Warning: The following options to parse-opts are unrecognized: :valid? 12:27:02.140 [main] INFO jepsen.cli - Test options: {:hashdump false, :ssh-private-key "./resources/vagrantkey", :enable-memcached-debug-log-level false, :package {:type :deb, :package #object[java.io.File 0x4b508371 "./couchbase-server-enterprise_6.5.1-6007-ubuntu16.04_amd64.deb"]}, :bucket-type :couchbase, :persist-to 0, :collect-core-files false, :manipulate-disks false, :concurrency 4, :autoreprovision-maxnodes 1, :skip-teardown false, :dcp-set-read false, :replicate-to 0, :cluster-run false, :server-groups-enabled false, :cas true, :ssh {:username "vagrant", :password "root", :strict-host-key-checking false, :private-key-path "./resources/vagrantkey"}, :rate 0, :doc-padding-size 3072, :durability (0 100 0 0), :nodes ["172.28.128.125" "172.28.128.126" "172.28.128.127" "172.28.128.128"], :failover-type :graceful, :test-count 1, :install-path "/opt/couchbase", :doc-count 30, :process-suspend-time 10, :perf-graphs false, :replicas 2, :autofailover false, :cbcollect true, :connect-timeout 10, :autoreprovision true, :disrupt-count 1, :collect-data-files false, :mixed-txns false, :kv-timeout #object[java.time.Duration 0x20027c44 "PT2S"], :server-group-count 1, :time-limit 60, :transactions false, :use-json-docs true, :eviction-policy "valueOnly", :workload "failover", :recovery-type :delta, :enable-tcp-capture false, :target-server-groups false} INFO [2019-12-17 12:27:02,265] jepsen test runner - jepsen.core Running test: {:hashdump false, :cycles 1, :node-count 4, :ssh-private-key "./resources/vagrantkey", :enable-memcached-debug-log-level false, :package {:type :deb, :package #object[java.io.File 0x4b508371 "./couchbase-server-enterprise_6.5.1-6007-ubuntu16.04_amd64.deb"]}, :bucket-type :couchbase, :persist-to 0, :collect-core-files false, :manipulate-disks false, :concurrency 90, :pool-size 6, :autoreprovision-maxnodes 1, :random-server-group nil, :txn-doc-count 3, :db #object[couchbase.core$couchbase_remote$reify__1269 0x60ed0b9d "couchbase.core$couchbase_remote$reify__1269@60ed0b9d"], :autofailover-maxcount 3, :skip-teardown false, :name "Couchbase", :sg-enabled false, :dcp-set-read false, :start-time #object[org.joda.time.DateTime 0x3ae2702a "2019-12-17T12:27:02.000Z"], :net #object[jepsen.net$reify__5211 0x6e948f1c "jepsen.net$reify__5211@6e948f1c"], :client {:cluster nil, :bucket nil, :collection nil, :env nil, :txn nil}, :replicate-to 0, :barrier #object[java.util.concurrent.CyclicBarrier 0x6f289728 "java.util.concurrent.CyclicBarrier@6f289728"], :cluster-run false, :server-groups-enabled false, :cas true, :ssh {:username "vagrant", :password "root", :strict-host-key-checking false, :private-key-path "./resources/vagrantkey"}, :rate 0, :autofailover-timeout 6, :doc-padding-size 3072, :checker #object[jepsen.checker$compose$reify__4751 0x23bd047f "jepsen.checker$compose$reify__4751@23bd047f"], :client-generator #object[jepsen.independent$concurrent_generator$reify__2879 0x7c226095 "jepsen.independent$concurrent_generator$reify__2879@7c226095"], :db-intialized #, :nemesis #object[couchbase.nemesis$couchbase$reify__1064 0x79b63325 "couchbase.nemesis$couchbase$reify__1064@79b63325"], :durability (0 100 0 0), :active-histories #, :control-atom #, :nodes ["172.28.128.125" "172.28.128.126" "172.28.128.127" "172.28.128.128"], :failover-type :graceful, :test-count 1, :install-path "/opt/couchbase", :doc-count 30, :process-suspend-time 10, :perf-graphs false, :replicas 2, :autofailover false, :cbcollect true, :connect-timeout 10, :autoreprovision true, :txn-bucket-count 1, :disrupt-count 1, :generator (gen/concat [(gen/on #{:nemesis} (gen/seqone ((gen/start-timeout 1800000) (gen/delayfn #object[clojure.core$constantly$fn__5672 0x1a6dc5ea "clojure.core$constantly$fn__5672@1a6dc5ea"] (gen/gvoid)) {:type :info, :f :failover, :failover-type :graceful, :targeter #object[couchbase.nemesis$basic_nodes_targeter 0x61957d9c "couchbase.nemesis$basic_nodes_targeter@61957d9c"], :target-count 1} (gen/delayfn #object[clojure.core$constantly$fn__5672 0x29c59b6e "clojure.core$constantly$fn__5672@29c59b6e"] (gen/gvoid)) {:type :info, :f :recover, :recovery-type :delta} (gen/delayfn #object[clojure.core$constantly$fn__5672 0x483b7dc4 "clojure.core$constantly$fn__5672@483b7dc4"] (gen/gvoid)) (gen/delayfn #object[clojure.core$constantly$fn__5672 0x6b8bdcc6 "clojure.core$constantly$fn__5672@6b8bdcc6"] (gen/gvoid)) (gen/set-atom-stop)))) (gen/on #object[clojure.core$complement$fn__5669 0x24a3d414 "clojure.core$complement$fn__5669@24a3d414"] (gen/while-atom-continue #object[jepsen.independent$concurrent_generator$reify__2879 0x37e55819 "jepsen.independent$concurrent_generator$reify__2879@37e55819"]))]), :collect-data-files false, :doc-threads 3, :mixed-txns false, :kv-timeout #object[java.time.Duration 0x20027c44 "PT2S"], :os #object[jepsen.os$reify__2349 0x54defd69 "jepsen.os$reify__2349@54defd69"], :server-group-count nil, :time-limit 60, :transactions false, :use-json-docs true, :eviction-policy "valueOnly", :workload "failover", :recovery-type :delta, :max-txn-size 3, :enable-tcp-capture false, :target-server-groups false, :model {:value :nil}} INFO [2019-12-17 12:27:02,820] jepsen test runner - jepsen.db Tearing down DB INFO [2019-12-17 12:27:02,822] jepsen node 172.28.128.128 - couchbase.util Tearing down couchbase node INFO [2019-12-17 12:27:02,822] jepsen node 172.28.128.125 - couchbase.util Tearing down couchbase node INFO [2019-12-17 12:27:02,822] jepsen node 172.28.128.127 - couchbase.util Tearing down couchbase node INFO [2019-12-17 12:27:02,822] jepsen node 172.28.128.126 - couchbase.util Tearing down couchbase node INFO [2019-12-17 12:27:05,866] jepsen node 172.28.128.128 - couchbase.util Teardown Complete INFO [2019-12-17 12:27:06,638] jepsen node 172.28.128.127 - couchbase.util Teardown Complete INFO [2019-12-17 12:27:06,859] jepsen node 172.28.128.125 - couchbase.util Teardown Complete INFO [2019-12-17 12:27:07,105] jepsen node 172.28.128.126 - couchbase.util Teardown Complete INFO [2019-12-17 12:27:07,108] jepsen test runner - jepsen.db Setting up DB INFO [2019-12-17 12:27:07,109] jepsen node 172.28.128.128 - couchbase.util Setting up Couchbase Server INFO [2019-12-17 12:27:07,109] jepsen node 172.28.128.125 - couchbase.util Setting up Couchbase Server INFO [2019-12-17 12:27:07,109] jepsen node 172.28.128.126 - couchbase.util Setting up Couchbase Server INFO [2019-12-17 12:27:07,109] jepsen node 172.28.128.127 - couchbase.util Setting up Couchbase Server INFO [2019-12-17 12:27:07,109] jepsen node 172.28.128.128 - couchbase.util Installing package INFO [2019-12-17 12:27:07,109] jepsen node 172.28.128.125 - couchbase.util Installing package INFO [2019-12-17 12:27:07,109] jepsen node 172.28.128.126 - couchbase.util Installing package INFO [2019-12-17 12:27:07,109] jepsen node 172.28.128.127 - couchbase.util Installing package INFO [2019-12-17 12:27:17,814] jepsen node 172.28.128.125 - couchbase.util Package installed INFO [2019-12-17 12:27:17,819] jepsen node 172.28.128.127 - couchbase.util Package installed INFO [2019-12-17 12:27:17,819] jepsen node 172.28.128.126 - couchbase.util Package installed INFO [2019-12-17 12:27:17,866] jepsen node 172.28.128.125 - couchbase.util Setting up data paths INFO [2019-12-17 12:27:17,866] jepsen node 172.28.128.127 - couchbase.util Setting up data paths INFO [2019-12-17 12:27:17,866] jepsen node 172.28.128.126 - couchbase.util Setting up data paths INFO [2019-12-17 12:27:17,928] jepsen node 172.28.128.128 - couchbase.util Package installed INFO [2019-12-17 12:27:17,928] jepsen node 172.28.128.128 - couchbase.util Setting up data paths INFO [2019-12-17 12:27:17,974] jepsen node 172.28.128.126 - couchbase.util Changing permissions for /opt/couchbase INFO [2019-12-17 12:27:17,974] jepsen node 172.28.128.127 - couchbase.util Changing permissions for /opt/couchbase INFO [2019-12-17 12:27:17,974] jepsen node 172.28.128.125 - couchbase.util Changing permissions for /opt/couchbase INFO [2019-12-17 12:27:18,037] jepsen node 172.28.128.128 - couchbase.util Changing permissions for /opt/couchbase INFO [2019-12-17 12:27:18,146] jepsen node 172.28.128.128 - couchbase.util Starting daemon INFO [2019-12-17 12:27:18,193] jepsen node 172.28.128.126 - couchbase.util Starting daemon INFO [2019-12-17 12:27:18,193] jepsen node 172.28.128.125 - couchbase.util Starting daemon INFO [2019-12-17 12:27:18,193] jepsen node 172.28.128.127 - couchbase.util Starting daemon INFO [2019-12-17 12:27:24,381] jepsen node 172.28.128.128 - couchbase.util Daemon started INFO [2019-12-17 12:27:24,381] jepsen node 172.28.128.126 - couchbase.util Daemon started INFO [2019-12-17 12:27:24,381] jepsen node 172.28.128.125 - couchbase.util Daemon started INFO [2019-12-17 12:27:24,381] jepsen node 172.28.128.128 - couchbase.util TCP packet capture is not enabled INFO [2019-12-17 12:27:24,381] jepsen node 172.28.128.126 - couchbase.util TCP packet capture is not enabled INFO [2019-12-17 12:27:24,381] jepsen node 172.28.128.125 - couchbase.util TCP packet capture is not enabled INFO [2019-12-17 12:27:24,382] jepsen node 172.28.128.127 - couchbase.util Daemon started INFO [2019-12-17 12:27:24,382] jepsen node 172.28.128.127 - couchbase.util TCP packet capture is not enabled INFO [2019-12-17 12:27:24,382] jepsen test runner - jepsen.db Setting up primary 172.28.128.125 INFO [2019-12-17 12:27:24,383] jepsen node 172.28.128.125 - couchbase.util Creating couchbase cluster from 172.28.128.125 INFO [2019-12-17 12:27:24,471] jepsen node 172.28.128.125 - couchbase.util Adding node 172.28.128.126 to cluster INFO [2019-12-17 12:27:26,350] jepsen node 172.28.128.125 - couchbase.util Adding node 172.28.128.127 to cluster INFO [2019-12-17 12:27:27,960] jepsen node 172.28.128.125 - couchbase.util Adding node 172.28.128.128 to cluster INFO [2019-12-17 12:27:29,497] jepsen node 172.28.128.125 - couchbase.util Rebalance status: {} INFO [2019-12-17 12:27:29,498] jepsen node 172.28.128.125 - couchbase.util Rebalance check count: 0 INFO [2019-12-17 12:27:29,498] jepsen node 172.28.128.125 - couchbase.util Rebalance stuck count: 0 INFO [2019-12-17 12:27:30,520] jepsen node 172.28.128.125 - couchbase.util Rebalance status: {:status none} INFO [2019-12-17 12:27:30,520] jepsen node 172.28.128.125 - couchbase.util Rebalance check count: 1 INFO [2019-12-17 12:27:30,520] jepsen node 172.28.128.125 - couchbase.util Rebalance stuck count: 0 INFO [2019-12-17 12:27:30,520] jepsen node 172.28.128.125 - couchbase.util Rebalance complete INFO [2019-12-17 12:27:30,549] jepsen node 172.28.128.125 - couchbase.util Waiting for bucket warmup to complete... INFO [2019-12-17 12:27:30,584] jepsen node 172.28.128.125 - couchbase.util Setup complete INFO [2019-12-17 12:27:30,588] jepsen test runner - jepsen.core Relative time begins now INFO [2019-12-17 12:27:30,598] jepsen nemesis - jepsen.core Setting up nemesis INFO [2019-12-17 12:27:30,599] jepsen worker 0 - jepsen.core Setting up worker 0 INFO [2019-12-17 12:27:30,600] jepsen worker 0 - couchbase.cbclients Opening new client INFO [2019-12-17 12:27:30,601] jepsen worker 1 - jepsen.core Setting up worker 1 INFO [2019-12-17 12:27:30,602] jepsen worker 2 - jepsen.core Setting up worker 2 INFO [2019-12-17 12:27:30,603] jepsen worker 3 - jepsen.core Setting up worker 3 INFO [2019-12-17 12:27:30,603] jepsen worker 4 - jepsen.core Setting up worker 4 INFO [2019-12-17 12:27:30,604] jepsen worker 5 - jepsen.core Setting up worker 5 INFO [2019-12-17 12:27:30,605] jepsen worker 6 - jepsen.core Setting up worker 6 INFO [2019-12-17 12:27:30,605] jepsen worker 7 - jepsen.core Setting up worker 7 INFO [2019-12-17 12:27:30,605] jepsen worker 8 - jepsen.core Setting up worker 8 INFO [2019-12-17 12:27:30,606] jepsen worker 9 - jepsen.core Setting up worker 9 INFO [2019-12-17 12:27:30,607] jepsen worker 10 - jepsen.core Setting up worker 10 INFO [2019-12-17 12:27:30,608] jepsen worker 11 - jepsen.core Setting up worker 11 INFO [2019-12-17 12:27:30,609] jepsen worker 12 - jepsen.core Setting up worker 12 INFO [2019-12-17 12:27:30,610] jepsen worker 13 - jepsen.core Setting up worker 13 INFO [2019-12-17 12:27:30,611] jepsen worker 14 - jepsen.core Setting up worker 14 INFO [2019-12-17 12:27:30,614] jepsen worker 15 - jepsen.core Setting up worker 15 INFO [2019-12-17 12:27:30,615] jepsen worker 16 - jepsen.core Setting up worker 16 INFO [2019-12-17 12:27:30,616] jepsen worker 17 - jepsen.core Setting up worker 17 INFO [2019-12-17 12:27:30,616] jepsen worker 18 - jepsen.core Setting up worker 18 INFO [2019-12-17 12:27:30,616] jepsen worker 19 - jepsen.core Setting up worker 19 INFO [2019-12-17 12:27:30,617] jepsen worker 20 - jepsen.core Setting up worker 20 INFO [2019-12-17 12:27:30,617] jepsen worker 21 - jepsen.core Setting up worker 21 INFO [2019-12-17 12:27:30,618] jepsen worker 22 - jepsen.core Setting up worker 22 INFO [2019-12-17 12:27:30,618] jepsen worker 23 - jepsen.core Setting up worker 23 INFO [2019-12-17 12:27:30,619] jepsen worker 24 - jepsen.core Setting up worker 24 INFO [2019-12-17 12:27:30,619] jepsen worker 25 - jepsen.core Setting up worker 25 INFO [2019-12-17 12:27:30,619] jepsen worker 26 - jepsen.core Setting up worker 26 INFO [2019-12-17 12:27:30,622] jepsen worker 28 - jepsen.core Setting up worker 28 INFO [2019-12-17 12:27:30,622] jepsen worker 29 - jepsen.core Setting up worker 29 INFO [2019-12-17 12:27:30,623] jepsen worker 30 - jepsen.core Setting up worker 30 INFO [2019-12-17 12:27:30,623] jepsen worker 27 - jepsen.core Setting up worker 27 INFO [2019-12-17 12:27:30,623] jepsen worker 31 - jepsen.core Setting up worker 31 INFO [2019-12-17 12:27:30,623] jepsen worker 32 - jepsen.core Setting up worker 32 INFO [2019-12-17 12:27:30,624] jepsen worker 33 - jepsen.core Setting up worker 33 INFO [2019-12-17 12:27:30,625] jepsen worker 34 - jepsen.core Setting up worker 34 INFO [2019-12-17 12:27:30,625] jepsen worker 35 - jepsen.core Setting up worker 35 INFO [2019-12-17 12:27:30,626] jepsen worker 36 - jepsen.core Setting up worker 36 INFO [2019-12-17 12:27:30,626] jepsen worker 37 - jepsen.core Setting up worker 37 INFO [2019-12-17 12:27:30,627] jepsen worker 38 - jepsen.core Setting up worker 38 INFO [2019-12-17 12:27:30,627] jepsen worker 39 - jepsen.core Setting up worker 39 INFO [2019-12-17 12:27:30,627] jepsen worker 40 - jepsen.core Setting up worker 40 INFO [2019-12-17 12:27:30,628] jepsen worker 41 - jepsen.core Setting up worker 41 INFO [2019-12-17 12:27:30,629] jepsen worker 42 - jepsen.core Setting up worker 42 INFO [2019-12-17 12:27:30,630] jepsen worker 43 - jepsen.core Setting up worker 43 INFO [2019-12-17 12:27:30,630] jepsen worker 44 - jepsen.core Setting up worker 44 INFO [2019-12-17 12:27:30,630] jepsen worker 45 - jepsen.core Setting up worker 45 INFO [2019-12-17 12:27:30,631] jepsen worker 46 - jepsen.core Setting up worker 46 INFO [2019-12-17 12:27:30,632] jepsen worker 47 - jepsen.core Setting up worker 47 INFO [2019-12-17 12:27:30,632] jepsen worker 48 - jepsen.core Setting up worker 48 INFO [2019-12-17 12:27:30,633] jepsen worker 49 - jepsen.core Setting up worker 49 INFO [2019-12-17 12:27:30,634] jepsen worker 51 - jepsen.core Setting up worker 51 INFO [2019-12-17 12:27:30,637] jepsen worker 52 - jepsen.core Setting up worker 52 INFO [2019-12-17 12:27:30,637] jepsen worker 53 - jepsen.core Setting up worker 53 INFO [2019-12-17 12:27:30,638] jepsen worker 54 - jepsen.core Setting up worker 54 INFO [2019-12-17 12:27:30,639] jepsen worker 55 - jepsen.core Setting up worker 55 INFO [2019-12-17 12:27:30,640] jepsen worker 50 - jepsen.core Setting up worker 50 INFO [2019-12-17 12:27:30,640] jepsen worker 56 - jepsen.core Setting up worker 56 INFO [2019-12-17 12:27:30,640] jepsen worker 57 - jepsen.core Setting up worker 57 INFO [2019-12-17 12:27:30,641] jepsen worker 58 - jepsen.core Setting up worker 58 INFO [2019-12-17 12:27:30,641] jepsen worker 59 - jepsen.core Setting up worker 59 INFO [2019-12-17 12:27:30,642] jepsen worker 60 - jepsen.core Setting up worker 60 INFO [2019-12-17 12:27:30,642] jepsen worker 61 - jepsen.core Setting up worker 61 INFO [2019-12-17 12:27:30,642] jepsen worker 62 - jepsen.core Setting up worker 62 INFO [2019-12-17 12:27:30,643] jepsen worker 64 - jepsen.core Setting up worker 64 INFO [2019-12-17 12:27:30,644] jepsen worker 65 - jepsen.core Setting up worker 65 INFO [2019-12-17 12:27:30,644] jepsen worker 63 - jepsen.core Setting up worker 63 INFO [2019-12-17 12:27:30,644] jepsen worker 66 - jepsen.core Setting up worker 66 INFO [2019-12-17 12:27:30,644] jepsen worker 67 - jepsen.core Setting up worker 67 INFO [2019-12-17 12:27:30,645] jepsen worker 68 - jepsen.core Setting up worker 68 INFO [2019-12-17 12:27:30,646] jepsen worker 69 - jepsen.core Setting up worker 69 INFO [2019-12-17 12:27:30,647] jepsen worker 70 - jepsen.core Setting up worker 70 INFO [2019-12-17 12:27:30,647] jepsen worker 71 - jepsen.core Setting up worker 71 INFO [2019-12-17 12:27:30,647] jepsen worker 72 - jepsen.core Setting up worker 72 INFO [2019-12-17 12:27:30,648] jepsen worker 73 - jepsen.core Setting up worker 73 INFO [2019-12-17 12:27:30,648] jepsen worker 74 - jepsen.core Setting up worker 74 INFO [2019-12-17 12:27:30,648] jepsen worker 75 - jepsen.core Setting up worker 75 INFO [2019-12-17 12:27:30,648] jepsen worker 76 - jepsen.core Setting up worker 76 INFO [2019-12-17 12:27:30,648] jepsen worker 77 - jepsen.core Setting up worker 77 INFO [2019-12-17 12:27:30,648] jepsen worker 78 - jepsen.core Setting up worker 78 INFO [2019-12-17 12:27:30,649] jepsen worker 79 - jepsen.core Setting up worker 79 INFO [2019-12-17 12:27:30,649] jepsen worker 80 - jepsen.core Setting up worker 80 INFO [2019-12-17 12:27:30,649] jepsen worker 81 - jepsen.core Setting up worker 81 INFO [2019-12-17 12:27:30,649] jepsen worker 82 - jepsen.core Setting up worker 82 INFO [2019-12-17 12:27:30,649] jepsen worker 83 - jepsen.core Setting up worker 83 INFO [2019-12-17 12:27:30,649] jepsen worker 84 - jepsen.core Setting up worker 84 INFO [2019-12-17 12:27:30,650] jepsen worker 85 - jepsen.core Setting up worker 85 INFO [2019-12-17 12:27:30,652] jepsen worker 86 - jepsen.core Setting up worker 86 INFO [2019-12-17 12:27:30,652] jepsen worker 87 - jepsen.core Setting up worker 87 INFO [2019-12-17 12:27:30,656] jepsen worker 88 - jepsen.core Setting up worker 88 INFO [2019-12-17 12:27:30,657] jepsen worker 89 - jepsen.core Setting up worker 89 INFO [2019-12-17 12:27:31,458] cb-events - com.couchbase.core [com.couchbase.core][CoreCreatedEvent] {"clientVersion":"3.0.0-beta.1","clientGitHash":"a6fd12eb","coreVersion":"2.0.0-beta.1","coreGitHash":"a6fd12eb","userAgent":"couchbase-java/3.0.0-beta.1 (Mac OS X 10.14.6 x86_64; OpenJDK 64-Bit Server VM 1.8.0_222-b10)","ioEnvironment":{"nativeIoEnabled":true,"eventLoopGroups":["KQueueEventLoopGroup"]},"ioConfig":{"allowedSaslMechs":["PLAIN","CRAM_MD5","SCRAM_SHA1","SCRAM_SHA256","SCRAM_SHA512"],"captureTraffic":[],"mutationTokensEnabled":true,"networkResolution":"auto","dnsSrvEnabled":false,"configPollIntervalMillis":2500,"kvCircuitBreakerConfig":"disabled","queryCircuitBreakerConfig":"disabled","viewCircuitBreakerConfig":"disabled","searchCircuitBreakerConfig":"disabled","analyticsCircuitBreakerConfig":"disabled","managerCircuitBreakerConfig":"disabled"},"compressionConfig":{"enabled":true,"minRatio":0.83,"minSize":32},"securityConfig":{"tlsEnabled":false,"nativeTlsEnabled":true,"hasKey":false,"hasTrustCertificates":false,"trustManagerFactory":null,"keyManagerFactory":null},"timeoutConfig":{"kvMs":2000,"managementMs":75000,"queryMs":75000,"viewMs":75000,"searchMs":75000,"analyticsMs":75000,"connectMs":10000,"disconnectMs":10000},"serviceConfig":{"kv":{"min":1,"max":1,"idleTimeMs":0},"query":{"min":0,"max":12,"idleTimeMs":300000},"view":{"min":0,"max":12,"idleTimeMs":300000},"search":{"min":0,"max":12,"idleTimeMs":300000},"analytics":{"min":0,"max":12,"idleTimeMs":300000}},"loggerConfig":{"customLogger":null,"fallbackToConsole":false,"disableSlf4j":false,"loggerName":"CouchbaseLogger","diagnosticContextEnabled":false},"retryStrategy":"BestEffortRetryStrategy"} {"coreId":1} INFO [2019-12-17 12:27:31,469] cb-events - com.couchbase.node [com.couchbase.node][NodeConnectedEvent] Node connected {"coreId":1,"managerPort":"8091","remote":"172.28.128.126"} INFO [2019-12-17 12:27:36,790] cb-events - com.couchbase.node [com.couchbase.node][NodeConnectedEvent] Node connected {"coreId":1,"managerPort":"8091","remote":"172.28.128.125"} INFO [2019-12-17 12:27:36,791] cb-events - com.couchbase.node [com.couchbase.node][NodeConnectedEvent] Node connected {"coreId":1,"managerPort":"8091","remote":"172.28.128.127"} INFO [2019-12-17 12:27:36,791] cb-events - com.couchbase.node [com.couchbase.node][NodeConnectedEvent] Node connected {"coreId":1,"managerPort":"8091","remote":"172.28.128.128"} INFO [2019-12-17 12:27:36,976] jepsen worker 0 - couchbase.cbclients Opening new client INFO [2019-12-17 12:27:37,016] cb-events - com.couchbase.core [com.couchbase.core][BucketOpenedEvent][173651µs] Opened bucket "default" {"coreId":1} INFO [2019-12-17 12:27:37,086] cb-events - com.couchbase.core [com.couchbase.core][CoreCreatedEvent] {"clientVersion":"3.0.0-beta.1","clientGitHash":"a6fd12eb","coreVersion":"2.0.0-beta.1","coreGitHash":"a6fd12eb","userAgent":"couchbase-java/3.0.0-beta.1 (Mac OS X 10.14.6 x86_64; OpenJDK 64-Bit Server VM 1.8.0_222-b10)","ioEnvironment":{"nativeIoEnabled":true,"eventLoopGroups":["KQueueEventLoopGroup"]},"ioConfig":{"allowedSaslMechs":["PLAIN","CRAM_MD5","SCRAM_SHA1","SCRAM_SHA256","SCRAM_SHA512"],"captureTraffic":[],"mutationTokensEnabled":true,"networkResolution":"auto","dnsSrvEnabled":false,"configPollIntervalMillis":2500,"kvCircuitBreakerConfig":"disabled","queryCircuitBreakerConfig":"disabled","viewCircuitBreakerConfig":"disabled","searchCircuitBreakerConfig":"disabled","analyticsCircuitBreakerConfig":"disabled","managerCircuitBreakerConfig":"disabled"},"compressionConfig":{"enabled":true,"minRatio":0.83,"minSize":32},"securityConfig":{"tlsEnabled":false,"nativeTlsEnabled":true,"hasKey":false,"hasTrustCertificates":false,"trustManagerFactory":null,"keyManagerFactory":null},"timeoutConfig":{"kvMs":2000,"managementMs":75000,"queryMs":75000,"viewMs":75000,"searchMs":75000,"analyticsMs":75000,"connectMs":10000,"disconnectMs":10000},"serviceConfig":{"kv":{"min":1,"max":1,"idleTimeMs":0},"query":{"min":0,"max":12,"idleTimeMs":300000},"view":{"min":0,"max":12,"idleTimeMs":300000},"search":{"min":0,"max":12,"idleTimeMs":300000},"analytics":{"min":0,"max":12,"idleTimeMs":300000}},"loggerConfig":{"customLogger":null,"fallbackToConsole":false,"disableSlf4j":false,"loggerName":"CouchbaseLogger","diagnosticContextEnabled":false},"retryStrategy":"BestEffortRetryStrategy"} {"coreId":2} INFO [2019-12-17 12:27:37,086] cb-events - com.couchbase.node [com.couchbase.node][NodeConnectedEvent] Node connected {"coreId":2,"managerPort":"8091","remote":"172.28.128.127"} INFO [2019-12-17 12:27:37,087] cb-events - com.couchbase.node [com.couchbase.node][NodeConnectedEvent] Node connected {"coreId":2,"managerPort":"8091","remote":"172.28.128.125"} INFO [2019-12-17 12:27:37,194] jepsen worker 0 - couchbase.cbclients Opening new client INFO [2019-12-17 12:27:37,197] cb-events - com.couchbase.node [com.couchbase.node][NodeConnectedEvent] Node connected {"coreId":2,"managerPort":"8091","remote":"172.28.128.126"} INFO [2019-12-17 12:27:37,197] cb-events - com.couchbase.node [com.couchbase.node][NodeConnectedEvent] Node connected {"coreId":2,"managerPort":"8091","remote":"172.28.128.128"} INFO [2019-12-17 12:27:37,206] cb-events - com.couchbase.core [com.couchbase.core][BucketOpenedEvent][95096µs] Opened bucket "default" {"coreId":2} INFO [2019-12-17 12:27:37,306] cb-events - com.couchbase.core [com.couchbase.core][CoreCreatedEvent] {"clientVersion":"3.0.0-beta.1","clientGitHash":"a6fd12eb","coreVersion":"2.0.0-beta.1","coreGitHash":"a6fd12eb","userAgent":"couchbase-java/3.0.0-beta.1 (Mac OS X 10.14.6 x86_64; OpenJDK 64-Bit Server VM 1.8.0_222-b10)","ioEnvironment":{"nativeIoEnabled":true,"eventLoopGroups":["KQueueEventLoopGroup"]},"ioConfig":{"allowedSaslMechs":["PLAIN","CRAM_MD5","SCRAM_SHA1","SCRAM_SHA256","SCRAM_SHA512"],"captureTraffic":[],"mutationTokensEnabled":true,"networkResolution":"auto","dnsSrvEnabled":false,"configPollIntervalMillis":2500,"kvCircuitBreakerConfig":"disabled","queryCircuitBreakerConfig":"disabled","viewCircuitBreakerConfig":"disabled","searchCircuitBreakerConfig":"disabled","analyticsCircuitBreakerConfig":"disabled","managerCircuitBreakerConfig":"disabled"},"compressionConfig":{"enabled":true,"minRatio":0.83,"minSize":32},"securityConfig":{"tlsEnabled":false,"nativeTlsEnabled":true,"hasKey":false,"hasTrustCertificates":false,"trustManagerFactory":null,"keyManagerFactory":null},"timeoutConfig":{"kvMs":2000,"managementMs":75000,"queryMs":75000,"viewMs":75000,"searchMs":75000,"analyticsMs":75000,"connectMs":10000,"disconnectMs":10000},"serviceConfig":{"kv":{"min":1,"max":1,"idleTimeMs":0},"query":{"min":0,"max":12,"idleTimeMs":300000},"view":{"min":0,"max":12,"idleTimeMs":300000},"search":{"min":0,"max":12,"idleTimeMs":300000},"analytics":{"min":0,"max":12,"idleTimeMs":300000}},"loggerConfig":{"customLogger":null,"fallbackToConsole":false,"disableSlf4j":false,"loggerName":"CouchbaseLogger","diagnosticContextEnabled":false},"retryStrategy":"BestEffortRetryStrategy"} {"coreId":3} INFO [2019-12-17 12:27:37,306] cb-events - com.couchbase.node [com.couchbase.node][NodeConnectedEvent] Node connected {"coreId":3,"managerPort":"8091","remote":"172.28.128.126"} INFO [2019-12-17 12:27:37,307] cb-events - com.couchbase.node [com.couchbase.node][NodeConnectedEvent] Node connected {"coreId":3,"managerPort":"8091","remote":"172.28.128.125"} INFO [2019-12-17 12:27:37,409] jepsen worker 0 - couchbase.cbclients Opening new client INFO [2019-12-17 12:27:37,418] cb-events - com.couchbase.node [com.couchbase.node][NodeConnectedEvent] Node connected {"coreId":3,"managerPort":"8091","remote":"172.28.128.127"} INFO [2019-12-17 12:27:37,419] cb-events - com.couchbase.node [com.couchbase.node][NodeConnectedEvent] Node connected {"coreId":3,"managerPort":"8091","remote":"172.28.128.128"} INFO [2019-12-17 12:27:37,424] cb-events - com.couchbase.core [com.couchbase.core][BucketOpenedEvent][90347µs] Opened bucket "default" {"coreId":3} INFO [2019-12-17 12:27:37,520] cb-events - com.couchbase.core [com.couchbase.core][CoreCreatedEvent] {"clientVersion":"3.0.0-beta.1","clientGitHash":"a6fd12eb","coreVersion":"2.0.0-beta.1","coreGitHash":"a6fd12eb","userAgent":"couchbase-java/3.0.0-beta.1 (Mac OS X 10.14.6 x86_64; OpenJDK 64-Bit Server VM 1.8.0_222-b10)","ioEnvironment":{"nativeIoEnabled":true,"eventLoopGroups":["KQueueEventLoopGroup"]},"ioConfig":{"allowedSaslMechs":["PLAIN","CRAM_MD5","SCRAM_SHA1","SCRAM_SHA256","SCRAM_SHA512"],"captureTraffic":[],"mutationTokensEnabled":true,"networkResolution":"auto","dnsSrvEnabled":false,"configPollIntervalMillis":2500,"kvCircuitBreakerConfig":"disabled","queryCircuitBreakerConfig":"disabled","viewCircuitBreakerConfig":"disabled","searchCircuitBreakerConfig":"disabled","analyticsCircuitBreakerConfig":"disabled","managerCircuitBreakerConfig":"disabled"},"compressionConfig":{"enabled":true,"minRatio":0.83,"minSize":32},"securityConfig":{"tlsEnabled":false,"nativeTlsEnabled":true,"hasKey":false,"hasTrustCertificates":false,"trustManagerFactory":null,"keyManagerFactory":null},"timeoutConfig":{"kvMs":2000,"managementMs":75000,"queryMs":75000,"viewMs":75000,"searchMs":75000,"analyticsMs":75000,"connectMs":10000,"disconnectMs":10000},"serviceConfig":{"kv":{"min":1,"max":1,"idleTimeMs":0},"query":{"min":0,"max":12,"idleTimeMs":300000},"view":{"min":0,"max":12,"idleTimeMs":300000},"search":{"min":0,"max":12,"idleTimeMs":300000},"analytics":{"min":0,"max":12,"idleTimeMs":300000}},"loggerConfig":{"customLogger":null,"fallbackToConsole":false,"disableSlf4j":false,"loggerName":"CouchbaseLogger","diagnosticContextEnabled":false},"retryStrategy":"BestEffortRetryStrategy"} {"coreId":4} INFO [2019-12-17 12:27:37,520] cb-events - com.couchbase.node [com.couchbase.node][NodeConnectedEvent] Node connected {"coreId":4,"managerPort":"8091","remote":"172.28.128.127"} INFO [2019-12-17 12:27:37,623] jepsen worker 0 - couchbase.cbclients Opening new client INFO [2019-12-17 12:27:37,630] cb-events - com.couchbase.node [com.couchbase.node][NodeConnectedEvent] Node connected {"coreId":4,"managerPort":"8091","remote":"172.28.128.125"} INFO [2019-12-17 12:27:37,631] cb-events - com.couchbase.node [com.couchbase.node][NodeConnectedEvent] Node connected {"coreId":4,"managerPort":"8091","remote":"172.28.128.126"} INFO [2019-12-17 12:27:37,631] cb-events - com.couchbase.node [com.couchbase.node][NodeConnectedEvent] Node connected {"coreId":4,"managerPort":"8091","remote":"172.28.128.128"} INFO [2019-12-17 12:27:37,635] cb-events - com.couchbase.core [com.couchbase.core][BucketOpenedEvent][88674µs] Opened bucket "default" {"coreId":4} INFO [2019-12-17 12:27:37,734] cb-events - com.couchbase.core [com.couchbase.core][CoreCreatedEvent] {"clientVersion":"3.0.0-beta.1","clientGitHash":"a6fd12eb","coreVersion":"2.0.0-beta.1","coreGitHash":"a6fd12eb","userAgent":"couchbase-java/3.0.0-beta.1 (Mac OS X 10.14.6 x86_64; OpenJDK 64-Bit Server VM 1.8.0_222-b10)","ioEnvironment":{"nativeIoEnabled":true,"eventLoopGroups":["KQueueEventLoopGroup"]},"ioConfig":{"allowedSaslMechs":["PLAIN","CRAM_MD5","SCRAM_SHA1","SCRAM_SHA256","SCRAM_SHA512"],"captureTraffic":[],"mutationTokensEnabled":true,"networkResolution":"auto","dnsSrvEnabled":false,"configPollIntervalMillis":2500,"kvCircuitBreakerConfig":"disabled","queryCircuitBreakerConfig":"disabled","viewCircuitBreakerConfig":"disabled","searchCircuitBreakerConfig":"disabled","analyticsCircuitBreakerConfig":"disabled","managerCircuitBreakerConfig":"disabled"},"compressionConfig":{"enabled":true,"minRatio":0.83,"minSize":32},"securityConfig":{"tlsEnabled":false,"nativeTlsEnabled":true,"hasKey":false,"hasTrustCertificates":false,"trustManagerFactory":null,"keyManagerFactory":null},"timeoutConfig":{"kvMs":2000,"managementMs":75000,"queryMs":75000,"viewMs":75000,"searchMs":75000,"analyticsMs":75000,"connectMs":10000,"disconnectMs":10000},"serviceConfig":{"kv":{"min":1,"max":1,"idleTimeMs":0},"query":{"min":0,"max":12,"idleTimeMs":300000},"view":{"min":0,"max":12,"idleTimeMs":300000},"search":{"min":0,"max":12,"idleTimeMs":300000},"analytics":{"min":0,"max":12,"idleTimeMs":300000}},"loggerConfig":{"customLogger":null,"fallbackToConsole":false,"disableSlf4j":false,"loggerName":"CouchbaseLogger","diagnosticContextEnabled":false},"retryStrategy":"BestEffortRetryStrategy"} {"coreId":5} INFO [2019-12-17 12:27:37,734] cb-events - com.couchbase.node [com.couchbase.node][NodeConnectedEvent] Node connected {"coreId":5,"managerPort":"8091","remote":"172.28.128.128"} INFO [2019-12-17 12:27:37,735] cb-events - com.couchbase.node [com.couchbase.node][NodeConnectedEvent] Node connected {"coreId":5,"managerPort":"8091","remote":"172.28.128.125"} INFO [2019-12-17 12:27:37,840] jepsen worker 0 - couchbase.cbclients Opening new client INFO [2019-12-17 12:27:37,844] cb-events - com.couchbase.node [com.couchbase.node][NodeConnectedEvent] Node connected {"coreId":5,"managerPort":"8091","remote":"172.28.128.126"} INFO [2019-12-17 12:27:37,844] cb-events - com.couchbase.node [com.couchbase.node][NodeConnectedEvent] Node connected {"coreId":5,"managerPort":"8091","remote":"172.28.128.127"} INFO [2019-12-17 12:27:37,847] cb-events - com.couchbase.core [com.couchbase.core][BucketOpenedEvent][93826µs] Opened bucket "default" {"coreId":5} INFO [2019-12-17 12:27:37,954] cb-events - com.couchbase.core [com.couchbase.core][CoreCreatedEvent] {"clientVersion":"3.0.0-beta.1","clientGitHash":"a6fd12eb","coreVersion":"2.0.0-beta.1","coreGitHash":"a6fd12eb","userAgent":"couchbase-java/3.0.0-beta.1 (Mac OS X 10.14.6 x86_64; OpenJDK 64-Bit Server VM 1.8.0_222-b10)","ioEnvironment":{"nativeIoEnabled":true,"eventLoopGroups":["KQueueEventLoopGroup"]},"ioConfig":{"allowedSaslMechs":["PLAIN","CRAM_MD5","SCRAM_SHA1","SCRAM_SHA256","SCRAM_SHA512"],"captureTraffic":[],"mutationTokensEnabled":true,"networkResolution":"auto","dnsSrvEnabled":false,"configPollIntervalMillis":2500,"kvCircuitBreakerConfig":"disabled","queryCircuitBreakerConfig":"disabled","viewCircuitBreakerConfig":"disabled","searchCircuitBreakerConfig":"disabled","analyticsCircuitBreakerConfig":"disabled","managerCircuitBreakerConfig":"disabled"},"compressionConfig":{"enabled":true,"minRatio":0.83,"minSize":32},"securityConfig":{"tlsEnabled":false,"nativeTlsEnabled":true,"hasKey":false,"hasTrustCertificates":false,"trustManagerFactory":null,"keyManagerFactory":null},"timeoutConfig":{"kvMs":2000,"managementMs":75000,"queryMs":75000,"viewMs":75000,"searchMs":75000,"analyticsMs":75000,"connectMs":10000,"disconnectMs":10000},"serviceConfig":{"kv":{"min":1,"max":1,"idleTimeMs":0},"query":{"min":0,"max":12,"idleTimeMs":300000},"view":{"min":0,"max":12,"idleTimeMs":300000},"search":{"min":0,"max":12,"idleTimeMs":300000},"analytics":{"min":0,"max":12,"idleTimeMs":300000}},"loggerConfig":{"customLogger":null,"fallbackToConsole":false,"disableSlf4j":false,"loggerName":"CouchbaseLogger","diagnosticContextEnabled":false},"retryStrategy":"BestEffortRetryStrategy"} {"coreId":6} INFO [2019-12-17 12:27:37,954] cb-events - com.couchbase.node [com.couchbase.node][NodeConnectedEvent] Node connected {"coreId":6,"managerPort":"8091","remote":"172.28.128.125"} INFO [2019-12-17 12:27:37,954] cb-events - com.couchbase.node [com.couchbase.node][NodeConnectedEvent] Node connected {"coreId":6,"managerPort":"8091","remote":"172.28.128.126"} INFO [2019-12-17 12:27:38,064] cb-events - com.couchbase.node [com.couchbase.node][NodeConnectedEvent] Node connected {"coreId":6,"managerPort":"8091","remote":"172.28.128.127"} INFO [2019-12-17 12:27:38,064] cb-events - com.couchbase.node [com.couchbase.node][NodeConnectedEvent] Node connected {"coreId":6,"managerPort":"8091","remote":"172.28.128.128"} INFO [2019-12-17 12:27:38,067] cb-events - com.couchbase.core [com.couchbase.core][BucketOpenedEvent][96689µs] Opened bucket "default" {"coreId":6} INFO [2019-12-17 12:27:43,074] jepsen nemesis - jepsen.core Running nemesis INFO [2019-12-17 12:27:43,074] jepsen worker 0 - jepsen.core Running worker 0 INFO [2019-12-17 12:27:43,074] jepsen worker 1 - jepsen.core Running worker 1 INFO [2019-12-17 12:27:43,075] jepsen worker 2 - jepsen.core Running worker 2 INFO [2019-12-17 12:27:43,075] jepsen worker 3 - jepsen.core Running worker 3 INFO [2019-12-17 12:27:43,075] jepsen worker 4 - jepsen.core Running worker 4 INFO [2019-12-17 12:27:43,075] jepsen worker 5 - jepsen.core Running worker 5 INFO [2019-12-17 12:27:43,075] jepsen worker 6 - jepsen.core Running worker 6 INFO [2019-12-17 12:27:43,075] jepsen worker 7 - jepsen.core Running worker 7 INFO [2019-12-17 12:27:43,075] jepsen worker 11 - jepsen.core Running worker 11 INFO [2019-12-17 12:27:43,076] jepsen worker 9 - jepsen.core Running worker 9 INFO [2019-12-17 12:27:43,076] jepsen worker 12 - jepsen.core Running worker 12 INFO [2019-12-17 12:27:43,076] jepsen worker 13 - jepsen.core Running worker 13 INFO [2019-12-17 12:27:43,076] jepsen worker 10 - jepsen.core Running worker 10 INFO [2019-12-17 12:27:43,076] jepsen worker 14 - jepsen.core Running worker 14 INFO [2019-12-17 12:27:43,076] jepsen worker 15 - jepsen.core Running worker 15 INFO [2019-12-17 12:27:43,076] jepsen worker 16 - jepsen.core Running worker 16 INFO [2019-12-17 12:27:43,076] jepsen worker 8 - jepsen.core Running worker 8 INFO [2019-12-17 12:27:43,076] jepsen worker 17 - jepsen.core Running worker 17 INFO [2019-12-17 12:27:43,076] jepsen worker 18 - jepsen.core Running worker 18 INFO [2019-12-17 12:27:43,076] jepsen worker 19 - jepsen.core Running worker 19 INFO [2019-12-17 12:27:43,076] jepsen worker 20 - jepsen.core Running worker 20 INFO [2019-12-17 12:27:43,076] jepsen worker 21 - jepsen.core Running worker 21 INFO [2019-12-17 12:27:43,077] jepsen worker 22 - jepsen.core Running worker 22 INFO [2019-12-17 12:27:43,077] jepsen worker 23 - jepsen.core Running worker 23 INFO [2019-12-17 12:27:43,077] jepsen worker 25 - jepsen.core Running worker 25 INFO [2019-12-17 12:27:43,077] jepsen worker 26 - jepsen.core Running worker 26 INFO [2019-12-17 12:27:43,077] jepsen worker 27 - jepsen.core Running worker 27 INFO [2019-12-17 12:27:43,077] jepsen worker 28 - jepsen.core Running worker 28 INFO [2019-12-17 12:27:43,077] jepsen worker 29 - jepsen.core Running worker 29 INFO [2019-12-17 12:27:43,077] jepsen worker 30 - jepsen.core Running worker 30 INFO [2019-12-17 12:27:43,077] jepsen worker 31 - jepsen.core Running worker 31 INFO [2019-12-17 12:27:43,077] jepsen worker 24 - jepsen.core Running worker 24 INFO [2019-12-17 12:27:43,078] jepsen worker 32 - jepsen.core Running worker 32 INFO [2019-12-17 12:27:43,078] jepsen worker 33 - jepsen.core Running worker 33 INFO [2019-12-17 12:27:43,078] jepsen worker 34 - jepsen.core Running worker 34 INFO [2019-12-17 12:27:43,078] jepsen worker 35 - jepsen.core Running worker 35 INFO [2019-12-17 12:27:43,078] jepsen worker 36 - jepsen.core Running worker 36 INFO [2019-12-17 12:27:43,078] jepsen worker 37 - jepsen.core Running worker 37 INFO [2019-12-17 12:27:43,079] jepsen worker 38 - jepsen.core Running worker 38 INFO [2019-12-17 12:27:43,079] jepsen worker 39 - jepsen.core Running worker 39 INFO [2019-12-17 12:27:43,079] jepsen worker 40 - jepsen.core Running worker 40 INFO [2019-12-17 12:27:43,079] jepsen worker 41 - jepsen.core Running worker 41 INFO [2019-12-17 12:27:43,079] jepsen worker 42 - jepsen.core Running worker 42 INFO [2019-12-17 12:27:43,079] jepsen worker 43 - jepsen.core Running worker 43 INFO [2019-12-17 12:27:43,079] jepsen worker 44 - jepsen.core Running worker 44 INFO [2019-12-17 12:27:43,079] jepsen worker 45 - jepsen.core Running worker 45 INFO [2019-12-17 12:27:43,079] jepsen worker 46 - jepsen.core Running worker 46 INFO [2019-12-17 12:27:43,079] jepsen worker 47 - jepsen.core Running worker 47 INFO [2019-12-17 12:27:43,080] jepsen worker 48 - jepsen.core Running worker 48 INFO [2019-12-17 12:27:43,080] jepsen worker 51 - jepsen.core Running worker 51 INFO [2019-12-17 12:27:43,080] jepsen worker 52 - jepsen.core Running worker 52 INFO [2019-12-17 12:27:43,080] jepsen worker 53 - jepsen.core Running worker 53 INFO [2019-12-17 12:27:43,080] jepsen worker 54 - jepsen.core Running worker 54 INFO [2019-12-17 12:27:43,080] jepsen worker 49 - jepsen.core Running worker 49 INFO [2019-12-17 12:27:43,080] jepsen worker 55 - jepsen.core Running worker 55 INFO [2019-12-17 12:27:43,080] jepsen worker 56 - jepsen.core Running worker 56 INFO [2019-12-17 12:27:43,081] jepsen worker 57 - jepsen.core Running worker 57 INFO [2019-12-17 12:27:43,081] jepsen worker 58 - jepsen.core Running worker 58 INFO [2019-12-17 12:27:43,081] jepsen worker 59 - jepsen.core Running worker 59 INFO [2019-12-17 12:27:43,081] jepsen worker 60 - jepsen.core Running worker 60 INFO [2019-12-17 12:27:43,082] jepsen worker 61 - jepsen.core Running worker 61 INFO [2019-12-17 12:27:43,082] jepsen worker 62 - jepsen.core Running worker 62 INFO [2019-12-17 12:27:43,082] jepsen worker 64 - jepsen.core Running worker 64 INFO [2019-12-17 12:27:43,082] jepsen worker 50 - jepsen.core Running worker 50 INFO [2019-12-17 12:27:43,082] jepsen worker 65 - jepsen.core Running worker 65 INFO [2019-12-17 12:27:43,082] jepsen worker 66 - jepsen.core Running worker 66 INFO [2019-12-17 12:27:43,082] jepsen worker 67 - jepsen.core Running worker 67 INFO [2019-12-17 12:27:43,082] jepsen worker 68 - jepsen.core Running worker 68 INFO [2019-12-17 12:27:43,082] jepsen worker 69 - jepsen.core Running worker 69 INFO [2019-12-17 12:27:43,082] jepsen worker 70 - jepsen.core Running worker 70 INFO [2019-12-17 12:27:43,082] jepsen worker 63 - jepsen.core Running worker 63 INFO [2019-12-17 12:27:43,082] jepsen worker 71 - jepsen.core Running worker 71 INFO [2019-12-17 12:27:43,082] jepsen worker 72 - jepsen.core Running worker 72 INFO [2019-12-17 12:27:43,082] jepsen worker 73 - jepsen.core Running worker 73 INFO [2019-12-17 12:27:43,082] jepsen worker 74 - jepsen.core Running worker 74 INFO [2019-12-17 12:27:43,082] jepsen worker 75 - jepsen.core Running worker 75 INFO [2019-12-17 12:27:43,082] jepsen worker 76 - jepsen.core Running worker 76 INFO [2019-12-17 12:27:43,083] jepsen worker 77 - jepsen.core Running worker 77 INFO [2019-12-17 12:27:43,083] jepsen worker 78 - jepsen.core Running worker 78 INFO [2019-12-17 12:27:43,083] jepsen worker 79 - jepsen.core Running worker 79 INFO [2019-12-17 12:27:43,083] jepsen worker 80 - jepsen.core Running worker 80 INFO [2019-12-17 12:27:43,083] jepsen worker 81 - jepsen.core Running worker 81 INFO [2019-12-17 12:27:43,083] jepsen worker 82 - jepsen.core Running worker 82 INFO [2019-12-17 12:27:43,083] jepsen worker 83 - jepsen.core Running worker 83 INFO [2019-12-17 12:27:43,083] jepsen worker 84 - jepsen.core Running worker 84 INFO [2019-12-17 12:27:43,083] jepsen worker 85 - jepsen.core Running worker 85 INFO [2019-12-17 12:27:43,083] jepsen worker 86 - jepsen.core Running worker 86 INFO [2019-12-17 12:27:43,083] jepsen worker 87 - jepsen.core Running worker 87 INFO [2019-12-17 12:27:43,083] jepsen worker 88 - jepsen.core Running worker 88 INFO [2019-12-17 12:27:43,083] jepsen worker 89 - jepsen.core Running worker 89 INFO [2019-12-17 12:27:43,102] jepsen worker 14 - jepsen.util 14 :invoke :read [4 nil] INFO [2019-12-17 12:27:43,102] jepsen worker 4 - jepsen.util 4 :invoke :read [1 nil] INFO [2019-12-17 12:27:43,107] jepsen worker 2 - jepsen.util 2 :invoke :cas [0 [44 32]] INFO [2019-12-17 12:27:43,107] jepsen worker 35 - jepsen.util 35 :invoke :read [11 nil] INFO [2019-12-17 12:27:43,107] jepsen worker 24 - jepsen.util 24 :invoke :read [8 nil] INFO [2019-12-17 12:27:43,116] jepsen worker 4 - jepsen.util 4 :ok :read [1 :nil] INFO [2019-12-17 12:27:43,116] jepsen worker 14 - jepsen.util 14 :ok :read [4 :nil] INFO [2019-12-17 12:27:43,116] jepsen worker 2 - jepsen.util 2 :fail :cas [0 [44 32]] :KeyNotFoundException INFO [2019-12-17 12:27:43,116] jepsen worker 35 - jepsen.util 35 :ok :read [11 :nil] INFO [2019-12-17 12:27:43,116] jepsen worker 24 - jepsen.util 24 :ok :read [8 :nil] INFO [2019-12-17 12:27:43,116] jepsen worker 14 - jepsen.util 14 :invoke :read [4 nil] INFO [2019-12-17 12:27:43,116] jepsen worker 4 - jepsen.util 4 :invoke :read [1 nil] INFO [2019-12-17 12:27:43,116] jepsen worker 2 - jepsen.util 2 :invoke :read [0 nil] INFO [2019-12-17 12:27:43,116] jepsen worker 35 - jepsen.util 35 :invoke :read [11 nil] INFO [2019-12-17 12:27:43,116] jepsen worker 24 - jepsen.util 24 :invoke :write [8 14] INFO [2019-12-17 12:27:43,117] jepsen worker 6 - jepsen.util 6 :invoke :write [2 15] INFO [2019-12-17 12:27:43,117] jepsen worker 27 - jepsen.util 27 :invoke :read [9 nil] INFO [2019-12-17 12:27:43,117] jepsen worker 34 - jepsen.util 34 :invoke :read [11 nil] INFO [2019-12-17 12:27:43,117] jepsen worker 36 - jepsen.util 36 :invoke :read [12 nil] INFO [2019-12-17 12:27:43,119] jepsen worker 35 - jepsen.util 35 :ok :read [11 :nil] INFO [2019-12-17 12:27:43,119] jepsen worker 14 - jepsen.util 14 :ok :read [4 :nil] INFO [2019-12-17 12:27:43,119] jepsen worker 35 - jepsen.util 35 :invoke :read [11 nil] INFO [2019-12-17 12:27:43,120] jepsen worker 14 - jepsen.util 14 :invoke :write [4 24] INFO [2019-12-17 12:27:43,121] jepsen worker 2 - jepsen.util 2 :ok :read [0 :nil] INFO [2019-12-17 12:27:43,122] jepsen worker 27 - jepsen.util 27 :ok :read [9 :nil] INFO [2019-12-17 12:27:43,122] jepsen worker 2 - jepsen.util 2 :invoke :write [0 34] INFO [2019-12-17 12:27:43,125] jepsen worker 34 - jepsen.util 34 :ok :read [11 :nil] INFO [2019-12-17 12:27:43,128] jepsen worker 27 - jepsen.util 27 :invoke :read [9 nil] INFO [2019-12-17 12:27:43,132] jepsen worker 36 - jepsen.util 36 :ok :read [12 :nil] INFO [2019-12-17 12:27:43,133] jepsen worker 35 - jepsen.util 35 :ok :read [11 :nil] INFO [2019-12-17 12:27:43,133] jepsen worker 4 - jepsen.util 4 :ok :read [1 :nil] INFO [2019-12-17 12:27:43,133] jepsen worker 34 - jepsen.util 34 :invoke :cas [11 [31 17]] INFO [2019-12-17 12:27:43,134] jepsen worker 36 - jepsen.util 36 :invoke :read [12 nil] INFO [2019-12-17 12:27:43,134] jepsen worker 35 - jepsen.util 35 :invoke :cas [11 [9 41]] INFO [2019-12-17 12:27:43,134] jepsen worker 4 - jepsen.util 4 :invoke :write [1 23] INFO [2019-12-17 12:27:43,138] jepsen worker 27 - jepsen.util 27 :ok :read [9 :nil] INFO [2019-12-17 12:27:43,139] jepsen worker 36 - jepsen.util 36 :ok :read [12 :nil] INFO [2019-12-17 12:27:43,139] jepsen worker 35 - jepsen.util 35 :fail :cas [11 [9 41]] :KeyNotFoundException INFO [2019-12-17 12:27:43,139] jepsen worker 34 - jepsen.util 34 :fail :cas [11 [31 17]] :KeyNotFoundException INFO [2019-12-17 12:27:43,139] jepsen worker 27 - jepsen.util 27 :invoke :cas [9 [3 16]] INFO [2019-12-17 12:27:43,139] jepsen worker 35 - jepsen.util 35 :invoke :cas [11 [18 40]] INFO [2019-12-17 12:27:43,139] jepsen worker 36 - jepsen.util 36 :invoke :cas [12 [22 33]] INFO [2019-12-17 12:27:43,139] jepsen worker 34 - jepsen.util 34 :invoke :read [11 nil] INFO [2019-12-17 12:27:43,145] jepsen worker 34 - jepsen.util 34 :ok :read [11 :nil] INFO [2019-12-17 12:27:43,145] jepsen worker 36 - jepsen.util 36 :fail :cas [12 [22 33]] :KeyNotFoundException INFO [2019-12-17 12:27:43,145] jepsen worker 27 - jepsen.util 27 :fail :cas [9 [3 16]] :KeyNotFoundException INFO [2019-12-17 12:27:43,147] jepsen worker 35 - jepsen.util 35 :fail :cas [11 [18 40]] :KeyNotFoundException INFO [2019-12-17 12:27:43,147] jepsen worker 34 - jepsen.util 34 :invoke :cas [11 [42 8]] INFO [2019-12-17 12:27:43,147] jepsen worker 27 - jepsen.util 27 :invoke :read [9 nil] INFO [2019-12-17 12:27:43,147] jepsen worker 36 - jepsen.util 36 :invoke :read [12 nil] INFO [2019-12-17 12:27:43,147] jepsen worker 35 - jepsen.util 35 :invoke :write [11 22] INFO [2019-12-17 12:27:43,147] jepsen worker 26 - jepsen.util 26 :invoke :read [8 nil] INFO [2019-12-17 12:27:43,147] jepsen worker 1 - jepsen.util 1 :invoke :cas [0 [19 9]] INFO [2019-12-17 12:27:43,150] jepsen worker 26 - jepsen.util 26 :ok :read [8 :nil] INFO [2019-12-17 12:27:43,150] jepsen worker 27 - jepsen.util 27 :ok :read [9 :nil] INFO [2019-12-17 12:27:43,150] jepsen worker 34 - jepsen.util 34 :fail :cas [11 [42 8]] :KeyNotFoundException INFO [2019-12-17 12:27:43,153] jepsen worker 36 - jepsen.util 36 :ok :read [12 :nil] INFO [2019-12-17 12:27:43,153] jepsen worker 1 - jepsen.util 1 :fail :cas [0 [19 9]] :KeyNotFoundException INFO [2019-12-17 12:27:43,153] jepsen worker 27 - jepsen.util 27 :invoke :cas [9 [32 33]] INFO [2019-12-17 12:27:43,153] jepsen worker 26 - jepsen.util 26 :invoke :cas [8 [47 16]] INFO [2019-12-17 12:27:43,153] jepsen worker 34 - jepsen.util 34 :invoke :write [11 7] INFO [2019-12-17 12:27:43,153] jepsen worker 36 - jepsen.util 36 :invoke :cas [12 [47 31]] INFO [2019-12-17 12:27:43,153] jepsen worker 1 - jepsen.util 1 :invoke :read [0 nil] INFO [2019-12-17 12:27:43,158] jepsen worker 1 - jepsen.util 1 :ok :read [0 :nil] INFO [2019-12-17 12:27:43,158] jepsen worker 26 - jepsen.util 26 :fail :cas [8 [47 16]] :KeyNotFoundException INFO [2019-12-17 12:27:43,158] jepsen worker 36 - jepsen.util 36 :fail :cas [12 [47 31]] :KeyNotFoundException INFO [2019-12-17 12:27:43,158] jepsen worker 27 - jepsen.util 27 :fail :cas [9 [32 33]] :KeyNotFoundException INFO [2019-12-17 12:27:43,159] jepsen worker 1 - jepsen.util 1 :invoke :write [0 14] INFO [2019-12-17 12:27:43,159] jepsen worker 26 - jepsen.util 26 :invoke :cas [8 [30 45]] INFO [2019-12-17 12:27:43,159] jepsen worker 36 - jepsen.util 36 :invoke :write [12 0] INFO [2019-12-17 12:27:43,159] jepsen worker 27 - jepsen.util 27 :invoke :write [9 7] INFO [2019-12-17 12:27:43,161] jepsen worker 26 - jepsen.util 26 :fail :cas [8 [30 45]] :KeyNotFoundException INFO [2019-12-17 12:27:43,162] jepsen worker 26 - jepsen.util 26 :invoke :read [8 nil] INFO [2019-12-17 12:27:43,164] jepsen worker 23 - jepsen.util 23 :invoke :write [7 48] INFO [2019-12-17 12:27:43,164] jepsen worker 9 - jepsen.util 9 :invoke :read [3 nil] INFO [2019-12-17 12:27:43,168] jepsen worker 29 - jepsen.util 29 :invoke :cas [9 [28 28]] INFO [2019-12-17 12:27:43,174] jepsen worker 8 - jepsen.util 8 :invoke :read [2 nil] INFO [2019-12-17 12:27:43,177] jepsen worker 26 - jepsen.util 26 :ok :read [8 :nil] INFO [2019-12-17 12:27:43,177] jepsen worker 26 - jepsen.util 26 :invoke :write [8 25] INFO [2019-12-17 12:27:43,184] jepsen worker 9 - jepsen.util 9 :ok :read [3 :nil] INFO [2019-12-17 12:27:43,184] jepsen worker 9 - jepsen.util 9 :invoke :cas [3 [26 23]] INFO [2019-12-17 12:27:43,193] jepsen worker 44 - jepsen.util 44 :invoke :write [14 27] INFO [2019-12-17 12:27:43,198] jepsen worker 29 - jepsen.util 29 :fail :cas [9 [28 28]] :KeyNotFoundException INFO [2019-12-17 12:27:43,199] jepsen worker 29 - jepsen.util 29 :invoke :read [9 nil] INFO [2019-12-17 12:27:43,207] jepsen worker 9 - jepsen.util 9 :fail :cas [3 [26 23]] :KeyNotFoundException INFO [2019-12-17 12:27:43,207] jepsen worker 9 - jepsen.util 9 :invoke :read [3 nil] INFO [2019-12-17 12:27:43,210] jepsen worker 8 - jepsen.util 8 :ok :read [2 :nil] INFO [2019-12-17 12:27:43,210] jepsen worker 8 - jepsen.util 8 :invoke :cas [2 [18 36]] INFO [2019-12-17 12:27:43,215] jepsen worker 8 - jepsen.util 8 :fail :cas [2 [18 36]] :KeyNotFoundException INFO [2019-12-17 12:27:43,215] jepsen worker 9 - jepsen.util 9 :ok :read [3 :nil] INFO [2019-12-17 12:27:43,215] jepsen worker 29 - jepsen.util 29 :ok :read [9 :nil] INFO [2019-12-17 12:27:43,215] jepsen worker 8 - jepsen.util 8 :invoke :read [2 nil] INFO [2019-12-17 12:27:43,216] jepsen worker 9 - jepsen.util 9 :invoke :cas [3 [14 42]] INFO [2019-12-17 12:27:43,217] jepsen worker 29 - jepsen.util 29 :invoke :read [9 nil] INFO [2019-12-17 12:27:43,231] jepsen worker 9 - jepsen.util 9 :fail :cas [3 [14 42]] :KeyNotFoundException INFO [2019-12-17 12:27:43,231] jepsen worker 29 - jepsen.util 29 :ok :read [9 :nil] INFO [2019-12-17 12:27:43,231] jepsen worker 8 - jepsen.util 8 :ok :read [2 :nil] INFO [2019-12-17 12:27:43,231] jepsen worker 9 - jepsen.util 9 :invoke :cas [3 [6 48]] INFO [2019-12-17 12:27:43,232] jepsen worker 29 - jepsen.util 29 :invoke :write [9 6] INFO [2019-12-17 12:27:43,232] jepsen worker 8 - jepsen.util 8 :invoke :write [2 0] INFO [2019-12-17 12:27:43,233] jepsen worker 9 - jepsen.util 9 :fail :cas [3 [6 48]] :KeyNotFoundException INFO [2019-12-17 12:27:43,233] jepsen worker 9 - jepsen.util 9 :invoke :cas [3 [3 18]] INFO [2019-12-17 12:27:43,234] jepsen worker 9 - jepsen.util 9 :fail :cas [3 [3 18]] :KeyNotFoundException INFO [2019-12-17 12:27:43,234] jepsen worker 83 - jepsen.util 83 :invoke :read [27 nil] INFO [2019-12-17 12:27:43,234] jepsen worker 9 - jepsen.util 9 :invoke :read [3 nil] INFO [2019-12-17 12:27:43,235] jepsen worker 9 - jepsen.util 9 :ok :read [3 :nil] INFO [2019-12-17 12:27:43,235] jepsen worker 83 - jepsen.util 83 :ok :read [27 :nil] INFO [2019-12-17 12:27:43,235] jepsen worker 83 - jepsen.util 83 :invoke :read [27 nil] INFO [2019-12-17 12:27:43,236] jepsen worker 9 - jepsen.util 9 :invoke :write [3 2] INFO [2019-12-17 12:27:43,236] jepsen worker 83 - jepsen.util 83 :ok :read [27 :nil] INFO [2019-12-17 12:27:43,236] jepsen worker 83 - jepsen.util 83 :invoke :read [27 nil] INFO [2019-12-17 12:27:43,237] jepsen worker 83 - jepsen.util 83 :ok :read [27 :nil] INFO [2019-12-17 12:27:43,237] jepsen worker 83 - jepsen.util 83 :invoke :read [27 nil] INFO [2019-12-17 12:27:43,238] jepsen worker 83 - jepsen.util 83 :ok :read [27 :nil] INFO [2019-12-17 12:27:43,238] jepsen worker 83 - jepsen.util 83 :invoke :write [27 21] INFO [2019-12-17 12:27:43,269] jepsen worker 16 - jepsen.util 16 :invoke :cas [5 [20 12]] INFO [2019-12-17 12:27:43,269] jepsen worker 39 - jepsen.util 39 :invoke :cas [13 [28 45]] INFO [2019-12-17 12:27:43,269] jepsen worker 28 - jepsen.util 28 :invoke :write [9 30] INFO [2019-12-17 12:27:43,269] jepsen worker 19 - jepsen.util 19 :invoke :cas [6 [20 25]] INFO [2019-12-17 12:27:43,269] jepsen worker 89 - jepsen.util 89 :invoke :read [29 nil] INFO [2019-12-17 12:27:43,269] jepsen worker 37 - jepsen.util 37 :invoke :write [12 23] INFO [2019-12-17 12:27:43,269] jepsen worker 18 - jepsen.util 18 :invoke :write [6 47] INFO [2019-12-17 12:27:43,269] jepsen worker 40 - jepsen.util 40 :invoke :read [13 nil] INFO [2019-12-17 12:27:43,269] jepsen worker 30 - jepsen.util 30 :invoke :write [10 23] INFO [2019-12-17 12:27:43,269] jepsen worker 11 - jepsen.util 11 :invoke :cas [3 [21 33]] INFO [2019-12-17 12:27:43,269] jepsen worker 42 - jepsen.util 42 :invoke :cas [14 [16 32]] INFO [2019-12-17 12:27:43,269] jepsen worker 22 - jepsen.util 22 :invoke :write [7 23] INFO [2019-12-17 12:27:43,269] jepsen worker 12 - jepsen.util 12 :invoke :cas [4 [32 1]] INFO [2019-12-17 12:27:43,269] jepsen worker 41 - jepsen.util 41 :invoke :write [13 9] INFO [2019-12-17 12:27:43,269] jepsen worker 17 - jepsen.util 17 :invoke :cas [5 [11 19]] INFO [2019-12-17 12:27:43,269] jepsen worker 32 - jepsen.util 32 :invoke :write [10 32] INFO [2019-12-17 12:27:43,269] jepsen worker 20 - jepsen.util 20 :invoke :cas [6 [24 0]] INFO [2019-12-17 12:27:43,269] jepsen worker 21 - jepsen.util 21 :invoke :write [7 20] INFO [2019-12-17 12:27:43,269] jepsen worker 5 - jepsen.util 5 :invoke :cas [1 [37 34]] INFO [2019-12-17 12:27:43,269] jepsen worker 15 - jepsen.util 15 :invoke :cas [5 [37 32]] INFO [2019-12-17 12:27:43,269] jepsen worker 0 - jepsen.util 0 :invoke :write [0 0] INFO [2019-12-17 12:27:43,269] jepsen worker 7 - jepsen.util 7 :invoke :write [2 33] INFO [2019-12-17 12:27:43,269] jepsen worker 43 - jepsen.util 43 :invoke :cas [14 [3 37]] INFO [2019-12-17 12:27:43,269] jepsen worker 31 - jepsen.util 31 :invoke :cas [10 [32 37]] INFO [2019-12-17 12:27:43,269] jepsen worker 33 - jepsen.util 33 :invoke :write [11 28] INFO [2019-12-17 12:27:43,269] jepsen worker 3 - jepsen.util 3 :invoke :write [1 29] INFO [2019-12-17 12:27:43,269] jepsen worker 13 - jepsen.util 13 :invoke :write [4 13] INFO [2019-12-17 12:27:43,269] jepsen worker 46 - jepsen.util 46 :invoke :read [15 nil] INFO [2019-12-17 12:27:43,269] jepsen worker 10 - jepsen.util 10 :invoke :write [3 42] INFO [2019-12-17 12:27:43,269] jepsen worker 45 - jepsen.util 45 :invoke :cas [15 [49 35]] INFO [2019-12-17 12:27:43,269] jepsen worker 48 - jepsen.util 48 :invoke :write [16 41] INFO [2019-12-17 12:27:43,270] jepsen worker 47 - jepsen.util 47 :invoke :read [15 nil] INFO [2019-12-17 12:27:43,270] jepsen worker 51 - jepsen.util 51 :invoke :write [17 0] INFO [2019-12-17 12:27:43,270] jepsen worker 52 - jepsen.util 52 :invoke :cas [17 [36 33]] INFO [2019-12-17 12:27:43,270] jepsen worker 53 - jepsen.util 53 :invoke :write [17 31] INFO [2019-12-17 12:27:43,270] jepsen worker 49 - jepsen.util 49 :invoke :read [16 nil] INFO [2019-12-17 12:27:43,270] jepsen worker 55 - jepsen.util 55 :invoke :write [18 34] INFO [2019-12-17 12:27:43,270] jepsen worker 57 - jepsen.util 57 :invoke :read [19 nil] INFO [2019-12-17 12:27:43,270] jepsen worker 58 - jepsen.util 58 :invoke :read [19 nil] INFO [2019-12-17 12:27:43,270] jepsen worker 59 - jepsen.util 59 :invoke :cas [19 [21 5]] INFO [2019-12-17 12:27:43,270] jepsen worker 60 - jepsen.util 60 :invoke :write [20 41] INFO [2019-12-17 12:27:43,270] jepsen worker 61 - jepsen.util 61 :invoke :write [20 7] INFO [2019-12-17 12:27:43,270] jepsen worker 64 - jepsen.util 64 :invoke :cas [21 [4 26]] INFO [2019-12-17 12:27:43,270] jepsen worker 38 - jepsen.util 38 :invoke :cas [12 [39 31]] INFO [2019-12-17 12:27:43,270] jepsen worker 65 - jepsen.util 65 :invoke :read [21 nil] INFO [2019-12-17 12:27:43,270] jepsen worker 67 - jepsen.util 67 :invoke :read [22 nil] INFO [2019-12-17 12:27:43,270] jepsen worker 70 - jepsen.util 70 :invoke :cas [23 [2 45]] INFO [2019-12-17 12:27:43,270] jepsen worker 68 - jepsen.util 68 :invoke :write [22 41] INFO [2019-12-17 12:27:43,270] jepsen worker 71 - jepsen.util 71 :invoke :write [23 12] INFO [2019-12-17 12:27:43,270] jepsen worker 69 - jepsen.util 69 :invoke :cas [23 [16 18]] INFO [2019-12-17 12:27:43,271] jepsen worker 63 - jepsen.util 63 :invoke :cas [21 [10 46]] INFO [2019-12-17 12:27:43,271] jepsen worker 72 - jepsen.util 72 :invoke :cas [24 [38 21]] INFO [2019-12-17 12:27:43,271] jepsen worker 73 - jepsen.util 73 :invoke :cas [24 [39 37]] INFO [2019-12-17 12:27:43,271] jepsen worker 74 - jepsen.util 74 :invoke :read [24 nil] INFO [2019-12-17 12:27:43,272] jepsen worker 19 - jepsen.util 19 :fail :cas [6 [20 25]] :KeyNotFoundException INFO [2019-12-17 12:27:43,272] jepsen worker 11 - jepsen.util 11 :fail :cas [3 [21 33]] :KeyNotFoundException INFO [2019-12-17 12:27:43,272] jepsen worker 16 - jepsen.util 16 :fail :cas [5 [20 12]] :KeyNotFoundException INFO [2019-12-17 12:27:43,272] jepsen worker 75 - jepsen.util 75 :invoke :read [25 nil] INFO [2019-12-17 12:27:43,272] jepsen worker 40 - jepsen.util 40 :ok :read [13 :nil] INFO [2019-12-17 12:27:43,272] jepsen worker 42 - jepsen.util 42 :fail :cas [14 [16 32]] :KeyNotFoundException INFO [2019-12-17 12:27:43,272] jepsen worker 76 - jepsen.util 76 :invoke :read [25 nil] INFO [2019-12-17 12:27:43,272] jepsen worker 17 - jepsen.util 17 :fail :cas [5 [11 19]] :KeyNotFoundException INFO [2019-12-17 12:27:43,272] jepsen worker 20 - jepsen.util 20 :fail :cas [6 [24 0]] :KeyNotFoundException INFO [2019-12-17 12:27:43,272] jepsen worker 77 - jepsen.util 77 :invoke :write [25 2] INFO [2019-12-17 12:27:43,273] jepsen worker 25 - jepsen.util 25 :invoke :write [8 8] INFO [2019-12-17 12:27:43,273] jepsen worker 78 - jepsen.util 78 :invoke :write [26 49] INFO [2019-12-17 12:27:43,273] jepsen worker 66 - jepsen.util 66 :invoke :write [22 40] INFO [2019-12-17 12:27:43,273] jepsen worker 5 - jepsen.util 5 :fail :cas [1 [37 34]] :KeyNotFoundException INFO [2019-12-17 12:27:43,275] jepsen worker 79 - jepsen.util 79 :invoke :read [26 nil] INFO [2019-12-17 12:27:43,276] jepsen worker 80 - jepsen.util 80 :invoke :cas [26 [20 39]] INFO [2019-12-17 12:27:43,280] jepsen worker 82 - jepsen.util 82 :invoke :cas [27 [33 26]] INFO [2019-12-17 12:27:43,280] jepsen worker 81 - jepsen.util 81 :invoke :cas [27 [46 6]] INFO [2019-12-17 12:27:43,282] jepsen worker 87 - jepsen.util 87 :invoke :read [29 nil] INFO [2019-12-17 12:27:43,282] jepsen worker 43 - jepsen.util 43 :fail :cas [14 [3 37]] :KeyNotFoundException INFO [2019-12-17 12:27:43,283] jepsen worker 85 - jepsen.util 85 :invoke :write [28 32] INFO [2019-12-17 12:27:43,284] jepsen worker 84 - jepsen.util 84 :invoke :read [28 nil] INFO [2019-12-17 12:27:43,284] jepsen worker 46 - jepsen.util 46 :ok :read [15 :nil] INFO [2019-12-17 12:27:43,287] jepsen worker 88 - jepsen.util 88 :invoke :write [29 40] INFO [2019-12-17 12:27:43,288] jepsen worker 45 - jepsen.util 45 :fail :cas [15 [49 35]] :KeyNotFoundException INFO [2019-12-17 12:27:43,290] jepsen worker 47 - jepsen.util 47 :ok :read [15 :nil] INFO [2019-12-17 12:27:43,293] jepsen worker 58 - jepsen.util 58 :ok :read [19 :nil] INFO [2019-12-17 12:27:43,295] jepsen worker 65 - jepsen.util 65 :ok :read [21 :nil] INFO [2019-12-17 12:27:43,295] jepsen worker 67 - jepsen.util 67 :ok :read [22 :nil] INFO [2019-12-17 12:27:43,295] jepsen worker 70 - jepsen.util 70 :fail :cas [23 [2 45]] :KeyNotFoundException INFO [2019-12-17 12:27:43,295] jepsen worker 69 - jepsen.util 69 :fail :cas [23 [16 18]] :KeyNotFoundException INFO [2019-12-17 12:27:43,312] jepsen worker 11 - jepsen.util 11 :invoke :cas [3 [22 45]] INFO [2019-12-17 12:27:43,312] jepsen worker 19 - jepsen.util 19 :invoke :write [6 45] INFO [2019-12-17 12:27:43,312] jepsen worker 16 - jepsen.util 16 :invoke :write [5 32] INFO [2019-12-17 12:27:43,312] jepsen worker 40 - jepsen.util 40 :invoke :cas [13 [30 18]] INFO [2019-12-17 12:27:43,312] jepsen worker 42 - jepsen.util 42 :invoke :write [14 29] INFO [2019-12-17 12:27:43,312] jepsen worker 17 - jepsen.util 17 :invoke :cas [5 [17 19]] INFO [2019-12-17 12:27:43,312] jepsen worker 20 - jepsen.util 20 :invoke :cas [6 [9 30]] INFO [2019-12-17 12:27:43,312] jepsen worker 38 - jepsen.util 38 :fail :cas [12 [39 31]] :KeyNotFoundException INFO [2019-12-17 12:27:43,312] jepsen worker 74 - jepsen.util 74 :ok :read [24 :nil] INFO [2019-12-17 12:27:43,313] jepsen worker 72 - jepsen.util 72 :fail :cas [24 [38 21]] :KeyNotFoundException INFO [2019-12-17 12:27:43,313] jepsen worker 64 - jepsen.util 64 :fail :cas [21 [4 26]] :KeyNotFoundException INFO [2019-12-17 12:27:43,313] jepsen worker 5 - jepsen.util 5 :invoke :read [1 nil] INFO [2019-12-17 12:27:43,313] jepsen worker 59 - jepsen.util 59 :fail :cas [19 [21 5]] :KeyNotFoundException INFO [2019-12-17 12:27:43,313] jepsen worker 63 - jepsen.util 63 :fail :cas [21 [10 46]] :KeyNotFoundException INFO [2019-12-17 12:27:43,313] jepsen worker 75 - jepsen.util 75 :ok :read [25 :nil] INFO [2019-12-17 12:27:43,313] jepsen worker 76 - jepsen.util 76 :ok :read [25 :nil] INFO [2019-12-17 12:27:43,313] jepsen worker 43 - jepsen.util 43 :invoke :cas [14 [43 32]] INFO [2019-12-17 12:27:43,314] jepsen worker 46 - jepsen.util 46 :invoke :write [15 32] INFO [2019-12-17 12:27:43,314] jepsen worker 79 - jepsen.util 79 :ok :read [26 :nil] INFO [2019-12-17 12:27:43,314] jepsen worker 80 - jepsen.util 80 :fail :cas [26 [20 39]] :KeyNotFoundException INFO [2019-12-17 12:27:43,314] jepsen worker 87 - jepsen.util 87 :ok :read [29 :nil] INFO [2019-12-17 12:27:43,314] jepsen worker 45 - jepsen.util 45 :invoke :cas [15 [38 42]] INFO [2019-12-17 12:27:43,314] jepsen worker 47 - jepsen.util 47 :invoke :write [15 44] INFO [2019-12-17 12:27:43,314] jepsen worker 58 - jepsen.util 58 :invoke :read [19 nil] INFO [2019-12-17 12:27:43,314] jepsen worker 67 - jepsen.util 67 :invoke :read [22 nil] INFO [2019-12-17 12:27:43,314] jepsen worker 65 - jepsen.util 65 :invoke :cas [21 [10 43]] INFO [2019-12-17 12:27:43,314] jepsen worker 70 - jepsen.util 70 :invoke :write [23 29] INFO [2019-12-17 12:27:43,314] jepsen worker 69 - jepsen.util 69 :invoke :read [23 nil] INFO [2019-12-17 12:27:43,315] jepsen worker 74 - jepsen.util 74 :invoke :read [24 nil] INFO [2019-12-17 12:27:43,315] jepsen worker 38 - jepsen.util 38 :invoke :read [12 nil] INFO [2019-12-17 12:27:43,315] jepsen worker 72 - jepsen.util 72 :invoke :write [24 14] INFO [2019-12-17 12:27:43,315] jepsen worker 64 - jepsen.util 64 :invoke :cas [21 [43 38]] INFO [2019-12-17 12:27:43,315] jepsen worker 63 - jepsen.util 63 :invoke :read [21 nil] INFO [2019-12-17 12:27:43,315] jepsen worker 59 - jepsen.util 59 :invoke :write [19 12] INFO [2019-12-17 12:27:43,315] jepsen worker 75 - jepsen.util 75 :invoke :cas [25 [30 38]] INFO [2019-12-17 12:27:43,315] jepsen worker 76 - jepsen.util 76 :invoke :read [25 nil] INFO [2019-12-17 12:27:43,316] jepsen worker 79 - jepsen.util 79 :invoke :cas [26 [35 34]] INFO [2019-12-17 12:27:43,316] jepsen worker 5 - jepsen.util 5 :ok :read [1 :nil] INFO [2019-12-17 12:27:43,316] jepsen worker 80 - jepsen.util 80 :invoke :read [26 nil] INFO [2019-12-17 12:27:43,316] jepsen worker 87 - jepsen.util 87 :invoke :cas [29 [6 8]] INFO [2019-12-17 12:27:43,316] jepsen worker 20 - jepsen.util 20 :fail :cas [6 [9 30]] :KeyNotFoundException INFO [2019-12-17 12:27:43,317] jepsen worker 11 - jepsen.util 11 :fail :cas [3 [22 45]] :KeyNotFoundException INFO [2019-12-17 12:27:43,317] jepsen worker 45 - jepsen.util 45 :fail :cas [15 [38 42]] :KeyNotFoundException INFO [2019-12-17 12:27:43,320] jepsen worker 43 - jepsen.util 43 :fail :cas [14 [43 32]] :KeyNotFoundException INFO [2019-12-17 12:27:43,324] jepsen worker 69 - jepsen.util 69 :ok :read [23 :nil] INFO [2019-12-17 12:27:43,327] jepsen worker 38 - jepsen.util 38 :ok :read [12 :nil] INFO [2019-12-17 12:27:43,328] jepsen worker 5 - jepsen.util 5 :invoke :write [1 16] INFO [2019-12-17 12:27:43,328] jepsen worker 20 - jepsen.util 20 :invoke :read [6 nil] INFO [2019-12-17 12:27:43,328] jepsen worker 11 - jepsen.util 11 :invoke :cas [3 [16 42]] INFO [2019-12-17 12:27:43,329] jepsen worker 45 - jepsen.util 45 :invoke :write [15 35] INFO [2019-12-17 12:27:43,329] jepsen worker 79 - jepsen.util 79 :fail :cas [26 [35 34]] :KeyNotFoundException INFO [2019-12-17 12:27:43,329] jepsen worker 80 - jepsen.util 80 :ok :read [26 :nil] INFO [2019-12-17 12:27:43,329] jepsen worker 43 - jepsen.util 43 :invoke :write [14 43] INFO [2019-12-17 12:27:43,330] jepsen worker 69 - jepsen.util 69 :invoke :read [23 nil] INFO [2019-12-17 12:27:43,330] jepsen worker 87 - jepsen.util 87 :fail :cas [29 [6 8]] :KeyNotFoundException INFO [2019-12-17 12:27:43,330] jepsen worker 38 - jepsen.util 38 :invoke :cas [12 [29 37]] INFO [2019-12-17 12:27:43,330] jepsen worker 79 - jepsen.util 79 :invoke :cas [26 [47 4]] INFO [2019-12-17 12:27:43,331] jepsen worker 20 - jepsen.util 20 :ok :read [6 :nil] INFO [2019-12-17 12:27:43,331] jepsen worker 80 - jepsen.util 80 :invoke :read [26 nil] INFO [2019-12-17 12:27:43,331] jepsen worker 65 - jepsen.util 65 :fail :cas [21 [10 43]] :KeyNotFoundException INFO [2019-12-17 12:27:43,331] jepsen worker 74 - jepsen.util 74 :ok :read [24 :nil] INFO [2019-12-17 12:27:43,332] jepsen worker 87 - jepsen.util 87 :invoke :cas [29 [30 7]] INFO [2019-12-17 12:27:43,333] jepsen worker 20 - jepsen.util 20 :invoke :write [6 3] INFO [2019-12-17 12:27:43,333] jepsen worker 69 - jepsen.util 69 :ok :read [23 :nil] INFO [2019-12-17 12:27:43,333] jepsen worker 65 - jepsen.util 65 :invoke :write [21 36] INFO [2019-12-17 12:27:43,334] jepsen worker 74 - jepsen.util 74 :invoke :write [24 35] INFO [2019-12-17 12:27:43,334] jepsen worker 79 - jepsen.util 79 :fail :cas [26 [47 4]] :KeyNotFoundException INFO [2019-12-17 12:27:43,334] jepsen worker 38 - jepsen.util 38 :fail :cas [12 [29 37]] :KeyNotFoundException INFO [2019-12-17 12:27:43,334] jepsen worker 80 - jepsen.util 80 :ok :read [26 :nil] INFO [2019-12-17 12:27:43,335] jepsen worker 69 - jepsen.util 69 :invoke :read [23 nil] INFO [2019-12-17 12:27:43,340] jepsen worker 87 - jepsen.util 87 :fail :cas [29 [30 7]] :KeyNotFoundException INFO [2019-12-17 12:27:43,340] jepsen worker 79 - jepsen.util 79 :invoke :write [26 47] INFO [2019-12-17 12:27:43,341] jepsen worker 38 - jepsen.util 38 :invoke :write [12 12] INFO [2019-12-17 12:27:43,341] jepsen worker 80 - jepsen.util 80 :invoke :read [26 nil] INFO [2019-12-17 12:27:43,341] jepsen worker 87 - jepsen.util 87 :invoke :read [29 nil] INFO [2019-12-17 12:27:43,344] jepsen worker 69 - jepsen.util 69 :ok :read [23 :nil] INFO [2019-12-17 12:27:43,344] jepsen worker 69 - jepsen.util 69 :invoke :cas [23 [20 47]] INFO [2019-12-17 12:27:43,345] jepsen worker 87 - jepsen.util 87 :ok :read [29 :nil] INFO [2019-12-17 12:27:43,345] jepsen worker 87 - jepsen.util 87 :invoke :write [29 45] INFO [2019-12-17 12:27:43,346] jepsen worker 80 - jepsen.util 80 :ok :read [26 :nil] INFO [2019-12-17 12:27:43,346] jepsen worker 80 - jepsen.util 80 :invoke :read [26 nil] INFO [2019-12-17 12:27:43,347] jepsen worker 69 - jepsen.util 69 :fail :cas [23 [20 47]] :KeyNotFoundException INFO [2019-12-17 12:27:43,347] jepsen worker 69 - jepsen.util 69 :invoke :read [23 nil] INFO [2019-12-17 12:27:43,348] jepsen worker 40 - jepsen.util 40 :fail :cas [13 [30 18]] :KeyNotFoundException INFO [2019-12-17 12:27:43,348] jepsen worker 82 - jepsen.util 82 :fail :cas [27 [33 26]] :KeyNotFoundException INFO [2019-12-17 12:27:43,348] jepsen worker 40 - jepsen.util 40 :invoke :cas [13 [15 44]] INFO [2019-12-17 12:27:43,348] jepsen worker 82 - jepsen.util 82 :invoke :cas [27 [36 39]] INFO [2019-12-17 12:27:43,348] jepsen worker 80 - jepsen.util 80 :ok :read [26 :nil] INFO [2019-12-17 12:27:43,348] jepsen worker 80 - jepsen.util 80 :invoke :write [26 46] INFO [2019-12-17 12:27:43,348] jepsen worker 84 - jepsen.util 84 :ok :read [28 :nil] INFO [2019-12-17 12:27:43,350] jepsen worker 67 - jepsen.util 67 :ok :read [22 :nil] INFO [2019-12-17 12:27:43,350] jepsen worker 84 - jepsen.util 84 :invoke :write [28 39] INFO [2019-12-17 12:27:43,350] jepsen worker 67 - jepsen.util 67 :invoke :cas [22 [18 11]] INFO [2019-12-17 12:27:43,350] jepsen worker 69 - jepsen.util 69 :ok :read [23 :nil] INFO [2019-12-17 12:27:43,352] jepsen worker 69 - jepsen.util 69 :invoke :cas [23 [18 25]] INFO [2019-12-17 12:27:43,353] jepsen worker 17 - jepsen.util 17 :fail :cas [5 [17 19]] :KeyNotFoundException INFO [2019-12-17 12:27:43,353] jepsen worker 58 - jepsen.util 58 :ok :read [19 :nil] INFO [2019-12-17 12:27:43,353] jepsen worker 17 - jepsen.util 17 :invoke :write [5 1] INFO [2019-12-17 12:27:43,353] jepsen worker 82 - jepsen.util 82 :fail :cas [27 [36 39]] :KeyNotFoundException INFO [2019-12-17 12:27:43,353] jepsen worker 40 - jepsen.util 40 :fail :cas [13 [15 44]] :KeyNotFoundException INFO [2019-12-17 12:27:43,353] jepsen worker 58 - jepsen.util 58 :invoke :read [19 nil] INFO [2019-12-17 12:27:43,353] jepsen worker 40 - jepsen.util 40 :invoke :write [13 26] INFO [2019-12-17 12:27:43,354] jepsen worker 82 - jepsen.util 82 :invoke :write [27 15] INFO [2019-12-17 12:27:43,354] jepsen worker 67 - jepsen.util 67 :fail :cas [22 [18 11]] :KeyNotFoundException INFO [2019-12-17 12:27:43,354] jepsen worker 67 - jepsen.util 67 :invoke :write [22 4] INFO [2019-12-17 12:27:43,356] jepsen worker 69 - jepsen.util 69 :fail :cas [23 [18 25]] :KeyNotFoundException INFO [2019-12-17 12:27:43,357] jepsen worker 69 - jepsen.util 69 :invoke :write [23 5] INFO [2019-12-17 12:27:43,361] jepsen worker 58 - jepsen.util 58 :ok :read [19 :nil] INFO [2019-12-17 12:27:43,361] jepsen worker 63 - jepsen.util 63 :ok :read [21 :nil] INFO [2019-12-17 12:27:43,361] jepsen worker 58 - jepsen.util 58 :invoke :cas [19 [9 27]] INFO [2019-12-17 12:27:43,361] jepsen worker 63 - jepsen.util 63 :invoke :cas [21 [15 28]] INFO [2019-12-17 12:27:43,362] jepsen worker 11 - jepsen.util 11 :fail :cas [3 [16 42]] :KeyNotFoundException INFO [2019-12-17 12:27:43,362] jepsen worker 11 - jepsen.util 11 :invoke :cas [3 [41 40]] INFO [2019-12-17 12:27:43,363] jepsen worker 64 - jepsen.util 64 :fail :cas [21 [43 38]] :KeyNotFoundException INFO [2019-12-17 12:27:43,363] jepsen worker 64 - jepsen.util 64 :invoke :cas [21 [48 18]] INFO [2019-12-17 12:27:43,363] jepsen worker 63 - jepsen.util 63 :fail :cas [21 [15 28]] :KeyNotFoundException INFO [2019-12-17 12:27:43,363] jepsen worker 63 - jepsen.util 63 :invoke :write [21 37] INFO [2019-12-17 12:27:43,363] jepsen worker 11 - jepsen.util 11 :fail :cas [3 [41 40]] :KeyNotFoundException INFO [2019-12-17 12:27:43,364] jepsen worker 64 - jepsen.util 64 :fail :cas [21 [48 18]] :KeyNotFoundException INFO [2019-12-17 12:27:43,364] jepsen worker 11 - jepsen.util 11 :invoke :cas [3 [6 28]] INFO [2019-12-17 12:27:43,364] jepsen worker 64 - jepsen.util 64 :invoke :read [21 nil] INFO [2019-12-17 12:27:43,364] jepsen worker 75 - jepsen.util 75 :fail :cas [25 [30 38]] :KeyNotFoundException INFO [2019-12-17 12:27:43,364] jepsen worker 76 - jepsen.util 76 :ok :read [25 :nil] INFO [2019-12-17 12:27:43,364] jepsen worker 75 - jepsen.util 75 :invoke :cas [25 [1 27]] INFO [2019-12-17 12:27:43,364] jepsen worker 76 - jepsen.util 76 :invoke :write [25 48] INFO [2019-12-17 12:27:43,364] jepsen worker 58 - jepsen.util 58 :fail :cas [19 [9 27]] :KeyNotFoundException INFO [2019-12-17 12:27:43,365] jepsen worker 11 - jepsen.util 11 :fail :cas [3 [6 28]] :KeyNotFoundException INFO [2019-12-17 12:27:43,365] jepsen worker 58 - jepsen.util 58 :invoke :read [19 nil] INFO [2019-12-17 12:27:43,365] jepsen worker 64 - jepsen.util 64 :ok :read [21 :nil] INFO [2019-12-17 12:27:43,365] jepsen worker 11 - jepsen.util 11 :invoke :cas [3 [8 1]] INFO [2019-12-17 12:27:43,365] jepsen worker 64 - jepsen.util 64 :invoke :cas [21 [1 25]] INFO [2019-12-17 12:27:43,365] jepsen worker 75 - jepsen.util 75 :fail :cas [25 [1 27]] :KeyNotFoundException INFO [2019-12-17 12:27:43,365] jepsen worker 58 - jepsen.util 58 :ok :read [19 :nil] INFO [2019-12-17 12:27:43,366] jepsen worker 75 - jepsen.util 75 :invoke :cas [25 [20 39]] INFO [2019-12-17 12:27:43,366] jepsen worker 58 - jepsen.util 58 :invoke :write [19 37] INFO [2019-12-17 12:27:43,366] jepsen worker 11 - jepsen.util 11 :fail :cas [3 [8 1]] :KeyNotFoundException INFO [2019-12-17 12:27:43,366] jepsen worker 11 - jepsen.util 11 :invoke :write [3 3] INFO [2019-12-17 12:27:43,366] jepsen worker 75 - jepsen.util 75 :fail :cas [25 [20 39]] :KeyNotFoundException INFO [2019-12-17 12:27:43,367] jepsen worker 75 - jepsen.util 75 :invoke :write [25 49] INFO [2019-12-17 12:27:43,367] jepsen worker 64 - jepsen.util 64 :fail :cas [21 [1 25]] :KeyNotFoundException INFO [2019-12-17 12:27:43,367] jepsen worker 64 - jepsen.util 64 :invoke :cas [21 [44 37]] INFO [2019-12-17 12:27:43,368] jepsen worker 64 - jepsen.util 64 :fail :cas [21 [44 37]] :KeyNotFoundException INFO [2019-12-17 12:27:43,368] jepsen worker 64 - jepsen.util 64 :invoke :read [21 nil] INFO [2019-12-17 12:27:43,369] jepsen worker 64 - jepsen.util 64 :ok :read [21 :nil] INFO [2019-12-17 12:27:43,369] jepsen worker 64 - jepsen.util 64 :invoke :cas [21 [33 21]] INFO [2019-12-17 12:27:43,369] jepsen worker 86 - jepsen.util 86 :invoke :cas [28 [22 29]] INFO [2019-12-17 12:27:43,369] jepsen worker 49 - jepsen.util 49 :ok :read [16 :nil] INFO [2019-12-17 12:27:43,369] jepsen worker 49 - jepsen.util 49 :invoke :read [16 nil] INFO [2019-12-17 12:27:43,270] jepsen worker 50 - jepsen.util 50 :invoke :cas [16 [3 25]] INFO [2019-12-17 12:27:43,270] jepsen worker 62 - jepsen.util 62 :invoke :cas [20 [11 33]] INFO [2019-12-17 12:27:43,371] jepsen worker 12 - jepsen.util 12 :fail :cas [4 [32 1]] :KeyNotFoundException INFO [2019-12-17 12:27:43,371] jepsen worker 12 - jepsen.util 12 :invoke :write [4 4] INFO [2019-12-17 12:27:43,372] jepsen worker 56 - jepsen.util 56 :invoke :cas [18 [28 37]] INFO [2019-12-17 12:27:43,372] jepsen worker 52 - jepsen.util 52 :fail :cas [17 [36 33]] :KeyNotFoundException INFO [2019-12-17 12:27:43,373] jepsen worker 52 - jepsen.util 52 :invoke :cas [17 [36 27]] INFO [2019-12-17 12:27:43,373] jepsen worker 54 - jepsen.util 54 :invoke :cas [18 [5 15]] INFO [2019-12-17 12:27:43,375] jepsen worker 89 - jepsen.util 89 :ok :read [29 :nil] INFO [2019-12-17 12:27:43,375] jepsen worker 89 - jepsen.util 89 :invoke :cas [29 [45 1]] INFO [2019-12-17 12:27:43,376] jepsen worker 56 - jepsen.util 56 :fail :cas [18 [28 37]] :KeyNotFoundException INFO [2019-12-17 12:27:43,376] jepsen worker 52 - jepsen.util 52 :fail :cas [17 [36 27]] :KeyNotFoundException INFO [2019-12-17 12:27:43,376] jepsen worker 56 - jepsen.util 56 :invoke :read [18 nil] INFO [2019-12-17 12:27:43,376] jepsen worker 52 - jepsen.util 52 :invoke :read [17 nil] INFO [2019-12-17 12:27:43,376] jepsen worker 54 - jepsen.util 54 :fail :cas [18 [5 15]] :KeyNotFoundException INFO [2019-12-17 12:27:43,377] jepsen worker 89 - jepsen.util 89 :fail :cas [29 [45 1]] :KeyNotFoundException INFO [2019-12-17 12:27:43,377] jepsen worker 54 - jepsen.util 54 :invoke :cas [18 [39 11]] INFO [2019-12-17 12:27:43,377] jepsen worker 89 - jepsen.util 89 :invoke :read [29 nil] INFO [2019-12-17 12:27:43,378] jepsen worker 52 - jepsen.util 52 :ok :read [17 :nil] INFO [2019-12-17 12:27:43,378] jepsen worker 56 - jepsen.util 56 :ok :read [18 :nil] INFO [2019-12-17 12:27:43,378] jepsen worker 52 - jepsen.util 52 :invoke :read [17 nil] INFO [2019-12-17 12:27:43,378] jepsen worker 54 - jepsen.util 54 :fail :cas [18 [39 11]] :KeyNotFoundException INFO [2019-12-17 12:27:43,378] jepsen worker 56 - jepsen.util 56 :invoke :read [18 nil] INFO [2019-12-17 12:27:43,379] jepsen worker 54 - jepsen.util 54 :invoke :cas [18 [5 46]] INFO [2019-12-17 12:27:43,379] jepsen worker 89 - jepsen.util 89 :ok :read [29 :nil] INFO [2019-12-17 12:27:43,379] jepsen worker 52 - jepsen.util 52 :ok :read [17 :nil] INFO [2019-12-17 12:27:43,379] jepsen worker 89 - jepsen.util 89 :invoke :read [29 nil] INFO [2019-12-17 12:27:43,379] jepsen worker 52 - jepsen.util 52 :invoke :cas [17 [5 1]] INFO [2019-12-17 12:27:43,379] jepsen worker 31 - jepsen.util 31 :fail :cas [10 [32 37]] :KeyNotFoundException INFO [2019-12-17 12:27:43,380] jepsen worker 56 - jepsen.util 56 :ok :read [18 :nil] INFO [2019-12-17 12:27:43,380] jepsen worker 64 - jepsen.util 64 :fail :cas [21 [33 21]] :KeyNotFoundException INFO [2019-12-17 12:27:43,380] jepsen worker 31 - jepsen.util 31 :invoke :read [10 nil] INFO [2019-12-17 12:27:43,380] jepsen worker 56 - jepsen.util 56 :invoke :write [18 28] INFO [2019-12-17 12:27:43,380] jepsen worker 54 - jepsen.util 54 :fail :cas [18 [5 46]] :KeyNotFoundException INFO [2019-12-17 12:27:43,380] jepsen worker 89 - jepsen.util 89 :ok :read [29 :nil] INFO [2019-12-17 12:27:43,380] jepsen worker 52 - jepsen.util 52 :fail :cas [17 [5 1]] :KeyNotFoundException INFO [2019-12-17 12:27:43,380] jepsen worker 64 - jepsen.util 64 :invoke :write [21 49] INFO [2019-12-17 12:27:43,380] jepsen worker 54 - jepsen.util 54 :invoke :read [18 nil] INFO [2019-12-17 12:27:43,380] jepsen worker 89 - jepsen.util 89 :invoke :write [29 5] INFO [2019-12-17 12:27:43,380] jepsen worker 52 - jepsen.util 52 :invoke :write [17 15] INFO [2019-12-17 12:27:43,381] jepsen worker 54 - jepsen.util 54 :ok :read [18 :nil] INFO [2019-12-17 12:27:43,381] jepsen worker 54 - jepsen.util 54 :invoke :write [18 47] INFO [2019-12-17 12:27:43,384] jepsen worker 73 - jepsen.util 73 :fail :cas [24 [39 37]] :KeyNotFoundException INFO [2019-12-17 12:27:43,384] jepsen worker 73 - jepsen.util 73 :invoke :read [24 nil] INFO [2019-12-17 12:27:43,389] jepsen worker 62 - jepsen.util 62 :fail :cas [20 [11 33]] :KeyNotFoundException INFO [2019-12-17 12:27:43,390] jepsen worker 62 - jepsen.util 62 :invoke :cas [20 [14 24]] INFO [2019-12-17 12:27:43,390] jepsen worker 73 - jepsen.util 73 :ok :read [24 :nil] INFO [2019-12-17 12:27:43,390] jepsen worker 31 - jepsen.util 31 :ok :read [10 :nil] INFO [2019-12-17 12:27:43,390] jepsen worker 73 - jepsen.util 73 :invoke :read [24 nil] INFO [2019-12-17 12:27:43,390] jepsen worker 31 - jepsen.util 31 :invoke :cas [10 [38 0]] INFO [2019-12-17 12:27:43,391] jepsen worker 86 - jepsen.util 86 :fail :cas [28 [22 29]] :KeyNotFoundException INFO [2019-12-17 12:27:43,391] jepsen worker 49 - jepsen.util 49 :ok :read [16 :nil] INFO [2019-12-17 12:27:43,391] jepsen worker 86 - jepsen.util 86 :invoke :cas [28 [28 36]] INFO [2019-12-17 12:27:43,391] jepsen worker 49 - jepsen.util 49 :invoke :write [16 8] INFO [2019-12-17 12:27:43,391] jepsen worker 39 - jepsen.util 39 :fail :cas [13 [28 45]] :KeyNotFoundException INFO [2019-12-17 12:27:43,391] jepsen worker 62 - jepsen.util 62 :fail :cas [20 [14 24]] :KeyNotFoundException INFO [2019-12-17 12:27:43,391] jepsen worker 39 - jepsen.util 39 :invoke :cas [13 [19 42]] INFO [2019-12-17 12:27:43,391] jepsen worker 62 - jepsen.util 62 :invoke :write [20 4] INFO [2019-12-17 12:27:43,392] jepsen worker 86 - jepsen.util 86 :fail :cas [28 [28 36]] :KeyNotFoundException INFO [2019-12-17 12:27:43,392] jepsen worker 86 - jepsen.util 86 :invoke :cas [28 [13 4]] INFO [2019-12-17 12:27:43,392] jepsen worker 15 - jepsen.util 15 :fail :cas [5 [37 32]] :KeyNotFoundException INFO [2019-12-17 12:27:43,392] jepsen worker 15 - jepsen.util 15 :invoke :cas [5 [25 32]] INFO [2019-12-17 12:27:43,392] jepsen worker 81 - jepsen.util 81 :fail :cas [27 [46 6]] :KeyNotFoundException INFO [2019-12-17 12:27:43,393] jepsen worker 81 - jepsen.util 81 :invoke :read [27 nil] INFO [2019-12-17 12:27:43,393] jepsen worker 57 - jepsen.util 57 :ok :read [19 :nil] INFO [2019-12-17 12:27:43,393] jepsen worker 57 - jepsen.util 57 :invoke :read [19 nil] INFO [2019-12-17 12:27:43,393] jepsen worker 50 - jepsen.util 50 :fail :cas [16 [3 25]] :KeyNotFoundException INFO [2019-12-17 12:27:43,394] jepsen worker 50 - jepsen.util 50 :invoke :write [16 10] INFO [2019-12-17 12:27:43,394] jepsen worker 86 - jepsen.util 86 :fail :cas [28 [13 4]] :KeyNotFoundException INFO [2019-12-17 12:27:43,394] jepsen worker 86 - jepsen.util 86 :invoke :write [28 49] INFO [2019-12-17 12:27:43,395] jepsen worker 39 - jepsen.util 39 :fail :cas [13 [19 42]] :KeyNotFoundException INFO [2019-12-17 12:27:43,396] jepsen worker 39 - jepsen.util 39 :invoke :read [13 nil] INFO [2019-12-17 12:27:43,396] jepsen worker 15 - jepsen.util 15 :fail :cas [5 [25 32]] :KeyNotFoundException INFO [2019-12-17 12:27:43,396] jepsen worker 15 - jepsen.util 15 :invoke :write [5 42] INFO [2019-12-17 12:27:43,396] jepsen worker 81 - jepsen.util 81 :ok :read [27 :nil] INFO [2019-12-17 12:27:43,396] jepsen worker 81 - jepsen.util 81 :invoke :read [27 nil] INFO [2019-12-17 12:27:43,397] jepsen worker 57 - jepsen.util 57 :ok :read [19 :nil] INFO [2019-12-17 12:27:43,397] jepsen worker 57 - jepsen.util 57 :invoke :write [19 2] INFO [2019-12-17 12:27:43,400] jepsen worker 39 - jepsen.util 39 :ok :read [13 :nil] INFO [2019-12-17 12:27:43,400] jepsen worker 73 - jepsen.util 73 :ok :read [24 :nil] INFO [2019-12-17 12:27:43,400] jepsen worker 39 - jepsen.util 39 :invoke :cas [13 [31 25]] INFO [2019-12-17 12:27:43,400] jepsen worker 73 - jepsen.util 73 :invoke :write [24 29] INFO [2019-12-17 12:27:43,400] jepsen worker 81 - jepsen.util 81 :ok :read [27 :nil] INFO [2019-12-17 12:27:43,400] jepsen worker 81 - jepsen.util 81 :invoke :read [27 nil] INFO [2019-12-17 12:27:43,401] jepsen worker 31 - jepsen.util 31 :fail :cas [10 [38 0]] :KeyNotFoundException INFO [2019-12-17 12:27:43,401] jepsen worker 31 - jepsen.util 31 :invoke :write [10 44] INFO [2019-12-17 12:27:43,406] jepsen worker 39 - jepsen.util 39 :fail :cas [13 [31 25]] :KeyNotFoundException INFO [2019-12-17 12:27:43,406] jepsen worker 39 - jepsen.util 39 :invoke :cas [13 [30 16]] INFO [2019-12-17 12:27:43,407] jepsen worker 81 - jepsen.util 81 :ok :read [27 :nil] INFO [2019-12-17 12:27:43,407] jepsen worker 81 - jepsen.util 81 :invoke :cas [27 [26 14]] INFO [2019-12-17 12:27:43,408] jepsen worker 39 - jepsen.util 39 :fail :cas [13 [30 16]] :KeyNotFoundException INFO [2019-12-17 12:27:43,408] jepsen worker 39 - jepsen.util 39 :invoke :cas [13 [37 19]] INFO [2019-12-17 12:27:43,408] jepsen worker 81 - jepsen.util 81 :fail :cas [27 [26 14]] :KeyNotFoundException INFO [2019-12-17 12:27:43,408] jepsen worker 81 - jepsen.util 81 :invoke :cas [27 [26 29]] INFO [2019-12-17 12:27:43,409] jepsen worker 39 - jepsen.util 39 :fail :cas [13 [37 19]] :KeyNotFoundException INFO [2019-12-17 12:27:43,410] jepsen worker 39 - jepsen.util 39 :invoke :cas [13 [36 36]] INFO [2019-12-17 12:27:43,410] jepsen worker 81 - jepsen.util 81 :fail :cas [27 [26 29]] :KeyNotFoundException INFO [2019-12-17 12:27:43,411] jepsen worker 81 - jepsen.util 81 :invoke :write [27 18] INFO [2019-12-17 12:27:43,412] jepsen worker 39 - jepsen.util 39 :fail :cas [13 [36 36]] :KeyNotFoundException INFO [2019-12-17 12:27:43,412] jepsen worker 39 - jepsen.util 39 :invoke :write [13 22] INFO [2019-12-17 12:27:43,898] jepsen worker 27 - jepsen.util 27 :ok :write [9 7] INFO [2019-12-17 12:27:43,898] jepsen worker 27 - jepsen.util 27 :invoke :read [9 nil] INFO [2019-12-17 12:27:43,922] jepsen worker 86 - jepsen.util 86 :ok :write [28 49] INFO [2019-12-17 12:27:43,922] jepsen worker 86 - jepsen.util 86 :invoke :write [28 14] INFO [2019-12-17 12:27:44,015] jepsen worker 71 - jepsen.util 71 :ok :write [23 12] INFO [2019-12-17 12:27:44,016] jepsen worker 71 - jepsen.util 71 :invoke :cas [23 [17 9]] INFO [2019-12-17 12:27:44,024] jepsen worker 67 - jepsen.util 67 :ok :write [22 4] INFO [2019-12-17 12:27:44,024] jepsen worker 67 - jepsen.util 67 :invoke :cas [22 [10 44]] INFO [2019-12-17 12:27:44,051] jepsen worker 10 - jepsen.util 10 :ok :write [3 42] INFO [2019-12-17 12:27:44,052] jepsen worker 10 - jepsen.util 10 :invoke :write [3 14] INFO [2019-12-17 12:27:44,058] jepsen worker 13 - jepsen.util 13 :ok :write [4 13] INFO [2019-12-17 12:27:44,059] jepsen worker 13 - jepsen.util 13 :invoke :write [4 43] INFO [2019-12-17 12:27:44,065] jepsen worker 3 - jepsen.util 3 :ok :write [1 29] INFO [2019-12-17 12:27:44,066] jepsen worker 3 - jepsen.util 3 :invoke :read [1 nil] INFO [2019-12-17 12:27:44,068] jepsen worker 32 - jepsen.util 32 :ok :write [10 32] INFO [2019-12-17 12:27:44,068] jepsen worker 32 - jepsen.util 32 :invoke :cas [10 [44 28]] INFO [2019-12-17 12:27:44,073] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":451,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:27:44,074] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":451,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:27:44,074] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":454,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:27:44,074] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":454,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:27:44,074] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":455,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:27:44,074] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":455,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:27:44,124] jepsen worker 41 - jepsen.util 41 :ok :write [13 9] INFO [2019-12-17 12:27:44,124] jepsen worker 41 - jepsen.util 41 :invoke :read [13 nil] INFO [2019-12-17 12:27:44,146] jepsen worker 0 - jepsen.util 0 :ok :write [0 0] INFO [2019-12-17 12:27:44,146] jepsen worker 0 - jepsen.util 0 :invoke :cas [0 [0 19]] INFO [2019-12-17 12:27:44,151] jepsen worker 23 - jepsen.util 23 :ok :write [7 48] INFO [2019-12-17 12:27:44,151] jepsen worker 23 - jepsen.util 23 :invoke :read [7 nil] INFO [2019-12-17 12:27:44,160] jepsen worker 67 - jepsen.util 67 :fail :cas [22 [10 44]] :ValueNotSwapFrom INFO [2019-12-17 12:27:44,161] jepsen worker 67 - jepsen.util 67 :invoke :read [22 nil] INFO [2019-12-17 12:27:44,168] jepsen worker 33 - jepsen.util 33 :ok :write [11 28] INFO [2019-12-17 12:27:44,168] jepsen worker 33 - jepsen.util 33 :invoke :cas [11 [14 0]] INFO [2019-12-17 12:27:44,184] jepsen worker 44 - jepsen.util 44 :ok :write [14 27] INFO [2019-12-17 12:27:44,184] jepsen worker 44 - jepsen.util 44 :invoke :cas [14 [15 40]] INFO [2019-12-17 12:27:44,250] jepsen worker 29 - jepsen.util 29 :ok :write [9 6] INFO [2019-12-17 12:27:44,251] jepsen worker 29 - jepsen.util 29 :invoke :read [9 nil] INFO [2019-12-17 12:27:44,261] jepsen worker 64 - jepsen.util 64 :ok :write [21 49] INFO [2019-12-17 12:27:44,261] jepsen worker 64 - jepsen.util 64 :invoke :read [21 nil] INFO [2019-12-17 12:27:44,277] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":472,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:27:44,278] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":472,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:27:44,278] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":476,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:27:44,278] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":476,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:27:44,354] jepsen worker 17 - jepsen.util 17 :ok :write [5 1] INFO [2019-12-17 12:27:44,354] jepsen worker 17 - jepsen.util 17 :invoke :write [5 6] INFO [2019-12-17 12:27:44,391] jepsen worker 56 - jepsen.util 56 :ok :write [18 28] INFO [2019-12-17 12:27:44,391] jepsen worker 56 - jepsen.util 56 :invoke :read [18 nil] INFO [2019-12-17 12:27:44,395] jepsen worker 83 - jepsen.util 83 :ok :write [27 21] INFO [2019-12-17 12:27:44,395] jepsen worker 83 - jepsen.util 83 :invoke :write [27 41] INFO [2019-12-17 12:27:44,395] jepsen worker 49 - jepsen.util 49 :ok :write [16 8] INFO [2019-12-17 12:27:44,395] jepsen worker 49 - jepsen.util 49 :invoke :write [16 35] INFO [2019-12-17 12:27:44,406] jepsen worker 19 - jepsen.util 19 :ok :write [6 45] INFO [2019-12-17 12:27:44,406] jepsen worker 72 - jepsen.util 72 :ok :write [24 14] INFO [2019-12-17 12:27:44,406] jepsen worker 19 - jepsen.util 19 :invoke :cas [6 [38 45]] INFO [2019-12-17 12:27:44,407] jepsen worker 72 - jepsen.util 72 :invoke :read [24 nil] INFO [2019-12-17 12:27:44,420] jepsen worker 52 - jepsen.util 52 :ok :write [17 15] INFO [2019-12-17 12:27:44,420] jepsen worker 52 - jepsen.util 52 :invoke :write [17 10] INFO [2019-12-17 12:27:44,443] jepsen worker 35 - jepsen.util 35 :ok :write [11 22] INFO [2019-12-17 12:27:44,443] jepsen worker 35 - jepsen.util 35 :invoke :read [11 nil] INFO [2019-12-17 12:27:44,451] jepsen worker 86 - jepsen.util 86 :ok :write [28 14] INFO [2019-12-17 12:27:44,451] jepsen worker 86 - jepsen.util 86 :invoke :cas [28 [43 33]] INFO [2019-12-17 12:27:44,475] jepsen worker 8 - jepsen.util 8 :ok :write [2 0] INFO [2019-12-17 12:27:44,475] jepsen worker 8 - jepsen.util 8 :invoke :cas [2 [18 11]] INFO [2019-12-17 12:27:44,476] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":486,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:27:44,476] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":486,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:27:44,476] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":489,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:27:44,476] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":489,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:27:44,506] jepsen worker 27 - jepsen.util 27 :ok :read [9 6] INFO [2019-12-17 12:27:44,506] jepsen worker 27 - jepsen.util 27 :invoke :read [9 nil] INFO [2019-12-17 12:27:44,514] jepsen worker 88 - jepsen.util 88 :ok :write [29 40] INFO [2019-12-17 12:27:44,514] jepsen worker 88 - jepsen.util 88 :invoke :write [29 15] INFO [2019-12-17 12:27:44,523] jepsen worker 65 - jepsen.util 65 :ok :write [21 36] INFO [2019-12-17 12:27:44,523] jepsen worker 65 - jepsen.util 65 :invoke :write [21 49] INFO [2019-12-17 12:27:44,546] jepsen worker 76 - jepsen.util 76 :ok :write [25 48] INFO [2019-12-17 12:27:44,546] jepsen worker 24 - jepsen.util 24 :ok :write [8 14] INFO [2019-12-17 12:27:44,546] jepsen worker 76 - jepsen.util 76 :invoke :read [25 nil] INFO [2019-12-17 12:27:44,546] jepsen worker 24 - jepsen.util 24 :invoke :cas [8 [11 7]] INFO [2019-12-17 12:27:44,561] jepsen worker 54 - jepsen.util 54 :ok :write [18 47] INFO [2019-12-17 12:27:44,562] jepsen worker 54 - jepsen.util 54 :invoke :write [18 0] INFO [2019-12-17 12:27:44,574] jepsen worker 69 - jepsen.util 69 :ok :write [23 5] INFO [2019-12-17 12:27:44,574] jepsen worker 69 - jepsen.util 69 :invoke :cas [23 [13 27]] INFO [2019-12-17 12:27:44,588] jepsen worker 67 - jepsen.util 67 :ok :read [22 4] INFO [2019-12-17 12:27:44,588] jepsen worker 67 - jepsen.util 67 :invoke :write [22 19] INFO [2019-12-17 12:27:44,631] jepsen worker 23 - jepsen.util 23 :ok :read [7 48] INFO [2019-12-17 12:27:44,631] jepsen worker 23 - jepsen.util 23 :invoke :read [7 nil] INFO [2019-12-17 12:27:44,647] jepsen worker 86 - jepsen.util 86 :fail :cas [28 [43 33]] :ValueNotSwapFrom INFO [2019-12-17 12:27:44,648] jepsen worker 86 - jepsen.util 86 :invoke :cas [28 [4 24]] INFO [2019-12-17 12:27:44,694] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":504,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:27:44,695] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":504,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:27:44,695] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":507,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:27:44,695] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":507,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:27:44,695] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":508,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:27:44,695] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":508,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:27:44,715] jepsen worker 19 - jepsen.util 19 :fail :cas [6 [38 45]] :ValueNotSwapFrom INFO [2019-12-17 12:27:44,715] jepsen worker 19 - jepsen.util 19 :invoke :cas [6 [43 33]] INFO [2019-12-17 12:27:44,725] jepsen worker 1 - jepsen.util 1 :ok :write [0 14] INFO [2019-12-17 12:27:44,726] jepsen worker 1 - jepsen.util 1 :invoke :cas [0 [0 34]] INFO [2019-12-17 12:27:44,728] jepsen worker 35 - jepsen.util 35 :ok :read [11 22] INFO [2019-12-17 12:27:44,728] jepsen worker 35 - jepsen.util 35 :invoke :cas [11 [36 17]] INFO [2019-12-17 12:27:44,752] jepsen worker 59 - jepsen.util 59 :ok :write [19 12] INFO [2019-12-17 12:27:44,753] jepsen worker 59 - jepsen.util 59 :invoke :cas [19 [44 19]] INFO [2019-12-17 12:27:44,780] jepsen worker 3 - jepsen.util 3 :ok :read [1 29] INFO [2019-12-17 12:27:44,780] jepsen worker 3 - jepsen.util 3 :invoke :cas [1 [10 44]] INFO [2019-12-17 12:27:44,801] jepsen worker 66 - jepsen.util 66 :ok :write [22 40] INFO [2019-12-17 12:27:44,801] jepsen worker 66 - jepsen.util 66 :invoke :read [22 nil] INFO [2019-12-17 12:27:44,833] jepsen worker 37 - jepsen.util 37 :ok :write [12 23] INFO [2019-12-17 12:27:44,834] jepsen worker 37 - jepsen.util 37 :invoke :cas [12 [8 37]] INFO [2019-12-17 12:27:44,842] jepsen worker 19 - jepsen.util 19 :fail :cas [6 [43 33]] :ValueNotSwapFrom INFO [2019-12-17 12:27:44,842] jepsen worker 19 - jepsen.util 19 :invoke :write [6 6] INFO [2019-12-17 12:27:44,894] jepsen worker 41 - jepsen.util 41 :ok :read [13 9] INFO [2019-12-17 12:27:44,895] jepsen worker 41 - jepsen.util 41 :invoke :cas [13 [7 6]] INFO [2019-12-17 12:27:44,902] jepsen worker 0 - jepsen.util 0 :fail :cas [0 [0 19]] :ValueNotSwapFrom INFO [2019-12-17 12:27:44,903] jepsen worker 0 - jepsen.util 0 :invoke :cas [0 [28 9]] INFO [2019-12-17 12:27:44,913] jepsen worker 21 - jepsen.util 21 :ok :write [7 20] INFO [2019-12-17 12:27:44,913] jepsen worker 21 - jepsen.util 21 :invoke :write [7 17] INFO [2019-12-17 12:27:44,946] jepsen worker 59 - jepsen.util 59 :fail :cas [19 [44 19]] :ValueNotSwapFrom INFO [2019-12-17 12:27:44,947] jepsen worker 59 - jepsen.util 59 :invoke :write [19 11] INFO [2019-12-17 12:27:44,965] jepsen worker 33 - jepsen.util 33 :fail :cas [11 [14 0]] :ValueNotSwapFrom INFO [2019-12-17 12:27:44,965] jepsen worker 33 - jepsen.util 33 :invoke :write [11 44] INFO [2019-12-17 12:27:44,979] jepsen worker 6 - jepsen.util 6 :ok :write [2 15] INFO [2019-12-17 12:27:44,979] jepsen worker 6 - jepsen.util 6 :invoke :cas [2 [12 20]] INFO [2019-12-17 12:27:44,985] jepsen worker 37 - jepsen.util 37 :fail :cas [12 [8 37]] :ValueNotSwapFrom INFO [2019-12-17 12:27:44,986] jepsen worker 37 - jepsen.util 37 :invoke :cas [12 [41 17]] INFO [2019-12-17 12:27:44,998] jepsen worker 78 - jepsen.util 78 :ok :write [26 49] INFO [2019-12-17 12:27:44,998] jepsen worker 78 - jepsen.util 78 :invoke :read [26 nil] INFO [2019-12-17 12:27:45,045] jepsen worker 51 - jepsen.util 51 :ok :write [17 0] INFO [2019-12-17 12:27:45,046] jepsen worker 51 - jepsen.util 51 :invoke :read [17 nil] INFO [2019-12-17 12:27:45,054] jepsen worker 30 - jepsen.util 30 :ok :write [10 23] INFO [2019-12-17 12:27:45,055] jepsen worker 30 - jepsen.util 30 :invoke :read [10 nil] INFO [2019-12-17 12:27:45,055] jepsen worker 45 - jepsen.util 45 :ok :write [15 35] INFO [2019-12-17 12:27:45,056] jepsen worker 45 - jepsen.util 45 :invoke :write [15 13] INFO [2019-12-17 12:27:45,061] jepsen worker 5 - jepsen.util 5 :ok :write [1 16] INFO [2019-12-17 12:27:45,061] jepsen worker 87 - jepsen.util 87 :ok :write [29 45] INFO [2019-12-17 12:27:45,061] jepsen worker 5 - jepsen.util 5 :invoke :read [1 nil] INFO [2019-12-17 12:27:45,061] jepsen worker 87 - jepsen.util 87 :invoke :read [29 nil] INFO [2019-12-17 12:27:45,073] jepsen worker 70 - jepsen.util 70 :ok :write [23 29] INFO [2019-12-17 12:27:45,073] jepsen worker 70 - jepsen.util 70 :invoke :write [23 16] INFO [2019-12-17 12:27:45,073] jepsen worker 16 - jepsen.util 16 :ok :write [5 32] INFO [2019-12-17 12:27:45,074] jepsen worker 16 - jepsen.util 16 :invoke :read [5 nil] INFO [2019-12-17 12:27:45,077] jepsen worker 34 - jepsen.util 34 :ok :write [11 7] INFO [2019-12-17 12:27:45,077] jepsen worker 34 - jepsen.util 34 :invoke :read [11 nil] INFO [2019-12-17 12:27:45,132] jepsen worker 61 - jepsen.util 61 :ok :write [20 7] INFO [2019-12-17 12:27:45,132] jepsen worker 61 - jepsen.util 61 :invoke :write [20 29] INFO [2019-12-17 12:27:45,141] jepsen worker 84 - jepsen.util 84 :ok :write [28 39] INFO [2019-12-17 12:27:45,142] jepsen worker 84 - jepsen.util 84 :invoke :read [28 nil] INFO [2019-12-17 12:27:45,162] jepsen worker 37 - jepsen.util 37 :fail :cas [12 [41 17]] :ValueNotSwapFrom INFO [2019-12-17 12:27:45,162] jepsen worker 37 - jepsen.util 37 :invoke :cas [12 [22 21]] INFO [2019-12-17 12:27:45,168] jepsen worker 69 - jepsen.util 69 :fail :cas [23 [13 27]] :ValueNotSwapFrom INFO [2019-12-17 12:27:45,168] jepsen worker 69 - jepsen.util 69 :invoke :write [23 40] INFO [2019-12-17 12:27:45,170] jepsen worker 28 - jepsen.util 28 :ok :write [9 30] INFO [2019-12-17 12:27:45,171] jepsen worker 28 - jepsen.util 28 :invoke :write [9 38] INFO [2019-12-17 12:27:45,191] jepsen worker 59 - jepsen.util 59 :ok :write [19 11] INFO [2019-12-17 12:27:45,192] jepsen worker 59 - jepsen.util 59 :invoke :write [19 37] INFO [2019-12-17 12:27:45,212] jepsen worker 76 - jepsen.util 76 :ok :read [25 48] INFO [2019-12-17 12:27:45,213] jepsen worker 76 - jepsen.util 76 :invoke :cas [25 [42 10]] INFO [2019-12-17 12:27:45,226] jepsen worker 0 - jepsen.util 0 :fail :cas [0 [28 9]] :ValueNotSwapFrom INFO [2019-12-17 12:27:45,227] jepsen worker 0 - jepsen.util 0 :invoke :cas [0 [11 40]] INFO [2019-12-17 12:27:45,228] jepsen worker 12 - jepsen.util 12 :ok :write [4 4] INFO [2019-12-17 12:27:45,229] jepsen worker 12 - jepsen.util 12 :invoke :cas [4 [34 28]] INFO [2019-12-17 12:27:45,244] jepsen worker 3 - jepsen.util 3 :fail :cas [1 [10 44]] :ValueNotSwapFrom INFO [2019-12-17 12:27:45,244] jepsen worker 3 - jepsen.util 3 :invoke :read [1 nil] INFO [2019-12-17 12:27:45,270] jepsen worker 41 - jepsen.util 41 :fail :cas [13 [7 6]] :ValueNotSwapFrom INFO [2019-12-17 12:27:45,270] jepsen worker 41 - jepsen.util 41 :invoke :write [13 18] INFO [2019-12-17 12:27:45,277] jepsen worker 66 - jepsen.util 66 :ok :read [22 40] INFO [2019-12-17 12:27:45,277] jepsen worker 66 - jepsen.util 66 :invoke :read [22 nil] INFO [2019-12-17 12:27:45,289] jepsen worker 34 - jepsen.util 34 :ok :read [11 7] INFO [2019-12-17 12:27:45,290] jepsen worker 34 - jepsen.util 34 :invoke :cas [11 [36 7]] INFO [2019-12-17 12:27:45,310] jepsen worker 84 - jepsen.util 84 :ok :read [28 39] INFO [2019-12-17 12:27:45,311] jepsen worker 84 - jepsen.util 84 :invoke :write [28 15] INFO [2019-12-17 12:27:45,312] jepsen worker 24 - jepsen.util 24 :fail :cas [8 [11 7]] :ValueNotSwapFrom INFO [2019-12-17 12:27:45,312] jepsen worker 24 - jepsen.util 24 :invoke :cas [8 [20 23]] INFO [2019-12-17 12:27:45,312] jepsen worker 44 - jepsen.util 44 :fail :cas [14 [15 40]] :ValueNotSwapFrom INFO [2019-12-17 12:27:45,312] jepsen worker 44 - jepsen.util 44 :invoke :read [14 nil] INFO [2019-12-17 12:27:45,315] jepsen worker 10 - jepsen.util 10 :ok :write [3 14] INFO [2019-12-17 12:27:45,315] jepsen worker 74 - jepsen.util 74 :ok :write [24 35] INFO [2019-12-17 12:27:45,315] jepsen worker 10 - jepsen.util 10 :invoke :write [3 32] INFO [2019-12-17 12:27:45,315] jepsen worker 74 - jepsen.util 74 :invoke :cas [24 [3 1]] INFO [2019-12-17 12:27:45,334] jepsen worker 76 - jepsen.util 76 :fail :cas [25 [42 10]] :ValueNotSwapFrom INFO [2019-12-17 12:27:45,335] jepsen worker 76 - jepsen.util 76 :invoke :cas [25 [49 6]] INFO [2019-12-17 12:27:45,376] jepsen worker 66 - jepsen.util 66 :ok :read [22 40] INFO [2019-12-17 12:27:45,376] jepsen worker 66 - jepsen.util 66 :invoke :cas [22 [25 44]] INFO [2019-12-17 12:27:45,392] jepsen worker 29 - jepsen.util 29 :ok :read [9 30] INFO [2019-12-17 12:27:45,392] jepsen worker 29 - jepsen.util 29 :invoke :write [9 5] INFO [2019-12-17 12:27:45,402] jepsen worker 0 - jepsen.util 0 :fail :cas [0 [11 40]] :ValueNotSwapFrom INFO [2019-12-17 12:27:45,402] jepsen worker 0 - jepsen.util 0 :invoke :read [0 nil] INFO [2019-12-17 12:27:45,402] jepsen worker 64 - jepsen.util 64 :ok :read [21 36] INFO [2019-12-17 12:27:45,403] jepsen worker 64 - jepsen.util 64 :invoke :cas [21 [16 0]] INFO [2019-12-17 12:27:45,432] jepsen worker 32 - jepsen.util 32 :fail :cas [10 [44 28]] :ValueNotSwapFrom INFO [2019-12-17 12:27:45,432] jepsen worker 32 - jepsen.util 32 :invoke :read [10 nil] INFO [2019-12-17 12:27:45,447] jepsen worker 34 - jepsen.util 34 :fail :cas [11 [36 7]] :ValueNotSwapFrom INFO [2019-12-17 12:27:45,447] jepsen worker 34 - jepsen.util 34 :invoke :read [11 nil] INFO [2019-12-17 12:27:45,451] jepsen worker 6 - jepsen.util 6 :fail :cas [2 [12 20]] :ValueNotSwapFrom INFO [2019-12-17 12:27:45,451] jepsen worker 6 - jepsen.util 6 :invoke :cas [2 [5 14]] INFO [2019-12-17 12:27:45,454] jepsen worker 81 - jepsen.util 81 :ok :write [27 18] INFO [2019-12-17 12:27:45,455] jepsen worker 81 - jepsen.util 81 :invoke :read [27 nil] INFO [2019-12-17 12:27:45,457] jepsen worker 87 - jepsen.util 87 :ok :read [29 45] INFO [2019-12-17 12:27:45,457] jepsen worker 87 - jepsen.util 87 :invoke :read [29 nil] INFO [2019-12-17 12:27:45,492] jepsen worker 74 - jepsen.util 74 :fail :cas [24 [3 1]] :ValueNotSwapFrom INFO [2019-12-17 12:27:45,492] jepsen worker 74 - jepsen.util 74 :invoke :read [24 nil] INFO [2019-12-17 12:27:45,506] jepsen worker 22 - jepsen.util 22 :info :write [7 23] :RequestCanceledException INFO [2019-12-17 12:27:45,507] jepsen worker 4 - jepsen.util 4 :info :write [1 23] :RequestCanceledException INFO [2019-12-17 12:27:45,507] jepsen worker 40 - jepsen.util 40 :info :write [13 26] :RequestCanceledException INFO [2019-12-17 12:27:45,508] jepsen worker 53 - jepsen.util 53 :info :write [17 31] :RequestCanceledException INFO [2019-12-17 12:27:45,517] jepsen worker 4 - jepsen.util 94 :invoke :read [1 nil] INFO [2019-12-17 12:27:45,517] jepsen worker 53 - jepsen.util 143 :invoke :write [17 4] INFO [2019-12-17 12:27:45,517] jepsen worker 22 - jepsen.util 112 :invoke :cas [7 [24 47]] INFO [2019-12-17 12:27:45,517] jepsen worker 40 - jepsen.util 130 :invoke :cas [13 [30 3]] INFO [2019-12-17 12:27:45,519] jepsen worker 55 - jepsen.util 55 :info :write [18 34] :RequestCanceledException INFO [2019-12-17 12:27:45,520] jepsen worker 60 - jepsen.util 60 :info :write [20 41] :RequestCanceledException INFO [2019-12-17 12:27:45,522] jepsen worker 36 - jepsen.util 36 :info :write [12 0] :RequestCanceledException INFO [2019-12-17 12:27:45,522] jepsen worker 9 - jepsen.util 9 :info :write [3 2] :RequestCanceledException INFO [2019-12-17 12:27:45,524] jepsen worker 39 - jepsen.util 39 :info :write [13 22] :RequestCanceledException INFO [2019-12-17 12:27:45,528] jepsen worker 36 - jepsen.util 126 :invoke :read [12 nil] INFO [2019-12-17 12:27:45,528] jepsen worker 60 - jepsen.util 150 :invoke :write [20 33] INFO [2019-12-17 12:27:45,528] jepsen worker 55 - jepsen.util 145 :invoke :read [18 nil] INFO [2019-12-17 12:27:45,528] jepsen worker 9 - jepsen.util 99 :invoke :read [3 nil] INFO [2019-12-17 12:27:45,529] jepsen worker 39 - jepsen.util 129 :invoke :write [13 37] INFO [2019-12-17 12:27:45,532] jepsen worker 14 - jepsen.util 14 :info :write [4 24] :RequestCanceledException INFO [2019-12-17 12:27:45,533] jepsen worker 26 - jepsen.util 26 :info :write [8 25] :RequestCanceledException INFO [2019-12-17 12:27:45,534] jepsen worker 2 - jepsen.util 2 :info :write [0 34] :RequestCanceledException INFO [2019-12-17 12:27:45,537] jepsen worker 14 - jepsen.util 104 :invoke :write [4 32] INFO [2019-12-17 12:27:45,539] jepsen worker 2 - jepsen.util 92 :invoke :cas [0 [34 24]] INFO [2019-12-17 12:27:45,539] jepsen worker 26 - jepsen.util 116 :invoke :write [8 20] INFO [2019-12-17 12:27:45,540] jepsen worker 89 - jepsen.util 89 :ok :write [29 5] INFO [2019-12-17 12:27:45,540] jepsen worker 89 - jepsen.util 89 :invoke :cas [29 [3 1]] INFO [2019-12-17 12:27:45,546] jepsen worker 85 - jepsen.util 85 :info :write [28 32] :RequestCanceledException INFO [2019-12-17 12:27:45,546] jepsen worker 48 - jepsen.util 48 :info :write [16 41] :RequestCanceledException INFO [2019-12-17 12:27:45,548] jepsen worker 25 - jepsen.util 25 :info :write [8 8] :RequestCanceledException INFO [2019-12-17 12:27:45,549] jepsen worker 79 - jepsen.util 79 :info :write [26 47] :RequestCanceledException INFO [2019-12-17 12:27:45,552] jepsen worker 68 - jepsen.util 68 :info :write [22 41] :RequestCanceledException INFO [2019-12-17 12:27:45,552] jepsen worker 32 - jepsen.util 32 :ok :read [10 23] INFO [2019-12-17 12:27:45,557] jepsen worker 73 - jepsen.util 73 :info :write [24 29] :RequestCanceledException INFO [2019-12-17 12:27:45,557] jepsen worker 31 - jepsen.util 31 :info :write [10 44] :RequestCanceledException INFO [2019-12-17 12:27:45,557] jepsen worker 7 - jepsen.util 7 :info :write [2 33] :RequestCanceledException INFO [2019-12-17 12:27:45,557] jepsen worker 50 - jepsen.util 50 :info :write [16 10] :RequestCanceledException INFO [2019-12-17 12:27:45,558] jepsen worker 32 - jepsen.util 32 :invoke :read [10 nil] INFO [2019-12-17 12:27:45,561] jepsen worker 85 - jepsen.util 175 :invoke :read [28 nil] INFO [2019-12-17 12:27:45,562] jepsen worker 25 - jepsen.util 115 :invoke :write [8 28] INFO [2019-12-17 12:27:45,562] jepsen worker 48 - jepsen.util 138 :invoke :cas [16 [4 37]] INFO [2019-12-17 12:27:45,562] jepsen worker 50 - jepsen.util 140 :invoke :cas [16 [15 13]] INFO [2019-12-17 12:27:45,562] jepsen worker 31 - jepsen.util 121 :invoke :write [10 29] INFO [2019-12-17 12:27:45,562] jepsen worker 7 - jepsen.util 97 :invoke :write [2 35] INFO [2019-12-17 12:27:45,562] jepsen worker 68 - jepsen.util 158 :invoke :write [22 13] INFO [2019-12-17 12:27:45,562] jepsen worker 79 - jepsen.util 169 :invoke :read [26 nil] INFO [2019-12-17 12:27:45,562] jepsen worker 73 - jepsen.util 163 :invoke :read [24 nil] INFO [2019-12-17 12:27:45,567] jepsen worker 3 - jepsen.util 3 :ok :read [1 16] INFO [2019-12-17 12:27:45,567] jepsen worker 3 - jepsen.util 3 :invoke :write [1 16] INFO [2019-12-17 12:27:45,586] jepsen worker 46 - jepsen.util 46 :info :write [15 32] :RequestCanceledException INFO [2019-12-17 12:27:45,586] jepsen worker 11 - jepsen.util 11 :info :write [3 3] :RequestCanceledException INFO [2019-12-17 12:27:45,587] jepsen worker 63 - jepsen.util 63 :info :write [21 37] :RequestCanceledException INFO [2019-12-17 12:27:45,589] jepsen worker 77 - jepsen.util 77 :info :write [25 2] :RequestCanceledException INFO [2019-12-17 12:27:45,589] jepsen worker 82 - jepsen.util 82 :info :write [27 15] :RequestCanceledException INFO [2019-12-17 12:27:45,592] jepsen worker 11 - jepsen.util 101 :invoke :read [3 nil] INFO [2019-12-17 12:27:45,592] jepsen worker 77 - jepsen.util 167 :invoke :read [25 nil] INFO [2019-12-17 12:27:45,592] jepsen worker 46 - jepsen.util 136 :invoke :write [15 49] INFO [2019-12-17 12:27:45,592] jepsen worker 63 - jepsen.util 153 :invoke :cas [21 [18 24]] INFO [2019-12-17 12:27:45,593] jepsen worker 82 - jepsen.util 172 :invoke :cas [27 [2 3]] INFO [2019-12-17 12:27:45,604] jepsen worker 47 - jepsen.util 47 :info :write [15 44] :RequestCanceledException INFO [2019-12-17 12:27:45,604] jepsen worker 58 - jepsen.util 58 :info :write [19 37] :RequestCanceledException INFO [2019-12-17 12:27:45,607] jepsen worker 58 - jepsen.util 148 :invoke :read [19 nil] INFO [2019-12-17 12:27:45,607] jepsen worker 47 - jepsen.util 137 :invoke :read [15 nil] INFO [2019-12-17 12:27:45,621] jepsen worker 18 - jepsen.util 18 :info :write [6 47] :RequestCanceledException INFO [2019-12-17 12:27:45,622] jepsen worker 15 - jepsen.util 15 :info :write [5 42] :RequestCanceledException INFO [2019-12-17 12:27:45,622] jepsen worker 42 - jepsen.util 42 :info :write [14 29] :RequestCanceledException INFO [2019-12-17 12:27:45,622] jepsen worker 57 - jepsen.util 57 :info :write [19 2] :RequestCanceledException INFO [2019-12-17 12:27:45,623] jepsen worker 75 - jepsen.util 75 :info :write [25 49] :RequestCanceledException INFO [2019-12-17 12:27:45,626] jepsen worker 75 - jepsen.util 165 :invoke :read [25 nil] INFO [2019-12-17 12:27:45,628] jepsen worker 15 - jepsen.util 105 :invoke :write [5 28] INFO [2019-12-17 12:27:45,628] jepsen worker 18 - jepsen.util 108 :invoke :cas [6 [3 34]] INFO [2019-12-17 12:27:45,628] jepsen worker 42 - jepsen.util 132 :invoke :write [14 28] INFO [2019-12-17 12:27:45,629] jepsen worker 43 - jepsen.util 43 :info :write [14 43] :RequestCanceledException INFO [2019-12-17 12:27:45,629] jepsen worker 80 - jepsen.util 80 :info :write [26 46] :RequestCanceledException INFO [2019-12-17 12:27:45,629] jepsen worker 57 - jepsen.util 147 :invoke :read [19 nil] INFO [2019-12-17 12:27:45,631] jepsen worker 20 - jepsen.util 20 :info :write [6 3] :RequestCanceledException INFO [2019-12-17 12:27:45,631] jepsen worker 38 - jepsen.util 38 :info :write [12 12] :RequestCanceledException INFO [2019-12-17 12:27:45,631] jepsen worker 62 - jepsen.util 62 :info :write [20 4] :RequestCanceledException INFO [2019-12-17 12:27:45,635] jepsen worker 38 - jepsen.util 128 :invoke :read [12 nil] INFO [2019-12-17 12:27:45,635] jepsen worker 62 - jepsen.util 152 :invoke :read [20 nil] INFO [2019-12-17 12:27:45,635] jepsen worker 80 - jepsen.util 170 :invoke :write [26 19] INFO [2019-12-17 12:27:45,635] jepsen worker 43 - jepsen.util 133 :invoke :write [14 27] INFO [2019-12-17 12:27:45,635] jepsen worker 20 - jepsen.util 110 :invoke :write [6 11] INFO [2019-12-17 12:27:45,636] jepsen worker 78 - jepsen.util 78 :ok :read [26 49] INFO [2019-12-17 12:27:45,636] jepsen worker 78 - jepsen.util 78 :invoke :write [26 26] INFO [2019-12-17 12:27:45,642] jepsen worker 74 - jepsen.util 74 :ok :read [24 35] INFO [2019-12-17 12:27:45,642] jepsen worker 74 - jepsen.util 74 :invoke :cas [24 [5 31]] INFO [2019-12-17 12:27:45,772] jepsen worker 76 - jepsen.util 76 :fail :cas [25 [49 6]] :ValueNotSwapFrom INFO [2019-12-17 12:27:45,772] jepsen worker 76 - jepsen.util 76 :invoke :cas [25 [27 48]] INFO [2019-12-17 12:27:45,995] jepsen worker 24 - jepsen.util 24 :fail :cas [8 [20 23]] :ValueNotSwapFrom INFO [2019-12-17 12:27:45,995] jepsen worker 24 - jepsen.util 24 :invoke :read [8 nil] INFO [2019-12-17 12:27:46,071] jepsen worker 87 - jepsen.util 87 :ok :read [29 5] INFO [2019-12-17 12:27:46,071] jepsen worker 87 - jepsen.util 87 :invoke :cas [29 [40 3]] INFO [2019-12-17 12:27:46,080] jepsen worker 44 - jepsen.util 44 :ok :read [14 27] INFO [2019-12-17 12:27:46,081] jepsen worker 44 - jepsen.util 44 :invoke :cas [14 [25 12]] INFO [2019-12-17 12:27:46,084] jepsen worker 23 - jepsen.util 23 :ok :read [7 20] INFO [2019-12-17 12:27:46,084] jepsen worker 23 - jepsen.util 23 :invoke :cas [7 [19 30]] INFO [2019-12-17 12:27:46,088] jepsen worker 71 - jepsen.util 71 :info :cas [23 [17 9]] :RequestCanceledException INFO [2019-12-17 12:27:46,091] jepsen worker 71 - jepsen.util 161 :invoke :cas [23 [49 3]] INFO [2019-12-17 12:27:46,134] jepsen worker 65 - jepsen.util 65 :ok :write [21 49] INFO [2019-12-17 12:27:46,134] jepsen worker 65 - jepsen.util 65 :invoke :write [21 38] INFO [2019-12-17 12:27:46,139] jepsen worker 6 - jepsen.util 6 :fail :cas [2 [5 14]] :ValueNotSwapFrom INFO [2019-12-17 12:27:46,139] jepsen worker 6 - jepsen.util 6 :invoke :read [2 nil] INFO [2019-12-17 12:27:46,152] jepsen worker 13 - jepsen.util 13 :info :write [4 43] :RequestCanceledException INFO [2019-12-17 12:27:46,155] jepsen worker 13 - jepsen.util 103 :invoke :cas [4 [17 16]] INFO [2019-12-17 12:27:46,194] jepsen worker 34 - jepsen.util 34 :ok :read [11 7] INFO [2019-12-17 12:27:46,194] jepsen worker 34 - jepsen.util 34 :invoke :write [11 46] INFO [2019-12-17 12:27:46,199] jepsen worker 12 - jepsen.util 12 :fail :cas [4 [34 28]] :ValueNotSwapFrom INFO [2019-12-17 12:27:46,199] jepsen worker 12 - jepsen.util 12 :invoke :write [4 44] INFO [2019-12-17 12:27:46,216] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":645,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:27:46,216] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":645,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:27:46,291] jepsen worker 32 - jepsen.util 32 :ok :read [10 23] INFO [2019-12-17 12:27:46,292] jepsen worker 32 - jepsen.util 32 :invoke :read [10 nil] INFO [2019-12-17 12:27:46,309] jepsen worker 84 - jepsen.util 84 :ok :write [28 15] INFO [2019-12-17 12:27:46,310] jepsen worker 84 - jepsen.util 84 :invoke :read [28 nil] INFO [2019-12-17 12:27:46,369] jepsen worker 72 - jepsen.util 72 :ok :read [24 35] INFO [2019-12-17 12:27:46,370] jepsen worker 72 - jepsen.util 72 :invoke :cas [24 [10 10]] INFO [2019-12-17 12:27:46,371] jepsen worker 0 - jepsen.util 0 :ok :read [0 14] INFO [2019-12-17 12:27:46,371] jepsen worker 0 - jepsen.util 0 :invoke :write [0 24] INFO [2019-12-17 12:27:46,386] jepsen worker 17 - jepsen.util 17 :info :write [5 6] :RequestCanceledException INFO [2019-12-17 12:27:46,388] jepsen worker 17 - jepsen.util 107 :invoke :write [5 20] INFO [2019-12-17 12:27:46,420] jepsen worker 27 - jepsen.util 27 :ok :read [9 30] INFO [2019-12-17 12:27:46,420] jepsen worker 49 - jepsen.util 49 :info :write [16 35] :RequestCanceledException INFO [2019-12-17 12:27:46,421] jepsen worker 27 - jepsen.util 27 :invoke :write [9 46] INFO [2019-12-17 12:27:46,423] jepsen worker 49 - jepsen.util 139 :invoke :read [16 nil] INFO [2019-12-17 12:27:46,434] jepsen worker 56 - jepsen.util 56 :fail :read [18 nil] :RequestTimeoutException INFO [2019-12-17 12:27:46,434] jepsen worker 56 - jepsen.util 56 :invoke :read [18 nil] INFO [2019-12-17 12:27:46,453] jepsen worker 81 - jepsen.util 81 :ok :read [27 18] INFO [2019-12-17 12:27:46,466] jepsen worker 63 - jepsen.util 153 :fail :cas [21 [18 24]] :ValueNotSwapFrom INFO [2019-12-17 12:27:46,472] jepsen worker 81 - jepsen.util 81 :invoke :read [27 nil] INFO [2019-12-17 12:27:46,476] jepsen worker 63 - jepsen.util 153 :invoke :write [21 19] INFO [2019-12-17 12:27:46,503] jepsen worker 18 - jepsen.util 108 :fail :cas [6 [3 34]] :ValueNotSwapFrom INFO [2019-12-17 12:27:46,504] jepsen worker 18 - jepsen.util 108 :invoke :cas [6 [16 20]] INFO [2019-12-17 12:27:46,505] jepsen worker 83 - jepsen.util 83 :info :write [27 41] :RequestCanceledException INFO [2019-12-17 12:27:46,508] jepsen worker 83 - jepsen.util 173 :invoke :write [27 37] INFO [2019-12-17 12:27:46,512] jepsen worker 52 - jepsen.util 52 :info :write [17 10] :RequestCanceledException INFO [2019-12-17 12:27:46,515] jepsen worker 52 - jepsen.util 142 :invoke :read [17 nil] INFO [2019-12-17 12:27:46,526] jepsen worker 8 - jepsen.util 8 :info :cas [2 [18 11]] :RequestCanceledException INFO [2019-12-17 12:27:46,527] jepsen worker 8 - jepsen.util 98 :invoke :read [2 nil] INFO [2019-12-17 12:27:46,539] jepsen worker 79 - jepsen.util 169 :ok :read [26 47] INFO [2019-12-17 12:27:46,540] jepsen worker 79 - jepsen.util 169 :invoke :write [26 29] INFO [2019-12-17 12:27:46,579] jepsen worker 54 - jepsen.util 54 :info :write [18 0] :RequestCanceledException INFO [2019-12-17 12:27:46,582] jepsen worker 54 - jepsen.util 144 :invoke :write [18 21] INFO [2019-12-17 12:27:46,593] jepsen worker 66 - jepsen.util 66 :fail :cas [22 [25 44]] :ValueNotSwapFrom INFO [2019-12-17 12:27:46,593] jepsen worker 66 - jepsen.util 66 :invoke :cas [22 [21 24]] INFO [2019-12-17 12:27:46,595] jepsen worker 72 - jepsen.util 72 :fail :cas [24 [10 10]] :ValueNotSwapFrom INFO [2019-12-17 12:27:46,595] jepsen worker 72 - jepsen.util 72 :invoke :cas [24 [14 33]] INFO [2019-12-17 12:27:46,604] jepsen worker 88 - jepsen.util 88 :info :write [29 15] :RequestCanceledException INFO [2019-12-17 12:27:46,606] jepsen worker 88 - jepsen.util 178 :invoke :cas [29 [48 14]] INFO [2019-12-17 12:27:46,634] jepsen worker 67 - jepsen.util 67 :info :write [22 19] :RequestCanceledException INFO [2019-12-17 12:27:46,636] jepsen worker 67 - jepsen.util 157 :invoke :write [22 5] INFO [2019-12-17 12:27:46,653] jepsen worker 85 - jepsen.util 175 :ok :read [28 15] INFO [2019-12-17 12:27:46,654] jepsen worker 85 - jepsen.util 175 :invoke :read [28 nil] INFO [2019-12-17 12:27:46,669] jepsen worker 5 - jepsen.util 5 :ok :read [1 23] INFO [2019-12-17 12:27:46,670] jepsen worker 5 - jepsen.util 5 :invoke :cas [1 [35 35]] INFO [2019-12-17 12:27:46,683] jepsen worker 24 - jepsen.util 24 :ok :read [8 14] INFO [2019-12-17 12:27:46,683] jepsen worker 24 - jepsen.util 24 :invoke :read [8 nil] INFO [2019-12-17 12:27:46,696] jepsen worker 35 - jepsen.util 35 :fail :cas [11 [36 17]] :ValueNotSwapFrom INFO [2019-12-17 12:27:46,696] jepsen worker 35 - jepsen.util 35 :invoke :read [11 nil] INFO [2019-12-17 12:27:46,710] jepsen worker 40 - jepsen.util 130 :fail :cas [13 [30 3]] :ValueNotSwapFrom INFO [2019-12-17 12:27:46,710] jepsen worker 40 - jepsen.util 130 :invoke :write [13 29] INFO [2019-12-17 12:27:46,713] jepsen worker 4 - jepsen.util 94 :ok :read [1 23] INFO [2019-12-17 12:27:46,713] jepsen worker 4 - jepsen.util 94 :invoke :write [1 1] INFO [2019-12-17 12:27:46,739] jepsen worker 6 - jepsen.util 6 :ok :read [2 15] INFO [2019-12-17 12:27:46,740] jepsen worker 6 - jepsen.util 6 :invoke :write [2 43] INFO [2019-12-17 12:27:46,746] jepsen worker 1 - jepsen.util 1 :info :cas [0 [0 34]] :RequestCanceledException INFO [2019-12-17 12:27:46,747] jepsen worker 86 - jepsen.util 86 :fail :cas [28 [4 24]] :ValueNotSwapFrom INFO [2019-12-17 12:27:46,747] jepsen worker 86 - jepsen.util 86 :invoke :write [28 34] INFO [2019-12-17 12:27:46,748] jepsen worker 1 - jepsen.util 91 :invoke :cas [0 [29 2]] INFO [2019-12-17 12:27:46,758] jepsen worker 37 - jepsen.util 37 :fail :cas [12 [22 21]] :ValueNotSwapFrom INFO [2019-12-17 12:27:46,758] jepsen worker 37 - jepsen.util 37 :invoke :cas [12 [6 12]] INFO [2019-12-17 12:27:46,773] jepsen worker 22 - jepsen.util 112 :fail :cas [7 [24 47]] :ValueNotSwapFrom INFO [2019-12-17 12:27:46,773] jepsen worker 22 - jepsen.util 112 :invoke :cas [7 [23 7]] INFO [2019-12-17 12:27:46,776] jepsen worker 76 - jepsen.util 76 :fail :cas [25 [27 48]] :ValueNotSwapFrom INFO [2019-12-17 12:27:46,776] jepsen worker 76 - jepsen.util 76 :invoke :read [25 nil] INFO [2019-12-17 12:27:46,820] jepsen worker 55 - jepsen.util 145 :ok :read [18 47] INFO [2019-12-17 12:27:46,820] jepsen worker 55 - jepsen.util 145 :invoke :read [18 nil] INFO [2019-12-17 12:27:46,821] jepsen worker 2 - jepsen.util 92 :fail :cas [0 [34 24]] :ValueNotSwapFrom INFO [2019-12-17 12:27:46,821] jepsen worker 2 - jepsen.util 92 :invoke :read [0 nil] INFO [2019-12-17 12:27:46,828] jepsen worker 50 - jepsen.util 140 :fail :cas [16 [15 13]] :ValueNotSwapFrom INFO [2019-12-17 12:27:46,829] jepsen worker 50 - jepsen.util 140 :invoke :write [16 0] INFO [2019-12-17 12:27:46,832] jepsen worker 89 - jepsen.util 89 :fail :cas [29 [3 1]] :ValueNotSwapFrom INFO [2019-12-17 12:27:46,833] jepsen worker 89 - jepsen.util 89 :invoke :read [29 nil] INFO [2019-12-17 12:27:46,872] jepsen worker 27 - jepsen.util 27 :ok :write [9 46] INFO [2019-12-17 12:27:46,873] jepsen worker 27 - jepsen.util 27 :invoke :read [9 nil] INFO [2019-12-17 12:27:46,875] jepsen worker 81 - jepsen.util 81 :ok :read [27 18] INFO [2019-12-17 12:27:46,875] jepsen worker 81 - jepsen.util 81 :invoke :read [27 nil] INFO [2019-12-17 12:27:46,890] jepsen worker 5 - jepsen.util 5 :fail :cas [1 [35 35]] :ValueNotSwapFrom INFO [2019-12-17 12:27:46,890] jepsen worker 5 - jepsen.util 5 :invoke :cas [1 [30 22]] INFO [2019-12-17 12:27:46,895] jepsen worker 68 - jepsen.util 158 :ok :write [22 13] INFO [2019-12-17 12:27:46,908] jepsen worker 8 - jepsen.util 98 :ok :read [2 15] INFO [2019-12-17 12:27:46,923] jepsen worker 84 - jepsen.util 84 :ok :read [28 15] INFO [2019-12-17 12:27:46,948] jepsen worker 19 - jepsen.util 19 :info :write [6 6] :RequestCanceledException INFO [2019-12-17 12:27:46,952] jepsen worker 9 - jepsen.util 99 :ok :read [3 14] INFO [2019-12-17 12:27:46,956] jepsen worker 52 - jepsen.util 142 :ok :read [17 0] INFO [2019-12-17 12:27:46,962] jepsen worker 71 - jepsen.util 161 :fail :cas [23 [49 3]] :ValueNotSwapFrom INFO [2019-12-17 12:27:46,980] jepsen worker 30 - jepsen.util 30 :ok :read [10 23] INFO [2019-12-17 12:27:46,988] jepsen worker 23 - jepsen.util 23 :fail :cas [7 [19 30]] :ValueNotSwapFrom INFO [2019-12-17 12:27:46,999] jepsen worker 8 - jepsen.util 98 :invoke :cas [2 [0 45]] INFO [2019-12-17 12:27:46,999] jepsen worker 68 - jepsen.util 158 :invoke :write [22 36] INFO [2019-12-17 12:27:46,999] jepsen worker 84 - jepsen.util 84 :invoke :read [28 nil] INFO [2019-12-17 12:27:47,000] jepsen worker 9 - jepsen.util 99 :invoke :cas [3 [8 32]] INFO [2019-12-17 12:27:47,000] jepsen worker 52 - jepsen.util 142 :invoke :read [17 nil] INFO [2019-12-17 12:27:47,000] jepsen worker 71 - jepsen.util 161 :invoke :write [23 2] INFO [2019-12-17 12:27:47,000] jepsen worker 30 - jepsen.util 30 :invoke :read [10 nil] INFO [2019-12-17 12:27:47,000] jepsen worker 23 - jepsen.util 23 :invoke :write [7 26] INFO [2019-12-17 12:27:47,001] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":703,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:27:47,001] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":703,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:27:47,001] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":706,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:27:47,002] jepsen worker 19 - jepsen.util 109 :invoke :cas [6 [21 49]] INFO [2019-12-17 12:27:47,002] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":706,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:27:47,017] jepsen worker 21 - jepsen.util 21 :info :write [7 17] :RequestCanceledException INFO [2019-12-17 12:27:47,017] jepsen worker 33 - jepsen.util 33 :info :write [11 44] :RequestCanceledException INFO [2019-12-17 12:27:47,019] jepsen worker 33 - jepsen.util 123 :invoke :write [11 13] INFO [2019-12-17 12:27:47,019] jepsen worker 21 - jepsen.util 111 :invoke :cas [7 [46 20]] INFO [2019-12-17 12:27:47,064] jepsen worker 35 - jepsen.util 35 :ok :read [11 7] INFO [2019-12-17 12:27:47,064] jepsen worker 35 - jepsen.util 35 :invoke :read [11 nil] INFO [2019-12-17 12:27:47,077] jepsen worker 89 - jepsen.util 89 :ok :read [29 5] INFO [2019-12-17 12:27:47,077] jepsen worker 89 - jepsen.util 89 :invoke :read [29 nil] INFO [2019-12-17 12:27:47,088] jepsen worker 48 - jepsen.util 138 :fail :cas [16 [4 37]] :ValueNotSwapFrom INFO [2019-12-17 12:27:47,088] jepsen worker 48 - jepsen.util 138 :invoke :write [16 16] INFO [2019-12-17 12:27:47,101] jepsen worker 16 - jepsen.util 16 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:27:47,102] jepsen worker 70 - jepsen.util 70 :info :write [23 16] :RequestCanceledException INFO [2019-12-17 12:27:47,102] jepsen worker 16 - jepsen.util 16 :invoke :read [5 nil] INFO [2019-12-17 12:27:47,103] jepsen worker 70 - jepsen.util 160 :invoke :cas [23 [40 49]] INFO [2019-12-17 12:27:47,112] jepsen worker 45 - jepsen.util 45 :info :write [15 13] :RequestCanceledException INFO [2019-12-17 12:27:47,114] jepsen worker 45 - jepsen.util 135 :invoke :cas [15 [42 13]] INFO [2019-12-17 12:27:47,127] jepsen worker 51 - jepsen.util 51 :fail :read [17 nil] :RequestTimeoutException INFO [2019-12-17 12:27:47,127] jepsen worker 51 - jepsen.util 51 :invoke :read [17 nil] INFO [2019-12-17 12:27:47,138] jepsen worker 37 - jepsen.util 37 :fail :cas [12 [6 12]] :ValueNotSwapFrom INFO [2019-12-17 12:27:47,139] jepsen worker 37 - jepsen.util 37 :invoke :cas [12 [9 11]] INFO [2019-12-17 12:27:47,140] jepsen worker 76 - jepsen.util 76 :ok :read [25 2] INFO [2019-12-17 12:27:47,140] jepsen worker 76 - jepsen.util 76 :invoke :cas [25 [31 9]] INFO [2019-12-17 12:27:47,143] jepsen worker 63 - jepsen.util 153 :ok :write [21 19] INFO [2019-12-17 12:27:47,144] jepsen worker 63 - jepsen.util 153 :invoke :write [21 26] INFO [2019-12-17 12:27:47,145] jepsen worker 58 - jepsen.util 148 :ok :read [19 2] INFO [2019-12-17 12:27:47,145] jepsen worker 58 - jepsen.util 148 :invoke :cas [19 [32 23]] INFO [2019-12-17 12:27:47,151] jepsen worker 61 - jepsen.util 61 :info :write [20 29] :RequestCanceledException INFO [2019-12-17 12:27:47,154] jepsen worker 61 - jepsen.util 151 :invoke :write [20 17] INFO [2019-12-17 12:27:47,263] jepsen worker 28 - jepsen.util 28 :info :write [9 38] :RequestCanceledException INFO [2019-12-17 12:27:47,263] jepsen worker 69 - jepsen.util 69 :info :write [23 40] :RequestCanceledException INFO [2019-12-17 12:27:47,269] jepsen worker 24 - jepsen.util 24 :ok :read [8 14] INFO [2019-12-17 12:27:47,270] jepsen worker 24 - jepsen.util 24 :invoke :read [8 nil] INFO [2019-12-17 12:27:47,271] jepsen worker 38 - jepsen.util 128 :ok :read [12 12] INFO [2019-12-17 12:27:47,272] jepsen worker 38 - jepsen.util 128 :invoke :cas [12 [1 40]] INFO [2019-12-17 12:27:47,272] jepsen worker 28 - jepsen.util 118 :invoke :write [9 15] INFO [2019-12-17 12:27:47,272] jepsen worker 69 - jepsen.util 159 :invoke :cas [23 [44 4]] INFO [2019-12-17 12:27:47,277] jepsen worker 18 - jepsen.util 108 :fail :cas [6 [16 20]] :ValueNotSwapFrom INFO [2019-12-17 12:27:47,277] jepsen worker 22 - jepsen.util 112 :fail :cas [7 [23 7]] :ValueNotSwapFrom INFO [2019-12-17 12:27:47,278] jepsen worker 18 - jepsen.util 108 :invoke :write [6 37] INFO [2019-12-17 12:27:47,278] jepsen worker 22 - jepsen.util 112 :invoke :write [7 24] INFO [2019-12-17 12:27:47,279] jepsen worker 55 - jepsen.util 145 :ok :read [18 47] INFO [2019-12-17 12:27:47,279] jepsen worker 55 - jepsen.util 145 :invoke :cas [18 [35 46]] INFO [2019-12-17 12:27:47,280] jepsen worker 5 - jepsen.util 5 :fail :cas [1 [30 22]] :ValueNotSwapFrom INFO [2019-12-17 12:27:47,280] jepsen worker 5 - jepsen.util 5 :invoke :read [1 nil] INFO [2019-12-17 12:27:47,284] jepsen worker 59 - jepsen.util 59 :info :write [19 37] :RequestCanceledException INFO [2019-12-17 12:27:47,284] jepsen worker 41 - jepsen.util 41 :info :write [13 18] :RequestCanceledException INFO [2019-12-17 12:27:47,285] jepsen worker 57 - jepsen.util 147 :ok :read [19 2] INFO [2019-12-17 12:27:47,285] jepsen worker 57 - jepsen.util 147 :invoke :read [19 nil] INFO [2019-12-17 12:27:47,286] jepsen worker 41 - jepsen.util 131 :invoke :read [13 nil] INFO [2019-12-17 12:27:47,286] jepsen worker 59 - jepsen.util 149 :invoke :cas [19 [47 37]] INFO [2019-12-17 12:27:47,303] jepsen worker 72 - jepsen.util 72 :fail :cas [24 [14 33]] :ValueNotSwapFrom INFO [2019-12-17 12:27:47,304] jepsen worker 72 - jepsen.util 72 :invoke :read [24 nil] INFO [2019-12-17 12:27:47,326] jepsen worker 81 - jepsen.util 81 :ok :read [27 18] INFO [2019-12-17 12:27:47,326] jepsen worker 81 - jepsen.util 81 :invoke :write [27 21] INFO [2019-12-17 12:27:47,327] jepsen worker 21 - jepsen.util 111 :fail :cas [7 [46 20]] :ValueNotSwapFrom INFO [2019-12-17 12:27:47,327] jepsen worker 21 - jepsen.util 111 :invoke :read [7 nil] INFO [2019-12-17 12:27:47,333] jepsen worker 76 - jepsen.util 76 :fail :cas [25 [31 9]] :ValueNotSwapFrom INFO [2019-12-17 12:27:47,333] jepsen worker 76 - jepsen.util 76 :invoke :read [25 nil] INFO [2019-12-17 12:27:47,336] jepsen worker 52 - jepsen.util 142 :ok :read [17 0] INFO [2019-12-17 12:27:47,337] jepsen worker 52 - jepsen.util 142 :invoke :cas [17 [0 21]] INFO [2019-12-17 12:27:47,345] jepsen worker 13 - jepsen.util 103 :fail :cas [4 [17 16]] :ValueNotSwapFrom INFO [2019-12-17 12:27:47,346] jepsen worker 13 - jepsen.util 103 :invoke :read [4 nil] INFO [2019-12-17 12:27:47,346] jepsen worker 82 - jepsen.util 172 :fail :cas [27 [2 3]] :ValueNotSwapFrom INFO [2019-12-17 12:27:47,347] jepsen worker 82 - jepsen.util 172 :invoke :write [27 30] INFO [2019-12-17 12:27:47,350] jepsen worker 27 - jepsen.util 27 :ok :read [9 46] INFO [2019-12-17 12:27:47,351] jepsen worker 27 - jepsen.util 27 :invoke :cas [9 [2 24]] INFO [2019-12-17 12:27:47,355] jepsen worker 45 - jepsen.util 135 :fail :cas [15 [42 13]] :ValueNotSwapFrom INFO [2019-12-17 12:27:47,356] jepsen worker 45 - jepsen.util 135 :invoke :read [15 nil] INFO [2019-12-17 12:27:47,356] jepsen worker 8 - jepsen.util 98 :fail :cas [2 [0 45]] :ValueNotSwapFrom INFO [2019-12-17 12:27:47,357] jepsen worker 8 - jepsen.util 98 :invoke :write [2 5] INFO [2019-12-17 12:27:47,377] jepsen worker 45 - jepsen.util 135 :ok :read [15 35] INFO [2019-12-17 12:27:47,377] jepsen worker 45 - jepsen.util 135 :invoke :write [15 18] INFO [2019-12-17 12:27:47,390] jepsen worker 89 - jepsen.util 89 :ok :read [29 5] INFO [2019-12-17 12:27:47,390] jepsen worker 89 - jepsen.util 89 :invoke :write [29 10] INFO [2019-12-17 12:27:47,402] jepsen worker 21 - jepsen.util 111 :ok :read [7 20] INFO [2019-12-17 12:27:47,402] jepsen worker 21 - jepsen.util 111 :invoke :cas [7 [3 46]] INFO [2019-12-17 12:27:47,405] jepsen worker 10 - jepsen.util 10 :info :write [3 32] :RequestCanceledException INFO [2019-12-17 12:27:47,407] jepsen worker 10 - jepsen.util 100 :invoke :cas [3 [17 16]] INFO [2019-12-17 12:27:47,420] jepsen worker 64 - jepsen.util 64 :info :cas [21 [16 0]] :RequestCanceledException INFO [2019-12-17 12:27:47,422] jepsen worker 64 - jepsen.util 154 :invoke :cas [21 [10 29]] INFO [2019-12-17 12:27:47,428] jepsen worker 36 - jepsen.util 126 :ok :read [12 12] INFO [2019-12-17 12:27:47,428] jepsen worker 36 - jepsen.util 126 :invoke :write [12 20] INFO [2019-12-17 12:27:47,438] jepsen worker 58 - jepsen.util 148 :fail :cas [19 [32 23]] :ValueNotSwapFrom INFO [2019-12-17 12:27:47,438] jepsen worker 58 - jepsen.util 148 :invoke :read [19 nil] INFO [2019-12-17 12:27:47,461] jepsen worker 37 - jepsen.util 37 :fail :cas [12 [9 11]] :ValueNotSwapFrom INFO [2019-12-17 12:27:47,461] jepsen worker 37 - jepsen.util 37 :invoke :cas [12 [23 9]] INFO [2019-12-17 12:27:47,469] jepsen worker 21 - jepsen.util 111 :fail :cas [7 [3 46]] :ValueNotSwapFrom INFO [2019-12-17 12:27:47,469] jepsen worker 21 - jepsen.util 111 :invoke :write [7 40] INFO [2019-12-17 12:27:47,478] jepsen worker 56 - jepsen.util 56 :ok :read [18 47] INFO [2019-12-17 12:27:47,478] jepsen worker 56 - jepsen.util 56 :invoke :write [18 39] INFO [2019-12-17 12:27:47,481] jepsen worker 69 - jepsen.util 159 :fail :cas [23 [44 4]] :ValueNotSwapFrom INFO [2019-12-17 12:27:47,482] jepsen worker 69 - jepsen.util 159 :invoke :write [23 8] INFO [2019-12-17 12:27:47,502] jepsen worker 86 - jepsen.util 86 :ok :write [28 34] INFO [2019-12-17 12:27:47,502] jepsen worker 86 - jepsen.util 86 :invoke :read [28 nil] INFO [2019-12-17 12:27:47,504] jepsen worker 29 - jepsen.util 29 :info :write [9 5] :RequestCanceledException INFO [2019-12-17 12:27:47,506] jepsen worker 29 - jepsen.util 119 :invoke :write [9 12] INFO [2019-12-17 12:27:47,510] jepsen worker 24 - jepsen.util 24 :ok :read [8 14] INFO [2019-12-17 12:27:47,511] jepsen worker 24 - jepsen.util 24 :invoke :cas [8 [16 30]] INFO [2019-12-17 12:27:47,512] jepsen worker 5 - jepsen.util 5 :ok :read [1 23] INFO [2019-12-17 12:27:47,512] jepsen worker 5 - jepsen.util 5 :invoke :cas [1 [29 33]] INFO [2019-12-17 12:27:47,527] jepsen worker 16 - jepsen.util 16 :ok :read [5 32] INFO [2019-12-17 12:27:47,528] jepsen worker 16 - jepsen.util 16 :invoke :cas [5 [6 44]] INFO [2019-12-17 12:27:47,552] jepsen worker 60 - jepsen.util 150 :info :write [20 33] :RequestCanceledException INFO [2019-12-17 12:27:47,554] jepsen worker 60 - jepsen.util 240 :invoke :read [20 nil] INFO [2019-12-17 12:27:47,561] jepsen worker 88 - jepsen.util 178 :fail :cas [29 [48 14]] :ValueNotSwapFrom INFO [2019-12-17 12:27:47,561] jepsen worker 88 - jepsen.util 178 :invoke :write [29 29] INFO [2019-12-17 12:27:47,572] jepsen worker 24 - jepsen.util 24 :fail :cas [8 [16 30]] :ValueNotSwapFrom INFO [2019-12-17 12:27:47,572] jepsen worker 11 - jepsen.util 101 :ok :read [3 14] INFO [2019-12-17 12:27:47,572] jepsen worker 24 - jepsen.util 24 :invoke :read [8 nil] INFO [2019-12-17 12:27:47,572] jepsen worker 11 - jepsen.util 101 :invoke :write [3 28] INFO [2019-12-17 12:27:47,573] jepsen worker 57 - jepsen.util 147 :ok :read [19 2] INFO [2019-12-17 12:27:47,573] jepsen worker 57 - jepsen.util 147 :invoke :write [19 41] INFO [2019-12-17 12:27:47,575] jepsen worker 63 - jepsen.util 153 :ok :write [21 26] INFO [2019-12-17 12:27:47,575] jepsen worker 63 - jepsen.util 153 :invoke :read [21 nil] INFO [2019-12-17 12:27:47,586] jepsen worker 25 - jepsen.util 115 :info :write [8 28] :RequestCanceledException INFO [2019-12-17 12:27:47,588] jepsen worker 25 - jepsen.util 205 :invoke :cas [8 [26 18]] INFO [2019-12-17 12:27:47,601] jepsen worker 39 - jepsen.util 129 :info :write [13 37] :RequestCanceledException INFO [2019-12-17 12:27:47,601] jepsen worker 73 - jepsen.util 163 :fail :read [24 nil] :RequestTimeoutException INFO [2019-12-17 12:27:47,601] jepsen worker 73 - jepsen.util 163 :invoke :read [24 nil] INFO [2019-12-17 12:27:47,603] jepsen worker 39 - jepsen.util 219 :invoke :cas [13 [23 1]] INFO [2019-12-17 12:27:47,606] jepsen worker 47 - jepsen.util 137 :ok :read [15 35] INFO [2019-12-17 12:27:47,606] jepsen worker 47 - jepsen.util 137 :invoke :cas [15 [46 48]] INFO [2019-12-17 12:27:47,607] jepsen worker 70 - jepsen.util 160 :fail :cas [23 [40 49]] :ValueNotSwapFrom INFO [2019-12-17 12:27:47,607] jepsen worker 70 - jepsen.util 160 :invoke :write [23 18] INFO [2019-12-17 12:27:47,610] jepsen worker 2 - jepsen.util 92 :ok :read [0 14] INFO [2019-12-17 12:27:47,610] jepsen worker 2 - jepsen.util 92 :invoke :read [0 nil] INFO [2019-12-17 12:27:47,620] jepsen worker 53 - jepsen.util 143 :info :write [17 4] :RequestCanceledException INFO [2019-12-17 12:27:47,620] jepsen worker 14 - jepsen.util 104 :info :write [4 32] :RequestCanceledException INFO [2019-12-17 12:27:47,620] jepsen worker 3 - jepsen.util 3 :info :write [1 16] :RequestCanceledException INFO [2019-12-17 12:27:47,622] jepsen worker 53 - jepsen.util 233 :invoke :cas [17 [48 25]] INFO [2019-12-17 12:27:47,622] jepsen worker 3 - jepsen.util 93 :invoke :cas [1 [48 20]] INFO [2019-12-17 12:27:47,622] jepsen worker 14 - jepsen.util 194 :invoke :cas [4 [14 49]] INFO [2019-12-17 12:27:47,635] jepsen worker 77 - jepsen.util 167 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:27:47,635] jepsen worker 31 - jepsen.util 121 :info :write [10 29] :RequestCanceledException INFO [2019-12-17 12:27:47,635] jepsen worker 77 - jepsen.util 167 :invoke :cas [25 [39 43]] INFO [2019-12-17 12:27:47,636] jepsen worker 31 - jepsen.util 211 :invoke :write [10 7] INFO [2019-12-17 12:27:47,643] jepsen worker 51 - jepsen.util 51 :ok :read [17 0] INFO [2019-12-17 12:27:47,644] jepsen worker 51 - jepsen.util 51 :invoke :read [17 nil] INFO [2019-12-17 12:27:47,652] jepsen worker 7 - jepsen.util 97 :info :write [2 35] :RequestCanceledException INFO [2019-12-17 12:27:47,652] jepsen worker 26 - jepsen.util 116 :info :write [8 20] :RequestCanceledException INFO [2019-12-17 12:27:47,652] jepsen worker 46 - jepsen.util 136 :info :write [15 49] :RequestCanceledException INFO [2019-12-17 12:27:47,652] jepsen worker 15 - jepsen.util 105 :info :write [5 28] :RequestCanceledException INFO [2019-12-17 12:27:47,654] jepsen worker 26 - jepsen.util 206 :invoke :read [8 nil] INFO [2019-12-17 12:27:47,654] jepsen worker 15 - jepsen.util 195 :invoke :write [5 16] INFO [2019-12-17 12:27:47,654] jepsen worker 46 - jepsen.util 226 :invoke :read [15 nil] INFO [2019-12-17 12:27:47,654] jepsen worker 7 - jepsen.util 187 :invoke :write [2 47] INFO [2019-12-17 12:27:47,662] jepsen worker 47 - jepsen.util 137 :fail :cas [15 [46 48]] :ValueNotSwapFrom INFO [2019-12-17 12:27:47,663] jepsen worker 47 - jepsen.util 137 :invoke :read [15 nil] INFO [2019-12-17 12:27:47,664] jepsen worker 72 - jepsen.util 72 :ok :read [24 35] INFO [2019-12-17 12:27:47,664] jepsen worker 72 - jepsen.util 72 :invoke :read [24 nil] INFO [2019-12-17 12:27:47,671] jepsen worker 27 - jepsen.util 27 :fail :cas [9 [2 24]] :ValueNotSwapFrom INFO [2019-12-17 12:27:47,671] jepsen worker 27 - jepsen.util 27 :invoke :write [9 23] INFO [2019-12-17 12:27:47,678] jepsen worker 25 - jepsen.util 205 :fail :cas [8 [26 18]] :ValueNotSwapFrom INFO [2019-12-17 12:27:47,678] jepsen worker 25 - jepsen.util 205 :invoke :write [8 15] INFO [2019-12-17 12:27:47,678] jepsen worker 42 - jepsen.util 132 :info :write [14 28] :RequestCanceledException INFO [2019-12-17 12:27:47,678] jepsen worker 20 - jepsen.util 110 :info :write [6 11] :RequestCanceledException INFO [2019-12-17 12:27:47,680] jepsen worker 20 - jepsen.util 200 :invoke :cas [6 [7 12]] INFO [2019-12-17 12:27:47,680] jepsen worker 42 - jepsen.util 222 :invoke :write [14 21] INFO [2019-12-17 12:27:47,694] jepsen worker 64 - jepsen.util 154 :fail :cas [21 [10 29]] :ValueNotSwapFrom INFO [2019-12-17 12:27:47,694] jepsen worker 64 - jepsen.util 154 :invoke :read [21 nil] INFO [2019-12-17 12:27:47,696] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":794,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:27:47,696] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":794,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:27:47,702] jepsen worker 47 - jepsen.util 137 :ok :read [15 35] INFO [2019-12-17 12:27:47,702] jepsen worker 47 - jepsen.util 137 :invoke :read [15 nil] INFO [2019-12-17 12:27:47,715] jepsen worker 76 - jepsen.util 76 :ok :read [25 2] INFO [2019-12-17 12:27:47,715] jepsen worker 76 - jepsen.util 76 :invoke :cas [25 [18 22]] INFO [2019-12-17 12:27:47,720] jepsen worker 43 - jepsen.util 133 :info :write [14 27] :RequestCanceledException INFO [2019-12-17 12:27:47,720] jepsen worker 78 - jepsen.util 78 :info :write [26 26] :RequestCanceledException INFO [2019-12-17 12:27:47,722] jepsen worker 78 - jepsen.util 168 :invoke :read [26 nil] INFO [2019-12-17 12:27:47,722] jepsen worker 43 - jepsen.util 223 :invoke :write [14 3] INFO [2019-12-17 12:27:47,725] jepsen worker 26 - jepsen.util 206 :ok :read [8 14] INFO [2019-12-17 12:27:47,725] jepsen worker 63 - jepsen.util 153 :ok :read [21 26] INFO [2019-12-17 12:27:47,725] jepsen worker 26 - jepsen.util 206 :invoke :read [8 nil] INFO [2019-12-17 12:27:47,725] jepsen worker 63 - jepsen.util 153 :invoke :read [21 nil] INFO [2019-12-17 12:27:47,735] jepsen worker 75 - jepsen.util 165 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:27:47,735] jepsen worker 62 - jepsen.util 152 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:27:47,735] jepsen worker 74 - jepsen.util 74 :info :cas [24 [5 31]] :RequestCanceledException INFO [2019-12-17 12:27:47,735] jepsen worker 75 - jepsen.util 165 :invoke :write [25 46] INFO [2019-12-17 12:27:47,735] jepsen worker 62 - jepsen.util 152 :invoke :cas [20 [31 14]] INFO [2019-12-17 12:27:47,736] jepsen worker 74 - jepsen.util 164 :invoke :cas [24 [25 41]] INFO [2019-12-17 12:27:47,746] jepsen worker 80 - jepsen.util 170 :info :write [26 19] :RequestCanceledException INFO [2019-12-17 12:27:47,748] jepsen worker 80 - jepsen.util 260 :invoke :write [26 22] INFO [2019-12-17 12:27:47,755] jepsen worker 72 - jepsen.util 72 :ok :read [24 35] INFO [2019-12-17 12:27:47,755] jepsen worker 72 - jepsen.util 72 :invoke :read [24 nil] INFO [2019-12-17 12:27:47,762] jepsen worker 47 - jepsen.util 137 :ok :read [15 35] INFO [2019-12-17 12:27:47,762] jepsen worker 47 - jepsen.util 137 :invoke :read [15 nil] INFO [2019-12-17 12:27:47,791] jepsen worker 47 - jepsen.util 137 :ok :read [15 35] INFO [2019-12-17 12:27:47,791] jepsen worker 47 - jepsen.util 137 :invoke :cas [15 [17 31]] INFO [2019-12-17 12:27:47,792] jepsen worker 14 - jepsen.util 194 :fail :cas [4 [14 49]] :ValueNotSwapFrom INFO [2019-12-17 12:27:47,792] jepsen worker 14 - jepsen.util 194 :invoke :read [4 nil] INFO [2019-12-17 12:27:47,796] jepsen worker 53 - jepsen.util 233 :fail :cas [17 [48 25]] :ValueNotSwapFrom INFO [2019-12-17 12:27:47,796] jepsen worker 53 - jepsen.util 233 :invoke :write [17 9] INFO [2019-12-17 12:27:47,803] jepsen worker 49 - jepsen.util 139 :ok :read [16 8] INFO [2019-12-17 12:27:47,803] jepsen worker 49 - jepsen.util 139 :invoke :read [16 nil] INFO [2019-12-17 12:27:47,809] jepsen worker 47 - jepsen.util 137 :fail :cas [15 [17 31]] :ValueNotSwapFrom INFO [2019-12-17 12:27:47,809] jepsen worker 47 - jepsen.util 137 :invoke :write [15 10] INFO [2019-12-17 12:27:47,812] jepsen worker 27 - jepsen.util 27 :ok :write [9 23] INFO [2019-12-17 12:27:47,812] jepsen worker 27 - jepsen.util 27 :invoke :write [9 7] INFO [2019-12-17 12:27:47,823] jepsen worker 37 - jepsen.util 37 :fail :cas [12 [23 9]] :ValueNotSwapFrom INFO [2019-12-17 12:27:47,823] jepsen worker 37 - jepsen.util 37 :invoke :read [12 nil] INFO [2019-12-17 12:27:47,837] jepsen worker 72 - jepsen.util 72 :ok :read [24 35] INFO [2019-12-17 12:27:47,837] jepsen worker 72 - jepsen.util 72 :invoke :write [24 12] INFO [2019-12-17 12:27:47,849] jepsen worker 24 - jepsen.util 24 :ok :read [8 14] INFO [2019-12-17 12:27:47,849] jepsen worker 24 - jepsen.util 24 :invoke :read [8 nil] INFO [2019-12-17 12:27:47,859] jepsen worker 3 - jepsen.util 93 :fail :cas [1 [48 20]] :ValueNotSwapFrom INFO [2019-12-17 12:27:47,859] jepsen worker 3 - jepsen.util 93 :invoke :read [1 nil] INFO [2019-12-17 12:27:47,878] jepsen worker 26 - jepsen.util 206 :ok :read [8 14] INFO [2019-12-17 12:27:47,879] jepsen worker 55 - jepsen.util 145 :fail :cas [18 [35 46]] :ValueNotSwapFrom INFO [2019-12-17 12:27:47,879] jepsen worker 26 - jepsen.util 206 :invoke :write [8 10] INFO [2019-12-17 12:27:47,879] jepsen worker 55 - jepsen.util 145 :invoke :write [18 36] INFO [2019-12-17 12:27:47,899] jepsen worker 67 - jepsen.util 157 :ok :write [22 5] INFO [2019-12-17 12:27:47,899] jepsen worker 67 - jepsen.util 157 :invoke :write [22 5] INFO [2019-12-17 12:27:47,902] jepsen worker 37 - jepsen.util 37 :ok :read [12 12] INFO [2019-12-17 12:27:47,902] jepsen worker 37 - jepsen.util 37 :invoke :read [12 nil] INFO [2019-12-17 12:27:47,924] jepsen worker 49 - jepsen.util 139 :ok :read [16 8] INFO [2019-12-17 12:27:47,924] jepsen worker 49 - jepsen.util 139 :invoke :cas [16 [32 4]] INFO [2019-12-17 12:27:47,928] jepsen worker 3 - jepsen.util 93 :ok :read [1 23] INFO [2019-12-17 12:27:47,928] jepsen worker 37 - jepsen.util 37 :ok :read [12 12] INFO [2019-12-17 12:27:47,928] jepsen worker 3 - jepsen.util 93 :invoke :read [1 nil] INFO [2019-12-17 12:27:47,928] jepsen worker 37 - jepsen.util 37 :invoke :write [12 25] INFO [2019-12-17 12:27:47,945] jepsen worker 27 - jepsen.util 27 :ok :write [9 7] INFO [2019-12-17 12:27:47,946] jepsen worker 27 - jepsen.util 27 :invoke :cas [9 [3 30]] INFO [2019-12-17 12:27:47,986] jepsen worker 64 - jepsen.util 154 :ok :read [21 26] INFO [2019-12-17 12:27:47,986] jepsen worker 64 - jepsen.util 154 :invoke :read [21 nil] INFO [2019-12-17 12:27:47,996] jepsen worker 60 - jepsen.util 240 :ok :read [20 7] INFO [2019-12-17 12:27:47,996] jepsen worker 60 - jepsen.util 240 :invoke :write [20 48] INFO [2019-12-17 12:27:48,005] jepsen worker 63 - jepsen.util 153 :ok :read [21 26] INFO [2019-12-17 12:27:48,005] jepsen worker 63 - jepsen.util 153 :invoke :cas [21 [27 36]] INFO [2019-12-17 12:27:48,018] jepsen worker 64 - jepsen.util 154 :ok :read [21 26] INFO [2019-12-17 12:27:48,019] jepsen worker 64 - jepsen.util 154 :invoke :write [21 25] INFO [2019-12-17 12:27:48,074] jepsen worker 51 - jepsen.util 51 :ok :read [17 0] INFO [2019-12-17 12:27:48,074] jepsen worker 51 - jepsen.util 51 :invoke :read [17 nil] INFO [2019-12-17 12:27:48,076] jepsen worker 38 - jepsen.util 128 :fail :cas [12 [1 40]] :ValueNotSwapFrom INFO [2019-12-17 12:27:48,076] jepsen worker 38 - jepsen.util 128 :invoke :cas [12 [38 17]] INFO [2019-12-17 12:27:48,089] jepsen nemesis - jepsen.util :nemesis :info :failover nil INFO [2019-12-17 12:27:48,090] jepsen nemesis - couchbase.nemesis Failing over node 172.28.128.125 INFO [2019-12-17 12:27:48,098] jepsen worker 13 - jepsen.util 103 :ok :read [4 4] INFO [2019-12-17 12:27:48,099] jepsen worker 13 - jepsen.util 103 :invoke :write [4 39] INFO [2019-12-17 12:27:48,106] jepsen worker 67 - jepsen.util 157 :ok :write [22 5] INFO [2019-12-17 12:27:48,107] jepsen worker 67 - jepsen.util 157 :invoke :read [22 nil] INFO [2019-12-17 12:27:48,121] jepsen worker 87 - jepsen.util 87 :info :cas [29 [40 3]] :RequestCanceledException INFO [2019-12-17 12:27:48,123] jepsen worker 87 - jepsen.util 177 :invoke :read [29 nil] INFO [2019-12-17 12:27:48,131] jepsen worker 49 - jepsen.util 139 :fail :cas [16 [32 4]] :ValueNotSwapFrom INFO [2019-12-17 12:27:48,132] jepsen worker 49 - jepsen.util 139 :invoke :read [16 nil] INFO [2019-12-17 12:27:48,146] jepsen worker 67 - jepsen.util 157 :ok :read [22 5] INFO [2019-12-17 12:27:48,229] jepsen worker 12 - jepsen.util 12 :info :write [4 44] :RequestCanceledException INFO [2019-12-17 12:27:48,230] jepsen worker 12 - jepsen.util 102 :invoke :read [4 nil] INFO [2019-12-17 12:27:48,235] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":863,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:27:48,235] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":863,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:27:48,235] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":868,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:27:48,236] jepsen worker 67 - jepsen.util 157 :invoke :write [22 1] INFO [2019-12-17 12:27:48,236] jepsen worker 65 - jepsen.util 65 :info :write [21 38] :RequestCanceledException INFO [2019-12-17 12:27:48,238] jepsen worker 65 - jepsen.util 155 :invoke :cas [21 [6 4]] INFO [2019-12-17 12:27:48,241] jepsen worker 76 - jepsen.util 76 :fail :cas [25 [18 22]] :ValueNotSwapFrom INFO [2019-12-17 12:27:48,242] jepsen worker 76 - jepsen.util 76 :invoke :cas [25 [26 37]] INFO [2019-12-17 12:27:48,245] jepsen worker 44 - jepsen.util 44 :fail :cas [14 [25 12]] :ValueNotSwapFrom INFO [2019-12-17 12:27:48,246] jepsen worker 44 - jepsen.util 44 :invoke :write [14 32] INFO [2019-12-17 12:27:48,246] jepsen worker 24 - jepsen.util 24 :ok :read [8 14] INFO [2019-12-17 12:27:48,246] jepsen worker 24 - jepsen.util 24 :invoke :write [8 32] INFO [2019-12-17 12:27:48,246] jepsen worker 35 - jepsen.util 35 :ok :read [11 7] INFO [2019-12-17 12:27:48,247] jepsen worker 35 - jepsen.util 35 :invoke :cas [11 [18 47]] INFO [2019-12-17 12:27:48,247] jepsen worker 32 - jepsen.util 32 :ok :read [10 23] INFO [2019-12-17 12:27:48,247] jepsen worker 49 - jepsen.util 139 :ok :read [16 8] INFO [2019-12-17 12:27:48,247] jepsen worker 78 - jepsen.util 168 :ok :read [26 47] INFO [2019-12-17 12:27:48,247] jepsen worker 32 - jepsen.util 32 :invoke :read [10 nil] INFO [2019-12-17 12:27:48,247] jepsen worker 49 - jepsen.util 139 :invoke :read [16 nil] INFO [2019-12-17 12:27:48,247] jepsen worker 78 - jepsen.util 168 :invoke :cas [26 [5 15]] INFO [2019-12-17 12:27:48,263] jepsen worker 63 - jepsen.util 153 :fail :cas [21 [27 36]] :ValueNotSwapFrom INFO [2019-12-17 12:27:48,263] jepsen worker 63 - jepsen.util 153 :invoke :read [21 nil] INFO [2019-12-17 12:27:48,282] jepsen worker 32 - jepsen.util 32 :ok :read [10 23] INFO [2019-12-17 12:27:48,282] jepsen worker 32 - jepsen.util 32 :invoke :write [10 15] INFO [2019-12-17 12:27:48,290] jepsen nemesis - couchbase.util Rebalance status: {} INFO [2019-12-17 12:27:48,290] jepsen nemesis - couchbase.util Rebalance check count: 0 INFO [2019-12-17 12:27:48,291] jepsen nemesis - couchbase.util Rebalance stuck count: 0 INFO [2019-12-17 12:27:48,306] jepsen worker 34 - jepsen.util 34 :info :write [11 46] :RequestCanceledException INFO [2019-12-17 12:27:48,307] jepsen worker 34 - jepsen.util 124 :invoke :cas [11 [21 19]] INFO [2019-12-17 12:27:48,345] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":868,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:27:48,345] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":871,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:27:48,345] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":871,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:27:48,369] jepsen worker 38 - jepsen.util 128 :fail :cas [12 [38 17]] :ValueNotSwapFrom INFO [2019-12-17 12:27:48,369] jepsen worker 38 - jepsen.util 128 :invoke :read [12 nil] INFO [2019-12-17 12:27:48,370] jepsen worker 67 - jepsen.util 157 :ok :write [22 1] INFO [2019-12-17 12:27:48,370] jepsen worker 67 - jepsen.util 157 :invoke :cas [22 [31 24]] INFO [2019-12-17 12:27:48,387] jepsen worker 49 - jepsen.util 139 :ok :read [16 8] INFO [2019-12-17 12:27:48,388] jepsen worker 49 - jepsen.util 139 :invoke :cas [16 [25 34]] INFO [2019-12-17 12:27:48,395] jepsen worker 78 - jepsen.util 168 :fail :cas [26 [5 15]] :ValueNotSwapFrom INFO [2019-12-17 12:27:48,395] jepsen worker 78 - jepsen.util 168 :invoke :read [26 nil] INFO [2019-12-17 12:27:48,400] jepsen worker 67 - jepsen.util 157 :fail :cas [22 [31 24]] :ValueNotSwapFrom INFO [2019-12-17 12:27:48,400] jepsen worker 67 - jepsen.util 157 :invoke :read [22 nil] INFO [2019-12-17 12:27:48,410] jepsen worker 49 - jepsen.util 139 :fail :cas [16 [25 34]] :ValueNotSwapFrom INFO [2019-12-17 12:27:48,410] jepsen worker 49 - jepsen.util 139 :invoke :read [16 nil] INFO [2019-12-17 12:27:48,421] jepsen worker 0 - jepsen.util 0 :info :write [0 24] :RequestCanceledException INFO [2019-12-17 12:27:48,423] jepsen worker 0 - jepsen.util 90 :invoke :read [0 nil] INFO [2019-12-17 12:27:48,423] jepsen worker 38 - jepsen.util 128 :ok :read [12 12] INFO [2019-12-17 12:27:48,423] jepsen worker 38 - jepsen.util 128 :invoke :write [12 41] INFO [2019-12-17 12:27:48,428] jepsen worker 17 - jepsen.util 107 :info :write [5 20] :RequestCanceledException INFO [2019-12-17 12:27:48,429] jepsen worker 17 - jepsen.util 197 :invoke :read [5 nil] INFO [2019-12-17 12:27:48,436] jepsen worker 67 - jepsen.util 157 :ok :read [22 1] INFO [2019-12-17 12:27:48,436] jepsen worker 67 - jepsen.util 157 :invoke :cas [22 [18 44]] INFO [2019-12-17 12:27:48,459] jepsen worker 49 - jepsen.util 139 :ok :read [16 8] INFO [2019-12-17 12:27:48,459] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":893,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:27:48,459] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":893,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:27:48,459] jepsen worker 49 - jepsen.util 139 :invoke :write [16 41] INFO [2019-12-17 12:27:48,459] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":895,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:27:48,459] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":895,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:27:48,475] jepsen worker 0 - jepsen.util 90 :ok :read [0 14] INFO [2019-12-17 12:27:48,475] jepsen worker 0 - jepsen.util 90 :invoke :write [0 32] INFO [2019-12-17 12:27:48,478] jepsen worker 28 - jepsen.util 118 :ok :write [9 15] INFO [2019-12-17 12:27:48,478] jepsen worker 28 - jepsen.util 118 :invoke :cas [9 [3 4]] INFO [2019-12-17 12:27:48,487] jepsen worker 67 - jepsen.util 157 :fail :cas [22 [18 44]] :ValueNotSwapFrom INFO [2019-12-17 12:27:48,487] jepsen worker 67 - jepsen.util 157 :invoke :write [22 48] INFO [2019-12-17 12:27:48,497] jepsen worker 73 - jepsen.util 163 :ok :read [24 35] INFO [2019-12-17 12:27:48,498] jepsen worker 73 - jepsen.util 163 :invoke :cas [24 [14 34]] INFO [2019-12-17 12:27:48,501] jepsen worker 51 - jepsen.util 51 :ok :read [17 0] INFO [2019-12-17 12:27:48,501] jepsen worker 51 - jepsen.util 51 :invoke :write [17 30] INFO [2019-12-17 12:27:48,511] jepsen worker 3 - jepsen.util 93 :ok :read [1 23] INFO [2019-12-17 12:27:48,512] jepsen worker 3 - jepsen.util 93 :invoke :cas [1 [30 0]] INFO [2019-12-17 12:27:48,522] jepsen worker 28 - jepsen.util 118 :fail :cas [9 [3 4]] :ValueNotSwapFrom INFO [2019-12-17 12:27:48,522] jepsen worker 28 - jepsen.util 118 :invoke :cas [9 [16 1]] INFO [2019-12-17 12:27:48,530] jepsen worker 78 - jepsen.util 168 :ok :read [26 47] INFO [2019-12-17 12:27:48,530] jepsen worker 78 - jepsen.util 168 :invoke :cas [26 [32 20]] INFO [2019-12-17 12:27:48,538] jepsen worker 73 - jepsen.util 163 :fail :cas [24 [14 34]] :ValueNotSwapFrom INFO [2019-12-17 12:27:48,539] jepsen worker 73 - jepsen.util 163 :invoke :cas [24 [44 41]] INFO [2019-12-17 12:27:48,563] jepsen worker 28 - jepsen.util 118 :fail :cas [9 [16 1]] :ValueNotSwapFrom INFO [2019-12-17 12:27:48,564] jepsen worker 28 - jepsen.util 118 :invoke :cas [9 [25 16]] INFO [2019-12-17 12:27:48,583] jepsen worker 79 - jepsen.util 169 :info :write [26 29] :RequestCanceledException INFO [2019-12-17 12:27:48,583] jepsen worker 73 - jepsen.util 163 :fail :cas [24 [44 41]] :ValueNotSwapFrom INFO [2019-12-17 12:27:48,584] jepsen worker 73 - jepsen.util 163 :invoke :cas [24 [28 16]] INFO [2019-12-17 12:27:48,584] jepsen worker 79 - jepsen.util 259 :invoke :cas [26 [3 34]] INFO [2019-12-17 12:27:48,597] jepsen worker 5 - jepsen.util 5 :fail :cas [1 [29 33]] :ValueNotSwapFrom INFO [2019-12-17 12:27:48,598] jepsen worker 5 - jepsen.util 5 :invoke :read [1 nil] INFO [2019-12-17 12:27:48,600] jepsen worker 3 - jepsen.util 93 :fail :cas [1 [30 0]] :ValueNotSwapFrom INFO [2019-12-17 12:27:48,600] jepsen worker 28 - jepsen.util 118 :fail :cas [9 [25 16]] :ValueNotSwapFrom INFO [2019-12-17 12:27:48,600] jepsen worker 3 - jepsen.util 93 :invoke :read [1 nil] INFO [2019-12-17 12:27:48,600] jepsen worker 28 - jepsen.util 118 :invoke :write [9 0] INFO [2019-12-17 12:27:48,613] jepsen worker 73 - jepsen.util 163 :fail :cas [24 [28 16]] :ValueNotSwapFrom INFO [2019-12-17 12:27:48,613] jepsen worker 73 - jepsen.util 163 :invoke :cas [24 [9 39]] INFO [2019-12-17 12:27:48,620] jepsen worker 83 - jepsen.util 173 :info :write [27 37] :RequestCanceledException INFO [2019-12-17 12:27:48,620] jepsen worker 66 - jepsen.util 66 :info :cas [22 [21 24]] :RequestCanceledException INFO [2019-12-17 12:27:48,621] jepsen worker 54 - jepsen.util 144 :info :write [18 21] :RequestCanceledException INFO [2019-12-17 12:27:48,622] jepsen worker 83 - jepsen.util 263 :invoke :read [27 nil] INFO [2019-12-17 12:27:48,622] jepsen worker 66 - jepsen.util 156 :invoke :write [22 12] INFO [2019-12-17 12:27:48,623] jepsen worker 54 - jepsen.util 234 :invoke :write [18 3] INFO [2019-12-17 12:27:48,668] jepsen worker 3 - jepsen.util 93 :ok :read [1 23] INFO [2019-12-17 12:27:48,668] jepsen worker 3 - jepsen.util 93 :invoke :cas [1 [7 19]] INFO [2019-12-17 12:27:48,678] jepsen worker 77 - jepsen.util 167 :fail :cas [25 [39 43]] :ValueNotSwapFrom INFO [2019-12-17 12:27:48,678] jepsen worker 77 - jepsen.util 167 :invoke :write [25 41] INFO [2019-12-17 12:27:48,699] jepsen worker 28 - jepsen.util 118 :ok :write [9 0] INFO [2019-12-17 12:27:48,699] jepsen worker 28 - jepsen.util 118 :invoke :write [9 11] INFO [2019-12-17 12:27:48,709] jepsen worker 67 - jepsen.util 157 :ok :write [22 48] INFO [2019-12-17 12:27:48,710] jepsen worker 67 - jepsen.util 157 :invoke :write [22 41] INFO [2019-12-17 12:27:48,714] jepsen worker 3 - jepsen.util 93 :fail :cas [1 [7 19]] :ValueNotSwapFrom INFO [2019-12-17 12:27:48,715] jepsen worker 3 - jepsen.util 93 :invoke :write [1 8] INFO [2019-12-17 12:27:48,721] jepsen worker 85 - jepsen.util 175 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:27:48,721] jepsen worker 85 - jepsen.util 175 :invoke :cas [28 [44 5]] INFO [2019-12-17 12:27:48,727] jepsen worker 62 - jepsen.util 152 :fail :cas [20 [31 14]] :ValueNotSwapFrom INFO [2019-12-17 12:27:48,727] jepsen worker 62 - jepsen.util 152 :invoke :cas [20 [17 27]] INFO [2019-12-17 12:27:48,788] jepsen worker 84 - jepsen.util 84 :ok :read [28 34] INFO [2019-12-17 12:27:48,788] jepsen worker 84 - jepsen.util 84 :invoke :cas [28 [39 14]] INFO [2019-12-17 12:27:48,795] jepsen worker 28 - jepsen.util 118 :ok :write [9 11] INFO [2019-12-17 12:27:48,795] jepsen worker 28 - jepsen.util 118 :invoke :cas [9 [5 13]] INFO [2019-12-17 12:27:48,805] jepsen worker 4 - jepsen.util 94 :info :write [1 1] :RequestCanceledException INFO [2019-12-17 12:27:48,806] jepsen worker 4 - jepsen.util 184 :invoke :cas [1 [41 46]] INFO [2019-12-17 12:27:48,810] jepsen worker 28 - jepsen.util 118 :fail :cas [9 [5 13]] :ValueNotSwapFrom INFO [2019-12-17 12:27:48,810] jepsen worker 28 - jepsen.util 118 :invoke :read [9 nil] INFO [2019-12-17 12:27:48,820] jepsen worker 40 - jepsen.util 130 :info :write [13 29] :RequestCanceledException INFO [2019-12-17 12:27:48,820] jepsen worker 6 - jepsen.util 6 :info :write [2 43] :RequestCanceledException INFO [2019-12-17 12:27:48,820] jepsen worker 1 - jepsen.util 91 :info :cas [0 [29 2]] :RequestCanceledException INFO [2019-12-17 12:27:48,822] jepsen worker 40 - jepsen.util 220 :invoke :cas [13 [26 24]] INFO [2019-12-17 12:27:48,822] jepsen worker 6 - jepsen.util 96 :invoke :write [2 43] INFO [2019-12-17 12:27:48,822] jepsen worker 1 - jepsen.util 181 :invoke :write [0 28] INFO [2019-12-17 12:27:48,828] jepsen worker 73 - jepsen.util 163 :fail :cas [24 [9 39]] :ValueNotSwapFrom INFO [2019-12-17 12:27:48,828] jepsen worker 73 - jepsen.util 163 :invoke :read [24 nil] INFO [2019-12-17 12:27:48,868] jepsen worker 73 - jepsen.util 163 :ok :read [24 35] INFO [2019-12-17 12:27:48,868] jepsen worker 73 - jepsen.util 163 :invoke :cas [24 [36 20]] INFO [2019-12-17 12:27:48,878] jepsen worker 35 - jepsen.util 35 :fail :cas [11 [18 47]] :ValueNotSwapFrom INFO [2019-12-17 12:27:48,878] jepsen worker 35 - jepsen.util 35 :invoke :read [11 nil] INFO [2019-12-17 12:27:48,892] jepsen worker 73 - jepsen.util 163 :fail :cas [24 [36 20]] :ValueNotSwapFrom INFO [2019-12-17 12:27:48,892] jepsen worker 73 - jepsen.util 163 :invoke :read [24 nil] INFO [2019-12-17 12:27:48,910] jepsen worker 73 - jepsen.util 163 :ok :read [24 35] INFO [2019-12-17 12:27:48,910] jepsen worker 73 - jepsen.util 163 :invoke :cas [24 [39 47]] INFO [2019-12-17 12:27:48,917] jepsen worker 76 - jepsen.util 76 :fail :cas [25 [26 37]] :ValueNotSwapFrom INFO [2019-12-17 12:27:48,917] jepsen worker 76 - jepsen.util 76 :invoke :write [25 19] INFO [2019-12-17 12:27:48,921] jepsen worker 50 - jepsen.util 140 :info :write [16 0] :RequestCanceledException INFO [2019-12-17 12:27:48,922] jepsen worker 50 - jepsen.util 230 :invoke :write [16 33] INFO [2019-12-17 12:27:48,936] jepsen worker 68 - jepsen.util 158 :ok :write [22 36] INFO [2019-12-17 12:27:48,936] jepsen worker 68 - jepsen.util 158 :invoke :cas [22 [29 29]] INFO [2019-12-17 12:27:48,939] jepsen worker 28 - jepsen.util 118 :ok :read [9 11] INFO [2019-12-17 12:27:48,940] jepsen worker 28 - jepsen.util 118 :invoke :read [9 nil] INFO [2019-12-17 12:27:48,952] jepsen worker 17 - jepsen.util 197 :ok :read [5 32] INFO [2019-12-17 12:27:48,953] jepsen worker 17 - jepsen.util 197 :invoke :cas [5 [0 45]] INFO [2019-12-17 12:27:48,969] jepsen worker 9 - jepsen.util 99 :fail :cas [3 [8 32]] :ValueNotSwapFrom INFO [2019-12-17 12:27:48,969] jepsen worker 9 - jepsen.util 99 :invoke :read [3 nil] INFO [2019-12-17 12:27:48,982] jepsen worker 30 - jepsen.util 30 :ok :read [10 23] INFO [2019-12-17 12:27:48,982] jepsen worker 30 - jepsen.util 30 :invoke :cas [10 [11 24]] INFO [2019-12-17 12:27:48,993] jepsen worker 85 - jepsen.util 175 :fail :cas [28 [44 5]] :ValueNotSwapFrom INFO [2019-12-17 12:27:48,993] jepsen worker 85 - jepsen.util 175 :invoke :cas [28 [18 47]] INFO [2019-12-17 12:27:49,009] jepsen worker 19 - jepsen.util 109 :fail :cas [6 [21 49]] :ValueNotSwapFrom INFO [2019-12-17 12:27:49,009] jepsen worker 19 - jepsen.util 109 :invoke :write [6 16] INFO [2019-12-17 12:27:49,012] jepsen worker 84 - jepsen.util 84 :fail :cas [28 [39 14]] :ValueNotSwapFrom INFO [2019-12-17 12:27:49,013] jepsen worker 84 - jepsen.util 84 :invoke :read [28 nil] INFO [2019-12-17 12:27:49,023] jepsen worker 62 - jepsen.util 152 :fail :cas [20 [17 27]] :ValueNotSwapFrom INFO [2019-12-17 12:27:49,024] jepsen worker 62 - jepsen.util 152 :invoke :write [20 27] INFO [2019-12-17 12:27:49,028] jepsen worker 40 - jepsen.util 220 :fail :cas [13 [26 24]] :ValueNotSwapFrom INFO [2019-12-17 12:27:49,028] jepsen worker 40 - jepsen.util 220 :invoke :cas [13 [34 37]] INFO [2019-12-17 12:27:49,041] jepsen worker 68 - jepsen.util 158 :fail :cas [22 [29 29]] :ValueNotSwapFrom INFO [2019-12-17 12:27:49,041] jepsen worker 68 - jepsen.util 158 :invoke :read [22 nil] INFO [2019-12-17 12:27:49,054] jepsen worker 87 - jepsen.util 177 :ok :read [29 5] INFO [2019-12-17 12:27:49,055] jepsen worker 87 - jepsen.util 177 :invoke :cas [29 [26 14]] INFO [2019-12-17 12:27:49,069] jepsen worker 17 - jepsen.util 197 :fail :cas [5 [0 45]] :ValueNotSwapFrom INFO [2019-12-17 12:27:49,069] jepsen worker 17 - jepsen.util 197 :invoke :cas [5 [41 14]] INFO [2019-12-17 12:27:49,074] jepsen worker 28 - jepsen.util 118 :ok :read [9 11] INFO [2019-12-17 12:27:49,074] jepsen worker 28 - jepsen.util 118 :invoke :write [9 0] INFO [2019-12-17 12:27:49,087] jepsen worker 23 - jepsen.util 23 :info :write [7 26] :RequestCanceledException INFO [2019-12-17 12:27:49,089] jepsen worker 23 - jepsen.util 113 :invoke :read [7 nil] INFO [2019-12-17 12:27:49,089] jepsen worker 34 - jepsen.util 124 :fail :cas [11 [21 19]] :ValueNotSwapFrom INFO [2019-12-17 12:27:49,090] jepsen worker 34 - jepsen.util 124 :invoke :write [11 42] INFO [2019-12-17 12:27:49,092] jepsen worker 85 - jepsen.util 175 :fail :cas [28 [18 47]] :ValueNotSwapFrom INFO [2019-12-17 12:27:49,093] jepsen worker 85 - jepsen.util 175 :invoke :cas [28 [48 16]] INFO [2019-12-17 12:27:49,096] jepsen worker 79 - jepsen.util 259 :fail :cas [26 [3 34]] :ValueNotSwapFrom INFO [2019-12-17 12:27:49,096] jepsen worker 79 - jepsen.util 259 :invoke :read [26 nil] INFO [2019-12-17 12:27:49,102] jepsen worker 33 - jepsen.util 123 :info :write [11 13] :RequestCanceledException INFO [2019-12-17 12:27:49,102] jepsen worker 71 - jepsen.util 161 :info :write [23 2] :RequestCanceledException INFO [2019-12-17 12:27:49,104] jepsen worker 71 - jepsen.util 251 :invoke :read [23 nil] INFO [2019-12-17 12:27:49,104] jepsen worker 33 - jepsen.util 213 :invoke :cas [11 [42 45]] INFO [2019-12-17 12:27:49,112] jepsen worker 84 - jepsen.util 84 :ok :read [28 34] INFO [2019-12-17 12:27:49,112] jepsen worker 84 - jepsen.util 84 :invoke :cas [28 [33 10]] INFO [2019-12-17 12:27:49,115] jepsen worker 73 - jepsen.util 163 :fail :cas [24 [39 47]] :ValueNotSwapFrom INFO [2019-12-17 12:27:49,115] jepsen worker 73 - jepsen.util 163 :invoke :read [24 nil] INFO [2019-12-17 12:27:49,134] jepsen worker 40 - jepsen.util 220 :fail :cas [13 [34 37]] :ValueNotSwapFrom INFO [2019-12-17 12:27:49,135] jepsen worker 40 - jepsen.util 220 :invoke :read [13 nil] INFO [2019-12-17 12:27:49,149] jepsen worker 29 - jepsen.util 119 :ok :write [9 12] INFO [2019-12-17 12:27:49,149] jepsen worker 29 - jepsen.util 119 :invoke :read [9 nil] INFO [2019-12-17 12:27:49,168] jepsen worker 68 - jepsen.util 158 :ok :read [22 36] INFO [2019-12-17 12:27:49,168] jepsen worker 68 - jepsen.util 158 :invoke :cas [22 [15 12]] INFO [2019-12-17 12:27:49,175] jepsen worker 12 - jepsen.util 102 :ok :read [4 4] INFO [2019-12-17 12:27:49,175] jepsen worker 12 - jepsen.util 102 :invoke :read [4 nil] INFO [2019-12-17 12:27:49,188] jepsen worker 17 - jepsen.util 197 :fail :cas [5 [41 14]] :ValueNotSwapFrom INFO [2019-12-17 12:27:49,188] jepsen worker 17 - jepsen.util 197 :invoke :cas [5 [19 20]] INFO [2019-12-17 12:27:49,197] jepsen worker 85 - jepsen.util 175 :fail :cas [28 [48 16]] :ValueNotSwapFrom INFO [2019-12-17 12:27:49,197] jepsen worker 85 - jepsen.util 175 :invoke :read [28 nil] INFO [2019-12-17 12:27:49,203] jepsen worker 48 - jepsen.util 138 :info :write [16 16] :RequestCanceledException INFO [2019-12-17 12:27:49,205] jepsen worker 48 - jepsen.util 228 :invoke :cas [16 [33 38]] INFO [2019-12-17 12:27:49,223] jepsen worker 84 - jepsen.util 84 :fail :cas [28 [33 10]] :ValueNotSwapFrom INFO [2019-12-17 12:27:49,224] jepsen worker 84 - jepsen.util 84 :invoke :cas [28 [28 42]] INFO [2019-12-17 12:27:49,236] jepsen worker 40 - jepsen.util 220 :ok :read [13 9] INFO [2019-12-17 12:27:49,236] jepsen worker 40 - jepsen.util 220 :invoke :write [13 3] INFO [2019-12-17 12:27:49,245] jepsen worker 45 - jepsen.util 135 :info :write [15 18] :SyncWriteAmbiguous INFO [2019-12-17 12:27:49,246] jepsen worker 45 - jepsen.util 225 :invoke :cas [15 [37 48]] INFO [2019-12-17 12:27:49,251] jepsen worker 4 - jepsen.util 184 :fail :cas [1 [41 46]] :ValueNotSwapFrom INFO [2019-12-17 12:27:49,252] jepsen worker 4 - jepsen.util 184 :invoke :read [1 nil] INFO [2019-12-17 12:27:49,254] jepsen worker 68 - jepsen.util 158 :fail :cas [22 [15 12]] :ValueNotSwapFrom INFO [2019-12-17 12:27:49,254] jepsen worker 68 - jepsen.util 158 :invoke :cas [22 [19 15]] INFO [2019-12-17 12:27:49,256] jepsen worker 78 - jepsen.util 168 :fail :cas [26 [32 20]] :ValueNotSwapFrom INFO [2019-12-17 12:27:49,257] jepsen worker 78 - jepsen.util 168 :invoke :write [26 48] INFO [2019-12-17 12:27:49,269] jepsen worker 17 - jepsen.util 197 :fail :cas [5 [19 20]] :ValueNotSwapFrom INFO [2019-12-17 12:27:49,270] jepsen worker 17 - jepsen.util 197 :invoke :cas [5 [21 37]] INFO [2019-12-17 12:27:49,288] jepsen worker 85 - jepsen.util 175 :ok :read [28 34] INFO [2019-12-17 12:27:49,289] jepsen worker 85 - jepsen.util 175 :invoke :write [28 1] INFO [2019-12-17 12:27:49,299] jepsen worker 35 - jepsen.util 35 :ok :read [11 7] INFO [2019-12-17 12:27:49,299] jepsen worker 35 - jepsen.util 35 :invoke :read [11 nil] INFO [2019-12-17 12:27:49,302] jepsen worker 10 - jepsen.util 100 :fail :cas [3 [17 16]] :ValueNotSwapFrom INFO [2019-12-17 12:27:49,302] jepsen worker 10 - jepsen.util 100 :invoke :write [3 10] INFO [2019-12-17 12:27:49,311] jepsen worker 84 - jepsen.util 84 :fail :cas [28 [28 42]] :ValueNotSwapFrom INFO [2019-12-17 12:27:49,311] jepsen worker 84 - jepsen.util 84 :invoke :read [28 nil] INFO [2019-12-17 12:27:49,317] jepsen worker 73 - jepsen.util 163 :ok :read [24 35] INFO [2019-12-17 12:27:49,317] jepsen worker 73 - jepsen.util 163 :invoke :write [24 36] INFO [2019-12-17 12:27:49,320] jepsen worker 61 - jepsen.util 151 :info :write [20 17] :RequestCanceledException INFO [2019-12-17 12:27:49,320] jepsen worker 18 - jepsen.util 108 :info :write [6 37] :RequestCanceledException INFO [2019-12-17 12:27:49,321] jepsen worker 41 - jepsen.util 131 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:27:49,321] jepsen worker 61 - jepsen.util 241 :invoke :cas [20 [16 35]] INFO [2019-12-17 12:27:49,321] jepsen worker 18 - jepsen.util 198 :invoke :cas [6 [24 39]] INFO [2019-12-17 12:27:49,322] jepsen worker 41 - jepsen.util 131 :invoke :read [13 nil] INFO [2019-12-17 12:27:49,330] jepsen worker 12 - jepsen.util 102 :ok :read [4 4] INFO [2019-12-17 12:27:49,330] jepsen worker 12 - jepsen.util 102 :invoke :cas [4 [0 19]] INFO [2019-12-17 12:27:49,334] jepsen worker 59 - jepsen.util 149 :info :cas [19 [47 37]] :RequestCanceledException INFO [2019-12-17 12:27:49,335] jepsen worker 59 - jepsen.util 239 :invoke :cas [19 [15 22]] INFO [2019-12-17 12:27:49,349] jepsen worker 66 - jepsen.util 156 :ok :write [22 12] INFO [2019-12-17 12:27:49,349] jepsen worker 66 - jepsen.util 156 :invoke :cas [22 [46 19]] INFO [2019-12-17 12:27:49,378] jepsen worker 12 - jepsen.util 102 :fail :cas [4 [0 19]] :ValueNotSwapFrom INFO [2019-12-17 12:27:49,378] jepsen worker 12 - jepsen.util 102 :invoke :write [4 28] INFO [2019-12-17 12:27:49,383] jepsen worker 82 - jepsen.util 172 :info :write [27 30] :RequestCanceledException INFO [2019-12-17 12:27:49,383] jepsen worker 22 - jepsen.util 112 :info :write [7 24] :RequestCanceledException INFO [2019-12-17 12:27:49,385] jepsen worker 22 - jepsen.util 202 :invoke :write [7 16] INFO [2019-12-17 12:27:49,385] jepsen worker 82 - jepsen.util 262 :invoke :write [27 27] INFO [2019-12-17 12:27:49,388] jepsen worker 71 - jepsen.util 251 :ok :read [23 29] INFO [2019-12-17 12:27:49,389] jepsen worker 71 - jepsen.util 251 :invoke :write [23 30] INFO [2019-12-17 12:27:49,393] jepsen worker 28 - jepsen.util 118 :ok :write [9 0] INFO [2019-12-17 12:27:49,394] jepsen worker 28 - jepsen.util 118 :invoke :cas [9 [11 39]] INFO [2019-12-17 12:27:49,401] jepsen worker 5 - jepsen.util 5 :ok :read [1 23] INFO [2019-12-17 12:27:49,402] jepsen worker 5 - jepsen.util 5 :invoke :cas [1 [44 25]] INFO [2019-12-17 12:27:49,405] jepsen worker 8 - jepsen.util 98 :info :write [2 5] :RequestCanceledException INFO [2019-12-17 12:27:49,407] jepsen worker 8 - jepsen.util 188 :invoke :read [2 nil] INFO [2019-12-17 12:27:49,407] jepsen worker 33 - jepsen.util 213 :fail :cas [11 [42 45]] :ValueNotSwapFrom INFO [2019-12-17 12:27:49,407] jepsen worker 33 - jepsen.util 213 :invoke :cas [11 [31 17]] INFO [2019-12-17 12:27:49,413] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0}, :ns_1@172.28.128.126 {:progress 0}, :ns_1@172.28.128.127 {:progress 0}, :ns_1@172.28.128.128 {:progress 0}} INFO [2019-12-17 12:27:49,414] jepsen nemesis - couchbase.util Rebalance check count: 1 INFO [2019-12-17 12:27:49,414] jepsen nemesis - couchbase.util Rebalance stuck count: 0 INFO [2019-12-17 12:27:49,421] jepsen worker 81 - jepsen.util 81 :info :write [27 21] :RequestCanceledException INFO [2019-12-17 12:27:49,423] jepsen worker 81 - jepsen.util 171 :invoke :write [27 46] INFO [2019-12-17 12:27:49,435] jepsen worker 36 - jepsen.util 126 :info :write [12 20] :RequestCanceledException INFO [2019-12-17 12:27:49,436] jepsen worker 36 - jepsen.util 216 :invoke :cas [12 [13 0]] INFO [2019-12-17 12:27:49,441] jepsen worker 58 - jepsen.util 148 :ok :read [19 2] INFO [2019-12-17 12:27:49,441] jepsen worker 58 - jepsen.util 148 :invoke :cas [19 [15 9]] INFO [2019-12-17 12:27:49,487] jepsen worker 89 - jepsen.util 89 :info :write [29 10] :RequestCanceledException INFO [2019-12-17 12:27:49,489] jepsen worker 89 - jepsen.util 179 :invoke :write [29 23] INFO [2019-12-17 12:27:49,500] jepsen worker 16 - jepsen.util 16 :fail :cas [5 [6 44]] :ValueNotSwapFrom INFO [2019-12-17 12:27:49,501] jepsen worker 16 - jepsen.util 16 :invoke :write [5 32] INFO [2019-12-17 12:27:49,502] jepsen worker 41 - jepsen.util 131 :ok :read [13 9] INFO [2019-12-17 12:27:49,502] jepsen worker 41 - jepsen.util 131 :invoke :write [13 37] INFO [2019-12-17 12:27:49,511] jepsen worker 66 - jepsen.util 156 :fail :cas [22 [46 19]] :ValueNotSwapFrom INFO [2019-12-17 12:27:49,511] jepsen worker 66 - jepsen.util 156 :invoke :write [22 15] INFO [2019-12-17 12:27:49,521] jepsen worker 21 - jepsen.util 111 :info :write [7 40] :RequestCanceledException INFO [2019-12-17 12:27:49,523] jepsen worker 21 - jepsen.util 201 :invoke :cas [7 [34 40]] INFO [2019-12-17 12:27:49,533] jepsen worker 56 - jepsen.util 56 :info :write [18 39] :RequestCanceledException INFO [2019-12-17 12:27:49,534] jepsen worker 56 - jepsen.util 146 :invoke :cas [18 [40 32]] INFO [2019-12-17 12:27:49,539] jepsen worker 33 - jepsen.util 213 :fail :cas [11 [31 17]] :ValueNotSwapFrom INFO [2019-12-17 12:27:49,540] jepsen worker 86 - jepsen.util 86 :ok :read [28 34] INFO [2019-12-17 12:27:49,540] jepsen worker 33 - jepsen.util 213 :invoke :cas [11 [44 32]] INFO [2019-12-17 12:27:49,540] jepsen worker 86 - jepsen.util 86 :invoke :read [28 nil] INFO [2019-12-17 12:27:49,543] jepsen worker 45 - jepsen.util 225 :fail :cas [15 [37 48]] :ValueNotSwapFrom INFO [2019-12-17 12:27:49,543] jepsen worker 45 - jepsen.util 225 :invoke :cas [15 [40 31]] INFO [2019-12-17 12:27:49,555] jepsen worker 5 - jepsen.util 5 :fail :cas [1 [44 25]] :ValueNotSwapFrom INFO [2019-12-17 12:27:49,555] jepsen worker 5 - jepsen.util 5 :invoke :read [1 nil] INFO [2019-12-17 12:27:49,556] jepsen worker 35 - jepsen.util 35 :ok :read [11 7] INFO [2019-12-17 12:27:49,557] jepsen worker 35 - jepsen.util 35 :invoke :cas [11 [41 33]] INFO [2019-12-17 12:27:49,566] jepsen worker 4 - jepsen.util 184 :ok :read [1 23] INFO [2019-12-17 12:27:49,567] jepsen worker 4 - jepsen.util 184 :invoke :cas [1 [36 19]] INFO [2019-12-17 12:27:49,569] jepsen worker 28 - jepsen.util 118 :fail :cas [9 [11 39]] :ValueNotSwapFrom INFO [2019-12-17 12:27:49,570] jepsen worker 28 - jepsen.util 118 :invoke :cas [9 [37 34]] INFO [2019-12-17 12:27:49,573] jepsen worker 39 - jepsen.util 219 :fail :cas [13 [23 1]] :ValueNotSwapFrom INFO [2019-12-17 12:27:49,574] jepsen worker 39 - jepsen.util 219 :invoke :cas [13 [11 48]] INFO [2019-12-17 12:27:49,587] jepsen worker 69 - jepsen.util 159 :info :write [23 8] :RequestCanceledException INFO [2019-12-17 12:27:49,587] jepsen worker 61 - jepsen.util 241 :fail :cas [20 [16 35]] :ValueNotSwapFrom INFO [2019-12-17 12:27:49,587] jepsen worker 61 - jepsen.util 241 :invoke :read [20 nil] INFO [2019-12-17 12:27:49,588] jepsen worker 69 - jepsen.util 249 :invoke :cas [23 [5 10]] INFO [2019-12-17 12:27:49,598] jepsen worker 28 - jepsen.util 118 :fail :cas [9 [37 34]] :ValueNotSwapFrom INFO [2019-12-17 12:27:49,599] jepsen worker 28 - jepsen.util 118 :invoke :read [9 nil] INFO [2019-12-17 12:27:49,603] jepsen worker 87 - jepsen.util 177 :fail :cas [29 [26 14]] :ValueNotSwapFrom INFO [2019-12-17 12:27:49,603] jepsen worker 87 - jepsen.util 177 :invoke :read [29 nil] INFO [2019-12-17 12:27:49,604] jepsen worker 57 - jepsen.util 147 :info :write [19 41] :RequestCanceledException INFO [2019-12-17 12:27:49,606] jepsen worker 57 - jepsen.util 237 :invoke :cas [19 [5 41]] INFO [2019-12-17 12:27:49,610] jepsen worker 48 - jepsen.util 228 :fail :cas [16 [33 38]] :ValueNotSwapFrom INFO [2019-12-17 12:27:49,610] jepsen worker 48 - jepsen.util 228 :invoke :write [16 17] INFO [2019-12-17 12:27:49,612] jepsen worker 2 - jepsen.util 92 :ok :read [0 14] INFO [2019-12-17 12:27:49,612] jepsen worker 2 - jepsen.util 92 :invoke :read [0 nil] INFO [2019-12-17 12:27:49,621] jepsen worker 11 - jepsen.util 101 :info :write [3 28] :RequestCanceledException INFO [2019-12-17 12:27:49,623] jepsen worker 11 - jepsen.util 191 :invoke :cas [3 [35 48]] INFO [2019-12-17 12:27:49,633] jepsen worker 28 - jepsen.util 118 :ok :read [9 0] INFO [2019-12-17 12:27:49,634] jepsen worker 28 - jepsen.util 118 :invoke :write [9 45] INFO [2019-12-17 12:27:49,634] jepsen worker 70 - jepsen.util 160 :info :write [23 18] :RequestCanceledException INFO [2019-12-17 12:27:49,634] jepsen worker 88 - jepsen.util 178 :info :write [29 29] :RequestCanceledException INFO [2019-12-17 12:27:49,636] jepsen worker 88 - jepsen.util 268 :invoke :cas [29 [18 26]] INFO [2019-12-17 12:27:49,636] jepsen worker 70 - jepsen.util 250 :invoke :cas [23 [8 26]] INFO [2019-12-17 12:27:49,669] jepsen worker 83 - jepsen.util 263 :ok :read [27 18] INFO [2019-12-17 12:27:49,669] jepsen worker 83 - jepsen.util 263 :invoke :write [27 48] INFO [2019-12-17 12:27:49,674] jepsen worker 79 - jepsen.util 259 :ok :read [26 47] INFO [2019-12-17 12:27:49,675] jepsen worker 79 - jepsen.util 259 :invoke :cas [26 [31 23]] INFO [2019-12-17 12:27:49,687] jepsen worker 52 - jepsen.util 142 :info :cas [17 [0 21]] :RequestCanceledException INFO [2019-12-17 12:27:49,688] jepsen worker 52 - jepsen.util 232 :invoke :cas [17 [25 30]] INFO [2019-12-17 12:27:49,690] jepsen worker 45 - jepsen.util 225 :fail :cas [15 [40 31]] :ValueNotSwapFrom INFO [2019-12-17 12:27:49,690] jepsen worker 45 - jepsen.util 225 :invoke :write [15 42] INFO [2019-12-17 12:27:49,697] jepsen worker 70 - jepsen.util 250 :fail :cas [23 [8 26]] :ValueNotSwapFrom INFO [2019-12-17 12:27:49,697] jepsen worker 70 - jepsen.util 250 :invoke :write [23 41] INFO [2019-12-17 12:27:49,705] jepsen worker 31 - jepsen.util 211 :info :write [10 7] :RequestCanceledException INFO [2019-12-17 12:27:49,705] jepsen worker 42 - jepsen.util 222 :info :write [14 21] :RequestCanceledException INFO [2019-12-17 12:27:49,707] jepsen worker 31 - jepsen.util 301 :invoke :read [10 nil] INFO [2019-12-17 12:27:49,707] jepsen worker 42 - jepsen.util 312 :invoke :read [14 nil] INFO [2019-12-17 12:27:49,720] jepsen worker 7 - jepsen.util 187 :info :write [2 47] :RequestCanceledException INFO [2019-12-17 12:27:49,721] jepsen worker 25 - jepsen.util 205 :info :write [8 15] :RequestCanceledException INFO [2019-12-17 12:27:49,722] jepsen worker 7 - jepsen.util 277 :invoke :cas [2 [44 26]] INFO [2019-12-17 12:27:49,723] jepsen worker 25 - jepsen.util 295 :invoke :read [8 nil] INFO [2019-12-17 12:27:49,726] jepsen worker 61 - jepsen.util 241 :ok :read [20 7] INFO [2019-12-17 12:27:49,727] jepsen worker 61 - jepsen.util 241 :invoke :read [20 nil] INFO [2019-12-17 12:27:49,734] jepsen worker 15 - jepsen.util 195 :info :write [5 16] :RequestCanceledException INFO [2019-12-17 12:27:49,736] jepsen worker 15 - jepsen.util 285 :invoke :read [5 nil] INFO [2019-12-17 12:27:49,750] jepsen worker 74 - jepsen.util 164 :info :cas [24 [25 41]] :RequestCanceledException INFO [2019-12-17 12:27:49,750] jepsen worker 46 - jepsen.util 226 :fail :read [15 nil] :RequestTimeoutException INFO [2019-12-17 12:27:49,750] jepsen worker 46 - jepsen.util 226 :invoke :write [15 2] INFO [2019-12-17 12:27:49,751] jepsen worker 74 - jepsen.util 254 :invoke :read [24 nil] INFO [2019-12-17 12:27:49,756] jepsen worker 31 - jepsen.util 301 :ok :read [10 23] INFO [2019-12-17 12:27:49,756] jepsen worker 31 - jepsen.util 301 :invoke :read [10 nil] INFO [2019-12-17 12:27:49,759] jepsen worker 58 - jepsen.util 148 :fail :cas [19 [15 9]] :ValueNotSwapFrom INFO [2019-12-17 12:27:49,760] jepsen worker 58 - jepsen.util 148 :invoke :read [19 nil] INFO [2019-12-17 12:27:49,772] jepsen worker 31 - jepsen.util 301 :ok :read [10 23] INFO [2019-12-17 12:27:49,772] jepsen worker 31 - jepsen.util 301 :invoke :write [10 43] INFO [2019-12-17 12:27:49,785] jepsen worker 20 - jepsen.util 200 :info :cas [6 [7 12]] :RequestCanceledException INFO [2019-12-17 12:27:49,786] jepsen worker 20 - jepsen.util 290 :invoke :read [6 nil] INFO [2019-12-17 12:27:49,800] jepsen worker 35 - jepsen.util 35 :fail :cas [11 [41 33]] :ValueNotSwapFrom INFO [2019-12-17 12:27:49,800] jepsen worker 35 - jepsen.util 35 :invoke :cas [11 [27 46]] INFO [2019-12-17 12:27:49,802] jepsen worker 58 - jepsen.util 148 :ok :read [19 41] INFO [2019-12-17 12:27:49,802] jepsen worker 58 - jepsen.util 148 :invoke :read [19 nil] INFO [2019-12-17 12:27:49,818] jepsen worker 80 - jepsen.util 260 :info :write [26 22] :RequestCanceledException INFO [2019-12-17 12:27:49,820] jepsen worker 80 - jepsen.util 350 :invoke :write [26 43] INFO [2019-12-17 12:27:49,834] jepsen worker 75 - jepsen.util 165 :info :write [25 46] :RequestCanceledException INFO [2019-12-17 12:27:49,834] jepsen worker 43 - jepsen.util 223 :info :write [14 3] :RequestCanceledException INFO [2019-12-17 12:27:49,835] jepsen worker 75 - jepsen.util 255 :invoke :read [25 nil] INFO [2019-12-17 12:27:49,835] jepsen worker 43 - jepsen.util 313 :invoke :cas [14 [34 25]] INFO [2019-12-17 12:27:49,844] jepsen worker 69 - jepsen.util 249 :fail :cas [23 [5 10]] :ValueNotSwapFrom INFO [2019-12-17 12:27:49,845] jepsen worker 69 - jepsen.util 249 :invoke :cas [23 [47 42]] INFO [2019-12-17 12:27:49,846] jepsen worker 36 - jepsen.util 216 :fail :cas [12 [13 0]] :ValueNotSwapFrom INFO [2019-12-17 12:27:49,846] jepsen worker 36 - jepsen.util 216 :invoke :cas [12 [12 25]] INFO [2019-12-17 12:27:49,849] jepsen worker 53 - jepsen.util 233 :info :write [17 9] :RequestCanceledException INFO [2019-12-17 12:27:49,849] jepsen worker 20 - jepsen.util 290 :ok :read [6 45] INFO [2019-12-17 12:27:49,850] jepsen worker 20 - jepsen.util 290 :invoke :read [6 nil] INFO [2019-12-17 12:27:49,851] jepsen worker 53 - jepsen.util 323 :invoke :read [17 nil] INFO [2019-12-17 12:27:49,888] jepsen worker 14 - jepsen.util 194 :fail :read [4 nil] :RequestTimeoutException INFO [2019-12-17 12:27:49,888] jepsen worker 14 - jepsen.util 194 :invoke :write [4 3] INFO [2019-12-17 12:27:49,913] jepsen worker 58 - jepsen.util 148 :ok :read [19 41] INFO [2019-12-17 12:27:49,913] jepsen worker 72 - jepsen.util 72 :info :write [24 12] :SyncWriteAmbiguous INFO [2019-12-17 12:27:49,913] jepsen worker 58 - jepsen.util 148 :invoke :read [19 nil] INFO [2019-12-17 12:27:49,915] jepsen worker 72 - jepsen.util 162 :invoke :read [24 nil] INFO [2019-12-17 12:27:49,920] jepsen worker 47 - jepsen.util 137 :info :write [15 10] :RequestCanceledException INFO [2019-12-17 12:27:49,920] jepsen worker 55 - jepsen.util 145 :info :write [18 36] :RequestCanceledException INFO [2019-12-17 12:27:49,920] jepsen worker 26 - jepsen.util 206 :info :write [8 10] :RequestCanceledException INFO [2019-12-17 12:27:49,921] jepsen worker 55 - jepsen.util 235 :invoke :write [18 15] INFO [2019-12-17 12:27:49,921] jepsen worker 47 - jepsen.util 227 :invoke :write [15 24] INFO [2019-12-17 12:27:49,921] jepsen worker 26 - jepsen.util 296 :invoke :cas [8 [47 9]] INFO [2019-12-17 12:27:49,944] jepsen worker 21 - jepsen.util 201 :fail :cas [7 [34 40]] :ValueNotSwapFrom INFO [2019-12-17 12:27:49,945] jepsen worker 21 - jepsen.util 201 :invoke :write [7 34] INFO [2019-12-17 12:27:49,950] jepsen worker 37 - jepsen.util 37 :info :write [12 25] :RequestCanceledException INFO [2019-12-17 12:27:49,951] jepsen worker 37 - jepsen.util 127 :invoke :cas [12 [41 23]] INFO [2019-12-17 12:27:49,967] jepsen worker 27 - jepsen.util 27 :fail :cas [9 [3 30]] :ValueNotSwapFrom INFO [2019-12-17 12:27:49,967] jepsen worker 27 - jepsen.util 27 :invoke :cas [9 [2 1]] INFO [2019-12-17 12:27:49,980] jepsen worker 28 - jepsen.util 118 :ok :write [9 45] INFO [2019-12-17 12:27:49,981] jepsen worker 28 - jepsen.util 118 :invoke :read [9 nil] INFO [2019-12-17 12:27:49,995] jepsen worker 8 - jepsen.util 188 :ok :read [2 15] INFO [2019-12-17 12:27:49,995] jepsen worker 8 - jepsen.util 188 :invoke :write [2 18] INFO [2019-12-17 12:27:49,996] jepsen worker 33 - jepsen.util 213 :fail :cas [11 [44 32]] :ValueNotSwapFrom INFO [2019-12-17 12:27:49,996] jepsen worker 33 - jepsen.util 213 :invoke :write [11 42] INFO [2019-12-17 12:27:50,003] jepsen worker 60 - jepsen.util 240 :info :write [20 48] :RequestCanceledException INFO [2019-12-17 12:27:50,005] jepsen worker 60 - jepsen.util 330 :invoke :write [20 3] INFO [2019-12-17 12:27:50,022] jepsen worker 63 - jepsen.util 153 :ok :read [21 38] INFO [2019-12-17 12:27:50,023] jepsen worker 63 - jepsen.util 153 :invoke :write [21 49] INFO [2019-12-17 12:27:50,046] jepsen worker 28 - jepsen.util 118 :ok :read [9 45] INFO [2019-12-17 12:27:50,046] jepsen worker 28 - jepsen.util 118 :invoke :write [9 36] INFO [2019-12-17 12:27:50,081] jepsen worker 42 - jepsen.util 312 :ok :read [14 27] INFO [2019-12-17 12:27:50,081] jepsen worker 42 - jepsen.util 312 :invoke :read [14 nil] INFO [2019-12-17 12:27:50,084] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1095,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:27:50,084] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1095,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:27:50,098] jepsen worker 20 - jepsen.util 290 :ok :read [6 45] INFO [2019-12-17 12:27:50,098] jepsen worker 20 - jepsen.util 290 :invoke :read [6 nil] INFO [2019-12-17 12:27:50,099] jepsen worker 56 - jepsen.util 146 :fail :cas [18 [40 32]] :ValueNotSwapFrom INFO [2019-12-17 12:27:50,099] jepsen worker 56 - jepsen.util 146 :invoke :cas [18 [28 11]] INFO [2019-12-17 12:27:50,113] jepsen worker 61 - jepsen.util 241 :ok :read [20 7] INFO [2019-12-17 12:27:50,113] jepsen worker 61 - jepsen.util 241 :invoke :write [20 49] INFO [2019-12-17 12:27:50,119] jepsen worker 27 - jepsen.util 27 :fail :cas [9 [2 1]] :ValueNotSwapFrom INFO [2019-12-17 12:27:50,119] jepsen worker 27 - jepsen.util 27 :invoke :write [9 41] INFO [2019-12-17 12:27:50,121] jepsen worker 13 - jepsen.util 103 :info :write [4 39] :RequestCanceledException INFO [2019-12-17 12:27:50,121] jepsen worker 64 - jepsen.util 154 :info :write [21 25] :RequestCanceledException INFO [2019-12-17 12:27:50,123] jepsen worker 13 - jepsen.util 193 :invoke :read [4 nil] INFO [2019-12-17 12:27:50,123] jepsen worker 64 - jepsen.util 244 :invoke :write [21 42] INFO [2019-12-17 12:27:50,123] jepsen worker 58 - jepsen.util 148 :ok :read [19 41] INFO [2019-12-17 12:27:50,123] jepsen worker 58 - jepsen.util 148 :invoke :cas [19 [31 25]] INFO [2019-12-17 12:27:50,125] jepsen worker 32 - jepsen.util 32 :info :write [10 15] :SyncWriteAmbiguous INFO [2019-12-17 12:27:50,127] jepsen worker 32 - jepsen.util 122 :invoke :write [10 16] INFO [2019-12-17 12:27:50,149] jepsen worker 69 - jepsen.util 249 :fail :cas [23 [47 42]] :ValueNotSwapFrom INFO [2019-12-17 12:27:50,150] jepsen worker 69 - jepsen.util 249 :invoke :cas [23 [29 44]] INFO [2019-12-17 12:27:50,157] jepsen worker 58 - jepsen.util 148 :fail :cas [19 [31 25]] :ValueNotSwapFrom INFO [2019-12-17 12:27:50,157] jepsen worker 58 - jepsen.util 148 :invoke :read [19 nil] INFO [2019-12-17 12:27:50,168] jepsen worker 18 - jepsen.util 198 :fail :cas [6 [24 39]] :ValueNotSwapFrom INFO [2019-12-17 12:27:50,169] jepsen worker 18 - jepsen.util 198 :invoke :cas [6 [21 31]] INFO [2019-12-17 12:27:50,179] jepsen worker 58 - jepsen.util 148 :ok :read [19 41] INFO [2019-12-17 12:27:50,179] jepsen worker 58 - jepsen.util 148 :invoke :cas [19 [15 38]] INFO [2019-12-17 12:27:50,184] jepsen worker 25 - jepsen.util 295 :ok :read [8 14] INFO [2019-12-17 12:27:50,184] jepsen worker 25 - jepsen.util 295 :invoke :write [8 7] INFO [2019-12-17 12:27:50,195] jepsen worker 42 - jepsen.util 312 :ok :read [14 27] INFO [2019-12-17 12:27:50,196] jepsen worker 42 - jepsen.util 312 :invoke :write [14 13] INFO [2019-12-17 12:27:50,202] jepsen worker 11 - jepsen.util 191 :fail :cas [3 [35 48]] :ValueNotSwapFrom INFO [2019-12-17 12:27:50,202] jepsen worker 11 - jepsen.util 191 :invoke :read [3 nil] INFO [2019-12-17 12:27:50,204] jepsen worker 58 - jepsen.util 148 :fail :cas [19 [15 38]] :ValueNotSwapFrom INFO [2019-12-17 12:27:50,204] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1117,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:27:50,204] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1117,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:27:50,204] jepsen worker 58 - jepsen.util 148 :invoke :cas [19 [30 29]] INFO [2019-12-17 12:27:50,223] jepsen worker 5 - jepsen.util 5 :ok :read [1 23] INFO [2019-12-17 12:27:50,223] jepsen worker 5 - jepsen.util 5 :invoke :cas [1 [10 40]] INFO [2019-12-17 12:27:50,236] jepsen worker 58 - jepsen.util 148 :fail :cas [19 [30 29]] :ValueNotSwapFrom INFO [2019-12-17 12:27:50,236] jepsen worker 58 - jepsen.util 148 :invoke :cas [19 [41 20]] INFO [2019-12-17 12:27:50,241] jepsen worker 18 - jepsen.util 198 :fail :cas [6 [21 31]] :ValueNotSwapFrom INFO [2019-12-17 12:27:50,241] jepsen worker 18 - jepsen.util 198 :invoke :write [6 27] INFO [2019-12-17 12:27:50,243] jepsen worker 87 - jepsen.util 177 :ok :read [29 5] INFO [2019-12-17 12:27:50,243] jepsen worker 87 - jepsen.util 177 :invoke :read [29 nil] INFO [2019-12-17 12:27:50,270] jepsen worker 63 - jepsen.util 153 :ok :write [21 49] INFO [2019-12-17 12:27:50,270] jepsen worker 63 - jepsen.util 153 :invoke :read [21 nil] INFO [2019-12-17 12:27:50,272] jepsen worker 11 - jepsen.util 191 :ok :read [3 14] INFO [2019-12-17 12:27:50,272] jepsen worker 11 - jepsen.util 191 :invoke :read [3 nil] INFO [2019-12-17 12:27:50,272] jepsen worker 79 - jepsen.util 259 :fail :cas [26 [31 23]] :ValueNotSwapFrom INFO [2019-12-17 12:27:50,272] jepsen worker 79 - jepsen.util 259 :invoke :cas [26 [12 40]] INFO [2019-12-17 12:27:50,291] jepsen worker 53 - jepsen.util 323 :ok :read [17 0] INFO [2019-12-17 12:27:50,291] jepsen worker 53 - jepsen.util 323 :invoke :cas [17 [20 23]] INFO [2019-12-17 12:27:50,301] jepsen worker 52 - jepsen.util 232 :fail :cas [17 [25 30]] :ValueNotSwapFrom INFO [2019-12-17 12:27:50,301] jepsen worker 52 - jepsen.util 232 :invoke :cas [17 [40 45]] INFO [2019-12-17 12:27:50,321] jepsen worker 65 - jepsen.util 155 :info :cas [21 [6 4]] :RequestCanceledException INFO [2019-12-17 12:27:50,321] jepsen worker 24 - jepsen.util 24 :info :write [8 32] :RequestCanceledException INFO [2019-12-17 12:27:50,322] jepsen worker 65 - jepsen.util 245 :invoke :cas [21 [19 12]] INFO [2019-12-17 12:27:50,322] jepsen worker 24 - jepsen.util 114 :invoke :read [8 nil] INFO [2019-12-17 12:27:50,332] jepsen worker 43 - jepsen.util 313 :fail :cas [14 [34 25]] :ValueNotSwapFrom INFO [2019-12-17 12:27:50,332] jepsen worker 43 - jepsen.util 313 :invoke :read [14 nil] INFO [2019-12-17 12:27:50,352] jepsen worker 44 - jepsen.util 44 :info :write [14 32] :RequestCanceledException INFO [2019-12-17 12:27:50,353] jepsen worker 44 - jepsen.util 134 :invoke :cas [14 [19 7]] INFO [2019-12-17 12:27:50,357] jepsen worker 11 - jepsen.util 191 :ok :read [3 14] INFO [2019-12-17 12:27:50,357] jepsen worker 11 - jepsen.util 191 :invoke :cas [3 [23 29]] INFO [2019-12-17 12:27:50,358] jepsen worker 53 - jepsen.util 323 :fail :cas [17 [20 23]] :ValueNotSwapFrom INFO [2019-12-17 12:27:50,358] jepsen worker 53 - jepsen.util 323 :invoke :read [17 nil] INFO [2019-12-17 12:27:50,360] jepsen worker 27 - jepsen.util 27 :ok :write [9 41] INFO [2019-12-17 12:27:50,361] jepsen worker 27 - jepsen.util 27 :invoke :write [9 34] INFO [2019-12-17 12:27:50,418] jepsen worker 11 - jepsen.util 191 :fail :cas [3 [23 29]] :ValueNotSwapFrom INFO [2019-12-17 12:27:50,419] jepsen worker 11 - jepsen.util 191 :invoke :cas [3 [2 46]] INFO [2019-12-17 12:27:50,419] jepsen worker 23 - jepsen.util 113 :ok :read [7 20] INFO [2019-12-17 12:27:50,419] jepsen worker 23 - jepsen.util 113 :invoke :cas [7 [46 39]] INFO [2019-12-17 12:27:50,423] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1147,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:27:50,423] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1147,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:27:50,424] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1150,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:27:50,424] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1150,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:27:50,424] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1151,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:27:50,424] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1151,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:27:50,435] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0}, :ns_1@172.28.128.126 {:progress 0}, :ns_1@172.28.128.127 {:progress 0}, :ns_1@172.28.128.128 {:progress 0.007547169811320753}} INFO [2019-12-17 12:27:50,436] jepsen nemesis - couchbase.util Rebalance check count: 2 INFO [2019-12-17 12:27:50,436] jepsen nemesis - couchbase.util Rebalance stuck count: 0 INFO [2019-12-17 12:27:50,444] jepsen worker 56 - jepsen.util 146 :fail :cas [18 [28 11]] :ValueNotSwapFrom INFO [2019-12-17 12:27:50,444] jepsen worker 56 - jepsen.util 146 :invoke :cas [18 [30 14]] INFO [2019-12-17 12:27:50,467] jepsen worker 11 - jepsen.util 191 :fail :cas [3 [2 46]] :ValueNotSwapFrom INFO [2019-12-17 12:27:50,467] jepsen worker 11 - jepsen.util 191 :invoke :read [3 nil] INFO [2019-12-17 12:27:50,478] jepsen worker 13 - jepsen.util 193 :ok :read [4 4] INFO [2019-12-17 12:27:50,478] jepsen worker 13 - jepsen.util 193 :invoke :read [4 nil] INFO [2019-12-17 12:27:50,498] jepsen worker 11 - jepsen.util 191 :ok :read [3 14] INFO [2019-12-17 12:27:50,498] jepsen worker 11 - jepsen.util 191 :invoke :write [3 21] INFO [2019-12-17 12:27:50,501] jepsen worker 27 - jepsen.util 27 :ok :write [9 34] INFO [2019-12-17 12:27:50,501] jepsen worker 27 - jepsen.util 27 :invoke :cas [9 [22 24]] INFO [2019-12-17 12:27:50,521] jepsen worker 38 - jepsen.util 128 :info :write [12 41] :RequestCanceledException INFO [2019-12-17 12:27:50,523] jepsen worker 38 - jepsen.util 218 :invoke :read [12 nil] INFO [2019-12-17 12:27:50,527] jepsen worker 27 - jepsen.util 27 :fail :cas [9 [22 24]] :ValueNotSwapFrom INFO [2019-12-17 12:27:50,528] jepsen worker 27 - jepsen.util 27 :invoke :cas [9 [19 26]] INFO [2019-12-17 12:27:50,534] jepsen worker 51 - jepsen.util 51 :info :write [17 30] :RequestCanceledException INFO [2019-12-17 12:27:50,536] jepsen worker 51 - jepsen.util 141 :invoke :read [17 nil] INFO [2019-12-17 12:27:50,540] jepsen worker 5 - jepsen.util 5 :fail :cas [1 [10 40]] :ValueNotSwapFrom INFO [2019-12-17 12:27:50,540] jepsen worker 5 - jepsen.util 5 :invoke :cas [1 [22 2]] INFO [2019-12-17 12:27:50,551] jepsen worker 0 - jepsen.util 90 :info :write [0 32] :RequestCanceledException INFO [2019-12-17 12:27:50,551] jepsen worker 49 - jepsen.util 139 :info :write [16 41] :RequestCanceledException INFO [2019-12-17 12:27:50,553] jepsen worker 0 - jepsen.util 180 :invoke :cas [0 [47 40]] INFO [2019-12-17 12:27:50,553] jepsen worker 49 - jepsen.util 229 :invoke :cas [16 [1 19]] INFO [2019-12-17 12:27:50,565] jepsen worker 27 - jepsen.util 27 :fail :cas [9 [19 26]] :ValueNotSwapFrom INFO [2019-12-17 12:27:50,565] jepsen worker 27 - jepsen.util 27 :invoke :cas [9 [47 49]] INFO [2019-12-17 12:27:50,570] jepsen worker 58 - jepsen.util 148 :ok :cas [19 [41 20]] INFO [2019-12-17 12:27:50,570] jepsen worker 58 - jepsen.util 148 :invoke :cas [19 [37 10]] INFO [2019-12-17 12:27:50,575] jepsen worker 87 - jepsen.util 177 :ok :read [29 5] INFO [2019-12-17 12:27:50,576] jepsen worker 87 - jepsen.util 177 :invoke :cas [29 [17 26]] INFO [2019-12-17 12:27:50,585] jepsen worker 27 - jepsen.util 27 :fail :cas [9 [47 49]] :ValueNotSwapFrom INFO [2019-12-17 12:27:50,585] jepsen worker 27 - jepsen.util 27 :invoke :write [9 28] INFO [2019-12-17 12:27:50,590] jepsen worker 58 - jepsen.util 148 :fail :cas [19 [37 10]] :ValueNotSwapFrom INFO [2019-12-17 12:27:50,591] jepsen worker 58 - jepsen.util 148 :invoke :write [19 17] INFO [2019-12-17 12:27:50,597] jepsen worker 43 - jepsen.util 313 :ok :read [14 27] INFO [2019-12-17 12:27:50,597] jepsen worker 43 - jepsen.util 313 :invoke :cas [14 [42 46]] INFO [2019-12-17 12:27:50,608] jepsen worker 4 - jepsen.util 184 :fail :cas [1 [36 19]] :ValueNotSwapFrom INFO [2019-12-17 12:27:50,609] jepsen worker 4 - jepsen.util 184 :invoke :cas [1 [5 23]] INFO [2019-12-17 12:27:50,612] jepsen worker 35 - jepsen.util 35 :fail :cas [11 [27 46]] :ValueNotSwapFrom INFO [2019-12-17 12:27:50,613] jepsen worker 35 - jepsen.util 35 :invoke :cas [11 [12 40]] INFO [2019-12-17 12:27:50,629] jepsen worker 79 - jepsen.util 259 :fail :cas [26 [12 40]] :ValueNotSwapFrom INFO [2019-12-17 12:27:50,629] jepsen worker 79 - jepsen.util 259 :invoke :read [26 nil] INFO [2019-12-17 12:27:50,635] jepsen worker 54 - jepsen.util 234 :info :write [18 3] :RequestCanceledException INFO [2019-12-17 12:27:50,636] jepsen worker 54 - jepsen.util 324 :invoke :read [18 nil] INFO [2019-12-17 12:27:50,653] jepsen worker 88 - jepsen.util 268 :fail :cas [29 [18 26]] :ValueNotSwapFrom INFO [2019-12-17 12:27:50,653] jepsen worker 88 - jepsen.util 268 :invoke :cas [29 [28 29]] INFO [2019-12-17 12:27:50,692] jepsen worker 27 - jepsen.util 27 :ok :write [9 28] INFO [2019-12-17 12:27:50,693] jepsen worker 27 - jepsen.util 27 :invoke :read [9 nil] INFO [2019-12-17 12:27:50,694] jepsen worker 52 - jepsen.util 232 :fail :cas [17 [40 45]] :ValueNotSwapFrom INFO [2019-12-17 12:27:50,694] jepsen worker 52 - jepsen.util 232 :invoke :cas [17 [15 37]] INFO [2019-12-17 12:27:50,711] jepsen worker 58 - jepsen.util 148 :ok :write [19 17] INFO [2019-12-17 12:27:50,712] jepsen worker 58 - jepsen.util 148 :invoke :cas [19 [31 16]] INFO [2019-12-17 12:27:50,715] jepsen worker 27 - jepsen.util 27 :ok :read [9 28] INFO [2019-12-17 12:27:50,716] jepsen worker 27 - jepsen.util 27 :invoke :read [9 nil] INFO [2019-12-17 12:27:50,734] jepsen worker 77 - jepsen.util 167 :info :write [25 41] :RequestCanceledException INFO [2019-12-17 12:27:50,736] jepsen worker 77 - jepsen.util 257 :invoke :cas [25 [45 27]] INFO [2019-12-17 12:27:50,738] jepsen worker 27 - jepsen.util 27 :ok :read [9 28] INFO [2019-12-17 12:27:50,738] jepsen worker 27 - jepsen.util 27 :invoke :cas [9 [18 4]] INFO [2019-12-17 12:27:50,746] jepsen worker 37 - jepsen.util 127 :fail :cas [12 [41 23]] :ValueNotSwapFrom INFO [2019-12-17 12:27:50,746] jepsen worker 37 - jepsen.util 127 :invoke :cas [12 [30 33]] INFO [2019-12-17 12:27:50,751] jepsen worker 67 - jepsen.util 157 :info :write [22 41] :RequestCanceledException INFO [2019-12-17 12:27:50,753] jepsen worker 67 - jepsen.util 247 :invoke :cas [22 [33 35]] INFO [2019-12-17 12:27:50,759] jepsen worker 43 - jepsen.util 313 :fail :cas [14 [42 46]] :ValueNotSwapFrom INFO [2019-12-17 12:27:50,759] jepsen worker 43 - jepsen.util 313 :invoke :write [14 36] INFO [2019-12-17 12:27:50,759] jepsen worker 27 - jepsen.util 27 :fail :cas [9 [18 4]] :ValueNotSwapFrom INFO [2019-12-17 12:27:50,759] jepsen worker 27 - jepsen.util 27 :invoke :write [9 42] INFO [2019-12-17 12:27:50,821] jepsen worker 3 - jepsen.util 93 :info :write [1 8] :RequestCanceledException INFO [2019-12-17 12:27:50,822] jepsen worker 3 - jepsen.util 183 :invoke :cas [1 [35 12]] INFO [2019-12-17 12:27:50,837] jepsen worker 53 - jepsen.util 323 :ok :read [17 0] INFO [2019-12-17 12:27:50,837] jepsen worker 53 - jepsen.util 323 :invoke :read [17 nil] INFO [2019-12-17 12:27:50,855] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1194,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:27:50,855] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1194,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:27:50,855] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1196,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:27:50,855] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1196,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:27:50,868] jepsen worker 37 - jepsen.util 127 :fail :cas [12 [30 33]] :ValueNotSwapFrom INFO [2019-12-17 12:27:50,868] jepsen worker 37 - jepsen.util 127 :invoke :read [12 nil] INFO [2019-12-17 12:27:50,878] jepsen worker 27 - jepsen.util 27 :ok :write [9 42] INFO [2019-12-17 12:27:50,878] jepsen worker 27 - jepsen.util 27 :invoke :read [9 nil] INFO [2019-12-17 12:27:50,888] jepsen worker 6 - jepsen.util 96 :info :write [2 43] :RequestCanceledException INFO [2019-12-17 12:27:50,889] jepsen worker 6 - jepsen.util 186 :invoke :cas [2 [28 13]] INFO [2019-12-17 12:27:50,903] jepsen worker 1 - jepsen.util 181 :info :write [0 28] :RequestCanceledException INFO [2019-12-17 12:27:50,905] jepsen worker 1 - jepsen.util 271 :invoke :cas [0 [2 25]] INFO [2019-12-17 12:27:50,907] jepsen worker 27 - jepsen.util 27 :ok :read [9 42] INFO [2019-12-17 12:27:50,907] jepsen worker 27 - jepsen.util 27 :invoke :write [9 17] INFO [2019-12-17 12:27:50,908] jepsen worker 58 - jepsen.util 148 :fail :cas [19 [31 16]] :ValueNotSwapFrom INFO [2019-12-17 12:27:50,909] jepsen worker 58 - jepsen.util 148 :invoke :write [19 11] INFO [2019-12-17 12:27:50,975] jepsen worker 51 - jepsen.util 141 :ok :read [17 0] INFO [2019-12-17 12:27:50,975] jepsen worker 51 - jepsen.util 141 :invoke :read [17 nil] INFO [2019-12-17 12:27:50,988] jepsen worker 9 - jepsen.util 99 :fail :read [3 nil] :RequestTimeoutException INFO [2019-12-17 12:27:50,988] jepsen worker 9 - jepsen.util 99 :invoke :cas [3 [5 12]] INFO [2019-12-17 12:27:50,988] jepsen worker 87 - jepsen.util 177 :fail :cas [29 [17 26]] :ValueNotSwapFrom INFO [2019-12-17 12:27:50,988] jepsen worker 87 - jepsen.util 177 :invoke :cas [29 [40 1]] INFO [2019-12-17 12:27:50,992] jepsen worker 19 - jepsen.util 109 :info :write [6 16] :SyncWriteAmbiguous INFO [2019-12-17 12:27:50,993] jepsen worker 19 - jepsen.util 199 :invoke :read [6 nil] INFO [2019-12-17 12:27:50,999] jepsen worker 6 - jepsen.util 186 :fail :cas [2 [28 13]] :ValueNotSwapFrom INFO [2019-12-17 12:27:50,999] jepsen worker 6 - jepsen.util 186 :invoke :write [2 36] INFO [2019-12-17 12:27:51,002] jepsen worker 44 - jepsen.util 134 :fail :cas [14 [19 7]] :ValueNotSwapFrom INFO [2019-12-17 12:27:51,002] jepsen worker 27 - jepsen.util 27 :ok :write [9 17] INFO [2019-12-17 12:27:51,002] jepsen worker 44 - jepsen.util 134 :invoke :write [14 48] INFO [2019-12-17 12:27:51,002] jepsen worker 27 - jepsen.util 27 :invoke :write [9 46] INFO [2019-12-17 12:27:51,004] jepsen worker 76 - jepsen.util 76 :info :write [25 19] :RequestCanceledException INFO [2019-12-17 12:27:51,005] jepsen worker 76 - jepsen.util 166 :invoke :cas [25 [43 8]] INFO [2019-12-17 12:27:51,020] jepsen worker 50 - jepsen.util 230 :info :write [16 33] :RequestCanceledException INFO [2019-12-17 12:27:51,022] jepsen worker 50 - jepsen.util 320 :invoke :read [16 nil] INFO [2019-12-17 12:27:51,031] jepsen worker 30 - jepsen.util 30 :fail :cas [10 [11 24]] :ValueNotSwapFrom INFO [2019-12-17 12:27:51,031] jepsen worker 30 - jepsen.util 30 :invoke :write [10 11] INFO [2019-12-17 12:27:51,040] jepsen worker 29 - jepsen.util 119 :ok :read [9 17] INFO [2019-12-17 12:27:51,041] jepsen worker 29 - jepsen.util 119 :invoke :write [9 36] INFO [2019-12-17 12:27:51,074] jepsen worker 13 - jepsen.util 193 :ok :read [4 4] INFO [2019-12-17 12:27:51,074] jepsen worker 13 - jepsen.util 193 :invoke :cas [4 [49 45]] INFO [2019-12-17 12:27:51,091] jepsen worker 74 - jepsen.util 254 :ok :read [24 35] INFO [2019-12-17 12:27:51,091] jepsen worker 74 - jepsen.util 254 :invoke :cas [24 [2 8]] INFO [2019-12-17 12:27:51,117] jepsen worker 72 - jepsen.util 162 :ok :read [24 35] INFO [2019-12-17 12:27:51,118] jepsen worker 72 - jepsen.util 162 :invoke :read [24 nil] INFO [2019-12-17 12:27:51,130] jepsen worker 35 - jepsen.util 35 :fail :cas [11 [12 40]] :ValueNotSwapFrom INFO [2019-12-17 12:27:51,130] jepsen worker 35 - jepsen.util 35 :invoke :read [11 nil] INFO [2019-12-17 12:27:51,135] jepsen worker 62 - jepsen.util 152 :info :write [20 27] :RequestCanceledException INFO [2019-12-17 12:27:51,135] jepsen worker 34 - jepsen.util 124 :info :write [11 42] :RequestCanceledException INFO [2019-12-17 12:27:51,137] jepsen worker 62 - jepsen.util 242 :invoke :cas [20 [30 8]] INFO [2019-12-17 12:27:51,137] jepsen worker 34 - jepsen.util 214 :invoke :write [11 33] INFO [2019-12-17 12:27:51,140] jepsen worker 40 - jepsen.util 220 :info :write [13 3] :SyncWriteAmbiguous INFO [2019-12-17 12:27:51,141] jepsen worker 40 - jepsen.util 310 :invoke :cas [13 [20 43]] INFO [2019-12-17 12:27:51,148] jepsen worker 9 - jepsen.util 99 :fail :cas [3 [5 12]] :ValueNotSwapFrom INFO [2019-12-17 12:27:51,148] jepsen worker 9 - jepsen.util 99 :invoke :write [3 42] INFO [2019-12-17 12:27:51,161] jepsen worker 58 - jepsen.util 148 :ok :write [19 11] INFO [2019-12-17 12:27:51,161] jepsen worker 58 - jepsen.util 148 :invoke :write [19 14] INFO [2019-12-17 12:27:51,172] jepsen worker 4 - jepsen.util 184 :fail :cas [1 [5 23]] :ValueNotSwapFrom INFO [2019-12-17 12:27:51,173] jepsen worker 4 - jepsen.util 184 :invoke :write [1 42] INFO [2019-12-17 12:27:51,190] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1230,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:27:51,190] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1230,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:27:51,206] jepsen worker 59 - jepsen.util 239 :fail :cas [19 [15 22]] :ValueNotSwapFrom INFO [2019-12-17 12:27:51,206] jepsen worker 59 - jepsen.util 239 :invoke :write [19 21] INFO [2019-12-17 12:27:51,207] jepsen worker 56 - jepsen.util 146 :fail :cas [18 [30 14]] :ValueNotSwapFrom INFO [2019-12-17 12:27:51,208] jepsen worker 56 - jepsen.util 146 :invoke :write [18 32] INFO [2019-12-17 12:27:51,217] jepsen worker 35 - jepsen.util 35 :ok :read [11 7] INFO [2019-12-17 12:27:51,217] jepsen worker 35 - jepsen.util 35 :invoke :cas [11 [42 26]] INFO [2019-12-17 12:27:51,227] jepsen worker 76 - jepsen.util 166 :fail :cas [25 [43 8]] :ValueNotSwapFrom INFO [2019-12-17 12:27:51,227] jepsen worker 76 - jepsen.util 166 :invoke :write [25 38] INFO [2019-12-17 12:27:51,250] jepsen worker 54 - jepsen.util 324 :ok :read [18 47] INFO [2019-12-17 12:27:51,250] jepsen worker 54 - jepsen.util 324 :invoke :write [18 2] INFO [2019-12-17 12:27:51,267] jepsen worker 27 - jepsen.util 27 :ok :write [9 46] INFO [2019-12-17 12:27:51,267] jepsen worker 27 - jepsen.util 27 :invoke :cas [9 [6 45]] INFO [2019-12-17 12:27:51,289] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1237,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:27:51,289] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1237,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:27:51,289] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1241,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:27:51,289] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1241,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:27:51,291] jepsen worker 38 - jepsen.util 218 :ok :read [12 12] INFO [2019-12-17 12:27:51,291] jepsen worker 38 - jepsen.util 218 :invoke :read [12 nil] INFO [2019-12-17 12:27:51,296] jepsen worker 27 - jepsen.util 27 :fail :cas [9 [6 45]] :ValueNotSwapFrom INFO [2019-12-17 12:27:51,296] jepsen worker 27 - jepsen.util 27 :invoke :write [9 18] INFO [2019-12-17 12:27:51,319] jepsen worker 24 - jepsen.util 114 :ok :read [8 14] INFO [2019-12-17 12:27:51,319] jepsen worker 24 - jepsen.util 114 :invoke :write [8 28] INFO [2019-12-17 12:27:51,320] jepsen worker 78 - jepsen.util 168 :info :write [26 48] :RequestCanceledException INFO [2019-12-17 12:27:51,321] jepsen worker 68 - jepsen.util 158 :info :cas [22 [19 15]] :RequestCanceledException INFO [2019-12-17 12:27:51,321] jepsen worker 17 - jepsen.util 197 :info :cas [5 [21 37]] :RequestCanceledException INFO [2019-12-17 12:27:51,321] jepsen worker 85 - jepsen.util 175 :info :write [28 1] :RequestCanceledException INFO [2019-12-17 12:27:51,322] jepsen worker 78 - jepsen.util 258 :invoke :write [26 27] INFO [2019-12-17 12:27:51,322] jepsen worker 68 - jepsen.util 248 :invoke :cas [22 [28 6]] INFO [2019-12-17 12:27:51,322] jepsen worker 17 - jepsen.util 287 :invoke :cas [5 [7 1]] INFO [2019-12-17 12:27:51,322] jepsen worker 85 - jepsen.util 265 :invoke :write [28 44] INFO [2019-12-17 12:27:51,329] jepsen worker 62 - jepsen.util 242 :fail :cas [20 [30 8]] :ValueNotSwapFrom INFO [2019-12-17 12:27:51,329] jepsen worker 62 - jepsen.util 242 :invoke :write [20 42] INFO [2019-12-17 12:27:51,350] jepsen worker 10 - jepsen.util 100 :info :write [3 10] :RequestCanceledException INFO [2019-12-17 12:27:51,351] jepsen worker 10 - jepsen.util 190 :invoke :cas [3 [24 19]] INFO [2019-12-17 12:27:51,352] jepsen worker 5 - jepsen.util 5 :fail :cas [1 [22 2]] :ValueNotSwapFrom INFO [2019-12-17 12:27:51,352] jepsen worker 5 - jepsen.util 5 :invoke :cas [1 [26 7]] INFO [2019-12-17 12:27:51,361] jepsen worker 88 - jepsen.util 268 :fail :cas [29 [28 29]] :ValueNotSwapFrom INFO [2019-12-17 12:27:51,361] jepsen worker 88 - jepsen.util 268 :invoke :read [29 nil] INFO [2019-12-17 12:27:51,387] jepsen worker 35 - jepsen.util 35 :fail :cas [11 [42 26]] :ValueNotSwapFrom INFO [2019-12-17 12:27:51,388] jepsen worker 35 - jepsen.util 35 :invoke :cas [11 [23 34]] INFO [2019-12-17 12:27:51,405] jepsen worker 73 - jepsen.util 163 :info :write [24 36] :RequestCanceledException INFO [2019-12-17 12:27:51,405] jepsen worker 12 - jepsen.util 102 :info :write [4 28] :RequestCanceledException INFO [2019-12-17 12:27:51,407] jepsen worker 73 - jepsen.util 253 :invoke :read [24 nil] INFO [2019-12-17 12:27:51,407] jepsen worker 12 - jepsen.util 192 :invoke :cas [4 [38 18]] INFO [2019-12-17 12:27:51,411] jepsen worker 3 - jepsen.util 183 :fail :cas [1 [35 12]] :ValueNotSwapFrom INFO [2019-12-17 12:27:51,411] jepsen worker 3 - jepsen.util 183 :invoke :cas [1 [41 17]] INFO [2019-12-17 12:27:51,418] jepsen worker 22 - jepsen.util 202 :info :write [7 16] :RequestCanceledException INFO [2019-12-17 12:27:51,420] jepsen worker 22 - jepsen.util 292 :invoke :read [7 nil] INFO [2019-12-17 12:27:51,422] jepsen worker 84 - jepsen.util 84 :ok :read [28 1] INFO [2019-12-17 12:27:51,422] jepsen worker 84 - jepsen.util 84 :invoke :read [28 nil] INFO [2019-12-17 12:27:51,426] jepsen worker 35 - jepsen.util 35 :fail :cas [11 [23 34]] :ValueNotSwapFrom INFO [2019-12-17 12:27:51,426] jepsen worker 35 - jepsen.util 35 :invoke :cas [11 [35 10]] INFO [2019-12-17 12:27:51,434] jepsen worker 53 - jepsen.util 323 :ok :read [17 0] INFO [2019-12-17 12:27:51,434] jepsen worker 53 - jepsen.util 323 :invoke :read [17 nil] INFO [2019-12-17 12:27:51,436] jepsen worker 79 - jepsen.util 259 :ok :read [26 47] INFO [2019-12-17 12:27:51,436] jepsen worker 79 - jepsen.util 259 :invoke :read [26 nil] INFO [2019-12-17 12:27:51,445] jepsen worker 27 - jepsen.util 27 :ok :write [9 18] INFO [2019-12-17 12:27:51,445] jepsen worker 27 - jepsen.util 27 :invoke :cas [9 [30 10]] INFO [2019-12-17 12:27:51,452] jepsen worker 71 - jepsen.util 251 :info :write [23 30] :RequestCanceledException INFO [2019-12-17 12:27:51,453] jepsen worker 23 - jepsen.util 113 :fail :cas [7 [46 39]] :ValueNotSwapFrom INFO [2019-12-17 12:27:51,453] jepsen worker 23 - jepsen.util 113 :invoke :cas [7 [9 24]] INFO [2019-12-17 12:27:51,454] jepsen worker 71 - jepsen.util 341 :invoke :read [23 nil] INFO [2019-12-17 12:27:51,471] jepsen worker 37 - jepsen.util 127 :ok :read [12 12] INFO [2019-12-17 12:27:51,472] jepsen worker 37 - jepsen.util 127 :invoke :read [12 nil] INFO [2019-12-17 12:27:51,473] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.00390625}, :ns_1@172.28.128.126 {:progress 0.003597122302158251}, :ns_1@172.28.128.127 {:progress 0.007782101167315147}, :ns_1@172.28.128.128 {:progress 0.03018867924528301}} INFO [2019-12-17 12:27:51,473] jepsen nemesis - couchbase.util Rebalance check count: 3 INFO [2019-12-17 12:27:51,473] jepsen nemesis - couchbase.util Rebalance stuck count: 0 INFO [2019-12-17 12:27:51,484] jepsen worker 27 - jepsen.util 27 :fail :cas [9 [30 10]] :ValueNotSwapFrom INFO [2019-12-17 12:27:51,484] jepsen worker 27 - jepsen.util 27 :invoke :write [9 7] INFO [2019-12-17 12:27:51,486] jepsen worker 82 - jepsen.util 262 :info :write [27 27] :RequestCanceledException INFO [2019-12-17 12:27:51,488] jepsen worker 82 - jepsen.util 352 :invoke :write [27 14] INFO [2019-12-17 12:27:51,488] jepsen worker 86 - jepsen.util 86 :ok :read [28 1] INFO [2019-12-17 12:27:51,488] jepsen worker 86 - jepsen.util 86 :invoke :cas [28 [41 27]] INFO [2019-12-17 12:27:51,504] jepsen worker 15 - jepsen.util 285 :ok :read [5 32] INFO [2019-12-17 12:27:51,504] jepsen worker 15 - jepsen.util 285 :invoke :write [5 25] INFO [2019-12-17 12:27:51,505] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1267,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:27:51,505] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1267,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:27:51,512] jepsen worker 39 - jepsen.util 219 :fail :cas [13 [11 48]] :ValueNotSwapFrom INFO [2019-12-17 12:27:51,513] jepsen worker 39 - jepsen.util 219 :invoke :read [13 nil] INFO [2019-12-17 12:27:51,515] jepsen worker 52 - jepsen.util 232 :fail :cas [17 [15 37]] :ValueNotSwapFrom INFO [2019-12-17 12:27:51,515] jepsen worker 52 - jepsen.util 232 :invoke :read [17 nil] INFO [2019-12-17 12:27:51,519] jepsen worker 41 - jepsen.util 131 :info :write [13 37] :RequestCanceledException INFO [2019-12-17 12:27:51,519] jepsen worker 81 - jepsen.util 171 :info :write [27 46] :RequestCanceledException INFO [2019-12-17 12:27:51,521] jepsen worker 41 - jepsen.util 221 :invoke :write [13 43] INFO [2019-12-17 12:27:51,521] jepsen worker 81 - jepsen.util 261 :invoke :read [27 nil] INFO [2019-12-17 12:27:51,530] jepsen worker 68 - jepsen.util 248 :fail :cas [22 [28 6]] :ValueNotSwapFrom INFO [2019-12-17 12:27:51,530] jepsen worker 68 - jepsen.util 248 :invoke :read [22 nil] INFO [2019-12-17 12:27:51,552] jepsen worker 89 - jepsen.util 179 :info :write [29 23] :RequestCanceledException INFO [2019-12-17 12:27:51,553] jepsen worker 89 - jepsen.util 269 :invoke :cas [29 [32 1]] INFO [2019-12-17 12:27:51,556] jepsen worker 2 - jepsen.util 92 :ok :read [0 14] INFO [2019-12-17 12:27:51,556] jepsen worker 2 - jepsen.util 92 :invoke :read [0 nil] INFO [2019-12-17 12:27:51,568] jepsen worker 53 - jepsen.util 323 :ok :read [17 0] INFO [2019-12-17 12:27:51,568] jepsen worker 53 - jepsen.util 323 :invoke :write [17 22] INFO [2019-12-17 12:27:51,583] jepsen worker 35 - jepsen.util 35 :fail :cas [11 [35 10]] :ValueNotSwapFrom INFO [2019-12-17 12:27:51,583] jepsen worker 35 - jepsen.util 35 :invoke :read [11 nil] INFO [2019-12-17 12:27:51,601] jepsen worker 16 - jepsen.util 16 :info :write [5 32] :RequestCanceledException INFO [2019-12-17 12:27:51,602] jepsen worker 16 - jepsen.util 106 :invoke :write [5 23] INFO [2019-12-17 12:27:51,606] jepsen worker 84 - jepsen.util 84 :ok :read [28 1] INFO [2019-12-17 12:27:51,606] jepsen worker 84 - jepsen.util 84 :invoke :cas [28 [23 12]] INFO [2019-12-17 12:27:51,620] jepsen worker 66 - jepsen.util 156 :info :write [22 15] :RequestCanceledException INFO [2019-12-17 12:27:51,621] jepsen worker 57 - jepsen.util 237 :info :cas [19 [5 41]] :RequestCanceledException INFO [2019-12-17 12:27:51,622] jepsen worker 66 - jepsen.util 246 :invoke :read [22 nil] INFO [2019-12-17 12:27:51,622] jepsen worker 57 - jepsen.util 327 :invoke :read [19 nil] INFO [2019-12-17 12:27:51,637] jepsen worker 58 - jepsen.util 148 :ok :write [19 14] INFO [2019-12-17 12:27:51,637] jepsen worker 58 - jepsen.util 148 :invoke :read [19 nil] INFO [2019-12-17 12:27:51,647] jepsen worker 49 - jepsen.util 229 :fail :cas [16 [1 19]] :ValueNotSwapFrom INFO [2019-12-17 12:27:51,647] jepsen worker 49 - jepsen.util 229 :invoke :read [16 nil] INFO [2019-12-17 12:27:51,653] jepsen worker 77 - jepsen.util 257 :fail :cas [25 [45 27]] :ValueNotSwapFrom INFO [2019-12-17 12:27:51,653] jepsen worker 77 - jepsen.util 257 :invoke :cas [25 [19 41]] INFO [2019-12-17 12:27:51,670] jepsen worker 27 - jepsen.util 27 :ok :write [9 7] INFO [2019-12-17 12:27:51,671] jepsen worker 27 - jepsen.util 27 :invoke :read [9 nil] INFO [2019-12-17 12:27:51,687] jepsen worker 48 - jepsen.util 228 :info :write [16 17] :RequestCanceledException INFO [2019-12-17 12:27:51,688] jepsen worker 48 - jepsen.util 318 :invoke :cas [16 [42 7]] INFO [2019-12-17 12:27:51,690] jepsen worker 27 - jepsen.util 27 :ok :read [9 7] INFO [2019-12-17 12:27:51,690] jepsen worker 27 - jepsen.util 27 :invoke :cas [9 [19 43]] INFO [2019-12-17 12:27:51,690] jepsen worker 87 - jepsen.util 177 :fail :cas [29 [40 1]] :ValueNotSwapFrom INFO [2019-12-17 12:27:51,690] jepsen worker 87 - jepsen.util 177 :invoke :cas [29 [39 35]] INFO [2019-12-17 12:27:51,698] jepsen worker 35 - jepsen.util 35 :ok :read [11 7] INFO [2019-12-17 12:27:51,698] jepsen worker 35 - jepsen.util 35 :invoke :read [11 nil] INFO [2019-12-17 12:27:51,705] jepsen worker 83 - jepsen.util 263 :info :write [27 48] :RequestCanceledException INFO [2019-12-17 12:27:51,707] jepsen worker 83 - jepsen.util 353 :invoke :read [27 nil] INFO [2019-12-17 12:27:51,709] jepsen worker 1 - jepsen.util 271 :fail :cas [0 [2 25]] :ValueNotSwapFrom INFO [2019-12-17 12:27:51,709] jepsen worker 27 - jepsen.util 27 :fail :cas [9 [19 43]] :ValueNotSwapFrom INFO [2019-12-17 12:27:51,710] jepsen worker 1 - jepsen.util 271 :invoke :read [0 nil] INFO [2019-12-17 12:27:51,710] jepsen worker 27 - jepsen.util 27 :invoke :cas [9 [26 34]] INFO [2019-12-17 12:27:51,716] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1300,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:27:51,717] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1300,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:27:51,717] jepsen worker 70 - jepsen.util 250 :info :write [23 41] :RequestCanceledException INFO [2019-12-17 12:27:51,717] jepsen worker 51 - jepsen.util 141 :ok :read [17 0] INFO [2019-12-17 12:27:51,718] jepsen worker 51 - jepsen.util 141 :invoke :read [17 nil] INFO [2019-12-17 12:27:51,718] jepsen worker 70 - jepsen.util 340 :invoke :write [23 19] INFO [2019-12-17 12:27:51,726] jepsen worker 7 - jepsen.util 277 :info :cas [2 [44 26]] :RequestCanceledException INFO [2019-12-17 12:27:51,727] jepsen worker 7 - jepsen.util 367 :invoke :cas [2 [26 34]] INFO [2019-12-17 12:27:51,735] jepsen worker 5 - jepsen.util 5 :fail :cas [1 [26 7]] :ValueNotSwapFrom INFO [2019-12-17 12:27:51,735] jepsen worker 5 - jepsen.util 5 :invoke :read [1 nil] INFO [2019-12-17 12:27:51,741] jepsen worker 27 - jepsen.util 27 :fail :cas [9 [26 34]] :ValueNotSwapFrom INFO [2019-12-17 12:27:51,741] jepsen worker 27 - jepsen.util 27 :invoke :cas [9 [36 16]] INFO [2019-12-17 12:27:51,752] jepsen worker 40 - jepsen.util 310 :fail :cas [13 [20 43]] :ValueNotSwapFrom INFO [2019-12-17 12:27:51,752] jepsen worker 40 - jepsen.util 310 :invoke :write [13 17] INFO [2019-12-17 12:27:51,768] jepsen worker 86 - jepsen.util 86 :fail :cas [28 [41 27]] :ValueNotSwapFrom INFO [2019-12-17 12:27:51,768] jepsen worker 86 - jepsen.util 86 :invoke :read [28 nil] INFO [2019-12-17 12:27:51,773] jepsen worker 27 - jepsen.util 27 :fail :cas [9 [36 16]] :ValueNotSwapFrom INFO [2019-12-17 12:27:51,773] jepsen worker 27 - jepsen.util 27 :invoke :write [9 14] INFO [2019-12-17 12:27:51,787] jepsen worker 77 - jepsen.util 257 :fail :cas [25 [19 41]] :ValueNotSwapFrom INFO [2019-12-17 12:27:51,787] jepsen worker 77 - jepsen.util 257 :invoke :read [25 nil] INFO [2019-12-17 12:27:51,802] jepsen worker 35 - jepsen.util 35 :ok :read [11 7] INFO [2019-12-17 12:27:51,802] jepsen worker 35 - jepsen.util 35 :invoke :read [11 nil] INFO [2019-12-17 12:27:51,804] jepsen worker 13 - jepsen.util 193 :fail :cas [4 [49 45]] :ValueNotSwapFrom INFO [2019-12-17 12:27:51,804] jepsen worker 13 - jepsen.util 193 :invoke :write [4 36] INFO [2019-12-17 12:27:51,805] jepsen worker 45 - jepsen.util 225 :info :write [15 42] :RequestCanceledException INFO [2019-12-17 12:27:51,805] jepsen worker 31 - jepsen.util 301 :info :write [10 43] :RequestCanceledException INFO [2019-12-17 12:27:51,806] jepsen worker 45 - jepsen.util 315 :invoke :cas [15 [27 28]] INFO [2019-12-17 12:27:51,806] jepsen worker 31 - jepsen.util 391 :invoke :write [10 9] INFO [2019-12-17 12:27:51,810] jepsen worker 39 - jepsen.util 219 :ok :read [13 9] INFO [2019-12-17 12:27:51,810] jepsen worker 39 - jepsen.util 219 :invoke :read [13 nil] INFO [2019-12-17 12:27:51,827] jepsen worker 45 - jepsen.util 315 :fail :cas [15 [27 28]] :ValueNotSwapFrom INFO [2019-12-17 12:27:51,827] jepsen worker 45 - jepsen.util 315 :invoke :write [15 6] INFO [2019-12-17 12:27:51,833] jepsen worker 81 - jepsen.util 261 :ok :read [27 18] INFO [2019-12-17 12:27:51,833] jepsen worker 81 - jepsen.util 261 :invoke :write [27 7] INFO [2019-12-17 12:27:51,851] jepsen worker 46 - jepsen.util 226 :info :write [15 2] :RequestCanceledException INFO [2019-12-17 12:27:51,853] jepsen worker 46 - jepsen.util 316 :invoke :cas [15 [16 30]] INFO [2019-12-17 12:27:51,873] jepsen worker 35 - jepsen.util 35 :ok :read [11 7] INFO [2019-12-17 12:27:51,873] jepsen worker 35 - jepsen.util 35 :invoke :write [11 3] INFO [2019-12-17 12:27:51,903] jepsen worker 2 - jepsen.util 92 :ok :read [0 14] INFO [2019-12-17 12:27:51,904] jepsen worker 2 - jepsen.util 92 :invoke :write [0 5] INFO [2019-12-17 12:27:51,921] jepsen worker 80 - jepsen.util 350 :info :write [26 43] :RequestCanceledException INFO [2019-12-17 12:27:51,923] jepsen worker 80 - jepsen.util 440 :invoke :read [26 nil] INFO [2019-12-17 12:27:51,934] jepsen worker 75 - jepsen.util 255 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:27:51,934] jepsen worker 26 - jepsen.util 296 :info :cas [8 [47 9]] :RequestCanceledException INFO [2019-12-17 12:27:51,934] jepsen worker 75 - jepsen.util 255 :invoke :cas [25 [43 45]] INFO [2019-12-17 12:27:51,935] jepsen worker 26 - jepsen.util 386 :invoke :write [8 33] INFO [2019-12-17 12:27:51,937] jepsen worker 49 - jepsen.util 229 :ok :read [16 8] INFO [2019-12-17 12:27:51,937] jepsen worker 49 - jepsen.util 229 :invoke :write [16 5] INFO [2019-12-17 12:27:51,946] jepsen worker 20 - jepsen.util 290 :ok :read [6 45] INFO [2019-12-17 12:27:51,947] jepsen worker 20 - jepsen.util 290 :invoke :read [6 nil] INFO [2019-12-17 12:27:51,956] jepsen worker 88 - jepsen.util 268 :ok :read [29 5] INFO [2019-12-17 12:27:51,956] jepsen worker 88 - jepsen.util 268 :invoke :cas [29 [23 9]] INFO [2019-12-17 12:27:51,956] jepsen worker 83 - jepsen.util 353 :ok :read [27 18] INFO [2019-12-17 12:27:51,957] jepsen worker 83 - jepsen.util 353 :invoke :write [27 33] INFO [2019-12-17 12:27:51,972] jepsen worker 1 - jepsen.util 271 :ok :read [0 14] INFO [2019-12-17 12:27:51,972] jepsen worker 1 - jepsen.util 271 :invoke :cas [0 [46 12]] INFO [2019-12-17 12:27:51,984] jepsen worker 59 - jepsen.util 239 :ok :write [19 21] INFO [2019-12-17 12:27:51,984] jepsen worker 59 - jepsen.util 239 :invoke :read [19 nil] INFO [2019-12-17 12:27:51,987] jepsen worker 14 - jepsen.util 194 :info :write [4 3] :RequestCanceledException INFO [2019-12-17 12:27:51,988] jepsen worker 14 - jepsen.util 284 :invoke :read [4 nil] INFO [2019-12-17 12:27:52,004] jepsen worker 55 - jepsen.util 235 :info :write [18 15] :RequestCanceledException INFO [2019-12-17 12:27:52,005] jepsen worker 12 - jepsen.util 192 :fail :cas [4 [38 18]] :ValueNotSwapFrom INFO [2019-12-17 12:27:52,005] jepsen worker 12 - jepsen.util 192 :invoke :read [4 nil] INFO [2019-12-17 12:27:52,005] jepsen worker 55 - jepsen.util 325 :invoke :write [18 12] INFO [2019-12-17 12:27:52,018] jepsen worker 20 - jepsen.util 290 :ok :read [6 45] INFO [2019-12-17 12:27:52,019] jepsen worker 59 - jepsen.util 239 :ok :read [19 21] INFO [2019-12-17 12:27:52,019] jepsen worker 20 - jepsen.util 290 :invoke :cas [6 [1 14]] INFO [2019-12-17 12:27:52,019] jepsen worker 59 - jepsen.util 239 :invoke :read [19 nil] INFO [2019-12-17 12:27:52,020] jepsen worker 47 - jepsen.util 227 :info :write [15 24] :RequestCanceledException INFO [2019-12-17 12:27:52,020] jepsen worker 21 - jepsen.util 201 :info :write [7 34] :RequestCanceledException INFO [2019-12-17 12:27:52,020] jepsen worker 33 - jepsen.util 213 :info :write [11 42] :RequestCanceledException INFO [2019-12-17 12:27:52,021] jepsen worker 21 - jepsen.util 291 :invoke :read [7 nil] INFO [2019-12-17 12:27:52,021] jepsen worker 47 - jepsen.util 317 :invoke :read [15 nil] INFO [2019-12-17 12:27:52,021] jepsen worker 33 - jepsen.util 303 :invoke :write [11 22] INFO [2019-12-17 12:27:52,068] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1346,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:27:52,068] jepsen worker 37 - jepsen.util 127 :ok :read [12 12] INFO [2019-12-17 12:27:52,069] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1346,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:27:52,069] jepsen worker 37 - jepsen.util 127 :invoke :cas [12 [32 15]] INFO [2019-12-17 12:27:52,069] jepsen worker 85 - jepsen.util 265 :ok :write [28 44] INFO [2019-12-17 12:27:52,069] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1350,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:27:52,069] jepsen worker 85 - jepsen.util 265 :invoke :read [28 nil] INFO [2019-12-17 12:27:52,069] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1350,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:27:52,076] jepsen worker 20 - jepsen.util 290 :fail :cas [6 [1 14]] :ValueNotSwapFrom INFO [2019-12-17 12:27:52,076] jepsen worker 20 - jepsen.util 290 :invoke :write [6 38] INFO [2019-12-17 12:27:52,085] jepsen worker 86 - jepsen.util 86 :ok :read [28 44] INFO [2019-12-17 12:27:52,085] jepsen worker 86 - jepsen.util 86 :invoke :read [28 nil] INFO [2019-12-17 12:27:52,106] jepsen worker 28 - jepsen.util 118 :info :write [9 36] :RequestCanceledException INFO [2019-12-17 12:27:52,106] jepsen worker 8 - jepsen.util 188 :info :write [2 18] :RequestCanceledException INFO [2019-12-17 12:27:52,107] jepsen worker 59 - jepsen.util 239 :ok :read [19 21] INFO [2019-12-17 12:27:52,107] jepsen worker 59 - jepsen.util 239 :invoke :read [19 nil] INFO [2019-12-17 12:27:52,107] jepsen worker 28 - jepsen.util 208 :invoke :write [9 28] INFO [2019-12-17 12:27:52,107] jepsen worker 8 - jepsen.util 278 :invoke :cas [2 [6 45]] INFO [2019-12-17 12:27:52,118] jepsen worker 32 - jepsen.util 122 :info :write [10 16] :SyncWriteAmbiguous INFO [2019-12-17 12:27:52,118] jepsen worker 60 - jepsen.util 330 :info :write [20 3] :RequestCanceledException INFO [2019-12-17 12:27:52,119] jepsen worker 32 - jepsen.util 212 :invoke :cas [10 [39 17]] INFO [2019-12-17 12:27:52,119] jepsen worker 60 - jepsen.util 420 :invoke :read [20 nil] INFO [2019-12-17 12:27:52,128] jepsen worker 3 - jepsen.util 183 :fail :cas [1 [41 17]] :ValueNotSwapFrom INFO [2019-12-17 12:27:52,128] jepsen worker 3 - jepsen.util 183 :invoke :write [1 42] INFO [2019-12-17 12:27:52,132] jepsen worker 36 - jepsen.util 216 :info :cas [12 [12 25]] :RequestCanceledException INFO [2019-12-17 12:27:52,134] jepsen worker 36 - jepsen.util 306 :invoke :cas [12 [22 38]] INFO [2019-12-17 12:27:52,136] jepsen worker 39 - jepsen.util 219 :ok :read [13 9] INFO [2019-12-17 12:27:52,136] jepsen worker 39 - jepsen.util 219 :invoke :write [13 37] INFO [2019-12-17 12:27:52,141] jepsen worker 59 - jepsen.util 239 :ok :read [19 21] INFO [2019-12-17 12:27:52,141] jepsen worker 59 - jepsen.util 239 :invoke :cas [19 [36 19]] INFO [2019-12-17 12:27:52,161] jepsen worker 59 - jepsen.util 239 :fail :cas [19 [36 19]] :ValueNotSwapFrom INFO [2019-12-17 12:27:52,161] jepsen worker 59 - jepsen.util 239 :invoke :write [19 35] INFO [2019-12-17 12:27:52,164] jepsen worker 58 - jepsen.util 148 :ok :read [19 21] INFO [2019-12-17 12:27:52,164] jepsen worker 58 - jepsen.util 148 :invoke :read [19 nil] INFO [2019-12-17 12:27:52,169] jepsen worker 32 - jepsen.util 212 :fail :cas [10 [39 17]] :ValueNotSwapFrom INFO [2019-12-17 12:27:52,169] jepsen worker 32 - jepsen.util 212 :invoke :read [10 nil] INFO [2019-12-17 12:27:52,176] jepsen worker 10 - jepsen.util 190 :fail :cas [3 [24 19]] :ValueNotSwapFrom INFO [2019-12-17 12:27:52,176] jepsen worker 10 - jepsen.util 190 :invoke :cas [3 [44 14]] INFO [2019-12-17 12:27:52,181] jepsen worker 27 - jepsen.util 27 :ok :write [9 14] INFO [2019-12-17 12:27:52,181] jepsen worker 27 - jepsen.util 27 :invoke :cas [9 [41 40]] INFO [2019-12-17 12:27:52,192] jepsen worker 46 - jepsen.util 316 :fail :cas [15 [16 30]] :ValueNotSwapFrom INFO [2019-12-17 12:27:52,192] jepsen worker 46 - jepsen.util 316 :invoke :read [15 nil] INFO [2019-12-17 12:27:52,206] jepsen worker 64 - jepsen.util 244 :info :write [21 42] :RequestCanceledException INFO [2019-12-17 12:27:52,207] jepsen worker 64 - jepsen.util 334 :invoke :write [21 47] INFO [2019-12-17 12:27:52,220] jepsen worker 61 - jepsen.util 241 :info :write [20 49] :RequestCanceledException INFO [2019-12-17 12:27:52,220] jepsen worker 42 - jepsen.util 312 :info :write [14 13] :RequestCanceledException INFO [2019-12-17 12:27:52,222] jepsen worker 61 - jepsen.util 331 :invoke :cas [20 [7 13]] INFO [2019-12-17 12:27:52,222] jepsen worker 42 - jepsen.util 402 :invoke :write [14 40] INFO [2019-12-17 12:27:52,227] jepsen worker 51 - jepsen.util 141 :ok :read [17 0] INFO [2019-12-17 12:27:52,227] jepsen worker 51 - jepsen.util 141 :invoke :cas [17 [42 32]] INFO [2019-12-17 12:27:52,248] jepsen worker 25 - jepsen.util 295 :info :write [8 7] :RequestCanceledException INFO [2019-12-17 12:27:52,249] jepsen worker 25 - jepsen.util 385 :invoke :cas [8 [15 27]] INFO [2019-12-17 12:27:52,251] jepsen worker 57 - jepsen.util 327 :ok :read [19 21] INFO [2019-12-17 12:27:52,251] jepsen worker 57 - jepsen.util 327 :invoke :read [19 nil] INFO [2019-12-17 12:27:52,253] jepsen worker 27 - jepsen.util 27 :fail :cas [9 [41 40]] :ValueNotSwapFrom INFO [2019-12-17 12:27:52,253] jepsen worker 27 - jepsen.util 27 :invoke :read [9 nil] INFO [2019-12-17 12:27:52,254] jepsen worker 47 - jepsen.util 317 :ok :read [15 35] INFO [2019-12-17 12:27:52,254] jepsen worker 47 - jepsen.util 317 :invoke :write [15 32] INFO [2019-12-17 12:27:52,259] jepsen worker 89 - jepsen.util 269 :fail :cas [29 [32 1]] :ValueNotSwapFrom INFO [2019-12-17 12:27:52,260] jepsen worker 89 - jepsen.util 269 :invoke :read [29 nil] INFO [2019-12-17 12:27:52,288] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1378,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:27:52,288] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1389,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:27:52,288] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1378,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:27:52,288] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1389,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:27:52,291] jepsen worker 32 - jepsen.util 212 :ok :read [10 23] INFO [2019-12-17 12:27:52,291] jepsen worker 32 - jepsen.util 212 :invoke :write [10 47] INFO [2019-12-17 12:27:52,300] jepsen worker 23 - jepsen.util 113 :fail :cas [7 [9 24]] :ValueNotSwapFrom INFO [2019-12-17 12:27:52,300] jepsen worker 23 - jepsen.util 113 :invoke :cas [7 [19 11]] INFO [2019-12-17 12:27:52,316] jepsen worker 10 - jepsen.util 190 :fail :cas [3 [44 14]] :ValueNotSwapFrom INFO [2019-12-17 12:27:52,317] jepsen worker 10 - jepsen.util 190 :invoke :write [3 12] INFO [2019-12-17 12:27:52,320] jepsen worker 63 - jepsen.util 153 :fail :read [21 nil] :RequestTimeoutException INFO [2019-12-17 12:27:52,320] jepsen worker 63 - jepsen.util 153 :invoke :write [21 42] INFO [2019-12-17 12:27:52,335] jepsen worker 18 - jepsen.util 198 :info :write [6 27] :RequestCanceledException INFO [2019-12-17 12:27:52,336] jepsen worker 18 - jepsen.util 288 :invoke :write [6 12] INFO [2019-12-17 12:27:52,348] jepsen worker 65 - jepsen.util 245 :info :cas [21 [19 12]] :RequestCanceledException INFO [2019-12-17 12:27:52,349] jepsen worker 65 - jepsen.util 335 :invoke :read [21 nil] INFO [2019-12-17 12:27:52,354] jepsen worker 25 - jepsen.util 385 :fail :cas [8 [15 27]] :ValueNotSwapFrom INFO [2019-12-17 12:27:52,355] jepsen worker 25 - jepsen.util 385 :invoke :cas [8 [21 45]] INFO [2019-12-17 12:27:52,374] jepsen worker 11 - jepsen.util 191 :info :write [3 21] :SyncWriteAmbiguous INFO [2019-12-17 12:27:52,376] jepsen worker 11 - jepsen.util 281 :invoke :read [3 nil] INFO [2019-12-17 12:27:52,386] jepsen worker 27 - jepsen.util 27 :ok :read [9 14] INFO [2019-12-17 12:27:52,387] jepsen worker 27 - jepsen.util 27 :invoke :cas [9 [25 37]] INFO [2019-12-17 12:27:52,394] jepsen worker 36 - jepsen.util 306 :fail :cas [12 [22 38]] :ValueNotSwapFrom INFO [2019-12-17 12:27:52,395] jepsen worker 36 - jepsen.util 306 :invoke :cas [12 [10 22]] INFO [2019-12-17 12:27:52,405] jepsen worker 88 - jepsen.util 268 :fail :cas [29 [23 9]] :ValueNotSwapFrom INFO [2019-12-17 12:27:52,405] jepsen worker 88 - jepsen.util 268 :invoke :write [29 40] INFO [2019-12-17 12:27:52,422] jepsen worker 46 - jepsen.util 316 :ok :read [15 35] INFO [2019-12-17 12:27:52,422] jepsen worker 46 - jepsen.util 316 :invoke :read [15 nil] INFO [2019-12-17 12:27:52,468] jepsen worker 73 - jepsen.util 253 :ok :read [24 35] INFO [2019-12-17 12:27:52,468] jepsen worker 73 - jepsen.util 253 :invoke :write [24 39] INFO [2019-12-17 12:27:52,497] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.01171875}, :ns_1@172.28.128.126 {:progress 0.01079136690647486}, :ns_1@172.28.128.127 {:progress 0.0311284046692607}, :ns_1@172.28.128.128 {:progress 0.03773584905660377}} INFO [2019-12-17 12:27:52,497] jepsen nemesis - couchbase.util Rebalance check count: 4 INFO [2019-12-17 12:27:52,497] jepsen nemesis - couchbase.util Rebalance stuck count: 0 INFO [2019-12-17 12:27:52,500] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1412,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:27:52,500] jepsen worker 14 - jepsen.util 284 :ok :read [4 4] INFO [2019-12-17 12:27:52,500] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1412,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:27:52,500] jepsen worker 14 - jepsen.util 284 :invoke :read [4 nil] INFO [2019-12-17 12:27:52,500] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1415,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:27:52,500] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1415,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:27:52,516] jepsen worker 64 - jepsen.util 334 :ok :write [21 47] INFO [2019-12-17 12:27:52,516] jepsen worker 64 - jepsen.util 334 :invoke :read [21 nil] INFO [2019-12-17 12:27:52,519] jepsen worker 59 - jepsen.util 239 :ok :write [19 35] INFO [2019-12-17 12:27:52,520] jepsen worker 59 - jepsen.util 239 :invoke :read [19 nil] INFO [2019-12-17 12:27:52,532] jepsen worker 36 - jepsen.util 306 :fail :cas [12 [10 22]] :ValueNotSwapFrom INFO [2019-12-17 12:27:52,532] jepsen worker 36 - jepsen.util 306 :invoke :write [12 5] INFO [2019-12-17 12:27:52,545] jepsen worker 59 - jepsen.util 239 :ok :read [19 35] INFO [2019-12-17 12:27:52,545] jepsen worker 59 - jepsen.util 239 :invoke :cas [19 [35 8]] INFO [2019-12-17 12:27:52,549] jepsen worker 38 - jepsen.util 218 :ok :read [12 12] INFO [2019-12-17 12:27:52,549] jepsen worker 75 - jepsen.util 255 :fail :cas [25 [43 45]] :ValueNotSwapFrom INFO [2019-12-17 12:27:52,549] jepsen worker 38 - jepsen.util 218 :invoke :read [12 nil] INFO [2019-12-17 12:27:52,549] jepsen worker 75 - jepsen.util 255 :invoke :write [25 13] INFO [2019-12-17 12:27:52,555] jepsen worker 12 - jepsen.util 192 :ok :read [4 4] INFO [2019-12-17 12:27:52,556] jepsen worker 12 - jepsen.util 192 :invoke :write [4 18] INFO [2019-12-17 12:27:52,576] jepsen worker 37 - jepsen.util 127 :fail :cas [12 [32 15]] :ValueNotSwapFrom INFO [2019-12-17 12:27:52,577] jepsen worker 37 - jepsen.util 127 :invoke :write [12 36] INFO [2019-12-17 12:27:52,608] jepsen worker 80 - jepsen.util 440 :ok :read [26 47] INFO [2019-12-17 12:27:52,609] jepsen worker 80 - jepsen.util 440 :invoke :write [26 13] INFO [2019-12-17 12:27:52,621] jepsen worker 0 - jepsen.util 180 :info :cas [0 [47 40]] :RequestCanceledException INFO [2019-12-17 12:27:52,622] jepsen worker 0 - jepsen.util 270 :invoke :cas [0 [40 4]] INFO [2019-12-17 12:27:52,655] jepsen worker 51 - jepsen.util 141 :fail :cas [17 [42 32]] :ValueNotSwapFrom INFO [2019-12-17 12:27:52,656] jepsen worker 51 - jepsen.util 141 :invoke :cas [17 [19 1]] INFO [2019-12-17 12:27:52,661] jepsen worker 59 - jepsen.util 239 :ok :cas [19 [35 8]] INFO [2019-12-17 12:27:52,662] jepsen worker 59 - jepsen.util 239 :invoke :cas [19 [7 4]] INFO [2019-12-17 12:27:52,678] jepsen worker 0 - jepsen.util 270 :fail :cas [0 [40 4]] :ValueNotSwapFrom INFO [2019-12-17 12:27:52,678] jepsen worker 0 - jepsen.util 270 :invoke :read [0 nil] INFO [2019-12-17 12:27:52,690] jepsen worker 59 - jepsen.util 239 :fail :cas [19 [7 4]] :ValueNotSwapFrom INFO [2019-12-17 12:27:52,691] jepsen worker 59 - jepsen.util 239 :invoke :cas [19 [6 5]] INFO [2019-12-17 12:27:52,695] jepsen worker 52 - jepsen.util 232 :ok :read [17 0] INFO [2019-12-17 12:27:52,695] jepsen worker 52 - jepsen.util 232 :invoke :write [17 23] INFO [2019-12-17 12:27:52,716] jepsen worker 0 - jepsen.util 270 :ok :read [0 14] INFO [2019-12-17 12:27:52,716] jepsen worker 0 - jepsen.util 270 :invoke :write [0 9] INFO [2019-12-17 12:27:52,716] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1433,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:27:52,717] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1433,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:27:52,717] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1436,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:27:52,717] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1436,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:27:52,717] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1437,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:27:52,717] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1437,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:27:52,731] jepsen worker 59 - jepsen.util 239 :fail :cas [19 [6 5]] :ValueNotSwapFrom INFO [2019-12-17 12:27:52,731] jepsen worker 59 - jepsen.util 239 :invoke :read [19 nil] INFO [2019-12-17 12:27:52,778] jepsen worker 67 - jepsen.util 247 :info :cas [22 [33 35]] :RequestCanceledException INFO [2019-12-17 12:27:52,780] jepsen worker 67 - jepsen.util 337 :invoke :write [22 34] INFO [2019-12-17 12:27:52,803] jepsen worker 69 - jepsen.util 249 :info :cas [23 [29 44]] :RequestCanceledException INFO [2019-12-17 12:27:52,805] jepsen worker 69 - jepsen.util 339 :invoke :cas [23 [12 12]] INFO [2019-12-17 12:27:52,831] jepsen worker 21 - jepsen.util 291 :ok :read [7 20] INFO [2019-12-17 12:27:52,831] jepsen worker 21 - jepsen.util 291 :invoke :read [7 nil] INFO [2019-12-17 12:27:52,848] jepsen worker 43 - jepsen.util 313 :info :write [14 36] :RequestCanceledException INFO [2019-12-17 12:27:52,849] jepsen worker 23 - jepsen.util 113 :fail :cas [7 [19 11]] :ValueNotSwapFrom INFO [2019-12-17 12:27:52,849] jepsen worker 23 - jepsen.util 113 :invoke :read [7 nil] INFO [2019-12-17 12:27:52,849] jepsen worker 43 - jepsen.util 403 :invoke :write [14 1] INFO [2019-12-17 12:27:52,851] jepsen worker 63 - jepsen.util 153 :ok :write [21 42] INFO [2019-12-17 12:27:52,852] jepsen worker 63 - jepsen.util 153 :invoke :read [21 nil] INFO [2019-12-17 12:27:52,897] jepsen worker 63 - jepsen.util 153 :ok :read [21 42] INFO [2019-12-17 12:27:52,897] jepsen worker 63 - jepsen.util 153 :invoke :cas [21 [24 45]] INFO [2019-12-17 12:27:52,906] jepsen worker 60 - jepsen.util 420 :ok :read [20 7] INFO [2019-12-17 12:27:52,907] jepsen worker 60 - jepsen.util 420 :invoke :write [20 22] INFO [2019-12-17 12:27:52,924] jepsen worker 51 - jepsen.util 141 :fail :cas [17 [19 1]] :ValueNotSwapFrom INFO [2019-12-17 12:27:52,924] jepsen worker 51 - jepsen.util 141 :invoke :cas [17 [20 46]] INFO [2019-12-17 12:27:52,929] jepsen worker 63 - jepsen.util 153 :fail :cas [21 [24 45]] :ValueNotSwapFrom INFO [2019-12-17 12:27:52,929] jepsen worker 63 - jepsen.util 153 :invoke :cas [21 [36 40]] INFO [2019-12-17 12:27:52,932] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1449,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:27:52,933] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1449,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:27:52,949] jepsen worker 63 - jepsen.util 153 :fail :cas [21 [36 40]] :ValueNotSwapFrom INFO [2019-12-17 12:27:52,949] jepsen worker 63 - jepsen.util 153 :invoke :cas [21 [0 13]] INFO [2019-12-17 12:27:52,966] jepsen worker 63 - jepsen.util 153 :fail :cas [21 [0 13]] :ValueNotSwapFrom INFO [2019-12-17 12:27:52,967] jepsen worker 63 - jepsen.util 153 :invoke :read [21 nil] INFO [2019-12-17 12:27:52,986] jepsen worker 63 - jepsen.util 153 :ok :read [21 42] INFO [2019-12-17 12:27:52,986] jepsen worker 63 - jepsen.util 153 :invoke :cas [21 [16 19]] INFO [2019-12-17 12:27:53,008] jepsen worker 63 - jepsen.util 153 :fail :cas [21 [16 19]] :ValueNotSwapFrom INFO [2019-12-17 12:27:53,008] jepsen worker 63 - jepsen.util 153 :invoke :cas [21 [48 6]] INFO [2019-12-17 12:27:53,021] jepsen worker 44 - jepsen.util 134 :info :write [14 48] :RequestCanceledException INFO [2019-12-17 12:27:53,021] jepsen worker 6 - jepsen.util 186 :info :write [2 36] :RequestCanceledException INFO [2019-12-17 12:27:53,023] jepsen worker 6 - jepsen.util 276 :invoke :cas [2 [3 34]] INFO [2019-12-17 12:27:53,023] jepsen worker 44 - jepsen.util 224 :invoke :cas [14 [21 11]] INFO [2019-12-17 12:27:53,039] jepsen worker 63 - jepsen.util 153 :fail :cas [21 [48 6]] :ValueNotSwapFrom INFO [2019-12-17 12:27:53,039] jepsen worker 63 - jepsen.util 153 :invoke :cas [21 [4 25]] INFO [2019-12-17 12:27:53,042] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1465,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:27:53,042] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1465,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:27:53,051] jepsen worker 19 - jepsen.util 199 :fail :read [6 nil] :RequestTimeoutException INFO [2019-12-17 12:27:53,051] jepsen worker 19 - jepsen.util 199 :invoke :write [6 14] INFO [2019-12-17 12:27:53,066] jepsen worker 63 - jepsen.util 153 :fail :cas [21 [4 25]] :ValueNotSwapFrom INFO [2019-12-17 12:27:53,066] jepsen worker 63 - jepsen.util 153 :invoke :write [21 22] INFO [2019-12-17 12:27:53,086] jepsen worker 50 - jepsen.util 320 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:27:53,086] jepsen worker 30 - jepsen.util 30 :info :write [10 11] :RequestCanceledException INFO [2019-12-17 12:27:53,086] jepsen worker 29 - jepsen.util 119 :info :write [9 36] :RequestCanceledException INFO [2019-12-17 12:27:53,086] jepsen worker 50 - jepsen.util 320 :invoke :write [16 0] INFO [2019-12-17 12:27:53,087] jepsen worker 30 - jepsen.util 120 :invoke :cas [10 [28 29]] INFO [2019-12-17 12:27:53,087] jepsen worker 29 - jepsen.util 209 :invoke :write [9 39] INFO [2019-12-17 12:27:53,125] jepsen worker 21 - jepsen.util 291 :ok :read [7 20] INFO [2019-12-17 12:27:53,125] jepsen worker 21 - jepsen.util 291 :invoke :read [7 nil] INFO [2019-12-17 12:27:53,154] jepsen worker 22 - jepsen.util 292 :ok :read [7 20] INFO [2019-12-17 12:27:53,154] jepsen worker 22 - jepsen.util 292 :invoke :read [7 nil] INFO [2019-12-17 12:27:53,177] jepsen worker 63 - jepsen.util 153 :ok :write [21 22] INFO [2019-12-17 12:27:53,178] jepsen worker 63 - jepsen.util 153 :invoke :read [21 nil] INFO [2019-12-17 12:27:53,181] jepsen worker 14 - jepsen.util 284 :ok :read [4 4] INFO [2019-12-17 12:27:53,181] jepsen worker 14 - jepsen.util 284 :invoke :cas [4 [3 26]] INFO [2019-12-17 12:27:53,186] jepsen worker 72 - jepsen.util 162 :fail :read [24 nil] :RequestTimeoutException INFO [2019-12-17 12:27:53,186] jepsen worker 74 - jepsen.util 254 :info :cas [24 [2 8]] :RequestCanceledException INFO [2019-12-17 12:27:53,186] jepsen worker 9 - jepsen.util 99 :info :write [3 42] :RequestCanceledException INFO [2019-12-17 12:27:53,186] jepsen worker 72 - jepsen.util 162 :invoke :read [24 nil] INFO [2019-12-17 12:27:53,187] jepsen worker 9 - jepsen.util 189 :invoke :read [3 nil] INFO [2019-12-17 12:27:53,187] jepsen worker 74 - jepsen.util 344 :invoke :read [24 nil] INFO [2019-12-17 12:27:53,198] jepsen worker 63 - jepsen.util 153 :ok :read [21 22] INFO [2019-12-17 12:27:53,199] jepsen worker 63 - jepsen.util 153 :invoke :write [21 17] INFO [2019-12-17 12:27:53,216] jepsen worker 34 - jepsen.util 214 :info :write [11 33] :RequestCanceledException INFO [2019-12-17 12:27:53,217] jepsen worker 34 - jepsen.util 304 :invoke :cas [11 [35 16]] INFO [2019-12-17 12:27:53,249] jepsen worker 63 - jepsen.util 153 :ok :write [21 17] INFO [2019-12-17 12:27:53,249] jepsen worker 4 - jepsen.util 184 :info :write [1 42] :RequestCanceledException INFO [2019-12-17 12:27:53,249] jepsen worker 63 - jepsen.util 153 :invoke :read [21 nil] INFO [2019-12-17 12:27:53,250] jepsen worker 4 - jepsen.util 274 :invoke :cas [1 [3 28]] INFO [2019-12-17 12:27:53,251] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1487,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:27:53,251] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1487,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:27:53,251] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1488,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:27:53,251] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1488,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:27:53,261] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1494,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:27:53,261] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1494,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:27:53,262] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1497,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:27:53,262] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1497,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:27:53,262] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1498,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:27:53,262] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1498,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:27:53,279] jepsen worker 63 - jepsen.util 153 :ok :read [21 17] INFO [2019-12-17 12:27:53,279] jepsen worker 63 - jepsen.util 153 :invoke :read [21 nil] INFO [2019-12-17 12:27:53,287] jepsen worker 56 - jepsen.util 146 :info :write [18 32] :RequestCanceledException INFO [2019-12-17 12:27:53,289] jepsen worker 56 - jepsen.util 236 :invoke :write [18 39] INFO [2019-12-17 12:27:53,299] jepsen worker 63 - jepsen.util 153 :ok :read [21 17] INFO [2019-12-17 12:27:53,299] jepsen worker 63 - jepsen.util 153 :invoke :write [21 49] INFO [2019-12-17 12:27:53,306] jepsen worker 87 - jepsen.util 177 :fail :cas [29 [39 35]] :ValueNotSwapFrom INFO [2019-12-17 12:27:53,306] jepsen worker 87 - jepsen.util 177 :invoke :cas [29 [11 47]] INFO [2019-12-17 12:27:53,321] jepsen worker 76 - jepsen.util 166 :info :write [25 38] :RequestCanceledException INFO [2019-12-17 12:27:53,322] jepsen worker 76 - jepsen.util 256 :invoke :read [25 nil] INFO [2019-12-17 12:27:53,332] jepsen worker 6 - jepsen.util 276 :fail :cas [2 [3 34]] :ValueNotSwapFrom INFO [2019-12-17 12:27:53,332] jepsen worker 6 - jepsen.util 276 :invoke :cas [2 [24 23]] INFO [2019-12-17 12:27:53,352] jepsen worker 54 - jepsen.util 324 :info :write [18 2] :RequestCanceledException INFO [2019-12-17 12:27:53,352] jepsen worker 78 - jepsen.util 258 :info :write [26 27] :RequestCanceledException INFO [2019-12-17 12:27:53,353] jepsen worker 54 - jepsen.util 414 :invoke :write [18 15] INFO [2019-12-17 12:27:53,353] jepsen worker 78 - jepsen.util 348 :invoke :read [26 nil] INFO [2019-12-17 12:27:53,376] jepsen worker 14 - jepsen.util 284 :fail :cas [4 [3 26]] :ValueNotSwapFrom INFO [2019-12-17 12:27:53,376] jepsen worker 14 - jepsen.util 284 :invoke :write [4 19] INFO [2019-12-17 12:27:53,378] jepsen worker 63 - jepsen.util 153 :ok :write [21 49] INFO [2019-12-17 12:27:53,378] jepsen worker 63 - jepsen.util 153 :invoke :read [21 nil] INFO [2019-12-17 12:27:53,385] jepsen worker 17 - jepsen.util 287 :info :cas [5 [7 1]] :RequestCanceledException INFO [2019-12-17 12:27:53,386] jepsen worker 17 - jepsen.util 377 :invoke :read [5 nil] INFO [2019-12-17 12:27:53,394] jepsen worker 25 - jepsen.util 385 :fail :cas [8 [21 45]] :ValueNotSwapFrom INFO [2019-12-17 12:27:53,394] jepsen worker 25 - jepsen.util 385 :invoke :write [8 24] INFO [2019-12-17 12:27:53,404] jepsen worker 62 - jepsen.util 242 :info :write [20 42] :RequestCanceledException INFO [2019-12-17 12:27:53,406] jepsen worker 62 - jepsen.util 332 :invoke :cas [20 [23 31]] INFO [2019-12-17 12:27:53,410] jepsen worker 63 - jepsen.util 153 :ok :read [21 49] INFO [2019-12-17 12:27:53,410] jepsen worker 63 - jepsen.util 153 :invoke :cas [21 [21 25]] INFO [2019-12-17 12:27:53,427] jepsen worker 24 - jepsen.util 114 :info :write [8 28] :RequestCanceledException INFO [2019-12-17 12:27:53,429] jepsen worker 63 - jepsen.util 153 :fail :cas [21 [21 25]] :ValueNotSwapFrom INFO [2019-12-17 12:27:53,429] jepsen worker 24 - jepsen.util 204 :invoke :write [8 22] INFO [2019-12-17 12:27:53,429] jepsen worker 63 - jepsen.util 153 :invoke :cas [21 [24 1]] INFO [2019-12-17 12:27:53,442] jepsen worker 15 - jepsen.util 285 :info :write [5 25] :SyncWriteAmbiguous INFO [2019-12-17 12:27:53,444] jepsen worker 15 - jepsen.util 375 :invoke :read [5 nil] INFO [2019-12-17 12:27:53,450] jepsen worker 63 - jepsen.util 153 :fail :cas [21 [24 1]] :ValueNotSwapFrom INFO [2019-12-17 12:27:53,450] jepsen worker 63 - jepsen.util 153 :invoke :cas [21 [16 33]] INFO [2019-12-17 12:27:53,476] jepsen worker 63 - jepsen.util 153 :fail :cas [21 [16 33]] :ValueNotSwapFrom INFO [2019-12-17 12:27:53,476] jepsen worker 63 - jepsen.util 153 :invoke :write [21 32] INFO [2019-12-17 12:27:53,479] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1521,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:27:53,479] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1521,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:27:53,479] jepsen worker 71 - jepsen.util 341 :fail :read [23 nil] :RequestTimeoutException INFO [2019-12-17 12:27:53,479] jepsen worker 79 - jepsen.util 259 :fail :read [26 nil] :RequestTimeoutException INFO [2019-12-17 12:27:53,479] jepsen worker 71 - jepsen.util 341 :invoke :cas [23 [2 44]] INFO [2019-12-17 12:27:53,479] jepsen worker 79 - jepsen.util 259 :invoke :write [26 9] INFO [2019-12-17 12:27:53,482] jepsen worker 84 - jepsen.util 84 :fail :cas [28 [23 12]] :ValueNotSwapFrom INFO [2019-12-17 12:27:53,483] jepsen worker 84 - jepsen.util 84 :invoke :read [28 nil] INFO [2019-12-17 12:27:53,504] jepsen worker 82 - jepsen.util 352 :info :write [27 14] :RequestCanceledException INFO [2019-12-17 12:27:53,505] jepsen worker 82 - jepsen.util 442 :invoke :read [27 nil] INFO [2019-12-17 12:27:53,509] jepsen worker 4 - jepsen.util 274 :fail :cas [1 [3 28]] :ValueNotSwapFrom INFO [2019-12-17 12:27:53,509] jepsen worker 4 - jepsen.util 274 :invoke :read [1 nil] INFO [2019-12-17 12:27:53,534] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.0234375}, :ns_1@172.28.128.126 {:progress 0.02158273381294962}, :ns_1@172.28.128.127 {:progress 0.05447470817120625}, :ns_1@172.28.128.128 {:progress 0.03773584905660377}} INFO [2019-12-17 12:27:53,534] jepsen nemesis - couchbase.util Rebalance check count: 5 INFO [2019-12-17 12:27:53,534] jepsen nemesis - couchbase.util Rebalance stuck count: 0 INFO [2019-12-17 12:27:53,535] jepsen worker 63 - jepsen.util 153 :ok :write [21 32] INFO [2019-12-17 12:27:53,535] jepsen worker 63 - jepsen.util 153 :invoke :write [21 0] INFO [2019-12-17 12:27:53,546] jepsen worker 22 - jepsen.util 292 :ok :read [7 20] INFO [2019-12-17 12:27:53,546] jepsen worker 22 - jepsen.util 292 :invoke :write [7 2] INFO [2019-12-17 12:27:53,558] jepsen worker 21 - jepsen.util 291 :ok :read [7 20] INFO [2019-12-17 12:27:53,559] jepsen worker 21 - jepsen.util 291 :invoke :cas [7 [33 39]] INFO [2019-12-17 12:27:53,577] jepsen worker 34 - jepsen.util 304 :fail :cas [11 [35 16]] :ValueNotSwapFrom INFO [2019-12-17 12:27:53,577] jepsen worker 34 - jepsen.util 304 :invoke :write [11 44] INFO [2019-12-17 12:27:53,612] jepsen worker 51 - jepsen.util 141 :fail :cas [17 [20 46]] :ValueNotSwapFrom INFO [2019-12-17 12:27:53,612] jepsen worker 51 - jepsen.util 141 :invoke :write [17 21] INFO [2019-12-17 12:27:53,618] jepsen worker 4 - jepsen.util 274 :ok :read [1 23] INFO [2019-12-17 12:27:53,618] jepsen worker 4 - jepsen.util 274 :invoke :cas [1 [46 0]] INFO [2019-12-17 12:27:53,621] jepsen worker 53 - jepsen.util 323 :info :write [17 22] :RequestCanceledException INFO [2019-12-17 12:27:53,621] jepsen worker 68 - jepsen.util 248 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:27:53,621] jepsen worker 16 - jepsen.util 106 :info :write [5 23] :RequestCanceledException INFO [2019-12-17 12:27:53,621] jepsen worker 68 - jepsen.util 248 :invoke :write [22 25] INFO [2019-12-17 12:27:53,621] jepsen worker 41 - jepsen.util 221 :info :write [13 43] :RequestCanceledException INFO [2019-12-17 12:27:53,622] jepsen worker 16 - jepsen.util 196 :invoke :write [5 40] INFO [2019-12-17 12:27:53,622] jepsen worker 53 - jepsen.util 413 :invoke :read [17 nil] INFO [2019-12-17 12:27:53,622] jepsen worker 41 - jepsen.util 311 :invoke :read [13 nil] INFO [2019-12-17 12:27:53,629] jepsen worker 76 - jepsen.util 256 :ok :read [25 2] INFO [2019-12-17 12:27:53,629] jepsen worker 76 - jepsen.util 256 :invoke :cas [25 [24 37]] INFO [2019-12-17 12:27:53,646] jepsen worker 63 - jepsen.util 153 :ok :write [21 0] INFO [2019-12-17 12:27:53,646] jepsen worker 63 - jepsen.util 153 :invoke :cas [21 [1 6]] INFO [2019-12-17 12:27:53,661] jepsen worker 63 - jepsen.util 153 :fail :cas [21 [1 6]] :ValueNotSwapFrom INFO [2019-12-17 12:27:53,662] jepsen worker 63 - jepsen.util 153 :invoke :read [21 nil] INFO [2019-12-17 12:27:53,677] jepsen worker 5 - jepsen.util 5 :ok :read [1 23] INFO [2019-12-17 12:27:53,677] jepsen worker 5 - jepsen.util 5 :invoke :cas [1 [30 45]] INFO [2019-12-17 12:27:53,679] jepsen worker 63 - jepsen.util 153 :ok :read [21 0] INFO [2019-12-17 12:27:53,680] jepsen worker 63 - jepsen.util 153 :invoke :cas [21 [46 39]] INFO [2019-12-17 12:27:53,687] jepsen worker 66 - jepsen.util 246 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:27:53,687] jepsen worker 66 - jepsen.util 246 :invoke :write [22 24] INFO [2019-12-17 12:27:53,755] jepsen worker 70 - jepsen.util 340 :info :write [23 19] :RequestCanceledException INFO [2019-12-17 12:27:53,755] jepsen worker 48 - jepsen.util 318 :info :cas [16 [42 7]] :RequestCanceledException INFO [2019-12-17 12:27:53,755] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1544,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:27:53,755] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1544,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:27:53,756] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1547,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:27:53,756] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1547,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:27:53,757] jepsen worker 70 - jepsen.util 430 :invoke :write [23 48] INFO [2019-12-17 12:27:53,757] jepsen worker 48 - jepsen.util 408 :invoke :read [16 nil] INFO [2019-12-17 12:27:53,764] jepsen worker 63 - jepsen.util 153 :fail :cas [21 [46 39]] :ValueNotSwapFrom INFO [2019-12-17 12:27:53,765] jepsen worker 63 - jepsen.util 153 :invoke :cas [21 [33 13]] INFO [2019-12-17 12:27:53,765] jepsen worker 89 - jepsen.util 269 :ok :read [29 5] INFO [2019-12-17 12:27:53,765] jepsen worker 89 - jepsen.util 269 :invoke :write [29 28] INFO [2019-12-17 12:27:53,778] jepsen worker 63 - jepsen.util 153 :fail :cas [21 [33 13]] :ValueNotSwapFrom INFO [2019-12-17 12:27:53,779] jepsen worker 63 - jepsen.util 153 :invoke :cas [21 [18 26]] INFO [2019-12-17 12:27:53,801] jepsen worker 63 - jepsen.util 153 :fail :cas [21 [18 26]] :ValueNotSwapFrom INFO [2019-12-17 12:27:53,801] jepsen worker 63 - jepsen.util 153 :invoke :read [21 nil] INFO [2019-12-17 12:27:53,812] jepsen worker 76 - jepsen.util 256 :fail :cas [25 [24 37]] :ValueNotSwapFrom INFO [2019-12-17 12:27:53,812] jepsen worker 76 - jepsen.util 256 :invoke :read [25 nil] INFO [2019-12-17 12:27:53,820] jepsen worker 7 - jepsen.util 367 :info :cas [2 [26 34]] :RequestCanceledException INFO [2019-12-17 12:27:53,820] jepsen worker 77 - jepsen.util 257 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:27:53,820] jepsen worker 77 - jepsen.util 257 :invoke :write [25 43] INFO [2019-12-17 12:27:53,821] jepsen worker 7 - jepsen.util 457 :invoke :cas [2 [34 37]] INFO [2019-12-17 12:27:53,822] jepsen worker 63 - jepsen.util 153 :ok :read [21 0] INFO [2019-12-17 12:27:53,822] jepsen worker 63 - jepsen.util 153 :invoke :cas [21 [9 35]] INFO [2019-12-17 12:27:53,827] jepsen worker 6 - jepsen.util 276 :fail :cas [2 [24 23]] :ValueNotSwapFrom INFO [2019-12-17 12:27:53,827] jepsen worker 6 - jepsen.util 276 :invoke :write [2 47] INFO [2019-12-17 12:27:53,834] jepsen worker 40 - jepsen.util 310 :info :write [13 17] :RequestCanceledException INFO [2019-12-17 12:27:53,835] jepsen worker 40 - jepsen.util 400 :invoke :write [13 11] INFO [2019-12-17 12:27:53,845] jepsen worker 63 - jepsen.util 153 :fail :cas [21 [9 35]] :ValueNotSwapFrom INFO [2019-12-17 12:27:53,845] jepsen worker 63 - jepsen.util 153 :invoke :read [21 nil] INFO [2019-12-17 12:27:53,862] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1568,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:27:53,862] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1568,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:27:53,867] jepsen worker 63 - jepsen.util 153 :ok :read [21 0] INFO [2019-12-17 12:27:53,867] jepsen worker 63 - jepsen.util 153 :invoke :read [21 nil] INFO [2019-12-17 12:27:53,884] jepsen worker 63 - jepsen.util 153 :ok :read [21 0] INFO [2019-12-17 12:27:53,884] jepsen worker 63 - jepsen.util 153 :invoke :cas [21 [14 11]] INFO [2019-12-17 12:27:53,887] jepsen worker 13 - jepsen.util 193 :info :write [4 36] :RequestCanceledException INFO [2019-12-17 12:27:53,887] jepsen worker 45 - jepsen.util 315 :info :write [15 6] :RequestCanceledException INFO [2019-12-17 12:27:53,887] jepsen worker 35 - jepsen.util 35 :info :write [11 3] :RequestCanceledException INFO [2019-12-17 12:27:53,888] jepsen worker 45 - jepsen.util 405 :invoke :read [15 nil] INFO [2019-12-17 12:27:53,888] jepsen worker 13 - jepsen.util 283 :invoke :cas [4 [11 32]] INFO [2019-12-17 12:27:53,888] jepsen worker 35 - jepsen.util 125 :invoke :write [11 34] INFO [2019-12-17 12:27:53,904] jepsen worker 63 - jepsen.util 153 :fail :cas [21 [14 11]] :ValueNotSwapFrom INFO [2019-12-17 12:27:53,905] jepsen worker 63 - jepsen.util 153 :invoke :write [21 1] INFO [2019-12-17 12:27:53,905] jepsen worker 31 - jepsen.util 391 :info :write [10 9] :RequestCanceledException INFO [2019-12-17 12:27:53,906] jepsen worker 31 - jepsen.util 481 :invoke :cas [10 [36 48]] INFO [2019-12-17 12:27:53,925] jepsen worker 23 - jepsen.util 113 :ok :read [7 20] INFO [2019-12-17 12:27:53,925] jepsen worker 23 - jepsen.util 113 :invoke :cas [7 [12 44]] INFO [2019-12-17 12:27:53,926] jepsen worker 4 - jepsen.util 274 :fail :cas [1 [46 0]] :ValueNotSwapFrom INFO [2019-12-17 12:27:53,926] jepsen worker 4 - jepsen.util 274 :invoke :write [1 29] INFO [2019-12-17 12:27:53,931] jepsen worker 81 - jepsen.util 261 :info :write [27 7] :RequestCanceledException INFO [2019-12-17 12:27:53,931] jepsen worker 2 - jepsen.util 92 :info :write [0 5] :RequestCanceledException INFO [2019-12-17 12:27:53,933] jepsen worker 81 - jepsen.util 351 :invoke :read [27 nil] INFO [2019-12-17 12:27:53,933] jepsen worker 2 - jepsen.util 182 :invoke :write [0 31] INFO [2019-12-17 12:27:53,943] jepsen worker 31 - jepsen.util 481 :fail :cas [10 [36 48]] :ValueNotSwapFrom INFO [2019-12-17 12:27:53,944] jepsen worker 31 - jepsen.util 481 :invoke :write [10 37] INFO [2019-12-17 12:27:53,952] jepsen worker 26 - jepsen.util 386 :info :write [8 33] :RequestCanceledException INFO [2019-12-17 12:27:53,954] jepsen worker 26 - jepsen.util 476 :invoke :read [8 nil] INFO [2019-12-17 12:27:53,962] jepsen worker 7 - jepsen.util 457 :fail :cas [2 [34 37]] :ValueNotSwapFrom INFO [2019-12-17 12:27:53,962] jepsen worker 7 - jepsen.util 457 :invoke :cas [2 [38 40]] INFO [2019-12-17 12:27:53,964] jepsen worker 20 - jepsen.util 290 :info :write [6 38] :SyncWriteAmbiguous INFO [2019-12-17 12:27:53,965] jepsen worker 20 - jepsen.util 380 :invoke :read [6 nil] INFO [2019-12-17 12:27:53,967] jepsen worker 45 - jepsen.util 405 :ok :read [15 35] INFO [2019-12-17 12:27:53,967] jepsen worker 45 - jepsen.util 405 :invoke :write [15 16] INFO [2019-12-17 12:27:54,007] jepsen worker 53 - jepsen.util 413 :ok :read [17 0] INFO [2019-12-17 12:27:54,007] jepsen worker 53 - jepsen.util 413 :invoke :write [17 34] INFO [2019-12-17 12:27:54,009] jepsen worker 44 - jepsen.util 224 :fail :cas [14 [21 11]] :ValueNotSwapFrom INFO [2019-12-17 12:27:54,009] jepsen worker 44 - jepsen.util 224 :invoke :cas [14 [47 17]] INFO [2019-12-17 12:27:54,034] jepsen worker 83 - jepsen.util 353 :info :write [27 33] :RequestCanceledException INFO [2019-12-17 12:27:54,034] jepsen worker 49 - jepsen.util 229 :info :write [16 5] :RequestCanceledException INFO [2019-12-17 12:27:54,035] jepsen worker 1 - jepsen.util 271 :info :cas [0 [46 12]] :RequestCanceledException INFO [2019-12-17 12:27:54,036] jepsen worker 49 - jepsen.util 319 :invoke :cas [16 [12 21]] INFO [2019-12-17 12:27:54,036] jepsen worker 83 - jepsen.util 443 :invoke :read [27 nil] INFO [2019-12-17 12:27:54,036] jepsen worker 1 - jepsen.util 361 :invoke :cas [0 [17 25]] INFO [2019-12-17 12:27:54,043] jepsen worker 78 - jepsen.util 348 :ok :read [26 47] INFO [2019-12-17 12:27:54,043] jepsen worker 78 - jepsen.util 348 :invoke :write [26 19] INFO [2019-12-17 12:27:54,046] jepsen worker 63 - jepsen.util 153 :ok :write [21 1] INFO [2019-12-17 12:27:54,046] jepsen worker 63 - jepsen.util 153 :invoke :write [21 19] INFO [2019-12-17 12:27:54,080] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1591,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:27:54,080] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1591,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:27:54,087] jepsen worker 55 - jepsen.util 325 :info :write [18 12] :RequestCanceledException INFO [2019-12-17 12:27:54,088] jepsen worker 55 - jepsen.util 415 :invoke :write [18 2] INFO [2019-12-17 12:27:54,099] jepsen worker 62 - jepsen.util 332 :fail :cas [20 [23 31]] :ValueNotSwapFrom INFO [2019-12-17 12:27:54,100] jepsen worker 62 - jepsen.util 332 :invoke :cas [20 [48 25]] INFO [2019-12-17 12:27:54,104] jepsen worker 85 - jepsen.util 265 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:27:54,105] jepsen worker 85 - jepsen.util 265 :invoke :cas [28 [48 25]] INFO [2019-12-17 12:27:54,121] jepsen worker 8 - jepsen.util 278 :info :cas [2 [6 45]] :RequestCanceledException INFO [2019-12-17 12:27:54,123] jepsen worker 7 - jepsen.util 457 :fail :cas [2 [38 40]] :ValueNotSwapFrom INFO [2019-12-17 12:27:54,123] jepsen worker 8 - jepsen.util 368 :invoke :write [2 10] INFO [2019-12-17 12:27:54,123] jepsen worker 7 - jepsen.util 457 :invoke :cas [2 [44 12]] INFO [2019-12-17 12:27:54,134] jepsen worker 86 - jepsen.util 86 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:27:54,134] jepsen worker 33 - jepsen.util 303 :info :write [11 22] :RequestCanceledException INFO [2019-12-17 12:27:54,135] jepsen worker 86 - jepsen.util 86 :invoke :write [28 33] INFO [2019-12-17 12:27:54,136] jepsen worker 33 - jepsen.util 393 :invoke :cas [11 [16 41]] INFO [2019-12-17 12:27:54,151] jepsen worker 63 - jepsen.util 153 :ok :write [21 19] INFO [2019-12-17 12:27:54,151] jepsen worker 63 - jepsen.util 153 :invoke :write [21 14] INFO [2019-12-17 12:27:54,151] jepsen worker 28 - jepsen.util 208 :info :write [9 28] :RequestCanceledException INFO [2019-12-17 12:27:54,153] jepsen worker 28 - jepsen.util 298 :invoke :write [9 44] INFO [2019-12-17 12:27:54,173] jepsen worker 5 - jepsen.util 5 :fail :cas [1 [30 45]] :ValueNotSwapFrom INFO [2019-12-17 12:27:54,173] jepsen worker 5 - jepsen.util 5 :invoke :write [1 2] INFO [2019-12-17 12:27:54,205] jepsen worker 58 - jepsen.util 148 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:27:54,205] jepsen worker 3 - jepsen.util 183 :info :write [1 42] :RequestCanceledException INFO [2019-12-17 12:27:54,205] jepsen worker 58 - jepsen.util 148 :invoke :read [19 nil] INFO [2019-12-17 12:27:54,206] jepsen worker 3 - jepsen.util 273 :invoke :read [1 nil] INFO [2019-12-17 12:27:54,226] jepsen worker 7 - jepsen.util 457 :fail :cas [2 [44 12]] :ValueNotSwapFrom INFO [2019-12-17 12:27:54,226] jepsen worker 7 - jepsen.util 457 :invoke :read [2 nil] INFO [2019-12-17 12:27:54,231] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1612,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:27:54,231] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1612,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:27:54,231] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1613,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:27:54,231] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1613,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:27:54,231] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1614,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:27:54,231] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1614,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:27:54,234] jepsen worker 39 - jepsen.util 219 :info :write [13 37] :RequestCanceledException INFO [2019-12-17 12:27:54,236] jepsen worker 39 - jepsen.util 309 :invoke :cas [13 [26 33]] INFO [2019-12-17 12:27:54,253] jepsen worker 63 - jepsen.util 153 :ok :write [21 14] INFO [2019-12-17 12:27:54,253] jepsen worker 63 - jepsen.util 153 :invoke :read [21 nil] INFO [2019-12-17 12:27:54,267] jepsen worker 87 - jepsen.util 177 :fail :cas [29 [11 47]] :ValueNotSwapFrom INFO [2019-12-17 12:27:54,267] jepsen worker 87 - jepsen.util 177 :invoke :write [29 4] INFO [2019-12-17 12:27:54,282] jepsen worker 63 - jepsen.util 153 :ok :read [21 14] INFO [2019-12-17 12:27:54,282] jepsen worker 63 - jepsen.util 153 :invoke :read [21 nil] INFO [2019-12-17 12:27:54,297] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1621,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:27:54,297] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1621,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:27:54,297] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1625,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:27:54,297] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1625,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:27:54,302] jepsen worker 63 - jepsen.util 153 :ok :read [21 14] INFO [2019-12-17 12:27:54,302] jepsen worker 63 - jepsen.util 153 :invoke :write [21 49] INFO [2019-12-17 12:27:54,302] jepsen worker 57 - jepsen.util 327 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:27:54,303] jepsen worker 57 - jepsen.util 327 :invoke :read [19 nil] INFO [2019-12-17 12:27:54,305] jepsen worker 32 - jepsen.util 212 :info :write [10 47] :SyncWriteAmbiguous INFO [2019-12-17 12:27:54,306] jepsen worker 32 - jepsen.util 302 :invoke :cas [10 [32 6]] INFO [2019-12-17 12:27:54,311] jepsen worker 21 - jepsen.util 291 :fail :cas [7 [33 39]] :ValueNotSwapFrom INFO [2019-12-17 12:27:54,311] jepsen worker 21 - jepsen.util 291 :invoke :read [7 nil] INFO [2019-12-17 12:27:54,318] jepsen worker 23 - jepsen.util 113 :fail :cas [7 [12 44]] :ValueNotSwapFrom INFO [2019-12-17 12:27:54,318] jepsen worker 23 - jepsen.util 113 :invoke :write [7 2] INFO [2019-12-17 12:27:54,320] jepsen worker 42 - jepsen.util 402 :info :write [14 40] :RequestCanceledException INFO [2019-12-17 12:27:54,321] jepsen worker 47 - jepsen.util 317 :info :write [15 32] :RequestCanceledException INFO [2019-12-17 12:27:54,322] jepsen worker 42 - jepsen.util 492 :invoke :write [14 27] INFO [2019-12-17 12:27:54,322] jepsen worker 47 - jepsen.util 407 :invoke :cas [15 [16 9]] INFO [2019-12-17 12:27:54,341] jepsen worker 72 - jepsen.util 162 :ok :read [24 35] INFO [2019-12-17 12:27:54,341] jepsen worker 72 - jepsen.util 162 :invoke :write [24 21] INFO [2019-12-17 12:27:54,368] jepsen worker 73 - jepsen.util 253 :info :write [24 39] :SyncWriteAmbiguous INFO [2019-12-17 12:27:54,369] jepsen worker 73 - jepsen.util 343 :invoke :cas [24 [7 15]] INFO [2019-12-17 12:27:54,370] jepsen worker 9 - jepsen.util 189 :ok :read [3 14] INFO [2019-12-17 12:27:54,370] jepsen worker 9 - jepsen.util 189 :invoke :write [3 43] INFO [2019-12-17 12:27:54,389] jepsen worker 7 - jepsen.util 457 :ok :read [2 15] INFO [2019-12-17 12:27:54,389] jepsen worker 7 - jepsen.util 457 :invoke :read [2 nil] INFO [2019-12-17 12:27:54,397] jepsen worker 71 - jepsen.util 341 :fail :cas [23 [2 44]] :ValueNotSwapFrom INFO [2019-12-17 12:27:54,398] jepsen worker 71 - jepsen.util 341 :invoke :read [23 nil] INFO [2019-12-17 12:27:54,399] jepsen worker 20 - jepsen.util 380 :ok :read [6 45] INFO [2019-12-17 12:27:54,399] jepsen worker 20 - jepsen.util 380 :invoke :read [6 nil] INFO [2019-12-17 12:27:54,405] jepsen worker 18 - jepsen.util 288 :info :write [6 12] :RequestCanceledException INFO [2019-12-17 12:27:54,406] jepsen worker 18 - jepsen.util 378 :invoke :write [6 10] INFO [2019-12-17 12:27:54,421] jepsen worker 47 - jepsen.util 407 :fail :cas [15 [16 9]] :ValueNotSwapFrom INFO [2019-12-17 12:27:54,421] jepsen worker 10 - jepsen.util 190 :info :write [3 12] :RequestCanceledException INFO [2019-12-17 12:27:54,421] jepsen worker 27 - jepsen.util 27 :info :cas [9 [25 37]] :RequestCanceledException INFO [2019-12-17 12:27:54,421] jepsen worker 65 - jepsen.util 335 :fail :read [21 nil] :RequestTimeoutException INFO [2019-12-17 12:27:54,421] jepsen worker 47 - jepsen.util 407 :invoke :write [15 29] INFO [2019-12-17 12:27:54,421] jepsen worker 65 - jepsen.util 335 :invoke :cas [21 [38 14]] INFO [2019-12-17 12:27:54,422] jepsen worker 10 - jepsen.util 280 :invoke :cas [3 [25 26]] INFO [2019-12-17 12:27:54,422] jepsen worker 27 - jepsen.util 117 :invoke :write [9 18] INFO [2019-12-17 12:27:54,424] jepsen worker 63 - jepsen.util 153 :ok :write [21 49] INFO [2019-12-17 12:27:54,424] jepsen worker 63 - jepsen.util 153 :invoke :cas [21 [38 7]] INFO [2019-12-17 12:27:54,434] jepsen worker 11 - jepsen.util 281 :fail :read [3 nil] :RequestTimeoutException INFO [2019-12-17 12:27:54,434] jepsen worker 11 - jepsen.util 281 :invoke :cas [3 [30 31]] INFO [2019-12-17 12:27:54,442] jepsen worker 76 - jepsen.util 256 :ok :read [25 2] INFO [2019-12-17 12:27:54,443] jepsen worker 76 - jepsen.util 256 :invoke :read [25 nil] INFO [2019-12-17 12:27:54,449] jepsen worker 20 - jepsen.util 380 :ok :read [6 45] INFO [2019-12-17 12:27:54,450] jepsen worker 20 - jepsen.util 380 :invoke :read [6 nil] INFO [2019-12-17 12:27:54,452] jepsen worker 88 - jepsen.util 268 :info :write [29 40] :RequestCanceledException INFO [2019-12-17 12:27:54,453] jepsen worker 88 - jepsen.util 358 :invoke :cas [29 [37 46]] INFO [2019-12-17 12:27:54,465] jepsen worker 13 - jepsen.util 283 :fail :cas [4 [11 32]] :ValueNotSwapFrom INFO [2019-12-17 12:27:54,465] jepsen worker 13 - jepsen.util 283 :invoke :read [4 nil] INFO [2019-12-17 12:27:54,502] jepsen worker 62 - jepsen.util 332 :fail :cas [20 [48 25]] :ValueNotSwapFrom INFO [2019-12-17 12:27:54,502] jepsen worker 62 - jepsen.util 332 :invoke :write [20 0] INFO [2019-12-17 12:27:54,510] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1649,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:27:54,510] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1649,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:27:54,521] jepsen worker 46 - jepsen.util 316 :fail :read [15 nil] :RequestTimeoutException INFO [2019-12-17 12:27:54,521] jepsen worker 46 - jepsen.util 316 :invoke :read [15 nil] INFO [2019-12-17 12:27:54,523] jepsen worker 26 - jepsen.util 476 :ok :read [8 14] INFO [2019-12-17 12:27:54,524] jepsen worker 26 - jepsen.util 476 :invoke :cas [8 [24 1]] INFO [2019-12-17 12:27:54,526] jepsen worker 13 - jepsen.util 283 :ok :read [4 4] INFO [2019-12-17 12:27:54,526] jepsen worker 13 - jepsen.util 283 :invoke :write [4 46] INFO [2019-12-17 12:27:54,534] jepsen worker 64 - jepsen.util 334 :fail :read [21 nil] :RequestTimeoutException INFO [2019-12-17 12:27:54,535] jepsen worker 64 - jepsen.util 334 :invoke :cas [21 [13 49]] INFO [2019-12-17 12:27:54,551] jepsen worker 61 - jepsen.util 331 :info :cas [20 [7 13]] :RequestCanceledException INFO [2019-12-17 12:27:54,553] jepsen worker 61 - jepsen.util 421 :invoke :read [20 nil] INFO [2019-12-17 12:27:54,553] jepsen worker 64 - jepsen.util 334 :fail :cas [21 [13 49]] :ValueNotSwapFrom INFO [2019-12-17 12:27:54,554] jepsen worker 64 - jepsen.util 334 :invoke :cas [21 [20 16]] INFO [2019-12-17 12:27:54,566] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.02734375}, :ns_1@172.28.128.126 {:progress 0.02517985611510787}, :ns_1@172.28.128.127 {:progress 0.0622568093385214}, :ns_1@172.28.128.128 {:progress 0.05283018867924527}} INFO [2019-12-17 12:27:54,567] jepsen nemesis - couchbase.util Rebalance check count: 6 INFO [2019-12-17 12:27:54,567] jepsen nemesis - couchbase.util Rebalance stuck count: 0 INFO [2019-12-17 12:27:54,571] jepsen worker 64 - jepsen.util 334 :fail :cas [21 [20 16]] :ValueNotSwapFrom INFO [2019-12-17 12:27:54,571] jepsen worker 64 - jepsen.util 334 :invoke :read [21 nil] INFO [2019-12-17 12:27:54,588] jepsen worker 38 - jepsen.util 218 :fail :read [12 nil] :RequestTimeoutException INFO [2019-12-17 12:27:54,588] jepsen worker 38 - jepsen.util 218 :invoke :read [12 nil] INFO [2019-12-17 12:27:54,592] jepsen worker 64 - jepsen.util 334 :ok :read [21 49] INFO [2019-12-17 12:27:54,592] jepsen worker 64 - jepsen.util 334 :invoke :write [21 7] INFO [2019-12-17 12:27:54,597] jepsen worker 44 - jepsen.util 224 :fail :cas [14 [47 17]] :ValueNotSwapFrom INFO [2019-12-17 12:27:54,598] jepsen worker 44 - jepsen.util 224 :invoke :cas [14 [47 16]] INFO [2019-12-17 12:27:54,610] jepsen worker 0 - jepsen.util 270 :info :write [0 9] :SyncWriteAmbiguous INFO [2019-12-17 12:27:54,611] jepsen worker 0 - jepsen.util 360 :invoke :write [0 9] INFO [2019-12-17 12:27:54,621] jepsen worker 36 - jepsen.util 306 :info :write [12 5] :RequestCanceledException INFO [2019-12-17 12:27:54,622] jepsen worker 36 - jepsen.util 396 :invoke :read [12 nil] INFO [2019-12-17 12:27:54,625] jepsen worker 59 - jepsen.util 239 :ok :read [19 8] INFO [2019-12-17 12:27:54,625] jepsen worker 59 - jepsen.util 239 :invoke :read [19 nil] INFO [2019-12-17 12:27:54,634] jepsen worker 80 - jepsen.util 440 :info :write [26 13] :RequestCanceledException INFO [2019-12-17 12:27:54,634] jepsen worker 75 - jepsen.util 255 :info :write [25 13] :RequestCanceledException INFO [2019-12-17 12:27:54,635] jepsen worker 49 - jepsen.util 319 :fail :cas [16 [12 21]] :ValueNotSwapFrom INFO [2019-12-17 12:27:54,635] jepsen worker 49 - jepsen.util 319 :invoke :read [16 nil] INFO [2019-12-17 12:27:54,636] jepsen worker 75 - jepsen.util 345 :invoke :read [25 nil] INFO [2019-12-17 12:27:54,636] jepsen worker 80 - jepsen.util 530 :invoke :write [26 42] INFO [2019-12-17 12:27:54,652] jepsen worker 64 - jepsen.util 334 :ok :write [21 7] INFO [2019-12-17 12:27:54,652] jepsen worker 64 - jepsen.util 334 :invoke :cas [21 [22 17]] INFO [2019-12-17 12:27:54,653] jepsen worker 12 - jepsen.util 192 :info :write [4 18] :RequestCanceledException INFO [2019-12-17 12:27:54,653] jepsen worker 37 - jepsen.util 127 :info :write [12 36] :RequestCanceledException INFO [2019-12-17 12:27:54,654] jepsen worker 12 - jepsen.util 282 :invoke :read [4 nil] INFO [2019-12-17 12:27:54,654] jepsen worker 37 - jepsen.util 217 :invoke :cas [12 [37 16]] INFO [2019-12-17 12:27:54,674] jepsen worker 64 - jepsen.util 334 :fail :cas [21 [22 17]] :ValueNotSwapFrom INFO [2019-12-17 12:27:54,674] jepsen worker 64 - jepsen.util 334 :invoke :cas [21 [26 38]] INFO [2019-12-17 12:27:54,703] jepsen worker 76 - jepsen.util 256 :ok :read [25 2] INFO [2019-12-17 12:27:54,703] jepsen worker 76 - jepsen.util 256 :invoke :write [25 36] INFO [2019-12-17 12:27:54,708] jepsen worker 64 - jepsen.util 334 :fail :cas [21 [26 38]] :ValueNotSwapFrom INFO [2019-12-17 12:27:54,708] jepsen worker 64 - jepsen.util 334 :invoke :write [21 31] INFO [2019-12-17 12:27:54,738] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1682,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:27:54,738] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1682,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:27:54,779] jepsen worker 64 - jepsen.util 334 :ok :write [21 31] INFO [2019-12-17 12:27:54,779] jepsen worker 64 - jepsen.util 334 :invoke :read [21 nil] INFO [2019-12-17 12:27:54,787] jepsen worker 52 - jepsen.util 232 :info :write [17 23] :RequestCanceledException INFO [2019-12-17 12:27:54,789] jepsen worker 52 - jepsen.util 322 :invoke :read [17 nil] INFO [2019-12-17 12:27:54,796] jepsen worker 64 - jepsen.util 334 :ok :read [21 31] INFO [2019-12-17 12:27:54,796] jepsen worker 64 - jepsen.util 334 :invoke :write [21 38] INFO [2019-12-17 12:27:54,798] jepsen worker 69 - jepsen.util 339 :fail :cas [23 [12 12]] :ValueNotSwapFrom INFO [2019-12-17 12:27:54,798] jepsen worker 69 - jepsen.util 339 :invoke :write [23 1] INFO [2019-12-17 12:27:54,813] jepsen worker 33 - jepsen.util 393 :fail :cas [11 [16 41]] :ValueNotSwapFrom INFO [2019-12-17 12:27:54,813] jepsen worker 33 - jepsen.util 393 :invoke :read [11 nil] INFO [2019-12-17 12:27:54,821] jepsen worker 67 - jepsen.util 337 :info :write [22 34] :RequestCanceledException INFO [2019-12-17 12:27:54,822] jepsen worker 67 - jepsen.util 427 :invoke :cas [22 [21 24]] INFO [2019-12-17 12:27:54,864] jepsen worker 64 - jepsen.util 334 :ok :write [21 38] INFO [2019-12-17 12:27:54,864] jepsen worker 64 - jepsen.util 334 :invoke :read [21 nil] INFO [2019-12-17 12:27:54,883] jepsen worker 64 - jepsen.util 334 :ok :read [21 38] INFO [2019-12-17 12:27:54,884] jepsen worker 64 - jepsen.util 334 :invoke :cas [21 [18 39]] INFO [2019-12-17 12:27:54,900] jepsen worker 64 - jepsen.util 334 :fail :cas [21 [18 39]] :ValueNotSwapFrom INFO [2019-12-17 12:27:54,900] jepsen worker 64 - jepsen.util 334 :invoke :write [21 47] INFO [2019-12-17 12:27:54,905] jepsen worker 43 - jepsen.util 403 :info :write [14 1] :RequestCanceledException INFO [2019-12-17 12:27:54,906] jepsen worker 43 - jepsen.util 493 :invoke :write [14 48] INFO [2019-12-17 12:27:54,954] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1701,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:27:54,955] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1701,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:27:54,956] jepsen worker 64 - jepsen.util 334 :ok :write [21 47] INFO [2019-12-17 12:27:54,956] jepsen worker 64 - jepsen.util 334 :invoke :cas [21 [26 10]] INFO [2019-12-17 12:27:54,985] jepsen worker 64 - jepsen.util 334 :fail :cas [21 [26 10]] :ValueNotSwapFrom INFO [2019-12-17 12:27:54,989] jepsen worker 64 - jepsen.util 334 :invoke :write [21 13] INFO [2019-12-17 12:27:55,005] jepsen worker 60 - jepsen.util 420 :info :write [20 22] :RequestCanceledException INFO [2019-12-17 12:27:55,006] jepsen worker 60 - jepsen.util 510 :invoke :read [20 nil] INFO [2019-12-17 12:27:55,053] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestTimeoutException: CarrierBucketConfigRequest {"retried":0,"reason":"TIMEOUT","requestId":1416,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:27:55,063] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1711,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:27:55,064] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1711,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:27:55,064] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1712,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:27:55,064] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1712,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:27:55,086] jepsen worker 3 - jepsen.util 273 :ok :read [1 23] INFO [2019-12-17 12:27:55,086] jepsen worker 3 - jepsen.util 273 :invoke :write [1 34] INFO [2019-12-17 12:27:55,087] jepsen worker 64 - jepsen.util 334 :ok :write [21 13] INFO [2019-12-17 12:27:55,088] jepsen worker 64 - jepsen.util 334 :invoke :write [21 45] INFO [2019-12-17 12:27:55,095] jepsen worker 28 - jepsen.util 298 :ok :write [9 44] INFO [2019-12-17 12:27:55,095] jepsen worker 28 - jepsen.util 298 :invoke :write [9 45] INFO [2019-12-17 12:27:55,111] jepsen worker 7 - jepsen.util 457 :ok :read [2 15] INFO [2019-12-17 12:27:55,111] jepsen worker 7 - jepsen.util 457 :invoke :cas [2 [18 15]] INFO [2019-12-17 12:27:55,121] jepsen worker 44 - jepsen.util 224 :fail :cas [14 [47 16]] :ValueNotSwapFrom INFO [2019-12-17 12:27:55,121] jepsen worker 29 - jepsen.util 209 :info :write [9 39] :RequestCanceledException INFO [2019-12-17 12:27:55,121] jepsen worker 44 - jepsen.util 224 :invoke :read [14 nil] INFO [2019-12-17 12:27:55,122] jepsen worker 29 - jepsen.util 299 :invoke :cas [9 [44 48]] INFO [2019-12-17 12:27:55,151] jepsen worker 19 - jepsen.util 199 :info :write [6 14] :RequestCanceledException INFO [2019-12-17 12:27:55,151] jepsen worker 30 - jepsen.util 120 :info :cas [10 [28 29]] :RequestCanceledException INFO [2019-12-17 12:27:55,152] jepsen worker 19 - jepsen.util 289 :invoke :read [6 nil] INFO [2019-12-17 12:27:55,152] jepsen worker 30 - jepsen.util 210 :invoke :read [10 nil] INFO [2019-12-17 12:27:55,154] jepsen worker 37 - jepsen.util 217 :fail :cas [12 [37 16]] :ValueNotSwapFrom INFO [2019-12-17 12:27:55,154] jepsen worker 37 - jepsen.util 217 :invoke :write [12 34] INFO [2019-12-17 12:27:55,160] jepsen worker 88 - jepsen.util 358 :fail :cas [29 [37 46]] :ValueNotSwapFrom INFO [2019-12-17 12:27:55,160] jepsen worker 88 - jepsen.util 358 :invoke :read [29 nil] INFO [2019-12-17 12:27:55,177] jepsen worker 64 - jepsen.util 334 :ok :write [21 45] INFO [2019-12-17 12:27:55,178] jepsen worker 64 - jepsen.util 334 :invoke :cas [21 [23 2]] INFO [2019-12-17 12:27:55,188] jepsen worker 50 - jepsen.util 320 :info :write [16 0] :RequestCanceledException INFO [2019-12-17 12:27:55,188] jepsen worker 28 - jepsen.util 298 :ok :write [9 45] INFO [2019-12-17 12:27:55,188] jepsen worker 28 - jepsen.util 298 :invoke :read [9 nil] INFO [2019-12-17 12:27:55,189] jepsen worker 50 - jepsen.util 410 :invoke :write [16 17] INFO [2019-12-17 12:27:55,198] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1727,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:27:55,198] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1727,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:27:55,198] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1728,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:27:55,198] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1728,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:27:55,203] jepsen worker 74 - jepsen.util 344 :fail :read [24 nil] :RequestTimeoutException INFO [2019-12-17 12:27:55,203] jepsen worker 74 - jepsen.util 344 :invoke :read [24 nil] INFO [2019-12-17 12:27:55,206] jepsen worker 30 - jepsen.util 210 :ok :read [10 23] INFO [2019-12-17 12:27:55,206] jepsen worker 30 - jepsen.util 210 :invoke :read [10 nil] INFO [2019-12-17 12:27:55,210] jepsen worker 75 - jepsen.util 345 :ok :read [25 2] INFO [2019-12-17 12:27:55,210] jepsen worker 75 - jepsen.util 345 :invoke :read [25 nil] INFO [2019-12-17 12:27:55,223] jepsen worker 28 - jepsen.util 298 :ok :read [9 45] INFO [2019-12-17 12:27:55,223] jepsen worker 28 - jepsen.util 298 :invoke :cas [9 [11 36]] INFO [2019-12-17 12:27:55,231] jepsen worker 19 - jepsen.util 289 :ok :read [6 45] INFO [2019-12-17 12:27:55,231] jepsen worker 19 - jepsen.util 289 :invoke :read [6 nil] INFO [2019-12-17 12:27:55,237] jepsen worker 30 - jepsen.util 210 :ok :read [10 23] INFO [2019-12-17 12:27:55,237] jepsen worker 30 - jepsen.util 210 :invoke :write [10 21] INFO [2019-12-17 12:27:55,255] jepsen worker 28 - jepsen.util 298 :fail :cas [9 [11 36]] :ValueNotSwapFrom INFO [2019-12-17 12:27:55,256] jepsen worker 28 - jepsen.util 298 :invoke :read [9 nil] INFO [2019-12-17 12:27:55,270] jepsen worker 64 - jepsen.util 334 :fail :cas [21 [23 2]] :ValueNotSwapFrom INFO [2019-12-17 12:27:55,270] jepsen worker 64 - jepsen.util 334 :invoke :cas [21 [1 44]] INFO [2019-12-17 12:27:55,272] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1735,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:27:55,273] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1735,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:27:55,273] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1738,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:27:55,273] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1738,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:27:55,273] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1739,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:27:55,273] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1739,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:27:55,278] jepsen worker 19 - jepsen.util 289 :ok :read [6 45] INFO [2019-12-17 12:27:55,278] jepsen worker 19 - jepsen.util 289 :invoke :read [6 nil] INFO [2019-12-17 12:27:55,291] jepsen worker 64 - jepsen.util 334 :fail :cas [21 [1 44]] :ValueNotSwapFrom INFO [2019-12-17 12:27:55,291] jepsen worker 64 - jepsen.util 334 :invoke :cas [21 [18 12]] INFO [2019-12-17 12:27:55,308] jepsen worker 19 - jepsen.util 289 :ok :read [6 45] INFO [2019-12-17 12:27:55,308] jepsen worker 19 - jepsen.util 289 :invoke :cas [6 [14 30]] INFO [2019-12-17 12:27:55,310] jepsen worker 12 - jepsen.util 282 :ok :read [4 4] INFO [2019-12-17 12:27:55,310] jepsen worker 12 - jepsen.util 282 :invoke :cas [4 [12 28]] INFO [2019-12-17 12:27:55,323] jepsen worker 61 - jepsen.util 421 :ok :read [20 7] INFO [2019-12-17 12:27:55,324] jepsen worker 61 - jepsen.util 421 :invoke :write [20 38] INFO [2019-12-17 12:27:55,329] jepsen worker 64 - jepsen.util 334 :fail :cas [21 [18 12]] :ValueNotSwapFrom INFO [2019-12-17 12:27:55,329] jepsen worker 64 - jepsen.util 334 :invoke :read [21 nil] INFO [2019-12-17 12:27:55,348] jepsen worker 19 - jepsen.util 289 :fail :cas [6 [14 30]] :ValueNotSwapFrom INFO [2019-12-17 12:27:55,348] jepsen worker 19 - jepsen.util 289 :invoke :write [6 15] INFO [2019-12-17 12:27:55,352] jepsen worker 56 - jepsen.util 236 :info :write [18 39] :RequestCanceledException INFO [2019-12-17 12:27:55,353] jepsen worker 56 - jepsen.util 326 :invoke :write [18 26] INFO [2019-12-17 12:27:55,388] jepsen worker 54 - jepsen.util 414 :info :write [18 15] :RequestCanceledException INFO [2019-12-17 12:27:55,389] jepsen worker 54 - jepsen.util 504 :invoke :cas [18 [16 33]] INFO [2019-12-17 12:27:55,396] jepsen worker 64 - jepsen.util 334 :ok :read [21 45] INFO [2019-12-17 12:27:55,396] jepsen worker 64 - jepsen.util 334 :invoke :cas [21 [2 18]] INFO [2019-12-17 12:27:55,420] jepsen worker 14 - jepsen.util 284 :info :write [4 19] :RequestCanceledException INFO [2019-12-17 12:27:55,420] jepsen worker 17 - jepsen.util 377 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:27:55,421] jepsen worker 17 - jepsen.util 377 :invoke :cas [5 [40 17]] INFO [2019-12-17 12:27:55,422] jepsen worker 14 - jepsen.util 374 :invoke :write [4 18] INFO [2019-12-17 12:27:55,425] jepsen worker 64 - jepsen.util 334 :fail :cas [21 [2 18]] :ValueNotSwapFrom INFO [2019-12-17 12:27:55,425] jepsen worker 64 - jepsen.util 334 :invoke :write [21 28] INFO [2019-12-17 12:27:55,450] jepsen worker 24 - jepsen.util 204 :info :write [8 22] :RequestCanceledException INFO [2019-12-17 12:27:55,452] jepsen worker 24 - jepsen.util 294 :invoke :cas [8 [26 2]] INFO [2019-12-17 12:27:55,465] jepsen worker 52 - jepsen.util 322 :ok :read [17 0] INFO [2019-12-17 12:27:55,465] jepsen worker 52 - jepsen.util 322 :invoke :read [17 nil] INFO [2019-12-17 12:27:55,475] jepsen worker 17 - jepsen.util 377 :fail :cas [5 [40 17]] :ValueNotSwapFrom INFO [2019-12-17 12:27:55,476] jepsen worker 17 - jepsen.util 377 :invoke :cas [5 [34 17]] INFO [2019-12-17 12:27:55,488] jepsen worker 25 - jepsen.util 385 :info :write [8 24] :RequestCanceledException INFO [2019-12-17 12:27:55,489] jepsen worker 25 - jepsen.util 475 :invoke :read [8 nil] INFO [2019-12-17 12:27:55,490] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1760,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:27:55,491] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1760,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:27:55,491] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1763,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:27:55,491] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1763,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:27:55,492] jepsen worker 17 - jepsen.util 377 :fail :cas [5 [34 17]] :ValueNotSwapFrom INFO [2019-12-17 12:27:55,492] jepsen worker 17 - jepsen.util 377 :invoke :write [5 22] INFO [2019-12-17 12:27:55,494] jepsen worker 64 - jepsen.util 334 :ok :write [21 28] INFO [2019-12-17 12:27:55,494] jepsen worker 64 - jepsen.util 334 :invoke :cas [21 [40 17]] INFO [2019-12-17 12:27:55,520] jepsen worker 15 - jepsen.util 375 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:27:55,520] jepsen worker 84 - jepsen.util 84 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:27:55,520] jepsen worker 15 - jepsen.util 375 :invoke :read [5 nil] INFO [2019-12-17 12:27:55,520] jepsen worker 84 - jepsen.util 84 :invoke :write [28 13] INFO [2019-12-17 12:27:55,530] jepsen worker 26 - jepsen.util 476 :fail :cas [8 [24 1]] :ValueNotSwapFrom INFO [2019-12-17 12:27:55,530] jepsen worker 26 - jepsen.util 476 :invoke :cas [8 [12 34]] INFO [2019-12-17 12:27:55,533] jepsen worker 64 - jepsen.util 334 :fail :cas [21 [40 17]] :ValueNotSwapFrom INFO [2019-12-17 12:27:55,534] jepsen worker 64 - jepsen.util 334 :invoke :cas [21 [23 8]] INFO [2019-12-17 12:27:55,541] jepsen worker 21 - jepsen.util 291 :ok :read [7 20] INFO [2019-12-17 12:27:55,541] jepsen worker 21 - jepsen.util 291 :invoke :write [7 48] INFO [2019-12-17 12:27:55,553] jepsen worker 64 - jepsen.util 334 :fail :cas [21 [23 8]] :ValueNotSwapFrom INFO [2019-12-17 12:27:55,553] jepsen worker 64 - jepsen.util 334 :invoke :cas [21 [3 4]] INFO [2019-12-17 12:27:55,571] jepsen worker 64 - jepsen.util 334 :fail :cas [21 [3 4]] :ValueNotSwapFrom INFO [2019-12-17 12:27:55,572] jepsen worker 88 - jepsen.util 358 :ok :read [29 5] INFO [2019-12-17 12:27:55,572] jepsen worker 64 - jepsen.util 334 :invoke :write [21 37] INFO [2019-12-17 12:27:55,572] jepsen worker 88 - jepsen.util 358 :invoke :cas [29 [32 34]] INFO [2019-12-17 12:27:55,573] jepsen worker 71 - jepsen.util 341 :ok :read [23 29] INFO [2019-12-17 12:27:55,573] jepsen worker 71 - jepsen.util 341 :invoke :read [23 nil] INFO [2019-12-17 12:27:55,588] jepsen worker 82 - jepsen.util 442 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:27:55,588] jepsen worker 79 - jepsen.util 259 :info :write [26 9] :RequestCanceledException INFO [2019-12-17 12:27:55,588] jepsen worker 82 - jepsen.util 442 :invoke :write [27 36] INFO [2019-12-17 12:27:55,589] jepsen worker 79 - jepsen.util 349 :invoke :write [26 13] INFO [2019-12-17 12:27:55,593] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.03515625}, :ns_1@172.28.128.126 {:progress 0.03237410071942448}, :ns_1@172.28.128.127 {:progress 0.0622568093385214}, :ns_1@172.28.128.128 {:progress 0.06037735849056602}} INFO [2019-12-17 12:27:55,593] jepsen nemesis - couchbase.util Rebalance check count: 7 INFO [2019-12-17 12:27:55,593] jepsen nemesis - couchbase.util Rebalance stuck count: 0 INFO [2019-12-17 12:27:55,615] jepsen worker 64 - jepsen.util 334 :ok :write [21 37] INFO [2019-12-17 12:27:55,615] jepsen worker 64 - jepsen.util 334 :invoke :write [21 11] INFO [2019-12-17 12:27:55,620] jepsen worker 22 - jepsen.util 292 :info :write [7 2] :RequestCanceledException INFO [2019-12-17 12:27:55,621] jepsen worker 34 - jepsen.util 304 :info :write [11 44] :RequestCanceledException INFO [2019-12-17 12:27:55,621] jepsen worker 33 - jepsen.util 393 :ok :read [11 7] INFO [2019-12-17 12:27:55,622] jepsen worker 22 - jepsen.util 382 :invoke :read [7 nil] INFO [2019-12-17 12:27:55,622] jepsen worker 33 - jepsen.util 393 :invoke :cas [11 [46 5]] INFO [2019-12-17 12:27:55,623] jepsen worker 34 - jepsen.util 394 :invoke :cas [11 [2 6]] INFO [2019-12-17 12:27:55,635] jepsen worker 41 - jepsen.util 311 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:27:55,635] jepsen worker 41 - jepsen.util 311 :invoke :read [13 nil] INFO [2019-12-17 12:27:55,684] jepsen worker 64 - jepsen.util 334 :ok :write [21 11] INFO [2019-12-17 12:27:55,684] jepsen worker 64 - jepsen.util 334 :invoke :cas [21 [29 35]] INFO [2019-12-17 12:27:55,694] jepsen worker 52 - jepsen.util 322 :ok :read [17 0] INFO [2019-12-17 12:27:55,694] jepsen worker 52 - jepsen.util 322 :invoke :read [17 nil] INFO [2019-12-17 12:27:55,705] jepsen worker 16 - jepsen.util 196 :info :write [5 40] :RequestCanceledException INFO [2019-12-17 12:27:55,705] jepsen worker 51 - jepsen.util 141 :info :write [17 21] :RequestCanceledException INFO [2019-12-17 12:27:55,706] jepsen worker 51 - jepsen.util 231 :invoke :read [17 nil] INFO [2019-12-17 12:27:55,706] jepsen worker 16 - jepsen.util 286 :invoke :cas [5 [22 24]] INFO [2019-12-17 12:27:55,714] jepsen worker 64 - jepsen.util 334 :fail :cas [21 [29 35]] :ValueNotSwapFrom INFO [2019-12-17 12:27:55,714] jepsen worker 64 - jepsen.util 334 :invoke :read [21 nil] INFO [2019-12-17 12:27:55,721] jepsen worker 68 - jepsen.util 248 :info :write [22 25] :RequestCanceledException INFO [2019-12-17 12:27:55,721] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1794,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:27:55,722] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1794,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:27:55,722] jepsen worker 68 - jepsen.util 338 :invoke :read [22 nil] INFO [2019-12-17 12:27:55,733] jepsen worker 38 - jepsen.util 218 :ok :read [12 12] INFO [2019-12-17 12:27:55,733] jepsen worker 38 - jepsen.util 218 :invoke :cas [12 [37 18]] INFO [2019-12-17 12:27:55,743] jepsen worker 64 - jepsen.util 334 :ok :read [21 11] INFO [2019-12-17 12:27:55,743] jepsen worker 64 - jepsen.util 334 :invoke :cas [21 [23 11]] INFO [2019-12-17 12:27:55,763] jepsen worker 64 - jepsen.util 334 :fail :cas [21 [23 11]] :ValueNotSwapFrom INFO [2019-12-17 12:27:55,763] jepsen worker 64 - jepsen.util 334 :invoke :write [21 4] INFO [2019-12-17 12:27:55,772] jepsen worker 26 - jepsen.util 476 :fail :cas [8 [12 34]] :ValueNotSwapFrom INFO [2019-12-17 12:27:55,772] jepsen worker 26 - jepsen.util 476 :invoke :read [8 nil] INFO [2019-12-17 12:27:55,781] jepsen worker 44 - jepsen.util 224 :ok :read [14 27] INFO [2019-12-17 12:27:55,781] jepsen worker 44 - jepsen.util 224 :invoke :read [14 nil] INFO [2019-12-17 12:27:55,786] jepsen worker 12 - jepsen.util 282 :fail :cas [4 [12 28]] :ValueNotSwapFrom INFO [2019-12-17 12:27:55,786] jepsen worker 12 - jepsen.util 282 :invoke :cas [4 [26 17]] INFO [2019-12-17 12:27:55,787] jepsen worker 66 - jepsen.util 246 :info :write [22 24] :RequestCanceledException INFO [2019-12-17 12:27:55,787] jepsen worker 66 - jepsen.util 336 :invoke :cas [22 [4 34]] INFO [2019-12-17 12:27:55,806] jepsen worker 52 - jepsen.util 322 :ok :read [17 0] INFO [2019-12-17 12:27:55,806] jepsen worker 52 - jepsen.util 322 :invoke :cas [17 [15 32]] INFO [2019-12-17 12:27:55,816] jepsen worker 32 - jepsen.util 302 :fail :cas [10 [32 6]] :ValueNotSwapFrom INFO [2019-12-17 12:27:55,817] jepsen worker 32 - jepsen.util 302 :invoke :write [10 33] INFO [2019-12-17 12:27:55,821] jepsen worker 48 - jepsen.util 408 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:27:55,822] jepsen worker 48 - jepsen.util 408 :invoke :read [16 nil] INFO [2019-12-17 12:27:55,831] jepsen worker 45 - jepsen.util 405 :info :write [15 16] :SyncWriteAmbiguous INFO [2019-12-17 12:27:55,833] jepsen worker 45 - jepsen.util 495 :invoke :write [15 40] INFO [2019-12-17 12:27:55,852] jepsen worker 89 - jepsen.util 269 :info :write [29 28] :RequestCanceledException INFO [2019-12-17 12:27:55,852] jepsen worker 70 - jepsen.util 430 :info :write [23 48] :RequestCanceledException INFO [2019-12-17 12:27:55,853] jepsen worker 89 - jepsen.util 359 :invoke :read [29 nil] INFO [2019-12-17 12:27:55,853] jepsen worker 70 - jepsen.util 520 :invoke :cas [23 [36 33]] INFO [2019-12-17 12:27:55,858] jepsen worker 25 - jepsen.util 475 :ok :read [8 14] INFO [2019-12-17 12:27:55,858] jepsen worker 25 - jepsen.util 475 :invoke :read [8 nil] INFO [2019-12-17 12:27:55,859] jepsen worker 65 - jepsen.util 335 :fail :cas [21 [38 14]] :ValueNotSwapFrom INFO [2019-12-17 12:27:55,859] jepsen worker 65 - jepsen.util 335 :invoke :cas [21 [4 25]] INFO [2019-12-17 12:27:55,872] jepsen worker 54 - jepsen.util 504 :fail :cas [18 [16 33]] :ValueNotSwapFrom INFO [2019-12-17 12:27:55,872] jepsen worker 54 - jepsen.util 504 :invoke :read [18 nil] INFO [2019-12-17 12:27:55,883] jepsen worker 88 - jepsen.util 358 :fail :cas [29 [32 34]] :ValueNotSwapFrom INFO [2019-12-17 12:27:55,883] jepsen worker 88 - jepsen.util 358 :invoke :cas [29 [18 14]] INFO [2019-12-17 12:27:55,889] jepsen worker 46 - jepsen.util 316 :ok :read [15 35] INFO [2019-12-17 12:27:55,889] jepsen worker 46 - jepsen.util 316 :invoke :cas [15 [18 10]] INFO [2019-12-17 12:27:55,894] jepsen worker 26 - jepsen.util 476 :ok :read [8 14] INFO [2019-12-17 12:27:55,894] jepsen worker 26 - jepsen.util 476 :invoke :read [8 nil] INFO [2019-12-17 12:27:55,904] jepsen worker 40 - jepsen.util 400 :info :write [13 11] :RequestCanceledException INFO [2019-12-17 12:27:55,906] jepsen worker 29 - jepsen.util 299 :fail :cas [9 [44 48]] :ValueNotSwapFrom INFO [2019-12-17 12:27:55,906] jepsen worker 40 - jepsen.util 490 :invoke :read [13 nil] INFO [2019-12-17 12:27:55,906] jepsen worker 29 - jepsen.util 299 :invoke :read [9 nil] INFO [2019-12-17 12:27:55,907] jepsen worker 52 - jepsen.util 322 :fail :cas [17 [15 32]] :ValueNotSwapFrom INFO [2019-12-17 12:27:55,907] jepsen worker 52 - jepsen.util 322 :invoke :write [17 19] INFO [2019-12-17 12:27:55,916] jepsen worker 65 - jepsen.util 335 :fail :cas [21 [4 25]] :ValueNotSwapFrom INFO [2019-12-17 12:27:55,917] jepsen worker 65 - jepsen.util 335 :invoke :write [21 42] INFO [2019-12-17 12:27:55,920] jepsen worker 7 - jepsen.util 457 :fail :cas [2 [18 15]] :ValueNotSwapFrom INFO [2019-12-17 12:27:55,920] jepsen worker 7 - jepsen.util 457 :invoke :read [2 nil] INFO [2019-12-17 12:27:55,920] jepsen worker 77 - jepsen.util 257 :info :write [25 43] :RequestCanceledException INFO [2019-12-17 12:27:55,921] jepsen worker 6 - jepsen.util 276 :info :write [2 47] :RequestCanceledException INFO [2019-12-17 12:27:55,921] jepsen worker 77 - jepsen.util 347 :invoke :write [25 29] INFO [2019-12-17 12:27:55,922] jepsen worker 6 - jepsen.util 366 :invoke :write [2 15] INFO [2019-12-17 12:27:55,935] jepsen worker 4 - jepsen.util 274 :info :write [1 29] :RequestCanceledException INFO [2019-12-17 12:27:55,937] jepsen worker 4 - jepsen.util 364 :invoke :cas [1 [44 2]] INFO [2019-12-17 12:27:55,937] jepsen worker 25 - jepsen.util 475 :ok :read [8 14] INFO [2019-12-17 12:27:55,937] jepsen worker 25 - jepsen.util 475 :invoke :read [8 nil] INFO [2019-12-17 12:27:55,939] jepsen worker 46 - jepsen.util 316 :fail :cas [15 [18 10]] :ValueNotSwapFrom INFO [2019-12-17 12:27:55,940] jepsen worker 46 - jepsen.util 316 :invoke :read [15 nil] INFO [2019-12-17 12:27:55,949] jepsen worker 35 - jepsen.util 125 :info :write [11 34] :RequestCanceledException INFO [2019-12-17 12:27:55,950] jepsen worker 35 - jepsen.util 215 :invoke :read [11 nil] INFO [2019-12-17 12:27:55,957] jepsen worker 60 - jepsen.util 510 :ok :read [20 7] INFO [2019-12-17 12:27:55,957] jepsen worker 60 - jepsen.util 510 :invoke :write [20 24] INFO [2019-12-17 12:27:55,976] jepsen worker 29 - jepsen.util 299 :ok :read [9 45] INFO [2019-12-17 12:27:55,976] jepsen worker 29 - jepsen.util 299 :invoke :write [9 47] INFO [2019-12-17 12:27:55,987] jepsen worker 81 - jepsen.util 351 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:27:55,987] jepsen worker 81 - jepsen.util 351 :invoke :write [27 23] INFO [2019-12-17 12:27:56,005] jepsen worker 2 - jepsen.util 182 :info :write [0 31] :RequestCanceledException INFO [2019-12-17 12:27:56,006] jepsen worker 2 - jepsen.util 272 :invoke :read [0 nil] INFO [2019-12-17 12:27:56,012] jepsen worker 64 - jepsen.util 334 :ok :write [21 4] INFO [2019-12-17 12:27:56,012] jepsen worker 64 - jepsen.util 334 :invoke :cas [21 [33 17]] INFO [2019-12-17 12:27:56,014] jepsen worker 27 - jepsen.util 117 :ok :write [9 18] INFO [2019-12-17 12:27:56,014] jepsen worker 27 - jepsen.util 117 :invoke :write [9 42] INFO [2019-12-17 12:27:56,021] jepsen worker 53 - jepsen.util 413 :info :write [17 34] :RequestCanceledException INFO [2019-12-17 12:27:56,021] jepsen worker 31 - jepsen.util 481 :info :write [10 37] :RequestCanceledException INFO [2019-12-17 12:27:56,023] jepsen worker 53 - jepsen.util 503 :invoke :read [17 nil] INFO [2019-12-17 12:27:56,023] jepsen worker 31 - jepsen.util 571 :invoke :write [10 38] INFO [2019-12-17 12:27:56,031] jepsen worker 75 - jepsen.util 345 :ok :read [25 2] INFO [2019-12-17 12:27:56,031] jepsen worker 75 - jepsen.util 345 :invoke :cas [25 [28 39]] INFO [2019-12-17 12:27:56,035] jepsen worker 64 - jepsen.util 334 :fail :cas [21 [33 17]] :ValueNotSwapFrom INFO [2019-12-17 12:27:56,035] jepsen worker 64 - jepsen.util 334 :invoke :cas [21 [45 14]] INFO [2019-12-17 12:27:56,047] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1840,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:27:56,048] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1840,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:27:56,048] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1844,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:27:56,048] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1844,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:27:56,057] jepsen worker 36 - jepsen.util 396 :ok :read [12 12] INFO [2019-12-17 12:27:56,057] jepsen worker 36 - jepsen.util 396 :invoke :write [12 12] INFO [2019-12-17 12:27:56,072] jepsen worker 65 - jepsen.util 335 :ok :write [21 42] INFO [2019-12-17 12:27:56,073] jepsen worker 65 - jepsen.util 335 :invoke :read [21 nil] INFO [2019-12-17 12:27:56,077] jepsen worker 64 - jepsen.util 334 :fail :cas [21 [45 14]] :ValueNotSwapFrom INFO [2019-12-17 12:27:56,078] jepsen worker 64 - jepsen.util 334 :invoke :read [21 nil] INFO [2019-12-17 12:27:56,087] jepsen worker 1 - jepsen.util 361 :info :cas [0 [17 25]] :RequestCanceledException INFO [2019-12-17 12:27:56,088] jepsen worker 1 - jepsen.util 451 :invoke :cas [0 [35 31]] INFO [2019-12-17 12:27:56,105] jepsen worker 65 - jepsen.util 335 :ok :read [21 42] INFO [2019-12-17 12:27:56,105] jepsen worker 65 - jepsen.util 335 :invoke :read [21 nil] INFO [2019-12-17 12:27:56,105] jepsen worker 55 - jepsen.util 415 :info :write [18 2] :RequestCanceledException INFO [2019-12-17 12:27:56,105] jepsen worker 78 - jepsen.util 348 :info :write [26 19] :RequestCanceledException INFO [2019-12-17 12:27:56,106] jepsen worker 78 - jepsen.util 438 :invoke :read [26 nil] INFO [2019-12-17 12:27:56,106] jepsen worker 55 - jepsen.util 505 :invoke :write [18 10] INFO [2019-12-17 12:27:56,116] jepsen worker 51 - jepsen.util 231 :ok :read [17 0] INFO [2019-12-17 12:27:56,116] jepsen worker 51 - jepsen.util 231 :invoke :cas [17 [47 32]] INFO [2019-12-17 12:27:56,120] jepsen worker 83 - jepsen.util 443 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:27:56,121] jepsen worker 83 - jepsen.util 443 :invoke :cas [27 [30 23]] INFO [2019-12-17 12:27:56,125] jepsen worker 64 - jepsen.util 334 :ok :read [21 42] INFO [2019-12-17 12:27:56,126] jepsen worker 64 - jepsen.util 334 :invoke :read [21 nil] INFO [2019-12-17 12:27:56,127] jepsen worker 34 - jepsen.util 394 :fail :cas [11 [2 6]] :ValueNotSwapFrom INFO [2019-12-17 12:27:56,127] jepsen worker 34 - jepsen.util 394 :invoke :write [11 3] INFO [2019-12-17 12:27:56,166] jepsen worker 89 - jepsen.util 359 :ok :read [29 5] INFO [2019-12-17 12:27:56,166] jepsen worker 89 - jepsen.util 359 :invoke :write [29 5] INFO [2019-12-17 12:27:56,175] jepsen worker 65 - jepsen.util 335 :ok :read [21 42] INFO [2019-12-17 12:27:56,175] jepsen worker 65 - jepsen.util 335 :invoke :cas [21 [36 39]] INFO [2019-12-17 12:27:56,180] jepsen worker 64 - jepsen.util 334 :ok :read [21 42] INFO [2019-12-17 12:27:56,181] jepsen worker 64 - jepsen.util 334 :invoke :write [21 4] INFO [2019-12-17 12:27:56,182] jepsen worker 33 - jepsen.util 393 :fail :cas [11 [46 5]] :ValueNotSwapFrom INFO [2019-12-17 12:27:56,182] jepsen worker 33 - jepsen.util 393 :invoke :cas [11 [8 40]] INFO [2019-12-17 12:27:56,191] jepsen worker 65 - jepsen.util 335 :fail :cas [21 [36 39]] :ValueNotSwapFrom INFO [2019-12-17 12:27:56,191] jepsen worker 65 - jepsen.util 335 :invoke :read [21 nil] INFO [2019-12-17 12:27:56,198] jepsen worker 85 - jepsen.util 265 :info :cas [28 [48 25]] :RequestCanceledException INFO [2019-12-17 12:27:56,200] jepsen worker 85 - jepsen.util 355 :invoke :write [28 25] INFO [2019-12-17 12:27:56,205] jepsen worker 54 - jepsen.util 504 :ok :read [18 47] INFO [2019-12-17 12:27:56,205] jepsen worker 54 - jepsen.util 504 :invoke :read [18 nil] INFO [2019-12-17 12:27:56,206] jepsen worker 12 - jepsen.util 282 :fail :cas [4 [26 17]] :ValueNotSwapFrom INFO [2019-12-17 12:27:56,206] jepsen worker 12 - jepsen.util 282 :invoke :cas [4 [35 18]] INFO [2019-12-17 12:27:56,221] jepsen worker 8 - jepsen.util 368 :info :write [2 10] :RequestCanceledException INFO [2019-12-17 12:27:56,222] jepsen worker 8 - jepsen.util 458 :invoke :write [2 11] INFO [2019-12-17 12:27:56,225] jepsen worker 22 - jepsen.util 382 :ok :read [7 20] INFO [2019-12-17 12:27:56,226] jepsen worker 22 - jepsen.util 382 :invoke :write [7 9] INFO [2019-12-17 12:27:56,234] jepsen worker 86 - jepsen.util 86 :info :write [28 33] :RequestCanceledException INFO [2019-12-17 12:27:56,234] jepsen worker 65 - jepsen.util 335 :ok :read [21 42] INFO [2019-12-17 12:27:56,235] jepsen worker 70 - jepsen.util 520 :fail :cas [23 [36 33]] :ValueNotSwapFrom INFO [2019-12-17 12:27:56,235] jepsen worker 65 - jepsen.util 335 :invoke :write [21 22] INFO [2019-12-17 12:27:56,235] jepsen worker 70 - jepsen.util 520 :invoke :read [23 nil] INFO [2019-12-17 12:27:56,235] jepsen worker 86 - jepsen.util 176 :invoke :write [28 12] INFO [2019-12-17 12:27:56,246] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1876,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:27:56,246] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1876,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:27:56,246] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1879,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:27:56,246] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1879,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:27:56,246] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1880,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:27:56,247] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1880,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:27:56,254] jepsen worker 72 - jepsen.util 162 :info :write [24 21] :SyncWriteAmbiguous INFO [2019-12-17 12:27:56,255] jepsen worker 72 - jepsen.util 252 :invoke :cas [24 [13 29]] INFO [2019-12-17 12:27:56,263] jepsen worker 25 - jepsen.util 475 :ok :read [8 14] INFO [2019-12-17 12:27:56,263] jepsen worker 25 - jepsen.util 475 :invoke :cas [8 [40 43]] INFO [2019-12-17 12:27:56,264] jepsen worker 26 - jepsen.util 476 :ok :read [8 14] INFO [2019-12-17 12:27:56,265] jepsen worker 26 - jepsen.util 476 :invoke :cas [8 [20 45]] INFO [2019-12-17 12:27:56,278] jepsen worker 53 - jepsen.util 503 :ok :read [17 0] INFO [2019-12-17 12:27:56,278] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1868,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:27:56,278] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1868,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:27:56,278] jepsen worker 53 - jepsen.util 503 :invoke :cas [17 [21 3]] INFO [2019-12-17 12:27:56,279] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1870,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:27:56,279] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1870,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:27:56,287] jepsen worker 39 - jepsen.util 309 :info :cas [13 [26 33]] :RequestCanceledException INFO [2019-12-17 12:27:56,287] jepsen worker 87 - jepsen.util 177 :info :write [29 4] :RequestCanceledException INFO [2019-12-17 12:27:56,287] jepsen worker 5 - jepsen.util 5 :info :write [1 2] :RequestCanceledException INFO [2019-12-17 12:27:56,288] jepsen worker 87 - jepsen.util 267 :invoke :write [29 29] INFO [2019-12-17 12:27:56,288] jepsen worker 5 - jepsen.util 95 :invoke :write [1 11] INFO [2019-12-17 12:27:56,288] jepsen worker 39 - jepsen.util 399 :invoke :write [13 31] INFO [2019-12-17 12:27:56,303] jepsen worker 58 - jepsen.util 148 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:27:56,303] jepsen worker 58 - jepsen.util 148 :invoke :write [19 33] INFO [2019-12-17 12:27:56,329] jepsen worker 72 - jepsen.util 252 :fail :cas [24 [13 29]] :ValueNotSwapFrom INFO [2019-12-17 12:27:56,329] jepsen worker 72 - jepsen.util 252 :invoke :write [24 25] INFO [2019-12-17 12:27:56,334] jepsen worker 23 - jepsen.util 113 :info :write [7 2] :RequestCanceledException INFO [2019-12-17 12:27:56,336] jepsen worker 23 - jepsen.util 203 :invoke :cas [7 [29 14]] INFO [2019-12-17 12:27:56,339] jepsen worker 7 - jepsen.util 457 :ok :read [2 15] INFO [2019-12-17 12:27:56,340] jepsen worker 7 - jepsen.util 457 :invoke :cas [2 [22 38]] INFO [2019-12-17 12:27:56,369] jepsen worker 64 - jepsen.util 334 :ok :write [21 4] INFO [2019-12-17 12:27:56,369] jepsen worker 64 - jepsen.util 334 :invoke :write [21 16] INFO [2019-12-17 12:27:56,378] jepsen worker 26 - jepsen.util 476 :fail :cas [8 [20 45]] :ValueNotSwapFrom INFO [2019-12-17 12:27:56,378] jepsen worker 26 - jepsen.util 476 :invoke :write [8 0] INFO [2019-12-17 12:27:56,387] jepsen worker 9 - jepsen.util 189 :info :write [3 43] :RequestCanceledException INFO [2019-12-17 12:27:56,388] jepsen worker 9 - jepsen.util 279 :invoke :cas [3 [32 42]] INFO [2019-12-17 12:27:56,402] jepsen worker 53 - jepsen.util 503 :fail :cas [17 [21 3]] :ValueNotSwapFrom INFO [2019-12-17 12:27:56,402] jepsen worker 53 - jepsen.util 503 :invoke :read [17 nil] INFO [2019-12-17 12:27:56,404] jepsen worker 57 - jepsen.util 327 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:27:56,404] jepsen worker 57 - jepsen.util 327 :invoke :write [19 18] INFO [2019-12-17 12:27:56,420] jepsen worker 18 - jepsen.util 378 :info :write [6 10] :RequestCanceledException INFO [2019-12-17 12:27:56,421] jepsen worker 42 - jepsen.util 492 :info :write [14 27] :RequestCanceledException INFO [2019-12-17 12:27:56,422] jepsen worker 18 - jepsen.util 468 :invoke :read [6 nil] INFO [2019-12-17 12:27:56,422] jepsen worker 88 - jepsen.util 358 :fail :cas [29 [18 14]] :ValueNotSwapFrom INFO [2019-12-17 12:27:56,422] jepsen worker 88 - jepsen.util 358 :invoke :cas [29 [33 12]] INFO [2019-12-17 12:27:56,422] jepsen worker 42 - jepsen.util 582 :invoke :cas [14 [29 14]] INFO [2019-12-17 12:27:56,435] jepsen worker 11 - jepsen.util 281 :info :cas [3 [30 31]] :RequestCanceledException INFO [2019-12-17 12:27:56,436] jepsen worker 11 - jepsen.util 371 :invoke :read [3 nil] INFO [2019-12-17 12:27:56,451] jepsen worker 64 - jepsen.util 334 :ok :write [21 16] INFO [2019-12-17 12:27:56,452] jepsen worker 73 - jepsen.util 343 :info :cas [24 [7 15]] :RequestCanceledException INFO [2019-12-17 12:27:56,452] jepsen worker 64 - jepsen.util 334 :invoke :cas [21 [41 23]] INFO [2019-12-17 12:27:56,453] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1910,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:27:56,453] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1910,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:27:56,453] jepsen worker 73 - jepsen.util 433 :invoke :read [24 nil] INFO [2019-12-17 12:27:56,459] jepsen worker 11 - jepsen.util 371 :ok :read [3 14] INFO [2019-12-17 12:27:56,460] jepsen worker 11 - jepsen.util 371 :invoke :cas [3 [19 46]] INFO [2019-12-17 12:27:56,487] jepsen worker 10 - jepsen.util 280 :info :cas [3 [25 26]] :RequestCanceledException INFO [2019-12-17 12:27:56,488] jepsen worker 10 - jepsen.util 370 :invoke :read [3 nil] INFO [2019-12-17 12:27:56,491] jepsen worker 0 - jepsen.util 360 :info :write [0 9] :SyncWriteAmbiguous INFO [2019-12-17 12:27:56,492] jepsen worker 0 - jepsen.util 450 :invoke :cas [0 [29 25]] INFO [2019-12-17 12:27:56,495] jepsen worker 75 - jepsen.util 345 :fail :cas [25 [28 39]] :ValueNotSwapFrom INFO [2019-12-17 12:27:56,496] jepsen worker 75 - jepsen.util 345 :invoke :read [25 nil] INFO [2019-12-17 12:27:56,496] jepsen worker 73 - jepsen.util 433 :ok :read [24 35] INFO [2019-12-17 12:27:56,496] jepsen worker 73 - jepsen.util 433 :invoke :cas [24 [35 31]] INFO [2019-12-17 12:27:56,501] jepsen worker 11 - jepsen.util 371 :fail :cas [3 [19 46]] :ValueNotSwapFrom INFO [2019-12-17 12:27:56,501] jepsen worker 64 - jepsen.util 334 :fail :cas [21 [41 23]] :ValueNotSwapFrom INFO [2019-12-17 12:27:56,501] jepsen worker 64 - jepsen.util 334 :invoke :write [21 10] INFO [2019-12-17 12:27:56,501] jepsen worker 11 - jepsen.util 371 :invoke :cas [3 [8 37]] INFO [2019-12-17 12:27:56,520] jepsen worker 63 - jepsen.util 153 :info :cas [21 [38 7]] :RequestCanceledException INFO [2019-12-17 12:27:56,522] jepsen worker 63 - jepsen.util 243 :invoke :write [21 19] INFO [2019-12-17 12:27:56,535] jepsen worker 47 - jepsen.util 407 :info :write [15 29] :RequestCanceledException INFO [2019-12-17 12:27:56,535] jepsen worker 20 - jepsen.util 380 :fail :read [6 nil] :RequestTimeoutException INFO [2019-12-17 12:27:56,535] jepsen worker 62 - jepsen.util 332 :info :write [20 0] :RequestCanceledException INFO [2019-12-17 12:27:56,535] jepsen worker 20 - jepsen.util 380 :invoke :write [6 1] INFO [2019-12-17 12:27:56,536] jepsen worker 47 - jepsen.util 497 :invoke :write [15 37] INFO [2019-12-17 12:27:56,536] jepsen worker 62 - jepsen.util 422 :invoke :write [20 36] INFO [2019-12-17 12:27:56,566] jepsen worker 59 - jepsen.util 239 :ok :read [19 8] INFO [2019-12-17 12:27:56,567] jepsen worker 59 - jepsen.util 239 :invoke :write [19 9] INFO [2019-12-17 12:27:56,586] jepsen worker 24 - jepsen.util 294 :fail :cas [8 [26 2]] :ValueNotSwapFrom INFO [2019-12-17 12:27:56,586] jepsen worker 24 - jepsen.util 294 :invoke :write [8 42] INFO [2019-12-17 12:27:56,592] jepsen worker 10 - jepsen.util 370 :ok :read [3 14] INFO [2019-12-17 12:27:56,592] jepsen worker 10 - jepsen.util 370 :invoke :read [3 nil] INFO [2019-12-17 12:27:56,593] jepsen worker 49 - jepsen.util 319 :ok :read [16 8] INFO [2019-12-17 12:27:56,593] jepsen worker 49 - jepsen.util 319 :invoke :read [16 nil] INFO [2019-12-17 12:27:56,635] jepsen worker 13 - jepsen.util 283 :info :write [4 46] :RequestCanceledException INFO [2019-12-17 12:27:56,636] jepsen worker 13 - jepsen.util 373 :invoke :read [4 nil] INFO [2019-12-17 12:27:56,640] jepsen worker 35 - jepsen.util 215 :ok :read [11 7] INFO [2019-12-17 12:27:56,640] jepsen worker 35 - jepsen.util 215 :invoke :cas [11 [35 39]] INFO [2019-12-17 12:27:56,675] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.03515625}, :ns_1@172.28.128.126 {:progress 0.03237410071942448}, :ns_1@172.28.128.127 {:progress 0.08560311284046696}, :ns_1@172.28.128.128 {:progress 0.07547169811320753}} INFO [2019-12-17 12:27:56,675] jepsen nemesis - couchbase.util Rebalance check count: 8 INFO [2019-12-17 12:27:56,675] jepsen nemesis - couchbase.util Rebalance stuck count: 0 INFO [2019-12-17 12:27:56,678] jepsen worker 7 - jepsen.util 457 :fail :cas [2 [22 38]] :ValueNotSwapFrom INFO [2019-12-17 12:27:56,678] jepsen worker 7 - jepsen.util 457 :invoke :write [2 20] INFO [2019-12-17 12:27:56,679] jepsen worker 11 - jepsen.util 371 :fail :cas [3 [8 37]] :ValueNotSwapFrom INFO [2019-12-17 12:27:56,680] jepsen worker 11 - jepsen.util 371 :invoke :cas [3 [25 40]] INFO [2019-12-17 12:27:56,698] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1938,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:27:56,698] jepsen worker 80 - jepsen.util 530 :info :write [26 42] :RequestCanceledException INFO [2019-12-17 12:27:56,698] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1938,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:27:56,699] jepsen worker 80 - jepsen.util 620 :invoke :cas [26 [9 28]] INFO [2019-12-17 12:27:56,720] jepsen worker 76 - jepsen.util 256 :info :write [25 36] :RequestCanceledException INFO [2019-12-17 12:27:56,722] jepsen worker 76 - jepsen.util 346 :invoke :write [25 19] INFO [2019-12-17 12:27:56,727] jepsen worker 53 - jepsen.util 503 :ok :read [17 0] INFO [2019-12-17 12:27:56,727] jepsen worker 11 - jepsen.util 371 :fail :cas [3 [25 40]] :ValueNotSwapFrom INFO [2019-12-17 12:27:56,727] jepsen worker 53 - jepsen.util 503 :invoke :read [17 nil] INFO [2019-12-17 12:27:56,727] jepsen worker 11 - jepsen.util 371 :invoke :cas [3 [23 31]] INFO [2019-12-17 12:27:56,749] jepsen worker 12 - jepsen.util 282 :fail :cas [4 [35 18]] :ValueNotSwapFrom INFO [2019-12-17 12:27:56,749] jepsen worker 12 - jepsen.util 282 :invoke :write [4 36] INFO [2019-12-17 12:27:56,759] jepsen worker 11 - jepsen.util 371 :fail :cas [3 [23 31]] :ValueNotSwapFrom INFO [2019-12-17 12:27:56,759] jepsen worker 11 - jepsen.util 371 :invoke :cas [3 [13 39]] INFO [2019-12-17 12:27:56,775] jepsen worker 64 - jepsen.util 334 :ok :write [21 10] INFO [2019-12-17 12:27:56,775] jepsen worker 64 - jepsen.util 334 :invoke :read [21 nil] INFO [2019-12-17 12:27:56,802] jepsen worker 71 - jepsen.util 341 :ok :read [23 29] INFO [2019-12-17 12:27:56,802] jepsen worker 71 - jepsen.util 341 :invoke :write [23 5] INFO [2019-12-17 12:27:56,806] jepsen worker 11 - jepsen.util 371 :fail :cas [3 [13 39]] :ValueNotSwapFrom INFO [2019-12-17 12:27:56,806] jepsen worker 11 - jepsen.util 371 :invoke :read [3 nil] INFO [2019-12-17 12:27:56,825] jepsen worker 64 - jepsen.util 334 :ok :read [21 10] INFO [2019-12-17 12:27:56,825] jepsen worker 64 - jepsen.util 334 :invoke :cas [21 [15 19]] INFO [2019-12-17 12:27:56,844] jepsen worker 11 - jepsen.util 371 :ok :read [3 14] INFO [2019-12-17 12:27:56,844] jepsen worker 11 - jepsen.util 371 :invoke :read [3 nil] INFO [2019-12-17 12:27:56,857] jepsen worker 64 - jepsen.util 334 :fail :cas [21 [15 19]] :ValueNotSwapFrom INFO [2019-12-17 12:27:56,857] jepsen worker 64 - jepsen.util 334 :invoke :write [21 0] INFO [2019-12-17 12:27:56,884] jepsen worker 51 - jepsen.util 231 :fail :cas [17 [47 32]] :ValueNotSwapFrom INFO [2019-12-17 12:27:56,884] jepsen worker 11 - jepsen.util 371 :ok :read [3 14] INFO [2019-12-17 12:27:56,884] jepsen worker 51 - jepsen.util 231 :invoke :cas [17 [20 34]] INFO [2019-12-17 12:27:56,884] jepsen worker 11 - jepsen.util 371 :invoke :read [3 nil] INFO [2019-12-17 12:27:56,887] jepsen worker 69 - jepsen.util 339 :info :write [23 1] :RequestCanceledException INFO [2019-12-17 12:27:56,889] jepsen worker 69 - jepsen.util 429 :invoke :write [23 1] INFO [2019-12-17 12:27:56,909] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1955,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:27:56,909] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1955,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:27:56,909] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1958,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:27:56,909] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1958,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:27:56,918] jepsen worker 80 - jepsen.util 620 :fail :cas [26 [9 28]] :ValueNotSwapFrom INFO [2019-12-17 12:27:56,918] jepsen worker 80 - jepsen.util 620 :invoke :write [26 19] INFO [2019-12-17 12:27:56,920] jepsen worker 67 - jepsen.util 427 :info :cas [22 [21 24]] :RequestCanceledException INFO [2019-12-17 12:27:56,921] jepsen worker 67 - jepsen.util 517 :invoke :write [22 45] INFO [2019-12-17 12:27:56,931] jepsen worker 11 - jepsen.util 371 :ok :read [3 14] INFO [2019-12-17 12:27:56,931] jepsen worker 11 - jepsen.util 371 :invoke :write [3 48] INFO [2019-12-17 12:27:56,948] jepsen worker 64 - jepsen.util 334 :ok :write [21 0] INFO [2019-12-17 12:27:56,948] jepsen worker 64 - jepsen.util 334 :invoke :read [21 nil] INFO [2019-12-17 12:27:56,970] jepsen worker 42 - jepsen.util 582 :fail :cas [14 [29 14]] :ValueNotSwapFrom INFO [2019-12-17 12:27:56,971] jepsen worker 42 - jepsen.util 582 :invoke :write [14 49] INFO [2019-12-17 12:27:56,987] jepsen worker 43 - jepsen.util 493 :info :write [14 48] :RequestCanceledException INFO [2019-12-17 12:27:56,988] jepsen worker 43 - jepsen.util 583 :invoke :cas [14 [13 29]] INFO [2019-12-17 12:27:56,990] jepsen worker 64 - jepsen.util 334 :ok :read [21 0] INFO [2019-12-17 12:27:56,990] jepsen worker 64 - jepsen.util 334 :invoke :read [21 nil] INFO [2019-12-17 12:27:56,995] jepsen worker 53 - jepsen.util 503 :ok :read [17 0] INFO [2019-12-17 12:27:56,995] jepsen worker 53 - jepsen.util 503 :invoke :cas [17 [16 40]] INFO [2019-12-17 12:27:56,996] jepsen worker 54 - jepsen.util 504 :ok :read [18 47] INFO [2019-12-17 12:27:56,996] jepsen worker 54 - jepsen.util 504 :invoke :read [18 nil] INFO [2019-12-17 12:27:57,016] jepsen worker 64 - jepsen.util 334 :ok :read [21 0] INFO [2019-12-17 12:27:57,016] jepsen worker 64 - jepsen.util 334 :invoke :write [21 14] INFO [2019-12-17 12:27:57,018] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1969,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:27:57,018] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1969,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:27:57,018] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1971,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:27:57,018] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1971,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:27:57,069] jepsen worker 25 - jepsen.util 475 :fail :cas [8 [40 43]] :ValueNotSwapFrom INFO [2019-12-17 12:27:57,070] jepsen worker 25 - jepsen.util 475 :invoke :read [8 nil] INFO [2019-12-17 12:27:57,094] jepsen worker 64 - jepsen.util 334 :ok :write [21 14] INFO [2019-12-17 12:27:57,095] jepsen worker 64 - jepsen.util 334 :invoke :cas [21 [24 45]] INFO [2019-12-17 12:27:57,109] jepsen worker 30 - jepsen.util 210 :info :write [10 21] :SyncWriteAmbiguous INFO [2019-12-17 12:27:57,110] jepsen worker 30 - jepsen.util 300 :invoke :write [10 9] INFO [2019-12-17 12:27:57,114] jepsen worker 3 - jepsen.util 273 :info :write [1 34] :RequestCanceledException INFO [2019-12-17 12:27:57,116] jepsen worker 3 - jepsen.util 363 :invoke :write [1 27] INFO [2019-12-17 12:27:57,116] jepsen worker 64 - jepsen.util 334 :fail :cas [21 [24 45]] :ValueNotSwapFrom INFO [2019-12-17 12:27:57,116] jepsen worker 64 - jepsen.util 334 :invoke :cas [21 [25 5]] INFO [2019-12-17 12:27:57,125] jepsen worker 75 - jepsen.util 345 :ok :read [25 2] INFO [2019-12-17 12:27:57,125] jepsen worker 75 - jepsen.util 345 :invoke :read [25 nil] INFO [2019-12-17 12:27:57,134] jepsen worker 64 - jepsen.util 334 :fail :cas [21 [25 5]] :ValueNotSwapFrom INFO [2019-12-17 12:27:57,134] jepsen worker 64 - jepsen.util 334 :invoke :cas [21 [34 9]] INFO [2019-12-17 12:27:57,153] jepsen worker 64 - jepsen.util 334 :fail :cas [21 [34 9]] :ValueNotSwapFrom INFO [2019-12-17 12:27:57,154] jepsen worker 64 - jepsen.util 334 :invoke :write [21 45] INFO [2019-12-17 12:27:57,183] jepsen worker 38 - jepsen.util 218 :fail :cas [12 [37 18]] :ValueNotSwapFrom INFO [2019-12-17 12:27:57,184] jepsen worker 38 - jepsen.util 218 :invoke :cas [12 [13 40]] INFO [2019-12-17 12:27:57,210] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1996,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:27:57,211] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1996,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:27:57,211] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1999,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:27:57,211] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1999,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:27:57,212] jepsen worker 50 - jepsen.util 410 :info :write [16 17] :RequestCanceledException INFO [2019-12-17 12:27:57,213] jepsen worker 70 - jepsen.util 520 :ok :read [23 29] INFO [2019-12-17 12:27:57,213] jepsen worker 70 - jepsen.util 520 :invoke :cas [23 [20 17]] INFO [2019-12-17 12:27:57,214] jepsen worker 50 - jepsen.util 500 :invoke :cas [16 [1 43]] INFO [2019-12-17 12:27:57,228] jepsen worker 64 - jepsen.util 334 :ok :write [21 45] INFO [2019-12-17 12:27:57,228] jepsen worker 64 - jepsen.util 334 :invoke :read [21 nil] INFO [2019-12-17 12:27:57,234] jepsen worker 37 - jepsen.util 217 :info :write [12 34] :RequestCanceledException INFO [2019-12-17 12:27:57,236] jepsen worker 37 - jepsen.util 307 :invoke :write [12 21] INFO [2019-12-17 12:27:57,245] jepsen worker 64 - jepsen.util 334 :ok :read [21 45] INFO [2019-12-17 12:27:57,245] jepsen worker 64 - jepsen.util 334 :invoke :write [21 36] INFO [2019-12-17 12:27:57,251] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1990,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:27:57,251] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1990,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:27:57,251] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1991,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:27:57,251] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1991,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:27:57,264] jepsen worker 54 - jepsen.util 504 :ok :read [18 47] INFO [2019-12-17 12:27:57,264] jepsen worker 54 - jepsen.util 504 :invoke :cas [18 [47 22]] INFO [2019-12-17 12:27:57,276] jepsen worker 44 - jepsen.util 224 :ok :read [14 27] INFO [2019-12-17 12:27:57,277] jepsen worker 44 - jepsen.util 224 :invoke :read [14 nil] INFO [2019-12-17 12:27:57,283] jepsen worker 4 - jepsen.util 364 :fail :cas [1 [44 2]] :ValueNotSwapFrom INFO [2019-12-17 12:27:57,283] jepsen worker 4 - jepsen.util 364 :invoke :read [1 nil] INFO [2019-12-17 12:27:57,286] jepsen worker 88 - jepsen.util 358 :fail :cas [29 [33 12]] :ValueNotSwapFrom INFO [2019-12-17 12:27:57,286] jepsen worker 88 - jepsen.util 358 :invoke :cas [29 [45 1]] INFO [2019-12-17 12:27:57,305] jepsen worker 74 - jepsen.util 344 :fail :read [24 nil] :RequestTimeoutException INFO [2019-12-17 12:27:57,305] jepsen worker 74 - jepsen.util 344 :invoke :read [24 nil] INFO [2019-12-17 12:27:57,324] jepsen worker 35 - jepsen.util 215 :fail :cas [11 [35 39]] :ValueNotSwapFrom INFO [2019-12-17 12:27:57,324] jepsen worker 35 - jepsen.util 215 :invoke :write [11 49] INFO [2019-12-17 12:27:57,330] jepsen worker 74 - jepsen.util 344 :ok :read [24 35] INFO [2019-12-17 12:27:57,330] jepsen worker 74 - jepsen.util 344 :invoke :cas [24 [21 46]] INFO [2019-12-17 12:27:57,351] jepsen worker 28 - jepsen.util 298 :fail :read [9 nil] :RequestTimeoutException INFO [2019-12-17 12:27:57,351] jepsen worker 28 - jepsen.util 298 :invoke :write [9 27] INFO [2019-12-17 12:27:57,355] jepsen worker 17 - jepsen.util 377 :info :write [5 22] :SyncWriteAmbiguous INFO [2019-12-17 12:27:57,357] jepsen worker 17 - jepsen.util 467 :invoke :write [5 9] INFO [2019-12-17 12:27:57,369] jepsen worker 74 - jepsen.util 344 :fail :cas [24 [21 46]] :ValueNotSwapFrom INFO [2019-12-17 12:27:57,369] jepsen worker 74 - jepsen.util 344 :invoke :read [24 nil] INFO [2019-12-17 12:27:57,379] jepsen worker 2 - jepsen.util 272 :ok :read [0 14] INFO [2019-12-17 12:27:57,379] jepsen worker 2 - jepsen.util 272 :invoke :write [0 2] INFO [2019-12-17 12:27:57,387] jepsen worker 56 - jepsen.util 326 :info :write [18 26] :RequestCanceledException INFO [2019-12-17 12:27:57,389] jepsen worker 56 - jepsen.util 416 :invoke :cas [18 [7 42]] INFO [2019-12-17 12:27:57,390] jepsen worker 74 - jepsen.util 344 :ok :read [24 35] INFO [2019-12-17 12:27:57,390] jepsen worker 74 - jepsen.util 344 :invoke :cas [24 [3 32]] INFO [2019-12-17 12:27:57,403] jepsen worker 78 - jepsen.util 438 :ok :read [26 47] INFO [2019-12-17 12:27:57,404] jepsen worker 78 - jepsen.util 438 :invoke :write [26 45] INFO [2019-12-17 12:27:57,418] jepsen worker 64 - jepsen.util 334 :ok :write [21 36] INFO [2019-12-17 12:27:57,418] jepsen worker 64 - jepsen.util 334 :invoke :write [21 14] INFO [2019-12-17 12:27:57,428] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2023,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:27:57,428] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2023,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:27:57,428] jepsen worker 74 - jepsen.util 344 :fail :cas [24 [3 32]] :ValueNotSwapFrom INFO [2019-12-17 12:27:57,428] jepsen worker 74 - jepsen.util 344 :invoke :write [24 21] INFO [2019-12-17 12:27:57,435] jepsen worker 19 - jepsen.util 289 :info :write [6 15] :RequestCanceledException INFO [2019-12-17 12:27:57,435] jepsen worker 61 - jepsen.util 421 :info :write [20 38] :RequestCanceledException INFO [2019-12-17 12:27:57,436] jepsen worker 19 - jepsen.util 379 :invoke :write [6 18] INFO [2019-12-17 12:27:57,436] jepsen worker 61 - jepsen.util 511 :invoke :write [20 25] INFO [2019-12-17 12:27:57,453] jepsen worker 53 - jepsen.util 503 :fail :cas [17 [16 40]] :ValueNotSwapFrom INFO [2019-12-17 12:27:57,453] jepsen worker 53 - jepsen.util 503 :invoke :cas [17 [24 9]] INFO [2019-12-17 12:27:57,493] jepsen worker 25 - jepsen.util 475 :ok :read [8 14] INFO [2019-12-17 12:27:57,493] jepsen worker 25 - jepsen.util 475 :invoke :read [8 nil] INFO [2019-12-17 12:27:57,512] jepsen worker 64 - jepsen.util 334 :ok :write [21 14] INFO [2019-12-17 12:27:57,512] jepsen worker 64 - jepsen.util 334 :invoke :write [21 34] INFO [2019-12-17 12:27:57,521] jepsen worker 14 - jepsen.util 374 :info :write [4 18] :RequestCanceledException INFO [2019-12-17 12:27:57,522] jepsen worker 14 - jepsen.util 464 :invoke :read [4 nil] INFO [2019-12-17 12:27:57,569] jepsen worker 64 - jepsen.util 334 :ok :write [21 34] INFO [2019-12-17 12:27:57,569] jepsen worker 64 - jepsen.util 334 :invoke :cas [21 [17 44]] INFO [2019-12-17 12:27:57,584] jepsen worker 64 - jepsen.util 334 :fail :cas [21 [17 44]] :ValueNotSwapFrom INFO [2019-12-17 12:27:57,584] jepsen worker 64 - jepsen.util 334 :invoke :cas [21 [8 2]] INFO [2019-12-17 12:27:57,586] jepsen worker 38 - jepsen.util 218 :fail :cas [12 [13 40]] :ValueNotSwapFrom INFO [2019-12-17 12:27:57,587] jepsen worker 38 - jepsen.util 218 :invoke :write [12 4] INFO [2019-12-17 12:27:57,605] jepsen worker 21 - jepsen.util 291 :info :write [7 48] :RequestCanceledException INFO [2019-12-17 12:27:57,605] jepsen worker 64 - jepsen.util 334 :fail :cas [21 [8 2]] :ValueNotSwapFrom INFO [2019-12-17 12:27:57,605] jepsen worker 64 - jepsen.util 334 :invoke :cas [21 [42 2]] INFO [2019-12-17 12:27:57,606] jepsen worker 21 - jepsen.util 381 :invoke :read [7 nil] INFO [2019-12-17 12:27:57,620] jepsen worker 15 - jepsen.util 375 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:27:57,620] jepsen worker 79 - jepsen.util 349 :info :write [26 13] :RequestCanceledException INFO [2019-12-17 12:27:57,620] jepsen worker 84 - jepsen.util 84 :info :write [28 13] :RequestCanceledException INFO [2019-12-17 12:27:57,621] jepsen worker 15 - jepsen.util 375 :invoke :write [5 44] INFO [2019-12-17 12:27:57,622] jepsen worker 79 - jepsen.util 439 :invoke :cas [26 [25 18]] INFO [2019-12-17 12:27:57,622] jepsen worker 84 - jepsen.util 174 :invoke :write [28 42] INFO [2019-12-17 12:27:57,631] jepsen worker 64 - jepsen.util 334 :fail :cas [21 [42 2]] :ValueNotSwapFrom INFO [2019-12-17 12:27:57,631] jepsen worker 64 - jepsen.util 334 :invoke :write [21 16] INFO [2019-12-17 12:27:57,639] jepsen worker 13 - jepsen.util 373 :ok :read [4 4] INFO [2019-12-17 12:27:57,639] jepsen worker 13 - jepsen.util 373 :invoke :write [4 29] INFO [2019-12-17 12:27:57,673] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestTimeoutException: CarrierBucketConfigRequest {"retried":0,"reason":"TIMEOUT","requestId":1708,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:27:57,673] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2044,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:27:57,674] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2044,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:27:57,674] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2047,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:27:57,674] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2047,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:27:57,687] jepsen worker 82 - jepsen.util 442 :info :write [27 36] :RequestCanceledException INFO [2019-12-17 12:27:57,687] jepsen worker 64 - jepsen.util 334 :ok :write [21 16] INFO [2019-12-17 12:27:57,687] jepsen worker 64 - jepsen.util 334 :invoke :write [21 46] INFO [2019-12-17 12:27:57,688] jepsen worker 82 - jepsen.util 532 :invoke :cas [27 [25 46]] INFO [2019-12-17 12:27:57,693] jepsen worker 4 - jepsen.util 364 :ok :read [1 23] INFO [2019-12-17 12:27:57,693] jepsen worker 4 - jepsen.util 364 :invoke :write [1 33] INFO [2019-12-17 12:27:57,693] jepsen worker 21 - jepsen.util 381 :ok :read [7 20] INFO [2019-12-17 12:27:57,693] jepsen worker 21 - jepsen.util 381 :invoke :cas [7 [49 17]] INFO [2019-12-17 12:27:57,716] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.046875}, :ns_1@172.28.128.126 {:progress 0.04316546762589923}, :ns_1@172.28.128.127 {:progress 0.08560311284046696}, :ns_1@172.28.128.128 {:progress 0.1056603773584905}} INFO [2019-12-17 12:27:57,716] jepsen nemesis - couchbase.util Rebalance check count: 9 INFO [2019-12-17 12:27:57,716] jepsen nemesis - couchbase.util Rebalance stuck count: 0 INFO [2019-12-17 12:27:57,735] jepsen worker 41 - jepsen.util 311 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:27:57,735] jepsen worker 16 - jepsen.util 286 :info :cas [5 [22 24]] :RequestCanceledException INFO [2019-12-17 12:27:57,735] jepsen worker 41 - jepsen.util 311 :invoke :cas [13 [37 13]] INFO [2019-12-17 12:27:57,736] jepsen worker 16 - jepsen.util 376 :invoke :write [5 45] INFO [2019-12-17 12:27:57,744] jepsen worker 21 - jepsen.util 381 :fail :cas [7 [49 17]] :ValueNotSwapFrom INFO [2019-12-17 12:27:57,744] jepsen worker 21 - jepsen.util 381 :invoke :write [7 7] INFO [2019-12-17 12:27:57,751] jepsen worker 68 - jepsen.util 338 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:27:57,752] jepsen worker 68 - jepsen.util 338 :invoke :read [22 nil] INFO [2019-12-17 12:27:57,768] jepsen worker 64 - jepsen.util 334 :ok :write [21 46] INFO [2019-12-17 12:27:57,769] jepsen worker 64 - jepsen.util 334 :invoke :read [21 nil] INFO [2019-12-17 12:27:57,789] jepsen worker 64 - jepsen.util 334 :ok :read [21 46] INFO [2019-12-17 12:27:57,789] jepsen worker 64 - jepsen.util 334 :invoke :read [21 nil] INFO [2019-12-17 12:27:57,821] jepsen worker 66 - jepsen.util 336 :info :cas [22 [4 34]] :RequestCanceledException INFO [2019-12-17 12:27:57,823] jepsen worker 51 - jepsen.util 231 :fail :cas [17 [20 34]] :ValueNotSwapFrom INFO [2019-12-17 12:27:57,823] jepsen worker 51 - jepsen.util 231 :invoke :write [17 40] INFO [2019-12-17 12:27:57,823] jepsen worker 66 - jepsen.util 426 :invoke :read [22 nil] INFO [2019-12-17 12:27:57,835] jepsen worker 64 - jepsen.util 334 :ok :read [21 46] INFO [2019-12-17 12:27:57,836] jepsen worker 64 - jepsen.util 334 :invoke :write [21 35] INFO [2019-12-17 12:27:57,887] jepsen worker 45 - jepsen.util 495 :info :write [15 40] :RequestCanceledException INFO [2019-12-17 12:27:57,888] jepsen worker 45 - jepsen.util 585 :invoke :write [15 26] INFO [2019-12-17 12:27:57,889] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2064,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:27:57,889] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2064,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:27:57,889] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2067,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:27:57,890] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2067,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:27:57,906] jepsen worker 32 - jepsen.util 302 :info :write [10 33] :RequestCanceledException INFO [2019-12-17 12:27:57,907] jepsen worker 32 - jepsen.util 392 :invoke :cas [10 [46 49]] INFO [2019-12-17 12:27:57,913] jepsen worker 64 - jepsen.util 334 :ok :write [21 35] INFO [2019-12-17 12:27:57,913] jepsen worker 64 - jepsen.util 334 :invoke :cas [21 [32 28]] INFO [2019-12-17 12:27:57,920] jepsen worker 48 - jepsen.util 408 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:27:57,921] jepsen worker 48 - jepsen.util 408 :invoke :cas [16 [7 33]] INFO [2019-12-17 12:27:57,924] jepsen worker 53 - jepsen.util 503 :fail :cas [17 [24 9]] :ValueNotSwapFrom INFO [2019-12-17 12:27:57,925] jepsen worker 53 - jepsen.util 503 :invoke :write [17 40] INFO [2019-12-17 12:27:57,929] jepsen worker 40 - jepsen.util 490 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:27:57,929] jepsen worker 40 - jepsen.util 490 :invoke :cas [13 [27 47]] INFO [2019-12-17 12:27:57,934] jepsen worker 64 - jepsen.util 334 :fail :cas [21 [32 28]] :ValueNotSwapFrom INFO [2019-12-17 12:27:57,934] jepsen worker 64 - jepsen.util 334 :invoke :write [21 39] INFO [2019-12-17 12:27:57,948] jepsen worker 52 - jepsen.util 322 :info :write [17 19] :RequestCanceledException INFO [2019-12-17 12:27:57,948] jepsen worker 77 - jepsen.util 347 :info :write [25 29] :RequestCanceledException INFO [2019-12-17 12:27:57,950] jepsen worker 52 - jepsen.util 412 :invoke :read [17 nil] INFO [2019-12-17 12:27:57,950] jepsen worker 77 - jepsen.util 437 :invoke :cas [25 [25 40]] INFO [2019-12-17 12:27:57,999] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2081,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:27:57,999] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2081,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:27:57,999] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2083,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:27:57,999] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2083,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:27:58,003] jepsen worker 60 - jepsen.util 510 :info :write [20 24] :RequestCanceledException INFO [2019-12-17 12:27:58,004] jepsen worker 60 - jepsen.util 600 :invoke :read [20 nil] INFO [2019-12-17 12:27:58,014] jepsen worker 64 - jepsen.util 334 :ok :write [21 39] INFO [2019-12-17 12:27:58,014] jepsen worker 64 - jepsen.util 334 :invoke :write [21 6] INFO [2019-12-17 12:27:58,021] jepsen worker 6 - jepsen.util 366 :info :write [2 15] :RequestCanceledException INFO [2019-12-17 12:27:58,021] jepsen worker 29 - jepsen.util 299 :info :write [9 47] :RequestCanceledException INFO [2019-12-17 12:27:58,021] jepsen worker 46 - jepsen.util 316 :fail :read [15 nil] :RequestTimeoutException INFO [2019-12-17 12:27:58,021] jepsen worker 46 - jepsen.util 316 :invoke :cas [15 [36 46]] INFO [2019-12-17 12:27:58,022] jepsen worker 6 - jepsen.util 456 :invoke :write [2 3] INFO [2019-12-17 12:27:58,022] jepsen worker 29 - jepsen.util 389 :invoke :write [9 43] INFO [2019-12-17 12:27:58,047] jepsen worker 31 - jepsen.util 571 :info :write [10 38] :RequestCanceledException INFO [2019-12-17 12:27:58,048] jepsen worker 31 - jepsen.util 661 :invoke :cas [10 [9 48]] INFO [2019-12-17 12:27:58,053] jepsen worker 70 - jepsen.util 520 :fail :cas [23 [20 17]] :ValueNotSwapFrom INFO [2019-12-17 12:27:58,053] jepsen worker 25 - jepsen.util 475 :ok :read [8 14] INFO [2019-12-17 12:27:58,053] jepsen worker 70 - jepsen.util 520 :invoke :write [23 49] INFO [2019-12-17 12:27:58,053] jepsen worker 25 - jepsen.util 475 :invoke :cas [8 [44 42]] INFO [2019-12-17 12:27:58,085] jepsen worker 81 - jepsen.util 351 :info :write [27 23] :RequestCanceledException INFO [2019-12-17 12:27:58,087] jepsen worker 81 - jepsen.util 441 :invoke :read [27 nil] INFO [2019-12-17 12:27:58,099] jepsen worker 14 - jepsen.util 464 :ok :read [4 4] INFO [2019-12-17 12:27:58,100] jepsen worker 14 - jepsen.util 464 :invoke :read [4 nil] INFO [2019-12-17 12:27:58,104] jepsen worker 27 - jepsen.util 117 :info :write [9 42] :RequestCanceledException INFO [2019-12-17 12:27:58,105] jepsen worker 27 - jepsen.util 207 :invoke :cas [9 [11 33]] INFO [2019-12-17 12:27:58,106] jepsen worker 64 - jepsen.util 334 :ok :write [21 6] INFO [2019-12-17 12:27:58,107] jepsen worker 64 - jepsen.util 334 :invoke :write [21 24] INFO [2019-12-17 12:27:58,121] jepsen worker 36 - jepsen.util 396 :info :write [12 12] :RequestCanceledException INFO [2019-12-17 12:27:58,121] jepsen worker 1 - jepsen.util 451 :info :cas [0 [35 31]] :RequestCanceledException INFO [2019-12-17 12:27:58,123] jepsen worker 36 - jepsen.util 486 :invoke :read [12 nil] INFO [2019-12-17 12:27:58,123] jepsen worker 1 - jepsen.util 541 :invoke :cas [0 [41 12]] INFO [2019-12-17 12:27:58,141] jepsen worker 88 - jepsen.util 358 :fail :cas [29 [45 1]] :ValueNotSwapFrom INFO [2019-12-17 12:27:58,141] jepsen worker 88 - jepsen.util 358 :invoke :read [29 nil] INFO [2019-12-17 12:27:58,179] jepsen worker 72 - jepsen.util 252 :info :write [24 25] :SyncWriteAmbiguous INFO [2019-12-17 12:27:58,181] jepsen worker 72 - jepsen.util 342 :invoke :write [24 29] INFO [2019-12-17 12:27:58,196] jepsen worker 18 - jepsen.util 468 :ok :read [6 45] INFO [2019-12-17 12:27:58,196] jepsen worker 18 - jepsen.util 468 :invoke :write [6 49] INFO [2019-12-17 12:27:58,199] jepsen worker 55 - jepsen.util 505 :info :write [18 10] :RequestCanceledException INFO [2019-12-17 12:27:58,199] jepsen worker 34 - jepsen.util 394 :info :write [11 3] :RequestCanceledException INFO [2019-12-17 12:27:58,199] jepsen worker 89 - jepsen.util 359 :info :write [29 5] :RequestCanceledException INFO [2019-12-17 12:27:58,200] jepsen worker 34 - jepsen.util 484 :invoke :cas [11 [15 1]] INFO [2019-12-17 12:27:58,200] jepsen worker 89 - jepsen.util 449 :invoke :write [29 5] INFO [2019-12-17 12:27:58,200] jepsen worker 55 - jepsen.util 595 :invoke :write [18 11] INFO [2019-12-17 12:27:58,212] jepsen worker 64 - jepsen.util 334 :ok :write [21 24] INFO [2019-12-17 12:27:58,212] jepsen worker 64 - jepsen.util 334 :invoke :read [21 nil] INFO [2019-12-17 12:27:58,221] jepsen worker 85 - jepsen.util 355 :info :write [28 25] :RequestCanceledException INFO [2019-12-17 12:27:58,221] jepsen worker 83 - jepsen.util 443 :info :cas [27 [30 23]] :RequestCanceledException INFO [2019-12-17 12:27:58,222] jepsen worker 85 - jepsen.util 445 :invoke :cas [28 [49 6]] INFO [2019-12-17 12:27:58,222] jepsen worker 83 - jepsen.util 533 :invoke :cas [27 [41 37]] INFO [2019-12-17 12:27:58,234] jepsen worker 64 - jepsen.util 334 :ok :read [21 24] INFO [2019-12-17 12:27:58,234] jepsen worker 64 - jepsen.util 334 :invoke :cas [21 [16 13]] INFO [2019-12-17 12:27:58,234] jepsen worker 33 - jepsen.util 393 :info :cas [11 [8 40]] :RequestCanceledException INFO [2019-12-17 12:27:58,235] jepsen worker 33 - jepsen.util 483 :invoke :cas [11 [25 18]] INFO [2019-12-17 12:27:58,247] jepsen worker 86 - jepsen.util 176 :info :write [28 12] :RequestCanceledException INFO [2019-12-17 12:27:58,248] jepsen worker 86 - jepsen.util 266 :invoke :read [28 nil] INFO [2019-12-17 12:27:58,252] jepsen worker 64 - jepsen.util 334 :fail :cas [21 [16 13]] :ValueNotSwapFrom INFO [2019-12-17 12:27:58,252] jepsen worker 64 - jepsen.util 334 :invoke :cas [21 [29 25]] INFO [2019-12-17 12:27:58,254] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2100,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:27:58,255] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2100,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:27:58,255] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2101,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:27:58,255] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2101,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:27:58,269] jepsen worker 64 - jepsen.util 334 :fail :cas [21 [29 25]] :ValueNotSwapFrom INFO [2019-12-17 12:27:58,269] jepsen worker 64 - jepsen.util 334 :invoke :cas [21 [4 26]] INFO [2019-12-17 12:27:58,283] jepsen worker 22 - jepsen.util 382 :info :write [7 9] :RequestCanceledException INFO [2019-12-17 12:27:58,284] jepsen worker 22 - jepsen.util 472 :invoke :cas [7 [41 46]] INFO [2019-12-17 12:27:58,286] jepsen worker 63 - jepsen.util 243 :ok :write [21 19] INFO [2019-12-17 12:27:58,286] jepsen worker 63 - jepsen.util 243 :invoke :write [21 30] INFO [2019-12-17 12:27:58,293] jepsen worker 64 - jepsen.util 334 :fail :cas [21 [4 26]] :ValueNotSwapFrom INFO [2019-12-17 12:27:58,293] jepsen worker 64 - jepsen.util 334 :invoke :read [21 nil] INFO [2019-12-17 12:27:58,297] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2107,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:27:58,297] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2107,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:27:58,298] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2111,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:27:58,298] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2111,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:27:58,301] jepsen worker 87 - jepsen.util 267 :info :write [29 29] :RequestCanceledException INFO [2019-12-17 12:27:58,302] jepsen worker 87 - jepsen.util 357 :invoke :read [29 nil] INFO [2019-12-17 12:27:58,319] jepsen worker 39 - jepsen.util 399 :info :write [13 31] :RequestCanceledException INFO [2019-12-17 12:27:58,319] jepsen worker 65 - jepsen.util 335 :info :write [21 22] :RequestCanceledException INFO [2019-12-17 12:27:58,320] jepsen worker 65 - jepsen.util 425 :invoke :cas [21 [29 21]] INFO [2019-12-17 12:27:58,320] jepsen worker 39 - jepsen.util 489 :invoke :write [13 15] INFO [2019-12-17 12:27:58,335] jepsen worker 8 - jepsen.util 458 :info :write [2 11] :RequestCanceledException INFO [2019-12-17 12:27:58,337] jepsen worker 8 - jepsen.util 548 :invoke :cas [2 [29 41]] INFO [2019-12-17 12:27:58,348] jepsen worker 43 - jepsen.util 583 :fail :cas [14 [13 29]] :ValueNotSwapFrom INFO [2019-12-17 12:27:58,348] jepsen worker 43 - jepsen.util 583 :invoke :write [14 37] INFO [2019-12-17 12:27:58,381] jepsen worker 64 - jepsen.util 334 :ok :read [21 19] INFO [2019-12-17 12:27:58,381] jepsen worker 64 - jepsen.util 334 :invoke :write [21 36] INFO [2019-12-17 12:27:58,386] jepsen worker 5 - jepsen.util 95 :info :write [1 11] :RequestCanceledException INFO [2019-12-17 12:27:58,387] jepsen worker 5 - jepsen.util 185 :invoke :read [1 nil] INFO [2019-12-17 12:27:58,388] jepsen worker 65 - jepsen.util 425 :fail :cas [21 [29 21]] :ValueNotSwapFrom INFO [2019-12-17 12:27:58,388] jepsen worker 65 - jepsen.util 425 :invoke :read [21 nil] INFO [2019-12-17 12:27:58,403] jepsen worker 58 - jepsen.util 148 :info :write [19 33] :RequestCanceledException INFO [2019-12-17 12:27:58,405] jepsen worker 58 - jepsen.util 238 :invoke :cas [19 [45 39]] INFO [2019-12-17 12:27:58,410] jepsen worker 52 - jepsen.util 412 :ok :read [17 0] INFO [2019-12-17 12:27:58,410] jepsen worker 52 - jepsen.util 412 :invoke :cas [17 [17 26]] INFO [2019-12-17 12:27:58,418] jepsen worker 26 - jepsen.util 476 :info :write [8 0] :RequestCanceledException INFO [2019-12-17 12:27:58,420] jepsen worker 26 - jepsen.util 566 :invoke :write [8 30] INFO [2019-12-17 12:27:58,422] jepsen worker 77 - jepsen.util 437 :fail :cas [25 [25 40]] :ValueNotSwapFrom INFO [2019-12-17 12:27:58,422] jepsen worker 77 - jepsen.util 437 :invoke :read [25 nil] INFO [2019-12-17 12:27:58,435] jepsen worker 23 - jepsen.util 203 :info :cas [7 [29 14]] :RequestCanceledException INFO [2019-12-17 12:27:58,437] jepsen worker 23 - jepsen.util 293 :invoke :read [7 nil] INFO [2019-12-17 12:27:58,438] jepsen worker 14 - jepsen.util 464 :ok :read [4 4] INFO [2019-12-17 12:27:58,438] jepsen worker 14 - jepsen.util 464 :invoke :read [4 nil] INFO [2019-12-17 12:27:58,446] jepsen worker 9 - jepsen.util 279 :info :cas [3 [32 42]] :RequestCanceledException INFO [2019-12-17 12:27:58,447] jepsen worker 9 - jepsen.util 369 :invoke :write [3 39] INFO [2019-12-17 12:27:58,456] jepsen worker 64 - jepsen.util 334 :ok :write [21 36] INFO [2019-12-17 12:27:58,456] jepsen worker 64 - jepsen.util 334 :invoke :read [21 nil] INFO [2019-12-17 12:27:58,472] jepsen worker 65 - jepsen.util 425 :ok :read [21 36] INFO [2019-12-17 12:27:58,472] jepsen worker 65 - jepsen.util 425 :invoke :read [21 nil] INFO [2019-12-17 12:27:58,491] jepsen worker 64 - jepsen.util 334 :ok :read [21 36] INFO [2019-12-17 12:27:58,491] jepsen worker 64 - jepsen.util 334 :invoke :read [21 nil] INFO [2019-12-17 12:27:58,499] jepsen worker 65 - jepsen.util 425 :ok :read [21 36] INFO [2019-12-17 12:27:58,499] jepsen worker 65 - jepsen.util 425 :invoke :write [21 12] INFO [2019-12-17 12:27:58,503] jepsen worker 57 - jepsen.util 327 :info :write [19 18] :RequestCanceledException INFO [2019-12-17 12:27:58,505] jepsen worker 57 - jepsen.util 417 :invoke :write [19 26] INFO [2019-12-17 12:27:58,516] jepsen worker 64 - jepsen.util 334 :ok :read [21 36] INFO [2019-12-17 12:27:58,517] jepsen worker 64 - jepsen.util 334 :invoke :cas [21 [17 4]] INFO [2019-12-17 12:27:58,520] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2142,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:27:58,520] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2142,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:27:58,520] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2143,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:27:58,520] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2143,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:27:58,551] jepsen worker 0 - jepsen.util 450 :info :cas [0 [29 25]] :RequestCanceledException INFO [2019-12-17 12:27:58,553] jepsen worker 0 - jepsen.util 540 :invoke :cas [0 [48 1]] INFO [2019-12-17 12:27:58,573] jepsen worker 36 - jepsen.util 486 :ok :read [12 12] INFO [2019-12-17 12:27:58,574] jepsen worker 36 - jepsen.util 486 :invoke :cas [12 [20 10]] INFO [2019-12-17 12:27:58,574] jepsen worker 79 - jepsen.util 439 :fail :cas [26 [25 18]] :ValueNotSwapFrom INFO [2019-12-17 12:27:58,575] jepsen worker 79 - jepsen.util 439 :invoke :cas [26 [21 37]] INFO [2019-12-17 12:27:58,584] jepsen worker 65 - jepsen.util 425 :ok :write [21 12] INFO [2019-12-17 12:27:58,584] jepsen worker 65 - jepsen.util 425 :invoke :write [21 23] INFO [2019-12-17 12:27:58,586] jepsen worker 47 - jepsen.util 497 :info :write [15 37] :RequestCanceledException INFO [2019-12-17 12:27:58,587] jepsen worker 47 - jepsen.util 587 :invoke :cas [15 [25 34]] INFO [2019-12-17 12:27:58,598] jepsen worker 62 - jepsen.util 422 :info :write [20 36] :RequestCanceledException INFO [2019-12-17 12:27:58,600] jepsen worker 62 - jepsen.util 512 :invoke :cas [20 [33 2]] INFO [2019-12-17 12:27:58,603] jepsen worker 33 - jepsen.util 483 :fail :cas [11 [25 18]] :ValueNotSwapFrom INFO [2019-12-17 12:27:58,603] jepsen worker 33 - jepsen.util 483 :invoke :cas [11 [8 21]] INFO [2019-12-17 12:27:58,618] jepsen worker 73 - jepsen.util 433 :info :cas [24 [35 31]] :RequestCanceledException INFO [2019-12-17 12:27:58,620] jepsen worker 73 - jepsen.util 523 :invoke :cas [24 [49 0]] INFO [2019-12-17 12:27:58,634] jepsen worker 24 - jepsen.util 294 :info :write [8 42] :RequestCanceledException INFO [2019-12-17 12:27:58,634] jepsen worker 20 - jepsen.util 380 :info :write [6 1] :RequestCanceledException INFO [2019-12-17 12:27:58,634] jepsen worker 10 - jepsen.util 370 :fail :read [3 nil] :RequestTimeoutException INFO [2019-12-17 12:27:58,635] jepsen worker 10 - jepsen.util 370 :invoke :cas [3 [4 45]] INFO [2019-12-17 12:27:58,636] jepsen worker 24 - jepsen.util 384 :invoke :cas [8 [8 29]] INFO [2019-12-17 12:27:58,636] jepsen worker 20 - jepsen.util 470 :invoke :write [6 48] INFO [2019-12-17 12:27:58,641] jepsen worker 64 - jepsen.util 334 :fail :cas [21 [17 4]] :ValueNotSwapFrom INFO [2019-12-17 12:27:58,641] jepsen worker 64 - jepsen.util 334 :invoke :read [21 nil] INFO [2019-12-17 12:27:58,652] jepsen worker 59 - jepsen.util 239 :info :write [19 9] :RequestCanceledException INFO [2019-12-17 12:27:58,652] jepsen worker 49 - jepsen.util 319 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:27:58,652] jepsen worker 49 - jepsen.util 319 :invoke :cas [16 [17 23]] INFO [2019-12-17 12:27:58,653] jepsen worker 59 - jepsen.util 329 :invoke :cas [19 [30 37]] INFO [2019-12-17 12:27:58,656] jepsen worker 44 - jepsen.util 224 :ok :read [14 27] INFO [2019-12-17 12:27:58,657] jepsen worker 44 - jepsen.util 224 :invoke :read [14 nil] INFO [2019-12-17 12:27:58,661] jepsen worker 47 - jepsen.util 587 :fail :cas [15 [25 34]] :ValueNotSwapFrom INFO [2019-12-17 12:27:58,661] jepsen worker 47 - jepsen.util 587 :invoke :write [15 19] INFO [2019-12-17 12:27:58,670] jepsen worker 75 - jepsen.util 345 :ok :read [25 2] INFO [2019-12-17 12:27:58,670] jepsen worker 75 - jepsen.util 345 :invoke :read [25 nil] INFO [2019-12-17 12:27:58,696] jepsen worker 14 - jepsen.util 464 :ok :read [4 4] INFO [2019-12-17 12:27:58,697] jepsen worker 14 - jepsen.util 464 :invoke :read [4 nil] INFO [2019-12-17 12:27:58,722] jepsen worker 65 - jepsen.util 425 :ok :write [21 23] INFO [2019-12-17 12:27:58,722] jepsen worker 65 - jepsen.util 425 :invoke :read [21 nil] INFO [2019-12-17 12:27:58,726] jepsen worker 36 - jepsen.util 486 :fail :cas [12 [20 10]] :ValueNotSwapFrom INFO [2019-12-17 12:27:58,726] jepsen worker 36 - jepsen.util 486 :invoke :read [12 nil] INFO [2019-12-17 12:27:58,737] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.046875}, :ns_1@172.28.128.126 {:progress 0.04316546762589923}, :ns_1@172.28.128.127 {:progress 0.0933852140077821}, :ns_1@172.28.128.128 {:progress 0.1283018867924528}} INFO [2019-12-17 12:27:58,737] jepsen nemesis - couchbase.util Rebalance check count: 10 INFO [2019-12-17 12:27:58,737] jepsen nemesis - couchbase.util Rebalance stuck count: 0 INFO [2019-12-17 12:27:58,740] jepsen worker 25 - jepsen.util 475 :fail :cas [8 [44 42]] :ValueNotSwapFrom INFO [2019-12-17 12:27:58,740] jepsen worker 25 - jepsen.util 475 :invoke :write [8 3] INFO [2019-12-17 12:27:58,748] jepsen worker 76 - jepsen.util 346 :info :write [25 19] :RequestCanceledException INFO [2019-12-17 12:27:58,750] jepsen worker 76 - jepsen.util 436 :invoke :write [25 38] INFO [2019-12-17 12:27:58,762] jepsen worker 10 - jepsen.util 370 :fail :cas [3 [4 45]] :ValueNotSwapFrom INFO [2019-12-17 12:27:58,762] jepsen worker 10 - jepsen.util 370 :invoke :cas [3 [20 37]] INFO [2019-12-17 12:27:58,787] jepsen worker 7 - jepsen.util 457 :info :write [2 20] :RequestCanceledException INFO [2019-12-17 12:27:58,789] jepsen worker 7 - jepsen.util 547 :invoke :write [2 24] INFO [2019-12-17 12:27:58,789] jepsen worker 64 - jepsen.util 334 :ok :read [21 23] INFO [2019-12-17 12:27:58,789] jepsen worker 64 - jepsen.util 334 :invoke :read [21 nil] INFO [2019-12-17 12:27:58,811] jepsen worker 65 - jepsen.util 425 :ok :read [21 23] INFO [2019-12-17 12:27:58,811] jepsen worker 65 - jepsen.util 425 :invoke :read [21 nil] INFO [2019-12-17 12:27:58,821] jepsen worker 12 - jepsen.util 282 :info :write [4 36] :RequestCanceledException INFO [2019-12-17 12:27:58,822] jepsen worker 12 - jepsen.util 372 :invoke :write [4 9] INFO [2019-12-17 12:27:58,845] jepsen worker 14 - jepsen.util 464 :ok :read [4 4] INFO [2019-12-17 12:27:58,845] jepsen worker 14 - jepsen.util 464 :invoke :write [4 12] INFO [2019-12-17 12:27:58,848] jepsen worker 10 - jepsen.util 370 :fail :cas [3 [20 37]] :ValueNotSwapFrom INFO [2019-12-17 12:27:58,848] jepsen worker 10 - jepsen.util 370 :invoke :write [3 40] INFO [2019-12-17 12:27:58,856] jepsen worker 60 - jepsen.util 600 :ok :read [20 7] INFO [2019-12-17 12:27:58,856] jepsen worker 60 - jepsen.util 600 :invoke :cas [20 [22 5]] INFO [2019-12-17 12:27:58,861] jepsen worker 64 - jepsen.util 334 :ok :read [21 23] INFO [2019-12-17 12:27:58,862] jepsen worker 64 - jepsen.util 334 :invoke :cas [21 [21 43]] INFO [2019-12-17 12:27:58,874] jepsen worker 65 - jepsen.util 425 :ok :read [21 23] INFO [2019-12-17 12:27:58,875] jepsen worker 65 - jepsen.util 425 :invoke :read [21 nil] INFO [2019-12-17 12:27:58,881] jepsen worker 71 - jepsen.util 341 :info :write [23 5] :RequestCanceledException INFO [2019-12-17 12:27:58,882] jepsen worker 88 - jepsen.util 358 :ok :read [29 5] INFO [2019-12-17 12:27:58,882] jepsen worker 71 - jepsen.util 431 :invoke :cas [23 [46 33]] INFO [2019-12-17 12:27:58,882] jepsen worker 88 - jepsen.util 358 :invoke :cas [29 [22 0]] INFO [2019-12-17 12:27:58,903] jepsen worker 52 - jepsen.util 412 :fail :cas [17 [17 26]] :ValueNotSwapFrom INFO [2019-12-17 12:27:58,904] jepsen worker 52 - jepsen.util 412 :invoke :read [17 nil] INFO [2019-12-17 12:27:58,914] jepsen worker 23 - jepsen.util 293 :ok :read [7 20] INFO [2019-12-17 12:27:58,915] jepsen worker 23 - jepsen.util 293 :invoke :write [7 29] INFO [2019-12-17 12:27:58,921] jepsen worker 69 - jepsen.util 429 :info :write [23 1] :RequestCanceledException INFO [2019-12-17 12:27:58,922] jepsen worker 69 - jepsen.util 519 :invoke :write [23 48] INFO [2019-12-17 12:27:58,934] jepsen worker 80 - jepsen.util 620 :info :write [26 19] :RequestCanceledException INFO [2019-12-17 12:27:58,936] jepsen worker 80 - jepsen.util 710 :invoke :read [26 nil] INFO [2019-12-17 12:27:58,942] jepsen worker 36 - jepsen.util 486 :ok :read [12 12] INFO [2019-12-17 12:27:58,942] jepsen worker 36 - jepsen.util 486 :invoke :read [12 nil] INFO [2019-12-17 12:27:58,963] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2195,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:27:58,963] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2195,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:27:58,968] jepsen worker 77 - jepsen.util 437 :ok :read [25 2] INFO [2019-12-17 12:27:58,968] jepsen worker 77 - jepsen.util 437 :invoke :cas [25 [13 30]] INFO [2019-12-17 12:27:58,975] jepsen worker 28 - jepsen.util 298 :ok :write [9 27] INFO [2019-12-17 12:27:58,975] jepsen worker 28 - jepsen.util 298 :invoke :read [9 nil] INFO [2019-12-17 12:27:58,988] jepsen worker 42 - jepsen.util 582 :info :write [14 49] :RequestCanceledException INFO [2019-12-17 12:27:58,988] jepsen worker 67 - jepsen.util 517 :info :write [22 45] :RequestCanceledException INFO [2019-12-17 12:27:58,989] jepsen worker 67 - jepsen.util 607 :invoke :write [22 43] INFO [2019-12-17 12:27:58,989] jepsen worker 42 - jepsen.util 672 :invoke :write [14 36] INFO [2019-12-17 12:27:59,002] jepsen worker 28 - jepsen.util 298 :ok :read [9 27] INFO [2019-12-17 12:27:59,002] jepsen worker 28 - jepsen.util 298 :invoke :read [9 nil] INFO [2019-12-17 12:27:59,004] jepsen worker 11 - jepsen.util 371 :info :write [3 48] :RequestCanceledException INFO [2019-12-17 12:27:59,006] jepsen worker 11 - jepsen.util 461 :invoke :write [3 26] INFO [2019-12-17 12:27:59,006] jepsen worker 52 - jepsen.util 412 :ok :read [17 0] INFO [2019-12-17 12:27:59,007] jepsen worker 52 - jepsen.util 412 :invoke :read [17 nil] INFO [2019-12-17 12:27:59,035] jepsen worker 28 - jepsen.util 298 :ok :read [9 27] INFO [2019-12-17 12:27:59,035] jepsen worker 28 - jepsen.util 298 :invoke :write [9 30] INFO [2019-12-17 12:27:59,049] jepsen worker 22 - jepsen.util 472 :fail :cas [7 [41 46]] :ValueNotSwapFrom INFO [2019-12-17 12:27:59,050] jepsen worker 22 - jepsen.util 472 :invoke :write [7 36] INFO [2019-12-17 12:27:59,073] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2209,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:27:59,073] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2209,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:27:59,073] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2212,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:27:59,073] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2212,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:27:59,085] jepsen worker 28 - jepsen.util 298 :ok :write [9 30] INFO [2019-12-17 12:27:59,086] jepsen worker 28 - jepsen.util 298 :invoke :write [9 47] INFO [2019-12-17 12:27:59,111] jepsen worker 62 - jepsen.util 512 :fail :cas [20 [33 2]] :ValueNotSwapFrom INFO [2019-12-17 12:27:59,112] jepsen worker 62 - jepsen.util 512 :invoke :write [20 2] INFO [2019-12-17 12:27:59,121] jepsen worker 44 - jepsen.util 224 :ok :read [14 27] INFO [2019-12-17 12:27:59,121] jepsen worker 44 - jepsen.util 224 :invoke :cas [14 [42 11]] INFO [2019-12-17 12:27:59,134] jepsen worker 3 - jepsen.util 363 :info :write [1 27] :RequestCanceledException INFO [2019-12-17 12:27:59,134] jepsen worker 3 - jepsen.util 453 :invoke :read [1 nil] INFO [2019-12-17 12:27:59,194] jepsen worker 28 - jepsen.util 298 :ok :write [9 47] INFO [2019-12-17 12:27:59,194] jepsen worker 28 - jepsen.util 298 :invoke :cas [9 [22 30]] INFO [2019-12-17 12:27:59,214] jepsen worker 8 - jepsen.util 548 :fail :cas [2 [29 41]] :ValueNotSwapFrom INFO [2019-12-17 12:27:59,215] jepsen worker 8 - jepsen.util 548 :invoke :cas [2 [36 43]] INFO [2019-12-17 12:27:59,215] jepsen worker 28 - jepsen.util 298 :fail :cas [9 [22 30]] :ValueNotSwapFrom INFO [2019-12-17 12:27:59,216] jepsen worker 28 - jepsen.util 298 :invoke :write [9 18] INFO [2019-12-17 12:27:59,221] jepsen worker 30 - jepsen.util 300 :info :write [10 9] :RequestCanceledException INFO [2019-12-17 12:27:59,223] jepsen worker 30 - jepsen.util 390 :invoke :cas [10 [35 12]] INFO [2019-12-17 12:27:59,239] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2226,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:27:59,240] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2226,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:27:59,246] jepsen worker 77 - jepsen.util 437 :fail :cas [25 [13 30]] :ValueNotSwapFrom INFO [2019-12-17 12:27:59,246] jepsen worker 77 - jepsen.util 437 :invoke :cas [25 [10 29]] INFO [2019-12-17 12:27:59,250] jepsen worker 34 - jepsen.util 484 :fail :cas [11 [15 1]] :ValueNotSwapFrom INFO [2019-12-17 12:27:59,251] jepsen worker 34 - jepsen.util 484 :invoke :cas [11 [12 18]] INFO [2019-12-17 12:27:59,252] jepsen worker 50 - jepsen.util 500 :info :cas [16 [1 43]] :RequestCanceledException INFO [2019-12-17 12:27:59,253] jepsen worker 50 - jepsen.util 590 :invoke :read [16 nil] INFO [2019-12-17 12:27:59,266] jepsen worker 3 - jepsen.util 453 :ok :read [1 23] INFO [2019-12-17 12:27:59,267] jepsen worker 3 - jepsen.util 453 :invoke :cas [1 [8 10]] INFO [2019-12-17 12:27:59,270] jepsen worker 5 - jepsen.util 185 :ok :read [1 23] INFO [2019-12-17 12:27:59,270] jepsen worker 5 - jepsen.util 185 :invoke :cas [1 [0 8]] INFO [2019-12-17 12:27:59,283] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2234,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:27:59,283] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2234,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:27:59,283] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2237,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:27:59,283] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2237,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:27:59,289] jepsen worker 36 - jepsen.util 486 :ok :read [12 12] INFO [2019-12-17 12:27:59,289] jepsen worker 36 - jepsen.util 486 :invoke :cas [12 [43 13]] INFO [2019-12-17 12:27:59,295] jepsen worker 28 - jepsen.util 298 :ok :write [9 18] INFO [2019-12-17 12:27:59,295] jepsen worker 28 - jepsen.util 298 :invoke :write [9 16] INFO [2019-12-17 12:27:59,301] jepsen worker 19 - jepsen.util 379 :info :write [6 18] :SyncWriteAmbiguous INFO [2019-12-17 12:27:59,302] jepsen worker 19 - jepsen.util 469 :invoke :cas [6 [28 42]] INFO [2019-12-17 12:27:59,318] jepsen worker 37 - jepsen.util 307 :info :write [12 21] :RequestCanceledException INFO [2019-12-17 12:27:59,320] jepsen worker 37 - jepsen.util 397 :invoke :write [12 16] INFO [2019-12-17 12:27:59,321] jepsen worker 56 - jepsen.util 416 :fail :cas [18 [7 42]] :ValueNotSwapFrom INFO [2019-12-17 12:27:59,321] jepsen worker 56 - jepsen.util 416 :invoke :read [18 nil] INFO [2019-12-17 12:27:59,332] jepsen worker 44 - jepsen.util 224 :fail :cas [14 [42 11]] :ValueNotSwapFrom INFO [2019-12-17 12:27:59,332] jepsen worker 19 - jepsen.util 469 :fail :cas [6 [28 42]] :ValueNotSwapFrom INFO [2019-12-17 12:27:59,332] jepsen worker 19 - jepsen.util 469 :invoke :write [6 34] INFO [2019-12-17 12:27:59,332] jepsen worker 44 - jepsen.util 224 :invoke :write [14 25] INFO [2019-12-17 12:27:59,333] jepsen worker 46 - jepsen.util 316 :fail :cas [15 [36 46]] :ValueNotSwapFrom INFO [2019-12-17 12:27:59,333] jepsen worker 46 - jepsen.util 316 :invoke :write [15 24] INFO [2019-12-17 12:27:59,341] jepsen worker 79 - jepsen.util 439 :fail :cas [26 [21 37]] :ValueNotSwapFrom INFO [2019-12-17 12:27:59,341] jepsen worker 79 - jepsen.util 439 :invoke :read [26 nil] INFO [2019-12-17 12:27:59,349] jepsen worker 27 - jepsen.util 207 :fail :cas [9 [11 33]] :ValueNotSwapFrom INFO [2019-12-17 12:27:59,350] jepsen worker 27 - jepsen.util 207 :invoke :write [9 42] INFO [2019-12-17 12:27:59,365] jepsen worker 33 - jepsen.util 483 :fail :cas [11 [8 21]] :ValueNotSwapFrom INFO [2019-12-17 12:27:59,365] jepsen worker 33 - jepsen.util 483 :invoke :read [11 nil] INFO [2019-12-17 12:27:59,376] jepsen worker 24 - jepsen.util 384 :fail :cas [8 [8 29]] :ValueNotSwapFrom INFO [2019-12-17 12:27:59,376] jepsen worker 24 - jepsen.util 384 :invoke :read [8 nil] INFO [2019-12-17 12:27:59,378] jepsen worker 60 - jepsen.util 600 :fail :cas [20 [22 5]] :ValueNotSwapFrom INFO [2019-12-17 12:27:59,378] jepsen worker 60 - jepsen.util 600 :invoke :cas [20 [3 31]] INFO [2019-12-17 12:27:59,386] jepsen worker 17 - jepsen.util 467 :info :write [5 9] :RequestCanceledException INFO [2019-12-17 12:27:59,386] jepsen worker 8 - jepsen.util 548 :fail :cas [2 [36 43]] :ValueNotSwapFrom INFO [2019-12-17 12:27:59,387] jepsen worker 8 - jepsen.util 548 :invoke :read [2 nil] INFO [2019-12-17 12:27:59,387] jepsen worker 17 - jepsen.util 557 :invoke :read [5 nil] INFO [2019-12-17 12:27:59,403] jepsen worker 35 - jepsen.util 215 :info :write [11 49] :RequestCanceledException INFO [2019-12-17 12:27:59,405] jepsen worker 35 - jepsen.util 305 :invoke :cas [11 [34 27]] INFO [2019-12-17 12:27:59,413] jepsen worker 71 - jepsen.util 431 :fail :cas [23 [46 33]] :ValueNotSwapFrom INFO [2019-12-17 12:27:59,413] jepsen worker 71 - jepsen.util 431 :invoke :read [23 nil] INFO [2019-12-17 12:27:59,414] jepsen worker 28 - jepsen.util 298 :ok :write [9 16] INFO [2019-12-17 12:27:59,414] jepsen worker 28 - jepsen.util 298 :invoke :cas [9 [6 21]] INFO [2019-12-17 12:27:59,421] jepsen worker 2 - jepsen.util 272 :info :write [0 2] :RequestCanceledException INFO [2019-12-17 12:27:59,423] jepsen worker 2 - jepsen.util 362 :invoke :cas [0 [45 22]] INFO [2019-12-17 12:27:59,429] jepsen worker 75 - jepsen.util 345 :ok :read [25 2] INFO [2019-12-17 12:27:59,429] jepsen worker 75 - jepsen.util 345 :invoke :read [25 nil] INFO [2019-12-17 12:27:59,436] jepsen worker 28 - jepsen.util 298 :fail :cas [9 [6 21]] :ValueNotSwapFrom INFO [2019-12-17 12:27:59,436] jepsen worker 28 - jepsen.util 298 :invoke :write [9 48] INFO [2019-12-17 12:27:59,473] jepsen worker 52 - jepsen.util 412 :ok :read [17 0] INFO [2019-12-17 12:27:59,473] jepsen worker 52 - jepsen.util 412 :invoke :cas [17 [17 31]] INFO [2019-12-17 12:27:59,474] jepsen worker 8 - jepsen.util 548 :ok :read [2 15] INFO [2019-12-17 12:27:59,475] jepsen worker 8 - jepsen.util 548 :invoke :read [2 nil] INFO [2019-12-17 12:27:59,487] jepsen worker 78 - jepsen.util 438 :info :write [26 45] :RequestCanceledException INFO [2019-12-17 12:27:59,488] jepsen worker 78 - jepsen.util 528 :invoke :read [26 nil] INFO [2019-12-17 12:27:59,503] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2265,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:27:59,503] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2265,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:27:59,503] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2268,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:27:59,503] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2268,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:27:59,505] jepsen worker 74 - jepsen.util 344 :info :write [24 21] :RequestCanceledException INFO [2019-12-17 12:27:59,507] jepsen worker 74 - jepsen.util 434 :invoke :write [24 18] INFO [2019-12-17 12:27:59,518] jepsen worker 28 - jepsen.util 298 :ok :write [9 48] INFO [2019-12-17 12:27:59,518] jepsen worker 28 - jepsen.util 298 :invoke :write [9 1] INFO [2019-12-17 12:27:59,526] jepsen worker 56 - jepsen.util 416 :ok :read [18 47] INFO [2019-12-17 12:27:59,527] jepsen worker 56 - jepsen.util 416 :invoke :read [18 nil] INFO [2019-12-17 12:27:59,534] jepsen worker 61 - jepsen.util 511 :info :write [20 25] :RequestCanceledException INFO [2019-12-17 12:27:59,536] jepsen worker 61 - jepsen.util 601 :invoke :read [20 nil] INFO [2019-12-17 12:27:59,578] jepsen worker 24 - jepsen.util 384 :ok :read [8 14] INFO [2019-12-17 12:27:59,578] jepsen worker 24 - jepsen.util 384 :invoke :read [8 nil] INFO [2019-12-17 12:27:59,608] jepsen worker 34 - jepsen.util 484 :fail :cas [11 [12 18]] :ValueNotSwapFrom INFO [2019-12-17 12:27:59,608] jepsen worker 34 - jepsen.util 484 :invoke :write [11 27] INFO [2019-12-17 12:27:59,619] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestTimeoutException: CarrierBucketConfigRequest {"retried":0,"reason":"TIMEOUT","requestId":1970,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:27:59,624] jepsen worker 87 - jepsen.util 357 :ok :read [29 5] INFO [2019-12-17 12:27:59,624] jepsen worker 87 - jepsen.util 357 :invoke :cas [29 [49 23]] INFO [2019-12-17 12:27:59,635] jepsen worker 28 - jepsen.util 298 :ok :write [9 1] INFO [2019-12-17 12:27:59,635] jepsen worker 28 - jepsen.util 298 :invoke :write [9 10] INFO [2019-12-17 12:27:59,648] jepsen worker 33 - jepsen.util 483 :ok :read [11 7] INFO [2019-12-17 12:27:59,648] jepsen worker 33 - jepsen.util 483 :invoke :write [11 33] INFO [2019-12-17 12:27:59,666] jepsen worker 41 - jepsen.util 311 :fail :cas [13 [37 13]] :ValueNotSwapFrom INFO [2019-12-17 12:27:59,667] jepsen worker 41 - jepsen.util 311 :invoke :read [13 nil] INFO [2019-12-17 12:27:59,687] jepsen worker 38 - jepsen.util 218 :info :write [12 4] :RequestCanceledException INFO [2019-12-17 12:27:59,688] jepsen worker 38 - jepsen.util 308 :invoke :cas [12 [47 40]] INFO [2019-12-17 12:27:59,702] jepsen worker 28 - jepsen.util 298 :ok :write [9 10] INFO [2019-12-17 12:27:59,703] jepsen worker 28 - jepsen.util 298 :invoke :read [9 nil] INFO [2019-12-17 12:27:59,704] jepsen worker 84 - jepsen.util 174 :info :write [28 42] :RequestCanceledException INFO [2019-12-17 12:27:59,706] jepsen worker 84 - jepsen.util 264 :invoke :write [28 1] INFO [2019-12-17 12:27:59,710] jepsen worker 3 - jepsen.util 453 :fail :cas [1 [8 10]] :ValueNotSwapFrom INFO [2019-12-17 12:27:59,710] jepsen worker 8 - jepsen.util 548 :ok :read [2 15] INFO [2019-12-17 12:27:59,710] jepsen worker 8 - jepsen.util 548 :invoke :read [2 nil] INFO [2019-12-17 12:27:59,710] jepsen worker 3 - jepsen.util 453 :invoke :read [1 nil] INFO [2019-12-17 12:27:59,713] jepsen worker 82 - jepsen.util 532 :info :cas [27 [25 46]] :RequestCanceledException INFO [2019-12-17 12:27:59,713] jepsen worker 13 - jepsen.util 373 :info :write [4 29] :RequestCanceledException INFO [2019-12-17 12:27:59,713] jepsen worker 15 - jepsen.util 375 :info :write [5 44] :RequestCanceledException INFO [2019-12-17 12:27:59,714] jepsen worker 82 - jepsen.util 622 :invoke :read [27 nil] INFO [2019-12-17 12:27:59,714] jepsen worker 15 - jepsen.util 465 :invoke :cas [5 [40 15]] INFO [2019-12-17 12:27:59,714] jepsen worker 13 - jepsen.util 463 :invoke :read [4 nil] INFO [2019-12-17 12:27:59,723] jepsen worker 28 - jepsen.util 298 :ok :read [9 10] INFO [2019-12-17 12:27:59,724] jepsen worker 28 - jepsen.util 298 :invoke :cas [9 [20 38]] INFO [2019-12-17 12:27:59,735] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2288,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:27:59,735] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2288,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:27:59,741] jepsen worker 36 - jepsen.util 486 :fail :cas [12 [43 13]] :ValueNotSwapFrom INFO [2019-12-17 12:27:59,741] jepsen worker 36 - jepsen.util 486 :invoke :write [12 0] INFO [2019-12-17 12:27:59,745] jepsen worker 28 - jepsen.util 298 :fail :cas [9 [20 38]] :ValueNotSwapFrom INFO [2019-12-17 12:27:59,745] jepsen worker 28 - jepsen.util 298 :invoke :write [9 2] INFO [2019-12-17 12:27:59,749] jepsen worker 75 - jepsen.util 345 :ok :read [25 2] INFO [2019-12-17 12:27:59,749] jepsen worker 35 - jepsen.util 305 :fail :cas [11 [34 27]] :ValueNotSwapFrom INFO [2019-12-17 12:27:59,749] jepsen worker 75 - jepsen.util 345 :invoke :write [25 25] INFO [2019-12-17 12:27:59,749] jepsen worker 35 - jepsen.util 305 :invoke :write [11 38] INFO [2019-12-17 12:27:59,752] jepsen worker 80 - jepsen.util 710 :ok :read [26 47] INFO [2019-12-17 12:27:59,752] jepsen worker 80 - jepsen.util 710 :invoke :read [26 nil] INFO [2019-12-17 12:27:59,756] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.05859375}, :ns_1@172.28.128.126 {:progress 0.0539568345323741}, :ns_1@172.28.128.127 {:progress 0.1011673151750972}, :ns_1@172.28.128.128 {:progress 0.1735849056603773}} INFO [2019-12-17 12:27:59,756] jepsen nemesis - couchbase.util Rebalance check count: 11 INFO [2019-12-17 12:27:59,756] jepsen nemesis - couchbase.util Rebalance stuck count: 0 INFO [2019-12-17 12:27:59,773] jepsen worker 71 - jepsen.util 431 :ok :read [23 29] INFO [2019-12-17 12:27:59,773] jepsen worker 71 - jepsen.util 431 :invoke :read [23 nil] INFO [2019-12-17 12:27:59,787] jepsen worker 4 - jepsen.util 364 :info :write [1 33] :RequestCanceledException INFO [2019-12-17 12:27:59,788] jepsen worker 4 - jepsen.util 454 :invoke :write [1 21] INFO [2019-12-17 12:27:59,811] jepsen worker 28 - jepsen.util 298 :ok :write [9 2] INFO [2019-12-17 12:27:59,811] jepsen worker 28 - jepsen.util 298 :invoke :read [9 nil] INFO [2019-12-17 12:27:59,821] jepsen worker 21 - jepsen.util 381 :info :write [7 7] :RequestCanceledException INFO [2019-12-17 12:27:59,822] jepsen worker 21 - jepsen.util 471 :invoke :write [7 38] INFO [2019-12-17 12:27:59,830] jepsen worker 28 - jepsen.util 298 :ok :read [9 2] INFO [2019-12-17 12:27:59,830] jepsen worker 28 - jepsen.util 298 :invoke :cas [9 [12 8]] INFO [2019-12-17 12:27:59,834] jepsen worker 16 - jepsen.util 376 :info :write [5 45] :RequestCanceledException INFO [2019-12-17 12:27:59,835] jepsen worker 16 - jepsen.util 466 :invoke :write [5 37] INFO [2019-12-17 12:27:59,849] jepsen worker 28 - jepsen.util 298 :fail :cas [9 [12 8]] :ValueNotSwapFrom INFO [2019-12-17 12:27:59,849] jepsen worker 28 - jepsen.util 298 :invoke :cas [9 [36 38]] INFO [2019-12-17 12:27:59,851] jepsen worker 66 - jepsen.util 426 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:27:59,851] jepsen worker 68 - jepsen.util 338 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:27:59,851] jepsen worker 66 - jepsen.util 426 :invoke :read [22 nil] INFO [2019-12-17 12:27:59,851] jepsen worker 68 - jepsen.util 338 :invoke :read [22 nil] INFO [2019-12-17 12:27:59,865] jepsen worker 28 - jepsen.util 298 :fail :cas [9 [36 38]] :ValueNotSwapFrom INFO [2019-12-17 12:27:59,866] jepsen worker 28 - jepsen.util 298 :invoke :read [9 nil] INFO [2019-12-17 12:27:59,876] jepsen worker 28 - jepsen.util 298 :ok :read [9 2] INFO [2019-12-17 12:27:59,877] jepsen worker 28 - jepsen.util 298 :invoke :read [9 nil] INFO [2019-12-17 12:27:59,890] jepsen worker 28 - jepsen.util 298 :ok :read [9 2] INFO [2019-12-17 12:27:59,890] jepsen worker 28 - jepsen.util 298 :invoke :read [9 nil] INFO [2019-12-17 12:27:59,900] jepsen worker 28 - jepsen.util 298 :ok :read [9 2] INFO [2019-12-17 12:27:59,900] jepsen worker 28 - jepsen.util 298 :invoke :cas [9 [46 25]] INFO [2019-12-17 12:27:59,909] jepsen worker 5 - jepsen.util 185 :fail :cas [1 [0 8]] :ValueNotSwapFrom INFO [2019-12-17 12:27:59,909] jepsen worker 5 - jepsen.util 185 :invoke :cas [1 [12 20]] INFO [2019-12-17 12:27:59,913] jepsen worker 28 - jepsen.util 298 :fail :cas [9 [46 25]] :ValueNotSwapFrom INFO [2019-12-17 12:27:59,913] jepsen worker 28 - jepsen.util 298 :invoke :cas [9 [18 31]] INFO [2019-12-17 12:27:59,919] jepsen worker 51 - jepsen.util 231 :info :write [17 40] :RequestCanceledException INFO [2019-12-17 12:27:59,919] jepsen worker 45 - jepsen.util 585 :info :write [15 26] :RequestCanceledException INFO [2019-12-17 12:27:59,920] jepsen worker 45 - jepsen.util 675 :invoke :read [15 nil] INFO [2019-12-17 12:27:59,920] jepsen worker 51 - jepsen.util 321 :invoke :write [17 30] INFO [2019-12-17 12:27:59,935] jepsen worker 53 - jepsen.util 503 :info :write [17 40] :RequestCanceledException INFO [2019-12-17 12:27:59,936] jepsen worker 53 - jepsen.util 593 :invoke :read [17 nil] INFO [2019-12-17 12:27:59,944] jepsen worker 28 - jepsen.util 298 :fail :cas [9 [18 31]] :ValueNotSwapFrom INFO [2019-12-17 12:27:59,944] jepsen worker 28 - jepsen.util 298 :invoke :read [9 nil] INFO [2019-12-17 12:27:59,947] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2316,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:27:59,947] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2316,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:27:59,947] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2319,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:27:59,947] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2319,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:27:59,947] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2320,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:27:59,947] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2320,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:27:59,957] jepsen worker 28 - jepsen.util 298 :ok :read [9 2] INFO [2019-12-17 12:27:59,957] jepsen worker 52 - jepsen.util 412 :fail :cas [17 [17 31]] :ValueNotSwapFrom INFO [2019-12-17 12:27:59,957] jepsen worker 28 - jepsen.util 298 :invoke :write [9 21] INFO [2019-12-17 12:27:59,957] jepsen worker 52 - jepsen.util 412 :invoke :write [17 0] INFO [2019-12-17 12:27:59,979] jepsen worker 38 - jepsen.util 308 :fail :cas [12 [47 40]] :ValueNotSwapFrom INFO [2019-12-17 12:27:59,979] jepsen worker 38 - jepsen.util 308 :invoke :write [12 20] INFO [2019-12-17 12:27:59,981] jepsen worker 56 - jepsen.util 416 :ok :read [18 47] INFO [2019-12-17 12:27:59,981] jepsen worker 56 - jepsen.util 416 :invoke :read [18 nil] INFO [2019-12-17 12:27:59,987] jepsen worker 32 - jepsen.util 392 :info :cas [10 [46 49]] :RequestCanceledException INFO [2019-12-17 12:27:59,989] jepsen worker 32 - jepsen.util 482 :invoke :cas [10 [4 24]] INFO [2019-12-17 12:28:00,010] jepsen worker 79 - jepsen.util 439 :ok :read [26 47] INFO [2019-12-17 12:28:00,010] jepsen worker 79 - jepsen.util 439 :invoke :read [26 nil] INFO [2019-12-17 12:28:00,011] jepsen worker 32 - jepsen.util 482 :fail :cas [10 [4 24]] :ValueNotSwapFrom INFO [2019-12-17 12:28:00,011] jepsen worker 32 - jepsen.util 482 :invoke :cas [10 [29 22]] INFO [2019-12-17 12:28:00,020] jepsen worker 48 - jepsen.util 408 :info :cas [16 [7 33]] :RequestCanceledException INFO [2019-12-17 12:28:00,022] jepsen worker 48 - jepsen.util 498 :invoke :cas [16 [10 36]] INFO [2019-12-17 12:28:00,025] jepsen worker 28 - jepsen.util 298 :ok :write [9 21] INFO [2019-12-17 12:28:00,026] jepsen worker 28 - jepsen.util 298 :invoke :write [9 26] INFO [2019-12-17 12:28:00,029] jepsen worker 32 - jepsen.util 482 :fail :cas [10 [29 22]] :ValueNotSwapFrom INFO [2019-12-17 12:28:00,029] jepsen worker 32 - jepsen.util 482 :invoke :cas [10 [9 22]] INFO [2019-12-17 12:28:00,034] jepsen worker 77 - jepsen.util 437 :fail :cas [25 [10 29]] :ValueNotSwapFrom INFO [2019-12-17 12:28:00,034] jepsen worker 77 - jepsen.util 437 :invoke :cas [25 [42 6]] INFO [2019-12-17 12:28:00,034] jepsen worker 40 - jepsen.util 490 :info :cas [13 [27 47]] :RequestCanceledException INFO [2019-12-17 12:28:00,036] jepsen worker 40 - jepsen.util 580 :invoke :write [13 2] INFO [2019-12-17 12:28:00,051] jepsen worker 32 - jepsen.util 482 :fail :cas [10 [9 22]] :ValueNotSwapFrom INFO [2019-12-17 12:28:00,051] jepsen worker 13 - jepsen.util 463 :ok :read [4 4] INFO [2019-12-17 12:28:00,051] jepsen worker 32 - jepsen.util 482 :invoke :write [10 45] INFO [2019-12-17 12:28:00,051] jepsen worker 13 - jepsen.util 463 :invoke :write [4 17] INFO [2019-12-17 12:28:00,051] jepsen worker 6 - jepsen.util 456 :info :write [2 3] :RequestCanceledException INFO [2019-12-17 12:28:00,053] jepsen worker 6 - jepsen.util 546 :invoke :cas [2 [29 46]] INFO [2019-12-17 12:28:00,058] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2335,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:28:00,058] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2335,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:28:00,058] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2336,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:28:00,058] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2336,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:28:00,080] jepsen worker 78 - jepsen.util 528 :ok :read [26 47] INFO [2019-12-17 12:28:00,080] jepsen worker 78 - jepsen.util 528 :invoke :cas [26 [2 3]] INFO [2019-12-17 12:28:00,088] jepsen worker 31 - jepsen.util 661 :info :cas [10 [9 48]] :RequestCanceledException INFO [2019-12-17 12:28:00,089] jepsen worker 31 - jepsen.util 751 :invoke :read [10 nil] INFO [2019-12-17 12:28:00,103] jepsen worker 28 - jepsen.util 298 :ok :write [9 26] INFO [2019-12-17 12:28:00,103] jepsen worker 28 - jepsen.util 298 :invoke :cas [9 [0 47]] INFO [2019-12-17 12:28:00,106] jepsen worker 81 - jepsen.util 441 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:28:00,106] jepsen worker 81 - jepsen.util 441 :invoke :cas [27 [6 17]] INFO [2019-12-17 12:28:00,120] jepsen worker 29 - jepsen.util 389 :info :write [9 43] :RequestCanceledException INFO [2019-12-17 12:28:00,121] jepsen worker 24 - jepsen.util 384 :ok :read [8 14] INFO [2019-12-17 12:28:00,121] jepsen worker 70 - jepsen.util 520 :info :write [23 49] :RequestCanceledException INFO [2019-12-17 12:28:00,121] jepsen worker 24 - jepsen.util 384 :invoke :cas [8 [37 7]] INFO [2019-12-17 12:28:00,122] jepsen worker 29 - jepsen.util 479 :invoke :write [9 45] INFO [2019-12-17 12:28:00,122] jepsen worker 70 - jepsen.util 610 :invoke :write [23 32] INFO [2019-12-17 12:28:00,128] jepsen worker 1 - jepsen.util 541 :info :cas [0 [41 12]] :RequestCanceledException INFO [2019-12-17 12:28:00,130] jepsen worker 1 - jepsen.util 631 :invoke :read [0 nil] INFO [2019-12-17 12:28:00,133] jepsen worker 28 - jepsen.util 298 :fail :cas [9 [0 47]] :ValueNotSwapFrom INFO [2019-12-17 12:28:00,133] jepsen worker 28 - jepsen.util 298 :invoke :cas [9 [45 33]] INFO [2019-12-17 12:28:00,155] jepsen worker 28 - jepsen.util 298 :fail :cas [9 [45 33]] :ValueNotSwapFrom INFO [2019-12-17 12:28:00,155] jepsen worker 28 - jepsen.util 298 :invoke :read [9 nil] INFO [2019-12-17 12:28:00,172] jepsen worker 72 - jepsen.util 342 :info :write [24 29] :SyncWriteAmbiguous INFO [2019-12-17 12:28:00,173] jepsen worker 72 - jepsen.util 432 :invoke :write [24 39] INFO [2019-12-17 12:28:00,178] jepsen worker 28 - jepsen.util 298 :ok :read [9 26] INFO [2019-12-17 12:28:00,178] jepsen worker 28 - jepsen.util 298 :invoke :cas [9 [5 40]] INFO [2019-12-17 12:28:00,199] jepsen worker 63 - jepsen.util 243 :ok :write [21 30] INFO [2019-12-17 12:28:00,200] jepsen worker 63 - jepsen.util 243 :invoke :cas [21 [1 29]] INFO [2019-12-17 12:28:00,221] jepsen worker 55 - jepsen.util 595 :info :write [18 11] :RequestCanceledException INFO [2019-12-17 12:28:00,221] jepsen worker 18 - jepsen.util 468 :info :write [6 49] :RequestCanceledException INFO [2019-12-17 12:28:00,222] jepsen worker 55 - jepsen.util 685 :invoke :read [18 nil] INFO [2019-12-17 12:28:00,222] jepsen worker 18 - jepsen.util 558 :invoke :read [6 nil] INFO [2019-12-17 12:28:00,222] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2364,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:28:00,222] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2364,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:28:00,222] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2366,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:28:00,222] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2366,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:28:00,229] jepsen worker 85 - jepsen.util 445 :info :cas [28 [49 6]] :RequestCanceledException INFO [2019-12-17 12:28:00,231] jepsen worker 85 - jepsen.util 535 :invoke :write [28 9] INFO [2019-12-17 12:28:00,233] jepsen worker 30 - jepsen.util 390 :fail :cas [10 [35 12]] :ValueNotSwapFrom INFO [2019-12-17 12:28:00,234] jepsen worker 30 - jepsen.util 390 :invoke :read [10 nil] INFO [2019-12-17 12:28:00,240] jepsen worker 45 - jepsen.util 675 :ok :read [15 35] INFO [2019-12-17 12:28:00,240] jepsen worker 45 - jepsen.util 675 :invoke :read [15 nil] INFO [2019-12-17 12:28:00,242] jepsen worker 53 - jepsen.util 593 :ok :read [17 0] INFO [2019-12-17 12:28:00,242] jepsen worker 53 - jepsen.util 593 :invoke :cas [17 [40 2]] INFO [2019-12-17 12:28:00,250] jepsen worker 83 - jepsen.util 533 :info :cas [27 [41 37]] :RequestCanceledException INFO [2019-12-17 12:28:00,252] jepsen worker 83 - jepsen.util 623 :invoke :read [27 nil] INFO [2019-12-17 12:28:00,254] jepsen worker 3 - jepsen.util 453 :ok :read [1 23] INFO [2019-12-17 12:28:00,254] jepsen worker 3 - jepsen.util 453 :invoke :cas [1 [19 25]] INFO [2019-12-17 12:28:00,254] jepsen worker 63 - jepsen.util 243 :fail :cas [21 [1 29]] :ValueNotSwapFrom INFO [2019-12-17 12:28:00,254] jepsen worker 63 - jepsen.util 243 :invoke :read [21 nil] INFO [2019-12-17 12:28:00,261] jepsen worker 30 - jepsen.util 390 :ok :read [10 23] INFO [2019-12-17 12:28:00,261] jepsen worker 30 - jepsen.util 390 :invoke :read [10 nil] INFO [2019-12-17 12:28:00,261] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2374,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:28:00,262] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2374,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:28:00,262] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2375,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:28:00,262] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2375,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:28:00,262] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2376,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:28:00,262] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2376,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:28:00,270] jepsen worker 45 - jepsen.util 675 :ok :read [15 35] INFO [2019-12-17 12:28:00,270] jepsen worker 45 - jepsen.util 675 :invoke :cas [15 [23 15]] INFO [2019-12-17 12:28:00,279] jepsen worker 63 - jepsen.util 243 :ok :read [21 30] INFO [2019-12-17 12:28:00,279] jepsen worker 63 - jepsen.util 243 :invoke :read [21 nil] INFO [2019-12-17 12:28:00,285] jepsen worker 86 - jepsen.util 266 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:28:00,285] jepsen worker 86 - jepsen.util 266 :invoke :cas [28 [48 7]] INFO [2019-12-17 12:28:00,287] jepsen worker 80 - jepsen.util 710 :ok :read [26 47] INFO [2019-12-17 12:28:00,287] jepsen worker 80 - jepsen.util 710 :invoke :read [26 nil] INFO [2019-12-17 12:28:00,291] jepsen worker 30 - jepsen.util 390 :ok :read [10 23] INFO [2019-12-17 12:28:00,291] jepsen worker 30 - jepsen.util 390 :invoke :read [10 nil] INFO [2019-12-17 12:28:00,298] jepsen worker 45 - jepsen.util 675 :fail :cas [15 [23 15]] :ValueNotSwapFrom INFO [2019-12-17 12:28:00,299] jepsen worker 45 - jepsen.util 675 :invoke :write [15 49] INFO [2019-12-17 12:28:00,306] jepsen worker 89 - jepsen.util 449 :info :write [29 5] :RequestCanceledException INFO [2019-12-17 12:28:00,307] jepsen worker 89 - jepsen.util 539 :invoke :cas [29 [46 27]] INFO [2019-12-17 12:28:00,312] jepsen worker 63 - jepsen.util 243 :ok :read [21 30] INFO [2019-12-17 12:28:00,312] jepsen worker 63 - jepsen.util 243 :invoke :read [21 nil] INFO [2019-12-17 12:28:00,314] jepsen worker 8 - jepsen.util 548 :ok :read [2 15] INFO [2019-12-17 12:28:00,315] jepsen worker 8 - jepsen.util 548 :invoke :cas [2 [23 13]] INFO [2019-12-17 12:28:00,327] jepsen worker 89 - jepsen.util 539 :fail :cas [29 [46 27]] :ValueNotSwapFrom INFO [2019-12-17 12:28:00,328] jepsen worker 89 - jepsen.util 539 :invoke :write [29 28] INFO [2019-12-17 12:28:00,328] jepsen worker 30 - jepsen.util 390 :ok :read [10 23] INFO [2019-12-17 12:28:00,328] jepsen worker 30 - jepsen.util 390 :invoke :write [10 26] INFO [2019-12-17 12:28:00,352] jepsen worker 39 - jepsen.util 489 :info :write [13 15] :RequestCanceledException INFO [2019-12-17 12:28:00,353] jepsen worker 39 - jepsen.util 579 :invoke :read [13 nil] INFO [2019-12-17 12:28:00,362] jepsen worker 88 - jepsen.util 358 :fail :cas [29 [22 0]] :ValueNotSwapFrom INFO [2019-12-17 12:28:00,362] jepsen worker 88 - jepsen.util 358 :invoke :read [29 nil] INFO [2019-12-17 12:28:00,391] jepsen worker 60 - jepsen.util 600 :fail :cas [20 [3 31]] :ValueNotSwapFrom INFO [2019-12-17 12:28:00,391] jepsen worker 60 - jepsen.util 600 :invoke :read [20 nil] INFO [2019-12-17 12:28:00,398] jepsen worker 56 - jepsen.util 416 :ok :read [18 47] INFO [2019-12-17 12:28:00,398] jepsen worker 56 - jepsen.util 416 :invoke :write [18 44] INFO [2019-12-17 12:28:00,406] jepsen worker 43 - jepsen.util 583 :info :write [14 37] :RequestCanceledException INFO [2019-12-17 12:28:00,406] jepsen worker 54 - jepsen.util 504 :info :cas [18 [47 22]] :RequestCanceledException INFO [2019-12-17 12:28:00,407] jepsen worker 43 - jepsen.util 673 :invoke :write [14 23] INFO [2019-12-17 12:28:00,407] jepsen worker 54 - jepsen.util 594 :invoke :read [18 nil] INFO [2019-12-17 12:28:00,480] jepsen worker 5 - jepsen.util 185 :fail :cas [1 [12 20]] :ValueNotSwapFrom INFO [2019-12-17 12:28:00,480] jepsen worker 5 - jepsen.util 185 :invoke :write [1 4] INFO [2019-12-17 12:28:00,483] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestTimeoutException: CarrierBucketConfigRequest {"retried":0,"reason":"TIMEOUT","requestId":2068,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:28:00,483] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2405,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:28:00,483] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2405,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:28:00,483] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2409,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:28:00,483] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2409,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:28:00,505] jepsen worker 58 - jepsen.util 238 :info :cas [19 [45 39]] :RequestCanceledException INFO [2019-12-17 12:28:00,506] jepsen worker 58 - jepsen.util 328 :invoke :cas [19 [29 8]] INFO [2019-12-17 12:28:00,521] jepsen worker 57 - jepsen.util 417 :info :write [19 26] :RequestCanceledException INFO [2019-12-17 12:28:00,521] jepsen worker 26 - jepsen.util 566 :info :write [8 30] :RequestCanceledException INFO [2019-12-17 12:28:00,522] jepsen worker 57 - jepsen.util 507 :invoke :write [19 31] INFO [2019-12-17 12:28:00,522] jepsen worker 26 - jepsen.util 656 :invoke :write [8 4] INFO [2019-12-17 12:28:00,549] jepsen worker 61 - jepsen.util 601 :ok :read [20 7] INFO [2019-12-17 12:28:00,550] jepsen worker 61 - jepsen.util 601 :invoke :write [20 31] INFO [2019-12-17 12:28:00,550] jepsen worker 87 - jepsen.util 357 :fail :cas [29 [49 23]] :ValueNotSwapFrom INFO [2019-12-17 12:28:00,550] jepsen worker 87 - jepsen.util 357 :invoke :cas [29 [24 13]] INFO [2019-12-17 12:28:00,552] jepsen worker 9 - jepsen.util 369 :info :write [3 39] :RequestCanceledException INFO [2019-12-17 12:28:00,553] jepsen worker 9 - jepsen.util 459 :invoke :write [3 18] INFO [2019-12-17 12:28:00,572] jepsen worker 24 - jepsen.util 384 :fail :cas [8 [37 7]] :ValueNotSwapFrom INFO [2019-12-17 12:28:00,573] jepsen worker 24 - jepsen.util 384 :invoke :write [8 0] INFO [2019-12-17 12:28:00,588] jepsen worker 0 - jepsen.util 540 :info :cas [0 [48 1]] :RequestCanceledException INFO [2019-12-17 12:28:00,589] jepsen worker 0 - jepsen.util 630 :invoke :write [0 39] INFO [2019-12-17 12:28:00,606] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestTimeoutException: CarrierBucketConfigRequest {"retried":0,"reason":"TIMEOUT","requestId":2079,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:28:00,617] jepsen worker 55 - jepsen.util 685 :ok :read [18 47] INFO [2019-12-17 12:28:00,618] jepsen worker 55 - jepsen.util 685 :invoke :cas [18 [40 20]] INFO [2019-12-17 12:28:00,620] jepsen worker 77 - jepsen.util 437 :fail :cas [25 [42 6]] :ValueNotSwapFrom INFO [2019-12-17 12:28:00,620] jepsen worker 77 - jepsen.util 437 :invoke :cas [25 [41 42]] INFO [2019-12-17 12:28:00,635] jepsen worker 73 - jepsen.util 523 :info :cas [24 [49 0]] :RequestCanceledException INFO [2019-12-17 12:28:00,636] jepsen worker 73 - jepsen.util 613 :invoke :read [24 nil] INFO [2019-12-17 12:28:00,687] jepsen worker 59 - jepsen.util 329 :info :cas [19 [30 37]] :RequestCanceledException INFO [2019-12-17 12:28:00,688] jepsen worker 59 - jepsen.util 419 :invoke :cas [19 [36 15]] INFO [2019-12-17 12:28:00,696] jepsen worker 55 - jepsen.util 685 :fail :cas [18 [40 20]] :ValueNotSwapFrom INFO [2019-12-17 12:28:00,696] jepsen worker 55 - jepsen.util 685 :invoke :cas [18 [13 13]] INFO [2019-12-17 12:28:00,705] jepsen worker 47 - jepsen.util 587 :info :write [15 19] :RequestCanceledException INFO [2019-12-17 12:28:00,706] jepsen worker 47 - jepsen.util 677 :invoke :write [15 4] INFO [2019-12-17 12:28:00,717] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2425,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:28:00,717] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2425,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:28:00,717] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2427,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:28:00,718] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2427,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:28:00,721] jepsen worker 20 - jepsen.util 470 :info :write [6 48] :RequestCanceledException INFO [2019-12-17 12:28:00,722] jepsen worker 20 - jepsen.util 560 :invoke :cas [6 [6 46]] INFO [2019-12-17 12:28:00,726] jepsen worker 10 - jepsen.util 370 :info :write [3 40] :SyncWriteAmbiguous INFO [2019-12-17 12:28:00,727] jepsen worker 10 - jepsen.util 460 :invoke :write [3 15] INFO [2019-12-17 12:28:00,729] jepsen worker 8 - jepsen.util 548 :fail :cas [2 [23 13]] :ValueNotSwapFrom INFO [2019-12-17 12:28:00,729] jepsen worker 8 - jepsen.util 548 :invoke :write [2 13] INFO [2019-12-17 12:28:00,752] jepsen worker 49 - jepsen.util 319 :info :cas [16 [17 23]] :RequestCanceledException INFO [2019-12-17 12:28:00,753] jepsen worker 49 - jepsen.util 409 :invoke :read [16 nil] INFO [2019-12-17 12:28:00,776] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.05859375}, :ns_1@172.28.128.126 {:progress 0.0539568345323741}, :ns_1@172.28.128.127 {:progress 0.1089494163424124}, :ns_1@172.28.128.128 {:progress 0.2339622641509433}} INFO [2019-12-17 12:28:00,776] jepsen nemesis - couchbase.util Rebalance check count: 12 INFO [2019-12-17 12:28:00,776] jepsen nemesis - couchbase.util Rebalance stuck count: 0 INFO [2019-12-17 12:28:00,805] jepsen worker 76 - jepsen.util 436 :info :write [25 38] :RequestCanceledException INFO [2019-12-17 12:28:00,806] jepsen worker 76 - jepsen.util 526 :invoke :cas [25 [16 5]] INFO [2019-12-17 12:28:00,820] jepsen worker 7 - jepsen.util 547 :info :write [2 24] :RequestCanceledException INFO [2019-12-17 12:28:00,821] jepsen worker 7 - jepsen.util 637 :invoke :cas [2 [38 46]] INFO [2019-12-17 12:28:00,836] jepsen worker 87 - jepsen.util 357 :fail :cas [29 [24 13]] :ValueNotSwapFrom INFO [2019-12-17 12:28:00,837] jepsen worker 87 - jepsen.util 357 :invoke :write [29 29] INFO [2019-12-17 12:28:00,849] jepsen worker 25 - jepsen.util 475 :info :write [8 3] :RequestCanceledException INFO [2019-12-17 12:28:00,850] jepsen worker 25 - jepsen.util 565 :invoke :write [8 28] INFO [2019-12-17 12:28:00,867] jepsen worker 78 - jepsen.util 528 :fail :cas [26 [2 3]] :ValueNotSwapFrom INFO [2019-12-17 12:28:00,867] jepsen worker 78 - jepsen.util 528 :invoke :cas [26 [27 30]] INFO [2019-12-17 12:28:00,896] jepsen worker 6 - jepsen.util 546 :fail :cas [2 [29 46]] :ValueNotSwapFrom INFO [2019-12-17 12:28:00,896] jepsen worker 6 - jepsen.util 546 :invoke :read [2 nil] INFO [2019-12-17 12:28:00,899] jepsen worker 71 - jepsen.util 431 :ok :read [23 29] INFO [2019-12-17 12:28:00,900] jepsen worker 71 - jepsen.util 431 :invoke :read [23 nil] INFO [2019-12-17 12:28:00,917] jepsen worker 7 - jepsen.util 637 :fail :cas [2 [38 46]] :ValueNotSwapFrom INFO [2019-12-17 12:28:00,918] jepsen worker 7 - jepsen.util 637 :invoke :write [2 13] INFO [2019-12-17 12:28:00,922] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2439,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:28:00,922] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2439,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:28:00,922] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2442,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:28:00,922] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2442,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:28:00,935] jepsen worker 65 - jepsen.util 425 :fail :read [21 nil] :RequestTimeoutException INFO [2019-12-17 12:28:00,935] jepsen worker 12 - jepsen.util 372 :info :write [4 9] :RequestCanceledException INFO [2019-12-17 12:28:00,935] jepsen worker 64 - jepsen.util 334 :info :cas [21 [21 43]] :RequestCanceledException INFO [2019-12-17 12:28:00,935] jepsen worker 23 - jepsen.util 293 :info :write [7 29] :RequestCanceledException INFO [2019-12-17 12:28:00,935] jepsen worker 65 - jepsen.util 425 :invoke :cas [21 [43 17]] INFO [2019-12-17 12:28:00,936] jepsen worker 23 - jepsen.util 383 :invoke :read [7 nil] INFO [2019-12-17 12:28:00,936] jepsen worker 64 - jepsen.util 424 :invoke :cas [21 [1 20]] INFO [2019-12-17 12:28:00,936] jepsen worker 12 - jepsen.util 462 :invoke :read [4 nil] INFO [2019-12-17 12:28:00,952] jepsen worker 14 - jepsen.util 464 :info :write [4 12] :RequestCanceledException INFO [2019-12-17 12:28:00,953] jepsen worker 14 - jepsen.util 554 :invoke :read [4 nil] INFO [2019-12-17 12:28:01,002] jepsen worker 67 - jepsen.util 607 :info :write [22 43] :RequestCanceledException INFO [2019-12-17 12:28:01,003] jepsen worker 67 - jepsen.util 697 :invoke :cas [22 [49 30]] INFO [2019-12-17 12:28:01,020] jepsen worker 69 - jepsen.util 519 :info :write [23 48] :RequestCanceledException INFO [2019-12-17 12:28:01,020] jepsen worker 42 - jepsen.util 672 :info :write [14 36] :RequestCanceledException INFO [2019-12-17 12:28:01,021] jepsen worker 69 - jepsen.util 609 :invoke :cas [23 [26 17]] INFO [2019-12-17 12:28:01,021] jepsen worker 42 - jepsen.util 762 :invoke :write [14 41] INFO [2019-12-17 12:28:01,028] jepsen worker 79 - jepsen.util 439 :ok :read [26 47] INFO [2019-12-17 12:28:01,028] jepsen worker 79 - jepsen.util 439 :invoke :cas [26 [1 10]] INFO [2019-12-17 12:28:01,032] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestTimeoutException: CarrierBucketConfigRequest {"retried":0,"reason":"TIMEOUT","requestId":2141,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:28:01,035] jepsen worker 11 - jepsen.util 461 :info :write [3 26] :RequestCanceledException INFO [2019-12-17 12:28:01,036] jepsen worker 11 - jepsen.util 551 :invoke :write [3 9] INFO [2019-12-17 12:28:01,041] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2459,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:28:01,042] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2459,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:28:01,096] jepsen worker 80 - jepsen.util 710 :ok :read [26 47] INFO [2019-12-17 12:28:01,096] jepsen worker 80 - jepsen.util 710 :invoke :read [26 nil] INFO [2019-12-17 12:28:01,103] jepsen worker 22 - jepsen.util 472 :info :write [7 36] :RequestCanceledException INFO [2019-12-17 12:28:01,103] jepsen worker 22 - jepsen.util 562 :invoke :read [7 nil] INFO [2019-12-17 12:28:01,119] jepsen worker 3 - jepsen.util 453 :fail :cas [1 [19 25]] :ValueNotSwapFrom INFO [2019-12-17 12:28:01,119] jepsen worker 3 - jepsen.util 453 :invoke :read [1 nil] INFO [2019-12-17 12:28:01,121] jepsen worker 62 - jepsen.util 512 :info :write [20 2] :RequestCanceledException INFO [2019-12-17 12:28:01,123] jepsen worker 62 - jepsen.util 602 :invoke :cas [20 [44 15]] INFO [2019-12-17 12:28:01,182] jepsen worker 54 - jepsen.util 594 :ok :read [18 47] INFO [2019-12-17 12:28:01,182] jepsen worker 77 - jepsen.util 437 :fail :cas [25 [41 42]] :ValueNotSwapFrom INFO [2019-12-17 12:28:01,182] jepsen worker 54 - jepsen.util 594 :invoke :read [18 nil] INFO [2019-12-17 12:28:01,182] jepsen worker 77 - jepsen.util 437 :invoke :cas [25 [2 14]] INFO [2019-12-17 12:28:01,192] jepsen worker 22 - jepsen.util 562 :ok :read [7 20] INFO [2019-12-17 12:28:01,192] jepsen worker 22 - jepsen.util 562 :invoke :read [7 nil] INFO [2019-12-17 12:28:01,204] jepsen worker 55 - jepsen.util 685 :fail :cas [18 [13 13]] :ValueNotSwapFrom INFO [2019-12-17 12:28:01,204] jepsen worker 55 - jepsen.util 685 :invoke :write [18 2] INFO [2019-12-17 12:28:01,206] jepsen worker 53 - jepsen.util 593 :fail :cas [17 [40 2]] :ValueNotSwapFrom INFO [2019-12-17 12:28:01,206] jepsen worker 19 - jepsen.util 469 :info :write [6 34] :SyncWriteAmbiguous INFO [2019-12-17 12:28:01,206] jepsen worker 53 - jepsen.util 593 :invoke :cas [17 [34 27]] INFO [2019-12-17 12:28:01,207] jepsen worker 19 - jepsen.util 559 :invoke :write [6 27] INFO [2019-12-17 12:28:01,207] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2472,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:28:01,208] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2472,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:28:01,208] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2473,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:28:01,208] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2473,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:28:01,240] jepsen worker 76 - jepsen.util 526 :fail :cas [25 [16 5]] :ValueNotSwapFrom INFO [2019-12-17 12:28:01,241] jepsen worker 76 - jepsen.util 526 :invoke :cas [25 [45 16]] INFO [2019-12-17 12:28:01,244] jepsen worker 12 - jepsen.util 462 :ok :read [4 4] INFO [2019-12-17 12:28:01,244] jepsen worker 12 - jepsen.util 462 :invoke :read [4 nil] INFO [2019-12-17 12:28:01,247] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2479,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:28:01,247] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2479,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:28:01,282] jepsen worker 50 - jepsen.util 590 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:28:01,283] jepsen worker 50 - jepsen.util 590 :invoke :read [16 nil] INFO [2019-12-17 12:28:01,287] jepsen worker 60 - jepsen.util 600 :ok :read [20 7] INFO [2019-12-17 12:28:01,287] jepsen worker 60 - jepsen.util 600 :invoke :cas [20 [19 39]] INFO [2019-12-17 12:28:01,290] jepsen worker 29 - jepsen.util 479 :ok :write [9 45] INFO [2019-12-17 12:28:01,290] jepsen worker 29 - jepsen.util 479 :invoke :read [9 nil] INFO [2019-12-17 12:28:01,371] jepsen worker 62 - jepsen.util 602 :fail :cas [20 [44 15]] :ValueNotSwapFrom INFO [2019-12-17 12:28:01,371] jepsen worker 62 - jepsen.util 602 :invoke :cas [20 [26 7]] INFO [2019-12-17 12:28:01,385] jepsen worker 71 - jepsen.util 431 :ok :read [23 29] INFO [2019-12-17 12:28:01,385] jepsen worker 71 - jepsen.util 431 :invoke :cas [23 [13 47]] INFO [2019-12-17 12:28:01,421] jepsen worker 27 - jepsen.util 207 :info :write [9 42] :RequestCanceledException INFO [2019-12-17 12:28:01,421] jepsen worker 37 - jepsen.util 397 :info :write [12 16] :RequestCanceledException INFO [2019-12-17 12:28:01,421] jepsen worker 46 - jepsen.util 316 :info :write [15 24] :RequestCanceledException INFO [2019-12-17 12:28:01,422] jepsen worker 27 - jepsen.util 297 :invoke :cas [9 [27 27]] INFO [2019-12-17 12:28:01,422] jepsen worker 46 - jepsen.util 406 :invoke :write [15 2] INFO [2019-12-17 12:28:01,423] jepsen worker 37 - jepsen.util 487 :invoke :cas [12 [23 19]] INFO [2019-12-17 12:28:01,429] jepsen worker 14 - jepsen.util 554 :ok :read [4 4] INFO [2019-12-17 12:28:01,429] jepsen worker 14 - jepsen.util 554 :invoke :write [4 43] INFO [2019-12-17 12:28:01,433] jepsen worker 76 - jepsen.util 526 :fail :cas [25 [45 16]] :ValueNotSwapFrom INFO [2019-12-17 12:28:01,433] jepsen worker 76 - jepsen.util 526 :invoke :cas [25 [44 32]] INFO [2019-12-17 12:28:01,434] jepsen worker 44 - jepsen.util 224 :info :write [14 25] :RequestCanceledException INFO [2019-12-17 12:28:01,434] jepsen worker 17 - jepsen.util 557 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:28:01,435] jepsen worker 17 - jepsen.util 557 :invoke :cas [5 [3 14]] INFO [2019-12-17 12:28:01,436] jepsen worker 44 - jepsen.util 314 :invoke :cas [14 [0 36]] INFO [2019-12-17 12:28:01,457] jepsen worker 78 - jepsen.util 528 :fail :cas [26 [27 30]] :ValueNotSwapFrom INFO [2019-12-17 12:28:01,457] jepsen worker 78 - jepsen.util 528 :invoke :write [26 38] INFO [2019-12-17 12:28:01,467] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2497,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:28:01,467] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2497,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:28:01,467] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2500,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:28:01,467] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2500,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:28:01,470] jepsen worker 62 - jepsen.util 602 :fail :cas [20 [26 7]] :ValueNotSwapFrom INFO [2019-12-17 12:28:01,471] jepsen worker 62 - jepsen.util 602 :invoke :cas [20 [35 46]] INFO [2019-12-17 12:28:01,483] jepsen worker 79 - jepsen.util 439 :fail :cas [26 [1 10]] :ValueNotSwapFrom INFO [2019-12-17 12:28:01,483] jepsen worker 79 - jepsen.util 439 :invoke :write [26 30] INFO [2019-12-17 12:28:01,520] jepsen worker 2 - jepsen.util 362 :info :cas [0 [45 22]] :RequestCanceledException INFO [2019-12-17 12:28:01,521] jepsen worker 2 - jepsen.util 452 :invoke :write [0 40] INFO [2019-12-17 12:28:01,541] jepsen worker 80 - jepsen.util 710 :ok :read [26 47] INFO [2019-12-17 12:28:01,541] jepsen worker 80 - jepsen.util 710 :invoke :read [26 nil] INFO [2019-12-17 12:28:01,547] jepsen worker 3 - jepsen.util 453 :ok :read [1 23] INFO [2019-12-17 12:28:01,548] jepsen worker 3 - jepsen.util 453 :invoke :read [1 nil] INFO [2019-12-17 12:28:01,554] jepsen worker 41 - jepsen.util 311 :ok :read [13 9] INFO [2019-12-17 12:28:01,554] jepsen worker 41 - jepsen.util 311 :invoke :cas [13 [39 14]] INFO [2019-12-17 12:28:01,556] jepsen worker 23 - jepsen.util 383 :ok :read [7 20] INFO [2019-12-17 12:28:01,556] jepsen worker 23 - jepsen.util 383 :invoke :write [7 44] INFO [2019-12-17 12:28:01,575] jepsen worker 48 - jepsen.util 498 :fail :cas [16 [10 36]] :ValueNotSwapFrom INFO [2019-12-17 12:28:01,576] jepsen worker 48 - jepsen.util 498 :invoke :write [16 42] INFO [2019-12-17 12:28:01,577] jepsen worker 54 - jepsen.util 594 :ok :read [18 47] INFO [2019-12-17 12:28:01,577] jepsen worker 54 - jepsen.util 594 :invoke :cas [18 [4 2]] INFO [2019-12-17 12:28:01,577] jepsen worker 76 - jepsen.util 526 :fail :cas [25 [44 32]] :ValueNotSwapFrom INFO [2019-12-17 12:28:01,578] jepsen worker 76 - jepsen.util 526 :invoke :write [25 49] INFO [2019-12-17 12:28:01,579] jepsen worker 71 - jepsen.util 431 :fail :cas [23 [13 47]] :ValueNotSwapFrom INFO [2019-12-17 12:28:01,580] jepsen worker 71 - jepsen.util 431 :invoke :read [23 nil] INFO [2019-12-17 12:28:01,582] jepsen worker 1 - jepsen.util 631 :ok :read [0 14] INFO [2019-12-17 12:28:01,582] jepsen worker 1 - jepsen.util 631 :invoke :write [0 24] INFO [2019-12-17 12:28:01,601] jepsen worker 17 - jepsen.util 557 :fail :cas [5 [3 14]] :ValueNotSwapFrom INFO [2019-12-17 12:28:01,602] jepsen worker 17 - jepsen.util 557 :invoke :write [5 13] INFO [2019-12-17 12:28:01,604] jepsen worker 74 - jepsen.util 434 :info :write [24 18] :RequestCanceledException INFO [2019-12-17 12:28:01,605] jepsen worker 74 - jepsen.util 524 :invoke :cas [24 [47 27]] INFO [2019-12-17 12:28:01,624] jepsen worker 41 - jepsen.util 311 :fail :cas [13 [39 14]] :ValueNotSwapFrom INFO [2019-12-17 12:28:01,625] jepsen worker 41 - jepsen.util 311 :invoke :read [13 nil] INFO [2019-12-17 12:28:01,641] jepsen worker 37 - jepsen.util 487 :fail :cas [12 [23 19]] :ValueNotSwapFrom INFO [2019-12-17 12:28:01,641] jepsen worker 37 - jepsen.util 487 :invoke :read [12 nil] INFO [2019-12-17 12:28:01,664] jepsen worker 62 - jepsen.util 602 :fail :cas [20 [35 46]] :ValueNotSwapFrom INFO [2019-12-17 12:28:01,665] jepsen worker 62 - jepsen.util 602 :invoke :write [20 25] INFO [2019-12-17 12:28:01,687] jepsen worker 34 - jepsen.util 484 :info :write [11 27] :RequestCanceledException INFO [2019-12-17 12:28:01,688] jepsen worker 34 - jepsen.util 574 :invoke :write [11 47] INFO [2019-12-17 12:28:01,704] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2527,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:28:01,704] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2527,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:28:01,720] jepsen worker 33 - jepsen.util 483 :info :write [11 33] :RequestCanceledException INFO [2019-12-17 12:28:01,721] jepsen worker 33 - jepsen.util 573 :invoke :read [11 nil] INFO [2019-12-17 12:28:01,752] jepsen worker 84 - jepsen.util 264 :info :write [28 1] :RequestCanceledException INFO [2019-12-17 12:28:01,753] jepsen worker 84 - jepsen.util 354 :invoke :read [28 nil] INFO [2019-12-17 12:28:01,772] jepsen worker 3 - jepsen.util 453 :ok :read [1 23] INFO [2019-12-17 12:28:01,772] jepsen worker 3 - jepsen.util 453 :invoke :write [1 37] INFO [2019-12-17 12:28:01,840] jepsen worker 4 - jepsen.util 454 :info :write [1 21] :RequestCanceledException INFO [2019-12-17 12:28:01,840] jepsen worker 21 - jepsen.util 471 :info :write [7 38] :RequestCanceledException INFO [2019-12-17 12:28:01,840] jepsen worker 82 - jepsen.util 622 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:28:01,840] jepsen worker 82 - jepsen.util 622 :invoke :cas [27 [11 39]] INFO [2019-12-17 12:28:01,840] jepsen worker 15 - jepsen.util 465 :info :cas [5 [40 15]] :RequestCanceledException INFO [2019-12-17 12:28:01,840] jepsen worker 75 - jepsen.util 345 :info :write [25 25] :RequestCanceledException INFO [2019-12-17 12:28:01,841] jepsen worker 4 - jepsen.util 544 :invoke :read [1 nil] INFO [2019-12-17 12:28:01,841] jepsen worker 21 - jepsen.util 561 :invoke :write [7 27] INFO [2019-12-17 12:28:01,841] jepsen worker 15 - jepsen.util 555 :invoke :write [5 1] INFO [2019-12-17 12:28:01,841] jepsen worker 75 - jepsen.util 435 :invoke :cas [25 [6 37]] INFO [2019-12-17 12:28:01,845] jepsen worker 33 - jepsen.util 573 :ok :read [11 7] INFO [2019-12-17 12:28:01,845] jepsen worker 33 - jepsen.util 573 :invoke :cas [11 [24 49]] INFO [2019-12-17 12:28:01,848] jepsen worker 36 - jepsen.util 486 :info :write [12 0] :RequestCanceledException INFO [2019-12-17 12:28:01,848] jepsen worker 35 - jepsen.util 305 :info :write [11 38] :RequestCanceledException INFO [2019-12-17 12:28:01,848] jepsen worker 16 - jepsen.util 466 :info :write [5 37] :RequestCanceledException INFO [2019-12-17 12:28:01,848] jepsen worker 31 - jepsen.util 751 :ok :read [10 23] INFO [2019-12-17 12:28:01,848] jepsen worker 31 - jepsen.util 751 :invoke :cas [10 [49 20]] INFO [2019-12-17 12:28:01,849] jepsen worker 35 - jepsen.util 395 :invoke :read [11 nil] INFO [2019-12-17 12:28:01,849] jepsen worker 16 - jepsen.util 556 :invoke :cas [5 [32 26]] INFO [2019-12-17 12:28:01,849] jepsen worker 36 - jepsen.util 576 :invoke :write [12 22] INFO [2019-12-17 12:28:01,855] jepsen worker 18 - jepsen.util 558 :ok :read [6 45] INFO [2019-12-17 12:28:01,855] jepsen worker 18 - jepsen.util 558 :invoke :cas [6 [46 31]] INFO [2019-12-17 12:28:01,855] jepsen worker 60 - jepsen.util 600 :fail :cas [20 [19 39]] :ValueNotSwapFrom INFO [2019-12-17 12:28:01,856] jepsen worker 60 - jepsen.util 600 :invoke :cas [20 [31 1]] INFO [2019-12-17 12:28:01,863] jepsen worker 73 - jepsen.util 613 :ok :read [24 35] INFO [2019-12-17 12:28:01,863] jepsen worker 73 - jepsen.util 613 :invoke :cas [24 [9 3]] INFO [2019-12-17 12:28:01,877] jepsen worker 31 - jepsen.util 751 :fail :cas [10 [49 20]] :ValueNotSwapFrom INFO [2019-12-17 12:28:01,877] jepsen worker 31 - jepsen.util 751 :invoke :cas [10 [40 13]] INFO [2019-12-17 12:28:01,883] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.05859375}, :ns_1@172.28.128.126 {:progress 0.0539568345323741}, :ns_1@172.28.128.127 {:progress 0.132295719844358}, :ns_1@172.28.128.128 {:progress 0.279245283018868}} INFO [2019-12-17 12:28:01,883] jepsen nemesis - couchbase.util Rebalance check count: 13 INFO [2019-12-17 12:28:01,883] jepsen nemesis - couchbase.util Rebalance stuck count: 0 INFO [2019-12-17 12:28:01,891] jepsen worker 18 - jepsen.util 558 :fail :cas [6 [46 31]] :ValueNotSwapFrom INFO [2019-12-17 12:28:01,892] jepsen worker 18 - jepsen.util 558 :invoke :cas [6 [7 47]] INFO [2019-12-17 12:28:01,903] jepsen worker 73 - jepsen.util 613 :fail :cas [24 [9 3]] :ValueNotSwapFrom INFO [2019-12-17 12:28:01,904] jepsen worker 73 - jepsen.util 613 :invoke :write [24 1] INFO [2019-12-17 12:28:01,917] jepsen worker 31 - jepsen.util 751 :fail :cas [10 [40 13]] :ValueNotSwapFrom INFO [2019-12-17 12:28:01,917] jepsen worker 31 - jepsen.util 751 :invoke :read [10 nil] INFO [2019-12-17 12:28:01,918] jepsen worker 32 - jepsen.util 482 :info :write [10 45] :SyncWriteAmbiguous INFO [2019-12-17 12:28:01,919] jepsen worker 32 - jepsen.util 572 :invoke :write [10 1] INFO [2019-12-17 12:28:01,946] jepsen worker 18 - jepsen.util 558 :fail :cas [6 [7 47]] :ValueNotSwapFrom INFO [2019-12-17 12:28:01,946] jepsen worker 18 - jepsen.util 558 :invoke :write [6 23] INFO [2019-12-17 12:28:01,950] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2546,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:28:01,950] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2546,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:28:01,951] jepsen worker 68 - jepsen.util 338 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:28:01,951] jepsen worker 66 - jepsen.util 426 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:28:01,951] jepsen worker 66 - jepsen.util 426 :invoke :cas [22 [19 33]] INFO [2019-12-17 12:28:01,951] jepsen worker 68 - jepsen.util 338 :invoke :cas [22 [4 42]] INFO [2019-12-17 12:28:01,973] jepsen worker 88 - jepsen.util 358 :ok :read [29 5] INFO [2019-12-17 12:28:01,973] jepsen worker 88 - jepsen.util 358 :invoke :write [29 37] INFO [2019-12-17 12:28:01,985] jepsen worker 51 - jepsen.util 321 :info :write [17 30] :RequestCanceledException INFO [2019-12-17 12:28:01,987] jepsen worker 51 - jepsen.util 411 :invoke :write [17 15] INFO [2019-12-17 12:28:02,002] jepsen worker 52 - jepsen.util 412 :info :write [17 0] :RequestCanceledException INFO [2019-12-17 12:28:02,003] jepsen worker 52 - jepsen.util 502 :invoke :read [17 nil] INFO [2019-12-17 12:28:02,017] jepsen worker 31 - jepsen.util 751 :ok :read [10 23] INFO [2019-12-17 12:28:02,017] jepsen worker 31 - jepsen.util 751 :invoke :cas [10 [1 5]] INFO [2019-12-17 12:28:02,024] jepsen worker 72 - jepsen.util 432 :info :write [24 39] :SyncWriteAmbiguous INFO [2019-12-17 12:28:02,025] jepsen worker 72 - jepsen.util 522 :invoke :cas [24 [11 29]] INFO [2019-12-17 12:28:02,031] jepsen worker 12 - jepsen.util 462 :ok :read [4 4] INFO [2019-12-17 12:28:02,031] jepsen worker 12 - jepsen.util 462 :invoke :write [4 41] INFO [2019-12-17 12:28:02,035] jepsen worker 38 - jepsen.util 308 :info :write [12 20] :RequestCanceledException INFO [2019-12-17 12:28:02,036] jepsen worker 38 - jepsen.util 398 :invoke :write [12 37] INFO [2019-12-17 12:28:02,043] jepsen worker 28 - jepsen.util 298 :fail :cas [9 [5 40]] :ValueNotSwapFrom INFO [2019-12-17 12:28:02,043] jepsen worker 28 - jepsen.util 298 :invoke :write [9 30] INFO [2019-12-17 12:28:02,052] jepsen worker 40 - jepsen.util 580 :info :write [13 2] :RequestCanceledException INFO [2019-12-17 12:28:02,053] jepsen worker 40 - jepsen.util 670 :invoke :cas [13 [36 40]] INFO [2019-12-17 12:28:02,059] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2566,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:28:02,059] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2566,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:28:02,059] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2568,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:28:02,059] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2568,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:28:02,078] jepsen worker 33 - jepsen.util 573 :fail :cas [11 [24 49]] :ValueNotSwapFrom INFO [2019-12-17 12:28:02,079] jepsen worker 33 - jepsen.util 573 :invoke :cas [11 [23 33]] INFO [2019-12-17 12:28:02,080] jepsen worker 54 - jepsen.util 594 :fail :cas [18 [4 2]] :ValueNotSwapFrom INFO [2019-12-17 12:28:02,081] jepsen worker 54 - jepsen.util 594 :invoke :write [18 44] INFO [2019-12-17 12:28:02,084] jepsen worker 64 - jepsen.util 424 :fail :cas [21 [1 20]] :ValueNotSwapFrom INFO [2019-12-17 12:28:02,084] jepsen worker 64 - jepsen.util 424 :invoke :cas [21 [5 24]] INFO [2019-12-17 12:28:02,105] jepsen worker 13 - jepsen.util 463 :info :write [4 17] :RequestCanceledException INFO [2019-12-17 12:28:02,106] jepsen worker 13 - jepsen.util 553 :invoke :write [4 28] INFO [2019-12-17 12:28:02,117] jepsen worker 28 - jepsen.util 298 :ok :write [9 30] INFO [2019-12-17 12:28:02,117] jepsen worker 28 - jepsen.util 298 :invoke :read [9 nil] INFO [2019-12-17 12:28:02,130] jepsen worker 35 - jepsen.util 395 :ok :read [11 7] INFO [2019-12-17 12:28:02,131] jepsen worker 35 - jepsen.util 395 :invoke :write [11 5] INFO [2019-12-17 12:28:02,146] jepsen worker 64 - jepsen.util 424 :fail :cas [21 [5 24]] :ValueNotSwapFrom INFO [2019-12-17 12:28:02,147] jepsen worker 64 - jepsen.util 424 :invoke :write [21 17] INFO [2019-12-17 12:28:02,147] jepsen worker 60 - jepsen.util 600 :fail :cas [20 [31 1]] :ValueNotSwapFrom INFO [2019-12-17 12:28:02,147] jepsen worker 60 - jepsen.util 600 :invoke :read [20 nil] INFO [2019-12-17 12:28:02,161] jepsen worker 4 - jepsen.util 544 :ok :read [1 23] INFO [2019-12-17 12:28:02,162] jepsen worker 4 - jepsen.util 544 :invoke :cas [1 [35 15]] INFO [2019-12-17 12:28:02,174] jepsen worker 28 - jepsen.util 298 :ok :read [9 30] INFO [2019-12-17 12:28:02,174] jepsen worker 28 - jepsen.util 298 :invoke :read [9 nil] INFO [2019-12-17 12:28:02,187] jepsen worker 37 - jepsen.util 487 :ok :read [12 12] INFO [2019-12-17 12:28:02,188] jepsen worker 37 - jepsen.util 487 :invoke :write [12 11] INFO [2019-12-17 12:28:02,205] jepsen worker 81 - jepsen.util 441 :info :cas [27 [6 17]] :RequestCanceledException INFO [2019-12-17 12:28:02,206] jepsen worker 81 - jepsen.util 531 :invoke :cas [27 [47 8]] INFO [2019-12-17 12:28:02,212] jepsen worker 64 - jepsen.util 424 :ok :write [21 17] INFO [2019-12-17 12:28:02,213] jepsen worker 64 - jepsen.util 424 :invoke :read [21 nil] INFO [2019-12-17 12:28:02,214] jepsen worker 71 - jepsen.util 431 :ok :read [23 29] INFO [2019-12-17 12:28:02,214] jepsen worker 71 - jepsen.util 431 :invoke :write [23 17] INFO [2019-12-17 12:28:02,221] jepsen worker 70 - jepsen.util 610 :info :write [23 32] :RequestCanceledException INFO [2019-12-17 12:28:02,222] jepsen worker 70 - jepsen.util 700 :invoke :write [23 39] INFO [2019-12-17 12:28:02,235] jepsen worker 64 - jepsen.util 424 :ok :read [21 17] INFO [2019-12-17 12:28:02,235] jepsen worker 64 - jepsen.util 424 :invoke :cas [21 [27 25]] INFO [2019-12-17 12:28:02,251] jepsen worker 64 - jepsen.util 424 :fail :cas [21 [27 25]] :ValueNotSwapFrom INFO [2019-12-17 12:28:02,251] jepsen worker 64 - jepsen.util 424 :invoke :cas [21 [15 37]] INFO [2019-12-17 12:28:02,261] jepsen worker 75 - jepsen.util 435 :fail :cas [25 [6 37]] :ValueNotSwapFrom INFO [2019-12-17 12:28:02,261] jepsen worker 75 - jepsen.util 435 :invoke :cas [25 [4 45]] INFO [2019-12-17 12:28:02,265] jepsen worker 64 - jepsen.util 424 :fail :cas [21 [15 37]] :ValueNotSwapFrom INFO [2019-12-17 12:28:02,265] jepsen worker 64 - jepsen.util 424 :invoke :read [21 nil] INFO [2019-12-17 12:28:02,278] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2595,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:28:02,278] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2595,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:28:02,278] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2599,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:28:02,278] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2599,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:28:02,282] jepsen worker 64 - jepsen.util 424 :ok :read [21 17] INFO [2019-12-17 12:28:02,282] jepsen worker 64 - jepsen.util 424 :invoke :read [21 nil] INFO [2019-12-17 12:28:02,296] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2587,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:28:02,296] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2587,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:28:02,296] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2589,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:28:02,296] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2589,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:28:02,297] jepsen worker 64 - jepsen.util 424 :ok :read [21 17] INFO [2019-12-17 12:28:02,297] jepsen worker 64 - jepsen.util 424 :invoke :cas [21 [3 25]] INFO [2019-12-17 12:28:02,304] jepsen worker 85 - jepsen.util 535 :info :write [28 9] :RequestCanceledException INFO [2019-12-17 12:28:02,305] jepsen worker 85 - jepsen.util 625 :invoke :cas [28 [33 19]] INFO [2019-12-17 12:28:02,316] jepsen worker 64 - jepsen.util 424 :fail :cas [21 [3 25]] :ValueNotSwapFrom INFO [2019-12-17 12:28:02,316] jepsen worker 64 - jepsen.util 424 :invoke :write [21 41] INFO [2019-12-17 12:28:02,320] jepsen worker 83 - jepsen.util 623 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:28:02,320] jepsen worker 45 - jepsen.util 675 :info :write [15 49] :RequestCanceledException INFO [2019-12-17 12:28:02,321] jepsen worker 83 - jepsen.util 623 :invoke :write [27 23] INFO [2019-12-17 12:28:02,322] jepsen worker 45 - jepsen.util 765 :invoke :read [15 nil] INFO [2019-12-17 12:28:02,384] jepsen worker 64 - jepsen.util 424 :ok :write [21 41] INFO [2019-12-17 12:28:02,384] jepsen worker 64 - jepsen.util 424 :invoke :cas [21 [21 34]] INFO [2019-12-17 12:28:02,387] jepsen worker 86 - jepsen.util 266 :info :cas [28 [48 7]] :RequestCanceledException INFO [2019-12-17 12:28:02,388] jepsen worker 86 - jepsen.util 356 :invoke :cas [28 [28 32]] INFO [2019-12-17 12:28:02,398] jepsen worker 45 - jepsen.util 765 :ok :read [15 35] INFO [2019-12-17 12:28:02,399] jepsen worker 45 - jepsen.util 765 :invoke :write [15 10] INFO [2019-12-17 12:28:02,410] jepsen worker 64 - jepsen.util 424 :fail :cas [21 [21 34]] :ValueNotSwapFrom INFO [2019-12-17 12:28:02,410] jepsen worker 64 - jepsen.util 424 :invoke :read [21 nil] INFO [2019-12-17 12:28:02,420] jepsen worker 30 - jepsen.util 390 :info :write [10 26] :RequestCanceledException INFO [2019-12-17 12:28:02,420] jepsen worker 63 - jepsen.util 243 :fail :read [21 nil] :RequestTimeoutException INFO [2019-12-17 12:28:02,420] jepsen worker 63 - jepsen.util 243 :invoke :read [21 nil] INFO [2019-12-17 12:28:02,421] jepsen worker 30 - jepsen.util 480 :invoke :write [10 38] INFO [2019-12-17 12:28:02,435] jepsen worker 89 - jepsen.util 539 :info :write [29 28] :RequestCanceledException INFO [2019-12-17 12:28:02,437] jepsen worker 89 - jepsen.util 629 :invoke :write [29 39] INFO [2019-12-17 12:28:02,442] jepsen worker 63 - jepsen.util 243 :ok :read [21 41] INFO [2019-12-17 12:28:02,443] jepsen worker 63 - jepsen.util 243 :invoke :cas [21 [39 26]] INFO [2019-12-17 12:28:02,451] jepsen worker 39 - jepsen.util 579 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:28:02,452] jepsen worker 39 - jepsen.util 579 :invoke :write [13 40] INFO [2019-12-17 12:28:02,454] jepsen worker 48 - jepsen.util 498 :ok :write [16 42] INFO [2019-12-17 12:28:02,454] jepsen worker 48 - jepsen.util 498 :invoke :write [16 27] INFO [2019-12-17 12:28:02,457] jepsen worker 33 - jepsen.util 573 :fail :cas [11 [23 33]] :ValueNotSwapFrom INFO [2019-12-17 12:28:02,457] jepsen worker 33 - jepsen.util 573 :invoke :write [11 30] INFO [2019-12-17 12:28:02,459] jepsen worker 63 - jepsen.util 243 :fail :cas [21 [39 26]] :ValueNotSwapFrom INFO [2019-12-17 12:28:02,459] jepsen worker 63 - jepsen.util 243 :invoke :read [21 nil] INFO [2019-12-17 12:28:02,477] jepsen worker 41 - jepsen.util 311 :ok :read [13 9] INFO [2019-12-17 12:28:02,477] jepsen worker 41 - jepsen.util 311 :invoke :write [13 36] INFO [2019-12-17 12:28:02,479] jepsen worker 63 - jepsen.util 243 :ok :read [21 41] INFO [2019-12-17 12:28:02,479] jepsen worker 63 - jepsen.util 243 :invoke :cas [21 [10 32]] INFO [2019-12-17 12:28:02,497] jepsen worker 80 - jepsen.util 710 :ok :read [26 47] INFO [2019-12-17 12:28:02,497] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2621,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:28:02,497] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2621,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:28:02,497] jepsen worker 80 - jepsen.util 710 :invoke :write [26 18] INFO [2019-12-17 12:28:02,497] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2625,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:28:02,497] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2625,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:28:02,499] jepsen worker 63 - jepsen.util 243 :fail :cas [21 [10 32]] :ValueNotSwapFrom INFO [2019-12-17 12:28:02,499] jepsen worker 63 - jepsen.util 243 :invoke :cas [21 [28 29]] INFO [2019-12-17 12:28:02,499] jepsen worker 56 - jepsen.util 416 :info :write [18 44] :RequestCanceledException INFO [2019-12-17 12:28:02,501] jepsen worker 56 - jepsen.util 506 :invoke :cas [18 [48 22]] INFO [2019-12-17 12:28:02,520] jepsen worker 43 - jepsen.util 673 :info :write [14 23] :RequestCanceledException INFO [2019-12-17 12:28:02,521] jepsen worker 43 - jepsen.util 763 :invoke :read [14 nil] INFO [2019-12-17 12:28:02,522] jepsen worker 63 - jepsen.util 243 :fail :cas [21 [28 29]] :ValueNotSwapFrom INFO [2019-12-17 12:28:02,523] jepsen worker 63 - jepsen.util 243 :invoke :read [21 nil] INFO [2019-12-17 12:28:02,544] jepsen worker 63 - jepsen.util 243 :ok :read [21 41] INFO [2019-12-17 12:28:02,545] jepsen worker 63 - jepsen.util 243 :invoke :write [21 9] INFO [2019-12-17 12:28:02,561] jepsen worker 75 - jepsen.util 435 :fail :cas [25 [4 45]] :ValueNotSwapFrom INFO [2019-12-17 12:28:02,562] jepsen worker 75 - jepsen.util 435 :invoke :write [25 17] INFO [2019-12-17 12:28:02,573] jepsen worker 20 - jepsen.util 560 :fail :cas [6 [6 46]] :ValueNotSwapFrom INFO [2019-12-17 12:28:02,573] jepsen worker 20 - jepsen.util 560 :invoke :read [6 nil] INFO [2019-12-17 12:28:02,583] jepsen worker 5 - jepsen.util 185 :info :write [1 4] :RequestCanceledException INFO [2019-12-17 12:28:02,584] jepsen worker 5 - jepsen.util 275 :invoke :read [1 nil] INFO [2019-12-17 12:28:02,596] jepsen worker 65 - jepsen.util 425 :fail :cas [21 [43 17]] :ValueNotSwapFrom INFO [2019-12-17 12:28:02,596] jepsen worker 65 - jepsen.util 425 :invoke :write [21 6] INFO [2019-12-17 12:28:02,604] jepsen worker 58 - jepsen.util 328 :info :cas [19 [29 8]] :RequestCanceledException INFO [2019-12-17 12:28:02,605] jepsen worker 58 - jepsen.util 418 :invoke :cas [19 [2 2]] INFO [2019-12-17 12:28:02,611] jepsen worker 1 - jepsen.util 631 :ok :write [0 24] INFO [2019-12-17 12:28:02,611] jepsen worker 1 - jepsen.util 631 :invoke :cas [0 [44 10]] INFO [2019-12-17 12:28:02,620] jepsen worker 61 - jepsen.util 601 :info :write [20 31] :RequestCanceledException INFO [2019-12-17 12:28:02,620] jepsen worker 57 - jepsen.util 507 :info :write [19 31] :RequestCanceledException INFO [2019-12-17 12:28:02,621] jepsen worker 0 - jepsen.util 630 :info :write [0 39] :RequestCanceledException INFO [2019-12-17 12:28:02,622] jepsen worker 61 - jepsen.util 691 :invoke :read [20 nil] INFO [2019-12-17 12:28:02,622] jepsen worker 57 - jepsen.util 597 :invoke :cas [19 [26 10]] INFO [2019-12-17 12:28:02,622] jepsen worker 0 - jepsen.util 720 :invoke :read [0 nil] INFO [2019-12-17 12:28:02,628] jepsen worker 6 - jepsen.util 546 :ok :read [2 15] INFO [2019-12-17 12:28:02,628] jepsen worker 6 - jepsen.util 546 :invoke :read [2 nil] INFO [2019-12-17 12:28:02,635] jepsen worker 26 - jepsen.util 656 :info :write [8 4] :RequestCanceledException INFO [2019-12-17 12:28:02,636] jepsen worker 26 - jepsen.util 746 :invoke :read [8 nil] INFO [2019-12-17 12:28:02,652] jepsen worker 24 - jepsen.util 384 :info :write [8 0] :RequestCanceledException INFO [2019-12-17 12:28:02,652] jepsen worker 9 - jepsen.util 459 :info :write [3 18] :RequestCanceledException INFO [2019-12-17 12:28:02,653] jepsen worker 24 - jepsen.util 474 :invoke :cas [8 [12 45]] INFO [2019-12-17 12:28:02,653] jepsen worker 9 - jepsen.util 549 :invoke :read [3 nil] INFO [2019-12-17 12:28:02,677] jepsen worker 3 - jepsen.util 453 :ok :write [1 37] INFO [2019-12-17 12:28:02,677] jepsen worker 3 - jepsen.util 453 :invoke :cas [1 [20 36]] INFO [2019-12-17 12:28:02,686] jepsen worker 72 - jepsen.util 522 :fail :cas [24 [11 29]] :ValueNotSwapFrom INFO [2019-12-17 12:28:02,686] jepsen worker 72 - jepsen.util 522 :invoke :cas [24 [43 10]] INFO [2019-12-17 12:28:02,700] jepsen worker 20 - jepsen.util 560 :ok :read [6 45] INFO [2019-12-17 12:28:02,700] jepsen worker 20 - jepsen.util 560 :invoke :read [6 nil] INFO [2019-12-17 12:28:02,705] jepsen worker 59 - jepsen.util 419 :info :cas [19 [36 15]] :RequestCanceledException INFO [2019-12-17 12:28:02,707] jepsen worker 59 - jepsen.util 509 :invoke :cas [19 [26 27]] INFO [2019-12-17 12:28:02,712] jepsen worker 53 - jepsen.util 593 :fail :cas [17 [34 27]] :ValueNotSwapFrom INFO [2019-12-17 12:28:02,712] jepsen worker 53 - jepsen.util 593 :invoke :read [17 nil] INFO [2019-12-17 12:28:02,716] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2652,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:28:02,716] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2652,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:28:02,716] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2656,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:28:02,716] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2656,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:28:02,717] jepsen worker 4 - jepsen.util 544 :fail :cas [1 [35 15]] :ValueNotSwapFrom INFO [2019-12-17 12:28:02,717] jepsen worker 4 - jepsen.util 544 :invoke :read [1 nil] INFO [2019-12-17 12:28:02,721] jepsen worker 47 - jepsen.util 677 :info :write [15 4] :RequestCanceledException INFO [2019-12-17 12:28:02,722] jepsen worker 47 - jepsen.util 767 :invoke :read [15 nil] INFO [2019-12-17 12:28:02,727] jepsen worker 63 - jepsen.util 243 :ok :write [21 9] INFO [2019-12-17 12:28:02,728] jepsen worker 63 - jepsen.util 243 :invoke :write [21 35] INFO [2019-12-17 12:28:02,745] jepsen worker 72 - jepsen.util 522 :fail :cas [24 [43 10]] :ValueNotSwapFrom INFO [2019-12-17 12:28:02,745] jepsen worker 72 - jepsen.util 522 :invoke :cas [24 [20 7]] INFO [2019-12-17 12:28:02,752] jepsen worker 10 - jepsen.util 460 :info :write [3 15] :RequestCanceledException INFO [2019-12-17 12:28:02,753] jepsen worker 10 - jepsen.util 550 :invoke :cas [3 [42 23]] INFO [2019-12-17 12:28:02,773] jepsen worker 52 - jepsen.util 502 :ok :read [17 0] INFO [2019-12-17 12:28:02,773] jepsen worker 52 - jepsen.util 502 :invoke :write [17 34] INFO [2019-12-17 12:28:02,791] jepsen worker 20 - jepsen.util 560 :ok :read [6 45] INFO [2019-12-17 12:28:02,791] jepsen worker 20 - jepsen.util 560 :invoke :cas [6 [0 48]] INFO [2019-12-17 12:28:02,796] jepsen worker 9 - jepsen.util 549 :ok :read [3 14] INFO [2019-12-17 12:28:02,796] jepsen worker 9 - jepsen.util 549 :invoke :cas [3 [37 3]] INFO [2019-12-17 12:28:02,821] jepsen worker 49 - jepsen.util 409 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:28:02,821] jepsen worker 8 - jepsen.util 548 :info :write [2 13] :RequestCanceledException INFO [2019-12-17 12:28:02,821] jepsen worker 49 - jepsen.util 409 :invoke :read [16 nil] INFO [2019-12-17 12:28:02,822] jepsen worker 8 - jepsen.util 638 :invoke :cas [2 [49 7]] INFO [2019-12-17 12:28:02,824] jepsen worker 60 - jepsen.util 600 :ok :read [20 7] INFO [2019-12-17 12:28:02,825] jepsen worker 60 - jepsen.util 600 :invoke :cas [20 [32 31]] INFO [2019-12-17 12:28:02,827] jepsen worker 72 - jepsen.util 522 :fail :cas [24 [20 7]] :ValueNotSwapFrom INFO [2019-12-17 12:28:02,827] jepsen worker 72 - jepsen.util 522 :invoke :cas [24 [20 14]] INFO [2019-12-17 12:28:02,843] jepsen worker 20 - jepsen.util 560 :fail :cas [6 [0 48]] :ValueNotSwapFrom INFO [2019-12-17 12:28:02,843] jepsen worker 20 - jepsen.util 560 :invoke :write [6 25] INFO [2019-12-17 12:28:02,867] jepsen worker 72 - jepsen.util 522 :fail :cas [24 [20 14]] :ValueNotSwapFrom INFO [2019-12-17 12:28:02,867] jepsen worker 72 - jepsen.util 522 :invoke :cas [24 [17 17]] INFO [2019-12-17 12:28:02,902] jepsen worker 63 - jepsen.util 243 :ok :write [21 35] INFO [2019-12-17 12:28:02,902] jepsen worker 63 - jepsen.util 243 :invoke :read [21 nil] INFO [2019-12-17 12:28:02,908] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.05859375}, :ns_1@172.28.128.126 {:progress 0.0539568345323741}, :ns_1@172.28.128.127 {:progress 0.132295719844358}, :ns_1@172.28.128.128 {:progress 0.279245283018868}} INFO [2019-12-17 12:28:02,908] jepsen nemesis - couchbase.util Rebalance check count: 14 INFO [2019-12-17 12:28:02,908] jepsen nemesis - couchbase.util Rebalance stuck count: 1 INFO [2019-12-17 12:28:02,917] jepsen worker 9 - jepsen.util 549 :fail :cas [3 [37 3]] :ValueNotSwapFrom INFO [2019-12-17 12:28:02,918] jepsen worker 9 - jepsen.util 549 :invoke :write [3 23] INFO [2019-12-17 12:28:02,921] jepsen worker 25 - jepsen.util 565 :info :write [8 28] :RequestCanceledException INFO [2019-12-17 12:28:02,921] jepsen worker 87 - jepsen.util 357 :info :write [29 29] :RequestCanceledException INFO [2019-12-17 12:28:02,923] jepsen worker 25 - jepsen.util 655 :invoke :write [8 18] INFO [2019-12-17 12:28:02,923] jepsen worker 87 - jepsen.util 447 :invoke :cas [29 [29 15]] INFO [2019-12-17 12:28:02,926] jepsen worker 63 - jepsen.util 243 :ok :read [21 35] INFO [2019-12-17 12:28:02,926] jepsen worker 63 - jepsen.util 243 :invoke :write [21 38] INFO [2019-12-17 12:28:02,934] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2681,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:28:02,934] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2681,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:28:02,934] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2683,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:28:02,934] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2683,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:28:02,941] jepsen worker 24 - jepsen.util 474 :fail :cas [8 [12 45]] :ValueNotSwapFrom INFO [2019-12-17 12:28:02,941] jepsen worker 24 - jepsen.util 474 :invoke :cas [8 [19 29]] INFO [2019-12-17 12:28:02,955] jepsen worker 17 - jepsen.util 557 :ok :write [5 13] INFO [2019-12-17 12:28:02,955] jepsen worker 17 - jepsen.util 557 :invoke :cas [5 [38 45]] INFO [2019-12-17 12:28:02,956] jepsen worker 8 - jepsen.util 638 :fail :cas [2 [49 7]] :ValueNotSwapFrom INFO [2019-12-17 12:28:02,957] jepsen worker 8 - jepsen.util 638 :invoke :cas [2 [33 38]] INFO [2019-12-17 12:28:03,001] jepsen worker 60 - jepsen.util 600 :fail :cas [20 [32 31]] :ValueNotSwapFrom INFO [2019-12-17 12:28:03,001] jepsen worker 60 - jepsen.util 600 :invoke :write [20 18] INFO [2019-12-17 12:28:03,005] jepsen worker 7 - jepsen.util 637 :info :write [2 13] :RequestCanceledException INFO [2019-12-17 12:28:03,007] jepsen worker 7 - jepsen.util 727 :invoke :cas [2 [16 38]] INFO [2019-12-17 12:28:03,042] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2695,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:28:03,042] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2695,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:28:03,043] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2698,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:28:03,043] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2698,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:28:03,043] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2699,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:28:03,043] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2699,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:28:03,048] jepsen worker 11 - jepsen.util 551 :info :write [3 9] :RequestCanceledException INFO [2019-12-17 12:28:03,049] jepsen worker 11 - jepsen.util 641 :invoke :cas [3 [19 27]] INFO [2019-12-17 12:28:03,073] jepsen worker 53 - jepsen.util 593 :ok :read [17 0] INFO [2019-12-17 12:28:03,073] jepsen worker 53 - jepsen.util 593 :invoke :write [17 43] INFO [2019-12-17 12:28:03,084] jepsen worker 4 - jepsen.util 544 :ok :read [1 37] INFO [2019-12-17 12:28:03,084] jepsen worker 4 - jepsen.util 544 :invoke :cas [1 [6 6]] INFO [2019-12-17 12:28:03,105] jepsen worker 67 - jepsen.util 697 :info :cas [22 [49 30]] :RequestCanceledException INFO [2019-12-17 12:28:03,106] jepsen worker 67 - jepsen.util 787 :invoke :cas [22 [12 1]] INFO [2019-12-17 12:28:03,121] jepsen worker 19 - jepsen.util 559 :info :write [6 27] :SyncWriteAmbiguous INFO [2019-12-17 12:28:03,121] jepsen worker 69 - jepsen.util 609 :info :cas [23 [26 17]] :RequestCanceledException INFO [2019-12-17 12:28:03,122] jepsen worker 19 - jepsen.util 649 :invoke :read [6 nil] INFO [2019-12-17 12:28:03,122] jepsen worker 69 - jepsen.util 699 :invoke :cas [23 [12 8]] INFO [2019-12-17 12:28:03,135] jepsen worker 42 - jepsen.util 762 :info :write [14 41] :RequestCanceledException INFO [2019-12-17 12:28:03,136] jepsen worker 42 - jepsen.util 852 :invoke :read [14 nil] INFO [2019-12-17 12:28:03,154] jepsen worker 74 - jepsen.util 524 :fail :cas [24 [47 27]] :ValueNotSwapFrom INFO [2019-12-17 12:28:03,155] jepsen worker 74 - jepsen.util 524 :invoke :cas [24 [8 44]] INFO [2019-12-17 12:28:03,170] jepsen worker 19 - jepsen.util 649 :ok :read [6 45] INFO [2019-12-17 12:28:03,170] jepsen worker 19 - jepsen.util 649 :invoke :cas [6 [43 1]] INFO [2019-12-17 12:28:03,173] jepsen worker 47 - jepsen.util 767 :ok :read [15 35] INFO [2019-12-17 12:28:03,173] jepsen worker 47 - jepsen.util 767 :invoke :write [15 20] INFO [2019-12-17 12:28:03,188] jepsen worker 11 - jepsen.util 641 :fail :cas [3 [19 27]] :ValueNotSwapFrom INFO [2019-12-17 12:28:03,189] jepsen worker 11 - jepsen.util 641 :invoke :write [3 27] INFO [2019-12-17 12:28:03,203] jepsen worker 43 - jepsen.util 763 :ok :read [14 27] INFO [2019-12-17 12:28:03,203] jepsen worker 43 - jepsen.util 763 :invoke :cas [14 [4 0]] INFO [2019-12-17 12:28:03,214] jepsen worker 19 - jepsen.util 649 :fail :cas [6 [43 1]] :ValueNotSwapFrom INFO [2019-12-17 12:28:03,214] jepsen worker 19 - jepsen.util 649 :invoke :write [6 49] INFO [2019-12-17 12:28:03,217] jepsen worker 74 - jepsen.util 524 :fail :cas [24 [8 44]] :ValueNotSwapFrom INFO [2019-12-17 12:28:03,217] jepsen worker 74 - jepsen.util 524 :invoke :cas [24 [14 5]] INFO [2019-12-17 12:28:03,218] jepsen worker 22 - jepsen.util 562 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:28:03,218] jepsen worker 55 - jepsen.util 685 :info :write [18 2] :RequestCanceledException INFO [2019-12-17 12:28:03,219] jepsen worker 22 - jepsen.util 562 :invoke :cas [7 [4 39]] INFO [2019-12-17 12:28:03,220] jepsen worker 55 - jepsen.util 775 :invoke :write [18 7] INFO [2019-12-17 12:28:03,228] jepsen worker 26 - jepsen.util 746 :ok :read [8 14] INFO [2019-12-17 12:28:03,228] jepsen worker 26 - jepsen.util 746 :invoke :cas [8 [46 11]] INFO [2019-12-17 12:28:03,244] jepsen worker 87 - jepsen.util 447 :fail :cas [29 [29 15]] :ValueNotSwapFrom INFO [2019-12-17 12:28:03,244] jepsen worker 87 - jepsen.util 447 :invoke :read [29 nil] INFO [2019-12-17 12:28:03,262] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2721,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:28:03,262] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2721,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:28:03,263] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2724,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:28:03,263] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2724,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:28:03,263] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2725,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:28:03,263] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2725,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:28:03,264] jepsen worker 69 - jepsen.util 699 :fail :cas [23 [12 8]] :ValueNotSwapFrom INFO [2019-12-17 12:28:03,264] jepsen worker 69 - jepsen.util 699 :invoke :read [23 nil] INFO [2019-12-17 12:28:03,270] jepsen worker 74 - jepsen.util 524 :fail :cas [24 [14 5]] :ValueNotSwapFrom INFO [2019-12-17 12:28:03,270] jepsen worker 74 - jepsen.util 524 :invoke :read [24 nil] INFO [2019-12-17 12:28:03,273] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2712,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:28:03,274] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2712,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:28:03,274] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2713,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:28:03,274] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2713,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:28:03,296] jepsen worker 74 - jepsen.util 524 :ok :read [24 35] INFO [2019-12-17 12:28:03,296] jepsen worker 74 - jepsen.util 524 :invoke :write [24 39] INFO [2019-12-17 12:28:03,296] jepsen worker 61 - jepsen.util 691 :ok :read [20 7] INFO [2019-12-17 12:28:03,296] jepsen worker 61 - jepsen.util 691 :invoke :read [20 nil] INFO [2019-12-17 12:28:03,305] jepsen worker 29 - jepsen.util 479 :fail :read [9 nil] :RequestTimeoutException INFO [2019-12-17 12:28:03,305] jepsen worker 29 - jepsen.util 479 :invoke :write [9 2] INFO [2019-12-17 12:28:03,307] jepsen worker 24 - jepsen.util 474 :fail :cas [8 [19 29]] :ValueNotSwapFrom INFO [2019-12-17 12:28:03,307] jepsen worker 24 - jepsen.util 474 :invoke :cas [8 [49 31]] INFO [2019-12-17 12:28:03,372] jepsen worker 56 - jepsen.util 506 :fail :cas [18 [48 22]] :ValueNotSwapFrom INFO [2019-12-17 12:28:03,372] jepsen worker 56 - jepsen.util 506 :invoke :cas [18 [35 16]] INFO [2019-12-17 12:28:03,387] jepsen worker 50 - jepsen.util 590 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:28:03,387] jepsen worker 50 - jepsen.util 590 :invoke :read [16 nil] INFO [2019-12-17 12:28:03,391] jepsen worker 87 - jepsen.util 447 :ok :read [29 5] INFO [2019-12-17 12:28:03,391] jepsen worker 87 - jepsen.util 447 :invoke :write [29 17] INFO [2019-12-17 12:28:03,428] jepsen worker 24 - jepsen.util 474 :fail :cas [8 [49 31]] :ValueNotSwapFrom INFO [2019-12-17 12:28:03,429] jepsen worker 24 - jepsen.util 474 :invoke :cas [8 [42 30]] INFO [2019-12-17 12:28:03,434] jepsen worker 46 - jepsen.util 406 :info :write [15 2] :RequestCanceledException INFO [2019-12-17 12:28:03,435] jepsen worker 46 - jepsen.util 496 :invoke :cas [15 [43 40]] INFO [2019-12-17 12:28:03,436] jepsen worker 56 - jepsen.util 506 :fail :cas [18 [35 16]] :ValueNotSwapFrom INFO [2019-12-17 12:28:03,436] jepsen worker 56 - jepsen.util 506 :invoke :write [18 31] INFO [2019-12-17 12:28:03,482] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2745,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:28:03,483] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2745,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:28:03,487] jepsen worker 78 - jepsen.util 528 :info :write [26 38] :RequestCanceledException INFO [2019-12-17 12:28:03,487] jepsen worker 14 - jepsen.util 554 :info :write [4 43] :RequestCanceledException INFO [2019-12-17 12:28:03,488] jepsen worker 14 - jepsen.util 644 :invoke :write [4 34] INFO [2019-12-17 12:28:03,488] jepsen worker 78 - jepsen.util 618 :invoke :read [26 nil] INFO [2019-12-17 12:28:03,503] jepsen worker 5 - jepsen.util 275 :ok :read [1 37] INFO [2019-12-17 12:28:03,503] jepsen worker 5 - jepsen.util 275 :invoke :write [1 16] INFO [2019-12-17 12:28:03,519] jepsen worker 4 - jepsen.util 544 :fail :cas [1 [6 6]] :ValueNotSwapFrom INFO [2019-12-17 12:28:03,519] jepsen worker 4 - jepsen.util 544 :invoke :write [1 18] INFO [2019-12-17 12:28:03,521] jepsen worker 27 - jepsen.util 297 :info :cas [9 [27 27]] :RequestCanceledException INFO [2019-12-17 12:28:03,521] jepsen worker 44 - jepsen.util 314 :info :cas [14 [0 36]] :RequestCanceledException INFO [2019-12-17 12:28:03,521] jepsen worker 44 - jepsen.util 404 :invoke :cas [14 [34 10]] INFO [2019-12-17 12:28:03,522] jepsen worker 27 - jepsen.util 387 :invoke :write [9 32] INFO [2019-12-17 12:28:03,549] jepsen worker 6 - jepsen.util 546 :ok :read [2 15] INFO [2019-12-17 12:28:03,549] jepsen worker 6 - jepsen.util 546 :invoke :read [2 nil] INFO [2019-12-17 12:28:03,551] jepsen worker 78 - jepsen.util 618 :ok :read [26 47] INFO [2019-12-17 12:28:03,552] jepsen worker 78 - jepsen.util 618 :invoke :read [26 nil] INFO [2019-12-17 12:28:03,575] jepsen worker 24 - jepsen.util 474 :fail :cas [8 [42 30]] :ValueNotSwapFrom INFO [2019-12-17 12:28:03,575] jepsen worker 24 - jepsen.util 474 :invoke :write [8 9] INFO [2019-12-17 12:28:03,584] jepsen worker 3 - jepsen.util 453 :fail :cas [1 [20 36]] :ValueNotSwapFrom INFO [2019-12-17 12:28:03,585] jepsen worker 3 - jepsen.util 453 :invoke :cas [1 [14 19]] INFO [2019-12-17 12:28:03,587] jepsen worker 2 - jepsen.util 452 :info :write [0 40] :RequestCanceledException INFO [2019-12-17 12:28:03,587] jepsen worker 79 - jepsen.util 439 :info :write [26 30] :RequestCanceledException INFO [2019-12-17 12:28:03,588] jepsen worker 2 - jepsen.util 542 :invoke :write [0 34] INFO [2019-12-17 12:28:03,588] jepsen worker 79 - jepsen.util 529 :invoke :write [26 1] INFO [2019-12-17 12:28:03,597] jepsen worker 78 - jepsen.util 618 :ok :read [26 47] INFO [2019-12-17 12:28:03,597] jepsen worker 78 - jepsen.util 618 :invoke :read [26 nil] INFO [2019-12-17 12:28:03,630] jepsen worker 78 - jepsen.util 618 :ok :read [26 47] INFO [2019-12-17 12:28:03,630] jepsen worker 78 - jepsen.util 618 :invoke :read [26 nil] INFO [2019-12-17 12:28:03,634] jepsen worker 23 - jepsen.util 383 :info :write [7 44] :RequestCanceledException INFO [2019-12-17 12:28:03,635] jepsen worker 23 - jepsen.util 473 :invoke :read [7 nil] INFO [2019-12-17 12:28:03,656] jepsen worker 42 - jepsen.util 852 :ok :read [14 27] INFO [2019-12-17 12:28:03,656] jepsen worker 42 - jepsen.util 852 :invoke :cas [14 [17 43]] INFO [2019-12-17 12:28:03,672] jepsen worker 19 - jepsen.util 649 :ok :write [6 49] INFO [2019-12-17 12:28:03,672] jepsen worker 19 - jepsen.util 649 :invoke :write [6 39] INFO [2019-12-17 12:28:03,676] jepsen worker 32 - jepsen.util 572 :ok :write [10 1] INFO [2019-12-17 12:28:03,676] jepsen worker 32 - jepsen.util 572 :invoke :read [10 nil] INFO [2019-12-17 12:28:03,678] jepsen worker 22 - jepsen.util 562 :fail :cas [7 [4 39]] :ValueNotSwapFrom INFO [2019-12-17 12:28:03,678] jepsen worker 22 - jepsen.util 562 :invoke :write [7 19] INFO [2019-12-17 12:28:03,684] jepsen worker 44 - jepsen.util 404 :fail :cas [14 [34 10]] :ValueNotSwapFrom INFO [2019-12-17 12:28:03,684] jepsen worker 44 - jepsen.util 404 :invoke :write [14 43] INFO [2019-12-17 12:28:03,686] jepsen worker 62 - jepsen.util 602 :info :write [20 25] :RequestCanceledException INFO [2019-12-17 12:28:03,686] jepsen worker 76 - jepsen.util 526 :info :write [25 49] :RequestCanceledException INFO [2019-12-17 12:28:03,688] jepsen worker 76 - jepsen.util 616 :invoke :write [25 17] INFO [2019-12-17 12:28:03,688] jepsen worker 62 - jepsen.util 692 :invoke :write [20 34] INFO [2019-12-17 12:28:03,688] jepsen worker 78 - jepsen.util 618 :ok :read [26 47] INFO [2019-12-17 12:28:03,689] jepsen worker 78 - jepsen.util 618 :invoke :read [26 nil] INFO [2019-12-17 12:28:03,694] jepsen worker 48 - jepsen.util 498 :ok :write [16 27] INFO [2019-12-17 12:28:03,694] jepsen worker 48 - jepsen.util 498 :invoke :write [16 12] INFO [2019-12-17 12:28:03,703] jepsen worker 3 - jepsen.util 453 :fail :cas [1 [14 19]] :ValueNotSwapFrom INFO [2019-12-17 12:28:03,703] jepsen worker 3 - jepsen.util 453 :invoke :cas [1 [18 10]] INFO [2019-12-17 12:28:03,710] jepsen worker 1 - jepsen.util 631 :fail :cas [0 [44 10]] :ValueNotSwapFrom INFO [2019-12-17 12:28:03,711] jepsen worker 1 - jepsen.util 631 :invoke :read [0 nil] INFO [2019-12-17 12:28:03,721] jepsen worker 34 - jepsen.util 574 :info :write [11 47] :RequestCanceledException INFO [2019-12-17 12:28:03,722] jepsen worker 34 - jepsen.util 664 :invoke :cas [11 [40 49]] INFO [2019-12-17 12:28:03,763] jepsen worker 78 - jepsen.util 618 :ok :read [26 47] INFO [2019-12-17 12:28:03,763] jepsen worker 78 - jepsen.util 618 :invoke :read [26 nil] INFO [2019-12-17 12:28:03,791] jepsen worker 10 - jepsen.util 550 :fail :cas [3 [42 23]] :ValueNotSwapFrom INFO [2019-12-17 12:28:03,792] jepsen worker 10 - jepsen.util 550 :invoke :read [3 nil] INFO [2019-12-17 12:28:03,792] jepsen worker 69 - jepsen.util 699 :ok :read [23 29] INFO [2019-12-17 12:28:03,792] jepsen worker 69 - jepsen.util 699 :invoke :cas [23 [12 11]] INFO [2019-12-17 12:28:03,803] jepsen worker 10 - jepsen.util 550 :ok :read [3 14] INFO [2019-12-17 12:28:03,803] jepsen worker 10 - jepsen.util 550 :invoke :write [3 44] INFO [2019-12-17 12:28:03,852] jepsen worker 84 - jepsen.util 354 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:28:03,853] jepsen worker 84 - jepsen.util 354 :invoke :write [28 22] INFO [2019-12-17 12:28:03,884] jepsen worker 21 - jepsen.util 561 :info :write [7 27] :RequestCanceledException INFO [2019-12-17 12:28:03,884] jepsen worker 7 - jepsen.util 727 :fail :cas [2 [16 38]] :ValueNotSwapFrom INFO [2019-12-17 12:28:03,885] jepsen worker 7 - jepsen.util 727 :invoke :write [2 30] INFO [2019-12-17 12:28:03,885] jepsen worker 21 - jepsen.util 651 :invoke :write [7 4] INFO [2019-12-17 12:28:03,905] jepsen worker 15 - jepsen.util 555 :info :write [5 1] :RequestCanceledException INFO [2019-12-17 12:28:03,906] jepsen worker 15 - jepsen.util 645 :invoke :read [5 nil] INFO [2019-12-17 12:28:03,909] jepsen worker 78 - jepsen.util 618 :ok :read [26 47] INFO [2019-12-17 12:28:03,909] jepsen worker 78 - jepsen.util 618 :invoke :cas [26 [14 42]] INFO [2019-12-17 12:28:03,910] jepsen worker 8 - jepsen.util 638 :fail :cas [2 [33 38]] :ValueNotSwapFrom INFO [2019-12-17 12:28:03,910] jepsen worker 8 - jepsen.util 638 :invoke :cas [2 [2 43]] INFO [2019-12-17 12:28:03,918] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2791,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:28:03,919] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2791,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:28:03,919] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2792,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:28:03,919] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2792,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:28:03,919] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2793,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:28:03,919] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2793,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:28:03,919] jepsen worker 82 - jepsen.util 622 :info :cas [27 [11 39]] :RequestCanceledException INFO [2019-12-17 12:28:03,919] jepsen worker 36 - jepsen.util 576 :info :write [12 22] :RequestCanceledException INFO [2019-12-17 12:28:03,920] jepsen worker 82 - jepsen.util 712 :invoke :cas [27 [20 13]] INFO [2019-12-17 12:28:03,920] jepsen worker 36 - jepsen.util 666 :invoke :write [12 39] INFO [2019-12-17 12:28:03,926] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.0703125}, :ns_1@172.28.128.126 {:progress 0.06474820143884896}, :ns_1@172.28.128.127 {:progress 0.132295719844358}, :ns_1@172.28.128.128 {:progress 0.309433962264151}} INFO [2019-12-17 12:28:03,926] jepsen nemesis - couchbase.util Rebalance check count: 15 INFO [2019-12-17 12:28:03,926] jepsen nemesis - couchbase.util Rebalance stuck count: 1 INFO [2019-12-17 12:28:03,952] jepsen worker 16 - jepsen.util 556 :info :cas [5 [32 26]] :RequestCanceledException INFO [2019-12-17 12:28:03,953] jepsen worker 16 - jepsen.util 646 :invoke :write [5 17] INFO [2019-12-17 12:28:03,972] jepsen worker 78 - jepsen.util 618 :fail :cas [26 [14 42]] :ValueNotSwapFrom INFO [2019-12-17 12:28:03,972] jepsen worker 78 - jepsen.util 618 :invoke :read [26 nil] INFO [2019-12-17 12:28:03,987] jepsen worker 73 - jepsen.util 613 :info :write [24 1] :RequestCanceledException INFO [2019-12-17 12:28:03,987] jepsen worker 18 - jepsen.util 558 :info :write [6 23] :RequestCanceledException INFO [2019-12-17 12:28:03,988] jepsen worker 73 - jepsen.util 703 :invoke :read [24 nil] INFO [2019-12-17 12:28:03,988] jepsen worker 18 - jepsen.util 648 :invoke :cas [6 [8 34]] INFO [2019-12-17 12:28:04,004] jepsen worker 51 - jepsen.util 411 :info :write [17 15] :RequestCanceledException INFO [2019-12-17 12:28:04,005] jepsen worker 51 - jepsen.util 501 :invoke :cas [17 [11 43]] INFO [2019-12-17 12:28:04,021] jepsen worker 88 - jepsen.util 358 :info :write [29 37] :RequestCanceledException INFO [2019-12-17 12:28:04,022] jepsen worker 88 - jepsen.util 448 :invoke :write [29 32] INFO [2019-12-17 12:28:04,027] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2805,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:28:04,027] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2805,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:28:04,027] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2809,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:28:04,028] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2809,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:28:04,051] jepsen worker 68 - jepsen.util 338 :info :cas [22 [4 42]] :RequestCanceledException INFO [2019-12-17 12:28:04,051] jepsen worker 38 - jepsen.util 398 :info :write [12 37] :RequestCanceledException INFO [2019-12-17 12:28:04,051] jepsen worker 66 - jepsen.util 426 :info :cas [22 [19 33]] :RequestCanceledException INFO [2019-12-17 12:28:04,053] jepsen worker 68 - jepsen.util 428 :invoke :cas [22 [26 8]] INFO [2019-12-17 12:28:04,053] jepsen worker 38 - jepsen.util 488 :invoke :cas [12 [2 24]] INFO [2019-12-17 12:28:04,053] jepsen worker 66 - jepsen.util 516 :invoke :read [22 nil] INFO [2019-12-17 12:28:04,060] jepsen worker 61 - jepsen.util 691 :ok :read [20 7] INFO [2019-12-17 12:28:04,060] jepsen worker 61 - jepsen.util 691 :invoke :write [20 4] INFO [2019-12-17 12:28:04,075] jepsen worker 3 - jepsen.util 453 :fail :cas [1 [18 10]] :ValueNotSwapFrom INFO [2019-12-17 12:28:04,075] jepsen worker 3 - jepsen.util 453 :invoke :cas [1 [5 27]] INFO [2019-12-17 12:28:04,075] jepsen worker 6 - jepsen.util 546 :ok :read [2 15] INFO [2019-12-17 12:28:04,075] jepsen worker 6 - jepsen.util 546 :invoke :write [2 17] INFO [2019-12-17 12:28:04,116] jepsen worker 69 - jepsen.util 699 :fail :cas [23 [12 11]] :ValueNotSwapFrom INFO [2019-12-17 12:28:04,116] jepsen worker 69 - jepsen.util 699 :invoke :read [23 nil] INFO [2019-12-17 12:28:04,121] jepsen worker 40 - jepsen.util 670 :info :cas [13 [36 40]] :RequestCanceledException INFO [2019-12-17 12:28:04,121] jepsen worker 12 - jepsen.util 462 :info :write [4 41] :RequestCanceledException INFO [2019-12-17 12:28:04,122] jepsen worker 40 - jepsen.util 760 :invoke :write [13 32] INFO [2019-12-17 12:28:04,122] jepsen worker 12 - jepsen.util 552 :invoke :cas [4 [23 17]] INFO [2019-12-17 12:28:04,187] jepsen worker 54 - jepsen.util 594 :info :write [18 44] :RequestCanceledException INFO [2019-12-17 12:28:04,187] jepsen worker 13 - jepsen.util 553 :info :write [4 28] :RequestCanceledException INFO [2019-12-17 12:28:04,188] jepsen worker 13 - jepsen.util 643 :invoke :write [4 46] INFO [2019-12-17 12:28:04,188] jepsen worker 54 - jepsen.util 684 :invoke :write [18 1] INFO [2019-12-17 12:28:04,224] jepsen worker 78 - jepsen.util 618 :ok :read [26 47] INFO [2019-12-17 12:28:04,225] jepsen worker 78 - jepsen.util 618 :invoke :write [26 37] INFO [2019-12-17 12:28:04,233] jepsen worker 42 - jepsen.util 852 :fail :cas [14 [17 43]] :ValueNotSwapFrom INFO [2019-12-17 12:28:04,233] jepsen worker 42 - jepsen.util 852 :invoke :cas [14 [27 1]] INFO [2019-12-17 12:28:04,235] jepsen worker 35 - jepsen.util 395 :info :write [11 5] :RequestCanceledException INFO [2019-12-17 12:28:04,236] jepsen worker 35 - jepsen.util 485 :invoke :read [11 nil] INFO [2019-12-17 12:28:04,245] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2832,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:28:04,245] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2832,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:28:04,246] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2835,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:28:04,246] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2835,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:28:04,250] jepsen worker 37 - jepsen.util 487 :info :write [12 11] :RequestCanceledException INFO [2019-12-17 12:28:04,250] jepsen worker 71 - jepsen.util 431 :info :write [23 17] :RequestCanceledException INFO [2019-12-17 12:28:04,250] jepsen worker 28 - jepsen.util 298 :fail :read [9 nil] :RequestTimeoutException INFO [2019-12-17 12:28:04,250] jepsen worker 28 - jepsen.util 298 :invoke :read [9 nil] INFO [2019-12-17 12:28:04,251] jepsen worker 71 - jepsen.util 521 :invoke :cas [23 [30 6]] INFO [2019-12-17 12:28:04,251] jepsen worker 37 - jepsen.util 577 :invoke :read [12 nil] INFO [2019-12-17 12:28:04,257] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2823,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:28:04,257] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2823,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:28:04,257] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2825,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:28:04,257] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2825,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:28:04,267] jepsen worker 28 - jepsen.util 298 :ok :read [9 30] INFO [2019-12-17 12:28:04,267] jepsen worker 28 - jepsen.util 298 :invoke :read [9 nil] INFO [2019-12-17 12:28:04,281] jepsen worker 28 - jepsen.util 298 :ok :read [9 30] INFO [2019-12-17 12:28:04,281] jepsen worker 28 - jepsen.util 298 :invoke :cas [9 [34 40]] INFO [2019-12-17 12:28:04,294] jepsen worker 28 - jepsen.util 298 :fail :cas [9 [34 40]] :ValueNotSwapFrom INFO [2019-12-17 12:28:04,295] jepsen worker 28 - jepsen.util 298 :invoke :read [9 nil] INFO [2019-12-17 12:28:04,305] jepsen worker 81 - jepsen.util 531 :info :cas [27 [47 8]] :RequestCanceledException INFO [2019-12-17 12:28:04,306] jepsen worker 81 - jepsen.util 621 :invoke :cas [27 [11 7]] INFO [2019-12-17 12:28:04,307] jepsen worker 28 - jepsen.util 298 :ok :read [9 30] INFO [2019-12-17 12:28:04,307] jepsen worker 28 - jepsen.util 298 :invoke :cas [9 [49 20]] INFO [2019-12-17 12:28:04,312] jepsen worker 26 - jepsen.util 746 :fail :cas [8 [46 11]] :ValueNotSwapFrom INFO [2019-12-17 12:28:04,312] jepsen worker 26 - jepsen.util 746 :invoke :write [8 47] INFO [2019-12-17 12:28:04,321] jepsen worker 70 - jepsen.util 700 :info :write [23 39] :RequestCanceledException INFO [2019-12-17 12:28:04,321] jepsen worker 77 - jepsen.util 437 :info :cas [25 [2 14]] :RequestCanceledException INFO [2019-12-17 12:28:04,322] jepsen worker 77 - jepsen.util 527 :invoke :cas [25 [7 22]] INFO [2019-12-17 12:28:04,322] jepsen worker 70 - jepsen.util 790 :invoke :write [23 24] INFO [2019-12-17 12:28:04,325] jepsen worker 28 - jepsen.util 298 :fail :cas [9 [49 20]] :ValueNotSwapFrom INFO [2019-12-17 12:28:04,325] jepsen worker 28 - jepsen.util 298 :invoke :read [9 nil] INFO [2019-12-17 12:28:04,342] jepsen worker 28 - jepsen.util 298 :ok :read [9 30] INFO [2019-12-17 12:28:04,343] jepsen worker 28 - jepsen.util 298 :invoke :write [9 30] INFO [2019-12-17 12:28:04,396] jepsen worker 28 - jepsen.util 298 :ok :write [9 30] INFO [2019-12-17 12:28:04,396] jepsen worker 28 - jepsen.util 298 :invoke :write [9 20] INFO [2019-12-17 12:28:04,413] jepsen worker 9 - jepsen.util 549 :ok :write [3 23] INFO [2019-12-17 12:28:04,413] jepsen worker 9 - jepsen.util 549 :invoke :read [3 nil] INFO [2019-12-17 12:28:04,420] jepsen worker 86 - jepsen.util 356 :info :cas [28 [28 32]] :RequestCanceledException INFO [2019-12-17 12:28:04,421] jepsen worker 86 - jepsen.util 446 :invoke :write [28 37] INFO [2019-12-17 12:28:04,421] jepsen worker 83 - jepsen.util 623 :info :write [27 23] :RequestCanceledException INFO [2019-12-17 12:28:04,422] jepsen worker 83 - jepsen.util 713 :invoke :write [27 28] INFO [2019-12-17 12:28:04,435] jepsen worker 34 - jepsen.util 664 :fail :cas [11 [40 49]] :ValueNotSwapFrom INFO [2019-12-17 12:28:04,435] jepsen worker 34 - jepsen.util 664 :invoke :read [11 nil] INFO [2019-12-17 12:28:04,439] jepsen worker 37 - jepsen.util 577 :ok :read [12 12] INFO [2019-12-17 12:28:04,439] jepsen worker 37 - jepsen.util 577 :invoke :write [12 6] INFO [2019-12-17 12:28:04,446] jepsen worker 28 - jepsen.util 298 :ok :write [9 20] INFO [2019-12-17 12:28:04,446] jepsen worker 28 - jepsen.util 298 :invoke :read [9 nil] INFO [2019-12-17 12:28:04,448] jepsen worker 64 - jepsen.util 424 :fail :read [21 nil] :RequestTimeoutException INFO [2019-12-17 12:28:04,448] jepsen worker 45 - jepsen.util 765 :info :write [15 10] :RequestCanceledException INFO [2019-12-17 12:28:04,448] jepsen worker 64 - jepsen.util 424 :invoke :read [21 nil] INFO [2019-12-17 12:28:04,449] jepsen worker 45 - jepsen.util 855 :invoke :write [15 11] INFO [2019-12-17 12:28:04,464] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2860,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:28:04,465] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2860,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:28:04,467] jepsen worker 28 - jepsen.util 298 :ok :read [9 20] INFO [2019-12-17 12:28:04,467] jepsen worker 28 - jepsen.util 298 :invoke :write [9 22] INFO [2019-12-17 12:28:04,482] jepsen worker 27 - jepsen.util 387 :ok :write [9 32] INFO [2019-12-17 12:28:04,482] jepsen worker 27 - jepsen.util 387 :invoke :cas [9 [32 5]] INFO [2019-12-17 12:28:04,487] jepsen worker 30 - jepsen.util 480 :info :write [10 38] :RequestCanceledException INFO [2019-12-17 12:28:04,488] jepsen worker 30 - jepsen.util 570 :invoke :write [10 30] INFO [2019-12-17 12:28:04,494] jepsen worker 64 - jepsen.util 424 :ok :read [21 35] INFO [2019-12-17 12:28:04,494] jepsen worker 64 - jepsen.util 424 :invoke :read [21 nil] INFO [2019-12-17 12:28:04,504] jepsen worker 51 - jepsen.util 501 :fail :cas [17 [11 43]] :ValueNotSwapFrom INFO [2019-12-17 12:28:04,504] jepsen worker 51 - jepsen.util 501 :invoke :write [17 14] INFO [2019-12-17 12:28:04,505] jepsen worker 89 - jepsen.util 629 :info :write [29 39] :RequestCanceledException INFO [2019-12-17 12:28:04,507] jepsen worker 89 - jepsen.util 719 :invoke :write [29 18] INFO [2019-12-17 12:28:04,516] jepsen worker 73 - jepsen.util 703 :ok :read [24 35] INFO [2019-12-17 12:28:04,516] jepsen worker 73 - jepsen.util 703 :invoke :write [24 8] INFO [2019-12-17 12:28:04,526] jepsen worker 9 - jepsen.util 549 :ok :read [3 23] INFO [2019-12-17 12:28:04,527] jepsen worker 9 - jepsen.util 549 :invoke :cas [3 [12 10]] INFO [2019-12-17 12:28:04,535] jepsen worker 41 - jepsen.util 311 :info :write [13 36] :RequestCanceledException INFO [2019-12-17 12:28:04,535] jepsen worker 33 - jepsen.util 573 :info :write [11 30] :RequestCanceledException INFO [2019-12-17 12:28:04,536] jepsen worker 41 - jepsen.util 401 :invoke :read [13 nil] INFO [2019-12-17 12:28:04,536] jepsen worker 33 - jepsen.util 663 :invoke :cas [11 [0 43]] INFO [2019-12-17 12:28:04,551] jepsen worker 39 - jepsen.util 579 :info :write [13 40] :RequestCanceledException INFO [2019-12-17 12:28:04,553] jepsen worker 23 - jepsen.util 473 :ok :read [7 20] INFO [2019-12-17 12:28:04,553] jepsen worker 39 - jepsen.util 669 :invoke :write [13 5] INFO [2019-12-17 12:28:04,553] jepsen worker 23 - jepsen.util 473 :invoke :write [7 21] INFO [2019-12-17 12:28:04,587] jepsen worker 80 - jepsen.util 710 :info :write [26 18] :RequestCanceledException INFO [2019-12-17 12:28:04,588] jepsen worker 80 - jepsen.util 800 :invoke :cas [26 [18 47]] INFO [2019-12-17 12:28:04,600] jepsen worker 28 - jepsen.util 298 :ok :write [9 22] INFO [2019-12-17 12:28:04,601] jepsen worker 28 - jepsen.util 298 :invoke :write [9 25] INFO [2019-12-17 12:28:04,605] jepsen worker 12 - jepsen.util 552 :fail :cas [4 [23 17]] :ValueNotSwapFrom INFO [2019-12-17 12:28:04,605] jepsen worker 77 - jepsen.util 527 :fail :cas [25 [7 22]] :ValueNotSwapFrom INFO [2019-12-17 12:28:04,605] jepsen worker 77 - jepsen.util 527 :invoke :write [25 23] INFO [2019-12-17 12:28:04,605] jepsen worker 12 - jepsen.util 552 :invoke :write [4 28] INFO [2019-12-17 12:28:04,621] jepsen worker 58 - jepsen.util 418 :info :cas [19 [2 2]] :RequestCanceledException INFO [2019-12-17 12:28:04,621] jepsen worker 75 - jepsen.util 435 :info :write [25 17] :RequestCanceledException INFO [2019-12-17 12:28:04,622] jepsen worker 75 - jepsen.util 525 :invoke :read [25 nil] INFO [2019-12-17 12:28:04,622] jepsen worker 58 - jepsen.util 508 :invoke :read [19 nil] INFO [2019-12-17 12:28:04,635] jepsen worker 65 - jepsen.util 425 :info :write [21 6] :RequestCanceledException INFO [2019-12-17 12:28:04,636] jepsen worker 65 - jepsen.util 515 :invoke :cas [21 [33 17]] INFO [2019-12-17 12:28:04,679] jepsen worker 28 - jepsen.util 298 :ok :write [9 25] INFO [2019-12-17 12:28:04,679] jepsen worker 28 - jepsen.util 298 :invoke :cas [9 [39 0]] INFO [2019-12-17 12:28:04,684] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2890,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:28:04,684] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2890,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:28:04,684] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2893,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:28:04,684] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2893,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:28:04,692] jepsen worker 28 - jepsen.util 298 :fail :cas [9 [39 0]] :ValueNotSwapFrom INFO [2019-12-17 12:28:04,692] jepsen worker 28 - jepsen.util 298 :invoke :cas [9 [10 7]] INFO [2019-12-17 12:28:04,695] jepsen worker 35 - jepsen.util 485 :ok :read [11 7] INFO [2019-12-17 12:28:04,695] jepsen worker 35 - jepsen.util 485 :invoke :write [11 10] INFO [2019-12-17 12:28:04,705] jepsen worker 19 - jepsen.util 649 :ok :write [6 39] INFO [2019-12-17 12:28:04,705] jepsen worker 19 - jepsen.util 649 :invoke :write [6 28] INFO [2019-12-17 12:28:04,706] jepsen worker 28 - jepsen.util 298 :fail :cas [9 [10 7]] :ValueNotSwapFrom INFO [2019-12-17 12:28:04,706] jepsen worker 57 - jepsen.util 597 :info :cas [19 [26 10]] :RequestCanceledException INFO [2019-12-17 12:28:04,706] jepsen worker 28 - jepsen.util 298 :invoke :read [9 nil] INFO [2019-12-17 12:28:04,707] jepsen worker 57 - jepsen.util 687 :invoke :read [19 nil] INFO [2019-12-17 12:28:04,721] jepsen worker 0 - jepsen.util 720 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:28:04,721] jepsen worker 0 - jepsen.util 720 :invoke :read [0 nil] INFO [2019-12-17 12:28:04,743] jepsen worker 65 - jepsen.util 515 :fail :cas [21 [33 17]] :ValueNotSwapFrom INFO [2019-12-17 12:28:04,743] jepsen worker 65 - jepsen.util 515 :invoke :read [21 nil] INFO [2019-12-17 12:28:04,754] jepsen worker 28 - jepsen.util 298 :ok :read [9 25] INFO [2019-12-17 12:28:04,754] jepsen worker 28 - jepsen.util 298 :invoke :cas [9 [38 19]] INFO [2019-12-17 12:28:04,760] jepsen worker 17 - jepsen.util 557 :fail :cas [5 [38 45]] :ValueNotSwapFrom INFO [2019-12-17 12:28:04,760] jepsen worker 17 - jepsen.util 557 :invoke :read [5 nil] INFO [2019-12-17 12:28:04,761] jepsen worker 32 - jepsen.util 572 :ok :read [10 38] INFO [2019-12-17 12:28:04,762] jepsen worker 32 - jepsen.util 572 :invoke :write [10 41] INFO [2019-12-17 12:28:04,781] jepsen worker 59 - jepsen.util 509 :info :cas [19 [26 27]] :RequestCanceledException INFO [2019-12-17 12:28:04,783] jepsen worker 59 - jepsen.util 599 :invoke :cas [19 [35 17]] INFO [2019-12-17 12:28:04,798] jepsen worker 31 - jepsen.util 751 :fail :cas [10 [1 5]] :CASMismatchException INFO [2019-12-17 12:28:04,799] jepsen worker 31 - jepsen.util 751 :invoke :cas [10 [15 41]] INFO [2019-12-17 12:28:04,819] jepsen worker 71 - jepsen.util 521 :fail :cas [23 [30 6]] :ValueNotSwapFrom INFO [2019-12-17 12:28:04,819] jepsen worker 71 - jepsen.util 521 :invoke :read [23 nil] INFO [2019-12-17 12:28:04,819] jepsen worker 52 - jepsen.util 502 :info :write [17 34] :RequestCanceledException INFO [2019-12-17 12:28:04,821] jepsen worker 52 - jepsen.util 592 :invoke :write [17 20] INFO [2019-12-17 12:28:04,831] jepsen worker 8 - jepsen.util 638 :fail :cas [2 [2 43]] :ValueNotSwapFrom INFO [2019-12-17 12:28:04,831] jepsen worker 8 - jepsen.util 638 :invoke :cas [2 [15 43]] INFO [2019-12-17 12:28:04,862] jepsen worker 28 - jepsen.util 298 :fail :cas [9 [38 19]] :ValueNotSwapFrom INFO [2019-12-17 12:28:04,862] jepsen worker 28 - jepsen.util 298 :invoke :read [9 nil] INFO [2019-12-17 12:28:04,881] jepsen worker 28 - jepsen.util 298 :ok :read [9 25] INFO [2019-12-17 12:28:04,881] jepsen worker 28 - jepsen.util 298 :invoke :read [9 nil] INFO [2019-12-17 12:28:04,883] jepsen worker 85 - jepsen.util 625 :ok :cas [28 [33 19]] INFO [2019-12-17 12:28:04,883] jepsen worker 85 - jepsen.util 625 :invoke :cas [28 [12 31]] INFO [2019-12-17 12:28:04,900] jepsen worker 28 - jepsen.util 298 :ok :read [9 25] INFO [2019-12-17 12:28:04,900] jepsen worker 28 - jepsen.util 298 :invoke :read [9 nil] INFO [2019-12-17 12:28:04,903] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2915,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:28:04,903] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2915,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:28:04,920] jepsen worker 49 - jepsen.util 409 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:28:04,921] jepsen worker 49 - jepsen.util 409 :invoke :write [16 12] INFO [2019-12-17 12:28:04,924] jepsen worker 28 - jepsen.util 298 :ok :read [9 25] INFO [2019-12-17 12:28:04,924] jepsen worker 28 - jepsen.util 298 :invoke :write [9 42] INFO [2019-12-17 12:28:04,925] jepsen worker 75 - jepsen.util 525 :ok :read [25 2] INFO [2019-12-17 12:28:04,925] jepsen worker 75 - jepsen.util 525 :invoke :read [25 nil] INFO [2019-12-17 12:28:04,934] jepsen worker 20 - jepsen.util 560 :info :write [6 25] :RequestCanceledException INFO [2019-12-17 12:28:04,934] jepsen worker 72 - jepsen.util 522 :info :cas [24 [17 17]] :RequestCanceledException INFO [2019-12-17 12:28:04,936] jepsen worker 20 - jepsen.util 650 :invoke :read [6 nil] INFO [2019-12-17 12:28:04,936] jepsen worker 72 - jepsen.util 612 :invoke :read [24 nil] INFO [2019-12-17 12:28:04,952] jepsen worker 25 - jepsen.util 655 :info :write [8 18] :RequestCanceledException INFO [2019-12-17 12:28:04,953] jepsen worker 25 - jepsen.util 745 :invoke :cas [8 [15 32]] INFO [2019-12-17 12:28:04,964] jepsen worker 38 - jepsen.util 488 :fail :cas [12 [2 24]] :ValueNotSwapFrom INFO [2019-12-17 12:28:04,964] jepsen worker 38 - jepsen.util 488 :invoke :write [12 13] INFO [2019-12-17 12:28:04,970] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.0703125}, :ns_1@172.28.128.126 {:progress 0.06474820143884896}, :ns_1@172.28.128.127 {:progress 0.1478599221789884}, :ns_1@172.28.128.128 {:progress 0.339622641509434}} INFO [2019-12-17 12:28:04,970] jepsen nemesis - couchbase.util Rebalance check count: 16 INFO [2019-12-17 12:28:04,970] jepsen nemesis - couchbase.util Rebalance stuck count: 1 INFO [2019-12-17 12:28:04,977] jepsen worker 72 - jepsen.util 612 :ok :read [24 35] INFO [2019-12-17 12:28:04,977] jepsen worker 72 - jepsen.util 612 :invoke :write [24 35] INFO [2019-12-17 12:28:04,992] jepsen worker 3 - jepsen.util 453 :fail :cas [1 [5 27]] :ValueNotSwapFrom INFO [2019-12-17 12:28:04,992] jepsen worker 3 - jepsen.util 453 :invoke :read [1 nil] INFO [2019-12-17 12:28:05,013] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2930,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:28:05,013] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2930,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:28:05,013] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2933,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:28:05,013] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2933,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:28:05,013] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2934,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:28:05,013] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2934,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:28:05,021] jepsen worker 63 - jepsen.util 243 :info :write [21 38] :RequestCanceledException INFO [2019-12-17 12:28:05,022] jepsen worker 63 - jepsen.util 333 :invoke :cas [21 [1 3]] INFO [2019-12-17 12:28:05,024] jepsen worker 28 - jepsen.util 298 :ok :write [9 42] INFO [2019-12-17 12:28:05,024] jepsen worker 28 - jepsen.util 298 :invoke :write [9 30] INFO [2019-12-17 12:28:05,035] jepsen worker 60 - jepsen.util 600 :info :write [20 18] :RequestCanceledException INFO [2019-12-17 12:28:05,036] jepsen worker 60 - jepsen.util 690 :invoke :read [20 nil] INFO [2019-12-17 12:28:05,044] jepsen worker 63 - jepsen.util 333 :fail :cas [21 [1 3]] :ValueNotSwapFrom INFO [2019-12-17 12:28:05,044] jepsen worker 63 - jepsen.util 333 :invoke :cas [21 [24 4]] INFO [2019-12-17 12:28:05,069] jepsen worker 74 - jepsen.util 524 :ok :write [24 39] INFO [2019-12-17 12:28:05,069] jepsen worker 74 - jepsen.util 524 :invoke :cas [24 [24 4]] INFO [2019-12-17 12:28:05,070] jepsen worker 69 - jepsen.util 699 :ok :read [23 29] INFO [2019-12-17 12:28:05,070] jepsen worker 69 - jepsen.util 699 :invoke :cas [23 [21 45]] INFO [2019-12-17 12:28:05,083] jepsen worker 28 - jepsen.util 298 :ok :write [9 30] INFO [2019-12-17 12:28:05,083] jepsen worker 28 - jepsen.util 298 :invoke :cas [9 [25 23]] INFO [2019-12-17 12:28:05,088] jepsen worker 75 - jepsen.util 525 :ok :read [25 2] INFO [2019-12-17 12:28:05,088] jepsen worker 75 - jepsen.util 525 :invoke :write [25 8] INFO [2019-12-17 12:28:05,091] jepsen worker 34 - jepsen.util 664 :ok :read [11 7] INFO [2019-12-17 12:28:05,091] jepsen worker 34 - jepsen.util 664 :invoke :read [11 nil] INFO [2019-12-17 12:28:05,106] jepsen worker 53 - jepsen.util 593 :info :write [17 43] :RequestCanceledException INFO [2019-12-17 12:28:05,106] jepsen worker 53 - jepsen.util 683 :invoke :write [17 37] INFO [2019-12-17 12:28:05,118] jepsen worker 28 - jepsen.util 298 :fail :cas [9 [25 23]] :ValueNotSwapFrom INFO [2019-12-17 12:28:05,119] jepsen worker 28 - jepsen.util 298 :invoke :read [9 nil] INFO [2019-12-17 12:28:05,121] jepsen worker 67 - jepsen.util 787 :info :cas [22 [12 1]] :RequestCanceledException INFO [2019-12-17 12:28:05,122] jepsen worker 67 - jepsen.util 877 :invoke :write [22 33] INFO [2019-12-17 12:28:05,125] jepsen worker 71 - jepsen.util 521 :ok :read [23 29] INFO [2019-12-17 12:28:05,125] jepsen worker 71 - jepsen.util 521 :invoke :read [23 nil] INFO [2019-12-17 12:28:05,138] jepsen worker 28 - jepsen.util 298 :ok :read [9 30] INFO [2019-12-17 12:28:05,138] jepsen worker 28 - jepsen.util 298 :invoke :write [9 13] INFO [2019-12-17 12:28:05,199] jepsen worker 28 - jepsen.util 298 :ok :write [9 13] INFO [2019-12-17 12:28:05,199] jepsen worker 28 - jepsen.util 298 :invoke :write [9 24] INFO [2019-12-17 12:28:05,205] jepsen worker 47 - jepsen.util 767 :info :write [15 20] :RequestCanceledException INFO [2019-12-17 12:28:05,205] jepsen worker 11 - jepsen.util 641 :info :write [3 27] :RequestCanceledException INFO [2019-12-17 12:28:05,207] jepsen worker 47 - jepsen.util 857 :invoke :read [15 nil] INFO [2019-12-17 12:28:05,207] jepsen worker 11 - jepsen.util 731 :invoke :write [3 1] INFO [2019-12-17 12:28:05,232] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2961,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:28:05,232] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2961,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:28:05,235] jepsen worker 43 - jepsen.util 763 :info :cas [14 [4 0]] :RequestCanceledException INFO [2019-12-17 12:28:05,236] jepsen worker 43 - jepsen.util 853 :invoke :write [14 18] INFO [2019-12-17 12:28:05,242] jepsen worker 47 - jepsen.util 857 :ok :read [15 35] INFO [2019-12-17 12:28:05,242] jepsen worker 47 - jepsen.util 857 :invoke :read [15 nil] INFO [2019-12-17 12:28:05,244] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2952,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:28:05,244] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2952,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:28:05,244] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2953,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:28:05,244] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2953,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:28:05,251] jepsen worker 25 - jepsen.util 745 :fail :cas [8 [15 32]] :ValueNotSwapFrom INFO [2019-12-17 12:28:05,251] jepsen worker 25 - jepsen.util 745 :invoke :write [8 36] INFO [2019-12-17 12:28:05,273] jepsen worker 47 - jepsen.util 857 :ok :read [15 35] INFO [2019-12-17 12:28:05,273] jepsen worker 47 - jepsen.util 857 :invoke :cas [15 [44 16]] INFO [2019-12-17 12:28:05,275] jepsen worker 3 - jepsen.util 453 :ok :read [1 37] INFO [2019-12-17 12:28:05,275] jepsen worker 3 - jepsen.util 453 :invoke :read [1 nil] INFO [2019-12-17 12:28:05,283] jepsen worker 28 - jepsen.util 298 :ok :write [9 24] INFO [2019-12-17 12:28:05,283] jepsen worker 28 - jepsen.util 298 :invoke :write [9 6] INFO [2019-12-17 12:28:05,288] jepsen worker 55 - jepsen.util 775 :info :write [18 7] :RequestCanceledException INFO [2019-12-17 12:28:05,290] jepsen worker 47 - jepsen.util 857 :fail :cas [15 [44 16]] :ValueNotSwapFrom INFO [2019-12-17 12:28:05,290] jepsen worker 47 - jepsen.util 857 :invoke :cas [15 [42 4]] INFO [2019-12-17 12:28:05,290] jepsen worker 55 - jepsen.util 865 :invoke :read [18 nil] INFO [2019-12-17 12:28:05,324] jepsen worker 47 - jepsen.util 857 :fail :cas [15 [42 4]] :ValueNotSwapFrom INFO [2019-12-17 12:28:05,324] jepsen worker 47 - jepsen.util 857 :invoke :cas [15 [18 13]] INFO [2019-12-17 12:28:05,349] jepsen worker 69 - jepsen.util 699 :fail :cas [23 [21 45]] :ValueNotSwapFrom INFO [2019-12-17 12:28:05,349] jepsen worker 69 - jepsen.util 699 :invoke :read [23 nil] INFO [2019-12-17 12:28:05,362] jepsen worker 47 - jepsen.util 857 :fail :cas [15 [18 13]] :ValueNotSwapFrom INFO [2019-12-17 12:28:05,362] jepsen worker 47 - jepsen.util 857 :invoke :cas [15 [5 27]] INFO [2019-12-17 12:28:05,368] jepsen worker 28 - jepsen.util 298 :ok :write [9 6] INFO [2019-12-17 12:28:05,369] jepsen worker 28 - jepsen.util 298 :invoke :write [9 42] INFO [2019-12-17 12:28:05,379] jepsen worker 47 - jepsen.util 857 :fail :cas [15 [5 27]] :ValueNotSwapFrom INFO [2019-12-17 12:28:05,379] jepsen worker 47 - jepsen.util 857 :invoke :write [15 6] INFO [2019-12-17 12:28:05,391] jepsen worker 71 - jepsen.util 521 :ok :read [23 29] INFO [2019-12-17 12:28:05,392] jepsen worker 71 - jepsen.util 521 :invoke :write [23 16] INFO [2019-12-17 12:28:05,406] jepsen worker 29 - jepsen.util 479 :info :write [9 2] :RequestCanceledException INFO [2019-12-17 12:28:05,408] jepsen worker 29 - jepsen.util 569 :invoke :write [9 48] INFO [2019-12-17 12:28:05,421] jepsen worker 87 - jepsen.util 447 :info :write [29 17] :RequestCanceledException INFO [2019-12-17 12:28:05,422] jepsen worker 87 - jepsen.util 537 :invoke :write [29 30] INFO [2019-12-17 12:28:05,445] jepsen worker 28 - jepsen.util 298 :ok :write [9 42] INFO [2019-12-17 12:28:05,445] jepsen worker 28 - jepsen.util 298 :invoke :cas [9 [19 10]] INFO [2019-12-17 12:28:05,451] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2985,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:28:05,451] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2985,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:28:05,451] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2986,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:28:05,451] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2986,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:28:05,459] jepsen worker 28 - jepsen.util 298 :fail :cas [9 [19 10]] :ValueNotSwapFrom INFO [2019-12-17 12:28:05,459] jepsen worker 28 - jepsen.util 298 :invoke :write [9 1] INFO [2019-12-17 12:28:05,484] jepsen worker 50 - jepsen.util 590 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:28:05,484] jepsen worker 50 - jepsen.util 590 :invoke :cas [16 [3 30]] INFO [2019-12-17 12:28:05,500] jepsen worker 33 - jepsen.util 663 :fail :cas [11 [0 43]] :ValueNotSwapFrom INFO [2019-12-17 12:28:05,500] jepsen worker 33 - jepsen.util 663 :invoke :read [11 nil] INFO [2019-12-17 12:28:05,504] jepsen worker 69 - jepsen.util 699 :ok :read [23 29] INFO [2019-12-17 12:28:05,505] jepsen worker 69 - jepsen.util 699 :invoke :write [23 43] INFO [2019-12-17 12:28:05,506] jepsen worker 46 - jepsen.util 496 :info :cas [15 [43 40]] :RequestCanceledException INFO [2019-12-17 12:28:05,507] jepsen worker 46 - jepsen.util 586 :invoke :cas [15 [49 21]] INFO [2019-12-17 12:28:05,517] jepsen worker 28 - jepsen.util 298 :ok :write [9 1] INFO [2019-12-17 12:28:05,517] jepsen worker 28 - jepsen.util 298 :invoke :cas [9 [33 3]] INFO [2019-12-17 12:28:05,521] jepsen worker 14 - jepsen.util 644 :info :write [4 34] :RequestCanceledException INFO [2019-12-17 12:28:05,521] jepsen worker 56 - jepsen.util 506 :info :write [18 31] :RequestCanceledException INFO [2019-12-17 12:28:05,523] jepsen worker 56 - jepsen.util 596 :invoke :write [18 12] INFO [2019-12-17 12:28:05,523] jepsen worker 14 - jepsen.util 734 :invoke :cas [4 [21 47]] INFO [2019-12-17 12:28:05,540] jepsen worker 55 - jepsen.util 865 :ok :read [18 47] INFO [2019-12-17 12:28:05,540] jepsen worker 46 - jepsen.util 586 :fail :cas [15 [49 21]] :ValueNotSwapFrom INFO [2019-12-17 12:28:05,540] jepsen worker 55 - jepsen.util 865 :invoke :cas [18 [18 20]] INFO [2019-12-17 12:28:05,540] jepsen worker 46 - jepsen.util 586 :invoke :write [15 23] INFO [2019-12-17 12:28:05,552] jepsen worker 5 - jepsen.util 275 :info :write [1 16] :RequestCanceledException INFO [2019-12-17 12:28:05,553] jepsen worker 5 - jepsen.util 365 :invoke :read [1 nil] INFO [2019-12-17 12:28:05,555] jepsen worker 28 - jepsen.util 298 :fail :cas [9 [33 3]] :ValueNotSwapFrom INFO [2019-12-17 12:28:05,555] jepsen worker 28 - jepsen.util 298 :invoke :cas [9 [37 48]] INFO [2019-12-17 12:28:05,576] jepsen worker 28 - jepsen.util 298 :fail :cas [9 [37 48]] :ValueNotSwapFrom INFO [2019-12-17 12:28:05,576] jepsen worker 28 - jepsen.util 298 :invoke :write [9 36] INFO [2019-12-17 12:28:05,605] jepsen worker 79 - jepsen.util 529 :info :write [26 1] :RequestCanceledException INFO [2019-12-17 12:28:05,606] jepsen worker 79 - jepsen.util 619 :invoke :cas [26 [35 19]] INFO [2019-12-17 12:28:05,611] jepsen worker 28 - jepsen.util 298 :ok :write [9 36] INFO [2019-12-17 12:28:05,611] jepsen worker 28 - jepsen.util 298 :invoke :read [9 nil] INFO [2019-12-17 12:28:05,621] jepsen worker 4 - jepsen.util 544 :info :write [1 18] :RequestCanceledException INFO [2019-12-17 12:28:05,622] jepsen worker 4 - jepsen.util 634 :invoke :read [1 nil] INFO [2019-12-17 12:28:05,624] jepsen worker 28 - jepsen.util 298 :ok :read [9 36] INFO [2019-12-17 12:28:05,624] jepsen worker 28 - jepsen.util 298 :invoke :cas [9 [7 43]] INFO [2019-12-17 12:28:05,637] jepsen worker 28 - jepsen.util 298 :fail :cas [9 [7 43]] :ValueNotSwapFrom INFO [2019-12-17 12:28:05,637] jepsen worker 28 - jepsen.util 298 :invoke :cas [9 [18 42]] INFO [2019-12-17 12:28:05,649] jepsen worker 28 - jepsen.util 298 :fail :cas [9 [18 42]] :ValueNotSwapFrom INFO [2019-12-17 12:28:05,649] jepsen worker 28 - jepsen.util 298 :invoke :write [9 10] INFO [2019-12-17 12:28:05,652] jepsen worker 24 - jepsen.util 474 :info :write [8 9] :RequestCanceledException INFO [2019-12-17 12:28:05,653] jepsen worker 24 - jepsen.util 564 :invoke :cas [8 [38 7]] INFO [2019-12-17 12:28:05,676] jepsen worker 30 - jepsen.util 570 :ok :write [10 30] INFO [2019-12-17 12:28:05,676] jepsen worker 30 - jepsen.util 570 :invoke :write [10 38] INFO [2019-12-17 12:28:05,677] jepsen worker 19 - jepsen.util 649 :ok :write [6 28] INFO [2019-12-17 12:28:05,677] jepsen worker 19 - jepsen.util 649 :invoke :write [6 25] INFO [2019-12-17 12:28:05,687] jepsen worker 2 - jepsen.util 542 :info :write [0 34] :RequestCanceledException INFO [2019-12-17 12:28:05,688] jepsen worker 2 - jepsen.util 632 :invoke :write [0 9] INFO [2019-12-17 12:28:05,718] jepsen worker 22 - jepsen.util 562 :info :write [7 19] :RequestCanceledException INFO [2019-12-17 12:28:05,720] jepsen worker 22 - jepsen.util 652 :invoke :cas [7 [44 47]] INFO [2019-12-17 12:28:05,722] jepsen worker 48 - jepsen.util 498 :ok :write [16 12] INFO [2019-12-17 12:28:05,722] jepsen worker 48 - jepsen.util 498 :invoke :write [16 17] INFO [2019-12-17 12:28:05,729] jepsen worker 76 - jepsen.util 616 :info :write [25 17] :RequestCanceledException INFO [2019-12-17 12:28:05,729] jepsen worker 1 - jepsen.util 631 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:28:05,729] jepsen worker 1 - jepsen.util 631 :invoke :cas [0 [49 45]] INFO [2019-12-17 12:28:05,730] jepsen worker 76 - jepsen.util 706 :invoke :read [25 nil] INFO [2019-12-17 12:28:05,731] jepsen worker 28 - jepsen.util 298 :ok :write [9 10] INFO [2019-12-17 12:28:05,731] jepsen worker 28 - jepsen.util 298 :invoke :read [9 nil] INFO [2019-12-17 12:28:05,732] jepsen worker 80 - jepsen.util 800 :fail :cas [26 [18 47]] :ValueNotSwapFrom INFO [2019-12-17 12:28:05,732] jepsen worker 80 - jepsen.util 800 :invoke :write [26 37] INFO [2019-12-17 12:28:05,733] jepsen worker 20 - jepsen.util 650 :ok :read [6 28] INFO [2019-12-17 12:28:05,733] jepsen worker 20 - jepsen.util 650 :invoke :cas [6 [4 0]] INFO [2019-12-17 12:28:05,749] jepsen worker 3 - jepsen.util 453 :ok :read [1 37] INFO [2019-12-17 12:28:05,749] jepsen worker 3 - jepsen.util 453 :invoke :read [1 nil] INFO [2019-12-17 12:28:05,750] jepsen worker 44 - jepsen.util 404 :info :write [14 43] :RequestCanceledException INFO [2019-12-17 12:28:05,750] jepsen worker 62 - jepsen.util 692 :info :write [20 34] :RequestCanceledException INFO [2019-12-17 12:28:05,752] jepsen worker 44 - jepsen.util 494 :invoke :cas [14 [45 32]] INFO [2019-12-17 12:28:05,752] jepsen worker 62 - jepsen.util 782 :invoke :write [20 39] INFO [2019-12-17 12:28:05,774] jepsen worker 68 - jepsen.util 428 :fail :cas [22 [26 8]] :ValueNotSwapFrom INFO [2019-12-17 12:28:05,775] jepsen worker 68 - jepsen.util 428 :invoke :write [22 10] INFO [2019-12-17 12:28:05,781] jepsen worker 74 - jepsen.util 524 :fail :cas [24 [24 4]] :ValueNotSwapFrom INFO [2019-12-17 12:28:05,782] jepsen worker 74 - jepsen.util 524 :invoke :read [24 nil] INFO [2019-12-17 12:28:05,787] jepsen worker 41 - jepsen.util 401 :ok :read [13 36] INFO [2019-12-17 12:28:05,787] jepsen worker 41 - jepsen.util 401 :invoke :cas [13 [29 15]] INFO [2019-12-17 12:28:05,800] jepsen worker 28 - jepsen.util 298 :ok :read [9 10] INFO [2019-12-17 12:28:05,800] jepsen worker 17 - jepsen.util 557 :ok :read [5 13] INFO [2019-12-17 12:28:05,800] jepsen worker 28 - jepsen.util 298 :invoke :write [9 9] INFO [2019-12-17 12:28:05,800] jepsen worker 17 - jepsen.util 557 :invoke :cas [5 [27 27]] INFO [2019-12-17 12:28:05,816] jepsen worker 85 - jepsen.util 625 :fail :cas [28 [12 31]] :ValueNotSwapFrom INFO [2019-12-17 12:28:05,816] jepsen worker 85 - jepsen.util 625 :invoke :write [28 38] INFO [2019-12-17 12:28:05,819] jepsen worker 10 - jepsen.util 550 :info :write [3 44] :RequestCanceledException INFO [2019-12-17 12:28:05,819] jepsen worker 55 - jepsen.util 865 :fail :cas [18 [18 20]] :ValueNotSwapFrom INFO [2019-12-17 12:28:05,820] jepsen worker 55 - jepsen.util 865 :invoke :read [18 nil] INFO [2019-12-17 12:28:05,821] jepsen worker 10 - jepsen.util 640 :invoke :read [3 nil] INFO [2019-12-17 12:28:05,847] jepsen worker 5 - jepsen.util 365 :ok :read [1 37] INFO [2019-12-17 12:28:05,847] jepsen worker 5 - jepsen.util 365 :invoke :write [1 46] INFO [2019-12-17 12:28:05,881] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3038,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:28:05,881] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3038,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:28:05,881] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3042,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:28:05,881] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3042,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:28:05,908] jepsen worker 10 - jepsen.util 640 :ok :read [3 23] INFO [2019-12-17 12:28:05,908] jepsen worker 10 - jepsen.util 640 :invoke :write [3 35] INFO [2019-12-17 12:28:05,910] jepsen worker 28 - jepsen.util 298 :ok :write [9 9] INFO [2019-12-17 12:28:05,910] jepsen worker 28 - jepsen.util 298 :invoke :cas [9 [30 10]] INFO [2019-12-17 12:28:05,936] jepsen worker 33 - jepsen.util 663 :ok :read [11 7] INFO [2019-12-17 12:28:05,936] jepsen worker 33 - jepsen.util 663 :invoke :write [11 33] INFO [2019-12-17 12:28:05,950] jepsen worker 65 - jepsen.util 515 :ok :read [21 35] INFO [2019-12-17 12:28:05,951] jepsen worker 65 - jepsen.util 515 :invoke :read [21 nil] INFO [2019-12-17 12:28:05,952] jepsen worker 84 - jepsen.util 354 :info :write [28 22] :RequestCanceledException INFO [2019-12-17 12:28:05,953] jepsen worker 84 - jepsen.util 444 :invoke :read [28 nil] INFO [2019-12-17 12:28:05,959] jepsen worker 28 - jepsen.util 298 :fail :cas [9 [30 10]] :ValueNotSwapFrom INFO [2019-12-17 12:28:05,960] jepsen worker 28 - jepsen.util 298 :invoke :read [9 nil] INFO [2019-12-17 12:28:05,984] jepsen worker 28 - jepsen.util 298 :ok :read [9 9] INFO [2019-12-17 12:28:05,984] jepsen worker 28 - jepsen.util 298 :invoke :read [9 nil] INFO [2019-12-17 12:28:05,985] jepsen worker 21 - jepsen.util 651 :info :write [7 4] :RequestCanceledException INFO [2019-12-17 12:28:05,985] jepsen worker 7 - jepsen.util 727 :info :write [2 30] :RequestCanceledException INFO [2019-12-17 12:28:05,986] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.0703125}, :ns_1@172.28.128.126 {:progress 0.06474820143884896}, :ns_1@172.28.128.127 {:progress 0.1556420233463035}, :ns_1@172.28.128.128 {:progress 0.3924528301886793}} INFO [2019-12-17 12:28:05,986] jepsen worker 7 - jepsen.util 817 :invoke :read [2 nil] INFO [2019-12-17 12:28:05,986] jepsen worker 21 - jepsen.util 741 :invoke :write [7 28] INFO [2019-12-17 12:28:05,986] jepsen nemesis - couchbase.util Rebalance check count: 17 INFO [2019-12-17 12:28:05,986] jepsen nemesis - couchbase.util Rebalance stuck count: 1 INFO [2019-12-17 12:28:05,992] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3055,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:28:05,992] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3055,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:28:05,992] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3056,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:28:05,992] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3056,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:28:06,003] jepsen worker 60 - jepsen.util 690 :ok :read [20 7] INFO [2019-12-17 12:28:06,003] jepsen worker 60 - jepsen.util 690 :invoke :write [20 15] INFO [2019-12-17 12:28:06,005] jepsen worker 15 - jepsen.util 645 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:28:06,006] jepsen worker 15 - jepsen.util 645 :invoke :read [5 nil] INFO [2019-12-17 12:28:06,010] jepsen worker 28 - jepsen.util 298 :ok :read [9 9] INFO [2019-12-17 12:28:06,010] jepsen worker 28 - jepsen.util 298 :invoke :read [9 nil] INFO [2019-12-17 12:28:06,014] jepsen worker 2 - jepsen.util 632 :ok :write [0 9] INFO [2019-12-17 12:28:06,014] jepsen worker 2 - jepsen.util 632 :invoke :write [0 37] INFO [2019-12-17 12:28:06,019] jepsen worker 82 - jepsen.util 712 :info :cas [27 [20 13]] :RequestCanceledException INFO [2019-12-17 12:28:06,020] jepsen worker 82 - jepsen.util 802 :invoke :cas [27 [3 15]] INFO [2019-12-17 12:28:06,026] jepsen worker 34 - jepsen.util 664 :ok :read [11 7] INFO [2019-12-17 12:28:06,026] jepsen worker 34 - jepsen.util 664 :invoke :write [11 29] INFO [2019-12-17 12:28:06,028] jepsen worker 11 - jepsen.util 731 :ok :write [3 1] INFO [2019-12-17 12:28:06,028] jepsen worker 11 - jepsen.util 731 :invoke :cas [3 [38 16]] INFO [2019-12-17 12:28:06,033] jepsen worker 36 - jepsen.util 666 :info :write [12 39] :RequestCanceledException INFO [2019-12-17 12:28:06,034] jepsen worker 36 - jepsen.util 756 :invoke :read [12 nil] INFO [2019-12-17 12:28:06,046] jepsen worker 28 - jepsen.util 298 :ok :read [9 9] INFO [2019-12-17 12:28:06,046] jepsen worker 28 - jepsen.util 298 :invoke :cas [9 [14 39]] INFO [2019-12-17 12:28:06,052] jepsen worker 1 - jepsen.util 631 :fail :cas [0 [49 45]] :ValueNotSwapFrom INFO [2019-12-17 12:28:06,052] jepsen worker 16 - jepsen.util 646 :info :write [5 17] :RequestCanceledException INFO [2019-12-17 12:28:06,053] jepsen worker 1 - jepsen.util 631 :invoke :cas [0 [33 47]] INFO [2019-12-17 12:28:06,054] jepsen worker 16 - jepsen.util 736 :invoke :read [5 nil] INFO [2019-12-17 12:28:06,057] jepsen worker 44 - jepsen.util 494 :fail :cas [14 [45 32]] :ValueNotSwapFrom INFO [2019-12-17 12:28:06,058] jepsen worker 44 - jepsen.util 494 :invoke :cas [14 [16 32]] INFO [2019-12-17 12:28:06,062] jepsen worker 30 - jepsen.util 570 :ok :write [10 38] INFO [2019-12-17 12:28:06,063] jepsen worker 30 - jepsen.util 570 :invoke :read [10 nil] INFO [2019-12-17 12:28:06,071] jepsen worker 28 - jepsen.util 298 :fail :cas [9 [14 39]] :ValueNotSwapFrom INFO [2019-12-17 12:28:06,072] jepsen worker 28 - jepsen.util 298 :invoke :write [9 31] INFO [2019-12-17 12:28:06,087] jepsen worker 18 - jepsen.util 648 :info :cas [6 [8 34]] :RequestCanceledException INFO [2019-12-17 12:28:06,087] jepsen worker 61 - jepsen.util 691 :info :write [20 4] :RequestCanceledException INFO [2019-12-17 12:28:06,088] jepsen worker 18 - jepsen.util 738 :invoke :cas [6 [46 24]] INFO [2019-12-17 12:28:06,089] jepsen worker 76 - jepsen.util 706 :ok :read [25 2] INFO [2019-12-17 12:28:06,089] jepsen worker 76 - jepsen.util 706 :invoke :read [25 nil] INFO [2019-12-17 12:28:06,089] jepsen worker 14 - jepsen.util 734 :fail :cas [4 [21 47]] :ValueNotSwapFrom INFO [2019-12-17 12:28:06,089] jepsen worker 61 - jepsen.util 781 :invoke :read [20 nil] INFO [2019-12-17 12:28:06,089] jepsen worker 14 - jepsen.util 734 :invoke :read [4 nil] INFO [2019-12-17 12:28:06,117] jepsen worker 66 - jepsen.util 516 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:28:06,117] jepsen worker 6 - jepsen.util 546 :info :write [2 17] :RequestCanceledException INFO [2019-12-17 12:28:06,117] jepsen worker 88 - jepsen.util 448 :info :write [29 32] :RequestCanceledException INFO [2019-12-17 12:28:06,117] jepsen worker 66 - jepsen.util 516 :invoke :read [22 nil] INFO [2019-12-17 12:28:06,118] jepsen worker 68 - jepsen.util 428 :ok :write [22 10] INFO [2019-12-17 12:28:06,118] jepsen worker 6 - jepsen.util 636 :invoke :read [2 nil] INFO [2019-12-17 12:28:06,118] jepsen worker 88 - jepsen.util 538 :invoke :write [29 14] INFO [2019-12-17 12:28:06,118] jepsen worker 68 - jepsen.util 428 :invoke :read [22 nil] INFO [2019-12-17 12:28:06,119] jepsen worker 30 - jepsen.util 570 :ok :read [10 38] INFO [2019-12-17 12:28:06,119] jepsen worker 30 - jepsen.util 570 :invoke :write [10 35] INFO [2019-12-17 12:28:06,149] jepsen worker 44 - jepsen.util 494 :fail :cas [14 [16 32]] :ValueNotSwapFrom INFO [2019-12-17 12:28:06,149] jepsen worker 44 - jepsen.util 494 :invoke :read [14 nil] INFO [2019-12-17 12:28:06,159] jepsen worker 41 - jepsen.util 401 :fail :cas [13 [29 15]] :ValueNotSwapFrom INFO [2019-12-17 12:28:06,159] jepsen worker 41 - jepsen.util 401 :invoke :read [13 nil] INFO [2019-12-17 12:28:06,169] jepsen worker 28 - jepsen.util 298 :ok :write [9 31] INFO [2019-12-17 12:28:06,169] jepsen worker 28 - jepsen.util 298 :invoke :write [9 43] INFO [2019-12-17 12:28:06,188] jepsen worker 40 - jepsen.util 760 :info :write [13 32] :RequestCanceledException INFO [2019-12-17 12:28:06,189] jepsen worker 40 - jepsen.util 850 :invoke :cas [13 [32 3]] INFO [2019-12-17 12:28:06,201] jepsen worker 76 - jepsen.util 706 :ok :read [25 2] INFO [2019-12-17 12:28:06,201] jepsen worker 76 - jepsen.util 706 :invoke :cas [25 [10 33]] INFO [2019-12-17 12:28:06,211] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3093,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:28:06,211] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3093,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:28:06,211] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3097,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:28:06,211] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3097,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:28:06,215] jepsen worker 28 - jepsen.util 298 :ok :write [9 43] INFO [2019-12-17 12:28:06,215] jepsen worker 28 - jepsen.util 298 :invoke :read [9 nil] INFO [2019-12-17 12:28:06,221] jepsen worker 13 - jepsen.util 643 :info :write [4 46] :RequestCanceledException INFO [2019-12-17 12:28:06,222] jepsen worker 13 - jepsen.util 733 :invoke :read [4 nil] INFO [2019-12-17 12:28:06,222] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3084,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:28:06,222] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3084,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:28:06,222] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3085,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:28:06,222] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3085,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:28:06,223] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3086,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:28:06,223] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3086,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:28:06,234] jepsen worker 54 - jepsen.util 684 :info :write [18 1] :RequestCanceledException INFO [2019-12-17 12:28:06,235] jepsen worker 78 - jepsen.util 618 :info :write [26 37] :RequestCanceledException INFO [2019-12-17 12:28:06,236] jepsen worker 78 - jepsen.util 708 :invoke :write [26 39] INFO [2019-12-17 12:28:06,237] jepsen worker 54 - jepsen.util 774 :invoke :cas [18 [23 19]] INFO [2019-12-17 12:28:06,243] jepsen worker 28 - jepsen.util 298 :ok :read [9 43] INFO [2019-12-17 12:28:06,243] jepsen worker 28 - jepsen.util 298 :invoke :cas [9 [25 17]] INFO [2019-12-17 12:28:06,255] jepsen worker 3 - jepsen.util 453 :ok :read [1 37] INFO [2019-12-17 12:28:06,255] jepsen worker 3 - jepsen.util 453 :invoke :cas [1 [34 42]] INFO [2019-12-17 12:28:06,260] jepsen worker 22 - jepsen.util 652 :fail :cas [7 [44 47]] :ValueNotSwapFrom INFO [2019-12-17 12:28:06,260] jepsen worker 22 - jepsen.util 652 :invoke :cas [7 [41 9]] INFO [2019-12-17 12:28:06,264] jepsen worker 44 - jepsen.util 494 :ok :read [14 27] INFO [2019-12-17 12:28:06,264] jepsen worker 44 - jepsen.util 494 :invoke :read [14 nil] INFO [2019-12-17 12:28:06,268] jepsen worker 28 - jepsen.util 298 :fail :cas [9 [25 17]] :ValueNotSwapFrom INFO [2019-12-17 12:28:06,268] jepsen worker 28 - jepsen.util 298 :invoke :write [9 27] INFO [2019-12-17 12:28:06,276] jepsen worker 55 - jepsen.util 865 :ok :read [18 47] INFO [2019-12-17 12:28:06,276] jepsen worker 55 - jepsen.util 865 :invoke :cas [18 [48 22]] INFO [2019-12-17 12:28:06,296] jepsen worker 4 - jepsen.util 634 :ok :read [1 37] INFO [2019-12-17 12:28:06,296] jepsen worker 4 - jepsen.util 634 :invoke :cas [1 [1 24]] INFO [2019-12-17 12:28:06,327] jepsen worker 28 - jepsen.util 298 :ok :write [9 27] INFO [2019-12-17 12:28:06,328] jepsen worker 28 - jepsen.util 298 :invoke :write [9 7] INFO [2019-12-17 12:28:06,334] jepsen worker 26 - jepsen.util 746 :info :write [8 47] :RequestCanceledException INFO [2019-12-17 12:28:06,336] jepsen worker 26 - jepsen.util 836 :invoke :cas [8 [46 32]] INFO [2019-12-17 12:28:06,359] jepsen worker 4 - jepsen.util 634 :fail :cas [1 [1 24]] :ValueNotSwapFrom INFO [2019-12-17 12:28:06,359] jepsen worker 4 - jepsen.util 634 :invoke :cas [1 [12 44]] INFO [2019-12-17 12:28:06,369] jepsen worker 7 - jepsen.util 817 :ok :read [2 15] INFO [2019-12-17 12:28:06,369] jepsen worker 7 - jepsen.util 817 :invoke :cas [2 [31 32]] INFO [2019-12-17 12:28:06,385] jepsen worker 28 - jepsen.util 298 :ok :write [9 7] INFO [2019-12-17 12:28:06,385] jepsen worker 28 - jepsen.util 298 :invoke :cas [9 [49 28]] INFO [2019-12-17 12:28:06,391] jepsen worker 36 - jepsen.util 756 :ok :read [12 12] INFO [2019-12-17 12:28:06,391] jepsen worker 36 - jepsen.util 756 :invoke :write [12 37] INFO [2019-12-17 12:28:06,405] jepsen worker 81 - jepsen.util 621 :info :cas [27 [11 7]] :RequestCanceledException INFO [2019-12-17 12:28:06,407] jepsen worker 81 - jepsen.util 711 :invoke :write [27 11] INFO [2019-12-17 12:28:06,418] jepsen worker 28 - jepsen.util 298 :fail :cas [9 [49 28]] :ValueNotSwapFrom INFO [2019-12-17 12:28:06,419] jepsen worker 28 - jepsen.util 298 :invoke :read [9 nil] INFO [2019-12-17 12:28:06,419] jepsen worker 44 - jepsen.util 494 :ok :read [14 27] INFO [2019-12-17 12:28:06,419] jepsen worker 44 - jepsen.util 494 :invoke :read [14 nil] INFO [2019-12-17 12:28:06,429] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3117,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:28:06,429] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3117,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:28:06,429] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3120,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:28:06,429] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3120,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:28:06,429] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3121,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:28:06,429] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3121,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:28:06,435] jepsen worker 70 - jepsen.util 790 :info :write [23 24] :RequestCanceledException INFO [2019-12-17 12:28:06,438] jepsen worker 70 - jepsen.util 880 :invoke :cas [23 [33 23]] INFO [2019-12-17 12:28:06,449] jepsen worker 86 - jepsen.util 446 :info :write [28 37] :RequestCanceledException INFO [2019-12-17 12:28:06,451] jepsen worker 86 - jepsen.util 536 :invoke :read [28 nil] INFO [2019-12-17 12:28:06,453] jepsen worker 28 - jepsen.util 298 :ok :read [9 7] INFO [2019-12-17 12:28:06,454] jepsen worker 28 - jepsen.util 298 :invoke :write [9 6] INFO [2019-12-17 12:28:06,454] jepsen worker 61 - jepsen.util 781 :ok :read [20 7] INFO [2019-12-17 12:28:06,454] jepsen worker 61 - jepsen.util 781 :invoke :cas [20 [35 46]] INFO [2019-12-17 12:28:06,455] jepsen worker 79 - jepsen.util 619 :fail :cas [26 [35 19]] :ValueNotSwapFrom INFO [2019-12-17 12:28:06,455] jepsen worker 79 - jepsen.util 619 :invoke :read [26 nil] INFO [2019-12-17 12:28:06,484] jepsen worker 14 - jepsen.util 734 :ok :read [4 4] INFO [2019-12-17 12:28:06,485] jepsen worker 14 - jepsen.util 734 :invoke :cas [4 [40 39]] INFO [2019-12-17 12:28:06,487] jepsen worker 37 - jepsen.util 577 :info :write [12 6] :RequestCanceledException INFO [2019-12-17 12:28:06,487] jepsen worker 45 - jepsen.util 855 :info :write [15 11] :RequestCanceledException INFO [2019-12-17 12:28:06,489] jepsen worker 45 - jepsen.util 945 :invoke :write [15 14] INFO [2019-12-17 12:28:06,489] jepsen worker 37 - jepsen.util 667 :invoke :write [12 12] INFO [2019-12-17 12:28:06,507] jepsen worker 28 - jepsen.util 298 :ok :write [9 6] INFO [2019-12-17 12:28:06,507] jepsen worker 28 - jepsen.util 298 :invoke :read [9 nil] INFO [2019-12-17 12:28:06,521] jepsen worker 89 - jepsen.util 719 :info :write [29 18] :RequestCanceledException INFO [2019-12-17 12:28:06,521] jepsen worker 83 - jepsen.util 713 :info :write [27 28] :RequestCanceledException INFO [2019-12-17 12:28:06,522] jepsen worker 89 - jepsen.util 809 :invoke :write [29 10] INFO [2019-12-17 12:28:06,522] jepsen worker 83 - jepsen.util 803 :invoke :write [27 1] INFO [2019-12-17 12:28:06,531] jepsen worker 28 - jepsen.util 298 :ok :read [9 6] INFO [2019-12-17 12:28:06,532] jepsen worker 28 - jepsen.util 298 :invoke :write [9 36] INFO [2019-12-17 12:28:06,533] jepsen worker 44 - jepsen.util 494 :ok :read [14 27] INFO [2019-12-17 12:28:06,533] jepsen worker 44 - jepsen.util 494 :invoke :read [14 nil] INFO [2019-12-17 12:28:06,548] jepsen worker 22 - jepsen.util 652 :fail :cas [7 [41 9]] :ValueNotSwapFrom INFO [2019-12-17 12:28:06,549] jepsen worker 22 - jepsen.util 652 :invoke :read [7 nil] INFO [2019-12-17 12:28:06,552] jepsen worker 64 - jepsen.util 424 :fail :read [21 nil] :RequestTimeoutException INFO [2019-12-17 12:28:06,552] jepsen worker 64 - jepsen.util 424 :invoke :cas [21 [49 39]] INFO [2019-12-17 12:28:06,578] jepsen worker 54 - jepsen.util 774 :fail :cas [18 [23 19]] :ValueNotSwapFrom INFO [2019-12-17 12:28:06,578] jepsen worker 54 - jepsen.util 774 :invoke :cas [18 [35 9]] INFO [2019-12-17 12:28:06,584] jepsen worker 28 - jepsen.util 298 :ok :write [9 36] INFO [2019-12-17 12:28:06,584] jepsen worker 28 - jepsen.util 298 :invoke :write [9 32] INFO [2019-12-17 12:28:06,600] jepsen worker 51 - jepsen.util 501 :info :write [17 14] :RequestCanceledException INFO [2019-12-17 12:28:06,608] jepsen worker 64 - jepsen.util 424 :fail :cas [21 [49 39]] :ValueNotSwapFrom INFO [2019-12-17 12:28:06,614] jepsen worker 9 - jepsen.util 549 :info :cas [3 [12 10]] :RequestCanceledException INFO [2019-12-17 12:28:06,614] jepsen worker 77 - jepsen.util 527 :info :write [25 23] :RequestCanceledException INFO [2019-12-17 12:28:06,614] jepsen worker 73 - jepsen.util 703 :info :write [24 8] :RequestCanceledException INFO [2019-12-17 12:28:06,614] jepsen worker 23 - jepsen.util 473 :info :write [7 21] :RequestCanceledException INFO [2019-12-17 12:28:06,614] jepsen worker 27 - jepsen.util 387 :info :cas [9 [32 5]] :RequestCanceledException INFO [2019-12-17 12:28:06,614] jepsen worker 39 - jepsen.util 669 :info :write [13 5] :RequestCanceledException INFO [2019-12-17 12:28:06,617] jepsen worker 44 - jepsen.util 494 :ok :read [14 27] INFO [2019-12-17 12:28:06,618] jepsen worker 55 - jepsen.util 865 :fail :cas [18 [48 22]] :ValueNotSwapFrom INFO [2019-12-17 12:28:06,619] jepsen worker 79 - jepsen.util 619 :ok :read [26 47] INFO [2019-12-17 12:28:06,620] jepsen worker 7 - jepsen.util 817 :fail :cas [2 [31 32]] :ValueNotSwapFrom INFO [2019-12-17 12:28:06,636] jepsen worker 3 - jepsen.util 453 :fail :cas [1 [34 42]] :ValueNotSwapFrom INFO [2019-12-17 12:28:06,643] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3142,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:28:06,653] jepsen worker 28 - jepsen.util 298 :ok :write [9 32] INFO [2019-12-17 12:28:06,657] jepsen worker 64 - jepsen.util 424 :invoke :write [21 31] INFO [2019-12-17 12:28:06,658] jepsen worker 44 - jepsen.util 494 :invoke :read [14 nil] INFO [2019-12-17 12:28:06,658] jepsen worker 55 - jepsen.util 865 :invoke :cas [18 [2 26]] INFO [2019-12-17 12:28:06,658] jepsen worker 79 - jepsen.util 619 :invoke :read [26 nil] INFO [2019-12-17 12:28:06,658] jepsen worker 7 - jepsen.util 817 :invoke :write [2 15] INFO [2019-12-17 12:28:06,658] jepsen worker 3 - jepsen.util 453 :invoke :read [1 nil] INFO [2019-12-17 12:28:06,658] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3142,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:28:06,658] jepsen worker 28 - jepsen.util 298 :invoke :cas [9 [43 36]] INFO [2019-12-17 12:28:06,658] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3145,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:28:06,658] jepsen worker 9 - jepsen.util 639 :invoke :cas [3 [26 10]] INFO [2019-12-17 12:28:06,659] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3145,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:28:06,659] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3146,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:28:06,659] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3146,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:28:06,659] jepsen worker 39 - jepsen.util 759 :invoke :read [13 nil] INFO [2019-12-17 12:28:06,659] jepsen worker 51 - jepsen.util 591 :invoke :cas [17 [12 32]] INFO [2019-12-17 12:28:06,659] jepsen worker 77 - jepsen.util 617 :invoke :read [25 nil] INFO [2019-12-17 12:28:06,659] jepsen worker 27 - jepsen.util 477 :invoke :read [9 nil] INFO [2019-12-17 12:28:06,659] jepsen worker 73 - jepsen.util 793 :invoke :read [24 nil] INFO [2019-12-17 12:28:06,659] jepsen worker 23 - jepsen.util 563 :invoke :read [7 nil] INFO [2019-12-17 12:28:06,684] jepsen worker 27 - jepsen.util 477 :ok :read [9 32] INFO [2019-12-17 12:28:06,684] jepsen worker 27 - jepsen.util 477 :invoke :write [9 27] INFO [2019-12-17 12:28:06,701] jepsen worker 14 - jepsen.util 734 :fail :cas [4 [40 39]] :ValueNotSwapFrom INFO [2019-12-17 12:28:06,701] jepsen worker 14 - jepsen.util 734 :invoke :read [4 nil] INFO [2019-12-17 12:28:06,703] jepsen worker 12 - jepsen.util 552 :info :write [4 28] :RequestCanceledException INFO [2019-12-17 12:28:06,704] jepsen worker 12 - jepsen.util 642 :invoke :read [4 nil] INFO [2019-12-17 12:28:06,706] jepsen worker 73 - jepsen.util 793 :ok :read [24 39] INFO [2019-12-17 12:28:06,706] jepsen worker 73 - jepsen.util 793 :invoke :read [24 nil] INFO [2019-12-17 12:28:06,715] jepsen worker 24 - jepsen.util 564 :fail :cas [8 [38 7]] :ValueNotSwapFrom INFO [2019-12-17 12:28:06,716] jepsen worker 24 - jepsen.util 564 :invoke :write [8 19] INFO [2019-12-17 12:28:06,717] jepsen worker 44 - jepsen.util 494 :ok :read [14 27] INFO [2019-12-17 12:28:06,718] jepsen worker 44 - jepsen.util 494 :invoke :cas [14 [37 0]] INFO [2019-12-17 12:28:06,718] jepsen worker 58 - jepsen.util 508 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:28:06,718] jepsen worker 42 - jepsen.util 852 :info :cas [14 [27 1]] :RequestCanceledException INFO [2019-12-17 12:28:06,718] jepsen worker 58 - jepsen.util 508 :invoke :cas [19 [10 9]] INFO [2019-12-17 12:28:06,721] jepsen worker 42 - jepsen.util 942 :invoke :read [14 nil] INFO [2019-12-17 12:28:06,733] jepsen worker 28 - jepsen.util 298 :fail :cas [9 [43 36]] :ValueNotSwapFrom INFO [2019-12-17 12:28:06,733] jepsen worker 28 - jepsen.util 298 :invoke :cas [9 [47 30]] INFO [2019-12-17 12:28:06,741] jepsen worker 73 - jepsen.util 793 :ok :read [24 39] INFO [2019-12-17 12:28:06,741] jepsen worker 73 - jepsen.util 793 :invoke :cas [24 [13 24]] INFO [2019-12-17 12:28:06,745] jepsen worker 44 - jepsen.util 494 :fail :cas [14 [37 0]] :ValueNotSwapFrom INFO [2019-12-17 12:28:06,746] jepsen worker 44 - jepsen.util 494 :invoke :cas [14 [5 46]] INFO [2019-12-17 12:28:06,752] jepsen worker 35 - jepsen.util 485 :info :write [11 10] :RequestCanceledException INFO [2019-12-17 12:28:06,752] jepsen worker 57 - jepsen.util 687 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:28:06,753] jepsen worker 57 - jepsen.util 687 :invoke :cas [19 [23 32]] INFO [2019-12-17 12:28:06,753] jepsen worker 13 - jepsen.util 733 :ok :read [4 4] INFO [2019-12-17 12:28:06,754] jepsen worker 13 - jepsen.util 733 :invoke :cas [4 [3 7]] INFO [2019-12-17 12:28:06,754] jepsen worker 35 - jepsen.util 575 :invoke :write [11 49] INFO [2019-12-17 12:28:06,780] jepsen worker 73 - jepsen.util 793 :fail :cas [24 [13 24]] :ValueNotSwapFrom INFO [2019-12-17 12:28:06,780] jepsen worker 73 - jepsen.util 793 :invoke :write [24 24] INFO [2019-12-17 12:28:06,784] jepsen worker 44 - jepsen.util 494 :fail :cas [14 [5 46]] :ValueNotSwapFrom INFO [2019-12-17 12:28:06,784] jepsen worker 32 - jepsen.util 572 :info :write [10 41] :RequestCanceledException INFO [2019-12-17 12:28:06,785] jepsen worker 44 - jepsen.util 494 :invoke :cas [14 [25 33]] INFO [2019-12-17 12:28:06,786] jepsen worker 32 - jepsen.util 662 :invoke :cas [10 [36 28]] INFO [2019-12-17 12:28:06,801] jepsen worker 27 - jepsen.util 477 :ok :write [9 27] INFO [2019-12-17 12:28:06,802] jepsen worker 27 - jepsen.util 477 :invoke :write [9 37] INFO [2019-12-17 12:28:06,821] jepsen worker 0 - jepsen.util 720 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:28:06,821] jepsen worker 0 - jepsen.util 720 :invoke :cas [0 [28 23]] INFO [2019-12-17 12:28:06,821] jepsen worker 59 - jepsen.util 599 :info :cas [19 [35 17]] :RequestCanceledException INFO [2019-12-17 12:28:06,823] jepsen worker 59 - jepsen.util 689 :invoke :write [19 1] INFO [2019-12-17 12:28:06,832] jepsen worker 51 - jepsen.util 591 :fail :cas [17 [12 32]] :ValueNotSwapFrom INFO [2019-12-17 12:28:06,832] jepsen worker 51 - jepsen.util 591 :invoke :cas [17 [21 20]] INFO [2019-12-17 12:28:06,845] jepsen worker 32 - jepsen.util 662 :fail :cas [10 [36 28]] :ValueNotSwapFrom INFO [2019-12-17 12:28:06,845] jepsen worker 32 - jepsen.util 662 :invoke :write [10 17] INFO [2019-12-17 12:28:06,850] jepsen worker 64 - jepsen.util 424 :ok :write [21 31] INFO [2019-12-17 12:28:06,851] jepsen worker 64 - jepsen.util 424 :invoke :cas [21 [0 37]] INFO [2019-12-17 12:28:06,861] jepsen worker 44 - jepsen.util 494 :fail :cas [14 [25 33]] :ValueNotSwapFrom INFO [2019-12-17 12:28:06,861] jepsen worker 44 - jepsen.util 494 :invoke :read [14 nil] INFO [2019-12-17 12:28:06,862] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3185,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:28:06,862] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3185,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:28:06,862] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3186,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:28:06,862] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3186,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:28:06,883] jepsen worker 28 - jepsen.util 298 :fail :cas [9 [47 30]] :ValueNotSwapFrom INFO [2019-12-17 12:28:06,884] jepsen worker 28 - jepsen.util 298 :invoke :cas [9 [26 47]] INFO [2019-12-17 12:28:06,887] jepsen worker 52 - jepsen.util 592 :info :write [17 20] :RequestCanceledException INFO [2019-12-17 12:28:06,887] jepsen worker 31 - jepsen.util 751 :info :cas [10 [15 41]] :RequestCanceledException INFO [2019-12-17 12:28:06,888] jepsen worker 52 - jepsen.util 682 :invoke :write [17 10] INFO [2019-12-17 12:28:06,889] jepsen worker 31 - jepsen.util 841 :invoke :cas [10 [13 38]] INFO [2019-12-17 12:28:06,896] jepsen worker 64 - jepsen.util 424 :fail :cas [21 [0 37]] :ValueNotSwapFrom INFO [2019-12-17 12:28:06,896] jepsen worker 64 - jepsen.util 424 :invoke :cas [21 [2 14]] INFO [2019-12-17 12:28:06,908] jepsen worker 13 - jepsen.util 733 :fail :cas [4 [3 7]] :ValueNotSwapFrom INFO [2019-12-17 12:28:06,908] jepsen worker 13 - jepsen.util 733 :invoke :cas [4 [34 48]] INFO [2019-12-17 12:28:06,912] jepsen worker 54 - jepsen.util 774 :fail :cas [18 [35 9]] :ValueNotSwapFrom INFO [2019-12-17 12:28:06,912] jepsen worker 54 - jepsen.util 774 :invoke :read [18 nil] INFO [2019-12-17 12:28:06,922] jepsen worker 44 - jepsen.util 494 :ok :read [14 27] INFO [2019-12-17 12:28:06,922] jepsen worker 44 - jepsen.util 494 :invoke :cas [14 [1 45]] INFO [2019-12-17 12:28:06,923] jepsen worker 48 - jepsen.util 498 :ok :write [16 17] INFO [2019-12-17 12:28:06,923] jepsen worker 48 - jepsen.util 498 :invoke :cas [16 [46 8]] INFO [2019-12-17 12:28:06,935] jepsen worker 12 - jepsen.util 642 :ok :read [4 4] INFO [2019-12-17 12:28:06,936] jepsen worker 12 - jepsen.util 642 :invoke :write [4 12] INFO [2019-12-17 12:28:06,936] jepsen worker 19 - jepsen.util 649 :ok :write [6 25] INFO [2019-12-17 12:28:06,937] jepsen worker 19 - jepsen.util 649 :invoke :write [6 23] INFO [2019-12-17 12:28:06,941] jepsen worker 30 - jepsen.util 570 :ok :write [10 35] INFO [2019-12-17 12:28:06,941] jepsen worker 30 - jepsen.util 570 :invoke :cas [10 [22 31]] INFO [2019-12-17 12:28:06,964] jepsen worker 17 - jepsen.util 557 :fail :cas [5 [27 27]] :ValueNotSwapFrom INFO [2019-12-17 12:28:06,964] jepsen worker 17 - jepsen.util 557 :invoke :cas [5 [12 23]] INFO [2019-12-17 12:28:06,967] jepsen worker 28 - jepsen.util 298 :fail :cas [9 [26 47]] :ValueNotSwapFrom INFO [2019-12-17 12:28:06,967] jepsen worker 28 - jepsen.util 298 :invoke :read [9 nil] INFO [2019-12-17 12:28:06,979] jepsen worker 42 - jepsen.util 942 :ok :read [14 27] INFO [2019-12-17 12:28:06,979] jepsen worker 42 - jepsen.util 942 :invoke :cas [14 [8 32]] INFO [2019-12-17 12:28:06,988] jepsen worker 51 - jepsen.util 591 :fail :cas [17 [21 20]] :ValueNotSwapFrom INFO [2019-12-17 12:28:06,989] jepsen worker 51 - jepsen.util 591 :invoke :cas [17 [35 5]] INFO [2019-12-17 12:28:07,013] jepsen worker 6 - jepsen.util 636 :ok :read [2 15] INFO [2019-12-17 12:28:07,013] jepsen worker 6 - jepsen.util 636 :invoke :cas [2 [9 9]] INFO [2019-12-17 12:28:07,019] jepsen worker 18 - jepsen.util 738 :fail :cas [6 [46 24]] :ValueNotSwapFrom INFO [2019-12-17 12:28:07,019] jepsen worker 18 - jepsen.util 738 :invoke :read [6 nil] INFO [2019-12-17 12:28:07,021] jepsen worker 49 - jepsen.util 409 :info :write [16 12] :RequestCanceledException INFO [2019-12-17 12:28:07,021] jepsen worker 72 - jepsen.util 612 :info :write [24 35] :RequestCanceledException INFO [2019-12-17 12:28:07,022] jepsen worker 72 - jepsen.util 702 :invoke :write [24 39] INFO [2019-12-17 12:28:07,023] jepsen worker 49 - jepsen.util 499 :invoke :read [16 nil] INFO [2019-12-17 12:28:07,035] jepsen worker 8 - jepsen.util 638 :info :cas [2 [15 43]] :RequestCanceledException INFO [2019-12-17 12:28:07,040] jepsen worker 8 - jepsen.util 728 :invoke :cas [2 [21 43]] INFO [2019-12-17 12:28:07,050] jepsen worker 27 - jepsen.util 477 :ok :write [9 37] INFO [2019-12-17 12:28:07,050] jepsen worker 31 - jepsen.util 841 :fail :cas [10 [13 38]] :ValueNotSwapFrom INFO [2019-12-17 12:28:07,051] jepsen worker 27 - jepsen.util 477 :invoke :read [9 nil] INFO [2019-12-17 12:28:07,051] jepsen worker 31 - jepsen.util 841 :invoke :write [10 45] INFO [2019-12-17 12:28:07,051] jepsen worker 38 - jepsen.util 488 :info :write [12 13] :RequestCanceledException INFO [2019-12-17 12:28:07,051] jepsen worker 63 - jepsen.util 333 :info :cas [21 [24 4]] :RequestCanceledException INFO [2019-12-17 12:28:07,053] jepsen worker 63 - jepsen.util 423 :invoke :cas [21 [47 33]] INFO [2019-12-17 12:28:07,053] jepsen worker 38 - jepsen.util 578 :invoke :cas [12 [44 19]] INFO [2019-12-17 12:28:07,056] jepsen worker 64 - jepsen.util 424 :fail :cas [21 [2 14]] :ValueNotSwapFrom INFO [2019-12-17 12:28:07,056] jepsen worker 64 - jepsen.util 424 :invoke :cas [21 [28 30]] INFO [2019-12-17 12:28:07,063] jepsen worker 44 - jepsen.util 494 :fail :cas [14 [1 45]] :ValueNotSwapFrom INFO [2019-12-17 12:28:07,063] jepsen worker 44 - jepsen.util 494 :invoke :read [14 nil] INFO [2019-12-17 12:28:07,068] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.08203125}, :ns_1@172.28.128.126 {:progress 0.07553956834532372}, :ns_1@172.28.128.127 {:progress 0.1556420233463035}, :ns_1@172.28.128.128 {:progress 0.6264150943396227}} INFO [2019-12-17 12:28:07,069] jepsen nemesis - couchbase.util Rebalance check count: 18 INFO [2019-12-17 12:28:07,069] jepsen nemesis - couchbase.util Rebalance stuck count: 1 INFO [2019-12-17 12:28:07,075] jepsen worker 28 - jepsen.util 298 :ok :read [9 37] INFO [2019-12-17 12:28:07,075] jepsen worker 28 - jepsen.util 298 :invoke :write [9 36] INFO [2019-12-17 12:28:07,079] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3206,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:28:07,080] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3206,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:28:07,080] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3209,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:28:07,080] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3209,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:28:07,092] jepsen worker 9 - jepsen.util 639 :fail :cas [3 [26 10]] :ValueNotSwapFrom INFO [2019-12-17 12:28:07,092] jepsen worker 9 - jepsen.util 639 :invoke :read [3 nil] INFO [2019-12-17 12:28:07,103] jepsen worker 75 - jepsen.util 525 :info :write [25 8] :RequestCanceledException INFO [2019-12-17 12:28:07,104] jepsen worker 75 - jepsen.util 615 :invoke :read [25 nil] INFO [2019-12-17 12:28:07,105] jepsen worker 64 - jepsen.util 424 :fail :cas [21 [28 30]] :ValueNotSwapFrom INFO [2019-12-17 12:28:07,105] jepsen worker 64 - jepsen.util 424 :invoke :write [21 26] INFO [2019-12-17 12:28:07,107] jepsen worker 13 - jepsen.util 733 :fail :cas [4 [34 48]] :ValueNotSwapFrom INFO [2019-12-17 12:28:07,107] jepsen worker 13 - jepsen.util 733 :invoke :write [4 17] INFO [2019-12-17 12:28:07,115] jepsen worker 27 - jepsen.util 477 :ok :read [9 37] INFO [2019-12-17 12:28:07,116] jepsen worker 27 - jepsen.util 477 :invoke :cas [9 [17 16]] INFO [2019-12-17 12:28:07,134] jepsen worker 30 - jepsen.util 570 :fail :cas [10 [22 31]] :ValueNotSwapFrom INFO [2019-12-17 12:28:07,134] jepsen worker 30 - jepsen.util 570 :invoke :write [10 39] INFO [2019-12-17 12:28:07,168] jepsen worker 44 - jepsen.util 494 :ok :read [14 27] INFO [2019-12-17 12:28:07,168] jepsen worker 44 - jepsen.util 494 :invoke :read [14 nil] INFO [2019-12-17 12:28:07,171] jepsen worker 75 - jepsen.util 615 :ok :read [25 2] INFO [2019-12-17 12:28:07,171] jepsen worker 75 - jepsen.util 615 :invoke :read [25 nil] INFO [2019-12-17 12:28:07,174] jepsen worker 51 - jepsen.util 591 :fail :cas [17 [35 5]] :ValueNotSwapFrom INFO [2019-12-17 12:28:07,174] jepsen worker 51 - jepsen.util 591 :invoke :read [17 nil] INFO [2019-12-17 12:28:07,188] jepsen worker 53 - jepsen.util 683 :info :write [17 37] :RequestCanceledException INFO [2019-12-17 12:28:07,189] jepsen worker 53 - jepsen.util 773 :invoke :write [17 7] INFO [2019-12-17 12:28:07,197] jepsen worker 9 - jepsen.util 639 :ok :read [3 1] INFO [2019-12-17 12:28:07,197] jepsen worker 9 - jepsen.util 639 :invoke :cas [3 [49 17]] INFO [2019-12-17 12:28:07,198] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3233,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:28:07,198] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3233,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:28:07,199] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3234,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:28:07,199] jepsen worker 67 - jepsen.util 877 :info :write [22 33] :RequestCanceledException INFO [2019-12-17 12:28:07,199] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3234,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:28:07,199] jepsen worker 27 - jepsen.util 477 :fail :cas [9 [17 16]] :ValueNotSwapFrom INFO [2019-12-17 12:28:07,199] jepsen worker 27 - jepsen.util 477 :invoke :cas [9 [42 5]] INFO [2019-12-17 12:28:07,201] jepsen worker 67 - jepsen.util 967 :invoke :write [22 19] INFO [2019-12-17 12:28:07,205] jepsen worker 85 - jepsen.util 625 :ok :write [28 38] INFO [2019-12-17 12:28:07,206] jepsen worker 85 - jepsen.util 625 :invoke :read [28 nil] INFO [2019-12-17 12:28:07,236] jepsen worker 68 - jepsen.util 428 :ok :read [22 10] INFO [2019-12-17 12:28:07,236] jepsen worker 68 - jepsen.util 428 :invoke :write [22 0] INFO [2019-12-17 12:28:07,237] jepsen worker 44 - jepsen.util 494 :ok :read [14 27] INFO [2019-12-17 12:28:07,238] jepsen worker 44 - jepsen.util 494 :invoke :cas [14 [29 10]] INFO [2019-12-17 12:28:07,252] jepsen worker 43 - jepsen.util 853 :info :write [14 18] :RequestCanceledException INFO [2019-12-17 12:28:07,254] jepsen worker 43 - jepsen.util 943 :invoke :read [14 nil] INFO [2019-12-17 12:28:07,256] jepsen worker 27 - jepsen.util 477 :fail :cas [9 [42 5]] :ValueNotSwapFrom INFO [2019-12-17 12:28:07,256] jepsen worker 27 - jepsen.util 477 :invoke :write [9 45] INFO [2019-12-17 12:28:07,264] jepsen worker 28 - jepsen.util 298 :ok :write [9 36] INFO [2019-12-17 12:28:07,264] jepsen worker 28 - jepsen.util 298 :invoke :cas [9 [30 33]] INFO [2019-12-17 12:28:07,265] jepsen worker 84 - jepsen.util 444 :ok :read [28 38] INFO [2019-12-17 12:28:07,265] jepsen worker 84 - jepsen.util 444 :invoke :read [28 nil] INFO [2019-12-17 12:28:07,294] jepsen worker 76 - jepsen.util 706 :fail :cas [25 [10 33]] :ValueNotSwapFrom INFO [2019-12-17 12:28:07,294] jepsen worker 76 - jepsen.util 706 :invoke :cas [25 [9 25]] INFO [2019-12-17 12:28:07,295] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3242,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:28:07,295] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3242,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:28:07,295] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3245,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:28:07,295] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3245,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:28:07,295] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3246,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:28:07,295] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3246,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:28:07,308] jepsen worker 28 - jepsen.util 298 :fail :cas [9 [30 33]] :ValueNotSwapFrom INFO [2019-12-17 12:28:07,309] jepsen worker 28 - jepsen.util 298 :invoke :cas [9 [19 10]] INFO [2019-12-17 12:28:07,315] jepsen worker 44 - jepsen.util 494 :fail :cas [14 [29 10]] :ValueNotSwapFrom INFO [2019-12-17 12:28:07,315] jepsen worker 44 - jepsen.util 494 :invoke :write [14 17] INFO [2019-12-17 12:28:07,333] jepsen worker 55 - jepsen.util 865 :fail :cas [18 [2 26]] :ValueNotSwapFrom INFO [2019-12-17 12:28:07,333] jepsen worker 55 - jepsen.util 865 :invoke :write [18 28] INFO [2019-12-17 12:28:07,335] jepsen worker 25 - jepsen.util 745 :info :write [8 36] :RequestCanceledException INFO [2019-12-17 12:28:07,335] jepsen worker 2 - jepsen.util 632 :ok :write [0 37] INFO [2019-12-17 12:28:07,335] jepsen worker 2 - jepsen.util 632 :invoke :read [0 nil] INFO [2019-12-17 12:28:07,336] jepsen worker 25 - jepsen.util 835 :invoke :cas [8 [29 31]] INFO [2019-12-17 12:28:07,353] jepsen worker 28 - jepsen.util 298 :fail :cas [9 [19 10]] :ValueNotSwapFrom INFO [2019-12-17 12:28:07,353] jepsen worker 28 - jepsen.util 298 :invoke :write [9 32] INFO [2019-12-17 12:28:07,356] jepsen worker 73 - jepsen.util 793 :ok :write [24 24] INFO [2019-12-17 12:28:07,356] jepsen worker 73 - jepsen.util 793 :invoke :write [24 2] INFO [2019-12-17 12:28:07,383] jepsen worker 63 - jepsen.util 423 :fail :cas [21 [47 33]] :ValueNotSwapFrom INFO [2019-12-17 12:28:07,383] jepsen worker 1 - jepsen.util 631 :fail :cas [0 [33 47]] :ValueNotSwapFrom INFO [2019-12-17 12:28:07,383] jepsen worker 63 - jepsen.util 423 :invoke :write [21 25] INFO [2019-12-17 12:28:07,383] jepsen worker 1 - jepsen.util 631 :invoke :write [0 4] INFO [2019-12-17 12:28:07,399] jepsen worker 41 - jepsen.util 401 :ok :read [13 36] INFO [2019-12-17 12:28:07,399] jepsen worker 75 - jepsen.util 615 :ok :read [25 2] INFO [2019-12-17 12:28:07,399] jepsen worker 41 - jepsen.util 401 :invoke :cas [13 [26 3]] INFO [2019-12-17 12:28:07,400] jepsen worker 75 - jepsen.util 615 :invoke :write [25 42] INFO [2019-12-17 12:28:07,409] jepsen worker 48 - jepsen.util 498 :fail :cas [16 [46 8]] :ValueNotSwapFrom INFO [2019-12-17 12:28:07,409] jepsen worker 48 - jepsen.util 498 :invoke :read [16 nil] INFO [2019-12-17 12:28:07,418] jepsen worker 47 - jepsen.util 857 :info :write [15 6] :RequestCanceledException INFO [2019-12-17 12:28:07,418] jepsen worker 71 - jepsen.util 521 :info :write [23 16] :RequestCanceledException INFO [2019-12-17 12:28:07,421] jepsen worker 47 - jepsen.util 947 :invoke :cas [15 [0 25]] INFO [2019-12-17 12:28:07,421] jepsen worker 71 - jepsen.util 611 :invoke :write [23 22] INFO [2019-12-17 12:28:07,421] jepsen worker 23 - jepsen.util 563 :ok :read [7 20] INFO [2019-12-17 12:28:07,421] jepsen worker 23 - jepsen.util 563 :invoke :cas [7 [41 33]] INFO [2019-12-17 12:28:07,425] jepsen worker 17 - jepsen.util 557 :fail :cas [5 [12 23]] :ValueNotSwapFrom INFO [2019-12-17 12:28:07,425] jepsen worker 17 - jepsen.util 557 :invoke :write [5 49] INFO [2019-12-17 12:28:07,443] jepsen worker 27 - jepsen.util 477 :ok :write [9 45] INFO [2019-12-17 12:28:07,444] jepsen worker 27 - jepsen.util 477 :invoke :write [9 6] INFO [2019-12-17 12:28:07,453] jepsen worker 85 - jepsen.util 625 :ok :read [28 38] INFO [2019-12-17 12:28:07,454] jepsen worker 85 - jepsen.util 625 :invoke :write [28 26] INFO [2019-12-17 12:28:07,454] jepsen worker 47 - jepsen.util 947 :fail :cas [15 [0 25]] :ValueNotSwapFrom INFO [2019-12-17 12:28:07,454] jepsen worker 47 - jepsen.util 947 :invoke :write [15 47] INFO [2019-12-17 12:28:07,456] jepsen worker 51 - jepsen.util 591 :ok :read [17 0] INFO [2019-12-17 12:28:07,457] jepsen worker 51 - jepsen.util 591 :invoke :read [17 nil] INFO [2019-12-17 12:28:07,476] jepsen worker 26 - jepsen.util 836 :fail :cas [8 [46 32]] :ValueNotSwapFrom INFO [2019-12-17 12:28:07,476] jepsen worker 26 - jepsen.util 836 :invoke :read [8 nil] INFO [2019-12-17 12:28:07,478] jepsen worker 29 - jepsen.util 569 :info :write [9 48] :RequestCanceledException INFO [2019-12-17 12:28:07,479] jepsen worker 29 - jepsen.util 659 :invoke :read [9 nil] INFO [2019-12-17 12:28:07,496] jepsen worker 87 - jepsen.util 537 :info :write [29 30] :RequestCanceledException INFO [2019-12-17 12:28:07,498] jepsen worker 87 - jepsen.util 627 :invoke :read [29 nil] INFO [2019-12-17 12:28:07,508] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3273,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:28:07,508] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3273,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:28:07,523] jepsen worker 68 - jepsen.util 428 :ok :write [22 0] INFO [2019-12-17 12:28:07,523] jepsen worker 68 - jepsen.util 428 :invoke :cas [22 [29 30]] INFO [2019-12-17 12:28:07,527] jepsen worker 31 - jepsen.util 841 :ok :write [10 45] INFO [2019-12-17 12:28:07,528] jepsen worker 31 - jepsen.util 841 :invoke :read [10 nil] INFO [2019-12-17 12:28:07,545] jepsen worker 69 - jepsen.util 699 :info :write [23 43] :RequestCanceledException INFO [2019-12-17 12:28:07,548] jepsen worker 69 - jepsen.util 789 :invoke :cas [23 [3 10]] INFO [2019-12-17 12:28:07,555] jepsen worker 31 - jepsen.util 841 :ok :read [10 45] INFO [2019-12-17 12:28:07,555] jepsen worker 31 - jepsen.util 841 :invoke :cas [10 [16 46]] INFO [2019-12-17 12:28:07,566] jepsen worker 27 - jepsen.util 477 :ok :write [9 6] INFO [2019-12-17 12:28:07,566] jepsen worker 27 - jepsen.util 477 :invoke :cas [9 [29 37]] INFO [2019-12-17 12:28:07,584] jepsen worker 61 - jepsen.util 781 :fail :cas [20 [35 46]] :ValueNotSwapFrom INFO [2019-12-17 12:28:07,584] jepsen worker 61 - jepsen.util 781 :invoke :cas [20 [19 13]] INFO [2019-12-17 12:28:07,584] jepsen worker 31 - jepsen.util 841 :fail :cas [10 [16 46]] :ValueNotSwapFrom INFO [2019-12-17 12:28:07,584] jepsen worker 31 - jepsen.util 841 :invoke :write [10 38] INFO [2019-12-17 12:28:07,587] jepsen worker 50 - jepsen.util 590 :info :cas [16 [3 30]] :RequestCanceledException INFO [2019-12-17 12:28:07,589] jepsen worker 50 - jepsen.util 680 :invoke :cas [16 [2 17]] INFO [2019-12-17 12:28:07,598] jepsen worker 27 - jepsen.util 477 :fail :cas [9 [29 37]] :ValueNotSwapFrom INFO [2019-12-17 12:28:07,599] jepsen worker 27 - jepsen.util 477 :invoke :cas [9 [3 0]] INFO [2019-12-17 12:28:07,617] jepsen worker 77 - jepsen.util 617 :ok :read [25 2] INFO [2019-12-17 12:28:07,618] jepsen worker 77 - jepsen.util 617 :invoke :read [25 nil] INFO [2019-12-17 12:28:07,621] jepsen worker 46 - jepsen.util 586 :info :write [15 23] :RequestCanceledException INFO [2019-12-17 12:28:07,623] jepsen worker 46 - jepsen.util 676 :invoke :cas [15 [21 28]] INFO [2019-12-17 12:28:07,635] jepsen worker 56 - jepsen.util 596 :info :write [18 12] :RequestCanceledException INFO [2019-12-17 12:28:07,637] jepsen worker 56 - jepsen.util 686 :invoke :read [18 nil] INFO [2019-12-17 12:28:07,643] jepsen worker 27 - jepsen.util 477 :fail :cas [9 [3 0]] :ValueNotSwapFrom INFO [2019-12-17 12:28:07,643] jepsen worker 27 - jepsen.util 477 :invoke :write [9 0] INFO [2019-12-17 12:28:07,644] jepsen worker 79 - jepsen.util 619 :ok :read [26 47] INFO [2019-12-17 12:28:07,644] jepsen worker 79 - jepsen.util 619 :invoke :read [26 nil] INFO [2019-12-17 12:28:07,660] jepsen worker 46 - jepsen.util 676 :fail :cas [15 [21 28]] :ValueNotSwapFrom INFO [2019-12-17 12:28:07,661] jepsen worker 46 - jepsen.util 676 :invoke :read [15 nil] INFO [2019-12-17 12:28:07,686] jepsen worker 46 - jepsen.util 676 :ok :read [15 35] INFO [2019-12-17 12:28:07,686] jepsen worker 46 - jepsen.util 676 :invoke :cas [15 [16 4]] INFO [2019-12-17 12:28:07,712] jepsen worker 27 - jepsen.util 477 :ok :write [9 0] INFO [2019-12-17 12:28:07,712] jepsen worker 27 - jepsen.util 477 :invoke :read [9 nil] INFO [2019-12-17 12:28:07,714] jepsen worker 46 - jepsen.util 676 :fail :cas [15 [16 4]] :ValueNotSwapFrom INFO [2019-12-17 12:28:07,715] jepsen worker 46 - jepsen.util 676 :invoke :read [15 nil] INFO [2019-12-17 12:28:07,739] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3298,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:28:07,739] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3298,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:28:07,739] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3299,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:28:07,739] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3299,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:28:07,743] jepsen worker 27 - jepsen.util 477 :ok :read [9 0] INFO [2019-12-17 12:28:07,744] jepsen worker 27 - jepsen.util 477 :invoke :cas [9 [25 13]] INFO [2019-12-17 12:28:07,745] jepsen worker 46 - jepsen.util 676 :ok :read [15 35] INFO [2019-12-17 12:28:07,745] jepsen worker 46 - jepsen.util 676 :invoke :cas [15 [3 19]] INFO [2019-12-17 12:28:07,774] jepsen worker 23 - jepsen.util 563 :fail :cas [7 [41 33]] :ValueNotSwapFrom INFO [2019-12-17 12:28:07,774] jepsen worker 23 - jepsen.util 563 :invoke :read [7 nil] INFO [2019-12-17 12:28:07,780] jepsen worker 46 - jepsen.util 676 :fail :cas [15 [3 19]] :ValueNotSwapFrom INFO [2019-12-17 12:28:07,780] jepsen worker 46 - jepsen.util 676 :invoke :write [15 1] INFO [2019-12-17 12:28:07,787] jepsen worker 80 - jepsen.util 800 :info :write [26 37] :RequestCanceledException INFO [2019-12-17 12:28:07,787] jepsen worker 27 - jepsen.util 477 :fail :cas [9 [25 13]] :ValueNotSwapFrom INFO [2019-12-17 12:28:07,787] jepsen worker 27 - jepsen.util 477 :invoke :cas [9 [29 12]] INFO [2019-12-17 12:28:07,789] jepsen worker 80 - jepsen.util 890 :invoke :read [26 nil] INFO [2019-12-17 12:28:07,800] jepsen worker 51 - jepsen.util 591 :ok :read [17 0] INFO [2019-12-17 12:28:07,800] jepsen worker 51 - jepsen.util 591 :invoke :cas [17 [35 3]] INFO [2019-12-17 12:28:07,806] jepsen worker 20 - jepsen.util 650 :info :cas [6 [4 0]] :RequestCanceledException INFO [2019-12-17 12:28:07,806] jepsen worker 74 - jepsen.util 524 :fail :read [24 nil] :RequestTimeoutException INFO [2019-12-17 12:28:07,806] jepsen worker 62 - jepsen.util 782 :info :write [20 39] :RequestCanceledException INFO [2019-12-17 12:28:07,806] jepsen worker 74 - jepsen.util 524 :invoke :cas [24 [36 47]] INFO [2019-12-17 12:28:07,807] jepsen worker 20 - jepsen.util 740 :invoke :cas [6 [17 40]] INFO [2019-12-17 12:28:07,808] jepsen worker 62 - jepsen.util 872 :invoke :read [20 nil] INFO [2019-12-17 12:28:07,810] jepsen worker 27 - jepsen.util 477 :fail :cas [9 [29 12]] :ValueNotSwapFrom INFO [2019-12-17 12:28:07,810] jepsen worker 27 - jepsen.util 477 :invoke :cas [9 [6 18]] INFO [2019-12-17 12:28:07,825] jepsen worker 6 - jepsen.util 636 :fail :cas [2 [9 9]] :ValueNotSwapFrom INFO [2019-12-17 12:28:07,825] jepsen worker 6 - jepsen.util 636 :invoke :write [2 37] INFO [2019-12-17 12:28:07,856] jepsen worker 27 - jepsen.util 477 :fail :cas [9 [6 18]] :ValueNotSwapFrom INFO [2019-12-17 12:28:07,856] jepsen worker 27 - jepsen.util 477 :invoke :write [9 33] INFO [2019-12-17 12:28:07,862] jepsen worker 20 - jepsen.util 740 :fail :cas [6 [17 40]] :ValueNotSwapFrom INFO [2019-12-17 12:28:07,862] jepsen worker 20 - jepsen.util 740 :invoke :cas [6 [47 10]] INFO [2019-12-17 12:28:07,879] jepsen worker 25 - jepsen.util 835 :fail :cas [8 [29 31]] :ValueNotSwapFrom INFO [2019-12-17 12:28:07,879] jepsen worker 25 - jepsen.util 835 :invoke :write [8 6] INFO [2019-12-17 12:28:07,904] jepsen worker 87 - jepsen.util 627 :ok :read [29 5] INFO [2019-12-17 12:28:07,904] jepsen worker 20 - jepsen.util 740 :fail :cas [6 [47 10]] :ValueNotSwapFrom INFO [2019-12-17 12:28:07,904] jepsen worker 23 - jepsen.util 563 :ok :read [7 20] INFO [2019-12-17 12:28:07,904] jepsen worker 87 - jepsen.util 627 :invoke :write [29 10] INFO [2019-12-17 12:28:07,904] jepsen worker 20 - jepsen.util 740 :invoke :read [6 nil] INFO [2019-12-17 12:28:07,904] jepsen worker 23 - jepsen.util 563 :invoke :cas [7 [21 8]] INFO [2019-12-17 12:28:07,907] jepsen worker 42 - jepsen.util 942 :fail :cas [14 [8 32]] :ValueNotSwapFrom INFO [2019-12-17 12:28:07,907] jepsen worker 42 - jepsen.util 942 :invoke :write [14 11] INFO [2019-12-17 12:28:07,919] jepsen worker 5 - jepsen.util 365 :info :write [1 46] :RequestCanceledException INFO [2019-12-17 12:28:07,921] jepsen worker 5 - jepsen.util 455 :invoke :write [1 49] INFO [2019-12-17 12:28:07,931] jepsen worker 20 - jepsen.util 740 :ok :read [6 25] INFO [2019-12-17 12:28:07,931] jepsen worker 20 - jepsen.util 740 :invoke :write [6 26] INFO [2019-12-17 12:28:07,943] jepsen worker 27 - jepsen.util 477 :ok :write [9 33] INFO [2019-12-17 12:28:07,943] jepsen worker 27 - jepsen.util 477 :invoke :write [9 1] INFO [2019-12-17 12:28:07,946] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3323,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:28:07,946] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3323,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:28:07,946] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3327,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:28:07,946] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3327,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:28:07,949] jepsen worker 33 - jepsen.util 663 :info :write [11 33] :RequestCanceledException INFO [2019-12-17 12:28:07,951] jepsen worker 33 - jepsen.util 753 :invoke :write [11 19] INFO [2019-12-17 12:28:07,967] jepsen worker 80 - jepsen.util 890 :ok :read [26 47] INFO [2019-12-17 12:28:07,968] jepsen worker 80 - jepsen.util 890 :invoke :read [26 nil] INFO [2019-12-17 12:28:07,991] jepsen worker 8 - jepsen.util 728 :fail :cas [2 [21 43]] :ValueNotSwapFrom INFO [2019-12-17 12:28:07,991] jepsen worker 8 - jepsen.util 728 :invoke :cas [2 [12 25]] INFO [2019-12-17 12:28:08,021] jepsen worker 21 - jepsen.util 741 :info :write [7 28] :RequestCanceledException INFO [2019-12-17 12:28:08,021] jepsen worker 60 - jepsen.util 690 :info :write [20 15] :RequestCanceledException INFO [2019-12-17 12:28:08,021] jepsen worker 10 - jepsen.util 640 :info :write [3 35] :RequestCanceledException INFO [2019-12-17 12:28:08,022] jepsen worker 54 - jepsen.util 774 :ok :read [18 47] INFO [2019-12-17 12:28:08,023] jepsen worker 54 - jepsen.util 774 :invoke :read [18 nil] INFO [2019-12-17 12:28:08,023] jepsen worker 21 - jepsen.util 831 :invoke :cas [7 [49 46]] INFO [2019-12-17 12:28:08,023] jepsen worker 60 - jepsen.util 780 :invoke :write [20 40] INFO [2019-12-17 12:28:08,023] jepsen worker 10 - jepsen.util 730 :invoke :read [3 nil] INFO [2019-12-17 12:28:08,034] jepsen worker 27 - jepsen.util 477 :ok :write [9 1] INFO [2019-12-17 12:28:08,034] jepsen worker 27 - jepsen.util 477 :invoke :cas [9 [38 44]] INFO [2019-12-17 12:28:08,035] jepsen worker 65 - jepsen.util 515 :fail :read [21 nil] :RequestTimeoutException INFO [2019-12-17 12:28:08,035] jepsen worker 65 - jepsen.util 515 :invoke :cas [21 [42 3]] INFO [2019-12-17 12:28:08,052] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3340,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:28:08,053] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3340,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:28:08,053] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3344,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:28:08,053] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3344,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:28:08,061] jepsen worker 27 - jepsen.util 477 :fail :cas [9 [38 44]] :ValueNotSwapFrom INFO [2019-12-17 12:28:08,061] jepsen worker 27 - jepsen.util 477 :invoke :write [9 48] INFO [2019-12-17 12:28:08,062] jepsen worker 43 - jepsen.util 943 :ok :read [14 27] INFO [2019-12-17 12:28:08,063] jepsen worker 43 - jepsen.util 943 :invoke :read [14 nil] INFO [2019-12-17 12:28:08,064] jepsen worker 23 - jepsen.util 563 :fail :cas [7 [21 8]] :ValueNotSwapFrom INFO [2019-12-17 12:28:08,064] jepsen worker 23 - jepsen.util 563 :invoke :read [7 nil] INFO [2019-12-17 12:28:08,081] jepsen worker 51 - jepsen.util 591 :fail :cas [17 [35 3]] :ValueNotSwapFrom INFO [2019-12-17 12:28:08,081] jepsen worker 51 - jepsen.util 591 :invoke :cas [17 [0 30]] INFO [2019-12-17 12:28:08,082] jepsen worker 76 - jepsen.util 706 :fail :cas [25 [9 25]] :ValueNotSwapFrom INFO [2019-12-17 12:28:08,082] jepsen worker 76 - jepsen.util 706 :invoke :cas [25 [6 46]] INFO [2019-12-17 12:28:08,087] jepsen worker 82 - jepsen.util 802 :info :cas [27 [3 15]] :RequestCanceledException INFO [2019-12-17 12:28:08,088] jepsen worker 82 - jepsen.util 892 :invoke :cas [27 [24 35]] INFO [2019-12-17 12:28:08,099] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.08984375}, :ns_1@172.28.128.126 {:progress 0.08273381294964033}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:28:08,099] jepsen nemesis - couchbase.util Rebalance check count: 19 INFO [2019-12-17 12:28:08,099] jepsen nemesis - couchbase.util Rebalance stuck count: 1 INFO [2019-12-17 12:28:08,106] jepsen worker 15 - jepsen.util 645 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:28:08,106] jepsen worker 15 - jepsen.util 645 :invoke :write [5 39] INFO [2019-12-17 12:28:08,121] jepsen worker 34 - jepsen.util 664 :info :write [11 29] :RequestCanceledException INFO [2019-12-17 12:28:08,121] jepsen worker 16 - jepsen.util 736 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:28:08,121] jepsen worker 16 - jepsen.util 736 :invoke :cas [5 [8 16]] INFO [2019-12-17 12:28:08,122] jepsen worker 34 - jepsen.util 754 :invoke :read [11 nil] INFO [2019-12-17 12:28:08,135] jepsen worker 11 - jepsen.util 731 :info :cas [3 [38 16]] :RequestCanceledException INFO [2019-12-17 12:28:08,136] jepsen worker 11 - jepsen.util 821 :invoke :cas [3 [46 39]] INFO [2019-12-17 12:28:08,142] jepsen worker 61 - jepsen.util 781 :fail :cas [20 [19 13]] :ValueNotSwapFrom INFO [2019-12-17 12:28:08,142] jepsen worker 61 - jepsen.util 781 :invoke :cas [20 [35 12]] INFO [2019-12-17 12:28:08,147] jepsen worker 27 - jepsen.util 477 :ok :write [9 48] INFO [2019-12-17 12:28:08,147] jepsen worker 27 - jepsen.util 477 :invoke :cas [9 [9 46]] INFO [2019-12-17 12:28:08,155] jepsen worker 56 - jepsen.util 686 :ok :read [18 47] INFO [2019-12-17 12:28:08,156] jepsen worker 56 - jepsen.util 686 :invoke :cas [18 [20 32]] INFO [2019-12-17 12:28:08,164] jepsen worker 11 - jepsen.util 821 :fail :cas [3 [46 39]] :ValueNotSwapFrom INFO [2019-12-17 12:28:08,165] jepsen worker 11 - jepsen.util 821 :invoke :cas [3 [10 6]] INFO [2019-12-17 12:28:08,170] jepsen worker 77 - jepsen.util 617 :ok :read [25 2] INFO [2019-12-17 12:28:08,171] jepsen worker 77 - jepsen.util 617 :invoke :read [25 nil] INFO [2019-12-17 12:28:08,177] jepsen worker 27 - jepsen.util 477 :fail :cas [9 [9 46]] :ValueNotSwapFrom INFO [2019-12-17 12:28:08,177] jepsen worker 27 - jepsen.util 477 :invoke :cas [9 [36 44]] INFO [2019-12-17 12:28:08,180] jepsen worker 88 - jepsen.util 538 :info :write [29 14] :RequestCanceledException INFO [2019-12-17 12:28:08,182] jepsen worker 88 - jepsen.util 628 :invoke :read [29 nil] INFO [2019-12-17 12:28:08,202] jepsen worker 40 - jepsen.util 850 :info :cas [13 [32 3]] :RequestCanceledException INFO [2019-12-17 12:28:08,204] jepsen worker 40 - jepsen.util 940 :invoke :cas [13 [38 28]] INFO [2019-12-17 12:28:08,221] jepsen worker 66 - jepsen.util 516 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:28:08,221] jepsen worker 66 - jepsen.util 516 :invoke :write [22 36] INFO [2019-12-17 12:28:08,263] jepsen worker 11 - jepsen.util 821 :fail :cas [3 [10 6]] :ValueNotSwapFrom INFO [2019-12-17 12:28:08,263] jepsen worker 11 - jepsen.util 821 :invoke :cas [3 [27 2]] INFO [2019-12-17 12:28:08,267] jepsen worker 27 - jepsen.util 477 :fail :cas [9 [36 44]] :ValueNotSwapFrom INFO [2019-12-17 12:28:08,267] jepsen worker 27 - jepsen.util 477 :invoke :read [9 nil] INFO [2019-12-17 12:28:08,273] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3378,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:28:08,273] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3378,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:28:08,273] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3382,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:28:08,273] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3382,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:28:08,274] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3369,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:28:08,275] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3369,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:28:08,275] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3371,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:28:08,275] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3371,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:28:08,286] jepsen worker 61 - jepsen.util 781 :fail :cas [20 [35 12]] :ValueNotSwapFrom INFO [2019-12-17 12:28:08,286] jepsen worker 61 - jepsen.util 781 :invoke :write [20 18] INFO [2019-12-17 12:28:08,303] jepsen worker 26 - jepsen.util 836 :ok :read [8 14] INFO [2019-12-17 12:28:08,303] jepsen worker 26 - jepsen.util 836 :invoke :cas [8 [0 3]] INFO [2019-12-17 12:28:08,315] jepsen worker 11 - jepsen.util 821 :fail :cas [3 [27 2]] :ValueNotSwapFrom INFO [2019-12-17 12:28:08,316] jepsen worker 11 - jepsen.util 821 :invoke :cas [3 [42 45]] INFO [2019-12-17 12:28:08,320] jepsen worker 27 - jepsen.util 477 :ok :read [9 48] INFO [2019-12-17 12:28:08,320] jepsen worker 27 - jepsen.util 477 :invoke :read [9 nil] INFO [2019-12-17 12:28:08,335] jepsen worker 78 - jepsen.util 708 :info :write [26 39] :RequestCanceledException INFO [2019-12-17 12:28:08,337] jepsen worker 78 - jepsen.util 798 :invoke :cas [26 [33 7]] INFO [2019-12-17 12:28:08,352] jepsen worker 4 - jepsen.util 634 :fail :cas [1 [12 44]] :ValueNotSwapFrom INFO [2019-12-17 12:28:08,352] jepsen worker 4 - jepsen.util 634 :invoke :cas [1 [26 48]] INFO [2019-12-17 12:28:08,355] jepsen worker 11 - jepsen.util 821 :fail :cas [3 [42 45]] :ValueNotSwapFrom INFO [2019-12-17 12:28:08,355] jepsen worker 11 - jepsen.util 821 :invoke :read [3 nil] INFO [2019-12-17 12:28:08,364] jepsen worker 21 - jepsen.util 831 :fail :cas [7 [49 46]] :ValueNotSwapFrom INFO [2019-12-17 12:28:08,365] jepsen worker 21 - jepsen.util 831 :invoke :read [7 nil] INFO [2019-12-17 12:28:08,368] jepsen worker 27 - jepsen.util 477 :ok :read [9 48] INFO [2019-12-17 12:28:08,369] jepsen worker 27 - jepsen.util 477 :invoke :write [9 43] INFO [2019-12-17 12:28:08,387] jepsen worker 11 - jepsen.util 821 :ok :read [3 1] INFO [2019-12-17 12:28:08,388] jepsen worker 11 - jepsen.util 821 :invoke :write [3 1] INFO [2019-12-17 12:28:08,405] jepsen worker 36 - jepsen.util 756 :info :write [12 37] :RequestCanceledException INFO [2019-12-17 12:28:08,428] jepsen worker 36 - jepsen.util 846 :invoke :cas [12 [6 4]] INFO [2019-12-17 12:28:08,449] jepsen worker 27 - jepsen.util 477 :ok :write [9 43] INFO [2019-12-17 12:28:08,449] jepsen worker 27 - jepsen.util 477 :invoke :read [9 nil] INFO [2019-12-17 12:28:08,501] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3399,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:28:08,501] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3399,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:28:08,502] jepsen worker 81 - jepsen.util 711 :info :write [27 11] :RequestCanceledException INFO [2019-12-17 12:28:08,510] jepsen worker 81 - jepsen.util 801 :invoke :write [27 12] INFO [2019-12-17 12:28:08,527] jepsen worker 86 - jepsen.util 536 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:28:08,531] jepsen worker 86 - jepsen.util 536 :invoke :write [28 47] INFO [2019-12-17 12:28:08,537] jepsen worker 45 - jepsen.util 945 :info :write [15 14] :RequestCanceledException INFO [2019-12-17 12:28:08,560] jepsen worker 37 - jepsen.util 667 :info :write [12 12] :RequestCanceledException INFO [2019-12-17 12:28:08,594] jepsen worker 83 - jepsen.util 803 :info :write [27 1] :RequestCanceledException INFO [2019-12-17 12:28:08,596] jepsen worker 45 - jepsen.util 1035 :invoke :write [15 31] INFO [2019-12-17 12:28:08,602] jepsen worker 26 - jepsen.util 836 :fail :cas [8 [0 3]] :ValueNotSwapFrom INFO [2019-12-17 12:28:08,635] jepsen worker 26 - jepsen.util 836 :invoke :read [8 nil] INFO [2019-12-17 12:28:08,637] jepsen worker 37 - jepsen.util 757 :invoke :write [12 45] INFO [2019-12-17 12:28:08,639] jepsen worker 89 - jepsen.util 809 :info :write [29 10] :RequestCanceledException INFO [2019-12-17 12:28:08,667] jepsen worker 22 - jepsen.util 652 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:28:08,667] jepsen worker 22 - jepsen.util 652 :invoke :read [7 nil] INFO [2019-12-17 12:28:08,676] jepsen worker 89 - jepsen.util 899 :invoke :write [29 48] INFO [2019-12-17 12:28:08,678] jepsen worker 83 - jepsen.util 893 :invoke :read [27 nil] INFO [2019-12-17 12:28:08,734] jepsen worker 39 - jepsen.util 759 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:28:08,739] jepsen worker 39 - jepsen.util 759 :invoke :read [13 nil] INFO [2019-12-17 12:28:08,759] jepsen worker 3 - jepsen.util 453 :fail :read [1 nil] :RequestTimeoutException INFO [2019-12-17 12:28:08,761] jepsen worker 14 - jepsen.util 734 :fail :read [4 nil] :RequestTimeoutException INFO [2019-12-17 12:28:08,774] jepsen worker 3 - jepsen.util 453 :invoke :cas [1 [25 25]] INFO [2019-12-17 12:28:08,774] jepsen worker 14 - jepsen.util 734 :invoke :write [4 49] INFO [2019-12-17 12:28:08,776] jepsen worker 7 - jepsen.util 817 :info :write [2 15] :RequestCanceledException INFO [2019-12-17 12:28:08,832] jepsen worker 24 - jepsen.util 564 :info :write [8 19] :RequestCanceledException INFO [2019-12-17 12:28:08,832] jepsen worker 58 - jepsen.util 508 :info :cas [19 [10 9]] :RequestCanceledException INFO [2019-12-17 12:28:08,958] jepsen worker 0 - jepsen.util 720 :info :cas [0 [28 23]] :RequestCanceledException INFO [2019-12-17 12:28:08,963] jepsen worker 62 - jepsen.util 872 :ok :read [20 7] INFO [2019-12-17 12:28:08,972] jepsen worker 32 - jepsen.util 662 :info :write [10 17] :RequestCanceledException INFO [2019-12-17 12:28:08,974] jepsen worker 78 - jepsen.util 798 :fail :cas [26 [33 7]] :ValueNotSwapFrom INFO [2019-12-17 12:28:08,975] jepsen worker 57 - jepsen.util 687 :info :cas [19 [23 32]] :RequestCanceledException INFO [2019-12-17 12:28:08,981] jepsen worker 35 - jepsen.util 575 :info :write [11 49] :RequestCanceledException INFO [2019-12-17 12:28:08,978] jepsen worker 59 - jepsen.util 689 :info :write [19 1] :RequestCanceledException INFO [2019-12-17 12:28:08,993] jepsen worker 52 - jepsen.util 682 :info :write [17 10] :RequestCanceledException INFO [2019-12-17 12:28:08,999] jepsen worker 70 - jepsen.util 880 :fail :cas [23 [33 23]] :ValueNotSwapFrom INFO [2019-12-17 12:28:09,012] jepsen worker 78 - jepsen.util 798 :invoke :cas [26 [24 37]] INFO [2019-12-17 12:28:09,012] jepsen worker 70 - jepsen.util 880 :invoke :cas [23 [3 36]] INFO [2019-12-17 12:28:09,013] jepsen worker 62 - jepsen.util 872 :invoke :cas [20 [22 15]] INFO [2019-12-17 12:28:09,027] jepsen worker 24 - jepsen.util 654 :invoke :read [8 nil] INFO [2019-12-17 12:28:09,028] jepsen worker 7 - jepsen.util 907 :invoke :cas [2 [48 11]] INFO [2019-12-17 12:28:09,032] jepsen worker 52 - jepsen.util 772 :invoke :read [17 nil] INFO [2019-12-17 12:28:09,054] jepsen worker 59 - jepsen.util 779 :invoke :read [19 nil] INFO [2019-12-17 12:28:09,054] jepsen worker 58 - jepsen.util 598 :invoke :write [19 1] INFO [2019-12-17 12:28:09,054] jepsen worker 0 - jepsen.util 810 :invoke :write [0 43] INFO [2019-12-17 12:28:09,073] jepsen worker 12 - jepsen.util 642 :info :write [4 12] :RequestCanceledException INFO [2019-12-17 12:28:09,092] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3426,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:28:09,107] jepsen worker 19 - jepsen.util 649 :info :write [6 23] :RequestCanceledException INFO [2019-12-17 12:28:09,128] jepsen worker 49 - jepsen.util 499 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:28:09,128] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3426,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:28:09,131] jepsen worker 31 - jepsen.util 841 :ok :write [10 38] INFO [2019-12-17 12:28:09,131] jepsen worker 35 - jepsen.util 665 :invoke :read [11 nil] INFO [2019-12-17 12:28:09,137] jepsen worker 38 - jepsen.util 578 :info :cas [12 [44 19]] :RequestCanceledException INFO [2019-12-17 12:28:09,138] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3435,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:28:09,140] jepsen worker 57 - jepsen.util 777 :invoke :read [19 nil] INFO [2019-12-17 12:28:09,139] jepsen worker 73 - jepsen.util 793 :ok :write [24 2] INFO [2019-12-17 12:28:09,144] jepsen worker 49 - jepsen.util 499 :invoke :cas [16 [2 21]] INFO [2019-12-17 12:28:09,148] jepsen worker 20 - jepsen.util 740 :ok :write [6 26] INFO [2019-12-17 12:28:09,147] jepsen worker 18 - jepsen.util 738 :fail :read [6 nil] :RequestTimeoutException INFO [2019-12-17 12:28:09,148] jepsen worker 1 - jepsen.util 631 :ok :write [0 4] INFO [2019-12-17 12:28:09,156] jepsen worker 18 - jepsen.util 738 :invoke :read [6 nil] INFO [2019-12-17 12:28:09,158] jepsen worker 20 - jepsen.util 740 :invoke :write [6 32] INFO [2019-12-17 12:28:09,162] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3435,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:28:09,167] jepsen worker 31 - jepsen.util 841 :invoke :cas [10 [14 29]] INFO [2019-12-17 12:28:09,178] jepsen worker 72 - jepsen.util 702 :info :write [24 39] :RequestCanceledException INFO [2019-12-17 12:28:09,182] jepsen worker 1 - jepsen.util 631 :invoke :write [0 5] INFO [2019-12-17 12:28:09,183] jepsen worker 73 - jepsen.util 793 :invoke :write [24 42] INFO [2019-12-17 12:28:09,192] jepsen worker 64 - jepsen.util 424 :info :write [21 26] :RequestCanceledException INFO [2019-12-17 12:28:09,212] jepsen worker 32 - jepsen.util 752 :invoke :cas [10 [24 29]] INFO [2019-12-17 12:28:09,224] jepsen worker 12 - jepsen.util 732 :invoke :write [4 28] INFO [2019-12-17 12:28:09,232] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3446,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:28:09,234] jepsen worker 30 - jepsen.util 570 :info :write [10 39] :RequestCanceledException INFO [2019-12-17 12:28:09,237] jepsen worker 53 - jepsen.util 773 :info :write [17 7] :RequestCanceledException INFO [2019-12-17 12:28:09,250] jepsen worker 13 - jepsen.util 733 :info :write [4 17] :RequestCanceledException INFO [2019-12-17 12:28:09,301] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3443,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:28:09,318] jepsen worker 19 - jepsen.util 739 :invoke :write [6 21] INFO [2019-12-17 12:28:09,328] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3443,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:28:09,360] jepsen worker 38 - jepsen.util 668 :invoke :cas [12 [11 23]] INFO [2019-12-17 12:28:09,370] jepsen worker 67 - jepsen.util 967 :info :write [22 19] :RequestCanceledException INFO [2019-12-17 12:28:09,392] jepsen worker 9 - jepsen.util 639 :info :cas [3 [49 17]] :RequestCanceledException INFO [2019-12-17 12:28:09,432] jepsen worker 84 - jepsen.util 444 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:28:09,433] jepsen worker 64 - jepsen.util 514 :invoke :cas [21 [19 45]] INFO [2019-12-17 12:28:09,432] jepsen worker 75 - jepsen.util 615 :info :write [25 42] :RequestCanceledException INFO [2019-12-17 12:28:09,440] jepsen worker 72 - jepsen.util 792 :invoke :cas [24 [6 49]] INFO [2019-12-17 12:28:09,441] jepsen worker 53 - jepsen.util 863 :invoke :write [17 46] INFO [2019-12-17 12:28:09,441] jepsen worker 55 - jepsen.util 865 :info :write [18 28] :RequestCanceledException INFO [2019-12-17 12:28:09,443] jepsen worker 44 - jepsen.util 494 :info :write [14 17] :RequestCanceledException INFO [2019-12-17 12:28:09,450] jepsen worker 13 - jepsen.util 823 :invoke :write [4 31] INFO [2019-12-17 12:28:09,452] jepsen worker 84 - jepsen.util 444 :invoke :write [28 24] INFO [2019-12-17 12:28:09,457] jepsen worker 41 - jepsen.util 401 :info :cas [13 [26 3]] :RequestCanceledException INFO [2019-12-17 12:28:09,469] jepsen worker 2 - jepsen.util 632 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:28:09,476] jepsen worker 2 - jepsen.util 632 :invoke :read [0 nil] INFO [2019-12-17 12:28:09,501] jepsen worker 28 - jepsen.util 298 :info :write [9 32] :RequestCanceledException INFO [2019-12-17 12:28:09,517] jepsen worker 27 - jepsen.util 477 :ok :read [9 43] INFO [2019-12-17 12:28:09,518] jepsen worker 27 - jepsen.util 477 :invoke :read [9 nil] INFO [2019-12-17 12:28:09,518] jepsen worker 48 - jepsen.util 498 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:28:09,520] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3462,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:28:09,520] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3462,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:28:09,520] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3463,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:28:09,520] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3463,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:28:09,530] jepsen worker 48 - jepsen.util 498 :invoke :cas [16 [38 19]] INFO [2019-12-17 12:28:09,532] jepsen worker 63 - jepsen.util 423 :info :write [21 25] :RequestCanceledException INFO [2019-12-17 12:28:09,534] jepsen worker 80 - jepsen.util 890 :ok :read [26 47] INFO [2019-12-17 12:28:09,540] jepsen worker 17 - jepsen.util 557 :info :write [5 49] :RequestCanceledException INFO [2019-12-17 12:28:09,544] jepsen worker 9 - jepsen.util 729 :invoke :cas [3 [19 23]] INFO [2019-12-17 12:28:09,547] jepsen worker 47 - jepsen.util 947 :info :write [15 47] :RequestCanceledException INFO [2019-12-17 12:28:09,553] jepsen worker 71 - jepsen.util 611 :info :write [23 22] :RequestCanceledException INFO [2019-12-17 12:28:09,558] jepsen worker 30 - jepsen.util 660 :invoke :cas [10 [3 26]] INFO [2019-12-17 12:28:09,559] jepsen worker 80 - jepsen.util 890 :invoke :cas [26 [8 26]] INFO [2019-12-17 12:28:09,564] jepsen worker 75 - jepsen.util 705 :invoke :cas [25 [32 25]] INFO [2019-12-17 12:28:09,566] jepsen worker 85 - jepsen.util 625 :info :write [28 26] :RequestCanceledException INFO [2019-12-17 12:28:09,643] jepsen worker 55 - jepsen.util 955 :invoke :cas [18 [42 4]] INFO [2019-12-17 12:28:09,653] jepsen worker 79 - jepsen.util 619 :ok :read [26 47] INFO [2019-12-17 12:28:09,654] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3446,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:28:09,659] jepsen worker 79 - jepsen.util 619 :invoke :read [26 nil] INFO [2019-12-17 12:28:09,673] jepsen worker 28 - jepsen.util 388 :invoke :write [9 38] INFO [2019-12-17 12:28:09,682] jepsen worker 67 - jepsen.util 1057 :invoke :write [22 32] INFO [2019-12-17 12:28:09,705] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3455,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:28:09,723] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3455,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:28:09,730] jepsen worker 50 - jepsen.util 680 :info :cas [16 [2 17]] :RequestCanceledException INFO [2019-12-17 12:28:09,730] jepsen worker 44 - jepsen.util 584 :invoke :cas [14 [44 46]] INFO [2019-12-17 12:28:09,738] jepsen worker 71 - jepsen.util 701 :invoke :write [23 22] INFO [2019-12-17 12:28:09,739] jepsen worker 41 - jepsen.util 491 :invoke :write [13 10] INFO [2019-12-17 12:28:09,740] jepsen worker 63 - jepsen.util 513 :invoke :cas [21 [21 29]] INFO [2019-12-17 12:28:09,740] jepsen worker 47 - jepsen.util 1037 :invoke :read [15 nil] INFO [2019-12-17 12:28:09,753] jepsen worker 17 - jepsen.util 647 :invoke :cas [5 [36 33]] INFO [2019-12-17 12:28:09,788] jepsen worker 85 - jepsen.util 715 :invoke :write [28 30] INFO [2019-12-17 12:28:09,810] jepsen worker 11 - jepsen.util 821 :ok :write [3 1] INFO [2019-12-17 12:28:09,817] jepsen worker 11 - jepsen.util 821 :invoke :write [3 8] INFO [2019-12-17 12:28:09,822] jepsen worker 46 - jepsen.util 676 :info :write [15 1] :RequestCanceledException INFO [2019-12-17 12:28:09,825] jepsen worker 50 - jepsen.util 770 :invoke :write [16 34] INFO [2019-12-17 12:28:09,867] jepsen worker 54 - jepsen.util 774 :ok :read [18 47] INFO [2019-12-17 12:28:09,868] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3494,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:28:09,869] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3494,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:28:09,869] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3498,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:28:09,869] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3498,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:28:09,870] jepsen worker 54 - jepsen.util 774 :invoke :cas [18 [24 37]] INFO [2019-12-17 12:28:09,875] jepsen worker 46 - jepsen.util 766 :invoke :read [15 nil] INFO [2019-12-17 12:28:09,905] jepsen worker 74 - jepsen.util 524 :info :cas [24 [36 47]] :RequestCanceledException INFO [2019-12-17 12:28:09,918] jepsen worker 6 - jepsen.util 636 :info :write [2 37] :RequestCanceledException INFO [2019-12-17 12:28:09,925] jepsen worker 4 - jepsen.util 634 :fail :cas [1 [26 48]] :ValueNotSwapFrom INFO [2019-12-17 12:28:09,927] jepsen worker 4 - jepsen.util 634 :invoke :read [1 nil] INFO [2019-12-17 12:28:09,933] jepsen worker 74 - jepsen.util 614 :invoke :cas [24 [9 34]] INFO [2019-12-17 12:28:09,935] jepsen worker 6 - jepsen.util 726 :invoke :read [2 nil] INFO [2019-12-17 12:28:09,935] jepsen worker 42 - jepsen.util 942 :info :write [14 11] :RequestCanceledException INFO [2019-12-17 12:28:09,938] jepsen worker 69 - jepsen.util 789 :fail :cas [23 [3 10]] :ValueNotSwapFrom INFO [2019-12-17 12:28:09,939] jepsen worker 42 - jepsen.util 1032 :invoke :cas [14 [11 12]] INFO [2019-12-17 12:28:09,941] jepsen worker 8 - jepsen.util 728 :fail :cas [2 [12 25]] :ValueNotSwapFrom INFO [2019-12-17 12:28:09,943] jepsen worker 8 - jepsen.util 728 :invoke :read [2 nil] INFO [2019-12-17 12:28:09,944] jepsen worker 69 - jepsen.util 789 :invoke :read [23 nil] INFO [2019-12-17 12:28:09,985] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:28:09,986] jepsen nemesis - couchbase.util Rebalance check count: 20 INFO [2019-12-17 12:28:09,986] jepsen nemesis - couchbase.util Rebalance stuck count: 1 INFO [2019-12-17 12:28:09,989] jepsen worker 56 - jepsen.util 686 :fail :cas [18 [20 32]] :ValueNotSwapFrom INFO [2019-12-17 12:28:09,989] jepsen worker 56 - jepsen.util 686 :invoke :write [18 13] INFO [2019-12-17 12:28:09,993] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3518,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:28:09,993] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3518,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:28:09,993] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3521,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:28:09,993] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3521,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:28:09,996] jepsen worker 87 - jepsen.util 627 :info :write [29 10] :RequestCanceledException INFO [2019-12-17 12:28:09,996] jepsen worker 5 - jepsen.util 455 :info :write [1 49] :RequestCanceledException INFO [2019-12-17 12:28:09,996] jepsen worker 25 - jepsen.util 835 :info :write [8 6] :RequestCanceledException INFO [2019-12-17 12:28:10,000] jepsen worker 21 - jepsen.util 831 :ok :read [7 20] INFO [2019-12-17 12:28:10,001] jepsen worker 25 - jepsen.util 925 :invoke :read [8 nil] INFO [2019-12-17 12:28:10,001] jepsen worker 21 - jepsen.util 831 :invoke :cas [7 [4 22]] INFO [2019-12-17 12:28:10,002] jepsen worker 5 - jepsen.util 545 :invoke :cas [1 [31 15]] INFO [2019-12-17 12:28:10,002] jepsen worker 87 - jepsen.util 717 :invoke :cas [29 [9 40]] INFO [2019-12-17 12:28:10,002] jepsen worker 76 - jepsen.util 706 :fail :cas [25 [6 46]] :ValueNotSwapFrom INFO [2019-12-17 12:28:10,003] jepsen worker 76 - jepsen.util 706 :invoke :write [25 37] INFO [2019-12-17 12:28:10,005] jepsen worker 33 - jepsen.util 753 :info :write [11 19] :RequestCanceledException INFO [2019-12-17 12:28:10,009] jepsen worker 33 - jepsen.util 843 :invoke :write [11 23] INFO [2019-12-17 12:28:10,048] jepsen worker 27 - jepsen.util 477 :ok :read [9 43] INFO [2019-12-17 12:28:10,050] jepsen worker 27 - jepsen.util 477 :invoke :cas [9 [20 21]] INFO [2019-12-17 12:28:10,053] jepsen worker 32 - jepsen.util 752 :fail :cas [10 [24 29]] :ValueNotSwapFrom INFO [2019-12-17 12:28:10,054] jepsen worker 32 - jepsen.util 752 :invoke :read [10 nil] INFO [2019-12-17 12:28:10,065] jepsen worker 43 - jepsen.util 943 :ok :read [14 27] INFO [2019-12-17 12:28:10,068] jepsen worker 43 - jepsen.util 943 :invoke :read [14 nil] INFO [2019-12-17 12:28:10,070] jepsen worker 88 - jepsen.util 628 :ok :read [29 5] INFO [2019-12-17 12:28:10,070] jepsen worker 88 - jepsen.util 628 :invoke :write [29 1] INFO [2019-12-17 12:28:10,081] jepsen worker 34 - jepsen.util 754 :ok :read [11 7] INFO [2019-12-17 12:28:10,081] jepsen worker 23 - jepsen.util 563 :ok :read [7 20] INFO [2019-12-17 12:28:10,082] jepsen worker 34 - jepsen.util 754 :invoke :write [11 25] INFO [2019-12-17 12:28:10,083] jepsen worker 68 - jepsen.util 428 :fail :cas [22 [29 30]] :ValueNotSwapFrom INFO [2019-12-17 12:28:10,084] jepsen worker 68 - jepsen.util 428 :invoke :read [22 nil] INFO [2019-12-17 12:28:10,086] jepsen worker 64 - jepsen.util 514 :fail :cas [21 [19 45]] :ValueNotSwapFrom INFO [2019-12-17 12:28:10,086] jepsen worker 23 - jepsen.util 563 :invoke :cas [7 [33 40]] INFO [2019-12-17 12:28:10,086] jepsen worker 64 - jepsen.util 514 :invoke :cas [21 [0 48]] INFO [2019-12-17 12:28:10,087] jepsen worker 30 - jepsen.util 660 :fail :cas [10 [3 26]] :ValueNotSwapFrom INFO [2019-12-17 12:28:10,087] jepsen worker 30 - jepsen.util 660 :invoke :write [10 28] INFO [2019-12-17 12:28:10,087] jepsen worker 29 - jepsen.util 659 :ok :read [9 43] INFO [2019-12-17 12:28:10,088] jepsen worker 29 - jepsen.util 659 :invoke :read [9 nil] INFO [2019-12-17 12:28:10,094] jepsen worker 10 - jepsen.util 730 :ok :read [3 1] INFO [2019-12-17 12:28:10,095] jepsen worker 10 - jepsen.util 730 :invoke :write [3 37] INFO [2019-12-17 12:28:10,096] jepsen worker 36 - jepsen.util 846 :fail :cas [12 [6 4]] :ValueNotSwapFrom INFO [2019-12-17 12:28:10,098] jepsen worker 31 - jepsen.util 841 :fail :cas [10 [14 29]] :ValueNotSwapFrom INFO [2019-12-17 12:28:10,098] jepsen worker 36 - jepsen.util 846 :invoke :cas [12 [48 49]] INFO [2019-12-17 12:28:10,099] jepsen worker 31 - jepsen.util 841 :invoke :read [10 nil] INFO [2019-12-17 12:28:10,103] jepsen worker 65 - jepsen.util 515 :fail :cas [21 [42 3]] :ValueNotSwapFrom INFO [2019-12-17 12:28:10,104] jepsen worker 65 - jepsen.util 515 :invoke :cas [21 [44 48]] INFO [2019-12-17 12:28:10,105] jepsen worker 72 - jepsen.util 792 :fail :cas [24 [6 49]] :ValueNotSwapFrom INFO [2019-12-17 12:28:10,105] jepsen worker 72 - jepsen.util 792 :invoke :read [24 nil] INFO [2019-12-17 12:28:10,107] jepsen worker 77 - jepsen.util 617 :ok :read [25 2] INFO [2019-12-17 12:28:10,107] jepsen worker 77 - jepsen.util 617 :invoke :cas [25 [3 29]] INFO [2019-12-17 12:28:10,117] jepsen worker 7 - jepsen.util 907 :fail :cas [2 [48 11]] :ValueNotSwapFrom INFO [2019-12-17 12:28:10,117] jepsen worker 7 - jepsen.util 907 :invoke :cas [2 [38 35]] INFO [2019-12-17 12:28:10,119] jepsen worker 82 - jepsen.util 892 :info :cas [27 [24 35]] :RequestCanceledException INFO [2019-12-17 12:28:10,123] jepsen worker 82 - jepsen.util 982 :invoke :cas [27 [7 19]] INFO [2019-12-17 12:28:10,128] jepsen worker 63 - jepsen.util 513 :fail :cas [21 [21 29]] :ValueNotSwapFrom INFO [2019-12-17 12:28:10,128] jepsen worker 63 - jepsen.util 513 :invoke :write [21 4] INFO [2019-12-17 12:28:10,133] jepsen worker 60 - jepsen.util 780 :info :write [20 40] :RequestCanceledException INFO [2019-12-17 12:28:10,136] jepsen worker 60 - jepsen.util 870 :invoke :write [20 15] INFO [2019-12-17 12:28:10,192] jepsen worker 46 - jepsen.util 766 :ok :read [15 35] INFO [2019-12-17 12:28:10,193] jepsen worker 46 - jepsen.util 766 :invoke :write [15 2] INFO [2019-12-17 12:28:10,196] jepsen worker 18 - jepsen.util 738 :ok :read [6 26] INFO [2019-12-17 12:28:10,196] jepsen worker 18 - jepsen.util 738 :invoke :write [6 48] INFO [2019-12-17 12:28:10,197] jepsen worker 15 - jepsen.util 645 :info :write [5 39] :RequestCanceledException INFO [2019-12-17 12:28:10,199] jepsen worker 15 - jepsen.util 735 :invoke :read [5 nil] INFO [2019-12-17 12:28:10,211] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3560,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:28:10,211] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3560,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:28:10,211] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3562,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:28:10,211] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3562,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:28:10,221] jepsen worker 16 - jepsen.util 736 :info :cas [5 [8 16]] :RequestCanceledException INFO [2019-12-17 12:28:10,223] jepsen worker 16 - jepsen.util 826 :invoke :read [5 nil] INFO [2019-12-17 12:28:10,234] jepsen worker 40 - jepsen.util 940 :info :cas [13 [38 28]] :RequestCanceledException INFO [2019-12-17 12:28:10,237] jepsen worker 40 - jepsen.util 1030 :invoke :write [13 23] INFO [2019-12-17 12:28:10,241] jepsen worker 26 - jepsen.util 836 :ok :read [8 14] INFO [2019-12-17 12:28:10,241] jepsen worker 26 - jepsen.util 836 :invoke :cas [8 [44 20]] INFO [2019-12-17 12:28:10,256] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3551,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:28:10,257] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3551,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:28:10,257] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3553,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:28:10,257] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3553,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:28:10,265] jepsen worker 32 - jepsen.util 752 :ok :read [10 38] INFO [2019-12-17 12:28:10,266] jepsen worker 32 - jepsen.util 752 :invoke :write [10 22] INFO [2019-12-17 12:28:10,275] jepsen worker 42 - jepsen.util 1032 :fail :cas [14 [11 12]] :ValueNotSwapFrom INFO [2019-12-17 12:28:10,275] jepsen worker 42 - jepsen.util 1032 :invoke :cas [14 [2 5]] INFO [2019-12-17 12:28:10,293] jepsen worker 55 - jepsen.util 955 :fail :cas [18 [42 4]] :ValueNotSwapFrom INFO [2019-12-17 12:28:10,293] jepsen worker 55 - jepsen.util 955 :invoke :cas [18 [15 4]] INFO [2019-12-17 12:28:10,319] jepsen worker 66 - jepsen.util 516 :info :write [22 36] :RequestCanceledException INFO [2019-12-17 12:28:10,321] jepsen worker 66 - jepsen.util 606 :invoke :cas [22 [18 7]] INFO [2019-12-17 12:28:10,329] jepsen worker 22 - jepsen.util 652 :ok :read [7 20] INFO [2019-12-17 12:28:10,330] jepsen worker 22 - jepsen.util 652 :invoke :read [7 nil] INFO [2019-12-17 12:28:10,338] jepsen worker 21 - jepsen.util 831 :fail :cas [7 [4 22]] :ValueNotSwapFrom INFO [2019-12-17 12:28:10,338] jepsen worker 21 - jepsen.util 831 :invoke :cas [7 [40 49]] INFO [2019-12-17 12:28:10,345] jepsen worker 2 - jepsen.util 632 :ok :read [0 4] INFO [2019-12-17 12:28:10,345] jepsen worker 2 - jepsen.util 632 :invoke :read [0 nil] INFO [2019-12-17 12:28:10,350] jepsen worker 61 - jepsen.util 781 :info :write [20 18] :RequestCanceledException INFO [2019-12-17 12:28:10,352] jepsen worker 61 - jepsen.util 871 :invoke :read [20 nil] INFO [2019-12-17 12:28:10,358] jepsen worker 31 - jepsen.util 841 :ok :read [10 38] INFO [2019-12-17 12:28:10,358] jepsen worker 31 - jepsen.util 841 :invoke :cas [10 [1 2]] INFO [2019-12-17 12:28:10,362] jepsen worker 9 - jepsen.util 729 :fail :cas [3 [19 23]] :ValueNotSwapFrom INFO [2019-12-17 12:28:10,362] jepsen worker 9 - jepsen.util 729 :invoke :read [3 nil] INFO [2019-12-17 12:28:10,371] jepsen worker 62 - jepsen.util 872 :fail :cas [20 [22 15]] :ValueNotSwapFrom INFO [2019-12-17 12:28:10,371] jepsen worker 48 - jepsen.util 498 :fail :cas [16 [38 19]] :ValueNotSwapFrom INFO [2019-12-17 12:28:10,371] jepsen worker 48 - jepsen.util 498 :invoke :read [16 nil] INFO [2019-12-17 12:28:10,371] jepsen worker 62 - jepsen.util 872 :invoke :write [20 23] INFO [2019-12-17 12:28:10,377] jepsen worker 27 - jepsen.util 477 :fail :cas [9 [20 21]] :ValueNotSwapFrom INFO [2019-12-17 12:28:10,378] jepsen worker 27 - jepsen.util 477 :invoke :write [9 42] INFO [2019-12-17 12:28:10,391] jepsen worker 79 - jepsen.util 619 :ok :read [26 47] INFO [2019-12-17 12:28:10,391] jepsen worker 79 - jepsen.util 619 :invoke :write [26 43] INFO [2019-12-17 12:28:10,398] jepsen worker 6 - jepsen.util 726 :ok :read [2 15] INFO [2019-12-17 12:28:10,399] jepsen worker 6 - jepsen.util 726 :invoke :cas [2 [21 44]] INFO [2019-12-17 12:28:10,429] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3585,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:28:10,429] jepsen worker 3 - jepsen.util 453 :fail :cas [1 [25 25]] :ValueNotSwapFrom INFO [2019-12-17 12:28:10,429] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3585,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:28:10,429] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3588,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:28:10,429] jepsen worker 3 - jepsen.util 453 :invoke :write [1 19] INFO [2019-12-17 12:28:10,429] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3588,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:28:10,456] jepsen worker 64 - jepsen.util 514 :fail :cas [21 [0 48]] :ValueNotSwapFrom INFO [2019-12-17 12:28:10,456] jepsen worker 64 - jepsen.util 514 :invoke :read [21 nil] INFO [2019-12-17 12:28:10,474] jepsen worker 77 - jepsen.util 617 :fail :cas [25 [3 29]] :ValueNotSwapFrom INFO [2019-12-17 12:28:10,474] jepsen worker 77 - jepsen.util 617 :invoke :cas [25 [2 3]] INFO [2019-12-17 12:28:10,491] jepsen worker 78 - jepsen.util 798 :fail :cas [26 [24 37]] :ValueNotSwapFrom INFO [2019-12-17 12:28:10,491] jepsen worker 78 - jepsen.util 798 :invoke :write [26 4] INFO [2019-12-17 12:28:10,494] jepsen worker 9 - jepsen.util 729 :ok :read [3 1] INFO [2019-12-17 12:28:10,494] jepsen worker 9 - jepsen.util 729 :invoke :read [3 nil] INFO [2019-12-17 12:28:10,530] jepsen worker 35 - jepsen.util 665 :ok :read [11 7] INFO [2019-12-17 12:28:10,530] jepsen worker 35 - jepsen.util 665 :invoke :cas [11 [41 30]] INFO [2019-12-17 12:28:10,534] jepsen worker 12 - jepsen.util 732 :ok :write [4 28] INFO [2019-12-17 12:28:10,534] jepsen worker 12 - jepsen.util 732 :invoke :write [4 21] INFO [2019-12-17 12:28:10,540] jepsen worker 37 - jepsen.util 757 :ok :write [12 45] INFO [2019-12-17 12:28:10,541] jepsen worker 37 - jepsen.util 757 :invoke :cas [12 [11 21]] INFO [2019-12-17 12:28:10,551] jepsen worker 9 - jepsen.util 729 :ok :read [3 1] INFO [2019-12-17 12:28:10,557] jepsen worker 9 - jepsen.util 729 :invoke :read [3 nil] INFO [2019-12-17 12:28:10,563] jepsen worker 55 - jepsen.util 955 :fail :cas [18 [15 4]] :ValueNotSwapFrom INFO [2019-12-17 12:28:10,563] jepsen worker 55 - jepsen.util 955 :invoke :write [18 2] INFO [2019-12-17 12:28:10,587] jepsen worker 1 - jepsen.util 631 :ok :write [0 5] INFO [2019-12-17 12:28:10,587] jepsen worker 1 - jepsen.util 631 :invoke :read [0 nil] INFO [2019-12-17 12:28:10,589] jepsen worker 23 - jepsen.util 563 :fail :cas [7 [33 40]] :ValueNotSwapFrom INFO [2019-12-17 12:28:10,590] jepsen worker 23 - jepsen.util 563 :invoke :write [7 41] INFO [2019-12-17 12:28:10,614] jepsen worker 9 - jepsen.util 729 :ok :read [3 1] INFO [2019-12-17 12:28:10,614] jepsen worker 9 - jepsen.util 729 :invoke :write [3 14] INFO [2019-12-17 12:28:10,639] jepsen worker 75 - jepsen.util 705 :fail :cas [25 [32 25]] :ValueNotSwapFrom INFO [2019-12-17 12:28:10,639] jepsen worker 75 - jepsen.util 705 :invoke :write [25 41] INFO [2019-12-17 12:28:10,672] jepsen worker 89 - jepsen.util 899 :ok :write [29 48] INFO [2019-12-17 12:28:10,673] jepsen worker 89 - jepsen.util 899 :invoke :read [29 nil] INFO [2019-12-17 12:28:10,673] jepsen worker 14 - jepsen.util 734 :ok :write [4 49] INFO [2019-12-17 12:28:10,673] jepsen worker 14 - jepsen.util 734 :invoke :read [4 nil] INFO [2019-12-17 12:28:10,675] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3604,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:28:10,675] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3604,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:28:10,695] jepsen worker 42 - jepsen.util 1032 :fail :cas [14 [2 5]] :ValueNotSwapFrom INFO [2019-12-17 12:28:10,695] jepsen worker 42 - jepsen.util 1032 :invoke :read [14 nil] INFO [2019-12-17 12:28:10,698] jepsen worker 84 - jepsen.util 444 :ok :write [28 24] INFO [2019-12-17 12:28:10,698] jepsen worker 84 - jepsen.util 444 :invoke :write [28 28] INFO [2019-12-17 12:28:10,718] jepsen worker 80 - jepsen.util 890 :fail :cas [26 [8 26]] :ValueNotSwapFrom INFO [2019-12-17 12:28:10,718] jepsen worker 80 - jepsen.util 890 :invoke :cas [26 [48 22]] INFO [2019-12-17 12:28:10,720] jepsen worker 86 - jepsen.util 536 :info :write [28 47] :RequestCanceledException INFO [2019-12-17 12:28:10,721] jepsen worker 86 - jepsen.util 626 :invoke :cas [28 [25 46]] INFO [2019-12-17 12:28:10,730] jepsen worker 38 - jepsen.util 668 :fail :cas [12 [11 23]] :ValueNotSwapFrom INFO [2019-12-17 12:28:10,730] jepsen worker 38 - jepsen.util 668 :invoke :write [12 22] INFO [2019-12-17 12:28:10,746] jepsen worker 61 - jepsen.util 871 :ok :read [20 7] INFO [2019-12-17 12:28:10,747] jepsen worker 61 - jepsen.util 871 :invoke :cas [20 [12 39]] INFO [2019-12-17 12:28:10,750] jepsen worker 83 - jepsen.util 893 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:28:10,751] jepsen worker 83 - jepsen.util 893 :invoke :write [27 46] INFO [2019-12-17 12:28:10,778] jepsen worker 8 - jepsen.util 728 :ok :read [2 15] INFO [2019-12-17 12:28:10,778] jepsen worker 8 - jepsen.util 728 :invoke :write [2 41] INFO [2019-12-17 12:28:10,782] jepsen worker 81 - jepsen.util 801 :info :write [27 12] :RequestCanceledException INFO [2019-12-17 12:28:10,785] jepsen worker 81 - jepsen.util 891 :invoke :read [27 nil] INFO [2019-12-17 12:28:10,788] jepsen worker 80 - jepsen.util 890 :fail :cas [26 [48 22]] :ValueNotSwapFrom INFO [2019-12-17 12:28:10,788] jepsen worker 22 - jepsen.util 652 :ok :read [7 20] INFO [2019-12-17 12:28:10,788] jepsen worker 80 - jepsen.util 890 :invoke :read [26 nil] INFO [2019-12-17 12:28:10,789] jepsen worker 22 - jepsen.util 652 :invoke :read [7 nil] INFO [2019-12-17 12:28:10,792] jepsen worker 85 - jepsen.util 715 :ok :write [28 30] INFO [2019-12-17 12:28:10,792] jepsen worker 85 - jepsen.util 715 :invoke :cas [28 [1 34]] INFO [2019-12-17 12:28:10,794] jepsen worker 54 - jepsen.util 774 :fail :cas [18 [24 37]] :ValueNotSwapFrom INFO [2019-12-17 12:28:10,794] jepsen worker 54 - jepsen.util 774 :invoke :write [18 20] INFO [2019-12-17 12:28:10,819] jepsen worker 39 - jepsen.util 759 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:28:10,819] jepsen worker 39 - jepsen.util 759 :invoke :cas [13 [16 10]] INFO [2019-12-17 12:28:10,829] jepsen worker 53 - jepsen.util 863 :ok :write [17 46] INFO [2019-12-17 12:28:10,829] jepsen worker 53 - jepsen.util 863 :invoke :cas [17 [42 44]] INFO [2019-12-17 12:28:10,848] jepsen worker 89 - jepsen.util 899 :ok :read [29 48] INFO [2019-12-17 12:28:10,848] jepsen worker 89 - jepsen.util 899 :invoke :read [29 nil] INFO [2019-12-17 12:28:10,852] jepsen worker 68 - jepsen.util 428 :ok :read [22 0] INFO [2019-12-17 12:28:10,852] jepsen worker 68 - jepsen.util 428 :invoke :read [22 nil] INFO [2019-12-17 12:28:10,859] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3633,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:28:10,859] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3633,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:28:10,859] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3634,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:28:10,859] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3634,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:28:10,900] jepsen worker 24 - jepsen.util 654 :ok :read [8 14] INFO [2019-12-17 12:28:10,900] jepsen worker 24 - jepsen.util 654 :invoke :read [8 nil] INFO [2019-12-17 12:28:10,911] jepsen worker 43 - jepsen.util 943 :ok :read [14 27] INFO [2019-12-17 12:28:10,911] jepsen worker 43 - jepsen.util 943 :invoke :read [14 nil] INFO [2019-12-17 12:28:10,925] jepsen worker 21 - jepsen.util 831 :fail :cas [7 [40 49]] :ValueNotSwapFrom INFO [2019-12-17 12:28:10,925] jepsen worker 21 - jepsen.util 831 :invoke :write [7 39] INFO [2019-12-17 12:28:10,969] jepsen worker 45 - jepsen.util 1035 :ok :write [15 31] INFO [2019-12-17 12:28:10,969] jepsen worker 45 - jepsen.util 1035 :invoke :cas [15 [13 26]] INFO [2019-12-17 12:28:10,971] jepsen worker 20 - jepsen.util 740 :ok :write [6 32] INFO [2019-12-17 12:28:10,971] jepsen worker 20 - jepsen.util 740 :invoke :read [6 nil] INFO [2019-12-17 12:28:10,972] jepsen worker 73 - jepsen.util 793 :ok :write [24 42] INFO [2019-12-17 12:28:10,972] jepsen worker 6 - jepsen.util 726 :fail :cas [2 [21 44]] :ValueNotSwapFrom INFO [2019-12-17 12:28:10,972] jepsen worker 73 - jepsen.util 793 :invoke :read [24 nil] INFO [2019-12-17 12:28:10,972] jepsen worker 6 - jepsen.util 726 :invoke :cas [2 [17 41]] INFO [2019-12-17 12:28:10,979] jepsen worker 36 - jepsen.util 846 :fail :cas [12 [48 49]] :ValueNotSwapFrom INFO [2019-12-17 12:28:10,979] jepsen worker 36 - jepsen.util 846 :invoke :write [12 3] INFO [2019-12-17 12:28:10,986] jepsen worker 79 - jepsen.util 619 :ok :write [26 43] INFO [2019-12-17 12:28:10,986] jepsen worker 79 - jepsen.util 619 :invoke :write [26 12] INFO [2019-12-17 12:28:10,986] jepsen worker 80 - jepsen.util 890 :ok :read [26 43] INFO [2019-12-17 12:28:10,987] jepsen worker 80 - jepsen.util 890 :invoke :read [26 nil] INFO [2019-12-17 12:28:10,993] jepsen worker 7 - jepsen.util 907 :fail :cas [2 [38 35]] :ValueNotSwapFrom INFO [2019-12-17 12:28:10,994] jepsen worker 7 - jepsen.util 907 :invoke :write [2 7] INFO [2019-12-17 12:28:11,008] jepsen worker 74 - jepsen.util 614 :fail :cas [24 [9 34]] :ValueNotSwapFrom INFO [2019-12-17 12:28:11,008] jepsen worker 74 - jepsen.util 614 :invoke :cas [24 [13 35]] INFO [2019-12-17 12:28:11,011] jepsen worker 20 - jepsen.util 740 :ok :read [6 32] INFO [2019-12-17 12:28:11,012] jepsen worker 20 - jepsen.util 740 :invoke :cas [6 [45 17]] INFO [2019-12-17 12:28:11,014] jepsen worker 73 - jepsen.util 793 :ok :read [24 42] INFO [2019-12-17 12:28:11,014] jepsen worker 73 - jepsen.util 793 :invoke :write [24 11] INFO [2019-12-17 12:28:11,052] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:28:11,053] jepsen nemesis - couchbase.util Rebalance check count: 21 INFO [2019-12-17 12:28:11,053] jepsen nemesis - couchbase.util Rebalance stuck count: 2 INFO [2019-12-17 12:28:11,054] jepsen worker 4 - jepsen.util 634 :ok :read [1 49] INFO [2019-12-17 12:28:11,054] jepsen worker 4 - jepsen.util 634 :invoke :cas [1 [1 22]] INFO [2019-12-17 12:28:11,076] jepsen worker 47 - jepsen.util 1037 :ok :read [15 31] INFO [2019-12-17 12:28:11,076] jepsen worker 47 - jepsen.util 1037 :invoke :read [15 nil] INFO [2019-12-17 12:28:11,080] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3643,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:28:11,080] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3643,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:28:11,081] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3647,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:28:11,081] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3647,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:28:11,087] jepsen worker 74 - jepsen.util 614 :fail :cas [24 [13 35]] :ValueNotSwapFrom INFO [2019-12-17 12:28:11,087] jepsen worker 74 - jepsen.util 614 :invoke :write [24 32] INFO [2019-12-17 12:28:11,096] jepsen worker 61 - jepsen.util 871 :fail :cas [20 [12 39]] :ValueNotSwapFrom INFO [2019-12-17 12:28:11,096] jepsen worker 61 - jepsen.util 871 :invoke :cas [20 [5 42]] INFO [2019-12-17 12:28:11,099] jepsen worker 20 - jepsen.util 740 :fail :cas [6 [45 17]] :ValueNotSwapFrom INFO [2019-12-17 12:28:11,099] jepsen worker 20 - jepsen.util 740 :invoke :write [6 18] INFO [2019-12-17 12:28:11,100] jepsen worker 52 - jepsen.util 772 :fail :read [17 nil] :RequestTimeoutException INFO [2019-12-17 12:28:11,100] jepsen worker 70 - jepsen.util 880 :info :cas [23 [3 36]] :RequestCanceledException INFO [2019-12-17 12:28:11,101] jepsen worker 52 - jepsen.util 772 :invoke :read [17 nil] INFO [2019-12-17 12:28:11,102] jepsen worker 70 - jepsen.util 970 :invoke :read [23 nil] INFO [2019-12-17 12:28:11,106] jepsen worker 44 - jepsen.util 584 :fail :cas [14 [44 46]] :ValueNotSwapFrom INFO [2019-12-17 12:28:11,106] jepsen worker 44 - jepsen.util 584 :invoke :cas [14 [32 46]] INFO [2019-12-17 12:28:11,114] jepsen worker 80 - jepsen.util 890 :ok :read [26 43] INFO [2019-12-17 12:28:11,114] jepsen worker 80 - jepsen.util 890 :invoke :write [26 43] INFO [2019-12-17 12:28:11,121] jepsen worker 59 - jepsen.util 779 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:28:11,121] jepsen worker 59 - jepsen.util 779 :invoke :read [19 nil] INFO [2019-12-17 12:28:11,136] jepsen worker 45 - jepsen.util 1035 :fail :cas [15 [13 26]] :ValueNotSwapFrom INFO [2019-12-17 12:28:11,137] jepsen worker 45 - jepsen.util 1035 :invoke :cas [15 [13 11]] INFO [2019-12-17 12:28:11,157] jepsen worker 47 - jepsen.util 1037 :ok :read [15 31] INFO [2019-12-17 12:28:11,157] jepsen worker 47 - jepsen.util 1037 :invoke :cas [15 [28 40]] INFO [2019-12-17 12:28:11,175] jepsen worker 6 - jepsen.util 726 :fail :cas [2 [17 41]] :ValueNotSwapFrom INFO [2019-12-17 12:28:11,175] jepsen worker 6 - jepsen.util 726 :invoke :read [2 nil] INFO [2019-12-17 12:28:11,177] jepsen worker 45 - jepsen.util 1035 :fail :cas [15 [13 11]] :ValueNotSwapFrom INFO [2019-12-17 12:28:11,177] jepsen worker 45 - jepsen.util 1035 :invoke :read [15 nil] INFO [2019-12-17 12:28:11,183] jepsen worker 57 - jepsen.util 777 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:28:11,184] jepsen worker 57 - jepsen.util 777 :invoke :cas [19 [4 49]] INFO [2019-12-17 12:28:11,185] jepsen worker 47 - jepsen.util 1037 :fail :cas [15 [28 40]] :ValueNotSwapFrom INFO [2019-12-17 12:28:11,185] jepsen worker 47 - jepsen.util 1037 :invoke :read [15 nil] INFO [2019-12-17 12:28:11,197] jepsen worker 69 - jepsen.util 789 :ok :read [23 29] INFO [2019-12-17 12:28:11,197] jepsen worker 69 - jepsen.util 789 :invoke :read [23 nil] INFO [2019-12-17 12:28:11,204] jepsen worker 45 - jepsen.util 1035 :ok :read [15 31] INFO [2019-12-17 12:28:11,205] jepsen worker 22 - jepsen.util 652 :ok :read [7 20] INFO [2019-12-17 12:28:11,208] jepsen worker 45 - jepsen.util 1035 :invoke :read [15 nil] INFO [2019-12-17 12:28:11,208] jepsen worker 22 - jepsen.util 652 :invoke :write [7 16] INFO [2019-12-17 12:28:11,214] jepsen worker 71 - jepsen.util 701 :ok :write [23 22] INFO [2019-12-17 12:28:11,214] jepsen worker 71 - jepsen.util 701 :invoke :read [23 nil] INFO [2019-12-17 12:28:11,215] jepsen worker 47 - jepsen.util 1037 :ok :read [15 31] INFO [2019-12-17 12:28:11,215] jepsen worker 47 - jepsen.util 1037 :invoke :write [15 1] INFO [2019-12-17 12:28:11,220] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3672,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:28:11,220] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3672,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:28:11,220] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3673,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:28:11,220] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3673,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:28:11,249] jepsen worker 49 - jepsen.util 499 :info :cas [16 [2 21]] :RequestCanceledException INFO [2019-12-17 12:28:11,252] jepsen worker 49 - jepsen.util 589 :invoke :write [16 19] INFO [2019-12-17 12:28:11,254] jepsen worker 61 - jepsen.util 871 :fail :cas [20 [5 42]] :ValueNotSwapFrom INFO [2019-12-17 12:28:11,254] jepsen worker 61 - jepsen.util 871 :invoke :write [20 27] INFO [2019-12-17 12:28:11,255] jepsen worker 87 - jepsen.util 717 :fail :cas [29 [9 40]] :ValueNotSwapFrom INFO [2019-12-17 12:28:11,255] jepsen worker 87 - jepsen.util 717 :invoke :cas [29 [17 27]] INFO [2019-12-17 12:28:11,265] jepsen worker 25 - jepsen.util 925 :ok :read [8 14] INFO [2019-12-17 12:28:11,265] jepsen worker 25 - jepsen.util 925 :invoke :cas [8 [45 48]] INFO [2019-12-17 12:28:11,280] jepsen worker 89 - jepsen.util 899 :ok :read [29 48] INFO [2019-12-17 12:28:11,280] jepsen worker 89 - jepsen.util 899 :invoke :write [29 49] INFO [2019-12-17 12:28:11,284] jepsen worker 45 - jepsen.util 1035 :ok :read [15 31] INFO [2019-12-17 12:28:11,284] jepsen worker 45 - jepsen.util 1035 :invoke :read [15 nil] INFO [2019-12-17 12:28:11,286] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3682,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:28:11,286] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3682,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:28:11,295] jepsen worker 5 - jepsen.util 545 :fail :cas [1 [31 15]] :ValueNotSwapFrom INFO [2019-12-17 12:28:11,295] jepsen worker 5 - jepsen.util 545 :invoke :cas [1 [17 13]] INFO [2019-12-17 12:28:11,318] jepsen worker 73 - jepsen.util 793 :ok :write [24 11] INFO [2019-12-17 12:28:11,318] jepsen worker 73 - jepsen.util 793 :invoke :cas [24 [9 7]] INFO [2019-12-17 12:28:11,318] jepsen worker 32 - jepsen.util 752 :ok :write [10 22] INFO [2019-12-17 12:28:11,318] jepsen worker 32 - jepsen.util 752 :invoke :write [10 6] INFO [2019-12-17 12:28:11,333] jepsen worker 9 - jepsen.util 729 :ok :write [3 14] INFO [2019-12-17 12:28:11,333] jepsen worker 9 - jepsen.util 729 :invoke :write [3 16] INFO [2019-12-17 12:28:11,367] jepsen worker 24 - jepsen.util 654 :ok :read [8 14] INFO [2019-12-17 12:28:11,367] jepsen worker 24 - jepsen.util 654 :invoke :cas [8 [1 34]] INFO [2019-12-17 12:28:11,369] jepsen worker 56 - jepsen.util 686 :ok :write [18 13] INFO [2019-12-17 12:28:11,369] jepsen worker 56 - jepsen.util 686 :invoke :write [18 38] INFO [2019-12-17 12:28:11,375] jepsen worker 20 - jepsen.util 740 :ok :write [6 18] INFO [2019-12-17 12:28:11,375] jepsen worker 20 - jepsen.util 740 :invoke :write [6 13] INFO [2019-12-17 12:28:11,392] jepsen worker 44 - jepsen.util 584 :fail :cas [14 [32 46]] :ValueNotSwapFrom INFO [2019-12-17 12:28:11,393] jepsen worker 44 - jepsen.util 584 :invoke :cas [14 [36 35]] INFO [2019-12-17 12:28:11,403] jepsen worker 42 - jepsen.util 1032 :ok :read [14 27] INFO [2019-12-17 12:28:11,403] jepsen worker 42 - jepsen.util 1032 :invoke :read [14 nil] INFO [2019-12-17 12:28:11,447] jepsen worker 53 - jepsen.util 863 :fail :cas [17 [42 44]] :ValueNotSwapFrom INFO [2019-12-17 12:28:11,447] jepsen worker 53 - jepsen.util 863 :invoke :read [17 nil] INFO [2019-12-17 12:28:11,472] jepsen worker 4 - jepsen.util 634 :fail :cas [1 [1 22]] :ValueNotSwapFrom INFO [2019-12-17 12:28:11,473] jepsen worker 4 - jepsen.util 634 :invoke :read [1 nil] INFO [2019-12-17 12:28:11,502] jepsen worker 24 - jepsen.util 654 :fail :cas [8 [1 34]] :ValueNotSwapFrom INFO [2019-12-17 12:28:11,502] jepsen worker 24 - jepsen.util 654 :invoke :write [8 37] INFO [2019-12-17 12:28:11,503] jepsen worker 52 - jepsen.util 772 :ok :read [17 46] INFO [2019-12-17 12:28:11,504] jepsen worker 52 - jepsen.util 772 :invoke :cas [17 [44 47]] INFO [2019-12-17 12:28:11,514] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3705,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:28:11,514] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3705,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:28:11,514] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3708,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:28:11,514] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3708,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:28:11,526] jepsen worker 44 - jepsen.util 584 :fail :cas [14 [36 35]] :ValueNotSwapFrom INFO [2019-12-17 12:28:11,526] jepsen worker 44 - jepsen.util 584 :invoke :write [14 4] INFO [2019-12-17 12:28:11,585] jepsen worker 25 - jepsen.util 925 :fail :cas [8 [45 48]] :ValueNotSwapFrom INFO [2019-12-17 12:28:11,585] jepsen worker 25 - jepsen.util 925 :invoke :write [8 14] INFO [2019-12-17 12:28:11,613] jepsen worker 6 - jepsen.util 726 :ok :read [2 15] INFO [2019-12-17 12:28:11,613] jepsen worker 6 - jepsen.util 726 :invoke :cas [2 [12 32]] INFO [2019-12-17 12:28:11,640] jepsen worker 37 - jepsen.util 757 :fail :cas [12 [11 21]] :ValueNotSwapFrom INFO [2019-12-17 12:28:11,640] jepsen worker 37 - jepsen.util 757 :invoke :read [12 nil] INFO [2019-12-17 12:28:11,648] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3721,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:28:11,648] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3721,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:28:11,677] jepsen worker 53 - jepsen.util 863 :ok :read [17 46] INFO [2019-12-17 12:28:11,677] jepsen worker 53 - jepsen.util 863 :invoke :cas [17 [16 28]] INFO [2019-12-17 12:28:11,695] jepsen worker 4 - jepsen.util 634 :ok :read [1 49] INFO [2019-12-17 12:28:11,695] jepsen worker 4 - jepsen.util 634 :invoke :read [1 nil] INFO [2019-12-17 12:28:11,696] jepsen worker 26 - jepsen.util 836 :fail :cas [8 [44 20]] :ValueNotSwapFrom INFO [2019-12-17 12:28:11,697] jepsen worker 26 - jepsen.util 836 :invoke :write [8 31] INFO [2019-12-17 12:28:11,735] jepsen worker 35 - jepsen.util 665 :fail :cas [11 [41 30]] :ValueNotSwapFrom INFO [2019-12-17 12:28:11,736] jepsen worker 35 - jepsen.util 665 :invoke :read [11 nil] INFO [2019-12-17 12:28:11,740] jepsen worker 69 - jepsen.util 789 :ok :read [23 22] INFO [2019-12-17 12:28:11,741] jepsen worker 69 - jepsen.util 789 :invoke :read [23 nil] INFO [2019-12-17 12:28:11,772] jepsen worker 6 - jepsen.util 726 :fail :cas [2 [12 32]] :ValueNotSwapFrom INFO [2019-12-17 12:28:11,772] jepsen worker 6 - jepsen.util 726 :invoke :write [2 14] INFO [2019-12-17 12:28:11,774] jepsen worker 52 - jepsen.util 772 :fail :cas [17 [44 47]] :ValueNotSwapFrom INFO [2019-12-17 12:28:11,774] jepsen worker 52 - jepsen.util 772 :invoke :write [17 7] INFO [2019-12-17 12:28:11,794] jepsen worker 87 - jepsen.util 717 :fail :cas [29 [17 27]] :ValueNotSwapFrom INFO [2019-12-17 12:28:11,794] jepsen worker 87 - jepsen.util 717 :invoke :cas [29 [37 36]] INFO [2019-12-17 12:28:11,821] jepsen worker 58 - jepsen.util 598 :info :write [19 1] :RequestCanceledException INFO [2019-12-17 12:28:11,821] jepsen worker 17 - jepsen.util 647 :info :cas [5 [36 33]] :RequestCanceledException INFO [2019-12-17 12:28:11,823] jepsen worker 58 - jepsen.util 688 :invoke :write [19 8] INFO [2019-12-17 12:28:11,823] jepsen worker 17 - jepsen.util 737 :invoke :write [5 25] INFO [2019-12-17 12:28:11,828] jepsen worker 14 - jepsen.util 734 :ok :read [4 49] INFO [2019-12-17 12:28:11,828] jepsen worker 14 - jepsen.util 734 :invoke :read [4 nil] INFO [2019-12-17 12:28:11,834] jepsen worker 19 - jepsen.util 739 :info :write [6 21] :RequestCanceledException INFO [2019-12-17 12:28:11,837] jepsen worker 19 - jepsen.util 829 :invoke :read [6 nil] INFO [2019-12-17 12:28:11,907] jepsen worker 28 - jepsen.util 388 :info :write [9 38] :RequestCanceledException INFO [2019-12-17 12:28:11,907] jepsen worker 70 - jepsen.util 970 :ok :read [23 22] INFO [2019-12-17 12:28:11,908] jepsen worker 70 - jepsen.util 970 :invoke :write [23 32] INFO [2019-12-17 12:28:11,909] jepsen worker 28 - jepsen.util 478 :invoke :cas [9 [43 46]] INFO [2019-12-17 12:28:11,921] jepsen worker 13 - jepsen.util 823 :info :write [4 31] :RequestCanceledException INFO [2019-12-17 12:28:11,921] jepsen worker 67 - jepsen.util 1057 :info :write [22 32] :RequestCanceledException INFO [2019-12-17 12:28:11,923] jepsen worker 13 - jepsen.util 913 :invoke :write [4 30] INFO [2019-12-17 12:28:11,923] jepsen worker 67 - jepsen.util 1147 :invoke :cas [22 [6 18]] INFO [2019-12-17 12:28:11,936] jepsen worker 71 - jepsen.util 701 :ok :read [23 22] INFO [2019-12-17 12:28:11,937] jepsen worker 71 - jepsen.util 701 :invoke :cas [23 [44 16]] INFO [2019-12-17 12:28:11,946] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3737,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:28:11,946] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3737,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:28:11,947] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3740,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:28:11,947] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3740,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:28:11,947] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3741,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:28:11,947] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3741,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:28:11,953] jepsen worker 0 - jepsen.util 810 :info :write [0 43] :RequestCanceledException INFO [2019-12-17 12:28:11,954] jepsen worker 0 - jepsen.util 900 :invoke :write [0 8] INFO [2019-12-17 12:28:11,964] jepsen worker 69 - jepsen.util 789 :ok :read [23 22] INFO [2019-12-17 12:28:11,964] jepsen worker 69 - jepsen.util 789 :invoke :write [23 21] INFO [2019-12-17 12:28:11,979] jepsen worker 87 - jepsen.util 717 :fail :cas [29 [37 36]] :ValueNotSwapFrom INFO [2019-12-17 12:28:11,979] jepsen worker 87 - jepsen.util 717 :invoke :write [29 37] INFO [2019-12-17 12:28:12,005] jepsen worker 11 - jepsen.util 821 :info :write [3 8] :RequestCanceledException INFO [2019-12-17 12:28:12,007] jepsen worker 11 - jepsen.util 911 :invoke :cas [3 [22 24]] INFO [2019-12-17 12:28:12,021] jepsen worker 41 - jepsen.util 491 :info :write [13 10] :RequestCanceledException INFO [2019-12-17 12:28:12,022] jepsen worker 41 - jepsen.util 581 :invoke :read [13 nil] INFO [2019-12-17 12:28:12,024] jepsen worker 5 - jepsen.util 545 :fail :cas [1 [17 13]] :ValueNotSwapFrom INFO [2019-12-17 12:28:12,024] jepsen worker 5 - jepsen.util 545 :invoke :cas [1 [2 13]] INFO [2019-12-17 12:28:12,052] jepsen worker 50 - jepsen.util 770 :info :write [16 34] :RequestCanceledException INFO [2019-12-17 12:28:12,054] jepsen worker 50 - jepsen.util 860 :invoke :cas [16 [5 43]] INFO [2019-12-17 12:28:12,056] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3751,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:28:12,056] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3751,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:28:12,066] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:28:12,066] jepsen nemesis - couchbase.util Rebalance check count: 22 INFO [2019-12-17 12:28:12,066] jepsen nemesis - couchbase.util Rebalance stuck count: 3 INFO [2019-12-17 12:28:12,084] jepsen worker 37 - jepsen.util 757 :ok :read [12 45] INFO [2019-12-17 12:28:12,084] jepsen worker 37 - jepsen.util 757 :invoke :cas [12 [12 47]] INFO [2019-12-17 12:28:12,087] jepsen worker 33 - jepsen.util 843 :info :write [11 23] :RequestCanceledException INFO [2019-12-17 12:28:12,088] jepsen worker 33 - jepsen.util 933 :invoke :write [11 20] INFO [2019-12-17 12:28:12,121] jepsen worker 76 - jepsen.util 706 :info :write [25 37] :RequestCanceledException INFO [2019-12-17 12:28:12,122] jepsen worker 76 - jepsen.util 796 :invoke :write [25 46] INFO [2019-12-17 12:28:12,135] jepsen worker 65 - jepsen.util 515 :info :cas [21 [44 48]] :RequestCanceledException INFO [2019-12-17 12:28:12,135] jepsen worker 29 - jepsen.util 659 :fail :read [9 nil] :RequestTimeoutException INFO [2019-12-17 12:28:12,135] jepsen worker 51 - jepsen.util 591 :info :cas [17 [0 30]] :RequestCanceledException INFO [2019-12-17 12:28:12,135] jepsen worker 29 - jepsen.util 659 :invoke :cas [9 [33 12]] INFO [2019-12-17 12:28:12,137] jepsen worker 51 - jepsen.util 681 :invoke :write [17 47] INFO [2019-12-17 12:28:12,137] jepsen worker 65 - jepsen.util 605 :invoke :cas [21 [36 39]] INFO [2019-12-17 12:28:12,140] jepsen worker 5 - jepsen.util 545 :fail :cas [1 [2 13]] :ValueNotSwapFrom INFO [2019-12-17 12:28:12,140] jepsen worker 5 - jepsen.util 545 :invoke :cas [1 [14 8]] INFO [2019-12-17 12:28:12,152] jepsen worker 30 - jepsen.util 660 :info :write [10 28] :RequestCanceledException INFO [2019-12-17 12:28:12,152] jepsen worker 10 - jepsen.util 730 :info :write [3 37] :RequestCanceledException INFO [2019-12-17 12:28:12,153] jepsen worker 10 - jepsen.util 820 :invoke :read [3 nil] INFO [2019-12-17 12:28:12,154] jepsen worker 30 - jepsen.util 750 :invoke :read [10 nil] INFO [2019-12-17 12:28:12,163] jepsen worker 42 - jepsen.util 1032 :ok :read [14 27] INFO [2019-12-17 12:28:12,163] jepsen worker 42 - jepsen.util 1032 :invoke :cas [14 [21 40]] INFO [2019-12-17 12:28:12,184] jepsen worker 72 - jepsen.util 792 :fail :read [24 nil] :RequestTimeoutException INFO [2019-12-17 12:28:12,184] jepsen worker 63 - jepsen.util 513 :info :write [21 4] :RequestCanceledException INFO [2019-12-17 12:28:12,184] jepsen worker 34 - jepsen.util 754 :info :write [11 25] :RequestCanceledException INFO [2019-12-17 12:28:12,184] jepsen worker 72 - jepsen.util 792 :invoke :write [24 42] INFO [2019-12-17 12:28:12,185] jepsen worker 34 - jepsen.util 844 :invoke :cas [11 [11 37]] INFO [2019-12-17 12:28:12,185] jepsen worker 63 - jepsen.util 603 :invoke :write [21 20] INFO [2019-12-17 12:28:12,199] jepsen worker 42 - jepsen.util 1032 :fail :cas [14 [21 40]] :ValueNotSwapFrom INFO [2019-12-17 12:28:12,199] jepsen worker 42 - jepsen.util 1032 :invoke :write [14 33] INFO [2019-12-17 12:28:12,206] jepsen worker 82 - jepsen.util 982 :info :cas [27 [7 19]] :RequestCanceledException INFO [2019-12-17 12:28:12,207] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3772,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:28:12,207] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3772,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:28:12,207] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3774,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:28:12,207] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3774,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:28:12,207] jepsen worker 82 - jepsen.util 1072 :invoke :write [27 6] INFO [2019-12-17 12:28:12,220] jepsen worker 46 - jepsen.util 766 :info :write [15 2] :RequestCanceledException INFO [2019-12-17 12:28:12,220] jepsen worker 60 - jepsen.util 870 :info :write [20 15] :RequestCanceledException INFO [2019-12-17 12:28:12,220] jepsen worker 88 - jepsen.util 628 :info :write [29 1] :RequestCanceledException INFO [2019-12-17 12:28:12,222] jepsen worker 60 - jepsen.util 960 :invoke :cas [20 [37 31]] INFO [2019-12-17 12:28:12,222] jepsen worker 46 - jepsen.util 856 :invoke :read [15 nil] INFO [2019-12-17 12:28:12,222] jepsen worker 88 - jepsen.util 718 :invoke :write [29 46] INFO [2019-12-17 12:28:12,226] jepsen worker 63 - jepsen.util 603 :ok :write [21 20] INFO [2019-12-17 12:28:12,227] jepsen worker 63 - jepsen.util 603 :invoke :read [21 nil] INFO [2019-12-17 12:28:12,234] jepsen worker 18 - jepsen.util 738 :info :write [6 48] :RequestCanceledException INFO [2019-12-17 12:28:12,234] jepsen worker 15 - jepsen.util 735 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:28:12,235] jepsen worker 15 - jepsen.util 735 :invoke :read [5 nil] INFO [2019-12-17 12:28:12,236] jepsen worker 18 - jepsen.util 828 :invoke :cas [6 [18 30]] INFO [2019-12-17 12:28:12,249] jepsen worker 63 - jepsen.util 603 :ok :read [21 20] INFO [2019-12-17 12:28:12,249] jepsen worker 63 - jepsen.util 603 :invoke :write [21 26] INFO [2019-12-17 12:28:12,252] jepsen worker 16 - jepsen.util 826 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:28:12,252] jepsen worker 16 - jepsen.util 826 :invoke :read [5 nil] INFO [2019-12-17 12:28:12,265] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3781,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:28:12,265] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3781,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:28:12,265] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3785,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:28:12,265] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3785,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:28:12,273] jepsen worker 4 - jepsen.util 634 :ok :read [1 49] INFO [2019-12-17 12:28:12,273] jepsen worker 4 - jepsen.util 634 :invoke :write [1 32] INFO [2019-12-17 12:28:12,280] jepsen worker 37 - jepsen.util 757 :fail :cas [12 [12 47]] :ValueNotSwapFrom INFO [2019-12-17 12:28:12,280] jepsen worker 37 - jepsen.util 757 :invoke :cas [12 [21 15]] INFO [2019-12-17 12:28:12,292] jepsen worker 63 - jepsen.util 603 :ok :write [21 26] INFO [2019-12-17 12:28:12,292] jepsen worker 63 - jepsen.util 603 :invoke :cas [21 [13 2]] INFO [2019-12-17 12:28:12,310] jepsen worker 63 - jepsen.util 603 :fail :cas [21 [13 2]] :ValueNotSwapFrom INFO [2019-12-17 12:28:12,310] jepsen worker 63 - jepsen.util 603 :invoke :cas [21 [22 35]] INFO [2019-12-17 12:28:12,321] jepsen worker 40 - jepsen.util 1030 :info :write [13 23] :RequestCanceledException INFO [2019-12-17 12:28:12,322] jepsen worker 40 - jepsen.util 1120 :invoke :write [13 20] INFO [2019-12-17 12:28:12,329] jepsen worker 63 - jepsen.util 603 :fail :cas [21 [22 35]] :ValueNotSwapFrom INFO [2019-12-17 12:28:12,329] jepsen worker 63 - jepsen.util 603 :invoke :read [21 nil] INFO [2019-12-17 12:28:12,345] jepsen worker 63 - jepsen.util 603 :ok :read [21 26] INFO [2019-12-17 12:28:12,345] jepsen worker 63 - jepsen.util 603 :invoke :cas [21 [49 26]] INFO [2019-12-17 12:28:12,360] jepsen worker 63 - jepsen.util 603 :fail :cas [21 [49 26]] :ValueNotSwapFrom INFO [2019-12-17 12:28:12,360] jepsen worker 63 - jepsen.util 603 :invoke :write [21 46] INFO [2019-12-17 12:28:12,373] jepsen worker 43 - jepsen.util 943 :ok :read [14 27] INFO [2019-12-17 12:28:12,373] jepsen worker 43 - jepsen.util 943 :invoke :cas [14 [34 12]] INFO [2019-12-17 12:28:12,380] jepsen worker 37 - jepsen.util 757 :fail :cas [12 [21 15]] :ValueNotSwapFrom INFO [2019-12-17 12:28:12,380] jepsen worker 37 - jepsen.util 757 :invoke :read [12 nil] INFO [2019-12-17 12:28:12,384] jepsen worker 66 - jepsen.util 606 :info :cas [22 [18 7]] :RequestCanceledException INFO [2019-12-17 12:28:12,386] jepsen worker 66 - jepsen.util 696 :invoke :read [22 nil] INFO [2019-12-17 12:28:12,396] jepsen worker 63 - jepsen.util 603 :ok :write [21 46] INFO [2019-12-17 12:28:12,396] jepsen worker 63 - jepsen.util 603 :invoke :write [21 16] INFO [2019-12-17 12:28:12,419] jepsen worker 62 - jepsen.util 872 :info :write [20 23] :RequestCanceledException INFO [2019-12-17 12:28:12,419] jepsen worker 27 - jepsen.util 477 :info :write [9 42] :RequestCanceledException INFO [2019-12-17 12:28:12,419] jepsen worker 31 - jepsen.util 841 :info :cas [10 [1 2]] :RequestCanceledException INFO [2019-12-17 12:28:12,421] jepsen worker 31 - jepsen.util 931 :invoke :write [10 23] INFO [2019-12-17 12:28:12,422] jepsen worker 62 - jepsen.util 962 :invoke :cas [20 [40 22]] INFO [2019-12-17 12:28:12,422] jepsen worker 27 - jepsen.util 567 :invoke :cas [9 [21 39]] INFO [2019-12-17 12:28:12,436] jepsen worker 2 - jepsen.util 632 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:28:12,436] jepsen worker 48 - jepsen.util 498 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:28:12,436] jepsen worker 2 - jepsen.util 632 :invoke :read [0 nil] INFO [2019-12-17 12:28:12,436] jepsen worker 48 - jepsen.util 498 :invoke :read [16 nil] INFO [2019-12-17 12:28:12,439] jepsen worker 63 - jepsen.util 603 :ok :write [21 16] INFO [2019-12-17 12:28:12,439] jepsen worker 63 - jepsen.util 603 :invoke :write [21 30] INFO [2019-12-17 12:28:12,452] jepsen worker 3 - jepsen.util 453 :info :write [1 19] :RequestCanceledException INFO [2019-12-17 12:28:12,453] jepsen worker 3 - jepsen.util 543 :invoke :write [1 22] INFO [2019-12-17 12:28:12,493] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3810,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:28:12,493] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3810,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:28:12,494] jepsen worker 53 - jepsen.util 863 :fail :cas [17 [16 28]] :ValueNotSwapFrom INFO [2019-12-17 12:28:12,494] jepsen worker 53 - jepsen.util 863 :invoke :write [17 3] INFO [2019-12-17 12:28:12,496] jepsen worker 63 - jepsen.util 603 :ok :write [21 30] INFO [2019-12-17 12:28:12,496] jepsen worker 63 - jepsen.util 603 :invoke :write [21 30] INFO [2019-12-17 12:28:12,504] jepsen worker 14 - jepsen.util 734 :ok :read [4 49] INFO [2019-12-17 12:28:12,504] jepsen worker 14 - jepsen.util 734 :invoke :cas [4 [30 41]] INFO [2019-12-17 12:28:12,521] jepsen worker 64 - jepsen.util 514 :fail :read [21 nil] :RequestTimeoutException INFO [2019-12-17 12:28:12,521] jepsen worker 64 - jepsen.util 514 :invoke :cas [21 [22 47]] INFO [2019-12-17 12:28:12,524] jepsen worker 60 - jepsen.util 960 :fail :cas [20 [37 31]] :ValueNotSwapFrom INFO [2019-12-17 12:28:12,525] jepsen worker 60 - jepsen.util 960 :invoke :read [20 nil] INFO [2019-12-17 12:28:12,549] jepsen worker 78 - jepsen.util 798 :info :write [26 4] :RequestCanceledException INFO [2019-12-17 12:28:12,551] jepsen worker 78 - jepsen.util 888 :invoke :write [26 33] INFO [2019-12-17 12:28:12,560] jepsen worker 63 - jepsen.util 603 :ok :write [21 30] INFO [2019-12-17 12:28:12,561] jepsen worker 63 - jepsen.util 603 :invoke :cas [21 [11 9]] INFO [2019-12-17 12:28:12,561] jepsen worker 71 - jepsen.util 701 :fail :cas [23 [44 16]] :ValueNotSwapFrom INFO [2019-12-17 12:28:12,561] jepsen worker 71 - jepsen.util 701 :invoke :read [23 nil] INFO [2019-12-17 12:28:12,564] jepsen worker 37 - jepsen.util 757 :ok :read [12 45] INFO [2019-12-17 12:28:12,564] jepsen worker 37 - jepsen.util 757 :invoke :read [12 nil] INFO [2019-12-17 12:28:12,572] jepsen worker 32 - jepsen.util 752 :ok :write [10 6] INFO [2019-12-17 12:28:12,573] jepsen worker 32 - jepsen.util 752 :invoke :read [10 nil] INFO [2019-12-17 12:28:12,578] jepsen worker 74 - jepsen.util 614 :ok :write [24 32] INFO [2019-12-17 12:28:12,579] jepsen worker 74 - jepsen.util 614 :invoke :cas [24 [45 5]] INFO [2019-12-17 12:28:12,585] jepsen worker 20 - jepsen.util 740 :ok :write [6 13] INFO [2019-12-17 12:28:12,585] jepsen worker 20 - jepsen.util 740 :invoke :cas [6 [44 15]] INFO [2019-12-17 12:28:12,587] jepsen worker 55 - jepsen.util 955 :info :write [18 2] :RequestCanceledException INFO [2019-12-17 12:28:12,588] jepsen worker 64 - jepsen.util 514 :fail :cas [21 [22 47]] :ValueNotSwapFrom INFO [2019-12-17 12:28:12,588] jepsen worker 64 - jepsen.util 514 :invoke :cas [21 [23 17]] INFO [2019-12-17 12:28:12,589] jepsen worker 55 - jepsen.util 1045 :invoke :write [18 0] INFO [2019-12-17 12:28:12,592] jepsen worker 47 - jepsen.util 1037 :ok :write [15 1] INFO [2019-12-17 12:28:12,592] jepsen worker 47 - jepsen.util 1037 :invoke :read [15 nil] INFO [2019-12-17 12:28:12,605] jepsen worker 12 - jepsen.util 732 :info :write [4 21] :RequestCanceledException INFO [2019-12-17 12:28:12,606] jepsen worker 23 - jepsen.util 563 :info :write [7 41] :RequestCanceledException INFO [2019-12-17 12:28:12,607] jepsen worker 12 - jepsen.util 822 :invoke :read [4 nil] INFO [2019-12-17 12:28:12,608] jepsen worker 23 - jepsen.util 653 :invoke :write [7 48] INFO [2019-12-17 12:28:12,624] jepsen worker 9 - jepsen.util 729 :ok :write [3 16] INFO [2019-12-17 12:28:12,624] jepsen worker 9 - jepsen.util 729 :invoke :cas [3 [27 45]] INFO [2019-12-17 12:28:12,626] jepsen worker 1 - jepsen.util 631 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:28:12,626] jepsen worker 1 - jepsen.util 631 :invoke :read [0 nil] INFO [2019-12-17 12:28:12,631] jepsen worker 34 - jepsen.util 844 :fail :cas [11 [11 37]] :ValueNotSwapFrom INFO [2019-12-17 12:28:12,631] jepsen worker 34 - jepsen.util 844 :invoke :cas [11 [21 13]] INFO [2019-12-17 12:28:12,649] jepsen worker 73 - jepsen.util 793 :fail :cas [24 [9 7]] :ValueNotSwapFrom INFO [2019-12-17 12:28:12,650] jepsen worker 14 - jepsen.util 734 :fail :cas [4 [30 41]] :ValueNotSwapFrom INFO [2019-12-17 12:28:12,650] jepsen worker 73 - jepsen.util 793 :invoke :read [24 nil] INFO [2019-12-17 12:28:12,650] jepsen worker 14 - jepsen.util 734 :invoke :read [4 nil] INFO [2019-12-17 12:28:12,691] jepsen worker 63 - jepsen.util 603 :fail :cas [21 [11 9]] :ValueNotSwapFrom INFO [2019-12-17 12:28:12,691] jepsen worker 63 - jepsen.util 603 :invoke :write [21 10] INFO [2019-12-17 12:28:12,706] jepsen worker 34 - jepsen.util 844 :fail :cas [11 [21 13]] :ValueNotSwapFrom INFO [2019-12-17 12:28:12,707] jepsen worker 34 - jepsen.util 844 :invoke :write [11 31] INFO [2019-12-17 12:28:12,709] jepsen worker 19 - jepsen.util 829 :ok :read [6 18] INFO [2019-12-17 12:28:12,710] jepsen worker 19 - jepsen.util 829 :invoke :write [6 1] INFO [2019-12-17 12:28:12,714] jepsen worker 75 - jepsen.util 705 :info :write [25 41] :RequestCanceledException INFO [2019-12-17 12:28:12,720] jepsen worker 18 - jepsen.util 828 :fail :cas [6 [18 30]] :ValueNotSwapFrom INFO [2019-12-17 12:28:12,721] jepsen worker 18 - jepsen.util 828 :invoke :write [6 23] INFO [2019-12-17 12:28:12,723] jepsen worker 75 - jepsen.util 795 :invoke :write [25 22] INFO [2019-12-17 12:28:12,728] jepsen worker 45 - jepsen.util 1035 :ok :read [15 1] INFO [2019-12-17 12:28:12,728] jepsen worker 45 - jepsen.util 1035 :invoke :write [15 37] INFO [2019-12-17 12:28:12,729] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3841,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:28:12,730] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3841,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:28:12,730] jepsen worker 84 - jepsen.util 444 :info :write [28 28] :RequestCanceledException INFO [2019-12-17 12:28:12,731] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3844,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:28:12,731] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3844,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:28:12,736] jepsen worker 84 - jepsen.util 534 :invoke :write [28 35] INFO [2019-12-17 12:28:12,752] jepsen worker 38 - jepsen.util 668 :info :write [12 22] :RequestCanceledException INFO [2019-12-17 12:28:12,765] jepsen worker 38 - jepsen.util 758 :invoke :cas [12 [36 23]] INFO [2019-12-17 12:28:12,779] jepsen worker 11 - jepsen.util 911 :fail :cas [3 [22 24]] :ValueNotSwapFrom INFO [2019-12-17 12:28:12,780] jepsen worker 11 - jepsen.util 911 :invoke :write [3 45] INFO [2019-12-17 12:28:12,790] jepsen worker 20 - jepsen.util 740 :fail :cas [6 [44 15]] :ValueNotSwapFrom INFO [2019-12-17 12:28:12,791] jepsen worker 20 - jepsen.util 740 :invoke :write [6 45] INFO [2019-12-17 12:28:12,809] jepsen worker 64 - jepsen.util 514 :fail :cas [21 [23 17]] :ValueNotSwapFrom INFO [2019-12-17 12:28:12,810] jepsen worker 64 - jepsen.util 514 :invoke :read [21 nil] INFO [2019-12-17 12:28:12,815] jepsen worker 8 - jepsen.util 728 :info :write [2 41] :RequestCanceledException INFO [2019-12-17 12:28:12,815] jepsen worker 86 - jepsen.util 626 :info :cas [28 [25 46]] :RequestCanceledException INFO [2019-12-17 12:28:12,818] jepsen worker 8 - jepsen.util 818 :invoke :cas [2 [40 32]] INFO [2019-12-17 12:28:12,821] jepsen worker 86 - jepsen.util 716 :invoke :cas [28 [24 38]] INFO [2019-12-17 12:28:12,834] jepsen worker 81 - jepsen.util 891 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:28:12,834] jepsen worker 85 - jepsen.util 715 :info :cas [28 [1 34]] :RequestCanceledException INFO [2019-12-17 12:28:12,834] jepsen worker 81 - jepsen.util 891 :invoke :cas [27 [21 32]] INFO [2019-12-17 12:28:12,838] jepsen worker 10 - jepsen.util 820 :ok :read [3 16] INFO [2019-12-17 12:28:12,838] jepsen worker 85 - jepsen.util 805 :invoke :cas [28 [48 39]] INFO [2019-12-17 12:28:12,838] jepsen worker 10 - jepsen.util 820 :invoke :write [3 17] INFO [2019-12-17 12:28:12,838] jepsen worker 29 - jepsen.util 659 :fail :cas [9 [33 12]] :ValueNotSwapFrom INFO [2019-12-17 12:28:12,838] jepsen worker 29 - jepsen.util 659 :invoke :write [9 33] INFO [2019-12-17 12:28:12,842] jepsen worker 65 - jepsen.util 605 :fail :cas [21 [36 39]] :ValueNotSwapFrom INFO [2019-12-17 12:28:12,842] jepsen worker 65 - jepsen.util 605 :invoke :write [21 12] INFO [2019-12-17 12:28:12,851] jepsen worker 83 - jepsen.util 893 :info :write [27 46] :RequestCanceledException INFO [2019-12-17 12:28:12,851] jepsen worker 54 - jepsen.util 774 :info :write [18 20] :RequestCanceledException INFO [2019-12-17 12:28:12,855] jepsen worker 54 - jepsen.util 864 :invoke :read [18 nil] INFO [2019-12-17 12:28:12,856] jepsen worker 83 - jepsen.util 983 :invoke :read [27 nil] INFO [2019-12-17 12:28:12,904] jepsen worker 46 - jepsen.util 856 :ok :read [15 1] INFO [2019-12-17 12:28:12,904] jepsen worker 46 - jepsen.util 856 :invoke :write [15 15] INFO [2019-12-17 12:28:12,919] jepsen worker 30 - jepsen.util 750 :ok :read [10 6] INFO [2019-12-17 12:28:12,919] jepsen worker 30 - jepsen.util 750 :invoke :write [10 31] INFO [2019-12-17 12:28:12,921] jepsen worker 39 - jepsen.util 759 :info :cas [13 [16 10]] :RequestCanceledException INFO [2019-12-17 12:28:12,923] jepsen worker 39 - jepsen.util 849 :invoke :write [13 42] INFO [2019-12-17 12:28:12,924] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3867,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:28:12,924] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3867,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:28:12,924] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3870,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:28:12,924] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3870,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:28:12,924] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3871,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:28:12,925] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3871,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:28:12,935] jepsen worker 68 - jepsen.util 428 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:28:12,935] jepsen worker 68 - jepsen.util 428 :invoke :write [22 15] INFO [2019-12-17 12:28:12,967] jepsen worker 27 - jepsen.util 567 :fail :cas [9 [21 39]] :ValueNotSwapFrom INFO [2019-12-17 12:28:12,967] jepsen worker 27 - jepsen.util 567 :invoke :write [9 33] INFO [2019-12-17 12:28:12,997] jepsen worker 32 - jepsen.util 752 :ok :read [10 6] INFO [2019-12-17 12:28:12,997] jepsen worker 32 - jepsen.util 752 :invoke :write [10 25] INFO [2019-12-17 12:28:13,007] jepsen worker 62 - jepsen.util 962 :fail :cas [20 [40 22]] :ValueNotSwapFrom INFO [2019-12-17 12:28:13,008] jepsen worker 62 - jepsen.util 962 :invoke :cas [20 [3 32]] INFO [2019-12-17 12:28:13,017] jepsen worker 9 - jepsen.util 729 :fail :cas [3 [27 45]] :ValueNotSwapFrom INFO [2019-12-17 12:28:13,017] jepsen worker 9 - jepsen.util 729 :invoke :cas [3 [33 38]] INFO [2019-12-17 12:28:13,021] jepsen worker 21 - jepsen.util 831 :info :write [7 39] :RequestCanceledException INFO [2019-12-17 12:28:13,021] jepsen worker 36 - jepsen.util 846 :info :write [12 3] :RequestCanceledException INFO [2019-12-17 12:28:13,022] jepsen worker 36 - jepsen.util 936 :invoke :cas [12 [38 0]] INFO [2019-12-17 12:28:13,022] jepsen worker 21 - jepsen.util 921 :invoke :cas [7 [27 20]] INFO [2019-12-17 12:28:13,033] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3880,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:28:13,033] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3880,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:28:13,034] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3882,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:28:13,034] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3882,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:28:13,038] jepsen worker 63 - jepsen.util 603 :ok :write [21 10] INFO [2019-12-17 12:28:13,038] jepsen worker 63 - jepsen.util 603 :invoke :cas [21 [2 3]] INFO [2019-12-17 12:28:13,049] jepsen worker 28 - jepsen.util 478 :ok :cas [9 [43 46]] INFO [2019-12-17 12:28:13,049] jepsen worker 28 - jepsen.util 478 :invoke :read [9 nil] INFO [2019-12-17 12:28:13,052] jepsen worker 77 - jepsen.util 617 :info :cas [25 [2 3]] :RequestCanceledException INFO [2019-12-17 12:28:13,054] jepsen worker 77 - jepsen.util 707 :invoke :write [25 22] INFO [2019-12-17 12:28:13,058] jepsen worker 63 - jepsen.util 603 :fail :cas [21 [2 3]] :ValueNotSwapFrom INFO [2019-12-17 12:28:13,058] jepsen worker 63 - jepsen.util 603 :invoke :cas [21 [46 5]] INFO [2019-12-17 12:28:13,068] jepsen worker 72 - jepsen.util 792 :ok :write [24 42] INFO [2019-12-17 12:28:13,068] jepsen worker 72 - jepsen.util 792 :invoke :write [24 17] INFO [2019-12-17 12:28:13,080] jepsen worker 9 - jepsen.util 729 :fail :cas [3 [33 38]] :ValueNotSwapFrom INFO [2019-12-17 12:28:13,080] jepsen worker 9 - jepsen.util 729 :invoke :write [3 45] INFO [2019-12-17 12:28:13,085] jepsen worker 7 - jepsen.util 907 :info :write [2 7] :RequestCanceledException INFO [2019-12-17 12:28:13,085] jepsen worker 79 - jepsen.util 619 :info :write [26 12] :RequestCanceledException INFO [2019-12-17 12:28:13,087] jepsen worker 79 - jepsen.util 709 :invoke :read [26 nil] INFO [2019-12-17 12:28:13,087] jepsen worker 7 - jepsen.util 997 :invoke :write [2 48] INFO [2019-12-17 12:28:13,102] jepsen worker 35 - jepsen.util 665 :ok :read [11 7] INFO [2019-12-17 12:28:13,103] jepsen worker 35 - jepsen.util 665 :invoke :write [11 21] INFO [2019-12-17 12:28:13,109] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:28:13,109] jepsen nemesis - couchbase.util Rebalance check count: 23 INFO [2019-12-17 12:28:13,109] jepsen nemesis - couchbase.util Rebalance stuck count: 4 INFO [2019-12-17 12:28:13,120] jepsen worker 63 - jepsen.util 603 :fail :cas [21 [46 5]] :ValueNotSwapFrom INFO [2019-12-17 12:28:13,120] jepsen worker 63 - jepsen.util 603 :invoke :cas [21 [26 1]] INFO [2019-12-17 12:28:13,128] jepsen worker 37 - jepsen.util 757 :ok :read [12 45] INFO [2019-12-17 12:28:13,128] jepsen worker 37 - jepsen.util 757 :invoke :write [12 49] INFO [2019-12-17 12:28:13,134] jepsen worker 80 - jepsen.util 890 :info :write [26 43] :RequestCanceledException INFO [2019-12-17 12:28:13,136] jepsen worker 80 - jepsen.util 980 :invoke :read [26 nil] INFO [2019-12-17 12:28:13,174] jepsen worker 8 - jepsen.util 818 :fail :cas [2 [40 32]] :ValueNotSwapFrom INFO [2019-12-17 12:28:13,174] jepsen worker 8 - jepsen.util 818 :invoke :cas [2 [47 10]] INFO [2019-12-17 12:28:13,192] jepsen worker 79 - jepsen.util 709 :ok :read [26 43] INFO [2019-12-17 12:28:13,192] jepsen worker 79 - jepsen.util 709 :invoke :cas [26 [2 31]] INFO [2019-12-17 12:28:13,201] jepsen worker 74 - jepsen.util 614 :fail :cas [24 [45 5]] :ValueNotSwapFrom INFO [2019-12-17 12:28:13,201] jepsen worker 74 - jepsen.util 614 :invoke :cas [24 [1 10]] INFO [2019-12-17 12:28:13,204] jepsen worker 28 - jepsen.util 478 :ok :read [9 46] INFO [2019-12-17 12:28:13,205] jepsen worker 28 - jepsen.util 478 :invoke :cas [9 [35 37]] INFO [2019-12-17 12:28:13,211] jepsen worker 43 - jepsen.util 943 :fail :cas [14 [34 12]] :ValueNotSwapFrom INFO [2019-12-17 12:28:13,211] jepsen worker 43 - jepsen.util 943 :invoke :read [14 nil] INFO [2019-12-17 12:28:13,221] jepsen worker 59 - jepsen.util 779 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:28:13,221] jepsen worker 59 - jepsen.util 779 :invoke :read [19 nil] INFO [2019-12-17 12:28:13,226] jepsen worker 5 - jepsen.util 545 :fail :cas [1 [14 8]] :ValueNotSwapFrom INFO [2019-12-17 12:28:13,226] jepsen worker 5 - jepsen.util 545 :invoke :read [1 nil] INFO [2019-12-17 12:28:13,237] jepsen worker 8 - jepsen.util 818 :fail :cas [2 [47 10]] :ValueNotSwapFrom INFO [2019-12-17 12:28:13,238] jepsen worker 8 - jepsen.util 818 :invoke :cas [2 [11 9]] INFO [2019-12-17 12:28:13,243] jepsen worker 63 - jepsen.util 603 :fail :cas [21 [26 1]] :ValueNotSwapFrom INFO [2019-12-17 12:28:13,244] jepsen worker 63 - jepsen.util 603 :invoke :write [21 8] INFO [2019-12-17 12:28:13,247] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3915,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:28:13,247] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3915,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:28:13,247] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3918,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:28:13,247] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3918,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:28:13,249] jepsen worker 22 - jepsen.util 652 :info :write [7 16] :RequestCanceledException INFO [2019-12-17 12:28:13,251] jepsen worker 22 - jepsen.util 742 :invoke :cas [7 [27 35]] INFO [2019-12-17 12:28:13,254] jepsen worker 73 - jepsen.util 793 :ok :read [24 42] INFO [2019-12-17 12:28:13,254] jepsen worker 73 - jepsen.util 793 :invoke :write [24 26] INFO [2019-12-17 12:28:13,278] jepsen worker 8 - jepsen.util 818 :fail :cas [2 [11 9]] :ValueNotSwapFrom INFO [2019-12-17 12:28:13,278] jepsen worker 8 - jepsen.util 818 :invoke :write [2 45] INFO [2019-12-17 12:28:13,287] jepsen worker 57 - jepsen.util 777 :info :cas [19 [4 49]] :RequestCanceledException INFO [2019-12-17 12:28:13,288] jepsen worker 57 - jepsen.util 867 :invoke :read [19 nil] INFO [2019-12-17 12:28:13,297] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3905,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:28:13,297] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3905,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:28:13,305] jepsen worker 22 - jepsen.util 742 :fail :cas [7 [27 35]] :ValueNotSwapFrom INFO [2019-12-17 12:28:13,305] jepsen worker 22 - jepsen.util 742 :invoke :write [7 2] INFO [2019-12-17 12:28:13,305] jepsen worker 61 - jepsen.util 871 :info :write [20 27] :RequestCanceledException INFO [2019-12-17 12:28:13,308] jepsen worker 61 - jepsen.util 961 :invoke :cas [20 [19 7]] INFO [2019-12-17 12:28:13,321] jepsen worker 49 - jepsen.util 589 :info :write [16 19] :RequestCanceledException INFO [2019-12-17 12:28:13,323] jepsen worker 49 - jepsen.util 679 :invoke :cas [16 [22 30]] INFO [2019-12-17 12:28:13,324] jepsen worker 63 - jepsen.util 603 :ok :write [21 8] INFO [2019-12-17 12:28:13,324] jepsen worker 63 - jepsen.util 603 :invoke :read [21 nil] INFO [2019-12-17 12:28:13,334] jepsen worker 12 - jepsen.util 822 :ok :read [4 49] INFO [2019-12-17 12:28:13,335] jepsen worker 12 - jepsen.util 822 :invoke :read [4 nil] INFO [2019-12-17 12:28:13,335] jepsen worker 89 - jepsen.util 899 :info :write [29 49] :RequestCanceledException INFO [2019-12-17 12:28:13,337] jepsen worker 89 - jepsen.util 989 :invoke :read [29 nil] INFO [2019-12-17 12:28:13,354] jepsen worker 60 - jepsen.util 960 :ok :read [20 7] INFO [2019-12-17 12:28:13,354] jepsen worker 60 - jepsen.util 960 :invoke :write [20 30] INFO [2019-12-17 12:28:13,363] jepsen worker 63 - jepsen.util 603 :ok :read [21 8] INFO [2019-12-17 12:28:13,363] jepsen worker 63 - jepsen.util 603 :invoke :write [21 45] INFO [2019-12-17 12:28:13,397] jepsen worker 54 - jepsen.util 864 :ok :read [18 13] INFO [2019-12-17 12:28:13,397] jepsen worker 54 - jepsen.util 864 :invoke :cas [18 [13 39]] INFO [2019-12-17 12:28:13,422] jepsen worker 56 - jepsen.util 686 :info :write [18 38] :RequestCanceledException INFO [2019-12-17 12:28:13,423] jepsen worker 56 - jepsen.util 776 :invoke :read [18 nil] INFO [2019-12-17 12:28:13,431] jepsen worker 63 - jepsen.util 603 :ok :write [21 45] INFO [2019-12-17 12:28:13,431] jepsen worker 63 - jepsen.util 603 :invoke :cas [21 [28 47]] INFO [2019-12-17 12:28:13,457] jepsen worker 63 - jepsen.util 603 :fail :cas [21 [28 47]] :ValueNotSwapFrom INFO [2019-12-17 12:28:13,458] jepsen worker 63 - jepsen.util 603 :invoke :write [21 42] INFO [2019-12-17 12:28:13,466] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3939,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:28:13,466] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3939,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:28:13,467] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3942,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:28:13,467] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3942,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:28:13,488] jepsen worker 43 - jepsen.util 943 :ok :read [14 27] INFO [2019-12-17 12:28:13,489] jepsen worker 43 - jepsen.util 943 :invoke :write [14 33] INFO [2019-12-17 12:28:13,514] jepsen worker 79 - jepsen.util 709 :fail :cas [26 [2 31]] :ValueNotSwapFrom INFO [2019-12-17 12:28:13,514] jepsen worker 79 - jepsen.util 709 :invoke :cas [26 [36 20]] INFO [2019-12-17 12:28:13,530] jepsen worker 63 - jepsen.util 603 :ok :write [21 42] INFO [2019-12-17 12:28:13,530] jepsen worker 63 - jepsen.util 603 :invoke :write [21 38] INFO [2019-12-17 12:28:13,535] jepsen worker 24 - jepsen.util 654 :info :write [8 37] :RequestCanceledException INFO [2019-12-17 12:28:13,536] jepsen worker 24 - jepsen.util 744 :invoke :cas [8 [15 7]] INFO [2019-12-17 12:28:13,588] jepsen worker 63 - jepsen.util 603 :ok :write [21 38] INFO [2019-12-17 12:28:13,589] jepsen worker 63 - jepsen.util 603 :invoke :read [21 nil] INFO [2019-12-17 12:28:13,617] jepsen worker 63 - jepsen.util 603 :ok :read [21 38] INFO [2019-12-17 12:28:13,618] jepsen worker 63 - jepsen.util 603 :invoke :read [21 nil] INFO [2019-12-17 12:28:13,638] jepsen worker 25 - jepsen.util 925 :info :write [8 14] :RequestCanceledException INFO [2019-12-17 12:28:13,638] jepsen worker 44 - jepsen.util 584 :info :write [14 4] :RequestCanceledException INFO [2019-12-17 12:28:13,659] jepsen worker 44 - jepsen.util 674 :invoke :read [14 nil] INFO [2019-12-17 12:28:13,661] jepsen worker 25 - jepsen.util 1015 :invoke :read [8 nil] INFO [2019-12-17 12:28:13,695] jepsen worker 62 - jepsen.util 962 :fail :cas [20 [3 32]] :ValueNotSwapFrom INFO [2019-12-17 12:28:13,695] jepsen worker 62 - jepsen.util 962 :invoke :cas [20 [47 1]] INFO [2019-12-17 12:28:13,697] jepsen worker 63 - jepsen.util 603 :ok :read [21 38] INFO [2019-12-17 12:28:13,697] jepsen worker 63 - jepsen.util 603 :invoke :cas [21 [26 37]] INFO [2019-12-17 12:28:13,700] jepsen worker 24 - jepsen.util 744 :fail :cas [8 [15 7]] :ValueNotSwapFrom INFO [2019-12-17 12:28:13,700] jepsen worker 24 - jepsen.util 744 :invoke :cas [8 [8 32]] INFO [2019-12-17 12:28:13,712] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3955,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:28:13,713] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3955,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:28:13,718] jepsen worker 26 - jepsen.util 836 :info :write [8 31] :RequestCanceledException INFO [2019-12-17 12:28:13,724] jepsen worker 26 - jepsen.util 926 :invoke :cas [8 [44 5]] INFO [2019-12-17 12:28:13,735] jepsen worker 63 - jepsen.util 603 :fail :cas [21 [26 37]] :ValueNotSwapFrom INFO [2019-12-17 12:28:13,735] jepsen worker 63 - jepsen.util 603 :invoke :cas [21 [27 14]] INFO [2019-12-17 12:28:13,759] jepsen worker 63 - jepsen.util 603 :fail :cas [21 [27 14]] :ValueNotSwapFrom INFO [2019-12-17 12:28:13,760] jepsen worker 63 - jepsen.util 603 :invoke :cas [21 [7 44]] INFO [2019-12-17 12:28:13,764] jepsen worker 36 - jepsen.util 936 :fail :cas [12 [38 0]] :ValueNotSwapFrom INFO [2019-12-17 12:28:13,764] jepsen worker 36 - jepsen.util 936 :invoke :write [12 47] INFO [2019-12-17 12:28:13,780] jepsen worker 63 - jepsen.util 603 :fail :cas [21 [7 44]] :ValueNotSwapFrom INFO [2019-12-17 12:28:13,780] jepsen worker 63 - jepsen.util 603 :invoke :read [21 nil] INFO [2019-12-17 12:28:13,793] jepsen worker 63 - jepsen.util 603 :ok :read [21 38] INFO [2019-12-17 12:28:13,794] jepsen worker 63 - jepsen.util 603 :invoke :cas [21 [28 12]] INFO [2019-12-17 12:28:13,806] jepsen worker 6 - jepsen.util 726 :info :write [2 14] :RequestCanceledException INFO [2019-12-17 12:28:13,806] jepsen worker 52 - jepsen.util 772 :info :write [17 7] :RequestCanceledException INFO [2019-12-17 12:28:13,807] jepsen worker 52 - jepsen.util 862 :invoke :read [17 nil] INFO [2019-12-17 12:28:13,807] jepsen worker 6 - jepsen.util 816 :invoke :read [2 nil] INFO [2019-12-17 12:28:13,812] jepsen worker 63 - jepsen.util 603 :fail :cas [21 [28 12]] :ValueNotSwapFrom INFO [2019-12-17 12:28:13,813] jepsen worker 63 - jepsen.util 603 :invoke :read [21 nil] INFO [2019-12-17 12:28:13,833] jepsen worker 26 - jepsen.util 926 :fail :cas [8 [44 5]] :ValueNotSwapFrom INFO [2019-12-17 12:28:13,833] jepsen worker 26 - jepsen.util 926 :invoke :write [8 6] INFO [2019-12-17 12:28:13,836] jepsen worker 63 - jepsen.util 603 :ok :read [21 38] INFO [2019-12-17 12:28:13,836] jepsen worker 63 - jepsen.util 603 :invoke :cas [21 [21 15]] INFO [2019-12-17 12:28:13,844] jepsen worker 6 - jepsen.util 816 :ok :read [2 15] INFO [2019-12-17 12:28:13,845] jepsen worker 6 - jepsen.util 816 :invoke :cas [2 [3 11]] INFO [2019-12-17 12:28:13,876] jepsen worker 38 - jepsen.util 758 :fail :cas [12 [36 23]] :ValueNotSwapFrom INFO [2019-12-17 12:28:13,876] jepsen worker 38 - jepsen.util 758 :invoke :read [12 nil] INFO [2019-12-17 12:28:13,881] jepsen worker 63 - jepsen.util 603 :fail :cas [21 [21 15]] :ValueNotSwapFrom INFO [2019-12-17 12:28:13,881] jepsen worker 63 - jepsen.util 603 :invoke :read [21 nil] INFO [2019-12-17 12:28:13,887] jepsen worker 58 - jepsen.util 688 :info :write [19 8] :RequestCanceledException INFO [2019-12-17 12:28:13,889] jepsen worker 58 - jepsen.util 778 :invoke :write [19 44] INFO [2019-12-17 12:28:13,895] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3977,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:28:13,895] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3977,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:28:13,896] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3981,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:28:13,896] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3981,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:28:13,905] jepsen worker 17 - jepsen.util 737 :info :write [5 25] :RequestCanceledException INFO [2019-12-17 12:28:13,907] jepsen worker 17 - jepsen.util 827 :invoke :write [5 28] INFO [2019-12-17 12:28:13,917] jepsen worker 63 - jepsen.util 603 :ok :read [21 38] INFO [2019-12-17 12:28:13,917] jepsen worker 63 - jepsen.util 603 :invoke :read [21 nil] INFO [2019-12-17 12:28:13,919] jepsen worker 80 - jepsen.util 980 :ok :read [26 43] INFO [2019-12-17 12:28:13,920] jepsen worker 80 - jepsen.util 980 :invoke :cas [26 [49 27]] INFO [2019-12-17 12:28:13,949] jepsen worker 63 - jepsen.util 603 :ok :read [21 38] INFO [2019-12-17 12:28:13,949] jepsen worker 63 - jepsen.util 603 :invoke :read [21 nil] INFO [2019-12-17 12:28:13,951] jepsen worker 14 - jepsen.util 734 :ok :read [4 49] INFO [2019-12-17 12:28:13,952] jepsen worker 14 - jepsen.util 734 :invoke :write [4 15] INFO [2019-12-17 12:28:13,952] jepsen worker 13 - jepsen.util 913 :info :write [4 30] :RequestCanceledException INFO [2019-12-17 12:28:13,952] jepsen worker 70 - jepsen.util 970 :info :write [23 32] :RequestCanceledException INFO [2019-12-17 12:28:13,955] jepsen worker 13 - jepsen.util 1003 :invoke :cas [4 [9 7]] INFO [2019-12-17 12:28:13,955] jepsen worker 70 - jepsen.util 1060 :invoke :write [23 24] INFO [2019-12-17 12:28:13,963] jepsen worker 6 - jepsen.util 816 :fail :cas [2 [3 11]] :ValueNotSwapFrom INFO [2019-12-17 12:28:13,963] jepsen worker 6 - jepsen.util 816 :invoke :write [2 9] INFO [2019-12-17 12:28:13,984] jepsen worker 63 - jepsen.util 603 :ok :read [21 38] INFO [2019-12-17 12:28:13,984] jepsen worker 63 - jepsen.util 603 :invoke :cas [21 [24 11]] INFO [2019-12-17 12:28:13,988] jepsen worker 0 - jepsen.util 900 :info :write [0 8] :RequestCanceledException INFO [2019-12-17 12:28:13,991] jepsen worker 0 - jepsen.util 990 :invoke :write [0 11] INFO [2019-12-17 12:28:14,000] jepsen worker 5 - jepsen.util 545 :ok :read [1 49] INFO [2019-12-17 12:28:14,001] jepsen worker 5 - jepsen.util 545 :invoke :write [1 45] INFO [2019-12-17 12:28:14,004] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3997,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:28:14,004] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3997,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:28:14,004] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3999,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:28:14,004] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3999,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:28:14,012] jepsen worker 63 - jepsen.util 603 :fail :cas [21 [24 11]] :ValueNotSwapFrom INFO [2019-12-17 12:28:14,013] jepsen worker 63 - jepsen.util 603 :invoke :write [21 21] INFO [2019-12-17 12:28:14,021] jepsen worker 67 - jepsen.util 1147 :info :cas [22 [6 18]] :RequestCanceledException INFO [2019-12-17 12:28:14,021] jepsen worker 69 - jepsen.util 789 :info :write [23 21] :RequestCanceledException INFO [2019-12-17 12:28:14,021] jepsen worker 87 - jepsen.util 717 :info :write [29 37] :RequestCanceledException INFO [2019-12-17 12:28:14,025] jepsen worker 67 - jepsen.util 1237 :invoke :read [22 nil] INFO [2019-12-17 12:28:14,025] jepsen worker 69 - jepsen.util 879 :invoke :read [23 nil] INFO [2019-12-17 12:28:14,025] jepsen worker 87 - jepsen.util 807 :invoke :write [29 21] INFO [2019-12-17 12:28:14,043] jepsen worker 89 - jepsen.util 989 :ok :read [29 48] INFO [2019-12-17 12:28:14,043] jepsen worker 89 - jepsen.util 989 :invoke :read [29 nil] INFO [2019-12-17 12:28:14,065] jepsen worker 63 - jepsen.util 603 :ok :write [21 21] INFO [2019-12-17 12:28:14,065] jepsen worker 63 - jepsen.util 603 :invoke :read [21 nil] INFO [2019-12-17 12:28:14,078] jepsen worker 12 - jepsen.util 822 :ok :read [4 49] INFO [2019-12-17 12:28:14,078] jepsen worker 12 - jepsen.util 822 :invoke :cas [4 [40 42]] INFO [2019-12-17 12:28:14,088] jepsen worker 63 - jepsen.util 603 :ok :read [21 21] INFO [2019-12-17 12:28:14,088] jepsen worker 63 - jepsen.util 603 :invoke :cas [21 [4 34]] INFO [2019-12-17 12:28:14,094] jepsen worker 52 - jepsen.util 862 :ok :read [17 46] INFO [2019-12-17 12:28:14,094] jepsen worker 52 - jepsen.util 862 :invoke :cas [17 [20 41]] INFO [2019-12-17 12:28:14,107] jepsen worker 63 - jepsen.util 603 :fail :cas [21 [4 34]] :ValueNotSwapFrom INFO [2019-12-17 12:28:14,107] jepsen worker 63 - jepsen.util 603 :invoke :cas [21 [16 34]] INFO [2019-12-17 12:28:14,121] jepsen worker 41 - jepsen.util 581 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:28:14,121] jepsen worker 41 - jepsen.util 581 :invoke :cas [13 [49 32]] INFO [2019-12-17 12:28:14,123] jepsen worker 71 - jepsen.util 701 :ok :read [23 22] INFO [2019-12-17 12:28:14,123] jepsen worker 71 - jepsen.util 701 :invoke :read [23 nil] INFO [2019-12-17 12:28:14,123] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:28:14,123] jepsen nemesis - couchbase.util Rebalance check count: 24 INFO [2019-12-17 12:28:14,123] jepsen nemesis - couchbase.util Rebalance stuck count: 5 INFO [2019-12-17 12:28:14,130] jepsen worker 72 - jepsen.util 792 :ok :write [24 17] INFO [2019-12-17 12:28:14,130] jepsen worker 31 - jepsen.util 931 :ok :write [10 23] INFO [2019-12-17 12:28:14,130] jepsen worker 72 - jepsen.util 792 :invoke :cas [24 [37 27]] INFO [2019-12-17 12:28:14,130] jepsen worker 31 - jepsen.util 931 :invoke :read [10 nil] INFO [2019-12-17 12:28:14,131] jepsen worker 18 - jepsen.util 828 :ok :write [6 23] INFO [2019-12-17 12:28:14,131] jepsen worker 18 - jepsen.util 828 :invoke :read [6 nil] INFO [2019-12-17 12:28:14,132] jepsen worker 63 - jepsen.util 603 :fail :cas [21 [16 34]] :ValueNotSwapFrom INFO [2019-12-17 12:28:14,132] jepsen worker 63 - jepsen.util 603 :invoke :cas [21 [33 29]] INFO [2019-12-17 12:28:14,133] jepsen worker 50 - jepsen.util 860 :info :cas [16 [5 43]] :RequestCanceledException INFO [2019-12-17 12:28:14,134] jepsen worker 50 - jepsen.util 950 :invoke :write [16 15] INFO [2019-12-17 12:28:14,141] jepsen worker 61 - jepsen.util 961 :fail :cas [20 [19 7]] :ValueNotSwapFrom INFO [2019-12-17 12:28:14,141] jepsen worker 61 - jepsen.util 961 :invoke :cas [20 [33 45]] INFO [2019-12-17 12:28:14,148] jepsen worker 9 - jepsen.util 729 :ok :write [3 45] INFO [2019-12-17 12:28:14,149] jepsen worker 9 - jepsen.util 729 :invoke :write [3 3] INFO [2019-12-17 12:28:14,153] jepsen worker 33 - jepsen.util 933 :info :write [11 20] :RequestCanceledException INFO [2019-12-17 12:28:14,154] jepsen worker 79 - jepsen.util 709 :fail :cas [26 [36 20]] :ValueNotSwapFrom INFO [2019-12-17 12:28:14,154] jepsen worker 79 - jepsen.util 709 :invoke :write [26 32] INFO [2019-12-17 12:28:14,155] jepsen worker 33 - jepsen.util 1023 :invoke :cas [11 [12 37]] INFO [2019-12-17 12:28:14,171] jepsen worker 25 - jepsen.util 1015 :ok :read [8 14] INFO [2019-12-17 12:28:14,171] jepsen worker 25 - jepsen.util 1015 :invoke :write [8 49] INFO [2019-12-17 12:28:14,183] jepsen worker 51 - jepsen.util 681 :info :write [17 47] :RequestCanceledException INFO [2019-12-17 12:28:14,185] jepsen worker 51 - jepsen.util 771 :invoke :cas [17 [21 24]] INFO [2019-12-17 12:28:14,190] jepsen worker 62 - jepsen.util 962 :fail :cas [20 [47 1]] :ValueNotSwapFrom INFO [2019-12-17 12:28:14,191] jepsen worker 62 - jepsen.util 962 :invoke :cas [20 [36 34]] INFO [2019-12-17 12:28:14,198] jepsen worker 18 - jepsen.util 828 :ok :read [6 23] INFO [2019-12-17 12:28:14,198] jepsen worker 18 - jepsen.util 828 :invoke :write [6 32] INFO [2019-12-17 12:28:14,199] jepsen worker 63 - jepsen.util 603 :fail :cas [21 [33 29]] :ValueNotSwapFrom INFO [2019-12-17 12:28:14,199] jepsen worker 63 - jepsen.util 603 :invoke :write [21 9] INFO [2019-12-17 12:28:14,221] jepsen worker 76 - jepsen.util 796 :info :write [25 46] :RequestCanceledException INFO [2019-12-17 12:28:14,221] jepsen worker 42 - jepsen.util 1032 :info :write [14 33] :RequestCanceledException INFO [2019-12-17 12:28:14,223] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4035,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:28:14,223] jepsen worker 42 - jepsen.util 1122 :invoke :read [14 nil] INFO [2019-12-17 12:28:14,223] jepsen worker 76 - jepsen.util 886 :invoke :write [25 22] INFO [2019-12-17 12:28:14,223] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4035,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:28:14,223] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4039,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:28:14,223] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4039,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:28:14,235] jepsen worker 88 - jepsen.util 718 :info :write [29 46] :RequestCanceledException INFO [2019-12-17 12:28:14,236] jepsen worker 88 - jepsen.util 808 :invoke :write [29 38] INFO [2019-12-17 12:28:14,258] jepsen worker 56 - jepsen.util 776 :ok :read [18 13] INFO [2019-12-17 12:28:14,258] jepsen worker 56 - jepsen.util 776 :invoke :read [18 nil] INFO [2019-12-17 12:28:14,275] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4025,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:28:14,275] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4025,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:28:14,275] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4026,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:28:14,275] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4026,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:28:14,278] jepsen worker 63 - jepsen.util 603 :ok :write [21 9] INFO [2019-12-17 12:28:14,278] jepsen worker 63 - jepsen.util 603 :invoke :cas [21 [7 10]] INFO [2019-12-17 12:28:14,288] jepsen worker 82 - jepsen.util 1072 :info :write [27 6] :RequestCanceledException INFO [2019-12-17 12:28:14,290] jepsen worker 82 - jepsen.util 1162 :invoke :cas [27 [27 0]] INFO [2019-12-17 12:28:14,295] jepsen worker 61 - jepsen.util 961 :fail :cas [20 [33 45]] :ValueNotSwapFrom INFO [2019-12-17 12:28:14,295] jepsen worker 61 - jepsen.util 961 :invoke :cas [20 [1 39]] INFO [2019-12-17 12:28:14,301] jepsen worker 4 - jepsen.util 634 :info :write [1 32] :RequestCanceledException INFO [2019-12-17 12:28:14,303] jepsen worker 4 - jepsen.util 724 :invoke :cas [1 [45 30]] INFO [2019-12-17 12:28:14,304] jepsen worker 45 - jepsen.util 1035 :ok :write [15 37] INFO [2019-12-17 12:28:14,305] jepsen worker 73 - jepsen.util 793 :ok :write [24 26] INFO [2019-12-17 12:28:14,305] jepsen worker 45 - jepsen.util 1035 :invoke :write [15 17] INFO [2019-12-17 12:28:14,305] jepsen worker 73 - jepsen.util 793 :invoke :write [24 37] INFO [2019-12-17 12:28:14,305] jepsen worker 63 - jepsen.util 603 :fail :cas [21 [7 10]] :ValueNotSwapFrom INFO [2019-12-17 12:28:14,306] jepsen worker 63 - jepsen.util 603 :invoke :read [21 nil] INFO [2019-12-17 12:28:14,335] jepsen worker 15 - jepsen.util 735 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:28:14,336] jepsen worker 15 - jepsen.util 735 :invoke :write [5 14] INFO [2019-12-17 12:28:14,353] jepsen worker 16 - jepsen.util 826 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:28:14,353] jepsen worker 16 - jepsen.util 826 :invoke :cas [5 [26 1]] INFO [2019-12-17 12:28:14,363] jepsen worker 31 - jepsen.util 931 :ok :read [10 23] INFO [2019-12-17 12:28:14,363] jepsen worker 31 - jepsen.util 931 :invoke :read [10 nil] INFO [2019-12-17 12:28:14,363] jepsen worker 72 - jepsen.util 792 :fail :cas [24 [37 27]] :ValueNotSwapFrom INFO [2019-12-17 12:28:14,363] jepsen worker 72 - jepsen.util 792 :invoke :cas [24 [45 36]] INFO [2019-12-17 12:28:14,366] jepsen worker 63 - jepsen.util 603 :ok :read [21 9] INFO [2019-12-17 12:28:14,366] jepsen worker 63 - jepsen.util 603 :invoke :write [21 28] INFO [2019-12-17 12:28:14,395] jepsen worker 89 - jepsen.util 989 :ok :read [29 48] INFO [2019-12-17 12:28:14,395] jepsen worker 89 - jepsen.util 989 :invoke :read [29 nil] INFO [2019-12-17 12:28:14,398] jepsen worker 61 - jepsen.util 961 :fail :cas [20 [1 39]] :ValueNotSwapFrom INFO [2019-12-17 12:28:14,399] jepsen worker 61 - jepsen.util 961 :invoke :write [20 26] INFO [2019-12-17 12:28:14,415] jepsen worker 24 - jepsen.util 744 :fail :cas [8 [8 32]] :ValueNotSwapFrom INFO [2019-12-17 12:28:14,416] jepsen worker 24 - jepsen.util 744 :invoke :write [8 44] INFO [2019-12-17 12:28:14,421] jepsen worker 40 - jepsen.util 1120 :info :write [13 20] :RequestCanceledException INFO [2019-12-17 12:28:14,422] jepsen worker 63 - jepsen.util 603 :ok :write [21 28] INFO [2019-12-17 12:28:14,422] jepsen worker 63 - jepsen.util 603 :invoke :write [21 39] INFO [2019-12-17 12:28:14,423] jepsen worker 40 - jepsen.util 1210 :invoke :cas [13 [38 42]] INFO [2019-12-17 12:28:14,442] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4060,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:28:14,442] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4060,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:28:14,442] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4063,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:28:14,443] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4063,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:28:14,443] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4064,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:28:14,443] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4064,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:28:14,452] jepsen worker 33 - jepsen.util 1023 :fail :cas [11 [12 37]] :ValueNotSwapFrom INFO [2019-12-17 12:28:14,452] jepsen worker 33 - jepsen.util 1023 :invoke :cas [11 [42 25]] INFO [2019-12-17 12:28:14,489] jepsen worker 66 - jepsen.util 696 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:28:14,491] jepsen worker 66 - jepsen.util 696 :invoke :cas [22 [19 15]] INFO [2019-12-17 12:28:14,493] jepsen worker 63 - jepsen.util 603 :ok :write [21 39] INFO [2019-12-17 12:28:14,496] jepsen worker 63 - jepsen.util 603 :invoke :read [21 nil] INFO [2019-12-17 12:28:14,502] jepsen worker 80 - jepsen.util 980 :fail :cas [26 [49 27]] :ValueNotSwapFrom INFO [2019-12-17 12:28:14,502] jepsen worker 80 - jepsen.util 980 :invoke :cas [26 [22 46]] INFO [2019-12-17 12:28:14,529] jepsen worker 38 - jepsen.util 758 :ok :read [12 45] INFO [2019-12-17 12:28:14,529] jepsen worker 38 - jepsen.util 758 :invoke :cas [12 [38 33]] INFO [2019-12-17 12:28:14,535] jepsen worker 2 - jepsen.util 632 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:28:14,536] jepsen worker 33 - jepsen.util 1023 :fail :cas [11 [42 25]] :ValueNotSwapFrom INFO [2019-12-17 12:28:14,536] jepsen worker 48 - jepsen.util 498 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:28:14,536] jepsen worker 2 - jepsen.util 632 :invoke :read [0 nil] INFO [2019-12-17 12:28:14,536] jepsen worker 33 - jepsen.util 1023 :invoke :write [11 15] INFO [2019-12-17 12:28:14,536] jepsen worker 48 - jepsen.util 498 :invoke :read [16 nil] INFO [2019-12-17 12:28:14,537] jepsen worker 42 - jepsen.util 1122 :ok :read [14 27] INFO [2019-12-17 12:28:14,538] jepsen worker 42 - jepsen.util 1122 :invoke :cas [14 [32 6]] INFO [2019-12-17 12:28:14,545] jepsen worker 3 - jepsen.util 543 :info :write [1 22] :RequestCanceledException INFO [2019-12-17 12:28:14,548] jepsen worker 3 - jepsen.util 633 :invoke :write [1 43] INFO [2019-12-17 12:28:14,573] jepsen worker 63 - jepsen.util 603 :ok :read [21 39] INFO [2019-12-17 12:28:14,573] jepsen worker 63 - jepsen.util 603 :invoke :cas [21 [29 7]] INFO [2019-12-17 12:28:14,603] jepsen worker 53 - jepsen.util 863 :info :write [17 3] :RequestCanceledException INFO [2019-12-17 12:28:14,603] jepsen worker 47 - jepsen.util 1037 :fail :read [15 nil] :RequestTimeoutException INFO [2019-12-17 12:28:14,604] jepsen worker 47 - jepsen.util 1037 :invoke :read [15 nil] INFO [2019-12-17 12:28:14,604] jepsen worker 63 - jepsen.util 603 :fail :cas [21 [29 7]] :ValueNotSwapFrom INFO [2019-12-17 12:28:14,605] jepsen worker 63 - jepsen.util 603 :invoke :write [21 22] INFO [2019-12-17 12:28:14,605] jepsen worker 53 - jepsen.util 953 :invoke :read [17 nil] INFO [2019-12-17 12:28:14,611] jepsen worker 78 - jepsen.util 888 :info :write [26 33] :RequestCanceledException INFO [2019-12-17 12:28:14,614] jepsen worker 78 - jepsen.util 978 :invoke :read [26 nil] INFO [2019-12-17 12:28:14,621] jepsen worker 56 - jepsen.util 776 :ok :read [18 13] INFO [2019-12-17 12:28:14,622] jepsen worker 56 - jepsen.util 776 :invoke :write [18 7] INFO [2019-12-17 12:28:14,668] jepsen worker 80 - jepsen.util 980 :fail :cas [26 [22 46]] :ValueNotSwapFrom INFO [2019-12-17 12:28:14,669] jepsen worker 63 - jepsen.util 603 :ok :write [21 22] INFO [2019-12-17 12:28:14,669] jepsen worker 80 - jepsen.util 980 :invoke :read [26 nil] INFO [2019-12-17 12:28:14,669] jepsen worker 63 - jepsen.util 603 :invoke :cas [21 [24 43]] INFO [2019-12-17 12:28:14,685] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4084,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:28:14,685] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4084,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:28:14,685] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4088,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:28:14,685] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4088,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:28:14,687] jepsen worker 55 - jepsen.util 1045 :info :write [18 0] :RequestCanceledException INFO [2019-12-17 12:28:14,690] jepsen worker 55 - jepsen.util 1135 :invoke :cas [18 [39 15]] INFO [2019-12-17 12:28:14,693] jepsen worker 38 - jepsen.util 758 :fail :cas [12 [38 33]] :ValueNotSwapFrom INFO [2019-12-17 12:28:14,693] jepsen worker 38 - jepsen.util 758 :invoke :write [12 12] INFO [2019-12-17 12:28:14,702] jepsen worker 63 - jepsen.util 603 :fail :cas [21 [24 43]] :ValueNotSwapFrom INFO [2019-12-17 12:28:14,702] jepsen worker 63 - jepsen.util 603 :invoke :cas [21 [14 19]] INFO [2019-12-17 12:28:14,721] jepsen worker 23 - jepsen.util 653 :info :write [7 48] :RequestCanceledException INFO [2019-12-17 12:28:14,723] jepsen worker 23 - jepsen.util 743 :invoke :cas [7 [26 26]] INFO [2019-12-17 12:28:14,728] jepsen worker 63 - jepsen.util 603 :fail :cas [21 [14 19]] :ValueNotSwapFrom INFO [2019-12-17 12:28:14,729] jepsen worker 63 - jepsen.util 603 :invoke :write [21 16] INFO [2019-12-17 12:28:14,734] jepsen worker 4 - jepsen.util 724 :fail :cas [1 [45 30]] :ValueNotSwapFrom INFO [2019-12-17 12:28:14,734] jepsen worker 4 - jepsen.util 724 :invoke :read [1 nil] INFO [2019-12-17 12:28:14,735] jepsen worker 1 - jepsen.util 631 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:28:14,735] jepsen worker 1 - jepsen.util 631 :invoke :write [0 19] INFO [2019-12-17 12:28:14,752] jepsen worker 34 - jepsen.util 844 :info :write [11 31] :RequestCanceledException INFO [2019-12-17 12:28:14,754] jepsen worker 34 - jepsen.util 934 :invoke :read [11 nil] INFO [2019-12-17 12:28:14,776] jepsen worker 52 - jepsen.util 862 :fail :cas [17 [20 41]] :ValueNotSwapFrom INFO [2019-12-17 12:28:14,776] jepsen worker 52 - jepsen.util 862 :invoke :cas [17 [36 42]] INFO [2019-12-17 12:28:14,786] jepsen worker 63 - jepsen.util 603 :ok :write [21 16] INFO [2019-12-17 12:28:14,786] jepsen worker 63 - jepsen.util 603 :invoke :read [21 nil] INFO [2019-12-17 12:28:14,805] jepsen worker 63 - jepsen.util 603 :ok :read [21 16] INFO [2019-12-17 12:28:14,806] jepsen worker 63 - jepsen.util 603 :invoke :cas [21 [24 7]] INFO [2019-12-17 12:28:14,821] jepsen worker 19 - jepsen.util 829 :info :write [6 1] :RequestCanceledException INFO [2019-12-17 12:28:14,821] jepsen worker 64 - jepsen.util 514 :fail :read [21 nil] :RequestTimeoutException INFO [2019-12-17 12:28:14,821] jepsen worker 64 - jepsen.util 514 :invoke :write [21 4] INFO [2019-12-17 12:28:14,823] jepsen worker 19 - jepsen.util 919 :invoke :write [6 9] INFO [2019-12-17 12:28:14,831] jepsen worker 63 - jepsen.util 603 :fail :cas [21 [24 7]] :ValueNotSwapFrom INFO [2019-12-17 12:28:14,831] jepsen worker 63 - jepsen.util 603 :invoke :write [21 8] INFO [2019-12-17 12:28:14,835] jepsen worker 75 - jepsen.util 795 :info :write [25 22] :RequestCanceledException INFO [2019-12-17 12:28:14,837] jepsen worker 75 - jepsen.util 885 :invoke :read [25 nil] INFO [2019-12-17 12:28:14,840] jepsen worker 62 - jepsen.util 962 :fail :cas [20 [36 34]] :ValueNotSwapFrom INFO [2019-12-17 12:28:14,840] jepsen worker 62 - jepsen.util 962 :invoke :cas [20 [25 19]] INFO [2019-12-17 12:28:14,842] jepsen worker 69 - jepsen.util 879 :ok :read [23 22] INFO [2019-12-17 12:28:14,842] jepsen worker 69 - jepsen.util 879 :invoke :read [23 nil] INFO [2019-12-17 12:28:14,850] jepsen worker 84 - jepsen.util 534 :info :write [28 35] :RequestCanceledException INFO [2019-12-17 12:28:14,850] jepsen worker 20 - jepsen.util 740 :info :write [6 45] :RequestCanceledException INFO [2019-12-17 12:28:14,852] jepsen worker 20 - jepsen.util 830 :invoke :write [6 1] INFO [2019-12-17 12:28:14,852] jepsen worker 84 - jepsen.util 624 :invoke :write [28 12] INFO [2019-12-17 12:28:14,852] jepsen worker 55 - jepsen.util 1135 :fail :cas [18 [39 15]] :ValueNotSwapFrom INFO [2019-12-17 12:28:14,852] jepsen worker 55 - jepsen.util 1135 :invoke :cas [18 [1 4]] INFO [2019-12-17 12:28:14,872] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4112,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:28:14,872] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4112,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:28:14,873] jepsen worker 71 - jepsen.util 701 :ok :read [23 22] INFO [2019-12-17 12:28:14,873] jepsen worker 71 - jepsen.util 701 :invoke :cas [23 [17 9]] INFO [2019-12-17 12:28:14,891] jepsen worker 53 - jepsen.util 953 :ok :read [17 46] INFO [2019-12-17 12:28:14,891] jepsen worker 53 - jepsen.util 953 :invoke :read [17 nil] INFO [2019-12-17 12:28:14,906] jepsen worker 11 - jepsen.util 911 :info :write [3 45] :RequestCanceledException INFO [2019-12-17 12:28:14,906] jepsen worker 86 - jepsen.util 716 :info :cas [28 [24 38]] :RequestCanceledException INFO [2019-12-17 12:28:14,906] jepsen worker 65 - jepsen.util 605 :info :write [21 12] :RequestCanceledException INFO [2019-12-17 12:28:14,907] jepsen worker 65 - jepsen.util 695 :invoke :read [21 nil] INFO [2019-12-17 12:28:14,907] jepsen worker 86 - jepsen.util 806 :invoke :cas [28 [14 34]] INFO [2019-12-17 12:28:14,907] jepsen worker 11 - jepsen.util 1001 :invoke :cas [3 [5 16]] INFO [2019-12-17 12:28:14,908] jepsen worker 64 - jepsen.util 514 :ok :write [21 4] INFO [2019-12-17 12:28:14,909] jepsen worker 64 - jepsen.util 514 :invoke :read [21 nil] INFO [2019-12-17 12:28:14,918] jepsen worker 69 - jepsen.util 879 :ok :read [23 22] INFO [2019-12-17 12:28:14,918] jepsen worker 69 - jepsen.util 879 :invoke :cas [23 [25 1]] INFO [2019-12-17 12:28:14,921] jepsen worker 10 - jepsen.util 820 :info :write [3 17] :RequestCanceledException INFO [2019-12-17 12:28:14,921] jepsen worker 85 - jepsen.util 805 :info :cas [28 [48 39]] :RequestCanceledException INFO [2019-12-17 12:28:14,921] jepsen worker 46 - jepsen.util 856 :info :write [15 15] :RequestCanceledException INFO [2019-12-17 12:28:14,923] jepsen worker 85 - jepsen.util 895 :invoke :read [28 nil] INFO [2019-12-17 12:28:14,923] jepsen worker 10 - jepsen.util 910 :invoke :cas [3 [5 12]] INFO [2019-12-17 12:28:14,923] jepsen worker 46 - jepsen.util 946 :invoke :write [15 38] INFO [2019-12-17 12:28:14,935] jepsen worker 29 - jepsen.util 659 :info :write [9 33] :RequestCanceledException INFO [2019-12-17 12:28:14,935] jepsen worker 81 - jepsen.util 891 :info :cas [27 [21 32]] :RequestCanceledException INFO [2019-12-17 12:28:14,938] jepsen worker 81 - jepsen.util 981 :invoke :write [27 35] INFO [2019-12-17 12:28:14,939] jepsen worker 29 - jepsen.util 749 :invoke :read [9 nil] INFO [2019-12-17 12:28:14,958] jepsen worker 83 - jepsen.util 983 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:28:14,959] jepsen worker 83 - jepsen.util 983 :invoke :write [27 22] INFO [2019-12-17 12:28:14,995] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4128,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:28:14,995] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4128,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:28:14,995] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4132,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:28:15,001] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4132,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:28:15,002] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4133,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:28:15,007] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4133,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:28:15,023] jepsen worker 32 - jepsen.util 752 :info :write [10 25] :RequestCanceledException INFO [2019-12-17 12:28:15,023] jepsen worker 23 - jepsen.util 743 :fail :cas [7 [26 26]] :ValueNotSwapFrom INFO [2019-12-17 12:28:15,026] jepsen worker 39 - jepsen.util 849 :info :write [13 42] :RequestCanceledException INFO [2019-12-17 12:28:15,026] jepsen worker 23 - jepsen.util 743 :invoke :read [7 nil] INFO [2019-12-17 12:28:15,040] jepsen worker 39 - jepsen.util 939 :invoke :write [13 38] INFO [2019-12-17 12:28:15,041] jepsen worker 32 - jepsen.util 842 :invoke :cas [10 [25 11]] INFO [2019-12-17 12:28:15,042] jepsen worker 68 - jepsen.util 428 :info :write [22 15] :RequestCanceledException INFO [2019-12-17 12:28:15,042] jepsen worker 27 - jepsen.util 567 :info :write [9 33] :RequestCanceledException INFO [2019-12-17 12:28:15,042] jepsen worker 30 - jepsen.util 750 :info :write [10 31] :RequestCanceledException INFO [2019-12-17 12:28:15,049] jepsen worker 30 - jepsen.util 840 :invoke :cas [10 [28 40]] INFO [2019-12-17 12:28:15,050] jepsen worker 27 - jepsen.util 657 :invoke :read [9 nil] INFO [2019-12-17 12:28:15,050] jepsen worker 68 - jepsen.util 518 :invoke :write [22 12] INFO [2019-12-17 12:28:15,098] jepsen worker 51 - jepsen.util 771 :fail :cas [17 [21 24]] :ValueNotSwapFrom INFO [2019-12-17 12:28:15,098] jepsen worker 13 - jepsen.util 1003 :fail :cas [4 [9 7]] :ValueNotSwapFrom INFO [2019-12-17 12:28:15,099] jepsen worker 51 - jepsen.util 771 :invoke :write [17 9] INFO [2019-12-17 12:28:15,099] jepsen worker 13 - jepsen.util 1003 :invoke :read [4 nil] INFO [2019-12-17 12:28:15,105] jepsen worker 21 - jepsen.util 921 :info :cas [7 [27 20]] :RequestCanceledException INFO [2019-12-17 12:28:15,110] jepsen worker 21 - jepsen.util 1011 :invoke :cas [7 [29 11]] INFO [2019-12-17 12:28:15,121] jepsen worker 35 - jepsen.util 665 :info :write [11 21] :RequestCanceledException INFO [2019-12-17 12:28:15,121] jepsen worker 77 - jepsen.util 707 :info :write [25 22] :RequestCanceledException INFO [2019-12-17 12:28:15,123] jepsen worker 12 - jepsen.util 822 :fail :cas [4 [40 42]] :ValueNotSwapFrom INFO [2019-12-17 12:28:15,123] jepsen worker 12 - jepsen.util 822 :invoke :write [4 11] INFO [2019-12-17 12:28:15,124] jepsen worker 77 - jepsen.util 797 :invoke :write [25 30] INFO [2019-12-17 12:28:15,124] jepsen worker 89 - jepsen.util 989 :ok :read [29 48] INFO [2019-12-17 12:28:15,124] jepsen worker 89 - jepsen.util 989 :invoke :cas [29 [17 1]] INFO [2019-12-17 12:28:15,125] jepsen worker 35 - jepsen.util 755 :invoke :read [11 nil] INFO [2019-12-17 12:28:15,146] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:28:15,148] jepsen nemesis - couchbase.util Rebalance check count: 25 INFO [2019-12-17 12:28:15,148] jepsen nemesis - couchbase.util Rebalance stuck count: 6 INFO [2019-12-17 12:28:15,154] jepsen worker 27 - jepsen.util 657 :ok :read [9 46] INFO [2019-12-17 12:28:15,154] jepsen worker 7 - jepsen.util 997 :info :write [2 48] :RequestCanceledException INFO [2019-12-17 12:28:15,154] jepsen worker 27 - jepsen.util 657 :invoke :read [9 nil] INFO [2019-12-17 12:28:15,156] jepsen worker 42 - jepsen.util 1122 :fail :cas [14 [32 6]] :ValueNotSwapFrom INFO [2019-12-17 12:28:15,158] jepsen worker 7 - jepsen.util 1087 :invoke :read [2 nil] INFO [2019-12-17 12:28:15,159] jepsen worker 42 - jepsen.util 1122 :invoke :cas [14 [19 36]] INFO [2019-12-17 12:28:15,167] jepsen worker 4 - jepsen.util 724 :ok :read [1 49] INFO [2019-12-17 12:28:15,168] jepsen worker 4 - jepsen.util 724 :invoke :cas [1 [18 31]] INFO [2019-12-17 12:28:15,183] jepsen worker 80 - jepsen.util 980 :ok :read [26 43] INFO [2019-12-17 12:28:15,184] jepsen worker 80 - jepsen.util 980 :invoke :cas [26 [38 0]] INFO [2019-12-17 12:28:15,199] jepsen worker 23 - jepsen.util 743 :ok :read [7 20] INFO [2019-12-17 12:28:15,199] jepsen worker 23 - jepsen.util 743 :invoke :write [7 40] INFO [2019-12-17 12:28:15,202] jepsen worker 27 - jepsen.util 657 :ok :read [9 46] INFO [2019-12-17 12:28:15,202] jepsen worker 27 - jepsen.util 657 :invoke :read [9 nil] INFO [2019-12-17 12:28:15,215] jepsen worker 37 - jepsen.util 757 :info :write [12 49] :RequestCanceledException INFO [2019-12-17 12:28:15,220] jepsen worker 37 - jepsen.util 847 :invoke :read [12 nil] INFO [2019-12-17 12:28:15,234] jepsen worker 28 - jepsen.util 478 :info :cas [9 [35 37]] :RequestCanceledException INFO [2019-12-17 12:28:15,235] jepsen worker 27 - jepsen.util 657 :ok :read [9 46] INFO [2019-12-17 12:28:15,235] jepsen worker 27 - jepsen.util 657 :invoke :read [9 nil] INFO [2019-12-17 12:28:15,235] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestTimeoutException: CarrierBucketConfigRequest {"retried":0,"reason":"TIMEOUT","requestId":3840,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:28:15,236] jepsen worker 28 - jepsen.util 568 :invoke :cas [9 [35 25]] INFO [2019-12-17 12:28:15,259] jepsen worker 27 - jepsen.util 657 :ok :read [9 46] INFO [2019-12-17 12:28:15,259] jepsen worker 27 - jepsen.util 657 :invoke :write [9 14] INFO [2019-12-17 12:28:15,263] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4153,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:28:15,263] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4153,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:28:15,263] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4154,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:28:15,263] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4154,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:28:15,273] jepsen worker 53 - jepsen.util 953 :ok :read [17 46] INFO [2019-12-17 12:28:15,273] jepsen worker 53 - jepsen.util 953 :invoke :read [17 nil] INFO [2019-12-17 12:28:15,282] jepsen worker 74 - jepsen.util 614 :info :cas [24 [1 10]] :RequestCanceledException INFO [2019-12-17 12:28:15,284] jepsen worker 74 - jepsen.util 704 :invoke :read [24 nil] INFO [2019-12-17 12:28:15,285] jepsen worker 13 - jepsen.util 1003 :ok :read [4 49] INFO [2019-12-17 12:28:15,285] jepsen worker 13 - jepsen.util 1003 :invoke :write [4 46] INFO [2019-12-17 12:28:15,293] jepsen worker 52 - jepsen.util 862 :fail :cas [17 [36 42]] :ValueNotSwapFrom INFO [2019-12-17 12:28:15,294] jepsen worker 75 - jepsen.util 885 :ok :read [25 2] INFO [2019-12-17 12:28:15,294] jepsen worker 52 - jepsen.util 862 :invoke :cas [17 [20 23]] INFO [2019-12-17 12:28:15,294] jepsen worker 75 - jepsen.util 885 :invoke :write [25 23] INFO [2019-12-17 12:28:15,303] jepsen worker 57 - jepsen.util 867 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:28:15,303] jepsen worker 57 - jepsen.util 867 :invoke :cas [19 [44 43]] INFO [2019-12-17 12:28:15,304] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4164,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:28:15,305] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4164,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:28:15,312] jepsen worker 59 - jepsen.util 779 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:28:15,312] jepsen worker 59 - jepsen.util 779 :invoke :write [19 0] INFO [2019-12-17 12:28:15,322] jepsen worker 27 - jepsen.util 657 :ok :write [9 14] INFO [2019-12-17 12:28:15,322] jepsen worker 27 - jepsen.util 657 :invoke :write [9 25] INFO [2019-12-17 12:28:15,325] jepsen worker 49 - jepsen.util 679 :info :cas [16 [22 30]] :RequestCanceledException INFO [2019-12-17 12:28:15,327] jepsen worker 49 - jepsen.util 769 :invoke :write [16 42] INFO [2019-12-17 12:28:15,343] jepsen worker 69 - jepsen.util 879 :fail :cas [23 [25 1]] :ValueNotSwapFrom INFO [2019-12-17 12:28:15,344] jepsen worker 69 - jepsen.util 879 :invoke :read [23 nil] INFO [2019-12-17 12:28:15,349] jepsen worker 55 - jepsen.util 1135 :fail :cas [18 [1 4]] :ValueNotSwapFrom INFO [2019-12-17 12:28:15,350] jepsen worker 55 - jepsen.util 1135 :invoke :write [18 9] INFO [2019-12-17 12:28:15,350] jepsen worker 71 - jepsen.util 701 :fail :cas [23 [17 9]] :ValueNotSwapFrom INFO [2019-12-17 12:28:15,351] jepsen worker 71 - jepsen.util 701 :invoke :read [23 nil] INFO [2019-12-17 12:28:15,375] jepsen worker 27 - jepsen.util 657 :ok :write [9 25] INFO [2019-12-17 12:28:15,375] jepsen worker 27 - jepsen.util 657 :invoke :write [9 40] INFO [2019-12-17 12:28:15,387] jepsen worker 22 - jepsen.util 742 :info :write [7 2] :RequestCanceledException INFO [2019-12-17 12:28:15,387] jepsen worker 8 - jepsen.util 818 :info :write [2 45] :RequestCanceledException INFO [2019-12-17 12:28:15,389] jepsen worker 22 - jepsen.util 832 :invoke :read [7 nil] INFO [2019-12-17 12:28:15,389] jepsen worker 8 - jepsen.util 908 :invoke :cas [2 [19 7]] INFO [2019-12-17 12:28:15,405] jepsen worker 60 - jepsen.util 960 :info :write [20 30] :RequestCanceledException INFO [2019-12-17 12:28:15,407] jepsen worker 60 - jepsen.util 1050 :invoke :cas [20 [38 13]] INFO [2019-12-17 12:28:15,429] jepsen worker 53 - jepsen.util 953 :ok :read [17 46] INFO [2019-12-17 12:28:15,429] jepsen worker 53 - jepsen.util 953 :invoke :write [17 23] INFO [2019-12-17 12:28:15,438] jepsen worker 89 - jepsen.util 989 :fail :cas [29 [17 1]] :ValueNotSwapFrom INFO [2019-12-17 12:28:15,438] jepsen worker 89 - jepsen.util 989 :invoke :read [29 nil] INFO [2019-12-17 12:28:15,441] jepsen worker 27 - jepsen.util 657 :ok :write [9 40] INFO [2019-12-17 12:28:15,442] jepsen worker 27 - jepsen.util 657 :invoke :write [9 9] INFO [2019-12-17 12:28:15,458] jepsen worker 34 - jepsen.util 934 :ok :read [11 7] INFO [2019-12-17 12:28:15,459] jepsen worker 34 - jepsen.util 934 :invoke :read [11 nil] INFO [2019-12-17 12:28:15,489] jepsen worker 27 - jepsen.util 657 :ok :write [9 9] INFO [2019-12-17 12:28:15,489] jepsen worker 27 - jepsen.util 657 :invoke :cas [9 [7 10]] INFO [2019-12-17 12:28:15,506] jepsen worker 62 - jepsen.util 962 :fail :cas [20 [25 19]] :ValueNotSwapFrom INFO [2019-12-17 12:28:15,507] jepsen worker 62 - jepsen.util 962 :invoke :read [20 nil] INFO [2019-12-17 12:28:15,510] jepsen worker 73 - jepsen.util 793 :ok :write [24 37] INFO [2019-12-17 12:28:15,510] jepsen worker 73 - jepsen.util 793 :invoke :cas [24 [26 39]] INFO [2019-12-17 12:28:15,512] jepsen worker 27 - jepsen.util 657 :fail :cas [9 [7 10]] :ValueNotSwapFrom INFO [2019-12-17 12:28:15,513] jepsen worker 27 - jepsen.util 657 :invoke :write [9 24] INFO [2019-12-17 12:28:15,518] jepsen worker 21 - jepsen.util 1011 :fail :cas [7 [29 11]] :ValueNotSwapFrom INFO [2019-12-17 12:28:15,518] jepsen worker 21 - jepsen.util 1011 :invoke :write [7 49] INFO [2019-12-17 12:28:15,520] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4189,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:28:15,520] jepsen worker 43 - jepsen.util 943 :info :write [14 33] :RequestCanceledException INFO [2019-12-17 12:28:15,520] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4189,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:28:15,521] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4192,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:28:15,521] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4192,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:28:15,523] jepsen worker 43 - jepsen.util 1033 :invoke :read [14 nil] INFO [2019-12-17 12:28:15,543] jepsen worker 34 - jepsen.util 934 :ok :read [11 7] INFO [2019-12-17 12:28:15,544] jepsen worker 34 - jepsen.util 934 :invoke :write [11 20] INFO [2019-12-17 12:28:15,553] jepsen worker 9 - jepsen.util 729 :ok :write [3 3] INFO [2019-12-17 12:28:15,553] jepsen worker 9 - jepsen.util 729 :invoke :write [3 5] INFO [2019-12-17 12:28:15,554] jepsen worker 45 - jepsen.util 1035 :ok :write [15 17] INFO [2019-12-17 12:28:15,554] jepsen worker 45 - jepsen.util 1035 :invoke :write [15 26] INFO [2019-12-17 12:28:15,563] jepsen worker 18 - jepsen.util 828 :ok :write [6 32] INFO [2019-12-17 12:28:15,564] jepsen worker 18 - jepsen.util 828 :invoke :read [6 nil] INFO [2019-12-17 12:28:15,564] jepsen worker 72 - jepsen.util 792 :fail :cas [24 [45 36]] :ValueNotSwapFrom INFO [2019-12-17 12:28:15,564] jepsen worker 72 - jepsen.util 792 :invoke :write [24 42] INFO [2019-12-17 12:28:15,587] jepsen worker 32 - jepsen.util 842 :fail :cas [10 [25 11]] :ValueNotSwapFrom INFO [2019-12-17 12:28:15,587] jepsen worker 32 - jepsen.util 842 :invoke :read [10 nil] INFO [2019-12-17 12:28:15,601] jepsen worker 73 - jepsen.util 793 :fail :cas [24 [26 39]] :ValueNotSwapFrom INFO [2019-12-17 12:28:15,602] jepsen worker 73 - jepsen.util 793 :invoke :cas [24 [37 38]] INFO [2019-12-17 12:28:15,627] jepsen worker 37 - jepsen.util 847 :ok :read [12 45] INFO [2019-12-17 12:28:15,629] jepsen worker 37 - jepsen.util 847 :invoke :cas [12 [41 2]] INFO [2019-12-17 12:28:15,634] jepsen worker 54 - jepsen.util 864 :info :cas [18 [13 39]] :RequestCanceledException INFO [2019-12-17 12:28:15,646] jepsen worker 54 - jepsen.util 954 :invoke :read [18 nil] INFO [2019-12-17 12:28:15,647] jepsen worker 62 - jepsen.util 962 :ok :read [20 7] INFO [2019-12-17 12:28:15,647] jepsen worker 65 - jepsen.util 695 :ok :read [21 4] INFO [2019-12-17 12:28:15,650] jepsen worker 65 - jepsen.util 695 :invoke :cas [21 [0 38]] INFO [2019-12-17 12:28:15,650] jepsen worker 62 - jepsen.util 962 :invoke :cas [20 [25 37]] INFO [2019-12-17 12:28:15,657] jepsen worker 28 - jepsen.util 568 :fail :cas [9 [35 25]] :ValueNotSwapFrom INFO [2019-12-17 12:28:15,657] jepsen worker 28 - jepsen.util 568 :invoke :cas [9 [20 27]] INFO [2019-12-17 12:28:15,668] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4213,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:28:15,668] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4213,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:28:15,669] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4216,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:28:15,669] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4216,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:28:15,669] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4217,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:28:15,669] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4217,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:28:15,676] jepsen worker 31 - jepsen.util 931 :ok :read [10 23] INFO [2019-12-17 12:28:15,677] jepsen worker 31 - jepsen.util 931 :invoke :write [10 0] INFO [2019-12-17 12:28:15,684] jepsen worker 35 - jepsen.util 755 :ok :read [11 7] INFO [2019-12-17 12:28:15,684] jepsen worker 35 - jepsen.util 755 :invoke :cas [11 [31 37]] INFO [2019-12-17 12:28:15,695] jepsen worker 69 - jepsen.util 879 :ok :read [23 22] INFO [2019-12-17 12:28:15,695] jepsen worker 69 - jepsen.util 879 :invoke :read [23 nil] INFO [2019-12-17 12:28:15,703] jepsen worker 44 - jepsen.util 674 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:28:15,703] jepsen worker 44 - jepsen.util 674 :invoke :cas [14 [38 43]] INFO [2019-12-17 12:28:15,704] jepsen worker 29 - jepsen.util 749 :ok :read [9 9] INFO [2019-12-17 12:28:15,704] jepsen worker 29 - jepsen.util 749 :invoke :write [9 28] INFO [2019-12-17 12:28:15,707] jepsen worker 13 - jepsen.util 1003 :ok :write [4 46] INFO [2019-12-17 12:28:15,707] jepsen worker 13 - jepsen.util 1003 :invoke :cas [4 [5 42]] INFO [2019-12-17 12:28:15,721] jepsen worker 32 - jepsen.util 842 :ok :read [10 23] INFO [2019-12-17 12:28:15,722] jepsen worker 32 - jepsen.util 842 :invoke :cas [10 [10 14]] INFO [2019-12-17 12:28:15,735] jepsen worker 89 - jepsen.util 989 :ok :read [29 48] INFO [2019-12-17 12:28:15,735] jepsen worker 89 - jepsen.util 989 :invoke :write [29 10] INFO [2019-12-17 12:28:15,749] jepsen worker 27 - jepsen.util 657 :ok :write [9 24] INFO [2019-12-17 12:28:15,749] jepsen worker 27 - jepsen.util 657 :invoke :write [9 21] INFO [2019-12-17 12:28:15,770] jepsen worker 74 - jepsen.util 704 :ok :read [24 37] INFO [2019-12-17 12:28:15,771] jepsen worker 74 - jepsen.util 704 :invoke :cas [24 [49 33]] INFO [2019-12-17 12:28:15,787] jepsen worker 36 - jepsen.util 936 :info :write [12 47] :RequestCanceledException INFO [2019-12-17 12:28:15,791] jepsen worker 47 - jepsen.util 1037 :ok :read [15 17] INFO [2019-12-17 12:28:15,791] jepsen worker 47 - jepsen.util 1037 :invoke :cas [15 [45 23]] INFO [2019-12-17 12:28:15,792] jepsen worker 36 - jepsen.util 1026 :invoke :read [12 nil] INFO [2019-12-17 12:28:15,794] jepsen worker 42 - jepsen.util 1122 :fail :cas [14 [19 36]] :ValueNotSwapFrom INFO [2019-12-17 12:28:15,794] jepsen worker 42 - jepsen.util 1122 :invoke :cas [14 [29 19]] INFO [2019-12-17 12:28:15,807] jepsen worker 18 - jepsen.util 828 :ok :read [6 32] INFO [2019-12-17 12:28:15,808] jepsen worker 18 - jepsen.util 828 :invoke :read [6 nil] INFO [2019-12-17 12:28:15,823] jepsen worker 37 - jepsen.util 847 :fail :cas [12 [41 2]] :ValueNotSwapFrom INFO [2019-12-17 12:28:15,824] jepsen worker 37 - jepsen.util 847 :invoke :read [12 nil] INFO [2019-12-17 12:28:15,869] jepsen worker 29 - jepsen.util 749 :ok :write [9 28] INFO [2019-12-17 12:28:15,869] jepsen worker 29 - jepsen.util 749 :invoke :cas [9 [6 35]] INFO [2019-12-17 12:28:15,891] jepsen worker 52 - jepsen.util 862 :fail :cas [17 [20 23]] :ValueNotSwapFrom INFO [2019-12-17 12:28:15,891] jepsen worker 52 - jepsen.util 862 :invoke :cas [17 [10 36]] INFO [2019-12-17 12:28:15,904] jepsen worker 60 - jepsen.util 1050 :fail :cas [20 [38 13]] :ValueNotSwapFrom INFO [2019-12-17 12:28:15,904] jepsen worker 60 - jepsen.util 1050 :invoke :cas [20 [11 41]] INFO [2019-12-17 12:28:15,911] jepsen worker 18 - jepsen.util 828 :ok :read [6 32] INFO [2019-12-17 12:28:15,911] jepsen worker 18 - jepsen.util 828 :invoke :cas [6 [21 17]] INFO [2019-12-17 12:28:15,932] jepsen worker 26 - jepsen.util 926 :info :write [8 6] :RequestCanceledException INFO [2019-12-17 12:28:15,934] jepsen worker 26 - jepsen.util 1016 :invoke :write [8 8] INFO [2019-12-17 12:28:15,942] jepsen worker 36 - jepsen.util 1026 :ok :read [12 45] INFO [2019-12-17 12:28:15,942] jepsen worker 36 - jepsen.util 1026 :invoke :cas [12 [29 17]] INFO [2019-12-17 12:28:15,944] jepsen worker 13 - jepsen.util 1003 :fail :cas [4 [5 42]] :ValueNotSwapFrom INFO [2019-12-17 12:28:15,944] jepsen worker 13 - jepsen.util 1003 :invoke :read [4 nil] INFO [2019-12-17 12:28:15,946] jepsen worker 29 - jepsen.util 749 :fail :cas [9 [6 35]] :ValueNotSwapFrom INFO [2019-12-17 12:28:15,947] jepsen worker 29 - jepsen.util 749 :invoke :cas [9 [9 38]] INFO [2019-12-17 12:28:15,949] jepsen worker 71 - jepsen.util 701 :ok :read [23 22] INFO [2019-12-17 12:28:15,949] jepsen worker 71 - jepsen.util 701 :invoke :write [23 16] INFO [2019-12-17 12:28:15,954] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4238,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:28:15,954] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4238,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:28:15,954] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4241,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:28:15,954] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4241,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:28:15,954] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4242,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:28:15,954] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4242,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:28:15,966] jepsen worker 18 - jepsen.util 828 :fail :cas [6 [21 17]] :ValueNotSwapFrom INFO [2019-12-17 12:28:15,966] jepsen worker 18 - jepsen.util 828 :invoke :read [6 nil] INFO [2019-12-17 12:28:15,984] jepsen worker 29 - jepsen.util 749 :fail :cas [9 [9 38]] :ValueNotSwapFrom INFO [2019-12-17 12:28:15,984] jepsen worker 29 - jepsen.util 749 :invoke :read [9 nil] INFO [2019-12-17 12:28:15,984] jepsen worker 58 - jepsen.util 778 :info :write [19 44] :RequestCanceledException INFO [2019-12-17 12:28:15,986] jepsen worker 58 - jepsen.util 868 :invoke :cas [19 [40 27]] INFO [2019-12-17 12:28:15,989] jepsen worker 43 - jepsen.util 1033 :ok :read [14 27] INFO [2019-12-17 12:28:15,989] jepsen worker 43 - jepsen.util 1033 :invoke :read [14 nil] INFO [2019-12-17 12:28:15,994] jepsen worker 80 - jepsen.util 980 :fail :cas [26 [38 0]] :ValueNotSwapFrom INFO [2019-12-17 12:28:15,994] jepsen worker 80 - jepsen.util 980 :invoke :write [26 29] INFO [2019-12-17 12:28:16,002] jepsen worker 17 - jepsen.util 827 :info :write [5 28] :RequestCanceledException INFO [2019-12-17 12:28:16,004] jepsen worker 17 - jepsen.util 917 :invoke :write [5 18] INFO [2019-12-17 12:28:16,015] jepsen worker 18 - jepsen.util 828 :ok :read [6 32] INFO [2019-12-17 12:28:16,015] jepsen worker 18 - jepsen.util 828 :invoke :write [6 47] INFO [2019-12-17 12:28:16,021] jepsen worker 6 - jepsen.util 816 :info :write [2 9] :RequestCanceledException INFO [2019-12-17 12:28:16,023] jepsen worker 6 - jepsen.util 906 :invoke :write [2 14] INFO [2019-12-17 12:28:16,028] jepsen worker 70 - jepsen.util 1060 :info :write [23 24] :RequestCanceledException INFO [2019-12-17 12:28:16,030] jepsen worker 70 - jepsen.util 1150 :invoke :write [23 6] INFO [2019-12-17 12:28:16,039] jepsen worker 29 - jepsen.util 749 :ok :read [9 28] INFO [2019-12-17 12:28:16,040] jepsen worker 29 - jepsen.util 749 :invoke :read [9 nil] INFO [2019-12-17 12:28:16,047] jepsen worker 5 - jepsen.util 545 :info :write [1 45] :RequestCanceledException INFO [2019-12-17 12:28:16,047] jepsen worker 0 - jepsen.util 990 :info :write [0 11] :RequestCanceledException INFO [2019-12-17 12:28:16,047] jepsen worker 14 - jepsen.util 734 :info :write [4 15] :RequestCanceledException INFO [2019-12-17 12:28:16,049] jepsen worker 14 - jepsen.util 824 :invoke :read [4 nil] INFO [2019-12-17 12:28:16,049] jepsen worker 0 - jepsen.util 1080 :invoke :read [0 nil] INFO [2019-12-17 12:28:16,049] jepsen worker 5 - jepsen.util 635 :invoke :read [1 nil] INFO [2019-12-17 12:28:16,081] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4256,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:28:16,082] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4256,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:28:16,082] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4259,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:28:16,082] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4259,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:28:16,082] jepsen worker 54 - jepsen.util 954 :ok :read [18 13] INFO [2019-12-17 12:28:16,082] jepsen worker 54 - jepsen.util 954 :invoke :write [18 26] INFO [2019-12-17 12:28:16,084] jepsen worker 9 - jepsen.util 729 :ok :write [3 5] INFO [2019-12-17 12:28:16,084] jepsen worker 9 - jepsen.util 729 :invoke :read [3 nil] INFO [2019-12-17 12:28:16,097] jepsen worker 72 - jepsen.util 792 :ok :write [24 42] INFO [2019-12-17 12:28:16,097] jepsen worker 72 - jepsen.util 792 :invoke :cas [24 [46 45]] INFO [2019-12-17 12:28:16,100] jepsen worker 87 - jepsen.util 807 :info :write [29 21] :RequestCanceledException INFO [2019-12-17 12:28:16,104] jepsen worker 87 - jepsen.util 897 :invoke :cas [29 [2 35]] INFO [2019-12-17 12:28:16,112] jepsen worker 43 - jepsen.util 1033 :ok :read [14 27] INFO [2019-12-17 12:28:16,112] jepsen worker 43 - jepsen.util 1033 :invoke :write [14 45] INFO [2019-12-17 12:28:16,118] jepsen worker 67 - jepsen.util 1237 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:28:16,118] jepsen worker 67 - jepsen.util 1237 :invoke :read [22 nil] INFO [2019-12-17 12:28:16,146] jepsen worker 65 - jepsen.util 695 :fail :cas [21 [0 38]] :ValueNotSwapFrom INFO [2019-12-17 12:28:16,146] jepsen worker 4 - jepsen.util 724 :fail :cas [1 [18 31]] :ValueNotSwapFrom INFO [2019-12-17 12:28:16,146] jepsen worker 65 - jepsen.util 695 :invoke :cas [21 [18 32]] INFO [2019-12-17 12:28:16,146] jepsen worker 4 - jepsen.util 724 :invoke :read [1 nil] INFO [2019-12-17 12:28:16,147] jepsen worker 42 - jepsen.util 1122 :fail :cas [14 [29 19]] :ValueNotSwapFrom INFO [2019-12-17 12:28:16,147] jepsen worker 42 - jepsen.util 1122 :invoke :cas [14 [25 6]] INFO [2019-12-17 12:28:16,152] jepsen worker 20 - jepsen.util 830 :ok :write [6 1] INFO [2019-12-17 12:28:16,153] jepsen worker 20 - jepsen.util 830 :invoke :write [6 6] INFO [2019-12-17 12:28:16,175] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:28:16,183] jepsen nemesis - couchbase.util Rebalance check count: 26 INFO [2019-12-17 12:28:16,183] jepsen nemesis - couchbase.util Rebalance stuck count: 7 INFO [2019-12-17 12:28:16,211] jepsen worker 73 - jepsen.util 793 :fail :cas [24 [37 38]] :ValueNotSwapFrom INFO [2019-12-17 12:28:16,215] jepsen worker 73 - jepsen.util 793 :invoke :write [24 0] INFO [2019-12-17 12:28:16,221] jepsen worker 41 - jepsen.util 581 :info :cas [13 [49 32]] :RequestCanceledException INFO [2019-12-17 12:28:16,227] jepsen worker 25 - jepsen.util 1015 :info :write [8 49] :RequestCanceledException INFO [2019-12-17 12:28:16,231] jepsen worker 50 - jepsen.util 950 :info :write [16 15] :RequestCanceledException INFO [2019-12-17 12:28:16,243] jepsen worker 79 - jepsen.util 709 :info :write [26 32] :RequestCanceledException INFO [2019-12-17 12:28:16,244] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4280,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:28:16,244] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4280,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:28:16,247] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4282,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:28:16,247] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4282,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:28:16,284] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4287,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:28:16,289] jepsen worker 25 - jepsen.util 1105 :invoke :cas [8 [11 11]] INFO [2019-12-17 12:28:16,293] jepsen worker 41 - jepsen.util 671 :invoke :cas [13 [6 8]] INFO [2019-12-17 12:28:16,295] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4287,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:28:16,296] jepsen worker 45 - jepsen.util 1035 :ok :write [15 26] INFO [2019-12-17 12:28:16,307] jepsen worker 45 - jepsen.util 1035 :invoke :write [15 25] INFO [2019-12-17 12:28:16,310] jepsen worker 88 - jepsen.util 808 :info :write [29 38] :RequestCanceledException INFO [2019-12-17 12:28:16,311] jepsen worker 76 - jepsen.util 886 :info :write [25 22] :RequestCanceledException INFO [2019-12-17 12:28:16,311] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4289,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:28:16,340] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4289,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:28:16,341] jepsen worker 82 - jepsen.util 1162 :info :cas [27 [27 0]] :RequestCanceledException INFO [2019-12-17 12:28:16,345] jepsen worker 79 - jepsen.util 799 :invoke :read [26 nil] INFO [2019-12-17 12:28:16,349] jepsen worker 50 - jepsen.util 1040 :invoke :cas [16 [28 33]] INFO [2019-12-17 12:28:16,372] jepsen worker 76 - jepsen.util 976 :invoke :read [25 nil] INFO [2019-12-17 12:28:16,374] jepsen worker 82 - jepsen.util 1252 :invoke :cas [27 [8 37]] INFO [2019-12-17 12:28:16,381] jepsen worker 88 - jepsen.util 898 :invoke :write [29 4] INFO [2019-12-17 12:28:16,419] jepsen worker 61 - jepsen.util 961 :info :write [20 26] :RequestCanceledException INFO [2019-12-17 12:28:16,436] jepsen worker 15 - jepsen.util 735 :info :write [5 14] :RequestCanceledException INFO [2019-12-17 12:28:16,468] jepsen worker 40 - jepsen.util 1210 :info :cas [13 [38 42]] :RequestCanceledException INFO [2019-12-17 12:28:16,478] jepsen worker 16 - jepsen.util 826 :info :cas [5 [26 1]] :RequestCanceledException INFO [2019-12-17 12:28:16,492] jepsen worker 24 - jepsen.util 744 :info :write [8 44] :RequestCanceledException INFO [2019-12-17 12:28:16,509] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4301,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:28:16,509] jepsen worker 78 - jepsen.util 978 :ok :read [26 43] INFO [2019-12-17 12:28:16,526] jepsen worker 71 - jepsen.util 701 :ok :write [23 16] INFO [2019-12-17 12:28:16,541] jepsen worker 61 - jepsen.util 1051 :invoke :cas [20 [29 37]] INFO [2019-12-17 12:28:16,542] jepsen worker 15 - jepsen.util 825 :invoke :cas [5 [38 30]] INFO [2019-12-17 12:28:16,580] jepsen worker 71 - jepsen.util 701 :invoke :read [23 nil] INFO [2019-12-17 12:28:16,581] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4301,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:28:16,582] jepsen worker 78 - jepsen.util 978 :invoke :cas [26 [33 29]] INFO [2019-12-17 12:28:16,597] jepsen worker 66 - jepsen.util 696 :info :cas [22 [19 15]] :RequestCanceledException INFO [2019-12-17 12:28:16,607] jepsen worker 40 - jepsen.util 1300 :invoke :write [13 2] INFO [2019-12-17 12:28:16,611] jepsen worker 16 - jepsen.util 916 :invoke :cas [5 [31 21]] INFO [2019-12-17 12:28:16,614] jepsen worker 24 - jepsen.util 834 :invoke :write [8 28] INFO [2019-12-17 12:28:16,623] jepsen worker 3 - jepsen.util 633 :info :write [1 43] :RequestCanceledException INFO [2019-12-17 12:28:16,625] jepsen worker 66 - jepsen.util 786 :invoke :cas [22 [40 36]] INFO [2019-12-17 12:28:16,638] jepsen worker 2 - jepsen.util 632 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:28:16,638] jepsen worker 2 - jepsen.util 632 :invoke :cas [0 [10 23]] INFO [2019-12-17 12:28:16,638] jepsen worker 48 - jepsen.util 498 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:28:16,639] jepsen worker 48 - jepsen.util 498 :invoke :read [16 nil] INFO [2019-12-17 12:28:16,639] jepsen worker 33 - jepsen.util 1023 :info :write [11 15] :RequestCanceledException INFO [2019-12-17 12:28:16,657] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4315,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:28:16,658] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4315,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:28:16,686] jepsen worker 3 - jepsen.util 723 :invoke :read [1 nil] INFO [2019-12-17 12:28:16,705] jepsen worker 33 - jepsen.util 1113 :invoke :cas [11 [21 6]] INFO [2019-12-17 12:28:16,717] jepsen worker 28 - jepsen.util 568 :fail :cas [9 [20 27]] :ValueNotSwapFrom INFO [2019-12-17 12:28:16,719] jepsen worker 28 - jepsen.util 568 :invoke :read [9 nil] INFO [2019-12-17 12:28:16,724] jepsen worker 56 - jepsen.util 776 :info :write [18 7] :RequestCanceledException INFO [2019-12-17 12:28:16,727] jepsen worker 38 - jepsen.util 758 :info :write [12 12] :RequestCanceledException INFO [2019-12-17 12:28:16,739] jepsen worker 55 - jepsen.util 1135 :ok :write [18 9] INFO [2019-12-17 12:28:16,740] jepsen worker 55 - jepsen.util 1135 :invoke :cas [18 [5 9]] INFO [2019-12-17 12:28:16,751] jepsen worker 56 - jepsen.util 866 :invoke :read [18 nil] INFO [2019-12-17 12:28:16,752] jepsen worker 38 - jepsen.util 848 :invoke :read [12 nil] INFO [2019-12-17 12:28:16,797] jepsen worker 9 - jepsen.util 729 :ok :read [3 5] INFO [2019-12-17 12:28:16,798] jepsen worker 9 - jepsen.util 729 :invoke :cas [3 [4 35]] INFO [2019-12-17 12:28:16,807] jepsen worker 69 - jepsen.util 879 :ok :read [23 22] INFO [2019-12-17 12:28:16,808] jepsen worker 32 - jepsen.util 842 :fail :cas [10 [10 14]] :ValueNotSwapFrom INFO [2019-12-17 12:28:16,808] jepsen worker 32 - jepsen.util 842 :invoke :write [10 27] INFO [2019-12-17 12:28:16,809] jepsen worker 69 - jepsen.util 879 :invoke :read [23 nil] INFO [2019-12-17 12:28:16,815] jepsen worker 60 - jepsen.util 1050 :fail :cas [20 [11 41]] :ValueNotSwapFrom INFO [2019-12-17 12:28:16,815] jepsen worker 60 - jepsen.util 1050 :invoke :cas [20 [43 23]] INFO [2019-12-17 12:28:16,827] jepsen worker 72 - jepsen.util 792 :fail :cas [24 [46 45]] :ValueNotSwapFrom INFO [2019-12-17 12:28:16,827] jepsen worker 52 - jepsen.util 862 :fail :cas [17 [10 36]] :ValueNotSwapFrom INFO [2019-12-17 12:28:16,827] jepsen worker 72 - jepsen.util 792 :invoke :cas [24 [10 12]] INFO [2019-12-17 12:28:16,827] jepsen worker 52 - jepsen.util 862 :invoke :read [17 nil] INFO [2019-12-17 12:28:16,831] jepsen worker 37 - jepsen.util 847 :ok :read [12 45] INFO [2019-12-17 12:28:16,831] jepsen worker 22 - jepsen.util 832 :ok :read [7 20] INFO [2019-12-17 12:28:16,833] jepsen worker 22 - jepsen.util 832 :invoke :write [7 2] INFO [2019-12-17 12:28:16,834] jepsen worker 37 - jepsen.util 847 :invoke :cas [12 [4 36]] INFO [2019-12-17 12:28:16,834] jepsen worker 4 - jepsen.util 724 :ok :read [1 49] INFO [2019-12-17 12:28:16,835] jepsen worker 4 - jepsen.util 724 :invoke :write [1 20] INFO [2019-12-17 12:28:16,835] jepsen worker 65 - jepsen.util 695 :fail :cas [21 [18 32]] :ValueNotSwapFrom INFO [2019-12-17 12:28:16,835] jepsen worker 1 - jepsen.util 631 :info :write [0 19] :RequestCanceledException INFO [2019-12-17 12:28:16,836] jepsen worker 65 - jepsen.util 695 :invoke :write [21 35] INFO [2019-12-17 12:28:16,836] jepsen worker 62 - jepsen.util 962 :fail :cas [20 [25 37]] :ValueNotSwapFrom INFO [2019-12-17 12:28:16,836] jepsen worker 62 - jepsen.util 962 :invoke :cas [20 [13 13]] INFO [2019-12-17 12:28:16,841] jepsen worker 87 - jepsen.util 897 :fail :cas [29 [2 35]] :ValueNotSwapFrom INFO [2019-12-17 12:28:16,841] jepsen worker 87 - jepsen.util 897 :invoke :read [29 nil] INFO [2019-12-17 12:28:16,844] jepsen worker 1 - jepsen.util 721 :invoke :read [0 nil] INFO [2019-12-17 12:28:16,853] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4341,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:28:16,853] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4341,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:28:16,854] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4344,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:28:16,854] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4344,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:28:16,858] jepsen worker 13 - jepsen.util 1003 :ok :read [4 46] INFO [2019-12-17 12:28:16,859] jepsen worker 13 - jepsen.util 1003 :invoke :read [4 nil] INFO [2019-12-17 12:28:16,861] jepsen worker 28 - jepsen.util 568 :ok :read [9 28] INFO [2019-12-17 12:28:16,862] jepsen worker 28 - jepsen.util 568 :invoke :write [9 6] INFO [2019-12-17 12:28:16,880] jepsen worker 9 - jepsen.util 729 :fail :cas [3 [4 35]] :ValueNotSwapFrom INFO [2019-12-17 12:28:16,880] jepsen worker 55 - jepsen.util 1135 :fail :cas [18 [5 9]] :ValueNotSwapFrom INFO [2019-12-17 12:28:16,880] jepsen worker 9 - jepsen.util 729 :invoke :cas [3 [14 33]] INFO [2019-12-17 12:28:16,880] jepsen worker 55 - jepsen.util 1135 :invoke :read [18 nil] INFO [2019-12-17 12:28:16,899] jepsen worker 53 - jepsen.util 953 :ok :write [17 23] INFO [2019-12-17 12:28:16,899] jepsen worker 53 - jepsen.util 953 :invoke :read [17 nil] INFO [2019-12-17 12:28:16,919] jepsen worker 84 - jepsen.util 624 :info :write [28 12] :RequestCanceledException INFO [2019-12-17 12:28:16,922] jepsen worker 19 - jepsen.util 919 :info :write [6 9] :RequestCanceledException INFO [2019-12-17 12:28:16,922] jepsen worker 63 - jepsen.util 603 :info :write [21 8] :RequestCanceledException INFO [2019-12-17 12:28:16,922] jepsen worker 11 - jepsen.util 1001 :info :cas [3 [5 16]] :RequestCanceledException INFO [2019-12-17 12:28:16,922] jepsen worker 64 - jepsen.util 514 :fail :read [21 nil] :RequestTimeoutException INFO [2019-12-17 12:28:16,922] jepsen worker 64 - jepsen.util 514 :invoke :cas [21 [15 17]] INFO [2019-12-17 12:28:16,923] jepsen worker 84 - jepsen.util 714 :invoke :cas [28 [4 40]] INFO [2019-12-17 12:28:16,924] jepsen worker 11 - jepsen.util 1091 :invoke :read [3 nil] INFO [2019-12-17 12:28:16,924] jepsen worker 19 - jepsen.util 1009 :invoke :write [6 39] INFO [2019-12-17 12:28:16,924] jepsen worker 63 - jepsen.util 693 :invoke :cas [21 [33 21]] INFO [2019-12-17 12:28:16,945] jepsen worker 42 - jepsen.util 1122 :fail :cas [14 [25 6]] :ValueNotSwapFrom INFO [2019-12-17 12:28:16,945] jepsen worker 42 - jepsen.util 1122 :invoke :read [14 nil] INFO [2019-12-17 12:28:16,949] jepsen worker 86 - jepsen.util 806 :info :cas [28 [14 34]] :RequestCanceledException INFO [2019-12-17 12:28:16,951] jepsen worker 86 - jepsen.util 896 :invoke :cas [28 [29 40]] INFO [2019-12-17 12:28:16,962] jepsen worker 11 - jepsen.util 1091 :ok :read [3 5] INFO [2019-12-17 12:28:16,963] jepsen worker 11 - jepsen.util 1091 :invoke :write [3 23] INFO [2019-12-17 12:28:16,987] jepsen worker 85 - jepsen.util 895 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:28:16,987] jepsen worker 85 - jepsen.util 895 :invoke :read [28 nil] INFO [2019-12-17 12:28:16,993] jepsen worker 65 - jepsen.util 695 :ok :write [21 35] INFO [2019-12-17 12:28:16,993] jepsen worker 65 - jepsen.util 695 :invoke :read [21 nil] INFO [2019-12-17 12:28:17,003] jepsen worker 10 - jepsen.util 910 :info :cas [3 [5 12]] :RequestCanceledException INFO [2019-12-17 12:28:17,005] jepsen worker 10 - jepsen.util 1000 :invoke :cas [3 [21 8]] INFO [2019-12-17 12:28:17,006] jepsen worker 64 - jepsen.util 514 :fail :cas [21 [15 17]] :ValueNotSwapFrom INFO [2019-12-17 12:28:17,006] jepsen worker 64 - jepsen.util 514 :invoke :read [21 nil] INFO [2019-12-17 12:28:17,017] jepsen worker 46 - jepsen.util 946 :info :write [15 38] :RequestCanceledException INFO [2019-12-17 12:28:17,020] jepsen worker 37 - jepsen.util 847 :fail :cas [12 [4 36]] :ValueNotSwapFrom INFO [2019-12-17 12:28:17,020] jepsen worker 37 - jepsen.util 847 :invoke :cas [12 [7 38]] INFO [2019-12-17 12:28:17,022] jepsen worker 46 - jepsen.util 1036 :invoke :cas [15 [21 22]] INFO [2019-12-17 12:28:17,025] jepsen worker 81 - jepsen.util 981 :info :write [27 35] :RequestCanceledException INFO [2019-12-17 12:28:17,029] jepsen worker 33 - jepsen.util 1113 :fail :cas [11 [21 6]] :ValueNotSwapFrom INFO [2019-12-17 12:28:17,030] jepsen worker 33 - jepsen.util 1113 :invoke :cas [11 [37 27]] INFO [2019-12-17 12:28:17,030] jepsen worker 81 - jepsen.util 1071 :invoke :cas [27 [31 35]] INFO [2019-12-17 12:28:17,055] jepsen worker 65 - jepsen.util 695 :ok :read [21 35] INFO [2019-12-17 12:28:17,056] jepsen worker 65 - jepsen.util 695 :invoke :read [21 nil] INFO [2019-12-17 12:28:17,079] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4363,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:28:17,079] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4363,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:28:17,080] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4366,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:28:17,080] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4366,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:28:17,080] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4367,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:28:17,080] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4367,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:28:17,082] jepsen worker 10 - jepsen.util 1000 :fail :cas [3 [21 8]] :ValueNotSwapFrom INFO [2019-12-17 12:28:17,082] jepsen worker 10 - jepsen.util 1000 :invoke :write [3 45] INFO [2019-12-17 12:28:17,082] jepsen worker 46 - jepsen.util 1036 :fail :cas [15 [21 22]] :ValueNotSwapFrom INFO [2019-12-17 12:28:17,083] jepsen worker 46 - jepsen.util 1036 :invoke :cas [15 [42 49]] INFO [2019-12-17 12:28:17,084] jepsen worker 42 - jepsen.util 1122 :ok :read [14 27] INFO [2019-12-17 12:28:17,084] jepsen worker 42 - jepsen.util 1122 :invoke :write [14 27] INFO [2019-12-17 12:28:17,103] jepsen worker 30 - jepsen.util 840 :info :cas [10 [28 40]] :RequestCanceledException INFO [2019-12-17 12:28:17,105] jepsen worker 30 - jepsen.util 930 :invoke :write [10 10] INFO [2019-12-17 12:28:17,118] jepsen worker 39 - jepsen.util 939 :info :write [13 38] :RequestCanceledException INFO [2019-12-17 12:28:17,120] jepsen worker 39 - jepsen.util 1029 :invoke :write [13 9] INFO [2019-12-17 12:28:17,128] jepsen worker 68 - jepsen.util 518 :info :write [22 12] :RequestCanceledException INFO [2019-12-17 12:28:17,133] jepsen worker 68 - jepsen.util 608 :invoke :read [22 nil] INFO [2019-12-17 12:28:17,139] jepsen worker 56 - jepsen.util 866 :ok :read [18 9] INFO [2019-12-17 12:28:17,139] jepsen worker 56 - jepsen.util 866 :invoke :write [18 14] INFO [2019-12-17 12:28:17,146] jepsen worker 51 - jepsen.util 771 :info :write [17 9] :RequestCanceledException INFO [2019-12-17 12:28:17,146] jepsen worker 83 - jepsen.util 983 :info :write [27 22] :RequestCanceledException INFO [2019-12-17 12:28:17,148] jepsen worker 83 - jepsen.util 1073 :invoke :write [27 18] INFO [2019-12-17 12:28:17,148] jepsen worker 51 - jepsen.util 861 :invoke :read [17 nil] INFO [2019-12-17 12:28:17,151] jepsen worker 65 - jepsen.util 695 :ok :read [21 35] INFO [2019-12-17 12:28:17,151] jepsen worker 65 - jepsen.util 695 :invoke :write [21 39] INFO [2019-12-17 12:28:17,200] jepsen worker 46 - jepsen.util 1036 :fail :cas [15 [42 49]] :ValueNotSwapFrom INFO [2019-12-17 12:28:17,200] jepsen worker 46 - jepsen.util 1036 :invoke :write [15 41] INFO [2019-12-17 12:28:17,200] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:28:17,200] jepsen nemesis - couchbase.util Rebalance check count: 27 INFO [2019-12-17 12:28:17,200] jepsen nemesis - couchbase.util Rebalance stuck count: 8 INFO [2019-12-17 12:28:17,206] jepsen worker 12 - jepsen.util 822 :info :write [4 11] :RequestCanceledException INFO [2019-12-17 12:28:17,206] jepsen worker 7 - jepsen.util 1087 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:28:17,207] jepsen worker 7 - jepsen.util 1087 :invoke :write [2 36] INFO [2019-12-17 12:28:17,211] jepsen worker 12 - jepsen.util 912 :invoke :cas [4 [42 20]] INFO [2019-12-17 12:28:17,220] jepsen worker 77 - jepsen.util 797 :info :write [25 30] :RequestCanceledException INFO [2019-12-17 12:28:17,223] jepsen worker 87 - jepsen.util 897 :ok :read [29 48] INFO [2019-12-17 12:28:17,223] jepsen worker 87 - jepsen.util 897 :invoke :write [29 23] INFO [2019-12-17 12:28:17,224] jepsen worker 77 - jepsen.util 887 :invoke :write [25 10] INFO [2019-12-17 12:28:17,225] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4394,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:28:17,226] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4394,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:28:17,226] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4398,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:28:17,226] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4398,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:28:17,226] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4387,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:28:17,226] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4387,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:28:17,226] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4388,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:28:17,226] jepsen worker 23 - jepsen.util 743 :info :write [7 40] :RequestCanceledException INFO [2019-12-17 12:28:17,226] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4388,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:28:17,227] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4389,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:28:17,227] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4389,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:28:17,234] jepsen worker 23 - jepsen.util 833 :invoke :cas [7 [11 10]] INFO [2019-12-17 12:28:17,279] jepsen worker 65 - jepsen.util 695 :ok :write [21 39] INFO [2019-12-17 12:28:17,298] jepsen worker 73 - jepsen.util 793 :ok :write [24 0] INFO [2019-12-17 12:28:17,308] jepsen worker 5 - jepsen.util 635 :ok :read [1 49] INFO [2019-12-17 12:28:17,332] jepsen worker 88 - jepsen.util 898 :ok :write [29 4] INFO [2019-12-17 12:28:17,361] jepsen worker 5 - jepsen.util 635 :invoke :read [1 nil] INFO [2019-12-17 12:28:17,361] jepsen worker 88 - jepsen.util 898 :invoke :cas [29 [39 49]] INFO [2019-12-17 12:28:17,363] jepsen worker 73 - jepsen.util 793 :invoke :cas [24 [11 46]] INFO [2019-12-17 12:28:17,366] jepsen worker 65 - jepsen.util 695 :invoke :read [21 nil] INFO [2019-12-17 12:28:17,403] jepsen worker 4 - jepsen.util 724 :ok :write [1 20] INFO [2019-12-17 12:28:17,404] jepsen worker 75 - jepsen.util 885 :info :write [25 23] :RequestCanceledException INFO [2019-12-17 12:28:17,416] jepsen worker 20 - jepsen.util 830 :ok :write [6 6] INFO [2019-12-17 12:28:17,419] jepsen worker 4 - jepsen.util 724 :invoke :cas [1 [35 41]] INFO [2019-12-17 12:28:17,425] jepsen worker 20 - jepsen.util 830 :invoke :write [6 47] INFO [2019-12-17 12:28:17,438] jepsen worker 49 - jepsen.util 769 :info :write [16 42] :RequestCanceledException INFO [2019-12-17 12:28:17,439] jepsen worker 59 - jepsen.util 779 :info :write [19 0] :RequestCanceledException INFO [2019-12-17 12:28:17,440] jepsen worker 70 - jepsen.util 1150 :ok :write [23 6] INFO [2019-12-17 12:28:17,443] jepsen worker 57 - jepsen.util 867 :info :cas [19 [44 43]] :RequestCanceledException INFO [2019-12-17 12:28:17,445] jepsen worker 70 - jepsen.util 1150 :invoke :read [23 nil] INFO [2019-12-17 12:28:17,445] jepsen worker 8 - jepsen.util 908 :info :cas [2 [19 7]] :RequestCanceledException INFO [2019-12-17 12:28:17,447] jepsen worker 75 - jepsen.util 975 :invoke :read [25 nil] INFO [2019-12-17 12:28:17,466] jepsen worker 11 - jepsen.util 1091 :ok :write [3 23] INFO [2019-12-17 12:28:17,477] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4414,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:28:17,493] jepsen worker 54 - jepsen.util 954 :ok :write [18 26] INFO [2019-12-17 12:28:17,493] jepsen worker 11 - jepsen.util 1091 :invoke :write [3 19] INFO [2019-12-17 12:28:17,502] jepsen worker 54 - jepsen.util 954 :invoke :write [18 39] INFO [2019-12-17 12:28:17,517] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4414,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:28:17,525] jepsen worker 59 - jepsen.util 869 :invoke :read [19 nil] INFO [2019-12-17 12:28:17,526] jepsen worker 45 - jepsen.util 1035 :ok :write [15 25] INFO [2019-12-17 12:28:17,526] jepsen worker 57 - jepsen.util 957 :invoke :cas [19 [9 28]] INFO [2019-12-17 12:28:17,536] jepsen worker 45 - jepsen.util 1035 :invoke :cas [15 [3 48]] INFO [2019-12-17 12:28:17,540] jepsen worker 32 - jepsen.util 842 :ok :write [10 27] INFO [2019-12-17 12:28:17,543] jepsen worker 32 - jepsen.util 842 :invoke :cas [10 [22 2]] INFO [2019-12-17 12:28:17,545] jepsen worker 49 - jepsen.util 859 :invoke :cas [16 [47 31]] INFO [2019-12-17 12:28:17,554] jepsen worker 56 - jepsen.util 866 :ok :write [18 14] INFO [2019-12-17 12:28:17,558] jepsen worker 56 - jepsen.util 866 :invoke :write [18 39] INFO [2019-12-17 12:28:17,558] jepsen worker 8 - jepsen.util 998 :invoke :cas [2 [25 49]] INFO [2019-12-17 12:28:17,558] jepsen worker 21 - jepsen.util 1011 :info :write [7 49] :RequestCanceledException INFO [2019-12-17 12:28:17,596] jepsen worker 89 - jepsen.util 989 :ok :write [29 10] INFO [2019-12-17 12:28:17,600] jepsen worker 89 - jepsen.util 989 :invoke :write [29 37] INFO [2019-12-17 12:28:17,637] jepsen worker 21 - jepsen.util 1101 :invoke :read [7 nil] INFO [2019-12-17 12:28:17,670] jepsen worker 34 - jepsen.util 934 :info :write [11 20] :RequestCanceledException INFO [2019-12-17 12:28:17,708] jepsen worker 31 - jepsen.util 931 :info :write [10 0] :RequestCanceledException INFO [2019-12-17 12:28:17,742] jepsen worker 34 - jepsen.util 1024 :invoke :cas [11 [30 27]] INFO [2019-12-17 12:28:17,782] jepsen worker 31 - jepsen.util 1021 :invoke :cas [10 [28 17]] INFO [2019-12-17 12:28:17,827] jepsen worker 47 - jepsen.util 1037 :info :cas [15 [45 23]] :RequestCanceledException INFO [2019-12-17 12:28:17,832] jepsen worker 44 - jepsen.util 674 :info :cas [14 [38 43]] :RequestCanceledException INFO [2019-12-17 12:28:17,850] jepsen worker 27 - jepsen.util 657 :info :write [9 21] :RequestCanceledException INFO [2019-12-17 12:28:17,866] jepsen worker 74 - jepsen.util 704 :info :cas [24 [49 33]] :RequestCanceledException INFO [2019-12-17 12:28:17,890] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4437,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:28:17,905] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4437,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:28:17,913] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4440,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:28:17,930] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4440,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:28:17,964] jepsen worker 69 - jepsen.util 879 :ok :read [23 16] INFO [2019-12-17 12:28:17,993] jepsen worker 69 - jepsen.util 879 :invoke :write [23 4] INFO [2019-12-17 12:28:18,009] jepsen worker 44 - jepsen.util 764 :invoke :cas [14 [29 29]] INFO [2019-12-17 12:28:18,010] jepsen worker 26 - jepsen.util 1016 :info :write [8 8] :RequestCanceledException INFO [2019-12-17 12:28:18,038] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4445,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:28:18,039] jepsen worker 47 - jepsen.util 1127 :invoke :cas [15 [35 33]] INFO [2019-12-17 12:28:18,041] jepsen worker 58 - jepsen.util 868 :info :cas [19 [40 27]] :RequestCanceledException INFO [2019-12-17 12:28:18,045] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4445,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:28:18,145] jepsen worker 80 - jepsen.util 980 :info :write [26 29] :RequestCanceledException INFO [2019-12-17 12:28:18,145] jepsen worker 74 - jepsen.util 794 :invoke :write [24 11] INFO [2019-12-17 12:28:18,146] jepsen worker 27 - jepsen.util 747 :invoke :cas [9 [45 11]] INFO [2019-12-17 12:28:18,146] jepsen worker 29 - jepsen.util 749 :fail :read [9 nil] :RequestTimeoutException INFO [2019-12-17 12:28:18,147] jepsen worker 18 - jepsen.util 828 :info :write [6 47] :RequestCanceledException INFO [2019-12-17 12:28:18,148] jepsen worker 29 - jepsen.util 749 :invoke :read [9 nil] INFO [2019-12-17 12:28:18,149] jepsen worker 58 - jepsen.util 958 :invoke :read [19 nil] INFO [2019-12-17 12:28:18,152] jepsen worker 35 - jepsen.util 755 :fail :cas [11 [31 37]] :ValueNotSwapFrom INFO [2019-12-17 12:28:18,152] jepsen worker 35 - jepsen.util 755 :invoke :read [11 nil] INFO [2019-12-17 12:28:18,155] jepsen worker 6 - jepsen.util 906 :info :write [2 14] :RequestCanceledException INFO [2019-12-17 12:28:18,166] jepsen worker 64 - jepsen.util 514 :ok :read [21 35] INFO [2019-12-17 12:28:18,166] jepsen worker 64 - jepsen.util 514 :invoke :write [21 21] INFO [2019-12-17 12:28:18,176] jepsen worker 17 - jepsen.util 917 :info :write [5 18] :RequestCanceledException INFO [2019-12-17 12:28:18,182] jepsen worker 0 - jepsen.util 1080 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:28:19,879] jepsen worker 0 - jepsen.util 1080 :invoke :write [0 6] INFO [2019-12-17 12:28:19,879] jepsen worker 26 - jepsen.util 1106 :invoke :read [8 nil] INFO [2019-12-17 12:28:19,879] jepsen worker 18 - jepsen.util 918 :invoke :cas [6 [37 25]] INFO [2019-12-17 12:28:19,879] jepsen worker 17 - jepsen.util 1007 :invoke :read [5 nil] INFO [2019-12-17 12:28:19,879] jepsen worker 80 - jepsen.util 1070 :invoke :write [26 49] INFO [2019-12-17 12:28:19,881] jepsen worker 6 - jepsen.util 996 :invoke :write [2 15] INFO [2019-12-17 12:28:19,883] jepsen worker 36 - jepsen.util 1026 :fail :cas [12 [29 17]] :ValueNotSwapFrom INFO [2019-12-17 12:28:19,884] jepsen worker 36 - jepsen.util 1026 :invoke :cas [12 [44 29]] INFO [2019-12-17 12:28:19,887] jepsen worker 32 - jepsen.util 842 :fail :cas [10 [22 2]] :ValueNotSwapFrom INFO [2019-12-17 12:28:19,887] jepsen worker 32 - jepsen.util 842 :invoke :read [10 nil] INFO [2019-12-17 12:28:19,890] jepsen worker 5 - jepsen.util 635 :ok :read [1 20] INFO [2019-12-17 12:28:19,890] jepsen worker 51 - jepsen.util 861 :ok :read [17 9] INFO [2019-12-17 12:28:19,890] jepsen worker 5 - jepsen.util 635 :invoke :read [1 nil] INFO [2019-12-17 12:28:19,891] jepsen worker 51 - jepsen.util 861 :invoke :read [17 nil] INFO [2019-12-17 12:28:19,893] jepsen worker 55 - jepsen.util 1135 :ok :read [18 9] INFO [2019-12-17 12:28:19,893] jepsen worker 25 - jepsen.util 1105 :fail :cas [8 [11 11]] :ValueNotSwapFrom INFO [2019-12-17 12:28:19,893] jepsen worker 25 - jepsen.util 1105 :invoke :write [8 3] INFO [2019-12-17 12:28:19,893] jepsen worker 55 - jepsen.util 1135 :invoke :cas [18 [45 34]] INFO [2019-12-17 12:28:19,893] jepsen worker 88 - jepsen.util 898 :fail :cas [29 [39 49]] :ValueNotSwapFrom INFO [2019-12-17 12:28:19,895] jepsen worker 88 - jepsen.util 898 :invoke :cas [29 [21 44]] INFO [2019-12-17 12:28:19,897] jepsen worker 71 - jepsen.util 701 :ok :read [23 6] INFO [2019-12-17 12:28:19,897] jepsen worker 71 - jepsen.util 701 :invoke :write [23 5] INFO [2019-12-17 12:28:19,898] jepsen worker 60 - jepsen.util 1050 :fail :cas [20 [43 23]] :ValueNotSwapFrom INFO [2019-12-17 12:28:19,899] jepsen worker 60 - jepsen.util 1050 :invoke :write [20 10] INFO [2019-12-17 12:28:19,904] jepsen worker 14 - jepsen.util 824 :ok :read [4 11] INFO [2019-12-17 12:28:19,904] jepsen worker 31 - jepsen.util 1021 :fail :cas [10 [28 17]] :ValueNotSwapFrom INFO [2019-12-17 12:28:19,904] jepsen worker 14 - jepsen.util 824 :invoke :read [4 nil] INFO [2019-12-17 12:28:19,904] jepsen worker 3 - jepsen.util 723 :ok :read [1 43] INFO [2019-12-17 12:28:19,904] jepsen worker 31 - jepsen.util 1021 :invoke :write [10 26] INFO [2019-12-17 12:28:19,907] jepsen worker 3 - jepsen.util 723 :invoke :write [1 21] INFO [2019-12-17 12:28:19,907] jepsen worker 78 - jepsen.util 978 :fail :cas [26 [33 29]] :ValueNotSwapFrom INFO [2019-12-17 12:28:19,908] jepsen worker 73 - jepsen.util 793 :fail :cas [24 [11 46]] :ValueNotSwapFrom INFO [2019-12-17 12:28:19,911] jepsen worker 65 - jepsen.util 695 :ok :read [21 39] INFO [2019-12-17 12:28:19,912] jepsen worker 45 - jepsen.util 1035 :fail :cas [15 [3 48]] :ValueNotSwapFrom INFO [2019-12-17 12:28:19,912] jepsen worker 78 - jepsen.util 978 :invoke :read [26 nil] INFO [2019-12-17 12:28:19,913] jepsen worker 72 - jepsen.util 792 :fail :cas [24 [10 12]] :ValueNotSwapFrom INFO [2019-12-17 12:28:19,913] jepsen worker 62 - jepsen.util 962 :fail :cas [20 [13 13]] :ValueNotSwapFrom INFO [2019-12-17 12:28:19,914] jepsen worker 73 - jepsen.util 793 :invoke :read [24 nil] INFO [2019-12-17 12:28:19,914] jepsen worker 53 - jepsen.util 953 :ok :read [17 23] INFO [2019-12-17 12:28:19,915] jepsen worker 65 - jepsen.util 695 :invoke :read [21 nil] INFO [2019-12-17 12:28:19,915] jepsen worker 37 - jepsen.util 847 :fail :cas [12 [7 38]] :ValueNotSwapFrom INFO [2019-12-17 12:28:19,915] jepsen worker 13 - jepsen.util 1003 :ok :read [4 46] INFO [2019-12-17 12:28:19,916] jepsen worker 61 - jepsen.util 1051 :fail :cas [20 [29 37]] :ValueNotSwapFrom INFO [2019-12-17 12:28:19,916] jepsen worker 45 - jepsen.util 1035 :invoke :read [15 nil] INFO [2019-12-17 12:28:19,916] jepsen worker 27 - jepsen.util 747 :fail :cas [9 [45 11]] :ValueNotSwapFrom INFO [2019-12-17 12:28:19,916] jepsen worker 72 - jepsen.util 792 :invoke :cas [24 [40 9]] INFO [2019-12-17 12:28:19,916] jepsen worker 4 - jepsen.util 724 :fail :cas [1 [35 41]] :ValueNotSwapFrom INFO [2019-12-17 12:28:19,917] jepsen worker 62 - jepsen.util 962 :invoke :write [20 11] INFO [2019-12-17 12:28:19,917] jepsen worker 53 - jepsen.util 953 :invoke :read [17 nil] INFO [2019-12-17 12:28:19,917] jepsen worker 37 - jepsen.util 847 :invoke :cas [12 [20 10]] INFO [2019-12-17 12:28:19,917] jepsen worker 13 - jepsen.util 1003 :invoke :read [4 nil] INFO [2019-12-17 12:28:19,918] jepsen worker 61 - jepsen.util 1051 :invoke :read [20 nil] INFO [2019-12-17 12:28:19,919] jepsen worker 38 - jepsen.util 848 :ok :read [12 12] INFO [2019-12-17 12:28:19,919] jepsen worker 27 - jepsen.util 747 :invoke :read [9 nil] INFO [2019-12-17 12:28:19,920] jepsen worker 85 - jepsen.util 895 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:28:19,920] jepsen worker 50 - jepsen.util 1040 :info :cas [16 [28 33]] :RequestCanceledException INFO [2019-12-17 12:28:19,920] jepsen worker 39 - jepsen.util 1029 :info :write [13 9] :RequestCanceledException INFO [2019-12-17 12:28:19,920] jepsen worker 4 - jepsen.util 724 :invoke :read [1 nil] INFO [2019-12-17 12:28:19,920] jepsen worker 22 - jepsen.util 832 :info :write [7 2] :RequestCanceledException INFO [2019-12-17 12:28:19,920] jepsen worker 40 - jepsen.util 1300 :info :write [13 2] :RequestCanceledException INFO [2019-12-17 12:28:19,920] jepsen worker 63 - jepsen.util 693 :info :cas [21 [33 21]] :RequestCanceledException INFO [2019-12-17 12:28:19,920] jepsen worker 21 - jepsen.util 1101 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:28:19,920] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4513,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:28:19,921] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4465,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:28:19,921] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4456,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:28:19,922] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4464,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:28:19,922] jepsen worker 38 - jepsen.util 848 :invoke :cas [12 [17 11]] INFO [2019-12-17 12:28:19,922] jepsen worker 43 - jepsen.util 1033 :info :write [14 45] :RequestCanceledException INFO [2019-12-17 12:28:19,922] jepsen worker 85 - jepsen.util 895 :invoke :read [28 nil] INFO [2019-12-17 12:28:19,922] jepsen worker 49 - jepsen.util 859 :info :cas [16 [47 31]] :RequestCanceledException INFO [2019-12-17 12:28:19,922] jepsen worker 81 - jepsen.util 1071 :info :cas [27 [31 35]] :RequestCanceledException INFO [2019-12-17 12:28:19,922] jepsen worker 57 - jepsen.util 957 :info :cas [19 [9 28]] :RequestCanceledException INFO [2019-12-17 12:28:19,922] jepsen worker 23 - jepsen.util 833 :fail :cas [7 [11 10]] :ValueNotSwapFrom INFO [2019-12-17 12:28:19,922] jepsen worker 41 - jepsen.util 671 :info :cas [13 [6 8]] :RequestCanceledException INFO [2019-12-17 12:28:19,923] jepsen worker 12 - jepsen.util 912 :info :cas [4 [42 20]] :RequestCanceledException INFO [2019-12-17 12:28:19,923] jepsen worker 66 - jepsen.util 786 :info :cas [22 [40 36]] :RequestCanceledException INFO [2019-12-17 12:28:19,923] jepsen worker 11 - jepsen.util 1091 :info :write [3 19] :RequestCanceledException INFO [2019-12-17 12:28:19,923] jepsen worker 70 - jepsen.util 1150 :fail :read [23 nil] :RequestTimeoutException INFO [2019-12-17 12:28:19,923] jepsen worker 67 - jepsen.util 1237 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:28:19,923] jepsen worker 2 - jepsen.util 632 :info :cas [0 [10 23]] :RequestCanceledException INFO [2019-12-17 12:28:19,923] jepsen worker 48 - jepsen.util 498 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:28:19,923] jepsen worker 79 - jepsen.util 799 :fail :read [26 nil] :RequestTimeoutException INFO [2019-12-17 12:28:19,923] jepsen worker 30 - jepsen.util 930 :info :write [10 10] :RequestCanceledException INFO [2019-12-17 12:28:19,923] jepsen worker 9 - jepsen.util 729 :info :cas [3 [14 33]] :RequestCanceledException INFO [2019-12-17 12:28:19,923] jepsen worker 28 - jepsen.util 568 :info :write [9 6] :RequestCanceledException INFO [2019-12-17 12:28:19,923] jepsen worker 52 - jepsen.util 862 :ok :read [17 9] INFO [2019-12-17 12:28:19,923] jepsen worker 15 - jepsen.util 825 :info :cas [5 [38 30]] :RequestCanceledException INFO [2019-12-17 12:28:19,923] jepsen worker 19 - jepsen.util 1009 :info :write [6 39] :RequestCanceledException INFO [2019-12-17 12:28:19,923] jepsen worker 84 - jepsen.util 714 :info :cas [28 [4 40]] :RequestCanceledException INFO [2019-12-17 12:28:19,923] jepsen worker 33 - jepsen.util 1113 :info :cas [11 [37 27]] :RequestCanceledException INFO [2019-12-17 12:28:19,923] jepsen worker 46 - jepsen.util 1036 :info :write [15 41] :RequestCanceledException INFO [2019-12-17 12:28:19,923] jepsen worker 42 - jepsen.util 1122 :info :write [14 27] :RequestCanceledException INFO [2019-12-17 12:28:19,923] jepsen worker 77 - jepsen.util 887 :info :write [25 10] :RequestCanceledException INFO [2019-12-17 12:28:19,923] jepsen worker 59 - jepsen.util 869 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:28:19,923] jepsen worker 82 - jepsen.util 1252 :info :cas [27 [8 37]] :RequestCanceledException INFO [2019-12-17 12:28:19,923] jepsen worker 83 - jepsen.util 1073 :info :write [27 18] :RequestCanceledException INFO [2019-12-17 12:28:19,923] jepsen worker 1 - jepsen.util 721 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:28:19,923] jepsen worker 8 - jepsen.util 998 :info :cas [2 [25 49]] :RequestCanceledException INFO [2019-12-17 12:28:19,923] jepsen worker 24 - jepsen.util 834 :info :write [8 28] :RequestCanceledException INFO [2019-12-17 12:28:19,923] jepsen worker 10 - jepsen.util 1000 :info :write [3 45] :RequestCanceledException INFO [2019-12-17 12:28:19,923] jepsen worker 87 - jepsen.util 897 :info :write [29 23] :RequestCanceledException INFO [2019-12-17 12:28:19,924] jepsen worker 21 - jepsen.util 1101 :invoke :read [7 nil] INFO [2019-12-17 12:28:19,924] jepsen worker 50 - jepsen.util 1130 :invoke :write [16 21] INFO [2019-12-17 12:28:19,925] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4513,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:28:19,925] jepsen worker 34 - jepsen.util 1024 :info :cas [11 [30 27]] :RequestCanceledException INFO [2019-12-17 12:28:19,925] jepsen worker 75 - jepsen.util 975 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:28:19,925] jepsen worker 76 - jepsen.util 976 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:28:19,925] jepsen worker 86 - jepsen.util 896 :info :cas [28 [29 40]] :RequestCanceledException INFO [2019-12-17 12:28:19,925] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4465,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:28:19,926] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4456,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:28:19,926] jepsen worker 7 - jepsen.util 1087 :info :write [2 36] :RequestCanceledException INFO [2019-12-17 12:28:19,926] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4464,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:28:19,926] jepsen worker 16 - jepsen.util 916 :info :cas [5 [31 21]] :RequestCanceledException INFO [2019-12-17 12:28:19,927] jepsen worker 20 - jepsen.util 830 :info :write [6 47] :RequestCanceledException INFO [2019-12-17 12:28:19,927] jepsen worker 63 - jepsen.util 783 :invoke :cas [21 [38 49]] INFO [2019-12-17 12:28:19,927] jepsen worker 68 - jepsen.util 608 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:28:19,928] jepsen worker 39 - jepsen.util 1119 :invoke :read [13 nil] INFO [2019-12-17 12:28:19,928] jepsen worker 40 - jepsen.util 1390 :invoke :write [13 32] INFO [2019-12-17 12:28:19,928] jepsen worker 22 - jepsen.util 922 :invoke :write [7 24] INFO [2019-12-17 12:28:19,928] jepsen worker 81 - jepsen.util 1161 :invoke :write [27 22] INFO [2019-12-17 12:28:19,929] jepsen worker 23 - jepsen.util 833 :invoke :write [7 2] INFO [2019-12-17 12:28:19,929] jepsen worker 49 - jepsen.util 949 :invoke :read [16 nil] INFO [2019-12-17 12:28:19,929] jepsen worker 57 - jepsen.util 1047 :invoke :read [19 nil] INFO [2019-12-17 12:28:19,930] jepsen worker 43 - jepsen.util 1123 :invoke :read [14 nil] INFO [2019-12-17 12:28:19,930] jepsen worker 12 - jepsen.util 1002 :invoke :cas [4 [3 13]] INFO [2019-12-17 12:28:19,930] jepsen worker 41 - jepsen.util 761 :invoke :write [13 13] INFO [2019-12-17 12:28:19,930] jepsen worker 70 - jepsen.util 1150 :invoke :read [23 nil] INFO [2019-12-17 12:28:19,930] jepsen worker 66 - jepsen.util 876 :invoke :read [22 nil] INFO [2019-12-17 12:28:19,930] jepsen worker 67 - jepsen.util 1237 :invoke :write [22 35] INFO [2019-12-17 12:28:19,931] jepsen worker 11 - jepsen.util 1181 :invoke :read [3 nil] INFO [2019-12-17 12:28:19,931] jepsen worker 48 - jepsen.util 498 :invoke :cas [16 [39 40]] INFO [2019-12-17 12:28:19,931] jepsen worker 79 - jepsen.util 799 :invoke :write [26 20] INFO [2019-12-17 12:28:19,932] jepsen worker 52 - jepsen.util 862 :invoke :cas [17 [2 49]] INFO [2019-12-17 12:28:19,932] jepsen worker 2 - jepsen.util 722 :invoke :cas [0 [43 28]] INFO [2019-12-17 12:28:19,932] jepsen worker 30 - jepsen.util 1020 :invoke :cas [10 [40 20]] INFO [2019-12-17 12:28:19,933] jepsen worker 28 - jepsen.util 658 :invoke :cas [9 [46 12]] INFO [2019-12-17 12:28:19,933] jepsen worker 9 - jepsen.util 819 :invoke :write [3 35] INFO [2019-12-17 12:28:19,933] jepsen worker 15 - jepsen.util 915 :invoke :read [5 nil] INFO [2019-12-17 12:28:19,933] jepsen worker 59 - jepsen.util 869 :invoke :read [19 nil] INFO [2019-12-17 12:28:19,933] jepsen worker 19 - jepsen.util 1099 :invoke :write [6 14] INFO [2019-12-17 12:28:19,933] jepsen worker 84 - jepsen.util 804 :invoke :write [28 17] INFO [2019-12-17 12:28:19,933] jepsen worker 33 - jepsen.util 1203 :invoke :write [11 39] INFO [2019-12-17 12:28:19,934] jepsen worker 46 - jepsen.util 1126 :invoke :write [15 6] INFO [2019-12-17 12:28:19,934] jepsen worker 1 - jepsen.util 721 :invoke :cas [0 [39 29]] INFO [2019-12-17 12:28:19,934] jepsen worker 42 - jepsen.util 1212 :invoke :write [14 7] INFO [2019-12-17 12:28:19,934] jepsen worker 77 - jepsen.util 977 :invoke :write [25 10] INFO [2019-12-17 12:28:19,935] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4514,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:28:19,935] jepsen worker 8 - jepsen.util 1088 :invoke :cas [2 [10 32]] INFO [2019-12-17 12:28:19,935] jepsen worker 83 - jepsen.util 1163 :invoke :write [27 9] INFO [2019-12-17 12:28:19,935] jepsen worker 24 - jepsen.util 924 :invoke :cas [8 [49 17]] INFO [2019-12-17 12:28:19,935] jepsen worker 82 - jepsen.util 1342 :invoke :write [27 35] INFO [2019-12-17 12:28:19,935] jepsen worker 75 - jepsen.util 975 :invoke :write [25 43] INFO [2019-12-17 12:28:19,935] jepsen worker 76 - jepsen.util 976 :invoke :cas [25 [17 5]] INFO [2019-12-17 12:28:19,936] jepsen worker 87 - jepsen.util 987 :invoke :cas [29 [45 10]] INFO [2019-12-17 12:28:19,936] jepsen worker 10 - jepsen.util 1090 :invoke :read [3 nil] INFO [2019-12-17 12:28:19,936] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4470,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:28:19,936] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4467,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:28:19,936] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4471,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:28:19,937] jepsen worker 68 - jepsen.util 608 :invoke :cas [22 [29 37]] INFO [2019-12-17 12:28:19,937] jepsen worker 34 - jepsen.util 1114 :invoke :read [11 nil] INFO [2019-12-17 12:28:19,939] jepsen worker 7 - jepsen.util 1177 :invoke :read [2 nil] INFO [2019-12-17 12:28:19,940] jepsen worker 16 - jepsen.util 1006 :invoke :read [5 nil] INFO [2019-12-17 12:28:19,941] jepsen worker 86 - jepsen.util 986 :invoke :write [28 9] INFO [2019-12-17 12:28:19,941] jepsen worker 20 - jepsen.util 920 :invoke :write [6 45] INFO [2019-12-17 12:28:19,942] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4514,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:28:19,944] jepsen worker 54 - jepsen.util 954 :info :write [18 39] :RequestCanceledException INFO [2019-12-17 12:28:19,945] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4470,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:28:19,945] jepsen worker 89 - jepsen.util 989 :info :write [29 37] :RequestCanceledException INFO [2019-12-17 12:28:19,945] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4467,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:28:19,946] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4471,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:28:19,947] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4515,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:28:19,947] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4483,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:28:19,948] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4481,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:28:19,948] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4477,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:28:19,949] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4515,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:28:19,949] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4481,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:28:19,950] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4483,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:28:19,950] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4477,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:28:19,950] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4519,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:28:19,952] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4519,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:28:19,952] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4521,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:28:19,952] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4521,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:28:19,962] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:28:19,962] jepsen nemesis - couchbase.util Rebalance check count: 28 INFO [2019-12-17 12:28:19,962] jepsen nemesis - couchbase.util Rebalance stuck count: 9 INFO [2019-12-17 12:28:19,964] jepsen worker 54 - jepsen.util 1044 :invoke :read [18 nil] INFO [2019-12-17 12:28:19,965] jepsen worker 89 - jepsen.util 1079 :invoke :write [29 46] INFO [2019-12-17 12:28:19,980] jepsen worker 56 - jepsen.util 866 :info :write [18 39] :RequestCanceledException INFO [2019-12-17 12:28:19,996] jepsen worker 56 - jepsen.util 956 :invoke :write [18 49] INFO [2019-12-17 12:28:20,028] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4533,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:28:20,028] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4533,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:28:20,028] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4538,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:28:20,028] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4535,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:28:20,028] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4538,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:28:20,029] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4535,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:28:20,029] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4539,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:28:20,029] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4540,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:28:20,029] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4539,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:28:20,029] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4540,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:28:20,029] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4541,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:28:20,029] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4545,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:28:20,029] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4541,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:28:20,029] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4545,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:28:20,029] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4549,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:28:20,030] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestTimeoutException: CarrierBucketConfigRequest {"retried":0,"reason":"TIMEOUT","requestId":4290,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:28:20,030] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4549,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:28:20,030] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestTimeoutException: CarrierBucketConfigRequest {"retried":0,"reason":"TIMEOUT","requestId":4397,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:28:20,030] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4590,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:28:20,030] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4590,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:28:20,030] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4592,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:28:20,030] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4592,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:28:20,031] jepsen worker 18 - jepsen.util 918 :fail :cas [6 [37 25]] :ValueNotSwapFrom INFO [2019-12-17 12:28:20,031] jepsen worker 18 - jepsen.util 918 :invoke :write [6 43] INFO [2019-12-17 12:28:20,035] jepsen worker 44 - jepsen.util 764 :info :cas [14 [29 29]] :RequestCanceledException INFO [2019-12-17 12:28:20,039] jepsen worker 44 - jepsen.util 854 :invoke :cas [14 [32 49]] INFO [2019-12-17 12:28:20,050] jepsen worker 47 - jepsen.util 1127 :info :cas [15 [35 33]] :RequestCanceledException INFO [2019-12-17 12:28:20,056] jepsen worker 47 - jepsen.util 1217 :invoke :cas [15 [46 30]] INFO [2019-12-17 12:28:20,058] jepsen worker 32 - jepsen.util 842 :ok :read [10 27] INFO [2019-12-17 12:28:20,059] jepsen worker 32 - jepsen.util 842 :invoke :read [10 nil] INFO [2019-12-17 12:28:20,067] jepsen worker 51 - jepsen.util 861 :ok :read [17 9] INFO [2019-12-17 12:28:20,067] jepsen worker 51 - jepsen.util 861 :invoke :read [17 nil] INFO [2019-12-17 12:28:20,075] jepsen worker 5 - jepsen.util 635 :ok :read [1 43] INFO [2019-12-17 12:28:20,075] jepsen worker 5 - jepsen.util 635 :invoke :cas [1 [12 15]] INFO [2019-12-17 12:28:20,080] jepsen worker 65 - jepsen.util 695 :ok :read [21 39] INFO [2019-12-17 12:28:20,080] jepsen worker 65 - jepsen.util 695 :invoke :read [21 nil] INFO [2019-12-17 12:28:20,095] jepsen worker 30 - jepsen.util 1020 :fail :cas [10 [40 20]] :ValueNotSwapFrom INFO [2019-12-17 12:28:20,096] jepsen worker 30 - jepsen.util 1020 :invoke :write [10 11] INFO [2019-12-17 12:28:20,112] jepsen worker 73 - jepsen.util 793 :ok :read [24 0] INFO [2019-12-17 12:28:20,113] jepsen worker 73 - jepsen.util 793 :invoke :cas [24 [26 49]] INFO [2019-12-17 12:28:20,144] jepsen worker 36 - jepsen.util 1026 :fail :cas [12 [44 29]] :ValueNotSwapFrom INFO [2019-12-17 12:28:20,144] jepsen worker 36 - jepsen.util 1026 :invoke :read [12 nil] INFO [2019-12-17 12:28:20,147] jepsen worker 88 - jepsen.util 898 :fail :cas [29 [21 44]] :ValueNotSwapFrom INFO [2019-12-17 12:28:20,147] jepsen worker 88 - jepsen.util 898 :invoke :cas [29 [41 28]] INFO [2019-12-17 12:28:20,151] jepsen worker 72 - jepsen.util 792 :fail :cas [24 [40 9]] :ValueNotSwapFrom INFO [2019-12-17 12:28:20,151] jepsen worker 72 - jepsen.util 792 :invoke :read [24 nil] INFO [2019-12-17 12:28:20,162] jepsen worker 27 - jepsen.util 747 :ok :read [9 28] INFO [2019-12-17 12:28:20,162] jepsen worker 27 - jepsen.util 747 :invoke :cas [9 [25 16]] INFO [2019-12-17 12:28:20,180] jepsen worker 10 - jepsen.util 1090 :ok :read [3 23] INFO [2019-12-17 12:28:20,181] jepsen worker 10 - jepsen.util 1090 :invoke :read [3 nil] INFO [2019-12-17 12:28:20,188] jepsen worker 35 - jepsen.util 755 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:28:20,188] jepsen worker 35 - jepsen.util 755 :invoke :read [11 nil] INFO [2019-12-17 12:28:20,190] jepsen worker 32 - jepsen.util 842 :ok :read [10 27] INFO [2019-12-17 12:28:20,190] jepsen worker 32 - jepsen.util 842 :invoke :cas [10 [10 38]] INFO [2019-12-17 12:28:20,205] jepsen worker 58 - jepsen.util 958 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:28:20,205] jepsen worker 58 - jepsen.util 958 :invoke :write [19 33] INFO [2019-12-17 12:28:20,217] jepsen worker 73 - jepsen.util 793 :fail :cas [24 [26 49]] :ValueNotSwapFrom INFO [2019-12-17 12:28:20,217] jepsen worker 73 - jepsen.util 793 :invoke :read [24 nil] INFO [2019-12-17 12:28:20,232] jepsen worker 72 - jepsen.util 792 :ok :read [24 0] INFO [2019-12-17 12:28:20,232] jepsen worker 72 - jepsen.util 792 :invoke :cas [24 [38 14]] INFO [2019-12-17 12:28:20,249] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4636,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:28:20,249] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4636,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:28:20,249] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4639,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:28:20,249] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4639,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:28:20,249] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4640,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:28:20,249] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4640,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:28:20,251] jepsen worker 78 - jepsen.util 978 :ok :read [26 29] INFO [2019-12-17 12:28:20,251] jepsen worker 78 - jepsen.util 978 :invoke :write [26 35] INFO [2019-12-17 12:28:20,252] jepsen worker 29 - jepsen.util 749 :fail :read [9 nil] :RequestTimeoutException INFO [2019-12-17 12:28:20,253] jepsen worker 29 - jepsen.util 749 :invoke :write [9 22] INFO [2019-12-17 12:28:20,262] jepsen worker 27 - jepsen.util 747 :fail :cas [9 [25 16]] :ValueNotSwapFrom INFO [2019-12-17 12:28:20,262] jepsen worker 27 - jepsen.util 747 :invoke :write [9 34] INFO [2019-12-17 12:28:20,273] jepsen worker 4 - jepsen.util 724 :ok :read [1 43] INFO [2019-12-17 12:28:20,273] jepsen worker 4 - jepsen.util 724 :invoke :read [1 nil] INFO [2019-12-17 12:28:20,274] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4628,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:28:20,274] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4628,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:28:20,274] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4629,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:28:20,274] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4629,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:28:20,274] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4630,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:28:20,274] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4630,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:28:20,287] jepsen worker 10 - jepsen.util 1090 :ok :read [3 23] INFO [2019-12-17 12:28:20,287] jepsen worker 10 - jepsen.util 1090 :invoke :cas [3 [5 33]] INFO [2019-12-17 12:28:20,305] jepsen worker 32 - jepsen.util 842 :fail :cas [10 [10 38]] :ValueNotSwapFrom INFO [2019-12-17 12:28:20,306] jepsen worker 32 - jepsen.util 842 :invoke :read [10 nil] INFO [2019-12-17 12:28:20,320] jepsen worker 73 - jepsen.util 793 :ok :read [24 0] INFO [2019-12-17 12:28:20,320] jepsen worker 73 - jepsen.util 793 :invoke :cas [24 [30 13]] INFO [2019-12-17 12:28:20,334] jepsen worker 72 - jepsen.util 792 :fail :cas [24 [38 14]] :ValueNotSwapFrom INFO [2019-12-17 12:28:20,335] jepsen worker 72 - jepsen.util 792 :invoke :read [24 nil] INFO [2019-12-17 12:28:20,368] jepsen worker 27 - jepsen.util 747 :ok :write [9 34] INFO [2019-12-17 12:28:20,368] jepsen worker 27 - jepsen.util 747 :invoke :cas [9 [38 9]] INFO [2019-12-17 12:28:20,376] jepsen worker 38 - jepsen.util 848 :fail :cas [12 [17 11]] :ValueNotSwapFrom INFO [2019-12-17 12:28:20,376] jepsen worker 38 - jepsen.util 848 :invoke :cas [12 [46 23]] INFO [2019-12-17 12:28:20,394] jepsen worker 32 - jepsen.util 842 :ok :read [10 27] INFO [2019-12-17 12:28:20,394] jepsen worker 32 - jepsen.util 842 :invoke :read [10 nil] INFO [2019-12-17 12:28:20,404] jepsen worker 73 - jepsen.util 793 :fail :cas [24 [30 13]] :ValueNotSwapFrom INFO [2019-12-17 12:28:20,404] jepsen worker 73 - jepsen.util 793 :invoke :read [24 nil] INFO [2019-12-17 12:28:20,415] jepsen worker 80 - jepsen.util 1070 :ok :write [26 49] INFO [2019-12-17 12:28:20,416] jepsen worker 80 - jepsen.util 1070 :invoke :write [26 47] INFO [2019-12-17 12:28:20,419] jepsen worker 51 - jepsen.util 861 :ok :read [17 9] INFO [2019-12-17 12:28:20,419] jepsen worker 51 - jepsen.util 861 :invoke :cas [17 [6 33]] INFO [2019-12-17 12:28:20,420] jepsen worker 72 - jepsen.util 792 :ok :read [24 0] INFO [2019-12-17 12:28:20,420] jepsen worker 72 - jepsen.util 792 :invoke :read [24 nil] INFO [2019-12-17 12:28:20,435] jepsen worker 61 - jepsen.util 1051 :ok :read [20 7] INFO [2019-12-17 12:28:20,435] jepsen worker 61 - jepsen.util 1051 :invoke :read [20 nil] INFO [2019-12-17 12:28:20,441] jepsen worker 89 - jepsen.util 1079 :ok :write [29 46] INFO [2019-12-17 12:28:20,441] jepsen worker 89 - jepsen.util 1079 :invoke :cas [29 [21 23]] INFO [2019-12-17 12:28:20,452] jepsen worker 27 - jepsen.util 747 :fail :cas [9 [38 9]] :ValueNotSwapFrom INFO [2019-12-17 12:28:20,452] jepsen worker 27 - jepsen.util 747 :invoke :write [9 0] INFO [2019-12-17 12:28:20,468] jepsen worker 32 - jepsen.util 842 :ok :read [10 27] INFO [2019-12-17 12:28:20,468] jepsen worker 32 - jepsen.util 842 :invoke :cas [10 [15 43]] INFO [2019-12-17 12:28:20,469] jepsen worker 21 - jepsen.util 1101 :ok :read [7 20] INFO [2019-12-17 12:28:20,469] jepsen worker 21 - jepsen.util 1101 :invoke :cas [7 [47 19]] INFO [2019-12-17 12:28:20,480] jepsen worker 14 - jepsen.util 824 :ok :read [4 11] INFO [2019-12-17 12:28:20,480] jepsen worker 14 - jepsen.util 824 :invoke :cas [4 [4 2]] INFO [2019-12-17 12:28:20,489] jepsen worker 61 - jepsen.util 1051 :ok :read [20 7] INFO [2019-12-17 12:28:20,489] jepsen worker 61 - jepsen.util 1051 :invoke :cas [20 [37 9]] INFO [2019-12-17 12:28:20,493] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestTimeoutException: CarrierBucketConfigRequest {"retried":0,"reason":"TIMEOUT","requestId":4439,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:28:20,493] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4660,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:28:20,493] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4660,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:28:20,494] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4663,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:28:20,494] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4663,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:28:20,494] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4664,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:28:20,494] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4664,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:28:20,507] jepsen worker 43 - jepsen.util 1123 :ok :read [14 27] INFO [2019-12-17 12:28:20,507] jepsen worker 43 - jepsen.util 1123 :invoke :cas [14 [20 23]] INFO [2019-12-17 12:28:20,512] jepsen worker 73 - jepsen.util 793 :ok :read [24 0] INFO [2019-12-17 12:28:20,512] jepsen worker 73 - jepsen.util 793 :invoke :write [24 7] INFO [2019-12-17 12:28:20,529] jepsen worker 87 - jepsen.util 987 :fail :cas [29 [45 10]] :ValueNotSwapFrom INFO [2019-12-17 12:28:20,529] jepsen worker 87 - jepsen.util 987 :invoke :write [29 14] INFO [2019-12-17 12:28:20,532] jepsen worker 24 - jepsen.util 924 :fail :cas [8 [49 17]] :ValueNotSwapFrom INFO [2019-12-17 12:28:20,532] jepsen worker 24 - jepsen.util 924 :invoke :write [8 37] INFO [2019-12-17 12:28:20,537] jepsen worker 72 - jepsen.util 792 :ok :read [24 0] INFO [2019-12-17 12:28:20,537] jepsen worker 72 - jepsen.util 792 :invoke :read [24 nil] INFO [2019-12-17 12:28:20,559] jepsen worker 27 - jepsen.util 747 :ok :write [9 0] INFO [2019-12-17 12:28:20,559] jepsen worker 27 - jepsen.util 747 :invoke :read [9 nil] INFO [2019-12-17 12:28:20,587] jepsen worker 44 - jepsen.util 854 :fail :cas [14 [32 49]] :ValueNotSwapFrom INFO [2019-12-17 12:28:20,587] jepsen worker 44 - jepsen.util 854 :invoke :write [14 12] INFO [2019-12-17 12:28:20,595] jepsen worker 37 - jepsen.util 847 :fail :cas [12 [20 10]] :ValueNotSwapFrom INFO [2019-12-17 12:28:20,595] jepsen worker 37 - jepsen.util 847 :invoke :read [12 nil] INFO [2019-12-17 12:28:20,597] jepsen worker 61 - jepsen.util 1051 :fail :cas [20 [37 9]] :ValueNotSwapFrom INFO [2019-12-17 12:28:20,597] jepsen worker 61 - jepsen.util 1051 :invoke :cas [20 [49 45]] INFO [2019-12-17 12:28:20,628] jepsen worker 88 - jepsen.util 898 :fail :cas [29 [41 28]] :ValueNotSwapFrom INFO [2019-12-17 12:28:20,628] jepsen worker 88 - jepsen.util 898 :invoke :write [29 14] INFO [2019-12-17 12:28:20,650] jepsen worker 13 - jepsen.util 1003 :ok :read [4 11] INFO [2019-12-17 12:28:20,650] jepsen worker 13 - jepsen.util 1003 :invoke :cas [4 [10 49]] INFO [2019-12-17 12:28:20,651] jepsen worker 79 - jepsen.util 799 :ok :write [26 20] INFO [2019-12-17 12:28:20,651] jepsen worker 79 - jepsen.util 799 :invoke :cas [26 [19 3]] INFO [2019-12-17 12:28:20,679] jepsen worker 55 - jepsen.util 1135 :fail :cas [18 [45 34]] :ValueNotSwapFrom INFO [2019-12-17 12:28:20,679] jepsen worker 55 - jepsen.util 1135 :invoke :read [18 nil] INFO [2019-12-17 12:28:20,713] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4683,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:28:20,713] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4683,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:28:20,724] jepsen worker 36 - jepsen.util 1026 :ok :read [12 12] INFO [2019-12-17 12:28:20,724] jepsen worker 36 - jepsen.util 1026 :invoke :read [12 nil] INFO [2019-12-17 12:28:20,742] jepsen worker 74 - jepsen.util 794 :ok :write [24 11] INFO [2019-12-17 12:28:20,743] jepsen worker 74 - jepsen.util 794 :invoke :write [24 31] INFO [2019-12-17 12:28:20,749] jepsen worker 69 - jepsen.util 879 :ok :write [23 4] INFO [2019-12-17 12:28:20,749] jepsen worker 69 - jepsen.util 879 :invoke :read [23 nil] INFO [2019-12-17 12:28:20,773] jepsen worker 31 - jepsen.util 1021 :ok :write [10 26] INFO [2019-12-17 12:28:20,773] jepsen worker 31 - jepsen.util 1021 :invoke :write [10 12] INFO [2019-12-17 12:28:20,823] jepsen worker 43 - jepsen.util 1123 :fail :cas [14 [20 23]] :ValueNotSwapFrom INFO [2019-12-17 12:28:20,824] jepsen worker 43 - jepsen.util 1123 :invoke :read [14 nil] INFO [2019-12-17 12:28:20,831] jepsen worker 78 - jepsen.util 978 :ok :write [26 35] INFO [2019-12-17 12:28:20,831] jepsen worker 78 - jepsen.util 978 :invoke :cas [26 [39 21]] INFO [2019-12-17 12:28:20,833] jepsen worker 61 - jepsen.util 1051 :fail :cas [20 [49 45]] :ValueNotSwapFrom INFO [2019-12-17 12:28:20,833] jepsen worker 61 - jepsen.util 1051 :invoke :write [20 18] INFO [2019-12-17 12:28:20,838] jepsen worker 53 - jepsen.util 953 :ok :read [17 9] INFO [2019-12-17 12:28:20,838] jepsen worker 53 - jepsen.util 953 :invoke :write [17 15] INFO [2019-12-17 12:28:20,860] jepsen worker 64 - jepsen.util 514 :ok :write [21 21] INFO [2019-12-17 12:28:20,860] jepsen worker 64 - jepsen.util 514 :invoke :cas [21 [30 14]] INFO [2019-12-17 12:28:20,869] jepsen worker 47 - jepsen.util 1217 :fail :cas [15 [46 30]] :ValueNotSwapFrom INFO [2019-12-17 12:28:20,869] jepsen worker 47 - jepsen.util 1217 :invoke :write [15 31] INFO [2019-12-17 12:28:20,883] jepsen worker 26 - jepsen.util 1106 :ok :read [8 14] INFO [2019-12-17 12:28:20,883] jepsen worker 26 - jepsen.util 1106 :invoke :read [8 nil] INFO [2019-12-17 12:28:20,898] jepsen worker 54 - jepsen.util 1044 :ok :read [18 39] INFO [2019-12-17 12:28:20,898] jepsen worker 54 - jepsen.util 1044 :invoke :cas [18 [7 17]] INFO [2019-12-17 12:28:20,923] jepsen worker 29 - jepsen.util 749 :ok :write [9 22] INFO [2019-12-17 12:28:20,923] jepsen worker 29 - jepsen.util 749 :invoke :read [9 nil] INFO [2019-12-17 12:28:20,940] jepsen worker 14 - jepsen.util 824 :fail :cas [4 [4 2]] :ValueNotSwapFrom INFO [2019-12-17 12:28:20,941] jepsen worker 14 - jepsen.util 824 :invoke :cas [4 [18 42]] INFO [2019-12-17 12:28:20,958] jepsen worker 4 - jepsen.util 724 :ok :read [1 43] INFO [2019-12-17 12:28:20,958] jepsen worker 4 - jepsen.util 724 :invoke :cas [1 [11 45]] INFO [2019-12-17 12:28:20,975] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:28:20,975] jepsen nemesis - couchbase.util Rebalance check count: 29 INFO [2019-12-17 12:28:20,975] jepsen nemesis - couchbase.util Rebalance stuck count: 10 INFO [2019-12-17 12:28:20,993] jepsen worker 7 - jepsen.util 1177 :ok :read [2 15] INFO [2019-12-17 12:28:20,993] jepsen worker 7 - jepsen.util 1177 :invoke :write [2 35] INFO [2019-12-17 12:28:21,144] jepsen worker 87 - jepsen.util 987 :ok :write [29 14] INFO [2019-12-17 12:28:21,145] jepsen worker 87 - jepsen.util 987 :invoke :cas [29 [21 9]] INFO [2019-12-17 12:28:21,145] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4713,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:28:21,145] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4713,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:28:21,145] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4717,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:28:21,145] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4717,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:28:21,146] jepsen worker 35 - jepsen.util 755 :ok :read [11 7] INFO [2019-12-17 12:28:21,146] jepsen worker 35 - jepsen.util 755 :invoke :write [11 22] INFO [2019-12-17 12:28:21,151] jepsen worker 69 - jepsen.util 879 :ok :read [23 4] INFO [2019-12-17 12:28:21,152] jepsen worker 69 - jepsen.util 879 :invoke :read [23 nil] INFO [2019-12-17 12:28:21,154] jepsen worker 5 - jepsen.util 635 :fail :cas [1 [12 15]] :ValueNotSwapFrom INFO [2019-12-17 12:28:21,154] jepsen worker 5 - jepsen.util 635 :invoke :read [1 nil] INFO [2019-12-17 12:28:21,157] jepsen worker 26 - jepsen.util 1106 :ok :read [8 14] INFO [2019-12-17 12:28:21,158] jepsen worker 26 - jepsen.util 1106 :invoke :cas [8 [23 29]] INFO [2019-12-17 12:28:21,158] jepsen worker 51 - jepsen.util 861 :fail :cas [17 [6 33]] :ValueNotSwapFrom INFO [2019-12-17 12:28:21,158] jepsen worker 51 - jepsen.util 861 :invoke :cas [17 [21 15]] INFO [2019-12-17 12:28:21,159] jepsen worker 38 - jepsen.util 848 :fail :cas [12 [46 23]] :ValueNotSwapFrom INFO [2019-12-17 12:28:21,159] jepsen worker 38 - jepsen.util 848 :invoke :cas [12 [28 22]] INFO [2019-12-17 12:28:21,188] jepsen worker 89 - jepsen.util 1079 :fail :cas [29 [21 23]] :ValueNotSwapFrom INFO [2019-12-17 12:28:21,188] jepsen worker 89 - jepsen.util 1079 :invoke :cas [29 [20 13]] INFO [2019-12-17 12:28:21,195] jepsen worker 56 - jepsen.util 956 :ok :write [18 49] INFO [2019-12-17 12:28:21,195] jepsen worker 56 - jepsen.util 956 :invoke :cas [18 [11 30]] INFO [2019-12-17 12:28:21,209] jepsen worker 21 - jepsen.util 1101 :fail :cas [7 [47 19]] :ValueNotSwapFrom INFO [2019-12-17 12:28:21,209] jepsen worker 21 - jepsen.util 1101 :invoke :cas [7 [45 28]] INFO [2019-12-17 12:28:21,214] jepsen worker 74 - jepsen.util 794 :ok :write [24 31] INFO [2019-12-17 12:28:21,214] jepsen worker 74 - jepsen.util 794 :invoke :read [24 nil] INFO [2019-12-17 12:28:21,215] jepsen worker 13 - jepsen.util 1003 :fail :cas [4 [10 49]] :ValueNotSwapFrom INFO [2019-12-17 12:28:21,216] jepsen worker 13 - jepsen.util 1003 :invoke :write [4 0] INFO [2019-12-17 12:28:21,235] jepsen worker 55 - jepsen.util 1135 :ok :read [18 49] INFO [2019-12-17 12:28:21,235] jepsen worker 55 - jepsen.util 1135 :invoke :cas [18 [34 11]] INFO [2019-12-17 12:28:21,235] jepsen worker 74 - jepsen.util 794 :ok :read [24 31] INFO [2019-12-17 12:28:21,235] jepsen worker 74 - jepsen.util 794 :invoke :write [24 3] INFO [2019-12-17 12:28:21,252] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4728,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:28:21,252] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4728,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:28:21,252] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4729,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:28:21,252] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4729,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:28:21,255] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4736,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:28:21,255] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4736,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:28:21,274] jepsen worker 37 - jepsen.util 847 :ok :read [12 12] INFO [2019-12-17 12:28:21,274] jepsen worker 37 - jepsen.util 847 :invoke :cas [12 [9 27]] INFO [2019-12-17 12:28:21,324] jepsen worker 56 - jepsen.util 956 :fail :cas [18 [11 30]] :ValueNotSwapFrom INFO [2019-12-17 12:28:21,324] jepsen worker 56 - jepsen.util 956 :invoke :cas [18 [12 21]] INFO [2019-12-17 12:28:21,326] jepsen worker 78 - jepsen.util 978 :fail :cas [26 [39 21]] :ValueNotSwapFrom INFO [2019-12-17 12:28:21,326] jepsen worker 78 - jepsen.util 978 :invoke :read [26 nil] INFO [2019-12-17 12:28:21,359] jepsen worker 5 - jepsen.util 635 :ok :read [1 43] INFO [2019-12-17 12:28:21,359] jepsen worker 5 - jepsen.util 635 :invoke :read [1 nil] INFO [2019-12-17 12:28:21,372] jepsen worker 31 - jepsen.util 1021 :ok :write [10 12] INFO [2019-12-17 12:28:21,373] jepsen worker 31 - jepsen.util 1021 :invoke :cas [10 [12 4]] INFO [2019-12-17 12:28:21,374] jepsen worker 43 - jepsen.util 1123 :ok :read [14 27] INFO [2019-12-17 12:28:21,374] jepsen worker 43 - jepsen.util 1123 :invoke :read [14 nil] INFO [2019-12-17 12:28:21,387] jepsen worker 38 - jepsen.util 848 :fail :cas [12 [28 22]] :ValueNotSwapFrom INFO [2019-12-17 12:28:21,388] jepsen worker 38 - jepsen.util 848 :invoke :read [12 nil] INFO [2019-12-17 12:28:21,430] jepsen worker 89 - jepsen.util 1079 :fail :cas [29 [20 13]] :ValueNotSwapFrom INFO [2019-12-17 12:28:21,430] jepsen worker 89 - jepsen.util 1079 :invoke :cas [29 [7 5]] INFO [2019-12-17 12:28:21,436] jepsen worker 71 - jepsen.util 701 :ok :write [23 5] INFO [2019-12-17 12:28:21,437] jepsen worker 71 - jepsen.util 701 :invoke :write [23 27] INFO [2019-12-17 12:28:21,445] jepsen worker 26 - jepsen.util 1106 :fail :cas [8 [23 29]] :ValueNotSwapFrom INFO [2019-12-17 12:28:21,445] jepsen worker 26 - jepsen.util 1106 :invoke :cas [8 [3 6]] INFO [2019-12-17 12:28:21,453] jepsen worker 47 - jepsen.util 1217 :ok :write [15 31] INFO [2019-12-17 12:28:21,453] jepsen worker 47 - jepsen.util 1217 :invoke :write [15 33] INFO [2019-12-17 12:28:21,471] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4755,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:28:21,471] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4755,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:28:21,471] jepsen worker 88 - jepsen.util 898 :ok :write [29 14] INFO [2019-12-17 12:28:21,471] jepsen worker 88 - jepsen.util 898 :invoke :write [29 18] INFO [2019-12-17 12:28:21,493] jepsen worker 74 - jepsen.util 794 :ok :write [24 3] INFO [2019-12-17 12:28:21,493] jepsen worker 74 - jepsen.util 794 :invoke :cas [24 [36 25]] INFO [2019-12-17 12:28:21,496] jepsen worker 79 - jepsen.util 799 :fail :cas [26 [19 3]] :ValueNotSwapFrom INFO [2019-12-17 12:28:21,496] jepsen worker 79 - jepsen.util 799 :invoke :write [26 0] INFO [2019-12-17 12:28:21,509] jepsen worker 74 - jepsen.util 794 :fail :cas [24 [36 25]] :ValueNotSwapFrom INFO [2019-12-17 12:28:21,509] jepsen worker 74 - jepsen.util 794 :invoke :cas [24 [39 41]] INFO [2019-12-17 12:28:21,510] jepsen worker 78 - jepsen.util 978 :ok :read [26 35] INFO [2019-12-17 12:28:21,510] jepsen worker 78 - jepsen.util 978 :invoke :cas [26 [43 16]] INFO [2019-12-17 12:28:21,511] jepsen worker 13 - jepsen.util 1003 :ok :write [4 0] INFO [2019-12-17 12:28:21,511] jepsen worker 13 - jepsen.util 1003 :invoke :write [4 41] INFO [2019-12-17 12:28:21,529] jepsen worker 3 - jepsen.util 723 :ok :write [1 21] INFO [2019-12-17 12:28:21,530] jepsen worker 3 - jepsen.util 723 :invoke :cas [1 [22 13]] INFO [2019-12-17 12:28:21,533] jepsen worker 74 - jepsen.util 794 :fail :cas [24 [39 41]] :ValueNotSwapFrom INFO [2019-12-17 12:28:21,536] jepsen worker 74 - jepsen.util 794 :invoke :write [24 10] INFO [2019-12-17 12:28:21,541] jepsen worker 55 - jepsen.util 1135 :fail :cas [18 [34 11]] :ValueNotSwapFrom INFO [2019-12-17 12:28:21,541] jepsen worker 55 - jepsen.util 1135 :invoke :write [18 14] INFO [2019-12-17 12:28:21,565] jepsen worker 4 - jepsen.util 724 :fail :cas [1 [11 45]] :ValueNotSwapFrom INFO [2019-12-17 12:28:21,566] jepsen worker 4 - jepsen.util 724 :invoke :cas [1 [13 4]] INFO [2019-12-17 12:28:21,598] jepsen worker 5 - jepsen.util 635 :ok :read [1 43] INFO [2019-12-17 12:28:21,599] jepsen worker 5 - jepsen.util 635 :invoke :write [1 47] INFO [2019-12-17 12:28:21,628] jepsen worker 37 - jepsen.util 847 :fail :cas [12 [9 27]] :ValueNotSwapFrom INFO [2019-12-17 12:28:21,628] jepsen worker 37 - jepsen.util 847 :invoke :write [12 43] INFO [2019-12-17 12:28:21,643] jepsen worker 43 - jepsen.util 1123 :ok :read [14 27] INFO [2019-12-17 12:28:21,644] jepsen worker 43 - jepsen.util 1123 :invoke :write [14 9] INFO [2019-12-17 12:28:21,651] jepsen worker 51 - jepsen.util 861 :fail :cas [17 [21 15]] :ValueNotSwapFrom INFO [2019-12-17 12:28:21,651] jepsen worker 51 - jepsen.util 861 :invoke :write [17 35] INFO [2019-12-17 12:28:21,661] jepsen worker 38 - jepsen.util 848 :ok :read [12 12] INFO [2019-12-17 12:28:21,661] jepsen worker 38 - jepsen.util 848 :invoke :cas [12 [26 17]] INFO [2019-12-17 12:28:21,679] jepsen worker 31 - jepsen.util 1021 :ok :cas [10 [12 4]] INFO [2019-12-17 12:28:21,679] jepsen worker 31 - jepsen.util 1021 :invoke :write [10 26] INFO [2019-12-17 12:28:21,681] jepsen worker 56 - jepsen.util 956 :fail :cas [18 [12 21]] :ValueNotSwapFrom INFO [2019-12-17 12:28:21,681] jepsen worker 56 - jepsen.util 956 :invoke :read [18 nil] INFO [2019-12-17 12:28:21,683] jepsen worker 12 - jepsen.util 1002 :fail :cas [4 [3 13]] :ValueNotSwapFrom INFO [2019-12-17 12:28:21,683] jepsen worker 12 - jepsen.util 1002 :invoke :cas [4 [43 4]] INFO [2019-12-17 12:28:21,690] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4776,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:28:21,691] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4776,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:28:21,692] jepsen worker 89 - jepsen.util 1079 :fail :cas [29 [7 5]] :ValueNotSwapFrom INFO [2019-12-17 12:28:21,692] jepsen worker 89 - jepsen.util 1079 :invoke :read [29 nil] INFO [2019-12-17 12:28:21,715] jepsen worker 26 - jepsen.util 1106 :fail :cas [8 [3 6]] :ValueNotSwapFrom INFO [2019-12-17 12:28:21,716] jepsen worker 26 - jepsen.util 1106 :invoke :read [8 nil] INFO [2019-12-17 12:28:21,717] jepsen worker 21 - jepsen.util 1101 :fail :cas [7 [45 28]] :ValueNotSwapFrom INFO [2019-12-17 12:28:21,717] jepsen worker 21 - jepsen.util 1101 :invoke :read [7 nil] INFO [2019-12-17 12:28:21,730] jepsen worker 78 - jepsen.util 978 :fail :cas [26 [43 16]] :ValueNotSwapFrom INFO [2019-12-17 12:28:21,730] jepsen worker 78 - jepsen.util 978 :invoke :write [26 17] INFO [2019-12-17 12:28:21,741] jepsen worker 70 - jepsen.util 1150 :ok :read [23 5] INFO [2019-12-17 12:28:21,741] jepsen worker 70 - jepsen.util 1150 :invoke :read [23 nil] INFO [2019-12-17 12:28:21,766] jepsen worker 52 - jepsen.util 862 :fail :cas [17 [2 49]] :ValueNotSwapFrom INFO [2019-12-17 12:28:21,766] jepsen worker 52 - jepsen.util 862 :invoke :cas [17 [22 32]] INFO [2019-12-17 12:28:21,768] jepsen worker 26 - jepsen.util 1106 :ok :read [8 14] INFO [2019-12-17 12:28:21,768] jepsen worker 26 - jepsen.util 1106 :invoke :write [8 45] INFO [2019-12-17 12:28:21,771] jepsen worker 47 - jepsen.util 1217 :ok :write [15 33] INFO [2019-12-17 12:28:21,772] jepsen worker 47 - jepsen.util 1217 :invoke :write [15 17] INFO [2019-12-17 12:28:21,781] jepsen worker 13 - jepsen.util 1003 :ok :write [4 41] INFO [2019-12-17 12:28:21,782] jepsen worker 13 - jepsen.util 1003 :invoke :cas [4 [13 28]] INFO [2019-12-17 12:28:21,830] jepsen worker 56 - jepsen.util 956 :ok :read [18 49] INFO [2019-12-17 12:28:21,830] jepsen worker 56 - jepsen.util 956 :invoke :write [18 2] INFO [2019-12-17 12:28:21,834] jepsen worker 31 - jepsen.util 1021 :ok :write [10 26] INFO [2019-12-17 12:28:21,834] jepsen worker 31 - jepsen.util 1021 :invoke :write [10 31] INFO [2019-12-17 12:28:21,834] jepsen worker 74 - jepsen.util 794 :ok :write [24 10] INFO [2019-12-17 12:28:21,835] jepsen worker 74 - jepsen.util 794 :invoke :write [24 27] INFO [2019-12-17 12:28:21,849] jepsen worker 53 - jepsen.util 953 :ok :write [17 15] INFO [2019-12-17 12:28:21,849] jepsen worker 53 - jepsen.util 953 :invoke :cas [17 [6 14]] INFO [2019-12-17 12:28:21,854] jepsen worker 71 - jepsen.util 701 :ok :write [23 27] INFO [2019-12-17 12:28:21,854] jepsen worker 71 - jepsen.util 701 :invoke :read [23 nil] INFO [2019-12-17 12:28:21,856] jepsen worker 88 - jepsen.util 898 :ok :write [29 18] INFO [2019-12-17 12:28:21,856] jepsen worker 88 - jepsen.util 898 :invoke :cas [29 [3 4]] INFO [2019-12-17 12:28:21,856] jepsen worker 80 - jepsen.util 1070 :ok :write [26 47] INFO [2019-12-17 12:28:21,856] jepsen worker 80 - jepsen.util 1070 :invoke :write [26 25] INFO [2019-12-17 12:28:21,865] jepsen worker 47 - jepsen.util 1217 :ok :write [15 17] INFO [2019-12-17 12:28:21,865] jepsen worker 47 - jepsen.util 1217 :invoke :write [15 13] INFO [2019-12-17 12:28:21,878] jepsen worker 36 - jepsen.util 1026 :ok :read [12 12] INFO [2019-12-17 12:28:21,878] jepsen worker 36 - jepsen.util 1026 :invoke :cas [12 [35 12]] INFO [2019-12-17 12:28:21,893] jepsen worker 69 - jepsen.util 879 :ok :read [23 27] INFO [2019-12-17 12:28:21,893] jepsen worker 69 - jepsen.util 879 :invoke :read [23 nil] INFO [2019-12-17 12:28:21,897] jepsen worker 52 - jepsen.util 862 :fail :cas [17 [22 32]] :ValueNotSwapFrom INFO [2019-12-17 12:28:21,898] jepsen worker 52 - jepsen.util 862 :invoke :write [17 17] INFO [2019-12-17 12:28:21,902] jepsen worker 5 - jepsen.util 635 :ok :write [1 47] INFO [2019-12-17 12:28:21,902] jepsen worker 5 - jepsen.util 635 :invoke :read [1 nil] INFO [2019-12-17 12:28:21,904] jepsen worker 3 - jepsen.util 723 :fail :cas [1 [22 13]] :ValueNotSwapFrom INFO [2019-12-17 12:28:21,904] jepsen worker 3 - jepsen.util 723 :invoke :cas [1 [12 13]] INFO [2019-12-17 12:28:21,909] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4804,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:28:21,909] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4804,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:28:21,935] jepsen worker 25 - jepsen.util 1105 :info :write [8 3] :RequestCanceledException INFO [2019-12-17 12:28:21,936] jepsen worker 25 - jepsen.util 1195 :invoke :write [8 6] INFO [2019-12-17 12:28:21,937] jepsen worker 14 - jepsen.util 824 :fail :cas [4 [18 42]] :ValueNotSwapFrom INFO [2019-12-17 12:28:21,937] jepsen worker 14 - jepsen.util 824 :invoke :read [4 nil] INFO [2019-12-17 12:28:21,952] jepsen worker 28 - jepsen.util 658 :info :cas [9 [46 12]] :RequestCanceledException INFO [2019-12-17 12:28:21,952] jepsen worker 17 - jepsen.util 1007 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:28:21,953] jepsen worker 17 - jepsen.util 1007 :invoke :cas [5 [9 23]] INFO [2019-12-17 12:28:21,954] jepsen worker 28 - jepsen.util 748 :invoke :read [9 nil] INFO [2019-12-17 12:28:21,957] jepsen worker 38 - jepsen.util 848 :fail :cas [12 [26 17]] :ValueNotSwapFrom INFO [2019-12-17 12:28:21,957] jepsen worker 38 - jepsen.util 848 :invoke :read [12 nil] INFO [2019-12-17 12:28:21,966] jepsen worker 55 - jepsen.util 1135 :ok :write [18 14] INFO [2019-12-17 12:28:21,966] jepsen worker 55 - jepsen.util 1135 :invoke :read [18 nil] INFO [2019-12-17 12:28:21,974] jepsen worker 28 - jepsen.util 748 :ok :read [9 22] INFO [2019-12-17 12:28:21,974] jepsen worker 28 - jepsen.util 748 :invoke :cas [9 [32 32]] INFO [2019-12-17 12:28:21,978] jepsen worker 53 - jepsen.util 953 :fail :cas [17 [6 14]] :ValueNotSwapFrom INFO [2019-12-17 12:28:21,979] jepsen worker 53 - jepsen.util 953 :invoke :write [17 17] INFO [2019-12-17 12:28:21,986] jepsen worker 12 - jepsen.util 1002 :fail :cas [4 [43 4]] :ValueNotSwapFrom INFO [2019-12-17 12:28:21,987] jepsen worker 12 - jepsen.util 1002 :invoke :read [4 nil] INFO [2019-12-17 12:28:21,988] jepsen worker 0 - jepsen.util 1080 :info :write [0 6] :RequestCanceledException INFO [2019-12-17 12:28:21,988] jepsen worker 6 - jepsen.util 996 :info :write [2 15] :RequestCanceledException INFO [2019-12-17 12:28:21,988] jepsen worker 85 - jepsen.util 895 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:28:21,988] jepsen worker 66 - jepsen.util 876 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:28:21,988] jepsen worker 15 - jepsen.util 915 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:28:21,988] jepsen worker 85 - jepsen.util 895 :invoke :write [28 36] INFO [2019-12-17 12:28:21,988] jepsen worker 66 - jepsen.util 876 :invoke :write [22 33] INFO [2019-12-17 12:28:21,988] jepsen worker 15 - jepsen.util 915 :invoke :read [5 nil] INFO [2019-12-17 12:28:21,989] jepsen worker 0 - jepsen.util 1170 :invoke :cas [0 [31 36]] INFO [2019-12-17 12:28:21,989] jepsen worker 6 - jepsen.util 1086 :invoke :write [2 12] INFO [2019-12-17 12:28:22,002] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:28:22,002] jepsen nemesis - couchbase.util Rebalance check count: 30 INFO [2019-12-17 12:28:22,002] jepsen nemesis - couchbase.util Rebalance stuck count: 11 INFO [2019-12-17 12:28:22,002] jepsen worker 28 - jepsen.util 748 :fail :cas [9 [32 32]] :ValueNotSwapFrom INFO [2019-12-17 12:28:22,002] jepsen worker 28 - jepsen.util 748 :invoke :cas [9 [9 36]] INFO [2019-12-17 12:28:22,003] jepsen worker 45 - jepsen.util 1035 :fail :read [15 nil] :RequestTimeoutException INFO [2019-12-17 12:28:22,003] jepsen worker 8 - jepsen.util 1088 :info :cas [2 [10 32]] :RequestCanceledException INFO [2019-12-17 12:28:22,003] jepsen worker 76 - jepsen.util 976 :info :cas [25 [17 5]] :RequestCanceledException INFO [2019-12-17 12:28:22,003] jepsen worker 49 - jepsen.util 949 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:28:22,003] jepsen worker 11 - jepsen.util 1181 :fail :read [3 nil] :RequestTimeoutException INFO [2019-12-17 12:28:22,003] jepsen worker 34 - jepsen.util 1114 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:28:22,003] jepsen worker 50 - jepsen.util 1130 :info :write [16 21] :RequestCanceledException INFO [2019-12-17 12:28:22,003] jepsen worker 68 - jepsen.util 608 :info :cas [22 [29 37]] :RequestCanceledException INFO [2019-12-17 12:28:22,003] jepsen worker 45 - jepsen.util 1035 :invoke :write [15 37] INFO [2019-12-17 12:28:22,004] jepsen worker 49 - jepsen.util 949 :invoke :cas [16 [13 4]] INFO [2019-12-17 12:28:22,004] jepsen worker 11 - jepsen.util 1181 :invoke :read [3 nil] INFO [2019-12-17 12:28:22,004] jepsen worker 34 - jepsen.util 1114 :invoke :write [11 17] INFO [2019-12-17 12:28:22,005] jepsen worker 76 - jepsen.util 1066 :invoke :cas [25 [12 16]] INFO [2019-12-17 12:28:22,005] jepsen worker 8 - jepsen.util 1178 :invoke :cas [2 [17 38]] INFO [2019-12-17 12:28:22,005] jepsen worker 50 - jepsen.util 1220 :invoke :read [16 nil] INFO [2019-12-17 12:28:22,005] jepsen worker 68 - jepsen.util 698 :invoke :cas [22 [30 4]] INFO [2019-12-17 12:28:22,020] jepsen worker 87 - jepsen.util 987 :fail :cas [29 [21 9]] :ValueNotSwapFrom INFO [2019-12-17 12:28:22,020] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4828,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:28:22,020] jepsen worker 87 - jepsen.util 987 :invoke :write [29 27] INFO [2019-12-17 12:28:22,021] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4828,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:28:22,021] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4831,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:28:22,021] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4831,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:28:22,021] jepsen worker 60 - jepsen.util 1050 :info :write [20 10] :RequestCanceledException INFO [2019-12-17 12:28:22,021] jepsen worker 9 - jepsen.util 819 :info :write [3 35] :RequestCanceledException INFO [2019-12-17 12:28:22,021] jepsen worker 62 - jepsen.util 962 :info :write [20 11] :RequestCanceledException INFO [2019-12-17 12:28:22,021] jepsen worker 22 - jepsen.util 922 :info :write [7 24] :RequestCanceledException INFO [2019-12-17 12:28:22,021] jepsen worker 41 - jepsen.util 761 :info :write [13 13] :RequestCanceledException INFO [2019-12-17 12:28:22,021] jepsen worker 67 - jepsen.util 1237 :info :write [22 35] :RequestCanceledException INFO [2019-12-17 12:28:22,021] jepsen worker 42 - jepsen.util 1212 :info :write [14 7] :RequestCanceledException INFO [2019-12-17 12:28:22,021] jepsen worker 63 - jepsen.util 783 :info :cas [21 [38 49]] :RequestCanceledException INFO [2019-12-17 12:28:22,021] jepsen worker 20 - jepsen.util 920 :info :write [6 45] :RequestCanceledException INFO [2019-12-17 12:28:22,022] jepsen worker 57 - jepsen.util 1047 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:28:22,022] jepsen worker 2 - jepsen.util 722 :info :cas [0 [43 28]] :RequestCanceledException INFO [2019-12-17 12:28:22,022] jepsen worker 59 - jepsen.util 869 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:28:22,022] jepsen worker 83 - jepsen.util 1163 :info :write [27 9] :RequestCanceledException INFO [2019-12-17 12:28:22,022] jepsen worker 1 - jepsen.util 721 :info :cas [0 [39 29]] :RequestCanceledException INFO [2019-12-17 12:28:22,022] jepsen worker 84 - jepsen.util 804 :info :write [28 17] :RequestCanceledException INFO [2019-12-17 12:28:22,023] jepsen worker 60 - jepsen.util 1140 :invoke :write [20 48] INFO [2019-12-17 12:28:22,023] jepsen worker 9 - jepsen.util 909 :invoke :write [3 18] INFO [2019-12-17 12:28:22,023] jepsen worker 62 - jepsen.util 1052 :invoke :cas [20 [24 41]] INFO [2019-12-17 12:28:22,023] jepsen worker 57 - jepsen.util 1047 :invoke :write [19 28] INFO [2019-12-17 12:28:22,023] jepsen worker 22 - jepsen.util 1012 :invoke :cas [7 [21 16]] INFO [2019-12-17 12:28:22,023] jepsen worker 41 - jepsen.util 851 :invoke :cas [13 [7 42]] INFO [2019-12-17 12:28:22,023] jepsen worker 42 - jepsen.util 1302 :invoke :read [14 nil] INFO [2019-12-17 12:28:22,023] jepsen worker 59 - jepsen.util 869 :invoke :write [19 18] INFO [2019-12-17 12:28:22,023] jepsen worker 67 - jepsen.util 1327 :invoke :write [22 10] INFO [2019-12-17 12:28:22,024] jepsen worker 63 - jepsen.util 873 :invoke :cas [21 [7 12]] INFO [2019-12-17 12:28:22,024] jepsen worker 20 - jepsen.util 1010 :invoke :write [6 39] INFO [2019-12-17 12:28:22,024] jepsen worker 2 - jepsen.util 812 :invoke :cas [0 [24 48]] INFO [2019-12-17 12:28:22,024] jepsen worker 1 - jepsen.util 811 :invoke :write [0 40] INFO [2019-12-17 12:28:22,024] jepsen worker 84 - jepsen.util 894 :invoke :cas [28 [26 0]] INFO [2019-12-17 12:28:22,025] jepsen worker 83 - jepsen.util 1253 :invoke :cas [27 [43 18]] INFO [2019-12-17 12:28:22,033] jepsen worker 39 - jepsen.util 1119 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:28:22,033] jepsen worker 48 - jepsen.util 498 :info :cas [16 [39 40]] :RequestCanceledException INFO [2019-12-17 12:28:22,033] jepsen worker 16 - jepsen.util 1006 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:28:22,033] jepsen worker 33 - jepsen.util 1203 :info :write [11 39] :RequestCanceledException INFO [2019-12-17 12:28:22,033] jepsen worker 39 - jepsen.util 1119 :invoke :cas [13 [14 45]] INFO [2019-12-17 12:28:22,033] jepsen worker 16 - jepsen.util 1006 :invoke :read [5 nil] INFO [2019-12-17 12:28:22,034] jepsen worker 28 - jepsen.util 748 :fail :cas [9 [9 36]] :ValueNotSwapFrom INFO [2019-12-17 12:28:22,034] jepsen worker 28 - jepsen.util 748 :invoke :write [9 8] INFO [2019-12-17 12:28:22,035] jepsen worker 48 - jepsen.util 588 :invoke :write [16 43] INFO [2019-12-17 12:28:22,035] jepsen worker 33 - jepsen.util 1293 :invoke :read [11 nil] INFO [2019-12-17 12:28:22,052] jepsen worker 46 - jepsen.util 1126 :info :write [15 6] :RequestCanceledException INFO [2019-12-17 12:28:22,052] jepsen worker 86 - jepsen.util 986 :info :write [28 9] :RequestCanceledException INFO [2019-12-17 12:28:22,052] jepsen worker 82 - jepsen.util 1342 :info :write [27 35] :RequestCanceledException INFO [2019-12-17 12:28:22,052] jepsen worker 40 - jepsen.util 1390 :info :write [13 32] :RequestCanceledException INFO [2019-12-17 12:28:22,053] jepsen worker 46 - jepsen.util 1216 :invoke :read [15 nil] INFO [2019-12-17 12:28:22,053] jepsen worker 86 - jepsen.util 1076 :invoke :read [28 nil] INFO [2019-12-17 12:28:22,053] jepsen worker 82 - jepsen.util 1432 :invoke :write [27 0] INFO [2019-12-17 12:28:22,053] jepsen worker 40 - jepsen.util 1480 :invoke :write [13 44] INFO [2019-12-17 12:28:22,054] jepsen worker 89 - jepsen.util 1079 :ok :read [29 18] INFO [2019-12-17 12:28:22,054] jepsen worker 89 - jepsen.util 1079 :invoke :cas [29 [14 35]] INFO [2019-12-17 12:28:22,060] jepsen worker 54 - jepsen.util 1044 :fail :cas [18 [7 17]] :ValueNotSwapFrom INFO [2019-12-17 12:28:22,060] jepsen worker 54 - jepsen.util 1044 :invoke :write [18 0] INFO [2019-12-17 12:28:22,072] jepsen worker 11 - jepsen.util 1181 :ok :read [3 19] INFO [2019-12-17 12:28:22,072] jepsen worker 11 - jepsen.util 1181 :invoke :write [3 14] INFO [2019-12-17 12:28:22,076] jepsen worker 69 - jepsen.util 879 :ok :read [23 27] INFO [2019-12-17 12:28:22,076] jepsen worker 69 - jepsen.util 879 :invoke :cas [23 [44 34]] INFO [2019-12-17 12:28:22,078] jepsen worker 70 - jepsen.util 1150 :ok :read [23 27] INFO [2019-12-17 12:28:22,078] jepsen worker 70 - jepsen.util 1150 :invoke :read [23 nil] INFO [2019-12-17 12:28:22,087] jepsen worker 77 - jepsen.util 977 :info :write [25 10] :RequestCanceledException INFO [2019-12-17 12:28:22,087] jepsen worker 81 - jepsen.util 1161 :info :write [27 22] :RequestCanceledException INFO [2019-12-17 12:28:22,087] jepsen worker 23 - jepsen.util 833 :info :write [7 2] :RequestCanceledException INFO [2019-12-17 12:28:22,088] jepsen worker 81 - jepsen.util 1251 :invoke :read [27 nil] INFO [2019-12-17 12:28:22,088] jepsen worker 77 - jepsen.util 1067 :invoke :write [25 35] INFO [2019-12-17 12:28:22,088] jepsen worker 23 - jepsen.util 923 :invoke :cas [7 [42 45]] INFO [2019-12-17 12:28:22,103] jepsen worker 4 - jepsen.util 724 :fail :cas [1 [13 4]] :ValueNotSwapFrom INFO [2019-12-17 12:28:22,103] jepsen worker 4 - jepsen.util 724 :invoke :read [1 nil] INFO [2019-12-17 12:28:22,105] jepsen worker 19 - jepsen.util 1099 :info :write [6 14] :RequestCanceledException INFO [2019-12-17 12:28:22,105] jepsen worker 75 - jepsen.util 975 :info :write [25 43] :RequestCanceledException INFO [2019-12-17 12:28:22,106] jepsen worker 19 - jepsen.util 1189 :invoke :read [6 nil] INFO [2019-12-17 12:28:22,106] jepsen worker 75 - jepsen.util 1065 :invoke :write [25 2] INFO [2019-12-17 12:28:22,119] jepsen worker 37 - jepsen.util 847 :ok :write [12 43] INFO [2019-12-17 12:28:22,119] jepsen worker 37 - jepsen.util 847 :invoke :cas [12 [48 48]] INFO [2019-12-17 12:28:22,127] jepsen worker 8 - jepsen.util 1178 :fail :cas [2 [17 38]] :ValueNotSwapFrom INFO [2019-12-17 12:28:22,127] jepsen worker 8 - jepsen.util 1178 :invoke :read [2 nil] INFO [2019-12-17 12:28:22,134] jepsen worker 18 - jepsen.util 918 :info :write [6 43] :RequestCanceledException INFO [2019-12-17 12:28:22,135] jepsen worker 65 - jepsen.util 695 :fail :read [21 nil] :RequestTimeoutException INFO [2019-12-17 12:28:22,135] jepsen worker 30 - jepsen.util 1020 :info :write [10 11] :RequestCanceledException INFO [2019-12-17 12:28:22,135] jepsen worker 65 - jepsen.util 695 :invoke :cas [21 [5 37]] INFO [2019-12-17 12:28:22,136] jepsen worker 30 - jepsen.util 1110 :invoke :read [10 nil] INFO [2019-12-17 12:28:22,136] jepsen worker 18 - jepsen.util 1008 :invoke :read [6 nil] INFO [2019-12-17 12:28:22,157] jepsen worker 76 - jepsen.util 1066 :fail :cas [25 [12 16]] :ValueNotSwapFrom INFO [2019-12-17 12:28:22,157] jepsen worker 76 - jepsen.util 1066 :invoke :write [25 14] INFO [2019-12-17 12:28:22,164] jepsen worker 62 - jepsen.util 1052 :fail :cas [20 [24 41]] :ValueNotSwapFrom INFO [2019-12-17 12:28:22,164] jepsen worker 62 - jepsen.util 1052 :invoke :read [20 nil] INFO [2019-12-17 12:28:22,167] jepsen worker 28 - jepsen.util 748 :ok :write [9 8] INFO [2019-12-17 12:28:22,167] jepsen worker 28 - jepsen.util 748 :invoke :read [9 nil] INFO [2019-12-17 12:28:22,171] jepsen worker 30 - jepsen.util 1110 :ok :read [10 26] INFO [2019-12-17 12:28:22,172] jepsen worker 30 - jepsen.util 1110 :invoke :read [10 nil] INFO [2019-12-17 12:28:22,194] jepsen worker 78 - jepsen.util 978 :ok :write [26 17] INFO [2019-12-17 12:28:22,194] jepsen worker 51 - jepsen.util 861 :ok :write [17 35] INFO [2019-12-17 12:28:22,194] jepsen worker 78 - jepsen.util 978 :invoke :write [26 13] INFO [2019-12-17 12:28:22,194] jepsen worker 51 - jepsen.util 861 :invoke :cas [17 [33 42]] INFO [2019-12-17 12:28:22,207] jepsen worker 71 - jepsen.util 701 :ok :read [23 27] INFO [2019-12-17 12:28:22,207] jepsen worker 71 - jepsen.util 701 :invoke :write [23 21] INFO [2019-12-17 12:28:22,221] jepsen worker 22 - jepsen.util 1012 :fail :cas [7 [21 16]] :ValueNotSwapFrom INFO [2019-12-17 12:28:22,221] jepsen worker 22 - jepsen.util 1012 :invoke :cas [7 [26 22]] INFO [2019-12-17 12:28:22,231] jepsen worker 28 - jepsen.util 748 :ok :read [9 8] INFO [2019-12-17 12:28:22,232] jepsen worker 28 - jepsen.util 748 :invoke :write [9 21] INFO [2019-12-17 12:28:22,238] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4889,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:28:22,238] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4889,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:28:22,238] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4890,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:28:22,239] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4890,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:28:22,240] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4897,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:28:22,240] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4897,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:28:22,241] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4902,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:28:22,241] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4902,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:28:22,282] jepsen worker 30 - jepsen.util 1110 :ok :read [10 26] INFO [2019-12-17 12:28:22,283] jepsen worker 30 - jepsen.util 1110 :invoke :write [10 21] INFO [2019-12-17 12:28:22,305] jepsen worker 13 - jepsen.util 1003 :fail :cas [4 [13 28]] :ValueNotSwapFrom INFO [2019-12-17 12:28:22,305] jepsen worker 62 - jepsen.util 1052 :ok :read [20 7] INFO [2019-12-17 12:28:22,306] jepsen worker 13 - jepsen.util 1003 :invoke :write [4 1] INFO [2019-12-17 12:28:22,306] jepsen worker 62 - jepsen.util 1052 :invoke :write [20 29] INFO [2019-12-17 12:28:22,306] jepsen worker 58 - jepsen.util 958 :info :write [19 33] :RequestCanceledException INFO [2019-12-17 12:28:22,313] jepsen worker 58 - jepsen.util 1048 :invoke :cas [19 [18 6]] INFO [2019-12-17 12:28:22,337] jepsen worker 89 - jepsen.util 1079 :fail :cas [29 [14 35]] :ValueNotSwapFrom INFO [2019-12-17 12:28:22,337] jepsen worker 89 - jepsen.util 1079 :invoke :read [29 nil] INFO [2019-12-17 12:28:22,363] jepsen worker 74 - jepsen.util 794 :ok :write [24 27] INFO [2019-12-17 12:28:22,363] jepsen worker 74 - jepsen.util 794 :invoke :read [24 nil] INFO [2019-12-17 12:28:22,365] jepsen worker 31 - jepsen.util 1021 :ok :write [10 31] INFO [2019-12-17 12:28:22,366] jepsen worker 47 - jepsen.util 1217 :ok :write [15 13] INFO [2019-12-17 12:28:22,366] jepsen worker 31 - jepsen.util 1021 :invoke :cas [10 [44 23]] INFO [2019-12-17 12:28:22,366] jepsen worker 47 - jepsen.util 1217 :invoke :cas [15 [16 6]] INFO [2019-12-17 12:28:22,379] jepsen worker 21 - jepsen.util 1101 :ok :read [7 20] INFO [2019-12-17 12:28:22,379] jepsen worker 21 - jepsen.util 1101 :invoke :read [7 nil] INFO [2019-12-17 12:28:22,380] jepsen worker 23 - jepsen.util 923 :fail :cas [7 [42 45]] :ValueNotSwapFrom INFO [2019-12-17 12:28:22,380] jepsen worker 23 - jepsen.util 923 :invoke :cas [7 [37 34]] INFO [2019-12-17 12:28:22,383] jepsen worker 5 - jepsen.util 635 :ok :read [1 47] INFO [2019-12-17 12:28:22,383] jepsen worker 5 - jepsen.util 635 :invoke :write [1 21] INFO [2019-12-17 12:28:22,384] jepsen worker 10 - jepsen.util 1090 :info :cas [3 [5 33]] :RequestCanceledException INFO [2019-12-17 12:28:22,390] jepsen worker 10 - jepsen.util 1180 :invoke :write [3 12] INFO [2019-12-17 12:28:22,418] jepsen worker 28 - jepsen.util 748 :ok :write [9 21] INFO [2019-12-17 12:28:22,418] jepsen worker 28 - jepsen.util 748 :invoke :write [9 13] INFO [2019-12-17 12:28:22,419] jepsen worker 20 - jepsen.util 1010 :ok :write [6 39] INFO [2019-12-17 12:28:22,419] jepsen worker 20 - jepsen.util 1010 :invoke :write [6 35] INFO [2019-12-17 12:28:22,422] jepsen worker 38 - jepsen.util 848 :ok :read [12 43] INFO [2019-12-17 12:28:22,423] jepsen worker 38 - jepsen.util 848 :invoke :write [12 20] INFO [2019-12-17 12:28:22,427] jepsen worker 42 - jepsen.util 1302 :ok :read [14 27] INFO [2019-12-17 12:28:22,427] jepsen worker 42 - jepsen.util 1302 :invoke :read [14 nil] INFO [2019-12-17 12:28:22,453] jepsen worker 55 - jepsen.util 1135 :ok :read [18 14] INFO [2019-12-17 12:28:22,552] jepsen worker 32 - jepsen.util 842 :info :cas [10 [15 43]] :RequestCanceledException INFO [2019-12-17 12:28:22,552] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4921,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:28:22,553] jepsen worker 63 - jepsen.util 873 :fail :cas [21 [7 12]] :ValueNotSwapFrom INFO [2019-12-17 12:28:22,554] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4921,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:28:22,554] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4922,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:28:22,554] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4922,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:28:22,554] jepsen worker 63 - jepsen.util 873 :invoke :write [21 12] INFO [2019-12-17 12:28:22,554] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4923,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:28:22,555] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4923,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:28:22,555] jepsen worker 74 - jepsen.util 794 :ok :read [24 27] INFO [2019-12-17 12:28:22,555] jepsen worker 74 - jepsen.util 794 :invoke :write [24 13] INFO [2019-12-17 12:28:22,555] jepsen worker 32 - jepsen.util 932 :invoke :cas [10 [0 49]] INFO [2019-12-17 12:28:22,556] jepsen worker 19 - jepsen.util 1189 :ok :read [6 47] INFO [2019-12-17 12:28:22,557] jepsen worker 19 - jepsen.util 1189 :invoke :write [6 43] INFO [2019-12-17 12:28:22,559] jepsen worker 55 - jepsen.util 1135 :invoke :write [18 14] INFO [2019-12-17 12:28:22,565] jepsen worker 46 - jepsen.util 1216 :ok :read [15 13] INFO [2019-12-17 12:28:22,565] jepsen worker 46 - jepsen.util 1216 :invoke :read [15 nil] INFO [2019-12-17 12:28:22,567] jepsen worker 4 - jepsen.util 724 :ok :read [1 47] INFO [2019-12-17 12:28:22,568] jepsen worker 4 - jepsen.util 724 :invoke :write [1 43] INFO [2019-12-17 12:28:22,568] jepsen worker 3 - jepsen.util 723 :fail :cas [1 [12 13]] :ValueNotSwapFrom INFO [2019-12-17 12:28:22,569] jepsen worker 3 - jepsen.util 723 :invoke :read [1 nil] INFO [2019-12-17 12:28:22,571] jepsen worker 37 - jepsen.util 847 :fail :cas [12 [48 48]] :ValueNotSwapFrom INFO [2019-12-17 12:28:22,571] jepsen worker 37 - jepsen.util 847 :invoke :read [12 nil] INFO [2019-12-17 12:28:22,572] jepsen worker 47 - jepsen.util 1217 :fail :cas [15 [16 6]] :ValueNotSwapFrom INFO [2019-12-17 12:28:22,572] jepsen worker 47 - jepsen.util 1217 :invoke :write [15 40] INFO [2019-12-17 12:28:22,588] jepsen worker 73 - jepsen.util 793 :info :write [24 7] :RequestCanceledException INFO [2019-12-17 12:28:22,588] jepsen worker 72 - jepsen.util 792 :fail :read [24 nil] :RequestTimeoutException INFO [2019-12-17 12:28:22,588] jepsen worker 27 - jepsen.util 747 :fail :read [9 nil] :RequestTimeoutException INFO [2019-12-17 12:28:22,588] jepsen worker 27 - jepsen.util 747 :invoke :read [9 nil] INFO [2019-12-17 12:28:22,588] jepsen worker 72 - jepsen.util 792 :invoke :cas [24 [0 49]] INFO [2019-12-17 12:28:22,589] jepsen worker 73 - jepsen.util 883 :invoke :write [24 42] INFO [2019-12-17 12:28:22,590] jepsen worker 8 - jepsen.util 1178 :ok :read [2 15] INFO [2019-12-17 12:28:22,591] jepsen worker 8 - jepsen.util 1178 :invoke :cas [2 [16 4]] INFO [2019-12-17 12:28:22,603] jepsen worker 65 - jepsen.util 695 :fail :cas [21 [5 37]] :ValueNotSwapFrom INFO [2019-12-17 12:28:22,603] jepsen worker 65 - jepsen.util 695 :invoke :write [21 17] INFO [2019-12-17 12:28:22,605] jepsen worker 32 - jepsen.util 932 :fail :cas [10 [0 49]] :ValueNotSwapFrom INFO [2019-12-17 12:28:22,605] jepsen worker 32 - jepsen.util 932 :invoke :cas [10 [34 23]] INFO [2019-12-17 12:28:22,614] jepsen worker 88 - jepsen.util 898 :fail :cas [29 [3 4]] :ValueNotSwapFrom INFO [2019-12-17 12:28:22,614] jepsen worker 88 - jepsen.util 898 :invoke :write [29 27] INFO [2019-12-17 12:28:22,627] jepsen worker 22 - jepsen.util 1012 :fail :cas [7 [26 22]] :ValueNotSwapFrom INFO [2019-12-17 12:28:22,627] jepsen worker 22 - jepsen.util 1012 :invoke :cas [7 [34 29]] INFO [2019-12-17 12:28:22,635] jepsen worker 24 - jepsen.util 924 :info :write [8 37] :RequestCanceledException INFO [2019-12-17 12:28:22,635] jepsen worker 44 - jepsen.util 854 :info :write [14 12] :RequestCanceledException INFO [2019-12-17 12:28:22,636] jepsen worker 44 - jepsen.util 944 :invoke :cas [14 [47 27]] INFO [2019-12-17 12:28:22,636] jepsen worker 24 - jepsen.util 1014 :invoke :cas [8 [39 36]] INFO [2019-12-17 12:28:22,645] jepsen worker 23 - jepsen.util 923 :fail :cas [7 [37 34]] :ValueNotSwapFrom INFO [2019-12-17 12:28:22,645] jepsen worker 23 - jepsen.util 923 :invoke :write [7 20] INFO [2019-12-17 12:28:22,661] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4949,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:28:22,661] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4949,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:28:22,661] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4953,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:28:22,661] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4953,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:28:22,665] jepsen worker 12 - jepsen.util 1002 :ok :read [4 41] INFO [2019-12-17 12:28:22,666] jepsen worker 12 - jepsen.util 1002 :invoke :read [4 nil] INFO [2019-12-17 12:28:22,678] jepsen worker 51 - jepsen.util 861 :fail :cas [17 [33 42]] :ValueNotSwapFrom INFO [2019-12-17 12:28:22,679] jepsen worker 51 - jepsen.util 861 :invoke :cas [17 [48 32]] INFO [2019-12-17 12:28:22,691] jepsen worker 70 - jepsen.util 1150 :ok :read [23 27] INFO [2019-12-17 12:28:22,692] jepsen worker 70 - jepsen.util 1150 :invoke :cas [23 [5 16]] INFO [2019-12-17 12:28:22,697] jepsen worker 8 - jepsen.util 1178 :fail :cas [2 [16 4]] :ValueNotSwapFrom INFO [2019-12-17 12:28:22,697] jepsen worker 8 - jepsen.util 1178 :invoke :cas [2 [9 28]] INFO [2019-12-17 12:28:22,705] jepsen worker 9 - jepsen.util 909 :ok :write [3 18] INFO [2019-12-17 12:28:22,705] jepsen worker 9 - jepsen.util 909 :invoke :read [3 nil] INFO [2019-12-17 12:28:22,718] jepsen worker 28 - jepsen.util 748 :ok :write [9 13] INFO [2019-12-17 12:28:22,718] jepsen worker 28 - jepsen.util 748 :invoke :cas [9 [15 29]] INFO [2019-12-17 12:28:22,720] jepsen worker 22 - jepsen.util 1012 :fail :cas [7 [34 29]] :ValueNotSwapFrom INFO [2019-12-17 12:28:22,720] jepsen worker 22 - jepsen.util 1012 :invoke :write [7 7] INFO [2019-12-17 12:28:22,747] jepsen worker 31 - jepsen.util 1021 :fail :cas [10 [44 23]] :ValueNotSwapFrom INFO [2019-12-17 12:28:22,747] jepsen worker 31 - jepsen.util 1021 :invoke :write [10 7] INFO [2019-12-17 12:28:22,762] jepsen worker 36 - jepsen.util 1026 :fail :cas [12 [35 12]] :ValueNotSwapFrom INFO [2019-12-17 12:28:22,763] jepsen worker 36 - jepsen.util 1026 :invoke :read [12 nil] INFO [2019-12-17 12:28:22,765] jepsen worker 28 - jepsen.util 748 :fail :cas [9 [15 29]] :ValueNotSwapFrom INFO [2019-12-17 12:28:22,765] jepsen worker 28 - jepsen.util 748 :invoke :cas [9 [37 24]] INFO [2019-12-17 12:28:22,775] jepsen worker 70 - jepsen.util 1150 :fail :cas [23 [5 16]] :ValueNotSwapFrom INFO [2019-12-17 12:28:22,775] jepsen worker 70 - jepsen.util 1150 :invoke :cas [23 [41 3]] INFO [2019-12-17 12:28:22,783] jepsen worker 14 - jepsen.util 824 :ok :read [4 41] INFO [2019-12-17 12:28:22,783] jepsen worker 14 - jepsen.util 824 :invoke :read [4 nil] INFO [2019-12-17 12:28:22,786] jepsen worker 51 - jepsen.util 861 :fail :cas [17 [48 32]] :ValueNotSwapFrom INFO [2019-12-17 12:28:22,787] jepsen worker 51 - jepsen.util 861 :invoke :read [17 nil] INFO [2019-12-17 12:28:22,824] jepsen worker 33 - jepsen.util 1293 :ok :read [11 7] INFO [2019-12-17 12:28:22,824] jepsen worker 33 - jepsen.util 1293 :invoke :cas [11 [0 7]] INFO [2019-12-17 12:28:22,863] jepsen worker 28 - jepsen.util 748 :fail :cas [9 [37 24]] :ValueNotSwapFrom INFO [2019-12-17 12:28:22,863] jepsen worker 28 - jepsen.util 748 :invoke :write [9 18] INFO [2019-12-17 12:28:22,876] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4971,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:28:22,877] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4971,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:28:22,877] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4974,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:28:22,877] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4974,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:28:22,878] jepsen worker 30 - jepsen.util 1110 :ok :write [10 21] INFO [2019-12-17 12:28:22,879] jepsen worker 30 - jepsen.util 1110 :invoke :write [10 5] INFO [2019-12-17 12:28:22,895] jepsen worker 69 - jepsen.util 879 :fail :cas [23 [44 34]] :ValueNotSwapFrom INFO [2019-12-17 12:28:22,895] jepsen worker 69 - jepsen.util 879 :invoke :cas [23 [20 15]] INFO [2019-12-17 12:28:22,909] jepsen worker 88 - jepsen.util 898 :ok :write [29 27] INFO [2019-12-17 12:28:22,909] jepsen worker 88 - jepsen.util 898 :invoke :read [29 nil] INFO [2019-12-17 12:28:22,915] jepsen worker 63 - jepsen.util 873 :ok :write [21 12] INFO [2019-12-17 12:28:22,915] jepsen worker 63 - jepsen.util 873 :invoke :cas [21 [43 29]] INFO [2019-12-17 12:28:22,921] jepsen worker 64 - jepsen.util 514 :info :cas [21 [30 14]] :RequestCanceledException INFO [2019-12-17 12:28:22,922] jepsen worker 64 - jepsen.util 604 :invoke :cas [21 [45 46]] INFO [2019-12-17 12:28:22,952] jepsen worker 61 - jepsen.util 1051 :info :write [20 18] :RequestCanceledException INFO [2019-12-17 12:28:22,952] jepsen worker 29 - jepsen.util 749 :fail :read [9 nil] :RequestTimeoutException INFO [2019-12-17 12:28:22,953] jepsen worker 29 - jepsen.util 749 :invoke :write [9 1] INFO [2019-12-17 12:28:22,954] jepsen worker 61 - jepsen.util 1141 :invoke :cas [20 [16 21]] INFO [2019-12-17 12:28:22,970] jepsen worker 63 - jepsen.util 873 :fail :cas [21 [43 29]] :ValueNotSwapFrom INFO [2019-12-17 12:28:22,970] jepsen worker 63 - jepsen.util 873 :invoke :cas [21 [17 45]] INFO [2019-12-17 12:28:22,981] jepsen worker 37 - jepsen.util 847 :ok :read [12 43] INFO [2019-12-17 12:28:22,982] jepsen worker 37 - jepsen.util 847 :invoke :cas [12 [43 14]] INFO [2019-12-17 12:28:23,004] jepsen worker 20 - jepsen.util 1010 :ok :write [6 35] INFO [2019-12-17 12:28:23,005] jepsen worker 20 - jepsen.util 1010 :invoke :write [6 2] INFO [2019-12-17 12:28:23,007] jepsen worker 21 - jepsen.util 1101 :ok :read [7 20] INFO [2019-12-17 12:28:23,007] jepsen worker 21 - jepsen.util 1101 :invoke :cas [7 [12 26]] INFO [2019-12-17 12:28:23,020] jepsen worker 7 - jepsen.util 1177 :info :write [2 35] :RequestCanceledException INFO [2019-12-17 12:28:23,021] jepsen worker 7 - jepsen.util 1267 :invoke :cas [2 [39 23]] INFO [2019-12-17 12:28:23,022] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:28:23,022] jepsen nemesis - couchbase.util Rebalance check count: 31 INFO [2019-12-17 12:28:23,022] jepsen nemesis - couchbase.util Rebalance stuck count: 12 INFO [2019-12-17 12:28:23,024] jepsen worker 64 - jepsen.util 604 :fail :cas [21 [45 46]] :ValueNotSwapFrom INFO [2019-12-17 12:28:23,024] jepsen worker 64 - jepsen.util 604 :invoke :cas [21 [42 11]] INFO [2019-12-17 12:28:23,027] jepsen worker 28 - jepsen.util 748 :ok :write [9 18] INFO [2019-12-17 12:28:23,027] jepsen worker 28 - jepsen.util 748 :invoke :write [9 38] INFO [2019-12-17 12:28:23,034] jepsen worker 63 - jepsen.util 873 :fail :cas [21 [17 45]] :ValueNotSwapFrom INFO [2019-12-17 12:28:23,035] jepsen worker 63 - jepsen.util 873 :invoke :write [21 11] INFO [2019-12-17 12:28:23,055] jepsen worker 89 - jepsen.util 1079 :ok :read [29 27] INFO [2019-12-17 12:28:23,055] jepsen worker 89 - jepsen.util 1079 :invoke :cas [29 [23 42]] INFO [2019-12-17 12:28:23,063] jepsen worker 42 - jepsen.util 1302 :ok :read [14 27] INFO [2019-12-17 12:28:23,063] jepsen worker 42 - jepsen.util 1302 :invoke :cas [14 [37 16]] INFO [2019-12-17 12:28:23,085] jepsen worker 3 - jepsen.util 723 :ok :read [1 47] INFO [2019-12-17 12:28:23,085] jepsen worker 3 - jepsen.util 723 :invoke :write [1 21] INFO [2019-12-17 12:28:23,092] jepsen worker 51 - jepsen.util 861 :ok :read [17 35] INFO [2019-12-17 12:28:23,092] jepsen worker 24 - jepsen.util 1014 :fail :cas [8 [39 36]] :ValueNotSwapFrom INFO [2019-12-17 12:28:23,092] jepsen worker 51 - jepsen.util 861 :invoke :read [17 nil] INFO [2019-12-17 12:28:23,092] jepsen worker 24 - jepsen.util 1014 :invoke :write [8 26] INFO [2019-12-17 12:28:23,094] jepsen worker 29 - jepsen.util 749 :ok :write [9 1] INFO [2019-12-17 12:28:23,094] jepsen worker 29 - jepsen.util 749 :invoke :cas [9 [13 24]] INFO [2019-12-17 12:28:23,096] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4986,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:28:23,096] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4986,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:28:23,096] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4990,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:28:23,096] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4990,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:28:23,098] jepsen worker 36 - jepsen.util 1026 :ok :read [12 43] INFO [2019-12-17 12:28:23,099] jepsen worker 36 - jepsen.util 1026 :invoke :read [12 nil] INFO [2019-12-17 12:28:23,106] jepsen worker 88 - jepsen.util 898 :ok :read [29 27] INFO [2019-12-17 12:28:23,106] jepsen worker 88 - jepsen.util 898 :invoke :write [29 40] INFO [2019-12-17 12:28:23,138] jepsen worker 12 - jepsen.util 1002 :ok :read [4 41] INFO [2019-12-17 12:28:23,138] jepsen worker 12 - jepsen.util 1002 :invoke :read [4 nil] INFO [2019-12-17 12:28:23,139] jepsen worker 7 - jepsen.util 1267 :fail :cas [2 [39 23]] :ValueNotSwapFrom INFO [2019-12-17 12:28:23,139] jepsen worker 7 - jepsen.util 1267 :invoke :write [2 7] INFO [2019-12-17 12:28:23,174] jepsen worker 63 - jepsen.util 873 :ok :write [21 11] INFO [2019-12-17 12:28:23,174] jepsen worker 63 - jepsen.util 873 :invoke :read [21 nil] INFO [2019-12-17 12:28:23,181] jepsen worker 33 - jepsen.util 1293 :fail :cas [11 [0 7]] :ValueNotSwapFrom INFO [2019-12-17 12:28:23,181] jepsen worker 33 - jepsen.util 1293 :invoke :write [11 17] INFO [2019-12-17 12:28:23,187] jepsen worker 29 - jepsen.util 749 :fail :cas [9 [13 24]] :ValueNotSwapFrom INFO [2019-12-17 12:28:23,187] jepsen worker 29 - jepsen.util 749 :invoke :write [9 20] INFO [2019-12-17 12:28:23,188] jepsen worker 35 - jepsen.util 755 :info :write [11 22] :RequestCanceledException INFO [2019-12-17 12:28:23,189] jepsen worker 35 - jepsen.util 845 :invoke :cas [11 [1 1]] INFO [2019-12-17 12:28:23,195] jepsen worker 12 - jepsen.util 1002 :ok :read [4 41] INFO [2019-12-17 12:28:23,195] jepsen worker 12 - jepsen.util 1002 :invoke :cas [4 [41 27]] INFO [2019-12-17 12:28:23,202] jepsen worker 63 - jepsen.util 873 :ok :read [21 11] INFO [2019-12-17 12:28:23,202] jepsen worker 63 - jepsen.util 873 :invoke :read [21 nil] INFO [2019-12-17 12:28:23,205] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":5009,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:28:23,205] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":5009,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:28:23,205] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":5010,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:28:23,205] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":5010,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:28:23,226] jepsen worker 51 - jepsen.util 861 :ok :read [17 35] INFO [2019-12-17 12:28:23,227] jepsen worker 51 - jepsen.util 861 :invoke :write [17 9] INFO [2019-12-17 12:28:23,228] jepsen worker 70 - jepsen.util 1150 :fail :cas [23 [41 3]] :ValueNotSwapFrom INFO [2019-12-17 12:28:23,229] jepsen worker 70 - jepsen.util 1150 :invoke :cas [23 [10 46]] INFO [2019-12-17 12:28:23,233] jepsen worker 14 - jepsen.util 824 :ok :read [4 41] INFO [2019-12-17 12:28:23,233] jepsen worker 14 - jepsen.util 824 :invoke :read [4 nil] INFO [2019-12-17 12:28:23,260] jepsen worker 28 - jepsen.util 748 :ok :write [9 38] INFO [2019-12-17 12:28:23,260] jepsen worker 28 - jepsen.util 748 :invoke :read [9 nil] INFO [2019-12-17 12:28:23,273] jepsen worker 63 - jepsen.util 873 :ok :read [21 11] INFO [2019-12-17 12:28:23,273] jepsen worker 63 - jepsen.util 873 :invoke :read [21 nil] INFO [2019-12-17 12:28:23,315] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":5022,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:28:23,315] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":5022,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:28:23,315] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":5024,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:28:23,315] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":5024,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:28:23,318] jepsen worker 63 - jepsen.util 873 :ok :read [21 11] INFO [2019-12-17 12:28:23,318] jepsen worker 63 - jepsen.util 873 :invoke :write [21 24] INFO [2019-12-17 12:28:23,323] jepsen worker 29 - jepsen.util 749 :ok :write [9 20] INFO [2019-12-17 12:28:23,323] jepsen worker 29 - jepsen.util 749 :invoke :read [9 nil] INFO [2019-12-17 12:28:23,327] jepsen worker 28 - jepsen.util 748 :ok :read [9 38] INFO [2019-12-17 12:28:23,327] jepsen worker 28 - jepsen.util 748 :invoke :read [9 nil] INFO [2019-12-17 12:28:23,356] jepsen worker 21 - jepsen.util 1101 :fail :cas [7 [12 26]] :ValueNotSwapFrom INFO [2019-12-17 12:28:23,356] jepsen worker 21 - jepsen.util 1101 :invoke :read [7 nil] INFO [2019-12-17 12:28:23,360] jepsen worker 5 - jepsen.util 635 :ok :write [1 21] INFO [2019-12-17 12:28:23,361] jepsen worker 5 - jepsen.util 635 :invoke :read [1 nil] INFO [2019-12-17 12:28:23,362] jepsen worker 30 - jepsen.util 1110 :ok :write [10 5] INFO [2019-12-17 12:28:23,362] jepsen worker 30 - jepsen.util 1110 :invoke :write [10 30] INFO [2019-12-17 12:28:23,365] jepsen worker 74 - jepsen.util 794 :ok :write [24 13] INFO [2019-12-17 12:28:23,365] jepsen worker 74 - jepsen.util 794 :invoke :cas [24 [15 45]] INFO [2019-12-17 12:28:23,375] jepsen worker 28 - jepsen.util 748 :ok :read [9 20] INFO [2019-12-17 12:28:23,375] jepsen worker 28 - jepsen.util 748 :invoke :read [9 nil] INFO [2019-12-17 12:28:23,375] jepsen worker 29 - jepsen.util 749 :ok :read [9 20] INFO [2019-12-17 12:28:23,375] jepsen worker 29 - jepsen.util 749 :invoke :read [9 nil] INFO [2019-12-17 12:28:23,379] jepsen worker 20 - jepsen.util 1010 :ok :write [6 2] INFO [2019-12-17 12:28:23,379] jepsen worker 20 - jepsen.util 1010 :invoke :write [6 22] INFO [2019-12-17 12:28:23,402] jepsen worker 64 - jepsen.util 604 :fail :cas [21 [42 11]] :ValueNotSwapFrom INFO [2019-12-17 12:28:23,403] jepsen worker 64 - jepsen.util 604 :invoke :read [21 nil] INFO [2019-12-17 12:28:23,403] jepsen worker 88 - jepsen.util 898 :ok :write [29 40] INFO [2019-12-17 12:28:23,403] jepsen worker 88 - jepsen.util 898 :invoke :cas [29 [39 36]] INFO [2019-12-17 12:28:23,406] jepsen worker 42 - jepsen.util 1302 :fail :cas [14 [37 16]] :ValueNotSwapFrom INFO [2019-12-17 12:28:23,406] jepsen worker 42 - jepsen.util 1302 :invoke :read [14 nil] INFO [2019-12-17 12:28:23,424] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":5044,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:28:23,424] jepsen worker 54 - jepsen.util 1044 :ok :write [18 0] INFO [2019-12-17 12:28:23,425] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":5044,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:28:23,425] jepsen worker 54 - jepsen.util 1044 :invoke :cas [18 [40 36]] INFO [2019-12-17 12:28:23,429] jepsen worker 28 - jepsen.util 748 :ok :read [9 20] INFO [2019-12-17 12:28:23,429] jepsen worker 28 - jepsen.util 748 :invoke :write [9 32] INFO [2019-12-17 12:28:23,452] jepsen worker 52 - jepsen.util 862 :ok :write [17 17] INFO [2019-12-17 12:28:23,453] jepsen worker 52 - jepsen.util 862 :invoke :write [17 48] INFO [2019-12-17 12:28:23,462] jepsen worker 64 - jepsen.util 604 :ok :read [21 11] INFO [2019-12-17 12:28:23,463] jepsen worker 64 - jepsen.util 604 :invoke :read [21 nil] INFO [2019-12-17 12:28:23,473] jepsen worker 63 - jepsen.util 873 :ok :write [21 24] INFO [2019-12-17 12:28:23,473] jepsen worker 63 - jepsen.util 873 :invoke :read [21 nil] INFO [2019-12-17 12:28:23,478] jepsen worker 29 - jepsen.util 749 :ok :read [9 20] INFO [2019-12-17 12:28:23,478] jepsen worker 29 - jepsen.util 749 :invoke :write [9 45] INFO [2019-12-17 12:28:23,490] jepsen worker 69 - jepsen.util 879 :fail :cas [23 [20 15]] :ValueNotSwapFrom INFO [2019-12-17 12:28:23,490] jepsen worker 69 - jepsen.util 879 :invoke :read [23 nil] INFO [2019-12-17 12:28:23,512] jepsen worker 64 - jepsen.util 604 :ok :read [21 24] INFO [2019-12-17 12:28:23,512] jepsen worker 64 - jepsen.util 604 :invoke :cas [21 [26 6]] INFO [2019-12-17 12:28:23,521] jepsen worker 79 - jepsen.util 799 :info :write [26 0] :RequestCanceledException INFO [2019-12-17 12:28:23,522] jepsen worker 79 - jepsen.util 889 :invoke :write [26 22] INFO [2019-12-17 12:28:23,584] jepsen worker 87 - jepsen.util 987 :ok :write [29 27] INFO [2019-12-17 12:28:23,584] jepsen worker 87 - jepsen.util 987 :invoke :cas [29 [9 26]] INFO [2019-12-17 12:28:23,596] jepsen worker 35 - jepsen.util 845 :fail :cas [11 [1 1]] :ValueNotSwapFrom INFO [2019-12-17 12:28:23,596] jepsen worker 35 - jepsen.util 845 :invoke :write [11 22] INFO [2019-12-17 12:28:23,614] jepsen worker 3 - jepsen.util 723 :ok :write [1 21] INFO [2019-12-17 12:28:23,615] jepsen worker 3 - jepsen.util 723 :invoke :read [1 nil] INFO [2019-12-17 12:28:23,617] jepsen worker 38 - jepsen.util 848 :ok :write [12 20] INFO [2019-12-17 12:28:23,617] jepsen worker 38 - jepsen.util 848 :invoke :read [12 nil] INFO [2019-12-17 12:28:23,625] jepsen worker 36 - jepsen.util 1026 :ok :read [12 43] INFO [2019-12-17 12:28:23,625] jepsen worker 36 - jepsen.util 1026 :invoke :write [12 31] INFO [2019-12-17 12:28:23,628] jepsen worker 63 - jepsen.util 873 :ok :read [21 24] INFO [2019-12-17 12:28:23,629] jepsen worker 63 - jepsen.util 873 :invoke :read [21 nil] INFO [2019-12-17 12:28:23,632] jepsen worker 53 - jepsen.util 953 :ok :write [17 17] INFO [2019-12-17 12:28:23,633] jepsen worker 53 - jepsen.util 953 :invoke :read [17 nil] INFO [2019-12-17 12:28:23,636] jepsen worker 64 - jepsen.util 604 :fail :cas [21 [26 6]] :ValueNotSwapFrom INFO [2019-12-17 12:28:23,636] jepsen worker 64 - jepsen.util 604 :invoke :cas [21 [48 11]] INFO [2019-12-17 12:28:23,644] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":5069,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:28:23,644] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":5069,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:28:23,644] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":5071,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:28:23,644] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":5071,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:28:23,677] jepsen worker 63 - jepsen.util 873 :ok :read [21 24] INFO [2019-12-17 12:28:23,677] jepsen worker 63 - jepsen.util 873 :invoke :write [21 11] INFO [2019-12-17 12:28:23,686] jepsen worker 64 - jepsen.util 604 :fail :cas [21 [48 11]] :ValueNotSwapFrom INFO [2019-12-17 12:28:23,686] jepsen worker 64 - jepsen.util 604 :invoke :cas [21 [33 38]] INFO [2019-12-17 12:28:23,699] jepsen worker 54 - jepsen.util 1044 :fail :cas [18 [40 36]] :ValueNotSwapFrom INFO [2019-12-17 12:28:23,699] jepsen worker 54 - jepsen.util 1044 :invoke :write [18 21] INFO [2019-12-17 12:28:23,703] jepsen worker 29 - jepsen.util 749 :ok :write [9 45] INFO [2019-12-17 12:28:23,703] jepsen worker 29 - jepsen.util 749 :invoke :cas [9 [17 7]] INFO [2019-12-17 12:28:23,723] jepsen worker 69 - jepsen.util 879 :ok :read [23 27] INFO [2019-12-17 12:28:23,724] jepsen worker 69 - jepsen.util 879 :invoke :read [23 nil] INFO [2019-12-17 12:28:23,735] jepsen worker 43 - jepsen.util 1123 :info :write [14 9] :RequestCanceledException INFO [2019-12-17 12:28:23,737] jepsen worker 43 - jepsen.util 1213 :invoke :read [14 nil] INFO [2019-12-17 12:28:23,744] jepsen worker 53 - jepsen.util 953 :ok :read [17 17] INFO [2019-12-17 12:28:23,744] jepsen worker 53 - jepsen.util 953 :invoke :write [17 35] INFO [2019-12-17 12:28:23,747] jepsen worker 64 - jepsen.util 604 :fail :cas [21 [33 38]] :ValueNotSwapFrom INFO [2019-12-17 12:28:23,748] jepsen worker 64 - jepsen.util 604 :invoke :cas [21 [48 12]] INFO [2019-12-17 12:28:23,764] jepsen worker 14 - jepsen.util 824 :ok :read [4 41] INFO [2019-12-17 12:28:23,764] jepsen worker 14 - jepsen.util 824 :invoke :read [4 nil] INFO [2019-12-17 12:28:23,787] jepsen worker 29 - jepsen.util 749 :fail :cas [9 [17 7]] :ValueNotSwapFrom INFO [2019-12-17 12:28:23,787] jepsen worker 29 - jepsen.util 749 :invoke :write [9 37] INFO [2019-12-17 12:28:23,794] jepsen worker 19 - jepsen.util 1189 :ok :write [6 43] INFO [2019-12-17 12:28:23,794] jepsen worker 19 - jepsen.util 1189 :invoke :cas [6 [20 9]] INFO [2019-12-17 12:28:23,802] jepsen worker 64 - jepsen.util 604 :fail :cas [21 [48 12]] :ValueNotSwapFrom INFO [2019-12-17 12:28:23,802] jepsen worker 64 - jepsen.util 604 :invoke :read [21 nil] INFO [2019-12-17 12:28:23,805] jepsen worker 87 - jepsen.util 987 :fail :cas [29 [9 26]] :ValueNotSwapFrom INFO [2019-12-17 12:28:23,805] jepsen worker 87 - jepsen.util 987 :invoke :cas [29 [9 46]] INFO [2019-12-17 12:28:23,818] jepsen worker 63 - jepsen.util 873 :ok :write [21 11] INFO [2019-12-17 12:28:23,818] jepsen worker 63 - jepsen.util 873 :invoke :cas [21 [40 24]] INFO [2019-12-17 12:28:23,821] jepsen worker 26 - jepsen.util 1106 :info :write [8 45] :RequestCanceledException INFO [2019-12-17 12:28:23,827] jepsen worker 26 - jepsen.util 1196 :invoke :cas [8 [25 17]] INFO [2019-12-17 12:28:23,827] jepsen worker 30 - jepsen.util 1110 :ok :write [10 30] INFO [2019-12-17 12:28:23,827] jepsen worker 30 - jepsen.util 1110 :invoke :cas [10 [27 42]] INFO [2019-12-17 12:28:23,851] jepsen worker 74 - jepsen.util 794 :fail :cas [24 [15 45]] :ValueNotSwapFrom INFO [2019-12-17 12:28:23,851] jepsen worker 74 - jepsen.util 794 :invoke :write [24 22] INFO [2019-12-17 12:28:23,862] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":5091,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:28:23,863] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":5091,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:28:23,863] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":5092,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:28:23,863] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":5092,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:28:23,864] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":5093,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:28:23,864] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":5093,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:28:23,870] jepsen worker 21 - jepsen.util 1101 :ok :read [7 20] INFO [2019-12-17 12:28:23,871] jepsen worker 21 - jepsen.util 1101 :invoke :read [7 nil] INFO [2019-12-17 12:28:23,873] jepsen worker 43 - jepsen.util 1213 :ok :read [14 27] INFO [2019-12-17 12:28:23,873] jepsen worker 43 - jepsen.util 1213 :invoke :read [14 nil] INFO [2019-12-17 12:28:23,876] jepsen worker 64 - jepsen.util 604 :ok :read [21 11] INFO [2019-12-17 12:28:23,876] jepsen worker 64 - jepsen.util 604 :invoke :read [21 nil] INFO [2019-12-17 12:28:23,879] jepsen worker 51 - jepsen.util 861 :ok :write [17 9] INFO [2019-12-17 12:28:23,879] jepsen worker 51 - jepsen.util 861 :invoke :cas [17 [42 8]] INFO [2019-12-17 12:28:23,884] jepsen worker 19 - jepsen.util 1189 :fail :cas [6 [20 9]] :ValueNotSwapFrom INFO [2019-12-17 12:28:23,885] jepsen worker 19 - jepsen.util 1189 :invoke :read [6 nil] INFO [2019-12-17 12:28:23,886] jepsen worker 63 - jepsen.util 873 :fail :cas [21 [40 24]] :ValueNotSwapFrom INFO [2019-12-17 12:28:23,886] jepsen worker 63 - jepsen.util 873 :invoke :cas [21 [19 29]] INFO [2019-12-17 12:28:23,889] jepsen worker 42 - jepsen.util 1302 :ok :read [14 27] INFO [2019-12-17 12:28:23,889] jepsen worker 42 - jepsen.util 1302 :invoke :read [14 nil] INFO [2019-12-17 12:28:23,903] jepsen worker 88 - jepsen.util 898 :fail :cas [29 [39 36]] :ValueNotSwapFrom INFO [2019-12-17 12:28:23,903] jepsen worker 88 - jepsen.util 898 :invoke :cas [29 [29 36]] INFO [2019-12-17 12:28:23,945] jepsen worker 80 - jepsen.util 1070 :info :write [26 25] :RequestCanceledException INFO [2019-12-17 12:28:23,945] jepsen worker 56 - jepsen.util 956 :info :write [18 2] :RequestCanceledException INFO [2019-12-17 12:28:23,945] jepsen worker 25 - jepsen.util 1195 :info :write [8 6] :RequestCanceledException INFO [2019-12-17 12:28:23,947] jepsen worker 56 - jepsen.util 1046 :invoke :read [18 nil] INFO [2019-12-17 12:28:23,947] jepsen worker 80 - jepsen.util 1160 :invoke :read [26 nil] INFO [2019-12-17 12:28:23,947] jepsen worker 25 - jepsen.util 1285 :invoke :cas [8 [0 47]] INFO [2019-12-17 12:28:23,947] jepsen worker 30 - jepsen.util 1110 :fail :cas [10 [27 42]] :ValueNotSwapFrom INFO [2019-12-17 12:28:23,948] jepsen worker 30 - jepsen.util 1110 :invoke :write [10 20] INFO [2019-12-17 12:28:23,955] jepsen worker 89 - jepsen.util 1079 :fail :cas [29 [23 42]] :ValueNotSwapFrom INFO [2019-12-17 12:28:23,955] jepsen worker 89 - jepsen.util 1079 :invoke :read [29 nil] INFO [2019-12-17 12:28:23,970] jepsen worker 5 - jepsen.util 635 :ok :read [1 21] INFO [2019-12-17 12:28:23,970] jepsen worker 5 - jepsen.util 635 :invoke :cas [1 [39 29]] INFO [2019-12-17 12:28:23,981] jepsen worker 21 - jepsen.util 1101 :ok :read [7 20] INFO [2019-12-17 12:28:23,982] jepsen worker 21 - jepsen.util 1101 :invoke :write [7 33] INFO [2019-12-17 12:28:24,005] jepsen worker 0 - jepsen.util 1170 :info :cas [0 [31 36]] :RequestCanceledException INFO [2019-12-17 12:28:24,006] jepsen worker 0 - jepsen.util 1260 :invoke :write [0 10] INFO [2019-12-17 12:28:24,015] jepsen worker 63 - jepsen.util 873 :fail :cas [21 [19 29]] :ValueNotSwapFrom INFO [2019-12-17 12:28:24,015] jepsen worker 63 - jepsen.util 873 :invoke :read [21 nil] INFO [2019-12-17 12:28:24,021] jepsen worker 6 - jepsen.util 1086 :info :write [2 12] :RequestCanceledException INFO [2019-12-17 12:28:24,021] jepsen worker 50 - jepsen.util 1220 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:28:24,021] jepsen worker 50 - jepsen.util 1220 :invoke :write [16 37] INFO [2019-12-17 12:28:24,022] jepsen worker 6 - jepsen.util 1176 :invoke :cas [2 [16 6]] INFO [2019-12-17 12:28:24,032] jepsen worker 38 - jepsen.util 848 :ok :read [12 20] INFO [2019-12-17 12:28:24,032] jepsen worker 38 - jepsen.util 848 :invoke :read [12 nil] INFO [2019-12-17 12:28:24,052] jepsen worker 2 - jepsen.util 812 :info :cas [0 [24 48]] :RequestCanceledException INFO [2019-12-17 12:28:24,052] jepsen worker 17 - jepsen.util 1007 :info :cas [5 [9 23]] :RequestCanceledException INFO [2019-12-17 12:28:24,052] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:28:24,052] jepsen nemesis - couchbase.util Rebalance check count: 32 INFO [2019-12-17 12:28:24,052] jepsen nemesis - couchbase.util Rebalance stuck count: 13 INFO [2019-12-17 12:28:24,053] jepsen worker 2 - jepsen.util 902 :invoke :cas [0 [32 31]] INFO [2019-12-17 12:28:24,053] jepsen worker 17 - jepsen.util 1097 :invoke :cas [5 [6 8]] INFO [2019-12-17 12:28:24,064] jepsen worker 29 - jepsen.util 749 :ok :write [9 37] INFO [2019-12-17 12:28:24,064] jepsen worker 29 - jepsen.util 749 :invoke :read [9 nil] INFO [2019-12-17 12:28:24,077] jepsen worker 63 - jepsen.util 873 :ok :read [21 11] INFO [2019-12-17 12:28:24,077] jepsen worker 63 - jepsen.util 873 :invoke :read [21 nil] INFO [2019-12-17 12:28:24,081] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":5111,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:28:24,081] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":5111,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:28:24,081] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":5114,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:28:24,081] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":5114,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:28:24,082] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":5115,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:28:24,082] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":5115,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:28:24,084] jepsen worker 64 - jepsen.util 604 :ok :read [21 11] INFO [2019-12-17 12:28:24,085] jepsen worker 64 - jepsen.util 604 :invoke :read [21 nil] INFO [2019-12-17 12:28:24,088] jepsen worker 15 - jepsen.util 915 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:28:24,088] jepsen worker 66 - jepsen.util 876 :info :write [22 33] :RequestCanceledException INFO [2019-12-17 12:28:24,088] jepsen worker 85 - jepsen.util 895 :info :write [28 36] :RequestCanceledException INFO [2019-12-17 12:28:24,088] jepsen worker 68 - jepsen.util 698 :info :cas [22 [30 4]] :RequestCanceledException INFO [2019-12-17 12:28:24,088] jepsen worker 82 - jepsen.util 1432 :info :write [27 0] :RequestCanceledException INFO [2019-12-17 12:28:24,088] jepsen worker 84 - jepsen.util 894 :info :cas [28 [26 0]] :RequestCanceledException INFO [2019-12-17 12:28:24,088] jepsen worker 15 - jepsen.util 915 :invoke :read [5 nil] INFO [2019-12-17 12:28:24,089] jepsen worker 66 - jepsen.util 966 :invoke :read [22 nil] INFO [2019-12-17 12:28:24,089] jepsen worker 84 - jepsen.util 984 :invoke :read [28 nil] INFO [2019-12-17 12:28:24,089] jepsen worker 85 - jepsen.util 985 :invoke :write [28 27] INFO [2019-12-17 12:28:24,089] jepsen worker 82 - jepsen.util 1522 :invoke :write [27 25] INFO [2019-12-17 12:28:24,089] jepsen worker 68 - jepsen.util 788 :invoke :write [22 20] INFO [2019-12-17 12:28:24,095] jepsen worker 38 - jepsen.util 848 :ok :read [12 20] INFO [2019-12-17 12:28:24,095] jepsen worker 38 - jepsen.util 848 :invoke :read [12 nil] INFO [2019-12-17 12:28:24,106] jepsen worker 49 - jepsen.util 949 :info :cas [16 [13 4]] :RequestCanceledException INFO [2019-12-17 12:28:24,106] jepsen worker 45 - jepsen.util 1035 :info :write [15 37] :RequestCanceledException INFO [2019-12-17 12:28:24,106] jepsen worker 83 - jepsen.util 1253 :info :cas [27 [43 18]] :RequestCanceledException INFO [2019-12-17 12:28:24,106] jepsen worker 34 - jepsen.util 1114 :info :write [11 17] :RequestCanceledException INFO [2019-12-17 12:28:24,106] jepsen worker 60 - jepsen.util 1140 :info :write [20 48] :RequestCanceledException INFO [2019-12-17 12:28:24,106] jepsen worker 67 - jepsen.util 1327 :info :write [22 10] :RequestCanceledException INFO [2019-12-17 12:28:24,106] jepsen worker 40 - jepsen.util 1480 :info :write [13 44] :RequestCanceledException INFO [2019-12-17 12:28:24,106] jepsen worker 11 - jepsen.util 1181 :info :write [3 14] :RequestCanceledException INFO [2019-12-17 12:28:24,107] jepsen worker 60 - jepsen.util 1230 :invoke :cas [20 [7 48]] INFO [2019-12-17 12:28:24,107] jepsen worker 83 - jepsen.util 1343 :invoke :write [27 10] INFO [2019-12-17 12:28:24,107] jepsen worker 34 - jepsen.util 1204 :invoke :write [11 19] INFO [2019-12-17 12:28:24,107] jepsen worker 67 - jepsen.util 1417 :invoke :write [22 24] INFO [2019-12-17 12:28:24,107] jepsen worker 49 - jepsen.util 1039 :invoke :cas [16 [38 38]] INFO [2019-12-17 12:28:24,107] jepsen worker 40 - jepsen.util 1570 :invoke :read [13 nil] INFO [2019-12-17 12:28:24,107] jepsen worker 45 - jepsen.util 1125 :invoke :write [15 46] INFO [2019-12-17 12:28:24,107] jepsen worker 11 - jepsen.util 1271 :invoke :write [3 47] INFO [2019-12-17 12:28:24,119] jepsen worker 59 - jepsen.util 869 :info :write [19 18] :RequestCanceledException INFO [2019-12-17 12:28:24,119] jepsen worker 1 - jepsen.util 811 :info :write [0 40] :RequestCanceledException INFO [2019-12-17 12:28:24,119] jepsen worker 57 - jepsen.util 1047 :info :write [19 28] :RequestCanceledException INFO [2019-12-17 12:28:24,119] jepsen worker 41 - jepsen.util 851 :info :cas [13 [7 42]] :RequestCanceledException INFO [2019-12-17 12:28:24,119] jepsen worker 81 - jepsen.util 1251 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:28:24,119] jepsen worker 86 - jepsen.util 1076 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:28:24,119] jepsen worker 81 - jepsen.util 1251 :invoke :write [27 26] INFO [2019-12-17 12:28:24,119] jepsen worker 86 - jepsen.util 1076 :invoke :write [28 37] INFO [2019-12-17 12:28:24,120] jepsen worker 57 - jepsen.util 1137 :invoke :cas [19 [23 31]] INFO [2019-12-17 12:28:24,120] jepsen worker 59 - jepsen.util 959 :invoke :cas [19 [24 47]] INFO [2019-12-17 12:28:24,120] jepsen worker 41 - jepsen.util 941 :invoke :cas [13 [1 3]] INFO [2019-12-17 12:28:24,120] jepsen worker 1 - jepsen.util 901 :invoke :write [0 0] INFO [2019-12-17 12:28:24,135] jepsen worker 39 - jepsen.util 1119 :info :cas [13 [14 45]] :RequestCanceledException INFO [2019-12-17 12:28:24,135] jepsen worker 48 - jepsen.util 588 :info :write [16 43] :RequestCanceledException INFO [2019-12-17 12:28:24,135] jepsen worker 16 - jepsen.util 1006 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:28:24,135] jepsen worker 77 - jepsen.util 1067 :info :write [25 35] :RequestCanceledException INFO [2019-12-17 12:28:24,135] jepsen worker 16 - jepsen.util 1006 :invoke :write [5 33] INFO [2019-12-17 12:28:24,136] jepsen worker 48 - jepsen.util 678 :invoke :read [16 nil] INFO [2019-12-17 12:28:24,136] jepsen worker 77 - jepsen.util 1157 :invoke :read [25 nil] INFO [2019-12-17 12:28:24,136] jepsen worker 39 - jepsen.util 1209 :invoke :cas [13 [28 47]] INFO [2019-12-17 12:28:24,182] jepsen worker 38 - jepsen.util 848 :ok :read [12 20] INFO [2019-12-17 12:28:24,183] jepsen worker 38 - jepsen.util 848 :invoke :read [12 nil] INFO [2019-12-17 12:28:24,185] jepsen worker 26 - jepsen.util 1196 :fail :cas [8 [25 17]] :ValueNotSwapFrom INFO [2019-12-17 12:28:24,185] jepsen worker 26 - jepsen.util 1196 :invoke :read [8 nil] INFO [2019-12-17 12:28:24,188] jepsen worker 75 - jepsen.util 1065 :info :write [25 2] :RequestCanceledException INFO [2019-12-17 12:28:24,189] jepsen worker 75 - jepsen.util 1155 :invoke :write [25 36] INFO [2019-12-17 12:28:24,191] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":5154,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:28:24,191] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":5154,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:28:24,191] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":5155,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:28:24,191] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":5155,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:28:24,201] jepsen worker 28 - jepsen.util 748 :ok :write [9 32] INFO [2019-12-17 12:28:24,201] jepsen worker 28 - jepsen.util 748 :invoke :read [9 nil] INFO [2019-12-17 12:28:24,208] jepsen worker 42 - jepsen.util 1302 :ok :read [14 27] INFO [2019-12-17 12:28:24,208] jepsen worker 42 - jepsen.util 1302 :invoke :write [14 30] INFO [2019-12-17 12:28:24,215] jepsen worker 4 - jepsen.util 724 :ok :write [1 43] INFO [2019-12-17 12:28:24,215] jepsen worker 4 - jepsen.util 724 :invoke :read [1 nil] INFO [2019-12-17 12:28:24,221] jepsen worker 18 - jepsen.util 1008 :fail :read [6 nil] :RequestTimeoutException INFO [2019-12-17 12:28:24,222] jepsen worker 18 - jepsen.util 1008 :invoke :write [6 30] INFO [2019-12-17 12:28:24,227] jepsen worker 29 - jepsen.util 749 :ok :read [9 37] INFO [2019-12-17 12:28:24,228] jepsen worker 29 - jepsen.util 749 :invoke :cas [9 [34 11]] INFO [2019-12-17 12:28:24,235] jepsen worker 76 - jepsen.util 1066 :info :write [25 14] :RequestCanceledException INFO [2019-12-17 12:28:24,236] jepsen worker 76 - jepsen.util 1156 :invoke :read [25 nil] INFO [2019-12-17 12:28:24,250] jepsen worker 37 - jepsen.util 847 :fail :cas [12 [43 14]] :ValueNotSwapFrom INFO [2019-12-17 12:28:24,251] jepsen worker 37 - jepsen.util 847 :invoke :write [12 8] INFO [2019-12-17 12:28:24,252] jepsen worker 71 - jepsen.util 701 :info :write [23 21] :RequestCanceledException INFO [2019-12-17 12:28:24,253] jepsen worker 71 - jepsen.util 791 :invoke :cas [23 [14 33]] INFO [2019-12-17 12:28:24,271] jepsen worker 77 - jepsen.util 1157 :ok :read [25 2] INFO [2019-12-17 12:28:24,272] jepsen worker 77 - jepsen.util 1157 :invoke :write [25 11] INFO [2019-12-17 12:28:24,278] jepsen worker 79 - jepsen.util 889 :ok :write [26 22] INFO [2019-12-17 12:28:24,279] jepsen worker 79 - jepsen.util 889 :invoke :write [26 21] INFO [2019-12-17 12:28:24,279] jepsen worker 13 - jepsen.util 1003 :ok :write [4 1] INFO [2019-12-17 12:28:24,279] jepsen worker 13 - jepsen.util 1003 :invoke :write [4 46] INFO [2019-12-17 12:28:24,285] jepsen worker 26 - jepsen.util 1196 :ok :read [8 14] INFO [2019-12-17 12:28:24,285] jepsen worker 26 - jepsen.util 1196 :invoke :write [8 27] INFO [2019-12-17 12:28:24,300] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":5162,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:28:24,300] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":5162,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:28:24,301] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":5166,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:28:24,301] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":5166,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:28:24,306] jepsen worker 78 - jepsen.util 978 :info :write [26 13] :RequestCanceledException INFO [2019-12-17 12:28:24,307] jepsen worker 78 - jepsen.util 1068 :invoke :cas [26 [19 42]] INFO [2019-12-17 12:28:24,309] jepsen worker 63 - jepsen.util 873 :ok :read [21 11] INFO [2019-12-17 12:28:24,309] jepsen worker 63 - jepsen.util 873 :invoke :write [21 38] INFO [2019-12-17 12:28:24,310] jepsen worker 64 - jepsen.util 604 :ok :read [21 11] INFO [2019-12-17 12:28:24,310] jepsen worker 64 - jepsen.util 604 :invoke :read [21 nil] INFO [2019-12-17 12:28:24,323] jepsen worker 36 - jepsen.util 1026 :ok :write [12 31] INFO [2019-12-17 12:28:24,324] jepsen worker 36 - jepsen.util 1026 :invoke :cas [12 [23 10]] INFO [2019-12-17 12:28:24,331] jepsen worker 89 - jepsen.util 1079 :ok :read [29 27] INFO [2019-12-17 12:28:24,331] jepsen worker 89 - jepsen.util 1079 :invoke :read [29 nil] INFO [2019-12-17 12:28:24,332] jepsen worker 28 - jepsen.util 748 :ok :read [9 32] INFO [2019-12-17 12:28:24,332] jepsen worker 28 - jepsen.util 748 :invoke :read [9 nil] INFO [2019-12-17 12:28:24,336] jepsen worker 58 - jepsen.util 1048 :info :cas [19 [18 6]] :RequestCanceledException INFO [2019-12-17 12:28:24,337] jepsen worker 58 - jepsen.util 1138 :invoke :read [19 nil] INFO [2019-12-17 12:28:24,339] jepsen worker 14 - jepsen.util 824 :ok :read [4 1] INFO [2019-12-17 12:28:24,340] jepsen worker 14 - jepsen.util 824 :invoke :read [4 nil] INFO [2019-12-17 12:28:24,347] jepsen worker 64 - jepsen.util 604 :ok :read [21 11] INFO [2019-12-17 12:28:24,348] jepsen worker 64 - jepsen.util 604 :invoke :cas [21 [41 24]] INFO [2019-12-17 12:28:24,355] jepsen worker 51 - jepsen.util 861 :fail :cas [17 [42 8]] :ValueNotSwapFrom INFO [2019-12-17 12:28:24,355] jepsen worker 51 - jepsen.util 861 :invoke :write [17 16] INFO [2019-12-17 12:28:24,369] jepsen worker 25 - jepsen.util 1285 :fail :cas [8 [0 47]] :ValueNotSwapFrom INFO [2019-12-17 12:28:24,369] jepsen worker 25 - jepsen.util 1285 :invoke :read [8 nil] INFO [2019-12-17 12:28:24,383] jepsen worker 87 - jepsen.util 987 :fail :cas [29 [9 46]] :ValueNotSwapFrom INFO [2019-12-17 12:28:24,383] jepsen worker 87 - jepsen.util 987 :invoke :write [29 17] INFO [2019-12-17 12:28:24,390] jepsen worker 5 - jepsen.util 635 :fail :cas [1 [39 29]] :ValueNotSwapFrom INFO [2019-12-17 12:28:24,390] jepsen worker 5 - jepsen.util 635 :invoke :write [1 39] INFO [2019-12-17 12:28:24,397] jepsen worker 28 - jepsen.util 748 :ok :read [9 32] INFO [2019-12-17 12:28:24,398] jepsen worker 28 - jepsen.util 748 :invoke :write [9 17] INFO [2019-12-17 12:28:24,405] jepsen worker 64 - jepsen.util 604 :fail :cas [21 [41 24]] :ValueNotSwapFrom INFO [2019-12-17 12:28:24,405] jepsen worker 64 - jepsen.util 604 :invoke :read [21 nil] INFO [2019-12-17 12:28:24,407] jepsen worker 54 - jepsen.util 1044 :ok :write [18 21] INFO [2019-12-17 12:28:24,407] jepsen worker 54 - jepsen.util 1044 :invoke :read [18 nil] INFO [2019-12-17 12:28:24,428] jepsen worker 43 - jepsen.util 1213 :ok :read [14 27] INFO [2019-12-17 12:28:24,428] jepsen worker 64 - jepsen.util 604 :ok :read [21 11] INFO [2019-12-17 12:28:24,428] jepsen worker 88 - jepsen.util 898 :fail :cas [29 [29 36]] :ValueNotSwapFrom INFO [2019-12-17 12:28:24,428] jepsen worker 43 - jepsen.util 1213 :invoke :cas [14 [46 27]] INFO [2019-12-17 12:28:24,428] jepsen worker 64 - jepsen.util 604 :invoke :cas [21 [9 17]] INFO [2019-12-17 12:28:24,428] jepsen worker 88 - jepsen.util 898 :invoke :write [29 27] INFO [2019-12-17 12:28:24,435] jepsen worker 62 - jepsen.util 1052 :info :write [20 29] :RequestCanceledException INFO [2019-12-17 12:28:24,436] jepsen worker 62 - jepsen.util 1142 :invoke :cas [20 [14 14]] INFO [2019-12-17 12:28:24,444] jepsen worker 6 - jepsen.util 1176 :fail :cas [2 [16 6]] :ValueNotSwapFrom INFO [2019-12-17 12:28:24,445] jepsen worker 6 - jepsen.util 1176 :invoke :read [2 nil] INFO [2019-12-17 12:28:24,449] jepsen worker 80 - jepsen.util 1160 :ok :read [26 22] INFO [2019-12-17 12:28:24,449] jepsen worker 80 - jepsen.util 1160 :invoke :cas [26 [32 11]] INFO [2019-12-17 12:28:24,452] jepsen worker 10 - jepsen.util 1180 :info :write [3 12] :RequestCanceledException INFO [2019-12-17 12:28:24,453] jepsen worker 10 - jepsen.util 1270 :invoke :cas [3 [31 37]] INFO [2019-12-17 12:28:24,469] jepsen worker 64 - jepsen.util 604 :fail :cas [21 [9 17]] :ValueNotSwapFrom INFO [2019-12-17 12:28:24,469] jepsen worker 64 - jepsen.util 604 :invoke :cas [21 [13 22]] INFO [2019-12-17 12:28:24,471] jepsen worker 12 - jepsen.util 1002 :fail :cas [4 [41 27]] :ValueNotSwapFrom INFO [2019-12-17 12:28:24,471] jepsen worker 12 - jepsen.util 1002 :invoke :cas [4 [13 46]] INFO [2019-12-17 12:28:24,490] jepsen worker 4 - jepsen.util 724 :ok :read [1 43] INFO [2019-12-17 12:28:24,490] jepsen worker 4 - jepsen.util 724 :invoke :cas [1 [16 8]] INFO [2019-12-17 12:28:24,491] jepsen worker 64 - jepsen.util 604 :fail :cas [21 [13 22]] :ValueNotSwapFrom INFO [2019-12-17 12:28:24,492] jepsen worker 64 - jepsen.util 604 :invoke :cas [21 [43 33]] INFO [2019-12-17 12:28:24,494] jepsen worker 76 - jepsen.util 1156 :ok :read [25 2] INFO [2019-12-17 12:28:24,494] jepsen worker 76 - jepsen.util 1156 :invoke :cas [25 [31 9]] INFO [2019-12-17 12:28:24,510] jepsen worker 64 - jepsen.util 604 :fail :cas [21 [43 33]] :ValueNotSwapFrom INFO [2019-12-17 12:28:24,510] jepsen worker 64 - jepsen.util 604 :invoke :write [21 48] INFO [2019-12-17 12:28:24,520] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":5195,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:28:24,520] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":5195,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:28:24,520] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":5199,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:28:24,520] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":5199,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:28:24,524] jepsen worker 28 - jepsen.util 748 :ok :write [9 17] INFO [2019-12-17 12:28:24,524] jepsen worker 28 - jepsen.util 748 :invoke :write [9 4] INFO [2019-12-17 12:28:24,538] jepsen worker 38 - jepsen.util 848 :ok :read [12 31] INFO [2019-12-17 12:28:24,538] jepsen worker 38 - jepsen.util 848 :invoke :cas [12 [18 28]] INFO [2019-12-17 12:28:24,552] jepsen worker 69 - jepsen.util 879 :ok :read [23 21] INFO [2019-12-17 12:28:24,552] jepsen worker 69 - jepsen.util 879 :invoke :cas [23 [32 22]] INFO [2019-12-17 12:28:24,563] jepsen worker 70 - jepsen.util 1150 :fail :cas [23 [10 46]] :ValueNotSwapFrom INFO [2019-12-17 12:28:24,563] jepsen worker 70 - jepsen.util 1150 :invoke :cas [23 [1 13]] INFO [2019-12-17 12:28:24,573] jepsen worker 36 - jepsen.util 1026 :fail :cas [12 [23 10]] :ValueNotSwapFrom INFO [2019-12-17 12:28:24,573] jepsen worker 36 - jepsen.util 1026 :invoke :cas [12 [4 21]] INFO [2019-12-17 12:28:24,580] jepsen worker 71 - jepsen.util 791 :fail :cas [23 [14 33]] :ValueNotSwapFrom INFO [2019-12-17 12:28:24,580] jepsen worker 71 - jepsen.util 791 :invoke :write [23 44] INFO [2019-12-17 12:28:24,593] jepsen worker 25 - jepsen.util 1285 :ok :read [8 14] INFO [2019-12-17 12:28:24,593] jepsen worker 25 - jepsen.util 1285 :invoke :cas [8 [14 42]] INFO [2019-12-17 12:28:24,599] jepsen worker 64 - jepsen.util 604 :ok :write [21 48] INFO [2019-12-17 12:28:24,599] jepsen worker 64 - jepsen.util 604 :invoke :write [21 17] INFO [2019-12-17 12:28:24,618] jepsen worker 62 - jepsen.util 1142 :fail :cas [20 [14 14]] :ValueNotSwapFrom INFO [2019-12-17 12:28:24,618] jepsen worker 62 - jepsen.util 1142 :invoke :read [20 nil] INFO [2019-12-17 12:28:24,618] jepsen worker 28 - jepsen.util 748 :ok :write [9 4] INFO [2019-12-17 12:28:24,618] jepsen worker 28 - jepsen.util 748 :invoke :read [9 nil] INFO [2019-12-17 12:28:24,621] jepsen worker 32 - jepsen.util 932 :info :cas [10 [34 23]] :RequestCanceledException INFO [2019-12-17 12:28:24,621] jepsen worker 55 - jepsen.util 1135 :info :write [18 14] :RequestCanceledException INFO [2019-12-17 12:28:24,622] jepsen worker 32 - jepsen.util 1022 :invoke :write [10 2] INFO [2019-12-17 12:28:24,622] jepsen worker 55 - jepsen.util 1225 :invoke :write [18 40] INFO [2019-12-17 12:28:24,635] jepsen worker 65 - jepsen.util 695 :info :write [21 17] :RequestCanceledException INFO [2019-12-17 12:28:24,636] jepsen worker 65 - jepsen.util 785 :invoke :cas [21 [9 2]] INFO [2019-12-17 12:28:24,653] jepsen worker 47 - jepsen.util 1217 :info :write [15 40] :RequestCanceledException INFO [2019-12-17 12:28:24,653] jepsen worker 46 - jepsen.util 1216 :fail :read [15 nil] :RequestTimeoutException INFO [2019-12-17 12:28:24,653] jepsen worker 46 - jepsen.util 1216 :invoke :write [15 30] INFO [2019-12-17 12:28:24,654] jepsen worker 47 - jepsen.util 1307 :invoke :write [15 12] INFO [2019-12-17 12:28:24,654] jepsen worker 28 - jepsen.util 748 :ok :read [9 4] INFO [2019-12-17 12:28:24,654] jepsen worker 28 - jepsen.util 748 :invoke :read [9 nil] INFO [2019-12-17 12:28:24,677] jepsen worker 28 - jepsen.util 748 :ok :read [9 4] INFO [2019-12-17 12:28:24,677] jepsen worker 28 - jepsen.util 748 :invoke :cas [9 [6 34]] INFO [2019-12-17 12:28:24,683] jepsen worker 54 - jepsen.util 1044 :ok :read [18 21] INFO [2019-12-17 12:28:24,683] jepsen worker 54 - jepsen.util 1044 :invoke :cas [18 [6 15]] INFO [2019-12-17 12:28:24,683] jepsen worker 27 - jepsen.util 747 :fail :read [9 nil] :RequestTimeoutException INFO [2019-12-17 12:28:24,683] jepsen worker 72 - jepsen.util 792 :info :cas [24 [0 49]] :RequestCanceledException INFO [2019-12-17 12:28:24,683] jepsen worker 73 - jepsen.util 883 :info :write [24 42] :RequestCanceledException INFO [2019-12-17 12:28:24,683] jepsen worker 27 - jepsen.util 747 :invoke :cas [9 [33 2]] INFO [2019-12-17 12:28:24,684] jepsen worker 72 - jepsen.util 882 :invoke :write [24 14] INFO [2019-12-17 12:28:24,684] jepsen worker 73 - jepsen.util 973 :invoke :cas [24 [5 15]] INFO [2019-12-17 12:28:24,705] jepsen worker 44 - jepsen.util 944 :info :cas [14 [47 27]] :RequestCanceledException INFO [2019-12-17 12:28:24,706] jepsen worker 44 - jepsen.util 1034 :invoke :write [14 25] INFO [2019-12-17 12:28:24,707] jepsen worker 14 - jepsen.util 824 :ok :read [4 1] INFO [2019-12-17 12:28:24,707] jepsen worker 14 - jepsen.util 824 :invoke :read [4 nil] INFO [2019-12-17 12:28:24,721] jepsen worker 4 - jepsen.util 724 :fail :cas [1 [16 8]] :ValueNotSwapFrom INFO [2019-12-17 12:28:24,721] jepsen worker 9 - jepsen.util 909 :fail :read [3 nil] :RequestTimeoutException INFO [2019-12-17 12:28:24,721] jepsen worker 4 - jepsen.util 724 :invoke :read [1 nil] INFO [2019-12-17 12:28:24,721] jepsen worker 89 - jepsen.util 1079 :ok :read [29 27] INFO [2019-12-17 12:28:24,721] jepsen worker 9 - jepsen.util 909 :invoke :write [3 37] INFO [2019-12-17 12:28:24,721] jepsen worker 89 - jepsen.util 1079 :invoke :cas [29 [34 33]] INFO [2019-12-17 12:28:24,730] jepsen worker 28 - jepsen.util 748 :fail :cas [9 [6 34]] :ValueNotSwapFrom INFO [2019-12-17 12:28:24,730] jepsen worker 28 - jepsen.util 748 :invoke :read [9 nil] INFO [2019-12-17 12:28:24,738] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":5227,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:28:24,738] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":5227,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:28:24,738] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":5230,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:28:24,738] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":5230,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:28:24,738] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":5231,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:28:24,738] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":5231,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:28:24,748] jepsen worker 56 - jepsen.util 1046 :ok :read [18 21] INFO [2019-12-17 12:28:24,749] jepsen worker 56 - jepsen.util 1046 :invoke :write [18 36] INFO [2019-12-17 12:28:24,752] jepsen worker 23 - jepsen.util 923 :info :write [7 20] :RequestCanceledException INFO [2019-12-17 12:28:24,752] jepsen worker 8 - jepsen.util 1178 :info :cas [2 [9 28]] :RequestCanceledException INFO [2019-12-17 12:28:24,752] jepsen worker 22 - jepsen.util 1012 :info :write [7 7] :RequestCanceledException INFO [2019-12-17 12:28:24,753] jepsen worker 22 - jepsen.util 1102 :invoke :read [7 nil] INFO [2019-12-17 12:28:24,753] jepsen worker 8 - jepsen.util 1268 :invoke :read [2 nil] INFO [2019-12-17 12:28:24,753] jepsen worker 23 - jepsen.util 1013 :invoke :write [7 10] INFO [2019-12-17 12:28:24,775] jepsen worker 4 - jepsen.util 724 :ok :read [1 43] INFO [2019-12-17 12:28:24,775] jepsen worker 4 - jepsen.util 724 :invoke :cas [1 [9 5]] INFO [2019-12-17 12:28:24,782] jepsen worker 22 - jepsen.util 1102 :ok :read [7 20] INFO [2019-12-17 12:28:24,782] jepsen worker 22 - jepsen.util 1102 :invoke :read [7 nil] INFO [2019-12-17 12:28:24,782] jepsen worker 3 - jepsen.util 723 :ok :read [1 43] INFO [2019-12-17 12:28:24,782] jepsen worker 3 - jepsen.util 723 :invoke :write [1 4] INFO [2019-12-17 12:28:24,804] jepsen worker 22 - jepsen.util 1102 :ok :read [7 20] INFO [2019-12-17 12:28:24,805] jepsen worker 22 - jepsen.util 1102 :invoke :read [7 nil] INFO [2019-12-17 12:28:24,819] jepsen worker 78 - jepsen.util 1068 :fail :cas [26 [19 42]] :ValueNotSwapFrom INFO [2019-12-17 12:28:24,820] jepsen worker 78 - jepsen.util 1068 :invoke :read [26 nil] INFO [2019-12-17 12:28:24,821] jepsen worker 31 - jepsen.util 1021 :info :write [10 7] :RequestCanceledException INFO [2019-12-17 12:28:24,822] jepsen worker 31 - jepsen.util 1111 :invoke :read [10 nil] INFO [2019-12-17 12:28:24,828] jepsen worker 27 - jepsen.util 747 :fail :cas [9 [33 2]] :ValueNotSwapFrom INFO [2019-12-17 12:28:24,828] jepsen worker 27 - jepsen.util 747 :invoke :read [9 nil] INFO [2019-12-17 12:28:24,834] jepsen worker 22 - jepsen.util 1102 :ok :read [7 20] INFO [2019-12-17 12:28:24,835] jepsen worker 22 - jepsen.util 1102 :invoke :write [7 23] INFO [2019-12-17 12:28:24,850] jepsen worker 12 - jepsen.util 1002 :fail :cas [4 [13 46]] :ValueNotSwapFrom INFO [2019-12-17 12:28:24,851] jepsen worker 12 - jepsen.util 1002 :invoke :read [4 nil] INFO [2019-12-17 12:28:24,886] jepsen worker 31 - jepsen.util 1111 :ok :read [10 30] INFO [2019-12-17 12:28:24,886] jepsen worker 31 - jepsen.util 1111 :invoke :cas [10 [46 5]] INFO [2019-12-17 12:28:24,893] jepsen worker 80 - jepsen.util 1160 :fail :cas [26 [32 11]] :ValueNotSwapFrom INFO [2019-12-17 12:28:24,893] jepsen worker 80 - jepsen.util 1160 :invoke :read [26 nil] INFO [2019-12-17 12:28:24,896] jepsen worker 27 - jepsen.util 747 :ok :read [9 4] INFO [2019-12-17 12:28:24,896] jepsen worker 27 - jepsen.util 747 :invoke :read [9 nil] INFO [2019-12-17 12:28:24,904] jepsen worker 43 - jepsen.util 1213 :fail :cas [14 [46 27]] :ValueNotSwapFrom INFO [2019-12-17 12:28:24,904] jepsen worker 43 - jepsen.util 1213 :invoke :write [14 46] INFO [2019-12-17 12:28:24,908] jepsen worker 54 - jepsen.util 1044 :fail :cas [18 [6 15]] :ValueNotSwapFrom INFO [2019-12-17 12:28:24,908] jepsen worker 54 - jepsen.util 1044 :invoke :read [18 nil] INFO [2019-12-17 12:28:24,916] jepsen worker 31 - jepsen.util 1111 :fail :cas [10 [46 5]] :ValueNotSwapFrom INFO [2019-12-17 12:28:24,916] jepsen worker 31 - jepsen.util 1111 :invoke :read [10 nil] INFO [2019-12-17 12:28:24,934] jepsen worker 64 - jepsen.util 604 :ok :write [21 17] INFO [2019-12-17 12:28:24,934] jepsen worker 64 - jepsen.util 604 :invoke :read [21 nil] INFO [2019-12-17 12:28:24,955] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":5259,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:28:24,955] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":5259,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:28:24,955] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":5262,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:28:24,955] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":5262,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:28:24,955] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":5263,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:28:24,955] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":5263,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:28:24,955] jepsen worker 27 - jepsen.util 747 :ok :read [9 4] INFO [2019-12-17 12:28:24,956] jepsen worker 27 - jepsen.util 747 :invoke :write [9 16] INFO [2019-12-17 12:28:24,958] jepsen worker 69 - jepsen.util 879 :fail :cas [23 [32 22]] :ValueNotSwapFrom INFO [2019-12-17 12:28:24,958] jepsen worker 69 - jepsen.util 879 :invoke :write [23 11] INFO [2019-12-17 12:28:24,964] jepsen worker 62 - jepsen.util 1142 :ok :read [20 7] INFO [2019-12-17 12:28:24,964] jepsen worker 62 - jepsen.util 1142 :invoke :cas [20 [44 48]] INFO [2019-12-17 12:28:24,970] jepsen worker 6 - jepsen.util 1176 :ok :read [2 15] INFO [2019-12-17 12:28:24,970] jepsen worker 6 - jepsen.util 1176 :invoke :write [2 17] INFO [2019-12-17 12:28:24,975] jepsen worker 31 - jepsen.util 1111 :ok :read [10 30] INFO [2019-12-17 12:28:24,976] jepsen worker 31 - jepsen.util 1111 :invoke :read [10 nil] INFO [2019-12-17 12:28:24,986] jepsen worker 76 - jepsen.util 1156 :fail :cas [25 [31 9]] :ValueNotSwapFrom INFO [2019-12-17 12:28:24,986] jepsen worker 76 - jepsen.util 1156 :invoke :write [25 15] INFO [2019-12-17 12:28:24,993] jepsen worker 64 - jepsen.util 604 :ok :read [21 17] INFO [2019-12-17 12:28:24,993] jepsen worker 64 - jepsen.util 604 :invoke :read [21 nil] INFO [2019-12-17 12:28:25,002] jepsen worker 70 - jepsen.util 1150 :fail :cas [23 [1 13]] :ValueNotSwapFrom INFO [2019-12-17 12:28:25,002] jepsen worker 70 - jepsen.util 1150 :invoke :cas [23 [43 16]] INFO [2019-12-17 12:28:25,017] jepsen worker 64 - jepsen.util 604 :ok :read [21 17] INFO [2019-12-17 12:28:25,017] jepsen worker 64 - jepsen.util 604 :invoke :read [21 nil] INFO [2019-12-17 12:28:25,020] jepsen worker 36 - jepsen.util 1026 :fail :cas [12 [4 21]] :ValueNotSwapFrom INFO [2019-12-17 12:28:25,021] jepsen worker 36 - jepsen.util 1026 :invoke :cas [12 [32 30]] INFO [2019-12-17 12:28:25,030] jepsen worker 74 - jepsen.util 794 :ok :write [24 22] INFO [2019-12-17 12:28:25,030] jepsen worker 74 - jepsen.util 794 :invoke :write [24 27] INFO [2019-12-17 12:28:25,034] jepsen worker 45 - jepsen.util 1125 :ok :write [15 46] INFO [2019-12-17 12:28:25,034] jepsen worker 45 - jepsen.util 1125 :invoke :read [15 nil] INFO [2019-12-17 12:28:25,035] jepsen worker 18 - jepsen.util 1008 :ok :write [6 30] INFO [2019-12-17 12:28:25,035] jepsen worker 18 - jepsen.util 1008 :invoke :write [6 36] INFO [2019-12-17 12:28:25,035] jepsen worker 64 - jepsen.util 604 :ok :read [21 17] INFO [2019-12-17 12:28:25,036] jepsen worker 64 - jepsen.util 604 :invoke :cas [21 [20 4]] INFO [2019-12-17 12:28:25,046] jepsen worker 14 - jepsen.util 824 :ok :read [4 1] INFO [2019-12-17 12:28:25,046] jepsen worker 14 - jepsen.util 824 :invoke :cas [4 [2 40]] INFO [2019-12-17 12:28:25,052] jepsen worker 61 - jepsen.util 1141 :info :cas [20 [16 21]] :RequestCanceledException INFO [2019-12-17 12:28:25,053] jepsen worker 61 - jepsen.util 1231 :invoke :read [20 nil] INFO [2019-12-17 12:28:25,061] jepsen worker 80 - jepsen.util 1160 :ok :read [26 22] INFO [2019-12-17 12:28:25,061] jepsen worker 80 - jepsen.util 1160 :invoke :read [26 nil] INFO [2019-12-17 12:28:25,062] jepsen worker 27 - jepsen.util 747 :ok :write [9 16] INFO [2019-12-17 12:28:25,062] jepsen worker 27 - jepsen.util 747 :invoke :cas [9 [22 33]] INFO [2019-12-17 12:28:25,064] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":5279,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:28:25,064] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":5279,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:28:25,064] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":5283,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:28:25,064] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":5283,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:28:25,073] jepsen worker 54 - jepsen.util 1044 :ok :read [18 21] INFO [2019-12-17 12:28:25,073] jepsen worker 54 - jepsen.util 1044 :invoke :read [18 nil] INFO [2019-12-17 12:28:25,078] jepsen worker 30 - jepsen.util 1110 :ok :write [10 20] INFO [2019-12-17 12:28:25,078] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:28:25,078] jepsen nemesis - couchbase.util Rebalance check count: 33 INFO [2019-12-17 12:28:25,078] jepsen worker 30 - jepsen.util 1110 :invoke :cas [10 [20 9]] INFO [2019-12-17 12:28:25,078] jepsen nemesis - couchbase.util Rebalance stuck count: 14 INFO [2019-12-17 12:28:25,085] jepsen worker 65 - jepsen.util 785 :fail :cas [21 [9 2]] :ValueNotSwapFrom INFO [2019-12-17 12:28:25,085] jepsen worker 65 - jepsen.util 785 :invoke :read [21 nil] INFO [2019-12-17 12:28:25,090] jepsen worker 64 - jepsen.util 604 :fail :cas [21 [20 4]] :ValueNotSwapFrom INFO [2019-12-17 12:28:25,090] jepsen worker 64 - jepsen.util 604 :invoke :write [21 1] INFO [2019-12-17 12:28:25,117] jepsen worker 80 - jepsen.util 1160 :ok :read [26 22] INFO [2019-12-17 12:28:25,117] jepsen worker 80 - jepsen.util 1160 :invoke :cas [26 [1 33]] INFO [2019-12-17 12:28:25,120] jepsen worker 27 - jepsen.util 747 :fail :cas [9 [22 33]] :ValueNotSwapFrom INFO [2019-12-17 12:28:25,121] jepsen worker 27 - jepsen.util 747 :invoke :read [9 nil] INFO [2019-12-17 12:28:25,155] jepsen worker 24 - jepsen.util 1014 :info :write [8 26] :RequestCanceledException INFO [2019-12-17 12:28:25,157] jepsen worker 24 - jepsen.util 1104 :invoke :cas [8 [20 42]] INFO [2019-12-17 12:28:25,166] jepsen worker 10 - jepsen.util 1270 :fail :cas [3 [31 37]] :ValueNotSwapFrom INFO [2019-12-17 12:28:25,166] jepsen worker 10 - jepsen.util 1270 :invoke :cas [3 [12 24]] INFO [2019-12-17 12:28:25,169] jepsen worker 54 - jepsen.util 1044 :ok :read [18 21] INFO [2019-12-17 12:28:25,169] jepsen worker 54 - jepsen.util 1044 :invoke :read [18 nil] INFO [2019-12-17 12:28:25,198] jepsen worker 78 - jepsen.util 1068 :ok :read [26 22] INFO [2019-12-17 12:28:25,198] jepsen worker 78 - jepsen.util 1068 :invoke :write [26 39] INFO [2019-12-17 12:28:25,216] jepsen worker 27 - jepsen.util 747 :ok :read [9 16] INFO [2019-12-17 12:28:25,216] jepsen worker 27 - jepsen.util 747 :invoke :write [9 34] INFO [2019-12-17 12:28:25,222] jepsen worker 7 - jepsen.util 1267 :info :write [2 7] :RequestCanceledException INFO [2019-12-17 12:28:25,222] jepsen worker 33 - jepsen.util 1293 :info :write [11 17] :RequestCanceledException INFO [2019-12-17 12:28:25,223] jepsen worker 73 - jepsen.util 973 :fail :cas [24 [5 15]] :ValueNotSwapFrom INFO [2019-12-17 12:28:25,223] jepsen worker 7 - jepsen.util 1357 :invoke :cas [2 [36 44]] INFO [2019-12-17 12:28:25,223] jepsen worker 73 - jepsen.util 973 :invoke :cas [24 [39 13]] INFO [2019-12-17 12:28:25,223] jepsen worker 33 - jepsen.util 1383 :invoke :write [11 35] INFO [2019-12-17 12:28:25,243] jepsen worker 38 - jepsen.util 848 :fail :cas [12 [18 28]] :ValueNotSwapFrom INFO [2019-12-17 12:28:25,243] jepsen worker 38 - jepsen.util 848 :invoke :read [12 nil] INFO [2019-12-17 12:28:25,262] jepsen worker 80 - jepsen.util 1160 :fail :cas [26 [1 33]] :ValueNotSwapFrom INFO [2019-12-17 12:28:25,263] jepsen worker 80 - jepsen.util 1160 :invoke :read [26 nil] INFO [2019-12-17 12:28:25,281] jepsen worker 10 - jepsen.util 1270 :fail :cas [3 [12 24]] :ValueNotSwapFrom INFO [2019-12-17 12:28:25,282] jepsen worker 10 - jepsen.util 1270 :invoke :read [3 nil] INFO [2019-12-17 12:28:25,283] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":5307,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:28:25,283] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":5314,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:28:25,283] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":5307,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:28:25,283] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":5314,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:28:25,283] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":5308,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:28:25,283] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":5317,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:28:25,283] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":5308,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:28:25,283] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":5317,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:28:25,314] jepsen worker 7 - jepsen.util 1357 :fail :cas [2 [36 44]] :ValueNotSwapFrom INFO [2019-12-17 12:28:25,314] jepsen worker 7 - jepsen.util 1357 :invoke :read [2 nil] INFO [2019-12-17 12:28:25,320] jepsen worker 73 - jepsen.util 973 :fail :cas [24 [39 13]] :ValueNotSwapFrom INFO [2019-12-17 12:28:25,320] jepsen worker 73 - jepsen.util 973 :invoke :cas [24 [3 23]] INFO [2019-12-17 12:28:25,322] jepsen worker 89 - jepsen.util 1079 :fail :cas [29 [34 33]] :ValueNotSwapFrom INFO [2019-12-17 12:28:25,323] jepsen worker 89 - jepsen.util 1079 :invoke :write [29 6] INFO [2019-12-17 12:28:25,325] jepsen worker 36 - jepsen.util 1026 :fail :cas [12 [32 30]] :ValueNotSwapFrom INFO [2019-12-17 12:28:25,325] jepsen worker 36 - jepsen.util 1026 :invoke :write [12 38] INFO [2019-12-17 12:28:25,356] jepsen worker 88 - jepsen.util 898 :ok :write [29 27] INFO [2019-12-17 12:28:25,356] jepsen worker 88 - jepsen.util 898 :invoke :cas [29 [40 14]] INFO [2019-12-17 12:28:25,357] jepsen worker 27 - jepsen.util 747 :ok :write [9 34] INFO [2019-12-17 12:28:25,357] jepsen worker 27 - jepsen.util 747 :invoke :cas [9 [34 2]] INFO [2019-12-17 12:28:25,359] jepsen worker 14 - jepsen.util 824 :fail :cas [4 [2 40]] :ValueNotSwapFrom INFO [2019-12-17 12:28:25,359] jepsen worker 14 - jepsen.util 824 :invoke :write [4 5] INFO [2019-12-17 12:28:25,370] jepsen worker 7 - jepsen.util 1357 :ok :read [2 15] INFO [2019-12-17 12:28:25,371] jepsen worker 7 - jepsen.util 1357 :invoke :write [2 9] INFO [2019-12-17 12:28:25,384] jepsen worker 64 - jepsen.util 604 :ok :write [21 1] INFO [2019-12-17 12:28:25,384] jepsen worker 64 - jepsen.util 604 :invoke :read [21 nil] INFO [2019-12-17 12:28:25,396] jepsen worker 4 - jepsen.util 724 :fail :cas [1 [9 5]] :ValueNotSwapFrom INFO [2019-12-17 12:28:25,396] jepsen worker 4 - jepsen.util 724 :invoke :write [1 48] INFO [2019-12-17 12:28:25,420] jepsen worker 64 - jepsen.util 604 :ok :read [21 1] INFO [2019-12-17 12:28:25,421] jepsen worker 64 - jepsen.util 604 :invoke :read [21 nil] INFO [2019-12-17 12:28:25,426] jepsen worker 5 - jepsen.util 635 :ok :write [1 39] INFO [2019-12-17 12:28:25,426] jepsen worker 5 - jepsen.util 635 :invoke :write [1 24] INFO [2019-12-17 12:28:25,436] jepsen worker 20 - jepsen.util 1010 :info :write [6 22] :RequestCanceledException INFO [2019-12-17 12:28:25,437] jepsen worker 20 - jepsen.util 1100 :invoke :write [6 30] INFO [2019-12-17 12:28:25,450] jepsen worker 64 - jepsen.util 604 :ok :read [21 1] INFO [2019-12-17 12:28:25,450] jepsen worker 64 - jepsen.util 604 :invoke :write [21 36] INFO [2019-12-17 12:28:25,471] jepsen worker 27 - jepsen.util 747 :ok :cas [9 [34 2]] INFO [2019-12-17 12:28:25,471] jepsen worker 27 - jepsen.util 747 :invoke :cas [9 [28 49]] INFO [2019-12-17 12:28:25,481] jepsen worker 53 - jepsen.util 953 :ok :write [17 35] INFO [2019-12-17 12:28:25,481] jepsen worker 53 - jepsen.util 953 :invoke :cas [17 [39 31]] INFO [2019-12-17 12:28:25,502] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":5342,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:28:25,502] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":5342,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:28:25,502] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":5344,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:28:25,502] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":5344,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:28:25,513] jepsen worker 27 - jepsen.util 747 :fail :cas [9 [28 49]] :ValueNotSwapFrom INFO [2019-12-17 12:28:25,513] jepsen worker 27 - jepsen.util 747 :invoke :cas [9 [36 38]] INFO [2019-12-17 12:28:25,514] jepsen worker 32 - jepsen.util 1022 :ok :write [10 2] INFO [2019-12-17 12:28:25,514] jepsen worker 46 - jepsen.util 1216 :ok :write [15 30] INFO [2019-12-17 12:28:25,514] jepsen worker 32 - jepsen.util 1022 :invoke :read [10 nil] INFO [2019-12-17 12:28:25,514] jepsen worker 46 - jepsen.util 1216 :invoke :read [15 nil] INFO [2019-12-17 12:28:25,521] jepsen worker 62 - jepsen.util 1142 :fail :cas [20 [44 48]] :ValueNotSwapFrom INFO [2019-12-17 12:28:25,521] jepsen worker 62 - jepsen.util 1142 :invoke :cas [20 [12 36]] INFO [2019-12-17 12:28:25,552] jepsen worker 52 - jepsen.util 862 :info :write [17 48] :RequestCanceledException INFO [2019-12-17 12:28:25,552] jepsen worker 80 - jepsen.util 1160 :ok :read [26 22] INFO [2019-12-17 12:28:25,552] jepsen worker 80 - jepsen.util 1160 :invoke :cas [26 [45 29]] INFO [2019-12-17 12:28:25,553] jepsen worker 52 - jepsen.util 952 :invoke :write [17 21] INFO [2019-12-17 12:28:25,553] jepsen worker 11 - jepsen.util 1271 :ok :write [3 47] INFO [2019-12-17 12:28:25,553] jepsen worker 11 - jepsen.util 1271 :invoke :write [3 24] INFO [2019-12-17 12:28:25,569] jepsen worker 27 - jepsen.util 747 :fail :cas [9 [36 38]] :ValueNotSwapFrom INFO [2019-12-17 12:28:25,569] jepsen worker 27 - jepsen.util 747 :invoke :cas [9 [46 47]] INFO [2019-12-17 12:28:25,572] jepsen worker 45 - jepsen.util 1125 :ok :read [15 30] INFO [2019-12-17 12:28:25,572] jepsen worker 45 - jepsen.util 1125 :invoke :read [15 nil] INFO [2019-12-17 12:28:25,586] jepsen worker 54 - jepsen.util 1044 :ok :read [18 21] INFO [2019-12-17 12:28:25,586] jepsen worker 54 - jepsen.util 1044 :invoke :write [18 32] INFO [2019-12-17 12:28:25,607] jepsen worker 32 - jepsen.util 1022 :ok :read [10 2] INFO [2019-12-17 12:28:25,607] jepsen worker 32 - jepsen.util 1022 :invoke :write [10 1] INFO [2019-12-17 12:28:25,633] jepsen worker 65 - jepsen.util 785 :ok :read [21 1] INFO [2019-12-17 12:28:25,633] jepsen worker 65 - jepsen.util 785 :invoke :read [21 nil] INFO [2019-12-17 12:28:25,634] jepsen worker 38 - jepsen.util 848 :ok :read [12 31] INFO [2019-12-17 12:28:25,634] jepsen worker 38 - jepsen.util 848 :invoke :read [12 nil] INFO [2019-12-17 12:28:25,645] jepsen worker 64 - jepsen.util 604 :ok :write [21 36] INFO [2019-12-17 12:28:25,645] jepsen worker 64 - jepsen.util 604 :invoke :cas [21 [41 31]] INFO [2019-12-17 12:28:25,686] jepsen worker 64 - jepsen.util 604 :fail :cas [21 [41 31]] :ValueNotSwapFrom INFO [2019-12-17 12:28:25,686] jepsen worker 64 - jepsen.util 604 :invoke :read [21 nil] INFO [2019-12-17 12:28:25,688] jepsen worker 35 - jepsen.util 845 :info :write [11 22] :RequestCanceledException INFO [2019-12-17 12:28:25,689] jepsen worker 35 - jepsen.util 935 :invoke :read [11 nil] INFO [2019-12-17 12:28:25,699] jepsen worker 37 - jepsen.util 847 :ok :write [12 8] INFO [2019-12-17 12:28:25,699] jepsen worker 37 - jepsen.util 847 :invoke :read [12 nil] INFO [2019-12-17 12:28:25,703] jepsen worker 24 - jepsen.util 1104 :fail :cas [8 [20 42]] :ValueNotSwapFrom INFO [2019-12-17 12:28:25,703] jepsen worker 24 - jepsen.util 1104 :invoke :read [8 nil] INFO [2019-12-17 12:28:25,710] jepsen worker 46 - jepsen.util 1216 :ok :read [15 30] INFO [2019-12-17 12:28:25,710] jepsen worker 46 - jepsen.util 1216 :invoke :write [15 22] INFO [2019-12-17 12:28:25,716] jepsen worker 27 - jepsen.util 747 :fail :cas [9 [46 47]] :ValueNotSwapFrom INFO [2019-12-17 12:28:25,717] jepsen worker 27 - jepsen.util 747 :invoke :cas [9 [36 28]] INFO [2019-12-17 12:28:25,720] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":5362,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:28:25,720] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":5362,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:28:25,753] jepsen worker 78 - jepsen.util 1068 :ok :write [26 39] INFO [2019-12-17 12:28:25,754] jepsen worker 78 - jepsen.util 1068 :invoke :read [26 nil] INFO [2019-12-17 12:28:25,765] jepsen worker 45 - jepsen.util 1125 :ok :read [15 30] INFO [2019-12-17 12:28:25,765] jepsen worker 45 - jepsen.util 1125 :invoke :write [15 18] INFO [2019-12-17 12:28:25,799] jepsen worker 12 - jepsen.util 1002 :ok :read [4 1] INFO [2019-12-17 12:28:25,799] jepsen worker 12 - jepsen.util 1002 :invoke :cas [4 [48 31]] INFO [2019-12-17 12:28:25,808] jepsen worker 64 - jepsen.util 604 :ok :read [21 36] INFO [2019-12-17 12:28:25,808] jepsen worker 64 - jepsen.util 604 :invoke :write [21 26] INFO [2019-12-17 12:28:25,832] jepsen worker 27 - jepsen.util 747 :fail :cas [9 [36 28]] :ValueNotSwapFrom INFO [2019-12-17 12:28:25,832] jepsen worker 27 - jepsen.util 747 :invoke :cas [9 [14 35]] INFO [2019-12-17 12:28:25,833] jepsen worker 65 - jepsen.util 785 :ok :read [21 36] INFO [2019-12-17 12:28:25,833] jepsen worker 65 - jepsen.util 785 :invoke :read [21 nil] INFO [2019-12-17 12:28:25,834] jepsen worker 70 - jepsen.util 1150 :fail :cas [23 [43 16]] :ValueNotSwapFrom INFO [2019-12-17 12:28:25,835] jepsen worker 70 - jepsen.util 1150 :invoke :write [23 29] INFO [2019-12-17 12:28:25,867] jepsen worker 89 - jepsen.util 1079 :ok :write [29 6] INFO [2019-12-17 12:28:25,867] jepsen worker 89 - jepsen.util 1079 :invoke :cas [29 [1 27]] INFO [2019-12-17 12:28:25,873] jepsen worker 52 - jepsen.util 952 :ok :write [17 21] INFO [2019-12-17 12:28:25,874] jepsen worker 52 - jepsen.util 952 :invoke :write [17 35] INFO [2019-12-17 12:28:25,874] jepsen worker 27 - jepsen.util 747 :fail :cas [9 [14 35]] :ValueNotSwapFrom INFO [2019-12-17 12:28:25,874] jepsen worker 27 - jepsen.util 747 :invoke :cas [9 [15 32]] INFO [2019-12-17 12:28:25,897] jepsen worker 53 - jepsen.util 953 :fail :cas [17 [39 31]] :ValueNotSwapFrom INFO [2019-12-17 12:28:25,897] jepsen worker 53 - jepsen.util 953 :invoke :write [17 44] INFO [2019-12-17 12:28:25,917] jepsen worker 27 - jepsen.util 747 :fail :cas [9 [15 32]] :ValueNotSwapFrom INFO [2019-12-17 12:28:25,917] jepsen worker 27 - jepsen.util 747 :invoke :cas [9 [29 17]] INFO [2019-12-17 12:28:25,920] jepsen worker 13 - jepsen.util 1003 :ok :write [4 46] INFO [2019-12-17 12:28:25,920] jepsen worker 13 - jepsen.util 1003 :invoke :read [4 nil] INFO [2019-12-17 12:28:25,922] jepsen worker 19 - jepsen.util 1189 :fail :read [6 nil] :RequestTimeoutException INFO [2019-12-17 12:28:25,922] jepsen worker 19 - jepsen.util 1189 :invoke :read [6 nil] INFO [2019-12-17 12:28:25,941] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":5384,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:28:25,941] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":5384,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:28:25,941] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":5388,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:28:25,941] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":5388,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:28:25,948] jepsen worker 87 - jepsen.util 987 :ok :write [29 17] INFO [2019-12-17 12:28:25,948] jepsen worker 87 - jepsen.util 987 :invoke :read [29 nil] INFO [2019-12-17 12:28:25,965] jepsen worker 27 - jepsen.util 747 :fail :cas [9 [29 17]] :ValueNotSwapFrom INFO [2019-12-17 12:28:25,965] jepsen worker 27 - jepsen.util 747 :invoke :read [9 nil] INFO [2019-12-17 12:28:25,988] jepsen worker 21 - jepsen.util 1101 :info :write [7 33] :RequestCanceledException INFO [2019-12-17 12:28:25,989] jepsen worker 21 - jepsen.util 1191 :invoke :cas [7 [34 14]] INFO [2019-12-17 12:28:25,991] jepsen worker 37 - jepsen.util 847 :ok :read [12 8] INFO [2019-12-17 12:28:25,991] jepsen worker 37 - jepsen.util 847 :invoke :cas [12 [32 6]] INFO [2019-12-17 12:28:25,993] jepsen worker 64 - jepsen.util 604 :ok :write [21 26] INFO [2019-12-17 12:28:25,993] jepsen worker 64 - jepsen.util 604 :invoke :write [21 14] INFO [2019-12-17 12:28:25,999] jepsen worker 27 - jepsen.util 747 :ok :read [9 2] INFO [2019-12-17 12:28:25,999] jepsen worker 27 - jepsen.util 747 :invoke :read [9 nil] INFO [2019-12-17 12:28:26,002] jepsen worker 80 - jepsen.util 1160 :fail :cas [26 [45 29]] :ValueNotSwapFrom INFO [2019-12-17 12:28:26,003] jepsen worker 80 - jepsen.util 1160 :invoke :write [26 8] INFO [2019-12-17 12:28:26,024] jepsen worker 13 - jepsen.util 1003 :ok :read [4 46] INFO [2019-12-17 12:28:26,024] jepsen worker 13 - jepsen.util 1003 :invoke :write [4 6] INFO [2019-12-17 12:28:26,031] jepsen worker 27 - jepsen.util 747 :ok :read [9 2] INFO [2019-12-17 12:28:26,031] jepsen worker 27 - jepsen.util 747 :invoke :read [9 nil] INFO [2019-12-17 12:28:26,051] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":5401,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:28:26,051] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":5401,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:28:26,051] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":5404,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:28:26,051] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":5404,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:28:26,052] jepsen worker 24 - jepsen.util 1104 :ok :read [8 14] INFO [2019-12-17 12:28:26,053] jepsen worker 24 - jepsen.util 1104 :invoke :cas [8 [30 46]] INFO [2019-12-17 12:28:26,072] jepsen worker 5 - jepsen.util 635 :ok :write [1 24] INFO [2019-12-17 12:28:26,072] jepsen worker 5 - jepsen.util 635 :invoke :cas [1 [0 31]] INFO [2019-12-17 12:28:26,086] jepsen worker 37 - jepsen.util 847 :fail :cas [12 [32 6]] :ValueNotSwapFrom INFO [2019-12-17 12:28:26,086] jepsen worker 37 - jepsen.util 847 :invoke :write [12 34] INFO [2019-12-17 12:28:26,089] jepsen worker 27 - jepsen.util 747 :ok :read [9 2] INFO [2019-12-17 12:28:26,089] jepsen worker 27 - jepsen.util 747 :invoke :read [9 nil] INFO [2019-12-17 12:28:26,096] jepsen worker 38 - jepsen.util 848 :ok :read [12 8] INFO [2019-12-17 12:28:26,096] jepsen worker 38 - jepsen.util 848 :invoke :write [12 16] INFO [2019-12-17 12:28:26,098] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:28:26,098] jepsen nemesis - couchbase.util Rebalance check count: 34 INFO [2019-12-17 12:28:26,098] jepsen nemesis - couchbase.util Rebalance stuck count: 15 INFO [2019-12-17 12:28:26,104] jepsen worker 64 - jepsen.util 604 :ok :write [21 14] INFO [2019-12-17 12:28:26,104] jepsen worker 64 - jepsen.util 604 :invoke :write [21 23] INFO [2019-12-17 12:28:26,106] jepsen worker 0 - jepsen.util 1260 :info :write [0 10] :RequestCanceledException INFO [2019-12-17 12:28:26,106] jepsen worker 85 - jepsen.util 985 :info :write [28 27] :RequestCanceledException INFO [2019-12-17 12:28:26,107] jepsen worker 0 - jepsen.util 1350 :invoke :read [0 nil] INFO [2019-12-17 12:28:26,107] jepsen worker 85 - jepsen.util 1075 :invoke :cas [28 [6 0]] INFO [2019-12-17 12:28:26,110] jepsen worker 89 - jepsen.util 1079 :fail :cas [29 [1 27]] :ValueNotSwapFrom INFO [2019-12-17 12:28:26,111] jepsen worker 89 - jepsen.util 1079 :invoke :write [29 42] INFO [2019-12-17 12:28:26,121] jepsen worker 49 - jepsen.util 1039 :info :cas [16 [38 38]] :RequestCanceledException INFO [2019-12-17 12:28:26,121] jepsen worker 82 - jepsen.util 1522 :info :write [27 25] :RequestCanceledException INFO [2019-12-17 12:28:26,121] jepsen worker 50 - jepsen.util 1220 :info :write [16 37] :RequestCanceledException INFO [2019-12-17 12:28:26,121] jepsen worker 66 - jepsen.util 966 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:28:26,122] jepsen worker 66 - jepsen.util 966 :invoke :write [22 42] INFO [2019-12-17 12:28:26,123] jepsen worker 49 - jepsen.util 1129 :invoke :read [16 nil] INFO [2019-12-17 12:28:26,123] jepsen worker 82 - jepsen.util 1612 :invoke :write [27 40] INFO [2019-12-17 12:28:26,123] jepsen worker 50 - jepsen.util 1310 :invoke :read [16 nil] INFO [2019-12-17 12:28:26,124] jepsen worker 27 - jepsen.util 747 :ok :read [9 2] INFO [2019-12-17 12:28:26,124] jepsen worker 27 - jepsen.util 747 :invoke :write [9 9] INFO [2019-12-17 12:28:26,135] jepsen worker 40 - jepsen.util 1570 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:28:26,135] jepsen worker 2 - jepsen.util 902 :info :cas [0 [32 31]] :RequestCanceledException INFO [2019-12-17 12:28:26,135] jepsen worker 68 - jepsen.util 788 :info :write [22 20] :RequestCanceledException INFO [2019-12-17 12:28:26,135] jepsen worker 40 - jepsen.util 1570 :invoke :cas [13 [37 2]] INFO [2019-12-17 12:28:26,137] jepsen worker 2 - jepsen.util 992 :invoke :write [0 23] INFO [2019-12-17 12:28:26,137] jepsen worker 68 - jepsen.util 878 :invoke :write [22 42] INFO [2019-12-17 12:28:26,152] jepsen worker 17 - jepsen.util 1097 :info :cas [5 [6 8]] :RequestCanceledException INFO [2019-12-17 12:28:26,152] jepsen worker 60 - jepsen.util 1230 :info :cas [20 [7 48]] :RequestCanceledException INFO [2019-12-17 12:28:26,152] jepsen worker 48 - jepsen.util 678 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:28:26,153] jepsen worker 48 - jepsen.util 678 :invoke :read [16 nil] INFO [2019-12-17 12:28:26,153] jepsen worker 60 - jepsen.util 1320 :invoke :write [20 16] INFO [2019-12-17 12:28:26,153] jepsen worker 17 - jepsen.util 1187 :invoke :write [5 1] INFO [2019-12-17 12:28:26,158] jepsen worker 64 - jepsen.util 604 :ok :write [21 23] INFO [2019-12-17 12:28:26,158] jepsen worker 64 - jepsen.util 604 :invoke :read [21 nil] INFO [2019-12-17 12:28:26,175] jepsen worker 62 - jepsen.util 1142 :fail :cas [20 [12 36]] :ValueNotSwapFrom INFO [2019-12-17 12:28:26,175] jepsen worker 62 - jepsen.util 1142 :invoke :cas [20 [46 27]] INFO [2019-12-17 12:28:26,182] jepsen worker 64 - jepsen.util 604 :ok :read [21 23] INFO [2019-12-17 12:28:26,182] jepsen worker 64 - jepsen.util 604 :invoke :read [21 nil] INFO [2019-12-17 12:28:26,184] jepsen worker 46 - jepsen.util 1216 :ok :write [15 22] INFO [2019-12-17 12:28:26,184] jepsen worker 46 - jepsen.util 1216 :invoke :cas [15 [18 16]] INFO [2019-12-17 12:28:26,188] jepsen worker 34 - jepsen.util 1204 :info :write [11 19] :RequestCanceledException INFO [2019-12-17 12:28:26,188] jepsen worker 15 - jepsen.util 915 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:28:26,188] jepsen worker 57 - jepsen.util 1137 :info :cas [19 [23 31]] :RequestCanceledException INFO [2019-12-17 12:28:26,188] jepsen worker 84 - jepsen.util 984 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:28:26,188] jepsen worker 39 - jepsen.util 1209 :info :cas [13 [28 47]] :RequestCanceledException INFO [2019-12-17 12:28:26,188] jepsen worker 15 - jepsen.util 915 :invoke :read [5 nil] INFO [2019-12-17 12:28:26,188] jepsen worker 84 - jepsen.util 984 :invoke :cas [28 [2 15]] INFO [2019-12-17 12:28:26,189] jepsen worker 34 - jepsen.util 1294 :invoke :cas [11 [42 8]] INFO [2019-12-17 12:28:26,189] jepsen worker 57 - jepsen.util 1227 :invoke :cas [19 [38 4]] INFO [2019-12-17 12:28:26,189] jepsen worker 39 - jepsen.util 1299 :invoke :write [13 27] INFO [2019-12-17 12:28:26,190] jepsen worker 20 - jepsen.util 1100 :ok :write [6 30] INFO [2019-12-17 12:28:26,190] jepsen worker 20 - jepsen.util 1100 :invoke :read [6 nil] INFO [2019-12-17 12:28:26,206] jepsen worker 59 - jepsen.util 959 :info :cas [19 [24 47]] :RequestCanceledException INFO [2019-12-17 12:28:26,206] jepsen worker 67 - jepsen.util 1417 :info :write [22 24] :RequestCanceledException INFO [2019-12-17 12:28:26,206] jepsen worker 75 - jepsen.util 1155 :info :write [25 36] :RequestCanceledException INFO [2019-12-17 12:28:26,207] jepsen worker 59 - jepsen.util 1049 :invoke :write [19 36] INFO [2019-12-17 12:28:26,207] jepsen worker 67 - jepsen.util 1507 :invoke :cas [22 [1 23]] INFO [2019-12-17 12:28:26,207] jepsen worker 75 - jepsen.util 1245 :invoke :write [25 38] INFO [2019-12-17 12:28:26,220] jepsen worker 41 - jepsen.util 941 :info :cas [13 [1 3]] :RequestCanceledException INFO [2019-12-17 12:28:26,220] jepsen worker 81 - jepsen.util 1251 :info :write [27 26] :RequestCanceledException INFO [2019-12-17 12:28:26,220] jepsen worker 83 - jepsen.util 1343 :info :write [27 10] :RequestCanceledException INFO [2019-12-17 12:28:26,220] jepsen worker 86 - jepsen.util 1076 :info :write [28 37] :RequestCanceledException INFO [2019-12-17 12:28:26,222] jepsen worker 41 - jepsen.util 1031 :invoke :read [13 nil] INFO [2019-12-17 12:28:26,222] jepsen worker 86 - jepsen.util 1166 :invoke :write [28 3] INFO [2019-12-17 12:28:26,222] jepsen worker 81 - jepsen.util 1341 :invoke :write [27 27] INFO [2019-12-17 12:28:26,222] jepsen worker 83 - jepsen.util 1433 :invoke :cas [27 [0 43]] INFO [2019-12-17 12:28:26,222] jepsen worker 64 - jepsen.util 604 :ok :read [21 23] INFO [2019-12-17 12:28:26,222] jepsen worker 64 - jepsen.util 604 :invoke :cas [21 [0 13]] INFO [2019-12-17 12:28:26,230] jepsen worker 19 - jepsen.util 1189 :ok :read [6 30] INFO [2019-12-17 12:28:26,230] jepsen worker 19 - jepsen.util 1189 :invoke :read [6 nil] INFO [2019-12-17 12:28:26,235] jepsen worker 16 - jepsen.util 1006 :info :write [5 33] :RequestCanceledException INFO [2019-12-17 12:28:26,235] jepsen worker 1 - jepsen.util 901 :info :write [0 0] :RequestCanceledException INFO [2019-12-17 12:28:26,236] jepsen worker 16 - jepsen.util 1096 :invoke :write [5 41] INFO [2019-12-17 12:28:26,236] jepsen worker 1 - jepsen.util 991 :invoke :cas [0 [48 28]] INFO [2019-12-17 12:28:26,237] jepsen worker 5 - jepsen.util 635 :fail :cas [1 [0 31]] :ValueNotSwapFrom INFO [2019-12-17 12:28:26,238] jepsen worker 5 - jepsen.util 635 :invoke :cas [1 [47 37]] INFO [2019-12-17 12:28:26,252] jepsen worker 29 - jepsen.util 749 :info :cas [9 [34 11]] :RequestCanceledException INFO [2019-12-17 12:28:26,253] jepsen worker 36 - jepsen.util 1026 :ok :write [12 38] INFO [2019-12-17 12:28:26,253] jepsen worker 36 - jepsen.util 1026 :invoke :cas [12 [14 8]] INFO [2019-12-17 12:28:26,253] jepsen worker 29 - jepsen.util 839 :invoke :read [9 nil] INFO [2019-12-17 12:28:26,257] jepsen worker 20 - jepsen.util 1100 :ok :read [6 30] INFO [2019-12-17 12:28:26,257] jepsen worker 20 - jepsen.util 1100 :invoke :write [6 23] INFO [2019-12-17 12:28:26,259] jepsen worker 13 - jepsen.util 1003 :ok :write [4 6] INFO [2019-12-17 12:28:26,260] jepsen worker 13 - jepsen.util 1003 :invoke :write [4 5] INFO [2019-12-17 12:28:26,267] jepsen worker 64 - jepsen.util 604 :fail :cas [21 [0 13]] :ValueNotSwapFrom INFO [2019-12-17 12:28:26,267] jepsen worker 64 - jepsen.util 604 :invoke :read [21 nil] INFO [2019-12-17 12:28:26,269] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":5437,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:28:26,269] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":5437,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:28:26,269] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":5438,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:28:26,269] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":5438,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:28:26,270] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":5449,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:28:26,270] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":5449,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:28:26,270] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":5453,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:28:26,270] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":5453,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:28:26,272] jepsen worker 19 - jepsen.util 1189 :ok :read [6 30] INFO [2019-12-17 12:28:26,272] jepsen worker 19 - jepsen.util 1189 :invoke :read [6 nil] INFO [2019-12-17 12:28:26,288] jepsen worker 42 - jepsen.util 1302 :info :write [14 30] :RequestCanceledException INFO [2019-12-17 12:28:26,288] jepsen worker 62 - jepsen.util 1142 :fail :cas [20 [46 27]] :ValueNotSwapFrom INFO [2019-12-17 12:28:26,288] jepsen worker 62 - jepsen.util 1142 :invoke :write [20 33] INFO [2019-12-17 12:28:26,289] jepsen worker 42 - jepsen.util 1392 :invoke :cas [14 [33 22]] INFO [2019-12-17 12:28:26,303] jepsen worker 65 - jepsen.util 785 :ok :read [21 23] INFO [2019-12-17 12:28:26,303] jepsen worker 65 - jepsen.util 785 :invoke :read [21 nil] INFO [2019-12-17 12:28:26,305] jepsen worker 27 - jepsen.util 747 :ok :write [9 9] INFO [2019-12-17 12:28:26,305] jepsen worker 27 - jepsen.util 747 :invoke :cas [9 [12 36]] INFO [2019-12-17 12:28:26,305] jepsen worker 32 - jepsen.util 1022 :ok :write [10 1] INFO [2019-12-17 12:28:26,305] jepsen worker 32 - jepsen.util 1022 :invoke :write [10 9] INFO [2019-12-17 12:28:26,306] jepsen worker 79 - jepsen.util 889 :info :write [26 21] :RequestCanceledException INFO [2019-12-17 12:28:26,307] jepsen worker 79 - jepsen.util 979 :invoke :cas [26 [1 5]] INFO [2019-12-17 12:28:26,321] jepsen worker 77 - jepsen.util 1157 :info :write [25 11] :RequestCanceledException INFO [2019-12-17 12:28:26,321] jepsen worker 63 - jepsen.util 873 :info :write [21 38] :RequestCanceledException INFO [2019-12-17 12:28:26,323] jepsen worker 63 - jepsen.util 963 :invoke :write [21 3] INFO [2019-12-17 12:28:26,323] jepsen worker 77 - jepsen.util 1247 :invoke :read [25 nil] INFO [2019-12-17 12:28:26,328] jepsen worker 64 - jepsen.util 604 :ok :read [21 23] INFO [2019-12-17 12:28:26,328] jepsen worker 64 - jepsen.util 604 :invoke :read [21 nil] INFO [2019-12-17 12:28:26,335] jepsen worker 26 - jepsen.util 1196 :info :write [8 27] :RequestCanceledException INFO [2019-12-17 12:28:26,337] jepsen worker 26 - jepsen.util 1286 :invoke :cas [8 [1 44]] INFO [2019-12-17 12:28:26,362] jepsen worker 34 - jepsen.util 1294 :fail :cas [11 [42 8]] :ValueNotSwapFrom INFO [2019-12-17 12:28:26,362] jepsen worker 34 - jepsen.util 1294 :invoke :cas [11 [11 29]] INFO [2019-12-17 12:28:26,385] jepsen worker 46 - jepsen.util 1216 :fail :cas [15 [18 16]] :ValueNotSwapFrom INFO [2019-12-17 12:28:26,385] jepsen worker 46 - jepsen.util 1216 :invoke :cas [15 [31 43]] INFO [2019-12-17 12:28:26,396] jepsen worker 87 - jepsen.util 987 :ok :read [29 17] INFO [2019-12-17 12:28:26,396] jepsen worker 87 - jepsen.util 987 :invoke :cas [29 [49 12]] INFO [2019-12-17 12:28:26,401] jepsen worker 29 - jepsen.util 839 :ok :read [9 9] INFO [2019-12-17 12:28:26,401] jepsen worker 29 - jepsen.util 839 :invoke :write [9 36] INFO [2019-12-17 12:28:26,409] jepsen worker 79 - jepsen.util 979 :fail :cas [26 [1 5]] :ValueNotSwapFrom INFO [2019-12-17 12:28:26,409] jepsen worker 79 - jepsen.util 979 :invoke :read [26 nil] INFO [2019-12-17 12:28:26,421] jepsen worker 58 - jepsen.util 1138 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:28:26,421] jepsen worker 58 - jepsen.util 1138 :invoke :write [19 16] INFO [2019-12-17 12:28:26,425] jepsen worker 65 - jepsen.util 785 :ok :read [21 23] INFO [2019-12-17 12:28:26,425] jepsen worker 65 - jepsen.util 785 :invoke :read [21 nil] INFO [2019-12-17 12:28:26,432] jepsen worker 27 - jepsen.util 747 :fail :cas [9 [12 36]] :ValueNotSwapFrom INFO [2019-12-17 12:28:26,432] jepsen worker 27 - jepsen.util 747 :invoke :cas [9 [40 46]] INFO [2019-12-17 12:28:26,435] jepsen worker 51 - jepsen.util 861 :info :write [17 16] :RequestCanceledException INFO [2019-12-17 12:28:26,436] jepsen worker 51 - jepsen.util 951 :invoke :read [17 nil] INFO [2019-12-17 12:28:26,460] jepsen worker 46 - jepsen.util 1216 :fail :cas [15 [31 43]] :ValueNotSwapFrom INFO [2019-12-17 12:28:26,460] jepsen worker 46 - jepsen.util 1216 :invoke :cas [15 [11 10]] INFO [2019-12-17 12:28:26,462] jepsen worker 5 - jepsen.util 635 :fail :cas [1 [47 37]] :ValueNotSwapFrom INFO [2019-12-17 12:28:26,462] jepsen worker 5 - jepsen.util 635 :invoke :cas [1 [13 11]] INFO [2019-12-17 12:28:26,463] jepsen worker 56 - jepsen.util 1046 :ok :write [18 36] INFO [2019-12-17 12:28:26,463] jepsen worker 56 - jepsen.util 1046 :invoke :write [18 17] INFO [2019-12-17 12:28:26,479] jepsen worker 13 - jepsen.util 1003 :ok :write [4 5] INFO [2019-12-17 12:28:26,479] jepsen worker 13 - jepsen.util 1003 :invoke :read [4 nil] INFO [2019-12-17 12:28:26,489] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":5487,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:28:26,489] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":5487,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:28:26,489] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":5490,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:28:26,489] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":5490,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:28:26,489] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":5491,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:28:26,489] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":5491,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:28:26,496] jepsen worker 71 - jepsen.util 791 :ok :write [23 44] INFO [2019-12-17 12:28:26,496] jepsen worker 71 - jepsen.util 791 :invoke :read [23 nil] INFO [2019-12-17 12:28:26,499] jepsen worker 65 - jepsen.util 785 :ok :read [21 23] INFO [2019-12-17 12:28:26,499] jepsen worker 65 - jepsen.util 785 :invoke :read [21 nil] INFO [2019-12-17 12:28:26,515] jepsen worker 46 - jepsen.util 1216 :fail :cas [15 [11 10]] :ValueNotSwapFrom INFO [2019-12-17 12:28:26,516] jepsen worker 46 - jepsen.util 1216 :invoke :cas [15 [46 48]] INFO [2019-12-17 12:28:26,517] jepsen worker 78 - jepsen.util 1068 :ok :read [26 39] INFO [2019-12-17 12:28:26,517] jepsen worker 78 - jepsen.util 1068 :invoke :write [26 32] INFO [2019-12-17 12:28:26,538] jepsen worker 65 - jepsen.util 785 :ok :read [21 23] INFO [2019-12-17 12:28:26,538] jepsen worker 65 - jepsen.util 785 :invoke :read [21 nil] INFO [2019-12-17 12:28:26,545] jepsen worker 12 - jepsen.util 1002 :fail :cas [4 [48 31]] :ValueNotSwapFrom INFO [2019-12-17 12:28:26,546] jepsen worker 12 - jepsen.util 1002 :invoke :write [4 35] INFO [2019-12-17 12:28:26,570] jepsen worker 46 - jepsen.util 1216 :fail :cas [15 [46 48]] :ValueNotSwapFrom INFO [2019-12-17 12:28:26,570] jepsen worker 46 - jepsen.util 1216 :invoke :write [15 32] INFO [2019-12-17 12:28:26,571] jepsen worker 26 - jepsen.util 1286 :fail :cas [8 [1 44]] :ValueNotSwapFrom INFO [2019-12-17 12:28:26,571] jepsen worker 26 - jepsen.util 1286 :invoke :cas [8 [46 9]] INFO [2019-12-17 12:28:26,583] jepsen worker 21 - jepsen.util 1191 :fail :cas [7 [34 14]] :ValueNotSwapFrom INFO [2019-12-17 12:28:26,583] jepsen worker 21 - jepsen.util 1191 :invoke :cas [7 [24 34]] INFO [2019-12-17 12:28:26,588] jepsen worker 24 - jepsen.util 1104 :fail :cas [8 [30 46]] :ValueNotSwapFrom INFO [2019-12-17 12:28:26,588] jepsen worker 24 - jepsen.util 1104 :invoke :cas [8 [43 20]] INFO [2019-12-17 12:28:26,597] jepsen worker 63 - jepsen.util 963 :ok :write [21 3] INFO [2019-12-17 12:28:26,597] jepsen worker 63 - jepsen.util 963 :invoke :write [21 6] INFO [2019-12-17 12:28:26,607] jepsen worker 88 - jepsen.util 898 :fail :cas [29 [40 14]] :ValueNotSwapFrom INFO [2019-12-17 12:28:26,607] jepsen worker 88 - jepsen.util 898 :invoke :read [29 nil] INFO [2019-12-17 12:28:26,622] jepsen worker 65 - jepsen.util 785 :ok :read [21 23] INFO [2019-12-17 12:28:26,622] jepsen worker 65 - jepsen.util 785 :invoke :cas [21 [48 49]] INFO [2019-12-17 12:28:26,642] jepsen worker 64 - jepsen.util 604 :ok :read [21 3] INFO [2019-12-17 12:28:26,642] jepsen worker 64 - jepsen.util 604 :invoke :write [21 44] INFO [2019-12-17 12:28:26,645] jepsen worker 29 - jepsen.util 839 :ok :write [9 36] INFO [2019-12-17 12:28:26,645] jepsen worker 29 - jepsen.util 839 :invoke :read [9 nil] INFO [2019-12-17 12:28:26,678] jepsen worker 36 - jepsen.util 1026 :fail :cas [12 [14 8]] :ValueNotSwapFrom INFO [2019-12-17 12:28:26,678] jepsen worker 36 - jepsen.util 1026 :invoke :read [12 nil] INFO [2019-12-17 12:28:26,688] jepsen worker 5 - jepsen.util 635 :fail :cas [1 [13 11]] :ValueNotSwapFrom INFO [2019-12-17 12:28:26,688] jepsen worker 5 - jepsen.util 635 :invoke :write [1 18] INFO [2019-12-17 12:28:26,699] jepsen worker 63 - jepsen.util 963 :ok :write [21 6] INFO [2019-12-17 12:28:26,699] jepsen worker 63 - jepsen.util 963 :invoke :cas [21 [41 26]] INFO [2019-12-17 12:28:26,700] jepsen worker 29 - jepsen.util 839 :ok :read [9 36] INFO [2019-12-17 12:28:26,701] jepsen worker 29 - jepsen.util 839 :invoke :read [9 nil] INFO [2019-12-17 12:28:26,705] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":5515,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:28:26,705] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":5515,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:28:26,720] jepsen worker 47 - jepsen.util 1307 :info :write [15 12] :RequestCanceledException INFO [2019-12-17 12:28:26,721] jepsen worker 44 - jepsen.util 1034 :info :write [14 25] :RequestCanceledException INFO [2019-12-17 12:28:26,723] jepsen worker 47 - jepsen.util 1397 :invoke :read [15 nil] INFO [2019-12-17 12:28:26,724] jepsen worker 44 - jepsen.util 1124 :invoke :cas [14 [6 46]] INFO [2019-12-17 12:28:26,730] jepsen worker 29 - jepsen.util 839 :ok :read [9 36] INFO [2019-12-17 12:28:26,731] jepsen worker 29 - jepsen.util 839 :invoke :write [9 7] INFO [2019-12-17 12:28:26,734] jepsen worker 55 - jepsen.util 1225 :info :write [18 40] :RequestCanceledException INFO [2019-12-17 12:28:26,736] jepsen worker 55 - jepsen.util 1315 :invoke :write [18 43] INFO [2019-12-17 12:28:26,741] jepsen worker 20 - jepsen.util 1100 :ok :write [6 23] INFO [2019-12-17 12:28:26,742] jepsen worker 20 - jepsen.util 1100 :invoke :cas [6 [41 39]] INFO [2019-12-17 12:28:26,744] jepsen worker 56 - jepsen.util 1046 :ok :write [18 17] INFO [2019-12-17 12:28:26,745] jepsen worker 56 - jepsen.util 1046 :invoke :cas [18 [23 46]] INFO [2019-12-17 12:28:26,776] jepsen worker 79 - jepsen.util 979 :ok :read [26 39] INFO [2019-12-17 12:28:26,776] jepsen worker 79 - jepsen.util 979 :invoke :read [26 nil] INFO [2019-12-17 12:28:26,786] jepsen worker 72 - jepsen.util 882 :info :write [24 14] :RequestCanceledException INFO [2019-12-17 12:28:26,786] jepsen worker 28 - jepsen.util 748 :fail :read [9 nil] :RequestTimeoutException INFO [2019-12-17 12:28:26,786] jepsen worker 28 - jepsen.util 748 :invoke :cas [9 [38 18]] INFO [2019-12-17 12:28:26,788] jepsen worker 72 - jepsen.util 972 :invoke :cas [24 [46 22]] INFO [2019-12-17 12:28:26,789] jepsen worker 64 - jepsen.util 604 :ok :write [21 44] INFO [2019-12-17 12:28:26,789] jepsen worker 64 - jepsen.util 604 :invoke :cas [21 [22 5]] INFO [2019-12-17 12:28:26,801] jepsen worker 13 - jepsen.util 1003 :ok :read [4 5] INFO [2019-12-17 12:28:26,801] jepsen worker 13 - jepsen.util 1003 :invoke :cas [4 [35 39]] INFO [2019-12-17 12:28:26,821] jepsen worker 9 - jepsen.util 909 :info :write [3 37] :RequestCanceledException INFO [2019-12-17 12:28:26,821] jepsen worker 23 - jepsen.util 1013 :info :write [7 10] :RequestCanceledException INFO [2019-12-17 12:28:26,828] jepsen worker 23 - jepsen.util 1103 :invoke :read [7 nil] INFO [2019-12-17 12:28:26,828] jepsen worker 9 - jepsen.util 999 :invoke :cas [3 [39 17]] INFO [2019-12-17 12:28:26,850] jepsen worker 8 - jepsen.util 1268 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:28:26,850] jepsen worker 8 - jepsen.util 1268 :invoke :read [2 nil] INFO [2019-12-17 12:28:26,851] jepsen worker 3 - jepsen.util 723 :info :write [1 4] :RequestCanceledException INFO [2019-12-17 12:28:26,852] jepsen worker 47 - jepsen.util 1397 :ok :read [15 22] INFO [2019-12-17 12:28:26,853] jepsen worker 47 - jepsen.util 1397 :invoke :write [15 12] INFO [2019-12-17 12:28:26,858] jepsen worker 3 - jepsen.util 813 :invoke :read [1 nil] INFO [2019-12-17 12:28:26,864] jepsen worker 26 - jepsen.util 1286 :fail :cas [8 [46 9]] :ValueNotSwapFrom INFO [2019-12-17 12:28:26,864] jepsen worker 26 - jepsen.util 1286 :invoke :write [8 27] INFO [2019-12-17 12:28:26,868] jepsen worker 64 - jepsen.util 604 :fail :cas [21 [22 5]] :ValueNotSwapFrom INFO [2019-12-17 12:28:26,869] jepsen worker 64 - jepsen.util 604 :invoke :read [21 nil] INFO [2019-12-17 12:28:26,871] jepsen worker 20 - jepsen.util 1100 :fail :cas [6 [41 39]] :ValueNotSwapFrom INFO [2019-12-17 12:28:26,871] jepsen worker 20 - jepsen.util 1100 :invoke :write [6 8] INFO [2019-12-17 12:28:26,887] jepsen worker 25 - jepsen.util 1285 :info :cas [8 [14 42]] :RequestCanceledException INFO [2019-12-17 12:28:26,894] jepsen worker 25 - jepsen.util 1375 :invoke :cas [8 [16 12]] INFO [2019-12-17 12:28:26,900] jepsen worker 88 - jepsen.util 898 :ok :read [29 17] INFO [2019-12-17 12:28:26,900] jepsen worker 88 - jepsen.util 898 :invoke :write [29 13] INFO [2019-12-17 12:28:26,913] jepsen worker 34 - jepsen.util 1294 :fail :cas [11 [11 29]] :ValueNotSwapFrom INFO [2019-12-17 12:28:26,913] jepsen worker 34 - jepsen.util 1294 :invoke :write [11 13] INFO [2019-12-17 12:28:26,921] jepsen worker 43 - jepsen.util 1213 :info :write [14 46] :RequestCanceledException INFO [2019-12-17 12:28:26,925] jepsen worker 43 - jepsen.util 1303 :invoke :write [14 14] INFO [2019-12-17 12:28:26,928] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":5541,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:28:26,928] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":5541,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:28:26,928] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":5544,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:28:26,928] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":5544,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:28:26,935] jepsen worker 22 - jepsen.util 1102 :info :write [7 23] :RequestCanceledException INFO [2019-12-17 12:28:26,940] jepsen worker 22 - jepsen.util 1192 :invoke :cas [7 [36 41]] INFO [2019-12-17 12:28:26,944] jepsen worker 64 - jepsen.util 604 :ok :read [21 44] INFO [2019-12-17 12:28:26,945] jepsen worker 64 - jepsen.util 604 :invoke :cas [21 [31 16]] INFO [2019-12-17 12:28:26,968] jepsen worker 71 - jepsen.util 791 :ok :read [23 44] INFO [2019-12-17 12:28:26,969] jepsen worker 71 - jepsen.util 791 :invoke :read [23 nil] INFO [2019-12-17 12:28:26,969] jepsen worker 51 - jepsen.util 951 :ok :read [17 21] INFO [2019-12-17 12:28:26,969] jepsen worker 51 - jepsen.util 951 :invoke :write [17 30] INFO [2019-12-17 12:28:26,981] jepsen worker 29 - jepsen.util 839 :ok :write [9 7] INFO [2019-12-17 12:28:26,981] jepsen worker 29 - jepsen.util 839 :invoke :read [9 nil] INFO [2019-12-17 12:28:26,983] jepsen worker 69 - jepsen.util 879 :info :write [23 11] :RequestCanceledException INFO [2019-12-17 12:28:26,983] jepsen worker 31 - jepsen.util 1111 :fail :read [10 nil] :RequestTimeoutException INFO [2019-12-17 12:28:26,984] jepsen worker 31 - jepsen.util 1111 :invoke :write [10 36] INFO [2019-12-17 12:28:26,987] jepsen worker 69 - jepsen.util 969 :invoke :cas [23 [4 17]] INFO [2019-12-17 12:28:27,007] jepsen worker 21 - jepsen.util 1191 :fail :cas [7 [24 34]] :ValueNotSwapFrom INFO [2019-12-17 12:28:27,007] jepsen worker 21 - jepsen.util 1191 :invoke :read [7 nil] INFO [2019-12-17 12:28:27,016] jepsen worker 24 - jepsen.util 1104 :fail :cas [8 [43 20]] :ValueNotSwapFrom INFO [2019-12-17 12:28:27,016] jepsen worker 24 - jepsen.util 1104 :invoke :write [8 47] INFO [2019-12-17 12:28:27,021] jepsen worker 6 - jepsen.util 1176 :info :write [2 17] :RequestCanceledException INFO [2019-12-17 12:28:27,021] jepsen worker 76 - jepsen.util 1156 :info :write [25 15] :RequestCanceledException INFO [2019-12-17 12:28:27,024] jepsen worker 6 - jepsen.util 1266 :invoke :write [2 29] INFO [2019-12-17 12:28:27,024] jepsen worker 76 - jepsen.util 1246 :invoke :cas [25 [25 9]] INFO [2019-12-17 12:28:27,033] jepsen worker 46 - jepsen.util 1216 :ok :write [15 32] INFO [2019-12-17 12:28:27,033] jepsen worker 32 - jepsen.util 1022 :ok :write [10 9] INFO [2019-12-17 12:28:27,033] jepsen worker 32 - jepsen.util 1022 :invoke :read [10 nil] INFO [2019-12-17 12:28:27,033] jepsen worker 46 - jepsen.util 1216 :invoke :write [15 0] INFO [2019-12-17 12:28:27,034] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":5560,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:28:27,034] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":5560,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:28:27,035] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":5563,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:28:27,035] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":5563,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:28:27,035] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":5564,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:28:27,036] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":5564,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:28:27,050] jepsen worker 28 - jepsen.util 748 :fail :cas [9 [38 18]] :ValueNotSwapFrom INFO [2019-12-17 12:28:27,050] jepsen worker 28 - jepsen.util 748 :invoke :write [9 42] INFO [2019-12-17 12:28:27,075] jepsen worker 64 - jepsen.util 604 :fail :cas [21 [31 16]] :ValueNotSwapFrom INFO [2019-12-17 12:28:27,075] jepsen worker 64 - jepsen.util 604 :invoke :cas [21 [23 27]] INFO [2019-12-17 12:28:27,094] jepsen worker 9 - jepsen.util 999 :fail :cas [3 [39 17]] :ValueNotSwapFrom INFO [2019-12-17 12:28:27,094] jepsen worker 23 - jepsen.util 1103 :ok :read [7 20] INFO [2019-12-17 12:28:27,094] jepsen worker 9 - jepsen.util 999 :invoke :cas [3 [44 19]] INFO [2019-12-17 12:28:27,095] jepsen worker 23 - jepsen.util 1103 :invoke :cas [7 [47 20]] INFO [2019-12-17 12:28:27,105] jepsen worker 61 - jepsen.util 1231 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:28:27,105] jepsen worker 61 - jepsen.util 1231 :invoke :cas [20 [14 41]] INFO [2019-12-17 12:28:27,118] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:28:27,118] jepsen nemesis - couchbase.util Rebalance check count: 35 INFO [2019-12-17 12:28:27,118] jepsen nemesis - couchbase.util Rebalance stuck count: 16 INFO [2019-12-17 12:28:27,121] jepsen worker 18 - jepsen.util 1008 :info :write [6 36] :RequestCanceledException INFO [2019-12-17 12:28:27,121] jepsen worker 74 - jepsen.util 794 :info :write [24 27] :RequestCanceledException INFO [2019-12-17 12:28:27,125] jepsen worker 18 - jepsen.util 1098 :invoke :read [6 nil] INFO [2019-12-17 12:28:27,126] jepsen worker 74 - jepsen.util 884 :invoke :read [24 nil] INFO [2019-12-17 12:28:27,128] jepsen worker 44 - jepsen.util 1124 :fail :cas [14 [6 46]] :ValueNotSwapFrom INFO [2019-12-17 12:28:27,128] jepsen worker 44 - jepsen.util 1124 :invoke :write [14 16] INFO [2019-12-17 12:28:27,150] jepsen worker 65 - jepsen.util 785 :fail :cas [21 [48 49]] :ValueNotSwapFrom INFO [2019-12-17 12:28:27,151] jepsen worker 65 - jepsen.util 785 :invoke :write [21 39] INFO [2019-12-17 12:28:27,153] jepsen worker 32 - jepsen.util 1022 :ok :read [10 9] INFO [2019-12-17 12:28:27,154] jepsen worker 32 - jepsen.util 1022 :invoke :read [10 nil] INFO [2019-12-17 12:28:27,179] jepsen worker 64 - jepsen.util 604 :fail :cas [21 [23 27]] :ValueNotSwapFrom INFO [2019-12-17 12:28:27,179] jepsen worker 29 - jepsen.util 839 :ok :read [9 7] INFO [2019-12-17 12:28:27,180] jepsen worker 29 - jepsen.util 839 :invoke :read [9 nil] INFO [2019-12-17 12:28:27,180] jepsen worker 64 - jepsen.util 604 :invoke :write [21 44] INFO [2019-12-17 12:28:27,204] jepsen worker 74 - jepsen.util 884 :ok :read [24 22] INFO [2019-12-17 12:28:27,211] jepsen worker 74 - jepsen.util 884 :invoke :read [24 nil] INFO [2019-12-17 12:28:27,217] jepsen worker 21 - jepsen.util 1191 :ok :read [7 20] INFO [2019-12-17 12:28:27,218] jepsen worker 21 - jepsen.util 1191 :invoke :read [7 nil] INFO [2019-12-17 12:28:27,227] jepsen worker 72 - jepsen.util 972 :fail :cas [24 [46 22]] :ValueNotSwapFrom INFO [2019-12-17 12:28:27,229] jepsen worker 72 - jepsen.util 972 :invoke :cas [24 [48 19]] INFO [2019-12-17 12:28:27,240] jepsen worker 18 - jepsen.util 1098 :ok :read [6 23] INFO [2019-12-17 12:28:27,240] jepsen worker 18 - jepsen.util 1098 :invoke :cas [6 [39 19]] INFO [2019-12-17 12:28:27,251] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":5585,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:28:27,252] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":5585,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:28:27,252] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":5586,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:28:27,252] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":5595,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:28:27,252] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":5586,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:28:27,252] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":5595,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:28:27,252] jepsen worker 33 - jepsen.util 1383 :info :write [11 35] :RequestCanceledException INFO [2019-12-17 12:28:27,252] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":5596,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:28:27,253] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":5596,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:28:27,255] jepsen worker 33 - jepsen.util 1473 :invoke :write [11 15] INFO [2019-12-17 12:28:27,261] jepsen worker 32 - jepsen.util 1022 :ok :read [10 9] INFO [2019-12-17 12:28:27,261] jepsen worker 32 - jepsen.util 1022 :invoke :cas [10 [44 26]] INFO [2019-12-17 12:28:27,271] jepsen worker 23 - jepsen.util 1103 :fail :cas [7 [47 20]] :ValueNotSwapFrom INFO [2019-12-17 12:28:27,271] jepsen worker 23 - jepsen.util 1103 :invoke :cas [7 [18 18]] INFO [2019-12-17 12:28:27,280] jepsen worker 18 - jepsen.util 1098 :fail :cas [6 [39 19]] :ValueNotSwapFrom INFO [2019-12-17 12:28:27,280] jepsen worker 18 - jepsen.util 1098 :invoke :cas [6 [29 33]] INFO [2019-12-17 12:28:27,302] jepsen worker 64 - jepsen.util 604 :ok :write [21 44] INFO [2019-12-17 12:28:27,302] jepsen worker 64 - jepsen.util 604 :invoke :cas [21 [7 10]] INFO [2019-12-17 12:28:27,303] jepsen worker 30 - jepsen.util 1110 :info :cas [10 [20 9]] :RequestCanceledException INFO [2019-12-17 12:28:27,303] jepsen worker 10 - jepsen.util 1270 :fail :read [3 nil] :RequestTimeoutException INFO [2019-12-17 12:28:27,304] jepsen worker 10 - jepsen.util 1270 :invoke :write [3 14] INFO [2019-12-17 12:28:27,307] jepsen worker 30 - jepsen.util 1200 :invoke :write [10 20] INFO [2019-12-17 12:28:27,318] jepsen worker 79 - jepsen.util 979 :ok :read [26 39] INFO [2019-12-17 12:28:27,318] jepsen worker 79 - jepsen.util 979 :invoke :read [26 nil] INFO [2019-12-17 12:28:27,349] jepsen worker 25 - jepsen.util 1375 :fail :cas [8 [16 12]] :ValueNotSwapFrom INFO [2019-12-17 12:28:27,349] jepsen worker 25 - jepsen.util 1375 :invoke :read [8 nil] INFO [2019-12-17 12:28:27,365] jepsen worker 13 - jepsen.util 1003 :fail :cas [4 [35 39]] :ValueNotSwapFrom INFO [2019-12-17 12:28:27,365] jepsen worker 13 - jepsen.util 1003 :invoke :write [4 37] INFO [2019-12-17 12:28:27,376] jepsen worker 32 - jepsen.util 1022 :fail :cas [10 [44 26]] :ValueNotSwapFrom INFO [2019-12-17 12:28:27,376] jepsen worker 32 - jepsen.util 1022 :invoke :write [10 33] INFO [2019-12-17 12:28:27,386] jepsen worker 64 - jepsen.util 604 :fail :cas [21 [7 10]] :ValueNotSwapFrom INFO [2019-12-17 12:28:27,386] jepsen worker 64 - jepsen.util 604 :invoke :cas [21 [21 27]] INFO [2019-12-17 12:28:27,405] jepsen worker 73 - jepsen.util 973 :info :cas [24 [3 23]] :RequestCanceledException INFO [2019-12-17 12:28:27,406] jepsen worker 73 - jepsen.util 1063 :invoke :write [24 31] INFO [2019-12-17 12:28:27,420] jepsen worker 18 - jepsen.util 1098 :fail :cas [6 [29 33]] :ValueNotSwapFrom INFO [2019-12-17 12:28:27,420] jepsen worker 18 - jepsen.util 1098 :invoke :write [6 17] INFO [2019-12-17 12:28:27,421] jepsen worker 14 - jepsen.util 824 :info :write [4 5] :RequestCanceledException INFO [2019-12-17 12:28:27,425] jepsen worker 14 - jepsen.util 914 :invoke :cas [4 [0 45]] INFO [2019-12-17 12:28:27,435] jepsen worker 7 - jepsen.util 1357 :info :write [2 9] :RequestCanceledException INFO [2019-12-17 12:28:27,435] jepsen worker 4 - jepsen.util 724 :info :write [1 48] :RequestCanceledException INFO [2019-12-17 12:28:27,438] jepsen worker 87 - jepsen.util 987 :fail :cas [29 [49 12]] :ValueNotSwapFrom INFO [2019-12-17 12:28:27,438] jepsen worker 87 - jepsen.util 987 :invoke :read [29 nil] INFO [2019-12-17 12:28:27,440] jepsen worker 4 - jepsen.util 814 :invoke :cas [1 [17 34]] INFO [2019-12-17 12:28:27,441] jepsen worker 7 - jepsen.util 1447 :invoke :read [2 nil] INFO [2019-12-17 12:28:27,460] jepsen worker 64 - jepsen.util 604 :fail :cas [21 [21 27]] :ValueNotSwapFrom INFO [2019-12-17 12:28:27,460] jepsen worker 64 - jepsen.util 604 :invoke :cas [21 [5 12]] INFO [2019-12-17 12:28:27,464] jepsen worker 79 - jepsen.util 979 :ok :read [26 39] INFO [2019-12-17 12:28:27,464] jepsen worker 79 - jepsen.util 979 :invoke :read [26 nil] INFO [2019-12-17 12:28:27,469] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":5617,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:28:27,470] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":5617,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:28:27,470] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":5619,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:28:27,470] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":5619,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:28:27,487] jepsen worker 64 - jepsen.util 604 :fail :cas [21 [5 12]] :ValueNotSwapFrom INFO [2019-12-17 12:28:27,487] jepsen worker 64 - jepsen.util 604 :invoke :write [21 6] INFO [2019-12-17 12:28:27,543] jepsen worker 7 - jepsen.util 1447 :ok :read [2 15] INFO [2019-12-17 12:28:27,544] jepsen worker 7 - jepsen.util 1447 :invoke :write [2 38] INFO [2019-12-17 12:28:27,552] jepsen worker 64 - jepsen.util 604 :ok :write [21 6] INFO [2019-12-17 12:28:27,552] jepsen worker 64 - jepsen.util 604 :invoke :read [21 nil] INFO [2019-12-17 12:28:27,580] jepsen worker 3 - jepsen.util 813 :ok :read [1 24] INFO [2019-12-17 12:28:27,580] jepsen worker 3 - jepsen.util 813 :invoke :cas [1 [43 23]] INFO [2019-12-17 12:28:27,595] jepsen worker 23 - jepsen.util 1103 :fail :cas [7 [18 18]] :ValueNotSwapFrom INFO [2019-12-17 12:28:27,595] jepsen worker 23 - jepsen.util 1103 :invoke :cas [7 [10 16]] INFO [2019-12-17 12:28:27,604] jepsen worker 64 - jepsen.util 604 :ok :read [21 6] INFO [2019-12-17 12:28:27,604] jepsen worker 64 - jepsen.util 604 :invoke :cas [21 [42 8]] INFO [2019-12-17 12:28:27,619] jepsen worker 36 - jepsen.util 1026 :ok :read [12 38] INFO [2019-12-17 12:28:27,619] jepsen worker 36 - jepsen.util 1026 :invoke :cas [12 [49 44]] INFO [2019-12-17 12:28:27,622] jepsen worker 11 - jepsen.util 1271 :info :write [3 24] :RequestCanceledException INFO [2019-12-17 12:28:27,625] jepsen worker 11 - jepsen.util 1361 :invoke :write [3 28] INFO [2019-12-17 12:28:27,649] jepsen worker 64 - jepsen.util 604 :fail :cas [21 [42 8]] :ValueNotSwapFrom INFO [2019-12-17 12:28:27,649] jepsen worker 64 - jepsen.util 604 :invoke :write [21 9] INFO [2019-12-17 12:28:27,663] jepsen worker 21 - jepsen.util 1191 :ok :read [7 20] INFO [2019-12-17 12:28:27,663] jepsen worker 21 - jepsen.util 1191 :invoke :cas [7 [37 41]] INFO [2019-12-17 12:28:27,683] jepsen worker 54 - jepsen.util 1044 :info :write [18 32] :RequestCanceledException INFO [2019-12-17 12:28:27,685] jepsen worker 54 - jepsen.util 1134 :invoke :cas [18 [41 14]] INFO [2019-12-17 12:28:27,686] jepsen worker 69 - jepsen.util 969 :fail :cas [23 [4 17]] :ValueNotSwapFrom INFO [2019-12-17 12:28:27,687] jepsen worker 69 - jepsen.util 969 :invoke :read [23 nil] INFO [2019-12-17 12:28:27,689] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":5637,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:28:27,689] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":5637,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:28:27,689] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":5640,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:28:27,689] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":5640,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:28:27,698] jepsen worker 35 - jepsen.util 935 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:28:27,699] jepsen worker 35 - jepsen.util 935 :invoke :cas [11 [29 26]] INFO [2019-12-17 12:28:27,710] jepsen worker 25 - jepsen.util 1375 :ok :read [8 14] INFO [2019-12-17 12:28:27,710] jepsen worker 25 - jepsen.util 1375 :invoke :write [8 11] INFO [2019-12-17 12:28:27,717] jepsen worker 87 - jepsen.util 987 :ok :read [29 17] INFO [2019-12-17 12:28:27,717] jepsen worker 23 - jepsen.util 1103 :fail :cas [7 [10 16]] :ValueNotSwapFrom INFO [2019-12-17 12:28:27,717] jepsen worker 87 - jepsen.util 987 :invoke :read [29 nil] INFO [2019-12-17 12:28:27,717] jepsen worker 23 - jepsen.util 1103 :invoke :read [7 nil] INFO [2019-12-17 12:28:27,750] jepsen worker 36 - jepsen.util 1026 :fail :cas [12 [49 44]] :ValueNotSwapFrom INFO [2019-12-17 12:28:27,750] jepsen worker 36 - jepsen.util 1026 :invoke :read [12 nil] INFO [2019-12-17 12:28:27,778] jepsen worker 64 - jepsen.util 604 :ok :write [21 9] INFO [2019-12-17 12:28:27,778] jepsen worker 64 - jepsen.util 604 :invoke :write [21 8] INFO [2019-12-17 12:28:27,850] jepsen worker 45 - jepsen.util 1125 :info :write [15 18] :RequestCanceledException INFO [2019-12-17 12:28:27,850] jepsen worker 70 - jepsen.util 1150 :info :write [23 29] :RequestCanceledException INFO [2019-12-17 12:28:27,852] jepsen worker 45 - jepsen.util 1215 :invoke :read [15 nil] INFO [2019-12-17 12:28:27,852] jepsen worker 70 - jepsen.util 1240 :invoke :read [23 nil] INFO [2019-12-17 12:28:27,862] jepsen worker 23 - jepsen.util 1103 :ok :read [7 20] INFO [2019-12-17 12:28:27,862] jepsen worker 23 - jepsen.util 1103 :invoke :read [7 nil] INFO [2019-12-17 12:28:27,876] jepsen worker 54 - jepsen.util 1134 :fail :cas [18 [41 14]] :ValueNotSwapFrom INFO [2019-12-17 12:28:27,876] jepsen worker 54 - jepsen.util 1134 :invoke :read [18 nil] INFO [2019-12-17 12:28:27,878] jepsen worker 64 - jepsen.util 604 :ok :write [21 8] INFO [2019-12-17 12:28:27,879] jepsen worker 64 - jepsen.util 604 :invoke :cas [21 [26 12]] INFO [2019-12-17 12:28:27,883] jepsen worker 52 - jepsen.util 952 :info :write [17 35] :RequestCanceledException INFO [2019-12-17 12:28:27,885] jepsen worker 52 - jepsen.util 1042 :invoke :write [17 4] INFO [2019-12-17 12:28:27,907] jepsen worker 64 - jepsen.util 604 :fail :cas [21 [26 12]] :ValueNotSwapFrom INFO [2019-12-17 12:28:27,908] jepsen worker 64 - jepsen.util 604 :invoke :cas [21 [28 30]] INFO [2019-12-17 12:28:27,908] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":5654,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:28:27,908] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":5654,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:28:27,908] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":5658,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:28:27,908] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":5658,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:28:27,916] jepsen worker 71 - jepsen.util 791 :ok :read [23 44] INFO [2019-12-17 12:28:27,916] jepsen worker 71 - jepsen.util 791 :invoke :read [23 nil] INFO [2019-12-17 12:28:27,929] jepsen worker 64 - jepsen.util 604 :fail :cas [21 [28 30]] :ValueNotSwapFrom INFO [2019-12-17 12:28:27,929] jepsen worker 64 - jepsen.util 604 :invoke :write [21 30] INFO [2019-12-17 12:28:27,938] jepsen worker 14 - jepsen.util 914 :fail :cas [4 [0 45]] :ValueNotSwapFrom INFO [2019-12-17 12:28:27,938] jepsen worker 14 - jepsen.util 914 :invoke :cas [4 [10 3]] INFO [2019-12-17 12:28:27,951] jepsen worker 69 - jepsen.util 969 :ok :read [23 44] INFO [2019-12-17 12:28:27,951] jepsen worker 69 - jepsen.util 969 :invoke :read [23 nil] INFO [2019-12-17 12:28:27,987] jepsen worker 53 - jepsen.util 953 :info :write [17 44] :RequestCanceledException INFO [2019-12-17 12:28:27,991] jepsen worker 53 - jepsen.util 1043 :invoke :cas [17 [38 1]] INFO [2019-12-17 12:28:28,003] jepsen worker 21 - jepsen.util 1191 :fail :cas [7 [37 41]] :ValueNotSwapFrom INFO [2019-12-17 12:28:28,003] jepsen worker 21 - jepsen.util 1191 :invoke :cas [7 [36 42]] INFO [2019-12-17 12:28:28,015] jepsen worker 64 - jepsen.util 604 :ok :write [21 30] INFO [2019-12-17 12:28:28,015] jepsen worker 64 - jepsen.util 604 :invoke :write [21 28] INFO [2019-12-17 12:28:28,021] jepsen worker 71 - jepsen.util 791 :ok :read [23 44] INFO [2019-12-17 12:28:28,021] jepsen worker 71 - jepsen.util 791 :invoke :read [23 nil] INFO [2019-12-17 12:28:28,021] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":5671,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:28:28,021] jepsen worker 80 - jepsen.util 1160 :info :write [26 8] :RequestCanceledException INFO [2019-12-17 12:28:28,021] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":5671,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:28:28,022] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":5673,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:28:28,022] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":5673,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:28:28,024] jepsen worker 80 - jepsen.util 1250 :invoke :read [26 nil] INFO [2019-12-17 12:28:28,029] jepsen worker 3 - jepsen.util 813 :fail :cas [1 [43 23]] :ValueNotSwapFrom INFO [2019-12-17 12:28:28,029] jepsen worker 3 - jepsen.util 813 :invoke :write [1 41] INFO [2019-12-17 12:28:28,046] jepsen worker 87 - jepsen.util 987 :ok :read [29 17] INFO [2019-12-17 12:28:28,046] jepsen worker 87 - jepsen.util 987 :invoke :cas [29 [17 8]] INFO [2019-12-17 12:28:28,072] jepsen worker 36 - jepsen.util 1026 :ok :read [12 38] INFO [2019-12-17 12:28:28,072] jepsen worker 36 - jepsen.util 1026 :invoke :write [12 48] INFO [2019-12-17 12:28:28,072] jepsen worker 21 - jepsen.util 1191 :fail :cas [7 [36 42]] :ValueNotSwapFrom INFO [2019-12-17 12:28:28,073] jepsen worker 21 - jepsen.util 1191 :invoke :cas [7 [42 10]] INFO [2019-12-17 12:28:28,113] jepsen worker 23 - jepsen.util 1103 :ok :read [7 20] INFO [2019-12-17 12:28:28,113] jepsen worker 23 - jepsen.util 1103 :invoke :write [7 37] INFO [2019-12-17 12:28:28,121] jepsen worker 37 - jepsen.util 847 :info :write [12 34] :RequestCanceledException INFO [2019-12-17 12:28:28,121] jepsen worker 64 - jepsen.util 604 :ok :write [21 28] INFO [2019-12-17 12:28:28,121] jepsen worker 64 - jepsen.util 604 :invoke :read [21 nil] INFO [2019-12-17 12:28:28,123] jepsen worker 37 - jepsen.util 937 :invoke :cas [12 [40 26]] INFO [2019-12-17 12:28:28,135] jepsen worker 0 - jepsen.util 1350 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:28:28,135] jepsen worker 0 - jepsen.util 1350 :invoke :write [0 1] INFO [2019-12-17 12:28:28,138] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:28:28,138] jepsen nemesis - couchbase.util Rebalance check count: 36 INFO [2019-12-17 12:28:28,138] jepsen nemesis - couchbase.util Rebalance stuck count: 17 INFO [2019-12-17 12:28:28,153] jepsen worker 85 - jepsen.util 1075 :info :cas [28 [6 0]] :RequestCanceledException INFO [2019-12-17 12:28:28,155] jepsen worker 85 - jepsen.util 1165 :invoke :cas [28 [17 3]] INFO [2019-12-17 12:28:28,158] jepsen worker 64 - jepsen.util 604 :ok :read [21 28] INFO [2019-12-17 12:28:28,158] jepsen worker 64 - jepsen.util 604 :invoke :cas [21 [47 26]] INFO [2019-12-17 12:28:28,160] jepsen worker 4 - jepsen.util 814 :fail :cas [1 [17 34]] :ValueNotSwapFrom INFO [2019-12-17 12:28:28,160] jepsen worker 4 - jepsen.util 814 :invoke :cas [1 [46 10]] INFO [2019-12-17 12:28:28,179] jepsen worker 64 - jepsen.util 604 :fail :cas [21 [47 26]] :ValueNotSwapFrom INFO [2019-12-17 12:28:28,180] jepsen worker 64 - jepsen.util 604 :invoke :read [21 nil] INFO [2019-12-17 12:28:28,186] jepsen worker 82 - jepsen.util 1612 :info :write [27 40] :RequestCanceledException INFO [2019-12-17 12:28:28,187] jepsen worker 82 - jepsen.util 1702 :invoke :write [27 40] INFO [2019-12-17 12:28:28,205] jepsen worker 38 - jepsen.util 848 :info :write [12 16] :RequestCanceledException INFO [2019-12-17 12:28:28,205] jepsen worker 89 - jepsen.util 1079 :info :write [29 42] :RequestCanceledException INFO [2019-12-17 12:28:28,205] jepsen worker 50 - jepsen.util 1310 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:28:28,205] jepsen worker 57 - jepsen.util 1227 :info :cas [19 [38 4]] :RequestCanceledException INFO [2019-12-17 12:28:28,206] jepsen worker 50 - jepsen.util 1310 :invoke :read [16 nil] INFO [2019-12-17 12:28:28,206] jepsen worker 38 - jepsen.util 938 :invoke :read [12 nil] INFO [2019-12-17 12:28:28,208] jepsen worker 89 - jepsen.util 1169 :invoke :read [29 nil] INFO [2019-12-17 12:28:28,208] jepsen worker 57 - jepsen.util 1317 :invoke :read [19 nil] INFO [2019-12-17 12:28:28,216] jepsen worker 64 - jepsen.util 604 :ok :read [21 28] INFO [2019-12-17 12:28:28,216] jepsen worker 64 - jepsen.util 604 :invoke :write [21 42] INFO [2019-12-17 12:28:28,221] jepsen worker 49 - jepsen.util 1129 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:28:28,221] jepsen worker 66 - jepsen.util 966 :info :write [22 42] :RequestCanceledException INFO [2019-12-17 12:28:28,221] jepsen worker 17 - jepsen.util 1187 :info :write [5 1] :RequestCanceledException INFO [2019-12-17 12:28:28,221] jepsen worker 49 - jepsen.util 1129 :invoke :read [16 nil] INFO [2019-12-17 12:28:28,221] jepsen worker 2 - jepsen.util 992 :info :write [0 23] :RequestCanceledException INFO [2019-12-17 12:28:28,222] jepsen worker 39 - jepsen.util 1299 :info :write [13 27] :RequestCanceledException INFO [2019-12-17 12:28:28,224] jepsen worker 2 - jepsen.util 1082 :invoke :read [0 nil] INFO [2019-12-17 12:28:28,224] jepsen worker 66 - jepsen.util 1056 :invoke :read [22 nil] INFO [2019-12-17 12:28:28,225] jepsen worker 17 - jepsen.util 1277 :invoke :read [5 nil] INFO [2019-12-17 12:28:28,225] jepsen worker 39 - jepsen.util 1389 :invoke :cas [13 [44 46]] INFO [2019-12-17 12:28:28,235] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":5691,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:28:28,236] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":5691,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:28:28,236] jepsen worker 40 - jepsen.util 1570 :info :cas [13 [37 2]] :RequestCanceledException INFO [2019-12-17 12:28:28,236] jepsen worker 68 - jepsen.util 878 :info :write [22 42] :RequestCanceledException INFO [2019-12-17 12:28:28,236] jepsen worker 59 - jepsen.util 1049 :info :write [19 36] :RequestCanceledException INFO [2019-12-17 12:28:28,236] jepsen worker 83 - jepsen.util 1433 :info :cas [27 [0 43]] :RequestCanceledException INFO [2019-12-17 12:28:28,236] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":5693,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:28:28,236] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":5693,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:28:28,239] jepsen worker 40 - jepsen.util 1660 :invoke :read [13 nil] INFO [2019-12-17 12:28:28,239] jepsen worker 59 - jepsen.util 1139 :invoke :cas [19 [36 6]] INFO [2019-12-17 12:28:28,239] jepsen worker 83 - jepsen.util 1523 :invoke :cas [27 [1 10]] INFO [2019-12-17 12:28:28,240] jepsen worker 68 - jepsen.util 968 :invoke :cas [22 [27 48]] INFO [2019-12-17 12:28:28,240] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":5699,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:28:28,240] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":5699,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:28:28,240] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":5704,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:28:28,240] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":5704,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:28:28,241] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":5705,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:28:28,241] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":5705,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:28:28,250] jepsen worker 48 - jepsen.util 678 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:28:28,250] jepsen worker 60 - jepsen.util 1320 :info :write [20 16] :RequestCanceledException INFO [2019-12-17 12:28:28,250] jepsen worker 16 - jepsen.util 1096 :info :write [5 41] :RequestCanceledException INFO [2019-12-17 12:28:28,250] jepsen worker 67 - jepsen.util 1507 :info :cas [22 [1 23]] :RequestCanceledException INFO [2019-12-17 12:28:28,250] jepsen worker 48 - jepsen.util 678 :invoke :cas [16 [25 45]] INFO [2019-12-17 12:28:28,251] jepsen worker 60 - jepsen.util 1410 :invoke :read [20 nil] INFO [2019-12-17 12:28:28,252] jepsen worker 67 - jepsen.util 1597 :invoke :cas [22 [24 8]] INFO [2019-12-17 12:28:28,252] jepsen worker 16 - jepsen.util 1186 :invoke :write [5 19] INFO [2019-12-17 12:28:28,272] jepsen worker 54 - jepsen.util 1134 :ok :read [18 17] INFO [2019-12-17 12:28:28,272] jepsen worker 54 - jepsen.util 1134 :invoke :write [18 49] INFO [2019-12-17 12:28:28,287] jepsen worker 15 - jepsen.util 915 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:28:28,287] jepsen worker 84 - jepsen.util 984 :info :cas [28 [2 15]] :RequestCanceledException INFO [2019-12-17 12:28:28,287] jepsen worker 41 - jepsen.util 1031 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:28:28,288] jepsen worker 15 - jepsen.util 915 :invoke :read [5 nil] INFO [2019-12-17 12:28:28,288] jepsen worker 41 - jepsen.util 1031 :invoke :cas [13 [47 18]] INFO [2019-12-17 12:28:28,290] jepsen worker 84 - jepsen.util 1074 :invoke :cas [28 [22 19]] INFO [2019-12-17 12:28:28,291] jepsen worker 79 - jepsen.util 979 :ok :read [26 39] INFO [2019-12-17 12:28:28,291] jepsen worker 79 - jepsen.util 979 :invoke :write [26 37] INFO [2019-12-17 12:28:28,304] jepsen worker 42 - jepsen.util 1392 :info :cas [14 [33 22]] :RequestCanceledException INFO [2019-12-17 12:28:28,304] jepsen worker 86 - jepsen.util 1166 :info :write [28 3] :RequestCanceledException INFO [2019-12-17 12:28:28,305] jepsen worker 42 - jepsen.util 1482 :invoke :cas [14 [46 39]] INFO [2019-12-17 12:28:28,308] jepsen worker 86 - jepsen.util 1256 :invoke :cas [28 [12 37]] INFO [2019-12-17 12:28:28,321] jepsen worker 64 - jepsen.util 604 :ok :write [21 42] INFO [2019-12-17 12:28:28,321] jepsen worker 75 - jepsen.util 1245 :info :write [25 38] :RequestCanceledException INFO [2019-12-17 12:28:28,321] jepsen worker 81 - jepsen.util 1341 :info :write [27 27] :RequestCanceledException INFO [2019-12-17 12:28:28,321] jepsen worker 1 - jepsen.util 991 :info :cas [0 [48 28]] :RequestCanceledException INFO [2019-12-17 12:28:28,322] jepsen worker 19 - jepsen.util 1189 :fail :read [6 nil] :RequestTimeoutException INFO [2019-12-17 12:28:28,322] jepsen worker 64 - jepsen.util 604 :invoke :write [21 26] INFO [2019-12-17 12:28:28,322] jepsen worker 19 - jepsen.util 1189 :invoke :read [6 nil] INFO [2019-12-17 12:28:28,325] jepsen worker 75 - jepsen.util 1335 :invoke :write [25 29] INFO [2019-12-17 12:28:28,327] jepsen worker 1 - jepsen.util 1081 :invoke :write [0 18] INFO [2019-12-17 12:28:28,328] jepsen worker 81 - jepsen.util 1431 :invoke :read [27 nil] INFO [2019-12-17 12:28:28,333] jepsen worker 21 - jepsen.util 1191 :fail :cas [7 [42 10]] :ValueNotSwapFrom INFO [2019-12-17 12:28:28,333] jepsen worker 21 - jepsen.util 1191 :invoke :read [7 nil] INFO [2019-12-17 12:28:28,352] jepsen worker 77 - jepsen.util 1247 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:28:28,353] jepsen worker 77 - jepsen.util 1247 :invoke :cas [25 [48 41]] INFO [2019-12-17 12:28:28,358] jepsen worker 69 - jepsen.util 969 :ok :read [23 44] INFO [2019-12-17 12:28:28,358] jepsen worker 69 - jepsen.util 969 :invoke :write [23 26] INFO [2019-12-17 12:28:28,380] jepsen worker 4 - jepsen.util 814 :fail :cas [1 [46 10]] :ValueNotSwapFrom INFO [2019-12-17 12:28:28,380] jepsen worker 4 - jepsen.util 814 :invoke :write [1 26] INFO [2019-12-17 12:28:28,388] jepsen worker 62 - jepsen.util 1142 :info :write [20 33] :RequestCanceledException INFO [2019-12-17 12:28:28,395] jepsen worker 62 - jepsen.util 1232 :invoke :write [20 6] INFO [2019-12-17 12:28:28,400] jepsen worker 64 - jepsen.util 604 :ok :write [21 26] INFO [2019-12-17 12:28:28,400] jepsen worker 64 - jepsen.util 604 :invoke :read [21 nil] INFO [2019-12-17 12:28:28,431] jepsen worker 38 - jepsen.util 938 :ok :read [12 38] INFO [2019-12-17 12:28:28,431] jepsen worker 38 - jepsen.util 938 :invoke :read [12 nil] INFO [2019-12-17 12:28:28,433] jepsen worker 42 - jepsen.util 1482 :fail :cas [14 [46 39]] :ValueNotSwapFrom INFO [2019-12-17 12:28:28,433] jepsen worker 42 - jepsen.util 1482 :invoke :write [14 21] INFO [2019-12-17 12:28:28,457] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":5742,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:28:28,458] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":5742,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:28:28,488] jepsen worker 27 - jepsen.util 747 :info :cas [9 [40 46]] :RequestCanceledException INFO [2019-12-17 12:28:28,498] jepsen worker 27 - jepsen.util 837 :invoke :write [9 37] INFO [2019-12-17 12:28:28,521] jepsen worker 58 - jepsen.util 1138 :info :write [19 16] :RequestCanceledException INFO [2019-12-17 12:28:28,529] jepsen worker 58 - jepsen.util 1228 :invoke :cas [19 [18 20]] INFO [2019-12-17 12:28:28,530] jepsen worker 64 - jepsen.util 604 :ok :read [21 26] INFO [2019-12-17 12:28:28,532] jepsen worker 31 - jepsen.util 1111 :ok :write [10 36] INFO [2019-12-17 12:28:28,533] jepsen worker 31 - jepsen.util 1111 :invoke :cas [10 [16 40]] INFO [2019-12-17 12:28:28,534] jepsen worker 64 - jepsen.util 604 :invoke :write [21 35] INFO [2019-12-17 12:28:28,534] jepsen worker 47 - jepsen.util 1397 :ok :write [15 12] INFO [2019-12-17 12:28:28,535] jepsen worker 47 - jepsen.util 1397 :invoke :cas [15 [44 9]] INFO [2019-12-17 12:28:28,544] jepsen worker 20 - jepsen.util 1100 :ok :write [6 8] INFO [2019-12-17 12:28:28,544] jepsen worker 20 - jepsen.util 1100 :invoke :read [6 nil] INFO [2019-12-17 12:28:28,548] jepsen worker 53 - jepsen.util 1043 :fail :cas [17 [38 1]] :ValueNotSwapFrom INFO [2019-12-17 12:28:28,549] jepsen worker 53 - jepsen.util 1043 :invoke :cas [17 [41 36]] INFO [2019-12-17 12:28:28,549] jepsen worker 71 - jepsen.util 791 :ok :read [23 44] INFO [2019-12-17 12:28:28,550] jepsen worker 71 - jepsen.util 791 :invoke :read [23 nil] INFO [2019-12-17 12:28:28,551] jepsen worker 78 - jepsen.util 1068 :info :write [26 32] :RequestCanceledException INFO [2019-12-17 12:28:28,551] jepsen worker 12 - jepsen.util 1002 :info :write [4 35] :RequestCanceledException INFO [2019-12-17 12:28:28,552] jepsen worker 12 - jepsen.util 1092 :invoke :read [4 nil] INFO [2019-12-17 12:28:28,553] jepsen worker 78 - jepsen.util 1158 :invoke :cas [26 [16 30]] INFO [2019-12-17 12:28:28,561] jepsen worker 11 - jepsen.util 1361 :ok :write [3 28] INFO [2019-12-17 12:28:28,561] jepsen worker 11 - jepsen.util 1361 :invoke :read [3 nil] INFO [2019-12-17 12:28:28,598] jepsen worker 11 - jepsen.util 1361 :ok :read [3 28] INFO [2019-12-17 12:28:28,598] jepsen worker 11 - jepsen.util 1361 :invoke :write [3 31] INFO [2019-12-17 12:28:28,648] jepsen worker 30 - jepsen.util 1200 :ok :write [10 20] INFO [2019-12-17 12:28:28,648] jepsen worker 30 - jepsen.util 1200 :invoke :cas [10 [47 6]] INFO [2019-12-17 12:28:28,667] jepsen worker 28 - jepsen.util 748 :ok :write [9 42] INFO [2019-12-17 12:28:28,668] jepsen worker 28 - jepsen.util 748 :invoke :read [9 nil] INFO [2019-12-17 12:28:28,674] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":5762,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:28:28,674] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":5762,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:28:28,674] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":5765,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:28:28,674] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":5765,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:28:28,674] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":5766,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:28:28,674] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":5766,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:28:28,678] jepsen worker 70 - jepsen.util 1240 :ok :read [23 44] INFO [2019-12-17 12:28:28,678] jepsen worker 70 - jepsen.util 1240 :invoke :read [23 nil] INFO [2019-12-17 12:28:28,690] jepsen worker 19 - jepsen.util 1189 :ok :read [6 8] INFO [2019-12-17 12:28:28,690] jepsen worker 19 - jepsen.util 1189 :invoke :write [6 33] INFO [2019-12-17 12:28:28,704] jepsen worker 64 - jepsen.util 604 :ok :write [21 35] INFO [2019-12-17 12:28:28,704] jepsen worker 64 - jepsen.util 604 :invoke :write [21 49] INFO [2019-12-17 12:28:28,712] jepsen worker 29 - jepsen.util 839 :ok :read [9 42] INFO [2019-12-17 12:28:28,712] jepsen worker 29 - jepsen.util 839 :invoke :cas [9 [21 39]] INFO [2019-12-17 12:28:28,734] jepsen worker 63 - jepsen.util 963 :info :cas [21 [41 26]] :RequestCanceledException INFO [2019-12-17 12:28:28,738] jepsen worker 63 - jepsen.util 1053 :invoke :read [21 nil] INFO [2019-12-17 12:28:28,747] jepsen worker 89 - jepsen.util 1169 :ok :read [29 17] INFO [2019-12-17 12:28:28,748] jepsen worker 89 - jepsen.util 1169 :invoke :write [29 12] INFO [2019-12-17 12:28:28,764] jepsen worker 10 - jepsen.util 1270 :ok :write [3 14] INFO [2019-12-17 12:28:28,764] jepsen worker 10 - jepsen.util 1270 :invoke :write [3 3] INFO [2019-12-17 12:28:28,773] jepsen worker 27 - jepsen.util 837 :ok :write [9 37] INFO [2019-12-17 12:28:28,774] jepsen worker 27 - jepsen.util 837 :invoke :cas [9 [42 19]] INFO [2019-12-17 12:28:28,788] jepsen worker 73 - jepsen.util 1063 :ok :write [24 31] INFO [2019-12-17 12:28:28,788] jepsen worker 73 - jepsen.util 1063 :invoke :cas [24 [22 36]] INFO [2019-12-17 12:28:28,803] jepsen worker 5 - jepsen.util 635 :info :write [1 18] :RequestCanceledException INFO [2019-12-17 12:28:28,807] jepsen worker 5 - jepsen.util 725 :invoke :read [1 nil] INFO [2019-12-17 12:28:28,808] jepsen worker 47 - jepsen.util 1397 :fail :cas [15 [44 9]] :ValueNotSwapFrom INFO [2019-12-17 12:28:28,808] jepsen worker 47 - jepsen.util 1397 :invoke :cas [15 [24 48]] INFO [2019-12-17 12:28:28,828] jepsen worker 64 - jepsen.util 604 :ok :write [21 49] INFO [2019-12-17 12:28:28,829] jepsen worker 64 - jepsen.util 604 :invoke :write [21 29] INFO [2019-12-17 12:28:28,829] jepsen worker 55 - jepsen.util 1315 :info :write [18 43] :RequestCanceledException INFO [2019-12-17 12:28:28,834] jepsen worker 80 - jepsen.util 1250 :ok :read [26 39] INFO [2019-12-17 12:28:28,834] jepsen worker 55 - jepsen.util 1405 :invoke :cas [18 [12 41]] INFO [2019-12-17 12:28:28,834] jepsen worker 80 - jepsen.util 1250 :invoke :write [26 17] INFO [2019-12-17 12:28:28,834] jepsen worker 14 - jepsen.util 914 :fail :cas [4 [10 3]] :ValueNotSwapFrom INFO [2019-12-17 12:28:28,835] jepsen worker 14 - jepsen.util 914 :invoke :write [4 35] INFO [2019-12-17 12:28:28,874] jepsen worker 31 - jepsen.util 1111 :fail :cas [10 [16 40]] :ValueNotSwapFrom INFO [2019-12-17 12:28:28,874] jepsen worker 31 - jepsen.util 1111 :invoke :cas [10 [49 45]] INFO [2019-12-17 12:28:28,884] jepsen worker 28 - jepsen.util 748 :ok :read [9 37] INFO [2019-12-17 12:28:28,884] jepsen worker 28 - jepsen.util 748 :invoke :write [9 40] INFO [2019-12-17 12:28:28,885] jepsen worker 56 - jepsen.util 1046 :fail :cas [18 [23 46]] :ValueNotSwapFrom INFO [2019-12-17 12:28:28,886] jepsen worker 56 - jepsen.util 1046 :invoke :read [18 nil] INFO [2019-12-17 12:28:28,896] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":5786,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:28:28,896] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":5786,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:28:28,919] jepsen worker 63 - jepsen.util 1053 :ok :read [21 49] INFO [2019-12-17 12:28:28,919] jepsen worker 26 - jepsen.util 1286 :info :write [8 27] :RequestCanceledException INFO [2019-12-17 12:28:28,920] jepsen worker 63 - jepsen.util 1053 :invoke :cas [21 [34 19]] INFO [2019-12-17 12:28:28,922] jepsen worker 26 - jepsen.util 1376 :invoke :read [8 nil] INFO [2019-12-17 12:28:28,928] jepsen worker 29 - jepsen.util 839 :fail :cas [9 [21 39]] :ValueNotSwapFrom INFO [2019-12-17 12:28:28,928] jepsen worker 29 - jepsen.util 839 :invoke :cas [9 [5 7]] INFO [2019-12-17 12:28:28,935] jepsen worker 88 - jepsen.util 898 :info :write [29 13] :RequestCanceledException INFO [2019-12-17 12:28:28,936] jepsen worker 88 - jepsen.util 988 :invoke :read [29 nil] INFO [2019-12-17 12:28:28,953] jepsen worker 8 - jepsen.util 1268 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:28:28,953] jepsen worker 22 - jepsen.util 1192 :info :cas [7 [36 41]] :RequestCanceledException INFO [2019-12-17 12:28:28,953] jepsen worker 8 - jepsen.util 1268 :invoke :read [2 nil] INFO [2019-12-17 12:28:28,955] jepsen worker 22 - jepsen.util 1282 :invoke :cas [7 [48 3]] INFO [2019-12-17 12:28:28,962] jepsen worker 73 - jepsen.util 1063 :fail :cas [24 [22 36]] :ValueNotSwapFrom INFO [2019-12-17 12:28:28,962] jepsen worker 73 - jepsen.util 1063 :invoke :write [24 28] INFO [2019-12-17 12:28:28,987] jepsen worker 31 - jepsen.util 1111 :fail :cas [10 [49 45]] :ValueNotSwapFrom INFO [2019-12-17 12:28:28,987] jepsen worker 31 - jepsen.util 1111 :invoke :cas [10 [22 46]] INFO [2019-12-17 12:28:28,988] jepsen worker 34 - jepsen.util 1294 :info :write [11 13] :RequestCanceledException INFO [2019-12-17 12:28:28,988] jepsen worker 51 - jepsen.util 951 :info :write [17 30] :RequestCanceledException INFO [2019-12-17 12:28:28,991] jepsen worker 51 - jepsen.util 1041 :invoke :cas [17 [32 2]] INFO [2019-12-17 12:28:28,991] jepsen worker 34 - jepsen.util 1384 :invoke :write [11 36] INFO [2019-12-17 12:28:28,995] jepsen worker 21 - jepsen.util 1191 :ok :read [7 20] INFO [2019-12-17 12:28:28,996] jepsen worker 21 - jepsen.util 1191 :invoke :read [7 nil] INFO [2019-12-17 12:28:29,005] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":5803,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:28:29,005] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":5803,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:28:29,036] jepsen worker 43 - jepsen.util 1303 :info :write [14 14] :RequestCanceledException INFO [2019-12-17 12:28:29,039] jepsen worker 43 - jepsen.util 1393 :invoke :cas [14 [45 20]] INFO [2019-12-17 12:28:29,043] jepsen worker 64 - jepsen.util 604 :ok :write [21 29] INFO [2019-12-17 12:28:29,044] jepsen worker 64 - jepsen.util 604 :invoke :read [21 nil] INFO [2019-12-17 12:28:29,049] jepsen worker 22 - jepsen.util 1282 :fail :cas [7 [48 3]] :ValueNotSwapFrom INFO [2019-12-17 12:28:29,049] jepsen worker 22 - jepsen.util 1282 :invoke :cas [7 [17 9]] INFO [2019-12-17 12:28:29,064] jepsen worker 19 - jepsen.util 1189 :ok :write [6 33] INFO [2019-12-17 12:28:29,064] jepsen worker 19 - jepsen.util 1189 :invoke :read [6 nil] INFO [2019-12-17 12:28:29,070] jepsen worker 21 - jepsen.util 1191 :ok :read [7 20] INFO [2019-12-17 12:28:29,071] jepsen worker 21 - jepsen.util 1191 :invoke :cas [7 [33 17]] INFO [2019-12-17 12:28:29,083] jepsen worker 28 - jepsen.util 748 :ok :write [9 40] INFO [2019-12-17 12:28:29,083] jepsen worker 28 - jepsen.util 748 :invoke :write [9 34] INFO [2019-12-17 12:28:29,088] jepsen worker 6 - jepsen.util 1266 :info :write [2 29] :RequestCanceledException INFO [2019-12-17 12:28:29,090] jepsen worker 6 - jepsen.util 1356 :invoke :read [2 nil] INFO [2019-12-17 12:28:29,095] jepsen worker 10 - jepsen.util 1270 :ok :write [3 3] INFO [2019-12-17 12:28:29,095] jepsen worker 10 - jepsen.util 1270 :invoke :cas [3 [22 27]] INFO [2019-12-17 12:28:29,102] jepsen worker 76 - jepsen.util 1246 :info :cas [25 [25 9]] :RequestCanceledException INFO [2019-12-17 12:28:29,104] jepsen worker 76 - jepsen.util 1336 :invoke :cas [25 [10 31]] INFO [2019-12-17 12:28:29,120] jepsen worker 24 - jepsen.util 1104 :info :write [8 47] :RequestCanceledException INFO [2019-12-17 12:28:29,122] jepsen worker 24 - jepsen.util 1194 :invoke :write [8 0] INFO [2019-12-17 12:28:29,125] jepsen worker 51 - jepsen.util 1041 :fail :cas [17 [32 2]] :ValueNotSwapFrom INFO [2019-12-17 12:28:29,125] jepsen worker 51 - jepsen.util 1041 :invoke :write [17 4] INFO [2019-12-17 12:28:29,143] jepsen worker 64 - jepsen.util 604 :ok :read [21 29] INFO [2019-12-17 12:28:29,143] jepsen worker 64 - jepsen.util 604 :invoke :cas [21 [48 49]] INFO [2019-12-17 12:28:29,149] jepsen worker 10 - jepsen.util 1270 :fail :cas [3 [22 27]] :ValueNotSwapFrom INFO [2019-12-17 12:28:29,150] jepsen worker 10 - jepsen.util 1270 :invoke :write [3 43] INFO [2019-12-17 12:28:29,152] jepsen worker 46 - jepsen.util 1216 :info :write [15 0] :RequestCanceledException INFO [2019-12-17 12:28:29,154] jepsen worker 46 - jepsen.util 1306 :invoke :cas [15 [48 46]] INFO [2019-12-17 12:28:29,173] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:28:29,173] jepsen nemesis - couchbase.util Rebalance check count: 37 INFO [2019-12-17 12:28:29,173] jepsen nemesis - couchbase.util Rebalance stuck count: 18 INFO [2019-12-17 12:28:29,176] jepsen worker 63 - jepsen.util 1053 :fail :cas [21 [34 19]] :ValueNotSwapFrom INFO [2019-12-17 12:28:29,177] jepsen worker 63 - jepsen.util 1053 :invoke :write [21 17] INFO [2019-12-17 12:28:29,180] jepsen worker 37 - jepsen.util 937 :fail :cas [12 [40 26]] :ValueNotSwapFrom INFO [2019-12-17 12:28:29,180] jepsen worker 37 - jepsen.util 937 :invoke :read [12 nil] INFO [2019-12-17 12:28:29,186] jepsen worker 9 - jepsen.util 999 :info :cas [3 [44 19]] :RequestCanceledException INFO [2019-12-17 12:28:29,189] jepsen worker 9 - jepsen.util 1089 :invoke :write [3 19] INFO [2019-12-17 12:28:29,219] jepsen worker 19 - jepsen.util 1189 :ok :read [6 33] INFO [2019-12-17 12:28:29,220] jepsen worker 31 - jepsen.util 1111 :fail :cas [10 [22 46]] :ValueNotSwapFrom INFO [2019-12-17 12:28:29,220] jepsen worker 19 - jepsen.util 1189 :invoke :cas [6 [42 37]] INFO [2019-12-17 12:28:29,220] jepsen worker 61 - jepsen.util 1231 :info :cas [20 [14 41]] :RequestCanceledException INFO [2019-12-17 12:28:29,220] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":5826,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:28:29,220] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":5826,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:28:29,220] jepsen worker 44 - jepsen.util 1124 :info :write [14 16] :RequestCanceledException INFO [2019-12-17 12:28:29,221] jepsen worker 31 - jepsen.util 1111 :invoke :read [10 nil] INFO [2019-12-17 12:28:29,221] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":5827,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:28:29,222] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":5827,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:28:29,222] jepsen worker 61 - jepsen.util 1321 :invoke :read [20 nil] INFO [2019-12-17 12:28:29,222] jepsen worker 44 - jepsen.util 1214 :invoke :cas [14 [3 16]] INFO [2019-12-17 12:28:29,235] jepsen worker 64 - jepsen.util 604 :fail :cas [21 [48 49]] :ValueNotSwapFrom INFO [2019-12-17 12:28:29,235] jepsen worker 64 - jepsen.util 604 :invoke :cas [21 [21 5]] INFO [2019-12-17 12:28:29,236] jepsen worker 74 - jepsen.util 884 :fail :read [24 nil] :RequestTimeoutException INFO [2019-12-17 12:28:29,236] jepsen worker 74 - jepsen.util 884 :invoke :cas [24 [31 32]] INFO [2019-12-17 12:28:29,253] jepsen worker 65 - jepsen.util 785 :info :write [21 39] :RequestCanceledException INFO [2019-12-17 12:28:29,253] jepsen worker 72 - jepsen.util 972 :info :cas [24 [48 19]] :RequestCanceledException INFO [2019-12-17 12:28:29,257] jepsen worker 72 - jepsen.util 1062 :invoke :cas [24 [35 26]] INFO [2019-12-17 12:28:29,259] jepsen worker 65 - jepsen.util 875 :invoke :write [21 8] INFO [2019-12-17 12:28:29,284] jepsen worker 46 - jepsen.util 1306 :fail :cas [15 [48 46]] :ValueNotSwapFrom INFO [2019-12-17 12:28:29,284] jepsen worker 46 - jepsen.util 1306 :invoke :write [15 1] INFO [2019-12-17 12:28:29,305] jepsen worker 5 - jepsen.util 725 :ok :read [1 24] INFO [2019-12-17 12:28:29,306] jepsen worker 5 - jepsen.util 725 :invoke :read [1 nil] INFO [2019-12-17 12:28:29,309] jepsen worker 53 - jepsen.util 1043 :fail :cas [17 [41 36]] :ValueNotSwapFrom INFO [2019-12-17 12:28:29,309] jepsen worker 53 - jepsen.util 1043 :invoke :write [17 22] INFO [2019-12-17 12:28:29,319] jepsen worker 71 - jepsen.util 791 :ok :read [23 44] INFO [2019-12-17 12:28:29,320] jepsen worker 71 - jepsen.util 791 :invoke :read [23 nil] INFO [2019-12-17 12:28:29,322] jepsen worker 28 - jepsen.util 748 :ok :write [9 34] INFO [2019-12-17 12:28:29,323] jepsen worker 28 - jepsen.util 748 :invoke :write [9 11] INFO [2019-12-17 12:28:29,329] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":5835,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:28:29,329] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":5835,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:28:29,330] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":5839,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:28:29,330] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":5839,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:28:29,333] jepsen worker 22 - jepsen.util 1282 :fail :cas [7 [17 9]] :ValueNotSwapFrom INFO [2019-12-17 12:28:29,333] jepsen worker 22 - jepsen.util 1282 :invoke :write [7 35] INFO [2019-12-17 12:28:29,336] jepsen worker 19 - jepsen.util 1189 :fail :cas [6 [42 37]] :ValueNotSwapFrom INFO [2019-12-17 12:28:29,336] jepsen worker 19 - jepsen.util 1189 :invoke :write [6 31] INFO [2019-12-17 12:28:29,345] jepsen worker 21 - jepsen.util 1191 :fail :cas [7 [33 17]] :ValueNotSwapFrom INFO [2019-12-17 12:28:29,345] jepsen worker 21 - jepsen.util 1191 :invoke :write [7 47] INFO [2019-12-17 12:28:29,349] jepsen worker 33 - jepsen.util 1473 :info :write [11 15] :RequestCanceledException INFO [2019-12-17 12:28:29,354] jepsen worker 33 - jepsen.util 1563 :invoke :write [11 46] INFO [2019-12-17 12:28:29,382] jepsen worker 38 - jepsen.util 938 :ok :read [12 38] INFO [2019-12-17 12:28:29,382] jepsen worker 38 - jepsen.util 938 :invoke :cas [12 [16 17]] INFO [2019-12-17 12:28:29,400] jepsen worker 80 - jepsen.util 1250 :ok :write [26 17] INFO [2019-12-17 12:28:29,401] jepsen worker 80 - jepsen.util 1250 :invoke :write [26 27] INFO [2019-12-17 12:28:29,418] jepsen worker 78 - jepsen.util 1158 :fail :cas [26 [16 30]] :ValueNotSwapFrom INFO [2019-12-17 12:28:29,418] jepsen worker 78 - jepsen.util 1158 :invoke :read [26 nil] INFO [2019-12-17 12:28:29,422] jepsen worker 32 - jepsen.util 1022 :info :write [10 33] :RequestCanceledException INFO [2019-12-17 12:28:29,422] jepsen worker 13 - jepsen.util 1003 :info :write [4 37] :RequestCanceledException INFO [2019-12-17 12:28:29,424] jepsen worker 32 - jepsen.util 1112 :invoke :write [10 2] INFO [2019-12-17 12:28:29,426] jepsen worker 13 - jepsen.util 1093 :invoke :cas [4 [44 2]] INFO [2019-12-17 12:28:29,429] jepsen worker 43 - jepsen.util 1393 :fail :cas [14 [45 20]] :ValueNotSwapFrom INFO [2019-12-17 12:28:29,429] jepsen worker 43 - jepsen.util 1393 :invoke :write [14 47] INFO [2019-12-17 12:28:29,438] jepsen worker 31 - jepsen.util 1111 :ok :read [10 20] INFO [2019-12-17 12:28:29,438] jepsen worker 31 - jepsen.util 1111 :invoke :read [10 nil] INFO [2019-12-17 12:28:29,439] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":5862,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:28:29,439] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":5862,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:28:29,439] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":5864,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:28:29,439] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":5864,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:28:29,459] jepsen worker 12 - jepsen.util 1092 :ok :read [4 5] INFO [2019-12-17 12:28:29,459] jepsen worker 12 - jepsen.util 1092 :invoke :read [4 nil] INFO [2019-12-17 12:28:29,469] jepsen worker 63 - jepsen.util 1053 :ok :write [21 17] INFO [2019-12-17 12:28:29,469] jepsen worker 63 - jepsen.util 1053 :invoke :cas [21 [27 7]] INFO [2019-12-17 12:28:29,475] jepsen worker 56 - jepsen.util 1046 :ok :read [18 17] INFO [2019-12-17 12:28:29,476] jepsen worker 56 - jepsen.util 1046 :invoke :cas [18 [33 44]] INFO [2019-12-17 12:28:29,479] jepsen worker 89 - jepsen.util 1169 :ok :write [29 12] INFO [2019-12-17 12:28:29,480] jepsen worker 89 - jepsen.util 1169 :invoke :write [29 27] INFO [2019-12-17 12:28:29,494] jepsen worker 28 - jepsen.util 748 :ok :write [9 11] INFO [2019-12-17 12:28:29,494] jepsen worker 28 - jepsen.util 748 :invoke :read [9 nil] INFO [2019-12-17 12:28:29,500] jepsen worker 70 - jepsen.util 1240 :ok :read [23 44] INFO [2019-12-17 12:28:29,500] jepsen worker 70 - jepsen.util 1240 :invoke :write [23 7] INFO [2019-12-17 12:28:29,522] jepsen worker 18 - jepsen.util 1098 :info :write [6 17] :RequestCanceledException INFO [2019-12-17 12:28:29,524] jepsen worker 18 - jepsen.util 1188 :invoke :write [6 30] INFO [2019-12-17 12:28:29,535] jepsen worker 31 - jepsen.util 1111 :ok :read [10 20] INFO [2019-12-17 12:28:29,535] jepsen worker 31 - jepsen.util 1111 :invoke :read [10 nil] INFO [2019-12-17 12:28:29,558] jepsen worker 28 - jepsen.util 748 :ok :read [9 11] INFO [2019-12-17 12:28:29,558] jepsen worker 28 - jepsen.util 748 :invoke :cas [9 [5 32]] INFO [2019-12-17 12:28:29,587] jepsen worker 88 - jepsen.util 988 :ok :read [29 12] INFO [2019-12-17 12:28:29,587] jepsen worker 88 - jepsen.util 988 :invoke :write [29 23] INFO [2019-12-17 12:28:29,623] jepsen worker 31 - jepsen.util 1111 :ok :read [10 20] INFO [2019-12-17 12:28:29,623] jepsen worker 31 - jepsen.util 1111 :invoke :write [10 29] INFO [2019-12-17 12:28:29,636] jepsen worker 7 - jepsen.util 1447 :info :write [2 38] :RequestCanceledException INFO [2019-12-17 12:28:29,642] jepsen worker 7 - jepsen.util 1537 :invoke :read [2 nil] INFO [2019-12-17 12:28:29,648] jepsen worker 78 - jepsen.util 1158 :ok :read [26 17] INFO [2019-12-17 12:28:29,649] jepsen worker 78 - jepsen.util 1158 :invoke :read [26 nil] INFO [2019-12-17 12:28:29,658] jepsen worker 10 - jepsen.util 1270 :ok :write [3 43] INFO [2019-12-17 12:28:29,659] jepsen worker 10 - jepsen.util 1270 :invoke :read [3 nil] INFO [2019-12-17 12:28:29,659] jepsen worker 19 - jepsen.util 1189 :ok :write [6 31] INFO [2019-12-17 12:28:29,660] jepsen worker 19 - jepsen.util 1189 :invoke :read [6 nil] INFO [2019-12-17 12:28:29,672] jepsen worker 28 - jepsen.util 748 :fail :cas [9 [5 32]] :ValueNotSwapFrom INFO [2019-12-17 12:28:29,672] jepsen worker 28 - jepsen.util 748 :invoke :read [9 nil] INFO [2019-12-17 12:28:29,694] jepsen worker 5 - jepsen.util 725 :ok :read [1 24] INFO [2019-12-17 12:28:29,694] jepsen worker 5 - jepsen.util 725 :invoke :read [1 nil] INFO [2019-12-17 12:28:29,716] jepsen worker 72 - jepsen.util 1062 :fail :cas [24 [35 26]] :ValueNotSwapFrom INFO [2019-12-17 12:28:29,716] jepsen worker 72 - jepsen.util 1062 :invoke :read [24 nil] INFO [2019-12-17 12:28:29,721] jepsen worker 61 - jepsen.util 1321 :ok :read [20 7] INFO [2019-12-17 12:28:29,721] jepsen worker 61 - jepsen.util 1321 :invoke :read [20 nil] INFO [2019-12-17 12:28:29,747] jepsen worker 19 - jepsen.util 1189 :ok :read [6 31] INFO [2019-12-17 12:28:29,747] jepsen worker 19 - jepsen.util 1189 :invoke :cas [6 [15 35]] INFO [2019-12-17 12:28:29,755] jepsen worker 10 - jepsen.util 1270 :ok :read [3 43] INFO [2019-12-17 12:28:29,755] jepsen worker 10 - jepsen.util 1270 :invoke :read [3 nil] INFO [2019-12-17 12:28:29,763] jepsen worker 38 - jepsen.util 938 :fail :cas [12 [16 17]] :ValueNotSwapFrom INFO [2019-12-17 12:28:29,763] jepsen worker 38 - jepsen.util 938 :invoke :read [12 nil] INFO [2019-12-17 12:28:29,798] jepsen worker 56 - jepsen.util 1046 :fail :cas [18 [33 44]] :ValueNotSwapFrom INFO [2019-12-17 12:28:29,799] jepsen worker 56 - jepsen.util 1046 :invoke :read [18 nil] INFO [2019-12-17 12:28:29,802] jepsen worker 12 - jepsen.util 1092 :ok :read [4 5] INFO [2019-12-17 12:28:29,802] jepsen worker 12 - jepsen.util 1092 :invoke :cas [4 [4 1]] INFO [2019-12-17 12:28:29,806] jepsen worker 35 - jepsen.util 935 :info :cas [11 [29 26]] :RequestCanceledException INFO [2019-12-17 12:28:29,806] jepsen worker 72 - jepsen.util 1062 :ok :read [24 31] INFO [2019-12-17 12:28:29,806] jepsen worker 72 - jepsen.util 1062 :invoke :cas [24 [6 8]] INFO [2019-12-17 12:28:29,808] jepsen worker 35 - jepsen.util 1025 :invoke :read [11 nil] INFO [2019-12-17 12:28:29,814] jepsen worker 19 - jepsen.util 1189 :fail :cas [6 [15 35]] :ValueNotSwapFrom INFO [2019-12-17 12:28:29,814] jepsen worker 19 - jepsen.util 1189 :invoke :read [6 nil] INFO [2019-12-17 12:28:29,821] jepsen worker 25 - jepsen.util 1375 :info :write [8 11] :RequestCanceledException INFO [2019-12-17 12:28:29,822] jepsen worker 10 - jepsen.util 1270 :ok :read [3 43] INFO [2019-12-17 12:28:29,822] jepsen worker 10 - jepsen.util 1270 :invoke :cas [3 [15 19]] INFO [2019-12-17 12:28:29,824] jepsen worker 25 - jepsen.util 1465 :invoke :write [8 31] INFO [2019-12-17 12:28:29,841] jepsen worker 72 - jepsen.util 1062 :fail :cas [24 [6 8]] :ValueNotSwapFrom INFO [2019-12-17 12:28:29,841] jepsen worker 72 - jepsen.util 1062 :invoke :read [24 nil] INFO [2019-12-17 12:28:29,860] jepsen worker 19 - jepsen.util 1189 :ok :read [6 31] INFO [2019-12-17 12:28:29,860] jepsen worker 19 - jepsen.util 1189 :invoke :cas [6 [20 4]] INFO [2019-12-17 12:28:29,870] jepsen worker 10 - jepsen.util 1270 :fail :cas [3 [15 19]] :ValueNotSwapFrom INFO [2019-12-17 12:28:29,870] jepsen worker 10 - jepsen.util 1270 :invoke :write [3 23] INFO [2019-12-17 12:28:29,879] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":5906,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:28:29,879] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":5906,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:28:29,879] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":5910,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:28:29,879] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":5910,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:28:29,884] jepsen worker 69 - jepsen.util 969 :ok :write [23 26] INFO [2019-12-17 12:28:29,884] jepsen worker 69 - jepsen.util 969 :invoke :cas [23 [4 0]] INFO [2019-12-17 12:28:29,887] jepsen worker 45 - jepsen.util 1215 :fail :read [15 nil] :RequestTimeoutException INFO [2019-12-17 12:28:29,888] jepsen worker 45 - jepsen.util 1215 :invoke :write [15 3] INFO [2019-12-17 12:28:29,904] jepsen worker 72 - jepsen.util 1062 :ok :read [24 31] INFO [2019-12-17 12:28:29,904] jepsen worker 72 - jepsen.util 1062 :invoke :read [24 nil] INFO [2019-12-17 12:28:29,910] jepsen worker 12 - jepsen.util 1092 :fail :cas [4 [4 1]] :ValueNotSwapFrom INFO [2019-12-17 12:28:29,910] jepsen worker 12 - jepsen.util 1092 :invoke :cas [4 [20 49]] INFO [2019-12-17 12:28:29,912] jepsen worker 19 - jepsen.util 1189 :fail :cas [6 [20 4]] :ValueNotSwapFrom INFO [2019-12-17 12:28:29,912] jepsen worker 19 - jepsen.util 1189 :invoke :cas [6 [10 0]] INFO [2019-12-17 12:28:29,921] jepsen worker 52 - jepsen.util 1042 :info :write [17 4] :RequestCanceledException INFO [2019-12-17 12:28:29,923] jepsen worker 52 - jepsen.util 1132 :invoke :read [17 nil] INFO [2019-12-17 12:28:29,942] jepsen worker 19 - jepsen.util 1189 :fail :cas [6 [10 0]] :ValueNotSwapFrom INFO [2019-12-17 12:28:29,942] jepsen worker 19 - jepsen.util 1189 :invoke :cas [6 [39 47]] INFO [2019-12-17 12:28:29,963] jepsen worker 19 - jepsen.util 1189 :fail :cas [6 [39 47]] :ValueNotSwapFrom INFO [2019-12-17 12:28:29,964] jepsen worker 19 - jepsen.util 1189 :invoke :read [6 nil] INFO [2019-12-17 12:28:29,981] jepsen worker 19 - jepsen.util 1189 :ok :read [6 31] INFO [2019-12-17 12:28:29,981] jepsen worker 19 - jepsen.util 1189 :invoke :cas [6 [29 3]] INFO [2019-12-17 12:28:30,004] jepsen worker 19 - jepsen.util 1189 :fail :cas [6 [29 3]] :ValueNotSwapFrom INFO [2019-12-17 12:28:30,005] jepsen worker 19 - jepsen.util 1189 :invoke :read [6 nil] INFO [2019-12-17 12:28:30,057] jepsen worker 19 - jepsen.util 1189 :ok :read [6 31] INFO [2019-12-17 12:28:30,058] jepsen worker 19 - jepsen.util 1189 :invoke :write [6 10] INFO [2019-12-17 12:28:30,059] jepsen worker 61 - jepsen.util 1321 :ok :read [20 7] INFO [2019-12-17 12:28:30,060] jepsen worker 61 - jepsen.util 1321 :invoke :cas [20 [3 41]] INFO [2019-12-17 12:28:30,096] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":5922,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:28:30,096] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":5922,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:28:30,096] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":5923,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:28:30,096] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":5923,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:28:30,106] jepsen worker 3 - jepsen.util 813 :info :write [1 41] :RequestCanceledException INFO [2019-12-17 12:28:30,108] jepsen worker 5 - jepsen.util 725 :ok :read [1 24] INFO [2019-12-17 12:28:30,108] jepsen worker 5 - jepsen.util 725 :invoke :write [1 23] INFO [2019-12-17 12:28:30,109] jepsen worker 3 - jepsen.util 903 :invoke :read [1 nil] INFO [2019-12-17 12:28:30,121] jepsen worker 36 - jepsen.util 1026 :info :write [12 48] :RequestCanceledException INFO [2019-12-17 12:28:30,123] jepsen worker 36 - jepsen.util 1116 :invoke :cas [12 [34 35]] INFO [2019-12-17 12:28:30,158] jepsen worker 61 - jepsen.util 1321 :fail :cas [20 [3 41]] :ValueNotSwapFrom INFO [2019-12-17 12:28:30,158] jepsen worker 61 - jepsen.util 1321 :invoke :cas [20 [26 1]] INFO [2019-12-17 12:28:30,184] jepsen worker 35 - jepsen.util 1025 :ok :read [11 7] INFO [2019-12-17 12:28:30,184] jepsen worker 35 - jepsen.util 1025 :invoke :read [11 nil] INFO [2019-12-17 12:28:30,188] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:28:30,188] jepsen worker 85 - jepsen.util 1165 :info :cas [28 [17 3]] :RequestCanceledException INFO [2019-12-17 12:28:30,188] jepsen nemesis - couchbase.util Rebalance check count: 38 INFO [2019-12-17 12:28:30,188] jepsen nemesis - couchbase.util Rebalance stuck count: 19 INFO [2019-12-17 12:28:30,191] jepsen worker 78 - jepsen.util 1158 :ok :read [26 17] INFO [2019-12-17 12:28:30,191] jepsen worker 78 - jepsen.util 1158 :invoke :cas [26 [5 48]] INFO [2019-12-17 12:28:30,191] jepsen worker 85 - jepsen.util 1255 :invoke :cas [28 [9 8]] INFO [2019-12-17 12:28:30,192] jepsen worker 56 - jepsen.util 1046 :ok :read [18 17] INFO [2019-12-17 12:28:30,192] jepsen worker 56 - jepsen.util 1046 :invoke :cas [18 [6 5]] INFO [2019-12-17 12:28:30,198] jepsen worker 36 - jepsen.util 1116 :fail :cas [12 [34 35]] :ValueNotSwapFrom INFO [2019-12-17 12:28:30,198] jepsen worker 36 - jepsen.util 1116 :invoke :write [12 17] INFO [2019-12-17 12:28:30,199] jepsen worker 71 - jepsen.util 791 :ok :read [23 26] INFO [2019-12-17 12:28:30,200] jepsen worker 71 - jepsen.util 791 :invoke :write [23 39] INFO [2019-12-17 12:28:30,206] jepsen worker 82 - jepsen.util 1702 :info :write [27 40] :RequestCanceledException INFO [2019-12-17 12:28:30,208] jepsen worker 82 - jepsen.util 1792 :invoke :read [27 nil] INFO [2019-12-17 12:28:30,208] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":5948,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:28:30,208] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":5948,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:28:30,208] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":5952,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:28:30,208] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":5952,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:28:30,209] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":5936,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:28:30,210] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":5936,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:28:30,210] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":5937,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:28:30,210] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":5937,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:28:30,218] jepsen worker 38 - jepsen.util 938 :ok :read [12 38] INFO [2019-12-17 12:28:30,218] jepsen worker 38 - jepsen.util 938 :invoke :read [12 nil] INFO [2019-12-17 12:28:30,222] jepsen worker 23 - jepsen.util 1103 :info :write [7 37] :RequestCanceledException INFO [2019-12-17 12:28:30,224] jepsen worker 23 - jepsen.util 1193 :invoke :write [7 13] INFO [2019-12-17 12:28:30,235] jepsen worker 0 - jepsen.util 1350 :info :write [0 1] :RequestCanceledException INFO [2019-12-17 12:28:30,237] jepsen worker 0 - jepsen.util 1440 :invoke :read [0 nil] INFO [2019-12-17 12:28:30,247] jepsen worker 78 - jepsen.util 1158 :fail :cas [26 [5 48]] :ValueNotSwapFrom INFO [2019-12-17 12:28:30,247] jepsen worker 78 - jepsen.util 1158 :invoke :cas [26 [4 7]] INFO [2019-12-17 12:28:30,249] jepsen worker 52 - jepsen.util 1132 :ok :read [17 21] INFO [2019-12-17 12:28:30,249] jepsen worker 52 - jepsen.util 1132 :invoke :write [17 9] INFO [2019-12-17 12:28:30,254] jepsen worker 57 - jepsen.util 1317 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:28:30,254] jepsen worker 59 - jepsen.util 1139 :info :cas [19 [36 6]] :RequestCanceledException INFO [2019-12-17 12:28:30,254] jepsen worker 57 - jepsen.util 1317 :invoke :write [19 25] INFO [2019-12-17 12:28:30,255] jepsen worker 59 - jepsen.util 1229 :invoke :read [19 nil] INFO [2019-12-17 12:28:30,285] jepsen worker 38 - jepsen.util 938 :ok :read [12 38] INFO [2019-12-17 12:28:30,286] jepsen worker 38 - jepsen.util 938 :invoke :read [12 nil] INFO [2019-12-17 12:28:30,288] jepsen worker 66 - jepsen.util 1056 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:28:30,288] jepsen worker 66 - jepsen.util 1056 :invoke :read [22 nil] INFO [2019-12-17 12:28:30,288] jepsen worker 68 - jepsen.util 968 :info :cas [22 [27 48]] :RequestCanceledException INFO [2019-12-17 12:28:30,290] jepsen worker 68 - jepsen.util 1058 :invoke :read [22 nil] INFO [2019-12-17 12:28:30,296] jepsen worker 78 - jepsen.util 1158 :fail :cas [26 [4 7]] :ValueNotSwapFrom INFO [2019-12-17 12:28:30,296] jepsen worker 78 - jepsen.util 1158 :invoke :cas [26 [29 11]] INFO [2019-12-17 12:28:30,296] jepsen worker 61 - jepsen.util 1321 :fail :cas [20 [26 1]] :ValueNotSwapFrom INFO [2019-12-17 12:28:30,296] jepsen worker 61 - jepsen.util 1321 :invoke :cas [20 [12 43]] INFO [2019-12-17 12:28:30,305] jepsen worker 50 - jepsen.util 1310 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:28:30,306] jepsen worker 50 - jepsen.util 1310 :invoke :write [16 6] INFO [2019-12-17 12:28:30,306] jepsen worker 17 - jepsen.util 1277 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:28:30,306] jepsen worker 60 - jepsen.util 1410 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:28:30,316] jepsen worker 60 - jepsen.util 1410 :invoke :write [20 3] INFO [2019-12-17 12:28:30,316] jepsen worker 17 - jepsen.util 1277 :invoke :cas [5 [2 39]] INFO [2019-12-17 12:28:30,322] jepsen worker 49 - jepsen.util 1129 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:28:30,322] jepsen worker 39 - jepsen.util 1389 :info :cas [13 [44 46]] :RequestCanceledException INFO [2019-12-17 12:28:30,322] jepsen worker 83 - jepsen.util 1523 :info :cas [27 [1 10]] :RequestCanceledException INFO [2019-12-17 12:28:30,322] jepsen worker 79 - jepsen.util 979 :info :write [26 37] :RequestCanceledException INFO [2019-12-17 12:28:30,322] jepsen worker 49 - jepsen.util 1129 :invoke :read [16 nil] INFO [2019-12-17 12:28:30,322] jepsen worker 54 - jepsen.util 1134 :info :write [18 49] :RequestCanceledException INFO [2019-12-17 12:28:30,322] jepsen worker 2 - jepsen.util 1082 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:28:30,322] jepsen worker 2 - jepsen.util 1082 :invoke :cas [0 [15 41]] INFO [2019-12-17 12:28:30,322] jepsen worker 67 - jepsen.util 1597 :info :cas [22 [24 8]] :RequestCanceledException INFO [2019-12-17 12:28:30,323] jepsen worker 38 - jepsen.util 938 :ok :read [12 38] INFO [2019-12-17 12:28:30,323] jepsen worker 38 - jepsen.util 938 :invoke :cas [12 [23 11]] INFO [2019-12-17 12:28:30,327] jepsen worker 83 - jepsen.util 1613 :invoke :cas [27 [14 7]] INFO [2019-12-17 12:28:30,327] jepsen worker 67 - jepsen.util 1687 :invoke :read [22 nil] INFO [2019-12-17 12:28:30,327] jepsen worker 79 - jepsen.util 1069 :invoke :read [26 nil] INFO [2019-12-17 12:28:30,328] jepsen worker 54 - jepsen.util 1224 :invoke :write [18 21] INFO [2019-12-17 12:28:30,328] jepsen worker 39 - jepsen.util 1479 :invoke :read [13 nil] INFO [2019-12-17 12:28:30,335] jepsen worker 40 - jepsen.util 1660 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:28:30,335] jepsen worker 16 - jepsen.util 1186 :info :write [5 19] :RequestCanceledException INFO [2019-12-17 12:28:30,336] jepsen worker 40 - jepsen.util 1660 :invoke :write [13 47] INFO [2019-12-17 12:28:30,340] jepsen worker 16 - jepsen.util 1276 :invoke :cas [5 [46 38]] INFO [2019-12-17 12:28:30,352] jepsen worker 48 - jepsen.util 678 :info :cas [16 [25 45]] :RequestCanceledException INFO [2019-12-17 12:28:30,352] jepsen worker 84 - jepsen.util 1074 :info :cas [28 [22 19]] :RequestCanceledException INFO [2019-12-17 12:28:30,356] jepsen worker 48 - jepsen.util 768 :invoke :cas [16 [29 0]] INFO [2019-12-17 12:28:30,357] jepsen worker 84 - jepsen.util 1164 :invoke :write [28 30] INFO [2019-12-17 12:28:30,389] jepsen worker 41 - jepsen.util 1031 :info :cas [13 [47 18]] :RequestCanceledException INFO [2019-12-17 12:28:30,389] jepsen worker 15 - jepsen.util 915 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:28:30,389] jepsen worker 15 - jepsen.util 915 :invoke :cas [5 [7 33]] INFO [2019-12-17 12:28:30,390] jepsen worker 86 - jepsen.util 1256 :info :cas [28 [12 37]] :RequestCanceledException INFO [2019-12-17 12:28:30,392] jepsen worker 41 - jepsen.util 1121 :invoke :read [13 nil] INFO [2019-12-17 12:28:30,392] jepsen worker 86 - jepsen.util 1346 :invoke :cas [28 [14 44]] INFO [2019-12-17 12:28:30,402] jepsen worker 78 - jepsen.util 1158 :fail :cas [26 [29 11]] :ValueNotSwapFrom INFO [2019-12-17 12:28:30,402] jepsen worker 78 - jepsen.util 1158 :invoke :write [26 27] INFO [2019-12-17 12:28:30,406] jepsen worker 75 - jepsen.util 1335 :info :write [25 29] :RequestCanceledException INFO [2019-12-17 12:28:30,409] jepsen worker 75 - jepsen.util 1425 :invoke :write [25 24] INFO [2019-12-17 12:28:30,422] jepsen worker 1 - jepsen.util 1081 :info :write [0 18] :RequestCanceledException INFO [2019-12-17 12:28:30,422] jepsen worker 1 - jepsen.util 1171 :invoke :cas [0 [43 18]] INFO [2019-12-17 12:28:30,423] jepsen worker 4 - jepsen.util 814 :info :write [1 26] :RequestCanceledException INFO [2019-12-17 12:28:30,425] jepsen worker 4 - jepsen.util 904 :invoke :read [1 nil] INFO [2019-12-17 12:28:30,427] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":5988,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:28:30,427] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":5988,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:28:30,427] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":5991,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:28:30,427] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":5991,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:28:30,427] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":5992,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:28:30,427] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":5992,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:28:30,435] jepsen worker 81 - jepsen.util 1431 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:28:30,435] jepsen worker 81 - jepsen.util 1431 :invoke :write [27 49] INFO [2019-12-17 12:28:30,448] jepsen worker 38 - jepsen.util 938 :fail :cas [12 [23 11]] :ValueNotSwapFrom INFO [2019-12-17 12:28:30,448] jepsen worker 38 - jepsen.util 938 :invoke :write [12 4] INFO [2019-12-17 12:28:30,450] jepsen worker 61 - jepsen.util 1321 :fail :cas [20 [12 43]] :ValueNotSwapFrom INFO [2019-12-17 12:28:30,450] jepsen worker 61 - jepsen.util 1321 :invoke :write [20 0] INFO [2019-12-17 12:28:30,452] jepsen worker 77 - jepsen.util 1247 :info :cas [25 [48 41]] :RequestCanceledException INFO [2019-12-17 12:28:30,452] jepsen worker 62 - jepsen.util 1232 :info :write [20 6] :RequestCanceledException INFO [2019-12-17 12:28:30,453] jepsen worker 62 - jepsen.util 1322 :invoke :read [20 nil] INFO [2019-12-17 12:28:30,454] jepsen worker 77 - jepsen.util 1337 :invoke :read [25 nil] INFO [2019-12-17 12:28:30,481] jepsen worker 35 - jepsen.util 1025 :ok :read [11 7] INFO [2019-12-17 12:28:30,482] jepsen worker 35 - jepsen.util 1025 :invoke :read [11 nil] INFO [2019-12-17 12:28:30,520] jepsen worker 42 - jepsen.util 1482 :info :write [14 21] :RequestCanceledException INFO [2019-12-17 12:28:30,521] jepsen worker 42 - jepsen.util 1572 :invoke :cas [14 [45 9]] INFO [2019-12-17 12:28:30,552] jepsen worker 13 - jepsen.util 1093 :fail :cas [4 [44 2]] :ValueNotSwapFrom INFO [2019-12-17 12:28:30,552] jepsen worker 13 - jepsen.util 1093 :invoke :write [4 5] INFO [2019-12-17 12:28:30,561] jepsen worker 12 - jepsen.util 1092 :fail :cas [4 [20 49]] :ValueNotSwapFrom INFO [2019-12-17 12:28:30,561] jepsen worker 12 - jepsen.util 1092 :invoke :cas [4 [3 40]] INFO [2019-12-17 12:28:30,578] jepsen worker 77 - jepsen.util 1337 :ok :read [25 2] INFO [2019-12-17 12:28:30,578] jepsen worker 77 - jepsen.util 1337 :invoke :read [25 nil] INFO [2019-12-17 12:28:30,588] jepsen worker 58 - jepsen.util 1228 :info :cas [19 [18 20]] :RequestCanceledException INFO [2019-12-17 12:28:30,589] jepsen worker 10 - jepsen.util 1270 :ok :write [3 23] INFO [2019-12-17 12:28:30,589] jepsen worker 10 - jepsen.util 1270 :invoke :read [3 nil] INFO [2019-12-17 12:28:30,591] jepsen worker 58 - jepsen.util 1318 :invoke :write [19 21] INFO [2019-12-17 12:28:30,609] jepsen worker 31 - jepsen.util 1111 :ok :write [10 29] INFO [2019-12-17 12:28:30,609] jepsen worker 31 - jepsen.util 1111 :invoke :write [10 36] INFO [2019-12-17 12:28:30,615] jepsen worker 46 - jepsen.util 1306 :ok :write [15 1] INFO [2019-12-17 12:28:30,616] jepsen worker 46 - jepsen.util 1306 :invoke :write [15 12] INFO [2019-12-17 12:28:30,620] jepsen worker 19 - jepsen.util 1189 :ok :write [6 10] INFO [2019-12-17 12:28:30,621] jepsen worker 19 - jepsen.util 1189 :invoke :write [6 2] INFO [2019-12-17 12:28:30,621] jepsen worker 20 - jepsen.util 1100 :fail :read [6 nil] :RequestTimeoutException INFO [2019-12-17 12:28:30,621] jepsen worker 87 - jepsen.util 987 :info :cas [29 [17 8]] :RequestCanceledException INFO [2019-12-17 12:28:30,621] jepsen worker 20 - jepsen.util 1100 :invoke :read [6 nil] INFO [2019-12-17 12:28:30,629] jepsen worker 87 - jepsen.util 1077 :invoke :cas [29 [2 20]] INFO [2019-12-17 12:28:30,641] jepsen worker 62 - jepsen.util 1322 :ok :read [20 7] INFO [2019-12-17 12:28:30,641] jepsen worker 62 - jepsen.util 1322 :invoke :read [20 nil] INFO [2019-12-17 12:28:30,645] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":6014,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:28:30,646] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":6014,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:28:30,649] jepsen worker 72 - jepsen.util 1062 :ok :read [24 31] INFO [2019-12-17 12:28:30,649] jepsen worker 72 - jepsen.util 1062 :invoke :write [24 15] INFO [2019-12-17 12:28:30,652] jepsen worker 11 - jepsen.util 1361 :info :write [3 31] :RequestCanceledException INFO [2019-12-17 12:28:30,653] jepsen worker 11 - jepsen.util 1451 :invoke :cas [3 [35 6]] INFO [2019-12-17 12:28:30,665] jepsen worker 10 - jepsen.util 1270 :ok :read [3 23] INFO [2019-12-17 12:28:30,666] jepsen worker 10 - jepsen.util 1270 :invoke :read [3 nil] INFO [2019-12-17 12:28:30,686] jepsen worker 28 - jepsen.util 748 :ok :read [9 11] INFO [2019-12-17 12:28:30,686] jepsen worker 28 - jepsen.util 748 :invoke :cas [9 [16 0]] INFO [2019-12-17 12:28:30,699] jepsen worker 20 - jepsen.util 1100 :ok :read [6 10] INFO [2019-12-17 12:28:30,699] jepsen worker 20 - jepsen.util 1100 :invoke :write [6 14] INFO [2019-12-17 12:28:30,703] jepsen worker 63 - jepsen.util 1053 :fail :cas [21 [27 7]] :ValueNotSwapFrom INFO [2019-12-17 12:28:30,703] jepsen worker 63 - jepsen.util 1053 :invoke :read [21 nil] INFO [2019-12-17 12:28:30,707] jepsen worker 11 - jepsen.util 1451 :fail :cas [3 [35 6]] :ValueNotSwapFrom INFO [2019-12-17 12:28:30,707] jepsen worker 11 - jepsen.util 1451 :invoke :write [3 46] INFO [2019-12-17 12:28:30,719] jepsen worker 30 - jepsen.util 1200 :info :cas [10 [47 6]] :RequestCanceledException INFO [2019-12-17 12:28:30,720] jepsen worker 10 - jepsen.util 1270 :ok :read [3 23] INFO [2019-12-17 12:28:30,720] jepsen worker 10 - jepsen.util 1270 :invoke :read [3 nil] INFO [2019-12-17 12:28:30,721] jepsen worker 30 - jepsen.util 1290 :invoke :read [10 nil] INFO [2019-12-17 12:28:30,751] jepsen worker 30 - jepsen.util 1290 :ok :read [10 29] INFO [2019-12-17 12:28:30,751] jepsen worker 30 - jepsen.util 1290 :invoke :cas [10 [0 7]] INFO [2019-12-17 12:28:30,753] jepsen worker 35 - jepsen.util 1025 :ok :read [11 7] INFO [2019-12-17 12:28:30,754] jepsen worker 35 - jepsen.util 1025 :invoke :write [11 14] INFO [2019-12-17 12:28:30,772] jepsen worker 30 - jepsen.util 1290 :fail :cas [10 [0 7]] :ValueNotSwapFrom INFO [2019-12-17 12:28:30,772] jepsen worker 30 - jepsen.util 1290 :invoke :read [10 nil] INFO [2019-12-17 12:28:30,794] jepsen worker 30 - jepsen.util 1290 :ok :read [10 29] INFO [2019-12-17 12:28:30,794] jepsen worker 30 - jepsen.util 1290 :invoke :read [10 nil] INFO [2019-12-17 12:28:30,818] jepsen worker 30 - jepsen.util 1290 :ok :read [10 29] INFO [2019-12-17 12:28:30,818] jepsen worker 30 - jepsen.util 1290 :invoke :write [10 46] INFO [2019-12-17 12:28:30,821] jepsen worker 87 - jepsen.util 1077 :fail :cas [29 [2 20]] :ValueNotSwapFrom INFO [2019-12-17 12:28:30,821] jepsen worker 87 - jepsen.util 1077 :invoke :write [29 8] INFO [2019-12-17 12:28:30,822] jepsen worker 27 - jepsen.util 837 :info :cas [9 [42 19]] :RequestCanceledException INFO [2019-12-17 12:28:30,824] jepsen worker 27 - jepsen.util 927 :invoke :read [9 nil] INFO [2019-12-17 12:28:30,848] jepsen worker 27 - jepsen.util 927 :ok :read [9 11] INFO [2019-12-17 12:28:30,848] jepsen worker 27 - jepsen.util 927 :invoke :write [9 41] INFO [2019-12-17 12:28:30,867] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":6039,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:28:30,867] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":6039,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:28:30,867] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":6040,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:28:30,867] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":6040,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:28:30,874] jepsen worker 77 - jepsen.util 1337 :ok :read [25 2] INFO [2019-12-17 12:28:30,874] jepsen worker 77 - jepsen.util 1337 :invoke :cas [25 [5 44]] INFO [2019-12-17 12:28:30,889] jepsen worker 79 - jepsen.util 1069 :ok :read [26 17] INFO [2019-12-17 12:28:30,889] jepsen worker 79 - jepsen.util 1069 :invoke :cas [26 [8 30]] INFO [2019-12-17 12:28:30,906] jepsen worker 47 - jepsen.util 1397 :info :cas [15 [24 48]] :RequestCanceledException INFO [2019-12-17 12:28:30,907] jepsen worker 14 - jepsen.util 914 :info :write [4 35] :RequestCanceledException INFO [2019-12-17 12:28:30,907] jepsen worker 14 - jepsen.util 1004 :invoke :cas [4 [16 48]] INFO [2019-12-17 12:28:30,909] jepsen worker 47 - jepsen.util 1487 :invoke :cas [15 [19 2]] INFO [2019-12-17 12:28:30,922] jepsen worker 55 - jepsen.util 1405 :info :cas [18 [12 41]] :RequestCanceledException INFO [2019-12-17 12:28:30,926] jepsen worker 55 - jepsen.util 1495 :invoke :write [18 1] INFO [2019-12-17 12:28:30,928] jepsen worker 27 - jepsen.util 927 :ok :write [9 41] INFO [2019-12-17 12:28:30,928] jepsen worker 27 - jepsen.util 927 :invoke :cas [9 [7 17]] INFO [2019-12-17 12:28:30,951] jepsen worker 27 - jepsen.util 927 :fail :cas [9 [7 17]] :ValueNotSwapFrom INFO [2019-12-17 12:28:30,951] jepsen worker 27 - jepsen.util 927 :invoke :write [9 2] INFO [2019-12-17 12:28:30,959] jepsen worker 69 - jepsen.util 969 :fail :cas [23 [4 0]] :ValueNotSwapFrom INFO [2019-12-17 12:28:30,959] jepsen worker 69 - jepsen.util 969 :invoke :cas [23 [18 32]] INFO [2019-12-17 12:28:30,986] jepsen worker 29 - jepsen.util 839 :info :cas [9 [5 7]] :RequestCanceledException INFO [2019-12-17 12:28:30,987] jepsen worker 29 - jepsen.util 929 :invoke :write [9 31] INFO [2019-12-17 12:28:30,989] jepsen worker 26 - jepsen.util 1376 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:28:30,989] jepsen worker 73 - jepsen.util 1063 :info :write [24 28] :RequestCanceledException INFO [2019-12-17 12:28:30,989] jepsen worker 26 - jepsen.util 1376 :invoke :read [8 nil] INFO [2019-12-17 12:28:30,991] jepsen worker 73 - jepsen.util 1153 :invoke :read [24 nil] INFO [2019-12-17 12:28:31,000] jepsen worker 27 - jepsen.util 927 :ok :write [9 2] INFO [2019-12-17 12:28:31,000] jepsen worker 27 - jepsen.util 927 :invoke :write [9 43] INFO [2019-12-17 12:28:31,018] jepsen worker 62 - jepsen.util 1322 :ok :read [20 7] INFO [2019-12-17 12:28:31,018] jepsen worker 62 - jepsen.util 1322 :invoke :read [20 nil] INFO [2019-12-17 12:28:31,050] jepsen worker 34 - jepsen.util 1384 :info :write [11 36] :RequestCanceledException INFO [2019-12-17 12:28:31,050] jepsen worker 8 - jepsen.util 1268 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:28:31,051] jepsen worker 8 - jepsen.util 1268 :invoke :write [2 25] INFO [2019-12-17 12:28:31,052] jepsen worker 34 - jepsen.util 1474 :invoke :read [11 nil] INFO [2019-12-17 12:28:31,065] jepsen worker 27 - jepsen.util 927 :ok :write [9 43] INFO [2019-12-17 12:28:31,066] jepsen worker 27 - jepsen.util 927 :invoke :write [9 48] INFO [2019-12-17 12:28:31,066] jepsen worker 4 - jepsen.util 904 :ok :read [1 24] INFO [2019-12-17 12:28:31,067] jepsen worker 4 - jepsen.util 904 :invoke :read [1 nil] INFO [2019-12-17 12:28:31,067] jepsen worker 62 - jepsen.util 1322 :ok :read [20 7] INFO [2019-12-17 12:28:31,067] jepsen worker 62 - jepsen.util 1322 :invoke :cas [20 [48 43]] INFO [2019-12-17 12:28:31,070] jepsen worker 3 - jepsen.util 903 :ok :read [1 24] INFO [2019-12-17 12:28:31,071] jepsen worker 3 - jepsen.util 903 :invoke :write [1 20] INFO [2019-12-17 12:28:31,084] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":6050,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:28:31,085] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":6050,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:28:31,085] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":6053,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:28:31,085] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":6053,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:28:31,085] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":6054,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:28:31,085] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":6054,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:28:31,110] jepsen worker 27 - jepsen.util 927 :ok :write [9 48] INFO [2019-12-17 12:28:31,111] jepsen worker 27 - jepsen.util 927 :invoke :read [9 nil] INFO [2019-12-17 12:28:31,126] jepsen worker 27 - jepsen.util 927 :ok :read [9 48] INFO [2019-12-17 12:28:31,127] jepsen worker 27 - jepsen.util 927 :invoke :cas [9 [7 8]] INFO [2019-12-17 12:28:31,152] jepsen worker 27 - jepsen.util 927 :fail :cas [9 [7 8]] :ValueNotSwapFrom INFO [2019-12-17 12:28:31,152] jepsen worker 27 - jepsen.util 927 :invoke :cas [9 [10 7]] INFO [2019-12-17 12:28:31,155] jepsen worker 69 - jepsen.util 969 :fail :cas [23 [18 32]] :ValueNotSwapFrom INFO [2019-12-17 12:28:31,155] jepsen worker 69 - jepsen.util 969 :invoke :write [23 44] INFO [2019-12-17 12:28:31,176] jepsen worker 27 - jepsen.util 927 :fail :cas [9 [10 7]] :ValueNotSwapFrom INFO [2019-12-17 12:28:31,176] jepsen worker 27 - jepsen.util 927 :invoke :cas [9 [19 6]] INFO [2019-12-17 12:28:31,183] jepsen worker 77 - jepsen.util 1337 :fail :cas [25 [5 44]] :ValueNotSwapFrom INFO [2019-12-17 12:28:31,183] jepsen worker 77 - jepsen.util 1337 :invoke :cas [25 [0 45]] INFO [2019-12-17 12:28:31,189] jepsen worker 6 - jepsen.util 1356 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:28:31,189] jepsen worker 6 - jepsen.util 1356 :invoke :cas [2 [5 37]] INFO [2019-12-17 12:28:31,191] jepsen worker 27 - jepsen.util 927 :fail :cas [9 [19 6]] :ValueNotSwapFrom INFO [2019-12-17 12:28:31,191] jepsen worker 27 - jepsen.util 927 :invoke :cas [9 [34 25]] INFO [2019-12-17 12:28:31,195] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":6073,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:28:31,195] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":6073,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:28:31,195] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":6074,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:28:31,195] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":6074,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:28:31,202] jepsen worker 12 - jepsen.util 1092 :fail :cas [4 [3 40]] :ValueNotSwapFrom INFO [2019-12-17 12:28:31,203] jepsen worker 12 - jepsen.util 1092 :invoke :write [4 0] INFO [2019-12-17 12:28:31,204] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:28:31,204] jepsen nemesis - couchbase.util Rebalance check count: 39 INFO [2019-12-17 12:28:31,205] jepsen nemesis - couchbase.util Rebalance stuck count: 20 INFO [2019-12-17 12:28:31,206] jepsen worker 76 - jepsen.util 1336 :info :cas [25 [10 31]] :RequestCanceledException INFO [2019-12-17 12:28:31,207] jepsen worker 76 - jepsen.util 1426 :invoke :read [25 nil] INFO [2019-12-17 12:28:31,213] jepsen worker 77 - jepsen.util 1337 :fail :cas [25 [0 45]] :ValueNotSwapFrom INFO [2019-12-17 12:28:31,213] jepsen worker 77 - jepsen.util 1337 :invoke :cas [25 [26 44]] INFO [2019-12-17 12:28:31,214] jepsen worker 27 - jepsen.util 927 :fail :cas [9 [34 25]] :ValueNotSwapFrom INFO [2019-12-17 12:28:31,214] jepsen worker 27 - jepsen.util 927 :invoke :write [9 32] INFO [2019-12-17 12:28:31,220] jepsen worker 37 - jepsen.util 937 :fail :read [12 nil] :RequestTimeoutException INFO [2019-12-17 12:28:31,220] jepsen worker 24 - jepsen.util 1194 :info :write [8 0] :RequestCanceledException INFO [2019-12-17 12:28:31,221] jepsen worker 37 - jepsen.util 937 :invoke :cas [12 [30 8]] INFO [2019-12-17 12:28:31,222] jepsen worker 24 - jepsen.util 1284 :invoke :cas [8 [13 15]] INFO [2019-12-17 12:28:31,236] jepsen worker 51 - jepsen.util 1041 :info :write [17 4] :RequestCanceledException INFO [2019-12-17 12:28:31,238] jepsen worker 51 - jepsen.util 1131 :invoke :read [17 nil] INFO [2019-12-17 12:28:31,245] jepsen worker 77 - jepsen.util 1337 :fail :cas [25 [26 44]] :ValueNotSwapFrom INFO [2019-12-17 12:28:31,245] jepsen worker 77 - jepsen.util 1337 :invoke :read [25 nil] INFO [2019-12-17 12:28:31,253] jepsen worker 9 - jepsen.util 1089 :info :write [3 19] :RequestCanceledException INFO [2019-12-17 12:28:31,254] jepsen worker 9 - jepsen.util 1179 :invoke :read [3 nil] INFO [2019-12-17 12:28:31,260] jepsen worker 27 - jepsen.util 927 :ok :write [9 32] INFO [2019-12-17 12:28:31,260] jepsen worker 27 - jepsen.util 927 :invoke :write [9 18] INFO [2019-12-17 12:28:31,263] jepsen worker 79 - jepsen.util 1069 :fail :cas [26 [8 30]] :ValueNotSwapFrom INFO [2019-12-17 12:28:31,263] jepsen worker 79 - jepsen.util 1069 :invoke :cas [26 [24 30]] INFO [2019-12-17 12:28:31,288] jepsen worker 44 - jepsen.util 1214 :info :cas [14 [3 16]] :RequestCanceledException INFO [2019-12-17 12:28:31,290] jepsen worker 44 - jepsen.util 1304 :invoke :read [14 nil] INFO [2019-12-17 12:28:31,304] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":6081,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:28:31,305] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":6081,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:28:31,305] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":6085,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:28:31,305] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":6085,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:28:31,319] jepsen worker 34 - jepsen.util 1474 :ok :read [11 7] INFO [2019-12-17 12:28:31,319] jepsen worker 34 - jepsen.util 1474 :invoke :write [11 40] INFO [2019-12-17 12:28:31,320] jepsen worker 44 - jepsen.util 1304 :ok :read [14 27] INFO [2019-12-17 12:28:31,320] jepsen worker 44 - jepsen.util 1304 :invoke :read [14 nil] INFO [2019-12-17 12:28:31,321] jepsen worker 65 - jepsen.util 875 :info :write [21 8] :RequestCanceledException INFO [2019-12-17 12:28:31,324] jepsen worker 27 - jepsen.util 927 :ok :write [9 18] INFO [2019-12-17 12:28:31,324] jepsen worker 27 - jepsen.util 927 :invoke :read [9 nil] INFO [2019-12-17 12:28:31,326] jepsen worker 65 - jepsen.util 965 :invoke :read [21 nil] INFO [2019-12-17 12:28:31,335] jepsen worker 74 - jepsen.util 884 :info :cas [24 [31 32]] :RequestCanceledException INFO [2019-12-17 12:28:31,335] jepsen worker 64 - jepsen.util 604 :info :cas [21 [21 5]] :RequestCanceledException INFO [2019-12-17 12:28:31,337] jepsen worker 74 - jepsen.util 974 :invoke :write [24 49] INFO [2019-12-17 12:28:31,338] jepsen worker 64 - jepsen.util 694 :invoke :read [21 nil] INFO [2019-12-17 12:28:31,353] jepsen worker 44 - jepsen.util 1304 :ok :read [14 27] INFO [2019-12-17 12:28:31,353] jepsen worker 44 - jepsen.util 1304 :invoke :write [14 2] INFO [2019-12-17 12:28:31,435] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":6106,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:28:31,435] jepsen worker 21 - jepsen.util 1191 :info :write [7 47] :RequestCanceledException INFO [2019-12-17 12:28:31,435] jepsen worker 33 - jepsen.util 1563 :info :write [11 46] :RequestCanceledException INFO [2019-12-17 12:28:31,435] jepsen worker 22 - jepsen.util 1282 :info :write [7 35] :RequestCanceledException INFO [2019-12-17 12:28:31,435] jepsen worker 53 - jepsen.util 1043 :info :write [17 22] :RequestCanceledException INFO [2019-12-17 12:28:31,435] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":6106,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:28:31,436] jepsen worker 22 - jepsen.util 1372 :invoke :cas [7 [11 14]] INFO [2019-12-17 12:28:31,436] jepsen worker 21 - jepsen.util 1281 :invoke :cas [7 [12 44]] INFO [2019-12-17 12:28:31,436] jepsen worker 53 - jepsen.util 1133 :invoke :cas [17 [1 40]] INFO [2019-12-17 12:28:31,436] jepsen worker 33 - jepsen.util 1653 :invoke :cas [11 [49 12]] INFO [2019-12-17 12:28:31,438] jepsen worker 65 - jepsen.util 965 :ok :read [21 17] INFO [2019-12-17 12:28:31,438] jepsen worker 65 - jepsen.util 965 :invoke :read [21 nil] INFO [2019-12-17 12:28:31,439] jepsen worker 27 - jepsen.util 927 :ok :read [9 18] INFO [2019-12-17 12:28:31,440] jepsen worker 27 - jepsen.util 927 :invoke :cas [9 [12 47]] INFO [2019-12-17 12:28:31,441] jepsen worker 62 - jepsen.util 1322 :fail :cas [20 [48 43]] :ValueNotSwapFrom INFO [2019-12-17 12:28:31,441] jepsen worker 62 - jepsen.util 1322 :invoke :cas [20 [6 5]] INFO [2019-12-17 12:28:31,452] jepsen worker 80 - jepsen.util 1250 :info :write [26 27] :RequestCanceledException INFO [2019-12-17 12:28:31,452] jepsen worker 32 - jepsen.util 1112 :info :write [10 2] :RequestCanceledException INFO [2019-12-17 12:28:31,453] jepsen worker 80 - jepsen.util 1340 :invoke :write [26 37] INFO [2019-12-17 12:28:31,453] jepsen worker 32 - jepsen.util 1202 :invoke :write [10 12] INFO [2019-12-17 12:28:31,460] jepsen worker 65 - jepsen.util 965 :ok :read [21 17] INFO [2019-12-17 12:28:31,461] jepsen worker 65 - jepsen.util 965 :invoke :write [21 47] INFO [2019-12-17 12:28:31,467] jepsen worker 27 - jepsen.util 927 :fail :cas [9 [12 47]] :ValueNotSwapFrom INFO [2019-12-17 12:28:31,467] jepsen worker 27 - jepsen.util 927 :invoke :write [9 15] INFO [2019-12-17 12:28:31,498] jepsen worker 4 - jepsen.util 904 :ok :read [1 24] INFO [2019-12-17 12:28:31,498] jepsen worker 4 - jepsen.util 904 :invoke :write [1 15] INFO [2019-12-17 12:28:31,508] jepsen worker 65 - jepsen.util 965 :ok :write [21 47] INFO [2019-12-17 12:28:31,508] jepsen worker 65 - jepsen.util 965 :invoke :read [21 nil] INFO [2019-12-17 12:28:31,510] jepsen worker 45 - jepsen.util 1215 :ok :write [15 3] INFO [2019-12-17 12:28:31,510] jepsen worker 45 - jepsen.util 1215 :invoke :write [15 25] INFO [2019-12-17 12:28:31,511] jepsen worker 19 - jepsen.util 1189 :ok :write [6 2] INFO [2019-12-17 12:28:31,511] jepsen worker 19 - jepsen.util 1189 :invoke :cas [6 [38 20]] INFO [2019-12-17 12:28:31,520] jepsen worker 30 - jepsen.util 1290 :ok :write [10 46] INFO [2019-12-17 12:28:31,520] jepsen worker 30 - jepsen.util 1290 :invoke :read [10 nil] INFO [2019-12-17 12:28:31,521] jepsen worker 43 - jepsen.util 1393 :info :write [14 47] :RequestCanceledException INFO [2019-12-17 12:28:31,523] jepsen worker 43 - jepsen.util 1483 :invoke :write [14 46] INFO [2019-12-17 12:28:31,536] jepsen worker 89 - jepsen.util 1169 :info :write [29 27] :RequestCanceledException INFO [2019-12-17 12:28:31,537] jepsen worker 89 - jepsen.util 1259 :invoke :write [29 0] INFO [2019-12-17 12:28:31,542] jepsen worker 72 - jepsen.util 1062 :ok :write [24 15] INFO [2019-12-17 12:28:31,542] jepsen worker 72 - jepsen.util 1062 :invoke :cas [24 [8 13]] INFO [2019-12-17 12:28:31,547] jepsen worker 79 - jepsen.util 1069 :fail :cas [26 [24 30]] :ValueNotSwapFrom INFO [2019-12-17 12:28:31,547] jepsen worker 79 - jepsen.util 1069 :invoke :write [26 8] INFO [2019-12-17 12:28:31,550] jepsen worker 33 - jepsen.util 1653 :fail :cas [11 [49 12]] :ValueNotSwapFrom INFO [2019-12-17 12:28:31,550] jepsen worker 33 - jepsen.util 1653 :invoke :cas [11 [6 29]] INFO [2019-12-17 12:28:31,559] jepsen worker 24 - jepsen.util 1284 :fail :cas [8 [13 15]] :ValueNotSwapFrom INFO [2019-12-17 12:28:31,559] jepsen worker 24 - jepsen.util 1284 :invoke :cas [8 [36 28]] INFO [2019-12-17 12:28:31,559] jepsen worker 73 - jepsen.util 1153 :ok :read [24 15] INFO [2019-12-17 12:28:31,559] jepsen worker 73 - jepsen.util 1153 :invoke :write [24 13] INFO [2019-12-17 12:28:31,588] jepsen worker 18 - jepsen.util 1188 :info :write [6 30] :RequestCanceledException INFO [2019-12-17 12:28:31,589] jepsen worker 18 - jepsen.util 1278 :invoke :read [6 nil] INFO [2019-12-17 12:28:31,590] jepsen worker 47 - jepsen.util 1487 :fail :cas [15 [19 2]] :ValueNotSwapFrom INFO [2019-12-17 12:28:31,590] jepsen worker 47 - jepsen.util 1487 :invoke :read [15 nil] INFO [2019-12-17 12:28:31,603] jepsen worker 30 - jepsen.util 1290 :ok :read [10 46] INFO [2019-12-17 12:28:31,603] jepsen worker 30 - jepsen.util 1290 :invoke :cas [10 [18 20]] INFO [2019-12-17 12:28:31,606] jepsen worker 88 - jepsen.util 988 :info :write [29 23] :RequestCanceledException INFO [2019-12-17 12:28:31,606] jepsen worker 70 - jepsen.util 1240 :info :write [23 7] :RequestCanceledException INFO [2019-12-17 12:28:31,607] jepsen worker 88 - jepsen.util 1078 :invoke :cas [29 [43 12]] INFO [2019-12-17 12:28:31,607] jepsen worker 70 - jepsen.util 1330 :invoke :write [23 38] INFO [2019-12-17 12:28:31,614] jepsen worker 62 - jepsen.util 1322 :fail :cas [20 [6 5]] :ValueNotSwapFrom INFO [2019-12-17 12:28:31,614] jepsen worker 62 - jepsen.util 1322 :invoke :cas [20 [20 5]] INFO [2019-12-17 12:28:31,654] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":6142,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:28:31,654] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":6142,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:28:31,654] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":6145,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:28:31,654] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":6145,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:28:31,661] jepsen worker 72 - jepsen.util 1062 :fail :cas [24 [8 13]] :ValueNotSwapFrom INFO [2019-12-17 12:28:31,661] jepsen worker 72 - jepsen.util 1062 :invoke :write [24 37] INFO [2019-12-17 12:28:31,662] jepsen worker 37 - jepsen.util 937 :fail :cas [12 [30 8]] :ValueNotSwapFrom INFO [2019-12-17 12:28:31,662] jepsen worker 37 - jepsen.util 937 :invoke :write [12 45] INFO [2019-12-17 12:28:31,670] jepsen worker 64 - jepsen.util 694 :ok :read [21 47] INFO [2019-12-17 12:28:31,670] jepsen worker 64 - jepsen.util 694 :invoke :write [21 18] INFO [2019-12-17 12:28:31,674] jepsen worker 88 - jepsen.util 1078 :fail :cas [29 [43 12]] :ValueNotSwapFrom INFO [2019-12-17 12:28:31,675] jepsen worker 88 - jepsen.util 1078 :invoke :write [29 21] INFO [2019-12-17 12:28:31,705] jepsen worker 7 - jepsen.util 1537 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:28:31,706] jepsen worker 7 - jepsen.util 1537 :invoke :write [2 40] INFO [2019-12-17 12:28:31,712] jepsen worker 27 - jepsen.util 927 :ok :write [9 15] INFO [2019-12-17 12:28:31,712] jepsen worker 27 - jepsen.util 927 :invoke :read [9 nil] INFO [2019-12-17 12:28:31,723] jepsen worker 24 - jepsen.util 1284 :fail :cas [8 [36 28]] :ValueNotSwapFrom INFO [2019-12-17 12:28:31,723] jepsen worker 24 - jepsen.util 1284 :invoke :write [8 26] INFO [2019-12-17 12:28:31,736] jepsen worker 27 - jepsen.util 927 :ok :read [9 15] INFO [2019-12-17 12:28:31,737] jepsen worker 27 - jepsen.util 927 :invoke :write [9 27] INFO [2019-12-17 12:28:31,739] jepsen worker 45 - jepsen.util 1215 :ok :write [15 25] INFO [2019-12-17 12:28:31,739] jepsen worker 45 - jepsen.util 1215 :invoke :read [15 nil] INFO [2019-12-17 12:28:31,752] jepsen worker 62 - jepsen.util 1322 :fail :cas [20 [20 5]] :ValueNotSwapFrom INFO [2019-12-17 12:28:31,752] jepsen worker 62 - jepsen.util 1322 :invoke :read [20 nil] INFO [2019-12-17 12:28:31,769] jepsen worker 45 - jepsen.util 1215 :ok :read [15 25] INFO [2019-12-17 12:28:31,769] jepsen worker 45 - jepsen.util 1215 :invoke :cas [15 [7 11]] INFO [2019-12-17 12:28:31,786] jepsen worker 45 - jepsen.util 1215 :fail :cas [15 [7 11]] :ValueNotSwapFrom INFO [2019-12-17 12:28:31,786] jepsen worker 45 - jepsen.util 1215 :invoke :read [15 nil] INFO [2019-12-17 12:28:31,810] jepsen worker 45 - jepsen.util 1215 :ok :read [15 25] INFO [2019-12-17 12:28:31,810] jepsen worker 45 - jepsen.util 1215 :invoke :read [15 nil] INFO [2019-12-17 12:28:31,810] jepsen worker 27 - jepsen.util 927 :ok :write [9 27] INFO [2019-12-17 12:28:31,810] jepsen worker 27 - jepsen.util 927 :invoke :write [9 34] INFO [2019-12-17 12:28:31,829] jepsen worker 45 - jepsen.util 1215 :ok :read [15 25] INFO [2019-12-17 12:28:31,830] jepsen worker 45 - jepsen.util 1215 :invoke :cas [15 [10 21]] INFO [2019-12-17 12:28:31,843] jepsen worker 62 - jepsen.util 1322 :ok :read [20 7] INFO [2019-12-17 12:28:31,843] jepsen worker 62 - jepsen.util 1322 :invoke :write [20 38] INFO [2019-12-17 12:28:31,852] jepsen worker 45 - jepsen.util 1215 :fail :cas [15 [10 21]] :ValueNotSwapFrom INFO [2019-12-17 12:28:31,852] jepsen worker 45 - jepsen.util 1215 :invoke :cas [15 [41 13]] INFO [2019-12-17 12:28:31,871] jepsen worker 45 - jepsen.util 1215 :fail :cas [15 [41 13]] :ValueNotSwapFrom INFO [2019-12-17 12:28:31,871] jepsen worker 45 - jepsen.util 1215 :invoke :cas [15 [3 47]] INFO [2019-12-17 12:28:31,872] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":6164,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:28:31,872] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":6164,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:28:31,872] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":6167,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:28:31,872] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":6167,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:28:31,872] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":6168,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:28:31,872] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":6168,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:28:31,891] jepsen worker 45 - jepsen.util 1215 :fail :cas [15 [3 47]] :ValueNotSwapFrom INFO [2019-12-17 12:28:31,891] jepsen worker 45 - jepsen.util 1215 :invoke :cas [15 [3 22]] INFO [2019-12-17 12:28:31,897] jepsen worker 27 - jepsen.util 927 :ok :write [9 34] INFO [2019-12-17 12:28:31,897] jepsen worker 27 - jepsen.util 927 :invoke :cas [9 [8 27]] INFO [2019-12-17 12:28:31,913] jepsen worker 45 - jepsen.util 1215 :fail :cas [15 [3 22]] :ValueNotSwapFrom INFO [2019-12-17 12:28:31,913] jepsen worker 45 - jepsen.util 1215 :invoke :cas [15 [30 11]] INFO [2019-12-17 12:28:31,917] jepsen worker 27 - jepsen.util 927 :fail :cas [9 [8 27]] :ValueNotSwapFrom INFO [2019-12-17 12:28:31,917] jepsen worker 27 - jepsen.util 927 :invoke :write [9 13] INFO [2019-12-17 12:28:31,935] jepsen worker 25 - jepsen.util 1465 :info :write [8 31] :RequestCanceledException INFO [2019-12-17 12:28:31,936] jepsen worker 25 - jepsen.util 1555 :invoke :read [8 nil] INFO [2019-12-17 12:28:31,938] jepsen worker 45 - jepsen.util 1215 :fail :cas [15 [30 11]] :ValueNotSwapFrom INFO [2019-12-17 12:28:31,938] jepsen worker 45 - jepsen.util 1215 :invoke :read [15 nil] INFO [2019-12-17 12:28:31,952] jepsen worker 73 - jepsen.util 1153 :ok :write [24 13] INFO [2019-12-17 12:28:31,952] jepsen worker 73 - jepsen.util 1153 :invoke :cas [24 [37 12]] INFO [2019-12-17 12:28:31,953] jepsen worker 45 - jepsen.util 1215 :ok :read [15 25] INFO [2019-12-17 12:28:31,954] jepsen worker 45 - jepsen.util 1215 :invoke :read [15 nil] INFO [2019-12-17 12:28:31,957] jepsen worker 32 - jepsen.util 1202 :ok :write [10 12] INFO [2019-12-17 12:28:31,957] jepsen worker 32 - jepsen.util 1202 :invoke :read [10 nil] INFO [2019-12-17 12:28:31,993] jepsen worker 45 - jepsen.util 1215 :ok :read [15 25] INFO [2019-12-17 12:28:31,993] jepsen worker 45 - jepsen.util 1215 :invoke :read [15 nil] INFO [2019-12-17 12:28:31,994] jepsen worker 73 - jepsen.util 1153 :fail :cas [24 [37 12]] :ValueNotSwapFrom INFO [2019-12-17 12:28:31,995] jepsen worker 73 - jepsen.util 1153 :invoke :write [24 47] INFO [2019-12-17 12:28:32,000] jepsen worker 47 - jepsen.util 1487 :ok :read [15 25] INFO [2019-12-17 12:28:32,000] jepsen worker 47 - jepsen.util 1487 :invoke :write [15 2] INFO [2019-12-17 12:28:32,023] jepsen worker 45 - jepsen.util 1215 :ok :read [15 25] INFO [2019-12-17 12:28:32,024] jepsen worker 45 - jepsen.util 1215 :invoke :write [15 4] INFO [2019-12-17 12:28:32,034] jepsen worker 27 - jepsen.util 927 :ok :write [9 13] INFO [2019-12-17 12:28:32,034] jepsen worker 27 - jepsen.util 927 :invoke :read [9 nil] INFO [2019-12-17 12:28:32,053] jepsen worker 27 - jepsen.util 927 :ok :read [9 13] INFO [2019-12-17 12:28:32,053] jepsen worker 27 - jepsen.util 927 :invoke :cas [9 [3 18]] INFO [2019-12-17 12:28:32,080] jepsen worker 27 - jepsen.util 927 :fail :cas [9 [3 18]] :ValueNotSwapFrom INFO [2019-12-17 12:28:32,080] jepsen worker 27 - jepsen.util 927 :invoke :cas [9 [11 26]] INFO [2019-12-17 12:28:32,090] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":6181,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:28:32,090] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":6181,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:28:32,102] jepsen worker 77 - jepsen.util 1337 :ok :read [25 2] INFO [2019-12-17 12:28:32,102] jepsen worker 77 - jepsen.util 1337 :invoke :cas [25 [12 8]] INFO [2019-12-17 12:28:32,107] jepsen worker 64 - jepsen.util 694 :ok :write [21 18] INFO [2019-12-17 12:28:32,107] jepsen worker 64 - jepsen.util 694 :invoke :write [21 9] INFO [2019-12-17 12:28:32,116] jepsen worker 27 - jepsen.util 927 :fail :cas [9 [11 26]] :ValueNotSwapFrom INFO [2019-12-17 12:28:32,116] jepsen worker 27 - jepsen.util 927 :invoke :read [9 nil] INFO [2019-12-17 12:28:32,153] jepsen worker 27 - jepsen.util 927 :ok :read [9 13] INFO [2019-12-17 12:28:32,153] jepsen worker 27 - jepsen.util 927 :invoke :write [9 21] INFO [2019-12-17 12:28:32,153] jepsen worker 5 - jepsen.util 725 :info :write [1 23] :RequestCanceledException INFO [2019-12-17 12:28:32,154] jepsen worker 5 - jepsen.util 815 :invoke :read [1 nil] INFO [2019-12-17 12:28:32,172] jepsen worker 51 - jepsen.util 1131 :ok :read [17 21] INFO [2019-12-17 12:28:32,173] jepsen worker 51 - jepsen.util 1131 :invoke :read [17 nil] INFO [2019-12-17 12:28:32,198] jepsen worker 53 - jepsen.util 1133 :fail :cas [17 [1 40]] :ValueNotSwapFrom INFO [2019-12-17 12:28:32,198] jepsen worker 53 - jepsen.util 1133 :invoke :cas [17 [37 46]] INFO [2019-12-17 12:28:32,199] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":6198,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:28:32,199] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":6198,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:28:32,199] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":6200,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:28:32,199] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":6200,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:28:32,204] jepsen worker 64 - jepsen.util 694 :ok :write [21 9] INFO [2019-12-17 12:28:32,205] jepsen worker 85 - jepsen.util 1255 :info :cas [28 [9 8]] :RequestCanceledException INFO [2019-12-17 12:28:32,208] jepsen worker 64 - jepsen.util 694 :invoke :read [21 nil] INFO [2019-12-17 12:28:32,209] jepsen worker 85 - jepsen.util 1345 :invoke :write [28 30] INFO [2019-12-17 12:28:32,220] jepsen worker 71 - jepsen.util 791 :ok :write [23 39] INFO [2019-12-17 12:28:32,220] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:28:32,220] jepsen nemesis - couchbase.util Rebalance check count: 40 INFO [2019-12-17 12:28:32,220] jepsen nemesis - couchbase.util Rebalance stuck count: 21 INFO [2019-12-17 12:28:32,220] jepsen worker 71 - jepsen.util 791 :invoke :cas [23 [33 45]] INFO [2019-12-17 12:28:32,221] jepsen worker 82 - jepsen.util 1792 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:28:32,221] jepsen worker 82 - jepsen.util 1792 :invoke :cas [27 [19 33]] INFO [2019-12-17 12:28:32,239] jepsen worker 64 - jepsen.util 694 :ok :read [21 9] INFO [2019-12-17 12:28:32,239] jepsen worker 64 - jepsen.util 694 :invoke :cas [21 [13 19]] INFO [2019-12-17 12:28:32,245] jepsen worker 25 - jepsen.util 1555 :ok :read [8 14] INFO [2019-12-17 12:28:32,245] jepsen worker 25 - jepsen.util 1555 :invoke :cas [8 [14 20]] INFO [2019-12-17 12:28:32,248] jepsen worker 27 - jepsen.util 927 :ok :write [9 21] INFO [2019-12-17 12:28:32,248] jepsen worker 27 - jepsen.util 927 :invoke :write [9 13] INFO [2019-12-17 12:28:32,253] jepsen worker 0 - jepsen.util 1440 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:28:32,253] jepsen worker 56 - jepsen.util 1046 :info :cas [18 [6 5]] :RequestCanceledException INFO [2019-12-17 12:28:32,253] jepsen worker 0 - jepsen.util 1440 :invoke :write [0 25] INFO [2019-12-17 12:28:32,254] jepsen worker 56 - jepsen.util 1136 :invoke :write [18 11] INFO [2019-12-17 12:28:32,263] jepsen worker 64 - jepsen.util 694 :fail :cas [21 [13 19]] :ValueNotSwapFrom INFO [2019-12-17 12:28:32,264] jepsen worker 64 - jepsen.util 694 :invoke :read [21 nil] INFO [2019-12-17 12:28:32,288] jepsen worker 36 - jepsen.util 1116 :info :write [12 17] :RequestCanceledException INFO [2019-12-17 12:28:32,289] jepsen worker 36 - jepsen.util 1206 :invoke :cas [12 [38 36]] INFO [2019-12-17 12:28:32,293] jepsen worker 73 - jepsen.util 1153 :ok :write [24 47] INFO [2019-12-17 12:28:32,293] jepsen worker 73 - jepsen.util 1153 :invoke :write [24 29] INFO [2019-12-17 12:28:32,306] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":6208,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:28:32,306] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":6208,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:28:32,306] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":6209,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:28:32,306] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":6209,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:28:32,306] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":6210,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:28:32,306] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":6210,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:28:32,312] jepsen worker 45 - jepsen.util 1215 :ok :write [15 4] INFO [2019-12-17 12:28:32,312] jepsen worker 45 - jepsen.util 1215 :invoke :write [15 18] INFO [2019-12-17 12:28:32,321] jepsen worker 59 - jepsen.util 1229 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:28:32,322] jepsen worker 59 - jepsen.util 1229 :invoke :cas [19 [38 49]] INFO [2019-12-17 12:28:32,325] jepsen worker 64 - jepsen.util 694 :ok :read [21 9] INFO [2019-12-17 12:28:32,325] jepsen worker 64 - jepsen.util 694 :invoke :read [21 nil] INFO [2019-12-17 12:28:32,335] jepsen worker 23 - jepsen.util 1193 :info :write [7 13] :RequestCanceledException INFO [2019-12-17 12:28:32,336] jepsen worker 23 - jepsen.util 1283 :invoke :cas [7 [5 12]] INFO [2019-12-17 12:28:32,346] jepsen worker 27 - jepsen.util 927 :ok :write [9 13] INFO [2019-12-17 12:28:32,346] jepsen worker 27 - jepsen.util 927 :invoke :read [9 nil] INFO [2019-12-17 12:28:32,351] jepsen worker 33 - jepsen.util 1653 :fail :cas [11 [6 29]] :ValueNotSwapFrom INFO [2019-12-17 12:28:32,351] jepsen worker 53 - jepsen.util 1133 :fail :cas [17 [37 46]] :ValueNotSwapFrom INFO [2019-12-17 12:28:32,351] jepsen worker 33 - jepsen.util 1653 :invoke :cas [11 [19 10]] INFO [2019-12-17 12:28:32,351] jepsen worker 53 - jepsen.util 1133 :invoke :read [17 nil] INFO [2019-12-17 12:28:32,352] jepsen worker 52 - jepsen.util 1132 :info :write [17 9] :RequestCanceledException INFO [2019-12-17 12:28:32,352] jepsen worker 57 - jepsen.util 1317 :info :write [19 25] :RequestCanceledException INFO [2019-12-17 12:28:32,352] jepsen worker 54 - jepsen.util 1224 :info :write [18 21] :RequestCanceledException INFO [2019-12-17 12:28:32,353] jepsen worker 57 - jepsen.util 1407 :invoke :read [19 nil] INFO [2019-12-17 12:28:32,353] jepsen worker 52 - jepsen.util 1222 :invoke :write [17 15] INFO [2019-12-17 12:28:32,353] jepsen worker 54 - jepsen.util 1314 :invoke :cas [18 [44 23]] INFO [2019-12-17 12:28:32,355] jepsen worker 64 - jepsen.util 694 :ok :read [21 9] INFO [2019-12-17 12:28:32,355] jepsen worker 64 - jepsen.util 694 :invoke :cas [21 [48 9]] INFO [2019-12-17 12:28:32,361] jepsen worker 27 - jepsen.util 927 :ok :read [9 13] INFO [2019-12-17 12:28:32,362] jepsen worker 27 - jepsen.util 927 :invoke :write [9 39] INFO [2019-12-17 12:28:32,378] jepsen worker 64 - jepsen.util 694 :fail :cas [21 [48 9]] :ValueNotSwapFrom INFO [2019-12-17 12:28:32,378] jepsen worker 64 - jepsen.util 694 :invoke :cas [21 [10 25]] INFO [2019-12-17 12:28:32,388] jepsen worker 66 - jepsen.util 1056 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:28:32,388] jepsen worker 68 - jepsen.util 1058 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:28:32,388] jepsen worker 16 - jepsen.util 1276 :info :cas [5 [46 38]] :RequestCanceledException INFO [2019-12-17 12:28:32,388] jepsen worker 66 - jepsen.util 1056 :invoke :write [22 4] INFO [2019-12-17 12:28:32,388] jepsen worker 68 - jepsen.util 1058 :invoke :read [22 nil] INFO [2019-12-17 12:28:32,389] jepsen worker 16 - jepsen.util 1366 :invoke :read [5 nil] INFO [2019-12-17 12:28:32,405] jepsen worker 64 - jepsen.util 694 :fail :cas [21 [10 25]] :ValueNotSwapFrom INFO [2019-12-17 12:28:32,405] jepsen worker 64 - jepsen.util 694 :invoke :read [21 nil] INFO [2019-12-17 12:28:32,406] jepsen worker 17 - jepsen.util 1277 :info :cas [5 [2 39]] :RequestCanceledException INFO [2019-12-17 12:28:32,406] jepsen worker 50 - jepsen.util 1310 :info :write [16 6] :RequestCanceledException INFO [2019-12-17 12:28:32,406] jepsen worker 83 - jepsen.util 1613 :info :cas [27 [14 7]] :RequestCanceledException INFO [2019-12-17 12:28:32,406] jepsen worker 60 - jepsen.util 1410 :info :write [20 3] :RequestCanceledException INFO [2019-12-17 12:28:32,406] jepsen worker 48 - jepsen.util 768 :info :cas [16 [29 0]] :RequestCanceledException INFO [2019-12-17 12:28:32,406] jepsen worker 73 - jepsen.util 1153 :ok :write [24 29] INFO [2019-12-17 12:28:32,406] jepsen worker 73 - jepsen.util 1153 :invoke :read [24 nil] INFO [2019-12-17 12:28:32,407] jepsen worker 50 - jepsen.util 1400 :invoke :cas [16 [44 35]] INFO [2019-12-17 12:28:32,407] jepsen worker 17 - jepsen.util 1367 :invoke :cas [5 [46 1]] INFO [2019-12-17 12:28:32,407] jepsen worker 83 - jepsen.util 1703 :invoke :read [27 nil] INFO [2019-12-17 12:28:32,407] jepsen worker 48 - jepsen.util 858 :invoke :write [16 13] INFO [2019-12-17 12:28:32,407] jepsen worker 60 - jepsen.util 1500 :invoke :cas [20 [21 9]] INFO [2019-12-17 12:28:32,421] jepsen worker 2 - jepsen.util 1082 :info :cas [0 [15 41]] :RequestCanceledException INFO [2019-12-17 12:28:32,421] jepsen worker 39 - jepsen.util 1479 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:28:32,421] jepsen worker 49 - jepsen.util 1129 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:28:32,422] jepsen worker 75 - jepsen.util 1425 :info :write [25 24] :RequestCanceledException INFO [2019-12-17 12:28:32,422] jepsen worker 67 - jepsen.util 1687 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:28:32,422] jepsen worker 84 - jepsen.util 1164 :info :write [28 30] :RequestCanceledException INFO [2019-12-17 12:28:32,422] jepsen worker 39 - jepsen.util 1479 :invoke :write [13 16] INFO [2019-12-17 12:28:32,422] jepsen worker 78 - jepsen.util 1158 :info :write [26 27] :RequestCanceledException INFO [2019-12-17 12:28:32,422] jepsen worker 49 - jepsen.util 1129 :invoke :write [16 41] INFO [2019-12-17 12:28:32,422] jepsen worker 67 - jepsen.util 1687 :invoke :read [22 nil] INFO [2019-12-17 12:28:32,423] jepsen worker 2 - jepsen.util 1172 :invoke :read [0 nil] INFO [2019-12-17 12:28:32,423] jepsen worker 75 - jepsen.util 1515 :invoke :write [25 36] INFO [2019-12-17 12:28:32,423] jepsen worker 84 - jepsen.util 1254 :invoke :cas [28 [16 31]] INFO [2019-12-17 12:28:32,423] jepsen worker 78 - jepsen.util 1248 :invoke :cas [26 [31 49]] INFO [2019-12-17 12:28:32,435] jepsen worker 40 - jepsen.util 1660 :info :write [13 47] :RequestCanceledException INFO [2019-12-17 12:28:32,435] jepsen worker 41 - jepsen.util 1121 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:28:32,436] jepsen worker 41 - jepsen.util 1121 :invoke :cas [13 [25 44]] INFO [2019-12-17 12:28:32,436] jepsen worker 40 - jepsen.util 1750 :invoke :read [13 nil] INFO [2019-12-17 12:28:32,440] jepsen worker 27 - jepsen.util 927 :ok :write [9 39] INFO [2019-12-17 12:28:32,440] jepsen worker 27 - jepsen.util 927 :invoke :cas [9 [10 39]] INFO [2019-12-17 12:28:32,440] jepsen worker 73 - jepsen.util 1153 :ok :read [24 29] INFO [2019-12-17 12:28:32,440] jepsen worker 73 - jepsen.util 1153 :invoke :cas [24 [23 9]] INFO [2019-12-17 12:28:32,447] jepsen worker 64 - jepsen.util 694 :ok :read [21 9] INFO [2019-12-17 12:28:32,447] jepsen worker 64 - jepsen.util 694 :invoke :cas [21 [18 21]] INFO [2019-12-17 12:28:32,452] jepsen worker 86 - jepsen.util 1346 :info :cas [28 [14 44]] :RequestCanceledException INFO [2019-12-17 12:28:32,453] jepsen worker 86 - jepsen.util 1436 :invoke :cas [28 [26 24]] INFO [2019-12-17 12:28:32,466] jepsen worker 71 - jepsen.util 791 :fail :cas [23 [33 45]] :ValueNotSwapFrom INFO [2019-12-17 12:28:32,467] jepsen worker 73 - jepsen.util 1153 :fail :cas [24 [23 9]] :ValueNotSwapFrom INFO [2019-12-17 12:28:32,467] jepsen worker 71 - jepsen.util 791 :invoke :write [23 33] INFO [2019-12-17 12:28:32,467] jepsen worker 73 - jepsen.util 1153 :invoke :cas [24 [12 24]] INFO [2019-12-17 12:28:32,488] jepsen worker 64 - jepsen.util 694 :fail :cas [21 [18 21]] :ValueNotSwapFrom INFO [2019-12-17 12:28:32,488] jepsen worker 15 - jepsen.util 915 :info :cas [5 [7 33]] :RequestCanceledException INFO [2019-12-17 12:28:32,488] jepsen worker 1 - jepsen.util 1171 :info :cas [0 [43 18]] :RequestCanceledException INFO [2019-12-17 12:28:32,488] jepsen worker 27 - jepsen.util 927 :fail :cas [9 [10 39]] :ValueNotSwapFrom INFO [2019-12-17 12:28:32,488] jepsen worker 64 - jepsen.util 694 :invoke :write [21 19] INFO [2019-12-17 12:28:32,489] jepsen worker 27 - jepsen.util 927 :invoke :write [9 31] INFO [2019-12-17 12:28:32,489] jepsen worker 73 - jepsen.util 1153 :fail :cas [24 [12 24]] :ValueNotSwapFrom INFO [2019-12-17 12:28:32,489] jepsen worker 73 - jepsen.util 1153 :invoke :read [24 nil] INFO [2019-12-17 12:28:32,490] jepsen worker 1 - jepsen.util 1261 :invoke :cas [0 [15 38]] INFO [2019-12-17 12:28:32,490] jepsen worker 15 - jepsen.util 1005 :invoke :cas [5 [2 41]] INFO [2019-12-17 12:28:32,504] jepsen worker 77 - jepsen.util 1337 :fail :cas [25 [12 8]] :ValueNotSwapFrom INFO [2019-12-17 12:28:32,504] jepsen worker 77 - jepsen.util 1337 :invoke :read [25 nil] INFO [2019-12-17 12:28:32,511] jepsen worker 73 - jepsen.util 1153 :ok :read [24 29] INFO [2019-12-17 12:28:32,511] jepsen worker 73 - jepsen.util 1153 :invoke :read [24 nil] INFO [2019-12-17 12:28:32,521] jepsen worker 61 - jepsen.util 1321 :info :write [20 0] :RequestCanceledException INFO [2019-12-17 12:28:32,521] jepsen worker 38 - jepsen.util 938 :info :write [12 4] :RequestCanceledException INFO [2019-12-17 12:28:32,522] jepsen worker 38 - jepsen.util 1028 :invoke :cas [12 [13 33]] INFO [2019-12-17 12:28:32,522] jepsen worker 61 - jepsen.util 1411 :invoke :write [20 42] INFO [2019-12-17 12:28:32,525] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":6254,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:28:32,526] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":6254,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:28:32,527] jepsen worker 73 - jepsen.util 1153 :ok :read [24 29] INFO [2019-12-17 12:28:32,527] jepsen worker 73 - jepsen.util 1153 :invoke :read [24 nil] INFO [2019-12-17 12:28:32,535] jepsen worker 81 - jepsen.util 1431 :info :write [27 49] :RequestCanceledException INFO [2019-12-17 12:28:32,537] jepsen worker 81 - jepsen.util 1521 :invoke :read [27 nil] INFO [2019-12-17 12:28:32,547] jepsen worker 73 - jepsen.util 1153 :ok :read [24 29] INFO [2019-12-17 12:28:32,547] jepsen worker 73 - jepsen.util 1153 :invoke :write [24 6] INFO [2019-12-17 12:28:32,552] jepsen worker 42 - jepsen.util 1572 :info :cas [14 [45 9]] :RequestCanceledException INFO [2019-12-17 12:28:32,553] jepsen worker 42 - jepsen.util 1662 :invoke :read [14 nil] INFO [2019-12-17 12:28:32,567] jepsen worker 77 - jepsen.util 1337 :ok :read [25 2] INFO [2019-12-17 12:28:32,567] jepsen worker 77 - jepsen.util 1337 :invoke :read [25 nil] INFO [2019-12-17 12:28:32,570] jepsen worker 27 - jepsen.util 927 :ok :write [9 31] INFO [2019-12-17 12:28:32,571] jepsen worker 27 - jepsen.util 927 :invoke :cas [9 [15 20]] INFO [2019-12-17 12:28:32,586] jepsen worker 27 - jepsen.util 927 :fail :cas [9 [15 20]] :ValueNotSwapFrom INFO [2019-12-17 12:28:32,586] jepsen worker 27 - jepsen.util 927 :invoke :read [9 nil] INFO [2019-12-17 12:28:32,592] jepsen worker 64 - jepsen.util 694 :ok :write [21 19] INFO [2019-12-17 12:28:32,592] jepsen worker 64 - jepsen.util 694 :invoke :read [21 nil] INFO [2019-12-17 12:28:32,601] jepsen worker 27 - jepsen.util 927 :ok :read [9 31] INFO [2019-12-17 12:28:32,601] jepsen worker 27 - jepsen.util 927 :invoke :write [9 38] INFO [2019-12-17 12:28:32,608] jepsen worker 64 - jepsen.util 694 :ok :read [21 19] INFO [2019-12-17 12:28:32,608] jepsen worker 64 - jepsen.util 694 :invoke :write [21 46] INFO [2019-12-17 12:28:32,620] jepsen worker 13 - jepsen.util 1093 :info :write [4 5] :RequestCanceledException INFO [2019-12-17 12:28:32,620] jepsen worker 58 - jepsen.util 1318 :info :write [19 21] :RequestCanceledException INFO [2019-12-17 12:28:32,621] jepsen worker 58 - jepsen.util 1408 :invoke :cas [19 [26 7]] INFO [2019-12-17 12:28:32,622] jepsen worker 13 - jepsen.util 1183 :invoke :cas [4 [33 17]] INFO [2019-12-17 12:28:32,638] jepsen worker 53 - jepsen.util 1133 :ok :read [17 21] INFO [2019-12-17 12:28:32,638] jepsen worker 53 - jepsen.util 1133 :invoke :read [17 nil] INFO [2019-12-17 12:28:32,662] jepsen worker 27 - jepsen.util 927 :ok :write [9 38] INFO [2019-12-17 12:28:32,662] jepsen worker 27 - jepsen.util 927 :invoke :write [9 10] INFO [2019-12-17 12:28:32,666] jepsen worker 38 - jepsen.util 1028 :fail :cas [12 [13 33]] :ValueNotSwapFrom INFO [2019-12-17 12:28:32,666] jepsen worker 38 - jepsen.util 1028 :invoke :read [12 nil] INFO [2019-12-17 12:28:32,678] jepsen worker 64 - jepsen.util 694 :ok :write [21 46] INFO [2019-12-17 12:28:32,678] jepsen worker 64 - jepsen.util 694 :invoke :read [21 nil] INFO [2019-12-17 12:28:32,687] jepsen worker 31 - jepsen.util 1111 :info :write [10 36] :RequestCanceledException INFO [2019-12-17 12:28:32,687] jepsen worker 28 - jepsen.util 748 :info :cas [9 [16 0]] :RequestCanceledException INFO [2019-12-17 12:28:32,688] jepsen worker 31 - jepsen.util 1201 :invoke :read [10 nil] INFO [2019-12-17 12:28:32,688] jepsen worker 28 - jepsen.util 838 :invoke :read [9 nil] INFO [2019-12-17 12:28:32,709] jepsen worker 33 - jepsen.util 1653 :fail :cas [11 [19 10]] :ValueNotSwapFrom INFO [2019-12-17 12:28:32,709] jepsen worker 53 - jepsen.util 1133 :ok :read [17 21] INFO [2019-12-17 12:28:32,709] jepsen worker 53 - jepsen.util 1133 :invoke :cas [17 [3 13]] INFO [2019-12-17 12:28:32,709] jepsen worker 33 - jepsen.util 1653 :invoke :write [11 45] INFO [2019-12-17 12:28:32,713] jepsen worker 31 - jepsen.util 1201 :ok :read [10 12] INFO [2019-12-17 12:28:32,713] jepsen worker 31 - jepsen.util 1201 :invoke :write [10 26] INFO [2019-12-17 12:28:32,717] jepsen worker 64 - jepsen.util 694 :ok :read [21 46] INFO [2019-12-17 12:28:32,717] jepsen worker 64 - jepsen.util 694 :invoke :cas [21 [39 9]] INFO [2019-12-17 12:28:32,718] jepsen worker 20 - jepsen.util 1100 :info :write [6 14] :RequestCanceledException INFO [2019-12-17 12:28:32,719] jepsen worker 20 - jepsen.util 1190 :invoke :write [6 35] INFO [2019-12-17 12:28:32,728] jepsen worker 38 - jepsen.util 1028 :ok :read [12 38] INFO [2019-12-17 12:28:32,729] jepsen worker 38 - jepsen.util 1028 :invoke :read [12 nil] INFO [2019-12-17 12:28:32,735] jepsen worker 46 - jepsen.util 1306 :info :write [15 12] :RequestCanceledException INFO [2019-12-17 12:28:32,735] jepsen worker 63 - jepsen.util 1053 :fail :read [21 nil] :RequestTimeoutException INFO [2019-12-17 12:28:32,735] jepsen worker 63 - jepsen.util 1053 :invoke :cas [21 [26 30]] INFO [2019-12-17 12:28:32,736] jepsen worker 46 - jepsen.util 1396 :invoke :read [15 nil] INFO [2019-12-17 12:28:32,740] jepsen worker 28 - jepsen.util 838 :ok :read [9 38] INFO [2019-12-17 12:28:32,740] jepsen worker 28 - jepsen.util 838 :invoke :read [9 nil] INFO [2019-12-17 12:28:32,745] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":6291,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:28:32,745] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":6291,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:28:32,745] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":6294,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:28:32,745] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":6294,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:28:32,745] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":6295,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:28:32,745] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":6295,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:28:32,756] jepsen worker 53 - jepsen.util 1133 :fail :cas [17 [3 13]] :ValueNotSwapFrom INFO [2019-12-17 12:28:32,756] jepsen worker 53 - jepsen.util 1133 :invoke :write [17 42] INFO [2019-12-17 12:28:32,763] jepsen worker 38 - jepsen.util 1028 :ok :read [12 38] INFO [2019-12-17 12:28:32,763] jepsen worker 38 - jepsen.util 1028 :invoke :write [12 41] INFO [2019-12-17 12:28:32,779] jepsen worker 64 - jepsen.util 694 :fail :cas [21 [39 9]] :ValueNotSwapFrom INFO [2019-12-17 12:28:32,779] jepsen worker 64 - jepsen.util 694 :invoke :read [21 nil] INFO [2019-12-17 12:28:32,805] jepsen worker 28 - jepsen.util 838 :ok :read [9 38] INFO [2019-12-17 12:28:32,805] jepsen worker 28 - jepsen.util 838 :invoke :write [9 7] INFO [2019-12-17 12:28:32,806] jepsen worker 10 - jepsen.util 1270 :fail :read [3 nil] :RequestTimeoutException INFO [2019-12-17 12:28:32,806] jepsen worker 11 - jepsen.util 1451 :info :write [3 46] :RequestCanceledException INFO [2019-12-17 12:28:32,807] jepsen worker 10 - jepsen.util 1270 :invoke :cas [3 [36 10]] INFO [2019-12-17 12:28:32,807] jepsen worker 46 - jepsen.util 1396 :ok :read [15 4] INFO [2019-12-17 12:28:32,807] jepsen worker 46 - jepsen.util 1396 :invoke :cas [15 [37 7]] INFO [2019-12-17 12:28:32,808] jepsen worker 11 - jepsen.util 1541 :invoke :write [3 12] INFO [2019-12-17 12:28:32,815] jepsen worker 64 - jepsen.util 694 :ok :read [21 46] INFO [2019-12-17 12:28:32,816] jepsen worker 64 - jepsen.util 694 :invoke :read [21 nil] INFO [2019-12-17 12:28:32,818] jepsen worker 35 - jepsen.util 1025 :info :write [11 14] :RequestCanceledException INFO [2019-12-17 12:28:32,819] jepsen worker 35 - jepsen.util 1115 :invoke :read [11 nil] INFO [2019-12-17 12:28:32,826] jepsen worker 27 - jepsen.util 927 :ok :write [9 10] INFO [2019-12-17 12:28:32,827] jepsen worker 27 - jepsen.util 927 :invoke :cas [9 [40 39]] INFO [2019-12-17 12:28:32,832] jepsen worker 10 - jepsen.util 1270 :fail :cas [3 [36 10]] :ValueNotSwapFrom INFO [2019-12-17 12:28:32,833] jepsen worker 10 - jepsen.util 1270 :invoke :cas [3 [35 35]] INFO [2019-12-17 12:28:32,848] jepsen worker 54 - jepsen.util 1314 :fail :cas [18 [44 23]] :ValueNotSwapFrom INFO [2019-12-17 12:28:32,848] jepsen worker 54 - jepsen.util 1314 :invoke :read [18 nil] INFO [2019-12-17 12:28:32,853] jepsen worker 14 - jepsen.util 1004 :fail :cas [4 [16 48]] :ValueNotSwapFrom INFO [2019-12-17 12:28:32,853] jepsen worker 14 - jepsen.util 1004 :invoke :write [4 27] INFO [2019-12-17 12:28:32,875] jepsen worker 64 - jepsen.util 694 :ok :read [21 46] INFO [2019-12-17 12:28:32,875] jepsen worker 64 - jepsen.util 694 :invoke :write [21 45] INFO [2019-12-17 12:28:32,882] jepsen worker 63 - jepsen.util 1053 :fail :cas [21 [26 30]] :ValueNotSwapFrom INFO [2019-12-17 12:28:32,882] jepsen worker 63 - jepsen.util 1053 :invoke :read [21 nil] INFO [2019-12-17 12:28:32,883] jepsen worker 46 - jepsen.util 1396 :fail :cas [15 [37 7]] :ValueNotSwapFrom INFO [2019-12-17 12:28:32,884] jepsen worker 46 - jepsen.util 1396 :invoke :read [15 nil] INFO [2019-12-17 12:28:32,884] jepsen worker 87 - jepsen.util 1077 :info :write [29 8] :RequestCanceledException INFO [2019-12-17 12:28:32,885] jepsen worker 87 - jepsen.util 1167 :invoke :cas [29 [24 47]] INFO [2019-12-17 12:28:32,895] jepsen worker 35 - jepsen.util 1115 :ok :read [11 7] INFO [2019-12-17 12:28:32,895] jepsen worker 35 - jepsen.util 1115 :invoke :write [11 35] INFO [2019-12-17 12:28:32,902] jepsen worker 77 - jepsen.util 1337 :ok :read [25 2] INFO [2019-12-17 12:28:32,902] jepsen worker 77 - jepsen.util 1337 :invoke :write [25 4] INFO [2019-12-17 12:28:32,921] jepsen worker 10 - jepsen.util 1270 :fail :cas [3 [35 35]] :ValueNotSwapFrom INFO [2019-12-17 12:28:32,921] jepsen worker 10 - jepsen.util 1270 :invoke :write [3 40] INFO [2019-12-17 12:28:32,940] jepsen worker 27 - jepsen.util 927 :fail :cas [9 [40 39]] :ValueNotSwapFrom INFO [2019-12-17 12:28:32,941] jepsen worker 27 - jepsen.util 927 :invoke :cas [9 [11 4]] INFO [2019-12-17 12:28:32,954] jepsen worker 46 - jepsen.util 1396 :ok :read [15 4] INFO [2019-12-17 12:28:32,954] jepsen worker 46 - jepsen.util 1396 :invoke :write [15 22] INFO [2019-12-17 12:28:32,959] jepsen worker 63 - jepsen.util 1053 :ok :read [21 46] INFO [2019-12-17 12:28:32,959] jepsen worker 63 - jepsen.util 1053 :invoke :cas [21 [48 14]] INFO [2019-12-17 12:28:32,964] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":6324,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:28:32,964] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":6324,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:28:32,964] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":6327,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:28:32,964] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":6327,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:28:32,974] jepsen worker 27 - jepsen.util 927 :fail :cas [9 [11 4]] :ValueNotSwapFrom INFO [2019-12-17 12:28:32,974] jepsen worker 27 - jepsen.util 927 :invoke :cas [9 [29 10]] INFO [2019-12-17 12:28:32,987] jepsen worker 55 - jepsen.util 1495 :info :write [18 1] :RequestCanceledException INFO [2019-12-17 12:28:32,988] jepsen worker 55 - jepsen.util 1585 :invoke :write [18 32] INFO [2019-12-17 12:28:33,000] jepsen worker 5 - jepsen.util 815 :ok :read [1 24] INFO [2019-12-17 12:28:33,000] jepsen worker 5 - jepsen.util 815 :invoke :read [1 nil] INFO [2019-12-17 12:28:33,005] jepsen worker 29 - jepsen.util 929 :info :write [9 31] :RequestCanceledException INFO [2019-12-17 12:28:33,006] jepsen worker 29 - jepsen.util 1019 :invoke :read [9 nil] INFO [2019-12-17 12:28:33,008] jepsen worker 63 - jepsen.util 1053 :fail :cas [21 [48 14]] :ValueNotSwapFrom INFO [2019-12-17 12:28:33,008] jepsen worker 63 - jepsen.util 1053 :invoke :write [21 14] INFO [2019-12-17 12:28:33,021] jepsen worker 27 - jepsen.util 927 :fail :cas [9 [29 10]] :ValueNotSwapFrom INFO [2019-12-17 12:28:33,021] jepsen worker 27 - jepsen.util 927 :invoke :write [9 42] INFO [2019-12-17 12:28:33,034] jepsen worker 51 - jepsen.util 1131 :ok :read [17 21] INFO [2019-12-17 12:28:33,034] jepsen worker 51 - jepsen.util 1131 :invoke :read [17 nil] INFO [2019-12-17 12:28:33,059] jepsen worker 28 - jepsen.util 838 :ok :write [9 7] INFO [2019-12-17 12:28:33,060] jepsen worker 28 - jepsen.util 838 :invoke :write [9 46] INFO [2019-12-17 12:28:33,060] jepsen worker 54 - jepsen.util 1314 :ok :read [18 17] INFO [2019-12-17 12:28:33,061] jepsen worker 54 - jepsen.util 1314 :invoke :write [18 32] INFO [2019-12-17 12:28:33,073] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":6342,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:28:33,073] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":6342,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:28:33,073] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":6346,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:28:33,073] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":6346,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:28:33,088] jepsen worker 26 - jepsen.util 1376 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:28:33,088] jepsen worker 26 - jepsen.util 1376 :invoke :write [8 10] INFO [2019-12-17 12:28:33,091] jepsen worker 63 - jepsen.util 1053 :ok :write [21 14] INFO [2019-12-17 12:28:33,091] jepsen worker 63 - jepsen.util 1053 :invoke :read [21 nil] INFO [2019-12-17 12:28:33,107] jepsen worker 63 - jepsen.util 1053 :ok :read [21 14] INFO [2019-12-17 12:28:33,107] jepsen worker 63 - jepsen.util 1053 :invoke :write [21 9] INFO [2019-12-17 12:28:33,118] jepsen worker 28 - jepsen.util 838 :ok :write [9 46] INFO [2019-12-17 12:28:33,118] jepsen worker 28 - jepsen.util 838 :invoke :cas [9 [36 5]] INFO [2019-12-17 12:28:33,122] jepsen worker 3 - jepsen.util 903 :info :write [1 20] :RequestCanceledException INFO [2019-12-17 12:28:33,123] jepsen worker 3 - jepsen.util 993 :invoke :write [1 40] INFO [2019-12-17 12:28:33,142] jepsen worker 28 - jepsen.util 838 :fail :cas [9 [36 5]] :ValueNotSwapFrom INFO [2019-12-17 12:28:33,142] jepsen worker 28 - jepsen.util 838 :invoke :write [9 48] INFO [2019-12-17 12:28:33,147] jepsen worker 73 - jepsen.util 1153 :ok :write [24 6] INFO [2019-12-17 12:28:33,147] jepsen worker 31 - jepsen.util 1201 :ok :write [10 26] INFO [2019-12-17 12:28:33,147] jepsen worker 45 - jepsen.util 1215 :ok :write [15 18] INFO [2019-12-17 12:28:33,148] jepsen worker 73 - jepsen.util 1153 :invoke :write [24 11] INFO [2019-12-17 12:28:33,148] jepsen worker 31 - jepsen.util 1201 :invoke :cas [10 [49 15]] INFO [2019-12-17 12:28:33,148] jepsen worker 45 - jepsen.util 1215 :invoke :write [15 34] INFO [2019-12-17 12:28:33,151] jepsen worker 10 - jepsen.util 1270 :ok :write [3 40] INFO [2019-12-17 12:28:33,151] jepsen worker 10 - jepsen.util 1270 :invoke :read [3 nil] INFO [2019-12-17 12:28:33,153] jepsen worker 8 - jepsen.util 1268 :info :write [2 25] :RequestCanceledException INFO [2019-12-17 12:28:33,154] jepsen worker 8 - jepsen.util 1358 :invoke :write [2 17] INFO [2019-12-17 12:28:33,155] jepsen worker 5 - jepsen.util 815 :ok :read [1 24] INFO [2019-12-17 12:28:33,155] jepsen worker 5 - jepsen.util 815 :invoke :cas [1 [33 28]] INFO [2019-12-17 12:28:33,189] jepsen worker 51 - jepsen.util 1131 :ok :read [17 21] INFO [2019-12-17 12:28:33,189] jepsen worker 51 - jepsen.util 1131 :invoke :write [17 34] INFO [2019-12-17 12:28:33,203] jepsen worker 78 - jepsen.util 1248 :fail :cas [26 [31 49]] :ValueNotSwapFrom INFO [2019-12-17 12:28:33,203] jepsen worker 78 - jepsen.util 1248 :invoke :write [26 45] INFO [2019-12-17 12:28:33,221] jepsen worker 69 - jepsen.util 969 :info :write [23 44] :RequestCanceledException INFO [2019-12-17 12:28:33,223] jepsen worker 69 - jepsen.util 1059 :invoke :read [23 nil] INFO [2019-12-17 12:28:33,242] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:28:33,243] jepsen worker 10 - jepsen.util 1270 :ok :read [3 40] INFO [2019-12-17 12:28:33,243] jepsen nemesis - couchbase.util Rebalance check count: 41 INFO [2019-12-17 12:28:33,243] jepsen nemesis - couchbase.util Rebalance stuck count: 22 INFO [2019-12-17 12:28:33,243] jepsen worker 10 - jepsen.util 1270 :invoke :cas [3 [25 36]] INFO [2019-12-17 12:28:33,250] jepsen worker 31 - jepsen.util 1201 :fail :cas [10 [49 15]] :ValueNotSwapFrom INFO [2019-12-17 12:28:33,250] jepsen worker 31 - jepsen.util 1201 :invoke :read [10 nil] INFO [2019-12-17 12:28:33,251] jepsen worker 76 - jepsen.util 1426 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:28:33,251] jepsen worker 76 - jepsen.util 1426 :invoke :cas [25 [30 28]] INFO [2019-12-17 12:28:33,272] jepsen worker 63 - jepsen.util 1053 :ok :write [21 9] INFO [2019-12-17 12:28:33,272] jepsen worker 63 - jepsen.util 1053 :invoke :write [21 46] INFO [2019-12-17 12:28:33,288] jepsen worker 6 - jepsen.util 1356 :info :cas [2 [5 37]] :RequestCanceledException INFO [2019-12-17 12:28:33,288] jepsen worker 6 - jepsen.util 1446 :invoke :write [2 17] INFO [2019-12-17 12:28:33,292] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":6375,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:28:33,292] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":6375,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:28:33,292] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":6378,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:28:33,292] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":6378,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:28:33,295] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":6367,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:28:33,295] jepsen worker 10 - jepsen.util 1270 :fail :cas [3 [25 36]] :ValueNotSwapFrom INFO [2019-12-17 12:28:33,295] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":6367,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:28:33,295] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":6368,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:28:33,295] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":6368,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:28:33,295] jepsen worker 10 - jepsen.util 1270 :invoke :write [3 48] INFO [2019-12-17 12:28:33,295] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":6369,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:28:33,295] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":6369,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:28:33,300] jepsen worker 5 - jepsen.util 815 :fail :cas [1 [33 28]] :ValueNotSwapFrom INFO [2019-12-17 12:28:33,300] jepsen worker 5 - jepsen.util 815 :invoke :cas [1 [16 31]] INFO [2019-12-17 12:28:33,306] jepsen worker 9 - jepsen.util 1179 :fail :read [3 nil] :RequestTimeoutException INFO [2019-12-17 12:28:33,306] jepsen worker 12 - jepsen.util 1092 :info :write [4 0] :RequestCanceledException INFO [2019-12-17 12:28:33,307] jepsen worker 9 - jepsen.util 1179 :invoke :cas [3 [41 0]] INFO [2019-12-17 12:28:33,307] jepsen worker 12 - jepsen.util 1182 :invoke :write [4 44] INFO [2019-12-17 12:28:33,336] jepsen worker 87 - jepsen.util 1167 :fail :cas [29 [24 47]] :ValueNotSwapFrom INFO [2019-12-17 12:28:33,340] jepsen worker 87 - jepsen.util 1167 :invoke :write [29 17] INFO [2019-12-17 12:28:33,344] jepsen worker 31 - jepsen.util 1201 :ok :read [10 26] INFO [2019-12-17 12:28:33,344] jepsen worker 31 - jepsen.util 1201 :invoke :cas [10 [19 45]] INFO [2019-12-17 12:28:33,352] jepsen worker 74 - jepsen.util 974 :info :write [24 49] :RequestCanceledException INFO [2019-12-17 12:28:33,353] jepsen worker 74 - jepsen.util 1064 :invoke :write [24 27] INFO [2019-12-17 12:28:33,391] jepsen worker 31 - jepsen.util 1201 :fail :cas [10 [19 45]] :ValueNotSwapFrom INFO [2019-12-17 12:28:33,391] jepsen worker 31 - jepsen.util 1201 :invoke :cas [10 [36 3]] INFO [2019-12-17 12:28:33,404] jepsen worker 28 - jepsen.util 838 :ok :write [9 48] INFO [2019-12-17 12:28:33,404] jepsen worker 28 - jepsen.util 838 :invoke :write [9 45] INFO [2019-12-17 12:28:33,421] jepsen worker 44 - jepsen.util 1304 :info :write [14 2] :RequestCanceledException INFO [2019-12-17 12:28:33,423] jepsen worker 44 - jepsen.util 1394 :invoke :read [14 nil] INFO [2019-12-17 12:28:33,435] jepsen worker 34 - jepsen.util 1474 :info :write [11 40] :RequestCanceledException INFO [2019-12-17 12:28:33,435] jepsen worker 34 - jepsen.util 1564 :invoke :write [11 16] INFO [2019-12-17 12:28:33,446] jepsen worker 31 - jepsen.util 1201 :fail :cas [10 [36 3]] :ValueNotSwapFrom INFO [2019-12-17 12:28:33,447] jepsen worker 31 - jepsen.util 1201 :invoke :cas [10 [14 14]] INFO [2019-12-17 12:28:33,461] jepsen worker 13 - jepsen.util 1183 :fail :cas [4 [33 17]] :ValueNotSwapFrom INFO [2019-12-17 12:28:33,461] jepsen worker 13 - jepsen.util 1183 :invoke :cas [4 [32 24]] INFO [2019-12-17 12:28:33,487] jepsen worker 22 - jepsen.util 1372 :info :cas [7 [11 14]] :RequestCanceledException INFO [2019-12-17 12:28:33,488] jepsen worker 22 - jepsen.util 1462 :invoke :write [7 26] INFO [2019-12-17 12:28:33,499] jepsen worker 5 - jepsen.util 815 :fail :cas [1 [16 31]] :ValueNotSwapFrom INFO [2019-12-17 12:28:33,499] jepsen worker 5 - jepsen.util 815 :invoke :write [1 21] INFO [2019-12-17 12:28:33,502] jepsen worker 31 - jepsen.util 1201 :fail :cas [10 [14 14]] :ValueNotSwapFrom INFO [2019-12-17 12:28:33,503] jepsen worker 31 - jepsen.util 1201 :invoke :cas [10 [17 45]] INFO [2019-12-17 12:28:33,505] jepsen worker 21 - jepsen.util 1281 :info :cas [7 [12 44]] :RequestCanceledException INFO [2019-12-17 12:28:33,506] jepsen worker 21 - jepsen.util 1371 :invoke :write [7 1] INFO [2019-12-17 12:28:33,511] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":6397,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:28:33,511] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":6397,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:28:33,511] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":6400,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:28:33,511] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":6400,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:28:33,535] jepsen worker 65 - jepsen.util 965 :fail :read [21 nil] :RequestTimeoutException INFO [2019-12-17 12:28:33,536] jepsen worker 65 - jepsen.util 965 :invoke :write [21 7] INFO [2019-12-17 12:28:33,541] jepsen worker 31 - jepsen.util 1201 :fail :cas [10 [17 45]] :ValueNotSwapFrom INFO [2019-12-17 12:28:33,541] jepsen worker 31 - jepsen.util 1201 :invoke :cas [10 [34 22]] INFO [2019-12-17 12:28:33,553] jepsen worker 80 - jepsen.util 1340 :info :write [26 37] :RequestCanceledException INFO [2019-12-17 12:28:33,554] jepsen worker 80 - jepsen.util 1430 :invoke :read [26 nil] INFO [2019-12-17 12:28:33,556] jepsen worker 11 - jepsen.util 1541 :ok :write [3 12] INFO [2019-12-17 12:28:33,556] jepsen worker 20 - jepsen.util 1190 :ok :write [6 35] INFO [2019-12-17 12:28:33,556] jepsen worker 11 - jepsen.util 1541 :invoke :write [3 37] INFO [2019-12-17 12:28:33,556] jepsen worker 20 - jepsen.util 1190 :invoke :cas [6 [2 10]] INFO [2019-12-17 12:28:33,579] jepsen worker 31 - jepsen.util 1201 :fail :cas [10 [34 22]] :ValueNotSwapFrom INFO [2019-12-17 12:28:33,579] jepsen worker 31 - jepsen.util 1201 :invoke :write [10 24] INFO [2019-12-17 12:28:33,582] jepsen worker 63 - jepsen.util 1053 :ok :write [21 46] INFO [2019-12-17 12:28:33,583] jepsen worker 63 - jepsen.util 1053 :invoke :cas [21 [17 1]] INFO [2019-12-17 12:28:33,588] jepsen worker 43 - jepsen.util 1483 :info :write [14 46] :RequestCanceledException INFO [2019-12-17 12:28:33,589] jepsen worker 43 - jepsen.util 1573 :invoke :write [14 45] INFO [2019-12-17 12:28:33,595] jepsen worker 28 - jepsen.util 838 :ok :write [9 45] INFO [2019-12-17 12:28:33,595] jepsen worker 28 - jepsen.util 838 :invoke :read [9 nil] INFO [2019-12-17 12:28:33,600] jepsen worker 29 - jepsen.util 1019 :ok :read [9 48] INFO [2019-12-17 12:28:33,600] jepsen worker 29 - jepsen.util 1019 :invoke :write [9 36] INFO [2019-12-17 12:28:33,606] jepsen worker 4 - jepsen.util 904 :info :write [1 15] :RequestCanceledException INFO [2019-12-17 12:28:33,606] jepsen worker 89 - jepsen.util 1259 :info :write [29 0] :RequestCanceledException INFO [2019-12-17 12:28:33,607] jepsen worker 4 - jepsen.util 994 :invoke :read [1 nil] INFO [2019-12-17 12:28:33,607] jepsen worker 89 - jepsen.util 1349 :invoke :cas [29 [19 30]] INFO [2019-12-17 12:28:33,622] jepsen worker 19 - jepsen.util 1189 :info :cas [6 [38 20]] :RequestCanceledException INFO [2019-12-17 12:28:33,622] jepsen worker 18 - jepsen.util 1278 :fail :read [6 nil] :RequestTimeoutException INFO [2019-12-17 12:28:33,622] jepsen worker 30 - jepsen.util 1290 :info :cas [10 [18 20]] :RequestCanceledException INFO [2019-12-17 12:28:33,622] jepsen worker 18 - jepsen.util 1278 :invoke :write [6 29] INFO [2019-12-17 12:28:33,623] jepsen worker 19 - jepsen.util 1279 :invoke :cas [6 [35 20]] INFO [2019-12-17 12:28:33,623] jepsen worker 30 - jepsen.util 1380 :invoke :cas [10 [34 15]] INFO [2019-12-17 12:28:33,635] jepsen worker 79 - jepsen.util 1069 :info :write [26 8] :RequestCanceledException INFO [2019-12-17 12:28:33,636] jepsen worker 79 - jepsen.util 1159 :invoke :cas [26 [30 5]] INFO [2019-12-17 12:28:33,651] jepsen worker 28 - jepsen.util 838 :ok :read [9 45] INFO [2019-12-17 12:28:33,651] jepsen worker 28 - jepsen.util 838 :invoke :write [9 41] INFO [2019-12-17 12:28:33,652] jepsen worker 70 - jepsen.util 1330 :info :write [23 38] :RequestCanceledException INFO [2019-12-17 12:28:33,653] jepsen worker 70 - jepsen.util 1420 :invoke :cas [23 [32 37]] INFO [2019-12-17 12:28:33,674] jepsen worker 89 - jepsen.util 1349 :fail :cas [29 [19 30]] :ValueNotSwapFrom INFO [2019-12-17 12:28:33,674] jepsen worker 89 - jepsen.util 1349 :invoke :write [29 44] INFO [2019-12-17 12:28:33,698] jepsen worker 65 - jepsen.util 965 :ok :write [21 7] INFO [2019-12-17 12:28:33,698] jepsen worker 65 - jepsen.util 965 :invoke :cas [21 [16 29]] INFO [2019-12-17 12:28:33,705] jepsen worker 72 - jepsen.util 1062 :info :write [24 37] :RequestCanceledException INFO [2019-12-17 12:28:33,706] jepsen worker 72 - jepsen.util 1152 :invoke :read [24 nil] INFO [2019-12-17 12:28:33,721] jepsen worker 37 - jepsen.util 937 :info :write [12 45] :RequestCanceledException INFO [2019-12-17 12:28:33,723] jepsen worker 37 - jepsen.util 1027 :invoke :cas [12 [9 23]] INFO [2019-12-17 12:28:33,728] jepsen worker 65 - jepsen.util 965 :fail :cas [21 [16 29]] :ValueNotSwapFrom INFO [2019-12-17 12:28:33,728] jepsen worker 65 - jepsen.util 965 :invoke :read [21 nil] INFO [2019-12-17 12:28:33,731] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":6427,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:28:33,731] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":6427,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:28:33,731] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":6430,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:28:33,731] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":6430,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:28:33,731] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":6431,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:28:33,732] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":6431,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:28:33,735] jepsen worker 88 - jepsen.util 1078 :info :write [29 21] :RequestCanceledException INFO [2019-12-17 12:28:33,736] jepsen worker 88 - jepsen.util 1168 :invoke :cas [29 [27 48]] INFO [2019-12-17 12:28:33,740] jepsen worker 28 - jepsen.util 838 :ok :write [9 41] INFO [2019-12-17 12:28:33,741] jepsen worker 28 - jepsen.util 838 :invoke :cas [9 [44 11]] INFO [2019-12-17 12:28:33,750] jepsen worker 65 - jepsen.util 965 :ok :read [21 7] INFO [2019-12-17 12:28:33,750] jepsen worker 65 - jepsen.util 965 :invoke :cas [21 [27 8]] INFO [2019-12-17 12:28:33,761] jepsen worker 28 - jepsen.util 838 :fail :cas [9 [44 11]] :ValueNotSwapFrom INFO [2019-12-17 12:28:33,761] jepsen worker 28 - jepsen.util 838 :invoke :read [9 nil] INFO [2019-12-17 12:28:33,773] jepsen worker 65 - jepsen.util 965 :fail :cas [21 [27 8]] :ValueNotSwapFrom INFO [2019-12-17 12:28:33,773] jepsen worker 65 - jepsen.util 965 :invoke :write [21 38] INFO [2019-12-17 12:28:33,796] jepsen worker 28 - jepsen.util 838 :ok :read [9 41] INFO [2019-12-17 12:28:33,796] jepsen worker 28 - jepsen.util 838 :invoke :read [9 nil] INFO [2019-12-17 12:28:33,805] jepsen worker 69 - jepsen.util 1059 :ok :read [23 39] INFO [2019-12-17 12:28:33,806] jepsen worker 7 - jepsen.util 1537 :info :write [2 40] :RequestCanceledException INFO [2019-12-17 12:28:33,811] jepsen worker 69 - jepsen.util 1059 :invoke :read [23 nil] INFO [2019-12-17 12:28:33,811] jepsen worker 7 - jepsen.util 1627 :invoke :write [2 26] INFO [2019-12-17 12:28:33,815] jepsen worker 28 - jepsen.util 838 :ok :read [9 41] INFO [2019-12-17 12:28:33,815] jepsen worker 28 - jepsen.util 838 :invoke :read [9 nil] INFO [2019-12-17 12:28:33,820] jepsen worker 24 - jepsen.util 1284 :info :write [8 26] :RequestCanceledException INFO [2019-12-17 12:28:33,821] jepsen worker 24 - jepsen.util 1374 :invoke :read [8 nil] INFO [2019-12-17 12:28:33,831] jepsen worker 45 - jepsen.util 1215 :ok :write [15 34] INFO [2019-12-17 12:28:33,831] jepsen worker 45 - jepsen.util 1215 :invoke :write [15 6] INFO [2019-12-17 12:28:33,832] jepsen worker 13 - jepsen.util 1183 :fail :cas [4 [32 24]] :ValueNotSwapFrom INFO [2019-12-17 12:28:33,832] jepsen worker 13 - jepsen.util 1183 :invoke :read [4 nil] INFO [2019-12-17 12:28:33,832] jepsen worker 10 - jepsen.util 1270 :ok :write [3 48] INFO [2019-12-17 12:28:33,832] jepsen worker 10 - jepsen.util 1270 :invoke :cas [3 [19 4]] INFO [2019-12-17 12:28:33,839] jepsen worker 28 - jepsen.util 838 :ok :read [9 41] INFO [2019-12-17 12:28:33,839] jepsen worker 28 - jepsen.util 838 :invoke :cas [9 [4 35]] INFO [2019-12-17 12:28:33,840] jepsen worker 73 - jepsen.util 1153 :ok :write [24 11] INFO [2019-12-17 12:28:33,840] jepsen worker 73 - jepsen.util 1153 :invoke :write [24 45] INFO [2019-12-17 12:28:33,874] jepsen worker 78 - jepsen.util 1248 :ok :write [26 45] INFO [2019-12-17 12:28:33,874] jepsen worker 78 - jepsen.util 1248 :invoke :read [26 nil] INFO [2019-12-17 12:28:33,878] jepsen worker 69 - jepsen.util 1059 :ok :read [23 39] INFO [2019-12-17 12:28:33,879] jepsen worker 69 - jepsen.util 1059 :invoke :read [23 nil] INFO [2019-12-17 12:28:33,890] jepsen worker 13 - jepsen.util 1183 :ok :read [4 5] INFO [2019-12-17 12:28:33,890] jepsen worker 13 - jepsen.util 1183 :invoke :read [4 nil] INFO [2019-12-17 12:28:33,898] jepsen worker 72 - jepsen.util 1152 :ok :read [24 6] INFO [2019-12-17 12:28:33,898] jepsen worker 72 - jepsen.util 1152 :invoke :read [24 nil] INFO [2019-12-17 12:28:33,907] jepsen worker 28 - jepsen.util 838 :fail :cas [9 [4 35]] :ValueNotSwapFrom INFO [2019-12-17 12:28:33,907] jepsen worker 28 - jepsen.util 838 :invoke :read [9 nil] INFO [2019-12-17 12:28:33,920] jepsen worker 10 - jepsen.util 1270 :fail :cas [3 [19 4]] :ValueNotSwapFrom INFO [2019-12-17 12:28:33,920] jepsen worker 10 - jepsen.util 1270 :invoke :read [3 nil] INFO [2019-12-17 12:28:33,921] jepsen worker 62 - jepsen.util 1322 :info :write [20 38] :RequestCanceledException INFO [2019-12-17 12:28:33,922] jepsen worker 62 - jepsen.util 1412 :invoke :cas [20 [20 20]] INFO [2019-12-17 12:28:33,950] jepsen worker 88 - jepsen.util 1168 :fail :cas [29 [27 48]] :ValueNotSwapFrom INFO [2019-12-17 12:28:33,950] jepsen worker 88 - jepsen.util 1168 :invoke :cas [29 [31 43]] INFO [2019-12-17 12:28:33,950] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":6455,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:28:33,950] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":6455,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:28:33,951] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":6458,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:28:33,951] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":6458,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:28:33,951] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":6459,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:28:33,951] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":6459,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:28:33,964] jepsen worker 53 - jepsen.util 1133 :ok :write [17 42] INFO [2019-12-17 12:28:33,973] jepsen worker 69 - jepsen.util 1059 :ok :read [23 39] INFO [2019-12-17 12:28:33,974] jepsen worker 28 - jepsen.util 838 :ok :read [9 41] INFO [2019-12-17 12:28:33,982] jepsen worker 53 - jepsen.util 1133 :invoke :cas [17 [4 22]] INFO [2019-12-17 12:28:33,982] jepsen worker 69 - jepsen.util 1059 :invoke :cas [23 [33 28]] INFO [2019-12-17 12:28:33,982] jepsen worker 28 - jepsen.util 838 :invoke :cas [9 [21 7]] INFO [2019-12-17 12:28:33,983] jepsen worker 65 - jepsen.util 965 :ok :write [21 38] INFO [2019-12-17 12:28:33,983] jepsen worker 72 - jepsen.util 1152 :ok :read [24 11] INFO [2019-12-17 12:28:33,984] jepsen worker 72 - jepsen.util 1152 :invoke :read [24 nil] INFO [2019-12-17 12:28:33,984] jepsen worker 65 - jepsen.util 965 :invoke :cas [21 [44 24]] INFO [2019-12-17 12:28:34,010] jepsen worker 24 - jepsen.util 1374 :ok :read [8 14] INFO [2019-12-17 12:28:34,010] jepsen worker 24 - jepsen.util 1374 :invoke :read [8 nil] INFO [2019-12-17 12:28:34,013] jepsen worker 10 - jepsen.util 1270 :ok :read [3 48] INFO [2019-12-17 12:28:34,013] jepsen worker 10 - jepsen.util 1270 :invoke :write [3 36] INFO [2019-12-17 12:28:34,024] jepsen worker 5 - jepsen.util 815 :ok :write [1 21] INFO [2019-12-17 12:28:34,024] jepsen worker 5 - jepsen.util 815 :invoke :read [1 nil] INFO [2019-12-17 12:28:34,029] jepsen worker 32 - jepsen.util 1202 :fail :read [10 nil] :RequestTimeoutException INFO [2019-12-17 12:28:34,029] jepsen worker 47 - jepsen.util 1487 :info :write [15 2] :RequestCanceledException INFO [2019-12-17 12:28:34,029] jepsen worker 32 - jepsen.util 1202 :invoke :cas [10 [40 13]] INFO [2019-12-17 12:28:34,031] jepsen worker 47 - jepsen.util 1577 :invoke :write [15 33] INFO [2019-12-17 12:28:34,037] jepsen worker 28 - jepsen.util 838 :fail :cas [9 [21 7]] :ValueNotSwapFrom INFO [2019-12-17 12:28:34,037] jepsen worker 28 - jepsen.util 838 :invoke :write [9 21] INFO [2019-12-17 12:28:34,059] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":6476,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:28:34,059] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":6476,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:28:34,088] jepsen worker 13 - jepsen.util 1183 :ok :read [4 5] INFO [2019-12-17 12:28:34,097] jepsen worker 65 - jepsen.util 965 :fail :cas [21 [44 24]] :ValueNotSwapFrom INFO [2019-12-17 12:28:34,109] jepsen worker 72 - jepsen.util 1152 :ok :read [24 11] INFO [2019-12-17 12:28:34,124] jepsen worker 73 - jepsen.util 1153 :ok :write [24 45] INFO [2019-12-17 12:28:34,132] jepsen worker 32 - jepsen.util 1202 :fail :cas [10 [40 13]] :ValueNotSwapFrom INFO [2019-12-17 12:28:34,135] jepsen worker 36 - jepsen.util 1206 :ok :cas [12 [38 36]] INFO [2019-12-17 12:28:34,152] jepsen worker 28 - jepsen.util 838 :ok :write [9 21] INFO [2019-12-17 12:28:34,158] jepsen worker 45 - jepsen.util 1215 :ok :write [15 6] INFO [2019-12-17 12:28:34,173] jepsen worker 24 - jepsen.util 1374 :ok :read [8 14] INFO [2019-12-17 12:28:34,177] jepsen worker 31 - jepsen.util 1201 :ok :write [10 24] INFO [2019-12-17 12:28:34,181] jepsen worker 10 - jepsen.util 1270 :ok :write [3 36] INFO [2019-12-17 12:28:34,182] jepsen worker 88 - jepsen.util 1168 :fail :cas [29 [31 43]] :ValueNotSwapFrom INFO [2019-12-17 12:28:34,196] jepsen worker 69 - jepsen.util 1059 :fail :cas [23 [33 28]] :ValueNotSwapFrom INFO [2019-12-17 12:28:34,227] jepsen worker 37 - jepsen.util 1027 :fail :cas [12 [9 23]] :ValueNotSwapFrom INFO [2019-12-17 12:28:34,233] jepsen worker 4 - jepsen.util 994 :ok :read [1 21] INFO [2019-12-17 12:28:34,253] jepsen worker 65 - jepsen.util 965 :invoke :cas [21 [1 26]] INFO [2019-12-17 12:28:34,253] jepsen worker 13 - jepsen.util 1183 :invoke :write [4 19] INFO [2019-12-17 12:28:34,253] jepsen worker 72 - jepsen.util 1152 :invoke :cas [24 [9 1]] INFO [2019-12-17 12:28:34,253] jepsen worker 73 - jepsen.util 1153 :invoke :write [24 24] INFO [2019-12-17 12:28:34,253] jepsen worker 32 - jepsen.util 1202 :invoke :cas [10 [44 1]] INFO [2019-12-17 12:28:34,253] jepsen worker 36 - jepsen.util 1206 :invoke :read [12 nil] INFO [2019-12-17 12:28:34,253] jepsen worker 28 - jepsen.util 838 :invoke :cas [9 [8 23]] INFO [2019-12-17 12:28:34,253] jepsen worker 45 - jepsen.util 1215 :invoke :read [15 nil] INFO [2019-12-17 12:28:34,253] jepsen worker 24 - jepsen.util 1374 :invoke :cas [8 [10 4]] INFO [2019-12-17 12:28:34,253] jepsen worker 31 - jepsen.util 1201 :invoke :cas [10 [36 30]] INFO [2019-12-17 12:28:34,253] jepsen worker 10 - jepsen.util 1270 :invoke :write [3 19] INFO [2019-12-17 12:28:34,253] jepsen worker 88 - jepsen.util 1168 :invoke :read [29 nil] INFO [2019-12-17 12:28:34,253] jepsen worker 69 - jepsen.util 1059 :invoke :cas [23 [34 39]] INFO [2019-12-17 12:28:34,253] jepsen worker 37 - jepsen.util 1027 :invoke :write [12 47] INFO [2019-12-17 12:28:34,253] jepsen worker 4 - jepsen.util 994 :invoke :cas [1 [7 20]] INFO [2019-12-17 12:28:34,268] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:28:34,268] jepsen nemesis - couchbase.util Rebalance check count: 42 INFO [2019-12-17 12:28:34,268] jepsen nemesis - couchbase.util Rebalance stuck count: 23 INFO [2019-12-17 12:28:34,279] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":6492,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:28:34,279] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":6492,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:28:34,280] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":6496,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:28:34,280] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":6496,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:28:34,281] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":6485,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:28:34,282] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":6485,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:28:34,282] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":6486,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:28:34,282] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":6486,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:28:34,287] jepsen worker 70 - jepsen.util 1420 :fail :cas [23 [32 37]] :ValueNotSwapFrom INFO [2019-12-17 12:28:34,287] jepsen worker 70 - jepsen.util 1420 :invoke :write [23 26] INFO [2019-12-17 12:28:34,293] jepsen worker 80 - jepsen.util 1430 :ok :read [26 45] INFO [2019-12-17 12:28:34,300] jepsen worker 80 - jepsen.util 1430 :invoke :cas [26 [19 10]] INFO [2019-12-17 12:28:34,306] jepsen worker 85 - jepsen.util 1345 :info :write [28 30] :RequestCanceledException INFO [2019-12-17 12:28:34,307] jepsen worker 85 - jepsen.util 1435 :invoke :write [28 42] INFO [2019-12-17 12:28:34,322] jepsen worker 56 - jepsen.util 1136 :info :write [18 11] :RequestCanceledException INFO [2019-12-17 12:28:34,322] jepsen worker 82 - jepsen.util 1792 :info :cas [27 [19 33]] :RequestCanceledException INFO [2019-12-17 12:28:34,323] jepsen worker 82 - jepsen.util 1882 :invoke :read [27 nil] INFO [2019-12-17 12:28:34,323] jepsen worker 56 - jepsen.util 1226 :invoke :cas [18 [35 27]] INFO [2019-12-17 12:28:34,325] jepsen worker 5 - jepsen.util 815 :ok :read [1 21] INFO [2019-12-17 12:28:34,325] jepsen worker 5 - jepsen.util 815 :invoke :read [1 nil] INFO [2019-12-17 12:28:34,352] jepsen worker 0 - jepsen.util 1440 :info :write [0 25] :RequestCanceledException INFO [2019-12-17 12:28:34,352] jepsen worker 23 - jepsen.util 1283 :info :cas [7 [5 12]] :RequestCanceledException INFO [2019-12-17 12:28:34,353] jepsen worker 23 - jepsen.util 1373 :invoke :cas [7 [10 21]] INFO [2019-12-17 12:28:34,353] jepsen worker 0 - jepsen.util 1530 :invoke :write [0 40] INFO [2019-12-17 12:28:34,379] jepsen worker 52 - jepsen.util 1222 :info :write [17 15] :RequestCanceledException INFO [2019-12-17 12:28:34,381] jepsen worker 52 - jepsen.util 1312 :invoke :write [17 5] INFO [2019-12-17 12:28:34,382] jepsen worker 79 - jepsen.util 1159 :fail :cas [26 [30 5]] :ValueNotSwapFrom INFO [2019-12-17 12:28:34,382] jepsen worker 24 - jepsen.util 1374 :fail :cas [8 [10 4]] :ValueNotSwapFrom INFO [2019-12-17 12:28:34,382] jepsen worker 79 - jepsen.util 1159 :invoke :read [26 nil] INFO [2019-12-17 12:28:34,382] jepsen worker 24 - jepsen.util 1374 :invoke :cas [8 [9 25]] INFO [2019-12-17 12:28:34,391] jepsen worker 72 - jepsen.util 1152 :fail :cas [24 [9 1]] :ValueNotSwapFrom INFO [2019-12-17 12:28:34,391] jepsen worker 72 - jepsen.util 1152 :invoke :cas [24 [0 43]] INFO [2019-12-17 12:28:34,408] jepsen worker 45 - jepsen.util 1215 :ok :read [15 6] INFO [2019-12-17 12:28:34,408] jepsen worker 45 - jepsen.util 1215 :invoke :write [15 3] INFO [2019-12-17 12:28:34,410] jepsen worker 32 - jepsen.util 1202 :fail :cas [10 [44 1]] :ValueNotSwapFrom INFO [2019-12-17 12:28:34,410] jepsen worker 32 - jepsen.util 1202 :invoke :write [10 19] INFO [2019-12-17 12:28:34,421] jepsen worker 57 - jepsen.util 1407 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:28:34,421] jepsen worker 59 - jepsen.util 1229 :info :cas [19 [38 49]] :RequestCanceledException INFO [2019-12-17 12:28:34,421] jepsen worker 83 - jepsen.util 1703 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:28:34,422] jepsen worker 83 - jepsen.util 1703 :invoke :write [27 27] INFO [2019-12-17 12:28:34,422] jepsen worker 57 - jepsen.util 1407 :invoke :write [19 39] INFO [2019-12-17 12:28:34,422] jepsen worker 16 - jepsen.util 1366 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:28:34,422] jepsen worker 16 - jepsen.util 1366 :invoke :cas [5 [14 22]] INFO [2019-12-17 12:28:34,423] jepsen worker 65 - jepsen.util 965 :fail :cas [21 [1 26]] :ValueNotSwapFrom INFO [2019-12-17 12:28:34,423] jepsen worker 65 - jepsen.util 965 :invoke :read [21 nil] INFO [2019-12-17 12:28:34,423] jepsen worker 59 - jepsen.util 1319 :invoke :read [19 nil] INFO [2019-12-17 12:28:34,428] jepsen worker 28 - jepsen.util 838 :fail :cas [9 [8 23]] :ValueNotSwapFrom INFO [2019-12-17 12:28:34,428] jepsen worker 28 - jepsen.util 838 :invoke :write [9 4] INFO [2019-12-17 12:28:34,436] jepsen worker 50 - jepsen.util 1400 :info :cas [16 [44 35]] :RequestCanceledException INFO [2019-12-17 12:28:34,437] jepsen worker 50 - jepsen.util 1490 :invoke :read [16 nil] INFO [2019-12-17 12:28:34,441] jepsen worker 31 - jepsen.util 1201 :fail :cas [10 [36 30]] :ValueNotSwapFrom INFO [2019-12-17 12:28:34,441] jepsen worker 31 - jepsen.util 1201 :invoke :read [10 nil] INFO [2019-12-17 12:28:34,453] jepsen worker 17 - jepsen.util 1367 :info :cas [5 [46 1]] :RequestCanceledException INFO [2019-12-17 12:28:34,453] jepsen worker 84 - jepsen.util 1254 :info :cas [28 [16 31]] :RequestCanceledException INFO [2019-12-17 12:28:34,454] jepsen worker 17 - jepsen.util 1457 :invoke :read [5 nil] INFO [2019-12-17 12:28:34,454] jepsen worker 84 - jepsen.util 1344 :invoke :write [28 28] INFO [2019-12-17 12:28:34,463] jepsen worker 23 - jepsen.util 1373 :fail :cas [7 [10 21]] :ValueNotSwapFrom INFO [2019-12-17 12:28:34,463] jepsen worker 23 - jepsen.util 1373 :invoke :write [7 20] INFO [2019-12-17 12:28:34,465] jepsen worker 65 - jepsen.util 965 :ok :read [21 38] INFO [2019-12-17 12:28:34,466] jepsen worker 65 - jepsen.util 965 :invoke :cas [21 [26 38]] INFO [2019-12-17 12:28:34,487] jepsen worker 24 - jepsen.util 1374 :fail :cas [8 [9 25]] :ValueNotSwapFrom INFO [2019-12-17 12:28:34,487] jepsen worker 24 - jepsen.util 1374 :invoke :write [8 37] INFO [2019-12-17 12:28:34,488] jepsen worker 40 - jepsen.util 1750 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:28:34,488] jepsen worker 66 - jepsen.util 1056 :info :write [22 4] :RequestCanceledException INFO [2019-12-17 12:28:34,488] jepsen worker 68 - jepsen.util 1058 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:28:34,488] jepsen worker 48 - jepsen.util 858 :info :write [16 13] :RequestCanceledException INFO [2019-12-17 12:28:34,489] jepsen worker 40 - jepsen.util 1750 :invoke :read [13 nil] INFO [2019-12-17 12:28:34,489] jepsen worker 68 - jepsen.util 1058 :invoke :read [22 nil] INFO [2019-12-17 12:28:34,489] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":6530,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:28:34,489] jepsen worker 36 - jepsen.util 1206 :ok :read [12 36] INFO [2019-12-17 12:28:34,489] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":6530,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:28:34,489] jepsen worker 36 - jepsen.util 1206 :invoke :read [12 nil] INFO [2019-12-17 12:28:34,489] jepsen worker 78 - jepsen.util 1248 :ok :read [26 45] INFO [2019-12-17 12:28:34,489] jepsen worker 78 - jepsen.util 1248 :invoke :read [26 nil] INFO [2019-12-17 12:28:34,490] jepsen worker 66 - jepsen.util 1146 :invoke :cas [22 [39 47]] INFO [2019-12-17 12:28:34,491] jepsen worker 48 - jepsen.util 948 :invoke :write [16 16] INFO [2019-12-17 12:28:34,501] jepsen worker 80 - jepsen.util 1430 :fail :cas [26 [19 10]] :ValueNotSwapFrom INFO [2019-12-17 12:28:34,503] jepsen worker 80 - jepsen.util 1430 :invoke :cas [26 [12 48]] INFO [2019-12-17 12:28:34,506] jepsen worker 86 - jepsen.util 1436 :info :cas [28 [26 24]] :RequestCanceledException INFO [2019-12-17 12:28:34,506] jepsen worker 60 - jepsen.util 1500 :info :cas [20 [21 9]] :RequestCanceledException INFO [2019-12-17 12:28:34,507] jepsen worker 60 - jepsen.util 1590 :invoke :cas [20 [6 28]] INFO [2019-12-17 12:28:34,507] jepsen worker 86 - jepsen.util 1526 :invoke :write [28 11] INFO [2019-12-17 12:28:34,521] jepsen worker 31 - jepsen.util 1201 :ok :read [10 24] INFO [2019-12-17 12:28:34,521] jepsen worker 49 - jepsen.util 1129 :info :write [16 41] :RequestCanceledException INFO [2019-12-17 12:28:34,521] jepsen worker 39 - jepsen.util 1479 :info :write [13 16] :RequestCanceledException INFO [2019-12-17 12:28:34,521] jepsen worker 67 - jepsen.util 1687 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:28:34,521] jepsen worker 2 - jepsen.util 1172 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:28:34,521] jepsen worker 1 - jepsen.util 1261 :info :cas [0 [15 38]] :RequestCanceledException INFO [2019-12-17 12:28:34,521] jepsen worker 25 - jepsen.util 1555 :info :cas [8 [14 20]] :RequestCanceledException INFO [2019-12-17 12:28:34,521] jepsen worker 31 - jepsen.util 1201 :invoke :write [10 38] INFO [2019-12-17 12:28:34,521] jepsen worker 67 - jepsen.util 1687 :invoke :write [22 48] INFO [2019-12-17 12:28:34,521] jepsen worker 2 - jepsen.util 1172 :invoke :write [0 39] INFO [2019-12-17 12:28:34,522] jepsen worker 49 - jepsen.util 1219 :invoke :cas [16 [22 35]] INFO [2019-12-17 12:28:34,522] jepsen worker 39 - jepsen.util 1569 :invoke :write [13 36] INFO [2019-12-17 12:28:34,522] jepsen worker 25 - jepsen.util 1645 :invoke :write [8 26] INFO [2019-12-17 12:28:34,522] jepsen worker 1 - jepsen.util 1351 :invoke :cas [0 [42 42]] INFO [2019-12-17 12:28:34,533] jepsen worker 65 - jepsen.util 965 :fail :cas [21 [26 38]] :ValueNotSwapFrom INFO [2019-12-17 12:28:34,533] jepsen worker 65 - jepsen.util 965 :invoke :write [21 3] INFO [2019-12-17 12:28:34,535] jepsen worker 41 - jepsen.util 1121 :info :cas [13 [25 44]] :RequestCanceledException INFO [2019-12-17 12:28:34,535] jepsen worker 75 - jepsen.util 1515 :info :write [25 36] :RequestCanceledException INFO [2019-12-17 12:28:34,535] jepsen worker 71 - jepsen.util 791 :info :write [23 33] :RequestCanceledException INFO [2019-12-17 12:28:34,535] jepsen worker 15 - jepsen.util 1005 :info :cas [5 [2 41]] :RequestCanceledException INFO [2019-12-17 12:28:34,536] jepsen worker 41 - jepsen.util 1211 :invoke :write [13 26] INFO [2019-12-17 12:28:34,536] jepsen worker 71 - jepsen.util 881 :invoke :write [23 44] INFO [2019-12-17 12:28:34,536] jepsen worker 75 - jepsen.util 1605 :invoke :cas [25 [2 15]] INFO [2019-12-17 12:28:34,536] jepsen worker 15 - jepsen.util 1095 :invoke :cas [5 [39 49]] INFO [2019-12-17 12:28:34,542] jepsen worker 60 - jepsen.util 1590 :fail :cas [20 [6 28]] :ValueNotSwapFrom INFO [2019-12-17 12:28:34,543] jepsen worker 60 - jepsen.util 1590 :invoke :cas [20 [27 19]] INFO [2019-12-17 12:28:34,550] jepsen worker 53 - jepsen.util 1133 :fail :cas [17 [4 22]] :ValueNotSwapFrom INFO [2019-12-17 12:28:34,550] jepsen worker 53 - jepsen.util 1133 :invoke :cas [17 [20 44]] INFO [2019-12-17 12:28:34,581] jepsen worker 4 - jepsen.util 994 :fail :cas [1 [7 20]] :ValueNotSwapFrom INFO [2019-12-17 12:28:34,581] jepsen worker 4 - jepsen.util 994 :invoke :write [1 42] INFO [2019-12-17 12:28:34,586] jepsen worker 88 - jepsen.util 1168 :ok :read [29 12] INFO [2019-12-17 12:28:34,586] jepsen worker 88 - jepsen.util 1168 :invoke :cas [29 [43 38]] INFO [2019-12-17 12:28:34,588] jepsen worker 81 - jepsen.util 1521 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:28:34,588] jepsen worker 81 - jepsen.util 1521 :invoke :read [27 nil] INFO [2019-12-17 12:28:34,600] jepsen worker 65 - jepsen.util 965 :ok :write [21 3] INFO [2019-12-17 12:28:34,606] jepsen worker 42 - jepsen.util 1662 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:28:34,612] jepsen worker 65 - jepsen.util 965 :invoke :cas [21 [28 38]] INFO [2019-12-17 12:28:34,612] jepsen worker 42 - jepsen.util 1662 :invoke :cas [14 [3 35]] INFO [2019-12-17 12:28:34,614] jepsen worker 69 - jepsen.util 1059 :fail :cas [23 [34 39]] :ValueNotSwapFrom INFO [2019-12-17 12:28:34,614] jepsen worker 69 - jepsen.util 1059 :invoke :cas [23 [17 39]] INFO [2019-12-17 12:28:34,621] jepsen worker 61 - jepsen.util 1411 :info :write [20 42] :RequestCanceledException INFO [2019-12-17 12:28:34,622] jepsen worker 61 - jepsen.util 1501 :invoke :cas [20 [23 2]] INFO [2019-12-17 12:28:34,628] jepsen worker 65 - jepsen.util 965 :fail :cas [21 [28 38]] :ValueNotSwapFrom INFO [2019-12-17 12:28:34,628] jepsen worker 65 - jepsen.util 965 :invoke :write [21 39] INFO [2019-12-17 12:28:34,667] jepsen worker 65 - jepsen.util 965 :ok :write [21 39] INFO [2019-12-17 12:28:34,667] jepsen worker 65 - jepsen.util 965 :invoke :cas [21 [25 44]] INFO [2019-12-17 12:28:34,670] jepsen worker 60 - jepsen.util 1590 :fail :cas [20 [27 19]] :ValueNotSwapFrom INFO [2019-12-17 12:28:34,671] jepsen worker 60 - jepsen.util 1590 :invoke :write [20 32] INFO [2019-12-17 12:28:34,682] jepsen worker 65 - jepsen.util 965 :fail :cas [21 [25 44]] :ValueNotSwapFrom INFO [2019-12-17 12:28:34,682] jepsen worker 65 - jepsen.util 965 :invoke :cas [21 [14 20]] INFO [2019-12-17 12:28:34,702] jepsen worker 65 - jepsen.util 965 :fail :cas [21 [14 20]] :ValueNotSwapFrom INFO [2019-12-17 12:28:34,710] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":6577,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:28:34,721] jepsen worker 58 - jepsen.util 1408 :info :cas [19 [26 7]] :RequestCanceledException INFO [2019-12-17 12:28:34,728] jepsen worker 56 - jepsen.util 1226 :fail :cas [18 [35 27]] :ValueNotSwapFrom INFO [2019-12-17 12:28:34,731] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":6577,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:28:34,731] jepsen worker 65 - jepsen.util 965 :invoke :read [21 nil] INFO [2019-12-17 12:28:34,731] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":6580,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:28:34,731] jepsen worker 56 - jepsen.util 1226 :invoke :cas [18 [46 43]] INFO [2019-12-17 12:28:34,731] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":6580,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:28:34,731] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":6581,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:28:34,731] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":6581,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:28:34,732] jepsen worker 58 - jepsen.util 1498 :invoke :read [19 nil] INFO [2019-12-17 12:28:34,736] jepsen worker 33 - jepsen.util 1653 :info :write [11 45] :RequestCanceledException INFO [2019-12-17 12:28:34,737] jepsen worker 33 - jepsen.util 1743 :invoke :read [11 nil] INFO [2019-12-17 12:28:34,749] jepsen worker 65 - jepsen.util 965 :ok :read [21 39] INFO [2019-12-17 12:28:34,750] jepsen worker 65 - jepsen.util 965 :invoke :write [21 9] INFO [2019-12-17 12:28:34,758] jepsen worker 88 - jepsen.util 1168 :fail :cas [29 [43 38]] :ValueNotSwapFrom INFO [2019-12-17 12:28:34,759] jepsen worker 88 - jepsen.util 1168 :invoke :read [29 nil] INFO [2019-12-17 12:28:34,801] jepsen worker 65 - jepsen.util 965 :ok :write [21 9] INFO [2019-12-17 12:28:34,801] jepsen worker 65 - jepsen.util 965 :invoke :read [21 nil] INFO [2019-12-17 12:28:34,807] jepsen worker 69 - jepsen.util 1059 :fail :cas [23 [17 39]] :ValueNotSwapFrom INFO [2019-12-17 12:28:34,807] jepsen worker 69 - jepsen.util 1059 :invoke :read [23 nil] INFO [2019-12-17 12:28:34,837] jepsen worker 65 - jepsen.util 965 :ok :read [21 9] INFO [2019-12-17 12:28:34,838] jepsen worker 65 - jepsen.util 965 :invoke :cas [21 [7 3]] INFO [2019-12-17 12:28:34,853] jepsen worker 38 - jepsen.util 1028 :info :write [12 41] :RequestCanceledException INFO [2019-12-17 12:28:34,855] jepsen worker 38 - jepsen.util 1118 :invoke :cas [12 [18 6]] INFO [2019-12-17 12:28:34,857] jepsen worker 65 - jepsen.util 965 :fail :cas [21 [7 3]] :ValueNotSwapFrom INFO [2019-12-17 12:28:34,857] jepsen worker 65 - jepsen.util 965 :invoke :write [21 14] INFO [2019-12-17 12:28:34,919] jepsen worker 65 - jepsen.util 965 :ok :write [21 14] INFO [2019-12-17 12:28:34,919] jepsen worker 65 - jepsen.util 965 :invoke :cas [21 [8 1]] INFO [2019-12-17 12:28:34,922] jepsen worker 64 - jepsen.util 694 :info :write [21 45] :RequestCanceledException INFO [2019-12-17 12:28:34,924] jepsen worker 64 - jepsen.util 784 :invoke :read [21 nil] INFO [2019-12-17 12:28:34,928] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":6597,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:28:34,928] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":6597,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:28:34,932] jepsen worker 35 - jepsen.util 1115 :info :write [11 35] :RequestCanceledException INFO [2019-12-17 12:28:34,932] jepsen worker 77 - jepsen.util 1337 :info :write [25 4] :RequestCanceledException INFO [2019-12-17 12:28:34,933] jepsen worker 35 - jepsen.util 1205 :invoke :read [11 nil] INFO [2019-12-17 12:28:34,933] jepsen worker 77 - jepsen.util 1427 :invoke :cas [25 [20 5]] INFO [2019-12-17 12:28:34,951] jepsen worker 14 - jepsen.util 1004 :info :write [4 27] :RequestCanceledException INFO [2019-12-17 12:28:34,952] jepsen worker 14 - jepsen.util 1094 :invoke :write [4 25] INFO [2019-12-17 12:28:34,962] jepsen worker 36 - jepsen.util 1206 :ok :read [12 36] INFO [2019-12-17 12:28:34,962] jepsen worker 36 - jepsen.util 1206 :invoke :write [12 19] INFO [2019-12-17 12:28:34,967] jepsen worker 64 - jepsen.util 784 :ok :read [21 14] INFO [2019-12-17 12:28:34,967] jepsen worker 64 - jepsen.util 784 :invoke :cas [21 [25 2]] INFO [2019-12-17 12:28:34,970] jepsen worker 65 - jepsen.util 965 :fail :cas [21 [8 1]] :ValueNotSwapFrom INFO [2019-12-17 12:28:34,970] jepsen worker 65 - jepsen.util 965 :invoke :cas [21 [44 24]] INFO [2019-12-17 12:28:34,990] jepsen worker 33 - jepsen.util 1743 :ok :read [11 7] INFO [2019-12-17 12:28:34,990] jepsen worker 33 - jepsen.util 1743 :invoke :write [11 25] INFO [2019-12-17 12:28:34,997] jepsen worker 65 - jepsen.util 965 :fail :cas [21 [44 24]] :ValueNotSwapFrom INFO [2019-12-17 12:28:34,997] jepsen worker 65 - jepsen.util 965 :invoke :read [21 nil] INFO [2019-12-17 12:28:35,002] jepsen worker 46 - jepsen.util 1396 :info :write [15 22] :RequestCanceledException INFO [2019-12-17 12:28:35,003] jepsen worker 46 - jepsen.util 1486 :invoke :write [15 37] INFO [2019-12-17 12:28:35,005] jepsen worker 64 - jepsen.util 784 :fail :cas [21 [25 2]] :ValueNotSwapFrom INFO [2019-12-17 12:28:35,005] jepsen worker 64 - jepsen.util 784 :invoke :cas [21 [41 19]] INFO [2019-12-17 12:28:35,021] jepsen worker 55 - jepsen.util 1585 :info :write [18 32] :RequestCanceledException INFO [2019-12-17 12:28:35,021] jepsen worker 64 - jepsen.util 784 :fail :cas [21 [41 19]] :ValueNotSwapFrom INFO [2019-12-17 12:28:35,022] jepsen worker 64 - jepsen.util 784 :invoke :cas [21 [16 37]] INFO [2019-12-17 12:28:35,023] jepsen worker 55 - jepsen.util 1675 :invoke :read [18 nil] INFO [2019-12-17 12:28:35,035] jepsen worker 27 - jepsen.util 927 :info :write [9 42] :RequestCanceledException INFO [2019-12-17 12:28:35,035] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":6611,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:28:35,035] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":6611,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:28:35,035] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":6615,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:28:35,035] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":6615,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:28:35,039] jepsen worker 27 - jepsen.util 1017 :invoke :write [9 49] INFO [2019-12-17 12:28:35,052] jepsen worker 65 - jepsen.util 965 :ok :read [21 14] INFO [2019-12-17 12:28:35,052] jepsen worker 65 - jepsen.util 965 :invoke :cas [21 [49 2]] INFO [2019-12-17 12:28:35,063] jepsen worker 56 - jepsen.util 1226 :fail :cas [18 [46 43]] :ValueNotSwapFrom INFO [2019-12-17 12:28:35,063] jepsen worker 56 - jepsen.util 1226 :invoke :read [18 nil] INFO [2019-12-17 12:28:35,077] jepsen worker 64 - jepsen.util 784 :fail :cas [21 [16 37]] :ValueNotSwapFrom INFO [2019-12-17 12:28:35,078] jepsen worker 64 - jepsen.util 784 :invoke :write [21 13] INFO [2019-12-17 12:28:35,084] jepsen worker 65 - jepsen.util 965 :fail :cas [21 [49 2]] :ValueNotSwapFrom INFO [2019-12-17 12:28:35,085] jepsen worker 65 - jepsen.util 965 :invoke :cas [21 [29 36]] INFO [2019-12-17 12:28:35,100] jepsen worker 65 - jepsen.util 965 :fail :cas [21 [29 36]] :ValueNotSwapFrom INFO [2019-12-17 12:28:35,100] jepsen worker 5 - jepsen.util 815 :ok :read [1 21] INFO [2019-12-17 12:28:35,100] jepsen worker 65 - jepsen.util 965 :invoke :read [21 nil] INFO [2019-12-17 12:28:35,100] jepsen worker 5 - jepsen.util 815 :invoke :cas [1 [10 43]] INFO [2019-12-17 12:28:35,106] jepsen worker 54 - jepsen.util 1314 :info :write [18 32] :RequestCanceledException INFO [2019-12-17 12:28:35,107] jepsen worker 54 - jepsen.util 1404 :invoke :cas [18 [47 3]] INFO [2019-12-17 12:28:35,113] jepsen worker 65 - jepsen.util 965 :ok :read [21 14] INFO [2019-12-17 12:28:35,113] jepsen worker 65 - jepsen.util 965 :invoke :cas [21 [37 41]] INFO [2019-12-17 12:28:35,124] jepsen worker 27 - jepsen.util 1017 :ok :write [9 49] INFO [2019-12-17 12:28:35,124] jepsen worker 27 - jepsen.util 1017 :invoke :write [9 44] INFO [2019-12-17 12:28:35,132] jepsen worker 79 - jepsen.util 1159 :ok :read [26 45] INFO [2019-12-17 12:28:35,132] jepsen worker 79 - jepsen.util 1159 :invoke :write [26 0] INFO [2019-12-17 12:28:35,137] jepsen worker 65 - jepsen.util 965 :fail :cas [21 [37 41]] :ValueNotSwapFrom INFO [2019-12-17 12:28:35,137] jepsen worker 65 - jepsen.util 965 :invoke :cas [21 [0 30]] INFO [2019-12-17 12:28:35,140] jepsen worker 35 - jepsen.util 1205 :ok :read [11 7] INFO [2019-12-17 12:28:35,140] jepsen worker 35 - jepsen.util 1205 :invoke :read [11 nil] INFO [2019-12-17 12:28:35,162] jepsen worker 78 - jepsen.util 1248 :ok :read [26 45] INFO [2019-12-17 12:28:35,162] jepsen worker 78 - jepsen.util 1248 :invoke :write [26 32] INFO [2019-12-17 12:28:35,172] jepsen worker 65 - jepsen.util 965 :fail :cas [21 [0 30]] :ValueNotSwapFrom INFO [2019-12-17 12:28:35,172] jepsen worker 65 - jepsen.util 965 :invoke :cas [21 [38 25]] INFO [2019-12-17 12:28:35,187] jepsen worker 26 - jepsen.util 1376 :info :write [8 10] :RequestCanceledException INFO [2019-12-17 12:28:35,189] jepsen worker 26 - jepsen.util 1466 :invoke :write [8 31] INFO [2019-12-17 12:28:35,193] jepsen worker 35 - jepsen.util 1205 :ok :read [11 7] INFO [2019-12-17 12:28:35,194] jepsen worker 35 - jepsen.util 1205 :invoke :cas [11 [10 46]] INFO [2019-12-17 12:28:35,195] jepsen worker 65 - jepsen.util 965 :fail :cas [21 [38 25]] :ValueNotSwapFrom INFO [2019-12-17 12:28:35,196] jepsen worker 65 - jepsen.util 965 :invoke :read [21 nil] INFO [2019-12-17 12:28:35,201] jepsen worker 64 - jepsen.util 784 :ok :write [21 13] INFO [2019-12-17 12:28:35,201] jepsen worker 64 - jepsen.util 784 :invoke :write [21 33] INFO [2019-12-17 12:28:35,204] jepsen worker 3 - jepsen.util 993 :info :write [1 40] :RequestCanceledException INFO [2019-12-17 12:28:35,205] jepsen worker 80 - jepsen.util 1430 :fail :cas [26 [12 48]] :ValueNotSwapFrom INFO [2019-12-17 12:28:35,205] jepsen worker 80 - jepsen.util 1430 :invoke :cas [26 [3 20]] INFO [2019-12-17 12:28:35,205] jepsen worker 3 - jepsen.util 1083 :invoke :cas [1 [35 36]] INFO [2019-12-17 12:28:35,207] jepsen worker 53 - jepsen.util 1133 :fail :cas [17 [20 44]] :ValueNotSwapFrom INFO [2019-12-17 12:28:35,207] jepsen worker 53 - jepsen.util 1133 :invoke :write [17 44] INFO [2019-12-17 12:28:35,215] jepsen worker 65 - jepsen.util 965 :ok :read [21 14] INFO [2019-12-17 12:28:35,215] jepsen worker 65 - jepsen.util 965 :invoke :read [21 nil] INFO [2019-12-17 12:28:35,221] jepsen worker 8 - jepsen.util 1358 :info :write [2 17] :RequestCanceledException INFO [2019-12-17 12:28:35,228] jepsen worker 8 - jepsen.util 1448 :invoke :read [2 nil] INFO [2019-12-17 12:28:35,249] jepsen worker 65 - jepsen.util 965 :ok :read [21 13] INFO [2019-12-17 12:28:35,250] jepsen worker 65 - jepsen.util 965 :invoke :write [21 30] INFO [2019-12-17 12:28:35,252] jepsen worker 35 - jepsen.util 1205 :fail :cas [11 [10 46]] :ValueNotSwapFrom INFO [2019-12-17 12:28:35,252] jepsen worker 35 - jepsen.util 1205 :invoke :read [11 nil] INFO [2019-12-17 12:28:35,254] jepsen worker 5 - jepsen.util 815 :fail :cas [1 [10 43]] :ValueNotSwapFrom INFO [2019-12-17 12:28:35,255] jepsen worker 5 - jepsen.util 815 :invoke :write [1 14] INFO [2019-12-17 12:28:35,256] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":6648,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:28:35,256] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":6648,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:28:35,256] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":6651,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:28:35,257] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":6651,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:28:35,261] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":6637,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:28:35,261] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":6637,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:28:35,278] jepsen worker 64 - jepsen.util 784 :ok :write [21 33] INFO [2019-12-17 12:28:35,279] jepsen worker 64 - jepsen.util 784 :invoke :cas [21 [37 32]] INFO [2019-12-17 12:28:35,286] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:28:35,286] jepsen nemesis - couchbase.util Rebalance check count: 43 INFO [2019-12-17 12:28:35,286] jepsen nemesis - couchbase.util Rebalance stuck count: 24 INFO [2019-12-17 12:28:35,288] jepsen worker 51 - jepsen.util 1131 :info :write [17 34] :RequestCanceledException INFO [2019-12-17 12:28:35,291] jepsen worker 51 - jepsen.util 1221 :invoke :cas [17 [32 7]] INFO [2019-12-17 12:28:35,297] jepsen worker 64 - jepsen.util 784 :fail :cas [21 [37 32]] :ValueNotSwapFrom INFO [2019-12-17 12:28:35,297] jepsen worker 64 - jepsen.util 784 :invoke :read [21 nil] INFO [2019-12-17 12:28:35,301] jepsen worker 10 - jepsen.util 1270 :ok :write [3 19] INFO [2019-12-17 12:28:35,301] jepsen worker 10 - jepsen.util 1270 :invoke :write [3 22] INFO [2019-12-17 12:28:35,318] jepsen worker 56 - jepsen.util 1226 :ok :read [18 17] INFO [2019-12-17 12:28:35,318] jepsen worker 56 - jepsen.util 1226 :invoke :cas [18 [1 34]] INFO [2019-12-17 12:28:35,319] jepsen worker 12 - jepsen.util 1182 :info :write [4 44] :RequestCanceledException INFO [2019-12-17 12:28:35,320] jepsen worker 12 - jepsen.util 1272 :invoke :read [4 nil] INFO [2019-12-17 12:28:35,339] jepsen worker 64 - jepsen.util 784 :ok :read [21 33] INFO [2019-12-17 12:28:35,339] jepsen worker 64 - jepsen.util 784 :invoke :cas [21 [15 31]] INFO [2019-12-17 12:28:35,348] jepsen worker 65 - jepsen.util 965 :ok :write [21 30] INFO [2019-12-17 12:28:35,348] jepsen worker 65 - jepsen.util 965 :invoke :write [21 9] INFO [2019-12-17 12:28:35,353] jepsen worker 76 - jepsen.util 1426 :info :cas [25 [30 28]] :RequestCanceledException INFO [2019-12-17 12:28:35,353] jepsen worker 6 - jepsen.util 1446 :info :write [2 17] :RequestCanceledException INFO [2019-12-17 12:28:35,354] jepsen worker 76 - jepsen.util 1516 :invoke :read [25 nil] INFO [2019-12-17 12:28:35,354] jepsen worker 6 - jepsen.util 1536 :invoke :cas [2 [32 11]] INFO [2019-12-17 12:28:35,369] jepsen worker 8 - jepsen.util 1448 :ok :read [2 15] INFO [2019-12-17 12:28:35,369] jepsen worker 8 - jepsen.util 1448 :invoke :read [2 nil] INFO [2019-12-17 12:28:35,373] jepsen worker 51 - jepsen.util 1221 :fail :cas [17 [32 7]] :ValueNotSwapFrom INFO [2019-12-17 12:28:35,373] jepsen worker 51 - jepsen.util 1221 :invoke :cas [17 [44 36]] INFO [2019-12-17 12:28:35,374] jepsen worker 35 - jepsen.util 1205 :ok :read [11 7] INFO [2019-12-17 12:28:35,374] jepsen worker 35 - jepsen.util 1205 :invoke :cas [11 [10 41]] INFO [2019-12-17 12:28:35,388] jepsen worker 74 - jepsen.util 1064 :info :write [24 27] :RequestCanceledException INFO [2019-12-17 12:28:35,389] jepsen worker 64 - jepsen.util 784 :fail :cas [21 [15 31]] :ValueNotSwapFrom INFO [2019-12-17 12:28:35,389] jepsen worker 64 - jepsen.util 784 :invoke :write [21 45] INFO [2019-12-17 12:28:35,389] jepsen worker 74 - jepsen.util 1154 :invoke :cas [24 [12 15]] INFO [2019-12-17 12:28:35,404] jepsen worker 88 - jepsen.util 1168 :ok :read [29 12] INFO [2019-12-17 12:28:35,405] jepsen worker 88 - jepsen.util 1168 :invoke :cas [29 [21 16]] INFO [2019-12-17 12:28:35,405] jepsen worker 9 - jepsen.util 1179 :info :cas [3 [41 0]] :RequestCanceledException INFO [2019-12-17 12:28:35,406] jepsen worker 9 - jepsen.util 1269 :invoke :read [3 nil] INFO [2019-12-17 12:28:35,423] jepsen worker 69 - jepsen.util 1059 :ok :read [23 39] INFO [2019-12-17 12:28:35,423] jepsen worker 69 - jepsen.util 1059 :invoke :read [23 nil] INFO [2019-12-17 12:28:35,444] jepsen worker 65 - jepsen.util 965 :ok :write [21 9] INFO [2019-12-17 12:28:35,444] jepsen worker 65 - jepsen.util 965 :invoke :cas [21 [39 28]] INFO [2019-12-17 12:28:35,452] jepsen worker 87 - jepsen.util 1167 :info :write [29 17] :RequestCanceledException INFO [2019-12-17 12:28:35,454] jepsen worker 87 - jepsen.util 1257 :invoke :read [29 nil] INFO [2019-12-17 12:28:35,460] jepsen worker 8 - jepsen.util 1448 :ok :read [2 15] INFO [2019-12-17 12:28:35,460] jepsen worker 8 - jepsen.util 1448 :invoke :cas [2 [1 45]] INFO [2019-12-17 12:28:35,462] jepsen worker 65 - jepsen.util 965 :fail :cas [21 [39 28]] :ValueNotSwapFrom INFO [2019-12-17 12:28:35,462] jepsen worker 65 - jepsen.util 965 :invoke :read [21 nil] INFO [2019-12-17 12:28:35,474] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":6679,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:28:35,474] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":6679,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:28:35,482] jepsen worker 65 - jepsen.util 965 :ok :read [21 9] INFO [2019-12-17 12:28:35,483] jepsen worker 65 - jepsen.util 965 :invoke :cas [21 [45 37]] INFO [2019-12-17 12:28:35,497] jepsen worker 38 - jepsen.util 1118 :fail :cas [12 [18 6]] :ValueNotSwapFrom INFO [2019-12-17 12:28:35,497] jepsen worker 38 - jepsen.util 1118 :invoke :write [12 8] INFO [2019-12-17 12:28:35,500] jepsen worker 65 - jepsen.util 965 :fail :cas [21 [45 37]] :ValueNotSwapFrom INFO [2019-12-17 12:28:35,500] jepsen worker 65 - jepsen.util 965 :invoke :write [21 16] INFO [2019-12-17 12:28:35,506] jepsen worker 44 - jepsen.util 1394 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:28:35,506] jepsen worker 44 - jepsen.util 1394 :invoke :cas [14 [29 39]] INFO [2019-12-17 12:28:35,506] jepsen worker 8 - jepsen.util 1448 :fail :cas [2 [1 45]] :ValueNotSwapFrom INFO [2019-12-17 12:28:35,507] jepsen worker 8 - jepsen.util 1448 :invoke :write [2 24] INFO [2019-12-17 12:28:35,521] jepsen worker 34 - jepsen.util 1564 :info :write [11 16] :RequestCanceledException INFO [2019-12-17 12:28:35,523] jepsen worker 34 - jepsen.util 1654 :invoke :cas [11 [43 33]] INFO [2019-12-17 12:28:35,535] jepsen worker 22 - jepsen.util 1462 :info :write [7 26] :RequestCanceledException INFO [2019-12-17 12:28:35,536] jepsen worker 22 - jepsen.util 1552 :invoke :cas [7 [2 17]] INFO [2019-12-17 12:28:35,549] jepsen worker 21 - jepsen.util 1371 :info :write [7 1] :RequestCanceledException INFO [2019-12-17 12:28:35,551] jepsen worker 21 - jepsen.util 1461 :invoke :cas [7 [2 17]] INFO [2019-12-17 12:28:35,551] jepsen worker 56 - jepsen.util 1226 :fail :cas [18 [1 34]] :ValueNotSwapFrom INFO [2019-12-17 12:28:35,552] jepsen worker 56 - jepsen.util 1226 :invoke :read [18 nil] INFO [2019-12-17 12:28:35,553] jepsen worker 54 - jepsen.util 1404 :fail :cas [18 [47 3]] :ValueNotSwapFrom INFO [2019-12-17 12:28:35,553] jepsen worker 54 - jepsen.util 1404 :invoke :read [18 nil] INFO [2019-12-17 12:28:35,564] jepsen worker 65 - jepsen.util 965 :ok :write [21 16] INFO [2019-12-17 12:28:35,564] jepsen worker 65 - jepsen.util 965 :invoke :write [21 18] INFO [2019-12-17 12:28:35,578] jepsen worker 55 - jepsen.util 1675 :ok :read [18 17] INFO [2019-12-17 12:28:35,579] jepsen worker 55 - jepsen.util 1675 :invoke :read [18 nil] INFO [2019-12-17 12:28:35,579] jepsen worker 76 - jepsen.util 1516 :ok :read [25 2] INFO [2019-12-17 12:28:35,579] jepsen worker 76 - jepsen.util 1516 :invoke :read [25 nil] INFO [2019-12-17 12:28:35,583] jepsen worker 20 - jepsen.util 1190 :info :cas [6 [2 10]] :RequestCanceledException INFO [2019-12-17 12:28:35,585] jepsen worker 20 - jepsen.util 1280 :invoke :write [6 13] INFO [2019-12-17 12:28:35,592] jepsen worker 35 - jepsen.util 1205 :fail :cas [11 [10 41]] :ValueNotSwapFrom INFO [2019-12-17 12:28:35,592] jepsen worker 35 - jepsen.util 1205 :invoke :cas [11 [21 10]] INFO [2019-12-17 12:28:35,603] jepsen worker 12 - jepsen.util 1272 :ok :read [4 5] INFO [2019-12-17 12:28:35,604] jepsen worker 12 - jepsen.util 1272 :invoke :cas [4 [24 2]] INFO [2019-12-17 12:28:35,622] jepsen worker 11 - jepsen.util 1541 :info :write [3 37] :RequestCanceledException INFO [2019-12-17 12:28:35,624] jepsen worker 11 - jepsen.util 1631 :invoke :read [3 nil] INFO [2019-12-17 12:28:35,626] jepsen worker 76 - jepsen.util 1516 :ok :read [25 2] INFO [2019-12-17 12:28:35,627] jepsen worker 76 - jepsen.util 1516 :invoke :read [25 nil] INFO [2019-12-17 12:28:35,628] jepsen worker 65 - jepsen.util 965 :ok :write [21 18] INFO [2019-12-17 12:28:35,628] jepsen worker 65 - jepsen.util 965 :invoke :cas [21 [27 46]] INFO [2019-12-17 12:28:35,636] jepsen worker 63 - jepsen.util 1053 :info :cas [21 [17 1]] :RequestCanceledException INFO [2019-12-17 12:28:35,636] jepsen worker 19 - jepsen.util 1279 :info :cas [6 [35 20]] :RequestCanceledException INFO [2019-12-17 12:28:35,637] jepsen worker 19 - jepsen.util 1369 :invoke :write [6 29] INFO [2019-12-17 12:28:35,637] jepsen worker 63 - jepsen.util 1143 :invoke :cas [21 [38 6]] INFO [2019-12-17 12:28:35,646] jepsen worker 35 - jepsen.util 1205 :fail :cas [11 [21 10]] :ValueNotSwapFrom INFO [2019-12-17 12:28:35,647] jepsen worker 35 - jepsen.util 1205 :invoke :cas [11 [0 46]] INFO [2019-12-17 12:28:35,653] jepsen worker 30 - jepsen.util 1380 :info :cas [10 [34 15]] :RequestCanceledException INFO [2019-12-17 12:28:35,654] jepsen worker 30 - jepsen.util 1470 :invoke :cas [10 [41 10]] INFO [2019-12-17 12:28:35,659] jepsen worker 22 - jepsen.util 1552 :fail :cas [7 [2 17]] :ValueNotSwapFrom INFO [2019-12-17 12:28:35,660] jepsen worker 22 - jepsen.util 1552 :invoke :cas [7 [17 5]] INFO [2019-12-17 12:28:35,660] jepsen worker 56 - jepsen.util 1226 :ok :read [18 17] INFO [2019-12-17 12:28:35,660] jepsen worker 56 - jepsen.util 1226 :invoke :write [18 35] INFO [2019-12-17 12:28:35,676] jepsen worker 65 - jepsen.util 965 :fail :cas [21 [27 46]] :ValueNotSwapFrom INFO [2019-12-17 12:28:35,676] jepsen worker 65 - jepsen.util 965 :invoke :write [21 3] INFO [2019-12-17 12:28:35,688] jepsen worker 43 - jepsen.util 1573 :info :write [14 45] :RequestCanceledException INFO [2019-12-17 12:28:35,688] jepsen worker 29 - jepsen.util 1019 :info :write [9 36] :RequestCanceledException INFO [2019-12-17 12:28:35,689] jepsen worker 29 - jepsen.util 1109 :invoke :write [9 8] INFO [2019-12-17 12:28:35,689] jepsen worker 43 - jepsen.util 1663 :invoke :cas [14 [6 15]] INFO [2019-12-17 12:28:35,694] jepsen worker 63 - jepsen.util 1143 :fail :cas [21 [38 6]] :ValueNotSwapFrom INFO [2019-12-17 12:28:35,696] jepsen worker 12 - jepsen.util 1272 :fail :cas [4 [24 2]] :ValueNotSwapFrom INFO [2019-12-17 12:28:35,701] jepsen worker 35 - jepsen.util 1205 :fail :cas [11 [0 46]] :ValueNotSwapFrom INFO [2019-12-17 12:28:35,711] jepsen worker 22 - jepsen.util 1552 :fail :cas [7 [17 5]] :ValueNotSwapFrom INFO [2019-12-17 12:28:35,714] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":6711,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:28:35,719] jepsen worker 18 - jepsen.util 1278 :info :write [6 29] :RequestCanceledException INFO [2019-12-17 12:28:35,719] jepsen worker 89 - jepsen.util 1349 :info :write [29 44] :RequestCanceledException INFO [2019-12-17 12:28:35,736] jepsen worker 12 - jepsen.util 1272 :invoke :write [4 7] INFO [2019-12-17 12:28:35,736] jepsen worker 63 - jepsen.util 1143 :invoke :read [21 nil] INFO [2019-12-17 12:28:35,736] jepsen worker 35 - jepsen.util 1205 :invoke :read [11 nil] INFO [2019-12-17 12:28:35,736] jepsen worker 22 - jepsen.util 1552 :invoke :write [7 45] INFO [2019-12-17 12:28:35,736] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":6711,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:28:35,737] jepsen worker 89 - jepsen.util 1439 :invoke :cas [29 [40 39]] INFO [2019-12-17 12:28:35,738] jepsen worker 18 - jepsen.util 1368 :invoke :write [6 36] INFO [2019-12-17 12:28:35,764] jepsen worker 63 - jepsen.util 1143 :ok :read [21 18] INFO [2019-12-17 12:28:35,764] jepsen worker 63 - jepsen.util 1143 :invoke :cas [21 [49 7]] INFO [2019-12-17 12:28:35,785] jepsen worker 63 - jepsen.util 1143 :fail :cas [21 [49 7]] :ValueNotSwapFrom INFO [2019-12-17 12:28:35,785] jepsen worker 63 - jepsen.util 1143 :invoke :cas [21 [26 23]] INFO [2019-12-17 12:28:35,796] jepsen worker 63 - jepsen.util 1143 :fail :cas [21 [26 23]] :ValueNotSwapFrom INFO [2019-12-17 12:28:35,796] jepsen worker 63 - jepsen.util 1143 :invoke :write [21 31] INFO [2019-12-17 12:28:35,818] jepsen worker 76 - jepsen.util 1516 :ok :read [25 2] INFO [2019-12-17 12:28:35,818] jepsen worker 76 - jepsen.util 1516 :invoke :cas [25 [26 30]] INFO [2019-12-17 12:28:35,822] jepsen worker 80 - jepsen.util 1430 :fail :cas [26 [3 20]] :ValueNotSwapFrom INFO [2019-12-17 12:28:35,822] jepsen worker 80 - jepsen.util 1430 :invoke :read [26 nil] INFO [2019-12-17 12:28:35,846] jepsen worker 51 - jepsen.util 1221 :fail :cas [17 [44 36]] :ValueNotSwapFrom INFO [2019-12-17 12:28:35,846] jepsen worker 51 - jepsen.util 1221 :invoke :read [17 nil] INFO [2019-12-17 12:28:35,848] jepsen worker 63 - jepsen.util 1143 :ok :write [21 31] INFO [2019-12-17 12:28:35,848] jepsen worker 63 - jepsen.util 1143 :invoke :read [21 nil] INFO [2019-12-17 12:28:35,848] jepsen worker 35 - jepsen.util 1205 :ok :read [11 7] INFO [2019-12-17 12:28:35,848] jepsen worker 35 - jepsen.util 1205 :invoke :cas [11 [31 25]] INFO [2019-12-17 12:28:35,852] jepsen worker 7 - jepsen.util 1627 :info :write [2 26] :RequestCanceledException INFO [2019-12-17 12:28:35,854] jepsen worker 7 - jepsen.util 1717 :invoke :write [2 10] INFO [2019-12-17 12:28:35,884] jepsen worker 63 - jepsen.util 1143 :ok :read [21 31] INFO [2019-12-17 12:28:35,884] jepsen worker 63 - jepsen.util 1143 :invoke :read [21 nil] INFO [2019-12-17 12:28:35,898] jepsen worker 88 - jepsen.util 1168 :fail :cas [29 [21 16]] :ValueNotSwapFrom INFO [2019-12-17 12:28:35,898] jepsen worker 88 - jepsen.util 1168 :invoke :read [29 nil] INFO [2019-12-17 12:28:35,901] jepsen worker 63 - jepsen.util 1143 :ok :read [21 31] INFO [2019-12-17 12:28:35,901] jepsen worker 63 - jepsen.util 1143 :invoke :cas [21 [10 18]] INFO [2019-12-17 12:28:35,911] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":6737,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:28:35,911] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":6737,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:28:35,911] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":6738,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:28:35,911] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":6738,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:28:35,911] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":6739,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:28:35,911] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":6739,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:28:35,913] jepsen worker 63 - jepsen.util 1143 :fail :cas [21 [10 18]] :ValueNotSwapFrom INFO [2019-12-17 12:28:35,913] jepsen worker 63 - jepsen.util 1143 :invoke :cas [21 [25 21]] INFO [2019-12-17 12:28:35,915] jepsen worker 76 - jepsen.util 1516 :fail :cas [25 [26 30]] :ValueNotSwapFrom INFO [2019-12-17 12:28:35,915] jepsen worker 76 - jepsen.util 1516 :invoke :read [25 nil] INFO [2019-12-17 12:28:35,924] jepsen worker 69 - jepsen.util 1059 :ok :read [23 39] INFO [2019-12-17 12:28:35,924] jepsen worker 69 - jepsen.util 1059 :invoke :cas [23 [27 15]] INFO [2019-12-17 12:28:35,933] jepsen worker 63 - jepsen.util 1143 :fail :cas [21 [25 21]] :ValueNotSwapFrom INFO [2019-12-17 12:28:35,934] jepsen worker 63 - jepsen.util 1143 :invoke :cas [21 [41 30]] INFO [2019-12-17 12:28:35,947] jepsen worker 87 - jepsen.util 1257 :ok :read [29 12] INFO [2019-12-17 12:28:35,947] jepsen worker 63 - jepsen.util 1143 :fail :cas [21 [41 30]] :ValueNotSwapFrom INFO [2019-12-17 12:28:35,947] jepsen worker 87 - jepsen.util 1257 :invoke :cas [29 [23 34]] INFO [2019-12-17 12:28:35,947] jepsen worker 63 - jepsen.util 1143 :invoke :cas [21 [45 44]] INFO [2019-12-17 12:28:35,958] jepsen worker 63 - jepsen.util 1143 :fail :cas [21 [45 44]] :ValueNotSwapFrom INFO [2019-12-17 12:28:35,958] jepsen worker 63 - jepsen.util 1143 :invoke :write [21 1] INFO [2019-12-17 12:28:35,988] jepsen worker 62 - jepsen.util 1412 :info :cas [20 [20 20]] :RequestCanceledException INFO [2019-12-17 12:28:35,989] jepsen worker 62 - jepsen.util 1502 :invoke :write [20 8] INFO [2019-12-17 12:28:36,000] jepsen worker 63 - jepsen.util 1143 :ok :write [21 1] INFO [2019-12-17 12:28:36,000] jepsen worker 63 - jepsen.util 1143 :invoke :write [21 18] INFO [2019-12-17 12:28:36,007] jepsen worker 54 - jepsen.util 1404 :ok :read [18 17] INFO [2019-12-17 12:28:36,007] jepsen worker 54 - jepsen.util 1404 :invoke :read [18 nil] INFO [2019-12-17 12:28:36,016] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":6757,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:28:36,016] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":6757,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:28:36,016] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":6759,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:28:36,016] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":6759,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:28:36,035] jepsen worker 54 - jepsen.util 1404 :ok :read [18 17] INFO [2019-12-17 12:28:36,035] jepsen worker 54 - jepsen.util 1404 :invoke :read [18 nil] INFO [2019-12-17 12:28:36,042] jepsen worker 63 - jepsen.util 1143 :ok :write [21 18] INFO [2019-12-17 12:28:36,042] jepsen worker 63 - jepsen.util 1143 :invoke :cas [21 [1 43]] INFO [2019-12-17 12:28:36,055] jepsen worker 63 - jepsen.util 1143 :fail :cas [21 [1 43]] :ValueNotSwapFrom INFO [2019-12-17 12:28:36,055] jepsen worker 63 - jepsen.util 1143 :invoke :write [21 30] INFO [2019-12-17 12:28:36,078] jepsen worker 55 - jepsen.util 1675 :ok :read [18 17] INFO [2019-12-17 12:28:36,079] jepsen worker 55 - jepsen.util 1675 :invoke :read [18 nil] INFO [2019-12-17 12:28:36,093] jepsen worker 63 - jepsen.util 1143 :ok :write [21 30] INFO [2019-12-17 12:28:36,093] jepsen worker 63 - jepsen.util 1143 :invoke :write [21 30] INFO [2019-12-17 12:28:36,106] jepsen worker 47 - jepsen.util 1577 :info :write [15 33] :RequestCanceledException INFO [2019-12-17 12:28:36,107] jepsen worker 47 - jepsen.util 1667 :invoke :read [15 nil] INFO [2019-12-17 12:28:36,134] jepsen worker 63 - jepsen.util 1143 :ok :write [21 30] INFO [2019-12-17 12:28:36,134] jepsen worker 63 - jepsen.util 1143 :invoke :read [21 nil] INFO [2019-12-17 12:28:36,148] jepsen worker 63 - jepsen.util 1143 :ok :read [21 30] INFO [2019-12-17 12:28:36,149] jepsen worker 63 - jepsen.util 1143 :invoke :read [21 nil] INFO [2019-12-17 12:28:36,161] jepsen worker 63 - jepsen.util 1143 :ok :read [21 30] INFO [2019-12-17 12:28:36,161] jepsen worker 63 - jepsen.util 1143 :invoke :cas [21 [21 19]] INFO [2019-12-17 12:28:36,174] jepsen worker 63 - jepsen.util 1143 :fail :cas [21 [21 19]] :ValueNotSwapFrom INFO [2019-12-17 12:28:36,174] jepsen worker 63 - jepsen.util 1143 :invoke :cas [21 [48 34]] INFO [2019-12-17 12:28:36,183] jepsen worker 63 - jepsen.util 1143 :fail :cas [21 [48 34]] :ValueNotSwapFrom INFO [2019-12-17 12:28:36,184] jepsen worker 63 - jepsen.util 1143 :invoke :cas [21 [6 12]] INFO [2019-12-17 12:28:36,196] jepsen worker 63 - jepsen.util 1143 :fail :cas [21 [6 12]] :ValueNotSwapFrom INFO [2019-12-17 12:28:36,196] jepsen worker 63 - jepsen.util 1143 :invoke :write [21 6] INFO [2019-12-17 12:28:36,204] jepsen worker 80 - jepsen.util 1430 :ok :read [26 45] INFO [2019-12-17 12:28:36,204] jepsen worker 80 - jepsen.util 1430 :invoke :read [26 nil] INFO [2019-12-17 12:28:36,233] jepsen worker 63 - jepsen.util 1143 :ok :write [21 6] INFO [2019-12-17 12:28:36,233] jepsen worker 63 - jepsen.util 1143 :invoke :read [21 nil] INFO [2019-12-17 12:28:36,240] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":6783,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:28:36,240] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":6774,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:28:36,241] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":6783,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:28:36,241] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":6774,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:28:36,241] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":6786,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:28:36,241] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":6776,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:28:36,241] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":6786,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:28:36,241] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":6776,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:28:36,241] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":6787,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:28:36,241] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":6787,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:28:36,246] jepsen worker 63 - jepsen.util 1143 :ok :read [21 6] INFO [2019-12-17 12:28:36,246] jepsen worker 63 - jepsen.util 1143 :invoke :cas [21 [18 15]] INFO [2019-12-17 12:28:36,257] jepsen worker 63 - jepsen.util 1143 :fail :cas [21 [18 15]] :ValueNotSwapFrom INFO [2019-12-17 12:28:36,257] jepsen worker 63 - jepsen.util 1143 :invoke :read [21 nil] INFO [2019-12-17 12:28:36,259] jepsen worker 73 - jepsen.util 1153 :info :write [24 24] :SyncWriteAmbiguous INFO [2019-12-17 12:28:36,259] jepsen worker 73 - jepsen.util 1243 :invoke :read [24 nil] INFO [2019-12-17 12:28:36,276] jepsen worker 9 - jepsen.util 1269 :ok :read [3 19] INFO [2019-12-17 12:28:36,277] jepsen worker 9 - jepsen.util 1269 :invoke :read [3 nil] INFO [2019-12-17 12:28:36,282] jepsen worker 63 - jepsen.util 1143 :ok :read [21 6] INFO [2019-12-17 12:28:36,282] jepsen worker 63 - jepsen.util 1143 :invoke :read [21 nil] INFO [2019-12-17 12:28:36,284] jepsen worker 11 - jepsen.util 1631 :ok :read [3 19] INFO [2019-12-17 12:28:36,285] jepsen worker 11 - jepsen.util 1631 :invoke :cas [3 [41 39]] INFO [2019-12-17 12:28:36,289] jepsen worker 9 - jepsen.util 1269 :ok :read [3 19] INFO [2019-12-17 12:28:36,290] jepsen worker 9 - jepsen.util 1269 :invoke :read [3 nil] INFO [2019-12-17 12:28:36,305] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:28:36,305] jepsen nemesis - couchbase.util Rebalance check count: 44 INFO [2019-12-17 12:28:36,305] jepsen nemesis - couchbase.util Rebalance stuck count: 25 INFO [2019-12-17 12:28:36,307] jepsen worker 3 - jepsen.util 1083 :fail :cas [1 [35 36]] :ValueNotSwapFrom INFO [2019-12-17 12:28:36,307] jepsen worker 3 - jepsen.util 1083 :invoke :write [1 20] INFO [2019-12-17 12:28:36,309] jepsen worker 63 - jepsen.util 1143 :ok :read [21 6] INFO [2019-12-17 12:28:36,309] jepsen worker 63 - jepsen.util 1143 :invoke :read [21 nil] INFO [2019-12-17 12:28:36,309] jepsen worker 11 - jepsen.util 1631 :fail :cas [3 [41 39]] :ValueNotSwapFrom INFO [2019-12-17 12:28:36,310] jepsen worker 11 - jepsen.util 1631 :invoke :write [3 34] INFO [2019-12-17 12:28:36,316] jepsen worker 9 - jepsen.util 1269 :ok :read [3 19] INFO [2019-12-17 12:28:36,316] jepsen worker 9 - jepsen.util 1269 :invoke :write [3 24] INFO [2019-12-17 12:28:36,322] jepsen worker 37 - jepsen.util 1027 :info :write [12 47] :RequestCanceledException INFO [2019-12-17 12:28:36,322] jepsen worker 85 - jepsen.util 1435 :info :write [28 42] :RequestCanceledException INFO [2019-12-17 12:28:36,324] jepsen worker 85 - jepsen.util 1525 :invoke :write [28 5] INFO [2019-12-17 12:28:36,324] jepsen worker 37 - jepsen.util 1117 :invoke :read [12 nil] INFO [2019-12-17 12:28:36,327] jepsen worker 63 - jepsen.util 1143 :ok :read [21 6] INFO [2019-12-17 12:28:36,327] jepsen worker 63 - jepsen.util 1143 :invoke :write [21 25] INFO [2019-12-17 12:28:36,331] jepsen worker 13 - jepsen.util 1183 :info :write [4 19] :RequestCanceledException INFO [2019-12-17 12:28:36,331] jepsen worker 82 - jepsen.util 1882 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:28:36,331] jepsen worker 89 - jepsen.util 1439 :fail :cas [29 [40 39]] :ValueNotSwapFrom INFO [2019-12-17 12:28:36,331] jepsen worker 32 - jepsen.util 1202 :info :write [10 19] :SyncWriteAmbiguous INFO [2019-12-17 12:28:36,331] jepsen worker 89 - jepsen.util 1439 :invoke :read [29 nil] INFO [2019-12-17 12:28:36,331] jepsen worker 82 - jepsen.util 1882 :invoke :cas [27 [35 35]] INFO [2019-12-17 12:28:36,332] jepsen worker 13 - jepsen.util 1273 :invoke :write [4 11] INFO [2019-12-17 12:28:36,332] jepsen worker 32 - jepsen.util 1292 :invoke :write [10 34] INFO [2019-12-17 12:28:36,349] jepsen worker 30 - jepsen.util 1470 :fail :cas [10 [41 10]] :ValueNotSwapFrom INFO [2019-12-17 12:28:36,350] jepsen worker 30 - jepsen.util 1470 :invoke :read [10 nil] INFO [2019-12-17 12:28:36,354] jepsen worker 76 - jepsen.util 1516 :ok :read [25 2] INFO [2019-12-17 12:28:36,354] jepsen worker 76 - jepsen.util 1516 :invoke :write [25 0] INFO [2019-12-17 12:28:36,366] jepsen worker 51 - jepsen.util 1221 :ok :read [17 42] INFO [2019-12-17 12:28:36,366] jepsen worker 51 - jepsen.util 1221 :invoke :write [17 9] INFO [2019-12-17 12:28:36,374] jepsen worker 45 - jepsen.util 1215 :info :write [15 3] :SyncWriteAmbiguous INFO [2019-12-17 12:28:36,375] jepsen worker 45 - jepsen.util 1305 :invoke :read [15 nil] INFO [2019-12-17 12:28:36,387] jepsen worker 35 - jepsen.util 1205 :fail :cas [11 [31 25]] :ValueNotSwapFrom INFO [2019-12-17 12:28:36,387] jepsen worker 35 - jepsen.util 1205 :invoke :cas [11 [4 37]] INFO [2019-12-17 12:28:36,388] jepsen worker 70 - jepsen.util 1420 :info :write [23 26] :RequestCanceledException INFO [2019-12-17 12:28:36,388] jepsen worker 0 - jepsen.util 1530 :info :write [0 40] :RequestCanceledException INFO [2019-12-17 12:28:36,389] jepsen worker 70 - jepsen.util 1510 :invoke :read [23 nil] INFO [2019-12-17 12:28:36,390] jepsen worker 0 - jepsen.util 1620 :invoke :cas [0 [29 46]] INFO [2019-12-17 12:28:36,405] jepsen worker 52 - jepsen.util 1312 :info :write [17 5] :RequestCanceledException INFO [2019-12-17 12:28:36,405] jepsen worker 72 - jepsen.util 1152 :info :cas [24 [0 43]] :RequestCanceledException INFO [2019-12-17 12:28:36,406] jepsen worker 52 - jepsen.util 1402 :invoke :read [17 nil] INFO [2019-12-17 12:28:36,407] jepsen worker 72 - jepsen.util 1242 :invoke :write [24 20] INFO [2019-12-17 12:28:36,413] jepsen worker 29 - jepsen.util 1109 :ok :write [9 8] INFO [2019-12-17 12:28:36,413] jepsen worker 29 - jepsen.util 1109 :invoke :read [9 nil] INFO [2019-12-17 12:28:36,421] jepsen worker 63 - jepsen.util 1143 :ok :write [21 25] INFO [2019-12-17 12:28:36,421] jepsen worker 63 - jepsen.util 1143 :invoke :cas [21 [34 34]] INFO [2019-12-17 12:28:36,438] jepsen worker 88 - jepsen.util 1168 :ok :read [29 12] INFO [2019-12-17 12:28:36,438] jepsen worker 88 - jepsen.util 1168 :invoke :cas [29 [27 14]] INFO [2019-12-17 12:28:36,443] jepsen worker 45 - jepsen.util 1305 :ok :read [15 6] INFO [2019-12-17 12:28:36,443] jepsen worker 45 - jepsen.util 1305 :invoke :cas [15 [34 11]] INFO [2019-12-17 12:28:36,455] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":6819,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:28:36,455] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":6819,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:28:36,455] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":6823,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:28:36,455] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":6823,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:28:36,469] jepsen worker 35 - jepsen.util 1205 :fail :cas [11 [4 37]] :ValueNotSwapFrom INFO [2019-12-17 12:28:36,469] jepsen worker 35 - jepsen.util 1205 :invoke :read [11 nil] INFO [2019-12-17 12:28:36,470] jepsen worker 29 - jepsen.util 1109 :ok :read [9 8] INFO [2019-12-17 12:28:36,471] jepsen worker 29 - jepsen.util 1109 :invoke :read [9 nil] INFO [2019-12-17 12:28:36,477] jepsen worker 69 - jepsen.util 1059 :fail :cas [23 [27 15]] :ValueNotSwapFrom INFO [2019-12-17 12:28:36,477] jepsen worker 69 - jepsen.util 1059 :invoke :write [23 23] INFO [2019-12-17 12:28:36,480] jepsen worker 45 - jepsen.util 1305 :fail :cas [15 [34 11]] :ValueNotSwapFrom INFO [2019-12-17 12:28:36,480] jepsen worker 45 - jepsen.util 1305 :invoke :write [15 15] INFO [2019-12-17 12:28:36,505] jepsen worker 35 - jepsen.util 1205 :ok :read [11 7] INFO [2019-12-17 12:28:36,506] jepsen worker 35 - jepsen.util 1205 :invoke :read [11 nil] INFO [2019-12-17 12:28:36,506] jepsen worker 48 - jepsen.util 948 :info :write [16 16] :RequestCanceledException INFO [2019-12-17 12:28:36,507] jepsen worker 48 - jepsen.util 1038 :invoke :cas [16 [19 16]] INFO [2019-12-17 12:28:36,509] jepsen worker 29 - jepsen.util 1109 :ok :read [9 8] INFO [2019-12-17 12:28:36,509] jepsen worker 29 - jepsen.util 1109 :invoke :cas [9 [34 49]] INFO [2019-12-17 12:28:36,520] jepsen worker 35 - jepsen.util 1205 :ok :read [11 7] INFO [2019-12-17 12:28:36,520] jepsen worker 35 - jepsen.util 1205 :invoke :write [11 11] INFO [2019-12-17 12:28:36,521] jepsen worker 57 - jepsen.util 1407 :info :write [19 39] :RequestCanceledException INFO [2019-12-17 12:28:36,521] jepsen worker 28 - jepsen.util 838 :info :write [9 4] :RequestCanceledException INFO [2019-12-17 12:28:36,521] jepsen worker 83 - jepsen.util 1703 :info :write [27 27] :RequestCanceledException INFO [2019-12-17 12:28:36,521] jepsen worker 23 - jepsen.util 1373 :info :write [7 20] :RequestCanceledException INFO [2019-12-17 12:28:36,521] jepsen worker 84 - jepsen.util 1344 :info :write [28 28] :RequestCanceledException INFO [2019-12-17 12:28:36,521] jepsen worker 24 - jepsen.util 1374 :info :write [8 37] :RequestCanceledException INFO [2019-12-17 12:28:36,522] jepsen worker 16 - jepsen.util 1366 :info :cas [5 [14 22]] :RequestCanceledException INFO [2019-12-17 12:28:36,522] jepsen worker 59 - jepsen.util 1319 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:28:36,522] jepsen worker 59 - jepsen.util 1319 :invoke :write [19 16] INFO [2019-12-17 12:28:36,522] jepsen worker 57 - jepsen.util 1497 :invoke :read [19 nil] INFO [2019-12-17 12:28:36,522] jepsen worker 23 - jepsen.util 1463 :invoke :read [7 nil] INFO [2019-12-17 12:28:36,522] jepsen worker 84 - jepsen.util 1434 :invoke :write [28 14] INFO [2019-12-17 12:28:36,522] jepsen worker 83 - jepsen.util 1793 :invoke :write [27 7] INFO [2019-12-17 12:28:36,522] jepsen worker 24 - jepsen.util 1464 :invoke :cas [8 [20 9]] INFO [2019-12-17 12:28:36,523] jepsen worker 16 - jepsen.util 1456 :invoke :cas [5 [22 18]] INFO [2019-12-17 12:28:36,527] jepsen worker 28 - jepsen.util 928 :invoke :cas [9 [12 23]] INFO [2019-12-17 12:28:36,529] jepsen worker 86 - jepsen.util 1526 :info :write [28 11] :RequestCanceledException INFO [2019-12-17 12:28:36,529] jepsen worker 50 - jepsen.util 1490 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:28:36,530] jepsen worker 50 - jepsen.util 1490 :invoke :cas [16 [38 18]] INFO [2019-12-17 12:28:36,530] jepsen worker 86 - jepsen.util 1616 :invoke :cas [28 [16 37]] INFO [2019-12-17 12:28:36,553] jepsen worker 75 - jepsen.util 1605 :info :cas [25 [2 15]] :RequestCanceledException INFO [2019-12-17 12:28:36,553] jepsen worker 31 - jepsen.util 1201 :info :write [10 38] :RequestCanceledException INFO [2019-12-17 12:28:36,553] jepsen worker 49 - jepsen.util 1219 :info :cas [16 [22 35]] :RequestCanceledException INFO [2019-12-17 12:28:36,553] jepsen worker 17 - jepsen.util 1457 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:28:36,553] jepsen worker 17 - jepsen.util 1457 :invoke :write [5 5] INFO [2019-12-17 12:28:36,553] jepsen worker 75 - jepsen.util 1695 :invoke :read [25 nil] INFO [2019-12-17 12:28:36,553] jepsen worker 31 - jepsen.util 1291 :invoke :read [10 nil] INFO [2019-12-17 12:28:36,554] jepsen worker 49 - jepsen.util 1309 :invoke :read [16 nil] INFO [2019-12-17 12:28:36,561] jepsen worker 87 - jepsen.util 1257 :fail :cas [29 [23 34]] :ValueNotSwapFrom INFO [2019-12-17 12:28:36,562] jepsen worker 87 - jepsen.util 1257 :invoke :read [29 nil] INFO [2019-12-17 12:28:36,587] jepsen worker 40 - jepsen.util 1750 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:28:36,587] jepsen worker 68 - jepsen.util 1058 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:28:36,587] jepsen worker 66 - jepsen.util 1146 :info :cas [22 [39 47]] :RequestCanceledException INFO [2019-12-17 12:28:36,587] jepsen worker 25 - jepsen.util 1645 :info :write [8 26] :RequestCanceledException INFO [2019-12-17 12:28:36,587] jepsen worker 40 - jepsen.util 1750 :invoke :read [13 nil] INFO [2019-12-17 12:28:36,588] jepsen worker 68 - jepsen.util 1058 :invoke :cas [22 [41 8]] INFO [2019-12-17 12:28:36,589] jepsen worker 66 - jepsen.util 1236 :invoke :cas [22 [47 25]] INFO [2019-12-17 12:28:36,589] jepsen worker 25 - jepsen.util 1735 :invoke :write [8 37] INFO [2019-12-17 12:28:36,589] jepsen worker 89 - jepsen.util 1439 :ok :read [29 12] INFO [2019-12-17 12:28:36,589] jepsen worker 89 - jepsen.util 1439 :invoke :read [29 nil] INFO [2019-12-17 12:28:36,606] jepsen worker 4 - jepsen.util 994 :info :write [1 42] :RequestCanceledException INFO [2019-12-17 12:28:36,606] jepsen worker 1 - jepsen.util 1351 :info :cas [0 [42 42]] :RequestCanceledException INFO [2019-12-17 12:28:36,607] jepsen worker 4 - jepsen.util 1084 :invoke :read [1 nil] INFO [2019-12-17 12:28:36,607] jepsen worker 1 - jepsen.util 1441 :invoke :cas [0 [34 49]] INFO [2019-12-17 12:28:36,614] jepsen worker 54 - jepsen.util 1404 :ok :read [18 17] INFO [2019-12-17 12:28:36,614] jepsen worker 54 - jepsen.util 1404 :invoke :write [18 25] INFO [2019-12-17 12:28:36,620] jepsen worker 2 - jepsen.util 1172 :info :write [0 39] :RequestCanceledException INFO [2019-12-17 12:28:36,620] jepsen worker 67 - jepsen.util 1687 :info :write [22 48] :RequestCanceledException INFO [2019-12-17 12:28:36,620] jepsen worker 39 - jepsen.util 1569 :info :write [13 36] :RequestCanceledException INFO [2019-12-17 12:28:36,620] jepsen worker 15 - jepsen.util 1095 :info :cas [5 [39 49]] :RequestCanceledException INFO [2019-12-17 12:28:36,620] jepsen worker 41 - jepsen.util 1211 :info :write [13 26] :RequestCanceledException INFO [2019-12-17 12:28:36,621] jepsen worker 39 - jepsen.util 1659 :invoke :write [13 27] INFO [2019-12-17 12:28:36,621] jepsen worker 41 - jepsen.util 1301 :invoke :write [13 41] INFO [2019-12-17 12:28:36,621] jepsen worker 15 - jepsen.util 1185 :invoke :write [5 44] INFO [2019-12-17 12:28:36,621] jepsen worker 67 - jepsen.util 1777 :invoke :cas [22 [4 42]] INFO [2019-12-17 12:28:36,621] jepsen worker 2 - jepsen.util 1262 :invoke :read [0 nil] INFO [2019-12-17 12:28:36,635] jepsen worker 71 - jepsen.util 881 :info :write [23 44] :RequestCanceledException INFO [2019-12-17 12:28:36,636] jepsen worker 71 - jepsen.util 971 :invoke :read [23 nil] INFO [2019-12-17 12:28:36,662] jepsen worker 89 - jepsen.util 1439 :ok :read [29 12] INFO [2019-12-17 12:28:36,662] jepsen worker 89 - jepsen.util 1439 :invoke :cas [29 [46 13]] INFO [2019-12-17 12:28:36,681] jepsen worker 80 - jepsen.util 1430 :ok :read [26 45] INFO [2019-12-17 12:28:36,681] jepsen worker 80 - jepsen.util 1430 :invoke :write [26 14] INFO [2019-12-17 12:28:36,688] jepsen worker 61 - jepsen.util 1501 :info :cas [20 [23 2]] :RequestCanceledException INFO [2019-12-17 12:28:36,688] jepsen worker 81 - jepsen.util 1521 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:28:36,688] jepsen worker 81 - jepsen.util 1521 :invoke :read [27 nil] INFO [2019-12-17 12:28:36,689] jepsen worker 61 - jepsen.util 1591 :invoke :write [20 43] INFO [2019-12-17 12:28:36,705] jepsen worker 42 - jepsen.util 1662 :info :cas [14 [3 35]] :RequestCanceledException INFO [2019-12-17 12:28:36,706] jepsen worker 42 - jepsen.util 1752 :invoke :write [14 35] INFO [2019-12-17 12:28:36,720] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":6863,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:28:36,721] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":6863,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:28:36,721] jepsen worker 60 - jepsen.util 1590 :info :write [20 32] :RequestCanceledException INFO [2019-12-17 12:28:36,721] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":6864,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:28:36,721] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":6864,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:28:36,721] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":6865,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:28:36,721] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":6865,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:28:36,721] jepsen worker 60 - jepsen.util 1680 :invoke :write [20 33] INFO [2019-12-17 12:28:36,740] jepsen worker 70 - jepsen.util 1510 :ok :read [23 39] INFO [2019-12-17 12:28:36,740] jepsen worker 70 - jepsen.util 1510 :invoke :read [23 nil] INFO [2019-12-17 12:28:36,754] jepsen worker 32 - jepsen.util 1292 :ok :write [10 34] INFO [2019-12-17 12:28:36,754] jepsen worker 32 - jepsen.util 1292 :invoke :cas [10 [1 41]] INFO [2019-12-17 12:28:36,767] jepsen worker 4 - jepsen.util 1084 :ok :read [1 21] INFO [2019-12-17 12:28:36,767] jepsen worker 75 - jepsen.util 1695 :ok :read [25 2] INFO [2019-12-17 12:28:36,768] jepsen worker 4 - jepsen.util 1084 :invoke :write [1 30] INFO [2019-12-17 12:28:36,768] jepsen worker 75 - jepsen.util 1695 :invoke :write [25 38] INFO [2019-12-17 12:28:36,769] jepsen worker 32 - jepsen.util 1292 :fail :cas [10 [1 41]] :ValueNotSwapFrom INFO [2019-12-17 12:28:36,769] jepsen worker 32 - jepsen.util 1292 :invoke :cas [10 [24 33]] INFO [2019-12-17 12:28:36,776] jepsen worker 72 - jepsen.util 1242 :ok :write [24 20] INFO [2019-12-17 12:28:36,776] jepsen worker 72 - jepsen.util 1242 :invoke :cas [24 [4 32]] INFO [2019-12-17 12:28:36,790] jepsen worker 70 - jepsen.util 1510 :ok :read [23 39] INFO [2019-12-17 12:28:36,790] jepsen worker 70 - jepsen.util 1510 :invoke :read [23 nil] INFO [2019-12-17 12:28:36,802] jepsen worker 19 - jepsen.util 1369 :ok :write [6 29] INFO [2019-12-17 12:28:36,802] jepsen worker 10 - jepsen.util 1270 :ok :write [3 22] INFO [2019-12-17 12:28:36,803] jepsen worker 10 - jepsen.util 1270 :invoke :cas [3 [5 11]] INFO [2019-12-17 12:28:36,803] jepsen worker 19 - jepsen.util 1369 :invoke :cas [6 [14 29]] INFO [2019-12-17 12:28:36,805] jepsen worker 72 - jepsen.util 1242 :fail :cas [24 [4 32]] :ValueNotSwapFrom INFO [2019-12-17 12:28:36,805] jepsen worker 72 - jepsen.util 1242 :invoke :write [24 3] INFO [2019-12-17 12:28:36,806] jepsen worker 58 - jepsen.util 1498 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:28:36,806] jepsen worker 58 - jepsen.util 1498 :invoke :read [19 nil] INFO [2019-12-17 12:28:36,819] jepsen worker 32 - jepsen.util 1292 :fail :cas [10 [24 33]] :ValueNotSwapFrom INFO [2019-12-17 12:28:36,819] jepsen worker 32 - jepsen.util 1292 :invoke :write [10 8] INFO [2019-12-17 12:28:36,874] jepsen worker 31 - jepsen.util 1291 :ok :read [10 34] INFO [2019-12-17 12:28:36,874] jepsen worker 31 - jepsen.util 1291 :invoke :read [10 nil] INFO [2019-12-17 12:28:36,899] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":6888,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:28:36,899] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":6888,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:28:36,899] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":6891,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:28:36,899] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":6891,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:28:36,903] jepsen worker 19 - jepsen.util 1369 :fail :cas [6 [14 29]] :ValueNotSwapFrom INFO [2019-12-17 12:28:36,903] jepsen worker 19 - jepsen.util 1369 :invoke :read [6 nil] INFO [2019-12-17 12:28:36,911] jepsen worker 31 - jepsen.util 1291 :ok :read [10 34] INFO [2019-12-17 12:28:36,912] jepsen worker 31 - jepsen.util 1291 :invoke :read [10 nil] INFO [2019-12-17 12:28:36,927] jepsen worker 19 - jepsen.util 1369 :ok :read [6 29] INFO [2019-12-17 12:28:36,927] jepsen worker 19 - jepsen.util 1369 :invoke :cas [6 [19 28]] INFO [2019-12-17 12:28:36,950] jepsen worker 71 - jepsen.util 971 :ok :read [23 39] INFO [2019-12-17 12:28:36,952] jepsen worker 71 - jepsen.util 971 :invoke :write [23 35] INFO [2019-12-17 12:28:36,956] jepsen worker 31 - jepsen.util 1291 :ok :read [10 34] INFO [2019-12-17 12:28:36,956] jepsen worker 31 - jepsen.util 1291 :invoke :write [10 44] INFO [2019-12-17 12:28:36,967] jepsen worker 19 - jepsen.util 1369 :fail :cas [6 [19 28]] :ValueNotSwapFrom INFO [2019-12-17 12:28:36,967] jepsen worker 19 - jepsen.util 1369 :invoke :read [6 nil] INFO [2019-12-17 12:28:36,979] jepsen worker 89 - jepsen.util 1439 :fail :cas [29 [46 13]] :ValueNotSwapFrom INFO [2019-12-17 12:28:36,979] jepsen worker 89 - jepsen.util 1439 :invoke :write [29 40] INFO [2019-12-17 12:28:36,987] jepsen worker 70 - jepsen.util 1510 :ok :read [23 39] INFO [2019-12-17 12:28:36,987] jepsen worker 70 - jepsen.util 1510 :invoke :read [23 nil] INFO [2019-12-17 12:28:36,988] jepsen worker 77 - jepsen.util 1427 :info :cas [25 [20 5]] :RequestCanceledException INFO [2019-12-17 12:28:36,988] jepsen worker 77 - jepsen.util 1517 :invoke :write [25 47] INFO [2019-12-17 12:28:37,004] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":6903,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:28:37,004] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":6903,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:28:37,004] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":6907,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:28:37,004] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":6907,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:28:37,005] jepsen worker 19 - jepsen.util 1369 :ok :read [6 29] INFO [2019-12-17 12:28:37,005] jepsen worker 19 - jepsen.util 1369 :invoke :write [6 48] INFO [2019-12-17 12:28:37,006] jepsen worker 14 - jepsen.util 1094 :info :write [4 25] :RequestCanceledException INFO [2019-12-17 12:28:37,007] jepsen worker 14 - jepsen.util 1184 :invoke :write [4 47] INFO [2019-12-17 12:28:37,022] jepsen worker 46 - jepsen.util 1486 :info :write [15 37] :RequestCanceledException INFO [2019-12-17 12:28:37,022] jepsen worker 33 - jepsen.util 1743 :info :write [11 25] :RequestCanceledException INFO [2019-12-17 12:28:37,023] jepsen worker 46 - jepsen.util 1576 :invoke :read [15 nil] INFO [2019-12-17 12:28:37,023] jepsen worker 33 - jepsen.util 1833 :invoke :cas [11 [20 40]] INFO [2019-12-17 12:28:37,036] jepsen worker 36 - jepsen.util 1206 :info :write [12 19] :RequestCanceledException INFO [2019-12-17 12:28:37,037] jepsen worker 36 - jepsen.util 1296 :invoke :write [12 34] INFO [2019-12-17 12:28:37,048] jepsen worker 46 - jepsen.util 1576 :ok :read [15 6] INFO [2019-12-17 12:28:37,048] jepsen worker 46 - jepsen.util 1576 :invoke :write [15 2] INFO [2019-12-17 12:28:37,091] jepsen worker 33 - jepsen.util 1833 :fail :cas [11 [20 40]] :ValueNotSwapFrom INFO [2019-12-17 12:28:37,091] jepsen worker 33 - jepsen.util 1833 :invoke :read [11 nil] INFO [2019-12-17 12:28:37,144] jepsen worker 33 - jepsen.util 1833 :ok :read [11 7] INFO [2019-12-17 12:28:37,144] jepsen worker 33 - jepsen.util 1833 :invoke :read [11 nil] INFO [2019-12-17 12:28:37,153] jepsen worker 27 - jepsen.util 1017 :info :write [9 44] :RequestCanceledException INFO [2019-12-17 12:28:37,154] jepsen worker 27 - jepsen.util 1107 :invoke :read [9 nil] INFO [2019-12-17 12:28:37,160] jepsen worker 45 - jepsen.util 1305 :ok :write [15 15] INFO [2019-12-17 12:28:37,160] jepsen worker 72 - jepsen.util 1242 :ok :write [24 3] INFO [2019-12-17 12:28:37,161] jepsen worker 72 - jepsen.util 1242 :invoke :read [24 nil] INFO [2019-12-17 12:28:37,161] jepsen worker 45 - jepsen.util 1305 :invoke :cas [15 [46 30]] INFO [2019-12-17 12:28:37,169] jepsen worker 20 - jepsen.util 1280 :ok :write [6 13] INFO [2019-12-17 12:28:37,169] jepsen worker 20 - jepsen.util 1280 :invoke :read [6 nil] INFO [2019-12-17 12:28:37,174] jepsen worker 32 - jepsen.util 1292 :ok :write [10 8] INFO [2019-12-17 12:28:37,174] jepsen worker 32 - jepsen.util 1292 :invoke :cas [10 [29 1]] INFO [2019-12-17 12:28:37,189] jepsen worker 28 - jepsen.util 928 :fail :cas [9 [12 23]] :ValueNotSwapFrom INFO [2019-12-17 12:28:37,189] jepsen worker 28 - jepsen.util 928 :invoke :write [9 5] INFO [2019-12-17 12:28:37,190] jepsen worker 72 - jepsen.util 1242 :ok :read [24 3] INFO [2019-12-17 12:28:37,191] jepsen worker 72 - jepsen.util 1242 :invoke :write [24 12] INFO [2019-12-17 12:28:37,217] jepsen worker 45 - jepsen.util 1305 :fail :cas [15 [46 30]] :ValueNotSwapFrom INFO [2019-12-17 12:28:37,217] jepsen worker 45 - jepsen.util 1305 :invoke :write [15 6] INFO [2019-12-17 12:28:37,221] jepsen worker 53 - jepsen.util 1133 :info :write [17 44] :RequestCanceledException INFO [2019-12-17 12:28:37,221] jepsen worker 79 - jepsen.util 1159 :info :write [26 0] :RequestCanceledException INFO [2019-12-17 12:28:37,221] jepsen worker 78 - jepsen.util 1248 :info :write [26 32] :RequestCanceledException INFO [2019-12-17 12:28:37,223] jepsen worker 79 - jepsen.util 1249 :invoke :read [26 nil] INFO [2019-12-17 12:28:37,223] jepsen worker 53 - jepsen.util 1223 :invoke :cas [17 [47 10]] INFO [2019-12-17 12:28:37,223] jepsen worker 78 - jepsen.util 1338 :invoke :write [26 46] INFO [2019-12-17 12:28:37,227] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":6923,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:28:37,227] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":6931,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:28:37,227] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":6923,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:28:37,227] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":6931,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:28:37,227] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":6924,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:28:37,228] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":6934,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:28:37,228] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":6924,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:28:37,228] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":6934,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:28:37,238] jepsen worker 70 - jepsen.util 1510 :ok :read [23 39] INFO [2019-12-17 12:28:37,238] jepsen worker 70 - jepsen.util 1510 :invoke :cas [23 [10 28]] INFO [2019-12-17 12:28:37,253] jepsen worker 10 - jepsen.util 1270 :fail :cas [3 [5 11]] :ValueNotSwapFrom INFO [2019-12-17 12:28:37,253] jepsen worker 10 - jepsen.util 1270 :invoke :cas [3 [29 35]] INFO [2019-12-17 12:28:37,257] jepsen worker 27 - jepsen.util 1107 :ok :read [9 8] INFO [2019-12-17 12:28:37,258] jepsen worker 27 - jepsen.util 1107 :invoke :write [9 1] INFO [2019-12-17 12:28:37,267] jepsen worker 32 - jepsen.util 1292 :fail :cas [10 [29 1]] :ValueNotSwapFrom INFO [2019-12-17 12:28:37,267] jepsen worker 32 - jepsen.util 1292 :invoke :write [10 47] INFO [2019-12-17 12:28:37,284] jepsen worker 26 - jepsen.util 1466 :info :write [8 31] :RequestCanceledException INFO [2019-12-17 12:28:37,284] jepsen worker 5 - jepsen.util 815 :info :write [1 14] :RequestCanceledException INFO [2019-12-17 12:28:37,285] jepsen worker 26 - jepsen.util 1556 :invoke :cas [8 [11 11]] INFO [2019-12-17 12:28:37,285] jepsen worker 5 - jepsen.util 905 :invoke :cas [1 [20 36]] INFO [2019-12-17 12:28:37,292] jepsen worker 28 - jepsen.util 928 :ok :write [9 5] INFO [2019-12-17 12:28:37,293] jepsen worker 28 - jepsen.util 928 :invoke :write [9 41] INFO [2019-12-17 12:28:37,293] jepsen worker 20 - jepsen.util 1280 :ok :read [6 13] INFO [2019-12-17 12:28:37,294] jepsen worker 20 - jepsen.util 1280 :invoke :read [6 nil] INFO [2019-12-17 12:28:37,321] jepsen worker 10 - jepsen.util 1270 :fail :cas [3 [29 35]] :ValueNotSwapFrom INFO [2019-12-17 12:28:37,321] jepsen worker 10 - jepsen.util 1270 :invoke :write [3 46] INFO [2019-12-17 12:28:37,324] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:28:37,325] jepsen nemesis - couchbase.util Rebalance check count: 45 INFO [2019-12-17 12:28:37,325] jepsen nemesis - couchbase.util Rebalance stuck count: 26 INFO [2019-12-17 12:28:37,325] jepsen worker 14 - jepsen.util 1184 :ok :write [4 47] INFO [2019-12-17 12:28:37,326] jepsen worker 14 - jepsen.util 1184 :invoke :read [4 nil] INFO [2019-12-17 12:28:37,342] jepsen worker 72 - jepsen.util 1242 :ok :write [24 12] INFO [2019-12-17 12:28:37,342] jepsen worker 72 - jepsen.util 1242 :invoke :read [24 nil] INFO [2019-12-17 12:28:37,373] jepsen worker 20 - jepsen.util 1280 :ok :read [6 13] INFO [2019-12-17 12:28:37,373] jepsen worker 20 - jepsen.util 1280 :invoke :cas [6 [43 10]] INFO [2019-12-17 12:28:37,388] jepsen worker 6 - jepsen.util 1536 :info :cas [2 [32 11]] :RequestCanceledException INFO [2019-12-17 12:28:37,389] jepsen worker 6 - jepsen.util 1626 :invoke :write [2 36] INFO [2019-12-17 12:28:37,389] jepsen worker 72 - jepsen.util 1242 :ok :read [24 12] INFO [2019-12-17 12:28:37,390] jepsen worker 72 - jepsen.util 1242 :invoke :cas [24 [17 12]] INFO [2019-12-17 12:28:37,403] jepsen worker 74 - jepsen.util 1154 :info :cas [24 [12 15]] :RequestCanceledException INFO [2019-12-17 12:28:37,405] jepsen worker 74 - jepsen.util 1244 :invoke :read [24 nil] INFO [2019-12-17 12:28:37,419] jepsen worker 28 - jepsen.util 928 :ok :write [9 41] INFO [2019-12-17 12:28:37,419] jepsen worker 28 - jepsen.util 928 :invoke :read [9 nil] INFO [2019-12-17 12:28:37,419] jepsen worker 72 - jepsen.util 1242 :fail :cas [24 [17 12]] :ValueNotSwapFrom INFO [2019-12-17 12:28:37,419] jepsen worker 72 - jepsen.util 1242 :invoke :write [24 30] INFO [2019-12-17 12:28:37,420] jepsen worker 55 - jepsen.util 1675 :ok :read [18 17] INFO [2019-12-17 12:28:37,420] jepsen worker 55 - jepsen.util 1675 :invoke :read [18 nil] INFO [2019-12-17 12:28:37,431] jepsen worker 10 - jepsen.util 1270 :ok :write [3 46] INFO [2019-12-17 12:28:37,431] jepsen worker 10 - jepsen.util 1270 :invoke :read [3 nil] INFO [2019-12-17 12:28:37,441] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":6958,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:28:37,441] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":6958,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:28:37,441] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":6962,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:28:37,441] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":6962,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:28:37,450] jepsen worker 64 - jepsen.util 784 :info :write [21 45] :RequestCanceledException INFO [2019-12-17 12:28:37,450] jepsen worker 74 - jepsen.util 1244 :ok :read [24 12] INFO [2019-12-17 12:28:37,450] jepsen worker 74 - jepsen.util 1244 :invoke :read [24 nil] INFO [2019-12-17 12:28:37,454] jepsen worker 64 - jepsen.util 874 :invoke :cas [21 [26 40]] INFO [2019-12-17 12:28:37,469] jepsen worker 20 - jepsen.util 1280 :fail :cas [6 [43 10]] :ValueNotSwapFrom INFO [2019-12-17 12:28:37,469] jepsen worker 20 - jepsen.util 1280 :invoke :read [6 nil] INFO [2019-12-17 12:28:37,479] jepsen worker 28 - jepsen.util 928 :ok :read [9 41] INFO [2019-12-17 12:28:37,479] jepsen worker 28 - jepsen.util 928 :invoke :write [9 9] INFO [2019-12-17 12:28:37,490] jepsen worker 79 - jepsen.util 1249 :ok :read [26 32] INFO [2019-12-17 12:28:37,491] jepsen worker 79 - jepsen.util 1249 :invoke :cas [26 [7 37]] INFO [2019-12-17 12:28:37,494] jepsen worker 33 - jepsen.util 1833 :ok :read [11 7] INFO [2019-12-17 12:28:37,494] jepsen worker 33 - jepsen.util 1833 :invoke :cas [11 [13 17]] INFO [2019-12-17 12:28:37,494] jepsen worker 37 - jepsen.util 1117 :ok :read [12 36] INFO [2019-12-17 12:28:37,495] jepsen worker 37 - jepsen.util 1117 :invoke :cas [12 [5 39]] INFO [2019-12-17 12:28:37,509] jepsen worker 10 - jepsen.util 1270 :ok :read [3 46] INFO [2019-12-17 12:28:37,510] jepsen worker 10 - jepsen.util 1270 :invoke :write [3 42] INFO [2019-12-17 12:28:37,516] jepsen worker 26 - jepsen.util 1556 :fail :cas [8 [11 11]] :ValueNotSwapFrom INFO [2019-12-17 12:28:37,516] jepsen worker 26 - jepsen.util 1556 :invoke :write [8 11] INFO [2019-12-17 12:28:37,519] jepsen worker 8 - jepsen.util 1448 :info :write [2 24] :RequestCanceledException INFO [2019-12-17 12:28:37,521] jepsen worker 8 - jepsen.util 1538 :invoke :cas [2 [4 16]] INFO [2019-12-17 12:28:37,534] jepsen worker 38 - jepsen.util 1118 :info :write [12 8] :RequestCanceledException INFO [2019-12-17 12:28:37,536] jepsen worker 38 - jepsen.util 1208 :invoke :write [12 37] INFO [2019-12-17 12:28:37,537] jepsen worker 64 - jepsen.util 874 :fail :cas [21 [26 40]] :ValueNotSwapFrom INFO [2019-12-17 12:28:37,537] jepsen worker 64 - jepsen.util 874 :invoke :write [21 13] INFO [2019-12-17 12:28:37,542] jepsen worker 74 - jepsen.util 1244 :ok :read [24 12] INFO [2019-12-17 12:28:37,542] jepsen worker 74 - jepsen.util 1244 :invoke :read [24 nil] INFO [2019-12-17 12:28:37,549] jepsen worker 34 - jepsen.util 1654 :info :cas [11 [43 33]] :RequestCanceledException INFO [2019-12-17 12:28:37,550] jepsen worker 34 - jepsen.util 1744 :invoke :cas [11 [27 9]] INFO [2019-12-17 12:28:37,559] jepsen worker 20 - jepsen.util 1280 :ok :read [6 13] INFO [2019-12-17 12:28:37,559] jepsen worker 20 - jepsen.util 1280 :invoke :cas [6 [25 37]] INFO [2019-12-17 12:28:37,560] jepsen worker 52 - jepsen.util 1402 :ok :read [17 42] INFO [2019-12-17 12:28:37,560] jepsen worker 52 - jepsen.util 1402 :invoke :cas [17 [24 43]] INFO [2019-12-17 12:28:37,588] jepsen worker 21 - jepsen.util 1461 :info :cas [7 [2 17]] :RequestCanceledException INFO [2019-12-17 12:28:37,589] jepsen worker 21 - jepsen.util 1551 :invoke :write [7 13] INFO [2019-12-17 12:28:37,598] jepsen worker 28 - jepsen.util 928 :ok :write [9 9] INFO [2019-12-17 12:28:37,598] jepsen worker 28 - jepsen.util 928 :invoke :write [9 34] INFO [2019-12-17 12:28:37,606] jepsen worker 44 - jepsen.util 1394 :info :cas [14 [29 39]] :RequestCanceledException INFO [2019-12-17 12:28:37,607] jepsen worker 44 - jepsen.util 1484 :invoke :cas [14 [30 25]] INFO [2019-12-17 12:28:37,611] jepsen worker 79 - jepsen.util 1249 :fail :cas [26 [7 37]] :ValueNotSwapFrom INFO [2019-12-17 12:28:37,611] jepsen worker 79 - jepsen.util 1249 :invoke :read [26 nil] INFO [2019-12-17 12:28:37,617] jepsen worker 5 - jepsen.util 905 :fail :cas [1 [20 36]] :ValueNotSwapFrom INFO [2019-12-17 12:28:37,617] jepsen worker 5 - jepsen.util 905 :invoke :cas [1 [6 14]] INFO [2019-12-17 12:28:37,646] jepsen worker 79 - jepsen.util 1249 :ok :read [26 32] INFO [2019-12-17 12:28:37,646] jepsen worker 79 - jepsen.util 1249 :invoke :write [26 1] INFO [2019-12-17 12:28:37,656] jepsen worker 87 - jepsen.util 1257 :ok :read [29 12] INFO [2019-12-17 12:28:37,656] jepsen worker 87 - jepsen.util 1257 :invoke :read [29 nil] INFO [2019-12-17 12:28:37,672] jepsen worker 28 - jepsen.util 928 :ok :write [9 34] INFO [2019-12-17 12:28:37,673] jepsen worker 28 - jepsen.util 928 :invoke :write [9 29] INFO [2019-12-17 12:28:37,695] jepsen worker 8 - jepsen.util 1538 :fail :cas [2 [4 16]] :ValueNotSwapFrom INFO [2019-12-17 12:28:37,696] jepsen worker 8 - jepsen.util 1538 :invoke :write [2 10] INFO [2019-12-17 12:28:37,704] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":6987,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:28:37,704] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":6987,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:28:37,705] jepsen worker 43 - jepsen.util 1663 :info :cas [14 [6 15]] :RequestCanceledException INFO [2019-12-17 12:28:37,706] jepsen worker 43 - jepsen.util 1753 :invoke :read [14 nil] INFO [2019-12-17 12:28:37,719] jepsen worker 28 - jepsen.util 928 :ok :write [9 29] INFO [2019-12-17 12:28:37,720] jepsen worker 28 - jepsen.util 928 :invoke :cas [9 [23 35]] INFO [2019-12-17 12:28:37,735] jepsen worker 65 - jepsen.util 965 :info :write [21 3] :RequestCanceledException INFO [2019-12-17 12:28:37,737] jepsen worker 65 - jepsen.util 1055 :invoke :read [21 nil] INFO [2019-12-17 12:28:37,744] jepsen worker 28 - jepsen.util 928 :fail :cas [9 [23 35]] :ValueNotSwapFrom INFO [2019-12-17 12:28:37,744] jepsen worker 28 - jepsen.util 928 :invoke :write [9 14] INFO [2019-12-17 12:28:37,746] jepsen worker 10 - jepsen.util 1270 :ok :write [3 42] INFO [2019-12-17 12:28:37,746] jepsen worker 10 - jepsen.util 1270 :invoke :read [3 nil] INFO [2019-12-17 12:28:37,749] jepsen worker 56 - jepsen.util 1226 :info :write [18 35] :RequestCanceledException INFO [2019-12-17 12:28:37,749] jepsen worker 12 - jepsen.util 1272 :info :write [4 7] :RequestCanceledException INFO [2019-12-17 12:28:37,751] jepsen worker 12 - jepsen.util 1362 :invoke :write [4 26] INFO [2019-12-17 12:28:37,751] jepsen worker 56 - jepsen.util 1316 :invoke :read [18 nil] INFO [2019-12-17 12:28:37,754] jepsen worker 65 - jepsen.util 1055 :ok :read [21 25] INFO [2019-12-17 12:28:37,754] jepsen worker 65 - jepsen.util 1055 :invoke :write [21 20] INFO [2019-12-17 12:28:37,769] jepsen worker 20 - jepsen.util 1280 :fail :cas [6 [25 37]] :ValueNotSwapFrom INFO [2019-12-17 12:28:37,769] jepsen worker 20 - jepsen.util 1280 :invoke :read [6 nil] INFO [2019-12-17 12:28:37,789] jepsen worker 10 - jepsen.util 1270 :ok :read [3 42] INFO [2019-12-17 12:28:37,790] jepsen worker 10 - jepsen.util 1270 :invoke :read [3 nil] INFO [2019-12-17 12:28:37,793] jepsen worker 72 - jepsen.util 1242 :ok :write [24 30] INFO [2019-12-17 12:28:37,793] jepsen worker 72 - jepsen.util 1242 :invoke :write [24 43] INFO [2019-12-17 12:28:37,807] jepsen worker 20 - jepsen.util 1280 :ok :read [6 13] INFO [2019-12-17 12:28:37,807] jepsen worker 20 - jepsen.util 1280 :invoke :cas [6 [10 46]] INFO [2019-12-17 12:28:37,821] jepsen worker 22 - jepsen.util 1552 :info :write [7 45] :RequestCanceledException INFO [2019-12-17 12:28:37,822] jepsen worker 22 - jepsen.util 1642 :invoke :read [7 nil] INFO [2019-12-17 12:28:37,836] jepsen worker 18 - jepsen.util 1368 :info :write [6 36] :RequestCanceledException INFO [2019-12-17 12:28:37,837] jepsen worker 18 - jepsen.util 1458 :invoke :read [6 nil] INFO [2019-12-17 12:28:37,842] jepsen worker 10 - jepsen.util 1270 :ok :read [3 42] INFO [2019-12-17 12:28:37,842] jepsen worker 10 - jepsen.util 1270 :invoke :cas [3 [48 43]] INFO [2019-12-17 12:28:37,845] jepsen worker 88 - jepsen.util 1168 :fail :cas [29 [27 14]] :ValueNotSwapFrom INFO [2019-12-17 12:28:37,845] jepsen worker 88 - jepsen.util 1168 :invoke :write [29 6] INFO [2019-12-17 12:28:37,864] jepsen worker 53 - jepsen.util 1223 :fail :cas [17 [47 10]] :ValueNotSwapFrom INFO [2019-12-17 12:28:37,864] jepsen worker 53 - jepsen.util 1223 :invoke :cas [17 [45 30]] INFO [2019-12-17 12:28:37,874] jepsen worker 20 - jepsen.util 1280 :fail :cas [6 [10 46]] :ValueNotSwapFrom INFO [2019-12-17 12:28:37,875] jepsen worker 20 - jepsen.util 1280 :invoke :write [6 46] INFO [2019-12-17 12:28:37,878] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":7013,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:28:37,878] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":7013,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:28:37,878] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":7016,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:28:37,878] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":7016,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:28:37,878] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":7017,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:28:37,878] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":7017,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:28:37,895] jepsen worker 33 - jepsen.util 1833 :fail :cas [11 [13 17]] :ValueNotSwapFrom INFO [2019-12-17 12:28:37,895] jepsen worker 70 - jepsen.util 1510 :fail :cas [23 [10 28]] :ValueNotSwapFrom INFO [2019-12-17 12:28:37,895] jepsen worker 33 - jepsen.util 1833 :invoke :read [11 nil] INFO [2019-12-17 12:28:37,895] jepsen worker 70 - jepsen.util 1510 :invoke :read [23 nil] INFO [2019-12-17 12:28:37,914] jepsen worker 28 - jepsen.util 928 :ok :write [9 14] INFO [2019-12-17 12:28:37,914] jepsen worker 28 - jepsen.util 928 :invoke :read [9 nil] INFO [2019-12-17 12:28:37,916] jepsen worker 10 - jepsen.util 1270 :fail :cas [3 [48 43]] :ValueNotSwapFrom INFO [2019-12-17 12:28:37,916] jepsen worker 10 - jepsen.util 1270 :invoke :cas [3 [0 17]] INFO [2019-12-17 12:28:37,951] jepsen worker 33 - jepsen.util 1833 :ok :read [11 7] INFO [2019-12-17 12:28:37,951] jepsen worker 33 - jepsen.util 1833 :invoke :cas [11 [38 27]] INFO [2019-12-17 12:28:37,953] jepsen worker 7 - jepsen.util 1717 :info :write [2 10] :RequestCanceledException INFO [2019-12-17 12:28:37,959] jepsen worker 7 - jepsen.util 1807 :invoke :cas [2 [5 11]] INFO [2019-12-17 12:28:37,959] jepsen worker 28 - jepsen.util 928 :ok :read [9 14] INFO [2019-12-17 12:28:37,959] jepsen worker 28 - jepsen.util 928 :invoke :read [9 nil] INFO [2019-12-17 12:28:37,975] jepsen worker 14 - jepsen.util 1184 :ok :read [4 47] INFO [2019-12-17 12:28:37,975] jepsen worker 14 - jepsen.util 1184 :invoke :write [4 16] INFO [2019-12-17 12:28:38,003] jepsen worker 28 - jepsen.util 928 :ok :read [9 14] INFO [2019-12-17 12:28:38,003] jepsen worker 28 - jepsen.util 928 :invoke :read [9 nil] INFO [2019-12-17 12:28:38,007] jepsen worker 65 - jepsen.util 1055 :ok :write [21 20] INFO [2019-12-17 12:28:38,007] jepsen worker 65 - jepsen.util 1055 :invoke :cas [21 [22 38]] INFO [2019-12-17 12:28:38,020] jepsen worker 72 - jepsen.util 1242 :ok :write [24 43] INFO [2019-12-17 12:28:38,020] jepsen worker 72 - jepsen.util 1242 :invoke :cas [24 [35 9]] INFO [2019-12-17 12:28:38,022] jepsen worker 62 - jepsen.util 1502 :info :write [20 8] :RequestCanceledException INFO [2019-12-17 12:28:38,023] jepsen worker 62 - jepsen.util 1592 :invoke :write [20 26] INFO [2019-12-17 12:28:38,043] jepsen worker 28 - jepsen.util 928 :ok :read [9 14] INFO [2019-12-17 12:28:38,043] jepsen worker 28 - jepsen.util 928 :invoke :read [9 nil] INFO [2019-12-17 12:28:38,055] jepsen worker 65 - jepsen.util 1055 :fail :cas [21 [22 38]] :ValueNotSwapFrom INFO [2019-12-17 12:28:38,055] jepsen worker 65 - jepsen.util 1055 :invoke :cas [21 [8 13]] INFO [2019-12-17 12:28:38,066] jepsen worker 72 - jepsen.util 1242 :fail :cas [24 [35 9]] :ValueNotSwapFrom INFO [2019-12-17 12:28:38,067] jepsen worker 72 - jepsen.util 1242 :invoke :cas [24 [12 2]] INFO [2019-12-17 12:28:38,075] jepsen worker 4 - jepsen.util 1084 :ok :write [1 30] INFO [2019-12-17 12:28:38,075] jepsen worker 4 - jepsen.util 1084 :invoke :read [1 nil] INFO [2019-12-17 12:28:38,088] jepsen worker 78 - jepsen.util 1338 :ok :write [26 46] INFO [2019-12-17 12:28:38,088] jepsen worker 78 - jepsen.util 1338 :invoke :write [26 27] INFO [2019-12-17 12:28:38,088] jepsen worker 36 - jepsen.util 1296 :ok :write [12 34] INFO [2019-12-17 12:28:38,088] jepsen worker 36 - jepsen.util 1296 :invoke :write [12 3] INFO [2019-12-17 12:28:38,097] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":7031,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:28:38,097] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":7031,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:28:38,108] jepsen worker 28 - jepsen.util 928 :ok :read [9 14] INFO [2019-12-17 12:28:38,108] jepsen worker 28 - jepsen.util 928 :invoke :cas [9 [10 39]] INFO [2019-12-17 12:28:38,146] jepsen worker 72 - jepsen.util 1242 :fail :cas [24 [12 2]] :ValueNotSwapFrom INFO [2019-12-17 12:28:38,146] jepsen worker 72 - jepsen.util 1242 :invoke :read [24 nil] INFO [2019-12-17 12:28:38,148] jepsen worker 70 - jepsen.util 1510 :ok :read [23 39] INFO [2019-12-17 12:28:38,149] jepsen worker 70 - jepsen.util 1510 :invoke :write [23 36] INFO [2019-12-17 12:28:38,175] jepsen worker 28 - jepsen.util 928 :fail :cas [9 [10 39]] :ValueNotSwapFrom INFO [2019-12-17 12:28:38,175] jepsen worker 28 - jepsen.util 928 :invoke :write [9 29] INFO [2019-12-17 12:28:38,175] jepsen worker 69 - jepsen.util 1059 :ok :write [23 23] INFO [2019-12-17 12:28:38,175] jepsen worker 69 - jepsen.util 1059 :invoke :read [23 nil] INFO [2019-12-17 12:28:38,177] jepsen worker 65 - jepsen.util 1055 :fail :cas [21 [8 13]] :ValueNotSwapFrom INFO [2019-12-17 12:28:38,177] jepsen worker 65 - jepsen.util 1055 :invoke :cas [21 [22 31]] INFO [2019-12-17 12:28:38,178] jepsen worker 5 - jepsen.util 905 :fail :cas [1 [6 14]] :ValueNotSwapFrom INFO [2019-12-17 12:28:38,178] jepsen worker 5 - jepsen.util 905 :invoke :write [1 4] INFO [2019-12-17 12:28:38,188] jepsen worker 47 - jepsen.util 1667 :fail :read [15 nil] :RequestTimeoutException INFO [2019-12-17 12:28:38,188] jepsen worker 47 - jepsen.util 1667 :invoke :write [15 22] INFO [2019-12-17 12:28:38,195] jepsen worker 72 - jepsen.util 1242 :ok :read [24 43] INFO [2019-12-17 12:28:38,195] jepsen worker 72 - jepsen.util 1242 :invoke :write [24 30] INFO [2019-12-17 12:28:38,201] jepsen worker 79 - jepsen.util 1249 :ok :write [26 1] INFO [2019-12-17 12:28:38,201] jepsen worker 79 - jepsen.util 1249 :invoke :cas [26 [16 47]] INFO [2019-12-17 12:28:38,207] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":7051,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:28:38,207] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":7061,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:28:38,207] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":7051,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:28:38,207] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":7061,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:28:38,208] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":7052,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:28:38,208] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":7065,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:28:38,208] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":7052,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:28:38,208] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":7065,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:28:38,213] jepsen worker 65 - jepsen.util 1055 :fail :cas [21 [22 31]] :ValueNotSwapFrom INFO [2019-12-17 12:28:38,214] jepsen worker 65 - jepsen.util 1055 :invoke :cas [21 [19 35]] INFO [2019-12-17 12:28:38,258] jepsen worker 79 - jepsen.util 1249 :fail :cas [26 [16 47]] :ValueNotSwapFrom INFO [2019-12-17 12:28:38,259] jepsen worker 79 - jepsen.util 1249 :invoke :read [26 nil] INFO [2019-12-17 12:28:38,267] jepsen worker 65 - jepsen.util 1055 :fail :cas [21 [19 35]] :ValueNotSwapFrom INFO [2019-12-17 12:28:38,267] jepsen worker 65 - jepsen.util 1055 :invoke :cas [21 [47 49]] INFO [2019-12-17 12:28:38,270] jepsen worker 20 - jepsen.util 1280 :ok :write [6 46] INFO [2019-12-17 12:28:38,270] jepsen worker 20 - jepsen.util 1280 :invoke :read [6 nil] INFO [2019-12-17 12:28:38,302] jepsen worker 28 - jepsen.util 928 :ok :write [9 29] INFO [2019-12-17 12:28:38,302] jepsen worker 28 - jepsen.util 928 :invoke :cas [9 [47 26]] INFO [2019-12-17 12:28:38,306] jepsen worker 55 - jepsen.util 1675 :ok :read [18 17] INFO [2019-12-17 12:28:38,306] jepsen worker 73 - jepsen.util 1243 :fail :read [24 nil] :RequestTimeoutException INFO [2019-12-17 12:28:38,306] jepsen worker 73 - jepsen.util 1243 :invoke :read [24 nil] INFO [2019-12-17 12:28:38,307] jepsen worker 55 - jepsen.util 1675 :invoke :cas [18 [41 19]] INFO [2019-12-17 12:28:38,315] jepsen worker 10 - jepsen.util 1270 :fail :cas [3 [0 17]] :ValueNotSwapFrom INFO [2019-12-17 12:28:38,315] jepsen worker 10 - jepsen.util 1270 :invoke :cas [3 [22 11]] INFO [2019-12-17 12:28:38,318] jepsen worker 13 - jepsen.util 1273 :ok :write [4 11] INFO [2019-12-17 12:28:38,318] jepsen worker 13 - jepsen.util 1273 :invoke :cas [4 [32 18]] INFO [2019-12-17 12:28:38,333] jepsen worker 37 - jepsen.util 1117 :fail :cas [12 [5 39]] :ValueNotSwapFrom INFO [2019-12-17 12:28:38,333] jepsen worker 37 - jepsen.util 1117 :invoke :write [12 37] INFO [2019-12-17 12:28:38,372] jepsen worker 20 - jepsen.util 1280 :ok :read [6 46] INFO [2019-12-17 12:28:38,372] jepsen worker 20 - jepsen.util 1280 :invoke :cas [6 [26 7]] INFO [2019-12-17 12:28:38,389] jepsen worker 56 - jepsen.util 1316 :ok :read [18 17] INFO [2019-12-17 12:28:38,389] jepsen worker 56 - jepsen.util 1316 :invoke :read [18 nil] INFO [2019-12-17 12:28:38,390] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:28:38,391] jepsen nemesis - couchbase.util Rebalance check count: 46 INFO [2019-12-17 12:28:38,391] jepsen nemesis - couchbase.util Rebalance stuck count: 27 INFO [2019-12-17 12:28:38,392] jepsen worker 79 - jepsen.util 1249 :ok :read [26 1] INFO [2019-12-17 12:28:38,392] jepsen worker 79 - jepsen.util 1249 :invoke :write [26 30] INFO [2019-12-17 12:28:38,396] jepsen worker 80 - jepsen.util 1430 :ok :write [26 14] INFO [2019-12-17 12:28:38,396] jepsen worker 3 - jepsen.util 1083 :ok :write [1 20] INFO [2019-12-17 12:28:38,397] jepsen worker 3 - jepsen.util 1083 :invoke :write [1 11] INFO [2019-12-17 12:28:38,397] jepsen worker 80 - jepsen.util 1430 :invoke :cas [26 [46 29]] INFO [2019-12-17 12:28:38,402] jepsen worker 28 - jepsen.util 928 :fail :cas [9 [47 26]] :ValueNotSwapFrom INFO [2019-12-17 12:28:38,402] jepsen worker 28 - jepsen.util 928 :invoke :write [9 8] INFO [2019-12-17 12:28:38,409] jepsen worker 33 - jepsen.util 1833 :fail :cas [11 [38 27]] :ValueNotSwapFrom INFO [2019-12-17 12:28:38,409] jepsen worker 33 - jepsen.util 1833 :invoke :write [11 12] INFO [2019-12-17 12:28:38,422] jepsen worker 30 - jepsen.util 1470 :fail :read [10 nil] :RequestTimeoutException INFO [2019-12-17 12:28:38,422] jepsen worker 11 - jepsen.util 1631 :info :write [3 34] :RequestCanceledException INFO [2019-12-17 12:28:38,422] jepsen worker 76 - jepsen.util 1516 :info :write [25 0] :RequestCanceledException INFO [2019-12-17 12:28:38,422] jepsen worker 9 - jepsen.util 1269 :info :write [3 24] :RequestCanceledException INFO [2019-12-17 12:28:38,422] jepsen worker 85 - jepsen.util 1525 :info :write [28 5] :RequestCanceledException INFO [2019-12-17 12:28:38,422] jepsen worker 30 - jepsen.util 1470 :invoke :read [10 nil] INFO [2019-12-17 12:28:38,422] jepsen worker 0 - jepsen.util 1620 :info :cas [0 [29 46]] :RequestCanceledException INFO [2019-12-17 12:28:38,423] jepsen worker 11 - jepsen.util 1721 :invoke :read [3 nil] INFO [2019-12-17 12:28:38,424] jepsen worker 65 - jepsen.util 1055 :fail :cas [21 [47 49]] :ValueNotSwapFrom INFO [2019-12-17 12:28:38,424] jepsen worker 65 - jepsen.util 1055 :invoke :write [21 42] INFO [2019-12-17 12:28:38,425] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":7084,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:28:38,426] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":7084,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:28:38,426] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":7088,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:28:38,426] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":7088,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:28:38,426] jepsen worker 76 - jepsen.util 1606 :invoke :read [25 nil] INFO [2019-12-17 12:28:38,426] jepsen worker 85 - jepsen.util 1615 :invoke :write [28 43] INFO [2019-12-17 12:28:38,426] jepsen worker 9 - jepsen.util 1359 :invoke :cas [3 [0 3]] INFO [2019-12-17 12:28:38,426] jepsen worker 0 - jepsen.util 1710 :invoke :write [0 43] INFO [2019-12-17 12:28:38,435] jepsen worker 82 - jepsen.util 1882 :info :cas [27 [35 35]] :RequestCanceledException INFO [2019-12-17 12:28:38,435] jepsen worker 51 - jepsen.util 1221 :info :write [17 9] :RequestCanceledException INFO [2019-12-17 12:28:38,436] jepsen worker 51 - jepsen.util 1311 :invoke :read [17 nil] INFO [2019-12-17 12:28:38,439] jepsen worker 82 - jepsen.util 1972 :invoke :cas [27 [8 8]] INFO [2019-12-17 12:28:38,444] jepsen worker 73 - jepsen.util 1243 :ok :read [24 43] INFO [2019-12-17 12:28:38,444] jepsen worker 73 - jepsen.util 1243 :invoke :write [24 30] INFO [2019-12-17 12:28:38,452] jepsen worker 63 - jepsen.util 1143 :info :cas [21 [34 34]] :RequestCanceledException INFO [2019-12-17 12:28:38,456] jepsen worker 63 - jepsen.util 1233 :invoke :write [21 3] INFO [2019-12-17 12:28:38,477] jepsen worker 10 - jepsen.util 1270 :fail :cas [3 [22 11]] :ValueNotSwapFrom INFO [2019-12-17 12:28:38,478] jepsen worker 10 - jepsen.util 1270 :invoke :read [3 nil] INFO [2019-12-17 12:28:38,485] jepsen worker 53 - jepsen.util 1223 :fail :cas [17 [45 30]] :ValueNotSwapFrom INFO [2019-12-17 12:28:38,486] jepsen worker 53 - jepsen.util 1223 :invoke :write [17 44] INFO [2019-12-17 12:28:38,493] jepsen worker 38 - jepsen.util 1208 :ok :write [12 37] INFO [2019-12-17 12:28:38,493] jepsen worker 38 - jepsen.util 1208 :invoke :read [12 nil] INFO [2019-12-17 12:28:38,500] jepsen worker 89 - jepsen.util 1439 :ok :write [29 40] INFO [2019-12-17 12:28:38,500] jepsen worker 89 - jepsen.util 1439 :invoke :write [29 32] INFO [2019-12-17 12:28:38,513] jepsen worker 30 - jepsen.util 1470 :ok :read [10 8] INFO [2019-12-17 12:28:38,513] jepsen worker 30 - jepsen.util 1470 :invoke :read [10 nil] INFO [2019-12-17 12:28:38,521] jepsen worker 48 - jepsen.util 1038 :info :cas [16 [19 16]] :RequestCanceledException INFO [2019-12-17 12:28:38,524] jepsen worker 48 - jepsen.util 1128 :invoke :cas [16 [29 39]] INFO [2019-12-17 12:28:38,553] jepsen worker 24 - jepsen.util 1464 :info :cas [8 [20 9]] :RequestCanceledException INFO [2019-12-17 12:28:38,554] jepsen worker 24 - jepsen.util 1554 :invoke :read [8 nil] INFO [2019-12-17 12:28:38,555] jepsen worker 11 - jepsen.util 1721 :ok :read [3 42] INFO [2019-12-17 12:28:38,556] jepsen worker 11 - jepsen.util 1721 :invoke :cas [3 [22 28]] INFO [2019-12-17 12:28:38,557] jepsen worker 56 - jepsen.util 1316 :ok :read [18 17] INFO [2019-12-17 12:28:38,557] jepsen worker 56 - jepsen.util 1316 :invoke :write [18 26] INFO [2019-12-17 12:28:38,560] jepsen worker 22 - jepsen.util 1642 :ok :read [7 20] INFO [2019-12-17 12:28:38,560] jepsen worker 22 - jepsen.util 1642 :invoke :read [7 nil] INFO [2019-12-17 12:28:38,569] jepsen worker 20 - jepsen.util 1280 :fail :cas [6 [26 7]] :ValueNotSwapFrom INFO [2019-12-17 12:28:38,569] jepsen worker 20 - jepsen.util 1280 :invoke :read [6 nil] INFO [2019-12-17 12:28:38,584] jepsen worker 29 - jepsen.util 1109 :info :cas [9 [34 49]] :RequestCanceledException INFO [2019-12-17 12:28:38,584] jepsen worker 57 - jepsen.util 1497 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:28:38,584] jepsen worker 57 - jepsen.util 1497 :invoke :read [19 nil] INFO [2019-12-17 12:28:38,585] jepsen worker 29 - jepsen.util 1199 :invoke :write [9 24] INFO [2019-12-17 12:28:38,594] jepsen worker 28 - jepsen.util 928 :ok :write [9 8] INFO [2019-12-17 12:28:38,595] jepsen worker 28 - jepsen.util 928 :invoke :read [9 nil] INFO [2019-12-17 12:28:38,606] jepsen worker 23 - jepsen.util 1463 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:28:38,606] jepsen worker 86 - jepsen.util 1616 :info :cas [28 [16 37]] :RequestCanceledException INFO [2019-12-17 12:28:38,606] jepsen worker 23 - jepsen.util 1463 :invoke :read [7 nil] INFO [2019-12-17 12:28:38,608] jepsen worker 86 - jepsen.util 1706 :invoke :read [28 nil] INFO [2019-12-17 12:28:38,615] jepsen worker 52 - jepsen.util 1402 :fail :cas [17 [24 43]] :ValueNotSwapFrom INFO [2019-12-17 12:28:38,616] jepsen worker 52 - jepsen.util 1402 :invoke :cas [17 [44 29]] INFO [2019-12-17 12:28:38,621] jepsen worker 16 - jepsen.util 1456 :info :cas [5 [22 18]] :RequestCanceledException INFO [2019-12-17 12:28:38,621] jepsen worker 66 - jepsen.util 1236 :info :cas [22 [47 25]] :RequestCanceledException INFO [2019-12-17 12:28:38,621] jepsen worker 35 - jepsen.util 1205 :info :write [11 11] :RequestCanceledException INFO [2019-12-17 12:28:38,622] jepsen worker 59 - jepsen.util 1319 :info :write [19 16] :RequestCanceledException INFO [2019-12-17 12:28:38,622] jepsen worker 84 - jepsen.util 1434 :info :write [28 14] :RequestCanceledException INFO [2019-12-17 12:28:38,622] jepsen worker 1 - jepsen.util 1441 :info :cas [0 [34 49]] :RequestCanceledException INFO [2019-12-17 12:28:38,623] jepsen worker 16 - jepsen.util 1546 :invoke :read [5 nil] INFO [2019-12-17 12:28:38,623] jepsen worker 35 - jepsen.util 1295 :invoke :cas [11 [36 32]] INFO [2019-12-17 12:28:38,625] jepsen worker 66 - jepsen.util 1326 :invoke :read [22 nil] INFO [2019-12-17 12:28:38,625] jepsen worker 1 - jepsen.util 1531 :invoke :cas [0 [16 39]] INFO [2019-12-17 12:28:38,626] jepsen worker 84 - jepsen.util 1524 :invoke :cas [28 [11 47]] INFO [2019-12-17 12:28:38,626] jepsen worker 59 - jepsen.util 1409 :invoke :read [19 nil] INFO [2019-12-17 12:28:38,635] jepsen worker 83 - jepsen.util 1793 :info :write [27 7] :RequestCanceledException INFO [2019-12-17 12:28:38,635] jepsen worker 50 - jepsen.util 1490 :info :cas [16 [38 18]] :RequestCanceledException INFO [2019-12-17 12:28:38,638] jepsen worker 83 - jepsen.util 1883 :invoke :cas [27 [20 16]] INFO [2019-12-17 12:28:38,638] jepsen worker 50 - jepsen.util 1580 :invoke :cas [16 [25 49]] INFO [2019-12-17 12:28:38,645] jepsen worker 87 - jepsen.util 1257 :ok :read [29 40] INFO [2019-12-17 12:28:38,645] jepsen worker 87 - jepsen.util 1257 :invoke :read [29 nil] INFO [2019-12-17 12:28:38,653] jepsen worker 49 - jepsen.util 1309 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:28:38,653] jepsen worker 17 - jepsen.util 1457 :info :write [5 5] :RequestCanceledException INFO [2019-12-17 12:28:38,653] jepsen worker 41 - jepsen.util 1301 :info :write [13 41] :RequestCanceledException INFO [2019-12-17 12:28:38,653] jepsen worker 49 - jepsen.util 1309 :invoke :read [16 nil] INFO [2019-12-17 12:28:38,658] jepsen worker 17 - jepsen.util 1547 :invoke :cas [5 [27 13]] INFO [2019-12-17 12:28:38,659] jepsen worker 41 - jepsen.util 1391 :invoke :write [13 14] INFO [2019-12-17 12:28:38,673] jepsen worker 79 - jepsen.util 1249 :ok :write [26 30] INFO [2019-12-17 12:28:38,674] jepsen worker 79 - jepsen.util 1249 :invoke :read [26 nil] INFO [2019-12-17 12:28:38,678] jepsen worker 55 - jepsen.util 1675 :fail :cas [18 [41 19]] :ValueNotSwapFrom INFO [2019-12-17 12:28:38,678] jepsen worker 55 - jepsen.util 1675 :invoke :cas [18 [34 42]] INFO [2019-12-17 12:28:38,679] jepsen worker 11 - jepsen.util 1721 :fail :cas [3 [22 28]] :ValueNotSwapFrom INFO [2019-12-17 12:28:38,679] jepsen worker 11 - jepsen.util 1721 :invoke :write [3 2] INFO [2019-12-17 12:28:38,687] jepsen worker 40 - jepsen.util 1750 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:28:38,687] jepsen worker 68 - jepsen.util 1058 :info :cas [22 [41 8]] :RequestCanceledException INFO [2019-12-17 12:28:38,687] jepsen worker 25 - jepsen.util 1735 :info :write [8 37] :RequestCanceledException INFO [2019-12-17 12:28:38,687] jepsen worker 40 - jepsen.util 1750 :invoke :write [13 24] INFO [2019-12-17 12:28:38,687] jepsen worker 67 - jepsen.util 1777 :info :cas [22 [4 42]] :RequestCanceledException INFO [2019-12-17 12:28:38,688] jepsen worker 68 - jepsen.util 1148 :invoke :write [22 13] INFO [2019-12-17 12:28:38,689] jepsen worker 67 - jepsen.util 1867 :invoke :write [22 4] INFO [2019-12-17 12:28:38,690] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":7128,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:28:38,690] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":7128,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:28:38,690] jepsen worker 25 - jepsen.util 1825 :invoke :cas [8 [32 39]] INFO [2019-12-17 12:28:38,692] jepsen worker 30 - jepsen.util 1470 :ok :read [10 8] INFO [2019-12-17 12:28:38,693] jepsen worker 30 - jepsen.util 1470 :invoke :cas [10 [22 36]] INFO [2019-12-17 12:28:38,702] jepsen worker 5 - jepsen.util 905 :ok :write [1 4] INFO [2019-12-17 12:28:38,702] jepsen worker 5 - jepsen.util 905 :invoke :read [1 nil] INFO [2019-12-17 12:28:38,706] jepsen worker 2 - jepsen.util 1262 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:28:38,706] jepsen worker 2 - jepsen.util 1262 :invoke :read [0 nil] INFO [2019-12-17 12:28:38,710] jepsen worker 79 - jepsen.util 1249 :ok :read [26 30] INFO [2019-12-17 12:28:38,710] jepsen worker 79 - jepsen.util 1249 :invoke :write [26 0] INFO [2019-12-17 12:28:38,712] jepsen worker 14 - jepsen.util 1184 :ok :write [4 16] INFO [2019-12-17 12:28:38,712] jepsen worker 14 - jepsen.util 1184 :invoke :read [4 nil] INFO [2019-12-17 12:28:38,720] jepsen worker 54 - jepsen.util 1404 :info :write [18 25] :RequestCanceledException INFO [2019-12-17 12:28:38,720] jepsen worker 15 - jepsen.util 1185 :info :write [5 44] :RequestCanceledException INFO [2019-12-17 12:28:38,724] jepsen worker 54 - jepsen.util 1494 :invoke :read [18 nil] INFO [2019-12-17 12:28:38,725] jepsen worker 15 - jepsen.util 1275 :invoke :cas [5 [30 3]] INFO [2019-12-17 12:28:38,729] jepsen worker 39 - jepsen.util 1659 :info :write [13 27] :RequestCanceledException INFO [2019-12-17 12:28:38,729] jepsen worker 61 - jepsen.util 1591 :info :write [20 43] :RequestCanceledException INFO [2019-12-17 12:28:38,730] jepsen worker 20 - jepsen.util 1280 :ok :read [6 46] INFO [2019-12-17 12:28:38,731] jepsen worker 20 - jepsen.util 1280 :invoke :write [6 30] INFO [2019-12-17 12:28:38,732] jepsen worker 39 - jepsen.util 1749 :invoke :write [13 40] INFO [2019-12-17 12:28:38,733] jepsen worker 61 - jepsen.util 1681 :invoke :cas [20 [21 48]] INFO [2019-12-17 12:28:38,753] jepsen worker 42 - jepsen.util 1752 :info :write [14 35] :RequestCanceledException INFO [2019-12-17 12:28:38,757] jepsen worker 42 - jepsen.util 1842 :invoke :read [14 nil] INFO [2019-12-17 12:28:38,762] jepsen worker 28 - jepsen.util 928 :ok :read [9 8] INFO [2019-12-17 12:28:38,762] jepsen worker 28 - jepsen.util 928 :invoke :read [9 nil] INFO [2019-12-17 12:28:38,768] jepsen worker 13 - jepsen.util 1273 :fail :cas [4 [32 18]] :ValueNotSwapFrom INFO [2019-12-17 12:28:38,769] jepsen worker 13 - jepsen.util 1273 :invoke :read [4 nil] INFO [2019-12-17 12:28:38,770] jepsen worker 30 - jepsen.util 1470 :fail :cas [10 [22 36]] :ValueNotSwapFrom INFO [2019-12-17 12:28:38,771] jepsen worker 30 - jepsen.util 1470 :invoke :read [10 nil] INFO [2019-12-17 12:28:38,787] jepsen worker 81 - jepsen.util 1521 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:28:38,787] jepsen worker 81 - jepsen.util 1521 :invoke :read [27 nil] INFO [2019-12-17 12:28:38,806] jepsen worker 65 - jepsen.util 1055 :ok :write [21 42] INFO [2019-12-17 12:28:38,806] jepsen worker 65 - jepsen.util 1055 :invoke :write [21 31] INFO [2019-12-17 12:28:38,821] jepsen worker 60 - jepsen.util 1680 :info :write [20 33] :RequestCanceledException INFO [2019-12-17 12:28:38,821] jepsen worker 75 - jepsen.util 1695 :info :write [25 38] :RequestCanceledException INFO [2019-12-17 12:28:38,824] jepsen worker 4 - jepsen.util 1084 :ok :read [1 4] INFO [2019-12-17 12:28:38,824] jepsen worker 4 - jepsen.util 1084 :invoke :write [1 22] INFO [2019-12-17 12:28:38,825] jepsen worker 60 - jepsen.util 1770 :invoke :read [20 nil] INFO [2019-12-17 12:28:38,825] jepsen worker 75 - jepsen.util 1785 :invoke :cas [25 [28 44]] INFO [2019-12-17 12:28:38,855] jepsen worker 28 - jepsen.util 928 :ok :read [9 8] INFO [2019-12-17 12:28:38,855] jepsen worker 28 - jepsen.util 928 :invoke :read [9 nil] INFO [2019-12-17 12:28:38,864] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":7163,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:28:38,865] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":7163,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:28:38,865] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":7166,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:28:38,865] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":7166,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:28:38,865] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":7167,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:28:38,865] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":7167,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:28:38,878] jepsen worker 42 - jepsen.util 1842 :ok :read [14 27] INFO [2019-12-17 12:28:38,878] jepsen worker 42 - jepsen.util 1842 :invoke :cas [14 [22 0]] INFO [2019-12-17 12:28:38,906] jepsen worker 58 - jepsen.util 1498 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:28:38,906] jepsen worker 58 - jepsen.util 1498 :invoke :read [19 nil] INFO [2019-12-17 12:28:38,927] jepsen worker 29 - jepsen.util 1199 :ok :write [9 24] INFO [2019-12-17 12:28:38,927] jepsen worker 29 - jepsen.util 1199 :invoke :read [9 nil] INFO [2019-12-17 12:28:38,940] jepsen worker 36 - jepsen.util 1296 :ok :write [12 3] INFO [2019-12-17 12:28:38,941] jepsen worker 36 - jepsen.util 1296 :invoke :read [12 nil] INFO [2019-12-17 12:28:38,983] jepsen worker 28 - jepsen.util 928 :ok :read [9 24] INFO [2019-12-17 12:28:38,983] jepsen worker 28 - jepsen.util 928 :invoke :cas [9 [7 49]] INFO [2019-12-17 12:28:38,995] jepsen worker 79 - jepsen.util 1249 :ok :write [26 0] INFO [2019-12-17 12:28:38,996] jepsen worker 79 - jepsen.util 1249 :invoke :cas [26 [45 0]] INFO [2019-12-17 12:28:39,008] jepsen worker 30 - jepsen.util 1470 :ok :read [10 8] INFO [2019-12-17 12:28:39,009] jepsen worker 30 - jepsen.util 1470 :invoke :write [10 30] INFO [2019-12-17 12:28:39,020] jepsen worker 71 - jepsen.util 971 :info :write [23 35] :RequestCanceledException INFO [2019-12-17 12:28:39,020] jepsen worker 42 - jepsen.util 1842 :fail :cas [14 [22 0]] :ValueNotSwapFrom INFO [2019-12-17 12:28:39,021] jepsen worker 42 - jepsen.util 1842 :invoke :read [14 nil] INFO [2019-12-17 12:28:39,022] jepsen worker 71 - jepsen.util 1061 :invoke :write [23 8] INFO [2019-12-17 12:28:39,035] jepsen worker 31 - jepsen.util 1291 :info :write [10 44] :RequestCanceledException INFO [2019-12-17 12:28:39,035] jepsen worker 19 - jepsen.util 1369 :info :write [6 48] :RequestCanceledException INFO [2019-12-17 12:28:39,038] jepsen worker 19 - jepsen.util 1459 :invoke :write [6 45] INFO [2019-12-17 12:28:39,038] jepsen worker 31 - jepsen.util 1381 :invoke :write [10 9] INFO [2019-12-17 12:28:39,055] jepsen worker 28 - jepsen.util 928 :fail :cas [9 [7 49]] :ValueNotSwapFrom INFO [2019-12-17 12:28:39,055] jepsen worker 28 - jepsen.util 928 :invoke :read [9 nil] INFO [2019-12-17 12:28:39,056] jepsen worker 72 - jepsen.util 1242 :ok :write [24 30] INFO [2019-12-17 12:28:39,056] jepsen worker 72 - jepsen.util 1242 :invoke :read [24 nil] INFO [2019-12-17 12:28:39,060] jepsen worker 65 - jepsen.util 1055 :ok :write [21 31] INFO [2019-12-17 12:28:39,060] jepsen worker 65 - jepsen.util 1055 :invoke :read [21 nil] INFO [2019-12-17 12:28:39,082] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":7173,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:28:39,082] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":7173,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:28:39,082] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":7176,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:28:39,083] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":7176,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:28:39,086] jepsen worker 53 - jepsen.util 1223 :ok :write [17 44] INFO [2019-12-17 12:28:39,086] jepsen worker 53 - jepsen.util 1223 :invoke :write [17 35] INFO [2019-12-17 12:28:39,088] jepsen worker 77 - jepsen.util 1517 :info :write [25 47] :RequestCanceledException INFO [2019-12-17 12:28:39,089] jepsen worker 79 - jepsen.util 1249 :fail :cas [26 [45 0]] :ValueNotSwapFrom INFO [2019-12-17 12:28:39,089] jepsen worker 79 - jepsen.util 1249 :invoke :cas [26 [31 37]] INFO [2019-12-17 12:28:39,089] jepsen worker 77 - jepsen.util 1607 :invoke :cas [25 [31 6]] INFO [2019-12-17 12:28:39,105] jepsen worker 65 - jepsen.util 1055 :ok :read [21 31] INFO [2019-12-17 12:28:39,105] jepsen worker 65 - jepsen.util 1055 :invoke :write [21 8] INFO [2019-12-17 12:28:39,111] jepsen worker 60 - jepsen.util 1770 :ok :read [20 7] INFO [2019-12-17 12:28:39,111] jepsen worker 60 - jepsen.util 1770 :invoke :cas [20 [33 9]] INFO [2019-12-17 12:28:39,114] jepsen worker 22 - jepsen.util 1642 :ok :read [7 20] INFO [2019-12-17 12:28:39,114] jepsen worker 22 - jepsen.util 1642 :invoke :read [7 nil] INFO [2019-12-17 12:28:39,118] jepsen worker 5 - jepsen.util 905 :ok :read [1 4] INFO [2019-12-17 12:28:39,118] jepsen worker 5 - jepsen.util 905 :invoke :write [1 24] INFO [2019-12-17 12:28:39,122] jepsen worker 46 - jepsen.util 1576 :info :write [15 2] :RequestCanceledException INFO [2019-12-17 12:28:39,123] jepsen worker 46 - jepsen.util 1666 :invoke :write [15 2] INFO [2019-12-17 12:28:39,170] jepsen worker 12 - jepsen.util 1362 :ok :write [4 26] INFO [2019-12-17 12:28:39,170] jepsen worker 65 - jepsen.util 1055 :ok :write [21 8] INFO [2019-12-17 12:28:39,173] jepsen worker 22 - jepsen.util 1642 :ok :read [7 20] INFO [2019-12-17 12:28:39,174] jepsen worker 22 - jepsen.util 1642 :invoke :cas [7 [41 27]] INFO [2019-12-17 12:28:39,174] jepsen worker 12 - jepsen.util 1362 :invoke :write [4 34] INFO [2019-12-17 12:28:39,174] jepsen worker 65 - jepsen.util 1055 :invoke :write [21 31] INFO [2019-12-17 12:28:39,191] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":7199,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:28:39,196] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":7199,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:28:39,247] jepsen worker 14 - jepsen.util 1184 :ok :read [4 16] INFO [2019-12-17 12:28:39,247] jepsen worker 14 - jepsen.util 1184 :invoke :cas [4 [17 35]] INFO [2019-12-17 12:28:39,259] jepsen worker 25 - jepsen.util 1825 :fail :cas [8 [32 39]] :ValueNotSwapFrom INFO [2019-12-17 12:28:39,260] jepsen worker 25 - jepsen.util 1825 :invoke :cas [8 [41 20]] INFO [2019-12-17 12:28:39,261] jepsen worker 79 - jepsen.util 1249 :fail :cas [26 [31 37]] :ValueNotSwapFrom INFO [2019-12-17 12:28:39,261] jepsen worker 79 - jepsen.util 1249 :invoke :read [26 nil] INFO [2019-12-17 12:28:39,264] jepsen worker 65 - jepsen.util 1055 :ok :write [21 31] INFO [2019-12-17 12:28:39,265] jepsen worker 65 - jepsen.util 1055 :invoke :cas [21 [9 48]] INFO [2019-12-17 12:28:39,267] jepsen worker 22 - jepsen.util 1642 :fail :cas [7 [41 27]] :ValueNotSwapFrom INFO [2019-12-17 12:28:39,267] jepsen worker 22 - jepsen.util 1642 :invoke :read [7 nil] INFO [2019-12-17 12:28:39,268] jepsen worker 54 - jepsen.util 1494 :ok :read [18 17] INFO [2019-12-17 12:28:39,268] jepsen worker 54 - jepsen.util 1494 :invoke :write [18 11] INFO [2019-12-17 12:28:39,280] jepsen worker 60 - jepsen.util 1770 :fail :cas [20 [33 9]] :ValueNotSwapFrom INFO [2019-12-17 12:28:39,281] jepsen worker 60 - jepsen.util 1770 :invoke :write [20 7] INFO [2019-12-17 12:28:39,284] jepsen worker 45 - jepsen.util 1305 :info :write [15 6] :RequestCanceledException INFO [2019-12-17 12:28:39,286] jepsen worker 45 - jepsen.util 1395 :invoke :cas [15 [27 5]] INFO [2019-12-17 12:28:39,297] jepsen worker 65 - jepsen.util 1055 :fail :cas [21 [9 48]] :ValueNotSwapFrom INFO [2019-12-17 12:28:39,297] jepsen worker 65 - jepsen.util 1055 :invoke :read [21 nil] INFO [2019-12-17 12:28:39,302] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":7207,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:28:39,302] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":7207,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:28:39,302] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":7210,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:28:39,302] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":7210,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:28:39,311] jepsen worker 13 - jepsen.util 1273 :ok :read [4 26] INFO [2019-12-17 12:28:39,312] jepsen worker 13 - jepsen.util 1273 :invoke :cas [4 [2 31]] INFO [2019-12-17 12:28:39,322] jepsen worker 27 - jepsen.util 1107 :info :write [9 1] :RequestCanceledException INFO [2019-12-17 12:28:39,322] jepsen worker 32 - jepsen.util 1292 :info :write [10 47] :RequestCanceledException INFO [2019-12-17 12:28:39,324] jepsen worker 27 - jepsen.util 1197 :invoke :read [9 nil] INFO [2019-12-17 12:28:39,326] jepsen worker 32 - jepsen.util 1382 :invoke :write [10 27] INFO [2019-12-17 12:28:39,326] jepsen worker 42 - jepsen.util 1842 :ok :read [14 27] INFO [2019-12-17 12:28:39,326] jepsen worker 42 - jepsen.util 1842 :invoke :read [14 nil] INFO [2019-12-17 12:28:39,349] jepsen worker 80 - jepsen.util 1430 :fail :cas [26 [46 29]] :ValueNotSwapFrom INFO [2019-12-17 12:28:39,349] jepsen worker 80 - jepsen.util 1430 :invoke :cas [26 [8 12]] INFO [2019-12-17 12:28:39,364] jepsen worker 63 - jepsen.util 1233 :ok :write [21 3] INFO [2019-12-17 12:28:39,365] jepsen worker 63 - jepsen.util 1233 :invoke :write [21 17] INFO [2019-12-17 12:28:39,365] jepsen worker 79 - jepsen.util 1249 :ok :read [26 0] INFO [2019-12-17 12:28:39,365] jepsen worker 79 - jepsen.util 1249 :invoke :read [26 nil] INFO [2019-12-17 12:28:39,378] jepsen worker 77 - jepsen.util 1607 :fail :cas [25 [31 6]] :ValueNotSwapFrom INFO [2019-12-17 12:28:39,379] jepsen worker 77 - jepsen.util 1607 :invoke :read [25 nil] INFO [2019-12-17 12:28:39,381] jepsen worker 65 - jepsen.util 1055 :ok :read [21 31] INFO [2019-12-17 12:28:39,382] jepsen worker 65 - jepsen.util 1055 :invoke :read [21 nil] INFO [2019-12-17 12:28:39,386] jepsen worker 36 - jepsen.util 1296 :ok :read [12 3] INFO [2019-12-17 12:28:39,387] jepsen worker 36 - jepsen.util 1296 :invoke :cas [12 [17 19]] INFO [2019-12-17 12:28:39,390] jepsen worker 70 - jepsen.util 1510 :ok :write [23 36] INFO [2019-12-17 12:28:39,390] jepsen worker 70 - jepsen.util 1510 :invoke :write [23 44] INFO [2019-12-17 12:28:39,401] jepsen worker 78 - jepsen.util 1338 :ok :write [26 27] INFO [2019-12-17 12:28:39,402] jepsen worker 78 - jepsen.util 1338 :invoke :cas [26 [14 6]] INFO [2019-12-17 12:28:39,409] jepsen worker 5 - jepsen.util 905 :ok :write [1 24] INFO [2019-12-17 12:28:39,409] jepsen worker 5 - jepsen.util 905 :invoke :read [1 nil] INFO [2019-12-17 12:28:39,414] jepsen worker 25 - jepsen.util 1825 :fail :cas [8 [41 20]] :ValueNotSwapFrom INFO [2019-12-17 12:28:39,414] jepsen worker 25 - jepsen.util 1825 :invoke :cas [8 [40 12]] INFO [2019-12-17 12:28:39,419] jepsen worker 6 - jepsen.util 1626 :info :write [2 36] :RequestCanceledException INFO [2019-12-17 12:28:39,421] jepsen worker 6 - jepsen.util 1716 :invoke :read [2 nil] INFO [2019-12-17 12:28:39,424] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:28:39,424] jepsen nemesis - couchbase.util Rebalance check count: 47 INFO [2019-12-17 12:28:39,424] jepsen nemesis - couchbase.util Rebalance stuck count: 28 INFO [2019-12-17 12:28:39,442] jepsen worker 65 - jepsen.util 1055 :ok :read [21 3] INFO [2019-12-17 12:28:39,442] jepsen worker 65 - jepsen.util 1055 :invoke :cas [21 [8 37]] INFO [2019-12-17 12:28:39,449] jepsen worker 42 - jepsen.util 1842 :ok :read [14 27] INFO [2019-12-17 12:28:39,449] jepsen worker 42 - jepsen.util 1842 :invoke :read [14 nil] INFO [2019-12-17 12:28:39,455] jepsen worker 77 - jepsen.util 1607 :ok :read [25 2] INFO [2019-12-17 12:28:39,455] jepsen worker 77 - jepsen.util 1607 :invoke :cas [25 [17 37]] INFO [2019-12-17 12:28:39,463] jepsen worker 71 - jepsen.util 1061 :ok :write [23 8] INFO [2019-12-17 12:28:39,464] jepsen worker 71 - jepsen.util 1061 :invoke :read [23 nil] INFO [2019-12-17 12:28:39,473] jepsen worker 11 - jepsen.util 1721 :ok :write [3 2] INFO [2019-12-17 12:28:39,474] jepsen worker 11 - jepsen.util 1721 :invoke :write [3 3] INFO [2019-12-17 12:28:39,486] jepsen worker 65 - jepsen.util 1055 :fail :cas [21 [8 37]] :ValueNotSwapFrom INFO [2019-12-17 12:28:39,486] jepsen worker 65 - jepsen.util 1055 :invoke :read [21 nil] INFO [2019-12-17 12:28:39,491] jepsen worker 63 - jepsen.util 1233 :ok :write [21 17] INFO [2019-12-17 12:28:39,491] jepsen worker 63 - jepsen.util 1233 :invoke :read [21 nil] INFO [2019-12-17 12:28:39,516] jepsen worker 79 - jepsen.util 1249 :ok :read [26 27] INFO [2019-12-17 12:28:39,516] jepsen worker 79 - jepsen.util 1249 :invoke :read [26 nil] INFO [2019-12-17 12:28:39,517] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":7235,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:28:39,517] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":7235,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:28:39,531] jepsen worker 65 - jepsen.util 1055 :ok :read [21 3] INFO [2019-12-17 12:28:39,531] jepsen worker 65 - jepsen.util 1055 :invoke :write [21 17] INFO [2019-12-17 12:28:39,531] jepsen worker 51 - jepsen.util 1311 :ok :read [17 44] INFO [2019-12-17 12:28:39,531] jepsen worker 51 - jepsen.util 1311 :invoke :cas [17 [38 7]] INFO [2019-12-17 12:28:39,536] jepsen worker 26 - jepsen.util 1556 :info :write [8 11] :RequestCanceledException INFO [2019-12-17 12:28:39,537] jepsen worker 26 - jepsen.util 1646 :invoke :write [8 13] INFO [2019-12-17 12:28:39,560] jepsen worker 63 - jepsen.util 1233 :ok :read [21 17] INFO [2019-12-17 12:28:39,561] jepsen worker 63 - jepsen.util 1233 :invoke :read [21 nil] INFO [2019-12-17 12:28:39,570] jepsen worker 80 - jepsen.util 1430 :fail :cas [26 [8 12]] :ValueNotSwapFrom INFO [2019-12-17 12:28:39,570] jepsen worker 80 - jepsen.util 1430 :invoke :read [26 nil] INFO [2019-12-17 12:28:39,576] jepsen worker 88 - jepsen.util 1168 :ok :write [29 6] INFO [2019-12-17 12:28:39,576] jepsen worker 88 - jepsen.util 1168 :invoke :cas [29 [21 34]] INFO [2019-12-17 12:28:39,586] jepsen worker 42 - jepsen.util 1842 :ok :read [14 27] INFO [2019-12-17 12:28:39,586] jepsen worker 42 - jepsen.util 1842 :invoke :read [14 nil] INFO [2019-12-17 12:28:39,588] jepsen worker 74 - jepsen.util 1244 :fail :read [24 nil] :RequestTimeoutException INFO [2019-12-17 12:28:39,588] jepsen worker 74 - jepsen.util 1244 :invoke :write [24 48] INFO [2019-12-17 12:28:39,590] jepsen worker 25 - jepsen.util 1825 :fail :cas [8 [40 12]] :ValueNotSwapFrom INFO [2019-12-17 12:28:39,590] jepsen worker 25 - jepsen.util 1825 :invoke :write [8 10] INFO [2019-12-17 12:28:39,592] jepsen worker 71 - jepsen.util 1061 :ok :read [23 8] INFO [2019-12-17 12:28:39,593] jepsen worker 71 - jepsen.util 1061 :invoke :write [23 28] INFO [2019-12-17 12:28:39,596] jepsen worker 22 - jepsen.util 1642 :ok :read [7 20] INFO [2019-12-17 12:28:39,596] jepsen worker 22 - jepsen.util 1642 :invoke :cas [7 [47 38]] INFO [2019-12-17 12:28:39,606] jepsen worker 64 - jepsen.util 874 :info :write [21 13] :RequestCanceledException INFO [2019-12-17 12:28:39,608] jepsen worker 64 - jepsen.util 964 :invoke :read [21 nil] INFO [2019-12-17 12:28:39,612] jepsen worker 77 - jepsen.util 1607 :fail :cas [25 [17 37]] :ValueNotSwapFrom INFO [2019-12-17 12:28:39,613] jepsen worker 77 - jepsen.util 1607 :invoke :cas [25 [28 39]] INFO [2019-12-17 12:28:39,622] jepsen worker 21 - jepsen.util 1551 :info :write [7 13] :RequestCanceledException INFO [2019-12-17 12:28:39,623] jepsen worker 21 - jepsen.util 1641 :invoke :write [7 12] INFO [2019-12-17 12:28:39,624] jepsen worker 63 - jepsen.util 1233 :ok :read [21 17] INFO [2019-12-17 12:28:39,624] jepsen worker 63 - jepsen.util 1233 :invoke :cas [21 [29 29]] INFO [2019-12-17 12:28:39,635] jepsen worker 22 - jepsen.util 1642 :fail :cas [7 [47 38]] :ValueNotSwapFrom INFO [2019-12-17 12:28:39,635] jepsen worker 22 - jepsen.util 1642 :invoke :write [7 42] INFO [2019-12-17 12:28:39,649] jepsen worker 34 - jepsen.util 1744 :info :cas [11 [27 9]] :RequestCanceledException INFO [2019-12-17 12:28:39,650] jepsen worker 34 - jepsen.util 1834 :invoke :write [11 1] INFO [2019-12-17 12:28:39,658] jepsen worker 79 - jepsen.util 1249 :ok :read [26 27] INFO [2019-12-17 12:28:39,658] jepsen worker 79 - jepsen.util 1249 :invoke :write [26 10] INFO [2019-12-17 12:28:39,666] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":7265,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:28:39,666] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":7265,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:28:39,666] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":7268,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:28:39,666] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":7268,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:28:39,667] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":7269,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:28:39,667] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":7269,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:28:39,675] jepsen worker 19 - jepsen.util 1459 :ok :write [6 45] INFO [2019-12-17 12:28:39,675] jepsen worker 19 - jepsen.util 1459 :invoke :cas [6 [18 39]] INFO [2019-12-17 12:28:39,678] jepsen worker 77 - jepsen.util 1607 :fail :cas [25 [28 39]] :ValueNotSwapFrom INFO [2019-12-17 12:28:39,678] jepsen worker 77 - jepsen.util 1607 :invoke :write [25 24] INFO [2019-12-17 12:28:39,687] jepsen worker 63 - jepsen.util 1233 :fail :cas [21 [29 29]] :ValueNotSwapFrom INFO [2019-12-17 12:28:39,688] jepsen worker 63 - jepsen.util 1233 :invoke :write [21 22] INFO [2019-12-17 12:28:39,689] jepsen worker 44 - jepsen.util 1484 :info :cas [14 [30 25]] :RequestCanceledException INFO [2019-12-17 12:28:39,690] jepsen worker 44 - jepsen.util 1574 :invoke :read [14 nil] INFO [2019-12-17 12:28:39,699] jepsen worker 65 - jepsen.util 1055 :ok :write [21 17] INFO [2019-12-17 12:28:39,700] jepsen worker 65 - jepsen.util 1055 :invoke :write [21 31] INFO [2019-12-17 12:28:39,721] jepsen worker 8 - jepsen.util 1538 :info :write [2 10] :RequestCanceledException INFO [2019-12-17 12:28:39,723] jepsen worker 8 - jepsen.util 1628 :invoke :cas [2 [37 43]] INFO [2019-12-17 12:28:39,742] jepsen worker 42 - jepsen.util 1842 :ok :read [14 27] INFO [2019-12-17 12:28:39,743] jepsen worker 42 - jepsen.util 1842 :invoke :cas [14 [21 16]] INFO [2019-12-17 12:28:39,753] jepsen worker 19 - jepsen.util 1459 :fail :cas [6 [18 39]] :ValueNotSwapFrom INFO [2019-12-17 12:28:39,753] jepsen worker 19 - jepsen.util 1459 :invoke :write [6 2] INFO [2019-12-17 12:28:39,771] jepsen worker 69 - jepsen.util 1059 :ok :read [23 8] INFO [2019-12-17 12:28:39,771] jepsen worker 69 - jepsen.util 1059 :invoke :read [23 nil] INFO [2019-12-17 12:28:39,783] jepsen worker 89 - jepsen.util 1439 :ok :write [29 32] INFO [2019-12-17 12:28:39,783] jepsen worker 56 - jepsen.util 1316 :ok :write [18 26] INFO [2019-12-17 12:28:39,783] jepsen worker 89 - jepsen.util 1439 :invoke :read [29 nil] INFO [2019-12-17 12:28:39,783] jepsen worker 56 - jepsen.util 1316 :invoke :read [18 nil] INFO [2019-12-17 12:28:39,806] jepsen worker 43 - jepsen.util 1753 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:28:39,806] jepsen worker 43 - jepsen.util 1753 :invoke :write [14 40] INFO [2019-12-17 12:28:39,813] jepsen worker 4 - jepsen.util 1084 :ok :write [1 22] INFO [2019-12-17 12:28:39,813] jepsen worker 4 - jepsen.util 1084 :invoke :read [1 nil] INFO [2019-12-17 12:28:39,830] jepsen worker 63 - jepsen.util 1233 :ok :write [21 22] INFO [2019-12-17 12:28:39,830] jepsen worker 63 - jepsen.util 1233 :invoke :read [21 nil] INFO [2019-12-17 12:28:39,835] jepsen worker 80 - jepsen.util 1430 :ok :read [26 27] INFO [2019-12-17 12:28:39,835] jepsen worker 80 - jepsen.util 1430 :invoke :read [26 nil] INFO [2019-12-17 12:28:39,875] jepsen worker 63 - jepsen.util 1233 :ok :read [21 22] INFO [2019-12-17 12:28:39,875] jepsen worker 63 - jepsen.util 1233 :invoke :cas [21 [49 40]] INFO [2019-12-17 12:28:39,878] jepsen worker 42 - jepsen.util 1842 :fail :cas [14 [21 16]] :ValueNotSwapFrom INFO [2019-12-17 12:28:39,879] jepsen worker 42 - jepsen.util 1842 :invoke :cas [14 [15 20]] INFO [2019-12-17 12:28:39,882] jepsen worker 13 - jepsen.util 1273 :fail :cas [4 [2 31]] :ValueNotSwapFrom INFO [2019-12-17 12:28:39,882] jepsen worker 13 - jepsen.util 1273 :invoke :read [4 nil] INFO [2019-12-17 12:28:39,889] jepsen worker 18 - jepsen.util 1458 :fail :read [6 nil] :RequestTimeoutException INFO [2019-12-17 12:28:39,889] jepsen worker 18 - jepsen.util 1458 :invoke :cas [6 [6 24]] INFO [2019-12-17 12:28:39,893] jepsen worker 38 - jepsen.util 1208 :ok :read [12 3] INFO [2019-12-17 12:28:39,893] jepsen worker 38 - jepsen.util 1208 :invoke :write [12 0] INFO [2019-12-17 12:28:39,907] jepsen worker 78 - jepsen.util 1338 :fail :cas [26 [14 6]] :ValueNotSwapFrom INFO [2019-12-17 12:28:39,908] jepsen worker 78 - jepsen.util 1338 :invoke :cas [26 [29 40]] INFO [2019-12-17 12:28:39,926] jepsen worker 63 - jepsen.util 1233 :fail :cas [21 [49 40]] :ValueNotSwapFrom INFO [2019-12-17 12:28:39,927] jepsen worker 30 - jepsen.util 1470 :ok :write [10 30] INFO [2019-12-17 12:28:39,927] jepsen worker 63 - jepsen.util 1233 :invoke :cas [21 [48 32]] INFO [2019-12-17 12:28:39,927] jepsen worker 30 - jepsen.util 1470 :invoke :cas [10 [1 3]] INFO [2019-12-17 12:28:39,927] jepsen worker 46 - jepsen.util 1666 :ok :write [15 2] INFO [2019-12-17 12:28:39,928] jepsen worker 46 - jepsen.util 1666 :invoke :write [15 24] INFO [2019-12-17 12:28:39,939] jepsen worker 65 - jepsen.util 1055 :ok :write [21 31] INFO [2019-12-17 12:28:39,939] jepsen worker 65 - jepsen.util 1055 :invoke :read [21 nil] INFO [2019-12-17 12:28:39,951] jepsen worker 51 - jepsen.util 1311 :fail :cas [17 [38 7]] :ValueNotSwapFrom INFO [2019-12-17 12:28:39,951] jepsen worker 51 - jepsen.util 1311 :invoke :cas [17 [27 44]] INFO [2019-12-17 12:28:39,959] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":7289,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:28:39,960] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":7289,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:28:39,960] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":7293,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:28:39,960] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":7293,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:28:39,989] jepsen worker 42 - jepsen.util 1842 :fail :cas [14 [15 20]] :ValueNotSwapFrom INFO [2019-12-17 12:28:39,989] jepsen worker 42 - jepsen.util 1842 :invoke :write [14 1] INFO [2019-12-17 12:28:39,999] jepsen worker 56 - jepsen.util 1316 :ok :read [18 26] INFO [2019-12-17 12:28:39,999] jepsen worker 56 - jepsen.util 1316 :invoke :cas [18 [40 12]] INFO [2019-12-17 12:28:40,006] jepsen worker 7 - jepsen.util 1807 :info :cas [2 [5 11]] :RequestCanceledException INFO [2019-12-17 12:28:40,008] jepsen worker 89 - jepsen.util 1439 :ok :read [29 32] INFO [2019-12-17 12:28:40,008] jepsen worker 89 - jepsen.util 1439 :invoke :read [29 nil] INFO [2019-12-17 12:28:40,008] jepsen worker 7 - jepsen.util 1897 :invoke :write [2 41] INFO [2019-12-17 12:28:40,023] jepsen worker 63 - jepsen.util 1233 :fail :cas [21 [48 32]] :ValueNotSwapFrom INFO [2019-12-17 12:28:40,023] jepsen worker 63 - jepsen.util 1233 :invoke :read [21 nil] INFO [2019-12-17 12:28:40,030] jepsen worker 27 - jepsen.util 1197 :ok :read [9 24] INFO [2019-12-17 12:28:40,030] jepsen worker 27 - jepsen.util 1197 :invoke :cas [9 [26 28]] INFO [2019-12-17 12:28:40,037] jepsen worker 30 - jepsen.util 1470 :fail :cas [10 [1 3]] :ValueNotSwapFrom INFO [2019-12-17 12:28:40,037] jepsen worker 30 - jepsen.util 1470 :invoke :read [10 nil] INFO [2019-12-17 12:28:40,037] jepsen worker 65 - jepsen.util 1055 :ok :read [21 31] INFO [2019-12-17 12:28:40,037] jepsen worker 65 - jepsen.util 1055 :invoke :read [21 nil] INFO [2019-12-17 12:28:40,042] jepsen worker 36 - jepsen.util 1296 :fail :cas [12 [17 19]] :ValueNotSwapFrom INFO [2019-12-17 12:28:40,042] jepsen worker 36 - jepsen.util 1296 :invoke :read [12 nil] INFO [2019-12-17 12:28:40,052] jepsen worker 5 - jepsen.util 905 :ok :read [1 22] INFO [2019-12-17 12:28:40,052] jepsen worker 5 - jepsen.util 905 :invoke :read [1 nil] INFO [2019-12-17 12:28:40,069] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":7307,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:28:40,069] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":7307,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:28:40,072] jepsen worker 13 - jepsen.util 1273 :ok :read [4 26] INFO [2019-12-17 12:28:40,072] jepsen worker 13 - jepsen.util 1273 :invoke :read [4 nil] INFO [2019-12-17 12:28:40,080] jepsen worker 63 - jepsen.util 1233 :ok :read [21 31] INFO [2019-12-17 12:28:40,080] jepsen worker 63 - jepsen.util 1233 :invoke :read [21 nil] INFO [2019-12-17 12:28:40,087] jepsen worker 78 - jepsen.util 1338 :fail :cas [26 [29 40]] :ValueNotSwapFrom INFO [2019-12-17 12:28:40,088] jepsen worker 78 - jepsen.util 1338 :invoke :cas [26 [17 4]] INFO [2019-12-17 12:28:40,093] jepsen worker 51 - jepsen.util 1311 :fail :cas [17 [27 44]] :ValueNotSwapFrom INFO [2019-12-17 12:28:40,093] jepsen worker 51 - jepsen.util 1311 :invoke :read [17 nil] INFO [2019-12-17 12:28:40,099] jepsen worker 56 - jepsen.util 1316 :fail :cas [18 [40 12]] :ValueNotSwapFrom INFO [2019-12-17 12:28:40,099] jepsen worker 56 - jepsen.util 1316 :invoke :write [18 39] INFO [2019-12-17 12:28:40,107] jepsen worker 27 - jepsen.util 1197 :fail :cas [9 [26 28]] :ValueNotSwapFrom INFO [2019-12-17 12:28:40,107] jepsen worker 27 - jepsen.util 1197 :invoke :read [9 nil] INFO [2019-12-17 12:28:40,112] jepsen worker 65 - jepsen.util 1055 :ok :read [21 31] INFO [2019-12-17 12:28:40,112] jepsen worker 65 - jepsen.util 1055 :invoke :write [21 29] INFO [2019-12-17 12:28:40,121] jepsen worker 62 - jepsen.util 1592 :info :write [20 26] :RequestCanceledException INFO [2019-12-17 12:28:40,123] jepsen worker 62 - jepsen.util 1682 :invoke :cas [20 [23 39]] INFO [2019-12-17 12:28:40,128] jepsen worker 89 - jepsen.util 1439 :ok :read [29 32] INFO [2019-12-17 12:28:40,128] jepsen worker 89 - jepsen.util 1439 :invoke :read [29 nil] INFO [2019-12-17 12:28:40,139] jepsen worker 30 - jepsen.util 1470 :ok :read [10 30] INFO [2019-12-17 12:28:40,139] jepsen worker 30 - jepsen.util 1470 :invoke :read [10 nil] INFO [2019-12-17 12:28:40,153] jepsen worker 27 - jepsen.util 1197 :ok :read [9 24] INFO [2019-12-17 12:28:40,153] jepsen worker 27 - jepsen.util 1197 :invoke :write [9 37] INFO [2019-12-17 12:28:40,159] jepsen worker 63 - jepsen.util 1233 :ok :read [21 31] INFO [2019-12-17 12:28:40,159] jepsen worker 63 - jepsen.util 1233 :invoke :cas [21 [4 3]] INFO [2019-12-17 12:28:40,168] jepsen worker 30 - jepsen.util 1470 :ok :read [10 30] INFO [2019-12-17 12:28:40,168] jepsen worker 30 - jepsen.util 1470 :invoke :read [10 nil] INFO [2019-12-17 12:28:40,197] jepsen worker 63 - jepsen.util 1233 :fail :cas [21 [4 3]] :ValueNotSwapFrom INFO [2019-12-17 12:28:40,198] jepsen worker 63 - jepsen.util 1233 :invoke :write [21 14] INFO [2019-12-17 12:28:40,199] jepsen worker 80 - jepsen.util 1430 :ok :read [26 27] INFO [2019-12-17 12:28:40,199] jepsen worker 80 - jepsen.util 1430 :invoke :cas [26 [30 48]] INFO [2019-12-17 12:28:40,199] jepsen worker 88 - jepsen.util 1168 :fail :cas [29 [21 34]] :ValueNotSwapFrom INFO [2019-12-17 12:28:40,199] jepsen worker 88 - jepsen.util 1168 :invoke :read [29 nil] INFO [2019-12-17 12:28:40,213] jepsen worker 27 - jepsen.util 1197 :ok :write [9 37] INFO [2019-12-17 12:28:40,213] jepsen worker 27 - jepsen.util 1197 :invoke :write [9 26] INFO [2019-12-17 12:28:40,239] jepsen worker 36 - jepsen.util 1296 :ok :read [12 3] INFO [2019-12-17 12:28:40,239] jepsen worker 36 - jepsen.util 1296 :invoke :read [12 nil] INFO [2019-12-17 12:28:40,240] jepsen worker 87 - jepsen.util 1257 :ok :read [29 32] INFO [2019-12-17 12:28:40,240] jepsen worker 87 - jepsen.util 1257 :invoke :read [29 nil] INFO [2019-12-17 12:28:40,248] jepsen worker 19 - jepsen.util 1459 :ok :write [6 2] INFO [2019-12-17 12:28:40,248] jepsen worker 19 - jepsen.util 1459 :invoke :write [6 28] INFO [2019-12-17 12:28:40,261] jepsen worker 55 - jepsen.util 1675 :fail :cas [18 [34 42]] :ValueNotSwapFrom INFO [2019-12-17 12:28:40,261] jepsen worker 55 - jepsen.util 1675 :invoke :write [18 12] INFO [2019-12-17 12:28:40,265] jepsen worker 65 - jepsen.util 1055 :ok :write [21 29] INFO [2019-12-17 12:28:40,266] jepsen worker 65 - jepsen.util 1055 :invoke :cas [21 [17 34]] INFO [2019-12-17 12:28:40,279] jepsen worker 18 - jepsen.util 1458 :fail :cas [6 [6 24]] :ValueNotSwapFrom INFO [2019-12-17 12:28:40,279] jepsen worker 18 - jepsen.util 1458 :invoke :cas [6 [22 35]] INFO [2019-12-17 12:28:40,286] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":7336,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:28:40,287] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":7336,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:28:40,287] jepsen worker 47 - jepsen.util 1667 :info :write [15 22] :RequestCanceledException INFO [2019-12-17 12:28:40,287] jepsen worker 78 - jepsen.util 1338 :fail :cas [26 [17 4]] :ValueNotSwapFrom INFO [2019-12-17 12:28:40,287] jepsen worker 78 - jepsen.util 1338 :invoke :write [26 11] INFO [2019-12-17 12:28:40,289] jepsen worker 47 - jepsen.util 1757 :invoke :cas [15 [13 0]] INFO [2019-12-17 12:28:40,289] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":7344,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:28:40,289] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":7344,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:28:40,289] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":7347,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:28:40,289] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":7347,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:28:40,289] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":7348,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:28:40,289] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":7348,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:28:40,295] jepsen worker 69 - jepsen.util 1059 :ok :read [23 8] INFO [2019-12-17 12:28:40,295] jepsen worker 69 - jepsen.util 1059 :invoke :cas [23 [46 42]] INFO [2019-12-17 12:28:40,316] jepsen worker 5 - jepsen.util 905 :ok :read [1 22] INFO [2019-12-17 12:28:40,317] jepsen worker 5 - jepsen.util 905 :invoke :write [1 27] INFO [2019-12-17 12:28:40,329] jepsen worker 27 - jepsen.util 1197 :ok :write [9 26] INFO [2019-12-17 12:28:40,329] jepsen worker 27 - jepsen.util 1197 :invoke :write [9 37] INFO [2019-12-17 12:28:40,335] jepsen worker 65 - jepsen.util 1055 :fail :cas [21 [17 34]] :ValueNotSwapFrom INFO [2019-12-17 12:28:40,336] jepsen worker 65 - jepsen.util 1055 :invoke :cas [21 [39 22]] INFO [2019-12-17 12:28:40,364] jepsen worker 13 - jepsen.util 1273 :ok :read [4 26] INFO [2019-12-17 12:28:40,364] jepsen worker 13 - jepsen.util 1273 :invoke :read [4 nil] INFO [2019-12-17 12:28:40,364] jepsen worker 63 - jepsen.util 1233 :ok :write [21 14] INFO [2019-12-17 12:28:40,364] jepsen worker 63 - jepsen.util 1233 :invoke :cas [21 [26 20]] INFO [2019-12-17 12:28:40,367] jepsen worker 51 - jepsen.util 1311 :ok :read [17 44] INFO [2019-12-17 12:28:40,368] jepsen worker 51 - jepsen.util 1311 :invoke :write [17 43] INFO [2019-12-17 12:28:40,389] jepsen worker 65 - jepsen.util 1055 :fail :cas [21 [39 22]] :ValueNotSwapFrom INFO [2019-12-17 12:28:40,390] jepsen worker 65 - jepsen.util 1055 :invoke :read [21 nil] INFO [2019-12-17 12:28:40,393] jepsen worker 47 - jepsen.util 1757 :fail :cas [15 [13 0]] :ValueNotSwapFrom INFO [2019-12-17 12:28:40,394] jepsen worker 47 - jepsen.util 1757 :invoke :write [15 14] INFO [2019-12-17 12:28:40,403] jepsen worker 27 - jepsen.util 1197 :ok :write [9 37] INFO [2019-12-17 12:28:40,403] jepsen worker 27 - jepsen.util 1197 :invoke :write [9 18] INFO [2019-12-17 12:28:40,404] jepsen worker 80 - jepsen.util 1430 :fail :cas [26 [30 48]] :ValueNotSwapFrom INFO [2019-12-17 12:28:40,404] jepsen worker 80 - jepsen.util 1430 :invoke :cas [26 [29 7]] INFO [2019-12-17 12:28:40,426] jepsen worker 63 - jepsen.util 1233 :fail :cas [21 [26 20]] :ValueNotSwapFrom INFO [2019-12-17 12:28:40,426] jepsen worker 63 - jepsen.util 1233 :invoke :read [21 nil] INFO [2019-12-17 12:28:40,435] jepsen worker 37 - jepsen.util 1117 :info :write [12 37] :RequestCanceledException INFO [2019-12-17 12:28:40,435] jepsen worker 33 - jepsen.util 1833 :info :write [11 12] :RequestCanceledException INFO [2019-12-17 12:28:40,436] jepsen worker 33 - jepsen.util 1923 :invoke :write [11 21] INFO [2019-12-17 12:28:40,436] jepsen worker 37 - jepsen.util 1207 :invoke :cas [12 [32 27]] INFO [2019-12-17 12:28:40,438] jepsen worker 65 - jepsen.util 1055 :ok :read [21 14] INFO [2019-12-17 12:28:40,439] jepsen worker 65 - jepsen.util 1055 :invoke :write [21 44] INFO [2019-12-17 12:28:40,445] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:28:40,445] jepsen nemesis - couchbase.util Rebalance check count: 48 INFO [2019-12-17 12:28:40,445] jepsen nemesis - couchbase.util Rebalance stuck count: 29 INFO [2019-12-17 12:28:40,452] jepsen worker 76 - jepsen.util 1606 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:28:40,452] jepsen worker 76 - jepsen.util 1606 :invoke :write [25 28] INFO [2019-12-17 12:28:40,480] jepsen worker 4 - jepsen.util 1084 :ok :read [1 22] INFO [2019-12-17 12:28:40,480] jepsen worker 4 - jepsen.util 1084 :invoke :write [1 21] INFO [2019-12-17 12:28:40,481] jepsen worker 27 - jepsen.util 1197 :ok :write [9 18] INFO [2019-12-17 12:28:40,481] jepsen worker 27 - jepsen.util 1197 :invoke :write [9 14] INFO [2019-12-17 12:28:40,484] jepsen worker 80 - jepsen.util 1430 :fail :cas [26 [29 7]] :ValueNotSwapFrom INFO [2019-12-17 12:28:40,484] jepsen worker 80 - jepsen.util 1430 :invoke :cas [26 [17 15]] INFO [2019-12-17 12:28:40,488] jepsen worker 9 - jepsen.util 1359 :info :cas [3 [0 3]] :RequestCanceledException INFO [2019-12-17 12:28:40,491] jepsen worker 9 - jepsen.util 1449 :invoke :cas [3 [17 28]] INFO [2019-12-17 12:28:40,502] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":7370,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:28:40,502] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":7370,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:28:40,502] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":7374,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:28:40,502] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":7374,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:28:40,506] jepsen worker 3 - jepsen.util 1083 :info :write [1 11] :RequestCanceledException INFO [2019-12-17 12:28:40,506] jepsen worker 0 - jepsen.util 1710 :info :write [0 43] :RequestCanceledException INFO [2019-12-17 12:28:40,506] jepsen worker 10 - jepsen.util 1270 :fail :read [3 nil] :RequestTimeoutException INFO [2019-12-17 12:28:40,506] jepsen worker 73 - jepsen.util 1243 :info :write [24 30] :RequestCanceledException INFO [2019-12-17 12:28:40,506] jepsen worker 10 - jepsen.util 1270 :invoke :read [3 nil] INFO [2019-12-17 12:28:40,507] jepsen worker 0 - jepsen.util 1800 :invoke :read [0 nil] INFO [2019-12-17 12:28:40,508] jepsen worker 3 - jepsen.util 1173 :invoke :write [1 5] INFO [2019-12-17 12:28:40,508] jepsen worker 73 - jepsen.util 1333 :invoke :write [24 5] INFO [2019-12-17 12:28:40,522] jepsen worker 85 - jepsen.util 1615 :info :write [28 43] :RequestCanceledException INFO [2019-12-17 12:28:40,525] jepsen worker 85 - jepsen.util 1705 :invoke :write [28 23] INFO [2019-12-17 12:28:40,536] jepsen worker 82 - jepsen.util 1972 :info :cas [27 [8 8]] :RequestCanceledException INFO [2019-12-17 12:28:40,539] jepsen worker 82 - jepsen.util 2062 :invoke :cas [27 [3 23]] INFO [2019-12-17 12:28:40,546] jepsen worker 9 - jepsen.util 1449 :fail :cas [3 [17 28]] :ValueNotSwapFrom INFO [2019-12-17 12:28:40,546] jepsen worker 9 - jepsen.util 1449 :invoke :cas [3 [20 0]] INFO [2019-12-17 12:28:40,551] jepsen worker 80 - jepsen.util 1430 :fail :cas [26 [17 15]] :ValueNotSwapFrom INFO [2019-12-17 12:28:40,552] jepsen worker 80 - jepsen.util 1430 :invoke :read [26 nil] INFO [2019-12-17 12:28:40,571] jepsen worker 10 - jepsen.util 1270 :ok :read [3 2] INFO [2019-12-17 12:28:40,571] jepsen worker 10 - jepsen.util 1270 :invoke :write [3 25] INFO [2019-12-17 12:28:40,589] jepsen worker 24 - jepsen.util 1554 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:28:40,589] jepsen worker 89 - jepsen.util 1439 :ok :read [29 32] INFO [2019-12-17 12:28:40,589] jepsen worker 24 - jepsen.util 1554 :invoke :read [8 nil] INFO [2019-12-17 12:28:40,589] jepsen worker 89 - jepsen.util 1439 :invoke :read [29 nil] INFO [2019-12-17 12:28:40,607] jepsen worker 9 - jepsen.util 1449 :fail :cas [3 [20 0]] :ValueNotSwapFrom INFO [2019-12-17 12:28:40,607] jepsen worker 9 - jepsen.util 1449 :invoke :write [3 16] INFO [2019-12-17 12:28:40,621] jepsen worker 86 - jepsen.util 1706 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:28:40,621] jepsen worker 48 - jepsen.util 1128 :info :cas [16 [29 39]] :RequestCanceledException INFO [2019-12-17 12:28:40,622] jepsen worker 86 - jepsen.util 1706 :invoke :cas [28 [34 6]] INFO [2019-12-17 12:28:40,624] jepsen worker 48 - jepsen.util 1218 :invoke :read [16 nil] INFO [2019-12-17 12:28:40,635] jepsen worker 52 - jepsen.util 1402 :info :cas [17 [44 29]] :RequestCanceledException INFO [2019-12-17 12:28:40,635] jepsen worker 16 - jepsen.util 1546 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:28:40,636] jepsen worker 16 - jepsen.util 1546 :invoke :cas [5 [20 46]] INFO [2019-12-17 12:28:40,639] jepsen worker 52 - jepsen.util 1492 :invoke :read [17 nil] INFO [2019-12-17 12:28:40,640] jepsen worker 27 - jepsen.util 1197 :ok :write [9 14] INFO [2019-12-17 12:28:40,641] jepsen worker 27 - jepsen.util 1197 :invoke :write [9 9] INFO [2019-12-17 12:28:40,651] jepsen worker 65 - jepsen.util 1055 :ok :write [21 44] INFO [2019-12-17 12:28:40,651] jepsen worker 65 - jepsen.util 1055 :invoke :read [21 nil] INFO [2019-12-17 12:28:40,653] jepsen worker 35 - jepsen.util 1295 :info :cas [11 [36 32]] :RequestCanceledException INFO [2019-12-17 12:28:40,653] jepsen worker 50 - jepsen.util 1580 :info :cas [16 [25 49]] :RequestCanceledException INFO [2019-12-17 12:28:40,654] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":7400,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:28:40,654] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":7400,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:28:40,655] jepsen worker 35 - jepsen.util 1385 :invoke :write [11 12] INFO [2019-12-17 12:28:40,655] jepsen worker 50 - jepsen.util 1670 :invoke :cas [16 [4 2]] INFO [2019-12-17 12:28:40,658] jepsen worker 80 - jepsen.util 1430 :ok :read [26 27] INFO [2019-12-17 12:28:40,658] jepsen worker 80 - jepsen.util 1430 :invoke :cas [26 [15 19]] INFO [2019-12-17 12:28:40,675] jepsen worker 65 - jepsen.util 1055 :ok :read [21 44] INFO [2019-12-17 12:28:40,675] jepsen worker 65 - jepsen.util 1055 :invoke :read [21 nil] INFO [2019-12-17 12:28:40,689] jepsen worker 57 - jepsen.util 1497 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:28:40,689] jepsen worker 1 - jepsen.util 1531 :info :cas [0 [16 39]] :RequestCanceledException INFO [2019-12-17 12:28:40,689] jepsen worker 41 - jepsen.util 1391 :info :write [13 14] :RequestCanceledException INFO [2019-12-17 12:28:40,689] jepsen worker 57 - jepsen.util 1497 :invoke :cas [19 [25 49]] INFO [2019-12-17 12:28:40,690] jepsen worker 41 - jepsen.util 1481 :invoke :cas [13 [32 12]] INFO [2019-12-17 12:28:40,691] jepsen worker 1 - jepsen.util 1621 :invoke :write [0 23] INFO [2019-12-17 12:28:40,693] jepsen worker 65 - jepsen.util 1055 :ok :read [21 44] INFO [2019-12-17 12:28:40,693] jepsen worker 65 - jepsen.util 1055 :invoke :read [21 nil] INFO [2019-12-17 12:28:40,706] jepsen worker 23 - jepsen.util 1463 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:28:40,706] jepsen worker 84 - jepsen.util 1524 :info :cas [28 [11 47]] :RequestCanceledException INFO [2019-12-17 12:28:40,706] jepsen worker 23 - jepsen.util 1463 :invoke :write [7 26] INFO [2019-12-17 12:28:40,708] jepsen worker 84 - jepsen.util 1614 :invoke :read [28 nil] INFO [2019-12-17 12:28:40,716] jepsen worker 65 - jepsen.util 1055 :ok :read [21 44] INFO [2019-12-17 12:28:40,716] jepsen worker 65 - jepsen.util 1055 :invoke :write [21 37] INFO [2019-12-17 12:28:40,720] jepsen worker 27 - jepsen.util 1197 :ok :write [9 9] INFO [2019-12-17 12:28:40,721] jepsen worker 66 - jepsen.util 1326 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:28:40,721] jepsen worker 17 - jepsen.util 1547 :info :cas [5 [27 13]] :RequestCanceledException INFO [2019-12-17 12:28:40,721] jepsen worker 27 - jepsen.util 1197 :invoke :cas [9 [23 17]] INFO [2019-12-17 12:28:40,721] jepsen worker 66 - jepsen.util 1326 :invoke :cas [22 [32 5]] INFO [2019-12-17 12:28:40,722] jepsen worker 67 - jepsen.util 1867 :info :write [22 4] :RequestCanceledException INFO [2019-12-17 12:28:40,722] jepsen worker 59 - jepsen.util 1409 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:28:40,722] jepsen worker 59 - jepsen.util 1409 :invoke :cas [19 [10 49]] INFO [2019-12-17 12:28:40,724] jepsen worker 17 - jepsen.util 1637 :invoke :cas [5 [45 10]] INFO [2019-12-17 12:28:40,726] jepsen worker 67 - jepsen.util 1957 :invoke :write [22 28] INFO [2019-12-17 12:28:40,734] jepsen worker 83 - jepsen.util 1883 :info :cas [27 [20 16]] :RequestCanceledException INFO [2019-12-17 12:28:40,740] jepsen worker 83 - jepsen.util 1973 :invoke :write [27 44] INFO [2019-12-17 12:28:40,751] jepsen worker 49 - jepsen.util 1309 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:28:40,752] jepsen worker 49 - jepsen.util 1309 :invoke :read [16 nil] INFO [2019-12-17 12:28:40,764] jepsen worker 88 - jepsen.util 1168 :ok :read [29 32] INFO [2019-12-17 12:28:40,765] jepsen worker 88 - jepsen.util 1168 :invoke :cas [29 [2 9]] INFO [2019-12-17 12:28:40,773] jepsen worker 27 - jepsen.util 1197 :fail :cas [9 [23 17]] :ValueNotSwapFrom INFO [2019-12-17 12:28:40,773] jepsen worker 27 - jepsen.util 1197 :invoke :cas [9 [1 23]] INFO [2019-12-17 12:28:40,786] jepsen worker 40 - jepsen.util 1750 :info :write [13 24] :RequestCanceledException INFO [2019-12-17 12:28:40,786] jepsen worker 39 - jepsen.util 1749 :info :write [13 40] :RequestCanceledException INFO [2019-12-17 12:28:40,789] jepsen worker 40 - jepsen.util 1840 :invoke :write [13 47] INFO [2019-12-17 12:28:40,790] jepsen worker 39 - jepsen.util 1839 :invoke :read [13 nil] INFO [2019-12-17 12:28:40,806] jepsen worker 68 - jepsen.util 1148 :info :write [22 13] :RequestCanceledException INFO [2019-12-17 12:28:40,806] jepsen worker 2 - jepsen.util 1262 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:28:40,806] jepsen worker 61 - jepsen.util 1681 :info :cas [20 [21 48]] :RequestCanceledException INFO [2019-12-17 12:28:40,806] jepsen worker 2 - jepsen.util 1262 :invoke :write [0 17] INFO [2019-12-17 12:28:40,806] jepsen worker 20 - jepsen.util 1280 :info :write [6 30] :RequestCanceledException INFO [2019-12-17 12:28:40,808] jepsen worker 61 - jepsen.util 1771 :invoke :read [20 nil] INFO [2019-12-17 12:28:40,810] jepsen worker 68 - jepsen.util 1238 :invoke :write [22 13] INFO [2019-12-17 12:28:40,810] jepsen worker 20 - jepsen.util 1370 :invoke :write [6 42] INFO [2019-12-17 12:28:40,821] jepsen worker 15 - jepsen.util 1275 :info :cas [5 [30 3]] :RequestCanceledException INFO [2019-12-17 12:28:40,829] jepsen worker 15 - jepsen.util 1365 :invoke :cas [5 [6 10]] INFO [2019-12-17 12:28:40,844] jepsen worker 62 - jepsen.util 1682 :fail :cas [20 [23 39]] :ValueNotSwapFrom INFO [2019-12-17 12:28:40,844] jepsen worker 62 - jepsen.util 1682 :invoke :read [20 nil] INFO [2019-12-17 12:28:40,844] jepsen worker 27 - jepsen.util 1197 :fail :cas [9 [1 23]] :ValueNotSwapFrom INFO [2019-12-17 12:28:40,844] jepsen worker 27 - jepsen.util 1197 :invoke :cas [9 [10 39]] INFO [2019-12-17 12:28:40,848] jepsen worker 36 - jepsen.util 1296 :ok :read [12 3] INFO [2019-12-17 12:28:40,848] jepsen worker 36 - jepsen.util 1296 :invoke :cas [12 [31 2]] INFO [2019-12-17 12:28:40,853] jepsen worker 75 - jepsen.util 1785 :info :cas [25 [28 44]] :RequestCanceledException INFO [2019-12-17 12:28:40,854] jepsen worker 65 - jepsen.util 1055 :ok :write [21 37] INFO [2019-12-17 12:28:40,854] jepsen worker 65 - jepsen.util 1055 :invoke :write [21 21] INFO [2019-12-17 12:28:40,855] jepsen worker 75 - jepsen.util 1875 :invoke :cas [25 [9 42]] INFO [2019-12-17 12:28:40,883] jepsen worker 80 - jepsen.util 1430 :fail :cas [26 [15 19]] :ValueNotSwapFrom INFO [2019-12-17 12:28:40,883] jepsen worker 80 - jepsen.util 1430 :invoke :cas [26 [7 25]] INFO [2019-12-17 12:28:40,889] jepsen worker 81 - jepsen.util 1521 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:28:40,889] jepsen worker 81 - jepsen.util 1521 :invoke :read [27 nil] INFO [2019-12-17 12:28:40,896] jepsen worker 27 - jepsen.util 1197 :fail :cas [9 [10 39]] :ValueNotSwapFrom INFO [2019-12-17 12:28:40,896] jepsen worker 27 - jepsen.util 1197 :invoke :read [9 nil] INFO [2019-12-17 12:28:40,906] jepsen worker 87 - jepsen.util 1257 :ok :read [29 32] INFO [2019-12-17 12:28:40,906] jepsen worker 87 - jepsen.util 1257 :invoke :read [29 nil] INFO [2019-12-17 12:28:40,924] jepsen worker 65 - jepsen.util 1055 :ok :write [21 21] INFO [2019-12-17 12:28:40,925] jepsen worker 65 - jepsen.util 1055 :invoke :cas [21 [30 3]] INFO [2019-12-17 12:28:40,931] jepsen worker 27 - jepsen.util 1197 :ok :read [9 9] INFO [2019-12-17 12:28:40,931] jepsen worker 27 - jepsen.util 1197 :invoke :read [9 nil] INFO [2019-12-17 12:28:40,943] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":7439,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:28:40,943] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":7439,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:28:40,943] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":7442,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:28:40,943] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":7442,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:28:40,943] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":7443,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:28:40,944] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":7443,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:28:40,948] jepsen worker 89 - jepsen.util 1439 :ok :read [29 32] INFO [2019-12-17 12:28:40,948] jepsen worker 89 - jepsen.util 1439 :invoke :cas [29 [42 48]] INFO [2019-12-17 12:28:40,957] jepsen worker 65 - jepsen.util 1055 :fail :cas [21 [30 3]] :ValueNotSwapFrom INFO [2019-12-17 12:28:40,957] jepsen worker 65 - jepsen.util 1055 :invoke :write [21 31] INFO [2019-12-17 12:28:40,973] jepsen worker 27 - jepsen.util 1197 :ok :read [9 9] INFO [2019-12-17 12:28:40,973] jepsen worker 27 - jepsen.util 1197 :invoke :write [9 44] INFO [2019-12-17 12:28:41,000] jepsen worker 69 - jepsen.util 1059 :fail :cas [23 [46 42]] :ValueNotSwapFrom INFO [2019-12-17 12:28:41,001] jepsen worker 69 - jepsen.util 1059 :invoke :write [23 10] INFO [2019-12-17 12:28:41,006] jepsen worker 58 - jepsen.util 1498 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:28:41,006] jepsen worker 29 - jepsen.util 1199 :fail :read [9 nil] :RequestTimeoutException INFO [2019-12-17 12:28:41,006] jepsen worker 29 - jepsen.util 1199 :invoke :cas [9 [6 9]] INFO [2019-12-17 12:28:41,007] jepsen worker 58 - jepsen.util 1498 :invoke :read [19 nil] INFO [2019-12-17 12:28:41,008] jepsen worker 37 - jepsen.util 1207 :fail :cas [12 [32 27]] :ValueNotSwapFrom INFO [2019-12-17 12:28:41,009] jepsen worker 37 - jepsen.util 1207 :invoke :cas [12 [26 14]] INFO [2019-12-17 12:28:41,050] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":7456,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:28:41,050] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":7456,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:28:41,050] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":7460,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:28:41,051] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":7460,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:28:41,058] jepsen worker 13 - jepsen.util 1273 :ok :read [4 26] INFO [2019-12-17 12:28:41,058] jepsen worker 13 - jepsen.util 1273 :invoke :read [4 nil] INFO [2019-12-17 12:28:41,071] jepsen worker 29 - jepsen.util 1199 :fail :cas [9 [6 9]] :ValueNotSwapFrom INFO [2019-12-17 12:28:41,071] jepsen worker 29 - jepsen.util 1199 :invoke :read [9 nil] INFO [2019-12-17 12:28:41,086] jepsen worker 27 - jepsen.util 1197 :ok :write [9 44] INFO [2019-12-17 12:28:41,087] jepsen worker 27 - jepsen.util 1197 :invoke :write [9 42] INFO [2019-12-17 12:28:41,088] jepsen worker 28 - jepsen.util 928 :fail :read [9 nil] :RequestTimeoutException INFO [2019-12-17 12:28:41,088] jepsen worker 28 - jepsen.util 928 :invoke :read [9 nil] INFO [2019-12-17 12:28:41,090] jepsen worker 65 - jepsen.util 1055 :ok :write [21 31] INFO [2019-12-17 12:28:41,090] jepsen worker 65 - jepsen.util 1055 :invoke :read [21 nil] INFO [2019-12-17 12:28:41,095] jepsen worker 47 - jepsen.util 1757 :ok :write [15 14] INFO [2019-12-17 12:28:41,096] jepsen worker 47 - jepsen.util 1757 :invoke :read [15 nil] INFO [2019-12-17 12:28:41,106] jepsen worker 53 - jepsen.util 1223 :info :write [17 35] :RequestCanceledException INFO [2019-12-17 12:28:41,106] jepsen worker 31 - jepsen.util 1381 :info :write [10 9] :RequestCanceledException INFO [2019-12-17 12:28:41,108] jepsen worker 31 - jepsen.util 1471 :invoke :write [10 27] INFO [2019-12-17 12:28:41,110] jepsen worker 53 - jepsen.util 1313 :invoke :cas [17 [36 22]] INFO [2019-12-17 12:28:41,123] jepsen worker 87 - jepsen.util 1257 :ok :read [29 32] INFO [2019-12-17 12:28:41,123] jepsen worker 87 - jepsen.util 1257 :invoke :cas [29 [44 19]] INFO [2019-12-17 12:28:41,130] jepsen worker 29 - jepsen.util 1199 :ok :read [9 9] INFO [2019-12-17 12:28:41,130] jepsen worker 29 - jepsen.util 1199 :invoke :read [9 nil] INFO [2019-12-17 12:28:41,138] jepsen worker 11 - jepsen.util 1721 :ok :write [3 3] INFO [2019-12-17 12:28:41,138] jepsen worker 11 - jepsen.util 1721 :invoke :cas [3 [37 1]] INFO [2019-12-17 12:28:41,151] jepsen worker 36 - jepsen.util 1296 :fail :cas [12 [31 2]] :ValueNotSwapFrom INFO [2019-12-17 12:28:41,151] jepsen worker 36 - jepsen.util 1296 :invoke :read [12 nil] INFO [2019-12-17 12:28:41,153] jepsen worker 19 - jepsen.util 1459 :ok :write [6 28] INFO [2019-12-17 12:28:41,153] jepsen worker 72 - jepsen.util 1242 :fail :read [24 nil] :RequestTimeoutException INFO [2019-12-17 12:28:41,153] jepsen worker 72 - jepsen.util 1242 :invoke :read [24 nil] INFO [2019-12-17 12:28:41,153] jepsen worker 19 - jepsen.util 1459 :invoke :cas [6 [33 5]] INFO [2019-12-17 12:28:41,155] jepsen worker 65 - jepsen.util 1055 :ok :read [21 31] INFO [2019-12-17 12:28:41,155] jepsen worker 65 - jepsen.util 1055 :invoke :cas [21 [0 42]] INFO [2019-12-17 12:28:41,160] jepsen worker 75 - jepsen.util 1875 :fail :cas [25 [9 42]] :ValueNotSwapFrom INFO [2019-12-17 12:28:41,160] jepsen worker 75 - jepsen.util 1875 :invoke :read [25 nil] INFO [2019-12-17 12:28:41,168] jepsen worker 27 - jepsen.util 1197 :ok :write [9 42] INFO [2019-12-17 12:28:41,168] jepsen worker 27 - jepsen.util 1197 :invoke :write [9 23] INFO [2019-12-17 12:28:41,187] jepsen worker 29 - jepsen.util 1199 :ok :read [9 44] INFO [2019-12-17 12:28:41,187] jepsen worker 29 - jepsen.util 1199 :invoke :write [9 15] INFO [2019-12-17 12:28:41,217] jepsen worker 62 - jepsen.util 1682 :ok :read [20 7] INFO [2019-12-17 12:28:41,217] jepsen worker 62 - jepsen.util 1682 :invoke :read [20 nil] INFO [2019-12-17 12:28:41,235] jepsen worker 12 - jepsen.util 1362 :info :write [4 34] :RequestCanceledException INFO [2019-12-17 12:28:41,239] jepsen worker 12 - jepsen.util 1452 :invoke :cas [4 [38 38]] INFO [2019-12-17 12:28:41,263] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":7482,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:28:41,264] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":7482,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:28:41,264] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":7483,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:28:41,265] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":7483,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:28:41,265] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":7484,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:28:41,266] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":7484,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:28:41,268] jepsen worker 88 - jepsen.util 1168 :fail :cas [29 [2 9]] :ValueNotSwapFrom INFO [2019-12-17 12:28:41,268] jepsen worker 88 - jepsen.util 1168 :invoke :read [29 nil] INFO [2019-12-17 12:28:41,269] jepsen worker 65 - jepsen.util 1055 :fail :cas [21 [0 42]] :ValueNotSwapFrom INFO [2019-12-17 12:28:41,270] jepsen worker 65 - jepsen.util 1055 :invoke :cas [21 [12 2]] INFO [2019-12-17 12:28:41,272] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":7490,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:28:41,273] jepsen worker 28 - jepsen.util 928 :ok :read [9 42] INFO [2019-12-17 12:28:41,304] jepsen worker 27 - jepsen.util 1197 :ok :write [9 23] INFO [2019-12-17 12:28:41,306] jepsen worker 14 - jepsen.util 1184 :fail :cas [4 [17 35]] :ValueNotSwapFrom INFO [2019-12-17 12:28:41,321] jepsen worker 65 - jepsen.util 1055 :fail :cas [21 [12 2]] :ValueNotSwapFrom INFO [2019-12-17 12:28:41,335] jepsen worker 60 - jepsen.util 1770 :info :write [20 7] :RequestCanceledException INFO [2019-12-17 12:28:41,336] jepsen worker 29 - jepsen.util 1199 :ok :write [9 15] INFO [2019-12-17 12:28:41,339] jepsen worker 73 - jepsen.util 1333 :ok :write [24 5] INFO [2019-12-17 12:28:41,341] jepsen worker 46 - jepsen.util 1666 :ok :write [15 24] INFO [2019-12-17 12:28:41,345] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":7490,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:28:41,345] jepsen worker 28 - jepsen.util 928 :invoke :write [9 34] INFO [2019-12-17 12:28:41,345] jepsen worker 27 - jepsen.util 1197 :invoke :write [9 13] INFO [2019-12-17 12:28:41,345] jepsen worker 14 - jepsen.util 1184 :invoke :read [4 nil] INFO [2019-12-17 12:28:41,345] jepsen worker 65 - jepsen.util 1055 :invoke :read [21 nil] INFO [2019-12-17 12:28:41,346] jepsen worker 29 - jepsen.util 1199 :invoke :cas [9 [41 47]] INFO [2019-12-17 12:28:41,346] jepsen worker 73 - jepsen.util 1333 :invoke :cas [24 [1 4]] INFO [2019-12-17 12:28:41,346] jepsen worker 46 - jepsen.util 1666 :invoke :cas [15 [23 12]] INFO [2019-12-17 12:28:41,346] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":7493,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:28:41,346] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":7493,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:28:41,346] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":7494,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:28:41,346] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":7494,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:28:41,348] jepsen worker 60 - jepsen.util 1860 :invoke :cas [20 [6 16]] INFO [2019-12-17 12:28:41,349] jepsen worker 62 - jepsen.util 1682 :ok :read [20 7] INFO [2019-12-17 12:28:41,349] jepsen worker 62 - jepsen.util 1682 :invoke :read [20 nil] INFO [2019-12-17 12:28:41,353] jepsen worker 54 - jepsen.util 1494 :info :write [18 11] :RequestCanceledException INFO [2019-12-17 12:28:41,353] jepsen worker 45 - jepsen.util 1395 :info :cas [15 [27 5]] :RequestCanceledException INFO [2019-12-17 12:28:41,355] jepsen worker 45 - jepsen.util 1485 :invoke :write [15 45] INFO [2019-12-17 12:28:41,357] jepsen worker 54 - jepsen.util 1584 :invoke :cas [18 [18 36]] INFO [2019-12-17 12:28:41,387] jepsen worker 32 - jepsen.util 1382 :info :write [10 27] :RequestCanceledException INFO [2019-12-17 12:28:41,389] jepsen worker 32 - jepsen.util 1472 :invoke :write [10 34] INFO [2019-12-17 12:28:41,391] jepsen worker 75 - jepsen.util 1875 :ok :read [25 2] INFO [2019-12-17 12:28:41,391] jepsen worker 75 - jepsen.util 1875 :invoke :cas [25 [47 21]] INFO [2019-12-17 12:28:41,406] jepsen worker 70 - jepsen.util 1510 :info :write [23 44] :RequestCanceledException INFO [2019-12-17 12:28:41,408] jepsen worker 70 - jepsen.util 1600 :invoke :read [23 nil] INFO [2019-12-17 12:28:41,434] jepsen worker 47 - jepsen.util 1757 :ok :read [15 24] INFO [2019-12-17 12:28:41,434] jepsen worker 47 - jepsen.util 1757 :invoke :read [15 nil] INFO [2019-12-17 12:28:41,435] jepsen worker 29 - jepsen.util 1199 :fail :cas [9 [41 47]] :ValueNotSwapFrom INFO [2019-12-17 12:28:41,435] jepsen worker 29 - jepsen.util 1199 :invoke :read [9 nil] INFO [2019-12-17 12:28:41,446] jepsen worker 53 - jepsen.util 1313 :fail :cas [17 [36 22]] :ValueNotSwapFrom INFO [2019-12-17 12:28:41,446] jepsen worker 53 - jepsen.util 1313 :invoke :write [17 48] INFO [2019-12-17 12:28:41,447] jepsen worker 65 - jepsen.util 1055 :ok :read [21 31] INFO [2019-12-17 12:28:41,448] jepsen worker 65 - jepsen.util 1055 :invoke :write [21 0] INFO [2019-12-17 12:28:41,468] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:28:41,468] jepsen nemesis - couchbase.util Rebalance check count: 49 INFO [2019-12-17 12:28:41,468] jepsen nemesis - couchbase.util Rebalance stuck count: 30 INFO [2019-12-17 12:28:41,476] jepsen worker 52 - jepsen.util 1492 :ok :read [17 44] INFO [2019-12-17 12:28:41,476] jepsen worker 52 - jepsen.util 1492 :invoke :cas [17 [20 15]] INFO [2019-12-17 12:28:41,481] jepsen worker 72 - jepsen.util 1242 :ok :read [24 5] INFO [2019-12-17 12:28:41,482] jepsen worker 72 - jepsen.util 1242 :invoke :write [24 3] INFO [2019-12-17 12:28:41,482] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":7516,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:28:41,482] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":7516,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:28:41,482] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":7518,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:28:41,483] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":7518,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:28:41,497] jepsen worker 89 - jepsen.util 1439 :fail :cas [29 [42 48]] :ValueNotSwapFrom INFO [2019-12-17 12:28:41,497] jepsen worker 89 - jepsen.util 1439 :invoke :read [29 nil] INFO [2019-12-17 12:28:41,506] jepsen worker 6 - jepsen.util 1716 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:28:41,506] jepsen worker 6 - jepsen.util 1716 :invoke :cas [2 [14 6]] INFO [2019-12-17 12:28:41,514] jepsen worker 29 - jepsen.util 1199 :ok :read [9 15] INFO [2019-12-17 12:28:41,515] jepsen worker 29 - jepsen.util 1199 :invoke :cas [9 [16 28]] INFO [2019-12-17 12:28:41,515] jepsen worker 73 - jepsen.util 1333 :fail :cas [24 [1 4]] :ValueNotSwapFrom INFO [2019-12-17 12:28:41,515] jepsen worker 73 - jepsen.util 1333 :invoke :write [24 43] INFO [2019-12-17 12:28:41,529] jepsen worker 80 - jepsen.util 1430 :fail :cas [26 [7 25]] :ValueNotSwapFrom INFO [2019-12-17 12:28:41,530] jepsen worker 80 - jepsen.util 1430 :invoke :cas [26 [31 37]] INFO [2019-12-17 12:28:41,572] jepsen worker 29 - jepsen.util 1199 :fail :cas [9 [16 28]] :ValueNotSwapFrom INFO [2019-12-17 12:28:41,573] jepsen worker 29 - jepsen.util 1199 :invoke :write [9 2] INFO [2019-12-17 12:28:41,575] jepsen worker 60 - jepsen.util 1860 :fail :cas [20 [6 16]] :ValueNotSwapFrom INFO [2019-12-17 12:28:41,575] jepsen worker 60 - jepsen.util 1860 :invoke :read [20 nil] INFO [2019-12-17 12:28:41,580] jepsen worker 27 - jepsen.util 1197 :ok :write [9 13] INFO [2019-12-17 12:28:41,580] jepsen worker 27 - jepsen.util 1197 :invoke :cas [9 [19 6]] INFO [2019-12-17 12:28:41,581] jepsen worker 62 - jepsen.util 1682 :ok :read [20 7] INFO [2019-12-17 12:28:41,581] jepsen worker 62 - jepsen.util 1682 :invoke :write [20 27] INFO [2019-12-17 12:28:41,620] jepsen worker 27 - jepsen.util 1197 :fail :cas [9 [19 6]] :ValueNotSwapFrom INFO [2019-12-17 12:28:41,620] jepsen worker 27 - jepsen.util 1197 :invoke :write [9 36] INFO [2019-12-17 12:28:41,620] jepsen worker 71 - jepsen.util 1061 :info :write [23 28] :RequestCanceledException INFO [2019-12-17 12:28:41,620] jepsen worker 26 - jepsen.util 1646 :info :write [8 13] :RequestCanceledException INFO [2019-12-17 12:28:41,624] jepsen worker 26 - jepsen.util 1736 :invoke :read [8 nil] INFO [2019-12-17 12:28:41,624] jepsen worker 71 - jepsen.util 1151 :invoke :cas [23 [7 43]] INFO [2019-12-17 12:28:41,636] jepsen worker 64 - jepsen.util 964 :fail :read [21 nil] :RequestTimeoutException INFO [2019-12-17 12:28:41,636] jepsen worker 25 - jepsen.util 1825 :info :write [8 10] :RequestCanceledException INFO [2019-12-17 12:28:41,636] jepsen worker 64 - jepsen.util 964 :invoke :read [21 nil] INFO [2019-12-17 12:28:41,639] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":7539,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:28:41,639] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":7539,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:28:41,641] jepsen worker 25 - jepsen.util 1915 :invoke :write [8 29] INFO [2019-12-17 12:28:41,646] jepsen worker 12 - jepsen.util 1452 :fail :cas [4 [38 38]] :ValueNotSwapFrom INFO [2019-12-17 12:28:41,646] jepsen worker 12 - jepsen.util 1452 :invoke :read [4 nil] INFO [2019-12-17 12:28:41,648] jepsen worker 21 - jepsen.util 1641 :info :write [7 12] :RequestCanceledException INFO [2019-12-17 12:28:41,649] jepsen worker 13 - jepsen.util 1273 :ok :read [4 26] INFO [2019-12-17 12:28:41,650] jepsen worker 13 - jepsen.util 1273 :invoke :read [4 nil] INFO [2019-12-17 12:28:41,650] jepsen worker 21 - jepsen.util 1731 :invoke :read [7 nil] INFO [2019-12-17 12:28:41,659] jepsen worker 65 - jepsen.util 1055 :ok :write [21 0] INFO [2019-12-17 12:28:41,659] jepsen worker 65 - jepsen.util 1055 :invoke :read [21 nil] INFO [2019-12-17 12:28:41,662] jepsen worker 60 - jepsen.util 1860 :ok :read [20 7] INFO [2019-12-17 12:28:41,662] jepsen worker 60 - jepsen.util 1860 :invoke :read [20 nil] INFO [2019-12-17 12:28:41,688] jepsen worker 74 - jepsen.util 1244 :info :write [24 48] :RequestCanceledException INFO [2019-12-17 12:28:41,688] jepsen worker 79 - jepsen.util 1249 :info :write [26 10] :RequestCanceledException INFO [2019-12-17 12:28:41,690] jepsen worker 79 - jepsen.util 1339 :invoke :write [26 37] INFO [2019-12-17 12:28:41,693] jepsen worker 74 - jepsen.util 1334 :invoke :write [24 30] INFO [2019-12-17 12:28:41,702] jepsen worker 87 - jepsen.util 1257 :fail :cas [29 [44 19]] :ValueNotSwapFrom INFO [2019-12-17 12:28:41,702] jepsen worker 87 - jepsen.util 1257 :invoke :read [29 nil] INFO [2019-12-17 12:28:41,720] jepsen worker 77 - jepsen.util 1607 :info :write [25 24] :RequestCanceledException INFO [2019-12-17 12:28:41,720] jepsen worker 22 - jepsen.util 1642 :info :write [7 42] :RequestCanceledException INFO [2019-12-17 12:28:41,720] jepsen worker 34 - jepsen.util 1834 :info :write [11 1] :RequestCanceledException INFO [2019-12-17 12:28:41,722] jepsen worker 22 - jepsen.util 1732 :invoke :cas [7 [11 29]] INFO [2019-12-17 12:28:41,724] jepsen worker 34 - jepsen.util 1924 :invoke :write [11 5] INFO [2019-12-17 12:28:41,728] jepsen worker 29 - jepsen.util 1199 :ok :write [9 2] INFO [2019-12-17 12:28:41,728] jepsen worker 29 - jepsen.util 1199 :invoke :read [9 nil] INFO [2019-12-17 12:28:41,728] jepsen worker 77 - jepsen.util 1697 :invoke :cas [25 [6 27]] INFO [2019-12-17 12:28:41,734] jepsen worker 54 - jepsen.util 1584 :fail :cas [18 [18 36]] :ValueNotSwapFrom INFO [2019-12-17 12:28:41,734] jepsen worker 54 - jepsen.util 1584 :invoke :cas [18 [6 18]] INFO [2019-12-17 12:28:41,740] jepsen worker 60 - jepsen.util 1860 :ok :read [20 7] INFO [2019-12-17 12:28:41,740] jepsen worker 60 - jepsen.util 1860 :invoke :read [20 nil] INFO [2019-12-17 12:28:41,765] jepsen worker 36 - jepsen.util 1296 :ok :read [12 3] INFO [2019-12-17 12:28:41,765] jepsen worker 36 - jepsen.util 1296 :invoke :write [12 4] INFO [2019-12-17 12:28:41,770] jepsen worker 65 - jepsen.util 1055 :ok :read [21 0] INFO [2019-12-17 12:28:41,770] jepsen worker 65 - jepsen.util 1055 :invoke :cas [21 [6 6]] INFO [2019-12-17 12:28:41,777] jepsen worker 14 - jepsen.util 1184 :ok :read [4 26] INFO [2019-12-17 12:28:41,777] jepsen worker 14 - jepsen.util 1184 :invoke :write [4 8] INFO [2019-12-17 12:28:41,784] jepsen worker 44 - jepsen.util 1574 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:28:41,784] jepsen worker 44 - jepsen.util 1574 :invoke :read [14 nil] INFO [2019-12-17 12:28:41,789] jepsen worker 70 - jepsen.util 1600 :ok :read [23 8] INFO [2019-12-17 12:28:41,790] jepsen worker 70 - jepsen.util 1600 :invoke :cas [23 [22 49]] INFO [2019-12-17 12:28:41,793] jepsen worker 27 - jepsen.util 1197 :ok :write [9 36] INFO [2019-12-17 12:28:41,793] jepsen worker 27 - jepsen.util 1197 :invoke :cas [9 [17 23]] INFO [2019-12-17 12:28:41,796] jepsen worker 26 - jepsen.util 1736 :ok :read [8 14] INFO [2019-12-17 12:28:41,796] jepsen worker 26 - jepsen.util 1736 :invoke :read [8 nil] INFO [2019-12-17 12:28:41,806] jepsen worker 8 - jepsen.util 1628 :info :cas [2 [37 43]] :RequestCanceledException INFO [2019-12-17 12:28:41,807] jepsen worker 8 - jepsen.util 1718 :invoke :write [2 49] INFO [2019-12-17 12:28:41,815] jepsen worker 65 - jepsen.util 1055 :fail :cas [21 [6 6]] :ValueNotSwapFrom INFO [2019-12-17 12:28:41,816] jepsen worker 65 - jepsen.util 1055 :invoke :cas [21 [3 22]] INFO [2019-12-17 12:28:41,845] jepsen worker 27 - jepsen.util 1197 :fail :cas [9 [17 23]] :ValueNotSwapFrom INFO [2019-12-17 12:28:41,845] jepsen worker 27 - jepsen.util 1197 :invoke :read [9 nil] INFO [2019-12-17 12:28:41,877] jepsen worker 52 - jepsen.util 1492 :fail :cas [17 [20 15]] :ValueNotSwapFrom INFO [2019-12-17 12:28:41,878] jepsen worker 52 - jepsen.util 1492 :invoke :cas [17 [25 47]] INFO [2019-12-17 12:28:41,888] jepsen worker 65 - jepsen.util 1055 :fail :cas [21 [3 22]] :ValueNotSwapFrom INFO [2019-12-17 12:28:41,888] jepsen worker 65 - jepsen.util 1055 :invoke :read [21 nil] INFO [2019-12-17 12:28:41,893] jepsen worker 22 - jepsen.util 1732 :fail :cas [7 [11 29]] :ValueNotSwapFrom INFO [2019-12-17 12:28:41,893] jepsen worker 22 - jepsen.util 1732 :invoke :read [7 nil] INFO [2019-12-17 12:28:41,898] jepsen worker 26 - jepsen.util 1736 :ok :read [8 14] INFO [2019-12-17 12:28:41,899] jepsen worker 26 - jepsen.util 1736 :invoke :read [8 nil] INFO [2019-12-17 12:28:41,906] jepsen worker 89 - jepsen.util 1439 :ok :read [29 32] INFO [2019-12-17 12:28:41,906] jepsen worker 89 - jepsen.util 1439 :invoke :read [29 nil] INFO [2019-12-17 12:28:41,906] jepsen worker 43 - jepsen.util 1753 :info :write [14 40] :RequestCanceledException INFO [2019-12-17 12:28:41,907] jepsen worker 43 - jepsen.util 1843 :invoke :cas [14 [11 20]] INFO [2019-12-17 12:28:41,912] jepsen worker 27 - jepsen.util 1197 :ok :read [9 36] INFO [2019-12-17 12:28:41,912] jepsen worker 27 - jepsen.util 1197 :invoke :read [9 nil] INFO [2019-12-17 12:28:41,918] jepsen worker 65 - jepsen.util 1055 :ok :read [21 0] INFO [2019-12-17 12:28:41,918] jepsen worker 65 - jepsen.util 1055 :invoke :read [21 nil] INFO [2019-12-17 12:28:41,925] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":7571,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:28:41,925] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":7571,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:28:41,925] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":7574,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:28:41,925] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":7574,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:28:41,935] jepsen worker 27 - jepsen.util 1197 :ok :read [9 36] INFO [2019-12-17 12:28:41,935] jepsen worker 27 - jepsen.util 1197 :invoke :cas [9 [28 13]] INFO [2019-12-17 12:28:41,936] jepsen worker 38 - jepsen.util 1208 :info :write [12 0] :RequestCanceledException INFO [2019-12-17 12:28:41,937] jepsen worker 38 - jepsen.util 1298 :invoke :cas [12 [27 1]] INFO [2019-12-17 12:28:41,942] jepsen worker 65 - jepsen.util 1055 :ok :read [21 0] INFO [2019-12-17 12:28:41,942] jepsen worker 65 - jepsen.util 1055 :invoke :read [21 nil] INFO [2019-12-17 12:28:41,975] jepsen worker 27 - jepsen.util 1197 :fail :cas [9 [28 13]] :ValueNotSwapFrom INFO [2019-12-17 12:28:41,975] jepsen worker 27 - jepsen.util 1197 :invoke :cas [9 [40 5]] INFO [2019-12-17 12:28:41,986] jepsen worker 12 - jepsen.util 1452 :ok :read [4 26] INFO [2019-12-17 12:28:41,986] jepsen worker 12 - jepsen.util 1452 :invoke :cas [4 [32 3]] INFO [2019-12-17 12:28:41,992] jepsen worker 65 - jepsen.util 1055 :ok :read [21 0] INFO [2019-12-17 12:28:41,992] jepsen worker 65 - jepsen.util 1055 :invoke :read [21 nil] INFO [2019-12-17 12:28:42,015] jepsen worker 80 - jepsen.util 1430 :fail :cas [26 [31 37]] :ValueNotSwapFrom INFO [2019-12-17 12:28:42,015] jepsen worker 80 - jepsen.util 1430 :invoke :read [26 nil] INFO [2019-12-17 12:28:42,018] jepsen worker 88 - jepsen.util 1168 :ok :read [29 32] INFO [2019-12-17 12:28:42,019] jepsen worker 88 - jepsen.util 1168 :invoke :write [29 24] INFO [2019-12-17 12:28:42,022] jepsen worker 42 - jepsen.util 1842 :info :write [14 1] :RequestCanceledException INFO [2019-12-17 12:28:42,024] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":7592,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:28:42,024] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":7592,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:28:42,028] jepsen worker 42 - jepsen.util 1932 :invoke :cas [14 [47 18]] INFO [2019-12-17 12:28:42,058] jepsen worker 75 - jepsen.util 1875 :fail :cas [25 [47 21]] :ValueNotSwapFrom INFO [2019-12-17 12:28:42,058] jepsen worker 27 - jepsen.util 1197 :fail :cas [9 [40 5]] :ValueNotSwapFrom INFO [2019-12-17 12:28:42,058] jepsen worker 27 - jepsen.util 1197 :invoke :read [9 nil] INFO [2019-12-17 12:28:42,058] jepsen worker 75 - jepsen.util 1875 :invoke :read [25 nil] INFO [2019-12-17 12:28:42,064] jepsen worker 21 - jepsen.util 1731 :ok :read [7 20] INFO [2019-12-17 12:28:42,065] jepsen worker 21 - jepsen.util 1731 :invoke :read [7 nil] INFO [2019-12-17 12:28:42,065] jepsen worker 65 - jepsen.util 1055 :ok :read [21 0] INFO [2019-12-17 12:28:42,065] jepsen worker 65 - jepsen.util 1055 :invoke :read [21 nil] INFO [2019-12-17 12:28:42,101] jepsen worker 27 - jepsen.util 1197 :ok :read [9 36] INFO [2019-12-17 12:28:42,101] jepsen worker 27 - jepsen.util 1197 :invoke :write [9 33] INFO [2019-12-17 12:28:42,114] jepsen worker 22 - jepsen.util 1732 :ok :read [7 20] INFO [2019-12-17 12:28:42,114] jepsen worker 22 - jepsen.util 1732 :invoke :cas [7 [2 33]] INFO [2019-12-17 12:28:42,118] jepsen worker 65 - jepsen.util 1055 :ok :read [21 0] INFO [2019-12-17 12:28:42,118] jepsen worker 65 - jepsen.util 1055 :invoke :read [21 nil] INFO [2019-12-17 12:28:42,119] jepsen worker 52 - jepsen.util 1492 :fail :cas [17 [25 47]] :ValueNotSwapFrom INFO [2019-12-17 12:28:42,119] jepsen worker 52 - jepsen.util 1492 :invoke :cas [17 [32 10]] INFO [2019-12-17 12:28:42,122] jepsen worker 7 - jepsen.util 1897 :info :write [2 41] :RequestCanceledException INFO [2019-12-17 12:28:42,128] jepsen worker 7 - jepsen.util 1987 :invoke :read [2 nil] INFO [2019-12-17 12:28:42,143] jepsen worker 54 - jepsen.util 1584 :fail :cas [18 [6 18]] :ValueNotSwapFrom INFO [2019-12-17 12:28:42,144] jepsen worker 54 - jepsen.util 1584 :invoke :read [18 nil] INFO [2019-12-17 12:28:42,153] jepsen worker 56 - jepsen.util 1316 :info :write [18 39] :RequestCanceledException INFO [2019-12-17 12:28:42,153] jepsen worker 26 - jepsen.util 1736 :ok :read [8 14] INFO [2019-12-17 12:28:42,153] jepsen worker 26 - jepsen.util 1736 :invoke :write [8 48] INFO [2019-12-17 12:28:42,156] jepsen worker 56 - jepsen.util 1406 :invoke :cas [18 [1 39]] INFO [2019-12-17 12:28:42,163] jepsen worker 65 - jepsen.util 1055 :ok :read [21 0] INFO [2019-12-17 12:28:42,163] jepsen worker 65 - jepsen.util 1055 :invoke :write [21 28] INFO [2019-12-17 12:28:42,194] jepsen worker 27 - jepsen.util 1197 :ok :write [9 33] INFO [2019-12-17 12:28:42,195] jepsen worker 27 - jepsen.util 1197 :invoke :write [9 16] INFO [2019-12-17 12:28:42,212] jepsen worker 80 - jepsen.util 1430 :ok :read [26 27] INFO [2019-12-17 12:28:42,213] jepsen worker 80 - jepsen.util 1430 :invoke :write [26 28] INFO [2019-12-17 12:28:42,213] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":7619,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:28:42,215] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":7619,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:28:42,215] jepsen worker 42 - jepsen.util 1932 :fail :cas [14 [47 18]] :ValueNotSwapFrom INFO [2019-12-17 12:28:42,217] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":7622,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:28:42,217] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":7622,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:28:42,218] jepsen worker 42 - jepsen.util 1932 :invoke :cas [14 [33 40]] INFO [2019-12-17 12:28:42,220] jepsen worker 30 - jepsen.util 1470 :fail :read [10 nil] :RequestTimeoutException INFO [2019-12-17 12:28:42,220] jepsen worker 30 - jepsen.util 1470 :invoke :read [10 nil] INFO [2019-12-17 12:28:42,243] jepsen worker 22 - jepsen.util 1732 :fail :cas [7 [2 33]] :ValueNotSwapFrom INFO [2019-12-17 12:28:42,243] jepsen worker 22 - jepsen.util 1732 :invoke :cas [7 [1 17]] INFO [2019-12-17 12:28:42,252] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":7611,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:28:42,253] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":7611,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:28:42,253] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":7613,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:28:42,253] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":7613,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:28:42,280] jepsen worker 60 - jepsen.util 1860 :ok :read [20 7] INFO [2019-12-17 12:28:42,280] jepsen worker 60 - jepsen.util 1860 :invoke :read [20 nil] INFO [2019-12-17 12:28:42,285] jepsen worker 54 - jepsen.util 1584 :ok :read [18 26] INFO [2019-12-17 12:28:42,285] jepsen worker 54 - jepsen.util 1584 :invoke :read [18 nil] INFO [2019-12-17 12:28:42,297] jepsen worker 71 - jepsen.util 1151 :fail :cas [23 [7 43]] :ValueNotSwapFrom INFO [2019-12-17 12:28:42,297] jepsen worker 71 - jepsen.util 1151 :invoke :cas [23 [26 14]] INFO [2019-12-17 12:28:42,300] jepsen worker 37 - jepsen.util 1207 :fail :cas [12 [26 14]] :ValueNotSwapFrom INFO [2019-12-17 12:28:42,300] jepsen worker 37 - jepsen.util 1207 :invoke :cas [12 [20 14]] INFO [2019-12-17 12:28:42,322] jepsen worker 78 - jepsen.util 1338 :info :write [26 11] :RequestCanceledException INFO [2019-12-17 12:28:42,324] jepsen worker 78 - jepsen.util 1428 :invoke :cas [26 [34 10]] INFO [2019-12-17 12:28:42,330] jepsen worker 89 - jepsen.util 1439 :ok :read [29 32] INFO [2019-12-17 12:28:42,330] jepsen worker 89 - jepsen.util 1439 :invoke :read [29 nil] INFO [2019-12-17 12:28:42,335] jepsen worker 55 - jepsen.util 1675 :info :write [18 12] :RequestCanceledException INFO [2019-12-17 12:28:42,340] jepsen worker 55 - jepsen.util 1765 :invoke :cas [18 [40 18]] INFO [2019-12-17 12:28:42,342] jepsen worker 30 - jepsen.util 1470 :ok :read [10 30] INFO [2019-12-17 12:28:42,343] jepsen worker 30 - jepsen.util 1470 :invoke :read [10 nil] INFO [2019-12-17 12:28:42,343] jepsen worker 12 - jepsen.util 1452 :fail :cas [4 [32 3]] :ValueNotSwapFrom INFO [2019-12-17 12:28:42,343] jepsen worker 12 - jepsen.util 1452 :invoke :read [4 nil] INFO [2019-12-17 12:28:42,347] jepsen worker 87 - jepsen.util 1257 :ok :read [29 32] INFO [2019-12-17 12:28:42,347] jepsen worker 87 - jepsen.util 1257 :invoke :cas [29 [14 49]] INFO [2019-12-17 12:28:42,352] jepsen worker 5 - jepsen.util 905 :info :write [1 27] :RequestCanceledException INFO [2019-12-17 12:28:42,354] jepsen worker 5 - jepsen.util 995 :invoke :write [1 16] INFO [2019-12-17 12:28:42,356] jepsen worker 7 - jepsen.util 1987 :ok :read [2 15] INFO [2019-12-17 12:28:42,356] jepsen worker 7 - jepsen.util 1987 :invoke :read [2 nil] INFO [2019-12-17 12:28:42,370] jepsen worker 65 - jepsen.util 1055 :ok :write [21 28] INFO [2019-12-17 12:28:42,370] jepsen worker 65 - jepsen.util 1055 :invoke :write [21 28] INFO [2019-12-17 12:28:42,374] jepsen worker 27 - jepsen.util 1197 :ok :write [9 16] INFO [2019-12-17 12:28:42,374] jepsen worker 27 - jepsen.util 1197 :invoke :cas [9 [9 37]] INFO [2019-12-17 12:28:42,387] jepsen worker 18 - jepsen.util 1458 :info :cas [6 [22 35]] :RequestCanceledException INFO [2019-12-17 12:28:42,392] jepsen worker 18 - jepsen.util 1548 :invoke :read [6 nil] INFO [2019-12-17 12:28:42,404] jepsen worker 42 - jepsen.util 1932 :fail :cas [14 [33 40]] :ValueNotSwapFrom INFO [2019-12-17 12:28:42,412] jepsen worker 22 - jepsen.util 1732 :fail :cas [7 [1 17]] :ValueNotSwapFrom INFO [2019-12-17 12:28:42,418] jepsen worker 30 - jepsen.util 1470 :ok :read [10 30] INFO [2019-12-17 12:28:42,421] jepsen worker 27 - jepsen.util 1197 :fail :cas [9 [9 37]] :ValueNotSwapFrom INFO [2019-12-17 12:28:42,422] jepsen worker 51 - jepsen.util 1311 :info :write [17 43] :RequestCanceledException INFO [2019-12-17 12:28:42,434] jepsen worker 22 - jepsen.util 1732 :invoke :read [7 nil] INFO [2019-12-17 12:28:42,434] jepsen worker 42 - jepsen.util 1932 :invoke :read [14 nil] INFO [2019-12-17 12:28:42,434] jepsen worker 30 - jepsen.util 1470 :invoke :write [10 8] INFO [2019-12-17 12:28:42,434] jepsen worker 27 - jepsen.util 1197 :invoke :cas [9 [15 20]] INFO [2019-12-17 12:28:42,435] jepsen worker 51 - jepsen.util 1401 :invoke :read [17 nil] INFO [2019-12-17 12:28:42,453] jepsen worker 63 - jepsen.util 1233 :fail :read [21 nil] :RequestTimeoutException INFO [2019-12-17 12:28:42,453] jepsen worker 63 - jepsen.util 1233 :invoke :cas [21 [3 31]] INFO [2019-12-17 12:28:42,457] jepsen worker 65 - jepsen.util 1055 :ok :write [21 28] INFO [2019-12-17 12:28:42,457] jepsen worker 65 - jepsen.util 1055 :invoke :read [21 nil] INFO [2019-12-17 12:28:42,467] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":7645,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:28:42,468] jepsen worker 27 - jepsen.util 1197 :fail :cas [9 [15 20]] :ValueNotSwapFrom INFO [2019-12-17 12:28:42,468] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":7645,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:28:42,468] jepsen worker 27 - jepsen.util 1197 :invoke :write [9 43] INFO [2019-12-17 12:28:42,489] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:28:42,489] jepsen nemesis - couchbase.util Rebalance check count: 50 INFO [2019-12-17 12:28:42,489] jepsen nemesis - couchbase.util Rebalance stuck count: 31 INFO [2019-12-17 12:28:42,506] jepsen worker 4 - jepsen.util 1084 :info :write [1 21] :RequestCanceledException INFO [2019-12-17 12:28:42,507] jepsen worker 4 - jepsen.util 1174 :invoke :read [1 nil] INFO [2019-12-17 12:28:42,519] jepsen worker 0 - jepsen.util 1800 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:28:42,519] jepsen worker 33 - jepsen.util 1923 :info :write [11 21] :RequestCanceledException INFO [2019-12-17 12:28:42,519] jepsen worker 0 - jepsen.util 1800 :invoke :cas [0 [21 12]] INFO [2019-12-17 12:28:42,525] jepsen worker 33 - jepsen.util 2013 :invoke :read [11 nil] INFO [2019-12-17 12:28:42,535] jepsen worker 3 - jepsen.util 1173 :info :write [1 5] :RequestCanceledException INFO [2019-12-17 12:28:42,537] jepsen worker 3 - jepsen.util 1263 :invoke :read [1 nil] INFO [2019-12-17 12:28:42,542] jepsen worker 18 - jepsen.util 1548 :ok :read [6 28] INFO [2019-12-17 12:28:42,543] jepsen worker 18 - jepsen.util 1548 :invoke :cas [6 [49 0]] INFO [2019-12-17 12:28:42,545] jepsen worker 60 - jepsen.util 1860 :ok :read [20 7] INFO [2019-12-17 12:28:42,545] jepsen worker 60 - jepsen.util 1860 :invoke :read [20 nil] INFO [2019-12-17 12:28:42,552] jepsen worker 13 - jepsen.util 1273 :ok :read [4 26] INFO [2019-12-17 12:28:42,552] jepsen worker 13 - jepsen.util 1273 :invoke :write [4 34] INFO [2019-12-17 12:28:42,553] jepsen worker 76 - jepsen.util 1606 :info :write [25 28] :RequestCanceledException INFO [2019-12-17 12:28:42,556] jepsen worker 76 - jepsen.util 1696 :invoke :read [25 nil] INFO [2019-12-17 12:28:42,556] jepsen worker 65 - jepsen.util 1055 :ok :read [21 28] INFO [2019-12-17 12:28:42,556] jepsen worker 65 - jepsen.util 1055 :invoke :write [21 33] INFO [2019-12-17 12:28:42,560] jepsen worker 75 - jepsen.util 1875 :ok :read [25 2] INFO [2019-12-17 12:28:42,560] jepsen worker 75 - jepsen.util 1875 :invoke :read [25 nil] INFO [2019-12-17 12:28:42,588] jepsen worker 82 - jepsen.util 2062 :info :cas [27 [3 23]] :RequestCanceledException INFO [2019-12-17 12:28:42,589] jepsen worker 7 - jepsen.util 1987 :ok :read [2 15] INFO [2019-12-17 12:28:42,589] jepsen worker 7 - jepsen.util 1987 :invoke :write [2 34] INFO [2019-12-17 12:28:42,593] jepsen worker 82 - jepsen.util 2152 :invoke :cas [27 [30 31]] INFO [2019-12-17 12:28:42,606] jepsen worker 10 - jepsen.util 1270 :info :write [3 25] :RequestCanceledException INFO [2019-12-17 12:28:42,613] jepsen worker 10 - jepsen.util 1360 :invoke :cas [3 [31 23]] INFO [2019-12-17 12:28:42,622] jepsen worker 85 - jepsen.util 1705 :info :write [28 23] :RequestCanceledException INFO [2019-12-17 12:28:42,625] jepsen worker 18 - jepsen.util 1548 :fail :cas [6 [49 0]] :ValueNotSwapFrom INFO [2019-12-17 12:28:42,626] jepsen worker 18 - jepsen.util 1548 :invoke :read [6 nil] INFO [2019-12-17 12:28:42,630] jepsen worker 85 - jepsen.util 1795 :invoke :cas [28 [33 28]] INFO [2019-12-17 12:28:42,634] jepsen worker 70 - jepsen.util 1600 :fail :cas [23 [22 49]] :ValueNotSwapFrom INFO [2019-12-17 12:28:42,634] jepsen worker 70 - jepsen.util 1600 :invoke :write [23 18] INFO [2019-12-17 12:28:42,656] jepsen worker 51 - jepsen.util 1401 :ok :read [17 44] INFO [2019-12-17 12:28:42,657] jepsen worker 51 - jepsen.util 1401 :invoke :read [17 nil] INFO [2019-12-17 12:28:42,658] jepsen worker 22 - jepsen.util 1732 :ok :read [7 20] INFO [2019-12-17 12:28:42,658] jepsen worker 22 - jepsen.util 1732 :invoke :read [7 nil] INFO [2019-12-17 12:28:42,669] jepsen worker 52 - jepsen.util 1492 :fail :cas [17 [32 10]] :ValueNotSwapFrom INFO [2019-12-17 12:28:42,670] jepsen worker 52 - jepsen.util 1492 :invoke :write [17 42] INFO [2019-12-17 12:28:42,673] jepsen worker 65 - jepsen.util 1055 :ok :write [21 33] INFO [2019-12-17 12:28:42,674] jepsen worker 65 - jepsen.util 1055 :invoke :cas [21 [2 22]] INFO [2019-12-17 12:28:42,678] jepsen worker 42 - jepsen.util 1932 :ok :read [14 27] INFO [2019-12-17 12:28:42,678] jepsen worker 42 - jepsen.util 1932 :invoke :write [14 29] INFO [2019-12-17 12:28:42,684] jepsen worker 24 - jepsen.util 1554 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:28:42,684] jepsen worker 24 - jepsen.util 1554 :invoke :write [8 47] INFO [2019-12-17 12:28:42,686] jepsen worker 21 - jepsen.util 1731 :ok :read [7 20] INFO [2019-12-17 12:28:42,686] jepsen worker 21 - jepsen.util 1731 :invoke :cas [7 [39 49]] INFO [2019-12-17 12:28:42,706] jepsen worker 48 - jepsen.util 1218 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:28:42,706] jepsen worker 9 - jepsen.util 1449 :info :write [3 16] :RequestCanceledException INFO [2019-12-17 12:28:42,707] jepsen worker 48 - jepsen.util 1218 :invoke :write [16 1] INFO [2019-12-17 12:28:42,711] jepsen worker 9 - jepsen.util 1539 :invoke :cas [3 [43 0]] INFO [2019-12-17 12:28:42,721] jepsen worker 41 - jepsen.util 1481 :info :cas [13 [32 12]] :RequestCanceledException INFO [2019-12-17 12:28:42,721] jepsen worker 86 - jepsen.util 1706 :info :cas [28 [34 6]] :RequestCanceledException INFO [2019-12-17 12:28:42,724] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":7677,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:28:42,724] jepsen worker 38 - jepsen.util 1298 :fail :cas [12 [27 1]] :ValueNotSwapFrom INFO [2019-12-17 12:28:42,725] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":7677,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:28:42,725] jepsen worker 38 - jepsen.util 1298 :invoke :write [12 26] INFO [2019-12-17 12:28:42,726] jepsen worker 86 - jepsen.util 1796 :invoke :write [28 29] INFO [2019-12-17 12:28:42,727] jepsen worker 41 - jepsen.util 1571 :invoke :cas [13 [45 10]] INFO [2019-12-17 12:28:42,736] jepsen worker 16 - jepsen.util 1546 :info :cas [5 [20 46]] :RequestCanceledException INFO [2019-12-17 12:28:42,736] jepsen worker 35 - jepsen.util 1385 :info :write [11 12] :RequestCanceledException INFO [2019-12-17 12:28:42,738] jepsen worker 35 - jepsen.util 1475 :invoke :cas [11 [38 35]] INFO [2019-12-17 12:28:42,741] jepsen worker 16 - jepsen.util 1636 :invoke :write [5 40] INFO [2019-12-17 12:28:42,753] jepsen worker 50 - jepsen.util 1670 :info :cas [16 [4 2]] :RequestCanceledException INFO [2019-12-17 12:28:42,758] jepsen worker 18 - jepsen.util 1548 :ok :read [6 28] INFO [2019-12-17 12:28:42,758] jepsen worker 18 - jepsen.util 1548 :invoke :cas [6 [31 23]] INFO [2019-12-17 12:28:42,759] jepsen worker 50 - jepsen.util 1760 :invoke :cas [16 [30 1]] INFO [2019-12-17 12:28:42,770] jepsen worker 89 - jepsen.util 1439 :ok :read [29 32] INFO [2019-12-17 12:28:42,770] jepsen worker 89 - jepsen.util 1439 :invoke :cas [29 [36 19]] INFO [2019-12-17 12:28:42,771] jepsen worker 65 - jepsen.util 1055 :fail :cas [21 [2 22]] :ValueNotSwapFrom INFO [2019-12-17 12:28:42,771] jepsen worker 65 - jepsen.util 1055 :invoke :write [21 7] INFO [2019-12-17 12:28:42,783] jepsen worker 22 - jepsen.util 1732 :ok :read [7 20] INFO [2019-12-17 12:28:42,783] jepsen worker 22 - jepsen.util 1732 :invoke :read [7 nil] INFO [2019-12-17 12:28:42,784] jepsen worker 60 - jepsen.util 1860 :ok :read [20 7] INFO [2019-12-17 12:28:42,784] jepsen worker 60 - jepsen.util 1860 :invoke :write [20 22] INFO [2019-12-17 12:28:42,788] jepsen worker 9 - jepsen.util 1539 :fail :cas [3 [43 0]] :ValueNotSwapFrom INFO [2019-12-17 12:28:42,788] jepsen worker 9 - jepsen.util 1539 :invoke :write [3 4] INFO [2019-12-17 12:28:42,788] jepsen worker 57 - jepsen.util 1497 :info :cas [19 [25 49]] :RequestCanceledException INFO [2019-12-17 12:28:42,788] jepsen worker 1 - jepsen.util 1621 :info :write [0 23] :RequestCanceledException INFO [2019-12-17 12:28:42,793] jepsen worker 57 - jepsen.util 1587 :invoke :write [19 20] INFO [2019-12-17 12:28:42,796] jepsen worker 1 - jepsen.util 1711 :invoke :cas [0 [21 11]] INFO [2019-12-17 12:28:42,803] jepsen worker 23 - jepsen.util 1463 :info :write [7 26] :RequestCanceledException INFO [2019-12-17 12:28:42,803] jepsen worker 84 - jepsen.util 1614 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:28:42,804] jepsen worker 84 - jepsen.util 1614 :invoke :write [28 37] INFO [2019-12-17 12:28:42,805] jepsen worker 23 - jepsen.util 1553 :invoke :cas [7 [31 5]] INFO [2019-12-17 12:28:42,819] jepsen worker 59 - jepsen.util 1409 :info :cas [19 [10 49]] :RequestCanceledException INFO [2019-12-17 12:28:42,819] jepsen worker 17 - jepsen.util 1637 :info :cas [5 [45 10]] :RequestCanceledException INFO [2019-12-17 12:28:42,819] jepsen worker 66 - jepsen.util 1326 :info :cas [22 [32 5]] :RequestCanceledException INFO [2019-12-17 12:28:42,819] jepsen worker 40 - jepsen.util 1840 :info :write [13 47] :RequestCanceledException INFO [2019-12-17 12:28:42,823] jepsen worker 17 - jepsen.util 1727 :invoke :cas [5 [8 6]] INFO [2019-12-17 12:28:42,823] jepsen worker 59 - jepsen.util 1499 :invoke :read [19 nil] INFO [2019-12-17 12:28:42,823] jepsen worker 66 - jepsen.util 1416 :invoke :cas [22 [9 37]] INFO [2019-12-17 12:28:42,824] jepsen worker 40 - jepsen.util 1930 :invoke :cas [13 [12 11]] INFO [2019-12-17 12:28:42,833] jepsen worker 12 - jepsen.util 1452 :ok :read [4 26] INFO [2019-12-17 12:28:42,833] jepsen worker 12 - jepsen.util 1452 :invoke :write [4 3] INFO [2019-12-17 12:28:42,834] jepsen worker 67 - jepsen.util 1957 :info :write [22 28] :RequestCanceledException INFO [2019-12-17 12:28:42,838] jepsen worker 67 - jepsen.util 2047 :invoke :write [22 25] INFO [2019-12-17 12:28:42,839] jepsen worker 18 - jepsen.util 1548 :fail :cas [6 [31 23]] :ValueNotSwapFrom INFO [2019-12-17 12:28:42,839] jepsen worker 18 - jepsen.util 1548 :invoke :write [6 46] INFO [2019-12-17 12:28:42,851] jepsen worker 33 - jepsen.util 2013 :ok :read [11 7] INFO [2019-12-17 12:28:42,851] jepsen worker 33 - jepsen.util 2013 :invoke :cas [11 [26 8]] INFO [2019-12-17 12:28:42,851] jepsen worker 49 - jepsen.util 1309 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:28:42,851] jepsen worker 83 - jepsen.util 1973 :info :write [27 44] :RequestCanceledException INFO [2019-12-17 12:28:42,851] jepsen worker 15 - jepsen.util 1365 :info :cas [5 [6 10]] :RequestCanceledException INFO [2019-12-17 12:28:42,851] jepsen worker 49 - jepsen.util 1309 :invoke :read [16 nil] INFO [2019-12-17 12:28:42,856] jepsen worker 15 - jepsen.util 1455 :invoke :read [5 nil] INFO [2019-12-17 12:28:42,856] jepsen worker 83 - jepsen.util 2063 :invoke :cas [27 [38 3]] INFO [2019-12-17 12:28:42,876] jepsen worker 22 - jepsen.util 1732 :ok :read [7 20] INFO [2019-12-17 12:28:42,877] jepsen worker 22 - jepsen.util 1732 :invoke :cas [7 [5 14]] INFO [2019-12-17 12:28:42,888] jepsen worker 39 - jepsen.util 1839 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:28:42,888] jepsen worker 65 - jepsen.util 1055 :ok :write [21 7] INFO [2019-12-17 12:28:42,889] jepsen worker 39 - jepsen.util 1839 :invoke :write [13 4] INFO [2019-12-17 12:28:42,889] jepsen worker 65 - jepsen.util 1055 :invoke :cas [21 [49 7]] INFO [2019-12-17 12:28:42,905] jepsen worker 21 - jepsen.util 1731 :fail :cas [7 [39 49]] :ValueNotSwapFrom INFO [2019-12-17 12:28:42,905] jepsen worker 21 - jepsen.util 1731 :invoke :cas [7 [14 26]] INFO [2019-12-17 12:28:42,906] jepsen worker 61 - jepsen.util 1771 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:28:42,906] jepsen worker 2 - jepsen.util 1262 :info :write [0 17] :RequestCanceledException INFO [2019-12-17 12:28:42,906] jepsen worker 61 - jepsen.util 1771 :invoke :write [20 0] INFO [2019-12-17 12:28:42,908] jepsen worker 2 - jepsen.util 1352 :invoke :write [0 43] INFO [2019-12-17 12:28:42,910] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":7715,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:28:42,910] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":7715,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:28:42,922] jepsen worker 68 - jepsen.util 1238 :info :write [22 13] :RequestCanceledException INFO [2019-12-17 12:28:42,924] jepsen worker 68 - jepsen.util 1328 :invoke :read [22 nil] INFO [2019-12-17 12:28:42,936] jepsen worker 20 - jepsen.util 1370 :info :write [6 42] :RequestCanceledException INFO [2019-12-17 12:28:42,937] jepsen worker 20 - jepsen.util 1460 :invoke :cas [6 [15 28]] INFO [2019-12-17 12:28:42,983] jepsen worker 35 - jepsen.util 1475 :fail :cas [11 [38 35]] :ValueNotSwapFrom INFO [2019-12-17 12:28:42,984] jepsen worker 35 - jepsen.util 1475 :invoke :write [11 14] INFO [2019-12-17 12:28:42,988] jepsen worker 81 - jepsen.util 1521 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:28:42,988] jepsen worker 81 - jepsen.util 1521 :invoke :cas [27 [35 15]] INFO [2019-12-17 12:28:42,996] jepsen worker 54 - jepsen.util 1584 :ok :read [18 26] INFO [2019-12-17 12:28:42,996] jepsen worker 54 - jepsen.util 1584 :invoke :read [18 nil] INFO [2019-12-17 12:28:43,005] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":7731,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:28:43,005] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":7731,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:28:43,006] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":7734,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:28:43,006] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":7734,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:28:43,006] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":7735,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:28:43,006] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":7735,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:28:43,019] jepsen worker 71 - jepsen.util 1151 :fail :cas [23 [26 14]] :ValueNotSwapFrom INFO [2019-12-17 12:28:43,020] jepsen worker 71 - jepsen.util 1151 :invoke :cas [23 [35 45]] INFO [2019-12-17 12:28:43,030] jepsen worker 89 - jepsen.util 1439 :fail :cas [29 [36 19]] :ValueNotSwapFrom INFO [2019-12-17 12:28:43,030] jepsen worker 89 - jepsen.util 1439 :invoke :read [29 nil] INFO [2019-12-17 12:28:43,031] jepsen worker 75 - jepsen.util 1875 :ok :read [25 2] INFO [2019-12-17 12:28:43,031] jepsen worker 75 - jepsen.util 1875 :invoke :cas [25 [13 21]] INFO [2019-12-17 12:28:43,036] jepsen worker 69 - jepsen.util 1059 :info :write [23 10] :RequestCanceledException INFO [2019-12-17 12:28:43,037] jepsen worker 69 - jepsen.util 1149 :invoke :cas [23 [27 25]] INFO [2019-12-17 12:28:43,058] jepsen worker 56 - jepsen.util 1406 :fail :cas [18 [1 39]] :ValueNotSwapFrom INFO [2019-12-17 12:28:43,058] jepsen worker 56 - jepsen.util 1406 :invoke :read [18 nil] INFO [2019-12-17 12:28:43,071] jepsen worker 37 - jepsen.util 1207 :fail :cas [12 [20 14]] :ValueNotSwapFrom INFO [2019-12-17 12:28:43,071] jepsen worker 31 - jepsen.util 1471 :info :write [10 27] :SyncWriteAmbiguous INFO [2019-12-17 12:28:43,071] jepsen worker 37 - jepsen.util 1207 :invoke :write [12 31] INFO [2019-12-17 12:28:43,072] jepsen worker 31 - jepsen.util 1561 :invoke :read [10 nil] INFO [2019-12-17 12:28:43,082] jepsen worker 55 - jepsen.util 1765 :fail :cas [18 [40 18]] :ValueNotSwapFrom INFO [2019-12-17 12:28:43,082] jepsen worker 55 - jepsen.util 1765 :invoke :read [18 nil] INFO [2019-12-17 12:28:43,099] jepsen worker 78 - jepsen.util 1428 :fail :cas [26 [34 10]] :ValueNotSwapFrom INFO [2019-12-17 12:28:43,100] jepsen worker 78 - jepsen.util 1428 :invoke :cas [26 [27 32]] INFO [2019-12-17 12:28:43,104] jepsen worker 58 - jepsen.util 1498 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:28:43,104] jepsen worker 58 - jepsen.util 1498 :invoke :cas [19 [6 0]] INFO [2019-12-17 12:28:43,112] jepsen worker 28 - jepsen.util 928 :ok :write [9 34] INFO [2019-12-17 12:28:43,113] jepsen worker 28 - jepsen.util 928 :invoke :read [9 nil] INFO [2019-12-17 12:28:43,140] jepsen worker 28 - jepsen.util 928 :ok :read [9 34] INFO [2019-12-17 12:28:43,140] jepsen worker 28 - jepsen.util 928 :invoke :read [9 nil] INFO [2019-12-17 12:28:43,163] jepsen worker 22 - jepsen.util 1732 :fail :cas [7 [5 14]] :ValueNotSwapFrom INFO [2019-12-17 12:28:43,164] jepsen worker 22 - jepsen.util 1732 :invoke :write [7 31] INFO [2019-12-17 12:28:43,168] jepsen worker 87 - jepsen.util 1257 :fail :cas [29 [14 49]] :ValueNotSwapFrom INFO [2019-12-17 12:28:43,168] jepsen worker 87 - jepsen.util 1257 :invoke :write [29 37] INFO [2019-12-17 12:28:43,176] jepsen worker 28 - jepsen.util 928 :ok :read [9 34] INFO [2019-12-17 12:28:43,177] jepsen worker 28 - jepsen.util 928 :invoke :write [9 38] INFO [2019-12-17 12:28:43,180] jepsen worker 89 - jepsen.util 1439 :ok :read [29 32] INFO [2019-12-17 12:28:43,180] jepsen worker 89 - jepsen.util 1439 :invoke :cas [29 [35 31]] INFO [2019-12-17 12:28:43,188] jepsen worker 19 - jepsen.util 1459 :info :cas [6 [33 5]] :RequestCanceledException INFO [2019-12-17 12:28:43,190] jepsen worker 19 - jepsen.util 1549 :invoke :cas [6 [41 22]] INFO [2019-12-17 12:28:43,200] jepsen worker 11 - jepsen.util 1721 :fail :cas [3 [37 1]] :ValueNotSwapFrom INFO [2019-12-17 12:28:43,201] jepsen worker 11 - jepsen.util 1721 :invoke :read [3 nil] INFO [2019-12-17 12:28:43,203] jepsen worker 46 - jepsen.util 1666 :fail :cas [15 [23 12]] :ValueNotSwapFrom INFO [2019-12-17 12:28:43,203] jepsen worker 46 - jepsen.util 1666 :invoke :read [15 nil] INFO [2019-12-17 12:28:43,206] jepsen worker 64 - jepsen.util 964 :ok :read [21 7] INFO [2019-12-17 12:28:43,206] jepsen worker 64 - jepsen.util 964 :invoke :read [21 nil] INFO [2019-12-17 12:28:43,245] jepsen worker 10 - jepsen.util 1360 :fail :cas [3 [31 23]] :ValueNotSwapFrom INFO [2019-12-17 12:28:43,245] jepsen worker 10 - jepsen.util 1360 :invoke :cas [3 [4 10]] INFO [2019-12-17 12:28:43,247] jepsen worker 19 - jepsen.util 1549 :fail :cas [6 [41 22]] :ValueNotSwapFrom INFO [2019-12-17 12:28:43,247] jepsen worker 19 - jepsen.util 1549 :invoke :read [6 nil] INFO [2019-12-17 12:28:43,259] jepsen worker 11 - jepsen.util 1721 :ok :read [3 3] INFO [2019-12-17 12:28:43,260] jepsen worker 11 - jepsen.util 1721 :invoke :cas [3 [41 16]] INFO [2019-12-17 12:28:43,270] jepsen worker 46 - jepsen.util 1666 :ok :read [15 24] INFO [2019-12-17 12:28:43,270] jepsen worker 46 - jepsen.util 1666 :invoke :read [15 nil] INFO [2019-12-17 12:28:43,288] jepsen worker 64 - jepsen.util 964 :ok :read [21 7] INFO [2019-12-17 12:28:43,289] jepsen worker 64 - jepsen.util 964 :invoke :read [21 nil] INFO [2019-12-17 12:28:43,294] jepsen worker 10 - jepsen.util 1360 :fail :cas [3 [4 10]] :ValueNotSwapFrom INFO [2019-12-17 12:28:43,294] jepsen worker 10 - jepsen.util 1360 :invoke :read [3 nil] INFO [2019-12-17 12:28:43,296] jepsen worker 89 - jepsen.util 1439 :fail :cas [29 [35 31]] :ValueNotSwapFrom INFO [2019-12-17 12:28:43,296] jepsen worker 89 - jepsen.util 1439 :invoke :read [29 nil] INFO [2019-12-17 12:28:43,306] jepsen worker 11 - jepsen.util 1721 :fail :cas [3 [41 16]] :ValueNotSwapFrom INFO [2019-12-17 12:28:43,306] jepsen worker 11 - jepsen.util 1721 :invoke :read [3 nil] INFO [2019-12-17 12:28:43,310] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":7767,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:28:43,310] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":7767,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:28:43,310] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":7769,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:28:43,310] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":7769,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:28:43,313] jepsen worker 46 - jepsen.util 1666 :ok :read [15 24] INFO [2019-12-17 12:28:43,313] jepsen worker 46 - jepsen.util 1666 :invoke :cas [15 [6 31]] INFO [2019-12-17 12:28:43,326] jepsen worker 28 - jepsen.util 928 :ok :write [9 38] INFO [2019-12-17 12:28:43,326] jepsen worker 28 - jepsen.util 928 :invoke :write [9 28] INFO [2019-12-17 12:28:43,364] jepsen worker 10 - jepsen.util 1360 :ok :read [3 3] INFO [2019-12-17 12:28:43,365] jepsen worker 10 - jepsen.util 1360 :invoke :write [3 15] INFO [2019-12-17 12:28:43,365] jepsen worker 19 - jepsen.util 1549 :ok :read [6 28] INFO [2019-12-17 12:28:43,365] jepsen worker 19 - jepsen.util 1549 :invoke :write [6 46] INFO [2019-12-17 12:28:43,365] jepsen worker 64 - jepsen.util 964 :ok :read [21 7] INFO [2019-12-17 12:28:43,365] jepsen worker 64 - jepsen.util 964 :invoke :read [21 nil] INFO [2019-12-17 12:28:43,382] jepsen worker 46 - jepsen.util 1666 :fail :cas [15 [6 31]] :ValueNotSwapFrom INFO [2019-12-17 12:28:43,382] jepsen worker 11 - jepsen.util 1721 :ok :read [3 3] INFO [2019-12-17 12:28:43,383] jepsen worker 45 - jepsen.util 1485 :info :write [15 45] :SyncWriteAmbiguous INFO [2019-12-17 12:28:43,386] jepsen worker 11 - jepsen.util 1721 :invoke :read [3 nil] INFO [2019-12-17 12:28:43,386] jepsen worker 46 - jepsen.util 1666 :invoke :cas [15 [42 47]] INFO [2019-12-17 12:28:43,388] jepsen worker 45 - jepsen.util 1575 :invoke :cas [15 [5 30]] INFO [2019-12-17 12:28:43,392] jepsen worker 54 - jepsen.util 1584 :ok :read [18 26] INFO [2019-12-17 12:28:43,392] jepsen worker 54 - jepsen.util 1584 :invoke :cas [18 [7 16]] INFO [2019-12-17 12:28:43,402] jepsen worker 64 - jepsen.util 964 :ok :read [21 7] INFO [2019-12-17 12:28:43,402] jepsen worker 64 - jepsen.util 964 :invoke :read [21 nil] INFO [2019-12-17 12:28:43,432] jepsen worker 11 - jepsen.util 1721 :ok :read [3 3] INFO [2019-12-17 12:28:43,433] jepsen worker 11 - jepsen.util 1721 :invoke :read [3 nil] INFO [2019-12-17 12:28:43,440] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":7792,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:28:43,440] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":7792,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:28:43,440] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":7794,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:28:43,440] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":7794,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:28:43,450] jepsen worker 33 - jepsen.util 2013 :fail :cas [11 [26 8]] :ValueNotSwapFrom INFO [2019-12-17 12:28:43,450] jepsen worker 33 - jepsen.util 2013 :invoke :write [11 37] INFO [2019-12-17 12:28:43,452] jepsen worker 47 - jepsen.util 1757 :fail :read [15 nil] :RequestTimeoutException INFO [2019-12-17 12:28:43,453] jepsen worker 47 - jepsen.util 1757 :invoke :cas [15 [42 29]] INFO [2019-12-17 12:28:43,459] jepsen worker 46 - jepsen.util 1666 :fail :cas [15 [42 47]] :ValueNotSwapFrom INFO [2019-12-17 12:28:43,460] jepsen worker 46 - jepsen.util 1666 :invoke :write [15 44] INFO [2019-12-17 12:28:43,467] jepsen worker 63 - jepsen.util 1233 :fail :cas [21 [3 31]] :ValueNotSwapFrom INFO [2019-12-17 12:28:43,467] jepsen worker 63 - jepsen.util 1233 :invoke :read [21 nil] INFO [2019-12-17 12:28:43,472] jepsen worker 21 - jepsen.util 1731 :fail :cas [7 [14 26]] :ValueNotSwapFrom INFO [2019-12-17 12:28:43,472] jepsen worker 21 - jepsen.util 1731 :invoke :cas [7 [19 48]] INFO [2019-12-17 12:28:43,488] jepsen worker 32 - jepsen.util 1472 :info :write [10 34] :RequestCanceledException INFO [2019-12-17 12:28:43,490] jepsen worker 51 - jepsen.util 1401 :ok :read [17 44] INFO [2019-12-17 12:28:43,490] jepsen worker 51 - jepsen.util 1401 :invoke :read [17 nil] INFO [2019-12-17 12:28:43,490] jepsen worker 32 - jepsen.util 1562 :invoke :cas [10 [20 40]] INFO [2019-12-17 12:28:43,494] jepsen worker 45 - jepsen.util 1575 :fail :cas [15 [5 30]] :ValueNotSwapFrom INFO [2019-12-17 12:28:43,494] jepsen worker 45 - jepsen.util 1575 :invoke :read [15 nil] INFO [2019-12-17 12:28:43,506] jepsen worker 53 - jepsen.util 1313 :info :write [17 48] :RequestCanceledException INFO [2019-12-17 12:28:43,507] jepsen worker 53 - jepsen.util 1403 :invoke :write [17 26] INFO [2019-12-17 12:28:43,528] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:28:43,528] jepsen nemesis - couchbase.util Rebalance check count: 51 INFO [2019-12-17 12:28:43,528] jepsen nemesis - couchbase.util Rebalance stuck count: 32 INFO [2019-12-17 12:28:43,546] jepsen worker 47 - jepsen.util 1757 :fail :cas [15 [42 29]] :ValueNotSwapFrom INFO [2019-12-17 12:28:43,547] jepsen worker 47 - jepsen.util 1757 :invoke :read [15 nil] INFO [2019-12-17 12:28:43,551] jepsen worker 64 - jepsen.util 964 :ok :read [21 7] INFO [2019-12-17 12:28:43,551] jepsen worker 64 - jepsen.util 964 :invoke :read [21 nil] INFO [2019-12-17 12:28:43,553] jepsen worker 73 - jepsen.util 1333 :info :write [24 43] :RequestCanceledException INFO [2019-12-17 12:28:43,553] jepsen worker 72 - jepsen.util 1242 :info :write [24 3] :RequestCanceledException INFO [2019-12-17 12:28:43,554] jepsen worker 73 - jepsen.util 1423 :invoke :read [24 nil] INFO [2019-12-17 12:28:43,555] jepsen worker 72 - jepsen.util 1332 :invoke :cas [24 [44 30]] INFO [2019-12-17 12:28:43,573] jepsen worker 89 - jepsen.util 1439 :ok :read [29 32] INFO [2019-12-17 12:28:43,573] jepsen worker 89 - jepsen.util 1439 :invoke :read [29 nil] INFO [2019-12-17 12:28:43,574] jepsen worker 63 - jepsen.util 1233 :ok :read [21 7] INFO [2019-12-17 12:28:43,574] jepsen worker 63 - jepsen.util 1233 :invoke :read [21 nil] INFO [2019-12-17 12:28:43,576] jepsen worker 11 - jepsen.util 1721 :ok :read [3 3] INFO [2019-12-17 12:28:43,576] jepsen worker 11 - jepsen.util 1721 :invoke :cas [3 [16 8]] INFO [2019-12-17 12:28:43,580] jepsen worker 32 - jepsen.util 1562 :fail :cas [10 [20 40]] :ValueNotSwapFrom INFO [2019-12-17 12:28:43,580] jepsen worker 32 - jepsen.util 1562 :invoke :write [10 38] INFO [2019-12-17 12:28:43,590] jepsen worker 71 - jepsen.util 1151 :fail :cas [23 [35 45]] :ValueNotSwapFrom INFO [2019-12-17 12:28:43,590] jepsen worker 71 - jepsen.util 1151 :invoke :read [23 nil] INFO [2019-12-17 12:28:43,597] jepsen worker 28 - jepsen.util 928 :ok :write [9 28] INFO [2019-12-17 12:28:43,598] jepsen worker 28 - jepsen.util 928 :invoke :cas [9 [2 34]] INFO [2019-12-17 12:28:43,602] jepsen worker 75 - jepsen.util 1875 :fail :cas [25 [13 21]] :ValueNotSwapFrom INFO [2019-12-17 12:28:43,602] jepsen worker 75 - jepsen.util 1875 :invoke :write [25 38] INFO [2019-12-17 12:28:43,604] jepsen worker 47 - jepsen.util 1757 :ok :read [15 24] INFO [2019-12-17 12:28:43,604] jepsen worker 47 - jepsen.util 1757 :invoke :write [15 47] INFO [2019-12-17 12:28:43,606] jepsen worker 6 - jepsen.util 1716 :info :cas [2 [14 6]] :RequestCanceledException INFO [2019-12-17 12:28:43,608] jepsen worker 6 - jepsen.util 1806 :invoke :cas [2 [9 39]] INFO [2019-12-17 12:28:43,612] jepsen worker 74 - jepsen.util 1334 :info :write [24 30] :SyncWriteAmbiguous INFO [2019-12-17 12:28:43,616] jepsen worker 74 - jepsen.util 1424 :invoke :write [24 44] INFO [2019-12-17 12:28:43,637] jepsen worker 62 - jepsen.util 1682 :info :write [20 27] :RequestCanceledException INFO [2019-12-17 12:28:43,640] jepsen worker 62 - jepsen.util 1772 :invoke :write [20 25] INFO [2019-12-17 12:28:43,678] jepsen worker 73 - jepsen.util 1423 :ok :read [24 5] INFO [2019-12-17 12:28:43,678] jepsen worker 73 - jepsen.util 1423 :invoke :cas [24 [39 21]] INFO [2019-12-17 12:28:43,681] jepsen worker 63 - jepsen.util 1233 :ok :read [21 7] INFO [2019-12-17 12:28:43,681] jepsen worker 63 - jepsen.util 1233 :invoke :write [21 48] INFO [2019-12-17 12:28:43,684] jepsen worker 31 - jepsen.util 1561 :ok :read [10 30] INFO [2019-12-17 12:28:43,684] jepsen worker 31 - jepsen.util 1561 :invoke :read [10 nil] INFO [2019-12-17 12:28:43,696] jepsen worker 3 - jepsen.util 1263 :ok :read [1 22] INFO [2019-12-17 12:28:43,696] jepsen worker 3 - jepsen.util 1263 :invoke :write [1 22] INFO [2019-12-17 12:28:43,709] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":7820,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:28:43,709] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":7820,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:28:43,709] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":7823,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:28:43,709] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":7823,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:28:43,709] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":7824,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:28:43,709] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":7824,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:28:43,717] jepsen worker 72 - jepsen.util 1332 :fail :cas [24 [44 30]] :ValueNotSwapFrom INFO [2019-12-17 12:28:43,717] jepsen worker 72 - jepsen.util 1332 :invoke :read [24 nil] INFO [2019-12-17 12:28:43,745] jepsen worker 4 - jepsen.util 1174 :ok :read [1 22] INFO [2019-12-17 12:28:43,745] jepsen worker 4 - jepsen.util 1174 :invoke :read [1 nil] INFO [2019-12-17 12:28:43,747] jepsen worker 28 - jepsen.util 928 :fail :cas [9 [2 34]] :ValueNotSwapFrom INFO [2019-12-17 12:28:43,747] jepsen worker 28 - jepsen.util 928 :invoke :read [9 nil] INFO [2019-12-17 12:28:43,753] jepsen worker 25 - jepsen.util 1915 :info :write [8 29] :RequestCanceledException INFO [2019-12-17 12:28:43,753] jepsen worker 77 - jepsen.util 1697 :info :cas [25 [6 27]] :RequestCanceledException INFO [2019-12-17 12:28:43,754] jepsen worker 31 - jepsen.util 1561 :ok :read [10 30] INFO [2019-12-17 12:28:43,755] jepsen worker 31 - jepsen.util 1561 :invoke :cas [10 [13 9]] INFO [2019-12-17 12:28:43,756] jepsen worker 25 - jepsen.util 2005 :invoke :read [8 nil] INFO [2019-12-17 12:28:43,756] jepsen worker 77 - jepsen.util 1787 :invoke :cas [25 [14 21]] INFO [2019-12-17 12:28:43,770] jepsen worker 73 - jepsen.util 1423 :fail :cas [24 [39 21]] :ValueNotSwapFrom INFO [2019-12-17 12:28:43,770] jepsen worker 73 - jepsen.util 1423 :invoke :read [24 nil] INFO [2019-12-17 12:28:43,772] jepsen worker 72 - jepsen.util 1332 :ok :read [24 5] INFO [2019-12-17 12:28:43,772] jepsen worker 72 - jepsen.util 1332 :invoke :write [24 40] INFO [2019-12-17 12:28:43,788] jepsen worker 79 - jepsen.util 1339 :info :write [26 37] :RequestCanceledException INFO [2019-12-17 12:28:43,790] jepsen worker 79 - jepsen.util 1429 :invoke :cas [26 [21 0]] INFO [2019-12-17 12:28:43,797] jepsen worker 21 - jepsen.util 1731 :fail :cas [7 [19 48]] :ValueNotSwapFrom INFO [2019-12-17 12:28:43,797] jepsen worker 21 - jepsen.util 1731 :invoke :write [7 24] INFO [2019-12-17 12:28:43,804] jepsen worker 29 - jepsen.util 1199 :fail :read [9 nil] :RequestTimeoutException INFO [2019-12-17 12:28:43,804] jepsen worker 14 - jepsen.util 1184 :info :write [4 8] :RequestCanceledException INFO [2019-12-17 12:28:43,804] jepsen worker 29 - jepsen.util 1199 :invoke :read [9 nil] INFO [2019-12-17 12:28:43,806] jepsen worker 14 - jepsen.util 1274 :invoke :write [4 20] INFO [2019-12-17 12:28:43,834] jepsen worker 34 - jepsen.util 1924 :info :write [11 5] :RequestCanceledException INFO [2019-12-17 12:28:43,839] jepsen worker 34 - jepsen.util 2014 :invoke :write [11 26] INFO [2019-12-17 12:28:43,847] jepsen worker 28 - jepsen.util 928 :ok :read [9 28] INFO [2019-12-17 12:28:43,847] jepsen worker 28 - jepsen.util 928 :invoke :cas [9 [11 7]] INFO [2019-12-17 12:28:43,850] jepsen worker 36 - jepsen.util 1296 :info :write [12 4] :RequestCanceledException INFO [2019-12-17 12:28:43,855] jepsen worker 36 - jepsen.util 1386 :invoke :cas [12 [39 20]] INFO [2019-12-17 12:28:43,874] jepsen worker 31 - jepsen.util 1561 :fail :cas [10 [13 9]] :ValueNotSwapFrom INFO [2019-12-17 12:28:43,874] jepsen worker 31 - jepsen.util 1561 :invoke :read [10 nil] INFO [2019-12-17 12:28:43,876] jepsen worker 73 - jepsen.util 1423 :ok :read [24 5] INFO [2019-12-17 12:28:43,876] jepsen worker 73 - jepsen.util 1423 :invoke :write [24 5] INFO [2019-12-17 12:28:43,878] jepsen worker 79 - jepsen.util 1429 :fail :cas [26 [21 0]] :ValueNotSwapFrom INFO [2019-12-17 12:28:43,879] jepsen worker 79 - jepsen.util 1429 :invoke :read [26 nil] INFO [2019-12-17 12:28:43,889] jepsen worker 44 - jepsen.util 1574 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:28:43,889] jepsen worker 44 - jepsen.util 1574 :invoke :read [14 nil] INFO [2019-12-17 12:28:43,890] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":7844,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:28:43,890] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":7844,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:28:43,890] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":7849,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:28:43,890] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":7849,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:28:43,893] jepsen worker 28 - jepsen.util 928 :fail :cas [9 [11 7]] :ValueNotSwapFrom INFO [2019-12-17 12:28:43,893] jepsen worker 28 - jepsen.util 928 :invoke :read [9 nil] INFO [2019-12-17 12:28:43,916] jepsen worker 89 - jepsen.util 1439 :ok :read [29 32] INFO [2019-12-17 12:28:43,916] jepsen worker 89 - jepsen.util 1439 :invoke :write [29 25] INFO [2019-12-17 12:28:43,921] jepsen worker 8 - jepsen.util 1718 :info :write [2 49] :RequestCanceledException INFO [2019-12-17 12:28:43,924] jepsen worker 8 - jepsen.util 1808 :invoke :write [2 12] INFO [2019-12-17 12:28:43,927] jepsen worker 9 - jepsen.util 1539 :ok :write [3 4] INFO [2019-12-17 12:28:43,927] jepsen worker 9 - jepsen.util 1539 :invoke :cas [3 [45 12]] INFO [2019-12-17 12:28:43,949] jepsen worker 20 - jepsen.util 1460 :fail :cas [6 [15 28]] :ValueNotSwapFrom INFO [2019-12-17 12:28:43,949] jepsen worker 20 - jepsen.util 1460 :invoke :read [6 nil] INFO [2019-12-17 12:28:43,958] jepsen worker 19 - jepsen.util 1549 :ok :write [6 46] INFO [2019-12-17 12:28:43,959] jepsen worker 19 - jepsen.util 1549 :invoke :cas [6 [7 45]] INFO [2019-12-17 12:28:43,975] jepsen worker 6 - jepsen.util 1806 :fail :cas [2 [9 39]] :ValueNotSwapFrom INFO [2019-12-17 12:28:43,975] jepsen worker 6 - jepsen.util 1806 :invoke :read [2 nil] INFO [2019-12-17 12:28:43,976] jepsen worker 9 - jepsen.util 1539 :fail :cas [3 [45 12]] :ValueNotSwapFrom INFO [2019-12-17 12:28:43,976] jepsen worker 9 - jepsen.util 1539 :invoke :write [3 7] INFO [2019-12-17 12:28:43,986] jepsen worker 20 - jepsen.util 1460 :ok :read [6 46] INFO [2019-12-17 12:28:43,987] jepsen worker 20 - jepsen.util 1460 :invoke :read [6 nil] INFO [2019-12-17 12:28:43,987] jepsen worker 43 - jepsen.util 1843 :info :cas [14 [11 20]] :RequestCanceledException INFO [2019-12-17 12:28:43,989] jepsen worker 43 - jepsen.util 1933 :invoke :read [14 nil] INFO [2019-12-17 12:28:44,002] jepsen worker 19 - jepsen.util 1549 :fail :cas [6 [7 45]] :ValueNotSwapFrom INFO [2019-12-17 12:28:44,002] jepsen worker 19 - jepsen.util 1549 :invoke :read [6 nil] INFO [2019-12-17 12:28:44,004] jepsen worker 54 - jepsen.util 1584 :fail :cas [18 [7 16]] :ValueNotSwapFrom INFO [2019-12-17 12:28:44,004] jepsen worker 54 - jepsen.util 1584 :invoke :read [18 nil] INFO [2019-12-17 12:28:44,025] jepsen worker 56 - jepsen.util 1406 :ok :read [18 26] INFO [2019-12-17 12:28:44,025] jepsen worker 56 - jepsen.util 1406 :invoke :write [18 27] INFO [2019-12-17 12:28:44,025] jepsen worker 19 - jepsen.util 1549 :ok :read [6 46] INFO [2019-12-17 12:28:44,026] jepsen worker 19 - jepsen.util 1549 :invoke :cas [6 [12 0]] INFO [2019-12-17 12:28:44,041] jepsen worker 19 - jepsen.util 1549 :fail :cas [6 [12 0]] :ValueNotSwapFrom INFO [2019-12-17 12:28:44,041] jepsen worker 19 - jepsen.util 1549 :invoke :cas [6 [1 29]] INFO [2019-12-17 12:28:44,060] jepsen worker 19 - jepsen.util 1549 :fail :cas [6 [1 29]] :ValueNotSwapFrom INFO [2019-12-17 12:28:44,061] jepsen worker 19 - jepsen.util 1549 :invoke :cas [6 [35 27]] INFO [2019-12-17 12:28:44,078] jepsen worker 19 - jepsen.util 1549 :fail :cas [6 [35 27]] :ValueNotSwapFrom INFO [2019-12-17 12:28:44,079] jepsen worker 19 - jepsen.util 1549 :invoke :cas [6 [37 44]] INFO [2019-12-17 12:28:44,090] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":7866,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:28:44,090] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":7866,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:28:44,090] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":7868,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:28:44,090] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":7868,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:28:44,094] jepsen worker 4 - jepsen.util 1174 :ok :read [1 22] INFO [2019-12-17 12:28:44,095] jepsen worker 4 - jepsen.util 1174 :invoke :write [1 41] INFO [2019-12-17 12:28:44,097] jepsen worker 6 - jepsen.util 1806 :ok :read [2 15] INFO [2019-12-17 12:28:44,097] jepsen worker 6 - jepsen.util 1806 :invoke :cas [2 [20 25]] INFO [2019-12-17 12:28:44,099] jepsen worker 19 - jepsen.util 1549 :fail :cas [6 [37 44]] :ValueNotSwapFrom INFO [2019-12-17 12:28:44,099] jepsen worker 19 - jepsen.util 1549 :invoke :write [6 10] INFO [2019-12-17 12:28:44,136] jepsen worker 88 - jepsen.util 1168 :info :write [29 24] :RequestCanceledException INFO [2019-12-17 12:28:44,138] jepsen worker 88 - jepsen.util 1258 :invoke :cas [29 [41 5]] INFO [2019-12-17 12:28:44,148] jepsen worker 51 - jepsen.util 1401 :ok :read [17 44] INFO [2019-12-17 12:28:44,148] jepsen worker 51 - jepsen.util 1401 :invoke :write [17 44] INFO [2019-12-17 12:28:44,166] jepsen worker 25 - jepsen.util 2005 :ok :read [8 14] INFO [2019-12-17 12:28:44,166] jepsen worker 25 - jepsen.util 2005 :invoke :cas [8 [8 44]] INFO [2019-12-17 12:28:44,168] jepsen worker 36 - jepsen.util 1386 :fail :cas [12 [39 20]] :ValueNotSwapFrom INFO [2019-12-17 12:28:44,168] jepsen worker 36 - jepsen.util 1386 :invoke :cas [12 [7 40]] INFO [2019-12-17 12:28:44,186] jepsen worker 71 - jepsen.util 1151 :ok :read [23 8] INFO [2019-12-17 12:28:44,186] jepsen worker 71 - jepsen.util 1151 :invoke :cas [23 [23 34]] INFO [2019-12-17 12:28:44,214] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":7886,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:28:44,219] jepsen worker 55 - jepsen.util 1765 :ok :read [18 26] INFO [2019-12-17 12:28:44,222] jepsen worker 26 - jepsen.util 1736 :info :write [8 48] :RequestCanceledException INFO [2019-12-17 12:28:44,264] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":7886,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:28:44,264] jepsen worker 55 - jepsen.util 1765 :invoke :cas [18 [49 24]] INFO [2019-12-17 12:28:44,264] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":7888,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:28:44,264] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":7888,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:28:44,266] jepsen worker 26 - jepsen.util 1826 :invoke :read [8 nil] INFO [2019-12-17 12:28:44,269] jepsen worker 6 - jepsen.util 1806 :fail :cas [2 [20 25]] :ValueNotSwapFrom INFO [2019-12-17 12:28:44,270] jepsen worker 6 - jepsen.util 1806 :invoke :cas [2 [15 11]] INFO [2019-12-17 12:28:44,274] jepsen worker 54 - jepsen.util 1584 :ok :read [18 26] INFO [2019-12-17 12:28:44,274] jepsen worker 54 - jepsen.util 1584 :invoke :read [18 nil] INFO [2019-12-17 12:28:44,291] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":7893,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:28:44,291] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":7893,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:28:44,291] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":7897,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:28:44,291] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":7897,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:28:44,306] jepsen worker 26 - jepsen.util 1826 :ok :read [8 14] INFO [2019-12-17 12:28:44,307] jepsen worker 26 - jepsen.util 1826 :invoke :write [8 5] INFO [2019-12-17 12:28:44,322] jepsen worker 80 - jepsen.util 1430 :info :write [26 28] :RequestCanceledException INFO [2019-12-17 12:28:44,323] jepsen worker 80 - jepsen.util 1520 :invoke :cas [26 [6 43]] INFO [2019-12-17 12:28:44,335] jepsen worker 79 - jepsen.util 1429 :ok :read [26 27] INFO [2019-12-17 12:28:44,335] jepsen worker 79 - jepsen.util 1429 :invoke :cas [26 [17 39]] INFO [2019-12-17 12:28:44,352] jepsen worker 25 - jepsen.util 2005 :fail :cas [8 [8 44]] :ValueNotSwapFrom INFO [2019-12-17 12:28:44,352] jepsen worker 25 - jepsen.util 2005 :invoke :write [8 3] INFO [2019-12-17 12:28:44,367] jepsen worker 80 - jepsen.util 1520 :fail :cas [26 [6 43]] :ValueNotSwapFrom INFO [2019-12-17 12:28:44,367] jepsen worker 80 - jepsen.util 1520 :invoke :write [26 24] INFO [2019-12-17 12:28:44,404] jepsen worker 5 - jepsen.util 995 :info :write [1 16] :RequestCanceledException INFO [2019-12-17 12:28:44,405] jepsen worker 5 - jepsen.util 1085 :invoke :cas [1 [10 0]] INFO [2019-12-17 12:28:44,441] jepsen worker 71 - jepsen.util 1151 :fail :cas [23 [23 34]] :ValueNotSwapFrom INFO [2019-12-17 12:28:44,441] jepsen worker 71 - jepsen.util 1151 :invoke :cas [23 [4 35]] INFO [2019-12-17 12:28:44,521] jepsen worker 30 - jepsen.util 1470 :info :write [10 8] :RequestCanceledException INFO [2019-12-17 12:28:44,521] jepsen worker 27 - jepsen.util 1197 :info :write [9 43] :RequestCanceledException INFO [2019-12-17 12:28:44,524] jepsen worker 27 - jepsen.util 1287 :invoke :cas [9 [40 3]] INFO [2019-12-17 12:28:44,524] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":7914,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:28:44,524] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":7914,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:28:44,525] jepsen worker 30 - jepsen.util 1560 :invoke :read [10 nil] INFO [2019-12-17 12:28:44,542] jepsen worker 30 - jepsen.util 1560 :ok :read [10 30] INFO [2019-12-17 12:28:44,542] jepsen worker 30 - jepsen.util 1560 :invoke :write [10 5] INFO [2019-12-17 12:28:44,543] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:28:44,543] jepsen nemesis - couchbase.util Rebalance check count: 52 INFO [2019-12-17 12:28:44,543] jepsen nemesis - couchbase.util Rebalance stuck count: 33 INFO [2019-12-17 12:28:44,563] jepsen worker 55 - jepsen.util 1765 :fail :cas [18 [49 24]] :ValueNotSwapFrom INFO [2019-12-17 12:28:44,563] jepsen worker 55 - jepsen.util 1765 :invoke :read [18 nil] INFO [2019-12-17 12:28:44,584] jepsen worker 88 - jepsen.util 1258 :fail :cas [29 [41 5]] :ValueNotSwapFrom INFO [2019-12-17 12:28:44,585] jepsen worker 88 - jepsen.util 1258 :invoke :write [29 24] INFO [2019-12-17 12:28:44,606] jepsen worker 76 - jepsen.util 1696 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:28:44,606] jepsen worker 76 - jepsen.util 1696 :invoke :cas [25 [36 30]] INFO [2019-12-17 12:28:44,619] jepsen worker 54 - jepsen.util 1584 :ok :read [18 26] INFO [2019-12-17 12:28:44,619] jepsen worker 54 - jepsen.util 1584 :invoke :cas [18 [23 22]] INFO [2019-12-17 12:28:44,622] jepsen worker 82 - jepsen.util 2152 :info :cas [27 [30 31]] :RequestCanceledException INFO [2019-12-17 12:28:44,622] jepsen worker 0 - jepsen.util 1800 :info :cas [0 [21 12]] :RequestCanceledException INFO [2019-12-17 12:28:44,623] jepsen worker 0 - jepsen.util 1890 :invoke :cas [0 [25 33]] INFO [2019-12-17 12:28:44,623] jepsen worker 82 - jepsen.util 2242 :invoke :read [27 nil] INFO [2019-12-17 12:28:44,636] jepsen worker 79 - jepsen.util 1429 :fail :cas [26 [17 39]] :ValueNotSwapFrom INFO [2019-12-17 12:28:44,636] jepsen worker 7 - jepsen.util 1987 :info :write [2 34] :RequestCanceledException INFO [2019-12-17 12:28:44,636] jepsen worker 79 - jepsen.util 1429 :invoke :cas [26 [5 11]] INFO [2019-12-17 12:28:44,637] jepsen worker 7 - jepsen.util 2077 :invoke :cas [2 [14 20]] INFO [2019-12-17 12:28:44,652] jepsen worker 13 - jepsen.util 1273 :info :write [4 34] :RequestCanceledException INFO [2019-12-17 12:28:44,652] jepsen worker 85 - jepsen.util 1795 :info :cas [28 [33 28]] :RequestCanceledException INFO [2019-12-17 12:28:44,653] jepsen worker 13 - jepsen.util 1363 :invoke :cas [4 [27 41]] INFO [2019-12-17 12:28:44,653] jepsen worker 85 - jepsen.util 1885 :invoke :cas [28 [21 34]] INFO [2019-12-17 12:28:44,676] jepsen worker 9 - jepsen.util 1539 :ok :write [3 7] INFO [2019-12-17 12:28:44,676] jepsen worker 9 - jepsen.util 1539 :invoke :cas [3 [47 38]] INFO [2019-12-17 12:28:44,678] jepsen worker 55 - jepsen.util 1765 :ok :read [18 26] INFO [2019-12-17 12:28:44,679] jepsen worker 55 - jepsen.util 1765 :invoke :cas [18 [15 13]] INFO [2019-12-17 12:28:44,682] jepsen worker 36 - jepsen.util 1386 :fail :cas [12 [7 40]] :ValueNotSwapFrom INFO [2019-12-17 12:28:44,682] jepsen worker 36 - jepsen.util 1386 :invoke :write [12 29] INFO [2019-12-17 12:28:44,684] jepsen worker 19 - jepsen.util 1549 :ok :write [6 10] INFO [2019-12-17 12:28:44,684] jepsen worker 19 - jepsen.util 1549 :invoke :read [6 nil] INFO [2019-12-17 12:28:44,689] jepsen worker 20 - jepsen.util 1460 :ok :read [6 46] INFO [2019-12-17 12:28:44,689] jepsen worker 20 - jepsen.util 1460 :invoke :write [6 21] INFO [2019-12-17 12:28:44,693] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":7927,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:28:44,693] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":7927,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:28:44,693] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":7931,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:28:44,693] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":7931,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:28:44,693] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":7932,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:28:44,693] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":7932,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:28:44,701] jepsen worker 27 - jepsen.util 1287 :fail :cas [9 [40 3]] :ValueNotSwapFrom INFO [2019-12-17 12:28:44,702] jepsen worker 27 - jepsen.util 1287 :invoke :read [9 nil] INFO [2019-12-17 12:28:44,706] jepsen worker 70 - jepsen.util 1600 :info :write [23 18] :RequestCanceledException INFO [2019-12-17 12:28:44,707] jepsen worker 70 - jepsen.util 1690 :invoke :cas [23 [10 6]] INFO [2019-12-17 12:28:44,710] jepsen worker 9 - jepsen.util 1539 :fail :cas [3 [47 38]] :ValueNotSwapFrom INFO [2019-12-17 12:28:44,710] jepsen worker 9 - jepsen.util 1539 :invoke :read [3 nil] INFO [2019-12-17 12:28:44,714] jepsen worker 32 - jepsen.util 1562 :ok :write [10 38] INFO [2019-12-17 12:28:44,714] jepsen worker 32 - jepsen.util 1562 :invoke :read [10 nil] INFO [2019-12-17 12:28:44,719] jepsen worker 52 - jepsen.util 1492 :info :write [17 42] :RequestCanceledException INFO [2019-12-17 12:28:44,719] jepsen worker 42 - jepsen.util 1932 :info :write [14 29] :RequestCanceledException INFO [2019-12-17 12:28:44,720] jepsen worker 19 - jepsen.util 1549 :ok :read [6 10] INFO [2019-12-17 12:28:44,720] jepsen worker 19 - jepsen.util 1549 :invoke :read [6 nil] INFO [2019-12-17 12:28:44,722] jepsen worker 52 - jepsen.util 1582 :invoke :write [17 21] INFO [2019-12-17 12:28:44,725] jepsen worker 46 - jepsen.util 1666 :ok :write [15 44] INFO [2019-12-17 12:28:44,725] jepsen worker 72 - jepsen.util 1332 :ok :write [24 40] INFO [2019-12-17 12:28:44,725] jepsen worker 46 - jepsen.util 1666 :invoke :cas [15 [26 46]] INFO [2019-12-17 12:28:44,726] jepsen worker 72 - jepsen.util 1332 :invoke :cas [24 [7 20]] INFO [2019-12-17 12:28:44,726] jepsen worker 42 - jepsen.util 2022 :invoke :write [14 33] INFO [2019-12-17 12:28:44,760] jepsen worker 64 - jepsen.util 964 :ok :read [21 7] INFO [2019-12-17 12:28:44,760] jepsen worker 64 - jepsen.util 964 :invoke :read [21 nil] INFO [2019-12-17 12:28:44,784] jepsen worker 86 - jepsen.util 1796 :info :write [28 29] :RequestCanceledException INFO [2019-12-17 12:28:44,784] jepsen worker 24 - jepsen.util 1554 :info :write [8 47] :RequestCanceledException INFO [2019-12-17 12:28:44,786] jepsen worker 86 - jepsen.util 1886 :invoke :write [28 9] INFO [2019-12-17 12:28:44,786] jepsen worker 24 - jepsen.util 1644 :invoke :cas [8 [48 29]] INFO [2019-12-17 12:28:44,792] jepsen worker 72 - jepsen.util 1332 :fail :cas [24 [7 20]] :ValueNotSwapFrom INFO [2019-12-17 12:28:44,793] jepsen worker 72 - jepsen.util 1332 :invoke :read [24 nil] INFO [2019-12-17 12:28:44,797] jepsen worker 54 - jepsen.util 1584 :fail :cas [18 [23 22]] :ValueNotSwapFrom INFO [2019-12-17 12:28:44,798] jepsen worker 54 - jepsen.util 1584 :invoke :read [18 nil] INFO [2019-12-17 12:28:44,803] jepsen worker 19 - jepsen.util 1549 :ok :read [6 10] INFO [2019-12-17 12:28:44,803] jepsen worker 19 - jepsen.util 1549 :invoke :read [6 nil] INFO [2019-12-17 12:28:44,807] jepsen worker 41 - jepsen.util 1571 :info :cas [13 [45 10]] :RequestCanceledException INFO [2019-12-17 12:28:44,807] jepsen worker 48 - jepsen.util 1218 :info :write [16 1] :RequestCanceledException INFO [2019-12-17 12:28:44,807] jepsen worker 38 - jepsen.util 1298 :info :write [12 26] :RequestCanceledException INFO [2019-12-17 12:28:44,809] jepsen worker 38 - jepsen.util 1388 :invoke :write [12 40] INFO [2019-12-17 12:28:44,809] jepsen worker 48 - jepsen.util 1308 :invoke :read [16 nil] INFO [2019-12-17 12:28:44,810] jepsen worker 41 - jepsen.util 1661 :invoke :cas [13 [47 38]] INFO [2019-12-17 12:28:44,819] jepsen worker 57 - jepsen.util 1587 :info :write [19 20] :RequestCanceledException INFO [2019-12-17 12:28:44,822] jepsen worker 57 - jepsen.util 1677 :invoke :read [19 nil] INFO [2019-12-17 12:28:44,832] jepsen worker 60 - jepsen.util 1860 :info :write [20 22] :RequestCanceledException INFO [2019-12-17 12:28:44,832] jepsen worker 16 - jepsen.util 1636 :info :write [5 40] :RequestCanceledException INFO [2019-12-17 12:28:44,832] jepsen worker 59 - jepsen.util 1499 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:28:44,832] jepsen worker 59 - jepsen.util 1499 :invoke :read [19 nil] INFO [2019-12-17 12:28:44,834] jepsen worker 16 - jepsen.util 1726 :invoke :write [5 5] INFO [2019-12-17 12:28:44,835] jepsen worker 60 - jepsen.util 1950 :invoke :read [20 nil] INFO [2019-12-17 12:28:44,848] jepsen worker 50 - jepsen.util 1760 :info :cas [16 [30 1]] :RequestCanceledException INFO [2019-12-17 12:28:44,848] jepsen worker 66 - jepsen.util 1416 :info :cas [22 [9 37]] :RequestCanceledException INFO [2019-12-17 12:28:44,852] jepsen worker 66 - jepsen.util 1506 :invoke :write [22 34] INFO [2019-12-17 12:28:44,854] jepsen worker 50 - jepsen.util 1850 :invoke :read [16 nil] INFO [2019-12-17 12:28:44,861] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":7966,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:28:44,862] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":7966,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:28:44,880] jepsen worker 67 - jepsen.util 2047 :info :write [22 25] :RequestCanceledException INFO [2019-12-17 12:28:44,880] jepsen worker 1 - jepsen.util 1711 :info :cas [0 [21 11]] :RequestCanceledException INFO [2019-12-17 12:28:44,884] jepsen worker 1 - jepsen.util 1801 :invoke :read [0 nil] INFO [2019-12-17 12:28:44,886] jepsen worker 67 - jepsen.util 2137 :invoke :write [22 2] INFO [2019-12-17 12:28:44,901] jepsen worker 23 - jepsen.util 1553 :info :cas [7 [31 5]] :RequestCanceledException INFO [2019-12-17 12:28:44,901] jepsen worker 84 - jepsen.util 1614 :info :write [28 37] :RequestCanceledException INFO [2019-12-17 12:28:44,901] jepsen worker 15 - jepsen.util 1455 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:28:44,902] jepsen worker 23 - jepsen.util 1643 :invoke :write [7 18] INFO [2019-12-17 12:28:44,902] jepsen worker 15 - jepsen.util 1455 :invoke :write [5 29] INFO [2019-12-17 12:28:44,903] jepsen worker 54 - jepsen.util 1584 :ok :read [18 26] INFO [2019-12-17 12:28:44,904] jepsen worker 54 - jepsen.util 1584 :invoke :write [18 27] INFO [2019-12-17 12:28:44,906] jepsen worker 84 - jepsen.util 1704 :invoke :write [28 1] INFO [2019-12-17 12:28:44,914] jepsen worker 17 - jepsen.util 1727 :info :cas [5 [8 6]] :RequestCanceledException INFO [2019-12-17 12:28:44,915] jepsen worker 18 - jepsen.util 1548 :info :write [6 46] :RequestCanceledException INFO [2019-12-17 12:28:44,915] jepsen worker 12 - jepsen.util 1452 :info :write [4 3] :RequestCanceledException INFO [2019-12-17 12:28:44,917] jepsen worker 65 - jepsen.util 1055 :info :cas [21 [49 7]] :RequestCanceledException INFO [2019-12-17 12:28:44,917] jepsen worker 83 - jepsen.util 2063 :info :cas [27 [38 3]] :RequestCanceledException INFO [2019-12-17 12:28:44,917] jepsen worker 40 - jepsen.util 1930 :info :cas [13 [12 11]] :RequestCanceledException INFO [2019-12-17 12:28:44,921] jepsen worker 40 - jepsen.util 2020 :invoke :cas [13 [29 31]] INFO [2019-12-17 12:28:44,924] jepsen worker 18 - jepsen.util 1638 :invoke :write [6 34] INFO [2019-12-17 12:28:44,927] jepsen worker 17 - jepsen.util 1817 :invoke :write [5 41] INFO [2019-12-17 12:28:44,928] jepsen worker 83 - jepsen.util 2153 :invoke :read [27 nil] INFO [2019-12-17 12:28:44,928] jepsen worker 65 - jepsen.util 1145 :invoke :cas [21 [6 0]] INFO [2019-12-17 12:28:44,932] jepsen worker 12 - jepsen.util 1542 :invoke :read [4 nil] INFO [2019-12-17 12:28:44,933] jepsen worker 2 - jepsen.util 1352 :info :write [0 43] :RequestCanceledException INFO [2019-12-17 12:28:44,940] jepsen worker 2 - jepsen.util 1442 :invoke :cas [0 [44 16]] INFO [2019-12-17 12:28:44,946] jepsen worker 11 - jepsen.util 1721 :fail :cas [3 [16 8]] :ValueNotSwapFrom INFO [2019-12-17 12:28:44,946] jepsen worker 11 - jepsen.util 1721 :invoke :read [3 nil] INFO [2019-12-17 12:28:44,951] jepsen worker 49 - jepsen.util 1309 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:28:44,951] jepsen worker 68 - jepsen.util 1328 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:28:44,952] jepsen worker 49 - jepsen.util 1309 :invoke :read [16 nil] INFO [2019-12-17 12:28:44,952] jepsen worker 68 - jepsen.util 1328 :invoke :cas [22 [7 13]] INFO [2019-12-17 12:28:44,966] jepsen worker 19 - jepsen.util 1549 :ok :read [6 10] INFO [2019-12-17 12:28:44,966] jepsen worker 19 - jepsen.util 1549 :invoke :read [6 nil] INFO [2019-12-17 12:28:44,968] jepsen worker 63 - jepsen.util 1233 :ok :write [21 48] INFO [2019-12-17 12:28:44,969] jepsen worker 63 - jepsen.util 1233 :invoke :cas [21 [5 27]] INFO [2019-12-17 12:28:44,972] jepsen worker 55 - jepsen.util 1765 :fail :cas [18 [15 13]] :ValueNotSwapFrom INFO [2019-12-17 12:28:44,972] jepsen worker 72 - jepsen.util 1332 :ok :read [24 40] INFO [2019-12-17 12:28:44,973] jepsen worker 72 - jepsen.util 1332 :invoke :read [24 nil] INFO [2019-12-17 12:28:44,973] jepsen worker 55 - jepsen.util 1765 :invoke :cas [18 [49 49]] INFO [2019-12-17 12:28:44,979] jepsen worker 39 - jepsen.util 1839 :info :write [13 4] :RequestCanceledException INFO [2019-12-17 12:28:44,980] jepsen worker 39 - jepsen.util 1929 :invoke :cas [13 [43 39]] INFO [2019-12-17 12:28:45,006] jepsen worker 61 - jepsen.util 1771 :info :write [20 0] :RequestCanceledException INFO [2019-12-17 12:28:45,008] jepsen worker 61 - jepsen.util 1861 :invoke :read [20 nil] INFO [2019-12-17 12:28:45,022] jepsen worker 35 - jepsen.util 1475 :info :write [11 14] :RequestCanceledException INFO [2019-12-17 12:28:45,024] jepsen worker 35 - jepsen.util 1565 :invoke :cas [11 [9 32]] INFO [2019-12-17 12:28:45,051] jepsen worker 72 - jepsen.util 1332 :ok :read [24 40] INFO [2019-12-17 12:28:45,051] jepsen worker 72 - jepsen.util 1332 :invoke :cas [24 [34 18]] INFO [2019-12-17 12:28:45,055] jepsen worker 32 - jepsen.util 1562 :ok :read [10 38] INFO [2019-12-17 12:28:45,056] jepsen worker 32 - jepsen.util 1562 :invoke :write [10 2] INFO [2019-12-17 12:28:45,058] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":7990,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:28:45,058] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":7990,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:28:45,058] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":7992,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:28:45,058] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":7992,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:28:45,058] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":7995,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:28:45,058] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":7995,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:28:45,068] jepsen worker 19 - jepsen.util 1549 :ok :read [6 10] INFO [2019-12-17 12:28:45,068] jepsen worker 19 - jepsen.util 1549 :invoke :write [6 13] INFO [2019-12-17 12:28:45,080] jepsen worker 63 - jepsen.util 1233 :fail :cas [21 [5 27]] :ValueNotSwapFrom INFO [2019-12-17 12:28:45,081] jepsen worker 63 - jepsen.util 1233 :invoke :cas [21 [15 35]] INFO [2019-12-17 12:28:45,084] jepsen worker 72 - jepsen.util 1332 :fail :cas [24 [34 18]] :ValueNotSwapFrom INFO [2019-12-17 12:28:45,084] jepsen worker 72 - jepsen.util 1332 :invoke :write [24 30] INFO [2019-12-17 12:28:45,088] jepsen worker 81 - jepsen.util 1521 :info :cas [27 [35 15]] :RequestCanceledException INFO [2019-12-17 12:28:45,088] jepsen worker 69 - jepsen.util 1149 :info :cas [23 [27 25]] :RequestCanceledException INFO [2019-12-17 12:28:45,088] jepsen worker 37 - jepsen.util 1207 :info :write [12 31] :RequestCanceledException INFO [2019-12-17 12:28:45,090] jepsen worker 81 - jepsen.util 1611 :invoke :cas [27 [38 24]] INFO [2019-12-17 12:28:45,091] jepsen worker 37 - jepsen.util 1297 :invoke :write [12 6] INFO [2019-12-17 12:28:45,091] jepsen worker 69 - jepsen.util 1239 :invoke :write [23 11] INFO [2019-12-17 12:28:45,192] jepsen worker 5 - jepsen.util 1085 :fail :cas [1 [10 0]] :ValueNotSwapFrom INFO [2019-12-17 12:28:45,192] jepsen worker 5 - jepsen.util 1085 :invoke :cas [1 [11 21]] INFO [2019-12-17 12:28:45,207] jepsen worker 58 - jepsen.util 1498 :info :cas [19 [6 0]] :RequestCanceledException INFO [2019-12-17 12:28:45,208] jepsen worker 58 - jepsen.util 1588 :invoke :read [19 nil] INFO [2019-12-17 12:28:45,222] jepsen worker 22 - jepsen.util 1732 :info :write [7 31] :RequestCanceledException INFO [2019-12-17 12:28:45,224] jepsen worker 22 - jepsen.util 1822 :invoke :write [7 30] INFO [2019-12-17 12:28:45,230] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8008,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:28:45,230] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8008,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:28:45,230] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8009,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:28:45,230] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8009,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:28:45,236] jepsen worker 87 - jepsen.util 1257 :info :write [29 37] :RequestCanceledException INFO [2019-12-17 12:28:45,237] jepsen worker 79 - jepsen.util 1429 :fail :cas [26 [5 11]] :ValueNotSwapFrom INFO [2019-12-17 12:28:45,237] jepsen worker 79 - jepsen.util 1429 :invoke :write [26 44] INFO [2019-12-17 12:28:45,237] jepsen worker 87 - jepsen.util 1347 :invoke :cas [29 [43 19]] INFO [2019-12-17 12:28:45,268] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8015,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:28:45,268] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8015,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:28:45,269] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8018,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:28:45,269] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8018,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:28:45,277] jepsen worker 35 - jepsen.util 1565 :fail :cas [11 [9 32]] :ValueNotSwapFrom INFO [2019-12-17 12:28:45,277] jepsen worker 35 - jepsen.util 1565 :invoke :write [11 11] INFO [2019-12-17 12:28:45,322] jepsen worker 78 - jepsen.util 1428 :info :cas [26 [27 32]] :RequestCanceledException INFO [2019-12-17 12:28:45,324] jepsen worker 78 - jepsen.util 1518 :invoke :read [26 nil] INFO [2019-12-17 12:28:45,327] jepsen worker 70 - jepsen.util 1690 :fail :cas [23 [10 6]] :ValueNotSwapFrom INFO [2019-12-17 12:28:45,327] jepsen worker 70 - jepsen.util 1690 :invoke :write [23 20] INFO [2019-12-17 12:28:45,333] jepsen worker 13 - jepsen.util 1363 :fail :cas [4 [27 41]] :ValueNotSwapFrom INFO [2019-12-17 12:28:45,333] jepsen worker 13 - jepsen.util 1363 :invoke :cas [4 [29 34]] INFO [2019-12-17 12:28:45,357] jepsen worker 71 - jepsen.util 1151 :fail :cas [23 [4 35]] :ValueNotSwapFrom INFO [2019-12-17 12:28:45,357] jepsen worker 71 - jepsen.util 1151 :invoke :read [23 nil] INFO [2019-12-17 12:28:45,399] jepsen worker 78 - jepsen.util 1518 :ok :read [26 27] INFO [2019-12-17 12:28:45,400] jepsen worker 78 - jepsen.util 1518 :invoke :write [26 24] INFO [2019-12-17 12:28:45,436] jepsen worker 10 - jepsen.util 1360 :info :write [3 15] :RequestCanceledException INFO [2019-12-17 12:28:45,438] jepsen worker 10 - jepsen.util 1450 :invoke :write [3 24] INFO [2019-12-17 12:28:45,471] jepsen worker 55 - jepsen.util 1765 :fail :cas [18 [49 49]] :ValueNotSwapFrom INFO [2019-12-17 12:28:45,471] jepsen worker 55 - jepsen.util 1765 :invoke :read [18 nil] INFO [2019-12-17 12:28:45,480] jepsen worker 13 - jepsen.util 1363 :fail :cas [4 [29 34]] :ValueNotSwapFrom INFO [2019-12-17 12:28:45,480] jepsen worker 13 - jepsen.util 1363 :invoke :read [4 nil] INFO [2019-12-17 12:28:45,492] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8033,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:28:45,492] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8033,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:28:45,493] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8035,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:28:45,493] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8035,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:28:45,522] jepsen worker 33 - jepsen.util 2013 :info :write [11 37] :RequestCanceledException INFO [2019-12-17 12:28:45,522] jepsen worker 53 - jepsen.util 1403 :info :write [17 26] :RequestCanceledException INFO [2019-12-17 12:28:45,523] jepsen worker 33 - jepsen.util 2103 :invoke :cas [11 [46 33]] INFO [2019-12-17 12:28:45,524] jepsen worker 53 - jepsen.util 1493 :invoke :read [17 nil] INFO [2019-12-17 12:28:45,534] jepsen worker 45 - jepsen.util 1575 :fail :read [15 nil] :RequestTimeoutException INFO [2019-12-17 12:28:45,534] jepsen worker 45 - jepsen.util 1575 :invoke :read [15 nil] INFO [2019-12-17 12:28:45,546] jepsen worker 12 - jepsen.util 1542 :ok :read [4 26] INFO [2019-12-17 12:28:45,547] jepsen worker 12 - jepsen.util 1542 :invoke :write [4 46] INFO [2019-12-17 12:28:45,555] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:28:45,556] jepsen nemesis - couchbase.util Rebalance check count: 53 INFO [2019-12-17 12:28:45,556] jepsen nemesis - couchbase.util Rebalance stuck count: 34 INFO [2019-12-17 12:28:45,591] jepsen worker 71 - jepsen.util 1151 :ok :read [23 8] INFO [2019-12-17 12:28:45,591] jepsen worker 71 - jepsen.util 1151 :invoke :cas [23 [8 32]] INFO [2019-12-17 12:28:45,612] jepsen worker 74 - jepsen.util 1424 :ok :write [24 44] INFO [2019-12-17 12:28:45,612] jepsen worker 20 - jepsen.util 1460 :ok :write [6 21] INFO [2019-12-17 12:28:45,612] jepsen worker 20 - jepsen.util 1460 :invoke :write [6 8] INFO [2019-12-17 12:28:45,612] jepsen worker 74 - jepsen.util 1424 :invoke :write [24 49] INFO [2019-12-17 12:28:45,615] jepsen worker 32 - jepsen.util 1562 :ok :write [10 2] INFO [2019-12-17 12:28:45,615] jepsen worker 32 - jepsen.util 1562 :invoke :write [10 23] INFO [2019-12-17 12:28:45,621] jepsen worker 75 - jepsen.util 1875 :info :write [25 38] :RequestCanceledException INFO [2019-12-17 12:28:45,624] jepsen worker 75 - jepsen.util 1965 :invoke :write [25 22] INFO [2019-12-17 12:28:45,624] jepsen worker 33 - jepsen.util 2103 :fail :cas [11 [46 33]] :ValueNotSwapFrom INFO [2019-12-17 12:28:45,624] jepsen worker 33 - jepsen.util 2103 :invoke :read [11 nil] INFO [2019-12-17 12:28:45,647] jepsen worker 46 - jepsen.util 1666 :fail :cas [15 [26 46]] :ValueNotSwapFrom INFO [2019-12-17 12:28:45,647] jepsen worker 46 - jepsen.util 1666 :invoke :write [15 49] INFO [2019-12-17 12:28:45,653] jepsen worker 47 - jepsen.util 1757 :info :write [15 47] :RequestCanceledException INFO [2019-12-17 12:28:45,655] jepsen worker 47 - jepsen.util 1847 :invoke :write [15 4] INFO [2019-12-17 12:28:45,669] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8050,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:28:45,669] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8050,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:28:45,669] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8054,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:28:45,670] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8054,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:28:45,676] jepsen worker 64 - jepsen.util 964 :ok :read [21 48] INFO [2019-12-17 12:28:45,676] jepsen worker 64 - jepsen.util 964 :invoke :cas [21 [25 25]] INFO [2019-12-17 12:28:45,686] jepsen worker 65 - jepsen.util 1145 :fail :cas [21 [6 0]] :ValueNotSwapFrom INFO [2019-12-17 12:28:45,686] jepsen worker 5 - jepsen.util 1085 :fail :cas [1 [11 21]] :ValueNotSwapFrom INFO [2019-12-17 12:28:45,686] jepsen worker 65 - jepsen.util 1145 :invoke :write [21 9] INFO [2019-12-17 12:28:45,686] jepsen worker 5 - jepsen.util 1085 :invoke :write [1 26] INFO [2019-12-17 12:28:45,692] jepsen worker 33 - jepsen.util 2103 :ok :read [11 7] INFO [2019-12-17 12:28:45,693] jepsen worker 33 - jepsen.util 2103 :invoke :read [11 nil] INFO [2019-12-17 12:28:45,698] jepsen worker 11 - jepsen.util 1721 :ok :read [3 7] INFO [2019-12-17 12:28:45,698] jepsen worker 11 - jepsen.util 1721 :invoke :read [3 nil] INFO [2019-12-17 12:28:45,704] jepsen worker 45 - jepsen.util 1575 :ok :read [15 44] INFO [2019-12-17 12:28:45,704] jepsen worker 45 - jepsen.util 1575 :invoke :cas [15 [34 9]] INFO [2019-12-17 12:28:45,726] jepsen worker 63 - jepsen.util 1233 :fail :cas [21 [15 35]] :ValueNotSwapFrom INFO [2019-12-17 12:28:45,727] jepsen worker 63 - jepsen.util 1233 :invoke :write [21 8] INFO [2019-12-17 12:28:45,729] jepsen worker 33 - jepsen.util 2103 :ok :read [11 7] INFO [2019-12-17 12:28:45,729] jepsen worker 33 - jepsen.util 2103 :invoke :write [11 43] INFO [2019-12-17 12:28:45,753] jepsen worker 62 - jepsen.util 1772 :info :write [20 25] :RequestCanceledException INFO [2019-12-17 12:28:45,754] jepsen worker 62 - jepsen.util 1862 :invoke :write [20 27] INFO [2019-12-17 12:28:45,777] jepsen worker 20 - jepsen.util 1460 :ok :write [6 8] INFO [2019-12-17 12:28:45,778] jepsen worker 20 - jepsen.util 1460 :invoke :read [6 nil] INFO [2019-12-17 12:28:45,788] jepsen worker 3 - jepsen.util 1263 :info :write [1 22] :RequestCanceledException INFO [2019-12-17 12:28:45,788] jepsen worker 77 - jepsen.util 1787 :info :cas [25 [14 21]] :RequestCanceledException INFO [2019-12-17 12:28:45,790] jepsen worker 3 - jepsen.util 1353 :invoke :cas [1 [26 9]] INFO [2019-12-17 12:28:45,791] jepsen worker 77 - jepsen.util 1877 :invoke :write [25 42] INFO [2019-12-17 12:28:45,820] jepsen worker 10 - jepsen.util 1450 :ok :write [3 24] INFO [2019-12-17 12:28:45,820] jepsen worker 10 - jepsen.util 1450 :invoke :write [3 26] INFO [2019-12-17 12:28:45,822] jepsen worker 21 - jepsen.util 1731 :info :write [7 24] :RequestCanceledException INFO [2019-12-17 12:28:45,829] jepsen worker 21 - jepsen.util 1821 :invoke :read [7 nil] INFO [2019-12-17 12:28:45,833] jepsen worker 20 - jepsen.util 1460 :ok :read [6 8] INFO [2019-12-17 12:28:45,833] jepsen worker 20 - jepsen.util 1460 :invoke :cas [6 [19 21]] INFO [2019-12-17 12:28:45,854] jepsen worker 20 - jepsen.util 1460 :fail :cas [6 [19 21]] :ValueNotSwapFrom INFO [2019-12-17 12:28:45,855] jepsen worker 20 - jepsen.util 1460 :invoke :write [6 41] INFO [2019-12-17 12:28:45,906] jepsen worker 29 - jepsen.util 1199 :fail :read [9 nil] :RequestTimeoutException INFO [2019-12-17 12:28:45,906] jepsen worker 31 - jepsen.util 1561 :fail :read [10 nil] :RequestTimeoutException INFO [2019-12-17 12:28:45,906] jepsen worker 29 - jepsen.util 1199 :invoke :read [9 nil] INFO [2019-12-17 12:28:45,906] jepsen worker 31 - jepsen.util 1561 :invoke :cas [10 [29 15]] INFO [2019-12-17 12:28:45,910] jepsen worker 87 - jepsen.util 1347 :fail :cas [29 [43 19]] :ValueNotSwapFrom INFO [2019-12-17 12:28:45,910] jepsen worker 87 - jepsen.util 1347 :invoke :read [29 nil] INFO [2019-12-17 12:28:45,921] jepsen worker 14 - jepsen.util 1274 :info :write [4 20] :RequestCanceledException INFO [2019-12-17 12:28:45,928] jepsen worker 14 - jepsen.util 1364 :invoke :write [4 36] INFO [2019-12-17 12:28:45,935] jepsen worker 34 - jepsen.util 2014 :info :write [11 26] :RequestCanceledException INFO [2019-12-17 12:28:45,941] jepsen worker 34 - jepsen.util 2104 :invoke :cas [11 [3 39]] INFO [2019-12-17 12:28:45,948] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8073,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:28:45,948] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8073,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:28:45,948] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8076,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:28:45,949] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8076,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:28:45,949] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8077,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:28:45,949] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8077,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:28:45,976] jepsen worker 29 - jepsen.util 1199 :ok :read [9 28] INFO [2019-12-17 12:28:45,977] jepsen worker 29 - jepsen.util 1199 :invoke :write [9 7] INFO [2019-12-17 12:28:45,988] jepsen worker 44 - jepsen.util 1574 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:28:45,988] jepsen worker 73 - jepsen.util 1423 :info :write [24 5] :RequestCanceledException INFO [2019-12-17 12:28:45,988] jepsen worker 44 - jepsen.util 1574 :invoke :write [14 11] INFO [2019-12-17 12:28:45,989] jepsen worker 28 - jepsen.util 928 :fail :read [9 nil] :RequestTimeoutException INFO [2019-12-17 12:28:45,990] jepsen worker 28 - jepsen.util 928 :invoke :cas [9 [40 17]] INFO [2019-12-17 12:28:45,999] jepsen worker 73 - jepsen.util 1513 :invoke :write [24 6] INFO [2019-12-17 12:28:46,024] jepsen worker 8 - jepsen.util 1808 :info :write [2 12] :RequestCanceledException INFO [2019-12-17 12:28:46,026] jepsen worker 89 - jepsen.util 1439 :info :write [29 25] :RequestCanceledException INFO [2019-12-17 12:28:46,041] jepsen worker 8 - jepsen.util 1898 :invoke :cas [2 [10 21]] INFO [2019-12-17 12:28:46,047] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8085,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:28:46,047] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8085,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:28:46,047] jepsen worker 89 - jepsen.util 1529 :invoke :read [29 nil] INFO [2019-12-17 12:28:46,049] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8087,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:28:46,050] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8087,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:28:46,054] jepsen worker 56 - jepsen.util 1406 :info :write [18 27] :RequestCanceledException INFO [2019-12-17 12:28:46,063] jepsen worker 56 - jepsen.util 1496 :invoke :read [18 nil] INFO [2019-12-17 12:28:46,073] jepsen worker 31 - jepsen.util 1561 :fail :cas [10 [29 15]] :ValueNotSwapFrom INFO [2019-12-17 12:28:46,073] jepsen worker 31 - jepsen.util 1561 :invoke :cas [10 [2 38]] INFO [2019-12-17 12:28:46,094] jepsen worker 43 - jepsen.util 1933 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:28:46,096] jepsen worker 43 - jepsen.util 1933 :invoke :cas [14 [26 37]] INFO [2019-12-17 12:28:46,105] jepsen worker 29 - jepsen.util 1199 :ok :write [9 7] INFO [2019-12-17 12:28:46,107] jepsen worker 29 - jepsen.util 1199 :invoke :read [9 nil] INFO [2019-12-17 12:28:46,153] jepsen worker 4 - jepsen.util 1174 :info :write [1 41] :RequestCanceledException INFO [2019-12-17 12:28:46,155] jepsen worker 4 - jepsen.util 1264 :invoke :cas [1 [6 47]] INFO [2019-12-17 12:28:46,162] jepsen worker 28 - jepsen.util 928 :fail :cas [9 [40 17]] :ValueNotSwapFrom INFO [2019-12-17 12:28:46,162] jepsen worker 28 - jepsen.util 928 :invoke :write [9 26] INFO [2019-12-17 12:28:46,173] jepsen worker 87 - jepsen.util 1347 :ok :read [29 32] INFO [2019-12-17 12:28:46,174] jepsen worker 87 - jepsen.util 1347 :invoke :cas [29 [26 0]] INFO [2019-12-17 12:28:46,223] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8102,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:28:46,235] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8102,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:28:46,237] jepsen worker 51 - jepsen.util 1401 :info :write [17 44] :RequestCanceledException INFO [2019-12-17 12:28:46,263] jepsen worker 51 - jepsen.util 1491 :invoke :cas [17 [31 29]] INFO [2019-12-17 12:28:46,269] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8109,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:28:46,273] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8109,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:28:46,275] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8111,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:28:46,278] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8111,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:28:46,319] jepsen worker 26 - jepsen.util 1826 :info :write [8 5] :RequestCanceledException INFO [2019-12-17 12:28:46,345] jepsen worker 47 - jepsen.util 1847 :ok :write [15 4] INFO [2019-12-17 12:28:46,348] jepsen worker 20 - jepsen.util 1460 :ok :write [6 41] INFO [2019-12-17 12:28:46,348] jepsen worker 32 - jepsen.util 1562 :ok :write [10 23] INFO [2019-12-17 12:28:46,352] jepsen worker 20 - jepsen.util 1460 :invoke :write [6 17] INFO [2019-12-17 12:28:46,352] jepsen worker 47 - jepsen.util 1847 :invoke :read [15 nil] INFO [2019-12-17 12:28:46,355] jepsen worker 32 - jepsen.util 1562 :invoke :write [10 22] INFO [2019-12-17 12:28:46,367] jepsen worker 74 - jepsen.util 1424 :ok :write [24 49] INFO [2019-12-17 12:28:46,372] jepsen worker 74 - jepsen.util 1424 :invoke :read [24 nil] INFO [2019-12-17 12:28:46,374] jepsen worker 26 - jepsen.util 1916 :invoke :write [8 34] INFO [2019-12-17 12:28:46,381] jepsen worker 28 - jepsen.util 928 :ok :write [9 26] INFO [2019-12-17 12:28:46,384] jepsen worker 28 - jepsen.util 928 :invoke :write [9 19] INFO [2019-12-17 12:28:46,387] jepsen worker 29 - jepsen.util 1199 :ok :read [9 7] INFO [2019-12-17 12:28:46,390] jepsen worker 29 - jepsen.util 1199 :invoke :write [9 38] INFO [2019-12-17 12:28:46,390] jepsen worker 34 - jepsen.util 2104 :fail :cas [11 [3 39]] :ValueNotSwapFrom INFO [2019-12-17 12:28:46,390] jepsen worker 34 - jepsen.util 2104 :invoke :read [11 nil] INFO [2019-12-17 12:28:46,409] jepsen worker 63 - jepsen.util 1233 :ok :write [21 8] INFO [2019-12-17 12:28:46,410] jepsen worker 63 - jepsen.util 1233 :invoke :write [21 1] INFO [2019-12-17 12:28:46,422] jepsen worker 25 - jepsen.util 2005 :info :write [8 3] :RequestCanceledException INFO [2019-12-17 12:28:46,423] jepsen worker 25 - jepsen.util 2095 :invoke :read [8 nil] INFO [2019-12-17 12:28:46,425] jepsen worker 6 - jepsen.util 1806 :info :cas [2 [15 11]] :RequestCanceledException INFO [2019-12-17 12:28:46,436] jepsen worker 89 - jepsen.util 1529 :ok :read [29 32] INFO [2019-12-17 12:28:46,437] jepsen worker 89 - jepsen.util 1529 :invoke :cas [29 [32 22]] INFO [2019-12-17 12:28:46,438] jepsen worker 80 - jepsen.util 1520 :info :write [26 24] :RequestCanceledException INFO [2019-12-17 12:28:46,455] jepsen worker 6 - jepsen.util 1896 :invoke :cas [2 [48 30]] INFO [2019-12-17 12:28:46,461] jepsen worker 80 - jepsen.util 1610 :invoke :read [26 nil] INFO [2019-12-17 12:28:46,464] jepsen worker 87 - jepsen.util 1347 :fail :cas [29 [26 0]] :ValueNotSwapFrom INFO [2019-12-17 12:28:46,465] jepsen worker 87 - jepsen.util 1347 :invoke :write [29 5] INFO [2019-12-17 12:28:46,474] jepsen worker 55 - jepsen.util 1765 :ok :read [18 26] INFO [2019-12-17 12:28:46,476] jepsen worker 55 - jepsen.util 1765 :invoke :write [18 26] INFO [2019-12-17 12:28:46,478] jepsen worker 73 - jepsen.util 1513 :ok :write [24 6] INFO [2019-12-17 12:28:46,478] jepsen worker 13 - jepsen.util 1363 :ok :read [4 26] INFO [2019-12-17 12:28:46,478] jepsen worker 73 - jepsen.util 1513 :invoke :read [24 nil] INFO [2019-12-17 12:28:46,478] jepsen worker 13 - jepsen.util 1363 :invoke :write [4 35] INFO [2019-12-17 12:28:46,479] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8120,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:28:46,480] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8120,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:28:46,537] jepsen worker 12 - jepsen.util 1542 :ok :write [4 46] INFO [2019-12-17 12:28:46,538] jepsen worker 12 - jepsen.util 1542 :invoke :write [4 30] INFO [2019-12-17 12:28:46,583] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:28:46,586] jepsen worker 30 - jepsen.util 1560 :info :write [10 5] :RequestCanceledException INFO [2019-12-17 12:28:46,588] jepsen worker 51 - jepsen.util 1491 :fail :cas [17 [31 29]] :ValueNotSwapFrom INFO [2019-12-17 12:28:46,602] jepsen worker 28 - jepsen.util 928 :ok :write [9 19] INFO [2019-12-17 12:28:46,604] jepsen worker 88 - jepsen.util 1258 :info :write [29 24] :RequestCanceledException INFO [2019-12-17 12:28:46,613] jepsen worker 53 - jepsen.util 1493 :ok :read [17 44] INFO [2019-12-17 12:28:46,620] jepsen worker 47 - jepsen.util 1847 :ok :read [15 4] INFO [2019-12-17 12:28:46,620] jepsen nemesis - couchbase.util Rebalance check count: 54 INFO [2019-12-17 12:28:46,621] jepsen worker 51 - jepsen.util 1491 :invoke :read [17 nil] INFO [2019-12-17 12:28:46,621] jepsen worker 53 - jepsen.util 1493 :invoke :read [17 nil] INFO [2019-12-17 12:28:46,621] jepsen nemesis - couchbase.util Rebalance stuck count: 35 INFO [2019-12-17 12:28:46,621] jepsen worker 28 - jepsen.util 928 :invoke :write [9 47] INFO [2019-12-17 12:28:46,622] jepsen worker 88 - jepsen.util 1348 :invoke :read [29 nil] INFO [2019-12-17 12:28:46,623] jepsen worker 47 - jepsen.util 1847 :invoke :read [15 nil] INFO [2019-12-17 12:28:46,629] jepsen worker 30 - jepsen.util 1650 :invoke :cas [10 [43 19]] INFO [2019-12-17 12:28:46,681] jepsen worker 52 - jepsen.util 1582 :ok :write [17 21] INFO [2019-12-17 12:28:46,681] jepsen worker 52 - jepsen.util 1582 :invoke :cas [17 [14 33]] INFO [2019-12-17 12:28:46,684] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8148,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:28:46,687] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8148,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:28:46,689] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8152,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:28:46,693] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8152,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:28:46,710] jepsen worker 76 - jepsen.util 1696 :info :cas [25 [36 30]] :RequestCanceledException INFO [2019-12-17 12:28:46,710] jepsen worker 0 - jepsen.util 1890 :info :cas [0 [25 33]] :RequestCanceledException INFO [2019-12-17 12:28:46,713] jepsen worker 74 - jepsen.util 1424 :ok :read [24 6] INFO [2019-12-17 12:28:46,713] jepsen worker 74 - jepsen.util 1424 :invoke :write [24 36] INFO [2019-12-17 12:28:46,718] jepsen worker 73 - jepsen.util 1513 :ok :read [24 6] INFO [2019-12-17 12:28:46,718] jepsen worker 73 - jepsen.util 1513 :invoke :cas [24 [40 1]] INFO [2019-12-17 12:28:46,730] jepsen worker 31 - jepsen.util 1561 :fail :cas [10 [2 38]] :CASMismatchException INFO [2019-12-17 12:28:46,732] jepsen worker 31 - jepsen.util 1561 :invoke :cas [10 [9 11]] INFO [2019-12-17 12:28:46,734] jepsen worker 7 - jepsen.util 2077 :info :cas [2 [14 20]] :RequestCanceledException INFO [2019-12-17 12:28:46,734] jepsen worker 0 - jepsen.util 1980 :invoke :read [0 nil] INFO [2019-12-17 12:28:46,736] jepsen worker 82 - jepsen.util 2242 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:28:46,737] jepsen worker 85 - jepsen.util 1885 :info :cas [28 [21 34]] :RequestCanceledException INFO [2019-12-17 12:28:46,737] jepsen worker 27 - jepsen.util 1287 :fail :read [9 nil] :RequestTimeoutException INFO [2019-12-17 12:28:46,737] jepsen worker 9 - jepsen.util 1539 :fail :read [3 nil] :RequestTimeoutException INFO [2019-12-17 12:28:46,737] jepsen worker 27 - jepsen.util 1287 :invoke :write [9 10] INFO [2019-12-17 12:28:46,738] jepsen worker 82 - jepsen.util 2242 :invoke :write [27 30] INFO [2019-12-17 12:28:46,738] jepsen worker 9 - jepsen.util 1539 :invoke :write [3 7] INFO [2019-12-17 12:28:46,741] jepsen worker 7 - jepsen.util 2167 :invoke :read [2 nil] INFO [2019-12-17 12:28:46,742] jepsen worker 76 - jepsen.util 1786 :invoke :cas [25 [43 19]] INFO [2019-12-17 12:28:46,755] jepsen worker 36 - jepsen.util 1386 :info :write [12 29] :RequestCanceledException INFO [2019-12-17 12:28:46,756] jepsen worker 38 - jepsen.util 1388 :ok :write [12 40] INFO [2019-12-17 12:28:46,756] jepsen worker 38 - jepsen.util 1388 :invoke :cas [12 [12 19]] INFO [2019-12-17 12:28:46,756] jepsen worker 36 - jepsen.util 1476 :invoke :read [12 nil] INFO [2019-12-17 12:28:46,757] jepsen worker 85 - jepsen.util 1975 :invoke :cas [28 [6 5]] INFO [2019-12-17 12:28:46,766] jepsen worker 56 - jepsen.util 1496 :ok :read [18 26] INFO [2019-12-17 12:28:46,769] jepsen worker 56 - jepsen.util 1496 :invoke :cas [18 [4 1]] INFO [2019-12-17 12:28:46,812] jepsen worker 34 - jepsen.util 2104 :ok :read [11 7] INFO [2019-12-17 12:28:46,814] jepsen worker 34 - jepsen.util 2104 :invoke :write [11 42] INFO [2019-12-17 12:28:46,823] jepsen worker 42 - jepsen.util 2022 :info :write [14 33] :RequestCanceledException INFO [2019-12-17 12:28:46,835] jepsen worker 86 - jepsen.util 1886 :info :write [28 9] :RequestCanceledException INFO [2019-12-17 12:28:46,837] jepsen worker 47 - jepsen.util 1847 :ok :read [15 4] INFO [2019-12-17 12:28:46,838] jepsen worker 42 - jepsen.util 2112 :invoke :read [14 nil] INFO [2019-12-17 12:28:46,842] jepsen worker 47 - jepsen.util 1847 :invoke :cas [15 [15 38]] INFO [2019-12-17 12:28:46,846] jepsen worker 86 - jepsen.util 1976 :invoke :read [28 nil] INFO [2019-12-17 12:28:46,852] jepsen worker 24 - jepsen.util 1644 :info :cas [8 [48 29]] :RequestCanceledException INFO [2019-12-17 12:28:46,853] jepsen worker 60 - jepsen.util 1950 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:28:46,853] jepsen worker 60 - jepsen.util 1950 :invoke :write [20 26] INFO [2019-12-17 12:28:46,870] jepsen worker 24 - jepsen.util 1734 :invoke :read [8 nil] INFO [2019-12-17 12:28:46,886] jepsen worker 50 - jepsen.util 1850 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:28:46,888] jepsen worker 48 - jepsen.util 1308 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:28:46,888] jepsen worker 50 - jepsen.util 1850 :invoke :write [16 2] INFO [2019-12-17 12:28:46,889] jepsen worker 48 - jepsen.util 1308 :invoke :write [16 16] INFO [2019-12-17 12:28:46,893] jepsen worker 30 - jepsen.util 1650 :fail :cas [10 [43 19]] :ValueNotSwapFrom INFO [2019-12-17 12:28:46,895] jepsen worker 30 - jepsen.util 1650 :invoke :read [10 nil] INFO [2019-12-17 12:28:46,895] jepsen worker 87 - jepsen.util 1347 :ok :write [29 5] INFO [2019-12-17 12:28:46,895] jepsen worker 87 - jepsen.util 1347 :invoke :read [29 nil] INFO [2019-12-17 12:28:46,905] jepsen worker 41 - jepsen.util 1661 :info :cas [13 [47 38]] :RequestCanceledException INFO [2019-12-17 12:28:46,909] jepsen worker 1 - jepsen.util 1801 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:28:46,910] jepsen worker 28 - jepsen.util 928 :ok :write [9 47] INFO [2019-12-17 12:28:46,912] jepsen worker 28 - jepsen.util 928 :invoke :write [9 11] INFO [2019-12-17 12:28:46,912] jepsen worker 1 - jepsen.util 1801 :invoke :write [0 28] INFO [2019-12-17 12:28:46,916] jepsen worker 41 - jepsen.util 1751 :invoke :cas [13 [0 31]] INFO [2019-12-17 12:28:46,921] jepsen worker 57 - jepsen.util 1677 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:28:46,924] jepsen worker 57 - jepsen.util 1677 :invoke :cas [19 [23 19]] INFO [2019-12-17 12:28:46,924] jepsen worker 67 - jepsen.util 2137 :info :write [22 2] :RequestCanceledException INFO [2019-12-17 12:28:46,925] jepsen worker 66 - jepsen.util 1506 :info :write [22 34] :RequestCanceledException INFO [2019-12-17 12:28:46,929] jepsen worker 73 - jepsen.util 1513 :fail :cas [24 [40 1]] :ValueNotSwapFrom INFO [2019-12-17 12:28:46,929] jepsen worker 73 - jepsen.util 1513 :invoke :cas [24 [47 30]] INFO [2019-12-17 12:28:46,930] jepsen worker 3 - jepsen.util 1353 :fail :cas [1 [26 9]] :ValueNotSwapFrom INFO [2019-12-17 12:28:46,930] jepsen worker 3 - jepsen.util 1353 :invoke :write [1 9] INFO [2019-12-17 12:28:46,933] jepsen worker 59 - jepsen.util 1499 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:28:46,934] jepsen worker 66 - jepsen.util 1596 :invoke :read [22 nil] INFO [2019-12-17 12:28:46,935] jepsen worker 59 - jepsen.util 1499 :invoke :cas [19 [29 32]] INFO [2019-12-17 12:28:46,936] jepsen worker 16 - jepsen.util 1726 :info :write [5 5] :RequestCanceledException INFO [2019-12-17 12:28:46,948] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8174,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:28:46,951] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8174,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:28:46,955] jepsen worker 84 - jepsen.util 1704 :info :write [28 1] :RequestCanceledException INFO [2019-12-17 12:28:46,959] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8175,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:28:46,960] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8175,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:28:46,962] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8176,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:28:46,962] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8176,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:28:46,965] jepsen worker 67 - jepsen.util 2227 :invoke :cas [22 [21 9]] INFO [2019-12-17 12:28:46,978] jepsen worker 16 - jepsen.util 1816 :invoke :cas [5 [11 21]] INFO [2019-12-17 12:28:46,992] jepsen worker 18 - jepsen.util 1638 :info :write [6 34] :RequestCanceledException INFO [2019-12-17 12:28:46,993] jepsen worker 37 - jepsen.util 1297 :ok :write [12 6] INFO [2019-12-17 12:28:46,993] jepsen worker 32 - jepsen.util 1562 :ok :write [10 22] INFO [2019-12-17 12:28:46,993] jepsen worker 32 - jepsen.util 1562 :invoke :cas [10 [6 11]] INFO [2019-12-17 12:28:46,995] jepsen worker 84 - jepsen.util 1794 :invoke :read [28 nil] INFO [2019-12-17 12:28:46,997] jepsen worker 37 - jepsen.util 1297 :invoke :cas [12 [14 45]] INFO [2019-12-17 12:28:47,006] jepsen worker 17 - jepsen.util 1817 :info :write [5 41] :RequestCanceledException INFO [2019-12-17 12:28:47,009] jepsen worker 15 - jepsen.util 1455 :info :write [5 29] :RequestCanceledException INFO [2019-12-17 12:28:47,010] jepsen worker 18 - jepsen.util 1728 :invoke :write [6 40] INFO [2019-12-17 12:28:47,012] jepsen worker 6 - jepsen.util 1896 :fail :cas [2 [48 30]] :ValueNotSwapFrom INFO [2019-12-17 12:28:47,012] jepsen worker 6 - jepsen.util 1896 :invoke :write [2 0] INFO [2019-12-17 12:28:47,017] jepsen worker 17 - jepsen.util 1907 :invoke :read [5 nil] INFO [2019-12-17 12:28:47,018] jepsen worker 20 - jepsen.util 1460 :ok :write [6 17] INFO [2019-12-17 12:28:47,018] jepsen worker 20 - jepsen.util 1460 :invoke :read [6 nil] INFO [2019-12-17 12:28:47,020] jepsen worker 80 - jepsen.util 1610 :ok :read [26 27] INFO [2019-12-17 12:28:47,021] jepsen worker 80 - jepsen.util 1610 :invoke :write [26 19] INFO [2019-12-17 12:28:47,025] jepsen worker 83 - jepsen.util 2153 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:28:47,025] jepsen worker 83 - jepsen.util 2153 :invoke :cas [27 [37 7]] INFO [2019-12-17 12:28:47,027] jepsen worker 23 - jepsen.util 1643 :info :write [7 18] :RequestCanceledException INFO [2019-12-17 12:28:47,027] jepsen worker 40 - jepsen.util 2020 :info :cas [13 [29 31]] :RequestCanceledException INFO [2019-12-17 12:28:47,028] jepsen worker 53 - jepsen.util 1493 :ok :read [17 21] INFO [2019-12-17 12:28:47,028] jepsen worker 54 - jepsen.util 1584 :info :write [18 27] :RequestCanceledException INFO [2019-12-17 12:28:47,029] jepsen worker 2 - jepsen.util 1442 :info :cas [0 [44 16]] :RequestCanceledException INFO [2019-12-17 12:28:47,030] jepsen worker 53 - jepsen.util 1493 :invoke :read [17 nil] INFO [2019-12-17 12:28:47,030] jepsen worker 15 - jepsen.util 1545 :invoke :write [5 10] INFO [2019-12-17 12:28:47,041] jepsen worker 2 - jepsen.util 1532 :invoke :cas [0 [40 35]] INFO [2019-12-17 12:28:47,042] jepsen worker 40 - jepsen.util 2110 :invoke :write [13 27] INFO [2019-12-17 12:28:47,042] jepsen worker 23 - jepsen.util 1733 :invoke :read [7 nil] INFO [2019-12-17 12:28:47,049] jepsen worker 49 - jepsen.util 1309 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:28:47,050] jepsen worker 49 - jepsen.util 1309 :invoke :read [16 nil] INFO [2019-12-17 12:28:47,050] jepsen worker 68 - jepsen.util 1328 :info :cas [22 [7 13]] :RequestCanceledException INFO [2019-12-17 12:28:47,052] jepsen worker 54 - jepsen.util 1674 :invoke :write [18 37] INFO [2019-12-17 12:28:47,064] jepsen worker 68 - jepsen.util 1418 :invoke :cas [22 [35 7]] INFO [2019-12-17 12:28:47,078] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8191,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:28:47,078] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8191,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:28:47,078] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8192,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:28:47,078] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8192,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:28:47,078] jepsen worker 74 - jepsen.util 1424 :ok :write [24 36] INFO [2019-12-17 12:28:47,079] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8193,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:28:47,079] jepsen worker 74 - jepsen.util 1424 :invoke :write [24 29] INFO [2019-12-17 12:28:47,079] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8193,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:28:47,081] jepsen worker 29 - jepsen.util 1199 :ok :write [9 38] INFO [2019-12-17 12:28:47,083] jepsen worker 29 - jepsen.util 1199 :invoke :read [9 nil] INFO [2019-12-17 12:28:47,089] jepsen worker 39 - jepsen.util 1929 :info :cas [13 [43 39]] :RequestCanceledException INFO [2019-12-17 12:28:47,099] jepsen worker 39 - jepsen.util 2019 :invoke :cas [13 [8 9]] INFO [2019-12-17 12:28:47,107] jepsen worker 72 - jepsen.util 1332 :info :write [24 30] :RequestCanceledException INFO [2019-12-17 12:28:47,108] jepsen worker 61 - jepsen.util 1861 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:28:47,110] jepsen worker 61 - jepsen.util 1861 :invoke :cas [20 [18 22]] INFO [2019-12-17 12:28:47,115] jepsen worker 71 - jepsen.util 1151 :ok :cas [23 [8 32]] INFO [2019-12-17 12:28:47,115] jepsen worker 71 - jepsen.util 1151 :invoke :cas [23 [0 18]] INFO [2019-12-17 12:28:47,118] jepsen worker 72 - jepsen.util 1422 :invoke :read [24 nil] INFO [2019-12-17 12:28:47,120] jepsen worker 19 - jepsen.util 1549 :info :write [6 13] :RequestCanceledException INFO [2019-12-17 12:28:47,121] jepsen worker 69 - jepsen.util 1239 :info :write [23 11] :RequestCanceledException INFO [2019-12-17 12:28:47,128] jepsen worker 19 - jepsen.util 1639 :invoke :cas [6 [36 26]] INFO [2019-12-17 12:28:47,136] jepsen worker 81 - jepsen.util 1611 :info :cas [27 [38 24]] :RequestCanceledException INFO [2019-12-17 12:28:47,136] jepsen worker 63 - jepsen.util 1233 :ok :write [21 1] INFO [2019-12-17 12:28:47,138] jepsen worker 63 - jepsen.util 1233 :invoke :cas [21 [36 40]] INFO [2019-12-17 12:28:47,138] jepsen worker 69 - jepsen.util 1329 :invoke :write [23 4] INFO [2019-12-17 12:28:47,156] jepsen worker 81 - jepsen.util 1701 :invoke :read [27 nil] INFO [2019-12-17 12:28:47,177] jepsen worker 51 - jepsen.util 1491 :ok :read [17 21] INFO [2019-12-17 12:28:47,177] jepsen worker 87 - jepsen.util 1347 :ok :read [29 5] INFO [2019-12-17 12:28:47,177] jepsen worker 51 - jepsen.util 1491 :invoke :cas [17 [31 0]] INFO [2019-12-17 12:28:47,178] jepsen worker 87 - jepsen.util 1347 :invoke :read [29 nil] INFO [2019-12-17 12:28:47,182] jepsen worker 47 - jepsen.util 1847 :fail :cas [15 [15 38]] :ValueNotSwapFrom INFO [2019-12-17 12:28:47,183] jepsen worker 47 - jepsen.util 1847 :invoke :read [15 nil] INFO [2019-12-17 12:28:47,221] jepsen worker 30 - jepsen.util 1650 :ok :read [10 23] INFO [2019-12-17 12:28:47,222] jepsen worker 30 - jepsen.util 1650 :invoke :cas [10 [43 41]] INFO [2019-12-17 12:28:47,223] jepsen worker 28 - jepsen.util 928 :ok :write [9 11] INFO [2019-12-17 12:28:47,224] jepsen worker 28 - jepsen.util 928 :invoke :write [9 11] INFO [2019-12-17 12:28:47,229] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8229,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:28:47,229] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8229,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:28:47,230] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8230,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:28:47,232] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8230,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:28:47,238] jepsen worker 76 - jepsen.util 1786 :fail :cas [25 [43 19]] :ValueNotSwapFrom INFO [2019-12-17 12:28:47,238] jepsen worker 76 - jepsen.util 1786 :invoke :write [25 42] INFO [2019-12-17 12:28:47,243] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8238,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:28:47,243] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8238,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:28:47,244] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8241,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:28:47,247] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8241,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:28:47,280] jepsen worker 73 - jepsen.util 1513 :fail :cas [24 [47 30]] :ValueNotSwapFrom INFO [2019-12-17 12:28:47,280] jepsen worker 29 - jepsen.util 1199 :ok :read [9 38] INFO [2019-12-17 12:28:47,280] jepsen worker 73 - jepsen.util 1513 :invoke :write [24 7] INFO [2019-12-17 12:28:47,280] jepsen worker 29 - jepsen.util 1199 :invoke :write [9 34] INFO [2019-12-17 12:28:47,282] jepsen worker 42 - jepsen.util 2112 :ok :read [14 27] INFO [2019-12-17 12:28:47,282] jepsen worker 42 - jepsen.util 2112 :invoke :write [14 25] INFO [2019-12-17 12:28:47,286] jepsen worker 58 - jepsen.util 1588 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:28:47,286] jepsen worker 58 - jepsen.util 1588 :invoke :read [19 nil] INFO [2019-12-17 12:28:47,296] jepsen worker 19 - jepsen.util 1639 :fail :cas [6 [36 26]] :ValueNotSwapFrom INFO [2019-12-17 12:28:47,297] jepsen worker 19 - jepsen.util 1639 :invoke :read [6 nil] INFO [2019-12-17 12:28:47,308] jepsen worker 22 - jepsen.util 1822 :info :write [7 30] :RequestCanceledException INFO [2019-12-17 12:28:47,309] jepsen worker 79 - jepsen.util 1429 :info :write [26 44] :RequestCanceledException INFO [2019-12-17 12:28:47,320] jepsen worker 79 - jepsen.util 1519 :invoke :cas [26 [27 7]] INFO [2019-12-17 12:28:47,322] jepsen worker 35 - jepsen.util 1565 :info :write [11 11] :RequestCanceledException INFO [2019-12-17 12:28:47,325] jepsen worker 22 - jepsen.util 1912 :invoke :write [7 32] INFO [2019-12-17 12:28:47,348] jepsen worker 35 - jepsen.util 1655 :invoke :read [11 nil] INFO [2019-12-17 12:28:47,354] jepsen worker 72 - jepsen.util 1422 :ok :read [24 36] INFO [2019-12-17 12:28:47,354] jepsen worker 72 - jepsen.util 1422 :invoke :write [24 20] INFO [2019-12-17 12:28:47,354] jepsen worker 28 - jepsen.util 928 :ok :write [9 11] INFO [2019-12-17 12:28:47,355] jepsen worker 28 - jepsen.util 928 :invoke :read [9 nil] INFO [2019-12-17 12:28:47,388] jepsen worker 47 - jepsen.util 1847 :ok :read [15 4] INFO [2019-12-17 12:28:47,388] jepsen worker 70 - jepsen.util 1690 :info :write [23 20] :RequestCanceledException INFO [2019-12-17 12:28:47,389] jepsen worker 47 - jepsen.util 1847 :invoke :read [15 nil] INFO [2019-12-17 12:28:47,398] jepsen worker 70 - jepsen.util 1780 :invoke :cas [23 [47 5]] INFO [2019-12-17 12:28:47,426] jepsen worker 87 - jepsen.util 1347 :ok :read [29 5] INFO [2019-12-17 12:28:47,427] jepsen worker 87 - jepsen.util 1347 :invoke :write [29 49] INFO [2019-12-17 12:28:47,434] jepsen worker 30 - jepsen.util 1650 :fail :cas [10 [43 41]] :ValueNotSwapFrom INFO [2019-12-17 12:28:47,434] jepsen worker 30 - jepsen.util 1650 :invoke :cas [10 [48 3]] INFO [2019-12-17 12:28:47,437] jepsen worker 78 - jepsen.util 1518 :info :write [26 24] :RequestCanceledException INFO [2019-12-17 12:28:47,443] jepsen worker 78 - jepsen.util 1608 :invoke :read [26 nil] INFO [2019-12-17 12:28:47,464] jepsen worker 29 - jepsen.util 1199 :ok :write [9 34] INFO [2019-12-17 12:28:47,466] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8258,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:28:47,480] jepsen worker 89 - jepsen.util 1529 :fail :cas [29 [32 22]] :ValueNotSwapFrom INFO [2019-12-17 12:28:47,483] jepsen worker 19 - jepsen.util 1639 :ok :read [6 17] INFO [2019-12-17 12:28:47,507] jepsen worker 28 - jepsen.util 928 :ok :read [9 11] INFO [2019-12-17 12:28:47,507] jepsen worker 51 - jepsen.util 1491 :fail :cas [17 [31 0]] :ValueNotSwapFrom INFO [2019-12-17 12:28:47,535] jepsen worker 29 - jepsen.util 1199 :invoke :read [9 nil] INFO [2019-12-17 12:28:47,535] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8258,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:28:47,536] jepsen worker 89 - jepsen.util 1529 :invoke :read [29 nil] INFO [2019-12-17 12:28:47,536] jepsen worker 19 - jepsen.util 1639 :invoke :cas [6 [15 20]] INFO [2019-12-17 12:28:47,537] jepsen worker 28 - jepsen.util 928 :invoke :cas [9 [11 27]] INFO [2019-12-17 12:28:47,537] jepsen worker 51 - jepsen.util 1491 :invoke :cas [17 [6 14]] INFO [2019-12-17 12:28:47,540] jepsen worker 70 - jepsen.util 1780 :fail :cas [23 [47 5]] :ValueNotSwapFrom INFO [2019-12-17 12:28:47,541] jepsen worker 70 - jepsen.util 1780 :invoke :write [23 19] INFO [2019-12-17 12:28:47,555] jepsen worker 47 - jepsen.util 1847 :ok :read [15 4] INFO [2019-12-17 12:28:47,556] jepsen worker 47 - jepsen.util 1847 :invoke :cas [15 [18 25]] INFO [2019-12-17 12:28:47,622] jepsen worker 56 - jepsen.util 1496 :fail :cas [18 [4 1]] :ValueNotSwapFrom INFO [2019-12-17 12:28:47,622] jepsen worker 56 - jepsen.util 1496 :invoke :cas [18 [30 43]] INFO [2019-12-17 12:28:47,624] jepsen worker 38 - jepsen.util 1388 :fail :cas [12 [12 19]] :ValueNotSwapFrom INFO [2019-12-17 12:28:47,624] jepsen worker 38 - jepsen.util 1388 :invoke :write [12 42] INFO [2019-12-17 12:28:47,647] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:28:47,648] jepsen nemesis - couchbase.util Rebalance check count: 55 INFO [2019-12-17 12:28:47,648] jepsen nemesis - couchbase.util Rebalance stuck count: 36 INFO [2019-12-17 12:28:47,653] jepsen worker 30 - jepsen.util 1650 :fail :cas [10 [48 3]] :ValueNotSwapFrom INFO [2019-12-17 12:28:47,656] jepsen worker 36 - jepsen.util 1476 :ok :read [12 6] INFO [2019-12-17 12:28:47,656] jepsen worker 36 - jepsen.util 1476 :invoke :read [12 nil] INFO [2019-12-17 12:28:47,658] jepsen worker 30 - jepsen.util 1650 :invoke :write [10 19] INFO [2019-12-17 12:28:47,670] jepsen worker 53 - jepsen.util 1493 :ok :read [17 21] INFO [2019-12-17 12:28:47,670] jepsen worker 53 - jepsen.util 1493 :invoke :cas [17 [0 20]] INFO [2019-12-17 12:28:47,691] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8276,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:28:47,692] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8276,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:28:47,692] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8278,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:28:47,693] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8278,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:28:47,707] jepsen worker 75 - jepsen.util 1965 :info :write [25 22] :RequestCanceledException INFO [2019-12-17 12:28:47,715] jepsen worker 75 - jepsen.util 2055 :invoke :cas [25 [49 3]] INFO [2019-12-17 12:28:47,719] jepsen worker 29 - jepsen.util 1199 :ok :read [9 34] INFO [2019-12-17 12:28:47,720] jepsen worker 29 - jepsen.util 1199 :invoke :write [9 29] INFO [2019-12-17 12:28:47,720] jepsen worker 19 - jepsen.util 1639 :fail :cas [6 [15 20]] :ValueNotSwapFrom INFO [2019-12-17 12:28:47,724] jepsen worker 19 - jepsen.util 1639 :invoke :cas [6 [5 42]] INFO [2019-12-17 12:28:47,736] jepsen worker 65 - jepsen.util 1145 :info :write [21 9] :RequestCanceledException INFO [2019-12-17 12:28:47,736] jepsen worker 45 - jepsen.util 1575 :info :cas [15 [34 9]] :RequestCanceledException INFO [2019-12-17 12:28:47,736] jepsen worker 9 - jepsen.util 1539 :ok :write [3 7] INFO [2019-12-17 12:28:47,737] jepsen worker 9 - jepsen.util 1539 :invoke :cas [3 [46 43]] INFO [2019-12-17 12:28:47,738] jepsen worker 11 - jepsen.util 1721 :fail :read [3 nil] :RequestTimeoutException INFO [2019-12-17 12:28:47,738] jepsen worker 11 - jepsen.util 1721 :invoke :read [3 nil] INFO [2019-12-17 12:28:47,738] jepsen worker 64 - jepsen.util 964 :info :cas [21 [25 25]] :RequestCanceledException INFO [2019-12-17 12:28:47,738] jepsen worker 46 - jepsen.util 1666 :info :write [15 49] :RequestCanceledException INFO [2019-12-17 12:28:47,755] jepsen worker 65 - jepsen.util 1235 :invoke :cas [21 [26 46]] INFO [2019-12-17 12:28:47,755] jepsen worker 45 - jepsen.util 1665 :invoke :read [15 nil] INFO [2019-12-17 12:28:47,755] jepsen worker 46 - jepsen.util 1756 :invoke :write [15 17] INFO [2019-12-17 12:28:47,756] jepsen worker 71 - jepsen.util 1151 :fail :cas [23 [0 18]] :ValueNotSwapFrom INFO [2019-12-17 12:28:47,756] jepsen worker 64 - jepsen.util 1054 :invoke :cas [21 [5 46]] INFO [2019-12-17 12:28:47,756] jepsen worker 71 - jepsen.util 1151 :invoke :read [23 nil] INFO [2019-12-17 12:28:47,757] jepsen worker 28 - jepsen.util 928 :fail :cas [9 [11 27]] :ValueNotSwapFrom INFO [2019-12-17 12:28:47,757] jepsen worker 74 - jepsen.util 1424 :ok :write [24 29] INFO [2019-12-17 12:28:47,757] jepsen worker 18 - jepsen.util 1728 :ok :write [6 40] INFO [2019-12-17 12:28:47,757] jepsen worker 18 - jepsen.util 1728 :invoke :read [6 nil] INFO [2019-12-17 12:28:47,757] jepsen worker 74 - jepsen.util 1424 :invoke :read [24 nil] INFO [2019-12-17 12:28:47,757] jepsen worker 78 - jepsen.util 1608 :ok :read [26 27] INFO [2019-12-17 12:28:47,757] jepsen worker 78 - jepsen.util 1608 :invoke :cas [26 [0 36]] INFO [2019-12-17 12:28:47,757] jepsen worker 28 - jepsen.util 928 :invoke :write [9 29] INFO [2019-12-17 12:28:47,758] jepsen worker 5 - jepsen.util 1085 :info :write [1 26] :RequestCanceledException INFO [2019-12-17 12:28:47,765] jepsen worker 5 - jepsen.util 1175 :invoke :write [1 11] INFO [2019-12-17 12:28:47,779] jepsen worker 27 - jepsen.util 1287 :ok :write [9 10] INFO [2019-12-17 12:28:47,779] jepsen worker 27 - jepsen.util 1287 :invoke :read [9 nil] INFO [2019-12-17 12:28:47,805] jepsen worker 37 - jepsen.util 1297 :fail :cas [12 [14 45]] :ValueNotSwapFrom INFO [2019-12-17 12:28:47,806] jepsen worker 37 - jepsen.util 1297 :invoke :read [12 nil] INFO [2019-12-17 12:28:47,814] jepsen worker 89 - jepsen.util 1529 :ok :read [29 5] INFO [2019-12-17 12:28:47,814] jepsen worker 89 - jepsen.util 1529 :invoke :cas [29 [43 23]] INFO [2019-12-17 12:28:47,823] jepsen worker 33 - jepsen.util 2103 :info :write [11 43] :RequestCanceledException INFO [2019-12-17 12:28:47,827] jepsen worker 77 - jepsen.util 1877 :info :write [25 42] :RequestCanceledException INFO [2019-12-17 12:28:47,834] jepsen worker 4 - jepsen.util 1264 :fail :cas [1 [6 47]] :ValueNotSwapFrom INFO [2019-12-17 12:28:47,834] jepsen worker 4 - jepsen.util 1264 :invoke :write [1 39] INFO [2019-12-17 12:28:47,834] jepsen worker 62 - jepsen.util 1862 :info :write [20 27] :RequestCanceledException INFO [2019-12-17 12:28:47,836] jepsen worker 62 - jepsen.util 1952 :invoke :read [20 nil] INFO [2019-12-17 12:28:47,836] jepsen worker 77 - jepsen.util 1967 :invoke :read [25 nil] INFO [2019-12-17 12:28:47,836] jepsen worker 33 - jepsen.util 2193 :invoke :cas [11 [44 12]] INFO [2019-12-17 12:28:47,854] jepsen worker 10 - jepsen.util 1450 :info :write [3 26] :RequestCanceledException INFO [2019-12-17 12:28:47,860] jepsen worker 10 - jepsen.util 1540 :invoke :cas [3 [10 20]] INFO [2019-12-17 12:28:47,891] jepsen worker 21 - jepsen.util 1821 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:28:47,892] jepsen worker 21 - jepsen.util 1821 :invoke :read [7 nil] INFO [2019-12-17 12:28:47,916] jepsen worker 18 - jepsen.util 1728 :ok :read [6 40] INFO [2019-12-17 12:28:47,916] jepsen worker 18 - jepsen.util 1728 :invoke :read [6 nil] INFO [2019-12-17 12:28:47,922] jepsen worker 9 - jepsen.util 1539 :fail :cas [3 [46 43]] :ValueNotSwapFrom INFO [2019-12-17 12:28:47,923] jepsen worker 9 - jepsen.util 1539 :invoke :cas [3 [25 13]] INFO [2019-12-17 12:28:47,923] jepsen worker 45 - jepsen.util 1665 :ok :read [15 4] INFO [2019-12-17 12:28:47,923] jepsen worker 45 - jepsen.util 1665 :invoke :cas [15 [3 46]] INFO [2019-12-17 12:28:47,924] jepsen worker 65 - jepsen.util 1235 :fail :cas [21 [26 46]] :ValueNotSwapFrom INFO [2019-12-17 12:28:47,925] jepsen worker 65 - jepsen.util 1235 :invoke :write [21 47] INFO [2019-12-17 12:28:47,942] jepsen worker 27 - jepsen.util 1287 :ok :read [9 10] INFO [2019-12-17 12:28:47,942] jepsen worker 47 - jepsen.util 1847 :fail :cas [15 [18 25]] :ValueNotSwapFrom INFO [2019-12-17 12:28:47,942] jepsen worker 47 - jepsen.util 1847 :invoke :cas [15 [34 41]] INFO [2019-12-17 12:28:47,943] jepsen worker 27 - jepsen.util 1287 :invoke :cas [9 [42 29]] INFO [2019-12-17 12:28:47,946] jepsen worker 64 - jepsen.util 1054 :fail :cas [21 [5 46]] :ValueNotSwapFrom INFO [2019-12-17 12:28:47,947] jepsen worker 64 - jepsen.util 1054 :invoke :write [21 41] INFO [2019-12-17 12:28:47,948] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8305,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:28:47,948] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8305,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:28:47,948] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8309,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:28:47,948] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8309,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:28:47,948] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8310,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:28:47,948] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8310,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:28:47,952] jepsen worker 19 - jepsen.util 1639 :fail :cas [6 [5 42]] :ValueNotSwapFrom INFO [2019-12-17 12:28:47,953] jepsen worker 19 - jepsen.util 1639 :invoke :cas [6 [49 9]] INFO [2019-12-17 12:28:47,968] jepsen worker 71 - jepsen.util 1151 :ok :read [23 32] INFO [2019-12-17 12:28:47,969] jepsen worker 71 - jepsen.util 1151 :invoke :write [23 7] INFO [2019-12-17 12:28:48,006] jepsen worker 14 - jepsen.util 1364 :info :write [4 36] :RequestCanceledException INFO [2019-12-17 12:28:48,008] jepsen worker 14 - jepsen.util 1454 :invoke :cas [4 [47 30]] INFO [2019-12-17 12:28:48,053] jepsen worker 8 - jepsen.util 1898 :info :cas [2 [10 21]] :RequestCanceledException INFO [2019-12-17 12:28:48,057] jepsen worker 8 - jepsen.util 1988 :invoke :read [2 nil] INFO [2019-12-17 12:28:48,076] jepsen worker 18 - jepsen.util 1728 :ok :read [6 40] INFO [2019-12-17 12:28:48,076] jepsen worker 18 - jepsen.util 1728 :invoke :cas [6 [32 14]] INFO [2019-12-17 12:28:48,085] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8325,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:28:48,085] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8325,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:28:48,086] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8328,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:28:48,086] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8328,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:28:48,087] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8329,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:28:48,087] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8329,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:28:48,088] jepsen worker 44 - jepsen.util 1574 :info :write [14 11] :RequestCanceledException INFO [2019-12-17 12:28:48,093] jepsen worker 44 - jepsen.util 1664 :invoke :read [14 nil] INFO [2019-12-17 12:28:48,110] jepsen worker 78 - jepsen.util 1608 :fail :cas [26 [0 36]] :ValueNotSwapFrom INFO [2019-12-17 12:28:48,110] jepsen worker 78 - jepsen.util 1608 :invoke :read [26 nil] INFO [2019-12-17 12:28:48,112] jepsen worker 77 - jepsen.util 1967 :ok :read [25 2] INFO [2019-12-17 12:28:48,112] jepsen worker 77 - jepsen.util 1967 :invoke :read [25 nil] INFO [2019-12-17 12:28:48,123] jepsen worker 38 - jepsen.util 1388 :ok :write [12 42] INFO [2019-12-17 12:28:48,123] jepsen worker 38 - jepsen.util 1388 :invoke :write [12 30] INFO [2019-12-17 12:28:48,132] jepsen worker 9 - jepsen.util 1539 :fail :cas [3 [25 13]] :ValueNotSwapFrom INFO [2019-12-17 12:28:48,132] jepsen worker 9 - jepsen.util 1539 :invoke :cas [3 [12 28]] INFO [2019-12-17 12:28:48,136] jepsen worker 65 - jepsen.util 1235 :ok :write [21 47] INFO [2019-12-17 12:28:48,137] jepsen worker 65 - jepsen.util 1235 :invoke :cas [21 [14 11]] INFO [2019-12-17 12:28:48,138] jepsen worker 18 - jepsen.util 1728 :fail :cas [6 [32 14]] :ValueNotSwapFrom INFO [2019-12-17 12:28:48,138] jepsen worker 18 - jepsen.util 1728 :invoke :read [6 nil] INFO [2019-12-17 12:28:48,181] jepsen worker 27 - jepsen.util 1287 :fail :cas [9 [42 29]] :ValueNotSwapFrom INFO [2019-12-17 12:28:48,181] jepsen worker 27 - jepsen.util 1287 :invoke :read [9 nil] INFO [2019-12-17 12:28:48,188] jepsen worker 43 - jepsen.util 1933 :info :cas [14 [26 37]] :RequestCanceledException INFO [2019-12-17 12:28:48,190] jepsen worker 5 - jepsen.util 1175 :ok :write [1 11] INFO [2019-12-17 12:28:48,190] jepsen worker 5 - jepsen.util 1175 :invoke :write [1 39] INFO [2019-12-17 12:28:48,190] jepsen worker 43 - jepsen.util 2023 :invoke :read [14 nil] INFO [2019-12-17 12:28:48,206] jepsen worker 18 - jepsen.util 1728 :ok :read [6 40] INFO [2019-12-17 12:28:48,206] jepsen worker 18 - jepsen.util 1728 :invoke :read [6 nil] INFO [2019-12-17 12:28:48,211] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8343,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:28:48,212] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8343,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:28:48,213] jepsen worker 9 - jepsen.util 1539 :fail :cas [3 [12 28]] :ValueNotSwapFrom INFO [2019-12-17 12:28:48,213] jepsen worker 9 - jepsen.util 1539 :invoke :write [3 29] INFO [2019-12-17 12:28:48,229] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8350,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:28:48,229] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestTimeoutException: CarrierBucketConfigRequest {"retried":0,"reason":"TIMEOUT","requestId":8053,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:28:48,229] jepsen worker 56 - jepsen.util 1496 :fail :cas [18 [30 43]] :ValueNotSwapFrom INFO [2019-12-17 12:28:48,229] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8350,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:28:48,229] jepsen worker 56 - jepsen.util 1496 :invoke :write [18 35] INFO [2019-12-17 12:28:48,229] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8353,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:28:48,229] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8353,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:28:48,230] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8354,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:28:48,230] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8354,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:28:48,264] jepsen worker 43 - jepsen.util 2023 :ok :read [14 27] INFO [2019-12-17 12:28:48,264] jepsen worker 70 - jepsen.util 1780 :ok :write [23 19] INFO [2019-12-17 12:28:48,264] jepsen worker 43 - jepsen.util 2023 :invoke :cas [14 [17 9]] INFO [2019-12-17 12:28:48,266] jepsen worker 70 - jepsen.util 1780 :invoke :read [23 nil] INFO [2019-12-17 12:28:48,268] jepsen worker 28 - jepsen.util 928 :ok :write [9 29] INFO [2019-12-17 12:28:48,268] jepsen worker 28 - jepsen.util 928 :invoke :cas [9 [20 29]] INFO [2019-12-17 12:28:48,308] jepsen worker 65 - jepsen.util 1235 :fail :cas [21 [14 11]] :ValueNotSwapFrom INFO [2019-12-17 12:28:48,310] jepsen worker 65 - jepsen.util 1235 :invoke :cas [21 [25 15]] INFO [2019-12-17 12:28:48,319] jepsen worker 73 - jepsen.util 1513 :ok :write [24 7] INFO [2019-12-17 12:28:48,321] jepsen worker 73 - jepsen.util 1513 :invoke :cas [24 [31 46]] INFO [2019-12-17 12:28:48,326] jepsen worker 4 - jepsen.util 1264 :ok :write [1 39] INFO [2019-12-17 12:28:48,326] jepsen worker 4 - jepsen.util 1264 :invoke :cas [1 [12 11]] INFO [2019-12-17 12:28:48,354] jepsen worker 27 - jepsen.util 1287 :ok :read [9 10] INFO [2019-12-17 12:28:48,355] jepsen worker 27 - jepsen.util 1287 :invoke :cas [9 [14 34]] INFO [2019-12-17 12:28:48,371] jepsen worker 36 - jepsen.util 1476 :ok :read [12 42] INFO [2019-12-17 12:28:48,371] jepsen worker 28 - jepsen.util 928 :fail :cas [9 [20 29]] :ValueNotSwapFrom INFO [2019-12-17 12:28:48,371] jepsen worker 18 - jepsen.util 1728 :ok :read [6 40] INFO [2019-12-17 12:28:48,371] jepsen worker 77 - jepsen.util 1967 :ok :read [25 2] INFO [2019-12-17 12:28:48,371] jepsen worker 79 - jepsen.util 1519 :fail :cas [26 [27 7]] :CASMismatchException INFO [2019-12-17 12:28:48,371] jepsen worker 36 - jepsen.util 1476 :invoke :write [12 10] INFO [2019-12-17 12:28:48,371] jepsen worker 28 - jepsen.util 928 :invoke :read [9 nil] INFO [2019-12-17 12:28:48,372] jepsen worker 18 - jepsen.util 1728 :invoke :write [6 48] INFO [2019-12-17 12:28:48,372] jepsen worker 77 - jepsen.util 1967 :invoke :read [25 nil] INFO [2019-12-17 12:28:48,372] jepsen worker 79 - jepsen.util 1519 :invoke :cas [26 [14 24]] INFO [2019-12-17 12:28:48,381] jepsen worker 51 - jepsen.util 1491 :fail :cas [17 [6 14]] :ValueNotSwapFrom INFO [2019-12-17 12:28:48,381] jepsen worker 53 - jepsen.util 1493 :fail :cas [17 [0 20]] :ValueNotSwapFrom INFO [2019-12-17 12:28:48,382] jepsen worker 53 - jepsen.util 1493 :invoke :read [17 nil] INFO [2019-12-17 12:28:48,382] jepsen worker 51 - jepsen.util 1491 :invoke :cas [17 [10 1]] INFO [2019-12-17 12:28:48,385] jepsen worker 54 - jepsen.util 1674 :ok :write [18 37] INFO [2019-12-17 12:28:48,386] jepsen worker 54 - jepsen.util 1674 :invoke :cas [18 [46 20]] INFO [2019-12-17 12:28:48,456] jepsen worker 11 - jepsen.util 1721 :ok :read [3 7] INFO [2019-12-17 12:28:48,456] jepsen worker 11 - jepsen.util 1721 :invoke :write [3 27] INFO [2019-12-17 12:28:48,457] jepsen worker 25 - jepsen.util 2095 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:28:48,457] jepsen worker 25 - jepsen.util 2095 :invoke :write [8 22] INFO [2019-12-17 12:28:48,528] jepsen worker 74 - jepsen.util 1424 :ok :read [24 7] INFO [2019-12-17 12:28:48,528] jepsen worker 74 - jepsen.util 1424 :invoke :read [24 nil] INFO [2019-12-17 12:28:48,530] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8374,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:28:48,530] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8374,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:28:48,531] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8378,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:28:48,531] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8378,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:28:48,536] jepsen worker 26 - jepsen.util 1916 :info :write [8 34] :RequestCanceledException INFO [2019-12-17 12:28:48,540] jepsen worker 26 - jepsen.util 2006 :invoke :write [8 21] INFO [2019-12-17 12:28:48,540] jepsen worker 13 - jepsen.util 1363 :ok :write [4 35] INFO [2019-12-17 12:28:48,540] jepsen worker 73 - jepsen.util 1513 :fail :cas [24 [31 46]] :ValueNotSwapFrom INFO [2019-12-17 12:28:48,541] jepsen worker 13 - jepsen.util 1363 :invoke :cas [4 [8 19]] INFO [2019-12-17 12:28:48,541] jepsen worker 73 - jepsen.util 1513 :invoke :cas [24 [10 31]] INFO [2019-12-17 12:28:48,542] jepsen worker 87 - jepsen.util 1347 :ok :write [29 49] INFO [2019-12-17 12:28:48,542] jepsen worker 87 - jepsen.util 1347 :invoke :write [29 31] INFO [2019-12-17 12:28:48,551] jepsen worker 89 - jepsen.util 1529 :fail :cas [29 [43 23]] :ValueNotSwapFrom INFO [2019-12-17 12:28:48,551] jepsen worker 89 - jepsen.util 1529 :invoke :read [29 nil] INFO [2019-12-17 12:28:48,559] jepsen worker 28 - jepsen.util 928 :ok :read [9 29] INFO [2019-12-17 12:28:48,559] jepsen worker 28 - jepsen.util 928 :invoke :write [9 45] INFO [2019-12-17 12:28:48,560] jepsen worker 88 - jepsen.util 1348 :ok :read [29 5] INFO [2019-12-17 12:28:48,561] jepsen worker 88 - jepsen.util 1348 :invoke :cas [29 [0 48]] INFO [2019-12-17 12:28:48,569] jepsen worker 52 - jepsen.util 1582 :fail :cas [17 [14 33]] :ValueNotSwapFrom INFO [2019-12-17 12:28:48,569] jepsen worker 52 - jepsen.util 1582 :invoke :read [17 nil] INFO [2019-12-17 12:28:48,589] jepsen worker 55 - jepsen.util 1765 :info :write [18 26] :RequestCanceledException INFO [2019-12-17 12:28:48,592] jepsen worker 74 - jepsen.util 1424 :ok :read [24 7] INFO [2019-12-17 12:28:48,593] jepsen worker 74 - jepsen.util 1424 :invoke :cas [24 [41 6]] INFO [2019-12-17 12:28:48,596] jepsen worker 55 - jepsen.util 1855 :invoke :read [18 nil] INFO [2019-12-17 12:28:48,604] jepsen worker 43 - jepsen.util 2023 :fail :cas [14 [17 9]] :ValueNotSwapFrom INFO [2019-12-17 12:28:48,605] jepsen worker 43 - jepsen.util 2023 :invoke :read [14 nil] INFO [2019-12-17 12:28:48,611] jepsen worker 14 - jepsen.util 1454 :fail :cas [4 [47 30]] :ValueNotSwapFrom INFO [2019-12-17 12:28:48,611] jepsen worker 14 - jepsen.util 1454 :invoke :read [4 nil] INFO [2019-12-17 12:28:48,647] jepsen worker 73 - jepsen.util 1513 :fail :cas [24 [10 31]] :ValueNotSwapFrom INFO [2019-12-17 12:28:48,648] jepsen worker 30 - jepsen.util 1650 :ok :write [10 19] INFO [2019-12-17 12:28:48,648] jepsen worker 73 - jepsen.util 1513 :invoke :cas [24 [36 0]] INFO [2019-12-17 12:28:48,648] jepsen worker 30 - jepsen.util 1650 :invoke :write [10 29] INFO [2019-12-17 12:28:48,648] jepsen worker 12 - jepsen.util 1542 :info :write [4 30] :RequestCanceledException INFO [2019-12-17 12:28:48,649] jepsen worker 46 - jepsen.util 1756 :ok :write [15 17] INFO [2019-12-17 12:28:48,650] jepsen worker 46 - jepsen.util 1756 :invoke :read [15 nil] INFO [2019-12-17 12:28:48,653] jepsen worker 12 - jepsen.util 1632 :invoke :read [4 nil] INFO [2019-12-17 12:28:48,659] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:28:48,660] jepsen nemesis - couchbase.util Rebalance check count: 56 INFO [2019-12-17 12:28:48,660] jepsen nemesis - couchbase.util Rebalance stuck count: 37 INFO [2019-12-17 12:28:48,675] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8397,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:28:48,676] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8397,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:28:48,682] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8398,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:28:48,689] jepsen worker 28 - jepsen.util 928 :ok :write [9 45] INFO [2019-12-17 12:28:48,691] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8398,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:28:48,694] jepsen worker 28 - jepsen.util 928 :invoke :cas [9 [32 28]] INFO [2019-12-17 12:28:48,696] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8399,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:28:48,702] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8399,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:28:48,733] jepsen worker 29 - jepsen.util 1199 :ok :write [9 29] INFO [2019-12-17 12:28:48,766] jepsen worker 29 - jepsen.util 1199 :invoke :read [9 nil] INFO [2019-12-17 12:28:48,767] jepsen worker 7 - jepsen.util 2167 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:28:48,767] jepsen worker 7 - jepsen.util 2167 :invoke :read [2 nil] INFO [2019-12-17 12:28:48,780] jepsen worker 9 - jepsen.util 1539 :ok :write [3 29] INFO [2019-12-17 12:28:48,780] jepsen worker 9 - jepsen.util 1539 :invoke :write [3 8] INFO [2019-12-17 12:28:48,813] jepsen worker 85 - jepsen.util 1975 :info :cas [28 [6 5]] :RequestCanceledException INFO [2019-12-17 12:28:48,814] jepsen worker 31 - jepsen.util 1561 :info :cas [10 [9 11]] :RequestCanceledException INFO [2019-12-17 12:28:48,821] jepsen worker 18 - jepsen.util 1728 :ok :write [6 48] INFO [2019-12-17 12:28:48,824] jepsen worker 18 - jepsen.util 1728 :invoke :cas [6 [1 40]] INFO [2019-12-17 12:28:48,831] jepsen worker 82 - jepsen.util 2242 :info :write [27 30] :RequestCanceledException INFO [2019-12-17 12:28:48,848] jepsen worker 0 - jepsen.util 1980 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:28:48,860] jepsen worker 0 - jepsen.util 1980 :invoke :cas [0 [35 48]] INFO [2019-12-17 12:28:48,864] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8409,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:28:48,867] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8409,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:28:48,901] jepsen worker 71 - jepsen.util 1151 :ok :write [23 7] INFO [2019-12-17 12:28:48,929] jepsen worker 71 - jepsen.util 1151 :invoke :write [23 7] INFO [2019-12-17 12:28:48,965] jepsen worker 24 - jepsen.util 1734 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:28:48,965] jepsen worker 34 - jepsen.util 2104 :info :write [11 42] :RequestCanceledException INFO [2019-12-17 12:28:48,967] jepsen worker 24 - jepsen.util 1734 :invoke :cas [8 [6 29]] INFO [2019-12-17 12:28:48,971] jepsen worker 60 - jepsen.util 1950 :info :write [20 26] :RequestCanceledException INFO [2019-12-17 12:28:48,988] jepsen worker 86 - jepsen.util 1976 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:28:48,992] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8413,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:28:49,002] jepsen worker 86 - jepsen.util 1976 :invoke :cas [28 [8 42]] INFO [2019-12-17 12:28:49,012] jepsen worker 85 - jepsen.util 2065 :invoke :write [28 22] INFO [2019-12-17 12:28:49,013] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8416,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:28:49,013] jepsen worker 66 - jepsen.util 1596 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:28:49,013] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8416,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:28:49,031] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8413,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:28:49,034] jepsen worker 66 - jepsen.util 1596 :invoke :read [22 nil] INFO [2019-12-17 12:28:49,040] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8421,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:28:49,041] jepsen worker 67 - jepsen.util 2227 :info :cas [22 [21 9]] :RequestCanceledException INFO [2019-12-17 12:28:49,061] jepsen worker 57 - jepsen.util 1677 :info :cas [19 [23 19]] :RequestCanceledException INFO [2019-12-17 12:28:49,076] jepsen worker 84 - jepsen.util 1794 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:28:49,093] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8421,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:28:49,105] jepsen worker 32 - jepsen.util 1562 :info :cas [10 [6 11]] :RequestCanceledException INFO [2019-12-17 12:28:49,120] jepsen worker 41 - jepsen.util 1751 :info :cas [13 [0 31]] :RequestCanceledException INFO [2019-12-17 12:28:49,122] jepsen worker 84 - jepsen.util 1794 :invoke :cas [28 [38 10]] INFO [2019-12-17 12:28:49,123] jepsen worker 16 - jepsen.util 1816 :info :cas [5 [11 21]] :RequestCanceledException INFO [2019-12-17 12:28:49,125] jepsen worker 31 - jepsen.util 1651 :invoke :write [10 33] INFO [2019-12-17 12:28:49,127] jepsen worker 1 - jepsen.util 1801 :info :write [0 28] :RequestCanceledException INFO [2019-12-17 12:28:49,127] jepsen worker 59 - jepsen.util 1499 :info :cas [19 [29 32]] :RequestCanceledException INFO [2019-12-17 12:28:49,171] jepsen worker 50 - jepsen.util 1850 :info :write [16 2] :RequestCanceledException INFO [2019-12-17 12:28:49,171] jepsen worker 23 - jepsen.util 1733 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:28:49,172] jepsen worker 20 - jepsen.util 1460 :fail :read [6 nil] :RequestTimeoutException INFO [2019-12-17 12:28:49,173] jepsen worker 3 - jepsen.util 1353 :info :write [1 9] :RequestCanceledException INFO [2019-12-17 12:28:49,179] jepsen worker 48 - jepsen.util 1308 :info :write [16 16] :RequestCanceledException INFO [2019-12-17 12:28:49,186] jepsen worker 39 - jepsen.util 2019 :info :cas [13 [8 9]] :RequestCanceledException INFO [2019-12-17 12:28:49,192] jepsen worker 15 - jepsen.util 1545 :info :write [5 10] :RequestCanceledException INFO [2019-12-17 12:28:49,205] jepsen worker 2 - jepsen.util 1532 :info :cas [0 [40 35]] :RequestCanceledException INFO [2019-12-17 12:28:49,214] jepsen worker 17 - jepsen.util 1907 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:28:49,215] jepsen worker 23 - jepsen.util 1733 :invoke :cas [7 [21 8]] INFO [2019-12-17 12:28:49,215] jepsen worker 83 - jepsen.util 2153 :info :cas [27 [37 7]] :RequestCanceledException INFO [2019-12-17 12:28:49,217] jepsen worker 20 - jepsen.util 1460 :invoke :write [6 13] INFO [2019-12-17 12:28:49,217] jepsen worker 61 - jepsen.util 1861 :info :cas [20 [18 22]] :RequestCanceledException INFO [2019-12-17 12:28:49,218] jepsen worker 40 - jepsen.util 2110 :info :write [13 27] :RequestCanceledException INFO [2019-12-17 12:28:49,218] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8429,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:28:49,235] jepsen worker 80 - jepsen.util 1610 :info :write [26 19] :RequestCanceledException INFO [2019-12-17 12:28:49,241] jepsen worker 17 - jepsen.util 1907 :invoke :read [5 nil] INFO [2019-12-17 12:28:49,246] jepsen worker 34 - jepsen.util 2194 :invoke :cas [11 [12 11]] INFO [2019-12-17 12:28:49,248] jepsen worker 68 - jepsen.util 1418 :info :cas [22 [35 7]] :RequestCanceledException INFO [2019-12-17 12:28:49,257] jepsen worker 82 - jepsen.util 2332 :invoke :write [27 37] INFO [2019-12-17 12:28:49,257] jepsen worker 49 - jepsen.util 1309 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:28:49,257] jepsen worker 63 - jepsen.util 1233 :info :cas [21 [36 40]] :RequestCanceledException INFO [2019-12-17 12:28:49,264] jepsen worker 49 - jepsen.util 1309 :invoke :read [16 nil] INFO [2019-12-17 12:28:49,279] jepsen worker 81 - jepsen.util 1701 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:28:49,281] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8429,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:28:49,282] jepsen worker 69 - jepsen.util 1329 :info :write [23 4] :RequestCanceledException INFO [2019-12-17 12:28:49,283] jepsen worker 6 - jepsen.util 1896 :info :write [2 0] :RequestCanceledException INFO [2019-12-17 12:28:49,283] jepsen worker 61 - jepsen.util 1951 :invoke :cas [20 [44 27]] INFO [2019-12-17 12:28:49,283] jepsen worker 80 - jepsen.util 1700 :invoke :cas [26 [9 16]] INFO [2019-12-17 12:28:49,284] jepsen worker 81 - jepsen.util 1701 :invoke :read [27 nil] INFO [2019-12-17 12:28:49,286] jepsen worker 68 - jepsen.util 1508 :invoke :cas [22 [48 49]] INFO [2019-12-17 12:28:49,329] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8432,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:28:49,350] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8432,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:28:49,355] jepsen worker 60 - jepsen.util 2040 :invoke :write [20 42] INFO [2019-12-17 12:28:49,669] jepsen worker 76 - jepsen.util 1786 :info :write [25 42] :RequestCanceledException INFO [2019-12-17 12:28:49,669] jepsen worker 67 - jepsen.util 2317 :invoke :write [22 8] INFO [2019-12-17 12:28:49,670] jepsen worker 57 - jepsen.util 1767 :invoke :write [19 30] INFO [2019-12-17 12:28:49,680] jepsen worker 30 - jepsen.util 1650 :ok :write [10 29] INFO [2019-12-17 12:28:49,691] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8449,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:28:49,692] jepsen worker 74 - jepsen.util 1424 :fail :cas [24 [41 6]] :ValueNotSwapFrom INFO [2019-12-17 12:28:49,692] jepsen worker 74 - jepsen.util 1424 :invoke :read [24 nil] INFO [2019-12-17 12:28:49,692] jepsen worker 58 - jepsen.util 1588 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:28:49,695] jepsen worker 16 - jepsen.util 1906 :invoke :read [5 nil] INFO [2019-12-17 12:28:49,699] jepsen worker 42 - jepsen.util 2112 :info :write [14 25] :RequestCanceledException INFO [2019-12-17 12:28:49,700] jepsen worker 58 - jepsen.util 1588 :invoke :write [19 38] INFO [2019-12-17 12:28:49,702] jepsen worker 79 - jepsen.util 1519 :fail :cas [26 [14 24]] :ValueNotSwapFrom INFO [2019-12-17 12:28:49,707] jepsen worker 72 - jepsen.util 1422 :info :write [24 20] :RequestCanceledException INFO [2019-12-17 12:28:49,708] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8449,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:28:49,708] jepsen worker 30 - jepsen.util 1650 :invoke :write [10 2] INFO [2019-12-17 12:28:49,743] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8454,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:28:49,744] jepsen worker 3 - jepsen.util 1443 :invoke :read [1 nil] INFO [2019-12-17 12:28:49,744] jepsen worker 2 - jepsen.util 1622 :invoke :cas [0 [43 18]] INFO [2019-12-17 12:28:49,745] jepsen worker 35 - jepsen.util 1655 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:28:49,750] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8454,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:28:49,750] jepsen worker 79 - jepsen.util 1519 :invoke :write [26 5] INFO [2019-12-17 12:28:49,750] jepsen worker 1 - jepsen.util 1891 :invoke :write [0 26] INFO [2019-12-17 12:28:49,750] jepsen worker 15 - jepsen.util 1635 :invoke :cas [5 [16 39]] INFO [2019-12-17 12:28:49,752] jepsen worker 22 - jepsen.util 1912 :info :write [7 32] :RequestCanceledException INFO [2019-12-17 12:28:49,753] jepsen worker 35 - jepsen.util 1655 :invoke :read [11 nil] INFO [2019-12-17 12:28:49,754] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8452,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:28:49,756] jepsen worker 75 - jepsen.util 2055 :info :cas [25 [49 3]] :RequestCanceledException INFO [2019-12-17 12:28:49,762] jepsen worker 63 - jepsen.util 1323 :invoke :cas [21 [36 19]] INFO [2019-12-17 12:28:49,765] jepsen worker 39 - jepsen.util 2109 :invoke :read [13 nil] INFO [2019-12-17 12:28:49,785] jepsen worker 83 - jepsen.util 2243 :invoke :read [27 nil] INFO [2019-12-17 12:28:49,785] jepsen worker 48 - jepsen.util 1398 :invoke :write [16 17] INFO [2019-12-17 12:28:49,786] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8452,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:28:49,786] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:28:49,786] jepsen worker 6 - jepsen.util 1986 :invoke :read [2 nil] INFO [2019-12-17 12:28:49,787] jepsen worker 41 - jepsen.util 1841 :invoke :cas [13 [35 6]] INFO [2019-12-17 12:28:49,787] jepsen worker 32 - jepsen.util 1652 :invoke :cas [10 [4 1]] INFO [2019-12-17 12:28:49,789] jepsen worker 50 - jepsen.util 1940 :invoke :cas [16 [43 4]] INFO [2019-12-17 12:28:49,790] jepsen worker 40 - jepsen.util 2200 :invoke :cas [13 [46 1]] INFO [2019-12-17 12:28:49,792] jepsen worker 69 - jepsen.util 1419 :invoke :cas [23 [6 39]] INFO [2019-12-17 12:28:49,803] jepsen nemesis - couchbase.util Rebalance check count: 57 INFO [2019-12-17 12:28:49,804] jepsen nemesis - couchbase.util Rebalance stuck count: 38 INFO [2019-12-17 12:28:49,849] jepsen worker 76 - jepsen.util 1876 :invoke :cas [25 [11 39]] INFO [2019-12-17 12:28:49,866] jepsen worker 62 - jepsen.util 1952 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:28:49,867] jepsen worker 62 - jepsen.util 1952 :invoke :write [20 20] INFO [2019-12-17 12:28:49,872] jepsen worker 75 - jepsen.util 2145 :invoke :read [25 nil] INFO [2019-12-17 12:28:49,894] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8475,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:28:49,910] jepsen worker 22 - jepsen.util 2002 :invoke :write [7 45] INFO [2019-12-17 12:28:49,926] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8475,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:28:49,927] jepsen worker 33 - jepsen.util 2193 :info :cas [11 [44 12]] :RequestCanceledException INFO [2019-12-17 12:28:49,928] jepsen worker 59 - jepsen.util 1589 :invoke :read [19 nil] INFO [2019-12-17 12:28:49,933] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8477,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:28:49,944] jepsen worker 72 - jepsen.util 1512 :invoke :write [24 36] INFO [2019-12-17 12:28:49,959] jepsen worker 10 - jepsen.util 1540 :info :cas [3 [10 20]] :RequestCanceledException INFO [2019-12-17 12:28:50,018] jepsen worker 42 - jepsen.util 2202 :invoke :read [14 nil] INFO [2019-12-17 12:28:50,019] jepsen worker 21 - jepsen.util 1821 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:28:50,044] jepsen worker 47 - jepsen.util 1847 :info :cas [15 [34 41]] :RequestCanceledException INFO [2019-12-17 12:28:50,050] jepsen worker 21 - jepsen.util 1821 :invoke :write [7 21] INFO [2019-12-17 12:28:50,061] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8477,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:28:50,091] jepsen worker 19 - jepsen.util 1639 :info :cas [6 [49 9]] :RequestCanceledException INFO [2019-12-17 12:28:50,112] jepsen worker 45 - jepsen.util 1665 :info :cas [15 [3 46]] :RequestCanceledException INFO [2019-12-17 12:28:50,165] jepsen worker 64 - jepsen.util 1054 :info :write [21 41] :RequestCanceledException INFO [2019-12-17 12:28:50,168] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8487,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:28:50,176] jepsen worker 8 - jepsen.util 1988 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:28:50,178] jepsen worker 8 - jepsen.util 1988 :invoke :read [2 nil] INFO [2019-12-17 12:28:50,191] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8487,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:28:50,198] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8488,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:28:50,200] jepsen worker 44 - jepsen.util 1664 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:28:50,222] jepsen worker 44 - jepsen.util 1664 :invoke :read [14 nil] INFO [2019-12-17 12:28:50,226] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8488,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:28:50,229] jepsen worker 38 - jepsen.util 1388 :info :write [12 30] :RequestCanceledException INFO [2019-12-17 12:28:50,233] jepsen worker 10 - jepsen.util 1630 :invoke :write [3 46] INFO [2019-12-17 12:28:50,263] jepsen worker 5 - jepsen.util 1175 :info :write [1 39] :RequestCanceledException INFO [2019-12-17 12:28:50,274] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8491,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:28:50,275] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8491,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:28:50,286] jepsen worker 33 - jepsen.util 2283 :invoke :read [11 nil] INFO [2019-12-17 12:28:50,299] jepsen worker 47 - jepsen.util 1937 :invoke :write [15 47] INFO [2019-12-17 12:28:50,315] jepsen worker 38 - jepsen.util 1478 :invoke :write [12 46] INFO [2019-12-17 12:28:50,321] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8500,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:28:50,329] jepsen worker 19 - jepsen.util 1729 :invoke :write [6 25] INFO [2019-12-17 12:28:50,330] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8500,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:28:50,330] jepsen worker 45 - jepsen.util 1755 :invoke :cas [15 [18 23]] INFO [2019-12-17 12:28:50,351] jepsen worker 64 - jepsen.util 1144 :invoke :write [21 26] INFO [2019-12-17 12:28:50,362] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8502,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:28:50,366] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8502,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:28:50,424] jepsen worker 56 - jepsen.util 1496 :info :write [18 35] :RequestCanceledException INFO [2019-12-17 12:28:50,425] jepsen worker 5 - jepsen.util 1265 :invoke :read [1 nil] INFO [2019-12-17 12:28:50,541] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8512,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:28:50,570] jepsen worker 53 - jepsen.util 1493 :fail :read [17 nil] :RequestTimeoutException INFO [2019-12-17 12:28:50,584] jepsen worker 36 - jepsen.util 1476 :info :write [12 10] :RequestCanceledException INFO [2019-12-17 12:28:50,585] jepsen worker 25 - jepsen.util 2095 :info :write [8 22] :RequestCanceledException INFO [2019-12-17 12:28:50,613] jepsen worker 26 - jepsen.util 2006 :info :write [8 21] :RequestCanceledException INFO [2019-12-17 12:28:50,619] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8512,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:28:50,630] jepsen worker 87 - jepsen.util 1347 :info :write [29 31] :RequestCanceledException INFO [2019-12-17 12:28:50,633] jepsen worker 53 - jepsen.util 1493 :invoke :write [17 2] INFO [2019-12-17 12:28:50,636] jepsen worker 52 - jepsen.util 1582 :fail :read [17 nil] :RequestTimeoutException INFO [2019-12-17 12:28:50,640] jepsen worker 14 - jepsen.util 1454 :fail :read [4 nil] :RequestTimeoutException INFO [2019-12-17 12:28:50,649] jepsen worker 13 - jepsen.util 1363 :info :cas [4 [8 19]] :RequestCanceledException INFO [2019-12-17 12:28:50,653] jepsen worker 88 - jepsen.util 1348 :info :cas [29 [0 48]] :RequestCanceledException INFO [2019-12-17 12:28:50,657] jepsen worker 52 - jepsen.util 1582 :invoke :cas [17 [5 25]] INFO [2019-12-17 12:28:50,665] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8521,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:28:50,669] jepsen worker 43 - jepsen.util 2023 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:28:50,794] jepsen worker 14 - jepsen.util 1454 :invoke :cas [4 [11 31]] INFO [2019-12-17 12:28:50,795] jepsen worker 43 - jepsen.util 2023 :invoke :read [14 nil] INFO [2019-12-17 12:28:50,796] jepsen worker 11 - jepsen.util 1721 :info :write [3 27] :RequestCanceledException INFO [2019-12-17 12:28:50,797] jepsen worker 26 - jepsen.util 2096 :invoke :write [8 8] INFO [2019-12-17 12:28:50,808] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8521,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:28:50,808] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8527,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:28:50,810] jepsen worker 25 - jepsen.util 2185 :invoke :write [8 6] INFO [2019-12-17 12:28:50,814] jepsen worker 12 - jepsen.util 1632 :fail :read [4 nil] :RequestTimeoutException INFO [2019-12-17 12:28:50,814] jepsen worker 46 - jepsen.util 1756 :fail :read [15 nil] :RequestTimeoutException INFO [2019-12-17 12:28:50,841] jepsen worker 56 - jepsen.util 1586 :invoke :cas [18 [28 24]] INFO [2019-12-17 12:28:50,851] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8527,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:28:50,880] jepsen worker 46 - jepsen.util 1756 :invoke :write [15 32] INFO [2019-12-17 12:28:50,889] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8540,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:28:50,891] jepsen worker 36 - jepsen.util 1566 :invoke :write [12 32] INFO [2019-12-17 12:28:50,894] jepsen worker 12 - jepsen.util 1632 :invoke :write [4 19] INFO [2019-12-17 12:28:50,916] jepsen worker 87 - jepsen.util 1437 :invoke :write [29 0] INFO [2019-12-17 12:28:50,917] jepsen worker 7 - jepsen.util 2167 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:28:50,938] jepsen worker 7 - jepsen.util 2167 :invoke :read [2 nil] INFO [2019-12-17 12:28:50,947] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8540,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:28:51,020] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8547,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:28:51,058] jepsen worker 0 - jepsen.util 1980 :info :cas [0 [35 48]] :RequestCanceledException INFO [2019-12-17 12:28:51,076] jepsen worker 24 - jepsen.util 1734 :info :cas [8 [6 29]] :RequestCanceledException INFO [2019-12-17 12:28:51,091] jepsen worker 13 - jepsen.util 1453 :invoke :read [4 nil] INFO [2019-12-17 12:28:51,104] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:28:51,111] jepsen worker 66 - jepsen.util 1596 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:28:51,112] jepsen nemesis - couchbase.util Rebalance check count: 58 INFO [2019-12-17 12:28:51,113] jepsen worker 66 - jepsen.util 1596 :invoke :write [22 32] INFO [2019-12-17 12:28:51,116] jepsen nemesis - couchbase.util Rebalance stuck count: 39 INFO [2019-12-17 12:28:51,140] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8547,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:28:51,154] jepsen worker 86 - jepsen.util 1976 :info :cas [28 [8 42]] :RequestCanceledException INFO [2019-12-17 12:28:51,154] jepsen worker 11 - jepsen.util 1811 :invoke :read [3 nil] INFO [2019-12-17 12:28:51,154] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8551,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:28:51,161] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8551,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:28:51,167] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8552,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:28:51,178] jepsen worker 88 - jepsen.util 1438 :invoke :write [29 42] INFO [2019-12-17 12:28:51,181] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8552,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:28:51,208] jepsen worker 0 - jepsen.util 2070 :invoke :cas [0 [9 0]] INFO [2019-12-17 12:28:51,241] jepsen worker 24 - jepsen.util 1824 :invoke :read [8 nil] INFO [2019-12-17 12:28:51,270] jepsen worker 84 - jepsen.util 1794 :info :cas [28 [38 10]] :RequestCanceledException INFO [2019-12-17 12:28:51,292] jepsen worker 86 - jepsen.util 2066 :invoke :write [28 42] INFO [2019-12-17 12:28:51,294] jepsen worker 84 - jepsen.util 1884 :invoke :cas [28 [1 29]] INFO [2019-12-17 12:28:51,337] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8566,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:28:51,342] jepsen worker 17 - jepsen.util 1907 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:28:51,346] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8566,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:28:51,363] jepsen worker 23 - jepsen.util 1733 :info :cas [7 [21 8]] :RequestCanceledException INFO [2019-12-17 12:28:51,364] jepsen worker 49 - jepsen.util 1309 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:28:51,373] jepsen worker 17 - jepsen.util 1907 :invoke :cas [5 [4 44]] INFO [2019-12-17 12:28:51,431] jepsen worker 49 - jepsen.util 1309 :invoke :write [16 1] INFO [2019-12-17 12:28:51,446] jepsen worker 61 - jepsen.util 1951 :info :cas [20 [44 27]] :RequestCanceledException INFO [2019-12-17 12:28:51,478] jepsen worker 68 - jepsen.util 1508 :info :cas [22 [48 49]] :RequestCanceledException INFO [2019-12-17 12:28:51,482] jepsen worker 81 - jepsen.util 1701 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:28:51,482] jepsen worker 81 - jepsen.util 1701 :invoke :read [27 nil] INFO [2019-12-17 12:28:51,562] jepsen worker 77 - jepsen.util 1967 :ok :read [25 2] INFO [2019-12-17 12:28:51,603] jepsen worker 77 - jepsen.util 1967 :invoke :cas [25 [11 36]] INFO [2019-12-17 12:28:51,702] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8591,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:28:51,703] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8591,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:28:51,713] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8592,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:28:51,735] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8592,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:28:51,743] jepsen worker 51 - jepsen.util 1491 :fail :cas [17 [10 1]] :ValueNotSwapFrom INFO [2019-12-17 12:28:51,749] jepsen worker 70 - jepsen.util 1780 :ok :read [23 19] INFO [2019-12-17 12:28:51,757] jepsen worker 16 - jepsen.util 1906 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:28:51,789] jepsen worker 37 - jepsen.util 1297 :ok :read [12 42] INFO [2019-12-17 12:28:51,789] jepsen worker 23 - jepsen.util 1823 :invoke :write [7 22] INFO [2019-12-17 12:28:51,790] jepsen worker 28 - jepsen.util 928 :fail :cas [9 [32 28]] :ValueNotSwapFrom INFO [2019-12-17 12:28:51,790] jepsen worker 9 - jepsen.util 1539 :info :write [3 8] :RequestCanceledException INFO [2019-12-17 12:28:51,790] jepsen worker 55 - jepsen.util 1855 :ok :read [18 37] INFO [2019-12-17 12:28:51,790] jepsen worker 51 - jepsen.util 1491 :invoke :cas [17 [1 9]] INFO [2019-12-17 12:28:51,790] jepsen worker 70 - jepsen.util 1780 :invoke :read [23 nil] INFO [2019-12-17 12:28:51,791] jepsen worker 68 - jepsen.util 1598 :invoke :write [22 48] INFO [2019-12-17 12:28:51,791] jepsen worker 61 - jepsen.util 2041 :invoke :write [20 24] INFO [2019-12-17 12:28:51,791] jepsen worker 73 - jepsen.util 1513 :fail :cas [24 [36 0]] :ValueNotSwapFrom INFO [2019-12-17 12:28:51,792] jepsen worker 16 - jepsen.util 1906 :invoke :read [5 nil] INFO [2019-12-17 12:28:51,792] jepsen worker 65 - jepsen.util 1235 :fail :cas [21 [25 15]] :ValueNotSwapFrom INFO [2019-12-17 12:28:51,792] jepsen worker 89 - jepsen.util 1529 :ok :read [29 49] INFO [2019-12-17 12:28:51,793] jepsen worker 37 - jepsen.util 1297 :invoke :write [12 15] INFO [2019-12-17 12:28:51,795] jepsen worker 28 - jepsen.util 928 :invoke :read [9 nil] INFO [2019-12-17 12:28:51,797] jepsen worker 55 - jepsen.util 1855 :invoke :read [18 nil] INFO [2019-12-17 12:28:51,798] jepsen worker 2 - jepsen.util 1622 :info :cas [0 [43 18]] :RequestCanceledException INFO [2019-12-17 12:28:51,815] jepsen worker 9 - jepsen.util 1629 :invoke :cas [3 [6 12]] INFO [2019-12-17 12:28:51,816] jepsen worker 32 - jepsen.util 1652 :info :cas [10 [4 1]] :RequestCanceledException INFO [2019-12-17 12:28:51,816] jepsen worker 3 - jepsen.util 1443 :fail :read [1 nil] :RequestTimeoutException INFO [2019-12-17 12:28:51,817] jepsen worker 15 - jepsen.util 1635 :info :cas [5 [16 39]] :RequestCanceledException INFO [2019-12-17 12:28:51,819] jepsen worker 27 - jepsen.util 1287 :fail :cas [9 [14 34]] :ValueNotSwapFrom INFO [2019-12-17 12:28:51,820] jepsen worker 18 - jepsen.util 1728 :fail :cas [6 [1 40]] :ValueNotSwapFrom INFO [2019-12-17 12:28:51,821] jepsen worker 35 - jepsen.util 1655 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:28:51,823] jepsen worker 73 - jepsen.util 1513 :invoke :read [24 nil] INFO [2019-12-17 12:28:51,824] jepsen worker 65 - jepsen.util 1235 :invoke :write [21 18] INFO [2019-12-17 12:28:51,824] jepsen worker 89 - jepsen.util 1529 :invoke :cas [29 [33 23]] INFO [2019-12-17 12:28:51,827] jepsen worker 3 - jepsen.util 1443 :invoke :read [1 nil] INFO [2019-12-17 12:28:51,828] jepsen worker 18 - jepsen.util 1728 :invoke :write [6 23] INFO [2019-12-17 12:28:51,828] jepsen worker 27 - jepsen.util 1287 :invoke :cas [9 [41 38]] INFO [2019-12-17 12:28:51,829] jepsen worker 35 - jepsen.util 1655 :invoke :cas [11 [11 39]] INFO [2019-12-17 12:28:51,832] jepsen worker 32 - jepsen.util 1742 :invoke :cas [10 [13 23]] INFO [2019-12-17 12:28:51,833] jepsen worker 15 - jepsen.util 1725 :invoke :cas [5 [46 29]] INFO [2019-12-17 12:28:51,833] jepsen worker 2 - jepsen.util 1712 :invoke :read [0 nil] INFO [2019-12-17 12:28:51,835] jepsen worker 6 - jepsen.util 1986 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:28:51,838] jepsen worker 6 - jepsen.util 1986 :invoke :write [2 12] INFO [2019-12-17 12:28:51,839] jepsen worker 29 - jepsen.util 1199 :ok :read [9 29] INFO [2019-12-17 12:28:51,839] jepsen worker 29 - jepsen.util 1199 :invoke :read [9 nil] INFO [2019-12-17 12:28:51,839] jepsen worker 4 - jepsen.util 1264 :fail :cas [1 [12 11]] :ValueNotSwapFrom INFO [2019-12-17 12:28:51,839] jepsen worker 54 - jepsen.util 1674 :fail :cas [18 [46 20]] :ValueNotSwapFrom INFO [2019-12-17 12:28:51,839] jepsen worker 4 - jepsen.util 1264 :invoke :cas [1 [14 42]] INFO [2019-12-17 12:28:51,840] jepsen worker 54 - jepsen.util 1674 :invoke :read [18 nil] INFO [2019-12-17 12:28:51,840] jepsen worker 78 - jepsen.util 1608 :ok :read [26 44] INFO [2019-12-17 12:28:51,840] jepsen worker 78 - jepsen.util 1608 :invoke :cas [26 [45 46]] INFO [2019-12-17 12:28:51,841] jepsen worker 74 - jepsen.util 1424 :ok :read [24 7] INFO [2019-12-17 12:28:51,841] jepsen worker 74 - jepsen.util 1424 :invoke :write [24 4] INFO [2019-12-17 12:28:51,841] jepsen worker 45 - jepsen.util 1755 :fail :cas [15 [18 23]] :ValueNotSwapFrom INFO [2019-12-17 12:28:51,841] jepsen worker 45 - jepsen.util 1755 :invoke :write [15 6] INFO [2019-12-17 12:28:51,851] jepsen worker 41 - jepsen.util 1841 :info :cas [13 [35 6]] :RequestCanceledException INFO [2019-12-17 12:28:51,857] jepsen worker 41 - jepsen.util 1931 :invoke :cas [13 [21 24]] INFO [2019-12-17 12:28:51,858] jepsen worker 11 - jepsen.util 1811 :ok :read [3 29] INFO [2019-12-17 12:28:51,858] jepsen worker 11 - jepsen.util 1811 :invoke :read [3 nil] INFO [2019-12-17 12:28:51,864] jepsen worker 43 - jepsen.util 2023 :ok :read [14 27] INFO [2019-12-17 12:28:51,864] jepsen worker 80 - jepsen.util 1700 :fail :cas [26 [9 16]] :ValueNotSwapFrom INFO [2019-12-17 12:28:51,865] jepsen worker 43 - jepsen.util 2023 :invoke :write [14 0] INFO [2019-12-17 12:28:51,865] jepsen worker 34 - jepsen.util 2194 :fail :cas [11 [12 11]] :ValueNotSwapFrom INFO [2019-12-17 12:28:51,865] jepsen worker 80 - jepsen.util 1700 :invoke :write [26 16] INFO [2019-12-17 12:28:51,866] jepsen worker 34 - jepsen.util 2194 :invoke :cas [11 [24 16]] INFO [2019-12-17 12:28:51,866] jepsen worker 33 - jepsen.util 2283 :ok :read [11 7] INFO [2019-12-17 12:28:51,868] jepsen worker 33 - jepsen.util 2283 :invoke :write [11 36] INFO [2019-12-17 12:28:51,880] jepsen worker 39 - jepsen.util 2109 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:28:51,880] jepsen worker 50 - jepsen.util 1940 :info :cas [16 [43 4]] :RequestCanceledException INFO [2019-12-17 12:28:51,880] jepsen worker 39 - jepsen.util 2109 :invoke :cas [13 [13 47]] INFO [2019-12-17 12:28:51,880] jepsen worker 52 - jepsen.util 1582 :fail :cas [17 [5 25]] :ValueNotSwapFrom INFO [2019-12-17 12:28:51,880] jepsen worker 52 - jepsen.util 1582 :invoke :cas [17 [11 19]] INFO [2019-12-17 12:28:51,882] jepsen worker 50 - jepsen.util 2030 :invoke :cas [16 [3 4]] INFO [2019-12-17 12:28:51,899] jepsen worker 83 - jepsen.util 2243 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:28:51,899] jepsen worker 40 - jepsen.util 2200 :info :cas [13 [46 1]] :RequestCanceledException INFO [2019-12-17 12:28:51,900] jepsen worker 83 - jepsen.util 2243 :invoke :read [27 nil] INFO [2019-12-17 12:28:51,901] jepsen worker 9 - jepsen.util 1629 :fail :cas [3 [6 12]] :ValueNotSwapFrom INFO [2019-12-17 12:28:51,901] jepsen worker 9 - jepsen.util 1629 :invoke :write [3 27] INFO [2019-12-17 12:28:51,902] jepsen worker 40 - jepsen.util 2290 :invoke :cas [13 [26 27]] INFO [2019-12-17 12:28:51,916] jepsen worker 69 - jepsen.util 1419 :fail :cas [23 [6 39]] :ValueNotSwapFrom INFO [2019-12-17 12:28:51,917] jepsen worker 69 - jepsen.util 1419 :invoke :read [23 nil] INFO [2019-12-17 12:28:51,923] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8624,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:28:51,924] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8624,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:28:51,924] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8627,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:28:51,924] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8627,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:28:51,924] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8628,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:28:51,924] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8628,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:28:51,926] jepsen worker 5 - jepsen.util 1265 :ok :read [1 39] INFO [2019-12-17 12:28:51,926] jepsen worker 5 - jepsen.util 1265 :invoke :read [1 nil] INFO [2019-12-17 12:28:51,934] jepsen worker 76 - jepsen.util 1876 :info :cas [25 [11 39]] :RequestCanceledException INFO [2019-12-17 12:28:51,935] jepsen worker 76 - jepsen.util 1966 :invoke :write [25 11] INFO [2019-12-17 12:28:51,945] jepsen worker 73 - jepsen.util 1513 :ok :read [24 7] INFO [2019-12-17 12:28:51,945] jepsen worker 73 - jepsen.util 1513 :invoke :read [24 nil] INFO [2019-12-17 12:28:51,953] jepsen worker 75 - jepsen.util 2145 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:28:51,953] jepsen worker 75 - jepsen.util 2145 :invoke :cas [25 [3 9]] INFO [2019-12-17 12:28:51,984] jepsen worker 77 - jepsen.util 1967 :fail :cas [25 [11 36]] :ValueNotSwapFrom INFO [2019-12-17 12:28:51,984] jepsen worker 77 - jepsen.util 1967 :invoke :cas [25 [1 44]] INFO [2019-12-17 12:28:52,008] jepsen worker 59 - jepsen.util 1589 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:28:52,008] jepsen worker 59 - jepsen.util 1589 :invoke :write [19 24] INFO [2019-12-17 12:28:52,020] jepsen worker 14 - jepsen.util 1454 :fail :cas [4 [11 31]] :ValueNotSwapFrom INFO [2019-12-17 12:28:52,020] jepsen worker 14 - jepsen.util 1454 :invoke :read [4 nil] INFO [2019-12-17 12:28:52,021] jepsen worker 63 - jepsen.util 1323 :info :cas [21 [36 19]] :RequestCanceledException INFO [2019-12-17 12:28:52,028] jepsen worker 63 - jepsen.util 1413 :invoke :read [21 nil] INFO [2019-12-17 12:28:52,031] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8652,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:28:52,033] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8652,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:28:52,033] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8655,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:28:52,033] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8655,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:28:52,036] jepsen worker 42 - jepsen.util 2202 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:28:52,036] jepsen worker 42 - jepsen.util 2202 :invoke :cas [14 [32 6]] INFO [2019-12-17 12:28:52,058] jepsen worker 34 - jepsen.util 2194 :fail :cas [11 [24 16]] :ValueNotSwapFrom INFO [2019-12-17 12:28:52,062] jepsen worker 34 - jepsen.util 2194 :invoke :cas [11 [0 28]] INFO [2019-12-17 12:28:52,069] jepsen worker 79 - jepsen.util 1519 :ok :write [26 5] INFO [2019-12-17 12:28:52,069] jepsen worker 79 - jepsen.util 1519 :invoke :write [26 31] INFO [2019-12-17 12:28:52,109] jepsen worker 74 - jepsen.util 1424 :ok :write [24 4] INFO [2019-12-17 12:28:52,111] jepsen worker 74 - jepsen.util 1424 :invoke :cas [24 [28 48]] INFO [2019-12-17 12:28:52,194] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:28:52,194] jepsen nemesis - couchbase.util Rebalance check count: 59 INFO [2019-12-17 12:28:52,194] jepsen nemesis - couchbase.util Rebalance stuck count: 40 INFO [2019-12-17 12:28:52,205] jepsen worker 46 - jepsen.util 1756 :ok :write [15 32] INFO [2019-12-17 12:28:52,209] jepsen worker 46 - jepsen.util 1756 :invoke :write [15 33] INFO [2019-12-17 12:28:52,218] jepsen worker 31 - jepsen.util 1651 :ok :write [10 33] INFO [2019-12-17 12:28:52,219] jepsen worker 31 - jepsen.util 1651 :invoke :read [10 nil] INFO [2019-12-17 12:28:52,239] jepsen worker 71 - jepsen.util 1151 :info :write [23 7] :RequestCanceledException INFO [2019-12-17 12:28:52,239] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8664,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:28:52,241] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8664,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:28:52,241] jepsen worker 8 - jepsen.util 1988 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:28:52,241] jepsen worker 8 - jepsen.util 1988 :invoke :read [2 nil] INFO [2019-12-17 12:28:52,246] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8665,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:28:52,252] jepsen worker 71 - jepsen.util 1241 :invoke :cas [23 [18 40]] INFO [2019-12-17 12:28:52,258] jepsen worker 44 - jepsen.util 1664 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:28:52,258] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8665,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:28:52,259] jepsen worker 44 - jepsen.util 1664 :invoke :read [14 nil] INFO [2019-12-17 12:28:52,264] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8669,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:28:52,269] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8669,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:28:52,289] jepsen worker 82 - jepsen.util 2332 :info :write [27 37] :RequestCanceledException INFO [2019-12-17 12:28:52,293] jepsen worker 82 - jepsen.util 2422 :invoke :cas [27 [3 47]] INFO [2019-12-17 12:28:52,295] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8670,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:28:52,295] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8670,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:28:52,296] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8674,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:28:52,296] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8674,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:28:52,296] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8675,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:28:52,296] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8675,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:28:52,302] jepsen worker 13 - jepsen.util 1453 :ok :read [4 36] INFO [2019-12-17 12:28:52,303] jepsen worker 13 - jepsen.util 1453 :invoke :cas [4 [34 42]] INFO [2019-12-17 12:28:52,306] jepsen worker 56 - jepsen.util 1586 :fail :cas [18 [28 24]] :ValueNotSwapFrom INFO [2019-12-17 12:28:52,306] jepsen worker 56 - jepsen.util 1586 :invoke :read [18 nil] INFO [2019-12-17 12:28:52,306] jepsen worker 89 - jepsen.util 1529 :fail :cas [29 [33 23]] :ValueNotSwapFrom INFO [2019-12-17 12:28:52,307] jepsen worker 89 - jepsen.util 1529 :invoke :write [29 19] INFO [2019-12-17 12:28:52,316] jepsen worker 5 - jepsen.util 1265 :ok :read [1 39] INFO [2019-12-17 12:28:52,316] jepsen worker 5 - jepsen.util 1265 :invoke :write [1 47] INFO [2019-12-17 12:28:52,319] jepsen worker 29 - jepsen.util 1199 :ok :read [9 29] INFO [2019-12-17 12:28:52,319] jepsen worker 29 - jepsen.util 1199 :invoke :cas [9 [39 22]] INFO [2019-12-17 12:28:52,319] jepsen worker 31 - jepsen.util 1651 :ok :read [10 33] INFO [2019-12-17 12:28:52,319] jepsen worker 31 - jepsen.util 1651 :invoke :read [10 nil] INFO [2019-12-17 12:28:52,322] jepsen worker 77 - jepsen.util 1967 :fail :cas [25 [1 44]] :ValueNotSwapFrom INFO [2019-12-17 12:28:52,322] jepsen worker 77 - jepsen.util 1967 :invoke :cas [25 [16 19]] INFO [2019-12-17 12:28:52,346] jepsen worker 31 - jepsen.util 1651 :ok :read [10 33] INFO [2019-12-17 12:28:52,346] jepsen worker 31 - jepsen.util 1651 :invoke :read [10 nil] INFO [2019-12-17 12:28:52,358] jepsen worker 12 - jepsen.util 1632 :ok :write [4 19] INFO [2019-12-17 12:28:52,358] jepsen worker 12 - jepsen.util 1632 :invoke :read [4 nil] INFO [2019-12-17 12:28:52,389] jepsen worker 71 - jepsen.util 1241 :fail :cas [23 [18 40]] :ValueNotSwapFrom INFO [2019-12-17 12:28:52,390] jepsen worker 71 - jepsen.util 1241 :invoke :write [23 26] INFO [2019-12-17 12:28:52,437] jepsen worker 85 - jepsen.util 2065 :info :write [28 22] :RequestCanceledException INFO [2019-12-17 12:28:52,462] jepsen worker 53 - jepsen.util 1493 :ok :write [17 2] INFO [2019-12-17 12:28:52,462] jepsen worker 53 - jepsen.util 1493 :invoke :write [17 43] INFO [2019-12-17 12:28:52,463] jepsen worker 87 - jepsen.util 1437 :ok :write [29 0] INFO [2019-12-17 12:28:52,464] jepsen worker 87 - jepsen.util 1437 :invoke :cas [29 [8 17]] INFO [2019-12-17 12:28:52,465] jepsen worker 85 - jepsen.util 2155 :invoke :write [28 25] INFO [2019-12-17 12:28:52,521] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8693,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:28:52,521] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8693,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:28:52,522] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8695,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:28:52,522] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8695,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:28:52,523] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8696,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:28:52,523] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8696,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:28:52,529] jepsen worker 30 - jepsen.util 1650 :ok :write [10 2] INFO [2019-12-17 12:28:52,529] jepsen worker 30 - jepsen.util 1650 :invoke :cas [10 [15 4]] INFO [2019-12-17 12:28:52,540] jepsen worker 31 - jepsen.util 1651 :ok :read [10 33] INFO [2019-12-17 12:28:52,541] jepsen worker 31 - jepsen.util 1651 :invoke :cas [10 [31 13]] INFO [2019-12-17 12:28:52,579] jepsen worker 20 - jepsen.util 1460 :ok :write [6 13] INFO [2019-12-17 12:28:52,579] jepsen worker 20 - jepsen.util 1460 :invoke :read [6 nil] INFO [2019-12-17 12:28:52,637] jepsen worker 46 - jepsen.util 1756 :ok :write [15 33] INFO [2019-12-17 12:28:52,637] jepsen worker 46 - jepsen.util 1756 :invoke :write [15 25] INFO [2019-12-17 12:28:52,642] jepsen worker 72 - jepsen.util 1512 :ok :write [24 36] INFO [2019-12-17 12:28:52,642] jepsen worker 72 - jepsen.util 1512 :invoke :cas [24 [49 47]] INFO [2019-12-17 12:28:52,688] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8708,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:28:52,691] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8708,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:28:52,692] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8710,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:28:52,693] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8710,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:28:52,712] jepsen worker 19 - jepsen.util 1729 :ok :write [6 25] INFO [2019-12-17 12:28:52,713] jepsen worker 19 - jepsen.util 1729 :invoke :read [6 nil] INFO [2019-12-17 12:28:52,928] jepsen worker 31 - jepsen.util 1651 :fail :cas [10 [31 13]] :ValueNotSwapFrom INFO [2019-12-17 12:28:52,929] jepsen worker 31 - jepsen.util 1651 :invoke :write [10 3] INFO [2019-12-17 12:28:52,931] jepsen worker 46 - jepsen.util 1756 :ok :write [15 25] INFO [2019-12-17 12:28:52,931] jepsen worker 46 - jepsen.util 1756 :invoke :cas [15 [28 43]] INFO [2019-12-17 12:28:52,932] jepsen worker 1 - jepsen.util 1891 :info :write [0 26] :RequestCanceledException INFO [2019-12-17 12:28:52,945] jepsen worker 58 - jepsen.util 1588 :info :write [19 38] :RequestCanceledException INFO [2019-12-17 12:28:52,955] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8716,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:28:52,956] jepsen worker 10 - jepsen.util 1630 :info :write [3 46] :RequestCanceledException INFO [2019-12-17 12:28:52,958] jepsen worker 57 - jepsen.util 1767 :info :write [19 30] :RequestCanceledException INFO [2019-12-17 12:28:52,958] jepsen worker 48 - jepsen.util 1398 :info :write [16 17] :RequestCanceledException INFO [2019-12-17 12:28:52,958] jepsen worker 60 - jepsen.util 2040 :info :write [20 42] :RequestCanceledException INFO [2019-12-17 12:28:52,958] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8716,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:28:52,959] jepsen worker 21 - jepsen.util 1821 :info :write [7 21] :RequestCanceledException INFO [2019-12-17 12:28:52,961] jepsen worker 62 - jepsen.util 1952 :info :write [20 20] :RequestCanceledException INFO [2019-12-17 12:28:52,973] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8717,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:28:52,976] jepsen worker 7 - jepsen.util 2167 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:28:52,978] jepsen worker 7 - jepsen.util 2167 :invoke :cas [2 [41 25]] INFO [2019-12-17 12:28:52,980] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8717,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:28:52,982] jepsen worker 22 - jepsen.util 2002 :info :write [7 45] :RequestCanceledException INFO [2019-12-17 12:28:52,985] jepsen worker 67 - jepsen.util 2317 :info :write [22 8] :RequestCanceledException INFO [2019-12-17 12:28:53,008] jepsen worker 89 - jepsen.util 1529 :ok :write [29 19] INFO [2019-12-17 12:28:53,011] jepsen worker 89 - jepsen.util 1529 :invoke :cas [29 [15 45]] INFO [2019-12-17 12:28:53,017] jepsen worker 48 - jepsen.util 1488 :invoke :read [16 nil] INFO [2019-12-17 12:28:53,031] jepsen worker 57 - jepsen.util 1857 :invoke :read [19 nil] INFO [2019-12-17 12:28:53,033] jepsen worker 62 - jepsen.util 2042 :invoke :cas [20 [5 10]] INFO [2019-12-17 12:28:53,041] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8723,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:28:53,043] jepsen worker 21 - jepsen.util 1911 :invoke :cas [7 [28 32]] INFO [2019-12-17 12:28:53,044] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8723,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:28:53,045] jepsen worker 58 - jepsen.util 1678 :invoke :write [19 9] INFO [2019-12-17 12:28:53,046] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8726,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:28:53,048] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8726,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:28:53,048] jepsen worker 10 - jepsen.util 1720 :invoke :cas [3 [2 0]] INFO [2019-12-17 12:28:53,061] jepsen worker 22 - jepsen.util 2092 :invoke :cas [7 [39 10]] INFO [2019-12-17 12:28:53,068] jepsen worker 60 - jepsen.util 2130 :invoke :cas [20 [5 45]] INFO [2019-12-17 12:28:53,076] jepsen worker 67 - jepsen.util 2407 :invoke :read [22 nil] INFO [2019-12-17 12:28:53,081] jepsen worker 1 - jepsen.util 1981 :invoke :cas [0 [5 42]] INFO [2019-12-17 12:28:53,168] jepsen worker 53 - jepsen.util 1493 :ok :write [17 43] INFO [2019-12-17 12:28:53,171] jepsen worker 53 - jepsen.util 1493 :invoke :read [17 nil] INFO [2019-12-17 12:28:53,216] jepsen worker 80 - jepsen.util 1700 :ok :write [26 16] INFO [2019-12-17 12:28:53,218] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8744,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:28:53,219] jepsen worker 80 - jepsen.util 1700 :invoke :cas [26 [42 43]] INFO [2019-12-17 12:28:53,234] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8744,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:28:53,235] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8746,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:28:53,235] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8746,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:28:53,235] jepsen worker 20 - jepsen.util 1460 :ok :read [6 13] INFO [2019-12-17 12:28:53,236] jepsen worker 20 - jepsen.util 1460 :invoke :read [6 nil] INFO [2019-12-17 12:28:53,243] jepsen worker 0 - jepsen.util 2070 :info :cas [0 [9 0]] :RequestCanceledException INFO [2019-12-17 12:28:53,276] jepsen worker 71 - jepsen.util 1241 :ok :write [23 26] INFO [2019-12-17 12:28:53,277] jepsen worker 71 - jepsen.util 1241 :invoke :cas [23 [43 27]] INFO [2019-12-17 12:28:53,277] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8739,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:28:53,277] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8739,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:28:53,277] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8740,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:28:53,277] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8740,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:28:53,297] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:28:53,298] jepsen nemesis - couchbase.util Rebalance check count: 60 INFO [2019-12-17 12:28:53,298] jepsen nemesis - couchbase.util Rebalance stuck count: 41 INFO [2019-12-17 12:28:53,301] jepsen worker 51 - jepsen.util 1491 :fail :cas [17 [1 9]] :ValueNotSwapFrom INFO [2019-12-17 12:28:53,301] jepsen worker 51 - jepsen.util 1491 :invoke :write [17 2] INFO [2019-12-17 12:28:53,302] jepsen worker 54 - jepsen.util 1674 :ok :read [18 35] INFO [2019-12-17 12:28:53,302] jepsen worker 54 - jepsen.util 1674 :invoke :read [18 nil] INFO [2019-12-17 12:28:53,305] jepsen worker 0 - jepsen.util 2160 :invoke :write [0 13] INFO [2019-12-17 12:28:53,327] jepsen worker 84 - jepsen.util 1884 :info :cas [28 [1 29]] :RequestCanceledException INFO [2019-12-17 12:28:53,341] jepsen worker 24 - jepsen.util 1824 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:28:53,346] jepsen worker 24 - jepsen.util 1824 :invoke :cas [8 [13 31]] INFO [2019-12-17 12:28:53,360] jepsen worker 64 - jepsen.util 1144 :info :write [21 26] :RequestCanceledException INFO [2019-12-17 12:28:53,364] jepsen worker 37 - jepsen.util 1297 :ok :write [12 15] INFO [2019-12-17 12:28:53,364] jepsen worker 37 - jepsen.util 1297 :invoke :read [12 nil] INFO [2019-12-17 12:28:53,369] jepsen worker 84 - jepsen.util 1974 :invoke :cas [28 [46 9]] INFO [2019-12-17 12:28:53,390] jepsen worker 47 - jepsen.util 1937 :info :write [15 47] :RequestCanceledException INFO [2019-12-17 12:28:53,415] jepsen worker 36 - jepsen.util 1566 :ok :write [12 32] INFO [2019-12-17 12:28:53,417] jepsen worker 38 - jepsen.util 1478 :info :write [12 46] :RequestCanceledException INFO [2019-12-17 12:28:53,420] jepsen worker 36 - jepsen.util 1566 :invoke :cas [12 [17 48]] INFO [2019-12-17 12:28:53,421] jepsen worker 64 - jepsen.util 1234 :invoke :write [21 6] INFO [2019-12-17 12:28:53,491] jepsen worker 38 - jepsen.util 1568 :invoke :read [12 nil] INFO [2019-12-17 12:28:53,522] jepsen worker 17 - jepsen.util 1907 :info :cas [5 [4 44]] :RequestCanceledException INFO [2019-12-17 12:28:53,534] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8764,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:28:53,541] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8764,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:28:53,542] jepsen worker 47 - jepsen.util 2027 :invoke :write [15 49] INFO [2019-12-17 12:28:53,567] jepsen worker 81 - jepsen.util 1701 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:28:53,567] jepsen worker 81 - jepsen.util 1701 :invoke :cas [27 [43 37]] INFO [2019-12-17 12:28:53,583] jepsen worker 19 - jepsen.util 1729 :ok :read [6 25] INFO [2019-12-17 12:28:53,583] jepsen worker 17 - jepsen.util 1997 :invoke :cas [5 [4 40]] INFO [2019-12-17 12:28:53,596] jepsen worker 26 - jepsen.util 2096 :info :write [8 8] :RequestCanceledException INFO [2019-12-17 12:28:53,597] jepsen worker 19 - jepsen.util 1729 :invoke :write [6 1] INFO [2019-12-17 12:28:53,640] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8772,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:28:53,640] jepsen worker 26 - jepsen.util 2186 :invoke :cas [8 [24 23]] INFO [2019-12-17 12:28:53,643] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8772,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:28:53,648] jepsen worker 78 - jepsen.util 1608 :fail :cas [26 [45 46]] :ValueNotSwapFrom INFO [2019-12-17 12:28:53,648] jepsen worker 78 - jepsen.util 1608 :invoke :read [26 nil] INFO [2019-12-17 12:28:53,649] jepsen worker 70 - jepsen.util 1780 :ok :read [23 7] INFO [2019-12-17 12:28:53,651] jepsen worker 70 - jepsen.util 1780 :invoke :cas [23 [29 3]] INFO [2019-12-17 12:28:53,672] jepsen worker 5 - jepsen.util 1265 :ok :write [1 47] INFO [2019-12-17 12:28:53,678] jepsen worker 5 - jepsen.util 1265 :invoke :write [1 31] INFO [2019-12-17 12:28:53,757] jepsen worker 88 - jepsen.util 1438 :ok :write [29 42] INFO [2019-12-17 12:28:53,762] jepsen worker 88 - jepsen.util 1438 :invoke :write [29 10] INFO [2019-12-17 12:28:53,764] jepsen worker 31 - jepsen.util 1651 :ok :write [10 3] INFO [2019-12-17 12:28:53,767] jepsen worker 31 - jepsen.util 1651 :invoke :write [10 36] INFO [2019-12-17 12:28:53,813] jepsen worker 25 - jepsen.util 2185 :info :write [8 6] :RequestCanceledException INFO [2019-12-17 12:28:53,854] jepsen worker 86 - jepsen.util 2066 :info :write [28 42] :RequestCanceledException INFO [2019-12-17 12:28:53,881] jepsen worker 49 - jepsen.util 1309 :info :write [16 1] :RequestCanceledException INFO [2019-12-17 12:28:53,904] jepsen worker 18 - jepsen.util 1728 :info :write [6 23] :RequestCanceledException INFO [2019-12-17 12:28:53,911] jepsen worker 28 - jepsen.util 928 :fail :read [9 nil] :RequestTimeoutException INFO [2019-12-17 12:28:53,932] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8783,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:28:54,410] jepsen worker 23 - jepsen.util 1823 :info :write [7 22] :RequestCanceledException INFO [2019-12-17 12:28:54,410] jepsen worker 32 - jepsen.util 1742 :info :cas [10 [13 23]] :RequestCanceledException INFO [2019-12-17 12:28:54,410] jepsen worker 16 - jepsen.util 1906 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:28:54,410] jepsen worker 3 - jepsen.util 1443 :ok :read [1 39] INFO [2019-12-17 12:28:54,413] jepsen worker 69 - jepsen.util 1419 :ok :read [23 7] INFO [2019-12-17 12:28:54,414] jepsen worker 13 - jepsen.util 1453 :fail :cas [4 [34 42]] :ValueNotSwapFrom INFO [2019-12-17 12:28:54,415] jepsen worker 14 - jepsen.util 1454 :ok :read [4 19] INFO [2019-12-17 12:28:54,415] jepsen worker 2 - jepsen.util 1712 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:28:54,416] jepsen worker 10 - jepsen.util 1720 :fail :cas [3 [2 0]] :ValueNotSwapFrom INFO [2019-12-17 12:28:54,416] jepsen worker 66 - jepsen.util 1596 :info :write [22 32] :RequestCanceledException INFO [2019-12-17 12:28:54,416] jepsen worker 6 - jepsen.util 1986 :info :write [2 12] :RequestCanceledException INFO [2019-12-17 12:28:54,417] jepsen worker 45 - jepsen.util 1755 :info :write [15 6] :RequestCanceledException INFO [2019-12-17 12:28:54,417] jepsen worker 4 - jepsen.util 1264 :fail :cas [1 [14 42]] :ValueNotSwapFrom INFO [2019-12-17 12:28:54,417] jepsen worker 27 - jepsen.util 1287 :info :cas [9 [41 38]] :RequestCanceledException INFO [2019-12-17 12:28:54,417] jepsen worker 28 - jepsen.util 928 :invoke :write [9 34] INFO [2019-12-17 12:28:54,417] jepsen worker 56 - jepsen.util 1586 :ok :read [18 35] INFO [2019-12-17 12:28:54,417] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8797,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:28:54,417] jepsen worker 55 - jepsen.util 1855 :ok :read [18 35] INFO [2019-12-17 12:28:54,417] jepsen worker 40 - jepsen.util 2290 :info :cas [13 [26 27]] :RequestCanceledException INFO [2019-12-17 12:28:54,417] jepsen worker 76 - jepsen.util 1966 :info :write [25 11] :RequestCanceledException INFO [2019-12-17 12:28:54,417] jepsen worker 50 - jepsen.util 2030 :info :cas [16 [3 4]] :RequestCanceledException INFO [2019-12-17 12:28:54,418] jepsen worker 39 - jepsen.util 2109 :info :cas [13 [13 47]] :RequestCanceledException INFO [2019-12-17 12:28:54,418] jepsen worker 46 - jepsen.util 1756 :fail :cas [15 [28 43]] :ValueNotSwapFrom INFO [2019-12-17 12:28:54,418] jepsen worker 15 - jepsen.util 1725 :info :cas [5 [46 29]] :RequestCanceledException INFO [2019-12-17 12:28:54,418] jepsen worker 43 - jepsen.util 2023 :info :write [14 0] :RequestCanceledException INFO [2019-12-17 12:28:54,419] jepsen worker 61 - jepsen.util 2041 :info :write [20 24] :RequestCanceledException INFO [2019-12-17 12:28:54,419] jepsen worker 11 - jepsen.util 1811 :fail :read [3 nil] :RequestTimeoutException INFO [2019-12-17 12:28:54,420] jepsen worker 75 - jepsen.util 2145 :info :cas [25 [3 9]] :RequestCanceledException INFO [2019-12-17 12:28:54,420] jepsen worker 44 - jepsen.util 1664 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:28:54,421] jepsen worker 35 - jepsen.util 1655 :info :cas [11 [11 39]] :RequestCanceledException INFO [2019-12-17 12:28:54,421] jepsen worker 73 - jepsen.util 1513 :fail :read [24 nil] :RequestTimeoutException INFO [2019-12-17 12:28:54,422] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8798,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:28:54,422] jepsen worker 33 - jepsen.util 2283 :info :write [11 36] :RequestCanceledException INFO [2019-12-17 12:28:54,422] jepsen worker 9 - jepsen.util 1629 :info :write [3 27] :RequestCanceledException INFO [2019-12-17 12:28:54,422] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8800,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:28:54,424] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8783,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:28:54,424] jepsen worker 41 - jepsen.util 1931 :info :cas [13 [21 24]] :RequestCanceledException INFO [2019-12-17 12:28:54,424] jepsen worker 65 - jepsen.util 1235 :info :write [21 18] :RequestCanceledException INFO [2019-12-17 12:28:54,424] jepsen worker 83 - jepsen.util 2243 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:28:54,424] jepsen worker 42 - jepsen.util 2202 :info :cas [14 [32 6]] :RequestCanceledException INFO [2019-12-17 12:28:54,425] jepsen worker 68 - jepsen.util 1598 :info :write [22 48] :RequestCanceledException INFO [2019-12-17 12:28:54,425] jepsen worker 63 - jepsen.util 1413 :fail :read [21 nil] :RequestTimeoutException INFO [2019-12-17 12:28:54,425] jepsen worker 59 - jepsen.util 1589 :info :write [19 24] :RequestCanceledException INFO [2019-12-17 12:28:54,426] jepsen worker 79 - jepsen.util 1519 :info :write [26 31] :RequestCanceledException INFO [2019-12-17 12:28:54,426] jepsen worker 8 - jepsen.util 1988 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:28:54,426] jepsen worker 69 - jepsen.util 1419 :invoke :cas [23 [48 31]] INFO [2019-12-17 12:28:54,426] jepsen worker 25 - jepsen.util 2275 :invoke :cas [8 [37 8]] INFO [2019-12-17 12:28:54,428] jepsen worker 10 - jepsen.util 1720 :invoke :read [3 nil] INFO [2019-12-17 12:28:54,429] jepsen worker 74 - jepsen.util 1424 :info :cas [24 [28 48]] :RequestCanceledException INFO [2019-12-17 12:28:54,429] jepsen worker 16 - jepsen.util 1906 :invoke :read [5 nil] INFO [2019-12-17 12:28:54,431] jepsen worker 3 - jepsen.util 1443 :invoke :read [1 nil] INFO [2019-12-17 12:28:54,431] jepsen worker 13 - jepsen.util 1453 :invoke :cas [4 [8 7]] INFO [2019-12-17 12:28:54,432] jepsen worker 29 - jepsen.util 1199 :info :cas [9 [39 22]] :RequestCanceledException INFO [2019-12-17 12:28:54,432] jepsen worker 82 - jepsen.util 2422 :info :cas [27 [3 47]] :RequestCanceledException INFO [2019-12-17 12:28:54,433] jepsen worker 14 - jepsen.util 1454 :invoke :cas [4 [44 43]] INFO [2019-12-17 12:28:54,433] jepsen worker 2 - jepsen.util 1712 :invoke :write [0 22] INFO [2019-12-17 12:28:54,435] jepsen worker 56 - jepsen.util 1586 :invoke :read [18 nil] INFO [2019-12-17 12:28:54,436] jepsen worker 4 - jepsen.util 1264 :invoke :write [1 0] INFO [2019-12-17 12:28:54,447] jepsen worker 86 - jepsen.util 2156 :invoke :write [28 11] INFO [2019-12-17 12:28:54,447] jepsen worker 55 - jepsen.util 1855 :invoke :cas [18 [9 32]] INFO [2019-12-17 12:28:54,448] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8797,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:28:54,470] jepsen worker 6 - jepsen.util 2076 :invoke :cas [2 [28 13]] INFO [2019-12-17 12:28:54,478] jepsen worker 11 - jepsen.util 1811 :invoke :cas [3 [47 12]] INFO [2019-12-17 12:28:54,482] jepsen worker 46 - jepsen.util 1756 :invoke :write [15 35] INFO [2019-12-17 12:28:54,484] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:28:54,484] jepsen worker 44 - jepsen.util 1664 :invoke :write [14 45] INFO [2019-12-17 12:28:54,492] jepsen worker 73 - jepsen.util 1513 :invoke :cas [24 [31 24]] INFO [2019-12-17 12:28:54,493] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8798,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:28:54,498] jepsen worker 49 - jepsen.util 1399 :invoke :write [16 48] INFO [2019-12-17 12:28:54,502] jepsen worker 18 - jepsen.util 1818 :invoke :write [6 41] INFO [2019-12-17 12:28:54,513] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8800,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:28:54,517] jepsen worker 23 - jepsen.util 1913 :invoke :read [7 nil] INFO [2019-12-17 12:28:54,521] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8785,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:28:54,529] jepsen worker 64 - jepsen.util 1234 :ok :write [21 6] INFO [2019-12-17 12:28:54,529] jepsen worker 83 - jepsen.util 2243 :invoke :write [27 23] INFO [2019-12-17 12:28:54,530] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8799,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:28:54,532] jepsen worker 63 - jepsen.util 1413 :invoke :write [21 37] INFO [2019-12-17 12:28:54,532] jepsen worker 47 - jepsen.util 2027 :ok :write [15 49] INFO [2019-12-17 12:28:54,535] jepsen worker 66 - jepsen.util 1686 :invoke :cas [22 [3 36]] INFO [2019-12-17 12:28:54,536] jepsen worker 8 - jepsen.util 1988 :invoke :read [2 nil] INFO [2019-12-17 12:28:54,539] jepsen worker 45 - jepsen.util 1845 :invoke :cas [15 [19 45]] INFO [2019-12-17 12:28:54,543] jepsen worker 75 - jepsen.util 2235 :invoke :write [25 8] INFO [2019-12-17 12:28:54,544] jepsen worker 27 - jepsen.util 1377 :invoke :write [9 14] INFO [2019-12-17 12:28:54,590] jepsen worker 32 - jepsen.util 1832 :invoke :write [10 26] INFO [2019-12-17 12:28:54,630] jepsen worker 9 - jepsen.util 1719 :invoke :write [3 28] INFO [2019-12-17 12:28:54,635] jepsen worker 79 - jepsen.util 1609 :invoke :cas [26 [38 8]] INFO [2019-12-17 12:28:54,658] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8824,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:28:54,676] jepsen worker 61 - jepsen.util 2131 :invoke :cas [20 [7 41]] INFO [2019-12-17 12:28:54,691] jepsen worker 35 - jepsen.util 1745 :invoke :write [11 18] INFO [2019-12-17 12:28:54,692] jepsen worker 76 - jepsen.util 2056 :invoke :cas [25 [22 8]] INFO [2019-12-17 12:28:54,691] jepsen nemesis - couchbase.util Rebalance check count: 61 INFO [2019-12-17 12:28:54,704] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8809,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:28:54,706] jepsen worker 30 - jepsen.util 1650 :info :cas [10 [15 4]] :RequestCanceledException INFO [2019-12-17 12:28:54,743] jepsen worker 85 - jepsen.util 2155 :info :write [28 25] :RequestCanceledException INFO [2019-12-17 12:28:54,763] jepsen worker 50 - jepsen.util 2120 :invoke :write [16 27] INFO [2019-12-17 12:28:54,771] jepsen worker 43 - jepsen.util 2113 :invoke :write [14 43] INFO [2019-12-17 12:28:54,773] jepsen worker 64 - jepsen.util 1234 :invoke :read [21 nil] INFO [2019-12-17 12:28:54,775] jepsen worker 47 - jepsen.util 2027 :invoke :write [15 18] INFO [2019-12-17 12:28:54,790] jepsen worker 72 - jepsen.util 1512 :info :cas [24 [49 47]] :RequestCanceledException INFO [2019-12-17 12:28:54,791] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8799,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:28:54,792] jepsen worker 40 - jepsen.util 2380 :invoke :read [13 nil] INFO [2019-12-17 12:28:54,796] jepsen nemesis - couchbase.util Rebalance stuck count: 42 INFO [2019-12-17 12:28:54,798] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8785,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:28:54,806] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8824,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:28:54,806] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8809,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:28:54,806] jepsen worker 68 - jepsen.util 1688 :invoke :cas [22 [3 3]] INFO [2019-12-17 12:28:54,806] jepsen worker 33 - jepsen.util 2373 :invoke :write [11 22] INFO [2019-12-17 12:28:54,807] jepsen worker 65 - jepsen.util 1325 :invoke :read [21 nil] INFO [2019-12-17 12:28:54,807] jepsen worker 82 - jepsen.util 2512 :invoke :cas [27 [41 43]] INFO [2019-12-17 12:28:54,808] jepsen worker 15 - jepsen.util 1815 :invoke :cas [5 [23 43]] INFO [2019-12-17 12:28:54,815] jepsen worker 39 - jepsen.util 2199 :invoke :cas [13 [38 16]] INFO [2019-12-17 12:28:54,818] jepsen worker 42 - jepsen.util 2292 :invoke :cas [14 [17 15]] INFO [2019-12-17 12:28:54,842] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8804,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:28:54,859] jepsen worker 59 - jepsen.util 1679 :invoke :write [19 7] INFO [2019-12-17 12:28:54,859] jepsen worker 41 - jepsen.util 2021 :invoke :cas [13 [32 7]] INFO [2019-12-17 12:28:54,865] jepsen worker 74 - jepsen.util 1514 :invoke :cas [24 [14 24]] INFO [2019-12-17 12:28:54,879] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8827,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:28:54,894] jepsen worker 29 - jepsen.util 1289 :invoke :cas [9 [24 0]] INFO [2019-12-17 12:28:54,895] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8804,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:28:54,895] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8846,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:28:54,899] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8827,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:28:54,907] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8846,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:28:54,926] jepsen worker 30 - jepsen.util 1740 :invoke :cas [10 [41 40]] INFO [2019-12-17 12:28:54,936] jepsen worker 34 - jepsen.util 2194 :fail :cas [11 [0 28]] :ValueNotSwapFrom INFO [2019-12-17 12:28:54,937] jepsen worker 34 - jepsen.util 2194 :invoke :read [11 nil] INFO [2019-12-17 12:28:54,956] jepsen worker 85 - jepsen.util 2245 :invoke :cas [28 [47 48]] INFO [2019-12-17 12:28:54,970] jepsen worker 20 - jepsen.util 1460 :ok :read [6 25] INFO [2019-12-17 12:28:54,976] jepsen worker 20 - jepsen.util 1460 :invoke :read [6 nil] INFO [2019-12-17 12:28:54,981] jepsen worker 53 - jepsen.util 1493 :ok :read [17 43] INFO [2019-12-17 12:28:54,981] jepsen worker 53 - jepsen.util 1493 :invoke :cas [17 [18 23]] INFO [2019-12-17 12:28:54,990] jepsen worker 72 - jepsen.util 1602 :invoke :cas [24 [35 41]] INFO [2019-12-17 12:28:54,995] jepsen worker 54 - jepsen.util 1674 :ok :read [18 35] INFO [2019-12-17 12:28:54,998] jepsen worker 54 - jepsen.util 1674 :invoke :read [18 nil] INFO [2019-12-17 12:28:55,018] jepsen worker 52 - jepsen.util 1582 :fail :cas [17 [11 19]] :ValueNotSwapFrom INFO [2019-12-17 12:28:55,024] jepsen worker 52 - jepsen.util 1582 :invoke :read [17 nil] INFO [2019-12-17 12:28:55,031] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8863,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:28:55,036] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8863,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:28:55,043] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8869,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:28:55,045] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8869,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:28:55,051] jepsen worker 48 - jepsen.util 1488 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:28:55,054] jepsen worker 48 - jepsen.util 1488 :invoke :cas [16 [27 34]] INFO [2019-12-17 12:28:55,058] jepsen worker 71 - jepsen.util 1241 :fail :cas [23 [43 27]] :ValueNotSwapFrom INFO [2019-12-17 12:28:55,061] jepsen worker 7 - jepsen.util 2167 :info :cas [2 [41 25]] :RequestCanceledException INFO [2019-12-17 12:28:55,062] jepsen worker 71 - jepsen.util 1241 :invoke :cas [23 [21 35]] INFO [2019-12-17 12:28:55,065] jepsen worker 57 - jepsen.util 1857 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:28:55,087] jepsen worker 87 - jepsen.util 1437 :fail :cas [29 [8 17]] :ValueNotSwapFrom INFO [2019-12-17 12:28:55,087] jepsen worker 87 - jepsen.util 1437 :invoke :read [29 nil] INFO [2019-12-17 12:28:55,089] jepsen worker 57 - jepsen.util 1857 :invoke :write [19 21] INFO [2019-12-17 12:28:55,093] jepsen worker 21 - jepsen.util 1911 :info :cas [7 [28 32]] :RequestCanceledException INFO [2019-12-17 12:28:55,140] jepsen worker 62 - jepsen.util 2042 :info :cas [20 [5 10]] :RequestCanceledException INFO [2019-12-17 12:28:55,152] jepsen worker 67 - jepsen.util 2407 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:28:55,166] jepsen worker 22 - jepsen.util 2092 :info :cas [7 [39 10]] :RequestCanceledException INFO [2019-12-17 12:28:55,202] jepsen worker 67 - jepsen.util 2407 :invoke :cas [22 [22 41]] INFO [2019-12-17 12:28:55,204] jepsen worker 1 - jepsen.util 1981 :info :cas [0 [5 42]] :RequestCanceledException INFO [2019-12-17 12:28:55,214] jepsen worker 60 - jepsen.util 2130 :info :cas [20 [5 45]] :RequestCanceledException INFO [2019-12-17 12:28:55,279] jepsen worker 21 - jepsen.util 2001 :invoke :cas [7 [11 21]] INFO [2019-12-17 12:28:55,333] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8876,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:28:55,337] jepsen worker 7 - jepsen.util 2257 :invoke :write [2 29] INFO [2019-12-17 12:28:55,361] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8876,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:28:55,363] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8882,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:28:55,419] jepsen worker 60 - jepsen.util 2220 :invoke :read [20 nil] INFO [2019-12-17 12:28:55,724] jepsen worker 26 - jepsen.util 2186 :info :cas [8 [24 23]] :RequestCanceledException INFO [2019-12-17 12:28:55,723] jepsen worker 58 - jepsen.util 1678 :info :write [19 9] :RequestCanceledException INFO [2019-12-17 12:28:55,725] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"requestId":8911,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"coreId":4,"completed":false} INFO [2019-12-17 12:28:55,739] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8882,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:28:55,747] jepsen worker 80 - jepsen.util 1700 :fail :cas [26 [42 43]] :ValueNotSwapFrom INFO [2019-12-17 12:28:55,748] jepsen worker 17 - jepsen.util 1997 :info :cas [5 [4 40]] :RequestCanceledException INFO [2019-12-17 12:28:55,748] jepsen worker 62 - jepsen.util 2132 :invoke :write [20 27] INFO [2019-12-17 12:28:55,750] jepsen worker 24 - jepsen.util 1824 :info :cas [8 [13 31]] :RequestCanceledException INFO [2019-12-17 12:28:55,760] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8911,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:28:55,766] jepsen worker 80 - jepsen.util 1700 :invoke :read [26 nil] INFO [2019-12-17 12:28:55,767] jepsen worker 36 - jepsen.util 1566 :fail :cas [12 [17 48]] :ValueNotSwapFrom INFO [2019-12-17 12:28:55,767] jepsen worker 31 - jepsen.util 1651 :ok :write [10 36] INFO [2019-12-17 12:28:55,771] jepsen worker 81 - jepsen.util 1701 :info :cas [27 [43 37]] :RequestCanceledException INFO [2019-12-17 12:28:55,777] jepsen worker 84 - jepsen.util 1974 :info :cas [28 [46 9]] :RequestCanceledException INFO [2019-12-17 12:28:55,777] jepsen worker 22 - jepsen.util 2182 :invoke :cas [7 [1 3]] INFO [2019-12-17 12:28:55,790] jepsen worker 5 - jepsen.util 1265 :ok :write [1 31] INFO [2019-12-17 12:28:55,799] jepsen worker 1 - jepsen.util 2071 :invoke :cas [0 [20 14]] INFO [2019-12-17 12:28:55,803] jepsen worker 70 - jepsen.util 1780 :info :cas [23 [29 3]] :RequestCanceledException INFO [2019-12-17 12:28:55,804] jepsen worker 5 - jepsen.util 1265 :invoke :read [1 nil] INFO [2019-12-17 12:28:55,816] jepsen worker 31 - jepsen.util 1651 :invoke :read [10 nil] INFO [2019-12-17 12:28:55,818] jepsen worker 36 - jepsen.util 1566 :invoke :cas [12 [47 23]] INFO [2019-12-17 12:28:55,838] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8912,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:28:55,840] jepsen worker 51 - jepsen.util 1491 :info :write [17 2] :RequestCanceledException INFO [2019-12-17 12:28:55,868] jepsen worker 17 - jepsen.util 2087 :invoke :read [5 nil] INFO [2019-12-17 12:28:55,883] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8928,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:28:55,900] jepsen worker 26 - jepsen.util 2276 :invoke :cas [8 [32 44]] INFO [2019-12-17 12:28:55,901] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8912,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:28:55,901] jepsen worker 58 - jepsen.util 1768 :invoke :cas [19 [46 15]] INFO [2019-12-17 12:28:55,906] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8928,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:28:55,929] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8930,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:28:55,929] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8930,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:28:55,947] jepsen worker 0 - jepsen.util 2160 :info :write [0 13] :RequestCanceledException INFO [2019-12-17 12:28:55,947] jepsen worker 81 - jepsen.util 1791 :invoke :read [27 nil] INFO [2019-12-17 12:28:55,956] jepsen worker 24 - jepsen.util 1914 :invoke :write [8 14] INFO [2019-12-17 12:28:55,965] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:28:55,966] jepsen nemesis - couchbase.util Rebalance check count: 62 INFO [2019-12-17 12:28:55,966] jepsen nemesis - couchbase.util Rebalance stuck count: 43 INFO [2019-12-17 12:28:55,982] jepsen worker 84 - jepsen.util 2064 :invoke :read [28 nil] INFO [2019-12-17 12:28:55,991] jepsen worker 70 - jepsen.util 1870 :invoke :cas [23 [10 18]] INFO [2019-12-17 12:28:56,007] jepsen worker 51 - jepsen.util 1581 :invoke :read [17 nil] INFO [2019-12-17 12:28:56,032] jepsen worker 37 - jepsen.util 1297 :ok :read [12 15] INFO [2019-12-17 12:28:56,033] jepsen worker 77 - jepsen.util 1967 :fail :cas [25 [16 19]] :ValueNotSwapFrom INFO [2019-12-17 12:28:56,036] jepsen worker 77 - jepsen.util 1967 :invoke :write [25 16] INFO [2019-12-17 12:28:56,037] jepsen worker 37 - jepsen.util 1297 :invoke :read [12 nil] INFO [2019-12-17 12:28:56,038] jepsen worker 12 - jepsen.util 1632 :ok :read [4 19] INFO [2019-12-17 12:28:56,039] jepsen worker 12 - jepsen.util 1632 :invoke :read [4 nil] INFO [2019-12-17 12:28:56,044] jepsen worker 0 - jepsen.util 2250 :invoke :cas [0 [16 28]] INFO [2019-12-17 12:28:56,047] jepsen worker 89 - jepsen.util 1529 :fail :cas [29 [15 45]] :ValueNotSwapFrom INFO [2019-12-17 12:28:56,048] jepsen worker 89 - jepsen.util 1529 :invoke :cas [29 [27 37]] INFO [2019-12-17 12:28:56,056] jepsen worker 78 - jepsen.util 1608 :ok :read [26 16] INFO [2019-12-17 12:28:56,060] jepsen worker 78 - jepsen.util 1608 :invoke :cas [26 [42 20]] INFO [2019-12-17 12:28:56,086] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8941,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:28:56,087] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8941,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:28:56,170] jepsen worker 38 - jepsen.util 1568 :ok :read [12 32] INFO [2019-12-17 12:28:56,170] jepsen worker 29 - jepsen.util 1289 :fail :cas [9 [24 0]] :ValueNotSwapFrom INFO [2019-12-17 12:28:56,172] jepsen worker 38 - jepsen.util 1568 :invoke :read [12 nil] INFO [2019-12-17 12:28:56,173] jepsen worker 55 - jepsen.util 1855 :fail :cas [18 [9 32]] :ValueNotSwapFrom INFO [2019-12-17 12:28:56,173] jepsen worker 29 - jepsen.util 1289 :invoke :read [9 nil] INFO [2019-12-17 12:28:56,174] jepsen worker 55 - jepsen.util 1855 :invoke :write [18 10] INFO [2019-12-17 12:28:56,175] jepsen worker 45 - jepsen.util 1845 :fail :cas [15 [19 45]] :ValueNotSwapFrom INFO [2019-12-17 12:28:56,175] jepsen worker 45 - jepsen.util 1845 :invoke :write [15 38] INFO [2019-12-17 12:28:56,181] jepsen worker 10 - jepsen.util 1720 :ok :read [3 8] INFO [2019-12-17 12:28:56,183] jepsen worker 65 - jepsen.util 1325 :ok :read [21 6] INFO [2019-12-17 12:28:56,183] jepsen worker 65 - jepsen.util 1325 :invoke :write [21 17] INFO [2019-12-17 12:28:56,183] jepsen worker 10 - jepsen.util 1720 :invoke :cas [3 [48 7]] INFO [2019-12-17 12:28:56,194] jepsen worker 11 - jepsen.util 1811 :fail :cas [3 [47 12]] :ValueNotSwapFrom INFO [2019-12-17 12:28:56,194] jepsen worker 11 - jepsen.util 1811 :invoke :cas [3 [14 5]] INFO [2019-12-17 12:28:56,204] jepsen worker 53 - jepsen.util 1493 :fail :cas [17 [18 23]] :ValueNotSwapFrom INFO [2019-12-17 12:28:56,205] jepsen worker 53 - jepsen.util 1493 :invoke :cas [17 [13 9]] INFO [2019-12-17 12:28:56,206] jepsen worker 73 - jepsen.util 1513 :fail :cas [24 [31 24]] :ValueNotSwapFrom INFO [2019-12-17 12:28:56,207] jepsen worker 73 - jepsen.util 1513 :invoke :write [24 17] INFO [2019-12-17 12:28:56,214] jepsen worker 79 - jepsen.util 1609 :fail :cas [26 [38 8]] :ValueNotSwapFrom INFO [2019-12-17 12:28:56,214] jepsen worker 54 - jepsen.util 1674 :ok :read [18 35] INFO [2019-12-17 12:28:56,214] jepsen worker 79 - jepsen.util 1609 :invoke :write [26 9] INFO [2019-12-17 12:28:56,214] jepsen worker 74 - jepsen.util 1514 :fail :cas [24 [14 24]] :ValueNotSwapFrom INFO [2019-12-17 12:28:56,215] jepsen worker 54 - jepsen.util 1674 :invoke :write [18 3] INFO [2019-12-17 12:28:56,217] jepsen worker 74 - jepsen.util 1514 :invoke :read [24 nil] INFO [2019-12-17 12:28:56,219] jepsen worker 20 - jepsen.util 1460 :ok :read [6 25] INFO [2019-12-17 12:28:56,219] jepsen worker 20 - jepsen.util 1460 :invoke :cas [6 [44 8]] INFO [2019-12-17 12:28:56,241] jepsen worker 13 - jepsen.util 1453 :fail :cas [4 [8 7]] :ValueNotSwapFrom INFO [2019-12-17 12:28:56,243] jepsen worker 70 - jepsen.util 1870 :fail :cas [23 [10 18]] :ValueNotSwapFrom INFO [2019-12-17 12:28:56,243] jepsen worker 72 - jepsen.util 1602 :fail :cas [24 [35 41]] :ValueNotSwapFrom INFO [2019-12-17 12:28:56,243] jepsen worker 70 - jepsen.util 1870 :invoke :read [23 nil] INFO [2019-12-17 12:28:56,243] jepsen worker 13 - jepsen.util 1453 :invoke :write [4 17] INFO [2019-12-17 12:28:56,243] jepsen worker 72 - jepsen.util 1602 :invoke :cas [24 [4 40]] INFO [2019-12-17 12:28:56,246] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8968,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:28:56,246] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8968,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:28:56,247] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8972,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:28:56,247] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8972,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:28:56,247] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8973,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:28:56,248] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8973,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:28:56,257] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8961,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:28:56,258] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8961,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:28:56,258] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8962,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:28:56,258] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8962,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:28:56,269] jepsen worker 36 - jepsen.util 1566 :fail :cas [12 [47 23]] :ValueNotSwapFrom INFO [2019-12-17 12:28:56,272] jepsen worker 36 - jepsen.util 1566 :invoke :cas [12 [39 28]] INFO [2019-12-17 12:28:56,286] jepsen worker 78 - jepsen.util 1608 :fail :cas [26 [42 20]] :ValueNotSwapFrom INFO [2019-12-17 12:28:56,287] jepsen worker 78 - jepsen.util 1608 :invoke :write [26 3] INFO [2019-12-17 12:28:56,290] jepsen worker 76 - jepsen.util 2056 :fail :cas [25 [22 8]] :ValueNotSwapFrom INFO [2019-12-17 12:28:56,292] jepsen worker 76 - jepsen.util 2056 :invoke :write [25 28] INFO [2019-12-17 12:28:56,307] jepsen worker 87 - jepsen.util 1437 :ok :read [29 42] INFO [2019-12-17 12:28:56,307] jepsen worker 87 - jepsen.util 1437 :invoke :write [29 31] INFO [2019-12-17 12:28:56,309] jepsen worker 34 - jepsen.util 2194 :ok :read [11 7] INFO [2019-12-17 12:28:56,310] jepsen worker 34 - jepsen.util 2194 :invoke :write [11 20] INFO [2019-12-17 12:28:56,322] jepsen worker 64 - jepsen.util 1234 :ok :read [21 6] INFO [2019-12-17 12:28:56,324] jepsen worker 5 - jepsen.util 1265 :ok :read [1 31] INFO [2019-12-17 12:28:56,324] jepsen worker 5 - jepsen.util 1265 :invoke :cas [1 [44 40]] INFO [2019-12-17 12:28:56,328] jepsen worker 71 - jepsen.util 1241 :fail :cas [23 [21 35]] :ValueNotSwapFrom INFO [2019-12-17 12:28:56,329] jepsen worker 64 - jepsen.util 1234 :invoke :write [21 35] INFO [2019-12-17 12:28:56,330] jepsen worker 71 - jepsen.util 1241 :invoke :cas [23 [43 26]] INFO [2019-12-17 12:28:56,351] jepsen worker 31 - jepsen.util 1651 :ok :read [10 36] INFO [2019-12-17 12:28:56,352] jepsen worker 31 - jepsen.util 1651 :invoke :cas [10 [16 48]] INFO [2019-12-17 12:28:56,353] jepsen worker 37 - jepsen.util 1297 :ok :read [12 32] INFO [2019-12-17 12:28:56,354] jepsen worker 37 - jepsen.util 1297 :invoke :cas [12 [36 4]] INFO [2019-12-17 12:28:56,383] jepsen worker 21 - jepsen.util 2001 :fail :cas [7 [11 21]] :ValueNotSwapFrom INFO [2019-12-17 12:28:56,383] jepsen worker 21 - jepsen.util 2001 :invoke :write [7 41] INFO [2019-12-17 12:28:56,424] jepsen worker 89 - jepsen.util 1529 :fail :cas [29 [27 37]] :ValueNotSwapFrom INFO [2019-12-17 12:28:56,424] jepsen worker 89 - jepsen.util 1529 :invoke :write [29 26] INFO [2019-12-17 12:28:56,442] jepsen worker 63 - jepsen.util 1413 :ok :write [21 37] INFO [2019-12-17 12:28:56,442] jepsen worker 63 - jepsen.util 1413 :invoke :cas [21 [27 29]] INFO [2019-12-17 12:28:56,464] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8992,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:28:56,464] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8992,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:28:56,465] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8997,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:28:56,465] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":8997,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:28:56,483] jepsen worker 28 - jepsen.util 928 :ok :write [9 34] INFO [2019-12-17 12:28:56,483] jepsen worker 28 - jepsen.util 928 :invoke :cas [9 [30 4]] INFO [2019-12-17 12:28:56,526] jepsen worker 51 - jepsen.util 1581 :ok :read [17 43] INFO [2019-12-17 12:28:56,526] jepsen worker 51 - jepsen.util 1581 :invoke :cas [17 [21 30]] INFO [2019-12-17 12:28:56,548] jepsen worker 53 - jepsen.util 1493 :fail :cas [17 [13 9]] :ValueNotSwapFrom INFO [2019-12-17 12:28:56,548] jepsen worker 38 - jepsen.util 1568 :ok :read [12 32] INFO [2019-12-17 12:28:56,549] jepsen worker 53 - jepsen.util 1493 :invoke :cas [17 [23 39]] INFO [2019-12-17 12:28:56,549] jepsen worker 38 - jepsen.util 1568 :invoke :write [12 6] INFO [2019-12-17 12:28:56,608] jepsen worker 16 - jepsen.util 1906 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:28:56,608] jepsen worker 16 - jepsen.util 1906 :invoke :read [5 nil] INFO [2019-12-17 12:28:56,616] jepsen worker 72 - jepsen.util 1602 :fail :cas [24 [4 40]] :ValueNotSwapFrom INFO [2019-12-17 12:28:56,617] jepsen worker 72 - jepsen.util 1602 :invoke :cas [24 [46 48]] INFO [2019-12-17 12:28:56,623] jepsen worker 69 - jepsen.util 1419 :info :cas [23 [48 31]] :RequestCanceledException INFO [2019-12-17 12:28:56,623] jepsen worker 14 - jepsen.util 1454 :info :cas [4 [44 43]] :RequestCanceledException INFO [2019-12-17 12:28:56,623] jepsen worker 3 - jepsen.util 1443 :fail :read [1 nil] :RequestTimeoutException INFO [2019-12-17 12:28:56,623] jepsen worker 56 - jepsen.util 1586 :fail :read [18 nil] :RequestTimeoutException INFO [2019-12-17 12:28:56,623] jepsen worker 25 - jepsen.util 2275 :info :cas [8 [37 8]] :RequestCanceledException INFO [2019-12-17 12:28:56,623] jepsen worker 56 - jepsen.util 1586 :invoke :cas [18 [46 49]] INFO [2019-12-17 12:28:56,623] jepsen worker 3 - jepsen.util 1443 :invoke :cas [1 [41 18]] INFO [2019-12-17 12:28:56,627] jepsen worker 14 - jepsen.util 1544 :invoke :cas [4 [22 34]] INFO [2019-12-17 12:28:56,627] jepsen worker 69 - jepsen.util 1509 :invoke :cas [23 [7 27]] INFO [2019-12-17 12:28:56,627] jepsen worker 25 - jepsen.util 2365 :invoke :read [8 nil] INFO [2019-12-17 12:28:56,627] jepsen worker 36 - jepsen.util 1566 :fail :cas [12 [39 28]] :ValueNotSwapFrom INFO [2019-12-17 12:28:56,628] jepsen worker 36 - jepsen.util 1566 :invoke :cas [12 [13 9]] INFO [2019-12-17 12:28:56,632] jepsen worker 31 - jepsen.util 1651 :fail :cas [10 [16 48]] :ValueNotSwapFrom INFO [2019-12-17 12:28:56,633] jepsen worker 31 - jepsen.util 1651 :invoke :write [10 15] INFO [2019-12-17 12:28:56,636] jepsen worker 70 - jepsen.util 1870 :ok :read [23 26] INFO [2019-12-17 12:28:56,637] jepsen worker 70 - jepsen.util 1870 :invoke :write [23 44] INFO [2019-12-17 12:28:56,638] jepsen worker 71 - jepsen.util 1241 :fail :cas [23 [43 26]] :ValueNotSwapFrom INFO [2019-12-17 12:28:56,638] jepsen worker 71 - jepsen.util 1241 :invoke :write [23 20] INFO [2019-12-17 12:28:56,681] jepsen worker 51 - jepsen.util 1581 :fail :cas [17 [21 30]] :ValueNotSwapFrom INFO [2019-12-17 12:28:56,682] jepsen worker 51 - jepsen.util 1581 :invoke :read [17 nil] INFO [2019-12-17 12:28:56,728] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9017,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:28:56,728] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9017,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:28:56,729] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9021,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:28:56,729] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9021,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:28:56,739] jepsen worker 5 - jepsen.util 1265 :fail :cas [1 [44 40]] :ValueNotSwapFrom INFO [2019-12-17 12:28:56,739] jepsen worker 5 - jepsen.util 1265 :invoke :read [1 nil] INFO [2019-12-17 12:28:56,748] jepsen worker 6 - jepsen.util 2076 :info :cas [2 [28 13]] :RequestCanceledException INFO [2019-12-17 12:28:56,751] jepsen worker 6 - jepsen.util 2166 :invoke :cas [2 [27 49]] INFO [2019-12-17 12:28:56,761] jepsen worker 53 - jepsen.util 1493 :fail :cas [17 [23 39]] :ValueNotSwapFrom INFO [2019-12-17 12:28:56,761] jepsen worker 53 - jepsen.util 1493 :invoke :read [17 nil] INFO [2019-12-17 12:28:56,785] jepsen worker 27 - jepsen.util 1377 :ok :write [9 14] INFO [2019-12-17 12:28:56,785] jepsen worker 27 - jepsen.util 1377 :invoke :cas [9 [24 29]] INFO [2019-12-17 12:28:56,806] jepsen worker 72 - jepsen.util 1602 :fail :cas [24 [46 48]] :ValueNotSwapFrom INFO [2019-12-17 12:28:56,806] jepsen worker 23 - jepsen.util 1913 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:28:56,806] jepsen worker 72 - jepsen.util 1602 :invoke :write [24 10] INFO [2019-12-17 12:28:56,807] jepsen worker 23 - jepsen.util 1913 :invoke :read [7 nil] INFO [2019-12-17 12:28:56,820] jepsen worker 88 - jepsen.util 1438 :ok :write [29 10] INFO [2019-12-17 12:28:56,821] jepsen worker 88 - jepsen.util 1438 :invoke :write [29 0] INFO [2019-12-17 12:28:56,821] jepsen worker 66 - jepsen.util 1686 :info :cas [22 [3 36]] :RequestCanceledException INFO [2019-12-17 12:28:56,821] jepsen worker 19 - jepsen.util 1729 :info :write [6 1] :RequestCanceledException INFO [2019-12-17 12:28:56,823] jepsen worker 19 - jepsen.util 1819 :invoke :read [6 nil] INFO [2019-12-17 12:28:56,823] jepsen worker 66 - jepsen.util 1776 :invoke :write [22 41] INFO [2019-12-17 12:28:56,836] jepsen worker 8 - jepsen.util 1988 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:28:56,836] jepsen worker 61 - jepsen.util 2131 :info :cas [20 [7 41]] :RequestCanceledException INFO [2019-12-17 12:28:56,836] jepsen worker 8 - jepsen.util 1988 :invoke :write [2 19] INFO [2019-12-17 12:28:56,838] jepsen worker 61 - jepsen.util 2221 :invoke :cas [20 [5 3]] INFO [2019-12-17 12:28:56,861] jepsen worker 27 - jepsen.util 1377 :fail :cas [9 [24 29]] :ValueNotSwapFrom INFO [2019-12-17 12:28:56,861] jepsen worker 27 - jepsen.util 1377 :invoke :write [9 11] INFO [2019-12-17 12:28:56,888] jepsen worker 82 - jepsen.util 2512 :info :cas [27 [41 43]] :RequestCanceledException INFO [2019-12-17 12:28:56,890] jepsen worker 82 - jepsen.util 2602 :invoke :read [27 nil] INFO [2019-12-17 12:28:56,900] jepsen worker 15 - jepsen.util 1815 :info :cas [5 [23 43]] :RequestCanceledException INFO [2019-12-17 12:28:56,902] jepsen worker 15 - jepsen.util 1905 :invoke :read [5 nil] INFO [2019-12-17 12:28:56,916] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9038,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:28:56,916] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9038,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:28:56,916] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9041,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:28:56,916] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9041,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:28:56,922] jepsen worker 39 - jepsen.util 2199 :info :cas [13 [38 16]] :RequestCanceledException INFO [2019-12-17 12:28:56,922] jepsen worker 40 - jepsen.util 2380 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:28:56,922] jepsen worker 41 - jepsen.util 2021 :info :cas [13 [32 7]] :RequestCanceledException INFO [2019-12-17 12:28:56,922] jepsen worker 40 - jepsen.util 2380 :invoke :cas [13 [30 6]] INFO [2019-12-17 12:28:56,924] jepsen worker 41 - jepsen.util 2111 :invoke :cas [13 [47 11]] INFO [2019-12-17 12:28:56,924] jepsen worker 39 - jepsen.util 2289 :invoke :cas [13 [13 21]] INFO [2019-12-17 12:28:56,936] jepsen worker 42 - jepsen.util 2292 :info :cas [14 [17 15]] :RequestCanceledException INFO [2019-12-17 12:28:56,936] jepsen worker 68 - jepsen.util 1688 :info :cas [22 [3 3]] :RequestCanceledException INFO [2019-12-17 12:28:56,937] jepsen worker 42 - jepsen.util 2382 :invoke :cas [14 [41 33]] INFO [2019-12-17 12:28:56,937] jepsen worker 68 - jepsen.util 1778 :invoke :write [22 43] INFO [2019-12-17 12:28:56,941] jepsen worker 46 - jepsen.util 1756 :ok :write [15 35] INFO [2019-12-17 12:28:56,942] jepsen worker 46 - jepsen.util 1756 :invoke :write [15 31] INFO [2019-12-17 12:28:56,944] jepsen worker 78 - jepsen.util 1608 :ok :write [26 3] INFO [2019-12-17 12:28:56,944] jepsen worker 78 - jepsen.util 1608 :invoke :read [26 nil] INFO [2019-12-17 12:28:56,944] jepsen worker 80 - jepsen.util 1700 :ok :read [26 3] INFO [2019-12-17 12:28:56,945] jepsen worker 80 - jepsen.util 1700 :invoke :read [26 nil] INFO [2019-12-17 12:28:56,991] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:28:56,992] jepsen nemesis - couchbase.util Rebalance check count: 63 INFO [2019-12-17 12:28:56,992] jepsen nemesis - couchbase.util Rebalance stuck count: 44 INFO [2019-12-17 12:28:56,992] jepsen worker 29 - jepsen.util 1289 :ok :read [9 14] INFO [2019-12-17 12:28:56,992] jepsen worker 29 - jepsen.util 1289 :invoke :write [9 8] INFO [2019-12-17 12:28:57,014] jepsen worker 30 - jepsen.util 1740 :info :cas [10 [41 40]] :RequestCanceledException INFO [2019-12-17 12:28:57,017] jepsen worker 30 - jepsen.util 1830 :invoke :write [10 5] INFO [2019-12-17 12:28:57,034] jepsen worker 85 - jepsen.util 2245 :info :cas [28 [47 48]] :RequestCanceledException INFO [2019-12-17 12:28:57,037] jepsen worker 12 - jepsen.util 1632 :ok :read [4 19] INFO [2019-12-17 12:28:57,037] jepsen worker 12 - jepsen.util 1632 :invoke :read [4 nil] INFO [2019-12-17 12:28:57,039] jepsen worker 85 - jepsen.util 2335 :invoke :cas [28 [39 10]] INFO [2019-12-17 12:28:57,056] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9055,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:28:57,056] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9055,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:28:57,057] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9058,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:28:57,057] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9058,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:28:57,057] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9059,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:28:57,057] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9059,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:28:57,060] jepsen worker 52 - jepsen.util 1582 :ok :read [17 2] INFO [2019-12-17 12:28:57,060] jepsen worker 52 - jepsen.util 1582 :invoke :read [17 nil] INFO [2019-12-17 12:28:57,071] jepsen worker 27 - jepsen.util 1377 :ok :write [9 11] INFO [2019-12-17 12:28:57,071] jepsen worker 27 - jepsen.util 1377 :invoke :write [9 42] INFO [2019-12-17 12:28:57,080] jepsen worker 64 - jepsen.util 1234 :ok :write [21 35] INFO [2019-12-17 12:28:57,081] jepsen worker 64 - jepsen.util 1234 :invoke :cas [21 [19 11]] INFO [2019-12-17 12:28:57,092] jepsen worker 56 - jepsen.util 1586 :fail :cas [18 [46 49]] :ValueNotSwapFrom INFO [2019-12-17 12:28:57,092] jepsen worker 56 - jepsen.util 1586 :invoke :read [18 nil] INFO [2019-12-17 12:28:57,109] jepsen worker 69 - jepsen.util 1509 :fail :cas [23 [7 27]] :ValueNotSwapFrom INFO [2019-12-17 12:28:57,109] jepsen worker 69 - jepsen.util 1509 :invoke :cas [23 [47 2]] INFO [2019-12-17 12:28:57,128] jepsen worker 48 - jepsen.util 1488 :info :cas [16 [27 34]] :RequestCanceledException INFO [2019-12-17 12:28:57,131] jepsen worker 48 - jepsen.util 1578 :invoke :write [16 11] INFO [2019-12-17 12:28:57,131] jepsen worker 3 - jepsen.util 1443 :fail :cas [1 [41 18]] :ValueNotSwapFrom INFO [2019-12-17 12:28:57,132] jepsen worker 3 - jepsen.util 1443 :invoke :write [1 13] INFO [2019-12-17 12:28:57,147] jepsen worker 36 - jepsen.util 1566 :fail :cas [12 [13 9]] :ValueNotSwapFrom INFO [2019-12-17 12:28:57,148] jepsen worker 36 - jepsen.util 1566 :invoke :cas [12 [13 8]] INFO [2019-12-17 12:28:57,177] jepsen worker 78 - jepsen.util 1608 :ok :read [26 3] INFO [2019-12-17 12:28:57,178] jepsen worker 78 - jepsen.util 1608 :invoke :read [26 nil] INFO [2019-12-17 12:28:57,191] jepsen worker 6 - jepsen.util 2166 :fail :cas [2 [27 49]] :ValueNotSwapFrom INFO [2019-12-17 12:28:57,191] jepsen worker 6 - jepsen.util 2166 :invoke :read [2 nil] INFO [2019-12-17 12:28:57,207] jepsen worker 14 - jepsen.util 1544 :fail :cas [4 [22 34]] :ValueNotSwapFrom INFO [2019-12-17 12:28:57,208] jepsen worker 14 - jepsen.util 1544 :invoke :write [4 1] INFO [2019-12-17 12:28:57,222] jepsen worker 67 - jepsen.util 2407 :info :cas [22 [22 41]] :RequestCanceledException INFO [2019-12-17 12:28:57,223] jepsen worker 67 - jepsen.util 2497 :invoke :cas [22 [23 42]] INFO [2019-12-17 12:28:57,224] jepsen worker 27 - jepsen.util 1377 :ok :write [9 42] INFO [2019-12-17 12:28:57,224] jepsen worker 27 - jepsen.util 1377 :invoke :read [9 nil] INFO [2019-12-17 12:28:57,228] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9081,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:28:57,228] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9081,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:28:57,228] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9084,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:28:57,228] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9084,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:28:57,228] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9085,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:28:57,229] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9085,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:28:57,238] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9074,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:28:57,238] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9074,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:28:57,238] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9075,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:28:57,238] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9075,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:28:57,253] jepsen worker 69 - jepsen.util 1509 :fail :cas [23 [47 2]] :ValueNotSwapFrom INFO [2019-12-17 12:28:57,253] jepsen worker 69 - jepsen.util 1509 :invoke :read [23 nil] INFO [2019-12-17 12:28:57,261] jepsen worker 27 - jepsen.util 1377 :ok :read [9 42] INFO [2019-12-17 12:28:57,262] jepsen worker 27 - jepsen.util 1377 :invoke :write [9 29] INFO [2019-12-17 12:28:57,292] jepsen worker 36 - jepsen.util 1566 :fail :cas [12 [13 8]] :ValueNotSwapFrom INFO [2019-12-17 12:28:57,293] jepsen worker 36 - jepsen.util 1566 :invoke :read [12 nil] INFO [2019-12-17 12:28:57,301] jepsen worker 80 - jepsen.util 1700 :ok :read [26 3] INFO [2019-12-17 12:28:57,301] jepsen worker 80 - jepsen.util 1700 :invoke :read [26 nil] INFO [2019-12-17 12:28:57,322] jepsen worker 32 - jepsen.util 1832 :info :write [10 26] :RequestCanceledException INFO [2019-12-17 12:28:57,324] jepsen worker 32 - jepsen.util 1922 :invoke :write [10 13] INFO [2019-12-17 12:28:57,337] jepsen worker 78 - jepsen.util 1608 :ok :read [26 3] INFO [2019-12-17 12:28:57,337] jepsen worker 78 - jepsen.util 1608 :invoke :read [26 nil] INFO [2019-12-17 12:28:57,347] jepsen worker 27 - jepsen.util 1377 :ok :write [9 29] INFO [2019-12-17 12:28:57,347] jepsen worker 27 - jepsen.util 1377 :invoke :read [9 nil] INFO [2019-12-17 12:28:57,364] jepsen worker 13 - jepsen.util 1453 :ok :write [4 17] INFO [2019-12-17 12:28:57,364] jepsen worker 13 - jepsen.util 1453 :invoke :read [4 nil] INFO [2019-12-17 12:28:57,371] jepsen worker 27 - jepsen.util 1377 :ok :read [9 29] INFO [2019-12-17 12:28:57,371] jepsen worker 27 - jepsen.util 1377 :invoke :write [9 38] INFO [2019-12-17 12:28:57,385] jepsen worker 12 - jepsen.util 1632 :ok :read [4 19] INFO [2019-12-17 12:28:57,385] jepsen worker 12 - jepsen.util 1632 :invoke :write [4 10] INFO [2019-12-17 12:28:57,411] jepsen worker 9 - jepsen.util 1719 :ok :write [3 28] INFO [2019-12-17 12:28:57,411] jepsen worker 9 - jepsen.util 1719 :invoke :read [3 nil] INFO [2019-12-17 12:28:57,422] jepsen worker 49 - jepsen.util 1399 :info :write [16 48] :RequestCanceledException INFO [2019-12-17 12:28:57,425] jepsen worker 56 - jepsen.util 1586 :ok :read [18 35] INFO [2019-12-17 12:28:57,425] jepsen worker 56 - jepsen.util 1586 :invoke :read [18 nil] INFO [2019-12-17 12:28:57,425] jepsen worker 49 - jepsen.util 1489 :invoke :write [16 12] INFO [2019-12-17 12:28:57,430] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9103,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:28:57,430] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9103,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:28:57,431] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9105,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:28:57,431] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9105,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:28:57,455] jepsen worker 44 - jepsen.util 1664 :info :write [14 45] :RequestCanceledException INFO [2019-12-17 12:28:57,463] jepsen worker 54 - jepsen.util 1674 :ok :write [18 3] INFO [2019-12-17 12:28:57,465] jepsen worker 54 - jepsen.util 1674 :invoke :read [18 nil] INFO [2019-12-17 12:28:57,469] jepsen worker 44 - jepsen.util 1754 :invoke :read [14 nil] INFO [2019-12-17 12:28:57,483] jepsen worker 27 - jepsen.util 1377 :ok :write [9 38] INFO [2019-12-17 12:28:57,486] jepsen worker 27 - jepsen.util 1377 :invoke :read [9 nil] INFO [2019-12-17 12:28:57,493] jepsen worker 2 - jepsen.util 1712 :info :write [0 22] :RequestCanceledException INFO [2019-12-17 12:28:57,497] jepsen worker 4 - jepsen.util 1264 :ok :write [1 0] INFO [2019-12-17 12:28:57,497] jepsen worker 4 - jepsen.util 1264 :invoke :write [1 47] INFO [2019-12-17 12:28:57,520] jepsen worker 2 - jepsen.util 1802 :invoke :cas [0 [37 20]] INFO [2019-12-17 12:28:57,569] jepsen worker 31 - jepsen.util 1651 :ok :write [10 15] INFO [2019-12-17 12:28:57,569] jepsen worker 31 - jepsen.util 1651 :invoke :write [10 36] INFO [2019-12-17 12:28:57,579] jepsen worker 71 - jepsen.util 1241 :ok :write [23 20] INFO [2019-12-17 12:28:57,582] jepsen worker 71 - jepsen.util 1241 :invoke :cas [23 [35 40]] INFO [2019-12-17 12:28:57,642] jepsen worker 36 - jepsen.util 1566 :ok :read [12 32] INFO [2019-12-17 12:28:57,643] jepsen worker 36 - jepsen.util 1566 :invoke :cas [12 [18 48]] INFO [2019-12-17 12:28:57,650] jepsen worker 89 - jepsen.util 1529 :ok :write [29 26] INFO [2019-12-17 12:28:57,654] jepsen worker 89 - jepsen.util 1529 :invoke :read [29 nil] INFO [2019-12-17 12:28:57,658] jepsen worker 46 - jepsen.util 1756 :ok :write [15 31] INFO [2019-12-17 12:28:57,659] jepsen worker 46 - jepsen.util 1756 :invoke :cas [15 [44 3]] INFO [2019-12-17 12:28:57,693] jepsen worker 87 - jepsen.util 1437 :ok :write [29 31] INFO [2019-12-17 12:28:57,694] jepsen worker 87 - jepsen.util 1437 :invoke :read [29 nil] INFO [2019-12-17 12:28:57,708] jepsen worker 29 - jepsen.util 1289 :ok :write [9 8] INFO [2019-12-17 12:28:57,709] jepsen worker 29 - jepsen.util 1289 :invoke :read [9 nil] INFO [2019-12-17 12:28:57,717] jepsen worker 18 - jepsen.util 1818 :ok :write [6 41] INFO [2019-12-17 12:28:57,717] jepsen worker 18 - jepsen.util 1818 :invoke :cas [6 [21 7]] INFO [2019-12-17 12:28:57,718] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9115,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:28:57,719] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9115,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:28:57,719] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9117,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:28:57,720] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9117,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:28:57,738] jepsen worker 86 - jepsen.util 2156 :info :write [28 11] :RequestCanceledException INFO [2019-12-17 12:28:57,756] jepsen worker 33 - jepsen.util 2373 :info :write [11 22] :RequestCanceledException INFO [2019-12-17 12:28:57,760] jepsen worker 59 - jepsen.util 1679 :info :write [19 7] :RequestCanceledException INFO [2019-12-17 12:28:57,766] jepsen worker 35 - jepsen.util 1745 :info :write [11 18] :RequestCanceledException INFO [2019-12-17 12:28:57,766] jepsen worker 75 - jepsen.util 2235 :info :write [25 8] :RequestCanceledException INFO [2019-12-17 12:28:57,769] jepsen worker 86 - jepsen.util 2246 :invoke :write [28 29] INFO [2019-12-17 12:28:57,784] jepsen worker 70 - jepsen.util 1870 :ok :write [23 44] INFO [2019-12-17 12:28:57,789] jepsen worker 70 - jepsen.util 1870 :invoke :read [23 nil] INFO [2019-12-17 12:28:57,799] jepsen worker 50 - jepsen.util 2120 :info :write [16 27] :RequestCanceledException INFO [2019-12-17 12:28:57,814] jepsen worker 75 - jepsen.util 2325 :invoke :cas [25 [22 48]] INFO [2019-12-17 12:28:57,818] jepsen worker 83 - jepsen.util 2243 :info :write [27 23] :RequestCanceledException INFO [2019-12-17 12:28:57,823] jepsen worker 32 - jepsen.util 1922 :ok :write [10 13] INFO [2019-12-17 12:28:57,823] jepsen worker 32 - jepsen.util 1922 :invoke :cas [10 [24 4]] INFO [2019-12-17 12:28:57,824] jepsen worker 60 - jepsen.util 2220 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:28:57,824] jepsen worker 60 - jepsen.util 2220 :invoke :read [20 nil] INFO [2019-12-17 12:28:57,825] jepsen worker 59 - jepsen.util 1769 :invoke :write [19 21] INFO [2019-12-17 12:28:57,827] jepsen worker 43 - jepsen.util 2113 :info :write [14 43] :RequestCanceledException INFO [2019-12-17 12:28:57,833] jepsen worker 33 - jepsen.util 2463 :invoke :write [11 14] INFO [2019-12-17 12:28:57,834] jepsen worker 47 - jepsen.util 2027 :info :write [15 18] :RequestCanceledException INFO [2019-12-17 12:28:57,836] jepsen worker 35 - jepsen.util 1835 :invoke :write [11 0] INFO [2019-12-17 12:28:57,845] jepsen worker 22 - jepsen.util 2182 :info :cas [7 [1 3]] :RequestCanceledException INFO [2019-12-17 12:28:57,857] jepsen worker 50 - jepsen.util 2210 :invoke :write [16 10] INFO [2019-12-17 12:28:57,860] jepsen worker 1 - jepsen.util 2071 :info :cas [0 [20 14]] :RequestCanceledException INFO [2019-12-17 12:28:58,901] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9140,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:28:58,901] jepsen worker 84 - jepsen.util 2064 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:28:58,903] jepsen worker 65 - jepsen.util 1325 :info :write [21 17] :RequestCanceledException INFO [2019-12-17 12:28:58,903] jepsen worker 20 - jepsen.util 1460 :info :cas [6 [44 8]] :RequestCanceledException INFO [2019-12-17 12:28:58,904] jepsen worker 24 - jepsen.util 1914 :info :write [8 14] :RequestCanceledException INFO [2019-12-17 12:28:58,904] jepsen worker 84 - jepsen.util 2064 :invoke :read [28 nil] INFO [2019-12-17 12:28:58,904] jepsen worker 62 - jepsen.util 2132 :info :write [20 27] :RequestCanceledException INFO [2019-12-17 12:28:58,904] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9133,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:28:58,906] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9149,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:28:58,906] jepsen worker 77 - jepsen.util 1967 :info :write [25 16] :RequestCanceledException INFO [2019-12-17 12:28:58,906] jepsen worker 10 - jepsen.util 1720 :info :cas [3 [48 7]] :RequestCanceledException INFO [2019-12-17 12:28:58,906] jepsen worker 73 - jepsen.util 1513 :info :write [24 17] :RequestCanceledException INFO [2019-12-17 12:28:58,906] jepsen worker 0 - jepsen.util 2250 :info :cas [0 [16 28]] :RequestCanceledException INFO [2019-12-17 12:28:58,907] jepsen worker 17 - jepsen.util 2087 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:28:58,907] jepsen worker 81 - jepsen.util 1791 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:28:58,908] jepsen worker 61 - jepsen.util 2221 :info :cas [20 [5 3]] :RequestCanceledException INFO [2019-12-17 12:28:58,908] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9133,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:28:58,909] jepsen worker 38 - jepsen.util 1568 :info :write [12 6] :RequestCanceledException INFO [2019-12-17 12:28:58,909] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9149,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:28:58,909] jepsen worker 21 - jepsen.util 2001 :info :write [7 41] :RequestCanceledException INFO [2019-12-17 12:28:58,910] jepsen worker 15 - jepsen.util 1905 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:28:58,911] jepsen worker 25 - jepsen.util 2365 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:28:58,912] jepsen worker 45 - jepsen.util 1845 :info :write [15 38] :RequestCanceledException INFO [2019-12-17 12:28:58,913] jepsen worker 17 - jepsen.util 2087 :invoke :write [5 4] INFO [2019-12-17 12:28:58,913] jepsen worker 76 - jepsen.util 2056 :info :write [25 28] :RequestCanceledException INFO [2019-12-17 12:28:58,913] jepsen worker 81 - jepsen.util 1791 :invoke :write [27 25] INFO [2019-12-17 12:28:58,914] jepsen worker 57 - jepsen.util 1857 :info :write [19 21] :RequestCanceledException INFO [2019-12-17 12:28:58,914] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9139,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:28:58,914] jepsen worker 72 - jepsen.util 1602 :info :write [24 10] :RequestCanceledException INFO [2019-12-17 12:28:58,914] jepsen worker 63 - jepsen.util 1413 :info :cas [21 [27 29]] :RequestCanceledException INFO [2019-12-17 12:28:58,914] jepsen worker 82 - jepsen.util 2602 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:28:58,914] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9136,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:28:58,915] jepsen worker 83 - jepsen.util 2333 :invoke :write [27 4] INFO [2019-12-17 12:28:58,915] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9141,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:28:58,915] jepsen worker 43 - jepsen.util 2203 :invoke :read [14 nil] INFO [2019-12-17 12:28:58,916] jepsen worker 34 - jepsen.util 2194 :info :write [11 20] :RequestCanceledException INFO [2019-12-17 12:28:58,917] jepsen worker 8 - jepsen.util 1988 :info :write [2 19] :RequestCanceledException INFO [2019-12-17 12:28:58,917] jepsen worker 7 - jepsen.util 2257 :info :write [2 29] :RequestCanceledException INFO [2019-12-17 12:28:58,917] jepsen worker 11 - jepsen.util 1811 :info :cas [3 [14 5]] :RequestCanceledException INFO [2019-12-17 12:28:58,917] jepsen worker 47 - jepsen.util 2117 :invoke :write [15 3] INFO [2019-12-17 12:28:58,918] jepsen worker 58 - jepsen.util 1768 :info :cas [19 [46 15]] :RequestCanceledException INFO [2019-12-17 12:28:58,919] jepsen worker 15 - jepsen.util 1905 :invoke :cas [5 [16 47]] INFO [2019-12-17 12:28:58,920] jepsen worker 25 - jepsen.util 2365 :invoke :write [8 33] INFO [2019-12-17 12:28:58,921] jepsen worker 16 - jepsen.util 1906 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:28:58,921] jepsen worker 74 - jepsen.util 1514 :fail :read [24 nil] :RequestTimeoutException INFO [2019-12-17 12:28:58,921] jepsen worker 26 - jepsen.util 2276 :info :cas [8 [32 44]] :RequestCanceledException INFO [2019-12-17 12:28:58,922] jepsen worker 55 - jepsen.util 1855 :info :write [18 10] :RequestCanceledException INFO [2019-12-17 12:28:58,922] jepsen worker 19 - jepsen.util 1819 :fail :read [6 nil] :RequestTimeoutException INFO [2019-12-17 12:28:58,922] jepsen worker 79 - jepsen.util 1609 :info :write [26 9] :RequestCanceledException INFO [2019-12-17 12:28:58,922] jepsen worker 28 - jepsen.util 928 :info :cas [9 [30 4]] :RequestCanceledException INFO [2019-12-17 12:28:58,924] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9139,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:28:58,924] jepsen worker 23 - jepsen.util 1913 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:28:58,925] jepsen worker 88 - jepsen.util 1438 :info :write [29 0] :RequestCanceledException INFO [2019-12-17 12:28:58,925] jepsen worker 66 - jepsen.util 1776 :info :write [22 41] :RequestCanceledException INFO [2019-12-17 12:28:58,927] jepsen worker 82 - jepsen.util 2602 :invoke :read [27 nil] INFO [2019-12-17 12:28:58,927] jepsen worker 22 - jepsen.util 2272 :invoke :read [7 nil] INFO [2019-12-17 12:28:58,927] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9136,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:28:58,929] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9141,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:28:58,930] jepsen worker 1 - jepsen.util 2161 :invoke :write [0 12] INFO [2019-12-17 12:28:58,932] jepsen worker 20 - jepsen.util 1550 :invoke :write [6 12] INFO [2019-12-17 12:28:58,936] jepsen worker 77 - jepsen.util 2057 :invoke :write [25 46] INFO [2019-12-17 12:28:58,936] jepsen worker 74 - jepsen.util 1514 :invoke :read [24 nil] INFO [2019-12-17 12:28:58,936] jepsen worker 16 - jepsen.util 1906 :invoke :cas [5 [33 39]] INFO [2019-12-17 12:28:58,937] jepsen worker 62 - jepsen.util 2222 :invoke :read [20 nil] INFO [2019-12-17 12:28:58,939] jepsen worker 19 - jepsen.util 1819 :invoke :write [6 27] INFO [2019-12-17 12:28:58,939] jepsen worker 65 - jepsen.util 1415 :invoke :write [21 44] INFO [2019-12-17 12:28:58,940] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9157,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:28:58,941] jepsen worker 10 - jepsen.util 1810 :invoke :write [3 24] INFO [2019-12-17 12:28:58,941] jepsen worker 23 - jepsen.util 1913 :invoke :write [7 49] INFO [2019-12-17 12:28:58,942] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9156,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:28:58,942] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9138,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:28:58,946] jepsen worker 63 - jepsen.util 1503 :invoke :cas [21 [34 22]] INFO [2019-12-17 12:28:58,946] jepsen worker 24 - jepsen.util 2004 :invoke :cas [8 [44 29]] INFO [2019-12-17 12:28:58,947] jepsen worker 7 - jepsen.util 2347 :invoke :read [2 nil] INFO [2019-12-17 12:28:58,947] jepsen worker 34 - jepsen.util 2284 :invoke :read [11 nil] INFO [2019-12-17 12:28:58,947] jepsen worker 26 - jepsen.util 2366 :invoke :read [8 nil] INFO [2019-12-17 12:28:58,947] jepsen worker 58 - jepsen.util 1858 :invoke :cas [19 [16 10]] INFO [2019-12-17 12:28:58,947] jepsen worker 11 - jepsen.util 1901 :invoke :cas [3 [22 21]] INFO [2019-12-17 12:28:58,947] jepsen worker 8 - jepsen.util 2078 :invoke :read [2 nil] INFO [2019-12-17 12:28:58,948] jepsen worker 28 - jepsen.util 1018 :invoke :write [9 25] INFO [2019-12-17 12:28:58,948] jepsen worker 79 - jepsen.util 1699 :invoke :read [26 nil] INFO [2019-12-17 12:28:58,948] jepsen worker 55 - jepsen.util 1945 :invoke :write [18 10] INFO [2019-12-17 12:28:58,949] jepsen worker 66 - jepsen.util 1866 :invoke :read [22 nil] INFO [2019-12-17 12:28:58,949] jepsen worker 88 - jepsen.util 1528 :invoke :write [29 32] INFO [2019-12-17 12:28:58,950] jepsen worker 76 - jepsen.util 2146 :invoke :cas [25 [4 33]] INFO [2019-12-17 12:28:58,950] jepsen worker 38 - jepsen.util 1658 :invoke :cas [12 [32 10]] INFO [2019-12-17 12:28:58,951] jepsen worker 73 - jepsen.util 1603 :invoke :read [24 nil] INFO [2019-12-17 12:28:58,952] jepsen worker 21 - jepsen.util 2091 :invoke :write [7 19] INFO [2019-12-17 12:28:58,953] jepsen worker 45 - jepsen.util 1935 :invoke :read [15 nil] INFO [2019-12-17 12:28:58,953] jepsen worker 72 - jepsen.util 1692 :invoke :read [24 nil] INFO [2019-12-17 12:28:58,953] jepsen worker 0 - jepsen.util 2340 :invoke :read [0 nil] INFO [2019-12-17 12:28:58,954] jepsen worker 57 - jepsen.util 1947 :invoke :read [19 nil] INFO [2019-12-17 12:28:58,955] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9157,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:28:58,955] jepsen worker 61 - jepsen.util 2311 :invoke :read [20 nil] INFO [2019-12-17 12:28:58,955] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9156,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:28:58,955] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9138,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:28:58,959] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9158,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:28:58,960] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9158,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:28:58,987] jepsen worker 4 - jepsen.util 1264 :ok :write [1 47] INFO [2019-12-17 12:28:58,987] jepsen worker 4 - jepsen.util 1264 :invoke :read [1 nil] INFO [2019-12-17 12:28:58,989] jepsen worker 39 - jepsen.util 2289 :info :cas [13 [13 21]] :RequestCanceledException INFO [2019-12-17 12:28:58,998] jepsen worker 39 - jepsen.util 2379 :invoke :cas [13 [27 41]] INFO [2019-12-17 12:28:59,006] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9150,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:28:59,006] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9150,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:28:59,006] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9200,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:28:59,006] jepsen worker 42 - jepsen.util 2382 :info :cas [14 [41 33]] :RequestCanceledException INFO [2019-12-17 12:28:59,007] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9200,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:28:59,007] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9202,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:28:59,007] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9202,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:28:59,012] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9140,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:28:59,013] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9166,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:28:59,013] jepsen worker 42 - jepsen.util 2472 :invoke :write [14 43] INFO [2019-12-17 12:28:59,013] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9166,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:28:59,019] jepsen worker 41 - jepsen.util 2111 :info :cas [13 [47 11]] :RequestCanceledException INFO [2019-12-17 12:28:59,019] jepsen worker 40 - jepsen.util 2380 :info :cas [13 [30 6]] :RequestCanceledException INFO [2019-12-17 12:28:59,019] jepsen worker 68 - jepsen.util 1778 :info :write [22 43] :RequestCanceledException INFO [2019-12-17 12:28:59,027] jepsen worker 68 - jepsen.util 1868 :invoke :read [22 nil] INFO [2019-12-17 12:28:59,027] jepsen worker 40 - jepsen.util 2470 :invoke :cas [13 [16 45]] INFO [2019-12-17 12:28:59,027] jepsen worker 41 - jepsen.util 2201 :invoke :write [13 29] INFO [2019-12-17 12:28:59,044] jepsen worker 85 - jepsen.util 2335 :info :cas [28 [39 10]] :RequestCanceledException INFO [2019-12-17 12:28:59,055] jepsen worker 85 - jepsen.util 2425 :invoke :read [28 nil] INFO [2019-12-17 12:28:59,060] jepsen worker 3 - jepsen.util 1443 :ok :write [1 13] INFO [2019-12-17 12:28:59,061] jepsen worker 3 - jepsen.util 1443 :invoke :write [1 9] INFO [2019-12-17 12:28:59,067] jepsen worker 51 - jepsen.util 1581 :ok :read [17 2] INFO [2019-12-17 12:28:59,068] jepsen worker 51 - jepsen.util 1581 :invoke :cas [17 [47 30]] INFO [2019-12-17 12:28:59,071] jepsen worker 37 - jepsen.util 1297 :fail :cas [12 [36 4]] :ValueNotSwapFrom INFO [2019-12-17 12:28:59,071] jepsen worker 37 - jepsen.util 1297 :invoke :cas [12 [45 41]] INFO [2019-12-17 12:28:59,071] jepsen worker 6 - jepsen.util 2166 :ok :read [2 15] INFO [2019-12-17 12:28:59,071] jepsen worker 6 - jepsen.util 2166 :invoke :cas [2 [40 24]] INFO [2019-12-17 12:28:59,072] jepsen worker 53 - jepsen.util 1493 :ok :read [17 2] INFO [2019-12-17 12:28:59,072] jepsen worker 53 - jepsen.util 1493 :invoke :read [17 nil] INFO [2019-12-17 12:28:59,074] jepsen worker 52 - jepsen.util 1582 :ok :read [17 2] INFO [2019-12-17 12:28:59,074] jepsen worker 27 - jepsen.util 1377 :ok :read [9 38] INFO [2019-12-17 12:28:59,075] jepsen worker 27 - jepsen.util 1377 :invoke :read [9 nil] INFO [2019-12-17 12:28:59,075] jepsen worker 52 - jepsen.util 1582 :invoke :cas [17 [14 26]] INFO [2019-12-17 12:28:59,118] jepsen worker 29 - jepsen.util 1289 :ok :read [9 8] INFO [2019-12-17 12:28:59,118] jepsen worker 29 - jepsen.util 1289 :invoke :cas [9 [5 6]] INFO [2019-12-17 12:28:59,137] jepsen worker 30 - jepsen.util 1830 :info :write [10 5] :RequestCanceledException INFO [2019-12-17 12:28:59,138] jepsen worker 64 - jepsen.util 1234 :info :cas [21 [19 11]] :RequestCanceledException INFO [2019-12-17 12:28:59,163] jepsen worker 30 - jepsen.util 1920 :invoke :write [10 29] INFO [2019-12-17 12:28:59,163] jepsen worker 64 - jepsen.util 1324 :invoke :cas [21 [41 38]] INFO [2019-12-17 12:28:59,172] jepsen worker 54 - jepsen.util 1674 :ok :read [18 3] INFO [2019-12-17 12:28:59,173] jepsen worker 54 - jepsen.util 1674 :invoke :read [18 nil] INFO [2019-12-17 12:28:59,200] jepsen worker 69 - jepsen.util 1509 :ok :read [23 26] INFO [2019-12-17 12:28:59,201] jepsen worker 69 - jepsen.util 1509 :invoke :read [23 nil] INFO [2019-12-17 12:28:59,208] jepsen worker 9 - jepsen.util 1719 :ok :read [3 28] INFO [2019-12-17 12:28:59,209] jepsen worker 9 - jepsen.util 1719 :invoke :cas [3 [47 38]] INFO [2019-12-17 12:28:59,209] jepsen worker 13 - jepsen.util 1453 :ok :read [4 17] INFO [2019-12-17 12:28:59,209] jepsen worker 13 - jepsen.util 1453 :invoke :cas [4 [35 1]] INFO [2019-12-17 12:28:59,211] jepsen worker 78 - jepsen.util 1608 :ok :read [26 3] INFO [2019-12-17 12:28:59,211] jepsen worker 78 - jepsen.util 1608 :invoke :write [26 21] INFO [2019-12-17 12:28:59,213] jepsen worker 5 - jepsen.util 1265 :ok :read [1 13] INFO [2019-12-17 12:28:59,214] jepsen worker 5 - jepsen.util 1265 :invoke :cas [1 [29 7]] INFO [2019-12-17 12:28:59,216] jepsen worker 71 - jepsen.util 1241 :fail :cas [23 [35 40]] :ValueNotSwapFrom INFO [2019-12-17 12:28:59,216] jepsen worker 18 - jepsen.util 1818 :fail :cas [6 [21 7]] :ValueNotSwapFrom INFO [2019-12-17 12:28:59,216] jepsen worker 71 - jepsen.util 1241 :invoke :read [23 nil] INFO [2019-12-17 12:28:59,216] jepsen worker 18 - jepsen.util 1818 :invoke :write [6 3] INFO [2019-12-17 12:28:59,219] jepsen worker 48 - jepsen.util 1578 :info :write [16 11] :RequestCanceledException INFO [2019-12-17 12:28:59,220] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9244,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:28:59,220] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9244,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:28:59,221] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9245,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:28:59,221] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9245,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:28:59,224] jepsen worker 46 - jepsen.util 1756 :fail :cas [15 [44 3]] :ValueNotSwapFrom INFO [2019-12-17 12:28:59,225] jepsen worker 46 - jepsen.util 1756 :invoke :read [15 nil] INFO [2019-12-17 12:28:59,225] jepsen worker 48 - jepsen.util 1668 :invoke :write [16 18] INFO [2019-12-17 12:28:59,242] jepsen worker 32 - jepsen.util 1922 :fail :cas [10 [24 4]] :ValueNotSwapFrom INFO [2019-12-17 12:28:59,242] jepsen worker 32 - jepsen.util 1922 :invoke :write [10 40] INFO [2019-12-17 12:28:59,258] jepsen worker 79 - jepsen.util 1699 :ok :read [26 3] INFO [2019-12-17 12:28:59,258] jepsen worker 79 - jepsen.util 1699 :invoke :cas [26 [34 45]] INFO [2019-12-17 12:28:59,258] jepsen worker 87 - jepsen.util 1437 :ok :read [29 31] INFO [2019-12-17 12:28:59,259] jepsen worker 87 - jepsen.util 1437 :invoke :read [29 nil] INFO [2019-12-17 12:28:59,269] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:28:59,269] jepsen nemesis - couchbase.util Rebalance check count: 64 INFO [2019-12-17 12:28:59,269] jepsen nemesis - couchbase.util Rebalance stuck count: 45 INFO [2019-12-17 12:28:59,283] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9251,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:28:59,284] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9251,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:28:59,284] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9254,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:28:59,284] jepsen worker 51 - jepsen.util 1581 :fail :cas [17 [47 30]] :ValueNotSwapFrom INFO [2019-12-17 12:28:59,284] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9254,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:28:59,284] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9255,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:28:59,284] jepsen worker 51 - jepsen.util 1581 :invoke :read [17 nil] INFO [2019-12-17 12:28:59,284] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9255,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:28:59,287] jepsen worker 67 - jepsen.util 2497 :info :cas [22 [23 42]] :RequestCanceledException INFO [2019-12-17 12:28:59,289] jepsen worker 67 - jepsen.util 2587 :invoke :cas [22 [26 2]] INFO [2019-12-17 12:28:59,314] jepsen worker 74 - jepsen.util 1514 :ok :read [24 36] INFO [2019-12-17 12:28:59,315] jepsen worker 74 - jepsen.util 1514 :invoke :write [24 21] INFO [2019-12-17 12:28:59,319] jepsen worker 80 - jepsen.util 1700 :fail :read [26 nil] :RequestTimeoutException INFO [2019-12-17 12:28:59,319] jepsen worker 14 - jepsen.util 1544 :info :write [4 1] :RequestCanceledException INFO [2019-12-17 12:28:59,320] jepsen worker 80 - jepsen.util 1700 :invoke :cas [26 [42 20]] INFO [2019-12-17 12:28:59,323] jepsen worker 14 - jepsen.util 1634 :invoke :write [4 8] INFO [2019-12-17 12:28:59,330] jepsen worker 89 - jepsen.util 1529 :ok :read [29 31] INFO [2019-12-17 12:28:59,330] jepsen worker 89 - jepsen.util 1529 :invoke :write [29 14] INFO [2019-12-17 12:28:59,350] jepsen worker 45 - jepsen.util 1935 :ok :read [15 31] INFO [2019-12-17 12:28:59,350] jepsen worker 45 - jepsen.util 1935 :invoke :read [15 nil] INFO [2019-12-17 12:28:59,363] jepsen worker 63 - jepsen.util 1503 :fail :cas [21 [34 22]] :ValueNotSwapFrom INFO [2019-12-17 12:28:59,364] jepsen worker 63 - jepsen.util 1503 :invoke :cas [21 [12 38]] INFO [2019-12-17 12:28:59,389] jepsen worker 73 - jepsen.util 1603 :ok :read [24 36] INFO [2019-12-17 12:28:59,389] jepsen worker 73 - jepsen.util 1603 :invoke :write [24 24] INFO [2019-12-17 12:28:59,416] jepsen worker 11 - jepsen.util 1901 :fail :cas [3 [22 21]] :ValueNotSwapFrom INFO [2019-12-17 12:28:59,417] jepsen worker 11 - jepsen.util 1901 :invoke :write [3 12] INFO [2019-12-17 12:28:59,422] jepsen worker 12 - jepsen.util 1632 :info :write [4 10] :RequestCanceledException INFO [2019-12-17 12:28:59,427] jepsen worker 12 - jepsen.util 1722 :invoke :read [4 nil] INFO [2019-12-17 12:28:59,438] jepsen worker 72 - jepsen.util 1692 :ok :read [24 36] INFO [2019-12-17 12:28:59,438] jepsen worker 72 - jepsen.util 1692 :invoke :write [24 37] INFO [2019-12-17 12:28:59,469] jepsen worker 37 - jepsen.util 1297 :fail :cas [12 [45 41]] :ValueNotSwapFrom INFO [2019-12-17 12:28:59,470] jepsen worker 37 - jepsen.util 1297 :invoke :read [12 nil] INFO [2019-12-17 12:28:59,482] jepsen worker 70 - jepsen.util 1870 :ok :read [23 44] INFO [2019-12-17 12:28:59,483] jepsen worker 70 - jepsen.util 1870 :invoke :read [23 nil] INFO [2019-12-17 12:28:59,486] jepsen worker 36 - jepsen.util 1566 :fail :cas [12 [18 48]] :ValueNotSwapFrom INFO [2019-12-17 12:28:59,486] jepsen worker 36 - jepsen.util 1566 :invoke :read [12 nil] INFO [2019-12-17 12:28:59,486] jepsen worker 63 - jepsen.util 1503 :fail :cas [21 [12 38]] :ValueNotSwapFrom INFO [2019-12-17 12:28:59,486] jepsen worker 63 - jepsen.util 1503 :invoke :read [21 nil] INFO [2019-12-17 12:28:59,501] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9277,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:28:59,501] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9277,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:28:59,501] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9281,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:28:59,502] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9281,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:28:59,502] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9282,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:28:59,502] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9282,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:28:59,515] jepsen worker 38 - jepsen.util 1658 :fail :cas [12 [32 10]] :ValueNotSwapFrom INFO [2019-12-17 12:28:59,516] jepsen worker 38 - jepsen.util 1658 :invoke :read [12 nil] INFO [2019-12-17 12:28:59,522] jepsen worker 56 - jepsen.util 1586 :fail :read [18 nil] :RequestTimeoutException INFO [2019-12-17 12:28:59,522] jepsen worker 49 - jepsen.util 1489 :info :write [16 12] :RequestCanceledException INFO [2019-12-17 12:28:59,522] jepsen worker 56 - jepsen.util 1586 :invoke :cas [18 [0 0]] INFO [2019-12-17 12:28:59,524] jepsen worker 49 - jepsen.util 1579 :invoke :read [16 nil] INFO [2019-12-17 12:28:59,532] jepsen worker 53 - jepsen.util 1493 :ok :read [17 2] INFO [2019-12-17 12:28:59,532] jepsen worker 53 - jepsen.util 1493 :invoke :write [17 2] INFO [2019-12-17 12:28:59,536] jepsen worker 44 - jepsen.util 1754 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:28:59,536] jepsen worker 44 - jepsen.util 1754 :invoke :read [14 nil] INFO [2019-12-17 12:28:59,550] jepsen worker 2 - jepsen.util 1802 :info :cas [0 [37 20]] :RequestCanceledException INFO [2019-12-17 12:28:59,553] jepsen worker 2 - jepsen.util 1892 :invoke :write [0 33] INFO [2019-12-17 12:28:59,554] jepsen worker 54 - jepsen.util 1674 :ok :read [18 3] INFO [2019-12-17 12:28:59,554] jepsen worker 54 - jepsen.util 1674 :invoke :write [18 11] INFO [2019-12-17 12:28:59,603] jepsen worker 28 - jepsen.util 1018 :ok :write [9 25] INFO [2019-12-17 12:28:59,604] jepsen worker 28 - jepsen.util 1018 :invoke :read [9 nil] INFO [2019-12-17 12:28:59,654] jepsen worker 31 - jepsen.util 1651 :info :write [10 36] :RequestCanceledException INFO [2019-12-17 12:28:59,662] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9301,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:28:59,663] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9301,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:28:59,664] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9303,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:28:59,664] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9303,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:28:59,667] jepsen worker 31 - jepsen.util 1741 :invoke :write [10 39] INFO [2019-12-17 12:28:59,689] jepsen worker 71 - jepsen.util 1241 :ok :read [23 44] INFO [2019-12-17 12:28:59,689] jepsen worker 71 - jepsen.util 1241 :invoke :cas [23 [31 4]] INFO [2019-12-17 12:28:59,691] jepsen worker 13 - jepsen.util 1453 :fail :cas [4 [35 1]] :ValueNotSwapFrom INFO [2019-12-17 12:28:59,692] jepsen worker 13 - jepsen.util 1453 :invoke :cas [4 [20 11]] INFO [2019-12-17 12:28:59,706] jepsen worker 28 - jepsen.util 1018 :ok :read [9 25] INFO [2019-12-17 12:28:59,707] jepsen worker 28 - jepsen.util 1018 :invoke :write [9 48] INFO [2019-12-17 12:28:59,718] jepsen worker 87 - jepsen.util 1437 :ok :read [29 31] INFO [2019-12-17 12:28:59,718] jepsen worker 87 - jepsen.util 1437 :invoke :write [29 21] INFO [2019-12-17 12:28:59,723] jepsen worker 7 - jepsen.util 2347 :ok :read [2 15] INFO [2019-12-17 12:28:59,723] jepsen worker 7 - jepsen.util 2347 :invoke :read [2 nil] INFO [2019-12-17 12:28:59,742] jepsen worker 5 - jepsen.util 1265 :fail :cas [1 [29 7]] :ValueNotSwapFrom INFO [2019-12-17 12:28:59,743] jepsen worker 5 - jepsen.util 1265 :invoke :read [1 nil] INFO [2019-12-17 12:28:59,819] jepsen worker 37 - jepsen.util 1297 :ok :read [12 6] INFO [2019-12-17 12:28:59,820] jepsen worker 37 - jepsen.util 1297 :invoke :read [12 nil] INFO [2019-12-17 12:28:59,842] jepsen worker 6 - jepsen.util 2166 :fail :cas [2 [40 24]] :ValueNotSwapFrom INFO [2019-12-17 12:28:59,843] jepsen worker 6 - jepsen.util 2166 :invoke :write [2 5] INFO [2019-12-17 12:28:59,868] jepsen worker 4 - jepsen.util 1264 :ok :read [1 47] INFO [2019-12-17 12:28:59,869] jepsen worker 4 - jepsen.util 1264 :invoke :read [1 nil] INFO [2019-12-17 12:28:59,875] jepsen worker 7 - jepsen.util 2347 :ok :read [2 15] INFO [2019-12-17 12:28:59,875] jepsen worker 7 - jepsen.util 2347 :invoke :read [2 nil] INFO [2019-12-17 12:28:59,879] jepsen worker 75 - jepsen.util 2325 :info :cas [25 [22 48]] :RequestCanceledException INFO [2019-12-17 12:28:59,882] jepsen worker 75 - jepsen.util 2415 :invoke :write [25 24] INFO [2019-12-17 12:28:59,884] jepsen worker 28 - jepsen.util 1018 :ok :write [9 48] INFO [2019-12-17 12:28:59,885] jepsen worker 28 - jepsen.util 1018 :invoke :cas [9 [37 11]] INFO [2019-12-17 12:28:59,895] jepsen worker 14 - jepsen.util 1634 :ok :write [4 8] INFO [2019-12-17 12:28:59,895] jepsen worker 14 - jepsen.util 1634 :invoke :read [4 nil] INFO [2019-12-17 12:28:59,908] jepsen worker 60 - jepsen.util 2220 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:28:59,908] jepsen worker 60 - jepsen.util 2220 :invoke :write [20 14] INFO [2019-12-17 12:28:59,926] jepsen worker 28 - jepsen.util 1018 :fail :cas [9 [37 11]] :ValueNotSwapFrom INFO [2019-12-17 12:28:59,927] jepsen worker 28 - jepsen.util 1018 :invoke :read [9 nil] INFO [2019-12-17 12:28:59,948] jepsen worker 79 - jepsen.util 1699 :fail :cas [26 [34 45]] :ValueNotSwapFrom INFO [2019-12-17 12:28:59,948] jepsen worker 79 - jepsen.util 1699 :invoke :read [26 nil] INFO [2019-12-17 12:28:59,950] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9315,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:28:59,950] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9315,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:28:59,950] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9317,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:28:59,950] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9317,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:28:59,972] jepsen worker 28 - jepsen.util 1018 :ok :read [9 48] INFO [2019-12-17 12:28:59,972] jepsen worker 28 - jepsen.util 1018 :invoke :write [9 40] INFO [2019-12-17 12:29:00,010] jepsen worker 37 - jepsen.util 1297 :ok :read [12 6] INFO [2019-12-17 12:29:00,010] jepsen worker 37 - jepsen.util 1297 :invoke :read [12 nil] INFO [2019-12-17 12:29:00,023] jepsen worker 7 - jepsen.util 2347 :ok :read [2 15] INFO [2019-12-17 12:29:00,023] jepsen worker 7 - jepsen.util 2347 :invoke :cas [2 [3 21]] INFO [2019-12-17 12:29:00,037] jepsen worker 51 - jepsen.util 1581 :ok :read [17 2] INFO [2019-12-17 12:29:00,037] jepsen worker 51 - jepsen.util 1581 :invoke :cas [17 [2 2]] INFO [2019-12-17 12:29:00,066] jepsen worker 28 - jepsen.util 1018 :ok :write [9 40] INFO [2019-12-17 12:29:00,067] jepsen worker 28 - jepsen.util 1018 :invoke :cas [9 [6 15]] INFO [2019-12-17 12:29:00,069] jepsen worker 12 - jepsen.util 1722 :ok :read [4 8] INFO [2019-12-17 12:29:00,070] jepsen worker 12 - jepsen.util 1722 :invoke :cas [4 [45 20]] INFO [2019-12-17 12:29:00,073] jepsen worker 55 - jepsen.util 1945 :ok :write [18 10] INFO [2019-12-17 12:29:00,074] jepsen worker 55 - jepsen.util 1945 :invoke :cas [18 [31 27]] INFO [2019-12-17 12:29:00,078] jepsen worker 88 - jepsen.util 1528 :ok :write [29 32] INFO [2019-12-17 12:29:00,079] jepsen worker 88 - jepsen.util 1528 :invoke :write [29 18] INFO [2019-12-17 12:29:00,081] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9329,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:29:00,081] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9329,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:29:00,081] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9332,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:29:00,082] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9332,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:29:00,091] jepsen worker 7 - jepsen.util 2347 :fail :cas [2 [3 21]] :ValueNotSwapFrom INFO [2019-12-17 12:29:00,091] jepsen worker 7 - jepsen.util 2347 :invoke :write [2 21] INFO [2019-12-17 12:29:00,107] jepsen worker 28 - jepsen.util 1018 :fail :cas [9 [6 15]] :ValueNotSwapFrom INFO [2019-12-17 12:29:00,107] jepsen worker 28 - jepsen.util 1018 :invoke :cas [9 [43 18]] INFO [2019-12-17 12:29:00,146] jepsen worker 28 - jepsen.util 1018 :fail :cas [9 [43 18]] :ValueNotSwapFrom INFO [2019-12-17 12:29:00,147] jepsen worker 28 - jepsen.util 1018 :invoke :write [9 24] INFO [2019-12-17 12:29:00,157] jepsen worker 52 - jepsen.util 1582 :fail :cas [17 [14 26]] :ValueNotSwapFrom INFO [2019-12-17 12:29:00,158] jepsen worker 52 - jepsen.util 1582 :invoke :read [17 nil] INFO [2019-12-17 12:29:00,185] jepsen worker 37 - jepsen.util 1297 :ok :read [12 6] INFO [2019-12-17 12:29:00,185] jepsen worker 37 - jepsen.util 1297 :invoke :write [12 33] INFO [2019-12-17 12:29:00,205] jepsen worker 73 - jepsen.util 1603 :ok :write [24 24] INFO [2019-12-17 12:29:00,205] jepsen worker 20 - jepsen.util 1550 :ok :write [6 12] INFO [2019-12-17 12:29:00,205] jepsen worker 73 - jepsen.util 1603 :invoke :read [24 nil] INFO [2019-12-17 12:29:00,205] jepsen worker 10 - jepsen.util 1810 :ok :write [3 24] INFO [2019-12-17 12:29:00,205] jepsen worker 20 - jepsen.util 1550 :invoke :cas [6 [25 20]] INFO [2019-12-17 12:29:00,205] jepsen worker 10 - jepsen.util 1810 :invoke :read [3 nil] INFO [2019-12-17 12:29:00,206] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9347,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:29:00,206] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9347,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:29:00,228] jepsen worker 89 - jepsen.util 1529 :ok :write [29 14] INFO [2019-12-17 12:29:00,228] jepsen worker 89 - jepsen.util 1529 :invoke :read [29 nil] INFO [2019-12-17 12:29:00,254] jepsen worker 47 - jepsen.util 2117 :ok :write [15 3] INFO [2019-12-17 12:29:00,255] jepsen worker 47 - jepsen.util 2117 :invoke :read [15 nil] INFO [2019-12-17 12:29:00,264] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9355,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:29:00,264] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9355,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:29:00,264] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9357,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:29:00,265] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9357,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:29:00,277] jepsen worker 10 - jepsen.util 1810 :ok :read [3 24] INFO [2019-12-17 12:29:00,277] jepsen worker 10 - jepsen.util 1810 :invoke :read [3 nil] INFO [2019-12-17 12:29:00,281] jepsen worker 73 - jepsen.util 1603 :ok :read [24 24] INFO [2019-12-17 12:29:00,281] jepsen worker 73 - jepsen.util 1603 :invoke :read [24 nil] INFO [2019-12-17 12:29:00,282] jepsen worker 20 - jepsen.util 1550 :fail :cas [6 [25 20]] :ValueNotSwapFrom INFO [2019-12-17 12:29:00,282] jepsen worker 20 - jepsen.util 1550 :invoke :read [6 nil] INFO [2019-12-17 12:29:00,289] jepsen worker 69 - jepsen.util 1509 :ok :read [23 44] INFO [2019-12-17 12:29:00,289] jepsen worker 69 - jepsen.util 1509 :invoke :write [23 40] INFO [2019-12-17 12:29:00,303] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:29:00,303] jepsen nemesis - couchbase.util Rebalance check count: 65 INFO [2019-12-17 12:29:00,303] jepsen nemesis - couchbase.util Rebalance stuck count: 46 INFO [2019-12-17 12:29:00,313] jepsen worker 31 - jepsen.util 1741 :ok :write [10 39] INFO [2019-12-17 12:29:00,313] jepsen worker 31 - jepsen.util 1741 :invoke :write [10 38] INFO [2019-12-17 12:29:00,363] jepsen worker 28 - jepsen.util 1018 :ok :write [9 24] INFO [2019-12-17 12:29:00,364] jepsen worker 28 - jepsen.util 1018 :invoke :read [9 nil] INFO [2019-12-17 12:29:00,371] jepsen worker 10 - jepsen.util 1810 :ok :read [3 24] INFO [2019-12-17 12:29:00,371] jepsen worker 10 - jepsen.util 1810 :invoke :write [3 35] INFO [2019-12-17 12:29:00,377] jepsen worker 55 - jepsen.util 1945 :fail :cas [18 [31 27]] :ValueNotSwapFrom INFO [2019-12-17 12:29:00,377] jepsen worker 55 - jepsen.util 1945 :invoke :cas [18 [2 15]] INFO [2019-12-17 12:29:00,396] jepsen worker 38 - jepsen.util 1658 :ok :read [12 6] INFO [2019-12-17 12:29:00,396] jepsen worker 73 - jepsen.util 1603 :ok :read [24 24] INFO [2019-12-17 12:29:00,397] jepsen worker 38 - jepsen.util 1658 :invoke :cas [12 [21 43]] INFO [2019-12-17 12:29:00,397] jepsen worker 73 - jepsen.util 1603 :invoke :cas [24 [46 6]] INFO [2019-12-17 12:29:00,405] jepsen worker 20 - jepsen.util 1550 :ok :read [6 12] INFO [2019-12-17 12:29:00,405] jepsen worker 20 - jepsen.util 1550 :invoke :read [6 nil] INFO [2019-12-17 12:29:00,464] jepsen worker 4 - jepsen.util 1264 :ok :read [1 47] INFO [2019-12-17 12:29:00,464] jepsen worker 4 - jepsen.util 1264 :invoke :cas [1 [20 35]] INFO [2019-12-17 12:29:00,466] jepsen worker 73 - jepsen.util 1603 :fail :cas [24 [46 6]] :ValueNotSwapFrom INFO [2019-12-17 12:29:00,466] jepsen worker 73 - jepsen.util 1603 :invoke :write [24 38] INFO [2019-12-17 12:29:00,477] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9375,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:29:00,477] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9375,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:29:00,477] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9378,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:29:00,478] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9378,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:29:00,478] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9379,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:29:00,478] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9379,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:29:00,480] jepsen worker 20 - jepsen.util 1550 :ok :read [6 12] INFO [2019-12-17 12:29:00,480] jepsen worker 20 - jepsen.util 1550 :invoke :write [6 20] INFO [2019-12-17 12:29:00,501] jepsen worker 65 - jepsen.util 1415 :ok :write [21 44] INFO [2019-12-17 12:29:00,501] jepsen worker 65 - jepsen.util 1415 :invoke :read [21 nil] INFO [2019-12-17 12:29:00,527] jepsen worker 28 - jepsen.util 1018 :ok :read [9 24] INFO [2019-12-17 12:29:00,528] jepsen worker 28 - jepsen.util 1018 :invoke :cas [9 [33 1]] INFO [2019-12-17 12:29:00,540] jepsen worker 70 - jepsen.util 1870 :ok :read [23 44] INFO [2019-12-17 12:29:00,540] jepsen worker 70 - jepsen.util 1870 :invoke :read [23 nil] INFO [2019-12-17 12:29:00,553] jepsen worker 47 - jepsen.util 2117 :ok :read [15 3] INFO [2019-12-17 12:29:00,553] jepsen worker 47 - jepsen.util 2117 :invoke :cas [15 [10 0]] INFO [2019-12-17 12:29:00,570] jepsen worker 52 - jepsen.util 1582 :ok :read [17 2] INFO [2019-12-17 12:29:00,570] jepsen worker 52 - jepsen.util 1582 :invoke :cas [17 [19 43]] INFO [2019-12-17 12:29:00,574] jepsen worker 65 - jepsen.util 1415 :ok :read [21 44] INFO [2019-12-17 12:29:00,574] jepsen worker 65 - jepsen.util 1415 :invoke :cas [21 [19 11]] INFO [2019-12-17 12:29:00,577] jepsen worker 54 - jepsen.util 1674 :ok :write [18 11] INFO [2019-12-17 12:29:00,577] jepsen worker 54 - jepsen.util 1674 :invoke :read [18 nil] INFO [2019-12-17 12:29:00,599] jepsen worker 28 - jepsen.util 1018 :fail :cas [9 [33 1]] :ValueNotSwapFrom INFO [2019-12-17 12:29:00,599] jepsen worker 28 - jepsen.util 1018 :invoke :read [9 nil] INFO [2019-12-17 12:29:00,624] jepsen worker 78 - jepsen.util 1608 :ok :write [26 21] INFO [2019-12-17 12:29:00,624] jepsen worker 78 - jepsen.util 1608 :invoke :cas [26 [34 40]] INFO [2019-12-17 12:29:00,630] jepsen worker 47 - jepsen.util 2117 :fail :cas [15 [10 0]] :ValueNotSwapFrom INFO [2019-12-17 12:29:00,630] jepsen worker 47 - jepsen.util 2117 :invoke :write [15 21] INFO [2019-12-17 12:29:00,639] jepsen worker 65 - jepsen.util 1415 :fail :cas [21 [19 11]] :ValueNotSwapFrom INFO [2019-12-17 12:29:00,639] jepsen worker 65 - jepsen.util 1415 :invoke :write [21 2] INFO [2019-12-17 12:29:00,646] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9392,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:29:00,646] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9392,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:29:00,646] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9396,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:29:00,647] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9396,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:29:00,647] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9397,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:29:00,648] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9397,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:29:00,678] jepsen worker 28 - jepsen.util 1018 :ok :read [9 24] INFO [2019-12-17 12:29:00,678] jepsen worker 28 - jepsen.util 1018 :invoke :write [9 30] INFO [2019-12-17 12:29:00,696] jepsen worker 53 - jepsen.util 1493 :ok :write [17 2] INFO [2019-12-17 12:29:00,696] jepsen worker 53 - jepsen.util 1493 :invoke :cas [17 [6 11]] INFO [2019-12-17 12:29:00,697] jepsen worker 55 - jepsen.util 1945 :fail :cas [18 [2 15]] :ValueNotSwapFrom INFO [2019-12-17 12:29:00,698] jepsen worker 55 - jepsen.util 1945 :invoke :write [18 34] INFO [2019-12-17 12:29:00,710] jepsen worker 13 - jepsen.util 1453 :fail :cas [4 [20 11]] :ValueNotSwapFrom INFO [2019-12-17 12:29:00,710] jepsen worker 13 - jepsen.util 1453 :invoke :write [4 49] INFO [2019-12-17 12:29:00,733] jepsen worker 4 - jepsen.util 1264 :fail :cas [1 [20 35]] :ValueNotSwapFrom INFO [2019-12-17 12:29:00,733] jepsen worker 4 - jepsen.util 1264 :invoke :cas [1 [25 48]] INFO [2019-12-17 12:29:00,739] jepsen worker 5 - jepsen.util 1265 :ok :read [1 47] INFO [2019-12-17 12:29:00,739] jepsen worker 5 - jepsen.util 1265 :invoke :write [1 34] INFO [2019-12-17 12:29:00,755] jepsen worker 80 - jepsen.util 1700 :fail :cas [26 [42 20]] :ValueNotSwapFrom INFO [2019-12-17 12:29:00,756] jepsen worker 80 - jepsen.util 1700 :invoke :cas [26 [7 43]] INFO [2019-12-17 12:29:00,899] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9408,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:29:00,900] jepsen worker 37 - jepsen.util 1297 :ok :write [12 33] INFO [2019-12-17 12:29:00,900] jepsen worker 37 - jepsen.util 1297 :invoke :write [12 9] INFO [2019-12-17 12:29:00,907] jepsen worker 36 - jepsen.util 1566 :ok :read [12 6] INFO [2019-12-17 12:29:00,907] jepsen worker 36 - jepsen.util 1566 :invoke :read [12 nil] INFO [2019-12-17 12:29:00,914] jepsen worker 56 - jepsen.util 1586 :fail :cas [18 [0 0]] :ValueNotSwapFrom INFO [2019-12-17 12:29:00,914] jepsen worker 71 - jepsen.util 1241 :fail :cas [23 [31 4]] :ValueNotSwapFrom INFO [2019-12-17 12:29:00,914] jepsen worker 14 - jepsen.util 1634 :ok :read [4 10] INFO [2019-12-17 12:29:00,914] jepsen worker 56 - jepsen.util 1586 :invoke :read [18 nil] INFO [2019-12-17 12:29:00,914] jepsen worker 71 - jepsen.util 1241 :invoke :cas [23 [21 7]] INFO [2019-12-17 12:29:00,914] jepsen worker 14 - jepsen.util 1634 :invoke :cas [4 [14 9]] INFO [2019-12-17 12:29:00,917] jepsen worker 52 - jepsen.util 1582 :fail :cas [17 [19 43]] :ValueNotSwapFrom INFO [2019-12-17 12:29:00,917] jepsen worker 52 - jepsen.util 1582 :invoke :cas [17 [16 16]] INFO [2019-12-17 12:29:00,929] jepsen worker 28 - jepsen.util 1018 :ok :write [9 30] INFO [2019-12-17 12:29:00,929] jepsen worker 28 - jepsen.util 1018 :invoke :cas [9 [38 30]] INFO [2019-12-17 12:29:00,946] jepsen worker 28 - jepsen.util 1018 :fail :cas [9 [38 30]] :ValueNotSwapFrom INFO [2019-12-17 12:29:00,946] jepsen worker 28 - jepsen.util 1018 :invoke :write [9 45] INFO [2019-12-17 12:29:00,953] jepsen worker 84 - jepsen.util 2064 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:29:00,953] jepsen worker 15 - jepsen.util 1905 :info :cas [5 [16 47]] :RequestCanceledException INFO [2019-12-17 12:29:00,953] jepsen worker 84 - jepsen.util 2064 :invoke :cas [28 [34 37]] INFO [2019-12-17 12:29:00,954] jepsen worker 15 - jepsen.util 1995 :invoke :cas [5 [2 32]] INFO [2019-12-17 12:29:00,965] jepsen worker 89 - jepsen.util 1529 :ok :read [29 14] INFO [2019-12-17 12:29:00,965] jepsen worker 89 - jepsen.util 1529 :invoke :write [29 19] INFO [2019-12-17 12:29:00,989] jepsen worker 38 - jepsen.util 1658 :fail :cas [12 [21 43]] :ValueNotSwapFrom INFO [2019-12-17 12:29:00,989] jepsen worker 38 - jepsen.util 1658 :invoke :write [12 39] INFO [2019-12-17 12:29:00,989] jepsen worker 62 - jepsen.util 2222 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:29:00,989] jepsen worker 43 - jepsen.util 2203 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:29:00,989] jepsen worker 34 - jepsen.util 2284 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:29:00,989] jepsen worker 61 - jepsen.util 2311 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:29:00,989] jepsen worker 62 - jepsen.util 2222 :invoke :read [20 nil] INFO [2019-12-17 12:29:00,989] jepsen worker 43 - jepsen.util 2203 :invoke :write [14 37] INFO [2019-12-17 12:29:00,989] jepsen worker 34 - jepsen.util 2284 :invoke :write [11 5] INFO [2019-12-17 12:29:00,989] jepsen worker 61 - jepsen.util 2311 :invoke :cas [20 [33 5]] INFO [2019-12-17 12:29:00,995] jepsen worker 28 - jepsen.util 1018 :ok :write [9 45] INFO [2019-12-17 12:29:00,995] jepsen worker 28 - jepsen.util 1018 :invoke :cas [9 [9 48]] INFO [2019-12-17 12:29:01,005] jepsen worker 26 - jepsen.util 2366 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:29:01,005] jepsen worker 16 - jepsen.util 1906 :info :cas [5 [33 39]] :RequestCanceledException INFO [2019-12-17 12:29:01,005] jepsen worker 23 - jepsen.util 1913 :info :write [7 49] :RequestCanceledException INFO [2019-12-17 12:29:01,006] jepsen worker 26 - jepsen.util 2366 :invoke :cas [8 [29 36]] INFO [2019-12-17 12:29:01,006] jepsen worker 16 - jepsen.util 1996 :invoke :write [5 12] INFO [2019-12-17 12:29:01,006] jepsen worker 23 - jepsen.util 2003 :invoke :read [7 nil] INFO [2019-12-17 12:29:01,008] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9408,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:29:01,008] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9427,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:29:01,008] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9411,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:29:01,008] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9427,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:29:01,008] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9411,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:29:01,008] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9431,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:29:01,008] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9431,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:29:01,019] jepsen worker 28 - jepsen.util 1018 :fail :cas [9 [9 48]] :ValueNotSwapFrom INFO [2019-12-17 12:29:01,019] jepsen worker 28 - jepsen.util 1018 :invoke :write [9 29] INFO [2019-12-17 12:29:01,023] jepsen worker 22 - jepsen.util 2272 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:29:01,023] jepsen worker 66 - jepsen.util 1866 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:29:01,023] jepsen worker 33 - jepsen.util 2463 :info :write [11 14] :RequestCanceledException INFO [2019-12-17 12:29:01,023] jepsen worker 58 - jepsen.util 1858 :info :cas [19 [16 10]] :RequestCanceledException INFO [2019-12-17 12:29:01,023] jepsen worker 86 - jepsen.util 2246 :info :write [28 29] :RequestCanceledException INFO [2019-12-17 12:29:01,023] jepsen worker 57 - jepsen.util 1947 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:29:01,023] jepsen worker 66 - jepsen.util 1866 :invoke :write [22 29] INFO [2019-12-17 12:29:01,023] jepsen worker 22 - jepsen.util 2272 :invoke :write [7 9] INFO [2019-12-17 12:29:01,023] jepsen worker 57 - jepsen.util 1947 :invoke :write [19 29] INFO [2019-12-17 12:29:01,024] jepsen worker 33 - jepsen.util 2553 :invoke :write [11 22] INFO [2019-12-17 12:29:01,024] jepsen worker 86 - jepsen.util 2336 :invoke :write [28 42] INFO [2019-12-17 12:29:01,024] jepsen worker 58 - jepsen.util 1948 :invoke :cas [19 [28 18]] INFO [2019-12-17 12:29:01,030] jepsen worker 35 - jepsen.util 1835 :info :write [11 0] :RequestCanceledException INFO [2019-12-17 12:29:01,030] jepsen worker 82 - jepsen.util 2602 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:29:01,030] jepsen worker 82 - jepsen.util 2602 :invoke :cas [27 [19 48]] INFO [2019-12-17 12:29:01,031] jepsen worker 70 - jepsen.util 1870 :ok :read [23 44] INFO [2019-12-17 12:29:01,031] jepsen worker 70 - jepsen.util 1870 :invoke :read [23 nil] INFO [2019-12-17 12:29:01,032] jepsen worker 35 - jepsen.util 1925 :invoke :write [11 18] INFO [2019-12-17 12:29:01,045] jepsen worker 79 - jepsen.util 1699 :ok :read [26 21] INFO [2019-12-17 12:29:01,045] jepsen worker 79 - jepsen.util 1699 :invoke :read [26 nil] INFO [2019-12-17 12:29:01,053] jepsen worker 76 - jepsen.util 2146 :info :cas [25 [4 33]] :RequestCanceledException INFO [2019-12-17 12:29:01,053] jepsen worker 8 - jepsen.util 2078 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:29:01,053] jepsen worker 24 - jepsen.util 2004 :info :cas [8 [44 29]] :RequestCanceledException INFO [2019-12-17 12:29:01,053] jepsen worker 25 - jepsen.util 2365 :info :write [8 33] :RequestCanceledException INFO [2019-12-17 12:29:01,053] jepsen worker 0 - jepsen.util 2340 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:29:01,053] jepsen worker 50 - jepsen.util 2210 :info :write [16 10] :RequestCanceledException INFO [2019-12-17 12:29:01,053] jepsen worker 40 - jepsen.util 2470 :info :cas [13 [16 45]] :RequestCanceledException INFO [2019-12-17 12:29:01,053] jepsen worker 8 - jepsen.util 2078 :invoke :read [2 nil] INFO [2019-12-17 12:29:01,053] jepsen worker 0 - jepsen.util 2340 :invoke :read [0 nil] INFO [2019-12-17 12:29:01,054] jepsen worker 76 - jepsen.util 2236 :invoke :write [25 23] INFO [2019-12-17 12:29:01,054] jepsen worker 24 - jepsen.util 2094 :invoke :read [8 nil] INFO [2019-12-17 12:29:01,054] jepsen worker 25 - jepsen.util 2455 :invoke :write [8 48] INFO [2019-12-17 12:29:01,054] jepsen worker 50 - jepsen.util 2300 :invoke :cas [16 [18 43]] INFO [2019-12-17 12:29:01,055] jepsen worker 40 - jepsen.util 2560 :invoke :write [13 8] INFO [2019-12-17 12:29:01,089] jepsen worker 85 - jepsen.util 2425 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:29:01,089] jepsen worker 19 - jepsen.util 1819 :info :write [6 27] :RequestCanceledException INFO [2019-12-17 12:29:01,089] jepsen worker 85 - jepsen.util 2425 :invoke :cas [28 [19 32]] INFO [2019-12-17 12:29:01,089] jepsen worker 19 - jepsen.util 1909 :invoke :cas [6 [17 29]] INFO [2019-12-17 12:29:01,103] jepsen worker 28 - jepsen.util 1018 :ok :write [9 29] INFO [2019-12-17 12:29:01,103] jepsen worker 28 - jepsen.util 1018 :invoke :cas [9 [24 5]] INFO [2019-12-17 12:29:01,104] jepsen worker 39 - jepsen.util 2379 :info :cas [13 [27 41]] :RequestCanceledException INFO [2019-12-17 12:29:01,104] jepsen worker 59 - jepsen.util 1769 :info :write [19 21] :RequestCanceledException INFO [2019-12-17 12:29:01,104] jepsen worker 21 - jepsen.util 2091 :info :write [7 19] :RequestCanceledException INFO [2019-12-17 12:29:01,105] jepsen worker 39 - jepsen.util 2469 :invoke :read [13 nil] INFO [2019-12-17 12:29:01,105] jepsen worker 59 - jepsen.util 1859 :invoke :cas [19 [22 39]] INFO [2019-12-17 12:29:01,105] jepsen worker 21 - jepsen.util 2181 :invoke :cas [7 [20 6]] INFO [2019-12-17 12:29:01,121] jepsen worker 17 - jepsen.util 2087 :info :write [5 4] :RequestCanceledException INFO [2019-12-17 12:29:01,121] jepsen worker 83 - jepsen.util 2333 :info :write [27 4] :RequestCanceledException INFO [2019-12-17 12:29:01,121] jepsen worker 27 - jepsen.util 1377 :fail :read [9 nil] :RequestTimeoutException INFO [2019-12-17 12:29:01,122] jepsen worker 77 - jepsen.util 2057 :info :write [25 46] :RequestCanceledException INFO [2019-12-17 12:29:01,122] jepsen worker 41 - jepsen.util 2201 :info :write [13 29] :RequestCanceledException INFO [2019-12-17 12:29:01,122] jepsen worker 27 - jepsen.util 1377 :invoke :cas [9 [2 48]] INFO [2019-12-17 12:29:01,123] jepsen worker 81 - jepsen.util 1791 :info :write [27 25] :RequestCanceledException INFO [2019-12-17 12:29:01,123] jepsen worker 42 - jepsen.util 2472 :info :write [14 43] :RequestCanceledException INFO [2019-12-17 12:29:01,123] jepsen worker 3 - jepsen.util 1443 :info :write [1 9] :RequestCanceledException INFO [2019-12-17 12:29:01,123] jepsen worker 83 - jepsen.util 2423 :invoke :write [27 11] INFO [2019-12-17 12:29:01,123] jepsen worker 77 - jepsen.util 2147 :invoke :cas [25 [48 13]] INFO [2019-12-17 12:29:01,123] jepsen worker 41 - jepsen.util 2291 :invoke :write [13 46] INFO [2019-12-17 12:29:01,123] jepsen worker 17 - jepsen.util 2177 :invoke :write [5 14] INFO [2019-12-17 12:29:01,124] jepsen worker 42 - jepsen.util 2562 :invoke :write [14 35] INFO [2019-12-17 12:29:01,124] jepsen worker 3 - jepsen.util 1533 :invoke :read [1 nil] INFO [2019-12-17 12:29:01,125] jepsen worker 81 - jepsen.util 1881 :invoke :cas [27 [34 1]] INFO [2019-12-17 12:29:01,130] jepsen worker 19 - jepsen.util 1909 :fail :cas [6 [17 29]] :ValueNotSwapFrom INFO [2019-12-17 12:29:01,130] jepsen worker 19 - jepsen.util 1909 :invoke :read [6 nil] INFO [2019-12-17 12:29:01,136] jepsen worker 68 - jepsen.util 1868 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:29:01,136] jepsen worker 1 - jepsen.util 2161 :info :write [0 12] :RequestCanceledException INFO [2019-12-17 12:29:01,136] jepsen worker 68 - jepsen.util 1868 :invoke :cas [22 [7 11]] INFO [2019-12-17 12:29:01,137] jepsen worker 1 - jepsen.util 2251 :invoke :write [0 11] INFO [2019-12-17 12:29:01,140] jepsen worker 28 - jepsen.util 1018 :fail :cas [9 [24 5]] :ValueNotSwapFrom INFO [2019-12-17 12:29:01,140] jepsen worker 28 - jepsen.util 1018 :invoke :read [9 nil] INFO [2019-12-17 12:29:01,148] jepsen worker 27 - jepsen.util 1377 :fail :cas [9 [2 48]] :ValueNotSwapFrom INFO [2019-12-17 12:29:01,148] jepsen worker 27 - jepsen.util 1377 :invoke :read [9 nil] INFO [2019-12-17 12:29:01,154] jepsen worker 19 - jepsen.util 1909 :ok :read [6 12] INFO [2019-12-17 12:29:01,154] jepsen worker 19 - jepsen.util 1909 :invoke :write [6 48] INFO [2019-12-17 12:29:01,163] jepsen worker 28 - jepsen.util 1018 :ok :read [9 29] INFO [2019-12-17 12:29:01,164] jepsen worker 28 - jepsen.util 1018 :invoke :cas [9 [29 42]] INFO [2019-12-17 12:29:01,175] jepsen worker 27 - jepsen.util 1377 :ok :read [9 29] INFO [2019-12-17 12:29:01,176] jepsen worker 27 - jepsen.util 1377 :invoke :write [9 7] INFO [2019-12-17 12:29:01,179] jepsen worker 12 - jepsen.util 1722 :fail :cas [4 [45 20]] :ValueNotSwapFrom INFO [2019-12-17 12:29:01,179] jepsen worker 12 - jepsen.util 1722 :invoke :write [4 26] INFO [2019-12-17 12:29:01,207] jepsen worker 29 - jepsen.util 1289 :info :cas [9 [5 6]] :RequestCanceledException INFO [2019-12-17 12:29:01,207] jepsen worker 29 - jepsen.util 1379 :invoke :cas [9 [16 34]] INFO [2019-12-17 12:29:01,216] jepsen worker 27 - jepsen.util 1377 :ok :write [9 7] INFO [2019-12-17 12:29:01,217] jepsen worker 27 - jepsen.util 1377 :invoke :read [9 nil] INFO [2019-12-17 12:29:01,223] jepsen worker 64 - jepsen.util 1324 :info :cas [21 [41 38]] :RequestCanceledException INFO [2019-12-17 12:29:01,224] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9489,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:29:01,224] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9482,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:29:01,224] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9489,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:29:01,224] jepsen worker 64 - jepsen.util 1414 :invoke :read [21 nil] INFO [2019-12-17 12:29:01,224] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9482,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:29:01,224] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9492,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:29:01,224] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9483,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:29:01,224] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9492,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:29:01,224] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9483,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:29:01,228] jepsen worker 29 - jepsen.util 1379 :fail :cas [9 [16 34]] :ValueNotSwapFrom INFO [2019-12-17 12:29:01,228] jepsen worker 29 - jepsen.util 1379 :invoke :read [9 nil] INFO [2019-12-17 12:29:01,241] jepsen worker 27 - jepsen.util 1377 :ok :read [9 7] INFO [2019-12-17 12:29:01,241] jepsen worker 27 - jepsen.util 1377 :invoke :write [9 10] INFO [2019-12-17 12:29:01,250] jepsen worker 29 - jepsen.util 1379 :ok :read [9 7] INFO [2019-12-17 12:29:01,251] jepsen worker 29 - jepsen.util 1379 :invoke :cas [9 [2 46]] INFO [2019-12-17 12:29:01,258] jepsen worker 87 - jepsen.util 1437 :ok :write [29 21] INFO [2019-12-17 12:29:01,258] jepsen worker 87 - jepsen.util 1437 :invoke :read [29 nil] INFO [2019-12-17 12:29:01,271] jepsen worker 31 - jepsen.util 1741 :ok :write [10 38] INFO [2019-12-17 12:29:01,271] jepsen worker 31 - jepsen.util 1741 :invoke :cas [10 [38 5]] INFO [2019-12-17 12:29:01,283] jepsen worker 80 - jepsen.util 1700 :fail :cas [26 [7 43]] :ValueNotSwapFrom INFO [2019-12-17 12:29:01,283] jepsen worker 80 - jepsen.util 1700 :invoke :write [26 34] INFO [2019-12-17 12:29:01,283] jepsen worker 53 - jepsen.util 1493 :fail :cas [17 [6 11]] :ValueNotSwapFrom INFO [2019-12-17 12:29:01,284] jepsen worker 53 - jepsen.util 1493 :invoke :cas [17 [15 29]] INFO [2019-12-17 12:29:01,288] jepsen worker 18 - jepsen.util 1818 :info :write [6 3] :RequestCanceledException INFO [2019-12-17 12:29:01,288] jepsen worker 9 - jepsen.util 1719 :info :cas [3 [47 38]] :RequestCanceledException INFO [2019-12-17 12:29:01,290] jepsen worker 9 - jepsen.util 1809 :invoke :read [3 nil] INFO [2019-12-17 12:29:01,290] jepsen worker 18 - jepsen.util 1908 :invoke :cas [6 [10 25]] INFO [2019-12-17 12:29:01,306] jepsen worker 30 - jepsen.util 1920 :info :write [10 29] :RequestCanceledException INFO [2019-12-17 12:29:01,306] jepsen worker 46 - jepsen.util 1756 :fail :read [15 nil] :RequestTimeoutException INFO [2019-12-17 12:29:01,306] jepsen worker 32 - jepsen.util 1922 :info :write [10 40] :RequestCanceledException INFO [2019-12-17 12:29:01,306] jepsen worker 46 - jepsen.util 1756 :invoke :write [15 12] INFO [2019-12-17 12:29:01,307] jepsen worker 30 - jepsen.util 2010 :invoke :read [10 nil] INFO [2019-12-17 12:29:01,307] jepsen worker 32 - jepsen.util 2012 :invoke :read [10 nil] INFO [2019-12-17 12:29:01,320] jepsen worker 56 - jepsen.util 1586 :ok :read [18 11] INFO [2019-12-17 12:29:01,320] jepsen worker 56 - jepsen.util 1586 :invoke :read [18 nil] INFO [2019-12-17 12:29:01,322] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:29:01,322] jepsen nemesis - couchbase.util Rebalance check count: 66 INFO [2019-12-17 12:29:01,322] jepsen nemesis - couchbase.util Rebalance stuck count: 47 INFO [2019-12-17 12:29:01,337] jepsen worker 48 - jepsen.util 1668 :info :write [16 18] :RequestCanceledException INFO [2019-12-17 12:29:01,338] jepsen worker 48 - jepsen.util 1758 :invoke :write [16 35] INFO [2019-12-17 12:29:01,345] jepsen worker 27 - jepsen.util 1377 :ok :write [9 10] INFO [2019-12-17 12:29:01,345] jepsen worker 27 - jepsen.util 1377 :invoke :read [9 nil] INFO [2019-12-17 12:29:01,348] jepsen worker 71 - jepsen.util 1241 :fail :cas [23 [21 7]] :ValueNotSwapFrom INFO [2019-12-17 12:29:01,348] jepsen worker 71 - jepsen.util 1241 :invoke :write [23 38] INFO [2019-12-17 12:29:01,352] jepsen worker 67 - jepsen.util 2587 :info :cas [22 [26 2]] :RequestCanceledException INFO [2019-12-17 12:29:01,353] jepsen worker 67 - jepsen.util 2677 :invoke :read [22 nil] INFO [2019-12-17 12:29:01,365] jepsen worker 3 - jepsen.util 1533 :ok :read [1 9] INFO [2019-12-17 12:29:01,365] jepsen worker 3 - jepsen.util 1533 :invoke :read [1 nil] INFO [2019-12-17 12:29:01,376] jepsen worker 27 - jepsen.util 1377 :ok :read [9 10] INFO [2019-12-17 12:29:01,376] jepsen worker 27 - jepsen.util 1377 :invoke :read [9 nil] INFO [2019-12-17 12:29:01,389] jepsen worker 74 - jepsen.util 1514 :info :write [24 21] :RequestCanceledException INFO [2019-12-17 12:29:01,391] jepsen worker 74 - jepsen.util 1604 :invoke :cas [24 [18 32]] INFO [2019-12-17 12:29:01,410] jepsen worker 27 - jepsen.util 1377 :ok :read [9 10] INFO [2019-12-17 12:29:01,410] jepsen worker 27 - jepsen.util 1377 :invoke :read [9 nil] INFO [2019-12-17 12:29:01,411] jepsen worker 79 - jepsen.util 1699 :ok :read [26 21] INFO [2019-12-17 12:29:01,412] jepsen worker 79 - jepsen.util 1699 :invoke :write [26 3] INFO [2019-12-17 12:29:01,421] jepsen worker 45 - jepsen.util 1935 :fail :read [15 nil] :RequestTimeoutException INFO [2019-12-17 12:29:01,422] jepsen worker 45 - jepsen.util 1935 :invoke :write [15 3] INFO [2019-12-17 12:29:01,434] jepsen worker 80 - jepsen.util 1700 :ok :write [26 34] INFO [2019-12-17 12:29:01,435] jepsen worker 80 - jepsen.util 1700 :invoke :write [26 21] INFO [2019-12-17 12:29:01,437] jepsen worker 11 - jepsen.util 1901 :info :write [3 12] :RequestCanceledException INFO [2019-12-17 12:29:01,438] jepsen worker 27 - jepsen.util 1377 :ok :read [9 10] INFO [2019-12-17 12:29:01,438] jepsen worker 11 - jepsen.util 1991 :invoke :write [3 30] INFO [2019-12-17 12:29:01,438] jepsen worker 27 - jepsen.util 1377 :invoke :read [9 nil] INFO [2019-12-17 12:29:01,443] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9522,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:29:01,443] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9522,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:29:01,452] jepsen worker 55 - jepsen.util 1945 :ok :write [18 34] INFO [2019-12-17 12:29:01,452] jepsen worker 13 - jepsen.util 1453 :ok :write [4 49] INFO [2019-12-17 12:29:01,453] jepsen worker 55 - jepsen.util 1945 :invoke :read [18 nil] INFO [2019-12-17 12:29:01,453] jepsen worker 13 - jepsen.util 1453 :invoke :cas [4 [5 46]] INFO [2019-12-17 12:29:01,461] jepsen worker 27 - jepsen.util 1377 :ok :read [9 10] INFO [2019-12-17 12:29:01,462] jepsen worker 27 - jepsen.util 1377 :invoke :read [9 nil] INFO [2019-12-17 12:29:01,480] jepsen worker 53 - jepsen.util 1493 :fail :cas [17 [15 29]] :ValueNotSwapFrom INFO [2019-12-17 12:29:01,480] jepsen worker 53 - jepsen.util 1493 :invoke :write [17 28] INFO [2019-12-17 12:29:01,481] jepsen worker 27 - jepsen.util 1377 :ok :read [9 10] INFO [2019-12-17 12:29:01,481] jepsen worker 27 - jepsen.util 1377 :invoke :read [9 nil] INFO [2019-12-17 12:29:01,494] jepsen worker 27 - jepsen.util 1377 :ok :read [9 10] INFO [2019-12-17 12:29:01,494] jepsen worker 56 - jepsen.util 1586 :ok :read [18 34] INFO [2019-12-17 12:29:01,494] jepsen worker 27 - jepsen.util 1377 :invoke :cas [9 [38 8]] INFO [2019-12-17 12:29:01,494] jepsen worker 56 - jepsen.util 1586 :invoke :cas [18 [17 34]] INFO [2019-12-17 12:29:01,496] jepsen worker 73 - jepsen.util 1603 :ok :write [24 38] INFO [2019-12-17 12:29:01,496] jepsen worker 10 - jepsen.util 1810 :ok :write [3 35] INFO [2019-12-17 12:29:01,496] jepsen worker 73 - jepsen.util 1603 :invoke :write [24 26] INFO [2019-12-17 12:29:01,496] jepsen worker 10 - jepsen.util 1810 :invoke :write [3 24] INFO [2019-12-17 12:29:01,496] jepsen worker 20 - jepsen.util 1550 :ok :write [6 20] INFO [2019-12-17 12:29:01,496] jepsen worker 20 - jepsen.util 1550 :invoke :write [6 18] INFO [2019-12-17 12:29:01,513] jepsen worker 4 - jepsen.util 1264 :fail :cas [1 [25 48]] :ValueNotSwapFrom INFO [2019-12-17 12:29:01,513] jepsen worker 4 - jepsen.util 1264 :invoke :write [1 47] INFO [2019-12-17 12:29:01,518] jepsen worker 3 - jepsen.util 1533 :ok :read [1 9] INFO [2019-12-17 12:29:01,518] jepsen worker 3 - jepsen.util 1533 :invoke :write [1 46] INFO [2019-12-17 12:29:01,536] jepsen worker 72 - jepsen.util 1692 :info :write [24 37] :RequestCanceledException INFO [2019-12-17 12:29:01,536] jepsen worker 63 - jepsen.util 1503 :fail :read [21 nil] :RequestTimeoutException INFO [2019-12-17 12:29:01,536] jepsen worker 63 - jepsen.util 1503 :invoke :cas [21 [49 19]] INFO [2019-12-17 12:29:01,537] jepsen worker 72 - jepsen.util 1782 :invoke :cas [24 [18 15]] INFO [2019-12-17 12:29:01,555] jepsen worker 52 - jepsen.util 1582 :fail :cas [17 [16 16]] :ValueNotSwapFrom INFO [2019-12-17 12:29:01,556] jepsen worker 52 - jepsen.util 1582 :invoke :read [17 nil] INFO [2019-12-17 12:29:01,574] jepsen worker 55 - jepsen.util 1945 :ok :read [18 34] INFO [2019-12-17 12:29:01,574] jepsen worker 55 - jepsen.util 1945 :invoke :cas [18 [0 11]] INFO [2019-12-17 12:29:01,593] jepsen worker 64 - jepsen.util 1414 :ok :read [21 44] INFO [2019-12-17 12:29:01,594] jepsen worker 64 - jepsen.util 1414 :invoke :write [21 25] INFO [2019-12-17 12:29:01,605] jepsen worker 18 - jepsen.util 1908 :fail :cas [6 [10 25]] :ValueNotSwapFrom INFO [2019-12-17 12:29:01,605] jepsen worker 18 - jepsen.util 1908 :invoke :read [6 nil] INFO [2019-12-17 12:29:01,607] jepsen worker 49 - jepsen.util 1579 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:29:01,607] jepsen worker 49 - jepsen.util 1579 :invoke :read [16 nil] INFO [2019-12-17 12:29:01,619] jepsen worker 27 - jepsen.util 1377 :fail :cas [9 [38 8]] :ValueNotSwapFrom INFO [2019-12-17 12:29:01,619] jepsen worker 27 - jepsen.util 1377 :invoke :cas [9 [21 12]] INFO [2019-12-17 12:29:01,622] jepsen worker 2 - jepsen.util 1892 :info :write [0 33] :RequestCanceledException INFO [2019-12-17 12:29:01,624] jepsen worker 32 - jepsen.util 2012 :ok :read [10 38] INFO [2019-12-17 12:29:01,624] jepsen worker 2 - jepsen.util 1982 :invoke :cas [0 [29 8]] INFO [2019-12-17 12:29:01,624] jepsen worker 32 - jepsen.util 2012 :invoke :read [10 nil] INFO [2019-12-17 12:29:01,636] jepsen worker 44 - jepsen.util 1754 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:29:01,636] jepsen worker 44 - jepsen.util 1754 :invoke :cas [14 [2 48]] INFO [2019-12-17 12:29:01,639] jepsen worker 27 - jepsen.util 1377 :fail :cas [9 [21 12]] :ValueNotSwapFrom INFO [2019-12-17 12:29:01,639] jepsen worker 27 - jepsen.util 1377 :invoke :cas [9 [28 3]] INFO [2019-12-17 12:29:01,660] jepsen worker 27 - jepsen.util 1377 :fail :cas [9 [28 3]] :ValueNotSwapFrom INFO [2019-12-17 12:29:01,660] jepsen worker 27 - jepsen.util 1377 :invoke :write [9 9] INFO [2019-12-17 12:29:01,662] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9553,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:29:01,662] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9553,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:29:01,662] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9556,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:29:01,662] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9556,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:29:01,699] jepsen worker 56 - jepsen.util 1586 :fail :cas [18 [17 34]] :ValueNotSwapFrom INFO [2019-12-17 12:29:01,699] jepsen worker 56 - jepsen.util 1586 :invoke :read [18 nil] INFO [2019-12-17 12:29:01,722] jepsen worker 27 - jepsen.util 1377 :ok :write [9 9] INFO [2019-12-17 12:29:01,723] jepsen worker 27 - jepsen.util 1377 :invoke :write [9 14] INFO [2019-12-17 12:29:01,746] jepsen worker 52 - jepsen.util 1582 :ok :read [17 2] INFO [2019-12-17 12:29:01,746] jepsen worker 52 - jepsen.util 1582 :invoke :read [17 nil] INFO [2019-12-17 12:29:01,777] jepsen worker 14 - jepsen.util 1634 :fail :cas [4 [14 9]] :ValueNotSwapFrom INFO [2019-12-17 12:29:01,778] jepsen worker 14 - jepsen.util 1634 :invoke :write [4 15] INFO [2019-12-17 12:29:01,786] jepsen worker 55 - jepsen.util 1945 :fail :cas [18 [0 11]] :ValueNotSwapFrom INFO [2019-12-17 12:29:01,786] jepsen worker 55 - jepsen.util 1945 :invoke :write [18 37] INFO [2019-12-17 12:29:01,796] jepsen worker 20 - jepsen.util 1550 :ok :write [6 18] INFO [2019-12-17 12:29:01,797] jepsen worker 20 - jepsen.util 1550 :invoke :cas [6 [21 30]] INFO [2019-12-17 12:29:01,811] jepsen worker 31 - jepsen.util 1741 :ok :cas [10 [38 5]] INFO [2019-12-17 12:29:01,812] jepsen worker 31 - jepsen.util 1741 :invoke :cas [10 [10 18]] INFO [2019-12-17 12:29:01,812] jepsen worker 47 - jepsen.util 2117 :ok :write [15 21] INFO [2019-12-17 12:29:01,813] jepsen worker 47 - jepsen.util 2117 :invoke :cas [15 [9 13]] INFO [2019-12-17 12:29:01,815] jepsen worker 56 - jepsen.util 1586 :ok :read [18 34] INFO [2019-12-17 12:29:01,815] jepsen worker 56 - jepsen.util 1586 :invoke :cas [18 [39 36]] INFO [2019-12-17 12:29:01,833] jepsen worker 27 - jepsen.util 1377 :ok :write [9 14] INFO [2019-12-17 12:29:01,833] jepsen worker 27 - jepsen.util 1377 :invoke :read [9 nil] INFO [2019-12-17 12:29:01,881] jepsen worker 63 - jepsen.util 1503 :fail :cas [21 [49 19]] :ValueNotSwapFrom INFO [2019-12-17 12:29:01,881] jepsen worker 63 - jepsen.util 1503 :invoke :read [21 nil] INFO [2019-12-17 12:29:01,882] jepsen worker 27 - jepsen.util 1377 :ok :read [9 14] INFO [2019-12-17 12:29:01,882] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9571,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:29:01,882] jepsen worker 27 - jepsen.util 1377 :invoke :cas [9 [45 29]] INFO [2019-12-17 12:29:01,882] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9571,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:29:01,882] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9575,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:29:01,882] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9575,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:29:01,889] jepsen worker 73 - jepsen.util 1603 :ok :write [24 26] INFO [2019-12-17 12:29:01,889] jepsen worker 73 - jepsen.util 1603 :invoke :write [24 33] INFO [2019-12-17 12:29:01,919] jepsen worker 72 - jepsen.util 1782 :fail :cas [24 [18 15]] :ValueNotSwapFrom INFO [2019-12-17 12:29:01,919] jepsen worker 72 - jepsen.util 1782 :invoke :cas [24 [49 46]] INFO [2019-12-17 12:29:01,922] jepsen worker 6 - jepsen.util 2166 :info :write [2 5] :RequestCanceledException INFO [2019-12-17 12:29:01,924] jepsen worker 6 - jepsen.util 2256 :invoke :read [2 nil] INFO [2019-12-17 12:29:01,934] jepsen worker 32 - jepsen.util 2012 :ok :read [10 5] INFO [2019-12-17 12:29:01,934] jepsen worker 32 - jepsen.util 2012 :invoke :read [10 nil] INFO [2019-12-17 12:29:01,953] jepsen worker 75 - jepsen.util 2415 :info :write [25 24] :RequestCanceledException INFO [2019-12-17 12:29:01,954] jepsen worker 75 - jepsen.util 2505 :invoke :write [25 21] INFO [2019-12-17 12:29:01,957] jepsen worker 27 - jepsen.util 1377 :fail :cas [9 [45 29]] :ValueNotSwapFrom INFO [2019-12-17 12:29:01,958] jepsen worker 27 - jepsen.util 1377 :invoke :write [9 2] INFO [2019-12-17 12:29:01,984] jepsen worker 65 - jepsen.util 1415 :ok :write [21 2] INFO [2019-12-17 12:29:01,985] jepsen worker 65 - jepsen.util 1415 :invoke :cas [21 [4 35]] INFO [2019-12-17 12:29:01,986] jepsen worker 6 - jepsen.util 2256 :ok :read [2 15] INFO [2019-12-17 12:29:01,986] jepsen worker 6 - jepsen.util 2256 :invoke :cas [2 [45 14]] INFO [2019-12-17 12:29:01,992] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9587,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:29:01,992] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9587,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:29:01,992] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9590,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:29:01,992] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9590,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:29:01,992] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9591,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:29:01,992] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9591,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:29:02,001] jepsen worker 31 - jepsen.util 1741 :fail :cas [10 [10 18]] :ValueNotSwapFrom INFO [2019-12-17 12:29:02,001] jepsen worker 31 - jepsen.util 1741 :invoke :write [10 2] INFO [2019-12-17 12:29:02,003] jepsen worker 6 - jepsen.util 2256 :fail :cas [2 [45 14]] :ValueNotSwapFrom INFO [2019-12-17 12:29:02,003] jepsen worker 6 - jepsen.util 2256 :invoke :write [2 35] INFO [2019-12-17 12:29:02,007] jepsen worker 60 - jepsen.util 2220 :info :write [20 14] :RequestCanceledException INFO [2019-12-17 12:29:02,008] jepsen worker 60 - jepsen.util 2310 :invoke :read [20 nil] INFO [2019-12-17 12:29:02,060] jepsen worker 36 - jepsen.util 1566 :ok :read [12 33] INFO [2019-12-17 12:29:02,061] jepsen worker 36 - jepsen.util 1566 :invoke :write [12 38] INFO [2019-12-17 12:29:02,072] jepsen worker 56 - jepsen.util 1586 :fail :cas [18 [39 36]] :ValueNotSwapFrom INFO [2019-12-17 12:29:02,072] jepsen worker 56 - jepsen.util 1586 :invoke :write [18 45] INFO [2019-12-17 12:29:02,091] jepsen worker 20 - jepsen.util 1550 :fail :cas [6 [21 30]] :ValueNotSwapFrom INFO [2019-12-17 12:29:02,091] jepsen worker 20 - jepsen.util 1550 :invoke :cas [6 [7 14]] INFO [2019-12-17 12:29:02,100] jepsen worker 63 - jepsen.util 1503 :ok :read [21 2] INFO [2019-12-17 12:29:02,101] jepsen worker 63 - jepsen.util 1503 :invoke :read [21 nil] INFO [2019-12-17 12:29:02,122] jepsen worker 7 - jepsen.util 2347 :info :write [2 21] :RequestCanceledException INFO [2019-12-17 12:29:02,123] jepsen worker 7 - jepsen.util 2437 :invoke :cas [2 [15 5]] INFO [2019-12-17 12:29:02,134] jepsen worker 88 - jepsen.util 1528 :info :write [29 18] :RequestCanceledException INFO [2019-12-17 12:29:02,135] jepsen worker 88 - jepsen.util 1618 :invoke :read [29 nil] INFO [2019-12-17 12:29:02,138] jepsen worker 32 - jepsen.util 2012 :ok :read [10 5] INFO [2019-12-17 12:29:02,139] jepsen worker 32 - jepsen.util 2012 :invoke :cas [10 [22 28]] INFO [2019-12-17 12:29:02,151] jepsen worker 9 - jepsen.util 1809 :ok :read [3 35] INFO [2019-12-17 12:29:02,152] jepsen worker 9 - jepsen.util 1809 :invoke :write [3 26] INFO [2019-12-17 12:29:02,155] jepsen worker 52 - jepsen.util 1582 :ok :read [17 2] INFO [2019-12-17 12:29:02,155] jepsen worker 52 - jepsen.util 1582 :invoke :write [17 45] INFO [2019-12-17 12:29:02,169] jepsen worker 70 - jepsen.util 1870 :ok :read [23 44] INFO [2019-12-17 12:29:02,169] jepsen worker 70 - jepsen.util 1870 :invoke :write [23 32] INFO [2019-12-17 12:29:02,180] jepsen worker 72 - jepsen.util 1782 :fail :cas [24 [49 46]] :ValueNotSwapFrom INFO [2019-12-17 12:29:02,180] jepsen worker 72 - jepsen.util 1782 :invoke :cas [24 [21 42]] INFO [2019-12-17 12:29:02,202] jepsen worker 20 - jepsen.util 1550 :fail :cas [6 [7 14]] :ValueNotSwapFrom INFO [2019-12-17 12:29:02,202] jepsen worker 20 - jepsen.util 1550 :invoke :cas [6 [5 45]] INFO [2019-12-17 12:29:02,210] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9607,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:29:02,210] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9615,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:29:02,210] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9607,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:29:02,210] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9615,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:29:02,210] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9608,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:29:02,210] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9618,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:29:02,210] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9608,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:29:02,210] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9618,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:29:02,210] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9619,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:29:02,211] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9619,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:29:02,215] jepsen worker 32 - jepsen.util 2012 :fail :cas [10 [22 28]] :ValueNotSwapFrom INFO [2019-12-17 12:29:02,216] jepsen worker 32 - jepsen.util 2012 :invoke :cas [10 [37 9]] INFO [2019-12-17 12:29:02,238] jepsen worker 20 - jepsen.util 1550 :fail :cas [6 [5 45]] :ValueNotSwapFrom INFO [2019-12-17 12:29:02,238] jepsen worker 20 - jepsen.util 1550 :invoke :write [6 45] INFO [2019-12-17 12:29:02,238] jepsen worker 27 - jepsen.util 1377 :ok :write [9 2] INFO [2019-12-17 12:29:02,238] jepsen worker 27 - jepsen.util 1377 :invoke :read [9 nil] INFO [2019-12-17 12:29:02,246] jepsen worker 64 - jepsen.util 1414 :ok :write [21 25] INFO [2019-12-17 12:29:02,246] jepsen worker 64 - jepsen.util 1414 :invoke :cas [21 [40 46]] INFO [2019-12-17 12:29:02,304] jepsen worker 27 - jepsen.util 1377 :ok :read [9 2] INFO [2019-12-17 12:29:02,304] jepsen worker 27 - jepsen.util 1377 :invoke :cas [9 [30 11]] INFO [2019-12-17 12:29:02,312] jepsen worker 32 - jepsen.util 2012 :fail :cas [10 [37 9]] :ValueNotSwapFrom INFO [2019-12-17 12:29:02,312] jepsen worker 32 - jepsen.util 2012 :invoke :read [10 nil] INFO [2019-12-17 12:29:02,334] jepsen worker 69 - jepsen.util 1509 :info :write [23 40] :RequestCanceledException INFO [2019-12-17 12:29:02,337] jepsen worker 69 - jepsen.util 1599 :invoke :write [23 48] INFO [2019-12-17 12:29:02,350] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:29:02,350] jepsen nemesis - couchbase.util Rebalance check count: 67 INFO [2019-12-17 12:29:02,350] jepsen nemesis - couchbase.util Rebalance stuck count: 48 INFO [2019-12-17 12:29:02,371] jepsen worker 27 - jepsen.util 1377 :fail :cas [9 [30 11]] :ValueNotSwapFrom INFO [2019-12-17 12:29:02,371] jepsen worker 27 - jepsen.util 1377 :invoke :read [9 nil] INFO [2019-12-17 12:29:02,378] jepsen worker 13 - jepsen.util 1453 :fail :cas [4 [5 46]] :ValueNotSwapFrom INFO [2019-12-17 12:29:02,378] jepsen worker 13 - jepsen.util 1453 :invoke :read [4 nil] INFO [2019-12-17 12:29:02,395] jepsen worker 46 - jepsen.util 1756 :ok :write [15 12] INFO [2019-12-17 12:29:02,395] jepsen worker 46 - jepsen.util 1756 :invoke :write [15 21] INFO [2019-12-17 12:29:02,404] jepsen worker 27 - jepsen.util 1377 :ok :read [9 2] INFO [2019-12-17 12:29:02,404] jepsen worker 27 - jepsen.util 1377 :invoke :write [9 39] INFO [2019-12-17 12:29:02,415] jepsen worker 87 - jepsen.util 1437 :ok :read [29 21] INFO [2019-12-17 12:29:02,415] jepsen worker 87 - jepsen.util 1437 :invoke :read [29 nil] INFO [2019-12-17 12:29:02,430] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9633,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:29:02,431] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9633,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:29:02,431] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9636,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:29:02,431] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9636,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:29:02,431] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9637,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:29:02,432] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9637,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:29:02,563] jepsen worker 27 - jepsen.util 1377 :ok :write [9 39] INFO [2019-12-17 12:29:02,563] jepsen worker 27 - jepsen.util 1377 :invoke :cas [9 [19 23]] INFO [2019-12-17 12:29:02,563] jepsen worker 47 - jepsen.util 2117 :fail :cas [15 [9 13]] :ValueNotSwapFrom INFO [2019-12-17 12:29:02,563] jepsen worker 47 - jepsen.util 2117 :invoke :write [15 32] INFO [2019-12-17 12:29:02,563] jepsen worker 65 - jepsen.util 1415 :fail :cas [21 [4 35]] :ValueNotSwapFrom INFO [2019-12-17 12:29:02,564] jepsen worker 65 - jepsen.util 1415 :invoke :cas [21 [15 34]] INFO [2019-12-17 12:29:02,580] jepsen worker 27 - jepsen.util 1377 :fail :cas [9 [19 23]] :ValueNotSwapFrom INFO [2019-12-17 12:29:02,580] jepsen worker 27 - jepsen.util 1377 :invoke :cas [9 [5 19]] INFO [2019-12-17 12:29:02,599] jepsen worker 27 - jepsen.util 1377 :fail :cas [9 [5 19]] :ValueNotSwapFrom INFO [2019-12-17 12:29:02,599] jepsen worker 27 - jepsen.util 1377 :invoke :write [9 26] INFO [2019-12-17 12:29:02,628] jepsen worker 31 - jepsen.util 1741 :ok :write [10 2] INFO [2019-12-17 12:29:02,628] jepsen worker 31 - jepsen.util 1741 :invoke :read [10 nil] INFO [2019-12-17 12:29:02,637] jepsen worker 20 - jepsen.util 1550 :ok :write [6 45] INFO [2019-12-17 12:29:02,637] jepsen worker 20 - jepsen.util 1550 :invoke :read [6 nil] INFO [2019-12-17 12:29:02,654] jepsen worker 54 - jepsen.util 1674 :fail :read [18 nil] :RequestTimeoutException INFO [2019-12-17 12:29:02,654] jepsen worker 54 - jepsen.util 1674 :invoke :cas [18 [37 29]] INFO [2019-12-17 12:29:02,660] jepsen worker 88 - jepsen.util 1618 :ok :read [29 21] INFO [2019-12-17 12:29:02,661] jepsen worker 88 - jepsen.util 1618 :invoke :write [29 40] INFO [2019-12-17 12:29:02,666] jepsen worker 31 - jepsen.util 1741 :ok :read [10 2] INFO [2019-12-17 12:29:02,666] jepsen worker 31 - jepsen.util 1741 :invoke :write [10 0] INFO [2019-12-17 12:29:02,672] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9645,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:29:02,672] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9645,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:29:02,672] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9650,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:29:02,672] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9650,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:29:02,676] jepsen worker 32 - jepsen.util 2012 :ok :read [10 2] INFO [2019-12-17 12:29:02,677] jepsen worker 32 - jepsen.util 2012 :invoke :read [10 nil] INFO [2019-12-17 12:29:02,690] jepsen worker 27 - jepsen.util 1377 :ok :write [9 26] INFO [2019-12-17 12:29:02,690] jepsen worker 27 - jepsen.util 1377 :invoke :write [9 34] INFO [2019-12-17 12:29:02,731] jepsen worker 20 - jepsen.util 1550 :ok :read [6 45] INFO [2019-12-17 12:29:02,731] jepsen worker 20 - jepsen.util 1550 :invoke :write [6 41] INFO [2019-12-17 12:29:02,743] jepsen worker 32 - jepsen.util 2012 :ok :read [10 2] INFO [2019-12-17 12:29:02,743] jepsen worker 32 - jepsen.util 2012 :invoke :cas [10 [40 38]] INFO [2019-12-17 12:29:02,756] jepsen worker 27 - jepsen.util 1377 :ok :write [9 34] INFO [2019-12-17 12:29:02,757] jepsen worker 27 - jepsen.util 1377 :invoke :write [9 43] INFO [2019-12-17 12:29:02,821] jepsen worker 27 - jepsen.util 1377 :ok :write [9 43] INFO [2019-12-17 12:29:02,822] jepsen worker 27 - jepsen.util 1377 :invoke :read [9 nil] INFO [2019-12-17 12:29:02,822] jepsen worker 5 - jepsen.util 1265 :info :write [1 34] :RequestCanceledException INFO [2019-12-17 12:29:02,823] jepsen worker 5 - jepsen.util 1355 :invoke :cas [1 [44 41]] INFO [2019-12-17 12:29:02,843] jepsen worker 27 - jepsen.util 1377 :ok :read [9 43] INFO [2019-12-17 12:29:02,843] jepsen worker 27 - jepsen.util 1377 :invoke :cas [9 [30 48]] INFO [2019-12-17 12:29:02,865] jepsen worker 27 - jepsen.util 1377 :fail :cas [9 [30 48]] :ValueNotSwapFrom INFO [2019-12-17 12:29:02,866] jepsen worker 27 - jepsen.util 1377 :invoke :cas [9 [22 4]] INFO [2019-12-17 12:29:02,880] jepsen worker 27 - jepsen.util 1377 :fail :cas [9 [22 4]] :ValueNotSwapFrom INFO [2019-12-17 12:29:02,880] jepsen worker 27 - jepsen.util 1377 :invoke :read [9 nil] INFO [2019-12-17 12:29:02,892] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9664,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:29:02,892] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9664,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:29:02,892] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9667,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:29:02,892] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9667,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:29:02,893] jepsen worker 27 - jepsen.util 1377 :ok :read [9 43] INFO [2019-12-17 12:29:02,893] jepsen worker 27 - jepsen.util 1377 :invoke :cas [9 [39 25]] INFO [2019-12-17 12:29:02,905] jepsen worker 27 - jepsen.util 1377 :fail :cas [9 [39 25]] :ValueNotSwapFrom INFO [2019-12-17 12:29:02,905] jepsen worker 27 - jepsen.util 1377 :invoke :read [9 nil] INFO [2019-12-17 12:29:02,916] jepsen worker 27 - jepsen.util 1377 :ok :read [9 43] INFO [2019-12-17 12:29:02,916] jepsen worker 27 - jepsen.util 1377 :invoke :write [9 45] INFO [2019-12-17 12:29:02,940] jepsen worker 87 - jepsen.util 1437 :ok :read [29 21] INFO [2019-12-17 12:29:02,940] jepsen worker 87 - jepsen.util 1437 :invoke :cas [29 [19 35]] INFO [2019-12-17 12:29:02,953] jepsen worker 37 - jepsen.util 1297 :info :write [12 9] :RequestCanceledException INFO [2019-12-17 12:29:02,954] jepsen worker 37 - jepsen.util 1387 :invoke :read [12 nil] INFO [2019-12-17 12:29:02,958] jepsen worker 27 - jepsen.util 1377 :ok :write [9 45] INFO [2019-12-17 12:29:02,958] jepsen worker 27 - jepsen.util 1377 :invoke :write [9 28] INFO [2019-12-17 12:29:03,002] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9677,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:29:03,002] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9677,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:29:03,002] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9680,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:29:03,002] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9680,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:29:03,002] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9681,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:29:03,002] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9681,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:29:03,005] jepsen worker 27 - jepsen.util 1377 :ok :write [9 28] INFO [2019-12-17 12:29:03,005] jepsen worker 27 - jepsen.util 1377 :invoke :cas [9 [36 44]] INFO [2019-12-17 12:29:03,016] jepsen worker 13 - jepsen.util 1453 :ok :read [4 49] INFO [2019-12-17 12:29:03,016] jepsen worker 13 - jepsen.util 1453 :invoke :read [4 nil] INFO [2019-12-17 12:29:03,022] jepsen worker 15 - jepsen.util 1995 :info :cas [5 [2 32]] :RequestCanceledException INFO [2019-12-17 12:29:03,022] jepsen worker 89 - jepsen.util 1529 :info :write [29 19] :RequestCanceledException INFO [2019-12-17 12:29:03,022] jepsen worker 38 - jepsen.util 1658 :info :write [12 39] :RequestCanceledException INFO [2019-12-17 12:29:03,023] jepsen worker 27 - jepsen.util 1377 :fail :cas [9 [36 44]] :ValueNotSwapFrom INFO [2019-12-17 12:29:03,023] jepsen worker 27 - jepsen.util 1377 :invoke :read [9 nil] INFO [2019-12-17 12:29:03,023] jepsen worker 38 - jepsen.util 1748 :invoke :write [12 16] INFO [2019-12-17 12:29:03,023] jepsen worker 89 - jepsen.util 1619 :invoke :read [29 nil] INFO [2019-12-17 12:29:03,024] jepsen worker 15 - jepsen.util 2085 :invoke :cas [5 [23 3]] INFO [2019-12-17 12:29:03,034] jepsen worker 27 - jepsen.util 1377 :ok :read [9 28] INFO [2019-12-17 12:29:03,034] jepsen worker 27 - jepsen.util 1377 :invoke :write [9 28] INFO [2019-12-17 12:29:03,053] jepsen worker 58 - jepsen.util 1948 :info :cas [19 [28 18]] :RequestCanceledException INFO [2019-12-17 12:29:03,053] jepsen worker 84 - jepsen.util 2064 :info :cas [28 [34 37]] :RequestCanceledException INFO [2019-12-17 12:29:03,054] jepsen worker 58 - jepsen.util 2038 :invoke :write [19 0] INFO [2019-12-17 12:29:03,054] jepsen worker 84 - jepsen.util 2154 :invoke :read [28 nil] INFO [2019-12-17 12:29:03,074] jepsen worker 27 - jepsen.util 1377 :ok :write [9 28] INFO [2019-12-17 12:29:03,074] jepsen worker 27 - jepsen.util 1377 :invoke :write [9 40] INFO [2019-12-17 12:29:03,089] jepsen worker 61 - jepsen.util 2311 :info :cas [20 [33 5]] :RequestCanceledException INFO [2019-12-17 12:29:03,089] jepsen worker 62 - jepsen.util 2222 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:29:03,089] jepsen worker 43 - jepsen.util 2203 :info :write [14 37] :RequestCanceledException INFO [2019-12-17 12:29:03,089] jepsen worker 34 - jepsen.util 2284 :info :write [11 5] :RequestCanceledException INFO [2019-12-17 12:29:03,089] jepsen worker 16 - jepsen.util 1996 :info :write [5 12] :RequestCanceledException INFO [2019-12-17 12:29:03,089] jepsen worker 76 - jepsen.util 2236 :info :write [25 23] :RequestCanceledException INFO [2019-12-17 12:29:03,089] jepsen worker 62 - jepsen.util 2222 :invoke :cas [20 [49 18]] INFO [2019-12-17 12:29:03,090] jepsen worker 34 - jepsen.util 2374 :invoke :read [11 nil] INFO [2019-12-17 12:29:03,090] jepsen worker 16 - jepsen.util 2086 :invoke :read [5 nil] INFO [2019-12-17 12:29:03,090] jepsen worker 43 - jepsen.util 2293 :invoke :cas [14 [39 22]] INFO [2019-12-17 12:29:03,090] jepsen worker 61 - jepsen.util 2401 :invoke :cas [20 [46 19]] INFO [2019-12-17 12:29:03,090] jepsen worker 76 - jepsen.util 2326 :invoke :read [25 nil] INFO [2019-12-17 12:29:03,098] jepsen worker 54 - jepsen.util 1674 :fail :cas [18 [37 29]] :ValueNotSwapFrom INFO [2019-12-17 12:29:03,098] jepsen worker 54 - jepsen.util 1674 :invoke :write [18 5] INFO [2019-12-17 12:29:03,107] jepsen worker 26 - jepsen.util 2366 :info :cas [8 [29 36]] :RequestCanceledException INFO [2019-12-17 12:29:03,107] jepsen worker 23 - jepsen.util 2003 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:29:03,107] jepsen worker 24 - jepsen.util 2094 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:29:03,107] jepsen worker 24 - jepsen.util 2094 :invoke :read [8 nil] INFO [2019-12-17 12:29:03,107] jepsen worker 23 - jepsen.util 2003 :invoke :write [7 11] INFO [2019-12-17 12:29:03,108] jepsen worker 26 - jepsen.util 2456 :invoke :write [8 9] INFO [2019-12-17 12:29:03,122] jepsen worker 35 - jepsen.util 1925 :info :write [11 18] :RequestCanceledException INFO [2019-12-17 12:29:03,122] jepsen worker 57 - jepsen.util 1947 :info :write [19 29] :RequestCanceledException INFO [2019-12-17 12:29:03,123] jepsen worker 22 - jepsen.util 2272 :info :write [7 9] :RequestCanceledException INFO [2019-12-17 12:29:03,123] jepsen worker 66 - jepsen.util 1866 :info :write [22 29] :RequestCanceledException INFO [2019-12-17 12:29:03,123] jepsen worker 25 - jepsen.util 2455 :info :write [8 48] :RequestCanceledException INFO [2019-12-17 12:29:03,123] jepsen worker 86 - jepsen.util 2336 :info :write [28 42] :RequestCanceledException INFO [2019-12-17 12:29:03,123] jepsen worker 21 - jepsen.util 2181 :info :cas [7 [20 6]] :RequestCanceledException INFO [2019-12-17 12:29:03,124] jepsen worker 35 - jepsen.util 2015 :invoke :read [11 nil] INFO [2019-12-17 12:29:03,124] jepsen worker 57 - jepsen.util 2037 :invoke :write [19 29] INFO [2019-12-17 12:29:03,124] jepsen worker 22 - jepsen.util 2362 :invoke :write [7 21] INFO [2019-12-17 12:29:03,124] jepsen worker 86 - jepsen.util 2426 :invoke :read [28 nil] INFO [2019-12-17 12:29:03,124] jepsen worker 21 - jepsen.util 2271 :invoke :cas [7 [14 11]] INFO [2019-12-17 12:29:03,124] jepsen worker 66 - jepsen.util 1956 :invoke :write [22 10] INFO [2019-12-17 12:29:03,124] jepsen worker 25 - jepsen.util 2545 :invoke :read [8 nil] INFO [2019-12-17 12:29:03,136] jepsen worker 33 - jepsen.util 2553 :info :write [11 22] :RequestCanceledException INFO [2019-12-17 12:29:03,136] jepsen worker 82 - jepsen.util 2602 :info :cas [27 [19 48]] :RequestCanceledException INFO [2019-12-17 12:29:03,136] jepsen worker 50 - jepsen.util 2300 :info :cas [16 [18 43]] :RequestCanceledException INFO [2019-12-17 12:29:03,136] jepsen worker 81 - jepsen.util 1881 :info :cas [27 [34 1]] :RequestCanceledException INFO [2019-12-17 12:29:03,137] jepsen worker 33 - jepsen.util 2643 :invoke :read [11 nil] INFO [2019-12-17 12:29:03,137] jepsen worker 50 - jepsen.util 2390 :invoke :cas [16 [8 11]] INFO [2019-12-17 12:29:03,137] jepsen worker 82 - jepsen.util 2692 :invoke :cas [27 [45 30]] INFO [2019-12-17 12:29:03,137] jepsen worker 81 - jepsen.util 1971 :invoke :write [27 23] INFO [2019-12-17 12:29:03,138] jepsen worker 27 - jepsen.util 1377 :ok :write [9 40] INFO [2019-12-17 12:29:03,138] jepsen worker 27 - jepsen.util 1377 :invoke :cas [9 [5 34]] INFO [2019-12-17 12:29:03,145] jepsen worker 37 - jepsen.util 1387 :ok :read [12 33] INFO [2019-12-17 12:29:03,145] jepsen worker 37 - jepsen.util 1387 :invoke :cas [12 [25 7]] INFO [2019-12-17 12:29:03,153] jepsen worker 0 - jepsen.util 2340 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:29:03,153] jepsen worker 8 - jepsen.util 2078 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:29:03,153] jepsen worker 40 - jepsen.util 2560 :info :write [13 8] :RequestCanceledException INFO [2019-12-17 12:29:03,153] jepsen worker 77 - jepsen.util 2147 :info :cas [25 [48 13]] :RequestCanceledException INFO [2019-12-17 12:29:03,153] jepsen worker 1 - jepsen.util 2251 :info :write [0 11] :RequestCanceledException INFO [2019-12-17 12:29:03,153] jepsen worker 0 - jepsen.util 2340 :invoke :write [0 11] INFO [2019-12-17 12:29:03,153] jepsen worker 8 - jepsen.util 2078 :invoke :cas [2 [28 7]] INFO [2019-12-17 12:29:03,154] jepsen worker 1 - jepsen.util 2341 :invoke :read [0 nil] INFO [2019-12-17 12:29:03,154] jepsen worker 77 - jepsen.util 2237 :invoke :cas [25 [47 23]] INFO [2019-12-17 12:29:03,154] jepsen worker 40 - jepsen.util 2650 :invoke :write [13 5] INFO [2019-12-17 12:29:03,156] jepsen worker 27 - jepsen.util 1377 :fail :cas [9 [5 34]] :ValueNotSwapFrom INFO [2019-12-17 12:29:03,156] jepsen worker 27 - jepsen.util 1377 :invoke :read [9 nil] INFO [2019-12-17 12:29:03,169] jepsen worker 27 - jepsen.util 1377 :ok :read [9 40] INFO [2019-12-17 12:29:03,169] jepsen worker 27 - jepsen.util 1377 :invoke :read [9 nil] INFO [2019-12-17 12:29:03,181] jepsen worker 27 - jepsen.util 1377 :ok :read [9 40] INFO [2019-12-17 12:29:03,181] jepsen worker 27 - jepsen.util 1377 :invoke :cas [9 [4 43]] INFO [2019-12-17 12:29:03,189] jepsen worker 39 - jepsen.util 2469 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:29:03,189] jepsen worker 51 - jepsen.util 1581 :info :cas [17 [2 2]] :RequestCanceledException INFO [2019-12-17 12:29:03,189] jepsen worker 85 - jepsen.util 2425 :info :cas [28 [19 32]] :RequestCanceledException INFO [2019-12-17 12:29:03,189] jepsen worker 17 - jepsen.util 2177 :info :write [5 14] :RequestCanceledException INFO [2019-12-17 12:29:03,189] jepsen worker 39 - jepsen.util 2469 :invoke :read [13 nil] INFO [2019-12-17 12:29:03,190] jepsen worker 51 - jepsen.util 1671 :invoke :read [17 nil] INFO [2019-12-17 12:29:03,190] jepsen worker 85 - jepsen.util 2515 :invoke :read [28 nil] INFO [2019-12-17 12:29:03,190] jepsen worker 17 - jepsen.util 2267 :invoke :write [5 42] INFO [2019-12-17 12:29:03,195] jepsen worker 27 - jepsen.util 1377 :fail :cas [9 [4 43]] :ValueNotSwapFrom INFO [2019-12-17 12:29:03,195] jepsen worker 27 - jepsen.util 1377 :invoke :read [9 nil] INFO [2019-12-17 12:29:03,206] jepsen worker 13 - jepsen.util 1453 :ok :read [4 49] INFO [2019-12-17 12:29:03,206] jepsen worker 13 - jepsen.util 1453 :invoke :write [4 1] INFO [2019-12-17 12:29:03,207] jepsen worker 42 - jepsen.util 2562 :info :write [14 35] :RequestCanceledException INFO [2019-12-17 12:29:03,207] jepsen worker 59 - jepsen.util 1859 :info :cas [19 [22 39]] :RequestCanceledException INFO [2019-12-17 12:29:03,208] jepsen worker 27 - jepsen.util 1377 :ok :read [9 40] INFO [2019-12-17 12:29:03,209] jepsen worker 42 - jepsen.util 2652 :invoke :read [14 nil] INFO [2019-12-17 12:29:03,209] jepsen worker 27 - jepsen.util 1377 :invoke :read [9 nil] INFO [2019-12-17 12:29:03,209] jepsen worker 59 - jepsen.util 1949 :invoke :read [19 nil] INFO [2019-12-17 12:29:03,219] jepsen worker 5 - jepsen.util 1355 :fail :cas [1 [44 41]] :ValueNotSwapFrom INFO [2019-12-17 12:29:03,219] jepsen worker 5 - jepsen.util 1355 :invoke :cas [1 [28 5]] INFO [2019-12-17 12:29:03,222] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9738,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:29:03,222] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9738,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:29:03,222] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9739,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:29:03,222] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9739,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:29:03,222] jepsen worker 19 - jepsen.util 1909 :info :write [6 48] :RequestCanceledException INFO [2019-12-17 12:29:03,222] jepsen worker 41 - jepsen.util 2291 :info :write [13 46] :RequestCanceledException INFO [2019-12-17 12:29:03,222] jepsen worker 28 - jepsen.util 1018 :info :cas [9 [29 42]] :RequestCanceledException INFO [2019-12-17 12:29:03,223] jepsen worker 27 - jepsen.util 1377 :ok :read [9 40] INFO [2019-12-17 12:29:03,223] jepsen worker 27 - jepsen.util 1377 :invoke :read [9 nil] INFO [2019-12-17 12:29:03,223] jepsen worker 19 - jepsen.util 1999 :invoke :cas [6 [23 16]] INFO [2019-12-17 12:29:03,223] jepsen worker 41 - jepsen.util 2381 :invoke :cas [13 [20 27]] INFO [2019-12-17 12:29:03,224] jepsen worker 28 - jepsen.util 1108 :invoke :write [9 46] INFO [2019-12-17 12:29:03,236] jepsen worker 83 - jepsen.util 2423 :info :write [27 11] :RequestCanceledException INFO [2019-12-17 12:29:03,236] jepsen worker 68 - jepsen.util 1868 :info :cas [22 [7 11]] :RequestCanceledException INFO [2019-12-17 12:29:03,237] jepsen worker 68 - jepsen.util 1958 :invoke :cas [22 [3 39]] INFO [2019-12-17 12:29:03,237] jepsen worker 83 - jepsen.util 2513 :invoke :cas [27 [6 38]] INFO [2019-12-17 12:29:03,240] jepsen worker 27 - jepsen.util 1377 :ok :read [9 40] INFO [2019-12-17 12:29:03,240] jepsen worker 27 - jepsen.util 1377 :invoke :read [9 nil] INFO [2019-12-17 12:29:03,243] jepsen worker 42 - jepsen.util 2652 :ok :read [14 27] INFO [2019-12-17 12:29:03,244] jepsen worker 42 - jepsen.util 2652 :invoke :read [14 nil] INFO [2019-12-17 12:29:03,251] jepsen worker 27 - jepsen.util 1377 :ok :read [9 40] INFO [2019-12-17 12:29:03,252] jepsen worker 27 - jepsen.util 1377 :invoke :read [9 nil] INFO [2019-12-17 12:29:03,264] jepsen worker 27 - jepsen.util 1377 :ok :read [9 40] INFO [2019-12-17 12:29:03,264] jepsen worker 27 - jepsen.util 1377 :invoke :cas [9 [6 11]] INFO [2019-12-17 12:29:03,274] jepsen worker 27 - jepsen.util 1377 :fail :cas [9 [6 11]] :ValueNotSwapFrom INFO [2019-12-17 12:29:03,274] jepsen worker 27 - jepsen.util 1377 :invoke :cas [9 [14 32]] INFO [2019-12-17 12:29:03,275] jepsen worker 42 - jepsen.util 2652 :ok :read [14 27] INFO [2019-12-17 12:29:03,275] jepsen worker 42 - jepsen.util 2652 :invoke :write [14 10] INFO [2019-12-17 12:29:03,285] jepsen worker 27 - jepsen.util 1377 :fail :cas [9 [14 32]] :ValueNotSwapFrom INFO [2019-12-17 12:29:03,285] jepsen worker 27 - jepsen.util 1377 :invoke :cas [9 [10 26]] INFO [2019-12-17 12:29:03,289] jepsen worker 12 - jepsen.util 1722 :info :write [4 26] :RequestCanceledException INFO [2019-12-17 12:29:03,290] jepsen worker 12 - jepsen.util 1812 :invoke :read [4 nil] INFO [2019-12-17 12:29:03,296] jepsen worker 27 - jepsen.util 1377 :fail :cas [9 [10 26]] :ValueNotSwapFrom INFO [2019-12-17 12:29:03,296] jepsen worker 27 - jepsen.util 1377 :invoke :read [9 nil] INFO [2019-12-17 12:29:03,299] jepsen worker 87 - jepsen.util 1437 :fail :cas [29 [19 35]] :ValueNotSwapFrom INFO [2019-12-17 12:29:03,299] jepsen worker 89 - jepsen.util 1619 :ok :read [29 21] INFO [2019-12-17 12:29:03,299] jepsen worker 87 - jepsen.util 1437 :invoke :write [29 19] INFO [2019-12-17 12:29:03,299] jepsen worker 89 - jepsen.util 1619 :invoke :write [29 49] INFO [2019-12-17 12:29:03,308] jepsen worker 27 - jepsen.util 1377 :ok :read [9 40] INFO [2019-12-17 12:29:03,308] jepsen worker 27 - jepsen.util 1377 :invoke :write [9 14] INFO [2019-12-17 12:29:03,319] jepsen worker 20 - jepsen.util 1550 :ok :write [6 41] INFO [2019-12-17 12:29:03,320] jepsen worker 20 - jepsen.util 1550 :invoke :cas [6 [0 0]] INFO [2019-12-17 12:29:03,320] jepsen worker 73 - jepsen.util 1603 :ok :write [24 33] INFO [2019-12-17 12:29:03,320] jepsen worker 73 - jepsen.util 1603 :invoke :read [24 nil] INFO [2019-12-17 12:29:03,322] jepsen worker 29 - jepsen.util 1379 :info :cas [9 [2 46]] :RequestCanceledException INFO [2019-12-17 12:29:03,323] jepsen worker 29 - jepsen.util 1469 :invoke :write [9 3] INFO [2019-12-17 12:29:03,332] jepsen worker 9 - jepsen.util 1809 :ok :write [3 26] INFO [2019-12-17 12:29:03,333] jepsen worker 9 - jepsen.util 1809 :invoke :cas [3 [13 36]] INFO [2019-12-17 12:29:03,333] jepsen worker 31 - jepsen.util 1741 :ok :write [10 0] INFO [2019-12-17 12:29:03,333] jepsen worker 31 - jepsen.util 1741 :invoke :read [10 nil] INFO [2019-12-17 12:29:03,336] jepsen worker 30 - jepsen.util 2010 :fail :read [10 nil] :RequestTimeoutException INFO [2019-12-17 12:29:03,336] jepsen worker 30 - jepsen.util 2010 :invoke :read [10 nil] INFO [2019-12-17 12:29:03,352] jepsen worker 20 - jepsen.util 1550 :fail :cas [6 [0 0]] :ValueNotSwapFrom INFO [2019-12-17 12:29:03,352] jepsen worker 20 - jepsen.util 1550 :invoke :write [6 32] INFO [2019-12-17 12:29:03,369] jepsen worker 73 - jepsen.util 1603 :ok :read [24 33] INFO [2019-12-17 12:29:03,369] jepsen worker 73 - jepsen.util 1603 :invoke :read [24 nil] INFO [2019-12-17 12:29:03,376] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:29:03,376] jepsen nemesis - couchbase.util Rebalance check count: 68 INFO [2019-12-17 12:29:03,376] jepsen nemesis - couchbase.util Rebalance stuck count: 49 INFO [2019-12-17 12:29:03,386] jepsen worker 73 - jepsen.util 1603 :ok :read [24 33] INFO [2019-12-17 12:29:03,386] jepsen worker 73 - jepsen.util 1603 :invoke :write [24 32] INFO [2019-12-17 12:29:03,389] jepsen worker 71 - jepsen.util 1241 :info :write [23 38] :RequestCanceledException INFO [2019-12-17 12:29:03,389] jepsen worker 67 - jepsen.util 2677 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:29:03,389] jepsen worker 67 - jepsen.util 2677 :invoke :cas [22 [10 39]] INFO [2019-12-17 12:29:03,390] jepsen worker 71 - jepsen.util 1331 :invoke :read [23 nil] INFO [2019-12-17 12:29:03,407] jepsen worker 74 - jepsen.util 1604 :info :cas [24 [18 32]] :RequestCanceledException INFO [2019-12-17 12:29:03,408] jepsen worker 74 - jepsen.util 1694 :invoke :read [24 nil] INFO [2019-12-17 12:29:03,422] jepsen worker 48 - jepsen.util 1758 :info :write [16 35] :RequestCanceledException INFO [2019-12-17 12:29:03,423] jepsen worker 48 - jepsen.util 1848 :invoke :cas [16 [13 3]] INFO [2019-12-17 12:29:03,440] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9777,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:29:03,440] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9777,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:29:03,456] jepsen worker 28 - jepsen.util 1108 :ok :write [9 46] INFO [2019-12-17 12:29:03,456] jepsen worker 28 - jepsen.util 1108 :invoke :cas [9 [48 5]] INFO [2019-12-17 12:29:03,476] jepsen worker 74 - jepsen.util 1694 :ok :read [24 33] INFO [2019-12-17 12:29:03,476] jepsen worker 74 - jepsen.util 1694 :invoke :cas [24 [3 42]] INFO [2019-12-17 12:29:03,486] jepsen worker 79 - jepsen.util 1699 :info :write [26 3] :RequestCanceledException INFO [2019-12-17 12:29:03,487] jepsen worker 79 - jepsen.util 1789 :invoke :read [26 nil] INFO [2019-12-17 12:29:03,490] jepsen worker 28 - jepsen.util 1108 :fail :cas [9 [48 5]] :ValueNotSwapFrom INFO [2019-12-17 12:29:03,490] jepsen worker 28 - jepsen.util 1108 :invoke :read [9 nil] INFO [2019-12-17 12:29:03,499] jepsen worker 74 - jepsen.util 1694 :fail :cas [24 [3 42]] :ValueNotSwapFrom INFO [2019-12-17 12:29:03,499] jepsen worker 74 - jepsen.util 1694 :invoke :read [24 nil] INFO [2019-12-17 12:29:03,511] jepsen worker 28 - jepsen.util 1108 :ok :read [9 46] INFO [2019-12-17 12:29:03,511] jepsen worker 28 - jepsen.util 1108 :invoke :cas [9 [8 24]] INFO [2019-12-17 12:29:03,520] jepsen worker 74 - jepsen.util 1694 :ok :read [24 33] INFO [2019-12-17 12:29:03,520] jepsen worker 74 - jepsen.util 1694 :invoke :write [24 34] INFO [2019-12-17 12:29:03,522] jepsen worker 27 - jepsen.util 1377 :ok :write [9 14] INFO [2019-12-17 12:29:03,522] jepsen worker 27 - jepsen.util 1377 :invoke :cas [9 [3 45]] INFO [2019-12-17 12:29:03,523] jepsen worker 80 - jepsen.util 1700 :info :write [26 21] :RequestCanceledException INFO [2019-12-17 12:29:03,523] jepsen worker 45 - jepsen.util 1935 :info :write [15 3] :RequestCanceledException INFO [2019-12-17 12:29:03,523] jepsen worker 11 - jepsen.util 1991 :info :write [3 30] :RequestCanceledException INFO [2019-12-17 12:29:03,523] jepsen worker 10 - jepsen.util 1810 :info :write [3 24] :RequestCanceledException INFO [2019-12-17 12:29:03,525] jepsen worker 45 - jepsen.util 2025 :invoke :cas [15 [10 34]] INFO [2019-12-17 12:29:03,525] jepsen worker 80 - jepsen.util 1790 :invoke :cas [26 [38 23]] INFO [2019-12-17 12:29:03,525] jepsen worker 10 - jepsen.util 1900 :invoke :cas [3 [14 35]] INFO [2019-12-17 12:29:03,525] jepsen worker 11 - jepsen.util 2081 :invoke :write [3 49] INFO [2019-12-17 12:29:03,539] jepsen worker 36 - jepsen.util 1566 :ok :write [12 38] INFO [2019-12-17 12:29:03,539] jepsen worker 36 - jepsen.util 1566 :invoke :cas [12 [4 13]] INFO [2019-12-17 12:29:03,547] jepsen worker 37 - jepsen.util 1387 :fail :cas [12 [25 7]] :ValueNotSwapFrom INFO [2019-12-17 12:29:03,547] jepsen worker 37 - jepsen.util 1387 :invoke :cas [12 [18 34]] INFO [2019-12-17 12:29:03,557] jepsen worker 28 - jepsen.util 1108 :fail :cas [9 [8 24]] :ValueNotSwapFrom INFO [2019-12-17 12:29:03,557] jepsen worker 28 - jepsen.util 1108 :invoke :read [9 nil] INFO [2019-12-17 12:29:03,577] jepsen worker 65 - jepsen.util 1415 :fail :cas [21 [15 34]] :ValueNotSwapFrom INFO [2019-12-17 12:29:03,577] jepsen worker 65 - jepsen.util 1415 :invoke :write [21 4] INFO [2019-12-17 12:29:03,583] jepsen worker 10 - jepsen.util 1900 :fail :cas [3 [14 35]] :ValueNotSwapFrom INFO [2019-12-17 12:29:03,583] jepsen worker 10 - jepsen.util 1900 :invoke :read [3 nil] INFO [2019-12-17 12:29:03,586] jepsen worker 47 - jepsen.util 2117 :ok :write [15 32] INFO [2019-12-17 12:29:03,586] jepsen worker 47 - jepsen.util 2117 :invoke :write [15 33] INFO [2019-12-17 12:29:03,588] jepsen worker 53 - jepsen.util 1493 :info :write [17 28] :RequestCanceledException INFO [2019-12-17 12:29:03,589] jepsen worker 53 - jepsen.util 1583 :invoke :cas [17 [27 29]] INFO [2019-12-17 12:29:03,590] jepsen worker 27 - jepsen.util 1377 :fail :cas [9 [3 45]] :ValueNotSwapFrom INFO [2019-12-17 12:29:03,590] jepsen worker 27 - jepsen.util 1377 :invoke :write [9 32] INFO [2019-12-17 12:29:03,606] jepsen worker 4 - jepsen.util 1264 :info :write [1 47] :RequestCanceledException INFO [2019-12-17 12:29:03,608] jepsen worker 4 - jepsen.util 1354 :invoke :read [1 nil] INFO [2019-12-17 12:29:03,620] jepsen worker 12 - jepsen.util 1812 :ok :read [4 49] INFO [2019-12-17 12:29:03,620] jepsen worker 12 - jepsen.util 1812 :invoke :write [4 38] INFO [2019-12-17 12:29:03,622] jepsen worker 3 - jepsen.util 1533 :info :write [1 46] :RequestCanceledException INFO [2019-12-17 12:29:03,622] jepsen worker 18 - jepsen.util 1908 :fail :read [6 nil] :RequestTimeoutException INFO [2019-12-17 12:29:03,622] jepsen worker 18 - jepsen.util 1908 :invoke :write [6 19] INFO [2019-12-17 12:29:03,623] jepsen worker 3 - jepsen.util 1623 :invoke :cas [1 [30 9]] INFO [2019-12-17 12:29:03,628] jepsen worker 10 - jepsen.util 1900 :ok :read [3 26] INFO [2019-12-17 12:29:03,628] jepsen worker 10 - jepsen.util 1900 :invoke :read [3 nil] INFO [2019-12-17 12:29:03,645] jepsen worker 19 - jepsen.util 1999 :fail :cas [6 [23 16]] :ValueNotSwapFrom INFO [2019-12-17 12:29:03,645] jepsen worker 19 - jepsen.util 1999 :invoke :write [6 47] INFO [2019-12-17 12:29:03,653] jepsen worker 2 - jepsen.util 1982 :info :cas [0 [29 8]] :RequestCanceledException INFO [2019-12-17 12:29:03,654] jepsen worker 2 - jepsen.util 2072 :invoke :cas [0 [25 16]] INFO [2019-12-17 12:29:03,660] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9810,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:29:03,660] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9810,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:29:03,660] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9812,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:29:03,660] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9812,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:29:03,682] jepsen worker 29 - jepsen.util 1469 :ok :write [9 3] INFO [2019-12-17 12:29:03,683] jepsen worker 29 - jepsen.util 1469 :invoke :read [9 nil] INFO [2019-12-17 12:29:03,698] jepsen worker 49 - jepsen.util 1579 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:29:03,699] jepsen worker 49 - jepsen.util 1579 :invoke :read [16 nil] INFO [2019-12-17 12:29:03,704] jepsen worker 20 - jepsen.util 1550 :ok :write [6 32] INFO [2019-12-17 12:29:03,704] jepsen worker 80 - jepsen.util 1790 :fail :cas [26 [38 23]] :ValueNotSwapFrom INFO [2019-12-17 12:29:03,704] jepsen worker 20 - jepsen.util 1550 :invoke :write [6 49] INFO [2019-12-17 12:29:03,704] jepsen worker 80 - jepsen.util 1790 :invoke :write [26 9] INFO [2019-12-17 12:29:03,715] jepsen worker 10 - jepsen.util 1900 :ok :read [3 26] INFO [2019-12-17 12:29:03,715] jepsen worker 10 - jepsen.util 1900 :invoke :cas [3 [4 11]] INFO [2019-12-17 12:29:03,737] jepsen worker 44 - jepsen.util 1754 :info :cas [14 [2 48]] :RequestCanceledException INFO [2019-12-17 12:29:03,738] jepsen worker 44 - jepsen.util 1844 :invoke :cas [14 [21 37]] INFO [2019-12-17 12:29:03,739] jepsen worker 37 - jepsen.util 1387 :fail :cas [12 [18 34]] :ValueNotSwapFrom INFO [2019-12-17 12:29:03,739] jepsen worker 37 - jepsen.util 1387 :invoke :cas [12 [37 43]] INFO [2019-12-17 12:29:03,742] jepsen worker 10 - jepsen.util 1900 :fail :cas [3 [4 11]] :ValueNotSwapFrom INFO [2019-12-17 12:29:03,742] jepsen worker 10 - jepsen.util 1900 :invoke :cas [3 [23 1]] INFO [2019-12-17 12:29:03,751] jepsen worker 51 - jepsen.util 1671 :ok :read [17 2] INFO [2019-12-17 12:29:03,751] jepsen worker 51 - jepsen.util 1671 :invoke :write [17 43] INFO [2019-12-17 12:29:03,776] jepsen worker 10 - jepsen.util 1900 :fail :cas [3 [23 1]] :ValueNotSwapFrom INFO [2019-12-17 12:29:03,776] jepsen worker 10 - jepsen.util 1900 :invoke :read [3 nil] INFO [2019-12-17 12:29:03,786] jepsen worker 73 - jepsen.util 1603 :ok :write [24 32] INFO [2019-12-17 12:29:03,787] jepsen worker 73 - jepsen.util 1603 :invoke :cas [24 [38 18]] INFO [2019-12-17 12:29:03,792] jepsen worker 3 - jepsen.util 1623 :fail :cas [1 [30 9]] :ValueNotSwapFrom INFO [2019-12-17 12:29:03,792] jepsen worker 3 - jepsen.util 1623 :invoke :write [1 13] INFO [2019-12-17 12:29:03,793] jepsen worker 10 - jepsen.util 1900 :ok :read [3 26] INFO [2019-12-17 12:29:03,793] jepsen worker 10 - jepsen.util 1900 :invoke :write [3 39] INFO [2019-12-17 12:29:03,798] jepsen worker 19 - jepsen.util 1999 :ok :write [6 47] INFO [2019-12-17 12:29:03,798] jepsen worker 19 - jepsen.util 1999 :invoke :cas [6 [21 11]] INFO [2019-12-17 12:29:03,800] jepsen worker 55 - jepsen.util 1945 :info :write [18 37] :RequestCanceledException INFO [2019-12-17 12:29:03,802] jepsen worker 55 - jepsen.util 2035 :invoke :read [18 nil] INFO [2019-12-17 12:29:03,807] jepsen worker 45 - jepsen.util 2025 :fail :cas [15 [10 34]] :ValueNotSwapFrom INFO [2019-12-17 12:29:03,807] jepsen worker 45 - jepsen.util 2025 :invoke :cas [15 [28 12]] INFO [2019-12-17 12:29:03,822] jepsen worker 14 - jepsen.util 1634 :info :write [4 15] :RequestCanceledException INFO [2019-12-17 12:29:03,824] jepsen worker 14 - jepsen.util 1724 :invoke :write [4 28] INFO [2019-12-17 12:29:03,829] jepsen worker 19 - jepsen.util 1999 :fail :cas [6 [21 11]] :ValueNotSwapFrom INFO [2019-12-17 12:29:03,829] jepsen worker 19 - jepsen.util 1999 :invoke :write [6 13] INFO [2019-12-17 12:29:03,835] jepsen worker 73 - jepsen.util 1603 :fail :cas [24 [38 18]] :ValueNotSwapFrom INFO [2019-12-17 12:29:03,836] jepsen worker 73 - jepsen.util 1603 :invoke :cas [24 [41 47]] INFO [2019-12-17 12:29:03,863] jepsen worker 45 - jepsen.util 2025 :fail :cas [15 [28 12]] :ValueNotSwapFrom INFO [2019-12-17 12:29:03,863] jepsen worker 45 - jepsen.util 2025 :invoke :write [15 14] INFO [2019-12-17 12:29:03,878] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9835,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:29:03,878] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9835,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:29:03,878] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9838,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:29:03,878] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9838,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:29:03,878] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9839,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:29:03,878] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9839,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:29:03,882] jepsen worker 73 - jepsen.util 1603 :fail :cas [24 [41 47]] :ValueNotSwapFrom INFO [2019-12-17 12:29:03,882] jepsen worker 73 - jepsen.util 1603 :invoke :write [24 49] INFO [2019-12-17 12:29:03,892] jepsen worker 12 - jepsen.util 1812 :ok :write [4 38] INFO [2019-12-17 12:29:03,892] jepsen worker 12 - jepsen.util 1812 :invoke :cas [4 [11 49]] INFO [2019-12-17 12:29:03,942] jepsen worker 36 - jepsen.util 1566 :fail :cas [12 [4 13]] :ValueNotSwapFrom INFO [2019-12-17 12:29:03,942] jepsen worker 36 - jepsen.util 1566 :invoke :cas [12 [29 23]] INFO [2019-12-17 12:29:03,986] jepsen worker 75 - jepsen.util 2505 :info :write [25 21] :RequestCanceledException INFO [2019-12-17 12:29:03,987] jepsen worker 75 - jepsen.util 2595 :invoke :cas [25 [10 30]] INFO [2019-12-17 12:29:03,998] jepsen worker 71 - jepsen.util 1331 :ok :read [23 44] INFO [2019-12-17 12:29:03,998] jepsen worker 71 - jepsen.util 1331 :invoke :write [23 6] INFO [2019-12-17 12:29:04,014] jepsen worker 19 - jepsen.util 1999 :ok :write [6 13] INFO [2019-12-17 12:29:04,014] jepsen worker 10 - jepsen.util 1900 :ok :write [3 39] INFO [2019-12-17 12:29:04,014] jepsen worker 19 - jepsen.util 1999 :invoke :read [6 nil] INFO [2019-12-17 12:29:04,014] jepsen worker 10 - jepsen.util 1900 :invoke :write [3 49] INFO [2019-12-17 12:29:04,021] jepsen worker 6 - jepsen.util 2256 :info :write [2 35] :RequestCanceledException INFO [2019-12-17 12:29:04,022] jepsen worker 6 - jepsen.util 2346 :invoke :read [2 nil] INFO [2019-12-17 12:29:04,027] jepsen worker 45 - jepsen.util 2025 :ok :write [15 14] INFO [2019-12-17 12:29:04,028] jepsen worker 45 - jepsen.util 2025 :invoke :read [15 nil] INFO [2019-12-17 12:29:04,029] jepsen worker 19 - jepsen.util 1999 :ok :read [6 13] INFO [2019-12-17 12:29:04,029] jepsen worker 19 - jepsen.util 1999 :invoke :write [6 6] INFO [2019-12-17 12:29:04,044] jepsen worker 5 - jepsen.util 1355 :fail :cas [1 [28 5]] :ValueNotSwapFrom INFO [2019-12-17 12:29:04,044] jepsen worker 5 - jepsen.util 1355 :invoke :cas [1 [44 49]] INFO [2019-12-17 12:29:04,049] jepsen worker 37 - jepsen.util 1387 :fail :cas [12 [37 43]] :ValueNotSwapFrom INFO [2019-12-17 12:29:04,049] jepsen worker 37 - jepsen.util 1387 :invoke :write [12 38] INFO [2019-12-17 12:29:04,064] jepsen worker 75 - jepsen.util 2595 :fail :cas [25 [10 30]] :ValueNotSwapFrom INFO [2019-12-17 12:29:04,064] jepsen worker 75 - jepsen.util 2595 :invoke :cas [25 [33 41]] INFO [2019-12-17 12:29:04,065] jepsen worker 45 - jepsen.util 2025 :ok :read [15 14] INFO [2019-12-17 12:29:04,065] jepsen worker 45 - jepsen.util 2025 :invoke :cas [15 [0 40]] INFO [2019-12-17 12:29:04,113] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9846,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:29:04,113] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9846,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:29:04,113] jepsen worker 60 - jepsen.util 2310 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:29:04,113] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9849,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:29:04,113] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9849,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:29:04,113] jepsen worker 60 - jepsen.util 2310 :invoke :write [20 2] INFO [2019-12-17 12:29:04,118] jepsen worker 75 - jepsen.util 2595 :fail :cas [25 [33 41]] :ValueNotSwapFrom INFO [2019-12-17 12:29:04,118] jepsen worker 75 - jepsen.util 2595 :invoke :read [25 nil] INFO [2019-12-17 12:29:04,121] jepsen worker 45 - jepsen.util 2025 :fail :cas [15 [0 40]] :ValueNotSwapFrom INFO [2019-12-17 12:29:04,121] jepsen worker 45 - jepsen.util 2025 :invoke :read [15 nil] INFO [2019-12-17 12:29:04,122] jepsen worker 56 - jepsen.util 1586 :info :write [18 45] :RequestCanceledException INFO [2019-12-17 12:29:04,123] jepsen worker 56 - jepsen.util 1676 :invoke :cas [18 [34 27]] INFO [2019-12-17 12:29:04,131] jepsen worker 36 - jepsen.util 1566 :fail :cas [12 [29 23]] :ValueNotSwapFrom INFO [2019-12-17 12:29:04,131] jepsen worker 36 - jepsen.util 1566 :invoke :cas [12 [49 32]] INFO [2019-12-17 12:29:04,134] jepsen worker 12 - jepsen.util 1812 :fail :cas [4 [11 49]] :ValueNotSwapFrom INFO [2019-12-17 12:29:04,135] jepsen worker 12 - jepsen.util 1812 :invoke :read [4 nil] INFO [2019-12-17 12:29:04,136] jepsen worker 63 - jepsen.util 1503 :fail :read [21 nil] :RequestTimeoutException INFO [2019-12-17 12:29:04,136] jepsen worker 78 - jepsen.util 1608 :info :cas [26 [34 40]] :RequestCanceledException INFO [2019-12-17 12:29:04,136] jepsen worker 63 - jepsen.util 1503 :invoke :read [21 nil] INFO [2019-12-17 12:29:04,137] jepsen worker 78 - jepsen.util 1698 :invoke :write [26 36] INFO [2019-12-17 12:29:04,138] jepsen worker 75 - jepsen.util 2595 :ok :read [25 2] INFO [2019-12-17 12:29:04,138] jepsen worker 75 - jepsen.util 2595 :invoke :read [25 nil] INFO [2019-12-17 12:29:04,144] jepsen worker 45 - jepsen.util 2025 :ok :read [15 14] INFO [2019-12-17 12:29:04,144] jepsen worker 45 - jepsen.util 2025 :invoke :cas [15 [5 49]] INFO [2019-12-17 12:29:04,151] jepsen worker 63 - jepsen.util 1503 :ok :read [21 25] INFO [2019-12-17 12:29:04,151] jepsen worker 63 - jepsen.util 1503 :invoke :read [21 nil] INFO [2019-12-17 12:29:04,157] jepsen worker 79 - jepsen.util 1789 :ok :read [26 34] INFO [2019-12-17 12:29:04,157] jepsen worker 79 - jepsen.util 1789 :invoke :write [26 30] INFO [2019-12-17 12:29:04,179] jepsen worker 63 - jepsen.util 1503 :ok :read [21 25] INFO [2019-12-17 12:29:04,179] jepsen worker 63 - jepsen.util 1503 :invoke :read [21 nil] INFO [2019-12-17 12:29:04,180] jepsen worker 4 - jepsen.util 1354 :ok :read [1 9] INFO [2019-12-17 12:29:04,180] jepsen worker 4 - jepsen.util 1354 :invoke :write [1 5] INFO [2019-12-17 12:29:04,180] jepsen worker 45 - jepsen.util 2025 :fail :cas [15 [5 49]] :ValueNotSwapFrom INFO [2019-12-17 12:29:04,180] jepsen worker 45 - jepsen.util 2025 :invoke :cas [15 [4 37]] INFO [2019-12-17 12:29:04,201] jepsen worker 63 - jepsen.util 1503 :ok :read [21 25] INFO [2019-12-17 12:29:04,201] jepsen worker 63 - jepsen.util 1503 :invoke :write [21 8] INFO [2019-12-17 12:29:04,205] jepsen worker 45 - jepsen.util 2025 :fail :cas [15 [4 37]] :ValueNotSwapFrom INFO [2019-12-17 12:29:04,205] jepsen worker 45 - jepsen.util 2025 :invoke :read [15 nil] INFO [2019-12-17 12:29:04,207] jepsen worker 52 - jepsen.util 1582 :info :write [17 45] :RequestCanceledException INFO [2019-12-17 12:29:04,208] jepsen worker 52 - jepsen.util 1672 :invoke :cas [17 [19 11]] INFO [2019-12-17 12:29:04,222] jepsen worker 7 - jepsen.util 2437 :info :cas [2 [15 5]] :RequestCanceledException INFO [2019-12-17 12:29:04,222] jepsen worker 70 - jepsen.util 1870 :info :write [23 32] :RequestCanceledException INFO [2019-12-17 12:29:04,222] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9876,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:29:04,222] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9885,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:29:04,222] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9876,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:29:04,223] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9885,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:29:04,223] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9878,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:29:04,223] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9888,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:29:04,223] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9878,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:29:04,223] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9888,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:29:04,223] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9889,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:29:04,223] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9889,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:29:04,223] jepsen worker 7 - jepsen.util 2527 :invoke :read [2 nil] INFO [2019-12-17 12:29:04,223] jepsen worker 70 - jepsen.util 1960 :invoke :read [23 nil] INFO [2019-12-17 12:29:04,226] jepsen worker 10 - jepsen.util 1900 :ok :write [3 49] INFO [2019-12-17 12:29:04,226] jepsen worker 10 - jepsen.util 1900 :invoke :cas [3 [2 43]] INFO [2019-12-17 12:29:04,229] jepsen worker 45 - jepsen.util 2025 :ok :read [15 14] INFO [2019-12-17 12:29:04,229] jepsen worker 45 - jepsen.util 2025 :invoke :write [15 7] INFO [2019-12-17 12:29:04,236] jepsen worker 72 - jepsen.util 1782 :info :cas [24 [21 42]] :RequestCanceledException INFO [2019-12-17 12:29:04,237] jepsen worker 72 - jepsen.util 1872 :invoke :cas [24 [49 20]] INFO [2019-12-17 12:29:04,239] jepsen worker 10 - jepsen.util 1900 :fail :cas [3 [2 43]] :ValueNotSwapFrom INFO [2019-12-17 12:29:04,239] jepsen worker 10 - jepsen.util 1900 :invoke :write [3 33] INFO [2019-12-17 12:29:04,245] jepsen worker 19 - jepsen.util 1999 :ok :write [6 6] INFO [2019-12-17 12:29:04,245] jepsen worker 19 - jepsen.util 1999 :invoke :read [6 nil] INFO [2019-12-17 12:29:04,260] jepsen worker 19 - jepsen.util 1999 :ok :read [6 6] INFO [2019-12-17 12:29:04,260] jepsen worker 63 - jepsen.util 1503 :ok :write [21 8] INFO [2019-12-17 12:29:04,260] jepsen worker 63 - jepsen.util 1503 :invoke :cas [21 [47 30]] INFO [2019-12-17 12:29:04,260] jepsen worker 19 - jepsen.util 1999 :invoke :cas [6 [38 11]] INFO [2019-12-17 12:29:04,267] jepsen worker 75 - jepsen.util 2595 :ok :read [25 2] INFO [2019-12-17 12:29:04,268] jepsen worker 75 - jepsen.util 2595 :invoke :read [25 nil] INFO [2019-12-17 12:29:04,282] jepsen worker 63 - jepsen.util 1503 :fail :cas [21 [47 30]] :ValueNotSwapFrom INFO [2019-12-17 12:29:04,282] jepsen worker 63 - jepsen.util 1503 :invoke :read [21 nil] INFO [2019-12-17 12:29:04,283] jepsen worker 19 - jepsen.util 1999 :fail :cas [6 [38 11]] :ValueNotSwapFrom INFO [2019-12-17 12:29:04,283] jepsen worker 19 - jepsen.util 1999 :invoke :read [6 nil] INFO [2019-12-17 12:29:04,284] jepsen worker 75 - jepsen.util 2595 :ok :read [25 2] INFO [2019-12-17 12:29:04,284] jepsen worker 75 - jepsen.util 2595 :invoke :read [25 nil] INFO [2019-12-17 12:29:04,289] jepsen worker 64 - jepsen.util 1414 :info :cas [21 [40 46]] :RequestCanceledException INFO [2019-12-17 12:29:04,291] jepsen worker 64 - jepsen.util 1504 :invoke :write [21 21] INFO [2019-12-17 12:29:04,301] jepsen worker 75 - jepsen.util 2595 :ok :read [25 2] INFO [2019-12-17 12:29:04,301] jepsen worker 75 - jepsen.util 2595 :invoke :cas [25 [31 14]] INFO [2019-12-17 12:29:04,307] jepsen worker 63 - jepsen.util 1503 :ok :read [21 8] INFO [2019-12-17 12:29:04,307] jepsen worker 63 - jepsen.util 1503 :invoke :read [21 nil] INFO [2019-12-17 12:29:04,316] jepsen worker 19 - jepsen.util 1999 :ok :read [6 6] INFO [2019-12-17 12:29:04,316] jepsen worker 19 - jepsen.util 1999 :invoke :read [6 nil] INFO [2019-12-17 12:29:04,316] jepsen worker 75 - jepsen.util 2595 :fail :cas [25 [31 14]] :ValueNotSwapFrom INFO [2019-12-17 12:29:04,316] jepsen worker 75 - jepsen.util 2595 :invoke :write [25 19] INFO [2019-12-17 12:29:04,336] jepsen worker 63 - jepsen.util 1503 :ok :read [21 8] INFO [2019-12-17 12:29:04,336] jepsen worker 63 - jepsen.util 1503 :invoke :read [21 nil] INFO [2019-12-17 12:29:04,337] jepsen worker 55 - jepsen.util 2035 :ok :read [18 34] INFO [2019-12-17 12:29:04,338] jepsen worker 55 - jepsen.util 2035 :invoke :write [18 20] INFO [2019-12-17 12:29:04,346] jepsen worker 12 - jepsen.util 1812 :ok :read [4 38] INFO [2019-12-17 12:29:04,346] jepsen worker 12 - jepsen.util 1812 :invoke :read [4 nil] INFO [2019-12-17 12:29:04,352] jepsen worker 53 - jepsen.util 1583 :fail :cas [17 [27 29]] :ValueNotSwapFrom INFO [2019-12-17 12:29:04,352] jepsen worker 53 - jepsen.util 1583 :invoke :cas [17 [17 12]] INFO [2019-12-17 12:29:04,364] jepsen worker 63 - jepsen.util 1503 :ok :read [21 8] INFO [2019-12-17 12:29:04,364] jepsen worker 63 - jepsen.util 1503 :invoke :cas [21 [0 13]] INFO [2019-12-17 12:29:04,383] jepsen worker 63 - jepsen.util 1503 :fail :cas [21 [0 13]] :ValueNotSwapFrom INFO [2019-12-17 12:29:04,383] jepsen worker 63 - jepsen.util 1503 :invoke :read [21 nil] INFO [2019-12-17 12:29:04,390] jepsen worker 64 - jepsen.util 1504 :ok :write [21 21] INFO [2019-12-17 12:29:04,390] jepsen worker 64 - jepsen.util 1504 :invoke :cas [21 [49 18]] INFO [2019-12-17 12:29:04,391] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:29:04,391] jepsen nemesis - couchbase.util Rebalance check count: 69 INFO [2019-12-17 12:29:04,391] jepsen nemesis - couchbase.util Rebalance stuck count: 50 INFO [2019-12-17 12:29:04,403] jepsen worker 63 - jepsen.util 1503 :ok :read [21 8] INFO [2019-12-17 12:29:04,403] jepsen worker 63 - jepsen.util 1503 :invoke :write [21 3] INFO [2019-12-17 12:29:04,409] jepsen worker 64 - jepsen.util 1504 :fail :cas [21 [49 18]] :ValueNotSwapFrom INFO [2019-12-17 12:29:04,409] jepsen worker 64 - jepsen.util 1504 :invoke :read [21 nil] INFO [2019-12-17 12:29:04,412] jepsen worker 5 - jepsen.util 1355 :fail :cas [1 [44 49]] :ValueNotSwapFrom INFO [2019-12-17 12:29:04,413] jepsen worker 5 - jepsen.util 1355 :invoke :cas [1 [10 0]] INFO [2019-12-17 12:29:04,427] jepsen worker 64 - jepsen.util 1504 :ok :read [21 21] INFO [2019-12-17 12:29:04,427] jepsen worker 64 - jepsen.util 1504 :invoke :read [21 nil] INFO [2019-12-17 12:29:04,433] jepsen worker 36 - jepsen.util 1566 :fail :cas [12 [49 32]] :ValueNotSwapFrom INFO [2019-12-17 12:29:04,434] jepsen worker 36 - jepsen.util 1566 :invoke :read [12 nil] INFO [2019-12-17 12:29:04,441] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9920,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:29:04,441] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9920,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:29:04,441] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9924,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:29:04,441] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9924,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:29:04,453] jepsen worker 69 - jepsen.util 1599 :info :write [23 48] :RequestCanceledException INFO [2019-12-17 12:29:04,454] jepsen worker 69 - jepsen.util 1689 :invoke :write [23 34] INFO [2019-12-17 12:29:04,461] jepsen worker 64 - jepsen.util 1504 :ok :read [21 21] INFO [2019-12-17 12:29:04,461] jepsen worker 64 - jepsen.util 1504 :invoke :read [21 nil] INFO [2019-12-17 12:29:04,463] jepsen worker 45 - jepsen.util 2025 :ok :write [15 7] INFO [2019-12-17 12:29:04,463] jepsen worker 45 - jepsen.util 2025 :invoke :write [15 5] INFO [2019-12-17 12:29:04,471] jepsen worker 63 - jepsen.util 1503 :ok :write [21 3] INFO [2019-12-17 12:29:04,472] jepsen worker 63 - jepsen.util 1503 :invoke :read [21 nil] INFO [2019-12-17 12:29:04,478] jepsen worker 64 - jepsen.util 1504 :ok :read [21 21] INFO [2019-12-17 12:29:04,478] jepsen worker 64 - jepsen.util 1504 :invoke :write [21 49] INFO [2019-12-17 12:29:04,502] jepsen worker 63 - jepsen.util 1503 :ok :read [21 3] INFO [2019-12-17 12:29:04,503] jepsen worker 63 - jepsen.util 1503 :invoke :write [21 2] INFO [2019-12-17 12:29:04,506] jepsen worker 46 - jepsen.util 1756 :info :write [15 21] :RequestCanceledException INFO [2019-12-17 12:29:04,507] jepsen worker 46 - jepsen.util 1846 :invoke :read [15 nil] INFO [2019-12-17 12:29:04,545] jepsen worker 64 - jepsen.util 1504 :ok :write [21 49] INFO [2019-12-17 12:29:04,545] jepsen worker 64 - jepsen.util 1504 :invoke :write [21 17] INFO [2019-12-17 12:29:04,550] jepsen worker 46 - jepsen.util 1846 :ok :read [15 7] INFO [2019-12-17 12:29:04,550] jepsen worker 46 - jepsen.util 1846 :invoke :cas [15 [41 22]] INFO [2019-12-17 12:29:04,578] jepsen worker 46 - jepsen.util 1846 :fail :cas [15 [41 22]] :ValueNotSwapFrom INFO [2019-12-17 12:29:04,578] jepsen worker 46 - jepsen.util 1846 :invoke :read [15 nil] INFO [2019-12-17 12:29:04,606] jepsen worker 46 - jepsen.util 1846 :ok :read [15 7] INFO [2019-12-17 12:29:04,606] jepsen worker 46 - jepsen.util 1846 :invoke :cas [15 [27 34]] INFO [2019-12-17 12:29:04,621] jepsen worker 64 - jepsen.util 1504 :ok :write [21 17] INFO [2019-12-17 12:29:04,621] jepsen worker 64 - jepsen.util 1504 :invoke :cas [21 [6 19]] INFO [2019-12-17 12:29:04,629] jepsen worker 46 - jepsen.util 1846 :fail :cas [15 [27 34]] :ValueNotSwapFrom INFO [2019-12-17 12:29:04,629] jepsen worker 46 - jepsen.util 1846 :invoke :read [15 nil] INFO [2019-12-17 12:29:04,651] jepsen worker 70 - jepsen.util 1960 :ok :read [23 44] INFO [2019-12-17 12:29:04,651] jepsen worker 70 - jepsen.util 1960 :invoke :read [23 nil] INFO [2019-12-17 12:29:04,660] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9941,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:29:04,660] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9941,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:29:04,661] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9945,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:29:04,661] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9945,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:29:04,665] jepsen worker 64 - jepsen.util 1504 :fail :cas [21 [6 19]] :ValueNotSwapFrom INFO [2019-12-17 12:29:04,666] jepsen worker 64 - jepsen.util 1504 :invoke :cas [21 [31 0]] INFO [2019-12-17 12:29:04,666] jepsen worker 46 - jepsen.util 1846 :ok :read [15 7] INFO [2019-12-17 12:29:04,666] jepsen worker 46 - jepsen.util 1846 :invoke :cas [15 [46 9]] INFO [2019-12-17 12:29:04,680] jepsen worker 64 - jepsen.util 1504 :fail :cas [21 [31 0]] :ValueNotSwapFrom INFO [2019-12-17 12:29:04,680] jepsen worker 64 - jepsen.util 1504 :invoke :cas [21 [0 40]] INFO [2019-12-17 12:29:04,681] jepsen worker 46 - jepsen.util 1846 :fail :cas [15 [46 9]] :ValueNotSwapFrom INFO [2019-12-17 12:29:04,681] jepsen worker 46 - jepsen.util 1846 :invoke :read [15 nil] INFO [2019-12-17 12:29:04,705] jepsen worker 46 - jepsen.util 1846 :ok :read [15 7] INFO [2019-12-17 12:29:04,705] jepsen worker 46 - jepsen.util 1846 :invoke :write [15 34] INFO [2019-12-17 12:29:04,707] jepsen worker 64 - jepsen.util 1504 :fail :cas [21 [0 40]] :ValueNotSwapFrom INFO [2019-12-17 12:29:04,707] jepsen worker 64 - jepsen.util 1504 :invoke :read [21 nil] INFO [2019-12-17 12:29:04,718] jepsen worker 12 - jepsen.util 1812 :ok :read [4 38] INFO [2019-12-17 12:29:04,718] jepsen worker 12 - jepsen.util 1812 :invoke :write [4 11] INFO [2019-12-17 12:29:04,734] jepsen worker 88 - jepsen.util 1618 :info :write [29 40] :RequestCanceledException INFO [2019-12-17 12:29:04,736] jepsen worker 88 - jepsen.util 1708 :invoke :cas [29 [9 33]] INFO [2019-12-17 12:29:04,737] jepsen worker 64 - jepsen.util 1504 :ok :read [21 17] INFO [2019-12-17 12:29:04,737] jepsen worker 64 - jepsen.util 1504 :invoke :cas [21 [8 4]] INFO [2019-12-17 12:29:04,754] jepsen worker 64 - jepsen.util 1504 :fail :cas [21 [8 4]] :ValueNotSwapFrom INFO [2019-12-17 12:29:04,754] jepsen worker 64 - jepsen.util 1504 :invoke :write [21 44] INFO [2019-12-17 12:29:04,765] jepsen worker 52 - jepsen.util 1672 :fail :cas [17 [19 11]] :ValueNotSwapFrom INFO [2019-12-17 12:29:04,765] jepsen worker 52 - jepsen.util 1672 :invoke :write [17 9] INFO [2019-12-17 12:29:04,790] jepsen worker 36 - jepsen.util 1566 :ok :read [12 38] INFO [2019-12-17 12:29:04,790] jepsen worker 36 - jepsen.util 1566 :invoke :write [12 14] INFO [2019-12-17 12:29:04,795] jepsen worker 64 - jepsen.util 1504 :ok :write [21 44] INFO [2019-12-17 12:29:04,795] jepsen worker 64 - jepsen.util 1504 :invoke :write [21 30] INFO [2019-12-17 12:29:04,815] jepsen worker 45 - jepsen.util 2025 :ok :write [15 5] INFO [2019-12-17 12:29:04,815] jepsen worker 45 - jepsen.util 2025 :invoke :write [15 48] INFO [2019-12-17 12:29:04,815] jepsen worker 10 - jepsen.util 1900 :ok :write [3 33] INFO [2019-12-17 12:29:04,815] jepsen worker 10 - jepsen.util 1900 :invoke :write [3 23] INFO [2019-12-17 12:29:04,845] jepsen worker 64 - jepsen.util 1504 :ok :write [21 30] INFO [2019-12-17 12:29:04,845] jepsen worker 64 - jepsen.util 1504 :invoke :write [21 20] INFO [2019-12-17 12:29:04,852] jepsen worker 32 - jepsen.util 2012 :info :cas [10 [40 38]] :RequestCanceledException INFO [2019-12-17 12:29:04,853] jepsen worker 32 - jepsen.util 2102 :invoke :read [10 nil] INFO [2019-12-17 12:29:04,880] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9966,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:29:04,880] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9966,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:29:04,880] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9969,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:29:04,880] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9969,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:29:04,886] jepsen worker 53 - jepsen.util 1583 :fail :cas [17 [17 12]] :ValueNotSwapFrom INFO [2019-12-17 12:29:04,886] jepsen worker 53 - jepsen.util 1583 :invoke :read [17 nil] INFO [2019-12-17 12:29:04,897] jepsen worker 32 - jepsen.util 2102 :ok :read [10 0] INFO [2019-12-17 12:29:04,897] jepsen worker 32 - jepsen.util 2102 :invoke :cas [10 [6 6]] INFO [2019-12-17 12:29:04,911] jepsen worker 64 - jepsen.util 1504 :ok :write [21 20] INFO [2019-12-17 12:29:04,912] jepsen worker 64 - jepsen.util 1504 :invoke :read [21 nil] INFO [2019-12-17 12:29:04,913] jepsen worker 32 - jepsen.util 2102 :fail :cas [10 [6 6]] :ValueNotSwapFrom INFO [2019-12-17 12:29:04,913] jepsen worker 32 - jepsen.util 2102 :invoke :write [10 16] INFO [2019-12-17 12:29:04,931] jepsen worker 5 - jepsen.util 1355 :fail :cas [1 [10 0]] :ValueNotSwapFrom INFO [2019-12-17 12:29:04,931] jepsen worker 5 - jepsen.util 1355 :invoke :read [1 nil] INFO [2019-12-17 12:29:04,937] jepsen worker 64 - jepsen.util 1504 :ok :read [21 20] INFO [2019-12-17 12:29:04,937] jepsen worker 64 - jepsen.util 1504 :invoke :write [21 36] INFO [2019-12-17 12:29:05,005] jepsen worker 64 - jepsen.util 1504 :ok :write [21 36] INFO [2019-12-17 12:29:05,008] jepsen worker 64 - jepsen.util 1504 :invoke :read [21 nil] INFO [2019-12-17 12:29:05,021] jepsen worker 64 - jepsen.util 1504 :ok :read [21 36] INFO [2019-12-17 12:29:05,021] jepsen worker 64 - jepsen.util 1504 :invoke :write [21 33] INFO [2019-12-17 12:29:05,035] jepsen worker 88 - jepsen.util 1708 :fail :cas [29 [9 33]] :ValueNotSwapFrom INFO [2019-12-17 12:29:05,035] jepsen worker 88 - jepsen.util 1708 :invoke :read [29 nil] INFO [2019-12-17 12:29:05,036] jepsen worker 15 - jepsen.util 2085 :info :cas [5 [23 3]] :RequestCanceledException INFO [2019-12-17 12:29:05,037] jepsen worker 15 - jepsen.util 2175 :invoke :read [5 nil] INFO [2019-12-17 12:29:05,065] jepsen worker 64 - jepsen.util 1504 :ok :write [21 33] INFO [2019-12-17 12:29:05,065] jepsen worker 64 - jepsen.util 1504 :invoke :cas [21 [25 8]] INFO [2019-12-17 12:29:05,078] jepsen worker 64 - jepsen.util 1504 :fail :cas [21 [25 8]] :ValueNotSwapFrom INFO [2019-12-17 12:29:05,078] jepsen worker 64 - jepsen.util 1504 :invoke :read [21 nil] INFO [2019-12-17 12:29:05,089] jepsen worker 64 - jepsen.util 1504 :ok :read [21 33] INFO [2019-12-17 12:29:05,089] jepsen worker 64 - jepsen.util 1504 :invoke :read [21 nil] INFO [2019-12-17 12:29:05,097] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9978,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:29:05,097] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9978,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:29:05,097] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9982,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:29:05,097] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":9982,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:29:05,101] jepsen worker 53 - jepsen.util 1583 :ok :read [17 2] INFO [2019-12-17 12:29:05,101] jepsen worker 53 - jepsen.util 1583 :invoke :cas [17 [6 30]] INFO [2019-12-17 12:29:05,103] jepsen worker 64 - jepsen.util 1504 :ok :read [21 33] INFO [2019-12-17 12:29:05,103] jepsen worker 64 - jepsen.util 1504 :invoke :cas [21 [46 27]] INFO [2019-12-17 12:29:05,105] jepsen worker 16 - jepsen.util 2086 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:29:05,105] jepsen worker 38 - jepsen.util 1748 :info :write [12 16] :RequestCanceledException INFO [2019-12-17 12:29:05,105] jepsen worker 16 - jepsen.util 2086 :invoke :write [5 6] INFO [2019-12-17 12:29:05,106] jepsen worker 38 - jepsen.util 1838 :invoke :read [12 nil] INFO [2019-12-17 12:29:05,119] jepsen worker 64 - jepsen.util 1504 :fail :cas [21 [46 27]] :ValueNotSwapFrom INFO [2019-12-17 12:29:05,119] jepsen worker 64 - jepsen.util 1504 :invoke :write [21 25] INFO [2019-12-17 12:29:05,122] jepsen worker 26 - jepsen.util 2456 :info :write [8 9] :RequestCanceledException INFO [2019-12-17 12:29:05,122] jepsen worker 84 - jepsen.util 2154 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:29:05,122] jepsen worker 43 - jepsen.util 2293 :info :cas [14 [39 22]] :RequestCanceledException INFO [2019-12-17 12:29:05,122] jepsen worker 84 - jepsen.util 2154 :invoke :read [28 nil] INFO [2019-12-17 12:29:05,124] jepsen worker 26 - jepsen.util 2546 :invoke :cas [8 [11 18]] INFO [2019-12-17 12:29:05,124] jepsen worker 43 - jepsen.util 2383 :invoke :cas [14 [16 5]] INFO [2019-12-17 12:29:05,136] jepsen worker 61 - jepsen.util 2401 :info :cas [20 [46 19]] :RequestCanceledException INFO [2019-12-17 12:29:05,136] jepsen worker 86 - jepsen.util 2426 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:29:05,136] jepsen worker 86 - jepsen.util 2426 :invoke :write [28 43] INFO [2019-12-17 12:29:05,137] jepsen worker 61 - jepsen.util 2491 :invoke :read [20 nil] INFO [2019-12-17 12:29:05,142] jepsen worker 70 - jepsen.util 1960 :ok :read [23 44] INFO [2019-12-17 12:29:05,142] jepsen worker 70 - jepsen.util 1960 :invoke :write [23 13] INFO [2019-12-17 12:29:05,153] jepsen worker 76 - jepsen.util 2326 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:29:05,153] jepsen worker 58 - jepsen.util 2038 :info :write [19 0] :RequestCanceledException INFO [2019-12-17 12:29:05,153] jepsen worker 54 - jepsen.util 1674 :info :write [18 5] :RequestCanceledException INFO [2019-12-17 12:29:05,153] jepsen worker 21 - jepsen.util 2271 :info :cas [7 [14 11]] :RequestCanceledException INFO [2019-12-17 12:29:05,153] jepsen worker 81 - jepsen.util 1971 :info :write [27 23] :RequestCanceledException INFO [2019-12-17 12:29:05,153] jepsen worker 76 - jepsen.util 2326 :invoke :cas [25 [22 41]] INFO [2019-12-17 12:29:05,155] jepsen worker 54 - jepsen.util 1764 :invoke :cas [18 [41 40]] INFO [2019-12-17 12:29:05,155] jepsen worker 81 - jepsen.util 2061 :invoke :read [27 nil] INFO [2019-12-17 12:29:05,155] jepsen worker 21 - jepsen.util 2361 :invoke :read [7 nil] INFO [2019-12-17 12:29:05,155] jepsen worker 58 - jepsen.util 2128 :invoke :cas [19 [1 21]] INFO [2019-12-17 12:29:05,158] jepsen worker 88 - jepsen.util 1708 :ok :read [29 21] INFO [2019-12-17 12:29:05,158] jepsen worker 88 - jepsen.util 1708 :invoke :write [29 45] INFO [2019-12-17 12:29:05,164] jepsen worker 64 - jepsen.util 1504 :ok :write [21 25] INFO [2019-12-17 12:29:05,164] jepsen worker 64 - jepsen.util 1504 :invoke :read [21 nil] INFO [2019-12-17 12:29:05,183] jepsen worker 64 - jepsen.util 1504 :ok :read [21 25] INFO [2019-12-17 12:29:05,183] jepsen worker 64 - jepsen.util 1504 :invoke :read [21 nil] INFO [2019-12-17 12:29:05,189] jepsen worker 62 - jepsen.util 2222 :info :cas [20 [49 18]] :RequestCanceledException INFO [2019-12-17 12:29:05,189] jepsen worker 34 - jepsen.util 2374 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:29:05,189] jepsen worker 25 - jepsen.util 2545 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:29:05,189] jepsen worker 35 - jepsen.util 2015 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:29:05,189] jepsen worker 77 - jepsen.util 2237 :info :cas [25 [47 23]] :RequestCanceledException INFO [2019-12-17 12:29:05,189] jepsen worker 34 - jepsen.util 2374 :invoke :cas [11 [1 27]] INFO [2019-12-17 12:29:05,189] jepsen worker 25 - jepsen.util 2545 :invoke :cas [8 [11 15]] INFO [2019-12-17 12:29:05,189] jepsen worker 35 - jepsen.util 2015 :invoke :cas [11 [4 11]] INFO [2019-12-17 12:29:05,190] jepsen worker 62 - jepsen.util 2312 :invoke :read [20 nil] INFO [2019-12-17 12:29:05,190] jepsen worker 77 - jepsen.util 2327 :invoke :write [25 29] INFO [2019-12-17 12:29:05,195] jepsen worker 64 - jepsen.util 1504 :ok :read [21 25] INFO [2019-12-17 12:29:05,195] jepsen worker 64 - jepsen.util 1504 :invoke :write [21 29] INFO [2019-12-17 12:29:05,206] jepsen worker 10 - jepsen.util 1900 :ok :write [3 23] INFO [2019-12-17 12:29:05,206] jepsen worker 45 - jepsen.util 2025 :ok :write [15 48] INFO [2019-12-17 12:29:05,206] jepsen worker 24 - jepsen.util 2094 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:29:05,206] jepsen worker 23 - jepsen.util 2003 :info :write [7 11] :RequestCanceledException INFO [2019-12-17 12:29:05,206] jepsen worker 57 - jepsen.util 2037 :info :write [19 29] :RequestCanceledException INFO [2019-12-17 12:29:05,206] jepsen worker 33 - jepsen.util 2643 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:29:05,206] jepsen worker 10 - jepsen.util 1900 :invoke :cas [3 [41 2]] INFO [2019-12-17 12:29:05,206] jepsen worker 40 - jepsen.util 2650 :info :write [13 5] :RequestCanceledException INFO [2019-12-17 12:29:05,206] jepsen worker 45 - jepsen.util 2025 :invoke :read [15 nil] INFO [2019-12-17 12:29:05,206] jepsen worker 24 - jepsen.util 2094 :invoke :read [8 nil] INFO [2019-12-17 12:29:05,206] jepsen worker 33 - jepsen.util 2643 :invoke :cas [11 [15 21]] INFO [2019-12-17 12:29:05,207] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":10010,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:29:05,207] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":10010,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:29:05,207] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":10011,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:29:05,207] jepsen worker 57 - jepsen.util 2127 :invoke :cas [19 [21 8]] INFO [2019-12-17 12:29:05,207] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":10011,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:29:05,207] jepsen worker 23 - jepsen.util 2093 :invoke :cas [7 [19 1]] INFO [2019-12-17 12:29:05,208] jepsen worker 40 - jepsen.util 2740 :invoke :write [13 3] INFO [2019-12-17 12:29:05,216] jepsen worker 32 - jepsen.util 2102 :ok :write [10 16] INFO [2019-12-17 12:29:05,216] jepsen worker 32 - jepsen.util 2102 :invoke :read [10 nil] INFO [2019-12-17 12:29:05,221] jepsen worker 1 - jepsen.util 2341 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:29:05,221] jepsen worker 66 - jepsen.util 1956 :info :write [22 10] :RequestCanceledException INFO [2019-12-17 12:29:05,221] jepsen worker 13 - jepsen.util 1453 :info :write [4 1] :RequestCanceledException INFO [2019-12-17 12:29:05,222] jepsen worker 1 - jepsen.util 2341 :invoke :read [0 nil] INFO [2019-12-17 12:29:05,222] jepsen worker 13 - jepsen.util 1543 :invoke :read [4 nil] INFO [2019-12-17 12:29:05,222] jepsen worker 66 - jepsen.util 2046 :invoke :write [22 25] INFO [2019-12-17 12:29:05,223] jepsen worker 22 - jepsen.util 2362 :info :write [7 21] :RequestCanceledException INFO [2019-12-17 12:29:05,223] jepsen worker 50 - jepsen.util 2390 :info :cas [16 [8 11]] :RequestCanceledException INFO [2019-12-17 12:29:05,224] jepsen worker 22 - jepsen.util 2452 :invoke :write [7 25] INFO [2019-12-17 12:29:05,224] jepsen worker 50 - jepsen.util 2480 :invoke :write [16 48] INFO [2019-12-17 12:29:05,236] jepsen worker 82 - jepsen.util 2692 :info :cas [27 [45 30]] :RequestCanceledException INFO [2019-12-17 12:29:05,236] jepsen worker 85 - jepsen.util 2515 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:29:05,236] jepsen worker 85 - jepsen.util 2515 :invoke :cas [28 [22 24]] INFO [2019-12-17 12:29:05,237] jepsen worker 82 - jepsen.util 2782 :invoke :read [27 nil] INFO [2019-12-17 12:29:05,246] jepsen worker 32 - jepsen.util 2102 :ok :read [10 16] INFO [2019-12-17 12:29:05,247] jepsen worker 32 - jepsen.util 2102 :invoke :read [10 nil] INFO [2019-12-17 12:29:05,253] jepsen worker 8 - jepsen.util 2078 :info :cas [2 [28 7]] :RequestCanceledException INFO [2019-12-17 12:29:05,253] jepsen worker 0 - jepsen.util 2340 :info :write [0 11] :RequestCanceledException INFO [2019-12-17 12:29:05,253] jepsen worker 17 - jepsen.util 2267 :info :write [5 42] :RequestCanceledException INFO [2019-12-17 12:29:05,253] jepsen worker 68 - jepsen.util 1958 :info :cas [22 [3 39]] :RequestCanceledException INFO [2019-12-17 12:29:05,254] jepsen worker 8 - jepsen.util 2168 :invoke :read [2 nil] INFO [2019-12-17 12:29:05,254] jepsen worker 0 - jepsen.util 2430 :invoke :write [0 44] INFO [2019-12-17 12:29:05,254] jepsen worker 17 - jepsen.util 2357 :invoke :write [5 32] INFO [2019-12-17 12:29:05,254] jepsen worker 68 - jepsen.util 2048 :invoke :read [22 nil] INFO [2019-12-17 12:29:05,258] jepsen worker 80 - jepsen.util 1790 :ok :write [26 9] INFO [2019-12-17 12:29:05,258] jepsen worker 80 - jepsen.util 1790 :invoke :write [26 42] INFO [2019-12-17 12:29:05,278] jepsen worker 10 - jepsen.util 1900 :fail :cas [3 [41 2]] :ValueNotSwapFrom INFO [2019-12-17 12:29:05,278] jepsen worker 10 - jepsen.util 1900 :invoke :read [3 nil] INFO [2019-12-17 12:29:05,284] jepsen worker 45 - jepsen.util 2025 :ok :read [15 48] INFO [2019-12-17 12:29:05,284] jepsen worker 45 - jepsen.util 2025 :invoke :cas [15 [7 0]] INFO [2019-12-17 12:29:05,285] jepsen worker 32 - jepsen.util 2102 :ok :read [10 16] INFO [2019-12-17 12:29:05,285] jepsen worker 32 - jepsen.util 2102 :invoke :write [10 42] INFO [2019-12-17 12:29:05,287] jepsen worker 59 - jepsen.util 1949 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:29:05,287] jepsen worker 39 - jepsen.util 2469 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:29:05,288] jepsen worker 59 - jepsen.util 1949 :invoke :cas [19 [34 1]] INFO [2019-12-17 12:29:05,288] jepsen worker 39 - jepsen.util 2469 :invoke :cas [13 [4 29]] INFO [2019-12-17 12:29:05,289] jepsen worker 37 - jepsen.util 1387 :ok :write [12 38] INFO [2019-12-17 12:29:05,289] jepsen worker 37 - jepsen.util 1387 :invoke :write [12 44] INFO [2019-12-17 12:29:05,317] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":10028,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:29:05,317] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":10028,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:29:05,317] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":10031,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:29:05,317] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":10031,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:29:05,321] jepsen worker 41 - jepsen.util 2381 :info :cas [13 [20 27]] :RequestCanceledException INFO [2019-12-17 12:29:05,321] jepsen worker 83 - jepsen.util 2513 :info :cas [27 [6 38]] :RequestCanceledException INFO [2019-12-17 12:29:05,321] jepsen worker 42 - jepsen.util 2652 :info :write [14 10] :RequestCanceledException INFO [2019-12-17 12:29:05,321] jepsen worker 89 - jepsen.util 1619 :info :write [29 49] :RequestCanceledException INFO [2019-12-17 12:29:05,328] jepsen worker 41 - jepsen.util 2471 :invoke :cas [13 [16 19]] INFO [2019-12-17 12:29:05,328] jepsen worker 83 - jepsen.util 2603 :invoke :cas [27 [21 11]] INFO [2019-12-17 12:29:05,328] jepsen worker 42 - jepsen.util 2742 :invoke :cas [14 [12 14]] INFO [2019-12-17 12:29:05,328] jepsen worker 89 - jepsen.util 1709 :invoke :read [29 nil] INFO [2019-12-17 12:29:05,328] jepsen worker 10 - jepsen.util 1900 :ok :read [3 23] INFO [2019-12-17 12:29:05,328] jepsen worker 10 - jepsen.util 1900 :invoke :read [3 nil] INFO [2019-12-17 12:29:05,332] jepsen worker 45 - jepsen.util 2025 :fail :cas [15 [7 0]] :ValueNotSwapFrom INFO [2019-12-17 12:29:05,332] jepsen worker 45 - jepsen.util 2025 :invoke :cas [15 [2 19]] INFO [2019-12-17 12:29:05,352] jepsen worker 52 - jepsen.util 1672 :ok :write [17 9] INFO [2019-12-17 12:29:05,353] jepsen worker 52 - jepsen.util 1672 :invoke :write [17 35] INFO [2019-12-17 12:29:05,353] jepsen worker 87 - jepsen.util 1437 :info :write [29 19] :RequestCanceledException INFO [2019-12-17 12:29:05,355] jepsen worker 87 - jepsen.util 1527 :invoke :cas [29 [40 4]] INFO [2019-12-17 12:29:05,357] jepsen worker 64 - jepsen.util 1504 :ok :write [21 29] INFO [2019-12-17 12:29:05,358] jepsen worker 64 - jepsen.util 1504 :invoke :read [21 nil] INFO [2019-12-17 12:29:05,363] jepsen worker 10 - jepsen.util 1900 :ok :read [3 23] INFO [2019-12-17 12:29:05,364] jepsen worker 10 - jepsen.util 1900 :invoke :read [3 nil] INFO [2019-12-17 12:29:05,366] jepsen worker 45 - jepsen.util 2025 :fail :cas [15 [2 19]] :ValueNotSwapFrom INFO [2019-12-17 12:29:05,366] jepsen worker 45 - jepsen.util 2025 :invoke :read [15 nil] INFO [2019-12-17 12:29:05,407] jepsen worker 9 - jepsen.util 1809 :info :cas [3 [13 36]] :RequestCanceledException INFO [2019-12-17 12:29:05,410] jepsen worker 9 - jepsen.util 1899 :invoke :cas [3 [24 18]] INFO [2019-12-17 12:29:05,417] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:29:05,417] jepsen nemesis - couchbase.util Rebalance check count: 70 INFO [2019-12-17 12:29:05,417] jepsen nemesis - couchbase.util Rebalance stuck count: 51 INFO [2019-12-17 12:29:05,425] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":10068,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:29:05,426] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":10068,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:29:05,426] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":10071,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:29:05,426] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":10071,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:29:05,427] jepsen worker 31 - jepsen.util 1741 :fail :read [10 nil] :RequestTimeoutException INFO [2019-12-17 12:29:05,427] jepsen worker 31 - jepsen.util 1741 :invoke :write [10 4] INFO [2019-12-17 12:29:05,428] jepsen worker 30 - jepsen.util 2010 :fail :read [10 nil] :RequestTimeoutException INFO [2019-12-17 12:29:05,428] jepsen worker 30 - jepsen.util 2010 :invoke :read [10 nil] INFO [2019-12-17 12:29:05,428] jepsen worker 10 - jepsen.util 1900 :ok :read [3 23] INFO [2019-12-17 12:29:05,428] jepsen worker 64 - jepsen.util 1504 :ok :read [21 29] INFO [2019-12-17 12:29:05,428] jepsen worker 10 - jepsen.util 1900 :invoke :cas [3 [8 0]] INFO [2019-12-17 12:29:05,428] jepsen worker 64 - jepsen.util 1504 :invoke :read [21 nil] INFO [2019-12-17 12:29:05,436] jepsen worker 45 - jepsen.util 2025 :ok :read [15 48] INFO [2019-12-17 12:29:05,436] jepsen worker 45 - jepsen.util 2025 :invoke :cas [15 [34 33]] INFO [2019-12-17 12:29:05,453] jepsen worker 48 - jepsen.util 1848 :info :cas [16 [13 3]] :RequestCanceledException INFO [2019-12-17 12:29:05,455] jepsen worker 48 - jepsen.util 1938 :invoke :cas [16 [31 38]] INFO [2019-12-17 12:29:05,468] jepsen worker 9 - jepsen.util 1899 :fail :cas [3 [24 18]] :ValueNotSwapFrom INFO [2019-12-17 12:29:05,468] jepsen worker 9 - jepsen.util 1899 :invoke :cas [3 [6 11]] INFO [2019-12-17 12:29:05,488] jepsen worker 67 - jepsen.util 2677 :info :cas [22 [10 39]] :RequestCanceledException INFO [2019-12-17 12:29:05,490] jepsen worker 67 - jepsen.util 2767 :invoke :read [22 nil] INFO [2019-12-17 12:29:05,501] jepsen worker 36 - jepsen.util 1566 :ok :write [12 14] INFO [2019-12-17 12:29:05,501] jepsen worker 36 - jepsen.util 1566 :invoke :write [12 38] INFO [2019-12-17 12:29:05,535] jepsen worker 30 - jepsen.util 2010 :ok :read [10 16] INFO [2019-12-17 12:29:05,535] jepsen worker 30 - jepsen.util 2010 :invoke :write [10 9] INFO [2019-12-17 12:29:05,536] jepsen worker 74 - jepsen.util 1694 :info :write [24 34] :RequestCanceledException INFO [2019-12-17 12:29:05,537] jepsen worker 74 - jepsen.util 1784 :invoke :cas [24 [41 15]] INFO [2019-12-17 12:29:05,576] jepsen worker 10 - jepsen.util 1900 :fail :cas [3 [8 0]] :ValueNotSwapFrom INFO [2019-12-17 12:29:05,576] jepsen worker 45 - jepsen.util 2025 :fail :cas [15 [34 33]] :ValueNotSwapFrom INFO [2019-12-17 12:29:05,576] jepsen worker 10 - jepsen.util 1900 :invoke :read [3 nil] INFO [2019-12-17 12:29:05,576] jepsen worker 45 - jepsen.util 2025 :invoke :read [15 nil] INFO [2019-12-17 12:29:05,578] jepsen worker 64 - jepsen.util 1504 :ok :read [21 29] INFO [2019-12-17 12:29:05,579] jepsen worker 64 - jepsen.util 1504 :invoke :read [21 nil] INFO [2019-12-17 12:29:05,582] jepsen worker 9 - jepsen.util 1899 :fail :cas [3 [6 11]] :ValueNotSwapFrom INFO [2019-12-17 12:29:05,583] jepsen worker 9 - jepsen.util 1899 :invoke :read [3 nil] INFO [2019-12-17 12:29:05,584] jepsen worker 3 - jepsen.util 1623 :ok :write [1 13] INFO [2019-12-17 12:29:05,585] jepsen worker 3 - jepsen.util 1623 :invoke :cas [1 [8 3]] INFO [2019-12-17 12:29:05,601] jepsen worker 79 - jepsen.util 1789 :ok :write [26 30] INFO [2019-12-17 12:29:05,601] jepsen worker 79 - jepsen.util 1789 :invoke :read [26 nil] INFO [2019-12-17 12:29:05,602] jepsen worker 74 - jepsen.util 1784 :fail :cas [24 [41 15]] :ValueNotSwapFrom INFO [2019-12-17 12:29:05,602] jepsen worker 74 - jepsen.util 1784 :invoke :read [24 nil] INFO [2019-12-17 12:29:05,606] jepsen worker 47 - jepsen.util 2117 :info :write [15 33] :RequestCanceledException INFO [2019-12-17 12:29:05,606] jepsen worker 65 - jepsen.util 1415 :info :write [21 4] :RequestCanceledException INFO [2019-12-17 12:29:05,607] jepsen worker 47 - jepsen.util 2207 :invoke :write [15 22] INFO [2019-12-17 12:29:05,607] jepsen worker 65 - jepsen.util 1505 :invoke :cas [21 [47 11]] INFO [2019-12-17 12:29:05,610] jepsen worker 88 - jepsen.util 1708 :ok :write [29 45] INFO [2019-12-17 12:29:05,610] jepsen worker 88 - jepsen.util 1708 :invoke :read [29 nil] INFO [2019-12-17 12:29:05,617] jepsen worker 53 - jepsen.util 1583 :fail :cas [17 [6 30]] :ValueNotSwapFrom INFO [2019-12-17 12:29:05,617] jepsen worker 53 - jepsen.util 1583 :invoke :write [17 43] INFO [2019-12-17 12:29:05,623] jepsen worker 27 - jepsen.util 1377 :info :write [9 32] :RequestCanceledException INFO [2019-12-17 12:29:05,623] jepsen worker 5 - jepsen.util 1355 :ok :read [1 13] INFO [2019-12-17 12:29:05,623] jepsen worker 5 - jepsen.util 1355 :invoke :cas [1 [8 29]] INFO [2019-12-17 12:29:05,624] jepsen worker 27 - jepsen.util 1467 :invoke :read [9 nil] INFO [2019-12-17 12:29:05,636] jepsen worker 28 - jepsen.util 1108 :fail :read [9 nil] :RequestTimeoutException INFO [2019-12-17 12:29:05,636] jepsen worker 11 - jepsen.util 2081 :info :write [3 49] :RequestCanceledException INFO [2019-12-17 12:29:05,636] jepsen worker 28 - jepsen.util 1108 :invoke :cas [9 [30 48]] INFO [2019-12-17 12:29:05,637] jepsen worker 11 - jepsen.util 2171 :invoke :write [3 15] INFO [2019-12-17 12:29:05,645] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":10101,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:29:05,645] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":10101,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:29:05,666] jepsen worker 64 - jepsen.util 1504 :ok :read [21 29] INFO [2019-12-17 12:29:05,666] jepsen worker 64 - jepsen.util 1504 :invoke :write [21 7] INFO [2019-12-17 12:29:05,675] jepsen worker 54 - jepsen.util 1764 :fail :cas [18 [41 40]] :ValueNotSwapFrom INFO [2019-12-17 12:29:05,675] jepsen worker 54 - jepsen.util 1764 :invoke :write [18 10] INFO [2019-12-17 12:29:05,687] jepsen worker 45 - jepsen.util 2025 :ok :read [15 48] INFO [2019-12-17 12:29:05,687] jepsen worker 45 - jepsen.util 2025 :invoke :write [15 38] INFO [2019-12-17 12:29:05,692] jepsen worker 65 - jepsen.util 1505 :fail :cas [21 [47 11]] :ValueNotSwapFrom INFO [2019-12-17 12:29:05,692] jepsen worker 65 - jepsen.util 1505 :invoke :read [21 nil] INFO [2019-12-17 12:29:05,706] jepsen worker 29 - jepsen.util 1469 :fail :read [9 nil] :RequestTimeoutException INFO [2019-12-17 12:29:05,706] jepsen worker 2 - jepsen.util 2072 :info :cas [0 [25 16]] :RequestCanceledException INFO [2019-12-17 12:29:05,706] jepsen worker 29 - jepsen.util 1469 :invoke :cas [9 [49 14]] INFO [2019-12-17 12:29:05,707] jepsen worker 2 - jepsen.util 2162 :invoke :read [0 nil] INFO [2019-12-17 12:29:05,710] jepsen worker 10 - jepsen.util 1900 :ok :read [3 23] INFO [2019-12-17 12:29:05,710] jepsen worker 10 - jepsen.util 1900 :invoke :write [3 17] INFO [2019-12-17 12:29:05,722] jepsen worker 18 - jepsen.util 1908 :info :write [6 19] :RequestCanceledException INFO [2019-12-17 12:29:05,724] jepsen worker 18 - jepsen.util 1998 :invoke :read [6 nil] INFO [2019-12-17 12:29:05,731] jepsen worker 37 - jepsen.util 1387 :ok :write [12 44] INFO [2019-12-17 12:29:05,731] jepsen worker 37 - jepsen.util 1387 :invoke :write [12 34] INFO [2019-12-17 12:29:05,738] jepsen worker 9 - jepsen.util 1899 :ok :read [3 23] INFO [2019-12-17 12:29:05,738] jepsen worker 9 - jepsen.util 1899 :invoke :write [3 40] INFO [2019-12-17 12:29:05,742] jepsen worker 14 - jepsen.util 1724 :ok :write [4 28] INFO [2019-12-17 12:29:05,743] jepsen worker 14 - jepsen.util 1724 :invoke :cas [4 [24 22]] INFO [2019-12-17 12:29:05,751] jepsen worker 20 - jepsen.util 1550 :info :write [6 49] :RequestCanceledException INFO [2019-12-17 12:29:05,752] jepsen worker 20 - jepsen.util 1640 :invoke :read [6 nil] INFO [2019-12-17 12:29:05,799] jepsen worker 74 - jepsen.util 1784 :ok :read [24 32] INFO [2019-12-17 12:29:05,799] jepsen worker 74 - jepsen.util 1784 :invoke :cas [24 [0 39]] INFO [2019-12-17 12:29:05,807] jepsen worker 49 - jepsen.util 1579 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:29:05,808] jepsen worker 49 - jepsen.util 1579 :invoke :cas [16 [23 3]] INFO [2019-12-17 12:29:05,808] jepsen worker 28 - jepsen.util 1108 :fail :cas [9 [30 48]] :ValueNotSwapFrom INFO [2019-12-17 12:29:05,809] jepsen worker 28 - jepsen.util 1108 :invoke :cas [9 [11 48]] INFO [2019-12-17 12:29:05,821] jepsen worker 51 - jepsen.util 1671 :info :write [17 43] :RequestCanceledException INFO [2019-12-17 12:29:05,821] jepsen worker 44 - jepsen.util 1844 :info :cas [14 [21 37]] :RequestCanceledException INFO [2019-12-17 12:29:05,823] jepsen worker 44 - jepsen.util 1934 :invoke :cas [14 [42 9]] INFO [2019-12-17 12:29:05,823] jepsen worker 51 - jepsen.util 1761 :invoke :write [17 18] INFO [2019-12-17 12:29:05,849] jepsen worker 65 - jepsen.util 1505 :ok :read [21 29] INFO [2019-12-17 12:29:05,850] jepsen worker 65 - jepsen.util 1505 :invoke :cas [21 [44 10]] INFO [2019-12-17 12:29:05,864] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":10123,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:29:05,864] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":10123,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:29:05,864] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":10127,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:29:05,865] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":10127,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:29:05,872] jepsen worker 4 - jepsen.util 1354 :ok :write [1 5] INFO [2019-12-17 12:29:05,872] jepsen worker 4 - jepsen.util 1354 :invoke :read [1 nil] INFO [2019-12-17 12:29:05,876] jepsen worker 27 - jepsen.util 1467 :ok :read [9 3] INFO [2019-12-17 12:29:05,876] jepsen worker 27 - jepsen.util 1467 :invoke :read [9 nil] INFO [2019-12-17 12:29:05,881] jepsen worker 28 - jepsen.util 1108 :fail :cas [9 [11 48]] :ValueNotSwapFrom INFO [2019-12-17 12:29:05,881] jepsen worker 28 - jepsen.util 1108 :invoke :write [9 20] INFO [2019-12-17 12:29:05,908] jepsen worker 65 - jepsen.util 1505 :fail :cas [21 [44 10]] :ValueNotSwapFrom INFO [2019-12-17 12:29:05,908] jepsen worker 65 - jepsen.util 1505 :invoke :cas [21 [11 25]] INFO [2019-12-17 12:29:05,926] jepsen worker 87 - jepsen.util 1527 :fail :cas [29 [40 4]] :ValueNotSwapFrom INFO [2019-12-17 12:29:05,926] jepsen worker 87 - jepsen.util 1527 :invoke :cas [29 [48 6]] INFO [2019-12-17 12:29:05,932] jepsen worker 71 - jepsen.util 1331 :ok :write [23 6] INFO [2019-12-17 12:29:05,932] jepsen worker 71 - jepsen.util 1331 :invoke :write [23 0] INFO [2019-12-17 12:29:05,942] jepsen worker 64 - jepsen.util 1504 :ok :write [21 7] INFO [2019-12-17 12:29:05,943] jepsen worker 64 - jepsen.util 1504 :invoke :read [21 nil] INFO [2019-12-17 12:29:05,968] jepsen worker 79 - jepsen.util 1789 :ok :read [26 30] INFO [2019-12-17 12:29:05,968] jepsen worker 79 - jepsen.util 1789 :invoke :write [26 43] INFO [2019-12-17 12:29:05,975] jepsen worker 74 - jepsen.util 1784 :fail :cas [24 [0 39]] :ValueNotSwapFrom INFO [2019-12-17 12:29:05,975] jepsen worker 74 - jepsen.util 1784 :invoke :cas [24 [27 46]] INFO [2019-12-17 12:29:05,984] jepsen worker 5 - jepsen.util 1355 :fail :cas [1 [8 29]] :ValueNotSwapFrom INFO [2019-12-17 12:29:05,984] jepsen worker 5 - jepsen.util 1355 :invoke :cas [1 [2 46]] INFO [2019-12-17 12:29:05,988] jepsen worker 73 - jepsen.util 1603 :info :write [24 49] :RequestCanceledException INFO [2019-12-17 12:29:05,990] jepsen worker 73 - jepsen.util 1693 :invoke :cas [24 [6 16]] INFO [2019-12-17 12:29:06,001] jepsen worker 53 - jepsen.util 1583 :ok :write [17 43] INFO [2019-12-17 12:29:06,001] jepsen worker 53 - jepsen.util 1583 :invoke :read [17 nil] INFO [2019-12-17 12:29:06,001] jepsen worker 12 - jepsen.util 1812 :ok :write [4 11] INFO [2019-12-17 12:29:06,001] jepsen worker 12 - jepsen.util 1812 :invoke :write [4 33] INFO [2019-12-17 12:29:06,021] jepsen worker 64 - jepsen.util 1504 :ok :read [21 7] INFO [2019-12-17 12:29:06,021] jepsen worker 64 - jepsen.util 1504 :invoke :cas [21 [12 37]] INFO [2019-12-17 12:29:06,021] jepsen worker 78 - jepsen.util 1698 :ok :write [26 36] INFO [2019-12-17 12:29:06,022] jepsen worker 78 - jepsen.util 1698 :invoke :cas [26 [16 1]] INFO [2019-12-17 12:29:06,051] jepsen worker 74 - jepsen.util 1784 :fail :cas [24 [27 46]] :ValueNotSwapFrom INFO [2019-12-17 12:29:06,051] jepsen worker 74 - jepsen.util 1784 :invoke :write [24 47] INFO [2019-12-17 12:29:06,061] jepsen worker 28 - jepsen.util 1108 :ok :write [9 20] INFO [2019-12-17 12:29:06,061] jepsen worker 28 - jepsen.util 1108 :invoke :write [9 13] INFO [2019-12-17 12:29:06,070] jepsen worker 64 - jepsen.util 1504 :fail :cas [21 [12 37]] :ValueNotSwapFrom INFO [2019-12-17 12:29:06,070] jepsen worker 64 - jepsen.util 1504 :invoke :read [21 nil] INFO [2019-12-17 12:29:06,083] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":10140,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:29:06,083] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":10140,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:29:06,083] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":10143,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:29:06,083] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":10143,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:29:06,083] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":10144,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:29:06,083] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":10144,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:29:06,089] jepsen worker 3 - jepsen.util 1623 :fail :cas [1 [8 3]] :ValueNotSwapFrom INFO [2019-12-17 12:29:06,089] jepsen worker 3 - jepsen.util 1623 :invoke :write [1 43] INFO [2019-12-17 12:29:06,089] jepsen worker 6 - jepsen.util 2346 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:29:06,089] jepsen worker 6 - jepsen.util 2346 :invoke :write [2 25] INFO [2019-12-17 12:29:06,151] jepsen worker 28 - jepsen.util 1108 :ok :write [9 13] INFO [2019-12-17 12:29:06,152] jepsen worker 28 - jepsen.util 1108 :invoke :cas [9 [39 41]] INFO [2019-12-17 12:29:06,178] jepsen worker 28 - jepsen.util 1108 :fail :cas [9 [39 41]] :ValueNotSwapFrom INFO [2019-12-17 12:29:06,179] jepsen worker 28 - jepsen.util 1108 :invoke :read [9 nil] INFO [2019-12-17 12:29:06,183] jepsen worker 4 - jepsen.util 1354 :ok :read [1 5] INFO [2019-12-17 12:29:06,183] jepsen worker 4 - jepsen.util 1354 :invoke :read [1 nil] INFO [2019-12-17 12:29:06,193] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":10157,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:29:06,193] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":10157,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:29:06,193] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":10158,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:29:06,193] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":10158,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:29:06,194] jepsen worker 88 - jepsen.util 1708 :ok :read [29 45] INFO [2019-12-17 12:29:06,194] jepsen worker 88 - jepsen.util 1708 :invoke :write [29 28] INFO [2019-12-17 12:29:06,206] jepsen worker 60 - jepsen.util 2310 :info :write [20 2] :RequestCanceledException INFO [2019-12-17 12:29:06,208] jepsen worker 60 - jepsen.util 2400 :invoke :write [20 34] INFO [2019-12-17 12:29:06,212] jepsen worker 32 - jepsen.util 2102 :ok :write [10 42] INFO [2019-12-17 12:29:06,212] jepsen worker 32 - jepsen.util 2102 :invoke :cas [10 [5 35]] INFO [2019-12-17 12:29:06,214] jepsen worker 28 - jepsen.util 1108 :ok :read [9 13] INFO [2019-12-17 12:29:06,215] jepsen worker 28 - jepsen.util 1108 :invoke :read [9 nil] INFO [2019-12-17 12:29:06,222] jepsen worker 56 - jepsen.util 1676 :ok :cas [18 [34 27]] INFO [2019-12-17 12:29:06,222] jepsen worker 56 - jepsen.util 1676 :invoke :cas [18 [5 28]] INFO [2019-12-17 12:29:06,247] jepsen worker 38 - jepsen.util 1838 :ok :read [12 34] INFO [2019-12-17 12:29:06,248] jepsen worker 38 - jepsen.util 1838 :invoke :read [12 nil] INFO [2019-12-17 12:29:06,253] jepsen worker 7 - jepsen.util 2527 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:29:06,253] jepsen worker 7 - jepsen.util 2527 :invoke :read [2 nil] INFO [2019-12-17 12:29:06,258] jepsen worker 28 - jepsen.util 1108 :ok :read [9 13] INFO [2019-12-17 12:29:06,258] jepsen worker 28 - jepsen.util 1108 :invoke :cas [9 [46 42]] INFO [2019-12-17 12:29:06,271] jepsen worker 28 - jepsen.util 1108 :fail :cas [9 [46 42]] :ValueNotSwapFrom INFO [2019-12-17 12:29:06,271] jepsen worker 28 - jepsen.util 1108 :invoke :cas [9 [47 21]] INFO [2019-12-17 12:29:06,289] jepsen worker 72 - jepsen.util 1872 :info :cas [24 [49 20]] :RequestCanceledException INFO [2019-12-17 12:29:06,290] jepsen worker 72 - jepsen.util 1962 :invoke :write [24 46] INFO [2019-12-17 12:29:06,293] jepsen worker 28 - jepsen.util 1108 :fail :cas [9 [47 21]] :ValueNotSwapFrom INFO [2019-12-17 12:29:06,293] jepsen worker 28 - jepsen.util 1108 :invoke :write [9 29] INFO [2019-12-17 12:29:06,296] jepsen worker 56 - jepsen.util 1676 :fail :cas [18 [5 28]] :ValueNotSwapFrom INFO [2019-12-17 12:29:06,296] jepsen worker 56 - jepsen.util 1676 :invoke :write [18 42] INFO [2019-12-17 12:29:06,303] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":10165,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:29:06,303] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":10165,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:29:06,303] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":10169,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:29:06,303] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":10169,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:29:06,325] jepsen worker 52 - jepsen.util 1672 :ok :write [17 35] INFO [2019-12-17 12:29:06,325] jepsen worker 52 - jepsen.util 1672 :invoke :write [17 9] INFO [2019-12-17 12:29:06,331] jepsen worker 38 - jepsen.util 1838 :ok :read [12 34] INFO [2019-12-17 12:29:06,331] jepsen worker 38 - jepsen.util 1838 :invoke :read [12 nil] INFO [2019-12-17 12:29:06,342] jepsen worker 14 - jepsen.util 1724 :fail :cas [4 [24 22]] :ValueNotSwapFrom INFO [2019-12-17 12:29:06,342] jepsen worker 14 - jepsen.util 1724 :invoke :read [4 nil] INFO [2019-12-17 12:29:06,360] jepsen worker 37 - jepsen.util 1387 :ok :write [12 34] INFO [2019-12-17 12:29:06,360] jepsen worker 37 - jepsen.util 1387 :invoke :write [12 42] INFO [2019-12-17 12:29:06,366] jepsen worker 53 - jepsen.util 1583 :ok :read [17 35] INFO [2019-12-17 12:29:06,366] jepsen worker 53 - jepsen.util 1583 :invoke :cas [17 [42 23]] INFO [2019-12-17 12:29:06,368] jepsen worker 28 - jepsen.util 1108 :ok :write [9 29] INFO [2019-12-17 12:29:06,369] jepsen worker 28 - jepsen.util 1108 :invoke :write [9 39] INFO [2019-12-17 12:29:06,388] jepsen worker 79 - jepsen.util 1789 :ok :write [26 43] INFO [2019-12-17 12:29:06,388] jepsen worker 79 - jepsen.util 1789 :invoke :write [26 10] INFO [2019-12-17 12:29:06,405] jepsen worker 9 - jepsen.util 1899 :ok :write [3 40] INFO [2019-12-17 12:29:06,406] jepsen worker 9 - jepsen.util 1899 :invoke :write [3 44] INFO [2019-12-17 12:29:06,407] jepsen worker 19 - jepsen.util 1999 :fail :read [6 nil] :RequestTimeoutException INFO [2019-12-17 12:29:06,407] jepsen worker 19 - jepsen.util 1999 :invoke :write [6 29] INFO [2019-12-17 12:29:06,407] jepsen worker 74 - jepsen.util 1784 :ok :write [24 47] INFO [2019-12-17 12:29:06,408] jepsen worker 74 - jepsen.util 1784 :invoke :read [24 nil] INFO [2019-12-17 12:29:06,412] jepsen worker 5 - jepsen.util 1355 :fail :cas [1 [2 46]] :ValueNotSwapFrom INFO [2019-12-17 12:29:06,412] jepsen worker 5 - jepsen.util 1355 :invoke :write [1 33] INFO [2019-12-17 12:29:06,423] jepsen worker 75 - jepsen.util 2595 :info :write [25 19] :RequestCanceledException INFO [2019-12-17 12:29:06,424] jepsen worker 75 - jepsen.util 2685 :invoke :cas [25 [9 27]] INFO [2019-12-17 12:29:06,426] jepsen worker 47 - jepsen.util 2207 :ok :write [15 22] INFO [2019-12-17 12:29:06,427] jepsen worker 47 - jepsen.util 2207 :invoke :cas [15 [30 47]] INFO [2019-12-17 12:29:06,433] jepsen worker 27 - jepsen.util 1467 :ok :read [9 29] INFO [2019-12-17 12:29:06,434] jepsen worker 27 - jepsen.util 1467 :invoke :cas [9 [5 7]] INFO [2019-12-17 12:29:06,435] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:29:06,435] jepsen nemesis - couchbase.util Rebalance check count: 71 INFO [2019-12-17 12:29:06,435] jepsen nemesis - couchbase.util Rebalance stuck count: 52 INFO [2019-12-17 12:29:06,435] jepsen worker 55 - jepsen.util 2035 :info :write [18 20] :RequestCanceledException INFO [2019-12-17 12:29:06,436] jepsen worker 55 - jepsen.util 2125 :invoke :read [18 nil] INFO [2019-12-17 12:29:06,445] jepsen worker 51 - jepsen.util 1761 :ok :write [17 18] INFO [2019-12-17 12:29:06,445] jepsen worker 51 - jepsen.util 1761 :invoke :cas [17 [42 39]] INFO [2019-12-17 12:29:06,446] jepsen worker 47 - jepsen.util 2207 :fail :cas [15 [30 47]] :ValueNotSwapFrom INFO [2019-12-17 12:29:06,446] jepsen worker 47 - jepsen.util 2207 :invoke :write [15 3] INFO [2019-12-17 12:29:06,451] jepsen worker 36 - jepsen.util 1566 :ok :write [12 38] INFO [2019-12-17 12:29:06,451] jepsen worker 36 - jepsen.util 1566 :invoke :cas [12 [28 37]] INFO [2019-12-17 12:29:06,485] jepsen worker 28 - jepsen.util 1108 :ok :write [9 39] INFO [2019-12-17 12:29:06,485] jepsen worker 28 - jepsen.util 1108 :invoke :read [9 nil] INFO [2019-12-17 12:29:06,488] jepsen worker 13 - jepsen.util 1543 :ok :read [4 11] INFO [2019-12-17 12:29:06,488] jepsen worker 13 - jepsen.util 1543 :invoke :cas [4 [26 48]] INFO [2019-12-17 12:29:06,508] jepsen worker 28 - jepsen.util 1108 :ok :read [9 39] INFO [2019-12-17 12:29:06,508] jepsen worker 28 - jepsen.util 1108 :invoke :cas [9 [30 35]] INFO [2019-12-17 12:29:06,511] jepsen worker 89 - jepsen.util 1709 :ok :read [29 45] INFO [2019-12-17 12:29:06,512] jepsen worker 53 - jepsen.util 1583 :fail :cas [17 [42 23]] :ValueNotSwapFrom INFO [2019-12-17 12:29:06,512] jepsen worker 89 - jepsen.util 1709 :invoke :write [29 17] INFO [2019-12-17 12:29:06,512] jepsen worker 53 - jepsen.util 1583 :invoke :read [17 nil] INFO [2019-12-17 12:29:06,521] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":10193,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:29:06,521] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":10193,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:29:06,522] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":10197,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:29:06,522] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":10197,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:29:06,522] jepsen worker 69 - jepsen.util 1689 :info :write [23 34] :RequestCanceledException INFO [2019-12-17 12:29:06,523] jepsen worker 69 - jepsen.util 1779 :invoke :write [23 5] INFO [2019-12-17 12:29:06,525] jepsen worker 28 - jepsen.util 1108 :fail :cas [9 [30 35]] :ValueNotSwapFrom INFO [2019-12-17 12:29:06,525] jepsen worker 28 - jepsen.util 1108 :invoke :cas [9 [17 16]] INFO [2019-12-17 12:29:06,536] jepsen worker 63 - jepsen.util 1503 :info :write [21 2] :RequestCanceledException INFO [2019-12-17 12:29:06,537] jepsen worker 63 - jepsen.util 1593 :invoke :read [21 nil] INFO [2019-12-17 12:29:06,544] jepsen worker 28 - jepsen.util 1108 :fail :cas [9 [17 16]] :ValueNotSwapFrom INFO [2019-12-17 12:29:06,545] jepsen worker 28 - jepsen.util 1108 :invoke :write [9 32] INFO [2019-12-17 12:29:06,553] jepsen worker 63 - jepsen.util 1593 :ok :read [21 7] INFO [2019-12-17 12:29:06,554] jepsen worker 63 - jepsen.util 1593 :invoke :read [21 nil] INFO [2019-12-17 12:29:06,572] jepsen worker 78 - jepsen.util 1698 :fail :cas [26 [16 1]] :ValueNotSwapFrom INFO [2019-12-17 12:29:06,572] jepsen worker 78 - jepsen.util 1698 :invoke :write [26 12] INFO [2019-12-17 12:29:06,580] jepsen worker 63 - jepsen.util 1593 :ok :read [21 7] INFO [2019-12-17 12:29:06,580] jepsen worker 63 - jepsen.util 1593 :invoke :read [21 nil] INFO [2019-12-17 12:29:06,604] jepsen worker 4 - jepsen.util 1354 :ok :read [1 5] INFO [2019-12-17 12:29:06,605] jepsen worker 4 - jepsen.util 1354 :invoke :read [1 nil] INFO [2019-12-17 12:29:06,614] jepsen worker 63 - jepsen.util 1593 :ok :read [21 7] INFO [2019-12-17 12:29:06,614] jepsen worker 63 - jepsen.util 1593 :invoke :read [21 nil] INFO [2019-12-17 12:29:06,614] jepsen worker 51 - jepsen.util 1761 :fail :cas [17 [42 39]] :ValueNotSwapFrom INFO [2019-12-17 12:29:06,615] jepsen worker 51 - jepsen.util 1761 :invoke :cas [17 [35 42]] INFO [2019-12-17 12:29:06,629] jepsen worker 28 - jepsen.util 1108 :ok :write [9 32] INFO [2019-12-17 12:29:06,629] jepsen worker 28 - jepsen.util 1108 :invoke :write [9 13] INFO [2019-12-17 12:29:06,630] jepsen worker 72 - jepsen.util 1962 :ok :write [24 46] INFO [2019-12-17 12:29:06,630] jepsen worker 72 - jepsen.util 1962 :invoke :write [24 35] INFO [2019-12-17 12:29:06,636] jepsen worker 63 - jepsen.util 1593 :ok :read [21 7] INFO [2019-12-17 12:29:06,637] jepsen worker 63 - jepsen.util 1593 :invoke :write [21 4] INFO [2019-12-17 12:29:06,639] jepsen worker 14 - jepsen.util 1724 :ok :read [4 11] INFO [2019-12-17 12:29:06,639] jepsen worker 14 - jepsen.util 1724 :invoke :read [4 nil] INFO [2019-12-17 12:29:06,654] jepsen worker 53 - jepsen.util 1583 :ok :read [17 18] INFO [2019-12-17 12:29:06,654] jepsen worker 53 - jepsen.util 1583 :invoke :cas [17 [27 25]] INFO [2019-12-17 12:29:06,669] jepsen worker 9 - jepsen.util 1899 :ok :write [3 44] INFO [2019-12-17 12:29:06,669] jepsen worker 9 - jepsen.util 1899 :invoke :write [3 44] INFO [2019-12-17 12:29:06,679] jepsen worker 47 - jepsen.util 2207 :ok :write [15 3] INFO [2019-12-17 12:29:06,679] jepsen worker 47 - jepsen.util 2207 :invoke :cas [15 [29 9]] INFO [2019-12-17 12:29:06,709] jepsen worker 47 - jepsen.util 2207 :fail :cas [15 [29 9]] :ValueNotSwapFrom INFO [2019-12-17 12:29:06,709] jepsen worker 47 - jepsen.util 2207 :invoke :cas [15 [31 22]] INFO [2019-12-17 12:29:06,736] jepsen worker 46 - jepsen.util 1846 :info :write [15 34] :RequestCanceledException INFO [2019-12-17 12:29:06,737] jepsen worker 46 - jepsen.util 1936 :invoke :write [15 14] INFO [2019-12-17 12:29:06,741] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":10222,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:29:06,741] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":10222,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:29:06,741] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":10226,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:29:06,741] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":10226,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:29:06,785] jepsen worker 47 - jepsen.util 2207 :fail :cas [15 [31 22]] :ValueNotSwapFrom INFO [2019-12-17 12:29:06,785] jepsen worker 47 - jepsen.util 2207 :invoke :cas [15 [48 48]] INFO [2019-12-17 12:29:06,794] jepsen worker 87 - jepsen.util 1527 :fail :cas [29 [48 6]] :ValueNotSwapFrom INFO [2019-12-17 12:29:06,794] jepsen worker 87 - jepsen.util 1527 :invoke :read [29 nil] INFO [2019-12-17 12:29:06,805] jepsen worker 55 - jepsen.util 2125 :ok :read [18 27] INFO [2019-12-17 12:29:06,805] jepsen worker 55 - jepsen.util 2125 :invoke :cas [18 [43 7]] INFO [2019-12-17 12:29:06,822] jepsen worker 36 - jepsen.util 1566 :fail :cas [12 [28 37]] :ValueNotSwapFrom INFO [2019-12-17 12:29:06,822] jepsen worker 36 - jepsen.util 1566 :invoke :read [12 nil] INFO [2019-12-17 12:29:06,853] jepsen worker 87 - jepsen.util 1527 :ok :read [29 45] INFO [2019-12-17 12:29:06,853] jepsen worker 87 - jepsen.util 1527 :invoke :cas [29 [7 6]] INFO [2019-12-17 12:29:06,871] jepsen worker 55 - jepsen.util 2125 :fail :cas [18 [43 7]] :ValueNotSwapFrom INFO [2019-12-17 12:29:06,872] jepsen worker 55 - jepsen.util 2125 :invoke :read [18 nil] INFO [2019-12-17 12:29:06,884] jepsen worker 47 - jepsen.util 2207 :fail :cas [15 [48 48]] :ValueNotSwapFrom INFO [2019-12-17 12:29:06,884] jepsen worker 47 - jepsen.util 2207 :invoke :write [15 25] INFO [2019-12-17 12:29:06,888] jepsen worker 28 - jepsen.util 1108 :ok :write [9 13] INFO [2019-12-17 12:29:06,888] jepsen worker 28 - jepsen.util 1108 :invoke :read [9 nil] INFO [2019-12-17 12:29:06,908] jepsen worker 36 - jepsen.util 1566 :ok :read [12 38] INFO [2019-12-17 12:29:06,908] jepsen worker 36 - jepsen.util 1566 :invoke :read [12 nil] INFO [2019-12-17 12:29:06,911] jepsen worker 63 - jepsen.util 1593 :ok :write [21 4] INFO [2019-12-17 12:29:06,912] jepsen worker 63 - jepsen.util 1593 :invoke :write [21 4] INFO [2019-12-17 12:29:06,925] jepsen worker 87 - jepsen.util 1527 :fail :cas [29 [7 6]] :ValueNotSwapFrom INFO [2019-12-17 12:29:06,925] jepsen worker 87 - jepsen.util 1527 :invoke :read [29 nil] INFO [2019-12-17 12:29:06,940] jepsen worker 28 - jepsen.util 1108 :ok :read [9 13] INFO [2019-12-17 12:29:06,940] jepsen worker 28 - jepsen.util 1108 :invoke :write [9 49] INFO [2019-12-17 12:29:06,956] jepsen worker 38 - jepsen.util 1838 :ok :read [12 38] INFO [2019-12-17 12:29:06,956] jepsen worker 38 - jepsen.util 1838 :invoke :read [12 nil] INFO [2019-12-17 12:29:06,957] jepsen worker 72 - jepsen.util 1962 :ok :write [24 35] INFO [2019-12-17 12:29:06,957] jepsen worker 72 - jepsen.util 1962 :invoke :read [24 nil] INFO [2019-12-17 12:29:06,960] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":10243,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:29:06,960] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":10243,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:29:06,960] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":10245,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:29:06,960] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":10245,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:29:06,970] jepsen worker 55 - jepsen.util 2125 :ok :read [18 27] INFO [2019-12-17 12:29:06,970] jepsen worker 55 - jepsen.util 2125 :invoke :read [18 nil] INFO [2019-12-17 12:29:06,994] jepsen worker 36 - jepsen.util 1566 :ok :read [12 38] INFO [2019-12-17 12:29:06,995] jepsen worker 36 - jepsen.util 1566 :invoke :write [12 4] INFO [2019-12-17 12:29:07,006] jepsen worker 63 - jepsen.util 1593 :ok :write [21 4] INFO [2019-12-17 12:29:07,006] jepsen worker 63 - jepsen.util 1593 :invoke :read [21 nil] INFO [2019-12-17 12:29:07,015] jepsen worker 72 - jepsen.util 1962 :ok :read [24 35] INFO [2019-12-17 12:29:07,015] jepsen worker 72 - jepsen.util 1962 :invoke :write [24 49] INFO [2019-12-17 12:29:07,030] jepsen worker 87 - jepsen.util 1527 :ok :read [29 45] INFO [2019-12-17 12:29:07,030] jepsen worker 87 - jepsen.util 1527 :invoke :write [29 39] INFO [2019-12-17 12:29:07,037] jepsen worker 63 - jepsen.util 1593 :ok :read [21 4] INFO [2019-12-17 12:29:07,038] jepsen worker 63 - jepsen.util 1593 :invoke :write [21 33] INFO [2019-12-17 12:29:07,041] jepsen worker 9 - jepsen.util 1899 :ok :write [3 44] INFO [2019-12-17 12:29:07,041] jepsen worker 9 - jepsen.util 1899 :invoke :cas [3 [34 31]] INFO [2019-12-17 12:29:07,065] jepsen worker 46 - jepsen.util 1936 :ok :write [15 14] INFO [2019-12-17 12:29:07,065] jepsen worker 46 - jepsen.util 1936 :invoke :read [15 nil] INFO [2019-12-17 12:29:07,069] jepsen worker 9 - jepsen.util 1899 :fail :cas [3 [34 31]] :ValueNotSwapFrom INFO [2019-12-17 12:29:07,069] jepsen worker 9 - jepsen.util 1899 :invoke :read [3 nil] INFO [2019-12-17 12:29:07,069] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":10257,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:29:07,069] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":10257,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:29:07,069] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":10260,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:29:07,069] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":10260,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:29:07,069] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":10261,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:29:07,070] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":10261,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:29:07,072] jepsen worker 13 - jepsen.util 1543 :fail :cas [4 [26 48]] :ValueNotSwapFrom INFO [2019-12-17 12:29:07,072] jepsen worker 13 - jepsen.util 1543 :invoke :cas [4 [28 34]] INFO [2019-12-17 12:29:07,080] jepsen worker 46 - jepsen.util 1936 :ok :read [15 14] INFO [2019-12-17 12:29:07,080] jepsen worker 46 - jepsen.util 1936 :invoke :read [15 nil] INFO [2019-12-17 12:29:07,084] jepsen worker 9 - jepsen.util 1899 :ok :read [3 44] INFO [2019-12-17 12:29:07,084] jepsen worker 9 - jepsen.util 1899 :invoke :write [3 19] INFO [2019-12-17 12:29:07,087] jepsen worker 28 - jepsen.util 1108 :ok :write [9 49] INFO [2019-12-17 12:29:07,087] jepsen worker 28 - jepsen.util 1108 :invoke :cas [9 [21 31]] INFO [2019-12-17 12:29:07,089] jepsen worker 15 - jepsen.util 2175 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:29:07,089] jepsen worker 15 - jepsen.util 2175 :invoke :write [5 25] INFO [2019-12-17 12:29:07,127] jepsen worker 46 - jepsen.util 1936 :ok :read [15 14] INFO [2019-12-17 12:29:07,127] jepsen worker 46 - jepsen.util 1936 :invoke :cas [15 [26 21]] INFO [2019-12-17 12:29:07,135] jepsen worker 43 - jepsen.util 2383 :info :cas [14 [16 5]] :RequestCanceledException INFO [2019-12-17 12:29:07,135] jepsen worker 28 - jepsen.util 1108 :fail :cas [9 [21 31]] :ValueNotSwapFrom INFO [2019-12-17 12:29:07,136] jepsen worker 28 - jepsen.util 1108 :invoke :read [9 nil] INFO [2019-12-17 12:29:07,136] jepsen worker 43 - jepsen.util 2473 :invoke :cas [14 [32 35]] INFO [2019-12-17 12:29:07,151] jepsen worker 61 - jepsen.util 2491 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:29:07,151] jepsen worker 61 - jepsen.util 2491 :invoke :read [20 nil] INFO [2019-12-17 12:29:07,189] jepsen worker 81 - jepsen.util 2061 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:29:07,190] jepsen worker 81 - jepsen.util 2061 :invoke :cas [27 [24 44]] INFO [2019-12-17 12:29:07,190] jepsen worker 46 - jepsen.util 1936 :fail :cas [15 [26 21]] :ValueNotSwapFrom INFO [2019-12-17 12:29:07,190] jepsen worker 46 - jepsen.util 1936 :invoke :read [15 nil] INFO [2019-12-17 12:29:07,197] jepsen worker 28 - jepsen.util 1108 :ok :read [9 49] INFO [2019-12-17 12:29:07,198] jepsen worker 28 - jepsen.util 1108 :invoke :cas [9 [48 18]] INFO [2019-12-17 12:29:07,207] jepsen worker 21 - jepsen.util 2361 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:29:07,207] jepsen worker 16 - jepsen.util 2086 :info :write [5 6] :RequestCanceledException INFO [2019-12-17 12:29:07,207] jepsen worker 21 - jepsen.util 2361 :invoke :read [7 nil] INFO [2019-12-17 12:29:07,208] jepsen worker 16 - jepsen.util 2176 :invoke :write [5 22] INFO [2019-12-17 12:29:07,218] jepsen worker 46 - jepsen.util 1936 :ok :read [15 14] INFO [2019-12-17 12:29:07,218] jepsen worker 46 - jepsen.util 1936 :invoke :write [15 36] INFO [2019-12-17 12:29:07,222] jepsen worker 70 - jepsen.util 1960 :info :write [23 13] :RequestCanceledException INFO [2019-12-17 12:29:07,222] jepsen worker 58 - jepsen.util 2128 :info :cas [19 [1 21]] :RequestCanceledException INFO [2019-12-17 12:29:07,222] jepsen worker 84 - jepsen.util 2154 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:29:07,222] jepsen worker 26 - jepsen.util 2546 :info :cas [8 [11 18]] :RequestCanceledException INFO [2019-12-17 12:29:07,223] jepsen worker 57 - jepsen.util 2127 :info :cas [19 [21 8]] :RequestCanceledException INFO [2019-12-17 12:29:07,223] jepsen worker 84 - jepsen.util 2154 :invoke :cas [28 [45 4]] INFO [2019-12-17 12:29:07,224] jepsen worker 70 - jepsen.util 2050 :invoke :cas [23 [44 32]] INFO [2019-12-17 12:29:07,224] jepsen worker 58 - jepsen.util 2218 :invoke :cas [19 [22 41]] INFO [2019-12-17 12:29:07,224] jepsen worker 26 - jepsen.util 2636 :invoke :read [8 nil] INFO [2019-12-17 12:29:07,224] jepsen worker 57 - jepsen.util 2217 :invoke :write [19 5] INFO [2019-12-17 12:29:07,236] jepsen worker 86 - jepsen.util 2426 :info :write [28 43] :RequestCanceledException INFO [2019-12-17 12:29:07,236] jepsen worker 62 - jepsen.util 2312 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:29:07,237] jepsen worker 62 - jepsen.util 2312 :invoke :read [20 nil] INFO [2019-12-17 12:29:07,237] jepsen worker 86 - jepsen.util 2516 :invoke :read [28 nil] INFO [2019-12-17 12:29:07,241] jepsen worker 63 - jepsen.util 1593 :ok :write [21 33] INFO [2019-12-17 12:29:07,242] jepsen worker 63 - jepsen.util 1593 :invoke :cas [21 [26 25]] INFO [2019-12-17 12:29:07,250] jepsen worker 28 - jepsen.util 1108 :fail :cas [9 [48 18]] :ValueNotSwapFrom INFO [2019-12-17 12:29:07,250] jepsen worker 28 - jepsen.util 1108 :invoke :cas [9 [20 26]] INFO [2019-12-17 12:29:07,253] jepsen worker 76 - jepsen.util 2326 :info :cas [25 [22 41]] :RequestCanceledException INFO [2019-12-17 12:29:07,253] jepsen worker 77 - jepsen.util 2327 :info :write [25 29] :RequestCanceledException INFO [2019-12-17 12:29:07,253] jepsen worker 22 - jepsen.util 2452 :info :write [7 25] :RequestCanceledException INFO [2019-12-17 12:29:07,254] jepsen worker 22 - jepsen.util 2542 :invoke :read [7 nil] INFO [2019-12-17 12:29:07,254] jepsen worker 76 - jepsen.util 2416 :invoke :write [25 6] INFO [2019-12-17 12:29:07,255] jepsen worker 77 - jepsen.util 2417 :invoke :write [25 5] INFO [2019-12-17 12:29:07,262] jepsen worker 63 - jepsen.util 1593 :fail :cas [21 [26 25]] :ValueNotSwapFrom INFO [2019-12-17 12:29:07,262] jepsen worker 63 - jepsen.util 1593 :invoke :cas [21 [29 30]] INFO [2019-12-17 12:29:07,270] jepsen worker 26 - jepsen.util 2636 :ok :read [8 14] INFO [2019-12-17 12:29:07,270] jepsen worker 26 - jepsen.util 2636 :invoke :read [8 nil] INFO [2019-12-17 12:29:07,278] jepsen worker 28 - jepsen.util 1108 :fail :cas [9 [20 26]] :ValueNotSwapFrom INFO [2019-12-17 12:29:07,278] jepsen worker 28 - jepsen.util 1108 :invoke :write [9 29] INFO [2019-12-17 12:29:07,280] jepsen worker 72 - jepsen.util 1962 :ok :write [24 49] INFO [2019-12-17 12:29:07,280] jepsen worker 72 - jepsen.util 1962 :invoke :read [24 nil] INFO [2019-12-17 12:29:07,289] jepsen worker 34 - jepsen.util 2374 :info :cas [11 [1 27]] :RequestCanceledException INFO [2019-12-17 12:29:07,289] jepsen worker 25 - jepsen.util 2545 :info :cas [8 [11 15]] :RequestCanceledException INFO [2019-12-17 12:29:07,289] jepsen worker 35 - jepsen.util 2015 :info :cas [11 [4 11]] :RequestCanceledException INFO [2019-12-17 12:29:07,289] jepsen worker 23 - jepsen.util 2093 :info :cas [7 [19 1]] :RequestCanceledException INFO [2019-12-17 12:29:07,289] jepsen worker 82 - jepsen.util 2782 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:29:07,289] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":10292,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:29:07,289] jepsen worker 82 - jepsen.util 2782 :invoke :write [27 49] INFO [2019-12-17 12:29:07,289] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":10292,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:29:07,290] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":10294,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:29:07,290] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":10294,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:29:07,291] jepsen worker 34 - jepsen.util 2464 :invoke :read [11 nil] INFO [2019-12-17 12:29:07,292] jepsen worker 25 - jepsen.util 2635 :invoke :write [8 48] INFO [2019-12-17 12:29:07,292] jepsen worker 35 - jepsen.util 2105 :invoke :read [11 nil] INFO [2019-12-17 12:29:07,292] jepsen worker 23 - jepsen.util 2183 :invoke :write [7 26] INFO [2019-12-17 12:29:07,297] jepsen worker 63 - jepsen.util 1593 :fail :cas [21 [29 30]] :ValueNotSwapFrom INFO [2019-12-17 12:29:07,298] jepsen worker 63 - jepsen.util 1593 :invoke :read [21 nil] INFO [2019-12-17 12:29:07,299] jepsen worker 26 - jepsen.util 2636 :ok :read [8 14] INFO [2019-12-17 12:29:07,300] jepsen worker 26 - jepsen.util 2636 :invoke :write [8 11] INFO [2019-12-17 12:29:07,303] jepsen worker 55 - jepsen.util 2125 :ok :read [18 27] INFO [2019-12-17 12:29:07,303] jepsen worker 55 - jepsen.util 2125 :invoke :read [18 nil] INFO [2019-12-17 12:29:07,307] jepsen worker 24 - jepsen.util 2094 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:29:07,307] jepsen worker 40 - jepsen.util 2740 :info :write [13 3] :RequestCanceledException INFO [2019-12-17 12:29:07,307] jepsen worker 8 - jepsen.util 2168 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:29:07,307] jepsen worker 33 - jepsen.util 2643 :info :cas [11 [15 21]] :RequestCanceledException INFO [2019-12-17 12:29:07,307] jepsen worker 80 - jepsen.util 1790 :info :write [26 42] :RequestCanceledException INFO [2019-12-17 12:29:07,307] jepsen worker 24 - jepsen.util 2094 :invoke :write [8 11] INFO [2019-12-17 12:29:07,307] jepsen worker 8 - jepsen.util 2168 :invoke :write [2 12] INFO [2019-12-17 12:29:07,308] jepsen worker 80 - jepsen.util 1880 :invoke :cas [26 [7 10]] INFO [2019-12-17 12:29:07,308] jepsen worker 40 - jepsen.util 2830 :invoke :cas [13 [27 12]] INFO [2019-12-17 12:29:07,308] jepsen worker 33 - jepsen.util 2733 :invoke :read [11 nil] INFO [2019-12-17 12:29:07,317] jepsen worker 72 - jepsen.util 1962 :ok :read [24 49] INFO [2019-12-17 12:29:07,317] jepsen worker 72 - jepsen.util 1962 :invoke :cas [24 [17 12]] INFO [2019-12-17 12:29:07,317] jepsen worker 38 - jepsen.util 1838 :ok :read [12 38] INFO [2019-12-17 12:29:07,317] jepsen worker 38 - jepsen.util 1838 :invoke :write [12 4] INFO [2019-12-17 12:29:07,322] jepsen worker 0 - jepsen.util 2430 :info :write [0 44] :RequestCanceledException INFO [2019-12-17 12:29:07,323] jepsen worker 50 - jepsen.util 2480 :info :write [16 48] :RequestCanceledException INFO [2019-12-17 12:29:07,323] jepsen worker 1 - jepsen.util 2341 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:29:07,323] jepsen worker 1 - jepsen.util 2341 :invoke :read [0 nil] INFO [2019-12-17 12:29:07,323] jepsen worker 0 - jepsen.util 2520 :invoke :write [0 24] INFO [2019-12-17 12:29:07,324] jepsen worker 50 - jepsen.util 2570 :invoke :write [16 36] INFO [2019-12-17 12:29:07,335] jepsen worker 51 - jepsen.util 1761 :fail :cas [17 [35 42]] :ValueNotSwapFrom INFO [2019-12-17 12:29:07,335] jepsen worker 51 - jepsen.util 1761 :invoke :cas [17 [6 36]] INFO [2019-12-17 12:29:07,336] jepsen worker 66 - jepsen.util 2046 :info :write [22 25] :RequestCanceledException INFO [2019-12-17 12:29:07,336] jepsen worker 85 - jepsen.util 2515 :info :cas [28 [22 24]] :RequestCanceledException INFO [2019-12-17 12:29:07,336] jepsen worker 17 - jepsen.util 2357 :info :write [5 32] :RequestCanceledException INFO [2019-12-17 12:29:07,338] jepsen worker 66 - jepsen.util 2136 :invoke :write [22 44] INFO [2019-12-17 12:29:07,338] jepsen worker 17 - jepsen.util 2447 :invoke :write [5 2] INFO [2019-12-17 12:29:07,338] jepsen worker 85 - jepsen.util 2605 :invoke :cas [28 [1 21]] INFO [2019-12-17 12:29:07,342] jepsen worker 34 - jepsen.util 2464 :ok :read [11 7] INFO [2019-12-17 12:29:07,342] jepsen worker 34 - jepsen.util 2464 :invoke :cas [11 [43 15]] INFO [2019-12-17 12:29:07,353] jepsen worker 68 - jepsen.util 2048 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:29:07,353] jepsen worker 68 - jepsen.util 2048 :invoke :cas [22 [25 30]] INFO [2019-12-17 12:29:07,358] jepsen worker 63 - jepsen.util 1593 :ok :read [21 33] INFO [2019-12-17 12:29:07,359] jepsen worker 63 - jepsen.util 1593 :invoke :write [21 23] INFO [2019-12-17 12:29:07,366] jepsen worker 9 - jepsen.util 1899 :ok :write [3 19] INFO [2019-12-17 12:29:07,366] jepsen worker 9 - jepsen.util 1899 :invoke :write [3 47] INFO [2019-12-17 12:29:07,388] jepsen worker 59 - jepsen.util 1949 :info :cas [19 [34 1]] :RequestCanceledException INFO [2019-12-17 12:29:07,388] jepsen worker 39 - jepsen.util 2469 :info :cas [13 [4 29]] :RequestCanceledException INFO [2019-12-17 12:29:07,388] jepsen worker 83 - jepsen.util 2603 :info :cas [27 [21 11]] :RequestCanceledException INFO [2019-12-17 12:29:07,392] jepsen worker 59 - jepsen.util 2039 :invoke :cas [19 [2 29]] INFO [2019-12-17 12:29:07,393] jepsen worker 83 - jepsen.util 2693 :invoke :read [27 nil] INFO [2019-12-17 12:29:07,394] jepsen worker 33 - jepsen.util 2733 :ok :read [11 7] INFO [2019-12-17 12:29:07,394] jepsen worker 39 - jepsen.util 2559 :invoke :write [13 29] INFO [2019-12-17 12:29:07,394] jepsen worker 33 - jepsen.util 2733 :invoke :cas [11 [13 38]] INFO [2019-12-17 12:29:07,407] jepsen worker 42 - jepsen.util 2742 :info :cas [14 [12 14]] :RequestCanceledException INFO [2019-12-17 12:29:07,413] jepsen worker 42 - jepsen.util 2832 :invoke :write [14 12] INFO [2019-12-17 12:29:07,423] jepsen worker 41 - jepsen.util 2471 :info :cas [13 [16 19]] :RequestCanceledException INFO [2019-12-17 12:29:07,429] jepsen worker 46 - jepsen.util 1936 :ok :write [15 36] INFO [2019-12-17 12:29:07,429] jepsen worker 46 - jepsen.util 1936 :invoke :cas [15 [12 34]] INFO [2019-12-17 12:29:07,431] jepsen worker 41 - jepsen.util 2561 :invoke :read [13 nil] INFO [2019-12-17 12:29:07,443] jepsen worker 55 - jepsen.util 2125 :ok :read [18 27] INFO [2019-12-17 12:29:07,443] jepsen worker 55 - jepsen.util 2125 :invoke :read [18 nil] INFO [2019-12-17 12:29:07,457] jepsen worker 34 - jepsen.util 2464 :fail :cas [11 [43 15]] :ValueNotSwapFrom INFO [2019-12-17 12:29:07,457] jepsen worker 34 - jepsen.util 2464 :invoke :read [11 nil] INFO [2019-12-17 12:29:07,470] jepsen worker 14 - jepsen.util 1724 :ok :read [4 11] INFO [2019-12-17 12:29:07,470] jepsen worker 14 - jepsen.util 1724 :invoke :read [4 nil] INFO [2019-12-17 12:29:07,472] jepsen worker 33 - jepsen.util 2733 :fail :cas [11 [13 38]] :ValueNotSwapFrom INFO [2019-12-17 12:29:07,473] jepsen worker 33 - jepsen.util 2733 :invoke :cas [11 [40 24]] INFO [2019-12-17 12:29:07,476] jepsen worker 53 - jepsen.util 1583 :fail :cas [17 [27 25]] :ValueNotSwapFrom INFO [2019-12-17 12:29:07,476] jepsen worker 53 - jepsen.util 1583 :invoke :cas [17 [25 12]] INFO [2019-12-17 12:29:07,477] jepsen worker 72 - jepsen.util 1962 :fail :cas [24 [17 12]] :ValueNotSwapFrom INFO [2019-12-17 12:29:07,477] jepsen worker 72 - jepsen.util 1962 :invoke :write [24 43] INFO [2019-12-17 12:29:07,489] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:29:07,489] jepsen nemesis - couchbase.util Rebalance check count: 72 INFO [2019-12-17 12:29:07,489] jepsen nemesis - couchbase.util Rebalance stuck count: 53 INFO [2019-12-17 12:29:07,507] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":10344,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:29:07,507] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":10344,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:29:07,507] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":10347,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:29:07,507] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":10347,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:29:07,514] jepsen worker 46 - jepsen.util 1936 :fail :cas [15 [12 34]] :ValueNotSwapFrom INFO [2019-12-17 12:29:07,514] jepsen worker 46 - jepsen.util 1936 :invoke :write [15 9] INFO [2019-12-17 12:29:07,522] jepsen worker 48 - jepsen.util 1938 :info :cas [16 [31 38]] :RequestCanceledException INFO [2019-12-17 12:29:07,525] jepsen worker 48 - jepsen.util 2028 :invoke :read [16 nil] INFO [2019-12-17 12:29:07,525] jepsen worker 9 - jepsen.util 1899 :ok :write [3 47] INFO [2019-12-17 12:29:07,525] jepsen worker 9 - jepsen.util 1899 :invoke :cas [3 [30 9]] INFO [2019-12-17 12:29:07,525] jepsen worker 28 - jepsen.util 1108 :ok :write [9 29] INFO [2019-12-17 12:29:07,525] jepsen worker 28 - jepsen.util 1108 :invoke :cas [9 [48 26]] INFO [2019-12-17 12:29:07,536] jepsen worker 31 - jepsen.util 1741 :info :write [10 4] :RequestCanceledException INFO [2019-12-17 12:29:07,537] jepsen worker 31 - jepsen.util 1831 :invoke :read [10 nil] INFO [2019-12-17 12:29:07,562] jepsen worker 33 - jepsen.util 2733 :fail :cas [11 [40 24]] :ValueNotSwapFrom INFO [2019-12-17 12:29:07,562] jepsen worker 33 - jepsen.util 2733 :invoke :write [11 26] INFO [2019-12-17 12:29:07,575] jepsen worker 80 - jepsen.util 1880 :fail :cas [26 [7 10]] :ValueNotSwapFrom INFO [2019-12-17 12:29:07,575] jepsen worker 80 - jepsen.util 1880 :invoke :read [26 nil] INFO [2019-12-17 12:29:07,575] jepsen worker 34 - jepsen.util 2464 :ok :read [11 7] INFO [2019-12-17 12:29:07,576] jepsen worker 34 - jepsen.util 2464 :invoke :write [11 28] INFO [2019-12-17 12:29:07,589] jepsen worker 67 - jepsen.util 2767 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:29:07,589] jepsen worker 67 - jepsen.util 2767 :invoke :write [22 24] INFO [2019-12-17 12:29:07,624] jepsen worker 63 - jepsen.util 1593 :ok :write [21 23] INFO [2019-12-17 12:29:07,624] jepsen worker 63 - jepsen.util 1593 :invoke :write [21 23] INFO [2019-12-17 12:29:07,634] jepsen worker 9 - jepsen.util 1899 :fail :cas [3 [30 9]] :ValueNotSwapFrom INFO [2019-12-17 12:29:07,634] jepsen worker 9 - jepsen.util 1899 :invoke :cas [3 [29 37]] INFO [2019-12-17 12:29:07,637] jepsen worker 30 - jepsen.util 2010 :info :write [10 9] :RequestCanceledException INFO [2019-12-17 12:29:07,643] jepsen worker 30 - jepsen.util 2100 :invoke :write [10 16] INFO [2019-12-17 12:29:07,646] jepsen worker 28 - jepsen.util 1108 :fail :cas [9 [48 26]] :ValueNotSwapFrom INFO [2019-12-17 12:29:07,647] jepsen worker 28 - jepsen.util 1108 :invoke :read [9 nil] INFO [2019-12-17 12:29:07,651] jepsen worker 4 - jepsen.util 1354 :ok :read [1 5] INFO [2019-12-17 12:29:07,651] jepsen worker 4 - jepsen.util 1354 :invoke :write [1 16] INFO [2019-12-17 12:29:07,694] jepsen worker 31 - jepsen.util 1831 :ok :read [10 42] INFO [2019-12-17 12:29:07,694] jepsen worker 31 - jepsen.util 1831 :invoke :write [10 12] INFO [2019-12-17 12:29:07,695] jepsen worker 9 - jepsen.util 1899 :fail :cas [3 [29 37]] :ValueNotSwapFrom INFO [2019-12-17 12:29:07,695] jepsen worker 9 - jepsen.util 1899 :invoke :cas [3 [26 14]] INFO [2019-12-17 12:29:07,720] jepsen worker 28 - jepsen.util 1108 :ok :read [9 29] INFO [2019-12-17 12:29:07,720] jepsen worker 28 - jepsen.util 1108 :invoke :write [9 5] INFO [2019-12-17 12:29:07,722] jepsen worker 11 - jepsen.util 2171 :info :write [3 15] :RequestCanceledException INFO [2019-12-17 12:29:07,722] jepsen worker 54 - jepsen.util 1764 :info :write [18 10] :RequestCanceledException INFO [2019-12-17 12:29:07,724] jepsen worker 46 - jepsen.util 1936 :ok :write [15 9] INFO [2019-12-17 12:29:07,724] jepsen worker 46 - jepsen.util 1936 :invoke :write [15 31] INFO [2019-12-17 12:29:07,726] jepsen worker 11 - jepsen.util 2261 :invoke :read [3 nil] INFO [2019-12-17 12:29:07,726] jepsen worker 54 - jepsen.util 1854 :invoke :read [18 nil] INFO [2019-12-17 12:29:07,726] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":10372,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:29:07,726] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":10372,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:29:07,727] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":10373,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:29:07,727] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":10373,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:29:07,734] jepsen worker 72 - jepsen.util 1962 :ok :write [24 43] INFO [2019-12-17 12:29:07,734] jepsen worker 72 - jepsen.util 1962 :invoke :write [24 35] INFO [2019-12-17 12:29:07,748] jepsen worker 9 - jepsen.util 1899 :fail :cas [3 [26 14]] :ValueNotSwapFrom INFO [2019-12-17 12:29:07,748] jepsen worker 9 - jepsen.util 1899 :invoke :cas [3 [19 27]] INFO [2019-12-17 12:29:07,767] jepsen worker 11 - jepsen.util 2261 :ok :read [3 47] INFO [2019-12-17 12:29:07,767] jepsen worker 11 - jepsen.util 2261 :invoke :write [3 30] INFO [2019-12-17 12:29:07,787] jepsen worker 45 - jepsen.util 2025 :info :write [15 38] :RequestCanceledException INFO [2019-12-17 12:29:07,787] jepsen worker 2 - jepsen.util 2162 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:29:07,788] jepsen worker 2 - jepsen.util 2162 :invoke :cas [0 [18 13]] INFO [2019-12-17 12:29:07,790] jepsen worker 45 - jepsen.util 2115 :invoke :read [15 nil] INFO [2019-12-17 12:29:07,794] jepsen worker 63 - jepsen.util 1593 :ok :write [21 23] INFO [2019-12-17 12:29:07,794] jepsen worker 63 - jepsen.util 1593 :invoke :write [21 13] INFO [2019-12-17 12:29:07,798] jepsen worker 29 - jepsen.util 1469 :info :cas [9 [49 14]] :RequestCanceledException INFO [2019-12-17 12:29:07,798] jepsen worker 18 - jepsen.util 1998 :fail :read [6 nil] :RequestTimeoutException INFO [2019-12-17 12:29:07,799] jepsen worker 18 - jepsen.util 1998 :invoke :write [6 30] INFO [2019-12-17 12:29:07,802] jepsen worker 29 - jepsen.util 1559 :invoke :write [9 26] INFO [2019-12-17 12:29:07,822] jepsen worker 10 - jepsen.util 1900 :info :write [3 17] :RequestCanceledException INFO [2019-12-17 12:29:07,822] jepsen worker 20 - jepsen.util 1640 :fail :read [6 nil] :RequestTimeoutException INFO [2019-12-17 12:29:07,823] jepsen worker 20 - jepsen.util 1640 :invoke :cas [6 [29 27]] INFO [2019-12-17 12:29:07,825] jepsen worker 10 - jepsen.util 1990 :invoke :read [3 nil] INFO [2019-12-17 12:29:07,835] jepsen worker 44 - jepsen.util 1934 :info :cas [14 [42 9]] :RequestCanceledException INFO [2019-12-17 12:29:07,837] jepsen worker 44 - jepsen.util 2024 :invoke :write [14 34] INFO [2019-12-17 12:29:07,873] jepsen worker 51 - jepsen.util 1761 :fail :cas [17 [6 36]] :ValueNotSwapFrom INFO [2019-12-17 12:29:07,873] jepsen worker 51 - jepsen.util 1761 :invoke :cas [17 [17 2]] INFO [2019-12-17 12:29:07,880] jepsen worker 9 - jepsen.util 1899 :fail :cas [3 [19 27]] :ValueNotSwapFrom INFO [2019-12-17 12:29:07,880] jepsen worker 9 - jepsen.util 1899 :invoke :write [3 15] INFO [2019-12-17 12:29:07,885] jepsen worker 28 - jepsen.util 1108 :ok :write [9 5] INFO [2019-12-17 12:29:07,886] jepsen worker 28 - jepsen.util 1108 :invoke :write [9 27] INFO [2019-12-17 12:29:07,899] jepsen worker 20 - jepsen.util 1640 :fail :cas [6 [29 27]] :ValueNotSwapFrom INFO [2019-12-17 12:29:07,899] jepsen worker 20 - jepsen.util 1640 :invoke :write [6 25] INFO [2019-12-17 12:29:07,906] jepsen worker 49 - jepsen.util 1579 :info :cas [16 [23 3]] :RequestCanceledException INFO [2019-12-17 12:29:07,909] jepsen worker 49 - jepsen.util 1669 :invoke :write [16 25] INFO [2019-12-17 12:29:07,936] jepsen worker 65 - jepsen.util 1505 :info :cas [21 [11 25]] :RequestCanceledException INFO [2019-12-17 12:29:07,937] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":10396,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:29:07,937] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":10396,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:29:07,937] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":10400,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:29:07,937] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":10400,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:29:07,940] jepsen worker 65 - jepsen.util 1595 :invoke :read [21 nil] INFO [2019-12-17 12:29:07,944] jepsen worker 45 - jepsen.util 2115 :ok :read [15 9] INFO [2019-12-17 12:29:07,944] jepsen worker 45 - jepsen.util 2115 :invoke :read [15 nil] INFO [2019-12-17 12:29:07,968] jepsen worker 29 - jepsen.util 1559 :ok :write [9 26] INFO [2019-12-17 12:29:07,969] jepsen worker 29 - jepsen.util 1559 :invoke :write [9 18] INFO [2019-12-17 12:29:08,004] jepsen worker 28 - jepsen.util 1108 :ok :write [9 27] INFO [2019-12-17 12:29:08,004] jepsen worker 28 - jepsen.util 1108 :invoke :cas [9 [25 27]] INFO [2019-12-17 12:29:08,022] jepsen worker 73 - jepsen.util 1693 :info :cas [24 [6 16]] :RequestCanceledException INFO [2019-12-17 12:29:08,022] jepsen worker 71 - jepsen.util 1331 :info :write [23 0] :RequestCanceledException INFO [2019-12-17 12:29:08,025] jepsen worker 73 - jepsen.util 1783 :invoke :read [24 nil] INFO [2019-12-17 12:29:08,027] jepsen worker 71 - jepsen.util 1421 :invoke :read [23 nil] INFO [2019-12-17 12:29:08,030] jepsen worker 14 - jepsen.util 1724 :ok :read [4 11] INFO [2019-12-17 12:29:08,030] jepsen worker 14 - jepsen.util 1724 :invoke :read [4 nil] INFO [2019-12-17 12:29:08,044] jepsen worker 13 - jepsen.util 1543 :fail :cas [4 [28 34]] :ValueNotSwapFrom INFO [2019-12-17 12:29:08,045] jepsen worker 13 - jepsen.util 1543 :invoke :cas [4 [29 25]] INFO [2019-12-17 12:29:08,047] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":10413,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:29:08,047] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":10413,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:29:08,048] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":10414,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:29:08,048] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":10414,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:29:08,063] jepsen worker 28 - jepsen.util 1108 :fail :cas [9 [25 27]] :ValueNotSwapFrom INFO [2019-12-17 12:29:08,063] jepsen worker 28 - jepsen.util 1108 :invoke :cas [9 [31 0]] INFO [2019-12-17 12:29:08,064] jepsen worker 29 - jepsen.util 1559 :ok :write [9 18] INFO [2019-12-17 12:29:08,064] jepsen worker 29 - jepsen.util 1559 :invoke :read [9 nil] INFO [2019-12-17 12:29:08,079] jepsen worker 53 - jepsen.util 1583 :fail :cas [17 [25 12]] :ValueNotSwapFrom INFO [2019-12-17 12:29:08,079] jepsen worker 53 - jepsen.util 1583 :invoke :read [17 nil] INFO [2019-12-17 12:29:08,080] jepsen worker 70 - jepsen.util 2050 :fail :cas [23 [44 32]] :ValueNotSwapFrom INFO [2019-12-17 12:29:08,080] jepsen worker 70 - jepsen.util 2050 :invoke :read [23 nil] INFO [2019-12-17 12:29:08,088] jepsen worker 12 - jepsen.util 1812 :info :write [4 33] :RequestCanceledException INFO [2019-12-17 12:29:08,090] jepsen worker 12 - jepsen.util 1902 :invoke :cas [4 [23 11]] INFO [2019-12-17 12:29:08,107] jepsen worker 64 - jepsen.util 1504 :fail :read [21 nil] :RequestTimeoutException INFO [2019-12-17 12:29:08,107] jepsen worker 64 - jepsen.util 1504 :invoke :write [21 35] INFO [2019-12-17 12:29:08,120] jepsen worker 29 - jepsen.util 1559 :ok :read [9 18] INFO [2019-12-17 12:29:08,120] jepsen worker 29 - jepsen.util 1559 :invoke :read [9 nil] INFO [2019-12-17 12:29:08,121] jepsen worker 73 - jepsen.util 1783 :ok :read [24 43] INFO [2019-12-17 12:29:08,121] jepsen worker 73 - jepsen.util 1783 :invoke :cas [24 [2 4]] INFO [2019-12-17 12:29:08,140] jepsen worker 28 - jepsen.util 1108 :fail :cas [9 [31 0]] :ValueNotSwapFrom INFO [2019-12-17 12:29:08,140] jepsen worker 28 - jepsen.util 1108 :invoke :write [9 20] INFO [2019-12-17 12:29:08,171] jepsen worker 29 - jepsen.util 1559 :ok :read [9 18] INFO [2019-12-17 12:29:08,171] jepsen worker 29 - jepsen.util 1559 :invoke :cas [9 [2 46]] INFO [2019-12-17 12:29:08,180] jepsen worker 73 - jepsen.util 1783 :fail :cas [24 [2 4]] :ValueNotSwapFrom INFO [2019-12-17 12:29:08,180] jepsen worker 73 - jepsen.util 1783 :invoke :cas [24 [17 39]] INFO [2019-12-17 12:29:08,185] jepsen worker 51 - jepsen.util 1761 :fail :cas [17 [17 2]] :ValueNotSwapFrom INFO [2019-12-17 12:29:08,185] jepsen worker 51 - jepsen.util 1761 :invoke :cas [17 [36 19]] INFO [2019-12-17 12:29:08,189] jepsen worker 3 - jepsen.util 1623 :info :write [1 43] :RequestCanceledException INFO [2019-12-17 12:29:08,189] jepsen worker 6 - jepsen.util 2346 :info :write [2 25] :RequestCanceledException INFO [2019-12-17 12:29:08,192] jepsen worker 3 - jepsen.util 1713 :invoke :read [1 nil] INFO [2019-12-17 12:29:08,192] jepsen worker 6 - jepsen.util 2436 :invoke :write [2 33] INFO [2019-12-17 12:29:08,217] jepsen worker 12 - jepsen.util 1902 :fail :cas [4 [23 11]] :ValueNotSwapFrom INFO [2019-12-17 12:29:08,218] jepsen worker 12 - jepsen.util 1902 :invoke :write [4 28] INFO [2019-12-17 12:29:08,238] jepsen worker 29 - jepsen.util 1559 :fail :cas [9 [2 46]] :ValueNotSwapFrom INFO [2019-12-17 12:29:08,239] jepsen worker 29 - jepsen.util 1559 :invoke :write [9 19] INFO [2019-12-17 12:29:08,244] jepsen worker 72 - jepsen.util 1962 :ok :write [24 35] INFO [2019-12-17 12:29:08,244] jepsen worker 72 - jepsen.util 1962 :invoke :write [24 15] INFO [2019-12-17 12:29:08,244] jepsen worker 30 - jepsen.util 2100 :ok :write [10 16] INFO [2019-12-17 12:29:08,244] jepsen worker 30 - jepsen.util 2100 :invoke :read [10 nil] INFO [2019-12-17 12:29:08,252] jepsen worker 88 - jepsen.util 1708 :info :write [29 28] :RequestCanceledException INFO [2019-12-17 12:29:08,255] jepsen worker 88 - jepsen.util 1798 :invoke :cas [29 [46 20]] INFO [2019-12-17 12:29:08,265] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":10432,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:29:08,265] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":10432,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:29:08,265] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":10441,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:29:08,265] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":10434,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:29:08,265] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":10441,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:29:08,265] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":10434,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:29:08,265] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":10445,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:29:08,266] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":10445,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:29:08,272] jepsen worker 28 - jepsen.util 1108 :ok :write [9 20] INFO [2019-12-17 12:29:08,272] jepsen worker 28 - jepsen.util 1108 :invoke :cas [9 [42 37]] INFO [2019-12-17 12:29:08,286] jepsen worker 55 - jepsen.util 2125 :ok :read [18 27] INFO [2019-12-17 12:29:08,286] jepsen worker 55 - jepsen.util 2125 :invoke :read [18 nil] INFO [2019-12-17 12:29:08,289] jepsen worker 60 - jepsen.util 2400 :info :write [20 34] :RequestCanceledException INFO [2019-12-17 12:29:08,290] jepsen worker 60 - jepsen.util 2490 :invoke :cas [20 [45 19]] INFO [2019-12-17 12:29:08,322] jepsen worker 32 - jepsen.util 2102 :info :cas [10 [5 35]] :RequestCanceledException INFO [2019-12-17 12:29:08,323] jepsen worker 32 - jepsen.util 2192 :invoke :write [10 48] INFO [2019-12-17 12:29:08,334] jepsen worker 80 - jepsen.util 1880 :ok :read [26 43] INFO [2019-12-17 12:29:08,334] jepsen worker 80 - jepsen.util 1880 :invoke :cas [26 [31 25]] INFO [2019-12-17 12:29:08,335] jepsen worker 29 - jepsen.util 1559 :ok :write [9 19] INFO [2019-12-17 12:29:08,335] jepsen worker 29 - jepsen.util 1559 :invoke :cas [9 [4 37]] INFO [2019-12-17 12:29:08,493] jepsen worker 79 - jepsen.util 1789 :info :write [26 10] :RequestCanceledException INFO [2019-12-17 12:29:08,493] jepsen worker 52 - jepsen.util 1672 :info :write [17 9] :RequestCanceledException INFO [2019-12-17 12:29:08,493] jepsen worker 37 - jepsen.util 1387 :info :write [12 42] :RequestCanceledException INFO [2019-12-17 12:29:08,493] jepsen worker 56 - jepsen.util 1676 :info :write [18 42] :RequestCanceledException INFO [2019-12-17 12:29:08,493] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":10458,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:29:08,494] jepsen worker 54 - jepsen.util 1854 :ok :read [18 27] INFO [2019-12-17 12:29:08,494] jepsen worker 64 - jepsen.util 1504 :ok :write [21 35] INFO [2019-12-17 12:29:08,494] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":10458,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:29:08,494] jepsen worker 7 - jepsen.util 2527 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:29:08,494] jepsen worker 27 - jepsen.util 1467 :info :cas [9 [5 7]] :RequestCanceledException INFO [2019-12-17 12:29:08,494] jepsen worker 54 - jepsen.util 1854 :invoke :read [18 nil] INFO [2019-12-17 12:29:08,494] jepsen worker 52 - jepsen.util 1762 :invoke :read [17 nil] INFO [2019-12-17 12:29:08,494] jepsen worker 79 - jepsen.util 1879 :invoke :read [26 nil] INFO [2019-12-17 12:29:08,494] jepsen worker 28 - jepsen.util 1108 :fail :cas [9 [42 37]] :ValueNotSwapFrom INFO [2019-12-17 12:29:08,495] jepsen worker 64 - jepsen.util 1504 :invoke :cas [21 [36 14]] INFO [2019-12-17 12:29:08,495] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":10462,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:29:08,495] jepsen worker 7 - jepsen.util 2527 :invoke :read [2 nil] INFO [2019-12-17 12:29:08,495] jepsen worker 37 - jepsen.util 1477 :invoke :write [12 45] INFO [2019-12-17 12:29:08,495] jepsen worker 56 - jepsen.util 1766 :invoke :write [18 12] INFO [2019-12-17 12:29:08,495] jepsen worker 28 - jepsen.util 1108 :invoke :write [9 11] INFO [2019-12-17 12:29:08,495] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":10462,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:29:08,496] jepsen worker 27 - jepsen.util 1557 :invoke :write [9 18] INFO [2019-12-17 12:29:08,501] jepsen worker 74 - jepsen.util 1784 :fail :read [24 nil] :RequestTimeoutException INFO [2019-12-17 12:29:08,501] jepsen worker 19 - jepsen.util 1999 :info :write [6 29] :RequestCanceledException INFO [2019-12-17 12:29:08,501] jepsen worker 74 - jepsen.util 1784 :invoke :read [24 nil] INFO [2019-12-17 12:29:08,504] jepsen worker 19 - jepsen.util 2089 :invoke :write [6 22] INFO [2019-12-17 12:29:08,507] jepsen worker 71 - jepsen.util 1421 :ok :read [23 6] INFO [2019-12-17 12:29:08,507] jepsen worker 71 - jepsen.util 1421 :invoke :read [23 nil] INFO [2019-12-17 12:29:08,510] jepsen worker 13 - jepsen.util 1543 :fail :cas [4 [29 25]] :ValueNotSwapFrom INFO [2019-12-17 12:29:08,511] jepsen worker 13 - jepsen.util 1543 :invoke :write [4 15] INFO [2019-12-17 12:29:08,513] jepsen worker 14 - jepsen.util 1724 :ok :read [4 11] INFO [2019-12-17 12:29:08,513] jepsen worker 14 - jepsen.util 1724 :invoke :write [4 20] INFO [2019-12-17 12:29:08,514] jepsen worker 75 - jepsen.util 2685 :info :cas [25 [9 27]] :RequestCanceledException INFO [2019-12-17 12:29:08,515] jepsen worker 29 - jepsen.util 1559 :fail :cas [9 [4 37]] :ValueNotSwapFrom INFO [2019-12-17 12:29:08,515] jepsen worker 75 - jepsen.util 2775 :invoke :read [25 nil] INFO [2019-12-17 12:29:08,515] jepsen worker 29 - jepsen.util 1559 :invoke :write [9 46] INFO [2019-12-17 12:29:08,516] jepsen worker 5 - jepsen.util 1355 :info :write [1 33] :RequestCanceledException INFO [2019-12-17 12:29:08,518] jepsen worker 5 - jepsen.util 1445 :invoke :cas [1 [45 45]] INFO [2019-12-17 12:29:08,533] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:29:08,533] jepsen nemesis - couchbase.util Rebalance check count: 73 INFO [2019-12-17 12:29:08,533] jepsen nemesis - couchbase.util Rebalance stuck count: 54 INFO [2019-12-17 12:29:08,538] jepsen worker 53 - jepsen.util 1583 :ok :read [17 18] INFO [2019-12-17 12:29:08,538] jepsen worker 53 - jepsen.util 1583 :invoke :cas [17 [43 2]] INFO [2019-12-17 12:29:08,544] jepsen worker 64 - jepsen.util 1504 :fail :cas [21 [36 14]] :ValueNotSwapFrom INFO [2019-12-17 12:29:08,544] jepsen worker 64 - jepsen.util 1504 :invoke :write [21 1] INFO [2019-12-17 12:29:08,554] jepsen worker 69 - jepsen.util 1779 :info :write [23 5] :RequestCanceledException INFO [2019-12-17 12:29:08,555] jepsen worker 69 - jepsen.util 1869 :invoke :read [23 nil] INFO [2019-12-17 12:29:08,558] jepsen worker 74 - jepsen.util 1784 :ok :read [24 35] INFO [2019-12-17 12:29:08,558] jepsen worker 74 - jepsen.util 1784 :invoke :read [24 nil] INFO [2019-12-17 12:29:08,584] jepsen worker 55 - jepsen.util 2125 :ok :read [18 27] INFO [2019-12-17 12:29:08,584] jepsen worker 55 - jepsen.util 2125 :invoke :write [18 17] INFO [2019-12-17 12:29:08,598] jepsen worker 64 - jepsen.util 1504 :ok :write [21 1] INFO [2019-12-17 12:29:08,598] jepsen worker 64 - jepsen.util 1504 :invoke :read [21 nil] INFO [2019-12-17 12:29:08,621] jepsen worker 51 - jepsen.util 1761 :fail :cas [17 [36 19]] :ValueNotSwapFrom INFO [2019-12-17 12:29:08,621] jepsen worker 51 - jepsen.util 1761 :invoke :read [17 nil] INFO [2019-12-17 12:29:08,623] jepsen worker 89 - jepsen.util 1709 :info :write [29 17] :RequestCanceledException INFO [2019-12-17 12:29:08,623] jepsen worker 78 - jepsen.util 1698 :info :write [26 12] :RequestCanceledException INFO [2019-12-17 12:29:08,624] jepsen worker 78 - jepsen.util 1788 :invoke :write [26 32] INFO [2019-12-17 12:29:08,624] jepsen worker 89 - jepsen.util 1799 :invoke :write [29 41] INFO [2019-12-17 12:29:08,630] jepsen worker 64 - jepsen.util 1504 :ok :read [21 1] INFO [2019-12-17 12:29:08,630] jepsen worker 64 - jepsen.util 1504 :invoke :read [21 nil] INFO [2019-12-17 12:29:08,644] jepsen worker 28 - jepsen.util 1108 :ok :write [9 11] INFO [2019-12-17 12:29:08,644] jepsen worker 28 - jepsen.util 1108 :invoke :cas [9 [15 12]] INFO [2019-12-17 12:29:08,650] jepsen worker 64 - jepsen.util 1504 :ok :read [21 1] INFO [2019-12-17 12:29:08,650] jepsen worker 64 - jepsen.util 1504 :invoke :read [21 nil] INFO [2019-12-17 12:29:08,665] jepsen worker 71 - jepsen.util 1421 :ok :read [23 6] INFO [2019-12-17 12:29:08,666] jepsen worker 71 - jepsen.util 1421 :invoke :read [23 nil] INFO [2019-12-17 12:29:08,672] jepsen worker 80 - jepsen.util 1880 :fail :cas [26 [31 25]] :ValueNotSwapFrom INFO [2019-12-17 12:29:08,672] jepsen worker 80 - jepsen.util 1880 :invoke :read [26 nil] INFO [2019-12-17 12:29:08,675] jepsen worker 64 - jepsen.util 1504 :ok :read [21 1] INFO [2019-12-17 12:29:08,675] jepsen worker 64 - jepsen.util 1504 :invoke :read [21 nil] INFO [2019-12-17 12:29:08,712] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":10491,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:29:08,712] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":10491,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:29:08,712] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":10494,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:29:08,712] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":10494,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:29:08,712] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":10495,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:29:08,712] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":10495,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:29:08,715] jepsen worker 64 - jepsen.util 1504 :ok :read [21 1] INFO [2019-12-17 12:29:08,716] jepsen worker 64 - jepsen.util 1504 :invoke :read [21 nil] INFO [2019-12-17 12:29:08,727] jepsen worker 27 - jepsen.util 1557 :ok :write [9 18] INFO [2019-12-17 12:29:08,728] jepsen worker 27 - jepsen.util 1557 :invoke :cas [9 [23 46]] INFO [2019-12-17 12:29:08,732] jepsen worker 64 - jepsen.util 1504 :ok :read [21 1] INFO [2019-12-17 12:29:08,732] jepsen worker 64 - jepsen.util 1504 :invoke :write [21 25] INFO [2019-12-17 12:29:08,757] jepsen worker 27 - jepsen.util 1557 :fail :cas [9 [23 46]] :ValueNotSwapFrom INFO [2019-12-17 12:29:08,757] jepsen worker 27 - jepsen.util 1557 :invoke :write [9 4] INFO [2019-12-17 12:29:08,763] jepsen worker 51 - jepsen.util 1761 :ok :read [17 18] INFO [2019-12-17 12:29:08,763] jepsen worker 51 - jepsen.util 1761 :invoke :read [17 nil] INFO [2019-12-17 12:29:08,789] jepsen worker 75 - jepsen.util 2775 :ok :read [25 2] INFO [2019-12-17 12:29:08,789] jepsen worker 75 - jepsen.util 2775 :invoke :cas [25 [49 36]] INFO [2019-12-17 12:29:08,799] jepsen worker 64 - jepsen.util 1504 :ok :write [21 25] INFO [2019-12-17 12:29:08,799] jepsen worker 64 - jepsen.util 1504 :invoke :cas [21 [26 8]] INFO [2019-12-17 12:29:08,811] jepsen worker 70 - jepsen.util 2050 :ok :read [23 6] INFO [2019-12-17 12:29:08,812] jepsen worker 70 - jepsen.util 2050 :invoke :write [23 9] INFO [2019-12-17 12:29:08,821] jepsen worker 53 - jepsen.util 1583 :fail :cas [17 [43 2]] :ValueNotSwapFrom INFO [2019-12-17 12:29:08,821] jepsen worker 53 - jepsen.util 1583 :invoke :read [17 nil] INFO [2019-12-17 12:29:08,828] jepsen worker 64 - jepsen.util 1504 :fail :cas [21 [26 8]] :ValueNotSwapFrom INFO [2019-12-17 12:29:08,828] jepsen worker 64 - jepsen.util 1504 :invoke :cas [21 [28 5]] INFO [2019-12-17 12:29:08,839] jepsen worker 27 - jepsen.util 1557 :ok :write [9 4] INFO [2019-12-17 12:29:08,839] jepsen worker 27 - jepsen.util 1557 :invoke :read [9 nil] INFO [2019-12-17 12:29:08,849] jepsen worker 64 - jepsen.util 1504 :fail :cas [21 [28 5]] :ValueNotSwapFrom INFO [2019-12-17 12:29:08,849] jepsen worker 64 - jepsen.util 1504 :invoke :read [21 nil] INFO [2019-12-17 12:29:08,860] jepsen worker 69 - jepsen.util 1869 :ok :read [23 6] INFO [2019-12-17 12:29:08,860] jepsen worker 69 - jepsen.util 1869 :invoke :write [23 30] INFO [2019-12-17 12:29:08,861] jepsen worker 52 - jepsen.util 1762 :ok :read [17 18] INFO [2019-12-17 12:29:08,861] jepsen worker 52 - jepsen.util 1762 :invoke :write [17 25] INFO [2019-12-17 12:29:08,878] jepsen worker 27 - jepsen.util 1557 :ok :read [9 4] INFO [2019-12-17 12:29:08,878] jepsen worker 27 - jepsen.util 1557 :invoke :cas [9 [35 5]] INFO [2019-12-17 12:29:08,879] jepsen worker 64 - jepsen.util 1504 :ok :read [21 25] INFO [2019-12-17 12:29:08,879] jepsen worker 64 - jepsen.util 1504 :invoke :write [21 8] INFO [2019-12-17 12:29:08,888] jepsen worker 71 - jepsen.util 1421 :ok :read [23 6] INFO [2019-12-17 12:29:08,888] jepsen worker 71 - jepsen.util 1421 :invoke :write [23 0] INFO [2019-12-17 12:29:08,891] jepsen worker 3 - jepsen.util 1713 :ok :read [1 5] INFO [2019-12-17 12:29:08,891] jepsen worker 3 - jepsen.util 1713 :invoke :cas [1 [25 20]] INFO [2019-12-17 12:29:08,896] jepsen worker 27 - jepsen.util 1557 :fail :cas [9 [35 5]] :ValueNotSwapFrom INFO [2019-12-17 12:29:08,896] jepsen worker 27 - jepsen.util 1557 :invoke :write [9 41] INFO [2019-12-17 12:29:08,911] jepsen worker 88 - jepsen.util 1798 :fail :cas [29 [46 20]] :ValueNotSwapFrom INFO [2019-12-17 12:29:08,911] jepsen worker 88 - jepsen.util 1798 :invoke :cas [29 [42 9]] INFO [2019-12-17 12:29:08,923] jepsen worker 47 - jepsen.util 2207 :info :write [15 25] :RequestCanceledException INFO [2019-12-17 12:29:08,924] jepsen worker 47 - jepsen.util 2297 :invoke :write [15 43] INFO [2019-12-17 12:29:08,929] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":10514,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:29:08,929] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":10514,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:29:08,930] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":10517,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:29:08,930] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":10517,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:29:08,943] jepsen worker 64 - jepsen.util 1504 :ok :write [21 8] INFO [2019-12-17 12:29:08,943] jepsen worker 64 - jepsen.util 1504 :invoke :write [21 0] INFO [2019-12-17 12:29:08,966] jepsen worker 79 - jepsen.util 1879 :ok :read [26 43] INFO [2019-12-17 12:29:08,966] jepsen worker 79 - jepsen.util 1879 :invoke :read [26 nil] INFO [2019-12-17 12:29:08,988] jepsen worker 53 - jepsen.util 1583 :ok :read [17 18] INFO [2019-12-17 12:29:08,988] jepsen worker 53 - jepsen.util 1583 :invoke :cas [17 [41 17]] INFO [2019-12-17 12:29:09,002] jepsen worker 27 - jepsen.util 1557 :ok :write [9 41] INFO [2019-12-17 12:29:09,002] jepsen worker 27 - jepsen.util 1557 :invoke :read [9 nil] INFO [2019-12-17 12:29:09,003] jepsen worker 54 - jepsen.util 1854 :ok :read [18 27] INFO [2019-12-17 12:29:09,003] jepsen worker 75 - jepsen.util 2775 :fail :cas [25 [49 36]] :ValueNotSwapFrom INFO [2019-12-17 12:29:09,003] jepsen worker 54 - jepsen.util 1854 :invoke :write [18 22] INFO [2019-12-17 12:29:09,003] jepsen worker 75 - jepsen.util 2775 :invoke :cas [25 [19 39]] INFO [2019-12-17 12:29:09,025] jepsen worker 75 - jepsen.util 2775 :fail :cas [25 [19 39]] :ValueNotSwapFrom INFO [2019-12-17 12:29:09,030] jepsen worker 75 - jepsen.util 2775 :invoke :read [25 nil] INFO [2019-12-17 12:29:09,036] jepsen worker 36 - jepsen.util 1566 :info :write [12 4] :RequestCanceledException INFO [2019-12-17 12:29:09,038] jepsen worker 36 - jepsen.util 1656 :invoke :write [12 32] INFO [2019-12-17 12:29:09,039] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":10531,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:29:09,039] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":10531,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:29:09,039] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":10535,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:29:09,039] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":10535,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:29:09,039] jepsen worker 27 - jepsen.util 1557 :ok :read [9 41] INFO [2019-12-17 12:29:09,040] jepsen worker 27 - jepsen.util 1557 :invoke :cas [9 [46 46]] INFO [2019-12-17 12:29:09,055] jepsen worker 64 - jepsen.util 1504 :ok :write [21 0] INFO [2019-12-17 12:29:09,056] jepsen worker 64 - jepsen.util 1504 :invoke :read [21 nil] INFO [2019-12-17 12:29:09,058] jepsen worker 27 - jepsen.util 1557 :fail :cas [9 [46 46]] :ValueNotSwapFrom INFO [2019-12-17 12:29:09,058] jepsen worker 27 - jepsen.util 1557 :invoke :write [9 15] INFO [2019-12-17 12:29:09,059] jepsen worker 5 - jepsen.util 1445 :fail :cas [1 [45 45]] :ValueNotSwapFrom INFO [2019-12-17 12:29:09,059] jepsen worker 5 - jepsen.util 1445 :invoke :cas [1 [11 43]] INFO [2019-12-17 12:29:09,079] jepsen worker 64 - jepsen.util 1504 :ok :read [21 0] INFO [2019-12-17 12:29:09,079] jepsen worker 64 - jepsen.util 1504 :invoke :read [21 nil] INFO [2019-12-17 12:29:09,096] jepsen worker 64 - jepsen.util 1504 :ok :read [21 0] INFO [2019-12-17 12:29:09,096] jepsen worker 64 - jepsen.util 1504 :invoke :cas [21 [8 43]] INFO [2019-12-17 12:29:09,099] jepsen worker 51 - jepsen.util 1761 :ok :read [17 18] INFO [2019-12-17 12:29:09,099] jepsen worker 51 - jepsen.util 1761 :invoke :cas [17 [9 15]] INFO [2019-12-17 12:29:09,116] jepsen worker 64 - jepsen.util 1504 :fail :cas [21 [8 43]] :ValueNotSwapFrom INFO [2019-12-17 12:29:09,117] jepsen worker 64 - jepsen.util 1504 :invoke :cas [21 [19 22]] INFO [2019-12-17 12:29:09,133] jepsen worker 27 - jepsen.util 1557 :ok :write [9 15] INFO [2019-12-17 12:29:09,133] jepsen worker 27 - jepsen.util 1557 :invoke :write [9 33] INFO [2019-12-17 12:29:09,136] jepsen worker 87 - jepsen.util 1527 :info :write [29 39] :RequestCanceledException INFO [2019-12-17 12:29:09,137] jepsen worker 87 - jepsen.util 1617 :invoke :cas [29 [19 45]] INFO [2019-12-17 12:29:09,141] jepsen worker 64 - jepsen.util 1504 :fail :cas [21 [19 22]] :ValueNotSwapFrom INFO [2019-12-17 12:29:09,142] jepsen worker 64 - jepsen.util 1504 :invoke :write [21 31] INFO [2019-12-17 12:29:09,146] jepsen worker 3 - jepsen.util 1713 :fail :cas [1 [25 20]] :ValueNotSwapFrom INFO [2019-12-17 12:29:09,146] jepsen worker 3 - jepsen.util 1713 :invoke :read [1 nil] INFO [2019-12-17 12:29:09,166] jepsen worker 79 - jepsen.util 1879 :ok :read [26 43] INFO [2019-12-17 12:29:09,167] jepsen worker 79 - jepsen.util 1879 :invoke :cas [26 [9 9]] INFO [2019-12-17 12:29:09,178] jepsen worker 75 - jepsen.util 2775 :ok :read [25 2] INFO [2019-12-17 12:29:09,178] jepsen worker 75 - jepsen.util 2775 :invoke :read [25 nil] INFO [2019-12-17 12:29:09,189] jepsen worker 15 - jepsen.util 2175 :info :write [5 25] :RequestCanceledException INFO [2019-12-17 12:29:09,190] jepsen worker 15 - jepsen.util 2265 :invoke :cas [5 [26 43]] INFO [2019-12-17 12:29:09,194] jepsen worker 75 - jepsen.util 2775 :ok :read [25 2] INFO [2019-12-17 12:29:09,196] jepsen worker 75 - jepsen.util 2775 :invoke :read [25 nil] INFO [2019-12-17 12:29:09,199] jepsen worker 5 - jepsen.util 1445 :fail :cas [1 [11 43]] :ValueNotSwapFrom INFO [2019-12-17 12:29:09,199] jepsen worker 5 - jepsen.util 1445 :invoke :write [1 37] INFO [2019-12-17 12:29:09,207] jepsen worker 43 - jepsen.util 2473 :info :cas [14 [32 35]] :RequestCanceledException INFO [2019-12-17 12:29:09,209] jepsen worker 43 - jepsen.util 2563 :invoke :read [14 nil] INFO [2019-12-17 12:29:09,210] jepsen worker 75 - jepsen.util 2775 :ok :read [25 2] INFO [2019-12-17 12:29:09,210] jepsen worker 75 - jepsen.util 2775 :invoke :write [25 36] INFO [2019-12-17 12:29:09,223] jepsen worker 16 - jepsen.util 2176 :info :write [5 22] :RequestCanceledException INFO [2019-12-17 12:29:09,224] jepsen worker 16 - jepsen.util 2266 :invoke :cas [5 [28 37]] INFO [2019-12-17 12:29:09,242] jepsen worker 64 - jepsen.util 1504 :ok :write [21 31] INFO [2019-12-17 12:29:09,242] jepsen worker 64 - jepsen.util 1504 :invoke :cas [21 [12 46]] INFO [2019-12-17 12:29:09,250] jepsen worker 27 - jepsen.util 1557 :ok :write [9 33] INFO [2019-12-17 12:29:09,250] jepsen worker 27 - jepsen.util 1557 :invoke :write [9 1] INFO [2019-12-17 12:29:09,250] jepsen worker 61 - jepsen.util 2491 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:29:09,250] jepsen worker 58 - jepsen.util 2218 :info :cas [19 [22 41]] :RequestCanceledException INFO [2019-12-17 12:29:09,251] jepsen worker 61 - jepsen.util 2491 :invoke :write [20 49] INFO [2019-12-17 12:29:09,252] jepsen worker 58 - jepsen.util 2308 :invoke :read [19 nil] INFO [2019-12-17 12:29:09,258] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":10557,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:29:09,258] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":10566,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:29:09,258] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":10557,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:29:09,258] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":10566,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:29:09,258] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":10558,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:29:09,258] jepsen worker 64 - jepsen.util 1504 :fail :cas [21 [12 46]] :ValueNotSwapFrom INFO [2019-12-17 12:29:09,258] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":10569,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:29:09,258] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":10558,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:29:09,259] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":10569,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:29:09,259] jepsen worker 64 - jepsen.util 1504 :invoke :write [21 15] INFO [2019-12-17 12:29:09,289] jepsen worker 57 - jepsen.util 2217 :info :write [19 5] :RequestCanceledException INFO [2019-12-17 12:29:09,289] jepsen worker 81 - jepsen.util 2061 :info :cas [27 [24 44]] :RequestCanceledException INFO [2019-12-17 12:29:09,307] jepsen worker 21 - jepsen.util 2361 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:29:09,307] jepsen worker 86 - jepsen.util 2516 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:29:09,307] jepsen worker 35 - jepsen.util 2105 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:29:09,308] jepsen worker 21 - jepsen.util 2361 :invoke :read [7 nil] INFO [2019-12-17 12:29:09,308] jepsen worker 86 - jepsen.util 2516 :invoke :read [28 nil] INFO [2019-12-17 12:29:09,308] jepsen worker 35 - jepsen.util 2105 :invoke :read [11 nil] INFO [2019-12-17 12:29:09,309] jepsen worker 57 - jepsen.util 2307 :invoke :read [19 nil] INFO [2019-12-17 12:29:09,309] jepsen worker 81 - jepsen.util 2151 :invoke :write [27 7] INFO [2019-12-17 12:29:09,322] jepsen worker 26 - jepsen.util 2636 :info :write [8 11] :RequestCanceledException INFO [2019-12-17 12:29:09,322] jepsen worker 84 - jepsen.util 2154 :info :cas [28 [45 4]] :RequestCanceledException INFO [2019-12-17 12:29:09,323] jepsen worker 22 - jepsen.util 2542 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:29:09,323] jepsen worker 22 - jepsen.util 2542 :invoke :write [7 40] INFO [2019-12-17 12:29:09,323] jepsen worker 23 - jepsen.util 2183 :info :write [7 26] :RequestCanceledException INFO [2019-12-17 12:29:09,324] jepsen worker 26 - jepsen.util 2726 :invoke :read [8 nil] INFO [2019-12-17 12:29:09,324] jepsen worker 84 - jepsen.util 2244 :invoke :write [28 3] INFO [2019-12-17 12:29:09,324] jepsen worker 23 - jepsen.util 2273 :invoke :read [7 nil] INFO [2019-12-17 12:29:09,327] jepsen worker 80 - jepsen.util 1880 :ok :read [26 43] INFO [2019-12-17 12:29:09,327] jepsen worker 80 - jepsen.util 1880 :invoke :read [26 nil] INFO [2019-12-17 12:29:09,332] jepsen worker 87 - jepsen.util 1617 :fail :cas [29 [19 45]] :ValueNotSwapFrom INFO [2019-12-17 12:29:09,333] jepsen worker 87 - jepsen.util 1617 :invoke :write [29 30] INFO [2019-12-17 12:29:09,334] jepsen worker 62 - jepsen.util 2312 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:29:09,334] jepsen worker 76 - jepsen.util 2416 :info :write [25 6] :RequestCanceledException INFO [2019-12-17 12:29:09,335] jepsen worker 62 - jepsen.util 2312 :invoke :read [20 nil] INFO [2019-12-17 12:29:09,336] jepsen worker 76 - jepsen.util 2506 :invoke :read [25 nil] INFO [2019-12-17 12:29:09,353] jepsen worker 85 - jepsen.util 2605 :info :cas [28 [1 21]] :RequestCanceledException INFO [2019-12-17 12:29:09,353] jepsen worker 77 - jepsen.util 2417 :info :write [25 5] :RequestCanceledException INFO [2019-12-17 12:29:09,353] jepsen worker 40 - jepsen.util 2830 :info :cas [13 [27 12]] :RequestCanceledException INFO [2019-12-17 12:29:09,354] jepsen worker 77 - jepsen.util 2507 :invoke :read [25 nil] INFO [2019-12-17 12:29:09,354] jepsen worker 40 - jepsen.util 2920 :invoke :read [13 nil] INFO [2019-12-17 12:29:09,354] jepsen worker 85 - jepsen.util 2695 :invoke :write [28 38] INFO [2019-12-17 12:29:09,356] jepsen worker 27 - jepsen.util 1557 :ok :write [9 1] INFO [2019-12-17 12:29:09,356] jepsen worker 27 - jepsen.util 1557 :invoke :cas [9 [36 44]] INFO [2019-12-17 12:29:09,372] jepsen worker 3 - jepsen.util 1713 :ok :read [1 5] INFO [2019-12-17 12:29:09,372] jepsen worker 3 - jepsen.util 1713 :invoke :read [1 nil] INFO [2019-12-17 12:29:09,379] jepsen worker 88 - jepsen.util 1798 :fail :cas [29 [42 9]] :ValueNotSwapFrom INFO [2019-12-17 12:29:09,379] jepsen worker 88 - jepsen.util 1798 :invoke :cas [29 [43 37]] INFO [2019-12-17 12:29:09,387] jepsen worker 27 - jepsen.util 1557 :fail :cas [9 [36 44]] :ValueNotSwapFrom INFO [2019-12-17 12:29:09,388] jepsen worker 27 - jepsen.util 1557 :invoke :write [9 32] INFO [2019-12-17 12:29:09,388] jepsen worker 25 - jepsen.util 2635 :info :write [8 48] :RequestCanceledException INFO [2019-12-17 12:29:09,388] jepsen worker 82 - jepsen.util 2782 :info :write [27 49] :RequestCanceledException INFO [2019-12-17 12:29:09,388] jepsen worker 0 - jepsen.util 2520 :info :write [0 24] :RequestCanceledException INFO [2019-12-17 12:29:09,390] jepsen worker 25 - jepsen.util 2725 :invoke :write [8 29] INFO [2019-12-17 12:29:09,390] jepsen worker 0 - jepsen.util 2610 :invoke :write [0 47] INFO [2019-12-17 12:29:09,390] jepsen worker 82 - jepsen.util 2872 :invoke :cas [27 [11 22]] INFO [2019-12-17 12:29:09,407] jepsen worker 64 - jepsen.util 1504 :ok :write [21 15] INFO [2019-12-17 12:29:09,407] jepsen worker 64 - jepsen.util 1504 :invoke :read [21 nil] INFO [2019-12-17 12:29:09,407] jepsen worker 24 - jepsen.util 2094 :info :write [8 11] :RequestCanceledException INFO [2019-12-17 12:29:09,407] jepsen worker 8 - jepsen.util 2168 :info :write [2 12] :RequestCanceledException INFO [2019-12-17 12:29:09,407] jepsen worker 38 - jepsen.util 1838 :info :write [12 4] :RequestCanceledException INFO [2019-12-17 12:29:09,407] jepsen worker 50 - jepsen.util 2570 :info :write [16 36] :RequestCanceledException INFO [2019-12-17 12:29:09,408] jepsen worker 50 - jepsen.util 2660 :invoke :read [16 nil] INFO [2019-12-17 12:29:09,408] jepsen worker 8 - jepsen.util 2258 :invoke :read [2 nil] INFO [2019-12-17 12:29:09,408] jepsen worker 24 - jepsen.util 2184 :invoke :cas [8 [46 13]] INFO [2019-12-17 12:29:09,408] jepsen worker 38 - jepsen.util 1928 :invoke :write [12 3] INFO [2019-12-17 12:29:09,422] jepsen worker 64 - jepsen.util 1504 :ok :read [21 15] INFO [2019-12-17 12:29:09,422] jepsen worker 1 - jepsen.util 2341 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:29:09,422] jepsen worker 59 - jepsen.util 2039 :info :cas [19 [2 29]] :RequestCanceledException INFO [2019-12-17 12:29:09,422] jepsen worker 64 - jepsen.util 1504 :invoke :write [21 1] INFO [2019-12-17 12:29:09,422] jepsen worker 1 - jepsen.util 2341 :invoke :write [0 46] INFO [2019-12-17 12:29:09,422] jepsen worker 66 - jepsen.util 2136 :info :write [22 44] :RequestCanceledException INFO [2019-12-17 12:29:09,423] jepsen worker 59 - jepsen.util 2129 :invoke :write [19 49] INFO [2019-12-17 12:29:09,423] jepsen worker 66 - jepsen.util 2226 :invoke :cas [22 [43 18]] INFO [2019-12-17 12:29:09,436] jepsen worker 17 - jepsen.util 2447 :info :write [5 2] :RequestCanceledException INFO [2019-12-17 12:29:09,436] jepsen worker 41 - jepsen.util 2561 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:29:09,436] jepsen worker 41 - jepsen.util 2561 :invoke :read [13 nil] INFO [2019-12-17 12:29:09,437] jepsen worker 17 - jepsen.util 2537 :invoke :write [5 47] INFO [2019-12-17 12:29:09,439] jepsen worker 79 - jepsen.util 1879 :fail :cas [26 [9 9]] :ValueNotSwapFrom INFO [2019-12-17 12:29:09,439] jepsen worker 79 - jepsen.util 1879 :invoke :read [26 nil] INFO [2019-12-17 12:29:09,441] jepsen worker 46 - jepsen.util 1936 :ok :write [15 31] INFO [2019-12-17 12:29:09,441] jepsen worker 46 - jepsen.util 1936 :invoke :cas [15 [41 1]] INFO [2019-12-17 12:29:09,453] jepsen worker 68 - jepsen.util 2048 :info :cas [22 [25 30]] :RequestCanceledException INFO [2019-12-17 12:29:09,454] jepsen worker 68 - jepsen.util 2138 :invoke :write [22 42] INFO [2019-12-17 12:29:09,456] jepsen worker 20 - jepsen.util 1640 :ok :write [6 25] INFO [2019-12-17 12:29:09,456] jepsen worker 20 - jepsen.util 1640 :invoke :read [6 nil] INFO [2019-12-17 12:29:09,456] jepsen worker 32 - jepsen.util 2192 :ok :write [10 48] INFO [2019-12-17 12:29:09,457] jepsen worker 32 - jepsen.util 2192 :invoke :read [10 nil] INFO [2019-12-17 12:29:09,458] jepsen worker 27 - jepsen.util 1557 :ok :write [9 32] INFO [2019-12-17 12:29:09,458] jepsen worker 27 - jepsen.util 1557 :invoke :read [9 nil] INFO [2019-12-17 12:29:09,460] jepsen worker 30 - jepsen.util 2100 :ok :read [10 16] INFO [2019-12-17 12:29:09,460] jepsen worker 30 - jepsen.util 2100 :invoke :write [10 26] INFO [2019-12-17 12:29:09,482] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":10611,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:29:09,482] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":10611,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:29:09,482] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":10613,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:29:09,482] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":10613,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:29:09,483] jepsen worker 3 - jepsen.util 1713 :ok :read [1 5] INFO [2019-12-17 12:29:09,483] jepsen worker 27 - jepsen.util 1557 :ok :read [9 32] INFO [2019-12-17 12:29:09,483] jepsen worker 3 - jepsen.util 1713 :invoke :cas [1 [8 17]] INFO [2019-12-17 12:29:09,483] jepsen worker 27 - jepsen.util 1557 :invoke :cas [9 [18 22]] INFO [2019-12-17 12:29:09,489] jepsen worker 83 - jepsen.util 2693 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:29:09,489] jepsen worker 83 - jepsen.util 2693 :invoke :cas [27 [41 45]] INFO [2019-12-17 12:29:09,501] jepsen worker 77 - jepsen.util 2507 :ok :read [25 2] INFO [2019-12-17 12:29:09,501] jepsen worker 77 - jepsen.util 2507 :invoke :cas [25 [3 10]] INFO [2019-12-17 12:29:09,503] jepsen worker 27 - jepsen.util 1557 :fail :cas [9 [18 22]] :ValueNotSwapFrom INFO [2019-12-17 12:29:09,503] jepsen worker 27 - jepsen.util 1557 :invoke :write [9 22] INFO [2019-12-17 12:29:09,506] jepsen worker 39 - jepsen.util 2559 :info :write [13 29] :RequestCanceledException INFO [2019-12-17 12:29:09,507] jepsen worker 39 - jepsen.util 2649 :invoke :cas [13 [49 11]] INFO [2019-12-17 12:29:09,511] jepsen worker 46 - jepsen.util 1936 :fail :cas [15 [41 1]] :ValueNotSwapFrom INFO [2019-12-17 12:29:09,511] jepsen worker 46 - jepsen.util 1936 :invoke :cas [15 [42 1]] INFO [2019-12-17 12:29:09,521] jepsen worker 65 - jepsen.util 1595 :ok :read [21 15] INFO [2019-12-17 12:29:09,521] jepsen worker 65 - jepsen.util 1595 :invoke :write [21 30] INFO [2019-12-17 12:29:09,522] jepsen worker 42 - jepsen.util 2832 :info :write [14 12] :RequestCanceledException INFO [2019-12-17 12:29:09,523] jepsen worker 42 - jepsen.util 2922 :invoke :read [14 nil] INFO [2019-12-17 12:29:09,530] jepsen worker 77 - jepsen.util 2507 :fail :cas [25 [3 10]] :ValueNotSwapFrom INFO [2019-12-17 12:29:09,530] jepsen worker 77 - jepsen.util 2507 :invoke :cas [25 [45 25]] INFO [2019-12-17 12:29:09,543] jepsen worker 77 - jepsen.util 2507 :fail :cas [25 [45 25]] :ValueNotSwapFrom INFO [2019-12-17 12:29:09,543] jepsen worker 77 - jepsen.util 2507 :invoke :read [25 nil] INFO [2019-12-17 12:29:09,551] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:29:09,551] jepsen nemesis - couchbase.util Rebalance check count: 74 INFO [2019-12-17 12:29:09,551] jepsen nemesis - couchbase.util Rebalance stuck count: 55 INFO [2019-12-17 12:29:09,554] jepsen worker 48 - jepsen.util 2028 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:29:09,554] jepsen worker 48 - jepsen.util 2028 :invoke :cas [16 [24 30]] INFO [2019-12-17 12:29:09,559] jepsen worker 77 - jepsen.util 2507 :ok :read [25 2] INFO [2019-12-17 12:29:09,559] jepsen worker 77 - jepsen.util 2507 :invoke :write [25 35] INFO [2019-12-17 12:29:09,564] jepsen worker 20 - jepsen.util 1640 :ok :read [6 25] INFO [2019-12-17 12:29:09,564] jepsen worker 20 - jepsen.util 1640 :invoke :read [6 nil] INFO [2019-12-17 12:29:09,580] jepsen worker 3 - jepsen.util 1713 :fail :cas [1 [8 17]] :ValueNotSwapFrom INFO [2019-12-17 12:29:09,580] jepsen worker 3 - jepsen.util 1713 :invoke :read [1 nil] INFO [2019-12-17 12:29:09,591] jepsen worker 79 - jepsen.util 1879 :ok :read [26 43] INFO [2019-12-17 12:29:09,591] jepsen worker 79 - jepsen.util 1879 :invoke :read [26 nil] INFO [2019-12-17 12:29:09,614] jepsen worker 64 - jepsen.util 1504 :ok :write [21 1] INFO [2019-12-17 12:29:09,614] jepsen worker 64 - jepsen.util 1504 :invoke :write [21 46] INFO [2019-12-17 12:29:09,620] jepsen worker 20 - jepsen.util 1640 :ok :read [6 25] INFO [2019-12-17 12:29:09,621] jepsen worker 20 - jepsen.util 1640 :invoke :write [6 17] INFO [2019-12-17 12:29:09,622] jepsen worker 33 - jepsen.util 2733 :info :write [11 26] :RequestCanceledException INFO [2019-12-17 12:29:09,622] jepsen worker 34 - jepsen.util 2464 :info :write [11 28] :RequestCanceledException INFO [2019-12-17 12:29:09,623] jepsen worker 33 - jepsen.util 2823 :invoke :cas [11 [32 8]] INFO [2019-12-17 12:29:09,623] jepsen worker 34 - jepsen.util 2554 :invoke :read [11 nil] INFO [2019-12-17 12:29:09,676] jepsen worker 80 - jepsen.util 1880 :ok :read [26 43] INFO [2019-12-17 12:29:09,676] jepsen worker 80 - jepsen.util 1880 :invoke :cas [26 [23 48]] INFO [2019-12-17 12:29:09,684] jepsen worker 30 - jepsen.util 2100 :ok :write [10 26] INFO [2019-12-17 12:29:09,684] jepsen worker 30 - jepsen.util 2100 :invoke :write [10 37] INFO [2019-12-17 12:29:09,684] jepsen worker 47 - jepsen.util 2297 :ok :write [15 43] INFO [2019-12-17 12:29:09,684] jepsen worker 47 - jepsen.util 2297 :invoke :read [15 nil] INFO [2019-12-17 12:29:09,690] jepsen worker 67 - jepsen.util 2767 :info :write [22 24] :RequestCanceledException INFO [2019-12-17 12:29:09,691] jepsen worker 67 - jepsen.util 2857 :invoke :write [22 44] INFO [2019-12-17 12:29:09,697] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":10647,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:29:09,697] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":10647,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:29:09,697] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":10650,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:29:09,697] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":10650,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:29:09,699] jepsen worker 32 - jepsen.util 2192 :ok :read [10 26] INFO [2019-12-17 12:29:09,699] jepsen worker 32 - jepsen.util 2192 :invoke :cas [10 [26 4]] INFO [2019-12-17 12:29:09,701] jepsen worker 27 - jepsen.util 1557 :ok :write [9 22] INFO [2019-12-17 12:29:09,701] jepsen worker 27 - jepsen.util 1557 :invoke :write [9 28] INFO [2019-12-17 12:29:09,711] jepsen worker 3 - jepsen.util 1713 :ok :read [1 5] INFO [2019-12-17 12:29:09,711] jepsen worker 3 - jepsen.util 1713 :invoke :read [1 nil] INFO [2019-12-17 12:29:09,714] jepsen worker 53 - jepsen.util 1583 :fail :cas [17 [41 17]] :ValueNotSwapFrom INFO [2019-12-17 12:29:09,714] jepsen worker 53 - jepsen.util 1583 :invoke :cas [17 [40 9]] INFO [2019-12-17 12:29:09,723] jepsen worker 31 - jepsen.util 1831 :info :write [10 12] :RequestCanceledException INFO [2019-12-17 12:29:09,723] jepsen worker 4 - jepsen.util 1354 :info :write [1 16] :RequestCanceledException INFO [2019-12-17 12:29:09,725] jepsen worker 4 - jepsen.util 1444 :invoke :write [1 10] INFO [2019-12-17 12:29:09,725] jepsen worker 47 - jepsen.util 2297 :ok :read [15 43] INFO [2019-12-17 12:29:09,725] jepsen worker 47 - jepsen.util 2297 :invoke :cas [15 [8 42]] INFO [2019-12-17 12:29:09,726] jepsen worker 31 - jepsen.util 1921 :invoke :read [10 nil] INFO [2019-12-17 12:29:09,780] jepsen worker 64 - jepsen.util 1504 :ok :write [21 46] INFO [2019-12-17 12:29:09,780] jepsen worker 64 - jepsen.util 1504 :invoke :cas [21 [41 28]] INFO [2019-12-17 12:29:09,791] jepsen worker 47 - jepsen.util 2297 :fail :cas [15 [8 42]] :ValueNotSwapFrom INFO [2019-12-17 12:29:09,791] jepsen worker 47 - jepsen.util 2297 :invoke :cas [15 [18 43]] INFO [2019-12-17 12:29:09,807] jepsen worker 11 - jepsen.util 2261 :info :write [3 30] :RequestCanceledException INFO [2019-12-17 12:29:09,808] jepsen worker 11 - jepsen.util 2351 :invoke :read [3 nil] INFO [2019-12-17 12:29:09,813] jepsen worker 31 - jepsen.util 1921 :ok :read [10 26] INFO [2019-12-17 12:29:09,813] jepsen worker 31 - jepsen.util 1921 :invoke :cas [10 [12 14]] INFO [2019-12-17 12:29:09,823] jepsen worker 63 - jepsen.util 1593 :info :write [21 13] :RequestCanceledException INFO [2019-12-17 12:29:09,824] jepsen worker 63 - jepsen.util 1683 :invoke :write [21 6] INFO [2019-12-17 12:29:09,854] jepsen worker 64 - jepsen.util 1504 :fail :cas [21 [41 28]] :ValueNotSwapFrom INFO [2019-12-17 12:29:09,854] jepsen worker 64 - jepsen.util 1504 :invoke :cas [21 [36 13]] INFO [2019-12-17 12:29:09,877] jepsen worker 51 - jepsen.util 1761 :fail :cas [17 [9 15]] :ValueNotSwapFrom INFO [2019-12-17 12:29:09,877] jepsen worker 51 - jepsen.util 1761 :invoke :read [17 nil] INFO [2019-12-17 12:29:09,886] jepsen worker 27 - jepsen.util 1557 :ok :write [9 28] INFO [2019-12-17 12:29:09,886] jepsen worker 27 - jepsen.util 1557 :invoke :cas [9 [3 16]] INFO [2019-12-17 12:29:09,886] jepsen worker 2 - jepsen.util 2162 :info :cas [0 [18 13]] :RequestCanceledException INFO [2019-12-17 12:29:09,886] jepsen worker 44 - jepsen.util 2024 :info :write [14 34] :RequestCanceledException INFO [2019-12-17 12:29:09,887] jepsen worker 44 - jepsen.util 2114 :invoke :read [14 nil] INFO [2019-12-17 12:29:09,887] jepsen worker 2 - jepsen.util 2252 :invoke :write [0 20] INFO [2019-12-17 12:29:09,892] jepsen worker 31 - jepsen.util 1921 :fail :cas [10 [12 14]] :ValueNotSwapFrom INFO [2019-12-17 12:29:09,892] jepsen worker 31 - jepsen.util 1921 :invoke :write [10 16] INFO [2019-12-17 12:29:09,902] jepsen worker 64 - jepsen.util 1504 :fail :cas [21 [36 13]] :ValueNotSwapFrom INFO [2019-12-17 12:29:09,902] jepsen worker 64 - jepsen.util 1504 :invoke :write [21 27] INFO [2019-12-17 12:29:09,906] jepsen worker 18 - jepsen.util 1998 :info :write [6 30] :RequestCanceledException INFO [2019-12-17 12:29:09,908] jepsen worker 18 - jepsen.util 2088 :invoke :write [6 28] INFO [2019-12-17 12:29:09,916] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":10671,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:29:09,916] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":10671,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:29:09,916] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":10675,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:29:09,916] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":10675,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:29:09,918] jepsen worker 87 - jepsen.util 1617 :ok :write [29 30] INFO [2019-12-17 12:29:09,918] jepsen worker 87 - jepsen.util 1617 :invoke :read [29 nil] INFO [2019-12-17 12:29:09,922] jepsen worker 10 - jepsen.util 1990 :fail :read [3 nil] :RequestTimeoutException INFO [2019-12-17 12:29:09,922] jepsen worker 10 - jepsen.util 1990 :invoke :read [3 nil] INFO [2019-12-17 12:29:09,945] jepsen worker 88 - jepsen.util 1798 :fail :cas [29 [43 37]] :ValueNotSwapFrom INFO [2019-12-17 12:29:09,945] jepsen worker 88 - jepsen.util 1798 :invoke :write [29 9] INFO [2019-12-17 12:29:09,953] jepsen worker 9 - jepsen.util 1899 :info :write [3 15] :RequestCanceledException INFO [2019-12-17 12:29:09,954] jepsen worker 9 - jepsen.util 1989 :invoke :cas [3 [2 7]] INFO [2019-12-17 12:29:09,958] jepsen worker 10 - jepsen.util 1990 :ok :read [3 47] INFO [2019-12-17 12:29:09,958] jepsen worker 10 - jepsen.util 1990 :invoke :write [3 29] INFO [2019-12-17 12:29:09,992] jepsen worker 79 - jepsen.util 1879 :ok :read [26 43] INFO [2019-12-17 12:29:09,992] jepsen worker 79 - jepsen.util 1879 :invoke :read [26 nil] INFO [2019-12-17 12:29:10,007] jepsen worker 49 - jepsen.util 1669 :info :write [16 25] :RequestCanceledException INFO [2019-12-17 12:29:10,008] jepsen worker 49 - jepsen.util 1759 :invoke :write [16 2] INFO [2019-12-17 12:29:10,020] jepsen worker 63 - jepsen.util 1683 :ok :write [21 6] INFO [2019-12-17 12:29:10,020] jepsen worker 63 - jepsen.util 1683 :invoke :read [21 nil] INFO [2019-12-17 12:29:10,021] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":10689,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:29:10,022] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":10689,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:29:10,022] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":10692,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:29:10,022] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":10692,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:29:10,022] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":10693,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:29:10,022] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":10693,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:29:10,027] jepsen worker 30 - jepsen.util 2100 :ok :write [10 37] INFO [2019-12-17 12:29:10,027] jepsen worker 30 - jepsen.util 2100 :invoke :cas [10 [32 27]] INFO [2019-12-17 12:29:10,036] jepsen worker 45 - jepsen.util 2115 :fail :read [15 nil] :RequestTimeoutException INFO [2019-12-17 12:29:10,036] jepsen worker 45 - jepsen.util 2115 :invoke :write [15 45] INFO [2019-12-17 12:29:10,053] jepsen worker 12 - jepsen.util 1902 :ok :write [4 28] INFO [2019-12-17 12:29:10,053] jepsen worker 12 - jepsen.util 1902 :invoke :read [4 nil] INFO [2019-12-17 12:29:10,055] jepsen worker 63 - jepsen.util 1683 :ok :read [21 6] INFO [2019-12-17 12:29:10,055] jepsen worker 63 - jepsen.util 1683 :invoke :write [21 9] INFO [2019-12-17 12:29:10,070] jepsen worker 20 - jepsen.util 1640 :ok :write [6 17] INFO [2019-12-17 12:29:10,070] jepsen worker 20 - jepsen.util 1640 :invoke :read [6 nil] INFO [2019-12-17 12:29:10,091] jepsen worker 20 - jepsen.util 1640 :ok :read [6 17] INFO [2019-12-17 12:29:10,091] jepsen worker 20 - jepsen.util 1640 :invoke :write [6 3] INFO [2019-12-17 12:29:10,100] jepsen worker 88 - jepsen.util 1798 :ok :write [29 9] INFO [2019-12-17 12:29:10,101] jepsen worker 88 - jepsen.util 1798 :invoke :cas [29 [11 8]] INFO [2019-12-17 12:29:10,124] jepsen worker 69 - jepsen.util 1869 :ok :write [23 30] INFO [2019-12-17 12:29:10,124] jepsen worker 69 - jepsen.util 1869 :invoke :write [23 10] INFO [2019-12-17 12:29:10,202] jepsen worker 80 - jepsen.util 1880 :fail :cas [26 [23 48]] :ValueNotSwapFrom INFO [2019-12-17 12:29:10,202] jepsen worker 80 - jepsen.util 1880 :invoke :cas [26 [32 41]] INFO [2019-12-17 12:29:10,211] jepsen worker 20 - jepsen.util 1640 :ok :write [6 3] INFO [2019-12-17 12:29:10,211] jepsen worker 20 - jepsen.util 1640 :invoke :cas [6 [19 16]] INFO [2019-12-17 12:29:10,216] jepsen worker 10 - jepsen.util 1990 :ok :write [3 29] INFO [2019-12-17 12:29:10,217] jepsen worker 10 - jepsen.util 1990 :invoke :read [3 nil] INFO [2019-12-17 12:29:10,229] jepsen worker 20 - jepsen.util 1640 :fail :cas [6 [19 16]] :ValueNotSwapFrom INFO [2019-12-17 12:29:10,229] jepsen worker 20 - jepsen.util 1640 :invoke :cas [6 [19 14]] INFO [2019-12-17 12:29:10,236] jepsen worker 73 - jepsen.util 1783 :info :cas [24 [17 39]] :RequestCanceledException INFO [2019-12-17 12:29:10,237] jepsen worker 73 - jepsen.util 1873 :invoke :write [24 47] INFO [2019-12-17 12:29:10,245] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":10705,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:29:10,245] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":10713,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:29:10,246] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":10705,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:29:10,246] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":10713,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:29:10,246] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":10717,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:29:10,246] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":10717,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:29:10,247] jepsen worker 10 - jepsen.util 1990 :ok :read [3 29] INFO [2019-12-17 12:29:10,247] jepsen worker 10 - jepsen.util 1990 :invoke :write [3 26] INFO [2019-12-17 12:29:10,251] jepsen worker 20 - jepsen.util 1640 :fail :cas [6 [19 14]] :ValueNotSwapFrom INFO [2019-12-17 12:29:10,251] jepsen worker 20 - jepsen.util 1640 :invoke :write [6 49] INFO [2019-12-17 12:29:10,304] jepsen worker 6 - jepsen.util 2436 :info :write [2 33] :RequestCanceledException INFO [2019-12-17 12:29:10,304] jepsen worker 72 - jepsen.util 1962 :info :write [24 15] :RequestCanceledException INFO [2019-12-17 12:29:10,306] jepsen worker 72 - jepsen.util 2052 :invoke :read [24 nil] INFO [2019-12-17 12:29:10,306] jepsen worker 6 - jepsen.util 2526 :invoke :write [2 14] INFO [2019-12-17 12:29:10,315] jepsen worker 12 - jepsen.util 1902 :ok :read [4 28] INFO [2019-12-17 12:29:10,316] jepsen worker 12 - jepsen.util 1902 :invoke :write [4 37] INFO [2019-12-17 12:29:10,322] jepsen worker 72 - jepsen.util 2052 :ok :read [24 35] INFO [2019-12-17 12:29:10,322] jepsen worker 72 - jepsen.util 2052 :invoke :cas [24 [48 2]] INFO [2019-12-17 12:29:10,334] jepsen worker 72 - jepsen.util 2052 :fail :cas [24 [48 2]] :ValueNotSwapFrom INFO [2019-12-17 12:29:10,334] jepsen worker 72 - jepsen.util 2052 :invoke :read [24 nil] INFO [2019-12-17 12:29:10,334] jepsen worker 60 - jepsen.util 2490 :info :cas [20 [45 19]] :RequestCanceledException INFO [2019-12-17 12:29:10,335] jepsen worker 60 - jepsen.util 2580 :invoke :read [20 nil] INFO [2019-12-17 12:29:10,345] jepsen worker 72 - jepsen.util 2052 :ok :read [24 35] INFO [2019-12-17 12:29:10,345] jepsen worker 72 - jepsen.util 2052 :invoke :write [24 21] INFO [2019-12-17 12:29:10,368] jepsen worker 3 - jepsen.util 1713 :ok :read [1 5] INFO [2019-12-17 12:29:10,369] jepsen worker 3 - jepsen.util 1713 :invoke :read [1 nil] INFO [2019-12-17 12:29:10,405] jepsen worker 53 - jepsen.util 1583 :fail :cas [17 [40 9]] :ValueNotSwapFrom INFO [2019-12-17 12:29:10,405] jepsen worker 53 - jepsen.util 1583 :invoke :cas [17 [25 45]] INFO [2019-12-17 12:29:10,418] jepsen worker 51 - jepsen.util 1761 :ok :read [17 18] INFO [2019-12-17 12:29:10,418] jepsen worker 51 - jepsen.util 1761 :invoke :read [17 nil] INFO [2019-12-17 12:29:10,418] jepsen worker 80 - jepsen.util 1880 :fail :cas [26 [32 41]] :ValueNotSwapFrom INFO [2019-12-17 12:29:10,418] jepsen worker 80 - jepsen.util 1880 :invoke :write [26 38] INFO [2019-12-17 12:29:10,469] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":10736,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:29:10,469] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":10736,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:29:10,469] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":10739,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:29:10,469] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":10739,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:29:10,469] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":10740,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:29:10,469] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":10740,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:29:10,505] jepsen worker 87 - jepsen.util 1617 :ok :read [29 9] INFO [2019-12-17 12:29:10,506] jepsen worker 87 - jepsen.util 1617 :invoke :read [29 nil] INFO [2019-12-17 12:29:10,519] jepsen worker 56 - jepsen.util 1766 :info :write [18 12] :RequestCanceledException INFO [2019-12-17 12:29:10,520] jepsen worker 56 - jepsen.util 1856 :invoke :cas [18 [5 1]] INFO [2019-12-17 12:29:10,553] jepsen worker 19 - jepsen.util 2089 :info :write [6 22] :RequestCanceledException INFO [2019-12-17 12:29:10,553] jepsen worker 29 - jepsen.util 1559 :info :write [9 46] :RequestCanceledException INFO [2019-12-17 12:29:10,553] jepsen worker 7 - jepsen.util 2527 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:29:10,553] jepsen worker 14 - jepsen.util 1724 :info :write [4 20] :RequestCanceledException INFO [2019-12-17 12:29:10,553] jepsen worker 7 - jepsen.util 2527 :invoke :read [2 nil] INFO [2019-12-17 12:29:10,554] jepsen worker 14 - jepsen.util 1814 :invoke :read [4 nil] INFO [2019-12-17 12:29:10,555] jepsen worker 19 - jepsen.util 2179 :invoke :read [6 nil] INFO [2019-12-17 12:29:10,558] jepsen worker 29 - jepsen.util 1649 :invoke :read [9 nil] INFO [2019-12-17 12:29:10,566] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:29:10,566] jepsen nemesis - couchbase.util Rebalance check count: 75 INFO [2019-12-17 12:29:10,566] jepsen nemesis - couchbase.util Rebalance stuck count: 56 INFO [2019-12-17 12:29:10,566] jepsen worker 79 - jepsen.util 1879 :ok :read [26 43] INFO [2019-12-17 12:29:10,566] jepsen worker 79 - jepsen.util 1879 :invoke :cas [26 [12 21]] INFO [2019-12-17 12:29:10,570] jepsen worker 88 - jepsen.util 1798 :fail :cas [29 [11 8]] :ValueNotSwapFrom INFO [2019-12-17 12:29:10,570] jepsen worker 88 - jepsen.util 1798 :invoke :write [29 32] INFO [2019-12-17 12:29:10,606] jepsen worker 37 - jepsen.util 1477 :info :write [12 45] :RequestCanceledException INFO [2019-12-17 12:29:10,606] jepsen worker 74 - jepsen.util 1784 :fail :read [24 nil] :RequestTimeoutException INFO [2019-12-17 12:29:10,606] jepsen worker 74 - jepsen.util 1784 :invoke :cas [24 [10 41]] INFO [2019-12-17 12:29:10,607] jepsen worker 37 - jepsen.util 1567 :invoke :read [12 nil] INFO [2019-12-17 12:29:10,608] jepsen worker 53 - jepsen.util 1583 :fail :cas [17 [25 45]] :ValueNotSwapFrom INFO [2019-12-17 12:29:10,608] jepsen worker 53 - jepsen.util 1583 :invoke :read [17 nil] INFO [2019-12-17 12:29:10,618] jepsen worker 74 - jepsen.util 1784 :fail :cas [24 [10 41]] :ValueNotSwapFrom INFO [2019-12-17 12:29:10,618] jepsen worker 74 - jepsen.util 1784 :invoke :write [24 19] INFO [2019-12-17 12:29:10,622] jepsen worker 13 - jepsen.util 1543 :info :write [4 15] :RequestCanceledException INFO [2019-12-17 12:29:10,623] jepsen worker 13 - jepsen.util 1633 :invoke :cas [4 [17 45]] INFO [2019-12-17 12:29:10,636] jepsen worker 55 - jepsen.util 2125 :info :write [18 17] :RequestCanceledException INFO [2019-12-17 12:29:10,637] jepsen worker 55 - jepsen.util 2215 :invoke :cas [18 [31 30]] INFO [2019-12-17 12:29:10,672] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":10756,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:29:10,672] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":10756,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:29:10,672] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":10760,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:29:10,672] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":10760,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:29:10,677] jepsen worker 20 - jepsen.util 1640 :ok :write [6 49] INFO [2019-12-17 12:29:10,677] jepsen worker 20 - jepsen.util 1640 :invoke :cas [6 [26 19]] INFO [2019-12-17 12:29:10,677] jepsen worker 10 - jepsen.util 1990 :ok :write [3 26] INFO [2019-12-17 12:29:10,677] jepsen worker 10 - jepsen.util 1990 :invoke :write [3 0] INFO [2019-12-17 12:29:10,686] jepsen worker 56 - jepsen.util 1856 :fail :cas [18 [5 1]] :ValueNotSwapFrom INFO [2019-12-17 12:29:10,686] jepsen worker 56 - jepsen.util 1856 :invoke :cas [18 [24 31]] INFO [2019-12-17 12:29:10,698] jepsen worker 3 - jepsen.util 1713 :ok :read [1 5] INFO [2019-12-17 12:29:10,698] jepsen worker 3 - jepsen.util 1713 :invoke :cas [1 [19 6]] INFO [2019-12-17 12:29:10,704] jepsen worker 45 - jepsen.util 2115 :ok :write [15 45] INFO [2019-12-17 12:29:10,704] jepsen worker 45 - jepsen.util 2115 :invoke :write [15 17] INFO [2019-12-17 12:29:10,706] jepsen worker 72 - jepsen.util 2052 :ok :write [24 21] INFO [2019-12-17 12:29:10,706] jepsen worker 72 - jepsen.util 2052 :invoke :cas [24 [9 5]] INFO [2019-12-17 12:29:10,708] jepsen worker 20 - jepsen.util 1640 :fail :cas [6 [26 19]] :ValueNotSwapFrom INFO [2019-12-17 12:29:10,709] jepsen worker 20 - jepsen.util 1640 :invoke :write [6 13] INFO [2019-12-17 12:29:10,711] jepsen worker 51 - jepsen.util 1761 :ok :read [17 18] INFO [2019-12-17 12:29:10,711] jepsen worker 51 - jepsen.util 1761 :invoke :write [17 26] INFO [2019-12-17 12:29:10,722] jepsen worker 78 - jepsen.util 1788 :info :write [26 32] :RequestCanceledException INFO [2019-12-17 12:29:10,723] jepsen worker 78 - jepsen.util 1878 :invoke :write [26 11] INFO [2019-12-17 12:29:10,735] jepsen worker 29 - jepsen.util 1649 :ok :read [9 28] INFO [2019-12-17 12:29:10,735] jepsen worker 29 - jepsen.util 1649 :invoke :write [9 8] INFO [2019-12-17 12:29:10,736] jepsen worker 89 - jepsen.util 1799 :info :write [29 41] :RequestCanceledException INFO [2019-12-17 12:29:10,736] jepsen worker 28 - jepsen.util 1108 :info :cas [9 [15 12]] :RequestCanceledException INFO [2019-12-17 12:29:10,737] jepsen worker 28 - jepsen.util 1198 :invoke :read [9 nil] INFO [2019-12-17 12:29:10,737] jepsen worker 89 - jepsen.util 1889 :invoke :cas [29 [9 35]] INFO [2019-12-17 12:29:10,748] jepsen worker 72 - jepsen.util 2052 :fail :cas [24 [9 5]] :ValueNotSwapFrom INFO [2019-12-17 12:29:10,748] jepsen worker 72 - jepsen.util 2052 :invoke :cas [24 [43 29]] INFO [2019-12-17 12:29:10,816] jepsen worker 29 - jepsen.util 1649 :ok :write [9 8] INFO [2019-12-17 12:29:10,817] jepsen worker 29 - jepsen.util 1649 :invoke :write [9 26] INFO [2019-12-17 12:29:10,819] jepsen worker 73 - jepsen.util 1873 :ok :write [24 47] INFO [2019-12-17 12:29:10,819] jepsen worker 73 - jepsen.util 1873 :invoke :read [24 nil] INFO [2019-12-17 12:29:10,834] jepsen worker 70 - jepsen.util 2050 :info :write [23 9] :RequestCanceledException INFO [2019-12-17 12:29:10,835] jepsen worker 70 - jepsen.util 2140 :invoke :read [23 nil] INFO [2019-12-17 12:29:10,884] jepsen worker 10 - jepsen.util 1990 :ok :write [3 0] INFO [2019-12-17 12:29:10,884] jepsen worker 10 - jepsen.util 1990 :invoke :write [3 2] INFO [2019-12-17 12:29:10,891] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":10780,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:29:10,891] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":10780,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:29:10,891] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":10782,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:29:10,891] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":10782,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:29:10,913] jepsen worker 29 - jepsen.util 1649 :ok :write [9 26] INFO [2019-12-17 12:29:10,913] jepsen worker 29 - jepsen.util 1649 :invoke :write [9 46] INFO [2019-12-17 12:29:10,922] jepsen worker 52 - jepsen.util 1762 :info :write [17 25] :RequestCanceledException INFO [2019-12-17 12:29:10,923] jepsen worker 52 - jepsen.util 1852 :invoke :write [17 11] INFO [2019-12-17 12:29:10,953] jepsen worker 71 - jepsen.util 1421 :info :write [23 0] :RequestCanceledException INFO [2019-12-17 12:29:10,954] jepsen worker 71 - jepsen.util 1511 :invoke :write [23 23] INFO [2019-12-17 12:29:10,956] jepsen worker 20 - jepsen.util 1640 :ok :write [6 13] INFO [2019-12-17 12:29:10,956] jepsen worker 20 - jepsen.util 1640 :invoke :read [6 nil] INFO [2019-12-17 12:29:10,977] jepsen worker 20 - jepsen.util 1640 :ok :read [6 13] INFO [2019-12-17 12:29:10,977] jepsen worker 20 - jepsen.util 1640 :invoke :read [6 nil] INFO [2019-12-17 12:29:10,991] jepsen worker 20 - jepsen.util 1640 :ok :read [6 13] INFO [2019-12-17 12:29:10,992] jepsen worker 20 - jepsen.util 1640 :invoke :read [6 nil] INFO [2019-12-17 12:29:10,993] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":10790,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:29:10,993] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":10790,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:29:10,993] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":10793,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:29:10,993] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":10793,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:29:10,993] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":10794,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:29:10,993] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":10794,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:29:10,999] jepsen worker 29 - jepsen.util 1649 :ok :write [9 46] INFO [2019-12-17 12:29:11,000] jepsen worker 29 - jepsen.util 1649 :invoke :read [9 nil] INFO [2019-12-17 12:29:11,006] jepsen worker 14 - jepsen.util 1814 :ok :read [4 28] INFO [2019-12-17 12:29:11,006] jepsen worker 14 - jepsen.util 1814 :invoke :write [4 25] INFO [2019-12-17 12:29:11,023] jepsen worker 54 - jepsen.util 1854 :info :write [18 22] :RequestCanceledException INFO [2019-12-17 12:29:11,025] jepsen worker 54 - jepsen.util 1944 :invoke :write [18 11] INFO [2019-12-17 12:29:11,028] jepsen worker 20 - jepsen.util 1640 :ok :read [6 13] INFO [2019-12-17 12:29:11,028] jepsen worker 20 - jepsen.util 1640 :invoke :read [6 nil] INFO [2019-12-17 12:29:11,034] jepsen worker 29 - jepsen.util 1649 :ok :read [9 46] INFO [2019-12-17 12:29:11,034] jepsen worker 29 - jepsen.util 1649 :invoke :cas [9 [23 41]] INFO [2019-12-17 12:29:11,046] jepsen worker 20 - jepsen.util 1640 :ok :read [6 13] INFO [2019-12-17 12:29:11,046] jepsen worker 20 - jepsen.util 1640 :invoke :write [6 39] INFO [2019-12-17 12:29:11,057] jepsen worker 29 - jepsen.util 1649 :fail :cas [9 [23 41]] :ValueNotSwapFrom INFO [2019-12-17 12:29:11,057] jepsen worker 29 - jepsen.util 1649 :invoke :write [9 21] INFO [2019-12-17 12:29:11,093] jepsen worker 55 - jepsen.util 2215 :fail :cas [18 [31 30]] :ValueNotSwapFrom INFO [2019-12-17 12:29:11,093] jepsen worker 55 - jepsen.util 2215 :invoke :cas [18 [42 15]] INFO [2019-12-17 12:29:11,098] jepsen worker 87 - jepsen.util 1617 :ok :read [29 9] INFO [2019-12-17 12:29:11,098] jepsen worker 87 - jepsen.util 1617 :invoke :cas [29 [36 16]] INFO [2019-12-17 12:29:11,106] jepsen worker 53 - jepsen.util 1583 :ok :read [17 18] INFO [2019-12-17 12:29:11,106] jepsen worker 53 - jepsen.util 1583 :invoke :write [17 47] INFO [2019-12-17 12:29:11,116] jepsen worker 56 - jepsen.util 1856 :fail :cas [18 [24 31]] :ValueNotSwapFrom INFO [2019-12-17 12:29:11,116] jepsen worker 56 - jepsen.util 1856 :invoke :write [18 11] INFO [2019-12-17 12:29:11,120] jepsen worker 36 - jepsen.util 1656 :info :write [12 32] :RequestCanceledException INFO [2019-12-17 12:29:11,121] jepsen worker 36 - jepsen.util 1746 :invoke :cas [12 [6 40]] INFO [2019-12-17 12:29:11,126] jepsen worker 37 - jepsen.util 1567 :ok :read [12 38] INFO [2019-12-17 12:29:11,126] jepsen worker 37 - jepsen.util 1567 :invoke :cas [12 [48 0]] INFO [2019-12-17 12:29:11,144] jepsen worker 29 - jepsen.util 1649 :ok :write [9 21] INFO [2019-12-17 12:29:11,144] jepsen worker 29 - jepsen.util 1649 :invoke :cas [9 [26 1]] INFO [2019-12-17 12:29:11,158] jepsen worker 29 - jepsen.util 1649 :fail :cas [9 [26 1]] :ValueNotSwapFrom INFO [2019-12-17 12:29:11,158] jepsen worker 29 - jepsen.util 1649 :invoke :write [9 7] INFO [2019-12-17 12:29:11,200] jepsen worker 29 - jepsen.util 1649 :ok :write [9 7] INFO [2019-12-17 12:29:11,200] jepsen worker 29 - jepsen.util 1649 :invoke :write [9 42] INFO [2019-12-17 12:29:11,207] jepsen worker 15 - jepsen.util 2265 :info :cas [5 [26 43]] :RequestCanceledException INFO [2019-12-17 12:29:11,208] jepsen worker 15 - jepsen.util 2355 :invoke :cas [5 [48 9]] INFO [2019-12-17 12:29:11,210] jepsen worker 70 - jepsen.util 2140 :ok :read [23 30] INFO [2019-12-17 12:29:11,210] jepsen worker 70 - jepsen.util 2140 :invoke :cas [23 [9 48]] INFO [2019-12-17 12:29:11,214] jepsen worker 79 - jepsen.util 1879 :fail :cas [26 [12 21]] :ValueNotSwapFrom INFO [2019-12-17 12:29:11,214] jepsen worker 79 - jepsen.util 1879 :invoke :write [26 17] INFO [2019-12-17 12:29:11,222] jepsen worker 36 - jepsen.util 1746 :fail :cas [12 [6 40]] :ValueNotSwapFrom INFO [2019-12-17 12:29:11,222] jepsen worker 43 - jepsen.util 2563 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:29:11,222] jepsen worker 36 - jepsen.util 1746 :invoke :read [12 nil] INFO [2019-12-17 12:29:11,222] jepsen worker 43 - jepsen.util 2563 :invoke :cas [14 [35 42]] INFO [2019-12-17 12:29:11,227] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":10811,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:29:11,227] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":10822,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:29:11,227] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":10811,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:29:11,227] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":10822,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:29:11,227] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":10824,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:29:11,227] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":10824,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:29:11,236] jepsen worker 16 - jepsen.util 2266 :info :cas [5 [28 37]] :RequestCanceledException INFO [2019-12-17 12:29:11,237] jepsen worker 16 - jepsen.util 2356 :invoke :write [5 32] INFO [2019-12-17 12:29:11,243] jepsen worker 29 - jepsen.util 1649 :ok :write [9 42] INFO [2019-12-17 12:29:11,244] jepsen worker 29 - jepsen.util 1649 :invoke :cas [9 [0 36]] INFO [2019-12-17 12:29:11,262] jepsen worker 29 - jepsen.util 1649 :fail :cas [9 [0 36]] :ValueNotSwapFrom INFO [2019-12-17 12:29:11,262] jepsen worker 29 - jepsen.util 1649 :invoke :cas [9 [24 6]] INFO [2019-12-17 12:29:11,275] jepsen worker 29 - jepsen.util 1649 :fail :cas [9 [24 6]] :ValueNotSwapFrom INFO [2019-12-17 12:29:11,275] jepsen worker 29 - jepsen.util 1649 :invoke :cas [9 [35 24]] INFO [2019-12-17 12:29:11,287] jepsen worker 29 - jepsen.util 1649 :fail :cas [9 [35 24]] :ValueNotSwapFrom INFO [2019-12-17 12:29:11,287] jepsen worker 29 - jepsen.util 1649 :invoke :read [9 nil] INFO [2019-12-17 12:29:11,288] jepsen worker 5 - jepsen.util 1445 :info :write [1 37] :RequestCanceledException INFO [2019-12-17 12:29:11,289] jepsen worker 5 - jepsen.util 1535 :invoke :write [1 38] INFO [2019-12-17 12:29:11,298] jepsen worker 29 - jepsen.util 1649 :ok :read [9 42] INFO [2019-12-17 12:29:11,298] jepsen worker 29 - jepsen.util 1649 :invoke :read [9 nil] INFO [2019-12-17 12:29:11,314] jepsen worker 36 - jepsen.util 1746 :ok :read [12 38] INFO [2019-12-17 12:29:11,314] jepsen worker 36 - jepsen.util 1746 :invoke :read [12 nil] INFO [2019-12-17 12:29:11,431] jepsen worker 81 - jepsen.util 2151 :info :write [27 7] :RequestCanceledException INFO [2019-12-17 12:29:11,431] jepsen worker 21 - jepsen.util 2361 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:29:11,431] jepsen worker 86 - jepsen.util 2516 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:29:11,431] jepsen worker 61 - jepsen.util 2491 :info :write [20 49] :RequestCanceledException INFO [2019-12-17 12:29:11,431] jepsen worker 35 - jepsen.util 2105 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:29:11,431] jepsen worker 85 - jepsen.util 2695 :info :write [28 38] :RequestCanceledException INFO [2019-12-17 12:29:11,431] jepsen worker 26 - jepsen.util 2726 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:29:11,431] jepsen worker 58 - jepsen.util 2308 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:29:11,431] jepsen worker 40 - jepsen.util 2920 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:29:11,431] jepsen worker 76 - jepsen.util 2506 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:29:11,431] jepsen worker 23 - jepsen.util 2273 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:29:11,431] jepsen worker 62 - jepsen.util 2312 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:29:11,431] jepsen worker 59 - jepsen.util 2129 :info :write [19 49] :RequestCanceledException INFO [2019-12-17 12:29:11,431] jepsen worker 21 - jepsen.util 2361 :invoke :write [7 24] INFO [2019-12-17 12:29:11,431] jepsen worker 0 - jepsen.util 2610 :info :write [0 47] :RequestCanceledException INFO [2019-12-17 12:29:11,431] jepsen worker 86 - jepsen.util 2516 :invoke :cas [28 [31 29]] INFO [2019-12-17 12:29:11,431] jepsen worker 35 - jepsen.util 2105 :invoke :cas [11 [2 28]] INFO [2019-12-17 12:29:11,431] jepsen worker 22 - jepsen.util 2542 :info :write [7 40] :RequestCanceledException INFO [2019-12-17 12:29:11,431] jepsen worker 84 - jepsen.util 2244 :info :write [28 3] :RequestCanceledException INFO [2019-12-17 12:29:11,431] jepsen worker 75 - jepsen.util 2775 :info :write [25 36] :RequestCanceledException INFO [2019-12-17 12:29:11,431] jepsen worker 25 - jepsen.util 2725 :info :write [8 29] :RequestCanceledException INFO [2019-12-17 12:29:11,431] jepsen worker 57 - jepsen.util 2307 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:29:11,431] jepsen worker 1 - jepsen.util 2341 :info :write [0 46] :RequestCanceledException INFO [2019-12-17 12:29:11,432] jepsen worker 26 - jepsen.util 2726 :invoke :cas [8 [8 16]] INFO [2019-12-17 12:29:11,432] jepsen worker 50 - jepsen.util 2660 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:29:11,432] jepsen worker 58 - jepsen.util 2308 :invoke :write [19 7] INFO [2019-12-17 12:29:11,432] jepsen worker 40 - jepsen.util 2920 :invoke :cas [13 [7 9]] INFO [2019-12-17 12:29:11,432] jepsen worker 81 - jepsen.util 2241 :invoke :read [27 nil] INFO [2019-12-17 12:29:11,432] jepsen worker 76 - jepsen.util 2506 :invoke :read [25 nil] INFO [2019-12-17 12:29:11,432] jepsen worker 23 - jepsen.util 2273 :invoke :cas [7 [5 8]] INFO [2019-12-17 12:29:11,432] jepsen worker 61 - jepsen.util 2581 :invoke :read [20 nil] INFO [2019-12-17 12:29:11,432] jepsen worker 62 - jepsen.util 2312 :invoke :read [20 nil] INFO [2019-12-17 12:29:11,432] jepsen worker 85 - jepsen.util 2785 :invoke :cas [28 [2 16]] INFO [2019-12-17 12:29:11,433] jepsen worker 59 - jepsen.util 2219 :invoke :write [19 25] INFO [2019-12-17 12:29:11,433] jepsen worker 0 - jepsen.util 2700 :invoke :write [0 4] INFO [2019-12-17 12:29:11,434] jepsen worker 57 - jepsen.util 2307 :invoke :read [19 nil] INFO [2019-12-17 12:29:11,434] jepsen worker 22 - jepsen.util 2632 :invoke :cas [7 [19 20]] INFO [2019-12-17 12:29:11,434] jepsen worker 84 - jepsen.util 2334 :invoke :read [28 nil] INFO [2019-12-17 12:29:11,434] jepsen worker 50 - jepsen.util 2660 :invoke :write [16 0] INFO [2019-12-17 12:29:11,434] jepsen worker 75 - jepsen.util 2865 :invoke :read [25 nil] INFO [2019-12-17 12:29:11,434] jepsen worker 25 - jepsen.util 2815 :invoke :write [8 36] INFO [2019-12-17 12:29:11,434] jepsen worker 1 - jepsen.util 2431 :invoke :cas [0 [6 9]] INFO [2019-12-17 12:29:11,439] jepsen worker 29 - jepsen.util 1649 :ok :read [9 42] INFO [2019-12-17 12:29:11,439] jepsen worker 29 - jepsen.util 1649 :invoke :read [9 nil] INFO [2019-12-17 12:29:11,450] jepsen worker 82 - jepsen.util 2872 :info :cas [27 [11 22]] :RequestCanceledException INFO [2019-12-17 12:29:11,450] jepsen worker 66 - jepsen.util 2226 :info :cas [22 [43 18]] :RequestCanceledException INFO [2019-12-17 12:29:11,451] jepsen worker 82 - jepsen.util 2962 :invoke :write [27 18] INFO [2019-12-17 12:29:11,451] jepsen worker 66 - jepsen.util 2316 :invoke :write [22 23] INFO [2019-12-17 12:29:11,456] jepsen worker 29 - jepsen.util 1649 :ok :read [9 42] INFO [2019-12-17 12:29:11,456] jepsen worker 29 - jepsen.util 1649 :invoke :read [9 nil] INFO [2019-12-17 12:29:11,468] jepsen worker 29 - jepsen.util 1649 :ok :read [9 42] INFO [2019-12-17 12:29:11,468] jepsen worker 29 - jepsen.util 1649 :invoke :write [9 45] INFO [2019-12-17 12:29:11,488] jepsen worker 8 - jepsen.util 2258 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:29:11,488] jepsen worker 68 - jepsen.util 2138 :info :write [22 42] :RequestCanceledException INFO [2019-12-17 12:29:11,489] jepsen worker 8 - jepsen.util 2258 :invoke :read [2 nil] INFO [2019-12-17 12:29:11,489] jepsen worker 68 - jepsen.util 2228 :invoke :write [22 9] INFO [2019-12-17 12:29:11,501] jepsen worker 36 - jepsen.util 1746 :ok :read [12 38] INFO [2019-12-17 12:29:11,501] jepsen worker 36 - jepsen.util 1746 :invoke :write [12 36] INFO [2019-12-17 12:29:11,502] jepsen worker 37 - jepsen.util 1567 :fail :cas [12 [48 0]] :ValueNotSwapFrom INFO [2019-12-17 12:29:11,502] jepsen worker 37 - jepsen.util 1567 :invoke :cas [12 [1 15]] INFO [2019-12-17 12:29:11,503] jepsen worker 10 - jepsen.util 1990 :ok :write [3 2] INFO [2019-12-17 12:29:11,504] jepsen worker 10 - jepsen.util 1990 :invoke :cas [3 [3 16]] INFO [2019-12-17 12:29:11,504] jepsen worker 45 - jepsen.util 2115 :ok :write [15 17] INFO [2019-12-17 12:29:11,504] jepsen worker 45 - jepsen.util 2115 :invoke :read [15 nil] INFO [2019-12-17 12:29:11,507] jepsen worker 24 - jepsen.util 2184 :info :cas [8 [46 13]] :RequestCanceledException INFO [2019-12-17 12:29:11,508] jepsen worker 24 - jepsen.util 2274 :invoke :cas [8 [40 3]] INFO [2019-12-17 12:29:11,519] jepsen worker 13 - jepsen.util 1633 :fail :cas [4 [17 45]] :ValueNotSwapFrom INFO [2019-12-17 12:29:11,519] jepsen worker 13 - jepsen.util 1633 :invoke :write [4 40] INFO [2019-12-17 12:29:11,521] jepsen worker 70 - jepsen.util 2140 :fail :cas [23 [9 48]] :ValueNotSwapFrom INFO [2019-12-17 12:29:11,521] jepsen worker 70 - jepsen.util 2140 :invoke :write [23 38] INFO [2019-12-17 12:29:11,522] jepsen worker 17 - jepsen.util 2537 :info :write [5 47] :RequestCanceledException INFO [2019-12-17 12:29:11,522] jepsen worker 38 - jepsen.util 1928 :info :write [12 3] :RequestCanceledException INFO [2019-12-17 12:29:11,523] jepsen worker 17 - jepsen.util 2627 :invoke :write [5 38] INFO [2019-12-17 12:29:11,524] jepsen worker 38 - jepsen.util 2018 :invoke :write [12 48] INFO [2019-12-17 12:29:11,535] jepsen worker 29 - jepsen.util 1649 :ok :write [9 45] INFO [2019-12-17 12:29:11,535] jepsen worker 20 - jepsen.util 1640 :ok :write [6 39] INFO [2019-12-17 12:29:11,535] jepsen worker 29 - jepsen.util 1649 :invoke :read [9 nil] INFO [2019-12-17 12:29:11,535] jepsen worker 20 - jepsen.util 1640 :invoke :read [6 nil] INFO [2019-12-17 12:29:11,536] jepsen worker 41 - jepsen.util 2561 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:29:11,536] jepsen worker 41 - jepsen.util 2561 :invoke :read [13 nil] INFO [2019-12-17 12:29:11,540] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":10839,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:29:11,540] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":10839,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:29:11,540] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":10842,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:29:11,540] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":10842,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:29:11,541] jepsen worker 37 - jepsen.util 1567 :fail :cas [12 [1 15]] :ValueNotSwapFrom INFO [2019-12-17 12:29:11,541] jepsen worker 37 - jepsen.util 1567 :invoke :write [12 27] INFO [2019-12-17 12:29:11,542] jepsen worker 10 - jepsen.util 1990 :fail :cas [3 [3 16]] :ValueNotSwapFrom INFO [2019-12-17 12:29:11,542] jepsen worker 10 - jepsen.util 1990 :invoke :write [3 30] INFO [2019-12-17 12:29:11,545] jepsen worker 45 - jepsen.util 2115 :ok :read [15 17] INFO [2019-12-17 12:29:11,545] jepsen worker 45 - jepsen.util 2115 :invoke :read [15 nil] INFO [2019-12-17 12:29:11,578] jepsen worker 45 - jepsen.util 2115 :ok :read [15 17] INFO [2019-12-17 12:29:11,578] jepsen worker 45 - jepsen.util 2115 :invoke :read [15 nil] INFO [2019-12-17 12:29:11,582] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:29:11,583] jepsen nemesis - couchbase.util Rebalance check count: 76 INFO [2019-12-17 12:29:11,583] jepsen nemesis - couchbase.util Rebalance stuck count: 57 INFO [2019-12-17 12:29:11,589] jepsen worker 46 - jepsen.util 1936 :info :cas [15 [42 1]] :RequestCanceledException INFO [2019-12-17 12:29:11,589] jepsen worker 83 - jepsen.util 2693 :info :cas [27 [41 45]] :RequestCanceledException INFO [2019-12-17 12:29:11,590] jepsen worker 20 - jepsen.util 1640 :ok :read [6 39] INFO [2019-12-17 12:29:11,590] jepsen worker 20 - jepsen.util 1640 :invoke :cas [6 [10 29]] INFO [2019-12-17 12:29:11,590] jepsen worker 29 - jepsen.util 1649 :ok :read [9 45] INFO [2019-12-17 12:29:11,590] jepsen worker 29 - jepsen.util 1649 :invoke :cas [9 [0 27]] INFO [2019-12-17 12:29:11,590] jepsen worker 46 - jepsen.util 2026 :invoke :read [15 nil] INFO [2019-12-17 12:29:11,590] jepsen worker 83 - jepsen.util 2783 :invoke :cas [27 [27 23]] INFO [2019-12-17 12:29:11,606] jepsen worker 39 - jepsen.util 2649 :info :cas [13 [49 11]] :RequestCanceledException INFO [2019-12-17 12:29:11,607] jepsen worker 39 - jepsen.util 2739 :invoke :write [13 14] INFO [2019-12-17 12:29:11,622] jepsen worker 77 - jepsen.util 2507 :info :write [25 35] :RequestCanceledException INFO [2019-12-17 12:29:11,623] jepsen worker 77 - jepsen.util 2597 :invoke :cas [25 [28 22]] INFO [2019-12-17 12:29:11,623] jepsen worker 42 - jepsen.util 2922 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:29:11,623] jepsen worker 65 - jepsen.util 1595 :info :write [21 30] :RequestCanceledException INFO [2019-12-17 12:29:11,623] jepsen worker 42 - jepsen.util 2922 :invoke :write [14 4] INFO [2019-12-17 12:29:11,624] jepsen worker 65 - jepsen.util 1685 :invoke :read [21 nil] INFO [2019-12-17 12:29:11,629] jepsen worker 3 - jepsen.util 1713 :fail :cas [1 [19 6]] :ValueNotSwapFrom INFO [2019-12-17 12:29:11,629] jepsen worker 3 - jepsen.util 1713 :invoke :read [1 nil] INFO [2019-12-17 12:29:11,633] jepsen worker 55 - jepsen.util 2215 :fail :cas [18 [42 15]] :ValueNotSwapFrom INFO [2019-12-17 12:29:11,633] jepsen worker 55 - jepsen.util 2215 :invoke :read [18 nil] INFO [2019-12-17 12:29:11,636] jepsen worker 33 - jepsen.util 2823 :info :cas [11 [32 8]] :RequestCanceledException INFO [2019-12-17 12:29:11,636] jepsen worker 33 - jepsen.util 2913 :invoke :read [11 nil] INFO [2019-12-17 12:29:11,649] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":10902,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:29:11,649] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":10902,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:29:11,649] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":10904,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:29:11,649] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":10904,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:29:11,652] jepsen worker 48 - jepsen.util 2028 :info :cas [16 [24 30]] :RequestCanceledException INFO [2019-12-17 12:29:11,652] jepsen worker 34 - jepsen.util 2554 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:29:11,653] jepsen worker 34 - jepsen.util 2554 :invoke :write [11 12] INFO [2019-12-17 12:29:11,653] jepsen worker 48 - jepsen.util 2118 :invoke :read [16 nil] INFO [2019-12-17 12:29:11,661] jepsen worker 45 - jepsen.util 2115 :ok :read [15 17] INFO [2019-12-17 12:29:11,661] jepsen worker 45 - jepsen.util 2115 :invoke :write [15 41] INFO [2019-12-17 12:29:11,664] jepsen worker 20 - jepsen.util 1640 :fail :cas [6 [10 29]] :ValueNotSwapFrom INFO [2019-12-17 12:29:11,665] jepsen worker 20 - jepsen.util 1640 :invoke :read [6 nil] INFO [2019-12-17 12:29:11,667] jepsen worker 29 - jepsen.util 1649 :fail :cas [9 [0 27]] :ValueNotSwapFrom INFO [2019-12-17 12:29:11,667] jepsen worker 29 - jepsen.util 1649 :invoke :write [9 11] INFO [2019-12-17 12:29:11,704] jepsen worker 4 - jepsen.util 1444 :ok :write [1 10] INFO [2019-12-17 12:29:11,704] jepsen worker 4 - jepsen.util 1444 :invoke :read [1 nil] INFO [2019-12-17 12:29:11,722] jepsen worker 65 - jepsen.util 1685 :ok :read [21 6] INFO [2019-12-17 12:29:11,722] jepsen worker 65 - jepsen.util 1685 :invoke :read [21 nil] INFO [2019-12-17 12:29:11,722] jepsen worker 67 - jepsen.util 2857 :info :write [22 44] :RequestCanceledException INFO [2019-12-17 12:29:11,724] jepsen worker 67 - jepsen.util 2947 :invoke :cas [22 [47 29]] INFO [2019-12-17 12:29:11,741] jepsen worker 20 - jepsen.util 1640 :ok :read [6 39] INFO [2019-12-17 12:29:11,741] jepsen worker 20 - jepsen.util 1640 :invoke :read [6 nil] INFO [2019-12-17 12:29:11,747] jepsen worker 12 - jepsen.util 1902 :ok :write [4 37] INFO [2019-12-17 12:29:11,747] jepsen worker 12 - jepsen.util 1902 :invoke :write [4 6] INFO [2019-12-17 12:29:11,793] jepsen worker 56 - jepsen.util 1856 :ok :write [18 11] INFO [2019-12-17 12:29:11,793] jepsen worker 56 - jepsen.util 1856 :invoke :write [18 32] INFO [2019-12-17 12:29:11,820] jepsen worker 20 - jepsen.util 1640 :ok :read [6 39] INFO [2019-12-17 12:29:11,820] jepsen worker 11 - jepsen.util 2351 :fail :read [3 nil] :RequestTimeoutException INFO [2019-12-17 12:29:11,820] jepsen worker 20 - jepsen.util 1640 :invoke :write [6 32] INFO [2019-12-17 12:29:11,820] jepsen worker 11 - jepsen.util 2351 :invoke :cas [3 [25 9]] INFO [2019-12-17 12:29:11,848] jepsen worker 29 - jepsen.util 1649 :ok :write [9 11] INFO [2019-12-17 12:29:11,848] jepsen worker 29 - jepsen.util 1649 :invoke :cas [9 [37 46]] INFO [2019-12-17 12:29:11,849] jepsen worker 11 - jepsen.util 2351 :fail :cas [3 [25 9]] :ValueNotSwapFrom INFO [2019-12-17 12:29:11,850] jepsen worker 11 - jepsen.util 2351 :invoke :write [3 5] INFO [2019-12-17 12:29:11,853] jepsen worker 47 - jepsen.util 2297 :info :cas [15 [18 43]] :RequestCanceledException INFO [2019-12-17 12:29:11,853] jepsen worker 32 - jepsen.util 2192 :info :cas [10 [26 4]] :RequestCanceledException INFO [2019-12-17 12:29:11,855] jepsen worker 47 - jepsen.util 2387 :invoke :cas [15 [36 28]] INFO [2019-12-17 12:29:11,855] jepsen worker 32 - jepsen.util 2282 :invoke :cas [10 [30 36]] INFO [2019-12-17 12:29:11,865] jepsen worker 51 - jepsen.util 1761 :ok :write [17 26] INFO [2019-12-17 12:29:11,866] jepsen worker 51 - jepsen.util 1761 :invoke :read [17 nil] INFO [2019-12-17 12:29:11,868] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":10920,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:29:11,868] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":10920,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:29:11,868] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":10923,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:29:11,869] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":10923,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:29:11,869] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":10924,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:29:11,869] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":10924,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:29:11,890] jepsen worker 47 - jepsen.util 2387 :fail :cas [15 [36 28]] :ValueNotSwapFrom INFO [2019-12-17 12:29:11,890] jepsen worker 47 - jepsen.util 2387 :invoke :read [15 nil] INFO [2019-12-17 12:29:11,897] jepsen worker 87 - jepsen.util 1617 :fail :cas [29 [36 16]] :ValueNotSwapFrom INFO [2019-12-17 12:29:11,897] jepsen worker 87 - jepsen.util 1617 :invoke :cas [29 [29 15]] INFO [2019-12-17 12:29:11,909] jepsen worker 47 - jepsen.util 2387 :ok :read [15 17] INFO [2019-12-17 12:29:11,909] jepsen worker 47 - jepsen.util 2387 :invoke :read [15 nil] INFO [2019-12-17 12:29:11,909] jepsen worker 10 - jepsen.util 1990 :ok :write [3 30] INFO [2019-12-17 12:29:11,909] jepsen worker 10 - jepsen.util 1990 :invoke :read [3 nil] INFO [2019-12-17 12:29:11,912] jepsen worker 29 - jepsen.util 1649 :fail :cas [9 [37 46]] :ValueNotSwapFrom INFO [2019-12-17 12:29:11,913] jepsen worker 29 - jepsen.util 1649 :invoke :cas [9 [45 5]] INFO [2019-12-17 12:29:11,922] jepsen worker 2 - jepsen.util 2252 :info :write [0 20] :RequestCanceledException INFO [2019-12-17 12:29:11,923] jepsen worker 2 - jepsen.util 2342 :invoke :read [0 nil] INFO [2019-12-17 12:29:11,936] jepsen worker 27 - jepsen.util 1557 :info :cas [9 [3 16]] :RequestCanceledException INFO [2019-12-17 12:29:11,937] jepsen worker 27 - jepsen.util 1647 :invoke :read [9 nil] INFO [2019-12-17 12:29:11,941] jepsen worker 46 - jepsen.util 2026 :ok :read [15 17] INFO [2019-12-17 12:29:11,941] jepsen worker 46 - jepsen.util 2026 :invoke :cas [15 [17 32]] INFO [2019-12-17 12:29:11,950] jepsen worker 47 - jepsen.util 2387 :ok :read [15 17] INFO [2019-12-17 12:29:11,950] jepsen worker 47 - jepsen.util 2387 :invoke :read [15 nil] INFO [2019-12-17 12:29:11,953] jepsen worker 44 - jepsen.util 2114 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:29:11,954] jepsen worker 44 - jepsen.util 2114 :invoke :write [14 8] INFO [2019-12-17 12:29:11,957] jepsen worker 10 - jepsen.util 1990 :ok :read [3 30] INFO [2019-12-17 12:29:11,957] jepsen worker 10 - jepsen.util 1990 :invoke :write [3 17] INFO [2019-12-17 12:29:11,989] jepsen worker 18 - jepsen.util 2088 :info :write [6 28] :RequestCanceledException INFO [2019-12-17 12:29:11,990] jepsen worker 18 - jepsen.util 2178 :invoke :read [6 nil] INFO [2019-12-17 12:29:11,996] jepsen worker 29 - jepsen.util 1649 :fail :cas [9 [45 5]] :ValueNotSwapFrom INFO [2019-12-17 12:29:11,996] jepsen worker 29 - jepsen.util 1649 :invoke :write [9 31] INFO [2019-12-17 12:29:11,999] jepsen worker 80 - jepsen.util 1880 :ok :write [26 38] INFO [2019-12-17 12:29:11,999] jepsen worker 80 - jepsen.util 1880 :invoke :write [26 15] INFO [2019-12-17 12:29:12,004] jepsen worker 31 - jepsen.util 1921 :info :write [10 16] :RequestCanceledException INFO [2019-12-17 12:29:12,004] jepsen worker 64 - jepsen.util 1504 :info :write [21 27] :RequestCanceledException INFO [2019-12-17 12:29:12,004] jepsen worker 9 - jepsen.util 1989 :info :cas [3 [2 7]] :RequestCanceledException INFO [2019-12-17 12:29:12,005] jepsen worker 31 - jepsen.util 2011 :invoke :cas [10 [33 27]] INFO [2019-12-17 12:29:12,005] jepsen worker 64 - jepsen.util 1594 :invoke :cas [21 [26 7]] INFO [2019-12-17 12:29:12,006] jepsen worker 9 - jepsen.util 2079 :invoke :cas [3 [34 24]] INFO [2019-12-17 12:29:12,007] jepsen worker 47 - jepsen.util 2387 :ok :read [15 17] INFO [2019-12-17 12:29:12,007] jepsen worker 47 - jepsen.util 2387 :invoke :write [15 37] INFO [2019-12-17 12:29:12,023] jepsen worker 49 - jepsen.util 1759 :info :write [16 2] :RequestCanceledException INFO [2019-12-17 12:29:12,024] jepsen worker 49 - jepsen.util 1849 :invoke :cas [16 [30 5]] INFO [2019-12-17 12:29:12,045] jepsen worker 27 - jepsen.util 1647 :ok :read [9 11] INFO [2019-12-17 12:29:12,045] jepsen worker 27 - jepsen.util 1647 :invoke :write [9 2] INFO [2019-12-17 12:29:12,066] jepsen worker 31 - jepsen.util 2011 :fail :cas [10 [33 27]] :ValueNotSwapFrom INFO [2019-12-17 12:29:12,066] jepsen worker 31 - jepsen.util 2011 :invoke :write [10 8] INFO [2019-12-17 12:29:12,088] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":10941,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:29:12,088] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":10941,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:29:12,088] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":10944,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:29:12,088] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":10944,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:29:12,089] jepsen worker 55 - jepsen.util 2215 :ok :read [18 11] INFO [2019-12-17 12:29:12,089] jepsen worker 55 - jepsen.util 2215 :invoke :read [18 nil] INFO [2019-12-17 12:29:12,089] jepsen worker 30 - jepsen.util 2100 :info :cas [10 [32 27]] :RequestCanceledException INFO [2019-12-17 12:29:12,090] jepsen worker 30 - jepsen.util 2190 :invoke :write [10 10] INFO [2019-12-17 12:29:12,102] jepsen worker 64 - jepsen.util 1594 :fail :cas [21 [26 7]] :ValueNotSwapFrom INFO [2019-12-17 12:29:12,102] jepsen worker 64 - jepsen.util 1594 :invoke :cas [21 [4 48]] INFO [2019-12-17 12:29:12,124] jepsen worker 45 - jepsen.util 2115 :ok :write [15 41] INFO [2019-12-17 12:29:12,124] jepsen worker 45 - jepsen.util 2115 :invoke :cas [15 [12 5]] INFO [2019-12-17 12:29:12,137] jepsen worker 63 - jepsen.util 1683 :info :write [21 9] :RequestCanceledException INFO [2019-12-17 12:29:12,138] jepsen worker 63 - jepsen.util 1773 :invoke :read [21 nil] INFO [2019-12-17 12:29:12,152] jepsen worker 45 - jepsen.util 2115 :fail :cas [15 [12 5]] :ValueNotSwapFrom INFO [2019-12-17 12:29:12,152] jepsen worker 45 - jepsen.util 2115 :invoke :write [15 45] INFO [2019-12-17 12:29:12,161] jepsen worker 55 - jepsen.util 2215 :ok :read [18 11] INFO [2019-12-17 12:29:12,161] jepsen worker 55 - jepsen.util 2215 :invoke :cas [18 [32 32]] INFO [2019-12-17 12:29:12,186] jepsen worker 29 - jepsen.util 1649 :ok :write [9 31] INFO [2019-12-17 12:29:12,186] jepsen worker 29 - jepsen.util 1649 :invoke :write [9 11] INFO [2019-12-17 12:29:12,197] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":10967,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:29:12,197] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":10967,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:29:12,201] jepsen worker 20 - jepsen.util 1640 :ok :write [6 32] INFO [2019-12-17 12:29:12,201] jepsen worker 20 - jepsen.util 1640 :invoke :write [6 46] INFO [2019-12-17 12:29:12,207] jepsen worker 69 - jepsen.util 1869 :info :write [23 10] :RequestCanceledException INFO [2019-12-17 12:29:12,208] jepsen worker 69 - jepsen.util 1959 :invoke :write [23 28] INFO [2019-12-17 12:29:12,233] jepsen worker 9 - jepsen.util 2079 :fail :cas [3 [34 24]] :ValueNotSwapFrom INFO [2019-12-17 12:29:12,233] jepsen worker 9 - jepsen.util 2079 :invoke :cas [3 [10 6]] INFO [2019-12-17 12:29:12,264] jepsen worker 10 - jepsen.util 1990 :ok :write [3 17] INFO [2019-12-17 12:29:12,264] jepsen worker 10 - jepsen.util 1990 :invoke :write [3 29] INFO [2019-12-17 12:29:12,266] jepsen worker 31 - jepsen.util 2011 :ok :write [10 8] INFO [2019-12-17 12:29:12,266] jepsen worker 31 - jepsen.util 2011 :invoke :write [10 27] INFO [2019-12-17 12:29:12,290] jepsen worker 29 - jepsen.util 1649 :ok :write [9 11] INFO [2019-12-17 12:29:12,290] jepsen worker 29 - jepsen.util 1649 :invoke :cas [9 [43 37]] INFO [2019-12-17 12:29:12,308] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":10973,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:29:12,308] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":10973,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:29:12,308] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":10976,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:29:12,308] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":10976,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:29:12,308] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":10977,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:29:12,308] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":10977,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:29:12,322] jepsen worker 6 - jepsen.util 2526 :info :write [2 14] :RequestCanceledException INFO [2019-12-17 12:29:12,324] jepsen worker 6 - jepsen.util 2616 :invoke :write [2 14] INFO [2019-12-17 12:29:12,325] jepsen worker 63 - jepsen.util 1773 :ok :read [21 6] INFO [2019-12-17 12:29:12,325] jepsen worker 63 - jepsen.util 1773 :invoke :cas [21 [1 8]] INFO [2019-12-17 12:29:12,335] jepsen worker 29 - jepsen.util 1649 :fail :cas [9 [43 37]] :ValueNotSwapFrom INFO [2019-12-17 12:29:12,335] jepsen worker 29 - jepsen.util 1649 :invoke :read [9 nil] INFO [2019-12-17 12:29:12,360] jepsen worker 29 - jepsen.util 1649 :ok :read [9 11] INFO [2019-12-17 12:29:12,360] jepsen worker 29 - jepsen.util 1649 :invoke :write [9 20] INFO [2019-12-17 12:29:12,384] jepsen worker 3 - jepsen.util 1713 :ok :read [1 10] INFO [2019-12-17 12:29:12,384] jepsen worker 3 - jepsen.util 1713 :invoke :read [1 nil] INFO [2019-12-17 12:29:12,389] jepsen worker 60 - jepsen.util 2580 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:29:12,389] jepsen worker 60 - jepsen.util 2580 :invoke :cas [20 [13 41]] INFO [2019-12-17 12:29:12,404] jepsen worker 29 - jepsen.util 1649 :ok :write [9 20] INFO [2019-12-17 12:29:12,404] jepsen worker 29 - jepsen.util 1649 :invoke :cas [9 [5 0]] INFO [2019-12-17 12:29:12,414] jepsen worker 4 - jepsen.util 1444 :ok :read [1 10] INFO [2019-12-17 12:29:12,414] jepsen worker 4 - jepsen.util 1444 :invoke :cas [1 [24 15]] INFO [2019-12-17 12:29:12,427] jepsen worker 29 - jepsen.util 1649 :fail :cas [9 [5 0]] :ValueNotSwapFrom INFO [2019-12-17 12:29:12,427] jepsen worker 29 - jepsen.util 1649 :invoke :write [9 24] INFO [2019-12-17 12:29:12,466] jepsen worker 29 - jepsen.util 1649 :ok :write [9 24] INFO [2019-12-17 12:29:12,466] jepsen worker 29 - jepsen.util 1649 :invoke :cas [9 [15 7]] INFO [2019-12-17 12:29:12,470] jepsen worker 55 - jepsen.util 2215 :fail :cas [18 [32 32]] :ValueNotSwapFrom INFO [2019-12-17 12:29:12,470] jepsen worker 55 - jepsen.util 2215 :invoke :read [18 nil] INFO [2019-12-17 12:29:12,484] jepsen worker 29 - jepsen.util 1649 :fail :cas [9 [15 7]] :ValueNotSwapFrom INFO [2019-12-17 12:29:12,484] jepsen worker 29 - jepsen.util 1649 :invoke :read [9 nil] INFO [2019-12-17 12:29:12,494] jepsen worker 29 - jepsen.util 1649 :ok :read [9 24] INFO [2019-12-17 12:29:12,494] jepsen worker 29 - jepsen.util 1649 :invoke :cas [9 [48 37]] INFO [2019-12-17 12:29:12,509] jepsen worker 29 - jepsen.util 1649 :fail :cas [9 [48 37]] :ValueNotSwapFrom INFO [2019-12-17 12:29:12,509] jepsen worker 29 - jepsen.util 1649 :invoke :write [9 41] INFO [2019-12-17 12:29:12,523] jepsen worker 87 - jepsen.util 1617 :fail :cas [29 [29 15]] :ValueNotSwapFrom INFO [2019-12-17 12:29:12,523] jepsen worker 87 - jepsen.util 1617 :invoke :write [29 17] INFO [2019-12-17 12:29:12,527] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":10992,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:29:12,527] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":10992,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:29:12,527] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":10996,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:29:12,527] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":10996,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:29:12,557] jepsen worker 29 - jepsen.util 1649 :ok :write [9 41] INFO [2019-12-17 12:29:12,557] jepsen worker 29 - jepsen.util 1649 :invoke :write [9 23] INFO [2019-12-17 12:29:12,596] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:29:12,596] jepsen nemesis - couchbase.util Rebalance check count: 77 INFO [2019-12-17 12:29:12,596] jepsen nemesis - couchbase.util Rebalance stuck count: 58 INFO [2019-12-17 12:29:12,599] jepsen worker 29 - jepsen.util 1649 :ok :write [9 23] INFO [2019-12-17 12:29:12,599] jepsen worker 29 - jepsen.util 1649 :invoke :cas [9 [46 27]] INFO [2019-12-17 12:29:12,616] jepsen worker 29 - jepsen.util 1649 :fail :cas [9 [46 27]] :ValueNotSwapFrom INFO [2019-12-17 12:29:12,616] jepsen worker 29 - jepsen.util 1649 :invoke :write [9 40] INFO [2019-12-17 12:29:12,622] jepsen worker 88 - jepsen.util 1798 :info :write [29 32] :RequestCanceledException INFO [2019-12-17 12:29:12,624] jepsen worker 88 - jepsen.util 1888 :invoke :write [29 30] INFO [2019-12-17 12:29:12,632] jepsen worker 51 - jepsen.util 1761 :ok :read [17 26] INFO [2019-12-17 12:29:12,633] jepsen worker 51 - jepsen.util 1761 :invoke :write [17 40] INFO [2019-12-17 12:29:12,636] jepsen worker 19 - jepsen.util 2179 :fail :read [6 nil] :RequestTimeoutException INFO [2019-12-17 12:29:12,636] jepsen worker 19 - jepsen.util 2179 :invoke :read [6 nil] INFO [2019-12-17 12:29:12,647] jepsen worker 19 - jepsen.util 2179 :ok :read [6 32] INFO [2019-12-17 12:29:12,648] jepsen worker 19 - jepsen.util 2179 :invoke :read [6 nil] INFO [2019-12-17 12:29:12,653] jepsen worker 7 - jepsen.util 2527 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:29:12,653] jepsen worker 7 - jepsen.util 2527 :invoke :write [2 1] INFO [2019-12-17 12:29:12,670] jepsen worker 29 - jepsen.util 1649 :ok :write [9 40] INFO [2019-12-17 12:29:12,670] jepsen worker 29 - jepsen.util 1649 :invoke :read [9 nil] INFO [2019-12-17 12:29:12,678] jepsen worker 19 - jepsen.util 2179 :ok :read [6 32] INFO [2019-12-17 12:29:12,678] jepsen worker 19 - jepsen.util 2179 :invoke :read [6 nil] INFO [2019-12-17 12:29:12,684] jepsen worker 29 - jepsen.util 1649 :ok :read [9 40] INFO [2019-12-17 12:29:12,684] jepsen worker 29 - jepsen.util 1649 :invoke :read [9 nil] INFO [2019-12-17 12:29:12,690] jepsen worker 19 - jepsen.util 2179 :ok :read [6 32] INFO [2019-12-17 12:29:12,690] jepsen worker 19 - jepsen.util 2179 :invoke :read [6 nil] INFO [2019-12-17 12:29:12,703] jepsen worker 19 - jepsen.util 2179 :ok :read [6 32] INFO [2019-12-17 12:29:12,703] jepsen worker 19 - jepsen.util 2179 :invoke :cas [6 [47 17]] INFO [2019-12-17 12:29:12,706] jepsen worker 29 - jepsen.util 1649 :ok :read [9 40] INFO [2019-12-17 12:29:12,706] jepsen worker 29 - jepsen.util 1649 :invoke :read [9 nil] INFO [2019-12-17 12:29:12,707] jepsen worker 74 - jepsen.util 1784 :info :write [24 19] :RequestCanceledException INFO [2019-12-17 12:29:12,708] jepsen worker 74 - jepsen.util 1874 :invoke :write [24 16] INFO [2019-12-17 12:29:12,724] jepsen worker 19 - jepsen.util 2179 :fail :cas [6 [47 17]] :ValueNotSwapFrom INFO [2019-12-17 12:29:12,724] jepsen worker 19 - jepsen.util 2179 :invoke :cas [6 [41 43]] INFO [2019-12-17 12:29:12,732] jepsen worker 29 - jepsen.util 1649 :ok :read [9 40] INFO [2019-12-17 12:29:12,732] jepsen worker 29 - jepsen.util 1649 :invoke :cas [9 [8 24]] INFO [2019-12-17 12:29:12,743] jepsen worker 29 - jepsen.util 1649 :fail :cas [9 [8 24]] :ValueNotSwapFrom INFO [2019-12-17 12:29:12,744] jepsen worker 29 - jepsen.util 1649 :invoke :read [9 nil] INFO [2019-12-17 12:29:12,745] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11011,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:29:12,745] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11011,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:29:12,745] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11014,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:29:12,745] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11014,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:29:12,745] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11015,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:29:12,745] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11015,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:29:12,753] jepsen worker 72 - jepsen.util 2052 :info :cas [24 [43 29]] :RequestCanceledException INFO [2019-12-17 12:29:12,754] jepsen worker 72 - jepsen.util 2142 :invoke :read [24 nil] INFO [2019-12-17 12:29:12,754] jepsen worker 10 - jepsen.util 1990 :ok :write [3 29] INFO [2019-12-17 12:29:12,755] jepsen worker 10 - jepsen.util 1990 :invoke :write [3 9] INFO [2019-12-17 12:29:12,755] jepsen worker 47 - jepsen.util 2387 :ok :write [15 37] INFO [2019-12-17 12:29:12,755] jepsen worker 47 - jepsen.util 2387 :invoke :cas [15 [19 30]] INFO [2019-12-17 12:29:12,764] jepsen worker 20 - jepsen.util 1640 :ok :write [6 46] INFO [2019-12-17 12:29:12,764] jepsen worker 31 - jepsen.util 2011 :ok :write [10 27] INFO [2019-12-17 12:29:12,764] jepsen worker 20 - jepsen.util 1640 :invoke :write [6 9] INFO [2019-12-17 12:29:12,764] jepsen worker 31 - jepsen.util 2011 :invoke :write [10 17] INFO [2019-12-17 12:29:12,778] jepsen worker 29 - jepsen.util 1649 :ok :read [9 40] INFO [2019-12-17 12:29:12,778] jepsen worker 29 - jepsen.util 1649 :invoke :cas [9 [9 33]] INFO [2019-12-17 12:29:12,799] jepsen worker 55 - jepsen.util 2215 :ok :read [18 11] INFO [2019-12-17 12:29:12,799] jepsen worker 55 - jepsen.util 2215 :invoke :write [18 26] INFO [2019-12-17 12:29:12,804] jepsen worker 38 - jepsen.util 2018 :ok :write [12 48] INFO [2019-12-17 12:29:12,804] jepsen worker 38 - jepsen.util 2018 :invoke :write [12 27] INFO [2019-12-17 12:29:12,822] jepsen worker 78 - jepsen.util 1878 :info :write [26 11] :RequestCanceledException INFO [2019-12-17 12:29:12,823] jepsen worker 78 - jepsen.util 1968 :invoke :write [26 36] INFO [2019-12-17 12:29:12,830] jepsen worker 64 - jepsen.util 1594 :fail :cas [21 [4 48]] :ValueNotSwapFrom INFO [2019-12-17 12:29:12,830] jepsen worker 64 - jepsen.util 1594 :invoke :cas [21 [25 16]] INFO [2019-12-17 12:29:12,836] jepsen worker 73 - jepsen.util 1873 :fail :read [24 nil] :RequestTimeoutException INFO [2019-12-17 12:29:12,836] jepsen worker 28 - jepsen.util 1198 :fail :read [9 nil] :RequestTimeoutException INFO [2019-12-17 12:29:12,836] jepsen worker 28 - jepsen.util 1198 :invoke :read [9 nil] INFO [2019-12-17 12:29:12,836] jepsen worker 73 - jepsen.util 1873 :invoke :write [24 2] INFO [2019-12-17 12:29:12,857] jepsen worker 3 - jepsen.util 1713 :ok :read [1 10] INFO [2019-12-17 12:29:12,857] jepsen worker 3 - jepsen.util 1713 :invoke :read [1 nil] INFO [2019-12-17 12:29:12,897] jepsen worker 47 - jepsen.util 2387 :fail :cas [15 [19 30]] :ValueNotSwapFrom INFO [2019-12-17 12:29:12,897] jepsen worker 47 - jepsen.util 2387 :invoke :read [15 nil] INFO [2019-12-17 12:29:12,908] jepsen worker 72 - jepsen.util 2142 :ok :read [24 47] INFO [2019-12-17 12:29:12,908] jepsen worker 72 - jepsen.util 2142 :invoke :cas [24 [13 2]] INFO [2019-12-17 12:29:12,910] jepsen worker 9 - jepsen.util 2079 :fail :cas [3 [10 6]] :ValueNotSwapFrom INFO [2019-12-17 12:29:12,910] jepsen worker 9 - jepsen.util 2079 :invoke :write [3 41] INFO [2019-12-17 12:29:12,922] jepsen worker 64 - jepsen.util 1594 :fail :cas [21 [25 16]] :ValueNotSwapFrom INFO [2019-12-17 12:29:12,922] jepsen worker 64 - jepsen.util 1594 :invoke :write [21 35] INFO [2019-12-17 12:29:12,931] jepsen worker 29 - jepsen.util 1649 :fail :cas [9 [9 33]] :ValueNotSwapFrom INFO [2019-12-17 12:29:12,932] jepsen worker 29 - jepsen.util 1649 :invoke :write [9 44] INFO [2019-12-17 12:29:12,944] jepsen worker 37 - jepsen.util 1567 :ok :write [12 27] INFO [2019-12-17 12:29:12,944] jepsen worker 37 - jepsen.util 1567 :invoke :write [12 1] INFO [2019-12-17 12:29:12,964] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11041,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:29:12,964] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11041,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:29:12,964] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11044,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:29:12,964] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11044,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:29:12,969] jepsen worker 47 - jepsen.util 2387 :ok :read [15 37] INFO [2019-12-17 12:29:12,969] jepsen worker 47 - jepsen.util 2387 :invoke :read [15 nil] INFO [2019-12-17 12:29:12,979] jepsen worker 89 - jepsen.util 1889 :ok :cas [29 [9 35]] INFO [2019-12-17 12:29:12,979] jepsen worker 89 - jepsen.util 1889 :invoke :write [29 25] INFO [2019-12-17 12:29:12,986] jepsen worker 72 - jepsen.util 2142 :fail :cas [24 [13 2]] :ValueNotSwapFrom INFO [2019-12-17 12:29:12,986] jepsen worker 72 - jepsen.util 2142 :invoke :write [24 40] INFO [2019-12-17 12:29:12,988] jepsen worker 52 - jepsen.util 1852 :info :write [17 11] :RequestCanceledException INFO [2019-12-17 12:29:12,989] jepsen worker 52 - jepsen.util 1942 :invoke :read [17 nil] INFO [2019-12-17 12:29:13,005] jepsen worker 71 - jepsen.util 1511 :info :write [23 23] :RequestCanceledException INFO [2019-12-17 12:29:13,006] jepsen worker 71 - jepsen.util 1601 :invoke :write [23 11] INFO [2019-12-17 12:29:13,012] jepsen worker 64 - jepsen.util 1594 :ok :write [21 35] INFO [2019-12-17 12:29:13,013] jepsen worker 64 - jepsen.util 1594 :invoke :read [21 nil] INFO [2019-12-17 12:29:13,013] jepsen worker 69 - jepsen.util 1959 :ok :write [23 28] INFO [2019-12-17 12:29:13,013] jepsen worker 69 - jepsen.util 1959 :invoke :read [23 nil] INFO [2019-12-17 12:29:13,016] jepsen worker 54 - jepsen.util 1944 :ok :write [18 11] INFO [2019-12-17 12:29:13,016] jepsen worker 54 - jepsen.util 1944 :invoke :read [18 nil] INFO [2019-12-17 12:29:13,022] jepsen worker 14 - jepsen.util 1814 :info :write [4 25] :RequestCanceledException INFO [2019-12-17 12:29:13,030] jepsen worker 14 - jepsen.util 1904 :invoke :read [4 nil] INFO [2019-12-17 12:29:13,037] jepsen worker 47 - jepsen.util 2387 :ok :read [15 37] INFO [2019-12-17 12:29:13,037] jepsen worker 47 - jepsen.util 2387 :invoke :write [15 34] INFO [2019-12-17 12:29:13,051] jepsen worker 64 - jepsen.util 1594 :ok :read [21 35] INFO [2019-12-17 12:29:13,052] jepsen worker 64 - jepsen.util 1594 :invoke :cas [21 [27 34]] INFO [2019-12-17 12:29:13,063] jepsen worker 4 - jepsen.util 1444 :fail :cas [1 [24 15]] :ValueNotSwapFrom INFO [2019-12-17 12:29:13,063] jepsen worker 4 - jepsen.util 1444 :invoke :cas [1 [28 49]] INFO [2019-12-17 12:29:13,073] jepsen worker 29 - jepsen.util 1649 :ok :write [9 44] INFO [2019-12-17 12:29:13,074] jepsen worker 29 - jepsen.util 1649 :invoke :read [9 nil] INFO [2019-12-17 12:29:13,075] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11057,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:29:13,075] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11057,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:29:13,075] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11058,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:29:13,075] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11058,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:29:13,075] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11059,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:29:13,075] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11059,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:29:13,096] jepsen worker 10 - jepsen.util 1990 :ok :write [3 9] INFO [2019-12-17 12:29:13,096] jepsen worker 10 - jepsen.util 1990 :invoke :read [3 nil] INFO [2019-12-17 12:29:13,096] jepsen worker 74 - jepsen.util 1874 :ok :write [24 16] INFO [2019-12-17 12:29:13,096] jepsen worker 74 - jepsen.util 1874 :invoke :write [24 47] INFO [2019-12-17 12:29:13,139] jepsen worker 28 - jepsen.util 1198 :ok :read [9 44] INFO [2019-12-17 12:29:13,140] jepsen worker 28 - jepsen.util 1198 :invoke :write [9 4] INFO [2019-12-17 12:29:13,154] jepsen worker 53 - jepsen.util 1583 :info :write [17 47] :RequestCanceledException INFO [2019-12-17 12:29:13,154] jepsen worker 53 - jepsen.util 1673 :invoke :write [17 42] INFO [2019-12-17 12:29:13,164] jepsen worker 10 - jepsen.util 1990 :ok :read [3 9] INFO [2019-12-17 12:29:13,164] jepsen worker 10 - jepsen.util 1990 :invoke :write [3 40] INFO [2019-12-17 12:29:13,209] jepsen worker 31 - jepsen.util 2011 :ok :write [10 17] INFO [2019-12-17 12:29:13,209] jepsen worker 20 - jepsen.util 1640 :ok :write [6 9] INFO [2019-12-17 12:29:13,209] jepsen worker 31 - jepsen.util 2011 :invoke :read [10 nil] INFO [2019-12-17 12:29:13,209] jepsen worker 20 - jepsen.util 1640 :invoke :read [6 nil] INFO [2019-12-17 12:29:13,221] jepsen worker 54 - jepsen.util 1944 :ok :read [18 11] INFO [2019-12-17 12:29:13,221] jepsen worker 54 - jepsen.util 1944 :invoke :read [18 nil] INFO [2019-12-17 12:29:13,250] jepsen worker 20 - jepsen.util 1640 :ok :read [6 9] INFO [2019-12-17 12:29:13,250] jepsen worker 20 - jepsen.util 1640 :invoke :cas [6 [11 37]] INFO [2019-12-17 12:29:13,253] jepsen worker 15 - jepsen.util 2355 :info :cas [5 [48 9]] :RequestCanceledException INFO [2019-12-17 12:29:13,254] jepsen worker 15 - jepsen.util 2445 :invoke :cas [5 [18 4]] INFO [2019-12-17 12:29:13,257] jepsen worker 31 - jepsen.util 2011 :ok :read [10 17] INFO [2019-12-17 12:29:13,257] jepsen worker 31 - jepsen.util 2011 :invoke :read [10 nil] INFO [2019-12-17 12:29:13,268] jepsen worker 20 - jepsen.util 1640 :fail :cas [6 [11 37]] :ValueNotSwapFrom INFO [2019-12-17 12:29:13,268] jepsen worker 20 - jepsen.util 1640 :invoke :write [6 1] INFO [2019-12-17 12:29:13,280] jepsen worker 31 - jepsen.util 2011 :ok :read [10 17] INFO [2019-12-17 12:29:13,280] jepsen worker 31 - jepsen.util 2011 :invoke :read [10 nil] INFO [2019-12-17 12:29:13,288] jepsen worker 79 - jepsen.util 1879 :info :write [26 17] :RequestCanceledException INFO [2019-12-17 12:29:13,290] jepsen worker 79 - jepsen.util 1969 :invoke :cas [26 [45 46]] INFO [2019-12-17 12:29:13,291] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11077,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:29:13,291] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11077,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:29:13,291] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11078,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:29:13,291] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11078,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:29:13,291] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11079,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:29:13,291] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11079,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:29:13,292] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11084,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:29:13,292] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11084,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:29:13,292] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11087,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:29:13,292] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11087,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:29:13,292] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11088,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:29:13,292] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11088,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:29:13,297] jepsen worker 31 - jepsen.util 2011 :ok :read [10 17] INFO [2019-12-17 12:29:13,297] jepsen worker 31 - jepsen.util 2011 :invoke :read [10 nil] INFO [2019-12-17 12:29:13,323] jepsen worker 16 - jepsen.util 2356 :info :write [5 32] :RequestCanceledException INFO [2019-12-17 12:29:13,323] jepsen worker 43 - jepsen.util 2563 :info :cas [14 [35 42]] :RequestCanceledException INFO [2019-12-17 12:29:13,324] jepsen worker 43 - jepsen.util 2653 :invoke :read [14 nil] INFO [2019-12-17 12:29:13,324] jepsen worker 16 - jepsen.util 2446 :invoke :read [5 nil] INFO [2019-12-17 12:29:13,332] jepsen worker 31 - jepsen.util 2011 :ok :read [10 17] INFO [2019-12-17 12:29:13,332] jepsen worker 31 - jepsen.util 2011 :invoke :cas [10 [40 21]] INFO [2019-12-17 12:29:13,360] jepsen worker 31 - jepsen.util 2011 :fail :cas [10 [40 21]] :ValueNotSwapFrom INFO [2019-12-17 12:29:13,360] jepsen worker 31 - jepsen.util 2011 :invoke :read [10 nil] INFO [2019-12-17 12:29:13,382] jepsen worker 31 - jepsen.util 2011 :ok :read [10 17] INFO [2019-12-17 12:29:13,382] jepsen worker 31 - jepsen.util 2011 :invoke :read [10 nil] INFO [2019-12-17 12:29:13,389] jepsen worker 5 - jepsen.util 1535 :info :write [1 38] :RequestCanceledException INFO [2019-12-17 12:29:13,390] jepsen worker 5 - jepsen.util 1625 :invoke :write [1 13] INFO [2019-12-17 12:29:13,393] jepsen worker 3 - jepsen.util 1713 :ok :read [1 10] INFO [2019-12-17 12:29:13,394] jepsen worker 3 - jepsen.util 1713 :invoke :read [1 nil] INFO [2019-12-17 12:29:13,397] jepsen worker 31 - jepsen.util 2011 :ok :read [10 17] INFO [2019-12-17 12:29:13,397] jepsen worker 31 - jepsen.util 2011 :invoke :write [10 29] INFO [2019-12-17 12:29:13,447] jepsen worker 52 - jepsen.util 1942 :ok :read [17 26] INFO [2019-12-17 12:29:13,447] jepsen worker 52 - jepsen.util 1942 :invoke :cas [17 [46 7]] INFO [2019-12-17 12:29:13,448] jepsen worker 14 - jepsen.util 1904 :ok :read [4 37] INFO [2019-12-17 12:29:13,448] jepsen worker 14 - jepsen.util 1904 :invoke :write [4 12] INFO [2019-12-17 12:29:13,453] jepsen worker 58 - jepsen.util 2308 :info :write [19 7] :RequestCanceledException INFO [2019-12-17 12:29:13,453] jepsen worker 76 - jepsen.util 2506 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:29:13,453] jepsen worker 1 - jepsen.util 2431 :info :cas [0 [6 9]] :RequestCanceledException INFO [2019-12-17 12:29:13,453] jepsen worker 59 - jepsen.util 2219 :info :write [19 25] :RequestCanceledException INFO [2019-12-17 12:29:13,454] jepsen worker 76 - jepsen.util 2506 :invoke :write [25 17] INFO [2019-12-17 12:29:13,455] jepsen worker 1 - jepsen.util 2521 :invoke :cas [0 [17 14]] INFO [2019-12-17 12:29:13,455] jepsen worker 58 - jepsen.util 2398 :invoke :cas [19 [41 40]] INFO [2019-12-17 12:29:13,455] jepsen worker 59 - jepsen.util 2309 :invoke :read [19 nil] INFO [2019-12-17 12:29:13,472] jepsen worker 69 - jepsen.util 1959 :ok :read [23 28] INFO [2019-12-17 12:29:13,472] jepsen worker 69 - jepsen.util 1959 :invoke :cas [23 [18 26]] INFO [2019-12-17 12:29:13,487] jepsen worker 66 - jepsen.util 2316 :info :write [22 23] :RequestCanceledException INFO [2019-12-17 12:29:13,487] jepsen worker 22 - jepsen.util 2632 :info :cas [7 [19 20]] :RequestCanceledException INFO [2019-12-17 12:29:13,487] jepsen worker 40 - jepsen.util 2920 :info :cas [13 [7 9]] :RequestCanceledException INFO [2019-12-17 12:29:13,488] jepsen worker 66 - jepsen.util 2406 :invoke :cas [22 [16 17]] INFO [2019-12-17 12:29:13,488] jepsen worker 22 - jepsen.util 2722 :invoke :read [7 nil] INFO [2019-12-17 12:29:13,488] jepsen worker 40 - jepsen.util 3010 :invoke :read [13 nil] INFO [2019-12-17 12:29:13,496] jepsen worker 54 - jepsen.util 1944 :ok :read [18 11] INFO [2019-12-17 12:29:13,496] jepsen worker 54 - jepsen.util 1944 :invoke :read [18 nil] INFO [2019-12-17 12:29:13,507] jepsen worker 86 - jepsen.util 2516 :info :cas [28 [31 29]] :RequestCanceledException INFO [2019-12-17 12:29:13,507] jepsen worker 26 - jepsen.util 2726 :info :cas [8 [8 16]] :RequestCanceledException INFO [2019-12-17 12:29:13,507] jepsen worker 35 - jepsen.util 2105 :info :cas [11 [2 28]] :RequestCanceledException INFO [2019-12-17 12:29:13,507] jepsen worker 81 - jepsen.util 2241 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:29:13,507] jepsen worker 84 - jepsen.util 2334 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:29:13,507] jepsen worker 21 - jepsen.util 2361 :info :write [7 24] :RequestCanceledException INFO [2019-12-17 12:29:13,507] jepsen worker 68 - jepsen.util 2228 :info :write [22 9] :RequestCanceledException INFO [2019-12-17 12:29:13,507] jepsen worker 81 - jepsen.util 2241 :invoke :read [27 nil] INFO [2019-12-17 12:29:13,507] jepsen worker 84 - jepsen.util 2334 :invoke :cas [28 [26 22]] INFO [2019-12-17 12:29:13,508] jepsen worker 21 - jepsen.util 2451 :invoke :write [7 5] INFO [2019-12-17 12:29:13,508] jepsen worker 35 - jepsen.util 2195 :invoke :read [11 nil] INFO [2019-12-17 12:29:13,508] jepsen worker 86 - jepsen.util 2606 :invoke :write [28 38] INFO [2019-12-17 12:29:13,508] jepsen worker 26 - jepsen.util 2816 :invoke :write [8 8] INFO [2019-12-17 12:29:13,508] jepsen worker 68 - jepsen.util 2318 :invoke :cas [22 [45 22]] INFO [2019-12-17 12:29:13,512] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11109,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:29:13,512] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11109,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:29:13,512] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11112,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:29:13,512] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11112,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:29:13,522] jepsen worker 57 - jepsen.util 2307 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:29:13,522] jepsen worker 61 - jepsen.util 2581 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:29:13,522] jepsen worker 75 - jepsen.util 2865 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:29:13,522] jepsen worker 0 - jepsen.util 2700 :info :write [0 4] :RequestCanceledException INFO [2019-12-17 12:29:13,522] jepsen worker 82 - jepsen.util 2962 :info :write [27 18] :RequestCanceledException INFO [2019-12-17 12:29:13,522] jepsen worker 24 - jepsen.util 2274 :info :cas [8 [40 3]] :RequestCanceledException INFO [2019-12-17 12:29:13,522] jepsen worker 50 - jepsen.util 2660 :info :write [16 0] :RequestCanceledException INFO [2019-12-17 12:29:13,523] jepsen worker 57 - jepsen.util 2307 :invoke :write [19 2] INFO [2019-12-17 12:29:13,523] jepsen worker 61 - jepsen.util 2581 :invoke :cas [20 [29 27]] INFO [2019-12-17 12:29:13,523] jepsen worker 75 - jepsen.util 2865 :invoke :read [25 nil] INFO [2019-12-17 12:29:13,523] jepsen worker 82 - jepsen.util 3052 :invoke :read [27 nil] INFO [2019-12-17 12:29:13,523] jepsen worker 0 - jepsen.util 2790 :invoke :cas [0 [6 44]] INFO [2019-12-17 12:29:13,524] jepsen worker 50 - jepsen.util 2750 :invoke :write [16 5] INFO [2019-12-17 12:29:13,524] jepsen worker 24 - jepsen.util 2364 :invoke :write [8 27] INFO [2019-12-17 12:29:13,536] jepsen worker 25 - jepsen.util 2815 :info :write [8 36] :RequestCanceledException INFO [2019-12-17 12:29:13,536] jepsen worker 62 - jepsen.util 2312 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:29:13,536] jepsen worker 23 - jepsen.util 2273 :info :cas [7 [5 8]] :RequestCanceledException INFO [2019-12-17 12:29:13,536] jepsen worker 85 - jepsen.util 2785 :info :cas [28 [2 16]] :RequestCanceledException INFO [2019-12-17 12:29:13,536] jepsen worker 36 - jepsen.util 1746 :info :write [12 36] :RequestCanceledException INFO [2019-12-17 12:29:13,536] jepsen worker 62 - jepsen.util 2312 :invoke :write [20 24] INFO [2019-12-17 12:29:13,536] jepsen worker 85 - jepsen.util 2875 :invoke :read [28 nil] INFO [2019-12-17 12:29:13,537] jepsen worker 25 - jepsen.util 2905 :invoke :write [8 30] INFO [2019-12-17 12:29:13,537] jepsen worker 36 - jepsen.util 1836 :invoke :read [12 nil] INFO [2019-12-17 12:29:13,537] jepsen worker 23 - jepsen.util 2363 :invoke :cas [7 [25 8]] INFO [2019-12-17 12:29:13,581] jepsen worker 4 - jepsen.util 1444 :fail :cas [1 [28 49]] :ValueNotSwapFrom INFO [2019-12-17 12:29:13,581] jepsen worker 4 - jepsen.util 1444 :invoke :read [1 nil] INFO [2019-12-17 12:29:13,589] jepsen worker 8 - jepsen.util 2258 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:29:13,589] jepsen worker 13 - jepsen.util 1633 :info :write [4 40] :RequestCanceledException INFO [2019-12-17 12:29:13,589] jepsen worker 8 - jepsen.util 2258 :invoke :cas [2 [32 16]] INFO [2019-12-17 12:29:13,590] jepsen worker 13 - jepsen.util 1723 :invoke :write [4 8] INFO [2019-12-17 12:29:13,612] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:29:13,613] jepsen nemesis - couchbase.util Rebalance check count: 78 INFO [2019-12-17 12:29:13,613] jepsen nemesis - couchbase.util Rebalance stuck count: 59 INFO [2019-12-17 12:29:13,623] jepsen worker 70 - jepsen.util 2140 :info :write [23 38] :RequestCanceledException INFO [2019-12-17 12:29:13,625] jepsen worker 70 - jepsen.util 2230 :invoke :write [23 6] INFO [2019-12-17 12:29:13,625] jepsen worker 3 - jepsen.util 1713 :ok :read [1 10] INFO [2019-12-17 12:29:13,625] jepsen worker 3 - jepsen.util 1713 :invoke :write [1 42] INFO [2019-12-17 12:29:13,636] jepsen worker 17 - jepsen.util 2627 :info :write [5 38] :RequestCanceledException INFO [2019-12-17 12:29:13,637] jepsen worker 41 - jepsen.util 2561 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:29:13,638] jepsen worker 41 - jepsen.util 2561 :invoke :write [13 24] INFO [2019-12-17 12:29:13,642] jepsen worker 17 - jepsen.util 2717 :invoke :write [5 25] INFO [2019-12-17 12:29:13,654] jepsen worker 33 - jepsen.util 2913 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:29:13,667] jepsen worker 33 - jepsen.util 2913 :invoke :read [11 nil] INFO [2019-12-17 12:29:13,679] jepsen worker 52 - jepsen.util 1942 :fail :cas [17 [46 7]] :ValueNotSwapFrom INFO [2019-12-17 12:29:13,679] jepsen worker 52 - jepsen.util 1942 :invoke :write [17 44] INFO [2019-12-17 12:29:13,689] jepsen worker 83 - jepsen.util 2783 :info :cas [27 [27 23]] :RequestCanceledException INFO [2019-12-17 12:29:13,691] jepsen worker 83 - jepsen.util 2873 :invoke :read [27 nil] INFO [2019-12-17 12:29:13,707] jepsen worker 39 - jepsen.util 2739 :info :write [13 14] :RequestCanceledException INFO [2019-12-17 12:29:13,707] jepsen worker 39 - jepsen.util 2829 :invoke :read [13 nil] INFO [2019-12-17 12:29:13,720] jepsen worker 42 - jepsen.util 2922 :info :write [14 4] :RequestCanceledException INFO [2019-12-17 12:29:13,720] jepsen worker 48 - jepsen.util 2118 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:29:13,720] jepsen worker 77 - jepsen.util 2597 :info :cas [25 [28 22]] :RequestCanceledException INFO [2019-12-17 12:29:13,721] jepsen worker 48 - jepsen.util 2118 :invoke :read [16 nil] INFO [2019-12-17 12:29:13,721] jepsen worker 42 - jepsen.util 3012 :invoke :write [14 16] INFO [2019-12-17 12:29:13,721] jepsen worker 77 - jepsen.util 2687 :invoke :read [25 nil] INFO [2019-12-17 12:29:13,730] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11148,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:29:13,730] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11148,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:29:13,736] jepsen worker 65 - jepsen.util 1685 :fail :read [21 nil] :RequestTimeoutException INFO [2019-12-17 12:29:13,736] jepsen worker 65 - jepsen.util 1685 :invoke :write [21 35] INFO [2019-12-17 12:29:13,736] jepsen worker 79 - jepsen.util 1969 :fail :cas [26 [45 46]] :ValueNotSwapFrom INFO [2019-12-17 12:29:13,736] jepsen worker 79 - jepsen.util 1969 :invoke :read [26 nil] INFO [2019-12-17 12:29:13,748] jepsen worker 69 - jepsen.util 1959 :fail :cas [23 [18 26]] :ValueNotSwapFrom INFO [2019-12-17 12:29:13,748] jepsen worker 69 - jepsen.util 1959 :invoke :cas [23 [15 3]] INFO [2019-12-17 12:29:13,754] jepsen worker 34 - jepsen.util 2554 :info :write [11 12] :RequestCanceledException INFO [2019-12-17 12:29:13,754] jepsen worker 34 - jepsen.util 2644 :invoke :write [11 23] INFO [2019-12-17 12:29:13,773] jepsen worker 65 - jepsen.util 1685 :ok :write [21 35] INFO [2019-12-17 12:29:13,773] jepsen worker 65 - jepsen.util 1685 :invoke :write [21 9] INFO [2019-12-17 12:29:13,776] jepsen worker 4 - jepsen.util 1444 :ok :read [1 10] INFO [2019-12-17 12:29:13,777] jepsen worker 4 - jepsen.util 1444 :invoke :read [1 nil] INFO [2019-12-17 12:29:13,789] jepsen worker 67 - jepsen.util 2947 :info :cas [22 [47 29]] :RequestCanceledException INFO [2019-12-17 12:29:13,791] jepsen worker 67 - jepsen.util 3037 :invoke :read [22 nil] INFO [2019-12-17 12:29:13,807] jepsen worker 56 - jepsen.util 1856 :info :write [18 32] :RequestCanceledException INFO [2019-12-17 12:29:13,809] jepsen worker 56 - jepsen.util 1946 :invoke :cas [18 [17 25]] INFO [2019-12-17 12:29:13,822] jepsen worker 12 - jepsen.util 1902 :info :write [4 6] :RequestCanceledException INFO [2019-12-17 12:29:13,823] jepsen worker 12 - jepsen.util 1992 :invoke :write [4 30] INFO [2019-12-17 12:29:13,824] jepsen worker 65 - jepsen.util 1685 :ok :write [21 9] INFO [2019-12-17 12:29:13,824] jepsen worker 65 - jepsen.util 1685 :invoke :read [21 nil] INFO [2019-12-17 12:29:13,842] jepsen worker 65 - jepsen.util 1685 :ok :read [21 9] INFO [2019-12-17 12:29:13,842] jepsen worker 65 - jepsen.util 1685 :invoke :write [21 2] INFO [2019-12-17 12:29:13,908] jepsen worker 65 - jepsen.util 1685 :ok :write [21 2] INFO [2019-12-17 12:29:13,908] jepsen worker 65 - jepsen.util 1685 :invoke :write [21 27] INFO [2019-12-17 12:29:13,923] jepsen worker 11 - jepsen.util 2351 :info :write [3 5] :RequestCanceledException INFO [2019-12-17 12:29:13,923] jepsen worker 32 - jepsen.util 2282 :info :cas [10 [30 36]] :RequestCanceledException INFO [2019-12-17 12:29:13,923] jepsen worker 31 - jepsen.util 2011 :ok :write [10 29] INFO [2019-12-17 12:29:13,923] jepsen worker 10 - jepsen.util 1990 :ok :write [3 40] INFO [2019-12-17 12:29:13,923] jepsen worker 10 - jepsen.util 1990 :invoke :read [3 nil] INFO [2019-12-17 12:29:13,923] jepsen worker 31 - jepsen.util 2011 :invoke :cas [10 [20 32]] INFO [2019-12-17 12:29:13,924] jepsen worker 32 - jepsen.util 2372 :invoke :write [10 12] INFO [2019-12-17 12:29:13,924] jepsen worker 11 - jepsen.util 2441 :invoke :write [3 17] INFO [2019-12-17 12:29:13,929] jepsen worker 20 - jepsen.util 1640 :ok :write [6 1] INFO [2019-12-17 12:29:13,929] jepsen worker 47 - jepsen.util 2387 :ok :write [15 34] INFO [2019-12-17 12:29:13,931] jepsen worker 69 - jepsen.util 1959 :fail :cas [23 [15 3]] :ValueNotSwapFrom INFO [2019-12-17 12:29:13,936] jepsen worker 2 - jepsen.util 2342 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:29:13,946] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11175,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:29:13,956] jepsen worker 20 - jepsen.util 1640 :invoke :cas [6 [12 49]] INFO [2019-12-17 12:29:13,956] jepsen worker 69 - jepsen.util 1959 :invoke :cas [23 [33 44]] INFO [2019-12-17 12:29:13,956] jepsen worker 47 - jepsen.util 2387 :invoke :cas [15 [41 35]] INFO [2019-12-17 12:29:13,956] jepsen worker 2 - jepsen.util 2342 :invoke :cas [0 [32 15]] INFO [2019-12-17 12:29:13,956] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11175,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:29:13,956] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11179,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:29:13,956] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11179,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:29:13,959] jepsen worker 31 - jepsen.util 2011 :fail :cas [10 [20 32]] :ValueNotSwapFrom INFO [2019-12-17 12:29:13,960] jepsen worker 31 - jepsen.util 2011 :invoke :write [10 15] INFO [2019-12-17 12:29:13,975] jepsen worker 10 - jepsen.util 1990 :ok :read [3 40] INFO [2019-12-17 12:29:13,975] jepsen worker 10 - jepsen.util 1990 :invoke :write [3 5] INFO [2019-12-17 12:29:13,989] jepsen worker 4 - jepsen.util 1444 :ok :read [1 10] INFO [2019-12-17 12:29:13,989] jepsen worker 4 - jepsen.util 1444 :invoke :read [1 nil] INFO [2019-12-17 12:29:14,004] jepsen worker 20 - jepsen.util 1640 :fail :cas [6 [12 49]] :ValueNotSwapFrom INFO [2019-12-17 12:29:14,005] jepsen worker 20 - jepsen.util 1640 :invoke :read [6 nil] INFO [2019-12-17 12:29:14,021] jepsen worker 47 - jepsen.util 2387 :fail :cas [15 [41 35]] :ValueNotSwapFrom INFO [2019-12-17 12:29:14,022] jepsen worker 47 - jepsen.util 2387 :invoke :write [15 11] INFO [2019-12-17 12:29:14,022] jepsen worker 18 - jepsen.util 2178 :fail :read [6 nil] :RequestTimeoutException INFO [2019-12-17 12:29:14,022] jepsen worker 46 - jepsen.util 2026 :info :cas [15 [17 32]] :RequestCanceledException INFO [2019-12-17 12:29:14,023] jepsen worker 18 - jepsen.util 2178 :invoke :read [6 nil] INFO [2019-12-17 12:29:14,029] jepsen worker 46 - jepsen.util 2116 :invoke :read [15 nil] INFO [2019-12-17 12:29:14,036] jepsen worker 80 - jepsen.util 1880 :info :write [26 15] :RequestCanceledException INFO [2019-12-17 12:29:14,036] jepsen worker 49 - jepsen.util 1849 :info :cas [16 [30 5]] :RequestCanceledException INFO [2019-12-17 12:29:14,038] jepsen worker 49 - jepsen.util 1939 :invoke :cas [16 [1 24]] INFO [2019-12-17 12:29:14,038] jepsen worker 80 - jepsen.util 1970 :invoke :read [26 nil] INFO [2019-12-17 12:29:14,063] jepsen worker 44 - jepsen.util 2114 :info :write [14 8] :RequestCanceledException INFO [2019-12-17 12:29:14,065] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11192,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:29:14,067] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11192,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:29:14,069] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11193,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:29:14,081] jepsen worker 65 - jepsen.util 1685 :ok :write [21 27] INFO [2019-12-17 12:29:14,085] jepsen worker 65 - jepsen.util 1685 :invoke :cas [21 [14 28]] INFO [2019-12-17 12:29:14,085] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11193,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:29:14,094] jepsen worker 44 - jepsen.util 2204 :invoke :write [14 30] INFO [2019-12-17 12:29:14,099] jepsen worker 20 - jepsen.util 1640 :ok :read [6 1] INFO [2019-12-17 12:29:14,099] jepsen worker 20 - jepsen.util 1640 :invoke :cas [6 [3 47]] INFO [2019-12-17 12:29:14,129] jepsen worker 20 - jepsen.util 1640 :fail :cas [6 [3 47]] :ValueNotSwapFrom INFO [2019-12-17 12:29:14,129] jepsen worker 20 - jepsen.util 1640 :invoke :read [6 nil] INFO [2019-12-17 12:29:14,147] jepsen worker 56 - jepsen.util 1946 :fail :cas [18 [17 25]] :ValueNotSwapFrom INFO [2019-12-17 12:29:14,148] jepsen worker 56 - jepsen.util 1946 :invoke :cas [18 [10 41]] INFO [2019-12-17 12:29:14,150] jepsen worker 20 - jepsen.util 1640 :ok :read [6 1] INFO [2019-12-17 12:29:14,150] jepsen worker 20 - jepsen.util 1640 :invoke :write [6 14] INFO [2019-12-17 12:29:14,154] jepsen worker 27 - jepsen.util 1647 :info :write [9 2] :RequestCanceledException INFO [2019-12-17 12:29:14,154] jepsen worker 30 - jepsen.util 2190 :info :write [10 10] :RequestCanceledException INFO [2019-12-17 12:29:14,156] jepsen worker 30 - jepsen.util 2280 :invoke :cas [10 [3 8]] INFO [2019-12-17 12:29:14,156] jepsen worker 27 - jepsen.util 1737 :invoke :write [9 25] INFO [2019-12-17 12:29:14,237] jepsen worker 45 - jepsen.util 2115 :info :write [15 45] :RequestCanceledException INFO [2019-12-17 12:29:14,239] jepsen worker 45 - jepsen.util 2205 :invoke :write [15 31] INFO [2019-12-17 12:29:14,245] jepsen worker 80 - jepsen.util 1970 :ok :read [26 38] INFO [2019-12-17 12:29:14,245] jepsen worker 80 - jepsen.util 1970 :invoke :read [26 nil] INFO [2019-12-17 12:29:14,279] jepsen worker 79 - jepsen.util 1969 :ok :read [26 38] INFO [2019-12-17 12:29:14,280] jepsen worker 79 - jepsen.util 1969 :invoke :write [26 11] INFO [2019-12-17 12:29:14,284] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11218,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:29:14,284] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11218,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:29:14,285] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11220,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:29:14,285] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11220,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:29:14,286] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11212,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:29:14,286] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11212,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:29:14,286] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11213,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:29:14,286] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11213,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:29:14,286] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11214,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:29:14,287] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11214,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:29:14,300] jepsen worker 69 - jepsen.util 1959 :fail :cas [23 [33 44]] :ValueNotSwapFrom INFO [2019-12-17 12:29:14,300] jepsen worker 69 - jepsen.util 1959 :invoke :read [23 nil] INFO [2019-12-17 12:29:14,329] jepsen worker 56 - jepsen.util 1946 :fail :cas [18 [10 41]] :ValueNotSwapFrom INFO [2019-12-17 12:29:14,329] jepsen worker 56 - jepsen.util 1946 :invoke :cas [18 [33 42]] INFO [2019-12-17 12:29:14,349] jepsen worker 54 - jepsen.util 1944 :ok :read [18 11] INFO [2019-12-17 12:29:14,350] jepsen worker 54 - jepsen.util 1944 :invoke :write [18 7] INFO [2019-12-17 12:29:14,352] jepsen worker 4 - jepsen.util 1444 :ok :read [1 10] INFO [2019-12-17 12:29:14,352] jepsen worker 4 - jepsen.util 1444 :invoke :cas [1 [45 38]] INFO [2019-12-17 12:29:14,406] jepsen worker 6 - jepsen.util 2616 :info :write [2 14] :RequestCanceledException INFO [2019-12-17 12:29:14,427] jepsen worker 63 - jepsen.util 1773 :info :cas [21 [1 8]] :RequestCanceledException INFO [2019-12-17 12:29:14,457] jepsen worker 6 - jepsen.util 2706 :invoke :read [2 nil] INFO [2019-12-17 12:29:14,503] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11234,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:29:14,503] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11234,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:29:14,503] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11236,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:29:14,503] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11236,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:29:14,503] jepsen worker 60 - jepsen.util 2580 :info :cas [20 [13 41]] :RequestCanceledException INFO [2019-12-17 12:29:14,508] jepsen worker 60 - jepsen.util 2670 :invoke :cas [20 [15 16]] INFO [2019-12-17 12:29:14,508] jepsen worker 63 - jepsen.util 1863 :invoke :read [21 nil] INFO [2019-12-17 12:29:14,532] jepsen worker 36 - jepsen.util 1836 :ok :read [12 27] INFO [2019-12-17 12:29:14,532] jepsen worker 36 - jepsen.util 1836 :invoke :read [12 nil] INFO [2019-12-17 12:29:14,598] jepsen worker 87 - jepsen.util 1617 :info :write [29 17] :RequestCanceledException INFO [2019-12-17 12:29:14,627] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:29:14,627] jepsen nemesis - couchbase.util Rebalance check count: 79 INFO [2019-12-17 12:29:14,628] jepsen nemesis - couchbase.util Rebalance stuck count: 60 INFO [2019-12-17 12:29:14,652] jepsen worker 51 - jepsen.util 1761 :info :write [17 40] :RequestCanceledException INFO [2019-12-17 12:29:14,654] jepsen worker 51 - jepsen.util 1851 :invoke :write [17 37] INFO [2019-12-17 12:29:14,663] jepsen worker 80 - jepsen.util 1970 :ok :read [26 38] INFO [2019-12-17 12:29:14,663] jepsen worker 80 - jepsen.util 1970 :invoke :read [26 nil] INFO [2019-12-17 12:29:14,683] jepsen worker 87 - jepsen.util 1707 :invoke :read [29 nil] INFO [2019-12-17 12:29:14,687] jepsen worker 20 - jepsen.util 1640 :ok :write [6 14] INFO [2019-12-17 12:29:14,687] jepsen worker 31 - jepsen.util 2011 :ok :write [10 15] INFO [2019-12-17 12:29:14,687] jepsen worker 10 - jepsen.util 1990 :ok :write [3 5] INFO [2019-12-17 12:29:14,687] jepsen worker 20 - jepsen.util 1640 :invoke :read [6 nil] INFO [2019-12-17 12:29:14,687] jepsen worker 72 - jepsen.util 2142 :ok :write [24 40] INFO [2019-12-17 12:29:14,687] jepsen worker 31 - jepsen.util 2011 :invoke :cas [10 [19 32]] INFO [2019-12-17 12:29:14,688] jepsen worker 10 - jepsen.util 1990 :invoke :cas [3 [30 30]] INFO [2019-12-17 12:29:14,688] jepsen worker 72 - jepsen.util 2142 :invoke :write [24 44] INFO [2019-12-17 12:29:14,689] jepsen worker 47 - jepsen.util 2387 :ok :write [15 11] INFO [2019-12-17 12:29:14,689] jepsen worker 47 - jepsen.util 2387 :invoke :write [15 21] INFO [2019-12-17 12:29:14,722] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11242,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:29:14,722] jepsen worker 88 - jepsen.util 1888 :info :write [29 30] :RequestCanceledException INFO [2019-12-17 12:29:14,722] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11242,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:29:14,723] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11246,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:29:14,723] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11246,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:29:14,725] jepsen worker 88 - jepsen.util 1978 :invoke :read [29 nil] INFO [2019-12-17 12:29:14,736] jepsen worker 19 - jepsen.util 2179 :info :cas [6 [41 43]] :RequestCanceledException INFO [2019-12-17 12:29:14,738] jepsen worker 19 - jepsen.util 2269 :invoke :cas [6 [1 27]] INFO [2019-12-17 12:29:14,747] jepsen worker 56 - jepsen.util 1946 :fail :cas [18 [33 42]] :ValueNotSwapFrom INFO [2019-12-17 12:29:14,748] jepsen worker 56 - jepsen.util 1946 :invoke :cas [18 [41 13]] INFO [2019-12-17 12:29:14,754] jepsen worker 63 - jepsen.util 1863 :ok :read [21 27] INFO [2019-12-17 12:29:14,754] jepsen worker 7 - jepsen.util 2527 :info :write [2 1] :RequestCanceledException INFO [2019-12-17 12:29:14,764] jepsen worker 63 - jepsen.util 1863 :invoke :cas [21 [35 11]] INFO [2019-12-17 12:29:14,765] jepsen worker 7 - jepsen.util 2617 :invoke :read [2 nil] INFO [2019-12-17 12:29:14,765] jepsen worker 31 - jepsen.util 2011 :fail :cas [10 [19 32]] :ValueNotSwapFrom INFO [2019-12-17 12:29:14,765] jepsen worker 31 - jepsen.util 2011 :invoke :read [10 nil] INFO [2019-12-17 12:29:14,785] jepsen worker 20 - jepsen.util 1640 :ok :read [6 14] INFO [2019-12-17 12:29:14,785] jepsen worker 20 - jepsen.util 1640 :invoke :read [6 nil] INFO [2019-12-17 12:29:14,817] jepsen worker 19 - jepsen.util 2269 :fail :cas [6 [1 27]] :ValueNotSwapFrom INFO [2019-12-17 12:29:14,818] jepsen worker 19 - jepsen.util 2269 :invoke :cas [6 [21 27]] INFO [2019-12-17 12:29:14,825] jepsen worker 63 - jepsen.util 1863 :fail :cas [21 [35 11]] :ValueNotSwapFrom INFO [2019-12-17 12:29:14,826] jepsen worker 63 - jepsen.util 1863 :invoke :read [21 nil] INFO [2019-12-17 12:29:14,830] jepsen worker 31 - jepsen.util 2011 :ok :read [10 15] INFO [2019-12-17 12:29:14,830] jepsen worker 31 - jepsen.util 2011 :invoke :cas [10 [30 7]] INFO [2019-12-17 12:29:14,854] jepsen worker 38 - jepsen.util 2018 :info :write [12 27] :RequestCanceledException INFO [2019-12-17 12:29:14,856] jepsen worker 38 - jepsen.util 2108 :invoke :cas [12 [4 18]] INFO [2019-12-17 12:29:14,861] jepsen worker 19 - jepsen.util 2269 :fail :cas [6 [21 27]] :ValueNotSwapFrom INFO [2019-12-17 12:29:14,862] jepsen worker 19 - jepsen.util 2269 :invoke :read [6 nil] INFO [2019-12-17 12:29:14,868] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11266,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:29:14,868] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11266,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:29:14,868] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11269,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:29:14,869] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11269,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:29:14,869] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11270,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:29:14,869] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11270,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:29:14,893] jepsen worker 63 - jepsen.util 1863 :ok :read [21 27] INFO [2019-12-17 12:29:14,893] jepsen worker 63 - jepsen.util 1863 :invoke :write [21 32] INFO [2019-12-17 12:29:14,907] jepsen worker 31 - jepsen.util 2011 :fail :cas [10 [30 7]] :ValueNotSwapFrom INFO [2019-12-17 12:29:14,907] jepsen worker 31 - jepsen.util 2011 :invoke :write [10 16] INFO [2019-12-17 12:29:14,907] jepsen worker 55 - jepsen.util 2215 :info :write [18 26] :RequestCanceledException INFO [2019-12-17 12:29:14,908] jepsen worker 55 - jepsen.util 2305 :invoke :write [18 29] INFO [2019-12-17 12:29:14,921] jepsen worker 19 - jepsen.util 2269 :ok :read [6 14] INFO [2019-12-17 12:29:14,922] jepsen worker 19 - jepsen.util 2269 :invoke :read [6 nil] INFO [2019-12-17 12:29:14,923] jepsen worker 20 - jepsen.util 1640 :ok :read [6 14] INFO [2019-12-17 12:29:14,923] jepsen worker 78 - jepsen.util 1968 :info :write [26 36] :RequestCanceledException INFO [2019-12-17 12:29:14,923] jepsen worker 20 - jepsen.util 1640 :invoke :write [6 23] INFO [2019-12-17 12:29:14,924] jepsen worker 78 - jepsen.util 2058 :invoke :read [26 nil] INFO [2019-12-17 12:29:14,937] jepsen worker 73 - jepsen.util 1873 :info :write [24 2] :RequestCanceledException INFO [2019-12-17 12:29:14,943] jepsen worker 73 - jepsen.util 1963 :invoke :read [24 nil] INFO [2019-12-17 12:29:15,002] jepsen worker 63 - jepsen.util 1863 :ok :write [21 32] INFO [2019-12-17 12:29:15,002] jepsen worker 63 - jepsen.util 1863 :invoke :read [21 nil] INFO [2019-12-17 12:29:15,017] jepsen worker 9 - jepsen.util 2079 :info :write [3 41] :RequestCanceledException INFO [2019-12-17 12:29:15,018] jepsen worker 37 - jepsen.util 1567 :info :write [12 1] :RequestCanceledException INFO [2019-12-17 12:29:15,020] jepsen worker 37 - jepsen.util 1657 :invoke :cas [12 [49 44]] INFO [2019-12-17 12:29:15,029] jepsen worker 9 - jepsen.util 2169 :invoke :cas [3 [8 3]] INFO [2019-12-17 12:29:15,053] jepsen worker 89 - jepsen.util 1889 :info :write [29 25] :RequestCanceledException INFO [2019-12-17 12:29:15,067] jepsen worker 89 - jepsen.util 1979 :invoke :cas [29 [34 38]] INFO [2019-12-17 12:29:15,084] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11281,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:29:15,085] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11281,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:29:15,086] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11284,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:29:15,087] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11284,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:29:15,088] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11285,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:29:15,089] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11285,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:29:15,104] jepsen worker 9 - jepsen.util 2169 :fail :cas [3 [8 3]] :ValueNotSwapFrom INFO [2019-12-17 12:29:15,105] jepsen worker 9 - jepsen.util 2169 :invoke :cas [3 [47 45]] INFO [2019-12-17 12:29:15,105] jepsen worker 71 - jepsen.util 1601 :info :write [23 11] :RequestCanceledException INFO [2019-12-17 12:29:15,106] jepsen worker 64 - jepsen.util 1594 :info :cas [21 [27 34]] :RequestCanceledException INFO [2019-12-17 12:29:15,121] jepsen worker 71 - jepsen.util 1691 :invoke :cas [23 [24 16]] INFO [2019-12-17 12:29:15,122] jepsen worker 64 - jepsen.util 1684 :invoke :read [21 nil] INFO [2019-12-17 12:29:15,137] jepsen worker 74 - jepsen.util 1874 :info :write [24 47] :RequestCanceledException INFO [2019-12-17 12:29:15,141] jepsen worker 74 - jepsen.util 1964 :invoke :cas [24 [9 35]] INFO [2019-12-17 12:29:15,148] jepsen worker 9 - jepsen.util 2169 :fail :cas [3 [47 45]] :ValueNotSwapFrom INFO [2019-12-17 12:29:15,148] jepsen worker 9 - jepsen.util 2169 :invoke :write [3 0] INFO [2019-12-17 12:29:15,151] jepsen worker 29 - jepsen.util 1649 :fail :read [9 nil] :RequestTimeoutException INFO [2019-12-17 12:29:15,151] jepsen worker 29 - jepsen.util 1649 :invoke :write [9 5] INFO [2019-12-17 12:29:15,181] jepsen worker 36 - jepsen.util 1836 :ok :read [12 27] INFO [2019-12-17 12:29:15,194] jepsen worker 36 - jepsen.util 1836 :invoke :cas [12 [17 29]] INFO [2019-12-17 12:29:15,237] jepsen worker 28 - jepsen.util 1198 :info :write [9 4] :RequestCanceledException INFO [2019-12-17 12:29:15,237] jepsen worker 53 - jepsen.util 1673 :info :write [17 42] :RequestCanceledException INFO [2019-12-17 12:29:15,239] jepsen worker 53 - jepsen.util 1763 :invoke :cas [17 [28 17]] INFO [2019-12-17 12:29:15,239] jepsen worker 28 - jepsen.util 1288 :invoke :write [9 24] INFO [2019-12-17 12:29:15,269] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11296,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:29:15,270] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11296,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:29:15,270] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11305,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:29:15,271] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11305,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:29:15,297] jepsen worker 28 - jepsen.util 1288 :ok :write [9 24] INFO [2019-12-17 12:29:15,298] jepsen worker 28 - jepsen.util 1288 :invoke :read [9 nil] INFO [2019-12-17 12:29:15,360] jepsen worker 15 - jepsen.util 2445 :info :cas [5 [18 4]] :RequestCanceledException INFO [2019-12-17 12:29:15,389] jepsen worker 43 - jepsen.util 2653 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:29:15,389] jepsen worker 43 - jepsen.util 2653 :invoke :cas [14 [21 10]] INFO [2019-12-17 12:29:15,391] jepsen worker 15 - jepsen.util 2535 :invoke :cas [5 [44 6]] INFO [2019-12-17 12:29:15,406] jepsen worker 16 - jepsen.util 2446 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:29:15,407] jepsen worker 16 - jepsen.util 2446 :invoke :cas [5 [38 3]] INFO [2019-12-17 12:29:15,422] jepsen worker 5 - jepsen.util 1625 :info :write [1 13] :RequestCanceledException INFO [2019-12-17 12:29:15,444] jepsen worker 5 - jepsen.util 1715 :invoke :read [1 nil] INFO [2019-12-17 12:29:15,504] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11318,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:29:15,505] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11318,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:29:15,508] jepsen worker 22 - jepsen.util 2722 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:29:15,509] jepsen worker 22 - jepsen.util 2722 :invoke :cas [7 [0 24]] INFO [2019-12-17 12:29:15,555] jepsen worker 58 - jepsen.util 2398 :info :cas [19 [41 40]] :RequestCanceledException INFO [2019-12-17 12:29:15,555] jepsen worker 76 - jepsen.util 2506 :info :write [25 17] :RequestCanceledException INFO [2019-12-17 12:29:15,555] jepsen worker 50 - jepsen.util 2750 :info :write [16 5] :RequestCanceledException INFO [2019-12-17 12:29:15,555] jepsen worker 35 - jepsen.util 2195 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:29:15,555] jepsen worker 80 - jepsen.util 1970 :ok :read [26 38] INFO [2019-12-17 12:29:15,556] jepsen worker 14 - jepsen.util 1904 :info :write [4 12] :RequestCanceledException INFO [2019-12-17 12:29:15,556] jepsen worker 80 - jepsen.util 1970 :invoke :cas [26 [42 19]] INFO [2019-12-17 12:29:15,557] jepsen worker 35 - jepsen.util 2195 :invoke :read [11 nil] INFO [2019-12-17 12:29:15,557] jepsen worker 40 - jepsen.util 3010 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:29:15,558] jepsen worker 59 - jepsen.util 2309 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:29:15,562] jepsen worker 1 - jepsen.util 2521 :info :cas [0 [17 14]] :RequestCanceledException INFO [2019-12-17 12:29:15,562] jepsen worker 21 - jepsen.util 2451 :info :write [7 5] :RequestCanceledException INFO [2019-12-17 12:29:15,562] jepsen worker 86 - jepsen.util 2606 :info :write [28 38] :RequestCanceledException INFO [2019-12-17 12:29:15,562] jepsen worker 0 - jepsen.util 2790 :info :cas [0 [6 44]] :RequestCanceledException INFO [2019-12-17 12:29:15,562] jepsen worker 40 - jepsen.util 3010 :invoke :write [13 39] INFO [2019-12-17 12:29:15,563] jepsen worker 58 - jepsen.util 2488 :invoke :read [19 nil] INFO [2019-12-17 12:29:15,563] jepsen worker 59 - jepsen.util 2309 :invoke :read [19 nil] INFO [2019-12-17 12:29:15,564] jepsen worker 50 - jepsen.util 2840 :invoke :read [16 nil] INFO [2019-12-17 12:29:15,565] jepsen worker 1 - jepsen.util 2611 :invoke :read [0 nil] INFO [2019-12-17 12:29:15,565] jepsen worker 0 - jepsen.util 2880 :invoke :read [0 nil] INFO [2019-12-17 12:29:15,565] jepsen worker 21 - jepsen.util 2541 :invoke :read [7 nil] INFO [2019-12-17 12:29:15,565] jepsen worker 28 - jepsen.util 1288 :ok :read [9 24] INFO [2019-12-17 12:29:15,565] jepsen worker 86 - jepsen.util 2696 :invoke :write [28 10] INFO [2019-12-17 12:29:15,565] jepsen worker 28 - jepsen.util 1288 :invoke :cas [9 [26 39]] INFO [2019-12-17 12:29:15,566] jepsen worker 76 - jepsen.util 2596 :invoke :read [25 nil] INFO [2019-12-17 12:29:15,588] jepsen worker 66 - jepsen.util 2406 :info :cas [22 [16 17]] :RequestCanceledException INFO [2019-12-17 12:29:15,594] jepsen worker 26 - jepsen.util 2816 :info :write [8 8] :RequestCanceledException INFO [2019-12-17 12:29:15,598] jepsen worker 85 - jepsen.util 2875 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:29:15,606] jepsen worker 85 - jepsen.util 2875 :invoke :cas [28 [22 6]] INFO [2019-12-17 12:29:15,607] jepsen worker 81 - jepsen.util 2241 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:29:15,607] jepsen worker 81 - jepsen.util 2241 :invoke :cas [27 [15 27]] INFO [2019-12-17 12:29:15,608] jepsen worker 84 - jepsen.util 2334 :info :cas [28 [26 22]] :RequestCanceledException INFO [2019-12-17 12:29:15,608] jepsen worker 68 - jepsen.util 2318 :info :cas [22 [45 22]] :RequestCanceledException INFO [2019-12-17 12:29:15,608] jepsen worker 25 - jepsen.util 2905 :info :write [8 30] :RequestCanceledException INFO [2019-12-17 12:29:15,618] jepsen worker 14 - jepsen.util 1994 :invoke :read [4 nil] INFO [2019-12-17 12:29:15,624] jepsen worker 61 - jepsen.util 2581 :info :cas [20 [29 27]] :RequestCanceledException INFO [2019-12-17 12:29:15,625] jepsen worker 57 - jepsen.util 2307 :info :write [19 2] :RequestCanceledException INFO [2019-12-17 12:29:15,625] jepsen worker 75 - jepsen.util 2865 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:29:15,625] jepsen worker 24 - jepsen.util 2364 :info :write [8 27] :RequestCanceledException INFO [2019-12-17 12:29:15,626] jepsen worker 75 - jepsen.util 2865 :invoke :cas [25 [35 21]] INFO [2019-12-17 12:29:15,632] jepsen worker 82 - jepsen.util 3052 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:29:15,633] jepsen worker 82 - jepsen.util 3052 :invoke :write [27 44] INFO [2019-12-17 12:29:15,635] jepsen worker 66 - jepsen.util 2496 :invoke :write [22 33] INFO [2019-12-17 12:29:15,643] jepsen worker 23 - jepsen.util 2363 :info :cas [7 [25 8]] :RequestCanceledException INFO [2019-12-17 12:29:15,643] jepsen worker 62 - jepsen.util 2312 :info :write [20 24] :RequestCanceledException INFO [2019-12-17 12:29:15,652] jepsen worker 68 - jepsen.util 2408 :invoke :read [22 nil] INFO [2019-12-17 12:29:15,655] jepsen worker 13 - jepsen.util 1723 :info :write [4 8] :RequestCanceledException INFO [2019-12-17 12:29:15,658] jepsen worker 26 - jepsen.util 2906 :invoke :cas [8 [18 30]] INFO [2019-12-17 12:29:15,658] jepsen worker 25 - jepsen.util 2995 :invoke :cas [8 [20 4]] INFO [2019-12-17 12:29:15,664] jepsen worker 84 - jepsen.util 2424 :invoke :read [28 nil] INFO [2019-12-17 12:29:15,669] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:29:15,670] jepsen worker 61 - jepsen.util 2671 :invoke :write [20 39] INFO [2019-12-17 12:29:15,670] jepsen nemesis - couchbase.util Rebalance check count: 80 INFO [2019-12-17 12:29:15,670] jepsen worker 24 - jepsen.util 2454 :invoke :read [8 nil] INFO [2019-12-17 12:29:15,671] jepsen nemesis - couchbase.util Rebalance stuck count: 61 INFO [2019-12-17 12:29:15,676] jepsen worker 57 - jepsen.util 2397 :invoke :cas [19 [19 27]] INFO [2019-12-17 12:29:15,681] jepsen worker 62 - jepsen.util 2402 :invoke :read [20 nil] INFO [2019-12-17 12:29:15,683] jepsen worker 23 - jepsen.util 2453 :invoke :write [7 48] INFO [2019-12-17 12:29:15,684] jepsen worker 13 - jepsen.util 1813 :invoke :write [4 10] INFO [2019-12-17 12:29:15,690] jepsen worker 3 - jepsen.util 1713 :info :write [1 42] :RequestCanceledException INFO [2019-12-17 12:29:15,690] jepsen worker 17 - jepsen.util 2717 :info :write [5 25] :RequestCanceledException INFO [2019-12-17 12:29:15,690] jepsen worker 8 - jepsen.util 2258 :info :cas [2 [32 16]] :RequestCanceledException INFO [2019-12-17 12:29:15,690] jepsen worker 88 - jepsen.util 1978 :ok :read [29 35] INFO [2019-12-17 12:29:15,691] jepsen worker 88 - jepsen.util 1978 :invoke :read [29 nil] INFO [2019-12-17 12:29:15,710] jepsen worker 83 - jepsen.util 2873 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:29:15,712] jepsen worker 83 - jepsen.util 2873 :invoke :cas [27 [3 37]] INFO [2019-12-17 12:29:15,713] jepsen worker 3 - jepsen.util 1803 :invoke :cas [1 [25 26]] INFO [2019-12-17 12:29:15,716] jepsen worker 17 - jepsen.util 2807 :invoke :read [5 nil] INFO [2019-12-17 12:29:15,717] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11342,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:29:15,719] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11342,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:29:15,719] jepsen worker 8 - jepsen.util 2348 :invoke :read [2 nil] INFO [2019-12-17 12:29:15,720] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11343,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:29:15,722] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11343,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:29:15,723] jepsen worker 70 - jepsen.util 2230 :info :write [23 6] :RequestCanceledException INFO [2019-12-17 12:29:15,723] jepsen worker 39 - jepsen.util 2829 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:29:15,725] jepsen worker 39 - jepsen.util 2829 :invoke :cas [13 [17 10]] INFO [2019-12-17 12:29:15,732] jepsen worker 31 - jepsen.util 2011 :ok :write [10 16] INFO [2019-12-17 12:29:15,732] jepsen worker 45 - jepsen.util 2205 :ok :write [15 31] INFO [2019-12-17 12:29:15,732] jepsen worker 45 - jepsen.util 2205 :invoke :write [15 49] INFO [2019-12-17 12:29:15,735] jepsen worker 31 - jepsen.util 2011 :invoke :write [10 27] INFO [2019-12-17 12:29:15,736] jepsen worker 42 - jepsen.util 3012 :info :write [14 16] :RequestCanceledException INFO [2019-12-17 12:29:15,737] jepsen worker 41 - jepsen.util 2561 :info :write [13 24] :RequestCanceledException INFO [2019-12-17 12:29:15,744] jepsen worker 11 - jepsen.util 2441 :ok :write [3 17] INFO [2019-12-17 12:29:15,744] jepsen worker 11 - jepsen.util 2441 :invoke :read [3 nil] INFO [2019-12-17 12:29:15,746] jepsen worker 20 - jepsen.util 1640 :ok :write [6 23] INFO [2019-12-17 12:29:15,746] jepsen worker 20 - jepsen.util 1640 :invoke :write [6 18] INFO [2019-12-17 12:29:15,757] jepsen worker 33 - jepsen.util 2913 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:29:15,757] jepsen worker 33 - jepsen.util 2913 :invoke :read [11 nil] INFO [2019-12-17 12:29:15,758] jepsen worker 77 - jepsen.util 2687 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:29:15,758] jepsen worker 77 - jepsen.util 2687 :invoke :read [25 nil] INFO [2019-12-17 12:29:15,758] jepsen worker 70 - jepsen.util 2320 :invoke :cas [23 [44 2]] INFO [2019-12-17 12:29:15,762] jepsen worker 41 - jepsen.util 2651 :invoke :read [13 nil] INFO [2019-12-17 12:29:15,775] jepsen worker 42 - jepsen.util 3102 :invoke :cas [14 [28 49]] INFO [2019-12-17 12:29:15,794] jepsen worker 52 - jepsen.util 1942 :info :write [17 44] :RequestCanceledException INFO [2019-12-17 12:29:15,814] jepsen worker 52 - jepsen.util 2032 :invoke :write [17 26] INFO [2019-12-17 12:29:15,825] jepsen worker 48 - jepsen.util 2118 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:29:15,826] jepsen worker 34 - jepsen.util 2644 :info :write [11 23] :RequestCanceledException INFO [2019-12-17 12:29:15,830] jepsen worker 48 - jepsen.util 2118 :invoke :read [16 nil] INFO [2019-12-17 12:29:15,859] jepsen worker 72 - jepsen.util 2142 :ok :write [24 44] INFO [2019-12-17 12:29:15,859] jepsen worker 72 - jepsen.util 2142 :invoke :write [24 48] INFO [2019-12-17 12:29:15,866] jepsen worker 34 - jepsen.util 2734 :invoke :write [11 6] INFO [2019-12-17 12:29:15,889] jepsen worker 67 - jepsen.util 3037 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:29:15,889] jepsen worker 67 - jepsen.util 3037 :invoke :write [22 44] INFO [2019-12-17 12:29:15,911] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11367,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:29:15,913] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11367,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:29:15,918] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11371,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:29:15,919] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11371,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:29:15,919] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11372,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:29:15,925] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11372,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:29:15,925] jepsen worker 12 - jepsen.util 1992 :info :write [4 30] :RequestCanceledException INFO [2019-12-17 12:29:15,992] jepsen worker 12 - jepsen.util 2082 :invoke :write [4 39] INFO [2019-12-17 12:29:16,027] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11377,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:29:16,034] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11377,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:29:16,040] jepsen worker 2 - jepsen.util 2342 :info :cas [0 [32 15]] :RequestCanceledException INFO [2019-12-17 12:29:16,045] jepsen worker 32 - jepsen.util 2372 :info :write [10 12] :RequestCanceledException INFO [2019-12-17 12:29:16,045] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11381,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:29:16,047] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11381,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:29:16,085] jepsen worker 32 - jepsen.util 2462 :invoke :cas [10 [16 47]] INFO [2019-12-17 12:29:16,092] jepsen worker 49 - jepsen.util 1939 :info :cas [16 [1 24]] :RequestCanceledException INFO [2019-12-17 12:29:16,093] jepsen worker 2 - jepsen.util 2432 :invoke :cas [0 [40 27]] INFO [2019-12-17 12:29:16,123] jepsen worker 18 - jepsen.util 2178 :fail :read [6 nil] :RequestTimeoutException INFO [2019-12-17 12:29:16,123] jepsen worker 44 - jepsen.util 2204 :info :write [14 30] :RequestCanceledException INFO [2019-12-17 12:29:16,123] jepsen worker 18 - jepsen.util 2178 :invoke :write [6 32] INFO [2019-12-17 12:29:16,129] jepsen worker 54 - jepsen.util 1944 :ok :write [18 7] INFO [2019-12-17 12:29:16,129] jepsen worker 54 - jepsen.util 1944 :invoke :read [18 nil] INFO [2019-12-17 12:29:16,132] jepsen worker 46 - jepsen.util 2116 :fail :read [15 nil] :RequestTimeoutException INFO [2019-12-17 12:29:16,132] jepsen worker 46 - jepsen.util 2116 :invoke :write [15 36] INFO [2019-12-17 12:29:16,146] jepsen worker 65 - jepsen.util 1685 :info :cas [21 [14 28]] :RequestCanceledException INFO [2019-12-17 12:29:16,171] jepsen worker 49 - jepsen.util 2029 :invoke :cas [16 [19 19]] INFO [2019-12-17 12:29:16,203] jepsen worker 44 - jepsen.util 2294 :invoke :cas [14 [20 44]] INFO [2019-12-17 12:29:16,207] jepsen worker 65 - jepsen.util 1775 :invoke :write [21 32] INFO [2019-12-17 12:29:16,236] jepsen worker 30 - jepsen.util 2280 :info :cas [10 [3 8]] :RequestCanceledException INFO [2019-12-17 12:29:16,258] jepsen worker 27 - jepsen.util 1737 :info :write [9 25] :RequestCanceledException INFO [2019-12-17 12:29:16,277] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11391,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:29:16,278] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11391,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:29:16,278] jepsen worker 27 - jepsen.util 1827 :invoke :cas [9 [12 41]] INFO [2019-12-17 12:29:16,278] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11393,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:29:16,284] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11393,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:29:16,286] jepsen worker 30 - jepsen.util 2370 :invoke :cas [10 [18 21]] INFO [2019-12-17 12:29:16,302] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11401,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:29:16,305] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11401,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:29:16,306] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11402,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:29:16,307] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11402,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:29:16,308] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11403,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:29:16,316] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11403,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:29:16,325] jepsen worker 51 - jepsen.util 1851 :ok :write [17 37] INFO [2019-12-17 12:29:16,325] jepsen worker 51 - jepsen.util 1851 :invoke :cas [17 [47 26]] INFO [2019-12-17 12:29:16,326] jepsen worker 79 - jepsen.util 1969 :info :write [26 11] :RequestCanceledException INFO [2019-12-17 12:29:16,340] jepsen worker 79 - jepsen.util 2059 :invoke :read [26 nil] INFO [2019-12-17 12:29:16,389] jepsen worker 4 - jepsen.util 1444 :info :cas [1 [45 38]] :RequestCanceledException INFO [2019-12-17 12:29:16,426] jepsen worker 4 - jepsen.util 1534 :invoke :read [1 nil] INFO [2019-12-17 12:29:16,456] jepsen worker 55 - jepsen.util 2305 :ok :write [18 29] INFO [2019-12-17 12:29:16,469] jepsen worker 56 - jepsen.util 1946 :fail :cas [18 [41 13]] :ValueNotSwapFrom INFO [2019-12-17 12:29:16,483] jepsen worker 38 - jepsen.util 2108 :fail :cas [12 [4 18]] :ValueNotSwapFrom INFO [2019-12-17 12:29:16,486] jepsen worker 6 - jepsen.util 2706 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:29:16,522] jepsen worker 55 - jepsen.util 2305 :invoke :cas [18 [41 16]] INFO [2019-12-17 12:29:16,523] jepsen worker 6 - jepsen.util 2706 :invoke :write [2 20] INFO [2019-12-17 12:29:16,523] jepsen worker 38 - jepsen.util 2108 :invoke :write [12 36] INFO [2019-12-17 12:29:16,523] jepsen worker 56 - jepsen.util 1946 :invoke :write [18 23] INFO [2019-12-17 12:29:16,540] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11414,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:29:16,544] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11414,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:29:16,545] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11418,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:29:16,552] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11418,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:29:16,558] jepsen worker 20 - jepsen.util 1640 :ok :write [6 18] INFO [2019-12-17 12:29:16,559] jepsen worker 20 - jepsen.util 1640 :invoke :cas [6 [36 6]] INFO [2019-12-17 12:29:16,570] jepsen worker 69 - jepsen.util 1959 :ok :read [23 28] INFO [2019-12-17 12:29:16,570] jepsen worker 69 - jepsen.util 1959 :invoke :read [23 nil] INFO [2019-12-17 12:29:16,574] jepsen worker 72 - jepsen.util 2142 :ok :write [24 48] INFO [2019-12-17 12:29:16,576] jepsen worker 72 - jepsen.util 2142 :invoke :cas [24 [14 14]] INFO [2019-12-17 12:29:16,640] jepsen worker 60 - jepsen.util 2670 :info :cas [20 [15 16]] :RequestCanceledException INFO [2019-12-17 12:29:16,684] jepsen worker 60 - jepsen.util 2760 :invoke :cas [20 [35 28]] INFO [2019-12-17 12:29:16,707] jepsen worker 47 - jepsen.util 2387 :info :write [15 21] :RequestCanceledException INFO [2019-12-17 12:29:16,710] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:29:16,711] jepsen nemesis - couchbase.util Rebalance check count: 81 INFO [2019-12-17 12:29:16,712] jepsen nemesis - couchbase.util Rebalance stuck count: 62 INFO [2019-12-17 12:29:16,725] jepsen worker 10 - jepsen.util 1990 :info :cas [3 [30 30]] :RequestCanceledException INFO [2019-12-17 12:29:16,736] jepsen worker 47 - jepsen.util 2477 :invoke :read [15 nil] INFO [2019-12-17 12:29:16,741] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11425,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:29:16,742] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11425,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:29:16,743] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11430,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:29:16,745] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11430,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:29:16,760] jepsen worker 10 - jepsen.util 2080 :invoke :read [3 nil] INFO [2019-12-17 12:29:16,762] jepsen worker 71 - jepsen.util 1691 :fail :cas [23 [24 16]] :ValueNotSwapFrom INFO [2019-12-17 12:29:16,763] jepsen worker 71 - jepsen.util 1691 :invoke :read [23 nil] INFO [2019-12-17 12:29:16,796] jepsen worker 78 - jepsen.util 2058 :ok :read [26 38] INFO [2019-12-17 12:29:16,797] jepsen worker 78 - jepsen.util 2058 :invoke :write [26 9] INFO [2019-12-17 12:29:16,806] jepsen worker 7 - jepsen.util 2617 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:29:16,808] jepsen worker 7 - jepsen.util 2617 :invoke :cas [2 [46 20]] INFO [2019-12-17 12:29:16,856] jepsen worker 89 - jepsen.util 1979 :fail :cas [29 [34 38]] :ValueNotSwapFrom INFO [2019-12-17 12:29:16,858] jepsen worker 89 - jepsen.util 1979 :invoke :cas [29 [8 6]] INFO [2019-12-17 12:29:16,865] jepsen worker 28 - jepsen.util 1288 :fail :cas [9 [26 39]] :ValueNotSwapFrom INFO [2019-12-17 12:29:16,866] jepsen worker 28 - jepsen.util 1288 :invoke :cas [9 [46 7]] INFO [2019-12-17 12:29:16,868] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11438,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:29:16,870] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11438,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:29:16,892] jepsen worker 36 - jepsen.util 1836 :fail :cas [12 [17 29]] :ValueNotSwapFrom INFO [2019-12-17 12:29:16,893] jepsen worker 36 - jepsen.util 1836 :invoke :cas [12 [44 14]] INFO [2019-12-17 12:29:16,918] jepsen worker 13 - jepsen.util 1813 :ok :write [4 10] INFO [2019-12-17 12:29:16,919] jepsen worker 13 - jepsen.util 1813 :invoke :cas [4 [25 13]] INFO [2019-12-17 12:29:16,938] jepsen worker 19 - jepsen.util 2269 :ok :read [6 14] INFO [2019-12-17 12:29:16,939] jepsen worker 19 - jepsen.util 2269 :invoke :read [6 nil] INFO [2019-12-17 12:29:16,946] jepsen worker 73 - jepsen.util 1963 :ok :read [24 40] INFO [2019-12-17 12:29:16,947] jepsen worker 73 - jepsen.util 1963 :invoke :write [24 11] INFO [2019-12-17 12:29:16,968] jepsen worker 74 - jepsen.util 1964 :fail :cas [24 [9 35]] :ValueNotSwapFrom INFO [2019-12-17 12:29:16,968] jepsen worker 74 - jepsen.util 1964 :invoke :read [24 nil] INFO [2019-12-17 12:29:16,993] jepsen worker 11 - jepsen.util 2441 :ok :read [3 17] INFO [2019-12-17 12:29:16,995] jepsen worker 11 - jepsen.util 2441 :invoke :cas [3 [24 5]] INFO [2019-12-17 12:29:17,023] jepsen worker 63 - jepsen.util 1863 :fail :read [21 nil] :RequestTimeoutException INFO [2019-12-17 12:29:17,024] jepsen worker 63 - jepsen.util 1863 :invoke :write [21 43] INFO [2019-12-17 12:29:17,053] jepsen worker 27 - jepsen.util 1827 :fail :cas [9 [12 41]] :ValueNotSwapFrom INFO [2019-12-17 12:29:17,054] jepsen worker 27 - jepsen.util 1827 :invoke :cas [9 [41 32]] INFO [2019-12-17 12:29:17,087] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11453,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:29:17,087] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11453,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:29:17,088] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11455,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:29:17,088] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11455,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:29:17,088] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11456,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:29:17,089] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11456,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:29:17,102] jepsen worker 51 - jepsen.util 1851 :fail :cas [17 [47 26]] :ValueNotSwapFrom INFO [2019-12-17 12:29:17,102] jepsen worker 51 - jepsen.util 1851 :invoke :cas [17 [48 1]] INFO [2019-12-17 12:29:17,103] jepsen worker 87 - jepsen.util 1707 :ok :read [29 35] INFO [2019-12-17 12:29:17,104] jepsen worker 87 - jepsen.util 1707 :invoke :cas [29 [26 24]] INFO [2019-12-17 12:29:17,105] jepsen worker 70 - jepsen.util 2320 :fail :cas [23 [44 2]] :ValueNotSwapFrom INFO [2019-12-17 12:29:17,105] jepsen worker 70 - jepsen.util 2320 :invoke :cas [23 [43 14]] INFO [2019-12-17 12:29:17,107] jepsen worker 30 - jepsen.util 2370 :fail :cas [10 [18 21]] :ValueNotSwapFrom INFO [2019-12-17 12:29:17,107] jepsen worker 30 - jepsen.util 2370 :invoke :read [10 nil] INFO [2019-12-17 12:29:17,112] jepsen worker 14 - jepsen.util 1994 :ok :read [4 37] INFO [2019-12-17 12:29:17,113] jepsen worker 14 - jepsen.util 1994 :invoke :cas [4 [24 2]] INFO [2019-12-17 12:29:17,116] jepsen worker 79 - jepsen.util 2059 :ok :read [26 38] INFO [2019-12-17 12:29:17,116] jepsen worker 79 - jepsen.util 2059 :invoke :read [26 nil] INFO [2019-12-17 12:29:17,116] jepsen worker 80 - jepsen.util 1970 :fail :cas [26 [42 19]] :ValueNotSwapFrom INFO [2019-12-17 12:29:17,116] jepsen worker 80 - jepsen.util 1970 :invoke :read [26 nil] INFO [2019-12-17 12:29:17,120] jepsen worker 4 - jepsen.util 1534 :ok :read [1 10] INFO [2019-12-17 12:29:17,120] jepsen worker 4 - jepsen.util 1534 :invoke :cas [1 [20 12]] INFO [2019-12-17 12:29:17,123] jepsen worker 5 - jepsen.util 1715 :ok :read [1 10] INFO [2019-12-17 12:29:17,123] jepsen worker 5 - jepsen.util 1715 :invoke :write [1 39] INFO [2019-12-17 12:29:17,123] jepsen worker 54 - jepsen.util 1944 :ok :read [18 7] INFO [2019-12-17 12:29:17,124] jepsen worker 54 - jepsen.util 1944 :invoke :cas [18 [12 5]] INFO [2019-12-17 12:29:17,125] jepsen worker 37 - jepsen.util 1657 :fail :cas [12 [49 44]] :ValueNotSwapFrom INFO [2019-12-17 12:29:17,125] jepsen worker 37 - jepsen.util 1657 :invoke :write [12 37] INFO [2019-12-17 12:29:17,132] jepsen worker 72 - jepsen.util 2142 :fail :cas [24 [14 14]] :ValueNotSwapFrom INFO [2019-12-17 12:29:17,132] jepsen worker 72 - jepsen.util 2142 :invoke :write [24 3] INFO [2019-12-17 12:29:17,146] jepsen worker 28 - jepsen.util 1288 :fail :cas [9 [46 7]] :ValueNotSwapFrom INFO [2019-12-17 12:29:17,146] jepsen worker 28 - jepsen.util 1288 :invoke :read [9 nil] INFO [2019-12-17 12:29:17,146] jepsen worker 64 - jepsen.util 1684 :fail :read [21 nil] :RequestTimeoutException INFO [2019-12-17 12:29:17,146] jepsen worker 64 - jepsen.util 1684 :invoke :read [21 nil] INFO [2019-12-17 12:29:17,158] jepsen worker 20 - jepsen.util 1640 :fail :cas [6 [36 6]] :ValueNotSwapFrom INFO [2019-12-17 12:29:17,158] jepsen worker 20 - jepsen.util 1640 :invoke :read [6 nil] INFO [2019-12-17 12:29:17,163] jepsen worker 11 - jepsen.util 2441 :fail :cas [3 [24 5]] :ValueNotSwapFrom INFO [2019-12-17 12:29:17,163] jepsen worker 11 - jepsen.util 2441 :invoke :cas [3 [12 12]] INFO [2019-12-17 12:29:17,164] jepsen worker 89 - jepsen.util 1979 :fail :cas [29 [8 6]] :ValueNotSwapFrom INFO [2019-12-17 12:29:17,164] jepsen worker 89 - jepsen.util 1979 :invoke :cas [29 [10 48]] INFO [2019-12-17 12:29:17,166] jepsen worker 10 - jepsen.util 2080 :ok :read [3 17] INFO [2019-12-17 12:29:17,166] jepsen worker 10 - jepsen.util 2080 :invoke :write [3 29] INFO [2019-12-17 12:29:17,204] jepsen worker 9 - jepsen.util 2169 :info :write [3 0] :RequestCanceledException INFO [2019-12-17 12:29:17,207] jepsen worker 9 - jepsen.util 2259 :invoke :write [3 17] INFO [2019-12-17 12:29:17,339] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11481,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:29:17,339] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11488,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:29:17,339] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11481,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:29:17,339] jepsen worker 53 - jepsen.util 1763 :info :cas [17 [28 17]] :RequestCanceledException INFO [2019-12-17 12:29:17,339] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11488,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:29:17,339] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11482,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:29:17,339] jepsen worker 29 - jepsen.util 1649 :info :write [9 5] :RequestCanceledException INFO [2019-12-17 12:29:17,339] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11491,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:29:17,339] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11482,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:29:17,340] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11491,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:29:17,340] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11483,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:29:17,340] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11492,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:29:17,340] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11483,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:29:17,340] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11492,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:29:17,341] jepsen worker 53 - jepsen.util 1853 :invoke :write [17 22] INFO [2019-12-17 12:29:17,341] jepsen worker 29 - jepsen.util 1739 :invoke :cas [9 [42 11]] INFO [2019-12-17 12:29:17,341] jepsen worker 64 - jepsen.util 1684 :ok :read [21 32] INFO [2019-12-17 12:29:17,341] jepsen worker 64 - jepsen.util 1684 :invoke :read [21 nil] INFO [2019-12-17 12:29:17,343] jepsen worker 27 - jepsen.util 1827 :fail :cas [9 [41 32]] :ValueNotSwapFrom INFO [2019-12-17 12:29:17,343] jepsen worker 27 - jepsen.util 1827 :invoke :write [9 40] INFO [2019-12-17 12:29:17,350] jepsen worker 36 - jepsen.util 1836 :fail :cas [12 [44 14]] :ValueNotSwapFrom INFO [2019-12-17 12:29:17,351] jepsen worker 28 - jepsen.util 1288 :ok :read [9 24] INFO [2019-12-17 12:29:17,351] jepsen worker 36 - jepsen.util 1836 :invoke :write [12 34] INFO [2019-12-17 12:29:17,351] jepsen worker 28 - jepsen.util 1288 :invoke :cas [9 [29 45]] INFO [2019-12-17 12:29:17,351] jepsen worker 13 - jepsen.util 1813 :fail :cas [4 [25 13]] :ValueNotSwapFrom INFO [2019-12-17 12:29:17,351] jepsen worker 13 - jepsen.util 1813 :invoke :write [4 39] INFO [2019-12-17 12:29:17,381] jepsen worker 55 - jepsen.util 2305 :fail :cas [18 [41 16]] :ValueNotSwapFrom INFO [2019-12-17 12:29:17,381] jepsen worker 55 - jepsen.util 2305 :invoke :read [18 nil] INFO [2019-12-17 12:29:17,417] jepsen worker 63 - jepsen.util 1863 :ok :write [21 43] INFO [2019-12-17 12:29:17,417] jepsen worker 63 - jepsen.util 1863 :invoke :read [21 nil] INFO [2019-12-17 12:29:17,423] jepsen worker 15 - jepsen.util 2535 :info :cas [5 [44 6]] :RequestCanceledException INFO [2019-12-17 12:29:17,424] jepsen worker 15 - jepsen.util 2625 :invoke :read [5 nil] INFO [2019-12-17 12:29:17,440] jepsen worker 30 - jepsen.util 2370 :ok :read [10 16] INFO [2019-12-17 12:29:17,441] jepsen worker 30 - jepsen.util 2370 :invoke :write [10 20] INFO [2019-12-17 12:29:17,448] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11506,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:29:17,449] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11506,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:29:17,449] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11508,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:29:17,449] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11508,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:29:17,452] jepsen worker 63 - jepsen.util 1863 :ok :read [21 43] INFO [2019-12-17 12:29:17,452] jepsen worker 63 - jepsen.util 1863 :invoke :read [21 nil] INFO [2019-12-17 12:29:17,489] jepsen worker 43 - jepsen.util 2653 :info :cas [14 [21 10]] :RequestCanceledException INFO [2019-12-17 12:29:17,490] jepsen worker 43 - jepsen.util 2743 :invoke :read [14 nil] INFO [2019-12-17 12:29:17,507] jepsen worker 16 - jepsen.util 2446 :info :cas [5 [38 3]] :RequestCanceledException INFO [2019-12-17 12:29:17,508] jepsen worker 16 - jepsen.util 2536 :invoke :write [5 43] INFO [2019-12-17 12:29:17,511] jepsen worker 69 - jepsen.util 1959 :ok :read [23 28] INFO [2019-12-17 12:29:17,511] jepsen worker 69 - jepsen.util 1959 :invoke :read [23 nil] INFO [2019-12-17 12:29:17,526] jepsen worker 71 - jepsen.util 1691 :ok :read [23 28] INFO [2019-12-17 12:29:17,526] jepsen worker 71 - jepsen.util 1691 :invoke :write [23 25] INFO [2019-12-17 12:29:17,558] jepsen worker 51 - jepsen.util 1851 :fail :cas [17 [48 1]] :ValueNotSwapFrom INFO [2019-12-17 12:29:17,559] jepsen worker 51 - jepsen.util 1851 :invoke :cas [17 [21 0]] INFO [2019-12-17 12:29:17,566] jepsen worker 88 - jepsen.util 1978 :ok :read [29 35] INFO [2019-12-17 12:29:17,566] jepsen worker 88 - jepsen.util 1978 :invoke :write [29 38] INFO [2019-12-17 12:29:17,586] jepsen worker 3 - jepsen.util 1803 :fail :cas [1 [25 26]] :ValueNotSwapFrom INFO [2019-12-17 12:29:17,589] jepsen worker 3 - jepsen.util 1803 :invoke :write [1 44] INFO [2019-12-17 12:29:17,589] jepsen worker 1 - jepsen.util 2611 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:29:17,590] jepsen worker 1 - jepsen.util 2611 :invoke :read [0 nil] INFO [2019-12-17 12:29:17,606] jepsen worker 22 - jepsen.util 2722 :info :cas [7 [0 24]] :RequestCanceledException INFO [2019-12-17 12:29:17,606] jepsen worker 0 - jepsen.util 2880 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:29:17,607] jepsen worker 0 - jepsen.util 2880 :invoke :write [0 1] INFO [2019-12-17 12:29:17,608] jepsen worker 22 - jepsen.util 2812 :invoke :write [7 49] INFO [2019-12-17 12:29:17,616] jepsen worker 10 - jepsen.util 2080 :ok :write [3 29] INFO [2019-12-17 12:29:17,616] jepsen worker 45 - jepsen.util 2205 :ok :write [15 49] INFO [2019-12-17 12:29:17,616] jepsen worker 45 - jepsen.util 2205 :invoke :read [15 nil] INFO [2019-12-17 12:29:17,616] jepsen worker 10 - jepsen.util 2080 :invoke :write [3 48] INFO [2019-12-17 12:29:17,621] jepsen worker 50 - jepsen.util 2840 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:29:17,621] jepsen worker 59 - jepsen.util 2309 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:29:17,621] jepsen worker 21 - jepsen.util 2541 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:29:17,621] jepsen worker 50 - jepsen.util 2840 :invoke :read [16 nil] INFO [2019-12-17 12:29:17,621] jepsen worker 59 - jepsen.util 2309 :invoke :cas [19 [45 44]] INFO [2019-12-17 12:29:17,621] jepsen worker 21 - jepsen.util 2541 :invoke :cas [7 [15 25]] INFO [2019-12-17 12:29:17,632] jepsen worker 31 - jepsen.util 2011 :ok :write [10 27] INFO [2019-12-17 12:29:17,633] jepsen worker 31 - jepsen.util 2011 :invoke :write [10 49] INFO [2019-12-17 12:29:17,634] jepsen worker 72 - jepsen.util 2142 :ok :write [24 3] INFO [2019-12-17 12:29:17,634] jepsen worker 72 - jepsen.util 2142 :invoke :write [24 36] INFO [2019-12-17 12:29:17,653] jepsen worker 35 - jepsen.util 2195 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:29:17,653] jepsen worker 58 - jepsen.util 2488 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:29:17,653] jepsen worker 76 - jepsen.util 2596 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:29:17,653] jepsen worker 35 - jepsen.util 2195 :invoke :read [11 nil] INFO [2019-12-17 12:29:17,653] jepsen worker 58 - jepsen.util 2488 :invoke :read [19 nil] INFO [2019-12-17 12:29:17,653] jepsen worker 76 - jepsen.util 2596 :invoke :write [25 43] INFO [2019-12-17 12:29:17,656] jepsen worker 79 - jepsen.util 2059 :ok :read [26 11] INFO [2019-12-17 12:29:17,656] jepsen worker 79 - jepsen.util 2059 :invoke :cas [26 [5 14]] INFO [2019-12-17 12:29:17,667] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11530,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:29:17,667] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11530,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:29:17,673] jepsen worker 11 - jepsen.util 2441 :fail :cas [3 [12 12]] :ValueNotSwapFrom INFO [2019-12-17 12:29:17,673] jepsen worker 11 - jepsen.util 2441 :invoke :cas [3 [40 44]] INFO [2019-12-17 12:29:17,689] jepsen worker 85 - jepsen.util 2875 :info :cas [28 [22 6]] :RequestCanceledException INFO [2019-12-17 12:29:17,689] jepsen worker 24 - jepsen.util 2454 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:29:17,690] jepsen worker 24 - jepsen.util 2454 :invoke :cas [8 [10 8]] INFO [2019-12-17 12:29:17,691] jepsen worker 85 - jepsen.util 2965 :invoke :write [28 28] INFO [2019-12-17 12:29:17,701] jepsen worker 68 - jepsen.util 2408 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:29:17,701] jepsen worker 81 - jepsen.util 2241 :info :cas [27 [15 27]] :RequestCanceledException INFO [2019-12-17 12:29:17,701] jepsen worker 57 - jepsen.util 2397 :info :cas [19 [19 27]] :RequestCanceledException INFO [2019-12-17 12:29:17,701] jepsen worker 68 - jepsen.util 2408 :invoke :cas [22 [15 3]] INFO [2019-12-17 12:29:17,702] jepsen worker 57 - jepsen.util 2487 :invoke :read [19 nil] INFO [2019-12-17 12:29:17,702] jepsen worker 81 - jepsen.util 2331 :invoke :read [27 nil] INFO [2019-12-17 12:29:17,719] jepsen worker 65 - jepsen.util 1775 :ok :write [21 32] INFO [2019-12-17 12:29:17,720] jepsen worker 65 - jepsen.util 1775 :invoke :cas [21 [28 48]] INFO [2019-12-17 12:29:17,722] jepsen worker 75 - jepsen.util 2865 :info :cas [25 [35 21]] :RequestCanceledException INFO [2019-12-17 12:29:17,722] jepsen worker 40 - jepsen.util 3010 :info :write [13 39] :RequestCanceledException INFO [2019-12-17 12:29:17,722] jepsen worker 26 - jepsen.util 2906 :info :cas [8 [18 30]] :RequestCanceledException INFO [2019-12-17 12:29:17,722] jepsen worker 62 - jepsen.util 2402 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:29:17,723] jepsen worker 62 - jepsen.util 2402 :invoke :cas [20 [23 10]] INFO [2019-12-17 12:29:17,724] jepsen worker 26 - jepsen.util 2996 :invoke :cas [8 [4 37]] INFO [2019-12-17 12:29:17,724] jepsen worker 40 - jepsen.util 3100 :invoke :read [13 nil] INFO [2019-12-17 12:29:17,724] jepsen worker 75 - jepsen.util 2955 :invoke :read [25 nil] INFO [2019-12-17 12:29:17,728] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:29:17,728] jepsen nemesis - couchbase.util Rebalance check count: 82 INFO [2019-12-17 12:29:17,728] jepsen nemesis - couchbase.util Rebalance stuck count: 63 INFO [2019-12-17 12:29:17,735] jepsen worker 25 - jepsen.util 2995 :info :cas [8 [20 4]] :RequestCanceledException INFO [2019-12-17 12:29:17,735] jepsen worker 86 - jepsen.util 2696 :info :write [28 10] :RequestCanceledException INFO [2019-12-17 12:29:17,736] jepsen worker 25 - jepsen.util 3085 :invoke :read [8 nil] INFO [2019-12-17 12:29:17,736] jepsen worker 86 - jepsen.util 2786 :invoke :cas [28 [23 0]] INFO [2019-12-17 12:29:17,750] jepsen worker 84 - jepsen.util 2424 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:29:17,750] jepsen worker 84 - jepsen.util 2424 :invoke :read [28 nil] INFO [2019-12-17 12:29:17,754] jepsen worker 12 - jepsen.util 2082 :ok :write [4 39] INFO [2019-12-17 12:29:17,755] jepsen worker 12 - jepsen.util 2082 :invoke :read [4 nil] INFO [2019-12-17 12:29:17,775] jepsen worker 69 - jepsen.util 1959 :ok :read [23 28] INFO [2019-12-17 12:29:17,776] jepsen worker 69 - jepsen.util 1959 :invoke :read [23 nil] INFO [2019-12-17 12:29:17,789] jepsen worker 17 - jepsen.util 2807 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:29:17,789] jepsen worker 17 - jepsen.util 2807 :invoke :write [5 4] INFO [2019-12-17 12:29:17,801] jepsen worker 69 - jepsen.util 1959 :ok :read [23 28] INFO [2019-12-17 12:29:17,801] jepsen worker 69 - jepsen.util 1959 :invoke :cas [23 [32 49]] INFO [2019-12-17 12:29:17,806] jepsen worker 8 - jepsen.util 2348 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:29:17,806] jepsen worker 83 - jepsen.util 2873 :info :cas [27 [3 37]] :RequestCanceledException INFO [2019-12-17 12:29:17,807] jepsen worker 8 - jepsen.util 2348 :invoke :write [2 19] INFO [2019-12-17 12:29:17,808] jepsen worker 83 - jepsen.util 2963 :invoke :cas [27 [36 47]] INFO [2019-12-17 12:29:17,822] jepsen worker 39 - jepsen.util 2829 :info :cas [13 [17 10]] :RequestCanceledException INFO [2019-12-17 12:29:17,823] jepsen worker 39 - jepsen.util 2919 :invoke :write [13 17] INFO [2019-12-17 12:29:17,836] jepsen worker 41 - jepsen.util 2651 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:29:17,836] jepsen worker 41 - jepsen.util 2651 :invoke :write [13 39] INFO [2019-12-17 12:29:17,837] jepsen worker 5 - jepsen.util 1715 :ok :write [1 39] INFO [2019-12-17 12:29:17,837] jepsen worker 5 - jepsen.util 1715 :invoke :write [1 25] INFO [2019-12-17 12:29:17,840] jepsen worker 38 - jepsen.util 2108 :ok :write [12 36] INFO [2019-12-17 12:29:17,840] jepsen worker 38 - jepsen.util 2108 :invoke :read [12 nil] INFO [2019-12-17 12:29:17,840] jepsen worker 78 - jepsen.util 2058 :ok :write [26 9] INFO [2019-12-17 12:29:17,840] jepsen worker 78 - jepsen.util 2058 :invoke :cas [26 [35 24]] INFO [2019-12-17 12:29:17,853] jepsen worker 52 - jepsen.util 2032 :ok :write [17 26] INFO [2019-12-17 12:29:17,853] jepsen worker 52 - jepsen.util 2032 :invoke :read [17 nil] INFO [2019-12-17 12:29:17,854] jepsen worker 33 - jepsen.util 2913 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:29:17,854] jepsen worker 42 - jepsen.util 3102 :info :cas [14 [28 49]] :RequestCanceledException INFO [2019-12-17 12:29:17,854] jepsen worker 77 - jepsen.util 2687 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:29:17,854] jepsen worker 33 - jepsen.util 2913 :invoke :read [11 nil] INFO [2019-12-17 12:29:17,854] jepsen worker 77 - jepsen.util 2687 :invoke :read [25 nil] INFO [2019-12-17 12:29:17,855] jepsen worker 42 - jepsen.util 3192 :invoke :cas [14 [6 18]] INFO [2019-12-17 12:29:17,886] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11569,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:29:17,886] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11569,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:29:17,886] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11573,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:29:17,886] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11573,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:29:17,895] jepsen worker 70 - jepsen.util 2320 :fail :cas [23 [43 14]] :ValueNotSwapFrom INFO [2019-12-17 12:29:17,895] jepsen worker 70 - jepsen.util 2320 :invoke :write [23 6] INFO [2019-12-17 12:29:17,898] jepsen worker 87 - jepsen.util 1707 :fail :cas [29 [26 24]] :ValueNotSwapFrom INFO [2019-12-17 12:29:17,899] jepsen worker 87 - jepsen.util 1707 :invoke :cas [29 [47 18]] INFO [2019-12-17 12:29:17,911] jepsen worker 54 - jepsen.util 1944 :fail :cas [18 [12 5]] :ValueNotSwapFrom INFO [2019-12-17 12:29:17,911] jepsen worker 54 - jepsen.util 1944 :invoke :write [18 10] INFO [2019-12-17 12:29:17,916] jepsen worker 18 - jepsen.util 2178 :ok :write [6 32] INFO [2019-12-17 12:29:17,917] jepsen worker 18 - jepsen.util 2178 :invoke :read [6 nil] INFO [2019-12-17 12:29:17,917] jepsen worker 10 - jepsen.util 2080 :ok :write [3 48] INFO [2019-12-17 12:29:17,918] jepsen worker 10 - jepsen.util 2080 :invoke :read [3 nil] INFO [2019-12-17 12:29:17,918] jepsen worker 72 - jepsen.util 2142 :ok :write [24 36] INFO [2019-12-17 12:29:17,918] jepsen worker 72 - jepsen.util 2142 :invoke :write [24 8] INFO [2019-12-17 12:29:17,922] jepsen worker 48 - jepsen.util 2118 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:29:17,923] jepsen worker 48 - jepsen.util 2118 :invoke :write [16 17] INFO [2019-12-17 12:29:17,923] jepsen worker 82 - jepsen.util 3052 :info :write [27 44] :RequestCanceledException INFO [2019-12-17 12:29:17,924] jepsen worker 82 - jepsen.util 3142 :invoke :read [27 nil] INFO [2019-12-17 12:29:17,955] jepsen worker 10 - jepsen.util 2080 :ok :read [3 48] INFO [2019-12-17 12:29:17,955] jepsen worker 10 - jepsen.util 2080 :invoke :cas [3 [30 33]] INFO [2019-12-17 12:29:17,967] jepsen worker 10 - jepsen.util 2080 :fail :cas [3 [30 33]] :ValueNotSwapFrom INFO [2019-12-17 12:29:17,967] jepsen worker 10 - jepsen.util 2080 :invoke :cas [3 [14 27]] INFO [2019-12-17 12:29:17,976] jepsen worker 37 - jepsen.util 1657 :ok :write [12 37] INFO [2019-12-17 12:29:17,976] jepsen worker 37 - jepsen.util 1657 :invoke :write [12 4] INFO [2019-12-17 12:29:17,983] jepsen worker 10 - jepsen.util 2080 :fail :cas [3 [14 27]] :ValueNotSwapFrom INFO [2019-12-17 12:29:17,983] jepsen worker 10 - jepsen.util 2080 :invoke :cas [3 [5 47]] INFO [2019-12-17 12:29:17,990] jepsen worker 66 - jepsen.util 2496 :info :write [22 33] :RequestCanceledException INFO [2019-12-17 12:29:17,991] jepsen worker 66 - jepsen.util 2586 :invoke :write [22 5] INFO [2019-12-17 12:29:17,996] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11589,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:29:17,996] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11589,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:29:17,996] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11591,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:29:17,996] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11591,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:29:17,999] jepsen worker 10 - jepsen.util 2080 :fail :cas [3 [5 47]] :ValueNotSwapFrom INFO [2019-12-17 12:29:17,999] jepsen worker 10 - jepsen.util 2080 :invoke :write [3 21] INFO [2019-12-17 12:29:17,999] jepsen worker 4 - jepsen.util 1534 :fail :cas [1 [20 12]] :ValueNotSwapFrom INFO [2019-12-17 12:29:17,999] jepsen worker 4 - jepsen.util 1534 :invoke :cas [1 [22 20]] INFO [2019-12-17 12:29:18,021] jepsen worker 31 - jepsen.util 2011 :ok :write [10 49] INFO [2019-12-17 12:29:18,022] jepsen worker 31 - jepsen.util 2011 :invoke :cas [10 [12 47]] INFO [2019-12-17 12:29:18,022] jepsen worker 61 - jepsen.util 2671 :info :write [20 39] :RequestCanceledException INFO [2019-12-17 12:29:18,023] jepsen worker 61 - jepsen.util 2761 :invoke :write [20 8] INFO [2019-12-17 12:29:18,034] jepsen worker 72 - jepsen.util 2142 :ok :write [24 8] INFO [2019-12-17 12:29:18,034] jepsen worker 72 - jepsen.util 2142 :invoke :cas [24 [42 16]] INFO [2019-12-17 12:29:18,040] jepsen worker 31 - jepsen.util 2011 :fail :cas [10 [12 47]] :ValueNotSwapFrom INFO [2019-12-17 12:29:18,040] jepsen worker 31 - jepsen.util 2011 :invoke :write [10 2] INFO [2019-12-17 12:29:18,052] jepsen worker 46 - jepsen.util 2116 :ok :write [15 36] INFO [2019-12-17 12:29:18,052] jepsen worker 46 - jepsen.util 2116 :invoke :write [15 31] INFO [2019-12-17 12:29:18,059] jepsen worker 89 - jepsen.util 1979 :fail :cas [29 [10 48]] :ValueNotSwapFrom INFO [2019-12-17 12:29:18,059] jepsen worker 89 - jepsen.util 1979 :invoke :write [29 44] INFO [2019-12-17 12:29:18,063] jepsen worker 72 - jepsen.util 2142 :fail :cas [24 [42 16]] :ValueNotSwapFrom INFO [2019-12-17 12:29:18,063] jepsen worker 72 - jepsen.util 2142 :invoke :read [24 nil] INFO [2019-12-17 12:29:18,092] jepsen worker 72 - jepsen.util 2142 :ok :read [24 8] INFO [2019-12-17 12:29:18,093] jepsen worker 72 - jepsen.util 2142 :invoke :cas [24 [2 8]] INFO [2019-12-17 12:29:18,096] jepsen worker 69 - jepsen.util 1959 :fail :cas [23 [32 49]] :ValueNotSwapFrom INFO [2019-12-17 12:29:18,096] jepsen worker 69 - jepsen.util 1959 :invoke :cas [23 [29 30]] INFO [2019-12-17 12:29:18,100] jepsen worker 87 - jepsen.util 1707 :fail :cas [29 [47 18]] :ValueNotSwapFrom INFO [2019-12-17 12:29:18,100] jepsen worker 87 - jepsen.util 1707 :invoke :write [29 45] INFO [2019-12-17 12:29:18,121] jepsen worker 72 - jepsen.util 2142 :fail :cas [24 [2 8]] :ValueNotSwapFrom INFO [2019-12-17 12:29:18,121] jepsen worker 72 - jepsen.util 2142 :invoke :cas [24 [21 19]] INFO [2019-12-17 12:29:18,134] jepsen worker 72 - jepsen.util 2142 :fail :cas [24 [21 19]] :ValueNotSwapFrom INFO [2019-12-17 12:29:18,134] jepsen worker 72 - jepsen.util 2142 :invoke :read [24 nil] INFO [2019-12-17 12:29:18,137] jepsen worker 23 - jepsen.util 2453 :info :write [7 48] :RequestCanceledException INFO [2019-12-17 12:29:18,137] jepsen worker 2 - jepsen.util 2432 :info :cas [0 [40 27]] :RequestCanceledException INFO [2019-12-17 12:29:18,138] jepsen worker 23 - jepsen.util 2543 :invoke :write [7 47] INFO [2019-12-17 12:29:18,138] jepsen worker 2 - jepsen.util 2522 :invoke :write [0 23] INFO [2019-12-17 12:29:18,148] jepsen worker 72 - jepsen.util 2142 :ok :read [24 8] INFO [2019-12-17 12:29:18,148] jepsen worker 72 - jepsen.util 2142 :invoke :cas [24 [31 23]] INFO [2019-12-17 12:29:18,156] jepsen worker 56 - jepsen.util 1946 :ok :write [18 23] INFO [2019-12-17 12:29:18,156] jepsen worker 14 - jepsen.util 1994 :fail :cas [4 [24 2]] :ValueNotSwapFrom INFO [2019-12-17 12:29:18,156] jepsen worker 56 - jepsen.util 1946 :invoke :cas [18 [34 6]] INFO [2019-12-17 12:29:18,156] jepsen worker 14 - jepsen.util 1994 :invoke :write [4 34] INFO [2019-12-17 12:29:18,162] jepsen worker 78 - jepsen.util 2058 :fail :cas [26 [35 24]] :ValueNotSwapFrom INFO [2019-12-17 12:29:18,162] jepsen worker 78 - jepsen.util 2058 :invoke :cas [26 [28 36]] INFO [2019-12-17 12:29:18,164] jepsen worker 72 - jepsen.util 2142 :fail :cas [24 [31 23]] :ValueNotSwapFrom INFO [2019-12-17 12:29:18,164] jepsen worker 72 - jepsen.util 2142 :invoke :write [24 18] INFO [2019-12-17 12:29:18,185] jepsen worker 80 - jepsen.util 1970 :ok :read [26 9] INFO [2019-12-17 12:29:18,185] jepsen worker 80 - jepsen.util 1970 :invoke :read [26 nil] INFO [2019-12-17 12:29:18,191] jepsen worker 36 - jepsen.util 1836 :ok :write [12 34] INFO [2019-12-17 12:29:18,191] jepsen worker 36 - jepsen.util 1836 :invoke :write [12 10] INFO [2019-12-17 12:29:18,201] jepsen worker 12 - jepsen.util 2082 :ok :read [4 39] INFO [2019-12-17 12:29:18,201] jepsen worker 12 - jepsen.util 2082 :invoke :cas [4 [15 10]] INFO [2019-12-17 12:29:18,211] jepsen worker 4 - jepsen.util 1534 :fail :cas [1 [22 20]] :ValueNotSwapFrom INFO [2019-12-17 12:29:18,211] jepsen worker 4 - jepsen.util 1534 :invoke :read [1 nil] INFO [2019-12-17 12:29:18,212] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11615,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:29:18,212] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11615,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:29:18,213] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11616,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:29:18,213] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11616,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:29:18,213] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11617,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:29:18,213] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11617,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:29:18,214] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11624,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:29:18,215] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11624,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:29:18,215] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11628,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:29:18,215] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11628,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:29:18,218] jepsen worker 44 - jepsen.util 2294 :info :cas [14 [20 44]] :RequestCanceledException INFO [2019-12-17 12:29:18,219] jepsen worker 44 - jepsen.util 2384 :invoke :cas [14 [43 19]] INFO [2019-12-17 12:29:18,245] jepsen worker 56 - jepsen.util 1946 :fail :cas [18 [34 6]] :ValueNotSwapFrom INFO [2019-12-17 12:29:18,253] jepsen worker 49 - jepsen.util 2029 :info :cas [16 [19 19]] :RequestCanceledException INFO [2019-12-17 12:29:18,267] jepsen worker 80 - jepsen.util 1970 :ok :read [26 9] INFO [2019-12-17 12:29:18,288] jepsen worker 67 - jepsen.util 3037 :info :write [22 44] :RequestCanceledException INFO [2019-12-17 12:29:18,304] jepsen worker 56 - jepsen.util 1946 :invoke :write [18 49] INFO [2019-12-17 12:29:18,304] jepsen worker 80 - jepsen.util 1970 :invoke :cas [26 [30 34]] INFO [2019-12-17 12:29:18,305] jepsen worker 49 - jepsen.util 2119 :invoke :read [16 nil] INFO [2019-12-17 12:29:18,305] jepsen worker 67 - jepsen.util 3127 :invoke :cas [22 [9 4]] INFO [2019-12-17 12:29:18,310] jepsen worker 13 - jepsen.util 1813 :ok :write [4 39] INFO [2019-12-17 12:29:18,310] jepsen worker 13 - jepsen.util 1813 :invoke :write [4 41] INFO [2019-12-17 12:29:18,327] jepsen worker 12 - jepsen.util 2082 :fail :cas [4 [15 10]] :ValueNotSwapFrom INFO [2019-12-17 12:29:18,327] jepsen worker 12 - jepsen.util 2082 :invoke :read [4 nil] INFO [2019-12-17 12:29:18,389] jepsen worker 34 - jepsen.util 2734 :info :write [11 6] :RequestCanceledException INFO [2019-12-17 12:29:18,390] jepsen worker 34 - jepsen.util 2824 :invoke :read [11 nil] INFO [2019-12-17 12:29:18,419] jepsen worker 71 - jepsen.util 1691 :ok :write [23 25] INFO [2019-12-17 12:29:18,419] jepsen worker 71 - jepsen.util 1691 :invoke :read [23 nil] INFO [2019-12-17 12:29:18,433] jepsen worker 80 - jepsen.util 1970 :fail :cas [26 [30 34]] :ValueNotSwapFrom INFO [2019-12-17 12:29:18,433] jepsen worker 80 - jepsen.util 1970 :invoke :read [26 nil] INFO [2019-12-17 12:29:18,434] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11639,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:29:18,434] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11639,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:29:18,434] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11640,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:29:18,434] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11640,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:29:18,434] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11642,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:29:18,434] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11642,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:29:18,436] jepsen worker 10 - jepsen.util 2080 :ok :write [3 21] INFO [2019-12-17 12:29:18,437] jepsen worker 10 - jepsen.util 2080 :invoke :read [3 nil] INFO [2019-12-17 12:29:18,437] jepsen worker 46 - jepsen.util 2116 :ok :write [15 31] INFO [2019-12-17 12:29:18,437] jepsen worker 46 - jepsen.util 2116 :invoke :cas [15 [9 17]] INFO [2019-12-17 12:29:18,438] jepsen worker 31 - jepsen.util 2011 :ok :write [10 2] INFO [2019-12-17 12:29:18,438] jepsen worker 31 - jepsen.util 2011 :invoke :cas [10 [4 35]] INFO [2019-12-17 12:29:18,466] jepsen worker 10 - jepsen.util 2080 :ok :read [3 21] INFO [2019-12-17 12:29:18,467] jepsen worker 10 - jepsen.util 2080 :invoke :write [3 11] INFO [2019-12-17 12:29:18,487] jepsen worker 12 - jepsen.util 2082 :ok :read [4 39] INFO [2019-12-17 12:29:18,487] jepsen worker 12 - jepsen.util 2082 :invoke :cas [4 [43 8]] INFO [2019-12-17 12:29:18,490] jepsen worker 46 - jepsen.util 2116 :fail :cas [15 [9 17]] :ValueNotSwapFrom INFO [2019-12-17 12:29:18,491] jepsen worker 46 - jepsen.util 2116 :invoke :write [15 42] INFO [2019-12-17 12:29:18,499] jepsen worker 31 - jepsen.util 2011 :fail :cas [10 [4 35]] :ValueNotSwapFrom INFO [2019-12-17 12:29:18,500] jepsen worker 31 - jepsen.util 2011 :invoke :read [10 nil] INFO [2019-12-17 12:29:18,519] jepsen worker 31 - jepsen.util 2011 :ok :read [10 2] INFO [2019-12-17 12:29:18,519] jepsen worker 31 - jepsen.util 2011 :invoke :read [10 nil] INFO [2019-12-17 12:29:18,527] jepsen worker 52 - jepsen.util 2032 :ok :read [17 26] INFO [2019-12-17 12:29:18,527] jepsen worker 52 - jepsen.util 2032 :invoke :write [17 18] INFO [2019-12-17 12:29:18,536] jepsen worker 31 - jepsen.util 2011 :ok :read [10 2] INFO [2019-12-17 12:29:18,536] jepsen worker 31 - jepsen.util 2011 :invoke :cas [10 [4 43]] INFO [2019-12-17 12:29:18,556] jepsen worker 31 - jepsen.util 2011 :fail :cas [10 [4 43]] :ValueNotSwapFrom INFO [2019-12-17 12:29:18,557] jepsen worker 31 - jepsen.util 2011 :invoke :write [10 14] INFO [2019-12-17 12:29:18,567] jepsen worker 69 - jepsen.util 1959 :fail :cas [23 [29 30]] :ValueNotSwapFrom INFO [2019-12-17 12:29:18,568] jepsen worker 69 - jepsen.util 1959 :invoke :read [23 nil] INFO [2019-12-17 12:29:18,572] jepsen worker 80 - jepsen.util 1970 :ok :read [26 9] INFO [2019-12-17 12:29:18,572] jepsen worker 80 - jepsen.util 1970 :invoke :cas [26 [42 7]] INFO [2019-12-17 12:29:18,578] jepsen worker 78 - jepsen.util 2058 :fail :cas [26 [28 36]] :ValueNotSwapFrom INFO [2019-12-17 12:29:18,579] jepsen worker 78 - jepsen.util 2058 :invoke :cas [26 [29 38]] INFO [2019-12-17 12:29:18,588] jepsen worker 12 - jepsen.util 2082 :fail :cas [4 [43 8]] :ValueNotSwapFrom INFO [2019-12-17 12:29:18,588] jepsen worker 12 - jepsen.util 2082 :invoke :read [4 nil] INFO [2019-12-17 12:29:18,595] jepsen worker 4 - jepsen.util 1534 :ok :read [1 39] INFO [2019-12-17 12:29:18,597] jepsen worker 4 - jepsen.util 1534 :invoke :write [1 28] INFO [2019-12-17 12:29:18,653] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11663,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:29:18,653] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11663,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:29:18,653] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11667,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:29:18,653] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11667,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:29:18,658] jepsen worker 78 - jepsen.util 2058 :fail :cas [26 [29 38]] :ValueNotSwapFrom INFO [2019-12-17 12:29:18,658] jepsen worker 78 - jepsen.util 2058 :invoke :write [26 2] INFO [2019-12-17 12:29:18,691] jepsen worker 80 - jepsen.util 1970 :fail :cas [26 [42 7]] :ValueNotSwapFrom INFO [2019-12-17 12:29:18,691] jepsen worker 80 - jepsen.util 1970 :invoke :cas [26 [25 36]] INFO [2019-12-17 12:29:18,709] jepsen worker 55 - jepsen.util 2305 :ok :read [18 23] INFO [2019-12-17 12:29:18,709] jepsen worker 55 - jepsen.util 2305 :invoke :write [18 27] INFO [2019-12-17 12:29:18,722] jepsen worker 60 - jepsen.util 2760 :info :cas [20 [35 28]] :RequestCanceledException INFO [2019-12-17 12:29:18,723] jepsen worker 60 - jepsen.util 2850 :invoke :cas [20 [41 45]] INFO [2019-12-17 12:29:18,729] jepsen worker 12 - jepsen.util 2082 :ok :read [4 39] INFO [2019-12-17 12:29:18,729] jepsen worker 12 - jepsen.util 2082 :invoke :read [4 nil] INFO [2019-12-17 12:29:18,741] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:29:18,741] jepsen nemesis - couchbase.util Rebalance check count: 83 INFO [2019-12-17 12:29:18,741] jepsen nemesis - couchbase.util Rebalance stuck count: 64 INFO [2019-12-17 12:29:18,759] jepsen worker 38 - jepsen.util 2108 :ok :read [12 34] INFO [2019-12-17 12:29:18,759] jepsen worker 38 - jepsen.util 2108 :invoke :cas [12 [49 20]] INFO [2019-12-17 12:29:18,772] jepsen worker 80 - jepsen.util 1970 :fail :cas [26 [25 36]] :ValueNotSwapFrom INFO [2019-12-17 12:29:18,772] jepsen worker 80 - jepsen.util 1970 :invoke :read [26 nil] INFO [2019-12-17 12:29:18,773] jepsen worker 51 - jepsen.util 1851 :fail :cas [17 [21 0]] :ValueNotSwapFrom INFO [2019-12-17 12:29:18,773] jepsen worker 51 - jepsen.util 1851 :invoke :read [17 nil] INFO [2019-12-17 12:29:18,789] jepsen worker 47 - jepsen.util 2477 :fail :read [15 nil] :RequestTimeoutException INFO [2019-12-17 12:29:18,789] jepsen worker 6 - jepsen.util 2706 :info :write [2 20] :RequestCanceledException INFO [2019-12-17 12:29:18,789] jepsen worker 47 - jepsen.util 2477 :invoke :read [15 nil] INFO [2019-12-17 12:29:18,790] jepsen worker 6 - jepsen.util 2796 :invoke :read [2 nil] INFO [2019-12-17 12:29:18,828] jepsen worker 80 - jepsen.util 1970 :ok :read [26 9] INFO [2019-12-17 12:29:18,828] jepsen worker 80 - jepsen.util 1970 :invoke :cas [26 [34 12]] INFO [2019-12-17 12:29:18,870] jepsen worker 79 - jepsen.util 2059 :fail :cas [26 [5 14]] :ValueNotSwapFrom INFO [2019-12-17 12:29:18,870] jepsen worker 79 - jepsen.util 2059 :invoke :cas [26 [29 40]] INFO [2019-12-17 12:29:18,872] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11682,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:29:18,872] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11682,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:29:18,872] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11684,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:29:18,872] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11684,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:29:18,907] jepsen worker 7 - jepsen.util 2617 :info :cas [2 [46 20]] :RequestCanceledException INFO [2019-12-17 12:29:18,907] jepsen worker 7 - jepsen.util 2707 :invoke :read [2 nil] INFO [2019-12-17 12:29:18,942] jepsen worker 69 - jepsen.util 1959 :ok :read [23 25] INFO [2019-12-17 12:29:18,942] jepsen worker 69 - jepsen.util 1959 :invoke :read [23 nil] INFO [2019-12-17 12:29:18,983] jepsen worker 69 - jepsen.util 1959 :ok :read [23 25] INFO [2019-12-17 12:29:18,983] jepsen worker 69 - jepsen.util 1959 :invoke :write [23 36] INFO [2019-12-17 12:29:18,983] jepsen worker 19 - jepsen.util 2269 :fail :read [6 nil] :RequestTimeoutException INFO [2019-12-17 12:29:18,983] jepsen worker 19 - jepsen.util 2269 :invoke :cas [6 [7 43]] INFO [2019-12-17 12:29:18,991] jepsen worker 71 - jepsen.util 1691 :ok :read [23 25] INFO [2019-12-17 12:29:18,992] jepsen worker 71 - jepsen.util 1691 :invoke :write [23 42] INFO [2019-12-17 12:29:19,001] jepsen worker 12 - jepsen.util 2082 :ok :read [4 39] INFO [2019-12-17 12:29:19,002] jepsen worker 12 - jepsen.util 2082 :invoke :write [4 5] INFO [2019-12-17 12:29:19,022] jepsen worker 74 - jepsen.util 1964 :fail :read [24 nil] :RequestTimeoutException INFO [2019-12-17 12:29:19,022] jepsen worker 74 - jepsen.util 1964 :invoke :read [24 nil] INFO [2019-12-17 12:29:19,063] jepsen worker 80 - jepsen.util 1970 :fail :cas [26 [34 12]] :ValueNotSwapFrom INFO [2019-12-17 12:29:19,063] jepsen worker 80 - jepsen.util 1970 :invoke :read [26 nil] INFO [2019-12-17 12:29:19,064] jepsen worker 51 - jepsen.util 1851 :ok :read [17 26] INFO [2019-12-17 12:29:19,064] jepsen worker 51 - jepsen.util 1851 :invoke :cas [17 [13 14]] INFO [2019-12-17 12:29:19,089] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11689,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:29:19,089] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11689,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:29:19,089] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11693,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:29:19,089] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11693,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:29:19,122] jepsen worker 73 - jepsen.util 1963 :info :write [24 11] :RequestCanceledException INFO [2019-12-17 12:29:19,123] jepsen worker 73 - jepsen.util 2053 :invoke :write [24 48] INFO [2019-12-17 12:29:19,181] jepsen worker 10 - jepsen.util 2080 :ok :write [3 11] INFO [2019-12-17 12:29:19,181] jepsen worker 10 - jepsen.util 2080 :invoke :read [3 nil] INFO [2019-12-17 12:29:19,181] jepsen worker 46 - jepsen.util 2116 :ok :write [15 42] INFO [2019-12-17 12:29:19,182] jepsen worker 46 - jepsen.util 2116 :invoke :cas [15 [46 10]] INFO [2019-12-17 12:29:19,190] jepsen worker 72 - jepsen.util 2142 :ok :write [24 18] INFO [2019-12-17 12:29:19,190] jepsen worker 72 - jepsen.util 2142 :invoke :write [24 18] INFO [2019-12-17 12:29:19,193] jepsen worker 31 - jepsen.util 2011 :ok :write [10 14] INFO [2019-12-17 12:29:19,193] jepsen worker 31 - jepsen.util 2011 :invoke :cas [10 [36 39]] INFO [2019-12-17 12:29:19,196] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11704,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:29:19,196] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11704,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:29:19,196] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11705,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:29:19,196] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11705,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:29:19,218] jepsen worker 74 - jepsen.util 1964 :ok :read [24 8] INFO [2019-12-17 12:29:19,218] jepsen worker 74 - jepsen.util 1964 :invoke :cas [24 [48 47]] INFO [2019-12-17 12:29:19,223] jepsen worker 32 - jepsen.util 2462 :info :cas [10 [16 47]] :RequestCanceledException INFO [2019-12-17 12:29:19,223] jepsen worker 20 - jepsen.util 1640 :fail :read [6 nil] :RequestTimeoutException INFO [2019-12-17 12:29:19,223] jepsen worker 20 - jepsen.util 1640 :invoke :read [6 nil] INFO [2019-12-17 12:29:19,224] jepsen worker 32 - jepsen.util 2552 :invoke :read [10 nil] INFO [2019-12-17 12:29:19,237] jepsen worker 80 - jepsen.util 1970 :ok :read [26 9] INFO [2019-12-17 12:29:19,238] jepsen worker 80 - jepsen.util 1970 :invoke :write [26 10] INFO [2019-12-17 12:29:19,246] jepsen worker 38 - jepsen.util 2108 :fail :cas [12 [49 20]] :ValueNotSwapFrom INFO [2019-12-17 12:29:19,246] jepsen worker 38 - jepsen.util 2108 :invoke :read [12 nil] INFO [2019-12-17 12:29:19,279] jepsen worker 10 - jepsen.util 2080 :ok :read [3 11] INFO [2019-12-17 12:29:19,279] jepsen worker 10 - jepsen.util 2080 :invoke :write [3 47] INFO [2019-12-17 12:29:19,305] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11714,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:29:19,306] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11714,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:29:19,314] jepsen worker 46 - jepsen.util 2116 :fail :cas [15 [46 10]] :ValueNotSwapFrom INFO [2019-12-17 12:29:19,314] jepsen worker 46 - jepsen.util 2116 :invoke :write [15 40] INFO [2019-12-17 12:29:19,327] jepsen worker 47 - jepsen.util 2477 :ok :read [15 42] INFO [2019-12-17 12:29:19,328] jepsen worker 47 - jepsen.util 2477 :invoke :read [15 nil] INFO [2019-12-17 12:29:19,354] jepsen worker 29 - jepsen.util 1739 :info :cas [9 [42 11]] :RequestCanceledException INFO [2019-12-17 12:29:19,354] jepsen worker 64 - jepsen.util 1684 :fail :read [21 nil] :RequestTimeoutException INFO [2019-12-17 12:29:19,354] jepsen worker 64 - jepsen.util 1684 :invoke :cas [21 [36 36]] INFO [2019-12-17 12:29:19,355] jepsen worker 29 - jepsen.util 1829 :invoke :cas [9 [48 31]] INFO [2019-12-17 12:29:19,392] jepsen worker 32 - jepsen.util 2552 :ok :read [10 14] INFO [2019-12-17 12:29:19,393] jepsen worker 32 - jepsen.util 2552 :invoke :read [10 nil] INFO [2019-12-17 12:29:19,400] jepsen worker 20 - jepsen.util 1640 :ok :read [6 32] INFO [2019-12-17 12:29:19,400] jepsen worker 20 - jepsen.util 1640 :invoke :cas [6 [14 46]] INFO [2019-12-17 12:29:19,406] jepsen worker 19 - jepsen.util 2269 :fail :cas [6 [7 43]] :ValueNotSwapFrom INFO [2019-12-17 12:29:19,406] jepsen worker 19 - jepsen.util 2269 :invoke :write [6 25] INFO [2019-12-17 12:29:19,407] jepsen worker 28 - jepsen.util 1288 :info :cas [9 [29 45]] :RequestCanceledException INFO [2019-12-17 12:29:19,407] jepsen worker 27 - jepsen.util 1827 :info :write [9 40] :RequestCanceledException INFO [2019-12-17 12:29:19,409] jepsen worker 27 - jepsen.util 1917 :invoke :write [9 21] INFO [2019-12-17 12:29:19,410] jepsen worker 28 - jepsen.util 1378 :invoke :write [9 1] INFO [2019-12-17 12:29:19,415] jepsen worker 74 - jepsen.util 1964 :fail :cas [24 [48 47]] :ValueNotSwapFrom INFO [2019-12-17 12:29:19,415] jepsen worker 74 - jepsen.util 1964 :invoke :cas [24 [4 42]] INFO [2019-12-17 12:29:19,422] jepsen worker 9 - jepsen.util 2259 :info :write [3 17] :RequestCanceledException INFO [2019-12-17 12:29:19,423] jepsen worker 9 - jepsen.util 2349 :invoke :cas [3 [12 42]] INFO [2019-12-17 12:29:19,434] jepsen worker 53 - jepsen.util 1853 :info :write [17 22] :RequestCanceledException INFO [2019-12-17 12:29:19,436] jepsen worker 53 - jepsen.util 1943 :invoke :cas [17 [44 21]] INFO [2019-12-17 12:29:19,437] jepsen worker 31 - jepsen.util 2011 :fail :cas [10 [36 39]] :ValueNotSwapFrom INFO [2019-12-17 12:29:19,438] jepsen worker 31 - jepsen.util 2011 :invoke :write [10 36] INFO [2019-12-17 12:29:19,456] jepsen worker 20 - jepsen.util 1640 :fail :cas [6 [14 46]] :ValueNotSwapFrom INFO [2019-12-17 12:29:19,456] jepsen worker 20 - jepsen.util 1640 :invoke :cas [6 [2 30]] INFO [2019-12-17 12:29:19,456] jepsen worker 32 - jepsen.util 2552 :ok :read [10 14] INFO [2019-12-17 12:29:19,456] jepsen worker 32 - jepsen.util 2552 :invoke :write [10 29] INFO [2019-12-17 12:29:19,479] jepsen worker 79 - jepsen.util 2059 :fail :cas [26 [29 40]] :ValueNotSwapFrom INFO [2019-12-17 12:29:19,479] jepsen worker 79 - jepsen.util 2059 :invoke :write [26 45] INFO [2019-12-17 12:29:19,479] jepsen worker 47 - jepsen.util 2477 :ok :read [15 42] INFO [2019-12-17 12:29:19,479] jepsen worker 47 - jepsen.util 2477 :invoke :cas [15 [23 3]] INFO [2019-12-17 12:29:19,499] jepsen worker 20 - jepsen.util 1640 :fail :cas [6 [2 30]] :ValueNotSwapFrom INFO [2019-12-17 12:29:19,499] jepsen worker 20 - jepsen.util 1640 :invoke :cas [6 [43 1]] INFO [2019-12-17 12:29:19,523] jepsen worker 15 - jepsen.util 2625 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:29:19,523] jepsen worker 15 - jepsen.util 2625 :invoke :read [5 nil] INFO [2019-12-17 12:29:19,523] jepsen worker 30 - jepsen.util 2370 :info :write [10 20] :RequestCanceledException INFO [2019-12-17 12:29:19,523] jepsen worker 63 - jepsen.util 1863 :fail :read [21 nil] :RequestTimeoutException INFO [2019-12-17 12:29:19,523] jepsen worker 63 - jepsen.util 1863 :invoke :read [21 nil] INFO [2019-12-17 12:29:19,524] jepsen worker 30 - jepsen.util 2460 :invoke :write [10 45] INFO [2019-12-17 12:29:19,525] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11738,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:29:19,525] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11738,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:29:19,526] jepsen worker 20 - jepsen.util 1640 :fail :cas [6 [43 1]] :ValueNotSwapFrom INFO [2019-12-17 12:29:19,527] jepsen worker 20 - jepsen.util 1640 :invoke :cas [6 [44 35]] INFO [2019-12-17 12:29:19,558] jepsen worker 63 - jepsen.util 1863 :ok :read [21 32] INFO [2019-12-17 12:29:19,559] jepsen worker 63 - jepsen.util 1863 :invoke :read [21 nil] INFO [2019-12-17 12:29:19,562] jepsen worker 20 - jepsen.util 1640 :fail :cas [6 [44 35]] :ValueNotSwapFrom INFO [2019-12-17 12:29:19,562] jepsen worker 20 - jepsen.util 1640 :invoke :cas [6 [10 15]] INFO [2019-12-17 12:29:19,564] jepsen worker 38 - jepsen.util 2108 :ok :read [12 34] INFO [2019-12-17 12:29:19,564] jepsen worker 38 - jepsen.util 2108 :invoke :read [12 nil] INFO [2019-12-17 12:29:19,574] jepsen worker 63 - jepsen.util 1863 :ok :read [21 32] INFO [2019-12-17 12:29:19,574] jepsen worker 63 - jepsen.util 1863 :invoke :read [21 nil] INFO [2019-12-17 12:29:19,589] jepsen worker 43 - jepsen.util 2743 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:29:19,589] jepsen worker 43 - jepsen.util 2743 :invoke :write [14 8] INFO [2019-12-17 12:29:19,592] jepsen worker 20 - jepsen.util 1640 :fail :cas [6 [10 15]] :ValueNotSwapFrom INFO [2019-12-17 12:29:19,592] jepsen worker 20 - jepsen.util 1640 :invoke :cas [6 [47 23]] INFO [2019-12-17 12:29:19,605] jepsen worker 63 - jepsen.util 1863 :ok :read [21 32] INFO [2019-12-17 12:29:19,605] jepsen worker 63 - jepsen.util 1863 :invoke :write [21 17] INFO [2019-12-17 12:29:19,607] jepsen worker 16 - jepsen.util 2536 :info :write [5 43] :RequestCanceledException INFO [2019-12-17 12:29:19,608] jepsen worker 16 - jepsen.util 2626 :invoke :write [5 43] INFO [2019-12-17 12:29:19,620] jepsen worker 20 - jepsen.util 1640 :fail :cas [6 [47 23]] :ValueNotSwapFrom INFO [2019-12-17 12:29:19,620] jepsen worker 20 - jepsen.util 1640 :invoke :read [6 nil] INFO [2019-12-17 12:29:19,622] jepsen worker 3 - jepsen.util 1803 :info :write [1 44] :RequestCanceledException INFO [2019-12-17 12:29:19,622] jepsen worker 88 - jepsen.util 1978 :info :write [29 38] :RequestCanceledException INFO [2019-12-17 12:29:19,623] jepsen worker 3 - jepsen.util 1893 :invoke :write [1 41] INFO [2019-12-17 12:29:19,623] jepsen worker 88 - jepsen.util 2068 :invoke :cas [29 [9 14]] INFO [2019-12-17 12:29:19,642] jepsen worker 10 - jepsen.util 2080 :ok :write [3 47] INFO [2019-12-17 12:29:19,642] jepsen worker 10 - jepsen.util 2080 :invoke :cas [3 [8 3]] INFO [2019-12-17 12:29:19,642] jepsen worker 72 - jepsen.util 2142 :ok :write [24 18] INFO [2019-12-17 12:29:19,642] jepsen worker 72 - jepsen.util 2142 :invoke :write [24 19] INFO [2019-12-17 12:29:19,656] jepsen worker 53 - jepsen.util 1943 :fail :cas [17 [44 21]] :ValueNotSwapFrom INFO [2019-12-17 12:29:19,656] jepsen worker 53 - jepsen.util 1943 :invoke :cas [17 [1 31]] INFO [2019-12-17 12:29:19,689] jepsen worker 1 - jepsen.util 2611 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:29:19,689] jepsen worker 1 - jepsen.util 2611 :invoke :cas [0 [9 33]] INFO [2019-12-17 12:29:19,706] jepsen worker 9 - jepsen.util 2349 :fail :cas [3 [12 42]] :ValueNotSwapFrom INFO [2019-12-17 12:29:19,706] jepsen worker 9 - jepsen.util 2349 :invoke :write [3 16] INFO [2019-12-17 12:29:19,707] jepsen worker 0 - jepsen.util 2880 :info :write [0 1] :RequestCanceledException INFO [2019-12-17 12:29:19,708] jepsen worker 0 - jepsen.util 2970 :invoke :write [0 49] INFO [2019-12-17 12:29:19,719] jepsen worker 64 - jepsen.util 1684 :fail :cas [21 [36 36]] :ValueNotSwapFrom INFO [2019-12-17 12:29:19,720] jepsen worker 64 - jepsen.util 1684 :invoke :cas [21 [28 3]] INFO [2019-12-17 12:29:19,722] jepsen worker 50 - jepsen.util 2840 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:29:19,722] jepsen worker 59 - jepsen.util 2309 :info :cas [19 [45 44]] :RequestCanceledException INFO [2019-12-17 12:29:19,722] jepsen worker 45 - jepsen.util 2205 :fail :read [15 nil] :RequestTimeoutException INFO [2019-12-17 12:29:19,722] jepsen worker 81 - jepsen.util 2331 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:29:19,722] jepsen worker 45 - jepsen.util 2205 :invoke :read [15 nil] INFO [2019-12-17 12:29:19,722] jepsen worker 50 - jepsen.util 2840 :invoke :write [16 14] INFO [2019-12-17 12:29:19,722] jepsen worker 81 - jepsen.util 2331 :invoke :read [27 nil] INFO [2019-12-17 12:29:19,723] jepsen worker 22 - jepsen.util 2812 :info :write [7 49] :RequestCanceledException INFO [2019-12-17 12:29:19,723] jepsen worker 21 - jepsen.util 2541 :info :cas [7 [15 25]] :RequestCanceledException INFO [2019-12-17 12:29:19,723] jepsen worker 59 - jepsen.util 2399 :invoke :write [19 6] INFO [2019-12-17 12:29:19,724] jepsen worker 21 - jepsen.util 2631 :invoke :cas [7 [33 12]] INFO [2019-12-17 12:29:19,724] jepsen worker 22 - jepsen.util 2902 :invoke :read [7 nil] INFO [2019-12-17 12:29:19,736] jepsen worker 85 - jepsen.util 2965 :info :write [28 28] :RequestCanceledException INFO [2019-12-17 12:29:19,737] jepsen worker 85 - jepsen.util 3055 :invoke :cas [28 [3 2]] INFO [2019-12-17 12:29:19,744] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11765,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:29:19,744] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11765,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:29:19,744] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11768,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:29:19,744] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11768,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:29:19,752] jepsen worker 35 - jepsen.util 2195 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:29:19,753] jepsen worker 76 - jepsen.util 2596 :info :write [25 43] :RequestCanceledException INFO [2019-12-17 12:29:19,753] jepsen worker 58 - jepsen.util 2488 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:29:19,753] jepsen worker 11 - jepsen.util 2441 :info :cas [3 [40 44]] :RequestCanceledException INFO [2019-12-17 12:29:19,753] jepsen worker 57 - jepsen.util 2487 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:29:19,753] jepsen worker 86 - jepsen.util 2786 :info :cas [28 [23 0]] :RequestCanceledException INFO [2019-12-17 12:29:19,753] jepsen worker 35 - jepsen.util 2195 :invoke :cas [11 [25 21]] INFO [2019-12-17 12:29:19,753] jepsen worker 58 - jepsen.util 2488 :invoke :read [19 nil] INFO [2019-12-17 12:29:19,753] jepsen worker 57 - jepsen.util 2487 :invoke :read [19 nil] INFO [2019-12-17 12:29:19,754] jepsen worker 76 - jepsen.util 2686 :invoke :read [25 nil] INFO [2019-12-17 12:29:19,754] jepsen worker 11 - jepsen.util 2531 :invoke :write [3 47] INFO [2019-12-17 12:29:19,754] jepsen worker 86 - jepsen.util 2876 :invoke :read [28 nil] INFO [2019-12-17 12:29:19,779] jepsen worker 10 - jepsen.util 2080 :fail :cas [3 [8 3]] :ValueNotSwapFrom INFO [2019-12-17 12:29:19,779] jepsen worker 10 - jepsen.util 2080 :invoke :write [3 16] INFO [2019-12-17 12:29:19,789] jepsen worker 26 - jepsen.util 2996 :info :cas [8 [4 37]] :RequestCanceledException INFO [2019-12-17 12:29:19,789] jepsen worker 65 - jepsen.util 1775 :info :cas [21 [28 48]] :RequestCanceledException INFO [2019-12-17 12:29:19,789] jepsen worker 24 - jepsen.util 2454 :info :cas [8 [10 8]] :RequestCanceledException INFO [2019-12-17 12:29:19,790] jepsen worker 65 - jepsen.util 1865 :invoke :write [21 25] INFO [2019-12-17 12:29:19,790] jepsen worker 24 - jepsen.util 2544 :invoke :write [8 8] INFO [2019-12-17 12:29:19,790] jepsen worker 26 - jepsen.util 3086 :invoke :read [8 nil] INFO [2019-12-17 12:29:19,792] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:29:19,792] jepsen nemesis - couchbase.util Rebalance check count: 84 INFO [2019-12-17 12:29:19,793] jepsen nemesis - couchbase.util Rebalance stuck count: 65 INFO [2019-12-17 12:29:19,797] jepsen worker 29 - jepsen.util 1829 :fail :cas [9 [48 31]] :ValueNotSwapFrom INFO [2019-12-17 12:29:19,797] jepsen worker 29 - jepsen.util 1829 :invoke :read [9 nil] INFO [2019-12-17 12:29:19,806] jepsen worker 63 - jepsen.util 1863 :ok :write [21 17] INFO [2019-12-17 12:29:19,806] jepsen worker 63 - jepsen.util 1863 :invoke :write [21 19] INFO [2019-12-17 12:29:19,806] jepsen worker 40 - jepsen.util 3100 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:29:19,806] jepsen worker 68 - jepsen.util 2408 :info :cas [22 [15 3]] :RequestCanceledException INFO [2019-12-17 12:29:19,807] jepsen worker 40 - jepsen.util 3100 :invoke :read [13 nil] INFO [2019-12-17 12:29:19,808] jepsen worker 68 - jepsen.util 2498 :invoke :read [22 nil] INFO [2019-12-17 12:29:19,822] jepsen worker 62 - jepsen.util 2402 :info :cas [20 [23 10]] :RequestCanceledException INFO [2019-12-17 12:29:19,822] jepsen worker 75 - jepsen.util 2955 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:29:19,822] jepsen worker 83 - jepsen.util 2963 :info :cas [27 [36 47]] :RequestCanceledException INFO [2019-12-17 12:29:19,822] jepsen worker 75 - jepsen.util 2955 :invoke :cas [25 [3 13]] INFO [2019-12-17 12:29:19,823] jepsen worker 62 - jepsen.util 2492 :invoke :cas [20 [11 12]] INFO [2019-12-17 12:29:19,823] jepsen worker 83 - jepsen.util 3053 :invoke :write [27 42] INFO [2019-12-17 12:29:19,833] jepsen worker 80 - jepsen.util 1970 :ok :write [26 10] INFO [2019-12-17 12:29:19,833] jepsen worker 80 - jepsen.util 1970 :invoke :cas [26 [3 12]] INFO [2019-12-17 12:29:19,837] jepsen worker 25 - jepsen.util 3085 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:29:19,837] jepsen worker 25 - jepsen.util 3085 :invoke :cas [8 [28 9]] INFO [2019-12-17 12:29:19,853] jepsen worker 84 - jepsen.util 2424 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:29:19,853] jepsen worker 84 - jepsen.util 2424 :invoke :cas [28 [40 16]] INFO [2019-12-17 12:29:19,889] jepsen worker 39 - jepsen.util 2919 :info :write [13 17] :RequestCanceledException INFO [2019-12-17 12:29:19,889] jepsen worker 17 - jepsen.util 2807 :info :write [5 4] :RequestCanceledException INFO [2019-12-17 12:29:19,890] jepsen worker 17 - jepsen.util 2897 :invoke :read [5 nil] INFO [2019-12-17 12:29:19,890] jepsen worker 39 - jepsen.util 3009 :invoke :read [13 nil] INFO [2019-12-17 12:29:19,895] jepsen worker 38 - jepsen.util 2108 :ok :read [12 34] INFO [2019-12-17 12:29:19,895] jepsen worker 38 - jepsen.util 2108 :invoke :read [12 nil] INFO [2019-12-17 12:29:19,903] jepsen worker 63 - jepsen.util 1863 :ok :write [21 19] INFO [2019-12-17 12:29:19,903] jepsen worker 63 - jepsen.util 1863 :invoke :read [21 nil] INFO [2019-12-17 12:29:19,907] jepsen worker 8 - jepsen.util 2348 :info :write [2 19] :RequestCanceledException INFO [2019-12-17 12:29:19,907] jepsen worker 42 - jepsen.util 3192 :info :cas [14 [6 18]] :RequestCanceledException INFO [2019-12-17 12:29:19,908] jepsen worker 8 - jepsen.util 2438 :invoke :write [2 38] INFO [2019-12-17 12:29:19,908] jepsen worker 42 - jepsen.util 3282 :invoke :read [14 nil] INFO [2019-12-17 12:29:19,911] jepsen worker 37 - jepsen.util 1657 :ok :write [12 4] INFO [2019-12-17 12:29:19,911] jepsen worker 37 - jepsen.util 1657 :invoke :cas [12 [0 31]] INFO [2019-12-17 12:29:19,923] jepsen worker 70 - jepsen.util 2320 :info :write [23 6] :RequestCanceledException INFO [2019-12-17 12:29:19,924] jepsen worker 70 - jepsen.util 2410 :invoke :write [23 33] INFO [2019-12-17 12:29:19,930] jepsen worker 63 - jepsen.util 1863 :ok :read [21 19] INFO [2019-12-17 12:29:19,930] jepsen worker 63 - jepsen.util 1863 :invoke :cas [21 [21 16]] INFO [2019-12-17 12:29:19,936] jepsen worker 5 - jepsen.util 1715 :info :write [1 25] :RequestCanceledException INFO [2019-12-17 12:29:19,936] jepsen worker 41 - jepsen.util 2651 :info :write [13 39] :RequestCanceledException INFO [2019-12-17 12:29:19,937] jepsen worker 5 - jepsen.util 1805 :invoke :cas [1 [6 20]] INFO [2019-12-17 12:29:19,937] jepsen worker 41 - jepsen.util 2741 :invoke :cas [13 [23 18]] INFO [2019-12-17 12:29:19,940] jepsen worker 46 - jepsen.util 2116 :ok :write [15 40] INFO [2019-12-17 12:29:19,941] jepsen worker 46 - jepsen.util 2116 :invoke :write [15 21] INFO [2019-12-17 12:29:19,947] jepsen worker 63 - jepsen.util 1863 :fail :cas [21 [21 16]] :ValueNotSwapFrom INFO [2019-12-17 12:29:19,948] jepsen worker 63 - jepsen.util 1863 :invoke :read [21 nil] INFO [2019-12-17 12:29:19,954] jepsen worker 33 - jepsen.util 2913 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:29:19,954] jepsen worker 77 - jepsen.util 2687 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:29:19,954] jepsen worker 33 - jepsen.util 2913 :invoke :read [11 nil] INFO [2019-12-17 12:29:19,954] jepsen worker 77 - jepsen.util 2687 :invoke :read [25 nil] INFO [2019-12-17 12:29:19,956] jepsen worker 45 - jepsen.util 2205 :ok :read [15 40] INFO [2019-12-17 12:29:19,956] jepsen worker 45 - jepsen.util 2205 :invoke :write [15 46] INFO [2019-12-17 12:29:19,956] jepsen worker 19 - jepsen.util 2269 :ok :write [6 25] INFO [2019-12-17 12:29:19,956] jepsen worker 19 - jepsen.util 2269 :invoke :cas [6 [2 7]] INFO [2019-12-17 12:29:19,959] jepsen worker 88 - jepsen.util 2068 :fail :cas [29 [9 14]] :ValueNotSwapFrom INFO [2019-12-17 12:29:19,960] jepsen worker 88 - jepsen.util 2068 :invoke :read [29 nil] INFO [2019-12-17 12:29:19,963] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11804,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:29:19,963] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11804,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:29:19,963] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11807,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:29:19,963] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11807,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:29:19,963] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11808,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:29:19,963] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11808,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:29:20,006] jepsen worker 63 - jepsen.util 1863 :ok :read [21 19] INFO [2019-12-17 12:29:20,007] jepsen worker 63 - jepsen.util 1863 :invoke :write [21 22] INFO [2019-12-17 12:29:20,017] jepsen worker 47 - jepsen.util 2477 :fail :cas [15 [23 3]] :ValueNotSwapFrom INFO [2019-12-17 12:29:20,017] jepsen worker 47 - jepsen.util 2477 :invoke :write [15 36] INFO [2019-12-17 12:29:20,022] jepsen worker 48 - jepsen.util 2118 :info :write [16 17] :RequestCanceledException INFO [2019-12-17 12:29:20,022] jepsen worker 18 - jepsen.util 2178 :fail :read [6 nil] :RequestTimeoutException INFO [2019-12-17 12:29:20,023] jepsen worker 54 - jepsen.util 1944 :info :write [18 10] :RequestCanceledException INFO [2019-12-17 12:29:20,023] jepsen worker 82 - jepsen.util 3142 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:29:20,023] jepsen worker 48 - jepsen.util 2208 :invoke :write [16 7] INFO [2019-12-17 12:29:20,023] jepsen worker 18 - jepsen.util 2178 :invoke :cas [6 [15 43]] INFO [2019-12-17 12:29:20,023] jepsen worker 82 - jepsen.util 3142 :invoke :cas [27 [6 46]] INFO [2019-12-17 12:29:20,024] jepsen worker 54 - jepsen.util 2034 :invoke :cas [18 [43 36]] INFO [2019-12-17 12:29:20,036] jepsen worker 66 - jepsen.util 2586 :info :write [22 5] :RequestCanceledException INFO [2019-12-17 12:29:20,037] jepsen worker 66 - jepsen.util 2676 :invoke :write [22 48] INFO [2019-12-17 12:29:20,042] jepsen worker 19 - jepsen.util 2269 :fail :cas [6 [2 7]] :ValueNotSwapFrom INFO [2019-12-17 12:29:20,042] jepsen worker 19 - jepsen.util 2269 :invoke :cas [6 [20 18]] INFO [2019-12-17 12:29:20,053] jepsen worker 61 - jepsen.util 2761 :info :write [20 8] :RequestCanceledException INFO [2019-12-17 12:29:20,054] jepsen worker 61 - jepsen.util 2851 :invoke :read [20 nil] INFO [2019-12-17 12:29:20,072] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11828,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:29:20,072] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11828,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:29:20,073] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11831,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:29:20,073] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11831,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:29:20,083] jepsen worker 32 - jepsen.util 2552 :ok :write [10 29] INFO [2019-12-17 12:29:20,084] jepsen worker 32 - jepsen.util 2552 :invoke :cas [10 [10 27]] INFO [2019-12-17 12:29:20,087] jepsen worker 72 - jepsen.util 2142 :ok :write [24 19] INFO [2019-12-17 12:29:20,087] jepsen worker 72 - jepsen.util 2142 :invoke :write [24 4] INFO [2019-12-17 12:29:20,102] jepsen worker 64 - jepsen.util 1684 :fail :cas [21 [28 3]] :ValueNotSwapFrom INFO [2019-12-17 12:29:20,102] jepsen worker 64 - jepsen.util 1684 :invoke :write [21 0] INFO [2019-12-17 12:29:20,105] jepsen worker 63 - jepsen.util 1863 :ok :write [21 22] INFO [2019-12-17 12:29:20,106] jepsen worker 63 - jepsen.util 1863 :invoke :read [21 nil] INFO [2019-12-17 12:29:20,118] jepsen worker 29 - jepsen.util 1829 :ok :read [9 24] INFO [2019-12-17 12:29:20,118] jepsen worker 29 - jepsen.util 1829 :invoke :read [9 nil] INFO [2019-12-17 12:29:20,121] jepsen worker 89 - jepsen.util 1979 :info :write [29 44] :RequestCanceledException INFO [2019-12-17 12:29:20,122] jepsen worker 89 - jepsen.util 2069 :invoke :cas [29 [15 39]] INFO [2019-12-17 12:29:20,142] jepsen worker 63 - jepsen.util 1863 :ok :read [21 22] INFO [2019-12-17 12:29:20,142] jepsen worker 63 - jepsen.util 1863 :invoke :cas [21 [45 20]] INFO [2019-12-17 12:29:20,152] jepsen worker 51 - jepsen.util 1851 :fail :cas [17 [13 14]] :ValueNotSwapFrom INFO [2019-12-17 12:29:20,152] jepsen worker 51 - jepsen.util 1851 :invoke :read [17 nil] INFO [2019-12-17 12:29:20,154] jepsen worker 87 - jepsen.util 1707 :info :write [29 45] :RequestCanceledException INFO [2019-12-17 12:29:20,155] jepsen worker 87 - jepsen.util 1797 :invoke :read [29 nil] INFO [2019-12-17 12:29:20,163] jepsen worker 80 - jepsen.util 1970 :fail :cas [26 [3 12]] :ValueNotSwapFrom INFO [2019-12-17 12:29:20,164] jepsen worker 80 - jepsen.util 1970 :invoke :write [26 20] INFO [2019-12-17 12:29:20,164] jepsen worker 63 - jepsen.util 1863 :fail :cas [21 [45 20]] :ValueNotSwapFrom INFO [2019-12-17 12:29:20,164] jepsen worker 63 - jepsen.util 1863 :invoke :cas [21 [0 40]] INFO [2019-12-17 12:29:20,178] jepsen worker 63 - jepsen.util 1863 :fail :cas [21 [0 40]] :ValueNotSwapFrom INFO [2019-12-17 12:29:20,178] jepsen worker 63 - jepsen.util 1863 :invoke :write [21 23] INFO [2019-12-17 12:29:20,181] jepsen worker 53 - jepsen.util 1943 :fail :cas [17 [1 31]] :ValueNotSwapFrom INFO [2019-12-17 12:29:20,182] jepsen worker 53 - jepsen.util 1943 :invoke :cas [17 [18 15]] INFO [2019-12-17 12:29:20,189] jepsen worker 2 - jepsen.util 2522 :info :write [0 23] :RequestCanceledException INFO [2019-12-17 12:29:20,189] jepsen worker 2 - jepsen.util 2612 :invoke :cas [0 [25 42]] INFO [2019-12-17 12:29:20,205] jepsen worker 9 - jepsen.util 2349 :ok :write [3 16] INFO [2019-12-17 12:29:20,205] jepsen worker 9 - jepsen.util 2349 :invoke :cas [3 [24 3]] INFO [2019-12-17 12:29:20,208] jepsen worker 47 - jepsen.util 2477 :ok :write [15 36] INFO [2019-12-17 12:29:20,208] jepsen worker 47 - jepsen.util 2477 :invoke :write [15 30] INFO [2019-12-17 12:29:20,220] jepsen worker 23 - jepsen.util 2543 :info :write [7 47] :RequestCanceledException INFO [2019-12-17 12:29:20,220] jepsen worker 36 - jepsen.util 1836 :info :write [12 10] :RequestCanceledException INFO [2019-12-17 12:29:20,220] jepsen worker 14 - jepsen.util 1994 :info :write [4 34] :RequestCanceledException INFO [2019-12-17 12:29:20,221] jepsen worker 38 - jepsen.util 2108 :ok :read [12 4] INFO [2019-12-17 12:29:20,221] jepsen worker 38 - jepsen.util 2108 :invoke :read [12 nil] INFO [2019-12-17 12:29:20,222] jepsen worker 23 - jepsen.util 2633 :invoke :read [7 nil] INFO [2019-12-17 12:29:20,222] jepsen worker 36 - jepsen.util 1926 :invoke :read [12 nil] INFO [2019-12-17 12:29:20,222] jepsen worker 14 - jepsen.util 2084 :invoke :cas [4 [30 13]] INFO [2019-12-17 12:29:20,225] jepsen worker 5 - jepsen.util 1805 :fail :cas [1 [6 20]] :ValueNotSwapFrom INFO [2019-12-17 12:29:20,225] jepsen worker 5 - jepsen.util 1805 :invoke :read [1 nil] INFO [2019-12-17 12:29:20,247] jepsen worker 9 - jepsen.util 2349 :fail :cas [3 [24 3]] :ValueNotSwapFrom INFO [2019-12-17 12:29:20,247] jepsen worker 9 - jepsen.util 2349 :invoke :cas [3 [22 44]] INFO [2019-12-17 12:29:20,259] jepsen worker 19 - jepsen.util 2269 :fail :cas [6 [20 18]] :ValueNotSwapFrom INFO [2019-12-17 12:29:20,259] jepsen worker 19 - jepsen.util 2269 :invoke :cas [6 [38 14]] INFO [2019-12-17 12:29:20,277] jepsen worker 37 - jepsen.util 1657 :fail :cas [12 [0 31]] :ValueNotSwapFrom INFO [2019-12-17 12:29:20,277] jepsen worker 37 - jepsen.util 1657 :invoke :read [12 nil] INFO [2019-12-17 12:29:20,280] jepsen worker 9 - jepsen.util 2349 :fail :cas [3 [22 44]] :ValueNotSwapFrom INFO [2019-12-17 12:29:20,280] jepsen worker 9 - jepsen.util 2349 :invoke :write [3 38] INFO [2019-12-17 12:29:20,288] jepsen worker 63 - jepsen.util 1863 :ok :write [21 23] INFO [2019-12-17 12:29:20,288] jepsen worker 63 - jepsen.util 1863 :invoke :cas [21 [31 48]] INFO [2019-12-17 12:29:20,292] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11856,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:29:20,292] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11864,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:29:20,292] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11856,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:29:20,292] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11864,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:29:20,292] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11858,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:29:20,292] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11867,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:29:20,292] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11858,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:29:20,292] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11867,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:29:20,292] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11868,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:29:20,292] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11868,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:29:20,300] jepsen worker 89 - jepsen.util 2069 :fail :cas [29 [15 39]] :ValueNotSwapFrom INFO [2019-12-17 12:29:20,301] jepsen worker 89 - jepsen.util 2069 :invoke :read [29 nil] INFO [2019-12-17 12:29:20,306] jepsen worker 44 - jepsen.util 2384 :info :cas [14 [43 19]] :RequestCanceledException INFO [2019-12-17 12:29:20,307] jepsen worker 44 - jepsen.util 2474 :invoke :read [14 nil] INFO [2019-12-17 12:29:20,317] jepsen worker 63 - jepsen.util 1863 :fail :cas [21 [31 48]] :ValueNotSwapFrom INFO [2019-12-17 12:29:20,317] jepsen worker 63 - jepsen.util 1863 :invoke :cas [21 [10 42]] INFO [2019-12-17 12:29:20,320] jepsen worker 88 - jepsen.util 2068 :ok :read [29 35] INFO [2019-12-17 12:29:20,320] jepsen worker 88 - jepsen.util 2068 :invoke :read [29 nil] INFO [2019-12-17 12:29:20,323] jepsen worker 49 - jepsen.util 2119 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:29:20,323] jepsen worker 49 - jepsen.util 2119 :invoke :read [16 nil] INFO [2019-12-17 12:29:20,337] jepsen worker 67 - jepsen.util 3127 :info :cas [22 [9 4]] :RequestCanceledException INFO [2019-12-17 12:29:20,338] jepsen worker 67 - jepsen.util 3217 :invoke :write [22 8] INFO [2019-12-17 12:29:20,345] jepsen worker 63 - jepsen.util 1863 :fail :cas [21 [10 42]] :ValueNotSwapFrom INFO [2019-12-17 12:29:20,345] jepsen worker 63 - jepsen.util 1863 :invoke :write [21 37] INFO [2019-12-17 12:29:20,353] jepsen worker 13 - jepsen.util 1813 :info :write [4 41] :RequestCanceledException INFO [2019-12-17 12:29:20,354] jepsen worker 13 - jepsen.util 1903 :invoke :read [4 nil] INFO [2019-12-17 12:29:20,374] jepsen worker 53 - jepsen.util 1943 :fail :cas [17 [18 15]] :ValueNotSwapFrom INFO [2019-12-17 12:29:20,374] jepsen worker 53 - jepsen.util 1943 :invoke :cas [17 [18 7]] INFO [2019-12-17 12:29:20,397] jepsen worker 63 - jepsen.util 1863 :ok :write [21 37] INFO [2019-12-17 12:29:20,397] jepsen worker 63 - jepsen.util 1863 :invoke :write [21 33] INFO [2019-12-17 12:29:20,407] jepsen worker 56 - jepsen.util 1946 :info :write [18 49] :RequestCanceledException INFO [2019-12-17 12:29:20,408] jepsen worker 56 - jepsen.util 2036 :invoke :cas [18 [8 7]] INFO [2019-12-17 12:29:20,416] jepsen worker 38 - jepsen.util 2108 :ok :read [12 4] INFO [2019-12-17 12:29:20,416] jepsen worker 38 - jepsen.util 2108 :invoke :write [12 34] INFO [2019-12-17 12:29:20,423] jepsen worker 36 - jepsen.util 1926 :ok :read [12 4] INFO [2019-12-17 12:29:20,423] jepsen worker 36 - jepsen.util 1926 :invoke :cas [12 [17 21]] INFO [2019-12-17 12:29:20,427] jepsen worker 5 - jepsen.util 1805 :ok :read [1 39] INFO [2019-12-17 12:29:20,428] jepsen worker 5 - jepsen.util 1805 :invoke :write [1 28] INFO [2019-12-17 12:29:20,443] jepsen worker 37 - jepsen.util 1657 :ok :read [12 4] INFO [2019-12-17 12:29:20,443] jepsen worker 37 - jepsen.util 1657 :invoke :read [12 nil] INFO [2019-12-17 12:29:20,447] jepsen worker 63 - jepsen.util 1863 :ok :write [21 33] INFO [2019-12-17 12:29:20,448] jepsen worker 63 - jepsen.util 1863 :invoke :read [21 nil] INFO [2019-12-17 12:29:20,453] jepsen worker 34 - jepsen.util 2824 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:29:20,454] jepsen worker 34 - jepsen.util 2824 :invoke :read [11 nil] INFO [2019-12-17 12:29:20,462] jepsen worker 63 - jepsen.util 1863 :ok :read [21 33] INFO [2019-12-17 12:29:20,462] jepsen worker 63 - jepsen.util 1863 :invoke :read [21 nil] INFO [2019-12-17 12:29:20,467] jepsen worker 3 - jepsen.util 1893 :ok :write [1 41] INFO [2019-12-17 12:29:20,467] jepsen worker 3 - jepsen.util 1893 :invoke :cas [1 [17 6]] INFO [2019-12-17 12:29:20,477] jepsen worker 63 - jepsen.util 1863 :ok :read [21 33] INFO [2019-12-17 12:29:20,478] jepsen worker 63 - jepsen.util 1863 :invoke :cas [21 [42 28]] INFO [2019-12-17 12:29:20,495] jepsen worker 63 - jepsen.util 1863 :fail :cas [21 [42 28]] :ValueNotSwapFrom INFO [2019-12-17 12:29:20,495] jepsen worker 63 - jepsen.util 1863 :invoke :cas [21 [24 36]] INFO [2019-12-17 12:29:20,504] jepsen worker 63 - jepsen.util 1863 :fail :cas [21 [24 36]] :ValueNotSwapFrom INFO [2019-12-17 12:29:20,504] jepsen worker 63 - jepsen.util 1863 :invoke :cas [21 [14 47]] INFO [2019-12-17 12:29:20,510] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11893,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:29:20,510] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11893,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:29:20,510] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11898,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:29:20,510] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11898,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:29:20,516] jepsen worker 63 - jepsen.util 1863 :fail :cas [21 [14 47]] :ValueNotSwapFrom INFO [2019-12-17 12:29:20,516] jepsen worker 63 - jepsen.util 1863 :invoke :write [21 39] INFO [2019-12-17 12:29:20,533] jepsen worker 54 - jepsen.util 2034 :fail :cas [18 [43 36]] :ValueNotSwapFrom INFO [2019-12-17 12:29:20,533] jepsen worker 54 - jepsen.util 2034 :invoke :write [18 24] INFO [2019-12-17 12:29:20,548] jepsen worker 89 - jepsen.util 2069 :ok :read [29 35] INFO [2019-12-17 12:29:20,549] jepsen worker 89 - jepsen.util 2069 :invoke :read [29 nil] INFO [2019-12-17 12:29:20,556] jepsen worker 88 - jepsen.util 2068 :ok :read [29 35] INFO [2019-12-17 12:29:20,557] jepsen worker 88 - jepsen.util 2068 :invoke :read [29 nil] INFO [2019-12-17 12:29:20,559] jepsen worker 55 - jepsen.util 2305 :ok :write [18 27] INFO [2019-12-17 12:29:20,560] jepsen worker 55 - jepsen.util 2305 :invoke :cas [18 [47 29]] INFO [2019-12-17 12:29:20,569] jepsen worker 56 - jepsen.util 2036 :fail :cas [18 [8 7]] :ValueNotSwapFrom INFO [2019-12-17 12:29:20,570] jepsen worker 56 - jepsen.util 2036 :invoke :read [18 nil] INFO [2019-12-17 12:29:20,580] jepsen worker 63 - jepsen.util 1863 :ok :write [21 39] INFO [2019-12-17 12:29:20,581] jepsen worker 63 - jepsen.util 1863 :invoke :write [21 2] INFO [2019-12-17 12:29:20,615] jepsen worker 53 - jepsen.util 1943 :fail :cas [17 [18 7]] :ValueNotSwapFrom INFO [2019-12-17 12:29:20,615] jepsen worker 53 - jepsen.util 1943 :invoke :read [17 nil] INFO [2019-12-17 12:29:20,622] jepsen worker 52 - jepsen.util 2032 :info :write [17 18] :RequestCanceledException INFO [2019-12-17 12:29:20,623] jepsen worker 52 - jepsen.util 2122 :invoke :cas [17 [48 12]] INFO [2019-12-17 12:29:20,635] jepsen worker 38 - jepsen.util 2108 :ok :write [12 34] INFO [2019-12-17 12:29:20,635] jepsen worker 38 - jepsen.util 2108 :invoke :write [12 41] INFO [2019-12-17 12:29:20,642] jepsen worker 63 - jepsen.util 1863 :ok :write [21 2] INFO [2019-12-17 12:29:20,642] jepsen worker 63 - jepsen.util 1863 :invoke :read [21 nil] INFO [2019-12-17 12:29:20,654] jepsen worker 4 - jepsen.util 1534 :info :write [1 28] :RequestCanceledException INFO [2019-12-17 12:29:20,655] jepsen worker 4 - jepsen.util 1624 :invoke :write [1 26] INFO [2019-12-17 12:29:20,668] jepsen worker 63 - jepsen.util 1863 :ok :read [21 2] INFO [2019-12-17 12:29:20,668] jepsen worker 63 - jepsen.util 1863 :invoke :write [21 16] INFO [2019-12-17 12:29:20,686] jepsen worker 89 - jepsen.util 2069 :ok :read [29 35] INFO [2019-12-17 12:29:20,687] jepsen worker 89 - jepsen.util 2069 :invoke :cas [29 [17 37]] INFO [2019-12-17 12:29:20,707] jepsen worker 72 - jepsen.util 2142 :ok :write [24 4] INFO [2019-12-17 12:29:20,707] jepsen worker 47 - jepsen.util 2477 :ok :write [15 30] INFO [2019-12-17 12:29:20,707] jepsen worker 72 - jepsen.util 2142 :invoke :write [24 7] INFO [2019-12-17 12:29:20,707] jepsen worker 47 - jepsen.util 2477 :invoke :read [15 nil] INFO [2019-12-17 12:29:20,713] jepsen worker 63 - jepsen.util 1863 :ok :write [21 16] INFO [2019-12-17 12:29:20,714] jepsen worker 63 - jepsen.util 1863 :invoke :write [21 27] INFO [2019-12-17 12:29:20,725] jepsen worker 53 - jepsen.util 1943 :ok :read [17 26] INFO [2019-12-17 12:29:20,725] jepsen worker 53 - jepsen.util 1943 :invoke :cas [17 [27 40]] INFO [2019-12-17 12:29:20,727] jepsen worker 3 - jepsen.util 1893 :fail :cas [1 [17 6]] :ValueNotSwapFrom INFO [2019-12-17 12:29:20,727] jepsen worker 3 - jepsen.util 1893 :invoke :write [1 11] INFO [2019-12-17 12:29:20,729] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11918,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:29:20,729] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11918,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:29:20,729] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11921,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:29:20,729] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11921,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:29:20,729] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11922,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:29:20,729] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11922,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:29:20,751] jepsen worker 78 - jepsen.util 2058 :info :write [26 2] :RequestCanceledException INFO [2019-12-17 12:29:20,753] jepsen worker 78 - jepsen.util 2148 :invoke :write [26 37] INFO [2019-12-17 12:29:20,768] jepsen worker 69 - jepsen.util 1959 :ok :write [23 36] INFO [2019-12-17 12:29:20,768] jepsen worker 69 - jepsen.util 1959 :invoke :read [23 nil] INFO [2019-12-17 12:29:20,783] jepsen worker 52 - jepsen.util 2122 :fail :cas [17 [48 12]] :ValueNotSwapFrom INFO [2019-12-17 12:29:20,783] jepsen worker 52 - jepsen.util 2122 :invoke :cas [17 [16 16]] INFO [2019-12-17 12:29:20,786] jepsen worker 29 - jepsen.util 1829 :ok :read [9 24] INFO [2019-12-17 12:29:20,786] jepsen worker 29 - jepsen.util 1829 :invoke :cas [9 [21 3]] INFO [2019-12-17 12:29:20,790] jepsen worker 36 - jepsen.util 1926 :fail :cas [12 [17 21]] :ValueNotSwapFrom INFO [2019-12-17 12:29:20,791] jepsen worker 36 - jepsen.util 1926 :invoke :cas [12 [10 35]] INFO [2019-12-17 12:29:20,803] jepsen worker 37 - jepsen.util 1657 :ok :read [12 34] INFO [2019-12-17 12:29:20,803] jepsen worker 37 - jepsen.util 1657 :invoke :cas [12 [10 28]] INFO [2019-12-17 12:29:20,810] jepsen worker 88 - jepsen.util 2068 :ok :read [29 35] INFO [2019-12-17 12:29:20,810] jepsen worker 88 - jepsen.util 2068 :invoke :read [29 nil] INFO [2019-12-17 12:29:20,813] jepsen worker 9 - jepsen.util 2349 :ok :write [3 38] INFO [2019-12-17 12:29:20,814] jepsen worker 9 - jepsen.util 2349 :invoke :read [3 nil] INFO [2019-12-17 12:29:20,815] jepsen worker 47 - jepsen.util 2477 :ok :read [15 30] INFO [2019-12-17 12:29:20,815] jepsen worker 47 - jepsen.util 2477 :invoke :read [15 nil] INFO [2019-12-17 12:29:20,822] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:29:20,823] jepsen nemesis - couchbase.util Rebalance check count: 85 INFO [2019-12-17 12:29:20,823] jepsen nemesis - couchbase.util Rebalance stuck count: 66 INFO [2019-12-17 12:29:20,823] jepsen worker 80 - jepsen.util 1970 :ok :write [26 20] INFO [2019-12-17 12:29:20,823] jepsen worker 60 - jepsen.util 2850 :info :cas [20 [41 45]] :RequestCanceledException INFO [2019-12-17 12:29:20,823] jepsen worker 80 - jepsen.util 1970 :invoke :write [26 31] INFO [2019-12-17 12:29:20,824] jepsen worker 60 - jepsen.util 2940 :invoke :cas [20 [20 23]] INFO [2019-12-17 12:29:20,826] jepsen worker 69 - jepsen.util 1959 :ok :read [23 36] INFO [2019-12-17 12:29:20,826] jepsen worker 69 - jepsen.util 1959 :invoke :read [23 nil] INFO [2019-12-17 12:29:20,835] jepsen worker 5 - jepsen.util 1805 :ok :write [1 28] INFO [2019-12-17 12:29:20,835] jepsen worker 5 - jepsen.util 1805 :invoke :write [1 28] INFO [2019-12-17 12:29:20,840] jepsen worker 52 - jepsen.util 2122 :fail :cas [17 [16 16]] :ValueNotSwapFrom INFO [2019-12-17 12:29:20,840] jepsen worker 47 - jepsen.util 2477 :ok :read [15 30] INFO [2019-12-17 12:29:20,840] jepsen worker 52 - jepsen.util 2122 :invoke :read [17 nil] INFO [2019-12-17 12:29:20,840] jepsen worker 47 - jepsen.util 2477 :invoke :cas [15 [9 24]] INFO [2019-12-17 12:29:20,865] jepsen worker 9 - jepsen.util 2349 :ok :read [3 38] INFO [2019-12-17 12:29:20,865] jepsen worker 9 - jepsen.util 2349 :invoke :write [3 42] INFO [2019-12-17 12:29:20,865] jepsen worker 36 - jepsen.util 1926 :fail :cas [12 [10 35]] :ValueNotSwapFrom INFO [2019-12-17 12:29:20,865] jepsen worker 36 - jepsen.util 1926 :invoke :cas [12 [26 42]] INFO [2019-12-17 12:29:20,866] jepsen worker 89 - jepsen.util 2069 :fail :cas [29 [17 37]] :ValueNotSwapFrom INFO [2019-12-17 12:29:20,866] jepsen worker 89 - jepsen.util 2069 :invoke :write [29 6] INFO [2019-12-17 12:29:20,888] jepsen worker 6 - jepsen.util 2796 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:29:20,888] jepsen worker 6 - jepsen.util 2796 :invoke :cas [2 [32 6]] INFO [2019-12-17 12:29:20,915] jepsen worker 47 - jepsen.util 2477 :fail :cas [15 [9 24]] :ValueNotSwapFrom INFO [2019-12-17 12:29:20,915] jepsen worker 47 - jepsen.util 2477 :invoke :write [15 43] INFO [2019-12-17 12:29:20,918] jepsen worker 37 - jepsen.util 1657 :fail :cas [12 [10 28]] :ValueNotSwapFrom INFO [2019-12-17 12:29:20,919] jepsen worker 37 - jepsen.util 1657 :invoke :write [12 46] INFO [2019-12-17 12:29:20,939] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11948,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:29:20,939] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11948,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:29:20,939] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11952,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:29:20,939] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11952,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:29:20,939] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11953,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:29:20,939] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11953,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:29:20,940] jepsen worker 38 - jepsen.util 2108 :ok :write [12 41] INFO [2019-12-17 12:29:20,940] jepsen worker 38 - jepsen.util 2108 :invoke :cas [12 [2 15]] INFO [2019-12-17 12:29:20,982] jepsen worker 53 - jepsen.util 1943 :fail :cas [17 [27 40]] :ValueNotSwapFrom INFO [2019-12-17 12:29:20,982] jepsen worker 53 - jepsen.util 1943 :invoke :write [17 12] INFO [2019-12-17 12:29:20,991] jepsen worker 63 - jepsen.util 1863 :ok :write [21 27] INFO [2019-12-17 12:29:20,991] jepsen worker 63 - jepsen.util 1863 :invoke :read [21 nil] INFO [2019-12-17 12:29:20,995] jepsen worker 88 - jepsen.util 2068 :ok :read [29 35] INFO [2019-12-17 12:29:20,995] jepsen worker 88 - jepsen.util 2068 :invoke :read [29 nil] INFO [2019-12-17 12:29:21,006] jepsen worker 7 - jepsen.util 2707 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:29:21,007] jepsen worker 7 - jepsen.util 2707 :invoke :read [2 nil] INFO [2019-12-17 12:29:21,008] jepsen worker 63 - jepsen.util 1863 :ok :read [21 27] INFO [2019-12-17 12:29:21,008] jepsen worker 63 - jepsen.util 1863 :invoke :read [21 nil] INFO [2019-12-17 12:29:21,034] jepsen worker 63 - jepsen.util 1863 :ok :read [21 27] INFO [2019-12-17 12:29:21,034] jepsen worker 63 - jepsen.util 1863 :invoke :read [21 nil] INFO [2019-12-17 12:29:21,036] jepsen worker 71 - jepsen.util 1691 :info :write [23 42] :RequestCanceledException INFO [2019-12-17 12:29:21,037] jepsen worker 71 - jepsen.util 1781 :invoke :write [23 34] INFO [2019-12-17 12:29:21,045] jepsen worker 63 - jepsen.util 1863 :ok :read [21 27] INFO [2019-12-17 12:29:21,045] jepsen worker 63 - jepsen.util 1863 :invoke :read [21 nil] INFO [2019-12-17 12:29:21,049] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11963,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:29:21,049] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11963,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:29:21,049] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11967,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:29:21,049] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11967,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:29:21,059] jepsen worker 69 - jepsen.util 1959 :ok :read [23 42] INFO [2019-12-17 12:29:21,059] jepsen worker 69 - jepsen.util 1959 :invoke :write [23 18] INFO [2019-12-17 12:29:21,064] jepsen worker 63 - jepsen.util 1863 :ok :read [21 27] INFO [2019-12-17 12:29:21,064] jepsen worker 63 - jepsen.util 1863 :invoke :write [21 1] INFO [2019-12-17 12:29:21,085] jepsen worker 52 - jepsen.util 2122 :ok :read [17 18] INFO [2019-12-17 12:29:21,085] jepsen worker 52 - jepsen.util 2122 :invoke :cas [17 [25 49]] INFO [2019-12-17 12:29:21,107] jepsen worker 12 - jepsen.util 2082 :info :write [4 5] :RequestCanceledException INFO [2019-12-17 12:29:21,109] jepsen worker 12 - jepsen.util 2172 :invoke :read [4 nil] INFO [2019-12-17 12:29:21,111] jepsen worker 36 - jepsen.util 1926 :fail :cas [12 [26 42]] :ValueNotSwapFrom INFO [2019-12-17 12:29:21,112] jepsen worker 36 - jepsen.util 1926 :invoke :read [12 nil] INFO [2019-12-17 12:29:21,113] jepsen worker 51 - jepsen.util 1851 :ok :read [17 18] INFO [2019-12-17 12:29:21,113] jepsen worker 51 - jepsen.util 1851 :invoke :cas [17 [0 28]] INFO [2019-12-17 12:29:21,117] jepsen worker 63 - jepsen.util 1863 :ok :write [21 1] INFO [2019-12-17 12:29:21,117] jepsen worker 63 - jepsen.util 1863 :invoke :read [21 nil] INFO [2019-12-17 12:29:21,132] jepsen worker 87 - jepsen.util 1797 :ok :read [29 35] INFO [2019-12-17 12:29:21,132] jepsen worker 87 - jepsen.util 1797 :invoke :read [29 nil] INFO [2019-12-17 12:29:21,138] jepsen worker 72 - jepsen.util 2142 :ok :write [24 7] INFO [2019-12-17 12:29:21,138] jepsen worker 9 - jepsen.util 2349 :ok :write [3 42] INFO [2019-12-17 12:29:21,138] jepsen worker 72 - jepsen.util 2142 :invoke :write [24 20] INFO [2019-12-17 12:29:21,138] jepsen worker 9 - jepsen.util 2349 :invoke :write [3 26] INFO [2019-12-17 12:29:21,148] jepsen worker 63 - jepsen.util 1863 :ok :read [21 1] INFO [2019-12-17 12:29:21,148] jepsen worker 63 - jepsen.util 1863 :invoke :cas [21 [4 5]] INFO [2019-12-17 12:29:21,152] jepsen worker 47 - jepsen.util 2477 :ok :write [15 43] INFO [2019-12-17 12:29:21,152] jepsen worker 47 - jepsen.util 2477 :invoke :write [15 47] INFO [2019-12-17 12:29:21,154] jepsen worker 70 - jepsen.util 2410 :ok :write [23 33] INFO [2019-12-17 12:29:21,155] jepsen worker 70 - jepsen.util 2410 :invoke :cas [23 [19 3]] INFO [2019-12-17 12:29:21,168] jepsen worker 63 - jepsen.util 1863 :fail :cas [21 [4 5]] :ValueNotSwapFrom INFO [2019-12-17 12:29:21,168] jepsen worker 63 - jepsen.util 1863 :invoke :write [21 9] INFO [2019-12-17 12:29:21,173] jepsen worker 14 - jepsen.util 2084 :fail :cas [4 [30 13]] :ValueNotSwapFrom INFO [2019-12-17 12:29:21,173] jepsen worker 14 - jepsen.util 2084 :invoke :read [4 nil] INFO [2019-12-17 12:29:21,195] jepsen worker 55 - jepsen.util 2305 :fail :cas [18 [47 29]] :ValueNotSwapFrom INFO [2019-12-17 12:29:21,196] jepsen worker 55 - jepsen.util 2305 :invoke :write [18 5] INFO [2019-12-17 12:29:21,199] jepsen worker 51 - jepsen.util 1851 :fail :cas [17 [0 28]] :ValueNotSwapFrom INFO [2019-12-17 12:29:21,199] jepsen worker 51 - jepsen.util 1851 :invoke :read [17 nil] INFO [2019-12-17 12:29:21,217] jepsen worker 87 - jepsen.util 1797 :ok :read [29 35] INFO [2019-12-17 12:29:21,217] jepsen worker 87 - jepsen.util 1797 :invoke :read [29 nil] INFO [2019-12-17 12:29:21,223] jepsen worker 73 - jepsen.util 2053 :info :write [24 48] :RequestCanceledException INFO [2019-12-17 12:29:21,224] jepsen worker 73 - jepsen.util 2143 :invoke :read [24 nil] INFO [2019-12-17 12:29:21,225] jepsen worker 51 - jepsen.util 1851 :ok :read [17 18] INFO [2019-12-17 12:29:21,225] jepsen worker 51 - jepsen.util 1851 :invoke :write [17 28] INFO [2019-12-17 12:29:21,231] jepsen worker 52 - jepsen.util 2122 :fail :cas [17 [25 49]] :ValueNotSwapFrom INFO [2019-12-17 12:29:21,232] jepsen worker 52 - jepsen.util 2122 :invoke :write [17 24] INFO [2019-12-17 12:29:21,259] jepsen worker 63 - jepsen.util 1863 :ok :write [21 9] INFO [2019-12-17 12:29:21,260] jepsen worker 63 - jepsen.util 1863 :invoke :cas [21 [11 47]] INFO [2019-12-17 12:29:21,275] jepsen worker 63 - jepsen.util 1863 :fail :cas [21 [11 47]] :ValueNotSwapFrom INFO [2019-12-17 12:29:21,275] jepsen worker 63 - jepsen.util 1863 :invoke :write [21 44] INFO [2019-12-17 12:29:21,275] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11991,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:29:21,275] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":12000,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:29:21,275] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11991,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:29:21,275] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":12000,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:29:21,275] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11992,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:29:21,275] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":11992,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:29:21,320] jepsen worker 88 - jepsen.util 2068 :ok :read [29 35] INFO [2019-12-17 12:29:21,320] jepsen worker 88 - jepsen.util 2068 :invoke :read [29 nil] INFO [2019-12-17 12:29:21,327] jepsen worker 63 - jepsen.util 1863 :ok :write [21 44] INFO [2019-12-17 12:29:21,327] jepsen worker 63 - jepsen.util 1863 :invoke :cas [21 [31 43]] INFO [2019-12-17 12:29:21,339] jepsen worker 63 - jepsen.util 1863 :fail :cas [21 [31 43]] :ValueNotSwapFrom INFO [2019-12-17 12:29:21,339] jepsen worker 63 - jepsen.util 1863 :invoke :cas [21 [18 0]] INFO [2019-12-17 12:29:21,353] jepsen worker 36 - jepsen.util 1926 :ok :read [12 41] INFO [2019-12-17 12:29:21,353] jepsen worker 36 - jepsen.util 1926 :invoke :cas [12 [8 7]] INFO [2019-12-17 12:29:21,363] jepsen worker 63 - jepsen.util 1863 :fail :cas [21 [18 0]] :ValueNotSwapFrom INFO [2019-12-17 12:29:21,364] jepsen worker 63 - jepsen.util 1863 :invoke :write [21 45] INFO [2019-12-17 12:29:21,401] jepsen worker 63 - jepsen.util 1863 :ok :write [21 45] INFO [2019-12-17 12:29:21,401] jepsen worker 63 - jepsen.util 1863 :invoke :read [21 nil] INFO [2019-12-17 12:29:21,409] jepsen worker 56 - jepsen.util 2036 :ok :read [18 27] INFO [2019-12-17 12:29:21,409] jepsen worker 56 - jepsen.util 2036 :invoke :read [18 nil] INFO [2019-12-17 12:29:21,414] jepsen worker 87 - jepsen.util 1797 :ok :read [29 35] INFO [2019-12-17 12:29:21,414] jepsen worker 87 - jepsen.util 1797 :invoke :cas [29 [13 15]] INFO [2019-12-17 12:29:21,419] jepsen worker 63 - jepsen.util 1863 :ok :read [21 45] INFO [2019-12-17 12:29:21,419] jepsen worker 63 - jepsen.util 1863 :invoke :cas [21 [45 10]] INFO [2019-12-17 12:29:21,468] jepsen worker 63 - jepsen.util 1863 :ok :cas [21 [45 10]] INFO [2019-12-17 12:29:21,468] jepsen worker 63 - jepsen.util 1863 :invoke :read [21 nil] INFO [2019-12-17 12:29:21,471] jepsen worker 87 - jepsen.util 1797 :fail :cas [29 [13 15]] :ValueNotSwapFrom INFO [2019-12-17 12:29:21,471] jepsen worker 87 - jepsen.util 1797 :invoke :write [29 4] INFO [2019-12-17 12:29:21,484] jepsen worker 63 - jepsen.util 1863 :ok :read [21 10] INFO [2019-12-17 12:29:21,484] jepsen worker 63 - jepsen.util 1863 :invoke :write [21 18] INFO [2019-12-17 12:29:21,489] jepsen worker 31 - jepsen.util 2011 :info :write [10 36] :RequestCanceledException INFO [2019-12-17 12:29:21,489] jepsen worker 28 - jepsen.util 1378 :info :write [9 1] :RequestCanceledException INFO [2019-12-17 12:29:21,489] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":12022,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:29:21,489] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":12022,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:29:21,490] jepsen worker 31 - jepsen.util 2101 :invoke :cas [10 [10 42]] INFO [2019-12-17 12:29:21,490] jepsen worker 28 - jepsen.util 1468 :invoke :write [9 9] INFO [2019-12-17 12:29:21,518] jepsen worker 70 - jepsen.util 2410 :fail :cas [23 [19 3]] :ValueNotSwapFrom INFO [2019-12-17 12:29:21,519] jepsen worker 70 - jepsen.util 2410 :invoke :cas [23 [39 6]] INFO [2019-12-17 12:29:21,522] jepsen worker 74 - jepsen.util 1964 :info :cas [24 [4 42]] :RequestCanceledException INFO [2019-12-17 12:29:21,522] jepsen worker 27 - jepsen.util 1917 :info :write [9 21] :RequestCanceledException INFO [2019-12-17 12:29:21,524] jepsen worker 27 - jepsen.util 2007 :invoke :cas [9 [6 31]] INFO [2019-12-17 12:29:21,524] jepsen worker 74 - jepsen.util 2054 :invoke :cas [24 [6 8]] INFO [2019-12-17 12:29:21,529] jepsen worker 63 - jepsen.util 1863 :ok :write [21 18] INFO [2019-12-17 12:29:21,529] jepsen worker 63 - jepsen.util 1863 :invoke :cas [21 [47 6]] INFO [2019-12-17 12:29:21,532] jepsen worker 13 - jepsen.util 1903 :ok :read [4 41] INFO [2019-12-17 12:29:21,532] jepsen worker 13 - jepsen.util 1903 :invoke :write [4 13] INFO [2019-12-17 12:29:21,535] jepsen worker 79 - jepsen.util 2059 :info :write [26 45] :RequestCanceledException INFO [2019-12-17 12:29:21,537] jepsen worker 79 - jepsen.util 2149 :invoke :read [26 nil] INFO [2019-12-17 12:29:21,544] jepsen worker 27 - jepsen.util 2007 :fail :cas [9 [6 31]] :ValueNotSwapFrom INFO [2019-12-17 12:29:21,544] jepsen worker 27 - jepsen.util 2007 :invoke :cas [9 [24 1]] INFO [2019-12-17 12:29:21,548] jepsen worker 63 - jepsen.util 1863 :fail :cas [21 [47 6]] :ValueNotSwapFrom INFO [2019-12-17 12:29:21,548] jepsen worker 63 - jepsen.util 1863 :invoke :read [21 nil] INFO [2019-12-17 12:29:21,570] jepsen worker 63 - jepsen.util 1863 :ok :read [21 18] INFO [2019-12-17 12:29:21,570] jepsen worker 63 - jepsen.util 1863 :invoke :read [21 nil] INFO [2019-12-17 12:29:21,582] jepsen worker 14 - jepsen.util 2084 :ok :read [4 41] INFO [2019-12-17 12:29:21,582] jepsen worker 14 - jepsen.util 2084 :invoke :read [4 nil] INFO [2019-12-17 12:29:21,590] jepsen worker 12 - jepsen.util 2172 :ok :read [4 41] INFO [2019-12-17 12:29:21,591] jepsen worker 12 - jepsen.util 2172 :invoke :read [4 nil] INFO [2019-12-17 12:29:21,591] jepsen worker 63 - jepsen.util 1863 :ok :read [21 18] INFO [2019-12-17 12:29:21,591] jepsen worker 63 - jepsen.util 1863 :invoke :write [21 29] INFO [2019-12-17 12:29:21,602] jepsen worker 38 - jepsen.util 2108 :fail :cas [12 [2 15]] :ValueNotSwapFrom INFO [2019-12-17 12:29:21,602] jepsen worker 38 - jepsen.util 2108 :invoke :read [12 nil] INFO [2019-12-17 12:29:21,607] jepsen worker 27 - jepsen.util 2007 :ok :cas [9 [24 1]] INFO [2019-12-17 12:29:21,607] jepsen worker 27 - jepsen.util 2007 :invoke :cas [9 [7 17]] INFO [2019-12-17 12:29:21,621] jepsen worker 27 - jepsen.util 2007 :fail :cas [9 [7 17]] :ValueNotSwapFrom INFO [2019-12-17 12:29:21,621] jepsen worker 27 - jepsen.util 2007 :invoke :write [9 27] INFO [2019-12-17 12:29:21,622] jepsen worker 15 - jepsen.util 2625 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:29:21,622] jepsen worker 15 - jepsen.util 2625 :invoke :write [5 45] INFO [2019-12-17 12:29:21,637] jepsen worker 30 - jepsen.util 2460 :info :write [10 45] :RequestCanceledException INFO [2019-12-17 12:29:21,638] jepsen worker 30 - jepsen.util 2550 :invoke :read [10 nil] INFO [2019-12-17 12:29:21,654] jepsen worker 16 - jepsen.util 2626 :info :write [5 43] :RequestCanceledException INFO [2019-12-17 12:29:21,655] jepsen worker 16 - jepsen.util 2716 :invoke :write [5 16] INFO [2019-12-17 12:29:21,666] jepsen worker 63 - jepsen.util 1863 :ok :write [21 29] INFO [2019-12-17 12:29:21,666] jepsen worker 63 - jepsen.util 1863 :invoke :cas [21 [15 16]] INFO [2019-12-17 12:29:21,675] jepsen worker 27 - jepsen.util 2007 :ok :write [9 27] INFO [2019-12-17 12:29:21,675] jepsen worker 27 - jepsen.util 2007 :invoke :cas [9 [6 43]] INFO [2019-12-17 12:29:21,685] jepsen worker 56 - jepsen.util 2036 :ok :read [18 27] INFO [2019-12-17 12:29:21,685] jepsen worker 56 - jepsen.util 2036 :invoke :write [18 36] INFO [2019-12-17 12:29:21,689] jepsen worker 43 - jepsen.util 2743 :info :write [14 8] :RequestCanceledException INFO [2019-12-17 12:29:21,692] jepsen worker 43 - jepsen.util 2833 :invoke :read [14 nil] INFO [2019-12-17 12:29:21,698] jepsen worker 79 - jepsen.util 2149 :ok :read [26 20] INFO [2019-12-17 12:29:21,698] jepsen worker 79 - jepsen.util 2149 :invoke :write [26 18] INFO [2019-12-17 12:29:21,703] jepsen worker 63 - jepsen.util 1863 :fail :cas [21 [15 16]] :ValueNotSwapFrom INFO [2019-12-17 12:29:21,703] jepsen worker 63 - jepsen.util 1863 :invoke :write [21 23] INFO [2019-12-17 12:29:21,704] jepsen worker 27 - jepsen.util 2007 :fail :cas [9 [6 43]] :ValueNotSwapFrom INFO [2019-12-17 12:29:21,704] jepsen worker 27 - jepsen.util 2007 :invoke :cas [9 [49 32]] INFO [2019-12-17 12:29:21,706] jepsen worker 38 - jepsen.util 2108 :ok :read [12 41] INFO [2019-12-17 12:29:21,706] jepsen worker 38 - jepsen.util 2108 :invoke :read [12 nil] INFO [2019-12-17 12:29:21,709] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":12049,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:29:21,709] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":12049,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:29:21,709] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":12052,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:29:21,709] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":12052,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:29:21,722] jepsen worker 20 - jepsen.util 1640 :fail :read [6 nil] :RequestTimeoutException INFO [2019-12-17 12:29:21,722] jepsen worker 20 - jepsen.util 1640 :invoke :write [6 8] INFO [2019-12-17 12:29:21,722] jepsen worker 88 - jepsen.util 2068 :ok :read [29 35] INFO [2019-12-17 12:29:21,722] jepsen worker 88 - jepsen.util 2068 :invoke :write [29 30] INFO [2019-12-17 12:29:21,725] jepsen worker 27 - jepsen.util 2007 :fail :cas [9 [49 32]] :ValueNotSwapFrom INFO [2019-12-17 12:29:21,725] jepsen worker 27 - jepsen.util 2007 :invoke :write [9 15] INFO [2019-12-17 12:29:21,736] jepsen worker 36 - jepsen.util 1926 :fail :cas [12 [8 7]] :ValueNotSwapFrom INFO [2019-12-17 12:29:21,736] jepsen worker 36 - jepsen.util 1926 :invoke :cas [12 [0 7]] INFO [2019-12-17 12:29:21,736] jepsen worker 21 - jepsen.util 2631 :info :cas [7 [33 12]] :RequestCanceledException INFO [2019-12-17 12:29:21,737] jepsen worker 21 - jepsen.util 2721 :invoke :cas [7 [29 4]] INFO [2019-12-17 12:29:21,753] jepsen worker 22 - jepsen.util 2902 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:29:21,753] jepsen worker 22 - jepsen.util 2902 :invoke :read [7 nil] INFO [2019-12-17 12:29:21,778] jepsen worker 63 - jepsen.util 1863 :ok :write [21 23] INFO [2019-12-17 12:29:21,778] jepsen worker 63 - jepsen.util 1863 :invoke :read [21 nil] INFO [2019-12-17 12:29:21,778] jepsen worker 12 - jepsen.util 2172 :ok :read [4 41] INFO [2019-12-17 12:29:21,779] jepsen worker 12 - jepsen.util 2172 :invoke :write [4 30] INFO [2019-12-17 12:29:21,781] jepsen worker 27 - jepsen.util 2007 :ok :write [9 15] INFO [2019-12-17 12:29:21,782] jepsen worker 27 - jepsen.util 2007 :invoke :write [9 48] INFO [2019-12-17 12:29:21,787] jepsen worker 1 - jepsen.util 2611 :info :cas [0 [9 33]] :RequestCanceledException INFO [2019-12-17 12:29:21,787] jepsen worker 76 - jepsen.util 2686 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:29:21,787] jepsen worker 76 - jepsen.util 2686 :invoke :cas [25 [32 16]] INFO [2019-12-17 12:29:21,790] jepsen worker 1 - jepsen.util 2701 :invoke :read [0 nil] INFO [2019-12-17 12:29:21,807] jepsen worker 0 - jepsen.util 2970 :info :write [0 49] :RequestCanceledException INFO [2019-12-17 12:29:21,807] jepsen worker 11 - jepsen.util 2531 :info :write [3 47] :RequestCanceledException INFO [2019-12-17 12:29:21,807] jepsen worker 10 - jepsen.util 2080 :info :write [3 16] :RequestCanceledException INFO [2019-12-17 12:29:21,809] jepsen worker 0 - jepsen.util 3060 :invoke :read [0 nil] INFO [2019-12-17 12:29:21,810] jepsen worker 11 - jepsen.util 2621 :invoke :read [3 nil] INFO [2019-12-17 12:29:21,810] jepsen worker 10 - jepsen.util 2170 :invoke :write [3 41] INFO [2019-12-17 12:29:21,822] jepsen worker 86 - jepsen.util 2876 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:29:21,822] jepsen worker 59 - jepsen.util 2399 :info :write [19 6] :RequestCanceledException INFO [2019-12-17 12:29:21,822] jepsen worker 81 - jepsen.util 2331 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:29:21,822] jepsen worker 50 - jepsen.util 2840 :info :write [16 14] :RequestCanceledException INFO [2019-12-17 12:29:21,822] jepsen worker 26 - jepsen.util 3086 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:29:21,822] jepsen worker 85 - jepsen.util 3055 :info :cas [28 [3 2]] :RequestCanceledException INFO [2019-12-17 12:29:21,823] jepsen worker 81 - jepsen.util 2331 :invoke :write [27 29] INFO [2019-12-17 12:29:21,823] jepsen worker 26 - jepsen.util 3086 :invoke :cas [8 [41 29]] INFO [2019-12-17 12:29:21,823] jepsen worker 86 - jepsen.util 2876 :invoke :read [28 nil] INFO [2019-12-17 12:29:21,827] jepsen worker 85 - jepsen.util 3145 :invoke :cas [28 [33 23]] INFO [2019-12-17 12:29:21,827] jepsen worker 59 - jepsen.util 2489 :invoke :cas [19 [48 47]] INFO [2019-12-17 12:29:21,827] jepsen worker 50 - jepsen.util 2930 :invoke :cas [16 [14 10]] INFO [2019-12-17 12:29:21,836] jepsen worker 65 - jepsen.util 1865 :info :write [21 25] :RequestCanceledException INFO [2019-12-17 12:29:21,840] jepsen worker 65 - jepsen.util 1955 :invoke :read [21 nil] INFO [2019-12-17 12:29:21,853] jepsen worker 58 - jepsen.util 2488 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:29:21,853] jepsen worker 35 - jepsen.util 2195 :info :cas [11 [25 21]] :RequestCanceledException INFO [2019-12-17 12:29:21,853] jepsen worker 57 - jepsen.util 2487 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:29:21,853] jepsen worker 24 - jepsen.util 2544 :info :write [8 8] :RequestCanceledException INFO [2019-12-17 12:29:21,854] jepsen worker 58 - jepsen.util 2488 :invoke :write [19 12] INFO [2019-12-17 12:29:21,854] jepsen worker 57 - jepsen.util 2487 :invoke :write [19 29] INFO [2019-12-17 12:29:21,857] jepsen worker 35 - jepsen.util 2285 :invoke :write [11 4] INFO [2019-12-17 12:29:21,858] jepsen worker 24 - jepsen.util 2634 :invoke :write [8 13] INFO [2019-12-17 12:29:21,861] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:29:21,861] jepsen nemesis - couchbase.util Rebalance check count: 86 INFO [2019-12-17 12:29:21,861] jepsen nemesis - couchbase.util Rebalance stuck count: 67 INFO [2019-12-17 12:29:21,885] jepsen worker 65 - jepsen.util 1955 :ok :read [21 23] INFO [2019-12-17 12:29:21,885] jepsen worker 65 - jepsen.util 1955 :invoke :cas [21 [14 47]] INFO [2019-12-17 12:29:21,885] jepsen worker 27 - jepsen.util 2007 :ok :write [9 48] INFO [2019-12-17 12:29:21,885] jepsen worker 27 - jepsen.util 2007 :invoke :write [9 10] INFO [2019-12-17 12:29:21,887] jepsen worker 68 - jepsen.util 2498 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:29:21,887] jepsen worker 68 - jepsen.util 2498 :invoke :read [22 nil] INFO [2019-12-17 12:29:21,907] jepsen worker 40 - jepsen.util 3100 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:29:21,907] jepsen worker 62 - jepsen.util 2492 :info :cas [20 [11 12]] :RequestCanceledException INFO [2019-12-17 12:29:21,907] jepsen worker 40 - jepsen.util 3100 :invoke :read [13 nil] INFO [2019-12-17 12:29:21,910] jepsen worker 62 - jepsen.util 2582 :invoke :write [20 9] INFO [2019-12-17 12:29:21,917] jepsen worker 65 - jepsen.util 1955 :fail :cas [21 [14 47]] :ValueNotSwapFrom INFO [2019-12-17 12:29:21,917] jepsen worker 65 - jepsen.util 1955 :invoke :cas [21 [4 2]] INFO [2019-12-17 12:29:21,923] jepsen worker 83 - jepsen.util 3053 :info :write [27 42] :RequestCanceledException INFO [2019-12-17 12:29:21,923] jepsen worker 75 - jepsen.util 2955 :info :cas [25 [3 13]] :RequestCanceledException INFO [2019-12-17 12:29:21,925] jepsen worker 75 - jepsen.util 3045 :invoke :cas [25 [24 31]] INFO [2019-12-17 12:29:21,925] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":12086,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:29:21,925] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":12086,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:29:21,926] jepsen worker 83 - jepsen.util 3143 :invoke :cas [27 [39 30]] INFO [2019-12-17 12:29:21,926] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":12090,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:29:21,926] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":12090,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:29:21,936] jepsen worker 25 - jepsen.util 3085 :info :cas [8 [28 9]] :RequestCanceledException INFO [2019-12-17 12:29:21,936] jepsen worker 17 - jepsen.util 2897 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:29:21,936] jepsen worker 17 - jepsen.util 2897 :invoke :read [5 nil] INFO [2019-12-17 12:29:21,939] jepsen worker 25 - jepsen.util 3175 :invoke :cas [8 [34 27]] INFO [2019-12-17 12:29:21,945] jepsen worker 65 - jepsen.util 1955 :fail :cas [21 [4 2]] :ValueNotSwapFrom INFO [2019-12-17 12:29:21,946] jepsen worker 65 - jepsen.util 1955 :invoke :write [21 36] INFO [2019-12-17 12:29:21,948] jepsen worker 27 - jepsen.util 2007 :ok :write [9 10] INFO [2019-12-17 12:29:21,948] jepsen worker 27 - jepsen.util 2007 :invoke :write [9 8] INFO [2019-12-17 12:29:21,953] jepsen worker 39 - jepsen.util 3009 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:29:21,953] jepsen worker 84 - jepsen.util 2424 :info :cas [28 [40 16]] :RequestCanceledException INFO [2019-12-17 12:29:21,953] jepsen worker 39 - jepsen.util 3009 :invoke :read [13 nil] INFO [2019-12-17 12:29:21,958] jepsen worker 84 - jepsen.util 2514 :invoke :write [28 14] INFO [2019-12-17 12:29:21,988] jepsen worker 42 - jepsen.util 3282 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:29:21,988] jepsen worker 42 - jepsen.util 3282 :invoke :read [14 nil] INFO [2019-12-17 12:29:22,023] jepsen worker 8 - jepsen.util 2438 :info :write [2 38] :RequestCanceledException INFO [2019-12-17 12:29:22,023] jepsen worker 46 - jepsen.util 2116 :info :write [15 21] :RequestCanceledException INFO [2019-12-17 12:29:22,023] jepsen worker 45 - jepsen.util 2205 :info :write [15 46] :RequestCanceledException INFO [2019-12-17 12:29:22,023] jepsen worker 27 - jepsen.util 2007 :ok :write [9 8] INFO [2019-12-17 12:29:22,023] jepsen worker 27 - jepsen.util 2007 :invoke :write [9 19] INFO [2019-12-17 12:29:22,025] jepsen worker 8 - jepsen.util 2528 :invoke :write [2 35] INFO [2019-12-17 12:29:22,025] jepsen worker 45 - jepsen.util 2295 :invoke :read [15 nil] INFO [2019-12-17 12:29:22,025] jepsen worker 46 - jepsen.util 2206 :invoke :cas [15 [18 13]] INFO [2019-12-17 12:29:22,032] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":12112,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:29:22,032] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":12112,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:29:22,036] jepsen worker 41 - jepsen.util 2741 :info :cas [13 [23 18]] :RequestCanceledException INFO [2019-12-17 12:29:22,039] jepsen worker 41 - jepsen.util 2831 :invoke :cas [13 [46 22]] INFO [2019-12-17 12:29:22,053] jepsen worker 77 - jepsen.util 2687 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:29:22,053] jepsen worker 33 - jepsen.util 2913 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:29:22,054] jepsen worker 48 - jepsen.util 2208 :info :write [16 7] :RequestCanceledException INFO [2019-12-17 12:29:22,054] jepsen worker 77 - jepsen.util 2687 :invoke :read [25 nil] INFO [2019-12-17 12:29:22,054] jepsen worker 33 - jepsen.util 2913 :invoke :cas [11 [20 32]] INFO [2019-12-17 12:29:22,058] jepsen worker 46 - jepsen.util 2206 :fail :cas [15 [18 13]] :ValueNotSwapFrom INFO [2019-12-17 12:29:22,059] jepsen worker 46 - jepsen.util 2206 :invoke :write [15 18] INFO [2019-12-17 12:29:22,059] jepsen worker 48 - jepsen.util 2298 :invoke :write [16 24] INFO [2019-12-17 12:29:22,089] jepsen worker 66 - jepsen.util 2676 :info :write [22 48] :RequestCanceledException INFO [2019-12-17 12:29:22,091] jepsen worker 66 - jepsen.util 2766 :invoke :cas [22 [1 40]] INFO [2019-12-17 12:29:22,096] jepsen worker 70 - jepsen.util 2410 :fail :cas [23 [39 6]] :ValueNotSwapFrom INFO [2019-12-17 12:29:22,096] jepsen worker 70 - jepsen.util 2410 :invoke :cas [23 [25 39]] INFO [2019-12-17 12:29:22,098] jepsen worker 65 - jepsen.util 1955 :ok :write [21 36] INFO [2019-12-17 12:29:22,098] jepsen worker 65 - jepsen.util 1955 :invoke :cas [21 [12 36]] INFO [2019-12-17 12:29:22,107] jepsen worker 61 - jepsen.util 2851 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:29:22,107] jepsen worker 61 - jepsen.util 2851 :invoke :cas [20 [13 4]] INFO [2019-12-17 12:29:22,118] jepsen worker 27 - jepsen.util 2007 :ok :write [9 19] INFO [2019-12-17 12:29:22,118] jepsen worker 27 - jepsen.util 2007 :invoke :read [9 nil] INFO [2019-12-17 12:29:22,122] jepsen worker 18 - jepsen.util 2178 :info :cas [6 [15 43]] :RequestCanceledException INFO [2019-12-17 12:29:22,122] jepsen worker 82 - jepsen.util 3142 :info :cas [27 [6 46]] :RequestCanceledException INFO [2019-12-17 12:29:22,125] jepsen worker 18 - jepsen.util 2268 :invoke :write [6 19] INFO [2019-12-17 12:29:22,125] jepsen worker 82 - jepsen.util 3232 :invoke :write [27 30] INFO [2019-12-17 12:29:22,130] jepsen worker 65 - jepsen.util 1955 :fail :cas [21 [12 36]] :ValueNotSwapFrom INFO [2019-12-17 12:29:22,130] jepsen worker 65 - jepsen.util 1955 :invoke :write [21 18] INFO [2019-12-17 12:29:22,136] jepsen worker 32 - jepsen.util 2552 :info :cas [10 [10 27]] :RequestCanceledException INFO [2019-12-17 12:29:22,139] jepsen worker 32 - jepsen.util 2642 :invoke :cas [10 [43 43]] INFO [2019-12-17 12:29:22,153] jepsen worker 64 - jepsen.util 1684 :info :write [21 0] :RequestCanceledException INFO [2019-12-17 12:29:22,156] jepsen worker 64 - jepsen.util 1774 :invoke :cas [21 [19 38]] INFO [2019-12-17 12:29:22,178] jepsen worker 32 - jepsen.util 2642 :fail :cas [10 [43 43]] :ValueNotSwapFrom INFO [2019-12-17 12:29:22,178] jepsen worker 32 - jepsen.util 2642 :invoke :write [10 28] INFO [2019-12-17 12:29:22,200] jepsen worker 14 - jepsen.util 2084 :ok :read [4 41] INFO [2019-12-17 12:29:22,200] jepsen worker 14 - jepsen.util 2084 :invoke :cas [4 [12 16]] INFO [2019-12-17 12:29:22,230] jepsen worker 36 - jepsen.util 1926 :fail :cas [12 [0 7]] :ValueNotSwapFrom INFO [2019-12-17 12:29:22,230] jepsen worker 36 - jepsen.util 1926 :invoke :read [12 nil] INFO [2019-12-17 12:29:22,236] jepsen worker 65 - jepsen.util 1955 :ok :write [21 18] INFO [2019-12-17 12:29:22,236] jepsen worker 65 - jepsen.util 1955 :invoke :read [21 nil] INFO [2019-12-17 12:29:22,253] jepsen worker 2 - jepsen.util 2612 :info :cas [0 [25 42]] :RequestCanceledException INFO [2019-12-17 12:29:22,255] jepsen worker 2 - jepsen.util 2702 :invoke :read [0 nil] INFO [2019-12-17 12:29:22,259] jepsen worker 70 - jepsen.util 2410 :fail :cas [23 [25 39]] :ValueNotSwapFrom INFO [2019-12-17 12:29:22,259] jepsen worker 70 - jepsen.util 2410 :invoke :write [23 14] INFO [2019-12-17 12:29:22,261] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":12144,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:29:22,261] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":12144,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:29:22,261] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":12147,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:29:22,261] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":12147,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:29:22,262] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":12148,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:29:22,262] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":12148,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:29:22,262] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":12136,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:29:22,262] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":12136,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:29:22,262] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":12137,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:29:22,262] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":12137,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:29:22,263] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":12138,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:29:22,263] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":12138,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:29:22,269] jepsen worker 65 - jepsen.util 1955 :ok :read [21 18] INFO [2019-12-17 12:29:22,270] jepsen worker 65 - jepsen.util 1955 :invoke :cas [21 [18 18]] INFO [2019-12-17 12:29:22,288] jepsen worker 19 - jepsen.util 2269 :info :cas [6 [38 14]] :RequestCanceledException INFO [2019-12-17 12:29:22,293] jepsen worker 19 - jepsen.util 2359 :invoke :read [6 nil] INFO [2019-12-17 12:29:22,322] jepsen worker 44 - jepsen.util 2474 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:29:22,322] jepsen worker 23 - jepsen.util 2633 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:29:22,322] jepsen worker 44 - jepsen.util 2474 :invoke :write [14 38] INFO [2019-12-17 12:29:22,322] jepsen worker 23 - jepsen.util 2633 :invoke :read [7 nil] INFO [2019-12-17 12:29:22,353] jepsen worker 36 - jepsen.util 1926 :ok :read [12 41] INFO [2019-12-17 12:29:22,353] jepsen worker 36 - jepsen.util 1926 :invoke :read [12 nil] INFO [2019-12-17 12:29:22,358] jepsen worker 65 - jepsen.util 1955 :ok :cas [21 [18 18]] INFO [2019-12-17 12:29:22,358] jepsen worker 65 - jepsen.util 1955 :invoke :write [21 4] INFO [2019-12-17 12:29:22,404] jepsen worker 65 - jepsen.util 1955 :ok :write [21 4] INFO [2019-12-17 12:29:22,404] jepsen worker 65 - jepsen.util 1955 :invoke :write [21 37] INFO [2019-12-17 12:29:22,423] jepsen worker 49 - jepsen.util 2119 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:29:22,423] jepsen worker 49 - jepsen.util 2119 :invoke :write [16 22] INFO [2019-12-17 12:29:22,436] jepsen worker 67 - jepsen.util 3217 :info :write [22 8] :RequestCanceledException INFO [2019-12-17 12:29:22,438] jepsen worker 67 - jepsen.util 3307 :invoke :read [22 nil] INFO [2019-12-17 12:29:22,461] jepsen worker 65 - jepsen.util 1955 :ok :write [21 37] INFO [2019-12-17 12:29:22,461] jepsen worker 65 - jepsen.util 1955 :invoke :write [21 38] INFO [2019-12-17 12:29:22,471] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":12162,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:29:22,471] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":12162,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:29:22,471] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":12166,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:29:22,471] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":12166,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:29:22,504] jepsen worker 65 - jepsen.util 1955 :ok :write [21 38] INFO [2019-12-17 12:29:22,504] jepsen worker 65 - jepsen.util 1955 :invoke :cas [21 [13 34]] INFO [2019-12-17 12:29:22,529] jepsen worker 65 - jepsen.util 1955 :fail :cas [21 [13 34]] :ValueNotSwapFrom INFO [2019-12-17 12:29:22,529] jepsen worker 65 - jepsen.util 1955 :invoke :read [21 nil] INFO [2019-12-17 12:29:22,551] jepsen worker 65 - jepsen.util 1955 :ok :read [21 38] INFO [2019-12-17 12:29:22,552] jepsen worker 65 - jepsen.util 1955 :invoke :read [21 nil] INFO [2019-12-17 12:29:22,554] jepsen worker 34 - jepsen.util 2824 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:29:22,554] jepsen worker 34 - jepsen.util 2824 :invoke :cas [11 [29 48]] INFO [2019-12-17 12:29:22,573] jepsen worker 65 - jepsen.util 1955 :ok :read [21 38] INFO [2019-12-17 12:29:22,574] jepsen worker 65 - jepsen.util 1955 :invoke :write [21 10] INFO [2019-12-17 12:29:22,588] jepsen worker 14 - jepsen.util 2084 :fail :cas [4 [12 16]] :ValueNotSwapFrom INFO [2019-12-17 12:29:22,588] jepsen worker 14 - jepsen.util 2084 :invoke :cas [4 [31 26]] INFO [2019-12-17 12:29:22,614] jepsen worker 38 - jepsen.util 2108 :ok :read [12 41] INFO [2019-12-17 12:29:22,614] jepsen worker 38 - jepsen.util 2108 :invoke :cas [12 [30 30]] INFO [2019-12-17 12:29:22,622] jepsen worker 54 - jepsen.util 2034 :info :write [18 24] :RequestCanceledException INFO [2019-12-17 12:29:22,625] jepsen worker 54 - jepsen.util 2124 :invoke :cas [18 [0 15]] INFO [2019-12-17 12:29:22,656] jepsen worker 14 - jepsen.util 2084 :fail :cas [4 [31 26]] :ValueNotSwapFrom INFO [2019-12-17 12:29:22,656] jepsen worker 14 - jepsen.util 2084 :invoke :write [4 15] INFO [2019-12-17 12:29:22,682] jepsen worker 65 - jepsen.util 1955 :ok :write [21 10] INFO [2019-12-17 12:29:22,682] jepsen worker 65 - jepsen.util 1955 :invoke :write [21 28] INFO [2019-12-17 12:29:22,689] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":12179,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:29:22,689] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":12179,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:29:22,689] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":12183,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:29:22,689] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":12183,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:29:22,707] jepsen worker 4 - jepsen.util 1624 :info :write [1 26] :RequestCanceledException INFO [2019-12-17 12:29:22,709] jepsen worker 4 - jepsen.util 1714 :invoke :cas [1 [19 49]] INFO [2019-12-17 12:29:22,745] jepsen worker 65 - jepsen.util 1955 :ok :write [21 28] INFO [2019-12-17 12:29:22,746] jepsen worker 65 - jepsen.util 1955 :invoke :cas [21 [44 41]] INFO [2019-12-17 12:29:22,768] jepsen worker 38 - jepsen.util 2108 :fail :cas [12 [30 30]] :ValueNotSwapFrom INFO [2019-12-17 12:29:22,768] jepsen worker 38 - jepsen.util 2108 :invoke :write [12 38] INFO [2019-12-17 12:29:22,788] jepsen worker 65 - jepsen.util 1955 :fail :cas [21 [44 41]] :ValueNotSwapFrom INFO [2019-12-17 12:29:22,788] jepsen worker 65 - jepsen.util 1955 :invoke :write [21 28] INFO [2019-12-17 12:29:22,823] jepsen worker 78 - jepsen.util 2148 :info :write [26 37] :RequestCanceledException INFO [2019-12-17 12:29:22,823] jepsen worker 3 - jepsen.util 1893 :info :write [1 11] :RequestCanceledException INFO [2019-12-17 12:29:22,824] jepsen worker 78 - jepsen.util 2238 :invoke :write [26 46] INFO [2019-12-17 12:29:22,824] jepsen worker 3 - jepsen.util 1983 :invoke :cas [1 [11 48]] INFO [2019-12-17 12:29:22,836] jepsen worker 65 - jepsen.util 1955 :ok :write [21 28] INFO [2019-12-17 12:29:22,836] jepsen worker 65 - jepsen.util 1955 :invoke :write [21 27] INFO [2019-12-17 12:29:22,837] jepsen worker 60 - jepsen.util 2940 :info :cas [20 [20 23]] :RequestCanceledException INFO [2019-12-17 12:29:22,840] jepsen worker 60 - jepsen.util 3030 :invoke :read [20 nil] INFO [2019-12-17 12:29:22,850] jepsen worker 29 - jepsen.util 1829 :info :cas [9 [21 3]] :RequestCanceledException INFO [2019-12-17 12:29:22,853] jepsen worker 29 - jepsen.util 1919 :invoke :read [9 nil] INFO [2019-12-17 12:29:22,879] jepsen worker 54 - jepsen.util 2124 :fail :cas [18 [0 15]] :ValueNotSwapFrom INFO [2019-12-17 12:29:22,880] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:29:22,880] jepsen worker 54 - jepsen.util 2124 :invoke :write [18 35] INFO [2019-12-17 12:29:22,880] jepsen nemesis - couchbase.util Rebalance check count: 87 INFO [2019-12-17 12:29:22,880] jepsen nemesis - couchbase.util Rebalance stuck count: 68 INFO [2019-12-17 12:29:22,900] jepsen worker 65 - jepsen.util 1955 :ok :write [21 27] INFO [2019-12-17 12:29:22,900] jepsen worker 65 - jepsen.util 1955 :invoke :read [21 nil] INFO [2019-12-17 12:29:22,907] jepsen worker 80 - jepsen.util 1970 :info :write [26 31] :RequestCanceledException INFO [2019-12-17 12:29:22,909] jepsen worker 80 - jepsen.util 2060 :invoke :cas [26 [43 40]] INFO [2019-12-17 12:29:22,915] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":12198,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:29:22,915] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":12198,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:29:22,915] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":12200,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:29:22,915] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":12200,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:29:22,923] jepsen worker 5 - jepsen.util 1805 :info :write [1 28] :RequestCanceledException INFO [2019-12-17 12:29:22,923] jepsen worker 89 - jepsen.util 2069 :info :write [29 6] :RequestCanceledException INFO [2019-12-17 12:29:22,924] jepsen worker 5 - jepsen.util 1895 :invoke :read [1 nil] INFO [2019-12-17 12:29:22,925] jepsen worker 89 - jepsen.util 2159 :invoke :cas [29 [27 2]] INFO [2019-12-17 12:29:22,932] jepsen worker 65 - jepsen.util 1955 :ok :read [21 27] INFO [2019-12-17 12:29:22,932] jepsen worker 65 - jepsen.util 1955 :invoke :cas [21 [33 39]] INFO [2019-12-17 12:29:22,978] jepsen worker 65 - jepsen.util 1955 :fail :cas [21 [33 39]] :ValueNotSwapFrom INFO [2019-12-17 12:29:22,979] jepsen worker 65 - jepsen.util 1955 :invoke :read [21 nil] INFO [2019-12-17 12:29:22,989] jepsen worker 6 - jepsen.util 2796 :info :cas [2 [32 6]] :RequestCanceledException INFO [2019-12-17 12:29:22,989] jepsen worker 37 - jepsen.util 1657 :info :write [12 46] :RequestCanceledException INFO [2019-12-17 12:29:22,992] jepsen worker 37 - jepsen.util 1747 :invoke :write [12 25] INFO [2019-12-17 12:29:22,994] jepsen worker 6 - jepsen.util 2886 :invoke :read [2 nil] INFO [2019-12-17 12:29:23,002] jepsen worker 89 - jepsen.util 2159 :fail :cas [29 [27 2]] :ValueNotSwapFrom INFO [2019-12-17 12:29:23,002] jepsen worker 89 - jepsen.util 2159 :invoke :cas [29 [39 28]] INFO [2019-12-17 12:29:23,011] jepsen worker 65 - jepsen.util 1955 :ok :read [21 27] INFO [2019-12-17 12:29:23,011] jepsen worker 65 - jepsen.util 1955 :invoke :read [21 nil] INFO [2019-12-17 12:29:23,020] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":12208,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:29:23,020] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":12208,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:29:23,020] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":12212,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:29:23,020] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":12212,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:29:23,020] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":12213,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:29:23,020] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":12213,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:29:23,023] jepsen worker 53 - jepsen.util 1943 :info :write [17 12] :RequestCanceledException INFO [2019-12-17 12:29:23,025] jepsen worker 53 - jepsen.util 2033 :invoke :cas [17 [33 17]] INFO [2019-12-17 12:29:23,031] jepsen worker 72 - jepsen.util 2142 :info :write [24 20] :SyncWriteAmbiguous INFO [2019-12-17 12:29:23,031] jepsen worker 47 - jepsen.util 2477 :info :write [15 47] :SyncWriteAmbiguous INFO [2019-12-17 12:29:23,036] jepsen worker 72 - jepsen.util 2232 :invoke :cas [24 [19 9]] INFO [2019-12-17 12:29:23,037] jepsen worker 47 - jepsen.util 2567 :invoke :cas [15 [16 47]] INFO [2019-12-17 12:29:23,047] jepsen worker 65 - jepsen.util 1955 :ok :read [21 27] INFO [2019-12-17 12:29:23,047] jepsen worker 65 - jepsen.util 1955 :invoke :write [21 28] INFO [2019-12-17 12:29:23,053] jepsen worker 71 - jepsen.util 1781 :info :write [23 34] :RequestCanceledException INFO [2019-12-17 12:29:23,061] jepsen worker 9 - jepsen.util 2349 :info :write [3 26] :SyncWriteAmbiguous INFO [2019-12-17 12:29:23,061] jepsen worker 4 - jepsen.util 1714 :fail :cas [1 [19 49]] :ValueNotSwapFrom INFO [2019-12-17 12:29:23,061] jepsen worker 4 - jepsen.util 1714 :invoke :write [1 44] INFO [2019-12-17 12:29:23,062] jepsen worker 71 - jepsen.util 1871 :invoke :read [23 nil] INFO [2019-12-17 12:29:23,066] jepsen worker 9 - jepsen.util 2439 :invoke :cas [3 [35 23]] INFO [2019-12-17 12:29:23,089] jepsen worker 69 - jepsen.util 1959 :info :write [23 18] :RequestCanceledException INFO [2019-12-17 12:29:23,095] jepsen worker 69 - jepsen.util 2049 :invoke :cas [23 [24 23]] INFO [2019-12-17 12:29:23,106] jepsen worker 7 - jepsen.util 2707 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:29:23,106] jepsen worker 7 - jepsen.util 2707 :invoke :read [2 nil] INFO [2019-12-17 12:29:23,108] jepsen worker 74 - jepsen.util 2054 :fail :cas [24 [6 8]] :ValueNotSwapFrom INFO [2019-12-17 12:29:23,108] jepsen worker 74 - jepsen.util 2054 :invoke :write [24 47] INFO [2019-12-17 12:29:23,117] jepsen worker 73 - jepsen.util 2143 :ok :read [24 7] INFO [2019-12-17 12:29:23,118] jepsen worker 73 - jepsen.util 2143 :invoke :write [24 48] INFO [2019-12-17 12:29:23,121] jepsen worker 36 - jepsen.util 1926 :ok :read [12 41] INFO [2019-12-17 12:29:23,121] jepsen worker 36 - jepsen.util 1926 :invoke :write [12 22] INFO [2019-12-17 12:29:23,139] jepsen worker 30 - jepsen.util 2550 :ok :read [10 29] INFO [2019-12-17 12:29:23,140] jepsen worker 30 - jepsen.util 2550 :invoke :cas [10 [0 42]] INFO [2019-12-17 12:29:23,151] jepsen worker 5 - jepsen.util 1895 :ok :read [1 28] INFO [2019-12-17 12:29:23,151] jepsen worker 5 - jepsen.util 1895 :invoke :cas [1 [20 29]] INFO [2019-12-17 12:29:23,164] jepsen worker 64 - jepsen.util 1774 :fail :cas [21 [19 38]] :ValueNotSwapFrom INFO [2019-12-17 12:29:23,164] jepsen worker 64 - jepsen.util 1774 :invoke :cas [21 [21 46]] INFO [2019-12-17 12:29:23,181] jepsen worker 31 - jepsen.util 2101 :fail :cas [10 [10 42]] :ValueNotSwapFrom INFO [2019-12-17 12:29:23,181] jepsen worker 31 - jepsen.util 2101 :invoke :write [10 13] INFO [2019-12-17 12:29:23,185] jepsen worker 11 - jepsen.util 2621 :ok :read [3 42] INFO [2019-12-17 12:29:23,186] jepsen worker 11 - jepsen.util 2621 :invoke :cas [3 [9 36]] INFO [2019-12-17 12:29:23,212] jepsen worker 19 - jepsen.util 2359 :ok :read [6 25] INFO [2019-12-17 12:29:23,212] jepsen worker 19 - jepsen.util 2359 :invoke :cas [6 [29 26]] INFO [2019-12-17 12:29:23,218] jepsen worker 9 - jepsen.util 2439 :fail :cas [3 [35 23]] :ValueNotSwapFrom INFO [2019-12-17 12:29:23,218] jepsen worker 9 - jepsen.util 2439 :invoke :read [3 nil] INFO [2019-12-17 12:29:23,236] jepsen worker 55 - jepsen.util 2305 :info :write [18 5] :RequestCanceledException INFO [2019-12-17 12:29:23,239] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":12234,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:29:23,239] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":12242,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:29:23,240] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":12234,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:29:23,240] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":12242,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:29:23,240] jepsen worker 55 - jepsen.util 2395 :invoke :cas [18 [7 24]] INFO [2019-12-17 12:29:23,240] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":12235,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:29:23,240] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":12245,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:29:23,240] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":12245,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:29:23,242] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":12235,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:29:23,248] jepsen worker 69 - jepsen.util 2049 :fail :cas [23 [24 23]] :ValueNotSwapFrom INFO [2019-12-17 12:29:23,248] jepsen worker 69 - jepsen.util 2049 :invoke :cas [23 [9 13]] INFO [2019-12-17 12:29:23,253] jepsen worker 65 - jepsen.util 1955 :ok :write [21 28] INFO [2019-12-17 12:29:23,254] jepsen worker 65 - jepsen.util 1955 :invoke :read [21 nil] INFO [2019-12-17 12:29:23,287] jepsen worker 52 - jepsen.util 2122 :info :write [17 24] :RequestCanceledException INFO [2019-12-17 12:29:23,290] jepsen worker 52 - jepsen.util 2212 :invoke :cas [17 [1 12]] INFO [2019-12-17 12:29:23,323] jepsen worker 89 - jepsen.util 2159 :fail :cas [29 [39 28]] :ValueNotSwapFrom INFO [2019-12-17 12:29:23,324] jepsen worker 89 - jepsen.util 2159 :invoke :cas [29 [4 47]] INFO [2019-12-17 12:29:23,336] jepsen worker 51 - jepsen.util 1851 :info :write [17 28] :RequestCanceledException INFO [2019-12-17 12:29:23,340] jepsen worker 9 - jepsen.util 2439 :ok :read [3 42] INFO [2019-12-17 12:29:23,340] jepsen worker 9 - jepsen.util 2439 :invoke :cas [3 [13 31]] INFO [2019-12-17 12:29:23,345] jepsen worker 51 - jepsen.util 1941 :invoke :read [17 nil] INFO [2019-12-17 12:29:23,349] jepsen worker 30 - jepsen.util 2550 :fail :cas [10 [0 42]] :ValueNotSwapFrom INFO [2019-12-17 12:29:23,349] jepsen worker 30 - jepsen.util 2550 :invoke :cas [10 [2 46]] INFO [2019-12-17 12:29:23,366] jepsen worker 65 - jepsen.util 1955 :ok :read [21 28] INFO [2019-12-17 12:29:23,366] jepsen worker 71 - jepsen.util 1871 :ok :read [23 33] INFO [2019-12-17 12:29:23,366] jepsen worker 65 - jepsen.util 1955 :invoke :write [21 18] INFO [2019-12-17 12:29:23,366] jepsen worker 71 - jepsen.util 1871 :invoke :cas [23 [29 40]] INFO [2019-12-17 12:29:23,366] jepsen worker 19 - jepsen.util 2359 :fail :cas [6 [29 26]] :ValueNotSwapFrom INFO [2019-12-17 12:29:23,367] jepsen worker 19 - jepsen.util 2359 :invoke :cas [6 [33 34]] INFO [2019-12-17 12:29:23,387] jepsen worker 3 - jepsen.util 1983 :fail :cas [1 [11 48]] :ValueNotSwapFrom INFO [2019-12-17 12:29:23,387] jepsen worker 3 - jepsen.util 1983 :invoke :read [1 nil] INFO [2019-12-17 12:29:23,522] jepsen worker 28 - jepsen.util 1468 :info :write [9 9] :RequestCanceledException INFO [2019-12-17 12:29:23,524] jepsen worker 28 - jepsen.util 1558 :invoke :write [9 3] INFO [2019-12-17 12:29:23,525] jepsen worker 30 - jepsen.util 2550 :fail :cas [10 [2 46]] :ValueNotSwapFrom INFO [2019-12-17 12:29:23,525] jepsen worker 9 - jepsen.util 2439 :fail :cas [3 [13 31]] :ValueNotSwapFrom INFO [2019-12-17 12:29:23,526] jepsen worker 30 - jepsen.util 2550 :invoke :write [10 36] INFO [2019-12-17 12:29:23,526] jepsen worker 9 - jepsen.util 2439 :invoke :read [3 nil] INFO [2019-12-17 12:29:23,529] jepsen worker 87 - jepsen.util 1797 :info :write [29 4] :RequestCanceledException INFO [2019-12-17 12:29:23,530] jepsen worker 87 - jepsen.util 1887 :invoke :write [29 4] INFO [2019-12-17 12:29:23,534] jepsen worker 19 - jepsen.util 2359 :fail :cas [6 [33 34]] :ValueNotSwapFrom INFO [2019-12-17 12:29:23,535] jepsen worker 52 - jepsen.util 2212 :fail :cas [17 [1 12]] :ValueNotSwapFrom INFO [2019-12-17 12:29:23,535] jepsen worker 19 - jepsen.util 2359 :invoke :read [6 nil] INFO [2019-12-17 12:29:23,535] jepsen worker 52 - jepsen.util 2212 :invoke :read [17 nil] INFO [2019-12-17 12:29:23,535] jepsen worker 89 - jepsen.util 2159 :fail :cas [29 [4 47]] :ValueNotSwapFrom INFO [2019-12-17 12:29:23,536] jepsen worker 89 - jepsen.util 2159 :invoke :write [29 5] INFO [2019-12-17 12:29:23,538] jepsen worker 80 - jepsen.util 2060 :fail :cas [26 [43 40]] :ValueNotSwapFrom INFO [2019-12-17 12:29:23,538] jepsen worker 80 - jepsen.util 2060 :invoke :cas [26 [8 49]] INFO [2019-12-17 12:29:23,547] jepsen worker 3 - jepsen.util 1983 :ok :read [1 28] INFO [2019-12-17 12:29:23,547] jepsen worker 3 - jepsen.util 1983 :invoke :read [1 nil] INFO [2019-12-17 12:29:23,553] jepsen worker 13 - jepsen.util 1903 :info :write [4 13] :RequestCanceledException INFO [2019-12-17 12:29:23,554] jepsen worker 13 - jepsen.util 1993 :invoke :cas [4 [28 25]] INFO [2019-12-17 12:29:23,568] jepsen worker 65 - jepsen.util 1955 :ok :write [21 18] INFO [2019-12-17 12:29:23,568] jepsen worker 65 - jepsen.util 1955 :invoke :cas [21 [41 38]] INFO [2019-12-17 12:29:23,583] jepsen worker 65 - jepsen.util 1955 :fail :cas [21 [41 38]] :ValueNotSwapFrom INFO [2019-12-17 12:29:23,583] jepsen worker 65 - jepsen.util 1955 :invoke :write [21 17] INFO [2019-12-17 12:29:23,607] jepsen worker 19 - jepsen.util 2359 :ok :read [6 25] INFO [2019-12-17 12:29:23,607] jepsen worker 19 - jepsen.util 2359 :invoke :read [6 nil] INFO [2019-12-17 12:29:23,629] jepsen worker 19 - jepsen.util 2359 :ok :read [6 25] INFO [2019-12-17 12:29:23,629] jepsen worker 19 - jepsen.util 2359 :invoke :read [6 nil] INFO [2019-12-17 12:29:23,630] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":12262,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:29:23,630] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":12262,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:29:23,630] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":12265,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:29:23,630] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":12265,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:29:23,634] jepsen worker 65 - jepsen.util 1955 :ok :write [21 17] INFO [2019-12-17 12:29:23,634] jepsen worker 65 - jepsen.util 1955 :invoke :cas [21 [29 39]] INFO [2019-12-17 12:29:23,657] jepsen worker 20 - jepsen.util 1640 :info :write [6 8] :SyncWriteAmbiguous INFO [2019-12-17 12:29:23,658] jepsen worker 20 - jepsen.util 1730 :invoke :write [6 49] INFO [2019-12-17 12:29:23,658] jepsen worker 65 - jepsen.util 1955 :fail :cas [21 [29 39]] :ValueNotSwapFrom INFO [2019-12-17 12:29:23,658] jepsen worker 65 - jepsen.util 1955 :invoke :write [21 0] INFO [2019-12-17 12:29:23,675] jepsen worker 13 - jepsen.util 1993 :fail :cas [4 [28 25]] :ValueNotSwapFrom INFO [2019-12-17 12:29:23,675] jepsen worker 13 - jepsen.util 1993 :invoke :read [4 nil] INFO [2019-12-17 12:29:23,678] jepsen worker 19 - jepsen.util 2359 :ok :read [6 25] INFO [2019-12-17 12:29:23,678] jepsen worker 19 - jepsen.util 2359 :invoke :write [6 48] INFO [2019-12-17 12:29:23,696] jepsen worker 45 - jepsen.util 2295 :ok :read [15 43] INFO [2019-12-17 12:29:23,696] jepsen worker 45 - jepsen.util 2295 :invoke :cas [15 [28 46]] INFO [2019-12-17 12:29:23,700] jepsen worker 5 - jepsen.util 1895 :fail :cas [1 [20 29]] :ValueNotSwapFrom INFO [2019-12-17 12:29:23,701] jepsen worker 5 - jepsen.util 1895 :invoke :write [1 33] INFO [2019-12-17 12:29:23,706] jepsen worker 16 - jepsen.util 2716 :info :write [5 16] :RequestCanceledException INFO [2019-12-17 12:29:23,707] jepsen worker 16 - jepsen.util 2806 :invoke :read [5 nil] INFO [2019-12-17 12:29:23,720] jepsen worker 69 - jepsen.util 2049 :fail :cas [23 [9 13]] :ValueNotSwapFrom INFO [2019-12-17 12:29:23,720] jepsen worker 72 - jepsen.util 2232 :fail :cas [24 [19 9]] :ValueNotSwapFrom INFO [2019-12-17 12:29:23,721] jepsen worker 69 - jepsen.util 2049 :invoke :cas [23 [49 21]] INFO [2019-12-17 12:29:23,721] jepsen worker 72 - jepsen.util 2232 :invoke :cas [24 [44 39]] INFO [2019-12-17 12:29:23,723] jepsen worker 43 - jepsen.util 2833 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:29:23,723] jepsen worker 15 - jepsen.util 2625 :info :write [5 45] :RequestCanceledException INFO [2019-12-17 12:29:23,723] jepsen worker 56 - jepsen.util 2036 :info :write [18 36] :RequestCanceledException INFO [2019-12-17 12:29:23,723] jepsen worker 79 - jepsen.util 2149 :info :write [26 18] :RequestCanceledException INFO [2019-12-17 12:29:23,723] jepsen worker 43 - jepsen.util 2833 :invoke :read [14 nil] INFO [2019-12-17 12:29:23,724] jepsen worker 15 - jepsen.util 2715 :invoke :read [5 nil] INFO [2019-12-17 12:29:23,724] jepsen worker 56 - jepsen.util 2126 :invoke :cas [18 [40 25]] INFO [2019-12-17 12:29:23,724] jepsen worker 79 - jepsen.util 2239 :invoke :read [26 nil] INFO [2019-12-17 12:29:23,739] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":12285,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:29:23,740] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":12285,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:29:23,767] jepsen worker 71 - jepsen.util 1871 :fail :cas [23 [29 40]] :ValueNotSwapFrom INFO [2019-12-17 12:29:23,768] jepsen worker 71 - jepsen.util 1871 :invoke :read [23 nil] INFO [2019-12-17 12:29:23,782] jepsen worker 45 - jepsen.util 2295 :fail :cas [15 [28 46]] :ValueNotSwapFrom INFO [2019-12-17 12:29:23,782] jepsen worker 45 - jepsen.util 2295 :invoke :read [15 nil] INFO [2019-12-17 12:29:23,789] jepsen worker 88 - jepsen.util 2068 :info :write [29 30] :RequestCanceledException INFO [2019-12-17 12:29:23,790] jepsen worker 88 - jepsen.util 2158 :invoke :write [29 17] INFO [2019-12-17 12:29:23,798] jepsen worker 72 - jepsen.util 2232 :fail :cas [24 [44 39]] :ValueNotSwapFrom INFO [2019-12-17 12:29:23,798] jepsen worker 72 - jepsen.util 2232 :invoke :cas [24 [21 7]] INFO [2019-12-17 12:29:23,804] jepsen worker 45 - jepsen.util 2295 :ok :read [15 43] INFO [2019-12-17 12:29:23,804] jepsen worker 45 - jepsen.util 2295 :invoke :read [15 nil] INFO [2019-12-17 12:29:23,810] jepsen worker 69 - jepsen.util 2049 :fail :cas [23 [49 21]] :ValueNotSwapFrom INFO [2019-12-17 12:29:23,810] jepsen worker 69 - jepsen.util 2049 :invoke :write [23 45] INFO [2019-12-17 12:29:23,816] jepsen worker 72 - jepsen.util 2232 :fail :cas [24 [21 7]] :ValueNotSwapFrom INFO [2019-12-17 12:29:23,816] jepsen worker 72 - jepsen.util 2232 :invoke :write [24 44] INFO [2019-12-17 12:29:23,818] jepsen worker 53 - jepsen.util 2033 :fail :cas [17 [33 17]] :ValueNotSwapFrom INFO [2019-12-17 12:29:23,819] jepsen worker 53 - jepsen.util 2033 :invoke :cas [17 [42 29]] INFO [2019-12-17 12:29:23,822] jepsen worker 63 - jepsen.util 1863 :fail :read [21 nil] :RequestTimeoutException INFO [2019-12-17 12:29:23,822] jepsen worker 63 - jepsen.util 1863 :invoke :write [21 3] INFO [2019-12-17 12:29:23,822] jepsen worker 0 - jepsen.util 3060 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:29:23,822] jepsen worker 12 - jepsen.util 2172 :info :write [4 30] :RequestCanceledException INFO [2019-12-17 12:29:23,823] jepsen worker 0 - jepsen.util 3060 :invoke :write [0 31] INFO [2019-12-17 12:29:23,823] jepsen worker 12 - jepsen.util 2262 :invoke :write [4 9] INFO [2019-12-17 12:29:23,836] jepsen worker 21 - jepsen.util 2721 :info :cas [7 [29 4]] :RequestCanceledException INFO [2019-12-17 12:29:23,837] jepsen worker 21 - jepsen.util 2811 :invoke :cas [7 [18 45]] INFO [2019-12-17 12:29:23,841] jepsen worker 51 - jepsen.util 1941 :ok :read [17 18] INFO [2019-12-17 12:29:23,842] jepsen worker 51 - jepsen.util 1941 :invoke :cas [17 [24 29]] INFO [2019-12-17 12:29:23,848] jepsen worker 45 - jepsen.util 2295 :ok :read [15 43] INFO [2019-12-17 12:29:23,849] jepsen worker 45 - jepsen.util 2295 :invoke :write [15 48] INFO [2019-12-17 12:29:23,854] jepsen worker 22 - jepsen.util 2902 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:29:23,854] jepsen worker 50 - jepsen.util 2930 :info :cas [16 [14 10]] :RequestCanceledException INFO [2019-12-17 12:29:23,854] jepsen worker 1 - jepsen.util 2701 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:29:23,854] jepsen worker 22 - jepsen.util 2902 :invoke :read [7 nil] INFO [2019-12-17 12:29:23,854] jepsen worker 1 - jepsen.util 2701 :invoke :cas [0 [43 14]] INFO [2019-12-17 12:29:23,855] jepsen worker 50 - jepsen.util 3020 :invoke :write [16 48] INFO [2019-12-17 12:29:23,867] jepsen worker 65 - jepsen.util 1955 :ok :write [21 0] INFO [2019-12-17 12:29:23,867] jepsen worker 65 - jepsen.util 1955 :invoke :cas [21 [36 32]] INFO [2019-12-17 12:29:23,883] jepsen worker 65 - jepsen.util 1955 :fail :cas [21 [36 32]] :ValueNotSwapFrom INFO [2019-12-17 12:29:23,884] jepsen worker 65 - jepsen.util 1955 :invoke :write [21 46] INFO [2019-12-17 12:29:23,889] jepsen worker 76 - jepsen.util 2686 :info :cas [25 [32 16]] :RequestCanceledException INFO [2019-12-17 12:29:23,889] jepsen worker 35 - jepsen.util 2285 :info :write [11 4] :RequestCanceledException INFO [2019-12-17 12:29:23,890] jepsen worker 76 - jepsen.util 2776 :invoke :write [25 49] INFO [2019-12-17 12:29:23,890] jepsen worker 35 - jepsen.util 2375 :invoke :cas [11 [40 12]] INFO [2019-12-17 12:29:23,896] jepsen worker 51 - jepsen.util 1941 :fail :cas [17 [24 29]] :ValueNotSwapFrom INFO [2019-12-17 12:29:23,897] jepsen worker 51 - jepsen.util 1941 :invoke :write [17 2] INFO [2019-12-17 12:29:23,903] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:29:23,903] jepsen nemesis - couchbase.util Rebalance check count: 88 INFO [2019-12-17 12:29:23,903] jepsen nemesis - couchbase.util Rebalance stuck count: 69 INFO [2019-12-17 12:29:23,907] jepsen worker 10 - jepsen.util 2170 :info :write [3 41] :RequestCanceledException INFO [2019-12-17 12:29:23,907] jepsen worker 24 - jepsen.util 2634 :info :write [8 13] :RequestCanceledException INFO [2019-12-17 12:29:23,908] jepsen worker 10 - jepsen.util 2260 :invoke :write [3 9] INFO [2019-12-17 12:29:23,908] jepsen worker 24 - jepsen.util 2724 :invoke :cas [8 [12 37]] INFO [2019-12-17 12:29:23,922] jepsen worker 81 - jepsen.util 2331 :info :write [27 29] :RequestCanceledException INFO [2019-12-17 12:29:23,922] jepsen worker 26 - jepsen.util 3086 :info :cas [8 [41 29]] :RequestCanceledException INFO [2019-12-17 12:29:23,923] jepsen worker 86 - jepsen.util 2876 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:29:23,923] jepsen worker 85 - jepsen.util 3145 :info :cas [28 [33 23]] :RequestCanceledException INFO [2019-12-17 12:29:23,923] jepsen worker 86 - jepsen.util 2876 :invoke :read [28 nil] INFO [2019-12-17 12:29:23,923] jepsen worker 81 - jepsen.util 2421 :invoke :write [27 48] INFO [2019-12-17 12:29:23,924] jepsen worker 26 - jepsen.util 3176 :invoke :write [8 47] INFO [2019-12-17 12:29:23,924] jepsen worker 85 - jepsen.util 3235 :invoke :read [28 nil] INFO [2019-12-17 12:29:23,935] jepsen worker 65 - jepsen.util 1955 :ok :write [21 46] INFO [2019-12-17 12:29:23,935] jepsen worker 65 - jepsen.util 1955 :invoke :read [21 nil] INFO [2019-12-17 12:29:23,936] jepsen worker 59 - jepsen.util 2489 :info :cas [19 [48 47]] :RequestCanceledException INFO [2019-12-17 12:29:23,936] jepsen worker 75 - jepsen.util 3045 :info :cas [25 [24 31]] :RequestCanceledException INFO [2019-12-17 12:29:23,937] jepsen worker 59 - jepsen.util 2579 :invoke :cas [19 [3 14]] INFO [2019-12-17 12:29:23,938] jepsen worker 75 - jepsen.util 3135 :invoke :cas [25 [49 40]] INFO [2019-12-17 12:29:23,949] jepsen worker 65 - jepsen.util 1955 :ok :read [21 46] INFO [2019-12-17 12:29:23,949] jepsen worker 65 - jepsen.util 1955 :invoke :read [21 nil] INFO [2019-12-17 12:29:23,953] jepsen worker 57 - jepsen.util 2487 :info :write [19 29] :RequestCanceledException INFO [2019-12-17 12:29:23,953] jepsen worker 58 - jepsen.util 2488 :info :write [19 12] :RequestCanceledException INFO [2019-12-17 12:29:23,953] jepsen worker 83 - jepsen.util 3143 :info :cas [27 [39 30]] :RequestCanceledException INFO [2019-12-17 12:29:23,954] jepsen worker 57 - jepsen.util 2577 :invoke :cas [19 [40 22]] INFO [2019-12-17 12:29:23,954] jepsen worker 83 - jepsen.util 3233 :invoke :write [27 8] INFO [2019-12-17 12:29:23,954] jepsen worker 58 - jepsen.util 2578 :invoke :read [19 nil] INFO [2019-12-17 12:29:23,955] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":12318,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:29:23,955] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":12318,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:29:23,955] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":12322,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:29:23,955] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":12322,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:29:23,962] jepsen worker 55 - jepsen.util 2395 :fail :cas [18 [7 24]] :ValueNotSwapFrom INFO [2019-12-17 12:29:23,962] jepsen worker 55 - jepsen.util 2395 :invoke :cas [18 [26 19]] INFO [2019-12-17 12:29:23,969] jepsen worker 65 - jepsen.util 1955 :ok :read [21 46] INFO [2019-12-17 12:29:23,969] jepsen worker 65 - jepsen.util 1955 :invoke :write [21 39] INFO [2019-12-17 12:29:23,982] jepsen worker 13 - jepsen.util 1993 :ok :read [4 41] INFO [2019-12-17 12:29:23,983] jepsen worker 13 - jepsen.util 1993 :invoke :write [4 3] INFO [2019-12-17 12:29:23,989] jepsen worker 84 - jepsen.util 2514 :info :write [28 14] :RequestCanceledException INFO [2019-12-17 12:29:23,989] jepsen worker 68 - jepsen.util 2498 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:29:23,989] jepsen worker 68 - jepsen.util 2498 :invoke :write [22 17] INFO [2019-12-17 12:29:23,990] jepsen worker 84 - jepsen.util 2604 :invoke :cas [28 [41 40]] INFO [2019-12-17 12:29:24,004] jepsen worker 32 - jepsen.util 2642 :ok :write [10 28] INFO [2019-12-17 12:29:24,004] jepsen worker 32 - jepsen.util 2642 :invoke :cas [10 [45 12]] INFO [2019-12-17 12:29:24,007] jepsen worker 40 - jepsen.util 3100 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:29:24,007] jepsen worker 40 - jepsen.util 3100 :invoke :cas [13 [21 9]] INFO [2019-12-17 12:29:24,015] jepsen worker 56 - jepsen.util 2126 :fail :cas [18 [40 25]] :ValueNotSwapFrom INFO [2019-12-17 12:29:24,016] jepsen worker 56 - jepsen.util 2126 :invoke :write [18 35] INFO [2019-12-17 12:29:24,022] jepsen worker 25 - jepsen.util 3175 :info :cas [8 [34 27]] :RequestCanceledException INFO [2019-12-17 12:29:24,022] jepsen worker 62 - jepsen.util 2582 :info :write [20 9] :RequestCanceledException INFO [2019-12-17 12:29:24,023] jepsen worker 47 - jepsen.util 2567 :fail :cas [15 [16 47]] :ValueNotSwapFrom INFO [2019-12-17 12:29:24,023] jepsen worker 47 - jepsen.util 2567 :invoke :cas [15 [27 28]] INFO [2019-12-17 12:29:24,023] jepsen worker 25 - jepsen.util 3265 :invoke :cas [8 [47 36]] INFO [2019-12-17 12:29:24,023] jepsen worker 62 - jepsen.util 2672 :invoke :cas [20 [5 29]] INFO [2019-12-17 12:29:24,035] jepsen worker 71 - jepsen.util 1871 :ok :read [23 33] INFO [2019-12-17 12:29:24,035] jepsen worker 71 - jepsen.util 1871 :invoke :cas [23 [22 1]] INFO [2019-12-17 12:29:24,036] jepsen worker 17 - jepsen.util 2897 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:29:24,036] jepsen worker 17 - jepsen.util 2897 :invoke :cas [5 [12 19]] INFO [2019-12-17 12:29:24,038] jepsen worker 32 - jepsen.util 2642 :fail :cas [10 [45 12]] :ValueNotSwapFrom INFO [2019-12-17 12:29:24,038] jepsen worker 32 - jepsen.util 2642 :invoke :read [10 nil] INFO [2019-12-17 12:29:24,048] jepsen worker 65 - jepsen.util 1955 :ok :write [21 39] INFO [2019-12-17 12:29:24,048] jepsen worker 65 - jepsen.util 1955 :invoke :read [21 nil] INFO [2019-12-17 12:29:24,053] jepsen worker 47 - jepsen.util 2567 :fail :cas [15 [27 28]] :ValueNotSwapFrom INFO [2019-12-17 12:29:24,053] jepsen worker 47 - jepsen.util 2567 :invoke :write [15 11] INFO [2019-12-17 12:29:24,054] jepsen worker 39 - jepsen.util 3009 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:29:24,054] jepsen worker 41 - jepsen.util 2831 :info :cas [13 [46 22]] :RequestCanceledException INFO [2019-12-17 12:29:24,054] jepsen worker 39 - jepsen.util 3009 :invoke :cas [13 [26 1]] INFO [2019-12-17 12:29:24,055] jepsen worker 41 - jepsen.util 2921 :invoke :cas [13 [22 18]] INFO [2019-12-17 12:29:24,064] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":12346,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:29:24,064] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":12346,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:29:24,064] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":12349,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:29:24,064] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":12349,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:29:24,064] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":12350,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:29:24,065] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":12350,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:29:24,068] jepsen worker 32 - jepsen.util 2642 :ok :read [10 28] INFO [2019-12-17 12:29:24,068] jepsen worker 32 - jepsen.util 2642 :invoke :read [10 nil] INFO [2019-12-17 12:29:24,069] jepsen worker 45 - jepsen.util 2295 :ok :write [15 48] INFO [2019-12-17 12:29:24,069] jepsen worker 74 - jepsen.util 2054 :ok :write [24 47] INFO [2019-12-17 12:29:24,070] jepsen worker 45 - jepsen.util 2295 :invoke :read [15 nil] INFO [2019-12-17 12:29:24,070] jepsen worker 74 - jepsen.util 2054 :invoke :write [24 15] INFO [2019-12-17 12:29:24,072] jepsen worker 65 - jepsen.util 1955 :ok :read [21 39] INFO [2019-12-17 12:29:24,072] jepsen worker 65 - jepsen.util 1955 :invoke :cas [21 [4 29]] INFO [2019-12-17 12:29:24,089] jepsen worker 42 - jepsen.util 3282 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:29:24,089] jepsen worker 42 - jepsen.util 3282 :invoke :write [14 0] INFO [2019-12-17 12:29:24,097] jepsen worker 19 - jepsen.util 2359 :ok :write [6 48] INFO [2019-12-17 12:29:24,097] jepsen worker 19 - jepsen.util 2359 :invoke :read [6 nil] INFO [2019-12-17 12:29:24,107] jepsen worker 8 - jepsen.util 2528 :info :write [2 35] :RequestCanceledException INFO [2019-12-17 12:29:24,108] jepsen worker 8 - jepsen.util 2618 :invoke :write [2 47] INFO [2019-12-17 12:29:24,115] jepsen worker 48 - jepsen.util 2298 :info :write [16 24] :RequestCanceledException INFO [2019-12-17 12:29:24,117] jepsen worker 48 - jepsen.util 2388 :invoke :read [16 nil] INFO [2019-12-17 12:29:24,119] jepsen worker 28 - jepsen.util 1558 :ok :write [9 3] INFO [2019-12-17 12:29:24,119] jepsen worker 28 - jepsen.util 1558 :invoke :write [9 44] INFO [2019-12-17 12:29:24,120] jepsen worker 45 - jepsen.util 2295 :ok :read [15 48] INFO [2019-12-17 12:29:24,120] jepsen worker 45 - jepsen.util 2295 :invoke :cas [15 [19 41]] INFO [2019-12-17 12:29:24,136] jepsen worker 27 - jepsen.util 2007 :fail :read [9 nil] :RequestTimeoutException INFO [2019-12-17 12:29:24,136] jepsen worker 46 - jepsen.util 2206 :info :write [15 18] :RequestCanceledException INFO [2019-12-17 12:29:24,136] jepsen worker 27 - jepsen.util 2007 :invoke :write [9 31] INFO [2019-12-17 12:29:24,137] jepsen worker 46 - jepsen.util 2296 :invoke :cas [15 [5 36]] INFO [2019-12-17 12:29:24,143] jepsen worker 52 - jepsen.util 2212 :ok :read [17 18] INFO [2019-12-17 12:29:24,143] jepsen worker 52 - jepsen.util 2212 :invoke :write [17 41] INFO [2019-12-17 12:29:24,154] jepsen worker 77 - jepsen.util 2687 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:29:24,154] jepsen worker 33 - jepsen.util 2913 :info :cas [11 [20 32]] :RequestCanceledException INFO [2019-12-17 12:29:24,154] jepsen worker 77 - jepsen.util 2687 :invoke :write [25 7] INFO [2019-12-17 12:29:24,155] jepsen worker 33 - jepsen.util 3003 :invoke :write [11 38] INFO [2019-12-17 12:29:24,166] jepsen worker 53 - jepsen.util 2033 :fail :cas [17 [42 29]] :ValueNotSwapFrom INFO [2019-12-17 12:29:24,166] jepsen worker 53 - jepsen.util 2033 :invoke :read [17 nil] INFO [2019-12-17 12:29:24,169] jepsen worker 65 - jepsen.util 1955 :fail :cas [21 [4 29]] :ValueNotSwapFrom INFO [2019-12-17 12:29:24,169] jepsen worker 65 - jepsen.util 1955 :invoke :write [21 45] INFO [2019-12-17 12:29:24,177] jepsen worker 45 - jepsen.util 2295 :fail :cas [15 [19 41]] :ValueNotSwapFrom INFO [2019-12-17 12:29:24,177] jepsen worker 45 - jepsen.util 2295 :invoke :write [15 11] INFO [2019-12-17 12:29:24,189] jepsen worker 66 - jepsen.util 2766 :info :cas [22 [1 40]] :RequestCanceledException INFO [2019-12-17 12:29:24,190] jepsen worker 66 - jepsen.util 2856 :invoke :read [22 nil] INFO [2019-12-17 12:29:24,207] jepsen worker 18 - jepsen.util 2268 :info :write [6 19] :RequestCanceledException INFO [2019-12-17 12:29:24,207] jepsen worker 61 - jepsen.util 2851 :info :cas [20 [13 4]] :RequestCanceledException INFO [2019-12-17 12:29:24,208] jepsen worker 18 - jepsen.util 2358 :invoke :cas [6 [0 22]] INFO [2019-12-17 12:29:24,208] jepsen worker 61 - jepsen.util 2941 :invoke :cas [20 [34 19]] INFO [2019-12-17 12:29:24,217] jepsen worker 80 - jepsen.util 2060 :fail :cas [26 [8 49]] :ValueNotSwapFrom INFO [2019-12-17 12:29:24,217] jepsen worker 80 - jepsen.util 2060 :invoke :read [26 nil] INFO [2019-12-17 12:29:24,222] jepsen worker 82 - jepsen.util 3232 :info :write [27 30] :RequestCanceledException INFO [2019-12-17 12:29:24,223] jepsen worker 82 - jepsen.util 3322 :invoke :write [27 12] INFO [2019-12-17 12:29:24,255] jepsen worker 65 - jepsen.util 1955 :ok :write [21 45] INFO [2019-12-17 12:29:24,255] jepsen worker 65 - jepsen.util 1955 :invoke :write [21 28] INFO [2019-12-17 12:29:24,283] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":12385,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:29:24,284] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":12385,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:29:24,284] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":12386,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:29:24,284] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":12394,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:29:24,284] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":12386,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:29:24,284] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":12394,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:29:24,284] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":12387,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:29:24,284] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":12397,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:29:24,284] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":12387,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:29:24,284] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":12397,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:29:24,284] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":12398,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:29:24,284] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":12398,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:29:24,306] jepsen worker 70 - jepsen.util 2410 :info :write [23 14] :RequestCanceledException INFO [2019-12-17 12:29:24,307] jepsen worker 70 - jepsen.util 2500 :invoke :read [23 nil] INFO [2019-12-17 12:29:24,314] jepsen worker 10 - jepsen.util 2260 :ok :write [3 9] INFO [2019-12-17 12:29:24,314] jepsen worker 10 - jepsen.util 2260 :invoke :cas [3 [8 30]] INFO [2019-12-17 12:29:24,320] jepsen worker 2 - jepsen.util 2702 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:29:24,320] jepsen worker 2 - jepsen.util 2702 :invoke :cas [0 [48 15]] INFO [2019-12-17 12:29:24,329] jepsen worker 65 - jepsen.util 1955 :ok :write [21 28] INFO [2019-12-17 12:29:24,329] jepsen worker 65 - jepsen.util 1955 :invoke :read [21 nil] INFO [2019-12-17 12:29:24,333] jepsen worker 19 - jepsen.util 2359 :ok :read [6 19] INFO [2019-12-17 12:29:24,333] jepsen worker 19 - jepsen.util 2359 :invoke :read [6 nil] INFO [2019-12-17 12:29:24,336] jepsen worker 29 - jepsen.util 1919 :ok :read [9 3] INFO [2019-12-17 12:29:24,337] jepsen worker 29 - jepsen.util 1919 :invoke :read [9 nil] INFO [2019-12-17 12:29:24,340] jepsen worker 38 - jepsen.util 2108 :ok :write [12 38] INFO [2019-12-17 12:29:24,340] jepsen worker 38 - jepsen.util 2108 :invoke :read [12 nil] INFO [2019-12-17 12:29:24,350] jepsen worker 74 - jepsen.util 2054 :ok :write [24 15] INFO [2019-12-17 12:29:24,350] jepsen worker 74 - jepsen.util 2054 :invoke :cas [24 [4 29]] INFO [2019-12-17 12:29:24,358] jepsen worker 47 - jepsen.util 2567 :ok :write [15 11] INFO [2019-12-17 12:29:24,358] jepsen worker 47 - jepsen.util 2567 :invoke :read [15 nil] INFO [2019-12-17 12:29:24,358] jepsen worker 10 - jepsen.util 2260 :fail :cas [3 [8 30]] :ValueNotSwapFrom INFO [2019-12-17 12:29:24,358] jepsen worker 10 - jepsen.util 2260 :invoke :read [3 nil] INFO [2019-12-17 12:29:24,375] jepsen worker 65 - jepsen.util 1955 :ok :read [21 28] INFO [2019-12-17 12:29:24,375] jepsen worker 65 - jepsen.util 1955 :invoke :read [21 nil] INFO [2019-12-17 12:29:24,387] jepsen worker 74 - jepsen.util 2054 :fail :cas [24 [4 29]] :ValueNotSwapFrom INFO [2019-12-17 12:29:24,388] jepsen worker 74 - jepsen.util 2054 :invoke :write [24 34] INFO [2019-12-17 12:29:24,394] jepsen worker 19 - jepsen.util 2359 :ok :read [6 19] INFO [2019-12-17 12:29:24,394] jepsen worker 19 - jepsen.util 2359 :invoke :write [6 21] INFO [2019-12-17 12:29:24,400] jepsen worker 55 - jepsen.util 2395 :fail :cas [18 [26 19]] :ValueNotSwapFrom INFO [2019-12-17 12:29:24,400] jepsen worker 55 - jepsen.util 2395 :invoke :write [18 15] INFO [2019-12-17 12:29:24,412] jepsen worker 10 - jepsen.util 2260 :ok :read [3 9] INFO [2019-12-17 12:29:24,412] jepsen worker 10 - jepsen.util 2260 :invoke :read [3 nil] INFO [2019-12-17 12:29:24,422] jepsen worker 44 - jepsen.util 2474 :info :write [14 38] :RequestCanceledException INFO [2019-12-17 12:29:24,422] jepsen worker 23 - jepsen.util 2633 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:29:24,422] jepsen worker 23 - jepsen.util 2633 :invoke :read [7 nil] INFO [2019-12-17 12:29:24,423] jepsen worker 44 - jepsen.util 2564 :invoke :cas [14 [15 40]] INFO [2019-12-17 12:29:24,443] jepsen worker 80 - jepsen.util 2060 :ok :read [26 20] INFO [2019-12-17 12:29:24,443] jepsen worker 80 - jepsen.util 2060 :invoke :read [26 nil] INFO [2019-12-17 12:29:24,456] jepsen worker 65 - jepsen.util 1955 :ok :read [21 28] INFO [2019-12-17 12:29:24,456] jepsen worker 65 - jepsen.util 1955 :invoke :write [21 2] INFO [2019-12-17 12:29:24,459] jepsen worker 53 - jepsen.util 2033 :ok :read [17 18] INFO [2019-12-17 12:29:24,460] jepsen worker 53 - jepsen.util 2033 :invoke :cas [17 [13 22]] INFO [2019-12-17 12:29:24,502] jepsen worker 53 - jepsen.util 2033 :fail :cas [17 [13 22]] :ValueNotSwapFrom INFO [2019-12-17 12:29:24,502] jepsen worker 53 - jepsen.util 2033 :invoke :read [17 nil] INFO [2019-12-17 12:29:24,503] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":12420,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:29:24,503] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":12420,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:29:24,503] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":12424,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:29:24,503] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":12424,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:29:24,506] jepsen worker 67 - jepsen.util 3307 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:29:24,507] jepsen worker 67 - jepsen.util 3307 :invoke :write [22 21] INFO [2019-12-17 12:29:24,509] jepsen worker 27 - jepsen.util 2007 :ok :write [9 31] INFO [2019-12-17 12:29:24,510] jepsen worker 27 - jepsen.util 2007 :invoke :cas [9 [12 24]] INFO [2019-12-17 12:29:24,522] jepsen worker 49 - jepsen.util 2119 :info :write [16 22] :RequestCanceledException INFO [2019-12-17 12:29:24,523] jepsen worker 49 - jepsen.util 2209 :invoke :read [16 nil] INFO [2019-12-17 12:29:24,534] jepsen worker 27 - jepsen.util 2007 :fail :cas [9 [12 24]] :ValueNotSwapFrom INFO [2019-12-17 12:29:24,534] jepsen worker 27 - jepsen.util 2007 :invoke :read [9 nil] INFO [2019-12-17 12:29:24,553] jepsen worker 53 - jepsen.util 2033 :ok :read [17 18] INFO [2019-12-17 12:29:24,553] jepsen worker 53 - jepsen.util 2033 :invoke :write [17 9] INFO [2019-12-17 12:29:24,566] jepsen worker 31 - jepsen.util 2101 :ok :write [10 13] INFO [2019-12-17 12:29:24,566] jepsen worker 31 - jepsen.util 2101 :invoke :read [10 nil] INFO [2019-12-17 12:29:24,570] jepsen worker 27 - jepsen.util 2007 :ok :read [9 31] INFO [2019-12-17 12:29:24,570] jepsen worker 27 - jepsen.util 2007 :invoke :cas [9 [0 14]] INFO [2019-12-17 12:29:24,586] jepsen worker 5 - jepsen.util 1895 :ok :write [1 33] INFO [2019-12-17 12:29:24,586] jepsen worker 5 - jepsen.util 1895 :invoke :cas [1 [39 18]] INFO [2019-12-17 12:29:24,595] jepsen worker 74 - jepsen.util 2054 :ok :write [24 34] INFO [2019-12-17 12:29:24,595] jepsen worker 19 - jepsen.util 2359 :ok :write [6 21] INFO [2019-12-17 12:29:24,596] jepsen worker 19 - jepsen.util 2359 :invoke :read [6 nil] INFO [2019-12-17 12:29:24,596] jepsen worker 74 - jepsen.util 2054 :invoke :cas [24 [21 3]] INFO [2019-12-17 12:29:24,600] jepsen worker 65 - jepsen.util 1955 :ok :write [21 2] INFO [2019-12-17 12:29:24,600] jepsen worker 65 - jepsen.util 1955 :invoke :write [21 6] INFO [2019-12-17 12:29:24,601] jepsen worker 38 - jepsen.util 2108 :ok :read [12 38] INFO [2019-12-17 12:29:24,602] jepsen worker 38 - jepsen.util 2108 :invoke :read [12 nil] INFO [2019-12-17 12:29:24,606] jepsen worker 29 - jepsen.util 1919 :ok :read [9 31] INFO [2019-12-17 12:29:24,606] jepsen worker 29 - jepsen.util 1919 :invoke :cas [9 [31 3]] INFO [2019-12-17 12:29:24,635] jepsen worker 3 - jepsen.util 1983 :ok :read [1 33] INFO [2019-12-17 12:29:24,636] jepsen worker 3 - jepsen.util 1983 :invoke :read [1 nil] INFO [2019-12-17 12:29:24,648] jepsen worker 19 - jepsen.util 2359 :ok :read [6 21] INFO [2019-12-17 12:29:24,648] jepsen worker 19 - jepsen.util 2359 :invoke :cas [6 [21 17]] INFO [2019-12-17 12:29:24,653] jepsen worker 34 - jepsen.util 2824 :info :cas [11 [29 48]] :RequestCanceledException INFO [2019-12-17 12:29:24,654] jepsen worker 34 - jepsen.util 2914 :invoke :write [11 21] INFO [2019-12-17 12:29:24,661] jepsen worker 74 - jepsen.util 2054 :fail :cas [24 [21 3]] :ValueNotSwapFrom INFO [2019-12-17 12:29:24,661] jepsen worker 74 - jepsen.util 2054 :invoke :cas [24 [35 29]] INFO [2019-12-17 12:29:24,662] jepsen worker 79 - jepsen.util 2239 :ok :read [26 20] INFO [2019-12-17 12:29:24,662] jepsen worker 79 - jepsen.util 2239 :invoke :read [26 nil] INFO [2019-12-17 12:29:24,678] jepsen worker 3 - jepsen.util 1983 :ok :read [1 33] INFO [2019-12-17 12:29:24,678] jepsen worker 3 - jepsen.util 1983 :invoke :read [1 nil] INFO [2019-12-17 12:29:24,698] jepsen worker 27 - jepsen.util 2007 :fail :cas [9 [0 14]] :ValueNotSwapFrom INFO [2019-12-17 12:29:24,698] jepsen worker 27 - jepsen.util 2007 :invoke :cas [9 [18 29]] INFO [2019-12-17 12:29:24,700] jepsen worker 31 - jepsen.util 2101 :ok :read [10 13] INFO [2019-12-17 12:29:24,700] jepsen worker 31 - jepsen.util 2101 :invoke :write [10 10] INFO [2019-12-17 12:29:24,706] jepsen worker 14 - jepsen.util 2084 :info :write [4 15] :RequestCanceledException INFO [2019-12-17 12:29:24,707] jepsen worker 14 - jepsen.util 2174 :invoke :cas [4 [18 6]] INFO [2019-12-17 12:29:24,722] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":12444,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:29:24,722] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":12444,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:29:24,722] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":12447,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:29:24,722] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":12447,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:29:24,722] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":12448,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:29:24,722] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":12448,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:29:24,750] jepsen worker 79 - jepsen.util 2239 :ok :read [26 20] INFO [2019-12-17 12:29:24,750] jepsen worker 79 - jepsen.util 2239 :invoke :read [26 nil] INFO [2019-12-17 12:29:24,779] jepsen worker 80 - jepsen.util 2060 :ok :read [26 20] INFO [2019-12-17 12:29:24,779] jepsen worker 80 - jepsen.util 2060 :invoke :cas [26 [11 47]] INFO [2019-12-17 12:29:24,826] jepsen worker 71 - jepsen.util 1871 :fail :cas [23 [22 1]] :ValueNotSwapFrom INFO [2019-12-17 12:29:24,826] jepsen worker 71 - jepsen.util 1871 :invoke :write [23 43] INFO [2019-12-17 12:29:24,853] jepsen worker 3 - jepsen.util 1983 :ok :read [1 33] INFO [2019-12-17 12:29:24,853] jepsen worker 78 - jepsen.util 2238 :info :write [26 46] :RequestCanceledException INFO [2019-12-17 12:29:24,854] jepsen worker 3 - jepsen.util 1983 :invoke :read [1 nil] INFO [2019-12-17 12:29:24,855] jepsen worker 78 - jepsen.util 2328 :invoke :write [26 46] INFO [2019-12-17 12:29:24,869] jepsen worker 52 - jepsen.util 2212 :ok :write [17 41] INFO [2019-12-17 12:29:24,869] jepsen worker 52 - jepsen.util 2212 :invoke :write [17 7] INFO [2019-12-17 12:29:24,889] jepsen worker 60 - jepsen.util 3030 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:29:24,889] jepsen worker 60 - jepsen.util 3030 :invoke :write [20 4] INFO [2019-12-17 12:29:24,896] jepsen worker 74 - jepsen.util 2054 :fail :cas [24 [35 29]] :ValueNotSwapFrom INFO [2019-12-17 12:29:24,897] jepsen worker 74 - jepsen.util 2054 :invoke :write [24 5] INFO [2019-12-17 12:29:24,905] jepsen worker 27 - jepsen.util 2007 :fail :cas [9 [18 29]] :ValueNotSwapFrom INFO [2019-12-17 12:29:24,905] jepsen worker 27 - jepsen.util 2007 :invoke :cas [9 [16 13]] INFO [2019-12-17 12:29:24,918] jepsen worker 70 - jepsen.util 2500 :ok :read [23 33] INFO [2019-12-17 12:29:24,918] jepsen worker 70 - jepsen.util 2500 :invoke :read [23 nil] INFO [2019-12-17 12:29:24,918] jepsen worker 4 - jepsen.util 1714 :ok :write [1 44] INFO [2019-12-17 12:29:24,918] jepsen worker 4 - jepsen.util 1714 :invoke :write [1 35] INFO [2019-12-17 12:29:24,919] jepsen worker 38 - jepsen.util 2108 :ok :read [12 38] INFO [2019-12-17 12:29:24,919] jepsen worker 38 - jepsen.util 2108 :invoke :write [12 36] INFO [2019-12-17 12:29:24,923] jepsen worker 54 - jepsen.util 2124 :info :write [18 35] :RequestCanceledException INFO [2019-12-17 12:29:24,924] jepsen worker 54 - jepsen.util 2214 :invoke :read [18 nil] INFO [2019-12-17 12:29:24,941] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":12463,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:29:24,941] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":12463,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:29:24,941] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":12466,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:29:24,941] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":12466,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:29:24,943] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:29:24,943] jepsen nemesis - couchbase.util Rebalance check count: 89 INFO [2019-12-17 12:29:24,943] jepsen nemesis - couchbase.util Rebalance stuck count: 70 INFO [2019-12-17 12:29:24,947] jepsen worker 80 - jepsen.util 2060 :fail :cas [26 [11 47]] :ValueNotSwapFrom INFO [2019-12-17 12:29:24,947] jepsen worker 80 - jepsen.util 2060 :invoke :read [26 nil] INFO [2019-12-17 12:29:24,981] jepsen worker 79 - jepsen.util 2239 :ok :read [26 20] INFO [2019-12-17 12:29:24,982] jepsen worker 79 - jepsen.util 2239 :invoke :read [26 nil] INFO [2019-12-17 12:29:24,986] jepsen worker 65 - jepsen.util 1955 :ok :write [21 6] INFO [2019-12-17 12:29:24,986] jepsen worker 65 - jepsen.util 1955 :invoke :read [21 nil] INFO [2019-12-17 12:29:24,987] jepsen worker 12 - jepsen.util 2262 :ok :write [4 9] INFO [2019-12-17 12:29:24,987] jepsen worker 12 - jepsen.util 2262 :invoke :read [4 nil] INFO [2019-12-17 12:29:24,988] jepsen worker 88 - jepsen.util 2158 :ok :write [29 17] INFO [2019-12-17 12:29:24,988] jepsen worker 88 - jepsen.util 2158 :invoke :read [29 nil] INFO [2019-12-17 12:29:24,991] jepsen worker 3 - jepsen.util 1983 :ok :read [1 44] INFO [2019-12-17 12:29:24,991] jepsen worker 3 - jepsen.util 1983 :invoke :cas [1 [3 18]] INFO [2019-12-17 12:29:25,005] jepsen worker 27 - jepsen.util 2007 :fail :cas [9 [16 13]] :ValueNotSwapFrom INFO [2019-12-17 12:29:25,005] jepsen worker 27 - jepsen.util 2007 :invoke :write [9 49] INFO [2019-12-17 12:29:25,022] jepsen worker 37 - jepsen.util 1747 :info :write [12 25] :RequestCanceledException INFO [2019-12-17 12:29:25,023] jepsen worker 37 - jepsen.util 1837 :invoke :cas [12 [39 28]] INFO [2019-12-17 12:29:25,026] jepsen worker 5 - jepsen.util 1895 :fail :cas [1 [39 18]] :ValueNotSwapFrom INFO [2019-12-17 12:29:25,026] jepsen worker 5 - jepsen.util 1895 :invoke :read [1 nil] INFO [2019-12-17 12:29:25,032] jepsen worker 79 - jepsen.util 2239 :ok :read [26 20] INFO [2019-12-17 12:29:25,032] jepsen worker 79 - jepsen.util 2239 :invoke :write [26 4] INFO [2019-12-17 12:29:25,046] jepsen worker 65 - jepsen.util 1955 :ok :read [21 6] INFO [2019-12-17 12:29:25,046] jepsen worker 65 - jepsen.util 1955 :invoke :write [21 29] INFO [2019-12-17 12:29:25,048] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":12484,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:29:25,049] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":12484,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:29:25,049] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":12488,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:29:25,049] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":12488,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:29:25,061] jepsen worker 14 - jepsen.util 2174 :fail :cas [4 [18 6]] :ValueNotSwapFrom INFO [2019-12-17 12:29:25,062] jepsen worker 14 - jepsen.util 2174 :invoke :cas [4 [42 25]] INFO [2019-12-17 12:29:25,089] jepsen worker 6 - jepsen.util 2886 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:29:25,089] jepsen worker 6 - jepsen.util 2886 :invoke :cas [2 [45 16]] INFO [2019-12-17 12:29:25,103] jepsen worker 5 - jepsen.util 1895 :ok :read [1 44] INFO [2019-12-17 12:29:25,103] jepsen worker 5 - jepsen.util 1895 :invoke :cas [1 [7 21]] INFO [2019-12-17 12:29:25,107] jepsen worker 7 - jepsen.util 2707 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:29:25,107] jepsen worker 7 - jepsen.util 2707 :invoke :cas [2 [19 2]] INFO [2019-12-17 12:29:25,111] jepsen worker 3 - jepsen.util 1983 :fail :cas [1 [3 18]] :ValueNotSwapFrom INFO [2019-12-17 12:29:25,111] jepsen worker 3 - jepsen.util 1983 :invoke :write [1 19] INFO [2019-12-17 12:29:25,152] jepsen worker 12 - jepsen.util 2262 :ok :read [4 9] INFO [2019-12-17 12:29:25,152] jepsen worker 12 - jepsen.util 2262 :invoke :write [4 7] INFO [2019-12-17 12:29:25,153] jepsen worker 52 - jepsen.util 2212 :ok :write [17 7] INFO [2019-12-17 12:29:25,153] jepsen worker 52 - jepsen.util 2212 :invoke :read [17 nil] INFO [2019-12-17 12:29:25,154] jepsen worker 56 - jepsen.util 2126 :ok :write [18 35] INFO [2019-12-17 12:29:25,154] jepsen worker 56 - jepsen.util 2126 :invoke :write [18 3] INFO [2019-12-17 12:29:25,154] jepsen worker 27 - jepsen.util 2007 :ok :write [9 49] INFO [2019-12-17 12:29:25,154] jepsen worker 27 - jepsen.util 2007 :invoke :read [9 nil] INFO [2019-12-17 12:29:25,164] jepsen worker 31 - jepsen.util 2101 :ok :write [10 10] INFO [2019-12-17 12:29:25,165] jepsen worker 31 - jepsen.util 2101 :invoke :read [10 nil] INFO [2019-12-17 12:29:25,169] jepsen worker 27 - jepsen.util 2007 :ok :read [9 49] INFO [2019-12-17 12:29:25,170] jepsen worker 27 - jepsen.util 2007 :invoke :cas [9 [26 35]] INFO [2019-12-17 12:29:25,188] jepsen worker 31 - jepsen.util 2101 :ok :read [10 10] INFO [2019-12-17 12:29:25,188] jepsen worker 31 - jepsen.util 2101 :invoke :read [10 nil] INFO [2019-12-17 12:29:25,190] jepsen worker 73 - jepsen.util 2143 :info :write [24 48] :RequestCanceledException INFO [2019-12-17 12:29:25,190] jepsen worker 36 - jepsen.util 1926 :info :write [12 22] :RequestCanceledException INFO [2019-12-17 12:29:25,191] jepsen worker 73 - jepsen.util 2233 :invoke :write [24 32] INFO [2019-12-17 12:29:25,191] jepsen worker 36 - jepsen.util 2016 :invoke :write [12 35] INFO [2019-12-17 12:29:25,191] jepsen worker 19 - jepsen.util 2359 :ok :cas [6 [21 17]] INFO [2019-12-17 12:29:25,191] jepsen worker 19 - jepsen.util 2359 :invoke :cas [6 [4 32]] INFO [2019-12-17 12:29:25,199] jepsen worker 27 - jepsen.util 2007 :fail :cas [9 [26 35]] :ValueNotSwapFrom INFO [2019-12-17 12:29:25,199] jepsen worker 27 - jepsen.util 2007 :invoke :read [9 nil] INFO [2019-12-17 12:29:25,201] jepsen worker 65 - jepsen.util 1955 :ok :write [21 29] INFO [2019-12-17 12:29:25,201] jepsen worker 65 - jepsen.util 1955 :invoke :cas [21 [26 25]] INFO [2019-12-17 12:29:25,236] jepsen worker 19 - jepsen.util 2359 :fail :cas [6 [4 32]] :ValueNotSwapFrom INFO [2019-12-17 12:29:25,236] jepsen worker 19 - jepsen.util 2359 :invoke :cas [6 [29 43]] INFO [2019-12-17 12:29:25,245] jepsen worker 80 - jepsen.util 2060 :ok :read [26 20] INFO [2019-12-17 12:29:25,245] jepsen worker 80 - jepsen.util 2060 :invoke :cas [26 [31 48]] INFO [2019-12-17 12:29:25,254] jepsen worker 64 - jepsen.util 1774 :info :cas [21 [21 46]] :RequestCanceledException INFO [2019-12-17 12:29:25,255] jepsen worker 64 - jepsen.util 1864 :invoke :cas [21 [18 41]] INFO [2019-12-17 12:29:25,267] jepsen worker 31 - jepsen.util 2101 :ok :read [10 10] INFO [2019-12-17 12:29:25,267] jepsen worker 31 - jepsen.util 2101 :invoke :read [10 nil] INFO [2019-12-17 12:29:25,268] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":12518,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:29:25,268] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":12518,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:29:25,268] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":12519,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:29:25,268] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":12508,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:29:25,268] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":12519,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:29:25,268] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":12508,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:29:25,289] jepsen worker 11 - jepsen.util 2621 :info :cas [3 [9 36]] :RequestCanceledException INFO [2019-12-17 12:29:25,290] jepsen worker 11 - jepsen.util 2711 :invoke :read [3 nil] INFO [2019-12-17 12:29:25,319] jepsen worker 14 - jepsen.util 2174 :fail :cas [4 [42 25]] :ValueNotSwapFrom INFO [2019-12-17 12:29:25,319] jepsen worker 14 - jepsen.util 2174 :invoke :cas [4 [45 16]] INFO [2019-12-17 12:29:25,341] jepsen worker 27 - jepsen.util 2007 :ok :read [9 49] INFO [2019-12-17 12:29:25,341] jepsen worker 27 - jepsen.util 2007 :invoke :read [9 nil] INFO [2019-12-17 12:29:25,342] jepsen worker 88 - jepsen.util 2158 :ok :read [29 17] INFO [2019-12-17 12:29:25,342] jepsen worker 88 - jepsen.util 2158 :invoke :read [29 nil] INFO [2019-12-17 12:29:25,343] jepsen worker 52 - jepsen.util 2212 :ok :read [17 7] INFO [2019-12-17 12:29:25,343] jepsen worker 52 - jepsen.util 2212 :invoke :read [17 nil] INFO [2019-12-17 12:29:25,363] jepsen worker 65 - jepsen.util 1955 :fail :cas [21 [26 25]] :ValueNotSwapFrom INFO [2019-12-17 12:29:25,363] jepsen worker 65 - jepsen.util 1955 :invoke :write [21 31] INFO [2019-12-17 12:29:25,391] jepsen worker 64 - jepsen.util 1864 :fail :cas [21 [18 41]] :ValueNotSwapFrom INFO [2019-12-17 12:29:25,391] jepsen worker 64 - jepsen.util 1864 :invoke :read [21 nil] INFO [2019-12-17 12:29:25,393] jepsen worker 31 - jepsen.util 2101 :ok :read [10 10] INFO [2019-12-17 12:29:25,394] jepsen worker 31 - jepsen.util 2101 :invoke :write [10 23] INFO [2019-12-17 12:29:25,396] jepsen worker 14 - jepsen.util 2174 :fail :cas [4 [45 16]] :ValueNotSwapFrom INFO [2019-12-17 12:29:25,397] jepsen worker 14 - jepsen.util 2174 :invoke :cas [4 [42 32]] INFO [2019-12-17 12:29:25,412] jepsen worker 88 - jepsen.util 2158 :ok :read [29 17] INFO [2019-12-17 12:29:25,412] jepsen worker 88 - jepsen.util 2158 :invoke :read [29 nil] INFO [2019-12-17 12:29:25,414] jepsen worker 80 - jepsen.util 2060 :fail :cas [26 [31 48]] :ValueNotSwapFrom INFO [2019-12-17 12:29:25,414] jepsen worker 80 - jepsen.util 2060 :invoke :write [26 15] INFO [2019-12-17 12:29:25,428] jepsen worker 11 - jepsen.util 2711 :ok :read [3 9] INFO [2019-12-17 12:29:25,428] jepsen worker 11 - jepsen.util 2711 :invoke :cas [3 [26 4]] INFO [2019-12-17 12:29:25,438] jepsen worker 64 - jepsen.util 1864 :ok :read [21 29] INFO [2019-12-17 12:29:25,438] jepsen worker 64 - jepsen.util 1864 :invoke :cas [21 [24 33]] INFO [2019-12-17 12:29:25,460] jepsen worker 64 - jepsen.util 1864 :fail :cas [21 [24 33]] :ValueNotSwapFrom INFO [2019-12-17 12:29:25,461] jepsen worker 64 - jepsen.util 1864 :invoke :cas [21 [18 8]] INFO [2019-12-17 12:29:25,472] jepsen worker 27 - jepsen.util 2007 :ok :read [9 49] INFO [2019-12-17 12:29:25,472] jepsen worker 27 - jepsen.util 2007 :invoke :cas [9 [9 28]] INFO [2019-12-17 12:29:25,475] jepsen worker 52 - jepsen.util 2212 :ok :read [17 7] INFO [2019-12-17 12:29:25,476] jepsen worker 52 - jepsen.util 2212 :invoke :write [17 10] INFO [2019-12-17 12:29:25,478] jepsen worker 88 - jepsen.util 2158 :ok :read [29 17] INFO [2019-12-17 12:29:25,479] jepsen worker 88 - jepsen.util 2158 :invoke :write [29 17] INFO [2019-12-17 12:29:25,487] jepsen worker 64 - jepsen.util 1864 :fail :cas [21 [18 8]] :ValueNotSwapFrom INFO [2019-12-17 12:29:25,487] jepsen worker 64 - jepsen.util 1864 :invoke :read [21 nil] INFO [2019-12-17 12:29:25,488] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":12542,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:29:25,488] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":12542,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:29:25,488] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":12544,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:29:25,488] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":12544,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:29:25,509] jepsen worker 14 - jepsen.util 2174 :fail :cas [4 [42 32]] :ValueNotSwapFrom INFO [2019-12-17 12:29:25,509] jepsen worker 14 - jepsen.util 2174 :invoke :read [4 nil] INFO [2019-12-17 12:29:25,517] jepsen worker 5 - jepsen.util 1895 :fail :cas [1 [7 21]] :ValueNotSwapFrom INFO [2019-12-17 12:29:25,517] jepsen worker 5 - jepsen.util 1895 :invoke :write [1 26] INFO [2019-12-17 12:29:25,526] jepsen worker 64 - jepsen.util 1864 :ok :read [21 29] INFO [2019-12-17 12:29:25,526] jepsen worker 64 - jepsen.util 1864 :invoke :read [21 nil] INFO [2019-12-17 12:29:25,545] jepsen worker 37 - jepsen.util 1837 :fail :cas [12 [39 28]] :ValueNotSwapFrom INFO [2019-12-17 12:29:25,545] jepsen worker 37 - jepsen.util 1837 :invoke :write [12 15] INFO [2019-12-17 12:29:25,551] jepsen worker 89 - jepsen.util 2159 :info :write [29 5] :RequestCanceledException INFO [2019-12-17 12:29:25,552] jepsen worker 89 - jepsen.util 2249 :invoke :cas [29 [1 23]] INFO [2019-12-17 12:29:25,555] jepsen worker 70 - jepsen.util 2500 :ok :read [23 33] INFO [2019-12-17 12:29:25,555] jepsen worker 70 - jepsen.util 2500 :invoke :cas [23 [38 44]] INFO [2019-12-17 12:29:25,566] jepsen worker 64 - jepsen.util 1864 :ok :read [21 29] INFO [2019-12-17 12:29:25,566] jepsen worker 64 - jepsen.util 1864 :invoke :read [21 nil] INFO [2019-12-17 12:29:25,573] jepsen worker 11 - jepsen.util 2711 :fail :cas [3 [26 4]] :ValueNotSwapFrom INFO [2019-12-17 12:29:25,573] jepsen worker 11 - jepsen.util 2711 :invoke :cas [3 [2 19]] INFO [2019-12-17 12:29:25,576] jepsen worker 54 - jepsen.util 2214 :ok :read [18 35] INFO [2019-12-17 12:29:25,576] jepsen worker 54 - jepsen.util 2214 :invoke :cas [18 [41 49]] INFO [2019-12-17 12:29:25,582] jepsen worker 74 - jepsen.util 2054 :ok :write [24 5] INFO [2019-12-17 12:29:25,582] jepsen worker 74 - jepsen.util 2054 :invoke :cas [24 [18 25]] INFO [2019-12-17 12:29:25,588] jepsen worker 30 - jepsen.util 2550 :info :write [10 36] :RequestCanceledException INFO [2019-12-17 12:29:25,589] jepsen worker 30 - jepsen.util 2640 :invoke :write [10 40] INFO [2019-12-17 12:29:25,620] jepsen worker 87 - jepsen.util 1887 :info :write [29 4] :RequestCanceledException INFO [2019-12-17 12:29:25,620] jepsen worker 9 - jepsen.util 2439 :fail :read [3 nil] :RequestTimeoutException INFO [2019-12-17 12:29:25,620] jepsen worker 9 - jepsen.util 2439 :invoke :write [3 30] INFO [2019-12-17 12:29:25,621] jepsen worker 87 - jepsen.util 1977 :invoke :cas [29 [25 11]] INFO [2019-12-17 12:29:25,643] jepsen worker 64 - jepsen.util 1864 :ok :read [21 29] INFO [2019-12-17 12:29:25,643] jepsen worker 64 - jepsen.util 1864 :invoke :read [21 nil] INFO [2019-12-17 12:29:25,653] jepsen worker 27 - jepsen.util 2007 :fail :cas [9 [9 28]] :ValueNotSwapFrom INFO [2019-12-17 12:29:25,653] jepsen worker 27 - jepsen.util 2007 :invoke :read [9 nil] INFO [2019-12-17 12:29:25,707] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":12567,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:29:25,707] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":12567,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:29:25,707] jepsen worker 20 - jepsen.util 1730 :info :write [6 49] :RequestCanceledException INFO [2019-12-17 12:29:25,708] jepsen worker 20 - jepsen.util 1820 :invoke :read [6 nil] INFO [2019-12-17 12:29:25,708] jepsen worker 89 - jepsen.util 2249 :fail :cas [29 [1 23]] :ValueNotSwapFrom INFO [2019-12-17 12:29:25,709] jepsen worker 89 - jepsen.util 2249 :invoke :write [29 34] INFO [2019-12-17 12:29:25,723] jepsen worker 16 - jepsen.util 2806 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:29:25,723] jepsen worker 16 - jepsen.util 2806 :invoke :cas [5 [18 22]] INFO [2019-12-17 12:29:25,734] jepsen worker 11 - jepsen.util 2711 :fail :cas [3 [2 19]] :ValueNotSwapFrom INFO [2019-12-17 12:29:25,734] jepsen worker 11 - jepsen.util 2711 :invoke :cas [3 [34 4]] INFO [2019-12-17 12:29:25,736] jepsen worker 64 - jepsen.util 1864 :ok :read [21 29] INFO [2019-12-17 12:29:25,736] jepsen worker 64 - jepsen.util 1864 :invoke :write [21 11] INFO [2019-12-17 12:29:25,737] jepsen worker 15 - jepsen.util 2715 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:29:25,737] jepsen worker 15 - jepsen.util 2715 :invoke :read [5 nil] INFO [2019-12-17 12:29:25,751] jepsen worker 74 - jepsen.util 2054 :fail :cas [24 [18 25]] :ValueNotSwapFrom INFO [2019-12-17 12:29:25,751] jepsen worker 74 - jepsen.util 2054 :invoke :read [24 nil] INFO [2019-12-17 12:29:25,766] jepsen worker 27 - jepsen.util 2007 :ok :read [9 49] INFO [2019-12-17 12:29:25,766] jepsen worker 27 - jepsen.util 2007 :invoke :read [9 nil] INFO [2019-12-17 12:29:25,781] jepsen worker 70 - jepsen.util 2500 :fail :cas [23 [38 44]] :ValueNotSwapFrom INFO [2019-12-17 12:29:25,781] jepsen worker 70 - jepsen.util 2500 :invoke :write [23 46] INFO [2019-12-17 12:29:25,789] jepsen worker 65 - jepsen.util 1955 :ok :write [21 31] INFO [2019-12-17 12:29:25,789] jepsen worker 65 - jepsen.util 1955 :invoke :write [21 0] INFO [2019-12-17 12:29:25,798] jepsen worker 54 - jepsen.util 2214 :fail :cas [18 [41 49]] :ValueNotSwapFrom INFO [2019-12-17 12:29:25,798] jepsen worker 54 - jepsen.util 2214 :invoke :write [18 23] INFO [2019-12-17 12:29:25,810] jepsen worker 11 - jepsen.util 2711 :fail :cas [3 [34 4]] :ValueNotSwapFrom INFO [2019-12-17 12:29:25,810] jepsen worker 11 - jepsen.util 2711 :invoke :read [3 nil] INFO [2019-12-17 12:29:25,822] jepsen worker 43 - jepsen.util 2833 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:29:25,822] jepsen worker 43 - jepsen.util 2833 :invoke :read [14 nil] INFO [2019-12-17 12:29:25,825] jepsen worker 27 - jepsen.util 2007 :ok :read [9 49] INFO [2019-12-17 12:29:25,825] jepsen worker 27 - jepsen.util 2007 :invoke :write [9 35] INFO [2019-12-17 12:29:25,839] jepsen worker 74 - jepsen.util 2054 :ok :read [24 5] INFO [2019-12-17 12:29:25,839] jepsen worker 74 - jepsen.util 2054 :invoke :cas [24 [15 9]] INFO [2019-12-17 12:29:25,851] jepsen worker 11 - jepsen.util 2711 :ok :read [3 9] INFO [2019-12-17 12:29:25,851] jepsen worker 11 - jepsen.util 2711 :invoke :write [3 8] INFO [2019-12-17 12:29:25,886] jepsen worker 74 - jepsen.util 2054 :fail :cas [24 [15 9]] :ValueNotSwapFrom INFO [2019-12-17 12:29:25,887] jepsen worker 74 - jepsen.util 2054 :invoke :write [24 44] INFO [2019-12-17 12:29:25,890] jepsen worker 69 - jepsen.util 2049 :info :write [23 45] :RequestCanceledException INFO [2019-12-17 12:29:25,890] jepsen worker 69 - jepsen.util 2139 :invoke :read [23 nil] INFO [2019-12-17 12:29:25,923] jepsen worker 0 - jepsen.util 3060 :info :write [0 31] :RequestCanceledException INFO [2019-12-17 12:29:25,923] jepsen worker 72 - jepsen.util 2232 :info :write [24 44] :RequestCanceledException INFO [2019-12-17 12:29:25,923] jepsen worker 35 - jepsen.util 2375 :info :cas [11 [40 12]] :RequestCanceledException INFO [2019-12-17 12:29:25,923] jepsen worker 63 - jepsen.util 1863 :info :write [21 3] :RequestCanceledException INFO [2019-12-17 12:29:25,923] jepsen worker 21 - jepsen.util 2811 :info :cas [7 [18 45]] :RequestCanceledException INFO [2019-12-17 12:29:25,924] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":12590,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:29:25,924] jepsen worker 0 - jepsen.util 3150 :invoke :read [0 nil] INFO [2019-12-17 12:29:25,924] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":12590,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:29:25,924] jepsen worker 72 - jepsen.util 2322 :invoke :read [24 nil] INFO [2019-12-17 12:29:25,924] jepsen worker 63 - jepsen.util 1953 :invoke :write [21 21] INFO [2019-12-17 12:29:25,924] jepsen worker 21 - jepsen.util 2901 :invoke :write [7 0] INFO [2019-12-17 12:29:25,924] jepsen worker 35 - jepsen.util 2465 :invoke :write [11 41] INFO [2019-12-17 12:29:25,924] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":12594,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:29:25,924] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":12594,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:29:25,935] jepsen worker 87 - jepsen.util 1977 :fail :cas [29 [25 11]] :ValueNotSwapFrom INFO [2019-12-17 12:29:25,935] jepsen worker 87 - jepsen.util 1977 :invoke :cas [29 [5 1]] INFO [2019-12-17 12:29:25,937] jepsen worker 50 - jepsen.util 3020 :info :write [16 48] :RequestCanceledException INFO [2019-12-17 12:29:25,937] jepsen worker 51 - jepsen.util 1941 :info :write [17 2] :RequestCanceledException INFO [2019-12-17 12:29:25,938] jepsen worker 51 - jepsen.util 2031 :invoke :write [17 12] INFO [2019-12-17 12:29:25,938] jepsen worker 50 - jepsen.util 3110 :invoke :write [16 21] INFO [2019-12-17 12:29:25,954] jepsen worker 22 - jepsen.util 2902 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:29:25,954] jepsen worker 76 - jepsen.util 2776 :info :write [25 49] :RequestCanceledException INFO [2019-12-17 12:29:25,954] jepsen worker 1 - jepsen.util 2701 :info :cas [0 [43 14]] :RequestCanceledException INFO [2019-12-17 12:29:25,954] jepsen worker 85 - jepsen.util 3235 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:29:25,954] jepsen worker 22 - jepsen.util 2902 :invoke :read [7 nil] INFO [2019-12-17 12:29:25,954] jepsen worker 85 - jepsen.util 3235 :invoke :read [28 nil] INFO [2019-12-17 12:29:25,955] jepsen worker 76 - jepsen.util 2866 :invoke :cas [25 [18 28]] INFO [2019-12-17 12:29:25,955] jepsen worker 1 - jepsen.util 2791 :invoke :read [0 nil] INFO [2019-12-17 12:29:25,961] jepsen worker 64 - jepsen.util 1864 :ok :write [21 11] INFO [2019-12-17 12:29:25,962] jepsen worker 64 - jepsen.util 1864 :invoke :write [21 25] INFO [2019-12-17 12:29:25,978] jepsen worker 27 - jepsen.util 2007 :ok :write [9 35] INFO [2019-12-17 12:29:25,978] jepsen worker 27 - jepsen.util 2007 :invoke :read [9 nil] INFO [2019-12-17 12:29:25,989] jepsen worker 75 - jepsen.util 3135 :info :cas [25 [49 40]] :RequestCanceledException INFO [2019-12-17 12:29:25,990] jepsen worker 75 - jepsen.util 3225 :invoke :cas [25 [45 1]] INFO [2019-12-17 12:29:26,007] jepsen worker 57 - jepsen.util 2577 :info :cas [19 [40 22]] :RequestCanceledException INFO [2019-12-17 12:29:26,007] jepsen worker 24 - jepsen.util 2724 :info :cas [8 [12 37]] :RequestCanceledException INFO [2019-12-17 12:29:26,009] jepsen worker 24 - jepsen.util 2814 :invoke :write [8 9] INFO [2019-12-17 12:29:26,009] jepsen worker 57 - jepsen.util 2667 :invoke :cas [19 [29 44]] INFO [2019-12-17 12:29:26,017] jepsen worker 14 - jepsen.util 2174 :ok :read [4 9] INFO [2019-12-17 12:29:26,017] jepsen worker 14 - jepsen.util 2174 :invoke :write [4 9] INFO [2019-12-17 12:29:26,021] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:29:26,021] jepsen nemesis - couchbase.util Rebalance check count: 90 INFO [2019-12-17 12:29:26,021] jepsen nemesis - couchbase.util Rebalance stuck count: 71 INFO [2019-12-17 12:29:26,022] jepsen worker 31 - jepsen.util 2101 :ok :write [10 23] INFO [2019-12-17 12:29:26,022] jepsen worker 31 - jepsen.util 2101 :invoke :cas [10 [8 23]] INFO [2019-12-17 12:29:26,022] jepsen worker 86 - jepsen.util 2876 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:29:26,022] jepsen worker 59 - jepsen.util 2579 :info :cas [19 [3 14]] :RequestCanceledException INFO [2019-12-17 12:29:26,022] jepsen worker 86 - jepsen.util 2876 :invoke :read [28 nil] INFO [2019-12-17 12:29:26,022] jepsen worker 27 - jepsen.util 2007 :ok :read [9 35] INFO [2019-12-17 12:29:26,022] jepsen worker 81 - jepsen.util 2421 :info :write [27 48] :RequestCanceledException INFO [2019-12-17 12:29:26,022] jepsen worker 83 - jepsen.util 3233 :info :write [27 8] :RequestCanceledException INFO [2019-12-17 12:29:26,022] jepsen worker 27 - jepsen.util 2007 :invoke :write [9 33] INFO [2019-12-17 12:29:26,023] jepsen worker 59 - jepsen.util 2669 :invoke :cas [19 [19 49]] INFO [2019-12-17 12:29:26,023] jepsen worker 81 - jepsen.util 2511 :invoke :write [27 16] INFO [2019-12-17 12:29:26,023] jepsen worker 83 - jepsen.util 3323 :invoke :cas [27 [41 40]] INFO [2019-12-17 12:29:26,033] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":12615,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:29:26,034] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":12615,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:29:26,034] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":12617,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:29:26,034] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":12617,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:29:26,036] jepsen worker 58 - jepsen.util 2578 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:29:26,036] jepsen worker 26 - jepsen.util 3176 :info :write [8 47] :RequestCanceledException INFO [2019-12-17 12:29:26,037] jepsen worker 58 - jepsen.util 2578 :invoke :write [19 48] INFO [2019-12-17 12:29:26,037] jepsen worker 26 - jepsen.util 3266 :invoke :read [8 nil] INFO [2019-12-17 12:29:26,053] jepsen worker 84 - jepsen.util 2604 :info :cas [28 [41 40]] :RequestCanceledException INFO [2019-12-17 12:29:26,054] jepsen worker 84 - jepsen.util 2694 :invoke :write [28 5] INFO [2019-12-17 12:29:26,062] jepsen worker 20 - jepsen.util 1820 :ok :read [6 17] INFO [2019-12-17 12:29:26,062] jepsen worker 20 - jepsen.util 1820 :invoke :cas [6 [21 32]] INFO [2019-12-17 12:29:26,069] jepsen worker 74 - jepsen.util 2054 :ok :write [24 44] INFO [2019-12-17 12:29:26,069] jepsen worker 74 - jepsen.util 2054 :invoke :read [24 nil] INFO [2019-12-17 12:29:26,085] jepsen worker 31 - jepsen.util 2101 :fail :cas [10 [8 23]] :ValueNotSwapFrom INFO [2019-12-17 12:29:26,085] jepsen worker 31 - jepsen.util 2101 :invoke :cas [10 [3 49]] INFO [2019-12-17 12:29:26,087] jepsen worker 62 - jepsen.util 2672 :info :cas [20 [5 29]] :RequestCanceledException INFO [2019-12-17 12:29:26,087] jepsen worker 68 - jepsen.util 2498 :info :write [22 17] :RequestCanceledException INFO [2019-12-17 12:29:26,087] jepsen worker 13 - jepsen.util 1993 :info :write [4 3] :RequestCanceledException INFO [2019-12-17 12:29:26,088] jepsen worker 62 - jepsen.util 2762 :invoke :read [20 nil] INFO [2019-12-17 12:29:26,088] jepsen worker 68 - jepsen.util 2588 :invoke :write [22 45] INFO [2019-12-17 12:29:26,088] jepsen worker 13 - jepsen.util 2083 :invoke :cas [4 [22 7]] INFO [2019-12-17 12:29:26,089] jepsen worker 63 - jepsen.util 1953 :ok :write [21 21] INFO [2019-12-17 12:29:26,089] jepsen worker 63 - jepsen.util 1953 :invoke :cas [21 [4 6]] INFO [2019-12-17 12:29:26,095] jepsen worker 72 - jepsen.util 2322 :ok :read [24 44] INFO [2019-12-17 12:29:26,095] jepsen worker 72 - jepsen.util 2322 :invoke :write [24 29] INFO [2019-12-17 12:29:26,107] jepsen worker 40 - jepsen.util 3100 :info :cas [13 [21 9]] :RequestCanceledException INFO [2019-12-17 12:29:26,107] jepsen worker 41 - jepsen.util 2921 :info :cas [13 [22 18]] :RequestCanceledException INFO [2019-12-17 12:29:26,108] jepsen worker 40 - jepsen.util 3190 :invoke :cas [13 [37 6]] INFO [2019-12-17 12:29:26,108] jepsen worker 41 - jepsen.util 3011 :invoke :write [13 28] INFO [2019-12-17 12:29:26,109] jepsen worker 74 - jepsen.util 2054 :ok :read [24 44] INFO [2019-12-17 12:29:26,109] jepsen worker 74 - jepsen.util 2054 :invoke :write [24 13] INFO [2019-12-17 12:29:26,123] jepsen worker 25 - jepsen.util 3265 :info :cas [8 [47 36]] :RequestCanceledException INFO [2019-12-17 12:29:26,123] jepsen worker 25 - jepsen.util 3355 :invoke :write [8 46] INFO [2019-12-17 12:29:26,131] jepsen worker 20 - jepsen.util 1820 :fail :cas [6 [21 32]] :ValueNotSwapFrom INFO [2019-12-17 12:29:26,131] jepsen worker 20 - jepsen.util 1820 :invoke :cas [6 [0 49]] INFO [2019-12-17 12:29:26,133] jepsen worker 64 - jepsen.util 1864 :ok :write [21 25] INFO [2019-12-17 12:29:26,133] jepsen worker 64 - jepsen.util 1864 :invoke :write [21 8] INFO [2019-12-17 12:29:26,136] jepsen worker 17 - jepsen.util 2897 :info :cas [5 [12 19]] :RequestCanceledException INFO [2019-12-17 12:29:26,136] jepsen worker 32 - jepsen.util 2642 :fail :read [10 nil] :RequestTimeoutException INFO [2019-12-17 12:29:26,136] jepsen worker 48 - jepsen.util 2388 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:29:26,136] jepsen worker 48 - jepsen.util 2388 :invoke :write [16 11] INFO [2019-12-17 12:29:26,136] jepsen worker 32 - jepsen.util 2642 :invoke :write [10 36] INFO [2019-12-17 12:29:26,137] jepsen worker 17 - jepsen.util 2987 :invoke :read [5 nil] INFO [2019-12-17 12:29:26,154] jepsen worker 39 - jepsen.util 3009 :info :cas [13 [26 1]] :RequestCanceledException INFO [2019-12-17 12:29:26,154] jepsen worker 46 - jepsen.util 2296 :info :cas [15 [5 36]] :RequestCanceledException INFO [2019-12-17 12:29:26,154] jepsen worker 28 - jepsen.util 1558 :info :write [9 44] :RequestCanceledException INFO [2019-12-17 12:29:26,155] jepsen worker 46 - jepsen.util 2386 :invoke :cas [15 [21 39]] INFO [2019-12-17 12:29:26,155] jepsen worker 39 - jepsen.util 3099 :invoke :cas [13 [43 19]] INFO [2019-12-17 12:29:26,155] jepsen worker 28 - jepsen.util 1648 :invoke :cas [9 [46 30]] INFO [2019-12-17 12:29:26,161] jepsen worker 87 - jepsen.util 1977 :fail :cas [29 [5 1]] :ValueNotSwapFrom INFO [2019-12-17 12:29:26,161] jepsen worker 87 - jepsen.util 1977 :invoke :cas [29 [28 47]] INFO [2019-12-17 12:29:26,164] jepsen worker 63 - jepsen.util 1953 :fail :cas [21 [4 6]] :ValueNotSwapFrom INFO [2019-12-17 12:29:26,164] jepsen worker 63 - jepsen.util 1953 :invoke :read [21 nil] INFO [2019-12-17 12:29:26,188] jepsen worker 27 - jepsen.util 2007 :ok :write [9 33] INFO [2019-12-17 12:29:26,188] jepsen worker 27 - jepsen.util 2007 :invoke :read [9 nil] INFO [2019-12-17 12:29:26,189] jepsen worker 31 - jepsen.util 2101 :fail :cas [10 [3 49]] :ValueNotSwapFrom INFO [2019-12-17 12:29:26,189] jepsen worker 31 - jepsen.util 2101 :invoke :write [10 24] INFO [2019-12-17 12:29:26,189] jepsen worker 42 - jepsen.util 3282 :info :write [14 0] :RequestCanceledException INFO [2019-12-17 12:29:26,190] jepsen worker 42 - jepsen.util 3372 :invoke :cas [14 [4 4]] INFO [2019-12-17 12:29:26,195] jepsen worker 28 - jepsen.util 1648 :fail :cas [9 [46 30]] :ValueNotSwapFrom INFO [2019-12-17 12:29:26,196] jepsen worker 28 - jepsen.util 1648 :invoke :read [9 nil] INFO [2019-12-17 12:29:26,213] jepsen worker 20 - jepsen.util 1820 :fail :cas [6 [0 49]] :ValueNotSwapFrom INFO [2019-12-17 12:29:26,213] jepsen worker 20 - jepsen.util 1820 :invoke :write [6 15] INFO [2019-12-17 12:29:26,217] jepsen worker 63 - jepsen.util 1953 :ok :read [21 25] INFO [2019-12-17 12:29:26,217] jepsen worker 63 - jepsen.util 1953 :invoke :write [21 23] INFO [2019-12-17 12:29:26,220] jepsen worker 8 - jepsen.util 2618 :info :write [2 47] :RequestCanceledException INFO [2019-12-17 12:29:26,220] jepsen worker 33 - jepsen.util 3003 :info :write [11 38] :RequestCanceledException INFO [2019-12-17 12:29:26,220] jepsen worker 45 - jepsen.util 2295 :info :write [15 11] :RequestCanceledException INFO [2019-12-17 12:29:26,220] jepsen worker 61 - jepsen.util 2941 :info :cas [20 [34 19]] :RequestCanceledException INFO [2019-12-17 12:29:26,221] jepsen worker 8 - jepsen.util 2708 :invoke :read [2 nil] INFO [2019-12-17 12:29:26,222] jepsen worker 33 - jepsen.util 3093 :invoke :cas [11 [10 4]] INFO [2019-12-17 12:29:26,222] jepsen worker 45 - jepsen.util 2385 :invoke :read [15 nil] INFO [2019-12-17 12:29:26,223] jepsen worker 61 - jepsen.util 3031 :invoke :cas [20 [6 42]] INFO [2019-12-17 12:29:26,252] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":12654,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:29:26,252] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":12662,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:29:26,252] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":12654,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:29:26,252] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":12662,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:29:26,252] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":12655,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:29:26,252] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":12666,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:29:26,252] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":12655,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:29:26,253] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":12666,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:29:26,253] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":12656,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:29:26,253] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":12667,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:29:26,253] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":12656,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:29:26,253] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":12667,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:29:26,254] jepsen worker 77 - jepsen.util 2687 :info :write [25 7] :RequestCanceledException INFO [2019-12-17 12:29:26,255] jepsen worker 77 - jepsen.util 2777 :invoke :write [25 38] INFO [2019-12-17 12:29:26,276] jepsen worker 72 - jepsen.util 2322 :ok :write [24 29] INFO [2019-12-17 12:29:26,276] jepsen worker 72 - jepsen.util 2322 :invoke :read [24 nil] INFO [2019-12-17 12:29:26,282] jepsen worker 69 - jepsen.util 2139 :ok :read [23 33] INFO [2019-12-17 12:29:26,282] jepsen worker 69 - jepsen.util 2139 :invoke :write [23 46] INFO [2019-12-17 12:29:26,285] jepsen worker 28 - jepsen.util 1648 :ok :read [9 33] INFO [2019-12-17 12:29:26,286] jepsen worker 28 - jepsen.util 1648 :invoke :read [9 nil] INFO [2019-12-17 12:29:26,289] jepsen worker 66 - jepsen.util 2856 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:29:26,289] jepsen worker 66 - jepsen.util 2856 :invoke :cas [22 [6 23]] INFO [2019-12-17 12:29:26,307] jepsen worker 18 - jepsen.util 2358 :info :cas [6 [0 22]] :RequestCanceledException INFO [2019-12-17 12:29:26,308] jepsen worker 18 - jepsen.util 2448 :invoke :write [6 9] INFO [2019-12-17 12:29:26,314] jepsen worker 72 - jepsen.util 2322 :ok :read [24 29] INFO [2019-12-17 12:29:26,314] jepsen worker 72 - jepsen.util 2322 :invoke :write [24 30] INFO [2019-12-17 12:29:26,337] jepsen worker 82 - jepsen.util 3322 :info :write [27 12] :RequestCanceledException INFO [2019-12-17 12:29:26,338] jepsen worker 82 - jepsen.util 3412 :invoke :write [27 17] INFO [2019-12-17 12:29:26,346] jepsen worker 87 - jepsen.util 1977 :fail :cas [29 [28 47]] :ValueNotSwapFrom INFO [2019-12-17 12:29:26,346] jepsen worker 87 - jepsen.util 1977 :invoke :cas [29 [24 9]] INFO [2019-12-17 12:29:26,354] jepsen worker 64 - jepsen.util 1864 :ok :write [21 8] INFO [2019-12-17 12:29:26,354] jepsen worker 64 - jepsen.util 1864 :invoke :cas [21 [9 10]] INFO [2019-12-17 12:29:26,391] jepsen worker 64 - jepsen.util 1864 :fail :cas [21 [9 10]] :ValueNotSwapFrom INFO [2019-12-17 12:29:26,391] jepsen worker 64 - jepsen.util 1864 :invoke :read [21 nil] INFO [2019-12-17 12:29:26,409] jepsen worker 54 - jepsen.util 2214 :ok :write [18 23] INFO [2019-12-17 12:29:26,409] jepsen worker 54 - jepsen.util 2214 :invoke :write [18 29] INFO [2019-12-17 12:29:26,415] jepsen worker 88 - jepsen.util 2158 :ok :write [29 17] INFO [2019-12-17 12:29:26,416] jepsen worker 88 - jepsen.util 2158 :invoke :read [29 nil] INFO [2019-12-17 12:29:26,422] jepsen worker 2 - jepsen.util 2702 :info :cas [0 [48 15]] :RequestCanceledException INFO [2019-12-17 12:29:26,424] jepsen worker 2 - jepsen.util 2792 :invoke :read [0 nil] INFO [2019-12-17 12:29:26,435] jepsen worker 47 - jepsen.util 2567 :fail :read [15 nil] :RequestTimeoutException INFO [2019-12-17 12:29:26,435] jepsen worker 47 - jepsen.util 2567 :invoke :write [15 24] INFO [2019-12-17 12:29:26,442] jepsen worker 64 - jepsen.util 1864 :ok :read [21 8] INFO [2019-12-17 12:29:26,442] jepsen worker 64 - jepsen.util 1864 :invoke :write [21 12] INFO [2019-12-17 12:29:26,463] jepsen worker 63 - jepsen.util 1953 :ok :write [21 23] INFO [2019-12-17 12:29:26,463] jepsen worker 63 - jepsen.util 1953 :invoke :cas [21 [30 35]] INFO [2019-12-17 12:29:26,473] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":12689,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:29:26,473] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":12689,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:29:26,473] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":12692,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:29:26,473] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":12692,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:29:26,473] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":12693,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:29:26,473] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":12693,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:29:26,483] jepsen worker 63 - jepsen.util 1953 :fail :cas [21 [30 35]] :ValueNotSwapFrom INFO [2019-12-17 12:29:26,483] jepsen worker 63 - jepsen.util 1953 :invoke :cas [21 [2 39]] INFO [2019-12-17 12:29:26,489] jepsen worker 10 - jepsen.util 2260 :fail :read [3 nil] :RequestTimeoutException INFO [2019-12-17 12:29:26,489] jepsen worker 10 - jepsen.util 2260 :invoke :read [3 nil] INFO [2019-12-17 12:29:26,507] jepsen worker 55 - jepsen.util 2395 :info :write [18 15] :RequestCanceledException INFO [2019-12-17 12:29:26,508] jepsen worker 55 - jepsen.util 2485 :invoke :cas [18 [11 34]] INFO [2019-12-17 12:29:26,519] jepsen worker 64 - jepsen.util 1864 :ok :write [21 12] INFO [2019-12-17 12:29:26,519] jepsen worker 64 - jepsen.util 1864 :invoke :write [21 10] INFO [2019-12-17 12:29:26,521] jepsen worker 63 - jepsen.util 1953 :fail :cas [21 [2 39]] :ValueNotSwapFrom INFO [2019-12-17 12:29:26,521] jepsen worker 63 - jepsen.util 1953 :invoke :write [21 19] INFO [2019-12-17 12:29:26,522] jepsen worker 23 - jepsen.util 2633 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:29:26,522] jepsen worker 44 - jepsen.util 2564 :info :cas [14 [15 40]] :RequestCanceledException INFO [2019-12-17 12:29:26,522] jepsen worker 23 - jepsen.util 2633 :invoke :write [7 20] INFO [2019-12-17 12:29:26,524] jepsen worker 44 - jepsen.util 2654 :invoke :read [14 nil] INFO [2019-12-17 12:29:26,544] jepsen worker 87 - jepsen.util 1977 :fail :cas [29 [24 9]] :ValueNotSwapFrom INFO [2019-12-17 12:29:26,544] jepsen worker 87 - jepsen.util 1977 :invoke :write [29 21] INFO [2019-12-17 12:29:26,573] jepsen worker 88 - jepsen.util 2158 :ok :read [29 17] INFO [2019-12-17 12:29:26,573] jepsen worker 88 - jepsen.util 2158 :invoke :cas [29 [36 23]] INFO [2019-12-17 12:29:26,589] jepsen worker 49 - jepsen.util 2209 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:29:26,589] jepsen worker 49 - jepsen.util 2209 :invoke :write [16 44] INFO [2019-12-17 12:29:26,603] jepsen worker 31 - jepsen.util 2101 :ok :write [10 24] INFO [2019-12-17 12:29:26,603] jepsen worker 31 - jepsen.util 2101 :invoke :write [10 27] INFO [2019-12-17 12:29:26,608] jepsen worker 67 - jepsen.util 3307 :info :write [22 21] :RequestCanceledException INFO [2019-12-17 12:29:26,608] jepsen worker 53 - jepsen.util 2033 :info :write [17 9] :RequestCanceledException INFO [2019-12-17 12:29:26,609] jepsen worker 53 - jepsen.util 2123 :invoke :cas [17 [9 8]] INFO [2019-12-17 12:29:26,609] jepsen worker 67 - jepsen.util 3397 :invoke :cas [22 [4 29]] INFO [2019-12-17 12:29:26,621] jepsen worker 89 - jepsen.util 2249 :ok :write [29 34] INFO [2019-12-17 12:29:26,621] jepsen worker 89 - jepsen.util 2249 :invoke :read [29 nil] INFO [2019-12-17 12:29:26,635] jepsen worker 18 - jepsen.util 2448 :ok :write [6 9] INFO [2019-12-17 12:29:26,635] jepsen worker 18 - jepsen.util 2448 :invoke :read [6 nil] INFO [2019-12-17 12:29:26,638] jepsen worker 5 - jepsen.util 1895 :ok :write [1 26] INFO [2019-12-17 12:29:26,638] jepsen worker 5 - jepsen.util 1895 :invoke :cas [1 [9 15]] INFO [2019-12-17 12:29:26,652] jepsen worker 10 - jepsen.util 2260 :ok :read [3 9] INFO [2019-12-17 12:29:26,652] jepsen worker 10 - jepsen.util 2260 :invoke :write [3 5] INFO [2019-12-17 12:29:26,676] jepsen worker 64 - jepsen.util 1864 :ok :write [21 10] INFO [2019-12-17 12:29:26,676] jepsen worker 64 - jepsen.util 1864 :invoke :cas [21 [40 3]] INFO [2019-12-17 12:29:26,678] jepsen worker 13 - jepsen.util 2083 :fail :cas [4 [22 7]] :ValueNotSwapFrom INFO [2019-12-17 12:29:26,678] jepsen worker 13 - jepsen.util 2083 :invoke :write [4 48] INFO [2019-12-17 12:29:26,684] jepsen worker 18 - jepsen.util 2448 :ok :read [6 9] INFO [2019-12-17 12:29:26,684] jepsen worker 18 - jepsen.util 2448 :invoke :write [6 10] INFO [2019-12-17 12:29:26,692] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":12713,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:29:26,692] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":12713,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:29:26,692] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":12718,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:29:26,692] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":12718,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:29:26,705] jepsen worker 34 - jepsen.util 2914 :info :write [11 21] :RequestCanceledException INFO [2019-12-17 12:29:26,706] jepsen worker 34 - jepsen.util 3004 :invoke :write [11 12] INFO [2019-12-17 12:29:26,728] jepsen worker 64 - jepsen.util 1864 :fail :cas [21 [40 3]] :ValueNotSwapFrom INFO [2019-12-17 12:29:26,728] jepsen worker 64 - jepsen.util 1864 :invoke :write [21 24] INFO [2019-12-17 12:29:26,739] jepsen worker 88 - jepsen.util 2158 :fail :cas [29 [36 23]] :ValueNotSwapFrom INFO [2019-12-17 12:29:26,739] jepsen worker 88 - jepsen.util 2158 :invoke :read [29 nil] INFO [2019-12-17 12:29:26,743] jepsen worker 72 - jepsen.util 2322 :ok :write [24 30] INFO [2019-12-17 12:29:26,743] jepsen worker 72 - jepsen.util 2322 :invoke :read [24 nil] INFO [2019-12-17 12:29:26,754] jepsen worker 79 - jepsen.util 2239 :ok :write [26 4] INFO [2019-12-17 12:29:26,754] jepsen worker 79 - jepsen.util 2239 :invoke :cas [26 [23 12]] INFO [2019-12-17 12:29:26,767] jepsen worker 63 - jepsen.util 1953 :ok :write [21 19] INFO [2019-12-17 12:29:26,767] jepsen worker 63 - jepsen.util 1953 :invoke :cas [21 [4 5]] INFO [2019-12-17 12:29:26,777] jepsen worker 51 - jepsen.util 2031 :ok :write [17 12] INFO [2019-12-17 12:29:26,777] jepsen worker 51 - jepsen.util 2031 :invoke :read [17 nil] INFO [2019-12-17 12:29:26,779] jepsen worker 63 - jepsen.util 1953 :fail :cas [21 [4 5]] :ValueNotSwapFrom INFO [2019-12-17 12:29:26,779] jepsen worker 63 - jepsen.util 1953 :invoke :cas [21 [12 34]] INFO [2019-12-17 12:29:26,784] jepsen worker 72 - jepsen.util 2322 :ok :read [24 30] INFO [2019-12-17 12:29:26,784] jepsen worker 72 - jepsen.util 2322 :invoke :cas [24 [38 48]] INFO [2019-12-17 12:29:26,811] jepsen worker 47 - jepsen.util 2567 :ok :write [15 24] INFO [2019-12-17 12:29:26,811] jepsen worker 47 - jepsen.util 2567 :invoke :cas [15 [29 23]] INFO [2019-12-17 12:29:26,816] jepsen worker 72 - jepsen.util 2322 :fail :cas [24 [38 48]] :ValueNotSwapFrom INFO [2019-12-17 12:29:26,817] jepsen worker 72 - jepsen.util 2322 :invoke :read [24 nil] INFO [2019-12-17 12:29:26,817] jepsen worker 63 - jepsen.util 1953 :fail :cas [21 [12 34]] :ValueNotSwapFrom INFO [2019-12-17 12:29:26,818] jepsen worker 63 - jepsen.util 1953 :invoke :write [21 20] INFO [2019-12-17 12:29:26,854] jepsen worker 71 - jepsen.util 1871 :info :write [23 43] :RequestCanceledException INFO [2019-12-17 12:29:26,855] jepsen worker 71 - jepsen.util 1961 :invoke :write [23 40] INFO [2019-12-17 12:29:26,886] jepsen worker 55 - jepsen.util 2485 :fail :cas [18 [11 34]] :ValueNotSwapFrom INFO [2019-12-17 12:29:26,887] jepsen worker 55 - jepsen.util 2485 :invoke :cas [18 [43 12]] INFO [2019-12-17 12:29:26,896] jepsen worker 72 - jepsen.util 2322 :ok :read [24 30] INFO [2019-12-17 12:29:26,897] jepsen worker 72 - jepsen.util 2322 :invoke :cas [24 [45 25]] INFO [2019-12-17 12:29:26,898] jepsen worker 47 - jepsen.util 2567 :fail :cas [15 [29 23]] :ValueNotSwapFrom INFO [2019-12-17 12:29:26,899] jepsen worker 47 - jepsen.util 2567 :invoke :read [15 nil] INFO [2019-12-17 12:29:26,907] jepsen worker 29 - jepsen.util 1919 :info :cas [9 [31 3]] :RequestCanceledException INFO [2019-12-17 12:29:26,909] jepsen worker 5 - jepsen.util 1895 :fail :cas [1 [9 15]] :ValueNotSwapFrom INFO [2019-12-17 12:29:26,909] jepsen worker 5 - jepsen.util 1895 :invoke :write [1 46] INFO [2019-12-17 12:29:26,909] jepsen worker 29 - jepsen.util 2009 :invoke :write [9 14] INFO [2019-12-17 12:29:26,910] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":12738,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:29:26,910] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":12738,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:29:26,910] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":12740,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:29:26,910] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":12740,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:29:26,922] jepsen worker 64 - jepsen.util 1864 :ok :write [21 24] INFO [2019-12-17 12:29:26,922] jepsen worker 64 - jepsen.util 1864 :invoke :write [21 27] INFO [2019-12-17 12:29:26,924] jepsen worker 10 - jepsen.util 2260 :ok :write [3 5] INFO [2019-12-17 12:29:26,925] jepsen worker 10 - jepsen.util 2260 :invoke :write [3 2] INFO [2019-12-17 12:29:26,928] jepsen worker 72 - jepsen.util 2322 :fail :cas [24 [45 25]] :ValueNotSwapFrom INFO [2019-12-17 12:29:26,928] jepsen worker 72 - jepsen.util 2322 :invoke :cas [24 [1 33]] INFO [2019-12-17 12:29:26,937] jepsen worker 78 - jepsen.util 2328 :info :write [26 46] :RequestCanceledException INFO [2019-12-17 12:29:26,937] jepsen worker 4 - jepsen.util 1714 :info :write [1 35] :RequestCanceledException INFO [2019-12-17 12:29:26,937] jepsen worker 78 - jepsen.util 2418 :invoke :cas [26 [21 1]] INFO [2019-12-17 12:29:26,938] jepsen worker 4 - jepsen.util 1804 :invoke :read [1 nil] INFO [2019-12-17 12:29:26,975] jepsen worker 55 - jepsen.util 2485 :fail :cas [18 [43 12]] :ValueNotSwapFrom INFO [2019-12-17 12:29:26,975] jepsen worker 55 - jepsen.util 2485 :invoke :write [18 27] INFO [2019-12-17 12:29:26,976] jepsen worker 47 - jepsen.util 2567 :ok :read [15 24] INFO [2019-12-17 12:29:26,976] jepsen worker 47 - jepsen.util 2567 :invoke :write [15 49] INFO [2019-12-17 12:29:26,984] jepsen worker 89 - jepsen.util 2249 :ok :read [29 34] INFO [2019-12-17 12:29:26,984] jepsen worker 89 - jepsen.util 2249 :invoke :read [29 nil] INFO [2019-12-17 12:29:26,988] jepsen worker 88 - jepsen.util 2158 :ok :read [29 34] INFO [2019-12-17 12:29:26,988] jepsen worker 88 - jepsen.util 2158 :invoke :read [29 nil] INFO [2019-12-17 12:29:26,989] jepsen worker 60 - jepsen.util 3030 :info :write [20 4] :RequestCanceledException INFO [2019-12-17 12:29:26,991] jepsen worker 60 - jepsen.util 3120 :invoke :cas [20 [1 15]] INFO [2019-12-17 12:29:27,019] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":12757,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:29:27,019] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":12757,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:29:27,020] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":12761,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:29:27,020] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":12761,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:29:27,020] jepsen worker 38 - jepsen.util 2108 :info :write [12 36] :RequestCanceledException INFO [2019-12-17 12:29:27,021] jepsen worker 38 - jepsen.util 2198 :invoke :write [12 18] INFO [2019-12-17 12:29:27,025] jepsen worker 72 - jepsen.util 2322 :fail :cas [24 [1 33]] :ValueNotSwapFrom INFO [2019-12-17 12:29:27,025] jepsen worker 72 - jepsen.util 2322 :invoke :write [24 23] INFO [2019-12-17 12:29:27,035] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:29:27,035] jepsen nemesis - couchbase.util Rebalance check count: 91 INFO [2019-12-17 12:29:27,035] jepsen nemesis - couchbase.util Rebalance stuck count: 72 INFO [2019-12-17 12:29:27,035] jepsen worker 63 - jepsen.util 1953 :ok :write [21 20] INFO [2019-12-17 12:29:27,035] jepsen worker 63 - jepsen.util 1953 :invoke :cas [21 [44 19]] INFO [2019-12-17 12:29:27,055] jepsen worker 63 - jepsen.util 1953 :fail :cas [21 [44 19]] :ValueNotSwapFrom INFO [2019-12-17 12:29:27,055] jepsen worker 63 - jepsen.util 1953 :invoke :write [21 9] INFO [2019-12-17 12:29:27,077] jepsen worker 88 - jepsen.util 2158 :ok :read [29 34] INFO [2019-12-17 12:29:27,077] jepsen worker 88 - jepsen.util 2158 :invoke :cas [29 [29 41]] INFO [2019-12-17 12:29:27,077] jepsen worker 29 - jepsen.util 2009 :ok :write [9 14] INFO [2019-12-17 12:29:27,077] jepsen worker 29 - jepsen.util 2009 :invoke :cas [9 [48 25]] INFO [2019-12-17 12:29:27,103] jepsen worker 29 - jepsen.util 2009 :fail :cas [9 [48 25]] :ValueNotSwapFrom INFO [2019-12-17 12:29:27,103] jepsen worker 29 - jepsen.util 2009 :invoke :cas [9 [40 24]] INFO [2019-12-17 12:29:27,133] jepsen worker 29 - jepsen.util 2009 :fail :cas [9 [40 24]] :ValueNotSwapFrom INFO [2019-12-17 12:29:27,133] jepsen worker 29 - jepsen.util 2009 :invoke :read [9 nil] INFO [2019-12-17 12:29:27,142] jepsen worker 10 - jepsen.util 2260 :ok :write [3 2] INFO [2019-12-17 12:29:27,142] jepsen worker 72 - jepsen.util 2322 :ok :write [24 23] INFO [2019-12-17 12:29:27,142] jepsen worker 10 - jepsen.util 2260 :invoke :write [3 14] INFO [2019-12-17 12:29:27,142] jepsen worker 72 - jepsen.util 2322 :invoke :write [24 33] INFO [2019-12-17 12:29:27,158] jepsen worker 31 - jepsen.util 2101 :ok :write [10 27] INFO [2019-12-17 12:29:27,159] jepsen worker 31 - jepsen.util 2101 :invoke :write [10 14] INFO [2019-12-17 12:29:27,187] jepsen worker 29 - jepsen.util 2009 :ok :read [9 14] INFO [2019-12-17 12:29:27,187] jepsen worker 29 - jepsen.util 2009 :invoke :read [9 nil] INFO [2019-12-17 12:29:27,189] jepsen worker 6 - jepsen.util 2886 :info :cas [2 [45 16]] :RequestCanceledException INFO [2019-12-17 12:29:27,190] jepsen worker 6 - jepsen.util 2976 :invoke :write [2 13] INFO [2019-12-17 12:29:27,205] jepsen worker 36 - jepsen.util 2016 :info :write [12 35] :RequestCanceledException INFO [2019-12-17 12:29:27,205] jepsen worker 7 - jepsen.util 2707 :info :cas [2 [19 2]] :RequestCanceledException INFO [2019-12-17 12:29:27,205] jepsen worker 12 - jepsen.util 2262 :info :write [4 7] :RequestCanceledException INFO [2019-12-17 12:29:27,206] jepsen worker 7 - jepsen.util 2797 :invoke :read [2 nil] INFO [2019-12-17 12:29:27,206] jepsen worker 36 - jepsen.util 2106 :invoke :write [12 27] INFO [2019-12-17 12:29:27,206] jepsen worker 12 - jepsen.util 2352 :invoke :write [4 26] INFO [2019-12-17 12:29:27,222] jepsen worker 63 - jepsen.util 1953 :ok :write [21 9] INFO [2019-12-17 12:29:27,222] jepsen worker 3 - jepsen.util 1983 :info :write [1 19] :RequestCanceledException INFO [2019-12-17 12:29:27,222] jepsen worker 63 - jepsen.util 1953 :invoke :cas [21 [28 24]] INFO [2019-12-17 12:29:27,224] jepsen worker 3 - jepsen.util 2073 :invoke :write [1 33] INFO [2019-12-17 12:29:27,230] jepsen worker 29 - jepsen.util 2009 :ok :read [9 14] INFO [2019-12-17 12:29:27,230] jepsen worker 29 - jepsen.util 2009 :invoke :write [9 21] INFO [2019-12-17 12:29:27,236] jepsen worker 63 - jepsen.util 1953 :fail :cas [21 [28 24]] :ValueNotSwapFrom INFO [2019-12-17 12:29:27,236] jepsen worker 63 - jepsen.util 1953 :invoke :cas [21 [45 45]] INFO [2019-12-17 12:29:27,239] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":12775,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:29:27,239] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":12784,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:29:27,239] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":12775,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:29:27,239] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":12784,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:29:27,239] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":12776,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:29:27,239] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":12785,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:29:27,239] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":12776,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:29:27,239] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":12785,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:29:27,247] jepsen worker 88 - jepsen.util 2158 :fail :cas [29 [29 41]] :ValueNotSwapFrom INFO [2019-12-17 12:29:27,247] jepsen worker 88 - jepsen.util 2158 :invoke :cas [29 [38 38]] INFO [2019-12-17 12:29:27,253] jepsen worker 56 - jepsen.util 2126 :info :write [18 3] :RequestCanceledException INFO [2019-12-17 12:29:27,254] jepsen worker 56 - jepsen.util 2216 :invoke :cas [18 [12 42]] INFO [2019-12-17 12:29:27,265] jepsen worker 63 - jepsen.util 1953 :fail :cas [21 [45 45]] :ValueNotSwapFrom INFO [2019-12-17 12:29:27,265] jepsen worker 63 - jepsen.util 1953 :invoke :read [21 nil] INFO [2019-12-17 12:29:27,288] jepsen worker 63 - jepsen.util 1953 :ok :read [21 9] INFO [2019-12-17 12:29:27,288] jepsen worker 63 - jepsen.util 1953 :invoke :write [21 30] INFO [2019-12-17 12:29:27,290] jepsen worker 19 - jepsen.util 2359 :info :cas [6 [29 43]] :RequestCanceledException INFO [2019-12-17 12:29:27,290] jepsen worker 73 - jepsen.util 2233 :info :write [24 32] :RequestCanceledException INFO [2019-12-17 12:29:27,291] jepsen worker 73 - jepsen.util 2323 :invoke :cas [24 [20 48]] INFO [2019-12-17 12:29:27,291] jepsen worker 19 - jepsen.util 2449 :invoke :cas [6 [42 38]] INFO [2019-12-17 12:29:27,297] jepsen worker 14 - jepsen.util 2174 :ok :write [4 9] INFO [2019-12-17 12:29:27,298] jepsen worker 14 - jepsen.util 2174 :invoke :read [4 nil] INFO [2019-12-17 12:29:27,306] jepsen worker 38 - jepsen.util 2198 :ok :write [12 18] INFO [2019-12-17 12:29:27,307] jepsen worker 38 - jepsen.util 2198 :invoke :cas [12 [30 8]] INFO [2019-12-17 12:29:27,316] jepsen worker 73 - jepsen.util 2323 :fail :cas [24 [20 48]] :ValueNotSwapFrom INFO [2019-12-17 12:29:27,317] jepsen worker 73 - jepsen.util 2323 :invoke :write [24 14] INFO [2019-12-17 12:29:27,317] jepsen worker 10 - jepsen.util 2260 :ok :write [3 14] INFO [2019-12-17 12:29:27,318] jepsen worker 10 - jepsen.util 2260 :invoke :write [3 10] INFO [2019-12-17 12:29:27,333] jepsen worker 31 - jepsen.util 2101 :ok :write [10 14] INFO [2019-12-17 12:29:27,333] jepsen worker 31 - jepsen.util 2101 :invoke :cas [10 [21 14]] INFO [2019-12-17 12:29:27,352] jepsen worker 31 - jepsen.util 2101 :fail :cas [10 [21 14]] :ValueNotSwapFrom INFO [2019-12-17 12:29:27,352] jepsen worker 31 - jepsen.util 2101 :invoke :cas [10 [21 37]] INFO [2019-12-17 12:29:27,383] jepsen worker 29 - jepsen.util 2009 :ok :write [9 21] INFO [2019-12-17 12:29:27,384] jepsen worker 29 - jepsen.util 2009 :invoke :cas [9 [23 49]] INFO [2019-12-17 12:29:27,390] jepsen worker 31 - jepsen.util 2101 :fail :cas [10 [21 37]] :ValueNotSwapFrom INFO [2019-12-17 12:29:27,390] jepsen worker 31 - jepsen.util 2101 :invoke :cas [10 [7 2]] INFO [2019-12-17 12:29:27,401] jepsen worker 19 - jepsen.util 2449 :fail :cas [6 [42 38]] :ValueNotSwapFrom INFO [2019-12-17 12:29:27,401] jepsen worker 19 - jepsen.util 2449 :invoke :read [6 nil] INFO [2019-12-17 12:29:27,417] jepsen worker 31 - jepsen.util 2101 :fail :cas [10 [7 2]] :ValueNotSwapFrom INFO [2019-12-17 12:29:27,417] jepsen worker 31 - jepsen.util 2101 :invoke :cas [10 [17 15]] INFO [2019-12-17 12:29:27,422] jepsen worker 80 - jepsen.util 2060 :info :write [26 15] :RequestCanceledException INFO [2019-12-17 12:29:27,424] jepsen worker 80 - jepsen.util 2150 :invoke :write [26 6] INFO [2019-12-17 12:29:27,424] jepsen worker 29 - jepsen.util 2009 :fail :cas [9 [23 49]] :ValueNotSwapFrom INFO [2019-12-17 12:29:27,424] jepsen worker 29 - jepsen.util 2009 :invoke :cas [9 [34 1]] INFO [2019-12-17 12:29:27,446] jepsen worker 19 - jepsen.util 2449 :ok :read [6 9] INFO [2019-12-17 12:29:27,446] jepsen worker 19 - jepsen.util 2449 :invoke :read [6 nil] INFO [2019-12-17 12:29:27,451] jepsen worker 63 - jepsen.util 1953 :ok :write [21 30] INFO [2019-12-17 12:29:27,451] jepsen worker 63 - jepsen.util 1953 :invoke :cas [21 [36 29]] INFO [2019-12-17 12:29:27,456] jepsen worker 31 - jepsen.util 2101 :fail :cas [10 [17 15]] :ValueNotSwapFrom INFO [2019-12-17 12:29:27,457] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":12812,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:29:27,457] jepsen worker 31 - jepsen.util 2101 :invoke :read [10 nil] INFO [2019-12-17 12:29:27,457] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":12812,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:29:27,457] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":12815,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:29:27,457] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":12815,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:29:27,462] jepsen worker 29 - jepsen.util 2009 :fail :cas [9 [34 1]] :ValueNotSwapFrom INFO [2019-12-17 12:29:27,462] jepsen worker 29 - jepsen.util 2009 :invoke :cas [9 [7 35]] INFO [2019-12-17 12:29:27,464] jepsen worker 38 - jepsen.util 2198 :fail :cas [12 [30 8]] :ValueNotSwapFrom INFO [2019-12-17 12:29:27,464] jepsen worker 38 - jepsen.util 2198 :invoke :cas [12 [24 25]] INFO [2019-12-17 12:29:27,472] jepsen worker 63 - jepsen.util 1953 :fail :cas [21 [36 29]] :ValueNotSwapFrom INFO [2019-12-17 12:29:27,472] jepsen worker 63 - jepsen.util 1953 :invoke :cas [21 [43 42]] INFO [2019-12-17 12:29:27,484] jepsen worker 51 - jepsen.util 2031 :ok :read [17 12] INFO [2019-12-17 12:29:27,484] jepsen worker 51 - jepsen.util 2031 :invoke :cas [17 [0 44]] INFO [2019-12-17 12:29:27,492] jepsen worker 78 - jepsen.util 2418 :fail :cas [26 [21 1]] :ValueNotSwapFrom INFO [2019-12-17 12:29:27,492] jepsen worker 78 - jepsen.util 2418 :invoke :cas [26 [46 10]] INFO [2019-12-17 12:29:27,503] jepsen worker 89 - jepsen.util 2249 :ok :read [29 34] INFO [2019-12-17 12:29:27,503] jepsen worker 89 - jepsen.util 2249 :invoke :cas [29 [8 37]] INFO [2019-12-17 12:29:27,514] jepsen worker 19 - jepsen.util 2449 :ok :read [6 9] INFO [2019-12-17 12:29:27,514] jepsen worker 19 - jepsen.util 2449 :invoke :write [6 45] INFO [2019-12-17 12:29:27,515] jepsen worker 31 - jepsen.util 2101 :ok :read [10 14] INFO [2019-12-17 12:29:27,516] jepsen worker 31 - jepsen.util 2101 :invoke :cas [10 [38 31]] INFO [2019-12-17 12:29:27,523] jepsen worker 52 - jepsen.util 2212 :info :write [17 10] :RequestCanceledException INFO [2019-12-17 12:29:27,524] jepsen worker 52 - jepsen.util 2302 :invoke :write [17 24] INFO [2019-12-17 12:29:27,540] jepsen worker 56 - jepsen.util 2216 :fail :cas [18 [12 42]] :ValueNotSwapFrom INFO [2019-12-17 12:29:27,540] jepsen worker 56 - jepsen.util 2216 :invoke :read [18 nil] INFO [2019-12-17 12:29:27,554] jepsen worker 38 - jepsen.util 2198 :fail :cas [12 [24 25]] :ValueNotSwapFrom INFO [2019-12-17 12:29:27,554] jepsen worker 38 - jepsen.util 2198 :invoke :cas [12 [8 11]] INFO [2019-12-17 12:29:27,566] jepsen worker 63 - jepsen.util 1953 :fail :cas [21 [43 42]] :ValueNotSwapFrom INFO [2019-12-17 12:29:27,566] jepsen worker 63 - jepsen.util 1953 :invoke :read [21 nil] INFO [2019-12-17 12:29:27,573] jepsen worker 56 - jepsen.util 2216 :ok :read [18 23] INFO [2019-12-17 12:29:27,573] jepsen worker 56 - jepsen.util 2216 :invoke :cas [18 [23 20]] INFO [2019-12-17 12:29:27,587] jepsen worker 29 - jepsen.util 2009 :fail :cas [9 [7 35]] :ValueNotSwapFrom INFO [2019-12-17 12:29:27,587] jepsen worker 29 - jepsen.util 2009 :invoke :read [9 nil] INFO [2019-12-17 12:29:27,591] jepsen worker 31 - jepsen.util 2101 :fail :cas [10 [38 31]] :ValueNotSwapFrom INFO [2019-12-17 12:29:27,592] jepsen worker 31 - jepsen.util 2101 :invoke :cas [10 [39 34]] INFO [2019-12-17 12:29:27,608] jepsen worker 63 - jepsen.util 1953 :ok :read [21 30] INFO [2019-12-17 12:29:27,608] jepsen worker 63 - jepsen.util 1953 :invoke :read [21 nil] INFO [2019-12-17 12:29:27,622] jepsen worker 37 - jepsen.util 1837 :info :write [12 15] :RequestCanceledException INFO [2019-12-17 12:29:27,622] jepsen worker 30 - jepsen.util 2640 :info :write [10 40] :RequestCanceledException INFO [2019-12-17 12:29:27,626] jepsen worker 30 - jepsen.util 2730 :invoke :cas [10 [8 39]] INFO [2019-12-17 12:29:27,627] jepsen worker 37 - jepsen.util 1927 :invoke :read [12 nil] INFO [2019-12-17 12:29:27,637] jepsen worker 31 - jepsen.util 2101 :fail :cas [10 [39 34]] :ValueNotSwapFrom INFO [2019-12-17 12:29:27,638] jepsen worker 31 - jepsen.util 2101 :invoke :cas [10 [27 46]] INFO [2019-12-17 12:29:27,642] jepsen worker 63 - jepsen.util 1953 :ok :read [21 30] INFO [2019-12-17 12:29:27,642] jepsen worker 63 - jepsen.util 1953 :invoke :read [21 nil] INFO [2019-12-17 12:29:27,648] jepsen worker 38 - jepsen.util 2198 :fail :cas [12 [8 11]] :ValueNotSwapFrom INFO [2019-12-17 12:29:27,648] jepsen worker 38 - jepsen.util 2198 :invoke :read [12 nil] INFO [2019-12-17 12:29:27,673] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":12842,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:29:27,673] jepsen worker 31 - jepsen.util 2101 :fail :cas [10 [27 46]] :ValueNotSwapFrom INFO [2019-12-17 12:29:27,674] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":12842,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:29:27,674] jepsen worker 31 - jepsen.util 2101 :invoke :cas [10 [38 32]] INFO [2019-12-17 12:29:27,674] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":12845,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:29:27,674] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":12845,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:29:27,677] jepsen worker 63 - jepsen.util 1953 :ok :read [21 30] INFO [2019-12-17 12:29:27,677] jepsen worker 63 - jepsen.util 1953 :invoke :cas [21 [12 43]] INFO [2019-12-17 12:29:27,696] jepsen worker 37 - jepsen.util 1927 :ok :read [12 18] INFO [2019-12-17 12:29:27,696] jepsen worker 37 - jepsen.util 1927 :invoke :cas [12 [38 9]] INFO [2019-12-17 12:29:27,701] jepsen worker 31 - jepsen.util 2101 :fail :cas [10 [38 32]] :ValueNotSwapFrom INFO [2019-12-17 12:29:27,701] jepsen worker 31 - jepsen.util 2101 :invoke :write [10 8] INFO [2019-12-17 12:29:27,705] jepsen worker 63 - jepsen.util 1953 :fail :cas [21 [12 43]] :ValueNotSwapFrom INFO [2019-12-17 12:29:27,705] jepsen worker 63 - jepsen.util 1953 :invoke :cas [21 [12 0]] INFO [2019-12-17 12:29:27,721] jepsen worker 9 - jepsen.util 2439 :info :write [3 30] :RequestCanceledException INFO [2019-12-17 12:29:27,722] jepsen worker 9 - jepsen.util 2529 :invoke :read [3 nil] INFO [2019-12-17 12:29:27,733] jepsen worker 88 - jepsen.util 2158 :fail :cas [29 [38 38]] :ValueNotSwapFrom INFO [2019-12-17 12:29:27,733] jepsen worker 88 - jepsen.util 2158 :invoke :cas [29 [5 8]] INFO [2019-12-17 12:29:27,742] jepsen worker 63 - jepsen.util 1953 :fail :cas [21 [12 0]] :ValueNotSwapFrom INFO [2019-12-17 12:29:27,742] jepsen worker 63 - jepsen.util 1953 :invoke :read [21 nil] INFO [2019-12-17 12:29:27,743] jepsen worker 38 - jepsen.util 2198 :ok :read [12 18] INFO [2019-12-17 12:29:27,743] jepsen worker 38 - jepsen.util 2198 :invoke :cas [12 [15 28]] INFO [2019-12-17 12:29:27,766] jepsen worker 9 - jepsen.util 2529 :ok :read [3 14] INFO [2019-12-17 12:29:27,766] jepsen worker 9 - jepsen.util 2529 :invoke :read [3 nil] INFO [2019-12-17 12:29:27,784] jepsen worker 63 - jepsen.util 1953 :ok :read [21 30] INFO [2019-12-17 12:29:27,784] jepsen worker 63 - jepsen.util 1953 :invoke :cas [21 [16 13]] INFO [2019-12-17 12:29:27,789] jepsen worker 88 - jepsen.util 2158 :fail :cas [29 [5 8]] :ValueNotSwapFrom INFO [2019-12-17 12:29:27,789] jepsen worker 88 - jepsen.util 2158 :invoke :write [29 12] INFO [2019-12-17 12:29:27,822] jepsen worker 65 - jepsen.util 1955 :info :write [21 0] :RequestCanceledException INFO [2019-12-17 12:29:27,822] jepsen worker 16 - jepsen.util 2806 :info :cas [5 [18 22]] :RequestCanceledException INFO [2019-12-17 12:29:27,824] jepsen worker 9 - jepsen.util 2529 :ok :read [3 14] INFO [2019-12-17 12:29:27,824] jepsen worker 63 - jepsen.util 1953 :fail :cas [21 [16 13]] :ValueNotSwapFrom INFO [2019-12-17 12:29:27,824] jepsen worker 9 - jepsen.util 2529 :invoke :read [3 nil] INFO [2019-12-17 12:29:27,824] jepsen worker 65 - jepsen.util 2045 :invoke :read [21 nil] INFO [2019-12-17 12:29:27,824] jepsen worker 16 - jepsen.util 2896 :invoke :read [5 nil] INFO [2019-12-17 12:29:27,824] jepsen worker 63 - jepsen.util 1953 :invoke :write [21 47] INFO [2019-12-17 12:29:27,836] jepsen worker 15 - jepsen.util 2715 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:29:27,836] jepsen worker 15 - jepsen.util 2715 :invoke :write [5 19] INFO [2019-12-17 12:29:27,840] jepsen worker 9 - jepsen.util 2529 :ok :read [3 14] INFO [2019-12-17 12:29:27,840] jepsen worker 9 - jepsen.util 2529 :invoke :read [3 nil] INFO [2019-12-17 12:29:27,852] jepsen worker 70 - jepsen.util 2500 :info :write [23 46] :RequestCanceledException INFO [2019-12-17 12:29:27,853] jepsen worker 70 - jepsen.util 2590 :invoke :cas [23 [17 1]] INFO [2019-12-17 12:29:27,858] jepsen worker 14 - jepsen.util 2174 :ok :read [4 9] INFO [2019-12-17 12:29:27,858] jepsen worker 14 - jepsen.util 2174 :invoke :read [4 nil] INFO [2019-12-17 12:29:27,870] jepsen worker 9 - jepsen.util 2529 :ok :read [3 14] INFO [2019-12-17 12:29:27,870] jepsen worker 9 - jepsen.util 2529 :invoke :read [3 nil] INFO [2019-12-17 12:29:27,893] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":12868,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:29:27,893] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":12868,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:29:27,893] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":12869,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:29:27,893] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":12869,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:29:27,893] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":12870,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:29:27,893] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":12870,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:29:27,900] jepsen worker 9 - jepsen.util 2529 :ok :read [3 14] INFO [2019-12-17 12:29:27,900] jepsen worker 9 - jepsen.util 2529 :invoke :write [3 18] INFO [2019-12-17 12:29:27,911] jepsen worker 63 - jepsen.util 1953 :ok :write [21 47] INFO [2019-12-17 12:29:27,911] jepsen worker 63 - jepsen.util 1953 :invoke :cas [21 [3 32]] INFO [2019-12-17 12:29:27,923] jepsen worker 43 - jepsen.util 2833 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:29:27,923] jepsen worker 43 - jepsen.util 2833 :invoke :read [14 nil] INFO [2019-12-17 12:29:27,933] jepsen worker 51 - jepsen.util 2031 :fail :cas [17 [0 44]] :ValueNotSwapFrom INFO [2019-12-17 12:29:27,933] jepsen worker 51 - jepsen.util 2031 :invoke :read [17 nil] INFO [2019-12-17 12:29:27,936] jepsen worker 63 - jepsen.util 1953 :fail :cas [21 [3 32]] :ValueNotSwapFrom INFO [2019-12-17 12:29:27,936] jepsen worker 63 - jepsen.util 1953 :invoke :cas [21 [18 44]] INFO [2019-12-17 12:29:27,936] jepsen worker 0 - jepsen.util 3150 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:29:27,936] jepsen worker 11 - jepsen.util 2711 :info :write [3 8] :RequestCanceledException INFO [2019-12-17 12:29:27,936] jepsen worker 0 - jepsen.util 3150 :invoke :read [0 nil] INFO [2019-12-17 12:29:27,938] jepsen worker 11 - jepsen.util 2801 :invoke :cas [3 [32 40]] INFO [2019-12-17 12:29:27,950] jepsen worker 63 - jepsen.util 1953 :fail :cas [21 [18 44]] :ValueNotSwapFrom INFO [2019-12-17 12:29:27,950] jepsen worker 63 - jepsen.util 1953 :invoke :cas [21 [13 20]] INFO [2019-12-17 12:29:27,964] jepsen worker 63 - jepsen.util 1953 :fail :cas [21 [13 20]] :ValueNotSwapFrom INFO [2019-12-17 12:29:27,967] jepsen worker 63 - jepsen.util 1953 :invoke :cas [21 [18 9]] INFO [2019-12-17 12:29:27,976] jepsen worker 63 - jepsen.util 1953 :fail :cas [21 [18 9]] :ValueNotSwapFrom INFO [2019-12-17 12:29:27,976] jepsen worker 63 - jepsen.util 1953 :invoke :write [21 49] INFO [2019-12-17 12:29:27,983] jepsen worker 79 - jepsen.util 2239 :fail :cas [26 [23 12]] :ValueNotSwapFrom INFO [2019-12-17 12:29:27,983] jepsen worker 79 - jepsen.util 2239 :invoke :cas [26 [12 15]] INFO [2019-12-17 12:29:27,989] jepsen worker 21 - jepsen.util 2901 :info :write [7 0] :RequestCanceledException INFO [2019-12-17 12:29:27,990] jepsen worker 21 - jepsen.util 2991 :invoke :read [7 nil] INFO [2019-12-17 12:29:27,991] jepsen worker 38 - jepsen.util 2198 :fail :cas [12 [15 28]] :ValueNotSwapFrom INFO [2019-12-17 12:29:27,991] jepsen worker 38 - jepsen.util 2198 :invoke :cas [12 [46 5]] INFO [2019-12-17 12:29:28,003] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":12890,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:29:28,003] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":12890,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:29:28,003] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":12893,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:29:28,003] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":12893,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:29:28,008] jepsen worker 35 - jepsen.util 2465 :info :write [11 41] :RequestCanceledException INFO [2019-12-17 12:29:28,009] jepsen worker 35 - jepsen.util 2555 :invoke :cas [11 [32 2]] INFO [2019-12-17 12:29:28,018] jepsen worker 63 - jepsen.util 1953 :ok :write [21 49] INFO [2019-12-17 12:29:28,018] jepsen worker 63 - jepsen.util 1953 :invoke :cas [21 [20 16]] INFO [2019-12-17 12:29:28,023] jepsen worker 57 - jepsen.util 2667 :info :cas [19 [29 44]] :RequestCanceledException INFO [2019-12-17 12:29:28,023] jepsen worker 1 - jepsen.util 2791 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:29:28,023] jepsen worker 1 - jepsen.util 2791 :invoke :read [0 nil] INFO [2019-12-17 12:29:28,023] jepsen worker 57 - jepsen.util 2757 :invoke :cas [19 [25 47]] INFO [2019-12-17 12:29:28,031] jepsen worker 63 - jepsen.util 1953 :fail :cas [21 [20 16]] :ValueNotSwapFrom INFO [2019-12-17 12:29:28,031] jepsen worker 63 - jepsen.util 1953 :invoke :write [21 6] INFO [2019-12-17 12:29:28,036] jepsen worker 50 - jepsen.util 3110 :info :write [16 21] :RequestCanceledException INFO [2019-12-17 12:29:28,036] jepsen worker 59 - jepsen.util 2669 :info :cas [19 [19 49]] :RequestCanceledException INFO [2019-12-17 12:29:28,037] jepsen worker 59 - jepsen.util 2759 :invoke :cas [19 [32 44]] INFO [2019-12-17 12:29:28,037] jepsen worker 50 - jepsen.util 3200 :invoke :write [16 40] INFO [2019-12-17 12:29:28,041] jepsen worker 4 - jepsen.util 1804 :ok :read [1 26] INFO [2019-12-17 12:29:28,041] jepsen worker 4 - jepsen.util 1804 :invoke :cas [1 [9 19]] INFO [2019-12-17 12:29:28,047] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:29:28,047] jepsen nemesis - couchbase.util Rebalance check count: 92 INFO [2019-12-17 12:29:28,047] jepsen nemesis - couchbase.util Rebalance stuck count: 73 INFO [2019-12-17 12:29:28,053] jepsen worker 85 - jepsen.util 3235 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:29:28,053] jepsen worker 22 - jepsen.util 2902 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:29:28,053] jepsen worker 81 - jepsen.util 2511 :info :write [27 16] :RequestCanceledException INFO [2019-12-17 12:29:28,053] jepsen worker 76 - jepsen.util 2866 :info :cas [25 [18 28]] :RequestCanceledException INFO [2019-12-17 12:29:28,053] jepsen worker 85 - jepsen.util 3235 :invoke :write [28 25] INFO [2019-12-17 12:29:28,053] jepsen worker 22 - jepsen.util 2902 :invoke :write [7 37] INFO [2019-12-17 12:29:28,054] jepsen worker 76 - jepsen.util 2956 :invoke :write [25 6] INFO [2019-12-17 12:29:28,054] jepsen worker 81 - jepsen.util 2601 :invoke :write [27 23] INFO [2019-12-17 12:29:28,072] jepsen worker 63 - jepsen.util 1953 :ok :write [21 6] INFO [2019-12-17 12:29:28,072] jepsen worker 63 - jepsen.util 1953 :invoke :cas [21 [25 31]] INFO [2019-12-17 12:29:28,089] jepsen worker 75 - jepsen.util 3225 :info :cas [25 [45 1]] :RequestCanceledException INFO [2019-12-17 12:29:28,089] jepsen worker 26 - jepsen.util 3266 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:29:28,089] jepsen worker 26 - jepsen.util 3266 :invoke :read [8 nil] INFO [2019-12-17 12:29:28,090] jepsen worker 75 - jepsen.util 3315 :invoke :read [25 nil] INFO [2019-12-17 12:29:28,095] jepsen worker 63 - jepsen.util 1953 :fail :cas [21 [25 31]] :ValueNotSwapFrom INFO [2019-12-17 12:29:28,095] jepsen worker 63 - jepsen.util 1953 :invoke :write [21 0] INFO [2019-12-17 12:29:28,108] jepsen worker 24 - jepsen.util 2814 :info :write [8 9] :RequestCanceledException INFO [2019-12-17 12:29:28,108] jepsen worker 84 - jepsen.util 2694 :info :write [28 5] :RequestCanceledException INFO [2019-12-17 12:29:28,109] jepsen worker 37 - jepsen.util 1927 :fail :cas [12 [38 9]] :ValueNotSwapFrom INFO [2019-12-17 12:29:28,109] jepsen worker 37 - jepsen.util 1927 :invoke :cas [12 [45 32]] INFO [2019-12-17 12:29:28,109] jepsen worker 24 - jepsen.util 2904 :invoke :cas [8 [24 23]] INFO [2019-12-17 12:29:28,109] jepsen worker 84 - jepsen.util 2784 :invoke :cas [28 [18 11]] INFO [2019-12-17 12:29:28,123] jepsen worker 83 - jepsen.util 3323 :info :cas [27 [41 40]] :RequestCanceledException INFO [2019-12-17 12:29:28,123] jepsen worker 86 - jepsen.util 2876 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:29:28,123] jepsen worker 62 - jepsen.util 2762 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:29:28,123] jepsen worker 40 - jepsen.util 3190 :info :cas [13 [37 6]] :RequestCanceledException INFO [2019-12-17 12:29:28,123] jepsen worker 86 - jepsen.util 2876 :invoke :write [28 38] INFO [2019-12-17 12:29:28,123] jepsen worker 62 - jepsen.util 2762 :invoke :read [20 nil] INFO [2019-12-17 12:29:28,124] jepsen worker 83 - jepsen.util 3413 :invoke :read [27 nil] INFO [2019-12-17 12:29:28,124] jepsen worker 40 - jepsen.util 3280 :invoke :read [13 nil] INFO [2019-12-17 12:29:28,137] jepsen worker 58 - jepsen.util 2578 :info :write [19 48] :RequestCanceledException INFO [2019-12-17 12:29:28,137] jepsen worker 68 - jepsen.util 2588 :info :write [22 45] :RequestCanceledException INFO [2019-12-17 12:29:28,137] jepsen worker 63 - jepsen.util 1953 :ok :write [21 0] INFO [2019-12-17 12:29:28,137] jepsen worker 63 - jepsen.util 1953 :invoke :read [21 nil] INFO [2019-12-17 12:29:28,138] jepsen worker 58 - jepsen.util 2668 :invoke :cas [19 [21 8]] INFO [2019-12-17 12:29:28,138] jepsen worker 68 - jepsen.util 2678 :invoke :cas [22 [34 20]] INFO [2019-12-17 12:29:28,148] jepsen worker 63 - jepsen.util 1953 :ok :read [21 0] INFO [2019-12-17 12:29:28,148] jepsen worker 63 - jepsen.util 1953 :invoke :cas [21 [21 26]] INFO [2019-12-17 12:29:28,149] jepsen worker 31 - jepsen.util 2101 :ok :write [10 8] INFO [2019-12-17 12:29:28,149] jepsen worker 31 - jepsen.util 2101 :invoke :cas [10 [27 25]] INFO [2019-12-17 12:29:28,154] jepsen worker 74 - jepsen.util 2054 :info :write [24 13] :RequestCanceledException INFO [2019-12-17 12:29:28,154] jepsen worker 72 - jepsen.util 2322 :ok :write [24 33] INFO [2019-12-17 12:29:28,154] jepsen worker 72 - jepsen.util 2322 :invoke :write [24 15] INFO [2019-12-17 12:29:28,156] jepsen worker 74 - jepsen.util 2144 :invoke :write [24 35] INFO [2019-12-17 12:29:28,176] jepsen worker 63 - jepsen.util 1953 :fail :cas [21 [21 26]] :ValueNotSwapFrom INFO [2019-12-17 12:29:28,176] jepsen worker 63 - jepsen.util 1953 :invoke :read [21 nil] INFO [2019-12-17 12:29:28,178] jepsen worker 14 - jepsen.util 2174 :ok :read [4 9] INFO [2019-12-17 12:29:28,178] jepsen worker 14 - jepsen.util 2174 :invoke :cas [4 [27 33]] INFO [2019-12-17 12:29:28,182] jepsen worker 31 - jepsen.util 2101 :fail :cas [10 [27 25]] :ValueNotSwapFrom INFO [2019-12-17 12:29:28,182] jepsen worker 31 - jepsen.util 2101 :invoke :cas [10 [9 6]] INFO [2019-12-17 12:29:28,184] jepsen worker 10 - jepsen.util 2260 :ok :write [3 10] INFO [2019-12-17 12:29:28,184] jepsen worker 10 - jepsen.util 2260 :invoke :cas [3 [23 35]] INFO [2019-12-17 12:29:28,189] jepsen worker 41 - jepsen.util 3011 :info :write [13 28] :RequestCanceledException INFO [2019-12-17 12:29:28,190] jepsen worker 41 - jepsen.util 3101 :invoke :cas [13 [39 34]] INFO [2019-12-17 12:29:28,196] jepsen worker 63 - jepsen.util 1953 :ok :read [21 0] INFO [2019-12-17 12:29:28,196] jepsen worker 63 - jepsen.util 1953 :invoke :read [21 nil] INFO [2019-12-17 12:29:28,207] jepsen worker 25 - jepsen.util 3355 :info :write [8 46] :RequestCanceledException INFO [2019-12-17 12:29:28,208] jepsen worker 38 - jepsen.util 2198 :fail :cas [12 [46 5]] :ValueNotSwapFrom INFO [2019-12-17 12:29:28,208] jepsen worker 38 - jepsen.util 2198 :invoke :read [12 nil] INFO [2019-12-17 12:29:28,208] jepsen worker 25 - jepsen.util 3445 :invoke :cas [8 [15 19]] INFO [2019-12-17 12:29:28,222] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":12941,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:29:28,222] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":12931,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:29:28,222] jepsen worker 17 - jepsen.util 2987 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:29:28,222] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":12941,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:29:28,222] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":12931,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:29:28,222] jepsen worker 17 - jepsen.util 2987 :invoke :cas [5 [37 15]] INFO [2019-12-17 12:29:28,222] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":12945,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:29:28,222] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":12933,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:29:28,222] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":12945,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:29:28,222] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":12933,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:29:28,226] jepsen worker 19 - jepsen.util 2449 :ok :write [6 45] INFO [2019-12-17 12:29:28,226] jepsen worker 19 - jepsen.util 2449 :invoke :write [6 21] INFO [2019-12-17 12:29:28,237] jepsen worker 32 - jepsen.util 2642 :info :write [10 36] :RequestCanceledException INFO [2019-12-17 12:29:28,237] jepsen worker 48 - jepsen.util 2388 :info :write [16 11] :RequestCanceledException INFO [2019-12-17 12:29:28,237] jepsen worker 46 - jepsen.util 2386 :info :cas [15 [21 39]] :RequestCanceledException INFO [2019-12-17 12:29:28,237] jepsen worker 27 - jepsen.util 2007 :fail :read [9 nil] :RequestTimeoutException INFO [2019-12-17 12:29:28,237] jepsen worker 45 - jepsen.util 2385 :fail :read [15 nil] :RequestTimeoutException INFO [2019-12-17 12:29:28,237] jepsen worker 27 - jepsen.util 2007 :invoke :read [9 nil] INFO [2019-12-17 12:29:28,237] jepsen worker 45 - jepsen.util 2385 :invoke :cas [15 [18 49]] INFO [2019-12-17 12:29:28,238] jepsen worker 48 - jepsen.util 2478 :invoke :read [16 nil] INFO [2019-12-17 12:29:28,238] jepsen worker 46 - jepsen.util 2476 :invoke :cas [15 [37 21]] INFO [2019-12-17 12:29:28,239] jepsen worker 32 - jepsen.util 2732 :invoke :read [10 nil] INFO [2019-12-17 12:29:28,252] jepsen worker 37 - jepsen.util 1927 :fail :cas [12 [45 32]] :ValueNotSwapFrom INFO [2019-12-17 12:29:28,252] jepsen worker 39 - jepsen.util 3099 :info :cas [13 [43 19]] :RequestCanceledException INFO [2019-12-17 12:29:28,252] jepsen worker 61 - jepsen.util 3031 :info :cas [20 [6 42]] :RequestCanceledException INFO [2019-12-17 12:29:28,252] jepsen worker 37 - jepsen.util 1927 :invoke :write [12 30] INFO [2019-12-17 12:29:28,253] jepsen worker 39 - jepsen.util 3189 :invoke :read [13 nil] INFO [2019-12-17 12:29:28,253] jepsen worker 61 - jepsen.util 3121 :invoke :write [20 19] INFO [2019-12-17 12:29:28,269] jepsen worker 31 - jepsen.util 2101 :fail :cas [10 [9 6]] :ValueNotSwapFrom INFO [2019-12-17 12:29:28,270] jepsen worker 31 - jepsen.util 2101 :invoke :cas [10 [10 25]] INFO [2019-12-17 12:29:28,270] jepsen worker 38 - jepsen.util 2198 :ok :read [12 18] INFO [2019-12-17 12:29:28,270] jepsen worker 38 - jepsen.util 2198 :invoke :write [12 40] INFO [2019-12-17 12:29:28,288] jepsen worker 11 - jepsen.util 2801 :fail :cas [3 [32 40]] :ValueNotSwapFrom INFO [2019-12-17 12:29:28,288] jepsen worker 11 - jepsen.util 2801 :invoke :cas [3 [6 35]] INFO [2019-12-17 12:29:28,290] jepsen worker 42 - jepsen.util 3372 :info :cas [14 [4 4]] :RequestCanceledException INFO [2019-12-17 12:29:28,291] jepsen worker 42 - jepsen.util 3462 :invoke :cas [14 [8 12]] INFO [2019-12-17 12:29:28,292] jepsen worker 51 - jepsen.util 2031 :ok :read [17 12] INFO [2019-12-17 12:29:28,292] jepsen worker 51 - jepsen.util 2031 :invoke :cas [17 [34 15]] INFO [2019-12-17 12:29:28,297] jepsen worker 78 - jepsen.util 2418 :fail :cas [26 [46 10]] :ValueNotSwapFrom INFO [2019-12-17 12:29:28,297] jepsen worker 78 - jepsen.util 2418 :invoke :read [26 nil] INFO [2019-12-17 12:29:28,307] jepsen worker 20 - jepsen.util 1820 :info :write [6 15] :RequestCanceledException INFO [2019-12-17 12:29:28,307] jepsen worker 8 - jepsen.util 2708 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:29:28,307] jepsen worker 8 - jepsen.util 2708 :invoke :cas [2 [23 41]] INFO [2019-12-17 12:29:28,308] jepsen worker 20 - jepsen.util 1910 :invoke :write [6 26] INFO [2019-12-17 12:29:28,313] jepsen worker 89 - jepsen.util 2249 :fail :cas [29 [8 37]] :ValueNotSwapFrom INFO [2019-12-17 12:29:28,313] jepsen worker 89 - jepsen.util 2249 :invoke :read [29 nil] INFO [2019-12-17 12:29:28,323] jepsen worker 33 - jepsen.util 3093 :info :cas [11 [10 4]] :RequestCanceledException INFO [2019-12-17 12:29:28,323] jepsen worker 77 - jepsen.util 2777 :info :write [25 38] :RequestCanceledException INFO [2019-12-17 12:29:28,323] jepsen worker 28 - jepsen.util 1648 :fail :read [9 nil] :RequestTimeoutException INFO [2019-12-17 12:29:28,323] jepsen worker 69 - jepsen.util 2139 :info :write [23 46] :RequestCanceledException INFO [2019-12-17 12:29:28,323] jepsen worker 28 - jepsen.util 1648 :invoke :read [9 nil] INFO [2019-12-17 12:29:28,324] jepsen worker 77 - jepsen.util 2867 :invoke :cas [25 [22 29]] INFO [2019-12-17 12:29:28,324] jepsen worker 33 - jepsen.util 3183 :invoke :cas [11 [36 48]] INFO [2019-12-17 12:29:28,324] jepsen worker 69 - jepsen.util 2229 :invoke :write [23 21] INFO [2019-12-17 12:29:28,338] jepsen worker 63 - jepsen.util 1953 :ok :read [21 0] INFO [2019-12-17 12:29:28,338] jepsen worker 63 - jepsen.util 1953 :invoke :cas [21 [38 28]] INFO [2019-12-17 12:29:28,362] jepsen worker 32 - jepsen.util 2732 :ok :read [10 8] INFO [2019-12-17 12:29:28,362] jepsen worker 32 - jepsen.util 2732 :invoke :write [10 8] INFO [2019-12-17 12:29:28,363] jepsen worker 65 - jepsen.util 2045 :ok :read [21 0] INFO [2019-12-17 12:29:28,363] jepsen worker 65 - jepsen.util 2045 :invoke :write [21 44] INFO [2019-12-17 12:29:28,365] jepsen worker 10 - jepsen.util 2260 :fail :cas [3 [23 35]] :ValueNotSwapFrom INFO [2019-12-17 12:29:28,365] jepsen worker 10 - jepsen.util 2260 :invoke :cas [3 [18 6]] INFO [2019-12-17 12:29:28,382] jepsen worker 46 - jepsen.util 2476 :fail :cas [15 [37 21]] :ValueNotSwapFrom INFO [2019-12-17 12:29:28,382] jepsen worker 46 - jepsen.util 2476 :invoke :read [15 nil] INFO [2019-12-17 12:29:28,390] jepsen worker 66 - jepsen.util 2856 :info :cas [22 [6 23]] :RequestCanceledException INFO [2019-12-17 12:29:28,390] jepsen worker 66 - jepsen.util 2946 :invoke :write [22 14] INFO [2019-12-17 12:29:28,402] jepsen worker 28 - jepsen.util 1648 :ok :read [9 21] INFO [2019-12-17 12:29:28,402] jepsen worker 28 - jepsen.util 1648 :invoke :read [9 nil] INFO [2019-12-17 12:29:28,407] jepsen worker 82 - jepsen.util 3412 :info :write [27 17] :RequestCanceledException INFO [2019-12-17 12:29:28,408] jepsen worker 82 - jepsen.util 3502 :invoke :cas [27 [18 41]] INFO [2019-12-17 12:29:28,416] jepsen worker 31 - jepsen.util 2101 :fail :cas [10 [10 25]] :ValueNotSwapFrom INFO [2019-12-17 12:29:28,416] jepsen worker 31 - jepsen.util 2101 :invoke :cas [10 [27 39]] INFO [2019-12-17 12:29:28,420] jepsen worker 63 - jepsen.util 1953 :fail :cas [21 [38 28]] :ValueNotSwapFrom INFO [2019-12-17 12:29:28,421] jepsen worker 63 - jepsen.util 1953 :invoke :write [21 44] INFO [2019-12-17 12:29:28,437] jepsen worker 46 - jepsen.util 2476 :ok :read [15 24] INFO [2019-12-17 12:29:28,437] jepsen worker 46 - jepsen.util 2476 :invoke :read [15 nil] INFO [2019-12-17 12:29:28,441] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":12980,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:29:28,441] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":12980,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:29:28,441] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":12983,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:29:28,441] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":12983,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:29:28,441] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":12984,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:29:28,441] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":12984,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:29:28,443] jepsen worker 53 - jepsen.util 2123 :fail :cas [17 [9 8]] :ValueNotSwapFrom INFO [2019-12-17 12:29:28,444] jepsen worker 53 - jepsen.util 2123 :invoke :write [17 23] INFO [2019-12-17 12:29:28,450] jepsen worker 28 - jepsen.util 1648 :ok :read [9 21] INFO [2019-12-17 12:29:28,450] jepsen worker 28 - jepsen.util 1648 :invoke :write [9 22] INFO [2019-12-17 12:29:28,453] jepsen worker 54 - jepsen.util 2214 :info :write [18 29] :RequestCanceledException INFO [2019-12-17 12:29:28,454] jepsen worker 54 - jepsen.util 2304 :invoke :cas [18 [10 47]] INFO [2019-12-17 12:29:28,468] jepsen worker 31 - jepsen.util 2101 :fail :cas [10 [27 39]] :ValueNotSwapFrom INFO [2019-12-17 12:29:28,468] jepsen worker 31 - jepsen.util 2101 :invoke :cas [10 [49 13]] INFO [2019-12-17 12:29:28,496] jepsen worker 46 - jepsen.util 2476 :ok :read [15 24] INFO [2019-12-17 12:29:28,496] jepsen worker 46 - jepsen.util 2476 :invoke :cas [15 [23 15]] INFO [2019-12-17 12:29:28,502] jepsen worker 70 - jepsen.util 2590 :fail :cas [23 [17 1]] :ValueNotSwapFrom INFO [2019-12-17 12:29:28,502] jepsen worker 70 - jepsen.util 2590 :invoke :write [23 46] INFO [2019-12-17 12:29:28,517] jepsen worker 31 - jepsen.util 2101 :fail :cas [10 [49 13]] :ValueNotSwapFrom INFO [2019-12-17 12:29:28,517] jepsen worker 31 - jepsen.util 2101 :invoke :read [10 nil] INFO [2019-12-17 12:29:28,522] jepsen worker 79 - jepsen.util 2239 :fail :cas [26 [12 15]] :ValueNotSwapFrom INFO [2019-12-17 12:29:28,522] jepsen worker 79 - jepsen.util 2239 :invoke :cas [26 [48 40]] INFO [2019-12-17 12:29:28,522] jepsen worker 2 - jepsen.util 2792 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:29:28,522] jepsen worker 2 - jepsen.util 2792 :invoke :cas [0 [33 4]] INFO [2019-12-17 12:29:28,527] jepsen worker 46 - jepsen.util 2476 :fail :cas [15 [23 15]] :ValueNotSwapFrom INFO [2019-12-17 12:29:28,527] jepsen worker 46 - jepsen.util 2476 :invoke :cas [15 [48 33]] INFO [2019-12-17 12:29:28,537] jepsen worker 63 - jepsen.util 1953 :ok :write [21 44] INFO [2019-12-17 12:29:28,538] jepsen worker 63 - jepsen.util 1953 :invoke :read [21 nil] INFO [2019-12-17 12:29:28,538] jepsen worker 14 - jepsen.util 2174 :fail :cas [4 [27 33]] :ValueNotSwapFrom INFO [2019-12-17 12:29:28,538] jepsen worker 14 - jepsen.util 2174 :invoke :read [4 nil] INFO [2019-12-17 12:29:28,554] jepsen worker 44 - jepsen.util 2654 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:29:28,554] jepsen worker 44 - jepsen.util 2654 :invoke :cas [14 [23 24]] INFO [2019-12-17 12:29:28,558] jepsen worker 31 - jepsen.util 2101 :ok :read [10 8] INFO [2019-12-17 12:29:28,558] jepsen worker 31 - jepsen.util 2101 :invoke :read [10 nil] INFO [2019-12-17 12:29:28,578] jepsen worker 4 - jepsen.util 1804 :fail :cas [1 [9 19]] :ValueNotSwapFrom INFO [2019-12-17 12:29:28,578] jepsen worker 4 - jepsen.util 1804 :invoke :read [1 nil] INFO [2019-12-17 12:29:28,583] jepsen worker 72 - jepsen.util 2322 :ok :write [24 15] INFO [2019-12-17 12:29:28,583] jepsen worker 72 - jepsen.util 2322 :invoke :read [24 nil] INFO [2019-12-17 12:29:28,585] jepsen worker 51 - jepsen.util 2031 :fail :cas [17 [34 15]] :ValueNotSwapFrom INFO [2019-12-17 12:29:28,585] jepsen worker 51 - jepsen.util 2031 :invoke :write [17 26] INFO [2019-12-17 12:29:28,585] jepsen worker 87 - jepsen.util 1977 :info :write [29 21] :RequestCanceledException INFO [2019-12-17 12:29:28,588] jepsen worker 87 - jepsen.util 2067 :invoke :read [29 nil] INFO [2019-12-17 12:29:28,611] jepsen worker 63 - jepsen.util 1953 :ok :read [21 44] INFO [2019-12-17 12:29:28,611] jepsen worker 63 - jepsen.util 1953 :invoke :write [21 2] INFO [2019-12-17 12:29:28,623] jepsen worker 23 - jepsen.util 2633 :info :write [7 20] :RequestCanceledException INFO [2019-12-17 12:29:28,628] jepsen worker 23 - jepsen.util 2723 :invoke :read [7 nil] INFO [2019-12-17 12:29:28,632] jepsen worker 72 - jepsen.util 2322 :ok :read [24 15] INFO [2019-12-17 12:29:28,633] jepsen worker 72 - jepsen.util 2322 :invoke :cas [24 [8 40]] INFO [2019-12-17 12:29:28,645] jepsen worker 46 - jepsen.util 2476 :fail :cas [15 [48 33]] :ValueNotSwapFrom INFO [2019-12-17 12:29:28,645] jepsen worker 46 - jepsen.util 2476 :invoke :cas [15 [9 0]] INFO [2019-12-17 12:29:28,655] jepsen worker 31 - jepsen.util 2101 :ok :read [10 8] INFO [2019-12-17 12:29:28,655] jepsen worker 31 - jepsen.util 2101 :invoke :read [10 nil] INFO [2019-12-17 12:29:28,660] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13010,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:29:28,661] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13010,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:29:28,661] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13012,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:29:28,661] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13012,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:29:28,661] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13013,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:29:28,661] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13013,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:29:28,683] jepsen worker 65 - jepsen.util 2045 :ok :write [21 44] INFO [2019-12-17 12:29:28,683] jepsen worker 65 - jepsen.util 2045 :invoke :cas [21 [1 48]] INFO [2019-12-17 12:29:28,686] jepsen worker 72 - jepsen.util 2322 :fail :cas [24 [8 40]] :ValueNotSwapFrom INFO [2019-12-17 12:29:28,686] jepsen worker 72 - jepsen.util 2322 :invoke :read [24 nil] INFO [2019-12-17 12:29:28,689] jepsen worker 49 - jepsen.util 2209 :info :write [16 44] :RequestCanceledException INFO [2019-12-17 12:29:28,689] jepsen worker 67 - jepsen.util 3397 :info :cas [22 [4 29]] :RequestCanceledException INFO [2019-12-17 12:29:28,689] jepsen worker 4 - jepsen.util 1804 :ok :read [1 26] INFO [2019-12-17 12:29:28,689] jepsen worker 4 - jepsen.util 1804 :invoke :read [1 nil] INFO [2019-12-17 12:29:28,693] jepsen worker 49 - jepsen.util 2299 :invoke :read [16 nil] INFO [2019-12-17 12:29:28,695] jepsen worker 67 - jepsen.util 3487 :invoke :write [22 34] INFO [2019-12-17 12:29:28,699] jepsen worker 30 - jepsen.util 2730 :ok :cas [10 [8 39]] INFO [2019-12-17 12:29:28,700] jepsen worker 30 - jepsen.util 2730 :invoke :cas [10 [46 37]] INFO [2019-12-17 12:29:28,716] jepsen worker 46 - jepsen.util 2476 :fail :cas [15 [9 0]] :ValueNotSwapFrom INFO [2019-12-17 12:29:28,716] jepsen worker 46 - jepsen.util 2476 :invoke :read [15 nil] INFO [2019-12-17 12:29:28,726] jepsen worker 79 - jepsen.util 2239 :fail :cas [26 [48 40]] :ValueNotSwapFrom INFO [2019-12-17 12:29:28,726] jepsen worker 79 - jepsen.util 2239 :invoke :write [26 3] INFO [2019-12-17 12:29:28,746] jepsen worker 72 - jepsen.util 2322 :ok :read [24 15] INFO [2019-12-17 12:29:28,746] jepsen worker 72 - jepsen.util 2322 :invoke :read [24 nil] INFO [2019-12-17 12:29:28,754] jepsen worker 13 - jepsen.util 2083 :info :write [4 48] :RequestCanceledException INFO [2019-12-17 12:29:28,754] jepsen worker 10 - jepsen.util 2260 :fail :cas [3 [18 6]] :ValueNotSwapFrom INFO [2019-12-17 12:29:28,754] jepsen worker 10 - jepsen.util 2260 :invoke :cas [3 [30 46]] INFO [2019-12-17 12:29:28,756] jepsen worker 13 - jepsen.util 2173 :invoke :write [4 18] INFO [2019-12-17 12:29:28,775] jepsen worker 30 - jepsen.util 2730 :fail :cas [10 [46 37]] :ValueNotSwapFrom INFO [2019-12-17 12:29:28,775] jepsen worker 30 - jepsen.util 2730 :invoke :write [10 41] INFO [2019-12-17 12:29:28,789] jepsen worker 18 - jepsen.util 2448 :info :write [6 10] :RequestCanceledException INFO [2019-12-17 12:29:28,793] jepsen worker 18 - jepsen.util 2538 :invoke :read [6 nil] INFO [2019-12-17 12:29:28,807] jepsen worker 34 - jepsen.util 3004 :info :write [11 12] :RequestCanceledException INFO [2019-12-17 12:29:28,809] jepsen worker 34 - jepsen.util 3094 :invoke :cas [11 [42 26]] INFO [2019-12-17 12:29:28,810] jepsen worker 72 - jepsen.util 2322 :ok :read [24 15] INFO [2019-12-17 12:29:28,810] jepsen worker 72 - jepsen.util 2322 :invoke :read [24 nil] INFO [2019-12-17 12:29:28,812] jepsen worker 10 - jepsen.util 2260 :fail :cas [3 [30 46]] :ValueNotSwapFrom INFO [2019-12-17 12:29:28,813] jepsen worker 10 - jepsen.util 2260 :invoke :write [3 21] INFO [2019-12-17 12:29:28,871] jepsen worker 31 - jepsen.util 2101 :ok :read [10 39] INFO [2019-12-17 12:29:28,872] jepsen worker 31 - jepsen.util 2101 :invoke :read [10 nil] INFO [2019-12-17 12:29:28,881] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13034,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:29:28,881] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13034,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:29:28,884] jepsen worker 65 - jepsen.util 2045 :fail :cas [21 [1 48]] :ValueNotSwapFrom INFO [2019-12-17 12:29:28,884] jepsen worker 65 - jepsen.util 2045 :invoke :cas [21 [39 32]] INFO [2019-12-17 12:29:28,886] jepsen worker 46 - jepsen.util 2476 :ok :read [15 24] INFO [2019-12-17 12:29:28,889] jepsen worker 63 - jepsen.util 1953 :ok :write [21 2] INFO [2019-12-17 12:29:28,894] jepsen worker 72 - jepsen.util 2322 :ok :read [24 15] INFO [2019-12-17 12:29:28,909] jepsen worker 31 - jepsen.util 2101 :ok :read [10 39] INFO [2019-12-17 12:29:28,918] jepsen worker 46 - jepsen.util 2476 :invoke :read [15 nil] INFO [2019-12-17 12:29:28,918] jepsen worker 63 - jepsen.util 1953 :invoke :cas [21 [11 39]] INFO [2019-12-17 12:29:28,918] jepsen worker 72 - jepsen.util 2322 :invoke :read [24 nil] INFO [2019-12-17 12:29:28,918] jepsen worker 31 - jepsen.util 2101 :invoke :read [10 nil] INFO [2019-12-17 12:29:28,922] jepsen worker 71 - jepsen.util 1961 :info :write [23 40] :RequestCanceledException INFO [2019-12-17 12:29:28,924] jepsen worker 71 - jepsen.util 2051 :invoke :write [23 7] INFO [2019-12-17 12:29:28,937] jepsen worker 5 - jepsen.util 1895 :info :write [1 46] :RequestCanceledException INFO [2019-12-17 12:29:28,939] jepsen worker 5 - jepsen.util 1985 :invoke :read [1 nil] INFO [2019-12-17 12:29:28,943] jepsen worker 78 - jepsen.util 2418 :ok :read [26 4] INFO [2019-12-17 12:29:28,943] jepsen worker 78 - jepsen.util 2418 :invoke :write [26 26] INFO [2019-12-17 12:29:28,994] jepsen worker 65 - jepsen.util 2045 :fail :cas [21 [39 32]] :ValueNotSwapFrom INFO [2019-12-17 12:29:28,995] jepsen worker 65 - jepsen.util 2045 :invoke :cas [21 [15 25]] INFO [2019-12-17 12:29:29,022] jepsen worker 64 - jepsen.util 1864 :info :write [21 27] :RequestCanceledException INFO [2019-12-17 12:29:29,023] jepsen worker 64 - jepsen.util 1954 :invoke :read [21 nil] INFO [2019-12-17 12:29:29,037] jepsen worker 55 - jepsen.util 2485 :info :write [18 27] :RequestCanceledException INFO [2019-12-17 12:29:29,037] jepsen worker 47 - jepsen.util 2567 :info :write [15 49] :RequestCanceledException INFO [2019-12-17 12:29:29,040] jepsen worker 31 - jepsen.util 2101 :ok :read [10 39] INFO [2019-12-17 12:29:29,040] jepsen worker 31 - jepsen.util 2101 :invoke :cas [10 [39 48]] INFO [2019-12-17 12:29:29,041] jepsen worker 89 - jepsen.util 2249 :ok :read [29 34] INFO [2019-12-17 12:29:29,042] jepsen worker 89 - jepsen.util 2249 :invoke :read [29 nil] INFO [2019-12-17 12:29:29,045] jepsen worker 47 - jepsen.util 2657 :invoke :write [15 23] INFO [2019-12-17 12:29:29,045] jepsen worker 55 - jepsen.util 2575 :invoke :write [18 23] INFO [2019-12-17 12:29:29,052] jepsen worker 46 - jepsen.util 2476 :ok :read [15 24] INFO [2019-12-17 12:29:29,052] jepsen worker 46 - jepsen.util 2476 :invoke :write [15 34] INFO [2019-12-17 12:29:29,058] jepsen worker 63 - jepsen.util 1953 :fail :cas [21 [11 39]] :ValueNotSwapFrom INFO [2019-12-17 12:29:29,059] jepsen worker 63 - jepsen.util 1953 :invoke :read [21 nil] INFO [2019-12-17 12:29:29,059] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:29:29,059] jepsen nemesis - couchbase.util Rebalance check count: 93 INFO [2019-12-17 12:29:29,059] jepsen nemesis - couchbase.util Rebalance stuck count: 74 INFO [2019-12-17 12:29:29,083] jepsen worker 72 - jepsen.util 2322 :ok :read [24 15] INFO [2019-12-17 12:29:29,084] jepsen worker 72 - jepsen.util 2322 :invoke :write [24 5] INFO [2019-12-17 12:29:29,089] jepsen worker 60 - jepsen.util 3120 :info :cas [20 [1 15]] :RequestCanceledException INFO [2019-12-17 12:29:29,096] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13047,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:29:29,097] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13047,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:29:29,097] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13052,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:29:29,097] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13052,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:29:29,099] jepsen worker 60 - jepsen.util 3210 :invoke :write [20 32] INFO [2019-12-17 12:29:29,109] jepsen worker 65 - jepsen.util 2045 :fail :cas [21 [15 25]] :ValueNotSwapFrom INFO [2019-12-17 12:29:29,109] jepsen worker 65 - jepsen.util 2045 :invoke :read [21 nil] INFO [2019-12-17 12:29:29,132] jepsen worker 64 - jepsen.util 1954 :ok :read [21 2] INFO [2019-12-17 12:29:29,132] jepsen worker 64 - jepsen.util 1954 :invoke :read [21 nil] INFO [2019-12-17 12:29:29,146] jepsen worker 63 - jepsen.util 1953 :ok :read [21 2] INFO [2019-12-17 12:29:29,146] jepsen worker 63 - jepsen.util 1953 :invoke :read [21 nil] INFO [2019-12-17 12:29:29,171] jepsen worker 14 - jepsen.util 2174 :ok :read [4 9] INFO [2019-12-17 12:29:29,171] jepsen worker 14 - jepsen.util 2174 :invoke :read [4 nil] INFO [2019-12-17 12:29:29,175] jepsen worker 65 - jepsen.util 2045 :ok :read [21 2] INFO [2019-12-17 12:29:29,176] jepsen worker 65 - jepsen.util 2045 :invoke :cas [21 [30 44]] INFO [2019-12-17 12:29:29,213] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13070,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:29:29,214] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13070,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:29:29,214] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13071,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:29:29,214] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13071,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:29:29,222] jepsen worker 6 - jepsen.util 2976 :info :write [2 13] :RequestCanceledException INFO [2019-12-17 12:29:29,222] jepsen worker 12 - jepsen.util 2352 :info :write [4 26] :RequestCanceledException INFO [2019-12-17 12:29:29,225] jepsen worker 12 - jepsen.util 2442 :invoke :cas [4 [22 21]] INFO [2019-12-17 12:29:29,225] jepsen worker 6 - jepsen.util 3066 :invoke :cas [2 [43 19]] INFO [2019-12-17 12:29:29,230] jepsen worker 65 - jepsen.util 2045 :fail :cas [21 [30 44]] :ValueNotSwapFrom INFO [2019-12-17 12:29:29,230] jepsen worker 65 - jepsen.util 2045 :invoke :write [21 37] INFO [2019-12-17 12:29:29,236] jepsen worker 7 - jepsen.util 2797 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:29:29,236] jepsen worker 7 - jepsen.util 2797 :invoke :write [2 9] INFO [2019-12-17 12:29:29,254] jepsen worker 36 - jepsen.util 2106 :info :write [12 27] :RequestCanceledException INFO [2019-12-17 12:29:29,259] jepsen worker 36 - jepsen.util 2196 :invoke :write [12 46] INFO [2019-12-17 12:29:29,288] jepsen worker 3 - jepsen.util 2073 :info :write [1 33] :RequestCanceledException INFO [2019-12-17 12:29:29,289] jepsen worker 65 - jepsen.util 2045 :ok :write [21 37] INFO [2019-12-17 12:29:29,290] jepsen worker 65 - jepsen.util 2045 :invoke :cas [21 [22 1]] INFO [2019-12-17 12:29:29,291] jepsen worker 3 - jepsen.util 2163 :invoke :cas [1 [48 8]] INFO [2019-12-17 12:29:29,304] jepsen worker 5 - jepsen.util 1985 :ok :read [1 26] INFO [2019-12-17 12:29:29,305] jepsen worker 5 - jepsen.util 1985 :invoke :read [1 nil] INFO [2019-12-17 12:29:29,316] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13076,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:29:29,316] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13076,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:29:29,316] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13079,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:29:29,316] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13079,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:29:29,329] jepsen worker 14 - jepsen.util 2174 :ok :read [4 9] INFO [2019-12-17 12:29:29,329] jepsen worker 14 - jepsen.util 2174 :invoke :cas [4 [26 7]] INFO [2019-12-17 12:29:29,341] jepsen worker 65 - jepsen.util 2045 :fail :cas [21 [22 1]] :ValueNotSwapFrom INFO [2019-12-17 12:29:29,341] jepsen worker 65 - jepsen.util 2045 :invoke :cas [21 [1 9]] INFO [2019-12-17 12:29:29,366] jepsen worker 87 - jepsen.util 2067 :ok :read [29 34] INFO [2019-12-17 12:29:29,367] jepsen worker 87 - jepsen.util 2067 :invoke :cas [29 [2 12]] INFO [2019-12-17 12:29:29,379] jepsen worker 65 - jepsen.util 2045 :fail :cas [21 [1 9]] :ValueNotSwapFrom INFO [2019-12-17 12:29:29,379] jepsen worker 65 - jepsen.util 2045 :invoke :cas [21 [29 7]] INFO [2019-12-17 12:29:29,386] jepsen worker 4 - jepsen.util 1804 :ok :read [1 26] INFO [2019-12-17 12:29:29,386] jepsen worker 4 - jepsen.util 1804 :invoke :read [1 nil] INFO [2019-12-17 12:29:29,400] jepsen worker 5 - jepsen.util 1985 :ok :read [1 26] INFO [2019-12-17 12:29:29,400] jepsen worker 5 - jepsen.util 1985 :invoke :cas [1 [1 19]] INFO [2019-12-17 12:29:29,417] jepsen worker 65 - jepsen.util 2045 :fail :cas [21 [29 7]] :ValueNotSwapFrom INFO [2019-12-17 12:29:29,417] jepsen worker 65 - jepsen.util 2045 :invoke :read [21 nil] INFO [2019-12-17 12:29:29,420] jepsen worker 73 - jepsen.util 2323 :info :write [24 14] :RequestCanceledException INFO [2019-12-17 12:29:29,423] jepsen worker 73 - jepsen.util 2413 :invoke :write [24 32] INFO [2019-12-17 12:29:29,424] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13099,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:29:29,424] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13099,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:29:29,424] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13101,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:29:29,424] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13101,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:29:29,450] jepsen worker 80 - jepsen.util 2150 :info :write [26 6] :RequestCanceledException INFO [2019-12-17 12:29:29,453] jepsen worker 80 - jepsen.util 2240 :invoke :write [26 14] INFO [2019-12-17 12:29:29,458] jepsen worker 65 - jepsen.util 2045 :ok :read [21 37] INFO [2019-12-17 12:29:29,458] jepsen worker 65 - jepsen.util 2045 :invoke :read [21 nil] INFO [2019-12-17 12:29:29,467] jepsen worker 54 - jepsen.util 2304 :fail :cas [18 [10 47]] :ValueNotSwapFrom INFO [2019-12-17 12:29:29,467] jepsen worker 54 - jepsen.util 2304 :invoke :cas [18 [33 36]] INFO [2019-12-17 12:29:29,490] jepsen worker 47 - jepsen.util 2657 :ok :write [15 23] INFO [2019-12-17 12:29:29,491] jepsen worker 47 - jepsen.util 2657 :invoke :cas [15 [7 48]] INFO [2019-12-17 12:29:29,493] jepsen worker 72 - jepsen.util 2322 :ok :write [24 5] INFO [2019-12-17 12:29:29,493] jepsen worker 72 - jepsen.util 2322 :invoke :write [24 49] INFO [2019-12-17 12:29:29,502] jepsen worker 30 - jepsen.util 2730 :ok :write [10 41] INFO [2019-12-17 12:29:29,502] jepsen worker 30 - jepsen.util 2730 :invoke :read [10 nil] INFO [2019-12-17 12:29:29,528] jepsen worker 65 - jepsen.util 2045 :ok :read [21 37] INFO [2019-12-17 12:29:29,539] jepsen worker 65 - jepsen.util 2045 :invoke :write [21 25] INFO [2019-12-17 12:29:29,620] jepsen worker 52 - jepsen.util 2302 :info :write [17 24] :RequestCanceledException INFO [2019-12-17 12:29:29,625] jepsen worker 52 - jepsen.util 2392 :invoke :cas [17 [28 14]] INFO [2019-12-17 12:29:29,637] jepsen worker 29 - jepsen.util 2009 :fail :read [9 nil] :RequestTimeoutException INFO [2019-12-17 12:29:29,637] jepsen worker 29 - jepsen.util 2009 :invoke :write [9 22] INFO [2019-12-17 12:29:29,647] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13114,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:29:29,650] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13114,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:29:29,651] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13115,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:29:29,653] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13115,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:29:29,708] jepsen worker 56 - jepsen.util 2216 :info :cas [18 [23 20]] :RequestCanceledException INFO [2019-12-17 12:29:29,736] jepsen worker 56 - jepsen.util 2306 :invoke :read [18 nil] INFO [2019-12-17 12:29:29,784] jepsen worker 65 - jepsen.util 2045 :ok :write [21 25] INFO [2019-12-17 12:29:29,785] jepsen worker 65 - jepsen.util 2045 :invoke :write [21 11] INFO [2019-12-17 12:29:29,785] jepsen worker 47 - jepsen.util 2657 :fail :cas [15 [7 48]] :ValueNotSwapFrom INFO [2019-12-17 12:29:29,786] jepsen worker 47 - jepsen.util 2657 :invoke :cas [15 [48 30]] INFO [2019-12-17 12:29:29,793] jepsen worker 63 - jepsen.util 1953 :ok :read [21 37] INFO [2019-12-17 12:29:29,794] jepsen worker 63 - jepsen.util 1953 :invoke :read [21 nil] INFO [2019-12-17 12:29:29,797] jepsen worker 64 - jepsen.util 1954 :ok :read [21 37] INFO [2019-12-17 12:29:29,797] jepsen worker 64 - jepsen.util 1954 :invoke :cas [21 [3 38]] INFO [2019-12-17 12:29:29,842] jepsen worker 5 - jepsen.util 1985 :fail :cas [1 [1 19]] :ValueNotSwapFrom INFO [2019-12-17 12:29:29,842] jepsen worker 5 - jepsen.util 1985 :invoke :cas [1 [6 11]] INFO [2019-12-17 12:29:29,851] jepsen worker 16 - jepsen.util 2896 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:29:29,852] jepsen worker 16 - jepsen.util 2896 :invoke :cas [5 [37 14]] INFO [2019-12-17 12:29:29,852] jepsen worker 12 - jepsen.util 2442 :fail :cas [4 [22 21]] :ValueNotSwapFrom INFO [2019-12-17 12:29:29,853] jepsen worker 12 - jepsen.util 2442 :invoke :cas [4 [20 17]] INFO [2019-12-17 12:29:29,863] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13126,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:29:29,863] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13126,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:29:29,863] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13131,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:29:29,863] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13131,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:29:29,876] jepsen worker 89 - jepsen.util 2249 :ok :read [29 34] INFO [2019-12-17 12:29:29,877] jepsen worker 89 - jepsen.util 2249 :invoke :read [29 nil] INFO [2019-12-17 12:29:29,884] jepsen worker 47 - jepsen.util 2657 :fail :cas [15 [48 30]] :ValueNotSwapFrom INFO [2019-12-17 12:29:29,885] jepsen worker 47 - jepsen.util 2657 :invoke :cas [15 [45 9]] INFO [2019-12-17 12:29:29,885] jepsen worker 73 - jepsen.util 2413 :ok :write [24 32] INFO [2019-12-17 12:29:29,885] jepsen worker 73 - jepsen.util 2413 :invoke :write [24 11] INFO [2019-12-17 12:29:29,886] jepsen worker 29 - jepsen.util 2009 :ok :write [9 22] INFO [2019-12-17 12:29:29,886] jepsen worker 29 - jepsen.util 2009 :invoke :read [9 nil] INFO [2019-12-17 12:29:29,887] jepsen worker 88 - jepsen.util 2158 :info :write [29 12] :RequestCanceledException INFO [2019-12-17 12:29:29,889] jepsen worker 88 - jepsen.util 2248 :invoke :write [29 28] INFO [2019-12-17 12:29:29,903] jepsen worker 64 - jepsen.util 1954 :fail :cas [21 [3 38]] :ValueNotSwapFrom INFO [2019-12-17 12:29:29,903] jepsen worker 64 - jepsen.util 1954 :invoke :write [21 47] INFO [2019-12-17 12:29:29,908] jepsen worker 63 - jepsen.util 1953 :ok :read [21 25] INFO [2019-12-17 12:29:29,908] jepsen worker 63 - jepsen.util 1953 :invoke :cas [21 [42 3]] INFO [2019-12-17 12:29:29,922] jepsen worker 9 - jepsen.util 2529 :info :write [3 18] :RequestCanceledException INFO [2019-12-17 12:29:29,926] jepsen worker 9 - jepsen.util 2619 :invoke :cas [3 [39 22]] INFO [2019-12-17 12:29:29,937] jepsen worker 15 - jepsen.util 2715 :info :write [5 19] :RequestCanceledException INFO [2019-12-17 12:29:29,943] jepsen worker 15 - jepsen.util 2805 :invoke :read [5 nil] INFO [2019-12-17 12:29:29,998] jepsen worker 65 - jepsen.util 2045 :ok :write [21 11] INFO [2019-12-17 12:29:29,998] jepsen worker 65 - jepsen.util 2045 :invoke :write [21 31] INFO [2019-12-17 12:29:30,007] jepsen worker 21 - jepsen.util 2991 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:29:30,008] jepsen worker 21 - jepsen.util 2991 :invoke :write [7 12] INFO [2019-12-17 12:29:30,013] jepsen worker 47 - jepsen.util 2657 :fail :cas [15 [45 9]] :ValueNotSwapFrom INFO [2019-12-17 12:29:30,014] jepsen worker 47 - jepsen.util 2657 :invoke :read [15 nil] INFO [2019-12-17 12:29:30,020] jepsen worker 43 - jepsen.util 2833 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:29:30,020] jepsen worker 35 - jepsen.util 2555 :info :cas [11 [32 2]] :RequestCanceledException INFO [2019-12-17 12:29:30,020] jepsen worker 43 - jepsen.util 2833 :invoke :read [14 nil] INFO [2019-12-17 12:29:30,028] jepsen worker 56 - jepsen.util 2306 :ok :read [18 23] INFO [2019-12-17 12:29:30,028] jepsen worker 56 - jepsen.util 2306 :invoke :read [18 nil] INFO [2019-12-17 12:29:30,029] jepsen worker 35 - jepsen.util 2645 :invoke :cas [11 [5 17]] INFO [2019-12-17 12:29:30,035] jepsen worker 0 - jepsen.util 3150 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:29:30,039] jepsen worker 0 - jepsen.util 3150 :invoke :cas [0 [25 35]] INFO [2019-12-17 12:29:30,039] jepsen worker 57 - jepsen.util 2757 :info :cas [19 [25 47]] :RequestCanceledException INFO [2019-12-17 12:29:30,044] jepsen worker 57 - jepsen.util 2847 :invoke :write [19 45] INFO [2019-12-17 12:29:30,061] jepsen worker 59 - jepsen.util 2759 :info :cas [19 [32 44]] :RequestCanceledException INFO [2019-12-17 12:29:30,068] jepsen worker 9 - jepsen.util 2619 :fail :cas [3 [39 22]] :ValueNotSwapFrom INFO [2019-12-17 12:29:30,068] jepsen worker 9 - jepsen.util 2619 :invoke :cas [3 [31 13]] INFO [2019-12-17 12:29:30,072] jepsen worker 64 - jepsen.util 1954 :ok :write [21 47] INFO [2019-12-17 12:29:30,073] jepsen worker 64 - jepsen.util 1954 :invoke :write [21 7] INFO [2019-12-17 12:29:30,081] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13145,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:29:30,082] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13145,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:29:30,083] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13146,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:29:30,084] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13146,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:29:30,084] jepsen worker 59 - jepsen.util 2849 :invoke :read [19 nil] INFO [2019-12-17 12:29:30,088] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:29:30,089] jepsen nemesis - couchbase.util Rebalance check count: 94 INFO [2019-12-17 12:29:30,090] jepsen worker 76 - jepsen.util 2956 :info :write [25 6] :RequestCanceledException INFO [2019-12-17 12:29:30,090] jepsen nemesis - couchbase.util Rebalance stuck count: 75 INFO [2019-12-17 12:29:30,105] jepsen worker 63 - jepsen.util 1953 :fail :cas [21 [42 3]] :ValueNotSwapFrom INFO [2019-12-17 12:29:30,106] jepsen worker 63 - jepsen.util 1953 :invoke :write [21 38] INFO [2019-12-17 12:29:30,108] jepsen worker 81 - jepsen.util 2601 :info :write [27 23] :RequestCanceledException INFO [2019-12-17 12:29:30,118] jepsen worker 76 - jepsen.util 3046 :invoke :write [25 48] INFO [2019-12-17 12:29:30,124] jepsen worker 75 - jepsen.util 3315 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:29:30,124] jepsen worker 75 - jepsen.util 3315 :invoke :read [25 nil] INFO [2019-12-17 12:29:30,124] jepsen worker 1 - jepsen.util 2791 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:29:30,124] jepsen worker 50 - jepsen.util 3200 :info :write [16 40] :RequestCanceledException INFO [2019-12-17 12:29:30,125] jepsen worker 1 - jepsen.util 2791 :invoke :cas [0 [47 17]] INFO [2019-12-17 12:29:30,133] jepsen worker 81 - jepsen.util 2691 :invoke :cas [27 [19 5]] INFO [2019-12-17 12:29:30,137] jepsen worker 24 - jepsen.util 2904 :info :cas [8 [24 23]] :RequestCanceledException INFO [2019-12-17 12:29:30,140] jepsen worker 50 - jepsen.util 3290 :invoke :cas [16 [47 26]] INFO [2019-12-17 12:29:30,154] jepsen worker 85 - jepsen.util 3235 :info :write [28 25] :RequestCanceledException INFO [2019-12-17 12:29:30,156] jepsen worker 84 - jepsen.util 2784 :info :cas [28 [18 11]] :RequestCanceledException INFO [2019-12-17 12:29:30,156] jepsen worker 22 - jepsen.util 2902 :info :write [7 37] :RequestCanceledException INFO [2019-12-17 12:29:30,156] jepsen worker 24 - jepsen.util 2994 :invoke :read [8 nil] INFO [2019-12-17 12:29:30,189] jepsen worker 26 - jepsen.util 3266 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:29:30,190] jepsen worker 26 - jepsen.util 3266 :invoke :write [8 0] INFO [2019-12-17 12:29:30,190] jepsen worker 85 - jepsen.util 3325 :invoke :read [28 nil] INFO [2019-12-17 12:29:30,191] jepsen worker 84 - jepsen.util 2874 :invoke :cas [28 [5 28]] INFO [2019-12-17 12:29:30,191] jepsen worker 40 - jepsen.util 3280 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:29:30,191] jepsen worker 40 - jepsen.util 3280 :invoke :write [13 35] INFO [2019-12-17 12:29:30,192] jepsen worker 22 - jepsen.util 2992 :invoke :write [7 7] INFO [2019-12-17 12:29:30,196] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13162,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:29:30,199] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13162,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:29:30,200] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13165,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:29:30,200] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13165,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:29:30,212] jepsen worker 58 - jepsen.util 2668 :info :cas [19 [21 8]] :RequestCanceledException INFO [2019-12-17 12:29:30,223] jepsen worker 83 - jepsen.util 3413 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:29:30,223] jepsen worker 62 - jepsen.util 2762 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:29:30,223] jepsen worker 83 - jepsen.util 3413 :invoke :cas [27 [15 33]] INFO [2019-12-17 12:29:30,223] jepsen worker 62 - jepsen.util 2762 :invoke :cas [20 [40 5]] INFO [2019-12-17 12:29:30,224] jepsen worker 25 - jepsen.util 3445 :info :cas [8 [15 19]] :RequestCanceledException INFO [2019-12-17 12:29:30,224] jepsen worker 86 - jepsen.util 2876 :info :write [28 38] :RequestCanceledException INFO [2019-12-17 12:29:30,227] jepsen worker 68 - jepsen.util 2678 :info :cas [22 [34 20]] :RequestCanceledException INFO [2019-12-17 12:29:30,229] jepsen worker 58 - jepsen.util 2758 :invoke :read [19 nil] INFO [2019-12-17 12:29:30,238] jepsen worker 74 - jepsen.util 2144 :info :write [24 35] :RequestCanceledException INFO [2019-12-17 12:29:30,242] jepsen worker 25 - jepsen.util 3535 :invoke :read [8 nil] INFO [2019-12-17 12:29:30,248] jepsen worker 86 - jepsen.util 2966 :invoke :cas [28 [25 23]] INFO [2019-12-17 12:29:30,252] jepsen worker 68 - jepsen.util 2768 :invoke :write [22 18] INFO [2019-12-17 12:29:30,256] jepsen worker 41 - jepsen.util 3101 :info :cas [13 [39 34]] :RequestCanceledException INFO [2019-12-17 12:29:30,271] jepsen worker 74 - jepsen.util 2234 :invoke :read [24 nil] INFO [2019-12-17 12:29:30,274] jepsen worker 65 - jepsen.util 2045 :ok :write [21 31] INFO [2019-12-17 12:29:30,275] jepsen worker 65 - jepsen.util 2045 :invoke :cas [21 [19 48]] INFO [2019-12-17 12:29:30,281] jepsen worker 14 - jepsen.util 2174 :fail :cas [4 [26 7]] :ValueNotSwapFrom INFO [2019-12-17 12:29:30,282] jepsen worker 14 - jepsen.util 2174 :invoke :write [4 20] INFO [2019-12-17 12:29:30,296] jepsen worker 48 - jepsen.util 2478 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:29:30,297] jepsen worker 48 - jepsen.util 2478 :invoke :read [16 nil] INFO [2019-12-17 12:29:30,299] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13173,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:29:30,300] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13173,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:29:30,305] jepsen worker 41 - jepsen.util 3191 :invoke :cas [13 [32 43]] INFO [2019-12-17 12:29:30,305] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13175,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:29:30,308] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13175,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:29:30,308] jepsen worker 37 - jepsen.util 1927 :info :write [12 30] :RequestCanceledException INFO [2019-12-17 12:29:30,310] jepsen worker 38 - jepsen.util 2198 :info :write [12 40] :RequestCanceledException INFO [2019-12-17 12:29:30,320] jepsen worker 42 - jepsen.util 3462 :info :cas [14 [8 12]] :RequestCanceledException INFO [2019-12-17 12:29:30,320] jepsen worker 17 - jepsen.util 2987 :info :cas [5 [37 15]] :RequestCanceledException INFO [2019-12-17 12:29:30,322] jepsen worker 37 - jepsen.util 2017 :invoke :read [12 nil] INFO [2019-12-17 12:29:30,324] jepsen worker 38 - jepsen.util 2288 :invoke :read [12 nil] INFO [2019-12-17 12:29:30,336] jepsen worker 19 - jepsen.util 2449 :info :write [6 21] :RequestCanceledException INFO [2019-12-17 12:29:30,336] jepsen worker 27 - jepsen.util 2007 :fail :read [9 nil] :RequestTimeoutException INFO [2019-12-17 12:29:30,337] jepsen worker 27 - jepsen.util 2007 :invoke :cas [9 [1 4]] INFO [2019-12-17 12:29:30,337] jepsen worker 45 - jepsen.util 2385 :info :cas [15 [18 49]] :RequestCanceledException INFO [2019-12-17 12:29:30,337] jepsen worker 39 - jepsen.util 3189 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:29:30,337] jepsen worker 39 - jepsen.util 3189 :invoke :write [13 7] INFO [2019-12-17 12:29:30,338] jepsen worker 45 - jepsen.util 2475 :invoke :write [15 12] INFO [2019-12-17 12:29:30,349] jepsen worker 42 - jepsen.util 3552 :invoke :cas [14 [18 3]] INFO [2019-12-17 12:29:30,350] jepsen worker 17 - jepsen.util 3077 :invoke :read [5 nil] INFO [2019-12-17 12:29:30,357] jepsen worker 19 - jepsen.util 2539 :invoke :write [6 1] INFO [2019-12-17 12:29:30,359] jepsen worker 61 - jepsen.util 3121 :info :write [20 19] :RequestCanceledException INFO [2019-12-17 12:29:30,389] jepsen worker 11 - jepsen.util 2801 :info :cas [3 [6 35]] :RequestCanceledException INFO [2019-12-17 12:29:30,389] jepsen worker 20 - jepsen.util 1910 :info :write [6 26] :RequestCanceledException INFO [2019-12-17 12:29:30,391] jepsen worker 61 - jepsen.util 3211 :invoke :write [20 45] INFO [2019-12-17 12:29:30,400] jepsen worker 63 - jepsen.util 1953 :ok :write [21 38] INFO [2019-12-17 12:29:30,400] jepsen worker 63 - jepsen.util 1953 :invoke :read [21 nil] INFO [2019-12-17 12:29:30,408] jepsen worker 8 - jepsen.util 2708 :info :cas [2 [23 41]] :RequestCanceledException INFO [2019-12-17 12:29:30,412] jepsen worker 77 - jepsen.util 2867 :info :cas [25 [22 29]] :RequestCanceledException INFO [2019-12-17 12:29:30,423] jepsen worker 82 - jepsen.util 3502 :info :cas [27 [18 41]] :RequestCanceledException INFO [2019-12-17 12:29:30,424] jepsen worker 33 - jepsen.util 3183 :info :cas [11 [36 48]] :RequestCanceledException INFO [2019-12-17 12:29:30,426] jepsen worker 32 - jepsen.util 2732 :info :write [10 8] :RequestCanceledException INFO [2019-12-17 12:29:30,431] jepsen worker 20 - jepsen.util 2000 :invoke :read [6 nil] INFO [2019-12-17 12:29:30,437] jepsen worker 69 - jepsen.util 2229 :info :write [23 21] :RequestCanceledException INFO [2019-12-17 12:29:30,437] jepsen worker 11 - jepsen.util 2891 :invoke :write [3 27] INFO [2019-12-17 12:29:30,440] jepsen worker 8 - jepsen.util 2798 :invoke :read [2 nil] INFO [2019-12-17 12:29:30,447] jepsen worker 77 - jepsen.util 2957 :invoke :cas [25 [1 42]] INFO [2019-12-17 12:29:30,454] jepsen worker 66 - jepsen.util 2946 :info :write [22 14] :RequestCanceledException INFO [2019-12-17 12:29:30,458] jepsen worker 32 - jepsen.util 2822 :invoke :cas [10 [20 15]] INFO [2019-12-17 12:29:30,462] jepsen worker 69 - jepsen.util 2319 :invoke :write [23 32] INFO [2019-12-17 12:29:30,470] jepsen worker 82 - jepsen.util 3592 :invoke :cas [27 [7 28]] INFO [2019-12-17 12:29:30,472] jepsen worker 47 - jepsen.util 2657 :ok :read [15 23] INFO [2019-12-17 12:29:30,473] jepsen worker 47 - jepsen.util 2657 :invoke :read [15 nil] INFO [2019-12-17 12:29:30,476] jepsen worker 33 - jepsen.util 3273 :invoke :read [11 nil] INFO [2019-12-17 12:29:30,491] jepsen worker 66 - jepsen.util 3036 :invoke :cas [22 [11 35]] INFO [2019-12-17 12:29:30,522] jepsen worker 3 - jepsen.util 2163 :fail :cas [1 [48 8]] :ValueNotSwapFrom INFO [2019-12-17 12:29:30,522] jepsen worker 3 - jepsen.util 2163 :invoke :read [1 nil] INFO [2019-12-17 12:29:30,522] jepsen worker 53 - jepsen.util 2123 :info :write [17 23] :RequestCanceledException INFO [2019-12-17 12:29:30,524] jepsen worker 28 - jepsen.util 1648 :info :write [9 22] :RequestCanceledException INFO [2019-12-17 12:29:30,526] jepsen worker 52 - jepsen.util 2392 :fail :cas [17 [28 14]] :ValueNotSwapFrom INFO [2019-12-17 12:29:30,527] jepsen worker 52 - jepsen.util 2392 :invoke :write [17 23] INFO [2019-12-17 12:29:30,527] jepsen worker 70 - jepsen.util 2590 :info :write [23 46] :RequestCanceledException INFO [2019-12-17 12:29:30,529] jepsen worker 70 - jepsen.util 2680 :invoke :read [23 nil] INFO [2019-12-17 12:29:30,534] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13198,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:29:30,535] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13198,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:29:30,535] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13202,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:29:30,537] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13202,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:29:30,592] jepsen worker 53 - jepsen.util 2213 :invoke :cas [17 [34 14]] INFO [2019-12-17 12:29:30,593] jepsen worker 28 - jepsen.util 1738 :invoke :read [9 nil] INFO [2019-12-17 12:29:30,622] jepsen worker 51 - jepsen.util 2031 :info :write [17 26] :RequestCanceledException INFO [2019-12-17 12:29:30,623] jepsen worker 2 - jepsen.util 2792 :info :cas [0 [33 4]] :RequestCanceledException INFO [2019-12-17 12:29:30,654] jepsen worker 51 - jepsen.util 2121 :invoke :read [17 nil] INFO [2019-12-17 12:29:30,655] jepsen worker 44 - jepsen.util 2654 :info :cas [14 [23 24]] :RequestCanceledException INFO [2019-12-17 12:29:30,656] jepsen worker 2 - jepsen.util 2882 :invoke :cas [0 [11 17]] INFO [2019-12-17 12:29:30,669] jepsen worker 73 - jepsen.util 2413 :ok :write [24 11] INFO [2019-12-17 12:29:30,671] jepsen worker 73 - jepsen.util 2413 :invoke :cas [24 [45 21]] INFO [2019-12-17 12:29:30,676] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13220,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:29:30,679] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13220,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:29:30,682] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13225,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:29:30,685] jepsen worker 44 - jepsen.util 2744 :invoke :write [14 34] INFO [2019-12-17 12:29:30,685] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13225,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:29:30,702] jepsen worker 9 - jepsen.util 2619 :fail :cas [3 [31 13]] :ValueNotSwapFrom INFO [2019-12-17 12:29:30,703] jepsen worker 9 - jepsen.util 2619 :invoke :read [3 nil] INFO [2019-12-17 12:29:30,712] jepsen worker 64 - jepsen.util 1954 :ok :write [21 7] INFO [2019-12-17 12:29:30,713] jepsen worker 64 - jepsen.util 1954 :invoke :write [21 31] INFO [2019-12-17 12:29:31,259] jepsen worker 13 - jepsen.util 2173 :info :write [4 18] :RequestCanceledException INFO [2019-12-17 12:29:31,259] jepsen worker 79 - jepsen.util 2239 :info :write [26 3] :RequestCanceledException INFO [2019-12-17 12:29:31,259] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13247,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:29:31,259] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13244,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:29:31,259] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13234,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:29:31,259] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13247,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:29:31,259] jepsen worker 18 - jepsen.util 2538 :fail :read [6 nil] :RequestTimeoutException INFO [2019-12-17 12:29:31,260] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13239,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:29:31,260] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13234,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:29:31,260] jepsen worker 10 - jepsen.util 2260 :info :write [3 21] :RequestCanceledException INFO [2019-12-17 12:29:31,260] jepsen worker 55 - jepsen.util 2575 :info :write [18 23] :RequestCanceledException INFO [2019-12-17 12:29:31,260] jepsen worker 31 - jepsen.util 2101 :info :cas [10 [39 48]] :RequestCanceledException INFO [2019-12-17 12:29:31,260] jepsen worker 78 - jepsen.util 2418 :info :write [26 26] :RequestCanceledException INFO [2019-12-17 12:29:31,260] jepsen worker 67 - jepsen.util 3487 :info :write [22 34] :RequestCanceledException INFO [2019-12-17 12:29:31,261] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13250,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:29:31,261] jepsen worker 79 - jepsen.util 2329 :invoke :write [26 26] INFO [2019-12-17 12:29:31,261] jepsen worker 60 - jepsen.util 3210 :info :write [20 32] :RequestCanceledException INFO [2019-12-17 12:29:31,261] jepsen worker 46 - jepsen.util 2476 :info :write [15 34] :RequestCanceledException INFO [2019-12-17 12:29:31,261] jepsen worker 34 - jepsen.util 3094 :info :cas [11 [42 26]] :RequestCanceledException INFO [2019-12-17 12:29:31,261] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13244,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:29:31,261] jepsen worker 18 - jepsen.util 2538 :invoke :cas [6 [38 25]] INFO [2019-12-17 12:29:31,261] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13239,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:29:31,261] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13237,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:29:31,262] jepsen worker 13 - jepsen.util 2263 :invoke :write [4 18] INFO [2019-12-17 12:29:31,264] jepsen worker 54 - jepsen.util 2304 :fail :cas [18 [33 36]] :ValueNotSwapFrom INFO [2019-12-17 12:29:31,264] jepsen worker 31 - jepsen.util 2191 :invoke :read [10 nil] INFO [2019-12-17 12:29:31,265] jepsen worker 71 - jepsen.util 2051 :info :write [23 7] :RequestCanceledException INFO [2019-12-17 12:29:31,265] jepsen worker 78 - jepsen.util 2508 :invoke :cas [26 [12 1]] INFO [2019-12-17 12:29:31,265] jepsen worker 23 - jepsen.util 2723 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:29:31,267] jepsen worker 65 - jepsen.util 2045 :fail :cas [21 [19 48]] :ValueNotSwapFrom INFO [2019-12-17 12:29:31,267] jepsen worker 67 - jepsen.util 3577 :invoke :read [22 nil] INFO [2019-12-17 12:29:31,267] jepsen worker 38 - jepsen.util 2288 :ok :read [12 18] INFO [2019-12-17 12:29:31,270] jepsen worker 12 - jepsen.util 2442 :fail :cas [4 [20 17]] :ValueNotSwapFrom INFO [2019-12-17 12:29:31,271] jepsen worker 27 - jepsen.util 2007 :fail :cas [9 [1 4]] :ValueNotSwapFrom INFO [2019-12-17 12:29:31,272] jepsen worker 74 - jepsen.util 2234 :ok :read [24 32] INFO [2019-12-17 12:29:31,273] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13250,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:29:31,273] jepsen worker 5 - jepsen.util 1985 :fail :cas [1 [6 11]] :ValueNotSwapFrom INFO [2019-12-17 12:29:31,273] jepsen worker 89 - jepsen.util 2249 :ok :read [29 34] INFO [2019-12-17 12:29:31,274] jepsen worker 49 - jepsen.util 2299 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:29:31,274] jepsen worker 6 - jepsen.util 3066 :info :cas [2 [43 19]] :RequestCanceledException INFO [2019-12-17 12:29:31,274] jepsen worker 7 - jepsen.util 2797 :info :write [2 9] :RequestCanceledException INFO [2019-12-17 12:29:31,276] jepsen worker 46 - jepsen.util 2566 :invoke :cas [15 [24 10]] INFO [2019-12-17 12:29:31,276] jepsen worker 60 - jepsen.util 3300 :invoke :write [20 0] INFO [2019-12-17 12:29:31,276] jepsen worker 47 - jepsen.util 2657 :ok :read [15 23] INFO [2019-12-17 12:29:31,277] jepsen worker 28 - jepsen.util 1738 :ok :read [9 22] INFO [2019-12-17 12:29:31,278] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13249,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:29:31,278] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13248,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:29:31,278] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13237,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:29:31,279] jepsen worker 34 - jepsen.util 3184 :invoke :read [11 nil] INFO [2019-12-17 12:29:31,279] jepsen worker 54 - jepsen.util 2304 :invoke :write [18 13] INFO [2019-12-17 12:29:31,280] jepsen worker 70 - jepsen.util 2680 :ok :read [23 33] INFO [2019-12-17 12:29:31,280] jepsen worker 23 - jepsen.util 2723 :invoke :write [7 41] INFO [2019-12-17 12:29:31,280] jepsen worker 65 - jepsen.util 2045 :invoke :write [21 35] INFO [2019-12-17 12:29:31,281] jepsen worker 71 - jepsen.util 2141 :invoke :cas [23 [35 39]] INFO [2019-12-17 12:29:31,281] jepsen worker 56 - jepsen.util 2306 :ok :read [18 23] INFO [2019-12-17 12:29:31,281] jepsen worker 55 - jepsen.util 2665 :invoke :read [18 nil] INFO [2019-12-17 12:29:31,282] jepsen worker 38 - jepsen.util 2288 :invoke :cas [12 [0 27]] INFO [2019-12-17 12:29:31,282] jepsen worker 12 - jepsen.util 2442 :invoke :read [4 nil] INFO [2019-12-17 12:29:31,282] jepsen worker 74 - jepsen.util 2234 :invoke :read [24 nil] INFO [2019-12-17 12:29:31,282] jepsen worker 27 - jepsen.util 2007 :invoke :cas [9 [17 20]] INFO [2019-12-17 12:29:31,283] jepsen worker 5 - jepsen.util 1985 :invoke :write [1 19] INFO [2019-12-17 12:29:31,283] jepsen worker 89 - jepsen.util 2249 :invoke :write [29 17] INFO [2019-12-17 12:29:31,283] jepsen worker 49 - jepsen.util 2299 :invoke :read [16 nil] INFO [2019-12-17 12:29:31,283] jepsen worker 63 - jepsen.util 1953 :ok :read [21 38] INFO [2019-12-17 12:29:31,284] jepsen worker 47 - jepsen.util 2657 :invoke :write [15 41] INFO [2019-12-17 12:29:31,284] jepsen worker 28 - jepsen.util 1738 :invoke :write [9 15] INFO [2019-12-17 12:29:31,284] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13249,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:29:31,285] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13248,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:29:31,285] jepsen worker 7 - jepsen.util 2887 :invoke :write [2 43] INFO [2019-12-17 12:29:31,286] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13246,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:29:31,286] jepsen worker 6 - jepsen.util 3156 :invoke :write [2 27] INFO [2019-12-17 12:29:31,287] jepsen worker 70 - jepsen.util 2680 :invoke :write [23 46] INFO [2019-12-17 12:29:31,288] jepsen worker 10 - jepsen.util 2350 :invoke :cas [3 [26 4]] INFO [2019-12-17 12:29:31,288] jepsen worker 56 - jepsen.util 2306 :invoke :cas [18 [28 15]] INFO [2019-12-17 12:29:31,289] jepsen worker 32 - jepsen.util 2822 :fail :cas [10 [20 15]] :ValueNotSwapFrom INFO [2019-12-17 12:29:31,289] jepsen worker 63 - jepsen.util 1953 :invoke :cas [21 [21 33]] INFO [2019-12-17 12:29:31,290] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13252,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:29:31,290] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13251,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:29:31,290] jepsen worker 4 - jepsen.util 1804 :ok :read [1 26] INFO [2019-12-17 12:29:31,290] jepsen worker 20 - jepsen.util 2000 :ok :read [6 45] INFO [2019-12-17 12:29:31,291] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13246,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:29:31,292] jepsen worker 32 - jepsen.util 2822 :invoke :write [10 23] INFO [2019-12-17 12:29:31,293] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13251,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:29:31,293] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13252,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:29:31,294] jepsen worker 4 - jepsen.util 1804 :invoke :cas [1 [0 45]] INFO [2019-12-17 12:29:31,295] jepsen worker 20 - jepsen.util 2000 :invoke :write [6 36] INFO [2019-12-17 12:29:31,298] jepsen worker 87 - jepsen.util 2067 :fail :cas [29 [2 12]] :ValueNotSwapFrom INFO [2019-12-17 12:29:31,298] jepsen worker 87 - jepsen.util 2067 :invoke :write [29 10] INFO [2019-12-17 12:29:31,323] jepsen worker 73 - jepsen.util 2413 :fail :cas [24 [45 21]] :ValueNotSwapFrom INFO [2019-12-17 12:29:31,324] jepsen worker 73 - jepsen.util 2413 :invoke :read [24 nil] INFO [2019-12-17 12:29:31,353] jepsen worker 36 - jepsen.util 2196 :info :write [12 46] :RequestCanceledException INFO [2019-12-17 12:29:31,354] jepsen worker 36 - jepsen.util 2286 :invoke :cas [12 [38 6]] INFO [2019-12-17 12:29:31,435] jepsen worker 18 - jepsen.util 2538 :fail :cas [6 [38 25]] :ValueNotSwapFrom INFO [2019-12-17 12:29:31,435] jepsen worker 18 - jepsen.util 2538 :invoke :cas [6 [39 41]] INFO [2019-12-17 12:29:31,437] jepsen worker 9 - jepsen.util 2619 :ok :read [3 10] INFO [2019-12-17 12:29:31,437] jepsen worker 9 - jepsen.util 2619 :invoke :cas [3 [42 38]] INFO [2019-12-17 12:29:31,478] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13296,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:29:31,479] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13296,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:29:31,479] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13299,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:29:31,479] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13299,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:29:31,486] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:29:31,487] jepsen nemesis - couchbase.util Rebalance check count: 95 INFO [2019-12-17 12:29:31,487] jepsen nemesis - couchbase.util Rebalance stuck count: 76 INFO [2019-12-17 12:29:31,488] jepsen worker 74 - jepsen.util 2234 :ok :read [24 11] INFO [2019-12-17 12:29:31,488] jepsen worker 74 - jepsen.util 2234 :invoke :cas [24 [21 44]] INFO [2019-12-17 12:29:31,498] jepsen worker 31 - jepsen.util 2191 :ok :read [10 41] INFO [2019-12-17 12:29:31,499] jepsen worker 31 - jepsen.util 2191 :invoke :write [10 1] INFO [2019-12-17 12:29:31,507] jepsen worker 80 - jepsen.util 2240 :info :write [26 14] :RequestCanceledException INFO [2019-12-17 12:29:31,507] jepsen worker 80 - jepsen.util 2330 :invoke :read [26 nil] INFO [2019-12-17 12:29:31,524] jepsen worker 80 - jepsen.util 2330 :ok :read [26 4] INFO [2019-12-17 12:29:31,524] jepsen worker 80 - jepsen.util 2330 :invoke :write [26 38] INFO [2019-12-17 12:29:31,534] jepsen worker 27 - jepsen.util 2007 :fail :cas [9 [17 20]] :ValueNotSwapFrom INFO [2019-12-17 12:29:31,534] jepsen worker 27 - jepsen.util 2007 :invoke :write [9 37] INFO [2019-12-17 12:29:31,544] jepsen worker 53 - jepsen.util 2213 :fail :cas [17 [34 14]] :ValueNotSwapFrom INFO [2019-12-17 12:29:31,544] jepsen worker 53 - jepsen.util 2213 :invoke :read [17 nil] INFO [2019-12-17 12:29:31,554] jepsen worker 72 - jepsen.util 2322 :info :write [24 49] :RequestCanceledException INFO [2019-12-17 12:29:31,555] jepsen worker 72 - jepsen.util 2412 :invoke :cas [24 [10 42]] INFO [2019-12-17 12:29:31,558] jepsen worker 3 - jepsen.util 2163 :ok :read [1 26] INFO [2019-12-17 12:29:31,559] jepsen worker 3 - jepsen.util 2163 :invoke :cas [1 [24 17]] INFO [2019-12-17 12:29:31,563] jepsen worker 63 - jepsen.util 1953 :fail :cas [21 [21 33]] :ValueNotSwapFrom INFO [2019-12-17 12:29:31,563] jepsen worker 63 - jepsen.util 1953 :invoke :cas [21 [11 1]] INFO [2019-12-17 12:29:31,570] jepsen worker 46 - jepsen.util 2566 :fail :cas [15 [24 10]] :ValueNotSwapFrom INFO [2019-12-17 12:29:31,570] jepsen worker 46 - jepsen.util 2566 :invoke :write [15 9] INFO [2019-12-17 12:29:31,589] jepsen worker 30 - jepsen.util 2730 :fail :read [10 nil] :RequestTimeoutException INFO [2019-12-17 12:29:31,589] jepsen worker 30 - jepsen.util 2730 :invoke :write [10 33] INFO [2019-12-17 12:29:31,598] jepsen worker 10 - jepsen.util 2350 :fail :cas [3 [26 4]] :ValueNotSwapFrom INFO [2019-12-17 12:29:31,599] jepsen worker 10 - jepsen.util 2350 :invoke :read [3 nil] INFO [2019-12-17 12:29:31,599] jepsen worker 37 - jepsen.util 2017 :ok :read [12 46] INFO [2019-12-17 12:29:31,599] jepsen worker 37 - jepsen.util 2017 :invoke :cas [12 [8 32]] INFO [2019-12-17 12:29:31,631] jepsen worker 88 - jepsen.util 2248 :ok :write [29 28] INFO [2019-12-17 12:29:31,631] jepsen worker 88 - jepsen.util 2248 :invoke :cas [29 [1 47]] INFO [2019-12-17 12:29:31,680] jepsen worker 71 - jepsen.util 2141 :fail :cas [23 [35 39]] :ValueNotSwapFrom INFO [2019-12-17 12:29:31,681] jepsen worker 71 - jepsen.util 2141 :invoke :write [23 27] INFO [2019-12-17 12:29:31,686] jepsen worker 45 - jepsen.util 2475 :ok :write [15 12] INFO [2019-12-17 12:29:31,687] jepsen worker 45 - jepsen.util 2475 :invoke :cas [15 [36 46]] INFO [2019-12-17 12:29:31,698] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13318,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:29:31,698] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13318,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:29:31,698] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13321,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:29:31,698] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13321,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:29:31,698] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13322,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:29:31,698] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13322,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:29:31,701] jepsen worker 20 - jepsen.util 2000 :ok :write [6 36] INFO [2019-12-17 12:29:31,702] jepsen worker 20 - jepsen.util 2000 :invoke :write [6 44] INFO [2019-12-17 12:29:31,702] jepsen worker 13 - jepsen.util 2263 :ok :write [4 18] INFO [2019-12-17 12:29:31,702] jepsen worker 13 - jepsen.util 2263 :invoke :write [4 23] INFO [2019-12-17 12:29:31,714] jepsen worker 28 - jepsen.util 1738 :ok :write [9 15] INFO [2019-12-17 12:29:31,714] jepsen worker 28 - jepsen.util 1738 :invoke :write [9 6] INFO [2019-12-17 12:29:31,721] jepsen worker 4 - jepsen.util 1804 :fail :cas [1 [0 45]] :ValueNotSwapFrom INFO [2019-12-17 12:29:31,721] jepsen worker 4 - jepsen.util 1804 :invoke :write [1 48] INFO [2019-12-17 12:29:31,736] jepsen worker 72 - jepsen.util 2412 :fail :cas [24 [10 42]] :ValueNotSwapFrom INFO [2019-12-17 12:29:31,736] jepsen worker 72 - jepsen.util 2412 :invoke :write [24 38] INFO [2019-12-17 12:29:31,759] jepsen worker 51 - jepsen.util 2121 :ok :read [17 23] INFO [2019-12-17 12:29:31,759] jepsen worker 51 - jepsen.util 2121 :invoke :write [17 20] INFO [2019-12-17 12:29:31,771] jepsen worker 89 - jepsen.util 2249 :ok :write [29 17] INFO [2019-12-17 12:29:31,771] jepsen worker 89 - jepsen.util 2249 :invoke :write [29 38] INFO [2019-12-17 12:29:31,791] jepsen worker 65 - jepsen.util 2045 :ok :write [21 35] INFO [2019-12-17 12:29:31,792] jepsen worker 65 - jepsen.util 2045 :invoke :write [21 19] INFO [2019-12-17 12:29:31,831] jepsen worker 11 - jepsen.util 2891 :ok :write [3 27] INFO [2019-12-17 12:29:31,831] jepsen worker 11 - jepsen.util 2891 :invoke :write [3 26] INFO [2019-12-17 12:29:31,837] jepsen worker 14 - jepsen.util 2174 :ok :write [4 20] INFO [2019-12-17 12:29:31,838] jepsen worker 14 - jepsen.util 2174 :invoke :write [4 9] INFO [2019-12-17 12:29:31,848] jepsen worker 36 - jepsen.util 2286 :fail :cas [12 [38 6]] :ValueNotSwapFrom INFO [2019-12-17 12:29:31,848] jepsen worker 36 - jepsen.util 2286 :invoke :read [12 nil] INFO [2019-12-17 12:29:31,863] jepsen worker 28 - jepsen.util 1738 :ok :write [9 6] INFO [2019-12-17 12:29:31,863] jepsen worker 28 - jepsen.util 1738 :invoke :write [9 34] INFO [2019-12-17 12:29:31,884] jepsen worker 30 - jepsen.util 2730 :ok :write [10 33] INFO [2019-12-17 12:29:31,884] jepsen worker 30 - jepsen.util 2730 :invoke :write [10 39] INFO [2019-12-17 12:29:31,892] jepsen worker 80 - jepsen.util 2330 :ok :write [26 38] INFO [2019-12-17 12:29:31,892] jepsen worker 80 - jepsen.util 2330 :invoke :write [26 11] INFO [2019-12-17 12:29:31,907] jepsen worker 65 - jepsen.util 2045 :ok :write [21 19] INFO [2019-12-17 12:29:31,907] jepsen worker 65 - jepsen.util 2045 :invoke :read [21 nil] INFO [2019-12-17 12:29:31,915] jepsen worker 3 - jepsen.util 2163 :fail :cas [1 [24 17]] :ValueNotSwapFrom INFO [2019-12-17 12:29:31,915] jepsen worker 3 - jepsen.util 2163 :invoke :read [1 nil] INFO [2019-12-17 12:29:31,915] jepsen worker 45 - jepsen.util 2475 :fail :cas [15 [36 46]] :ValueNotSwapFrom INFO [2019-12-17 12:29:31,915] jepsen worker 45 - jepsen.util 2475 :invoke :cas [15 [33 0]] INFO [2019-12-17 12:29:31,937] jepsen worker 29 - jepsen.util 2009 :fail :read [9 nil] :RequestTimeoutException INFO [2019-12-17 12:29:31,937] jepsen worker 29 - jepsen.util 2009 :invoke :cas [9 [20 33]] INFO [2019-12-17 12:29:31,942] jepsen worker 37 - jepsen.util 2017 :fail :cas [12 [8 32]] :ValueNotSwapFrom INFO [2019-12-17 12:29:31,943] jepsen worker 37 - jepsen.util 2017 :invoke :cas [12 [49 33]] INFO [2019-12-17 12:29:31,947] jepsen worker 65 - jepsen.util 2045 :ok :read [21 19] INFO [2019-12-17 12:29:31,947] jepsen worker 65 - jepsen.util 2045 :invoke :write [21 20] INFO [2019-12-17 12:29:31,954] jepsen worker 16 - jepsen.util 2896 :info :cas [5 [37 14]] :RequestCanceledException INFO [2019-12-17 12:29:31,955] jepsen worker 16 - jepsen.util 2986 :invoke :cas [5 [36 7]] INFO [2019-12-17 12:29:31,956] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13337,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:29:31,956] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13337,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:29:31,956] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13340,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:29:31,956] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13340,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:29:31,956] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13341,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:29:31,956] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13341,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:29:31,972] jepsen worker 36 - jepsen.util 2286 :ok :read [12 46] INFO [2019-12-17 12:29:31,972] jepsen worker 36 - jepsen.util 2286 :invoke :cas [12 [3 18]] INFO [2019-12-17 12:29:31,989] jepsen worker 15 - jepsen.util 2805 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:29:31,989] jepsen worker 15 - jepsen.util 2805 :invoke :write [5 4] INFO [2019-12-17 12:29:32,004] jepsen worker 29 - jepsen.util 2009 :fail :cas [9 [20 33]] :ValueNotSwapFrom INFO [2019-12-17 12:29:32,004] jepsen worker 29 - jepsen.util 2009 :invoke :read [9 nil] INFO [2019-12-17 12:29:32,031] jepsen worker 53 - jepsen.util 2213 :ok :read [17 23] INFO [2019-12-17 12:29:32,031] jepsen worker 53 - jepsen.util 2213 :invoke :write [17 5] INFO [2019-12-17 12:29:32,047] jepsen worker 28 - jepsen.util 1738 :ok :write [9 34] INFO [2019-12-17 12:29:32,047] jepsen worker 28 - jepsen.util 1738 :invoke :read [9 nil] INFO [2019-12-17 12:29:32,065] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13353,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:29:32,065] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13353,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:29:32,066] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13354,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:29:32,066] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13354,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:29:32,067] jepsen worker 28 - jepsen.util 1738 :ok :read [9 34] INFO [2019-12-17 12:29:32,067] jepsen worker 28 - jepsen.util 1738 :invoke :cas [9 [15 17]] INFO [2019-12-17 12:29:32,072] jepsen worker 65 - jepsen.util 2045 :ok :write [21 20] INFO [2019-12-17 12:29:32,072] jepsen worker 65 - jepsen.util 2045 :invoke :cas [21 [31 39]] INFO [2019-12-17 12:29:32,081] jepsen worker 28 - jepsen.util 1738 :fail :cas [9 [15 17]] :ValueNotSwapFrom INFO [2019-12-17 12:29:32,082] jepsen worker 28 - jepsen.util 1738 :invoke :read [9 nil] INFO [2019-12-17 12:29:32,093] jepsen worker 65 - jepsen.util 2045 :fail :cas [21 [31 39]] :ValueNotSwapFrom INFO [2019-12-17 12:29:32,093] jepsen worker 65 - jepsen.util 2045 :invoke :write [21 33] INFO [2019-12-17 12:29:32,094] jepsen worker 51 - jepsen.util 2121 :ok :write [17 20] INFO [2019-12-17 12:29:32,094] jepsen worker 51 - jepsen.util 2121 :invoke :cas [17 [33 36]] INFO [2019-12-17 12:29:32,104] jepsen worker 28 - jepsen.util 1738 :ok :read [9 34] INFO [2019-12-17 12:29:32,104] jepsen worker 28 - jepsen.util 1738 :invoke :cas [9 [43 21]] INFO [2019-12-17 12:29:32,106] jepsen worker 88 - jepsen.util 2248 :fail :cas [29 [1 47]] :ValueNotSwapFrom INFO [2019-12-17 12:29:32,106] jepsen worker 88 - jepsen.util 2248 :invoke :write [29 24] INFO [2019-12-17 12:29:32,108] jepsen worker 35 - jepsen.util 2645 :info :cas [11 [5 17]] :RequestCanceledException INFO [2019-12-17 12:29:32,108] jepsen worker 21 - jepsen.util 2991 :info :write [7 12] :RequestCanceledException INFO [2019-12-17 12:29:32,109] jepsen worker 21 - jepsen.util 3081 :invoke :write [7 30] INFO [2019-12-17 12:29:32,109] jepsen worker 35 - jepsen.util 2735 :invoke :write [11 28] INFO [2019-12-17 12:29:32,123] jepsen worker 43 - jepsen.util 2833 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:29:32,123] jepsen worker 43 - jepsen.util 2833 :invoke :cas [14 [16 4]] INFO [2019-12-17 12:29:32,137] jepsen worker 0 - jepsen.util 3150 :info :cas [0 [25 35]] :RequestCanceledException INFO [2019-12-17 12:29:32,137] jepsen worker 59 - jepsen.util 2849 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:29:32,137] jepsen worker 59 - jepsen.util 2849 :invoke :write [19 32] INFO [2019-12-17 12:29:32,137] jepsen worker 28 - jepsen.util 1738 :fail :cas [9 [43 21]] :ValueNotSwapFrom INFO [2019-12-17 12:29:32,137] jepsen worker 28 - jepsen.util 1738 :invoke :cas [9 [24 43]] INFO [2019-12-17 12:29:32,138] jepsen worker 0 - jepsen.util 3240 :invoke :read [0 nil] INFO [2019-12-17 12:29:32,156] jepsen worker 78 - jepsen.util 2508 :fail :cas [26 [12 1]] :ValueNotSwapFrom INFO [2019-12-17 12:29:32,156] jepsen worker 78 - jepsen.util 2508 :invoke :write [26 7] INFO [2019-12-17 12:29:32,157] jepsen worker 28 - jepsen.util 1738 :fail :cas [9 [24 43]] :ValueNotSwapFrom INFO [2019-12-17 12:29:32,157] jepsen worker 28 - jepsen.util 1738 :invoke :write [9 10] INFO [2019-12-17 12:29:32,169] jepsen worker 65 - jepsen.util 2045 :ok :write [21 33] INFO [2019-12-17 12:29:32,170] jepsen worker 65 - jepsen.util 2045 :invoke :cas [21 [31 13]] INFO [2019-12-17 12:29:32,176] jepsen worker 79 - jepsen.util 2329 :ok :write [26 26] INFO [2019-12-17 12:29:32,176] jepsen worker 79 - jepsen.util 2329 :invoke :write [26 12] INFO [2019-12-17 12:29:32,189] jepsen worker 50 - jepsen.util 3290 :info :cas [16 [47 26]] :RequestCanceledException INFO [2019-12-17 12:29:32,190] jepsen worker 50 - jepsen.util 3380 :invoke :write [16 31] INFO [2019-12-17 12:29:32,200] jepsen worker 55 - jepsen.util 2665 :ok :read [18 23] INFO [2019-12-17 12:29:32,200] jepsen worker 55 - jepsen.util 2665 :invoke :write [18 6] INFO [2019-12-17 12:29:32,206] jepsen worker 65 - jepsen.util 2045 :fail :cas [21 [31 13]] :ValueNotSwapFrom INFO [2019-12-17 12:29:32,206] jepsen worker 65 - jepsen.util 2045 :invoke :cas [21 [7 21]] INFO [2019-12-17 12:29:32,207] jepsen worker 24 - jepsen.util 2994 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:29:32,208] jepsen worker 24 - jepsen.util 2994 :invoke :read [8 nil] INFO [2019-12-17 12:29:32,223] jepsen worker 81 - jepsen.util 2691 :info :cas [27 [19 5]] :RequestCanceledException INFO [2019-12-17 12:29:32,223] jepsen worker 1 - jepsen.util 2791 :info :cas [0 [47 17]] :RequestCanceledException INFO [2019-12-17 12:29:32,223] jepsen worker 75 - jepsen.util 3315 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:29:32,223] jepsen worker 85 - jepsen.util 3325 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:29:32,223] jepsen worker 57 - jepsen.util 2847 :info :write [19 45] :RequestCanceledException INFO [2019-12-17 12:29:32,224] jepsen worker 75 - jepsen.util 3315 :invoke :cas [25 [6 7]] INFO [2019-12-17 12:29:32,224] jepsen worker 85 - jepsen.util 3325 :invoke :cas [28 [24 43]] INFO [2019-12-17 12:29:32,225] jepsen worker 1 - jepsen.util 2881 :invoke :cas [0 [40 9]] INFO [2019-12-17 12:29:32,225] jepsen worker 57 - jepsen.util 2937 :invoke :write [19 11] INFO [2019-12-17 12:29:32,225] jepsen worker 81 - jepsen.util 2781 :invoke :cas [27 [11 22]] INFO [2019-12-17 12:29:32,237] jepsen worker 84 - jepsen.util 2874 :info :cas [28 [5 28]] :RequestCanceledException INFO [2019-12-17 12:29:32,238] jepsen worker 65 - jepsen.util 2045 :fail :cas [21 [7 21]] :ValueNotSwapFrom INFO [2019-12-17 12:29:32,238] jepsen worker 65 - jepsen.util 2045 :invoke :read [21 nil] INFO [2019-12-17 12:29:32,238] jepsen worker 84 - jepsen.util 2964 :invoke :cas [28 [7 25]] INFO [2019-12-17 12:29:32,238] jepsen worker 28 - jepsen.util 1738 :ok :write [9 10] INFO [2019-12-17 12:29:32,238] jepsen worker 28 - jepsen.util 1738 :invoke :read [9 nil] INFO [2019-12-17 12:29:32,249] jepsen worker 38 - jepsen.util 2288 :fail :cas [12 [0 27]] :ValueNotSwapFrom INFO [2019-12-17 12:29:32,249] jepsen worker 38 - jepsen.util 2288 :invoke :read [12 nil] INFO [2019-12-17 12:29:32,254] jepsen worker 76 - jepsen.util 3046 :info :write [25 48] :RequestCanceledException INFO [2019-12-17 12:29:32,255] jepsen worker 13 - jepsen.util 2263 :ok :write [4 23] INFO [2019-12-17 12:29:32,255] jepsen worker 13 - jepsen.util 2263 :invoke :cas [4 [37 13]] INFO [2019-12-17 12:29:32,255] jepsen worker 76 - jepsen.util 3136 :invoke :read [25 nil] INFO [2019-12-17 12:29:32,256] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13380,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:29:32,256] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13380,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:29:32,256] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13382,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:29:32,256] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13382,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:29:32,275] jepsen worker 72 - jepsen.util 2412 :ok :write [24 38] INFO [2019-12-17 12:29:32,276] jepsen worker 72 - jepsen.util 2412 :invoke :read [24 nil] INFO [2019-12-17 12:29:32,277] jepsen worker 65 - jepsen.util 2045 :ok :read [21 33] INFO [2019-12-17 12:29:32,277] jepsen worker 65 - jepsen.util 2045 :invoke :read [21 nil] INFO [2019-12-17 12:29:32,280] jepsen worker 28 - jepsen.util 1738 :ok :read [9 10] INFO [2019-12-17 12:29:32,281] jepsen worker 28 - jepsen.util 1738 :invoke :cas [9 [39 43]] INFO [2019-12-17 12:29:32,282] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13391,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:29:32,282] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13391,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:29:32,282] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13394,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:29:32,282] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13394,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:29:32,282] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13395,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:29:32,282] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13395,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:29:32,287] jepsen worker 25 - jepsen.util 3535 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:29:32,287] jepsen worker 25 - jepsen.util 3535 :invoke :read [8 nil] INFO [2019-12-17 12:29:32,299] jepsen worker 52 - jepsen.util 2392 :ok :write [17 23] INFO [2019-12-17 12:29:32,299] jepsen worker 52 - jepsen.util 2392 :invoke :cas [17 [42 1]] INFO [2019-12-17 12:29:32,303] jepsen worker 11 - jepsen.util 2891 :ok :write [3 26] INFO [2019-12-17 12:29:32,303] jepsen worker 11 - jepsen.util 2891 :invoke :write [3 48] INFO [2019-12-17 12:29:32,307] jepsen worker 86 - jepsen.util 2966 :info :cas [28 [25 23]] :RequestCanceledException INFO [2019-12-17 12:29:32,309] jepsen worker 86 - jepsen.util 3056 :invoke :read [28 nil] INFO [2019-12-17 12:29:32,320] jepsen worker 62 - jepsen.util 2762 :info :cas [20 [40 5]] :RequestCanceledException INFO [2019-12-17 12:29:32,320] jepsen worker 83 - jepsen.util 3413 :info :cas [27 [15 33]] :RequestCanceledException INFO [2019-12-17 12:29:32,320] jepsen worker 58 - jepsen.util 2758 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:29:32,320] jepsen worker 58 - jepsen.util 2758 :invoke :cas [19 [26 22]] INFO [2019-12-17 12:29:32,321] jepsen worker 62 - jepsen.util 2852 :invoke :cas [20 [33 14]] INFO [2019-12-17 12:29:32,321] jepsen worker 83 - jepsen.util 3503 :invoke :cas [27 [11 0]] INFO [2019-12-17 12:29:32,322] jepsen worker 87 - jepsen.util 2067 :ok :write [29 10] INFO [2019-12-17 12:29:32,322] jepsen worker 87 - jepsen.util 2067 :invoke :write [29 6] INFO [2019-12-17 12:29:32,323] jepsen worker 12 - jepsen.util 2442 :ok :read [4 23] INFO [2019-12-17 12:29:32,323] jepsen worker 12 - jepsen.util 2442 :invoke :read [4 nil] INFO [2019-12-17 12:29:32,354] jepsen worker 41 - jepsen.util 3191 :info :cas [13 [32 43]] :RequestCanceledException INFO [2019-12-17 12:29:32,354] jepsen worker 22 - jepsen.util 2992 :info :write [7 7] :RequestCanceledException INFO [2019-12-17 12:29:32,355] jepsen worker 22 - jepsen.util 3082 :invoke :write [7 3] INFO [2019-12-17 12:29:32,355] jepsen worker 12 - jepsen.util 2442 :ok :read [4 23] INFO [2019-12-17 12:29:32,355] jepsen worker 41 - jepsen.util 3281 :invoke :cas [13 [28 19]] INFO [2019-12-17 12:29:32,355] jepsen worker 12 - jepsen.util 2442 :invoke :cas [4 [10 34]] INFO [2019-12-17 12:29:32,359] jepsen worker 65 - jepsen.util 2045 :ok :read [21 33] INFO [2019-12-17 12:29:32,359] jepsen worker 72 - jepsen.util 2412 :ok :read [24 38] INFO [2019-12-17 12:29:32,359] jepsen worker 65 - jepsen.util 2045 :invoke :cas [21 [43 44]] INFO [2019-12-17 12:29:32,359] jepsen worker 72 - jepsen.util 2412 :invoke :write [24 33] INFO [2019-12-17 12:29:32,370] jepsen worker 53 - jepsen.util 2213 :ok :write [17 5] INFO [2019-12-17 12:29:32,370] jepsen worker 53 - jepsen.util 2213 :invoke :write [17 33] INFO [2019-12-17 12:29:32,376] jepsen worker 79 - jepsen.util 2329 :ok :write [26 12] INFO [2019-12-17 12:29:32,376] jepsen worker 79 - jepsen.util 2329 :invoke :write [26 36] INFO [2019-12-17 12:29:32,377] jepsen worker 12 - jepsen.util 2442 :fail :cas [4 [10 34]] :ValueNotSwapFrom INFO [2019-12-17 12:29:32,377] jepsen worker 12 - jepsen.util 2442 :invoke :cas [4 [17 47]] INFO [2019-12-17 12:29:32,380] jepsen worker 26 - jepsen.util 3266 :info :write [8 0] :RequestCanceledException INFO [2019-12-17 12:29:32,380] jepsen worker 40 - jepsen.util 3280 :info :write [13 35] :RequestCanceledException INFO [2019-12-17 12:29:32,380] jepsen worker 48 - jepsen.util 2478 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:29:32,381] jepsen worker 48 - jepsen.util 2478 :invoke :cas [16 [20 36]] INFO [2019-12-17 12:29:32,381] jepsen worker 26 - jepsen.util 3356 :invoke :write [8 49] INFO [2019-12-17 12:29:32,381] jepsen worker 40 - jepsen.util 3370 :invoke :write [13 31] INFO [2019-12-17 12:29:32,385] jepsen worker 28 - jepsen.util 1738 :fail :cas [9 [39 43]] :ValueNotSwapFrom INFO [2019-12-17 12:29:32,385] jepsen worker 28 - jepsen.util 1738 :invoke :read [9 nil] INFO [2019-12-17 12:29:32,404] jepsen worker 12 - jepsen.util 2442 :fail :cas [4 [17 47]] :ValueNotSwapFrom INFO [2019-12-17 12:29:32,404] jepsen worker 12 - jepsen.util 2442 :invoke :read [4 nil] INFO [2019-12-17 12:29:32,421] jepsen worker 28 - jepsen.util 1738 :ok :read [9 10] INFO [2019-12-17 12:29:32,421] jepsen worker 28 - jepsen.util 1738 :invoke :read [9 nil] INFO [2019-12-17 12:29:32,423] jepsen worker 42 - jepsen.util 3552 :info :cas [14 [18 3]] :RequestCanceledException INFO [2019-12-17 12:29:32,425] jepsen worker 65 - jepsen.util 2045 :fail :cas [21 [43 44]] :ValueNotSwapFrom INFO [2019-12-17 12:29:32,425] jepsen worker 65 - jepsen.util 2045 :invoke :cas [21 [43 34]] INFO [2019-12-17 12:29:32,425] jepsen worker 42 - jepsen.util 3642 :invoke :cas [14 [6 10]] INFO [2019-12-17 12:29:32,435] jepsen worker 87 - jepsen.util 2067 :ok :write [29 6] INFO [2019-12-17 12:29:32,435] jepsen worker 87 - jepsen.util 2067 :invoke :write [29 14] INFO [2019-12-17 12:29:32,436] jepsen worker 17 - jepsen.util 3077 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:29:32,436] jepsen worker 17 - jepsen.util 3077 :invoke :cas [5 [41 30]] INFO [2019-12-17 12:29:32,437] jepsen worker 72 - jepsen.util 2412 :ok :write [24 33] INFO [2019-12-17 12:29:32,438] jepsen worker 72 - jepsen.util 2412 :invoke :cas [24 [28 9]] INFO [2019-12-17 12:29:32,444] jepsen worker 12 - jepsen.util 2442 :ok :read [4 23] INFO [2019-12-17 12:29:32,444] jepsen worker 12 - jepsen.util 2442 :invoke :read [4 nil] INFO [2019-12-17 12:29:32,462] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13434,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:29:32,463] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13434,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:29:32,463] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13438,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:29:32,463] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13438,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:29:32,466] jepsen worker 52 - jepsen.util 2392 :fail :cas [17 [42 1]] :ValueNotSwapFrom INFO [2019-12-17 12:29:32,466] jepsen worker 52 - jepsen.util 2392 :invoke :write [17 47] INFO [2019-12-17 12:29:32,469] jepsen worker 72 - jepsen.util 2412 :fail :cas [24 [28 9]] :ValueNotSwapFrom INFO [2019-12-17 12:29:32,469] jepsen worker 72 - jepsen.util 2412 :invoke :read [24 nil] INFO [2019-12-17 12:29:32,485] jepsen worker 82 - jepsen.util 3592 :info :cas [27 [7 28]] :RequestCanceledException INFO [2019-12-17 12:29:32,486] jepsen worker 82 - jepsen.util 3682 :invoke :cas [27 [19 32]] INFO [2019-12-17 12:29:32,503] jepsen worker 69 - jepsen.util 2319 :ok :write [23 32] INFO [2019-12-17 12:29:32,503] jepsen worker 69 - jepsen.util 2319 :invoke :read [23 nil] INFO [2019-12-17 12:29:32,507] jepsen worker 33 - jepsen.util 3273 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:29:32,507] jepsen worker 33 - jepsen.util 3273 :invoke :cas [11 [2 40]] INFO [2019-12-17 12:29:32,507] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:29:32,508] jepsen nemesis - couchbase.util Rebalance check count: 96 INFO [2019-12-17 12:29:32,508] jepsen nemesis - couchbase.util Rebalance stuck count: 77 INFO [2019-12-17 12:29:32,516] jepsen worker 72 - jepsen.util 2412 :ok :read [24 33] INFO [2019-12-17 12:29:32,516] jepsen worker 72 - jepsen.util 2412 :invoke :cas [24 [10 22]] INFO [2019-12-17 12:29:32,522] jepsen worker 54 - jepsen.util 2304 :ok :write [18 13] INFO [2019-12-17 12:29:32,522] jepsen worker 54 - jepsen.util 2304 :invoke :write [18 11] INFO [2019-12-17 12:29:32,523] jepsen worker 8 - jepsen.util 2798 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:29:32,523] jepsen worker 8 - jepsen.util 2798 :invoke :cas [2 [18 45]] INFO [2019-12-17 12:29:32,523] jepsen worker 68 - jepsen.util 2768 :info :write [22 18] :RequestCanceledException INFO [2019-12-17 12:29:32,523] jepsen worker 66 - jepsen.util 3036 :info :cas [22 [11 35]] :RequestCanceledException INFO [2019-12-17 12:29:32,524] jepsen worker 68 - jepsen.util 2858 :invoke :read [22 nil] INFO [2019-12-17 12:29:32,524] jepsen worker 66 - jepsen.util 3126 :invoke :write [22 31] INFO [2019-12-17 12:29:32,537] jepsen worker 77 - jepsen.util 2957 :info :cas [25 [1 42]] :RequestCanceledException INFO [2019-12-17 12:29:32,538] jepsen worker 77 - jepsen.util 3047 :invoke :read [25 nil] INFO [2019-12-17 12:29:32,552] jepsen worker 72 - jepsen.util 2412 :fail :cas [24 [10 22]] :ValueNotSwapFrom INFO [2019-12-17 12:29:32,552] jepsen worker 72 - jepsen.util 2412 :invoke :read [24 nil] INFO [2019-12-17 12:29:32,554] jepsen worker 19 - jepsen.util 2539 :info :write [6 1] :RequestCanceledException INFO [2019-12-17 12:29:32,555] jepsen worker 65 - jepsen.util 2045 :fail :cas [21 [43 34]] :ValueNotSwapFrom INFO [2019-12-17 12:29:32,555] jepsen worker 19 - jepsen.util 2629 :invoke :read [6 nil] INFO [2019-12-17 12:29:32,555] jepsen worker 65 - jepsen.util 2045 :invoke :read [21 nil] INFO [2019-12-17 12:29:32,561] jepsen worker 28 - jepsen.util 1738 :ok :read [9 10] INFO [2019-12-17 12:29:32,561] jepsen worker 28 - jepsen.util 1738 :invoke :read [9 nil] INFO [2019-12-17 12:29:32,568] jepsen worker 56 - jepsen.util 2306 :fail :cas [18 [28 15]] :ValueNotSwapFrom INFO [2019-12-17 12:29:32,568] jepsen worker 56 - jepsen.util 2306 :invoke :cas [18 [19 14]] INFO [2019-12-17 12:29:32,572] jepsen worker 14 - jepsen.util 2174 :ok :write [4 9] INFO [2019-12-17 12:29:32,588] jepsen worker 72 - jepsen.util 2412 :ok :read [24 33] INFO [2019-12-17 12:29:32,597] jepsen worker 11 - jepsen.util 2891 :ok :write [3 48] INFO [2019-12-17 12:29:32,601] jepsen worker 12 - jepsen.util 2442 :ok :read [4 9] INFO [2019-12-17 12:29:32,605] jepsen worker 3 - jepsen.util 2163 :ok :read [1 26] INFO [2019-12-17 12:29:32,610] jepsen worker 65 - jepsen.util 2045 :ok :read [21 33] INFO [2019-12-17 12:29:32,615] jepsen worker 19 - jepsen.util 2629 :ok :read [6 36] INFO [2019-12-17 12:29:32,624] jepsen worker 14 - jepsen.util 2174 :invoke :write [4 33] INFO [2019-12-17 12:29:32,625] jepsen worker 72 - jepsen.util 2412 :invoke :write [24 47] INFO [2019-12-17 12:29:32,625] jepsen worker 11 - jepsen.util 2891 :invoke :read [3 nil] INFO [2019-12-17 12:29:32,625] jepsen worker 12 - jepsen.util 2442 :invoke :read [4 nil] INFO [2019-12-17 12:29:32,625] jepsen worker 3 - jepsen.util 2163 :invoke :write [1 31] INFO [2019-12-17 12:29:32,625] jepsen worker 65 - jepsen.util 2045 :invoke :read [21 nil] INFO [2019-12-17 12:29:32,625] jepsen worker 19 - jepsen.util 2629 :invoke :write [6 27] INFO [2019-12-17 12:29:32,634] jepsen worker 28 - jepsen.util 1738 :ok :read [9 10] INFO [2019-12-17 12:29:32,634] jepsen worker 28 - jepsen.util 1738 :invoke :cas [9 [37 45]] INFO [2019-12-17 12:29:32,636] jepsen worker 39 - jepsen.util 3189 :info :write [13 7] :RequestCanceledException INFO [2019-12-17 12:29:32,637] jepsen worker 39 - jepsen.util 3279 :invoke :read [13 nil] INFO [2019-12-17 12:29:32,652] jepsen worker 12 - jepsen.util 2442 :ok :read [4 9] INFO [2019-12-17 12:29:32,652] jepsen worker 12 - jepsen.util 2442 :invoke :read [4 nil] INFO [2019-12-17 12:29:32,657] jepsen worker 65 - jepsen.util 2045 :ok :read [21 33] INFO [2019-12-17 12:29:32,657] jepsen worker 65 - jepsen.util 2045 :invoke :write [21 16] INFO [2019-12-17 12:29:32,659] jepsen worker 11 - jepsen.util 2891 :ok :read [3 48] INFO [2019-12-17 12:29:32,659] jepsen worker 11 - jepsen.util 2891 :invoke :write [3 15] INFO [2019-12-17 12:29:32,682] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13471,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:29:32,682] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13471,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:29:32,682] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13474,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:29:32,682] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13474,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:29:32,682] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13475,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:29:32,682] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13475,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:29:32,687] jepsen worker 12 - jepsen.util 2442 :ok :read [4 9] INFO [2019-12-17 12:29:32,687] jepsen worker 12 - jepsen.util 2442 :invoke :cas [4 [39 37]] INFO [2019-12-17 12:29:32,706] jepsen worker 38 - jepsen.util 2288 :ok :read [12 46] INFO [2019-12-17 12:29:32,740] jepsen worker 2 - jepsen.util 2882 :info :cas [0 [11 17]] :RequestCanceledException INFO [2019-12-17 12:29:32,741] jepsen worker 61 - jepsen.util 3211 :info :write [20 45] :RequestCanceledException INFO [2019-12-17 12:29:32,741] jepsen worker 38 - jepsen.util 2288 :invoke :write [12 39] INFO [2019-12-17 12:29:32,742] jepsen worker 61 - jepsen.util 3301 :invoke :write [20 4] INFO [2019-12-17 12:29:32,742] jepsen worker 2 - jepsen.util 2972 :invoke :write [0 27] INFO [2019-12-17 12:29:32,800] jepsen worker 65 - jepsen.util 2045 :ok :write [21 16] INFO [2019-12-17 12:29:32,800] jepsen worker 65 - jepsen.util 2045 :invoke :cas [21 [25 26]] INFO [2019-12-17 12:29:32,816] jepsen worker 65 - jepsen.util 2045 :fail :cas [21 [25 26]] :ValueNotSwapFrom INFO [2019-12-17 12:29:32,816] jepsen worker 65 - jepsen.util 2045 :invoke :write [21 3] INFO [2019-12-17 12:29:32,839] jepsen worker 36 - jepsen.util 2286 :fail :cas [12 [3 18]] :ValueNotSwapFrom INFO [2019-12-17 12:29:32,839] jepsen worker 36 - jepsen.util 2286 :invoke :cas [12 [41 7]] INFO [2019-12-17 12:29:32,844] jepsen worker 72 - jepsen.util 2412 :ok :write [24 47] INFO [2019-12-17 12:29:32,845] jepsen worker 72 - jepsen.util 2412 :invoke :cas [24 [31 31]] INFO [2019-12-17 12:29:32,848] jepsen worker 70 - jepsen.util 2680 :ok :write [23 46] INFO [2019-12-17 12:29:32,848] jepsen worker 70 - jepsen.util 2680 :invoke :write [23 25] INFO [2019-12-17 12:29:32,855] jepsen worker 11 - jepsen.util 2891 :ok :write [3 15] INFO [2019-12-17 12:29:32,855] jepsen worker 19 - jepsen.util 2629 :ok :write [6 27] INFO [2019-12-17 12:29:32,855] jepsen worker 11 - jepsen.util 2891 :invoke :cas [3 [22 17]] INFO [2019-12-17 12:29:32,855] jepsen worker 19 - jepsen.util 2629 :invoke :cas [6 [22 21]] INFO [2019-12-17 12:29:32,862] jepsen worker 72 - jepsen.util 2412 :fail :cas [24 [31 31]] :ValueNotSwapFrom INFO [2019-12-17 12:29:32,862] jepsen worker 72 - jepsen.util 2412 :invoke :write [24 45] INFO [2019-12-17 12:29:32,874] jepsen worker 13 - jepsen.util 2263 :fail :cas [4 [37 13]] :ValueNotSwapFrom INFO [2019-12-17 12:29:32,875] jepsen worker 13 - jepsen.util 2263 :invoke :cas [4 [4 21]] INFO [2019-12-17 12:29:32,892] jepsen worker 51 - jepsen.util 2121 :fail :cas [17 [33 36]] :ValueNotSwapFrom INFO [2019-12-17 12:29:32,892] jepsen worker 51 - jepsen.util 2121 :invoke :write [17 49] INFO [2019-12-17 12:29:32,895] jepsen worker 80 - jepsen.util 2330 :ok :write [26 11] INFO [2019-12-17 12:29:32,896] jepsen worker 80 - jepsen.util 2330 :invoke :read [26 nil] INFO [2019-12-17 12:29:32,902] jepsen worker 19 - jepsen.util 2629 :fail :cas [6 [22 21]] :ValueNotSwapFrom INFO [2019-12-17 12:29:32,902] jepsen worker 19 - jepsen.util 2629 :invoke :cas [6 [38 42]] INFO [2019-12-17 12:29:32,907] jepsen worker 11 - jepsen.util 2891 :fail :cas [3 [22 17]] :ValueNotSwapFrom INFO [2019-12-17 12:29:32,907] jepsen worker 11 - jepsen.util 2891 :invoke :read [3 nil] INFO [2019-12-17 12:29:32,920] jepsen worker 12 - jepsen.util 2442 :fail :cas [4 [39 37]] :ValueNotSwapFrom INFO [2019-12-17 12:29:32,920] jepsen worker 12 - jepsen.util 2442 :invoke :read [4 nil] INFO [2019-12-17 12:29:32,935] jepsen worker 19 - jepsen.util 2629 :fail :cas [6 [38 42]] :ValueNotSwapFrom INFO [2019-12-17 12:29:32,936] jepsen worker 19 - jepsen.util 2629 :invoke :read [6 nil] INFO [2019-12-17 12:29:32,938] jepsen worker 11 - jepsen.util 2891 :ok :read [3 15] INFO [2019-12-17 12:29:32,939] jepsen worker 11 - jepsen.util 2891 :invoke :cas [3 [44 43]] INFO [2019-12-17 12:29:32,960] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13492,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:29:32,960] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13492,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:29:32,960] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13494,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:29:32,960] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13494,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:29:32,967] jepsen worker 65 - jepsen.util 2045 :ok :write [21 3] INFO [2019-12-17 12:29:32,967] jepsen worker 65 - jepsen.util 2045 :invoke :read [21 nil] INFO [2019-12-17 12:29:32,977] jepsen worker 19 - jepsen.util 2629 :ok :read [6 27] INFO [2019-12-17 12:29:32,978] jepsen worker 19 - jepsen.util 2629 :invoke :cas [6 [33 15]] INFO [2019-12-17 12:29:32,978] jepsen worker 37 - jepsen.util 2017 :fail :cas [12 [49 33]] :ValueNotSwapFrom INFO [2019-12-17 12:29:32,978] jepsen worker 37 - jepsen.util 2017 :invoke :read [12 nil] INFO [2019-12-17 12:29:32,985] jepsen worker 11 - jepsen.util 2891 :fail :cas [3 [44 43]] :ValueNotSwapFrom INFO [2019-12-17 12:29:32,985] jepsen worker 11 - jepsen.util 2891 :invoke :cas [3 [15 45]] INFO [2019-12-17 12:29:33,006] jepsen worker 65 - jepsen.util 2045 :ok :read [21 3] INFO [2019-12-17 12:29:33,007] jepsen worker 65 - jepsen.util 2045 :invoke :read [21 nil] INFO [2019-12-17 12:29:33,012] jepsen worker 19 - jepsen.util 2629 :fail :cas [6 [33 15]] :ValueNotSwapFrom INFO [2019-12-17 12:29:33,012] jepsen worker 19 - jepsen.util 2629 :invoke :write [6 26] INFO [2019-12-17 12:29:33,041] jepsen worker 69 - jepsen.util 2319 :ok :read [23 46] INFO [2019-12-17 12:29:33,041] jepsen worker 69 - jepsen.util 2319 :invoke :cas [23 [21 35]] INFO [2019-12-17 12:29:33,063] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13510,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:29:33,063] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13510,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:29:33,063] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13514,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:29:33,063] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13514,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:29:33,064] jepsen worker 65 - jepsen.util 2045 :ok :read [21 3] INFO [2019-12-17 12:29:33,064] jepsen worker 65 - jepsen.util 2045 :invoke :read [21 nil] INFO [2019-12-17 12:29:33,104] jepsen worker 65 - jepsen.util 2045 :ok :read [21 3] INFO [2019-12-17 12:29:33,105] jepsen worker 65 - jepsen.util 2045 :invoke :read [21 nil] INFO [2019-12-17 12:29:33,112] jepsen worker 89 - jepsen.util 2249 :ok :write [29 38] INFO [2019-12-17 12:29:33,112] jepsen worker 89 - jepsen.util 2249 :invoke :read [29 nil] INFO [2019-12-17 12:29:33,134] jepsen worker 65 - jepsen.util 2045 :ok :read [21 3] INFO [2019-12-17 12:29:33,134] jepsen worker 65 - jepsen.util 2045 :invoke :read [21 nil] INFO [2019-12-17 12:29:33,139] jepsen worker 14 - jepsen.util 2174 :ok :write [4 33] INFO [2019-12-17 12:29:33,139] jepsen worker 14 - jepsen.util 2174 :invoke :cas [4 [3 10]] INFO [2019-12-17 12:29:33,151] jepsen worker 65 - jepsen.util 2045 :ok :read [21 3] INFO [2019-12-17 12:29:33,151] jepsen worker 65 - jepsen.util 2045 :invoke :write [21 29] INFO [2019-12-17 12:29:33,166] jepsen worker 12 - jepsen.util 2442 :ok :read [4 33] INFO [2019-12-17 12:29:33,167] jepsen worker 12 - jepsen.util 2442 :invoke :cas [4 [24 31]] INFO [2019-12-17 12:29:33,193] jepsen worker 72 - jepsen.util 2412 :ok :write [24 45] INFO [2019-12-17 12:29:33,194] jepsen worker 72 - jepsen.util 2412 :invoke :write [24 42] INFO [2019-12-17 12:29:33,195] jepsen worker 4 - jepsen.util 1804 :ok :write [1 48] INFO [2019-12-17 12:29:33,195] jepsen worker 4 - jepsen.util 1804 :invoke :read [1 nil] INFO [2019-12-17 12:29:33,219] jepsen worker 55 - jepsen.util 2665 :ok :write [18 6] INFO [2019-12-17 12:29:33,219] jepsen worker 55 - jepsen.util 2665 :invoke :write [18 33] INFO [2019-12-17 12:29:33,238] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13527,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:29:33,238] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13527,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:29:33,238] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13528,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:29:33,238] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13528,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:29:33,252] jepsen worker 5 - jepsen.util 1985 :ok :write [1 19] INFO [2019-12-17 12:29:33,252] jepsen worker 5 - jepsen.util 1985 :invoke :write [1 43] INFO [2019-12-17 12:29:33,258] jepsen worker 19 - jepsen.util 2629 :ok :write [6 26] INFO [2019-12-17 12:29:33,258] jepsen worker 19 - jepsen.util 2629 :invoke :write [6 29] INFO [2019-12-17 12:29:33,261] jepsen worker 65 - jepsen.util 2045 :ok :write [21 29] INFO [2019-12-17 12:29:33,261] jepsen worker 65 - jepsen.util 2045 :invoke :cas [21 [1 4]] INFO [2019-12-17 12:29:33,274] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13535,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:29:33,274] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13535,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:29:33,274] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13538,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:29:33,274] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13538,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:29:33,285] jepsen worker 65 - jepsen.util 2045 :fail :cas [21 [1 4]] :ValueNotSwapFrom INFO [2019-12-17 12:29:33,285] jepsen worker 65 - jepsen.util 2045 :invoke :read [21 nil] INFO [2019-12-17 12:29:33,299] jepsen worker 72 - jepsen.util 2412 :ok :write [24 42] INFO [2019-12-17 12:29:33,299] jepsen worker 72 - jepsen.util 2412 :invoke :read [24 nil] INFO [2019-12-17 12:29:33,307] jepsen worker 67 - jepsen.util 3577 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:29:33,307] jepsen worker 67 - jepsen.util 3577 :invoke :write [22 45] INFO [2019-12-17 12:29:33,316] jepsen worker 72 - jepsen.util 2412 :ok :read [24 42] INFO [2019-12-17 12:29:33,316] jepsen worker 72 - jepsen.util 2412 :invoke :cas [24 [9 7]] INFO [2019-12-17 12:29:33,323] jepsen worker 47 - jepsen.util 2657 :info :write [15 41] :RequestCanceledException INFO [2019-12-17 12:29:33,323] jepsen worker 23 - jepsen.util 2723 :info :write [7 41] :RequestCanceledException INFO [2019-12-17 12:29:33,323] jepsen worker 44 - jepsen.util 2744 :info :write [14 34] :RequestCanceledException INFO [2019-12-17 12:29:33,323] jepsen worker 6 - jepsen.util 3156 :info :write [2 27] :RequestCanceledException INFO [2019-12-17 12:29:33,324] jepsen worker 23 - jepsen.util 2813 :invoke :write [7 12] INFO [2019-12-17 12:29:33,324] jepsen worker 47 - jepsen.util 2747 :invoke :write [15 41] INFO [2019-12-17 12:29:33,324] jepsen worker 44 - jepsen.util 2834 :invoke :write [14 19] INFO [2019-12-17 12:29:33,324] jepsen worker 6 - jepsen.util 3246 :invoke :cas [2 [23 11]] INFO [2019-12-17 12:29:33,336] jepsen worker 60 - jepsen.util 3300 :info :write [20 0] :RequestCanceledException INFO [2019-12-17 12:29:33,336] jepsen worker 49 - jepsen.util 2299 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:29:33,336] jepsen worker 49 - jepsen.util 2299 :invoke :write [16 22] INFO [2019-12-17 12:29:33,337] jepsen worker 60 - jepsen.util 3390 :invoke :read [20 nil] INFO [2019-12-17 12:29:33,346] jepsen worker 11 - jepsen.util 2891 :ok :cas [3 [15 45]] INFO [2019-12-17 12:29:33,346] jepsen worker 72 - jepsen.util 2412 :fail :cas [24 [9 7]] :ValueNotSwapFrom INFO [2019-12-17 12:29:33,346] jepsen worker 11 - jepsen.util 2891 :invoke :read [3 nil] INFO [2019-12-17 12:29:33,346] jepsen worker 72 - jepsen.util 2412 :invoke :cas [24 [18 30]] INFO [2019-12-17 12:29:33,353] jepsen worker 34 - jepsen.util 3184 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:29:33,353] jepsen worker 64 - jepsen.util 1954 :info :write [21 31] :RequestCanceledException INFO [2019-12-17 12:29:33,354] jepsen worker 32 - jepsen.util 2822 :info :write [10 23] :RequestCanceledException INFO [2019-12-17 12:29:33,354] jepsen worker 34 - jepsen.util 3184 :invoke :read [11 nil] INFO [2019-12-17 12:29:33,355] jepsen worker 64 - jepsen.util 2044 :invoke :read [21 nil] INFO [2019-12-17 12:29:33,355] jepsen worker 32 - jepsen.util 2912 :invoke :write [10 49] INFO [2019-12-17 12:29:33,387] jepsen worker 65 - jepsen.util 2045 :ok :read [21 29] INFO [2019-12-17 12:29:33,387] jepsen worker 65 - jepsen.util 2045 :invoke :cas [21 [46 4]] INFO [2019-12-17 12:29:33,388] jepsen worker 72 - jepsen.util 2412 :fail :cas [24 [18 30]] :ValueNotSwapFrom INFO [2019-12-17 12:29:33,388] jepsen worker 73 - jepsen.util 2413 :fail :read [24 nil] :RequestTimeoutException INFO [2019-12-17 12:29:33,388] jepsen worker 72 - jepsen.util 2412 :invoke :read [24 nil] INFO [2019-12-17 12:29:33,388] jepsen worker 73 - jepsen.util 2413 :invoke :cas [24 [15 40]] INFO [2019-12-17 12:29:33,405] jepsen worker 11 - jepsen.util 2891 :ok :read [3 45] INFO [2019-12-17 12:29:33,405] jepsen worker 11 - jepsen.util 2891 :invoke :write [3 46] INFO [2019-12-17 12:29:33,406] jepsen worker 64 - jepsen.util 2044 :ok :read [21 29] INFO [2019-12-17 12:29:33,406] jepsen worker 64 - jepsen.util 2044 :invoke :write [21 32] INFO [2019-12-17 12:29:33,408] jepsen worker 7 - jepsen.util 2887 :info :write [2 43] :RequestCanceledException INFO [2019-12-17 12:29:33,409] jepsen worker 7 - jepsen.util 2977 :invoke :read [2 nil] INFO [2019-12-17 12:29:33,411] jepsen worker 56 - jepsen.util 2306 :fail :cas [18 [19 14]] :ValueNotSwapFrom INFO [2019-12-17 12:29:33,411] jepsen worker 56 - jepsen.util 2306 :invoke :write [18 15] INFO [2019-12-17 12:29:33,416] jepsen worker 72 - jepsen.util 2412 :ok :read [24 42] INFO [2019-12-17 12:29:33,416] jepsen worker 72 - jepsen.util 2412 :invoke :write [24 28] INFO [2019-12-17 12:29:33,440] jepsen worker 36 - jepsen.util 2286 :fail :cas [12 [41 7]] :ValueNotSwapFrom INFO [2019-12-17 12:29:33,440] jepsen worker 36 - jepsen.util 2286 :invoke :write [12 32] INFO [2019-12-17 12:29:33,444] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13568,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:29:33,444] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13568,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:29:33,444] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13571,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:29:33,444] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13571,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:29:33,468] jepsen worker 73 - jepsen.util 2413 :fail :cas [24 [15 40]] :ValueNotSwapFrom INFO [2019-12-17 12:29:33,468] jepsen worker 73 - jepsen.util 2413 :invoke :write [24 5] INFO [2019-12-17 12:29:33,489] jepsen worker 18 - jepsen.util 2538 :info :cas [6 [39 41]] :RequestCanceledException INFO [2019-12-17 12:29:33,490] jepsen worker 18 - jepsen.util 2628 :invoke :cas [6 [28 23]] INFO [2019-12-17 12:29:33,498] jepsen worker 13 - jepsen.util 2263 :fail :cas [4 [4 21]] :ValueNotSwapFrom INFO [2019-12-17 12:29:33,498] jepsen worker 13 - jepsen.util 2263 :invoke :write [4 15] INFO [2019-12-17 12:29:33,513] jepsen worker 80 - jepsen.util 2330 :ok :read [26 11] INFO [2019-12-17 12:29:33,513] jepsen worker 80 - jepsen.util 2330 :invoke :cas [26 [3 33]] INFO [2019-12-17 12:29:33,521] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:29:33,521] jepsen nemesis - couchbase.util Rebalance check count: 97 INFO [2019-12-17 12:29:33,521] jepsen nemesis - couchbase.util Rebalance stuck count: 78 INFO [2019-12-17 12:29:33,523] jepsen worker 9 - jepsen.util 2619 :info :cas [3 [42 38]] :RequestCanceledException INFO [2019-12-17 12:29:33,523] jepsen worker 31 - jepsen.util 2191 :info :write [10 1] :RequestCanceledException INFO [2019-12-17 12:29:33,524] jepsen worker 9 - jepsen.util 2709 :invoke :cas [3 [40 26]] INFO [2019-12-17 12:29:33,524] jepsen worker 31 - jepsen.util 2281 :invoke :read [10 nil] INFO [2019-12-17 12:29:33,536] jepsen worker 74 - jepsen.util 2234 :info :cas [24 [21 44]] :RequestCanceledException INFO [2019-12-17 12:29:33,537] jepsen worker 74 - jepsen.util 2324 :invoke :write [24 39] INFO [2019-12-17 12:29:33,537] jepsen worker 64 - jepsen.util 2044 :ok :write [21 32] INFO [2019-12-17 12:29:33,537] jepsen worker 64 - jepsen.util 2044 :invoke :cas [21 [3 2]] INFO [2019-12-17 12:29:33,551] jepsen worker 64 - jepsen.util 2044 :fail :cas [21 [3 2]] :ValueNotSwapFrom INFO [2019-12-17 12:29:33,551] jepsen worker 64 - jepsen.util 2044 :invoke :write [21 25] INFO [2019-12-17 12:29:33,558] jepsen worker 37 - jepsen.util 2017 :ok :read [12 46] INFO [2019-12-17 12:29:33,558] jepsen worker 37 - jepsen.util 2017 :invoke :write [12 40] INFO [2019-12-17 12:29:33,586] jepsen worker 89 - jepsen.util 2249 :ok :read [29 38] INFO [2019-12-17 12:29:33,587] jepsen worker 89 - jepsen.util 2249 :invoke :write [29 37] INFO [2019-12-17 12:29:33,623] jepsen worker 63 - jepsen.util 1953 :info :cas [21 [11 1]] :RequestCanceledException INFO [2019-12-17 12:29:33,623] jepsen worker 46 - jepsen.util 2566 :info :write [15 9] :RequestCanceledException INFO [2019-12-17 12:29:33,624] jepsen worker 63 - jepsen.util 2043 :invoke :cas [21 [6 47]] INFO [2019-12-17 12:29:33,624] jepsen worker 46 - jepsen.util 2656 :invoke :write [15 7] INFO [2019-12-17 12:29:33,636] jepsen worker 27 - jepsen.util 2007 :info :write [9 37] :RequestCanceledException INFO [2019-12-17 12:29:33,636] jepsen worker 10 - jepsen.util 2350 :fail :read [3 nil] :RequestTimeoutException INFO [2019-12-17 12:29:33,636] jepsen worker 10 - jepsen.util 2350 :invoke :write [3 21] INFO [2019-12-17 12:29:33,637] jepsen worker 27 - jepsen.util 2097 :invoke :cas [9 [48 4]] INFO [2019-12-17 12:29:33,647] jepsen worker 69 - jepsen.util 2319 :fail :cas [23 [21 35]] :ValueNotSwapFrom INFO [2019-12-17 12:29:33,647] jepsen worker 69 - jepsen.util 2319 :invoke :cas [23 [9 10]] INFO [2019-12-17 12:29:33,662] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13589,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:29:33,663] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13589,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:29:33,663] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13592,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:29:33,663] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13592,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:29:33,666] jepsen worker 14 - jepsen.util 2174 :fail :cas [4 [3 10]] :ValueNotSwapFrom INFO [2019-12-17 12:29:33,666] jepsen worker 14 - jepsen.util 2174 :invoke :read [4 nil] INFO [2019-12-17 12:29:33,675] jepsen worker 4 - jepsen.util 1804 :ok :read [1 19] INFO [2019-12-17 12:29:33,675] jepsen worker 4 - jepsen.util 1804 :invoke :read [1 nil] INFO [2019-12-17 12:29:33,690] jepsen worker 12 - jepsen.util 2442 :fail :cas [4 [24 31]] :ValueNotSwapFrom INFO [2019-12-17 12:29:33,690] jepsen worker 12 - jepsen.util 2442 :invoke :write [4 6] INFO [2019-12-17 12:29:33,722] jepsen worker 71 - jepsen.util 2141 :info :write [23 27] :RequestCanceledException INFO [2019-12-17 12:29:33,723] jepsen worker 71 - jepsen.util 2231 :invoke :write [23 16] INFO [2019-12-17 12:29:33,743] jepsen worker 80 - jepsen.util 2330 :fail :cas [26 [3 33]] :ValueNotSwapFrom INFO [2019-12-17 12:29:33,743] jepsen worker 80 - jepsen.util 2330 :invoke :cas [26 [16 16]] INFO [2019-12-17 12:29:33,773] jepsen worker 14 - jepsen.util 2174 :ok :read [4 33] INFO [2019-12-17 12:29:33,773] jepsen worker 14 - jepsen.util 2174 :invoke :cas [4 [26 43]] INFO [2019-12-17 12:29:33,789] jepsen worker 20 - jepsen.util 2000 :info :write [6 44] :RequestCanceledException INFO [2019-12-17 12:29:33,790] jepsen worker 20 - jepsen.util 2090 :invoke :read [6 nil] INFO [2019-12-17 12:29:33,871] jepsen worker 69 - jepsen.util 2319 :fail :cas [23 [9 10]] :ValueNotSwapFrom INFO [2019-12-17 12:29:33,871] jepsen worker 69 - jepsen.util 2319 :invoke :write [23 48] INFO [2019-12-17 12:29:33,914] jepsen worker 4 - jepsen.util 1804 :ok :read [1 19] INFO [2019-12-17 12:29:33,914] jepsen worker 4 - jepsen.util 1804 :invoke :read [1 nil] INFO [2019-12-17 12:29:33,930] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13604,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:29:33,930] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13604,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:29:33,930] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13607,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:29:33,930] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13607,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:29:33,930] jepsen worker 80 - jepsen.util 2330 :fail :cas [26 [16 16]] :ValueNotSwapFrom INFO [2019-12-17 12:29:33,930] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13608,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:29:33,930] jepsen worker 80 - jepsen.util 2330 :invoke :cas [26 [18 24]] INFO [2019-12-17 12:29:33,930] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13608,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:29:33,979] jepsen worker 4 - jepsen.util 1804 :ok :read [1 19] INFO [2019-12-17 12:29:33,979] jepsen worker 4 - jepsen.util 1804 :invoke :cas [1 [3 17]] INFO [2019-12-17 12:29:33,989] jepsen worker 30 - jepsen.util 2730 :info :write [10 39] :RequestCanceledException INFO [2019-12-17 12:29:33,990] jepsen worker 30 - jepsen.util 2820 :invoke :cas [10 [22 15]] INFO [2019-12-17 12:29:34,002] jepsen worker 30 - jepsen.util 2820 :fail :cas [10 [22 15]] :ValueNotSwapFrom INFO [2019-12-17 12:29:34,002] jepsen worker 30 - jepsen.util 2820 :invoke :cas [10 [39 3]] INFO [2019-12-17 12:29:34,007] jepsen worker 45 - jepsen.util 2475 :info :cas [15 [33 0]] :RequestCanceledException INFO [2019-12-17 12:29:34,007] jepsen worker 16 - jepsen.util 2986 :info :cas [5 [36 7]] :RequestCanceledException INFO [2019-12-17 12:29:34,008] jepsen worker 16 - jepsen.util 3076 :invoke :cas [5 [2 40]] INFO [2019-12-17 12:29:34,008] jepsen worker 45 - jepsen.util 2565 :invoke :cas [15 [31 15]] INFO [2019-12-17 12:29:34,011] jepsen worker 30 - jepsen.util 2820 :fail :cas [10 [39 3]] :ValueNotSwapFrom INFO [2019-12-17 12:29:34,011] jepsen worker 30 - jepsen.util 2820 :invoke :read [10 nil] INFO [2019-12-17 12:29:34,020] jepsen worker 30 - jepsen.util 2820 :ok :read [10 33] INFO [2019-12-17 12:29:34,020] jepsen worker 30 - jepsen.util 2820 :invoke :write [10 29] INFO [2019-12-17 12:29:34,037] jepsen worker 29 - jepsen.util 2009 :fail :read [9 nil] :RequestTimeoutException INFO [2019-12-17 12:29:34,037] jepsen worker 29 - jepsen.util 2009 :invoke :write [9 23] INFO [2019-12-17 12:29:34,040] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13615,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:29:34,040] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13615,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:29:34,040] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13618,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:29:34,040] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13618,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:29:34,041] jepsen worker 14 - jepsen.util 2174 :fail :cas [4 [26 43]] :ValueNotSwapFrom INFO [2019-12-17 12:29:34,041] jepsen worker 14 - jepsen.util 2174 :invoke :write [4 47] INFO [2019-12-17 12:29:34,088] jepsen worker 15 - jepsen.util 2805 :info :write [5 4] :RequestCanceledException INFO [2019-12-17 12:29:34,089] jepsen worker 15 - jepsen.util 2895 :invoke :read [5 nil] INFO [2019-12-17 12:29:34,136] jepsen worker 35 - jepsen.util 2735 :info :write [11 28] :RequestCanceledException INFO [2019-12-17 12:29:34,137] jepsen worker 35 - jepsen.util 2825 :invoke :cas [11 [37 21]] INFO [2019-12-17 12:29:34,153] jepsen worker 88 - jepsen.util 2248 :info :write [29 24] :RequestCanceledException INFO [2019-12-17 12:29:34,153] jepsen worker 0 - jepsen.util 3240 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:29:34,153] jepsen worker 0 - jepsen.util 3240 :invoke :write [0 42] INFO [2019-12-17 12:29:34,153] jepsen worker 88 - jepsen.util 2338 :invoke :cas [29 [41 24]] INFO [2019-12-17 12:29:34,169] jepsen worker 80 - jepsen.util 2330 :fail :cas [26 [18 24]] :ValueNotSwapFrom INFO [2019-12-17 12:29:34,169] jepsen worker 80 - jepsen.util 2330 :invoke :read [26 nil] INFO [2019-12-17 12:29:34,187] jepsen worker 78 - jepsen.util 2508 :info :write [26 7] :RequestCanceledException INFO [2019-12-17 12:29:34,187] jepsen worker 78 - jepsen.util 2598 :invoke :write [26 16] INFO [2019-12-17 12:29:34,222] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13632,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:29:34,222] jepsen worker 50 - jepsen.util 3380 :info :write [16 31] :RequestCanceledException INFO [2019-12-17 12:29:34,222] jepsen worker 21 - jepsen.util 3081 :info :write [7 30] :RequestCanceledException INFO [2019-12-17 12:29:34,222] jepsen worker 43 - jepsen.util 2833 :info :cas [14 [16 4]] :RequestCanceledException INFO [2019-12-17 12:29:34,222] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13632,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:29:34,223] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13633,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:29:34,223] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13633,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:29:34,223] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13634,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:29:34,223] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13634,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:29:34,223] jepsen worker 43 - jepsen.util 2923 :invoke :read [14 nil] INFO [2019-12-17 12:29:34,223] jepsen worker 50 - jepsen.util 3470 :invoke :write [16 20] INFO [2019-12-17 12:29:34,223] jepsen worker 21 - jepsen.util 3171 :invoke :write [7 5] INFO [2019-12-17 12:29:34,237] jepsen worker 59 - jepsen.util 2849 :info :write [19 32] :RequestCanceledException INFO [2019-12-17 12:29:34,238] jepsen worker 59 - jepsen.util 2939 :invoke :write [19 13] INFO [2019-12-17 12:29:34,259] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13640,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:29:34,260] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13640,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:29:34,260] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13644,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:29:34,260] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13644,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:29:34,263] jepsen worker 4 - jepsen.util 1804 :fail :cas [1 [3 17]] :ValueNotSwapFrom INFO [2019-12-17 12:29:34,263] jepsen worker 4 - jepsen.util 1804 :invoke :cas [1 [49 26]] INFO [2019-12-17 12:29:34,289] jepsen worker 1 - jepsen.util 2881 :info :cas [0 [40 9]] :RequestCanceledException INFO [2019-12-17 12:29:34,290] jepsen worker 1 - jepsen.util 2971 :invoke :read [0 nil] INFO [2019-12-17 12:29:34,304] jepsen worker 4 - jepsen.util 1804 :fail :cas [1 [49 26]] :ValueNotSwapFrom INFO [2019-12-17 12:29:34,305] jepsen worker 4 - jepsen.util 1804 :invoke :read [1 nil] INFO [2019-12-17 12:29:34,307] jepsen worker 24 - jepsen.util 2994 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:29:34,307] jepsen worker 57 - jepsen.util 2937 :info :write [19 11] :RequestCanceledException INFO [2019-12-17 12:29:34,307] jepsen worker 24 - jepsen.util 2994 :invoke :read [8 nil] INFO [2019-12-17 12:29:34,308] jepsen worker 57 - jepsen.util 3027 :invoke :read [19 nil] INFO [2019-12-17 12:29:34,322] jepsen worker 85 - jepsen.util 3325 :info :cas [28 [24 43]] :RequestCanceledException INFO [2019-12-17 12:29:34,322] jepsen worker 86 - jepsen.util 3056 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:29:34,322] jepsen worker 75 - jepsen.util 3315 :info :cas [25 [6 7]] :RequestCanceledException INFO [2019-12-17 12:29:34,322] jepsen worker 81 - jepsen.util 2781 :info :cas [27 [11 22]] :RequestCanceledException INFO [2019-12-17 12:29:34,323] jepsen worker 86 - jepsen.util 3056 :invoke :write [28 13] INFO [2019-12-17 12:29:34,324] jepsen worker 75 - jepsen.util 3405 :invoke :write [25 16] INFO [2019-12-17 12:29:34,324] jepsen worker 85 - jepsen.util 3415 :invoke :cas [28 [22 26]] INFO [2019-12-17 12:29:34,324] jepsen worker 81 - jepsen.util 2871 :invoke :read [27 nil] INFO [2019-12-17 12:29:34,337] jepsen worker 84 - jepsen.util 2964 :info :cas [28 [7 25]] :RequestCanceledException INFO [2019-12-17 12:29:34,338] jepsen worker 84 - jepsen.util 3054 :invoke :write [28 24] INFO [2019-12-17 12:29:34,353] jepsen worker 76 - jepsen.util 3136 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:29:34,353] jepsen worker 76 - jepsen.util 3136 :invoke :write [25 0] INFO [2019-12-17 12:29:34,373] jepsen worker 4 - jepsen.util 1804 :ok :read [1 19] INFO [2019-12-17 12:29:34,373] jepsen worker 4 - jepsen.util 1804 :invoke :write [1 8] INFO [2019-12-17 12:29:34,382] jepsen worker 88 - jepsen.util 2338 :fail :cas [29 [41 24]] :ValueNotSwapFrom INFO [2019-12-17 12:29:34,382] jepsen worker 88 - jepsen.util 2338 :invoke :read [29 nil] INFO [2019-12-17 12:29:34,389] jepsen worker 25 - jepsen.util 3535 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:29:34,389] jepsen worker 62 - jepsen.util 2852 :info :cas [20 [33 14]] :RequestCanceledException INFO [2019-12-17 12:29:34,390] jepsen worker 25 - jepsen.util 3535 :invoke :write [8 31] INFO [2019-12-17 12:29:34,390] jepsen worker 62 - jepsen.util 2942 :invoke :cas [20 [39 17]] INFO [2019-12-17 12:29:34,407] jepsen worker 83 - jepsen.util 3503 :info :cas [27 [11 0]] :RequestCanceledException INFO [2019-12-17 12:29:34,407] jepsen worker 83 - jepsen.util 3593 :invoke :read [27 nil] INFO [2019-12-17 12:29:34,423] jepsen worker 58 - jepsen.util 2758 :info :cas [19 [26 22]] :RequestCanceledException INFO [2019-12-17 12:29:34,423] jepsen worker 22 - jepsen.util 3082 :info :write [7 3] :RequestCanceledException INFO [2019-12-17 12:29:34,423] jepsen worker 40 - jepsen.util 3370 :info :write [13 31] :RequestCanceledException INFO [2019-12-17 12:29:34,425] jepsen worker 58 - jepsen.util 2848 :invoke :cas [19 [18 47]] INFO [2019-12-17 12:29:34,425] jepsen worker 22 - jepsen.util 3172 :invoke :read [7 nil] INFO [2019-12-17 12:29:34,425] jepsen worker 40 - jepsen.util 3460 :invoke :write [13 8] INFO [2019-12-17 12:29:34,425] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13665,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:29:34,425] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13665,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:29:34,425] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13669,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:29:34,426] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13669,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:29:34,426] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13670,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:29:34,426] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13670,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:29:34,437] jepsen worker 41 - jepsen.util 3281 :info :cas [13 [28 19]] :RequestCanceledException INFO [2019-12-17 12:29:34,437] jepsen worker 79 - jepsen.util 2329 :info :write [26 36] :RequestCanceledException INFO [2019-12-17 12:29:34,437] jepsen worker 41 - jepsen.util 3371 :invoke :write [13 11] INFO [2019-12-17 12:29:34,437] jepsen worker 79 - jepsen.util 2419 :invoke :cas [26 [0 12]] INFO [2019-12-17 12:29:34,454] jepsen worker 53 - jepsen.util 2213 :info :write [17 33] :RequestCanceledException INFO [2019-12-17 12:29:34,454] jepsen worker 26 - jepsen.util 3356 :info :write [8 49] :RequestCanceledException INFO [2019-12-17 12:29:34,455] jepsen worker 53 - jepsen.util 2303 :invoke :read [17 nil] INFO [2019-12-17 12:29:34,455] jepsen worker 26 - jepsen.util 3446 :invoke :read [8 nil] INFO [2019-12-17 12:29:34,459] jepsen worker 79 - jepsen.util 2419 :fail :cas [26 [0 12]] :ValueNotSwapFrom INFO [2019-12-17 12:29:34,459] jepsen worker 79 - jepsen.util 2419 :invoke :write [26 21] INFO [2019-12-17 12:29:34,489] jepsen worker 48 - jepsen.util 2478 :info :cas [16 [20 36]] :RequestCanceledException INFO [2019-12-17 12:29:34,489] jepsen worker 42 - jepsen.util 3642 :info :cas [14 [6 10]] :RequestCanceledException INFO [2019-12-17 12:29:34,490] jepsen worker 48 - jepsen.util 2568 :invoke :cas [16 [38 11]] INFO [2019-12-17 12:29:34,490] jepsen worker 42 - jepsen.util 3732 :invoke :write [14 1] INFO [2019-12-17 12:29:34,494] jepsen worker 53 - jepsen.util 2303 :ok :read [17 5] INFO [2019-12-17 12:29:34,494] jepsen worker 53 - jepsen.util 2303 :invoke :write [17 42] INFO [2019-12-17 12:29:34,507] jepsen worker 82 - jepsen.util 3682 :info :cas [27 [19 32]] :RequestCanceledException INFO [2019-12-17 12:29:34,507] jepsen worker 87 - jepsen.util 2067 :info :write [29 14] :RequestCanceledException INFO [2019-12-17 12:29:34,508] jepsen worker 82 - jepsen.util 3772 :invoke :write [27 26] INFO [2019-12-17 12:29:34,508] jepsen worker 87 - jepsen.util 2157 :invoke :cas [29 [22 35]] INFO [2019-12-17 12:29:34,523] jepsen worker 52 - jepsen.util 2392 :info :write [17 47] :RequestCanceledException INFO [2019-12-17 12:29:34,524] jepsen worker 52 - jepsen.util 2482 :invoke :write [17 8] INFO [2019-12-17 12:29:34,535] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:29:34,535] jepsen nemesis - couchbase.util Rebalance check count: 98 INFO [2019-12-17 12:29:34,535] jepsen nemesis - couchbase.util Rebalance stuck count: 79 INFO [2019-12-17 12:29:34,536] jepsen worker 17 - jepsen.util 3077 :info :cas [5 [41 30]] :RequestCanceledException INFO [2019-12-17 12:29:34,537] jepsen worker 17 - jepsen.util 3167 :invoke :read [5 nil] INFO [2019-12-17 12:29:34,554] jepsen worker 77 - jepsen.util 3047 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:29:34,554] jepsen worker 77 - jepsen.util 3047 :invoke :cas [25 [47 48]] INFO [2019-12-17 12:29:34,580] jepsen worker 54 - jepsen.util 2304 :info :write [18 11] :RequestCanceledException INFO [2019-12-17 12:29:34,581] jepsen worker 54 - jepsen.util 2394 :invoke :read [18 nil] INFO [2019-12-17 12:29:34,592] jepsen worker 80 - jepsen.util 2330 :ok :read [26 11] INFO [2019-12-17 12:29:34,592] jepsen worker 80 - jepsen.util 2330 :invoke :read [26 nil] INFO [2019-12-17 12:29:34,605] jepsen worker 33 - jepsen.util 3273 :info :cas [11 [2 40]] :RequestCanceledException INFO [2019-12-17 12:29:34,606] jepsen worker 33 - jepsen.util 3363 :invoke :write [11 26] INFO [2019-12-17 12:29:34,623] jepsen worker 8 - jepsen.util 2798 :info :cas [2 [18 45]] :RequestCanceledException INFO [2019-12-17 12:29:34,623] jepsen worker 68 - jepsen.util 2858 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:29:34,623] jepsen worker 68 - jepsen.util 2858 :invoke :cas [22 [15 2]] INFO [2019-12-17 12:29:34,624] jepsen worker 8 - jepsen.util 2888 :invoke :write [2 15] INFO [2019-12-17 12:29:34,636] jepsen worker 47 - jepsen.util 2747 :ok :write [15 41] INFO [2019-12-17 12:29:34,636] jepsen worker 47 - jepsen.util 2747 :invoke :read [15 nil] INFO [2019-12-17 12:29:34,637] jepsen worker 66 - jepsen.util 3126 :info :write [22 31] :RequestCanceledException INFO [2019-12-17 12:29:34,637] jepsen worker 66 - jepsen.util 3216 :invoke :write [22 41] INFO [2019-12-17 12:29:34,640] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13694,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:29:34,640] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13694,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:29:34,640] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13698,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:29:34,640] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13698,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:29:34,645] jepsen worker 32 - jepsen.util 2912 :ok :write [10 49] INFO [2019-12-17 12:29:34,645] jepsen worker 11 - jepsen.util 2891 :ok :write [3 46] INFO [2019-12-17 12:29:34,645] jepsen worker 19 - jepsen.util 2629 :ok :write [6 29] INFO [2019-12-17 12:29:34,645] jepsen worker 32 - jepsen.util 2912 :invoke :read [10 nil] INFO [2019-12-17 12:29:34,646] jepsen worker 19 - jepsen.util 2629 :invoke :cas [6 [30 32]] INFO [2019-12-17 12:29:34,646] jepsen worker 11 - jepsen.util 2891 :invoke :cas [3 [1 8]] INFO [2019-12-17 12:29:34,647] jepsen worker 72 - jepsen.util 2412 :ok :write [24 28] INFO [2019-12-17 12:29:34,647] jepsen worker 72 - jepsen.util 2412 :invoke :write [24 21] INFO [2019-12-17 12:29:34,688] jepsen worker 31 - jepsen.util 2281 :ok :read [10 49] INFO [2019-12-17 12:29:34,688] jepsen worker 79 - jepsen.util 2419 :ok :write [26 21] INFO [2019-12-17 12:29:34,688] jepsen worker 31 - jepsen.util 2281 :invoke :write [10 27] INFO [2019-12-17 12:29:34,688] jepsen worker 79 - jepsen.util 2419 :invoke :read [26 nil] INFO [2019-12-17 12:29:34,691] jepsen worker 39 - jepsen.util 3279 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:29:34,691] jepsen worker 39 - jepsen.util 3279 :invoke :write [13 40] INFO [2019-12-17 12:29:34,694] jepsen worker 63 - jepsen.util 2043 :fail :cas [21 [6 47]] :ValueNotSwapFrom INFO [2019-12-17 12:29:34,694] jepsen worker 63 - jepsen.util 2043 :invoke :write [21 3] INFO [2019-12-17 12:29:34,702] jepsen worker 18 - jepsen.util 2628 :fail :cas [6 [28 23]] :ValueNotSwapFrom INFO [2019-12-17 12:29:34,702] jepsen worker 18 - jepsen.util 2628 :invoke :write [6 15] INFO [2019-12-17 12:29:34,707] jepsen worker 80 - jepsen.util 2330 :ok :read [26 11] INFO [2019-12-17 12:29:34,707] jepsen worker 80 - jepsen.util 2330 :invoke :read [26 nil] INFO [2019-12-17 12:29:34,722] jepsen worker 3 - jepsen.util 2163 :info :write [1 31] :RequestCanceledException INFO [2019-12-17 12:29:34,722] jepsen worker 28 - jepsen.util 1738 :info :cas [9 [37 45]] :RequestCanceledException INFO [2019-12-17 12:29:34,723] jepsen worker 28 - jepsen.util 1828 :invoke :read [9 nil] INFO [2019-12-17 12:29:34,723] jepsen worker 3 - jepsen.util 2253 :invoke :read [1 nil] INFO [2019-12-17 12:29:34,738] jepsen worker 20 - jepsen.util 2090 :ok :read [6 29] INFO [2019-12-17 12:29:34,738] jepsen worker 20 - jepsen.util 2090 :invoke :read [6 nil] INFO [2019-12-17 12:29:34,743] jepsen worker 27 - jepsen.util 2097 :fail :cas [9 [48 4]] :ValueNotSwapFrom INFO [2019-12-17 12:29:34,743] jepsen worker 27 - jepsen.util 2097 :invoke :read [9 nil] INFO [2019-12-17 12:29:34,789] jepsen worker 38 - jepsen.util 2288 :info :write [12 39] :RequestCanceledException INFO [2019-12-17 12:29:34,795] jepsen worker 38 - jepsen.util 2378 :invoke :write [12 6] INFO [2019-12-17 12:29:34,804] jepsen worker 28 - jepsen.util 1828 :ok :read [9 10] INFO [2019-12-17 12:29:34,804] jepsen worker 28 - jepsen.util 1828 :invoke :cas [9 [48 5]] INFO [2019-12-17 12:29:34,808] jepsen worker 61 - jepsen.util 3301 :info :write [20 4] :RequestCanceledException INFO [2019-12-17 12:29:34,810] jepsen worker 88 - jepsen.util 2338 :ok :read [29 38] INFO [2019-12-17 12:29:34,810] jepsen worker 88 - jepsen.util 2338 :invoke :write [29 22] INFO [2019-12-17 12:29:34,814] jepsen worker 11 - jepsen.util 2891 :fail :cas [3 [1 8]] :ValueNotSwapFrom INFO [2019-12-17 12:29:34,814] jepsen worker 61 - jepsen.util 3391 :invoke :cas [20 [8 25]] INFO [2019-12-17 12:29:34,814] jepsen worker 11 - jepsen.util 2891 :invoke :read [3 nil] INFO [2019-12-17 12:29:34,816] jepsen worker 19 - jepsen.util 2629 :fail :cas [6 [30 32]] :ValueNotSwapFrom INFO [2019-12-17 12:29:34,816] jepsen worker 19 - jepsen.util 2629 :invoke :read [6 nil] INFO [2019-12-17 12:29:34,821] jepsen worker 2 - jepsen.util 2972 :info :write [0 27] :RequestCanceledException INFO [2019-12-17 12:29:34,828] jepsen worker 2 - jepsen.util 3062 :invoke :cas [0 [33 37]] INFO [2019-12-17 12:29:34,844] jepsen worker 87 - jepsen.util 2157 :fail :cas [29 [22 35]] :ValueNotSwapFrom INFO [2019-12-17 12:29:34,844] jepsen worker 87 - jepsen.util 2157 :invoke :write [29 26] INFO [2019-12-17 12:29:34,859] jepsen worker 28 - jepsen.util 1828 :fail :cas [9 [48 5]] :ValueNotSwapFrom INFO [2019-12-17 12:29:34,859] jepsen worker 28 - jepsen.util 1828 :invoke :write [9 12] INFO [2019-12-17 12:29:34,880] jepsen worker 79 - jepsen.util 2419 :ok :read [26 21] INFO [2019-12-17 12:29:34,880] jepsen worker 79 - jepsen.util 2419 :invoke :read [26 nil] INFO [2019-12-17 12:29:34,904] jepsen worker 5 - jepsen.util 1985 :ok :write [1 43] INFO [2019-12-17 12:29:34,904] jepsen worker 5 - jepsen.util 1985 :invoke :write [1 41] INFO [2019-12-17 12:29:34,908] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13724,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:29:34,909] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13724,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:29:34,909] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13726,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:29:34,909] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13726,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:29:34,936] jepsen worker 70 - jepsen.util 2680 :info :write [23 25] :RequestCanceledException INFO [2019-12-17 12:29:34,938] jepsen worker 28 - jepsen.util 1828 :ok :write [9 12] INFO [2019-12-17 12:29:34,938] jepsen worker 28 - jepsen.util 1828 :invoke :read [9 nil] INFO [2019-12-17 12:29:34,939] jepsen worker 70 - jepsen.util 2770 :invoke :write [23 36] INFO [2019-12-17 12:29:34,956] jepsen worker 3 - jepsen.util 2253 :ok :read [1 43] INFO [2019-12-17 12:29:34,956] jepsen worker 3 - jepsen.util 2253 :invoke :write [1 38] INFO [2019-12-17 12:29:34,979] jepsen worker 28 - jepsen.util 1828 :ok :read [9 12] INFO [2019-12-17 12:29:34,980] jepsen worker 28 - jepsen.util 1828 :invoke :write [9 16] INFO [2019-12-17 12:29:34,982] jepsen worker 4 - jepsen.util 1804 :ok :write [1 8] INFO [2019-12-17 12:29:34,982] jepsen worker 4 - jepsen.util 1804 :invoke :write [1 11] INFO [2019-12-17 12:29:34,990] jepsen worker 51 - jepsen.util 2121 :info :write [17 49] :RequestCanceledException INFO [2019-12-17 12:29:34,993] jepsen worker 51 - jepsen.util 2211 :invoke :read [17 nil] INFO [2019-12-17 12:29:35,012] jepsen worker 13 - jepsen.util 2263 :ok :write [4 15] INFO [2019-12-17 12:29:35,012] jepsen worker 13 - jepsen.util 2263 :invoke :read [4 nil] INFO [2019-12-17 12:29:35,019] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13735,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:29:35,019] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13735,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:29:35,019] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13739,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:29:35,019] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13739,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:29:35,019] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13740,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:29:35,020] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13740,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:29:35,055] jepsen worker 28 - jepsen.util 1828 :ok :write [9 16] INFO [2019-12-17 12:29:35,056] jepsen worker 28 - jepsen.util 1828 :invoke :write [9 10] INFO [2019-12-17 12:29:35,068] jepsen worker 56 - jepsen.util 2306 :ok :write [18 15] INFO [2019-12-17 12:29:35,068] jepsen worker 56 - jepsen.util 2306 :invoke :read [18 nil] INFO [2019-12-17 12:29:35,118] jepsen worker 18 - jepsen.util 2628 :ok :write [6 15] INFO [2019-12-17 12:29:35,118] jepsen worker 10 - jepsen.util 2350 :ok :write [3 21] INFO [2019-12-17 12:29:35,118] jepsen worker 18 - jepsen.util 2628 :invoke :read [6 nil] INFO [2019-12-17 12:29:35,118] jepsen worker 10 - jepsen.util 2350 :invoke :read [3 nil] INFO [2019-12-17 12:29:35,118] jepsen worker 73 - jepsen.util 2413 :ok :write [24 5] INFO [2019-12-17 12:29:35,118] jepsen worker 73 - jepsen.util 2413 :invoke :read [24 nil] INFO [2019-12-17 12:29:35,121] jepsen worker 46 - jepsen.util 2656 :ok :write [15 7] INFO [2019-12-17 12:29:35,121] jepsen worker 46 - jepsen.util 2656 :invoke :read [15 nil] INFO [2019-12-17 12:29:35,136] jepsen worker 36 - jepsen.util 2286 :ok :write [12 32] INFO [2019-12-17 12:29:35,136] jepsen worker 36 - jepsen.util 2286 :invoke :write [12 35] INFO [2019-12-17 12:29:35,138] jepsen worker 78 - jepsen.util 2598 :ok :write [26 16] INFO [2019-12-17 12:29:35,138] jepsen worker 78 - jepsen.util 2598 :invoke :cas [26 [8 32]] INFO [2019-12-17 12:29:35,141] jepsen worker 69 - jepsen.util 2319 :ok :write [23 48] INFO [2019-12-17 12:29:35,141] jepsen worker 69 - jepsen.util 2319 :invoke :cas [23 [32 18]] INFO [2019-12-17 12:29:35,146] jepsen worker 88 - jepsen.util 2338 :ok :write [29 22] INFO [2019-12-17 12:29:35,146] jepsen worker 88 - jepsen.util 2338 :invoke :write [29 27] INFO [2019-12-17 12:29:35,163] jepsen worker 45 - jepsen.util 2565 :fail :cas [15 [31 15]] :ValueNotSwapFrom INFO [2019-12-17 12:29:35,163] jepsen worker 45 - jepsen.util 2565 :invoke :write [15 14] INFO [2019-12-17 12:29:35,169] jepsen worker 28 - jepsen.util 1828 :ok :write [9 10] INFO [2019-12-17 12:29:35,169] jepsen worker 28 - jepsen.util 1828 :invoke :write [9 11] INFO [2019-12-17 12:29:35,184] jepsen worker 32 - jepsen.util 2912 :ok :read [10 49] INFO [2019-12-17 12:29:35,184] jepsen worker 32 - jepsen.util 2912 :invoke :read [10 nil] INFO [2019-12-17 12:29:35,191] jepsen worker 19 - jepsen.util 2629 :ok :read [6 15] INFO [2019-12-17 12:29:35,192] jepsen worker 19 - jepsen.util 2629 :invoke :write [6 21] INFO [2019-12-17 12:29:35,207] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13757,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:29:35,207] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13757,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:29:35,207] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13758,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:29:35,207] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13758,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:29:35,212] jepsen worker 47 - jepsen.util 2747 :ok :read [15 7] INFO [2019-12-17 12:29:35,212] jepsen worker 47 - jepsen.util 2747 :invoke :write [15 21] INFO [2019-12-17 12:29:35,220] jepsen worker 31 - jepsen.util 2281 :ok :write [10 27] INFO [2019-12-17 12:29:35,221] jepsen worker 31 - jepsen.util 2281 :invoke :read [10 nil] INFO [2019-12-17 12:29:35,231] jepsen worker 46 - jepsen.util 2656 :ok :read [15 7] INFO [2019-12-17 12:29:35,231] jepsen worker 46 - jepsen.util 2656 :invoke :write [15 2] INFO [2019-12-17 12:29:35,237] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13767,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:29:35,238] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13767,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:29:35,238] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13769,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:29:35,238] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13769,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:29:35,255] jepsen worker 18 - jepsen.util 2628 :ok :read [6 15] INFO [2019-12-17 12:29:35,255] jepsen worker 18 - jepsen.util 2628 :invoke :write [6 45] INFO [2019-12-17 12:29:35,256] jepsen worker 11 - jepsen.util 2891 :ok :read [3 21] INFO [2019-12-17 12:29:35,256] jepsen worker 11 - jepsen.util 2891 :invoke :write [3 4] INFO [2019-12-17 12:29:35,273] jepsen worker 3 - jepsen.util 2253 :ok :write [1 38] INFO [2019-12-17 12:29:35,273] jepsen worker 3 - jepsen.util 2253 :invoke :cas [1 [41 49]] INFO [2019-12-17 12:29:35,288] jepsen worker 31 - jepsen.util 2281 :ok :read [10 27] INFO [2019-12-17 12:29:35,288] jepsen worker 31 - jepsen.util 2281 :invoke :write [10 23] INFO [2019-12-17 12:29:35,290] jepsen worker 55 - jepsen.util 2665 :info :write [18 33] :RequestCanceledException INFO [2019-12-17 12:29:35,292] jepsen worker 55 - jepsen.util 2755 :invoke :write [18 16] INFO [2019-12-17 12:29:35,329] jepsen worker 72 - jepsen.util 2412 :ok :write [24 21] INFO [2019-12-17 12:29:35,329] jepsen worker 72 - jepsen.util 2412 :invoke :cas [24 [2 20]] INFO [2019-12-17 12:29:35,351] jepsen worker 63 - jepsen.util 2043 :ok :write [21 3] INFO [2019-12-17 12:29:35,351] jepsen worker 63 - jepsen.util 2043 :invoke :cas [21 [28 17]] INFO [2019-12-17 12:29:35,374] jepsen worker 80 - jepsen.util 2330 :ok :read [26 16] INFO [2019-12-17 12:29:35,374] jepsen worker 80 - jepsen.util 2330 :invoke :write [26 35] INFO [2019-12-17 12:29:35,375] jepsen worker 72 - jepsen.util 2412 :fail :cas [24 [2 20]] :ValueNotSwapFrom INFO [2019-12-17 12:29:35,375] jepsen worker 72 - jepsen.util 2412 :invoke :write [24 25] INFO [2019-12-17 12:29:35,377] jepsen worker 28 - jepsen.util 1828 :ok :write [9 11] INFO [2019-12-17 12:29:35,377] jepsen worker 28 - jepsen.util 1828 :invoke :cas [9 [28 0]] INFO [2019-12-17 12:29:35,382] jepsen worker 6 - jepsen.util 3246 :info :cas [2 [23 11]] :RequestCanceledException INFO [2019-12-17 12:29:35,386] jepsen worker 6 - jepsen.util 3336 :invoke :read [2 nil] INFO [2019-12-17 12:29:35,395] jepsen worker 56 - jepsen.util 2306 :ok :read [18 15] INFO [2019-12-17 12:29:35,395] jepsen worker 56 - jepsen.util 2306 :invoke :read [18 nil] INFO [2019-12-17 12:29:35,399] jepsen worker 79 - jepsen.util 2419 :ok :read [26 16] INFO [2019-12-17 12:29:35,399] jepsen worker 79 - jepsen.util 2419 :invoke :read [26 nil] INFO [2019-12-17 12:29:35,407] jepsen worker 67 - jepsen.util 3577 :info :write [22 45] :RequestCanceledException INFO [2019-12-17 12:29:35,407] jepsen worker 60 - jepsen.util 3390 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:29:35,407] jepsen worker 60 - jepsen.util 3390 :invoke :write [20 15] INFO [2019-12-17 12:29:35,409] jepsen worker 67 - jepsen.util 3667 :invoke :read [22 nil] INFO [2019-12-17 12:29:35,422] jepsen worker 44 - jepsen.util 2834 :info :write [14 19] :RequestCanceledException INFO [2019-12-17 12:29:35,426] jepsen worker 44 - jepsen.util 2924 :invoke :cas [14 [17 48]] INFO [2019-12-17 12:29:35,437] jepsen worker 65 - jepsen.util 2045 :info :cas [21 [46 4]] :RequestCanceledException INFO [2019-12-17 12:29:35,437] jepsen worker 23 - jepsen.util 2813 :info :write [7 12] :RequestCanceledException INFO [2019-12-17 12:29:35,437] jepsen worker 49 - jepsen.util 2299 :info :write [16 22] :RequestCanceledException INFO [2019-12-17 12:29:35,439] jepsen worker 65 - jepsen.util 2135 :invoke :cas [21 [11 31]] INFO [2019-12-17 12:29:35,441] jepsen worker 49 - jepsen.util 2389 :invoke :write [16 43] INFO [2019-12-17 12:29:35,442] jepsen worker 23 - jepsen.util 2903 :invoke :write [7 10] INFO [2019-12-17 12:29:35,442] jepsen worker 28 - jepsen.util 1828 :fail :cas [9 [28 0]] :ValueNotSwapFrom INFO [2019-12-17 12:29:35,442] jepsen worker 28 - jepsen.util 1828 :invoke :cas [9 [44 1]] INFO [2019-12-17 12:29:35,448] jepsen worker 63 - jepsen.util 2043 :fail :cas [21 [28 17]] :ValueNotSwapFrom INFO [2019-12-17 12:29:35,448] jepsen worker 63 - jepsen.util 2043 :invoke :read [21 nil] INFO [2019-12-17 12:29:35,453] jepsen worker 34 - jepsen.util 3184 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:29:35,453] jepsen worker 7 - jepsen.util 2977 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:29:35,454] jepsen worker 34 - jepsen.util 3184 :invoke :write [11 9] INFO [2019-12-17 12:29:35,454] jepsen worker 7 - jepsen.util 2977 :invoke :cas [2 [35 4]] INFO [2019-12-17 12:29:35,465] jepsen worker 69 - jepsen.util 2319 :fail :cas [23 [32 18]] :ValueNotSwapFrom INFO [2019-12-17 12:29:35,465] jepsen worker 69 - jepsen.util 2319 :invoke :read [23 nil] INFO [2019-12-17 12:29:35,515] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13789,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:29:35,516] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13789,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:29:35,516] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13792,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:29:35,516] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13792,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:29:35,552] jepsen worker 63 - jepsen.util 2043 :ok :read [21 3] INFO [2019-12-17 12:29:35,552] jepsen worker 63 - jepsen.util 2043 :invoke :read [21 nil] INFO [2019-12-17 12:29:35,554] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:29:35,554] jepsen nemesis - couchbase.util Rebalance check count: 99 INFO [2019-12-17 12:29:35,554] jepsen nemesis - couchbase.util Rebalance stuck count: 80 INFO [2019-12-17 12:29:35,555] jepsen worker 28 - jepsen.util 1828 :fail :cas [9 [44 1]] :ValueNotSwapFrom INFO [2019-12-17 12:29:35,555] jepsen worker 28 - jepsen.util 1828 :invoke :write [9 44] INFO [2019-12-17 12:29:35,563] jepsen worker 54 - jepsen.util 2394 :ok :read [18 15] INFO [2019-12-17 12:29:35,563] jepsen worker 54 - jepsen.util 2394 :invoke :write [18 12] INFO [2019-12-17 12:29:35,565] jepsen worker 45 - jepsen.util 2565 :ok :write [15 14] INFO [2019-12-17 12:29:35,565] jepsen worker 45 - jepsen.util 2565 :invoke :write [15 0] INFO [2019-12-17 12:29:35,590] jepsen worker 9 - jepsen.util 2709 :info :cas [3 [40 26]] :RequestCanceledException INFO [2019-12-17 12:29:35,590] jepsen worker 9 - jepsen.util 2799 :invoke :write [3 1] INFO [2019-12-17 12:29:35,600] jepsen worker 63 - jepsen.util 2043 :ok :read [21 3] INFO [2019-12-17 12:29:35,600] jepsen worker 63 - jepsen.util 2043 :invoke :cas [21 [13 14]] INFO [2019-12-17 12:29:35,614] jepsen worker 3 - jepsen.util 2253 :fail :cas [1 [41 49]] :ValueNotSwapFrom INFO [2019-12-17 12:29:35,614] jepsen worker 3 - jepsen.util 2253 :invoke :write [1 11] INFO [2019-12-17 12:29:35,623] jepsen worker 64 - jepsen.util 2044 :info :write [21 25] :RequestCanceledException INFO [2019-12-17 12:29:35,623] jepsen worker 74 - jepsen.util 2324 :info :write [24 39] :RequestCanceledException INFO [2019-12-17 12:29:35,623] jepsen worker 37 - jepsen.util 2017 :info :write [12 40] :RequestCanceledException INFO [2019-12-17 12:29:35,624] jepsen worker 64 - jepsen.util 2134 :invoke :read [21 nil] INFO [2019-12-17 12:29:35,624] jepsen worker 37 - jepsen.util 2107 :invoke :read [12 nil] INFO [2019-12-17 12:29:35,624] jepsen worker 74 - jepsen.util 2414 :invoke :write [24 15] INFO [2019-12-17 12:29:35,636] jepsen worker 63 - jepsen.util 2043 :fail :cas [21 [13 14]] :ValueNotSwapFrom INFO [2019-12-17 12:29:35,637] jepsen worker 63 - jepsen.util 2043 :invoke :read [21 nil] INFO [2019-12-17 12:29:35,638] jepsen worker 79 - jepsen.util 2419 :ok :read [26 16] INFO [2019-12-17 12:29:35,638] jepsen worker 79 - jepsen.util 2419 :invoke :cas [26 [36 12]] INFO [2019-12-17 12:29:35,654] jepsen worker 89 - jepsen.util 2249 :info :write [29 37] :RequestCanceledException INFO [2019-12-17 12:29:35,655] jepsen worker 89 - jepsen.util 2339 :invoke :cas [29 [18 25]] INFO [2019-12-17 12:29:35,681] jepsen worker 63 - jepsen.util 2043 :ok :read [21 3] INFO [2019-12-17 12:29:35,681] jepsen worker 63 - jepsen.util 2043 :invoke :cas [21 [2 14]] INFO [2019-12-17 12:29:35,703] jepsen worker 63 - jepsen.util 2043 :fail :cas [21 [2 14]] :ValueNotSwapFrom INFO [2019-12-17 12:29:35,703] jepsen worker 63 - jepsen.util 2043 :invoke :read [21 nil] INFO [2019-12-17 12:29:35,705] jepsen worker 28 - jepsen.util 1828 :ok :write [9 44] INFO [2019-12-17 12:29:35,705] jepsen worker 28 - jepsen.util 1828 :invoke :write [9 11] INFO [2019-12-17 12:29:35,721] jepsen worker 12 - jepsen.util 2442 :info :write [4 6] :RequestCanceledException INFO [2019-12-17 12:29:35,723] jepsen worker 12 - jepsen.util 2532 :invoke :cas [4 [3 8]] INFO [2019-12-17 12:29:35,731] jepsen worker 63 - jepsen.util 2043 :ok :read [21 3] INFO [2019-12-17 12:29:35,732] jepsen worker 63 - jepsen.util 2043 :invoke :write [21 43] INFO [2019-12-17 12:29:35,734] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13817,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:29:35,734] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13817,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:29:35,734] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13819,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:29:35,734] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13819,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:29:35,760] jepsen worker 56 - jepsen.util 2306 :ok :read [18 15] INFO [2019-12-17 12:29:35,760] jepsen worker 56 - jepsen.util 2306 :invoke :cas [18 [24 27]] INFO [2019-12-17 12:29:35,765] jepsen worker 13 - jepsen.util 2263 :ok :read [4 15] INFO [2019-12-17 12:29:35,765] jepsen worker 13 - jepsen.util 2263 :invoke :read [4 nil] INFO [2019-12-17 12:29:35,775] jepsen worker 12 - jepsen.util 2532 :fail :cas [4 [3 8]] :ValueNotSwapFrom INFO [2019-12-17 12:29:35,776] jepsen worker 12 - jepsen.util 2532 :invoke :read [4 nil] INFO [2019-12-17 12:29:35,789] jepsen worker 71 - jepsen.util 2231 :info :write [23 16] :RequestCanceledException INFO [2019-12-17 12:29:35,790] jepsen worker 28 - jepsen.util 1828 :ok :write [9 11] INFO [2019-12-17 12:29:35,790] jepsen worker 28 - jepsen.util 1828 :invoke :read [9 nil] INFO [2019-12-17 12:29:35,791] jepsen worker 71 - jepsen.util 2321 :invoke :read [23 nil] INFO [2019-12-17 12:29:35,809] jepsen worker 71 - jepsen.util 2321 :ok :read [23 48] INFO [2019-12-17 12:29:35,809] jepsen worker 71 - jepsen.util 2321 :invoke :write [23 44] INFO [2019-12-17 12:29:35,813] jepsen worker 72 - jepsen.util 2412 :ok :write [24 25] INFO [2019-12-17 12:29:35,813] jepsen worker 72 - jepsen.util 2412 :invoke :cas [24 [36 33]] INFO [2019-12-17 12:29:35,814] jepsen worker 11 - jepsen.util 2891 :ok :write [3 4] INFO [2019-12-17 12:29:35,814] jepsen worker 11 - jepsen.util 2891 :invoke :write [3 9] INFO [2019-12-17 12:29:35,815] jepsen worker 28 - jepsen.util 1828 :ok :read [9 11] INFO [2019-12-17 12:29:35,816] jepsen worker 28 - jepsen.util 1828 :invoke :write [9 42] INFO [2019-12-17 12:29:35,830] jepsen worker 37 - jepsen.util 2107 :ok :read [12 32] INFO [2019-12-17 12:29:35,830] jepsen worker 37 - jepsen.util 2107 :invoke :write [12 2] INFO [2019-12-17 12:29:35,889] jepsen worker 72 - jepsen.util 2412 :fail :cas [24 [36 33]] :ValueNotSwapFrom INFO [2019-12-17 12:29:35,889] jepsen worker 72 - jepsen.util 2412 :invoke :read [24 nil] INFO [2019-12-17 12:29:35,895] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13838,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:29:35,895] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13838,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:29:35,895] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13841,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:29:35,895] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13841,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:29:35,895] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13842,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:29:35,895] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13842,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:29:35,898] jepsen worker 65 - jepsen.util 2135 :fail :cas [21 [11 31]] :ValueNotSwapFrom INFO [2019-12-17 12:29:35,898] jepsen worker 65 - jepsen.util 2135 :invoke :write [21 33] INFO [2019-12-17 12:29:35,904] jepsen worker 63 - jepsen.util 2043 :ok :write [21 43] INFO [2019-12-17 12:29:35,904] jepsen worker 63 - jepsen.util 2043 :invoke :write [21 40] INFO [2019-12-17 12:29:35,926] jepsen worker 72 - jepsen.util 2412 :ok :read [24 25] INFO [2019-12-17 12:29:35,926] jepsen worker 72 - jepsen.util 2412 :invoke :write [24 36] INFO [2019-12-17 12:29:35,946] jepsen worker 19 - jepsen.util 2629 :ok :write [6 21] INFO [2019-12-17 12:29:35,946] jepsen worker 19 - jepsen.util 2629 :invoke :write [6 45] INFO [2019-12-17 12:29:35,999] jepsen worker 51 - jepsen.util 2211 :ok :read [17 5] INFO [2019-12-17 12:29:35,999] jepsen worker 51 - jepsen.util 2211 :invoke :write [17 45] INFO [2019-12-17 12:29:36,002] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13848,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:29:36,002] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13848,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:29:36,002] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13849,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:29:36,002] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13849,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:29:36,002] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13851,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:29:36,002] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13851,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:29:36,019] jepsen worker 78 - jepsen.util 2598 :fail :cas [26 [8 32]] :ValueNotSwapFrom INFO [2019-12-17 12:29:36,019] jepsen worker 78 - jepsen.util 2598 :invoke :cas [26 [16 8]] INFO [2019-12-17 12:29:36,027] jepsen worker 64 - jepsen.util 2134 :ok :read [21 43] INFO [2019-12-17 12:29:36,027] jepsen worker 64 - jepsen.util 2134 :invoke :read [21 nil] INFO [2019-12-17 12:29:36,037] jepsen worker 16 - jepsen.util 3076 :info :cas [5 [2 40]] :RequestCanceledException INFO [2019-12-17 12:29:36,039] jepsen worker 16 - jepsen.util 3166 :invoke :cas [5 [27 26]] INFO [2019-12-17 12:29:36,046] jepsen worker 46 - jepsen.util 2656 :ok :write [15 2] INFO [2019-12-17 12:29:36,046] jepsen worker 46 - jepsen.util 2656 :invoke :write [15 35] INFO [2019-12-17 12:29:36,049] jepsen worker 63 - jepsen.util 2043 :ok :write [21 40] INFO [2019-12-17 12:29:36,049] jepsen worker 63 - jepsen.util 2043 :invoke :read [21 nil] INFO [2019-12-17 12:29:36,060] jepsen worker 74 - jepsen.util 2414 :ok :write [24 15] INFO [2019-12-17 12:29:36,060] jepsen worker 74 - jepsen.util 2414 :invoke :read [24 nil] INFO [2019-12-17 12:29:36,083] jepsen worker 69 - jepsen.util 2319 :ok :read [23 48] INFO [2019-12-17 12:29:36,083] jepsen worker 69 - jepsen.util 2319 :invoke :cas [23 [35 15]] INFO [2019-12-17 12:29:36,096] jepsen worker 63 - jepsen.util 2043 :ok :read [21 40] INFO [2019-12-17 12:29:36,096] jepsen worker 63 - jepsen.util 2043 :invoke :read [21 nil] INFO [2019-12-17 12:29:36,122] jepsen worker 15 - jepsen.util 2895 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:29:36,122] jepsen worker 30 - jepsen.util 2820 :info :write [10 29] :RequestCanceledException INFO [2019-12-17 12:29:36,122] jepsen worker 14 - jepsen.util 2174 :info :write [4 47] :RequestCanceledException INFO [2019-12-17 12:29:36,122] jepsen worker 15 - jepsen.util 2895 :invoke :write [5 13] INFO [2019-12-17 12:29:36,123] jepsen worker 14 - jepsen.util 2264 :invoke :write [4 1] INFO [2019-12-17 12:29:36,123] jepsen worker 30 - jepsen.util 2910 :invoke :read [10 nil] INFO [2019-12-17 12:29:36,137] jepsen worker 29 - jepsen.util 2009 :info :write [9 23] :RequestCanceledException INFO [2019-12-17 12:29:36,139] jepsen worker 29 - jepsen.util 2099 :invoke :cas [9 [47 21]] INFO [2019-12-17 12:29:36,140] jepsen worker 63 - jepsen.util 2043 :ok :read [21 40] INFO [2019-12-17 12:29:36,141] jepsen worker 63 - jepsen.util 2043 :invoke :cas [21 [37 3]] INFO [2019-12-17 12:29:36,151] jepsen worker 65 - jepsen.util 2135 :ok :write [21 33] INFO [2019-12-17 12:29:36,151] jepsen worker 65 - jepsen.util 2135 :invoke :read [21 nil] INFO [2019-12-17 12:29:36,163] jepsen worker 19 - jepsen.util 2629 :ok :write [6 45] INFO [2019-12-17 12:29:36,164] jepsen worker 19 - jepsen.util 2629 :invoke :write [6 19] INFO [2019-12-17 12:29:36,173] jepsen worker 28 - jepsen.util 1828 :ok :write [9 42] INFO [2019-12-17 12:29:36,173] jepsen worker 28 - jepsen.util 1828 :invoke :read [9 nil] INFO [2019-12-17 12:29:36,184] jepsen worker 89 - jepsen.util 2339 :fail :cas [29 [18 25]] :ValueNotSwapFrom INFO [2019-12-17 12:29:36,184] jepsen worker 89 - jepsen.util 2339 :invoke :cas [29 [34 4]] INFO [2019-12-17 12:29:36,187] jepsen worker 63 - jepsen.util 2043 :fail :cas [21 [37 3]] :ValueNotSwapFrom INFO [2019-12-17 12:29:36,187] jepsen worker 63 - jepsen.util 2043 :invoke :read [21 nil] INFO [2019-12-17 12:29:36,189] jepsen worker 35 - jepsen.util 2825 :info :cas [11 [37 21]] :RequestCanceledException INFO [2019-12-17 12:29:36,190] jepsen worker 35 - jepsen.util 2915 :invoke :cas [11 [14 10]] INFO [2019-12-17 12:29:36,193] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13874,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:29:36,193] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13874,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:29:36,193] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13875,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:29:36,193] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13875,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:29:36,193] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13876,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:29:36,193] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13876,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:29:36,201] jepsen worker 30 - jepsen.util 2910 :ok :read [10 27] INFO [2019-12-17 12:29:36,201] jepsen worker 30 - jepsen.util 2910 :invoke :write [10 11] INFO [2019-12-17 12:29:36,221] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13882,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:29:36,221] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13882,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:29:36,222] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13885,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:29:36,222] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13885,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:29:36,222] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13886,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:29:36,222] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13886,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:29:36,226] jepsen worker 69 - jepsen.util 2319 :fail :cas [23 [35 15]] :ValueNotSwapFrom INFO [2019-12-17 12:29:36,227] jepsen worker 69 - jepsen.util 2319 :invoke :read [23 nil] INFO [2019-12-17 12:29:36,237] jepsen worker 43 - jepsen.util 2923 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:29:36,237] jepsen worker 43 - jepsen.util 2923 :invoke :write [14 20] INFO [2019-12-17 12:29:36,254] jepsen worker 0 - jepsen.util 3240 :info :write [0 42] :RequestCanceledException INFO [2019-12-17 12:29:36,254] jepsen worker 50 - jepsen.util 3470 :info :write [16 20] :RequestCanceledException INFO [2019-12-17 12:29:36,255] jepsen worker 0 - jepsen.util 3330 :invoke :write [0 5] INFO [2019-12-17 12:29:36,255] jepsen worker 50 - jepsen.util 3560 :invoke :cas [16 [46 14]] INFO [2019-12-17 12:29:36,286] jepsen worker 59 - jepsen.util 2939 :info :write [19 13] :RequestCanceledException INFO [2019-12-17 12:29:36,287] jepsen worker 29 - jepsen.util 2099 :fail :cas [9 [47 21]] :ValueNotSwapFrom INFO [2019-12-17 12:29:36,287] jepsen worker 29 - jepsen.util 2099 :invoke :write [9 24] INFO [2019-12-17 12:29:36,287] jepsen worker 59 - jepsen.util 3029 :invoke :write [19 38] INFO [2019-12-17 12:29:36,307] jepsen worker 1 - jepsen.util 2971 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:29:36,307] jepsen worker 1 - jepsen.util 2971 :invoke :read [0 nil] INFO [2019-12-17 12:29:36,322] jepsen worker 57 - jepsen.util 3027 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:29:36,322] jepsen worker 21 - jepsen.util 3171 :info :write [7 5] :RequestCanceledException INFO [2019-12-17 12:29:36,322] jepsen worker 57 - jepsen.util 3027 :invoke :read [19 nil] INFO [2019-12-17 12:29:36,324] jepsen worker 21 - jepsen.util 3261 :invoke :read [7 nil] INFO [2019-12-17 12:29:36,350] jepsen worker 63 - jepsen.util 2043 :ok :read [21 33] INFO [2019-12-17 12:29:36,350] jepsen worker 28 - jepsen.util 1828 :ok :read [9 42] INFO [2019-12-17 12:29:36,350] jepsen worker 63 - jepsen.util 2043 :invoke :read [21 nil] INFO [2019-12-17 12:29:36,350] jepsen worker 28 - jepsen.util 1828 :invoke :write [9 38] INFO [2019-12-17 12:29:36,354] jepsen worker 85 - jepsen.util 3415 :info :cas [28 [22 26]] :RequestCanceledException INFO [2019-12-17 12:29:36,355] jepsen worker 85 - jepsen.util 3505 :invoke :cas [28 [23 36]] INFO [2019-12-17 12:29:36,374] jepsen worker 65 - jepsen.util 2135 :ok :read [21 33] INFO [2019-12-17 12:29:36,375] jepsen worker 65 - jepsen.util 2135 :invoke :write [21 17] INFO [2019-12-17 12:29:36,389] jepsen worker 84 - jepsen.util 3054 :info :write [28 24] :RequestCanceledException INFO [2019-12-17 12:29:36,389] jepsen worker 79 - jepsen.util 2419 :fail :cas [26 [36 12]] :ValueNotSwapFrom INFO [2019-12-17 12:29:36,389] jepsen worker 79 - jepsen.util 2419 :invoke :cas [26 [3 8]] INFO [2019-12-17 12:29:36,390] jepsen worker 84 - jepsen.util 3144 :invoke :write [28 9] INFO [2019-12-17 12:29:36,408] jepsen worker 62 - jepsen.util 2942 :info :cas [20 [39 17]] :RequestCanceledException INFO [2019-12-17 12:29:36,408] jepsen worker 24 - jepsen.util 2994 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:29:36,408] jepsen worker 24 - jepsen.util 2994 :invoke :cas [8 [7 30]] INFO [2019-12-17 12:29:36,409] jepsen worker 62 - jepsen.util 3032 :invoke :cas [20 [37 8]] INFO [2019-12-17 12:29:36,420] jepsen worker 63 - jepsen.util 2043 :ok :read [21 33] INFO [2019-12-17 12:29:36,420] jepsen worker 63 - jepsen.util 2043 :invoke :cas [21 [48 2]] INFO [2019-12-17 12:29:36,422] jepsen worker 29 - jepsen.util 2099 :ok :write [9 24] INFO [2019-12-17 12:29:36,422] jepsen worker 29 - jepsen.util 2099 :invoke :cas [9 [24 9]] INFO [2019-12-17 12:29:36,423] jepsen worker 86 - jepsen.util 3056 :info :write [28 13] :RequestCanceledException INFO [2019-12-17 12:29:36,423] jepsen worker 81 - jepsen.util 2871 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:29:36,423] jepsen worker 83 - jepsen.util 3593 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:29:36,423] jepsen worker 83 - jepsen.util 3593 :invoke :cas [27 [36 36]] INFO [2019-12-17 12:29:36,423] jepsen worker 81 - jepsen.util 2871 :invoke :cas [27 [32 44]] INFO [2019-12-17 12:29:36,424] jepsen worker 86 - jepsen.util 3146 :invoke :cas [28 [19 37]] INFO [2019-12-17 12:29:36,434] jepsen worker 38 - jepsen.util 2378 :ok :write [12 6] INFO [2019-12-17 12:29:36,434] jepsen worker 38 - jepsen.util 2378 :invoke :read [12 nil] INFO [2019-12-17 12:29:36,436] jepsen worker 58 - jepsen.util 2848 :info :cas [19 [18 47]] :RequestCanceledException INFO [2019-12-17 12:29:36,436] jepsen worker 75 - jepsen.util 3405 :info :write [25 16] :RequestCanceledException INFO [2019-12-17 12:29:36,437] jepsen worker 75 - jepsen.util 3495 :invoke :read [25 nil] INFO [2019-12-17 12:29:36,437] jepsen worker 58 - jepsen.util 2938 :invoke :cas [19 [46 47]] INFO [2019-12-17 12:29:36,453] jepsen worker 22 - jepsen.util 3172 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:29:36,453] jepsen worker 76 - jepsen.util 3136 :info :write [25 0] :RequestCanceledException INFO [2019-12-17 12:29:36,453] jepsen worker 22 - jepsen.util 3172 :invoke :read [7 nil] INFO [2019-12-17 12:29:36,454] jepsen worker 76 - jepsen.util 3226 :invoke :cas [25 [43 28]] INFO [2019-12-17 12:29:36,466] jepsen worker 13 - jepsen.util 2263 :ok :read [4 6] INFO [2019-12-17 12:29:36,466] jepsen worker 13 - jepsen.util 2263 :invoke :cas [4 [25 16]] INFO [2019-12-17 12:29:36,467] jepsen worker 63 - jepsen.util 2043 :fail :cas [21 [48 2]] :ValueNotSwapFrom INFO [2019-12-17 12:29:36,467] jepsen worker 63 - jepsen.util 2043 :invoke :cas [21 [8 23]] INFO [2019-12-17 12:29:36,479] jepsen worker 56 - jepsen.util 2306 :fail :cas [18 [24 27]] :ValueNotSwapFrom INFO [2019-12-17 12:29:36,480] jepsen worker 56 - jepsen.util 2306 :invoke :read [18 nil] INFO [2019-12-17 12:29:36,481] jepsen worker 63 - jepsen.util 2043 :fail :cas [21 [8 23]] :ValueNotSwapFrom INFO [2019-12-17 12:29:36,481] jepsen worker 63 - jepsen.util 2043 :invoke :read [21 nil] INFO [2019-12-17 12:29:36,485] jepsen worker 28 - jepsen.util 1828 :ok :write [9 38] INFO [2019-12-17 12:29:36,485] jepsen worker 28 - jepsen.util 1828 :invoke :write [9 8] INFO [2019-12-17 12:29:36,490] jepsen worker 41 - jepsen.util 3371 :info :write [13 11] :RequestCanceledException INFO [2019-12-17 12:29:36,490] jepsen worker 25 - jepsen.util 3535 :info :write [8 31] :RequestCanceledException INFO [2019-12-17 12:29:36,491] jepsen worker 41 - jepsen.util 3461 :invoke :write [13 1] INFO [2019-12-17 12:29:36,491] jepsen worker 25 - jepsen.util 3625 :invoke :write [8 46] INFO [2019-12-17 12:29:36,496] jepsen worker 12 - jepsen.util 2532 :ok :read [4 6] INFO [2019-12-17 12:29:36,496] jepsen worker 12 - jepsen.util 2532 :invoke :write [4 31] INFO [2019-12-17 12:29:36,502] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13910,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:29:36,502] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13910,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:29:36,502] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13914,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:29:36,502] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13914,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:29:36,502] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13915,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:29:36,502] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13915,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:29:36,510] jepsen worker 19 - jepsen.util 2629 :ok :write [6 19] INFO [2019-12-17 12:29:36,511] jepsen worker 46 - jepsen.util 2656 :ok :write [15 35] INFO [2019-12-17 12:29:36,511] jepsen worker 19 - jepsen.util 2629 :invoke :write [6 0] INFO [2019-12-17 12:29:36,511] jepsen worker 46 - jepsen.util 2656 :invoke :read [15 nil] INFO [2019-12-17 12:29:36,512] jepsen worker 63 - jepsen.util 2043 :ok :read [21 33] INFO [2019-12-17 12:29:36,512] jepsen worker 63 - jepsen.util 2043 :invoke :read [21 nil] INFO [2019-12-17 12:29:36,522] jepsen worker 53 - jepsen.util 2303 :info :write [17 42] :RequestCanceledException INFO [2019-12-17 12:29:36,523] jepsen worker 42 - jepsen.util 3732 :info :write [14 1] :RequestCanceledException INFO [2019-12-17 12:29:36,523] jepsen worker 40 - jepsen.util 3460 :info :write [13 8] :RequestCanceledException INFO [2019-12-17 12:29:36,523] jepsen worker 53 - jepsen.util 2393 :invoke :cas [17 [18 25]] INFO [2019-12-17 12:29:36,524] jepsen worker 42 - jepsen.util 3822 :invoke :cas [14 [28 31]] INFO [2019-12-17 12:29:36,524] jepsen worker 40 - jepsen.util 3550 :invoke :read [13 nil] INFO [2019-12-17 12:29:36,529] jepsen worker 30 - jepsen.util 2910 :ok :write [10 11] INFO [2019-12-17 12:29:36,529] jepsen worker 30 - jepsen.util 2910 :invoke :write [10 28] INFO [2019-12-17 12:29:36,532] jepsen worker 46 - jepsen.util 2656 :ok :read [15 35] INFO [2019-12-17 12:29:36,532] jepsen worker 46 - jepsen.util 2656 :invoke :cas [15 [38 14]] INFO [2019-12-17 12:29:36,537] jepsen worker 26 - jepsen.util 3446 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:29:36,537] jepsen worker 26 - jepsen.util 3446 :invoke :cas [8 [48 7]] INFO [2019-12-17 12:29:36,554] jepsen worker 48 - jepsen.util 2568 :info :cas [16 [38 11]] :RequestCanceledException INFO [2019-12-17 12:29:36,555] jepsen worker 48 - jepsen.util 2658 :invoke :read [16 nil] INFO [2019-12-17 12:29:36,584] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:29:36,584] jepsen nemesis - couchbase.util Rebalance check count: 100 INFO [2019-12-17 12:29:36,585] jepsen nemesis - couchbase.util Rebalance stuck count: 81 INFO [2019-12-17 12:29:36,589] jepsen worker 82 - jepsen.util 3772 :info :write [27 26] :RequestCanceledException INFO [2019-12-17 12:29:36,590] jepsen worker 82 - jepsen.util 3862 :invoke :read [27 nil] INFO [2019-12-17 12:29:36,618] jepsen worker 33 - jepsen.util 3363 :info :write [11 26] :RequestCanceledException INFO [2019-12-17 12:29:36,618] jepsen worker 52 - jepsen.util 2482 :info :write [17 8] :RequestCanceledException INFO [2019-12-17 12:29:36,619] jepsen worker 33 - jepsen.util 3453 :invoke :write [11 42] INFO [2019-12-17 12:29:36,619] jepsen worker 52 - jepsen.util 2572 :invoke :read [17 nil] INFO [2019-12-17 12:29:36,623] jepsen worker 89 - jepsen.util 2339 :fail :cas [29 [34 4]] :ValueNotSwapFrom INFO [2019-12-17 12:29:36,623] jepsen worker 89 - jepsen.util 2339 :invoke :read [29 nil] INFO [2019-12-17 12:29:36,627] jepsen worker 46 - jepsen.util 2656 :fail :cas [15 [38 14]] :ValueNotSwapFrom INFO [2019-12-17 12:29:36,628] jepsen worker 46 - jepsen.util 2656 :invoke :read [15 nil] INFO [2019-12-17 12:29:36,628] jepsen worker 63 - jepsen.util 2043 :ok :read [21 33] INFO [2019-12-17 12:29:36,629] jepsen worker 63 - jepsen.util 2043 :invoke :write [21 45] INFO [2019-12-17 12:29:36,629] jepsen worker 69 - jepsen.util 2319 :ok :read [23 48] INFO [2019-12-17 12:29:36,629] jepsen worker 69 - jepsen.util 2319 :invoke :cas [23 [27 13]] INFO [2019-12-17 12:29:36,637] jepsen worker 17 - jepsen.util 3167 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:29:36,637] jepsen worker 17 - jepsen.util 3167 :invoke :read [5 nil] INFO [2019-12-17 12:29:36,651] jepsen worker 46 - jepsen.util 2656 :ok :read [15 35] INFO [2019-12-17 12:29:36,651] jepsen worker 46 - jepsen.util 2656 :invoke :read [15 nil] INFO [2019-12-17 12:29:36,652] jepsen worker 77 - jepsen.util 3047 :info :cas [25 [47 48]] :RequestCanceledException INFO [2019-12-17 12:29:36,653] jepsen worker 77 - jepsen.util 3137 :invoke :write [25 11] INFO [2019-12-17 12:29:36,689] jepsen worker 46 - jepsen.util 2656 :ok :read [15 35] INFO [2019-12-17 12:29:36,689] jepsen worker 46 - jepsen.util 2656 :invoke :write [15 39] INFO [2019-12-17 12:29:36,690] jepsen worker 8 - jepsen.util 2888 :info :write [2 15] :RequestCanceledException INFO [2019-12-17 12:29:36,691] jepsen worker 8 - jepsen.util 2978 :invoke :read [2 nil] INFO [2019-12-17 12:29:36,707] jepsen worker 66 - jepsen.util 3216 :info :write [22 41] :RequestCanceledException INFO [2019-12-17 12:29:36,708] jepsen worker 66 - jepsen.util 3306 :invoke :read [22 nil] INFO [2019-12-17 12:29:36,709] jepsen worker 19 - jepsen.util 2629 :ok :write [6 0] INFO [2019-12-17 12:29:36,709] jepsen worker 19 - jepsen.util 2629 :invoke :cas [6 [9 11]] INFO [2019-12-17 12:29:36,714] jepsen worker 65 - jepsen.util 2135 :ok :write [21 17] INFO [2019-12-17 12:29:36,714] jepsen worker 65 - jepsen.util 2135 :invoke :write [21 1] INFO [2019-12-17 12:29:36,717] jepsen worker 28 - jepsen.util 1828 :ok :write [9 8] INFO [2019-12-17 12:29:36,717] jepsen worker 28 - jepsen.util 1828 :invoke :write [9 45] INFO [2019-12-17 12:29:36,723] jepsen worker 68 - jepsen.util 2858 :info :cas [22 [15 2]] :RequestCanceledException INFO [2019-12-17 12:29:36,724] jepsen worker 68 - jepsen.util 2948 :invoke :read [22 nil] INFO [2019-12-17 12:29:36,725] jepsen worker 19 - jepsen.util 2629 :fail :cas [6 [9 11]] :ValueNotSwapFrom INFO [2019-12-17 12:29:36,725] jepsen worker 19 - jepsen.util 2629 :invoke :write [6 26] INFO [2019-12-17 12:29:36,727] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13948,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:29:36,727] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13948,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:29:36,727] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13951,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:29:36,727] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13951,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:29:36,727] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13952,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:29:36,727] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13952,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:29:36,731] jepsen worker 38 - jepsen.util 2378 :ok :read [12 6] INFO [2019-12-17 12:29:36,731] jepsen worker 38 - jepsen.util 2378 :invoke :write [12 17] INFO [2019-12-17 12:29:36,789] jepsen worker 28 - jepsen.util 1828 :ok :write [9 45] INFO [2019-12-17 12:29:36,789] jepsen worker 28 - jepsen.util 1828 :invoke :read [9 nil] INFO [2019-12-17 12:29:36,789] jepsen worker 39 - jepsen.util 3279 :info :write [13 40] :RequestCanceledException INFO [2019-12-17 12:29:36,790] jepsen worker 39 - jepsen.util 3369 :invoke :read [13 nil] INFO [2019-12-17 12:29:36,807] jepsen worker 20 - jepsen.util 2090 :fail :read [6 nil] :RequestTimeoutException INFO [2019-12-17 12:29:36,807] jepsen worker 20 - jepsen.util 2090 :invoke :cas [6 [34 41]] INFO [2019-12-17 12:29:36,812] jepsen worker 28 - jepsen.util 1828 :ok :read [9 45] INFO [2019-12-17 12:29:36,812] jepsen worker 28 - jepsen.util 1828 :invoke :read [9 nil] INFO [2019-12-17 12:29:36,823] jepsen worker 27 - jepsen.util 2097 :fail :read [9 nil] :RequestTimeoutException INFO [2019-12-17 12:29:36,823] jepsen worker 27 - jepsen.util 2097 :invoke :read [9 nil] INFO [2019-12-17 12:29:36,840] jepsen worker 28 - jepsen.util 1828 :ok :read [9 45] INFO [2019-12-17 12:29:36,840] jepsen worker 28 - jepsen.util 1828 :invoke :read [9 nil] INFO [2019-12-17 12:29:36,875] jepsen worker 56 - jepsen.util 2306 :ok :read [18 15] INFO [2019-12-17 12:29:36,875] jepsen worker 28 - jepsen.util 1828 :ok :read [9 45] INFO [2019-12-17 12:29:36,875] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13973,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:29:36,875] jepsen worker 56 - jepsen.util 2306 :invoke :write [18 27] INFO [2019-12-17 12:29:36,875] jepsen worker 28 - jepsen.util 1828 :invoke :write [9 16] INFO [2019-12-17 12:29:36,875] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13973,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:29:36,875] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13974,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:29:36,876] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13974,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:29:36,888] jepsen worker 2 - jepsen.util 3062 :info :cas [0 [33 37]] :RequestCanceledException INFO [2019-12-17 12:29:36,889] jepsen worker 2 - jepsen.util 3152 :invoke :write [0 30] INFO [2019-12-17 12:29:36,895] jepsen worker 53 - jepsen.util 2393 :fail :cas [17 [18 25]] :ValueNotSwapFrom INFO [2019-12-17 12:29:36,895] jepsen worker 53 - jepsen.util 2393 :invoke :write [17 37] INFO [2019-12-17 12:29:36,899] jepsen worker 65 - jepsen.util 2135 :ok :write [21 1] INFO [2019-12-17 12:29:36,899] jepsen worker 65 - jepsen.util 2135 :invoke :read [21 nil] INFO [2019-12-17 12:29:36,907] jepsen worker 87 - jepsen.util 2157 :info :write [29 26] :RequestCanceledException INFO [2019-12-17 12:29:36,908] jepsen worker 87 - jepsen.util 2247 :invoke :read [29 nil] INFO [2019-12-17 12:29:36,915] jepsen worker 65 - jepsen.util 2135 :ok :read [21 1] INFO [2019-12-17 12:29:36,915] jepsen worker 65 - jepsen.util 2135 :invoke :write [21 13] INFO [2019-12-17 12:29:36,923] jepsen worker 61 - jepsen.util 3391 :info :cas [20 [8 25]] :RequestCanceledException INFO [2019-12-17 12:29:36,924] jepsen worker 61 - jepsen.util 3481 :invoke :cas [20 [27 42]] INFO [2019-12-17 12:29:36,937] jepsen worker 5 - jepsen.util 1985 :info :write [1 41] :RequestCanceledException INFO [2019-12-17 12:29:36,939] jepsen worker 5 - jepsen.util 2075 :invoke :cas [1 [8 19]] INFO [2019-12-17 12:29:36,953] jepsen worker 28 - jepsen.util 1828 :ok :write [9 16] INFO [2019-12-17 12:29:36,953] jepsen worker 28 - jepsen.util 1828 :invoke :write [9 43] INFO [2019-12-17 12:29:36,982] jepsen worker 52 - jepsen.util 2572 :ok :read [17 5] INFO [2019-12-17 12:29:36,982] jepsen worker 52 - jepsen.util 2572 :invoke :write [17 18] INFO [2019-12-17 12:29:36,986] jepsen worker 46 - jepsen.util 2656 :ok :write [15 39] INFO [2019-12-17 12:29:36,986] jepsen worker 46 - jepsen.util 2656 :invoke :read [15 nil] INFO [2019-12-17 12:29:36,987] jepsen worker 19 - jepsen.util 2629 :ok :write [6 26] INFO [2019-12-17 12:29:36,987] jepsen worker 19 - jepsen.util 2629 :invoke :read [6 nil] INFO [2019-12-17 12:29:37,005] jepsen worker 69 - jepsen.util 2319 :fail :cas [23 [27 13]] :ValueNotSwapFrom INFO [2019-12-17 12:29:37,005] jepsen worker 70 - jepsen.util 2770 :info :write [23 36] :RequestCanceledException INFO [2019-12-17 12:29:37,006] jepsen worker 69 - jepsen.util 2319 :invoke :read [23 nil] INFO [2019-12-17 12:29:37,006] jepsen worker 70 - jepsen.util 2860 :invoke :read [23 nil] INFO [2019-12-17 12:29:37,023] jepsen worker 4 - jepsen.util 1804 :info :write [1 11] :RequestCanceledException INFO [2019-12-17 12:29:37,025] jepsen worker 4 - jepsen.util 1894 :invoke :cas [1 [42 36]] INFO [2019-12-17 12:29:37,032] jepsen worker 46 - jepsen.util 2656 :ok :read [15 39] INFO [2019-12-17 12:29:37,032] jepsen worker 46 - jepsen.util 2656 :invoke :write [15 17] INFO [2019-12-17 12:29:37,046] jepsen worker 27 - jepsen.util 2097 :ok :read [9 16] INFO [2019-12-17 12:29:37,046] jepsen worker 27 - jepsen.util 2097 :invoke :read [9 nil] INFO [2019-12-17 12:29:37,077] jepsen worker 65 - jepsen.util 2135 :ok :write [21 13] INFO [2019-12-17 12:29:37,077] jepsen worker 65 - jepsen.util 2135 :invoke :write [21 30] INFO [2019-12-17 12:29:37,088] jepsen worker 19 - jepsen.util 2629 :ok :read [6 26] INFO [2019-12-17 12:29:37,088] jepsen worker 19 - jepsen.util 2629 :invoke :read [6 nil] INFO [2019-12-17 12:29:37,094] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13989,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:29:37,094] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13989,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:29:37,095] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13991,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:29:37,095] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":13991,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:29:37,096] jepsen worker 27 - jepsen.util 2097 :ok :read [9 43] INFO [2019-12-17 12:29:37,097] jepsen worker 27 - jepsen.util 2097 :invoke :read [9 nil] INFO [2019-12-17 12:29:37,100] jepsen worker 28 - jepsen.util 1828 :ok :write [9 43] INFO [2019-12-17 12:29:37,101] jepsen worker 28 - jepsen.util 1828 :invoke :cas [9 [31 46]] INFO [2019-12-17 12:29:37,114] jepsen worker 87 - jepsen.util 2247 :ok :read [29 22] INFO [2019-12-17 12:29:37,115] jepsen worker 87 - jepsen.util 2247 :invoke :read [29 nil] INFO [2019-12-17 12:29:37,137] jepsen worker 10 - jepsen.util 2350 :fail :read [3 nil] :RequestTimeoutException INFO [2019-12-17 12:29:37,137] jepsen worker 10 - jepsen.util 2350 :invoke :read [3 nil] INFO [2019-12-17 12:29:37,153] jepsen worker 36 - jepsen.util 2286 :info :write [12 35] :RequestCanceledException INFO [2019-12-17 12:29:37,154] jepsen worker 36 - jepsen.util 2376 :invoke :read [12 nil] INFO [2019-12-17 12:29:37,176] jepsen worker 89 - jepsen.util 2339 :ok :read [29 22] INFO [2019-12-17 12:29:37,177] jepsen worker 89 - jepsen.util 2339 :invoke :cas [29 [33 39]] INFO [2019-12-17 12:29:37,186] jepsen worker 73 - jepsen.util 2413 :fail :read [24 nil] :RequestTimeoutException INFO [2019-12-17 12:29:37,186] jepsen worker 73 - jepsen.util 2413 :invoke :cas [24 [47 19]] INFO [2019-12-17 12:29:37,190] jepsen worker 28 - jepsen.util 1828 :fail :cas [9 [31 46]] :ValueNotSwapFrom INFO [2019-12-17 12:29:37,190] jepsen worker 28 - jepsen.util 1828 :invoke :read [9 nil] INFO [2019-12-17 12:29:37,194] jepsen worker 19 - jepsen.util 2629 :ok :read [6 26] INFO [2019-12-17 12:29:37,195] jepsen worker 19 - jepsen.util 2629 :invoke :cas [6 [47 12]] INFO [2019-12-17 12:29:37,205] jepsen worker 88 - jepsen.util 2338 :info :write [29 27] :RequestCanceledException INFO [2019-12-17 12:29:37,207] jepsen worker 88 - jepsen.util 2428 :invoke :write [29 4] INFO [2019-12-17 12:29:37,237] jepsen worker 32 - jepsen.util 2912 :fail :read [10 nil] :RequestTimeoutException INFO [2019-12-17 12:29:37,237] jepsen worker 32 - jepsen.util 2912 :invoke :write [10 25] INFO [2019-12-17 12:29:37,253] jepsen worker 47 - jepsen.util 2747 :info :write [15 21] :RequestCanceledException INFO [2019-12-17 12:29:37,255] jepsen worker 47 - jepsen.util 2837 :invoke :cas [15 [41 33]] INFO [2019-12-17 12:29:37,261] jepsen worker 79 - jepsen.util 2419 :fail :cas [26 [3 8]] :ValueNotSwapFrom INFO [2019-12-17 12:29:37,262] jepsen worker 79 - jepsen.util 2419 :invoke :write [26 7] INFO [2019-12-17 12:29:37,264] jepsen worker 28 - jepsen.util 1828 :ok :read [9 43] INFO [2019-12-17 12:29:37,264] jepsen worker 28 - jepsen.util 1828 :invoke :cas [9 [2 46]] INFO [2019-12-17 12:29:37,265] jepsen worker 73 - jepsen.util 2413 :fail :cas [24 [47 19]] :ValueNotSwapFrom INFO [2019-12-17 12:29:37,265] jepsen worker 73 - jepsen.util 2413 :invoke :write [24 2] INFO [2019-12-17 12:29:37,278] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":14011,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:29:37,278] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":14011,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:29:37,278] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":14012,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:29:37,278] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":14012,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:29:37,291] jepsen worker 87 - jepsen.util 2247 :ok :read [29 22] INFO [2019-12-17 12:29:37,292] jepsen worker 87 - jepsen.util 2247 :invoke :cas [29 [33 20]] INFO [2019-12-17 12:29:37,308] jepsen worker 31 - jepsen.util 2281 :info :write [10 23] :RequestCanceledException INFO [2019-12-17 12:29:37,308] jepsen worker 10 - jepsen.util 2350 :ok :read [3 4] INFO [2019-12-17 12:29:37,308] jepsen worker 10 - jepsen.util 2350 :invoke :read [3 nil] INFO [2019-12-17 12:29:37,309] jepsen worker 31 - jepsen.util 2371 :invoke :cas [10 [27 40]] INFO [2019-12-17 12:29:37,312] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":14018,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:29:37,312] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":14018,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:29:37,312] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":14021,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:29:37,312] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":14021,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:29:37,323] jepsen worker 18 - jepsen.util 2628 :info :write [6 45] :RequestCanceledException INFO [2019-12-17 12:29:37,324] jepsen worker 18 - jepsen.util 2718 :invoke :read [6 nil] INFO [2019-12-17 12:29:37,327] jepsen worker 5 - jepsen.util 2075 :fail :cas [1 [8 19]] :ValueNotSwapFrom INFO [2019-12-17 12:29:37,327] jepsen worker 5 - jepsen.util 2075 :invoke :cas [1 [38 36]] INFO [2019-12-17 12:29:37,336] jepsen worker 55 - jepsen.util 2755 :info :write [18 16] :RequestCanceledException INFO [2019-12-17 12:29:37,337] jepsen worker 55 - jepsen.util 2845 :invoke :read [18 nil] INFO [2019-12-17 12:29:37,343] jepsen worker 27 - jepsen.util 2097 :ok :read [9 43] INFO [2019-12-17 12:29:37,344] jepsen worker 27 - jepsen.util 2097 :invoke :write [9 48] INFO [2019-12-17 12:29:37,361] jepsen worker 13 - jepsen.util 2263 :fail :cas [4 [25 16]] :ValueNotSwapFrom INFO [2019-12-17 12:29:37,361] jepsen worker 13 - jepsen.util 2263 :invoke :read [4 nil] INFO [2019-12-17 12:29:37,392] jepsen worker 78 - jepsen.util 2598 :ok :cas [26 [16 8]] INFO [2019-12-17 12:29:37,393] jepsen worker 78 - jepsen.util 2598 :invoke :cas [26 [21 22]] INFO [2019-12-17 12:29:37,397] jepsen worker 28 - jepsen.util 1828 :fail :cas [9 [2 46]] :ValueNotSwapFrom INFO [2019-12-17 12:29:37,397] jepsen worker 28 - jepsen.util 1828 :invoke :cas [9 [42 6]] INFO [2019-12-17 12:29:37,421] jepsen worker 69 - jepsen.util 2319 :ok :read [23 48] INFO [2019-12-17 12:29:37,421] jepsen worker 69 - jepsen.util 2319 :invoke :cas [23 [12 17]] INFO [2019-12-17 12:29:37,424] jepsen worker 80 - jepsen.util 2330 :info :write [26 35] :RequestCanceledException INFO [2019-12-17 12:29:37,424] jepsen worker 67 - jepsen.util 3667 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:29:37,424] jepsen worker 67 - jepsen.util 3667 :invoke :read [22 nil] INFO [2019-12-17 12:29:37,425] jepsen worker 80 - jepsen.util 2420 :invoke :write [26 4] INFO [2019-12-17 12:29:37,449] jepsen worker 19 - jepsen.util 2629 :fail :cas [6 [47 12]] :ValueNotSwapFrom INFO [2019-12-17 12:29:37,449] jepsen worker 19 - jepsen.util 2629 :invoke :write [6 16] INFO [2019-12-17 12:29:37,453] jepsen worker 6 - jepsen.util 3336 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:29:37,454] jepsen worker 6 - jepsen.util 3336 :invoke :cas [2 [18 8]] INFO [2019-12-17 12:29:37,454] jepsen worker 31 - jepsen.util 2371 :fail :cas [10 [27 40]] :ValueNotSwapFrom INFO [2019-12-17 12:29:37,454] jepsen worker 31 - jepsen.util 2371 :invoke :read [10 nil] INFO [2019-12-17 12:29:37,462] jepsen worker 69 - jepsen.util 2319 :fail :cas [23 [12 17]] :ValueNotSwapFrom INFO [2019-12-17 12:29:37,462] jepsen worker 69 - jepsen.util 2319 :invoke :read [23 nil] INFO [2019-12-17 12:29:37,469] jepsen worker 28 - jepsen.util 1828 :fail :cas [9 [42 6]] :ValueNotSwapFrom INFO [2019-12-17 12:29:37,470] jepsen worker 28 - jepsen.util 1828 :invoke :cas [9 [33 22]] INFO [2019-12-17 12:29:37,482] jepsen worker 36 - jepsen.util 2376 :ok :read [12 6] INFO [2019-12-17 12:29:37,482] jepsen worker 36 - jepsen.util 2376 :invoke :read [12 nil] INFO [2019-12-17 12:29:37,485] jepsen worker 70 - jepsen.util 2860 :ok :read [23 48] INFO [2019-12-17 12:29:37,486] jepsen worker 70 - jepsen.util 2860 :invoke :cas [23 [6 41]] INFO [2019-12-17 12:29:37,489] jepsen worker 44 - jepsen.util 2924 :info :cas [14 [17 48]] :RequestCanceledException INFO [2019-12-17 12:29:37,490] jepsen worker 44 - jepsen.util 3014 :invoke :cas [14 [34 40]] INFO [2019-12-17 12:29:37,491] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":14042,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:29:37,491] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":14042,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:29:37,491] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":14043,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:29:37,491] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":14043,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:29:37,507] jepsen worker 60 - jepsen.util 3390 :info :write [20 15] :RequestCanceledException INFO [2019-12-17 12:29:37,509] jepsen worker 60 - jepsen.util 3480 :invoke :cas [20 [19 40]] INFO [2019-12-17 12:29:37,515] jepsen worker 31 - jepsen.util 2371 :ok :read [10 11] INFO [2019-12-17 12:29:37,515] jepsen worker 31 - jepsen.util 2371 :invoke :cas [10 [37 36]] INFO [2019-12-17 12:29:37,520] jepsen worker 49 - jepsen.util 2389 :info :write [16 43] :RequestCanceledException INFO [2019-12-17 12:29:37,521] jepsen worker 87 - jepsen.util 2247 :fail :cas [29 [33 20]] :ValueNotSwapFrom INFO [2019-12-17 12:29:37,521] jepsen worker 87 - jepsen.util 2247 :invoke :write [29 42] INFO [2019-12-17 12:29:37,521] jepsen worker 49 - jepsen.util 2479 :invoke :cas [16 [4 46]] INFO [2019-12-17 12:29:37,536] jepsen worker 23 - jepsen.util 2903 :info :write [7 10] :RequestCanceledException INFO [2019-12-17 12:29:37,538] jepsen worker 23 - jepsen.util 2993 :invoke :cas [7 [24 12]] INFO [2019-12-17 12:29:37,539] jepsen worker 28 - jepsen.util 1828 :fail :cas [9 [33 22]] :ValueNotSwapFrom INFO [2019-12-17 12:29:37,539] jepsen worker 28 - jepsen.util 1828 :invoke :write [9 5] INFO [2019-12-17 12:29:37,547] jepsen worker 65 - jepsen.util 2135 :ok :write [21 30] INFO [2019-12-17 12:29:37,547] jepsen worker 65 - jepsen.util 2135 :invoke :read [21 nil] INFO [2019-12-17 12:29:37,550] jepsen worker 7 - jepsen.util 2977 :info :cas [2 [35 4]] :RequestCanceledException INFO [2019-12-17 12:29:37,550] jepsen worker 34 - jepsen.util 3184 :info :write [11 9] :RequestCanceledException INFO [2019-12-17 12:29:37,552] jepsen worker 34 - jepsen.util 3274 :invoke :write [11 39] INFO [2019-12-17 12:29:37,552] jepsen worker 7 - jepsen.util 3067 :invoke :write [2 11] INFO [2019-12-17 12:29:37,582] jepsen worker 27 - jepsen.util 2097 :ok :write [9 48] INFO [2019-12-17 12:29:37,582] jepsen worker 27 - jepsen.util 2097 :invoke :write [9 48] INFO [2019-12-17 12:29:37,589] jepsen worker 47 - jepsen.util 2837 :fail :cas [15 [41 33]] :ValueNotSwapFrom INFO [2019-12-17 12:29:37,590] jepsen worker 47 - jepsen.util 2837 :invoke :cas [15 [37 42]] INFO [2019-12-17 12:29:37,592] jepsen worker 31 - jepsen.util 2371 :fail :cas [10 [37 36]] :ValueNotSwapFrom INFO [2019-12-17 12:29:37,592] jepsen worker 31 - jepsen.util 2371 :invoke :cas [10 [18 36]] INFO [2019-12-17 12:29:37,607] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:29:37,607] jepsen nemesis - couchbase.util Rebalance check count: 101 INFO [2019-12-17 12:29:37,607] jepsen nemesis - couchbase.util Rebalance stuck count: 82 INFO [2019-12-17 12:29:37,629] jepsen worker 89 - jepsen.util 2339 :fail :cas [29 [33 39]] :ValueNotSwapFrom INFO [2019-12-17 12:29:37,629] jepsen worker 89 - jepsen.util 2339 :invoke :write [29 22] INFO [2019-12-17 12:29:37,637] jepsen worker 3 - jepsen.util 2253 :info :write [1 11] :RequestCanceledException INFO [2019-12-17 12:29:37,638] jepsen worker 3 - jepsen.util 2343 :invoke :write [1 38] INFO [2019-12-17 12:29:37,640] jepsen worker 65 - jepsen.util 2135 :ok :read [21 30] INFO [2019-12-17 12:29:37,640] jepsen worker 65 - jepsen.util 2135 :invoke :cas [21 [23 3]] INFO [2019-12-17 12:29:37,654] jepsen worker 47 - jepsen.util 2837 :fail :cas [15 [37 42]] :ValueNotSwapFrom INFO [2019-12-17 12:29:37,654] jepsen worker 47 - jepsen.util 2837 :invoke :write [15 3] INFO [2019-12-17 12:29:37,654] jepsen worker 54 - jepsen.util 2394 :info :write [18 12] :RequestCanceledException INFO [2019-12-17 12:29:37,654] jepsen worker 45 - jepsen.util 2565 :info :write [15 0] :RequestCanceledException INFO [2019-12-17 12:29:37,654] jepsen worker 9 - jepsen.util 2799 :info :write [3 1] :RequestCanceledException INFO [2019-12-17 12:29:37,655] jepsen worker 54 - jepsen.util 2484 :invoke :read [18 nil] INFO [2019-12-17 12:29:37,655] jepsen worker 45 - jepsen.util 2655 :invoke :write [15 13] INFO [2019-12-17 12:29:37,655] jepsen worker 9 - jepsen.util 2889 :invoke :write [3 26] INFO [2019-12-17 12:29:37,658] jepsen worker 65 - jepsen.util 2135 :fail :cas [21 [23 3]] :ValueNotSwapFrom INFO [2019-12-17 12:29:37,658] jepsen worker 65 - jepsen.util 2135 :invoke :write [21 41] INFO [2019-12-17 12:29:37,707] jepsen worker 28 - jepsen.util 1828 :ok :write [9 5] INFO [2019-12-17 12:29:37,707] jepsen worker 28 - jepsen.util 1828 :invoke :cas [9 [12 36]] INFO [2019-12-17 12:29:37,710] jepsen worker 46 - jepsen.util 2656 :ok :write [15 17] INFO [2019-12-17 12:29:37,710] jepsen worker 46 - jepsen.util 2656 :invoke :read [15 nil] INFO [2019-12-17 12:29:37,712] jepsen worker 73 - jepsen.util 2413 :ok :write [24 2] INFO [2019-12-17 12:29:37,712] jepsen worker 73 - jepsen.util 2413 :invoke :cas [24 [13 0]] INFO [2019-12-17 12:29:37,713] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":14073,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:29:37,713] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":14073,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:29:37,723] jepsen worker 78 - jepsen.util 2598 :fail :cas [26 [21 22]] :ValueNotSwapFrom INFO [2019-12-17 12:29:37,723] jepsen worker 78 - jepsen.util 2598 :invoke :cas [26 [1 25]] INFO [2019-12-17 12:29:37,749] jepsen worker 70 - jepsen.util 2860 :fail :cas [23 [6 41]] :ValueNotSwapFrom INFO [2019-12-17 12:29:37,750] jepsen worker 70 - jepsen.util 2860 :invoke :read [23 nil] INFO [2019-12-17 12:29:37,759] jepsen worker 73 - jepsen.util 2413 :fail :cas [24 [13 0]] :ValueNotSwapFrom INFO [2019-12-17 12:29:37,759] jepsen worker 73 - jepsen.util 2413 :invoke :write [24 15] INFO [2019-12-17 12:29:37,764] jepsen worker 46 - jepsen.util 2656 :ok :read [15 17] INFO [2019-12-17 12:29:37,765] jepsen worker 46 - jepsen.util 2656 :invoke :cas [15 [49 27]] INFO [2019-12-17 12:29:37,776] jepsen worker 32 - jepsen.util 2912 :ok :write [10 25] INFO [2019-12-17 12:29:37,776] jepsen worker 32 - jepsen.util 2912 :invoke :write [10 16] INFO [2019-12-17 12:29:37,777] jepsen worker 19 - jepsen.util 2629 :ok :write [6 16] INFO [2019-12-17 12:29:37,777] jepsen worker 19 - jepsen.util 2629 :invoke :write [6 6] INFO [2019-12-17 12:29:37,782] jepsen worker 46 - jepsen.util 2656 :fail :cas [15 [49 27]] :ValueNotSwapFrom INFO [2019-12-17 12:29:37,782] jepsen worker 46 - jepsen.util 2656 :invoke :write [15 30] INFO [2019-12-17 12:29:37,790] jepsen worker 70 - jepsen.util 2860 :ok :read [23 48] INFO [2019-12-17 12:29:37,790] jepsen worker 70 - jepsen.util 2860 :invoke :cas [23 [11 10]] INFO [2019-12-17 12:29:37,854] jepsen worker 71 - jepsen.util 2321 :info :write [23 44] :RequestCanceledException INFO [2019-12-17 12:29:37,855] jepsen worker 71 - jepsen.util 2411 :invoke :cas [23 [37 38]] INFO [2019-12-17 12:29:37,860] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":14096,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:29:37,860] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":14096,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:29:37,860] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":14100,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:29:37,860] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":14100,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:29:37,869] jepsen worker 80 - jepsen.util 2420 :ok :write [26 4] INFO [2019-12-17 12:29:37,869] jepsen worker 80 - jepsen.util 2420 :invoke :cas [26 [3 42]] INFO [2019-12-17 12:29:37,883] jepsen worker 70 - jepsen.util 2860 :fail :cas [23 [11 10]] :ValueNotSwapFrom INFO [2019-12-17 12:29:37,883] jepsen worker 70 - jepsen.util 2860 :invoke :read [23 nil] INFO [2019-12-17 12:29:37,890] jepsen worker 37 - jepsen.util 2107 :info :write [12 2] :RequestCanceledException INFO [2019-12-17 12:29:37,891] jepsen worker 37 - jepsen.util 2197 :invoke :read [12 nil] INFO [2019-12-17 12:29:37,906] jepsen worker 69 - jepsen.util 2319 :ok :read [23 48] INFO [2019-12-17 12:29:37,906] jepsen worker 69 - jepsen.util 2319 :invoke :cas [23 [39 11]] INFO [2019-12-17 12:29:37,907] jepsen worker 11 - jepsen.util 2891 :info :write [3 9] :RequestCanceledException INFO [2019-12-17 12:29:37,907] jepsen worker 11 - jepsen.util 2981 :invoke :read [3 nil] INFO [2019-12-17 12:29:37,941] jepsen worker 9 - jepsen.util 2889 :ok :write [3 26] INFO [2019-12-17 12:29:37,941] jepsen worker 9 - jepsen.util 2889 :invoke :read [3 nil] INFO [2019-12-17 12:29:37,941] jepsen worker 73 - jepsen.util 2413 :ok :write [24 15] INFO [2019-12-17 12:29:37,941] jepsen worker 45 - jepsen.util 2655 :ok :write [15 13] INFO [2019-12-17 12:29:37,941] jepsen worker 73 - jepsen.util 2413 :invoke :cas [24 [24 30]] INFO [2019-12-17 12:29:37,941] jepsen worker 45 - jepsen.util 2655 :invoke :read [15 nil] INFO [2019-12-17 12:29:37,942] jepsen worker 36 - jepsen.util 2376 :ok :read [12 6] INFO [2019-12-17 12:29:37,942] jepsen worker 36 - jepsen.util 2376 :invoke :write [12 36] INFO [2019-12-17 12:29:37,961] jepsen worker 55 - jepsen.util 2845 :ok :read [18 15] INFO [2019-12-17 12:29:37,961] jepsen worker 55 - jepsen.util 2845 :invoke :write [18 20] INFO [2019-12-17 12:29:37,985] jepsen worker 11 - jepsen.util 2981 :ok :read [3 26] INFO [2019-12-17 12:29:37,985] jepsen worker 11 - jepsen.util 2981 :invoke :read [3 nil] INFO [2019-12-17 12:29:37,989] jepsen worker 72 - jepsen.util 2412 :info :write [24 36] :RequestCanceledException INFO [2019-12-17 12:29:37,990] jepsen worker 72 - jepsen.util 2502 :invoke :read [24 nil] INFO [2019-12-17 12:29:38,004] jepsen worker 45 - jepsen.util 2655 :ok :read [15 13] INFO [2019-12-17 12:29:38,004] jepsen worker 45 - jepsen.util 2655 :invoke :cas [15 [42 9]] INFO [2019-12-17 12:29:38,006] jepsen worker 13 - jepsen.util 2263 :ok :read [4 6] INFO [2019-12-17 12:29:38,006] jepsen worker 13 - jepsen.util 2263 :invoke :read [4 nil] INFO [2019-12-17 12:29:38,008] jepsen worker 9 - jepsen.util 2889 :ok :read [3 26] INFO [2019-12-17 12:29:38,008] jepsen worker 9 - jepsen.util 2889 :invoke :write [3 23] INFO [2019-12-17 12:29:38,018] jepsen worker 73 - jepsen.util 2413 :fail :cas [24 [24 30]] :ValueNotSwapFrom INFO [2019-12-17 12:29:38,018] jepsen worker 73 - jepsen.util 2413 :invoke :read [24 nil] INFO [2019-12-17 12:29:38,023] jepsen worker 51 - jepsen.util 2211 :info :write [17 45] :RequestCanceledException INFO [2019-12-17 12:29:38,024] jepsen worker 71 - jepsen.util 2411 :fail :cas [23 [37 38]] :ValueNotSwapFrom INFO [2019-12-17 12:29:38,024] jepsen worker 71 - jepsen.util 2411 :invoke :write [23 49] INFO [2019-12-17 12:29:38,024] jepsen worker 51 - jepsen.util 2301 :invoke :cas [17 [41 37]] INFO [2019-12-17 12:29:38,039] jepsen worker 19 - jepsen.util 2629 :ok :write [6 6] INFO [2019-12-17 12:29:38,039] jepsen worker 19 - jepsen.util 2629 :invoke :cas [6 [43 39]] INFO [2019-12-17 12:29:38,063] jepsen worker 54 - jepsen.util 2484 :ok :read [18 15] INFO [2019-12-17 12:29:38,063] jepsen worker 54 - jepsen.util 2484 :invoke :write [18 24] INFO [2019-12-17 12:29:38,079] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":14114,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:29:38,079] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":14114,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:29:38,079] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":14115,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:29:38,079] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":14115,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:29:38,080] jepsen worker 72 - jepsen.util 2502 :ok :read [24 15] INFO [2019-12-17 12:29:38,080] jepsen worker 72 - jepsen.util 2502 :invoke :read [24 nil] INFO [2019-12-17 12:29:38,094] jepsen worker 45 - jepsen.util 2655 :fail :cas [15 [42 9]] :ValueNotSwapFrom INFO [2019-12-17 12:29:38,094] jepsen worker 45 - jepsen.util 2655 :invoke :cas [15 [3 13]] INFO [2019-12-17 12:29:38,100] jepsen worker 11 - jepsen.util 2981 :ok :read [3 26] INFO [2019-12-17 12:29:38,100] jepsen worker 11 - jepsen.util 2981 :invoke :write [3 10] INFO [2019-12-17 12:29:38,107] jepsen worker 4 - jepsen.util 1894 :fail :cas [1 [42 36]] :ValueNotSwapFrom INFO [2019-12-17 12:29:38,107] jepsen worker 4 - jepsen.util 1894 :invoke :cas [1 [49 42]] INFO [2019-12-17 12:29:38,108] jepsen worker 74 - jepsen.util 2414 :fail :read [24 nil] :RequestTimeoutException INFO [2019-12-17 12:29:38,108] jepsen worker 74 - jepsen.util 2414 :invoke :cas [24 [13 41]] INFO [2019-12-17 12:29:38,122] jepsen worker 19 - jepsen.util 2629 :fail :cas [6 [43 39]] :ValueNotSwapFrom INFO [2019-12-17 12:29:38,122] jepsen worker 19 - jepsen.util 2629 :invoke :cas [6 [17 35]] INFO [2019-12-17 12:29:38,122] jepsen worker 64 - jepsen.util 2134 :fail :read [21 nil] :RequestTimeoutException INFO [2019-12-17 12:29:38,122] jepsen worker 16 - jepsen.util 3166 :info :cas [5 [27 26]] :RequestCanceledException INFO [2019-12-17 12:29:38,122] jepsen worker 64 - jepsen.util 2134 :invoke :read [21 nil] INFO [2019-12-17 12:29:38,123] jepsen worker 16 - jepsen.util 3256 :invoke :read [5 nil] INFO [2019-12-17 12:29:38,131] jepsen worker 45 - jepsen.util 2655 :fail :cas [15 [3 13]] :ValueNotSwapFrom INFO [2019-12-17 12:29:38,131] jepsen worker 45 - jepsen.util 2655 :invoke :cas [15 [46 41]] INFO [2019-12-17 12:29:38,132] jepsen worker 70 - jepsen.util 2860 :ok :read [23 48] INFO [2019-12-17 12:29:38,133] jepsen worker 70 - jepsen.util 2860 :invoke :write [23 46] INFO [2019-12-17 12:29:38,137] jepsen worker 72 - jepsen.util 2502 :ok :read [24 15] INFO [2019-12-17 12:29:38,137] jepsen worker 72 - jepsen.util 2502 :invoke :write [24 21] INFO [2019-12-17 12:29:38,139] jepsen worker 73 - jepsen.util 2413 :ok :read [24 15] INFO [2019-12-17 12:29:38,139] jepsen worker 73 - jepsen.util 2413 :invoke :cas [24 [19 42]] INFO [2019-12-17 12:29:38,152] jepsen worker 45 - jepsen.util 2655 :fail :cas [15 [46 41]] :ValueNotSwapFrom INFO [2019-12-17 12:29:38,152] jepsen worker 45 - jepsen.util 2655 :invoke :cas [15 [15 10]] INFO [2019-12-17 12:29:38,168] jepsen worker 19 - jepsen.util 2629 :fail :cas [6 [17 35]] :ValueNotSwapFrom INFO [2019-12-17 12:29:38,168] jepsen worker 19 - jepsen.util 2629 :invoke :write [6 49] INFO [2019-12-17 12:29:38,189] jepsen worker 14 - jepsen.util 2264 :info :write [4 1] :RequestCanceledException INFO [2019-12-17 12:29:38,190] jepsen worker 14 - jepsen.util 2354 :invoke :read [4 nil] INFO [2019-12-17 12:29:38,193] jepsen worker 74 - jepsen.util 2414 :fail :cas [24 [13 41]] :ValueNotSwapFrom INFO [2019-12-17 12:29:38,193] jepsen worker 74 - jepsen.util 2414 :invoke :cas [24 [1 49]] INFO [2019-12-17 12:29:38,197] jepsen worker 64 - jepsen.util 2134 :ok :read [21 30] INFO [2019-12-17 12:29:38,197] jepsen worker 64 - jepsen.util 2134 :invoke :read [21 nil] INFO [2019-12-17 12:29:38,201] jepsen worker 78 - jepsen.util 2598 :fail :cas [26 [1 25]] :ValueNotSwapFrom INFO [2019-12-17 12:29:38,201] jepsen worker 78 - jepsen.util 2598 :invoke :read [26 nil] INFO [2019-12-17 12:29:38,208] jepsen worker 45 - jepsen.util 2655 :fail :cas [15 [15 10]] :ValueNotSwapFrom INFO [2019-12-17 12:29:38,208] jepsen worker 45 - jepsen.util 2655 :invoke :write [15 23] INFO [2019-12-17 12:29:38,223] jepsen worker 15 - jepsen.util 2895 :info :write [5 13] :RequestCanceledException INFO [2019-12-17 12:29:38,226] jepsen worker 15 - jepsen.util 2985 :invoke :read [5 nil] INFO [2019-12-17 12:29:38,237] jepsen worker 35 - jepsen.util 2915 :info :cas [11 [14 10]] :RequestCanceledException INFO [2019-12-17 12:29:38,238] jepsen worker 35 - jepsen.util 3005 :invoke :write [11 7] INFO [2019-12-17 12:29:38,258] jepsen worker 78 - jepsen.util 2598 :ok :read [26 4] INFO [2019-12-17 12:29:38,259] jepsen worker 78 - jepsen.util 2598 :invoke :cas [26 [43 49]] INFO [2019-12-17 12:29:38,262] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":14142,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:29:38,262] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":14142,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:29:38,262] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":14144,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:29:38,262] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":14144,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:29:38,299] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":14152,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:29:38,299] jepsen worker 72 - jepsen.util 2502 :ok :write [24 21] INFO [2019-12-17 12:29:38,299] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":14152,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:29:38,299] jepsen worker 72 - jepsen.util 2502 :invoke :read [24 nil] INFO [2019-12-17 12:29:38,299] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":14156,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:29:38,299] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":14156,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:29:38,301] jepsen worker 9 - jepsen.util 2889 :ok :write [3 23] INFO [2019-12-17 12:29:38,302] jepsen worker 9 - jepsen.util 2889 :invoke :read [3 nil] INFO [2019-12-17 12:29:38,321] jepsen worker 50 - jepsen.util 3560 :info :cas [16 [46 14]] :RequestCanceledException INFO [2019-12-17 12:29:38,323] jepsen worker 50 - jepsen.util 3650 :invoke :write [16 49] INFO [2019-12-17 12:29:38,329] jepsen worker 80 - jepsen.util 2420 :fail :cas [26 [3 42]] :ValueNotSwapFrom INFO [2019-12-17 12:29:38,329] jepsen worker 80 - jepsen.util 2420 :invoke :write [26 27] INFO [2019-12-17 12:29:38,332] jepsen worker 74 - jepsen.util 2414 :fail :cas [24 [1 49]] :ValueNotSwapFrom INFO [2019-12-17 12:29:38,332] jepsen worker 74 - jepsen.util 2414 :invoke :cas [24 [49 39]] INFO [2019-12-17 12:29:38,337] jepsen worker 43 - jepsen.util 2923 :info :write [14 20] :RequestCanceledException INFO [2019-12-17 12:29:38,341] jepsen worker 43 - jepsen.util 3013 :invoke :read [14 nil] INFO [2019-12-17 12:29:38,353] jepsen worker 9 - jepsen.util 2889 :ok :read [3 23] INFO [2019-12-17 12:29:38,353] jepsen worker 9 - jepsen.util 2889 :invoke :read [3 nil] INFO [2019-12-17 12:29:38,354] jepsen worker 0 - jepsen.util 3330 :info :write [0 5] :RequestCanceledException INFO [2019-12-17 12:29:38,358] jepsen worker 0 - jepsen.util 3420 :invoke :read [0 nil] INFO [2019-12-17 12:29:38,361] jepsen worker 72 - jepsen.util 2502 :ok :read [24 21] INFO [2019-12-17 12:29:38,361] jepsen worker 72 - jepsen.util 2502 :invoke :cas [24 [29 36]] INFO [2019-12-17 12:29:38,365] jepsen worker 37 - jepsen.util 2197 :ok :read [12 6] INFO [2019-12-17 12:29:38,366] jepsen worker 37 - jepsen.util 2197 :invoke :read [12 nil] INFO [2019-12-17 12:29:38,369] jepsen worker 74 - jepsen.util 2414 :fail :cas [24 [49 39]] :ValueNotSwapFrom INFO [2019-12-17 12:29:38,369] jepsen worker 74 - jepsen.util 2414 :invoke :write [24 49] INFO [2019-12-17 12:29:38,371] jepsen worker 9 - jepsen.util 2889 :ok :read [3 23] INFO [2019-12-17 12:29:38,371] jepsen worker 9 - jepsen.util 2889 :invoke :write [3 29] INFO [2019-12-17 12:29:38,389] jepsen worker 59 - jepsen.util 3029 :info :write [19 38] :RequestCanceledException INFO [2019-12-17 12:29:38,390] jepsen worker 59 - jepsen.util 3119 :invoke :cas [19 [35 27]] INFO [2019-12-17 12:29:38,399] jepsen worker 72 - jepsen.util 2502 :fail :cas [24 [29 36]] :ValueNotSwapFrom INFO [2019-12-17 12:29:38,400] jepsen worker 72 - jepsen.util 2502 :invoke :cas [24 [27 12]] INFO [2019-12-17 12:29:38,407] jepsen worker 1 - jepsen.util 2971 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:29:38,407] jepsen worker 21 - jepsen.util 3261 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:29:38,408] jepsen worker 1 - jepsen.util 2971 :invoke :read [0 nil] INFO [2019-12-17 12:29:38,408] jepsen worker 21 - jepsen.util 3261 :invoke :cas [7 [28 30]] INFO [2019-12-17 12:29:38,420] jepsen worker 45 - jepsen.util 2655 :ok :write [15 23] INFO [2019-12-17 12:29:38,420] jepsen worker 45 - jepsen.util 2655 :invoke :read [15 nil] INFO [2019-12-17 12:29:38,423] jepsen worker 57 - jepsen.util 3027 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:29:38,423] jepsen worker 85 - jepsen.util 3505 :info :cas [28 [23 36]] :RequestCanceledException INFO [2019-12-17 12:29:38,423] jepsen worker 57 - jepsen.util 3027 :invoke :write [19 26] INFO [2019-12-17 12:29:38,424] jepsen worker 69 - jepsen.util 2319 :fail :cas [23 [39 11]] :ValueNotSwapFrom INFO [2019-12-17 12:29:38,424] jepsen worker 69 - jepsen.util 2319 :invoke :write [23 4] INFO [2019-12-17 12:29:38,424] jepsen worker 85 - jepsen.util 3595 :invoke :read [28 nil] INFO [2019-12-17 12:29:38,437] jepsen worker 84 - jepsen.util 3144 :info :write [28 9] :RequestCanceledException INFO [2019-12-17 12:29:38,438] jepsen worker 84 - jepsen.util 3234 :invoke :cas [28 [33 33]] INFO [2019-12-17 12:29:38,438] jepsen worker 45 - jepsen.util 2655 :ok :read [15 23] INFO [2019-12-17 12:29:38,438] jepsen worker 45 - jepsen.util 2655 :invoke :read [15 nil] INFO [2019-12-17 12:29:38,454] jepsen worker 62 - jepsen.util 3032 :info :cas [20 [37 8]] :RequestCanceledException INFO [2019-12-17 12:29:38,455] jepsen worker 62 - jepsen.util 3122 :invoke :cas [20 [29 48]] INFO [2019-12-17 12:29:38,456] jepsen worker 45 - jepsen.util 2655 :ok :read [15 23] INFO [2019-12-17 12:29:38,457] jepsen worker 45 - jepsen.util 2655 :invoke :cas [15 [27 46]] INFO [2019-12-17 12:29:38,457] jepsen worker 19 - jepsen.util 2629 :ok :write [6 49] INFO [2019-12-17 12:29:38,457] jepsen worker 19 - jepsen.util 2629 :invoke :write [6 26] INFO [2019-12-17 12:29:38,477] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":14179,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:29:38,478] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":14179,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:29:38,478] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":14182,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:29:38,478] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":14182,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:29:38,478] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":14184,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:29:38,478] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":14184,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:29:38,489] jepsen worker 75 - jepsen.util 3495 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:29:38,489] jepsen worker 75 - jepsen.util 3495 :invoke :read [25 nil] INFO [2019-12-17 12:29:38,491] jepsen worker 45 - jepsen.util 2655 :fail :cas [15 [27 46]] :ValueNotSwapFrom INFO [2019-12-17 12:29:38,491] jepsen worker 45 - jepsen.util 2655 :invoke :write [15 36] INFO [2019-12-17 12:29:38,505] jepsen worker 74 - jepsen.util 2414 :ok :write [24 49] INFO [2019-12-17 12:29:38,505] jepsen worker 74 - jepsen.util 2414 :invoke :write [24 31] INFO [2019-12-17 12:29:38,507] jepsen worker 24 - jepsen.util 2994 :info :cas [8 [7 30]] :RequestCanceledException INFO [2019-12-17 12:29:38,507] jepsen worker 58 - jepsen.util 2938 :info :cas [19 [46 47]] :RequestCanceledException INFO [2019-12-17 12:29:38,508] jepsen worker 24 - jepsen.util 3084 :invoke :cas [8 [37 6]] INFO [2019-12-17 12:29:38,508] jepsen worker 58 - jepsen.util 3028 :invoke :cas [19 [40 25]] INFO [2019-12-17 12:29:38,515] jepsen worker 9 - jepsen.util 2889 :ok :write [3 29] INFO [2019-12-17 12:29:38,515] jepsen worker 9 - jepsen.util 2889 :invoke :read [3 nil] INFO [2019-12-17 12:29:38,523] jepsen worker 29 - jepsen.util 2099 :info :cas [9 [24 9]] :RequestCanceledException INFO [2019-12-17 12:29:38,523] jepsen worker 86 - jepsen.util 3146 :info :cas [28 [19 37]] :RequestCanceledException INFO [2019-12-17 12:29:38,523] jepsen worker 81 - jepsen.util 2871 :info :cas [27 [32 44]] :RequestCanceledException INFO [2019-12-17 12:29:38,523] jepsen worker 83 - jepsen.util 3593 :info :cas [27 [36 36]] :RequestCanceledException INFO [2019-12-17 12:29:38,523] jepsen worker 76 - jepsen.util 3226 :info :cas [25 [43 28]] :RequestCanceledException INFO [2019-12-17 12:29:38,524] jepsen worker 29 - jepsen.util 2189 :invoke :read [9 nil] INFO [2019-12-17 12:29:38,524] jepsen worker 86 - jepsen.util 3236 :invoke :read [28 nil] INFO [2019-12-17 12:29:38,524] jepsen worker 83 - jepsen.util 3683 :invoke :cas [27 [11 24]] INFO [2019-12-17 12:29:38,524] jepsen worker 81 - jepsen.util 2961 :invoke :cas [27 [5 30]] INFO [2019-12-17 12:29:38,524] jepsen worker 76 - jepsen.util 3316 :invoke :read [25 nil] INFO [2019-12-17 12:29:38,536] jepsen worker 12 - jepsen.util 2532 :info :write [4 31] :RequestCanceledException INFO [2019-12-17 12:29:38,536] jepsen worker 41 - jepsen.util 3461 :info :write [13 1] :RequestCanceledException INFO [2019-12-17 12:29:38,537] jepsen worker 41 - jepsen.util 3551 :invoke :read [13 nil] INFO [2019-12-17 12:29:38,537] jepsen worker 12 - jepsen.util 2622 :invoke :write [4 0] INFO [2019-12-17 12:29:38,545] jepsen worker 9 - jepsen.util 2889 :ok :read [3 29] INFO [2019-12-17 12:29:38,545] jepsen worker 9 - jepsen.util 2889 :invoke :write [3 47] INFO [2019-12-17 12:29:38,554] jepsen worker 25 - jepsen.util 3625 :info :write [8 46] :RequestCanceledException INFO [2019-12-17 12:29:38,554] jepsen worker 22 - jepsen.util 3172 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:29:38,554] jepsen worker 22 - jepsen.util 3172 :invoke :write [7 3] INFO [2019-12-17 12:29:38,555] jepsen worker 25 - jepsen.util 3715 :invoke :cas [8 [37 41]] INFO [2019-12-17 12:29:38,563] jepsen worker 51 - jepsen.util 2301 :fail :cas [17 [41 37]] :ValueNotSwapFrom INFO [2019-12-17 12:29:38,563] jepsen worker 51 - jepsen.util 2301 :invoke :read [17 nil] INFO [2019-12-17 12:29:38,580] jepsen worker 29 - jepsen.util 2189 :ok :read [9 5] INFO [2019-12-17 12:29:38,581] jepsen worker 29 - jepsen.util 2189 :invoke :read [9 nil] INFO [2019-12-17 12:29:38,588] jepsen worker 42 - jepsen.util 3822 :info :cas [14 [28 31]] :RequestCanceledException INFO [2019-12-17 12:29:38,589] jepsen worker 42 - jepsen.util 3912 :invoke :write [14 23] INFO [2019-12-17 12:29:38,593] jepsen worker 29 - jepsen.util 2189 :ok :read [9 5] INFO [2019-12-17 12:29:38,594] jepsen worker 29 - jepsen.util 2189 :invoke :write [9 46] INFO [2019-12-17 12:29:38,606] jepsen worker 40 - jepsen.util 3550 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:29:38,606] jepsen worker 30 - jepsen.util 2910 :info :write [10 28] :RequestCanceledException INFO [2019-12-17 12:29:38,606] jepsen worker 40 - jepsen.util 3550 :invoke :read [13 nil] INFO [2019-12-17 12:29:38,607] jepsen worker 30 - jepsen.util 3000 :invoke :write [10 5] INFO [2019-12-17 12:29:38,619] jepsen worker 48 - jepsen.util 2658 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:29:38,619] jepsen worker 48 - jepsen.util 2658 :invoke :read [16 nil] INFO [2019-12-17 12:29:38,622] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:29:38,622] jepsen nemesis - couchbase.util Rebalance check count: 102 INFO [2019-12-17 12:29:38,622] jepsen nemesis - couchbase.util Rebalance stuck count: 83 INFO [2019-12-17 12:29:38,638] jepsen worker 26 - jepsen.util 3446 :info :cas [8 [48 7]] :RequestCanceledException INFO [2019-12-17 12:29:38,638] jepsen worker 82 - jepsen.util 3862 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:29:38,638] jepsen worker 82 - jepsen.util 3862 :invoke :write [27 40] INFO [2019-12-17 12:29:38,639] jepsen worker 26 - jepsen.util 3536 :invoke :cas [8 [37 44]] INFO [2019-12-17 12:29:38,643] jepsen worker 29 - jepsen.util 2189 :ok :write [9 46] INFO [2019-12-17 12:29:38,643] jepsen worker 29 - jepsen.util 2189 :invoke :write [9 35] INFO [2019-12-17 12:29:38,654] jepsen worker 33 - jepsen.util 3453 :info :write [11 42] :RequestCanceledException INFO [2019-12-17 12:29:38,655] jepsen worker 33 - jepsen.util 3543 :invoke :cas [11 [33 19]] INFO [2019-12-17 12:29:38,666] jepsen worker 45 - jepsen.util 2655 :ok :write [15 36] INFO [2019-12-17 12:29:38,666] jepsen worker 45 - jepsen.util 2655 :invoke :read [15 nil] INFO [2019-12-17 12:29:38,680] jepsen worker 13 - jepsen.util 2263 :ok :read [4 6] INFO [2019-12-17 12:29:38,680] jepsen worker 13 - jepsen.util 2263 :invoke :cas [4 [10 44]] INFO [2019-12-17 12:29:38,690] jepsen worker 77 - jepsen.util 3137 :info :write [25 11] :RequestCanceledException INFO [2019-12-17 12:29:38,691] jepsen worker 77 - jepsen.util 3227 :invoke :read [25 nil] INFO [2019-12-17 12:29:38,696] jepsen worker 45 - jepsen.util 2655 :ok :read [15 36] INFO [2019-12-17 12:29:38,696] jepsen worker 45 - jepsen.util 2655 :invoke :read [15 nil] INFO [2019-12-17 12:29:38,700] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":14216,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:29:38,700] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":14216,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:29:38,700] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":14220,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:29:38,700] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":14220,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:29:38,707] jepsen worker 8 - jepsen.util 2978 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:29:38,707] jepsen worker 8 - jepsen.util 2978 :invoke :cas [2 [10 28]] INFO [2019-12-17 12:29:38,712] jepsen worker 4 - jepsen.util 1894 :fail :cas [1 [49 42]] :ValueNotSwapFrom INFO [2019-12-17 12:29:38,712] jepsen worker 4 - jepsen.util 1894 :invoke :cas [1 [37 28]] INFO [2019-12-17 12:29:38,717] jepsen worker 29 - jepsen.util 2189 :ok :write [9 35] INFO [2019-12-17 12:29:38,717] jepsen worker 29 - jepsen.util 2189 :invoke :write [9 43] INFO [2019-12-17 12:29:38,717] jepsen worker 45 - jepsen.util 2655 :ok :read [15 36] INFO [2019-12-17 12:29:38,717] jepsen worker 45 - jepsen.util 2655 :invoke :cas [15 [8 28]] INFO [2019-12-17 12:29:38,723] jepsen worker 66 - jepsen.util 3306 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:29:38,723] jepsen worker 66 - jepsen.util 3306 :invoke :read [22 nil] INFO [2019-12-17 12:29:38,736] jepsen worker 63 - jepsen.util 2043 :info :write [21 45] :RequestCanceledException INFO [2019-12-17 12:29:38,736] jepsen worker 17 - jepsen.util 3167 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:29:38,736] jepsen worker 68 - jepsen.util 2948 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:29:38,737] jepsen worker 17 - jepsen.util 3167 :invoke :cas [5 [1 15]] INFO [2019-12-17 12:29:38,737] jepsen worker 68 - jepsen.util 2948 :invoke :write [22 23] INFO [2019-12-17 12:29:38,737] jepsen worker 63 - jepsen.util 2133 :invoke :read [21 nil] INFO [2019-12-17 12:29:38,745] jepsen worker 45 - jepsen.util 2655 :fail :cas [15 [8 28]] :ValueNotSwapFrom INFO [2019-12-17 12:29:38,745] jepsen worker 45 - jepsen.util 2655 :invoke :cas [15 [45 44]] INFO [2019-12-17 12:29:38,754] jepsen worker 38 - jepsen.util 2378 :info :write [12 17] :RequestCanceledException INFO [2019-12-17 12:29:38,756] jepsen worker 38 - jepsen.util 2468 :invoke :cas [12 [33 45]] INFO [2019-12-17 12:29:38,781] jepsen worker 45 - jepsen.util 2655 :fail :cas [15 [45 44]] :ValueNotSwapFrom INFO [2019-12-17 12:29:38,782] jepsen worker 45 - jepsen.util 2655 :invoke :cas [15 [21 0]] INFO [2019-12-17 12:29:38,786] jepsen worker 14 - jepsen.util 2354 :ok :read [4 6] INFO [2019-12-17 12:29:38,786] jepsen worker 14 - jepsen.util 2354 :invoke :read [4 nil] INFO [2019-12-17 12:29:38,811] jepsen worker 45 - jepsen.util 2655 :fail :cas [15 [21 0]] :ValueNotSwapFrom INFO [2019-12-17 12:29:38,811] jepsen worker 45 - jepsen.util 2655 :invoke :write [15 0] INFO [2019-12-17 12:29:38,825] jepsen worker 37 - jepsen.util 2197 :ok :read [12 6] INFO [2019-12-17 12:29:38,825] jepsen worker 37 - jepsen.util 2197 :invoke :cas [12 [20 41]] INFO [2019-12-17 12:29:38,842] jepsen worker 29 - jepsen.util 2189 :ok :write [9 43] INFO [2019-12-17 12:29:38,842] jepsen worker 29 - jepsen.util 2189 :invoke :write [9 20] INFO [2019-12-17 12:29:38,851] jepsen worker 39 - jepsen.util 3369 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:29:38,851] jepsen worker 39 - jepsen.util 3369 :invoke :read [13 nil] INFO [2019-12-17 12:29:38,866] jepsen worker 4 - jepsen.util 1894 :fail :cas [1 [37 28]] :ValueNotSwapFrom INFO [2019-12-17 12:29:38,866] jepsen worker 4 - jepsen.util 1894 :invoke :write [1 25] INFO [2019-12-17 12:29:38,891] jepsen worker 29 - jepsen.util 2189 :ok :write [9 20] INFO [2019-12-17 12:29:38,891] jepsen worker 29 - jepsen.util 2189 :invoke :cas [9 [5 26]] INFO [2019-12-17 12:29:38,907] jepsen worker 20 - jepsen.util 2090 :info :cas [6 [34 41]] :RequestCanceledException INFO [2019-12-17 12:29:38,908] jepsen worker 20 - jepsen.util 2180 :invoke :write [6 29] INFO [2019-12-17 12:29:38,912] jepsen worker 29 - jepsen.util 2189 :fail :cas [9 [5 26]] :ValueNotSwapFrom INFO [2019-12-17 12:29:38,912] jepsen worker 29 - jepsen.util 2189 :invoke :write [9 3] INFO [2019-12-17 12:29:38,923] jepsen worker 2 - jepsen.util 3152 :info :write [0 30] :RequestCanceledException INFO [2019-12-17 12:29:38,923] jepsen worker 53 - jepsen.util 2393 :info :write [17 37] :RequestCanceledException INFO [2019-12-17 12:29:38,924] jepsen worker 2 - jepsen.util 3242 :invoke :read [0 nil] INFO [2019-12-17 12:29:38,925] jepsen worker 53 - jepsen.util 2483 :invoke :cas [17 [38 30]] INFO [2019-12-17 12:29:38,938] jepsen worker 56 - jepsen.util 2306 :info :write [18 27] :RequestCanceledException INFO [2019-12-17 12:29:38,940] jepsen worker 56 - jepsen.util 2396 :invoke :cas [18 [35 39]] INFO [2019-12-17 12:29:38,953] jepsen worker 29 - jepsen.util 2189 :ok :write [9 3] INFO [2019-12-17 12:29:38,954] jepsen worker 29 - jepsen.util 2189 :invoke :read [9 nil] INFO [2019-12-17 12:29:38,954] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":14246,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:29:38,954] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":14246,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:29:38,954] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":14249,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:29:38,954] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":14249,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:29:38,954] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":14250,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:29:38,955] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":14250,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:29:38,966] jepsen worker 29 - jepsen.util 2189 :ok :read [9 3] INFO [2019-12-17 12:29:38,966] jepsen worker 29 - jepsen.util 2189 :invoke :write [9 33] INFO [2019-12-17 12:29:38,993] jepsen worker 53 - jepsen.util 2483 :fail :cas [17 [38 30]] :ValueNotSwapFrom INFO [2019-12-17 12:29:38,993] jepsen worker 53 - jepsen.util 2483 :invoke :cas [17 [20 28]] INFO [2019-12-17 12:29:39,003] jepsen worker 78 - jepsen.util 2598 :fail :cas [26 [43 49]] :ValueNotSwapFrom INFO [2019-12-17 12:29:39,003] jepsen worker 78 - jepsen.util 2598 :invoke :read [26 nil] INFO [2019-12-17 12:29:39,006] jepsen worker 29 - jepsen.util 2189 :ok :write [9 33] INFO [2019-12-17 12:29:39,006] jepsen worker 29 - jepsen.util 2189 :invoke :write [9 32] INFO [2019-12-17 12:29:39,008] jepsen worker 61 - jepsen.util 3481 :info :cas [20 [27 42]] :RequestCanceledException INFO [2019-12-17 12:29:39,009] jepsen worker 61 - jepsen.util 3571 :invoke :write [20 36] INFO [2019-12-17 12:29:39,023] jepsen worker 52 - jepsen.util 2572 :info :write [17 18] :RequestCanceledException INFO [2019-12-17 12:29:39,024] jepsen worker 52 - jepsen.util 2662 :invoke :read [17 nil] INFO [2019-12-17 12:29:39,050] jepsen worker 29 - jepsen.util 2189 :ok :write [9 32] INFO [2019-12-17 12:29:39,050] jepsen worker 29 - jepsen.util 2189 :invoke :write [9 27] INFO [2019-12-17 12:29:39,058] jepsen worker 51 - jepsen.util 2301 :ok :read [17 5] INFO [2019-12-17 12:29:39,059] jepsen worker 51 - jepsen.util 2301 :invoke :read [17 nil] INFO [2019-12-17 12:29:39,064] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":14261,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:29:39,064] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":14261,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:29:39,064] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":14264,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:29:39,064] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":14264,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:29:39,064] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":14265,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:29:39,064] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":14265,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:29:39,087] jepsen worker 29 - jepsen.util 2189 :ok :write [9 27] INFO [2019-12-17 12:29:39,087] jepsen worker 29 - jepsen.util 2189 :invoke :cas [9 [34 47]] INFO [2019-12-17 12:29:39,106] jepsen worker 29 - jepsen.util 2189 :fail :cas [9 [34 47]] :ValueNotSwapFrom INFO [2019-12-17 12:29:39,106] jepsen worker 29 - jepsen.util 2189 :invoke :read [9 nil] INFO [2019-12-17 12:29:39,117] jepsen worker 29 - jepsen.util 2189 :ok :read [9 27] INFO [2019-12-17 12:29:39,117] jepsen worker 29 - jepsen.util 2189 :invoke :write [9 38] INFO [2019-12-17 12:29:39,123] jepsen worker 78 - jepsen.util 2598 :ok :read [26 4] INFO [2019-12-17 12:29:39,123] jepsen worker 78 - jepsen.util 2598 :invoke :cas [26 [25 49]] INFO [2019-12-17 12:29:39,161] jepsen worker 29 - jepsen.util 2189 :ok :write [9 38] INFO [2019-12-17 12:29:39,161] jepsen worker 29 - jepsen.util 2189 :invoke :cas [9 [19 42]] INFO [2019-12-17 12:29:39,174] jepsen worker 29 - jepsen.util 2189 :fail :cas [9 [19 42]] :ValueNotSwapFrom INFO [2019-12-17 12:29:39,174] jepsen worker 29 - jepsen.util 2189 :invoke :write [9 42] INFO [2019-12-17 12:29:39,202] jepsen worker 19 - jepsen.util 2629 :ok :write [6 26] INFO [2019-12-17 12:29:39,203] jepsen worker 74 - jepsen.util 2414 :ok :write [24 31] INFO [2019-12-17 12:29:39,203] jepsen worker 19 - jepsen.util 2629 :invoke :read [6 nil] INFO [2019-12-17 12:29:39,203] jepsen worker 74 - jepsen.util 2414 :invoke :read [24 nil] INFO [2019-12-17 12:29:39,203] jepsen worker 9 - jepsen.util 2889 :ok :write [3 47] INFO [2019-12-17 12:29:39,203] jepsen worker 9 - jepsen.util 2889 :invoke :write [3 43] INFO [2019-12-17 12:29:39,206] jepsen worker 45 - jepsen.util 2655 :ok :write [15 0] INFO [2019-12-17 12:29:39,206] jepsen worker 45 - jepsen.util 2655 :invoke :cas [15 [36 11]] INFO [2019-12-17 12:29:39,224] jepsen worker 38 - jepsen.util 2468 :fail :cas [12 [33 45]] :ValueNotSwapFrom INFO [2019-12-17 12:29:39,224] jepsen worker 38 - jepsen.util 2468 :invoke :read [12 nil] INFO [2019-12-17 12:29:39,232] jepsen worker 63 - jepsen.util 2133 :ok :read [21 30] INFO [2019-12-17 12:29:39,233] jepsen worker 63 - jepsen.util 2133 :invoke :read [21 nil] INFO [2019-12-17 12:29:39,240] jepsen worker 52 - jepsen.util 2662 :ok :read [17 5] INFO [2019-12-17 12:29:39,240] jepsen worker 52 - jepsen.util 2662 :invoke :read [17 nil] INFO [2019-12-17 12:29:39,244] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":14279,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:29:39,244] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":14279,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:29:39,244] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":14280,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:29:39,244] jepsen worker 29 - jepsen.util 2189 :ok :write [9 42] INFO [2019-12-17 12:29:39,245] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":14280,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:29:39,245] jepsen worker 29 - jepsen.util 2189 :invoke :write [9 18] INFO [2019-12-17 12:29:39,252] jepsen worker 45 - jepsen.util 2655 :fail :cas [15 [36 11]] :ValueNotSwapFrom INFO [2019-12-17 12:29:39,252] jepsen worker 45 - jepsen.util 2655 :invoke :cas [15 [17 38]] INFO [2019-12-17 12:29:39,272] jepsen worker 37 - jepsen.util 2197 :fail :cas [12 [20 41]] :ValueNotSwapFrom INFO [2019-12-17 12:29:39,272] jepsen worker 37 - jepsen.util 2197 :invoke :read [12 nil] INFO [2019-12-17 12:29:39,273] jepsen worker 53 - jepsen.util 2483 :fail :cas [17 [20 28]] :ValueNotSwapFrom INFO [2019-12-17 12:29:39,274] jepsen worker 53 - jepsen.util 2483 :invoke :cas [17 [14 1]] INFO [2019-12-17 12:29:39,279] jepsen worker 19 - jepsen.util 2629 :ok :read [6 26] INFO [2019-12-17 12:29:39,279] jepsen worker 19 - jepsen.util 2629 :invoke :cas [6 [21 18]] INFO [2019-12-17 12:29:39,284] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":14290,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:29:39,284] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":14290,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:29:39,284] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":14293,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:29:39,284] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":14293,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:29:39,284] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":14294,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:29:39,284] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":14294,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:29:39,290] jepsen worker 88 - jepsen.util 2428 :info :write [29 4] :RequestCanceledException INFO [2019-12-17 12:29:39,291] jepsen worker 88 - jepsen.util 2518 :invoke :write [29 16] INFO [2019-12-17 12:29:39,308] jepsen worker 79 - jepsen.util 2419 :info :write [26 7] :RequestCanceledException INFO [2019-12-17 12:29:39,309] jepsen worker 79 - jepsen.util 2509 :invoke :read [26 nil] INFO [2019-12-17 12:29:39,317] jepsen worker 5 - jepsen.util 2075 :ok :cas [1 [38 36]] INFO [2019-12-17 12:29:39,317] jepsen worker 5 - jepsen.util 2075 :invoke :write [1 27] INFO [2019-12-17 12:29:39,318] jepsen worker 12 - jepsen.util 2622 :ok :write [4 0] INFO [2019-12-17 12:29:39,318] jepsen worker 12 - jepsen.util 2622 :invoke :read [4 nil] INFO [2019-12-17 12:29:39,319] jepsen worker 63 - jepsen.util 2133 :ok :read [21 30] INFO [2019-12-17 12:29:39,319] jepsen worker 63 - jepsen.util 2133 :invoke :write [21 48] INFO [2019-12-17 12:29:39,331] jepsen worker 45 - jepsen.util 2655 :fail :cas [15 [17 38]] :ValueNotSwapFrom INFO [2019-12-17 12:29:39,331] jepsen worker 45 - jepsen.util 2655 :invoke :read [15 nil] INFO [2019-12-17 12:29:39,337] jepsen worker 10 - jepsen.util 2350 :fail :read [3 nil] :RequestTimeoutException INFO [2019-12-17 12:29:39,337] jepsen worker 18 - jepsen.util 2718 :fail :read [6 nil] :RequestTimeoutException INFO [2019-12-17 12:29:39,337] jepsen worker 10 - jepsen.util 2350 :invoke :read [3 nil] INFO [2019-12-17 12:29:39,337] jepsen worker 18 - jepsen.util 2718 :invoke :cas [6 [37 44]] INFO [2019-12-17 12:29:39,353] jepsen worker 45 - jepsen.util 2655 :ok :read [15 0] INFO [2019-12-17 12:29:39,353] jepsen worker 45 - jepsen.util 2655 :invoke :write [15 25] INFO [2019-12-17 12:29:39,391] jepsen worker 19 - jepsen.util 2629 :fail :cas [6 [21 18]] :ValueNotSwapFrom INFO [2019-12-17 12:29:39,392] jepsen worker 19 - jepsen.util 2629 :invoke :cas [6 [27 14]] INFO [2019-12-17 12:29:39,396] jepsen worker 79 - jepsen.util 2509 :ok :read [26 4] INFO [2019-12-17 12:29:39,396] jepsen worker 79 - jepsen.util 2509 :invoke :write [26 21] INFO [2019-12-17 12:29:39,399] jepsen worker 13 - jepsen.util 2263 :fail :cas [4 [10 44]] :ValueNotSwapFrom INFO [2019-12-17 12:29:39,399] jepsen worker 13 - jepsen.util 2263 :invoke :cas [4 [19 17]] INFO [2019-12-17 12:29:39,433] jepsen worker 78 - jepsen.util 2598 :fail :cas [26 [25 49]] :ValueNotSwapFrom INFO [2019-12-17 12:29:39,433] jepsen worker 78 - jepsen.util 2598 :invoke :write [26 25] INFO [2019-12-17 12:29:39,435] jepsen worker 14 - jepsen.util 2354 :ok :read [4 0] INFO [2019-12-17 12:29:39,435] jepsen worker 14 - jepsen.util 2354 :invoke :read [4 nil] INFO [2019-12-17 12:29:39,440] jepsen worker 29 - jepsen.util 2189 :ok :write [9 18] INFO [2019-12-17 12:29:39,440] jepsen worker 29 - jepsen.util 2189 :invoke :cas [9 [29 42]] INFO [2019-12-17 12:29:39,446] jepsen worker 63 - jepsen.util 2133 :ok :write [21 48] INFO [2019-12-17 12:29:39,446] jepsen worker 63 - jepsen.util 2133 :invoke :read [21 nil] INFO [2019-12-17 12:29:39,456] jepsen worker 51 - jepsen.util 2301 :ok :read [17 5] INFO [2019-12-17 12:29:39,457] jepsen worker 51 - jepsen.util 2301 :invoke :cas [17 [14 35]] INFO [2019-12-17 12:29:39,457] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":14316,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:29:39,457] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":14316,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:29:39,457] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":14319,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:29:39,457] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":14319,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:29:39,480] jepsen worker 29 - jepsen.util 2189 :fail :cas [9 [29 42]] :ValueNotSwapFrom INFO [2019-12-17 12:29:39,480] jepsen worker 29 - jepsen.util 2189 :invoke :read [9 nil] INFO [2019-12-17 12:29:39,501] jepsen worker 29 - jepsen.util 2189 :ok :read [9 18] INFO [2019-12-17 12:29:39,501] jepsen worker 29 - jepsen.util 2189 :invoke :write [9 39] INFO [2019-12-17 12:29:39,514] jepsen worker 19 - jepsen.util 2629 :fail :cas [6 [27 14]] :ValueNotSwapFrom INFO [2019-12-17 12:29:39,514] jepsen worker 19 - jepsen.util 2629 :invoke :read [6 nil] INFO [2019-12-17 12:29:39,520] jepsen worker 4 - jepsen.util 1894 :ok :write [1 25] INFO [2019-12-17 12:29:39,520] jepsen worker 4 - jepsen.util 1894 :invoke :cas [1 [34 12]] INFO [2019-12-17 12:29:39,523] jepsen worker 67 - jepsen.util 3667 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:29:39,523] jepsen worker 67 - jepsen.util 3667 :invoke :cas [22 [14 30]] INFO [2019-12-17 12:29:39,544] jepsen worker 56 - jepsen.util 2396 :fail :cas [18 [35 39]] :ValueNotSwapFrom INFO [2019-12-17 12:29:39,544] jepsen worker 56 - jepsen.util 2396 :invoke :write [18 5] INFO [2019-12-17 12:29:39,553] jepsen worker 63 - jepsen.util 2133 :ok :read [21 48] INFO [2019-12-17 12:29:39,553] jepsen worker 63 - jepsen.util 2133 :invoke :write [21 17] INFO [2019-12-17 12:29:39,554] jepsen worker 6 - jepsen.util 3336 :info :cas [2 [18 8]] :RequestCanceledException INFO [2019-12-17 12:29:39,555] jepsen worker 6 - jepsen.util 3426 :invoke :read [2 nil] INFO [2019-12-17 12:29:39,561] jepsen worker 19 - jepsen.util 2629 :ok :read [6 26] INFO [2019-12-17 12:29:39,561] jepsen worker 19 - jepsen.util 2629 :invoke :read [6 nil] INFO [2019-12-17 12:29:39,572] jepsen worker 53 - jepsen.util 2483 :fail :cas [17 [14 1]] :ValueNotSwapFrom INFO [2019-12-17 12:29:39,572] jepsen worker 53 - jepsen.util 2483 :invoke :write [17 6] INFO [2019-12-17 12:29:39,589] jepsen worker 87 - jepsen.util 2247 :info :write [29 42] :RequestCanceledException INFO [2019-12-17 12:29:39,589] jepsen worker 44 - jepsen.util 3014 :info :cas [14 [34 40]] :RequestCanceledException INFO [2019-12-17 12:29:39,589] jepsen worker 29 - jepsen.util 2189 :ok :write [9 39] INFO [2019-12-17 12:29:39,590] jepsen worker 29 - jepsen.util 2189 :invoke :write [9 9] INFO [2019-12-17 12:29:39,590] jepsen worker 87 - jepsen.util 2337 :invoke :read [29 nil] INFO [2019-12-17 12:29:39,590] jepsen worker 44 - jepsen.util 3104 :invoke :cas [14 [45 35]] INFO [2019-12-17 12:29:39,607] jepsen worker 60 - jepsen.util 3480 :info :cas [20 [19 40]] :RequestCanceledException INFO [2019-12-17 12:29:39,608] jepsen worker 60 - jepsen.util 3570 :invoke :read [20 nil] INFO [2019-12-17 12:29:39,622] jepsen worker 49 - jepsen.util 2479 :info :cas [16 [4 46]] :RequestCanceledException INFO [2019-12-17 12:29:39,622] jepsen worker 7 - jepsen.util 3067 :info :write [2 11] :RequestCanceledException INFO [2019-12-17 12:29:39,622] jepsen worker 27 - jepsen.util 2097 :info :write [9 48] :RequestCanceledException INFO [2019-12-17 12:29:39,622] jepsen worker 31 - jepsen.util 2371 :info :cas [10 [18 36]] :RequestCanceledException INFO [2019-12-17 12:29:39,623] jepsen worker 49 - jepsen.util 2569 :invoke :cas [16 [15 18]] INFO [2019-12-17 12:29:39,623] jepsen worker 7 - jepsen.util 3157 :invoke :write [2 14] INFO [2019-12-17 12:29:39,624] jepsen worker 31 - jepsen.util 2461 :invoke :read [10 nil] INFO [2019-12-17 12:29:39,624] jepsen worker 27 - jepsen.util 2187 :invoke :read [9 nil] INFO [2019-12-17 12:29:39,632] jepsen worker 23 - jepsen.util 2993 :info :cas [7 [24 12]] :RequestCanceledException INFO [2019-12-17 12:29:39,633] jepsen worker 23 - jepsen.util 3083 :invoke :write [7 18] INFO [2019-12-17 12:29:39,640] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:29:39,641] jepsen nemesis - couchbase.util Rebalance check count: 103 INFO [2019-12-17 12:29:39,641] jepsen nemesis - couchbase.util Rebalance stuck count: 84 INFO [2019-12-17 12:29:39,646] jepsen worker 63 - jepsen.util 2133 :ok :write [21 17] INFO [2019-12-17 12:29:39,646] jepsen worker 63 - jepsen.util 2133 :invoke :write [21 12] INFO [2019-12-17 12:29:39,651] jepsen worker 34 - jepsen.util 3274 :info :write [11 39] :RequestCanceledException INFO [2019-12-17 12:29:39,653] jepsen worker 34 - jepsen.util 3364 :invoke :cas [11 [36 33]] INFO [2019-12-17 12:29:39,667] jepsen worker 29 - jepsen.util 2189 :ok :write [9 9] INFO [2019-12-17 12:29:39,667] jepsen worker 29 - jepsen.util 2189 :invoke :cas [9 [23 33]] INFO [2019-12-17 12:29:39,680] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":14348,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:29:39,680] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":14348,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:29:39,680] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":14349,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:29:39,680] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":14349,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:29:39,690] jepsen worker 3 - jepsen.util 2343 :info :write [1 38] :RequestCanceledException INFO [2019-12-17 12:29:39,691] jepsen worker 27 - jepsen.util 2187 :ok :read [9 9] INFO [2019-12-17 12:29:39,691] jepsen worker 27 - jepsen.util 2187 :invoke :read [9 nil] INFO [2019-12-17 12:29:39,691] jepsen worker 3 - jepsen.util 2433 :invoke :write [1 39] INFO [2019-12-17 12:29:39,695] jepsen worker 54 - jepsen.util 2484 :ok :write [18 24] INFO [2019-12-17 12:29:39,695] jepsen worker 54 - jepsen.util 2484 :invoke :cas [18 [8 3]] INFO [2019-12-17 12:29:39,700] jepsen worker 29 - jepsen.util 2189 :fail :cas [9 [23 33]] :ValueNotSwapFrom INFO [2019-12-17 12:29:39,700] jepsen worker 29 - jepsen.util 2189 :invoke :cas [9 [34 41]] INFO [2019-12-17 12:29:39,706] jepsen worker 65 - jepsen.util 2135 :info :write [21 41] :RequestCanceledException INFO [2019-12-17 12:29:39,706] jepsen worker 47 - jepsen.util 2837 :info :write [15 3] :RequestCanceledException INFO [2019-12-17 12:29:39,708] jepsen worker 65 - jepsen.util 2225 :invoke :read [21 nil] INFO [2019-12-17 12:29:39,708] jepsen worker 47 - jepsen.util 2927 :invoke :read [15 nil] INFO [2019-12-17 12:29:39,714] jepsen worker 27 - jepsen.util 2187 :ok :read [9 9] INFO [2019-12-17 12:29:39,714] jepsen worker 27 - jepsen.util 2187 :invoke :write [9 26] INFO [2019-12-17 12:29:39,723] jepsen worker 89 - jepsen.util 2339 :info :write [29 22] :RequestCanceledException INFO [2019-12-17 12:29:39,723] jepsen worker 28 - jepsen.util 1828 :info :cas [9 [12 36]] :RequestCanceledException INFO [2019-12-17 12:29:39,724] jepsen worker 89 - jepsen.util 2429 :invoke :cas [29 [43 25]] INFO [2019-12-17 12:29:39,724] jepsen worker 28 - jepsen.util 1918 :invoke :read [9 nil] INFO [2019-12-17 12:29:39,749] jepsen worker 54 - jepsen.util 2484 :fail :cas [18 [8 3]] :ValueNotSwapFrom INFO [2019-12-17 12:29:39,750] jepsen worker 54 - jepsen.util 2484 :invoke :write [18 22] INFO [2019-12-17 12:29:39,758] jepsen worker 29 - jepsen.util 2189 :fail :cas [9 [34 41]] :ValueNotSwapFrom INFO [2019-12-17 12:29:39,758] jepsen worker 29 - jepsen.util 2189 :invoke :write [9 4] INFO [2019-12-17 12:29:39,762] jepsen worker 53 - jepsen.util 2483 :ok :write [17 6] INFO [2019-12-17 12:29:39,763] jepsen worker 53 - jepsen.util 2483 :invoke :read [17 nil] INFO [2019-12-17 12:29:39,765] jepsen worker 69 - jepsen.util 2319 :ok :write [23 4] INFO [2019-12-17 12:29:39,766] jepsen worker 69 - jepsen.util 2319 :invoke :write [23 10] INFO [2019-12-17 12:29:39,769] jepsen worker 89 - jepsen.util 2429 :fail :cas [29 [43 25]] :ValueNotSwapFrom INFO [2019-12-17 12:29:39,770] jepsen worker 89 - jepsen.util 2429 :invoke :read [29 nil] INFO [2019-12-17 12:29:39,770] jepsen worker 63 - jepsen.util 2133 :ok :write [21 12] INFO [2019-12-17 12:29:39,770] jepsen worker 63 - jepsen.util 2133 :invoke :cas [21 [39 18]] INFO [2019-12-17 12:29:39,796] jepsen worker 28 - jepsen.util 1918 :ok :read [9 9] INFO [2019-12-17 12:29:39,796] jepsen worker 28 - jepsen.util 1918 :invoke :cas [9 [15 32]] INFO [2019-12-17 12:29:39,812] jepsen worker 27 - jepsen.util 2187 :ok :write [9 26] INFO [2019-12-17 12:29:39,813] jepsen worker 27 - jepsen.util 2187 :invoke :write [9 27] INFO [2019-12-17 12:29:39,816] jepsen worker 63 - jepsen.util 2133 :fail :cas [21 [39 18]] :ValueNotSwapFrom INFO [2019-12-17 12:29:39,817] jepsen worker 63 - jepsen.util 2133 :invoke :read [21 nil] INFO [2019-12-17 12:29:39,836] jepsen worker 28 - jepsen.util 1918 :fail :cas [9 [15 32]] :ValueNotSwapFrom INFO [2019-12-17 12:29:39,837] jepsen worker 28 - jepsen.util 1918 :invoke :cas [9 [48 42]] INFO [2019-12-17 12:29:39,837] jepsen worker 32 - jepsen.util 2912 :info :write [10 16] :RequestCanceledException INFO [2019-12-17 12:29:39,838] jepsen worker 32 - jepsen.util 3002 :invoke :write [10 19] INFO [2019-12-17 12:29:39,842] jepsen worker 63 - jepsen.util 2133 :ok :read [21 12] INFO [2019-12-17 12:29:39,843] jepsen worker 63 - jepsen.util 2133 :invoke :cas [21 [15 30]] INFO [2019-12-17 12:29:39,854] jepsen worker 46 - jepsen.util 2656 :info :write [15 30] :RequestCanceledException INFO [2019-12-17 12:29:39,855] jepsen worker 46 - jepsen.util 2746 :invoke :write [15 32] INFO [2019-12-17 12:29:39,874] jepsen worker 28 - jepsen.util 1918 :fail :cas [9 [48 42]] :ValueNotSwapFrom INFO [2019-12-17 12:29:39,874] jepsen worker 28 - jepsen.util 1918 :invoke :cas [9 [37 37]] INFO [2019-12-17 12:29:39,887] jepsen worker 63 - jepsen.util 2133 :fail :cas [21 [15 30]] :ValueNotSwapFrom INFO [2019-12-17 12:29:39,887] jepsen worker 63 - jepsen.util 2133 :invoke :cas [21 [20 45]] INFO [2019-12-17 12:29:39,901] jepsen worker 53 - jepsen.util 2483 :ok :read [17 6] INFO [2019-12-17 12:29:39,901] jepsen worker 53 - jepsen.util 2483 :invoke :write [17 3] INFO [2019-12-17 12:29:39,901] jepsen worker 9 - jepsen.util 2889 :ok :write [3 43] INFO [2019-12-17 12:29:39,901] jepsen worker 9 - jepsen.util 2889 :invoke :cas [3 [15 45]] INFO [2019-12-17 12:29:39,905] jepsen worker 30 - jepsen.util 3000 :ok :write [10 5] INFO [2019-12-17 12:29:39,906] jepsen worker 30 - jepsen.util 3000 :invoke :read [10 nil] INFO [2019-12-17 12:29:39,908] jepsen worker 28 - jepsen.util 1918 :fail :cas [9 [37 37]] :ValueNotSwapFrom INFO [2019-12-17 12:29:39,909] jepsen worker 28 - jepsen.util 1918 :invoke :cas [9 [26 24]] INFO [2019-12-17 12:29:39,935] jepsen worker 36 - jepsen.util 2376 :ok :write [12 36] INFO [2019-12-17 12:29:39,935] jepsen worker 36 - jepsen.util 2376 :invoke :read [12 nil] INFO [2019-12-17 12:29:39,936] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":14376,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:29:39,936] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":14376,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:29:39,936] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":14379,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:29:39,936] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":14379,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:29:39,936] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":14380,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:29:39,936] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":14380,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:29:39,948] jepsen worker 63 - jepsen.util 2133 :fail :cas [21 [20 45]] :ValueNotSwapFrom INFO [2019-12-17 12:29:39,948] jepsen worker 63 - jepsen.util 2133 :invoke :cas [21 [39 0]] INFO [2019-12-17 12:29:39,976] jepsen worker 37 - jepsen.util 2197 :ok :read [12 36] INFO [2019-12-17 12:29:39,977] jepsen worker 37 - jepsen.util 2197 :invoke :write [12 33] INFO [2019-12-17 12:29:39,984] jepsen worker 28 - jepsen.util 1918 :fail :cas [9 [26 24]] :ValueNotSwapFrom INFO [2019-12-17 12:29:39,984] jepsen worker 28 - jepsen.util 1918 :invoke :cas [9 [17 17]] INFO [2019-12-17 12:29:39,988] jepsen worker 10 - jepsen.util 2350 :ok :read [3 43] INFO [2019-12-17 12:29:39,989] jepsen worker 10 - jepsen.util 2350 :invoke :write [3 12] INFO [2019-12-17 12:29:39,992] jepsen worker 38 - jepsen.util 2468 :ok :read [12 36] INFO [2019-12-17 12:29:39,992] jepsen worker 38 - jepsen.util 2468 :invoke :write [12 28] INFO [2019-12-17 12:29:40,004] jepsen worker 45 - jepsen.util 2655 :ok :write [15 25] INFO [2019-12-17 12:29:40,004] jepsen worker 45 - jepsen.util 2655 :invoke :cas [15 [14 49]] INFO [2019-12-17 12:29:40,004] jepsen worker 29 - jepsen.util 2189 :ok :write [9 4] INFO [2019-12-17 12:29:40,004] jepsen worker 29 - jepsen.util 2189 :invoke :read [9 nil] INFO [2019-12-17 12:29:40,032] jepsen worker 79 - jepsen.util 2509 :ok :write [26 21] INFO [2019-12-17 12:29:40,032] jepsen worker 79 - jepsen.util 2509 :invoke :read [26 nil] INFO [2019-12-17 12:29:40,045] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":14394,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:29:40,045] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":14394,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:29:40,051] jepsen worker 52 - jepsen.util 2662 :ok :read [17 6] INFO [2019-12-17 12:29:40,051] jepsen worker 52 - jepsen.util 2662 :invoke :write [17 42] INFO [2019-12-17 12:29:40,053] jepsen worker 55 - jepsen.util 2845 :info :write [18 20] :RequestCanceledException INFO [2019-12-17 12:29:40,053] jepsen worker 71 - jepsen.util 2411 :info :write [23 49] :RequestCanceledException INFO [2019-12-17 12:29:40,054] jepsen worker 71 - jepsen.util 2501 :invoke :cas [23 [15 22]] INFO [2019-12-17 12:29:40,055] jepsen worker 55 - jepsen.util 2935 :invoke :read [18 nil] INFO [2019-12-17 12:29:40,081] jepsen worker 18 - jepsen.util 2718 :fail :cas [6 [37 44]] :ValueNotSwapFrom INFO [2019-12-17 12:29:40,081] jepsen worker 18 - jepsen.util 2718 :invoke :read [6 nil] INFO [2019-12-17 12:29:40,095] jepsen worker 37 - jepsen.util 2197 :ok :write [12 33] INFO [2019-12-17 12:29:40,096] jepsen worker 37 - jepsen.util 2197 :invoke :write [12 41] INFO [2019-12-17 12:29:40,128] jepsen worker 36 - jepsen.util 2376 :ok :read [12 33] INFO [2019-12-17 12:29:40,128] jepsen worker 36 - jepsen.util 2376 :invoke :read [12 nil] INFO [2019-12-17 12:29:40,135] jepsen worker 16 - jepsen.util 3256 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:29:40,135] jepsen worker 16 - jepsen.util 3256 :invoke :write [5 38] INFO [2019-12-17 12:29:40,136] jepsen worker 65 - jepsen.util 2225 :ok :read [21 12] INFO [2019-12-17 12:29:40,136] jepsen worker 65 - jepsen.util 2225 :invoke :read [21 nil] INFO [2019-12-17 12:29:40,177] jepsen worker 63 - jepsen.util 2133 :fail :cas [21 [39 0]] :ValueNotSwapFrom INFO [2019-12-17 12:29:40,177] jepsen worker 63 - jepsen.util 2133 :invoke :write [21 48] INFO [2019-12-17 12:29:40,185] jepsen worker 78 - jepsen.util 2598 :ok :write [26 25] INFO [2019-12-17 12:29:40,185] jepsen worker 78 - jepsen.util 2598 :invoke :cas [26 [26 13]] INFO [2019-12-17 12:29:40,189] jepsen worker 29 - jepsen.util 2189 :ok :read [9 4] INFO [2019-12-17 12:29:40,189] jepsen worker 29 - jepsen.util 2189 :invoke :write [9 25] INFO [2019-12-17 12:29:40,190] jepsen worker 11 - jepsen.util 2981 :info :write [3 10] :RequestCanceledException INFO [2019-12-17 12:29:40,190] jepsen worker 73 - jepsen.util 2413 :info :cas [24 [19 42]] :RequestCanceledException INFO [2019-12-17 12:29:40,191] jepsen worker 11 - jepsen.util 3071 :invoke :cas [3 [26 1]] INFO [2019-12-17 12:29:40,191] jepsen worker 73 - jepsen.util 2503 :invoke :read [24 nil] INFO [2019-12-17 12:29:40,219] jepsen worker 45 - jepsen.util 2655 :fail :cas [15 [14 49]] :ValueNotSwapFrom INFO [2019-12-17 12:29:40,219] jepsen worker 45 - jepsen.util 2655 :invoke :cas [15 [36 23]] INFO [2019-12-17 12:29:40,219] jepsen worker 51 - jepsen.util 2301 :fail :cas [17 [14 35]] :ValueNotSwapFrom INFO [2019-12-17 12:29:40,219] jepsen worker 51 - jepsen.util 2301 :invoke :write [17 31] INFO [2019-12-17 12:29:40,223] jepsen worker 64 - jepsen.util 2134 :fail :read [21 nil] :RequestTimeoutException INFO [2019-12-17 12:29:40,223] jepsen worker 64 - jepsen.util 2134 :invoke :cas [21 [41 42]] INFO [2019-12-17 12:29:40,226] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":14413,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:29:40,226] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":14413,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:29:40,227] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":14414,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:29:40,227] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":14414,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:29:40,227] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":14417,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:29:40,227] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":14417,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:29:40,234] jepsen worker 38 - jepsen.util 2468 :ok :write [12 28] INFO [2019-12-17 12:29:40,234] jepsen worker 38 - jepsen.util 2468 :invoke :write [12 42] INFO [2019-12-17 12:29:40,237] jepsen worker 31 - jepsen.util 2461 :ok :read [10 5] INFO [2019-12-17 12:29:40,237] jepsen worker 31 - jepsen.util 2461 :invoke :cas [10 [13 44]] INFO [2019-12-17 12:29:40,237] jepsen worker 70 - jepsen.util 2860 :info :write [23 46] :RequestCanceledException INFO [2019-12-17 12:29:40,238] jepsen worker 70 - jepsen.util 2950 :invoke :cas [23 [12 49]] INFO [2019-12-17 12:29:40,265] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":14423,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:29:40,265] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":14423,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:29:40,265] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":14426,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:29:40,265] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":14426,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:29:40,265] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":14427,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:29:40,265] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":14427,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:29:40,278] jepsen worker 5 - jepsen.util 2075 :ok :write [1 27] INFO [2019-12-17 12:29:40,278] jepsen worker 5 - jepsen.util 2075 :invoke :write [1 7] INFO [2019-12-17 12:29:40,289] jepsen worker 35 - jepsen.util 3005 :info :write [11 7] :RequestCanceledException INFO [2019-12-17 12:29:40,290] jepsen worker 35 - jepsen.util 3095 :invoke :write [11 33] INFO [2019-12-17 12:29:40,297] jepsen worker 11 - jepsen.util 3071 :fail :cas [3 [26 1]] :ValueNotSwapFrom INFO [2019-12-17 12:29:40,297] jepsen worker 11 - jepsen.util 3071 :invoke :cas [3 [23 25]] INFO [2019-12-17 12:29:40,314] jepsen worker 79 - jepsen.util 2509 :ok :read [26 25] INFO [2019-12-17 12:29:40,314] jepsen worker 79 - jepsen.util 2509 :invoke :read [26 nil] INFO [2019-12-17 12:29:40,320] jepsen worker 15 - jepsen.util 2985 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:29:40,320] jepsen worker 15 - jepsen.util 2985 :invoke :cas [5 [26 30]] INFO [2019-12-17 12:29:40,320] jepsen worker 52 - jepsen.util 2662 :ok :write [17 42] INFO [2019-12-17 12:29:40,321] jepsen worker 52 - jepsen.util 2662 :invoke :read [17 nil] INFO [2019-12-17 12:29:40,324] jepsen worker 28 - jepsen.util 1918 :fail :cas [9 [17 17]] :ValueNotSwapFrom INFO [2019-12-17 12:29:40,324] jepsen worker 28 - jepsen.util 1918 :invoke :cas [9 [36 35]] INFO [2019-12-17 12:29:40,332] jepsen worker 71 - jepsen.util 2501 :fail :cas [23 [15 22]] :ValueNotSwapFrom INFO [2019-12-17 12:29:40,332] jepsen worker 71 - jepsen.util 2501 :invoke :read [23 nil] INFO [2019-12-17 12:29:40,347] jepsen worker 47 - jepsen.util 2927 :ok :read [15 25] INFO [2019-12-17 12:29:40,347] jepsen worker 47 - jepsen.util 2927 :invoke :cas [15 [20 33]] INFO [2019-12-17 12:29:40,354] jepsen worker 78 - jepsen.util 2598 :fail :cas [26 [26 13]] :ValueNotSwapFrom INFO [2019-12-17 12:29:40,354] jepsen worker 78 - jepsen.util 2598 :invoke :write [26 14] INFO [2019-12-17 12:29:40,382] jepsen worker 9 - jepsen.util 2889 :fail :cas [3 [15 45]] :ValueNotSwapFrom INFO [2019-12-17 12:29:40,382] jepsen worker 9 - jepsen.util 2889 :invoke :write [3 21] INFO [2019-12-17 12:29:40,385] jepsen worker 37 - jepsen.util 2197 :ok :write [12 41] INFO [2019-12-17 12:29:40,385] jepsen worker 37 - jepsen.util 2197 :invoke :cas [12 [2 13]] INFO [2019-12-17 12:29:40,389] jepsen worker 65 - jepsen.util 2225 :ok :read [21 12] INFO [2019-12-17 12:29:40,389] jepsen worker 65 - jepsen.util 2225 :invoke :read [21 nil] INFO [2019-12-17 12:29:40,402] jepsen worker 36 - jepsen.util 2376 :ok :read [12 41] INFO [2019-12-17 12:29:40,402] jepsen worker 36 - jepsen.util 2376 :invoke :cas [12 [20 20]] INFO [2019-12-17 12:29:40,402] jepsen worker 45 - jepsen.util 2655 :fail :cas [15 [36 23]] :ValueNotSwapFrom INFO [2019-12-17 12:29:40,403] jepsen worker 45 - jepsen.util 2655 :invoke :read [15 nil] INFO [2019-12-17 12:29:40,408] jepsen worker 50 - jepsen.util 3650 :info :write [16 49] :RequestCanceledException INFO [2019-12-17 12:29:40,409] jepsen worker 50 - jepsen.util 3740 :invoke :read [16 nil] INFO [2019-12-17 12:29:40,422] jepsen worker 43 - jepsen.util 3013 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:29:40,422] jepsen worker 80 - jepsen.util 2420 :info :write [26 27] :RequestCanceledException INFO [2019-12-17 12:29:40,423] jepsen worker 43 - jepsen.util 3013 :invoke :write [14 3] INFO [2019-12-17 12:29:40,425] jepsen worker 80 - jepsen.util 2510 :invoke :cas [26 [45 23]] INFO [2019-12-17 12:29:40,437] jepsen worker 88 - jepsen.util 2518 :ok :write [29 16] INFO [2019-12-17 12:29:40,437] jepsen worker 0 - jepsen.util 3420 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:29:40,437] jepsen worker 88 - jepsen.util 2518 :invoke :cas [29 [34 7]] INFO [2019-12-17 12:29:40,437] jepsen worker 0 - jepsen.util 3420 :invoke :write [0 44] INFO [2019-12-17 12:29:40,442] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":14450,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:29:40,442] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":14450,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:29:40,443] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":14452,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:29:40,444] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":14452,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:29:40,448] jepsen worker 65 - jepsen.util 2225 :ok :read [21 12] INFO [2019-12-17 12:29:40,448] jepsen worker 65 - jepsen.util 2225 :invoke :cas [21 [40 13]] INFO [2019-12-17 12:29:40,454] jepsen worker 59 - jepsen.util 3119 :info :cas [19 [35 27]] :RequestCanceledException INFO [2019-12-17 12:29:40,455] jepsen worker 59 - jepsen.util 3209 :invoke :cas [19 [10 25]] INFO [2019-12-17 12:29:40,460] jepsen worker 45 - jepsen.util 2655 :ok :read [15 25] INFO [2019-12-17 12:29:40,460] jepsen worker 45 - jepsen.util 2655 :invoke :cas [15 [44 35]] INFO [2019-12-17 12:29:40,462] jepsen worker 37 - jepsen.util 2197 :fail :cas [12 [2 13]] :ValueNotSwapFrom INFO [2019-12-17 12:29:40,462] jepsen worker 37 - jepsen.util 2197 :invoke :write [12 33] INFO [2019-12-17 12:29:40,467] jepsen worker 65 - jepsen.util 2225 :fail :cas [21 [40 13]] :ValueNotSwapFrom INFO [2019-12-17 12:29:40,467] jepsen worker 65 - jepsen.util 2225 :invoke :cas [21 [23 49]] INFO [2019-12-17 12:29:40,476] jepsen worker 51 - jepsen.util 2301 :ok :write [17 31] INFO [2019-12-17 12:29:40,476] jepsen worker 51 - jepsen.util 2301 :invoke :cas [17 [8 13]] INFO [2019-12-17 12:29:40,477] jepsen worker 45 - jepsen.util 2655 :fail :cas [15 [44 35]] :ValueNotSwapFrom INFO [2019-12-17 12:29:40,477] jepsen worker 45 - jepsen.util 2655 :invoke :cas [15 [38 18]] INFO [2019-12-17 12:29:40,489] jepsen worker 84 - jepsen.util 3234 :info :cas [28 [33 33]] :RequestCanceledException INFO [2019-12-17 12:29:40,490] jepsen worker 65 - jepsen.util 2225 :fail :cas [21 [23 49]] :ValueNotSwapFrom INFO [2019-12-17 12:29:40,490] jepsen worker 84 - jepsen.util 3324 :invoke :read [28 nil] INFO [2019-12-17 12:29:40,490] jepsen worker 65 - jepsen.util 2225 :invoke :write [21 33] INFO [2019-12-17 12:29:40,491] jepsen worker 38 - jepsen.util 2468 :ok :write [12 42] INFO [2019-12-17 12:29:40,491] jepsen worker 38 - jepsen.util 2468 :invoke :cas [12 [2 33]] INFO [2019-12-17 12:29:40,495] jepsen worker 29 - jepsen.util 2189 :ok :write [9 25] INFO [2019-12-17 12:29:40,495] jepsen worker 29 - jepsen.util 2189 :invoke :write [9 11] INFO [2019-12-17 12:29:40,505] jepsen worker 71 - jepsen.util 2501 :ok :read [23 4] INFO [2019-12-17 12:29:40,506] jepsen worker 71 - jepsen.util 2501 :invoke :write [23 12] INFO [2019-12-17 12:29:40,508] jepsen worker 72 - jepsen.util 2502 :info :cas [24 [27 12]] :RequestCanceledException INFO [2019-12-17 12:29:40,508] jepsen worker 1 - jepsen.util 2971 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:29:40,508] jepsen worker 62 - jepsen.util 3122 :info :cas [20 [29 48]] :RequestCanceledException INFO [2019-12-17 12:29:40,508] jepsen worker 21 - jepsen.util 3261 :info :cas [7 [28 30]] :RequestCanceledException INFO [2019-12-17 12:29:40,508] jepsen worker 1 - jepsen.util 2971 :invoke :read [0 nil] INFO [2019-12-17 12:29:40,509] jepsen worker 72 - jepsen.util 2592 :invoke :write [24 28] INFO [2019-12-17 12:29:40,509] jepsen worker 62 - jepsen.util 3212 :invoke :cas [20 [41 39]] INFO [2019-12-17 12:29:40,509] jepsen worker 21 - jepsen.util 3351 :invoke :read [7 nil] INFO [2019-12-17 12:29:40,523] jepsen worker 24 - jepsen.util 3084 :info :cas [8 [37 6]] :RequestCanceledException INFO [2019-12-17 12:29:40,523] jepsen worker 85 - jepsen.util 3595 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:29:40,523] jepsen worker 57 - jepsen.util 3027 :info :write [19 26] :RequestCanceledException INFO [2019-12-17 12:29:40,523] jepsen worker 85 - jepsen.util 3595 :invoke :cas [28 [27 9]] INFO [2019-12-17 12:29:40,524] jepsen worker 57 - jepsen.util 3117 :invoke :read [19 nil] INFO [2019-12-17 12:29:40,524] jepsen worker 24 - jepsen.util 3174 :invoke :write [8 16] INFO [2019-12-17 12:29:40,527] jepsen worker 70 - jepsen.util 2950 :fail :cas [23 [12 49]] :ValueNotSwapFrom INFO [2019-12-17 12:29:40,527] jepsen worker 70 - jepsen.util 2950 :invoke :read [23 nil] INFO [2019-12-17 12:29:40,536] jepsen worker 58 - jepsen.util 3028 :info :cas [19 [40 25]] :RequestCanceledException INFO [2019-12-17 12:29:40,538] jepsen worker 58 - jepsen.util 3118 :invoke :cas [19 [24 4]] INFO [2019-12-17 12:29:40,552] jepsen worker 52 - jepsen.util 2662 :ok :read [17 31] INFO [2019-12-17 12:29:40,552] jepsen worker 52 - jepsen.util 2662 :invoke :write [17 10] INFO [2019-12-17 12:29:40,563] jepsen worker 38 - jepsen.util 2468 :fail :cas [12 [2 33]] :ValueNotSwapFrom INFO [2019-12-17 12:29:40,563] jepsen worker 38 - jepsen.util 2468 :invoke :write [12 18] INFO [2019-12-17 12:29:40,569] jepsen worker 11 - jepsen.util 3071 :fail :cas [3 [23 25]] :ValueNotSwapFrom INFO [2019-12-17 12:29:40,569] jepsen worker 11 - jepsen.util 3071 :invoke :read [3 nil] INFO [2019-12-17 12:29:40,587] jepsen worker 70 - jepsen.util 2950 :ok :read [23 4] INFO [2019-12-17 12:29:40,588] jepsen worker 70 - jepsen.util 2950 :invoke :write [23 39] INFO [2019-12-17 12:29:40,589] jepsen worker 83 - jepsen.util 3683 :info :cas [27 [11 24]] :RequestCanceledException INFO [2019-12-17 12:29:40,589] jepsen worker 75 - jepsen.util 3495 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:29:40,590] jepsen worker 75 - jepsen.util 3495 :invoke :read [25 nil] INFO [2019-12-17 12:29:40,591] jepsen worker 36 - jepsen.util 2376 :fail :cas [12 [20 20]] :ValueNotSwapFrom INFO [2019-12-17 12:29:40,591] jepsen worker 83 - jepsen.util 3773 :invoke :cas [27 [4 42]] INFO [2019-12-17 12:29:40,591] jepsen worker 36 - jepsen.util 2376 :invoke :read [12 nil] INFO [2019-12-17 12:29:40,607] jepsen worker 81 - jepsen.util 2961 :info :cas [27 [5 30]] :RequestCanceledException INFO [2019-12-17 12:29:40,608] jepsen worker 81 - jepsen.util 3051 :invoke :write [27 10] INFO [2019-12-17 12:29:40,623] jepsen worker 25 - jepsen.util 3715 :info :cas [8 [37 41]] :RequestCanceledException INFO [2019-12-17 12:29:40,623] jepsen worker 86 - jepsen.util 3236 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:29:40,623] jepsen worker 76 - jepsen.util 3316 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:29:40,623] jepsen worker 86 - jepsen.util 3236 :invoke :cas [28 [42 36]] INFO [2019-12-17 12:29:40,623] jepsen worker 76 - jepsen.util 3316 :invoke :cas [25 [9 48]] INFO [2019-12-17 12:29:40,624] jepsen worker 25 - jepsen.util 3805 :invoke :read [8 nil] INFO [2019-12-17 12:29:40,638] jepsen worker 41 - jepsen.util 3551 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:29:40,638] jepsen worker 41 - jepsen.util 3551 :invoke :cas [13 [18 1]] INFO [2019-12-17 12:29:40,645] jepsen worker 3 - jepsen.util 2433 :ok :write [1 39] INFO [2019-12-17 12:29:40,645] jepsen worker 3 - jepsen.util 2433 :invoke :write [1 1] INFO [2019-12-17 12:29:40,654] jepsen worker 22 - jepsen.util 3172 :info :write [7 3] :RequestCanceledException INFO [2019-12-17 12:29:40,655] jepsen worker 22 - jepsen.util 3262 :invoke :read [7 nil] INFO [2019-12-17 12:29:40,660] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:29:40,660] jepsen nemesis - couchbase.util Rebalance check count: 104 INFO [2019-12-17 12:29:40,660] jepsen nemesis - couchbase.util Rebalance stuck count: 85 INFO [2019-12-17 12:29:40,665] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":14494,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:29:40,665] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":14494,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:29:40,665] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":14498,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:29:40,665] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":14498,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:29:40,680] jepsen worker 78 - jepsen.util 2598 :ok :write [26 14] INFO [2019-12-17 12:29:40,681] jepsen worker 78 - jepsen.util 2598 :invoke :cas [26 [9 20]] INFO [2019-12-17 12:29:40,682] jepsen worker 45 - jepsen.util 2655 :fail :cas [15 [38 18]] :ValueNotSwapFrom INFO [2019-12-17 12:29:40,682] jepsen worker 45 - jepsen.util 2655 :invoke :write [15 36] INFO [2019-12-17 12:29:40,687] jepsen worker 42 - jepsen.util 3912 :info :write [14 23] :RequestCanceledException INFO [2019-12-17 12:29:40,688] jepsen worker 42 - jepsen.util 4002 :invoke :write [14 37] INFO [2019-12-17 12:29:40,702] jepsen worker 29 - jepsen.util 2189 :ok :write [9 11] INFO [2019-12-17 12:29:40,702] jepsen worker 29 - jepsen.util 2189 :invoke :cas [9 [34 26]] INFO [2019-12-17 12:29:40,704] jepsen worker 79 - jepsen.util 2509 :ok :read [26 14] INFO [2019-12-17 12:29:40,704] jepsen worker 79 - jepsen.util 2509 :invoke :write [26 20] INFO [2019-12-17 12:29:40,705] jepsen worker 40 - jepsen.util 3550 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:29:40,706] jepsen worker 40 - jepsen.util 3550 :invoke :write [13 37] INFO [2019-12-17 12:29:40,723] jepsen worker 26 - jepsen.util 3536 :info :cas [8 [37 44]] :RequestCanceledException INFO [2019-12-17 12:29:40,723] jepsen worker 48 - jepsen.util 2658 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:29:40,723] jepsen worker 48 - jepsen.util 2658 :invoke :cas [16 [8 34]] INFO [2019-12-17 12:29:40,724] jepsen worker 26 - jepsen.util 3626 :invoke :read [8 nil] INFO [2019-12-17 12:29:40,738] jepsen worker 63 - jepsen.util 2133 :ok :write [21 48] INFO [2019-12-17 12:29:40,738] jepsen worker 82 - jepsen.util 3862 :info :write [27 40] :RequestCanceledException INFO [2019-12-17 12:29:40,738] jepsen worker 33 - jepsen.util 3543 :info :cas [11 [33 19]] :RequestCanceledException INFO [2019-12-17 12:29:40,738] jepsen worker 63 - jepsen.util 2133 :invoke :write [21 6] INFO [2019-12-17 12:29:40,739] jepsen worker 82 - jepsen.util 3952 :invoke :cas [27 [27 41]] INFO [2019-12-17 12:29:40,739] jepsen worker 33 - jepsen.util 3633 :invoke :cas [11 [26 37]] INFO [2019-12-17 12:29:40,739] jepsen worker 29 - jepsen.util 2189 :fail :cas [9 [34 26]] :ValueNotSwapFrom INFO [2019-12-17 12:29:40,740] jepsen worker 29 - jepsen.util 2189 :invoke :write [9 5] INFO [2019-12-17 12:29:40,750] jepsen worker 52 - jepsen.util 2662 :ok :write [17 10] INFO [2019-12-17 12:29:40,750] jepsen worker 52 - jepsen.util 2662 :invoke :read [17 nil] INFO [2019-12-17 12:29:40,752] jepsen worker 77 - jepsen.util 3227 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:29:40,753] jepsen worker 77 - jepsen.util 3227 :invoke :read [25 nil] INFO [2019-12-17 12:29:40,771] jepsen worker 51 - jepsen.util 2301 :fail :cas [17 [8 13]] :ValueNotSwapFrom INFO [2019-12-17 12:29:40,771] jepsen worker 51 - jepsen.util 2301 :invoke :write [17 45] INFO [2019-12-17 12:29:40,802] jepsen worker 37 - jepsen.util 2197 :ok :write [12 33] INFO [2019-12-17 12:29:40,803] jepsen worker 37 - jepsen.util 2197 :invoke :write [12 32] INFO [2019-12-17 12:29:40,807] jepsen worker 8 - jepsen.util 2978 :info :cas [2 [10 28]] :RequestCanceledException INFO [2019-12-17 12:29:40,808] jepsen worker 8 - jepsen.util 3068 :invoke :read [2 nil] INFO [2019-12-17 12:29:40,810] jepsen worker 3 - jepsen.util 2433 :ok :write [1 1] INFO [2019-12-17 12:29:40,810] jepsen worker 3 - jepsen.util 2433 :invoke :read [1 nil] INFO [2019-12-17 12:29:40,820] jepsen worker 66 - jepsen.util 3306 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:29:40,820] jepsen worker 66 - jepsen.util 3306 :invoke :cas [22 [25 12]] INFO [2019-12-17 12:29:40,834] jepsen worker 88 - jepsen.util 2518 :fail :cas [29 [34 7]] :ValueNotSwapFrom INFO [2019-12-17 12:29:40,834] jepsen worker 88 - jepsen.util 2518 :invoke :cas [29 [33 11]] INFO [2019-12-17 12:29:40,836] jepsen worker 17 - jepsen.util 3167 :info :cas [5 [1 15]] :RequestCanceledException INFO [2019-12-17 12:29:40,836] jepsen worker 68 - jepsen.util 2948 :info :write [22 23] :RequestCanceledException INFO [2019-12-17 12:29:40,837] jepsen worker 17 - jepsen.util 3257 :invoke :write [5 2] INFO [2019-12-17 12:29:40,837] jepsen worker 68 - jepsen.util 3038 :invoke :write [22 48] INFO [2019-12-17 12:29:40,840] jepsen worker 56 - jepsen.util 2396 :ok :write [18 5] INFO [2019-12-17 12:29:40,840] jepsen worker 56 - jepsen.util 2396 :invoke :cas [18 [14 27]] INFO [2019-12-17 12:29:40,871] jepsen worker 63 - jepsen.util 2133 :ok :write [21 6] INFO [2019-12-17 12:29:40,871] jepsen worker 63 - jepsen.util 2133 :invoke :read [21 nil] INFO [2019-12-17 12:29:40,899] jepsen worker 12 - jepsen.util 2622 :ok :read [4 0] INFO [2019-12-17 12:29:40,899] jepsen worker 12 - jepsen.util 2622 :invoke :cas [4 [0 30]] INFO [2019-12-17 12:29:40,900] jepsen worker 29 - jepsen.util 2189 :ok :write [9 5] INFO [2019-12-17 12:29:40,900] jepsen worker 29 - jepsen.util 2189 :invoke :cas [9 [11 40]] INFO [2019-12-17 12:29:40,902] jepsen worker 32 - jepsen.util 3002 :ok :write [10 19] INFO [2019-12-17 12:29:40,902] jepsen worker 32 - jepsen.util 3002 :invoke :read [10 nil] INFO [2019-12-17 12:29:40,902] jepsen worker 63 - jepsen.util 2133 :ok :read [21 6] INFO [2019-12-17 12:29:40,902] jepsen worker 63 - jepsen.util 2133 :invoke :read [21 nil] INFO [2019-12-17 12:29:40,905] jepsen worker 10 - jepsen.util 2350 :ok :write [3 12] INFO [2019-12-17 12:29:40,905] jepsen worker 10 - jepsen.util 2350 :invoke :read [3 nil] INFO [2019-12-17 12:29:40,907] jepsen worker 53 - jepsen.util 2483 :ok :write [17 3] INFO [2019-12-17 12:29:40,907] jepsen worker 53 - jepsen.util 2483 :invoke :read [17 nil] INFO [2019-12-17 12:29:40,922] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":14528,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:29:40,923] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":14528,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:29:40,940] jepsen worker 29 - jepsen.util 2189 :fail :cas [9 [11 40]] :ValueNotSwapFrom INFO [2019-12-17 12:29:40,941] jepsen worker 29 - jepsen.util 2189 :invoke :write [9 42] INFO [2019-12-17 12:29:40,954] jepsen worker 39 - jepsen.util 3369 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:29:40,954] jepsen worker 39 - jepsen.util 3369 :invoke :read [13 nil] INFO [2019-12-17 12:29:40,964] jepsen worker 13 - jepsen.util 2263 :fail :cas [4 [19 17]] :ValueNotSwapFrom INFO [2019-12-17 12:29:40,964] jepsen worker 13 - jepsen.util 2263 :invoke :cas [4 [4 46]] INFO [2019-12-17 12:29:40,967] jepsen worker 63 - jepsen.util 2133 :ok :read [21 6] INFO [2019-12-17 12:29:40,967] jepsen worker 63 - jepsen.util 2133 :invoke :cas [21 [5 20]] INFO [2019-12-17 12:29:40,971] jepsen worker 47 - jepsen.util 2927 :fail :cas [15 [20 33]] :ValueNotSwapFrom INFO [2019-12-17 12:29:40,971] jepsen worker 47 - jepsen.util 2927 :invoke :read [15 nil] INFO [2019-12-17 12:29:41,007] jepsen worker 20 - jepsen.util 2180 :info :write [6 29] :RequestCanceledException INFO [2019-12-17 12:29:41,008] jepsen worker 20 - jepsen.util 2270 :invoke :write [6 1] INFO [2019-12-17 12:29:41,015] jepsen worker 36 - jepsen.util 2376 :ok :read [12 33] INFO [2019-12-17 12:29:41,015] jepsen worker 36 - jepsen.util 2376 :invoke :read [12 nil] INFO [2019-12-17 12:29:41,023] jepsen worker 46 - jepsen.util 2746 :ok :write [15 32] INFO [2019-12-17 12:29:41,023] jepsen worker 46 - jepsen.util 2746 :invoke :read [15 nil] INFO [2019-12-17 12:29:41,023] jepsen worker 2 - jepsen.util 3242 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:29:41,023] jepsen worker 2 - jepsen.util 3242 :invoke :cas [0 [24 15]] INFO [2019-12-17 12:29:41,032] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":14543,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:29:41,032] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":14543,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:29:41,032] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":14546,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:29:41,032] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":14546,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:29:41,032] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":14547,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:29:41,032] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":14547,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:29:41,035] jepsen worker 55 - jepsen.util 2935 :ok :read [18 5] INFO [2019-12-17 12:29:41,035] jepsen worker 55 - jepsen.util 2935 :invoke :read [18 nil] INFO [2019-12-17 12:29:41,036] jepsen worker 14 - jepsen.util 2354 :ok :read [4 0] INFO [2019-12-17 12:29:41,036] jepsen worker 14 - jepsen.util 2354 :invoke :write [4 19] INFO [2019-12-17 12:29:41,040] jepsen worker 63 - jepsen.util 2133 :fail :cas [21 [5 20]] :ValueNotSwapFrom INFO [2019-12-17 12:29:41,040] jepsen worker 63 - jepsen.util 2133 :invoke :write [21 25] INFO [2019-12-17 12:29:41,065] jepsen worker 46 - jepsen.util 2746 :ok :read [15 32] INFO [2019-12-17 12:29:41,065] jepsen worker 46 - jepsen.util 2746 :invoke :write [15 37] INFO [2019-12-17 12:29:41,106] jepsen worker 29 - jepsen.util 2189 :ok :write [9 42] INFO [2019-12-17 12:29:41,106] jepsen worker 29 - jepsen.util 2189 :invoke :write [9 26] INFO [2019-12-17 12:29:41,122] jepsen worker 4 - jepsen.util 1894 :fail :cas [1 [34 12]] :ValueNotSwapFrom INFO [2019-12-17 12:29:41,122] jepsen worker 4 - jepsen.util 1894 :invoke :cas [1 [18 31]] INFO [2019-12-17 12:29:41,123] jepsen worker 61 - jepsen.util 3571 :info :write [20 36] :RequestCanceledException INFO [2019-12-17 12:29:41,124] jepsen worker 61 - jepsen.util 3661 :invoke :write [20 8] INFO [2019-12-17 12:29:41,141] jepsen worker 80 - jepsen.util 2510 :fail :cas [26 [45 23]] :ValueNotSwapFrom INFO [2019-12-17 12:29:41,141] jepsen worker 80 - jepsen.util 2510 :invoke :cas [26 [33 12]] INFO [2019-12-17 12:29:41,162] jepsen worker 29 - jepsen.util 2189 :ok :write [9 26] INFO [2019-12-17 12:29:41,162] jepsen worker 29 - jepsen.util 2189 :invoke :cas [9 [4 27]] INFO [2019-12-17 12:29:41,177] jepsen worker 29 - jepsen.util 2189 :fail :cas [9 [4 27]] :ValueNotSwapFrom INFO [2019-12-17 12:29:41,177] jepsen worker 29 - jepsen.util 2189 :invoke :write [9 25] INFO [2019-12-17 12:29:41,198] jepsen worker 87 - jepsen.util 2337 :ok :read [29 16] INFO [2019-12-17 12:29:41,198] jepsen worker 87 - jepsen.util 2337 :invoke :write [29 24] INFO [2019-12-17 12:29:41,209] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":14563,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:29:41,210] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":14563,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:29:41,210] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":14564,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:29:41,210] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":14564,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:29:41,221] jepsen worker 89 - jepsen.util 2429 :ok :read [29 16] INFO [2019-12-17 12:29:41,221] jepsen worker 89 - jepsen.util 2429 :invoke :write [29 29] INFO [2019-12-17 12:29:41,227] jepsen worker 29 - jepsen.util 2189 :ok :write [9 25] INFO [2019-12-17 12:29:41,227] jepsen worker 29 - jepsen.util 2189 :invoke :read [9 nil] INFO [2019-12-17 12:29:41,247] jepsen worker 29 - jepsen.util 2189 :ok :read [9 25] INFO [2019-12-17 12:29:41,248] jepsen worker 29 - jepsen.util 2189 :invoke :write [9 48] INFO [2019-12-17 12:29:41,249] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":14571,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:29:41,250] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":14571,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:29:41,254] jepsen worker 53 - jepsen.util 2483 :ok :read [17 3] INFO [2019-12-17 12:29:41,254] jepsen worker 53 - jepsen.util 2483 :invoke :cas [17 [44 35]] INFO [2019-12-17 12:29:41,264] jepsen worker 56 - jepsen.util 2396 :fail :cas [18 [14 27]] :ValueNotSwapFrom INFO [2019-12-17 12:29:41,264] jepsen worker 56 - jepsen.util 2396 :invoke :read [18 nil] INFO [2019-12-17 12:29:41,287] jepsen worker 29 - jepsen.util 2189 :ok :write [9 48] INFO [2019-12-17 12:29:41,287] jepsen worker 29 - jepsen.util 2189 :invoke :write [9 24] INFO [2019-12-17 12:29:41,304] jepsen worker 74 - jepsen.util 2414 :fail :read [24 nil] :RequestTimeoutException INFO [2019-12-17 12:29:41,305] jepsen worker 74 - jepsen.util 2414 :invoke :read [24 nil] INFO [2019-12-17 12:29:41,310] jepsen worker 55 - jepsen.util 2935 :ok :read [18 5] INFO [2019-12-17 12:29:41,311] jepsen worker 55 - jepsen.util 2935 :invoke :cas [18 [2 34]] INFO [2019-12-17 12:29:41,344] jepsen worker 29 - jepsen.util 2189 :ok :write [9 24] INFO [2019-12-17 12:29:41,344] jepsen worker 29 - jepsen.util 2189 :invoke :read [9 nil] INFO [2019-12-17 12:29:41,361] jepsen worker 29 - jepsen.util 2189 :ok :read [9 24] INFO [2019-12-17 12:29:41,361] jepsen worker 29 - jepsen.util 2189 :invoke :write [9 6] INFO [2019-12-17 12:29:41,404] jepsen worker 29 - jepsen.util 2189 :ok :write [9 6] INFO [2019-12-17 12:29:41,404] jepsen worker 29 - jepsen.util 2189 :invoke :read [9 nil] INFO [2019-12-17 12:29:41,423] jepsen worker 29 - jepsen.util 2189 :ok :read [9 6] INFO [2019-12-17 12:29:41,423] jepsen worker 29 - jepsen.util 2189 :invoke :cas [9 [8 27]] INFO [2019-12-17 12:29:41,428] jepsen worker 80 - jepsen.util 2510 :fail :cas [26 [33 12]] :ValueNotSwapFrom INFO [2019-12-17 12:29:41,429] jepsen worker 80 - jepsen.util 2510 :invoke :read [26 nil] INFO [2019-12-17 12:29:41,429] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":14590,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:29:41,429] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":14590,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:29:41,429] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":14593,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:29:41,429] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":14593,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:29:41,437] jepsen worker 13 - jepsen.util 2263 :fail :cas [4 [4 46]] :ValueNotSwapFrom INFO [2019-12-17 12:29:41,438] jepsen worker 13 - jepsen.util 2263 :invoke :write [4 32] INFO [2019-12-17 12:29:41,440] jepsen worker 29 - jepsen.util 2189 :fail :cas [9 [8 27]] :ValueNotSwapFrom INFO [2019-12-17 12:29:41,440] jepsen worker 29 - jepsen.util 2189 :invoke :read [9 nil] INFO [2019-12-17 12:29:41,442] jepsen worker 54 - jepsen.util 2484 :ok :write [18 22] INFO [2019-12-17 12:29:41,442] jepsen worker 54 - jepsen.util 2484 :invoke :write [18 29] INFO [2019-12-17 12:29:41,447] jepsen worker 53 - jepsen.util 2483 :fail :cas [17 [44 35]] :ValueNotSwapFrom INFO [2019-12-17 12:29:41,447] jepsen worker 53 - jepsen.util 2483 :invoke :read [17 nil] INFO [2019-12-17 12:29:41,460] jepsen worker 71 - jepsen.util 2501 :ok :write [23 12] INFO [2019-12-17 12:29:41,461] jepsen worker 71 - jepsen.util 2501 :invoke :cas [23 [26 11]] INFO [2019-12-17 12:29:41,463] jepsen worker 29 - jepsen.util 2189 :ok :read [9 6] INFO [2019-12-17 12:29:41,463] jepsen worker 29 - jepsen.util 2189 :invoke :write [9 16] INFO [2019-12-17 12:29:41,472] jepsen worker 55 - jepsen.util 2935 :fail :cas [18 [2 34]] :ValueNotSwapFrom INFO [2019-12-17 12:29:41,472] jepsen worker 55 - jepsen.util 2935 :invoke :cas [18 [10 36]] INFO [2019-12-17 12:29:41,500] jepsen worker 37 - jepsen.util 2197 :ok :write [12 32] INFO [2019-12-17 12:29:41,500] jepsen worker 37 - jepsen.util 2197 :invoke :cas [12 [6 37]] INFO [2019-12-17 12:29:41,508] jepsen worker 80 - jepsen.util 2510 :ok :read [26 14] INFO [2019-12-17 12:29:41,509] jepsen worker 80 - jepsen.util 2510 :invoke :write [26 36] INFO [2019-12-17 12:29:41,515] jepsen worker 29 - jepsen.util 2189 :ok :write [9 16] INFO [2019-12-17 12:29:41,515] jepsen worker 29 - jepsen.util 2189 :invoke :write [9 3] INFO [2019-12-17 12:29:41,560] jepsen worker 14 - jepsen.util 2354 :ok :write [4 19] INFO [2019-12-17 12:29:41,561] jepsen worker 14 - jepsen.util 2354 :invoke :cas [4 [5 22]] INFO [2019-12-17 12:29:41,571] jepsen worker 36 - jepsen.util 2376 :ok :read [12 32] INFO [2019-12-17 12:29:41,571] jepsen worker 36 - jepsen.util 2376 :invoke :cas [12 [29 32]] INFO [2019-12-17 12:29:41,573] jepsen worker 29 - jepsen.util 2189 :ok :write [9 3] INFO [2019-12-17 12:29:41,573] jepsen worker 29 - jepsen.util 2189 :invoke :read [9 nil] INFO [2019-12-17 12:29:41,583] jepsen worker 53 - jepsen.util 2483 :ok :read [17 3] INFO [2019-12-17 12:29:41,583] jepsen worker 53 - jepsen.util 2483 :invoke :cas [17 [14 14]] INFO [2019-12-17 12:29:41,590] jepsen worker 29 - jepsen.util 2189 :ok :read [9 3] INFO [2019-12-17 12:29:41,590] jepsen worker 29 - jepsen.util 2189 :invoke :write [9 34] INFO [2019-12-17 12:29:41,611] jepsen worker 88 - jepsen.util 2518 :fail :cas [29 [33 11]] :ValueNotSwapFrom INFO [2019-12-17 12:29:41,612] jepsen worker 88 - jepsen.util 2518 :invoke :read [29 nil] INFO [2019-12-17 12:29:41,623] jepsen worker 67 - jepsen.util 3667 :info :cas [22 [14 30]] :RequestCanceledException INFO [2019-12-17 12:29:41,623] jepsen worker 19 - jepsen.util 2629 :fail :read [6 nil] :RequestTimeoutException INFO [2019-12-17 12:29:41,623] jepsen worker 44 - jepsen.util 3104 :info :cas [14 [45 35]] :RequestCanceledException INFO [2019-12-17 12:29:41,623] jepsen worker 19 - jepsen.util 2629 :invoke :read [6 nil] INFO [2019-12-17 12:29:41,624] jepsen worker 67 - jepsen.util 3757 :invoke :write [22 31] INFO [2019-12-17 12:29:41,624] jepsen worker 44 - jepsen.util 3194 :invoke :cas [14 [7 49]] INFO [2019-12-17 12:29:41,637] jepsen worker 6 - jepsen.util 3426 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:29:41,637] jepsen worker 6 - jepsen.util 3426 :invoke :cas [2 [8 16]] INFO [2019-12-17 12:29:41,640] jepsen worker 29 - jepsen.util 2189 :ok :write [9 34] INFO [2019-12-17 12:29:41,640] jepsen worker 29 - jepsen.util 2189 :invoke :cas [9 [25 6]] INFO [2019-12-17 12:29:41,648] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":14618,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:29:41,648] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":14618,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:29:41,649] jepsen worker 51 - jepsen.util 2301 :ok :write [17 45] INFO [2019-12-17 12:29:41,649] jepsen worker 51 - jepsen.util 2301 :invoke :cas [17 [49 35]] INFO [2019-12-17 12:29:41,653] jepsen worker 55 - jepsen.util 2935 :fail :cas [18 [10 36]] :ValueNotSwapFrom INFO [2019-12-17 12:29:41,654] jepsen worker 55 - jepsen.util 2935 :invoke :read [18 nil] INFO [2019-12-17 12:29:41,659] jepsen worker 29 - jepsen.util 2189 :fail :cas [9 [25 6]] :ValueNotSwapFrom INFO [2019-12-17 12:29:41,659] jepsen worker 29 - jepsen.util 2189 :invoke :write [9 8] INFO [2019-12-17 12:29:41,664] jepsen worker 9 - jepsen.util 2889 :ok :write [3 21] INFO [2019-12-17 12:29:41,664] jepsen worker 9 - jepsen.util 2889 :invoke :cas [3 [35 38]] INFO [2019-12-17 12:29:41,670] jepsen worker 78 - jepsen.util 2598 :fail :cas [26 [9 20]] :ValueNotSwapFrom INFO [2019-12-17 12:29:41,670] jepsen worker 78 - jepsen.util 2598 :invoke :cas [26 [46 31]] INFO [2019-12-17 12:29:41,685] jepsen worker 34 - jepsen.util 3364 :info :cas [11 [36 33]] :RequestCanceledException INFO [2019-12-17 12:29:41,685] jepsen worker 60 - jepsen.util 3570 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:29:41,686] jepsen worker 60 - jepsen.util 3570 :invoke :read [20 nil] INFO [2019-12-17 12:29:41,686] jepsen worker 34 - jepsen.util 3454 :invoke :write [11 42] INFO [2019-12-17 12:29:41,695] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:29:41,695] jepsen nemesis - couchbase.util Rebalance check count: 105 INFO [2019-12-17 12:29:41,696] jepsen nemesis - couchbase.util Rebalance stuck count: 86 INFO [2019-12-17 12:29:41,696] jepsen worker 9 - jepsen.util 2889 :fail :cas [3 [35 38]] :ValueNotSwapFrom INFO [2019-12-17 12:29:41,696] jepsen worker 9 - jepsen.util 2889 :invoke :cas [3 [39 31]] INFO [2019-12-17 12:29:41,707] jepsen worker 9 - jepsen.util 2889 :fail :cas [3 [39 31]] :ValueNotSwapFrom INFO [2019-12-17 12:29:41,707] jepsen worker 9 - jepsen.util 2889 :invoke :read [3 nil] INFO [2019-12-17 12:29:41,707] jepsen worker 49 - jepsen.util 2569 :info :cas [16 [15 18]] :RequestCanceledException INFO [2019-12-17 12:29:41,708] jepsen worker 49 - jepsen.util 2659 :invoke :cas [16 [5 0]] INFO [2019-12-17 12:29:41,722] jepsen worker 23 - jepsen.util 3083 :info :write [7 18] :RequestCanceledException INFO [2019-12-17 12:29:41,723] jepsen worker 23 - jepsen.util 3173 :invoke :write [7 11] INFO [2019-12-17 12:29:41,724] jepsen worker 7 - jepsen.util 3157 :info :write [2 14] :RequestCanceledException INFO [2019-12-17 12:29:41,724] jepsen worker 3 - jepsen.util 2433 :ok :read [1 1] INFO [2019-12-17 12:29:41,724] jepsen worker 3 - jepsen.util 2433 :invoke :cas [1 [45 9]] INFO [2019-12-17 12:29:41,725] jepsen worker 7 - jepsen.util 3247 :invoke :cas [2 [10 20]] INFO [2019-12-17 12:29:41,727] jepsen worker 9 - jepsen.util 2889 :ok :read [3 21] INFO [2019-12-17 12:29:41,727] jepsen worker 9 - jepsen.util 2889 :invoke :write [3 23] INFO [2019-12-17 12:29:41,740] jepsen worker 37 - jepsen.util 2197 :fail :cas [12 [6 37]] :ValueNotSwapFrom INFO [2019-12-17 12:29:41,740] jepsen worker 37 - jepsen.util 2197 :invoke :cas [12 [13 0]] INFO [2019-12-17 12:29:41,742] jepsen worker 29 - jepsen.util 2189 :ok :write [9 8] INFO [2019-12-17 12:29:41,742] jepsen worker 29 - jepsen.util 2189 :invoke :read [9 nil] INFO [2019-12-17 12:29:41,754] jepsen worker 29 - jepsen.util 2189 :ok :read [9 8] INFO [2019-12-17 12:29:41,754] jepsen worker 29 - jepsen.util 2189 :invoke :write [9 45] INFO [2019-12-17 12:29:41,800] jepsen worker 87 - jepsen.util 2337 :ok :write [29 24] INFO [2019-12-17 12:29:41,800] jepsen worker 87 - jepsen.util 2337 :invoke :cas [29 [14 27]] INFO [2019-12-17 12:29:41,813] jepsen worker 29 - jepsen.util 2189 :ok :write [9 45] INFO [2019-12-17 12:29:41,813] jepsen worker 29 - jepsen.util 2189 :invoke :write [9 43] INFO [2019-12-17 12:29:41,821] jepsen worker 80 - jepsen.util 2510 :ok :write [26 36] INFO [2019-12-17 12:29:41,821] jepsen worker 80 - jepsen.util 2510 :invoke :cas [26 [14 33]] INFO [2019-12-17 12:29:41,822] jepsen worker 69 - jepsen.util 2319 :info :write [23 10] :RequestCanceledException INFO [2019-12-17 12:29:41,824] jepsen worker 69 - jepsen.util 2409 :invoke :write [23 13] INFO [2019-12-17 12:29:41,826] jepsen worker 36 - jepsen.util 2376 :fail :cas [12 [29 32]] :ValueNotSwapFrom INFO [2019-12-17 12:29:41,826] jepsen worker 36 - jepsen.util 2376 :invoke :read [12 nil] INFO [2019-12-17 12:29:41,851] jepsen worker 27 - jepsen.util 2187 :info :write [9 27] :RequestCanceledException INFO [2019-12-17 12:29:41,852] jepsen worker 53 - jepsen.util 2483 :fail :cas [17 [14 14]] :ValueNotSwapFrom INFO [2019-12-17 12:29:41,852] jepsen worker 53 - jepsen.util 2483 :invoke :cas [17 [8 30]] INFO [2019-12-17 12:29:41,852] jepsen worker 27 - jepsen.util 2277 :invoke :read [9 nil] INFO [2019-12-17 12:29:41,877] jepsen worker 37 - jepsen.util 2197 :fail :cas [12 [13 0]] :ValueNotSwapFrom INFO [2019-12-17 12:29:41,878] jepsen worker 37 - jepsen.util 2197 :invoke :cas [12 [16 26]] INFO [2019-12-17 12:29:41,883] jepsen worker 55 - jepsen.util 2935 :ok :read [18 22] INFO [2019-12-17 12:29:41,883] jepsen worker 55 - jepsen.util 2935 :invoke :write [18 7] INFO [2019-12-17 12:29:41,904] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":14647,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:29:41,904] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":14647,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:29:41,905] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":14651,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:29:41,905] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":14651,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:29:41,914] jepsen worker 4 - jepsen.util 1894 :fail :cas [1 [18 31]] :ValueNotSwapFrom INFO [2019-12-17 12:29:41,915] jepsen worker 4 - jepsen.util 1894 :invoke :cas [1 [0 30]] INFO [2019-12-17 12:29:41,930] jepsen worker 36 - jepsen.util 2376 :ok :read [12 32] INFO [2019-12-17 12:29:41,930] jepsen worker 36 - jepsen.util 2376 :invoke :read [12 nil] INFO [2019-12-17 12:29:41,941] jepsen worker 27 - jepsen.util 2277 :ok :read [9 45] INFO [2019-12-17 12:29:41,941] jepsen worker 27 - jepsen.util 2277 :invoke :cas [9 [48 11]] INFO [2019-12-17 12:29:41,942] jepsen worker 5 - jepsen.util 2075 :ok :write [1 7] INFO [2019-12-17 12:29:41,942] jepsen worker 5 - jepsen.util 2075 :invoke :write [1 2] INFO [2019-12-17 12:29:41,945] jepsen worker 29 - jepsen.util 2189 :ok :write [9 43] INFO [2019-12-17 12:29:41,945] jepsen worker 29 - jepsen.util 2189 :invoke :cas [9 [9 17]] INFO [2019-12-17 12:29:41,956] jepsen worker 70 - jepsen.util 2950 :ok :write [23 39] INFO [2019-12-17 12:29:41,956] jepsen worker 70 - jepsen.util 2950 :invoke :write [23 28] INFO [2019-12-17 12:29:41,961] jepsen worker 80 - jepsen.util 2510 :fail :cas [26 [14 33]] :ValueNotSwapFrom INFO [2019-12-17 12:29:41,961] jepsen worker 80 - jepsen.util 2510 :invoke :cas [26 [34 0]] INFO [2019-12-17 12:29:41,970] jepsen worker 27 - jepsen.util 2277 :fail :cas [9 [48 11]] :ValueNotSwapFrom INFO [2019-12-17 12:29:41,970] jepsen worker 27 - jepsen.util 2277 :invoke :cas [9 [17 22]] INFO [2019-12-17 12:29:41,970] jepsen worker 29 - jepsen.util 2189 :fail :cas [9 [9 17]] :ValueNotSwapFrom INFO [2019-12-17 12:29:41,970] jepsen worker 29 - jepsen.util 2189 :invoke :cas [9 [17 36]] INFO [2019-12-17 12:29:41,986] jepsen worker 71 - jepsen.util 2501 :fail :cas [23 [26 11]] :ValueNotSwapFrom INFO [2019-12-17 12:29:41,987] jepsen worker 71 - jepsen.util 2501 :invoke :read [23 nil] INFO [2019-12-17 12:29:42,007] jepsen worker 30 - jepsen.util 3000 :fail :read [10 nil] :RequestTimeoutException INFO [2019-12-17 12:29:42,007] jepsen worker 30 - jepsen.util 3000 :invoke :write [10 19] INFO [2019-12-17 12:29:42,014] jepsen worker 29 - jepsen.util 2189 :fail :cas [9 [17 36]] :ValueNotSwapFrom INFO [2019-12-17 12:29:42,014] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":14667,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:29:42,014] jepsen worker 29 - jepsen.util 2189 :invoke :read [9 nil] INFO [2019-12-17 12:29:42,014] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":14667,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:29:42,014] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":14669,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:29:42,014] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":14669,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:29:42,015] jepsen worker 27 - jepsen.util 2277 :fail :cas [9 [17 22]] :ValueNotSwapFrom INFO [2019-12-17 12:29:42,015] jepsen worker 27 - jepsen.util 2277 :invoke :cas [9 [46 37]] INFO [2019-12-17 12:29:42,016] jepsen worker 51 - jepsen.util 2301 :fail :cas [17 [49 35]] :ValueNotSwapFrom INFO [2019-12-17 12:29:42,016] jepsen worker 51 - jepsen.util 2301 :invoke :write [17 22] INFO [2019-12-17 12:29:42,028] jepsen worker 37 - jepsen.util 2197 :fail :cas [12 [16 26]] :ValueNotSwapFrom INFO [2019-12-17 12:29:42,028] jepsen worker 37 - jepsen.util 2197 :invoke :write [12 15] INFO [2019-12-17 12:29:42,044] jepsen worker 29 - jepsen.util 2189 :ok :read [9 43] INFO [2019-12-17 12:29:42,044] jepsen worker 29 - jepsen.util 2189 :invoke :read [9 nil] INFO [2019-12-17 12:29:42,046] jepsen worker 27 - jepsen.util 2277 :fail :cas [9 [46 37]] :ValueNotSwapFrom INFO [2019-12-17 12:29:42,046] jepsen worker 27 - jepsen.util 2277 :invoke :read [9 nil] INFO [2019-12-17 12:29:42,058] jepsen worker 89 - jepsen.util 2429 :ok :write [29 29] INFO [2019-12-17 12:29:42,058] jepsen worker 89 - jepsen.util 2429 :invoke :write [29 5] INFO [2019-12-17 12:29:42,060] jepsen worker 36 - jepsen.util 2376 :ok :read [12 32] INFO [2019-12-17 12:29:42,061] jepsen worker 36 - jepsen.util 2376 :invoke :read [12 nil] INFO [2019-12-17 12:29:42,074] jepsen worker 78 - jepsen.util 2598 :fail :cas [26 [46 31]] :ValueNotSwapFrom INFO [2019-12-17 12:29:42,075] jepsen worker 78 - jepsen.util 2598 :invoke :cas [26 [4 45]] INFO [2019-12-17 12:29:42,075] jepsen worker 53 - jepsen.util 2483 :fail :cas [17 [8 30]] :ValueNotSwapFrom INFO [2019-12-17 12:29:42,075] jepsen worker 53 - jepsen.util 2483 :invoke :read [17 nil] INFO [2019-12-17 12:29:42,086] jepsen worker 20 - jepsen.util 2270 :ok :write [6 1] INFO [2019-12-17 12:29:42,086] jepsen worker 20 - jepsen.util 2270 :invoke :cas [6 [11 24]] INFO [2019-12-17 12:29:42,086] jepsen worker 29 - jepsen.util 2189 :ok :read [9 43] INFO [2019-12-17 12:29:42,086] jepsen worker 29 - jepsen.util 2189 :invoke :read [9 nil] INFO [2019-12-17 12:29:42,087] jepsen worker 27 - jepsen.util 2277 :ok :read [9 43] INFO [2019-12-17 12:29:42,087] jepsen worker 27 - jepsen.util 2277 :invoke :write [9 26] INFO [2019-12-17 12:29:42,094] jepsen worker 46 - jepsen.util 2746 :ok :write [15 37] INFO [2019-12-17 12:29:42,094] jepsen worker 46 - jepsen.util 2746 :invoke :write [15 8] INFO [2019-12-17 12:29:42,105] jepsen worker 80 - jepsen.util 2510 :fail :cas [26 [34 0]] :ValueNotSwapFrom INFO [2019-12-17 12:29:42,106] jepsen worker 80 - jepsen.util 2510 :invoke :cas [26 [37 47]] INFO [2019-12-17 12:29:42,115] jepsen worker 38 - jepsen.util 2468 :ok :write [12 18] INFO [2019-12-17 12:29:42,115] jepsen worker 38 - jepsen.util 2468 :invoke :read [12 nil] INFO [2019-12-17 12:29:42,134] jepsen worker 53 - jepsen.util 2483 :ok :read [17 45] INFO [2019-12-17 12:29:42,137] jepsen worker 18 - jepsen.util 2718 :fail :read [6 nil] :RequestTimeoutException INFO [2019-12-17 12:29:42,168] jepsen worker 71 - jepsen.util 2501 :ok :read [23 39] INFO [2019-12-17 12:29:42,170] jepsen worker 56 - jepsen.util 2396 :ok :read [18 22] INFO [2019-12-17 12:29:42,177] jepsen worker 19 - jepsen.util 2629 :ok :read [6 1] INFO [2019-12-17 12:29:42,178] jepsen worker 80 - jepsen.util 2510 :fail :cas [26 [37 47]] :ValueNotSwapFrom INFO [2019-12-17 12:29:42,194] jepsen worker 12 - jepsen.util 2622 :fail :cas [4 [0 30]] :CASMismatchException INFO [2019-12-17 12:29:42,196] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":14687,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:29:42,216] jepsen worker 29 - jepsen.util 2189 :ok :read [9 43] INFO [2019-12-17 12:29:42,217] jepsen worker 53 - jepsen.util 2483 :invoke :write [17 2] INFO [2019-12-17 12:29:42,217] jepsen worker 18 - jepsen.util 2718 :invoke :cas [6 [34 21]] INFO [2019-12-17 12:29:42,217] jepsen worker 71 - jepsen.util 2501 :invoke :read [23 nil] INFO [2019-12-17 12:29:42,217] jepsen worker 56 - jepsen.util 2396 :invoke :read [18 nil] INFO [2019-12-17 12:29:42,217] jepsen worker 19 - jepsen.util 2629 :invoke :write [6 14] INFO [2019-12-17 12:29:42,217] jepsen worker 80 - jepsen.util 2510 :invoke :read [26 nil] INFO [2019-12-17 12:29:42,217] jepsen worker 12 - jepsen.util 2622 :invoke :write [4 49] INFO [2019-12-17 12:29:42,217] jepsen worker 29 - jepsen.util 2189 :invoke :read [9 nil] INFO [2019-12-17 12:29:42,217] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":14687,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:29:42,217] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":14689,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:29:42,217] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":14689,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:29:42,221] jepsen worker 73 - jepsen.util 2503 :fail :read [24 nil] :RequestTimeoutException INFO [2019-12-17 12:29:42,222] jepsen worker 73 - jepsen.util 2503 :invoke :write [24 25] INFO [2019-12-17 12:29:42,233] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":14694,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:29:42,233] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":14694,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:29:42,233] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":14697,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:29:42,233] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":14697,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:29:42,233] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":14698,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:29:42,233] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":14698,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:29:42,236] jepsen worker 16 - jepsen.util 3256 :info :write [5 38] :RequestCanceledException INFO [2019-12-17 12:29:42,237] jepsen worker 16 - jepsen.util 3346 :invoke :cas [5 [26 45]] INFO [2019-12-17 12:29:42,257] jepsen worker 20 - jepsen.util 2270 :fail :cas [6 [11 24]] :ValueNotSwapFrom INFO [2019-12-17 12:29:42,258] jepsen worker 20 - jepsen.util 2270 :invoke :write [6 11] INFO [2019-12-17 12:29:42,264] jepsen worker 74 - jepsen.util 2414 :ok :read [24 31] INFO [2019-12-17 12:29:42,265] jepsen worker 74 - jepsen.util 2414 :invoke :write [24 42] INFO [2019-12-17 12:29:42,271] jepsen worker 37 - jepsen.util 2197 :ok :write [12 15] INFO [2019-12-17 12:29:42,271] jepsen worker 37 - jepsen.util 2197 :invoke :write [12 39] INFO [2019-12-17 12:29:42,275] jepsen worker 71 - jepsen.util 2501 :ok :read [23 39] INFO [2019-12-17 12:29:42,275] jepsen worker 71 - jepsen.util 2501 :invoke :write [23 22] INFO [2019-12-17 12:29:42,276] jepsen worker 80 - jepsen.util 2510 :ok :read [26 36] INFO [2019-12-17 12:29:42,276] jepsen worker 80 - jepsen.util 2510 :invoke :write [26 20] INFO [2019-12-17 12:29:42,307] jepsen worker 29 - jepsen.util 2189 :ok :read [9 43] INFO [2019-12-17 12:29:42,308] jepsen worker 29 - jepsen.util 2189 :invoke :cas [9 [30 21]] INFO [2019-12-17 12:29:42,309] jepsen worker 35 - jepsen.util 3095 :info :write [11 33] :RequestCanceledException INFO [2019-12-17 12:29:42,322] jepsen worker 64 - jepsen.util 2134 :info :cas [21 [41 42]] :RequestCanceledException INFO [2019-12-17 12:29:42,323] jepsen worker 35 - jepsen.util 3185 :invoke :write [11 44] INFO [2019-12-17 12:29:42,331] jepsen worker 54 - jepsen.util 2484 :ok :write [18 29] INFO [2019-12-17 12:29:42,331] jepsen worker 54 - jepsen.util 2484 :invoke :cas [18 [24 22]] INFO [2019-12-17 12:29:42,334] jepsen worker 64 - jepsen.util 2224 :invoke :write [21 40] INFO [2019-12-17 12:29:42,337] jepsen worker 31 - jepsen.util 2461 :info :cas [10 [13 44]] :RequestCanceledException INFO [2019-12-17 12:29:42,338] jepsen worker 31 - jepsen.util 2551 :invoke :cas [10 [22 46]] INFO [2019-12-17 12:29:42,362] jepsen worker 27 - jepsen.util 2277 :ok :write [9 26] INFO [2019-12-17 12:29:42,362] jepsen worker 27 - jepsen.util 2277 :invoke :read [9 nil] INFO [2019-12-17 12:29:42,383] jepsen worker 72 - jepsen.util 2592 :ok :write [24 28] INFO [2019-12-17 12:29:42,383] jepsen worker 72 - jepsen.util 2592 :invoke :read [24 nil] INFO [2019-12-17 12:29:42,384] jepsen worker 79 - jepsen.util 2509 :ok :write [26 20] INFO [2019-12-17 12:29:42,384] jepsen worker 79 - jepsen.util 2509 :invoke :read [26 nil] INFO [2019-12-17 12:29:42,422] jepsen worker 29 - jepsen.util 2189 :fail :cas [9 [30 21]] :ValueNotSwapFrom INFO [2019-12-17 12:29:42,422] jepsen worker 29 - jepsen.util 2189 :invoke :write [9 38] INFO [2019-12-17 12:29:42,423] jepsen worker 50 - jepsen.util 3740 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:29:42,423] jepsen worker 50 - jepsen.util 3740 :invoke :write [16 3] INFO [2019-12-17 12:29:42,423] jepsen worker 15 - jepsen.util 2985 :info :cas [5 [26 30]] :RequestCanceledException INFO [2019-12-17 12:29:42,423] jepsen worker 28 - jepsen.util 1918 :info :cas [9 [36 35]] :RequestCanceledException INFO [2019-12-17 12:29:42,424] jepsen worker 15 - jepsen.util 3075 :invoke :write [5 20] INFO [2019-12-17 12:29:42,424] jepsen worker 28 - jepsen.util 2008 :invoke :read [9 nil] INFO [2019-12-17 12:29:42,453] jepsen worker 36 - jepsen.util 2376 :ok :read [12 15] INFO [2019-12-17 12:29:42,454] jepsen worker 36 - jepsen.util 2376 :invoke :write [12 1] INFO [2019-12-17 12:29:42,507] jepsen worker 27 - jepsen.util 2277 :ok :read [9 26] INFO [2019-12-17 12:29:42,508] jepsen worker 27 - jepsen.util 2277 :invoke :write [9 1] INFO [2019-12-17 12:29:42,511] jepsen worker 78 - jepsen.util 2598 :fail :cas [26 [4 45]] :ValueNotSwapFrom INFO [2019-12-17 12:29:42,511] jepsen worker 78 - jepsen.util 2598 :invoke :cas [26 [47 38]] INFO [2019-12-17 12:29:42,512] jepsen worker 31 - jepsen.util 2551 :fail :cas [10 [22 46]] :ValueNotSwapFrom INFO [2019-12-17 12:29:42,512] jepsen worker 31 - jepsen.util 2551 :invoke :write [10 19] INFO [2019-12-17 12:29:42,514] jepsen worker 14 - jepsen.util 2354 :fail :cas [4 [5 22]] :ValueNotSwapFrom INFO [2019-12-17 12:29:42,514] jepsen worker 14 - jepsen.util 2354 :invoke :write [4 33] INFO [2019-12-17 12:29:42,515] jepsen worker 72 - jepsen.util 2592 :ok :read [24 28] INFO [2019-12-17 12:29:42,515] jepsen worker 72 - jepsen.util 2592 :invoke :write [24 20] INFO [2019-12-17 12:29:42,522] jepsen worker 28 - jepsen.util 2008 :ok :read [9 26] INFO [2019-12-17 12:29:42,522] jepsen worker 28 - jepsen.util 2008 :invoke :read [9 nil] INFO [2019-12-17 12:29:42,523] jepsen worker 84 - jepsen.util 3324 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:29:42,523] jepsen worker 84 - jepsen.util 3324 :invoke :read [28 nil] INFO [2019-12-17 12:29:42,524] jepsen worker 65 - jepsen.util 2225 :info :write [21 33] :RequestCanceledException INFO [2019-12-17 12:29:42,524] jepsen worker 43 - jepsen.util 3013 :info :write [14 3] :RequestCanceledException INFO [2019-12-17 12:29:42,524] jepsen worker 21 - jepsen.util 3351 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:29:42,524] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":14722,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:29:42,524] jepsen worker 21 - jepsen.util 3351 :invoke :read [7 nil] INFO [2019-12-17 12:29:42,524] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":14722,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:29:42,524] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":14726,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:29:42,524] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":14726,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:29:42,525] jepsen worker 65 - jepsen.util 2315 :invoke :read [21 nil] INFO [2019-12-17 12:29:42,525] jepsen worker 43 - jepsen.util 3103 :invoke :read [14 nil] INFO [2019-12-17 12:29:42,533] jepsen worker 87 - jepsen.util 2337 :fail :cas [29 [14 27]] :ValueNotSwapFrom INFO [2019-12-17 12:29:42,533] jepsen worker 87 - jepsen.util 2337 :invoke :read [29 nil] INFO [2019-12-17 12:29:42,537] jepsen worker 57 - jepsen.util 3117 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:29:42,537] jepsen worker 0 - jepsen.util 3420 :info :write [0 44] :RequestCanceledException INFO [2019-12-17 12:29:42,537] jepsen worker 57 - jepsen.util 3117 :invoke :read [19 nil] INFO [2019-12-17 12:29:42,538] jepsen worker 0 - jepsen.util 3510 :invoke :write [0 6] INFO [2019-12-17 12:29:42,550] jepsen worker 9 - jepsen.util 2889 :ok :write [3 23] INFO [2019-12-17 12:29:42,550] jepsen worker 9 - jepsen.util 2889 :invoke :write [3 19] INFO [2019-12-17 12:29:42,551] jepsen worker 24 - jepsen.util 3174 :info :write [8 16] :RequestCanceledException INFO [2019-12-17 12:29:42,551] jepsen worker 59 - jepsen.util 3209 :info :cas [19 [10 25]] :RequestCanceledException INFO [2019-12-17 12:29:42,552] jepsen worker 59 - jepsen.util 3299 :invoke :cas [19 [8 4]] INFO [2019-12-17 12:29:42,552] jepsen worker 24 - jepsen.util 3264 :invoke :write [8 25] INFO [2019-12-17 12:29:42,571] jepsen worker 88 - jepsen.util 2518 :ok :read [29 29] INFO [2019-12-17 12:29:42,572] jepsen worker 88 - jepsen.util 2518 :invoke :write [29 9] INFO [2019-12-17 12:29:42,588] jepsen worker 18 - jepsen.util 2718 :fail :cas [6 [34 21]] :ValueNotSwapFrom INFO [2019-12-17 12:29:42,589] jepsen worker 18 - jepsen.util 2718 :invoke :cas [6 [38 14]] INFO [2019-12-17 12:29:42,607] jepsen worker 62 - jepsen.util 3212 :info :cas [20 [41 39]] :RequestCanceledException INFO [2019-12-17 12:29:42,607] jepsen worker 1 - jepsen.util 2971 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:29:42,607] jepsen worker 1 - jepsen.util 2971 :invoke :read [0 nil] INFO [2019-12-17 12:29:42,608] jepsen worker 62 - jepsen.util 3302 :invoke :cas [20 [20 1]] INFO [2019-12-17 12:29:42,615] jepsen worker 65 - jepsen.util 2315 :ok :read [21 6] INFO [2019-12-17 12:29:42,615] jepsen worker 65 - jepsen.util 2315 :invoke :write [21 18] INFO [2019-12-17 12:29:42,615] jepsen worker 3 - jepsen.util 2433 :fail :cas [1 [45 9]] :ValueNotSwapFrom INFO [2019-12-17 12:29:42,615] jepsen worker 3 - jepsen.util 2433 :invoke :read [1 nil] INFO [2019-12-17 12:29:42,623] jepsen worker 58 - jepsen.util 3118 :info :cas [19 [24 4]] :RequestCanceledException INFO [2019-12-17 12:29:42,623] jepsen worker 85 - jepsen.util 3595 :info :cas [28 [27 9]] :RequestCanceledException INFO [2019-12-17 12:29:42,624] jepsen worker 58 - jepsen.util 3208 :invoke :write [19 19] INFO [2019-12-17 12:29:42,624] jepsen worker 85 - jepsen.util 3685 :invoke :cas [28 [23 43]] INFO [2019-12-17 12:29:42,624] jepsen worker 29 - jepsen.util 2189 :ok :write [9 38] INFO [2019-12-17 12:29:42,624] jepsen worker 29 - jepsen.util 2189 :invoke :read [9 nil] INFO [2019-12-17 12:29:42,651] jepsen worker 29 - jepsen.util 2189 :ok :read [9 38] INFO [2019-12-17 12:29:42,651] jepsen worker 29 - jepsen.util 2189 :invoke :cas [9 [34 44]] INFO [2019-12-17 12:29:42,654] jepsen worker 11 - jepsen.util 3071 :fail :read [3 nil] :RequestTimeoutException INFO [2019-12-17 12:29:42,654] jepsen worker 11 - jepsen.util 3071 :invoke :cas [3 [41 28]] INFO [2019-12-17 12:29:42,664] jepsen worker 19 - jepsen.util 2629 :ok :write [6 14] INFO [2019-12-17 12:29:42,664] jepsen worker 19 - jepsen.util 2629 :invoke :write [6 15] INFO [2019-12-17 12:29:42,672] jepsen worker 30 - jepsen.util 3000 :ok :write [10 19] INFO [2019-12-17 12:29:42,672] jepsen worker 30 - jepsen.util 3000 :invoke :write [10 29] INFO [2019-12-17 12:29:42,688] jepsen worker 29 - jepsen.util 2189 :fail :cas [9 [34 44]] :ValueNotSwapFrom INFO [2019-12-17 12:29:42,688] jepsen worker 29 - jepsen.util 2189 :invoke :read [9 nil] INFO [2019-12-17 12:29:42,689] jepsen worker 75 - jepsen.util 3495 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:29:42,689] jepsen worker 83 - jepsen.util 3773 :info :cas [27 [4 42]] :RequestCanceledException INFO [2019-12-17 12:29:42,689] jepsen worker 75 - jepsen.util 3495 :invoke :read [25 nil] INFO [2019-12-17 12:29:42,690] jepsen worker 83 - jepsen.util 3863 :invoke :cas [27 [41 5]] INFO [2019-12-17 12:29:42,691] jepsen worker 4 - jepsen.util 1894 :fail :cas [1 [0 30]] :ValueNotSwapFrom INFO [2019-12-17 12:29:42,691] jepsen worker 4 - jepsen.util 1894 :invoke :read [1 nil] INFO [2019-12-17 12:29:42,707] jepsen worker 81 - jepsen.util 3051 :info :write [27 10] :RequestCanceledException INFO [2019-12-17 12:29:42,709] jepsen worker 81 - jepsen.util 3141 :invoke :write [27 43] INFO [2019-12-17 12:29:42,710] jepsen worker 54 - jepsen.util 2484 :fail :cas [18 [24 22]] :ValueNotSwapFrom INFO [2019-12-17 12:29:42,710] jepsen worker 54 - jepsen.util 2484 :invoke :read [18 nil] INFO [2019-12-17 12:29:42,722] jepsen worker 86 - jepsen.util 3236 :info :cas [28 [42 36]] :RequestCanceledException INFO [2019-12-17 12:29:42,723] jepsen worker 86 - jepsen.util 3326 :invoke :write [28 20] INFO [2019-12-17 12:29:42,723] jepsen worker 76 - jepsen.util 3316 :info :cas [25 [9 48]] :RequestCanceledException INFO [2019-12-17 12:29:42,723] jepsen worker 25 - jepsen.util 3805 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:29:42,723] jepsen worker 45 - jepsen.util 2655 :info :write [15 36] :RequestCanceledException INFO [2019-12-17 12:29:42,723] jepsen worker 25 - jepsen.util 3805 :invoke :cas [8 [49 22]] INFO [2019-12-17 12:29:42,724] jepsen worker 76 - jepsen.util 3406 :invoke :read [25 nil] INFO [2019-12-17 12:29:42,724] jepsen worker 45 - jepsen.util 2745 :invoke :read [15 nil] INFO [2019-12-17 12:29:42,735] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":14759,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:29:42,735] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":14759,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:29:42,735] jepsen worker 22 - jepsen.util 3262 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:29:42,735] jepsen worker 41 - jepsen.util 3551 :info :cas [13 [18 1]] :RequestCanceledException INFO [2019-12-17 12:29:42,735] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:29:42,735] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":14762,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:29:42,735] jepsen worker 22 - jepsen.util 3262 :invoke :write [7 46] INFO [2019-12-17 12:29:42,735] jepsen nemesis - couchbase.util Rebalance check count: 106 INFO [2019-12-17 12:29:42,735] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":14762,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:29:42,735] jepsen nemesis - couchbase.util Rebalance stuck count: 87 INFO [2019-12-17 12:29:42,735] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":14763,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:29:42,736] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":14763,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:29:42,736] jepsen worker 41 - jepsen.util 3641 :invoke :read [13 nil] INFO [2019-12-17 12:29:42,746] jepsen worker 11 - jepsen.util 3071 :fail :cas [3 [41 28]] :ValueNotSwapFrom INFO [2019-12-17 12:29:42,746] jepsen worker 11 - jepsen.util 3071 :invoke :cas [3 [34 4]] INFO [2019-12-17 12:29:42,751] jepsen worker 27 - jepsen.util 2277 :ok :write [9 1] INFO [2019-12-17 12:29:42,751] jepsen worker 27 - jepsen.util 2277 :invoke :cas [9 [11 48]] INFO [2019-12-17 12:29:42,754] jepsen worker 42 - jepsen.util 4002 :info :write [14 37] :RequestCanceledException INFO [2019-12-17 12:29:42,755] jepsen worker 42 - jepsen.util 4092 :invoke :cas [14 [42 1]] INFO [2019-12-17 12:29:42,758] jepsen worker 79 - jepsen.util 2509 :ok :read [26 20] INFO [2019-12-17 12:29:42,758] jepsen worker 79 - jepsen.util 2509 :invoke :read [26 nil] INFO [2019-12-17 12:29:42,769] jepsen worker 52 - jepsen.util 2662 :ok :read [17 45] INFO [2019-12-17 12:29:42,769] jepsen worker 52 - jepsen.util 2662 :invoke :read [17 nil] INFO [2019-12-17 12:29:42,777] jepsen worker 38 - jepsen.util 2468 :ok :read [12 15] INFO [2019-12-17 12:29:42,777] jepsen worker 38 - jepsen.util 2468 :invoke :read [12 nil] INFO [2019-12-17 12:29:42,789] jepsen worker 82 - jepsen.util 3952 :info :cas [27 [27 41]] :RequestCanceledException INFO [2019-12-17 12:29:42,789] jepsen worker 29 - jepsen.util 2189 :ok :read [9 38] INFO [2019-12-17 12:29:42,789] jepsen worker 29 - jepsen.util 2189 :invoke :read [9 nil] INFO [2019-12-17 12:29:42,790] jepsen worker 82 - jepsen.util 4042 :invoke :read [27 nil] INFO [2019-12-17 12:29:42,801] jepsen worker 3 - jepsen.util 2433 :ok :read [1 7] INFO [2019-12-17 12:29:42,801] jepsen worker 3 - jepsen.util 2433 :invoke :read [1 nil] INFO [2019-12-17 12:29:42,807] jepsen worker 40 - jepsen.util 3550 :info :write [13 37] :RequestCanceledException INFO [2019-12-17 12:29:42,807] jepsen worker 33 - jepsen.util 3633 :info :cas [11 [26 37]] :RequestCanceledException INFO [2019-12-17 12:29:42,808] jepsen worker 33 - jepsen.util 3723 :invoke :read [11 nil] INFO [2019-12-17 12:29:42,808] jepsen worker 40 - jepsen.util 3640 :invoke :write [13 15] INFO [2019-12-17 12:29:42,822] jepsen worker 26 - jepsen.util 3626 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:29:42,822] jepsen worker 8 - jepsen.util 3068 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:29:42,822] jepsen worker 48 - jepsen.util 2658 :info :cas [16 [8 34]] :RequestCanceledException INFO [2019-12-17 12:29:42,822] jepsen worker 26 - jepsen.util 3626 :invoke :cas [8 [19 7]] INFO [2019-12-17 12:29:42,822] jepsen worker 8 - jepsen.util 3068 :invoke :read [2 nil] INFO [2019-12-17 12:29:42,823] jepsen worker 48 - jepsen.util 2748 :invoke :write [16 6] INFO [2019-12-17 12:29:42,832] jepsen worker 27 - jepsen.util 2277 :fail :cas [9 [11 48]] :ValueNotSwapFrom INFO [2019-12-17 12:29:42,832] jepsen worker 27 - jepsen.util 2277 :invoke :write [9 47] INFO [2019-12-17 12:29:42,834] jepsen worker 11 - jepsen.util 3071 :fail :cas [3 [34 4]] :ValueNotSwapFrom INFO [2019-12-17 12:29:42,834] jepsen worker 11 - jepsen.util 3071 :invoke :cas [3 [23 26]] INFO [2019-12-17 12:29:42,853] jepsen worker 68 - jepsen.util 3038 :info :write [22 48] :RequestCanceledException INFO [2019-12-17 12:29:42,853] jepsen worker 77 - jepsen.util 3227 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:29:42,854] jepsen worker 77 - jepsen.util 3227 :invoke :read [25 nil] INFO [2019-12-17 12:29:42,855] jepsen worker 68 - jepsen.util 3128 :invoke :cas [22 [21 25]] INFO [2019-12-17 12:29:42,855] jepsen worker 78 - jepsen.util 2598 :fail :cas [26 [47 38]] :ValueNotSwapFrom INFO [2019-12-17 12:29:42,855] jepsen worker 78 - jepsen.util 2598 :invoke :cas [26 [35 30]] INFO [2019-12-17 12:29:42,877] jepsen worker 29 - jepsen.util 2189 :ok :read [9 1] INFO [2019-12-17 12:29:42,877] jepsen worker 29 - jepsen.util 2189 :invoke :read [9 nil] INFO [2019-12-17 12:29:42,889] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":14802,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:29:42,889] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":14802,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:29:42,889] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":14804,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:29:42,889] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":14804,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:29:42,903] jepsen worker 3 - jepsen.util 2433 :ok :read [1 7] INFO [2019-12-17 12:29:42,903] jepsen worker 3 - jepsen.util 2433 :invoke :read [1 nil] INFO [2019-12-17 12:29:42,923] jepsen worker 66 - jepsen.util 3306 :info :cas [22 [25 12]] :RequestCanceledException INFO [2019-12-17 12:29:42,924] jepsen worker 66 - jepsen.util 3396 :invoke :write [22 27] INFO [2019-12-17 12:29:42,928] jepsen worker 54 - jepsen.util 2484 :ok :read [18 29] INFO [2019-12-17 12:29:42,929] jepsen worker 54 - jepsen.util 2484 :invoke :write [18 43] INFO [2019-12-17 12:29:42,936] jepsen worker 64 - jepsen.util 2224 :ok :write [21 40] INFO [2019-12-17 12:29:42,936] jepsen worker 17 - jepsen.util 3257 :info :write [5 2] :RequestCanceledException INFO [2019-12-17 12:29:42,936] jepsen worker 10 - jepsen.util 2350 :fail :read [3 nil] :RequestTimeoutException INFO [2019-12-17 12:29:42,937] jepsen worker 64 - jepsen.util 2224 :invoke :read [21 nil] INFO [2019-12-17 12:29:42,937] jepsen worker 10 - jepsen.util 2350 :invoke :write [3 11] INFO [2019-12-17 12:29:42,938] jepsen worker 17 - jepsen.util 3347 :invoke :read [5 nil] INFO [2019-12-17 12:29:42,953] jepsen worker 29 - jepsen.util 2189 :ok :read [9 1] INFO [2019-12-17 12:29:42,953] jepsen worker 29 - jepsen.util 2189 :invoke :read [9 nil] INFO [2019-12-17 12:29:42,958] jepsen worker 64 - jepsen.util 2224 :ok :read [21 40] INFO [2019-12-17 12:29:42,958] jepsen worker 64 - jepsen.util 2224 :invoke :read [21 nil] INFO [2019-12-17 12:29:42,965] jepsen worker 78 - jepsen.util 2598 :fail :cas [26 [35 30]] :ValueNotSwapFrom INFO [2019-12-17 12:29:42,966] jepsen worker 78 - jepsen.util 2598 :invoke :cas [26 [11 1]] INFO [2019-12-17 12:29:42,969] jepsen worker 56 - jepsen.util 2396 :ok :read [18 29] INFO [2019-12-17 12:29:42,969] jepsen worker 56 - jepsen.util 2396 :invoke :read [18 nil] INFO [2019-12-17 12:29:42,983] jepsen worker 64 - jepsen.util 2224 :ok :read [21 40] INFO [2019-12-17 12:29:42,983] jepsen worker 64 - jepsen.util 2224 :invoke :read [21 nil] INFO [2019-12-17 12:29:42,989] jepsen worker 32 - jepsen.util 3002 :fail :read [10 nil] :RequestTimeoutException INFO [2019-12-17 12:29:42,989] jepsen worker 32 - jepsen.util 3002 :invoke :read [10 nil] INFO [2019-12-17 12:29:42,999] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":14819,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:29:42,999] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":14819,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:29:42,999] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":14822,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:29:42,999] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":14822,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:29:42,999] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":14823,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:29:42,999] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":14823,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:29:43,002] jepsen worker 9 - jepsen.util 2889 :ok :write [3 19] INFO [2019-12-17 12:29:43,002] jepsen worker 9 - jepsen.util 2889 :invoke :write [3 39] INFO [2019-12-17 12:29:43,004] jepsen worker 11 - jepsen.util 3071 :fail :cas [3 [23 26]] :CASMismatchException INFO [2019-12-17 12:29:43,004] jepsen worker 11 - jepsen.util 3071 :invoke :cas [3 [44 20]] INFO [2019-12-17 12:29:43,018] jepsen worker 27 - jepsen.util 2277 :ok :write [9 47] INFO [2019-12-17 12:29:43,019] jepsen worker 27 - jepsen.util 2277 :invoke :read [9 nil] INFO [2019-12-17 12:29:43,021] jepsen worker 3 - jepsen.util 2433 :ok :read [1 7] INFO [2019-12-17 12:29:43,021] jepsen worker 3 - jepsen.util 2433 :invoke :read [1 nil] INFO [2019-12-17 12:29:43,037] jepsen worker 47 - jepsen.util 2927 :fail :read [15 nil] :RequestTimeoutException INFO [2019-12-17 12:29:43,037] jepsen worker 47 - jepsen.util 2927 :invoke :read [15 nil] INFO [2019-12-17 12:29:43,039] jepsen worker 11 - jepsen.util 3071 :fail :cas [3 [44 20]] :ValueNotSwapFrom INFO [2019-12-17 12:29:43,039] jepsen worker 11 - jepsen.util 3071 :invoke :write [3 6] INFO [2019-12-17 12:29:43,048] jepsen worker 27 - jepsen.util 2277 :ok :read [9 47] INFO [2019-12-17 12:29:43,049] jepsen worker 27 - jepsen.util 2277 :invoke :cas [9 [12 28]] INFO [2019-12-17 12:29:43,054] jepsen worker 39 - jepsen.util 3369 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:29:43,054] jepsen worker 39 - jepsen.util 3369 :invoke :cas [13 [34 29]] INFO [2019-12-17 12:29:43,062] jepsen worker 79 - jepsen.util 2509 :ok :read [26 20] INFO [2019-12-17 12:29:43,063] jepsen worker 79 - jepsen.util 2509 :invoke :read [26 nil] INFO [2019-12-17 12:29:43,072] jepsen worker 27 - jepsen.util 2277 :fail :cas [9 [12 28]] :ValueNotSwapFrom INFO [2019-12-17 12:29:43,072] jepsen worker 27 - jepsen.util 2277 :invoke :cas [9 [29 41]] INFO [2019-12-17 12:29:43,081] jepsen worker 64 - jepsen.util 2224 :ok :read [21 40] INFO [2019-12-17 12:29:43,081] jepsen worker 64 - jepsen.util 2224 :invoke :cas [21 [49 13]] INFO [2019-12-17 12:29:43,122] jepsen worker 27 - jepsen.util 2277 :fail :cas [9 [29 41]] :ValueNotSwapFrom INFO [2019-12-17 12:29:43,122] jepsen worker 27 - jepsen.util 2277 :invoke :cas [9 [16 22]] INFO [2019-12-17 12:29:43,123] jepsen worker 63 - jepsen.util 2133 :info :write [21 25] :RequestCanceledException INFO [2019-12-17 12:29:43,124] jepsen worker 2 - jepsen.util 3242 :info :cas [0 [24 15]] :RequestCanceledException INFO [2019-12-17 12:29:43,125] jepsen worker 2 - jepsen.util 3332 :invoke :read [0 nil] INFO [2019-12-17 12:29:43,125] jepsen worker 63 - jepsen.util 2223 :invoke :cas [21 [10 16]] INFO [2019-12-17 12:29:43,130] jepsen worker 65 - jepsen.util 2315 :ok :write [21 18] INFO [2019-12-17 12:29:43,131] jepsen worker 65 - jepsen.util 2315 :invoke :read [21 nil] INFO [2019-12-17 12:29:43,131] jepsen worker 64 - jepsen.util 2224 :fail :cas [21 [49 13]] :ValueNotSwapFrom INFO [2019-12-17 12:29:43,131] jepsen worker 64 - jepsen.util 2224 :invoke :cas [21 [30 4]] INFO [2019-12-17 12:29:43,141] jepsen worker 52 - jepsen.util 2662 :ok :read [17 45] INFO [2019-12-17 12:29:43,141] jepsen worker 52 - jepsen.util 2662 :invoke :write [17 0] INFO [2019-12-17 12:29:43,161] jepsen worker 27 - jepsen.util 2277 :fail :cas [9 [16 22]] :ValueNotSwapFrom INFO [2019-12-17 12:29:43,161] jepsen worker 27 - jepsen.util 2277 :invoke :write [9 13] INFO [2019-12-17 12:29:43,182] jepsen worker 38 - jepsen.util 2468 :ok :read [12 15] INFO [2019-12-17 12:29:43,182] jepsen worker 38 - jepsen.util 2468 :invoke :write [12 2] INFO [2019-12-17 12:29:43,189] jepsen worker 61 - jepsen.util 3661 :info :write [20 8] :RequestCanceledException INFO [2019-12-17 12:29:43,189] jepsen worker 61 - jepsen.util 3751 :invoke :write [20 21] INFO [2019-12-17 12:29:43,200] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":14844,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:29:43,200] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":14844,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:29:43,200] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":14845,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:29:43,200] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":14845,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:29:43,218] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":14854,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:29:43,218] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":14854,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:29:43,218] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":14856,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:29:43,218] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":14856,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:29:43,219] jepsen worker 63 - jepsen.util 2223 :fail :cas [21 [10 16]] :ValueNotSwapFrom INFO [2019-12-17 12:29:43,219] jepsen worker 63 - jepsen.util 2223 :invoke :write [21 4] INFO [2019-12-17 12:29:43,243] jepsen worker 64 - jepsen.util 2224 :fail :cas [21 [30 4]] :ValueNotSwapFrom INFO [2019-12-17 12:29:43,243] jepsen worker 64 - jepsen.util 2224 :invoke :cas [21 [23 1]] INFO [2019-12-17 12:29:43,261] jepsen worker 64 - jepsen.util 2224 :fail :cas [21 [23 1]] :ValueNotSwapFrom INFO [2019-12-17 12:29:43,261] jepsen worker 64 - jepsen.util 2224 :invoke :write [21 35] INFO [2019-12-17 12:29:43,270] jepsen worker 27 - jepsen.util 2277 :ok :write [9 13] INFO [2019-12-17 12:29:43,270] jepsen worker 27 - jepsen.util 2277 :invoke :cas [9 [43 0]] INFO [2019-12-17 12:29:43,289] jepsen worker 27 - jepsen.util 2277 :fail :cas [9 [43 0]] :ValueNotSwapFrom INFO [2019-12-17 12:29:43,289] jepsen worker 27 - jepsen.util 2277 :invoke :read [9 nil] INFO [2019-12-17 12:29:43,301] jepsen worker 27 - jepsen.util 2277 :ok :read [9 13] INFO [2019-12-17 12:29:43,301] jepsen worker 27 - jepsen.util 2277 :invoke :write [9 30] INFO [2019-12-17 12:29:43,327] jepsen worker 64 - jepsen.util 2224 :ok :write [21 35] INFO [2019-12-17 12:29:43,327] jepsen worker 64 - jepsen.util 2224 :invoke :write [21 14] INFO [2019-12-17 12:29:43,332] jepsen worker 3 - jepsen.util 2433 :ok :read [1 7] INFO [2019-12-17 12:29:43,332] jepsen worker 3 - jepsen.util 2433 :invoke :cas [1 [45 8]] INFO [2019-12-17 12:29:43,372] jepsen worker 27 - jepsen.util 2277 :ok :write [9 30] INFO [2019-12-17 12:29:43,372] jepsen worker 27 - jepsen.util 2277 :invoke :cas [9 [12 7]] INFO [2019-12-17 12:29:43,385] jepsen worker 64 - jepsen.util 2224 :ok :write [21 14] INFO [2019-12-17 12:29:43,385] jepsen worker 64 - jepsen.util 2224 :invoke :cas [21 [45 14]] INFO [2019-12-17 12:29:43,390] jepsen worker 27 - jepsen.util 2277 :fail :cas [9 [12 7]] :ValueNotSwapFrom INFO [2019-12-17 12:29:43,390] jepsen worker 27 - jepsen.util 2277 :invoke :write [9 4] INFO [2019-12-17 12:29:43,402] jepsen worker 64 - jepsen.util 2224 :fail :cas [21 [45 14]] :ValueNotSwapFrom INFO [2019-12-17 12:29:43,402] jepsen worker 64 - jepsen.util 2224 :invoke :cas [21 [4 27]] INFO [2019-12-17 12:29:43,410] jepsen worker 78 - jepsen.util 2598 :fail :cas [26 [11 1]] :ValueNotSwapFrom INFO [2019-12-17 12:29:43,410] jepsen worker 78 - jepsen.util 2598 :invoke :write [26 40] INFO [2019-12-17 12:29:43,418] jepsen worker 64 - jepsen.util 2224 :fail :cas [21 [4 27]] :ValueNotSwapFrom INFO [2019-12-17 12:29:43,419] jepsen worker 64 - jepsen.util 2224 :invoke :cas [21 [0 28]] INFO [2019-12-17 12:29:43,430] jepsen worker 79 - jepsen.util 2509 :ok :read [26 20] INFO [2019-12-17 12:29:43,431] jepsen worker 79 - jepsen.util 2509 :invoke :read [26 nil] INFO [2019-12-17 12:29:43,431] jepsen worker 64 - jepsen.util 2224 :fail :cas [21 [0 28]] :ValueNotSwapFrom INFO [2019-12-17 12:29:43,431] jepsen worker 64 - jepsen.util 2224 :invoke :write [21 12] INFO [2019-12-17 12:29:43,454] jepsen worker 13 - jepsen.util 2263 :info :write [4 32] :RequestCanceledException INFO [2019-12-17 12:29:43,455] jepsen worker 13 - jepsen.util 2353 :invoke :cas [4 [45 7]] INFO [2019-12-17 12:29:43,465] jepsen worker 27 - jepsen.util 2277 :ok :write [9 4] INFO [2019-12-17 12:29:43,466] jepsen worker 27 - jepsen.util 2277 :invoke :write [9 42] INFO [2019-12-17 12:29:43,481] jepsen worker 64 - jepsen.util 2224 :ok :write [21 12] INFO [2019-12-17 12:29:43,481] jepsen worker 64 - jepsen.util 2224 :invoke :cas [21 [44 46]] INFO [2019-12-17 12:29:43,492] jepsen worker 64 - jepsen.util 2224 :fail :cas [21 [44 46]] :ValueNotSwapFrom INFO [2019-12-17 12:29:43,492] jepsen worker 64 - jepsen.util 2224 :invoke :cas [21 [42 26]] INFO [2019-12-17 12:29:43,502] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":14872,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:29:43,502] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":14872,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:29:43,502] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":14875,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:29:43,502] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":14875,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:29:43,503] jepsen worker 64 - jepsen.util 2224 :fail :cas [21 [42 26]] :ValueNotSwapFrom INFO [2019-12-17 12:29:43,503] jepsen worker 64 - jepsen.util 2224 :invoke :read [21 nil] INFO [2019-12-17 12:29:43,513] jepsen worker 64 - jepsen.util 2224 :ok :read [21 12] INFO [2019-12-17 12:29:43,513] jepsen worker 64 - jepsen.util 2224 :invoke :cas [21 [32 45]] INFO [2019-12-17 12:29:43,539] jepsen worker 64 - jepsen.util 2224 :fail :cas [21 [32 45]] :ValueNotSwapFrom INFO [2019-12-17 12:29:43,540] jepsen worker 64 - jepsen.util 2224 :invoke :write [21 33] INFO [2019-12-17 12:29:43,552] jepsen worker 27 - jepsen.util 2277 :ok :write [9 42] INFO [2019-12-17 12:29:43,552] jepsen worker 27 - jepsen.util 2277 :invoke :read [9 nil] INFO [2019-12-17 12:29:43,571] jepsen worker 27 - jepsen.util 2277 :ok :read [9 42] INFO [2019-12-17 12:29:43,572] jepsen worker 27 - jepsen.util 2277 :invoke :read [9 nil] INFO [2019-12-17 12:29:43,593] jepsen worker 27 - jepsen.util 2277 :ok :read [9 42] INFO [2019-12-17 12:29:43,593] jepsen worker 27 - jepsen.util 2277 :invoke :cas [9 [2 35]] INFO [2019-12-17 12:29:43,593] jepsen worker 64 - jepsen.util 2224 :ok :write [21 33] INFO [2019-12-17 12:29:43,593] jepsen worker 64 - jepsen.util 2224 :invoke :cas [21 [16 44]] INFO [2019-12-17 12:29:43,607] jepsen worker 64 - jepsen.util 2224 :fail :cas [21 [16 44]] :ValueNotSwapFrom INFO [2019-12-17 12:29:43,607] jepsen worker 64 - jepsen.util 2224 :invoke :read [21 nil] INFO [2019-12-17 12:29:43,612] jepsen worker 11 - jepsen.util 3071 :ok :write [3 6] INFO [2019-12-17 12:29:43,612] jepsen worker 11 - jepsen.util 3071 :invoke :write [3 45] INFO [2019-12-17 12:29:43,615] jepsen worker 31 - jepsen.util 2551 :ok :write [10 19] INFO [2019-12-17 12:29:43,616] jepsen worker 31 - jepsen.util 2551 :invoke :cas [10 [44 7]] INFO [2019-12-17 12:29:43,623] jepsen worker 64 - jepsen.util 2224 :ok :read [21 33] INFO [2019-12-17 12:29:43,624] jepsen worker 64 - jepsen.util 2224 :invoke :cas [21 [18 38]] INFO [2019-12-17 12:29:43,624] jepsen worker 27 - jepsen.util 2277 :fail :cas [9 [2 35]] :ValueNotSwapFrom INFO [2019-12-17 12:29:43,624] jepsen worker 27 - jepsen.util 2277 :invoke :write [9 45] INFO [2019-12-17 12:29:43,678] jepsen worker 32 - jepsen.util 3002 :ok :read [10 19] INFO [2019-12-17 12:29:43,678] jepsen worker 32 - jepsen.util 3002 :invoke :write [10 25] INFO [2019-12-17 12:29:43,679] jepsen worker 64 - jepsen.util 2224 :fail :cas [21 [18 38]] :ValueNotSwapFrom INFO [2019-12-17 12:29:43,679] jepsen worker 64 - jepsen.util 2224 :invoke :cas [21 [45 3]] INFO [2019-12-17 12:29:43,708] jepsen worker 67 - jepsen.util 3757 :info :write [22 31] :RequestCanceledException INFO [2019-12-17 12:29:43,710] jepsen worker 67 - jepsen.util 3847 :invoke :cas [22 [2 2]] INFO [2019-12-17 12:29:43,719] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":14897,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:29:43,720] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":14897,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:29:43,720] jepsen worker 44 - jepsen.util 3194 :info :cas [14 [7 49]] :RequestCanceledException INFO [2019-12-17 12:29:43,720] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":14900,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:29:43,721] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":14900,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:29:43,722] jepsen worker 31 - jepsen.util 2551 :fail :cas [10 [44 7]] :ValueNotSwapFrom INFO [2019-12-17 12:29:43,722] jepsen worker 31 - jepsen.util 2551 :invoke :write [10 21] INFO [2019-12-17 12:29:43,726] jepsen worker 44 - jepsen.util 3284 :invoke :write [14 37] INFO [2019-12-17 12:29:43,737] jepsen worker 64 - jepsen.util 2224 :fail :cas [21 [45 3]] :ValueNotSwapFrom INFO [2019-12-17 12:29:43,737] jepsen worker 34 - jepsen.util 3454 :info :write [11 42] :RequestCanceledException INFO [2019-12-17 12:29:43,737] jepsen worker 6 - jepsen.util 3426 :info :cas [2 [8 16]] :RequestCanceledException INFO [2019-12-17 12:29:43,737] jepsen worker 64 - jepsen.util 2224 :invoke :write [21 10] INFO [2019-12-17 12:29:43,738] jepsen worker 6 - jepsen.util 3516 :invoke :write [2 29] INFO [2019-12-17 12:29:43,738] jepsen worker 34 - jepsen.util 3544 :invoke :write [11 41] INFO [2019-12-17 12:29:43,747] jepsen worker 3 - jepsen.util 2433 :fail :cas [1 [45 8]] :ValueNotSwapFrom INFO [2019-12-17 12:29:43,748] jepsen worker 3 - jepsen.util 2433 :invoke :read [1 nil] INFO [2019-12-17 12:29:43,754] jepsen worker 49 - jepsen.util 2659 :info :cas [16 [5 0]] :RequestCanceledException INFO [2019-12-17 12:29:43,755] jepsen worker 49 - jepsen.util 2749 :invoke :cas [16 [23 26]] INFO [2019-12-17 12:29:43,764] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:29:43,764] jepsen nemesis - couchbase.util Rebalance check count: 107 INFO [2019-12-17 12:29:43,764] jepsen nemesis - couchbase.util Rebalance stuck count: 88 INFO [2019-12-17 12:29:43,789] jepsen worker 60 - jepsen.util 3570 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:29:43,789] jepsen worker 7 - jepsen.util 3247 :info :cas [2 [10 20]] :RequestCanceledException INFO [2019-12-17 12:29:43,790] jepsen worker 60 - jepsen.util 3570 :invoke :write [20 44] INFO [2019-12-17 12:29:43,791] jepsen worker 7 - jepsen.util 3337 :invoke :cas [2 [43 9]] INFO [2019-12-17 12:29:43,791] jepsen worker 87 - jepsen.util 2337 :ok :read [29 29] INFO [2019-12-17 12:29:43,791] jepsen worker 87 - jepsen.util 2337 :invoke :read [29 nil] INFO [2019-12-17 12:29:43,808] jepsen worker 4 - jepsen.util 1894 :ok :read [1 7] INFO [2019-12-17 12:29:43,808] jepsen worker 4 - jepsen.util 1894 :invoke :write [1 35] INFO [2019-12-17 12:29:43,817] jepsen worker 64 - jepsen.util 2224 :ok :write [21 10] INFO [2019-12-17 12:29:43,817] jepsen worker 64 - jepsen.util 2224 :invoke :cas [21 [36 28]] INFO [2019-12-17 12:29:43,823] jepsen worker 23 - jepsen.util 3173 :info :write [7 11] :RequestCanceledException INFO [2019-12-17 12:29:43,826] jepsen worker 23 - jepsen.util 3263 :invoke :cas [7 [5 41]] INFO [2019-12-17 12:29:43,830] jepsen worker 27 - jepsen.util 2277 :ok :write [9 45] INFO [2019-12-17 12:29:43,830] jepsen worker 27 - jepsen.util 2277 :invoke :cas [9 [21 5]] INFO [2019-12-17 12:29:43,838] jepsen worker 64 - jepsen.util 2224 :fail :cas [21 [36 28]] :ValueNotSwapFrom INFO [2019-12-17 12:29:43,838] jepsen worker 64 - jepsen.util 2224 :invoke :write [21 43] INFO [2019-12-17 12:29:43,852] jepsen worker 27 - jepsen.util 2277 :fail :cas [9 [21 5]] :ValueNotSwapFrom INFO [2019-12-17 12:29:43,852] jepsen worker 27 - jepsen.util 2277 :invoke :read [9 nil] INFO [2019-12-17 12:29:43,876] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":14921,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:29:43,876] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":14921,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:29:43,876] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":14924,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:29:43,876] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":14924,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:29:43,876] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":14925,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:29:43,876] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":14925,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:29:43,892] jepsen worker 27 - jepsen.util 2277 :ok :read [9 45] INFO [2019-12-17 12:29:43,892] jepsen worker 27 - jepsen.util 2277 :invoke :cas [9 [1 46]] INFO [2019-12-17 12:29:43,904] jepsen worker 69 - jepsen.util 2409 :info :write [23 13] :RequestCanceledException INFO [2019-12-17 12:29:43,906] jepsen worker 69 - jepsen.util 2499 :invoke :read [23 nil] INFO [2019-12-17 12:29:43,909] jepsen worker 27 - jepsen.util 2277 :fail :cas [9 [1 46]] :ValueNotSwapFrom INFO [2019-12-17 12:29:43,909] jepsen worker 27 - jepsen.util 2277 :invoke :write [9 16] INFO [2019-12-17 12:29:43,935] jepsen worker 55 - jepsen.util 2935 :info :write [18 7] :RequestCanceledException INFO [2019-12-17 12:29:43,937] jepsen worker 55 - jepsen.util 3025 :invoke :read [18 nil] INFO [2019-12-17 12:29:43,971] jepsen worker 64 - jepsen.util 2224 :ok :write [21 43] INFO [2019-12-17 12:29:43,971] jepsen worker 64 - jepsen.util 2224 :invoke :cas [21 [16 18]] INFO [2019-12-17 12:29:43,988] jepsen worker 79 - jepsen.util 2509 :ok :read [26 20] INFO [2019-12-17 12:29:43,988] jepsen worker 79 - jepsen.util 2509 :invoke :read [26 nil] INFO [2019-12-17 12:29:43,989] jepsen worker 70 - jepsen.util 2950 :info :write [23 28] :RequestCanceledException INFO [2019-12-17 12:29:43,992] jepsen worker 70 - jepsen.util 3040 :invoke :write [23 23] INFO [2019-12-17 12:29:43,998] jepsen worker 64 - jepsen.util 2224 :fail :cas [21 [16 18]] :ValueNotSwapFrom INFO [2019-12-17 12:29:43,998] jepsen worker 64 - jepsen.util 2224 :invoke :write [21 38] INFO [2019-12-17 12:29:44,019] jepsen worker 27 - jepsen.util 2277 :ok :write [9 16] INFO [2019-12-17 12:29:44,019] jepsen worker 27 - jepsen.util 2277 :invoke :read [9 nil] INFO [2019-12-17 12:29:44,021] jepsen worker 5 - jepsen.util 2075 :info :write [1 2] :RequestCanceledException INFO [2019-12-17 12:29:44,022] jepsen worker 5 - jepsen.util 2165 :invoke :write [1 15] INFO [2019-12-17 12:29:44,082] jepsen worker 64 - jepsen.util 2224 :ok :write [21 38] INFO [2019-12-17 12:29:44,082] jepsen worker 64 - jepsen.util 2224 :invoke :write [21 16] INFO [2019-12-17 12:29:44,083] jepsen worker 27 - jepsen.util 2277 :ok :read [9 16] INFO [2019-12-17 12:29:44,083] jepsen worker 27 - jepsen.util 2277 :invoke :write [9 29] INFO [2019-12-17 12:29:44,090] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":14932,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:29:44,090] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":14932,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:29:44,091] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":14933,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:29:44,091] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":14933,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:29:44,092] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":14937,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:29:44,093] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":14937,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:29:44,109] jepsen worker 46 - jepsen.util 2746 :info :write [15 8] :RequestCanceledException INFO [2019-12-17 12:29:44,120] jepsen worker 46 - jepsen.util 2836 :invoke :write [15 42] INFO [2019-12-17 12:29:44,124] jepsen worker 51 - jepsen.util 2301 :info :write [17 22] :RequestCanceledException INFO [2019-12-17 12:29:44,125] jepsen worker 51 - jepsen.util 2391 :invoke :read [17 nil] INFO [2019-12-17 12:29:44,134] jepsen worker 79 - jepsen.util 2509 :ok :read [26 20] INFO [2019-12-17 12:29:44,134] jepsen worker 79 - jepsen.util 2509 :invoke :read [26 nil] INFO [2019-12-17 12:29:44,155] jepsen worker 89 - jepsen.util 2429 :info :write [29 5] :RequestCanceledException INFO [2019-12-17 12:29:44,161] jepsen worker 89 - jepsen.util 2519 :invoke :cas [29 [10 47]] INFO [2019-12-17 12:29:44,167] jepsen worker 56 - jepsen.util 2396 :ok :read [18 29] INFO [2019-12-17 12:29:44,167] jepsen worker 56 - jepsen.util 2396 :invoke :read [18 nil] INFO [2019-12-17 12:29:44,198] jepsen worker 3 - jepsen.util 2433 :ok :read [1 7] INFO [2019-12-17 12:29:44,198] jepsen worker 55 - jepsen.util 3025 :ok :read [18 29] INFO [2019-12-17 12:29:44,198] jepsen worker 3 - jepsen.util 2433 :invoke :cas [1 [19 13]] INFO [2019-12-17 12:29:44,198] jepsen worker 55 - jepsen.util 3025 :invoke :write [18 41] INFO [2019-12-17 12:29:44,228] jepsen worker 64 - jepsen.util 2224 :ok :write [21 16] INFO [2019-12-17 12:29:44,228] jepsen worker 51 - jepsen.util 2391 :ok :read [17 45] INFO [2019-12-17 12:29:44,229] jepsen worker 64 - jepsen.util 2224 :invoke :write [21 46] INFO [2019-12-17 12:29:44,229] jepsen worker 51 - jepsen.util 2391 :invoke :read [17 nil] INFO [2019-12-17 12:29:44,237] jepsen worker 53 - jepsen.util 2483 :info :write [17 2] :RequestCanceledException INFO [2019-12-17 12:29:44,246] jepsen worker 27 - jepsen.util 2277 :ok :write [9 29] INFO [2019-12-17 12:29:44,247] jepsen worker 27 - jepsen.util 2277 :invoke :read [9 nil] INFO [2019-12-17 12:29:44,252] jepsen worker 12 - jepsen.util 2622 :info :write [4 49] :RequestCanceledException INFO [2019-12-17 12:29:44,252] jepsen worker 53 - jepsen.util 2573 :invoke :write [17 24] INFO [2019-12-17 12:29:44,255] jepsen worker 12 - jepsen.util 2712 :invoke :read [4 nil] INFO [2019-12-17 12:29:44,279] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":14949,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:29:44,279] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":14949,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:29:44,279] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":14951,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:29:44,279] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":14951,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:29:44,290] jepsen worker 3 - jepsen.util 2433 :fail :cas [1 [19 13]] :ValueNotSwapFrom INFO [2019-12-17 12:29:44,290] jepsen worker 3 - jepsen.util 2433 :invoke :cas [1 [12 34]] INFO [2019-12-17 12:29:44,298] jepsen worker 27 - jepsen.util 2277 :ok :read [9 29] INFO [2019-12-17 12:29:44,298] jepsen worker 27 - jepsen.util 2277 :invoke :cas [9 [26 32]] INFO [2019-12-17 12:29:44,307] jepsen worker 74 - jepsen.util 2414 :info :write [24 42] :RequestCanceledException INFO [2019-12-17 12:29:44,309] jepsen worker 74 - jepsen.util 2504 :invoke :cas [24 [8 45]] INFO [2019-12-17 12:29:44,309] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":14958,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:29:44,310] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":14958,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:29:44,310] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":14962,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:29:44,310] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":14962,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:29:44,322] jepsen worker 64 - jepsen.util 2224 :ok :write [21 46] INFO [2019-12-17 12:29:44,322] jepsen worker 64 - jepsen.util 2224 :invoke :cas [21 [46 5]] INFO [2019-12-17 12:29:44,323] jepsen worker 71 - jepsen.util 2501 :info :write [23 22] :RequestCanceledException INFO [2019-12-17 12:29:44,323] jepsen worker 73 - jepsen.util 2503 :info :write [24 25] :RequestCanceledException INFO [2019-12-17 12:29:44,324] jepsen worker 20 - jepsen.util 2270 :info :write [6 11] :RequestCanceledException INFO [2019-12-17 12:29:44,324] jepsen worker 37 - jepsen.util 2197 :info :write [12 39] :RequestCanceledException INFO [2019-12-17 12:29:44,325] jepsen worker 20 - jepsen.util 2360 :invoke :read [6 nil] INFO [2019-12-17 12:29:44,335] jepsen worker 16 - jepsen.util 3346 :info :cas [5 [26 45]] :RequestCanceledException INFO [2019-12-17 12:29:44,336] jepsen worker 80 - jepsen.util 2510 :info :write [26 20] :RequestCanceledException INFO [2019-12-17 12:29:44,341] jepsen worker 37 - jepsen.util 2287 :invoke :read [12 nil] INFO [2019-12-17 12:29:44,341] jepsen worker 73 - jepsen.util 2593 :invoke :cas [24 [41 14]] INFO [2019-12-17 12:29:44,344] jepsen worker 71 - jepsen.util 2591 :invoke :cas [23 [7 3]] INFO [2019-12-17 12:29:44,344] jepsen worker 80 - jepsen.util 2600 :invoke :cas [26 [24 27]] INFO [2019-12-17 12:29:44,345] jepsen worker 16 - jepsen.util 3436 :invoke :cas [5 [5 12]] INFO [2019-12-17 12:29:44,354] jepsen worker 35 - jepsen.util 3185 :info :write [11 44] :RequestCanceledException INFO [2019-12-17 12:29:44,366] jepsen worker 35 - jepsen.util 3275 :invoke :write [11 10] INFO [2019-12-17 12:29:44,398] jepsen worker 11 - jepsen.util 3071 :ok :write [3 45] INFO [2019-12-17 12:29:44,398] jepsen worker 32 - jepsen.util 3002 :ok :write [10 25] INFO [2019-12-17 12:29:44,398] jepsen worker 11 - jepsen.util 3071 :invoke :read [3 nil] INFO [2019-12-17 12:29:44,398] jepsen worker 32 - jepsen.util 3002 :invoke :write [10 40] INFO [2019-12-17 12:29:44,401] jepsen worker 74 - jepsen.util 2504 :fail :cas [24 [8 45]] :ValueNotSwapFrom INFO [2019-12-17 12:29:44,401] jepsen worker 74 - jepsen.util 2504 :invoke :read [24 nil] INFO [2019-12-17 12:29:44,411] jepsen worker 27 - jepsen.util 2277 :fail :cas [9 [26 32]] :ValueNotSwapFrom INFO [2019-12-17 12:29:44,411] jepsen worker 27 - jepsen.util 2277 :invoke :read [9 nil] INFO [2019-12-17 12:29:44,483] jepsen worker 79 - jepsen.util 2509 :ok :read [26 20] INFO [2019-12-17 12:29:44,483] jepsen worker 79 - jepsen.util 2509 :invoke :write [26 25] INFO [2019-12-17 12:29:44,483] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":14982,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:29:44,484] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":14982,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:29:44,484] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":14987,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:29:44,487] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":14987,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:29:44,507] jepsen worker 15 - jepsen.util 3075 :info :write [5 20] :RequestCanceledException INFO [2019-12-17 12:29:44,514] jepsen worker 46 - jepsen.util 2836 :ok :write [15 42] INFO [2019-12-17 12:29:44,514] jepsen worker 46 - jepsen.util 2836 :invoke :cas [15 [19 13]] INFO [2019-12-17 12:29:44,524] jepsen worker 36 - jepsen.util 2376 :info :write [12 1] :RequestCanceledException INFO [2019-12-17 12:29:44,524] jepsen worker 50 - jepsen.util 3740 :info :write [16 3] :RequestCanceledException INFO [2019-12-17 12:29:44,530] jepsen worker 15 - jepsen.util 3165 :invoke :read [5 nil] INFO [2019-12-17 12:29:44,540] jepsen worker 36 - jepsen.util 2466 :invoke :read [12 nil] INFO [2019-12-17 12:29:44,551] jepsen worker 50 - jepsen.util 3830 :invoke :read [16 nil] INFO [2019-12-17 12:29:44,554] jepsen worker 43 - jepsen.util 3103 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:29:44,555] jepsen worker 14 - jepsen.util 2354 :info :write [4 33] :RequestCanceledException INFO [2019-12-17 12:29:44,555] jepsen worker 43 - jepsen.util 3103 :invoke :read [14 nil] INFO [2019-12-17 12:29:44,574] jepsen worker 14 - jepsen.util 2444 :invoke :cas [4 [7 48]] INFO [2019-12-17 12:29:44,589] jepsen worker 59 - jepsen.util 3299 :info :cas [19 [8 4]] :RequestCanceledException INFO [2019-12-17 12:29:44,590] jepsen worker 72 - jepsen.util 2592 :info :write [24 20] :RequestCanceledException INFO [2019-12-17 12:29:44,607] jepsen worker 24 - jepsen.util 3264 :info :write [8 25] :RequestCanceledException INFO [2019-12-17 12:29:44,608] jepsen worker 88 - jepsen.util 2518 :info :write [29 9] :RequestCanceledException INFO [2019-12-17 12:29:44,619] jepsen worker 59 - jepsen.util 3389 :invoke :cas [19 [20 34]] INFO [2019-12-17 12:29:44,626] jepsen worker 84 - jepsen.util 3324 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:29:44,626] jepsen worker 0 - jepsen.util 3510 :info :write [0 6] :RequestCanceledException INFO [2019-12-17 12:29:44,627] jepsen worker 28 - jepsen.util 2008 :fail :read [9 nil] :RequestTimeoutException INFO [2019-12-17 12:29:44,627] jepsen worker 72 - jepsen.util 2682 :invoke :cas [24 [35 28]] INFO [2019-12-17 12:29:44,627] jepsen worker 84 - jepsen.util 3324 :invoke :cas [28 [38 5]] INFO [2019-12-17 12:29:44,628] jepsen worker 28 - jepsen.util 2008 :invoke :read [9 nil] INFO [2019-12-17 12:29:44,630] jepsen worker 62 - jepsen.util 3302 :info :cas [20 [20 1]] :RequestCanceledException INFO [2019-12-17 12:29:44,638] jepsen worker 21 - jepsen.util 3351 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:29:44,638] jepsen worker 57 - jepsen.util 3117 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:29:44,638] jepsen worker 18 - jepsen.util 2718 :info :cas [6 [38 14]] :RequestCanceledException INFO [2019-12-17 12:29:44,639] jepsen worker 57 - jepsen.util 3117 :invoke :write [19 26] INFO [2019-12-17 12:29:44,639] jepsen worker 21 - jepsen.util 3351 :invoke :write [7 8] INFO [2019-12-17 12:29:44,656] jepsen worker 85 - jepsen.util 3685 :info :cas [28 [23 43]] :RequestCanceledException INFO [2019-12-17 12:29:44,663] jepsen worker 0 - jepsen.util 3600 :invoke :write [0 41] INFO [2019-12-17 12:29:44,663] jepsen worker 24 - jepsen.util 3354 :invoke :write [8 29] INFO [2019-12-17 12:29:44,663] jepsen worker 88 - jepsen.util 2608 :invoke :cas [29 [14 38]] INFO [2019-12-17 12:29:44,665] jepsen worker 62 - jepsen.util 3392 :invoke :read [20 nil] INFO [2019-12-17 12:29:44,668] jepsen worker 18 - jepsen.util 2808 :invoke :write [6 28] INFO [2019-12-17 12:29:44,682] jepsen worker 85 - jepsen.util 3775 :invoke :cas [28 [6 24]] INFO [2019-12-17 12:29:44,709] jepsen worker 1 - jepsen.util 2971 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:29:44,712] jepsen worker 30 - jepsen.util 3000 :info :write [10 29] :RequestCanceledException INFO [2019-12-17 12:29:44,714] jepsen worker 1 - jepsen.util 2971 :invoke :cas [0 [24 27]] INFO [2019-12-17 12:29:44,714] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15001,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:29:44,715] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15001,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:29:44,715] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15004,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:29:44,716] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15004,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:29:44,717] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15005,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:29:44,718] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15005,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:29:44,726] jepsen worker 19 - jepsen.util 2629 :info :write [6 15] :RequestCanceledException INFO [2019-12-17 12:29:44,734] jepsen worker 83 - jepsen.util 3863 :info :cas [27 [41 5]] :RequestCanceledException INFO [2019-12-17 12:29:44,742] jepsen worker 45 - jepsen.util 2745 :fail :read [15 nil] :RequestTimeoutException INFO [2019-12-17 12:29:44,742] jepsen worker 45 - jepsen.util 2745 :invoke :write [15 27] INFO [2019-12-17 12:29:44,745] jepsen worker 31 - jepsen.util 2551 :ok :write [10 21] INFO [2019-12-17 12:29:44,747] jepsen worker 31 - jepsen.util 2551 :invoke :write [10 12] INFO [2019-12-17 12:29:44,749] jepsen worker 58 - jepsen.util 3208 :info :write [19 19] :RequestCanceledException INFO [2019-12-17 12:29:44,756] jepsen worker 41 - jepsen.util 3641 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:29:44,757] jepsen worker 41 - jepsen.util 3641 :invoke :cas [13 [43 8]] INFO [2019-12-17 12:29:44,785] jepsen worker 30 - jepsen.util 3090 :invoke :cas [10 [40 25]] INFO [2019-12-17 12:29:44,795] jepsen worker 75 - jepsen.util 3495 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:29:44,796] jepsen worker 75 - jepsen.util 3495 :invoke :cas [25 [18 1]] INFO [2019-12-17 12:29:44,804] jepsen worker 81 - jepsen.util 3141 :info :write [27 43] :RequestCanceledException INFO [2019-12-17 12:29:44,810] jepsen worker 86 - jepsen.util 3326 :info :write [28 20] :RequestCanceledException INFO [2019-12-17 12:29:44,812] jepsen worker 19 - jepsen.util 2719 :invoke :read [6 nil] INFO [2019-12-17 12:29:44,819] jepsen worker 83 - jepsen.util 3953 :invoke :read [27 nil] INFO [2019-12-17 12:29:44,829] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:29:44,830] jepsen worker 40 - jepsen.util 3640 :info :write [13 15] :RequestCanceledException INFO [2019-12-17 12:29:44,830] jepsen worker 25 - jepsen.util 3805 :info :cas [8 [49 22]] :RequestCanceledException INFO [2019-12-17 12:29:44,831] jepsen nemesis - couchbase.util Rebalance check count: 108 INFO [2019-12-17 12:29:44,836] jepsen worker 42 - jepsen.util 4092 :info :cas [14 [42 1]] :RequestCanceledException INFO [2019-12-17 12:29:44,836] jepsen nemesis - couchbase.util Rebalance stuck count: 89 INFO [2019-12-17 12:29:44,837] jepsen worker 76 - jepsen.util 3406 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:29:44,844] jepsen worker 22 - jepsen.util 3262 :info :write [7 46] :RequestCanceledException INFO [2019-12-17 12:29:44,845] jepsen worker 76 - jepsen.util 3406 :invoke :write [25 6] INFO [2019-12-17 12:29:44,878] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15017,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:29:44,885] jepsen worker 81 - jepsen.util 3231 :invoke :write [27 38] INFO [2019-12-17 12:29:44,885] jepsen worker 58 - jepsen.util 3298 :invoke :write [19 43] INFO [2019-12-17 12:29:44,892] jepsen worker 82 - jepsen.util 4042 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:29:44,892] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15017,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:29:44,892] jepsen worker 82 - jepsen.util 4042 :invoke :write [27 25] INFO [2019-12-17 12:29:44,894] jepsen worker 64 - jepsen.util 2224 :ok :cas [21 [46 5]] INFO [2019-12-17 12:29:44,894] jepsen worker 64 - jepsen.util 2224 :invoke :write [21 34] INFO [2019-12-17 12:29:44,914] jepsen worker 33 - jepsen.util 3723 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:29:44,914] jepsen worker 33 - jepsen.util 3723 :invoke :read [11 nil] INFO [2019-12-17 12:29:44,915] jepsen worker 86 - jepsen.util 3416 :invoke :write [28 25] INFO [2019-12-17 12:29:44,928] jepsen worker 8 - jepsen.util 3068 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:29:44,928] jepsen worker 8 - jepsen.util 3068 :invoke :read [2 nil] INFO [2019-12-17 12:29:44,930] jepsen worker 26 - jepsen.util 3626 :info :cas [8 [19 7]] :RequestCanceledException INFO [2019-12-17 12:29:44,949] jepsen worker 48 - jepsen.util 2748 :info :write [16 6] :RequestCanceledException INFO [2019-12-17 12:29:44,962] jepsen worker 77 - jepsen.util 3227 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:29:44,963] jepsen worker 77 - jepsen.util 3227 :invoke :write [25 29] INFO [2019-12-17 12:29:44,967] jepsen worker 68 - jepsen.util 3128 :info :cas [22 [21 25]] :RequestCanceledException INFO [2019-12-17 12:29:44,973] jepsen worker 40 - jepsen.util 3730 :invoke :read [13 nil] INFO [2019-12-17 12:29:44,983] jepsen worker 22 - jepsen.util 3352 :invoke :cas [7 [13 30]] INFO [2019-12-17 12:29:44,990] jepsen worker 42 - jepsen.util 4182 :invoke :read [14 nil] INFO [2019-12-17 12:29:44,990] jepsen worker 17 - jepsen.util 3347 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:29:44,995] jepsen worker 17 - jepsen.util 3347 :invoke :cas [5 [22 39]] INFO [2019-12-17 12:29:44,998] jepsen worker 25 - jepsen.util 3895 :invoke :write [8 21] INFO [2019-12-17 12:29:45,010] jepsen worker 54 - jepsen.util 2484 :info :write [18 43] :RequestCanceledException INFO [2019-12-17 12:29:45,083] jepsen worker 48 - jepsen.util 2838 :invoke :cas [16 [29 12]] INFO [2019-12-17 12:29:45,083] jepsen worker 68 - jepsen.util 3218 :invoke :write [22 13] INFO [2019-12-17 12:29:45,084] jepsen worker 13 - jepsen.util 2353 :fail :cas [4 [45 7]] :ValueNotSwapFrom INFO [2019-12-17 12:29:45,085] jepsen worker 13 - jepsen.util 2353 :invoke :read [4 nil] INFO [2019-12-17 12:29:45,086] jepsen worker 32 - jepsen.util 3002 :ok :write [10 40] INFO [2019-12-17 12:29:45,086] jepsen worker 27 - jepsen.util 2277 :ok :read [9 29] INFO [2019-12-17 12:29:45,086] jepsen worker 87 - jepsen.util 2337 :ok :read [29 29] INFO [2019-12-17 12:29:45,086] jepsen worker 32 - jepsen.util 3002 :invoke :write [10 14] INFO [2019-12-17 12:29:45,086] jepsen worker 74 - jepsen.util 2504 :ok :read [24 28] INFO [2019-12-17 12:29:45,087] jepsen worker 27 - jepsen.util 2277 :invoke :write [9 42] INFO [2019-12-17 12:29:45,087] jepsen worker 87 - jepsen.util 2337 :invoke :read [29 nil] INFO [2019-12-17 12:29:45,087] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15027,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:29:45,087] jepsen worker 10 - jepsen.util 2350 :info :write [3 11] :RequestCanceledException INFO [2019-12-17 12:29:45,088] jepsen worker 26 - jepsen.util 3716 :invoke :cas [8 [10 25]] INFO [2019-12-17 12:29:45,087] jepsen worker 74 - jepsen.util 2504 :invoke :cas [24 [46 7]] INFO [2019-12-17 12:29:45,089] jepsen worker 66 - jepsen.util 3396 :info :write [22 27] :RequestCanceledException INFO [2019-12-17 12:29:45,089] jepsen worker 29 - jepsen.util 2189 :fail :read [9 nil] :RequestTimeoutException INFO [2019-12-17 12:29:45,089] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15027,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:29:45,090] jepsen worker 9 - jepsen.util 2889 :info :write [3 39] :RequestCanceledException INFO [2019-12-17 12:29:45,090] jepsen worker 29 - jepsen.util 2189 :invoke :cas [9 [43 9]] INFO [2019-12-17 12:29:45,092] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15033,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:29:45,094] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15033,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:29:45,101] jepsen worker 54 - jepsen.util 2574 :invoke :read [18 nil] INFO [2019-12-17 12:29:45,137] jepsen worker 47 - jepsen.util 2927 :fail :read [15 nil] :RequestTimeoutException INFO [2019-12-17 12:29:45,139] jepsen worker 47 - jepsen.util 2927 :invoke :read [15 nil] INFO [2019-12-17 12:29:45,159] jepsen worker 39 - jepsen.util 3369 :info :cas [13 [34 29]] :RequestCanceledException INFO [2019-12-17 12:29:45,192] jepsen worker 52 - jepsen.util 2662 :info :write [17 0] :RequestCanceledException INFO [2019-12-17 12:29:45,206] jepsen worker 9 - jepsen.util 2979 :invoke :write [3 27] INFO [2019-12-17 12:29:45,206] jepsen worker 66 - jepsen.util 3486 :invoke :read [22 nil] INFO [2019-12-17 12:29:45,207] jepsen worker 2 - jepsen.util 3332 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:29:45,208] jepsen worker 2 - jepsen.util 3332 :invoke :read [0 nil] INFO [2019-12-17 12:29:45,232] jepsen worker 10 - jepsen.util 2440 :invoke :read [3 nil] INFO [2019-12-17 12:29:45,243] jepsen worker 65 - jepsen.util 2315 :fail :read [21 nil] :RequestTimeoutException INFO [2019-12-17 12:29:45,244] jepsen worker 65 - jepsen.util 2315 :invoke :read [21 nil] INFO [2019-12-17 12:29:45,251] jepsen worker 61 - jepsen.util 3751 :info :write [20 21] :RequestCanceledException INFO [2019-12-17 12:29:45,260] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15051,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:29:45,264] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15051,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:29:45,266] jepsen worker 39 - jepsen.util 3459 :invoke :read [13 nil] INFO [2019-12-17 12:29:45,267] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15058,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:29:45,274] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15058,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:29:45,282] jepsen worker 52 - jepsen.util 2752 :invoke :read [17 nil] INFO [2019-12-17 12:29:45,291] jepsen worker 38 - jepsen.util 2468 :info :write [12 2] :RequestCanceledException INFO [2019-12-17 12:29:45,314] jepsen worker 61 - jepsen.util 3841 :invoke :write [20 48] INFO [2019-12-17 12:29:45,325] jepsen worker 63 - jepsen.util 2223 :info :write [21 4] :RequestCanceledException INFO [2019-12-17 12:29:45,363] jepsen worker 38 - jepsen.util 2558 :invoke :cas [12 [37 12]] INFO [2019-12-17 12:29:45,370] jepsen worker 63 - jepsen.util 2313 :invoke :cas [21 [40 45]] INFO [2019-12-17 12:29:45,408] jepsen worker 11 - jepsen.util 3071 :ok :read [3 45] INFO [2019-12-17 12:29:45,408] jepsen worker 11 - jepsen.util 3071 :invoke :write [3 9] INFO [2019-12-17 12:29:45,429] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15078,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:29:45,430] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15078,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:29:45,430] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15081,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:29:45,433] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15081,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:29:45,438] jepsen worker 28 - jepsen.util 2008 :ok :read [9 29] INFO [2019-12-17 12:29:45,439] jepsen worker 28 - jepsen.util 2008 :invoke :read [9 nil] INFO [2019-12-17 12:29:45,524] jepsen worker 78 - jepsen.util 2598 :info :write [26 40] :RequestCanceledException INFO [2019-12-17 12:29:45,543] jepsen worker 46 - jepsen.util 2836 :fail :cas [15 [19 13]] :ValueNotSwapFrom INFO [2019-12-17 12:29:45,543] jepsen worker 46 - jepsen.util 2836 :invoke :cas [15 [32 12]] INFO [2019-12-17 12:29:45,548] jepsen worker 78 - jepsen.util 2688 :invoke :write [26 33] INFO [2019-12-17 12:29:45,560] jepsen worker 72 - jepsen.util 2682 :fail :cas [24 [35 28]] :ValueNotSwapFrom INFO [2019-12-17 12:29:45,561] jepsen worker 72 - jepsen.util 2682 :invoke :cas [24 [45 34]] INFO [2019-12-17 12:29:45,574] jepsen worker 51 - jepsen.util 2391 :ok :read [17 45] INFO [2019-12-17 12:29:45,575] jepsen worker 51 - jepsen.util 2391 :invoke :cas [17 [21 8]] INFO [2019-12-17 12:29:45,607] jepsen worker 63 - jepsen.util 2313 :fail :cas [21 [40 45]] :ValueNotSwapFrom INFO [2019-12-17 12:29:45,607] jepsen worker 37 - jepsen.util 2287 :ok :read [12 15] INFO [2019-12-17 12:29:45,607] jepsen worker 37 - jepsen.util 2287 :invoke :read [12 nil] INFO [2019-12-17 12:29:45,607] jepsen worker 63 - jepsen.util 2313 :invoke :write [21 40] INFO [2019-12-17 12:29:45,611] jepsen worker 56 - jepsen.util 2396 :ok :read [18 29] INFO [2019-12-17 12:29:45,612] jepsen worker 56 - jepsen.util 2396 :invoke :write [18 43] INFO [2019-12-17 12:29:45,620] jepsen worker 30 - jepsen.util 3090 :fail :cas [10 [40 25]] :ValueNotSwapFrom INFO [2019-12-17 12:29:45,620] jepsen worker 30 - jepsen.util 3090 :invoke :write [10 13] INFO [2019-12-17 12:29:45,621] jepsen worker 12 - jepsen.util 2712 :ok :read [4 19] INFO [2019-12-17 12:29:45,622] jepsen worker 12 - jepsen.util 2712 :invoke :write [4 36] INFO [2019-12-17 12:29:45,623] jepsen worker 64 - jepsen.util 2224 :ok :write [21 34] INFO [2019-12-17 12:29:45,624] jepsen worker 64 - jepsen.util 2224 :invoke :read [21 nil] INFO [2019-12-17 12:29:45,625] jepsen worker 73 - jepsen.util 2593 :fail :cas [24 [41 14]] :ValueNotSwapFrom INFO [2019-12-17 12:29:45,626] jepsen worker 73 - jepsen.util 2593 :invoke :read [24 nil] INFO [2019-12-17 12:29:45,643] jepsen worker 29 - jepsen.util 2189 :fail :cas [9 [43 9]] :ValueNotSwapFrom INFO [2019-12-17 12:29:45,643] jepsen worker 29 - jepsen.util 2189 :invoke :write [9 14] INFO [2019-12-17 12:29:45,646] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15099,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:29:45,647] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15099,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:29:45,647] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15100,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:29:45,647] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15100,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:29:45,651] jepsen worker 80 - jepsen.util 2600 :fail :cas [26 [24 27]] :ValueNotSwapFrom INFO [2019-12-17 12:29:45,652] jepsen worker 80 - jepsen.util 2600 :invoke :read [26 nil] INFO [2019-12-17 12:29:45,663] jepsen worker 52 - jepsen.util 2752 :ok :read [17 45] INFO [2019-12-17 12:29:45,664] jepsen worker 52 - jepsen.util 2752 :invoke :write [17 4] INFO [2019-12-17 12:29:45,689] jepsen worker 19 - jepsen.util 2719 :ok :read [6 14] INFO [2019-12-17 12:29:45,689] jepsen worker 19 - jepsen.util 2719 :invoke :cas [6 [37 33]] INFO [2019-12-17 12:29:45,695] jepsen worker 27 - jepsen.util 2277 :ok :write [9 42] INFO [2019-12-17 12:29:45,695] jepsen worker 27 - jepsen.util 2277 :invoke :cas [9 [6 24]] INFO [2019-12-17 12:29:45,720] jepsen worker 67 - jepsen.util 3847 :info :cas [22 [2 2]] :RequestCanceledException INFO [2019-12-17 12:29:45,722] jepsen worker 67 - jepsen.util 3937 :invoke :write [22 18] INFO [2019-12-17 12:29:45,786] jepsen worker 44 - jepsen.util 3284 :info :write [14 37] :RequestCanceledException INFO [2019-12-17 12:29:45,796] jepsen worker 63 - jepsen.util 2313 :ok :write [21 40] INFO [2019-12-17 12:29:45,798] jepsen worker 63 - jepsen.util 2313 :invoke :write [21 18] INFO [2019-12-17 12:29:45,805] jepsen worker 6 - jepsen.util 3516 :info :write [2 29] :RequestCanceledException INFO [2019-12-17 12:29:45,815] jepsen worker 44 - jepsen.util 3374 :invoke :write [14 41] INFO [2019-12-17 12:29:45,821] jepsen worker 34 - jepsen.util 3544 :info :write [11 41] :RequestCanceledException INFO [2019-12-17 12:29:45,830] jepsen worker 49 - jepsen.util 2749 :info :cas [16 [23 26]] :RequestCanceledException INFO [2019-12-17 12:29:45,853] jepsen worker 7 - jepsen.util 3337 :info :cas [2 [43 9]] :RequestCanceledException INFO [2019-12-17 12:29:45,854] jepsen worker 4 - jepsen.util 1894 :info :write [1 35] :RequestCanceledException INFO [2019-12-17 12:29:45,862] jepsen worker 6 - jepsen.util 3606 :invoke :write [2 3] INFO [2019-12-17 12:29:45,862] jepsen worker 34 - jepsen.util 3634 :invoke :cas [11 [15 26]] INFO [2019-12-17 12:29:45,864] jepsen worker 49 - jepsen.util 2839 :invoke :cas [16 [33 5]] INFO [2019-12-17 12:29:45,869] jepsen worker 7 - jepsen.util 3427 :invoke :read [2 nil] INFO [2019-12-17 12:29:45,875] jepsen worker 51 - jepsen.util 2391 :fail :cas [17 [21 8]] :ValueNotSwapFrom INFO [2019-12-17 12:29:45,875] jepsen worker 51 - jepsen.util 2391 :invoke :read [17 nil] INFO [2019-12-17 12:29:45,876] jepsen worker 4 - jepsen.util 1984 :invoke :cas [1 [21 33]] INFO [2019-12-17 12:29:45,880] jepsen worker 60 - jepsen.util 3570 :info :write [20 44] :RequestCanceledException INFO [2019-12-17 12:29:45,888] jepsen worker 60 - jepsen.util 3660 :invoke :cas [20 [22 33]] INFO [2019-12-17 12:29:45,888] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:29:45,889] jepsen nemesis - couchbase.util Rebalance check count: 109 INFO [2019-12-17 12:29:45,889] jepsen nemesis - couchbase.util Rebalance stuck count: 90 INFO [2019-12-17 12:29:45,892] jepsen worker 28 - jepsen.util 2008 :ok :read [9 42] INFO [2019-12-17 12:29:45,892] jepsen worker 28 - jepsen.util 2008 :invoke :write [9 22] INFO [2019-12-17 12:29:45,906] jepsen worker 64 - jepsen.util 2224 :ok :read [21 34] INFO [2019-12-17 12:29:45,906] jepsen worker 64 - jepsen.util 2224 :invoke :read [21 nil] INFO [2019-12-17 12:29:45,919] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15114,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:29:45,920] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15114,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:29:45,920] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15116,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:29:45,921] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15116,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:29:45,921] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15117,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:29:45,922] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15117,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:29:45,924] jepsen worker 23 - jepsen.util 3263 :info :cas [7 [5 41]] :RequestCanceledException INFO [2019-12-17 12:29:45,933] jepsen worker 23 - jepsen.util 3353 :invoke :cas [7 [45 1]] INFO [2019-12-17 12:29:45,936] jepsen worker 14 - jepsen.util 2444 :fail :cas [4 [7 48]] :ValueNotSwapFrom INFO [2019-12-17 12:29:45,936] jepsen worker 14 - jepsen.util 2444 :invoke :read [4 nil] INFO [2019-12-17 12:29:45,986] jepsen worker 69 - jepsen.util 2499 :ok :read [23 39] INFO [2019-12-17 12:29:45,987] jepsen worker 69 - jepsen.util 2499 :invoke :read [23 nil] INFO [2019-12-17 12:29:46,023] jepsen worker 70 - jepsen.util 3040 :info :write [23 23] :RequestCanceledException INFO [2019-12-17 12:29:46,024] jepsen worker 13 - jepsen.util 2353 :ok :read [4 19] INFO [2019-12-17 12:29:46,024] jepsen worker 13 - jepsen.util 2353 :invoke :read [4 nil] INFO [2019-12-17 12:29:46,034] jepsen worker 70 - jepsen.util 3130 :invoke :read [23 nil] INFO [2019-12-17 12:29:46,043] jepsen worker 89 - jepsen.util 2519 :fail :cas [29 [10 47]] :ValueNotSwapFrom INFO [2019-12-17 12:29:46,043] jepsen worker 89 - jepsen.util 2519 :invoke :cas [29 [44 5]] INFO [2019-12-17 12:29:46,059] jepsen worker 63 - jepsen.util 2313 :ok :write [21 18] INFO [2019-12-17 12:29:46,060] jepsen worker 63 - jepsen.util 2313 :invoke :read [21 nil] INFO [2019-12-17 12:29:46,124] jepsen worker 37 - jepsen.util 2287 :ok :read [12 15] INFO [2019-12-17 12:29:46,125] jepsen worker 37 - jepsen.util 2287 :invoke :write [12 16] INFO [2019-12-17 12:29:46,126] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15130,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:29:46,127] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15130,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:29:46,127] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15131,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:29:46,128] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15131,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:29:46,137] jepsen worker 5 - jepsen.util 2165 :info :write [1 15] :RequestCanceledException INFO [2019-12-17 12:29:46,147] jepsen worker 28 - jepsen.util 2008 :ok :write [9 22] INFO [2019-12-17 12:29:46,147] jepsen worker 28 - jepsen.util 2008 :invoke :cas [9 [25 9]] INFO [2019-12-17 12:29:46,185] jepsen worker 5 - jepsen.util 2255 :invoke :cas [1 [25 24]] INFO [2019-12-17 12:29:46,201] jepsen worker 18 - jepsen.util 2808 :ok :write [6 28] INFO [2019-12-17 12:29:46,201] jepsen worker 18 - jepsen.util 2808 :invoke :read [6 nil] INFO [2019-12-17 12:29:46,204] jepsen worker 11 - jepsen.util 3071 :ok :write [3 9] INFO [2019-12-17 12:29:46,204] jepsen worker 11 - jepsen.util 3071 :invoke :read [3 nil] INFO [2019-12-17 12:29:46,223] jepsen worker 31 - jepsen.util 2551 :ok :write [10 12] INFO [2019-12-17 12:29:46,224] jepsen worker 31 - jepsen.util 2551 :invoke :cas [10 [6 28]] INFO [2019-12-17 12:29:46,237] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15143,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:29:46,239] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15143,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:29:46,239] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15147,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:29:46,239] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15147,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:29:46,276] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15152,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:29:46,278] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15152,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:29:46,278] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15156,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:29:46,280] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15156,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:29:46,280] jepsen worker 64 - jepsen.util 2224 :ok :read [21 40] INFO [2019-12-17 12:29:46,281] jepsen worker 87 - jepsen.util 2337 :ok :read [29 29] INFO [2019-12-17 12:29:46,281] jepsen worker 64 - jepsen.util 2224 :invoke :write [21 35] INFO [2019-12-17 12:29:46,282] jepsen worker 87 - jepsen.util 2337 :invoke :write [29 29] INFO [2019-12-17 12:29:46,304] jepsen worker 55 - jepsen.util 3025 :info :write [18 41] :RequestCanceledException INFO [2019-12-17 12:29:46,306] jepsen worker 53 - jepsen.util 2573 :info :write [17 24] :RequestCanceledException INFO [2019-12-17 12:29:46,307] jepsen worker 3 - jepsen.util 2433 :info :cas [1 [12 34]] :RequestCanceledException INFO [2019-12-17 12:29:46,317] jepsen worker 55 - jepsen.util 3115 :invoke :write [18 25] INFO [2019-12-17 12:29:46,318] jepsen worker 53 - jepsen.util 2663 :invoke :write [17 28] INFO [2019-12-17 12:29:46,320] jepsen worker 3 - jepsen.util 2523 :invoke :write [1 15] INFO [2019-12-17 12:29:46,352] jepsen worker 20 - jepsen.util 2360 :fail :read [6 nil] :RequestTimeoutException INFO [2019-12-17 12:29:46,352] jepsen worker 20 - jepsen.util 2360 :invoke :read [6 nil] INFO [2019-12-17 12:29:46,375] jepsen worker 36 - jepsen.util 2466 :ok :read [12 15] INFO [2019-12-17 12:29:46,375] jepsen worker 36 - jepsen.util 2466 :invoke :read [12 nil] INFO [2019-12-17 12:29:46,380] jepsen worker 28 - jepsen.util 2008 :fail :cas [9 [25 9]] :ValueNotSwapFrom INFO [2019-12-17 12:29:46,380] jepsen worker 28 - jepsen.util 2008 :invoke :write [9 49] INFO [2019-12-17 12:29:46,384] jepsen worker 54 - jepsen.util 2574 :ok :read [18 29] INFO [2019-12-17 12:29:46,385] jepsen worker 54 - jepsen.util 2574 :invoke :write [18 41] INFO [2019-12-17 12:29:46,391] jepsen worker 46 - jepsen.util 2836 :fail :cas [15 [32 12]] :ValueNotSwapFrom INFO [2019-12-17 12:29:46,391] jepsen worker 46 - jepsen.util 2836 :invoke :cas [15 [48 32]] INFO [2019-12-17 12:29:46,396] jepsen worker 89 - jepsen.util 2519 :fail :cas [29 [44 5]] :ValueNotSwapFrom INFO [2019-12-17 12:29:46,397] jepsen worker 89 - jepsen.util 2519 :invoke :write [29 15] INFO [2019-12-17 12:29:46,400] jepsen worker 69 - jepsen.util 2499 :ok :read [23 39] INFO [2019-12-17 12:29:46,400] jepsen worker 69 - jepsen.util 2499 :invoke :read [23 nil] INFO [2019-12-17 12:29:46,401] jepsen worker 80 - jepsen.util 2600 :ok :read [26 20] INFO [2019-12-17 12:29:46,401] jepsen worker 80 - jepsen.util 2600 :invoke :write [26 30] INFO [2019-12-17 12:29:46,402] jepsen worker 72 - jepsen.util 2682 :fail :cas [24 [45 34]] :ValueNotSwapFrom INFO [2019-12-17 12:29:46,402] jepsen worker 72 - jepsen.util 2682 :invoke :read [24 nil] INFO [2019-12-17 12:29:46,404] jepsen worker 71 - jepsen.util 2591 :info :cas [23 [7 3]] :RequestCanceledException INFO [2019-12-17 12:29:46,406] jepsen worker 71 - jepsen.util 2681 :invoke :write [23 17] INFO [2019-12-17 12:29:46,437] jepsen worker 16 - jepsen.util 3436 :info :cas [5 [5 12]] :RequestCanceledException INFO [2019-12-17 12:29:46,440] jepsen worker 16 - jepsen.util 3526 :invoke :write [5 2] INFO [2019-12-17 12:29:46,452] jepsen worker 35 - jepsen.util 3275 :info :write [11 10] :RequestCanceledException INFO [2019-12-17 12:29:46,454] jepsen worker 35 - jepsen.util 3365 :invoke :read [11 nil] INFO [2019-12-17 12:29:46,462] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15175,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:29:46,462] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15175,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:29:46,462] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15179,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:29:46,463] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15179,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:29:46,467] jepsen worker 64 - jepsen.util 2224 :ok :write [21 35] INFO [2019-12-17 12:29:46,467] jepsen worker 64 - jepsen.util 2224 :invoke :read [21 nil] INFO [2019-12-17 12:29:46,487] jepsen worker 29 - jepsen.util 2189 :ok :write [9 14] INFO [2019-12-17 12:29:46,488] jepsen worker 29 - jepsen.util 2189 :invoke :cas [9 [31 5]] INFO [2019-12-17 12:29:46,497] jepsen worker 64 - jepsen.util 2224 :ok :read [21 35] INFO [2019-12-17 12:29:46,497] jepsen worker 64 - jepsen.util 2224 :invoke :cas [21 [34 7]] INFO [2019-12-17 12:29:46,533] jepsen worker 28 - jepsen.util 2008 :ok :write [9 49] INFO [2019-12-17 12:29:46,533] jepsen worker 28 - jepsen.util 2008 :invoke :cas [9 [45 19]] INFO [2019-12-17 12:29:46,543] jepsen worker 64 - jepsen.util 2224 :fail :cas [21 [34 7]] :ValueNotSwapFrom INFO [2019-12-17 12:29:46,543] jepsen worker 64 - jepsen.util 2224 :invoke :cas [21 [35 2]] INFO [2019-12-17 12:29:46,556] jepsen worker 11 - jepsen.util 3071 :ok :read [3 9] INFO [2019-12-17 12:29:46,556] jepsen worker 11 - jepsen.util 3071 :invoke :cas [3 [3 15]] INFO [2019-12-17 12:29:46,583] jepsen worker 45 - jepsen.util 2745 :ok :write [15 27] INFO [2019-12-17 12:29:46,583] jepsen worker 45 - jepsen.util 2745 :invoke :write [15 1] INFO [2019-12-17 12:29:46,587] jepsen worker 20 - jepsen.util 2360 :ok :read [6 28] INFO [2019-12-17 12:29:46,587] jepsen worker 20 - jepsen.util 2360 :invoke :read [6 nil] INFO [2019-12-17 12:29:46,607] jepsen worker 50 - jepsen.util 3830 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:29:46,623] jepsen worker 15 - jepsen.util 3165 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:29:46,641] jepsen worker 59 - jepsen.util 3389 :info :cas [19 [20 34]] :RequestCanceledException INFO [2019-12-17 12:29:46,647] jepsen worker 46 - jepsen.util 2836 :fail :cas [15 [48 32]] :ValueNotSwapFrom INFO [2019-12-17 12:29:46,654] jepsen worker 43 - jepsen.util 3103 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:29:46,658] jepsen worker 28 - jepsen.util 2008 :fail :cas [9 [45 19]] :ValueNotSwapFrom INFO [2019-12-17 12:29:46,673] jepsen worker 36 - jepsen.util 2466 :ok :read [12 15] INFO [2019-12-17 12:29:46,686] jepsen worker 50 - jepsen.util 3830 :invoke :write [16 42] INFO [2019-12-17 12:29:46,686] jepsen worker 15 - jepsen.util 3165 :invoke :read [5 nil] INFO [2019-12-17 12:29:46,686] jepsen worker 46 - jepsen.util 2836 :invoke :write [15 46] INFO [2019-12-17 12:29:46,686] jepsen worker 43 - jepsen.util 3103 :invoke :cas [14 [23 13]] INFO [2019-12-17 12:29:46,694] jepsen worker 59 - jepsen.util 3479 :invoke :cas [19 [24 9]] INFO [2019-12-17 12:29:46,704] jepsen worker 29 - jepsen.util 2189 :fail :cas [9 [31 5]] :ValueNotSwapFrom INFO [2019-12-17 12:29:46,704] jepsen worker 72 - jepsen.util 2682 :ok :read [24 28] INFO [2019-12-17 12:29:46,705] jepsen worker 36 - jepsen.util 2466 :invoke :read [12 nil] INFO [2019-12-17 12:29:46,705] jepsen worker 28 - jepsen.util 2008 :invoke :cas [9 [32 42]] INFO [2019-12-17 12:29:46,705] jepsen worker 88 - jepsen.util 2608 :info :cas [29 [14 38]] :RequestCanceledException INFO [2019-12-17 12:29:46,705] jepsen worker 29 - jepsen.util 2189 :invoke :cas [9 [49 39]] INFO [2019-12-17 12:29:46,706] jepsen worker 72 - jepsen.util 2682 :invoke :cas [24 [38 21]] INFO [2019-12-17 12:29:46,708] jepsen worker 88 - jepsen.util 2698 :invoke :read [29 nil] INFO [2019-12-17 12:29:46,719] jepsen worker 47 - jepsen.util 2927 :ok :read [15 27] INFO [2019-12-17 12:29:46,719] jepsen worker 47 - jepsen.util 2927 :invoke :cas [15 [36 37]] INFO [2019-12-17 12:29:46,722] jepsen worker 74 - jepsen.util 2504 :fail :cas [24 [46 7]] :ValueNotSwapFrom INFO [2019-12-17 12:29:46,722] jepsen worker 74 - jepsen.util 2504 :invoke :cas [24 [32 18]] INFO [2019-12-17 12:29:46,722] jepsen worker 62 - jepsen.util 3392 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:29:46,722] jepsen worker 84 - jepsen.util 3324 :info :cas [28 [38 5]] :RequestCanceledException INFO [2019-12-17 12:29:46,722] jepsen worker 11 - jepsen.util 3071 :fail :cas [3 [3 15]] :ValueNotSwapFrom INFO [2019-12-17 12:29:46,723] jepsen worker 62 - jepsen.util 3392 :invoke :read [20 nil] INFO [2019-12-17 12:29:46,723] jepsen worker 20 - jepsen.util 2360 :ok :read [6 28] INFO [2019-12-17 12:29:46,723] jepsen worker 11 - jepsen.util 3071 :invoke :cas [3 [44 4]] INFO [2019-12-17 12:29:46,723] jepsen worker 20 - jepsen.util 2360 :invoke :write [6 42] INFO [2019-12-17 12:29:46,727] jepsen worker 84 - jepsen.util 3414 :invoke :cas [28 [22 39]] INFO [2019-12-17 12:29:46,736] jepsen worker 65 - jepsen.util 2315 :ok :read [21 35] INFO [2019-12-17 12:29:46,736] jepsen worker 65 - jepsen.util 2315 :invoke :cas [21 [32 46]] INFO [2019-12-17 12:29:46,739] jepsen worker 10 - jepsen.util 2440 :ok :read [3 9] INFO [2019-12-17 12:29:46,739] jepsen worker 10 - jepsen.util 2440 :invoke :write [3 6] INFO [2019-12-17 12:29:46,740] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15192,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:29:46,740] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15192,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:29:46,740] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15195,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:29:46,740] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15195,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:29:46,741] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15196,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:29:46,741] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15196,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:29:46,742] jepsen worker 69 - jepsen.util 2499 :ok :read [23 39] INFO [2019-12-17 12:29:46,742] jepsen worker 69 - jepsen.util 2499 :invoke :write [23 4] INFO [2019-12-17 12:29:46,752] jepsen worker 32 - jepsen.util 3002 :ok :write [10 14] INFO [2019-12-17 12:29:46,752] jepsen worker 32 - jepsen.util 3002 :invoke :write [10 3] INFO [2019-12-17 12:29:46,754] jepsen worker 85 - jepsen.util 3775 :info :cas [28 [6 24]] :RequestCanceledException INFO [2019-12-17 12:29:46,757] jepsen worker 85 - jepsen.util 3865 :invoke :write [28 37] INFO [2019-12-17 12:29:46,797] jepsen worker 51 - jepsen.util 2391 :ok :read [17 45] INFO [2019-12-17 12:29:46,797] jepsen worker 51 - jepsen.util 2391 :invoke :cas [17 [18 10]] INFO [2019-12-17 12:29:46,802] jepsen worker 28 - jepsen.util 2008 :fail :cas [9 [32 42]] :ValueNotSwapFrom INFO [2019-12-17 12:29:46,802] jepsen worker 28 - jepsen.util 2008 :invoke :write [9 1] INFO [2019-12-17 12:29:46,807] jepsen worker 1 - jepsen.util 2971 :info :cas [0 [24 27]] :RequestCanceledException INFO [2019-12-17 12:29:46,810] jepsen worker 1 - jepsen.util 3061 :invoke :read [0 nil] INFO [2019-12-17 12:29:46,821] jepsen worker 79 - jepsen.util 2509 :info :write [26 25] :RequestCanceledException INFO [2019-12-17 12:29:46,824] jepsen worker 79 - jepsen.util 2599 :invoke :read [26 nil] INFO [2019-12-17 12:29:46,848] jepsen worker 18 - jepsen.util 2808 :ok :read [6 28] INFO [2019-12-17 12:29:46,849] jepsen worker 18 - jepsen.util 2808 :invoke :write [6 31] INFO [2019-12-17 12:29:46,854] jepsen worker 31 - jepsen.util 2551 :fail :cas [10 [6 28]] :ValueNotSwapFrom INFO [2019-12-17 12:29:46,854] jepsen worker 31 - jepsen.util 2551 :invoke :read [10 nil] INFO [2019-12-17 12:29:46,855] jepsen worker 41 - jepsen.util 3641 :info :cas [13 [43 8]] :RequestCanceledException INFO [2019-12-17 12:29:46,856] jepsen worker 72 - jepsen.util 2682 :fail :cas [24 [38 21]] :ValueNotSwapFrom INFO [2019-12-17 12:29:46,856] jepsen worker 72 - jepsen.util 2682 :invoke :cas [24 [33 26]] INFO [2019-12-17 12:29:46,858] jepsen worker 41 - jepsen.util 3731 :invoke :read [13 nil] INFO [2019-12-17 12:29:46,880] jepsen worker 11 - jepsen.util 3071 :fail :cas [3 [44 4]] :ValueNotSwapFrom INFO [2019-12-17 12:29:46,880] jepsen worker 11 - jepsen.util 3071 :invoke :read [3 nil] INFO [2019-12-17 12:29:46,890] jepsen worker 75 - jepsen.util 3495 :info :cas [25 [18 1]] :RequestCanceledException INFO [2019-12-17 12:29:46,894] jepsen worker 64 - jepsen.util 2224 :ok :cas [21 [35 2]] INFO [2019-12-17 12:29:46,894] jepsen worker 75 - jepsen.util 3585 :invoke :write [25 49] INFO [2019-12-17 12:29:46,894] jepsen worker 64 - jepsen.util 2224 :invoke :cas [21 [38 18]] INFO [2019-12-17 12:29:46,899] jepsen worker 51 - jepsen.util 2391 :fail :cas [17 [18 10]] :ValueNotSwapFrom INFO [2019-12-17 12:29:46,899] jepsen worker 51 - jepsen.util 2391 :invoke :read [17 nil] INFO [2019-12-17 12:29:46,908] jepsen worker 83 - jepsen.util 3953 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:29:46,908] jepsen worker 83 - jepsen.util 3953 :invoke :read [27 nil] INFO [2019-12-17 12:29:46,913] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15228,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:29:46,913] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15228,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:29:46,913] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15229,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:29:46,913] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15229,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:29:46,914] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15230,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:29:46,914] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15230,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:29:46,918] jepsen worker 4 - jepsen.util 1984 :fail :cas [1 [21 33]] :ValueNotSwapFrom INFO [2019-12-17 12:29:46,918] jepsen worker 4 - jepsen.util 1984 :invoke :cas [1 [24 22]] INFO [2019-12-17 12:29:46,918] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:29:46,918] jepsen nemesis - couchbase.util Rebalance check count: 110 INFO [2019-12-17 12:29:46,918] jepsen nemesis - couchbase.util Rebalance stuck count: 91 INFO [2019-12-17 12:29:46,940] jepsen worker 64 - jepsen.util 2224 :fail :cas [21 [38 18]] :ValueNotSwapFrom INFO [2019-12-17 12:29:46,940] jepsen worker 64 - jepsen.util 2224 :invoke :cas [21 [46 19]] INFO [2019-12-17 12:29:46,962] jepsen worker 14 - jepsen.util 2444 :ok :read [4 19] INFO [2019-12-17 12:29:46,962] jepsen worker 14 - jepsen.util 2444 :invoke :read [4 nil] INFO [2019-12-17 12:29:46,977] jepsen worker 5 - jepsen.util 2255 :fail :cas [1 [25 24]] :ValueNotSwapFrom INFO [2019-12-17 12:29:46,977] jepsen worker 5 - jepsen.util 2255 :invoke :cas [1 [31 46]] INFO [2019-12-17 12:29:46,999] jepsen worker 64 - jepsen.util 2224 :fail :cas [21 [46 19]] :ValueNotSwapFrom INFO [2019-12-17 12:29:46,999] jepsen worker 64 - jepsen.util 2224 :invoke :cas [21 [20 34]] INFO [2019-12-17 12:29:47,007] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15243,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:29:47,007] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15243,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:29:47,007] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15246,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:29:47,007] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15246,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:29:47,007] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15247,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:29:47,007] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15247,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:29:47,008] jepsen worker 33 - jepsen.util 3723 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:29:47,008] jepsen worker 42 - jepsen.util 4182 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:29:47,008] jepsen worker 33 - jepsen.util 3723 :invoke :read [11 nil] INFO [2019-12-17 12:29:47,008] jepsen worker 42 - jepsen.util 4182 :invoke :read [14 nil] INFO [2019-12-17 12:29:47,022] jepsen worker 40 - jepsen.util 3730 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:29:47,022] jepsen worker 8 - jepsen.util 3068 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:29:47,022] jepsen worker 21 - jepsen.util 3351 :info :write [7 8] :RequestCanceledException INFO [2019-12-17 12:29:47,023] jepsen worker 40 - jepsen.util 3730 :invoke :cas [13 [40 48]] INFO [2019-12-17 12:29:47,023] jepsen worker 8 - jepsen.util 3068 :invoke :read [2 nil] INFO [2019-12-17 12:29:47,025] jepsen worker 21 - jepsen.util 3441 :invoke :write [7 35] INFO [2019-12-17 12:29:47,042] jepsen worker 13 - jepsen.util 2353 :ok :read [4 19] INFO [2019-12-17 12:29:47,042] jepsen worker 46 - jepsen.util 2836 :ok :write [15 46] INFO [2019-12-17 12:29:47,042] jepsen worker 13 - jepsen.util 2353 :invoke :write [4 17] INFO [2019-12-17 12:29:47,042] jepsen worker 46 - jepsen.util 2836 :invoke :read [15 nil] INFO [2019-12-17 12:29:47,057] jepsen worker 64 - jepsen.util 2224 :fail :cas [21 [20 34]] :ValueNotSwapFrom INFO [2019-12-17 12:29:47,057] jepsen worker 64 - jepsen.util 2224 :invoke :cas [21 [21 17]] INFO [2019-12-17 12:29:47,084] jepsen worker 28 - jepsen.util 2008 :ok :write [9 1] INFO [2019-12-17 12:29:47,084] jepsen worker 28 - jepsen.util 2008 :invoke :cas [9 [17 34]] INFO [2019-12-17 12:29:47,088] jepsen worker 22 - jepsen.util 3352 :info :cas [7 [13 30]] :RequestCanceledException INFO [2019-12-17 12:29:47,088] jepsen worker 17 - jepsen.util 3347 :info :cas [5 [22 39]] :RequestCanceledException INFO [2019-12-17 12:29:47,092] jepsen worker 22 - jepsen.util 3442 :invoke :write [7 37] INFO [2019-12-17 12:29:47,092] jepsen worker 17 - jepsen.util 3437 :invoke :write [5 5] INFO [2019-12-17 12:29:47,094] jepsen worker 4 - jepsen.util 1984 :fail :cas [1 [24 22]] :ValueNotSwapFrom INFO [2019-12-17 12:29:47,094] jepsen worker 4 - jepsen.util 1984 :invoke :cas [1 [37 33]] INFO [2019-12-17 12:29:47,101] jepsen worker 64 - jepsen.util 2224 :fail :cas [21 [21 17]] :ValueNotSwapFrom INFO [2019-12-17 12:29:47,101] jepsen worker 64 - jepsen.util 2224 :invoke :read [21 nil] INFO [2019-12-17 12:29:47,123] jepsen worker 0 - jepsen.util 3600 :info :write [0 41] :RequestCanceledException INFO [2019-12-17 12:29:47,123] jepsen worker 26 - jepsen.util 3716 :info :cas [8 [10 25]] :RequestCanceledException INFO [2019-12-17 12:29:47,126] jepsen worker 0 - jepsen.util 3690 :invoke :write [0 10] INFO [2019-12-17 12:29:47,126] jepsen worker 26 - jepsen.util 3806 :invoke :read [8 nil] INFO [2019-12-17 12:29:47,137] jepsen worker 57 - jepsen.util 3117 :info :write [19 26] :RequestCanceledException INFO [2019-12-17 12:29:47,137] jepsen worker 48 - jepsen.util 2838 :info :cas [16 [29 12]] :RequestCanceledException INFO [2019-12-17 12:29:47,142] jepsen worker 48 - jepsen.util 2928 :invoke :cas [16 [19 37]] INFO [2019-12-17 12:29:47,142] jepsen worker 57 - jepsen.util 3207 :invoke :cas [19 [36 14]] INFO [2019-12-17 12:29:47,636] jepsen worker 20 - jepsen.util 2360 :ok :write [6 42] INFO [2019-12-17 12:29:47,636] jepsen worker 32 - jepsen.util 3002 :ok :write [10 3] INFO [2019-12-17 12:29:47,636] jepsen worker 32 - jepsen.util 3002 :invoke :read [10 nil] INFO [2019-12-17 12:29:47,636] jepsen worker 20 - jepsen.util 2360 :invoke :cas [6 [32 37]] INFO [2019-12-17 12:29:47,638] jepsen worker 86 - jepsen.util 3416 :info :write [28 25] :RequestCanceledException INFO [2019-12-17 12:29:47,638] jepsen worker 58 - jepsen.util 3298 :info :write [19 43] :RequestCanceledException INFO [2019-12-17 12:29:47,641] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15268,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:29:47,641] jepsen worker 78 - jepsen.util 2688 :info :write [26 33] :RequestCanceledException INFO [2019-12-17 12:29:47,642] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15268,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:29:47,643] jepsen worker 77 - jepsen.util 3227 :info :write [25 29] :RequestCanceledException INFO [2019-12-17 12:29:47,643] jepsen worker 2 - jepsen.util 3332 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:29:47,643] jepsen worker 24 - jepsen.util 3354 :info :write [8 29] :RequestCanceledException INFO [2019-12-17 12:29:47,644] jepsen worker 68 - jepsen.util 3218 :info :write [22 13] :RequestCanceledException INFO [2019-12-17 12:29:47,644] jepsen worker 66 - jepsen.util 3486 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:29:47,646] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15267,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:29:47,646] jepsen worker 82 - jepsen.util 4042 :info :write [27 25] :RequestCanceledException INFO [2019-12-17 12:29:47,646] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15281,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:29:47,647] jepsen worker 2 - jepsen.util 3332 :invoke :cas [0 [5 48]] INFO [2019-12-17 12:29:47,648] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15270,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:29:47,648] jepsen worker 66 - jepsen.util 3486 :invoke :cas [22 [39 0]] INFO [2019-12-17 12:29:47,649] jepsen worker 9 - jepsen.util 2979 :info :write [3 27] :RequestCanceledException INFO [2019-12-17 12:29:47,649] jepsen worker 61 - jepsen.util 3841 :info :write [20 48] :RequestCanceledException INFO [2019-12-17 12:29:47,650] jepsen worker 38 - jepsen.util 2558 :info :cas [12 [37 12]] :RequestCanceledException INFO [2019-12-17 12:29:47,650] jepsen worker 76 - jepsen.util 3406 :info :write [25 6] :RequestCanceledException INFO [2019-12-17 12:29:47,650] jepsen worker 39 - jepsen.util 3459 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:29:47,651] jepsen worker 73 - jepsen.util 2593 :fail :read [24 nil] :RequestTimeoutException INFO [2019-12-17 12:29:47,652] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15281,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:29:47,652] jepsen worker 81 - jepsen.util 3231 :info :write [27 38] :RequestCanceledException INFO [2019-12-17 12:29:47,652] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15267,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:29:47,655] jepsen worker 73 - jepsen.util 2593 :invoke :write [24 42] INFO [2019-12-17 12:29:47,655] jepsen worker 25 - jepsen.util 3895 :info :write [8 21] :RequestCanceledException INFO [2019-12-17 12:29:47,655] jepsen worker 39 - jepsen.util 3459 :invoke :read [13 nil] INFO [2019-12-17 12:29:47,657] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15270,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:29:47,658] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15280,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:29:47,659] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15280,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:29:47,664] jepsen worker 58 - jepsen.util 3388 :invoke :cas [19 [41 11]] INFO [2019-12-17 12:29:47,664] jepsen worker 77 - jepsen.util 3317 :invoke :cas [25 [27 37]] INFO [2019-12-17 12:29:47,667] jepsen worker 82 - jepsen.util 4132 :invoke :cas [27 [36 9]] INFO [2019-12-17 12:29:47,667] jepsen worker 61 - jepsen.util 3931 :invoke :write [20 22] INFO [2019-12-17 12:29:47,667] jepsen worker 76 - jepsen.util 3496 :invoke :cas [25 [16 39]] INFO [2019-12-17 12:29:47,668] jepsen worker 86 - jepsen.util 3506 :invoke :write [28 15] INFO [2019-12-17 12:29:47,669] jepsen worker 78 - jepsen.util 2778 :invoke :write [26 15] INFO [2019-12-17 12:29:47,669] jepsen worker 68 - jepsen.util 3308 :invoke :cas [22 [48 31]] INFO [2019-12-17 12:29:47,669] jepsen worker 81 - jepsen.util 3321 :invoke :cas [27 [31 7]] INFO [2019-12-17 12:29:47,669] jepsen worker 25 - jepsen.util 3985 :invoke :cas [8 [48 33]] INFO [2019-12-17 12:29:47,669] jepsen worker 24 - jepsen.util 3444 :invoke :read [8 nil] INFO [2019-12-17 12:29:47,670] jepsen worker 9 - jepsen.util 3069 :invoke :read [3 nil] INFO [2019-12-17 12:29:47,670] jepsen worker 38 - jepsen.util 2648 :invoke :write [12 3] INFO [2019-12-17 12:29:47,714] jepsen worker 52 - jepsen.util 2752 :info :write [17 4] :RequestCanceledException INFO [2019-12-17 12:29:47,714] jepsen worker 30 - jepsen.util 3090 :info :write [10 13] :RequestCanceledException INFO [2019-12-17 12:29:47,714] jepsen worker 12 - jepsen.util 2712 :info :write [4 36] :RequestCanceledException INFO [2019-12-17 12:29:47,714] jepsen worker 27 - jepsen.util 2277 :info :cas [9 [6 24]] :RequestCanceledException INFO [2019-12-17 12:29:47,719] jepsen worker 12 - jepsen.util 2802 :invoke :write [4 41] INFO [2019-12-17 12:29:47,720] jepsen worker 52 - jepsen.util 2842 :invoke :write [17 11] INFO [2019-12-17 12:29:47,720] jepsen worker 30 - jepsen.util 3180 :invoke :read [10 nil] INFO [2019-12-17 12:29:47,720] jepsen worker 27 - jepsen.util 2367 :invoke :write [9 34] INFO [2019-12-17 12:29:47,743] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15277,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:29:47,743] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15277,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:29:47,744] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15283,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:29:47,744] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15283,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:29:47,747] jepsen worker 67 - jepsen.util 3937 :info :write [22 18] :RequestCanceledException INFO [2019-12-17 12:29:47,747] jepsen worker 56 - jepsen.util 2396 :info :write [18 43] :RequestCanceledException INFO [2019-12-17 12:29:47,754] jepsen worker 67 - jepsen.util 4027 :invoke :cas [22 [32 19]] INFO [2019-12-17 12:29:47,754] jepsen worker 56 - jepsen.util 2486 :invoke :read [18 nil] INFO [2019-12-17 12:29:47,754] jepsen worker 88 - jepsen.util 2698 :ok :read [29 29] INFO [2019-12-17 12:29:47,755] jepsen worker 88 - jepsen.util 2698 :invoke :read [29 nil] INFO [2019-12-17 12:29:47,761] jepsen worker 70 - jepsen.util 3130 :ok :read [23 39] INFO [2019-12-17 12:29:47,761] jepsen worker 70 - jepsen.util 3130 :invoke :write [23 48] INFO [2019-12-17 12:29:47,763] jepsen worker 51 - jepsen.util 2391 :ok :read [17 45] INFO [2019-12-17 12:29:47,763] jepsen worker 51 - jepsen.util 2391 :invoke :write [17 42] INFO [2019-12-17 12:29:47,764] jepsen worker 28 - jepsen.util 2008 :fail :cas [9 [17 34]] :ValueNotSwapFrom INFO [2019-12-17 12:29:47,764] jepsen worker 28 - jepsen.util 2008 :invoke :read [9 nil] INFO [2019-12-17 12:29:47,765] jepsen worker 64 - jepsen.util 2224 :ok :read [21 2] INFO [2019-12-17 12:29:47,765] jepsen worker 64 - jepsen.util 2224 :invoke :read [21 nil] INFO [2019-12-17 12:29:47,788] jepsen worker 19 - jepsen.util 2719 :info :cas [6 [37 33]] :RequestCanceledException INFO [2019-12-17 12:29:47,791] jepsen worker 19 - jepsen.util 2809 :invoke :cas [6 [32 40]] INFO [2019-12-17 12:29:47,791] jepsen worker 20 - jepsen.util 2360 :fail :cas [6 [32 37]] :ValueNotSwapFrom INFO [2019-12-17 12:29:47,792] jepsen worker 20 - jepsen.util 2360 :invoke :read [6 nil] INFO [2019-12-17 12:29:47,824] jepsen worker 9 - jepsen.util 3069 :ok :read [3 9] INFO [2019-12-17 12:29:47,825] jepsen worker 9 - jepsen.util 3069 :invoke :write [3 30] INFO [2019-12-17 12:29:47,828] jepsen worker 30 - jepsen.util 3180 :ok :read [10 3] INFO [2019-12-17 12:29:47,828] jepsen worker 30 - jepsen.util 3180 :invoke :write [10 40] INFO [2019-12-17 12:29:47,855] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15325,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:29:47,855] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15325,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:29:47,855] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15329,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:29:47,855] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15329,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:29:47,856] jepsen worker 19 - jepsen.util 2809 :fail :cas [6 [32 40]] :ValueNotSwapFrom INFO [2019-12-17 12:29:47,856] jepsen worker 19 - jepsen.util 2809 :invoke :write [6 9] INFO [2019-12-17 12:29:47,883] jepsen worker 64 - jepsen.util 2224 :ok :read [21 2] INFO [2019-12-17 12:29:47,883] jepsen worker 64 - jepsen.util 2224 :invoke :cas [21 [25 24]] INFO [2019-12-17 12:29:47,892] jepsen worker 32 - jepsen.util 3002 :ok :read [10 3] INFO [2019-12-17 12:29:47,892] jepsen worker 32 - jepsen.util 3002 :invoke :cas [10 [35 14]] INFO [2019-12-17 12:29:47,907] jepsen worker 34 - jepsen.util 3634 :info :cas [11 [15 26]] :RequestCanceledException INFO [2019-12-17 12:29:47,909] jepsen worker 34 - jepsen.util 3724 :invoke :write [11 15] INFO [2019-12-17 12:29:47,920] jepsen worker 49 - jepsen.util 2839 :info :cas [16 [33 5]] :RequestCanceledException INFO [2019-12-17 12:29:47,920] jepsen worker 60 - jepsen.util 3660 :info :cas [20 [22 33]] :RequestCanceledException INFO [2019-12-17 12:29:47,920] jepsen worker 44 - jepsen.util 3374 :info :write [14 41] :RequestCanceledException INFO [2019-12-17 12:29:47,923] jepsen worker 44 - jepsen.util 3464 :invoke :cas [14 [11 4]] INFO [2019-12-17 12:29:47,923] jepsen worker 49 - jepsen.util 2929 :invoke :read [16 nil] INFO [2019-12-17 12:29:47,923] jepsen worker 60 - jepsen.util 3750 :invoke :cas [20 [42 20]] INFO [2019-12-17 12:29:47,933] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:29:47,933] jepsen nemesis - couchbase.util Rebalance check count: 111 INFO [2019-12-17 12:29:47,933] jepsen nemesis - couchbase.util Rebalance stuck count: 92 INFO [2019-12-17 12:29:47,937] jepsen worker 7 - jepsen.util 3427 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:29:47,937] jepsen worker 7 - jepsen.util 3427 :invoke :read [2 nil] INFO [2019-12-17 12:29:47,952] jepsen worker 89 - jepsen.util 2519 :ok :write [29 15] INFO [2019-12-17 12:29:47,952] jepsen worker 89 - jepsen.util 2519 :invoke :read [29 nil] INFO [2019-12-17 12:29:47,955] jepsen worker 80 - jepsen.util 2600 :ok :write [26 30] INFO [2019-12-17 12:29:47,955] jepsen worker 80 - jepsen.util 2600 :invoke :read [26 nil] INFO [2019-12-17 12:29:47,962] jepsen worker 28 - jepsen.util 2008 :ok :read [9 1] INFO [2019-12-17 12:29:47,962] jepsen worker 28 - jepsen.util 2008 :invoke :read [9 nil] INFO [2019-12-17 12:29:47,983] jepsen worker 12 - jepsen.util 2802 :ok :write [4 41] INFO [2019-12-17 12:29:47,984] jepsen worker 12 - jepsen.util 2802 :invoke :write [4 2] INFO [2019-12-17 12:29:47,989] jepsen worker 23 - jepsen.util 3353 :info :cas [7 [45 1]] :RequestCanceledException INFO [2019-12-17 12:29:47,990] jepsen worker 6 - jepsen.util 3606 :info :write [2 3] :RequestCanceledException INFO [2019-12-17 12:29:47,992] jepsen worker 23 - jepsen.util 3443 :invoke :write [7 25] INFO [2019-12-17 12:29:47,992] jepsen worker 6 - jepsen.util 3696 :invoke :cas [2 [5 29]] INFO [2019-12-17 12:29:48,007] jepsen worker 27 - jepsen.util 2367 :ok :write [9 34] INFO [2019-12-17 12:29:48,007] jepsen worker 27 - jepsen.util 2367 :invoke :cas [9 [6 1]] INFO [2019-12-17 12:29:48,019] jepsen worker 79 - jepsen.util 2599 :ok :read [26 30] INFO [2019-12-17 12:29:48,019] jepsen worker 79 - jepsen.util 2599 :invoke :read [26 nil] INFO [2019-12-17 12:29:48,026] jepsen worker 28 - jepsen.util 2008 :ok :read [9 1] INFO [2019-12-17 12:29:48,026] jepsen worker 28 - jepsen.util 2008 :invoke :read [9 nil] INFO [2019-12-17 12:29:48,040] jepsen worker 27 - jepsen.util 2367 :fail :cas [9 [6 1]] :ValueNotSwapFrom INFO [2019-12-17 12:29:48,040] jepsen worker 27 - jepsen.util 2367 :invoke :cas [9 [17 23]] INFO [2019-12-17 12:29:48,064] jepsen worker 14 - jepsen.util 2444 :ok :read [4 41] INFO [2019-12-17 12:29:48,065] jepsen worker 14 - jepsen.util 2444 :invoke :read [4 nil] INFO [2019-12-17 12:29:48,066] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15341,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:29:48,067] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15341,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:29:48,067] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15342,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:29:48,067] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15342,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:29:48,069] jepsen worker 28 - jepsen.util 2008 :ok :read [9 34] INFO [2019-12-17 12:29:48,069] jepsen worker 28 - jepsen.util 2008 :invoke :read [9 nil] INFO [2019-12-17 12:29:48,082] jepsen worker 27 - jepsen.util 2367 :fail :cas [9 [17 23]] :ValueNotSwapFrom INFO [2019-12-17 12:29:48,082] jepsen worker 27 - jepsen.util 2367 :invoke :read [9 nil] INFO [2019-12-17 12:29:48,093] jepsen worker 5 - jepsen.util 2255 :fail :cas [1 [31 46]] :ValueNotSwapFrom INFO [2019-12-17 12:29:48,094] jepsen worker 5 - jepsen.util 2255 :invoke :read [1 nil] INFO [2019-12-17 12:29:48,094] jepsen worker 13 - jepsen.util 2353 :ok :write [4 17] INFO [2019-12-17 12:29:48,094] jepsen worker 13 - jepsen.util 2353 :invoke :write [4 17] INFO [2019-12-17 12:29:48,116] jepsen worker 27 - jepsen.util 2367 :ok :read [9 34] INFO [2019-12-17 12:29:48,116] jepsen worker 27 - jepsen.util 2367 :invoke :cas [9 [19 46]] INFO [2019-12-17 12:29:48,116] jepsen worker 28 - jepsen.util 2008 :ok :read [9 34] INFO [2019-12-17 12:29:48,117] jepsen worker 28 - jepsen.util 2008 :invoke :read [9 nil] INFO [2019-12-17 12:29:48,123] jepsen worker 63 - jepsen.util 2313 :fail :read [21 nil] :RequestTimeoutException INFO [2019-12-17 12:29:48,123] jepsen worker 63 - jepsen.util 2313 :invoke :read [21 nil] INFO [2019-12-17 12:29:48,127] jepsen worker 3 - jepsen.util 2523 :ok :write [1 15] INFO [2019-12-17 12:29:48,128] jepsen worker 3 - jepsen.util 2523 :invoke :cas [1 [14 42]] INFO [2019-12-17 12:29:48,156] jepsen worker 27 - jepsen.util 2367 :fail :cas [9 [19 46]] :ValueNotSwapFrom INFO [2019-12-17 12:29:48,156] jepsen worker 27 - jepsen.util 2367 :invoke :read [9 nil] INFO [2019-12-17 12:29:48,170] jepsen worker 28 - jepsen.util 2008 :ok :read [9 34] INFO [2019-12-17 12:29:48,171] jepsen worker 28 - jepsen.util 2008 :invoke :write [9 11] INFO [2019-12-17 12:29:48,181] jepsen worker 63 - jepsen.util 2313 :ok :read [21 2] INFO [2019-12-17 12:29:48,181] jepsen worker 63 - jepsen.util 2313 :invoke :write [21 46] INFO [2019-12-17 12:29:48,201] jepsen worker 27 - jepsen.util 2367 :ok :read [9 34] INFO [2019-12-17 12:29:48,201] jepsen worker 27 - jepsen.util 2367 :invoke :cas [9 [1 46]] INFO [2019-12-17 12:29:48,218] jepsen worker 70 - jepsen.util 3130 :ok :write [23 48] INFO [2019-12-17 12:29:48,219] jepsen worker 70 - jepsen.util 3130 :invoke :cas [23 [23 37]] INFO [2019-12-17 12:29:48,232] jepsen worker 27 - jepsen.util 2367 :fail :cas [9 [1 46]] :ValueNotSwapFrom INFO [2019-12-17 12:29:48,232] jepsen worker 27 - jepsen.util 2367 :invoke :write [9 28] INFO [2019-12-17 12:29:48,247] jepsen worker 52 - jepsen.util 2842 :ok :write [17 11] INFO [2019-12-17 12:29:48,247] jepsen worker 52 - jepsen.util 2842 :invoke :read [17 nil] INFO [2019-12-17 12:29:48,260] jepsen worker 28 - jepsen.util 2008 :ok :write [9 11] INFO [2019-12-17 12:29:48,260] jepsen worker 28 - jepsen.util 2008 :invoke :write [9 12] INFO [2019-12-17 12:29:48,286] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15366,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:29:48,286] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15366,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:29:48,286] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15368,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:29:48,286] jepsen worker 79 - jepsen.util 2599 :ok :read [26 30] INFO [2019-12-17 12:29:48,286] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15368,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:29:48,286] jepsen worker 79 - jepsen.util 2599 :invoke :cas [26 [19 1]] INFO [2019-12-17 12:29:48,309] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15374,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:29:48,309] jepsen worker 9 - jepsen.util 3069 :ok :write [3 30] INFO [2019-12-17 12:29:48,309] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15374,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:29:48,309] jepsen worker 9 - jepsen.util 3069 :invoke :write [3 38] INFO [2019-12-17 12:29:48,309] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15378,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:29:48,309] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15378,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:29:48,311] jepsen worker 80 - jepsen.util 2600 :ok :read [26 30] INFO [2019-12-17 12:29:48,311] jepsen worker 80 - jepsen.util 2600 :invoke :read [26 nil] INFO [2019-12-17 12:29:48,311] jepsen worker 19 - jepsen.util 2809 :ok :write [6 9] INFO [2019-12-17 12:29:48,311] jepsen worker 19 - jepsen.util 2809 :invoke :write [6 38] INFO [2019-12-17 12:29:48,312] jepsen worker 30 - jepsen.util 3180 :ok :write [10 40] INFO [2019-12-17 12:29:48,312] jepsen worker 30 - jepsen.util 3180 :invoke :write [10 27] INFO [2019-12-17 12:29:48,321] jepsen worker 37 - jepsen.util 2287 :info :write [12 16] :RequestCanceledException INFO [2019-12-17 12:29:48,327] jepsen worker 37 - jepsen.util 2377 :invoke :write [12 30] INFO [2019-12-17 12:29:48,342] jepsen worker 14 - jepsen.util 2444 :ok :read [4 17] INFO [2019-12-17 12:29:48,343] jepsen worker 14 - jepsen.util 2444 :invoke :write [4 6] INFO [2019-12-17 12:29:48,348] jepsen worker 87 - jepsen.util 2337 :info :write [29 29] :RequestCanceledException INFO [2019-12-17 12:29:48,352] jepsen worker 87 - jepsen.util 2427 :invoke :cas [29 [28 7]] INFO [2019-12-17 12:29:48,371] jepsen worker 64 - jepsen.util 2224 :fail :cas [21 [25 24]] :ValueNotSwapFrom INFO [2019-12-17 12:29:48,371] jepsen worker 64 - jepsen.util 2224 :invoke :read [21 nil] INFO [2019-12-17 12:29:48,373] jepsen worker 12 - jepsen.util 2802 :ok :write [4 2] INFO [2019-12-17 12:29:48,373] jepsen worker 12 - jepsen.util 2802 :invoke :read [4 nil] INFO [2019-12-17 12:29:48,375] jepsen worker 73 - jepsen.util 2593 :ok :write [24 42] INFO [2019-12-17 12:29:48,376] jepsen worker 73 - jepsen.util 2593 :invoke :write [24 2] INFO [2019-12-17 12:29:48,380] jepsen worker 5 - jepsen.util 2255 :ok :read [1 15] INFO [2019-12-17 12:29:48,380] jepsen worker 5 - jepsen.util 2255 :invoke :read [1 nil] INFO [2019-12-17 12:29:48,411] jepsen worker 52 - jepsen.util 2842 :ok :read [17 28] INFO [2019-12-17 12:29:48,423] jepsen worker 79 - jepsen.util 2599 :fail :cas [26 [19 1]] :ValueNotSwapFrom INFO [2019-12-17 12:29:48,435] jepsen worker 70 - jepsen.util 3130 :fail :cas [23 [23 37]] :ValueNotSwapFrom INFO [2019-12-17 12:29:48,437] jepsen worker 55 - jepsen.util 3115 :info :write [18 25] :RequestCanceledException INFO [2019-12-17 12:29:48,438] jepsen worker 71 - jepsen.util 2681 :ok :write [23 17] INFO [2019-12-17 12:29:48,443] jepsen worker 53 - jepsen.util 2663 :ok :write [17 28] INFO [2019-12-17 12:29:48,443] jepsen worker 27 - jepsen.util 2367 :ok :write [9 28] INFO [2019-12-17 12:29:48,475] jepsen worker 80 - jepsen.util 2600 :ok :read [26 30] INFO [2019-12-17 12:29:48,490] jepsen worker 54 - jepsen.util 2574 :info :write [18 41] :RequestCanceledException INFO [2019-12-17 12:29:48,501] jepsen worker 12 - jepsen.util 2802 :ok :read [4 2] INFO [2019-12-17 12:29:48,504] jepsen worker 52 - jepsen.util 2842 :invoke :read [17 nil] INFO [2019-12-17 12:29:48,504] jepsen worker 79 - jepsen.util 2599 :invoke :cas [26 [3 44]] INFO [2019-12-17 12:29:48,504] jepsen worker 70 - jepsen.util 3130 :invoke :write [23 41] INFO [2019-12-17 12:29:48,504] jepsen worker 71 - jepsen.util 2681 :invoke :cas [23 [0 48]] INFO [2019-12-17 12:29:48,505] jepsen worker 53 - jepsen.util 2663 :invoke :cas [17 [28 6]] INFO [2019-12-17 12:29:48,505] jepsen worker 80 - jepsen.util 2600 :invoke :read [26 nil] INFO [2019-12-17 12:29:48,505] jepsen worker 12 - jepsen.util 2802 :invoke :cas [4 [23 25]] INFO [2019-12-17 12:29:48,505] jepsen worker 27 - jepsen.util 2367 :invoke :cas [9 [27 38]] INFO [2019-12-17 12:29:48,507] jepsen worker 54 - jepsen.util 2664 :invoke :write [18 15] INFO [2019-12-17 12:29:48,507] jepsen worker 55 - jepsen.util 3205 :invoke :read [18 nil] INFO [2019-12-17 12:29:48,508] jepsen worker 16 - jepsen.util 3526 :info :write [5 2] :RequestCanceledException INFO [2019-12-17 12:29:48,511] jepsen worker 16 - jepsen.util 3616 :invoke :read [5 nil] INFO [2019-12-17 12:29:48,523] jepsen worker 35 - jepsen.util 3365 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:29:48,523] jepsen worker 35 - jepsen.util 3365 :invoke :read [11 nil] INFO [2019-12-17 12:29:48,527] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15395,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:29:48,527] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15395,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:29:48,527] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15400,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:29:48,528] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15400,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:29:48,551] jepsen worker 27 - jepsen.util 2367 :fail :cas [9 [27 38]] :ValueNotSwapFrom INFO [2019-12-17 12:29:48,551] jepsen worker 27 - jepsen.util 2367 :invoke :read [9 nil] INFO [2019-12-17 12:29:48,552] jepsen worker 12 - jepsen.util 2802 :fail :cas [4 [23 25]] :ValueNotSwapFrom INFO [2019-12-17 12:29:48,552] jepsen worker 12 - jepsen.util 2802 :invoke :write [4 12] INFO [2019-12-17 12:29:48,577] jepsen worker 27 - jepsen.util 2367 :ok :read [9 28] INFO [2019-12-17 12:29:48,577] jepsen worker 27 - jepsen.util 2367 :invoke :write [9 18] INFO [2019-12-17 12:29:48,619] jepsen worker 36 - jepsen.util 2466 :ok :read [12 15] INFO [2019-12-17 12:29:48,620] jepsen worker 36 - jepsen.util 2466 :invoke :read [12 nil] INFO [2019-12-17 12:29:48,637] jepsen worker 45 - jepsen.util 2745 :info :write [15 1] :RequestCanceledException INFO [2019-12-17 12:29:48,638] jepsen worker 45 - jepsen.util 2835 :invoke :write [15 43] INFO [2019-12-17 12:29:48,657] jepsen worker 70 - jepsen.util 3130 :ok :write [23 41] INFO [2019-12-17 12:29:48,658] jepsen worker 70 - jepsen.util 3130 :invoke :write [23 4] INFO [2019-12-17 12:29:48,662] jepsen worker 37 - jepsen.util 2377 :ok :write [12 30] INFO [2019-12-17 12:29:48,663] jepsen worker 37 - jepsen.util 2377 :invoke :cas [12 [36 27]] INFO [2019-12-17 12:29:48,671] jepsen worker 27 - jepsen.util 2367 :ok :write [9 18] INFO [2019-12-17 12:29:48,672] jepsen worker 27 - jepsen.util 2367 :invoke :read [9 nil] INFO [2019-12-17 12:29:48,700] jepsen worker 4 - jepsen.util 1984 :fail :cas [1 [37 33]] :ValueNotSwapFrom INFO [2019-12-17 12:29:48,700] jepsen worker 4 - jepsen.util 1984 :invoke :cas [1 [1 41]] INFO [2019-12-17 12:29:48,720] jepsen worker 15 - jepsen.util 3165 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:29:48,720] jepsen worker 15 - jepsen.util 3165 :invoke :write [5 25] INFO [2019-12-17 12:29:48,723] jepsen worker 52 - jepsen.util 2842 :ok :read [17 28] INFO [2019-12-17 12:29:48,723] jepsen worker 52 - jepsen.util 2842 :invoke :read [17 nil] INFO [2019-12-17 12:29:48,728] jepsen worker 27 - jepsen.util 2367 :ok :read [9 18] INFO [2019-12-17 12:29:48,728] jepsen worker 27 - jepsen.util 2367 :invoke :cas [9 [5 42]] INFO [2019-12-17 12:29:48,738] jepsen worker 59 - jepsen.util 3479 :info :cas [19 [24 9]] :RequestCanceledException INFO [2019-12-17 12:29:48,738] jepsen worker 74 - jepsen.util 2504 :info :cas [24 [32 18]] :RequestCanceledException INFO [2019-12-17 12:29:48,738] jepsen worker 47 - jepsen.util 2927 :info :cas [15 [36 37]] :RequestCanceledException INFO [2019-12-17 12:29:48,740] jepsen worker 74 - jepsen.util 2594 :invoke :write [24 28] INFO [2019-12-17 12:29:48,740] jepsen worker 47 - jepsen.util 3017 :invoke :write [15 14] INFO [2019-12-17 12:29:48,740] jepsen worker 59 - jepsen.util 3569 :invoke :read [19 nil] INFO [2019-12-17 12:29:48,740] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15418,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:29:48,741] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15418,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:29:48,741] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15421,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:29:48,741] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15421,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:29:48,741] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15422,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:29:48,741] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15422,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:29:48,754] jepsen worker 43 - jepsen.util 3103 :info :cas [14 [23 13]] :RequestCanceledException INFO [2019-12-17 12:29:48,759] jepsen worker 43 - jepsen.util 3193 :invoke :cas [14 [10 20]] INFO [2019-12-17 12:29:48,776] jepsen worker 27 - jepsen.util 2367 :fail :cas [9 [5 42]] :ValueNotSwapFrom INFO [2019-12-17 12:29:48,776] jepsen worker 27 - jepsen.util 2367 :invoke :read [9 nil] INFO [2019-12-17 12:29:48,780] jepsen worker 88 - jepsen.util 2698 :ok :read [29 15] INFO [2019-12-17 12:29:48,780] jepsen worker 88 - jepsen.util 2698 :invoke :read [29 nil] INFO [2019-12-17 12:29:48,790] jepsen worker 69 - jepsen.util 2499 :info :write [23 4] :RequestCanceledException INFO [2019-12-17 12:29:48,790] jepsen worker 85 - jepsen.util 3865 :info :write [28 37] :RequestCanceledException INFO [2019-12-17 12:29:48,792] jepsen worker 85 - jepsen.util 3955 :invoke :write [28 17] INFO [2019-12-17 12:29:48,792] jepsen worker 69 - jepsen.util 2589 :invoke :cas [23 [37 13]] INFO [2019-12-17 12:29:48,804] jepsen worker 51 - jepsen.util 2391 :ok :write [17 42] INFO [2019-12-17 12:29:48,805] jepsen worker 51 - jepsen.util 2391 :invoke :write [17 22] INFO [2019-12-17 12:29:48,808] jepsen worker 50 - jepsen.util 3830 :info :write [16 42] :RequestCanceledException INFO [2019-12-17 12:29:48,810] jepsen worker 50 - jepsen.util 3920 :invoke :cas [16 [42 12]] INFO [2019-12-17 12:29:48,819] jepsen worker 27 - jepsen.util 2367 :ok :read [9 18] INFO [2019-12-17 12:29:48,819] jepsen worker 27 - jepsen.util 2367 :invoke :write [9 26] INFO [2019-12-17 12:29:48,821] jepsen worker 12 - jepsen.util 2802 :ok :write [4 12] INFO [2019-12-17 12:29:48,821] jepsen worker 12 - jepsen.util 2802 :invoke :read [4 nil] INFO [2019-12-17 12:29:48,823] jepsen worker 84 - jepsen.util 3414 :info :cas [28 [22 39]] :RequestCanceledException INFO [2019-12-17 12:29:48,824] jepsen worker 62 - jepsen.util 3392 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:29:48,824] jepsen worker 62 - jepsen.util 3392 :invoke :read [20 nil] INFO [2019-12-17 12:29:48,826] jepsen worker 84 - jepsen.util 3504 :invoke :read [28 nil] INFO [2019-12-17 12:29:48,837] jepsen worker 10 - jepsen.util 2440 :info :write [3 6] :RequestCanceledException INFO [2019-12-17 12:29:48,837] jepsen worker 65 - jepsen.util 2315 :info :cas [21 [32 46]] :RequestCanceledException INFO [2019-12-17 12:29:48,839] jepsen worker 65 - jepsen.util 2405 :invoke :read [21 nil] INFO [2019-12-17 12:29:48,839] jepsen worker 10 - jepsen.util 2530 :invoke :write [3 11] INFO [2019-12-17 12:29:48,840] jepsen worker 87 - jepsen.util 2427 :fail :cas [29 [28 7]] :ValueNotSwapFrom INFO [2019-12-17 12:29:48,840] jepsen worker 87 - jepsen.util 2427 :invoke :cas [29 [13 30]] INFO [2019-12-17 12:29:48,855] jepsen worker 29 - jepsen.util 2189 :info :cas [9 [49 39]] :RequestCanceledException INFO [2019-12-17 12:29:48,857] jepsen worker 29 - jepsen.util 2279 :invoke :read [9 nil] INFO [2019-12-17 12:29:48,865] jepsen worker 3 - jepsen.util 2523 :fail :cas [1 [14 42]] :ValueNotSwapFrom INFO [2019-12-17 12:29:48,865] jepsen worker 3 - jepsen.util 2523 :invoke :read [1 nil] INFO [2019-12-17 12:29:48,890] jepsen worker 31 - jepsen.util 2551 :fail :read [10 nil] :RequestTimeoutException INFO [2019-12-17 12:29:48,890] jepsen worker 31 - jepsen.util 2551 :invoke :cas [10 [0 33]] INFO [2019-12-17 12:29:48,905] jepsen worker 1 - jepsen.util 3061 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:29:48,905] jepsen worker 1 - jepsen.util 3061 :invoke :read [0 nil] INFO [2019-12-17 12:29:48,927] jepsen worker 79 - jepsen.util 2599 :fail :cas [26 [3 44]] :ValueNotSwapFrom INFO [2019-12-17 12:29:48,927] jepsen worker 79 - jepsen.util 2599 :invoke :write [26 25] INFO [2019-12-17 12:29:48,931] jepsen worker 87 - jepsen.util 2427 :fail :cas [29 [13 30]] :ValueNotSwapFrom INFO [2019-12-17 12:29:48,931] jepsen worker 87 - jepsen.util 2427 :invoke :read [29 nil] INFO [2019-12-17 12:29:48,937] jepsen worker 55 - jepsen.util 3205 :ok :read [18 29] INFO [2019-12-17 12:29:48,937] jepsen worker 41 - jepsen.util 3731 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:29:48,937] jepsen worker 18 - jepsen.util 2808 :info :write [6 31] :RequestCanceledException INFO [2019-12-17 12:29:48,937] jepsen worker 55 - jepsen.util 3205 :invoke :read [18 nil] INFO [2019-12-17 12:29:48,937] jepsen worker 41 - jepsen.util 3731 :invoke :cas [13 [39 2]] INFO [2019-12-17 12:29:48,940] jepsen worker 18 - jepsen.util 2898 :invoke :read [6 nil] INFO [2019-12-17 12:29:48,944] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15449,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:29:48,944] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15449,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:29:48,944] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15452,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:29:48,944] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15452,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:29:48,953] jepsen worker 80 - jepsen.util 2600 :ok :read [26 33] INFO [2019-12-17 12:29:48,953] jepsen worker 80 - jepsen.util 2600 :invoke :cas [26 [34 32]] INFO [2019-12-17 12:29:48,954] jepsen worker 11 - jepsen.util 3071 :fail :read [3 nil] :RequestTimeoutException INFO [2019-12-17 12:29:48,954] jepsen worker 72 - jepsen.util 2682 :info :cas [24 [33 26]] :RequestCanceledException INFO [2019-12-17 12:29:48,954] jepsen worker 75 - jepsen.util 3585 :info :write [25 49] :RequestCanceledException INFO [2019-12-17 12:29:48,954] jepsen worker 11 - jepsen.util 3071 :invoke :cas [3 [17 0]] INFO [2019-12-17 12:29:48,957] jepsen worker 72 - jepsen.util 2772 :invoke :cas [24 [22 19]] INFO [2019-12-17 12:29:48,957] jepsen worker 75 - jepsen.util 3675 :invoke :read [25 nil] INFO [2019-12-17 12:29:48,965] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:29:48,965] jepsen nemesis - couchbase.util Rebalance check count: 112 INFO [2019-12-17 12:29:48,965] jepsen nemesis - couchbase.util Rebalance stuck count: 93 INFO [2019-12-17 12:29:49,007] jepsen worker 83 - jepsen.util 3953 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:29:49,008] jepsen worker 83 - jepsen.util 3953 :invoke :read [27 nil] INFO [2019-12-17 12:29:49,008] jepsen worker 27 - jepsen.util 2367 :ok :write [9 26] INFO [2019-12-17 12:29:49,008] jepsen worker 27 - jepsen.util 2367 :invoke :cas [9 [45 17]] INFO [2019-12-17 12:29:49,016] jepsen worker 72 - jepsen.util 2772 :fail :cas [24 [22 19]] :ValueNotSwapFrom INFO [2019-12-17 12:29:49,016] jepsen worker 72 - jepsen.util 2772 :invoke :read [24 nil] INFO [2019-12-17 12:29:49,038] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15467,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:29:49,038] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15467,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:29:49,039] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15469,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:29:49,039] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15469,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:29:49,041] jepsen worker 71 - jepsen.util 2681 :fail :cas [23 [0 48]] :ValueNotSwapFrom INFO [2019-12-17 12:29:49,041] jepsen worker 71 - jepsen.util 2681 :invoke :write [23 5] INFO [2019-12-17 12:29:49,052] jepsen worker 46 - jepsen.util 2836 :fail :read [15 nil] :RequestTimeoutException INFO [2019-12-17 12:29:49,052] jepsen worker 46 - jepsen.util 2836 :invoke :write [15 11] INFO [2019-12-17 12:29:49,082] jepsen worker 72 - jepsen.util 2772 :ok :read [24 42] INFO [2019-12-17 12:29:49,082] jepsen worker 72 - jepsen.util 2772 :invoke :write [24 46] INFO [2019-12-17 12:29:49,087] jepsen worker 11 - jepsen.util 3071 :fail :cas [3 [17 0]] :ValueNotSwapFrom INFO [2019-12-17 12:29:49,087] jepsen worker 11 - jepsen.util 3071 :invoke :read [3 nil] INFO [2019-12-17 12:29:49,103] jepsen worker 27 - jepsen.util 2367 :fail :cas [9 [45 17]] :ValueNotSwapFrom INFO [2019-12-17 12:29:49,104] jepsen worker 27 - jepsen.util 2367 :invoke :write [9 36] INFO [2019-12-17 12:29:49,107] jepsen worker 36 - jepsen.util 2466 :ok :read [12 30] INFO [2019-12-17 12:29:49,108] jepsen worker 36 - jepsen.util 2466 :invoke :write [12 19] INFO [2019-12-17 12:29:49,108] jepsen worker 33 - jepsen.util 3723 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:29:49,108] jepsen worker 42 - jepsen.util 4182 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:29:49,108] jepsen worker 33 - jepsen.util 3723 :invoke :write [11 42] INFO [2019-12-17 12:29:49,109] jepsen worker 42 - jepsen.util 4182 :invoke :read [14 nil] INFO [2019-12-17 12:29:49,119] jepsen worker 73 - jepsen.util 2593 :ok :write [24 2] INFO [2019-12-17 12:29:49,120] jepsen worker 73 - jepsen.util 2593 :invoke :read [24 nil] INFO [2019-12-17 12:29:49,122] jepsen worker 30 - jepsen.util 3180 :ok :write [10 27] INFO [2019-12-17 12:29:49,122] jepsen worker 30 - jepsen.util 3180 :invoke :read [10 nil] INFO [2019-12-17 12:29:49,123] jepsen worker 21 - jepsen.util 3441 :info :write [7 35] :RequestCanceledException INFO [2019-12-17 12:29:49,123] jepsen worker 40 - jepsen.util 3730 :info :cas [13 [40 48]] :RequestCanceledException INFO [2019-12-17 12:29:49,123] jepsen worker 14 - jepsen.util 2444 :ok :write [4 6] INFO [2019-12-17 12:29:49,123] jepsen worker 14 - jepsen.util 2444 :invoke :read [4 nil] INFO [2019-12-17 12:29:49,123] jepsen worker 8 - jepsen.util 3068 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:29:49,124] jepsen worker 8 - jepsen.util 3068 :invoke :cas [2 [34 35]] INFO [2019-12-17 12:29:49,125] jepsen worker 9 - jepsen.util 3069 :ok :write [3 38] INFO [2019-12-17 12:29:49,125] jepsen worker 9 - jepsen.util 3069 :invoke :cas [3 [32 46]] INFO [2019-12-17 12:29:49,126] jepsen worker 40 - jepsen.util 3820 :invoke :cas [13 [38 30]] INFO [2019-12-17 12:29:49,126] jepsen worker 21 - jepsen.util 3531 :invoke :write [7 23] INFO [2019-12-17 12:29:49,137] jepsen worker 26 - jepsen.util 3806 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:29:49,138] jepsen worker 26 - jepsen.util 3806 :invoke :cas [8 [2 47]] INFO [2019-12-17 12:29:49,144] jepsen worker 89 - jepsen.util 2519 :ok :read [29 29] INFO [2019-12-17 12:29:49,144] jepsen worker 89 - jepsen.util 2519 :invoke :read [29 nil] INFO [2019-12-17 12:29:49,154] jepsen worker 57 - jepsen.util 3207 :info :cas [19 [36 14]] :RequestCanceledException INFO [2019-12-17 12:29:49,156] jepsen worker 57 - jepsen.util 3297 :invoke :write [19 18] INFO [2019-12-17 12:29:49,189] jepsen worker 29 - jepsen.util 2279 :ok :read [9 26] INFO [2019-12-17 12:29:49,189] jepsen worker 29 - jepsen.util 2279 :invoke :read [9 nil] INFO [2019-12-17 12:29:49,190] jepsen worker 22 - jepsen.util 3442 :info :write [7 37] :RequestCanceledException INFO [2019-12-17 12:29:49,193] jepsen worker 22 - jepsen.util 3532 :invoke :cas [7 [39 44]] INFO [2019-12-17 12:29:49,207] jepsen worker 17 - jepsen.util 3437 :info :write [5 5] :RequestCanceledException INFO [2019-12-17 12:29:49,209] jepsen worker 17 - jepsen.util 3527 :invoke :write [5 23] INFO [2019-12-17 12:29:49,223] jepsen worker 0 - jepsen.util 3690 :info :write [0 10] :RequestCanceledException INFO [2019-12-17 12:29:49,223] jepsen worker 48 - jepsen.util 2928 :info :cas [16 [19 37]] :RequestCanceledException INFO [2019-12-17 12:29:49,227] jepsen worker 0 - jepsen.util 3780 :invoke :cas [0 [35 39]] INFO [2019-12-17 12:29:49,227] jepsen worker 48 - jepsen.util 3018 :invoke :write [16 32] INFO [2019-12-17 12:29:49,239] jepsen worker 31 - jepsen.util 2551 :fail :cas [10 [0 33]] :ValueNotSwapFrom INFO [2019-12-17 12:29:49,239] jepsen worker 31 - jepsen.util 2551 :invoke :cas [10 [24 20]] INFO [2019-12-17 12:29:49,251] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15496,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:29:49,251] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15496,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:29:49,251] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15497,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:29:49,251] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15497,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:29:49,252] jepsen worker 5 - jepsen.util 2255 :ok :read [1 15] INFO [2019-12-17 12:29:49,252] jepsen worker 5 - jepsen.util 2255 :invoke :cas [1 [23 0]] INFO [2019-12-17 12:29:49,267] jepsen worker 73 - jepsen.util 2593 :ok :read [24 2] INFO [2019-12-17 12:29:49,267] jepsen worker 73 - jepsen.util 2593 :invoke :read [24 nil] INFO [2019-12-17 12:29:49,271] jepsen worker 37 - jepsen.util 2377 :fail :cas [12 [36 27]] :ValueNotSwapFrom INFO [2019-12-17 12:29:49,272] jepsen worker 37 - jepsen.util 2377 :invoke :read [12 nil] INFO [2019-12-17 12:29:49,272] jepsen worker 29 - jepsen.util 2279 :ok :read [9 26] INFO [2019-12-17 12:29:49,272] jepsen worker 29 - jepsen.util 2279 :invoke :cas [9 [3 29]] INFO [2019-12-17 12:29:49,275] jepsen worker 31 - jepsen.util 2551 :fail :cas [10 [24 20]] :ValueNotSwapFrom INFO [2019-12-17 12:29:49,276] jepsen worker 31 - jepsen.util 2551 :invoke :read [10 nil] INFO [2019-12-17 12:29:49,283] jepsen worker 87 - jepsen.util 2427 :ok :read [29 29] INFO [2019-12-17 12:29:49,284] jepsen worker 87 - jepsen.util 2427 :invoke :cas [29 [2 44]] INFO [2019-12-17 12:29:49,286] jepsen worker 73 - jepsen.util 2593 :ok :read [24 2] INFO [2019-12-17 12:29:49,286] jepsen worker 73 - jepsen.util 2593 :invoke :write [24 15] INFO [2019-12-17 12:29:49,290] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15504,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:29:49,290] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15504,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:29:49,303] jepsen worker 29 - jepsen.util 2279 :fail :cas [9 [3 29]] :ValueNotSwapFrom INFO [2019-12-17 12:29:49,303] jepsen worker 29 - jepsen.util 2279 :invoke :write [9 11] INFO [2019-12-17 12:29:49,339] jepsen worker 31 - jepsen.util 2551 :ok :read [10 27] INFO [2019-12-17 12:29:49,339] jepsen worker 31 - jepsen.util 2551 :invoke :read [10 nil] INFO [2019-12-17 12:29:49,347] jepsen worker 69 - jepsen.util 2589 :fail :cas [23 [37 13]] :ValueNotSwapFrom INFO [2019-12-17 12:29:49,347] jepsen worker 69 - jepsen.util 2589 :invoke :cas [23 [35 23]] INFO [2019-12-17 12:29:49,370] jepsen worker 3 - jepsen.util 2523 :ok :read [1 15] INFO [2019-12-17 12:29:49,370] jepsen worker 3 - jepsen.util 2523 :invoke :write [1 4] INFO [2019-12-17 12:29:49,404] jepsen worker 19 - jepsen.util 2809 :ok :write [6 38] INFO [2019-12-17 12:29:49,405] jepsen worker 19 - jepsen.util 2809 :invoke :cas [6 [8 2]] INFO [2019-12-17 12:29:49,406] jepsen worker 46 - jepsen.util 2836 :ok :write [15 11] INFO [2019-12-17 12:29:49,406] jepsen worker 46 - jepsen.util 2836 :invoke :write [15 36] INFO [2019-12-17 12:29:49,410] jepsen worker 56 - jepsen.util 2486 :ok :read [18 29] INFO [2019-12-17 12:29:49,411] jepsen worker 56 - jepsen.util 2486 :invoke :write [18 5] INFO [2019-12-17 12:29:49,424] jepsen worker 55 - jepsen.util 3205 :ok :read [18 29] INFO [2019-12-17 12:29:49,424] jepsen worker 55 - jepsen.util 3205 :invoke :write [18 23] INFO [2019-12-17 12:29:49,441] jepsen worker 52 - jepsen.util 2842 :ok :read [17 42] INFO [2019-12-17 12:29:49,442] jepsen worker 52 - jepsen.util 2842 :invoke :write [17 44] INFO [2019-12-17 12:29:49,474] jepsen worker 89 - jepsen.util 2519 :ok :read [29 29] INFO [2019-12-17 12:29:49,474] jepsen worker 89 - jepsen.util 2519 :invoke :cas [29 [32 13]] INFO [2019-12-17 12:29:49,485] jepsen worker 72 - jepsen.util 2772 :ok :write [24 46] INFO [2019-12-17 12:29:49,485] jepsen worker 72 - jepsen.util 2772 :invoke :read [24 nil] INFO [2019-12-17 12:29:49,496] jepsen worker 79 - jepsen.util 2599 :ok :write [26 25] INFO [2019-12-17 12:29:49,497] jepsen worker 79 - jepsen.util 2599 :invoke :write [26 13] INFO [2019-12-17 12:29:49,502] jepsen worker 72 - jepsen.util 2772 :ok :read [24 46] INFO [2019-12-17 12:29:49,502] jepsen worker 72 - jepsen.util 2772 :invoke :cas [24 [10 4]] INFO [2019-12-17 12:29:49,509] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15529,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:29:49,509] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15529,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:29:49,515] jepsen worker 72 - jepsen.util 2772 :fail :cas [24 [10 4]] :ValueNotSwapFrom INFO [2019-12-17 12:29:49,516] jepsen worker 72 - jepsen.util 2772 :invoke :write [24 20] INFO [2019-12-17 12:29:49,557] jepsen worker 14 - jepsen.util 2444 :ok :read [4 6] INFO [2019-12-17 12:29:49,557] jepsen worker 14 - jepsen.util 2444 :invoke :write [4 33] INFO [2019-12-17 12:29:49,586] jepsen worker 80 - jepsen.util 2600 :fail :cas [26 [34 32]] :ValueNotSwapFrom INFO [2019-12-17 12:29:49,586] jepsen worker 80 - jepsen.util 2600 :invoke :read [26 nil] INFO [2019-12-17 12:29:49,616] jepsen worker 5 - jepsen.util 2255 :fail :cas [1 [23 0]] :ValueNotSwapFrom INFO [2019-12-17 12:29:49,617] jepsen worker 5 - jepsen.util 2255 :invoke :write [1 30] INFO [2019-12-17 12:29:49,619] jepsen worker 88 - jepsen.util 2698 :ok :read [29 29] INFO [2019-12-17 12:29:49,619] jepsen worker 88 - jepsen.util 2698 :invoke :write [29 34] INFO [2019-12-17 12:29:49,640] jepsen worker 37 - jepsen.util 2377 :ok :read [12 30] INFO [2019-12-17 12:29:49,641] jepsen worker 37 - jepsen.util 2377 :invoke :read [12 nil] INFO [2019-12-17 12:29:49,660] jepsen worker 10 - jepsen.util 2530 :ok :write [3 11] INFO [2019-12-17 12:29:49,661] jepsen worker 10 - jepsen.util 2530 :invoke :cas [3 [49 48]] INFO [2019-12-17 12:29:49,687] jepsen worker 12 - jepsen.util 2802 :ok :read [4 6] INFO [2019-12-17 12:29:49,687] jepsen worker 12 - jepsen.util 2802 :invoke :cas [4 [2 48]] INFO [2019-12-17 12:29:49,690] jepsen worker 58 - jepsen.util 3388 :info :cas [19 [41 11]] :RequestCanceledException INFO [2019-12-17 12:29:49,691] jepsen worker 58 - jepsen.util 3478 :invoke :write [19 17] INFO [2019-12-17 12:29:49,692] jepsen worker 87 - jepsen.util 2427 :fail :cas [29 [2 44]] :ValueNotSwapFrom INFO [2019-12-17 12:29:49,692] jepsen worker 87 - jepsen.util 2427 :invoke :write [29 46] INFO [2019-12-17 12:29:49,698] jepsen worker 45 - jepsen.util 2835 :ok :write [15 43] INFO [2019-12-17 12:29:49,698] jepsen worker 45 - jepsen.util 2835 :invoke :read [15 nil] INFO [2019-12-17 12:29:49,701] jepsen worker 10 - jepsen.util 2530 :fail :cas [3 [49 48]] :ValueNotSwapFrom INFO [2019-12-17 12:29:49,701] jepsen worker 10 - jepsen.util 2530 :invoke :read [3 nil] INFO [2019-12-17 12:29:49,706] jepsen worker 72 - jepsen.util 2772 :ok :write [24 20] INFO [2019-12-17 12:29:49,706] jepsen worker 72 - jepsen.util 2772 :invoke :write [24 40] INFO [2019-12-17 12:29:49,708] jepsen worker 2 - jepsen.util 3332 :info :cas [0 [5 48]] :RequestCanceledException INFO [2019-12-17 12:29:49,708] jepsen worker 77 - jepsen.util 3317 :info :cas [25 [27 37]] :RequestCanceledException INFO [2019-12-17 12:29:49,708] jepsen worker 68 - jepsen.util 3308 :info :cas [22 [48 31]] :RequestCanceledException INFO [2019-12-17 12:29:49,709] jepsen worker 2 - jepsen.util 3422 :invoke :read [0 nil] INFO [2019-12-17 12:29:49,709] jepsen worker 68 - jepsen.util 3398 :invoke :cas [22 [36 13]] INFO [2019-12-17 12:29:49,709] jepsen worker 77 - jepsen.util 3407 :invoke :read [25 nil] INFO [2019-12-17 12:29:49,736] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15546,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:29:49,736] jepsen worker 86 - jepsen.util 3506 :info :write [28 15] :RequestCanceledException INFO [2019-12-17 12:29:49,736] jepsen worker 61 - jepsen.util 3931 :info :write [20 22] :RequestCanceledException INFO [2019-12-17 12:29:49,736] jepsen worker 66 - jepsen.util 3486 :info :cas [22 [39 0]] :RequestCanceledException INFO [2019-12-17 12:29:49,736] jepsen worker 25 - jepsen.util 3985 :info :cas [8 [48 33]] :RequestCanceledException INFO [2019-12-17 12:29:49,736] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15546,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:29:49,737] jepsen worker 81 - jepsen.util 3321 :info :cas [27 [31 7]] :RequestCanceledException INFO [2019-12-17 12:29:49,737] jepsen worker 82 - jepsen.util 4132 :info :cas [27 [36 9]] :RequestCanceledException INFO [2019-12-17 12:29:49,738] jepsen worker 36 - jepsen.util 2466 :ok :write [12 19] INFO [2019-12-17 12:29:49,738] jepsen worker 61 - jepsen.util 4021 :invoke :read [20 nil] INFO [2019-12-17 12:29:49,738] jepsen worker 86 - jepsen.util 3596 :invoke :write [28 30] INFO [2019-12-17 12:29:49,738] jepsen worker 66 - jepsen.util 3576 :invoke :cas [22 [45 20]] INFO [2019-12-17 12:29:49,738] jepsen worker 25 - jepsen.util 4075 :invoke :cas [8 [6 27]] INFO [2019-12-17 12:29:49,738] jepsen worker 36 - jepsen.util 2466 :invoke :write [12 35] INFO [2019-12-17 12:29:49,739] jepsen worker 81 - jepsen.util 3411 :invoke :write [27 6] INFO [2019-12-17 12:29:49,739] jepsen worker 82 - jepsen.util 4222 :invoke :write [27 25] INFO [2019-12-17 12:29:49,740] jepsen worker 10 - jepsen.util 2530 :ok :read [3 11] INFO [2019-12-17 12:29:49,740] jepsen worker 10 - jepsen.util 2530 :invoke :read [3 nil] INFO [2019-12-17 12:29:49,747] jepsen worker 19 - jepsen.util 2809 :fail :cas [6 [8 2]] :ValueNotSwapFrom INFO [2019-12-17 12:29:49,747] jepsen worker 19 - jepsen.util 2809 :invoke :cas [6 [21 48]] INFO [2019-12-17 12:29:49,754] jepsen worker 76 - jepsen.util 3496 :info :cas [25 [16 39]] :RequestCanceledException INFO [2019-12-17 12:29:49,754] jepsen worker 39 - jepsen.util 3459 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:29:49,754] jepsen worker 24 - jepsen.util 3444 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:29:49,754] jepsen worker 39 - jepsen.util 3459 :invoke :cas [13 [9 36]] INFO [2019-12-17 12:29:49,754] jepsen worker 24 - jepsen.util 3444 :invoke :cas [8 [8 28]] INFO [2019-12-17 12:29:49,755] jepsen worker 76 - jepsen.util 3586 :invoke :cas [25 [27 26]] INFO [2019-12-17 12:29:49,770] jepsen worker 10 - jepsen.util 2530 :ok :read [3 11] INFO [2019-12-17 12:29:49,771] jepsen worker 10 - jepsen.util 2530 :invoke :cas [3 [24 37]] INFO [2019-12-17 12:29:49,775] jepsen worker 45 - jepsen.util 2835 :ok :read [15 43] INFO [2019-12-17 12:29:49,775] jepsen worker 45 - jepsen.util 2835 :invoke :cas [15 [6 19]] INFO [2019-12-17 12:29:49,790] jepsen worker 78 - jepsen.util 2778 :info :write [26 15] :RequestCanceledException INFO [2019-12-17 12:29:49,790] jepsen worker 38 - jepsen.util 2648 :info :write [12 3] :RequestCanceledException INFO [2019-12-17 12:29:49,791] jepsen worker 38 - jepsen.util 2738 :invoke :read [12 nil] INFO [2019-12-17 12:29:49,791] jepsen worker 78 - jepsen.util 2868 :invoke :read [26 nil] INFO [2019-12-17 12:29:49,805] jepsen worker 19 - jepsen.util 2809 :fail :cas [6 [21 48]] :ValueNotSwapFrom INFO [2019-12-17 12:29:49,806] jepsen worker 19 - jepsen.util 2809 :invoke :write [6 33] INFO [2019-12-17 12:29:49,815] jepsen worker 45 - jepsen.util 2835 :fail :cas [15 [6 19]] :ValueNotSwapFrom INFO [2019-12-17 12:29:49,815] jepsen worker 45 - jepsen.util 2835 :invoke :read [15 nil] INFO [2019-12-17 12:29:49,817] jepsen worker 13 - jepsen.util 2353 :ok :write [4 17] INFO [2019-12-17 12:29:49,817] jepsen worker 13 - jepsen.util 2353 :invoke :write [4 8] INFO [2019-12-17 12:29:49,821] jepsen worker 67 - jepsen.util 4027 :info :cas [22 [32 19]] :RequestCanceledException INFO [2019-12-17 12:29:49,822] jepsen worker 67 - jepsen.util 4117 :invoke :write [22 16] INFO [2019-12-17 12:29:49,823] jepsen worker 72 - jepsen.util 2772 :ok :write [24 40] INFO [2019-12-17 12:29:49,823] jepsen worker 72 - jepsen.util 2772 :invoke :cas [24 [26 42]] INFO [2019-12-17 12:29:49,842] jepsen worker 89 - jepsen.util 2519 :fail :cas [29 [32 13]] :ValueNotSwapFrom INFO [2019-12-17 12:29:49,842] jepsen worker 89 - jepsen.util 2519 :invoke :cas [29 [35 19]] INFO [2019-12-17 12:29:49,851] jepsen worker 4 - jepsen.util 1984 :fail :cas [1 [1 41]] :ValueNotSwapFrom INFO [2019-12-17 12:29:49,851] jepsen worker 4 - jepsen.util 1984 :invoke :cas [1 [47 11]] INFO [2019-12-17 12:29:49,854] jepsen worker 20 - jepsen.util 2360 :fail :read [6 nil] :RequestTimeoutException INFO [2019-12-17 12:29:49,854] jepsen worker 20 - jepsen.util 2360 :invoke :cas [6 [42 41]] INFO [2019-12-17 12:29:49,873] jepsen worker 10 - jepsen.util 2530 :fail :cas [3 [24 37]] :ValueNotSwapFrom INFO [2019-12-17 12:29:49,873] jepsen worker 10 - jepsen.util 2530 :invoke :read [3 nil] INFO [2019-12-17 12:29:49,874] jepsen worker 72 - jepsen.util 2772 :fail :cas [24 [26 42]] :ValueNotSwapFrom INFO [2019-12-17 12:29:49,874] jepsen worker 72 - jepsen.util 2772 :invoke :write [24 12] INFO [2019-12-17 12:29:49,878] jepsen worker 70 - jepsen.util 3130 :ok :write [23 4] INFO [2019-12-17 12:29:49,878] jepsen worker 70 - jepsen.util 3130 :invoke :cas [23 [45 33]] INFO [2019-12-17 12:29:49,898] jepsen worker 69 - jepsen.util 2589 :fail :cas [23 [35 23]] :ValueNotSwapFrom INFO [2019-12-17 12:29:49,898] jepsen worker 69 - jepsen.util 2589 :invoke :read [23 nil] INFO [2019-12-17 12:29:49,915] jepsen worker 20 - jepsen.util 2360 :fail :cas [6 [42 41]] :ValueNotSwapFrom INFO [2019-12-17 12:29:49,915] jepsen worker 20 - jepsen.util 2360 :invoke :cas [6 [15 22]] INFO [2019-12-17 12:29:49,916] jepsen worker 12 - jepsen.util 2802 :fail :cas [4 [2 48]] :ValueNotSwapFrom INFO [2019-12-17 12:29:49,916] jepsen worker 12 - jepsen.util 2802 :invoke :write [4 49] INFO [2019-12-17 12:29:49,924] jepsen worker 10 - jepsen.util 2530 :ok :read [3 11] INFO [2019-12-17 12:29:49,924] jepsen worker 10 - jepsen.util 2530 :invoke :write [3 47] INFO [2019-12-17 12:29:49,928] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15585,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:29:49,928] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15585,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:29:49,928] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15589,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:29:49,929] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15589,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:29:49,937] jepsen worker 44 - jepsen.util 3464 :info :cas [14 [11 4]] :RequestCanceledException INFO [2019-12-17 12:29:49,938] jepsen worker 44 - jepsen.util 3554 :invoke :write [14 5] INFO [2019-12-17 12:29:49,948] jepsen worker 70 - jepsen.util 3130 :fail :cas [23 [45 33]] :ValueNotSwapFrom INFO [2019-12-17 12:29:49,948] jepsen worker 70 - jepsen.util 3130 :invoke :read [23 nil] INFO [2019-12-17 12:29:49,951] jepsen worker 49 - jepsen.util 2929 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:29:49,951] jepsen worker 49 - jepsen.util 2929 :invoke :read [16 nil] INFO [2019-12-17 12:29:49,971] jepsen worker 20 - jepsen.util 2360 :fail :cas [6 [15 22]] :ValueNotSwapFrom INFO [2019-12-17 12:29:49,971] jepsen worker 20 - jepsen.util 2360 :invoke :cas [6 [5 42]] INFO [2019-12-17 12:29:49,972] jepsen worker 69 - jepsen.util 2589 :ok :read [23 4] INFO [2019-12-17 12:29:49,972] jepsen worker 69 - jepsen.util 2589 :invoke :read [23 nil] INFO [2019-12-17 12:29:49,979] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:29:49,979] jepsen nemesis - couchbase.util Rebalance check count: 113 INFO [2019-12-17 12:29:49,979] jepsen nemesis - couchbase.util Rebalance stuck count: 94 INFO [2019-12-17 12:29:49,990] jepsen worker 32 - jepsen.util 3002 :info :cas [10 [35 14]] :RequestCanceledException INFO [2019-12-17 12:29:49,992] jepsen worker 32 - jepsen.util 3092 :invoke :read [10 nil] INFO [2019-12-17 12:29:49,992] jepsen worker 20 - jepsen.util 2360 :fail :cas [6 [5 42]] :ValueNotSwapFrom INFO [2019-12-17 12:29:49,992] jepsen worker 20 - jepsen.util 2360 :invoke :cas [6 [41 49]] INFO [2019-12-17 12:29:50,008] jepsen worker 6 - jepsen.util 3696 :info :cas [2 [5 29]] :RequestCanceledException INFO [2019-12-17 12:29:50,010] jepsen worker 6 - jepsen.util 3786 :invoke :write [2 31] INFO [2019-12-17 12:29:50,013] jepsen worker 20 - jepsen.util 2360 :fail :cas [6 [41 49]] :ValueNotSwapFrom INFO [2019-12-17 12:29:50,013] jepsen worker 20 - jepsen.util 2360 :invoke :read [6 nil] INFO [2019-12-17 12:29:50,018] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15604,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:29:50,019] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15604,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:29:50,019] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15607,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:29:50,019] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15607,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:29:50,023] jepsen worker 34 - jepsen.util 3724 :info :write [11 15] :RequestCanceledException INFO [2019-12-17 12:29:50,023] jepsen worker 60 - jepsen.util 3750 :info :cas [20 [42 20]] :RequestCanceledException INFO [2019-12-17 12:29:50,024] jepsen worker 34 - jepsen.util 3814 :invoke :write [11 7] INFO [2019-12-17 12:29:50,024] jepsen worker 60 - jepsen.util 3840 :invoke :cas [20 [16 4]] INFO [2019-12-17 12:29:50,036] jepsen worker 20 - jepsen.util 2360 :ok :read [6 38] INFO [2019-12-17 12:29:50,036] jepsen worker 20 - jepsen.util 2360 :invoke :read [6 nil] INFO [2019-12-17 12:29:50,038] jepsen worker 7 - jepsen.util 3427 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:29:50,038] jepsen worker 7 - jepsen.util 3427 :invoke :cas [2 [46 42]] INFO [2019-12-17 12:29:50,058] jepsen worker 20 - jepsen.util 2360 :ok :read [6 38] INFO [2019-12-17 12:29:50,058] jepsen worker 20 - jepsen.util 2360 :invoke :write [6 41] INFO [2019-12-17 12:29:50,073] jepsen worker 19 - jepsen.util 2809 :ok :write [6 33] INFO [2019-12-17 12:29:50,073] jepsen worker 19 - jepsen.util 2809 :invoke :cas [6 [40 6]] INFO [2019-12-17 12:29:50,076] jepsen worker 72 - jepsen.util 2772 :ok :write [24 12] INFO [2019-12-17 12:29:50,076] jepsen worker 72 - jepsen.util 2772 :invoke :read [24 nil] INFO [2019-12-17 12:29:50,090] jepsen worker 23 - jepsen.util 3443 :info :write [7 25] :RequestCanceledException INFO [2019-12-17 12:29:50,092] jepsen worker 23 - jepsen.util 3533 :invoke :read [7 nil] INFO [2019-12-17 12:29:50,109] jepsen worker 19 - jepsen.util 2809 :fail :cas [6 [40 6]] :ValueNotSwapFrom INFO [2019-12-17 12:29:50,109] jepsen worker 19 - jepsen.util 2809 :invoke :cas [6 [43 5]] INFO [2019-12-17 12:29:50,115] jepsen worker 32 - jepsen.util 3092 :ok :read [10 27] INFO [2019-12-17 12:29:50,116] jepsen worker 32 - jepsen.util 3092 :invoke :read [10 nil] INFO [2019-12-17 12:29:50,136] jepsen worker 4 - jepsen.util 1984 :fail :cas [1 [47 11]] :ValueNotSwapFrom INFO [2019-12-17 12:29:50,136] jepsen worker 4 - jepsen.util 1984 :invoke :read [1 nil] INFO [2019-12-17 12:29:50,142] jepsen worker 19 - jepsen.util 2809 :fail :cas [6 [43 5]] :ValueNotSwapFrom INFO [2019-12-17 12:29:50,142] jepsen worker 19 - jepsen.util 2809 :invoke :write [6 33] INFO [2019-12-17 12:29:50,145] jepsen worker 10 - jepsen.util 2530 :ok :write [3 47] INFO [2019-12-17 12:29:50,145] jepsen worker 10 - jepsen.util 2530 :invoke :cas [3 [21 41]] INFO [2019-12-17 12:29:50,158] jepsen worker 89 - jepsen.util 2519 :fail :cas [29 [35 19]] :ValueNotSwapFrom INFO [2019-12-17 12:29:50,158] jepsen worker 89 - jepsen.util 2519 :invoke :write [29 20] INFO [2019-12-17 12:29:50,173] jepsen worker 10 - jepsen.util 2530 :fail :cas [3 [21 41]] :ValueNotSwapFrom INFO [2019-12-17 12:29:50,174] jepsen worker 10 - jepsen.util 2530 :invoke :read [3 nil] INFO [2019-12-17 12:29:50,182] jepsen worker 78 - jepsen.util 2868 :ok :read [26 25] INFO [2019-12-17 12:29:50,182] jepsen worker 78 - jepsen.util 2868 :invoke :read [26 nil] INFO [2019-12-17 12:29:50,190] jepsen worker 72 - jepsen.util 2772 :ok :read [24 12] INFO [2019-12-17 12:29:50,190] jepsen worker 72 - jepsen.util 2772 :invoke :write [24 47] INFO [2019-12-17 12:29:50,199] jepsen worker 32 - jepsen.util 3092 :ok :read [10 27] INFO [2019-12-17 12:29:50,199] jepsen worker 32 - jepsen.util 3092 :invoke :cas [10 [42 36]] INFO [2019-12-17 12:29:50,223] jepsen worker 63 - jepsen.util 2313 :info :write [21 46] :RequestCanceledException INFO [2019-12-17 12:29:50,224] jepsen worker 10 - jepsen.util 2530 :ok :read [3 47] INFO [2019-12-17 12:29:50,224] jepsen worker 10 - jepsen.util 2530 :invoke :read [3 nil] INFO [2019-12-17 12:29:50,224] jepsen worker 63 - jepsen.util 2403 :invoke :read [21 nil] INFO [2019-12-17 12:29:50,229] jepsen worker 12 - jepsen.util 2802 :ok :write [4 49] INFO [2019-12-17 12:29:50,229] jepsen worker 12 - jepsen.util 2802 :invoke :cas [4 [29 30]] INFO [2019-12-17 12:29:50,230] jepsen worker 38 - jepsen.util 2738 :ok :read [12 19] INFO [2019-12-17 12:29:50,230] jepsen worker 38 - jepsen.util 2738 :invoke :read [12 nil] INFO [2019-12-17 12:29:50,238] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15628,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:29:50,238] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15628,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:29:50,238] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15629,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:29:50,238] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15629,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:29:50,238] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15630,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:29:50,238] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15630,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:29:50,256] jepsen worker 10 - jepsen.util 2530 :ok :read [3 47] INFO [2019-12-17 12:29:50,256] jepsen worker 10 - jepsen.util 2530 :invoke :read [3 nil] INFO [2019-12-17 12:29:50,272] jepsen worker 63 - jepsen.util 2403 :ok :read [21 2] INFO [2019-12-17 12:29:50,272] jepsen worker 63 - jepsen.util 2403 :invoke :cas [21 [12 23]] INFO [2019-12-17 12:29:50,280] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15637,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:29:50,280] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15637,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:29:50,280] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15641,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:29:50,280] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15641,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:29:50,286] jepsen worker 10 - jepsen.util 2530 :ok :read [3 47] INFO [2019-12-17 12:29:50,286] jepsen worker 10 - jepsen.util 2530 :invoke :read [3 nil] INFO [2019-12-17 12:29:50,296] jepsen worker 63 - jepsen.util 2403 :fail :cas [21 [12 23]] :ValueNotSwapFrom INFO [2019-12-17 12:29:50,296] jepsen worker 63 - jepsen.util 2403 :invoke :read [21 nil] INFO [2019-12-17 12:29:50,306] jepsen worker 10 - jepsen.util 2530 :ok :read [3 47] INFO [2019-12-17 12:29:50,306] jepsen worker 10 - jepsen.util 2530 :invoke :cas [3 [19 30]] INFO [2019-12-17 12:29:50,316] jepsen worker 63 - jepsen.util 2403 :ok :read [21 2] INFO [2019-12-17 12:29:50,316] jepsen worker 63 - jepsen.util 2403 :invoke :write [21 39] INFO [2019-12-17 12:29:50,323] jepsen worker 28 - jepsen.util 2008 :info :write [9 12] :RequestCanceledException INFO [2019-12-17 12:29:50,324] jepsen worker 28 - jepsen.util 2098 :invoke :cas [9 [20 8]] INFO [2019-12-17 12:29:50,330] jepsen worker 71 - jepsen.util 2681 :ok :write [23 5] INFO [2019-12-17 12:29:50,330] jepsen worker 71 - jepsen.util 2681 :invoke :write [23 26] INFO [2019-12-17 12:29:50,336] jepsen worker 20 - jepsen.util 2360 :ok :write [6 41] INFO [2019-12-17 12:29:50,336] jepsen worker 20 - jepsen.util 2360 :invoke :cas [6 [40 25]] INFO [2019-12-17 12:29:50,340] jepsen worker 10 - jepsen.util 2530 :fail :cas [3 [19 30]] :ValueNotSwapFrom INFO [2019-12-17 12:29:50,340] jepsen worker 10 - jepsen.util 2530 :invoke :write [3 17] INFO [2019-12-17 12:29:50,360] jepsen worker 20 - jepsen.util 2360 :fail :cas [6 [40 25]] :ValueNotSwapFrom INFO [2019-12-17 12:29:50,360] jepsen worker 20 - jepsen.util 2360 :invoke :cas [6 [13 46]] INFO [2019-12-17 12:29:50,373] jepsen worker 69 - jepsen.util 2589 :ok :read [23 4] INFO [2019-12-17 12:29:50,373] jepsen worker 69 - jepsen.util 2589 :invoke :write [23 19] INFO [2019-12-17 12:29:50,379] jepsen worker 72 - jepsen.util 2772 :ok :write [24 47] INFO [2019-12-17 12:29:50,379] jepsen worker 72 - jepsen.util 2772 :invoke :cas [24 [45 11]] INFO [2019-12-17 12:29:50,398] jepsen worker 70 - jepsen.util 3130 :ok :read [23 5] INFO [2019-12-17 12:29:50,398] jepsen worker 70 - jepsen.util 3130 :invoke :read [23 nil] INFO [2019-12-17 12:29:50,410] jepsen worker 20 - jepsen.util 2360 :fail :cas [6 [13 46]] :ValueNotSwapFrom INFO [2019-12-17 12:29:50,410] jepsen worker 20 - jepsen.util 2360 :invoke :cas [6 [31 12]] INFO [2019-12-17 12:29:50,415] jepsen worker 28 - jepsen.util 2098 :fail :cas [9 [20 8]] :ValueNotSwapFrom INFO [2019-12-17 12:29:50,415] jepsen worker 28 - jepsen.util 2098 :invoke :cas [9 [11 7]] INFO [2019-12-17 12:29:50,421] jepsen worker 64 - jepsen.util 2224 :fail :read [21 nil] :RequestTimeoutException INFO [2019-12-17 12:29:50,421] jepsen worker 64 - jepsen.util 2224 :invoke :read [21 nil] INFO [2019-12-17 12:29:50,444] jepsen worker 12 - jepsen.util 2802 :fail :cas [4 [29 30]] :ValueNotSwapFrom INFO [2019-12-17 12:29:50,444] jepsen worker 12 - jepsen.util 2802 :invoke :read [4 nil] INFO [2019-12-17 12:29:50,456] jepsen worker 53 - jepsen.util 2663 :fail :cas [17 [28 6]] :ValueNotSwapFrom INFO [2019-12-17 12:29:50,456] jepsen worker 53 - jepsen.util 2663 :invoke :write [17 12] INFO [2019-12-17 12:29:50,473] jepsen worker 20 - jepsen.util 2360 :fail :cas [6 [31 12]] :ValueNotSwapFrom INFO [2019-12-17 12:29:50,474] jepsen worker 20 - jepsen.util 2360 :invoke :cas [6 [14 21]] INFO [2019-12-17 12:29:50,474] jepsen worker 72 - jepsen.util 2772 :fail :cas [24 [45 11]] :ValueNotSwapFrom INFO [2019-12-17 12:29:50,474] jepsen worker 72 - jepsen.util 2772 :invoke :cas [24 [41 14]] INFO [2019-12-17 12:29:50,499] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15663,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:29:50,499] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15663,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:29:50,499] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15666,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:29:50,499] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15666,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:29:50,500] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15667,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:29:50,500] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15667,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:29:50,501] jepsen worker 28 - jepsen.util 2098 :fail :cas [9 [11 7]] :ValueNotSwapFrom INFO [2019-12-17 12:29:50,501] jepsen worker 28 - jepsen.util 2098 :invoke :cas [9 [18 44]] INFO [2019-12-17 12:29:50,504] jepsen worker 78 - jepsen.util 2868 :ok :read [26 25] INFO [2019-12-17 12:29:50,504] jepsen worker 78 - jepsen.util 2868 :invoke :read [26 nil] INFO [2019-12-17 12:29:50,528] jepsen worker 20 - jepsen.util 2360 :fail :cas [6 [14 21]] :ValueNotSwapFrom INFO [2019-12-17 12:29:50,528] jepsen worker 20 - jepsen.util 2360 :invoke :write [6 19] INFO [2019-12-17 12:29:50,540] jepsen worker 72 - jepsen.util 2772 :fail :cas [24 [41 14]] :ValueNotSwapFrom INFO [2019-12-17 12:29:50,540] jepsen worker 72 - jepsen.util 2772 :invoke :write [24 28] INFO [2019-12-17 12:29:50,555] jepsen worker 54 - jepsen.util 2664 :info :write [18 15] :RequestCanceledException INFO [2019-12-17 12:29:50,555] jepsen worker 54 - jepsen.util 2754 :invoke :read [18 nil] INFO [2019-12-17 12:29:50,570] jepsen worker 28 - jepsen.util 2098 :fail :cas [9 [18 44]] :ValueNotSwapFrom INFO [2019-12-17 12:29:50,570] jepsen worker 28 - jepsen.util 2098 :invoke :read [9 nil] INFO [2019-12-17 12:29:50,586] jepsen worker 54 - jepsen.util 2754 :ok :read [18 29] INFO [2019-12-17 12:29:50,586] jepsen worker 54 - jepsen.util 2754 :invoke :write [18 20] INFO [2019-12-17 12:29:50,590] jepsen worker 16 - jepsen.util 3616 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:29:50,590] jepsen worker 16 - jepsen.util 3616 :invoke :write [5 36] INFO [2019-12-17 12:29:50,601] jepsen worker 63 - jepsen.util 2403 :ok :write [21 39] INFO [2019-12-17 12:29:50,602] jepsen worker 63 - jepsen.util 2403 :invoke :cas [21 [19 34]] INFO [2019-12-17 12:29:50,622] jepsen worker 63 - jepsen.util 2403 :fail :cas [21 [19 34]] :ValueNotSwapFrom INFO [2019-12-17 12:29:50,623] jepsen worker 35 - jepsen.util 3365 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:29:50,629] jepsen worker 20 - jepsen.util 2360 :ok :write [6 19] INFO [2019-12-17 12:29:50,637] jepsen worker 72 - jepsen.util 2772 :ok :write [24 28] INFO [2019-12-17 12:29:50,651] jepsen worker 10 - jepsen.util 2530 :ok :write [3 17] INFO [2019-12-17 12:29:50,685] jepsen worker 63 - jepsen.util 2403 :invoke :write [21 37] INFO [2019-12-17 12:29:50,685] jepsen worker 35 - jepsen.util 3365 :invoke :cas [11 [24 19]] INFO [2019-12-17 12:29:50,685] jepsen worker 20 - jepsen.util 2360 :invoke :cas [6 [16 43]] INFO [2019-12-17 12:29:50,685] jepsen worker 72 - jepsen.util 2772 :invoke :write [24 17] INFO [2019-12-17 12:29:50,685] jepsen worker 10 - jepsen.util 2530 :invoke :cas [3 [43 16]] INFO [2019-12-17 12:29:50,703] jepsen worker 20 - jepsen.util 2360 :fail :cas [6 [16 43]] :ValueNotSwapFrom INFO [2019-12-17 12:29:50,704] jepsen worker 20 - jepsen.util 2360 :invoke :write [6 19] INFO [2019-12-17 12:29:50,715] jepsen worker 10 - jepsen.util 2530 :fail :cas [3 [43 16]] :ValueNotSwapFrom INFO [2019-12-17 12:29:50,715] jepsen worker 10 - jepsen.util 2530 :invoke :write [3 34] INFO [2019-12-17 12:29:50,718] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15683,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:29:50,719] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15683,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:29:50,719] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15687,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:29:50,719] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15687,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:29:50,754] jepsen worker 59 - jepsen.util 3569 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:29:50,754] jepsen worker 59 - jepsen.util 3569 :invoke :read [19 nil] INFO [2019-12-17 12:29:50,778] jepsen worker 70 - jepsen.util 3130 :ok :read [23 5] INFO [2019-12-17 12:29:50,778] jepsen worker 70 - jepsen.util 3130 :invoke :write [23 21] INFO [2019-12-17 12:29:50,792] jepsen worker 63 - jepsen.util 2403 :ok :write [21 37] INFO [2019-12-17 12:29:50,793] jepsen worker 63 - jepsen.util 2403 :invoke :cas [21 [43 4]] INFO [2019-12-17 12:29:50,807] jepsen worker 12 - jepsen.util 2802 :ok :read [4 49] INFO [2019-12-17 12:29:50,807] jepsen worker 12 - jepsen.util 2802 :invoke :write [4 38] INFO [2019-12-17 12:29:50,808] jepsen worker 63 - jepsen.util 2403 :fail :cas [21 [43 4]] :ValueNotSwapFrom INFO [2019-12-17 12:29:50,808] jepsen worker 63 - jepsen.util 2403 :invoke :read [21 nil] INFO [2019-12-17 12:29:50,823] jepsen worker 15 - jepsen.util 3165 :info :write [5 25] :RequestCanceledException INFO [2019-12-17 12:29:50,823] jepsen worker 43 - jepsen.util 3193 :info :cas [14 [10 20]] :RequestCanceledException INFO [2019-12-17 12:29:50,823] jepsen worker 74 - jepsen.util 2594 :info :write [24 28] :RequestCanceledException INFO [2019-12-17 12:29:50,823] jepsen worker 50 - jepsen.util 3920 :info :cas [16 [42 12]] :RequestCanceledException INFO [2019-12-17 12:29:50,823] jepsen worker 51 - jepsen.util 2391 :info :write [17 22] :RequestCanceledException INFO [2019-12-17 12:29:50,825] jepsen worker 43 - jepsen.util 3283 :invoke :cas [14 [16 27]] INFO [2019-12-17 12:29:50,825] jepsen worker 15 - jepsen.util 3255 :invoke :write [5 22] INFO [2019-12-17 12:29:50,825] jepsen worker 74 - jepsen.util 2684 :invoke :write [24 30] INFO [2019-12-17 12:29:50,825] jepsen worker 51 - jepsen.util 2481 :invoke :write [17 5] INFO [2019-12-17 12:29:50,825] jepsen worker 50 - jepsen.util 4010 :invoke :read [16 nil] INFO [2019-12-17 12:29:50,834] jepsen worker 63 - jepsen.util 2403 :ok :read [21 37] INFO [2019-12-17 12:29:50,834] jepsen worker 63 - jepsen.util 2403 :invoke :read [21 nil] INFO [2019-12-17 12:29:50,837] jepsen worker 47 - jepsen.util 3017 :info :write [15 14] :RequestCanceledException INFO [2019-12-17 12:29:50,837] jepsen worker 84 - jepsen.util 3504 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:29:50,837] jepsen worker 84 - jepsen.util 3504 :invoke :read [28 nil] INFO [2019-12-17 12:29:50,838] jepsen worker 47 - jepsen.util 3107 :invoke :write [15 33] INFO [2019-12-17 12:29:50,854] jepsen worker 65 - jepsen.util 2405 :fail :read [21 nil] :RequestTimeoutException INFO [2019-12-17 12:29:50,854] jepsen worker 65 - jepsen.util 2405 :invoke :read [21 nil] INFO [2019-12-17 12:29:50,873] jepsen worker 63 - jepsen.util 2403 :ok :read [21 37] INFO [2019-12-17 12:29:50,873] jepsen worker 63 - jepsen.util 2403 :invoke :write [21 17] INFO [2019-12-17 12:29:50,889] jepsen worker 85 - jepsen.util 3955 :info :write [28 17] :RequestCanceledException INFO [2019-12-17 12:29:50,891] jepsen worker 85 - jepsen.util 4045 :invoke :cas [28 [13 14]] INFO [2019-12-17 12:29:50,900] jepsen worker 80 - jepsen.util 2600 :ok :read [26 25] INFO [2019-12-17 12:29:50,901] jepsen worker 80 - jepsen.util 2600 :invoke :cas [26 [5 22]] INFO [2019-12-17 12:29:50,914] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15711,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:29:50,914] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15711,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:29:50,915] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15714,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:29:50,915] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15714,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:29:50,923] jepsen worker 62 - jepsen.util 3392 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:29:50,923] jepsen worker 62 - jepsen.util 3392 :invoke :write [20 29] INFO [2019-12-17 12:29:50,932] jepsen worker 78 - jepsen.util 2868 :ok :read [26 25] INFO [2019-12-17 12:29:50,932] jepsen worker 78 - jepsen.util 2868 :invoke :write [26 45] INFO [2019-12-17 12:29:50,941] jepsen worker 4 - jepsen.util 1984 :ok :read [1 15] INFO [2019-12-17 12:29:50,941] jepsen worker 4 - jepsen.util 1984 :invoke :read [1 nil] INFO [2019-12-17 12:29:50,962] jepsen worker 63 - jepsen.util 2403 :ok :write [21 17] INFO [2019-12-17 12:29:50,962] jepsen worker 63 - jepsen.util 2403 :invoke :cas [21 [26 35]] INFO [2019-12-17 12:29:50,983] jepsen worker 38 - jepsen.util 2738 :ok :read [12 19] INFO [2019-12-17 12:29:50,984] jepsen worker 38 - jepsen.util 2738 :invoke :cas [12 [7 22]] INFO [2019-12-17 12:29:50,984] jepsen worker 63 - jepsen.util 2403 :fail :cas [21 [26 35]] :ValueNotSwapFrom INFO [2019-12-17 12:29:50,984] jepsen worker 63 - jepsen.util 2403 :invoke :cas [21 [22 23]] INFO [2019-12-17 12:29:50,994] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:29:50,994] jepsen nemesis - couchbase.util Rebalance check count: 114 INFO [2019-12-17 12:29:50,994] jepsen nemesis - couchbase.util Rebalance stuck count: 95 INFO [2019-12-17 12:29:50,995] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15725,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:29:50,995] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15725,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:29:50,995] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15728,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:29:50,996] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15728,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:29:50,996] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15729,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:29:50,996] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15729,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:29:50,997] jepsen worker 63 - jepsen.util 2403 :fail :cas [21 [22 23]] :ValueNotSwapFrom INFO [2019-12-17 12:29:50,997] jepsen worker 63 - jepsen.util 2403 :invoke :cas [21 [48 45]] INFO [2019-12-17 12:29:51,008] jepsen worker 1 - jepsen.util 3061 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:29:51,008] jepsen worker 63 - jepsen.util 2403 :fail :cas [21 [48 45]] :ValueNotSwapFrom INFO [2019-12-17 12:29:51,008] jepsen worker 18 - jepsen.util 2898 :fail :read [6 nil] :RequestTimeoutException INFO [2019-12-17 12:29:51,008] jepsen worker 1 - jepsen.util 3061 :invoke :read [0 nil] INFO [2019-12-17 12:29:51,008] jepsen worker 63 - jepsen.util 2403 :invoke :write [21 3] INFO [2019-12-17 12:29:51,008] jepsen worker 18 - jepsen.util 2898 :invoke :read [6 nil] INFO [2019-12-17 12:29:51,022] jepsen worker 18 - jepsen.util 2898 :ok :read [6 19] INFO [2019-12-17 12:29:51,023] jepsen worker 18 - jepsen.util 2898 :invoke :read [6 nil] INFO [2019-12-17 12:29:51,033] jepsen worker 37 - jepsen.util 2377 :ok :read [12 19] INFO [2019-12-17 12:29:51,033] jepsen worker 37 - jepsen.util 2377 :invoke :cas [12 [36 4]] INFO [2019-12-17 12:29:51,035] jepsen worker 75 - jepsen.util 3675 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:29:51,035] jepsen worker 41 - jepsen.util 3731 :info :cas [13 [39 2]] :RequestCanceledException INFO [2019-12-17 12:29:51,035] jepsen worker 75 - jepsen.util 3675 :invoke :cas [25 [12 4]] INFO [2019-12-17 12:29:51,036] jepsen worker 41 - jepsen.util 3821 :invoke :write [13 12] INFO [2019-12-17 12:29:51,058] jepsen worker 63 - jepsen.util 2403 :ok :write [21 3] INFO [2019-12-17 12:29:51,058] jepsen worker 63 - jepsen.util 2403 :invoke :read [21 nil] INFO [2019-12-17 12:29:51,082] jepsen worker 63 - jepsen.util 2403 :ok :read [21 3] INFO [2019-12-17 12:29:51,082] jepsen worker 63 - jepsen.util 2403 :invoke :cas [21 [8 7]] INFO [2019-12-17 12:29:51,099] jepsen worker 63 - jepsen.util 2403 :fail :cas [21 [8 7]] :ValueNotSwapFrom INFO [2019-12-17 12:29:51,099] jepsen worker 63 - jepsen.util 2403 :invoke :read [21 nil] INFO [2019-12-17 12:29:51,108] jepsen worker 83 - jepsen.util 3953 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:29:51,108] jepsen worker 83 - jepsen.util 3953 :invoke :read [27 nil] INFO [2019-12-17 12:29:51,111] jepsen worker 63 - jepsen.util 2403 :ok :read [21 3] INFO [2019-12-17 12:29:51,111] jepsen worker 63 - jepsen.util 2403 :invoke :write [21 26] INFO [2019-12-17 12:29:51,125] jepsen worker 4 - jepsen.util 1984 :ok :read [1 15] INFO [2019-12-17 12:29:51,125] jepsen worker 4 - jepsen.util 1984 :invoke :cas [1 [36 4]] INFO [2019-12-17 12:29:51,136] jepsen worker 30 - jepsen.util 3180 :fail :read [10 nil] :RequestTimeoutException INFO [2019-12-17 12:29:51,137] jepsen worker 30 - jepsen.util 3180 :invoke :read [10 nil] INFO [2019-12-17 12:29:51,154] jepsen worker 11 - jepsen.util 3071 :fail :read [3 nil] :RequestTimeoutException INFO [2019-12-17 12:29:51,154] jepsen worker 27 - jepsen.util 2367 :info :write [9 36] :RequestCanceledException INFO [2019-12-17 12:29:51,154] jepsen worker 40 - jepsen.util 3820 :info :cas [13 [38 30]] :RequestCanceledException INFO [2019-12-17 12:29:51,154] jepsen worker 11 - jepsen.util 3071 :invoke :cas [3 [8 22]] INFO [2019-12-17 12:29:51,155] jepsen worker 63 - jepsen.util 2403 :ok :write [21 26] INFO [2019-12-17 12:29:51,155] jepsen worker 63 - jepsen.util 2403 :invoke :write [21 38] INFO [2019-12-17 12:29:51,155] jepsen worker 27 - jepsen.util 2457 :invoke :write [9 40] INFO [2019-12-17 12:29:51,155] jepsen worker 40 - jepsen.util 3910 :invoke :read [13 nil] INFO [2019-12-17 12:29:51,189] jepsen worker 57 - jepsen.util 3297 :info :write [19 18] :RequestCanceledException INFO [2019-12-17 12:29:51,190] jepsen worker 57 - jepsen.util 3387 :invoke :cas [19 [7 36]] INFO [2019-12-17 12:29:51,207] jepsen worker 33 - jepsen.util 3723 :info :write [11 42] :RequestCanceledException INFO [2019-12-17 12:29:51,207] jepsen worker 42 - jepsen.util 4182 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:29:51,207] jepsen worker 22 - jepsen.util 3532 :info :cas [7 [39 44]] :RequestCanceledException INFO [2019-12-17 12:29:51,207] jepsen worker 42 - jepsen.util 4182 :invoke :cas [14 [38 22]] INFO [2019-12-17 12:29:51,208] jepsen worker 33 - jepsen.util 3813 :invoke :cas [11 [10 35]] INFO [2019-12-17 12:29:51,208] jepsen worker 22 - jepsen.util 3622 :invoke :cas [7 [44 12]] INFO [2019-12-17 12:29:51,214] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15750,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:29:51,215] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15750,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:29:51,215] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15751,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:29:51,215] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15751,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:29:51,223] jepsen worker 9 - jepsen.util 3069 :info :cas [3 [32 46]] :RequestCanceledException INFO [2019-12-17 12:29:51,223] jepsen worker 8 - jepsen.util 3068 :info :cas [2 [34 35]] :RequestCanceledException INFO [2019-12-17 12:29:51,223] jepsen worker 21 - jepsen.util 3531 :info :write [7 23] :RequestCanceledException INFO [2019-12-17 12:29:51,224] jepsen worker 9 - jepsen.util 3159 :invoke :read [3 nil] INFO [2019-12-17 12:29:51,224] jepsen worker 21 - jepsen.util 3621 :invoke :read [7 nil] INFO [2019-12-17 12:29:51,224] jepsen worker 8 - jepsen.util 3158 :invoke :read [2 nil] INFO [2019-12-17 12:29:51,237] jepsen worker 26 - jepsen.util 3806 :info :cas [8 [2 47]] :RequestCanceledException INFO [2019-12-17 12:29:51,238] jepsen worker 26 - jepsen.util 3896 :invoke :read [8 nil] INFO [2019-12-17 12:29:51,244] jepsen worker 27 - jepsen.util 2457 :ok :write [9 40] INFO [2019-12-17 12:29:51,244] jepsen worker 27 - jepsen.util 2457 :invoke :read [9 nil] INFO [2019-12-17 12:29:51,250] jepsen worker 48 - jepsen.util 3018 :info :write [16 32] :RequestCanceledException INFO [2019-12-17 12:29:51,251] jepsen worker 48 - jepsen.util 3108 :invoke :read [16 nil] INFO [2019-12-17 12:29:51,260] jepsen worker 27 - jepsen.util 2457 :ok :read [9 40] INFO [2019-12-17 12:29:51,260] jepsen worker 27 - jepsen.util 2457 :invoke :read [9 nil] INFO [2019-12-17 12:29:51,262] jepsen worker 63 - jepsen.util 2403 :ok :write [21 38] INFO [2019-12-17 12:29:51,262] jepsen worker 63 - jepsen.util 2403 :invoke :write [21 14] INFO [2019-12-17 12:29:51,264] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15762,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:29:51,264] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15762,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:29:51,264] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15764,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:29:51,264] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15764,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:29:51,276] jepsen worker 27 - jepsen.util 2457 :ok :read [9 40] INFO [2019-12-17 12:29:51,276] jepsen worker 27 - jepsen.util 2457 :invoke :write [9 25] INFO [2019-12-17 12:29:51,323] jepsen worker 17 - jepsen.util 3527 :info :write [5 23] :RequestCanceledException INFO [2019-12-17 12:29:51,323] jepsen worker 17 - jepsen.util 3617 :invoke :write [5 48] INFO [2019-12-17 12:29:51,337] jepsen worker 0 - jepsen.util 3780 :info :cas [0 [35 39]] :RequestCanceledException INFO [2019-12-17 12:29:51,338] jepsen worker 0 - jepsen.util 3870 :invoke :cas [0 [17 15]] INFO [2019-12-17 12:29:51,341] jepsen worker 63 - jepsen.util 2403 :ok :write [21 14] INFO [2019-12-17 12:29:51,341] jepsen worker 63 - jepsen.util 2403 :invoke :read [21 nil] INFO [2019-12-17 12:29:51,349] jepsen worker 27 - jepsen.util 2457 :ok :write [9 25] INFO [2019-12-17 12:29:51,349] jepsen worker 27 - jepsen.util 2457 :invoke :cas [9 [6 28]] INFO [2019-12-17 12:29:51,355] jepsen worker 63 - jepsen.util 2403 :ok :read [21 14] INFO [2019-12-17 12:29:51,355] jepsen worker 63 - jepsen.util 2403 :invoke :write [21 45] INFO [2019-12-17 12:29:51,366] jepsen worker 27 - jepsen.util 2457 :fail :cas [9 [6 28]] :ValueNotSwapFrom INFO [2019-12-17 12:29:51,366] jepsen worker 27 - jepsen.util 2457 :invoke :write [9 31] INFO [2019-12-17 12:29:51,390] jepsen worker 29 - jepsen.util 2279 :info :write [9 11] :RequestCanceledException INFO [2019-12-17 12:29:51,390] jepsen worker 73 - jepsen.util 2593 :info :write [24 15] :RequestCanceledException INFO [2019-12-17 12:29:51,390] jepsen worker 31 - jepsen.util 2551 :fail :read [10 nil] :RequestTimeoutException INFO [2019-12-17 12:29:51,390] jepsen worker 31 - jepsen.util 2551 :invoke :write [10 47] INFO [2019-12-17 12:29:51,391] jepsen worker 73 - jepsen.util 2683 :invoke :cas [24 [12 26]] INFO [2019-12-17 12:29:51,391] jepsen worker 29 - jepsen.util 2369 :invoke :read [9 nil] INFO [2019-12-17 12:29:51,423] jepsen worker 3 - jepsen.util 2523 :info :write [1 4] :RequestCanceledException INFO [2019-12-17 12:29:51,425] jepsen worker 3 - jepsen.util 2613 :invoke :read [1 nil] INFO [2019-12-17 12:29:51,427] jepsen worker 27 - jepsen.util 2457 :ok :write [9 31] INFO [2019-12-17 12:29:51,428] jepsen worker 27 - jepsen.util 2457 :invoke :read [9 nil] INFO [2019-12-17 12:29:51,450] jepsen worker 63 - jepsen.util 2403 :ok :write [21 45] INFO [2019-12-17 12:29:51,450] jepsen worker 63 - jepsen.util 2403 :invoke :cas [21 [41 25]] INFO [2019-12-17 12:29:51,455] jepsen worker 46 - jepsen.util 2836 :info :write [15 36] :RequestCanceledException INFO [2019-12-17 12:29:51,456] jepsen worker 46 - jepsen.util 2926 :invoke :read [15 nil] INFO [2019-12-17 12:29:51,479] jepsen worker 73 - jepsen.util 2683 :fail :cas [24 [12 26]] :ValueNotSwapFrom INFO [2019-12-17 12:29:51,479] jepsen worker 73 - jepsen.util 2683 :invoke :read [24 nil] INFO [2019-12-17 12:29:51,483] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15783,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:29:51,483] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15783,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:29:51,483] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15784,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:29:51,483] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15784,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:29:51,490] jepsen worker 56 - jepsen.util 2486 :info :write [18 5] :RequestCanceledException INFO [2019-12-17 12:29:51,491] jepsen worker 56 - jepsen.util 2576 :invoke :cas [18 [49 4]] INFO [2019-12-17 12:29:51,492] jepsen worker 29 - jepsen.util 2369 :ok :read [9 31] INFO [2019-12-17 12:29:51,493] jepsen worker 29 - jepsen.util 2369 :invoke :read [9 nil] INFO [2019-12-17 12:29:51,503] jepsen worker 63 - jepsen.util 2403 :fail :cas [21 [41 25]] :ValueNotSwapFrom INFO [2019-12-17 12:29:51,503] jepsen worker 63 - jepsen.util 2403 :invoke :read [21 nil] INFO [2019-12-17 12:29:51,504] jepsen worker 38 - jepsen.util 2738 :fail :cas [12 [7 22]] :ValueNotSwapFrom INFO [2019-12-17 12:29:51,505] jepsen worker 38 - jepsen.util 2738 :invoke :cas [12 [1 38]] INFO [2019-12-17 12:29:51,513] jepsen worker 29 - jepsen.util 2369 :ok :read [9 31] INFO [2019-12-17 12:29:51,513] jepsen worker 29 - jepsen.util 2369 :invoke :cas [9 [8 45]] INFO [2019-12-17 12:29:51,538] jepsen worker 79 - jepsen.util 2599 :info :write [26 13] :RequestCanceledException INFO [2019-12-17 12:29:51,538] jepsen worker 55 - jepsen.util 3205 :info :write [18 23] :RequestCanceledException INFO [2019-12-17 12:29:51,538] jepsen worker 52 - jepsen.util 2842 :info :write [17 44] :RequestCanceledException INFO [2019-12-17 12:29:51,539] jepsen worker 55 - jepsen.util 3295 :invoke :read [18 nil] INFO [2019-12-17 12:29:51,539] jepsen worker 52 - jepsen.util 2932 :invoke :write [17 10] INFO [2019-12-17 12:29:51,539] jepsen worker 79 - jepsen.util 2689 :invoke :write [26 48] INFO [2019-12-17 12:29:51,546] jepsen worker 63 - jepsen.util 2403 :ok :read [21 45] INFO [2019-12-17 12:29:51,546] jepsen worker 63 - jepsen.util 2403 :invoke :read [21 nil] INFO [2019-12-17 12:29:51,557] jepsen worker 29 - jepsen.util 2369 :fail :cas [9 [8 45]] :ValueNotSwapFrom INFO [2019-12-17 12:29:51,557] jepsen worker 29 - jepsen.util 2369 :invoke :cas [9 [18 7]] INFO [2019-12-17 12:29:51,568] jepsen worker 63 - jepsen.util 2403 :ok :read [21 45] INFO [2019-12-17 12:29:51,568] jepsen worker 63 - jepsen.util 2403 :invoke :write [21 49] INFO [2019-12-17 12:29:51,577] jepsen worker 29 - jepsen.util 2369 :fail :cas [9 [18 7]] :ValueNotSwapFrom INFO [2019-12-17 12:29:51,577] jepsen worker 29 - jepsen.util 2369 :invoke :write [9 11] INFO [2019-12-17 12:29:51,615] jepsen worker 63 - jepsen.util 2403 :ok :write [21 49] INFO [2019-12-17 12:29:51,615] jepsen worker 63 - jepsen.util 2403 :invoke :read [21 nil] INFO [2019-12-17 12:29:51,623] jepsen worker 14 - jepsen.util 2444 :info :write [4 33] :RequestCanceledException INFO [2019-12-17 12:29:51,623] jepsen worker 5 - jepsen.util 2255 :info :write [1 30] :RequestCanceledException INFO [2019-12-17 12:29:51,624] jepsen worker 14 - jepsen.util 2534 :invoke :cas [4 [27 32]] INFO [2019-12-17 12:29:51,624] jepsen worker 5 - jepsen.util 2345 :invoke :write [1 47] INFO [2019-12-17 12:29:51,655] jepsen worker 88 - jepsen.util 2698 :info :write [29 34] :RequestCanceledException INFO [2019-12-17 12:29:51,656] jepsen worker 88 - jepsen.util 2788 :invoke :read [29 nil] INFO [2019-12-17 12:29:51,656] jepsen worker 29 - jepsen.util 2369 :ok :write [9 11] INFO [2019-12-17 12:29:51,657] jepsen worker 29 - jepsen.util 2369 :invoke :read [9 nil] INFO [2019-12-17 12:29:51,681] jepsen worker 29 - jepsen.util 2369 :ok :read [9 11] INFO [2019-12-17 12:29:51,681] jepsen worker 29 - jepsen.util 2369 :invoke :read [9 nil] INFO [2019-12-17 12:29:51,693] jepsen worker 80 - jepsen.util 2600 :fail :cas [26 [5 22]] :ValueNotSwapFrom INFO [2019-12-17 12:29:51,693] jepsen worker 80 - jepsen.util 2600 :invoke :read [26 nil] INFO [2019-12-17 12:29:51,696] jepsen worker 29 - jepsen.util 2369 :ok :read [9 11] INFO [2019-12-17 12:29:51,696] jepsen worker 29 - jepsen.util 2369 :invoke :cas [9 [32 39]] INFO [2019-12-17 12:29:51,702] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15812,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:29:51,702] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15812,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:29:51,702] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15814,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:29:51,703] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15814,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:29:51,712] jepsen worker 29 - jepsen.util 2369 :fail :cas [9 [32 39]] :ValueNotSwapFrom INFO [2019-12-17 12:29:51,713] jepsen worker 29 - jepsen.util 2369 :invoke :write [9 3] INFO [2019-12-17 12:29:51,715] jepsen worker 4 - jepsen.util 1984 :fail :cas [1 [36 4]] :ValueNotSwapFrom INFO [2019-12-17 12:29:51,715] jepsen worker 4 - jepsen.util 1984 :invoke :cas [1 [12 35]] INFO [2019-12-17 12:29:51,716] jepsen worker 37 - jepsen.util 2377 :fail :cas [12 [36 4]] :ValueNotSwapFrom INFO [2019-12-17 12:29:51,717] jepsen worker 37 - jepsen.util 2377 :invoke :write [12 33] INFO [2019-12-17 12:29:51,723] jepsen worker 58 - jepsen.util 3478 :info :write [19 17] :RequestCanceledException INFO [2019-12-17 12:29:51,723] jepsen worker 77 - jepsen.util 3407 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:29:51,723] jepsen worker 77 - jepsen.util 3407 :invoke :read [25 nil] INFO [2019-12-17 12:29:51,724] jepsen worker 58 - jepsen.util 3568 :invoke :write [19 7] INFO [2019-12-17 12:29:51,736] jepsen worker 87 - jepsen.util 2427 :info :write [29 46] :RequestCanceledException INFO [2019-12-17 12:29:51,737] jepsen worker 87 - jepsen.util 2517 :invoke :read [29 nil] INFO [2019-12-17 12:29:51,753] jepsen worker 56 - jepsen.util 2576 :fail :cas [18 [49 4]] :ValueNotSwapFrom INFO [2019-12-17 12:29:51,754] jepsen worker 56 - jepsen.util 2576 :invoke :read [18 nil] INFO [2019-12-17 12:29:51,769] jepsen worker 29 - jepsen.util 2369 :ok :write [9 3] INFO [2019-12-17 12:29:51,770] jepsen worker 29 - jepsen.util 2369 :invoke :write [9 22] INFO [2019-12-17 12:29:51,786] jepsen worker 25 - jepsen.util 4075 :info :cas [8 [6 27]] :RequestCanceledException INFO [2019-12-17 12:29:51,786] jepsen worker 2 - jepsen.util 3422 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:29:51,786] jepsen worker 36 - jepsen.util 2466 :info :write [12 35] :RequestCanceledException INFO [2019-12-17 12:29:51,786] jepsen worker 2 - jepsen.util 3422 :invoke :write [0 17] INFO [2019-12-17 12:29:51,787] jepsen worker 25 - jepsen.util 4165 :invoke :cas [8 [38 33]] INFO [2019-12-17 12:29:51,789] jepsen worker 36 - jepsen.util 2556 :invoke :cas [12 [17 32]] INFO [2019-12-17 12:29:51,805] jepsen worker 81 - jepsen.util 3411 :info :write [27 6] :RequestCanceledException INFO [2019-12-17 12:29:51,805] jepsen worker 68 - jepsen.util 3398 :info :cas [22 [36 13]] :RequestCanceledException INFO [2019-12-17 12:29:51,805] jepsen worker 3 - jepsen.util 2613 :ok :read [1 15] INFO [2019-12-17 12:29:51,805] jepsen worker 3 - jepsen.util 2613 :invoke :write [1 35] INFO [2019-12-17 12:29:51,806] jepsen worker 68 - jepsen.util 3488 :invoke :write [22 43] INFO [2019-12-17 12:29:51,806] jepsen worker 81 - jepsen.util 3501 :invoke :cas [27 [33 7]] INFO [2019-12-17 12:29:51,811] jepsen worker 38 - jepsen.util 2738 :fail :cas [12 [1 38]] :ValueNotSwapFrom INFO [2019-12-17 12:29:51,811] jepsen worker 38 - jepsen.util 2738 :invoke :write [12 14] INFO [2019-12-17 12:29:51,823] jepsen worker 66 - jepsen.util 3576 :info :cas [22 [45 20]] :RequestCanceledException INFO [2019-12-17 12:29:51,824] jepsen worker 82 - jepsen.util 4222 :info :write [27 25] :RequestCanceledException INFO [2019-12-17 12:29:51,824] jepsen worker 66 - jepsen.util 3666 :invoke :cas [22 [37 13]] INFO [2019-12-17 12:29:51,825] jepsen worker 82 - jepsen.util 4312 :invoke :write [27 3] INFO [2019-12-17 12:29:51,830] jepsen worker 76 - jepsen.util 3586 :info :cas [25 [27 26]] :RequestCanceledException INFO [2019-12-17 12:29:51,830] jepsen worker 61 - jepsen.util 4021 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:29:51,830] jepsen worker 61 - jepsen.util 4021 :invoke :cas [20 [4 49]] INFO [2019-12-17 12:29:51,831] jepsen worker 76 - jepsen.util 3676 :invoke :write [25 20] INFO [2019-12-17 12:29:51,840] jepsen worker 29 - jepsen.util 2369 :ok :write [9 22] INFO [2019-12-17 12:29:51,840] jepsen worker 29 - jepsen.util 2369 :invoke :write [9 17] INFO [2019-12-17 12:29:51,851] jepsen worker 39 - jepsen.util 3459 :info :cas [13 [9 36]] :RequestCanceledException INFO [2019-12-17 12:29:51,851] jepsen worker 86 - jepsen.util 3596 :info :write [28 30] :RequestCanceledException INFO [2019-12-17 12:29:51,851] jepsen worker 13 - jepsen.util 2353 :info :write [4 8] :RequestCanceledException INFO [2019-12-17 12:29:51,851] jepsen worker 24 - jepsen.util 3444 :info :cas [8 [8 28]] :RequestCanceledException INFO [2019-12-17 12:29:51,852] jepsen worker 13 - jepsen.util 2443 :invoke :cas [4 [49 40]] INFO [2019-12-17 12:29:51,852] jepsen worker 86 - jepsen.util 3686 :invoke :write [28 12] INFO [2019-12-17 12:29:51,852] jepsen worker 24 - jepsen.util 3534 :invoke :write [8 12] INFO [2019-12-17 12:29:51,852] jepsen worker 39 - jepsen.util 3549 :invoke :write [13 13] INFO [2019-12-17 12:29:51,857] jepsen worker 87 - jepsen.util 2517 :ok :read [29 29] INFO [2019-12-17 12:29:51,857] jepsen worker 87 - jepsen.util 2517 :invoke :write [29 21] INFO [2019-12-17 12:29:51,882] jepsen worker 14 - jepsen.util 2534 :fail :cas [4 [27 32]] :ValueNotSwapFrom INFO [2019-12-17 12:29:51,882] jepsen worker 14 - jepsen.util 2534 :invoke :read [4 nil] INFO [2019-12-17 12:29:51,889] jepsen worker 67 - jepsen.util 4117 :info :write [22 16] :RequestCanceledException INFO [2019-12-17 12:29:51,890] jepsen worker 67 - jepsen.util 4207 :invoke :read [22 nil] INFO [2019-12-17 12:29:51,893] jepsen worker 4 - jepsen.util 1984 :fail :cas [1 [12 35]] :ValueNotSwapFrom INFO [2019-12-17 12:29:51,893] jepsen worker 4 - jepsen.util 1984 :invoke :write [1 8] INFO [2019-12-17 12:29:51,895] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15841,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:29:51,896] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15841,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:29:51,896] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15845,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:29:51,896] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15845,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:29:51,896] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15846,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:29:51,896] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15846,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:29:51,907] jepsen worker 45 - jepsen.util 2835 :fail :read [15 nil] :RequestTimeoutException INFO [2019-12-17 12:29:51,907] jepsen worker 45 - jepsen.util 2835 :invoke :read [15 nil] INFO [2019-12-17 12:29:51,915] jepsen worker 29 - jepsen.util 2369 :ok :write [9 17] INFO [2019-12-17 12:29:51,915] jepsen worker 29 - jepsen.util 2369 :invoke :read [9 nil] INFO [2019-12-17 12:29:51,920] jepsen worker 88 - jepsen.util 2788 :ok :read [29 29] INFO [2019-12-17 12:29:51,921] jepsen worker 88 - jepsen.util 2788 :invoke :cas [29 [21 22]] INFO [2019-12-17 12:29:51,931] jepsen worker 56 - jepsen.util 2576 :ok :read [18 29] INFO [2019-12-17 12:29:51,931] jepsen worker 56 - jepsen.util 2576 :invoke :cas [18 [31 35]] INFO [2019-12-17 12:29:51,933] jepsen worker 45 - jepsen.util 2835 :ok :read [15 43] INFO [2019-12-17 12:29:51,933] jepsen worker 45 - jepsen.util 2835 :invoke :read [15 nil] INFO [2019-12-17 12:29:51,950] jepsen worker 29 - jepsen.util 2369 :ok :read [9 17] INFO [2019-12-17 12:29:51,951] jepsen worker 29 - jepsen.util 2369 :invoke :cas [9 [26 45]] INFO [2019-12-17 12:29:51,960] jepsen worker 88 - jepsen.util 2788 :fail :cas [29 [21 22]] :ValueNotSwapFrom INFO [2019-12-17 12:29:51,960] jepsen worker 88 - jepsen.util 2788 :invoke :write [29 42] INFO [2019-12-17 12:29:51,963] jepsen worker 45 - jepsen.util 2835 :ok :read [15 43] INFO [2019-12-17 12:29:51,963] jepsen worker 45 - jepsen.util 2835 :invoke :read [15 nil] INFO [2019-12-17 12:29:51,972] jepsen worker 29 - jepsen.util 2369 :fail :cas [9 [26 45]] :ValueNotSwapFrom INFO [2019-12-17 12:29:51,972] jepsen worker 29 - jepsen.util 2369 :invoke :cas [9 [34 18]] INFO [2019-12-17 12:29:51,979] jepsen worker 80 - jepsen.util 2600 :ok :read [26 25] INFO [2019-12-17 12:29:51,979] jepsen worker 80 - jepsen.util 2600 :invoke :cas [26 [30 35]] INFO [2019-12-17 12:29:51,981] jepsen worker 45 - jepsen.util 2835 :ok :read [15 43] INFO [2019-12-17 12:29:51,981] jepsen worker 45 - jepsen.util 2835 :invoke :cas [15 [41 35]] INFO [2019-12-17 12:29:51,988] jepsen worker 29 - jepsen.util 2369 :fail :cas [9 [34 18]] :ValueNotSwapFrom INFO [2019-12-17 12:29:51,988] jepsen worker 29 - jepsen.util 2369 :invoke :read [9 nil] INFO [2019-12-17 12:29:51,995] jepsen worker 45 - jepsen.util 2835 :fail :cas [15 [41 35]] :ValueNotSwapFrom INFO [2019-12-17 12:29:51,995] jepsen worker 45 - jepsen.util 2835 :invoke :write [15 10] INFO [2019-12-17 12:29:52,002] jepsen worker 29 - jepsen.util 2369 :ok :read [9 17] INFO [2019-12-17 12:29:52,002] jepsen worker 29 - jepsen.util 2369 :invoke :read [9 nil] INFO [2019-12-17 12:29:52,008] jepsen worker 44 - jepsen.util 3554 :info :write [14 5] :RequestCanceledException INFO [2019-12-17 12:29:52,009] jepsen worker 44 - jepsen.util 3644 :invoke :write [14 0] INFO [2019-12-17 12:29:52,009] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:29:52,009] jepsen nemesis - couchbase.util Rebalance check count: 115 INFO [2019-12-17 12:29:52,009] jepsen nemesis - couchbase.util Rebalance stuck count: 96 INFO [2019-12-17 12:29:52,014] jepsen worker 56 - jepsen.util 2576 :fail :cas [18 [31 35]] :ValueNotSwapFrom INFO [2019-12-17 12:29:52,014] jepsen worker 56 - jepsen.util 2576 :invoke :read [18 nil] INFO [2019-12-17 12:29:52,031] jepsen worker 55 - jepsen.util 3295 :ok :read [18 29] INFO [2019-12-17 12:29:52,031] jepsen worker 55 - jepsen.util 3295 :invoke :read [18 nil] INFO [2019-12-17 12:29:52,037] jepsen worker 6 - jepsen.util 3786 :info :write [2 31] :RequestCanceledException INFO [2019-12-17 12:29:52,038] jepsen worker 6 - jepsen.util 3876 :invoke :read [2 nil] INFO [2019-12-17 12:29:52,055] jepsen worker 49 - jepsen.util 2929 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:29:52,055] jepsen worker 34 - jepsen.util 3814 :info :write [11 7] :RequestCanceledException INFO [2019-12-17 12:29:52,055] jepsen worker 72 - jepsen.util 2772 :ok :write [24 17] INFO [2019-12-17 12:29:52,055] jepsen worker 49 - jepsen.util 2929 :invoke :cas [16 [14 15]] INFO [2019-12-17 12:29:52,056] jepsen worker 72 - jepsen.util 2772 :invoke :write [24 47] INFO [2019-12-17 12:29:52,056] jepsen worker 34 - jepsen.util 3904 :invoke :cas [11 [2 8]] INFO [2019-12-17 12:29:52,091] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15867,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:29:52,092] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15867,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:29:52,093] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15870,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:29:52,093] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15870,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:29:52,112] jepsen worker 47 - jepsen.util 3107 :ok :write [15 33] INFO [2019-12-17 12:29:52,112] jepsen worker 47 - jepsen.util 3107 :invoke :cas [15 [38 1]] INFO [2019-12-17 12:29:52,123] jepsen worker 60 - jepsen.util 3840 :info :cas [20 [16 4]] :RequestCanceledException INFO [2019-12-17 12:29:52,126] jepsen worker 60 - jepsen.util 3930 :invoke :cas [20 [38 12]] INFO [2019-12-17 12:29:52,130] jepsen worker 31 - jepsen.util 2551 :ok :write [10 47] INFO [2019-12-17 12:29:52,130] jepsen worker 31 - jepsen.util 2551 :invoke :write [10 33] INFO [2019-12-17 12:29:52,131] jepsen worker 20 - jepsen.util 2360 :ok :write [6 19] INFO [2019-12-17 12:29:52,131] jepsen worker 20 - jepsen.util 2360 :invoke :write [6 20] INFO [2019-12-17 12:29:52,134] jepsen worker 72 - jepsen.util 2772 :ok :write [24 47] INFO [2019-12-17 12:29:52,134] jepsen worker 72 - jepsen.util 2772 :invoke :cas [24 [20 44]] INFO [2019-12-17 12:29:52,137] jepsen worker 7 - jepsen.util 3427 :info :cas [2 [46 42]] :RequestCanceledException INFO [2019-12-17 12:29:52,138] jepsen worker 36 - jepsen.util 2556 :fail :cas [12 [17 32]] :ValueNotSwapFrom INFO [2019-12-17 12:29:52,138] jepsen worker 36 - jepsen.util 2556 :invoke :cas [12 [28 30]] INFO [2019-12-17 12:29:52,139] jepsen worker 7 - jepsen.util 3517 :invoke :write [2 36] INFO [2019-12-17 12:29:52,140] jepsen worker 10 - jepsen.util 2530 :ok :write [3 34] INFO [2019-12-17 12:29:52,140] jepsen worker 10 - jepsen.util 2530 :invoke :read [3 nil] INFO [2019-12-17 12:29:52,166] jepsen worker 30 - jepsen.util 3180 :ok :read [10 27] INFO [2019-12-17 12:29:52,166] jepsen worker 30 - jepsen.util 3180 :invoke :read [10 nil] INFO [2019-12-17 12:29:52,176] jepsen worker 55 - jepsen.util 3295 :ok :read [18 29] INFO [2019-12-17 12:29:52,177] jepsen worker 55 - jepsen.util 3295 :invoke :cas [18 [17 31]] INFO [2019-12-17 12:29:52,189] jepsen worker 23 - jepsen.util 3533 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:29:52,189] jepsen worker 89 - jepsen.util 2519 :info :write [29 20] :RequestCanceledException INFO [2019-12-17 12:29:52,190] jepsen worker 23 - jepsen.util 3533 :invoke :write [7 12] INFO [2019-12-17 12:29:52,190] jepsen worker 89 - jepsen.util 2609 :invoke :write [29 28] INFO [2019-12-17 12:29:52,201] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15891,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:29:52,201] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15891,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:29:52,202] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15893,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:29:52,202] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15893,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:29:52,208] jepsen worker 19 - jepsen.util 2809 :info :write [6 33] :RequestCanceledException INFO [2019-12-17 12:29:52,209] jepsen worker 19 - jepsen.util 2899 :invoke :write [6 41] INFO [2019-12-17 12:29:52,223] jepsen worker 32 - jepsen.util 3092 :info :cas [10 [42 36]] :RequestCanceledException INFO [2019-12-17 12:29:52,224] jepsen worker 32 - jepsen.util 3182 :invoke :write [10 2] INFO [2019-12-17 12:29:52,224] jepsen worker 73 - jepsen.util 2683 :ok :read [24 17] INFO [2019-12-17 12:29:52,225] jepsen worker 73 - jepsen.util 2683 :invoke :cas [24 [6 22]] INFO [2019-12-17 12:29:52,243] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15900,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:29:52,243] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15900,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:29:52,243] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15903,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:29:52,243] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15903,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:29:52,243] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15904,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:29:52,243] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15904,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:29:52,245] jepsen worker 55 - jepsen.util 3295 :fail :cas [18 [17 31]] :ValueNotSwapFrom INFO [2019-12-17 12:29:52,246] jepsen worker 55 - jepsen.util 3295 :invoke :read [18 nil] INFO [2019-12-17 12:29:52,255] jepsen worker 46 - jepsen.util 2926 :ok :read [15 33] INFO [2019-12-17 12:29:52,255] jepsen worker 9 - jepsen.util 3159 :ok :read [3 34] INFO [2019-12-17 12:29:52,255] jepsen worker 46 - jepsen.util 2926 :invoke :write [15 20] INFO [2019-12-17 12:29:52,255] jepsen worker 9 - jepsen.util 3159 :invoke :write [3 46] INFO [2019-12-17 12:29:52,269] jepsen worker 72 - jepsen.util 2772 :fail :cas [24 [20 44]] :ValueNotSwapFrom INFO [2019-12-17 12:29:52,269] jepsen worker 72 - jepsen.util 2772 :invoke :write [24 22] INFO [2019-12-17 12:29:52,271] jepsen worker 65 - jepsen.util 2405 :ok :read [21 49] INFO [2019-12-17 12:29:52,271] jepsen worker 65 - jepsen.util 2405 :invoke :cas [21 [47 31]] INFO [2019-12-17 12:29:52,276] jepsen worker 55 - jepsen.util 3295 :ok :read [18 29] INFO [2019-12-17 12:29:52,276] jepsen worker 55 - jepsen.util 3295 :invoke :read [18 nil] INFO [2019-12-17 12:29:52,300] jepsen worker 11 - jepsen.util 3071 :fail :cas [3 [8 22]] :ValueNotSwapFrom INFO [2019-12-17 12:29:52,300] jepsen worker 11 - jepsen.util 3071 :invoke :read [3 nil] INFO [2019-12-17 12:29:52,318] jepsen worker 10 - jepsen.util 2530 :ok :read [3 34] INFO [2019-12-17 12:29:52,319] jepsen worker 10 - jepsen.util 2530 :invoke :write [3 26] INFO [2019-12-17 12:29:52,336] jepsen worker 47 - jepsen.util 3107 :fail :cas [15 [38 1]] :ValueNotSwapFrom INFO [2019-12-17 12:29:52,337] jepsen worker 47 - jepsen.util 3107 :invoke :cas [15 [47 20]] INFO [2019-12-17 12:29:52,342] jepsen worker 14 - jepsen.util 2534 :ok :read [4 49] INFO [2019-12-17 12:29:52,342] jepsen worker 14 - jepsen.util 2534 :invoke :cas [4 [2 2]] INFO [2019-12-17 12:29:52,351] jepsen worker 30 - jepsen.util 3180 :ok :read [10 47] INFO [2019-12-17 12:29:52,351] jepsen worker 30 - jepsen.util 3180 :invoke :read [10 nil] INFO [2019-12-17 12:29:52,383] jepsen worker 31 - jepsen.util 2551 :ok :write [10 33] INFO [2019-12-17 12:29:52,383] jepsen worker 31 - jepsen.util 2551 :invoke :cas [10 [13 47]] INFO [2019-12-17 12:29:52,390] jepsen worker 71 - jepsen.util 2681 :info :write [23 26] :RequestCanceledException INFO [2019-12-17 12:29:52,391] jepsen worker 71 - jepsen.util 2771 :invoke :read [23 nil] INFO [2019-12-17 12:29:52,395] jepsen worker 55 - jepsen.util 3295 :ok :read [18 29] INFO [2019-12-17 12:29:52,395] jepsen worker 55 - jepsen.util 3295 :invoke :write [18 16] INFO [2019-12-17 12:29:52,407] jepsen worker 69 - jepsen.util 2589 :info :write [23 19] :RequestCanceledException INFO [2019-12-17 12:29:52,409] jepsen worker 69 - jepsen.util 2679 :invoke :write [23 30] INFO [2019-12-17 12:29:52,414] jepsen worker 80 - jepsen.util 2600 :fail :cas [26 [30 35]] :ValueNotSwapFrom INFO [2019-12-17 12:29:52,414] jepsen worker 80 - jepsen.util 2600 :invoke :write [26 23] INFO [2019-12-17 12:29:52,415] jepsen worker 73 - jepsen.util 2683 :fail :cas [24 [6 22]] :ValueNotSwapFrom INFO [2019-12-17 12:29:52,416] jepsen worker 73 - jepsen.util 2683 :invoke :read [24 nil] INFO [2019-12-17 12:29:52,429] jepsen worker 31 - jepsen.util 2551 :fail :cas [10 [13 47]] :ValueNotSwapFrom INFO [2019-12-17 12:29:52,429] jepsen worker 31 - jepsen.util 2551 :invoke :read [10 nil] INFO [2019-12-17 12:29:52,444] jepsen worker 73 - jepsen.util 2683 :ok :read [24 47] INFO [2019-12-17 12:29:52,444] jepsen worker 73 - jepsen.util 2683 :invoke :write [24 36] INFO [2019-12-17 12:29:52,461] jepsen worker 31 - jepsen.util 2551 :ok :read [10 33] INFO [2019-12-17 12:29:52,461] jepsen worker 31 - jepsen.util 2551 :invoke :write [10 34] INFO [2019-12-17 12:29:52,461] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15926,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:29:52,461] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15926,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:29:52,462] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15931,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:29:52,462] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15931,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:29:52,482] jepsen worker 19 - jepsen.util 2899 :ok :write [6 41] INFO [2019-12-17 12:29:52,482] jepsen worker 19 - jepsen.util 2899 :invoke :read [6 nil] INFO [2019-12-17 12:29:52,495] jepsen worker 46 - jepsen.util 2926 :ok :write [15 20] INFO [2019-12-17 12:29:52,495] jepsen worker 46 - jepsen.util 2926 :invoke :write [15 47] INFO [2019-12-17 12:29:52,497] jepsen worker 19 - jepsen.util 2899 :ok :read [6 41] INFO [2019-12-17 12:29:52,498] jepsen worker 19 - jepsen.util 2899 :invoke :write [6 37] INFO [2019-12-17 12:29:52,517] jepsen worker 30 - jepsen.util 3180 :ok :read [10 33] INFO [2019-12-17 12:29:52,517] jepsen worker 30 - jepsen.util 3180 :invoke :cas [10 [23 40]] INFO [2019-12-17 12:29:52,523] jepsen worker 64 - jepsen.util 2224 :fail :read [21 nil] :RequestTimeoutException INFO [2019-12-17 12:29:52,523] jepsen worker 64 - jepsen.util 2224 :invoke :write [21 38] INFO [2019-12-17 12:29:52,553] jepsen worker 53 - jepsen.util 2663 :info :write [17 12] :RequestCanceledException INFO [2019-12-17 12:29:52,555] jepsen worker 53 - jepsen.util 2753 :invoke :cas [17 [33 0]] INFO [2019-12-17 12:29:52,622] jepsen worker 36 - jepsen.util 2556 :fail :cas [12 [28 30]] :ValueNotSwapFrom INFO [2019-12-17 12:29:52,622] jepsen worker 36 - jepsen.util 2556 :invoke :read [12 nil] INFO [2019-12-17 12:29:52,624] jepsen worker 28 - jepsen.util 2098 :fail :read [9 nil] :RequestTimeoutException INFO [2019-12-17 12:29:52,624] jepsen worker 28 - jepsen.util 2098 :invoke :cas [9 [9 28]] INFO [2019-12-17 12:29:52,637] jepsen worker 54 - jepsen.util 2754 :info :write [18 20] :RequestCanceledException INFO [2019-12-17 12:29:52,638] jepsen worker 54 - jepsen.util 2844 :invoke :read [18 nil] INFO [2019-12-17 12:29:52,645] jepsen worker 28 - jepsen.util 2098 :fail :cas [9 [9 28]] :ValueNotSwapFrom INFO [2019-12-17 12:29:52,646] jepsen worker 28 - jepsen.util 2098 :invoke :cas [9 [37 49]] INFO [2019-12-17 12:29:52,661] jepsen worker 28 - jepsen.util 2098 :fail :cas [9 [37 49]] :ValueNotSwapFrom INFO [2019-12-17 12:29:52,661] jepsen worker 28 - jepsen.util 2098 :invoke :read [9 nil] INFO [2019-12-17 12:29:52,673] jepsen worker 71 - jepsen.util 2771 :ok :read [23 5] INFO [2019-12-17 12:29:52,674] jepsen worker 71 - jepsen.util 2771 :invoke :read [23 nil] INFO [2019-12-17 12:29:52,678] jepsen worker 28 - jepsen.util 2098 :ok :read [9 17] INFO [2019-12-17 12:29:52,678] jepsen worker 28 - jepsen.util 2098 :invoke :write [9 48] INFO [2019-12-17 12:29:52,681] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15945,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:29:52,681] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15945,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:29:52,681] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15948,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:29:52,681] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15948,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:29:52,681] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15949,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:29:52,681] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15949,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:29:52,690] jepsen worker 16 - jepsen.util 3616 :info :write [5 36] :RequestCanceledException INFO [2019-12-17 12:29:52,691] jepsen worker 16 - jepsen.util 3706 :invoke :write [5 4] INFO [2019-12-17 12:29:52,723] jepsen worker 35 - jepsen.util 3365 :info :cas [11 [24 19]] :RequestCanceledException INFO [2019-12-17 12:29:52,725] jepsen worker 35 - jepsen.util 3455 :invoke :cas [11 [15 20]] INFO [2019-12-17 12:29:52,732] jepsen worker 28 - jepsen.util 2098 :ok :write [9 48] INFO [2019-12-17 12:29:52,732] jepsen worker 28 - jepsen.util 2098 :invoke :cas [9 [21 36]] INFO [2019-12-17 12:29:52,734] jepsen worker 56 - jepsen.util 2576 :ok :read [18 29] INFO [2019-12-17 12:29:52,734] jepsen worker 56 - jepsen.util 2576 :invoke :cas [18 [42 5]] INFO [2019-12-17 12:29:52,755] jepsen worker 28 - jepsen.util 2098 :fail :cas [9 [21 36]] :ValueNotSwapFrom INFO [2019-12-17 12:29:52,755] jepsen worker 28 - jepsen.util 2098 :invoke :cas [9 [25 40]] INFO [2019-12-17 12:29:52,778] jepsen worker 28 - jepsen.util 2098 :fail :cas [9 [25 40]] :ValueNotSwapFrom INFO [2019-12-17 12:29:52,778] jepsen worker 28 - jepsen.util 2098 :invoke :write [9 9] INFO [2019-12-17 12:29:52,781] jepsen worker 36 - jepsen.util 2556 :ok :read [12 19] INFO [2019-12-17 12:29:52,781] jepsen worker 36 - jepsen.util 2556 :invoke :cas [12 [40 43]] INFO [2019-12-17 12:29:52,807] jepsen worker 70 - jepsen.util 3130 :info :write [23 21] :RequestCanceledException INFO [2019-12-17 12:29:52,808] jepsen worker 70 - jepsen.util 3220 :invoke :write [23 0] INFO [2019-12-17 12:29:52,837] jepsen worker 5 - jepsen.util 2345 :ok :write [1 47] INFO [2019-12-17 12:29:52,837] jepsen worker 5 - jepsen.util 2345 :invoke :write [1 18] INFO [2019-12-17 12:29:52,849] jepsen worker 59 - jepsen.util 3569 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:29:52,849] jepsen worker 43 - jepsen.util 3283 :info :cas [14 [16 27]] :RequestCanceledException INFO [2019-12-17 12:29:52,849] jepsen worker 59 - jepsen.util 3569 :invoke :cas [19 [43 48]] INFO [2019-12-17 12:29:52,850] jepsen worker 43 - jepsen.util 3373 :invoke :write [14 11] INFO [2019-12-17 12:29:52,860] jepsen worker 4 - jepsen.util 1984 :ok :write [1 8] INFO [2019-12-17 12:29:52,860] jepsen worker 4 - jepsen.util 1984 :invoke :cas [1 [7 4]] INFO [2019-12-17 12:29:52,880] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15965,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:29:52,881] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15965,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:29:52,881] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15969,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:29:52,881] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15969,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:29:52,883] jepsen worker 28 - jepsen.util 2098 :ok :write [9 9] INFO [2019-12-17 12:29:52,883] jepsen worker 28 - jepsen.util 2098 :invoke :write [9 22] INFO [2019-12-17 12:29:52,890] jepsen worker 74 - jepsen.util 2684 :info :write [24 30] :RequestCanceledException INFO [2019-12-17 12:29:52,891] jepsen worker 74 - jepsen.util 2774 :invoke :write [24 39] INFO [2019-12-17 12:29:52,908] jepsen worker 12 - jepsen.util 2802 :info :write [4 38] :RequestCanceledException INFO [2019-12-17 12:29:52,908] jepsen worker 51 - jepsen.util 2481 :info :write [17 5] :RequestCanceledException INFO [2019-12-17 12:29:52,909] jepsen worker 12 - jepsen.util 2892 :invoke :read [4 nil] INFO [2019-12-17 12:29:52,909] jepsen worker 51 - jepsen.util 2571 :invoke :read [17 nil] INFO [2019-12-17 12:29:52,921] jepsen worker 9 - jepsen.util 3159 :ok :write [3 46] INFO [2019-12-17 12:29:52,922] jepsen worker 9 - jepsen.util 3159 :invoke :cas [3 [21 23]] INFO [2019-12-17 12:29:52,923] jepsen worker 50 - jepsen.util 4010 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:29:52,923] jepsen worker 15 - jepsen.util 3255 :info :write [5 22] :RequestCanceledException INFO [2019-12-17 12:29:52,923] jepsen worker 50 - jepsen.util 4010 :invoke :write [16 14] INFO [2019-12-17 12:29:52,924] jepsen worker 15 - jepsen.util 3345 :invoke :cas [5 [28 45]] INFO [2019-12-17 12:29:52,932] jepsen worker 73 - jepsen.util 2683 :ok :write [24 36] INFO [2019-12-17 12:29:52,933] jepsen worker 73 - jepsen.util 2683 :invoke :cas [24 [31 30]] INFO [2019-12-17 12:29:52,935] jepsen worker 20 - jepsen.util 2360 :ok :write [6 20] INFO [2019-12-17 12:29:52,935] jepsen worker 20 - jepsen.util 2360 :invoke :read [6 nil] INFO [2019-12-17 12:29:52,938] jepsen worker 84 - jepsen.util 3504 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:29:52,938] jepsen worker 84 - jepsen.util 3504 :invoke :read [28 nil] INFO [2019-12-17 12:29:52,942] jepsen worker 46 - jepsen.util 2926 :ok :write [15 47] INFO [2019-12-17 12:29:52,943] jepsen worker 46 - jepsen.util 2926 :invoke :cas [15 [46 49]] INFO [2019-12-17 12:29:52,953] jepsen worker 85 - jepsen.util 4045 :info :cas [28 [13 14]] :RequestCanceledException INFO [2019-12-17 12:29:52,955] jepsen worker 85 - jepsen.util 4135 :invoke :write [28 25] INFO [2019-12-17 12:29:52,958] jepsen worker 78 - jepsen.util 2868 :ok :write [26 45] INFO [2019-12-17 12:29:52,958] jepsen worker 78 - jepsen.util 2868 :invoke :cas [26 [32 6]] INFO [2019-12-17 12:29:52,964] jepsen worker 46 - jepsen.util 2926 :fail :cas [15 [46 49]] :ValueNotSwapFrom INFO [2019-12-17 12:29:52,964] jepsen worker 46 - jepsen.util 2926 :invoke :read [15 nil] INFO [2019-12-17 12:29:52,983] jepsen worker 46 - jepsen.util 2926 :ok :read [15 47] INFO [2019-12-17 12:29:52,983] jepsen worker 46 - jepsen.util 2926 :invoke :cas [15 [19 23]] INFO [2019-12-17 12:29:53,012] jepsen worker 46 - jepsen.util 2926 :fail :cas [15 [19 23]] :ValueNotSwapFrom INFO [2019-12-17 12:29:53,013] jepsen worker 46 - jepsen.util 2926 :invoke :read [15 nil] INFO [2019-12-17 12:29:53,018] jepsen worker 73 - jepsen.util 2683 :fail :cas [24 [31 30]] :ValueNotSwapFrom INFO [2019-12-17 12:29:53,018] jepsen worker 73 - jepsen.util 2683 :invoke :read [24 nil] INFO [2019-12-17 12:29:53,019] jepsen worker 71 - jepsen.util 2771 :ok :read [23 5] INFO [2019-12-17 12:29:53,019] jepsen worker 71 - jepsen.util 2771 :invoke :write [23 17] INFO [2019-12-17 12:29:53,022] jepsen worker 53 - jepsen.util 2753 :fail :cas [17 [33 0]] :ValueNotSwapFrom INFO [2019-12-17 12:29:53,022] jepsen worker 53 - jepsen.util 2753 :invoke :read [17 nil] INFO [2019-12-17 12:29:53,023] jepsen worker 62 - jepsen.util 3392 :info :write [20 29] :RequestCanceledException INFO [2019-12-17 12:29:53,025] jepsen worker 62 - jepsen.util 3482 :invoke :cas [20 [28 15]] INFO [2019-12-17 12:29:53,039] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:29:53,039] jepsen nemesis - couchbase.util Rebalance check count: 116 INFO [2019-12-17 12:29:53,040] jepsen nemesis - couchbase.util Rebalance stuck count: 97 INFO [2019-12-17 12:29:53,054] jepsen worker 46 - jepsen.util 2926 :ok :read [15 47] INFO [2019-12-17 12:29:53,055] jepsen worker 46 - jepsen.util 2926 :invoke :read [15 nil] INFO [2019-12-17 12:29:53,080] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15988,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:29:53,080] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15988,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:29:53,080] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15992,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:29:53,080] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":15992,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:29:53,099] jepsen worker 73 - jepsen.util 2683 :ok :read [24 36] INFO [2019-12-17 12:29:53,099] jepsen worker 73 - jepsen.util 2683 :invoke :cas [24 [22 49]] INFO [2019-12-17 12:29:53,101] jepsen worker 20 - jepsen.util 2360 :ok :read [6 20] INFO [2019-12-17 12:29:53,101] jepsen worker 20 - jepsen.util 2360 :invoke :write [6 45] INFO [2019-12-17 12:29:53,102] jepsen worker 46 - jepsen.util 2926 :ok :read [15 47] INFO [2019-12-17 12:29:53,102] jepsen worker 46 - jepsen.util 2926 :invoke :write [15 43] INFO [2019-12-17 12:29:53,106] jepsen worker 1 - jepsen.util 3061 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:29:53,107] jepsen worker 18 - jepsen.util 2898 :fail :read [6 nil] :RequestTimeoutException INFO [2019-12-17 12:29:53,107] jepsen worker 1 - jepsen.util 3061 :invoke :cas [0 [27 44]] INFO [2019-12-17 12:29:53,107] jepsen worker 18 - jepsen.util 2898 :invoke :write [6 15] INFO [2019-12-17 12:29:53,107] jepsen worker 64 - jepsen.util 2224 :ok :write [21 38] INFO [2019-12-17 12:29:53,108] jepsen worker 64 - jepsen.util 2224 :invoke :write [21 35] INFO [2019-12-17 12:29:53,120] jepsen worker 41 - jepsen.util 3821 :info :write [13 12] :RequestCanceledException INFO [2019-12-17 12:29:53,122] jepsen worker 41 - jepsen.util 3911 :invoke :read [13 nil] INFO [2019-12-17 12:29:53,135] jepsen worker 14 - jepsen.util 2534 :fail :cas [4 [2 2]] :ValueNotSwapFrom INFO [2019-12-17 12:29:53,135] jepsen worker 14 - jepsen.util 2534 :invoke :write [4 24] INFO [2019-12-17 12:29:53,137] jepsen worker 75 - jepsen.util 3675 :info :cas [25 [12 4]] :RequestCanceledException INFO [2019-12-17 12:29:53,138] jepsen worker 75 - jepsen.util 3765 :invoke :write [25 29] INFO [2019-12-17 12:29:53,156] jepsen worker 5 - jepsen.util 2345 :ok :write [1 18] INFO [2019-12-17 12:29:53,156] jepsen worker 5 - jepsen.util 2345 :invoke :cas [1 [25 13]] INFO [2019-12-17 12:29:53,161] jepsen worker 53 - jepsen.util 2753 :ok :read [17 42] INFO [2019-12-17 12:29:53,161] jepsen worker 53 - jepsen.util 2753 :invoke :read [17 nil] INFO [2019-12-17 12:29:53,192] jepsen worker 38 - jepsen.util 2738 :ok :write [12 14] INFO [2019-12-17 12:29:53,192] jepsen worker 38 - jepsen.util 2738 :invoke :cas [12 [44 18]] INFO [2019-12-17 12:29:53,195] jepsen worker 28 - jepsen.util 2098 :ok :write [9 22] INFO [2019-12-17 12:29:53,195] jepsen worker 28 - jepsen.util 2098 :invoke :cas [9 [4 10]] INFO [2019-12-17 12:29:53,199] jepsen worker 73 - jepsen.util 2683 :fail :cas [24 [22 49]] :ValueNotSwapFrom INFO [2019-12-17 12:29:53,199] jepsen worker 73 - jepsen.util 2683 :invoke :cas [24 [14 38]] INFO [2019-12-17 12:29:53,207] jepsen worker 83 - jepsen.util 3953 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:29:53,207] jepsen worker 40 - jepsen.util 3910 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:29:53,208] jepsen worker 83 - jepsen.util 3953 :invoke :read [27 nil] INFO [2019-12-17 12:29:53,208] jepsen worker 40 - jepsen.util 3910 :invoke :cas [13 [9 0]] INFO [2019-12-17 12:29:53,209] jepsen worker 87 - jepsen.util 2517 :ok :write [29 21] INFO [2019-12-17 12:29:53,209] jepsen worker 87 - jepsen.util 2517 :invoke :cas [29 [9 15]] INFO [2019-12-17 12:29:53,217] jepsen worker 28 - jepsen.util 2098 :fail :cas [9 [4 10]] :ValueNotSwapFrom INFO [2019-12-17 12:29:53,217] jepsen worker 28 - jepsen.util 2098 :invoke :cas [9 [45 0]] INFO [2019-12-17 12:29:53,218] jepsen worker 57 - jepsen.util 3387 :info :cas [19 [7 36]] :RequestCanceledException INFO [2019-12-17 12:29:53,220] jepsen worker 57 - jepsen.util 3477 :invoke :read [19 nil] INFO [2019-12-17 12:29:53,229] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16020,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:29:53,229] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16020,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:29:53,229] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16023,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:29:53,229] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16023,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:29:53,237] jepsen worker 33 - jepsen.util 3813 :info :cas [11 [10 35]] :RequestCanceledException INFO [2019-12-17 12:29:53,238] jepsen worker 33 - jepsen.util 3903 :invoke :cas [11 [49 0]] INFO [2019-12-17 12:29:53,254] jepsen worker 22 - jepsen.util 3622 :info :cas [7 [44 12]] :RequestCanceledException INFO [2019-12-17 12:29:53,255] jepsen worker 22 - jepsen.util 3712 :invoke :cas [7 [6 47]] INFO [2019-12-17 12:29:53,264] jepsen worker 73 - jepsen.util 2683 :fail :cas [24 [14 38]] :ValueNotSwapFrom INFO [2019-12-17 12:29:53,264] jepsen worker 73 - jepsen.util 2683 :invoke :cas [24 [25 39]] INFO [2019-12-17 12:29:53,267] jepsen worker 28 - jepsen.util 2098 :fail :cas [9 [45 0]] :ValueNotSwapFrom INFO [2019-12-17 12:29:53,267] jepsen worker 28 - jepsen.util 2098 :invoke :cas [9 [36 5]] INFO [2019-12-17 12:29:53,289] jepsen worker 21 - jepsen.util 3621 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:29:53,289] jepsen worker 21 - jepsen.util 3621 :invoke :read [7 nil] INFO [2019-12-17 12:29:53,290] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16010,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:29:53,291] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16010,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:29:53,291] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16011,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:29:53,291] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16011,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:29:53,291] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16012,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:29:53,291] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16012,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:29:53,296] jepsen worker 78 - jepsen.util 2868 :fail :cas [26 [32 6]] :ValueNotSwapFrom INFO [2019-12-17 12:29:53,296] jepsen worker 78 - jepsen.util 2868 :invoke :write [26 19] INFO [2019-12-17 12:29:53,299] jepsen worker 73 - jepsen.util 2683 :fail :cas [24 [25 39]] :ValueNotSwapFrom INFO [2019-12-17 12:29:53,299] jepsen worker 73 - jepsen.util 2683 :invoke :cas [24 [47 37]] INFO [2019-12-17 12:29:53,308] jepsen worker 42 - jepsen.util 4182 :info :cas [14 [38 22]] :RequestCanceledException INFO [2019-12-17 12:29:53,308] jepsen worker 8 - jepsen.util 3158 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:29:53,308] jepsen worker 8 - jepsen.util 3158 :invoke :read [2 nil] INFO [2019-12-17 12:29:53,309] jepsen worker 42 - jepsen.util 4272 :invoke :read [14 nil] INFO [2019-12-17 12:29:53,324] jepsen worker 26 - jepsen.util 3896 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:29:53,324] jepsen worker 26 - jepsen.util 3896 :invoke :write [8 44] INFO [2019-12-17 12:29:53,329] jepsen worker 28 - jepsen.util 2098 :fail :cas [9 [36 5]] :ValueNotSwapFrom INFO [2019-12-17 12:29:53,329] jepsen worker 28 - jepsen.util 2098 :invoke :write [9 6] INFO [2019-12-17 12:29:53,338] jepsen worker 48 - jepsen.util 3108 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:29:53,338] jepsen worker 48 - jepsen.util 3108 :invoke :read [16 nil] INFO [2019-12-17 12:29:53,354] jepsen worker 17 - jepsen.util 3617 :info :write [5 48] :RequestCanceledException INFO [2019-12-17 12:29:53,355] jepsen worker 17 - jepsen.util 3707 :invoke :write [5 7] INFO [2019-12-17 12:29:53,363] jepsen worker 73 - jepsen.util 2683 :fail :cas [24 [47 37]] :ValueNotSwapFrom INFO [2019-12-17 12:29:53,363] jepsen worker 69 - jepsen.util 2679 :ok :write [23 30] INFO [2019-12-17 12:29:53,364] jepsen worker 69 - jepsen.util 2679 :invoke :read [23 nil] INFO [2019-12-17 12:29:53,364] jepsen worker 73 - jepsen.util 2683 :invoke :write [24 7] INFO [2019-12-17 12:29:53,381] jepsen worker 36 - jepsen.util 2556 :fail :cas [12 [40 43]] :ValueNotSwapFrom INFO [2019-12-17 12:29:53,381] jepsen worker 36 - jepsen.util 2556 :invoke :write [12 35] INFO [2019-12-17 12:29:53,389] jepsen worker 64 - jepsen.util 2224 :ok :write [21 35] INFO [2019-12-17 12:29:53,390] jepsen worker 64 - jepsen.util 2224 :invoke :cas [21 [16 35]] INFO [2019-12-17 12:29:53,423] jepsen worker 0 - jepsen.util 3870 :info :cas [0 [17 15]] :RequestCanceledException INFO [2019-12-17 12:29:53,424] jepsen worker 0 - jepsen.util 3960 :invoke :write [0 35] INFO [2019-12-17 12:29:53,429] jepsen worker 64 - jepsen.util 2224 :fail :cas [21 [16 35]] :ValueNotSwapFrom INFO [2019-12-17 12:29:53,430] jepsen worker 64 - jepsen.util 2224 :invoke :write [21 34] INFO [2019-12-17 12:29:53,436] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16048,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:29:53,436] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16048,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:29:53,436] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16051,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:29:53,436] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16051,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:29:53,436] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16052,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:29:53,436] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16052,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:29:53,456] jepsen worker 53 - jepsen.util 2753 :ok :read [17 42] INFO [2019-12-17 12:29:53,458] jepsen worker 53 - jepsen.util 2753 :invoke :read [17 nil] INFO [2019-12-17 12:29:53,485] jepsen worker 70 - jepsen.util 3220 :ok :write [23 0] INFO [2019-12-17 12:29:53,486] jepsen worker 70 - jepsen.util 3220 :invoke :write [23 12] INFO [2019-12-17 12:29:53,487] jepsen worker 64 - jepsen.util 2224 :ok :write [21 34] INFO [2019-12-17 12:29:53,488] jepsen worker 64 - jepsen.util 2224 :invoke :cas [21 [41 18]] INFO [2019-12-17 12:29:53,489] jepsen worker 54 - jepsen.util 2844 :ok :read [18 29] INFO [2019-12-17 12:29:53,489] jepsen worker 27 - jepsen.util 2457 :fail :read [9 nil] :RequestTimeoutException INFO [2019-12-17 12:29:53,489] jepsen worker 27 - jepsen.util 2457 :invoke :read [9 nil] INFO [2019-12-17 12:29:53,489] jepsen worker 54 - jepsen.util 2844 :invoke :cas [18 [21 33]] INFO [2019-12-17 12:29:53,498] jepsen worker 51 - jepsen.util 2571 :ok :read [17 42] INFO [2019-12-17 12:29:53,499] jepsen worker 51 - jepsen.util 2571 :invoke :read [17 nil] INFO [2019-12-17 12:29:53,523] jepsen worker 5 - jepsen.util 2345 :fail :cas [1 [25 13]] :ValueNotSwapFrom INFO [2019-12-17 12:29:53,523] jepsen worker 5 - jepsen.util 2345 :invoke :cas [1 [12 29]] INFO [2019-12-17 12:29:53,533] jepsen worker 64 - jepsen.util 2224 :fail :cas [21 [41 18]] :ValueNotSwapFrom INFO [2019-12-17 12:29:53,533] jepsen worker 64 - jepsen.util 2224 :invoke :write [21 16] INFO [2019-12-17 12:29:53,534] jepsen worker 51 - jepsen.util 2571 :ok :read [17 42] INFO [2019-12-17 12:29:53,534] jepsen worker 51 - jepsen.util 2571 :invoke :write [17 45] INFO [2019-12-17 12:29:53,582] jepsen worker 46 - jepsen.util 2926 :ok :write [15 43] INFO [2019-12-17 12:29:53,582] jepsen worker 46 - jepsen.util 2926 :invoke :write [15 27] INFO [2019-12-17 12:29:53,590] jepsen worker 52 - jepsen.util 2932 :info :write [17 10] :RequestCanceledException INFO [2019-12-17 12:29:53,593] jepsen worker 52 - jepsen.util 3022 :invoke :cas [17 [19 12]] INFO [2019-12-17 12:29:53,602] jepsen worker 12 - jepsen.util 2892 :ok :read [4 49] INFO [2019-12-17 12:29:53,602] jepsen worker 12 - jepsen.util 2892 :invoke :cas [4 [1 12]] INFO [2019-12-17 12:29:53,608] jepsen worker 79 - jepsen.util 2689 :info :write [26 48] :RequestCanceledException INFO [2019-12-17 12:29:53,609] jepsen worker 79 - jepsen.util 2779 :invoke :read [26 nil] INFO [2019-12-17 12:29:53,611] jepsen worker 55 - jepsen.util 3295 :ok :write [18 16] INFO [2019-12-17 12:29:53,611] jepsen worker 55 - jepsen.util 3295 :invoke :cas [18 [46 10]] INFO [2019-12-17 12:29:53,617] jepsen worker 28 - jepsen.util 2098 :ok :write [9 6] INFO [2019-12-17 12:29:53,618] jepsen worker 28 - jepsen.util 2098 :invoke :read [9 nil] INFO [2019-12-17 12:29:53,632] jepsen worker 64 - jepsen.util 2224 :ok :write [21 16] INFO [2019-12-17 12:29:53,632] jepsen worker 64 - jepsen.util 2224 :invoke :write [21 7] INFO [2019-12-17 12:29:53,639] jepsen worker 28 - jepsen.util 2098 :ok :read [9 6] INFO [2019-12-17 12:29:53,639] jepsen worker 28 - jepsen.util 2098 :invoke :write [9 17] INFO [2019-12-17 12:29:53,650] jepsen worker 52 - jepsen.util 3022 :fail :cas [17 [19 12]] :ValueNotSwapFrom INFO [2019-12-17 12:29:53,651] jepsen worker 52 - jepsen.util 3022 :invoke :read [17 nil] INFO [2019-12-17 12:29:53,664] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16074,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:29:53,664] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16074,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:29:53,705] jepsen worker 89 - jepsen.util 2609 :ok :write [29 28] INFO [2019-12-17 12:29:53,705] jepsen worker 89 - jepsen.util 2609 :invoke :cas [29 [2 36]] INFO [2019-12-17 12:29:53,707] jepsen worker 63 - jepsen.util 2403 :fail :read [21 nil] :RequestTimeoutException INFO [2019-12-17 12:29:53,707] jepsen worker 63 - jepsen.util 2403 :invoke :cas [21 [28 4]] INFO [2019-12-17 12:29:53,711] jepsen worker 18 - jepsen.util 2898 :ok :write [6 15] INFO [2019-12-17 12:29:53,711] jepsen worker 18 - jepsen.util 2898 :invoke :cas [6 [32 25]] INFO [2019-12-17 12:29:53,741] jepsen worker 5 - jepsen.util 2345 :fail :cas [1 [12 29]] :ValueNotSwapFrom INFO [2019-12-17 12:29:53,742] jepsen worker 5 - jepsen.util 2345 :invoke :write [1 2] INFO [2019-12-17 12:29:53,743] jepsen worker 52 - jepsen.util 3022 :ok :read [17 42] INFO [2019-12-17 12:29:53,743] jepsen worker 52 - jepsen.util 3022 :invoke :read [17 nil] INFO [2019-12-17 12:29:53,752] jepsen worker 63 - jepsen.util 2403 :fail :cas [21 [28 4]] :ValueNotSwapFrom INFO [2019-12-17 12:29:53,752] jepsen worker 63 - jepsen.util 2403 :invoke :write [21 14] INFO [2019-12-17 12:29:53,783] jepsen worker 28 - jepsen.util 2098 :ok :write [9 17] INFO [2019-12-17 12:29:53,783] jepsen worker 28 - jepsen.util 2098 :invoke :cas [9 [20 30]] INFO [2019-12-17 12:29:53,808] jepsen worker 25 - jepsen.util 4165 :info :cas [8 [38 33]] :RequestCanceledException INFO [2019-12-17 12:29:53,809] jepsen worker 25 - jepsen.util 4255 :invoke :read [8 nil] INFO [2019-12-17 12:29:53,824] jepsen worker 64 - jepsen.util 2224 :ok :write [21 7] INFO [2019-12-17 12:29:53,824] jepsen worker 37 - jepsen.util 2377 :info :write [12 33] :RequestCanceledException INFO [2019-12-17 12:29:53,824] jepsen worker 58 - jepsen.util 3568 :info :write [19 7] :RequestCanceledException INFO [2019-12-17 12:29:53,824] jepsen worker 77 - jepsen.util 3407 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:29:53,824] jepsen worker 3 - jepsen.util 2613 :info :write [1 35] :RequestCanceledException INFO [2019-12-17 12:29:53,824] jepsen worker 64 - jepsen.util 2224 :invoke :read [21 nil] INFO [2019-12-17 12:29:53,824] jepsen worker 77 - jepsen.util 3407 :invoke :cas [25 [0 27]] INFO [2019-12-17 12:29:53,825] jepsen worker 37 - jepsen.util 2467 :invoke :write [12 43] INFO [2019-12-17 12:29:53,825] jepsen worker 58 - jepsen.util 3658 :invoke :cas [19 [28 21]] INFO [2019-12-17 12:29:53,825] jepsen worker 3 - jepsen.util 2703 :invoke :read [1 nil] INFO [2019-12-17 12:29:53,827] jepsen worker 87 - jepsen.util 2517 :fail :cas [29 [9 15]] :ValueNotSwapFrom INFO [2019-12-17 12:29:53,828] jepsen worker 87 - jepsen.util 2517 :invoke :write [29 17] INFO [2019-12-17 12:29:53,837] jepsen worker 68 - jepsen.util 3488 :info :write [22 43] :RequestCanceledException INFO [2019-12-17 12:29:53,837] jepsen worker 18 - jepsen.util 2898 :fail :cas [6 [32 25]] :ValueNotSwapFrom INFO [2019-12-17 12:29:53,837] jepsen worker 18 - jepsen.util 2898 :invoke :write [6 49] INFO [2019-12-17 12:29:53,838] jepsen worker 68 - jepsen.util 3578 :invoke :read [22 nil] INFO [2019-12-17 12:29:53,843] jepsen worker 28 - jepsen.util 2098 :fail :cas [9 [20 30]] :ValueNotSwapFrom INFO [2019-12-17 12:29:53,844] jepsen worker 28 - jepsen.util 2098 :invoke :read [9 nil] INFO [2019-12-17 12:29:53,844] jepsen worker 73 - jepsen.util 2683 :ok :write [24 7] INFO [2019-12-17 12:29:53,844] jepsen worker 73 - jepsen.util 2683 :invoke :read [24 nil] INFO [2019-12-17 12:29:53,854] jepsen worker 81 - jepsen.util 3501 :info :cas [27 [33 7]] :RequestCanceledException INFO [2019-12-17 12:29:53,855] jepsen worker 81 - jepsen.util 3591 :invoke :cas [27 [16 1]] INFO [2019-12-17 12:29:53,882] jepsen worker 52 - jepsen.util 3022 :ok :read [17 42] INFO [2019-12-17 12:29:53,883] jepsen worker 52 - jepsen.util 3022 :invoke :read [17 nil] INFO [2019-12-17 12:29:53,884] jepsen worker 53 - jepsen.util 2753 :ok :read [17 42] INFO [2019-12-17 12:29:53,884] jepsen worker 53 - jepsen.util 2753 :invoke :read [17 nil] INFO [2019-12-17 12:29:53,887] jepsen worker 82 - jepsen.util 4312 :info :write [27 3] :RequestCanceledException INFO [2019-12-17 12:29:53,887] jepsen worker 2 - jepsen.util 3422 :info :write [0 17] :RequestCanceledException INFO [2019-12-17 12:29:53,889] jepsen worker 2 - jepsen.util 3512 :invoke :read [0 nil] INFO [2019-12-17 12:29:53,889] jepsen worker 82 - jepsen.util 4402 :invoke :cas [27 [45 37]] INFO [2019-12-17 12:29:53,903] jepsen worker 55 - jepsen.util 3295 :fail :cas [18 [46 10]] :ValueNotSwapFrom INFO [2019-12-17 12:29:53,903] jepsen worker 64 - jepsen.util 2224 :ok :read [21 7] INFO [2019-12-17 12:29:53,903] jepsen worker 55 - jepsen.util 3295 :invoke :cas [18 [4 45]] INFO [2019-12-17 12:29:53,903] jepsen worker 64 - jepsen.util 2224 :invoke :write [21 27] INFO [2019-12-17 12:29:53,905] jepsen worker 76 - jepsen.util 3676 :info :write [25 20] :RequestCanceledException INFO [2019-12-17 12:29:53,907] jepsen worker 76 - jepsen.util 3766 :invoke :read [25 nil] INFO [2019-12-17 12:29:53,913] jepsen worker 27 - jepsen.util 2457 :ok :read [9 17] INFO [2019-12-17 12:29:53,913] jepsen worker 27 - jepsen.util 2457 :invoke :write [9 33] INFO [2019-12-17 12:29:53,916] jepsen worker 39 - jepsen.util 3549 :info :write [13 13] :RequestCanceledException INFO [2019-12-17 12:29:53,916] jepsen worker 66 - jepsen.util 3666 :info :cas [22 [37 13]] :RequestCanceledException INFO [2019-12-17 12:29:53,918] jepsen worker 66 - jepsen.util 3756 :invoke :cas [22 [34 31]] INFO [2019-12-17 12:29:53,918] jepsen worker 39 - jepsen.util 3639 :invoke :write [13 32] INFO [2019-12-17 12:29:53,918] jepsen worker 28 - jepsen.util 2098 :ok :read [9 17] INFO [2019-12-17 12:29:53,921] jepsen worker 28 - jepsen.util 2098 :invoke :cas [9 [8 26]] INFO [2019-12-17 12:29:53,937] jepsen worker 61 - jepsen.util 4021 :info :cas [20 [4 49]] :RequestCanceledException INFO [2019-12-17 12:29:53,937] jepsen worker 86 - jepsen.util 3686 :info :write [28 12] :RequestCanceledException INFO [2019-12-17 12:29:53,938] jepsen worker 54 - jepsen.util 2844 :fail :cas [18 [21 33]] :ValueNotSwapFrom INFO [2019-12-17 12:29:53,938] jepsen worker 54 - jepsen.util 2844 :invoke :cas [18 [6 32]] INFO [2019-12-17 12:29:53,938] jepsen worker 61 - jepsen.util 4111 :invoke :cas [20 [21 26]] INFO [2019-12-17 12:29:53,938] jepsen worker 86 - jepsen.util 3776 :invoke :cas [28 [33 24]] INFO [2019-12-17 12:29:53,941] jepsen worker 73 - jepsen.util 2683 :ok :read [24 7] INFO [2019-12-17 12:29:53,942] jepsen worker 73 - jepsen.util 2683 :invoke :cas [24 [19 15]] INFO [2019-12-17 12:29:53,952] jepsen worker 24 - jepsen.util 3534 :info :write [8 12] :RequestCanceledException INFO [2019-12-17 12:29:53,953] jepsen worker 24 - jepsen.util 3624 :invoke :read [8 nil] INFO [2019-12-17 12:29:53,955] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16099,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:29:53,956] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16099,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:29:53,956] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16103,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:29:53,956] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16103,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:29:53,966] jepsen worker 28 - jepsen.util 2098 :fail :cas [9 [8 26]] :ValueNotSwapFrom INFO [2019-12-17 12:29:53,966] jepsen worker 28 - jepsen.util 2098 :invoke :read [9 nil] INFO [2019-12-17 12:29:53,991] jepsen worker 67 - jepsen.util 4207 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:29:53,991] jepsen worker 67 - jepsen.util 4207 :invoke :read [22 nil] INFO [2019-12-17 12:29:53,992] jepsen worker 53 - jepsen.util 2753 :ok :read [17 42] INFO [2019-12-17 12:29:53,992] jepsen worker 52 - jepsen.util 3022 :ok :read [17 42] INFO [2019-12-17 12:29:53,992] jepsen worker 53 - jepsen.util 2753 :invoke :write [17 3] INFO [2019-12-17 12:29:53,993] jepsen worker 52 - jepsen.util 3022 :invoke :cas [17 [34 3]] INFO [2019-12-17 12:29:54,021] jepsen worker 28 - jepsen.util 2098 :ok :read [9 17] INFO [2019-12-17 12:29:54,022] jepsen worker 28 - jepsen.util 2098 :invoke :cas [9 [14 42]] INFO [2019-12-17 12:29:54,033] jepsen worker 55 - jepsen.util 3295 :fail :cas [18 [4 45]] :ValueNotSwapFrom INFO [2019-12-17 12:29:54,033] jepsen worker 55 - jepsen.util 3295 :invoke :write [18 34] INFO [2019-12-17 12:29:54,040] jepsen worker 46 - jepsen.util 2926 :ok :write [15 27] INFO [2019-12-17 12:29:54,040] jepsen worker 46 - jepsen.util 2926 :invoke :write [15 3] INFO [2019-12-17 12:29:54,054] jepsen worker 88 - jepsen.util 2788 :info :write [29 42] :RequestCanceledException INFO [2019-12-17 12:29:54,055] jepsen worker 88 - jepsen.util 2878 :invoke :cas [29 [6 43]] INFO [2019-12-17 12:29:54,058] jepsen worker 12 - jepsen.util 2892 :fail :cas [4 [1 12]] :ValueNotSwapFrom INFO [2019-12-17 12:29:54,058] jepsen worker 12 - jepsen.util 2892 :invoke :cas [4 [49 24]] INFO [2019-12-17 12:29:54,063] jepsen worker 28 - jepsen.util 2098 :fail :cas [9 [14 42]] :ValueNotSwapFrom INFO [2019-12-17 12:29:54,063] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16124,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:29:54,063] jepsen worker 28 - jepsen.util 2098 :invoke :read [9 nil] INFO [2019-12-17 12:29:54,063] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16124,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:29:54,064] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16127,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:29:54,064] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16127,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:29:54,083] jepsen worker 27 - jepsen.util 2457 :ok :write [9 33] INFO [2019-12-17 12:29:54,083] jepsen worker 27 - jepsen.util 2457 :invoke :cas [9 [48 42]] INFO [2019-12-17 12:29:54,086] jepsen worker 28 - jepsen.util 2098 :ok :read [9 17] INFO [2019-12-17 12:29:54,086] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:29:54,086] jepsen worker 28 - jepsen.util 2098 :invoke :cas [9 [11 23]] INFO [2019-12-17 12:29:54,086] jepsen nemesis - couchbase.util Rebalance check count: 117 INFO [2019-12-17 12:29:54,086] jepsen nemesis - couchbase.util Rebalance stuck count: 98 INFO [2019-12-17 12:29:54,089] jepsen worker 89 - jepsen.util 2609 :fail :cas [29 [2 36]] :ValueNotSwapFrom INFO [2019-12-17 12:29:54,089] jepsen worker 89 - jepsen.util 2609 :invoke :read [29 nil] INFO [2019-12-17 12:29:54,102] jepsen worker 45 - jepsen.util 2835 :info :write [15 10] :RequestCanceledException INFO [2019-12-17 12:29:54,102] jepsen worker 29 - jepsen.util 2369 :fail :read [9 nil] :RequestTimeoutException INFO [2019-12-17 12:29:54,103] jepsen worker 29 - jepsen.util 2369 :invoke :read [9 nil] INFO [2019-12-17 12:29:54,103] jepsen worker 44 - jepsen.util 3644 :info :write [14 0] :RequestCanceledException INFO [2019-12-17 12:29:54,104] jepsen worker 45 - jepsen.util 2925 :invoke :write [15 7] INFO [2019-12-17 12:29:54,104] jepsen worker 44 - jepsen.util 3734 :invoke :cas [14 [35 17]] INFO [2019-12-17 12:29:54,109] jepsen worker 27 - jepsen.util 2457 :fail :cas [9 [48 42]] :ValueNotSwapFrom INFO [2019-12-17 12:29:54,110] jepsen worker 27 - jepsen.util 2457 :invoke :write [9 14] INFO [2019-12-17 12:29:54,111] jepsen worker 3 - jepsen.util 2703 :ok :read [1 18] INFO [2019-12-17 12:29:54,111] jepsen worker 3 - jepsen.util 2703 :invoke :cas [1 [43 22]] INFO [2019-12-17 12:29:54,118] jepsen worker 63 - jepsen.util 2403 :ok :write [21 14] INFO [2019-12-17 12:29:54,118] jepsen worker 63 - jepsen.util 2403 :invoke :cas [21 [6 24]] INFO [2019-12-17 12:29:54,123] jepsen worker 6 - jepsen.util 3876 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:29:54,123] jepsen worker 6 - jepsen.util 3876 :invoke :read [2 nil] INFO [2019-12-17 12:29:54,126] jepsen worker 28 - jepsen.util 2098 :fail :cas [9 [11 23]] :ValueNotSwapFrom INFO [2019-12-17 12:29:54,126] jepsen worker 28 - jepsen.util 2098 :invoke :read [9 nil] INFO [2019-12-17 12:29:54,135] jepsen worker 52 - jepsen.util 3022 :fail :cas [17 [34 3]] :ValueNotSwapFrom INFO [2019-12-17 12:29:54,135] jepsen worker 52 - jepsen.util 3022 :invoke :read [17 nil] INFO [2019-12-17 12:29:54,137] jepsen worker 34 - jepsen.util 3904 :info :cas [11 [2 8]] :RequestCanceledException INFO [2019-12-17 12:29:54,138] jepsen worker 34 - jepsen.util 3994 :invoke :read [11 nil] INFO [2019-12-17 12:29:54,154] jepsen worker 49 - jepsen.util 2929 :info :cas [16 [14 15]] :RequestCanceledException INFO [2019-12-17 12:29:54,154] jepsen worker 60 - jepsen.util 3930 :info :cas [20 [38 12]] :RequestCanceledException INFO [2019-12-17 12:29:54,155] jepsen worker 49 - jepsen.util 3019 :invoke :write [16 44] INFO [2019-12-17 12:29:54,155] jepsen worker 60 - jepsen.util 4020 :invoke :read [20 nil] INFO [2019-12-17 12:29:54,158] jepsen worker 29 - jepsen.util 2369 :ok :read [9 33] INFO [2019-12-17 12:29:54,158] jepsen worker 29 - jepsen.util 2369 :invoke :cas [9 [18 30]] INFO [2019-12-17 12:29:54,189] jepsen worker 63 - jepsen.util 2403 :fail :cas [21 [6 24]] :ValueNotSwapFrom INFO [2019-12-17 12:29:54,189] jepsen worker 63 - jepsen.util 2403 :invoke :read [21 nil] INFO [2019-12-17 12:29:54,210] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16160,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:29:54,210] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16160,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:29:54,210] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16164,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:29:54,210] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16164,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:29:54,213] jepsen worker 52 - jepsen.util 3022 :ok :read [17 42] INFO [2019-12-17 12:29:54,213] jepsen worker 52 - jepsen.util 3022 :invoke :cas [17 [47 5]] INFO [2019-12-17 12:29:54,213] jepsen worker 28 - jepsen.util 2098 :ok :read [9 33] INFO [2019-12-17 12:29:54,213] jepsen worker 28 - jepsen.util 2098 :invoke :read [9 nil] INFO [2019-12-17 12:29:54,223] jepsen worker 7 - jepsen.util 3517 :info :write [2 36] :RequestCanceledException INFO [2019-12-17 12:29:54,224] jepsen worker 7 - jepsen.util 3607 :invoke :read [2 nil] INFO [2019-12-17 12:29:54,225] jepsen worker 29 - jepsen.util 2369 :fail :cas [9 [18 30]] :ValueNotSwapFrom INFO [2019-12-17 12:29:54,225] jepsen worker 29 - jepsen.util 2369 :invoke :cas [9 [21 28]] INFO [2019-12-17 12:29:54,247] jepsen worker 79 - jepsen.util 2779 :ok :read [26 45] INFO [2019-12-17 12:29:54,247] jepsen worker 79 - jepsen.util 2779 :invoke :cas [26 [35 25]] INFO [2019-12-17 12:29:54,252] jepsen worker 54 - jepsen.util 2844 :fail :cas [18 [6 32]] :ValueNotSwapFrom INFO [2019-12-17 12:29:54,252] jepsen worker 54 - jepsen.util 2844 :invoke :write [18 47] INFO [2019-12-17 12:29:54,272] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16153,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:29:54,272] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16153,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:29:54,273] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16154,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:29:54,273] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16154,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:29:54,273] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16155,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:29:54,273] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16155,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:29:54,274] jepsen worker 56 - jepsen.util 2576 :fail :cas [18 [42 5]] :ValueNotSwapFrom INFO [2019-12-17 12:29:54,274] jepsen worker 56 - jepsen.util 2576 :invoke :write [18 39] INFO [2019-12-17 12:29:54,283] jepsen worker 28 - jepsen.util 2098 :ok :read [9 33] INFO [2019-12-17 12:29:54,284] jepsen worker 28 - jepsen.util 2098 :invoke :read [9 nil] INFO [2019-12-17 12:29:54,285] jepsen worker 4 - jepsen.util 1984 :fail :cas [1 [7 4]] :ValueNotSwapFrom INFO [2019-12-17 12:29:54,285] jepsen worker 4 - jepsen.util 1984 :invoke :write [1 10] INFO [2019-12-17 12:29:54,290] jepsen worker 23 - jepsen.util 3533 :info :write [7 12] :RequestCanceledException INFO [2019-12-17 12:29:54,291] jepsen worker 23 - jepsen.util 3623 :invoke :write [7 0] INFO [2019-12-17 12:29:54,323] jepsen worker 32 - jepsen.util 3182 :info :write [10 2] :RequestCanceledException INFO [2019-12-17 12:29:54,323] jepsen worker 72 - jepsen.util 2772 :info :write [24 22] :RequestCanceledException INFO [2019-12-17 12:29:54,324] jepsen worker 13 - jepsen.util 2443 :info :cas [4 [49 40]] :RequestCanceledException INFO [2019-12-17 12:29:54,325] jepsen worker 32 - jepsen.util 3272 :invoke :write [10 1] INFO [2019-12-17 12:29:54,326] jepsen worker 72 - jepsen.util 2862 :invoke :cas [24 [10 44]] INFO [2019-12-17 12:29:54,326] jepsen worker 13 - jepsen.util 2533 :invoke :write [4 2] INFO [2019-12-17 12:29:54,339] jepsen worker 38 - jepsen.util 2738 :fail :cas [12 [44 18]] :ValueNotSwapFrom INFO [2019-12-17 12:29:54,340] jepsen worker 38 - jepsen.util 2738 :invoke :cas [12 [24 20]] INFO [2019-12-17 12:29:54,343] jepsen worker 63 - jepsen.util 2403 :ok :read [21 14] INFO [2019-12-17 12:29:54,344] jepsen worker 28 - jepsen.util 2098 :ok :read [9 33] INFO [2019-12-17 12:29:54,344] jepsen worker 63 - jepsen.util 2403 :invoke :write [21 28] INFO [2019-12-17 12:29:54,344] jepsen worker 28 - jepsen.util 2098 :invoke :write [9 13] INFO [2019-12-17 12:29:54,354] jepsen worker 65 - jepsen.util 2405 :info :cas [21 [47 31]] :RequestCanceledException INFO [2019-12-17 12:29:54,354] jepsen worker 11 - jepsen.util 3071 :fail :read [3 nil] :RequestTimeoutException INFO [2019-12-17 12:29:54,355] jepsen worker 11 - jepsen.util 3071 :invoke :read [3 nil] INFO [2019-12-17 12:29:54,355] jepsen worker 65 - jepsen.util 2495 :invoke :cas [21 [38 29]] INFO [2019-12-17 12:29:54,367] jepsen worker 88 - jepsen.util 2878 :fail :cas [29 [6 43]] :ValueNotSwapFrom INFO [2019-12-17 12:29:54,368] jepsen worker 88 - jepsen.util 2878 :invoke :write [29 44] INFO [2019-12-17 12:29:54,375] jepsen worker 52 - jepsen.util 3022 :fail :cas [17 [47 5]] :ValueNotSwapFrom INFO [2019-12-17 12:29:54,375] jepsen worker 52 - jepsen.util 3022 :invoke :read [17 nil] INFO [2019-12-17 12:29:54,387] jepsen worker 89 - jepsen.util 2609 :ok :read [29 28] INFO [2019-12-17 12:29:54,387] jepsen worker 89 - jepsen.util 2609 :invoke :cas [29 [0 25]] INFO [2019-12-17 12:29:54,391] jepsen worker 69 - jepsen.util 2679 :ok :read [23 0] INFO [2019-12-17 12:29:54,392] jepsen worker 69 - jepsen.util 2679 :invoke :write [23 49] INFO [2019-12-17 12:29:54,410] jepsen worker 29 - jepsen.util 2369 :fail :cas [9 [21 28]] :ValueNotSwapFrom INFO [2019-12-17 12:29:54,410] jepsen worker 29 - jepsen.util 2369 :invoke :write [9 38] INFO [2019-12-17 12:29:54,416] jepsen worker 72 - jepsen.util 2862 :fail :cas [24 [10 44]] :ValueNotSwapFrom INFO [2019-12-17 12:29:54,416] jepsen worker 72 - jepsen.util 2862 :invoke :write [24 3] INFO [2019-12-17 12:29:54,423] jepsen worker 10 - jepsen.util 2530 :info :write [3 26] :RequestCanceledException INFO [2019-12-17 12:29:54,424] jepsen worker 10 - jepsen.util 2620 :invoke :read [3 nil] INFO [2019-12-17 12:29:54,437] jepsen worker 47 - jepsen.util 3107 :info :cas [15 [47 20]] :RequestCanceledException INFO [2019-12-17 12:29:54,438] jepsen worker 47 - jepsen.util 3197 :invoke :cas [15 [5 8]] INFO [2019-12-17 12:29:54,440] jepsen worker 52 - jepsen.util 3022 :ok :read [17 42] INFO [2019-12-17 12:29:54,440] jepsen worker 52 - jepsen.util 3022 :invoke :cas [17 [0 40]] INFO [2019-12-17 12:29:54,456] jepsen worker 11 - jepsen.util 3071 :ok :read [3 46] INFO [2019-12-17 12:29:54,456] jepsen worker 11 - jepsen.util 3071 :invoke :cas [3 [35 45]] INFO [2019-12-17 12:29:54,460] jepsen worker 45 - jepsen.util 2925 :ok :write [15 7] INFO [2019-12-17 12:29:54,460] jepsen worker 45 - jepsen.util 2925 :invoke :write [15 10] INFO [2019-12-17 12:29:54,482] jepsen worker 65 - jepsen.util 2495 :fail :cas [21 [38 29]] :ValueNotSwapFrom INFO [2019-12-17 12:29:54,482] jepsen worker 65 - jepsen.util 2495 :invoke :cas [21 [47 12]] INFO [2019-12-17 12:29:54,489] jepsen worker 31 - jepsen.util 2551 :info :write [10 34] :RequestCanceledException INFO [2019-12-17 12:29:54,491] jepsen worker 31 - jepsen.util 2641 :invoke :read [10 nil] INFO [2019-12-17 12:29:54,499] jepsen worker 3 - jepsen.util 2703 :fail :cas [1 [43 22]] :ValueNotSwapFrom INFO [2019-12-17 12:29:54,499] jepsen worker 3 - jepsen.util 2703 :invoke :cas [1 [10 11]] INFO [2019-12-17 12:29:54,517] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16191,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:29:54,517] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16191,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:29:54,517] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16195,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:29:54,517] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16195,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:29:54,517] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16196,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:29:54,517] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16196,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:29:54,524] jepsen worker 80 - jepsen.util 2600 :info :write [26 23] :RequestCanceledException INFO [2019-12-17 12:29:54,524] jepsen worker 80 - jepsen.util 2690 :invoke :write [26 14] INFO [2019-12-17 12:29:54,537] jepsen worker 30 - jepsen.util 3180 :info :cas [10 [23 40]] :RequestCanceledException INFO [2019-12-17 12:29:54,538] jepsen worker 30 - jepsen.util 3270 :invoke :write [10 24] INFO [2019-12-17 12:29:54,546] jepsen worker 52 - jepsen.util 3022 :fail :cas [17 [0 40]] :ValueNotSwapFrom INFO [2019-12-17 12:29:54,546] jepsen worker 52 - jepsen.util 3022 :invoke :cas [17 [44 13]] INFO [2019-12-17 12:29:54,559] jepsen worker 47 - jepsen.util 3197 :fail :cas [15 [5 8]] :ValueNotSwapFrom INFO [2019-12-17 12:29:54,559] jepsen worker 47 - jepsen.util 3197 :invoke :cas [15 [23 1]] INFO [2019-12-17 12:29:54,578] jepsen worker 10 - jepsen.util 2620 :ok :read [3 46] INFO [2019-12-17 12:29:54,578] jepsen worker 10 - jepsen.util 2620 :invoke :read [3 nil] INFO [2019-12-17 12:29:54,582] jepsen worker 52 - jepsen.util 3022 :fail :cas [17 [44 13]] :ValueNotSwapFrom INFO [2019-12-17 12:29:54,582] jepsen worker 52 - jepsen.util 3022 :invoke :read [17 nil] INFO [2019-12-17 12:29:54,596] jepsen worker 11 - jepsen.util 3071 :fail :cas [3 [35 45]] :ValueNotSwapFrom INFO [2019-12-17 12:29:54,597] jepsen worker 11 - jepsen.util 3071 :invoke :cas [3 [5 14]] INFO [2019-12-17 12:29:54,607] jepsen worker 19 - jepsen.util 2899 :info :write [6 37] :RequestCanceledException INFO [2019-12-17 12:29:54,608] jepsen worker 19 - jepsen.util 2989 :invoke :write [6 36] INFO [2019-12-17 12:29:54,631] jepsen worker 65 - jepsen.util 2495 :fail :cas [21 [47 12]] :ValueNotSwapFrom INFO [2019-12-17 12:29:54,631] jepsen worker 65 - jepsen.util 2495 :invoke :write [21 30] INFO [2019-12-17 12:29:54,647] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16214,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:29:54,647] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16214,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:29:54,647] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16217,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:29:54,647] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16217,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:29:54,647] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16218,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:29:54,647] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16218,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:29:54,655] jepsen worker 28 - jepsen.util 2098 :ok :write [9 13] INFO [2019-12-17 12:29:54,656] jepsen worker 28 - jepsen.util 2098 :invoke :cas [9 [0 48]] INFO [2019-12-17 12:29:54,659] jepsen worker 52 - jepsen.util 3022 :ok :read [17 42] INFO [2019-12-17 12:29:54,659] jepsen worker 52 - jepsen.util 3022 :invoke :write [17 5] INFO [2019-12-17 12:29:54,679] jepsen worker 11 - jepsen.util 3071 :fail :cas [3 [5 14]] :ValueNotSwapFrom INFO [2019-12-17 12:29:54,679] jepsen worker 11 - jepsen.util 3071 :invoke :write [3 28] INFO [2019-12-17 12:29:54,708] jepsen worker 16 - jepsen.util 3706 :info :write [5 4] :RequestCanceledException INFO [2019-12-17 12:29:54,709] jepsen worker 16 - jepsen.util 3796 :invoke :write [5 27] INFO [2019-12-17 12:29:54,715] jepsen worker 79 - jepsen.util 2779 :fail :cas [26 [35 25]] :ValueNotSwapFrom INFO [2019-12-17 12:29:54,715] jepsen worker 79 - jepsen.util 2779 :invoke :write [26 16] INFO [2019-12-17 12:29:54,721] jepsen worker 89 - jepsen.util 2609 :fail :cas [29 [0 25]] :ValueNotSwapFrom INFO [2019-12-17 12:29:54,721] jepsen worker 89 - jepsen.util 2609 :invoke :read [29 nil] INFO [2019-12-17 12:29:54,743] jepsen worker 10 - jepsen.util 2620 :ok :read [3 46] INFO [2019-12-17 12:29:54,743] jepsen worker 10 - jepsen.util 2620 :invoke :cas [3 [28 27]] INFO [2019-12-17 12:29:54,770] jepsen worker 10 - jepsen.util 2620 :fail :cas [3 [28 27]] :ValueNotSwapFrom INFO [2019-12-17 12:29:54,770] jepsen worker 10 - jepsen.util 2620 :invoke :write [3 39] INFO [2019-12-17 12:29:54,791] jepsen worker 3 - jepsen.util 2703 :fail :cas [1 [10 11]] :ValueNotSwapFrom INFO [2019-12-17 12:29:54,791] jepsen worker 3 - jepsen.util 2703 :invoke :cas [1 [47 41]] INFO [2019-12-17 12:29:54,824] jepsen worker 35 - jepsen.util 3455 :info :cas [11 [15 20]] :RequestCanceledException INFO [2019-12-17 12:29:54,825] jepsen worker 35 - jepsen.util 3545 :invoke :cas [11 [42 18]] INFO [2019-12-17 12:29:54,842] jepsen worker 32 - jepsen.util 3272 :ok :write [10 1] INFO [2019-12-17 12:29:54,842] jepsen worker 32 - jepsen.util 3272 :invoke :cas [10 [8 45]] INFO [2019-12-17 12:29:54,842] jepsen worker 45 - jepsen.util 2925 :ok :write [15 10] INFO [2019-12-17 12:29:54,842] jepsen worker 45 - jepsen.util 2925 :invoke :cas [15 [26 30]] INFO [2019-12-17 12:29:54,861] jepsen worker 45 - jepsen.util 2925 :fail :cas [15 [26 30]] :ValueNotSwapFrom INFO [2019-12-17 12:29:54,861] jepsen worker 45 - jepsen.util 2925 :invoke :write [15 27] INFO [2019-12-17 12:29:54,904] jepsen worker 29 - jepsen.util 2369 :ok :write [9 38] INFO [2019-12-17 12:29:54,904] jepsen worker 29 - jepsen.util 2369 :invoke :read [9 nil] INFO [2019-12-17 12:29:54,916] jepsen worker 32 - jepsen.util 3272 :fail :cas [10 [8 45]] :ValueNotSwapFrom INFO [2019-12-17 12:29:54,917] jepsen worker 32 - jepsen.util 3272 :invoke :cas [10 [16 2]] INFO [2019-12-17 12:29:54,923] jepsen worker 29 - jepsen.util 2369 :ok :read [9 38] INFO [2019-12-17 12:29:54,923] jepsen worker 29 - jepsen.util 2369 :invoke :cas [9 [16 12]] INFO [2019-12-17 12:29:54,923] jepsen worker 74 - jepsen.util 2774 :info :write [24 39] :RequestCanceledException INFO [2019-12-17 12:29:54,924] jepsen worker 74 - jepsen.util 2864 :invoke :cas [24 [49 39]] INFO [2019-12-17 12:29:54,932] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16234,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:29:54,932] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16234,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:29:54,932] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16236,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:29:54,932] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16236,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:29:54,933] jepsen worker 32 - jepsen.util 3272 :fail :cas [10 [16 2]] :ValueNotSwapFrom INFO [2019-12-17 12:29:54,933] jepsen worker 32 - jepsen.util 3272 :invoke :write [10 12] INFO [2019-12-17 12:29:54,954] jepsen worker 59 - jepsen.util 3569 :info :cas [19 [43 48]] :RequestCanceledException INFO [2019-12-17 12:29:54,954] jepsen worker 43 - jepsen.util 3373 :info :write [14 11] :RequestCanceledException INFO [2019-12-17 12:29:54,955] jepsen worker 59 - jepsen.util 3659 :invoke :write [19 43] INFO [2019-12-17 12:29:54,955] jepsen worker 43 - jepsen.util 3463 :invoke :write [14 16] INFO [2019-12-17 12:29:54,979] jepsen worker 29 - jepsen.util 2369 :fail :cas [9 [16 12]] :ValueNotSwapFrom INFO [2019-12-17 12:29:54,979] jepsen worker 29 - jepsen.util 2369 :invoke :write [9 4] INFO [2019-12-17 12:29:55,023] jepsen worker 15 - jepsen.util 3345 :info :cas [5 [28 45]] :RequestCanceledException INFO [2019-12-17 12:29:55,023] jepsen worker 50 - jepsen.util 4010 :info :write [16 14] :RequestCanceledException INFO [2019-12-17 12:29:55,023] jepsen worker 9 - jepsen.util 3159 :info :cas [3 [21 23]] :RequestCanceledException INFO [2019-12-17 12:29:55,024] jepsen worker 15 - jepsen.util 3435 :invoke :read [5 nil] INFO [2019-12-17 12:29:55,025] jepsen worker 74 - jepsen.util 2864 :fail :cas [24 [49 39]] :ValueNotSwapFrom INFO [2019-12-17 12:29:55,025] jepsen worker 74 - jepsen.util 2864 :invoke :read [24 nil] INFO [2019-12-17 12:29:55,028] jepsen worker 50 - jepsen.util 4100 :invoke :cas [16 [1 14]] INFO [2019-12-17 12:29:55,028] jepsen worker 9 - jepsen.util 3249 :invoke :cas [3 [21 35]] INFO [2019-12-17 12:29:55,036] jepsen worker 72 - jepsen.util 2862 :ok :write [24 3] INFO [2019-12-17 12:29:55,036] jepsen worker 19 - jepsen.util 2989 :ok :write [6 36] INFO [2019-12-17 12:29:55,036] jepsen worker 72 - jepsen.util 2862 :invoke :read [24 nil] INFO [2019-12-17 12:29:55,036] jepsen worker 19 - jepsen.util 2989 :invoke :write [6 6] INFO [2019-12-17 12:29:55,037] jepsen worker 85 - jepsen.util 4135 :info :write [28 25] :RequestCanceledException INFO [2019-12-17 12:29:55,037] jepsen worker 84 - jepsen.util 3504 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:29:55,037] jepsen worker 71 - jepsen.util 2771 :info :write [23 17] :RequestCanceledException INFO [2019-12-17 12:29:55,037] jepsen worker 84 - jepsen.util 3504 :invoke :write [28 7] INFO [2019-12-17 12:29:55,038] jepsen worker 71 - jepsen.util 2861 :invoke :cas [23 [26 30]] INFO [2019-12-17 12:29:55,038] jepsen worker 85 - jepsen.util 4225 :invoke :cas [28 [29 23]] INFO [2019-12-17 12:29:55,052] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16247,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:29:55,052] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16247,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:29:55,052] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16250,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:29:55,052] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16250,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:29:55,052] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16251,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:29:55,052] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16251,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:29:55,054] jepsen worker 62 - jepsen.util 3482 :info :cas [20 [28 15]] :RequestCanceledException INFO [2019-12-17 12:29:55,055] jepsen worker 62 - jepsen.util 3572 :invoke :read [20 nil] INFO [2019-12-17 12:29:55,085] jepsen worker 28 - jepsen.util 2098 :fail :cas [9 [0 48]] :ValueNotSwapFrom INFO [2019-12-17 12:29:55,085] jepsen worker 28 - jepsen.util 2098 :invoke :read [9 nil] INFO [2019-12-17 12:29:55,126] jepsen worker 89 - jepsen.util 2609 :ok :read [29 28] INFO [2019-12-17 12:29:55,127] jepsen worker 89 - jepsen.util 2609 :invoke :cas [29 [9 29]] INFO [2019-12-17 12:29:55,128] jepsen worker 72 - jepsen.util 2862 :ok :read [24 3] INFO [2019-12-17 12:29:55,128] jepsen worker 72 - jepsen.util 2862 :invoke :write [24 4] INFO [2019-12-17 12:29:55,137] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:29:55,137] jepsen nemesis - couchbase.util Rebalance check count: 118 INFO [2019-12-17 12:29:55,137] jepsen nemesis - couchbase.util Rebalance stuck count: 99 INFO [2019-12-17 12:29:55,151] jepsen worker 20 - jepsen.util 2360 :info :write [6 45] :RequestCanceledException INFO [2019-12-17 12:29:55,152] jepsen worker 20 - jepsen.util 2450 :invoke :write [6 27] INFO [2019-12-17 12:29:55,155] jepsen worker 3 - jepsen.util 2703 :fail :cas [1 [47 41]] :ValueNotSwapFrom INFO [2019-12-17 12:29:55,155] jepsen worker 3 - jepsen.util 2703 :invoke :cas [1 [4 36]] INFO [2019-12-17 12:29:55,180] jepsen worker 28 - jepsen.util 2098 :ok :read [9 38] INFO [2019-12-17 12:29:55,180] jepsen worker 28 - jepsen.util 2098 :invoke :read [9 nil] INFO [2019-12-17 12:29:55,190] jepsen worker 75 - jepsen.util 3765 :info :write [25 29] :RequestCanceledException INFO [2019-12-17 12:29:55,191] jepsen worker 75 - jepsen.util 3855 :invoke :cas [25 [28 46]] INFO [2019-12-17 12:29:55,207] jepsen worker 1 - jepsen.util 3061 :info :cas [0 [27 44]] :RequestCanceledException INFO [2019-12-17 12:29:55,209] jepsen worker 1 - jepsen.util 3151 :invoke :write [0 12] INFO [2019-12-17 12:29:55,214] jepsen worker 71 - jepsen.util 2861 :fail :cas [23 [26 30]] :ValueNotSwapFrom INFO [2019-12-17 12:29:55,214] jepsen worker 71 - jepsen.util 2861 :invoke :cas [23 [47 36]] INFO [2019-12-17 12:29:55,223] jepsen worker 14 - jepsen.util 2534 :info :write [4 24] :RequestCanceledException INFO [2019-12-17 12:29:55,223] jepsen worker 41 - jepsen.util 3911 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:29:55,223] jepsen worker 41 - jepsen.util 3911 :invoke :write [13 3] INFO [2019-12-17 12:29:55,224] jepsen worker 14 - jepsen.util 2624 :invoke :write [4 4] INFO [2019-12-17 12:29:55,257] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16269,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:29:55,257] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16269,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:29:55,306] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16278,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:29:55,306] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16278,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:29:55,306] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16280,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:29:55,306] jepsen worker 10 - jepsen.util 2620 :ok :write [3 39] INFO [2019-12-17 12:29:55,306] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16280,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:29:55,306] jepsen worker 10 - jepsen.util 2620 :invoke :read [3 nil] INFO [2019-12-17 12:29:55,307] jepsen worker 83 - jepsen.util 3953 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:29:55,307] jepsen worker 40 - jepsen.util 3910 :info :cas [13 [9 0]] :RequestCanceledException INFO [2019-12-17 12:29:55,307] jepsen worker 57 - jepsen.util 3477 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:29:55,307] jepsen worker 83 - jepsen.util 3953 :invoke :read [27 nil] INFO [2019-12-17 12:29:55,307] jepsen worker 57 - jepsen.util 3477 :invoke :write [19 48] INFO [2019-12-17 12:29:55,308] jepsen worker 40 - jepsen.util 4000 :invoke :cas [13 [29 7]] INFO [2019-12-17 12:29:55,323] jepsen worker 33 - jepsen.util 3903 :info :cas [11 [49 0]] :RequestCanceledException INFO [2019-12-17 12:29:55,323] jepsen worker 78 - jepsen.util 2868 :info :write [26 19] :RequestCanceledException INFO [2019-12-17 12:29:55,325] jepsen worker 33 - jepsen.util 3993 :invoke :write [11 31] INFO [2019-12-17 12:29:55,325] jepsen worker 78 - jepsen.util 2958 :invoke :write [26 36] INFO [2019-12-17 12:29:55,325] jepsen worker 10 - jepsen.util 2620 :ok :read [3 39] INFO [2019-12-17 12:29:55,325] jepsen worker 10 - jepsen.util 2620 :invoke :cas [3 [3 34]] INFO [2019-12-17 12:29:55,337] jepsen worker 22 - jepsen.util 3712 :info :cas [7 [6 47]] :RequestCanceledException INFO [2019-12-17 12:29:55,338] jepsen worker 22 - jepsen.util 3802 :invoke :read [7 nil] INFO [2019-12-17 12:29:55,349] jepsen worker 3 - jepsen.util 2703 :fail :cas [1 [4 36]] :ValueNotSwapFrom INFO [2019-12-17 12:29:55,349] jepsen worker 3 - jepsen.util 2703 :invoke :cas [1 [41 18]] INFO [2019-12-17 12:29:55,352] jepsen worker 10 - jepsen.util 2620 :fail :cas [3 [3 34]] :ValueNotSwapFrom INFO [2019-12-17 12:29:55,352] jepsen worker 10 - jepsen.util 2620 :invoke :cas [3 [20 19]] INFO [2019-12-17 12:29:55,354] jepsen worker 42 - jepsen.util 4272 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:29:55,355] jepsen worker 42 - jepsen.util 4272 :invoke :write [14 26] INFO [2019-12-17 12:29:55,357] jepsen worker 38 - jepsen.util 2738 :fail :cas [12 [24 20]] :ValueNotSwapFrom INFO [2019-12-17 12:29:55,357] jepsen worker 38 - jepsen.util 2738 :invoke :write [12 46] INFO [2019-12-17 12:29:55,366] jepsen worker 10 - jepsen.util 2620 :fail :cas [3 [20 19]] :ValueNotSwapFrom INFO [2019-12-17 12:29:55,366] jepsen worker 10 - jepsen.util 2620 :invoke :cas [3 [17 3]] INFO [2019-12-17 12:29:55,378] jepsen worker 10 - jepsen.util 2620 :fail :cas [3 [17 3]] :ValueNotSwapFrom INFO [2019-12-17 12:29:55,378] jepsen worker 10 - jepsen.util 2620 :invoke :cas [3 [9 10]] INFO [2019-12-17 12:29:55,390] jepsen worker 21 - jepsen.util 3621 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:29:55,390] jepsen worker 21 - jepsen.util 3621 :invoke :write [7 20] INFO [2019-12-17 12:29:55,391] jepsen worker 10 - jepsen.util 2620 :fail :cas [3 [9 10]] :ValueNotSwapFrom INFO [2019-12-17 12:29:55,392] jepsen worker 10 - jepsen.util 2620 :invoke :read [3 nil] INFO [2019-12-17 12:29:55,411] jepsen worker 8 - jepsen.util 3158 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:29:55,412] jepsen worker 8 - jepsen.util 3158 :invoke :cas [2 [2 31]] INFO [2019-12-17 12:29:55,421] jepsen worker 10 - jepsen.util 2620 :ok :read [3 39] INFO [2019-12-17 12:29:55,421] jepsen worker 10 - jepsen.util 2620 :invoke :write [3 42] INFO [2019-12-17 12:29:55,422] jepsen worker 17 - jepsen.util 3707 :info :write [5 7] :RequestCanceledException INFO [2019-12-17 12:29:55,424] jepsen worker 26 - jepsen.util 3896 :info :write [8 44] :RequestCanceledException INFO [2019-12-17 12:29:55,424] jepsen worker 17 - jepsen.util 3797 :invoke :cas [5 [31 38]] INFO [2019-12-17 12:29:55,424] jepsen worker 36 - jepsen.util 2556 :info :write [12 35] :RequestCanceledException INFO [2019-12-17 12:29:55,425] jepsen worker 36 - jepsen.util 2646 :invoke :write [12 3] INFO [2019-12-17 12:29:55,425] jepsen worker 26 - jepsen.util 3986 :invoke :cas [8 [20 49]] INFO [2019-12-17 12:29:55,438] jepsen worker 48 - jepsen.util 3108 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:29:55,441] jepsen worker 48 - jepsen.util 3108 :invoke :write [16 13] INFO [2019-12-17 12:29:55,490] jepsen worker 0 - jepsen.util 3960 :info :write [0 35] :RequestCanceledException INFO [2019-12-17 12:29:55,491] jepsen worker 0 - jepsen.util 4050 :invoke :cas [0 [47 22]] INFO [2019-12-17 12:29:55,504] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16303,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:29:55,505] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16303,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:29:55,505] jepsen worker 71 - jepsen.util 2861 :fail :cas [23 [47 36]] :ValueNotSwapFrom INFO [2019-12-17 12:29:55,505] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16307,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:29:55,507] jepsen worker 71 - jepsen.util 2861 :invoke :read [23 nil] INFO [2019-12-17 12:29:55,507] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16307,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:29:55,537] jepsen worker 70 - jepsen.util 3220 :info :write [23 12] :RequestCanceledException INFO [2019-12-17 12:29:55,538] jepsen worker 70 - jepsen.util 3310 :invoke :read [23 nil] INFO [2019-12-17 12:29:55,606] jepsen worker 51 - jepsen.util 2571 :info :write [17 45] :RequestCanceledException INFO [2019-12-17 12:29:55,607] jepsen worker 51 - jepsen.util 2661 :invoke :read [17 nil] INFO [2019-12-17 12:29:55,700] jepsen worker 71 - jepsen.util 2861 :ok :read [23 0] INFO [2019-12-17 12:29:55,700] jepsen worker 71 - jepsen.util 2861 :invoke :read [23 nil] INFO [2019-12-17 12:29:55,707] jepsen worker 51 - jepsen.util 2661 :ok :read [17 42] INFO [2019-12-17 12:29:55,707] jepsen worker 51 - jepsen.util 2661 :invoke :write [17 34] INFO [2019-12-17 12:29:55,744] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16320,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:29:55,744] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16320,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:29:55,744] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16322,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:29:55,744] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16322,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:29:55,835] jepsen worker 5 - jepsen.util 2345 :info :write [1 2] :RequestCanceledException INFO [2019-12-17 12:29:55,835] jepsen worker 25 - jepsen.util 4255 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:29:55,835] jepsen worker 25 - jepsen.util 4255 :invoke :write [8 24] INFO [2019-12-17 12:29:55,836] jepsen worker 5 - jepsen.util 2435 :invoke :write [1 11] INFO [2019-12-17 12:29:55,855] jepsen worker 37 - jepsen.util 2467 :info :write [12 43] :RequestCanceledException INFO [2019-12-17 12:29:55,856] jepsen worker 37 - jepsen.util 2557 :invoke :write [12 11] INFO [2019-12-17 12:29:55,887] jepsen worker 87 - jepsen.util 2517 :info :write [29 17] :RequestCanceledException INFO [2019-12-17 12:29:55,888] jepsen worker 87 - jepsen.util 2607 :invoke :read [29 nil] INFO [2019-12-17 12:29:55,892] jepsen worker 89 - jepsen.util 2609 :fail :cas [29 [9 29]] :ValueNotSwapFrom INFO [2019-12-17 12:29:55,892] jepsen worker 89 - jepsen.util 2609 :invoke :write [29 45] INFO [2019-12-17 12:29:55,908] jepsen worker 18 - jepsen.util 2898 :info :write [6 49] :RequestCanceledException INFO [2019-12-17 12:29:55,908] jepsen worker 68 - jepsen.util 3578 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:29:55,908] jepsen worker 68 - jepsen.util 3578 :invoke :read [22 nil] INFO [2019-12-17 12:29:55,909] jepsen worker 18 - jepsen.util 2988 :invoke :read [6 nil] INFO [2019-12-17 12:29:55,919] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16328,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:29:55,919] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16328,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:29:55,919] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16331,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:29:55,919] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16331,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:29:55,919] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16332,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:29:55,919] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16332,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:29:55,924] jepsen worker 77 - jepsen.util 3407 :info :cas [25 [0 27]] :RequestCanceledException INFO [2019-12-17 12:29:55,924] jepsen worker 81 - jepsen.util 3591 :info :cas [27 [16 1]] :RequestCanceledException INFO [2019-12-17 12:29:55,924] jepsen worker 76 - jepsen.util 3766 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:29:55,924] jepsen worker 58 - jepsen.util 3658 :info :cas [19 [28 21]] :RequestCanceledException INFO [2019-12-17 12:29:55,924] jepsen worker 64 - jepsen.util 2224 :info :write [21 27] :RequestCanceledException INFO [2019-12-17 12:29:55,924] jepsen worker 76 - jepsen.util 3766 :invoke :write [25 19] INFO [2019-12-17 12:29:55,925] jepsen worker 81 - jepsen.util 3681 :invoke :write [27 12] INFO [2019-12-17 12:29:55,925] jepsen worker 77 - jepsen.util 3497 :invoke :write [25 4] INFO [2019-12-17 12:29:55,925] jepsen worker 58 - jepsen.util 3748 :invoke :read [19 nil] INFO [2019-12-17 12:29:55,925] jepsen worker 64 - jepsen.util 2314 :invoke :write [21 49] INFO [2019-12-17 12:29:55,930] jepsen worker 18 - jepsen.util 2988 :ok :read [6 36] INFO [2019-12-17 12:29:55,931] jepsen worker 18 - jepsen.util 2988 :invoke :cas [6 [20 8]] INFO [2019-12-17 12:29:55,937] jepsen worker 2 - jepsen.util 3512 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:29:55,937] jepsen worker 2 - jepsen.util 3512 :invoke :read [0 nil] INFO [2019-12-17 12:29:55,955] jepsen worker 82 - jepsen.util 4402 :info :cas [27 [45 37]] :RequestCanceledException INFO [2019-12-17 12:29:55,956] jepsen worker 82 - jepsen.util 4492 :invoke :cas [27 [9 43]] INFO [2019-12-17 12:29:55,958] jepsen worker 18 - jepsen.util 2988 :fail :cas [6 [20 8]] :ValueNotSwapFrom INFO [2019-12-17 12:29:55,958] jepsen worker 18 - jepsen.util 2988 :invoke :cas [6 [19 1]] INFO [2019-12-17 12:29:55,970] jepsen worker 18 - jepsen.util 2988 :fail :cas [6 [19 1]] :ValueNotSwapFrom INFO [2019-12-17 12:29:55,970] jepsen worker 18 - jepsen.util 2988 :invoke :read [6 nil] INFO [2019-12-17 12:29:55,985] jepsen worker 18 - jepsen.util 2988 :ok :read [6 36] INFO [2019-12-17 12:29:55,985] jepsen worker 18 - jepsen.util 2988 :invoke :read [6 nil] INFO [2019-12-17 12:29:55,987] jepsen worker 73 - jepsen.util 2683 :info :cas [24 [19 15]] :RequestCanceledException INFO [2019-12-17 12:29:55,987] jepsen worker 66 - jepsen.util 3756 :info :cas [22 [34 31]] :RequestCanceledException INFO [2019-12-17 12:29:55,988] jepsen worker 73 - jepsen.util 2773 :invoke :write [24 31] INFO [2019-12-17 12:29:55,988] jepsen worker 66 - jepsen.util 3846 :invoke :cas [22 [31 49]] INFO [2019-12-17 12:29:56,000] jepsen worker 18 - jepsen.util 2988 :ok :read [6 36] INFO [2019-12-17 12:29:56,001] jepsen worker 18 - jepsen.util 2988 :invoke :cas [6 [37 29]] INFO [2019-12-17 12:29:56,008] jepsen worker 39 - jepsen.util 3639 :info :write [13 32] :RequestCanceledException INFO [2019-12-17 12:29:56,009] jepsen worker 39 - jepsen.util 3729 :invoke :read [13 nil] INFO [2019-12-17 12:29:56,023] jepsen worker 61 - jepsen.util 4111 :info :cas [20 [21 26]] :RequestCanceledException INFO [2019-12-17 12:29:56,023] jepsen worker 53 - jepsen.util 2753 :info :write [17 3] :RequestCanceledException INFO [2019-12-17 12:29:56,024] jepsen worker 61 - jepsen.util 4201 :invoke :cas [20 [43 19]] INFO [2019-12-17 12:29:56,024] jepsen worker 53 - jepsen.util 2843 :invoke :cas [17 [18 38]] INFO [2019-12-17 12:29:56,035] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16350,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:29:56,035] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16350,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:29:56,035] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16354,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:29:56,035] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16354,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:29:56,037] jepsen worker 86 - jepsen.util 3776 :info :cas [28 [33 24]] :RequestCanceledException INFO [2019-12-17 12:29:56,038] jepsen worker 86 - jepsen.util 3866 :invoke :write [28 18] INFO [2019-12-17 12:29:56,054] jepsen worker 24 - jepsen.util 3624 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:29:56,054] jepsen worker 24 - jepsen.util 3624 :invoke :read [8 nil] INFO [2019-12-17 12:29:56,065] jepsen worker 70 - jepsen.util 3310 :ok :read [23 0] INFO [2019-12-17 12:29:56,066] jepsen worker 70 - jepsen.util 3310 :invoke :read [23 nil] INFO [2019-12-17 12:29:56,072] jepsen worker 3 - jepsen.util 2703 :fail :cas [1 [41 18]] :ValueNotSwapFrom INFO [2019-12-17 12:29:56,072] jepsen worker 3 - jepsen.util 2703 :invoke :write [1 46] INFO [2019-12-17 12:29:56,078] jepsen worker 71 - jepsen.util 2861 :ok :read [23 0] INFO [2019-12-17 12:29:56,078] jepsen worker 71 - jepsen.util 2861 :invoke :cas [23 [49 16]] INFO [2019-12-17 12:29:56,087] jepsen worker 67 - jepsen.util 4207 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:29:56,087] jepsen worker 67 - jepsen.util 4207 :invoke :write [22 14] INFO [2019-12-17 12:29:56,124] jepsen worker 55 - jepsen.util 3295 :info :write [18 34] :RequestCanceledException INFO [2019-12-17 12:29:56,137] jepsen worker 46 - jepsen.util 2926 :info :write [15 3] :RequestCanceledException INFO [2019-12-17 12:29:56,149] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:29:56,162] jepsen worker 87 - jepsen.util 2607 :ok :read [29 28] INFO [2019-12-17 12:29:56,186] jepsen worker 27 - jepsen.util 2457 :info :write [9 14] :RequestCanceledException INFO [2019-12-17 12:29:56,208] jepsen worker 44 - jepsen.util 3734 :info :cas [14 [35 17]] :RequestCanceledException INFO [2019-12-17 12:29:56,223] jepsen worker 34 - jepsen.util 3994 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:29:56,223] jepsen worker 6 - jepsen.util 3876 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:29:56,235] jepsen worker 49 - jepsen.util 3019 :info :write [16 44] :RequestCanceledException INFO [2019-12-17 12:29:56,242] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16367,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:29:56,250] jepsen worker 19 - jepsen.util 2989 :ok :write [6 6] INFO [2019-12-17 12:29:56,250] jepsen worker 72 - jepsen.util 2862 :ok :write [24 4] INFO [2019-12-17 12:29:56,250] jepsen worker 10 - jepsen.util 2620 :ok :write [3 42] INFO [2019-12-17 12:29:56,251] jepsen worker 32 - jepsen.util 3272 :ok :write [10 12] INFO [2019-12-17 12:29:56,252] jepsen worker 60 - jepsen.util 4020 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:29:56,260] jepsen nemesis - couchbase.util Rebalance check count: 119 INFO [2019-12-17 12:29:56,260] jepsen worker 87 - jepsen.util 2607 :invoke :cas [29 [7 33]] INFO [2019-12-17 12:29:56,261] jepsen worker 34 - jepsen.util 3994 :invoke :read [11 nil] INFO [2019-12-17 12:29:56,261] jepsen worker 6 - jepsen.util 3876 :invoke :write [2 29] INFO [2019-12-17 12:29:56,261] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16367,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:29:56,261] jepsen worker 19 - jepsen.util 2989 :invoke :read [6 nil] INFO [2019-12-17 12:29:56,261] jepsen worker 45 - jepsen.util 2925 :ok :write [15 27] INFO [2019-12-17 12:29:56,261] jepsen worker 72 - jepsen.util 2862 :invoke :read [24 nil] INFO [2019-12-17 12:29:56,261] jepsen worker 10 - jepsen.util 2620 :invoke :write [3 10] INFO [2019-12-17 12:29:56,261] jepsen worker 32 - jepsen.util 3272 :invoke :cas [10 [24 33]] INFO [2019-12-17 12:29:56,261] jepsen worker 60 - jepsen.util 4020 :invoke :write [20 14] INFO [2019-12-17 12:29:56,261] jepsen nemesis - couchbase.util Rebalance stuck count: 100 INFO [2019-12-17 12:29:56,261] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16369,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:29:56,261] jepsen worker 45 - jepsen.util 2925 :invoke :read [15 nil] INFO [2019-12-17 12:29:56,261] jepsen worker 46 - jepsen.util 3016 :invoke :read [15 nil] INFO [2019-12-17 12:29:56,261] jepsen worker 55 - jepsen.util 3385 :invoke :read [18 nil] INFO [2019-12-17 12:29:56,261] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16369,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:29:56,261] jepsen worker 27 - jepsen.util 2547 :invoke :cas [9 [3 32]] INFO [2019-12-17 12:29:56,261] jepsen worker 44 - jepsen.util 3824 :invoke :write [14 32] INFO [2019-12-17 12:29:56,262] jepsen worker 49 - jepsen.util 3109 :invoke :cas [16 [18 5]] INFO [2019-12-17 12:29:56,265] jepsen worker 74 - jepsen.util 2864 :ok :read [24 4] INFO [2019-12-17 12:29:56,265] jepsen worker 74 - jepsen.util 2864 :invoke :write [24 30] INFO [2019-12-17 12:29:56,285] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16374,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:29:56,285] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16374,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:29:56,286] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16378,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:29:56,286] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16378,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:29:56,290] jepsen worker 54 - jepsen.util 2844 :info :write [18 47] :RequestCanceledException INFO [2019-12-17 12:29:56,293] jepsen worker 54 - jepsen.util 2934 :invoke :write [18 47] INFO [2019-12-17 12:29:56,293] jepsen worker 9 - jepsen.util 3249 :fail :cas [3 [21 35]] :ValueNotSwapFrom INFO [2019-12-17 12:29:56,293] jepsen worker 9 - jepsen.util 3249 :invoke :read [3 nil] INFO [2019-12-17 12:29:56,307] jepsen worker 72 - jepsen.util 2862 :ok :read [24 4] INFO [2019-12-17 12:29:56,307] jepsen worker 72 - jepsen.util 2862 :invoke :write [24 11] INFO [2019-12-17 12:29:56,316] jepsen worker 87 - jepsen.util 2607 :fail :cas [29 [7 33]] :ValueNotSwapFrom INFO [2019-12-17 12:29:56,316] jepsen worker 87 - jepsen.util 2607 :invoke :cas [29 [28 34]] INFO [2019-12-17 12:29:56,323] jepsen worker 7 - jepsen.util 3607 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:29:56,323] jepsen worker 7 - jepsen.util 3607 :invoke :read [2 nil] INFO [2019-12-17 12:29:56,329] jepsen worker 45 - jepsen.util 2925 :ok :read [15 27] INFO [2019-12-17 12:29:56,329] jepsen worker 45 - jepsen.util 2925 :invoke :read [15 nil] INFO [2019-12-17 12:29:56,337] jepsen worker 46 - jepsen.util 3016 :ok :read [15 27] INFO [2019-12-17 12:29:56,337] jepsen worker 13 - jepsen.util 2533 :info :write [4 2] :RequestCanceledException INFO [2019-12-17 12:29:56,337] jepsen worker 46 - jepsen.util 3016 :invoke :write [15 33] INFO [2019-12-17 12:29:56,340] jepsen worker 13 - jepsen.util 2623 :invoke :write [4 26] INFO [2019-12-17 12:29:56,354] jepsen worker 56 - jepsen.util 2576 :info :write [18 39] :RequestCanceledException INFO [2019-12-17 12:29:56,354] jepsen worker 4 - jepsen.util 1984 :info :write [1 10] :RequestCanceledException INFO [2019-12-17 12:29:56,355] jepsen worker 56 - jepsen.util 2666 :invoke :read [18 nil] INFO [2019-12-17 12:29:56,355] jepsen worker 4 - jepsen.util 2074 :invoke :cas [1 [0 35]] INFO [2019-12-17 12:29:56,359] jepsen worker 53 - jepsen.util 2843 :fail :cas [17 [18 38]] :ValueNotSwapFrom INFO [2019-12-17 12:29:56,359] jepsen worker 53 - jepsen.util 2843 :invoke :cas [17 [9 31]] INFO [2019-12-17 12:29:56,363] jepsen worker 27 - jepsen.util 2547 :fail :cas [9 [3 32]] :ValueNotSwapFrom INFO [2019-12-17 12:29:56,363] jepsen worker 27 - jepsen.util 2547 :invoke :cas [9 [49 10]] INFO [2019-12-17 12:29:56,367] jepsen worker 9 - jepsen.util 3249 :ok :read [3 42] INFO [2019-12-17 12:29:56,367] jepsen worker 9 - jepsen.util 3249 :invoke :read [3 nil] INFO [2019-12-17 12:29:56,372] jepsen worker 71 - jepsen.util 2861 :fail :cas [23 [49 16]] :ValueNotSwapFrom INFO [2019-12-17 12:29:56,372] jepsen worker 71 - jepsen.util 2861 :invoke :read [23 nil] INFO [2019-12-17 12:29:56,382] jepsen worker 64 - jepsen.util 2314 :ok :write [21 49] INFO [2019-12-17 12:29:56,382] jepsen worker 64 - jepsen.util 2314 :invoke :read [21 nil] INFO [2019-12-17 12:29:56,390] jepsen worker 23 - jepsen.util 3623 :info :write [7 0] :RequestCanceledException INFO [2019-12-17 12:29:56,390] jepsen worker 12 - jepsen.util 2892 :info :cas [4 [49 24]] :RequestCanceledException INFO [2019-12-17 12:29:56,391] jepsen worker 23 - jepsen.util 3713 :invoke :cas [7 [45 48]] INFO [2019-12-17 12:29:56,391] jepsen worker 12 - jepsen.util 2982 :invoke :write [4 29] INFO [2019-12-17 12:29:56,408] jepsen worker 63 - jepsen.util 2403 :info :write [21 28] :RequestCanceledException INFO [2019-12-17 12:29:56,409] jepsen worker 63 - jepsen.util 2493 :invoke :write [21 14] INFO [2019-12-17 12:29:56,418] jepsen worker 53 - jepsen.util 2843 :fail :cas [17 [9 31]] :ValueNotSwapFrom INFO [2019-12-17 12:29:56,418] jepsen worker 53 - jepsen.util 2843 :invoke :cas [17 [30 37]] INFO [2019-12-17 12:29:56,422] jepsen worker 88 - jepsen.util 2878 :info :write [29 44] :RequestCanceledException INFO [2019-12-17 12:29:56,423] jepsen worker 88 - jepsen.util 2968 :invoke :cas [29 [46 10]] INFO [2019-12-17 12:29:56,427] jepsen worker 32 - jepsen.util 3272 :fail :cas [10 [24 33]] :ValueNotSwapFrom INFO [2019-12-17 12:29:56,427] jepsen worker 32 - jepsen.util 3272 :invoke :write [10 45] INFO [2019-12-17 12:29:56,447] jepsen worker 71 - jepsen.util 2861 :ok :read [23 0] INFO [2019-12-17 12:29:56,447] jepsen worker 71 - jepsen.util 2861 :invoke :write [23 23] INFO [2019-12-17 12:29:56,455] jepsen worker 69 - jepsen.util 2679 :info :write [23 49] :RequestCanceledException INFO [2019-12-17 12:29:56,456] jepsen worker 69 - jepsen.util 2769 :invoke :cas [23 [5 9]] INFO [2019-12-17 12:29:56,473] jepsen worker 45 - jepsen.util 2925 :ok :read [15 27] INFO [2019-12-17 12:29:56,473] jepsen worker 45 - jepsen.util 2925 :invoke :read [15 nil] INFO [2019-12-17 12:29:56,482] jepsen worker 9 - jepsen.util 3249 :ok :read [3 42] INFO [2019-12-17 12:29:56,482] jepsen worker 9 - jepsen.util 3249 :invoke :read [3 nil] INFO [2019-12-17 12:29:56,492] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16414,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:29:56,492] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16414,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:29:56,492] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16417,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:29:56,493] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16417,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:29:56,493] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16418,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:29:56,493] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16418,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:29:56,493] jepsen worker 45 - jepsen.util 2925 :ok :read [15 27] INFO [2019-12-17 12:29:56,493] jepsen worker 45 - jepsen.util 2925 :invoke :write [15 45] INFO [2019-12-17 12:29:56,497] jepsen worker 56 - jepsen.util 2666 :ok :read [18 16] INFO [2019-12-17 12:29:56,497] jepsen worker 56 - jepsen.util 2666 :invoke :cas [18 [11 37]] INFO [2019-12-17 12:29:56,507] jepsen worker 31 - jepsen.util 2641 :fail :read [10 nil] :RequestTimeoutException INFO [2019-12-17 12:29:56,507] jepsen worker 31 - jepsen.util 2641 :invoke :read [10 nil] INFO [2019-12-17 12:29:56,516] jepsen worker 9 - jepsen.util 3249 :ok :read [3 42] INFO [2019-12-17 12:29:56,517] jepsen worker 9 - jepsen.util 3249 :invoke :read [3 nil] INFO [2019-12-17 12:29:56,527] jepsen worker 70 - jepsen.util 3310 :ok :read [23 0] INFO [2019-12-17 12:29:56,527] jepsen worker 70 - jepsen.util 3310 :invoke :read [23 nil] INFO [2019-12-17 12:29:56,563] jepsen worker 4 - jepsen.util 2074 :fail :cas [1 [0 35]] :ValueNotSwapFrom INFO [2019-12-17 12:29:56,564] jepsen worker 4 - jepsen.util 2074 :invoke :write [1 40] INFO [2019-12-17 12:29:56,590] jepsen worker 47 - jepsen.util 3197 :info :cas [15 [23 1]] :RequestCanceledException INFO [2019-12-17 12:29:56,591] jepsen worker 47 - jepsen.util 3287 :invoke :read [15 nil] INFO [2019-12-17 12:29:56,614] jepsen worker 54 - jepsen.util 2934 :ok :write [18 47] INFO [2019-12-17 12:29:56,615] jepsen worker 54 - jepsen.util 2934 :invoke :write [18 36] INFO [2019-12-17 12:29:56,624] jepsen worker 80 - jepsen.util 2690 :info :write [26 14] :RequestCanceledException INFO [2019-12-17 12:29:56,624] jepsen worker 80 - jepsen.util 2780 :invoke :cas [26 [25 30]] INFO [2019-12-17 12:29:56,637] jepsen worker 30 - jepsen.util 3270 :info :write [10 24] :RequestCanceledException INFO [2019-12-17 12:29:56,638] jepsen worker 30 - jepsen.util 3360 :invoke :write [10 28] INFO [2019-12-17 12:29:56,654] jepsen worker 65 - jepsen.util 2495 :info :write [21 30] :RequestCanceledException INFO [2019-12-17 12:29:56,655] jepsen worker 65 - jepsen.util 2585 :invoke :read [21 nil] INFO [2019-12-17 12:29:56,686] jepsen worker 10 - jepsen.util 2620 :ok :write [3 10] INFO [2019-12-17 12:29:56,686] jepsen worker 46 - jepsen.util 3016 :ok :write [15 33] INFO [2019-12-17 12:29:56,686] jepsen worker 10 - jepsen.util 2620 :invoke :read [3 nil] INFO [2019-12-17 12:29:56,686] jepsen worker 46 - jepsen.util 3016 :invoke :read [15 nil] INFO [2019-12-17 12:29:56,695] jepsen worker 20 - jepsen.util 2450 :ok :write [6 27] INFO [2019-12-17 12:29:56,695] jepsen worker 20 - jepsen.util 2450 :invoke :read [6 nil] INFO [2019-12-17 12:29:56,703] jepsen worker 55 - jepsen.util 3385 :ok :read [18 47] INFO [2019-12-17 12:29:56,703] jepsen worker 55 - jepsen.util 3385 :invoke :read [18 nil] INFO [2019-12-17 12:29:56,707] jepsen worker 52 - jepsen.util 3022 :info :write [17 5] :RequestCanceledException INFO [2019-12-17 12:29:56,708] jepsen worker 52 - jepsen.util 3112 :invoke :read [17 nil] INFO [2019-12-17 12:29:56,723] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16435,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:29:56,723] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16435,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:29:56,723] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16438,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:29:56,724] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16438,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:29:56,724] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16439,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:29:56,724] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16439,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:29:56,754] jepsen worker 11 - jepsen.util 3071 :info :write [3 28] :RequestCanceledException INFO [2019-12-17 12:29:56,756] jepsen worker 11 - jepsen.util 3161 :invoke :cas [3 [36 38]] INFO [2019-12-17 12:29:56,762] jepsen worker 46 - jepsen.util 3016 :ok :read [15 33] INFO [2019-12-17 12:29:56,762] jepsen worker 27 - jepsen.util 2547 :fail :cas [9 [49 10]] :ValueNotSwapFrom INFO [2019-12-17 12:29:56,762] jepsen worker 46 - jepsen.util 3016 :invoke :read [15 nil] INFO [2019-12-17 12:29:56,762] jepsen worker 27 - jepsen.util 2547 :invoke :read [9 nil] INFO [2019-12-17 12:29:56,788] jepsen worker 47 - jepsen.util 3287 :ok :read [15 33] INFO [2019-12-17 12:29:56,788] jepsen worker 19 - jepsen.util 2989 :ok :read [6 27] INFO [2019-12-17 12:29:56,788] jepsen worker 47 - jepsen.util 3287 :invoke :read [15 nil] INFO [2019-12-17 12:29:56,788] jepsen worker 19 - jepsen.util 2989 :invoke :write [6 28] INFO [2019-12-17 12:29:56,823] jepsen worker 79 - jepsen.util 2779 :info :write [26 16] :RequestCanceledException INFO [2019-12-17 12:29:56,823] jepsen worker 16 - jepsen.util 3796 :info :write [5 27] :RequestCanceledException INFO [2019-12-17 12:29:56,824] jepsen worker 16 - jepsen.util 3886 :invoke :write [5 13] INFO [2019-12-17 12:29:56,824] jepsen worker 79 - jepsen.util 2869 :invoke :cas [26 [29 32]] INFO [2019-12-17 12:29:56,831] jepsen worker 46 - jepsen.util 3016 :ok :read [15 33] INFO [2019-12-17 12:29:56,831] jepsen worker 46 - jepsen.util 3016 :invoke :read [15 nil] INFO [2019-12-17 12:29:56,857] jepsen worker 20 - jepsen.util 2450 :ok :read [6 27] INFO [2019-12-17 12:29:56,858] jepsen worker 20 - jepsen.util 2450 :invoke :cas [6 [23 13]] INFO [2019-12-17 12:29:56,858] jepsen worker 53 - jepsen.util 2843 :fail :cas [17 [30 37]] :ValueNotSwapFrom INFO [2019-12-17 12:29:56,858] jepsen worker 53 - jepsen.util 2843 :invoke :read [17 nil] INFO [2019-12-17 12:29:56,863] jepsen worker 10 - jepsen.util 2620 :ok :read [3 10] INFO [2019-12-17 12:29:56,863] jepsen worker 10 - jepsen.util 2620 :invoke :write [3 17] INFO [2019-12-17 12:29:56,883] jepsen worker 35 - jepsen.util 3545 :info :cas [11 [42 18]] :RequestCanceledException INFO [2019-12-17 12:29:56,884] jepsen worker 35 - jepsen.util 3635 :invoke :write [11 43] INFO [2019-12-17 12:29:56,890] jepsen worker 69 - jepsen.util 2769 :fail :cas [23 [5 9]] :ValueNotSwapFrom INFO [2019-12-17 12:29:56,891] jepsen worker 69 - jepsen.util 2769 :invoke :cas [23 [25 22]] INFO [2019-12-17 12:29:56,903] jepsen worker 11 - jepsen.util 3161 :fail :cas [3 [36 38]] :ValueNotSwapFrom INFO [2019-12-17 12:29:56,904] jepsen worker 11 - jepsen.util 3161 :invoke :write [3 39] INFO [2019-12-17 12:29:56,906] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16457,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:29:56,907] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16457,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:29:56,907] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16460,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:29:56,907] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16460,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:29:56,907] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16461,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:29:56,907] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16461,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:29:56,908] jepsen worker 46 - jepsen.util 3016 :ok :read [15 33] INFO [2019-12-17 12:29:56,908] jepsen worker 46 - jepsen.util 3016 :invoke :write [15 19] INFO [2019-12-17 12:29:56,925] jepsen worker 27 - jepsen.util 2547 :ok :read [9 38] INFO [2019-12-17 12:29:56,925] jepsen worker 27 - jepsen.util 2547 :invoke :write [9 16] INFO [2019-12-17 12:29:56,956] jepsen worker 56 - jepsen.util 2666 :fail :cas [18 [11 37]] :ValueNotSwapFrom INFO [2019-12-17 12:29:56,956] jepsen worker 56 - jepsen.util 2666 :invoke :write [18 35] INFO [2019-12-17 12:29:56,966] jepsen worker 47 - jepsen.util 3287 :ok :read [15 33] INFO [2019-12-17 12:29:56,967] jepsen worker 47 - jepsen.util 3287 :invoke :read [15 nil] INFO [2019-12-17 12:29:56,984] jepsen worker 70 - jepsen.util 3310 :ok :read [23 0] INFO [2019-12-17 12:29:56,984] jepsen worker 70 - jepsen.util 3310 :invoke :cas [23 [47 28]] INFO [2019-12-17 12:29:56,993] jepsen worker 55 - jepsen.util 3385 :ok :read [18 47] INFO [2019-12-17 12:29:56,993] jepsen worker 55 - jepsen.util 3385 :invoke :write [18 41] INFO [2019-12-17 12:29:57,008] jepsen worker 29 - jepsen.util 2369 :info :write [9 4] :RequestCanceledException INFO [2019-12-17 12:29:57,009] jepsen worker 29 - jepsen.util 2459 :invoke :cas [9 [48 9]] INFO [2019-12-17 12:29:57,017] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16474,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:29:57,017] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16474,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:29:57,017] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16478,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:29:57,017] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16478,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:29:57,023] jepsen worker 59 - jepsen.util 3659 :info :write [19 43] :RequestCanceledException INFO [2019-12-17 12:29:57,025] jepsen worker 59 - jepsen.util 3749 :invoke :read [19 nil] INFO [2019-12-17 12:29:57,037] jepsen worker 43 - jepsen.util 3463 :info :write [14 16] :RequestCanceledException INFO [2019-12-17 12:29:57,038] jepsen worker 43 - jepsen.util 3553 :invoke :read [14 nil] INFO [2019-12-17 12:29:57,054] jepsen worker 15 - jepsen.util 3435 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:29:57,055] jepsen worker 15 - jepsen.util 3435 :invoke :cas [5 [28 4]] INFO [2019-12-17 12:29:57,057] jepsen worker 88 - jepsen.util 2968 :fail :cas [29 [46 10]] :ValueNotSwapFrom INFO [2019-12-17 12:29:57,057] jepsen worker 88 - jepsen.util 2968 :invoke :cas [29 [23 37]] INFO [2019-12-17 12:29:57,084] jepsen worker 70 - jepsen.util 3310 :fail :cas [23 [47 28]] :ValueNotSwapFrom INFO [2019-12-17 12:29:57,084] jepsen worker 70 - jepsen.util 3310 :invoke :write [23 5] INFO [2019-12-17 12:29:57,116] jepsen worker 52 - jepsen.util 3112 :ok :read [17 42] INFO [2019-12-17 12:29:57,116] jepsen worker 52 - jepsen.util 3112 :invoke :write [17 13] INFO [2019-12-17 12:29:57,123] jepsen worker 85 - jepsen.util 4225 :info :cas [28 [29 23]] :RequestCanceledException INFO [2019-12-17 12:29:57,123] jepsen worker 50 - jepsen.util 4100 :info :cas [16 [1 14]] :RequestCanceledException INFO [2019-12-17 12:29:57,124] jepsen worker 50 - jepsen.util 4190 :invoke :cas [16 [37 7]] INFO [2019-12-17 12:29:57,124] jepsen worker 85 - jepsen.util 4315 :invoke :write [28 26] INFO [2019-12-17 12:29:57,136] jepsen worker 62 - jepsen.util 3572 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:29:57,136] jepsen worker 84 - jepsen.util 3504 :info :write [28 7] :RequestCanceledException INFO [2019-12-17 12:29:57,136] jepsen worker 62 - jepsen.util 3572 :invoke :cas [20 [34 26]] INFO [2019-12-17 12:29:57,137] jepsen worker 84 - jepsen.util 3594 :invoke :write [28 11] INFO [2019-12-17 12:29:57,221] jepsen worker 88 - jepsen.util 2968 :fail :cas [29 [23 37]] :ValueNotSwapFrom INFO [2019-12-17 12:29:57,221] jepsen worker 88 - jepsen.util 2968 :invoke :cas [29 [43 6]] INFO [2019-12-17 12:29:57,223] jepsen worker 75 - jepsen.util 3855 :info :cas [25 [28 46]] :RequestCanceledException INFO [2019-12-17 12:29:57,224] jepsen worker 28 - jepsen.util 2098 :fail :read [9 nil] :RequestTimeoutException INFO [2019-12-17 12:29:57,224] jepsen worker 28 - jepsen.util 2098 :invoke :read [9 nil] INFO [2019-12-17 12:29:57,224] jepsen worker 75 - jepsen.util 3945 :invoke :read [25 nil] INFO [2019-12-17 12:29:57,235] jepsen worker 32 - jepsen.util 3272 :ok :write [10 45] INFO [2019-12-17 12:29:57,236] jepsen worker 32 - jepsen.util 3272 :invoke :write [10 10] INFO [2019-12-17 12:29:57,238] jepsen worker 72 - jepsen.util 2862 :ok :write [24 11] INFO [2019-12-17 12:29:57,238] jepsen worker 72 - jepsen.util 2862 :invoke :read [24 nil] INFO [2019-12-17 12:29:57,246] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16492,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:29:57,247] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16492,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:29:57,247] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16493,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:29:57,247] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16493,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:29:57,253] jepsen worker 31 - jepsen.util 2641 :ok :read [10 45] INFO [2019-12-17 12:29:57,253] jepsen worker 31 - jepsen.util 2641 :invoke :read [10 nil] INFO [2019-12-17 12:29:57,261] jepsen worker 19 - jepsen.util 2989 :ok :write [6 28] INFO [2019-12-17 12:29:57,261] jepsen worker 19 - jepsen.util 2989 :invoke :read [6 nil] INFO [2019-12-17 12:29:57,269] jepsen worker 10 - jepsen.util 2620 :ok :write [3 17] INFO [2019-12-17 12:29:57,270] jepsen worker 10 - jepsen.util 2620 :invoke :write [3 8] INFO [2019-12-17 12:29:57,271] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16497,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:29:57,271] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16497,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:29:57,271] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16501,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:29:57,271] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16501,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:29:57,279] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:29:57,279] jepsen nemesis - couchbase.util Rebalance check count: 120 INFO [2019-12-17 12:29:57,279] jepsen nemesis - couchbase.util Rebalance stuck count: 101 INFO [2019-12-17 12:29:57,284] jepsen worker 46 - jepsen.util 3016 :ok :write [15 19] INFO [2019-12-17 12:29:57,284] jepsen worker 46 - jepsen.util 3016 :invoke :cas [15 [24 7]] INFO [2019-12-17 12:29:57,289] jepsen worker 1 - jepsen.util 3151 :info :write [0 12] :RequestCanceledException INFO [2019-12-17 12:29:57,289] jepsen worker 63 - jepsen.util 2493 :ok :write [21 14] INFO [2019-12-17 12:29:57,289] jepsen worker 63 - jepsen.util 2493 :invoke :cas [21 [16 35]] INFO [2019-12-17 12:29:57,292] jepsen worker 1 - jepsen.util 3241 :invoke :write [0 26] INFO [2019-12-17 12:29:57,301] jepsen worker 14 - jepsen.util 2624 :info :write [4 4] :RequestCanceledException INFO [2019-12-17 12:29:57,303] jepsen worker 14 - jepsen.util 2714 :invoke :write [4 19] INFO [2019-12-17 12:29:57,316] jepsen worker 47 - jepsen.util 3287 :ok :read [15 19] INFO [2019-12-17 12:29:57,316] jepsen worker 47 - jepsen.util 3287 :invoke :cas [15 [41 43]] INFO [2019-12-17 12:29:57,323] jepsen worker 40 - jepsen.util 4000 :info :cas [13 [29 7]] :RequestCanceledException INFO [2019-12-17 12:29:57,323] jepsen worker 41 - jepsen.util 3911 :info :write [13 3] :RequestCanceledException INFO [2019-12-17 12:29:57,326] jepsen worker 41 - jepsen.util 4001 :invoke :read [13 nil] INFO [2019-12-17 12:29:57,326] jepsen worker 40 - jepsen.util 4090 :invoke :read [13 nil] INFO [2019-12-17 12:29:57,354] jepsen worker 78 - jepsen.util 2958 :info :write [26 36] :RequestCanceledException INFO [2019-12-17 12:29:57,357] jepsen worker 78 - jepsen.util 3048 :invoke :write [26 44] INFO [2019-12-17 12:29:57,360] jepsen worker 29 - jepsen.util 2459 :fail :cas [9 [48 9]] :ValueNotSwapFrom INFO [2019-12-17 12:29:57,360] jepsen worker 29 - jepsen.util 2459 :invoke :write [9 46] INFO [2019-12-17 12:29:57,382] jepsen worker 46 - jepsen.util 3016 :fail :cas [15 [24 7]] :ValueNotSwapFrom INFO [2019-12-17 12:29:57,382] jepsen worker 46 - jepsen.util 3016 :invoke :write [15 8] INFO [2019-12-17 12:29:57,401] jepsen worker 69 - jepsen.util 2769 :fail :cas [23 [25 22]] :ValueNotSwapFrom INFO [2019-12-17 12:29:57,401] jepsen worker 69 - jepsen.util 2769 :invoke :read [23 nil] INFO [2019-12-17 12:29:57,405] jepsen worker 57 - jepsen.util 3477 :info :write [19 48] :RequestCanceledException INFO [2019-12-17 12:29:57,405] jepsen worker 83 - jepsen.util 3953 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:29:57,406] jepsen worker 83 - jepsen.util 3953 :invoke :cas [27 [43 28]] INFO [2019-12-17 12:29:57,409] jepsen worker 57 - jepsen.util 3567 :invoke :read [19 nil] INFO [2019-12-17 12:29:57,412] jepsen worker 28 - jepsen.util 2098 :ok :read [9 38] INFO [2019-12-17 12:29:57,412] jepsen worker 28 - jepsen.util 2098 :invoke :cas [9 [13 41]] INFO [2019-12-17 12:29:57,421] jepsen worker 22 - jepsen.util 3802 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:29:57,422] jepsen worker 22 - jepsen.util 3802 :invoke :write [7 49] INFO [2019-12-17 12:29:57,425] jepsen worker 72 - jepsen.util 2862 :ok :read [24 11] INFO [2019-12-17 12:29:57,425] jepsen worker 72 - jepsen.util 2862 :invoke :read [24 nil] INFO [2019-12-17 12:29:57,436] jepsen worker 33 - jepsen.util 3993 :info :write [11 31] :RequestCanceledException INFO [2019-12-17 12:29:57,437] jepsen worker 88 - jepsen.util 2968 :fail :cas [29 [43 6]] :ValueNotSwapFrom INFO [2019-12-17 12:29:57,437] jepsen worker 88 - jepsen.util 2968 :invoke :cas [29 [18 12]] INFO [2019-12-17 12:29:57,439] jepsen worker 33 - jepsen.util 4083 :invoke :cas [11 [46 17]] INFO [2019-12-17 12:29:57,450] jepsen worker 38 - jepsen.util 2738 :info :write [12 46] :RequestCanceledException INFO [2019-12-17 12:29:57,450] jepsen worker 42 - jepsen.util 4272 :info :write [14 26] :RequestCanceledException INFO [2019-12-17 12:29:57,454] jepsen worker 38 - jepsen.util 2828 :invoke :cas [12 [15 46]] INFO [2019-12-17 12:29:57,455] jepsen worker 42 - jepsen.util 4362 :invoke :write [14 36] INFO [2019-12-17 12:29:57,462] jepsen worker 63 - jepsen.util 2493 :fail :cas [21 [16 35]] :ValueNotSwapFrom INFO [2019-12-17 12:29:57,463] jepsen worker 63 - jepsen.util 2493 :invoke :write [21 34] INFO [2019-12-17 12:29:57,473] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16528,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:29:57,474] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16528,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:29:57,474] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16530,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:29:57,474] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16530,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:29:57,490] jepsen worker 21 - jepsen.util 3621 :info :write [7 20] :RequestCanceledException INFO [2019-12-17 12:29:57,490] jepsen worker 17 - jepsen.util 3797 :info :cas [5 [31 38]] :RequestCanceledException INFO [2019-12-17 12:29:57,492] jepsen worker 17 - jepsen.util 3887 :invoke :read [5 nil] INFO [2019-12-17 12:29:57,493] jepsen worker 21 - jepsen.util 3711 :invoke :cas [7 [26 29]] INFO [2019-12-17 12:29:57,506] jepsen worker 36 - jepsen.util 2646 :info :write [12 3] :RequestCanceledException INFO [2019-12-17 12:29:57,506] jepsen worker 8 - jepsen.util 3158 :info :cas [2 [2 31]] :RequestCanceledException INFO [2019-12-17 12:29:57,508] jepsen worker 36 - jepsen.util 2736 :invoke :write [12 35] INFO [2019-12-17 12:29:57,508] jepsen worker 8 - jepsen.util 3248 :invoke :write [2 6] INFO [2019-12-17 12:29:57,523] jepsen worker 26 - jepsen.util 3986 :info :cas [8 [20 49]] :RequestCanceledException INFO [2019-12-17 12:29:57,525] jepsen worker 26 - jepsen.util 4076 :invoke :read [8 nil] INFO [2019-12-17 12:29:57,527] jepsen worker 30 - jepsen.util 3360 :ok :write [10 28] INFO [2019-12-17 12:29:57,527] jepsen worker 30 - jepsen.util 3360 :invoke :write [10 33] INFO [2019-12-17 12:29:57,535] jepsen worker 48 - jepsen.util 3108 :info :write [16 13] :RequestCanceledException INFO [2019-12-17 12:29:57,536] jepsen worker 0 - jepsen.util 4050 :info :cas [0 [47 22]] :RequestCanceledException INFO [2019-12-17 12:29:57,539] jepsen worker 0 - jepsen.util 4140 :invoke :write [0 22] INFO [2019-12-17 12:29:57,539] jepsen worker 48 - jepsen.util 3198 :invoke :cas [16 [22 15]] INFO [2019-12-17 12:29:57,568] jepsen worker 29 - jepsen.util 2459 :ok :write [9 46] INFO [2019-12-17 12:29:57,568] jepsen worker 29 - jepsen.util 2459 :invoke :read [9 nil] INFO [2019-12-17 12:29:57,607] jepsen worker 72 - jepsen.util 2862 :ok :read [24 11] INFO [2019-12-17 12:29:57,607] jepsen worker 72 - jepsen.util 2862 :invoke :read [24 nil] INFO [2019-12-17 12:29:57,625] jepsen worker 11 - jepsen.util 3161 :ok :write [3 39] INFO [2019-12-17 12:29:57,625] jepsen worker 11 - jepsen.util 3161 :invoke :read [3 nil] INFO [2019-12-17 12:29:57,649] jepsen worker 53 - jepsen.util 2843 :ok :read [17 42] INFO [2019-12-17 12:29:57,649] jepsen worker 53 - jepsen.util 2843 :invoke :write [17 45] INFO [2019-12-17 12:29:57,667] jepsen worker 19 - jepsen.util 2989 :ok :read [6 28] INFO [2019-12-17 12:29:57,667] jepsen worker 19 - jepsen.util 2989 :invoke :cas [6 [44 24]] INFO [2019-12-17 12:29:57,673] jepsen worker 11 - jepsen.util 3161 :ok :read [3 39] INFO [2019-12-17 12:29:57,673] jepsen worker 11 - jepsen.util 3161 :invoke :cas [3 [12 37]] INFO [2019-12-17 12:29:57,690] jepsen worker 63 - jepsen.util 2493 :ok :write [21 34] INFO [2019-12-17 12:29:57,691] jepsen worker 63 - jepsen.util 2493 :invoke :write [21 33] INFO [2019-12-17 12:29:57,697] jepsen worker 19 - jepsen.util 2989 :fail :cas [6 [44 24]] :ValueNotSwapFrom INFO [2019-12-17 12:29:57,697] jepsen worker 19 - jepsen.util 2989 :invoke :cas [6 [43 29]] INFO [2019-12-17 12:29:57,702] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16553,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:29:57,703] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16553,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:29:57,714] jepsen worker 11 - jepsen.util 3161 :fail :cas [3 [12 37]] :ValueNotSwapFrom INFO [2019-12-17 12:29:57,714] jepsen worker 11 - jepsen.util 3161 :invoke :write [3 29] INFO [2019-12-17 12:29:57,724] jepsen worker 51 - jepsen.util 2661 :info :write [17 34] :RequestCanceledException INFO [2019-12-17 12:29:57,726] jepsen worker 69 - jepsen.util 2769 :ok :read [23 0] INFO [2019-12-17 12:29:57,726] jepsen worker 69 - jepsen.util 2769 :invoke :cas [23 [31 22]] INFO [2019-12-17 12:29:57,727] jepsen worker 51 - jepsen.util 2751 :invoke :read [17 nil] INFO [2019-12-17 12:29:57,733] jepsen worker 72 - jepsen.util 2862 :ok :read [24 11] INFO [2019-12-17 12:29:57,733] jepsen worker 72 - jepsen.util 2862 :invoke :read [24 nil] INFO [2019-12-17 12:29:57,763] jepsen worker 19 - jepsen.util 2989 :fail :cas [6 [43 29]] :ValueNotSwapFrom INFO [2019-12-17 12:29:57,764] jepsen worker 19 - jepsen.util 2989 :invoke :write [6 13] INFO [2019-12-17 12:29:57,793] jepsen worker 32 - jepsen.util 3272 :ok :write [10 10] INFO [2019-12-17 12:29:57,793] jepsen worker 32 - jepsen.util 3272 :invoke :read [10 nil] INFO [2019-12-17 12:29:57,839] jepsen worker 63 - jepsen.util 2493 :ok :write [21 33] INFO [2019-12-17 12:29:57,839] jepsen worker 63 - jepsen.util 2493 :invoke :write [21 17] INFO [2019-12-17 12:29:57,886] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16568,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:29:57,886] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16568,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:29:57,886] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16572,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:29:57,887] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16572,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:29:57,890] jepsen worker 5 - jepsen.util 2435 :info :write [1 11] :RequestCanceledException INFO [2019-12-17 12:29:57,892] jepsen worker 5 - jepsen.util 2525 :invoke :read [1 nil] INFO [2019-12-17 12:29:57,899] jepsen worker 38 - jepsen.util 2828 :fail :cas [12 [15 46]] :ValueNotSwapFrom INFO [2019-12-17 12:29:57,899] jepsen worker 38 - jepsen.util 2828 :invoke :read [12 nil] INFO [2019-12-17 12:29:57,908] jepsen worker 37 - jepsen.util 2557 :info :write [12 11] :RequestCanceledException INFO [2019-12-17 12:29:57,909] jepsen worker 37 - jepsen.util 2647 :invoke :read [12 nil] INFO [2019-12-17 12:29:57,926] jepsen worker 63 - jepsen.util 2493 :ok :write [21 17] INFO [2019-12-17 12:29:57,926] jepsen worker 63 - jepsen.util 2493 :invoke :read [21 nil] INFO [2019-12-17 12:29:57,937] jepsen worker 25 - jepsen.util 4255 :info :write [8 24] :RequestCanceledException INFO [2019-12-17 12:29:57,939] jepsen worker 25 - jepsen.util 4345 :invoke :read [8 nil] INFO [2019-12-17 12:29:57,954] jepsen worker 63 - jepsen.util 2493 :ok :read [21 17] INFO [2019-12-17 12:29:57,954] jepsen worker 81 - jepsen.util 3681 :info :write [27 12] :RequestCanceledException INFO [2019-12-17 12:29:57,955] jepsen worker 63 - jepsen.util 2493 :invoke :write [21 39] INFO [2019-12-17 12:29:57,956] jepsen worker 81 - jepsen.util 3771 :invoke :cas [27 [42 36]] INFO [2019-12-17 12:29:57,990] jepsen worker 89 - jepsen.util 2609 :info :write [29 45] :RequestCanceledException INFO [2019-12-17 12:29:57,990] jepsen worker 58 - jepsen.util 3748 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:29:57,990] jepsen worker 58 - jepsen.util 3748 :invoke :write [19 49] INFO [2019-12-17 12:29:57,991] jepsen worker 89 - jepsen.util 2699 :invoke :read [29 nil] INFO [2019-12-17 12:29:58,001] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16584,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:29:58,001] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16584,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:29:58,002] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16586,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:29:58,002] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16586,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:29:58,005] jepsen worker 68 - jepsen.util 3578 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:29:58,005] jepsen worker 68 - jepsen.util 3578 :invoke :read [22 nil] INFO [2019-12-17 12:29:58,024] jepsen worker 76 - jepsen.util 3766 :info :write [25 19] :RequestCanceledException INFO [2019-12-17 12:29:58,024] jepsen worker 39 - jepsen.util 3729 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:29:58,024] jepsen worker 82 - jepsen.util 4492 :info :cas [27 [9 43]] :RequestCanceledException INFO [2019-12-17 12:29:58,024] jepsen worker 77 - jepsen.util 3497 :info :write [25 4] :RequestCanceledException INFO [2019-12-17 12:29:58,024] jepsen worker 30 - jepsen.util 3360 :ok :write [10 33] INFO [2019-12-17 12:29:58,024] jepsen worker 39 - jepsen.util 3729 :invoke :read [13 nil] INFO [2019-12-17 12:29:58,024] jepsen worker 30 - jepsen.util 3360 :invoke :write [10 46] INFO [2019-12-17 12:29:58,025] jepsen worker 82 - jepsen.util 4582 :invoke :cas [27 [27 47]] INFO [2019-12-17 12:29:58,025] jepsen worker 77 - jepsen.util 3587 :invoke :read [25 nil] INFO [2019-12-17 12:29:58,025] jepsen worker 76 - jepsen.util 3856 :invoke :cas [25 [13 15]] INFO [2019-12-17 12:29:58,026] jepsen worker 69 - jepsen.util 2769 :fail :cas [23 [31 22]] :ValueNotSwapFrom INFO [2019-12-17 12:29:58,026] jepsen worker 69 - jepsen.util 2769 :invoke :read [23 nil] INFO [2019-12-17 12:29:58,028] jepsen worker 46 - jepsen.util 3016 :ok :write [15 8] INFO [2019-12-17 12:29:58,028] jepsen worker 46 - jepsen.util 3016 :invoke :read [15 nil] INFO [2019-12-17 12:29:58,038] jepsen worker 73 - jepsen.util 2773 :info :write [24 31] :RequestCanceledException INFO [2019-12-17 12:29:58,038] jepsen worker 2 - jepsen.util 3512 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:29:58,038] jepsen worker 18 - jepsen.util 2988 :info :cas [6 [37 29]] :RequestCanceledException INFO [2019-12-17 12:29:58,038] jepsen worker 2 - jepsen.util 3512 :invoke :read [0 nil] INFO [2019-12-17 12:29:58,040] jepsen worker 73 - jepsen.util 2863 :invoke :write [24 38] INFO [2019-12-17 12:29:58,040] jepsen worker 18 - jepsen.util 3078 :invoke :read [6 nil] INFO [2019-12-17 12:29:58,040] jepsen worker 63 - jepsen.util 2493 :ok :write [21 39] INFO [2019-12-17 12:29:58,041] jepsen worker 63 - jepsen.util 2493 :invoke :read [21 nil] INFO [2019-12-17 12:29:58,054] jepsen worker 66 - jepsen.util 3846 :info :cas [22 [31 49]] :RequestCanceledException INFO [2019-12-17 12:29:58,055] jepsen worker 66 - jepsen.util 3936 :invoke :read [22 nil] INFO [2019-12-17 12:29:58,056] jepsen worker 32 - jepsen.util 3272 :ok :read [10 33] INFO [2019-12-17 12:29:58,056] jepsen worker 32 - jepsen.util 3272 :invoke :write [10 36] INFO [2019-12-17 12:29:58,071] jepsen worker 11 - jepsen.util 3161 :ok :write [3 29] INFO [2019-12-17 12:29:58,071] jepsen worker 11 - jepsen.util 3161 :invoke :cas [3 [20 34]] INFO [2019-12-17 12:29:58,089] jepsen worker 3 - jepsen.util 2703 :info :write [1 46] :RequestCanceledException INFO [2019-12-17 12:29:58,089] jepsen worker 61 - jepsen.util 4201 :info :cas [20 [43 19]] :RequestCanceledException INFO [2019-12-17 12:29:58,091] jepsen worker 3 - jepsen.util 2793 :invoke :read [1 nil] INFO [2019-12-17 12:29:58,091] jepsen worker 61 - jepsen.util 4291 :invoke :cas [20 [10 1]] INFO [2019-12-17 12:29:58,106] jepsen worker 46 - jepsen.util 3016 :ok :read [15 8] INFO [2019-12-17 12:29:58,106] jepsen worker 46 - jepsen.util 3016 :invoke :cas [15 [0 28]] INFO [2019-12-17 12:29:58,123] jepsen worker 63 - jepsen.util 2493 :ok :read [21 39] INFO [2019-12-17 12:29:58,123] jepsen worker 63 - jepsen.util 2493 :invoke :cas [21 [11 38]] INFO [2019-12-17 12:29:58,123] jepsen worker 86 - jepsen.util 3866 :info :write [28 18] :RequestCanceledException INFO [2019-12-17 12:29:58,125] jepsen worker 86 - jepsen.util 3956 :invoke :cas [28 [2 34]] INFO [2019-12-17 12:29:58,144] jepsen worker 18 - jepsen.util 3078 :ok :read [6 28] INFO [2019-12-17 12:29:58,144] jepsen worker 18 - jepsen.util 3078 :invoke :write [6 31] INFO [2019-12-17 12:29:58,155] jepsen worker 24 - jepsen.util 3624 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:29:58,155] jepsen worker 24 - jepsen.util 3624 :invoke :write [8 21] INFO [2019-12-17 12:29:58,167] jepsen worker 46 - jepsen.util 3016 :fail :cas [15 [0 28]] :ValueNotSwapFrom INFO [2019-12-17 12:29:58,167] jepsen worker 46 - jepsen.util 3016 :invoke :cas [15 [29 43]] INFO [2019-12-17 12:29:58,168] jepsen worker 63 - jepsen.util 2493 :fail :cas [21 [11 38]] :ValueNotSwapFrom INFO [2019-12-17 12:29:58,168] jepsen worker 63 - jepsen.util 2493 :invoke :write [21 37] INFO [2019-12-17 12:29:58,190] jepsen worker 67 - jepsen.util 4207 :info :write [22 14] :RequestCanceledException INFO [2019-12-17 12:29:58,191] jepsen worker 67 - jepsen.util 4297 :invoke :write [22 43] INFO [2019-12-17 12:29:58,200] jepsen worker 46 - jepsen.util 3016 :fail :cas [15 [29 43]] :ValueNotSwapFrom INFO [2019-12-17 12:29:58,201] jepsen worker 46 - jepsen.util 3016 :invoke :read [15 nil] INFO [2019-12-17 12:29:58,227] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16612,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:29:58,227] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16612,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:29:58,227] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16613,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:29:58,228] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16613,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:29:58,228] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16614,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:29:58,228] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16614,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:29:58,243] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16621,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:29:58,243] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16621,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:29:58,243] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16625,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:29:58,243] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16625,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:29:58,250] jepsen worker 51 - jepsen.util 2751 :ok :read [17 42] INFO [2019-12-17 12:29:58,250] jepsen worker 51 - jepsen.util 2751 :invoke :cas [17 [41 16]] INFO [2019-12-17 12:29:58,259] jepsen worker 46 - jepsen.util 3016 :ok :read [15 8] INFO [2019-12-17 12:29:58,259] jepsen worker 46 - jepsen.util 3016 :invoke :cas [15 [27 48]] INFO [2019-12-17 12:29:58,278] jepsen worker 63 - jepsen.util 2493 :ok :write [21 37] INFO [2019-12-17 12:29:58,278] jepsen worker 63 - jepsen.util 2493 :invoke :write [21 40] INFO [2019-12-17 12:29:58,287] jepsen worker 44 - jepsen.util 3824 :info :write [14 32] :RequestCanceledException INFO [2019-12-17 12:29:58,288] jepsen worker 44 - jepsen.util 3914 :invoke :cas [14 [32 46]] INFO [2019-12-17 12:29:58,294] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:29:58,295] jepsen nemesis - couchbase.util Rebalance check count: 121 INFO [2019-12-17 12:29:58,295] jepsen nemesis - couchbase.util Rebalance stuck count: 102 INFO [2019-12-17 12:29:58,297] jepsen worker 88 - jepsen.util 2968 :fail :cas [29 [18 12]] :ValueNotSwapFrom INFO [2019-12-17 12:29:58,298] jepsen worker 88 - jepsen.util 2968 :invoke :cas [29 [27 36]] INFO [2019-12-17 12:29:58,305] jepsen worker 49 - jepsen.util 3109 :info :cas [16 [18 5]] :RequestCanceledException INFO [2019-12-17 12:29:58,305] jepsen worker 74 - jepsen.util 2864 :info :write [24 30] :RequestCanceledException INFO [2019-12-17 12:29:58,306] jepsen worker 74 - jepsen.util 2954 :invoke :read [24 nil] INFO [2019-12-17 12:29:58,306] jepsen worker 49 - jepsen.util 3199 :invoke :write [16 13] INFO [2019-12-17 12:29:58,311] jepsen worker 46 - jepsen.util 3016 :fail :cas [15 [27 48]] :ValueNotSwapFrom INFO [2019-12-17 12:29:58,312] jepsen worker 46 - jepsen.util 3016 :invoke :write [15 33] INFO [2019-12-17 12:29:58,314] jepsen worker 80 - jepsen.util 2780 :fail :cas [26 [25 30]] :ValueNotSwapFrom INFO [2019-12-17 12:29:58,314] jepsen worker 80 - jepsen.util 2780 :invoke :cas [26 [14 23]] INFO [2019-12-17 12:29:58,324] jepsen worker 34 - jepsen.util 3994 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:29:58,324] jepsen worker 6 - jepsen.util 3876 :info :write [2 29] :RequestCanceledException INFO [2019-12-17 12:29:58,324] jepsen worker 34 - jepsen.util 3994 :invoke :read [11 nil] INFO [2019-12-17 12:29:58,325] jepsen worker 6 - jepsen.util 3966 :invoke :cas [2 [25 43]] INFO [2019-12-17 12:29:58,332] jepsen worker 37 - jepsen.util 2647 :ok :read [12 14] INFO [2019-12-17 12:29:58,333] jepsen worker 37 - jepsen.util 2647 :invoke :cas [12 [20 29]] INFO [2019-12-17 12:29:58,354] jepsen worker 60 - jepsen.util 4020 :info :write [20 14] :RequestCanceledException INFO [2019-12-17 12:29:58,355] jepsen worker 60 - jepsen.util 4110 :invoke :write [20 27] INFO [2019-12-17 12:29:58,358] jepsen worker 74 - jepsen.util 2954 :ok :read [24 11] INFO [2019-12-17 12:29:58,358] jepsen worker 74 - jepsen.util 2954 :invoke :read [24 nil] INFO [2019-12-17 12:29:58,360] jepsen worker 79 - jepsen.util 2869 :fail :cas [26 [29 32]] :ValueNotSwapFrom INFO [2019-12-17 12:29:58,360] jepsen worker 79 - jepsen.util 2869 :invoke :cas [26 [22 10]] INFO [2019-12-17 12:29:58,373] jepsen worker 89 - jepsen.util 2699 :ok :read [29 28] INFO [2019-12-17 12:29:58,373] jepsen worker 89 - jepsen.util 2699 :invoke :read [29 nil] INFO [2019-12-17 12:29:58,379] jepsen worker 74 - jepsen.util 2954 :ok :read [24 11] INFO [2019-12-17 12:29:58,380] jepsen worker 74 - jepsen.util 2954 :invoke :cas [24 [40 1]] INFO [2019-12-17 12:29:58,395] jepsen worker 74 - jepsen.util 2954 :fail :cas [24 [40 1]] :ValueNotSwapFrom INFO [2019-12-17 12:29:58,396] jepsen worker 74 - jepsen.util 2954 :invoke :write [24 16] INFO [2019-12-17 12:29:58,401] jepsen worker 88 - jepsen.util 2968 :fail :cas [29 [27 36]] :ValueNotSwapFrom INFO [2019-12-17 12:29:58,402] jepsen worker 88 - jepsen.util 2968 :invoke :read [29 nil] INFO [2019-12-17 12:29:58,404] jepsen worker 63 - jepsen.util 2493 :ok :write [21 40] INFO [2019-12-17 12:29:58,404] jepsen worker 63 - jepsen.util 2493 :invoke :cas [21 [27 37]] INFO [2019-12-17 12:29:58,408] jepsen worker 12 - jepsen.util 2982 :info :write [4 29] :RequestCanceledException INFO [2019-12-17 12:29:58,408] jepsen worker 64 - jepsen.util 2314 :fail :read [21 nil] :RequestTimeoutException INFO [2019-12-17 12:29:58,408] jepsen worker 64 - jepsen.util 2314 :invoke :read [21 nil] INFO [2019-12-17 12:29:58,409] jepsen worker 12 - jepsen.util 3072 :invoke :read [4 nil] INFO [2019-12-17 12:29:58,411] jepsen worker 80 - jepsen.util 2780 :fail :cas [26 [14 23]] :ValueNotSwapFrom INFO [2019-12-17 12:29:58,412] jepsen worker 80 - jepsen.util 2780 :invoke :read [26 nil] INFO [2019-12-17 12:29:58,423] jepsen worker 7 - jepsen.util 3607 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:29:58,424] jepsen worker 7 - jepsen.util 3607 :invoke :read [2 nil] INFO [2019-12-17 12:29:58,437] jepsen worker 13 - jepsen.util 2623 :info :write [4 26] :RequestCanceledException INFO [2019-12-17 12:29:58,439] jepsen worker 13 - jepsen.util 2713 :invoke :write [4 26] INFO [2019-12-17 12:29:58,444] jepsen worker 30 - jepsen.util 3360 :ok :write [10 46] INFO [2019-12-17 12:29:58,444] jepsen worker 30 - jepsen.util 3360 :invoke :cas [10 [38 14]] INFO [2019-12-17 12:29:58,444] jepsen worker 73 - jepsen.util 2863 :ok :write [24 38] INFO [2019-12-17 12:29:58,444] jepsen worker 73 - jepsen.util 2863 :invoke :write [24 33] INFO [2019-12-17 12:29:58,447] jepsen worker 37 - jepsen.util 2647 :fail :cas [12 [20 29]] :ValueNotSwapFrom INFO [2019-12-17 12:29:58,447] jepsen worker 37 - jepsen.util 2647 :invoke :write [12 38] INFO [2019-12-17 12:29:58,449] jepsen worker 63 - jepsen.util 2493 :fail :cas [21 [27 37]] :ValueNotSwapFrom INFO [2019-12-17 12:29:58,449] jepsen worker 63 - jepsen.util 2493 :invoke :cas [21 [48 2]] INFO [2019-12-17 12:29:58,458] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16650,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:29:58,458] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16650,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:29:58,458] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16653,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:29:58,458] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16653,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:29:58,458] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16655,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:29:58,458] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16655,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:29:58,467] jepsen worker 64 - jepsen.util 2314 :ok :read [21 40] INFO [2019-12-17 12:29:58,467] jepsen worker 64 - jepsen.util 2314 :invoke :write [21 10] INFO [2019-12-17 12:29:58,490] jepsen worker 23 - jepsen.util 3713 :info :cas [7 [45 48]] :RequestCanceledException INFO [2019-12-17 12:29:58,491] jepsen worker 23 - jepsen.util 3803 :invoke :write [7 3] INFO [2019-12-17 12:29:58,492] jepsen worker 30 - jepsen.util 3360 :fail :cas [10 [38 14]] :ValueNotSwapFrom INFO [2019-12-17 12:29:58,493] jepsen worker 30 - jepsen.util 3360 :invoke :write [10 35] INFO [2019-12-17 12:29:58,495] jepsen worker 63 - jepsen.util 2493 :fail :cas [21 [48 2]] :ValueNotSwapFrom INFO [2019-12-17 12:29:58,495] jepsen worker 63 - jepsen.util 2493 :invoke :read [21 nil] INFO [2019-12-17 12:29:58,499] jepsen worker 79 - jepsen.util 2869 :fail :cas [26 [22 10]] :ValueNotSwapFrom INFO [2019-12-17 12:29:58,499] jepsen worker 79 - jepsen.util 2869 :invoke :cas [26 [33 45]] INFO [2019-12-17 12:29:58,506] jepsen worker 71 - jepsen.util 2861 :info :write [23 23] :RequestCanceledException INFO [2019-12-17 12:29:58,508] jepsen worker 71 - jepsen.util 2951 :invoke :read [23 nil] INFO [2019-12-17 12:29:58,520] jepsen worker 63 - jepsen.util 2493 :ok :read [21 40] INFO [2019-12-17 12:29:58,521] jepsen worker 63 - jepsen.util 2493 :invoke :cas [21 [13 13]] INFO [2019-12-17 12:29:58,543] jepsen worker 64 - jepsen.util 2314 :ok :write [21 10] INFO [2019-12-17 12:29:58,543] jepsen worker 64 - jepsen.util 2314 :invoke :read [21 nil] INFO [2019-12-17 12:29:58,547] jepsen worker 88 - jepsen.util 2968 :ok :read [29 28] INFO [2019-12-17 12:29:58,547] jepsen worker 88 - jepsen.util 2968 :invoke :read [29 nil] INFO [2019-12-17 12:29:58,557] jepsen worker 12 - jepsen.util 3072 :ok :read [4 49] INFO [2019-12-17 12:29:58,558] jepsen worker 12 - jepsen.util 3072 :invoke :read [4 nil] INFO [2019-12-17 12:29:58,566] jepsen worker 89 - jepsen.util 2699 :ok :read [29 28] INFO [2019-12-17 12:29:58,566] jepsen worker 89 - jepsen.util 2699 :invoke :cas [29 [3 39]] INFO [2019-12-17 12:29:58,573] jepsen worker 64 - jepsen.util 2314 :ok :read [21 10] INFO [2019-12-17 12:29:58,573] jepsen worker 64 - jepsen.util 2314 :invoke :read [21 nil] INFO [2019-12-17 12:29:58,585] jepsen worker 63 - jepsen.util 2493 :fail :cas [21 [13 13]] :ValueNotSwapFrom INFO [2019-12-17 12:29:58,585] jepsen worker 63 - jepsen.util 2493 :invoke :cas [21 [25 43]] INFO [2019-12-17 12:29:58,586] jepsen worker 45 - jepsen.util 2925 :info :write [15 45] :RequestCanceledException INFO [2019-12-17 12:29:58,586] jepsen worker 9 - jepsen.util 3249 :fail :read [3 nil] :RequestTimeoutException INFO [2019-12-17 12:29:58,587] jepsen worker 9 - jepsen.util 3249 :invoke :write [3 7] INFO [2019-12-17 12:29:58,589] jepsen worker 45 - jepsen.util 3015 :invoke :write [15 1] INFO [2019-12-17 12:29:58,589] jepsen worker 64 - jepsen.util 2314 :ok :read [21 10] INFO [2019-12-17 12:29:58,589] jepsen worker 64 - jepsen.util 2314 :invoke :cas [21 [8 12]] INFO [2019-12-17 12:29:58,619] jepsen worker 38 - jepsen.util 2828 :ok :read [12 14] INFO [2019-12-17 12:29:58,619] jepsen worker 38 - jepsen.util 2828 :invoke :read [12 nil] INFO [2019-12-17 12:29:58,624] jepsen worker 4 - jepsen.util 2074 :info :write [1 40] :RequestCanceledException INFO [2019-12-17 12:29:58,626] jepsen worker 4 - jepsen.util 2164 :invoke :read [1 nil] INFO [2019-12-17 12:29:58,627] jepsen worker 64 - jepsen.util 2314 :fail :cas [21 [8 12]] :ValueNotSwapFrom INFO [2019-12-17 12:29:58,628] jepsen worker 64 - jepsen.util 2314 :invoke :cas [21 [25 7]] INFO [2019-12-17 12:29:58,640] jepsen worker 88 - jepsen.util 2968 :ok :read [29 28] INFO [2019-12-17 12:29:58,641] jepsen worker 88 - jepsen.util 2968 :invoke :read [29 nil] INFO [2019-12-17 12:29:58,678] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16681,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:29:58,678] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16681,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:29:58,679] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16685,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:29:58,679] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16685,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:29:58,685] jepsen worker 69 - jepsen.util 2769 :ok :read [23 0] INFO [2019-12-17 12:29:58,686] jepsen worker 69 - jepsen.util 2769 :invoke :read [23 nil] INFO [2019-12-17 12:29:58,692] jepsen worker 63 - jepsen.util 2493 :fail :cas [21 [25 43]] :ValueNotSwapFrom INFO [2019-12-17 12:29:58,692] jepsen worker 63 - jepsen.util 2493 :invoke :write [21 4] INFO [2019-12-17 12:29:58,701] jepsen worker 64 - jepsen.util 2314 :fail :cas [21 [25 7]] :ValueNotSwapFrom INFO [2019-12-17 12:29:58,701] jepsen worker 64 - jepsen.util 2314 :invoke :write [21 4] INFO [2019-12-17 12:29:58,704] jepsen worker 18 - jepsen.util 3078 :ok :write [6 31] INFO [2019-12-17 12:29:58,704] jepsen worker 18 - jepsen.util 3078 :invoke :read [6 nil] INFO [2019-12-17 12:29:58,710] jepsen worker 46 - jepsen.util 3016 :ok :write [15 33] INFO [2019-12-17 12:29:58,710] jepsen worker 46 - jepsen.util 3016 :invoke :cas [15 [22 46]] INFO [2019-12-17 12:29:58,722] jepsen worker 4 - jepsen.util 2164 :ok :read [1 18] INFO [2019-12-17 12:29:58,723] jepsen worker 65 - jepsen.util 2585 :fail :read [21 nil] :RequestTimeoutException INFO [2019-12-17 12:29:58,723] jepsen worker 54 - jepsen.util 2934 :info :write [18 36] :RequestCanceledException INFO [2019-12-17 12:29:58,723] jepsen worker 4 - jepsen.util 2164 :invoke :write [1 43] INFO [2019-12-17 12:29:58,723] jepsen worker 65 - jepsen.util 2585 :invoke :cas [21 [5 35]] INFO [2019-12-17 12:29:58,725] jepsen worker 54 - jepsen.util 3024 :invoke :cas [18 [38 0]] INFO [2019-12-17 12:29:58,762] jepsen worker 18 - jepsen.util 3078 :ok :read [6 31] INFO [2019-12-17 12:29:58,762] jepsen worker 18 - jepsen.util 3078 :invoke :cas [6 [47 21]] INFO [2019-12-17 12:29:58,771] jepsen worker 73 - jepsen.util 2863 :ok :write [24 33] INFO [2019-12-17 12:29:58,772] jepsen worker 73 - jepsen.util 2863 :invoke :read [24 nil] INFO [2019-12-17 12:29:58,777] jepsen worker 46 - jepsen.util 3016 :fail :cas [15 [22 46]] :ValueNotSwapFrom INFO [2019-12-17 12:29:58,777] jepsen worker 46 - jepsen.util 3016 :invoke :cas [15 [22 41]] INFO [2019-12-17 12:29:58,800] jepsen worker 73 - jepsen.util 2863 :ok :read [24 33] INFO [2019-12-17 12:29:58,800] jepsen worker 73 - jepsen.util 2863 :invoke :read [24 nil] INFO [2019-12-17 12:29:58,820] jepsen worker 87 - jepsen.util 2607 :info :cas [29 [28 34]] :RequestCanceledException INFO [2019-12-17 12:29:58,822] jepsen worker 87 - jepsen.util 2697 :invoke :write [29 41] INFO [2019-12-17 12:29:58,873] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16703,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:29:58,873] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16703,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:29:58,880] jepsen worker 18 - jepsen.util 3078 :fail :cas [6 [47 21]] :ValueNotSwapFrom INFO [2019-12-17 12:29:58,880] jepsen worker 18 - jepsen.util 3078 :invoke :read [6 nil] INFO [2019-12-17 12:29:58,890] jepsen worker 46 - jepsen.util 3016 :fail :cas [15 [22 41]] :ValueNotSwapFrom INFO [2019-12-17 12:29:58,890] jepsen worker 46 - jepsen.util 3016 :invoke :write [15 28] INFO [2019-12-17 12:29:58,892] jepsen worker 73 - jepsen.util 2863 :ok :read [24 33] INFO [2019-12-17 12:29:58,893] jepsen worker 73 - jepsen.util 2863 :invoke :read [24 nil] INFO [2019-12-17 12:29:58,894] jepsen worker 64 - jepsen.util 2314 :ok :write [21 4] INFO [2019-12-17 12:29:58,894] jepsen worker 64 - jepsen.util 2314 :invoke :cas [21 [46 38]] INFO [2019-12-17 12:29:58,907] jepsen worker 35 - jepsen.util 3635 :info :write [11 43] :RequestCanceledException INFO [2019-12-17 12:29:58,909] jepsen worker 35 - jepsen.util 3725 :invoke :cas [11 [29 25]] INFO [2019-12-17 12:29:58,921] jepsen worker 16 - jepsen.util 3886 :info :write [5 13] :RequestCanceledException INFO [2019-12-17 12:29:58,925] jepsen worker 16 - jepsen.util 3976 :invoke :write [5 2] INFO [2019-12-17 12:29:58,937] jepsen worker 80 - jepsen.util 2780 :ok :read [26 45] INFO [2019-12-17 12:29:58,937] jepsen worker 80 - jepsen.util 2780 :invoke :write [26 2] INFO [2019-12-17 12:29:58,954] jepsen worker 20 - jepsen.util 2450 :info :cas [6 [23 13]] :RequestCanceledException INFO [2019-12-17 12:29:58,955] jepsen worker 73 - jepsen.util 2863 :ok :read [24 33] INFO [2019-12-17 12:29:58,955] jepsen worker 73 - jepsen.util 2863 :invoke :read [24 nil] INFO [2019-12-17 12:29:58,956] jepsen worker 18 - jepsen.util 3078 :ok :read [6 31] INFO [2019-12-17 12:29:58,957] jepsen worker 18 - jepsen.util 3078 :invoke :write [6 48] INFO [2019-12-17 12:29:58,958] jepsen worker 20 - jepsen.util 2540 :invoke :cas [6 [24 45]] INFO [2019-12-17 12:29:58,961] jepsen worker 64 - jepsen.util 2314 :fail :cas [21 [46 38]] :ValueNotSwapFrom INFO [2019-12-17 12:29:58,961] jepsen worker 64 - jepsen.util 2314 :invoke :cas [21 [7 4]] INFO [2019-12-17 12:29:58,991] jepsen worker 73 - jepsen.util 2863 :ok :read [24 33] INFO [2019-12-17 12:29:58,992] jepsen worker 73 - jepsen.util 2863 :invoke :read [24 nil] INFO [2019-12-17 12:29:58,999] jepsen worker 79 - jepsen.util 2869 :fail :cas [26 [33 45]] :ValueNotSwapFrom INFO [2019-12-17 12:29:59,000] jepsen worker 79 - jepsen.util 2869 :invoke :cas [26 [31 2]] INFO [2019-12-17 12:29:59,023] jepsen worker 27 - jepsen.util 2547 :info :write [9 16] :RequestCanceledException INFO [2019-12-17 12:29:59,024] jepsen worker 27 - jepsen.util 2637 :invoke :cas [9 [14 46]] INFO [2019-12-17 12:29:59,036] jepsen worker 59 - jepsen.util 3749 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:29:59,037] jepsen worker 59 - jepsen.util 3749 :invoke :write [19 34] INFO [2019-12-17 12:29:59,046] jepsen worker 73 - jepsen.util 2863 :ok :read [24 33] INFO [2019-12-17 12:29:59,046] jepsen worker 73 - jepsen.util 2863 :invoke :write [24 41] INFO [2019-12-17 12:29:59,054] jepsen worker 56 - jepsen.util 2666 :info :write [18 35] :RequestCanceledException INFO [2019-12-17 12:29:59,054] jepsen worker 55 - jepsen.util 3385 :info :write [18 41] :RequestCanceledException INFO [2019-12-17 12:29:59,055] jepsen worker 43 - jepsen.util 3553 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:29:59,055] jepsen worker 43 - jepsen.util 3553 :invoke :write [14 17] INFO [2019-12-17 12:29:59,058] jepsen worker 55 - jepsen.util 3475 :invoke :cas [18 [25 9]] INFO [2019-12-17 12:29:59,059] jepsen worker 56 - jepsen.util 2756 :invoke :write [18 11] INFO [2019-12-17 12:29:59,069] jepsen worker 27 - jepsen.util 2637 :fail :cas [9 [14 46]] :ValueNotSwapFrom INFO [2019-12-17 12:29:59,069] jepsen worker 27 - jepsen.util 2637 :invoke :read [9 nil] INFO [2019-12-17 12:29:59,076] jepsen worker 64 - jepsen.util 2314 :fail :cas [21 [7 4]] :ValueNotSwapFrom INFO [2019-12-17 12:29:59,077] jepsen worker 64 - jepsen.util 2314 :invoke :cas [21 [39 25]] INFO [2019-12-17 12:29:59,078] jepsen worker 89 - jepsen.util 2699 :fail :cas [29 [3 39]] :ValueNotSwapFrom INFO [2019-12-17 12:29:59,078] jepsen worker 89 - jepsen.util 2699 :invoke :read [29 nil] INFO [2019-12-17 12:29:59,092] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16718,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:29:59,092] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16718,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:29:59,092] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16721,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:29:59,093] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16721,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:29:59,093] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16722,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:29:59,093] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16722,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:29:59,137] jepsen worker 52 - jepsen.util 3112 :info :write [17 13] :RequestCanceledException INFO [2019-12-17 12:29:59,139] jepsen worker 64 - jepsen.util 2314 :fail :cas [21 [39 25]] :ValueNotSwapFrom INFO [2019-12-17 12:29:59,139] jepsen worker 64 - jepsen.util 2314 :invoke :cas [21 [23 39]] INFO [2019-12-17 12:29:59,141] jepsen worker 52 - jepsen.util 3202 :invoke :write [17 16] INFO [2019-12-17 12:29:59,148] jepsen worker 27 - jepsen.util 2637 :ok :read [9 46] INFO [2019-12-17 12:29:59,148] jepsen worker 27 - jepsen.util 2637 :invoke :read [9 nil] INFO [2019-12-17 12:29:59,154] jepsen worker 70 - jepsen.util 3310 :info :write [23 5] :RequestCanceledException INFO [2019-12-17 12:29:59,154] jepsen worker 15 - jepsen.util 3435 :info :cas [5 [28 4]] :RequestCanceledException INFO [2019-12-17 12:29:59,156] jepsen worker 15 - jepsen.util 3525 :invoke :write [5 39] INFO [2019-12-17 12:29:59,156] jepsen worker 70 - jepsen.util 3400 :invoke :read [23 nil] INFO [2019-12-17 12:29:59,189] jepsen worker 64 - jepsen.util 2314 :fail :cas [21 [23 39]] :ValueNotSwapFrom INFO [2019-12-17 12:29:59,189] jepsen worker 64 - jepsen.util 2314 :invoke :read [21 nil] INFO [2019-12-17 12:29:59,190] jepsen worker 85 - jepsen.util 4315 :info :write [28 26] :RequestCanceledException INFO [2019-12-17 12:29:59,194] jepsen worker 27 - jepsen.util 2637 :ok :read [9 46] INFO [2019-12-17 12:29:59,194] jepsen worker 85 - jepsen.util 4405 :invoke :cas [28 [32 10]] INFO [2019-12-17 12:29:59,194] jepsen worker 27 - jepsen.util 2637 :invoke :cas [9 [26 39]] INFO [2019-12-17 12:29:59,201] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16740,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:29:59,201] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16740,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:29:59,201] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16741,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:29:59,202] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16741,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:29:59,208] jepsen worker 84 - jepsen.util 3594 :info :write [28 11] :RequestCanceledException INFO [2019-12-17 12:29:59,212] jepsen worker 84 - jepsen.util 3684 :invoke :write [28 9] INFO [2019-12-17 12:29:59,224] jepsen worker 50 - jepsen.util 4190 :info :cas [16 [37 7]] :RequestCanceledException INFO [2019-12-17 12:29:59,226] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16751,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:29:59,227] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16751,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:29:59,227] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16753,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:29:59,227] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16753,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:29:59,227] jepsen worker 50 - jepsen.util 4280 :invoke :write [16 13] INFO [2019-12-17 12:29:59,236] jepsen worker 62 - jepsen.util 3572 :info :cas [20 [34 26]] :RequestCanceledException INFO [2019-12-17 12:29:59,241] jepsen worker 12 - jepsen.util 3072 :ok :read [4 49] INFO [2019-12-17 12:29:59,241] jepsen worker 12 - jepsen.util 3072 :invoke :cas [4 [35 10]] INFO [2019-12-17 12:29:59,241] jepsen worker 64 - jepsen.util 2314 :ok :read [21 4] INFO [2019-12-17 12:29:59,241] jepsen worker 62 - jepsen.util 3662 :invoke :cas [20 [24 9]] INFO [2019-12-17 12:29:59,241] jepsen worker 64 - jepsen.util 2314 :invoke :write [21 2] INFO [2019-12-17 12:29:59,247] jepsen worker 27 - jepsen.util 2637 :fail :cas [9 [26 39]] :ValueNotSwapFrom INFO [2019-12-17 12:29:59,248] jepsen worker 27 - jepsen.util 2637 :invoke :cas [9 [29 29]] INFO [2019-12-17 12:29:59,308] jepsen worker 31 - jepsen.util 2641 :fail :read [10 nil] :RequestTimeoutException INFO [2019-12-17 12:29:59,308] jepsen worker 31 - jepsen.util 2641 :invoke :cas [10 [47 16]] INFO [2019-12-17 12:29:59,314] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:29:59,315] jepsen nemesis - couchbase.util Rebalance check count: 122 INFO [2019-12-17 12:29:59,315] jepsen nemesis - couchbase.util Rebalance stuck count: 103 INFO [2019-12-17 12:29:59,319] jepsen worker 88 - jepsen.util 2968 :ok :read [29 28] INFO [2019-12-17 12:29:59,319] jepsen worker 88 - jepsen.util 2968 :invoke :cas [29 [49 17]] INFO [2019-12-17 12:29:59,320] jepsen worker 27 - jepsen.util 2637 :fail :cas [9 [29 29]] :ValueNotSwapFrom INFO [2019-12-17 12:29:59,320] jepsen worker 27 - jepsen.util 2637 :invoke :cas [9 [32 18]] INFO [2019-12-17 12:29:59,324] jepsen worker 10 - jepsen.util 2620 :info :write [3 8] :RequestCanceledException INFO [2019-12-17 12:29:59,324] jepsen worker 75 - jepsen.util 3945 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:29:59,324] jepsen worker 75 - jepsen.util 3945 :invoke :write [25 27] INFO [2019-12-17 12:29:59,328] jepsen worker 10 - jepsen.util 2710 :invoke :write [3 16] INFO [2019-12-17 12:29:59,337] jepsen worker 1 - jepsen.util 3241 :info :write [0 26] :RequestCanceledException INFO [2019-12-17 12:29:59,341] jepsen worker 1 - jepsen.util 3331 :invoke :write [0 33] INFO [2019-12-17 12:29:59,354] jepsen worker 14 - jepsen.util 2714 :info :write [4 19] :RequestCanceledException INFO [2019-12-17 12:29:59,354] jepsen worker 38 - jepsen.util 2828 :ok :read [12 14] INFO [2019-12-17 12:29:59,355] jepsen worker 38 - jepsen.util 2828 :invoke :cas [12 [25 30]] INFO [2019-12-17 12:29:59,356] jepsen worker 14 - jepsen.util 2804 :invoke :cas [4 [7 48]] INFO [2019-12-17 12:29:59,381] jepsen worker 27 - jepsen.util 2637 :fail :cas [9 [32 18]] :ValueNotSwapFrom INFO [2019-12-17 12:29:59,381] jepsen worker 27 - jepsen.util 2637 :invoke :write [9 4] INFO [2019-12-17 12:29:59,387] jepsen worker 79 - jepsen.util 2869 :fail :cas [26 [31 2]] :ValueNotSwapFrom INFO [2019-12-17 12:29:59,387] jepsen worker 79 - jepsen.util 2869 :invoke :cas [26 [31 40]] INFO [2019-12-17 12:29:59,390] jepsen worker 40 - jepsen.util 4090 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:29:59,390] jepsen worker 40 - jepsen.util 4090 :invoke :write [13 40] INFO [2019-12-17 12:29:59,405] jepsen worker 5 - jepsen.util 2525 :ok :read [1 18] INFO [2019-12-17 12:29:59,405] jepsen worker 5 - jepsen.util 2525 :invoke :write [1 3] INFO [2019-12-17 12:29:59,407] jepsen worker 78 - jepsen.util 3048 :info :write [26 44] :RequestCanceledException INFO [2019-12-17 12:29:59,410] jepsen worker 78 - jepsen.util 3138 :invoke :write [26 41] INFO [2019-12-17 12:29:59,418] jepsen worker 64 - jepsen.util 2314 :ok :write [21 2] INFO [2019-12-17 12:29:59,418] jepsen worker 64 - jepsen.util 2314 :invoke :cas [21 [8 41]] INFO [2019-12-17 12:29:59,423] jepsen worker 47 - jepsen.util 3287 :info :cas [15 [41 43]] :RequestCanceledException INFO [2019-12-17 12:29:59,423] jepsen worker 57 - jepsen.util 3567 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:29:59,423] jepsen worker 28 - jepsen.util 2098 :info :cas [9 [13 41]] :RequestCanceledException INFO [2019-12-17 12:29:59,423] jepsen worker 41 - jepsen.util 4001 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:29:59,423] jepsen worker 57 - jepsen.util 3567 :invoke :write [19 24] INFO [2019-12-17 12:29:59,423] jepsen worker 41 - jepsen.util 4001 :invoke :cas [13 [48 39]] INFO [2019-12-17 12:29:59,424] jepsen worker 28 - jepsen.util 2188 :invoke :cas [9 [44 7]] INFO [2019-12-17 12:29:59,425] jepsen worker 47 - jepsen.util 3377 :invoke :write [15 3] INFO [2019-12-17 12:29:59,432] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16776,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:29:59,432] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16776,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:29:59,432] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16779,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:29:59,432] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16779,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:29:59,456] jepsen worker 70 - jepsen.util 3400 :ok :read [23 0] INFO [2019-12-17 12:29:59,456] jepsen worker 70 - jepsen.util 3400 :invoke :read [23 nil] INFO [2019-12-17 12:29:59,462] jepsen worker 31 - jepsen.util 2641 :fail :cas [10 [47 16]] :ValueNotSwapFrom INFO [2019-12-17 12:29:59,462] jepsen worker 31 - jepsen.util 2641 :invoke :cas [10 [41 39]] INFO [2019-12-17 12:29:59,471] jepsen worker 89 - jepsen.util 2699 :ok :read [29 28] INFO [2019-12-17 12:29:59,471] jepsen worker 89 - jepsen.util 2699 :invoke :read [29 nil] INFO [2019-12-17 12:29:59,484] jepsen worker 31 - jepsen.util 2641 :fail :cas [10 [41 39]] :ValueNotSwapFrom INFO [2019-12-17 12:29:59,484] jepsen worker 31 - jepsen.util 2641 :invoke :read [10 nil] INFO [2019-12-17 12:29:59,503] jepsen worker 83 - jepsen.util 3953 :info :cas [27 [43 28]] :RequestCanceledException INFO [2019-12-17 12:29:59,503] jepsen worker 21 - jepsen.util 3711 :info :cas [7 [26 29]] :RequestCanceledException INFO [2019-12-17 12:29:59,504] jepsen worker 83 - jepsen.util 4043 :invoke :read [27 nil] INFO [2019-12-17 12:29:59,504] jepsen worker 21 - jepsen.util 3801 :invoke :write [7 41] INFO [2019-12-17 12:29:59,511] jepsen worker 31 - jepsen.util 2641 :ok :read [10 46] INFO [2019-12-17 12:29:59,512] jepsen worker 31 - jepsen.util 2641 :invoke :write [10 28] INFO [2019-12-17 12:29:59,518] jepsen worker 27 - jepsen.util 2637 :ok :write [9 4] INFO [2019-12-17 12:29:59,519] jepsen worker 27 - jepsen.util 2637 :invoke :write [9 15] INFO [2019-12-17 12:29:59,520] jepsen worker 22 - jepsen.util 3802 :info :write [7 49] :RequestCanceledException INFO [2019-12-17 12:29:59,520] jepsen worker 42 - jepsen.util 4362 :info :write [14 36] :RequestCanceledException INFO [2019-12-17 12:29:59,520] jepsen worker 36 - jepsen.util 2736 :info :write [12 35] :RequestCanceledException INFO [2019-12-17 12:29:59,521] jepsen worker 22 - jepsen.util 3892 :invoke :write [7 21] INFO [2019-12-17 12:29:59,521] jepsen worker 36 - jepsen.util 2826 :invoke :read [12 nil] INFO [2019-12-17 12:29:59,521] jepsen worker 42 - jepsen.util 4452 :invoke :write [14 1] INFO [2019-12-17 12:29:59,533] jepsen worker 88 - jepsen.util 2968 :fail :cas [29 [49 17]] :ValueNotSwapFrom INFO [2019-12-17 12:29:59,534] jepsen worker 88 - jepsen.util 2968 :invoke :read [29 nil] INFO [2019-12-17 12:29:59,537] jepsen worker 8 - jepsen.util 3248 :info :write [2 6] :RequestCanceledException INFO [2019-12-17 12:29:59,537] jepsen worker 33 - jepsen.util 4083 :info :cas [11 [46 17]] :RequestCanceledException INFO [2019-12-17 12:29:59,539] jepsen worker 8 - jepsen.util 3338 :invoke :read [2 nil] INFO [2019-12-17 12:29:59,541] jepsen worker 33 - jepsen.util 4173 :invoke :read [11 nil] INFO [2019-12-17 12:29:59,541] jepsen worker 3 - jepsen.util 2793 :ok :read [1 18] INFO [2019-12-17 12:29:59,541] jepsen worker 3 - jepsen.util 2793 :invoke :read [1 nil] INFO [2019-12-17 12:29:59,554] jepsen worker 26 - jepsen.util 4076 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:29:59,554] jepsen worker 26 - jepsen.util 4076 :invoke :write [8 30] INFO [2019-12-17 12:29:59,558] jepsen worker 51 - jepsen.util 2751 :fail :cas [17 [41 16]] :ValueNotSwapFrom INFO [2019-12-17 12:29:59,558] jepsen worker 51 - jepsen.util 2751 :invoke :read [17 nil] INFO [2019-12-17 12:29:59,575] jepsen worker 71 - jepsen.util 2951 :ok :read [23 0] INFO [2019-12-17 12:29:59,575] jepsen worker 71 - jepsen.util 2951 :invoke :read [23 nil] INFO [2019-12-17 12:29:59,590] jepsen worker 69 - jepsen.util 2769 :ok :read [23 0] INFO [2019-12-17 12:29:59,590] jepsen worker 69 - jepsen.util 2769 :invoke :write [23 22] INFO [2019-12-17 12:29:59,590] jepsen worker 17 - jepsen.util 3887 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:29:59,590] jepsen worker 48 - jepsen.util 3198 :info :cas [16 [22 15]] :RequestCanceledException INFO [2019-12-17 12:29:59,590] jepsen worker 17 - jepsen.util 3887 :invoke :cas [5 [2 6]] INFO [2019-12-17 12:29:59,591] jepsen worker 27 - jepsen.util 2637 :ok :write [9 15] INFO [2019-12-17 12:29:59,591] jepsen worker 27 - jepsen.util 2637 :invoke :cas [9 [30 6]] INFO [2019-12-17 12:29:59,591] jepsen worker 48 - jepsen.util 3288 :invoke :write [16 48] INFO [2019-12-17 12:29:59,612] jepsen worker 27 - jepsen.util 2637 :fail :cas [9 [30 6]] :ValueNotSwapFrom INFO [2019-12-17 12:29:59,613] jepsen worker 27 - jepsen.util 2637 :invoke :cas [9 [16 45]] INFO [2019-12-17 12:29:59,624] jepsen worker 29 - jepsen.util 2459 :fail :read [9 nil] :RequestTimeoutException INFO [2019-12-17 12:29:59,624] jepsen worker 0 - jepsen.util 4140 :info :write [0 22] :RequestCanceledException INFO [2019-12-17 12:29:59,624] jepsen worker 29 - jepsen.util 2459 :invoke :read [9 nil] INFO [2019-12-17 12:29:59,625] jepsen worker 54 - jepsen.util 3024 :fail :cas [18 [38 0]] :ValueNotSwapFrom INFO [2019-12-17 12:29:59,626] jepsen worker 54 - jepsen.util 3024 :invoke :read [18 nil] INFO [2019-12-17 12:29:59,626] jepsen worker 0 - jepsen.util 4230 :invoke :cas [0 [18 12]] INFO [2019-12-17 12:29:59,631] jepsen worker 27 - jepsen.util 2637 :fail :cas [9 [16 45]] :ValueNotSwapFrom INFO [2019-12-17 12:29:59,631] jepsen worker 27 - jepsen.util 2637 :invoke :write [9 3] INFO [2019-12-17 12:29:59,666] jepsen worker 27 - jepsen.util 2637 :ok :write [9 3] INFO [2019-12-17 12:29:59,666] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16812,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:29:59,667] jepsen worker 27 - jepsen.util 2637 :invoke :read [9 nil] INFO [2019-12-17 12:29:59,667] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16812,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:29:59,667] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16815,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:29:59,668] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16815,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:29:59,668] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16816,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:29:59,668] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16816,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:29:59,688] jepsen worker 27 - jepsen.util 2637 :ok :read [9 3] INFO [2019-12-17 12:29:59,689] jepsen worker 27 - jepsen.util 2637 :invoke :cas [9 [19 7]] INFO [2019-12-17 12:29:59,690] jepsen worker 89 - jepsen.util 2699 :ok :read [29 28] INFO [2019-12-17 12:29:59,690] jepsen worker 89 - jepsen.util 2699 :invoke :read [29 nil] INFO [2019-12-17 12:29:59,692] jepsen worker 51 - jepsen.util 2751 :ok :read [17 42] INFO [2019-12-17 12:29:59,693] jepsen worker 51 - jepsen.util 2751 :invoke :cas [17 [5 14]] INFO [2019-12-17 12:29:59,699] jepsen worker 12 - jepsen.util 3072 :fail :cas [4 [35 10]] :ValueNotSwapFrom INFO [2019-12-17 12:29:59,699] jepsen worker 12 - jepsen.util 3072 :invoke :read [4 nil] INFO [2019-12-17 12:29:59,706] jepsen worker 27 - jepsen.util 2637 :fail :cas [9 [19 7]] :ValueNotSwapFrom INFO [2019-12-17 12:29:59,707] jepsen worker 27 - jepsen.util 2637 :invoke :cas [9 [30 13]] INFO [2019-12-17 12:29:59,708] jepsen worker 53 - jepsen.util 2843 :info :write [17 45] :RequestCanceledException INFO [2019-12-17 12:29:59,710] jepsen worker 53 - jepsen.util 2933 :invoke :read [17 nil] INFO [2019-12-17 12:29:59,721] jepsen worker 55 - jepsen.util 3475 :fail :cas [18 [25 9]] :ValueNotSwapFrom INFO [2019-12-17 12:29:59,721] jepsen worker 55 - jepsen.util 3475 :invoke :write [18 44] INFO [2019-12-17 12:29:59,722] jepsen worker 27 - jepsen.util 2637 :fail :cas [9 [30 13]] :ValueNotSwapFrom INFO [2019-12-17 12:29:59,722] jepsen worker 27 - jepsen.util 2637 :invoke :read [9 nil] INFO [2019-12-17 12:29:59,730] jepsen worker 79 - jepsen.util 2869 :fail :cas [26 [31 40]] :ValueNotSwapFrom INFO [2019-12-17 12:29:59,731] jepsen worker 79 - jepsen.util 2869 :invoke :write [26 20] INFO [2019-12-17 12:29:59,738] jepsen worker 27 - jepsen.util 2637 :ok :read [9 3] INFO [2019-12-17 12:29:59,738] jepsen worker 27 - jepsen.util 2637 :invoke :cas [9 [13 24]] INFO [2019-12-17 12:29:59,754] jepsen worker 27 - jepsen.util 2637 :fail :cas [9 [13 24]] :ValueNotSwapFrom INFO [2019-12-17 12:29:59,754] jepsen worker 27 - jepsen.util 2637 :invoke :read [9 nil] INFO [2019-12-17 12:29:59,757] jepsen worker 71 - jepsen.util 2951 :ok :read [23 0] INFO [2019-12-17 12:29:59,757] jepsen worker 71 - jepsen.util 2951 :invoke :write [23 14] INFO [2019-12-17 12:29:59,769] jepsen worker 27 - jepsen.util 2637 :ok :read [9 3] INFO [2019-12-17 12:29:59,769] jepsen worker 27 - jepsen.util 2637 :invoke :read [9 nil] INFO [2019-12-17 12:29:59,781] jepsen worker 27 - jepsen.util 2637 :ok :read [9 3] INFO [2019-12-17 12:29:59,782] jepsen worker 27 - jepsen.util 2637 :invoke :write [9 28] INFO [2019-12-17 12:29:59,820] jepsen worker 72 - jepsen.util 2862 :fail :read [24 nil] :RequestTimeoutException INFO [2019-12-17 12:29:59,820] jepsen worker 72 - jepsen.util 2862 :invoke :write [24 48] INFO [2019-12-17 12:29:59,834] jepsen worker 27 - jepsen.util 2637 :ok :write [9 28] INFO [2019-12-17 12:29:59,834] jepsen worker 27 - jepsen.util 2637 :invoke :read [9 nil] INFO [2019-12-17 12:29:59,838] jepsen worker 38 - jepsen.util 2828 :fail :cas [12 [25 30]] :ValueNotSwapFrom INFO [2019-12-17 12:29:59,838] jepsen worker 38 - jepsen.util 2828 :invoke :read [12 nil] INFO [2019-12-17 12:29:59,851] jepsen worker 19 - jepsen.util 2989 :info :write [6 13] :RequestCanceledException INFO [2019-12-17 12:29:59,853] jepsen worker 19 - jepsen.util 3079 :invoke :read [6 nil] INFO [2019-12-17 12:29:59,854] jepsen worker 3 - jepsen.util 2793 :ok :read [1 18] INFO [2019-12-17 12:29:59,854] jepsen worker 3 - jepsen.util 2793 :invoke :write [1 12] INFO [2019-12-17 12:29:59,857] jepsen worker 88 - jepsen.util 2968 :ok :read [29 28] INFO [2019-12-17 12:29:59,857] jepsen worker 88 - jepsen.util 2968 :invoke :read [29 nil] INFO [2019-12-17 12:29:59,863] jepsen worker 27 - jepsen.util 2637 :ok :read [9 28] INFO [2019-12-17 12:29:59,863] jepsen worker 27 - jepsen.util 2637 :invoke :cas [9 [18 3]] INFO [2019-12-17 12:29:59,886] jepsen worker 27 - jepsen.util 2637 :fail :cas [9 [18 3]] :ValueNotSwapFrom INFO [2019-12-17 12:29:59,886] jepsen worker 27 - jepsen.util 2637 :invoke :read [9 nil] INFO [2019-12-17 12:29:59,893] jepsen worker 51 - jepsen.util 2751 :fail :cas [17 [5 14]] :ValueNotSwapFrom INFO [2019-12-17 12:29:59,894] jepsen worker 51 - jepsen.util 2751 :invoke :cas [17 [40 38]] INFO [2019-12-17 12:29:59,897] jepsen worker 27 - jepsen.util 2637 :ok :read [9 28] INFO [2019-12-17 12:29:59,897] jepsen worker 27 - jepsen.util 2637 :invoke :write [9 3] INFO [2019-12-17 12:29:59,923] jepsen worker 14 - jepsen.util 2804 :fail :cas [4 [7 48]] :ValueNotSwapFrom INFO [2019-12-17 12:29:59,923] jepsen worker 14 - jepsen.util 2804 :invoke :cas [4 [23 6]] INFO [2019-12-17 12:29:59,945] jepsen worker 27 - jepsen.util 2637 :ok :write [9 3] INFO [2019-12-17 12:29:59,945] jepsen worker 27 - jepsen.util 2637 :invoke :read [9 nil] INFO [2019-12-17 12:29:59,954] jepsen worker 25 - jepsen.util 4345 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:29:59,954] jepsen worker 25 - jepsen.util 4345 :invoke :read [8 nil] INFO [2019-12-17 12:29:59,958] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16839,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:29:59,958] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16839,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:29:59,958] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16842,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:29:59,958] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16842,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:29:59,965] jepsen worker 27 - jepsen.util 2637 :ok :read [9 3] INFO [2019-12-17 12:29:59,965] jepsen worker 27 - jepsen.util 2637 :invoke :write [9 14] INFO [2019-12-17 12:30:00,003] jepsen worker 9 - jepsen.util 3249 :ok :write [3 7] INFO [2019-12-17 12:30:00,003] jepsen worker 31 - jepsen.util 2641 :ok :write [10 28] INFO [2019-12-17 12:30:00,003] jepsen worker 73 - jepsen.util 2863 :ok :write [24 41] INFO [2019-12-17 12:30:00,003] jepsen worker 9 - jepsen.util 3249 :invoke :write [3 42] INFO [2019-12-17 12:30:00,004] jepsen worker 31 - jepsen.util 2641 :invoke :cas [10 [36 37]] INFO [2019-12-17 12:30:00,004] jepsen worker 73 - jepsen.util 2863 :invoke :cas [24 [37 32]] INFO [2019-12-17 12:30:00,004] jepsen worker 45 - jepsen.util 3015 :ok :write [15 1] INFO [2019-12-17 12:30:00,005] jepsen worker 45 - jepsen.util 3015 :invoke :read [15 nil] INFO [2019-12-17 12:30:00,010] jepsen worker 18 - jepsen.util 3078 :ok :write [6 48] INFO [2019-12-17 12:30:00,010] jepsen worker 18 - jepsen.util 3078 :invoke :write [6 36] INFO [2019-12-17 12:30:00,011] jepsen worker 14 - jepsen.util 2804 :fail :cas [4 [23 6]] :ValueNotSwapFrom INFO [2019-12-17 12:30:00,011] jepsen worker 14 - jepsen.util 2804 :invoke :write [4 36] INFO [2019-12-17 12:30:00,024] jepsen worker 81 - jepsen.util 3771 :info :cas [27 [42 36]] :RequestCanceledException INFO [2019-12-17 12:30:00,025] jepsen worker 81 - jepsen.util 3861 :invoke :write [27 4] INFO [2019-12-17 12:30:00,026] jepsen worker 36 - jepsen.util 2826 :ok :read [12 14] INFO [2019-12-17 12:30:00,026] jepsen worker 36 - jepsen.util 2826 :invoke :cas [12 [30 44]] INFO [2019-12-17 12:30:00,037] jepsen worker 76 - jepsen.util 3856 :info :cas [25 [13 15]] :RequestCanceledException INFO [2019-12-17 12:30:00,038] jepsen worker 76 - jepsen.util 3946 :invoke :cas [25 [18 16]] INFO [2019-12-17 12:30:00,059] jepsen worker 54 - jepsen.util 3024 :ok :read [18 47] INFO [2019-12-17 12:30:00,060] jepsen worker 54 - jepsen.util 3024 :invoke :cas [18 [28 21]] INFO [2019-12-17 12:30:00,067] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16855,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:30:00,067] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16855,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:30:00,067] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16856,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:30:00,067] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16856,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:30:00,072] jepsen worker 20 - jepsen.util 2540 :fail :cas [6 [24 45]] :ValueNotSwapFrom INFO [2019-12-17 12:30:00,072] jepsen worker 20 - jepsen.util 2540 :invoke :write [6 27] INFO [2019-12-17 12:30:00,073] jepsen worker 89 - jepsen.util 2699 :ok :read [29 28] INFO [2019-12-17 12:30:00,073] jepsen worker 89 - jepsen.util 2699 :invoke :read [29 nil] INFO [2019-12-17 12:30:00,084] jepsen worker 88 - jepsen.util 2968 :ok :read [29 28] INFO [2019-12-17 12:30:00,085] jepsen worker 88 - jepsen.util 2968 :invoke :cas [29 [21 46]] INFO [2019-12-17 12:30:00,088] jepsen worker 58 - jepsen.util 3748 :info :write [19 49] :RequestCanceledException INFO [2019-12-17 12:30:00,088] jepsen worker 66 - jepsen.util 3936 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:30:00,089] jepsen worker 66 - jepsen.util 3936 :invoke :read [22 nil] INFO [2019-12-17 12:30:00,089] jepsen worker 58 - jepsen.util 3838 :invoke :write [19 27] INFO [2019-12-17 12:30:00,091] jepsen worker 73 - jepsen.util 2863 :fail :cas [24 [37 32]] :ValueNotSwapFrom INFO [2019-12-17 12:30:00,091] jepsen worker 73 - jepsen.util 2863 :invoke :write [24 45] INFO [2019-12-17 12:30:00,092] jepsen worker 12 - jepsen.util 3072 :ok :read [4 49] INFO [2019-12-17 12:30:00,092] jepsen worker 12 - jepsen.util 3072 :invoke :write [4 29] INFO [2019-12-17 12:30:00,099] jepsen worker 28 - jepsen.util 2188 :fail :cas [9 [44 7]] :ValueNotSwapFrom INFO [2019-12-17 12:30:00,099] jepsen worker 28 - jepsen.util 2188 :invoke :read [9 nil] INFO [2019-12-17 12:30:00,100] jepsen worker 31 - jepsen.util 2641 :fail :cas [10 [36 37]] :ValueNotSwapFrom INFO [2019-12-17 12:30:00,100] jepsen worker 31 - jepsen.util 2641 :invoke :read [10 nil] INFO [2019-12-17 12:30:00,108] jepsen worker 32 - jepsen.util 3272 :info :write [10 36] :RequestCanceledException INFO [2019-12-17 12:30:00,108] jepsen worker 68 - jepsen.util 3578 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:30:00,108] jepsen worker 82 - jepsen.util 4582 :info :cas [27 [27 47]] :RequestCanceledException INFO [2019-12-17 12:30:00,109] jepsen worker 68 - jepsen.util 3578 :invoke :cas [22 [23 1]] INFO [2019-12-17 12:30:00,110] jepsen worker 82 - jepsen.util 4672 :invoke :read [27 nil] INFO [2019-12-17 12:30:00,110] jepsen worker 32 - jepsen.util 3362 :invoke :cas [10 [44 45]] INFO [2019-12-17 12:30:00,118] jepsen worker 70 - jepsen.util 3400 :ok :read [23 0] INFO [2019-12-17 12:30:00,118] jepsen worker 70 - jepsen.util 3400 :invoke :write [23 33] INFO [2019-12-17 12:30:00,123] jepsen worker 39 - jepsen.util 3729 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:30:00,123] jepsen worker 61 - jepsen.util 4291 :info :cas [20 [10 1]] :RequestCanceledException INFO [2019-12-17 12:30:00,123] jepsen worker 77 - jepsen.util 3587 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:30:00,124] jepsen worker 39 - jepsen.util 3729 :invoke :write [13 30] INFO [2019-12-17 12:30:00,124] jepsen worker 77 - jepsen.util 3587 :invoke :read [25 nil] INFO [2019-12-17 12:30:00,124] jepsen worker 61 - jepsen.util 4381 :invoke :write [20 3] INFO [2019-12-17 12:30:00,137] jepsen worker 2 - jepsen.util 3512 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:30:00,137] jepsen worker 86 - jepsen.util 3956 :info :cas [28 [2 34]] :RequestCanceledException INFO [2019-12-17 12:30:00,137] jepsen worker 2 - jepsen.util 3512 :invoke :write [0 0] INFO [2019-12-17 12:30:00,138] jepsen worker 86 - jepsen.util 4046 :invoke :write [28 19] INFO [2019-12-17 12:30:00,154] jepsen worker 11 - jepsen.util 3161 :info :cas [3 [20 34]] :RequestCanceledException INFO [2019-12-17 12:30:00,155] jepsen worker 11 - jepsen.util 3251 :invoke :write [3 23] INFO [2019-12-17 12:30:00,180] jepsen worker 53 - jepsen.util 2933 :ok :read [17 42] INFO [2019-12-17 12:30:00,180] jepsen worker 53 - jepsen.util 2933 :invoke :read [17 nil] INFO [2019-12-17 12:30:00,185] jepsen worker 19 - jepsen.util 3079 :ok :read [6 48] INFO [2019-12-17 12:30:00,186] jepsen worker 19 - jepsen.util 3079 :invoke :cas [6 [39 24]] INFO [2019-12-17 12:30:00,187] jepsen worker 31 - jepsen.util 2641 :ok :read [10 28] INFO [2019-12-17 12:30:00,188] jepsen worker 31 - jepsen.util 2641 :invoke :write [10 44] INFO [2019-12-17 12:30:00,191] jepsen worker 29 - jepsen.util 2459 :ok :read [9 3] INFO [2019-12-17 12:30:00,191] jepsen worker 29 - jepsen.util 2459 :invoke :read [9 nil] INFO [2019-12-17 12:30:00,217] jepsen worker 27 - jepsen.util 2637 :ok :write [9 14] INFO [2019-12-17 12:30:00,218] jepsen worker 27 - jepsen.util 2637 :invoke :write [9 4] INFO [2019-12-17 12:30:00,220] jepsen worker 88 - jepsen.util 2968 :fail :cas [29 [21 46]] :ValueNotSwapFrom INFO [2019-12-17 12:30:00,220] jepsen worker 88 - jepsen.util 2968 :invoke :write [29 18] INFO [2019-12-17 12:30:00,223] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16898,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:30:00,223] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16898,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:30:00,223] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16901,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:30:00,223] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16901,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:30:00,223] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16902,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:30:00,223] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16902,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:30:00,254] jepsen worker 67 - jepsen.util 4297 :info :write [22 43] :RequestCanceledException INFO [2019-12-17 12:30:00,254] jepsen worker 24 - jepsen.util 3624 :info :write [8 21] :RequestCanceledException INFO [2019-12-17 12:30:00,255] jepsen worker 24 - jepsen.util 3714 :invoke :cas [8 [47 22]] INFO [2019-12-17 12:30:00,255] jepsen worker 67 - jepsen.util 4387 :invoke :read [22 nil] INFO [2019-12-17 12:30:00,283] jepsen worker 38 - jepsen.util 2828 :ok :read [12 14] INFO [2019-12-17 12:30:00,284] jepsen worker 38 - jepsen.util 2828 :invoke :cas [12 [22 16]] INFO [2019-12-17 12:30:00,284] jepsen worker 27 - jepsen.util 2637 :ok :write [9 4] INFO [2019-12-17 12:30:00,284] jepsen worker 27 - jepsen.util 2637 :invoke :cas [9 [6 43]] INFO [2019-12-17 12:30:00,287] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16890,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:30:00,287] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16890,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:30:00,287] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16891,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:30:00,287] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16891,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:30:00,287] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16892,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:30:00,287] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16892,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:30:00,314] jepsen worker 27 - jepsen.util 2637 :fail :cas [9 [6 43]] :ValueNotSwapFrom INFO [2019-12-17 12:30:00,314] jepsen worker 27 - jepsen.util 2637 :invoke :read [9 nil] INFO [2019-12-17 12:30:00,323] jepsen worker 44 - jepsen.util 3914 :info :cas [14 [32 46]] :RequestCanceledException INFO [2019-12-17 12:30:00,324] jepsen worker 44 - jepsen.util 4004 :invoke :cas [14 [31 13]] INFO [2019-12-17 12:30:00,330] jepsen worker 47 - jepsen.util 3377 :ok :write [15 3] INFO [2019-12-17 12:30:00,330] jepsen worker 47 - jepsen.util 3377 :invoke :read [15 nil] INFO [2019-12-17 12:30:00,334] jepsen worker 27 - jepsen.util 2637 :ok :read [9 4] INFO [2019-12-17 12:30:00,334] jepsen worker 27 - jepsen.util 2637 :invoke :read [9 nil] INFO [2019-12-17 12:30:00,356] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:30:00,356] jepsen nemesis - couchbase.util Rebalance check count: 123 INFO [2019-12-17 12:30:00,356] jepsen nemesis - couchbase.util Rebalance stuck count: 104 INFO [2019-12-17 12:30:00,361] jepsen worker 27 - jepsen.util 2637 :ok :read [9 4] INFO [2019-12-17 12:30:00,361] jepsen worker 27 - jepsen.util 2637 :invoke :cas [9 [4 24]] INFO [2019-12-17 12:30:00,374] jepsen worker 79 - jepsen.util 2869 :ok :write [26 20] INFO [2019-12-17 12:30:00,374] jepsen worker 79 - jepsen.util 2869 :invoke :read [26 nil] INFO [2019-12-17 12:30:00,408] jepsen worker 49 - jepsen.util 3199 :info :write [16 13] :RequestCanceledException INFO [2019-12-17 12:30:00,409] jepsen worker 49 - jepsen.util 3289 :invoke :write [16 41] INFO [2019-12-17 12:30:00,423] jepsen worker 34 - jepsen.util 3994 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:30:00,423] jepsen worker 6 - jepsen.util 3966 :info :cas [2 [25 43]] :RequestCanceledException INFO [2019-12-17 12:30:00,423] jepsen worker 34 - jepsen.util 3994 :invoke :write [11 41] INFO [2019-12-17 12:30:00,424] jepsen worker 6 - jepsen.util 4056 :invoke :write [2 9] INFO [2019-12-17 12:30:00,437] jepsen worker 60 - jepsen.util 4110 :info :write [20 27] :RequestCanceledException INFO [2019-12-17 12:30:00,438] jepsen worker 60 - jepsen.util 4200 :invoke :read [20 nil] INFO [2019-12-17 12:30:00,440] jepsen worker 47 - jepsen.util 3377 :ok :read [15 3] INFO [2019-12-17 12:30:00,440] jepsen worker 47 - jepsen.util 3377 :invoke :read [15 nil] INFO [2019-12-17 12:30:00,443] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16918,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:30:00,444] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16918,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:30:00,444] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16922,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:30:00,444] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16922,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:30:00,448] jepsen worker 37 - jepsen.util 2647 :ok :write [12 38] INFO [2019-12-17 12:30:00,448] jepsen worker 37 - jepsen.util 2647 :invoke :write [12 25] INFO [2019-12-17 12:30:00,452] jepsen worker 78 - jepsen.util 3138 :ok :write [26 41] INFO [2019-12-17 12:30:00,452] jepsen worker 78 - jepsen.util 3138 :invoke :read [26 nil] INFO [2019-12-17 12:30:00,463] jepsen worker 47 - jepsen.util 3377 :ok :read [15 3] INFO [2019-12-17 12:30:00,463] jepsen worker 47 - jepsen.util 3377 :invoke :read [15 nil] INFO [2019-12-17 12:30:00,490] jepsen worker 74 - jepsen.util 2954 :info :write [24 16] :RequestCanceledException INFO [2019-12-17 12:30:00,491] jepsen worker 78 - jepsen.util 3138 :ok :read [26 41] INFO [2019-12-17 12:30:00,491] jepsen worker 74 - jepsen.util 3044 :invoke :cas [24 [11 25]] INFO [2019-12-17 12:30:00,491] jepsen worker 78 - jepsen.util 3138 :invoke :write [26 2] INFO [2019-12-17 12:30:00,492] jepsen worker 27 - jepsen.util 2637 :ok :cas [9 [4 24]] INFO [2019-12-17 12:30:00,493] jepsen worker 27 - jepsen.util 2637 :invoke :cas [9 [36 28]] INFO [2019-12-17 12:30:00,496] jepsen worker 47 - jepsen.util 3377 :ok :read [15 3] INFO [2019-12-17 12:30:00,496] jepsen worker 47 - jepsen.util 3377 :invoke :cas [15 [19 38]] INFO [2019-12-17 12:30:00,522] jepsen worker 27 - jepsen.util 2637 :fail :cas [9 [36 28]] :ValueNotSwapFrom INFO [2019-12-17 12:30:00,522] jepsen worker 27 - jepsen.util 2637 :invoke :read [9 nil] INFO [2019-12-17 12:30:00,523] jepsen worker 7 - jepsen.util 3607 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:30:00,523] jepsen worker 13 - jepsen.util 2713 :info :write [4 26] :RequestCanceledException INFO [2019-12-17 12:30:00,523] jepsen worker 7 - jepsen.util 3607 :invoke :read [2 nil] INFO [2019-12-17 12:30:00,524] jepsen worker 13 - jepsen.util 2803 :invoke :cas [4 [12 23]] INFO [2019-12-17 12:30:00,555] jepsen worker 87 - jepsen.util 2697 :ok :write [29 41] INFO [2019-12-17 12:30:00,555] jepsen worker 87 - jepsen.util 2697 :invoke :write [29 40] INFO [2019-12-17 12:30:00,587] jepsen worker 23 - jepsen.util 3803 :info :write [7 3] :RequestCanceledException INFO [2019-12-17 12:30:00,588] jepsen worker 23 - jepsen.util 3893 :invoke :read [7 nil] INFO [2019-12-17 12:30:00,598] jepsen worker 47 - jepsen.util 3377 :fail :cas [15 [19 38]] :ValueNotSwapFrom INFO [2019-12-17 12:30:00,599] jepsen worker 47 - jepsen.util 3377 :invoke :cas [15 [32 5]] INFO [2019-12-17 12:30:00,602] jepsen worker 74 - jepsen.util 3044 :fail :cas [24 [11 25]] :ValueNotSwapFrom INFO [2019-12-17 12:30:00,603] jepsen worker 74 - jepsen.util 3044 :invoke :write [24 39] INFO [2019-12-17 12:30:00,605] jepsen worker 30 - jepsen.util 3360 :info :write [10 35] :RequestCanceledException INFO [2019-12-17 12:30:00,606] jepsen worker 30 - jepsen.util 3450 :invoke :cas [10 [29 28]] INFO [2019-12-17 12:30:00,625] jepsen worker 27 - jepsen.util 2637 :ok :read [9 24] INFO [2019-12-17 12:30:00,626] jepsen worker 27 - jepsen.util 2637 :invoke :read [9 nil] INFO [2019-12-17 12:30:00,655] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16944,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:30:00,656] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16944,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:30:00,656] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16947,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:30:00,656] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16947,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:30:00,656] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16948,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:30:00,656] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16948,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:30:00,665] jepsen worker 47 - jepsen.util 3377 :fail :cas [15 [32 5]] :ValueNotSwapFrom INFO [2019-12-17 12:30:00,665] jepsen worker 47 - jepsen.util 3377 :invoke :cas [15 [24 11]] INFO [2019-12-17 12:30:00,675] jepsen worker 36 - jepsen.util 2826 :fail :cas [12 [30 44]] :ValueNotSwapFrom INFO [2019-12-17 12:30:00,676] jepsen worker 36 - jepsen.util 2826 :invoke :write [12 20] INFO [2019-12-17 12:30:00,690] jepsen worker 27 - jepsen.util 2637 :ok :read [9 24] INFO [2019-12-17 12:30:00,690] jepsen worker 27 - jepsen.util 2637 :invoke :cas [9 [16 28]] INFO [2019-12-17 12:30:00,699] jepsen worker 4 - jepsen.util 2164 :ok :write [1 43] INFO [2019-12-17 12:30:00,699] jepsen worker 4 - jepsen.util 2164 :invoke :read [1 nil] INFO [2019-12-17 12:30:00,713] jepsen worker 52 - jepsen.util 3202 :ok :write [17 16] INFO [2019-12-17 12:30:00,713] jepsen worker 52 - jepsen.util 3202 :invoke :read [17 nil] INFO [2019-12-17 12:30:00,723] jepsen worker 63 - jepsen.util 2493 :info :write [21 4] :RequestCanceledException INFO [2019-12-17 12:30:00,724] jepsen worker 63 - jepsen.util 2583 :invoke :write [21 13] INFO [2019-12-17 12:30:00,737] jepsen worker 27 - jepsen.util 2637 :fail :cas [9 [16 28]] :ValueNotSwapFrom INFO [2019-12-17 12:30:00,738] jepsen worker 27 - jepsen.util 2637 :invoke :cas [9 [15 28]] INFO [2019-12-17 12:30:00,738] jepsen worker 47 - jepsen.util 3377 :fail :cas [15 [24 11]] :ValueNotSwapFrom INFO [2019-12-17 12:30:00,738] jepsen worker 47 - jepsen.util 3377 :invoke :cas [15 [19 46]] INFO [2019-12-17 12:30:00,759] jepsen worker 27 - jepsen.util 2637 :fail :cas [9 [15 28]] :ValueNotSwapFrom INFO [2019-12-17 12:30:00,759] jepsen worker 27 - jepsen.util 2637 :invoke :read [9 nil] INFO [2019-12-17 12:30:00,766] jepsen worker 70 - jepsen.util 3400 :ok :write [23 33] INFO [2019-12-17 12:30:00,766] jepsen worker 70 - jepsen.util 3400 :invoke :read [23 nil] INFO [2019-12-17 12:30:00,775] jepsen worker 27 - jepsen.util 2637 :ok :read [9 24] INFO [2019-12-17 12:30:00,775] jepsen worker 27 - jepsen.util 2637 :invoke :write [9 9] INFO [2019-12-17 12:30:00,799] jepsen worker 54 - jepsen.util 3024 :fail :cas [18 [28 21]] :ValueNotSwapFrom INFO [2019-12-17 12:30:00,799] jepsen worker 54 - jepsen.util 3024 :invoke :read [18 nil] INFO [2019-12-17 12:30:00,814] jepsen worker 3 - jepsen.util 2793 :ok :write [1 12] INFO [2019-12-17 12:30:00,815] jepsen worker 3 - jepsen.util 2793 :invoke :write [1 4] INFO [2019-12-17 12:30:00,823] jepsen worker 65 - jepsen.util 2585 :info :cas [21 [5 35]] :RequestCanceledException INFO [2019-12-17 12:30:00,825] jepsen worker 65 - jepsen.util 2675 :invoke :write [21 21] INFO [2019-12-17 12:30:00,828] jepsen worker 63 - jepsen.util 2583 :ok :write [21 13] INFO [2019-12-17 12:30:00,828] jepsen worker 63 - jepsen.util 2583 :invoke :cas [21 [36 40]] INFO [2019-12-17 12:30:00,840] jepsen worker 14 - jepsen.util 2804 :ok :write [4 36] INFO [2019-12-17 12:30:00,840] jepsen worker 14 - jepsen.util 2804 :invoke :cas [4 [40 12]] INFO [2019-12-17 12:30:00,843] jepsen worker 51 - jepsen.util 2751 :fail :cas [17 [40 38]] :ValueNotSwapFrom INFO [2019-12-17 12:30:00,843] jepsen worker 51 - jepsen.util 2751 :invoke :read [17 nil] INFO [2019-12-17 12:30:00,848] jepsen worker 27 - jepsen.util 2637 :ok :write [9 9] INFO [2019-12-17 12:30:00,848] jepsen worker 27 - jepsen.util 2637 :invoke :cas [9 [7 9]] INFO [2019-12-17 12:30:00,867] jepsen worker 89 - jepsen.util 2699 :ok :read [29 41] INFO [2019-12-17 12:30:00,868] jepsen worker 89 - jepsen.util 2699 :invoke :cas [29 [10 46]] INFO [2019-12-17 12:30:00,881] jepsen worker 63 - jepsen.util 2583 :fail :cas [21 [36 40]] :ValueNotSwapFrom INFO [2019-12-17 12:30:00,881] jepsen worker 63 - jepsen.util 2583 :invoke :read [21 nil] INFO [2019-12-17 12:30:00,885] jepsen worker 27 - jepsen.util 2637 :fail :cas [9 [7 9]] :ValueNotSwapFrom INFO [2019-12-17 12:30:00,885] jepsen worker 27 - jepsen.util 2637 :invoke :write [9 6] INFO [2019-12-17 12:30:00,901] jepsen worker 63 - jepsen.util 2583 :ok :read [21 13] INFO [2019-12-17 12:30:00,901] jepsen worker 63 - jepsen.util 2583 :invoke :read [21 nil] INFO [2019-12-17 12:30:00,905] jepsen worker 79 - jepsen.util 2869 :ok :read [26 41] INFO [2019-12-17 12:30:00,905] jepsen worker 79 - jepsen.util 2869 :invoke :cas [26 [32 6]] INFO [2019-12-17 12:30:00,926] jepsen worker 63 - jepsen.util 2583 :ok :read [21 13] INFO [2019-12-17 12:30:00,926] jepsen worker 63 - jepsen.util 2583 :invoke :cas [21 [41 21]] INFO [2019-12-17 12:30:00,937] jepsen worker 46 - jepsen.util 3016 :info :write [15 28] :RequestCanceledException INFO [2019-12-17 12:30:00,938] jepsen worker 46 - jepsen.util 3106 :invoke :cas [15 [24 20]] INFO [2019-12-17 12:30:00,944] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16968,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:30:00,944] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16968,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:30:00,944] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16971,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:30:00,944] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16971,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:30:00,958] jepsen worker 88 - jepsen.util 2968 :ok :write [29 18] INFO [2019-12-17 12:30:00,958] jepsen worker 88 - jepsen.util 2968 :invoke :cas [29 [46 45]] INFO [2019-12-17 12:30:00,973] jepsen worker 63 - jepsen.util 2583 :fail :cas [21 [41 21]] :ValueNotSwapFrom INFO [2019-12-17 12:30:00,973] jepsen worker 63 - jepsen.util 2583 :invoke :read [21 nil] INFO [2019-12-17 12:30:00,990] jepsen worker 80 - jepsen.util 2780 :info :write [26 2] :RequestCanceledException INFO [2019-12-17 12:30:00,990] jepsen worker 35 - jepsen.util 3725 :info :cas [11 [29 25]] :RequestCanceledException INFO [2019-12-17 12:30:00,991] jepsen worker 80 - jepsen.util 2870 :invoke :write [26 27] INFO [2019-12-17 12:30:00,991] jepsen worker 35 - jepsen.util 3815 :invoke :cas [11 [43 18]] INFO [2019-12-17 12:30:00,995] jepsen worker 5 - jepsen.util 2525 :ok :write [1 3] INFO [2019-12-17 12:30:00,995] jepsen worker 5 - jepsen.util 2525 :invoke :cas [1 [5 29]] INFO [2019-12-17 12:30:01,008] jepsen worker 16 - jepsen.util 3976 :info :write [5 2] :RequestCanceledException INFO [2019-12-17 12:30:01,009] jepsen worker 16 - jepsen.util 4066 :invoke :cas [5 [5 27]] INFO [2019-12-17 12:30:01,028] jepsen worker 63 - jepsen.util 2583 :ok :read [21 13] INFO [2019-12-17 12:30:01,028] jepsen worker 63 - jepsen.util 2583 :invoke :cas [21 [26 41]] INFO [2019-12-17 12:30:01,032] jepsen worker 46 - jepsen.util 3106 :fail :cas [15 [24 20]] :ValueNotSwapFrom INFO [2019-12-17 12:30:01,032] jepsen worker 46 - jepsen.util 3106 :invoke :read [15 nil] INFO [2019-12-17 12:30:01,032] jepsen worker 70 - jepsen.util 3400 :ok :read [23 33] INFO [2019-12-17 12:30:01,033] jepsen worker 70 - jepsen.util 3400 :invoke :read [23 nil] INFO [2019-12-17 12:30:01,041] jepsen worker 37 - jepsen.util 2647 :ok :write [12 25] INFO [2019-12-17 12:30:01,041] jepsen worker 37 - jepsen.util 2647 :invoke :write [12 36] INFO [2019-12-17 12:30:01,052] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16983,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:30:01,052] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16983,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:30:01,052] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16987,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:30:01,052] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":16987,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:30:01,057] jepsen worker 63 - jepsen.util 2583 :fail :cas [21 [26 41]] :ValueNotSwapFrom INFO [2019-12-17 12:30:01,057] jepsen worker 63 - jepsen.util 2583 :invoke :read [21 nil] INFO [2019-12-17 12:30:01,061] jepsen worker 46 - jepsen.util 3106 :ok :read [15 3] INFO [2019-12-17 12:30:01,061] jepsen worker 46 - jepsen.util 3106 :invoke :write [15 13] INFO [2019-12-17 12:30:01,072] jepsen worker 13 - jepsen.util 2803 :fail :cas [4 [12 23]] :ValueNotSwapFrom INFO [2019-12-17 12:30:01,072] jepsen worker 13 - jepsen.util 2803 :invoke :write [4 2] INFO [2019-12-17 12:30:01,077] jepsen worker 5 - jepsen.util 2525 :fail :cas [1 [5 29]] :ValueNotSwapFrom INFO [2019-12-17 12:30:01,077] jepsen worker 5 - jepsen.util 2525 :invoke :write [1 20] INFO [2019-12-17 12:30:01,081] jepsen worker 27 - jepsen.util 2637 :ok :write [9 6] INFO [2019-12-17 12:30:01,081] jepsen worker 27 - jepsen.util 2637 :invoke :cas [9 [43 43]] INFO [2019-12-17 12:30:01,091] jepsen worker 63 - jepsen.util 2583 :ok :read [21 13] INFO [2019-12-17 12:30:01,091] jepsen worker 63 - jepsen.util 2583 :invoke :cas [21 [45 45]] INFO [2019-12-17 12:30:01,112] jepsen worker 70 - jepsen.util 3400 :ok :read [23 33] INFO [2019-12-17 12:30:01,113] jepsen worker 70 - jepsen.util 3400 :invoke :write [23 49] INFO [2019-12-17 12:30:01,116] jepsen worker 27 - jepsen.util 2637 :fail :cas [9 [43 43]] :ValueNotSwapFrom INFO [2019-12-17 12:30:01,116] jepsen worker 27 - jepsen.util 2637 :invoke :write [9 4] INFO [2019-12-17 12:30:01,123] jepsen worker 56 - jepsen.util 2756 :info :write [18 11] :RequestCanceledException INFO [2019-12-17 12:30:01,124] jepsen worker 56 - jepsen.util 2846 :invoke :write [18 31] INFO [2019-12-17 12:30:01,132] jepsen worker 73 - jepsen.util 2863 :ok :write [24 45] INFO [2019-12-17 12:30:01,132] jepsen worker 73 - jepsen.util 2863 :invoke :cas [24 [34 1]] INFO [2019-12-17 12:30:01,135] jepsen worker 59 - jepsen.util 3749 :info :write [19 34] :RequestCanceledException INFO [2019-12-17 12:30:01,137] jepsen worker 59 - jepsen.util 3839 :invoke :cas [19 [38 9]] INFO [2019-12-17 12:30:01,140] jepsen worker 11 - jepsen.util 3251 :ok :write [3 23] INFO [2019-12-17 12:30:01,140] jepsen worker 11 - jepsen.util 3251 :invoke :write [3 12] INFO [2019-12-17 12:30:01,148] jepsen worker 63 - jepsen.util 2583 :fail :cas [21 [45 45]] :ValueNotSwapFrom INFO [2019-12-17 12:30:01,149] jepsen worker 63 - jepsen.util 2583 :invoke :cas [21 [11 24]] INFO [2019-12-17 12:30:01,151] jepsen worker 43 - jepsen.util 3553 :info :write [14 17] :RequestCanceledException INFO [2019-12-17 12:30:01,152] jepsen worker 43 - jepsen.util 3643 :invoke :read [14 nil] INFO [2019-12-17 12:30:01,168] jepsen worker 52 - jepsen.util 3202 :ok :read [17 16] INFO [2019-12-17 12:30:01,168] jepsen worker 52 - jepsen.util 3202 :invoke :write [17 20] INFO [2019-12-17 12:30:01,174] jepsen worker 80 - jepsen.util 2870 :ok :write [26 27] INFO [2019-12-17 12:30:01,174] jepsen worker 80 - jepsen.util 2870 :invoke :cas [26 [14 24]] INFO [2019-12-17 12:30:01,187] jepsen worker 18 - jepsen.util 3078 :ok :write [6 36] INFO [2019-12-17 12:30:01,188] jepsen worker 18 - jepsen.util 3078 :invoke :cas [6 [18 20]] INFO [2019-12-17 12:30:01,190] jepsen worker 12 - jepsen.util 3072 :ok :write [4 29] INFO [2019-12-17 12:30:01,190] jepsen worker 12 - jepsen.util 3072 :invoke :read [4 nil] INFO [2019-12-17 12:30:01,196] jepsen worker 63 - jepsen.util 2583 :fail :cas [21 [11 24]] :ValueNotSwapFrom INFO [2019-12-17 12:30:01,196] jepsen worker 63 - jepsen.util 2583 :invoke :cas [21 [6 9]] INFO [2019-12-17 12:30:01,208] jepsen worker 73 - jepsen.util 2863 :fail :cas [24 [34 1]] :ValueNotSwapFrom INFO [2019-12-17 12:30:01,208] jepsen worker 15 - jepsen.util 3525 :info :write [5 39] :RequestCanceledException INFO [2019-12-17 12:30:01,208] jepsen worker 73 - jepsen.util 2863 :invoke :read [24 nil] INFO [2019-12-17 12:30:01,209] jepsen worker 15 - jepsen.util 3615 :invoke :read [5 nil] INFO [2019-12-17 12:30:01,212] jepsen worker 71 - jepsen.util 2951 :ok :write [23 14] INFO [2019-12-17 12:30:01,213] jepsen worker 71 - jepsen.util 2951 :invoke :read [23 nil] INFO [2019-12-17 12:30:01,216] jepsen worker 30 - jepsen.util 3450 :fail :cas [10 [29 28]] :ValueNotSwapFrom INFO [2019-12-17 12:30:01,216] jepsen worker 30 - jepsen.util 3450 :invoke :write [10 37] INFO [2019-12-17 12:30:01,229] jepsen worker 85 - jepsen.util 4405 :info :cas [28 [32 10]] :RequestCanceledException INFO [2019-12-17 12:30:01,230] jepsen worker 85 - jepsen.util 4495 :invoke :read [28 nil] INFO [2019-12-17 12:30:01,239] jepsen worker 53 - jepsen.util 2933 :ok :read [17 16] INFO [2019-12-17 12:30:01,239] jepsen worker 53 - jepsen.util 2933 :invoke :cas [17 [24 47]] INFO [2019-12-17 12:30:01,255] jepsen worker 84 - jepsen.util 3684 :info :write [28 9] :RequestCanceledException INFO [2019-12-17 12:30:01,256] jepsen worker 84 - jepsen.util 3774 :invoke :write [28 0] INFO [2019-12-17 12:30:01,266] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":17015,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:30:01,266] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":17015,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:30:01,266] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":17016,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:30:01,266] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":17016,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:30:01,270] jepsen worker 18 - jepsen.util 3078 :fail :cas [6 [18 20]] :ValueNotSwapFrom INFO [2019-12-17 12:30:01,270] jepsen worker 18 - jepsen.util 3078 :invoke :write [6 2] INFO [2019-12-17 12:30:01,275] jepsen worker 89 - jepsen.util 2699 :fail :cas [29 [10 46]] :ValueNotSwapFrom INFO [2019-12-17 12:30:01,275] jepsen worker 89 - jepsen.util 2699 :invoke :cas [29 [25 18]] INFO [2019-12-17 12:30:01,287] jepsen worker 62 - jepsen.util 3662 :info :cas [20 [24 9]] :RequestCanceledException INFO [2019-12-17 12:30:01,288] jepsen worker 51 - jepsen.util 2751 :ok :read [17 16] INFO [2019-12-17 12:30:01,288] jepsen worker 51 - jepsen.util 2751 :invoke :write [17 27] INFO [2019-12-17 12:30:01,289] jepsen worker 62 - jepsen.util 3752 :invoke :cas [20 [47 44]] INFO [2019-12-17 12:30:01,305] jepsen worker 63 - jepsen.util 2583 :fail :cas [21 [6 9]] :ValueNotSwapFrom INFO [2019-12-17 12:30:01,305] jepsen worker 63 - jepsen.util 2583 :invoke :read [21 nil] INFO [2019-12-17 12:30:01,311] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":17023,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:30:01,312] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":17023,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:30:01,312] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":17026,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:30:01,312] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":17026,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:30:01,313] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":17027,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:30:01,313] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":17027,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:30:01,317] jepsen worker 79 - jepsen.util 2869 :fail :cas [26 [32 6]] :ValueNotSwapFrom INFO [2019-12-17 12:30:01,318] jepsen worker 79 - jepsen.util 2869 :invoke :cas [26 [1 21]] INFO [2019-12-17 12:30:01,324] jepsen worker 50 - jepsen.util 4280 :info :write [16 13] :RequestCanceledException INFO [2019-12-17 12:30:01,325] jepsen worker 50 - jepsen.util 4370 :invoke :cas [16 [15 24]] INFO [2019-12-17 12:30:01,326] jepsen worker 73 - jepsen.util 2863 :ok :read [24 45] INFO [2019-12-17 12:30:01,326] jepsen worker 73 - jepsen.util 2863 :invoke :read [24 nil] INFO [2019-12-17 12:30:01,332] jepsen worker 27 - jepsen.util 2637 :ok :write [9 4] INFO [2019-12-17 12:30:01,332] jepsen worker 27 - jepsen.util 2637 :invoke :read [9 nil] INFO [2019-12-17 12:30:01,334] jepsen worker 13 - jepsen.util 2803 :ok :write [4 2] INFO [2019-12-17 12:30:01,334] jepsen worker 13 - jepsen.util 2803 :invoke :read [4 nil] INFO [2019-12-17 12:30:01,335] jepsen worker 63 - jepsen.util 2583 :ok :read [21 13] INFO [2019-12-17 12:30:01,335] jepsen worker 63 - jepsen.util 2583 :invoke :cas [21 [2 42]] INFO [2019-12-17 12:30:01,342] jepsen worker 5 - jepsen.util 2525 :ok :write [1 20] INFO [2019-12-17 12:30:01,342] jepsen worker 5 - jepsen.util 2525 :invoke :write [1 35] INFO [2019-12-17 12:30:01,358] jepsen worker 27 - jepsen.util 2637 :ok :read [9 4] INFO [2019-12-17 12:30:01,358] jepsen worker 27 - jepsen.util 2637 :invoke :cas [9 [38 19]] INFO [2019-12-17 12:30:01,371] jepsen worker 4 - jepsen.util 2164 :ok :read [1 20] INFO [2019-12-17 12:30:01,371] jepsen worker 4 - jepsen.util 2164 :invoke :cas [1 [25 37]] INFO [2019-12-17 12:30:01,385] jepsen worker 73 - jepsen.util 2863 :ok :read [24 45] INFO [2019-12-17 12:30:01,385] jepsen worker 73 - jepsen.util 2863 :invoke :cas [24 [40 37]] INFO [2019-12-17 12:30:01,386] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:30:01,386] jepsen nemesis - couchbase.util Rebalance check count: 124 INFO [2019-12-17 12:30:01,386] jepsen nemesis - couchbase.util Rebalance stuck count: 105 INFO [2019-12-17 12:30:01,389] jepsen worker 27 - jepsen.util 2637 :fail :cas [9 [38 19]] :ValueNotSwapFrom INFO [2019-12-17 12:30:01,390] jepsen worker 27 - jepsen.util 2637 :invoke :write [9 8] INFO [2019-12-17 12:30:01,404] jepsen worker 63 - jepsen.util 2583 :fail :cas [21 [2 42]] :ValueNotSwapFrom INFO [2019-12-17 12:30:01,404] jepsen worker 63 - jepsen.util 2583 :invoke :write [21 26] INFO [2019-12-17 12:30:01,408] jepsen worker 10 - jepsen.util 2710 :info :write [3 16] :RequestCanceledException INFO [2019-12-17 12:30:01,409] jepsen worker 10 - jepsen.util 2800 :invoke :read [3 nil] INFO [2019-12-17 12:30:01,423] jepsen worker 75 - jepsen.util 3945 :info :write [25 27] :RequestCanceledException INFO [2019-12-17 12:30:01,423] jepsen worker 1 - jepsen.util 3331 :info :write [0 33] :RequestCanceledException INFO [2019-12-17 12:30:01,424] jepsen worker 75 - jepsen.util 4035 :invoke :write [25 29] INFO [2019-12-17 12:30:01,424] jepsen worker 1 - jepsen.util 3421 :invoke :cas [0 [20 0]] INFO [2019-12-17 12:30:01,428] jepsen worker 14 - jepsen.util 2804 :fail :cas [4 [40 12]] :ValueNotSwapFrom INFO [2019-12-17 12:30:01,428] jepsen worker 14 - jepsen.util 2804 :invoke :write [4 34] INFO [2019-12-17 12:30:01,429] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":17055,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:30:01,429] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":17055,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:30:01,434] jepsen worker 52 - jepsen.util 3202 :ok :write [17 20] INFO [2019-12-17 12:30:01,434] jepsen worker 52 - jepsen.util 3202 :invoke :read [17 nil] INFO [2019-12-17 12:30:01,461] jepsen worker 73 - jepsen.util 2863 :fail :cas [24 [40 37]] :ValueNotSwapFrom INFO [2019-12-17 12:30:01,461] jepsen worker 73 - jepsen.util 2863 :invoke :read [24 nil] INFO [2019-12-17 12:30:01,475] jepsen worker 89 - jepsen.util 2699 :fail :cas [29 [25 18]] :ValueNotSwapFrom INFO [2019-12-17 12:30:01,475] jepsen worker 89 - jepsen.util 2699 :invoke :read [29 nil] INFO [2019-12-17 12:30:01,480] jepsen worker 88 - jepsen.util 2968 :fail :cas [29 [46 45]] :ValueNotSwapFrom INFO [2019-12-17 12:30:01,480] jepsen worker 88 - jepsen.util 2968 :invoke :write [29 5] INFO [2019-12-17 12:30:01,490] jepsen worker 40 - jepsen.util 4090 :info :write [13 40] :RequestCanceledException INFO [2019-12-17 12:30:01,491] jepsen worker 40 - jepsen.util 4180 :invoke :write [13 20] INFO [2019-12-17 12:30:01,503] jepsen worker 52 - jepsen.util 3202 :ok :read [17 20] INFO [2019-12-17 12:30:01,503] jepsen worker 52 - jepsen.util 3202 :invoke :read [17 nil] INFO [2019-12-17 12:30:01,507] jepsen worker 64 - jepsen.util 2314 :info :cas [21 [8 41]] :RequestCanceledException INFO [2019-12-17 12:30:01,508] jepsen worker 64 - jepsen.util 2404 :invoke :read [21 nil] INFO [2019-12-17 12:30:01,523] jepsen worker 41 - jepsen.util 4001 :info :cas [13 [48 39]] :RequestCanceledException INFO [2019-12-17 12:30:01,524] jepsen worker 41 - jepsen.util 4091 :invoke :write [13 25] INFO [2019-12-17 12:30:01,524] jepsen worker 21 - jepsen.util 3801 :info :write [7 41] :RequestCanceledException INFO [2019-12-17 12:30:01,524] jepsen worker 57 - jepsen.util 3567 :info :write [19 24] :RequestCanceledException INFO [2019-12-17 12:30:01,525] jepsen worker 57 - jepsen.util 3657 :invoke :write [19 9] INFO [2019-12-17 12:30:01,525] jepsen worker 21 - jepsen.util 3891 :invoke :cas [7 [8 13]] INFO [2019-12-17 12:30:01,531] jepsen worker 73 - jepsen.util 2863 :ok :read [24 45] INFO [2019-12-17 12:30:01,531] jepsen worker 73 - jepsen.util 2863 :invoke :write [24 18] INFO [2019-12-17 12:30:01,537] jepsen worker 22 - jepsen.util 3892 :info :write [7 21] :RequestCanceledException INFO [2019-12-17 12:30:01,538] jepsen worker 22 - jepsen.util 3982 :invoke :write [7 39] INFO [2019-12-17 12:30:01,554] jepsen worker 27 - jepsen.util 2637 :ok :write [9 8] INFO [2019-12-17 12:30:01,554] jepsen worker 27 - jepsen.util 2637 :invoke :read [9 nil] INFO [2019-12-17 12:30:01,563] jepsen worker 63 - jepsen.util 2583 :ok :write [21 26] INFO [2019-12-17 12:30:01,563] jepsen worker 63 - jepsen.util 2583 :invoke :read [21 nil] INFO [2019-12-17 12:30:01,584] jepsen worker 89 - jepsen.util 2699 :ok :read [29 18] INFO [2019-12-17 12:30:01,585] jepsen worker 89 - jepsen.util 2699 :invoke :read [29 nil] INFO [2019-12-17 12:30:01,590] jepsen worker 8 - jepsen.util 3338 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:30:01,590] jepsen worker 8 - jepsen.util 3338 :invoke :read [2 nil] INFO [2019-12-17 12:30:01,590] jepsen worker 27 - jepsen.util 2637 :ok :read [9 8] INFO [2019-12-17 12:30:01,591] jepsen worker 27 - jepsen.util 2637 :invoke :read [9 nil] INFO [2019-12-17 12:30:01,599] jepsen worker 79 - jepsen.util 2869 :fail :cas [26 [1 21]] :ValueNotSwapFrom INFO [2019-12-17 12:30:01,599] jepsen worker 79 - jepsen.util 2869 :invoke :read [26 nil] INFO [2019-12-17 12:30:01,607] jepsen worker 64 - jepsen.util 2404 :ok :read [21 26] INFO [2019-12-17 12:30:01,607] jepsen worker 64 - jepsen.util 2404 :invoke :cas [21 [13 27]] INFO [2019-12-17 12:30:01,608] jepsen worker 83 - jepsen.util 4043 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:30:01,608] jepsen worker 33 - jepsen.util 4173 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:30:01,608] jepsen worker 83 - jepsen.util 4043 :invoke :write [27 11] INFO [2019-12-17 12:30:01,608] jepsen worker 33 - jepsen.util 4173 :invoke :write [11 42] INFO [2019-12-17 12:30:01,624] jepsen worker 42 - jepsen.util 4452 :info :write [14 1] :RequestCanceledException INFO [2019-12-17 12:30:01,624] jepsen worker 48 - jepsen.util 3288 :info :write [16 48] :RequestCanceledException INFO [2019-12-17 12:30:01,625] jepsen worker 48 - jepsen.util 3378 :invoke :write [16 41] INFO [2019-12-17 12:30:01,625] jepsen worker 42 - jepsen.util 4542 :invoke :write [14 24] INFO [2019-12-17 12:30:01,633] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":17085,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:30:01,633] jepsen worker 0 - jepsen.util 4230 :info :cas [0 [18 12]] :RequestCanceledException INFO [2019-12-17 12:30:01,633] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":17085,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:30:01,633] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":17088,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:30:01,633] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":17088,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:30:01,634] jepsen worker 0 - jepsen.util 4320 :invoke :write [0 48] INFO [2019-12-17 12:30:01,636] jepsen worker 89 - jepsen.util 2699 :ok :read [29 18] INFO [2019-12-17 12:30:01,636] jepsen worker 89 - jepsen.util 2699 :invoke :write [29 42] INFO [2019-12-17 12:30:01,654] jepsen worker 26 - jepsen.util 4076 :info :write [8 30] :RequestCanceledException INFO [2019-12-17 12:30:01,654] jepsen worker 69 - jepsen.util 2769 :info :write [23 22] :RequestCanceledException INFO [2019-12-17 12:30:01,655] jepsen worker 69 - jepsen.util 2859 :invoke :write [23 33] INFO [2019-12-17 12:30:01,655] jepsen worker 26 - jepsen.util 4166 :invoke :write [8 1] INFO [2019-12-17 12:30:01,666] jepsen worker 27 - jepsen.util 2637 :ok :read [9 8] INFO [2019-12-17 12:30:01,667] jepsen worker 27 - jepsen.util 2637 :invoke :cas [9 [36 47]] INFO [2019-12-17 12:30:01,668] jepsen worker 63 - jepsen.util 2583 :ok :read [21 26] INFO [2019-12-17 12:30:01,668] jepsen worker 63 - jepsen.util 2583 :invoke :read [21 nil] INFO [2019-12-17 12:30:01,681] jepsen worker 52 - jepsen.util 3202 :ok :read [17 20] INFO [2019-12-17 12:30:01,681] jepsen worker 52 - jepsen.util 3202 :invoke :read [17 nil] INFO [2019-12-17 12:30:01,690] jepsen worker 17 - jepsen.util 3887 :info :cas [5 [2 6]] :RequestCanceledException INFO [2019-12-17 12:30:01,691] jepsen worker 17 - jepsen.util 3977 :invoke :read [5 nil] INFO [2019-12-17 12:30:01,717] jepsen worker 27 - jepsen.util 2637 :fail :cas [9 [36 47]] :ValueNotSwapFrom INFO [2019-12-17 12:30:01,717] jepsen worker 27 - jepsen.util 2637 :invoke :write [9 6] INFO [2019-12-17 12:30:01,717] jepsen worker 64 - jepsen.util 2404 :fail :cas [21 [13 27]] :ValueNotSwapFrom INFO [2019-12-17 12:30:01,717] jepsen worker 64 - jepsen.util 2404 :invoke :cas [21 [42 27]] INFO [2019-12-17 12:30:01,733] jepsen worker 63 - jepsen.util 2583 :ok :read [21 26] INFO [2019-12-17 12:30:01,734] jepsen worker 63 - jepsen.util 2583 :invoke :cas [21 [13 15]] INFO [2019-12-17 12:30:01,736] jepsen worker 51 - jepsen.util 2751 :ok :write [17 27] INFO [2019-12-17 12:30:01,737] jepsen worker 51 - jepsen.util 2751 :invoke :cas [17 [10 8]] INFO [2019-12-17 12:30:01,752] jepsen worker 55 - jepsen.util 3475 :info :write [18 44] :RequestCanceledException INFO [2019-12-17 12:30:01,753] jepsen worker 55 - jepsen.util 3565 :invoke :write [18 29] INFO [2019-12-17 12:30:01,753] jepsen worker 64 - jepsen.util 2404 :fail :cas [21 [42 27]] :ValueNotSwapFrom INFO [2019-12-17 12:30:01,753] jepsen worker 64 - jepsen.util 2404 :invoke :cas [21 [17 49]] INFO [2019-12-17 12:30:01,758] jepsen worker 89 - jepsen.util 2699 :ok :write [29 42] INFO [2019-12-17 12:30:01,758] jepsen worker 89 - jepsen.util 2699 :invoke :cas [29 [15 41]] INFO [2019-12-17 12:30:01,770] jepsen worker 63 - jepsen.util 2583 :fail :cas [21 [13 15]] :ValueNotSwapFrom INFO [2019-12-17 12:30:01,770] jepsen worker 63 - jepsen.util 2583 :invoke :read [21 nil] INFO [2019-12-17 12:30:01,780] jepsen worker 52 - jepsen.util 3202 :ok :read [17 27] INFO [2019-12-17 12:30:01,780] jepsen worker 52 - jepsen.util 3202 :invoke :read [17 nil] INFO [2019-12-17 12:30:01,788] jepsen worker 64 - jepsen.util 2404 :fail :cas [21 [17 49]] :ValueNotSwapFrom INFO [2019-12-17 12:30:01,788] jepsen worker 64 - jepsen.util 2404 :invoke :write [21 24] INFO [2019-12-17 12:30:01,795] jepsen worker 79 - jepsen.util 2869 :ok :read [26 27] INFO [2019-12-17 12:30:01,795] jepsen worker 79 - jepsen.util 2869 :invoke :read [26 nil] INFO [2019-12-17 12:30:01,818] jepsen worker 31 - jepsen.util 2641 :ok :write [10 44] INFO [2019-12-17 12:30:01,818] jepsen worker 31 - jepsen.util 2641 :invoke :cas [10 [44 45]] INFO [2019-12-17 12:30:01,819] jepsen worker 63 - jepsen.util 2583 :ok :read [21 26] INFO [2019-12-17 12:30:01,819] jepsen worker 63 - jepsen.util 2583 :invoke :write [21 26] INFO [2019-12-17 12:30:01,837] jepsen worker 52 - jepsen.util 3202 :ok :read [17 27] INFO [2019-12-17 12:30:01,837] jepsen worker 52 - jepsen.util 3202 :invoke :cas [17 [18 40]] INFO [2019-12-17 12:30:01,839] jepsen worker 27 - jepsen.util 2637 :ok :write [9 6] INFO [2019-12-17 12:30:01,840] jepsen worker 27 - jepsen.util 2637 :invoke :cas [9 [38 22]] INFO [2019-12-17 12:30:01,841] jepsen worker 70 - jepsen.util 3400 :ok :write [23 49] INFO [2019-12-17 12:30:01,841] jepsen worker 70 - jepsen.util 3400 :invoke :write [23 43] INFO [2019-12-17 12:30:01,867] jepsen worker 27 - jepsen.util 2637 :fail :cas [9 [38 22]] :ValueNotSwapFrom INFO [2019-12-17 12:30:01,867] jepsen worker 27 - jepsen.util 2637 :invoke :read [9 nil] INFO [2019-12-17 12:30:01,882] jepsen worker 65 - jepsen.util 2675 :ok :write [21 21] INFO [2019-12-17 12:30:01,882] jepsen worker 89 - jepsen.util 2699 :fail :cas [29 [15 41]] :ValueNotSwapFrom INFO [2019-12-17 12:30:01,883] jepsen worker 89 - jepsen.util 2699 :invoke :cas [29 [2 44]] INFO [2019-12-17 12:30:01,883] jepsen worker 65 - jepsen.util 2675 :invoke :cas [21 [47 29]] INFO [2019-12-17 12:30:01,887] jepsen worker 37 - jepsen.util 2647 :ok :write [12 36] INFO [2019-12-17 12:30:01,887] jepsen worker 37 - jepsen.util 2647 :invoke :read [12 nil] INFO [2019-12-17 12:30:01,907] jepsen worker 27 - jepsen.util 2637 :ok :read [9 6] INFO [2019-12-17 12:30:01,907] jepsen worker 27 - jepsen.util 2637 :invoke :write [9 13] INFO [2019-12-17 12:30:01,923] jepsen worker 72 - jepsen.util 2862 :info :write [24 48] :RequestCanceledException INFO [2019-12-17 12:30:01,924] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":17115,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:30:01,924] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":17115,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:30:01,924] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":17119,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:30:01,924] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":17119,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:30:01,925] jepsen worker 72 - jepsen.util 2952 :invoke :write [24 47] INFO [2019-12-17 12:30:01,966] jepsen worker 71 - jepsen.util 2951 :ok :read [23 49] INFO [2019-12-17 12:30:01,966] jepsen worker 71 - jepsen.util 2951 :invoke :write [23 30] INFO [2019-12-17 12:30:01,984] jepsen worker 65 - jepsen.util 2675 :fail :cas [21 [47 29]] :ValueNotSwapFrom INFO [2019-12-17 12:30:01,984] jepsen worker 65 - jepsen.util 2675 :invoke :write [21 5] INFO [2019-12-17 12:30:02,004] jepsen worker 64 - jepsen.util 2404 :ok :write [21 24] INFO [2019-12-17 12:30:02,004] jepsen worker 64 - jepsen.util 2404 :invoke :write [21 5] INFO [2019-12-17 12:30:02,024] jepsen worker 45 - jepsen.util 3015 :fail :read [15 nil] :RequestTimeoutException INFO [2019-12-17 12:30:02,024] jepsen worker 45 - jepsen.util 3015 :invoke :write [15 38] INFO [2019-12-17 12:30:02,032] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":17130,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:30:02,032] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":17130,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:30:02,032] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":17133,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:30:02,033] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":17133,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:30:02,033] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":17134,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:30:02,033] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":17134,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:30:02,042] jepsen worker 80 - jepsen.util 2870 :fail :cas [26 [14 24]] :ValueNotSwapFrom INFO [2019-12-17 12:30:02,042] jepsen worker 80 - jepsen.util 2870 :invoke :cas [26 [49 34]] INFO [2019-12-17 12:30:02,046] jepsen worker 46 - jepsen.util 3106 :ok :write [15 13] INFO [2019-12-17 12:30:02,047] jepsen worker 46 - jepsen.util 3106 :invoke :write [15 49] INFO [2019-12-17 12:30:02,050] jepsen worker 25 - jepsen.util 4345 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:30:02,050] jepsen worker 25 - jepsen.util 4345 :invoke :write [8 22] INFO [2019-12-17 12:30:02,060] jepsen worker 38 - jepsen.util 2828 :fail :cas [12 [22 16]] :ValueNotSwapFrom INFO [2019-12-17 12:30:02,060] jepsen worker 38 - jepsen.util 2828 :invoke :write [12 23] INFO [2019-12-17 12:30:02,061] jepsen worker 79 - jepsen.util 2869 :ok :read [26 27] INFO [2019-12-17 12:30:02,061] jepsen worker 79 - jepsen.util 2869 :invoke :write [26 10] INFO [2019-12-17 12:30:02,072] jepsen worker 10 - jepsen.util 2800 :ok :read [3 23] INFO [2019-12-17 12:30:02,072] jepsen worker 10 - jepsen.util 2800 :invoke :cas [3 [4 18]] INFO [2019-12-17 12:30:02,084] jepsen worker 27 - jepsen.util 2637 :ok :write [9 13] INFO [2019-12-17 12:30:02,084] jepsen worker 27 - jepsen.util 2637 :invoke :cas [9 [29 30]] INFO [2019-12-17 12:30:02,090] jepsen worker 9 - jepsen.util 3249 :info :write [3 42] :RequestCanceledException INFO [2019-12-17 12:30:02,091] jepsen worker 81 - jepsen.util 3861 :info :write [27 4] :RequestCanceledException INFO [2019-12-17 12:30:02,092] jepsen worker 9 - jepsen.util 3339 :invoke :write [3 27] INFO [2019-12-17 12:30:02,092] jepsen worker 81 - jepsen.util 3951 :invoke :write [27 31] INFO [2019-12-17 12:30:02,105] jepsen worker 76 - jepsen.util 3946 :info :cas [25 [18 16]] :RequestCanceledException INFO [2019-12-17 12:30:02,107] jepsen worker 76 - jepsen.util 4036 :invoke :cas [25 [37 26]] INFO [2019-12-17 12:30:02,110] jepsen worker 27 - jepsen.util 2637 :fail :cas [9 [29 30]] :ValueNotSwapFrom INFO [2019-12-17 12:30:02,111] jepsen worker 27 - jepsen.util 2637 :invoke :write [9 3] INFO [2019-12-17 12:30:02,112] jepsen worker 52 - jepsen.util 3202 :fail :cas [17 [18 40]] :ValueNotSwapFrom INFO [2019-12-17 12:30:02,113] jepsen worker 52 - jepsen.util 3202 :invoke :cas [17 [32 35]] INFO [2019-12-17 12:30:02,114] jepsen worker 63 - jepsen.util 2583 :ok :write [21 26] INFO [2019-12-17 12:30:02,114] jepsen worker 63 - jepsen.util 2583 :invoke :read [21 nil] INFO [2019-12-17 12:30:02,123] jepsen worker 28 - jepsen.util 2188 :fail :read [9 nil] :RequestTimeoutException INFO [2019-12-17 12:30:02,123] jepsen worker 28 - jepsen.util 2188 :invoke :cas [9 [19 12]] INFO [2019-12-17 12:30:02,124] jepsen worker 20 - jepsen.util 2540 :info :write [6 27] :RequestCanceledException INFO [2019-12-17 12:30:02,124] jepsen worker 58 - jepsen.util 3838 :info :write [19 27] :RequestCanceledException INFO [2019-12-17 12:30:02,125] jepsen worker 58 - jepsen.util 3928 :invoke :read [19 nil] INFO [2019-12-17 12:30:02,125] jepsen worker 20 - jepsen.util 2630 :invoke :write [6 38] INFO [2019-12-17 12:30:02,126] jepsen worker 5 - jepsen.util 2525 :ok :write [1 35] INFO [2019-12-17 12:30:02,127] jepsen worker 5 - jepsen.util 2525 :invoke :cas [1 [29 27]] INFO [2019-12-17 12:30:02,132] jepsen worker 89 - jepsen.util 2699 :fail :cas [29 [2 44]] :ValueNotSwapFrom INFO [2019-12-17 12:30:02,132] jepsen worker 89 - jepsen.util 2699 :invoke :write [29 17] INFO [2019-12-17 12:30:02,138] jepsen worker 82 - jepsen.util 4672 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:30:02,138] jepsen worker 82 - jepsen.util 4672 :invoke :write [27 17] INFO [2019-12-17 12:30:02,149] jepsen worker 63 - jepsen.util 2583 :ok :read [21 26] INFO [2019-12-17 12:30:02,150] jepsen worker 63 - jepsen.util 2583 :invoke :cas [21 [1 36]] INFO [2019-12-17 12:30:02,150] jepsen worker 32 - jepsen.util 3362 :info :cas [10 [44 45]] :RequestCanceledException INFO [2019-12-17 12:30:02,151] jepsen worker 32 - jepsen.util 3452 :invoke :write [10 49] INFO [2019-12-17 12:30:02,158] jepsen worker 73 - jepsen.util 2863 :ok :write [24 18] INFO [2019-12-17 12:30:02,158] jepsen worker 73 - jepsen.util 2863 :invoke :read [24 nil] INFO [2019-12-17 12:30:02,173] jepsen worker 28 - jepsen.util 2188 :fail :cas [9 [19 12]] :ValueNotSwapFrom INFO [2019-12-17 12:30:02,173] jepsen worker 28 - jepsen.util 2188 :invoke :write [9 24] INFO [2019-12-17 12:30:02,182] jepsen worker 63 - jepsen.util 2583 :fail :cas [21 [1 36]] :ValueNotSwapFrom INFO [2019-12-17 12:30:02,182] jepsen worker 63 - jepsen.util 2583 :invoke :cas [21 [0 1]] INFO [2019-12-17 12:30:02,190] jepsen worker 86 - jepsen.util 4046 :info :write [28 19] :RequestCanceledException INFO [2019-12-17 12:30:02,190] jepsen worker 66 - jepsen.util 3936 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:30:02,190] jepsen worker 66 - jepsen.util 3936 :invoke :read [22 nil] INFO [2019-12-17 12:30:02,191] jepsen worker 86 - jepsen.util 4136 :invoke :cas [28 [18 13]] INFO [2019-12-17 12:30:02,208] jepsen worker 68 - jepsen.util 3578 :info :cas [22 [23 1]] :RequestCanceledException INFO [2019-12-17 12:30:02,209] jepsen worker 68 - jepsen.util 3668 :invoke :write [22 18] INFO [2019-12-17 12:30:02,221] jepsen worker 77 - jepsen.util 3587 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:30:02,221] jepsen worker 39 - jepsen.util 3729 :info :write [13 30] :RequestCanceledException INFO [2019-12-17 12:30:02,222] jepsen worker 29 - jepsen.util 2459 :fail :read [9 nil] :RequestTimeoutException INFO [2019-12-17 12:30:02,222] jepsen worker 61 - jepsen.util 4381 :info :write [20 3] :RequestCanceledException INFO [2019-12-17 12:30:02,222] jepsen worker 77 - jepsen.util 3587 :invoke :write [25 49] INFO [2019-12-17 12:30:02,222] jepsen worker 19 - jepsen.util 3079 :info :cas [6 [39 24]] :RequestCanceledException INFO [2019-12-17 12:30:02,222] jepsen worker 14 - jepsen.util 2804 :ok :write [4 34] INFO [2019-12-17 12:30:02,222] jepsen worker 29 - jepsen.util 2459 :invoke :read [9 nil] INFO [2019-12-17 12:30:02,223] jepsen worker 39 - jepsen.util 3819 :invoke :write [13 49] INFO [2019-12-17 12:30:02,223] jepsen worker 14 - jepsen.util 2804 :invoke :cas [4 [41 1]] INFO [2019-12-17 12:30:02,223] jepsen worker 61 - jepsen.util 4471 :invoke :read [20 nil] INFO [2019-12-17 12:30:02,224] jepsen worker 19 - jepsen.util 3169 :invoke :read [6 nil] INFO [2019-12-17 12:30:02,225] jepsen worker 87 - jepsen.util 2697 :ok :write [29 40] INFO [2019-12-17 12:30:02,225] jepsen worker 87 - jepsen.util 2697 :invoke :read [29 nil] INFO [2019-12-17 12:30:02,236] jepsen worker 2 - jepsen.util 3512 :info :write [0 0] :RequestCanceledException INFO [2019-12-17 12:30:02,237] jepsen worker 2 - jepsen.util 3602 :invoke :cas [0 [18 8]] INFO [2019-12-17 12:30:02,243] jepsen worker 73 - jepsen.util 2863 :ok :read [24 18] INFO [2019-12-17 12:30:02,243] jepsen worker 73 - jepsen.util 2863 :invoke :write [24 20] INFO [2019-12-17 12:30:02,251] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":17162,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:30:02,251] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":17162,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:30:02,251] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":17163,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:30:02,251] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":17163,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:30:02,251] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":17164,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:30:02,251] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":17164,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:30:02,259] jepsen worker 13 - jepsen.util 2803 :ok :read [4 34] INFO [2019-12-17 12:30:02,259] jepsen worker 13 - jepsen.util 2803 :invoke :cas [4 [11 37]] INFO [2019-12-17 12:30:02,271] jepsen worker 51 - jepsen.util 2751 :fail :cas [17 [10 8]] :ValueNotSwapFrom INFO [2019-12-17 12:30:02,271] jepsen worker 51 - jepsen.util 2751 :invoke :cas [17 [0 23]] INFO [2019-12-17 12:30:02,274] jepsen worker 27 - jepsen.util 2637 :ok :write [9 3] INFO [2019-12-17 12:30:02,275] jepsen worker 27 - jepsen.util 2637 :invoke :read [9 nil] INFO [2019-12-17 12:30:02,282] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":17172,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:30:02,282] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":17172,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:30:02,282] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":17174,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:30:02,283] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":17174,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:30:02,292] jepsen worker 4 - jepsen.util 2164 :fail :cas [1 [25 37]] :ValueNotSwapFrom INFO [2019-12-17 12:30:02,292] jepsen worker 4 - jepsen.util 2164 :invoke :read [1 nil] INFO [2019-12-17 12:30:02,306] jepsen worker 27 - jepsen.util 2637 :ok :read [9 3] INFO [2019-12-17 12:30:02,306] jepsen worker 27 - jepsen.util 2637 :invoke :read [9 nil] INFO [2019-12-17 12:30:02,324] jepsen worker 24 - jepsen.util 3714 :info :cas [8 [47 22]] :RequestCanceledException INFO [2019-12-17 12:30:02,325] jepsen worker 24 - jepsen.util 3804 :invoke :write [8 13] INFO [2019-12-17 12:30:02,328] jepsen worker 27 - jepsen.util 2637 :ok :read [9 3] INFO [2019-12-17 12:30:02,328] jepsen worker 27 - jepsen.util 2637 :invoke :cas [9 [12 28]] INFO [2019-12-17 12:30:02,337] jepsen worker 67 - jepsen.util 4387 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:30:02,337] jepsen worker 67 - jepsen.util 4387 :invoke :write [22 7] INFO [2019-12-17 12:30:02,346] jepsen worker 31 - jepsen.util 2641 :ok :cas [10 [44 45]] INFO [2019-12-17 12:30:02,346] jepsen worker 31 - jepsen.util 2641 :invoke :read [10 nil] INFO [2019-12-17 12:30:02,352] jepsen worker 27 - jepsen.util 2637 :fail :cas [9 [12 28]] :ValueNotSwapFrom INFO [2019-12-17 12:30:02,352] jepsen worker 27 - jepsen.util 2637 :invoke :read [9 nil] INFO [2019-12-17 12:30:02,354] jepsen worker 44 - jepsen.util 4004 :info :cas [14 [31 13]] :RequestCanceledException INFO [2019-12-17 12:30:02,355] jepsen worker 44 - jepsen.util 4094 :invoke :write [14 17] INFO [2019-12-17 12:30:02,367] jepsen worker 88 - jepsen.util 2968 :ok :write [29 5] INFO [2019-12-17 12:30:02,368] jepsen worker 88 - jepsen.util 2968 :invoke :cas [29 [42 25]] INFO [2019-12-17 12:30:02,391] jepsen worker 27 - jepsen.util 2637 :ok :read [9 3] INFO [2019-12-17 12:30:02,391] jepsen worker 27 - jepsen.util 2637 :invoke :write [9 16] INFO [2019-12-17 12:30:02,398] jepsen worker 65 - jepsen.util 2675 :ok :write [21 5] INFO [2019-12-17 12:30:02,398] jepsen worker 65 - jepsen.util 2675 :invoke :write [21 26] INFO [2019-12-17 12:30:02,408] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:30:02,408] jepsen nemesis - couchbase.util Rebalance check count: 125 INFO [2019-12-17 12:30:02,408] jepsen nemesis - couchbase.util Rebalance stuck count: 106 INFO [2019-12-17 12:30:02,414] jepsen worker 4 - jepsen.util 2164 :ok :read [1 35] INFO [2019-12-17 12:30:02,414] jepsen worker 4 - jepsen.util 2164 :invoke :write [1 26] INFO [2019-12-17 12:30:02,417] jepsen worker 54 - jepsen.util 3024 :ok :read [18 44] INFO [2019-12-17 12:30:02,417] jepsen worker 54 - jepsen.util 3024 :invoke :read [18 nil] INFO [2019-12-17 12:30:02,451] jepsen worker 12 - jepsen.util 3072 :ok :read [4 34] INFO [2019-12-17 12:30:02,452] jepsen worker 12 - jepsen.util 3072 :invoke :cas [4 [28 32]] INFO [2019-12-17 12:30:02,454] jepsen worker 31 - jepsen.util 2641 :ok :read [10 45] INFO [2019-12-17 12:30:02,454] jepsen worker 31 - jepsen.util 2641 :invoke :write [10 38] INFO [2019-12-17 12:30:02,455] jepsen worker 37 - jepsen.util 2647 :ok :read [12 36] INFO [2019-12-17 12:30:02,455] jepsen worker 37 - jepsen.util 2647 :invoke :read [12 nil] INFO [2019-12-17 12:30:02,478] jepsen worker 45 - jepsen.util 3015 :ok :write [15 38] INFO [2019-12-17 12:30:02,478] jepsen worker 45 - jepsen.util 3015 :invoke :write [15 7] INFO [2019-12-17 12:30:02,480] jepsen worker 20 - jepsen.util 2630 :ok :write [6 38] INFO [2019-12-17 12:30:02,480] jepsen worker 20 - jepsen.util 2630 :invoke :read [6 nil] INFO [2019-12-17 12:30:02,490] jepsen worker 6 - jepsen.util 4056 :info :write [2 9] :RequestCanceledException INFO [2019-12-17 12:30:02,490] jepsen worker 53 - jepsen.util 2933 :fail :cas [17 [24 47]] :ValueNotSwapFrom INFO [2019-12-17 12:30:02,490] jepsen worker 53 - jepsen.util 2933 :invoke :write [17 38] INFO [2019-12-17 12:30:02,491] jepsen worker 6 - jepsen.util 4146 :invoke :write [2 19] INFO [2019-12-17 12:30:02,492] jepsen worker 73 - jepsen.util 2863 :ok :write [24 20] INFO [2019-12-17 12:30:02,492] jepsen worker 73 - jepsen.util 2863 :invoke :cas [24 [9 48]] INFO [2019-12-17 12:30:02,506] jepsen worker 10 - jepsen.util 2800 :fail :cas [3 [4 18]] :ValueNotSwapFrom INFO [2019-12-17 12:30:02,506] jepsen worker 10 - jepsen.util 2800 :invoke :read [3 nil] INFO [2019-12-17 12:30:02,508] jepsen worker 60 - jepsen.util 4200 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:30:02,509] jepsen worker 60 - jepsen.util 4200 :invoke :cas [20 [20 4]] INFO [2019-12-17 12:30:02,522] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":17202,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:30:02,523] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":17202,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:30:02,523] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":17205,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:30:02,523] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":17205,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:30:02,523] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":17206,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:30:02,523] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":17206,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:30:02,523] jepsen worker 34 - jepsen.util 3994 :info :write [11 41] :RequestCanceledException INFO [2019-12-17 12:30:02,523] jepsen worker 49 - jepsen.util 3289 :info :write [16 41] :RequestCanceledException INFO [2019-12-17 12:30:02,524] jepsen worker 80 - jepsen.util 2870 :fail :cas [26 [49 34]] :ValueNotSwapFrom INFO [2019-12-17 12:30:02,524] jepsen worker 49 - jepsen.util 3379 :invoke :write [16 40] INFO [2019-12-17 12:30:02,524] jepsen worker 34 - jepsen.util 4084 :invoke :read [11 nil] INFO [2019-12-17 12:30:02,524] jepsen worker 80 - jepsen.util 2870 :invoke :read [26 nil] INFO [2019-12-17 12:30:02,542] jepsen worker 14 - jepsen.util 2804 :fail :cas [4 [41 1]] :ValueNotSwapFrom INFO [2019-12-17 12:30:02,542] jepsen worker 14 - jepsen.util 2804 :invoke :cas [4 [26 19]] INFO [2019-12-17 12:30:02,555] jepsen worker 78 - jepsen.util 3138 :info :write [26 2] :RequestCanceledException INFO [2019-12-17 12:30:02,556] jepsen worker 78 - jepsen.util 3228 :invoke :cas [26 [43 26]] INFO [2019-12-17 12:30:02,600] jepsen worker 19 - jepsen.util 3169 :ok :read [6 38] INFO [2019-12-17 12:30:02,600] jepsen worker 19 - jepsen.util 3169 :invoke :read [6 nil] INFO [2019-12-17 12:30:02,605] jepsen worker 13 - jepsen.util 2803 :fail :cas [4 [11 37]] :ValueNotSwapFrom INFO [2019-12-17 12:30:02,605] jepsen worker 13 - jepsen.util 2803 :invoke :cas [4 [10 15]] INFO [2019-12-17 12:30:02,618] jepsen worker 73 - jepsen.util 2863 :fail :cas [24 [9 48]] :ValueNotSwapFrom INFO [2019-12-17 12:30:02,618] jepsen worker 73 - jepsen.util 2863 :invoke :write [24 21] INFO [2019-12-17 12:30:02,619] jepsen worker 29 - jepsen.util 2459 :ok :read [9 3] INFO [2019-12-17 12:30:02,619] jepsen worker 29 - jepsen.util 2459 :invoke :cas [9 [22 0]] INFO [2019-12-17 12:30:02,620] jepsen worker 7 - jepsen.util 3607 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:30:02,620] jepsen worker 74 - jepsen.util 3044 :info :write [24 39] :RequestCanceledException INFO [2019-12-17 12:30:02,621] jepsen worker 7 - jepsen.util 3607 :invoke :write [2 33] INFO [2019-12-17 12:30:02,622] jepsen worker 74 - jepsen.util 3134 :invoke :write [24 5] INFO [2019-12-17 12:30:02,650] jepsen worker 88 - jepsen.util 2968 :fail :cas [29 [42 25]] :ValueNotSwapFrom INFO [2019-12-17 12:30:02,651] jepsen worker 88 - jepsen.util 2968 :invoke :read [29 nil] INFO [2019-12-17 12:30:02,651] jepsen worker 20 - jepsen.util 2630 :ok :read [6 38] INFO [2019-12-17 12:30:02,651] jepsen worker 20 - jepsen.util 2630 :invoke :read [6 nil] INFO [2019-12-17 12:30:02,655] jepsen worker 23 - jepsen.util 3893 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:30:02,655] jepsen worker 23 - jepsen.util 3893 :invoke :cas [7 [23 47]] INFO [2019-12-17 12:30:02,655] jepsen worker 65 - jepsen.util 2675 :ok :write [21 26] INFO [2019-12-17 12:30:02,655] jepsen worker 65 - jepsen.util 2675 :invoke :read [21 nil] INFO [2019-12-17 12:30:02,673] jepsen worker 19 - jepsen.util 3169 :ok :read [6 38] INFO [2019-12-17 12:30:02,673] jepsen worker 19 - jepsen.util 3169 :invoke :cas [6 [16 18]] INFO [2019-12-17 12:30:02,683] jepsen worker 20 - jepsen.util 2630 :ok :read [6 38] INFO [2019-12-17 12:30:02,683] jepsen worker 20 - jepsen.util 2630 :invoke :read [6 nil] INFO [2019-12-17 12:30:02,685] jepsen worker 87 - jepsen.util 2697 :ok :read [29 5] INFO [2019-12-17 12:30:02,685] jepsen worker 87 - jepsen.util 2697 :invoke :write [29 12] INFO [2019-12-17 12:30:02,711] jepsen worker 4 - jepsen.util 2164 :ok :write [1 26] INFO [2019-12-17 12:30:02,712] jepsen worker 4 - jepsen.util 2164 :invoke :write [1 1] INFO [2019-12-17 12:30:02,713] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":17229,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:30:02,714] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":17229,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:30:02,714] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":17232,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:30:02,714] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":17232,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:30:02,714] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":17233,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:30:02,715] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":17233,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:30:02,720] jepsen worker 19 - jepsen.util 3169 :fail :cas [6 [16 18]] :ValueNotSwapFrom INFO [2019-12-17 12:30:02,721] jepsen worker 19 - jepsen.util 3169 :invoke :read [6 nil] INFO [2019-12-17 12:30:02,747] jepsen worker 20 - jepsen.util 2630 :ok :read [6 38] INFO [2019-12-17 12:30:02,747] jepsen worker 20 - jepsen.util 2630 :invoke :read [6 nil] INFO [2019-12-17 12:30:02,755] jepsen worker 36 - jepsen.util 2826 :info :write [12 20] :RequestCanceledException INFO [2019-12-17 12:30:02,755] jepsen worker 36 - jepsen.util 2916 :invoke :read [12 nil] INFO [2019-12-17 12:30:02,758] jepsen worker 51 - jepsen.util 2751 :fail :cas [17 [0 23]] :ValueNotSwapFrom INFO [2019-12-17 12:30:02,758] jepsen worker 51 - jepsen.util 2751 :invoke :cas [17 [28 5]] INFO [2019-12-17 12:30:02,771] jepsen worker 19 - jepsen.util 3169 :ok :read [6 38] INFO [2019-12-17 12:30:02,771] jepsen worker 19 - jepsen.util 3169 :invoke :write [6 49] INFO [2019-12-17 12:30:02,790] jepsen worker 47 - jepsen.util 3377 :info :cas [15 [19 46]] :RequestCanceledException INFO [2019-12-17 12:30:02,791] jepsen worker 47 - jepsen.util 3467 :invoke :write [15 2] INFO [2019-12-17 12:30:02,797] jepsen worker 80 - jepsen.util 2870 :ok :read [26 27] INFO [2019-12-17 12:30:02,797] jepsen worker 80 - jepsen.util 2870 :invoke :read [26 nil] INFO [2019-12-17 12:30:02,801] jepsen worker 27 - jepsen.util 2637 :ok :write [9 16] INFO [2019-12-17 12:30:02,801] jepsen worker 27 - jepsen.util 2637 :invoke :cas [9 [12 48]] INFO [2019-12-17 12:30:02,807] jepsen worker 20 - jepsen.util 2630 :ok :read [6 38] INFO [2019-12-17 12:30:02,807] jepsen worker 20 - jepsen.util 2630 :invoke :read [6 nil] INFO [2019-12-17 12:30:02,825] jepsen worker 37 - jepsen.util 2647 :ok :read [12 36] INFO [2019-12-17 12:30:02,825] jepsen worker 37 - jepsen.util 2647 :invoke :read [12 nil] INFO [2019-12-17 12:30:02,867] jepsen worker 56 - jepsen.util 2846 :ok :write [18 31] INFO [2019-12-17 12:30:02,867] jepsen worker 56 - jepsen.util 2846 :invoke :cas [18 [7 1]] INFO [2019-12-17 12:30:02,905] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":17255,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:30:02,905] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":17255,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:30:02,905] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":17257,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:30:02,905] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":17257,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:30:02,905] jepsen worker 3 - jepsen.util 2793 :info :write [1 4] :RequestCanceledException INFO [2019-12-17 12:30:02,906] jepsen worker 3 - jepsen.util 2883 :invoke :read [1 nil] INFO [2019-12-17 12:30:02,910] jepsen worker 14 - jepsen.util 2804 :fail :cas [4 [26 19]] :ValueNotSwapFrom INFO [2019-12-17 12:30:02,910] jepsen worker 14 - jepsen.util 2804 :invoke :read [4 nil] INFO [2019-12-17 12:30:02,950] jepsen worker 13 - jepsen.util 2803 :fail :cas [4 [10 15]] :ValueNotSwapFrom INFO [2019-12-17 12:30:02,951] jepsen worker 13 - jepsen.util 2803 :invoke :write [4 7] INFO [2019-12-17 12:30:02,979] jepsen worker 70 - jepsen.util 3400 :ok :write [23 43] INFO [2019-12-17 12:30:02,979] jepsen worker 70 - jepsen.util 3400 :invoke :write [23 13] INFO [2019-12-17 12:30:03,008] jepsen worker 54 - jepsen.util 3024 :ok :read [18 31] INFO [2019-12-17 12:30:03,008] jepsen worker 54 - jepsen.util 3024 :invoke :read [18 nil] INFO [2019-12-17 12:30:03,014] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":17263,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:30:03,014] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":17263,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:30:03,014] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":17264,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:30:03,014] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":17264,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:30:03,014] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":17265,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:30:03,014] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":17265,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:30:03,021] jepsen worker 87 - jepsen.util 2697 :ok :write [29 12] INFO [2019-12-17 12:30:03,021] jepsen worker 87 - jepsen.util 2697 :invoke :write [29 37] INFO [2019-12-17 12:30:03,023] jepsen worker 16 - jepsen.util 4066 :info :cas [5 [5 27]] :RequestCanceledException INFO [2019-12-17 12:30:03,024] jepsen worker 16 - jepsen.util 4156 :invoke :cas [5 [31 19]] INFO [2019-12-17 12:30:03,042] jepsen worker 5 - jepsen.util 2525 :fail :cas [1 [29 27]] :ValueNotSwapFrom INFO [2019-12-17 12:30:03,042] jepsen worker 5 - jepsen.util 2525 :invoke :cas [1 [18 7]] INFO [2019-12-17 12:30:03,055] jepsen worker 35 - jepsen.util 3815 :info :cas [11 [43 18]] :RequestCanceledException INFO [2019-12-17 12:30:03,056] jepsen worker 35 - jepsen.util 3905 :invoke :read [11 nil] INFO [2019-12-17 12:30:03,094] jepsen worker 88 - jepsen.util 2968 :ok :read [29 12] INFO [2019-12-17 12:30:03,095] jepsen worker 88 - jepsen.util 2968 :invoke :cas [29 [45 40]] INFO [2019-12-17 12:30:03,110] jepsen worker 73 - jepsen.util 2863 :ok :write [24 21] INFO [2019-12-17 12:30:03,110] jepsen worker 31 - jepsen.util 2641 :ok :write [10 38] INFO [2019-12-17 12:30:03,110] jepsen worker 73 - jepsen.util 2863 :invoke :cas [24 [9 22]] INFO [2019-12-17 12:30:03,110] jepsen worker 31 - jepsen.util 2641 :invoke :write [10 41] INFO [2019-12-17 12:30:03,135] jepsen worker 89 - jepsen.util 2699 :ok :write [29 17] INFO [2019-12-17 12:30:03,135] jepsen worker 89 - jepsen.util 2699 :invoke :write [29 43] INFO [2019-12-17 12:30:03,138] jepsen worker 73 - jepsen.util 2863 :fail :cas [24 [9 22]] :ValueNotSwapFrom INFO [2019-12-17 12:30:03,138] jepsen worker 73 - jepsen.util 2863 :invoke :read [24 nil] INFO [2019-12-17 12:30:03,141] jepsen worker 65 - jepsen.util 2675 :ok :read [21 26] INFO [2019-12-17 12:30:03,141] jepsen worker 65 - jepsen.util 2675 :invoke :cas [21 [1 4]] INFO [2019-12-17 12:30:03,147] jepsen worker 12 - jepsen.util 3072 :fail :cas [4 [28 32]] :ValueNotSwapFrom INFO [2019-12-17 12:30:03,147] jepsen worker 12 - jepsen.util 3072 :invoke :read [4 nil] INFO [2019-12-17 12:30:03,160] jepsen worker 73 - jepsen.util 2863 :ok :read [24 21] INFO [2019-12-17 12:30:03,161] jepsen worker 73 - jepsen.util 2863 :invoke :cas [24 [24 5]] INFO [2019-12-17 12:30:03,182] jepsen worker 73 - jepsen.util 2863 :fail :cas [24 [24 5]] :ValueNotSwapFrom INFO [2019-12-17 12:30:03,182] jepsen worker 73 - jepsen.util 2863 :invoke :read [24 nil] INFO [2019-12-17 12:30:03,195] jepsen worker 36 - jepsen.util 2916 :ok :read [12 20] INFO [2019-12-17 12:30:03,195] jepsen worker 36 - jepsen.util 2916 :invoke :write [12 38] INFO [2019-12-17 12:30:03,199] jepsen worker 73 - jepsen.util 2863 :ok :read [24 21] INFO [2019-12-17 12:30:03,199] jepsen worker 73 - jepsen.util 2863 :invoke :cas [24 [20 7]] INFO [2019-12-17 12:30:03,205] jepsen worker 59 - jepsen.util 3839 :info :cas [19 [38 9]] :RequestCanceledException INFO [2019-12-17 12:30:03,205] jepsen worker 11 - jepsen.util 3251 :info :write [3 12] :RequestCanceledException INFO [2019-12-17 12:30:03,206] jepsen worker 14 - jepsen.util 2804 :ok :read [4 34] INFO [2019-12-17 12:30:03,206] jepsen worker 14 - jepsen.util 2804 :invoke :cas [4 [33 45]] INFO [2019-12-17 12:30:03,206] jepsen worker 11 - jepsen.util 3341 :invoke :read [3 nil] INFO [2019-12-17 12:30:03,206] jepsen worker 59 - jepsen.util 3929 :invoke :read [19 nil] INFO [2019-12-17 12:30:03,220] jepsen worker 73 - jepsen.util 2863 :fail :cas [24 [20 7]] :ValueNotSwapFrom INFO [2019-12-17 12:30:03,220] jepsen worker 73 - jepsen.util 2863 :invoke :write [24 18] INFO [2019-12-17 12:30:03,223] jepsen worker 43 - jepsen.util 3643 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:30:03,223] jepsen worker 43 - jepsen.util 3643 :invoke :write [14 35] INFO [2019-12-17 12:30:03,233] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":17283,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:30:03,233] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":17283,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:30:03,233] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":17284,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:30:03,233] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":17284,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:30:03,237] jepsen worker 15 - jepsen.util 3615 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:30:03,238] jepsen worker 15 - jepsen.util 3615 :invoke :write [5 19] INFO [2019-12-17 12:30:03,255] jepsen worker 85 - jepsen.util 4495 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:30:03,255] jepsen worker 45 - jepsen.util 3015 :ok :write [15 7] INFO [2019-12-17 12:30:03,255] jepsen worker 85 - jepsen.util 4495 :invoke :write [28 38] INFO [2019-12-17 12:30:03,255] jepsen worker 45 - jepsen.util 3015 :invoke :write [15 43] INFO [2019-12-17 12:30:03,268] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":17297,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:30:03,268] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":17297,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:30:03,268] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":17299,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:30:03,268] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":17299,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:30:03,307] jepsen worker 52 - jepsen.util 3202 :fail :cas [17 [32 35]] :ValueNotSwapFrom INFO [2019-12-17 12:30:03,307] jepsen worker 52 - jepsen.util 3202 :invoke :cas [17 [15 19]] INFO [2019-12-17 12:30:03,323] jepsen worker 30 - jepsen.util 3450 :info :write [10 37] :RequestCanceledException INFO [2019-12-17 12:30:03,323] jepsen worker 18 - jepsen.util 3078 :info :write [6 2] :RequestCanceledException INFO [2019-12-17 12:30:03,323] jepsen worker 84 - jepsen.util 3774 :info :write [28 0] :RequestCanceledException INFO [2019-12-17 12:30:03,324] jepsen worker 18 - jepsen.util 3168 :invoke :cas [6 [48 18]] INFO [2019-12-17 12:30:03,324] jepsen worker 84 - jepsen.util 3864 :invoke :cas [28 [25 31]] INFO [2019-12-17 12:30:03,324] jepsen worker 30 - jepsen.util 3540 :invoke :write [10 31] INFO [2019-12-17 12:30:03,336] jepsen worker 80 - jepsen.util 2870 :ok :read [26 27] INFO [2019-12-17 12:30:03,337] jepsen worker 80 - jepsen.util 2870 :invoke :cas [26 [47 25]] INFO [2019-12-17 12:30:03,339] jepsen worker 18 - jepsen.util 3168 :fail :cas [6 [48 18]] :ValueNotSwapFrom INFO [2019-12-17 12:30:03,339] jepsen worker 18 - jepsen.util 3168 :invoke :cas [6 [12 21]] INFO [2019-12-17 12:30:03,352] jepsen worker 37 - jepsen.util 2647 :ok :read [12 20] INFO [2019-12-17 12:30:03,352] jepsen worker 37 - jepsen.util 2647 :invoke :write [12 46] INFO [2019-12-17 12:30:03,354] jepsen worker 18 - jepsen.util 3168 :fail :cas [6 [12 21]] :ValueNotSwapFrom INFO [2019-12-17 12:30:03,355] jepsen worker 18 - jepsen.util 3168 :invoke :cas [6 [16 33]] INFO [2019-12-17 12:30:03,367] jepsen worker 18 - jepsen.util 3168 :fail :cas [6 [16 33]] :ValueNotSwapFrom INFO [2019-12-17 12:30:03,368] jepsen worker 18 - jepsen.util 3168 :invoke :read [6 nil] INFO [2019-12-17 12:30:03,370] jepsen worker 71 - jepsen.util 2951 :ok :write [23 30] INFO [2019-12-17 12:30:03,370] jepsen worker 71 - jepsen.util 2951 :invoke :read [23 nil] INFO [2019-12-17 12:30:03,385] jepsen worker 18 - jepsen.util 3168 :ok :read [6 38] INFO [2019-12-17 12:30:03,386] jepsen worker 18 - jepsen.util 3168 :invoke :cas [6 [30 25]] INFO [2019-12-17 12:30:03,391] jepsen worker 62 - jepsen.util 3752 :info :cas [20 [47 44]] :RequestCanceledException INFO [2019-12-17 12:30:03,392] jepsen worker 78 - jepsen.util 3228 :fail :cas [26 [43 26]] :ValueNotSwapFrom INFO [2019-12-17 12:30:03,392] jepsen worker 62 - jepsen.util 3842 :invoke :write [20 19] INFO [2019-12-17 12:30:03,393] jepsen worker 78 - jepsen.util 3228 :invoke :cas [26 [1 26]] INFO [2019-12-17 12:30:03,393] jepsen worker 5 - jepsen.util 2525 :fail :cas [1 [18 7]] :ValueNotSwapFrom INFO [2019-12-17 12:30:03,393] jepsen worker 5 - jepsen.util 2525 :invoke :write [1 46] INFO [2019-12-17 12:30:03,400] jepsen worker 18 - jepsen.util 3168 :fail :cas [6 [30 25]] :ValueNotSwapFrom INFO [2019-12-17 12:30:03,400] jepsen worker 18 - jepsen.util 3168 :invoke :read [6 nil] INFO [2019-12-17 12:30:03,408] jepsen worker 50 - jepsen.util 4370 :info :cas [16 [15 24]] :RequestCanceledException INFO [2019-12-17 12:30:03,409] jepsen worker 50 - jepsen.util 4460 :invoke :cas [16 [19 35]] INFO [2019-12-17 12:30:03,414] jepsen worker 18 - jepsen.util 3168 :ok :read [6 38] INFO [2019-12-17 12:30:03,415] jepsen worker 18 - jepsen.util 3168 :invoke :cas [6 [22 35]] INFO [2019-12-17 12:30:03,425] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:30:03,425] jepsen nemesis - couchbase.util Rebalance check count: 126 INFO [2019-12-17 12:30:03,425] jepsen nemesis - couchbase.util Rebalance stuck count: 107 INFO [2019-12-17 12:30:03,429] jepsen worker 18 - jepsen.util 3168 :fail :cas [6 [22 35]] :ValueNotSwapFrom INFO [2019-12-17 12:30:03,429] jepsen worker 18 - jepsen.util 3168 :invoke :cas [6 [14 21]] INFO [2019-12-17 12:30:03,437] jepsen worker 51 - jepsen.util 2751 :fail :cas [17 [28 5]] :ValueNotSwapFrom INFO [2019-12-17 12:30:03,437] jepsen worker 51 - jepsen.util 2751 :invoke :read [17 nil] INFO [2019-12-17 12:30:03,442] jepsen worker 19 - jepsen.util 3169 :ok :write [6 49] INFO [2019-12-17 12:30:03,443] jepsen worker 19 - jepsen.util 3169 :invoke :cas [6 [43 31]] INFO [2019-12-17 12:30:03,444] jepsen worker 18 - jepsen.util 3168 :fail :cas [6 [14 21]] :ValueNotSwapFrom INFO [2019-12-17 12:30:03,444] jepsen worker 18 - jepsen.util 3168 :invoke :write [6 2] INFO [2019-12-17 12:30:03,447] jepsen worker 73 - jepsen.util 2863 :ok :write [24 18] INFO [2019-12-17 12:30:03,447] jepsen worker 73 - jepsen.util 2863 :invoke :cas [24 [13 34]] INFO [2019-12-17 12:30:03,454] jepsen worker 1 - jepsen.util 3421 :info :cas [0 [20 0]] :RequestCanceledException INFO [2019-12-17 12:30:03,455] jepsen worker 1 - jepsen.util 3511 :invoke :write [0 37] INFO [2019-12-17 12:30:03,465] jepsen worker 73 - jepsen.util 2863 :fail :cas [24 [13 34]] :ValueNotSwapFrom INFO [2019-12-17 12:30:03,465] jepsen worker 73 - jepsen.util 2863 :invoke :cas [24 [5 32]] INFO [2019-12-17 12:30:03,488] jepsen worker 56 - jepsen.util 2846 :fail :cas [18 [7 1]] :ValueNotSwapFrom INFO [2019-12-17 12:30:03,488] jepsen worker 73 - jepsen.util 2863 :fail :cas [24 [5 32]] :ValueNotSwapFrom INFO [2019-12-17 12:30:03,488] jepsen worker 56 - jepsen.util 2846 :invoke :cas [18 [17 29]] INFO [2019-12-17 12:30:03,488] jepsen worker 73 - jepsen.util 2863 :invoke :write [24 19] INFO [2019-12-17 12:30:03,488] jepsen worker 31 - jepsen.util 2641 :ok :write [10 41] INFO [2019-12-17 12:30:03,489] jepsen worker 31 - jepsen.util 2641 :invoke :cas [10 [4 48]] INFO [2019-12-17 12:30:03,500] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":17323,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:30:03,500] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":17323,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:30:03,501] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":17326,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:30:03,501] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":17326,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:30:03,505] jepsen worker 65 - jepsen.util 2675 :fail :cas [21 [1 4]] :ValueNotSwapFrom INFO [2019-12-17 12:30:03,505] jepsen worker 65 - jepsen.util 2675 :invoke :write [21 20] INFO [2019-12-17 12:30:03,513] jepsen worker 11 - jepsen.util 3341 :ok :read [3 23] INFO [2019-12-17 12:30:03,513] jepsen worker 11 - jepsen.util 3341 :invoke :write [3 49] INFO [2019-12-17 12:30:03,514] jepsen worker 40 - jepsen.util 4180 :info :write [13 20] :RequestCanceledException INFO [2019-12-17 12:30:03,515] jepsen worker 40 - jepsen.util 4270 :invoke :read [13 nil] INFO [2019-12-17 12:30:03,527] jepsen worker 12 - jepsen.util 3072 :ok :read [4 34] INFO [2019-12-17 12:30:03,527] jepsen worker 12 - jepsen.util 3072 :invoke :cas [4 [5 19]] INFO [2019-12-17 12:30:03,534] jepsen worker 71 - jepsen.util 2951 :ok :read [23 30] INFO [2019-12-17 12:30:03,534] jepsen worker 71 - jepsen.util 2951 :invoke :cas [23 [29 4]] INFO [2019-12-17 12:30:03,536] jepsen worker 75 - jepsen.util 4035 :info :write [25 29] :RequestCanceledException INFO [2019-12-17 12:30:03,536] jepsen worker 21 - jepsen.util 3891 :info :cas [7 [8 13]] :RequestCanceledException INFO [2019-12-17 12:30:03,537] jepsen worker 21 - jepsen.util 3981 :invoke :read [7 nil] INFO [2019-12-17 12:30:03,537] jepsen worker 75 - jepsen.util 4125 :invoke :write [25 7] INFO [2019-12-17 12:30:03,542] jepsen worker 31 - jepsen.util 2641 :fail :cas [10 [4 48]] :ValueNotSwapFrom INFO [2019-12-17 12:30:03,542] jepsen worker 31 - jepsen.util 2641 :invoke :write [10 6] INFO [2019-12-17 12:30:03,554] jepsen worker 22 - jepsen.util 3982 :info :write [7 39] :RequestCanceledException INFO [2019-12-17 12:30:03,555] jepsen worker 22 - jepsen.util 4072 :invoke :write [7 32] INFO [2019-12-17 12:30:03,575] jepsen worker 51 - jepsen.util 2751 :ok :read [17 27] INFO [2019-12-17 12:30:03,575] jepsen worker 51 - jepsen.util 2751 :invoke :write [17 7] INFO [2019-12-17 12:30:03,591] jepsen worker 65 - jepsen.util 2675 :ok :write [21 20] INFO [2019-12-17 12:30:03,591] jepsen worker 65 - jepsen.util 2675 :invoke :cas [21 [5 25]] INFO [2019-12-17 12:30:03,595] jepsen worker 80 - jepsen.util 2870 :fail :cas [26 [47 25]] :ValueNotSwapFrom INFO [2019-12-17 12:30:03,596] jepsen worker 80 - jepsen.util 2870 :invoke :read [26 nil] INFO [2019-12-17 12:30:03,605] jepsen worker 41 - jepsen.util 4091 :info :write [13 25] :RequestCanceledException INFO [2019-12-17 12:30:03,606] jepsen worker 41 - jepsen.util 4181 :invoke :read [13 nil] INFO [2019-12-17 12:30:03,624] jepsen worker 57 - jepsen.util 3657 :info :write [19 9] :RequestCanceledException INFO [2019-12-17 12:30:03,625] jepsen worker 57 - jepsen.util 3747 :invoke :read [19 nil] INFO [2019-12-17 12:30:03,641] jepsen worker 3 - jepsen.util 2883 :ok :read [1 26] INFO [2019-12-17 12:30:03,641] jepsen worker 3 - jepsen.util 2883 :invoke :write [1 25] INFO [2019-12-17 12:30:03,649] jepsen worker 45 - jepsen.util 3015 :ok :write [15 43] INFO [2019-12-17 12:30:03,649] jepsen worker 45 - jepsen.util 3015 :invoke :read [15 nil] INFO [2019-12-17 12:30:03,669] jepsen worker 45 - jepsen.util 3015 :ok :read [15 43] INFO [2019-12-17 12:30:03,669] jepsen worker 45 - jepsen.util 3015 :invoke :write [15 32] INFO [2019-12-17 12:30:03,676] jepsen worker 88 - jepsen.util 2968 :fail :cas [29 [45 40]] :ValueNotSwapFrom INFO [2019-12-17 12:30:03,676] jepsen worker 88 - jepsen.util 2968 :invoke :write [29 28] INFO [2019-12-17 12:30:03,679] jepsen worker 71 - jepsen.util 2951 :fail :cas [23 [29 4]] :ValueNotSwapFrom INFO [2019-12-17 12:30:03,680] jepsen worker 71 - jepsen.util 2951 :invoke :read [23 nil] INFO [2019-12-17 12:30:03,689] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":17355,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:30:03,689] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":17355,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:30:03,689] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":17357,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:30:03,689] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":17357,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:30:03,690] jepsen worker 8 - jepsen.util 3338 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:30:03,690] jepsen worker 42 - jepsen.util 4542 :info :write [14 24] :RequestCanceledException INFO [2019-12-17 12:30:03,690] jepsen worker 8 - jepsen.util 3338 :invoke :read [2 nil] INFO [2019-12-17 12:30:03,691] jepsen worker 42 - jepsen.util 4632 :invoke :write [14 21] INFO [2019-12-17 12:30:03,707] jepsen worker 83 - jepsen.util 4043 :info :write [27 11] :RequestCanceledException INFO [2019-12-17 12:30:03,707] jepsen worker 0 - jepsen.util 4320 :info :write [0 48] :RequestCanceledException INFO [2019-12-17 12:30:03,707] jepsen worker 33 - jepsen.util 4173 :info :write [11 42] :RequestCanceledException INFO [2019-12-17 12:30:03,709] jepsen worker 33 - jepsen.util 4263 :invoke :write [11 21] INFO [2019-12-17 12:30:03,709] jepsen worker 0 - jepsen.util 4410 :invoke :write [0 27] INFO [2019-12-17 12:30:03,709] jepsen worker 83 - jepsen.util 4133 :invoke :cas [27 [49 26]] INFO [2019-12-17 12:30:03,723] jepsen worker 48 - jepsen.util 3378 :info :write [16 41] :RequestCanceledException INFO [2019-12-17 12:30:03,723] jepsen worker 69 - jepsen.util 2859 :info :write [23 33] :RequestCanceledException INFO [2019-12-17 12:30:03,724] jepsen worker 48 - jepsen.util 3468 :invoke :read [16 nil] INFO [2019-12-17 12:30:03,724] jepsen worker 69 - jepsen.util 2949 :invoke :cas [23 [18 7]] INFO [2019-12-17 12:30:03,738] jepsen worker 26 - jepsen.util 4166 :info :write [8 1] :RequestCanceledException INFO [2019-12-17 12:30:03,739] jepsen worker 26 - jepsen.util 4256 :invoke :write [8 4] INFO [2019-12-17 12:30:03,755] jepsen worker 17 - jepsen.util 3977 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:30:03,755] jepsen worker 17 - jepsen.util 3977 :invoke :cas [5 [31 16]] INFO [2019-12-17 12:30:03,757] jepsen worker 54 - jepsen.util 3024 :ok :read [18 31] INFO [2019-12-17 12:30:03,757] jepsen worker 54 - jepsen.util 3024 :invoke :write [18 27] INFO [2019-12-17 12:30:03,824] jepsen worker 55 - jepsen.util 3565 :info :write [18 29] :RequestCanceledException INFO [2019-12-17 12:30:03,826] jepsen worker 55 - jepsen.util 3655 :invoke :write [18 5] INFO [2019-12-17 12:30:03,846] jepsen worker 18 - jepsen.util 3168 :ok :write [6 2] INFO [2019-12-17 12:30:03,846] jepsen worker 18 - jepsen.util 3168 :invoke :read [6 nil] INFO [2019-12-17 12:30:03,847] jepsen worker 31 - jepsen.util 2641 :ok :write [10 6] INFO [2019-12-17 12:30:03,847] jepsen worker 31 - jepsen.util 2641 :invoke :cas [10 [18 16]] INFO [2019-12-17 12:30:03,864] jepsen worker 73 - jepsen.util 2863 :ok :write [24 19] INFO [2019-12-17 12:30:03,864] jepsen worker 45 - jepsen.util 3015 :ok :write [15 32] INFO [2019-12-17 12:30:03,864] jepsen worker 73 - jepsen.util 2863 :invoke :cas [24 [39 21]] INFO [2019-12-17 12:30:03,864] jepsen worker 45 - jepsen.util 3015 :invoke :write [15 19] INFO [2019-12-17 12:30:03,869] jepsen worker 14 - jepsen.util 2804 :fail :cas [4 [33 45]] :ValueNotSwapFrom INFO [2019-12-17 12:30:03,869] jepsen worker 14 - jepsen.util 2804 :invoke :cas [4 [42 18]] INFO [2019-12-17 12:30:03,879] jepsen worker 18 - jepsen.util 3168 :ok :read [6 2] INFO [2019-12-17 12:30:03,879] jepsen worker 18 - jepsen.util 3168 :invoke :write [6 6] INFO [2019-12-17 12:30:03,891] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":17377,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:30:03,891] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":17377,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:30:03,891] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":17380,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:30:03,891] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":17380,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:30:03,891] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":17381,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:30:03,891] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":17381,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:30:03,908] jepsen worker 31 - jepsen.util 2641 :fail :cas [10 [18 16]] :ValueNotSwapFrom INFO [2019-12-17 12:30:03,908] jepsen worker 31 - jepsen.util 2641 :invoke :write [10 44] INFO [2019-12-17 12:30:03,915] jepsen worker 73 - jepsen.util 2863 :fail :cas [24 [39 21]] :ValueNotSwapFrom INFO [2019-12-17 12:30:03,916] jepsen worker 73 - jepsen.util 2863 :invoke :read [24 nil] INFO [2019-12-17 12:30:03,942] jepsen worker 56 - jepsen.util 2846 :fail :cas [18 [17 29]] :ValueNotSwapFrom INFO [2019-12-17 12:30:03,942] jepsen worker 56 - jepsen.util 2846 :invoke :cas [18 [10 19]] INFO [2019-12-17 12:30:03,955] jepsen worker 73 - jepsen.util 2863 :ok :read [24 19] INFO [2019-12-17 12:30:03,956] jepsen worker 73 - jepsen.util 2863 :invoke :write [24 44] INFO [2019-12-17 12:30:03,982] jepsen worker 52 - jepsen.util 3202 :fail :cas [17 [15 19]] :ValueNotSwapFrom INFO [2019-12-17 12:30:03,982] jepsen worker 52 - jepsen.util 3202 :invoke :read [17 nil] INFO [2019-12-17 12:30:03,990] jepsen worker 72 - jepsen.util 2952 :info :write [24 47] :RequestCanceledException INFO [2019-12-17 12:30:03,991] jepsen worker 72 - jepsen.util 3042 :invoke :cas [24 [5 8]] INFO [2019-12-17 12:30:04,000] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":17391,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:30:04,001] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":17391,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:30:04,001] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":17393,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:30:04,001] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":17393,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:30:04,009] jepsen worker 72 - jepsen.util 3042 :fail :cas [24 [5 8]] :ValueNotSwapFrom INFO [2019-12-17 12:30:04,009] jepsen worker 72 - jepsen.util 3042 :invoke :write [24 36] INFO [2019-12-17 12:30:04,031] jepsen worker 78 - jepsen.util 3228 :fail :cas [26 [1 26]] :ValueNotSwapFrom INFO [2019-12-17 12:30:04,032] jepsen worker 78 - jepsen.util 3228 :invoke :read [26 nil] INFO [2019-12-17 12:30:04,090] jepsen worker 64 - jepsen.util 2404 :info :write [21 5] :RequestCanceledException INFO [2019-12-17 12:30:04,090] jepsen worker 79 - jepsen.util 2869 :info :write [26 10] :RequestCanceledException INFO [2019-12-17 12:30:04,091] jepsen worker 64 - jepsen.util 2494 :invoke :read [21 nil] INFO [2019-12-17 12:30:04,091] jepsen worker 79 - jepsen.util 2959 :invoke :read [26 nil] INFO [2019-12-17 12:30:04,104] jepsen worker 64 - jepsen.util 2494 :ok :read [21 20] INFO [2019-12-17 12:30:04,104] jepsen worker 64 - jepsen.util 2494 :invoke :write [21 8] INFO [2019-12-17 12:30:04,108] jepsen worker 9 - jepsen.util 3339 :info :write [3 27] :RequestCanceledException INFO [2019-12-17 12:30:04,109] jepsen worker 9 - jepsen.util 3429 :invoke :read [3 nil] INFO [2019-12-17 12:30:04,122] jepsen worker 31 - jepsen.util 2641 :ok :write [10 44] INFO [2019-12-17 12:30:04,122] jepsen worker 18 - jepsen.util 3168 :ok :write [6 6] INFO [2019-12-17 12:30:04,122] jepsen worker 31 - jepsen.util 2641 :invoke :cas [10 [24 17]] INFO [2019-12-17 12:30:04,122] jepsen worker 18 - jepsen.util 3168 :invoke :write [6 17] INFO [2019-12-17 12:30:04,124] jepsen worker 81 - jepsen.util 3951 :info :write [27 31] :RequestCanceledException INFO [2019-12-17 12:30:04,124] jepsen worker 46 - jepsen.util 3106 :info :write [15 49] :RequestCanceledException INFO [2019-12-17 12:30:04,125] jepsen worker 46 - jepsen.util 3196 :invoke :write [15 39] INFO [2019-12-17 12:30:04,125] jepsen worker 81 - jepsen.util 4041 :invoke :read [27 nil] INFO [2019-12-17 12:30:04,133] jepsen worker 45 - jepsen.util 3015 :ok :write [15 19] INFO [2019-12-17 12:30:04,136] jepsen worker 45 - jepsen.util 3015 :invoke :write [15 48] INFO [2019-12-17 12:30:04,137] jepsen worker 76 - jepsen.util 4036 :info :cas [25 [37 26]] :RequestCanceledException INFO [2019-12-17 12:30:04,138] jepsen worker 76 - jepsen.util 4126 :invoke :cas [25 [6 40]] INFO [2019-12-17 12:30:04,143] jepsen worker 9 - jepsen.util 3429 :ok :read [3 23] INFO [2019-12-17 12:30:04,144] jepsen worker 9 - jepsen.util 3429 :invoke :write [3 18] INFO [2019-12-17 12:30:04,152] jepsen worker 31 - jepsen.util 2641 :fail :cas [10 [24 17]] :ValueNotSwapFrom INFO [2019-12-17 12:30:04,152] jepsen worker 31 - jepsen.util 2641 :invoke :write [10 9] INFO [2019-12-17 12:30:04,155] jepsen worker 25 - jepsen.util 4345 :info :write [8 22] :RequestCanceledException INFO [2019-12-17 12:30:04,155] jepsen worker 58 - jepsen.util 3928 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:30:04,155] jepsen worker 38 - jepsen.util 2828 :info :write [12 23] :RequestCanceledException INFO [2019-12-17 12:30:04,155] jepsen worker 58 - jepsen.util 3928 :invoke :cas [19 [45 35]] INFO [2019-12-17 12:30:04,156] jepsen worker 25 - jepsen.util 4435 :invoke :write [8 49] INFO [2019-12-17 12:30:04,160] jepsen worker 38 - jepsen.util 2918 :invoke :read [12 nil] INFO [2019-12-17 12:30:04,169] jepsen worker 71 - jepsen.util 2951 :ok :read [23 30] INFO [2019-12-17 12:30:04,170] jepsen worker 64 - jepsen.util 2494 :ok :write [21 8] INFO [2019-12-17 12:30:04,190] jepsen worker 32 - jepsen.util 3452 :info :write [10 49] :RequestCanceledException INFO [2019-12-17 12:30:04,207] jepsen worker 86 - jepsen.util 4136 :info :cas [28 [18 13]] :RequestCanceledException INFO [2019-12-17 12:30:04,220] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":17415,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:30:04,223] jepsen worker 28 - jepsen.util 2188 :info :write [9 24] :RequestCanceledException INFO [2019-12-17 12:30:04,236] jepsen worker 82 - jepsen.util 4672 :info :write [27 17] :RequestCanceledException INFO [2019-12-17 12:30:04,246] jepsen worker 12 - jepsen.util 3072 :fail :cas [4 [5 19]] :ValueNotSwapFrom INFO [2019-12-17 12:30:04,248] jepsen worker 69 - jepsen.util 2949 :fail :cas [23 [18 7]] :ValueNotSwapFrom INFO [2019-12-17 12:30:04,254] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":17422,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:30:04,254] jepsen worker 61 - jepsen.util 4471 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:30:04,258] jepsen worker 52 - jepsen.util 3202 :ok :read [17 27] INFO [2019-12-17 12:30:04,268] jepsen worker 71 - jepsen.util 2951 :invoke :write [23 23] INFO [2019-12-17 12:30:04,268] jepsen worker 64 - jepsen.util 2494 :invoke :write [21 7] INFO [2019-12-17 12:30:04,274] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":17415,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:30:04,274] jepsen worker 12 - jepsen.util 3072 :invoke :write [4 44] INFO [2019-12-17 12:30:04,274] jepsen worker 69 - jepsen.util 2949 :invoke :cas [23 [11 36]] INFO [2019-12-17 12:30:04,274] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":17422,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:30:04,275] jepsen worker 61 - jepsen.util 4471 :invoke :cas [20 [32 33]] INFO [2019-12-17 12:30:04,275] jepsen worker 52 - jepsen.util 3202 :invoke :read [17 nil] INFO [2019-12-17 12:30:04,275] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":17417,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:30:04,275] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":17425,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:30:04,275] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":17417,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:30:04,275] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":17425,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:30:04,275] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":17426,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:30:04,275] jepsen worker 86 - jepsen.util 4226 :invoke :write [28 41] INFO [2019-12-17 12:30:04,275] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":17426,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:30:04,275] jepsen worker 32 - jepsen.util 3542 :invoke :cas [10 [41 10]] INFO [2019-12-17 12:30:04,275] jepsen worker 82 - jepsen.util 4762 :invoke :cas [27 [23 40]] INFO [2019-12-17 12:30:04,275] jepsen worker 28 - jepsen.util 2278 :invoke :cas [9 [13 11]] INFO [2019-12-17 12:30:04,290] jepsen worker 63 - jepsen.util 2583 :info :cas [21 [0 1]] :RequestCanceledException INFO [2019-12-17 12:30:04,290] jepsen worker 66 - jepsen.util 3936 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:30:04,290] jepsen worker 2 - jepsen.util 3602 :info :cas [0 [18 8]] :RequestCanceledException INFO [2019-12-17 12:30:04,291] jepsen worker 66 - jepsen.util 3936 :invoke :write [22 42] INFO [2019-12-17 12:30:04,291] jepsen worker 2 - jepsen.util 3692 :invoke :write [0 1] INFO [2019-12-17 12:30:04,291] jepsen worker 63 - jepsen.util 2673 :invoke :read [21 nil] INFO [2019-12-17 12:30:04,298] jepsen worker 28 - jepsen.util 2278 :fail :cas [9 [13 11]] :ValueNotSwapFrom INFO [2019-12-17 12:30:04,298] jepsen worker 28 - jepsen.util 2278 :invoke :cas [9 [27 44]] INFO [2019-12-17 12:30:04,314] jepsen worker 28 - jepsen.util 2278 :fail :cas [9 [27 44]] :ValueNotSwapFrom INFO [2019-12-17 12:30:04,314] jepsen worker 28 - jepsen.util 2278 :invoke :read [9 nil] INFO [2019-12-17 12:30:04,323] jepsen worker 68 - jepsen.util 3668 :info :write [22 18] :RequestCanceledException INFO [2019-12-17 12:30:04,323] jepsen worker 77 - jepsen.util 3587 :info :write [25 49] :RequestCanceledException INFO [2019-12-17 12:30:04,324] jepsen worker 68 - jepsen.util 3758 :invoke :write [22 0] INFO [2019-12-17 12:30:04,324] jepsen worker 77 - jepsen.util 3677 :invoke :write [25 23] INFO [2019-12-17 12:30:04,330] jepsen worker 56 - jepsen.util 2846 :fail :cas [18 [10 19]] :ValueNotSwapFrom INFO [2019-12-17 12:30:04,330] jepsen worker 56 - jepsen.util 2846 :invoke :write [18 32] INFO [2019-12-17 12:30:04,330] jepsen worker 28 - jepsen.util 2278 :ok :read [9 16] INFO [2019-12-17 12:30:04,331] jepsen worker 28 - jepsen.util 2278 :invoke :write [9 49] INFO [2019-12-17 12:30:04,337] jepsen worker 64 - jepsen.util 2494 :ok :write [21 7] INFO [2019-12-17 12:30:04,337] jepsen worker 64 - jepsen.util 2494 :invoke :cas [21 [45 42]] INFO [2019-12-17 12:30:04,338] jepsen worker 39 - jepsen.util 3819 :info :write [13 49] :RequestCanceledException INFO [2019-12-17 12:30:04,339] jepsen worker 39 - jepsen.util 3909 :invoke :read [13 nil] INFO [2019-12-17 12:30:04,355] jepsen worker 79 - jepsen.util 2959 :ok :read [26 27] INFO [2019-12-17 12:30:04,355] jepsen worker 79 - jepsen.util 2959 :invoke :cas [26 [11 32]] INFO [2019-12-17 12:30:04,362] jepsen worker 64 - jepsen.util 2494 :fail :cas [21 [45 42]] :ValueNotSwapFrom INFO [2019-12-17 12:30:04,362] jepsen worker 64 - jepsen.util 2494 :invoke :read [21 nil] INFO [2019-12-17 12:30:04,371] jepsen worker 78 - jepsen.util 3228 :ok :read [26 27] INFO [2019-12-17 12:30:04,371] jepsen worker 78 - jepsen.util 3228 :invoke :write [26 19] INFO [2019-12-17 12:30:04,386] jepsen worker 64 - jepsen.util 2494 :ok :read [21 7] INFO [2019-12-17 12:30:04,386] jepsen worker 64 - jepsen.util 2494 :invoke :cas [21 [7 29]] INFO [2019-12-17 12:30:04,407] jepsen worker 24 - jepsen.util 3804 :info :write [8 13] :RequestCanceledException INFO [2019-12-17 12:30:04,408] jepsen worker 24 - jepsen.util 3894 :invoke :cas [8 [44 25]] INFO [2019-12-17 12:30:04,412] jepsen worker 80 - jepsen.util 2870 :ok :read [26 27] INFO [2019-12-17 12:30:04,412] jepsen worker 80 - jepsen.util 2870 :invoke :read [26 nil] INFO [2019-12-17 12:30:04,424] jepsen worker 44 - jepsen.util 4094 :info :write [14 17] :RequestCanceledException INFO [2019-12-17 12:30:04,425] jepsen worker 44 - jepsen.util 4184 :invoke :write [14 7] INFO [2019-12-17 12:30:04,437] jepsen worker 67 - jepsen.util 4387 :info :write [22 7] :RequestCanceledException INFO [2019-12-17 12:30:04,438] jepsen worker 67 - jepsen.util 4477 :invoke :read [22 nil] INFO [2019-12-17 12:30:04,440] jepsen worker 28 - jepsen.util 2278 :ok :write [9 49] INFO [2019-12-17 12:30:04,440] jepsen worker 28 - jepsen.util 2278 :invoke :write [9 24] INFO [2019-12-17 12:30:04,443] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:30:04,443] jepsen nemesis - couchbase.util Rebalance check count: 127 INFO [2019-12-17 12:30:04,443] jepsen nemesis - couchbase.util Rebalance stuck count: 108 INFO [2019-12-17 12:30:04,480] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":17456,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:30:04,480] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":17456,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:30:04,480] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":17460,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:30:04,480] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":17460,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:30:04,481] jepsen worker 64 - jepsen.util 2494 :ok :cas [21 [7 29]] INFO [2019-12-17 12:30:04,481] jepsen worker 64 - jepsen.util 2494 :invoke :cas [21 [9 30]] INFO [2019-12-17 12:30:04,495] jepsen worker 64 - jepsen.util 2494 :fail :cas [21 [9 30]] :ValueNotSwapFrom INFO [2019-12-17 12:30:04,495] jepsen worker 64 - jepsen.util 2494 :invoke :write [21 43] INFO [2019-12-17 12:30:04,511] jepsen worker 14 - jepsen.util 2804 :fail :cas [4 [42 18]] :ValueNotSwapFrom INFO [2019-12-17 12:30:04,511] jepsen worker 14 - jepsen.util 2804 :invoke :write [4 37] INFO [2019-12-17 12:30:04,517] jepsen worker 28 - jepsen.util 2278 :ok :write [9 24] INFO [2019-12-17 12:30:04,517] jepsen worker 28 - jepsen.util 2278 :invoke :read [9 nil] INFO [2019-12-17 12:30:04,523] jepsen worker 10 - jepsen.util 2800 :fail :read [3 nil] :RequestTimeoutException INFO [2019-12-17 12:30:04,523] jepsen worker 10 - jepsen.util 2800 :invoke :cas [3 [21 26]] INFO [2019-12-17 12:30:04,537] jepsen worker 6 - jepsen.util 4146 :info :write [2 19] :RequestCanceledException INFO [2019-12-17 12:30:04,538] jepsen worker 6 - jepsen.util 4236 :invoke :cas [2 [32 20]] INFO [2019-12-17 12:30:04,542] jepsen worker 28 - jepsen.util 2278 :ok :read [9 24] INFO [2019-12-17 12:30:04,542] jepsen worker 28 - jepsen.util 2278 :invoke :cas [9 [2 47]] INFO [2019-12-17 12:30:04,554] jepsen worker 28 - jepsen.util 2278 :fail :cas [9 [2 47]] :ValueNotSwapFrom INFO [2019-12-17 12:30:04,554] jepsen worker 49 - jepsen.util 3379 :info :write [16 40] :RequestCanceledException INFO [2019-12-17 12:30:04,554] jepsen worker 53 - jepsen.util 2933 :info :write [17 38] :RequestCanceledException INFO [2019-12-17 12:30:04,554] jepsen worker 28 - jepsen.util 2278 :invoke :write [9 34] INFO [2019-12-17 12:30:04,555] jepsen worker 53 - jepsen.util 3023 :invoke :read [17 nil] INFO [2019-12-17 12:30:04,555] jepsen worker 49 - jepsen.util 3469 :invoke :write [16 49] INFO [2019-12-17 12:30:04,577] jepsen worker 64 - jepsen.util 2494 :ok :write [21 43] INFO [2019-12-17 12:30:04,577] jepsen worker 64 - jepsen.util 2494 :invoke :write [21 45] INFO [2019-12-17 12:30:04,607] jepsen worker 60 - jepsen.util 4200 :info :cas [20 [20 4]] :RequestCanceledException INFO [2019-12-17 12:30:04,608] jepsen worker 60 - jepsen.util 4290 :invoke :write [20 26] INFO [2019-12-17 12:30:04,617] jepsen worker 28 - jepsen.util 2278 :ok :write [9 34] INFO [2019-12-17 12:30:04,617] jepsen worker 28 - jepsen.util 2278 :invoke :cas [9 [35 29]] INFO [2019-12-17 12:30:04,622] jepsen worker 34 - jepsen.util 4084 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:30:04,622] jepsen worker 34 - jepsen.util 4084 :invoke :read [11 nil] INFO [2019-12-17 12:30:04,638] jepsen worker 28 - jepsen.util 2278 :fail :cas [9 [35 29]] :ValueNotSwapFrom INFO [2019-12-17 12:30:04,638] jepsen worker 28 - jepsen.util 2278 :invoke :write [9 16] INFO [2019-12-17 12:30:04,673] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":17479,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:30:04,673] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":17479,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:30:04,673] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":17482,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:30:04,673] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":17482,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:30:04,673] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":17483,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:30:04,673] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":17483,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:30:04,687] jepsen worker 64 - jepsen.util 2494 :ok :write [21 45] INFO [2019-12-17 12:30:04,688] jepsen worker 64 - jepsen.util 2494 :invoke :cas [21 [15 12]] INFO [2019-12-17 12:30:04,706] jepsen worker 64 - jepsen.util 2494 :fail :cas [21 [15 12]] :ValueNotSwapFrom INFO [2019-12-17 12:30:04,707] jepsen worker 64 - jepsen.util 2494 :invoke :cas [21 [36 37]] INFO [2019-12-17 12:30:04,708] jepsen worker 74 - jepsen.util 3134 :info :write [24 5] :RequestCanceledException INFO [2019-12-17 12:30:04,709] jepsen worker 74 - jepsen.util 3224 :invoke :read [24 nil] INFO [2019-12-17 12:30:04,724] jepsen worker 7 - jepsen.util 3607 :info :write [2 33] :RequestCanceledException INFO [2019-12-17 12:30:04,724] jepsen worker 29 - jepsen.util 2459 :info :cas [9 [22 0]] :RequestCanceledException INFO [2019-12-17 12:30:04,725] jepsen worker 29 - jepsen.util 2549 :invoke :read [9 nil] INFO [2019-12-17 12:30:04,725] jepsen worker 7 - jepsen.util 3697 :invoke :cas [2 [0 21]] INFO [2019-12-17 12:30:04,734] jepsen worker 64 - jepsen.util 2494 :fail :cas [21 [36 37]] :ValueNotSwapFrom INFO [2019-12-17 12:30:04,734] jepsen worker 64 - jepsen.util 2494 :invoke :cas [21 [23 44]] INFO [2019-12-17 12:30:04,735] jepsen worker 28 - jepsen.util 2278 :ok :write [9 16] INFO [2019-12-17 12:30:04,735] jepsen worker 28 - jepsen.util 2278 :invoke :write [9 28] INFO [2019-12-17 12:30:04,737] jepsen worker 4 - jepsen.util 2164 :info :write [1 1] :RequestCanceledException INFO [2019-12-17 12:30:04,738] jepsen worker 4 - jepsen.util 2254 :invoke :read [1 nil] INFO [2019-12-17 12:30:04,746] jepsen worker 74 - jepsen.util 3224 :ok :read [24 19] INFO [2019-12-17 12:30:04,747] jepsen worker 74 - jepsen.util 3224 :invoke :read [24 nil] INFO [2019-12-17 12:30:04,755] jepsen worker 23 - jepsen.util 3893 :info :cas [7 [23 47]] :RequestCanceledException INFO [2019-12-17 12:30:04,756] jepsen worker 23 - jepsen.util 3983 :invoke :read [7 nil] INFO [2019-12-17 12:30:04,785] jepsen worker 64 - jepsen.util 2494 :fail :cas [21 [23 44]] :ValueNotSwapFrom INFO [2019-12-17 12:30:04,785] jepsen worker 64 - jepsen.util 2494 :invoke :read [21 nil] INFO [2019-12-17 12:30:04,795] jepsen worker 74 - jepsen.util 3224 :ok :read [24 19] INFO [2019-12-17 12:30:04,795] jepsen worker 74 - jepsen.util 3224 :invoke :read [24 nil] INFO [2019-12-17 12:30:04,812] jepsen worker 64 - jepsen.util 2494 :ok :read [21 45] INFO [2019-12-17 12:30:04,812] jepsen worker 64 - jepsen.util 2494 :invoke :cas [21 [46 7]] INFO [2019-12-17 12:30:04,813] jepsen worker 28 - jepsen.util 2278 :ok :write [9 28] INFO [2019-12-17 12:30:04,813] jepsen worker 28 - jepsen.util 2278 :invoke :write [9 4] INFO [2019-12-17 12:30:04,823] jepsen worker 20 - jepsen.util 2630 :fail :read [6 nil] :RequestTimeoutException INFO [2019-12-17 12:30:04,823] jepsen worker 20 - jepsen.util 2630 :invoke :read [6 nil] INFO [2019-12-17 12:30:04,834] jepsen worker 74 - jepsen.util 3224 :ok :read [24 19] INFO [2019-12-17 12:30:04,835] jepsen worker 74 - jepsen.util 3224 :invoke :read [24 nil] INFO [2019-12-17 12:30:04,837] jepsen worker 47 - jepsen.util 3467 :info :write [15 2] :RequestCanceledException INFO [2019-12-17 12:30:04,837] jepsen worker 27 - jepsen.util 2637 :info :cas [9 [12 48]] :RequestCanceledException INFO [2019-12-17 12:30:04,839] jepsen worker 27 - jepsen.util 2727 :invoke :cas [9 [11 32]] INFO [2019-12-17 12:30:04,839] jepsen worker 47 - jepsen.util 3557 :invoke :write [15 28] INFO [2019-12-17 12:30:04,842] jepsen worker 64 - jepsen.util 2494 :fail :cas [21 [46 7]] :ValueNotSwapFrom INFO [2019-12-17 12:30:04,842] jepsen worker 64 - jepsen.util 2494 :invoke :write [21 21] INFO [2019-12-17 12:30:04,852] jepsen worker 69 - jepsen.util 2949 :fail :cas [23 [11 36]] :ValueNotSwapFrom INFO [2019-12-17 12:30:04,853] jepsen worker 69 - jepsen.util 2949 :invoke :read [23 nil] INFO [2019-12-17 12:30:04,872] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":17506,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:30:04,872] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":17506,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:30:04,873] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":17509,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:30:04,873] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":17509,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:30:04,878] jepsen worker 74 - jepsen.util 3224 :ok :read [24 19] INFO [2019-12-17 12:30:04,878] jepsen worker 74 - jepsen.util 3224 :invoke :write [24 18] INFO [2019-12-17 12:30:04,886] jepsen worker 28 - jepsen.util 2278 :ok :write [9 4] INFO [2019-12-17 12:30:04,886] jepsen worker 28 - jepsen.util 2278 :invoke :read [9 nil] INFO [2019-12-17 12:30:04,892] jepsen worker 80 - jepsen.util 2870 :ok :read [26 27] INFO [2019-12-17 12:30:04,892] jepsen worker 80 - jepsen.util 2870 :invoke :read [26 nil] INFO [2019-12-17 12:30:04,902] jepsen worker 27 - jepsen.util 2727 :fail :cas [9 [11 32]] :ValueNotSwapFrom INFO [2019-12-17 12:30:04,902] jepsen worker 27 - jepsen.util 2727 :invoke :write [9 12] INFO [2019-12-17 12:30:04,914] jepsen worker 28 - jepsen.util 2278 :ok :read [9 4] INFO [2019-12-17 12:30:04,914] jepsen worker 28 - jepsen.util 2278 :invoke :read [9 nil] INFO [2019-12-17 12:30:04,947] jepsen worker 27 - jepsen.util 2727 :ok :write [9 12] INFO [2019-12-17 12:30:04,947] jepsen worker 27 - jepsen.util 2727 :invoke :write [9 9] INFO [2019-12-17 12:30:04,952] jepsen worker 52 - jepsen.util 3202 :ok :read [17 27] INFO [2019-12-17 12:30:04,952] jepsen worker 52 - jepsen.util 3202 :invoke :read [17 nil] INFO [2019-12-17 12:30:04,988] jepsen worker 38 - jepsen.util 2918 :ok :read [12 20] INFO [2019-12-17 12:30:04,988] jepsen worker 38 - jepsen.util 2918 :invoke :write [12 28] INFO [2019-12-17 12:30:04,997] jepsen worker 27 - jepsen.util 2727 :ok :write [9 9] INFO [2019-12-17 12:30:04,997] jepsen worker 27 - jepsen.util 2727 :invoke :read [9 nil] INFO [2019-12-17 12:30:05,010] jepsen worker 69 - jepsen.util 2949 :ok :read [23 30] INFO [2019-12-17 12:30:05,010] jepsen worker 69 - jepsen.util 2949 :invoke :cas [23 [4 20]] INFO [2019-12-17 12:30:05,012] jepsen worker 27 - jepsen.util 2727 :ok :read [9 9] INFO [2019-12-17 12:30:05,012] jepsen worker 27 - jepsen.util 2727 :invoke :write [9 29] INFO [2019-12-17 12:30:05,023] jepsen worker 70 - jepsen.util 3400 :info :write [23 13] :RequestCanceledException INFO [2019-12-17 12:30:05,024] jepsen worker 70 - jepsen.util 3490 :invoke :write [23 37] INFO [2019-12-17 12:30:05,037] jepsen worker 13 - jepsen.util 2803 :info :write [4 7] :RequestCanceledException INFO [2019-12-17 12:30:05,038] jepsen worker 13 - jepsen.util 2893 :invoke :write [4 18] INFO [2019-12-17 12:30:05,044] jepsen worker 18 - jepsen.util 3168 :ok :write [6 17] INFO [2019-12-17 12:30:05,044] jepsen worker 18 - jepsen.util 3168 :invoke :read [6 nil] INFO [2019-12-17 12:30:05,045] jepsen worker 31 - jepsen.util 2641 :ok :write [10 9] INFO [2019-12-17 12:30:05,045] jepsen worker 31 - jepsen.util 2641 :invoke :cas [10 [36 31]] INFO [2019-12-17 12:30:05,047] jepsen worker 45 - jepsen.util 3015 :ok :write [15 48] INFO [2019-12-17 12:30:05,047] jepsen worker 73 - jepsen.util 2863 :ok :write [24 44] INFO [2019-12-17 12:30:05,047] jepsen worker 45 - jepsen.util 3015 :invoke :read [15 nil] INFO [2019-12-17 12:30:05,047] jepsen worker 73 - jepsen.util 2863 :invoke :cas [24 [23 38]] INFO [2019-12-17 12:30:05,063] jepsen worker 10 - jepsen.util 2800 :fail :cas [3 [21 26]] :ValueNotSwapFrom INFO [2019-12-17 12:30:05,063] jepsen worker 10 - jepsen.util 2800 :invoke :write [3 33] INFO [2019-12-17 12:30:05,089] jepsen worker 32 - jepsen.util 3542 :fail :cas [10 [41 10]] :ValueNotSwapFrom INFO [2019-12-17 12:30:05,090] jepsen worker 32 - jepsen.util 3542 :invoke :read [10 nil] INFO [2019-12-17 12:30:05,090] jepsen worker 35 - jepsen.util 3905 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:30:05,090] jepsen worker 35 - jepsen.util 3905 :invoke :write [11 26] INFO [2019-12-17 12:30:05,092] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":17518,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:30:05,092] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":17518,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:30:05,092] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":17520,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:30:05,092] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":17520,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:30:05,123] jepsen worker 87 - jepsen.util 2697 :info :write [29 37] :RequestCanceledException INFO [2019-12-17 12:30:05,123] jepsen worker 16 - jepsen.util 4156 :info :cas [5 [31 19]] :RequestCanceledException INFO [2019-12-17 12:30:05,124] jepsen worker 87 - jepsen.util 2787 :invoke :write [29 13] INFO [2019-12-17 12:30:05,125] jepsen worker 16 - jepsen.util 4246 :invoke :write [5 32] INFO [2019-12-17 12:30:05,134] jepsen worker 63 - jepsen.util 2673 :ok :read [21 45] INFO [2019-12-17 12:30:05,135] jepsen worker 63 - jepsen.util 2673 :invoke :read [21 nil] INFO [2019-12-17 12:30:05,165] jepsen worker 73 - jepsen.util 2863 :fail :cas [24 [23 38]] :ValueNotSwapFrom INFO [2019-12-17 12:30:05,166] jepsen worker 73 - jepsen.util 2863 :invoke :write [24 26] INFO [2019-12-17 12:30:05,176] jepsen worker 45 - jepsen.util 3015 :ok :read [15 48] INFO [2019-12-17 12:30:05,176] jepsen worker 45 - jepsen.util 3015 :invoke :read [15 nil] INFO [2019-12-17 12:30:05,183] jepsen worker 18 - jepsen.util 3168 :ok :read [6 17] INFO [2019-12-17 12:30:05,183] jepsen worker 18 - jepsen.util 3168 :invoke :write [6 18] INFO [2019-12-17 12:30:05,190] jepsen worker 89 - jepsen.util 2699 :info :write [29 43] :RequestCanceledException INFO [2019-12-17 12:30:05,192] jepsen worker 89 - jepsen.util 2789 :invoke :write [29 21] INFO [2019-12-17 12:30:05,195] jepsen worker 29 - jepsen.util 2549 :ok :read [9 9] INFO [2019-12-17 12:30:05,195] jepsen worker 29 - jepsen.util 2549 :invoke :read [9 nil] INFO [2019-12-17 12:30:05,203] jepsen worker 27 - jepsen.util 2727 :ok :write [9 29] INFO [2019-12-17 12:30:05,204] jepsen worker 27 - jepsen.util 2727 :invoke :read [9 nil] INFO [2019-12-17 12:30:05,224] jepsen worker 36 - jepsen.util 2916 :info :write [12 38] :RequestCanceledException INFO [2019-12-17 12:30:05,224] jepsen worker 59 - jepsen.util 3929 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:30:05,224] jepsen worker 59 - jepsen.util 3929 :invoke :cas [19 [49 19]] INFO [2019-12-17 12:30:05,225] jepsen worker 36 - jepsen.util 3006 :invoke :read [12 nil] INFO [2019-12-17 12:30:05,240] jepsen worker 53 - jepsen.util 3023 :ok :read [17 27] INFO [2019-12-17 12:30:05,240] jepsen worker 53 - jepsen.util 3023 :invoke :read [17 nil] INFO [2019-12-17 12:30:05,247] jepsen worker 32 - jepsen.util 3542 :ok :read [10 9] INFO [2019-12-17 12:30:05,247] jepsen worker 32 - jepsen.util 3542 :invoke :read [10 nil] INFO [2019-12-17 12:30:05,251] jepsen worker 31 - jepsen.util 2641 :fail :cas [10 [36 31]] :ValueNotSwapFrom INFO [2019-12-17 12:30:05,252] jepsen worker 31 - jepsen.util 2641 :invoke :write [10 7] INFO [2019-12-17 12:30:05,260] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":17544,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:30:05,260] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":17553,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:30:05,260] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":17544,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:30:05,260] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":17553,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:30:05,260] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":17556,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:30:05,260] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":17556,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:30:05,322] jepsen worker 27 - jepsen.util 2727 :ok :read [9 29] INFO [2019-12-17 12:30:05,322] jepsen worker 27 - jepsen.util 2727 :invoke :read [9 nil] INFO [2019-12-17 12:30:05,323] jepsen worker 43 - jepsen.util 3643 :info :write [14 35] :RequestCanceledException INFO [2019-12-17 12:30:05,324] jepsen worker 43 - jepsen.util 3733 :invoke :read [14 nil] INFO [2019-12-17 12:30:05,330] jepsen worker 63 - jepsen.util 2673 :ok :read [21 45] INFO [2019-12-17 12:30:05,330] jepsen worker 63 - jepsen.util 2673 :invoke :write [21 25] INFO [2019-12-17 12:30:05,336] jepsen worker 15 - jepsen.util 3615 :info :write [5 19] :RequestCanceledException INFO [2019-12-17 12:30:05,337] jepsen worker 15 - jepsen.util 3705 :invoke :cas [5 [28 14]] INFO [2019-12-17 12:30:05,354] jepsen worker 85 - jepsen.util 4495 :info :write [28 38] :RequestCanceledException INFO [2019-12-17 12:30:05,354] jepsen worker 84 - jepsen.util 3864 :info :cas [28 [25 31]] :RequestCanceledException INFO [2019-12-17 12:30:05,356] jepsen worker 84 - jepsen.util 3954 :invoke :write [28 15] INFO [2019-12-17 12:30:05,356] jepsen worker 85 - jepsen.util 4585 :invoke :read [28 nil] INFO [2019-12-17 12:30:05,361] jepsen worker 79 - jepsen.util 2959 :fail :cas [26 [11 32]] :ValueNotSwapFrom INFO [2019-12-17 12:30:05,361] jepsen worker 79 - jepsen.util 2959 :invoke :write [26 21] INFO [2019-12-17 12:30:05,368] jepsen worker 29 - jepsen.util 2549 :ok :read [9 29] INFO [2019-12-17 12:30:05,369] jepsen worker 29 - jepsen.util 2549 :invoke :read [9 nil] INFO [2019-12-17 12:30:05,373] jepsen worker 27 - jepsen.util 2727 :ok :read [9 29] INFO [2019-12-17 12:30:05,376] jepsen worker 27 - jepsen.util 2727 :invoke :read [9 nil] INFO [2019-12-17 12:30:05,399] jepsen worker 27 - jepsen.util 2727 :ok :read [9 29] INFO [2019-12-17 12:30:05,399] jepsen worker 27 - jepsen.util 2727 :invoke :cas [9 [40 19]] INFO [2019-12-17 12:30:05,424] jepsen worker 30 - jepsen.util 3540 :info :write [10 31] :RequestCanceledException INFO [2019-12-17 12:30:05,424] jepsen worker 5 - jepsen.util 2525 :info :write [1 46] :RequestCanceledException INFO [2019-12-17 12:30:05,425] jepsen worker 30 - jepsen.util 3630 :invoke :read [10 nil] INFO [2019-12-17 12:30:05,425] jepsen worker 5 - jepsen.util 2615 :invoke :read [1 nil] INFO [2019-12-17 12:30:05,435] jepsen worker 37 - jepsen.util 2647 :info :write [12 46] :RequestCanceledException INFO [2019-12-17 12:30:05,437] jepsen worker 37 - jepsen.util 2737 :invoke :read [12 nil] INFO [2019-12-17 12:30:05,441] jepsen worker 27 - jepsen.util 2727 :fail :cas [9 [40 19]] :ValueNotSwapFrom INFO [2019-12-17 12:30:05,441] jepsen worker 27 - jepsen.util 2727 :invoke :cas [9 [18 49]] INFO [2019-12-17 12:30:05,454] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":17575,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:30:05,454] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":17575,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:30:05,454] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":17578,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:30:05,455] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":17578,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:30:05,456] jepsen worker 30 - jepsen.util 3630 :ok :read [10 9] INFO [2019-12-17 12:30:05,456] jepsen worker 30 - jepsen.util 3630 :invoke :read [10 nil] INFO [2019-12-17 12:30:05,459] jepsen worker 52 - jepsen.util 3202 :ok :read [17 27] INFO [2019-12-17 12:30:05,459] jepsen worker 52 - jepsen.util 3202 :invoke :cas [17 [41 33]] INFO [2019-12-17 12:30:05,471] jepsen worker 27 - jepsen.util 2727 :fail :cas [9 [18 49]] :ValueNotSwapFrom INFO [2019-12-17 12:30:05,471] jepsen worker 27 - jepsen.util 2727 :invoke :cas [9 [16 12]] INFO [2019-12-17 12:30:05,472] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:30:05,472] jepsen nemesis - couchbase.util Rebalance check count: 128 INFO [2019-12-17 12:30:05,472] jepsen nemesis - couchbase.util Rebalance stuck count: 109 INFO [2019-12-17 12:30:05,484] jepsen worker 30 - jepsen.util 3630 :ok :read [10 9] INFO [2019-12-17 12:30:05,485] jepsen worker 30 - jepsen.util 3630 :invoke :write [10 0] INFO [2019-12-17 12:30:05,490] jepsen worker 62 - jepsen.util 3842 :info :write [20 19] :RequestCanceledException INFO [2019-12-17 12:30:05,491] jepsen worker 27 - jepsen.util 2727 :fail :cas [9 [16 12]] :ValueNotSwapFrom INFO [2019-12-17 12:30:05,491] jepsen worker 27 - jepsen.util 2727 :invoke :read [9 nil] INFO [2019-12-17 12:30:05,491] jepsen worker 62 - jepsen.util 3932 :invoke :write [20 25] INFO [2019-12-17 12:30:05,508] jepsen worker 50 - jepsen.util 4460 :info :cas [16 [19 35]] :RequestCanceledException INFO [2019-12-17 12:30:05,510] jepsen worker 50 - jepsen.util 4550 :invoke :write [16 29] INFO [2019-12-17 12:30:05,523] jepsen worker 19 - jepsen.util 3169 :info :cas [6 [43 31]] :RequestCanceledException INFO [2019-12-17 12:30:05,523] jepsen worker 1 - jepsen.util 3511 :info :write [0 37] :RequestCanceledException INFO [2019-12-17 12:30:05,524] jepsen worker 1 - jepsen.util 3601 :invoke :read [0 nil] INFO [2019-12-17 12:30:05,524] jepsen worker 19 - jepsen.util 3259 :invoke :read [6 nil] INFO [2019-12-17 12:30:05,533] jepsen worker 4 - jepsen.util 2254 :ok :read [1 26] INFO [2019-12-17 12:30:05,533] jepsen worker 4 - jepsen.util 2254 :invoke :cas [1 [2 44]] INFO [2019-12-17 12:30:05,537] jepsen worker 40 - jepsen.util 4270 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:30:05,537] jepsen worker 40 - jepsen.util 4270 :invoke :cas [13 [1 6]] INFO [2019-12-17 12:30:05,538] jepsen worker 19 - jepsen.util 3259 :ok :read [6 17] INFO [2019-12-17 12:30:05,538] jepsen worker 19 - jepsen.util 3259 :invoke :read [6 nil] INFO [2019-12-17 12:30:05,550] jepsen worker 19 - jepsen.util 3259 :ok :read [6 17] INFO [2019-12-17 12:30:05,551] jepsen worker 19 - jepsen.util 3259 :invoke :write [6 9] INFO [2019-12-17 12:30:05,554] jepsen worker 21 - jepsen.util 3981 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:30:05,555] jepsen worker 21 - jepsen.util 3981 :invoke :cas [7 [24 45]] INFO [2019-12-17 12:30:05,590] jepsen worker 22 - jepsen.util 4072 :info :write [7 32] :RequestCanceledException INFO [2019-12-17 12:30:05,591] jepsen worker 22 - jepsen.util 4162 :invoke :cas [7 [44 6]] INFO [2019-12-17 12:30:05,608] jepsen worker 65 - jepsen.util 2675 :info :cas [21 [5 25]] :RequestCanceledException INFO [2019-12-17 12:30:05,608] jepsen worker 51 - jepsen.util 2751 :info :write [17 7] :RequestCanceledException INFO [2019-12-17 12:30:05,608] jepsen worker 11 - jepsen.util 3341 :info :write [3 49] :RequestCanceledException INFO [2019-12-17 12:30:05,609] jepsen worker 51 - jepsen.util 2841 :invoke :cas [17 [32 26]] INFO [2019-12-17 12:30:05,609] jepsen worker 11 - jepsen.util 3431 :invoke :write [3 2] INFO [2019-12-17 12:30:05,609] jepsen worker 65 - jepsen.util 2765 :invoke :read [21 nil] INFO [2019-12-17 12:30:05,623] jepsen worker 69 - jepsen.util 2949 :fail :cas [23 [4 20]] :ValueNotSwapFrom INFO [2019-12-17 12:30:05,623] jepsen worker 69 - jepsen.util 2949 :invoke :write [23 26] INFO [2019-12-17 12:30:05,623] jepsen worker 75 - jepsen.util 4125 :info :write [25 7] :RequestCanceledException INFO [2019-12-17 12:30:05,624] jepsen worker 75 - jepsen.util 4215 :invoke :write [25 44] INFO [2019-12-17 12:30:05,654] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":17604,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:30:05,654] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":17604,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:30:05,654] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":17607,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:30:05,654] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":17607,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:30:05,670] jepsen worker 80 - jepsen.util 2870 :ok :read [26 27] INFO [2019-12-17 12:30:05,670] jepsen worker 80 - jepsen.util 2870 :invoke :cas [26 [32 19]] INFO [2019-12-17 12:30:05,708] jepsen worker 41 - jepsen.util 4181 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:30:05,708] jepsen worker 41 - jepsen.util 4181 :invoke :write [13 42] INFO [2019-12-17 12:30:05,711] jepsen worker 53 - jepsen.util 3023 :ok :read [17 27] INFO [2019-12-17 12:30:05,711] jepsen worker 53 - jepsen.util 3023 :invoke :read [17 nil] INFO [2019-12-17 12:30:05,723] jepsen worker 0 - jepsen.util 4410 :info :write [0 27] :RequestCanceledException INFO [2019-12-17 12:30:05,723] jepsen worker 57 - jepsen.util 3747 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:30:05,723] jepsen worker 57 - jepsen.util 3747 :invoke :cas [19 [43 48]] INFO [2019-12-17 12:30:05,724] jepsen worker 0 - jepsen.util 4500 :invoke :write [0 7] INFO [2019-12-17 12:30:05,735] jepsen worker 88 - jepsen.util 2968 :info :write [29 28] :RequestCanceledException INFO [2019-12-17 12:30:05,735] jepsen worker 42 - jepsen.util 4632 :info :write [14 21] :RequestCanceledException INFO [2019-12-17 12:30:05,736] jepsen worker 42 - jepsen.util 4722 :invoke :write [14 6] INFO [2019-12-17 12:30:05,736] jepsen worker 88 - jepsen.util 3058 :invoke :read [29 nil] INFO [2019-12-17 12:30:05,754] jepsen worker 3 - jepsen.util 2883 :info :write [1 25] :RequestCanceledException INFO [2019-12-17 12:30:05,754] jepsen worker 33 - jepsen.util 4263 :info :write [11 21] :RequestCanceledException INFO [2019-12-17 12:30:05,755] jepsen worker 33 - jepsen.util 4353 :invoke :write [11 36] INFO [2019-12-17 12:30:05,755] jepsen worker 3 - jepsen.util 2973 :invoke :write [1 26] INFO [2019-12-17 12:30:05,789] jepsen worker 8 - jepsen.util 3338 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:30:05,789] jepsen worker 83 - jepsen.util 4133 :info :cas [27 [49 26]] :RequestCanceledException INFO [2019-12-17 12:30:05,790] jepsen worker 8 - jepsen.util 3338 :invoke :cas [2 [33 5]] INFO [2019-12-17 12:30:05,791] jepsen worker 83 - jepsen.util 4223 :invoke :read [27 nil] INFO [2019-12-17 12:30:05,800] jepsen worker 36 - jepsen.util 3006 :ok :read [12 20] INFO [2019-12-17 12:30:05,800] jepsen worker 36 - jepsen.util 3006 :invoke :cas [12 [3 44]] INFO [2019-12-17 12:30:05,808] jepsen worker 48 - jepsen.util 3468 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:30:05,808] jepsen worker 48 - jepsen.util 3468 :invoke :write [16 32] INFO [2019-12-17 12:30:05,823] jepsen worker 53 - jepsen.util 3023 :ok :read [17 27] INFO [2019-12-17 12:30:05,823] jepsen worker 53 - jepsen.util 3023 :invoke :write [17 32] INFO [2019-12-17 12:30:05,837] jepsen worker 26 - jepsen.util 4256 :info :write [8 4] :RequestCanceledException INFO [2019-12-17 12:30:05,838] jepsen worker 26 - jepsen.util 4346 :invoke :cas [8 [5 40]] INFO [2019-12-17 12:30:05,849] jepsen worker 4 - jepsen.util 2254 :fail :cas [1 [2 44]] :ValueNotSwapFrom INFO [2019-12-17 12:30:05,849] jepsen worker 4 - jepsen.util 2254 :invoke :write [1 18] INFO [2019-12-17 12:30:05,852] jepsen worker 54 - jepsen.util 3024 :info :write [18 27] :RequestCanceledException INFO [2019-12-17 12:30:05,852] jepsen worker 55 - jepsen.util 3655 :info :write [18 5] :RequestCanceledException INFO [2019-12-17 12:30:05,852] jepsen worker 17 - jepsen.util 3977 :info :cas [5 [31 16]] :RequestCanceledException INFO [2019-12-17 12:30:05,853] jepsen worker 54 - jepsen.util 3114 :invoke :cas [18 [39 0]] INFO [2019-12-17 12:30:05,854] jepsen worker 55 - jepsen.util 3745 :invoke :cas [18 [1 39]] INFO [2019-12-17 12:30:05,854] jepsen worker 17 - jepsen.util 4067 :invoke :read [5 nil] INFO [2019-12-17 12:30:05,858] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":17629,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:30:05,858] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":17629,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:30:05,858] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":17633,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:30:05,858] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":17633,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:30:05,902] jepsen worker 5 - jepsen.util 2615 :ok :read [1 26] INFO [2019-12-17 12:30:05,902] jepsen worker 5 - jepsen.util 2615 :invoke :cas [1 [35 30]] INFO [2019-12-17 12:30:05,916] jepsen worker 52 - jepsen.util 3202 :fail :cas [17 [41 33]] :ValueNotSwapFrom INFO [2019-12-17 12:30:05,916] jepsen worker 52 - jepsen.util 3202 :invoke :cas [17 [42 37]] INFO [2019-12-17 12:30:05,922] jepsen worker 80 - jepsen.util 2870 :fail :cas [26 [32 19]] :ValueNotSwapFrom INFO [2019-12-17 12:30:05,922] jepsen worker 80 - jepsen.util 2870 :invoke :cas [26 [25 2]] INFO [2019-12-17 12:30:05,994] jepsen worker 80 - jepsen.util 2870 :fail :cas [26 [25 2]] :ValueNotSwapFrom INFO [2019-12-17 12:30:05,994] jepsen worker 80 - jepsen.util 2870 :invoke :cas [26 [20 0]] INFO [2019-12-17 12:30:06,023] jepsen worker 72 - jepsen.util 3042 :info :write [24 36] :RequestCanceledException INFO [2019-12-17 12:30:06,024] jepsen worker 72 - jepsen.util 3132 :invoke :read [24 nil] INFO [2019-12-17 12:30:06,054] jepsen worker 36 - jepsen.util 3006 :fail :cas [12 [3 44]] :ValueNotSwapFrom INFO [2019-12-17 12:30:06,054] jepsen worker 36 - jepsen.util 3006 :invoke :read [12 nil] INFO [2019-12-17 12:30:06,067] jepsen worker 37 - jepsen.util 2737 :ok :read [12 20] INFO [2019-12-17 12:30:06,068] jepsen worker 37 - jepsen.util 2737 :invoke :write [12 41] INFO [2019-12-17 12:30:06,077] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":17640,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:30:06,077] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":17640,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:30:06,106] jepsen worker 54 - jepsen.util 3114 :fail :cas [18 [39 0]] :ValueNotSwapFrom INFO [2019-12-17 12:30:06,106] jepsen worker 54 - jepsen.util 3114 :invoke :cas [18 [22 49]] INFO [2019-12-17 12:30:06,128] jepsen worker 5 - jepsen.util 2615 :fail :cas [1 [35 30]] :ValueNotSwapFrom INFO [2019-12-17 12:30:06,128] jepsen worker 5 - jepsen.util 2615 :invoke :read [1 nil] INFO [2019-12-17 12:30:06,137] jepsen worker 51 - jepsen.util 2841 :fail :cas [17 [32 26]] :ValueNotSwapFrom INFO [2019-12-17 12:30:06,138] jepsen worker 51 - jepsen.util 2841 :invoke :cas [17 [35 46]] INFO [2019-12-17 12:30:06,141] jepsen worker 52 - jepsen.util 3202 :fail :cas [17 [42 37]] :ValueNotSwapFrom INFO [2019-12-17 12:30:06,141] jepsen worker 52 - jepsen.util 3202 :invoke :cas [17 [18 22]] INFO [2019-12-17 12:30:06,154] jepsen worker 81 - jepsen.util 4041 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:30:06,155] jepsen worker 81 - jepsen.util 4041 :invoke :write [27 9] INFO [2019-12-17 12:30:06,190] jepsen worker 25 - jepsen.util 4435 :info :write [8 49] :RequestCanceledException INFO [2019-12-17 12:30:06,190] jepsen worker 25 - jepsen.util 4525 :invoke :cas [8 [38 29]] INFO [2019-12-17 12:30:06,202] jepsen worker 36 - jepsen.util 3006 :ok :read [12 20] INFO [2019-12-17 12:30:06,202] jepsen worker 36 - jepsen.util 3006 :invoke :read [12 nil] INFO [2019-12-17 12:30:06,222] jepsen worker 76 - jepsen.util 4126 :info :cas [25 [6 40]] :RequestCanceledException INFO [2019-12-17 12:30:06,222] jepsen worker 9 - jepsen.util 3429 :info :write [3 18] :RequestCanceledException INFO [2019-12-17 12:30:06,223] jepsen worker 76 - jepsen.util 4216 :invoke :read [25 nil] INFO [2019-12-17 12:30:06,223] jepsen worker 9 - jepsen.util 3519 :invoke :read [3 nil] INFO [2019-12-17 12:30:06,236] jepsen worker 9 - jepsen.util 3519 :ok :read [3 23] INFO [2019-12-17 12:30:06,236] jepsen worker 9 - jepsen.util 3519 :invoke :write [3 12] INFO [2019-12-17 12:30:06,237] jepsen worker 46 - jepsen.util 3196 :info :write [15 39] :RequestCanceledException INFO [2019-12-17 12:30:06,238] jepsen worker 46 - jepsen.util 3286 :invoke :cas [15 [45 0]] INFO [2019-12-17 12:30:06,247] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":17652,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:30:06,247] jepsen worker 5 - jepsen.util 2615 :ok :read [1 26] INFO [2019-12-17 12:30:06,247] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":17652,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:30:06,247] jepsen worker 5 - jepsen.util 2615 :invoke :cas [1 [26 15]] INFO [2019-12-17 12:30:06,247] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":17653,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:30:06,247] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":17653,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:30:06,248] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":17654,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:30:06,248] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":17661,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:30:06,248] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":17654,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:30:06,248] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":17661,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:30:06,248] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":17665,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:30:06,248] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":17665,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:30:06,254] jepsen worker 58 - jepsen.util 3928 :info :cas [19 [45 35]] :RequestCanceledException INFO [2019-12-17 12:30:06,255] jepsen worker 58 - jepsen.util 4018 :invoke :read [19 nil] INFO [2019-12-17 12:30:06,277] jepsen worker 51 - jepsen.util 2841 :fail :cas [17 [35 46]] :ValueNotSwapFrom INFO [2019-12-17 12:30:06,277] jepsen worker 51 - jepsen.util 2841 :invoke :write [17 33] INFO [2019-12-17 12:30:06,307] jepsen worker 71 - jepsen.util 2951 :info :write [23 23] :RequestCanceledException INFO [2019-12-17 12:30:06,308] jepsen worker 71 - jepsen.util 3041 :invoke :read [23 nil] INFO [2019-12-17 12:30:06,318] jepsen worker 52 - jepsen.util 3202 :fail :cas [17 [18 22]] :ValueNotSwapFrom INFO [2019-12-17 12:30:06,318] jepsen worker 52 - jepsen.util 3202 :invoke :read [17 nil] INFO [2019-12-17 12:30:06,324] jepsen worker 86 - jepsen.util 4226 :info :write [28 41] :RequestCanceledException INFO [2019-12-17 12:30:06,329] jepsen worker 86 - jepsen.util 4316 :invoke :cas [28 [33 45]] INFO [2019-12-17 12:30:06,352] jepsen worker 54 - jepsen.util 3114 :fail :cas [18 [22 49]] :ValueNotSwapFrom INFO [2019-12-17 12:30:06,353] jepsen worker 54 - jepsen.util 3114 :invoke :write [18 35] INFO [2019-12-17 12:30:06,355] jepsen worker 61 - jepsen.util 4471 :info :cas [20 [32 33]] :RequestCanceledException INFO [2019-12-17 12:30:06,355] jepsen worker 82 - jepsen.util 4762 :info :cas [27 [23 40]] :RequestCanceledException INFO [2019-12-17 12:30:06,355] jepsen worker 12 - jepsen.util 3072 :info :write [4 44] :RequestCanceledException INFO [2019-12-17 12:30:06,355] jepsen worker 39 - jepsen.util 3909 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:30:06,355] jepsen worker 39 - jepsen.util 3909 :invoke :read [13 nil] INFO [2019-12-17 12:30:06,356] jepsen worker 82 - jepsen.util 4852 :invoke :cas [27 [4 20]] INFO [2019-12-17 12:30:06,356] jepsen worker 12 - jepsen.util 3162 :invoke :write [4 23] INFO [2019-12-17 12:30:06,356] jepsen worker 61 - jepsen.util 4561 :invoke :cas [20 [23 45]] INFO [2019-12-17 12:30:06,363] jepsen worker 88 - jepsen.util 3058 :ok :read [29 17] INFO [2019-12-17 12:30:06,363] jepsen worker 88 - jepsen.util 3058 :invoke :read [29 nil] INFO [2019-12-17 12:30:06,386] jepsen worker 56 - jepsen.util 2846 :info :write [18 32] :RequestCanceledException INFO [2019-12-17 12:30:06,386] jepsen worker 2 - jepsen.util 3692 :info :write [0 1] :RequestCanceledException INFO [2019-12-17 12:30:06,386] jepsen worker 66 - jepsen.util 3936 :info :write [22 42] :RequestCanceledException INFO [2019-12-17 12:30:06,387] jepsen worker 78 - jepsen.util 3228 :info :write [26 19] :RequestCanceledException INFO [2019-12-17 12:30:06,387] jepsen worker 78 - jepsen.util 3318 :invoke :write [26 8] INFO [2019-12-17 12:30:06,387] jepsen worker 2 - jepsen.util 3782 :invoke :read [0 nil] INFO [2019-12-17 12:30:06,388] jepsen worker 66 - jepsen.util 4026 :invoke :read [22 nil] INFO [2019-12-17 12:30:06,389] jepsen worker 56 - jepsen.util 2936 :invoke :read [18 nil] INFO [2019-12-17 12:30:06,401] jepsen worker 55 - jepsen.util 3745 :fail :cas [18 [1 39]] :ValueNotSwapFrom INFO [2019-12-17 12:30:06,401] jepsen worker 55 - jepsen.util 3745 :invoke :cas [18 [11 40]] INFO [2019-12-17 12:30:06,421] jepsen worker 68 - jepsen.util 3758 :info :write [22 0] :RequestCanceledException INFO [2019-12-17 12:30:06,421] jepsen worker 24 - jepsen.util 3894 :info :cas [8 [44 25]] :RequestCanceledException INFO [2019-12-17 12:30:06,421] jepsen worker 36 - jepsen.util 3006 :ok :read [12 20] INFO [2019-12-17 12:30:06,421] jepsen worker 36 - jepsen.util 3006 :invoke :write [12 0] INFO [2019-12-17 12:30:06,422] jepsen worker 24 - jepsen.util 3984 :invoke :read [8 nil] INFO [2019-12-17 12:30:06,422] jepsen worker 68 - jepsen.util 3848 :invoke :write [22 19] INFO [2019-12-17 12:30:06,437] jepsen worker 77 - jepsen.util 3677 :info :write [25 23] :RequestCanceledException INFO [2019-12-17 12:30:06,438] jepsen worker 77 - jepsen.util 3767 :invoke :cas [25 [34 22]] INFO [2019-12-17 12:30:06,441] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":17689,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:30:06,442] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":17689,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:30:06,442] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":17690,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:30:06,442] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":17690,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:30:06,442] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":17691,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:30:06,442] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":17691,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:30:06,486] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:30:06,490] jepsen worker 44 - jepsen.util 4184 :info :write [14 7] :RequestCanceledException INFO [2019-12-17 12:30:06,491] jepsen worker 80 - jepsen.util 2870 :fail :cas [26 [20 0]] :ValueNotSwapFrom INFO [2019-12-17 12:30:06,505] jepsen worker 67 - jepsen.util 4477 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:30:06,537] jepsen worker 14 - jepsen.util 2804 :info :write [4 37] :RequestCanceledException INFO [2019-12-17 12:30:06,551] jepsen worker 56 - jepsen.util 2936 :ok :read [18 31] INFO [2019-12-17 12:30:06,586] jepsen nemesis - couchbase.util Rebalance check count: 129 INFO [2019-12-17 12:30:06,586] jepsen worker 80 - jepsen.util 2870 :invoke :read [26 nil] INFO [2019-12-17 12:30:06,586] jepsen worker 67 - jepsen.util 4477 :invoke :write [22 16] INFO [2019-12-17 12:30:06,586] jepsen worker 56 - jepsen.util 2936 :invoke :cas [18 [42 6]] INFO [2019-12-17 12:30:06,586] jepsen nemesis - couchbase.util Rebalance stuck count: 110 INFO [2019-12-17 12:30:06,586] jepsen worker 44 - jepsen.util 4274 :invoke :write [14 46] INFO [2019-12-17 12:30:06,587] jepsen worker 14 - jepsen.util 2894 :invoke :cas [4 [33 49]] INFO [2019-12-17 12:30:06,613] jepsen worker 56 - jepsen.util 2936 :fail :cas [18 [42 6]] :ValueNotSwapFrom INFO [2019-12-17 12:30:06,613] jepsen worker 56 - jepsen.util 2936 :invoke :read [18 nil] INFO [2019-12-17 12:30:06,623] jepsen worker 6 - jepsen.util 4236 :info :cas [2 [32 20]] :RequestCanceledException INFO [2019-12-17 12:30:06,623] jepsen worker 60 - jepsen.util 4290 :info :write [20 26] :RequestCanceledException INFO [2019-12-17 12:30:06,624] jepsen worker 60 - jepsen.util 4380 :invoke :read [20 nil] INFO [2019-12-17 12:30:06,624] jepsen worker 6 - jepsen.util 4326 :invoke :write [2 11] INFO [2019-12-17 12:30:06,655] jepsen worker 49 - jepsen.util 3469 :info :write [16 49] :RequestCanceledException INFO [2019-12-17 12:30:06,655] jepsen worker 49 - jepsen.util 3559 :invoke :write [16 14] INFO [2019-12-17 12:30:06,672] jepsen worker 56 - jepsen.util 2936 :ok :read [18 31] INFO [2019-12-17 12:30:06,672] jepsen worker 56 - jepsen.util 2936 :invoke :read [18 nil] INFO [2019-12-17 12:30:06,678] jepsen worker 55 - jepsen.util 3745 :fail :cas [18 [11 40]] :ValueNotSwapFrom INFO [2019-12-17 12:30:06,678] jepsen worker 55 - jepsen.util 3745 :invoke :cas [18 [25 4]] INFO [2019-12-17 12:30:06,717] jepsen worker 52 - jepsen.util 3202 :ok :read [17 27] INFO [2019-12-17 12:30:06,717] jepsen worker 52 - jepsen.util 3202 :invoke :write [17 23] INFO [2019-12-17 12:30:06,724] jepsen worker 34 - jepsen.util 4084 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:30:06,724] jepsen worker 34 - jepsen.util 4084 :invoke :write [11 2] INFO [2019-12-17 12:30:06,741] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":17707,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:30:06,741] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":17707,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:30:06,741] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":17711,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:30:06,741] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":17711,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:30:06,750] jepsen worker 14 - jepsen.util 2894 :fail :cas [4 [33 49]] :ValueNotSwapFrom INFO [2019-12-17 12:30:06,750] jepsen worker 14 - jepsen.util 2894 :invoke :cas [4 [39 45]] INFO [2019-12-17 12:30:06,769] jepsen worker 14 - jepsen.util 2894 :fail :cas [4 [39 45]] :ValueNotSwapFrom INFO [2019-12-17 12:30:06,769] jepsen worker 14 - jepsen.util 2894 :invoke :read [4 nil] INFO [2019-12-17 12:30:06,809] jepsen worker 14 - jepsen.util 2894 :ok :read [4 34] INFO [2019-12-17 12:30:06,809] jepsen worker 14 - jepsen.util 2894 :invoke :write [4 41] INFO [2019-12-17 12:30:06,822] jepsen worker 7 - jepsen.util 3697 :info :cas [2 [0 21]] :RequestCanceledException INFO [2019-12-17 12:30:06,822] jepsen worker 88 - jepsen.util 3058 :ok :read [29 17] INFO [2019-12-17 12:30:06,822] jepsen worker 88 - jepsen.util 3058 :invoke :write [29 33] INFO [2019-12-17 12:30:06,823] jepsen worker 7 - jepsen.util 3787 :invoke :cas [2 [14 39]] INFO [2019-12-17 12:30:06,855] jepsen worker 23 - jepsen.util 3983 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:30:06,855] jepsen worker 23 - jepsen.util 3983 :invoke :read [7 nil] INFO [2019-12-17 12:30:06,889] jepsen worker 47 - jepsen.util 3557 :info :write [15 28] :RequestCanceledException INFO [2019-12-17 12:30:06,890] jepsen worker 47 - jepsen.util 3647 :invoke :write [15 47] INFO [2019-12-17 12:30:06,894] jepsen worker 64 - jepsen.util 2494 :info :write [21 21] :RequestCanceledException INFO [2019-12-17 12:30:06,895] jepsen worker 64 - jepsen.util 2584 :invoke :read [21 nil] INFO [2019-12-17 12:30:06,897] jepsen worker 56 - jepsen.util 2936 :ok :read [18 31] INFO [2019-12-17 12:30:06,897] jepsen worker 56 - jepsen.util 2936 :invoke :write [18 35] INFO [2019-12-17 12:30:06,900] jepsen worker 73 - jepsen.util 2863 :ok :write [24 26] INFO [2019-12-17 12:30:06,900] jepsen worker 73 - jepsen.util 2863 :invoke :write [24 48] INFO [2019-12-17 12:30:06,920] jepsen worker 31 - jepsen.util 2641 :ok :write [10 7] INFO [2019-12-17 12:30:06,920] jepsen worker 31 - jepsen.util 2641 :invoke :read [10 nil] INFO [2019-12-17 12:30:06,921] jepsen worker 18 - jepsen.util 3168 :ok :write [6 18] INFO [2019-12-17 12:30:06,921] jepsen worker 18 - jepsen.util 3168 :invoke :write [6 30] INFO [2019-12-17 12:30:06,922] jepsen worker 20 - jepsen.util 2630 :fail :read [6 nil] :RequestTimeoutException INFO [2019-12-17 12:30:06,922] jepsen worker 20 - jepsen.util 2630 :invoke :write [6 43] INFO [2019-12-17 12:30:06,924] jepsen worker 64 - jepsen.util 2584 :ok :read [21 45] INFO [2019-12-17 12:30:06,925] jepsen worker 64 - jepsen.util 2584 :invoke :write [21 11] INFO [2019-12-17 12:30:06,935] jepsen worker 10 - jepsen.util 2800 :ok :write [3 33] INFO [2019-12-17 12:30:06,935] jepsen worker 10 - jepsen.util 2800 :invoke :cas [3 [15 13]] INFO [2019-12-17 12:30:06,954] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":17725,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:30:06,954] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":17725,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:30:06,980] jepsen worker 71 - jepsen.util 3041 :ok :read [23 30] INFO [2019-12-17 12:30:06,980] jepsen worker 71 - jepsen.util 3041 :invoke :cas [23 [13 12]] INFO [2019-12-17 12:30:06,990] jepsen worker 74 - jepsen.util 3224 :info :write [24 18] :RequestCanceledException INFO [2019-12-17 12:30:06,991] jepsen worker 74 - jepsen.util 3314 :invoke :read [24 nil] INFO [2019-12-17 12:30:07,006] jepsen worker 38 - jepsen.util 2918 :info :write [12 28] :RequestCanceledException INFO [2019-12-17 12:30:07,007] jepsen worker 38 - jepsen.util 3008 :invoke :read [12 nil] INFO [2019-12-17 12:30:07,010] jepsen worker 65 - jepsen.util 2765 :ok :read [21 45] INFO [2019-12-17 12:30:07,011] jepsen worker 65 - jepsen.util 2765 :invoke :write [21 44] INFO [2019-12-17 12:30:07,022] jepsen worker 28 - jepsen.util 2278 :fail :read [9 nil] :RequestTimeoutException INFO [2019-12-17 12:30:07,022] jepsen worker 28 - jepsen.util 2278 :invoke :read [9 nil] INFO [2019-12-17 12:30:07,033] jepsen worker 46 - jepsen.util 3286 :fail :cas [15 [45 0]] :ValueNotSwapFrom INFO [2019-12-17 12:30:07,033] jepsen worker 46 - jepsen.util 3286 :invoke :read [15 nil] INFO [2019-12-17 12:30:07,041] jepsen worker 45 - jepsen.util 3015 :ok :read [15 48] INFO [2019-12-17 12:30:07,042] jepsen worker 55 - jepsen.util 3745 :fail :cas [18 [25 4]] :ValueNotSwapFrom INFO [2019-12-17 12:30:07,042] jepsen worker 55 - jepsen.util 3745 :invoke :write [18 4] INFO [2019-12-17 12:30:07,043] jepsen worker 45 - jepsen.util 3015 :invoke :read [15 nil] INFO [2019-12-17 12:30:07,050] jepsen worker 72 - jepsen.util 3132 :ok :read [24 26] INFO [2019-12-17 12:30:07,050] jepsen worker 72 - jepsen.util 3132 :invoke :read [24 nil] INFO [2019-12-17 12:30:07,060] jepsen worker 31 - jepsen.util 2641 :ok :read [10 7] INFO [2019-12-17 12:30:07,061] jepsen worker 31 - jepsen.util 2641 :invoke :cas [10 [0 34]] INFO [2019-12-17 12:30:07,063] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":17740,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:30:07,063] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":17740,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:30:07,063] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":17743,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:30:07,063] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":17743,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:30:07,064] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":17745,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:30:07,064] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":17745,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:30:07,080] jepsen worker 46 - jepsen.util 3286 :ok :read [15 48] INFO [2019-12-17 12:30:07,080] jepsen worker 46 - jepsen.util 3286 :invoke :write [15 5] INFO [2019-12-17 12:30:07,100] jepsen worker 31 - jepsen.util 2641 :fail :cas [10 [0 34]] :ValueNotSwapFrom INFO [2019-12-17 12:30:07,100] jepsen worker 31 - jepsen.util 2641 :invoke :read [10 nil] INFO [2019-12-17 12:30:07,108] jepsen worker 70 - jepsen.util 3490 :info :write [23 37] :RequestCanceledException INFO [2019-12-17 12:30:07,109] jepsen worker 70 - jepsen.util 3580 :invoke :write [23 8] INFO [2019-12-17 12:30:07,123] jepsen worker 13 - jepsen.util 2893 :info :write [4 18] :RequestCanceledException INFO [2019-12-17 12:30:07,124] jepsen worker 13 - jepsen.util 2983 :invoke :read [4 nil] INFO [2019-12-17 12:30:07,141] jepsen worker 73 - jepsen.util 2863 :ok :write [24 48] INFO [2019-12-17 12:30:07,141] jepsen worker 11 - jepsen.util 3431 :ok :write [3 2] INFO [2019-12-17 12:30:07,141] jepsen worker 73 - jepsen.util 2863 :invoke :read [24 nil] INFO [2019-12-17 12:30:07,141] jepsen worker 11 - jepsen.util 3431 :invoke :write [3 19] INFO [2019-12-17 12:30:07,154] jepsen worker 16 - jepsen.util 4246 :info :write [5 32] :RequestCanceledException INFO [2019-12-17 12:30:07,155] jepsen worker 16 - jepsen.util 4336 :invoke :write [5 6] INFO [2019-12-17 12:30:07,156] jepsen worker 10 - jepsen.util 2800 :fail :cas [3 [15 13]] :ValueNotSwapFrom INFO [2019-12-17 12:30:07,157] jepsen worker 10 - jepsen.util 2800 :invoke :write [3 2] INFO [2019-12-17 12:30:07,183] jepsen worker 45 - jepsen.util 3015 :ok :read [15 48] INFO [2019-12-17 12:30:07,187] jepsen worker 45 - jepsen.util 3015 :invoke :read [15 nil] INFO [2019-12-17 12:30:07,188] jepsen worker 73 - jepsen.util 2863 :ok :read [24 48] INFO [2019-12-17 12:30:07,188] jepsen worker 73 - jepsen.util 2863 :invoke :read [24 nil] INFO [2019-12-17 12:30:07,189] jepsen worker 35 - jepsen.util 3905 :info :write [11 26] :RequestCanceledException INFO [2019-12-17 12:30:07,190] jepsen worker 35 - jepsen.util 3995 :invoke :write [11 12] INFO [2019-12-17 12:30:07,224] jepsen worker 89 - jepsen.util 2789 :info :write [29 21] :RequestCanceledException INFO [2019-12-17 12:30:07,225] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":17764,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:30:07,225] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":17764,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:30:07,225] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":17765,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:30:07,225] jepsen worker 89 - jepsen.util 2879 :invoke :read [29 nil] INFO [2019-12-17 12:30:07,225] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":17765,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:30:07,229] jepsen worker 80 - jepsen.util 2870 :ok :read [26 27] INFO [2019-12-17 12:30:07,229] jepsen worker 80 - jepsen.util 2870 :invoke :write [26 38] INFO [2019-12-17 12:30:07,233] jepsen worker 73 - jepsen.util 2863 :ok :read [24 48] INFO [2019-12-17 12:30:07,233] jepsen worker 73 - jepsen.util 2863 :invoke :read [24 nil] INFO [2019-12-17 12:30:07,238] jepsen worker 87 - jepsen.util 2787 :info :write [29 13] :RequestCanceledException INFO [2019-12-17 12:30:07,239] jepsen worker 87 - jepsen.util 2877 :invoke :cas [29 [7 7]] INFO [2019-12-17 12:30:07,242] jepsen worker 73 - jepsen.util 2863 :ok :read [24 48] INFO [2019-12-17 12:30:07,243] jepsen worker 73 - jepsen.util 2863 :invoke :write [24 39] INFO [2019-12-17 12:30:07,276] jepsen worker 20 - jepsen.util 2630 :ok :write [6 43] INFO [2019-12-17 12:30:07,276] jepsen worker 20 - jepsen.util 2630 :invoke :cas [6 [49 33]] INFO [2019-12-17 12:30:07,276] jepsen worker 47 - jepsen.util 3647 :ok :write [15 47] INFO [2019-12-17 12:30:07,276] jepsen worker 47 - jepsen.util 3647 :invoke :cas [15 [20 8]] INFO [2019-12-17 12:30:07,301] jepsen worker 11 - jepsen.util 3431 :ok :write [3 19] INFO [2019-12-17 12:30:07,301] jepsen worker 11 - jepsen.util 3431 :invoke :write [3 11] INFO [2019-12-17 12:30:07,312] jepsen worker 73 - jepsen.util 2863 :ok :write [24 39] INFO [2019-12-17 12:30:07,315] jepsen worker 73 - jepsen.util 2863 :invoke :cas [24 [2 49]] INFO [2019-12-17 12:30:07,321] jepsen worker 20 - jepsen.util 2630 :fail :cas [6 [49 33]] :ValueNotSwapFrom INFO [2019-12-17 12:30:07,321] jepsen worker 20 - jepsen.util 2630 :invoke :read [6 nil] INFO [2019-12-17 12:30:07,321] jepsen worker 59 - jepsen.util 3929 :info :cas [19 [49 19]] :RequestCanceledException INFO [2019-12-17 12:30:07,322] jepsen worker 59 - jepsen.util 4019 :invoke :cas [19 [20 11]] INFO [2019-12-17 12:30:07,327] jepsen worker 47 - jepsen.util 3647 :fail :cas [15 [20 8]] :ValueNotSwapFrom INFO [2019-12-17 12:30:07,328] jepsen worker 47 - jepsen.util 3647 :invoke :cas [15 [15 27]] INFO [2019-12-17 12:30:07,328] jepsen worker 72 - jepsen.util 3132 :ok :read [24 48] INFO [2019-12-17 12:30:07,328] jepsen worker 72 - jepsen.util 3132 :invoke :write [24 26] INFO [2019-12-17 12:30:07,338] jepsen worker 32 - jepsen.util 3542 :fail :read [10 nil] :RequestTimeoutException INFO [2019-12-17 12:30:07,338] jepsen worker 32 - jepsen.util 3542 :invoke :cas [10 [47 6]] INFO [2019-12-17 12:30:07,355] jepsen worker 13 - jepsen.util 2983 :ok :read [4 18] INFO [2019-12-17 12:30:07,357] jepsen worker 13 - jepsen.util 2983 :invoke :read [4 nil] INFO [2019-12-17 12:30:07,372] jepsen worker 71 - jepsen.util 3041 :fail :cas [23 [13 12]] :ValueNotSwapFrom INFO [2019-12-17 12:30:07,372] jepsen worker 71 - jepsen.util 3041 :invoke :cas [23 [47 14]] INFO [2019-12-17 12:30:07,397] jepsen worker 13 - jepsen.util 2983 :ok :read [4 18] INFO [2019-12-17 12:30:07,397] jepsen worker 13 - jepsen.util 2983 :invoke :cas [4 [29 38]] INFO [2019-12-17 12:30:07,408] jepsen worker 63 - jepsen.util 2673 :info :write [21 25] :RequestCanceledException INFO [2019-12-17 12:30:07,408] jepsen worker 43 - jepsen.util 3733 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:30:07,408] jepsen worker 79 - jepsen.util 2959 :info :write [26 21] :RequestCanceledException INFO [2019-12-17 12:30:07,408] jepsen worker 29 - jepsen.util 2549 :fail :read [9 nil] :RequestTimeoutException INFO [2019-12-17 12:30:07,408] jepsen worker 43 - jepsen.util 3733 :invoke :read [14 nil] INFO [2019-12-17 12:30:07,409] jepsen worker 29 - jepsen.util 2549 :invoke :write [9 18] INFO [2019-12-17 12:30:07,409] jepsen worker 63 - jepsen.util 2763 :invoke :write [21 43] INFO [2019-12-17 12:30:07,409] jepsen worker 79 - jepsen.util 3049 :invoke :read [26 nil] INFO [2019-12-17 12:30:07,421] jepsen worker 20 - jepsen.util 2630 :ok :read [6 43] INFO [2019-12-17 12:30:07,421] jepsen worker 20 - jepsen.util 2630 :invoke :read [6 nil] INFO [2019-12-17 12:30:07,422] jepsen worker 14 - jepsen.util 2894 :ok :write [4 41] INFO [2019-12-17 12:30:07,422] jepsen worker 14 - jepsen.util 2894 :invoke :write [4 42] INFO [2019-12-17 12:30:07,424] jepsen worker 15 - jepsen.util 3705 :info :cas [5 [28 14]] :RequestCanceledException INFO [2019-12-17 12:30:07,425] jepsen worker 15 - jepsen.util 3795 :invoke :cas [5 [18 43]] INFO [2019-12-17 12:30:07,427] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":17800,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:30:07,427] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":17800,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:30:07,427] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":17802,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:30:07,427] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":17802,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:30:07,430] jepsen worker 45 - jepsen.util 3015 :ok :read [15 47] INFO [2019-12-17 12:30:07,430] jepsen worker 45 - jepsen.util 3015 :invoke :cas [15 [38 18]] INFO [2019-12-17 12:30:07,438] jepsen worker 84 - jepsen.util 3954 :info :write [28 15] :RequestCanceledException INFO [2019-12-17 12:30:07,446] jepsen worker 84 - jepsen.util 4044 :invoke :write [28 29] INFO [2019-12-17 12:30:07,448] jepsen worker 13 - jepsen.util 2983 :fail :cas [4 [29 38]] :ValueNotSwapFrom INFO [2019-12-17 12:30:07,448] jepsen worker 13 - jepsen.util 2983 :invoke :write [4 21] INFO [2019-12-17 12:30:07,454] jepsen worker 85 - jepsen.util 4585 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:30:07,454] jepsen worker 85 - jepsen.util 4585 :invoke :cas [28 [26 48]] INFO [2019-12-17 12:30:07,464] jepsen worker 73 - jepsen.util 2863 :fail :cas [24 [2 49]] :ValueNotSwapFrom INFO [2019-12-17 12:30:07,464] jepsen worker 73 - jepsen.util 2863 :invoke :cas [24 [3 24]] INFO [2019-12-17 12:30:07,471] jepsen worker 47 - jepsen.util 3647 :fail :cas [15 [15 27]] :ValueNotSwapFrom INFO [2019-12-17 12:30:07,471] jepsen worker 47 - jepsen.util 3647 :invoke :cas [15 [48 27]] INFO [2019-12-17 12:30:07,492] jepsen worker 20 - jepsen.util 2630 :ok :read [6 43] INFO [2019-12-17 12:30:07,492] jepsen worker 20 - jepsen.util 2630 :invoke :cas [6 [19 43]] INFO [2019-12-17 12:30:07,495] jepsen worker 73 - jepsen.util 2863 :fail :cas [24 [3 24]] :ValueNotSwapFrom INFO [2019-12-17 12:30:07,495] jepsen worker 73 - jepsen.util 2863 :invoke :read [24 nil] INFO [2019-12-17 12:30:07,501] jepsen worker 38 - jepsen.util 3008 :ok :read [12 20] INFO [2019-12-17 12:30:07,502] jepsen worker 38 - jepsen.util 3008 :invoke :write [12 47] INFO [2019-12-17 12:30:07,517] jepsen worker 20 - jepsen.util 2630 :fail :cas [6 [19 43]] :ValueNotSwapFrom INFO [2019-12-17 12:30:07,517] jepsen worker 20 - jepsen.util 2630 :invoke :write [6 39] INFO [2019-12-17 12:30:07,523] jepsen worker 27 - jepsen.util 2727 :fail :read [9 nil] :RequestTimeoutException INFO [2019-12-17 12:30:07,523] jepsen worker 30 - jepsen.util 3630 :info :write [10 0] :RequestCanceledException INFO [2019-12-17 12:30:07,523] jepsen worker 62 - jepsen.util 3932 :info :write [20 25] :RequestCanceledException INFO [2019-12-17 12:30:07,524] jepsen worker 27 - jepsen.util 2727 :invoke :read [9 nil] INFO [2019-12-17 12:30:07,525] jepsen worker 30 - jepsen.util 3720 :invoke :read [10 nil] INFO [2019-12-17 12:30:07,525] jepsen worker 62 - jepsen.util 4022 :invoke :read [20 nil] INFO [2019-12-17 12:30:07,531] jepsen worker 73 - jepsen.util 2863 :ok :read [24 39] INFO [2019-12-17 12:30:07,531] jepsen worker 73 - jepsen.util 2863 :invoke :read [24 nil] INFO [2019-12-17 12:30:07,537] jepsen worker 50 - jepsen.util 4550 :info :write [16 29] :RequestCanceledException INFO [2019-12-17 12:30:07,538] jepsen worker 50 - jepsen.util 4640 :invoke :write [16 38] INFO [2019-12-17 12:30:07,555] jepsen worker 1 - jepsen.util 3601 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:30:07,555] jepsen worker 1 - jepsen.util 3601 :invoke :read [0 nil] INFO [2019-12-17 12:30:07,571] jepsen worker 89 - jepsen.util 2879 :ok :read [29 17] INFO [2019-12-17 12:30:07,571] jepsen worker 89 - jepsen.util 2879 :invoke :cas [29 [36 34]] INFO [2019-12-17 12:30:07,589] jepsen worker 30 - jepsen.util 3720 :ok :read [10 7] INFO [2019-12-17 12:30:07,589] jepsen worker 30 - jepsen.util 3720 :invoke :read [10 nil] INFO [2019-12-17 12:30:07,623] jepsen worker 19 - jepsen.util 3259 :info :write [6 9] :RequestCanceledException INFO [2019-12-17 12:30:07,627] jepsen worker 19 - jepsen.util 3349 :invoke :read [6 nil] INFO [2019-12-17 12:30:07,630] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:30:07,630] jepsen nemesis - couchbase.util Rebalance check count: 130 INFO [2019-12-17 12:30:07,630] jepsen nemesis - couchbase.util Rebalance stuck count: 111 INFO [2019-12-17 12:30:07,636] jepsen worker 73 - jepsen.util 2863 :ok :read [24 39] INFO [2019-12-17 12:30:07,636] jepsen worker 73 - jepsen.util 2863 :invoke :write [24 16] INFO [2019-12-17 12:30:07,637] jepsen worker 40 - jepsen.util 4270 :info :cas [13 [1 6]] :RequestCanceledException INFO [2019-12-17 12:30:07,638] jepsen worker 40 - jepsen.util 4360 :invoke :write [13 26] INFO [2019-12-17 12:30:07,647] jepsen worker 21 - jepsen.util 3981 :info :cas [7 [24 45]] :RequestCanceledException INFO [2019-12-17 12:30:07,647] jepsen worker 75 - jepsen.util 4215 :info :write [25 44] :RequestCanceledException INFO [2019-12-17 12:30:07,648] jepsen worker 75 - jepsen.util 4305 :invoke :read [25 nil] INFO [2019-12-17 12:30:07,648] jepsen worker 21 - jepsen.util 4071 :invoke :read [7 nil] INFO [2019-12-17 12:30:07,654] jepsen worker 30 - jepsen.util 3720 :ok :read [10 7] INFO [2019-12-17 12:30:07,654] jepsen worker 30 - jepsen.util 3720 :invoke :read [10 nil] INFO [2019-12-17 12:30:07,672] jepsen worker 11 - jepsen.util 3431 :ok :write [3 11] INFO [2019-12-17 12:30:07,672] jepsen worker 11 - jepsen.util 3431 :invoke :cas [3 [48 7]] INFO [2019-12-17 12:30:07,685] jepsen worker 29 - jepsen.util 2549 :ok :write [9 18] INFO [2019-12-17 12:30:07,685] jepsen worker 29 - jepsen.util 2549 :invoke :write [9 39] INFO [2019-12-17 12:30:07,690] jepsen worker 22 - jepsen.util 4162 :info :cas [7 [44 6]] :RequestCanceledException INFO [2019-12-17 12:30:07,691] jepsen worker 22 - jepsen.util 4252 :invoke :read [7 nil] INFO [2019-12-17 12:30:07,693] jepsen worker 30 - jepsen.util 3720 :ok :read [10 7] INFO [2019-12-17 12:30:07,693] jepsen worker 30 - jepsen.util 3720 :invoke :read [10 nil] INFO [2019-12-17 12:30:07,723] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":17827,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:30:07,723] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":17827,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:30:07,723] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":17829,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:30:07,723] jepsen worker 69 - jepsen.util 2949 :info :write [23 26] :RequestCanceledException INFO [2019-12-17 12:30:07,723] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":17829,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:30:07,724] jepsen worker 69 - jepsen.util 3039 :invoke :cas [23 [2 23]] INFO [2019-12-17 12:30:07,742] jepsen worker 30 - jepsen.util 3720 :ok :read [10 7] INFO [2019-12-17 12:30:07,742] jepsen worker 30 - jepsen.util 3720 :invoke :write [10 1] INFO [2019-12-17 12:30:07,784] jepsen worker 72 - jepsen.util 3132 :ok :write [24 26] INFO [2019-12-17 12:30:07,784] jepsen worker 72 - jepsen.util 3132 :invoke :cas [24 [34 6]] INFO [2019-12-17 12:30:07,789] jepsen worker 63 - jepsen.util 2763 :ok :write [21 43] INFO [2019-12-17 12:30:07,790] jepsen worker 63 - jepsen.util 2763 :invoke :read [21 nil] INFO [2019-12-17 12:30:07,791] jepsen worker 42 - jepsen.util 4722 :info :write [14 6] :RequestCanceledException INFO [2019-12-17 12:30:07,792] jepsen worker 42 - jepsen.util 4812 :invoke :cas [14 [27 10]] INFO [2019-12-17 12:30:07,808] jepsen worker 41 - jepsen.util 4181 :info :write [13 42] :RequestCanceledException INFO [2019-12-17 12:30:07,809] jepsen worker 41 - jepsen.util 4271 :invoke :read [13 nil] INFO [2019-12-17 12:30:07,819] jepsen worker 29 - jepsen.util 2549 :ok :write [9 39] INFO [2019-12-17 12:30:07,819] jepsen worker 29 - jepsen.util 2549 :invoke :read [9 nil] INFO [2019-12-17 12:30:07,824] jepsen worker 57 - jepsen.util 3747 :info :cas [19 [43 48]] :RequestCanceledException INFO [2019-12-17 12:30:07,824] jepsen worker 0 - jepsen.util 4500 :info :write [0 7] :RequestCanceledException INFO [2019-12-17 12:30:07,824] jepsen worker 33 - jepsen.util 4353 :info :write [11 36] :RequestCanceledException INFO [2019-12-17 12:30:07,825] jepsen worker 33 - jepsen.util 4443 :invoke :read [11 nil] INFO [2019-12-17 12:30:07,825] jepsen worker 57 - jepsen.util 3837 :invoke :write [19 6] INFO [2019-12-17 12:30:07,825] jepsen worker 0 - jepsen.util 4590 :invoke :write [0 5] INFO [2019-12-17 12:30:07,827] jepsen worker 19 - jepsen.util 3349 :ok :read [6 43] INFO [2019-12-17 12:30:07,827] jepsen worker 19 - jepsen.util 3349 :invoke :cas [6 [14 8]] INFO [2019-12-17 12:30:07,828] jepsen worker 63 - jepsen.util 2763 :ok :read [21 43] INFO [2019-12-17 12:30:07,828] jepsen worker 63 - jepsen.util 2763 :invoke :read [21 nil] INFO [2019-12-17 12:30:07,837] jepsen worker 3 - jepsen.util 2973 :info :write [1 26] :RequestCanceledException INFO [2019-12-17 12:30:07,837] jepsen worker 53 - jepsen.util 3023 :info :write [17 32] :RequestCanceledException INFO [2019-12-17 12:30:07,838] jepsen worker 3 - jepsen.util 3063 :invoke :write [1 10] INFO [2019-12-17 12:30:07,838] jepsen worker 53 - jepsen.util 3113 :invoke :read [17 nil] INFO [2019-12-17 12:30:07,848] jepsen worker 72 - jepsen.util 3132 :fail :cas [24 [34 6]] :ValueNotSwapFrom INFO [2019-12-17 12:30:07,848] jepsen worker 72 - jepsen.util 3132 :invoke :read [24 nil] INFO [2019-12-17 12:30:07,854] jepsen worker 83 - jepsen.util 4223 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:30:07,854] jepsen worker 83 - jepsen.util 4223 :invoke :cas [27 [27 14]] INFO [2019-12-17 12:30:07,857] jepsen worker 63 - jepsen.util 2763 :ok :read [21 43] INFO [2019-12-17 12:30:07,857] jepsen worker 63 - jepsen.util 2763 :invoke :read [21 nil] INFO [2019-12-17 12:30:07,868] jepsen worker 19 - jepsen.util 3349 :fail :cas [6 [14 8]] :ValueNotSwapFrom INFO [2019-12-17 12:30:07,868] jepsen worker 19 - jepsen.util 3349 :invoke :write [6 46] INFO [2019-12-17 12:30:07,868] jepsen worker 29 - jepsen.util 2549 :ok :read [9 39] INFO [2019-12-17 12:30:07,868] jepsen worker 29 - jepsen.util 2549 :invoke :read [9 nil] INFO [2019-12-17 12:30:07,885] jepsen worker 63 - jepsen.util 2763 :ok :read [21 43] INFO [2019-12-17 12:30:07,885] jepsen worker 63 - jepsen.util 2763 :invoke :read [21 nil] INFO [2019-12-17 12:30:07,890] jepsen worker 8 - jepsen.util 3338 :info :cas [2 [33 5]] :RequestCanceledException INFO [2019-12-17 12:30:07,890] jepsen worker 8 - jepsen.util 3428 :invoke :write [2 37] INFO [2019-12-17 12:30:07,898] jepsen worker 72 - jepsen.util 3132 :ok :read [24 26] INFO [2019-12-17 12:30:07,899] jepsen worker 72 - jepsen.util 3132 :invoke :read [24 nil] INFO [2019-12-17 12:30:07,907] jepsen worker 48 - jepsen.util 3468 :info :write [16 32] :RequestCanceledException INFO [2019-12-17 12:30:07,908] jepsen worker 48 - jepsen.util 3558 :invoke :write [16 17] INFO [2019-12-17 12:30:07,918] jepsen worker 89 - jepsen.util 2879 :fail :cas [29 [36 34]] :ValueNotSwapFrom INFO [2019-12-17 12:30:07,918] jepsen worker 89 - jepsen.util 2879 :invoke :cas [29 [37 30]] INFO [2019-12-17 12:30:07,922] jepsen worker 29 - jepsen.util 2549 :ok :read [9 39] INFO [2019-12-17 12:30:07,922] jepsen worker 29 - jepsen.util 2549 :invoke :cas [9 [29 46]] INFO [2019-12-17 12:30:07,923] jepsen worker 17 - jepsen.util 4067 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:30:07,923] jepsen worker 26 - jepsen.util 4346 :info :cas [8 [5 40]] :RequestCanceledException INFO [2019-12-17 12:30:07,923] jepsen worker 17 - jepsen.util 4067 :invoke :read [5 nil] INFO [2019-12-17 12:30:07,924] jepsen worker 26 - jepsen.util 4436 :invoke :cas [8 [6 13]] INFO [2019-12-17 12:30:07,932] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":17856,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:30:07,932] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":17856,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:30:07,932] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":17860,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:30:07,932] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":17860,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:30:07,937] jepsen worker 4 - jepsen.util 2254 :info :write [1 18] :RequestCanceledException INFO [2019-12-17 12:30:07,937] jepsen worker 63 - jepsen.util 2763 :ok :read [21 43] INFO [2019-12-17 12:30:07,937] jepsen worker 63 - jepsen.util 2763 :invoke :read [21 nil] INFO [2019-12-17 12:30:07,938] jepsen worker 4 - jepsen.util 2344 :invoke :write [1 5] INFO [2019-12-17 12:30:07,963] jepsen worker 20 - jepsen.util 2630 :ok :write [6 39] INFO [2019-12-17 12:30:07,963] jepsen worker 20 - jepsen.util 2630 :invoke :write [6 42] INFO [2019-12-17 12:30:07,990] jepsen worker 69 - jepsen.util 3039 :fail :cas [23 [2 23]] :ValueNotSwapFrom INFO [2019-12-17 12:30:07,990] jepsen worker 69 - jepsen.util 3039 :invoke :cas [23 [2 37]] INFO [2019-12-17 12:30:07,993] jepsen worker 29 - jepsen.util 2549 :fail :cas [9 [29 46]] :ValueNotSwapFrom INFO [2019-12-17 12:30:07,993] jepsen worker 29 - jepsen.util 2549 :invoke :read [9 nil] INFO [2019-12-17 12:30:08,001] jepsen worker 63 - jepsen.util 2763 :ok :read [21 43] INFO [2019-12-17 12:30:08,001] jepsen worker 63 - jepsen.util 2763 :invoke :read [21 nil] INFO [2019-12-17 12:30:08,023] jepsen worker 27 - jepsen.util 2727 :ok :read [9 39] INFO [2019-12-17 12:30:08,023] jepsen worker 27 - jepsen.util 2727 :invoke :cas [9 [23 38]] INFO [2019-12-17 12:30:08,039] jepsen worker 29 - jepsen.util 2549 :ok :read [9 39] INFO [2019-12-17 12:30:08,039] jepsen worker 29 - jepsen.util 2549 :invoke :write [9 44] INFO [2019-12-17 12:30:08,041] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":17876,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:30:08,042] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":17876,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:30:08,042] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":17877,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:30:08,042] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":17877,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:30:08,064] jepsen worker 63 - jepsen.util 2763 :ok :read [21 43] INFO [2019-12-17 12:30:08,064] jepsen worker 63 - jepsen.util 2763 :invoke :cas [21 [5 37]] INFO [2019-12-17 12:30:08,078] jepsen worker 87 - jepsen.util 2877 :fail :cas [29 [7 7]] :ValueNotSwapFrom INFO [2019-12-17 12:30:08,079] jepsen worker 87 - jepsen.util 2877 :invoke :read [29 nil] INFO [2019-12-17 12:30:08,082] jepsen worker 63 - jepsen.util 2763 :fail :cas [21 [5 37]] :ValueNotSwapFrom INFO [2019-12-17 12:30:08,083] jepsen worker 63 - jepsen.util 2763 :invoke :read [21 nil] INFO [2019-12-17 12:30:08,104] jepsen worker 63 - jepsen.util 2763 :ok :read [21 43] INFO [2019-12-17 12:30:08,104] jepsen worker 63 - jepsen.util 2763 :invoke :cas [21 [5 16]] INFO [2019-12-17 12:30:08,108] jepsen worker 37 - jepsen.util 2737 :info :write [12 41] :RequestCanceledException INFO [2019-12-17 12:30:08,109] jepsen worker 37 - jepsen.util 2827 :invoke :write [12 42] INFO [2019-12-17 12:30:08,142] jepsen worker 63 - jepsen.util 2763 :fail :cas [21 [5 16]] :ValueNotSwapFrom INFO [2019-12-17 12:30:08,142] jepsen worker 63 - jepsen.util 2763 :invoke :write [21 42] INFO [2019-12-17 12:30:08,144] jepsen worker 29 - jepsen.util 2549 :ok :write [9 44] INFO [2019-12-17 12:30:08,144] jepsen worker 29 - jepsen.util 2549 :invoke :write [9 33] INFO [2019-12-17 12:30:08,169] jepsen worker 71 - jepsen.util 3041 :fail :cas [23 [47 14]] :ValueNotSwapFrom INFO [2019-12-17 12:30:08,169] jepsen worker 71 - jepsen.util 3041 :invoke :read [23 nil] INFO [2019-12-17 12:30:08,179] jepsen worker 89 - jepsen.util 2879 :fail :cas [29 [37 30]] :ValueNotSwapFrom INFO [2019-12-17 12:30:08,179] jepsen worker 89 - jepsen.util 2879 :invoke :cas [29 [30 35]] INFO [2019-12-17 12:30:08,202] jepsen worker 29 - jepsen.util 2549 :ok :write [9 33] INFO [2019-12-17 12:30:08,202] jepsen worker 29 - jepsen.util 2549 :invoke :read [9 nil] INFO [2019-12-17 12:30:08,206] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":17895,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:30:08,206] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":17895,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:30:08,206] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":17897,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:30:08,207] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":17897,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:30:08,207] jepsen worker 63 - jepsen.util 2763 :ok :write [21 42] INFO [2019-12-17 12:30:08,207] jepsen worker 63 - jepsen.util 2763 :invoke :cas [21 [8 8]] INFO [2019-12-17 12:30:08,221] jepsen worker 89 - jepsen.util 2879 :fail :cas [29 [30 35]] :ValueNotSwapFrom INFO [2019-12-17 12:30:08,221] jepsen worker 89 - jepsen.util 2879 :invoke :cas [29 [35 32]] INFO [2019-12-17 12:30:08,228] jepsen worker 79 - jepsen.util 3049 :ok :read [26 27] INFO [2019-12-17 12:30:08,229] jepsen worker 79 - jepsen.util 3049 :invoke :read [26 nil] INFO [2019-12-17 12:30:08,236] jepsen worker 63 - jepsen.util 2763 :fail :cas [21 [8 8]] :ValueNotSwapFrom INFO [2019-12-17 12:30:08,236] jepsen worker 63 - jepsen.util 2763 :invoke :write [21 24] INFO [2019-12-17 12:30:08,237] jepsen worker 29 - jepsen.util 2549 :ok :read [9 33] INFO [2019-12-17 12:30:08,237] jepsen worker 29 - jepsen.util 2549 :invoke :write [9 6] INFO [2019-12-17 12:30:08,254] jepsen worker 25 - jepsen.util 4525 :info :cas [8 [38 29]] :RequestCanceledException INFO [2019-12-17 12:30:08,254] jepsen worker 81 - jepsen.util 4041 :info :write [27 9] :RequestCanceledException INFO [2019-12-17 12:30:08,255] jepsen worker 25 - jepsen.util 4615 :invoke :cas [8 [9 27]] INFO [2019-12-17 12:30:08,255] jepsen worker 81 - jepsen.util 4131 :invoke :write [27 15] INFO [2019-12-17 12:30:08,278] jepsen worker 89 - jepsen.util 2879 :fail :cas [29 [35 32]] :ValueNotSwapFrom INFO [2019-12-17 12:30:08,279] jepsen worker 89 - jepsen.util 2879 :invoke :cas [29 [15 23]] INFO [2019-12-17 12:30:08,280] jepsen worker 63 - jepsen.util 2763 :ok :write [21 24] INFO [2019-12-17 12:30:08,280] jepsen worker 63 - jepsen.util 2763 :invoke :write [21 31] INFO [2019-12-17 12:30:08,290] jepsen worker 9 - jepsen.util 3519 :info :write [3 12] :RequestCanceledException INFO [2019-12-17 12:30:08,294] jepsen worker 9 - jepsen.util 3609 :invoke :read [3 nil] INFO [2019-12-17 12:30:08,308] jepsen worker 51 - jepsen.util 2841 :info :write [17 33] :RequestCanceledException INFO [2019-12-17 12:30:08,309] jepsen worker 51 - jepsen.util 2931 :invoke :write [17 16] INFO [2019-12-17 12:30:08,314] jepsen worker 9 - jepsen.util 3609 :ok :read [3 11] INFO [2019-12-17 12:30:08,314] jepsen worker 9 - jepsen.util 3609 :invoke :read [3 nil] INFO [2019-12-17 12:30:08,316] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":17904,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:30:08,316] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":17904,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:30:08,316] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":17907,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:30:08,316] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":17907,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:30:08,316] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":17908,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:30:08,316] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":17908,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:30:08,323] jepsen worker 76 - jepsen.util 4216 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:30:08,323] jepsen worker 76 - jepsen.util 4216 :invoke :cas [25 [1 14]] INFO [2019-12-17 12:30:08,324] jepsen worker 58 - jepsen.util 4018 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:30:08,324] jepsen worker 58 - jepsen.util 4018 :invoke :cas [19 [33 12]] INFO [2019-12-17 12:30:08,349] jepsen worker 9 - jepsen.util 3609 :ok :read [3 11] INFO [2019-12-17 12:30:08,349] jepsen worker 9 - jepsen.util 3609 :invoke :cas [3 [5 35]] INFO [2019-12-17 12:30:08,353] jepsen worker 86 - jepsen.util 4316 :info :cas [28 [33 45]] :RequestCanceledException INFO [2019-12-17 12:30:08,354] jepsen worker 29 - jepsen.util 2549 :ok :write [9 6] INFO [2019-12-17 12:30:08,354] jepsen worker 29 - jepsen.util 2549 :invoke :read [9 nil] INFO [2019-12-17 12:30:08,354] jepsen worker 86 - jepsen.util 4406 :invoke :read [28 nil] INFO [2019-12-17 12:30:08,368] jepsen worker 63 - jepsen.util 2763 :ok :write [21 31] INFO [2019-12-17 12:30:08,368] jepsen worker 63 - jepsen.util 2763 :invoke :write [21 10] INFO [2019-12-17 12:30:08,390] jepsen worker 12 - jepsen.util 3162 :info :write [4 23] :RequestCanceledException INFO [2019-12-17 12:30:08,390] jepsen worker 54 - jepsen.util 3114 :info :write [18 35] :RequestCanceledException INFO [2019-12-17 12:30:08,390] jepsen worker 54 - jepsen.util 3204 :invoke :read [18 nil] INFO [2019-12-17 12:30:08,391] jepsen worker 12 - jepsen.util 3252 :invoke :cas [4 [38 14]] INFO [2019-12-17 12:30:08,406] jepsen worker 87 - jepsen.util 2877 :ok :read [29 17] INFO [2019-12-17 12:30:08,406] jepsen worker 87 - jepsen.util 2877 :invoke :cas [29 [46 33]] INFO [2019-12-17 12:30:08,409] jepsen worker 61 - jepsen.util 4561 :info :cas [20 [23 45]] :RequestCanceledException INFO [2019-12-17 12:30:08,409] jepsen worker 61 - jepsen.util 4651 :invoke :write [20 38] INFO [2019-12-17 12:30:08,417] jepsen worker 9 - jepsen.util 3609 :fail :cas [3 [5 35]] :ValueNotSwapFrom INFO [2019-12-17 12:30:08,418] jepsen worker 9 - jepsen.util 3609 :invoke :cas [3 [34 22]] INFO [2019-12-17 12:30:08,422] jepsen worker 78 - jepsen.util 3318 :info :write [26 8] :RequestCanceledException INFO [2019-12-17 12:30:08,422] jepsen worker 82 - jepsen.util 4852 :info :cas [27 [4 20]] :RequestCanceledException INFO [2019-12-17 12:30:08,423] jepsen worker 82 - jepsen.util 4942 :invoke :write [27 5] INFO [2019-12-17 12:30:08,423] jepsen worker 78 - jepsen.util 3408 :invoke :cas [26 [39 1]] INFO [2019-12-17 12:30:08,437] jepsen worker 2 - jepsen.util 3782 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:30:08,438] jepsen worker 2 - jepsen.util 3782 :invoke :write [0 25] INFO [2019-12-17 12:30:08,454] jepsen worker 29 - jepsen.util 2549 :ok :read [9 6] INFO [2019-12-17 12:30:08,454] jepsen worker 39 - jepsen.util 3909 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:30:08,454] jepsen worker 66 - jepsen.util 4026 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:30:08,454] jepsen worker 29 - jepsen.util 2549 :invoke :write [9 24] INFO [2019-12-17 12:30:08,454] jepsen worker 39 - jepsen.util 3909 :invoke :cas [13 [39 44]] INFO [2019-12-17 12:30:08,454] jepsen worker 66 - jepsen.util 4026 :invoke :write [22 33] INFO [2019-12-17 12:30:08,460] jepsen worker 9 - jepsen.util 3609 :fail :cas [3 [34 22]] :ValueNotSwapFrom INFO [2019-12-17 12:30:08,460] jepsen worker 9 - jepsen.util 3609 :invoke :write [3 23] INFO [2019-12-17 12:30:08,475] jepsen worker 30 - jepsen.util 3720 :ok :write [10 1] INFO [2019-12-17 12:30:08,475] jepsen worker 30 - jepsen.util 3720 :invoke :write [10 27] INFO [2019-12-17 12:30:08,477] jepsen worker 89 - jepsen.util 2879 :fail :cas [29 [15 23]] :ValueNotSwapFrom INFO [2019-12-17 12:30:08,477] jepsen worker 89 - jepsen.util 2879 :invoke :read [29 nil] INFO [2019-12-17 12:30:08,484] jepsen worker 19 - jepsen.util 3349 :ok :write [6 46] INFO [2019-12-17 12:30:08,487] jepsen worker 19 - jepsen.util 3349 :invoke :write [6 14] INFO [2019-12-17 12:30:08,490] jepsen worker 5 - jepsen.util 2615 :info :cas [1 [26 15]] :RequestCanceledException INFO [2019-12-17 12:30:08,490] jepsen worker 36 - jepsen.util 3006 :info :write [12 0] :RequestCanceledException INFO [2019-12-17 12:30:08,490] jepsen worker 24 - jepsen.util 3984 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:30:08,491] jepsen worker 24 - jepsen.util 3984 :invoke :write [8 12] INFO [2019-12-17 12:30:08,491] jepsen worker 5 - jepsen.util 2705 :invoke :read [1 nil] INFO [2019-12-17 12:30:08,492] jepsen worker 63 - jepsen.util 2763 :ok :write [21 10] INFO [2019-12-17 12:30:08,493] jepsen worker 63 - jepsen.util 2763 :invoke :write [21 21] INFO [2019-12-17 12:30:08,501] jepsen worker 36 - jepsen.util 3096 :invoke :cas [12 [35 31]] INFO [2019-12-17 12:30:08,508] jepsen worker 68 - jepsen.util 3848 :info :write [22 19] :RequestCanceledException INFO [2019-12-17 12:30:08,509] jepsen worker 68 - jepsen.util 3938 :invoke :write [22 34] INFO [2019-12-17 12:30:08,523] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":17933,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:30:08,523] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":17933,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:30:08,523] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":17936,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:30:08,523] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":17936,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:30:08,523] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":17937,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:30:08,523] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":17937,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:30:08,524] jepsen worker 77 - jepsen.util 3767 :info :cas [25 [34 22]] :RequestCanceledException INFO [2019-12-17 12:30:08,525] jepsen worker 77 - jepsen.util 3857 :invoke :read [25 nil] INFO [2019-12-17 12:30:08,553] jepsen worker 29 - jepsen.util 2549 :ok :write [9 24] INFO [2019-12-17 12:30:08,553] jepsen worker 29 - jepsen.util 2549 :invoke :cas [9 [5 9]] INFO [2019-12-17 12:30:08,565] jepsen worker 89 - jepsen.util 2879 :ok :read [29 17] INFO [2019-12-17 12:30:08,565] jepsen worker 89 - jepsen.util 2879 :invoke :write [29 37] INFO [2019-12-17 12:30:08,567] jepsen worker 63 - jepsen.util 2763 :ok :write [21 21] INFO [2019-12-17 12:30:08,567] jepsen worker 63 - jepsen.util 2763 :invoke :read [21 nil] INFO [2019-12-17 12:30:08,579] jepsen worker 55 - jepsen.util 3745 :ok :write [18 4] INFO [2019-12-17 12:30:08,579] jepsen worker 55 - jepsen.util 3745 :invoke :write [18 48] INFO [2019-12-17 12:30:08,583] jepsen worker 79 - jepsen.util 3049 :ok :read [26 27] INFO [2019-12-17 12:30:08,583] jepsen worker 79 - jepsen.util 3049 :invoke :read [26 nil] INFO [2019-12-17 12:30:08,592] jepsen worker 69 - jepsen.util 3039 :fail :cas [23 [2 37]] :ValueNotSwapFrom INFO [2019-12-17 12:30:08,592] jepsen worker 69 - jepsen.util 3039 :invoke :cas [23 [30 14]] INFO [2019-12-17 12:30:08,595] jepsen worker 53 - jepsen.util 3113 :ok :read [17 27] INFO [2019-12-17 12:30:08,595] jepsen worker 53 - jepsen.util 3113 :invoke :write [17 13] INFO [2019-12-17 12:30:08,608] jepsen worker 67 - jepsen.util 4477 :info :write [22 16] :RequestCanceledException INFO [2019-12-17 12:30:08,609] jepsen worker 67 - jepsen.util 4567 :invoke :read [22 nil] INFO [2019-12-17 12:30:08,620] jepsen worker 71 - jepsen.util 3041 :ok :read [23 30] INFO [2019-12-17 12:30:08,620] jepsen worker 71 - jepsen.util 3041 :invoke :write [23 22] INFO [2019-12-17 12:30:08,637] jepsen worker 44 - jepsen.util 4274 :info :write [14 46] :RequestCanceledException INFO [2019-12-17 12:30:08,647] jepsen worker 44 - jepsen.util 4364 :invoke :write [14 36] INFO [2019-12-17 12:30:08,651] jepsen worker 12 - jepsen.util 3252 :fail :cas [4 [38 14]] :ValueNotSwapFrom INFO [2019-12-17 12:30:08,651] jepsen worker 12 - jepsen.util 3252 :invoke :write [4 3] INFO [2019-12-17 12:30:08,658] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:30:08,658] jepsen nemesis - couchbase.util Rebalance check count: 131 INFO [2019-12-17 12:30:08,658] jepsen nemesis - couchbase.util Rebalance stuck count: 112 INFO [2019-12-17 12:30:08,669] jepsen worker 29 - jepsen.util 2549 :fail :cas [9 [5 9]] :ValueNotSwapFrom INFO [2019-12-17 12:30:08,669] jepsen worker 29 - jepsen.util 2549 :invoke :write [9 6] INFO [2019-12-17 12:30:08,692] jepsen worker 6 - jepsen.util 4326 :info :write [2 11] :RequestCanceledException INFO [2019-12-17 12:30:08,693] jepsen worker 6 - jepsen.util 4416 :invoke :cas [2 [1 2]] INFO [2019-12-17 12:30:08,707] jepsen worker 49 - jepsen.util 3559 :info :write [16 14] :RequestCanceledException INFO [2019-12-17 12:30:08,708] jepsen worker 49 - jepsen.util 3649 :invoke :cas [16 [4 15]] INFO [2019-12-17 12:30:08,712] jepsen worker 63 - jepsen.util 2763 :ok :read [21 21] INFO [2019-12-17 12:30:08,712] jepsen worker 63 - jepsen.util 2763 :invoke :cas [21 [39 2]] INFO [2019-12-17 12:30:08,713] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":17963,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:30:08,713] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":17963,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:30:08,714] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":17966,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:30:08,714] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":17966,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:30:08,714] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":17967,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:30:08,714] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":17967,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:30:08,720] jepsen worker 5 - jepsen.util 2705 :ok :read [1 26] INFO [2019-12-17 12:30:08,720] jepsen worker 5 - jepsen.util 2705 :invoke :write [1 24] INFO [2019-12-17 12:30:08,723] jepsen worker 60 - jepsen.util 4380 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:30:08,723] jepsen worker 60 - jepsen.util 4380 :invoke :cas [20 [31 6]] INFO [2019-12-17 12:30:08,732] jepsen worker 29 - jepsen.util 2549 :ok :write [9 6] INFO [2019-12-17 12:30:08,732] jepsen worker 29 - jepsen.util 2549 :invoke :read [9 nil] INFO [2019-12-17 12:30:08,740] jepsen worker 63 - jepsen.util 2763 :fail :cas [21 [39 2]] :ValueNotSwapFrom INFO [2019-12-17 12:30:08,740] jepsen worker 63 - jepsen.util 2763 :invoke :read [21 nil] INFO [2019-12-17 12:30:08,749] jepsen worker 29 - jepsen.util 2549 :ok :read [9 6] INFO [2019-12-17 12:30:08,750] jepsen worker 29 - jepsen.util 2549 :invoke :write [9 26] INFO [2019-12-17 12:30:08,772] jepsen worker 63 - jepsen.util 2763 :ok :read [21 21] INFO [2019-12-17 12:30:08,772] jepsen worker 63 - jepsen.util 2763 :invoke :read [21 nil] INFO [2019-12-17 12:30:08,788] jepsen worker 87 - jepsen.util 2877 :fail :cas [29 [46 33]] :ValueNotSwapFrom INFO [2019-12-17 12:30:08,788] jepsen worker 87 - jepsen.util 2877 :invoke :read [29 nil] INFO [2019-12-17 12:30:08,790] jepsen worker 52 - jepsen.util 3202 :info :write [17 23] :RequestCanceledException INFO [2019-12-17 12:30:08,791] jepsen worker 52 - jepsen.util 3292 :invoke :read [17 nil] INFO [2019-12-17 12:30:08,792] jepsen worker 63 - jepsen.util 2763 :ok :read [21 21] INFO [2019-12-17 12:30:08,792] jepsen worker 63 - jepsen.util 2763 :invoke :read [21 nil] INFO [2019-12-17 12:30:08,804] jepsen worker 63 - jepsen.util 2763 :ok :read [21 21] INFO [2019-12-17 12:30:08,805] jepsen worker 63 - jepsen.util 2763 :invoke :cas [21 [25 6]] INFO [2019-12-17 12:30:08,805] jepsen worker 79 - jepsen.util 3049 :ok :read [26 27] INFO [2019-12-17 12:30:08,805] jepsen worker 79 - jepsen.util 3049 :invoke :read [26 nil] INFO [2019-12-17 12:30:08,807] jepsen worker 78 - jepsen.util 3408 :fail :cas [26 [39 1]] :ValueNotSwapFrom INFO [2019-12-17 12:30:08,807] jepsen worker 78 - jepsen.util 3408 :invoke :read [26 nil] INFO [2019-12-17 12:30:08,823] jepsen worker 34 - jepsen.util 4084 :info :write [11 2] :RequestCanceledException INFO [2019-12-17 12:30:08,824] jepsen worker 34 - jepsen.util 4174 :invoke :write [11 35] INFO [2019-12-17 12:30:08,833] jepsen worker 29 - jepsen.util 2549 :ok :write [9 26] INFO [2019-12-17 12:30:08,833] jepsen worker 29 - jepsen.util 2549 :invoke :write [9 5] INFO [2019-12-17 12:30:08,835] jepsen worker 63 - jepsen.util 2763 :fail :cas [21 [25 6]] :ValueNotSwapFrom INFO [2019-12-17 12:30:08,835] jepsen worker 63 - jepsen.util 2763 :invoke :read [21 nil] INFO [2019-12-17 12:30:08,852] jepsen worker 87 - jepsen.util 2877 :ok :read [29 17] INFO [2019-12-17 12:30:08,852] jepsen worker 87 - jepsen.util 2877 :invoke :write [29 47] INFO [2019-12-17 12:30:08,870] jepsen worker 63 - jepsen.util 2763 :ok :read [21 21] INFO [2019-12-17 12:30:08,870] jepsen worker 63 - jepsen.util 2763 :invoke :write [21 38] INFO [2019-12-17 12:30:08,887] jepsen worker 79 - jepsen.util 3049 :ok :read [26 27] INFO [2019-12-17 12:30:08,887] jepsen worker 79 - jepsen.util 3049 :invoke :cas [26 [25 4]] INFO [2019-12-17 12:30:08,910] jepsen worker 88 - jepsen.util 3058 :info :write [29 33] :RequestCanceledException INFO [2019-12-17 12:30:08,912] jepsen worker 88 - jepsen.util 3148 :invoke :write [29 21] INFO [2019-12-17 12:30:08,916] jepsen worker 78 - jepsen.util 3408 :ok :read [26 27] INFO [2019-12-17 12:30:08,916] jepsen worker 78 - jepsen.util 3408 :invoke :cas [26 [16 36]] INFO [2019-12-17 12:30:08,923] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":17991,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:30:08,923] jepsen worker 7 - jepsen.util 3787 :info :cas [2 [14 39]] :RequestCanceledException INFO [2019-12-17 12:30:08,923] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":17991,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:30:08,923] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":17994,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:30:08,923] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":17994,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:30:08,924] jepsen worker 56 - jepsen.util 2936 :info :write [18 35] :RequestCanceledException INFO [2019-12-17 12:30:08,924] jepsen worker 7 - jepsen.util 3877 :invoke :write [2 26] INFO [2019-12-17 12:30:08,925] jepsen worker 56 - jepsen.util 3026 :invoke :write [18 43] INFO [2019-12-17 12:30:08,935] jepsen worker 63 - jepsen.util 2763 :ok :write [21 38] INFO [2019-12-17 12:30:08,935] jepsen worker 63 - jepsen.util 2763 :invoke :read [21 nil] INFO [2019-12-17 12:30:08,937] jepsen worker 18 - jepsen.util 3168 :info :write [6 30] :RequestCanceledException INFO [2019-12-17 12:30:08,938] jepsen worker 18 - jepsen.util 3258 :invoke :write [6 46] INFO [2019-12-17 12:30:08,946] jepsen worker 29 - jepsen.util 2549 :ok :write [9 5] INFO [2019-12-17 12:30:08,946] jepsen worker 29 - jepsen.util 2549 :invoke :write [9 21] INFO [2019-12-17 12:30:08,955] jepsen worker 64 - jepsen.util 2584 :info :write [21 11] :RequestCanceledException INFO [2019-12-17 12:30:08,955] jepsen worker 23 - jepsen.util 3983 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:30:08,955] jepsen worker 23 - jepsen.util 3983 :invoke :write [7 30] INFO [2019-12-17 12:30:08,956] jepsen worker 64 - jepsen.util 2674 :invoke :read [21 nil] INFO [2019-12-17 12:30:08,964] jepsen worker 63 - jepsen.util 2763 :ok :read [21 38] INFO [2019-12-17 12:30:08,964] jepsen worker 63 - jepsen.util 2763 :invoke :cas [21 [29 34]] INFO [2019-12-17 12:30:08,977] jepsen worker 54 - jepsen.util 3204 :ok :read [18 4] INFO [2019-12-17 12:30:08,977] jepsen worker 54 - jepsen.util 3204 :invoke :cas [18 [47 12]] INFO [2019-12-17 12:30:09,024] jepsen worker 74 - jepsen.util 3314 :fail :read [24 nil] :RequestTimeoutException INFO [2019-12-17 12:30:09,024] jepsen worker 74 - jepsen.util 3314 :invoke :read [24 nil] INFO [2019-12-17 12:30:09,028] jepsen worker 63 - jepsen.util 2763 :fail :cas [21 [29 34]] :ValueNotSwapFrom INFO [2019-12-17 12:30:09,028] jepsen worker 63 - jepsen.util 2763 :invoke :cas [21 [37 30]] INFO [2019-12-17 12:30:09,030] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18010,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:30:09,030] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18010,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:30:09,030] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18014,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:30:09,030] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18014,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:30:09,037] jepsen worker 65 - jepsen.util 2765 :info :write [21 44] :RequestCanceledException INFO [2019-12-17 12:30:09,038] jepsen worker 65 - jepsen.util 2855 :invoke :cas [21 [26 48]] INFO [2019-12-17 12:30:09,041] jepsen worker 36 - jepsen.util 3096 :fail :cas [12 [35 31]] :ValueNotSwapFrom INFO [2019-12-17 12:30:09,041] jepsen worker 63 - jepsen.util 2763 :fail :cas [21 [37 30]] :ValueNotSwapFrom INFO [2019-12-17 12:30:09,041] jepsen worker 36 - jepsen.util 3096 :invoke :write [12 14] INFO [2019-12-17 12:30:09,041] jepsen worker 63 - jepsen.util 2763 :invoke :read [21 nil] INFO [2019-12-17 12:30:09,050] jepsen worker 29 - jepsen.util 2549 :ok :write [9 21] INFO [2019-12-17 12:30:09,050] jepsen worker 29 - jepsen.util 2549 :invoke :write [9 28] INFO [2019-12-17 12:30:09,065] jepsen worker 63 - jepsen.util 2763 :ok :read [21 38] INFO [2019-12-17 12:30:09,066] jepsen worker 63 - jepsen.util 2763 :invoke :write [21 9] INFO [2019-12-17 12:30:09,072] jepsen worker 65 - jepsen.util 2855 :fail :cas [21 [26 48]] :ValueNotSwapFrom INFO [2019-12-17 12:30:09,072] jepsen worker 65 - jepsen.util 2855 :invoke :write [21 41] INFO [2019-12-17 12:30:09,102] jepsen worker 9 - jepsen.util 3609 :ok :write [3 23] INFO [2019-12-17 12:30:09,102] jepsen worker 9 - jepsen.util 3609 :invoke :cas [3 [7 46]] INFO [2019-12-17 12:30:09,102] jepsen worker 19 - jepsen.util 3349 :ok :write [6 14] INFO [2019-12-17 12:30:09,102] jepsen worker 19 - jepsen.util 3349 :invoke :read [6 nil] INFO [2019-12-17 12:30:09,106] jepsen worker 30 - jepsen.util 3720 :ok :write [10 27] INFO [2019-12-17 12:30:09,106] jepsen worker 30 - jepsen.util 3720 :invoke :cas [10 [27 44]] INFO [2019-12-17 12:30:09,107] jepsen worker 46 - jepsen.util 3286 :info :write [15 5] :RequestCanceledException INFO [2019-12-17 12:30:09,109] jepsen worker 46 - jepsen.util 3376 :invoke :cas [15 [1 30]] INFO [2019-12-17 12:30:09,123] jepsen worker 28 - jepsen.util 2278 :fail :read [9 nil] :RequestTimeoutException INFO [2019-12-17 12:30:09,124] jepsen worker 28 - jepsen.util 2278 :invoke :write [9 16] INFO [2019-12-17 12:30:09,155] jepsen worker 74 - jepsen.util 3314 :ok :read [24 26] INFO [2019-12-17 12:30:09,155] jepsen worker 74 - jepsen.util 3314 :invoke :cas [24 [47 10]] INFO [2019-12-17 12:30:09,192] jepsen worker 78 - jepsen.util 3408 :fail :cas [26 [16 36]] :ValueNotSwapFrom INFO [2019-12-17 12:30:09,192] jepsen worker 78 - jepsen.util 3408 :invoke :cas [26 [10 32]] INFO [2019-12-17 12:30:09,201] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18030,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:30:09,201] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18030,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:30:09,201] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18031,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:30:09,201] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18031,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:30:09,203] jepsen worker 64 - jepsen.util 2674 :ok :read [21 38] INFO [2019-12-17 12:30:09,203] jepsen worker 64 - jepsen.util 2674 :invoke :write [21 0] INFO [2019-12-17 12:30:09,209] jepsen worker 31 - jepsen.util 2641 :fail :read [10 nil] :RequestTimeoutException INFO [2019-12-17 12:30:09,209] jepsen worker 70 - jepsen.util 3580 :info :write [23 8] :RequestCanceledException INFO [2019-12-17 12:30:09,209] jepsen worker 31 - jepsen.util 2641 :invoke :cas [10 [20 7]] INFO [2019-12-17 12:30:09,210] jepsen worker 70 - jepsen.util 3670 :invoke :write [23 41] INFO [2019-12-17 12:30:09,221] jepsen worker 10 - jepsen.util 2800 :info :write [3 2] :RequestCanceledException INFO [2019-12-17 12:30:09,224] jepsen worker 10 - jepsen.util 2890 :invoke :write [3 9] INFO [2019-12-17 12:30:09,228] jepsen worker 9 - jepsen.util 3609 :fail :cas [3 [7 46]] :ValueNotSwapFrom INFO [2019-12-17 12:30:09,228] jepsen worker 9 - jepsen.util 3609 :invoke :write [3 12] INFO [2019-12-17 12:30:09,228] jepsen worker 16 - jepsen.util 4336 :info :write [5 6] :RequestCanceledException INFO [2019-12-17 12:30:09,231] jepsen worker 16 - jepsen.util 4426 :invoke :write [5 39] INFO [2019-12-17 12:30:09,251] jepsen worker 52 - jepsen.util 3292 :ok :read [17 27] INFO [2019-12-17 12:30:09,251] jepsen worker 52 - jepsen.util 3292 :invoke :write [17 1] INFO [2019-12-17 12:30:09,254] jepsen worker 35 - jepsen.util 3995 :info :write [11 12] :RequestCanceledException INFO [2019-12-17 12:30:09,255] jepsen worker 35 - jepsen.util 4085 :invoke :read [11 nil] INFO [2019-12-17 12:30:09,259] jepsen worker 19 - jepsen.util 3349 :ok :read [6 14] INFO [2019-12-17 12:30:09,259] jepsen worker 19 - jepsen.util 3349 :invoke :write [6 32] INFO [2019-12-17 12:30:09,282] jepsen worker 54 - jepsen.util 3204 :fail :cas [18 [47 12]] :ValueNotSwapFrom INFO [2019-12-17 12:30:09,283] jepsen worker 54 - jepsen.util 3204 :invoke :cas [18 [9 40]] INFO [2019-12-17 12:30:09,287] jepsen worker 46 - jepsen.util 3376 :fail :cas [15 [1 30]] :ValueNotSwapFrom INFO [2019-12-17 12:30:09,288] jepsen worker 46 - jepsen.util 3376 :invoke :write [15 7] INFO [2019-12-17 12:30:09,296] jepsen worker 65 - jepsen.util 2855 :ok :write [21 41] INFO [2019-12-17 12:30:09,296] jepsen worker 65 - jepsen.util 2855 :invoke :read [21 nil] INFO [2019-12-17 12:30:09,301] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18041,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:30:09,301] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18041,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:30:09,301] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18044,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:30:09,301] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18044,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:30:09,303] jepsen worker 29 - jepsen.util 2549 :ok :write [9 28] INFO [2019-12-17 12:30:09,303] jepsen worker 29 - jepsen.util 2549 :invoke :write [9 0] INFO [2019-12-17 12:30:09,338] jepsen worker 80 - jepsen.util 2870 :info :write [26 38] :RequestCanceledException INFO [2019-12-17 12:30:09,339] jepsen worker 65 - jepsen.util 2855 :ok :read [21 41] INFO [2019-12-17 12:30:09,339] jepsen worker 80 - jepsen.util 2960 :invoke :write [26 3] INFO [2019-12-17 12:30:09,339] jepsen worker 65 - jepsen.util 2855 :invoke :read [21 nil] INFO [2019-12-17 12:30:09,396] jepsen worker 31 - jepsen.util 2641 :fail :cas [10 [20 7]] :ValueNotSwapFrom INFO [2019-12-17 12:30:09,396] jepsen worker 31 - jepsen.util 2641 :invoke :read [10 nil] INFO [2019-12-17 12:30:09,408] jepsen worker 59 - jepsen.util 4019 :info :cas [19 [20 11]] :RequestCanceledException INFO [2019-12-17 12:30:09,409] jepsen worker 59 - jepsen.util 4109 :invoke :cas [19 [20 41]] INFO [2019-12-17 12:30:09,409] jepsen worker 65 - jepsen.util 2855 :ok :read [21 41] INFO [2019-12-17 12:30:09,409] jepsen worker 65 - jepsen.util 2855 :invoke :cas [21 [30 40]] INFO [2019-12-17 12:30:09,430] jepsen worker 28 - jepsen.util 2278 :ok :write [9 16] INFO [2019-12-17 12:30:09,430] jepsen worker 28 - jepsen.util 2278 :invoke :cas [9 [20 24]] INFO [2019-12-17 12:30:09,438] jepsen worker 32 - jepsen.util 3542 :info :cas [10 [47 6]] :RequestCanceledException INFO [2019-12-17 12:30:09,439] jepsen worker 32 - jepsen.util 3632 :invoke :write [10 7] INFO [2019-12-17 12:30:09,440] jepsen worker 65 - jepsen.util 2855 :fail :cas [21 [30 40]] :ValueNotSwapFrom INFO [2019-12-17 12:30:09,440] jepsen worker 65 - jepsen.util 2855 :invoke :cas [21 [9 8]] INFO [2019-12-17 12:30:09,450] jepsen worker 14 - jepsen.util 2894 :info :write [4 42] :RequestCanceledException INFO [2019-12-17 12:30:09,450] jepsen worker 15 - jepsen.util 3795 :info :cas [5 [18 43]] :RequestCanceledException INFO [2019-12-17 12:30:09,451] jepsen worker 14 - jepsen.util 2984 :invoke :read [4 nil] INFO [2019-12-17 12:30:09,452] jepsen worker 15 - jepsen.util 3885 :invoke :read [5 nil] INFO [2019-12-17 12:30:09,460] jepsen worker 30 - jepsen.util 3720 :ok :cas [10 [27 44]] INFO [2019-12-17 12:30:09,460] jepsen worker 30 - jepsen.util 3720 :invoke :read [10 nil] INFO [2019-12-17 12:30:09,470] jepsen worker 64 - jepsen.util 2674 :ok :write [21 0] INFO [2019-12-17 12:30:09,470] jepsen worker 64 - jepsen.util 2674 :invoke :cas [21 [35 38]] INFO [2019-12-17 12:30:09,471] jepsen worker 78 - jepsen.util 3408 :fail :cas [26 [10 32]] :ValueNotSwapFrom INFO [2019-12-17 12:30:09,471] jepsen worker 78 - jepsen.util 3408 :invoke :read [26 nil] INFO [2019-12-17 12:30:09,477] jepsen worker 65 - jepsen.util 2855 :fail :cas [21 [9 8]] :ValueNotSwapFrom INFO [2019-12-17 12:30:09,477] jepsen worker 65 - jepsen.util 2855 :invoke :read [21 nil] INFO [2019-12-17 12:30:09,509] jepsen worker 43 - jepsen.util 3733 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:30:09,509] jepsen worker 43 - jepsen.util 3733 :invoke :write [14 31] INFO [2019-12-17 12:30:09,512] jepsen worker 64 - jepsen.util 2674 :fail :cas [21 [35 38]] :ValueNotSwapFrom INFO [2019-12-17 12:30:09,512] jepsen worker 64 - jepsen.util 2674 :invoke :write [21 4] INFO [2019-12-17 12:30:09,513] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18063,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:30:09,514] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18063,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:30:09,514] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18064,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:30:09,514] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18064,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:30:09,521] jepsen worker 65 - jepsen.util 2855 :ok :read [21 0] INFO [2019-12-17 12:30:09,521] jepsen worker 65 - jepsen.util 2855 :invoke :read [21 nil] INFO [2019-12-17 12:30:09,524] jepsen worker 13 - jepsen.util 2983 :info :write [4 21] :RequestCanceledException INFO [2019-12-17 12:30:09,524] jepsen worker 84 - jepsen.util 4044 :info :write [28 29] :RequestCanceledException INFO [2019-12-17 12:30:09,524] jepsen worker 45 - jepsen.util 3015 :info :cas [15 [38 18]] :RequestCanceledException INFO [2019-12-17 12:30:09,525] jepsen worker 45 - jepsen.util 3105 :invoke :cas [15 [41 48]] INFO [2019-12-17 12:30:09,530] jepsen worker 84 - jepsen.util 4134 :invoke :read [28 nil] INFO [2019-12-17 12:30:09,538] jepsen worker 47 - jepsen.util 3647 :info :cas [15 [48 27]] :RequestCanceledException INFO [2019-12-17 12:30:09,539] jepsen worker 47 - jepsen.util 3737 :invoke :write [15 38] INFO [2019-12-17 12:30:09,540] jepsen worker 65 - jepsen.util 2855 :ok :read [21 0] INFO [2019-12-17 12:30:09,540] jepsen worker 65 - jepsen.util 2855 :invoke :read [21 nil] INFO [2019-12-17 12:30:09,541] jepsen worker 13 - jepsen.util 3073 :invoke :read [4 nil] INFO [2019-12-17 12:30:09,555] jepsen worker 85 - jepsen.util 4585 :info :cas [28 [26 48]] :RequestCanceledException INFO [2019-12-17 12:30:09,557] jepsen worker 85 - jepsen.util 4675 :invoke :write [28 28] INFO [2019-12-17 12:30:09,578] jepsen worker 65 - jepsen.util 2855 :ok :read [21 0] INFO [2019-12-17 12:30:09,578] jepsen worker 65 - jepsen.util 2855 :invoke :read [21 nil] INFO [2019-12-17 12:30:09,580] jepsen worker 79 - jepsen.util 3049 :fail :cas [26 [25 4]] :ValueNotSwapFrom INFO [2019-12-17 12:30:09,580] jepsen worker 79 - jepsen.util 3049 :invoke :cas [26 [26 22]] INFO [2019-12-17 12:30:09,590] jepsen worker 38 - jepsen.util 3008 :info :write [12 47] :RequestCanceledException INFO [2019-12-17 12:30:09,591] jepsen worker 38 - jepsen.util 3098 :invoke :read [12 nil] INFO [2019-12-17 12:30:09,605] jepsen worker 65 - jepsen.util 2855 :ok :read [21 0] INFO [2019-12-17 12:30:09,605] jepsen worker 65 - jepsen.util 2855 :invoke :write [21 29] INFO [2019-12-17 12:30:09,608] jepsen worker 62 - jepsen.util 4022 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:30:09,608] jepsen worker 62 - jepsen.util 4022 :invoke :read [20 nil] INFO [2019-12-17 12:30:09,623] jepsen worker 50 - jepsen.util 4640 :info :write [16 38] :RequestCanceledException INFO [2019-12-17 12:30:09,624] jepsen worker 50 - jepsen.util 4730 :invoke :read [16 nil] INFO [2019-12-17 12:30:09,625] jepsen worker 10 - jepsen.util 2890 :ok :write [3 9] INFO [2019-12-17 12:30:09,626] jepsen worker 10 - jepsen.util 2890 :invoke :read [3 nil] INFO [2019-12-17 12:30:09,628] jepsen worker 32 - jepsen.util 3632 :ok :write [10 7] INFO [2019-12-17 12:30:09,628] jepsen worker 32 - jepsen.util 3632 :invoke :read [10 nil] INFO [2019-12-17 12:30:09,635] jepsen worker 46 - jepsen.util 3376 :ok :write [15 7] INFO [2019-12-17 12:30:09,635] jepsen worker 46 - jepsen.util 3376 :invoke :cas [15 [11 26]] INFO [2019-12-17 12:30:09,650] jepsen worker 64 - jepsen.util 2674 :ok :write [21 4] INFO [2019-12-17 12:30:09,651] jepsen worker 64 - jepsen.util 2674 :invoke :write [21 14] INFO [2019-12-17 12:30:09,655] jepsen worker 1 - jepsen.util 3601 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:30:09,655] jepsen worker 73 - jepsen.util 2863 :info :write [24 16] :RequestCanceledException INFO [2019-12-17 12:30:09,655] jepsen worker 40 - jepsen.util 4360 :info :write [13 26] :RequestCanceledException INFO [2019-12-17 12:30:09,655] jepsen worker 1 - jepsen.util 3601 :invoke :write [0 43] INFO [2019-12-17 12:30:09,656] jepsen worker 73 - jepsen.util 2953 :invoke :read [24 nil] INFO [2019-12-17 12:30:09,656] jepsen worker 40 - jepsen.util 4450 :invoke :write [13 13] INFO [2019-12-17 12:30:09,661] jepsen worker 54 - jepsen.util 3204 :fail :cas [18 [9 40]] :ValueNotSwapFrom INFO [2019-12-17 12:30:09,662] jepsen worker 54 - jepsen.util 3204 :invoke :write [18 46] INFO [2019-12-17 12:30:09,681] jepsen worker 14 - jepsen.util 2984 :ok :read [4 41] INFO [2019-12-17 12:30:09,681] jepsen worker 14 - jepsen.util 2984 :invoke :cas [4 [47 3]] INFO [2019-12-17 12:30:09,690] jepsen worker 21 - jepsen.util 4071 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:30:09,690] jepsen worker 21 - jepsen.util 4071 :invoke :write [7 36] INFO [2019-12-17 12:30:09,692] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18096,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:30:09,692] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18096,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:30:09,692] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18099,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:30:09,692] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18099,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:30:09,697] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:30:09,697] jepsen nemesis - couchbase.util Rebalance check count: 132 INFO [2019-12-17 12:30:09,697] jepsen nemesis - couchbase.util Rebalance stuck count: 113 INFO [2019-12-17 12:30:09,698] jepsen worker 31 - jepsen.util 2641 :ok :read [10 44] INFO [2019-12-17 12:30:09,698] jepsen worker 31 - jepsen.util 2641 :invoke :cas [10 [29 40]] INFO [2019-12-17 12:30:09,705] jepsen worker 22 - jepsen.util 4252 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:30:09,705] jepsen worker 22 - jepsen.util 4252 :invoke :write [7 15] INFO [2019-12-17 12:30:09,723] jepsen worker 75 - jepsen.util 4305 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:30:09,723] jepsen worker 75 - jepsen.util 4305 :invoke :read [25 nil] INFO [2019-12-17 12:30:09,724] jepsen worker 45 - jepsen.util 3105 :fail :cas [15 [41 48]] :ValueNotSwapFrom INFO [2019-12-17 12:30:09,724] jepsen worker 11 - jepsen.util 3431 :info :cas [3 [48 7]] :RequestCanceledException INFO [2019-12-17 12:30:09,725] jepsen worker 45 - jepsen.util 3105 :invoke :read [15 nil] INFO [2019-12-17 12:30:09,726] jepsen worker 11 - jepsen.util 3521 :invoke :read [3 nil] INFO [2019-12-17 12:30:09,756] jepsen worker 32 - jepsen.util 3632 :ok :read [10 7] INFO [2019-12-17 12:30:09,756] jepsen worker 32 - jepsen.util 3632 :invoke :read [10 nil] INFO [2019-12-17 12:30:09,773] jepsen worker 11 - jepsen.util 3521 :ok :read [3 9] INFO [2019-12-17 12:30:09,773] jepsen worker 11 - jepsen.util 3521 :invoke :write [3 20] INFO [2019-12-17 12:30:09,793] jepsen worker 10 - jepsen.util 2890 :ok :read [3 9] INFO [2019-12-17 12:30:09,793] jepsen worker 10 - jepsen.util 2890 :invoke :read [3 nil] INFO [2019-12-17 12:30:09,835] jepsen worker 31 - jepsen.util 2641 :fail :cas [10 [29 40]] :ValueNotSwapFrom INFO [2019-12-17 12:30:09,835] jepsen worker 31 - jepsen.util 2641 :invoke :read [10 nil] INFO [2019-12-17 12:30:09,838] jepsen worker 42 - jepsen.util 4812 :info :cas [14 [27 10]] :RequestCanceledException INFO [2019-12-17 12:30:09,847] jepsen worker 42 - jepsen.util 4902 :invoke :write [14 32] INFO [2019-12-17 12:30:09,854] jepsen worker 41 - jepsen.util 4271 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:30:09,854] jepsen worker 41 - jepsen.util 4271 :invoke :read [13 nil] INFO [2019-12-17 12:30:09,865] jepsen worker 45 - jepsen.util 3105 :ok :read [15 7] INFO [2019-12-17 12:30:09,866] jepsen worker 45 - jepsen.util 3105 :invoke :write [15 39] INFO [2019-12-17 12:30:09,888] jepsen worker 57 - jepsen.util 3837 :info :write [19 6] :RequestCanceledException INFO [2019-12-17 12:30:09,895] jepsen worker 57 - jepsen.util 3927 :invoke :write [19 43] INFO [2019-12-17 12:30:09,905] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18119,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:30:09,905] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18119,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:30:09,905] jepsen worker 0 - jepsen.util 4590 :info :write [0 5] :RequestCanceledException INFO [2019-12-17 12:30:09,905] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18123,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:30:09,905] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18123,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:30:09,909] jepsen worker 0 - jepsen.util 4680 :invoke :write [0 31] INFO [2019-12-17 12:30:09,919] jepsen worker 3 - jepsen.util 3063 :info :write [1 10] :RequestCanceledException INFO [2019-12-17 12:30:09,919] jepsen worker 33 - jepsen.util 4443 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:30:09,919] jepsen worker 48 - jepsen.util 3558 :info :write [16 17] :RequestCanceledException INFO [2019-12-17 12:30:09,919] jepsen worker 33 - jepsen.util 4443 :invoke :cas [11 [32 46]] INFO [2019-12-17 12:30:09,920] jepsen worker 3 - jepsen.util 3153 :invoke :read [1 nil] INFO [2019-12-17 12:30:09,923] jepsen worker 48 - jepsen.util 3648 :invoke :write [16 25] INFO [2019-12-17 12:30:09,929] jepsen worker 32 - jepsen.util 3632 :ok :read [10 7] INFO [2019-12-17 12:30:09,930] jepsen worker 32 - jepsen.util 3632 :invoke :cas [10 [45 31]] INFO [2019-12-17 12:30:09,936] jepsen worker 72 - jepsen.util 3132 :fail :read [24 nil] :RequestTimeoutException INFO [2019-12-17 12:30:09,936] jepsen worker 72 - jepsen.util 3132 :invoke :read [24 nil] INFO [2019-12-17 12:30:09,939] jepsen worker 65 - jepsen.util 2855 :ok :write [21 29] INFO [2019-12-17 12:30:09,939] jepsen worker 65 - jepsen.util 2855 :invoke :read [21 nil] INFO [2019-12-17 12:30:09,952] jepsen worker 8 - jepsen.util 3428 :info :write [2 37] :RequestCanceledException INFO [2019-12-17 12:30:09,952] jepsen worker 83 - jepsen.util 4223 :info :cas [27 [27 14]] :RequestCanceledException INFO [2019-12-17 12:30:09,954] jepsen worker 10 - jepsen.util 2890 :ok :read [3 9] INFO [2019-12-17 12:30:09,954] jepsen worker 10 - jepsen.util 2890 :invoke :cas [3 [41 35]] INFO [2019-12-17 12:30:09,955] jepsen worker 8 - jepsen.util 3518 :invoke :write [2 39] INFO [2019-12-17 12:30:09,955] jepsen worker 83 - jepsen.util 4313 :invoke :cas [27 [33 34]] INFO [2019-12-17 12:30:09,985] jepsen worker 26 - jepsen.util 4436 :info :cas [8 [6 13]] :RequestCanceledException INFO [2019-12-17 12:30:09,986] jepsen worker 31 - jepsen.util 2641 :ok :read [10 7] INFO [2019-12-17 12:30:09,986] jepsen worker 31 - jepsen.util 2641 :invoke :read [10 nil] INFO [2019-12-17 12:30:09,989] jepsen worker 26 - jepsen.util 4526 :invoke :read [8 nil] INFO [2019-12-17 12:30:10,008] jepsen worker 4 - jepsen.util 2344 :info :write [1 5] :RequestCanceledException INFO [2019-12-17 12:30:10,010] jepsen worker 4 - jepsen.util 2434 :invoke :cas [1 [22 30]] INFO [2019-12-17 12:30:10,014] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18140,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:30:10,015] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18140,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:30:10,015] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18144,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:30:10,015] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18144,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:30:10,023] jepsen worker 17 - jepsen.util 4067 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:30:10,024] jepsen worker 17 - jepsen.util 4067 :invoke :read [5 nil] INFO [2019-12-17 12:30:10,024] jepsen worker 20 - jepsen.util 2630 :info :write [6 42] :RequestCanceledException INFO [2019-12-17 12:30:10,025] jepsen worker 20 - jepsen.util 2720 :invoke :write [6 34] INFO [2019-12-17 12:30:10,034] jepsen worker 73 - jepsen.util 2953 :ok :read [24 26] INFO [2019-12-17 12:30:10,034] jepsen worker 73 - jepsen.util 2953 :invoke :write [24 47] INFO [2019-12-17 12:30:10,041] jepsen worker 56 - jepsen.util 3026 :ok :write [18 43] INFO [2019-12-17 12:30:10,041] jepsen worker 56 - jepsen.util 3026 :invoke :write [18 46] INFO [2019-12-17 12:30:10,082] jepsen worker 65 - jepsen.util 2855 :ok :read [21 29] INFO [2019-12-17 12:30:10,082] jepsen worker 65 - jepsen.util 2855 :invoke :read [21 nil] INFO [2019-12-17 12:30:10,093] jepsen worker 10 - jepsen.util 2890 :fail :cas [3 [41 35]] :ValueNotSwapFrom INFO [2019-12-17 12:30:10,093] jepsen worker 10 - jepsen.util 2890 :invoke :write [3 6] INFO [2019-12-17 12:30:10,100] jepsen worker 31 - jepsen.util 2641 :ok :read [10 7] INFO [2019-12-17 12:30:10,100] jepsen worker 31 - jepsen.util 2641 :invoke :cas [10 [6 11]] INFO [2019-12-17 12:30:10,124] jepsen worker 27 - jepsen.util 2727 :info :cas [9 [23 38]] :RequestCanceledException INFO [2019-12-17 12:30:10,125] jepsen worker 27 - jepsen.util 2817 :invoke :read [9 nil] INFO [2019-12-17 12:30:10,148] jepsen worker 38 - jepsen.util 3098 :ok :read [12 20] INFO [2019-12-17 12:30:10,148] jepsen worker 38 - jepsen.util 3098 :invoke :cas [12 [28 30]] INFO [2019-12-17 12:30:10,184] jepsen worker 31 - jepsen.util 2641 :fail :cas [10 [6 11]] :ValueNotSwapFrom INFO [2019-12-17 12:30:10,184] jepsen worker 31 - jepsen.util 2641 :invoke :read [10 nil] INFO [2019-12-17 12:30:10,203] jepsen worker 27 - jepsen.util 2817 :ok :read [9 16] INFO [2019-12-17 12:30:10,203] jepsen worker 27 - jepsen.util 2817 :invoke :read [9 nil] INFO [2019-12-17 12:30:10,214] jepsen worker 14 - jepsen.util 2984 :fail :cas [4 [47 3]] :ValueNotSwapFrom INFO [2019-12-17 12:30:10,214] jepsen worker 14 - jepsen.util 2984 :invoke :write [4 13] INFO [2019-12-17 12:30:10,225] jepsen worker 37 - jepsen.util 2827 :info :write [12 42] :RequestCanceledException INFO [2019-12-17 12:30:10,229] jepsen worker 37 - jepsen.util 2917 :invoke :write [12 40] INFO [2019-12-17 12:30:10,249] jepsen worker 31 - jepsen.util 2641 :ok :read [10 7] INFO [2019-12-17 12:30:10,249] jepsen worker 31 - jepsen.util 2641 :invoke :cas [10 [35 37]] INFO [2019-12-17 12:30:10,256] jepsen worker 27 - jepsen.util 2817 :ok :read [9 16] INFO [2019-12-17 12:30:10,256] jepsen worker 27 - jepsen.util 2817 :invoke :write [9 1] INFO [2019-12-17 12:30:10,259] jepsen worker 11 - jepsen.util 3521 :ok :write [3 20] INFO [2019-12-17 12:30:10,259] jepsen worker 11 - jepsen.util 3521 :invoke :write [3 49] INFO [2019-12-17 12:30:10,286] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18164,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:30:10,286] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18164,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:30:10,286] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18168,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:30:10,286] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18168,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:30:10,297] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18156,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:30:10,297] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18156,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:30:10,297] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18157,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:30:10,297] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18157,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:30:10,297] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18158,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:30:10,297] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18158,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:30:10,298] jepsen worker 31 - jepsen.util 2641 :fail :cas [10 [35 37]] :ValueNotSwapFrom INFO [2019-12-17 12:30:10,298] jepsen worker 31 - jepsen.util 2641 :invoke :cas [10 [18 16]] INFO [2019-12-17 12:30:10,330] jepsen worker 32 - jepsen.util 3632 :fail :cas [10 [45 31]] :ValueNotSwapFrom INFO [2019-12-17 12:30:10,331] jepsen worker 32 - jepsen.util 3632 :invoke :cas [10 [21 7]] INFO [2019-12-17 12:30:10,336] jepsen worker 27 - jepsen.util 2817 :ok :write [9 1] INFO [2019-12-17 12:30:10,337] jepsen worker 27 - jepsen.util 2817 :invoke :cas [9 [39 1]] INFO [2019-12-17 12:30:10,337] jepsen worker 25 - jepsen.util 4615 :info :cas [8 [9 27]] :RequestCanceledException INFO [2019-12-17 12:30:10,338] jepsen worker 25 - jepsen.util 4705 :invoke :read [8 nil] INFO [2019-12-17 12:30:10,355] jepsen worker 81 - jepsen.util 4131 :info :write [27 15] :RequestCanceledException INFO [2019-12-17 12:30:10,357] jepsen worker 81 - jepsen.util 4221 :invoke :read [27 nil] INFO [2019-12-17 12:30:10,376] jepsen worker 31 - jepsen.util 2641 :fail :cas [10 [18 16]] :ValueNotSwapFrom INFO [2019-12-17 12:30:10,376] jepsen worker 31 - jepsen.util 2641 :invoke :write [10 5] INFO [2019-12-17 12:30:10,390] jepsen worker 51 - jepsen.util 2931 :info :write [17 16] :RequestCanceledException INFO [2019-12-17 12:30:10,391] jepsen worker 51 - jepsen.util 3021 :invoke :cas [17 [49 13]] INFO [2019-12-17 12:30:10,393] jepsen worker 27 - jepsen.util 2817 :fail :cas [9 [39 1]] :ValueNotSwapFrom INFO [2019-12-17 12:30:10,393] jepsen worker 27 - jepsen.util 2817 :invoke :read [9 nil] INFO [2019-12-17 12:30:10,408] jepsen worker 86 - jepsen.util 4406 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:30:10,408] jepsen worker 86 - jepsen.util 4406 :invoke :read [28 nil] INFO [2019-12-17 12:30:10,423] jepsen worker 76 - jepsen.util 4216 :info :cas [25 [1 14]] :RequestCanceledException INFO [2019-12-17 12:30:10,423] jepsen worker 58 - jepsen.util 4018 :info :cas [19 [33 12]] :RequestCanceledException INFO [2019-12-17 12:30:10,427] jepsen worker 76 - jepsen.util 4306 :invoke :cas [25 [31 43]] INFO [2019-12-17 12:30:10,427] jepsen worker 58 - jepsen.util 4108 :invoke :cas [19 [1 22]] INFO [2019-12-17 12:30:10,449] jepsen worker 27 - jepsen.util 2817 :ok :read [9 1] INFO [2019-12-17 12:30:10,449] jepsen worker 27 - jepsen.util 2817 :invoke :read [9 nil] INFO [2019-12-17 12:30:10,450] jepsen worker 78 - jepsen.util 3408 :ok :read [26 27] INFO [2019-12-17 12:30:10,450] jepsen worker 78 - jepsen.util 3408 :invoke :write [26 27] INFO [2019-12-17 12:30:10,454] jepsen worker 61 - jepsen.util 4651 :info :write [20 38] :RequestCanceledException INFO [2019-12-17 12:30:10,458] jepsen worker 61 - jepsen.util 4741 :invoke :cas [20 [15 24]] INFO [2019-12-17 12:30:10,503] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18184,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:30:10,503] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18184,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:30:10,503] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18188,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:30:10,503] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18188,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:30:10,523] jepsen worker 38 - jepsen.util 3098 :fail :cas [12 [28 30]] :ValueNotSwapFrom INFO [2019-12-17 12:30:10,524] jepsen worker 38 - jepsen.util 3098 :invoke :read [12 nil] INFO [2019-12-17 12:30:10,524] jepsen worker 82 - jepsen.util 4942 :info :write [27 5] :RequestCanceledException INFO [2019-12-17 12:30:10,525] jepsen worker 27 - jepsen.util 2817 :ok :read [9 1] INFO [2019-12-17 12:30:10,525] jepsen worker 27 - jepsen.util 2817 :invoke :write [9 31] INFO [2019-12-17 12:30:10,527] jepsen worker 13 - jepsen.util 3073 :ok :read [4 41] INFO [2019-12-17 12:30:10,527] jepsen worker 13 - jepsen.util 3073 :invoke :read [4 nil] INFO [2019-12-17 12:30:10,529] jepsen worker 82 - jepsen.util 5032 :invoke :write [27 25] INFO [2019-12-17 12:30:10,537] jepsen worker 2 - jepsen.util 3782 :info :write [0 25] :RequestCanceledException INFO [2019-12-17 12:30:10,537] jepsen worker 68 - jepsen.util 3938 :info :write [22 34] :RequestCanceledException INFO [2019-12-17 12:30:10,541] jepsen worker 2 - jepsen.util 3872 :invoke :read [0 nil] INFO [2019-12-17 12:30:10,542] jepsen worker 68 - jepsen.util 4028 :invoke :read [22 nil] INFO [2019-12-17 12:30:10,546] jepsen worker 3 - jepsen.util 3153 :ok :read [1 26] INFO [2019-12-17 12:30:10,546] jepsen worker 3 - jepsen.util 3153 :invoke :write [1 48] INFO [2019-12-17 12:30:10,555] jepsen worker 77 - jepsen.util 3857 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:30:10,555] jepsen worker 66 - jepsen.util 4026 :info :write [22 33] :RequestCanceledException INFO [2019-12-17 12:30:10,555] jepsen worker 77 - jepsen.util 3857 :invoke :write [25 7] INFO [2019-12-17 12:30:10,555] jepsen worker 39 - jepsen.util 3909 :info :cas [13 [39 44]] :RequestCanceledException INFO [2019-12-17 12:30:10,562] jepsen worker 39 - jepsen.util 3999 :invoke :read [13 nil] INFO [2019-12-17 12:30:10,564] jepsen worker 66 - jepsen.util 4116 :invoke :read [22 nil] INFO [2019-12-17 12:30:10,588] jepsen worker 27 - jepsen.util 2817 :ok :write [9 31] INFO [2019-12-17 12:30:10,588] jepsen worker 27 - jepsen.util 2817 :invoke :cas [9 [7 10]] INFO [2019-12-17 12:30:10,590] jepsen worker 24 - jepsen.util 3984 :info :write [8 12] :RequestCanceledException INFO [2019-12-17 12:30:10,594] jepsen worker 24 - jepsen.util 4074 :invoke :write [8 17] INFO [2019-12-17 12:30:10,608] jepsen worker 55 - jepsen.util 3745 :info :write [18 48] :RequestCanceledException INFO [2019-12-17 12:30:10,609] jepsen worker 55 - jepsen.util 3835 :invoke :cas [18 [6 1]] INFO [2019-12-17 12:30:10,624] jepsen worker 89 - jepsen.util 2879 :info :write [29 37] :RequestCanceledException INFO [2019-12-17 12:30:10,624] jepsen worker 67 - jepsen.util 4567 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:30:10,624] jepsen worker 67 - jepsen.util 4567 :invoke :write [22 16] INFO [2019-12-17 12:30:10,630] jepsen worker 89 - jepsen.util 2969 :invoke :write [29 8] INFO [2019-12-17 12:30:10,630] jepsen worker 79 - jepsen.util 3049 :fail :cas [26 [26 22]] :ValueNotSwapFrom INFO [2019-12-17 12:30:10,631] jepsen worker 79 - jepsen.util 3049 :invoke :cas [26 [12 24]] INFO [2019-12-17 12:30:10,638] jepsen worker 53 - jepsen.util 3113 :info :write [17 13] :RequestCanceledException INFO [2019-12-17 12:30:10,638] jepsen worker 71 - jepsen.util 3041 :info :write [23 22] :RequestCanceledException INFO [2019-12-17 12:30:10,639] jepsen worker 71 - jepsen.util 3131 :invoke :cas [23 [14 14]] INFO [2019-12-17 12:30:10,641] jepsen worker 27 - jepsen.util 2817 :fail :cas [9 [7 10]] :ValueNotSwapFrom INFO [2019-12-17 12:30:10,641] jepsen worker 27 - jepsen.util 2817 :invoke :read [9 nil] INFO [2019-12-17 12:30:10,643] jepsen worker 53 - jepsen.util 3203 :invoke :read [17 nil] INFO [2019-12-17 12:30:10,672] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18209,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:30:10,672] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18209,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:30:10,672] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18213,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:30:10,672] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18213,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:30:10,690] jepsen worker 44 - jepsen.util 4364 :info :write [14 36] :RequestCanceledException INFO [2019-12-17 12:30:10,697] jepsen worker 27 - jepsen.util 2817 :ok :read [9 31] INFO [2019-12-17 12:30:10,697] jepsen worker 27 - jepsen.util 2817 :invoke :read [9 nil] INFO [2019-12-17 12:30:10,697] jepsen worker 44 - jepsen.util 4454 :invoke :cas [14 [39 6]] INFO [2019-12-17 12:30:10,708] jepsen worker 6 - jepsen.util 4416 :info :cas [2 [1 2]] :RequestCanceledException INFO [2019-12-17 12:30:10,709] jepsen worker 6 - jepsen.util 4506 :invoke :cas [2 [40 28]] INFO [2019-12-17 12:30:10,711] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:30:10,711] jepsen nemesis - couchbase.util Rebalance check count: 133 INFO [2019-12-17 12:30:10,711] jepsen nemesis - couchbase.util Rebalance stuck count: 114 INFO [2019-12-17 12:30:10,724] jepsen worker 49 - jepsen.util 3649 :info :cas [16 [4 15]] :RequestCanceledException INFO [2019-12-17 12:30:10,731] jepsen worker 49 - jepsen.util 3739 :invoke :read [16 nil] INFO [2019-12-17 12:30:10,736] jepsen worker 53 - jepsen.util 3203 :ok :read [17 27] INFO [2019-12-17 12:30:10,736] jepsen worker 53 - jepsen.util 3203 :invoke :write [17 0] INFO [2019-12-17 12:30:10,738] jepsen worker 12 - jepsen.util 3252 :info :write [4 3] :RequestCanceledException INFO [2019-12-17 12:30:10,738] jepsen worker 12 - jepsen.util 3342 :invoke :cas [4 [29 43]] INFO [2019-12-17 12:30:10,745] jepsen worker 27 - jepsen.util 2817 :ok :read [9 31] INFO [2019-12-17 12:30:10,745] jepsen worker 27 - jepsen.util 2817 :invoke :cas [9 [29 31]] INFO [2019-12-17 12:30:10,754] jepsen worker 79 - jepsen.util 3049 :fail :cas [26 [12 24]] :ValueNotSwapFrom INFO [2019-12-17 12:30:10,754] jepsen worker 79 - jepsen.util 3049 :invoke :write [26 41] INFO [2019-12-17 12:30:10,802] jepsen worker 27 - jepsen.util 2817 :fail :cas [9 [29 31]] :ValueNotSwapFrom INFO [2019-12-17 12:30:10,802] jepsen worker 27 - jepsen.util 2817 :invoke :read [9 nil] INFO [2019-12-17 12:30:10,808] jepsen worker 5 - jepsen.util 2705 :info :write [1 24] :RequestCanceledException INFO [2019-12-17 12:30:10,812] jepsen worker 5 - jepsen.util 2795 :invoke :write [1 11] INFO [2019-12-17 12:30:10,823] jepsen worker 60 - jepsen.util 4380 :info :cas [20 [31 6]] :RequestCanceledException INFO [2019-12-17 12:30:10,827] jepsen worker 60 - jepsen.util 4470 :invoke :read [20 nil] INFO [2019-12-17 12:30:10,849] jepsen worker 31 - jepsen.util 2641 :ok :write [10 5] INFO [2019-12-17 12:30:10,849] jepsen worker 31 - jepsen.util 2641 :invoke :cas [10 [41 29]] INFO [2019-12-17 12:30:10,849] jepsen worker 45 - jepsen.util 3105 :ok :write [15 39] INFO [2019-12-17 12:30:10,850] jepsen worker 45 - jepsen.util 3105 :invoke :write [15 13] INFO [2019-12-17 12:30:10,853] jepsen worker 34 - jepsen.util 4174 :info :write [11 35] :RequestCanceledException INFO [2019-12-17 12:30:10,854] jepsen worker 11 - jepsen.util 3521 :ok :write [3 49] INFO [2019-12-17 12:30:10,854] jepsen worker 20 - jepsen.util 2720 :ok :write [6 34] INFO [2019-12-17 12:30:10,854] jepsen worker 11 - jepsen.util 3521 :invoke :write [3 48] INFO [2019-12-17 12:30:10,854] jepsen worker 20 - jepsen.util 2720 :invoke :cas [6 [19 18]] INFO [2019-12-17 12:30:10,856] jepsen worker 27 - jepsen.util 2817 :ok :read [9 31] INFO [2019-12-17 12:30:10,856] jepsen worker 27 - jepsen.util 2817 :invoke :read [9 nil] INFO [2019-12-17 12:30:10,860] jepsen worker 34 - jepsen.util 4264 :invoke :read [11 nil] INFO [2019-12-17 12:30:10,891] jepsen worker 87 - jepsen.util 2877 :info :write [29 47] :RequestCanceledException INFO [2019-12-17 12:30:10,891] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18234,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:30:10,891] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18234,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:30:10,892] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18237,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:30:10,892] jepsen worker 87 - jepsen.util 2967 :invoke :cas [29 [18 25]] INFO [2019-12-17 12:30:10,892] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18237,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:30:10,931] jepsen worker 38 - jepsen.util 3098 :ok :read [12 20] INFO [2019-12-17 12:30:10,931] jepsen worker 38 - jepsen.util 3098 :invoke :cas [12 [24 8]] INFO [2019-12-17 12:30:10,933] jepsen worker 32 - jepsen.util 3632 :fail :cas [10 [21 7]] :ValueNotSwapFrom INFO [2019-12-17 12:30:10,933] jepsen worker 32 - jepsen.util 3632 :invoke :write [10 15] INFO [2019-12-17 12:30:10,947] jepsen worker 65 - jepsen.util 2855 :ok :read [21 29] INFO [2019-12-17 12:30:10,947] jepsen worker 65 - jepsen.util 2855 :invoke :cas [21 [30 31]] INFO [2019-12-17 12:30:10,950] jepsen worker 80 - jepsen.util 2960 :ok :write [26 3] INFO [2019-12-17 12:30:10,951] jepsen worker 80 - jepsen.util 2960 :invoke :write [26 5] INFO [2019-12-17 12:30:10,979] jepsen worker 20 - jepsen.util 2720 :fail :cas [6 [19 18]] :ValueNotSwapFrom INFO [2019-12-17 12:30:10,979] jepsen worker 20 - jepsen.util 2720 :invoke :write [6 22] INFO [2019-12-17 12:30:10,990] jepsen worker 7 - jepsen.util 3877 :info :write [2 26] :RequestCanceledException INFO [2019-12-17 12:30:10,997] jepsen worker 7 - jepsen.util 3967 :invoke :write [2 1] INFO [2019-12-17 12:30:10,999] jepsen worker 31 - jepsen.util 2641 :fail :cas [10 [41 29]] :ValueNotSwapFrom INFO [2019-12-17 12:30:10,999] jepsen worker 31 - jepsen.util 2641 :invoke :cas [10 [13 36]] INFO [2019-12-17 12:30:11,002] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18248,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:30:11,002] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18248,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:30:11,002] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18252,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:30:11,002] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18252,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:30:11,003] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18253,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:30:11,003] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18253,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:30:11,005] jepsen worker 11 - jepsen.util 3521 :ok :write [3 48] INFO [2019-12-17 12:30:11,005] jepsen worker 11 - jepsen.util 3521 :invoke :read [3 nil] INFO [2019-12-17 12:30:11,015] jepsen worker 27 - jepsen.util 2817 :ok :read [9 31] INFO [2019-12-17 12:30:11,015] jepsen worker 27 - jepsen.util 2817 :invoke :read [9 nil] INFO [2019-12-17 12:30:11,023] jepsen worker 18 - jepsen.util 3258 :info :write [6 46] :RequestCanceledException INFO [2019-12-17 12:30:11,024] jepsen worker 88 - jepsen.util 3148 :info :write [29 21] :RequestCanceledException INFO [2019-12-17 12:30:11,024] jepsen worker 18 - jepsen.util 3348 :invoke :read [6 nil] INFO [2019-12-17 12:30:11,026] jepsen worker 65 - jepsen.util 2855 :fail :cas [21 [30 31]] :ValueNotSwapFrom INFO [2019-12-17 12:30:11,026] jepsen worker 65 - jepsen.util 2855 :invoke :write [21 37] INFO [2019-12-17 12:30:11,027] jepsen worker 88 - jepsen.util 3238 :invoke :cas [29 [35 16]] INFO [2019-12-17 12:30:11,050] jepsen worker 70 - jepsen.util 3670 :ok :write [23 41] INFO [2019-12-17 12:30:11,050] jepsen worker 70 - jepsen.util 3670 :invoke :read [23 nil] INFO [2019-12-17 12:30:11,051] jepsen worker 23 - jepsen.util 3983 :info :write [7 30] :RequestCanceledException INFO [2019-12-17 12:30:11,063] jepsen worker 23 - jepsen.util 4073 :invoke :cas [7 [11 30]] INFO [2019-12-17 12:30:11,099] jepsen worker 31 - jepsen.util 2641 :fail :cas [10 [13 36]] :ValueNotSwapFrom INFO [2019-12-17 12:30:11,099] jepsen worker 31 - jepsen.util 2641 :invoke :cas [10 [25 49]] INFO [2019-12-17 12:30:11,134] jepsen worker 18 - jepsen.util 3348 :ok :read [6 34] INFO [2019-12-17 12:30:11,134] jepsen worker 27 - jepsen.util 2817 :ok :read [9 31] INFO [2019-12-17 12:30:11,134] jepsen worker 32 - jepsen.util 3632 :ok :write [10 15] INFO [2019-12-17 12:30:11,134] jepsen worker 45 - jepsen.util 3105 :ok :write [15 13] INFO [2019-12-17 12:30:11,134] jepsen worker 18 - jepsen.util 3348 :invoke :cas [6 [44 12]] INFO [2019-12-17 12:30:11,134] jepsen worker 27 - jepsen.util 2817 :invoke :write [9 41] INFO [2019-12-17 12:30:11,134] jepsen worker 32 - jepsen.util 3632 :invoke :write [10 5] INFO [2019-12-17 12:30:11,134] jepsen worker 36 - jepsen.util 3096 :info :write [12 14] :RequestCanceledException INFO [2019-12-17 12:30:11,134] jepsen worker 45 - jepsen.util 3105 :invoke :cas [15 [24 32]] INFO [2019-12-17 12:30:11,135] jepsen worker 63 - jepsen.util 2763 :info :write [21 9] :RequestCanceledException INFO [2019-12-17 12:30:11,135] jepsen worker 69 - jepsen.util 3039 :info :cas [23 [30 14]] :RequestCanceledException INFO [2019-12-17 12:30:11,136] jepsen worker 63 - jepsen.util 2853 :invoke :cas [21 [0 18]] INFO [2019-12-17 12:30:11,140] jepsen worker 36 - jepsen.util 3186 :invoke :write [12 18] INFO [2019-12-17 12:30:11,140] jepsen worker 69 - jepsen.util 3129 :invoke :cas [23 [9 17]] INFO [2019-12-17 12:30:11,205] jepsen worker 12 - jepsen.util 3342 :fail :cas [4 [29 43]] :ValueNotSwapFrom INFO [2019-12-17 12:30:11,205] jepsen worker 12 - jepsen.util 3342 :invoke :read [4 nil] INFO [2019-12-17 12:30:11,224] jepsen worker 74 - jepsen.util 3314 :info :cas [24 [47 10]] :RequestCanceledException INFO [2019-12-17 12:30:11,224] jepsen worker 63 - jepsen.util 2853 :fail :cas [21 [0 18]] :ValueNotSwapFrom INFO [2019-12-17 12:30:11,224] jepsen worker 63 - jepsen.util 2853 :invoke :write [21 29] INFO [2019-12-17 12:30:11,232] jepsen worker 74 - jepsen.util 3404 :invoke :read [24 nil] INFO [2019-12-17 12:30:11,248] jepsen worker 13 - jepsen.util 3073 :ok :read [4 41] INFO [2019-12-17 12:30:11,248] jepsen worker 13 - jepsen.util 3073 :invoke :write [4 47] INFO [2019-12-17 12:30:11,249] jepsen worker 45 - jepsen.util 3105 :fail :cas [15 [24 32]] :ValueNotSwapFrom INFO [2019-12-17 12:30:11,249] jepsen worker 45 - jepsen.util 3105 :invoke :write [15 35] INFO [2019-12-17 12:30:11,255] jepsen worker 18 - jepsen.util 3348 :fail :cas [6 [44 12]] :ValueNotSwapFrom INFO [2019-12-17 12:30:11,255] jepsen worker 31 - jepsen.util 2641 :fail :cas [10 [25 49]] :ValueNotSwapFrom INFO [2019-12-17 12:30:11,255] jepsen worker 18 - jepsen.util 3348 :invoke :read [6 nil] INFO [2019-12-17 12:30:11,255] jepsen worker 31 - jepsen.util 2641 :invoke :cas [10 [43 39]] INFO [2019-12-17 12:30:11,271] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18279,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:30:11,271] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18279,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:30:11,271] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18283,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:30:11,271] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18283,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:30:11,279] jepsen worker 4 - jepsen.util 2434 :fail :cas [1 [22 30]] :ValueNotSwapFrom INFO [2019-12-17 12:30:11,279] jepsen worker 4 - jepsen.util 2434 :invoke :read [1 nil] INFO [2019-12-17 12:30:11,281] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18272,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:30:11,281] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18272,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:30:11,304] jepsen worker 74 - jepsen.util 3404 :ok :read [24 26] INFO [2019-12-17 12:30:11,304] jepsen worker 74 - jepsen.util 3404 :invoke :write [24 13] INFO [2019-12-17 12:30:11,314] jepsen worker 51 - jepsen.util 3021 :fail :cas [17 [49 13]] :ValueNotSwapFrom INFO [2019-12-17 12:30:11,314] jepsen worker 51 - jepsen.util 3021 :invoke :cas [17 [23 1]] INFO [2019-12-17 12:30:11,319] jepsen worker 71 - jepsen.util 3131 :fail :cas [23 [14 14]] :ValueNotSwapFrom INFO [2019-12-17 12:30:11,319] jepsen worker 71 - jepsen.util 3131 :invoke :cas [23 [35 37]] INFO [2019-12-17 12:30:11,324] jepsen worker 9 - jepsen.util 3609 :info :write [3 12] :RequestCanceledException INFO [2019-12-17 12:30:11,324] jepsen worker 16 - jepsen.util 4426 :info :write [5 39] :RequestCanceledException INFO [2019-12-17 12:30:11,330] jepsen worker 16 - jepsen.util 4516 :invoke :write [5 41] INFO [2019-12-17 12:30:11,334] jepsen worker 9 - jepsen.util 3699 :invoke :write [3 5] INFO [2019-12-17 12:30:11,336] jepsen worker 52 - jepsen.util 3292 :info :write [17 1] :RequestCanceledException INFO [2019-12-17 12:30:11,336] jepsen worker 35 - jepsen.util 4085 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:30:11,336] jepsen worker 19 - jepsen.util 3349 :info :write [6 32] :RequestCanceledException INFO [2019-12-17 12:30:11,336] jepsen worker 35 - jepsen.util 4085 :invoke :read [11 nil] INFO [2019-12-17 12:30:11,337] jepsen worker 52 - jepsen.util 3382 :invoke :cas [17 [14 21]] INFO [2019-12-17 12:30:11,337] jepsen worker 19 - jepsen.util 3439 :invoke :cas [6 [0 31]] INFO [2019-12-17 12:30:11,357] jepsen worker 18 - jepsen.util 3348 :ok :read [6 34] INFO [2019-12-17 12:30:11,357] jepsen worker 18 - jepsen.util 3348 :invoke :write [6 37] INFO [2019-12-17 12:30:11,370] jepsen worker 27 - jepsen.util 2817 :ok :write [9 41] INFO [2019-12-17 12:30:11,370] jepsen worker 27 - jepsen.util 2817 :invoke :cas [9 [14 40]] INFO [2019-12-17 12:30:11,376] jepsen worker 38 - jepsen.util 3098 :fail :cas [12 [24 8]] :ValueNotSwapFrom INFO [2019-12-17 12:30:11,376] jepsen worker 38 - jepsen.util 3098 :invoke :write [12 16] INFO [2019-12-17 12:30:11,384] jepsen worker 31 - jepsen.util 2641 :fail :cas [10 [43 39]] :ValueNotSwapFrom INFO [2019-12-17 12:30:11,385] jepsen worker 31 - jepsen.util 2641 :invoke :read [10 nil] INFO [2019-12-17 12:30:11,408] jepsen worker 29 - jepsen.util 2549 :info :write [9 0] :RequestCanceledException INFO [2019-12-17 12:30:11,409] jepsen worker 29 - jepsen.util 2639 :invoke :read [9 nil] INFO [2019-12-17 12:30:11,415] jepsen worker 55 - jepsen.util 3835 :fail :cas [18 [6 1]] :ValueNotSwapFrom INFO [2019-12-17 12:30:11,415] jepsen worker 55 - jepsen.util 3835 :invoke :read [18 nil] INFO [2019-12-17 12:30:11,423] jepsen worker 59 - jepsen.util 4109 :info :cas [19 [20 41]] :RequestCanceledException INFO [2019-12-17 12:30:11,429] jepsen worker 59 - jepsen.util 4199 :invoke :cas [19 [48 16]] INFO [2019-12-17 12:30:11,448] jepsen worker 63 - jepsen.util 2853 :ok :write [21 29] INFO [2019-12-17 12:30:11,448] jepsen worker 63 - jepsen.util 2853 :invoke :read [21 nil] INFO [2019-12-17 12:30:11,452] jepsen worker 31 - jepsen.util 2641 :ok :read [10 15] INFO [2019-12-17 12:30:11,453] jepsen worker 31 - jepsen.util 2641 :invoke :read [10 nil] INFO [2019-12-17 12:30:11,472] jepsen worker 29 - jepsen.util 2639 :ok :read [9 41] INFO [2019-12-17 12:30:11,472] jepsen worker 29 - jepsen.util 2639 :invoke :write [9 22] INFO [2019-12-17 12:30:11,489] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18306,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:30:11,489] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18306,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:30:11,489] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18309,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:30:11,489] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18309,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:30:11,489] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18310,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:30:11,489] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18310,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:30:11,490] jepsen worker 30 - jepsen.util 3720 :fail :read [10 nil] :RequestTimeoutException INFO [2019-12-17 12:30:11,490] jepsen worker 30 - jepsen.util 3720 :invoke :write [10 2] INFO [2019-12-17 12:30:11,523] jepsen worker 87 - jepsen.util 2967 :fail :cas [29 [18 25]] :ValueNotSwapFrom INFO [2019-12-17 12:30:11,523] jepsen worker 87 - jepsen.util 2967 :invoke :cas [29 [44 24]] INFO [2019-12-17 12:30:11,523] jepsen worker 15 - jepsen.util 3885 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:30:11,524] jepsen worker 15 - jepsen.util 3885 :invoke :write [5 10] INFO [2019-12-17 12:30:11,524] jepsen worker 28 - jepsen.util 2278 :info :cas [9 [20 24]] :RequestCanceledException INFO [2019-12-17 12:30:11,525] jepsen worker 28 - jepsen.util 2368 :invoke :write [9 15] INFO [2019-12-17 12:30:11,539] jepsen worker 52 - jepsen.util 3382 :fail :cas [17 [14 21]] :ValueNotSwapFrom INFO [2019-12-17 12:30:11,539] jepsen worker 52 - jepsen.util 3382 :invoke :write [17 14] INFO [2019-12-17 12:30:11,546] jepsen worker 63 - jepsen.util 2853 :ok :read [21 29] INFO [2019-12-17 12:30:11,546] jepsen worker 63 - jepsen.util 2853 :invoke :cas [21 [15 11]] INFO [2019-12-17 12:30:11,555] jepsen worker 84 - jepsen.util 4134 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:30:11,555] jepsen worker 84 - jepsen.util 4134 :invoke :read [28 nil] INFO [2019-12-17 12:30:11,568] jepsen worker 29 - jepsen.util 2639 :ok :write [9 22] INFO [2019-12-17 12:30:11,568] jepsen worker 29 - jepsen.util 2639 :invoke :write [9 29] INFO [2019-12-17 12:30:11,582] jepsen worker 31 - jepsen.util 2641 :ok :read [10 15] INFO [2019-12-17 12:30:11,582] jepsen worker 31 - jepsen.util 2641 :invoke :read [10 nil] INFO [2019-12-17 12:30:11,583] jepsen worker 88 - jepsen.util 3238 :fail :cas [29 [35 16]] :ValueNotSwapFrom INFO [2019-12-17 12:30:11,583] jepsen worker 88 - jepsen.util 3238 :invoke :read [29 nil] INFO [2019-12-17 12:30:11,608] jepsen worker 43 - jepsen.util 3733 :info :write [14 31] :RequestCanceledException INFO [2019-12-17 12:30:11,608] jepsen worker 85 - jepsen.util 4675 :info :write [28 28] :RequestCanceledException INFO [2019-12-17 12:30:11,613] jepsen worker 85 - jepsen.util 4765 :invoke :read [28 nil] INFO [2019-12-17 12:30:11,614] jepsen worker 43 - jepsen.util 3823 :invoke :cas [14 [40 3]] INFO [2019-12-17 12:30:11,624] jepsen worker 47 - jepsen.util 3737 :info :write [15 38] :RequestCanceledException INFO [2019-12-17 12:30:11,630] jepsen worker 47 - jepsen.util 3827 :invoke :read [15 nil] INFO [2019-12-17 12:30:11,637] jepsen worker 50 - jepsen.util 4730 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:30:11,637] jepsen worker 50 - jepsen.util 4730 :invoke :read [16 nil] INFO [2019-12-17 12:30:11,642] jepsen worker 63 - jepsen.util 2853 :fail :cas [21 [15 11]] :ValueNotSwapFrom INFO [2019-12-17 12:30:11,642] jepsen worker 63 - jepsen.util 2853 :invoke :cas [21 [19 24]] INFO [2019-12-17 12:30:11,646] jepsen worker 31 - jepsen.util 2641 :ok :read [10 15] INFO [2019-12-17 12:30:11,646] jepsen worker 31 - jepsen.util 2641 :invoke :cas [10 [10 43]] INFO [2019-12-17 12:30:11,653] jepsen worker 12 - jepsen.util 3342 :ok :read [4 41] INFO [2019-12-17 12:30:11,653] jepsen worker 12 - jepsen.util 3342 :invoke :write [4 18] INFO [2019-12-17 12:30:11,653] jepsen worker 69 - jepsen.util 3129 :fail :cas [23 [9 17]] :ValueNotSwapFrom INFO [2019-12-17 12:30:11,653] jepsen worker 69 - jepsen.util 3129 :invoke :cas [23 [22 27]] INFO [2019-12-17 12:30:11,658] jepsen worker 70 - jepsen.util 3670 :ok :read [23 41] INFO [2019-12-17 12:30:11,658] jepsen worker 70 - jepsen.util 3670 :invoke :cas [23 [46 44]] INFO [2019-12-17 12:30:11,677] jepsen worker 29 - jepsen.util 2639 :ok :write [9 29] INFO [2019-12-17 12:30:11,677] jepsen worker 29 - jepsen.util 2639 :invoke :write [9 29] INFO [2019-12-17 12:30:11,682] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18329,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:30:11,682] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18329,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:30:11,682] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18333,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:30:11,682] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18333,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:30:11,682] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18334,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:30:11,682] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18334,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:30:11,708] jepsen worker 45 - jepsen.util 3105 :ok :write [15 35] INFO [2019-12-17 12:30:11,708] jepsen worker 62 - jepsen.util 4022 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:30:11,708] jepsen worker 62 - jepsen.util 4022 :invoke :read [20 nil] INFO [2019-12-17 12:30:11,708] jepsen worker 45 - jepsen.util 3105 :invoke :cas [15 [22 46]] INFO [2019-12-17 12:30:11,710] jepsen worker 32 - jepsen.util 3632 :ok :write [10 5] INFO [2019-12-17 12:30:11,711] jepsen worker 32 - jepsen.util 3632 :invoke :cas [10 [48 42]] INFO [2019-12-17 12:30:11,711] jepsen worker 74 - jepsen.util 3404 :ok :write [24 13] INFO [2019-12-17 12:30:11,711] jepsen worker 74 - jepsen.util 3404 :invoke :read [24 nil] INFO [2019-12-17 12:30:11,713] jepsen worker 20 - jepsen.util 2720 :ok :write [6 22] INFO [2019-12-17 12:30:11,713] jepsen worker 20 - jepsen.util 2720 :invoke :cas [6 [17 35]] INFO [2019-12-17 12:30:11,715] jepsen worker 63 - jepsen.util 2853 :fail :cas [21 [19 24]] :ValueNotSwapFrom INFO [2019-12-17 12:30:11,715] jepsen worker 63 - jepsen.util 2853 :invoke :cas [21 [46 34]] INFO [2019-12-17 12:30:11,724] jepsen worker 40 - jepsen.util 4450 :info :write [13 13] :RequestCanceledException INFO [2019-12-17 12:30:11,724] jepsen worker 46 - jepsen.util 3376 :info :cas [15 [11 26]] :RequestCanceledException INFO [2019-12-17 12:30:11,724] jepsen worker 54 - jepsen.util 3204 :info :write [18 46] :RequestCanceledException INFO [2019-12-17 12:30:11,728] jepsen worker 46 - jepsen.util 3466 :invoke :cas [15 [16 0]] INFO [2019-12-17 12:30:11,728] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:30:11,728] jepsen nemesis - couchbase.util Rebalance check count: 134 INFO [2019-12-17 12:30:11,728] jepsen nemesis - couchbase.util Rebalance stuck count: 115 INFO [2019-12-17 12:30:11,729] jepsen worker 40 - jepsen.util 4540 :invoke :cas [13 [12 17]] INFO [2019-12-17 12:30:11,729] jepsen worker 54 - jepsen.util 3294 :invoke :write [18 26] INFO [2019-12-17 12:30:11,738] jepsen worker 64 - jepsen.util 2674 :info :write [21 14] :RequestCanceledException INFO [2019-12-17 12:30:11,739] jepsen worker 64 - jepsen.util 2764 :invoke :read [21 nil] INFO [2019-12-17 12:30:11,747] jepsen worker 31 - jepsen.util 2641 :fail :cas [10 [10 43]] :ValueNotSwapFrom INFO [2019-12-17 12:30:11,747] jepsen worker 31 - jepsen.util 2641 :invoke :read [10 nil] INFO [2019-12-17 12:30:11,752] jepsen worker 65 - jepsen.util 2855 :ok :write [21 37] INFO [2019-12-17 12:30:11,753] jepsen worker 65 - jepsen.util 2855 :invoke :write [21 19] INFO [2019-12-17 12:30:11,754] jepsen worker 1 - jepsen.util 3601 :info :write [0 43] :RequestCanceledException INFO [2019-12-17 12:30:11,757] jepsen worker 1 - jepsen.util 3691 :invoke :cas [0 [44 27]] INFO [2019-12-17 12:30:11,772] jepsen worker 28 - jepsen.util 2368 :ok :write [9 15] INFO [2019-12-17 12:30:11,772] jepsen worker 28 - jepsen.util 2368 :invoke :cas [9 [19 22]] INFO [2019-12-17 12:30:11,779] jepsen worker 63 - jepsen.util 2853 :fail :cas [21 [46 34]] :ValueNotSwapFrom INFO [2019-12-17 12:30:11,780] jepsen worker 63 - jepsen.util 2853 :invoke :read [21 nil] INFO [2019-12-17 12:30:11,790] jepsen worker 21 - jepsen.util 4071 :info :write [7 36] :RequestCanceledException INFO [2019-12-17 12:30:11,791] jepsen worker 21 - jepsen.util 4161 :invoke :write [7 45] INFO [2019-12-17 12:30:11,797] jepsen worker 19 - jepsen.util 3439 :fail :cas [6 [0 31]] :ValueNotSwapFrom INFO [2019-12-17 12:30:11,797] jepsen worker 19 - jepsen.util 3439 :invoke :read [6 nil] INFO [2019-12-17 12:30:11,809] jepsen worker 22 - jepsen.util 4252 :info :write [7 15] :RequestCanceledException INFO [2019-12-17 12:30:11,814] jepsen worker 22 - jepsen.util 4342 :invoke :cas [7 [12 34]] INFO [2019-12-17 12:30:11,818] jepsen worker 29 - jepsen.util 2639 :ok :write [9 29] INFO [2019-12-17 12:30:11,818] jepsen worker 29 - jepsen.util 2639 :invoke :read [9 nil] INFO [2019-12-17 12:30:11,820] jepsen worker 75 - jepsen.util 4305 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:30:11,820] jepsen worker 75 - jepsen.util 4305 :invoke :write [25 43] INFO [2019-12-17 12:30:11,830] jepsen worker 28 - jepsen.util 2368 :fail :cas [9 [19 22]] :ValueNotSwapFrom INFO [2019-12-17 12:30:11,830] jepsen worker 28 - jepsen.util 2368 :invoke :read [9 nil] INFO [2019-12-17 12:30:11,859] jepsen worker 63 - jepsen.util 2853 :ok :read [21 37] INFO [2019-12-17 12:30:11,859] jepsen worker 63 - jepsen.util 2853 :invoke :read [21 nil] INFO [2019-12-17 12:30:11,877] jepsen worker 31 - jepsen.util 2641 :ok :read [10 5] INFO [2019-12-17 12:30:11,877] jepsen worker 31 - jepsen.util 2641 :invoke :cas [10 [2 46]] INFO [2019-12-17 12:30:11,901] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18363,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:30:11,901] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18363,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:30:11,901] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18366,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:30:11,901] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18366,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:30:11,901] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18367,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:30:11,901] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18367,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:30:11,904] jepsen worker 29 - jepsen.util 2639 :ok :read [9 29] INFO [2019-12-17 12:30:11,905] jepsen worker 29 - jepsen.util 2639 :invoke :write [9 36] INFO [2019-12-17 12:30:11,905] jepsen worker 42 - jepsen.util 4902 :info :write [14 32] :RequestCanceledException INFO [2019-12-17 12:30:11,911] jepsen worker 42 - jepsen.util 4992 :invoke :write [14 26] INFO [2019-12-17 12:30:11,936] jepsen worker 28 - jepsen.util 2368 :ok :read [9 29] INFO [2019-12-17 12:30:11,936] jepsen worker 28 - jepsen.util 2368 :invoke :read [9 nil] INFO [2019-12-17 12:30:11,942] jepsen worker 55 - jepsen.util 3835 :ok :read [18 43] INFO [2019-12-17 12:30:11,942] jepsen worker 55 - jepsen.util 3835 :invoke :read [18 nil] INFO [2019-12-17 12:30:11,955] jepsen worker 41 - jepsen.util 4271 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:30:11,955] jepsen worker 41 - jepsen.util 4271 :invoke :cas [13 [19 16]] INFO [2019-12-17 12:30:11,969] jepsen worker 63 - jepsen.util 2853 :ok :read [21 37] INFO [2019-12-17 12:30:11,969] jepsen worker 63 - jepsen.util 2853 :invoke :read [21 nil] INFO [2019-12-17 12:30:11,978] jepsen worker 31 - jepsen.util 2641 :fail :cas [10 [2 46]] :ValueNotSwapFrom INFO [2019-12-17 12:30:11,978] jepsen worker 31 - jepsen.util 2641 :invoke :write [10 38] INFO [2019-12-17 12:30:11,992] jepsen worker 8 - jepsen.util 3518 :info :write [2 39] :RequestCanceledException INFO [2019-12-17 12:30:11,998] jepsen worker 8 - jepsen.util 3608 :invoke :write [2 30] INFO [2019-12-17 12:30:12,000] jepsen worker 4 - jepsen.util 2434 :ok :read [1 26] INFO [2019-12-17 12:30:12,000] jepsen worker 4 - jepsen.util 2434 :invoke :cas [1 [37 7]] INFO [2019-12-17 12:30:12,008] jepsen worker 83 - jepsen.util 4313 :info :cas [27 [33 34]] :RequestCanceledException INFO [2019-12-17 12:30:12,010] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18378,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:30:12,011] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18378,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:30:12,016] jepsen worker 83 - jepsen.util 4403 :invoke :cas [27 [16 47]] INFO [2019-12-17 12:30:12,024] jepsen worker 33 - jepsen.util 4443 :info :cas [11 [32 46]] :RequestCanceledException INFO [2019-12-17 12:30:12,024] jepsen worker 48 - jepsen.util 3648 :info :write [16 25] :RequestCanceledException INFO [2019-12-17 12:30:12,024] jepsen worker 57 - jepsen.util 3927 :info :write [19 43] :RequestCanceledException INFO [2019-12-17 12:30:12,024] jepsen worker 26 - jepsen.util 4526 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:30:12,024] jepsen worker 26 - jepsen.util 4526 :invoke :cas [8 [21 20]] INFO [2019-12-17 12:30:12,025] jepsen worker 28 - jepsen.util 2368 :ok :read [9 29] INFO [2019-12-17 12:30:12,025] jepsen worker 28 - jepsen.util 2368 :invoke :write [9 44] INFO [2019-12-17 12:30:12,026] jepsen worker 57 - jepsen.util 4017 :invoke :read [19 nil] INFO [2019-12-17 12:30:12,026] jepsen worker 33 - jepsen.util 4533 :invoke :cas [11 [22 14]] INFO [2019-12-17 12:30:12,028] jepsen worker 48 - jepsen.util 3738 :invoke :write [16 10] INFO [2019-12-17 12:30:12,038] jepsen worker 0 - jepsen.util 4680 :info :write [0 31] :RequestCanceledException INFO [2019-12-17 12:30:12,038] jepsen worker 72 - jepsen.util 3132 :fail :read [24 nil] :RequestTimeoutException INFO [2019-12-17 12:30:12,039] jepsen worker 72 - jepsen.util 3132 :invoke :read [24 nil] INFO [2019-12-17 12:30:12,042] jepsen worker 0 - jepsen.util 4770 :invoke :write [0 34] INFO [2019-12-17 12:30:12,047] jepsen worker 29 - jepsen.util 2639 :ok :write [9 36] INFO [2019-12-17 12:30:12,048] jepsen worker 29 - jepsen.util 2639 :invoke :cas [9 [42 23]] INFO [2019-12-17 12:30:12,089] jepsen worker 63 - jepsen.util 2853 :ok :read [21 37] INFO [2019-12-17 12:30:12,089] jepsen worker 63 - jepsen.util 2853 :invoke :read [21 nil] INFO [2019-12-17 12:30:12,108] jepsen worker 56 - jepsen.util 3026 :info :write [18 46] :RequestCanceledException INFO [2019-12-17 12:30:12,115] jepsen worker 56 - jepsen.util 3116 :invoke :write [18 12] INFO [2019-12-17 12:30:12,117] jepsen worker 69 - jepsen.util 3129 :fail :cas [23 [22 27]] :ValueNotSwapFrom INFO [2019-12-17 12:30:12,118] jepsen worker 69 - jepsen.util 3129 :invoke :write [23 1] INFO [2019-12-17 12:30:12,118] jepsen worker 72 - jepsen.util 3132 :ok :read [24 13] INFO [2019-12-17 12:30:12,118] jepsen worker 72 - jepsen.util 3132 :invoke :write [24 32] INFO [2019-12-17 12:30:12,120] jepsen worker 51 - jepsen.util 3021 :fail :cas [17 [23 1]] :ValueNotSwapFrom INFO [2019-12-17 12:30:12,120] jepsen worker 51 - jepsen.util 3021 :invoke :cas [17 [41 27]] INFO [2019-12-17 12:30:12,123] jepsen worker 17 - jepsen.util 4067 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:30:12,123] jepsen worker 17 - jepsen.util 4067 :invoke :write [5 40] INFO [2019-12-17 12:30:12,152] jepsen worker 73 - jepsen.util 2953 :info :write [24 47] :RequestCanceledException INFO [2019-12-17 12:30:12,158] jepsen worker 73 - jepsen.util 3043 :invoke :read [24 nil] INFO [2019-12-17 12:30:12,163] jepsen worker 71 - jepsen.util 3131 :fail :cas [23 [35 37]] :ValueNotSwapFrom INFO [2019-12-17 12:30:12,163] jepsen worker 71 - jepsen.util 3131 :invoke :write [23 36] INFO [2019-12-17 12:30:12,165] jepsen worker 28 - jepsen.util 2368 :ok :write [9 44] INFO [2019-12-17 12:30:12,165] jepsen worker 28 - jepsen.util 2368 :invoke :read [9 nil] INFO [2019-12-17 12:30:12,171] jepsen worker 55 - jepsen.util 3835 :ok :read [18 43] INFO [2019-12-17 12:30:12,171] jepsen worker 55 - jepsen.util 3835 :invoke :read [18 nil] INFO [2019-12-17 12:30:12,184] jepsen worker 63 - jepsen.util 2853 :ok :read [21 37] INFO [2019-12-17 12:30:12,184] jepsen worker 63 - jepsen.util 2853 :invoke :read [21 nil] INFO [2019-12-17 12:30:12,208] jepsen worker 10 - jepsen.util 2890 :info :write [3 6] :RequestCanceledException INFO [2019-12-17 12:30:12,210] jepsen worker 10 - jepsen.util 2980 :invoke :write [3 12] INFO [2019-12-17 12:30:12,228] jepsen worker 88 - jepsen.util 3238 :ok :read [29 17] INFO [2019-12-17 12:30:12,228] jepsen worker 88 - jepsen.util 3238 :invoke :read [29 nil] INFO [2019-12-17 12:30:12,229] jepsen worker 9 - jepsen.util 3699 :ok :write [3 5] INFO [2019-12-17 12:30:12,229] jepsen worker 9 - jepsen.util 3699 :invoke :cas [3 [48 47]] INFO [2019-12-17 12:30:12,230] jepsen worker 72 - jepsen.util 3132 :ok :write [24 32] INFO [2019-12-17 12:30:12,230] jepsen worker 72 - jepsen.util 3132 :invoke :cas [24 [3 46]] INFO [2019-12-17 12:30:12,231] jepsen worker 18 - jepsen.util 3348 :ok :write [6 37] INFO [2019-12-17 12:30:12,231] jepsen worker 18 - jepsen.util 3348 :invoke :cas [6 [1 27]] INFO [2019-12-17 12:30:12,242] jepsen worker 30 - jepsen.util 3720 :ok :write [10 2] INFO [2019-12-17 12:30:12,242] jepsen worker 30 - jepsen.util 3720 :invoke :read [10 nil] INFO [2019-12-17 12:30:12,257] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18411,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:30:12,257] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18411,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:30:12,257] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18415,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:30:12,258] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18415,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:30:12,258] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18416,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:30:12,258] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18416,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:30:12,259] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18405,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:30:12,259] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18405,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:30:12,259] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18407,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:30:12,259] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18407,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:30:12,261] jepsen worker 63 - jepsen.util 2853 :ok :read [21 37] INFO [2019-12-17 12:30:12,261] jepsen worker 63 - jepsen.util 2853 :invoke :cas [21 [37 39]] INFO [2019-12-17 12:30:12,290] jepsen worker 37 - jepsen.util 2917 :info :write [12 40] :RequestCanceledException INFO [2019-12-17 12:30:12,291] jepsen worker 37 - jepsen.util 3007 :invoke :read [12 nil] INFO [2019-12-17 12:30:12,306] jepsen worker 14 - jepsen.util 2984 :info :write [4 13] :RequestCanceledException INFO [2019-12-17 12:30:12,309] jepsen worker 14 - jepsen.util 3074 :invoke :write [4 38] INFO [2019-12-17 12:30:12,315] jepsen worker 73 - jepsen.util 3043 :ok :read [24 13] INFO [2019-12-17 12:30:12,315] jepsen worker 73 - jepsen.util 3043 :invoke :cas [24 [7 32]] INFO [2019-12-17 12:30:12,337] jepsen worker 72 - jepsen.util 3132 :fail :cas [24 [3 46]] :ValueNotSwapFrom INFO [2019-12-17 12:30:12,337] jepsen worker 72 - jepsen.util 3132 :invoke :write [24 30] INFO [2019-12-17 12:30:12,401] jepsen worker 9 - jepsen.util 3699 :fail :cas [3 [48 47]] :ValueNotSwapFrom INFO [2019-12-17 12:30:12,401] jepsen worker 9 - jepsen.util 3699 :invoke :write [3 27] INFO [2019-12-17 12:30:12,405] jepsen worker 74 - jepsen.util 3404 :ok :read [24 32] INFO [2019-12-17 12:30:12,405] jepsen worker 32 - jepsen.util 3632 :fail :cas [10 [48 42]] :ValueNotSwapFrom INFO [2019-12-17 12:30:12,405] jepsen worker 74 - jepsen.util 3404 :invoke :write [24 9] INFO [2019-12-17 12:30:12,405] jepsen worker 32 - jepsen.util 3632 :invoke :cas [10 [27 48]] INFO [2019-12-17 12:30:12,408] jepsen worker 25 - jepsen.util 4705 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:30:12,408] jepsen worker 25 - jepsen.util 4705 :invoke :read [8 nil] INFO [2019-12-17 12:30:12,424] jepsen worker 81 - jepsen.util 4221 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:30:12,424] jepsen worker 81 - jepsen.util 4221 :invoke :read [27 nil] INFO [2019-12-17 12:30:12,454] jepsen worker 76 - jepsen.util 4306 :info :cas [25 [31 43]] :RequestCanceledException INFO [2019-12-17 12:30:12,466] jepsen worker 76 - jepsen.util 4396 :invoke :write [25 25] INFO [2019-12-17 12:30:12,475] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18431,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:30:12,475] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18431,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:30:12,475] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18436,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:30:12,475] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18436,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:30:12,481] jepsen worker 73 - jepsen.util 3043 :fail :cas [24 [7 32]] :ValueNotSwapFrom INFO [2019-12-17 12:30:12,482] jepsen worker 73 - jepsen.util 3043 :invoke :read [24 nil] INFO [2019-12-17 12:30:12,484] jepsen worker 87 - jepsen.util 2967 :fail :cas [29 [44 24]] :ValueNotSwapFrom INFO [2019-12-17 12:30:12,485] jepsen worker 87 - jepsen.util 2967 :invoke :read [29 nil] INFO [2019-12-17 12:30:12,486] jepsen worker 61 - jepsen.util 4741 :info :cas [20 [15 24]] :RequestCanceledException INFO [2019-12-17 12:30:12,487] jepsen worker 78 - jepsen.util 3408 :info :write [26 27] :RequestCanceledException INFO [2019-12-17 12:30:12,487] jepsen worker 19 - jepsen.util 3439 :ok :read [6 37] INFO [2019-12-17 12:30:12,488] jepsen worker 19 - jepsen.util 3439 :invoke :write [6 44] INFO [2019-12-17 12:30:12,492] jepsen worker 61 - jepsen.util 4831 :invoke :write [20 20] INFO [2019-12-17 12:30:12,492] jepsen worker 78 - jepsen.util 3498 :invoke :read [26 nil] INFO [2019-12-17 12:30:12,500] jepsen worker 18 - jepsen.util 3348 :fail :cas [6 [1 27]] :ValueNotSwapFrom INFO [2019-12-17 12:30:12,501] jepsen worker 18 - jepsen.util 3348 :invoke :read [6 nil] INFO [2019-12-17 12:30:12,507] jepsen worker 30 - jepsen.util 3720 :ok :read [10 2] INFO [2019-12-17 12:30:12,508] jepsen worker 30 - jepsen.util 3720 :invoke :read [10 nil] INFO [2019-12-17 12:30:12,508] jepsen worker 70 - jepsen.util 3670 :fail :cas [23 [46 44]] :ValueNotSwapFrom INFO [2019-12-17 12:30:12,508] jepsen worker 70 - jepsen.util 3670 :invoke :read [23 nil] INFO [2019-12-17 12:30:12,508] jepsen worker 86 - jepsen.util 4406 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:30:12,508] jepsen worker 86 - jepsen.util 4406 :invoke :cas [28 [37 33]] INFO [2019-12-17 12:30:12,523] jepsen worker 58 - jepsen.util 4108 :info :cas [19 [1 22]] :RequestCanceledException INFO [2019-12-17 12:30:12,524] jepsen worker 32 - jepsen.util 3632 :fail :cas [10 [27 48]] :ValueNotSwapFrom INFO [2019-12-17 12:30:12,524] jepsen worker 32 - jepsen.util 3632 :invoke :cas [10 [12 8]] INFO [2019-12-17 12:30:12,530] jepsen worker 58 - jepsen.util 4198 :invoke :read [19 nil] INFO [2019-12-17 12:30:12,566] jepsen worker 87 - jepsen.util 2967 :ok :read [29 17] INFO [2019-12-17 12:30:12,566] jepsen worker 87 - jepsen.util 2967 :invoke :cas [29 [27 10]] INFO [2019-12-17 12:30:12,608] jepsen worker 82 - jepsen.util 5032 :info :write [27 25] :RequestCanceledException INFO [2019-12-17 12:30:12,611] jepsen worker 10 - jepsen.util 2980 :ok :write [3 12] INFO [2019-12-17 12:30:12,611] jepsen worker 10 - jepsen.util 2980 :invoke :write [3 6] INFO [2019-12-17 12:30:12,613] jepsen worker 82 - jepsen.util 5122 :invoke :write [27 2] INFO [2019-12-17 12:30:12,620] jepsen worker 37 - jepsen.util 3007 :ok :read [12 20] INFO [2019-12-17 12:30:12,621] jepsen worker 37 - jepsen.util 3007 :invoke :read [12 nil] INFO [2019-12-17 12:30:12,624] jepsen worker 66 - jepsen.util 4116 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:30:12,624] jepsen worker 66 - jepsen.util 4116 :invoke :write [22 4] INFO [2019-12-17 12:30:12,624] jepsen worker 2 - jepsen.util 3872 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:30:12,624] jepsen worker 2 - jepsen.util 3872 :invoke :cas [0 [20 14]] INFO [2019-12-17 12:30:12,630] jepsen worker 73 - jepsen.util 3043 :ok :read [24 32] INFO [2019-12-17 12:30:12,630] jepsen worker 73 - jepsen.util 3043 :invoke :write [24 12] INFO [2019-12-17 12:30:12,637] jepsen worker 68 - jepsen.util 4028 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:30:12,637] jepsen worker 68 - jepsen.util 4028 :invoke :read [22 nil] INFO [2019-12-17 12:30:12,638] jepsen worker 30 - jepsen.util 3720 :ok :read [10 2] INFO [2019-12-17 12:30:12,638] jepsen worker 30 - jepsen.util 3720 :invoke :read [10 nil] INFO [2019-12-17 12:30:12,654] jepsen worker 3 - jepsen.util 3153 :info :write [1 48] :RequestCanceledException INFO [2019-12-17 12:30:12,654] jepsen worker 77 - jepsen.util 3857 :info :write [25 7] :RequestCanceledException INFO [2019-12-17 12:30:12,654] jepsen worker 39 - jepsen.util 3999 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:30:12,654] jepsen worker 39 - jepsen.util 3999 :invoke :write [13 44] INFO [2019-12-17 12:30:12,657] jepsen worker 77 - jepsen.util 3947 :invoke :write [25 16] INFO [2019-12-17 12:30:12,657] jepsen worker 3 - jepsen.util 3243 :invoke :cas [1 [20 3]] INFO [2019-12-17 12:30:12,663] jepsen worker 13 - jepsen.util 3073 :ok :write [4 47] INFO [2019-12-17 12:30:12,663] jepsen worker 13 - jepsen.util 3073 :invoke :write [4 13] INFO [2019-12-17 12:30:12,670] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18459,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:30:12,670] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18459,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:30:12,670] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18462,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:30:12,670] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18462,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:30:12,670] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18463,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:30:12,670] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18463,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:30:12,672] jepsen worker 19 - jepsen.util 3439 :ok :write [6 44] INFO [2019-12-17 12:30:12,672] jepsen worker 19 - jepsen.util 3439 :invoke :read [6 nil] INFO [2019-12-17 12:30:12,680] jepsen worker 72 - jepsen.util 3132 :ok :write [24 30] INFO [2019-12-17 12:30:12,680] jepsen worker 72 - jepsen.util 3132 :invoke :cas [24 [25 23]] INFO [2019-12-17 12:30:12,690] jepsen worker 24 - jepsen.util 4074 :info :write [8 17] :RequestCanceledException INFO [2019-12-17 12:30:12,697] jepsen worker 24 - jepsen.util 4164 :invoke :read [8 nil] INFO [2019-12-17 12:30:12,722] jepsen worker 88 - jepsen.util 3238 :ok :read [29 17] INFO [2019-12-17 12:30:12,722] jepsen worker 88 - jepsen.util 3238 :invoke :read [29 nil] INFO [2019-12-17 12:30:12,724] jepsen worker 67 - jepsen.util 4567 :info :write [22 16] :RequestCanceledException INFO [2019-12-17 12:30:12,724] jepsen worker 44 - jepsen.util 4454 :info :cas [14 [39 6]] :RequestCanceledException INFO [2019-12-17 12:30:12,725] jepsen worker 89 - jepsen.util 2969 :info :write [29 8] :RequestCanceledException INFO [2019-12-17 12:30:12,726] jepsen worker 67 - jepsen.util 4657 :invoke :write [22 14] INFO [2019-12-17 12:30:12,727] jepsen worker 44 - jepsen.util 4544 :invoke :cas [14 [19 49]] INFO [2019-12-17 12:30:12,730] jepsen worker 89 - jepsen.util 3059 :invoke :cas [29 [38 21]] INFO [2019-12-17 12:30:12,745] jepsen worker 32 - jepsen.util 3632 :fail :cas [10 [12 8]] :ValueNotSwapFrom INFO [2019-12-17 12:30:12,745] jepsen worker 32 - jepsen.util 3632 :invoke :write [10 17] INFO [2019-12-17 12:30:12,751] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:30:12,751] jepsen nemesis - couchbase.util Rebalance check count: 135 INFO [2019-12-17 12:30:12,751] jepsen nemesis - couchbase.util Rebalance stuck count: 116 INFO [2019-12-17 12:30:12,755] jepsen worker 72 - jepsen.util 3132 :fail :cas [24 [25 23]] :ValueNotSwapFrom INFO [2019-12-17 12:30:12,755] jepsen worker 72 - jepsen.util 3132 :invoke :cas [24 [35 34]] INFO [2019-12-17 12:30:12,791] jepsen worker 6 - jepsen.util 4506 :info :cas [2 [40 28]] :RequestCanceledException INFO [2019-12-17 12:30:12,793] jepsen worker 30 - jepsen.util 3720 :ok :read [10 2] INFO [2019-12-17 12:30:12,793] jepsen worker 30 - jepsen.util 3720 :invoke :read [10 nil] INFO [2019-12-17 12:30:12,796] jepsen worker 6 - jepsen.util 4596 :invoke :read [2 nil] INFO [2019-12-17 12:30:12,803] jepsen worker 19 - jepsen.util 3439 :ok :read [6 44] INFO [2019-12-17 12:30:12,803] jepsen worker 19 - jepsen.util 3439 :invoke :read [6 nil] INFO [2019-12-17 12:30:12,808] jepsen worker 49 - jepsen.util 3739 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:30:12,809] jepsen worker 49 - jepsen.util 3739 :invoke :cas [16 [19 39]] INFO [2019-12-17 12:30:12,814] jepsen worker 10 - jepsen.util 2980 :ok :write [3 6] INFO [2019-12-17 12:30:12,815] jepsen worker 10 - jepsen.util 2980 :invoke :read [3 nil] INFO [2019-12-17 12:30:12,821] jepsen worker 72 - jepsen.util 3132 :fail :cas [24 [35 34]] :ValueNotSwapFrom INFO [2019-12-17 12:30:12,822] jepsen worker 72 - jepsen.util 3132 :invoke :write [24 39] INFO [2019-12-17 12:30:12,828] jepsen worker 70 - jepsen.util 3670 :ok :read [23 41] INFO [2019-12-17 12:30:12,828] jepsen worker 70 - jepsen.util 3670 :invoke :write [23 25] INFO [2019-12-17 12:30:12,838] jepsen worker 79 - jepsen.util 3049 :info :write [26 41] :RequestCanceledException INFO [2019-12-17 12:30:12,839] jepsen worker 73 - jepsen.util 3043 :ok :write [24 12] INFO [2019-12-17 12:30:12,839] jepsen worker 73 - jepsen.util 3043 :invoke :write [24 40] INFO [2019-12-17 12:30:12,840] jepsen worker 79 - jepsen.util 3139 :invoke :read [26 nil] INFO [2019-12-17 12:30:12,847] jepsen worker 32 - jepsen.util 3632 :ok :write [10 17] INFO [2019-12-17 12:30:12,847] jepsen worker 32 - jepsen.util 3632 :invoke :write [10 46] INFO [2019-12-17 12:30:12,854] jepsen worker 53 - jepsen.util 3203 :info :write [17 0] :RequestCanceledException INFO [2019-12-17 12:30:12,854] jepsen worker 60 - jepsen.util 4470 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:30:12,855] jepsen worker 60 - jepsen.util 4470 :invoke :cas [20 [28 38]] INFO [2019-12-17 12:30:12,859] jepsen worker 53 - jepsen.util 3293 :invoke :read [17 nil] INFO [2019-12-17 12:30:12,870] jepsen worker 10 - jepsen.util 2980 :ok :read [3 6] INFO [2019-12-17 12:30:12,870] jepsen worker 10 - jepsen.util 2980 :invoke :write [3 0] INFO [2019-12-17 12:30:12,889] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18490,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:30:12,889] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18490,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:30:12,890] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18493,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:30:12,890] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18493,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:30:12,924] jepsen worker 34 - jepsen.util 4264 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:30:12,924] jepsen worker 34 - jepsen.util 4264 :invoke :write [11 34] INFO [2019-12-17 12:30:12,934] jepsen worker 19 - jepsen.util 3439 :ok :read [6 44] INFO [2019-12-17 12:30:12,935] jepsen worker 19 - jepsen.util 3439 :invoke :write [6 21] INFO [2019-12-17 12:30:12,938] jepsen worker 5 - jepsen.util 2795 :info :write [1 11] :RequestCanceledException INFO [2019-12-17 12:30:12,940] jepsen worker 5 - jepsen.util 2885 :invoke :cas [1 [47 35]] INFO [2019-12-17 12:30:12,956] jepsen worker 55 - jepsen.util 3835 :ok :read [18 43] INFO [2019-12-17 12:30:12,956] jepsen worker 55 - jepsen.util 3835 :invoke :write [18 31] INFO [2019-12-17 12:30:12,980] jepsen worker 79 - jepsen.util 3139 :ok :read [26 3] INFO [2019-12-17 12:30:12,980] jepsen worker 79 - jepsen.util 3139 :invoke :cas [26 [2 19]] INFO [2019-12-17 12:30:12,997] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18504,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:30:12,997] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18504,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:30:12,997] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18507,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:30:12,997] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18507,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:30:12,997] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18508,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:30:12,997] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18508,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:30:13,025] jepsen worker 3 - jepsen.util 3243 :fail :cas [1 [20 3]] :ValueNotSwapFrom INFO [2019-12-17 12:30:13,025] jepsen worker 3 - jepsen.util 3243 :invoke :cas [1 [11 47]] INFO [2019-12-17 12:30:13,054] jepsen worker 80 - jepsen.util 2960 :info :write [26 5] :RequestCanceledException INFO [2019-12-17 12:30:13,056] jepsen worker 80 - jepsen.util 3050 :invoke :read [26 nil] INFO [2019-12-17 12:30:13,082] jepsen worker 51 - jepsen.util 3021 :fail :cas [17 [41 27]] :ValueNotSwapFrom INFO [2019-12-17 12:30:13,082] jepsen worker 51 - jepsen.util 3021 :invoke :cas [17 [4 49]] INFO [2019-12-17 12:30:13,087] jepsen worker 4 - jepsen.util 2434 :fail :cas [1 [37 7]] :ValueNotSwapFrom INFO [2019-12-17 12:30:13,087] jepsen worker 4 - jepsen.util 2434 :invoke :write [1 26] INFO [2019-12-17 12:30:13,090] jepsen worker 11 - jepsen.util 3521 :fail :read [3 nil] :RequestTimeoutException INFO [2019-12-17 12:30:13,090] jepsen worker 11 - jepsen.util 3521 :invoke :cas [3 [6 28]] INFO [2019-12-17 12:30:13,108] jepsen worker 7 - jepsen.util 3967 :info :write [2 1] :RequestCanceledException INFO [2019-12-17 12:30:13,110] jepsen worker 7 - jepsen.util 4057 :invoke :write [2 44] INFO [2019-12-17 12:30:13,113] jepsen worker 79 - jepsen.util 3139 :fail :cas [26 [2 19]] :ValueNotSwapFrom INFO [2019-12-17 12:30:13,113] jepsen worker 79 - jepsen.util 3139 :invoke :write [26 33] INFO [2019-12-17 12:30:13,155] jepsen worker 23 - jepsen.util 4073 :info :cas [7 [11 30]] :RequestCanceledException INFO [2019-12-17 12:30:13,162] jepsen worker 23 - jepsen.util 4163 :invoke :read [7 nil] INFO [2019-12-17 12:30:13,186] jepsen worker 3 - jepsen.util 3243 :fail :cas [1 [11 47]] :ValueNotSwapFrom INFO [2019-12-17 12:30:13,186] jepsen worker 3 - jepsen.util 3243 :invoke :write [1 46] INFO [2019-12-17 12:30:13,190] jepsen worker 36 - jepsen.util 3186 :info :write [12 18] :RequestCanceledException INFO [2019-12-17 12:30:13,192] jepsen worker 36 - jepsen.util 3276 :invoke :write [12 25] INFO [2019-12-17 12:30:13,219] jepsen worker 80 - jepsen.util 3050 :ok :read [26 3] INFO [2019-12-17 12:30:13,220] jepsen worker 80 - jepsen.util 3050 :invoke :write [26 10] INFO [2019-12-17 12:30:13,245] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18518,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:30:13,246] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18526,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:30:13,246] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18518,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:30:13,246] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18526,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:30:13,246] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18519,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:30:13,246] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18529,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:30:13,246] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18519,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:30:13,246] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18529,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:30:13,249] jepsen worker 87 - jepsen.util 2967 :fail :cas [29 [27 10]] :ValueNotSwapFrom INFO [2019-12-17 12:30:13,249] jepsen worker 87 - jepsen.util 2967 :invoke :write [29 1] INFO [2019-12-17 12:30:13,258] jepsen worker 78 - jepsen.util 3498 :ok :read [26 3] INFO [2019-12-17 12:30:13,258] jepsen worker 78 - jepsen.util 3498 :invoke :read [26 nil] INFO [2019-12-17 12:30:13,316] jepsen worker 5 - jepsen.util 2885 :fail :cas [1 [47 35]] :ValueNotSwapFrom INFO [2019-12-17 12:30:13,317] jepsen worker 5 - jepsen.util 2885 :invoke :read [1 nil] INFO [2019-12-17 12:30:13,366] jepsen worker 88 - jepsen.util 3238 :ok :read [29 17] INFO [2019-12-17 12:30:13,366] jepsen worker 88 - jepsen.util 3238 :invoke :read [29 nil] INFO [2019-12-17 12:30:13,394] jepsen worker 78 - jepsen.util 3498 :ok :read [26 3] INFO [2019-12-17 12:30:13,394] jepsen worker 78 - jepsen.util 3498 :invoke :write [26 49] INFO [2019-12-17 12:30:13,401] jepsen worker 53 - jepsen.util 3293 :ok :read [17 27] INFO [2019-12-17 12:30:13,401] jepsen worker 53 - jepsen.util 3293 :invoke :read [17 nil] INFO [2019-12-17 12:30:13,423] jepsen worker 38 - jepsen.util 3098 :info :write [12 16] :RequestCanceledException INFO [2019-12-17 12:30:13,424] jepsen worker 27 - jepsen.util 2817 :info :cas [9 [14 40]] :RequestCanceledException INFO [2019-12-17 12:30:13,427] jepsen worker 38 - jepsen.util 3188 :invoke :read [12 nil] INFO [2019-12-17 12:30:13,428] jepsen worker 27 - jepsen.util 2907 :invoke :cas [9 [37 43]] INFO [2019-12-17 12:30:13,438] jepsen worker 35 - jepsen.util 4085 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:30:13,438] jepsen worker 37 - jepsen.util 3007 :ok :read [12 20] INFO [2019-12-17 12:30:13,438] jepsen worker 16 - jepsen.util 4516 :info :write [5 41] :RequestCanceledException INFO [2019-12-17 12:30:13,438] jepsen worker 35 - jepsen.util 4085 :invoke :write [11 16] INFO [2019-12-17 12:30:13,438] jepsen worker 37 - jepsen.util 3007 :invoke :read [12 nil] INFO [2019-12-17 12:30:13,445] jepsen worker 16 - jepsen.util 4606 :invoke :write [5 49] INFO [2019-12-17 12:30:13,463] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18542,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:30:13,463] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18542,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:30:13,464] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18543,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:30:13,464] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18543,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:30:13,483] jepsen worker 27 - jepsen.util 2907 :fail :cas [9 [37 43]] :ValueNotSwapFrom INFO [2019-12-17 12:30:13,483] jepsen worker 27 - jepsen.util 2907 :invoke :read [9 nil] INFO [2019-12-17 12:30:13,492] jepsen worker 89 - jepsen.util 3059 :fail :cas [29 [38 21]] :ValueNotSwapFrom INFO [2019-12-17 12:30:13,492] jepsen worker 89 - jepsen.util 3059 :invoke :read [29 nil] INFO [2019-12-17 12:30:13,496] jepsen worker 5 - jepsen.util 2885 :ok :read [1 26] INFO [2019-12-17 12:30:13,497] jepsen worker 5 - jepsen.util 2885 :invoke :cas [1 [13 22]] INFO [2019-12-17 12:30:13,508] jepsen worker 51 - jepsen.util 3021 :fail :cas [17 [4 49]] :ValueNotSwapFrom INFO [2019-12-17 12:30:13,508] jepsen worker 51 - jepsen.util 3021 :invoke :read [17 nil] INFO [2019-12-17 12:30:13,524] jepsen worker 59 - jepsen.util 4199 :info :cas [19 [48 16]] :RequestCanceledException INFO [2019-12-17 12:30:13,527] jepsen worker 59 - jepsen.util 4289 :invoke :write [19 34] INFO [2019-12-17 12:30:13,528] jepsen worker 27 - jepsen.util 2907 :ok :read [9 44] INFO [2019-12-17 12:30:13,528] jepsen worker 27 - jepsen.util 2907 :invoke :read [9 nil] INFO [2019-12-17 12:30:13,565] jepsen worker 27 - jepsen.util 2907 :ok :read [9 44] INFO [2019-12-17 12:30:13,565] jepsen worker 27 - jepsen.util 2907 :invoke :read [9 nil] INFO [2019-12-17 12:30:13,618] jepsen worker 27 - jepsen.util 2907 :ok :read [9 44] INFO [2019-12-17 12:30:13,618] jepsen worker 27 - jepsen.util 2907 :invoke :write [9 13] INFO [2019-12-17 12:30:13,623] jepsen worker 52 - jepsen.util 3382 :info :write [17 14] :RequestCanceledException INFO [2019-12-17 12:30:13,624] jepsen worker 15 - jepsen.util 3885 :info :write [5 10] :RequestCanceledException INFO [2019-12-17 12:30:13,626] jepsen worker 52 - jepsen.util 3472 :invoke :read [17 nil] INFO [2019-12-17 12:30:13,629] jepsen worker 15 - jepsen.util 3975 :invoke :read [5 nil] INFO [2019-12-17 12:30:13,639] jepsen worker 32 - jepsen.util 3632 :ok :write [10 46] INFO [2019-12-17 12:30:13,639] jepsen worker 10 - jepsen.util 2980 :ok :write [3 0] INFO [2019-12-17 12:30:13,639] jepsen worker 32 - jepsen.util 3632 :invoke :cas [10 [48 42]] INFO [2019-12-17 12:30:13,639] jepsen worker 72 - jepsen.util 3132 :ok :write [24 39] INFO [2019-12-17 12:30:13,639] jepsen worker 10 - jepsen.util 2980 :invoke :write [3 15] INFO [2019-12-17 12:30:13,639] jepsen worker 72 - jepsen.util 3132 :invoke :write [24 4] INFO [2019-12-17 12:30:13,651] jepsen worker 84 - jepsen.util 4134 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:30:13,651] jepsen worker 85 - jepsen.util 4765 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:30:13,652] jepsen worker 84 - jepsen.util 4134 :invoke :cas [28 [5 15]] INFO [2019-12-17 12:30:13,652] jepsen worker 85 - jepsen.util 4765 :invoke :write [28 6] INFO [2019-12-17 12:30:13,656] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18560,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:30:13,657] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18560,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:30:13,657] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18564,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:30:13,658] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18564,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:30:13,691] jepsen worker 47 - jepsen.util 3827 :fail :read [15 nil] :RequestTimeoutException INFO [2019-12-17 12:30:13,691] jepsen worker 47 - jepsen.util 3827 :invoke :write [15 19] INFO [2019-12-17 12:30:13,709] jepsen worker 5 - jepsen.util 2885 :fail :cas [1 [13 22]] :ValueNotSwapFrom INFO [2019-12-17 12:30:13,709] jepsen worker 5 - jepsen.util 2885 :invoke :write [1 45] INFO [2019-12-17 12:30:13,724] jepsen worker 43 - jepsen.util 3823 :info :cas [14 [40 3]] :RequestCanceledException INFO [2019-12-17 12:30:13,724] jepsen worker 12 - jepsen.util 3342 :info :write [4 18] :RequestCanceledException INFO [2019-12-17 12:30:13,727] jepsen worker 12 - jepsen.util 3432 :invoke :cas [4 [0 25]] INFO [2019-12-17 12:30:13,730] jepsen worker 43 - jepsen.util 3913 :invoke :cas [14 [44 0]] INFO [2019-12-17 12:30:13,731] jepsen worker 50 - jepsen.util 4730 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:30:13,731] jepsen worker 50 - jepsen.util 4730 :invoke :cas [16 [18 22]] INFO [2019-12-17 12:30:13,731] jepsen worker 45 - jepsen.util 3105 :info :cas [15 [22 46]] :RequestCanceledException INFO [2019-12-17 12:30:13,735] jepsen worker 45 - jepsen.util 3195 :invoke :cas [15 [16 46]] INFO [2019-12-17 12:30:13,738] jepsen worker 37 - jepsen.util 3007 :ok :read [12 20] INFO [2019-12-17 12:30:13,738] jepsen worker 37 - jepsen.util 3007 :invoke :write [12 46] INFO [2019-12-17 12:30:13,754] jepsen worker 64 - jepsen.util 2764 :fail :read [21 nil] :RequestTimeoutException INFO [2019-12-17 12:30:13,754] jepsen worker 20 - jepsen.util 2720 :info :cas [6 [17 35]] :RequestCanceledException INFO [2019-12-17 12:30:13,754] jepsen worker 64 - jepsen.util 2764 :invoke :write [21 38] INFO [2019-12-17 12:30:13,756] jepsen worker 20 - jepsen.util 2810 :invoke :write [6 42] INFO [2019-12-17 12:30:13,780] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:30:13,780] jepsen nemesis - couchbase.util Rebalance check count: 136 INFO [2019-12-17 12:30:13,780] jepsen nemesis - couchbase.util Rebalance stuck count: 117 INFO [2019-12-17 12:30:13,782] jepsen worker 27 - jepsen.util 2907 :ok :write [9 13] INFO [2019-12-17 12:30:13,782] jepsen worker 27 - jepsen.util 2907 :invoke :cas [9 [48 46]] INFO [2019-12-17 12:30:13,808] jepsen worker 62 - jepsen.util 4022 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:30:13,808] jepsen worker 62 - jepsen.util 4022 :invoke :write [20 38] INFO [2019-12-17 12:30:13,809] jepsen worker 46 - jepsen.util 3466 :info :cas [15 [16 0]] :RequestCanceledException INFO [2019-12-17 12:30:13,812] jepsen worker 89 - jepsen.util 3059 :ok :read [29 17] INFO [2019-12-17 12:30:13,812] jepsen worker 89 - jepsen.util 3059 :invoke :cas [29 [40 36]] INFO [2019-12-17 12:30:13,813] jepsen worker 46 - jepsen.util 3556 :invoke :write [15 42] INFO [2019-12-17 12:30:13,814] jepsen worker 19 - jepsen.util 3439 :ok :write [6 21] INFO [2019-12-17 12:30:13,814] jepsen worker 19 - jepsen.util 3439 :invoke :cas [6 [22 0]] INFO [2019-12-17 12:30:13,820] jepsen worker 1 - jepsen.util 3691 :info :cas [0 [44 27]] :RequestCanceledException INFO [2019-12-17 12:30:13,820] jepsen worker 40 - jepsen.util 4540 :info :cas [13 [12 17]] :RequestCanceledException INFO [2019-12-17 12:30:13,824] jepsen worker 1 - jepsen.util 3781 :invoke :cas [0 [11 24]] INFO [2019-12-17 12:30:13,826] jepsen worker 40 - jepsen.util 4630 :invoke :write [13 20] INFO [2019-12-17 12:30:13,830] jepsen worker 88 - jepsen.util 3238 :ok :read [29 17] INFO [2019-12-17 12:30:13,830] jepsen worker 88 - jepsen.util 3238 :invoke :read [29 nil] INFO [2019-12-17 12:30:13,837] jepsen worker 54 - jepsen.util 3294 :info :write [18 26] :RequestCanceledException INFO [2019-12-17 12:30:13,838] jepsen worker 72 - jepsen.util 3132 :ok :write [24 4] INFO [2019-12-17 12:30:13,839] jepsen worker 72 - jepsen.util 3132 :invoke :write [24 26] INFO [2019-12-17 12:30:13,839] jepsen worker 54 - jepsen.util 3384 :invoke :write [18 39] INFO [2019-12-17 12:30:13,852] jepsen worker 64 - jepsen.util 2764 :ok :write [21 38] INFO [2019-12-17 12:30:13,852] jepsen worker 64 - jepsen.util 2764 :invoke :cas [21 [43 0]] INFO [2019-12-17 12:30:13,855] jepsen worker 65 - jepsen.util 2855 :info :write [21 19] :RequestCanceledException INFO [2019-12-17 12:30:13,860] jepsen worker 65 - jepsen.util 2945 :invoke :read [21 nil] INFO [2019-12-17 12:30:13,874] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18592,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:30:13,874] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18592,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:30:13,875] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18595,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:30:13,875] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18595,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:30:13,878] jepsen worker 11 - jepsen.util 3521 :fail :cas [3 [6 28]] :ValueNotSwapFrom INFO [2019-12-17 12:30:13,879] jepsen worker 11 - jepsen.util 3521 :invoke :write [3 44] INFO [2019-12-17 12:30:13,887] jepsen worker 21 - jepsen.util 4161 :info :write [7 45] :RequestCanceledException INFO [2019-12-17 12:30:13,892] jepsen worker 21 - jepsen.util 4251 :invoke :read [7 nil] INFO [2019-12-17 12:30:13,907] jepsen worker 32 - jepsen.util 3632 :fail :cas [10 [48 42]] :ValueNotSwapFrom INFO [2019-12-17 12:30:13,907] jepsen worker 32 - jepsen.util 3632 :invoke :write [10 10] INFO [2019-12-17 12:30:13,908] jepsen worker 22 - jepsen.util 4342 :info :cas [7 [12 34]] :RequestCanceledException INFO [2019-12-17 12:30:13,912] jepsen worker 22 - jepsen.util 4432 :invoke :cas [7 [37 39]] INFO [2019-12-17 12:30:13,923] jepsen worker 75 - jepsen.util 4305 :info :write [25 43] :RequestCanceledException INFO [2019-12-17 12:30:13,929] jepsen worker 75 - jepsen.util 4395 :invoke :write [25 20] INFO [2019-12-17 12:30:13,930] jepsen worker 64 - jepsen.util 2764 :fail :cas [21 [43 0]] :ValueNotSwapFrom INFO [2019-12-17 12:30:13,930] jepsen worker 64 - jepsen.util 2764 :invoke :cas [21 [13 26]] INFO [2019-12-17 12:30:13,947] jepsen worker 47 - jepsen.util 3827 :ok :write [15 19] INFO [2019-12-17 12:30:13,947] jepsen worker 47 - jepsen.util 3827 :invoke :cas [15 [7 16]] INFO [2019-12-17 12:30:13,956] jepsen worker 53 - jepsen.util 3293 :ok :read [17 27] INFO [2019-12-17 12:30:13,956] jepsen worker 53 - jepsen.util 3293 :invoke :read [17 nil] INFO [2019-12-17 12:30:13,985] jepsen worker 64 - jepsen.util 2764 :fail :cas [21 [13 26]] :ValueNotSwapFrom INFO [2019-12-17 12:30:13,985] jepsen worker 64 - jepsen.util 2764 :invoke :write [21 29] INFO [2019-12-17 12:30:14,008] jepsen worker 31 - jepsen.util 2641 :info :write [10 38] :RequestCanceledException INFO [2019-12-17 12:30:14,011] jepsen worker 38 - jepsen.util 3188 :ok :read [12 20] INFO [2019-12-17 12:30:14,011] jepsen worker 38 - jepsen.util 3188 :invoke :cas [12 [12 21]] INFO [2019-12-17 12:30:14,012] jepsen worker 31 - jepsen.util 2731 :invoke :cas [10 [27 33]] INFO [2019-12-17 12:30:14,023] jepsen worker 42 - jepsen.util 4992 :info :write [14 26] :RequestCanceledException INFO [2019-12-17 12:30:14,030] jepsen worker 42 - jepsen.util 5082 :invoke :cas [14 [22 11]] INFO [2019-12-17 12:30:14,038] jepsen worker 8 - jepsen.util 3608 :info :write [2 30] :RequestCanceledException INFO [2019-12-17 12:30:14,047] jepsen worker 8 - jepsen.util 3698 :invoke :cas [2 [0 27]] INFO [2019-12-17 12:30:14,054] jepsen worker 64 - jepsen.util 2764 :ok :write [21 29] INFO [2019-12-17 12:30:14,055] jepsen worker 64 - jepsen.util 2764 :invoke :cas [21 [27 16]] INFO [2019-12-17 12:30:14,055] jepsen worker 41 - jepsen.util 4271 :info :cas [13 [19 16]] :RequestCanceledException INFO [2019-12-17 12:30:14,055] jepsen worker 83 - jepsen.util 4403 :info :cas [27 [16 47]] :RequestCanceledException INFO [2019-12-17 12:30:14,060] jepsen worker 41 - jepsen.util 4361 :invoke :write [13 31] INFO [2019-12-17 12:30:14,060] jepsen worker 83 - jepsen.util 4493 :invoke :cas [27 [19 27]] INFO [2019-12-17 12:30:14,068] jepsen worker 51 - jepsen.util 3021 :ok :read [17 27] INFO [2019-12-17 12:30:14,068] jepsen worker 51 - jepsen.util 3021 :invoke :cas [17 [40 38]] INFO [2019-12-17 12:30:14,071] jepsen worker 72 - jepsen.util 3132 :ok :write [24 26] INFO [2019-12-17 12:30:14,071] jepsen worker 11 - jepsen.util 3521 :ok :write [3 44] INFO [2019-12-17 12:30:14,071] jepsen worker 72 - jepsen.util 3132 :invoke :cas [24 [49 3]] INFO [2019-12-17 12:30:14,071] jepsen worker 11 - jepsen.util 3521 :invoke :write [3 14] INFO [2019-12-17 12:30:14,090] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18606,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:30:14,090] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18606,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:30:14,090] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18610,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:30:14,090] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18610,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:30:14,091] jepsen worker 33 - jepsen.util 4533 :info :cas [11 [22 14]] :RequestCanceledException INFO [2019-12-17 12:30:14,099] jepsen worker 33 - jepsen.util 4623 :invoke :write [11 3] INFO [2019-12-17 12:30:14,107] jepsen worker 48 - jepsen.util 3738 :info :write [16 10] :RequestCanceledException INFO [2019-12-17 12:30:14,107] jepsen worker 29 - jepsen.util 2639 :info :cas [9 [42 23]] :RequestCanceledException INFO [2019-12-17 12:30:14,111] jepsen worker 29 - jepsen.util 2729 :invoke :read [9 nil] INFO [2019-12-17 12:30:14,114] jepsen worker 48 - jepsen.util 3828 :invoke :read [16 nil] INFO [2019-12-17 12:30:14,119] jepsen worker 64 - jepsen.util 2764 :fail :cas [21 [27 16]] :ValueNotSwapFrom INFO [2019-12-17 12:30:14,119] jepsen worker 64 - jepsen.util 2764 :invoke :cas [21 [14 48]] INFO [2019-12-17 12:30:14,124] jepsen worker 26 - jepsen.util 4526 :info :cas [8 [21 20]] :RequestCanceledException INFO [2019-12-17 12:30:14,124] jepsen worker 57 - jepsen.util 4017 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:30:14,124] jepsen worker 0 - jepsen.util 4770 :info :write [0 34] :RequestCanceledException INFO [2019-12-17 12:30:14,124] jepsen worker 57 - jepsen.util 4017 :invoke :read [19 nil] INFO [2019-12-17 12:30:14,129] jepsen worker 0 - jepsen.util 4860 :invoke :cas [0 [19 4]] INFO [2019-12-17 12:30:14,130] jepsen worker 26 - jepsen.util 4616 :invoke :write [8 17] INFO [2019-12-17 12:30:14,157] jepsen worker 31 - jepsen.util 2731 :fail :cas [10 [27 33]] :ValueNotSwapFrom INFO [2019-12-17 12:30:14,158] jepsen worker 31 - jepsen.util 2731 :invoke :cas [10 [37 41]] INFO [2019-12-17 12:30:14,171] jepsen worker 72 - jepsen.util 3132 :fail :cas [24 [49 3]] :ValueNotSwapFrom INFO [2019-12-17 12:30:14,171] jepsen worker 72 - jepsen.util 3132 :invoke :read [24 nil] INFO [2019-12-17 12:30:14,179] jepsen worker 29 - jepsen.util 2729 :ok :read [9 13] INFO [2019-12-17 12:30:14,179] jepsen worker 29 - jepsen.util 2729 :invoke :write [9 10] INFO [2019-12-17 12:30:14,186] jepsen worker 51 - jepsen.util 3021 :fail :cas [17 [40 38]] :ValueNotSwapFrom INFO [2019-12-17 12:30:14,186] jepsen worker 51 - jepsen.util 3021 :invoke :cas [17 [41 49]] INFO [2019-12-17 12:30:14,209] jepsen worker 69 - jepsen.util 3129 :info :write [23 1] :RequestCanceledException INFO [2019-12-17 12:30:14,212] jepsen worker 69 - jepsen.util 3219 :invoke :cas [23 [38 46]] INFO [2019-12-17 12:30:14,223] jepsen worker 17 - jepsen.util 4067 :info :write [5 40] :RequestCanceledException INFO [2019-12-17 12:30:14,228] jepsen worker 17 - jepsen.util 4157 :invoke :write [5 7] INFO [2019-12-17 12:30:14,233] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18632,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:30:14,233] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18639,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:30:14,233] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18639,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:30:14,233] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18632,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:30:14,233] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18642,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:30:14,233] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18642,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:30:14,233] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18634,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:30:14,234] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18634,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:30:14,235] jepsen worker 31 - jepsen.util 2731 :fail :cas [10 [37 41]] :ValueNotSwapFrom INFO [2019-12-17 12:30:14,235] jepsen worker 31 - jepsen.util 2731 :invoke :read [10 nil] INFO [2019-12-17 12:30:14,238] jepsen worker 28 - jepsen.util 2368 :fail :read [9 nil] :RequestTimeoutException INFO [2019-12-17 12:30:14,238] jepsen worker 56 - jepsen.util 3116 :info :write [18 12] :RequestCanceledException INFO [2019-12-17 12:30:14,238] jepsen worker 71 - jepsen.util 3131 :info :write [23 36] :RequestCanceledException INFO [2019-12-17 12:30:14,238] jepsen worker 28 - jepsen.util 2368 :invoke :read [9 nil] INFO [2019-12-17 12:30:14,239] jepsen worker 64 - jepsen.util 2764 :fail :cas [21 [14 48]] :ValueNotSwapFrom INFO [2019-12-17 12:30:14,239] jepsen worker 64 - jepsen.util 2764 :invoke :cas [21 [44 31]] INFO [2019-12-17 12:30:14,241] jepsen worker 56 - jepsen.util 3206 :invoke :write [18 12] INFO [2019-12-17 12:30:14,243] jepsen worker 71 - jepsen.util 3221 :invoke :cas [23 [23 46]] INFO [2019-12-17 12:30:14,250] jepsen worker 72 - jepsen.util 3132 :ok :read [24 26] INFO [2019-12-17 12:30:14,251] jepsen worker 72 - jepsen.util 3132 :invoke :write [24 35] INFO [2019-12-17 12:30:14,273] jepsen worker 52 - jepsen.util 3472 :ok :read [17 27] INFO [2019-12-17 12:30:14,273] jepsen worker 52 - jepsen.util 3472 :invoke :read [17 nil] INFO [2019-12-17 12:30:14,290] jepsen worker 29 - jepsen.util 2729 :ok :write [9 10] INFO [2019-12-17 12:30:14,290] jepsen worker 29 - jepsen.util 2729 :invoke :read [9 nil] INFO [2019-12-17 12:30:14,316] jepsen worker 64 - jepsen.util 2764 :fail :cas [21 [44 31]] :ValueNotSwapFrom INFO [2019-12-17 12:30:14,316] jepsen worker 64 - jepsen.util 2764 :invoke :cas [21 [14 9]] INFO [2019-12-17 12:30:14,324] jepsen worker 63 - jepsen.util 2853 :info :cas [21 [37 39]] :RequestCanceledException INFO [2019-12-17 12:30:14,330] jepsen worker 63 - jepsen.util 2943 :invoke :read [21 nil] INFO [2019-12-17 12:30:14,360] jepsen worker 52 - jepsen.util 3472 :ok :read [17 27] INFO [2019-12-17 12:30:14,360] jepsen worker 52 - jepsen.util 3472 :invoke :read [17 nil] INFO [2019-12-17 12:30:14,364] jepsen worker 28 - jepsen.util 2368 :ok :read [9 10] INFO [2019-12-17 12:30:14,364] jepsen worker 28 - jepsen.util 2368 :invoke :cas [9 [12 17]] INFO [2019-12-17 12:30:14,408] jepsen worker 14 - jepsen.util 3074 :info :write [4 38] :RequestCanceledException INFO [2019-12-17 12:30:14,410] jepsen worker 14 - jepsen.util 3164 :invoke :write [4 49] INFO [2019-12-17 12:30:14,412] jepsen worker 64 - jepsen.util 2764 :fail :cas [21 [14 9]] :ValueNotSwapFrom INFO [2019-12-17 12:30:14,412] jepsen worker 64 - jepsen.util 2764 :invoke :write [21 29] INFO [2019-12-17 12:30:14,451] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18661,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:30:14,452] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18661,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:30:14,452] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18665,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:30:14,453] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18665,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:30:14,455] jepsen worker 9 - jepsen.util 3699 :info :write [3 27] :RequestCanceledException INFO [2019-12-17 12:30:14,463] jepsen worker 9 - jepsen.util 3789 :invoke :cas [3 [25 46]] INFO [2019-12-17 12:30:14,467] jepsen worker 12 - jepsen.util 3432 :fail :cas [4 [0 25]] :ValueNotSwapFrom INFO [2019-12-17 12:30:14,467] jepsen worker 12 - jepsen.util 3432 :invoke :cas [4 [35 19]] INFO [2019-12-17 12:30:14,472] jepsen worker 88 - jepsen.util 3238 :ok :read [29 17] INFO [2019-12-17 12:30:14,472] jepsen worker 88 - jepsen.util 3238 :invoke :write [29 19] INFO [2019-12-17 12:30:14,508] jepsen worker 25 - jepsen.util 4705 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:30:14,508] jepsen worker 25 - jepsen.util 4705 :invoke :read [8 nil] INFO [2019-12-17 12:30:14,519] jepsen worker 9 - jepsen.util 3789 :fail :cas [3 [25 46]] :ValueNotSwapFrom INFO [2019-12-17 12:30:14,520] jepsen worker 9 - jepsen.util 3789 :invoke :cas [3 [40 33]] INFO [2019-12-17 12:30:14,524] jepsen worker 81 - jepsen.util 4221 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:30:14,524] jepsen worker 81 - jepsen.util 4221 :invoke :write [27 11] INFO [2019-12-17 12:30:14,524] jepsen worker 74 - jepsen.util 3404 :info :write [24 9] :RequestCanceledException INFO [2019-12-17 12:30:14,524] jepsen worker 76 - jepsen.util 4396 :info :write [25 25] :RequestCanceledException INFO [2019-12-17 12:30:14,524] jepsen worker 18 - jepsen.util 3348 :fail :read [6 nil] :RequestTimeoutException INFO [2019-12-17 12:30:14,525] jepsen worker 18 - jepsen.util 3348 :invoke :cas [6 [19 3]] INFO [2019-12-17 12:30:14,528] jepsen worker 74 - jepsen.util 3494 :invoke :read [24 nil] INFO [2019-12-17 12:30:14,529] jepsen worker 53 - jepsen.util 3293 :ok :read [17 27] INFO [2019-12-17 12:30:14,529] jepsen worker 53 - jepsen.util 3293 :invoke :cas [17 [16 21]] INFO [2019-12-17 12:30:14,533] jepsen worker 76 - jepsen.util 4486 :invoke :write [25 7] INFO [2019-12-17 12:30:14,536] jepsen worker 64 - jepsen.util 2764 :ok :write [21 29] INFO [2019-12-17 12:30:14,537] jepsen worker 61 - jepsen.util 4831 :info :write [20 20] :RequestCanceledException INFO [2019-12-17 12:30:14,538] jepsen worker 64 - jepsen.util 2764 :invoke :write [21 39] INFO [2019-12-17 12:30:14,549] jepsen worker 61 - jepsen.util 4921 :invoke :cas [20 [36 5]] INFO [2019-12-17 12:30:14,573] jepsen worker 38 - jepsen.util 3188 :fail :cas [12 [12 21]] :ValueNotSwapFrom INFO [2019-12-17 12:30:14,573] jepsen worker 38 - jepsen.util 3188 :invoke :read [12 nil] INFO [2019-12-17 12:30:14,588] jepsen worker 9 - jepsen.util 3789 :fail :cas [3 [40 33]] :ValueNotSwapFrom INFO [2019-12-17 12:30:14,588] jepsen worker 9 - jepsen.util 3789 :invoke :write [3 18] INFO [2019-12-17 12:30:14,599] jepsen worker 18 - jepsen.util 3348 :fail :cas [6 [19 3]] :ValueNotSwapFrom INFO [2019-12-17 12:30:14,599] jepsen worker 18 - jepsen.util 3348 :invoke :read [6 nil] INFO [2019-12-17 12:30:14,603] jepsen worker 74 - jepsen.util 3494 :ok :read [24 26] INFO [2019-12-17 12:30:14,604] jepsen worker 74 - jepsen.util 3494 :invoke :cas [24 [31 40]] INFO [2019-12-17 12:30:14,608] jepsen worker 86 - jepsen.util 4406 :info :cas [28 [37 33]] :RequestCanceledException INFO [2019-12-17 12:30:14,611] jepsen worker 86 - jepsen.util 4496 :invoke :write [28 20] INFO [2019-12-17 12:30:14,621] jepsen worker 51 - jepsen.util 3021 :fail :cas [17 [41 49]] :ValueNotSwapFrom INFO [2019-12-17 12:30:14,621] jepsen worker 51 - jepsen.util 3021 :invoke :read [17 nil] INFO [2019-12-17 12:30:14,623] jepsen worker 58 - jepsen.util 4198 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:30:14,624] jepsen worker 58 - jepsen.util 4198 :invoke :write [19 19] INFO [2019-12-17 12:30:14,640] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18686,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:30:14,641] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18686,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:30:14,641] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18691,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:30:14,641] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18691,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:30:14,657] jepsen worker 64 - jepsen.util 2764 :ok :write [21 39] INFO [2019-12-17 12:30:14,659] jepsen worker 11 - jepsen.util 3521 :ok :write [3 14] INFO [2019-12-17 12:30:14,659] jepsen worker 11 - jepsen.util 3521 :invoke :read [3 nil] INFO [2019-12-17 12:30:14,660] jepsen worker 64 - jepsen.util 2764 :invoke :write [21 27] INFO [2019-12-17 12:30:14,660] jepsen worker 72 - jepsen.util 3132 :ok :write [24 35] INFO [2019-12-17 12:30:14,660] jepsen worker 72 - jepsen.util 3132 :invoke :write [24 46] INFO [2019-12-17 12:30:14,678] jepsen worker 18 - jepsen.util 3348 :ok :read [6 21] INFO [2019-12-17 12:30:14,679] jepsen worker 18 - jepsen.util 3348 :invoke :read [6 nil] INFO [2019-12-17 12:30:14,690] jepsen worker 82 - jepsen.util 5122 :info :write [27 2] :RequestCanceledException INFO [2019-12-17 12:30:14,694] jepsen worker 82 - jepsen.util 5212 :invoke :read [27 nil] INFO [2019-12-17 12:30:14,702] jepsen worker 74 - jepsen.util 3494 :fail :cas [24 [31 40]] :ValueNotSwapFrom INFO [2019-12-17 12:30:14,702] jepsen worker 74 - jepsen.util 3494 :invoke :read [24 nil] INFO [2019-12-17 12:30:14,708] jepsen worker 77 - jepsen.util 3947 :info :write [25 16] :RequestCanceledException INFO [2019-12-17 12:30:14,715] jepsen worker 77 - jepsen.util 4037 :invoke :cas [25 [44 40]] INFO [2019-12-17 12:30:14,724] jepsen worker 66 - jepsen.util 4116 :info :write [22 4] :RequestCanceledException INFO [2019-12-17 12:30:14,724] jepsen worker 2 - jepsen.util 3872 :info :cas [0 [20 14]] :RequestCanceledException INFO [2019-12-17 12:30:14,728] jepsen worker 2 - jepsen.util 3962 :invoke :read [0 nil] INFO [2019-12-17 12:30:14,728] jepsen worker 66 - jepsen.util 4206 :invoke :write [22 7] INFO [2019-12-17 12:30:14,739] jepsen worker 68 - jepsen.util 4028 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:30:14,740] jepsen worker 24 - jepsen.util 4164 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:30:14,740] jepsen worker 24 - jepsen.util 4164 :invoke :read [8 nil] INFO [2019-12-17 12:30:14,740] jepsen worker 68 - jepsen.util 4028 :invoke :read [22 nil] INFO [2019-12-17 12:30:14,750] jepsen worker 18 - jepsen.util 3348 :ok :read [6 21] INFO [2019-12-17 12:30:14,750] jepsen worker 18 - jepsen.util 3348 :invoke :write [6 34] INFO [2019-12-17 12:30:14,756] jepsen worker 39 - jepsen.util 3999 :info :write [13 44] :RequestCanceledException INFO [2019-12-17 12:30:14,763] jepsen worker 11 - jepsen.util 3521 :ok :read [3 14] INFO [2019-12-17 12:30:14,763] jepsen worker 11 - jepsen.util 3521 :invoke :cas [3 [40 27]] INFO [2019-12-17 12:30:14,764] jepsen worker 39 - jepsen.util 4089 :invoke :read [13 nil] INFO [2019-12-17 12:30:14,765] jepsen worker 28 - jepsen.util 2368 :fail :cas [9 [12 17]] :ValueNotSwapFrom INFO [2019-12-17 12:30:14,765] jepsen worker 28 - jepsen.util 2368 :invoke :write [9 42] INFO [2019-12-17 12:30:14,783] jepsen worker 64 - jepsen.util 2764 :ok :write [21 27] INFO [2019-12-17 12:30:14,783] jepsen worker 64 - jepsen.util 2764 :invoke :cas [21 [33 11]] INFO [2019-12-17 12:30:14,786] jepsen worker 74 - jepsen.util 3494 :ok :read [24 35] INFO [2019-12-17 12:30:14,787] jepsen worker 74 - jepsen.util 3494 :invoke :cas [24 [39 4]] INFO [2019-12-17 12:30:14,790] jepsen worker 13 - jepsen.util 3073 :info :write [4 13] :RequestCanceledException INFO [2019-12-17 12:30:14,799] jepsen worker 13 - jepsen.util 3163 :invoke :write [4 21] INFO [2019-12-17 12:30:14,799] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:30:14,799] jepsen nemesis - couchbase.util Rebalance check count: 137 INFO [2019-12-17 12:30:14,800] jepsen nemesis - couchbase.util Rebalance stuck count: 118 INFO [2019-12-17 12:30:14,805] jepsen worker 38 - jepsen.util 3188 :ok :read [12 20] INFO [2019-12-17 12:30:14,805] jepsen worker 38 - jepsen.util 3188 :invoke :cas [12 [26 4]] INFO [2019-12-17 12:30:14,808] jepsen worker 6 - jepsen.util 4596 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:30:14,808] jepsen worker 6 - jepsen.util 4596 :invoke :cas [2 [38 6]] INFO [2019-12-17 12:30:14,809] jepsen worker 69 - jepsen.util 3219 :fail :cas [23 [38 46]] :ValueNotSwapFrom INFO [2019-12-17 12:30:14,810] jepsen worker 69 - jepsen.util 3219 :invoke :write [23 45] INFO [2019-12-17 12:30:14,823] jepsen worker 44 - jepsen.util 4544 :info :cas [14 [19 49]] :RequestCanceledException INFO [2019-12-17 12:30:14,827] jepsen worker 44 - jepsen.util 4634 :invoke :cas [14 [49 4]] INFO [2019-12-17 12:30:14,848] jepsen worker 64 - jepsen.util 2764 :fail :cas [21 [33 11]] :ValueNotSwapFrom INFO [2019-12-17 12:30:14,848] jepsen worker 64 - jepsen.util 2764 :invoke :cas [21 [34 22]] INFO [2019-12-17 12:30:14,850] jepsen worker 11 - jepsen.util 3521 :fail :cas [3 [40 27]] :ValueNotSwapFrom INFO [2019-12-17 12:30:14,850] jepsen worker 11 - jepsen.util 3521 :invoke :write [3 13] INFO [2019-12-17 12:30:14,854] jepsen worker 67 - jepsen.util 4657 :info :write [22 14] :RequestCanceledException INFO [2019-12-17 12:30:14,857] jepsen worker 51 - jepsen.util 3021 :ok :read [17 27] INFO [2019-12-17 12:30:14,857] jepsen worker 51 - jepsen.util 3021 :invoke :cas [17 [22 44]] INFO [2019-12-17 12:30:14,859] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18720,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:30:14,859] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18720,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:30:14,859] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18721,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:30:14,859] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18721,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:30:14,867] jepsen worker 67 - jepsen.util 4747 :invoke :cas [22 [48 9]] INFO [2019-12-17 12:30:14,884] jepsen worker 53 - jepsen.util 3293 :fail :cas [17 [16 21]] :ValueNotSwapFrom INFO [2019-12-17 12:30:14,884] jepsen worker 53 - jepsen.util 3293 :invoke :read [17 nil] INFO [2019-12-17 12:30:14,890] jepsen worker 70 - jepsen.util 3670 :info :write [23 25] :RequestCanceledException INFO [2019-12-17 12:30:14,890] jepsen worker 30 - jepsen.util 3720 :fail :read [10 nil] :RequestTimeoutException INFO [2019-12-17 12:30:14,890] jepsen worker 30 - jepsen.util 3720 :invoke :cas [10 [32 2]] INFO [2019-12-17 12:30:14,892] jepsen worker 70 - jepsen.util 3760 :invoke :cas [23 [10 41]] INFO [2019-12-17 12:30:14,902] jepsen worker 64 - jepsen.util 2764 :fail :cas [21 [34 22]] :ValueNotSwapFrom INFO [2019-12-17 12:30:14,902] jepsen worker 64 - jepsen.util 2764 :invoke :cas [21 [10 29]] INFO [2019-12-17 12:30:14,907] jepsen worker 49 - jepsen.util 3739 :info :cas [16 [19 39]] :RequestCanceledException INFO [2019-12-17 12:30:14,913] jepsen worker 49 - jepsen.util 3829 :invoke :read [16 nil] INFO [2019-12-17 12:30:14,942] jepsen worker 72 - jepsen.util 3132 :ok :write [24 46] INFO [2019-12-17 12:30:14,943] jepsen worker 72 - jepsen.util 3132 :invoke :read [24 nil] INFO [2019-12-17 12:30:14,948] jepsen worker 64 - jepsen.util 2764 :fail :cas [21 [10 29]] :ValueNotSwapFrom INFO [2019-12-17 12:30:14,948] jepsen worker 64 - jepsen.util 2764 :invoke :cas [21 [46 14]] INFO [2019-12-17 12:30:14,955] jepsen worker 73 - jepsen.util 3043 :info :write [24 40] :RequestCanceledException INFO [2019-12-17 12:30:14,956] jepsen worker 60 - jepsen.util 4470 :info :cas [20 [28 38]] :RequestCanceledException INFO [2019-12-17 12:30:14,957] jepsen worker 73 - jepsen.util 3133 :invoke :read [24 nil] INFO [2019-12-17 12:30:14,964] jepsen worker 60 - jepsen.util 4560 :invoke :write [20 41] INFO [2019-12-17 12:30:14,979] jepsen worker 12 - jepsen.util 3432 :fail :cas [4 [35 19]] :ValueNotSwapFrom INFO [2019-12-17 12:30:14,979] jepsen worker 12 - jepsen.util 3432 :invoke :write [4 48] INFO [2019-12-17 12:30:14,981] jepsen worker 71 - jepsen.util 3221 :fail :cas [23 [23 46]] :ValueNotSwapFrom INFO [2019-12-17 12:30:14,981] jepsen worker 71 - jepsen.util 3221 :invoke :write [23 38] INFO [2019-12-17 12:30:15,008] jepsen worker 55 - jepsen.util 3835 :info :write [18 31] :RequestCanceledException INFO [2019-12-17 12:30:15,013] jepsen worker 55 - jepsen.util 3925 :invoke :cas [18 [12 31]] INFO [2019-12-17 12:30:15,024] jepsen worker 34 - jepsen.util 4264 :info :write [11 34] :RequestCanceledException INFO [2019-12-17 12:30:15,026] jepsen worker 64 - jepsen.util 2764 :fail :cas [21 [46 14]] :ValueNotSwapFrom INFO [2019-12-17 12:30:15,026] jepsen worker 64 - jepsen.util 2764 :invoke :cas [21 [13 48]] INFO [2019-12-17 12:30:15,027] jepsen worker 28 - jepsen.util 2368 :ok :write [9 42] INFO [2019-12-17 12:30:15,027] jepsen worker 28 - jepsen.util 2368 :invoke :read [9 nil] INFO [2019-12-17 12:30:15,028] jepsen worker 34 - jepsen.util 4354 :invoke :write [11 2] INFO [2019-12-17 12:30:15,034] jepsen worker 73 - jepsen.util 3133 :ok :read [24 46] INFO [2019-12-17 12:30:15,034] jepsen worker 73 - jepsen.util 3133 :invoke :cas [24 [22 28]] INFO [2019-12-17 12:30:15,066] jepsen worker 18 - jepsen.util 3348 :ok :write [6 34] INFO [2019-12-17 12:30:15,066] jepsen worker 18 - jepsen.util 3348 :invoke :write [6 30] INFO [2019-12-17 12:30:15,077] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18734,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:30:15,077] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18734,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:30:15,078] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18737,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:30:15,078] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18737,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:30:15,079] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18738,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:30:15,079] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18738,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:30:15,085] jepsen worker 72 - jepsen.util 3132 :ok :read [24 46] INFO [2019-12-17 12:30:15,085] jepsen worker 72 - jepsen.util 3132 :invoke :cas [24 [3 38]] INFO [2019-12-17 12:30:15,138] jepsen worker 4 - jepsen.util 2434 :info :write [1 26] :RequestCanceledException INFO [2019-12-17 12:30:15,138] jepsen worker 64 - jepsen.util 2764 :fail :cas [21 [13 48]] :ValueNotSwapFrom INFO [2019-12-17 12:30:15,138] jepsen worker 64 - jepsen.util 2764 :invoke :read [21 nil] INFO [2019-12-17 12:30:15,142] jepsen worker 28 - jepsen.util 2368 :ok :read [9 42] INFO [2019-12-17 12:30:15,142] jepsen worker 28 - jepsen.util 2368 :invoke :write [9 21] INFO [2019-12-17 12:30:15,146] jepsen worker 4 - jepsen.util 2524 :invoke :cas [1 [13 19]] INFO [2019-12-17 12:30:15,147] jepsen worker 89 - jepsen.util 3059 :fail :cas [29 [40 36]] :ValueNotSwapFrom INFO [2019-12-17 12:30:15,148] jepsen worker 89 - jepsen.util 3059 :invoke :write [29 8] INFO [2019-12-17 12:30:15,170] jepsen worker 11 - jepsen.util 3521 :ok :write [3 13] INFO [2019-12-17 12:30:15,170] jepsen worker 11 - jepsen.util 3521 :invoke :read [3 nil] INFO [2019-12-17 12:30:15,183] jepsen worker 72 - jepsen.util 3132 :fail :cas [24 [3 38]] :ValueNotSwapFrom INFO [2019-12-17 12:30:15,183] jepsen worker 72 - jepsen.util 3132 :invoke :write [24 44] INFO [2019-12-17 12:30:15,191] jepsen worker 7 - jepsen.util 4057 :info :write [2 44] :RequestCanceledException INFO [2019-12-17 12:30:15,194] jepsen worker 74 - jepsen.util 3494 :fail :cas [24 [39 4]] :ValueNotSwapFrom INFO [2019-12-17 12:30:15,194] jepsen worker 74 - jepsen.util 3494 :invoke :read [24 nil] INFO [2019-12-17 12:30:15,198] jepsen worker 73 - jepsen.util 3133 :fail :cas [24 [22 28]] :ValueNotSwapFrom INFO [2019-12-17 12:30:15,198] jepsen worker 73 - jepsen.util 3133 :invoke :read [24 nil] INFO [2019-12-17 12:30:15,199] jepsen worker 7 - jepsen.util 4147 :invoke :read [2 nil] INFO [2019-12-17 12:30:15,208] jepsen worker 23 - jepsen.util 4163 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:30:15,208] jepsen worker 23 - jepsen.util 4163 :invoke :cas [7 [9 23]] INFO [2019-12-17 12:30:15,218] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18767,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:30:15,218] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18767,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:30:15,219] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18753,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:30:15,219] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18753,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:30:15,219] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18754,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:30:15,220] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18754,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:30:15,223] jepsen worker 79 - jepsen.util 3139 :info :write [26 33] :RequestCanceledException INFO [2019-12-17 12:30:15,223] jepsen worker 36 - jepsen.util 3276 :info :write [12 25] :RequestCanceledException INFO [2019-12-17 12:30:15,223] jepsen worker 3 - jepsen.util 3243 :info :write [1 46] :RequestCanceledException INFO [2019-12-17 12:30:15,227] jepsen worker 79 - jepsen.util 3229 :invoke :cas [26 [27 44]] INFO [2019-12-17 12:30:15,227] jepsen worker 36 - jepsen.util 3366 :invoke :write [12 36] INFO [2019-12-17 12:30:15,230] jepsen worker 3 - jepsen.util 3333 :invoke :cas [1 [11 43]] INFO [2019-12-17 12:30:15,236] jepsen worker 13 - jepsen.util 3163 :ok :write [4 21] INFO [2019-12-17 12:30:15,237] jepsen worker 13 - jepsen.util 3163 :invoke :read [4 nil] INFO [2019-12-17 12:30:15,248] jepsen worker 18 - jepsen.util 3348 :ok :write [6 30] INFO [2019-12-17 12:30:15,249] jepsen worker 18 - jepsen.util 3348 :invoke :cas [6 [26 9]] INFO [2019-12-17 12:30:15,249] jepsen worker 30 - jepsen.util 3720 :fail :cas [10 [32 2]] :ValueNotSwapFrom INFO [2019-12-17 12:30:15,249] jepsen worker 30 - jepsen.util 3720 :invoke :read [10 nil] INFO [2019-12-17 12:30:15,250] jepsen worker 64 - jepsen.util 2764 :ok :read [21 27] INFO [2019-12-17 12:30:15,250] jepsen worker 64 - jepsen.util 2764 :invoke :cas [21 [25 21]] INFO [2019-12-17 12:30:15,280] jepsen worker 11 - jepsen.util 3521 :ok :read [3 13] INFO [2019-12-17 12:30:15,280] jepsen worker 11 - jepsen.util 3521 :invoke :write [3 12] INFO [2019-12-17 12:30:15,281] jepsen worker 73 - jepsen.util 3133 :ok :read [24 46] INFO [2019-12-17 12:30:15,281] jepsen worker 73 - jepsen.util 3133 :invoke :cas [24 [5 41]] INFO [2019-12-17 12:30:15,290] jepsen worker 87 - jepsen.util 2967 :info :write [29 1] :RequestCanceledException INFO [2019-12-17 12:30:15,293] jepsen worker 87 - jepsen.util 3057 :invoke :cas [29 [1 11]] INFO [2019-12-17 12:30:15,313] jepsen worker 89 - jepsen.util 3059 :ok :write [29 8] INFO [2019-12-17 12:30:15,314] jepsen worker 89 - jepsen.util 3059 :invoke :cas [29 [0 6]] INFO [2019-12-17 12:30:15,324] jepsen worker 80 - jepsen.util 3050 :info :write [26 10] :RequestCanceledException INFO [2019-12-17 12:30:15,326] jepsen worker 80 - jepsen.util 3140 :invoke :cas [26 [21 41]] INFO [2019-12-17 12:30:15,358] jepsen worker 74 - jepsen.util 3494 :ok :read [24 46] INFO [2019-12-17 12:30:15,358] jepsen worker 74 - jepsen.util 3494 :invoke :cas [24 [39 36]] INFO [2019-12-17 12:30:15,371] jepsen worker 64 - jepsen.util 2764 :fail :cas [21 [25 21]] :ValueNotSwapFrom INFO [2019-12-17 12:30:15,372] jepsen worker 64 - jepsen.util 2764 :invoke :cas [21 [36 45]] INFO [2019-12-17 12:30:15,382] jepsen worker 79 - jepsen.util 3229 :fail :cas [26 [27 44]] :ValueNotSwapFrom INFO [2019-12-17 12:30:15,382] jepsen worker 79 - jepsen.util 3229 :invoke :write [26 19] INFO [2019-12-17 12:30:15,389] jepsen worker 28 - jepsen.util 2368 :ok :write [9 21] INFO [2019-12-17 12:30:15,389] jepsen worker 28 - jepsen.util 2368 :invoke :cas [9 [10 40]] INFO [2019-12-17 12:30:15,390] jepsen worker 30 - jepsen.util 3720 :ok :read [10 46] INFO [2019-12-17 12:30:15,390] jepsen worker 30 - jepsen.util 3720 :invoke :read [10 nil] INFO [2019-12-17 12:30:15,413] jepsen worker 18 - jepsen.util 3348 :fail :cas [6 [26 9]] :ValueNotSwapFrom INFO [2019-12-17 12:30:15,413] jepsen worker 18 - jepsen.util 3348 :invoke :cas [6 [18 27]] INFO [2019-12-17 12:30:15,417] jepsen worker 73 - jepsen.util 3133 :fail :cas [24 [5 41]] :ValueNotSwapFrom INFO [2019-12-17 12:30:15,417] jepsen worker 73 - jepsen.util 3133 :invoke :read [24 nil] INFO [2019-12-17 12:30:15,435] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18792,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:30:15,435] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18792,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:30:15,438] jepsen worker 70 - jepsen.util 3760 :fail :cas [23 [10 41]] :ValueNotSwapFrom INFO [2019-12-17 12:30:15,438] jepsen worker 70 - jepsen.util 3760 :invoke :cas [23 [33 31]] INFO [2019-12-17 12:30:15,455] jepsen worker 78 - jepsen.util 3498 :info :write [26 49] :RequestCanceledException INFO [2019-12-17 12:30:15,463] jepsen worker 78 - jepsen.util 3588 :invoke :write [26 41] INFO [2019-12-17 12:30:15,486] jepsen worker 4 - jepsen.util 2524 :fail :cas [1 [13 19]] :ValueNotSwapFrom INFO [2019-12-17 12:30:15,487] jepsen worker 4 - jepsen.util 2524 :invoke :cas [1 [14 42]] INFO [2019-12-17 12:30:15,489] jepsen worker 38 - jepsen.util 3188 :fail :cas [12 [26 4]] :ValueNotSwapFrom INFO [2019-12-17 12:30:15,489] jepsen worker 38 - jepsen.util 3188 :invoke :cas [12 [27 40]] INFO [2019-12-17 12:30:15,519] jepsen worker 80 - jepsen.util 3140 :fail :cas [26 [21 41]] :ValueNotSwapFrom INFO [2019-12-17 12:30:15,519] jepsen worker 80 - jepsen.util 3140 :invoke :write [26 26] INFO [2019-12-17 12:30:15,524] jepsen worker 16 - jepsen.util 4606 :info :write [5 49] :RequestCanceledException INFO [2019-12-17 12:30:15,537] jepsen worker 16 - jepsen.util 4696 :invoke :write [5 34] INFO [2019-12-17 12:30:15,538] jepsen worker 35 - jepsen.util 4085 :info :write [11 16] :RequestCanceledException INFO [2019-12-17 12:30:15,548] jepsen worker 35 - jepsen.util 4175 :invoke :cas [11 [41 32]] INFO [2019-12-17 12:30:15,559] jepsen worker 64 - jepsen.util 2764 :fail :cas [21 [36 45]] :ValueNotSwapFrom INFO [2019-12-17 12:30:15,560] jepsen worker 64 - jepsen.util 2764 :invoke :read [21 nil] INFO [2019-12-17 12:30:15,569] jepsen worker 18 - jepsen.util 3348 :fail :cas [6 [18 27]] :ValueNotSwapFrom INFO [2019-12-17 12:30:15,569] jepsen worker 18 - jepsen.util 3348 :invoke :read [6 nil] INFO [2019-12-17 12:30:15,590] jepsen worker 3 - jepsen.util 3333 :fail :cas [1 [11 43]] :ValueNotSwapFrom INFO [2019-12-17 12:30:15,590] jepsen worker 3 - jepsen.util 3333 :invoke :write [1 8] INFO [2019-12-17 12:30:15,590] jepsen worker 59 - jepsen.util 4289 :info :write [19 34] :RequestCanceledException INFO [2019-12-17 12:30:15,599] jepsen worker 74 - jepsen.util 3494 :fail :cas [24 [39 36]] :ValueNotSwapFrom INFO [2019-12-17 12:30:15,599] jepsen worker 74 - jepsen.util 3494 :invoke :write [24 37] INFO [2019-12-17 12:30:15,604] jepsen worker 59 - jepsen.util 4379 :invoke :write [19 0] INFO [2019-12-17 12:30:15,604] jepsen worker 73 - jepsen.util 3133 :ok :read [24 46] INFO [2019-12-17 12:30:15,604] jepsen worker 73 - jepsen.util 3133 :invoke :cas [24 [28 25]] INFO [2019-12-17 12:30:15,621] jepsen worker 4 - jepsen.util 2524 :fail :cas [1 [14 42]] :ValueNotSwapFrom INFO [2019-12-17 12:30:15,621] jepsen worker 4 - jepsen.util 2524 :invoke :read [1 nil] INFO [2019-12-17 12:30:15,629] jepsen worker 70 - jepsen.util 3760 :fail :cas [23 [33 31]] :ValueNotSwapFrom INFO [2019-12-17 12:30:15,629] jepsen worker 70 - jepsen.util 3760 :invoke :read [23 nil] INFO [2019-12-17 12:30:15,636] jepsen worker 51 - jepsen.util 3021 :fail :cas [17 [22 44]] :ValueNotSwapFrom INFO [2019-12-17 12:30:15,636] jepsen worker 51 - jepsen.util 3021 :invoke :read [17 nil] INFO [2019-12-17 12:30:15,650] jepsen worker 53 - jepsen.util 3293 :ok :read [17 27] INFO [2019-12-17 12:30:15,650] jepsen worker 53 - jepsen.util 3293 :invoke :read [17 nil] INFO [2019-12-17 12:30:15,661] jepsen worker 64 - jepsen.util 2764 :ok :read [21 27] INFO [2019-12-17 12:30:15,661] jepsen worker 64 - jepsen.util 2764 :invoke :cas [21 [10 3]] INFO [2019-12-17 12:30:15,671] jepsen worker 18 - jepsen.util 3348 :ok :read [6 30] INFO [2019-12-17 12:30:15,671] jepsen worker 18 - jepsen.util 3348 :invoke :cas [6 [7 28]] INFO [2019-12-17 12:30:15,684] jepsen worker 55 - jepsen.util 3925 :fail :cas [18 [12 31]] :ValueNotSwapFrom INFO [2019-12-17 12:30:15,684] jepsen worker 55 - jepsen.util 3925 :invoke :write [18 44] INFO [2019-12-17 12:30:15,692] jepsen worker 73 - jepsen.util 3133 :fail :cas [24 [28 25]] :ValueNotSwapFrom INFO [2019-12-17 12:30:15,692] jepsen worker 73 - jepsen.util 3133 :invoke :read [24 nil] INFO [2019-12-17 12:30:15,711] jepsen worker 72 - jepsen.util 3132 :ok :write [24 44] INFO [2019-12-17 12:30:15,711] jepsen worker 72 - jepsen.util 3132 :invoke :read [24 nil] INFO [2019-12-17 12:30:15,711] jepsen worker 11 - jepsen.util 3521 :ok :write [3 12] INFO [2019-12-17 12:30:15,711] jepsen worker 11 - jepsen.util 3521 :invoke :read [3 nil] INFO [2019-12-17 12:30:15,723] jepsen worker 10 - jepsen.util 2980 :info :write [3 15] :RequestCanceledException INFO [2019-12-17 12:30:15,723] jepsen worker 89 - jepsen.util 3059 :fail :cas [29 [0 6]] :ValueNotSwapFrom INFO [2019-12-17 12:30:15,723] jepsen worker 89 - jepsen.util 3059 :invoke :write [29 37] INFO [2019-12-17 12:30:15,724] jepsen worker 15 - jepsen.util 3975 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:30:15,724] jepsen worker 15 - jepsen.util 3975 :invoke :read [5 nil] INFO [2019-12-17 12:30:15,726] jepsen worker 10 - jepsen.util 3070 :invoke :cas [3 [13 10]] INFO [2019-12-17 12:30:15,726] jepsen worker 52 - jepsen.util 3472 :ok :read [17 27] INFO [2019-12-17 12:30:15,726] jepsen worker 52 - jepsen.util 3472 :invoke :read [17 nil] INFO [2019-12-17 12:30:15,732] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18813,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:30:15,732] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18813,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:30:15,732] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18815,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:30:15,733] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18815,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:30:15,754] jepsen worker 64 - jepsen.util 2764 :fail :cas [21 [10 3]] :ValueNotSwapFrom INFO [2019-12-17 12:30:15,754] jepsen worker 64 - jepsen.util 2764 :invoke :write [21 12] INFO [2019-12-17 12:30:15,755] jepsen worker 85 - jepsen.util 4765 :info :write [28 6] :RequestCanceledException INFO [2019-12-17 12:30:15,755] jepsen worker 43 - jepsen.util 3913 :info :cas [14 [44 0]] :RequestCanceledException INFO [2019-12-17 12:30:15,755] jepsen worker 5 - jepsen.util 2885 :info :write [1 45] :RequestCanceledException INFO [2019-12-17 12:30:15,755] jepsen worker 84 - jepsen.util 4134 :info :cas [28 [5 15]] :RequestCanceledException INFO [2019-12-17 12:30:15,762] jepsen worker 5 - jepsen.util 2975 :invoke :cas [1 [48 38]] INFO [2019-12-17 12:30:15,762] jepsen worker 43 - jepsen.util 4003 :invoke :write [14 3] INFO [2019-12-17 12:30:15,762] jepsen worker 85 - jepsen.util 4855 :invoke :read [28 nil] INFO [2019-12-17 12:30:15,767] jepsen worker 84 - jepsen.util 4224 :invoke :cas [28 [45 35]] INFO [2019-12-17 12:30:15,778] jepsen worker 18 - jepsen.util 3348 :fail :cas [6 [7 28]] :ValueNotSwapFrom INFO [2019-12-17 12:30:15,779] jepsen worker 18 - jepsen.util 3348 :invoke :write [6 15] INFO [2019-12-17 12:30:15,782] jepsen worker 73 - jepsen.util 3133 :ok :read [24 46] INFO [2019-12-17 12:30:15,783] jepsen worker 73 - jepsen.util 3133 :invoke :read [24 nil] INFO [2019-12-17 12:30:15,791] jepsen worker 37 - jepsen.util 3007 :info :write [12 46] :RequestCanceledException INFO [2019-12-17 12:30:15,791] jepsen worker 20 - jepsen.util 2810 :info :write [6 42] :RequestCanceledException INFO [2019-12-17 12:30:15,793] jepsen worker 20 - jepsen.util 2900 :invoke :read [6 nil] INFO [2019-12-17 12:30:15,798] jepsen worker 37 - jepsen.util 3097 :invoke :read [12 nil] INFO [2019-12-17 12:30:15,813] jepsen worker 10 - jepsen.util 3070 :fail :cas [3 [13 10]] :ValueNotSwapFrom INFO [2019-12-17 12:30:15,813] jepsen worker 10 - jepsen.util 3070 :invoke :read [3 nil] INFO [2019-12-17 12:30:15,814] jepsen worker 72 - jepsen.util 3132 :ok :read [24 44] INFO [2019-12-17 12:30:15,814] jepsen worker 72 - jepsen.util 3132 :invoke :write [24 29] INFO [2019-12-17 12:30:15,819] jepsen worker 30 - jepsen.util 3720 :ok :read [10 46] INFO [2019-12-17 12:30:15,819] jepsen worker 30 - jepsen.util 3720 :invoke :read [10 nil] INFO [2019-12-17 12:30:15,823] jepsen worker 45 - jepsen.util 3195 :info :cas [15 [16 46]] :RequestCanceledException INFO [2019-12-17 12:30:15,826] jepsen worker 45 - jepsen.util 3285 :invoke :cas [15 [3 12]] INFO [2019-12-17 12:30:15,827] jepsen worker 11 - jepsen.util 3521 :ok :read [3 12] INFO [2019-12-17 12:30:15,827] jepsen worker 11 - jepsen.util 3521 :invoke :cas [3 [0 45]] INFO [2019-12-17 12:30:15,838] jepsen worker 50 - jepsen.util 4730 :info :cas [16 [18 22]] :RequestCanceledException INFO [2019-12-17 12:30:15,841] jepsen worker 50 - jepsen.util 4820 :invoke :cas [16 [7 7]] INFO [2019-12-17 12:30:15,844] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:30:15,844] jepsen nemesis - couchbase.util Rebalance check count: 138 INFO [2019-12-17 12:30:15,844] jepsen nemesis - couchbase.util Rebalance stuck count: 119 INFO [2019-12-17 12:30:15,853] jepsen worker 20 - jepsen.util 2900 :ok :read [6 30] INFO [2019-12-17 12:30:15,853] jepsen worker 20 - jepsen.util 2900 :invoke :write [6 44] INFO [2019-12-17 12:30:15,855] jepsen worker 27 - jepsen.util 2907 :info :cas [9 [48 46]] :RequestCanceledException INFO [2019-12-17 12:30:15,861] jepsen worker 27 - jepsen.util 2997 :invoke :cas [9 [29 14]] INFO [2019-12-17 12:30:15,868] jepsen worker 73 - jepsen.util 3133 :ok :read [24 44] INFO [2019-12-17 12:30:15,868] jepsen worker 73 - jepsen.util 3133 :invoke :write [24 41] INFO [2019-12-17 12:30:15,890] jepsen worker 19 - jepsen.util 3439 :info :cas [6 [22 0]] :RequestCanceledException INFO [2019-12-17 12:30:15,891] jepsen worker 10 - jepsen.util 3070 :ok :read [3 12] INFO [2019-12-17 12:30:15,891] jepsen worker 10 - jepsen.util 3070 :invoke :read [3 nil] INFO [2019-12-17 12:30:15,893] jepsen worker 19 - jepsen.util 3529 :invoke :read [6 nil] INFO [2019-12-17 12:30:15,908] jepsen worker 62 - jepsen.util 4022 :info :write [20 38] :RequestCanceledException INFO [2019-12-17 12:30:15,908] jepsen worker 46 - jepsen.util 3556 :info :write [15 42] :RequestCanceledException INFO [2019-12-17 12:30:15,911] jepsen worker 46 - jepsen.util 3646 :invoke :read [15 nil] INFO [2019-12-17 12:30:15,914] jepsen worker 62 - jepsen.util 4112 :invoke :cas [20 [18 29]] INFO [2019-12-17 12:30:15,923] jepsen worker 1 - jepsen.util 3781 :info :cas [0 [11 24]] :RequestCanceledException INFO [2019-12-17 12:30:15,924] jepsen worker 65 - jepsen.util 2945 :fail :read [21 nil] :RequestTimeoutException INFO [2019-12-17 12:30:15,924] jepsen worker 65 - jepsen.util 2945 :invoke :read [21 nil] INFO [2019-12-17 12:30:15,928] jepsen worker 1 - jepsen.util 3871 :invoke :cas [0 [34 27]] INFO [2019-12-17 12:30:15,936] jepsen worker 40 - jepsen.util 4630 :info :write [13 20] :RequestCanceledException INFO [2019-12-17 12:30:15,941] jepsen worker 40 - jepsen.util 4720 :invoke :read [13 nil] INFO [2019-12-17 12:30:15,950] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18844,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:30:15,950] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18844,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:30:15,950] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18847,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:30:15,950] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18847,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:30:15,950] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18848,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:30:15,952] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18848,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:30:15,955] jepsen worker 54 - jepsen.util 3384 :info :write [18 39] :RequestCanceledException INFO [2019-12-17 12:30:15,963] jepsen worker 54 - jepsen.util 3474 :invoke :write [18 22] INFO [2019-12-17 12:30:15,967] jepsen worker 30 - jepsen.util 3720 :ok :read [10 46] INFO [2019-12-17 12:30:15,968] jepsen worker 30 - jepsen.util 3720 :invoke :cas [10 [2 6]] INFO [2019-12-17 12:30:15,976] jepsen worker 13 - jepsen.util 3163 :ok :read [4 21] INFO [2019-12-17 12:30:15,976] jepsen worker 13 - jepsen.util 3163 :invoke :read [4 nil] INFO [2019-12-17 12:30:15,982] jepsen worker 11 - jepsen.util 3521 :fail :cas [3 [0 45]] :ValueNotSwapFrom INFO [2019-12-17 12:30:15,982] jepsen worker 11 - jepsen.util 3521 :invoke :cas [3 [20 3]] INFO [2019-12-17 12:30:15,985] jepsen worker 19 - jepsen.util 3529 :ok :read [6 30] INFO [2019-12-17 12:30:15,986] jepsen worker 19 - jepsen.util 3529 :invoke :write [6 6] INFO [2019-12-17 12:30:15,990] jepsen worker 64 - jepsen.util 2764 :ok :write [21 12] INFO [2019-12-17 12:30:15,990] jepsen worker 64 - jepsen.util 2764 :invoke :cas [21 [30 36]] INFO [2019-12-17 12:30:15,991] jepsen worker 21 - jepsen.util 4251 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:30:15,991] jepsen worker 21 - jepsen.util 4251 :invoke :write [7 26] INFO [2019-12-17 12:30:15,991] jepsen worker 32 - jepsen.util 3632 :info :write [10 10] :RequestCanceledException INFO [2019-12-17 12:30:15,991] jepsen worker 47 - jepsen.util 3827 :info :cas [15 [7 16]] :RequestCanceledException INFO [2019-12-17 12:30:15,996] jepsen worker 47 - jepsen.util 3917 :invoke :cas [15 [1 21]] INFO [2019-12-17 12:30:15,996] jepsen worker 32 - jepsen.util 3722 :invoke :read [10 nil] INFO [2019-12-17 12:30:15,996] jepsen worker 65 - jepsen.util 2945 :ok :read [21 27] INFO [2019-12-17 12:30:15,997] jepsen worker 65 - jepsen.util 2945 :invoke :cas [21 [27 18]] INFO [2019-12-17 12:30:16,004] jepsen worker 10 - jepsen.util 3070 :ok :read [3 12] INFO [2019-12-17 12:30:16,005] jepsen worker 10 - jepsen.util 3070 :invoke :read [3 nil] INFO [2019-12-17 12:30:16,008] jepsen worker 22 - jepsen.util 4432 :info :cas [7 [37 39]] :RequestCanceledException INFO [2019-12-17 12:30:16,011] jepsen worker 22 - jepsen.util 4522 :invoke :read [7 nil] INFO [2019-12-17 12:30:16,024] jepsen worker 75 - jepsen.util 4395 :info :write [25 20] :RequestCanceledException INFO [2019-12-17 12:30:16,026] jepsen worker 75 - jepsen.util 4485 :invoke :write [25 25] INFO [2019-12-17 12:30:16,047] jepsen worker 47 - jepsen.util 3917 :fail :cas [15 [1 21]] :ValueNotSwapFrom INFO [2019-12-17 12:30:16,047] jepsen worker 47 - jepsen.util 3917 :invoke :read [15 nil] INFO [2019-12-17 12:30:16,051] jepsen worker 42 - jepsen.util 5082 :info :cas [14 [22 11]] :RequestCanceledException INFO [2019-12-17 12:30:16,053] jepsen worker 42 - jepsen.util 5172 :invoke :write [14 2] INFO [2019-12-17 12:30:16,066] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18865,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:30:16,066] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18865,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:30:16,066] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18868,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:30:16,067] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18868,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:30:16,109] jepsen worker 83 - jepsen.util 4493 :info :cas [27 [19 27]] :RequestCanceledException INFO [2019-12-17 12:30:16,119] jepsen worker 83 - jepsen.util 4583 :invoke :read [27 nil] INFO [2019-12-17 12:30:16,123] jepsen worker 8 - jepsen.util 3698 :info :cas [2 [0 27]] :RequestCanceledException INFO [2019-12-17 12:30:16,132] jepsen worker 8 - jepsen.util 3788 :invoke :cas [2 [14 0]] INFO [2019-12-17 12:30:16,146] jepsen worker 4 - jepsen.util 2524 :ok :read [1 26] INFO [2019-12-17 12:30:16,146] jepsen worker 4 - jepsen.util 2524 :invoke :read [1 nil] INFO [2019-12-17 12:30:16,154] jepsen worker 48 - jepsen.util 3828 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:30:16,155] jepsen worker 48 - jepsen.util 3828 :invoke :read [16 nil] INFO [2019-12-17 12:30:16,173] jepsen worker 65 - jepsen.util 2945 :fail :cas [21 [27 18]] :ValueNotSwapFrom INFO [2019-12-17 12:30:16,173] jepsen worker 65 - jepsen.util 2945 :invoke :cas [21 [43 4]] INFO [2019-12-17 12:30:16,175] jepsen worker 64 - jepsen.util 2764 :fail :cas [21 [30 36]] :ValueNotSwapFrom INFO [2019-12-17 12:30:16,175] jepsen worker 64 - jepsen.util 2764 :invoke :read [21 nil] INFO [2019-12-17 12:30:16,191] jepsen worker 10 - jepsen.util 3070 :ok :read [3 12] INFO [2019-12-17 12:30:16,191] jepsen worker 70 - jepsen.util 3760 :ok :read [23 41] INFO [2019-12-17 12:30:16,191] jepsen worker 5 - jepsen.util 2975 :fail :cas [1 [48 38]] :ValueNotSwapFrom INFO [2019-12-17 12:30:16,191] jepsen worker 45 - jepsen.util 3285 :fail :cas [15 [3 12]] :ValueNotSwapFrom INFO [2019-12-17 12:30:16,191] jepsen worker 70 - jepsen.util 3760 :invoke :write [23 49] INFO [2019-12-17 12:30:16,191] jepsen worker 5 - jepsen.util 2975 :invoke :cas [1 [23 37]] INFO [2019-12-17 12:30:16,191] jepsen worker 10 - jepsen.util 3070 :invoke :write [3 28] INFO [2019-12-17 12:30:16,191] jepsen worker 26 - jepsen.util 4616 :info :write [8 17] :RequestCanceledException INFO [2019-12-17 12:30:16,191] jepsen worker 41 - jepsen.util 4361 :info :write [13 31] :RequestCanceledException INFO [2019-12-17 12:30:16,191] jepsen worker 45 - jepsen.util 3285 :invoke :read [15 nil] INFO [2019-12-17 12:30:16,196] jepsen worker 26 - jepsen.util 4706 :invoke :write [8 4] INFO [2019-12-17 12:30:16,203] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18886,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:30:16,203] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18886,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:30:16,203] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18888,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:30:16,203] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18888,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:30:16,212] jepsen worker 41 - jepsen.util 4451 :invoke :cas [13 [44 42]] INFO [2019-12-17 12:30:16,223] jepsen worker 33 - jepsen.util 4623 :info :write [11 3] :RequestCanceledException INFO [2019-12-17 12:30:16,224] jepsen worker 57 - jepsen.util 4017 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:30:16,224] jepsen worker 0 - jepsen.util 4860 :info :cas [0 [19 4]] :RequestCanceledException INFO [2019-12-17 12:30:16,224] jepsen worker 57 - jepsen.util 4017 :invoke :write [19 32] INFO [2019-12-17 12:30:16,234] jepsen worker 33 - jepsen.util 4713 :invoke :write [11 0] INFO [2019-12-17 12:30:16,234] jepsen worker 0 - jepsen.util 4950 :invoke :read [0 nil] INFO [2019-12-17 12:30:16,252] jepsen worker 64 - jepsen.util 2764 :ok :read [21 12] INFO [2019-12-17 12:30:16,252] jepsen worker 64 - jepsen.util 2764 :invoke :write [21 7] INFO [2019-12-17 12:30:16,264] jepsen worker 65 - jepsen.util 2945 :fail :cas [21 [43 4]] :ValueNotSwapFrom INFO [2019-12-17 12:30:16,265] jepsen worker 65 - jepsen.util 2945 :invoke :cas [21 [27 41]] INFO [2019-12-17 12:30:16,276] jepsen worker 45 - jepsen.util 3285 :ok :read [15 19] INFO [2019-12-17 12:30:16,276] jepsen worker 45 - jepsen.util 3285 :invoke :read [15 nil] INFO [2019-12-17 12:30:16,311] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18894,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:30:16,311] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18894,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:30:16,311] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18898,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:30:16,311] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18898,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:30:16,312] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18899,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:30:16,312] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18899,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:30:16,320] jepsen worker 65 - jepsen.util 2945 :fail :cas [21 [27 41]] :ValueNotSwapFrom INFO [2019-12-17 12:30:16,320] jepsen worker 65 - jepsen.util 2945 :invoke :write [21 10] INFO [2019-12-17 12:30:16,324] jepsen worker 13 - jepsen.util 3163 :ok :read [4 21] INFO [2019-12-17 12:30:16,324] jepsen worker 17 - jepsen.util 4157 :info :write [5 7] :RequestCanceledException INFO [2019-12-17 12:30:16,324] jepsen worker 13 - jepsen.util 3163 :invoke :cas [4 [36 37]] INFO [2019-12-17 12:30:16,328] jepsen worker 17 - jepsen.util 4247 :invoke :read [5 nil] INFO [2019-12-17 12:30:16,338] jepsen worker 56 - jepsen.util 3206 :info :write [18 12] :RequestCanceledException INFO [2019-12-17 12:30:16,338] jepsen worker 31 - jepsen.util 2731 :fail :read [10 nil] :RequestTimeoutException INFO [2019-12-17 12:30:16,338] jepsen worker 29 - jepsen.util 2729 :fail :read [9 nil] :RequestTimeoutException INFO [2019-12-17 12:30:16,338] jepsen worker 29 - jepsen.util 2729 :invoke :write [9 19] INFO [2019-12-17 12:30:16,338] jepsen worker 31 - jepsen.util 2731 :invoke :cas [10 [24 49]] INFO [2019-12-17 12:30:16,342] jepsen worker 56 - jepsen.util 3296 :invoke :write [18 10] INFO [2019-12-17 12:30:16,353] jepsen worker 45 - jepsen.util 3285 :ok :read [15 19] INFO [2019-12-17 12:30:16,353] jepsen worker 45 - jepsen.util 3285 :invoke :write [15 17] INFO [2019-12-17 12:30:16,362] jepsen worker 64 - jepsen.util 2764 :ok :write [21 7] INFO [2019-12-17 12:30:16,363] jepsen worker 64 - jepsen.util 2764 :invoke :read [21 nil] INFO [2019-12-17 12:30:16,366] jepsen worker 51 - jepsen.util 3021 :ok :read [17 27] INFO [2019-12-17 12:30:16,366] jepsen worker 51 - jepsen.util 3021 :invoke :read [17 nil] INFO [2019-12-17 12:30:16,404] jepsen worker 65 - jepsen.util 2945 :ok :write [21 10] INFO [2019-12-17 12:30:16,404] jepsen worker 65 - jepsen.util 2945 :invoke :cas [21 [46 23]] INFO [2019-12-17 12:30:16,405] jepsen worker 64 - jepsen.util 2764 :ok :read [21 7] INFO [2019-12-17 12:30:16,405] jepsen worker 64 - jepsen.util 2764 :invoke :write [21 36] INFO [2019-12-17 12:30:16,424] jepsen worker 63 - jepsen.util 2943 :fail :read [21 nil] :RequestTimeoutException INFO [2019-12-17 12:30:16,424] jepsen worker 63 - jepsen.util 2943 :invoke :write [21 20] INFO [2019-12-17 12:30:16,460] jepsen worker 65 - jepsen.util 2945 :fail :cas [21 [46 23]] :ValueNotSwapFrom INFO [2019-12-17 12:30:16,460] jepsen worker 65 - jepsen.util 2945 :invoke :write [21 32] INFO [2019-12-17 12:30:16,490] jepsen worker 14 - jepsen.util 3164 :info :write [4 49] :RequestCanceledException INFO [2019-12-17 12:30:16,493] jepsen worker 14 - jepsen.util 3254 :invoke :cas [4 [30 3]] INFO [2019-12-17 12:30:16,495] jepsen worker 64 - jepsen.util 2764 :ok :write [21 36] INFO [2019-12-17 12:30:16,496] jepsen worker 64 - jepsen.util 2764 :invoke :write [21 29] INFO [2019-12-17 12:30:16,511] jepsen worker 4 - jepsen.util 2524 :ok :read [1 26] INFO [2019-12-17 12:30:16,511] jepsen worker 4 - jepsen.util 2524 :invoke :cas [1 [27 48]] INFO [2019-12-17 12:30:16,520] jepsen worker 37 - jepsen.util 3097 :ok :read [12 20] INFO [2019-12-17 12:30:16,520] jepsen worker 37 - jepsen.util 3097 :invoke :read [12 nil] INFO [2019-12-17 12:30:16,527] jepsen worker 38 - jepsen.util 3188 :fail :cas [12 [27 40]] :ValueNotSwapFrom INFO [2019-12-17 12:30:16,527] jepsen worker 38 - jepsen.util 3188 :invoke :write [12 34] INFO [2019-12-17 12:30:16,530] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18922,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:30:16,530] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18922,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:30:16,530] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18926,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:30:16,530] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18926,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:30:16,537] jepsen worker 88 - jepsen.util 3238 :info :write [29 19] :RequestCanceledException INFO [2019-12-17 12:30:16,540] jepsen worker 88 - jepsen.util 3328 :invoke :cas [29 [40 47]] INFO [2019-12-17 12:30:16,553] jepsen worker 5 - jepsen.util 2975 :fail :cas [1 [23 37]] :ValueNotSwapFrom INFO [2019-12-17 12:30:16,553] jepsen worker 5 - jepsen.util 2975 :invoke :read [1 nil] INFO [2019-12-17 12:30:16,568] jepsen worker 64 - jepsen.util 2764 :ok :write [21 29] INFO [2019-12-17 12:30:16,607] jepsen worker 25 - jepsen.util 4705 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:30:16,612] jepsen worker 64 - jepsen.util 2764 :invoke :write [21 33] INFO [2019-12-17 12:30:16,612] jepsen worker 25 - jepsen.util 4705 :invoke :read [8 nil] INFO [2019-12-17 12:30:16,624] jepsen worker 81 - jepsen.util 4221 :info :write [27 11] :RequestCanceledException INFO [2019-12-17 12:30:16,625] jepsen worker 87 - jepsen.util 3057 :fail :cas [29 [1 11]] :ValueNotSwapFrom INFO [2019-12-17 12:30:16,628] jepsen worker 53 - jepsen.util 3293 :ok :read [17 27] INFO [2019-12-17 12:30:16,638] jepsen worker 76 - jepsen.util 4486 :info :write [25 7] :RequestCanceledException INFO [2019-12-17 12:30:16,653] jepsen worker 87 - jepsen.util 3057 :invoke :cas [29 [24 5]] INFO [2019-12-17 12:30:16,653] jepsen worker 53 - jepsen.util 3293 :invoke :write [17 1] INFO [2019-12-17 12:30:16,654] jepsen worker 61 - jepsen.util 4921 :info :cas [20 [36 5]] :RequestCanceledException INFO [2019-12-17 12:30:16,657] jepsen worker 81 - jepsen.util 4311 :invoke :cas [27 [11 10]] INFO [2019-12-17 12:30:16,657] jepsen worker 76 - jepsen.util 4576 :invoke :read [25 nil] INFO [2019-12-17 12:30:16,657] jepsen worker 61 - jepsen.util 5011 :invoke :cas [20 [14 36]] INFO [2019-12-17 12:30:16,683] jepsen worker 64 - jepsen.util 2764 :ok :write [21 33] INFO [2019-12-17 12:30:16,683] jepsen worker 64 - jepsen.util 2764 :invoke :read [21 nil] INFO [2019-12-17 12:30:16,697] jepsen worker 13 - jepsen.util 3163 :fail :cas [4 [36 37]] :ValueNotSwapFrom INFO [2019-12-17 12:30:16,697] jepsen worker 13 - jepsen.util 3163 :invoke :cas [4 [7 13]] INFO [2019-12-17 12:30:16,706] jepsen worker 9 - jepsen.util 3789 :info :write [3 18] :RequestCanceledException INFO [2019-12-17 12:30:16,709] jepsen worker 9 - jepsen.util 3879 :invoke :read [3 nil] INFO [2019-12-17 12:30:16,717] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18939,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:30:16,717] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18939,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:30:16,717] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18942,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:30:16,717] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18942,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:30:16,718] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18943,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:30:16,718] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18943,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:30:16,720] jepsen worker 52 - jepsen.util 3472 :ok :read [17 27] INFO [2019-12-17 12:30:16,720] jepsen worker 52 - jepsen.util 3472 :invoke :read [17 nil] INFO [2019-12-17 12:30:16,724] jepsen worker 58 - jepsen.util 4198 :info :write [19 19] :RequestCanceledException INFO [2019-12-17 12:30:16,724] jepsen worker 86 - jepsen.util 4496 :info :write [28 20] :RequestCanceledException INFO [2019-12-17 12:30:16,726] jepsen worker 58 - jepsen.util 4288 :invoke :write [19 10] INFO [2019-12-17 12:30:16,727] jepsen worker 86 - jepsen.util 4586 :invoke :write [28 34] INFO [2019-12-17 12:30:16,746] jepsen worker 64 - jepsen.util 2764 :ok :read [21 33] INFO [2019-12-17 12:30:16,746] jepsen worker 64 - jepsen.util 2764 :invoke :cas [21 [21 43]] INFO [2019-12-17 12:30:16,772] jepsen worker 9 - jepsen.util 3879 :ok :read [3 12] INFO [2019-12-17 12:30:16,772] jepsen worker 9 - jepsen.util 3879 :invoke :cas [3 [44 19]] INFO [2019-12-17 12:30:16,781] jepsen worker 51 - jepsen.util 3021 :ok :read [17 27] INFO [2019-12-17 12:30:16,781] jepsen worker 51 - jepsen.util 3021 :invoke :read [17 nil] INFO [2019-12-17 12:30:16,785] jepsen worker 82 - jepsen.util 5212 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:30:16,799] jepsen worker 14 - jepsen.util 3254 :fail :cas [4 [30 3]] :ValueNotSwapFrom INFO [2019-12-17 12:30:16,808] jepsen worker 77 - jepsen.util 4037 :info :cas [25 [44 40]] :RequestCanceledException INFO [2019-12-17 12:30:16,817] jepsen worker 64 - jepsen.util 2764 :fail :cas [21 [21 43]] :ValueNotSwapFrom INFO [2019-12-17 12:30:16,823] jepsen worker 2 - jepsen.util 3962 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:30:16,837] jepsen worker 9 - jepsen.util 3879 :fail :cas [3 [44 19]] :ValueNotSwapFrom INFO [2019-12-17 12:30:16,838] jepsen worker 24 - jepsen.util 4164 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:30:16,838] jepsen worker 68 - jepsen.util 4028 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:30:16,838] jepsen worker 66 - jepsen.util 4206 :info :write [22 7] :RequestCanceledException INFO [2019-12-17 12:30:16,856] jepsen worker 39 - jepsen.util 4089 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:30:16,862] jepsen worker 37 - jepsen.util 3097 :ok :read [12 20] INFO [2019-12-17 12:30:16,862] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:30:16,870] jepsen worker 82 - jepsen.util 5212 :invoke :cas [27 [10 24]] INFO [2019-12-17 12:30:16,870] jepsen worker 14 - jepsen.util 3254 :invoke :read [4 nil] INFO [2019-12-17 12:30:16,870] jepsen worker 64 - jepsen.util 2764 :invoke :write [21 14] INFO [2019-12-17 12:30:16,870] jepsen worker 2 - jepsen.util 3962 :invoke :write [0 42] INFO [2019-12-17 12:30:16,870] jepsen worker 9 - jepsen.util 3879 :invoke :cas [3 [1 13]] INFO [2019-12-17 12:30:16,870] jepsen worker 68 - jepsen.util 4028 :invoke :read [22 nil] INFO [2019-12-17 12:30:16,870] jepsen worker 24 - jepsen.util 4164 :invoke :cas [8 [5 0]] INFO [2019-12-17 12:30:16,871] jepsen worker 39 - jepsen.util 4089 :invoke :read [13 nil] INFO [2019-12-17 12:30:16,871] jepsen nemesis - couchbase.util Rebalance check count: 139 INFO [2019-12-17 12:30:16,871] jepsen worker 37 - jepsen.util 3097 :invoke :cas [12 [46 18]] INFO [2019-12-17 12:30:16,871] jepsen nemesis - couchbase.util Rebalance stuck count: 120 INFO [2019-12-17 12:30:16,873] jepsen worker 77 - jepsen.util 4127 :invoke :cas [25 [22 25]] INFO [2019-12-17 12:30:16,873] jepsen worker 66 - jepsen.util 4296 :invoke :write [22 4] INFO [2019-12-17 12:30:16,890] jepsen worker 44 - jepsen.util 4634 :info :cas [14 [49 4]] :RequestCanceledException INFO [2019-12-17 12:30:16,894] jepsen worker 44 - jepsen.util 4724 :invoke :cas [14 [43 23]] INFO [2019-12-17 12:30:16,909] jepsen worker 6 - jepsen.util 4596 :info :cas [2 [38 6]] :RequestCanceledException INFO [2019-12-17 12:30:16,909] jepsen worker 69 - jepsen.util 3219 :info :write [23 45] :RequestCanceledException INFO [2019-12-17 12:30:16,909] jepsen worker 67 - jepsen.util 4747 :info :cas [22 [48 9]] :RequestCanceledException INFO [2019-12-17 12:30:16,911] jepsen worker 69 - jepsen.util 3309 :invoke :cas [23 [7 7]] INFO [2019-12-17 12:30:16,912] jepsen worker 67 - jepsen.util 4837 :invoke :read [22 nil] INFO [2019-12-17 12:30:16,913] jepsen worker 6 - jepsen.util 4686 :invoke :read [2 nil] INFO [2019-12-17 12:30:16,914] jepsen worker 9 - jepsen.util 3879 :fail :cas [3 [1 13]] :ValueNotSwapFrom INFO [2019-12-17 12:30:16,914] jepsen worker 9 - jepsen.util 3879 :invoke :write [3 20] INFO [2019-12-17 12:30:16,931] jepsen worker 64 - jepsen.util 2764 :ok :write [21 14] INFO [2019-12-17 12:30:16,931] jepsen worker 64 - jepsen.util 2764 :invoke :cas [21 [33 44]] INFO [2019-12-17 12:30:16,937] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18959,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:30:16,937] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18959,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:30:16,937] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18962,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:30:16,937] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18962,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:30:16,938] jepsen worker 49 - jepsen.util 3829 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:30:16,938] jepsen worker 49 - jepsen.util 3829 :invoke :write [16 49] INFO [2019-12-17 12:30:16,995] jepsen worker 64 - jepsen.util 2764 :fail :cas [21 [33 44]] :ValueNotSwapFrom INFO [2019-12-17 12:30:16,996] jepsen worker 64 - jepsen.util 2764 :invoke :write [21 14] INFO [2019-12-17 12:30:17,001] jepsen worker 88 - jepsen.util 3328 :fail :cas [29 [40 47]] :ValueNotSwapFrom INFO [2019-12-17 12:30:17,001] jepsen worker 88 - jepsen.util 3328 :invoke :cas [29 [48 19]] INFO [2019-12-17 12:30:17,008] jepsen worker 69 - jepsen.util 3309 :fail :cas [23 [7 7]] :ValueNotSwapFrom INFO [2019-12-17 12:30:17,008] jepsen worker 69 - jepsen.util 3309 :invoke :cas [23 [32 10]] INFO [2019-12-17 12:30:17,024] jepsen worker 60 - jepsen.util 4560 :info :write [20 41] :RequestCanceledException INFO [2019-12-17 12:30:17,029] jepsen worker 60 - jepsen.util 4650 :invoke :cas [20 [11 8]] INFO [2019-12-17 12:30:17,031] jepsen worker 87 - jepsen.util 3057 :fail :cas [29 [24 5]] :ValueNotSwapFrom INFO [2019-12-17 12:30:17,031] jepsen worker 87 - jepsen.util 3057 :invoke :write [29 30] INFO [2019-12-17 12:30:17,038] jepsen worker 12 - jepsen.util 3432 :info :write [4 48] :RequestCanceledException INFO [2019-12-17 12:30:17,041] jepsen worker 12 - jepsen.util 3522 :invoke :write [4 2] INFO [2019-12-17 12:30:17,053] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18983,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:30:17,053] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18983,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:30:17,053] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18987,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:30:17,054] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":18987,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:30:17,055] jepsen worker 71 - jepsen.util 3221 :info :write [23 38] :RequestCanceledException INFO [2019-12-17 12:30:17,057] jepsen worker 71 - jepsen.util 3311 :invoke :write [23 3] INFO [2019-12-17 12:30:17,070] jepsen worker 64 - jepsen.util 2764 :ok :write [21 14] INFO [2019-12-17 12:30:17,071] jepsen worker 64 - jepsen.util 2764 :invoke :write [21 38] INFO [2019-12-17 12:30:17,078] jepsen worker 52 - jepsen.util 3472 :ok :read [17 27] INFO [2019-12-17 12:30:17,079] jepsen worker 52 - jepsen.util 3472 :invoke :read [17 nil] INFO [2019-12-17 12:30:17,093] jepsen worker 13 - jepsen.util 3163 :fail :cas [4 [7 13]] :ValueNotSwapFrom INFO [2019-12-17 12:30:17,093] jepsen worker 13 - jepsen.util 3163 :invoke :cas [4 [19 21]] INFO [2019-12-17 12:30:17,108] jepsen worker 34 - jepsen.util 4354 :info :write [11 2] :RequestCanceledException INFO [2019-12-17 12:30:17,114] jepsen worker 34 - jepsen.util 4444 :invoke :read [11 nil] INFO [2019-12-17 12:30:17,126] jepsen worker 64 - jepsen.util 2764 :ok :write [21 38] INFO [2019-12-17 12:30:17,127] jepsen worker 64 - jepsen.util 2764 :invoke :cas [21 [46 33]] INFO [2019-12-17 12:30:17,140] jepsen worker 14 - jepsen.util 3254 :ok :read [4 21] INFO [2019-12-17 12:30:17,141] jepsen worker 14 - jepsen.util 3254 :invoke :write [4 48] INFO [2019-12-17 12:30:17,176] jepsen worker 64 - jepsen.util 2764 :fail :cas [21 [46 33]] :ValueNotSwapFrom INFO [2019-12-17 12:30:17,176] jepsen worker 64 - jepsen.util 2764 :invoke :cas [21 [22 16]] INFO [2019-12-17 12:30:17,181] jepsen worker 69 - jepsen.util 3309 :fail :cas [23 [32 10]] :ValueNotSwapFrom INFO [2019-12-17 12:30:17,182] jepsen worker 69 - jepsen.util 3309 :invoke :cas [23 [13 48]] INFO [2019-12-17 12:30:17,191] jepsen worker 45 - jepsen.util 3285 :ok :write [15 17] INFO [2019-12-17 12:30:17,191] jepsen worker 45 - jepsen.util 3285 :invoke :write [15 36] INFO [2019-12-17 12:30:17,219] jepsen worker 37 - jepsen.util 3097 :fail :cas [12 [46 18]] :ValueNotSwapFrom INFO [2019-12-17 12:30:17,220] jepsen worker 37 - jepsen.util 3097 :invoke :write [12 30] INFO [2019-12-17 12:30:17,237] jepsen worker 7 - jepsen.util 4147 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:30:17,238] jepsen worker 7 - jepsen.util 4147 :invoke :write [2 41] INFO [2019-12-17 12:30:17,253] jepsen worker 64 - jepsen.util 2764 :fail :cas [21 [22 16]] :ValueNotSwapFrom INFO [2019-12-17 12:30:17,253] jepsen worker 64 - jepsen.util 2764 :invoke :read [21 nil] INFO [2019-12-17 12:30:17,261] jepsen worker 88 - jepsen.util 3328 :fail :cas [29 [48 19]] :ValueNotSwapFrom INFO [2019-12-17 12:30:17,262] jepsen worker 88 - jepsen.util 3328 :invoke :cas [29 [36 13]] INFO [2019-12-17 12:30:17,297] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19004,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:30:17,297] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19012,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:30:17,297] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19004,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:30:17,298] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19012,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:30:17,298] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19005,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:30:17,298] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19005,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:30:17,305] jepsen worker 23 - jepsen.util 4163 :info :cas [7 [9 23]] :RequestCanceledException INFO [2019-12-17 12:30:17,306] jepsen worker 64 - jepsen.util 2764 :ok :read [21 38] INFO [2019-12-17 12:30:17,306] jepsen worker 64 - jepsen.util 2764 :invoke :cas [21 [26 7]] INFO [2019-12-17 12:30:17,308] jepsen worker 23 - jepsen.util 4253 :invoke :cas [7 [47 15]] INFO [2019-12-17 12:30:17,324] jepsen worker 36 - jepsen.util 3366 :info :write [12 36] :RequestCanceledException INFO [2019-12-17 12:30:17,325] jepsen worker 36 - jepsen.util 3456 :invoke :write [12 45] INFO [2019-12-17 12:30:17,348] jepsen worker 64 - jepsen.util 2764 :fail :cas [21 [26 7]] :ValueNotSwapFrom INFO [2019-12-17 12:30:17,348] jepsen worker 64 - jepsen.util 2764 :invoke :write [21 31] INFO [2019-12-17 12:30:17,362] jepsen worker 52 - jepsen.util 3472 :ok :read [17 27] INFO [2019-12-17 12:30:17,362] jepsen worker 52 - jepsen.util 3472 :invoke :write [17 45] INFO [2019-12-17 12:30:17,410] jepsen worker 64 - jepsen.util 2764 :ok :write [21 31] INFO [2019-12-17 12:30:17,410] jepsen worker 64 - jepsen.util 2764 :invoke :read [21 nil] INFO [2019-12-17 12:30:17,436] jepsen worker 28 - jepsen.util 2368 :info :cas [9 [10 40]] :RequestCanceledException INFO [2019-12-17 12:30:17,438] jepsen worker 28 - jepsen.util 2458 :invoke :read [9 nil] INFO [2019-12-17 12:30:17,455] jepsen worker 79 - jepsen.util 3229 :info :write [26 19] :RequestCanceledException INFO [2019-12-17 12:30:17,457] jepsen worker 79 - jepsen.util 3319 :invoke :write [26 31] INFO [2019-12-17 12:30:17,459] jepsen worker 64 - jepsen.util 2764 :ok :read [21 31] INFO [2019-12-17 12:30:17,459] jepsen worker 64 - jepsen.util 2764 :invoke :read [21 nil] INFO [2019-12-17 12:30:17,486] jepsen worker 51 - jepsen.util 3021 :ok :read [17 27] INFO [2019-12-17 12:30:17,486] jepsen worker 51 - jepsen.util 3021 :invoke :cas [17 [17 25]] INFO [2019-12-17 12:30:17,500] jepsen worker 4 - jepsen.util 2524 :fail :cas [1 [27 48]] :ValueNotSwapFrom INFO [2019-12-17 12:30:17,500] jepsen worker 4 - jepsen.util 2524 :invoke :write [1 12] INFO [2019-12-17 12:30:17,501] jepsen worker 64 - jepsen.util 2764 :ok :read [21 31] INFO [2019-12-17 12:30:17,501] jepsen worker 64 - jepsen.util 2764 :invoke :write [21 25] INFO [2019-12-17 12:30:17,518] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19027,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:30:17,518] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19027,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:30:17,518] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19028,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:30:17,518] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19028,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:30:17,538] jepsen worker 78 - jepsen.util 3588 :info :write [26 41] :RequestCanceledException INFO [2019-12-17 12:30:17,542] jepsen worker 78 - jepsen.util 3678 :invoke :write [26 45] INFO [2019-12-17 12:30:17,568] jepsen worker 5 - jepsen.util 2975 :ok :read [1 26] INFO [2019-12-17 12:30:17,568] jepsen worker 5 - jepsen.util 2975 :invoke :cas [1 [8 30]] INFO [2019-12-17 12:30:17,589] jepsen worker 64 - jepsen.util 2764 :ok :write [21 25] INFO [2019-12-17 12:30:17,589] jepsen worker 64 - jepsen.util 2764 :invoke :cas [21 [40 46]] INFO [2019-12-17 12:30:17,610] jepsen worker 88 - jepsen.util 3328 :fail :cas [29 [36 13]] :ValueNotSwapFrom INFO [2019-12-17 12:30:17,610] jepsen worker 88 - jepsen.util 3328 :invoke :read [29 nil] INFO [2019-12-17 12:30:17,623] jepsen worker 35 - jepsen.util 4175 :info :cas [11 [41 32]] :RequestCanceledException INFO [2019-12-17 12:30:17,624] jepsen worker 80 - jepsen.util 3140 :info :write [26 26] :RequestCanceledException INFO [2019-12-17 12:30:17,626] jepsen worker 35 - jepsen.util 4265 :invoke :read [11 nil] INFO [2019-12-17 12:30:17,628] jepsen worker 80 - jepsen.util 3230 :invoke :cas [26 [24 48]] INFO [2019-12-17 12:30:17,629] jepsen worker 64 - jepsen.util 2764 :fail :cas [21 [40 46]] :ValueNotSwapFrom INFO [2019-12-17 12:30:17,629] jepsen worker 64 - jepsen.util 2764 :invoke :read [21 nil] INFO [2019-12-17 12:30:17,653] jepsen worker 16 - jepsen.util 4696 :info :write [5 34] :RequestCanceledException INFO [2019-12-17 12:30:17,655] jepsen worker 16 - jepsen.util 4786 :invoke :cas [5 [9 32]] INFO [2019-12-17 12:30:17,660] jepsen worker 64 - jepsen.util 2764 :ok :read [21 25] INFO [2019-12-17 12:30:17,660] jepsen worker 64 - jepsen.util 2764 :invoke :cas [21 [17 34]] INFO [2019-12-17 12:30:17,688] jepsen worker 3 - jepsen.util 3333 :info :write [1 8] :RequestCanceledException INFO [2019-12-17 12:30:17,688] jepsen worker 59 - jepsen.util 4379 :info :write [19 0] :RequestCanceledException INFO [2019-12-17 12:30:17,695] jepsen worker 64 - jepsen.util 2764 :fail :cas [21 [17 34]] :ValueNotSwapFrom INFO [2019-12-17 12:30:17,698] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19048,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:30:17,698] jepsen worker 64 - jepsen.util 2764 :invoke :cas [21 [4 29]] INFO [2019-12-17 12:30:17,698] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19048,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:30:17,698] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19049,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:30:17,698] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19049,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:30:17,701] jepsen worker 3 - jepsen.util 3423 :invoke :cas [1 [20 48]] INFO [2019-12-17 12:30:17,701] jepsen worker 59 - jepsen.util 4469 :invoke :read [19 nil] INFO [2019-12-17 12:30:17,724] jepsen worker 74 - jepsen.util 3494 :info :write [24 37] :RequestCanceledException INFO [2019-12-17 12:30:17,731] jepsen worker 72 - jepsen.util 3132 :info :write [24 29] :SyncWriteAmbiguous INFO [2019-12-17 12:30:17,732] jepsen worker 74 - jepsen.util 3584 :invoke :write [24 39] INFO [2019-12-17 12:30:17,733] jepsen worker 72 - jepsen.util 3222 :invoke :read [24 nil] INFO [2019-12-17 12:30:17,760] jepsen worker 64 - jepsen.util 2764 :fail :cas [21 [4 29]] :ValueNotSwapFrom INFO [2019-12-17 12:30:17,760] jepsen worker 64 - jepsen.util 2764 :invoke :write [21 21] INFO [2019-12-17 12:30:17,776] jepsen worker 31 - jepsen.util 2731 :fail :cas [10 [24 49]] :ValueNotSwapFrom INFO [2019-12-17 12:30:17,776] jepsen worker 31 - jepsen.util 2731 :invoke :read [10 nil] INFO [2019-12-17 12:30:17,778] jepsen worker 29 - jepsen.util 2729 :ok :write [9 19] INFO [2019-12-17 12:30:17,779] jepsen worker 29 - jepsen.util 2729 :invoke :write [9 37] INFO [2019-12-17 12:30:17,790] jepsen worker 72 - jepsen.util 3222 :ok :read [24 44] INFO [2019-12-17 12:30:17,790] jepsen worker 72 - jepsen.util 3222 :invoke :write [24 47] INFO [2019-12-17 12:30:17,791] jepsen worker 55 - jepsen.util 3925 :info :write [18 44] :RequestCanceledException INFO [2019-12-17 12:30:17,791] jepsen worker 89 - jepsen.util 3059 :info :write [29 37] :RequestCanceledException INFO [2019-12-17 12:30:17,793] jepsen worker 55 - jepsen.util 4015 :invoke :write [18 10] INFO [2019-12-17 12:30:17,797] jepsen worker 89 - jepsen.util 3149 :invoke :cas [29 [17 18]] INFO [2019-12-17 12:30:17,828] jepsen worker 20 - jepsen.util 2900 :info :write [6 44] :SyncWriteAmbiguous INFO [2019-12-17 12:30:17,828] jepsen worker 15 - jepsen.util 3975 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:30:17,828] jepsen worker 18 - jepsen.util 3348 :info :write [6 15] :RequestCanceledException INFO [2019-12-17 12:30:17,828] jepsen worker 84 - jepsen.util 4224 :info :cas [28 [45 35]] :RequestCanceledException INFO [2019-12-17 12:30:17,828] jepsen worker 15 - jepsen.util 3975 :invoke :cas [5 [25 18]] INFO [2019-12-17 12:30:17,834] jepsen worker 20 - jepsen.util 2990 :invoke :cas [6 [12 28]] INFO [2019-12-17 12:30:17,834] jepsen worker 18 - jepsen.util 3438 :invoke :read [6 nil] INFO [2019-12-17 12:30:17,835] jepsen worker 84 - jepsen.util 4314 :invoke :read [28 nil] INFO [2019-12-17 12:30:17,838] jepsen worker 43 - jepsen.util 4003 :info :write [14 3] :RequestCanceledException INFO [2019-12-17 12:30:17,841] jepsen worker 31 - jepsen.util 2731 :ok :read [10 46] INFO [2019-12-17 12:30:17,841] jepsen worker 31 - jepsen.util 2731 :invoke :write [10 4] INFO [2019-12-17 12:30:17,842] jepsen worker 43 - jepsen.util 4093 :invoke :cas [14 [39 28]] INFO [2019-12-17 12:30:17,855] jepsen worker 85 - jepsen.util 4855 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:30:17,856] jepsen worker 85 - jepsen.util 4855 :invoke :write [28 18] INFO [2019-12-17 12:30:17,892] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:30:17,893] jepsen nemesis - couchbase.util Rebalance check count: 140 INFO [2019-12-17 12:30:17,893] jepsen nemesis - couchbase.util Rebalance stuck count: 121 INFO [2019-12-17 12:30:17,902] jepsen worker 80 - jepsen.util 3230 :fail :cas [26 [24 48]] :ValueNotSwapFrom INFO [2019-12-17 12:30:17,902] jepsen worker 80 - jepsen.util 3230 :invoke :read [26 nil] INFO [2019-12-17 12:30:17,921] jepsen worker 29 - jepsen.util 2729 :ok :write [9 37] INFO [2019-12-17 12:30:17,921] jepsen worker 29 - jepsen.util 2729 :invoke :cas [9 [35 24]] INFO [2019-12-17 12:30:17,924] jepsen worker 30 - jepsen.util 3720 :fail :cas [10 [2 6]] :ValueNotSwapFrom INFO [2019-12-17 12:30:17,925] jepsen worker 30 - jepsen.util 3720 :invoke :write [10 32] INFO [2019-12-17 12:30:17,938] jepsen worker 50 - jepsen.util 4820 :info :cas [16 [7 7]] :RequestCanceledException INFO [2019-12-17 12:30:17,938] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19070,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:30:17,938] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19070,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:30:17,938] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19072,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:30:17,938] jepsen worker 11 - jepsen.util 3521 :fail :cas [3 [20 3]] :ValueNotSwapFrom INFO [2019-12-17 12:30:17,938] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19072,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:30:17,939] jepsen worker 11 - jepsen.util 3521 :invoke :cas [3 [36 32]] INFO [2019-12-17 12:30:17,939] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19073,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:30:17,939] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19073,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:30:17,942] jepsen worker 32 - jepsen.util 3722 :ok :read [10 46] INFO [2019-12-17 12:30:17,942] jepsen worker 32 - jepsen.util 3722 :invoke :read [10 nil] INFO [2019-12-17 12:30:17,942] jepsen worker 50 - jepsen.util 4910 :invoke :read [16 nil] INFO [2019-12-17 12:30:17,946] jepsen worker 3 - jepsen.util 3423 :fail :cas [1 [20 48]] :ValueNotSwapFrom INFO [2019-12-17 12:30:17,946] jepsen worker 3 - jepsen.util 3423 :invoke :write [1 39] INFO [2019-12-17 12:30:17,955] jepsen worker 27 - jepsen.util 2997 :info :cas [9 [29 14]] :RequestCanceledException INFO [2019-12-17 12:30:17,955] jepsen worker 73 - jepsen.util 3133 :info :write [24 41] :RequestCanceledException INFO [2019-12-17 12:30:17,957] jepsen worker 27 - jepsen.util 3087 :invoke :cas [9 [27 48]] INFO [2019-12-17 12:30:17,959] jepsen worker 73 - jepsen.util 3223 :invoke :write [24 29] INFO [2019-12-17 12:30:17,991] jepsen worker 46 - jepsen.util 3646 :fail :read [15 nil] :RequestTimeoutException INFO [2019-12-17 12:30:17,991] jepsen worker 46 - jepsen.util 3646 :invoke :read [15 nil] INFO [2019-12-17 12:30:18,001] jepsen worker 11 - jepsen.util 3521 :fail :cas [3 [36 32]] :ValueNotSwapFrom INFO [2019-12-17 12:30:18,001] jepsen worker 11 - jepsen.util 3521 :invoke :write [3 9] INFO [2019-12-17 12:30:18,008] jepsen worker 62 - jepsen.util 4112 :info :cas [20 [18 29]] :RequestCanceledException INFO [2019-12-17 12:30:18,013] jepsen worker 62 - jepsen.util 4202 :invoke :write [20 19] INFO [2019-12-17 12:30:18,013] jepsen worker 32 - jepsen.util 3722 :ok :read [10 46] INFO [2019-12-17 12:30:18,014] jepsen worker 32 - jepsen.util 3722 :invoke :write [10 47] INFO [2019-12-17 12:30:18,024] jepsen worker 19 - jepsen.util 3529 :info :write [6 6] :RequestCanceledException INFO [2019-12-17 12:30:18,024] jepsen worker 1 - jepsen.util 3871 :info :cas [0 [34 27]] :RequestCanceledException INFO [2019-12-17 12:30:18,029] jepsen worker 19 - jepsen.util 3619 :invoke :cas [6 [2 0]] INFO [2019-12-17 12:30:18,030] jepsen worker 1 - jepsen.util 3961 :invoke :write [0 11] INFO [2019-12-17 12:30:18,037] jepsen worker 40 - jepsen.util 4720 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:30:18,038] jepsen worker 40 - jepsen.util 4720 :invoke :cas [13 [2 12]] INFO [2019-12-17 12:30:18,040] jepsen worker 46 - jepsen.util 3646 :ok :read [15 17] INFO [2019-12-17 12:30:18,047] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19090,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:30:18,051] jepsen worker 54 - jepsen.util 3474 :info :write [18 22] :RequestCanceledException INFO [2019-12-17 12:30:18,061] jepsen worker 13 - jepsen.util 3163 :fail :cas [4 [19 21]] :ValueNotSwapFrom INFO [2019-12-17 12:30:18,066] jepsen worker 89 - jepsen.util 3149 :fail :cas [29 [17 18]] :ValueNotSwapFrom INFO [2019-12-17 12:30:18,072] jepsen worker 10 - jepsen.util 3070 :info :write [3 28] :SyncWriteAmbiguous INFO [2019-12-17 12:30:18,090] jepsen worker 21 - jepsen.util 4251 :info :write [7 26] :RequestCanceledException INFO [2019-12-17 12:30:18,107] jepsen worker 22 - jepsen.util 4522 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:30:18,111] jepsen worker 51 - jepsen.util 3021 :fail :cas [17 [17 25]] :ValueNotSwapFrom INFO [2019-12-17 12:30:18,113] jepsen worker 46 - jepsen.util 3646 :invoke :write [15 31] INFO [2019-12-17 12:30:18,113] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19090,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:30:18,113] jepsen worker 89 - jepsen.util 3149 :invoke :write [29 34] INFO [2019-12-17 12:30:18,113] jepsen worker 13 - jepsen.util 3163 :invoke :write [4 15] INFO [2019-12-17 12:30:18,113] jepsen worker 22 - jepsen.util 4522 :invoke :write [7 35] INFO [2019-12-17 12:30:18,113] jepsen worker 51 - jepsen.util 3021 :invoke :cas [17 [13 15]] INFO [2019-12-17 12:30:18,116] jepsen worker 10 - jepsen.util 3160 :invoke :read [3 nil] INFO [2019-12-17 12:30:18,116] jepsen worker 21 - jepsen.util 4341 :invoke :write [7 13] INFO [2019-12-17 12:30:18,116] jepsen worker 54 - jepsen.util 3564 :invoke :write [18 33] INFO [2019-12-17 12:30:18,121] jepsen worker 27 - jepsen.util 3087 :fail :cas [9 [27 48]] :ValueNotSwapFrom INFO [2019-12-17 12:30:18,121] jepsen worker 27 - jepsen.util 3087 :invoke :cas [9 [8 16]] INFO [2019-12-17 12:30:18,123] jepsen worker 47 - jepsen.util 3917 :fail :read [15 nil] :RequestTimeoutException INFO [2019-12-17 12:30:18,123] jepsen worker 75 - jepsen.util 4485 :info :write [25 25] :RequestCanceledException INFO [2019-12-17 12:30:18,123] jepsen worker 47 - jepsen.util 3917 :invoke :read [15 nil] INFO [2019-12-17 12:30:18,128] jepsen worker 75 - jepsen.util 4575 :invoke :read [25 nil] INFO [2019-12-17 12:30:18,138] jepsen worker 42 - jepsen.util 5172 :info :write [14 2] :RequestCanceledException INFO [2019-12-17 12:30:18,145] jepsen worker 42 - jepsen.util 5262 :invoke :write [14 23] INFO [2019-12-17 12:30:18,155] jepsen worker 69 - jepsen.util 3309 :fail :cas [23 [13 48]] :ValueNotSwapFrom INFO [2019-12-17 12:30:18,155] jepsen worker 83 - jepsen.util 4583 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:30:18,155] jepsen worker 69 - jepsen.util 3309 :invoke :cas [23 [34 38]] INFO [2019-12-17 12:30:18,156] jepsen worker 83 - jepsen.util 4583 :invoke :cas [27 [34 42]] INFO [2019-12-17 12:30:18,179] jepsen worker 10 - jepsen.util 3160 :ok :read [3 12] INFO [2019-12-17 12:30:18,179] jepsen worker 10 - jepsen.util 3160 :invoke :cas [3 [26 13]] INFO [2019-12-17 12:30:18,184] jepsen worker 80 - jepsen.util 3230 :ok :read [26 3] INFO [2019-12-17 12:30:18,184] jepsen worker 80 - jepsen.util 3230 :invoke :read [26 nil] INFO [2019-12-17 12:30:18,199] jepsen worker 47 - jepsen.util 3917 :ok :read [15 17] INFO [2019-12-17 12:30:18,200] jepsen worker 47 - jepsen.util 3917 :invoke :write [15 33] INFO [2019-12-17 12:30:18,204] jepsen worker 27 - jepsen.util 3087 :fail :cas [9 [8 16]] :ValueNotSwapFrom INFO [2019-12-17 12:30:18,204] jepsen worker 27 - jepsen.util 3087 :invoke :cas [9 [24 41]] INFO [2019-12-17 12:30:18,223] jepsen worker 8 - jepsen.util 3788 :info :cas [2 [14 0]] :RequestCanceledException INFO [2019-12-17 12:30:18,224] jepsen worker 70 - jepsen.util 3760 :info :write [23 49] :RequestCanceledException INFO [2019-12-17 12:30:18,224] jepsen worker 10 - jepsen.util 3160 :fail :cas [3 [26 13]] :ValueNotSwapFrom INFO [2019-12-17 12:30:18,224] jepsen worker 10 - jepsen.util 3160 :invoke :cas [3 [6 17]] INFO [2019-12-17 12:30:18,228] jepsen worker 8 - jepsen.util 3878 :invoke :read [2 nil] INFO [2019-12-17 12:30:18,228] jepsen worker 70 - jepsen.util 3850 :invoke :read [23 nil] INFO [2019-12-17 12:30:18,242] jepsen worker 27 - jepsen.util 3087 :fail :cas [9 [24 41]] :ValueNotSwapFrom INFO [2019-12-17 12:30:18,242] jepsen worker 27 - jepsen.util 3087 :invoke :write [9 45] INFO [2019-12-17 12:30:18,255] jepsen worker 48 - jepsen.util 3828 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:30:18,255] jepsen worker 48 - jepsen.util 3828 :invoke :read [16 nil] INFO [2019-12-17 12:30:18,281] jepsen worker 88 - jepsen.util 3328 :ok :read [29 8] INFO [2019-12-17 12:30:18,281] jepsen worker 88 - jepsen.util 3328 :invoke :write [29 33] INFO [2019-12-17 12:30:18,283] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19123,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:30:18,283] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19123,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:30:18,283] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19127,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:30:18,284] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19127,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:30:18,284] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19115,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:30:18,284] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19115,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:30:18,284] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19116,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:30:18,284] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19116,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:30:18,284] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19117,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:30:18,285] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19117,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:30:18,286] jepsen worker 10 - jepsen.util 3160 :fail :cas [3 [6 17]] :ValueNotSwapFrom INFO [2019-12-17 12:30:18,287] jepsen worker 10 - jepsen.util 3160 :invoke :read [3 nil] INFO [2019-12-17 12:30:18,290] jepsen worker 26 - jepsen.util 4706 :info :write [8 4] :RequestCanceledException INFO [2019-12-17 12:30:18,295] jepsen worker 26 - jepsen.util 4796 :invoke :read [8 nil] INFO [2019-12-17 12:30:18,309] jepsen worker 41 - jepsen.util 4451 :info :cas [13 [44 42]] :RequestCanceledException INFO [2019-12-17 12:30:18,309] jepsen worker 5 - jepsen.util 2975 :fail :cas [1 [8 30]] :ValueNotSwapFrom INFO [2019-12-17 12:30:18,309] jepsen worker 5 - jepsen.util 2975 :invoke :write [1 43] INFO [2019-12-17 12:30:18,311] jepsen worker 41 - jepsen.util 4541 :invoke :read [13 nil] INFO [2019-12-17 12:30:18,312] jepsen worker 27 - jepsen.util 3087 :ok :write [9 45] INFO [2019-12-17 12:30:18,312] jepsen worker 27 - jepsen.util 3087 :invoke :read [9 nil] INFO [2019-12-17 12:30:18,323] jepsen worker 57 - jepsen.util 4017 :info :write [19 32] :RequestCanceledException INFO [2019-12-17 12:30:18,324] jepsen worker 33 - jepsen.util 4713 :info :write [11 0] :RequestCanceledException INFO [2019-12-17 12:30:18,329] jepsen worker 57 - jepsen.util 4107 :invoke :write [19 14] INFO [2019-12-17 12:30:18,335] jepsen worker 33 - jepsen.util 4803 :invoke :read [11 nil] INFO [2019-12-17 12:30:18,337] jepsen worker 0 - jepsen.util 4950 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:30:18,337] jepsen worker 0 - jepsen.util 4950 :invoke :read [0 nil] INFO [2019-12-17 12:30:18,339] jepsen worker 10 - jepsen.util 3160 :ok :read [3 12] INFO [2019-12-17 12:30:18,339] jepsen worker 10 - jepsen.util 3160 :invoke :write [3 18] INFO [2019-12-17 12:30:18,352] jepsen worker 27 - jepsen.util 3087 :ok :read [9 45] INFO [2019-12-17 12:30:18,352] jepsen worker 27 - jepsen.util 3087 :invoke :read [9 nil] INFO [2019-12-17 12:30:18,355] jepsen worker 17 - jepsen.util 4247 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:30:18,355] jepsen worker 17 - jepsen.util 4247 :invoke :write [5 31] INFO [2019-12-17 12:30:18,379] jepsen worker 80 - jepsen.util 3230 :ok :read [26 3] INFO [2019-12-17 12:30:18,379] jepsen worker 80 - jepsen.util 3230 :invoke :write [26 22] INFO [2019-12-17 12:30:18,394] jepsen worker 27 - jepsen.util 3087 :ok :read [9 45] INFO [2019-12-17 12:30:18,394] jepsen worker 27 - jepsen.util 3087 :invoke :read [9 nil] INFO [2019-12-17 12:30:18,424] jepsen worker 56 - jepsen.util 3296 :info :write [18 10] :RequestCanceledException INFO [2019-12-17 12:30:18,426] jepsen worker 56 - jepsen.util 3386 :invoke :read [18 nil] INFO [2019-12-17 12:30:18,453] jepsen worker 27 - jepsen.util 3087 :ok :read [9 45] INFO [2019-12-17 12:30:18,453] jepsen worker 27 - jepsen.util 3087 :invoke :cas [9 [0 35]] INFO [2019-12-17 12:30:18,502] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19149,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:30:18,502] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19149,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:30:18,502] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19152,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:30:18,502] jepsen worker 27 - jepsen.util 3087 :fail :cas [9 [0 35]] :ValueNotSwapFrom INFO [2019-12-17 12:30:18,503] jepsen worker 27 - jepsen.util 3087 :invoke :cas [9 [0 23]] INFO [2019-12-17 12:30:18,503] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19152,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:30:18,503] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19153,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:30:18,503] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19153,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:30:18,523] jepsen worker 65 - jepsen.util 2945 :info :write [21 32] :RequestCanceledException INFO [2019-12-17 12:30:18,523] jepsen worker 63 - jepsen.util 2943 :info :write [21 20] :RequestCanceledException INFO [2019-12-17 12:30:18,526] jepsen worker 63 - jepsen.util 3033 :invoke :write [21 37] INFO [2019-12-17 12:30:18,526] jepsen worker 65 - jepsen.util 3035 :invoke :read [21 nil] INFO [2019-12-17 12:30:18,551] jepsen worker 69 - jepsen.util 3309 :fail :cas [23 [34 38]] :ValueNotSwapFrom INFO [2019-12-17 12:30:18,551] jepsen worker 69 - jepsen.util 3309 :invoke :cas [23 [40 46]] INFO [2019-12-17 12:30:18,572] jepsen worker 65 - jepsen.util 3035 :ok :read [21 25] INFO [2019-12-17 12:30:18,572] jepsen worker 65 - jepsen.util 3035 :invoke :write [21 49] INFO [2019-12-17 12:30:18,584] jepsen worker 27 - jepsen.util 3087 :fail :cas [9 [0 23]] :ValueNotSwapFrom INFO [2019-12-17 12:30:18,584] jepsen worker 27 - jepsen.util 3087 :invoke :read [9 nil] INFO [2019-12-17 12:30:18,632] jepsen worker 65 - jepsen.util 3035 :ok :write [21 49] INFO [2019-12-17 12:30:18,638] jepsen worker 38 - jepsen.util 3188 :info :write [12 34] :RequestCanceledException INFO [2019-12-17 12:30:18,651] jepsen worker 65 - jepsen.util 3035 :invoke :read [21 nil] INFO [2019-12-17 12:30:18,653] jepsen worker 38 - jepsen.util 3278 :invoke :cas [12 [28 26]] INFO [2019-12-17 12:30:18,675] jepsen worker 51 - jepsen.util 3021 :fail :cas [17 [13 15]] :ValueNotSwapFrom INFO [2019-12-17 12:30:18,675] jepsen worker 51 - jepsen.util 3021 :invoke :cas [17 [19 3]] INFO [2019-12-17 12:30:18,679] jepsen worker 27 - jepsen.util 3087 :ok :read [9 45] INFO [2019-12-17 12:30:18,679] jepsen worker 27 - jepsen.util 3087 :invoke :cas [9 [3 32]] INFO [2019-12-17 12:30:18,706] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19163,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:30:18,707] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19163,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:30:18,707] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19167,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:30:18,707] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19167,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:30:18,708] jepsen worker 25 - jepsen.util 4705 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:30:18,708] jepsen worker 25 - jepsen.util 4705 :invoke :read [8 nil] INFO [2019-12-17 12:30:18,723] jepsen worker 65 - jepsen.util 3035 :ok :read [21 49] INFO [2019-12-17 12:30:18,724] jepsen worker 65 - jepsen.util 3035 :invoke :read [21 nil] INFO [2019-12-17 12:30:18,724] jepsen worker 81 - jepsen.util 4311 :info :cas [27 [11 10]] :RequestCanceledException INFO [2019-12-17 12:30:18,729] jepsen worker 81 - jepsen.util 4401 :invoke :read [27 nil] INFO [2019-12-17 12:30:18,737] jepsen worker 76 - jepsen.util 4576 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:30:18,737] jepsen worker 53 - jepsen.util 3293 :info :write [17 1] :RequestCanceledException INFO [2019-12-17 12:30:18,738] jepsen worker 76 - jepsen.util 4576 :invoke :write [25 28] INFO [2019-12-17 12:30:18,747] jepsen worker 53 - jepsen.util 3383 :invoke :write [17 14] INFO [2019-12-17 12:30:18,754] jepsen worker 61 - jepsen.util 5011 :info :cas [20 [14 36]] :RequestCanceledException INFO [2019-12-17 12:30:18,759] jepsen worker 27 - jepsen.util 3087 :fail :cas [9 [3 32]] :ValueNotSwapFrom INFO [2019-12-17 12:30:18,759] jepsen worker 27 - jepsen.util 3087 :invoke :read [9 nil] INFO [2019-12-17 12:30:18,761] jepsen worker 56 - jepsen.util 3386 :ok :read [18 31] INFO [2019-12-17 12:30:18,761] jepsen worker 56 - jepsen.util 3386 :invoke :read [18 nil] INFO [2019-12-17 12:30:18,762] jepsen worker 61 - jepsen.util 5101 :invoke :read [20 nil] INFO [2019-12-17 12:30:18,791] jepsen worker 58 - jepsen.util 4288 :info :write [19 10] :RequestCanceledException INFO [2019-12-17 12:30:18,797] jepsen worker 58 - jepsen.util 4378 :invoke :cas [19 [32 26]] INFO [2019-12-17 12:30:18,804] jepsen worker 65 - jepsen.util 3035 :ok :read [21 49] INFO [2019-12-17 12:30:18,804] jepsen worker 65 - jepsen.util 3035 :invoke :read [21 nil] INFO [2019-12-17 12:30:18,808] jepsen worker 86 - jepsen.util 4586 :info :write [28 34] :RequestCanceledException INFO [2019-12-17 12:30:18,813] jepsen worker 86 - jepsen.util 4676 :invoke :read [28 nil] INFO [2019-12-17 12:30:18,823] jepsen worker 27 - jepsen.util 3087 :ok :read [9 45] INFO [2019-12-17 12:30:18,823] jepsen worker 27 - jepsen.util 3087 :invoke :cas [9 [29 49]] INFO [2019-12-17 12:30:18,840] jepsen worker 51 - jepsen.util 3021 :fail :cas [17 [19 3]] :ValueNotSwapFrom INFO [2019-12-17 12:30:18,841] jepsen worker 51 - jepsen.util 3021 :invoke :write [17 27] INFO [2019-12-17 12:30:18,883] jepsen worker 27 - jepsen.util 3087 :fail :cas [9 [29 49]] :ValueNotSwapFrom INFO [2019-12-17 12:30:18,883] jepsen worker 27 - jepsen.util 3087 :invoke :read [9 nil] INFO [2019-12-17 12:30:18,891] jepsen worker 82 - jepsen.util 5212 :info :cas [27 [10 24]] :RequestCanceledException INFO [2019-12-17 12:30:18,893] jepsen worker 82 - jepsen.util 5302 :invoke :read [27 nil] INFO [2019-12-17 12:30:18,894] jepsen worker 65 - jepsen.util 3035 :ok :read [21 49] INFO [2019-12-17 12:30:18,894] jepsen worker 65 - jepsen.util 3035 :invoke :read [21 nil] INFO [2019-12-17 12:30:18,907] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:30:18,907] jepsen nemesis - couchbase.util Rebalance check count: 141 INFO [2019-12-17 12:30:18,907] jepsen nemesis - couchbase.util Rebalance stuck count: 122 INFO [2019-12-17 12:30:18,923] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19188,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:30:18,923] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19188,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:30:18,923] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19190,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:30:18,924] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19190,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:30:18,924] jepsen worker 2 - jepsen.util 3962 :info :write [0 42] :RequestCanceledException INFO [2019-12-17 12:30:18,924] jepsen worker 77 - jepsen.util 4127 :info :cas [25 [22 25]] :RequestCanceledException INFO [2019-12-17 12:30:18,927] jepsen worker 2 - jepsen.util 4052 :invoke :write [0 47] INFO [2019-12-17 12:30:18,928] jepsen worker 77 - jepsen.util 4217 :invoke :cas [25 [23 48]] INFO [2019-12-17 12:30:18,934] jepsen worker 27 - jepsen.util 3087 :ok :read [9 45] INFO [2019-12-17 12:30:18,934] jepsen worker 27 - jepsen.util 3087 :invoke :cas [9 [30 6]] INFO [2019-12-17 12:30:18,936] jepsen worker 68 - jepsen.util 4028 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:30:18,936] jepsen worker 66 - jepsen.util 4296 :info :write [22 4] :RequestCanceledException INFO [2019-12-17 12:30:18,936] jepsen worker 24 - jepsen.util 4164 :info :cas [8 [5 0]] :RequestCanceledException INFO [2019-12-17 12:30:18,936] jepsen worker 68 - jepsen.util 4028 :invoke :read [22 nil] INFO [2019-12-17 12:30:18,942] jepsen worker 24 - jepsen.util 4254 :invoke :write [8 41] INFO [2019-12-17 12:30:18,943] jepsen worker 66 - jepsen.util 4386 :invoke :read [22 nil] INFO [2019-12-17 12:30:18,952] jepsen worker 39 - jepsen.util 4089 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:30:18,952] jepsen worker 44 - jepsen.util 4724 :info :cas [14 [43 23]] :RequestCanceledException INFO [2019-12-17 12:30:18,953] jepsen worker 39 - jepsen.util 4089 :invoke :cas [13 [12 30]] INFO [2019-12-17 12:30:18,959] jepsen worker 44 - jepsen.util 4814 :invoke :write [14 39] INFO [2019-12-17 12:30:18,964] jepsen worker 65 - jepsen.util 3035 :ok :read [21 49] INFO [2019-12-17 12:30:18,965] jepsen worker 65 - jepsen.util 3035 :invoke :write [21 49] INFO [2019-12-17 12:30:18,983] jepsen worker 69 - jepsen.util 3309 :fail :cas [23 [40 46]] :ValueNotSwapFrom INFO [2019-12-17 12:30:18,984] jepsen worker 69 - jepsen.util 3309 :invoke :read [23 nil] INFO [2019-12-17 12:30:18,990] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19205,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:30:18,990] jepsen worker 67 - jepsen.util 4837 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:30:18,990] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19205,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:30:18,990] jepsen worker 67 - jepsen.util 4837 :invoke :read [22 nil] INFO [2019-12-17 12:30:18,990] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19208,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:30:18,991] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19208,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:30:18,991] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19209,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:30:18,991] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19209,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:30:18,992] jepsen worker 27 - jepsen.util 3087 :fail :cas [9 [30 6]] :ValueNotSwapFrom INFO [2019-12-17 12:30:18,992] jepsen worker 27 - jepsen.util 3087 :invoke :cas [9 [42 20]] INFO [2019-12-17 12:30:19,007] jepsen worker 6 - jepsen.util 4686 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:30:19,008] jepsen worker 6 - jepsen.util 4686 :invoke :cas [2 [40 0]] INFO [2019-12-17 12:30:19,024] jepsen worker 9 - jepsen.util 3879 :info :write [3 20] :RequestCanceledException INFO [2019-12-17 12:30:19,028] jepsen worker 65 - jepsen.util 3035 :ok :write [21 49] INFO [2019-12-17 12:30:19,028] jepsen worker 65 - jepsen.util 3035 :invoke :read [21 nil] INFO [2019-12-17 12:30:19,031] jepsen worker 9 - jepsen.util 3969 :invoke :read [3 nil] INFO [2019-12-17 12:30:19,038] jepsen worker 49 - jepsen.util 3829 :info :write [16 49] :RequestCanceledException INFO [2019-12-17 12:30:19,040] jepsen worker 49 - jepsen.util 3919 :invoke :write [16 11] INFO [2019-12-17 12:30:19,069] jepsen worker 45 - jepsen.util 3285 :info :write [15 36] :SyncWriteAmbiguous INFO [2019-12-17 12:30:19,071] jepsen worker 45 - jepsen.util 3375 :invoke :read [15 nil] INFO [2019-12-17 12:30:19,081] jepsen worker 27 - jepsen.util 3087 :fail :cas [9 [42 20]] :ValueNotSwapFrom INFO [2019-12-17 12:30:19,081] jepsen worker 27 - jepsen.util 3087 :invoke :read [9 nil] INFO [2019-12-17 12:30:19,099] jepsen worker 65 - jepsen.util 3035 :ok :read [21 49] INFO [2019-12-17 12:30:19,109] jepsen worker 87 - jepsen.util 3057 :info :write [29 30] :RequestCanceledException INFO [2019-12-17 12:30:19,117] jepsen worker 28 - jepsen.util 2458 :ok :read [9 45] INFO [2019-12-17 12:30:19,123] jepsen worker 60 - jepsen.util 4650 :info :cas [20 [11 8]] :RequestCanceledException INFO [2019-12-17 12:30:19,134] jepsen worker 20 - jepsen.util 2990 :fail :cas [6 [12 28]] :ValueNotSwapFrom INFO [2019-12-17 12:30:19,138] jepsen worker 12 - jepsen.util 3522 :info :write [4 2] :RequestCanceledException INFO [2019-12-17 12:30:19,145] jepsen worker 65 - jepsen.util 3035 :invoke :read [21 nil] INFO [2019-12-17 12:30:19,145] jepsen worker 28 - jepsen.util 2458 :invoke :write [9 1] INFO [2019-12-17 12:30:19,145] jepsen worker 20 - jepsen.util 2990 :invoke :read [6 nil] INFO [2019-12-17 12:30:19,147] jepsen worker 12 - jepsen.util 3612 :invoke :cas [4 [25 2]] INFO [2019-12-17 12:30:19,150] jepsen worker 87 - jepsen.util 3147 :invoke :cas [29 [43 34]] INFO [2019-12-17 12:30:19,152] jepsen worker 60 - jepsen.util 4740 :invoke :write [20 31] INFO [2019-12-17 12:30:19,155] jepsen worker 71 - jepsen.util 3311 :info :write [23 3] :RequestCanceledException INFO [2019-12-17 12:30:19,156] jepsen worker 70 - jepsen.util 3850 :ok :read [23 41] INFO [2019-12-17 12:30:19,156] jepsen worker 70 - jepsen.util 3850 :invoke :cas [23 [35 27]] INFO [2019-12-17 12:30:19,157] jepsen worker 71 - jepsen.util 3401 :invoke :read [23 nil] INFO [2019-12-17 12:30:19,165] jepsen worker 27 - jepsen.util 3087 :ok :read [9 45] INFO [2019-12-17 12:30:19,165] jepsen worker 27 - jepsen.util 3087 :invoke :cas [9 [18 34]] INFO [2019-12-17 12:30:19,170] jepsen worker 69 - jepsen.util 3309 :ok :read [23 41] INFO [2019-12-17 12:30:19,170] jepsen worker 69 - jepsen.util 3309 :invoke :write [23 49] INFO [2019-12-17 12:30:19,175] jepsen worker 9 - jepsen.util 3969 :ok :read [3 12] INFO [2019-12-17 12:30:19,176] jepsen worker 9 - jepsen.util 3969 :invoke :read [3 nil] INFO [2019-12-17 12:30:19,190] jepsen worker 14 - jepsen.util 3254 :info :write [4 48] :RequestCanceledException INFO [2019-12-17 12:30:19,193] jepsen worker 14 - jepsen.util 3344 :invoke :write [4 13] INFO [2019-12-17 12:30:19,202] jepsen worker 56 - jepsen.util 3386 :ok :read [18 31] INFO [2019-12-17 12:30:19,202] jepsen worker 56 - jepsen.util 3386 :invoke :cas [18 [21 30]] INFO [2019-12-17 12:30:19,219] jepsen worker 20 - jepsen.util 2990 :ok :read [6 30] INFO [2019-12-17 12:30:19,220] jepsen worker 20 - jepsen.util 2990 :invoke :cas [6 [39 43]] INFO [2019-12-17 12:30:19,224] jepsen worker 34 - jepsen.util 4444 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:30:19,224] jepsen worker 34 - jepsen.util 4444 :invoke :read [11 nil] INFO [2019-12-17 12:30:19,238] jepsen worker 27 - jepsen.util 3087 :fail :cas [9 [18 34]] :ValueNotSwapFrom INFO [2019-12-17 12:30:19,238] jepsen worker 27 - jepsen.util 3087 :invoke :cas [9 [26 42]] INFO [2019-12-17 12:30:19,248] jepsen worker 65 - jepsen.util 3035 :ok :read [21 49] INFO [2019-12-17 12:30:19,248] jepsen worker 65 - jepsen.util 3035 :invoke :read [21 nil] INFO [2019-12-17 12:30:19,253] jepsen worker 28 - jepsen.util 2458 :ok :write [9 1] INFO [2019-12-17 12:30:19,253] jepsen worker 28 - jepsen.util 2458 :invoke :cas [9 [27 17]] INFO [2019-12-17 12:30:19,266] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19229,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:30:19,266] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19229,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:30:19,266] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19230,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:30:19,266] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19230,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:30:19,273] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19239,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:30:19,273] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19239,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:30:19,300] jepsen worker 27 - jepsen.util 3087 :fail :cas [9 [26 42]] :ValueNotSwapFrom INFO [2019-12-17 12:30:19,301] jepsen worker 27 - jepsen.util 3087 :invoke :write [9 44] INFO [2019-12-17 12:30:19,308] jepsen worker 37 - jepsen.util 3097 :info :write [12 30] :RequestCanceledException INFO [2019-12-17 12:30:19,310] jepsen worker 37 - jepsen.util 3187 :invoke :write [12 4] INFO [2019-12-17 12:30:19,324] jepsen worker 20 - jepsen.util 2990 :fail :cas [6 [39 43]] :ValueNotSwapFrom INFO [2019-12-17 12:30:19,324] jepsen worker 20 - jepsen.util 2990 :invoke :write [6 14] INFO [2019-12-17 12:30:19,327] jepsen worker 9 - jepsen.util 3969 :ok :read [3 12] INFO [2019-12-17 12:30:19,328] jepsen worker 9 - jepsen.util 3969 :invoke :write [3 32] INFO [2019-12-17 12:30:19,337] jepsen worker 7 - jepsen.util 4147 :info :write [2 41] :RequestCanceledException INFO [2019-12-17 12:30:19,345] jepsen worker 7 - jepsen.util 4237 :invoke :cas [2 [48 7]] INFO [2019-12-17 12:30:19,349] jepsen worker 65 - jepsen.util 3035 :ok :read [21 49] INFO [2019-12-17 12:30:19,349] jepsen worker 65 - jepsen.util 3035 :invoke :read [21 nil] INFO [2019-12-17 12:30:19,359] jepsen worker 12 - jepsen.util 3612 :fail :cas [4 [25 2]] :ValueNotSwapFrom INFO [2019-12-17 12:30:19,359] jepsen worker 12 - jepsen.util 3612 :invoke :cas [4 [33 6]] INFO [2019-12-17 12:30:19,366] jepsen worker 28 - jepsen.util 2458 :fail :cas [9 [27 17]] :ValueNotSwapFrom INFO [2019-12-17 12:30:19,366] jepsen worker 28 - jepsen.util 2458 :invoke :read [9 nil] INFO [2019-12-17 12:30:19,388] jepsen worker 38 - jepsen.util 3278 :fail :cas [12 [28 26]] :ValueNotSwapFrom INFO [2019-12-17 12:30:19,389] jepsen worker 38 - jepsen.util 3278 :invoke :read [12 nil] INFO [2019-12-17 12:30:19,390] jepsen worker 23 - jepsen.util 4253 :info :cas [7 [47 15]] :RequestCanceledException INFO [2019-12-17 12:30:19,392] jepsen worker 23 - jepsen.util 4343 :invoke :read [7 nil] INFO [2019-12-17 12:30:19,392] jepsen worker 71 - jepsen.util 3401 :ok :read [23 41] INFO [2019-12-17 12:30:19,392] jepsen worker 71 - jepsen.util 3401 :invoke :cas [23 [10 20]] INFO [2019-12-17 12:30:19,408] jepsen worker 36 - jepsen.util 3456 :info :write [12 45] :RequestCanceledException INFO [2019-12-17 12:30:19,408] jepsen worker 52 - jepsen.util 3472 :info :write [17 45] :RequestCanceledException INFO [2019-12-17 12:30:19,410] jepsen worker 36 - jepsen.util 3546 :invoke :read [12 nil] INFO [2019-12-17 12:30:19,414] jepsen worker 52 - jepsen.util 3562 :invoke :read [17 nil] INFO [2019-12-17 12:30:19,418] jepsen worker 27 - jepsen.util 3087 :ok :write [9 44] INFO [2019-12-17 12:30:19,418] jepsen worker 27 - jepsen.util 3087 :invoke :cas [9 [27 21]] INFO [2019-12-17 12:30:19,462] jepsen worker 65 - jepsen.util 3035 :ok :read [21 49] INFO [2019-12-17 12:30:19,463] jepsen worker 65 - jepsen.util 3035 :invoke :read [21 nil] INFO [2019-12-17 12:30:19,477] jepsen worker 27 - jepsen.util 3087 :fail :cas [9 [27 21]] :ValueNotSwapFrom INFO [2019-12-17 12:30:19,477] jepsen worker 27 - jepsen.util 3087 :invoke :write [9 15] INFO [2019-12-17 12:30:19,488] jepsen worker 36 - jepsen.util 3546 :ok :read [12 20] INFO [2019-12-17 12:30:19,489] jepsen worker 36 - jepsen.util 3546 :invoke :cas [12 [21 10]] INFO [2019-12-17 12:30:19,491] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19263,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:30:19,491] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19263,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:30:19,491] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19266,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:30:19,491] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19266,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:30:19,491] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19267,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:30:19,491] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19267,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:30:19,504] jepsen worker 65 - jepsen.util 3035 :ok :read [21 49] INFO [2019-12-17 12:30:19,504] jepsen worker 65 - jepsen.util 3035 :invoke :read [21 nil] INFO [2019-12-17 12:30:19,522] jepsen worker 4 - jepsen.util 2524 :info :write [1 12] :RequestCanceledException INFO [2019-12-17 12:30:19,529] jepsen worker 4 - jepsen.util 2614 :invoke :write [1 8] INFO [2019-12-17 12:30:19,535] jepsen worker 27 - jepsen.util 3087 :ok :write [9 15] INFO [2019-12-17 12:30:19,535] jepsen worker 27 - jepsen.util 3087 :invoke :cas [9 [13 1]] INFO [2019-12-17 12:30:19,537] jepsen worker 79 - jepsen.util 3319 :info :write [26 31] :RequestCanceledException INFO [2019-12-17 12:30:19,539] jepsen worker 79 - jepsen.util 3409 :invoke :write [26 14] INFO [2019-12-17 12:30:19,553] jepsen worker 56 - jepsen.util 3386 :fail :cas [18 [21 30]] :ValueNotSwapFrom INFO [2019-12-17 12:30:19,554] jepsen worker 56 - jepsen.util 3386 :invoke :write [18 9] INFO [2019-12-17 12:30:19,591] jepsen worker 36 - jepsen.util 3546 :fail :cas [12 [21 10]] :ValueNotSwapFrom INFO [2019-12-17 12:30:19,591] jepsen worker 36 - jepsen.util 3546 :invoke :write [12 33] INFO [2019-12-17 12:30:19,602] jepsen worker 65 - jepsen.util 3035 :ok :read [21 49] INFO [2019-12-17 12:30:19,602] jepsen worker 65 - jepsen.util 3035 :invoke :cas [21 [6 13]] INFO [2019-12-17 12:30:19,608] jepsen worker 74 - jepsen.util 3584 :info :write [24 39] :SyncWriteAmbiguous INFO [2019-12-17 12:30:19,615] jepsen worker 74 - jepsen.util 3674 :invoke :write [24 5] INFO [2019-12-17 12:30:19,620] jepsen worker 27 - jepsen.util 3087 :fail :cas [9 [13 1]] :ValueNotSwapFrom INFO [2019-12-17 12:30:19,620] jepsen worker 27 - jepsen.util 3087 :invoke :read [9 nil] INFO [2019-12-17 12:30:19,638] jepsen worker 52 - jepsen.util 3562 :ok :read [17 27] INFO [2019-12-17 12:30:19,639] jepsen worker 52 - jepsen.util 3562 :invoke :write [17 38] INFO [2019-12-17 12:30:19,646] jepsen worker 64 - jepsen.util 2764 :ok :write [21 21] INFO [2019-12-17 12:30:19,647] jepsen worker 64 - jepsen.util 2764 :invoke :write [21 23] INFO [2019-12-17 12:30:19,655] jepsen worker 78 - jepsen.util 3678 :info :write [26 45] :RequestCanceledException INFO [2019-12-17 12:30:19,659] jepsen worker 78 - jepsen.util 3768 :invoke :write [26 39] INFO [2019-12-17 12:30:19,673] jepsen worker 65 - jepsen.util 3035 :fail :cas [21 [6 13]] :ValueNotSwapFrom INFO [2019-12-17 12:30:19,674] jepsen worker 65 - jepsen.util 3035 :invoke :write [21 7] INFO [2019-12-17 12:30:19,700] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19283,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:30:19,700] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19283,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:30:19,701] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19285,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:30:19,701] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19285,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:30:19,701] jepsen worker 27 - jepsen.util 3087 :ok :read [9 15] INFO [2019-12-17 12:30:19,703] jepsen worker 27 - jepsen.util 3087 :invoke :cas [9 [27 22]] INFO [2019-12-17 12:30:19,708] jepsen worker 16 - jepsen.util 4786 :info :cas [5 [9 32]] :RequestCanceledException INFO [2019-12-17 12:30:19,713] jepsen worker 16 - jepsen.util 4876 :invoke :cas [5 [7 33]] INFO [2019-12-17 12:30:19,724] jepsen worker 35 - jepsen.util 4265 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:30:19,724] jepsen worker 35 - jepsen.util 4265 :invoke :read [11 nil] INFO [2019-12-17 12:30:19,727] jepsen worker 19 - jepsen.util 3619 :fail :cas [6 [2 0]] :ValueNotSwapFrom INFO [2019-12-17 12:30:19,727] jepsen worker 19 - jepsen.util 3619 :invoke :read [6 nil] INFO [2019-12-17 12:30:19,735] jepsen worker 45 - jepsen.util 3375 :ok :read [15 17] INFO [2019-12-17 12:30:19,736] jepsen worker 45 - jepsen.util 3375 :invoke :write [15 5] INFO [2019-12-17 12:30:19,737] jepsen worker 59 - jepsen.util 4469 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:30:19,738] jepsen worker 59 - jepsen.util 4469 :invoke :read [19 nil] INFO [2019-12-17 12:30:19,741] jepsen worker 12 - jepsen.util 3612 :fail :cas [4 [33 6]] :ValueNotSwapFrom INFO [2019-12-17 12:30:19,741] jepsen worker 12 - jepsen.util 3612 :invoke :cas [4 [26 12]] INFO [2019-12-17 12:30:19,746] jepsen worker 64 - jepsen.util 2764 :ok :write [21 23] INFO [2019-12-17 12:30:19,746] jepsen worker 64 - jepsen.util 2764 :invoke :cas [21 [47 36]] INFO [2019-12-17 12:30:19,778] jepsen worker 38 - jepsen.util 3278 :ok :read [12 20] INFO [2019-12-17 12:30:19,778] jepsen worker 31 - jepsen.util 2731 :info :write [10 4] :SyncWriteAmbiguous INFO [2019-12-17 12:30:19,778] jepsen worker 38 - jepsen.util 3278 :invoke :read [12 nil] INFO [2019-12-17 12:30:19,783] jepsen worker 65 - jepsen.util 3035 :ok :write [21 7] INFO [2019-12-17 12:30:19,783] jepsen worker 65 - jepsen.util 3035 :invoke :write [21 27] INFO [2019-12-17 12:30:19,784] jepsen worker 31 - jepsen.util 2821 :invoke :cas [10 [32 43]] INFO [2019-12-17 12:30:19,802] jepsen worker 71 - jepsen.util 3401 :fail :cas [23 [10 20]] :ValueNotSwapFrom INFO [2019-12-17 12:30:19,802] jepsen worker 71 - jepsen.util 3401 :invoke :read [23 nil] INFO [2019-12-17 12:30:19,814] jepsen worker 19 - jepsen.util 3619 :ok :read [6 30] INFO [2019-12-17 12:30:19,814] jepsen worker 19 - jepsen.util 3619 :invoke :read [6 nil] INFO [2019-12-17 12:30:19,836] jepsen worker 27 - jepsen.util 3087 :fail :cas [9 [27 22]] :ValueNotSwapFrom INFO [2019-12-17 12:30:19,837] jepsen worker 27 - jepsen.util 3087 :invoke :write [9 12] INFO [2019-12-17 12:30:19,852] jepsen worker 64 - jepsen.util 2764 :fail :cas [21 [47 36]] :ValueNotSwapFrom INFO [2019-12-17 12:30:19,853] jepsen worker 64 - jepsen.util 2764 :invoke :write [21 43] INFO [2019-12-17 12:30:19,855] jepsen worker 84 - jepsen.util 4314 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:30:19,855] jepsen worker 84 - jepsen.util 4314 :invoke :cas [28 [22 39]] INFO [2019-12-17 12:30:19,891] jepsen worker 55 - jepsen.util 4015 :info :write [18 10] :RequestCanceledException INFO [2019-12-17 12:30:19,893] jepsen worker 55 - jepsen.util 4105 :invoke :write [18 12] INFO [2019-12-17 12:30:19,909] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19306,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:30:19,909] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19306,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:30:19,909] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19309,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:30:19,910] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19309,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:30:19,910] jepsen worker 65 - jepsen.util 3035 :ok :write [21 27] INFO [2019-12-17 12:30:19,910] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19310,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:30:19,910] jepsen worker 65 - jepsen.util 3035 :invoke :write [21 13] INFO [2019-12-17 12:30:19,910] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19310,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:30:19,923] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:30:19,924] jepsen nemesis - couchbase.util Rebalance check count: 142 INFO [2019-12-17 12:30:19,924] jepsen nemesis - couchbase.util Rebalance stuck count: 123 INFO [2019-12-17 12:30:19,924] jepsen worker 72 - jepsen.util 3222 :info :write [24 47] :RequestCanceledException INFO [2019-12-17 12:30:19,924] jepsen worker 15 - jepsen.util 3975 :info :cas [5 [25 18]] :RequestCanceledException INFO [2019-12-17 12:30:19,924] jepsen worker 43 - jepsen.util 4093 :info :cas [14 [39 28]] :RequestCanceledException INFO [2019-12-17 12:30:19,926] jepsen worker 15 - jepsen.util 4065 :invoke :cas [5 [37 19]] INFO [2019-12-17 12:30:19,926] jepsen worker 43 - jepsen.util 4183 :invoke :read [14 nil] INFO [2019-12-17 12:30:19,926] jepsen worker 72 - jepsen.util 3312 :invoke :cas [24 [41 38]] INFO [2019-12-17 12:30:19,937] jepsen worker 29 - jepsen.util 2729 :info :cas [9 [35 24]] :RequestCanceledException INFO [2019-12-17 12:30:19,937] jepsen worker 18 - jepsen.util 3438 :fail :read [6 nil] :RequestTimeoutException INFO [2019-12-17 12:30:19,937] jepsen worker 18 - jepsen.util 3438 :invoke :write [6 24] INFO [2019-12-17 12:30:19,939] jepsen worker 29 - jepsen.util 2819 :invoke :read [9 nil] INFO [2019-12-17 12:30:19,955] jepsen worker 85 - jepsen.util 4855 :info :write [28 18] :RequestCanceledException INFO [2019-12-17 12:30:19,959] jepsen worker 85 - jepsen.util 4945 :invoke :write [28 27] INFO [2019-12-17 12:30:19,975] jepsen worker 38 - jepsen.util 3278 :ok :read [12 20] INFO [2019-12-17 12:30:19,975] jepsen worker 38 - jepsen.util 3278 :invoke :cas [12 [9 3]] INFO [2019-12-17 12:30:19,988] jepsen worker 87 - jepsen.util 3147 :fail :cas [29 [43 34]] :ValueNotSwapFrom INFO [2019-12-17 12:30:19,988] jepsen worker 87 - jepsen.util 3147 :invoke :read [29 nil] INFO [2019-12-17 12:30:19,991] jepsen worker 50 - jepsen.util 4910 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:30:19,991] jepsen worker 30 - jepsen.util 3720 :info :write [10 32] :RequestCanceledException INFO [2019-12-17 12:30:19,991] jepsen worker 50 - jepsen.util 4910 :invoke :cas [16 [32 41]] INFO [2019-12-17 12:30:19,993] jepsen worker 30 - jepsen.util 3810 :invoke :cas [10 [15 46]] INFO [2019-12-17 12:30:20,004] jepsen worker 65 - jepsen.util 3035 :ok :write [21 13] INFO [2019-12-17 12:30:20,004] jepsen worker 65 - jepsen.util 3035 :invoke :cas [21 [31 20]] INFO [2019-12-17 12:30:20,006] jepsen worker 70 - jepsen.util 3850 :fail :cas [23 [35 27]] :ValueNotSwapFrom INFO [2019-12-17 12:30:20,006] jepsen worker 70 - jepsen.util 3850 :invoke :write [23 32] INFO [2019-12-17 12:30:20,024] jepsen worker 3 - jepsen.util 3423 :info :write [1 39] :RequestCanceledException INFO [2019-12-17 12:30:20,024] jepsen worker 73 - jepsen.util 3223 :info :write [24 29] :RequestCanceledException INFO [2019-12-17 12:30:20,027] jepsen worker 3 - jepsen.util 3513 :invoke :cas [1 [37 23]] INFO [2019-12-17 12:30:20,027] jepsen worker 73 - jepsen.util 3313 :invoke :cas [24 [18 44]] INFO [2019-12-17 12:30:20,059] jepsen worker 65 - jepsen.util 3035 :fail :cas [21 [31 20]] :ValueNotSwapFrom INFO [2019-12-17 12:30:20,060] jepsen worker 65 - jepsen.util 3035 :invoke :read [21 nil] INFO [2019-12-17 12:30:20,063] jepsen worker 71 - jepsen.util 3401 :ok :read [23 41] INFO [2019-12-17 12:30:20,063] jepsen worker 71 - jepsen.util 3401 :invoke :read [23 nil] INFO [2019-12-17 12:30:20,083] jepsen worker 38 - jepsen.util 3278 :fail :cas [12 [9 3]] :ValueNotSwapFrom INFO [2019-12-17 12:30:20,084] jepsen worker 38 - jepsen.util 3278 :invoke :write [12 47] INFO [2019-12-17 12:30:20,085] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19324,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:30:20,085] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19324,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:30:20,085] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19327,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:30:20,086] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19327,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:30:20,088] jepsen worker 30 - jepsen.util 3810 :fail :cas [10 [15 46]] :ValueNotSwapFrom INFO [2019-12-17 12:30:20,088] jepsen worker 30 - jepsen.util 3810 :invoke :read [10 nil] INFO [2019-12-17 12:30:20,091] jepsen worker 11 - jepsen.util 3521 :info :write [3 9] :RequestCanceledException INFO [2019-12-17 12:30:20,091] jepsen worker 32 - jepsen.util 3722 :info :write [10 47] :RequestCanceledException INFO [2019-12-17 12:30:20,093] jepsen worker 11 - jepsen.util 3611 :invoke :write [3 2] INFO [2019-12-17 12:30:20,093] jepsen worker 32 - jepsen.util 3812 :invoke :read [10 nil] INFO [2019-12-17 12:30:20,111] jepsen worker 65 - jepsen.util 3035 :ok :read [21 13] INFO [2019-12-17 12:30:20,111] jepsen worker 65 - jepsen.util 3035 :invoke :read [21 nil] INFO [2019-12-17 12:30:20,124] jepsen worker 1 - jepsen.util 3961 :info :write [0 11] :RequestCanceledException INFO [2019-12-17 12:30:20,130] jepsen worker 1 - jepsen.util 4051 :invoke :write [0 23] INFO [2019-12-17 12:30:20,138] jepsen worker 62 - jepsen.util 4202 :info :write [20 19] :RequestCanceledException INFO [2019-12-17 12:30:20,138] jepsen worker 40 - jepsen.util 4720 :info :cas [13 [2 12]] :RequestCanceledException INFO [2019-12-17 12:30:20,142] jepsen worker 40 - jepsen.util 4810 :invoke :read [13 nil] INFO [2019-12-17 12:30:20,142] jepsen worker 62 - jepsen.util 4292 :invoke :read [20 nil] INFO [2019-12-17 12:30:20,149] jepsen worker 30 - jepsen.util 3810 :ok :read [10 46] INFO [2019-12-17 12:30:20,149] jepsen worker 30 - jepsen.util 3810 :invoke :cas [10 [17 43]] INFO [2019-12-17 12:30:20,162] jepsen worker 32 - jepsen.util 3812 :ok :read [10 46] INFO [2019-12-17 12:30:20,162] jepsen worker 32 - jepsen.util 3812 :invoke :cas [10 [37 15]] INFO [2019-12-17 12:30:20,182] jepsen worker 30 - jepsen.util 3810 :fail :cas [10 [17 43]] :ValueNotSwapFrom INFO [2019-12-17 12:30:20,182] jepsen worker 30 - jepsen.util 3810 :invoke :read [10 nil] INFO [2019-12-17 12:30:20,190] jepsen worker 46 - jepsen.util 3646 :info :write [15 31] :RequestCanceledException INFO [2019-12-17 12:30:20,192] jepsen worker 46 - jepsen.util 3736 :invoke :cas [15 [3 37]] INFO [2019-12-17 12:30:20,193] jepsen worker 65 - jepsen.util 3035 :ok :read [21 13] INFO [2019-12-17 12:30:20,193] jepsen worker 65 - jepsen.util 3035 :invoke :write [21 11] INFO [2019-12-17 12:30:20,212] jepsen worker 10 - jepsen.util 3160 :info :write [3 18] :SyncWriteAmbiguous INFO [2019-12-17 12:30:20,213] jepsen worker 32 - jepsen.util 3812 :fail :cas [10 [37 15]] :ValueNotSwapFrom INFO [2019-12-17 12:30:20,213] jepsen worker 32 - jepsen.util 3812 :invoke :read [10 nil] INFO [2019-12-17 12:30:20,214] jepsen worker 21 - jepsen.util 4341 :info :write [7 13] :RequestCanceledException INFO [2019-12-17 12:30:20,214] jepsen worker 89 - jepsen.util 3149 :info :write [29 34] :RequestCanceledException INFO [2019-12-17 12:30:20,215] jepsen worker 22 - jepsen.util 4522 :info :write [7 35] :RequestCanceledException INFO [2019-12-17 12:30:20,215] jepsen worker 10 - jepsen.util 3250 :invoke :cas [3 [20 38]] INFO [2019-12-17 12:30:20,217] jepsen worker 89 - jepsen.util 3239 :invoke :cas [29 [41 45]] INFO [2019-12-17 12:30:20,220] jepsen worker 22 - jepsen.util 4612 :invoke :cas [7 [35 20]] INFO [2019-12-17 12:30:20,221] jepsen worker 21 - jepsen.util 4431 :invoke :write [7 7] INFO [2019-12-17 12:30:20,222] jepsen worker 54 - jepsen.util 3564 :info :write [18 33] :RequestCanceledException INFO [2019-12-17 12:30:20,223] jepsen worker 13 - jepsen.util 3163 :info :write [4 15] :RequestCanceledException INFO [2019-12-17 12:30:20,223] jepsen worker 47 - jepsen.util 3917 :info :write [15 33] :RequestCanceledException INFO [2019-12-17 12:30:20,224] jepsen worker 54 - jepsen.util 3654 :invoke :cas [18 [32 34]] INFO [2019-12-17 12:30:20,226] jepsen worker 13 - jepsen.util 3253 :invoke :write [4 1] INFO [2019-12-17 12:30:20,228] jepsen worker 47 - jepsen.util 4007 :invoke :read [15 nil] INFO [2019-12-17 12:30:20,238] jepsen worker 75 - jepsen.util 4575 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:30:20,238] jepsen worker 75 - jepsen.util 4575 :invoke :cas [25 [7 17]] INFO [2019-12-17 12:30:20,243] jepsen worker 30 - jepsen.util 3810 :ok :read [10 46] INFO [2019-12-17 12:30:20,243] jepsen worker 30 - jepsen.util 3810 :invoke :cas [10 [17 23]] INFO [2019-12-17 12:30:20,252] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19347,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:30:20,252] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19347,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:30:20,253] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19349,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:30:20,253] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19349,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:30:20,256] jepsen worker 83 - jepsen.util 4583 :info :cas [27 [34 42]] :RequestCanceledException INFO [2019-12-17 12:30:20,256] jepsen worker 42 - jepsen.util 5262 :info :write [14 23] :RequestCanceledException INFO [2019-12-17 12:30:20,259] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19356,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:30:20,259] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19356,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:30:20,259] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19361,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:30:20,259] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19361,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:30:20,259] jepsen worker 83 - jepsen.util 4673 :invoke :cas [27 [32 49]] INFO [2019-12-17 12:30:20,260] jepsen worker 42 - jepsen.util 5352 :invoke :cas [14 [47 34]] INFO [2019-12-17 12:30:20,283] jepsen worker 63 - jepsen.util 3033 :ok :write [21 37] INFO [2019-12-17 12:30:20,283] jepsen worker 63 - jepsen.util 3033 :invoke :write [21 7] INFO [2019-12-17 12:30:20,308] jepsen worker 26 - jepsen.util 4796 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:30:20,308] jepsen worker 88 - jepsen.util 3328 :info :write [29 33] :RequestCanceledException INFO [2019-12-17 12:30:20,308] jepsen worker 26 - jepsen.util 4796 :invoke :read [8 nil] INFO [2019-12-17 12:30:20,310] jepsen worker 88 - jepsen.util 3418 :invoke :write [29 6] INFO [2019-12-17 12:30:20,324] jepsen worker 8 - jepsen.util 3878 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:30:20,324] jepsen worker 8 - jepsen.util 3878 :invoke :read [2 nil] INFO [2019-12-17 12:30:20,324] jepsen worker 41 - jepsen.util 4541 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:30:20,324] jepsen worker 41 - jepsen.util 4541 :invoke :write [13 13] INFO [2019-12-17 12:30:20,326] jepsen worker 30 - jepsen.util 3810 :fail :cas [10 [17 23]] :ValueNotSwapFrom INFO [2019-12-17 12:30:20,326] jepsen worker 30 - jepsen.util 3810 :invoke :write [10 32] INFO [2019-12-17 12:30:20,332] jepsen worker 65 - jepsen.util 3035 :ok :write [21 11] INFO [2019-12-17 12:30:20,332] jepsen worker 65 - jepsen.util 3035 :invoke :read [21 nil] INFO [2019-12-17 12:30:20,338] jepsen worker 72 - jepsen.util 3312 :fail :cas [24 [41 38]] :ValueNotSwapFrom INFO [2019-12-17 12:30:20,338] jepsen worker 72 - jepsen.util 3312 :invoke :cas [24 [30 17]] INFO [2019-12-17 12:30:20,354] jepsen worker 48 - jepsen.util 3828 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:30:20,354] jepsen worker 33 - jepsen.util 4803 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:30:20,355] jepsen worker 33 - jepsen.util 4803 :invoke :read [11 nil] INFO [2019-12-17 12:30:20,355] jepsen worker 48 - jepsen.util 3828 :invoke :cas [16 [41 43]] INFO [2019-12-17 12:30:20,389] jepsen worker 32 - jepsen.util 3812 :ok :read [10 46] INFO [2019-12-17 12:30:20,389] jepsen worker 32 - jepsen.util 3812 :invoke :read [10 nil] INFO [2019-12-17 12:30:20,391] jepsen worker 80 - jepsen.util 3230 :info :write [26 22] :RequestCanceledException INFO [2019-12-17 12:30:20,393] jepsen worker 80 - jepsen.util 3320 :invoke :cas [26 [46 45]] INFO [2019-12-17 12:30:20,412] jepsen worker 65 - jepsen.util 3035 :ok :read [21 11] INFO [2019-12-17 12:30:20,412] jepsen worker 65 - jepsen.util 3035 :invoke :cas [21 [8 20]] INFO [2019-12-17 12:30:20,424] jepsen worker 5 - jepsen.util 2975 :info :write [1 43] :RequestCanceledException INFO [2019-12-17 12:30:20,432] jepsen worker 5 - jepsen.util 3065 :invoke :write [1 1] INFO [2019-12-17 12:30:20,437] jepsen worker 0 - jepsen.util 4950 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:30:20,437] jepsen worker 57 - jepsen.util 4107 :info :write [19 14] :RequestCanceledException INFO [2019-12-17 12:30:20,438] jepsen worker 0 - jepsen.util 4950 :invoke :cas [0 [14 43]] INFO [2019-12-17 12:30:20,443] jepsen worker 57 - jepsen.util 4197 :invoke :cas [19 [16 29]] INFO [2019-12-17 12:30:20,449] jepsen worker 32 - jepsen.util 3812 :ok :read [10 46] INFO [2019-12-17 12:30:20,449] jepsen worker 32 - jepsen.util 3812 :invoke :write [10 29] INFO [2019-12-17 12:30:20,454] jepsen worker 3 - jepsen.util 3513 :fail :cas [1 [37 23]] :ValueNotSwapFrom INFO [2019-12-17 12:30:20,454] jepsen worker 3 - jepsen.util 3513 :invoke :read [1 nil] INFO [2019-12-17 12:30:20,455] jepsen worker 17 - jepsen.util 4247 :info :write [5 31] :RequestCanceledException INFO [2019-12-17 12:30:20,459] jepsen worker 17 - jepsen.util 4337 :invoke :cas [5 [19 20]] INFO [2019-12-17 12:30:20,479] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19387,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:30:20,479] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19387,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:30:20,479] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19391,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:30:20,479] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19391,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:30:20,511] jepsen worker 65 - jepsen.util 3035 :fail :cas [21 [8 20]] :ValueNotSwapFrom INFO [2019-12-17 12:30:20,511] jepsen worker 65 - jepsen.util 3035 :invoke :read [21 nil] INFO [2019-12-17 12:30:20,536] jepsen worker 79 - jepsen.util 3409 :ok :write [26 14] INFO [2019-12-17 12:30:20,536] jepsen worker 79 - jepsen.util 3409 :invoke :read [26 nil] INFO [2019-12-17 12:30:20,550] jepsen worker 12 - jepsen.util 3612 :fail :cas [4 [26 12]] :ValueNotSwapFrom INFO [2019-12-17 12:30:20,550] jepsen worker 12 - jepsen.util 3612 :invoke :cas [4 [5 23]] INFO [2019-12-17 12:30:20,556] jepsen worker 20 - jepsen.util 2990 :ok :write [6 14] INFO [2019-12-17 12:30:20,556] jepsen worker 20 - jepsen.util 2990 :invoke :read [6 nil] INFO [2019-12-17 12:30:20,569] jepsen worker 45 - jepsen.util 3375 :ok :write [15 5] INFO [2019-12-17 12:30:20,569] jepsen worker 45 - jepsen.util 3375 :invoke :write [15 13] INFO [2019-12-17 12:30:20,583] jepsen worker 65 - jepsen.util 3035 :ok :read [21 11] INFO [2019-12-17 12:30:20,584] jepsen worker 65 - jepsen.util 3035 :invoke :read [21 nil] INFO [2019-12-17 12:30:20,613] jepsen worker 52 - jepsen.util 3562 :ok :write [17 38] INFO [2019-12-17 12:30:20,613] jepsen worker 52 - jepsen.util 3562 :invoke :cas [17 [47 48]] INFO [2019-12-17 12:30:20,646] jepsen worker 89 - jepsen.util 3239 :fail :cas [29 [41 45]] :ValueNotSwapFrom INFO [2019-12-17 12:30:20,646] jepsen worker 89 - jepsen.util 3239 :invoke :read [29 nil] INFO [2019-12-17 12:30:20,676] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19406,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:30:20,676] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19406,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:30:20,677] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19409,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:30:20,677] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19409,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:30:20,677] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19410,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:30:20,677] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19410,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:30:20,708] jepsen worker 65 - jepsen.util 3035 :ok :read [21 11] INFO [2019-12-17 12:30:20,709] jepsen worker 65 - jepsen.util 3035 :invoke :read [21 nil] INFO [2019-12-17 12:30:20,724] jepsen worker 19 - jepsen.util 3619 :ok :read [6 14] INFO [2019-12-17 12:30:20,753] jepsen worker 19 - jepsen.util 3619 :invoke :read [6 nil] INFO [2019-12-17 12:30:20,756] jepsen worker 71 - jepsen.util 3401 :ok :read [23 41] INFO [2019-12-17 12:30:20,756] jepsen worker 71 - jepsen.util 3401 :invoke :cas [23 [33 44]] INFO [2019-12-17 12:30:20,759] jepsen worker 28 - jepsen.util 2458 :ok :read [9 15] INFO [2019-12-17 12:30:20,759] jepsen worker 28 - jepsen.util 2458 :invoke :write [9 20] INFO [2019-12-17 12:30:20,760] jepsen worker 31 - jepsen.util 2821 :fail :cas [10 [32 43]] :ValueNotSwapFrom INFO [2019-12-17 12:30:20,760] jepsen worker 31 - jepsen.util 2821 :invoke :write [10 11] INFO [2019-12-17 12:30:20,774] jepsen worker 30 - jepsen.util 3810 :ok :write [10 32] INFO [2019-12-17 12:30:20,774] jepsen worker 74 - jepsen.util 3674 :ok :write [24 5] INFO [2019-12-17 12:30:20,778] jepsen worker 54 - jepsen.util 3654 :fail :cas [18 [32 34]] :ValueNotSwapFrom INFO [2019-12-17 12:30:20,792] jepsen worker 20 - jepsen.util 2990 :ok :read [6 14] INFO [2019-12-17 12:30:20,804] jepsen worker 11 - jepsen.util 3611 :ok :write [3 2] INFO [2019-12-17 12:30:20,808] jepsen worker 25 - jepsen.util 4705 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:30:20,811] jepsen worker 64 - jepsen.util 2764 :ok :write [21 43] INFO [2019-12-17 12:30:20,823] jepsen worker 61 - jepsen.util 5101 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:30:20,824] jepsen worker 14 - jepsen.util 3344 :ok :write [4 13] INFO [2019-12-17 12:30:20,838] jepsen worker 76 - jepsen.util 4576 :info :write [25 28] :RequestCanceledException INFO [2019-12-17 12:30:20,838] jepsen worker 81 - jepsen.util 4401 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:30:20,852] jepsen worker 53 - jepsen.util 3383 :info :write [17 14] :RequestCanceledException INFO [2019-12-17 12:30:20,883] jepsen worker 65 - jepsen.util 3035 :ok :read [21 11] INFO [2019-12-17 12:30:20,893] jepsen worker 58 - jepsen.util 4378 :info :cas [19 [32 26]] :RequestCanceledException INFO [2019-12-17 12:30:20,897] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19418,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:30:20,909] jepsen worker 86 - jepsen.util 4676 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:30:20,912] jepsen worker 73 - jepsen.util 3313 :fail :cas [24 [18 44]] :ValueNotSwapFrom INFO [2019-12-17 12:30:20,922] jepsen worker 82 - jepsen.util 5302 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:30:20,926] jepsen worker 46 - jepsen.util 3736 :fail :cas [15 [3 37]] :ValueNotSwapFrom INFO [2019-12-17 12:30:20,934] jepsen worker 28 - jepsen.util 2458 :ok :write [9 20] INFO [2019-12-17 12:30:20,935] jepsen worker 19 - jepsen.util 3619 :ok :read [6 14] INFO [2019-12-17 12:30:20,938] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:30:20,938] jepsen worker 51 - jepsen.util 3021 :info :write [17 27] :RequestCanceledException INFO [2019-12-17 12:30:20,956] jepsen worker 77 - jepsen.util 4217 :info :cas [25 [23 48]] :RequestCanceledException INFO [2019-12-17 12:30:20,977] jepsen worker 31 - jepsen.util 2821 :ok :write [10 11] INFO [2019-12-17 12:30:20,984] jepsen worker 30 - jepsen.util 3810 :invoke :read [10 nil] INFO [2019-12-17 12:30:20,984] jepsen worker 74 - jepsen.util 3674 :invoke :read [24 nil] INFO [2019-12-17 12:30:20,984] jepsen worker 54 - jepsen.util 3654 :invoke :write [18 19] INFO [2019-12-17 12:30:20,984] jepsen worker 20 - jepsen.util 2990 :invoke :read [6 nil] INFO [2019-12-17 12:30:20,984] jepsen worker 11 - jepsen.util 3611 :invoke :write [3 0] INFO [2019-12-17 12:30:20,984] jepsen worker 25 - jepsen.util 4705 :invoke :write [8 37] INFO [2019-12-17 12:30:20,984] jepsen worker 64 - jepsen.util 2764 :invoke :read [21 nil] INFO [2019-12-17 12:30:20,985] jepsen worker 61 - jepsen.util 5101 :invoke :cas [20 [21 44]] INFO [2019-12-17 12:30:20,985] jepsen worker 14 - jepsen.util 3344 :invoke :write [4 44] INFO [2019-12-17 12:30:20,985] jepsen worker 81 - jepsen.util 4401 :invoke :cas [27 [27 20]] INFO [2019-12-17 12:30:20,985] jepsen worker 65 - jepsen.util 3035 :invoke :read [21 nil] INFO [2019-12-17 12:30:20,985] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19418,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:30:20,985] jepsen worker 86 - jepsen.util 4676 :invoke :write [28 17] INFO [2019-12-17 12:30:20,985] jepsen worker 73 - jepsen.util 3313 :invoke :read [24 nil] INFO [2019-12-17 12:30:20,985] jepsen worker 82 - jepsen.util 5302 :invoke :write [27 41] INFO [2019-12-17 12:30:20,985] jepsen worker 46 - jepsen.util 3736 :invoke :write [15 3] INFO [2019-12-17 12:30:20,986] jepsen worker 28 - jepsen.util 2458 :invoke :write [9 16] INFO [2019-12-17 12:30:20,986] jepsen worker 66 - jepsen.util 4386 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:30:20,986] jepsen worker 19 - jepsen.util 3619 :invoke :write [6 46] INFO [2019-12-17 12:30:20,986] jepsen nemesis - couchbase.util Rebalance check count: 143 INFO [2019-12-17 12:30:20,987] jepsen worker 31 - jepsen.util 2821 :invoke :read [10 nil] INFO [2019-12-17 12:30:20,987] jepsen worker 72 - jepsen.util 3312 :fail :cas [24 [30 17]] :ValueNotSwapFrom INFO [2019-12-17 12:30:20,987] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19421,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:30:20,988] jepsen worker 66 - jepsen.util 4386 :invoke :read [22 nil] INFO [2019-12-17 12:30:20,988] jepsen nemesis - couchbase.util Rebalance stuck count: 124 INFO [2019-12-17 12:30:20,988] jepsen worker 58 - jepsen.util 4468 :invoke :write [19 46] INFO [2019-12-17 12:30:20,988] jepsen worker 72 - jepsen.util 3312 :invoke :write [24 31] INFO [2019-12-17 12:30:20,988] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19421,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:30:20,991] jepsen worker 53 - jepsen.util 3473 :invoke :read [17 nil] INFO [2019-12-17 12:30:20,991] jepsen worker 51 - jepsen.util 3111 :invoke :cas [17 [42 25]] INFO [2019-12-17 12:30:20,992] jepsen worker 76 - jepsen.util 4666 :invoke :read [25 nil] INFO [2019-12-17 12:30:20,994] jepsen worker 77 - jepsen.util 4307 :invoke :cas [25 [46 35]] INFO [2019-12-17 12:30:20,994] jepsen worker 3 - jepsen.util 3513 :ok :read [1 26] INFO [2019-12-17 12:30:20,995] jepsen worker 3 - jepsen.util 3513 :invoke :write [1 32] INFO [2019-12-17 12:30:21,002] jepsen worker 44 - jepsen.util 4814 :info :write [14 39] :RequestCanceledException INFO [2019-12-17 12:30:21,013] jepsen worker 44 - jepsen.util 4904 :invoke :write [14 23] INFO [2019-12-17 12:30:21,014] jepsen worker 24 - jepsen.util 4254 :info :write [8 41] :RequestCanceledException INFO [2019-12-17 12:30:21,018] jepsen worker 80 - jepsen.util 3320 :fail :cas [26 [46 45]] :ValueNotSwapFrom INFO [2019-12-17 12:30:21,018] jepsen worker 80 - jepsen.util 3320 :invoke :write [26 41] INFO [2019-12-17 12:30:21,024] jepsen worker 24 - jepsen.util 4344 :invoke :cas [8 [14 29]] INFO [2019-12-17 12:30:21,025] jepsen worker 89 - jepsen.util 3239 :ok :read [29 33] INFO [2019-12-17 12:30:21,025] jepsen worker 89 - jepsen.util 3239 :invoke :cas [29 [42 25]] INFO [2019-12-17 12:30:21,034] jepsen worker 68 - jepsen.util 4028 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:30:21,034] jepsen worker 2 - jepsen.util 4052 :info :write [0 47] :RequestCanceledException INFO [2019-12-17 12:30:21,034] jepsen worker 68 - jepsen.util 4028 :invoke :write [22 39] INFO [2019-12-17 12:30:21,041] jepsen worker 2 - jepsen.util 4142 :invoke :write [0 49] INFO [2019-12-17 12:30:21,055] jepsen worker 39 - jepsen.util 4089 :info :cas [13 [12 30]] :RequestCanceledException INFO [2019-12-17 12:30:21,062] jepsen worker 39 - jepsen.util 4179 :invoke :read [13 nil] INFO [2019-12-17 12:30:21,072] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19437,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:30:21,072] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19437,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:30:21,072] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19440,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:30:21,072] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19440,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:30:21,086] jepsen worker 79 - jepsen.util 3409 :ok :read [26 14] INFO [2019-12-17 12:30:21,086] jepsen worker 79 - jepsen.util 3409 :invoke :cas [26 [22 41]] INFO [2019-12-17 12:30:21,089] jepsen worker 4 - jepsen.util 2614 :ok :write [1 8] INFO [2019-12-17 12:30:21,090] jepsen worker 4 - jepsen.util 2614 :invoke :read [1 nil] INFO [2019-12-17 12:30:21,090] jepsen worker 67 - jepsen.util 4837 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:30:21,091] jepsen worker 67 - jepsen.util 4837 :invoke :write [22 4] INFO [2019-12-17 12:30:21,108] jepsen worker 6 - jepsen.util 4686 :info :cas [2 [40 0]] :RequestCanceledException INFO [2019-12-17 12:30:21,111] jepsen worker 6 - jepsen.util 4776 :invoke :cas [2 [41 13]] INFO [2019-12-17 12:30:21,137] jepsen worker 30 - jepsen.util 3810 :ok :read [10 11] INFO [2019-12-17 12:30:21,137] jepsen worker 74 - jepsen.util 3674 :ok :read [24 5] INFO [2019-12-17 12:30:21,137] jepsen worker 74 - jepsen.util 3674 :invoke :cas [24 [37 36]] INFO [2019-12-17 12:30:21,137] jepsen worker 30 - jepsen.util 3810 :invoke :write [10 41] INFO [2019-12-17 12:30:21,138] jepsen worker 49 - jepsen.util 3919 :info :write [16 11] :RequestCanceledException INFO [2019-12-17 12:30:21,144] jepsen worker 49 - jepsen.util 4009 :invoke :cas [16 [42 24]] INFO [2019-12-17 12:30:21,151] jepsen worker 64 - jepsen.util 2764 :ok :read [21 43] INFO [2019-12-17 12:30:21,151] jepsen worker 64 - jepsen.util 2764 :invoke :read [21 nil] INFO [2019-12-17 12:30:21,199] jepsen worker 87 - jepsen.util 3147 :ok :read [29 33] INFO [2019-12-17 12:30:21,199] jepsen worker 87 - jepsen.util 3147 :invoke :read [29 nil] INFO [2019-12-17 12:30:21,202] jepsen worker 20 - jepsen.util 2990 :ok :read [6 14] INFO [2019-12-17 12:30:21,202] jepsen worker 20 - jepsen.util 2990 :invoke :cas [6 [48 33]] INFO [2019-12-17 12:30:21,208] jepsen worker 60 - jepsen.util 4740 :info :write [20 31] :RequestCanceledException INFO [2019-12-17 12:30:21,213] jepsen worker 60 - jepsen.util 4830 :invoke :read [20 nil] INFO [2019-12-17 12:30:21,214] jepsen worker 52 - jepsen.util 3562 :fail :cas [17 [47 48]] :ValueNotSwapFrom INFO [2019-12-17 12:30:21,223] jepsen worker 52 - jepsen.util 3562 :invoke :read [17 nil] INFO [2019-12-17 12:30:21,224] jepsen worker 69 - jepsen.util 3309 :info :write [23 49] :RequestCanceledException INFO [2019-12-17 12:30:21,232] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19479,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:30:21,233] jepsen worker 31 - jepsen.util 2821 :ok :read [10 11] INFO [2019-12-17 12:30:21,233] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19471,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:30:21,234] jepsen worker 73 - jepsen.util 3313 :ok :read [24 5] INFO [2019-12-17 12:30:21,235] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19479,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:30:21,235] jepsen worker 31 - jepsen.util 2821 :invoke :cas [10 [48 36]] INFO [2019-12-17 12:30:21,235] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19471,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:30:21,235] jepsen worker 73 - jepsen.util 3313 :invoke :write [24 4] INFO [2019-12-17 12:30:21,235] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19482,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:30:21,235] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19472,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:30:21,235] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19482,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:30:21,235] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19472,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:30:21,237] jepsen worker 69 - jepsen.util 3399 :invoke :cas [23 [10 12]] INFO [2019-12-17 12:30:21,238] jepsen worker 65 - jepsen.util 3035 :ok :read [21 43] INFO [2019-12-17 12:30:21,238] jepsen worker 65 - jepsen.util 3035 :invoke :cas [21 [8 12]] INFO [2019-12-17 12:30:21,270] jepsen worker 71 - jepsen.util 3401 :fail :cas [23 [33 44]] :ValueNotSwapFrom INFO [2019-12-17 12:30:21,270] jepsen worker 71 - jepsen.util 3401 :invoke :read [23 nil] INFO [2019-12-17 12:30:21,296] jepsen worker 65 - jepsen.util 3035 :fail :cas [21 [8 12]] :ValueNotSwapFrom INFO [2019-12-17 12:30:21,296] jepsen worker 65 - jepsen.util 3035 :invoke :cas [21 [12 30]] INFO [2019-12-17 12:30:21,323] jepsen worker 34 - jepsen.util 4444 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:30:21,324] jepsen worker 34 - jepsen.util 4444 :invoke :cas [11 [22 4]] INFO [2019-12-17 12:30:21,330] jepsen worker 28 - jepsen.util 2458 :ok :write [9 16] INFO [2019-12-17 12:30:21,330] jepsen worker 28 - jepsen.util 2458 :invoke :write [9 33] INFO [2019-12-17 12:30:21,343] jepsen worker 70 - jepsen.util 3850 :ok :write [23 32] INFO [2019-12-17 12:30:21,343] jepsen worker 70 - jepsen.util 3850 :invoke :write [23 38] INFO [2019-12-17 12:30:21,355] jepsen worker 37 - jepsen.util 3187 :info :write [12 4] :RequestCanceledException INFO [2019-12-17 12:30:21,358] jepsen worker 37 - jepsen.util 3277 :invoke :write [12 22] INFO [2019-12-17 12:30:21,377] jepsen worker 71 - jepsen.util 3401 :ok :read [23 41] INFO [2019-12-17 12:30:21,377] jepsen worker 71 - jepsen.util 3401 :invoke :read [23 nil] INFO [2019-12-17 12:30:21,409] jepsen worker 46 - jepsen.util 3736 :ok :write [15 3] INFO [2019-12-17 12:30:21,409] jepsen worker 46 - jepsen.util 3736 :invoke :write [15 32] INFO [2019-12-17 12:30:21,412] jepsen worker 89 - jepsen.util 3239 :fail :cas [29 [42 25]] :ValueNotSwapFrom INFO [2019-12-17 12:30:21,412] jepsen worker 89 - jepsen.util 3239 :invoke :cas [29 [48 14]] INFO [2019-12-17 12:30:21,415] jepsen worker 65 - jepsen.util 3035 :fail :cas [21 [12 30]] :ValueNotSwapFrom INFO [2019-12-17 12:30:21,415] jepsen worker 65 - jepsen.util 3035 :invoke :read [21 nil] INFO [2019-12-17 12:30:21,424] jepsen worker 9 - jepsen.util 3969 :info :write [3 32] :RequestCanceledException INFO [2019-12-17 12:30:21,424] jepsen worker 7 - jepsen.util 4237 :info :cas [2 [48 7]] :RequestCanceledException INFO [2019-12-17 12:30:21,426] jepsen worker 9 - jepsen.util 4059 :invoke :write [3 46] INFO [2019-12-17 12:30:21,431] jepsen worker 7 - jepsen.util 4327 :invoke :read [2 nil] INFO [2019-12-17 12:30:21,449] jepsen worker 28 - jepsen.util 2458 :ok :write [9 33] INFO [2019-12-17 12:30:21,452] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19503,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:30:21,458] jepsen worker 12 - jepsen.util 3612 :fail :cas [4 [5 23]] :ValueNotSwapFrom INFO [2019-12-17 12:30:21,473] jepsen worker 28 - jepsen.util 2458 :invoke :write [9 4] INFO [2019-12-17 12:30:21,473] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19503,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:30:21,473] jepsen worker 12 - jepsen.util 3612 :invoke :cas [4 [29 45]] INFO [2019-12-17 12:30:21,473] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19505,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:30:21,474] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19505,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:30:21,480] jepsen worker 65 - jepsen.util 3035 :ok :read [21 43] INFO [2019-12-17 12:30:21,480] jepsen worker 65 - jepsen.util 3035 :invoke :read [21 nil] INFO [2019-12-17 12:30:21,490] jepsen worker 23 - jepsen.util 4343 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:30:21,491] jepsen worker 23 - jepsen.util 4343 :invoke :write [7 28] INFO [2019-12-17 12:30:21,542] jepsen worker 13 - jepsen.util 3253 :ok :write [4 1] INFO [2019-12-17 12:30:21,542] jepsen worker 13 - jepsen.util 3253 :invoke :cas [4 [16 8]] INFO [2019-12-17 12:30:21,547] jepsen worker 30 - jepsen.util 3810 :ok :write [10 41] INFO [2019-12-17 12:30:21,547] jepsen worker 30 - jepsen.util 3810 :invoke :read [10 nil] INFO [2019-12-17 12:30:21,554] jepsen worker 65 - jepsen.util 3035 :ok :read [21 43] INFO [2019-12-17 12:30:21,554] jepsen worker 65 - jepsen.util 3035 :invoke :read [21 nil] INFO [2019-12-17 12:30:21,558] jepsen worker 19 - jepsen.util 3619 :ok :write [6 46] INFO [2019-12-17 12:30:21,559] jepsen worker 19 - jepsen.util 3619 :invoke :read [6 nil] INFO [2019-12-17 12:30:21,604] jepsen worker 53 - jepsen.util 3473 :ok :read [17 38] INFO [2019-12-17 12:30:21,604] jepsen worker 53 - jepsen.util 3473 :invoke :read [17 nil] INFO [2019-12-17 12:30:21,621] jepsen worker 28 - jepsen.util 2458 :ok :write [9 4] INFO [2019-12-17 12:30:21,621] jepsen worker 28 - jepsen.util 2458 :invoke :cas [9 [31 5]] INFO [2019-12-17 12:30:21,624] jepsen worker 56 - jepsen.util 3386 :info :write [18 9] :RequestCanceledException INFO [2019-12-17 12:30:21,636] jepsen worker 56 - jepsen.util 3476 :invoke :write [18 40] INFO [2019-12-17 12:30:21,663] jepsen worker 30 - jepsen.util 3810 :ok :read [10 41] INFO [2019-12-17 12:30:21,664] jepsen worker 30 - jepsen.util 3810 :invoke :cas [10 [29 24]] INFO [2019-12-17 12:30:21,670] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19520,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:30:21,670] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19520,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:30:21,672] jepsen worker 52 - jepsen.util 3562 :ok :read [17 38] INFO [2019-12-17 12:30:21,672] jepsen worker 52 - jepsen.util 3562 :invoke :cas [17 [24 33]] INFO [2019-12-17 12:30:21,691] jepsen worker 78 - jepsen.util 3768 :info :write [26 39] :RequestCanceledException INFO [2019-12-17 12:30:21,702] jepsen worker 78 - jepsen.util 3858 :invoke :read [26 nil] INFO [2019-12-17 12:30:21,708] jepsen worker 36 - jepsen.util 3546 :info :write [12 33] :RequestCanceledException INFO [2019-12-17 12:30:21,713] jepsen worker 36 - jepsen.util 3636 :invoke :read [12 nil] INFO [2019-12-17 12:30:21,718] jepsen worker 69 - jepsen.util 3399 :fail :cas [23 [10 12]] :ValueNotSwapFrom INFO [2019-12-17 12:30:21,719] jepsen worker 69 - jepsen.util 3399 :invoke :read [23 nil] INFO [2019-12-17 12:30:21,725] jepsen worker 65 - jepsen.util 3035 :ok :read [21 43] INFO [2019-12-17 12:30:21,725] jepsen worker 65 - jepsen.util 3035 :invoke :read [21 nil] INFO [2019-12-17 12:30:21,736] jepsen worker 28 - jepsen.util 2458 :fail :cas [9 [31 5]] :ValueNotSwapFrom INFO [2019-12-17 12:30:21,736] jepsen worker 28 - jepsen.util 2458 :invoke :cas [9 [49 36]] INFO [2019-12-17 12:30:21,753] jepsen worker 12 - jepsen.util 3612 :fail :cas [4 [29 45]] :ValueNotSwapFrom INFO [2019-12-17 12:30:21,754] jepsen worker 12 - jepsen.util 3612 :invoke :write [4 12] INFO [2019-12-17 12:30:21,775] jepsen worker 71 - jepsen.util 3401 :ok :read [23 32] INFO [2019-12-17 12:30:21,776] jepsen worker 71 - jepsen.util 3401 :invoke :write [23 38] INFO [2019-12-17 12:30:21,809] jepsen worker 16 - jepsen.util 4876 :info :cas [5 [7 33]] :RequestCanceledException INFO [2019-12-17 12:30:21,811] jepsen worker 46 - jepsen.util 3736 :ok :write [15 32] INFO [2019-12-17 12:30:21,812] jepsen worker 46 - jepsen.util 3736 :invoke :cas [15 [19 48]] INFO [2019-12-17 12:30:21,814] jepsen worker 16 - jepsen.util 4966 :invoke :cas [5 [34 2]] INFO [2019-12-17 12:30:21,818] jepsen worker 72 - jepsen.util 3312 :ok :write [24 31] INFO [2019-12-17 12:30:21,818] jepsen worker 72 - jepsen.util 3312 :invoke :write [24 48] INFO [2019-12-17 12:30:21,820] jepsen worker 51 - jepsen.util 3111 :fail :cas [17 [42 25]] :ValueNotSwapFrom INFO [2019-12-17 12:30:21,820] jepsen worker 51 - jepsen.util 3111 :invoke :read [17 nil] INFO [2019-12-17 12:30:21,824] jepsen worker 35 - jepsen.util 4265 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:30:21,824] jepsen worker 35 - jepsen.util 4265 :invoke :read [11 nil] INFO [2019-12-17 12:30:21,827] jepsen worker 30 - jepsen.util 3810 :fail :cas [10 [29 24]] :ValueNotSwapFrom INFO [2019-12-17 12:30:21,827] jepsen worker 30 - jepsen.util 3810 :invoke :read [10 nil] INFO [2019-12-17 12:30:21,838] jepsen worker 59 - jepsen.util 4469 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:30:21,838] jepsen worker 59 - jepsen.util 4469 :invoke :read [19 nil] INFO [2019-12-17 12:30:21,839] jepsen worker 4 - jepsen.util 2614 :ok :read [1 8] INFO [2019-12-17 12:30:21,839] jepsen worker 4 - jepsen.util 2614 :invoke :write [1 7] INFO [2019-12-17 12:30:21,868] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19542,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:30:21,868] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19542,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:30:21,868] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19546,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:30:21,869] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19546,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:30:21,880] jepsen worker 28 - jepsen.util 2458 :fail :cas [9 [49 36]] :ValueNotSwapFrom INFO [2019-12-17 12:30:21,880] jepsen worker 28 - jepsen.util 2458 :invoke :read [9 nil] INFO [2019-12-17 12:30:21,889] jepsen worker 65 - jepsen.util 3035 :ok :read [21 43] INFO [2019-12-17 12:30:21,890] jepsen worker 65 - jepsen.util 3035 :invoke :read [21 nil] INFO [2019-12-17 12:30:21,907] jepsen worker 9 - jepsen.util 4059 :ok :write [3 46] INFO [2019-12-17 12:30:21,907] jepsen worker 9 - jepsen.util 4059 :invoke :read [3 nil] INFO [2019-12-17 12:30:21,909] jepsen worker 27 - jepsen.util 3087 :info :write [9 12] :RequestCanceledException INFO [2019-12-17 12:30:21,910] jepsen worker 53 - jepsen.util 3473 :ok :read [17 38] INFO [2019-12-17 12:30:21,911] jepsen worker 53 - jepsen.util 3473 :invoke :write [17 19] INFO [2019-12-17 12:30:21,911] jepsen worker 27 - jepsen.util 3177 :invoke :write [9 23] INFO [2019-12-17 12:30:21,939] jepsen worker 55 - jepsen.util 4105 :info :write [18 12] :RequestCanceledException INFO [2019-12-17 12:30:21,946] jepsen worker 55 - jepsen.util 4195 :invoke :write [18 32] INFO [2019-12-17 12:30:21,954] jepsen worker 84 - jepsen.util 4314 :info :cas [28 [22 39]] :RequestCanceledException INFO [2019-12-17 12:30:21,955] jepsen worker 15 - jepsen.util 4065 :info :cas [5 [37 19]] :RequestCanceledException INFO [2019-12-17 12:30:21,961] jepsen worker 15 - jepsen.util 4155 :invoke :read [5 nil] INFO [2019-12-17 12:30:21,973] jepsen worker 84 - jepsen.util 4404 :invoke :read [28 nil] INFO [2019-12-17 12:30:22,009] jepsen worker 29 - jepsen.util 2819 :fail :read [9 nil] :RequestTimeoutException INFO [2019-12-17 12:30:22,009] jepsen worker 29 - jepsen.util 2819 :invoke :cas [9 [22 29]] INFO [2019-12-17 12:30:22,024] jepsen worker 43 - jepsen.util 4183 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:30:22,024] jepsen worker 43 - jepsen.util 4183 :invoke :cas [14 [18 32]] INFO [2019-12-17 12:30:22,025] jepsen worker 85 - jepsen.util 4945 :info :write [28 27] :RequestCanceledException INFO [2019-12-17 12:30:22,038] jepsen worker 18 - jepsen.util 3438 :info :write [6 24] :RequestCanceledException INFO [2019-12-17 12:30:22,040] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:30:22,040] jepsen nemesis - couchbase.util Rebalance check count: 144 INFO [2019-12-17 12:30:22,040] jepsen nemesis - couchbase.util Rebalance stuck count: 125 INFO [2019-12-17 12:30:22,053] jepsen worker 18 - jepsen.util 3528 :invoke :cas [6 [42 39]] INFO [2019-12-17 12:30:22,058] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19555,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:30:22,059] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19555,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:30:22,059] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19559,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:30:22,060] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19559,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:30:22,063] jepsen worker 85 - jepsen.util 5035 :invoke :write [28 15] INFO [2019-12-17 12:30:22,068] jepsen worker 46 - jepsen.util 3736 :fail :cas [15 [19 48]] :ValueNotSwapFrom INFO [2019-12-17 12:30:22,068] jepsen worker 46 - jepsen.util 3736 :invoke :write [15 43] INFO [2019-12-17 12:30:22,068] jepsen worker 87 - jepsen.util 3147 :ok :read [29 33] INFO [2019-12-17 12:30:22,069] jepsen worker 87 - jepsen.util 3147 :invoke :write [29 22] INFO [2019-12-17 12:30:22,092] jepsen worker 50 - jepsen.util 4910 :info :cas [16 [32 41]] :RequestCanceledException INFO [2019-12-17 12:30:22,108] jepsen worker 50 - jepsen.util 5000 :invoke :write [16 4] INFO [2019-12-17 12:30:22,125] jepsen worker 38 - jepsen.util 3278 :info :write [12 47] :RequestCanceledException INFO [2019-12-17 12:30:22,155] jepsen worker 62 - jepsen.util 4292 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:30:22,156] jepsen worker 62 - jepsen.util 4292 :invoke :cas [20 [46 38]] INFO [2019-12-17 12:30:22,163] jepsen worker 38 - jepsen.util 3368 :invoke :cas [12 [6 32]] INFO [2019-12-17 12:30:22,223] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19573,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:30:22,223] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19573,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:30:22,223] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19580,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:30:22,223] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19580,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:30:22,224] jepsen worker 1 - jepsen.util 4051 :info :write [0 23] :RequestCanceledException INFO [2019-12-17 12:30:22,229] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19569,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:30:22,229] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19569,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:30:22,230] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19578,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:30:22,230] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19578,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:30:22,235] jepsen worker 30 - jepsen.util 3810 :ok :read [10 41] INFO [2019-12-17 12:30:22,236] jepsen worker 30 - jepsen.util 3810 :invoke :cas [10 [39 26]] INFO [2019-12-17 12:30:22,238] jepsen worker 40 - jepsen.util 4810 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:30:22,238] jepsen worker 40 - jepsen.util 4810 :invoke :read [13 nil] INFO [2019-12-17 12:30:22,244] jepsen worker 1 - jepsen.util 4141 :invoke :read [0 nil] INFO [2019-12-17 12:30:22,251] jepsen worker 27 - jepsen.util 3177 :ok :write [9 23] INFO [2019-12-17 12:30:22,251] jepsen worker 27 - jepsen.util 3177 :invoke :read [9 nil] INFO [2019-12-17 12:30:22,286] jepsen worker 89 - jepsen.util 3239 :fail :cas [29 [48 14]] :ValueNotSwapFrom INFO [2019-12-17 12:30:22,286] jepsen worker 89 - jepsen.util 3239 :invoke :cas [29 [4 20]] INFO [2019-12-17 12:30:22,290] jepsen worker 10 - jepsen.util 3250 :info :cas [3 [20 38]] :RequestCanceledException INFO [2019-12-17 12:30:22,290] jepsen worker 47 - jepsen.util 4007 :fail :read [15 nil] :RequestTimeoutException INFO [2019-12-17 12:30:22,290] jepsen worker 47 - jepsen.util 4007 :invoke :read [15 nil] INFO [2019-12-17 12:30:22,293] jepsen worker 10 - jepsen.util 3340 :invoke :cas [3 [28 0]] INFO [2019-12-17 12:30:22,309] jepsen worker 83 - jepsen.util 4673 :info :cas [27 [32 49]] :RequestCanceledException INFO [2019-12-17 12:30:22,309] jepsen worker 28 - jepsen.util 2458 :ok :read [9 4] INFO [2019-12-17 12:30:22,311] jepsen worker 28 - jepsen.util 2458 :invoke :read [9 nil] INFO [2019-12-17 12:30:22,324] jepsen worker 22 - jepsen.util 4612 :info :cas [7 [35 20]] :RequestCanceledException INFO [2019-12-17 12:30:22,326] jepsen worker 42 - jepsen.util 5352 :info :cas [14 [47 34]] :RequestCanceledException INFO [2019-12-17 12:30:22,338] jepsen worker 75 - jepsen.util 4575 :info :cas [25 [7 17]] :RequestCanceledException INFO [2019-12-17 12:30:22,340] jepsen worker 21 - jepsen.util 4431 :info :write [7 7] :RequestCanceledException INFO [2019-12-17 12:30:22,341] jepsen worker 88 - jepsen.util 3418 :info :write [29 6] :RequestCanceledException INFO [2019-12-17 12:30:22,347] jepsen worker 83 - jepsen.util 4763 :invoke :read [27 nil] INFO [2019-12-17 12:30:22,350] jepsen worker 88 - jepsen.util 3508 :invoke :cas [29 [1 29]] INFO [2019-12-17 12:30:22,353] jepsen worker 29 - jepsen.util 2819 :fail :cas [9 [22 29]] :ValueNotSwapFrom INFO [2019-12-17 12:30:22,356] jepsen worker 29 - jepsen.util 2819 :invoke :cas [9 [18 27]] INFO [2019-12-17 12:30:22,357] jepsen worker 42 - jepsen.util 5442 :invoke :cas [14 [19 26]] INFO [2019-12-17 12:30:22,360] jepsen worker 22 - jepsen.util 4702 :invoke :read [7 nil] INFO [2019-12-17 12:30:22,365] jepsen worker 75 - jepsen.util 4665 :invoke :read [25 nil] INFO [2019-12-17 12:30:22,380] jepsen worker 21 - jepsen.util 4521 :invoke :write [7 8] INFO [2019-12-17 12:30:22,382] jepsen worker 78 - jepsen.util 3858 :ok :read [26 14] INFO [2019-12-17 12:30:22,382] jepsen worker 78 - jepsen.util 3858 :invoke :read [26 nil] INFO [2019-12-17 12:30:22,392] jepsen worker 9 - jepsen.util 4059 :ok :read [3 46] INFO [2019-12-17 12:30:22,392] jepsen worker 9 - jepsen.util 4059 :invoke :read [3 nil] INFO [2019-12-17 12:30:22,398] jepsen worker 63 - jepsen.util 3033 :info :write [21 7] :RequestCanceledException INFO [2019-12-17 12:30:22,406] jepsen worker 26 - jepsen.util 4796 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:30:22,408] jepsen worker 26 - jepsen.util 4796 :invoke :read [8 nil] INFO [2019-12-17 12:30:22,422] jepsen worker 8 - jepsen.util 3878 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:30:22,422] jepsen worker 8 - jepsen.util 3878 :invoke :read [2 nil] INFO [2019-12-17 12:30:22,422] jepsen worker 63 - jepsen.util 3123 :invoke :read [21 nil] INFO [2019-12-17 12:30:22,422] jepsen worker 41 - jepsen.util 4541 :info :write [13 13] :RequestCanceledException INFO [2019-12-17 12:30:22,425] jepsen worker 65 - jepsen.util 3035 :ok :read [21 43] INFO [2019-12-17 12:30:22,426] jepsen worker 65 - jepsen.util 3035 :invoke :cas [21 [46 28]] INFO [2019-12-17 12:30:22,427] jepsen worker 36 - jepsen.util 3636 :ok :read [12 33] INFO [2019-12-17 12:30:22,427] jepsen worker 18 - jepsen.util 3528 :fail :cas [6 [42 39]] :ValueNotSwapFrom INFO [2019-12-17 12:30:22,427] jepsen worker 36 - jepsen.util 3636 :invoke :cas [12 [39 8]] INFO [2019-12-17 12:30:22,427] jepsen worker 18 - jepsen.util 3528 :invoke :cas [6 [38 25]] INFO [2019-12-17 12:30:22,445] jepsen worker 52 - jepsen.util 3562 :fail :cas [17 [24 33]] :ValueNotSwapFrom INFO [2019-12-17 12:30:22,445] jepsen worker 41 - jepsen.util 4631 :invoke :cas [13 [48 44]] INFO [2019-12-17 12:30:22,446] jepsen worker 52 - jepsen.util 3562 :invoke :cas [17 [38 48]] INFO [2019-12-17 12:30:22,456] jepsen worker 33 - jepsen.util 4803 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:30:22,456] jepsen worker 33 - jepsen.util 4803 :invoke :cas [11 [19 10]] INFO [2019-12-17 12:30:22,457] jepsen worker 48 - jepsen.util 3828 :info :cas [16 [41 43]] :RequestCanceledException INFO [2019-12-17 12:30:22,462] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19603,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:30:22,462] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19603,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:30:22,463] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19607,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:30:22,463] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19607,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:30:22,488] jepsen worker 48 - jepsen.util 3918 :invoke :read [16 nil] INFO [2019-12-17 12:30:22,490] jepsen worker 57 - jepsen.util 4197 :info :cas [19 [16 29]] :RequestCanceledException INFO [2019-12-17 12:30:22,494] jepsen worker 13 - jepsen.util 3253 :fail :cas [4 [16 8]] :ValueNotSwapFrom INFO [2019-12-17 12:30:22,494] jepsen worker 13 - jepsen.util 3253 :invoke :write [4 44] INFO [2019-12-17 12:30:22,497] jepsen worker 57 - jepsen.util 4287 :invoke :read [19 nil] INFO [2019-12-17 12:30:22,509] jepsen worker 17 - jepsen.util 4337 :info :cas [5 [19 20]] :RequestCanceledException INFO [2019-12-17 12:30:22,509] jepsen worker 32 - jepsen.util 3812 :info :write [10 29] :RequestCanceledException INFO [2019-12-17 12:30:22,524] jepsen worker 5 - jepsen.util 3065 :info :write [1 1] :RequestCanceledException INFO [2019-12-17 12:30:22,525] jepsen worker 32 - jepsen.util 3902 :invoke :cas [10 [39 0]] INFO [2019-12-17 12:30:22,534] jepsen worker 5 - jepsen.util 3155 :invoke :cas [1 [13 1]] INFO [2019-12-17 12:30:22,539] jepsen worker 0 - jepsen.util 4950 :info :cas [0 [14 43]] :RequestCanceledException INFO [2019-12-17 12:30:22,542] jepsen worker 17 - jepsen.util 4427 :invoke :read [5 nil] INFO [2019-12-17 12:30:22,593] jepsen worker 0 - jepsen.util 5040 :invoke :read [0 nil] INFO [2019-12-17 12:30:22,636] jepsen worker 46 - jepsen.util 3736 :ok :write [15 43] INFO [2019-12-17 12:30:22,636] jepsen worker 46 - jepsen.util 3736 :invoke :read [15 nil] INFO [2019-12-17 12:30:22,642] jepsen worker 47 - jepsen.util 4007 :ok :read [15 32] INFO [2019-12-17 12:30:22,643] jepsen worker 47 - jepsen.util 4007 :invoke :cas [15 [4 13]] INFO [2019-12-17 12:30:22,657] jepsen worker 45 - jepsen.util 3375 :info :write [15 13] :RequestCanceledException INFO [2019-12-17 12:30:22,669] jepsen worker 45 - jepsen.util 3465 :invoke :read [15 nil] INFO [2019-12-17 12:30:22,672] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19623,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:30:22,672] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19623,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:30:22,673] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19627,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:30:22,673] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19627,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:30:22,693] jepsen worker 10 - jepsen.util 3340 :fail :cas [3 [28 0]] :ValueNotSwapFrom INFO [2019-12-17 12:30:22,693] jepsen worker 10 - jepsen.util 3340 :invoke :write [3 21] INFO [2019-12-17 12:30:22,794] jepsen worker 30 - jepsen.util 3810 :fail :cas [10 [39 26]] :ValueNotSwapFrom INFO [2019-12-17 12:30:22,795] jepsen worker 30 - jepsen.util 3810 :invoke :read [10 nil] INFO [2019-12-17 12:30:22,802] jepsen worker 27 - jepsen.util 3177 :ok :read [9 23] INFO [2019-12-17 12:30:22,802] jepsen worker 27 - jepsen.util 3177 :invoke :read [9 nil] INFO [2019-12-17 12:30:22,818] jepsen worker 69 - jepsen.util 3399 :ok :read [23 32] INFO [2019-12-17 12:30:22,818] jepsen worker 69 - jepsen.util 3399 :invoke :read [23 nil] INFO [2019-12-17 12:30:22,858] jepsen worker 28 - jepsen.util 2458 :ok :read [9 23] INFO [2019-12-17 12:30:22,859] jepsen worker 28 - jepsen.util 2458 :invoke :read [9 nil] INFO [2019-12-17 12:30:22,870] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19639,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:30:22,871] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19639,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:30:23,028] jepsen worker 76 - jepsen.util 4666 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:30:23,028] jepsen worker 61 - jepsen.util 5101 :info :cas [20 [21 44]] :RequestCanceledException INFO [2019-12-17 12:30:23,028] jepsen worker 76 - jepsen.util 4666 :invoke :cas [25 [32 34]] INFO [2019-12-17 12:30:23,040] jepsen worker 14 - jepsen.util 3344 :info :write [4 44] :RequestCanceledException INFO [2019-12-17 12:30:23,040] jepsen worker 81 - jepsen.util 4401 :info :cas [27 [27 20]] :RequestCanceledException INFO [2019-12-17 12:30:23,062] jepsen worker 10 - jepsen.util 3340 :ok :write [3 21] INFO [2019-12-17 12:30:23,062] jepsen worker 54 - jepsen.util 3654 :info :write [18 19] :RequestCanceledException INFO [2019-12-17 12:30:23,062] jepsen worker 10 - jepsen.util 3340 :invoke :read [3 nil] INFO [2019-12-17 12:30:23,062] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19643,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:30:23,063] jepsen worker 3 - jepsen.util 3513 :info :write [1 32] :RequestCanceledException INFO [2019-12-17 12:30:23,063] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19643,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:30:23,064] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19646,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:30:23,065] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19646,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:30:23,066] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19647,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:30:23,067] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19647,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:30:23,087] jepsen worker 3 - jepsen.util 3603 :invoke :cas [1 [33 11]] INFO [2019-12-17 12:30:23,094] jepsen worker 66 - jepsen.util 4386 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:30:23,095] jepsen worker 66 - jepsen.util 4386 :invoke :write [22 27] INFO [2019-12-17 12:30:23,102] jepsen worker 77 - jepsen.util 4307 :info :cas [25 [46 35]] :RequestCanceledException INFO [2019-12-17 12:30:23,107] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:30:23,111] jepsen nemesis - couchbase.util Rebalance check count: 145 INFO [2019-12-17 12:30:23,115] jepsen nemesis - couchbase.util Rebalance stuck count: 126 INFO [2019-12-17 12:30:23,125] jepsen worker 86 - jepsen.util 4676 :info :write [28 17] :RequestCanceledException INFO [2019-12-17 12:30:23,125] jepsen worker 11 - jepsen.util 3611 :info :write [3 0] :RequestCanceledException INFO [2019-12-17 12:30:23,126] jepsen worker 24 - jepsen.util 4344 :info :cas [8 [14 29]] :RequestCanceledException INFO [2019-12-17 12:30:23,126] jepsen worker 6 - jepsen.util 4776 :info :cas [2 [41 13]] :RequestCanceledException INFO [2019-12-17 12:30:23,126] jepsen worker 82 - jepsen.util 5302 :info :write [27 41] :RequestCanceledException INFO [2019-12-17 12:30:23,126] jepsen worker 61 - jepsen.util 5191 :invoke :read [20 nil] INFO [2019-12-17 12:30:23,128] jepsen worker 44 - jepsen.util 4904 :info :write [14 23] :RequestCanceledException INFO [2019-12-17 12:30:23,128] jepsen worker 25 - jepsen.util 4705 :info :write [8 37] :RequestCanceledException INFO [2019-12-17 12:30:23,128] jepsen worker 58 - jepsen.util 4468 :info :write [19 46] :RequestCanceledException INFO [2019-12-17 12:30:23,152] jepsen worker 68 - jepsen.util 4028 :info :write [22 39] :RequestCanceledException INFO [2019-12-17 12:30:23,153] jepsen worker 2 - jepsen.util 4142 :info :write [0 49] :RequestCanceledException INFO [2019-12-17 12:30:23,153] jepsen worker 39 - jepsen.util 4179 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:30:23,154] jepsen worker 80 - jepsen.util 3320 :info :write [26 41] :RequestCanceledException INFO [2019-12-17 12:30:23,154] jepsen worker 39 - jepsen.util 4179 :invoke :write [13 25] INFO [2019-12-17 12:30:23,155] jepsen worker 79 - jepsen.util 3409 :info :cas [26 [22 41]] :RequestCanceledException INFO [2019-12-17 12:30:23,179] jepsen worker 81 - jepsen.util 4491 :invoke :cas [27 [5 7]] INFO [2019-12-17 12:30:23,186] jepsen worker 32 - jepsen.util 3902 :fail :cas [10 [39 0]] :ValueNotSwapFrom INFO [2019-12-17 12:30:23,189] jepsen worker 32 - jepsen.util 3902 :invoke :cas [10 [42 49]] INFO [2019-12-17 12:30:23,195] jepsen worker 67 - jepsen.util 4837 :info :write [22 4] :RequestCanceledException INFO [2019-12-17 12:30:23,196] jepsen worker 14 - jepsen.util 3434 :invoke :read [4 nil] INFO [2019-12-17 12:30:23,204] jepsen worker 11 - jepsen.util 3701 :invoke :cas [3 [10 42]] INFO [2019-12-17 12:30:23,205] jepsen worker 49 - jepsen.util 4009 :info :cas [16 [42 24]] :RequestCanceledException INFO [2019-12-17 12:30:23,211] jepsen worker 54 - jepsen.util 3744 :invoke :write [18 34] INFO [2019-12-17 12:30:23,218] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19664,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:30:23,221] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19664,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:30:23,229] jepsen worker 20 - jepsen.util 2990 :info :cas [6 [48 33]] :RequestCanceledException INFO [2019-12-17 12:30:23,232] jepsen worker 74 - jepsen.util 3674 :info :cas [24 [37 36]] :RequestCanceledException INFO [2019-12-17 12:30:23,461] jepsen worker 79 - jepsen.util 3499 :invoke :write [26 48] INFO [2019-12-17 12:30:23,463] jepsen worker 63 - jepsen.util 3123 :ok :read [21 43] INFO [2019-12-17 12:30:23,463] jepsen worker 89 - jepsen.util 3239 :fail :cas [29 [4 20]] :ValueNotSwapFrom INFO [2019-12-17 12:30:23,463] jepsen worker 63 - jepsen.util 3123 :invoke :write [21 0] INFO [2019-12-17 12:30:23,463] jepsen worker 89 - jepsen.util 3239 :invoke :write [29 37] INFO [2019-12-17 12:30:23,464] jepsen worker 20 - jepsen.util 3080 :invoke :read [6 nil] INFO [2019-12-17 12:30:23,466] jepsen worker 68 - jepsen.util 4118 :invoke :cas [22 [31 17]] INFO [2019-12-17 12:30:23,467] jepsen worker 77 - jepsen.util 4397 :invoke :cas [25 [22 38]] INFO [2019-12-17 12:30:23,467] jepsen worker 5 - jepsen.util 3155 :fail :cas [1 [13 1]] :ValueNotSwapFrom INFO [2019-12-17 12:30:23,469] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19672,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:30:23,473] jepsen worker 70 - jepsen.util 3850 :info :write [23 38] :RequestCanceledException INFO [2019-12-17 12:30:23,475] jepsen worker 34 - jepsen.util 4444 :info :cas [11 [22 4]] :RequestCanceledException INFO [2019-12-17 12:30:23,475] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19655,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:30:23,475] jepsen worker 73 - jepsen.util 3313 :info :write [24 4] :RequestCanceledException INFO [2019-12-17 12:30:23,475] jepsen worker 64 - jepsen.util 2764 :fail :read [21 nil] :RequestTimeoutException INFO [2019-12-17 12:30:23,475] jepsen worker 31 - jepsen.util 2821 :info :cas [10 [48 36]] :RequestCanceledException INFO [2019-12-17 12:30:23,476] jepsen worker 7 - jepsen.util 4327 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:30:23,476] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19672,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:30:23,476] jepsen worker 60 - jepsen.util 4830 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:30:23,476] jepsen worker 78 - jepsen.util 3858 :ok :read [26 14] INFO [2019-12-17 12:30:23,477] jepsen worker 5 - jepsen.util 3155 :invoke :read [1 nil] INFO [2019-12-17 12:30:23,477] jepsen worker 64 - jepsen.util 2764 :invoke :write [21 48] INFO [2019-12-17 12:30:23,484] jepsen worker 6 - jepsen.util 4866 :invoke :write [2 13] INFO [2019-12-17 12:30:23,486] jepsen worker 60 - jepsen.util 4830 :invoke :write [20 10] INFO [2019-12-17 12:30:23,486] jepsen worker 7 - jepsen.util 4327 :invoke :write [2 17] INFO [2019-12-17 12:30:23,486] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19655,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:30:23,486] jepsen worker 29 - jepsen.util 2819 :fail :cas [9 [18 27]] :ValueNotSwapFrom INFO [2019-12-17 12:30:23,487] jepsen worker 78 - jepsen.util 3858 :invoke :cas [26 [40 11]] INFO [2019-12-17 12:30:23,487] jepsen worker 86 - jepsen.util 4766 :invoke :read [28 nil] INFO [2019-12-17 12:30:23,487] jepsen worker 37 - jepsen.util 3277 :info :write [12 22] :RequestCanceledException INFO [2019-12-17 12:30:23,488] jepsen worker 80 - jepsen.util 3410 :invoke :write [26 49] INFO [2019-12-17 12:30:23,490] jepsen worker 82 - jepsen.util 5392 :invoke :write [27 34] INFO [2019-12-17 12:30:23,502] jepsen worker 29 - jepsen.util 2819 :invoke :cas [9 [20 47]] INFO [2019-12-17 12:30:23,504] jepsen worker 24 - jepsen.util 4434 :invoke :read [8 nil] INFO [2019-12-17 12:30:23,517] jepsen worker 2 - jepsen.util 4232 :invoke :write [0 17] INFO [2019-12-17 12:30:23,535] jepsen worker 67 - jepsen.util 4927 :invoke :write [22 40] INFO [2019-12-17 12:30:23,549] jepsen worker 25 - jepsen.util 4795 :invoke :write [8 26] INFO [2019-12-17 12:30:23,557] jepsen worker 49 - jepsen.util 4099 :invoke :read [16 nil] INFO [2019-12-17 12:30:23,559] jepsen worker 73 - jepsen.util 3403 :invoke :write [24 11] INFO [2019-12-17 12:30:23,561] jepsen worker 37 - jepsen.util 3367 :invoke :write [12 30] INFO [2019-12-17 12:30:23,561] jepsen worker 70 - jepsen.util 3940 :invoke :write [23 14] INFO [2019-12-17 12:30:23,581] jepsen worker 74 - jepsen.util 3764 :invoke :write [24 35] INFO [2019-12-17 12:30:23,584] jepsen worker 44 - jepsen.util 4994 :invoke :read [14 nil] INFO [2019-12-17 12:30:23,591] jepsen worker 23 - jepsen.util 4343 :info :write [7 28] :RequestCanceledException INFO [2019-12-17 12:30:23,605] jepsen worker 58 - jepsen.util 4558 :invoke :write [19 1] INFO [2019-12-17 12:30:23,626] jepsen worker 34 - jepsen.util 4534 :invoke :cas [11 [7 9]] INFO [2019-12-17 12:30:23,658] jepsen worker 19 - jepsen.util 3619 :fail :read [6 nil] :RequestTimeoutException INFO [2019-12-17 12:30:23,662] jepsen worker 19 - jepsen.util 3619 :invoke :cas [6 [19 46]] INFO [2019-12-17 12:30:23,689] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19686,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:30:23,702] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19686,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:30:23,786] jepsen worker 9 - jepsen.util 4059 :ok :read [3 46] INFO [2019-12-17 12:30:23,795] jepsen worker 31 - jepsen.util 2911 :invoke :cas [10 [12 21]] INFO [2019-12-17 12:30:23,797] jepsen worker 9 - jepsen.util 4059 :invoke :cas [3 [27 35]] INFO [2019-12-17 12:30:23,797] jepsen worker 56 - jepsen.util 3476 :info :write [18 40] :RequestCanceledException INFO [2019-12-17 12:30:23,835] jepsen worker 23 - jepsen.util 4433 :invoke :cas [7 [42 40]] INFO [2019-12-17 12:30:23,835] jepsen worker 12 - jepsen.util 3612 :info :write [4 12] :RequestCanceledException INFO [2019-12-17 12:30:23,852] jepsen worker 16 - jepsen.util 4966 :info :cas [5 [34 2]] :RequestCanceledException INFO [2019-12-17 12:30:23,920] jepsen worker 72 - jepsen.util 3312 :info :write [24 48] :RequestCanceledException INFO [2019-12-17 12:30:23,927] jepsen worker 35 - jepsen.util 4265 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:30:23,933] jepsen worker 71 - jepsen.util 3401 :info :write [23 38] :RequestCanceledException INFO [2019-12-17 12:30:23,942] jepsen worker 35 - jepsen.util 4265 :invoke :write [11 22] INFO [2019-12-17 12:30:23,949] jepsen worker 59 - jepsen.util 4469 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:30:23,950] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19694,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:30:23,952] jepsen worker 4 - jepsen.util 2614 :info :write [1 7] :RequestCanceledException INFO [2019-12-17 12:30:23,957] jepsen worker 59 - jepsen.util 4469 :invoke :read [19 nil] INFO [2019-12-17 12:30:23,975] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19694,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:30:23,981] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19696,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:30:23,985] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19696,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:30:23,993] jepsen worker 72 - jepsen.util 3402 :invoke :read [24 nil] INFO [2019-12-17 12:30:23,993] jepsen worker 56 - jepsen.util 3566 :invoke :write [18 36] INFO [2019-12-17 12:30:23,994] jepsen worker 15 - jepsen.util 4155 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:30:23,994] jepsen worker 15 - jepsen.util 4155 :invoke :write [5 18] INFO [2019-12-17 12:30:24,014] jepsen worker 84 - jepsen.util 4404 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:30:24,021] jepsen worker 84 - jepsen.util 4404 :invoke :read [28 nil] INFO [2019-12-17 12:30:24,038] jepsen worker 16 - jepsen.util 5056 :invoke :write [5 34] INFO [2019-12-17 12:30:24,043] jepsen worker 53 - jepsen.util 3473 :info :write [17 19] :RequestCanceledException INFO [2019-12-17 12:30:24,059] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19704,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:30:24,065] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19704,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:30:24,066] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19708,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:30:24,073] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19708,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:30:24,077] jepsen worker 4 - jepsen.util 2704 :invoke :read [1 nil] INFO [2019-12-17 12:30:24,085] jepsen worker 12 - jepsen.util 3702 :invoke :cas [4 [33 8]] INFO [2019-12-17 12:30:24,096] jepsen worker 53 - jepsen.util 3563 :invoke :read [17 nil] INFO [2019-12-17 12:30:24,129] jepsen worker 71 - jepsen.util 3491 :invoke :read [23 nil] INFO [2019-12-17 12:30:24,130] jepsen worker 43 - jepsen.util 4183 :info :cas [14 [18 32]] :RequestCanceledException INFO [2019-12-17 12:30:24,132] jepsen worker 55 - jepsen.util 4195 :info :write [18 32] :RequestCanceledException INFO [2019-12-17 12:30:24,168] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:30:24,170] jepsen nemesis - couchbase.util Rebalance check count: 146 INFO [2019-12-17 12:30:24,172] jepsen nemesis - couchbase.util Rebalance stuck count: 127 INFO [2019-12-17 12:30:24,193] jepsen worker 87 - jepsen.util 3147 :info :write [29 22] :RequestCanceledException INFO [2019-12-17 12:30:24,193] jepsen worker 43 - jepsen.util 4273 :invoke :read [14 nil] INFO [2019-12-17 12:30:24,195] jepsen worker 55 - jepsen.util 4285 :invoke :write [18 30] INFO [2019-12-17 12:30:24,200] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19718,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:30:24,203] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19718,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:30:24,206] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19720,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:30:24,206] jepsen worker 87 - jepsen.util 3237 :invoke :read [29 nil] INFO [2019-12-17 12:30:24,213] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19720,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:30:24,231] jepsen worker 38 - jepsen.util 3368 :info :cas [12 [6 32]] :RequestCanceledException INFO [2019-12-17 12:30:24,242] jepsen worker 85 - jepsen.util 5035 :info :write [28 15] :RequestCanceledException INFO [2019-12-17 12:30:24,253] jepsen worker 62 - jepsen.util 4292 :info :cas [20 [46 38]] :RequestCanceledException INFO [2019-12-17 12:30:24,253] jepsen worker 50 - jepsen.util 5000 :info :write [16 4] :RequestCanceledException INFO [2019-12-17 12:30:24,282] jepsen worker 38 - jepsen.util 3458 :invoke :write [12 38] INFO [2019-12-17 12:30:24,297] jepsen worker 1 - jepsen.util 4141 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:30:24,298] jepsen worker 1 - jepsen.util 4141 :invoke :write [0 13] INFO [2019-12-17 12:30:24,316] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19726,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:30:24,323] jepsen worker 85 - jepsen.util 5125 :invoke :read [28 nil] INFO [2019-12-17 12:30:24,324] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19726,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:30:24,336] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19728,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:30:24,337] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19728,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:30:24,341] jepsen worker 40 - jepsen.util 4810 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:30:24,344] jepsen worker 50 - jepsen.util 5090 :invoke :read [16 nil] INFO [2019-12-17 12:30:24,349] jepsen worker 40 - jepsen.util 4810 :invoke :cas [13 [21 5]] INFO [2019-12-17 12:30:24,394] jepsen worker 75 - jepsen.util 4665 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:30:24,395] jepsen worker 75 - jepsen.util 4665 :invoke :read [25 nil] INFO [2019-12-17 12:30:24,402] jepsen worker 62 - jepsen.util 4382 :invoke :read [20 nil] INFO [2019-12-17 12:30:24,408] jepsen worker 65 - jepsen.util 3035 :fail :cas [21 [46 28]] :ValueNotSwapFrom INFO [2019-12-17 12:30:24,408] jepsen worker 65 - jepsen.util 3035 :invoke :write [21 23] INFO [2019-12-17 12:30:24,421] jepsen worker 26 - jepsen.util 4796 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:30:24,421] jepsen worker 26 - jepsen.util 4796 :invoke :cas [8 [0 14]] INFO [2019-12-17 12:30:24,423] jepsen worker 45 - jepsen.util 3465 :ok :read [15 43] INFO [2019-12-17 12:30:24,424] jepsen worker 45 - jepsen.util 3465 :invoke :write [15 4] INFO [2019-12-17 12:30:24,426] jepsen worker 22 - jepsen.util 4702 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:30:24,427] jepsen worker 83 - jepsen.util 4763 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:30:24,427] jepsen worker 22 - jepsen.util 4702 :invoke :write [7 16] INFO [2019-12-17 12:30:24,427] jepsen worker 83 - jepsen.util 4763 :invoke :read [27 nil] INFO [2019-12-17 12:30:24,429] jepsen worker 8 - jepsen.util 3878 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:30:24,429] jepsen worker 8 - jepsen.util 3878 :invoke :write [2 14] INFO [2019-12-17 12:30:24,439] jepsen worker 88 - jepsen.util 3508 :info :cas [29 [1 29]] :RequestCanceledException INFO [2019-12-17 12:30:24,442] jepsen worker 46 - jepsen.util 3736 :ok :read [15 43] INFO [2019-12-17 12:30:24,443] jepsen worker 46 - jepsen.util 3736 :invoke :write [15 21] INFO [2019-12-17 12:30:24,454] jepsen worker 18 - jepsen.util 3528 :fail :cas [6 [38 25]] :ValueNotSwapFrom INFO [2019-12-17 12:30:24,455] jepsen worker 18 - jepsen.util 3528 :invoke :read [6 nil] INFO [2019-12-17 12:30:24,456] jepsen worker 42 - jepsen.util 5442 :info :cas [14 [19 26]] :RequestCanceledException INFO [2019-12-17 12:30:24,485] jepsen worker 88 - jepsen.util 3598 :invoke :read [29 nil] INFO [2019-12-17 12:30:24,489] jepsen worker 47 - jepsen.util 4007 :fail :cas [15 [4 13]] :ValueNotSwapFrom INFO [2019-12-17 12:30:24,489] jepsen worker 47 - jepsen.util 4007 :invoke :read [15 nil] INFO [2019-12-17 12:30:24,492] jepsen worker 42 - jepsen.util 5532 :invoke :read [14 nil] INFO [2019-12-17 12:30:24,510] jepsen worker 21 - jepsen.util 4521 :info :write [7 8] :RequestCanceledException INFO [2019-12-17 12:30:24,525] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19750,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:30:24,526] jepsen worker 57 - jepsen.util 4287 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:30:24,526] jepsen worker 57 - jepsen.util 4287 :invoke :cas [19 [30 43]] INFO [2019-12-17 12:30:24,527] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19750,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:30:24,531] jepsen worker 21 - jepsen.util 4611 :invoke :write [7 33] INFO [2019-12-17 12:30:24,532] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19754,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:30:24,536] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19754,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:30:24,537] jepsen worker 41 - jepsen.util 4631 :info :cas [13 [48 44]] :RequestCanceledException INFO [2019-12-17 12:30:24,565] jepsen worker 48 - jepsen.util 3918 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:30:24,565] jepsen worker 33 - jepsen.util 4803 :info :cas [11 [19 10]] :RequestCanceledException INFO [2019-12-17 12:30:24,566] jepsen worker 48 - jepsen.util 3918 :invoke :write [16 13] INFO [2019-12-17 12:30:24,576] jepsen worker 63 - jepsen.util 3123 :ok :write [21 0] INFO [2019-12-17 12:30:24,580] jepsen worker 63 - jepsen.util 3123 :invoke :read [21 nil] INFO [2019-12-17 12:30:24,618] jepsen worker 41 - jepsen.util 4721 :invoke :cas [13 [28 13]] INFO [2019-12-17 12:30:24,619] jepsen worker 33 - jepsen.util 4893 :invoke :read [11 nil] INFO [2019-12-17 12:30:24,631] jepsen worker 17 - jepsen.util 4427 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:30:24,632] jepsen worker 17 - jepsen.util 4427 :invoke :read [5 nil] INFO [2019-12-17 12:30:24,639] jepsen worker 0 - jepsen.util 5040 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:30:24,640] jepsen worker 0 - jepsen.util 5040 :invoke :cas [0 [43 3]] INFO [2019-12-17 12:30:24,728] jepsen worker 13 - jepsen.util 3253 :info :write [4 44] :RequestCanceledException INFO [2019-12-17 12:30:24,746] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19776,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:30:24,748] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19776,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:30:24,748] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19779,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:30:24,748] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19779,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:30:24,749] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19780,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:30:24,750] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19780,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:30:24,781] jepsen worker 13 - jepsen.util 3343 :invoke :cas [4 [45 21]] INFO [2019-12-17 12:30:24,787] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestTimeoutException: CarrierBucketConfigRequest {"retried":0,"reason":"TIMEOUT","requestId":19577,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:30:24,800] jepsen worker 74 - jepsen.util 3764 :ok :write [24 35] INFO [2019-12-17 12:30:24,807] jepsen worker 74 - jepsen.util 3764 :invoke :read [24 nil] INFO [2019-12-17 12:30:24,888] jepsen worker 51 - jepsen.util 3111 :ok :read [17 38] INFO [2019-12-17 12:30:24,889] jepsen worker 51 - jepsen.util 3111 :invoke :read [17 nil] INFO [2019-12-17 12:30:24,918] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19793,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:30:24,924] jepsen worker 30 - jepsen.util 3810 :ok :read [10 41] INFO [2019-12-17 12:30:24,928] jepsen worker 30 - jepsen.util 3810 :invoke :read [10 nil] INFO [2019-12-17 12:30:24,928] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19793,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:30:24,930] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19796,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:30:24,931] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19796,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:30:24,940] jepsen worker 65 - jepsen.util 3035 :ok :write [21 23] INFO [2019-12-17 12:30:24,940] jepsen worker 65 - jepsen.util 3035 :invoke :read [21 nil] INFO [2019-12-17 12:30:25,039] jepsen worker 36 - jepsen.util 3636 :fail :cas [12 [39 8]] :ValueNotSwapFrom INFO [2019-12-17 12:30:25,039] jepsen worker 36 - jepsen.util 3636 :invoke :cas [12 [47 12]] INFO [2019-12-17 12:30:25,040] jepsen worker 46 - jepsen.util 3736 :ok :write [15 21] INFO [2019-12-17 12:30:25,041] jepsen worker 46 - jepsen.util 3736 :invoke :read [15 nil] INFO [2019-12-17 12:30:25,070] jepsen worker 20 - jepsen.util 3080 :ok :read [6 46] INFO [2019-12-17 12:30:25,073] jepsen worker 20 - jepsen.util 3080 :invoke :write [6 49] INFO [2019-12-17 12:30:25,096] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19803,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:30:25,096] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19803,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:30:25,098] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19807,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:30:25,105] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19807,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:30:25,125] jepsen worker 76 - jepsen.util 4666 :info :cas [25 [32 34]] :RequestCanceledException INFO [2019-12-17 12:30:25,126] jepsen worker 3 - jepsen.util 3603 :info :cas [1 [33 11]] :RequestCanceledException INFO [2019-12-17 12:30:25,144] jepsen worker 3 - jepsen.util 3693 :invoke :read [1 nil] INFO [2019-12-17 12:30:25,163] jepsen worker 76 - jepsen.util 4756 :invoke :read [25 nil] INFO [2019-12-17 12:30:25,174] jepsen worker 73 - jepsen.util 3403 :ok :write [24 11] INFO [2019-12-17 12:30:25,175] jepsen worker 73 - jepsen.util 3403 :invoke :read [24 nil] INFO [2019-12-17 12:30:25,196] jepsen worker 61 - jepsen.util 5191 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:30:25,196] jepsen worker 61 - jepsen.util 5191 :invoke :cas [20 [24 38]] INFO [2019-12-17 12:30:25,202] jepsen worker 11 - jepsen.util 3701 :fail :cas [3 [10 42]] :ValueNotSwapFrom INFO [2019-12-17 12:30:25,206] jepsen worker 11 - jepsen.util 3701 :invoke :write [3 21] INFO [2019-12-17 12:30:25,209] jepsen worker 27 - jepsen.util 3177 :ok :read [9 23] INFO [2019-12-17 12:30:25,209] jepsen worker 27 - jepsen.util 3177 :invoke :cas [9 [3 3]] INFO [2019-12-17 12:30:25,209] jepsen worker 81 - jepsen.util 4491 :info :cas [27 [5 7]] :RequestCanceledException INFO [2019-12-17 12:30:25,210] jepsen worker 10 - jepsen.util 3340 :ok :read [3 21] INFO [2019-12-17 12:30:25,212] jepsen worker 10 - jepsen.util 3340 :invoke :cas [3 [34 8]] INFO [2019-12-17 12:30:25,221] jepsen worker 28 - jepsen.util 2458 :ok :read [9 23] INFO [2019-12-17 12:30:25,221] jepsen worker 28 - jepsen.util 2458 :invoke :read [9 nil] INFO [2019-12-17 12:30:25,235] jepsen worker 14 - jepsen.util 3434 :fail :read [4 nil] :RequestTimeoutException INFO [2019-12-17 12:30:25,235] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19824,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:30:25,236] jepsen worker 14 - jepsen.util 3434 :invoke :write [4 16] INFO [2019-12-17 12:30:25,236] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19824,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:30:25,238] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19826,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:30:25,239] jepsen worker 29 - jepsen.util 2819 :fail :cas [9 [20 47]] :ValueNotSwapFrom INFO [2019-12-17 12:30:25,239] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19826,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:30:25,241] jepsen worker 29 - jepsen.util 2819 :invoke :write [9 40] INFO [2019-12-17 12:30:25,245] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19814,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:30:25,249] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19814,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:30:25,250] jepsen worker 81 - jepsen.util 4581 :invoke :cas [27 [3 10]] INFO [2019-12-17 12:30:25,252] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19816,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:30:25,256] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19816,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:30:25,268] jepsen worker 31 - jepsen.util 2911 :fail :cas [10 [12 21]] :ValueNotSwapFrom INFO [2019-12-17 12:30:25,269] jepsen worker 31 - jepsen.util 2911 :invoke :read [10 nil] INFO [2019-12-17 12:30:25,273] jepsen worker 72 - jepsen.util 3402 :ok :read [24 31] INFO [2019-12-17 12:30:25,274] jepsen worker 72 - jepsen.util 3402 :invoke :write [24 46] INFO [2019-12-17 12:30:25,278] jepsen worker 32 - jepsen.util 3902 :fail :cas [10 [42 49]] :ValueNotSwapFrom INFO [2019-12-17 12:30:25,279] jepsen worker 32 - jepsen.util 3902 :invoke :read [10 nil] INFO [2019-12-17 12:30:25,286] jepsen worker 19 - jepsen.util 3619 :fail :cas [6 [19 46]] :ValueNotSwapFrom INFO [2019-12-17 12:30:25,287] jepsen worker 19 - jepsen.util 3619 :invoke :read [6 nil] INFO [2019-12-17 12:30:25,409] jepsen worker 69 - jepsen.util 3399 :ok :read [23 32] INFO [2019-12-17 12:30:25,410] jepsen worker 69 - jepsen.util 3399 :invoke :read [23 nil] INFO [2019-12-17 12:30:25,436] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19840,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:30:25,439] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19840,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:30:25,442] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19841,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:30:25,442] jepsen worker 5 - jepsen.util 3155 :ok :read [1 8] INFO [2019-12-17 12:30:25,442] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19841,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:30:25,443] jepsen worker 5 - jepsen.util 3155 :invoke :read [1 nil] INFO [2019-12-17 12:30:25,492] jepsen worker 87 - jepsen.util 3237 :ok :read [29 22] INFO [2019-12-17 12:30:25,493] jepsen worker 87 - jepsen.util 3237 :invoke :read [29 nil] INFO [2019-12-17 12:30:25,493] jepsen worker 4 - jepsen.util 2704 :ok :read [1 8] INFO [2019-12-17 12:30:25,494] jepsen worker 4 - jepsen.util 2704 :invoke :read [1 nil] INFO [2019-12-17 12:30:25,500] jepsen worker 78 - jepsen.util 3858 :fail :cas [26 [40 11]] :ValueNotSwapFrom INFO [2019-12-17 12:30:25,501] jepsen worker 78 - jepsen.util 3858 :invoke :cas [26 [47 20]] INFO [2019-12-17 12:30:25,510] jepsen worker 68 - jepsen.util 4118 :info :cas [22 [31 17]] :RequestCanceledException INFO [2019-12-17 12:30:25,522] jepsen worker 68 - jepsen.util 4208 :invoke :cas [22 [25 44]] INFO [2019-12-17 12:30:25,524] jepsen worker 77 - jepsen.util 4397 :info :cas [25 [22 38]] :RequestCanceledException INFO [2019-12-17 12:30:25,545] jepsen worker 77 - jepsen.util 4487 :invoke :write [25 49] INFO [2019-12-17 12:30:25,549] jepsen worker 18 - jepsen.util 3528 :ok :read [6 46] INFO [2019-12-17 12:30:25,551] jepsen worker 18 - jepsen.util 3528 :invoke :read [6 nil] INFO [2019-12-17 12:30:25,553] jepsen worker 86 - jepsen.util 4766 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:30:25,553] jepsen worker 86 - jepsen.util 4766 :invoke :read [28 nil] INFO [2019-12-17 12:30:25,581] jepsen worker 74 - jepsen.util 3764 :ok :read [24 35] INFO [2019-12-17 12:30:25,582] jepsen worker 74 - jepsen.util 3764 :invoke :cas [24 [31 14]] INFO [2019-12-17 12:30:25,586] jepsen worker 65 - jepsen.util 3035 :ok :read [21 23] INFO [2019-12-17 12:30:25,587] jepsen worker 65 - jepsen.util 3035 :invoke :cas [21 [16 29]] INFO [2019-12-17 12:30:25,588] jepsen worker 63 - jepsen.util 3123 :ok :read [21 0] INFO [2019-12-17 12:30:25,589] jepsen worker 63 - jepsen.util 3123 :invoke :cas [21 [2 39]] INFO [2019-12-17 12:30:25,601] jepsen worker 24 - jepsen.util 4434 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:30:25,602] jepsen worker 24 - jepsen.util 4434 :invoke :cas [8 [11 29]] INFO [2019-12-17 12:30:25,625] jepsen worker 49 - jepsen.util 4099 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:30:25,625] jepsen worker 49 - jepsen.util 4099 :invoke :read [16 nil] INFO [2019-12-17 12:30:25,639] jepsen worker 64 - jepsen.util 2764 :ok :write [21 48] INFO [2019-12-17 12:30:25,640] jepsen worker 64 - jepsen.util 2764 :invoke :write [21 26] INFO [2019-12-17 12:30:25,656] jepsen worker 44 - jepsen.util 4994 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:30:25,657] jepsen worker 44 - jepsen.util 4994 :invoke :cas [14 [11 16]] INFO [2019-12-17 12:30:25,657] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19861,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:30:25,658] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19861,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:30:25,658] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19863,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:30:25,659] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19863,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:30:25,665] jepsen worker 29 - jepsen.util 2819 :ok :write [9 40] INFO [2019-12-17 12:30:25,665] jepsen worker 29 - jepsen.util 2819 :invoke :cas [9 [15 48]] INFO [2019-12-17 12:30:25,700] jepsen worker 34 - jepsen.util 4534 :info :cas [11 [7 9]] :RequestCanceledException INFO [2019-12-17 12:30:25,706] jepsen worker 46 - jepsen.util 3736 :ok :read [15 21] INFO [2019-12-17 12:30:25,707] jepsen worker 46 - jepsen.util 3736 :invoke :write [15 30] INFO [2019-12-17 12:30:25,763] jepsen worker 34 - jepsen.util 4624 :invoke :cas [11 [21 38]] INFO [2019-12-17 12:30:25,796] jepsen worker 56 - jepsen.util 3566 :ok :write [18 36] INFO [2019-12-17 12:30:25,797] jepsen worker 56 - jepsen.util 3566 :invoke :write [18 9] INFO [2019-12-17 12:30:25,856] jepsen worker 30 - jepsen.util 3810 :ok :read [10 41] INFO [2019-12-17 12:30:25,857] jepsen worker 30 - jepsen.util 3810 :invoke :write [10 11] INFO [2019-12-17 12:30:25,892] jepsen worker 45 - jepsen.util 3465 :ok :write [15 4] INFO [2019-12-17 12:30:25,893] jepsen worker 45 - jepsen.util 3465 :invoke :write [15 43] INFO [2019-12-17 12:30:25,935] jepsen worker 71 - jepsen.util 3491 :ok :read [23 38] INFO [2019-12-17 12:30:25,936] jepsen worker 71 - jepsen.util 3491 :invoke :cas [23 [35 2]] INFO [2019-12-17 12:30:25,940] jepsen worker 47 - jepsen.util 4007 :ok :read [15 21] INFO [2019-12-17 12:30:25,942] jepsen worker 47 - jepsen.util 4007 :invoke :cas [15 [40 41]] INFO [2019-12-17 12:30:25,944] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19870,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:30:25,948] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19870,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:30:25,955] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19874,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:30:25,958] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19874,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:30:25,978] jepsen worker 11 - jepsen.util 3701 :ok :write [3 21] INFO [2019-12-17 12:30:25,978] jepsen worker 11 - jepsen.util 3701 :invoke :read [3 nil] INFO [2019-12-17 12:30:25,997] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19882,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:30:25,999] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19882,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:30:26,015] jepsen worker 37 - jepsen.util 3367 :ok :write [12 30] INFO [2019-12-17 12:30:26,016] jepsen worker 37 - jepsen.util 3367 :invoke :read [12 nil] INFO [2019-12-17 12:30:26,022] jepsen worker 23 - jepsen.util 4433 :info :cas [7 [42 40]] :RequestCanceledException INFO [2019-12-17 12:30:26,022] jepsen worker 79 - jepsen.util 3499 :info :write [26 48] :RequestCanceledException INFO [2019-12-17 12:30:26,031] jepsen worker 9 - jepsen.util 4059 :fail :cas [3 [27 35]] :ValueNotSwapFrom INFO [2019-12-17 12:30:26,031] jepsen worker 9 - jepsen.util 4059 :invoke :read [3 nil] INFO [2019-12-17 12:30:26,032] jepsen worker 79 - jepsen.util 3589 :invoke :read [26 nil] INFO [2019-12-17 12:30:26,033] jepsen worker 23 - jepsen.util 4523 :invoke :cas [7 [10 18]] INFO [2019-12-17 12:30:26,041] jepsen worker 59 - jepsen.util 4469 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:30:26,041] jepsen worker 59 - jepsen.util 4469 :invoke :cas [19 [44 3]] INFO [2019-12-17 12:30:26,044] jepsen worker 28 - jepsen.util 2458 :ok :read [9 23] INFO [2019-12-17 12:30:26,045] jepsen worker 28 - jepsen.util 2458 :invoke :read [9 nil] INFO [2019-12-17 12:30:26,048] jepsen worker 10 - jepsen.util 3340 :fail :cas [3 [34 8]] :ValueNotSwapFrom INFO [2019-12-17 12:30:26,048] jepsen worker 10 - jepsen.util 3340 :invoke :write [3 18] INFO [2019-12-17 12:30:26,090] jepsen worker 54 - jepsen.util 3744 :ok :write [18 34] INFO [2019-12-17 12:30:26,090] jepsen worker 54 - jepsen.util 3744 :invoke :write [18 32] INFO [2019-12-17 12:30:26,114] jepsen worker 84 - jepsen.util 4404 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:30:26,114] jepsen worker 84 - jepsen.util 4404 :invoke :write [28 17] INFO [2019-12-17 12:30:26,151] jepsen worker 89 - jepsen.util 3239 :ok :write [29 37] INFO [2019-12-17 12:30:26,151] jepsen worker 89 - jepsen.util 3239 :invoke :write [29 40] INFO [2019-12-17 12:30:26,215] jepsen worker 31 - jepsen.util 2911 :ok :read [10 41] INFO [2019-12-17 12:30:26,216] jepsen worker 31 - jepsen.util 2911 :invoke :write [10 18] INFO [2019-12-17 12:30:26,227] jepsen worker 56 - jepsen.util 3566 :ok :write [18 9] INFO [2019-12-17 12:30:26,228] jepsen worker 56 - jepsen.util 3566 :invoke :read [18 nil] INFO [2019-12-17 12:30:26,239] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19903,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:30:26,239] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19903,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:30:26,240] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19905,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:30:26,241] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19905,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:30:26,255] jepsen worker 43 - jepsen.util 4273 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:30:26,256] jepsen worker 39 - jepsen.util 4179 :info :write [13 25] :RequestCanceledException INFO [2019-12-17 12:30:26,256] jepsen worker 43 - jepsen.util 4273 :invoke :read [14 nil] INFO [2019-12-17 12:30:26,274] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19894,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:30:26,281] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19894,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:30:26,283] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19895,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:30:26,286] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19895,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:30:26,290] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19896,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:30:26,292] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19896,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:30:26,293] jepsen worker 66 - jepsen.util 4386 :info :write [22 27] :RequestCanceledException INFO [2019-12-17 12:30:26,311] jepsen worker 39 - jepsen.util 4269 :invoke :write [13 32] INFO [2019-12-17 12:30:26,320] jepsen worker 38 - jepsen.util 3458 :ok :write [12 38] INFO [2019-12-17 12:30:26,321] jepsen worker 38 - jepsen.util 3458 :invoke :write [12 38] INFO [2019-12-17 12:30:26,322] jepsen worker 73 - jepsen.util 3403 :ok :read [24 11] INFO [2019-12-17 12:30:26,323] jepsen worker 73 - jepsen.util 3403 :invoke :read [24 nil] INFO [2019-12-17 12:30:26,337] jepsen worker 66 - jepsen.util 4476 :invoke :cas [22 [28 12]] INFO [2019-12-17 12:30:26,355] jepsen worker 80 - jepsen.util 3410 :ok :write [26 49] INFO [2019-12-17 12:30:26,355] jepsen worker 80 - jepsen.util 3410 :invoke :read [26 nil] INFO [2019-12-17 12:30:26,405] jepsen worker 63 - jepsen.util 3123 :fail :cas [21 [2 39]] :ValueNotSwapFrom INFO [2019-12-17 12:30:26,406] jepsen worker 63 - jepsen.util 3123 :invoke :read [21 nil] INFO [2019-12-17 12:30:26,422] jepsen worker 85 - jepsen.util 5125 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:30:26,422] jepsen worker 2 - jepsen.util 4232 :info :write [0 17] :RequestCanceledException INFO [2019-12-17 12:30:26,422] jepsen worker 58 - jepsen.util 4558 :info :write [19 1] :RequestCanceledException INFO [2019-12-17 12:30:26,422] jepsen worker 70 - jepsen.util 3940 :info :write [23 14] :RequestCanceledException INFO [2019-12-17 12:30:26,422] jepsen worker 85 - jepsen.util 5125 :invoke :read [28 nil] INFO [2019-12-17 12:30:26,422] jepsen worker 7 - jepsen.util 4327 :info :write [2 17] :RequestCanceledException INFO [2019-12-17 12:30:26,438] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19921,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:30:26,439] jepsen worker 50 - jepsen.util 5090 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:30:26,440] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19921,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:30:26,440] jepsen worker 6 - jepsen.util 4866 :info :write [2 13] :RequestCanceledException INFO [2019-12-17 12:30:26,440] jepsen worker 50 - jepsen.util 5090 :invoke :read [16 nil] INFO [2019-12-17 12:30:26,440] jepsen worker 40 - jepsen.util 4810 :info :cas [13 [21 5]] :RequestCanceledException INFO [2019-12-17 12:30:26,446] jepsen worker 70 - jepsen.util 4030 :invoke :cas [23 [37 17]] INFO [2019-12-17 12:30:26,447] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19922,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:30:26,449] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19922,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:30:26,450] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19923,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:30:26,452] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19923,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:30:26,456] jepsen worker 62 - jepsen.util 4382 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:30:26,456] jepsen worker 25 - jepsen.util 4795 :info :write [8 26] :RequestCanceledException INFO [2019-12-17 12:30:26,456] jepsen worker 62 - jepsen.util 4382 :invoke :cas [20 [15 28]] INFO [2019-12-17 12:30:26,457] jepsen worker 58 - jepsen.util 4648 :invoke :cas [19 [30 4]] INFO [2019-12-17 12:30:26,460] jepsen worker 29 - jepsen.util 2819 :fail :cas [9 [15 48]] :ValueNotSwapFrom INFO [2019-12-17 12:30:26,460] jepsen worker 29 - jepsen.util 2819 :invoke :cas [9 [40 24]] INFO [2019-12-17 12:30:26,464] jepsen worker 87 - jepsen.util 3237 :ok :read [29 22] INFO [2019-12-17 12:30:26,464] jepsen worker 7 - jepsen.util 4417 :invoke :read [2 nil] INFO [2019-12-17 12:30:26,464] jepsen worker 87 - jepsen.util 3237 :invoke :read [29 nil] INFO [2019-12-17 12:30:26,466] jepsen worker 2 - jepsen.util 4322 :invoke :write [0 31] INFO [2019-12-17 12:30:26,466] jepsen worker 18 - jepsen.util 3528 :ok :read [6 46] INFO [2019-12-17 12:30:26,474] jepsen worker 4 - jepsen.util 2704 :ok :read [1 32] INFO [2019-12-17 12:30:26,474] jepsen worker 40 - jepsen.util 4900 :invoke :write [13 11] INFO [2019-12-17 12:30:26,474] jepsen worker 4 - jepsen.util 2704 :invoke :cas [1 [36 3]] INFO [2019-12-17 12:30:26,478] jepsen worker 18 - jepsen.util 3528 :invoke :read [6 nil] INFO [2019-12-17 12:30:26,479] jepsen worker 25 - jepsen.util 4885 :invoke :cas [8 [23 16]] INFO [2019-12-17 12:30:26,482] jepsen worker 12 - jepsen.util 3702 :fail :cas [4 [33 8]] :ValueNotSwapFrom INFO [2019-12-17 12:30:26,483] jepsen worker 6 - jepsen.util 4956 :invoke :read [2 nil] INFO [2019-12-17 12:30:26,483] jepsen worker 12 - jepsen.util 3702 :invoke :read [4 nil] INFO [2019-12-17 12:30:26,491] jepsen worker 82 - jepsen.util 5392 :info :write [27 34] :RequestCanceledException INFO [2019-12-17 12:30:26,491] jepsen worker 75 - jepsen.util 4665 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:30:26,492] jepsen worker 75 - jepsen.util 4665 :invoke :read [25 nil] INFO [2019-12-17 12:30:26,510] jepsen worker 82 - jepsen.util 5482 :invoke :cas [27 [19 28]] INFO [2019-12-17 12:30:26,517] jepsen worker 60 - jepsen.util 4830 :info :write [20 10] :RequestCanceledException INFO [2019-12-17 12:30:26,518] jepsen worker 26 - jepsen.util 4796 :info :cas [8 [0 14]] :RequestCanceledException INFO [2019-12-17 12:30:26,533] jepsen worker 83 - jepsen.util 4763 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:30:26,534] jepsen worker 83 - jepsen.util 4763 :invoke :read [27 nil] INFO [2019-12-17 12:30:26,534] jepsen worker 88 - jepsen.util 3598 :fail :read [29 nil] :RequestTimeoutException INFO [2019-12-17 12:30:26,534] jepsen worker 88 - jepsen.util 3598 :invoke :write [29 45] INFO [2019-12-17 12:30:26,542] jepsen worker 67 - jepsen.util 4927 :info :write [22 40] :RequestCanceledException INFO [2019-12-17 12:30:26,564] jepsen worker 60 - jepsen.util 4920 :invoke :read [20 nil] INFO [2019-12-17 12:30:26,564] jepsen worker 32 - jepsen.util 3902 :ok :read [10 41] INFO [2019-12-17 12:30:26,565] jepsen worker 32 - jepsen.util 3902 :invoke :write [10 11] INFO [2019-12-17 12:30:26,568] jepsen worker 13 - jepsen.util 3343 :fail :cas [4 [45 21]] :ValueNotSwapFrom INFO [2019-12-17 12:30:26,568] jepsen worker 26 - jepsen.util 4886 :invoke :cas [8 [29 34]] INFO [2019-12-17 12:30:26,568] jepsen worker 13 - jepsen.util 3343 :invoke :write [4 38] INFO [2019-12-17 12:30:26,573] jepsen worker 67 - jepsen.util 5017 :invoke :read [22 nil] INFO [2019-12-17 12:30:26,587] jepsen worker 65 - jepsen.util 3035 :fail :cas [21 [16 29]] :ValueNotSwapFrom INFO [2019-12-17 12:30:26,587] jepsen worker 42 - jepsen.util 5532 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:30:26,587] jepsen worker 65 - jepsen.util 3035 :invoke :read [21 nil] INFO [2019-12-17 12:30:26,587] jepsen worker 42 - jepsen.util 5532 :invoke :read [14 nil] INFO [2019-12-17 12:30:26,611] jepsen worker 55 - jepsen.util 4285 :ok :write [18 30] INFO [2019-12-17 12:30:26,611] jepsen worker 55 - jepsen.util 4285 :invoke :read [18 nil] INFO [2019-12-17 12:30:26,626] jepsen worker 57 - jepsen.util 4287 :info :cas [19 [30 43]] :RequestCanceledException INFO [2019-12-17 12:30:26,627] jepsen worker 16 - jepsen.util 5056 :info :write [5 34] :RequestCanceledException INFO [2019-12-17 12:30:26,627] jepsen worker 35 - jepsen.util 4265 :info :write [11 22] :RequestCanceledException INFO [2019-12-17 12:30:26,628] jepsen worker 22 - jepsen.util 4702 :info :write [7 16] :RequestCanceledException INFO [2019-12-17 12:30:26,635] jepsen worker 53 - jepsen.util 3563 :ok :read [17 19] INFO [2019-12-17 12:30:26,636] jepsen worker 53 - jepsen.util 3563 :invoke :read [17 nil] INFO [2019-12-17 12:30:26,638] jepsen worker 33 - jepsen.util 4893 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:30:26,639] jepsen worker 33 - jepsen.util 4893 :invoke :cas [11 [18 47]] INFO [2019-12-17 12:30:26,649] jepsen worker 22 - jepsen.util 4792 :invoke :cas [7 [41 46]] INFO [2019-12-17 12:30:26,649] jepsen worker 35 - jepsen.util 4355 :invoke :cas [11 [9 6]] INFO [2019-12-17 12:30:26,651] jepsen worker 57 - jepsen.util 4377 :invoke :write [19 17] INFO [2019-12-17 12:30:26,651] jepsen worker 16 - jepsen.util 5146 :invoke :write [5 25] INFO [2019-12-17 12:30:26,665] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19952,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:30:26,665] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19952,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:30:26,665] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19956,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:30:26,665] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19956,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:30:26,683] jepsen worker 51 - jepsen.util 3111 :ok :read [17 19] INFO [2019-12-17 12:30:26,684] jepsen worker 51 - jepsen.util 3111 :invoke :cas [17 [35 49]] INFO [2019-12-17 12:30:26,692] jepsen worker 15 - jepsen.util 4155 :info :write [5 18] :RequestCanceledException INFO [2019-12-17 12:30:26,716] jepsen worker 15 - jepsen.util 4245 :invoke :write [5 28] INFO [2019-12-17 12:30:26,722] jepsen worker 14 - jepsen.util 3434 :ok :write [4 16] INFO [2019-12-17 12:30:26,723] jepsen worker 14 - jepsen.util 3434 :invoke :cas [4 [24 39]] INFO [2019-12-17 12:30:26,724] jepsen worker 8 - jepsen.util 3878 :info :write [2 14] :RequestCanceledException INFO [2019-12-17 12:30:26,728] jepsen worker 17 - jepsen.util 4427 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:30:26,729] jepsen worker 17 - jepsen.util 4427 :invoke :read [5 nil] INFO [2019-12-17 12:30:26,732] jepsen worker 41 - jepsen.util 4721 :info :cas [13 [28 13]] :RequestCanceledException INFO [2019-12-17 12:30:26,742] jepsen worker 0 - jepsen.util 5040 :info :cas [0 [43 3]] :RequestCanceledException INFO [2019-12-17 12:30:26,754] jepsen worker 8 - jepsen.util 3968 :invoke :read [2 nil] INFO [2019-12-17 12:30:26,755] jepsen worker 41 - jepsen.util 4811 :invoke :cas [13 [18 15]] INFO [2019-12-17 12:30:26,755] jepsen worker 48 - jepsen.util 3918 :info :write [16 13] :RequestCanceledException INFO [2019-12-17 12:30:26,771] jepsen worker 0 - jepsen.util 5130 :invoke :read [0 nil] INFO [2019-12-17 12:30:26,793] jepsen worker 1 - jepsen.util 4141 :info :write [0 13] :RequestCanceledException INFO [2019-12-17 12:30:26,796] jepsen worker 48 - jepsen.util 4008 :invoke :cas [16 [20 41]] INFO [2019-12-17 12:30:27,168] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19971,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:30:27,168] jepsen worker 76 - jepsen.util 4756 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:30:27,168] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19971,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:30:27,168] jepsen worker 76 - jepsen.util 4756 :invoke :read [25 nil] INFO [2019-12-17 12:30:27,169] jepsen worker 71 - jepsen.util 3491 :fail :cas [23 [35 2]] :ValueNotSwapFrom INFO [2019-12-17 12:30:27,169] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19977,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:30:27,169] jepsen worker 71 - jepsen.util 3491 :invoke :write [23 22] INFO [2019-12-17 12:30:27,169] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19977,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:30:27,170] jepsen worker 1 - jepsen.util 4231 :invoke :read [0 nil] INFO [2019-12-17 12:30:27,193] jepsen worker 9 - jepsen.util 4059 :ok :read [3 21] INFO [2019-12-17 12:30:27,193] jepsen worker 52 - jepsen.util 3562 :fail :cas [17 [38 48]] :CASMismatchException INFO [2019-12-17 12:30:27,193] jepsen worker 9 - jepsen.util 4059 :invoke :read [3 nil] INFO [2019-12-17 12:30:27,193] jepsen worker 4 - jepsen.util 2704 :fail :cas [1 [36 3]] :ValueNotSwapFrom INFO [2019-12-17 12:30:27,193] jepsen worker 21 - jepsen.util 4611 :info :write [7 33] :RequestCanceledException INFO [2019-12-17 12:30:27,193] jepsen worker 78 - jepsen.util 3858 :fail :cas [26 [47 20]] :ValueNotSwapFrom INFO [2019-12-17 12:30:27,193] jepsen worker 28 - jepsen.util 2458 :ok :read [9 40] INFO [2019-12-17 12:30:27,193] jepsen worker 63 - jepsen.util 3123 :ok :read [21 48] INFO [2019-12-17 12:30:27,193] jepsen worker 4 - jepsen.util 2704 :invoke :write [1 18] INFO [2019-12-17 12:30:27,193] jepsen worker 56 - jepsen.util 3566 :ok :read [18 9] INFO [2019-12-17 12:30:27,194] jepsen worker 3 - jepsen.util 3693 :ok :read [1 32] INFO [2019-12-17 12:30:27,194] jepsen worker 52 - jepsen.util 3562 :invoke :write [17 33] INFO [2019-12-17 12:30:27,194] jepsen worker 28 - jepsen.util 2458 :invoke :read [9 nil] INFO [2019-12-17 12:30:27,194] jepsen worker 78 - jepsen.util 3858 :invoke :cas [26 [37 48]] INFO [2019-12-17 12:30:27,194] jepsen worker 63 - jepsen.util 3123 :invoke :cas [21 [5 25]] INFO [2019-12-17 12:30:27,195] jepsen worker 3 - jepsen.util 3693 :invoke :cas [1 [24 10]] INFO [2019-12-17 12:30:27,195] jepsen worker 56 - jepsen.util 3566 :invoke :write [18 36] INFO [2019-12-17 12:30:27,204] jepsen worker 21 - jepsen.util 4701 :invoke :write [7 17] INFO [2019-12-17 12:30:27,220] jepsen worker 54 - jepsen.util 3744 :ok :write [18 32] INFO [2019-12-17 12:30:27,220] jepsen worker 54 - jepsen.util 3744 :invoke :read [18 nil] INFO [2019-12-17 12:30:27,232] jepsen worker 36 - jepsen.util 3636 :fail :cas [12 [47 12]] :ValueNotSwapFrom INFO [2019-12-17 12:30:27,233] jepsen worker 36 - jepsen.util 3636 :invoke :write [12 11] INFO [2019-12-17 12:30:27,238] jepsen worker 18 - jepsen.util 3528 :ok :read [6 46] INFO [2019-12-17 12:30:27,239] jepsen worker 18 - jepsen.util 3528 :invoke :read [6 nil] INFO [2019-12-17 12:30:27,240] jepsen worker 87 - jepsen.util 3237 :ok :read [29 37] INFO [2019-12-17 12:30:27,240] jepsen worker 87 - jepsen.util 3237 :invoke :write [29 48] INFO [2019-12-17 12:30:27,241] jepsen worker 70 - jepsen.util 4030 :fail :cas [23 [37 17]] :ValueNotSwapFrom INFO [2019-12-17 12:30:27,242] jepsen worker 70 - jepsen.util 4030 :invoke :cas [23 [6 4]] INFO [2019-12-17 12:30:27,252] jepsen worker 27 - jepsen.util 3177 :info :cas [9 [3 3]] :RequestCanceledException INFO [2019-12-17 12:30:27,262] jepsen worker 27 - jepsen.util 3267 :invoke :read [9 nil] INFO [2019-12-17 12:30:27,278] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19988,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:30:27,278] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19980,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:30:27,279] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19988,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:30:27,279] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":19980,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:30:27,279] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20000,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:30:27,279] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20000,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:30:27,279] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20002,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:30:27,279] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20002,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:30:27,283] jepsen worker 61 - jepsen.util 5191 :info :cas [20 [24 38]] :RequestCanceledException INFO [2019-12-17 12:30:27,292] jepsen worker 61 - jepsen.util 5281 :invoke :write [20 3] INFO [2019-12-17 12:30:27,299] jepsen worker 69 - jepsen.util 3399 :ok :read [23 14] INFO [2019-12-17 12:30:27,299] jepsen worker 69 - jepsen.util 3399 :invoke :cas [23 [43 39]] INFO [2019-12-17 12:30:27,308] jepsen worker 81 - jepsen.util 4581 :info :cas [27 [3 10]] :RequestCanceledException INFO [2019-12-17 12:30:27,315] jepsen worker 79 - jepsen.util 3589 :ok :read [26 49] INFO [2019-12-17 12:30:27,315] jepsen worker 81 - jepsen.util 4671 :invoke :cas [27 [35 11]] INFO [2019-12-17 12:30:27,315] jepsen worker 79 - jepsen.util 3589 :invoke :read [26 nil] INFO [2019-12-17 12:30:27,316] jepsen worker 5 - jepsen.util 3155 :ok :read [1 32] INFO [2019-12-17 12:30:27,316] jepsen worker 5 - jepsen.util 3155 :invoke :cas [1 [20 47]] INFO [2019-12-17 12:30:27,320] jepsen worker 28 - jepsen.util 2458 :ok :read [9 40] INFO [2019-12-17 12:30:27,320] jepsen worker 28 - jepsen.util 2458 :invoke :cas [9 [44 45]] INFO [2019-12-17 12:30:27,321] jepsen worker 9 - jepsen.util 4059 :ok :read [3 21] INFO [2019-12-17 12:30:27,321] jepsen worker 9 - jepsen.util 4059 :invoke :write [3 32] INFO [2019-12-17 12:30:27,335] jepsen worker 18 - jepsen.util 3528 :ok :read [6 46] INFO [2019-12-17 12:30:27,335] jepsen worker 18 - jepsen.util 3528 :invoke :write [6 23] INFO [2019-12-17 12:30:27,336] jepsen worker 63 - jepsen.util 3123 :fail :cas [21 [5 25]] :ValueNotSwapFrom INFO [2019-12-17 12:30:27,336] jepsen worker 63 - jepsen.util 3123 :invoke :write [21 17] INFO [2019-12-17 12:30:27,355] jepsen worker 19 - jepsen.util 3619 :fail :read [6 nil] :RequestTimeoutException INFO [2019-12-17 12:30:27,355] jepsen worker 19 - jepsen.util 3619 :invoke :cas [6 [27 7]] INFO [2019-12-17 12:30:27,365] jepsen worker 28 - jepsen.util 2458 :fail :cas [9 [44 45]] :ValueNotSwapFrom INFO [2019-12-17 12:30:27,366] jepsen worker 28 - jepsen.util 2458 :invoke :write [9 27] INFO [2019-12-17 12:30:27,382] jepsen worker 20 - jepsen.util 3080 :info :write [6 49] :RequestCanceledException INFO [2019-12-17 12:30:27,386] jepsen worker 20 - jepsen.util 3170 :invoke :write [6 40] INFO [2019-12-17 12:30:27,399] jepsen worker 37 - jepsen.util 3367 :ok :read [12 38] INFO [2019-12-17 12:30:27,399] jepsen worker 37 - jepsen.util 3367 :invoke :write [12 11] INFO [2019-12-17 12:30:27,410] jepsen worker 80 - jepsen.util 3410 :ok :read [26 49] INFO [2019-12-17 12:30:27,411] jepsen worker 80 - jepsen.util 3410 :invoke :write [26 37] INFO [2019-12-17 12:30:27,415] jepsen worker 14 - jepsen.util 3434 :fail :cas [4 [24 39]] :ValueNotSwapFrom INFO [2019-12-17 12:30:27,415] jepsen worker 14 - jepsen.util 3434 :invoke :cas [4 [29 26]] INFO [2019-12-17 12:30:27,436] jepsen worker 53 - jepsen.util 3563 :ok :read [17 19] INFO [2019-12-17 12:30:27,436] jepsen worker 53 - jepsen.util 3563 :invoke :cas [17 [30 15]] INFO [2019-12-17 12:30:27,442] jepsen worker 70 - jepsen.util 4030 :fail :cas [23 [6 4]] :ValueNotSwapFrom INFO [2019-12-17 12:30:27,443] jepsen worker 70 - jepsen.util 4030 :invoke :write [23 16] INFO [2019-12-17 12:30:27,448] jepsen worker 19 - jepsen.util 3619 :fail :cas [6 [27 7]] :ValueNotSwapFrom INFO [2019-12-17 12:30:27,448] jepsen worker 19 - jepsen.util 3619 :invoke :write [6 45] INFO [2019-12-17 12:30:27,485] jepsen worker 88 - jepsen.util 3598 :ok :write [29 45] INFO [2019-12-17 12:30:27,485] jepsen worker 88 - jepsen.util 3598 :invoke :read [29 nil] INFO [2019-12-17 12:30:27,488] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20030,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:30:27,488] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20030,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:30:27,488] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20033,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:30:27,488] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20033,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:30:27,488] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20034,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:30:27,488] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20034,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:30:27,538] jepsen worker 72 - jepsen.util 3402 :info :write [24 46] :RequestCanceledException INFO [2019-12-17 12:30:27,538] jepsen worker 55 - jepsen.util 4285 :ok :read [18 32] INFO [2019-12-17 12:30:27,538] jepsen worker 55 - jepsen.util 4285 :invoke :cas [18 [25 25]] INFO [2019-12-17 12:30:27,540] jepsen worker 72 - jepsen.util 3492 :invoke :cas [24 [30 26]] INFO [2019-12-17 12:30:27,575] jepsen worker 46 - jepsen.util 3736 :ok :write [15 30] INFO [2019-12-17 12:30:27,575] jepsen worker 46 - jepsen.util 3736 :invoke :read [15 nil] INFO [2019-12-17 12:30:27,590] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:30:27,590] jepsen nemesis - couchbase.util Rebalance check count: 147 INFO [2019-12-17 12:30:27,590] jepsen nemesis - couchbase.util Rebalance stuck count: 128 INFO [2019-12-17 12:30:27,624] jepsen worker 68 - jepsen.util 4208 :info :cas [22 [25 44]] :RequestCanceledException INFO [2019-12-17 12:30:27,627] jepsen worker 88 - jepsen.util 3598 :ok :read [29 45] INFO [2019-12-17 12:30:27,628] jepsen worker 68 - jepsen.util 4298 :invoke :read [22 nil] INFO [2019-12-17 12:30:27,638] jepsen worker 74 - jepsen.util 3764 :info :cas [24 [31 14]] :RequestCanceledException INFO [2019-12-17 12:30:27,646] jepsen worker 11 - jepsen.util 3701 :ok :read [3 21] INFO [2019-12-17 12:30:27,654] jepsen worker 86 - jepsen.util 4766 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:30:27,661] jepsen worker 13 - jepsen.util 3343 :ok :write [4 38] INFO [2019-12-17 12:30:27,666] jepsen worker 73 - jepsen.util 3403 :ok :read [24 46] INFO [2019-12-17 12:30:27,706] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20045,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:30:27,707] jepsen worker 24 - jepsen.util 4434 :info :cas [8 [11 29]] :RequestCanceledException INFO [2019-12-17 12:30:27,709] jepsen worker 51 - jepsen.util 3111 :fail :cas [17 [35 49]] :ValueNotSwapFrom INFO [2019-12-17 12:30:27,724] jepsen worker 49 - jepsen.util 4099 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:30:27,731] jepsen worker 88 - jepsen.util 3598 :invoke :write [29 5] INFO [2019-12-17 12:30:27,731] jepsen worker 11 - jepsen.util 3701 :invoke :read [3 nil] INFO [2019-12-17 12:30:27,731] jepsen worker 86 - jepsen.util 4766 :invoke :write [28 19] INFO [2019-12-17 12:30:27,731] jepsen worker 13 - jepsen.util 3343 :invoke :read [4 nil] INFO [2019-12-17 12:30:27,731] jepsen worker 73 - jepsen.util 3403 :invoke :cas [24 [49 12]] INFO [2019-12-17 12:30:27,732] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20045,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:30:27,732] jepsen worker 51 - jepsen.util 3111 :invoke :cas [17 [39 6]] INFO [2019-12-17 12:30:27,732] jepsen worker 49 - jepsen.util 4099 :invoke :write [16 6] INFO [2019-12-17 12:30:27,732] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20047,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:30:27,733] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20047,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:30:27,734] jepsen worker 24 - jepsen.util 4524 :invoke :cas [8 [1 36]] INFO [2019-12-17 12:30:27,734] jepsen worker 74 - jepsen.util 3854 :invoke :read [24 nil] INFO [2019-12-17 12:30:27,736] jepsen worker 77 - jepsen.util 4487 :info :write [25 49] :RequestCanceledException INFO [2019-12-17 12:30:27,741] jepsen worker 77 - jepsen.util 4577 :invoke :read [25 nil] INFO [2019-12-17 12:30:27,753] jepsen worker 44 - jepsen.util 4994 :info :cas [14 [11 16]] :RequestCanceledException INFO [2019-12-17 12:30:27,753] jepsen worker 64 - jepsen.util 2764 :info :write [21 26] :RequestCanceledException INFO [2019-12-17 12:30:27,758] jepsen worker 54 - jepsen.util 3744 :ok :read [18 32] INFO [2019-12-17 12:30:27,764] jepsen worker 72 - jepsen.util 3492 :fail :cas [24 [30 26]] :ValueNotSwapFrom INFO [2019-12-17 12:30:27,769] jepsen worker 71 - jepsen.util 3491 :ok :write [23 22] INFO [2019-12-17 12:30:27,810] jepsen worker 54 - jepsen.util 3744 :invoke :cas [18 [42 20]] INFO [2019-12-17 12:30:27,810] jepsen worker 72 - jepsen.util 3492 :invoke :cas [24 [8 24]] INFO [2019-12-17 12:30:27,810] jepsen worker 71 - jepsen.util 3491 :invoke :read [23 nil] INFO [2019-12-17 12:30:27,812] jepsen worker 44 - jepsen.util 5084 :invoke :write [14 11] INFO [2019-12-17 12:30:27,814] jepsen worker 64 - jepsen.util 2854 :invoke :read [21 nil] INFO [2019-12-17 12:30:27,818] jepsen worker 74 - jepsen.util 3854 :ok :read [24 46] INFO [2019-12-17 12:30:27,818] jepsen worker 74 - jepsen.util 3854 :invoke :cas [24 [10 11]] INFO [2019-12-17 12:30:27,826] jepsen worker 46 - jepsen.util 3736 :ok :read [15 30] INFO [2019-12-17 12:30:27,827] jepsen worker 46 - jepsen.util 3736 :invoke :read [15 nil] INFO [2019-12-17 12:30:27,831] jepsen worker 73 - jepsen.util 3403 :fail :cas [24 [49 12]] :ValueNotSwapFrom INFO [2019-12-17 12:30:27,831] jepsen worker 73 - jepsen.util 3403 :invoke :read [24 nil] INFO [2019-12-17 12:30:27,832] jepsen worker 11 - jepsen.util 3701 :ok :read [3 21] INFO [2019-12-17 12:30:27,832] jepsen worker 11 - jepsen.util 3701 :invoke :write [3 11] INFO [2019-12-17 12:30:27,852] jepsen worker 4 - jepsen.util 2704 :ok :write [1 18] INFO [2019-12-17 12:30:27,852] jepsen worker 4 - jepsen.util 2704 :invoke :write [1 2] INFO [2019-12-17 12:30:27,856] jepsen worker 34 - jepsen.util 4624 :info :cas [11 [21 38]] :RequestCanceledException INFO [2019-12-17 12:30:27,859] jepsen worker 34 - jepsen.util 4714 :invoke :cas [11 [19 20]] INFO [2019-12-17 12:30:27,867] jepsen worker 80 - jepsen.util 3410 :ok :write [26 37] INFO [2019-12-17 12:30:27,867] jepsen worker 80 - jepsen.util 3410 :invoke :cas [26 [11 32]] INFO [2019-12-17 12:30:27,902] jepsen worker 38 - jepsen.util 3458 :ok :write [12 38] INFO [2019-12-17 12:30:27,917] jepsen worker 52 - jepsen.util 3562 :ok :write [17 33] INFO [2019-12-17 12:30:27,925] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20069,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:30:27,940] jepsen worker 79 - jepsen.util 3589 :ok :read [26 37] INFO [2019-12-17 12:30:27,950] jepsen worker 38 - jepsen.util 3458 :invoke :read [12 nil] INFO [2019-12-17 12:30:27,950] jepsen worker 52 - jepsen.util 3562 :invoke :read [17 nil] INFO [2019-12-17 12:30:27,950] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20069,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:30:27,950] jepsen worker 79 - jepsen.util 3589 :invoke :write [26 46] INFO [2019-12-17 12:30:27,950] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20073,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:30:27,950] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20073,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:30:27,962] jepsen worker 10 - jepsen.util 3340 :ok :write [3 18] INFO [2019-12-17 12:30:27,962] jepsen worker 10 - jepsen.util 3340 :invoke :write [3 25] INFO [2019-12-17 12:30:27,986] jepsen worker 47 - jepsen.util 4007 :info :cas [15 [40 41]] :RequestCanceledException INFO [2019-12-17 12:30:27,994] jepsen worker 47 - jepsen.util 4097 :invoke :cas [15 [3 19]] INFO [2019-12-17 12:30:28,000] jepsen worker 74 - jepsen.util 3854 :fail :cas [24 [10 11]] :ValueNotSwapFrom INFO [2019-12-17 12:30:28,000] jepsen worker 74 - jepsen.util 3854 :invoke :write [24 13] INFO [2019-12-17 12:30:28,008] jepsen worker 73 - jepsen.util 3403 :ok :read [24 46] INFO [2019-12-17 12:30:28,018] jepsen worker 12 - jepsen.util 3702 :ok :read [4 38] INFO [2019-12-17 12:30:28,022] jepsen worker 72 - jepsen.util 3492 :fail :cas [24 [8 24]] :ValueNotSwapFrom INFO [2019-12-17 12:30:28,031] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20084,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:30:28,053] jepsen worker 45 - jepsen.util 3465 :info :write [15 43] :RequestCanceledException INFO [2019-12-17 12:30:28,077] jepsen worker 3 - jepsen.util 3693 :fail :cas [1 [24 10]] :ValueNotSwapFrom INFO [2019-12-17 12:30:28,079] jepsen worker 87 - jepsen.util 3237 :ok :write [29 48] INFO [2019-12-17 12:30:28,079] jepsen worker 73 - jepsen.util 3403 :invoke :read [24 nil] INFO [2019-12-17 12:30:28,079] jepsen worker 72 - jepsen.util 3492 :invoke :cas [24 [2 4]] INFO [2019-12-17 12:30:28,079] jepsen worker 12 - jepsen.util 3702 :invoke :write [4 37] INFO [2019-12-17 12:30:28,079] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20084,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:30:28,079] jepsen worker 3 - jepsen.util 3693 :invoke :cas [1 [26 1]] INFO [2019-12-17 12:30:28,079] jepsen worker 87 - jepsen.util 3237 :invoke :read [29 nil] INFO [2019-12-17 12:30:28,082] jepsen worker 45 - jepsen.util 3555 :invoke :read [15 nil] INFO [2019-12-17 12:30:28,089] jepsen worker 23 - jepsen.util 4523 :info :cas [7 [10 18]] :RequestCanceledException INFO [2019-12-17 12:30:28,092] jepsen worker 23 - jepsen.util 4613 :invoke :read [7 nil] INFO [2019-12-17 12:30:28,099] jepsen worker 64 - jepsen.util 2854 :ok :read [21 48] INFO [2019-12-17 12:30:28,099] jepsen worker 64 - jepsen.util 2854 :invoke :cas [21 [21 35]] INFO [2019-12-17 12:30:28,138] jepsen worker 59 - jepsen.util 4469 :info :cas [19 [44 3]] :RequestCanceledException INFO [2019-12-17 12:30:28,140] jepsen worker 46 - jepsen.util 3736 :ok :read [15 30] INFO [2019-12-17 12:30:28,140] jepsen worker 46 - jepsen.util 3736 :invoke :cas [15 [13 44]] INFO [2019-12-17 12:30:28,141] jepsen worker 59 - jepsen.util 4559 :invoke :write [19 26] INFO [2019-12-17 12:30:28,194] jepsen worker 65 - jepsen.util 3035 :ok :read [21 48] INFO [2019-12-17 12:30:28,194] jepsen worker 65 - jepsen.util 3035 :invoke :read [21 nil] INFO [2019-12-17 12:30:28,207] jepsen worker 45 - jepsen.util 3555 :ok :read [15 30] INFO [2019-12-17 12:30:28,207] jepsen worker 55 - jepsen.util 4285 :fail :cas [18 [25 25]] :ValueNotSwapFrom INFO [2019-12-17 12:30:28,207] jepsen worker 45 - jepsen.util 3555 :invoke :read [15 nil] INFO [2019-12-17 12:30:28,207] jepsen worker 55 - jepsen.util 4285 :invoke :write [18 33] INFO [2019-12-17 12:30:28,233] jepsen worker 30 - jepsen.util 3810 :info :write [10 11] :RequestCanceledException INFO [2019-12-17 12:30:28,235] jepsen worker 5 - jepsen.util 3155 :fail :cas [1 [20 47]] :ValueNotSwapFrom INFO [2019-12-17 12:30:28,235] jepsen worker 5 - jepsen.util 3155 :invoke :read [1 nil] INFO [2019-12-17 12:30:28,235] jepsen worker 73 - jepsen.util 3403 :ok :read [24 46] INFO [2019-12-17 12:30:28,236] jepsen worker 73 - jepsen.util 3403 :invoke :read [24 nil] INFO [2019-12-17 12:30:28,236] jepsen worker 30 - jepsen.util 3900 :invoke :cas [10 [21 45]] INFO [2019-12-17 12:30:28,240] jepsen worker 72 - jepsen.util 3492 :fail :cas [24 [2 4]] :ValueNotSwapFrom INFO [2019-12-17 12:30:28,241] jepsen worker 72 - jepsen.util 3492 :invoke :write [24 40] INFO [2019-12-17 12:30:28,247] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20101,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:30:28,247] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20110,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:30:28,248] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20101,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:30:28,248] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20110,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:30:28,248] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20103,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:30:28,248] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20113,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:30:28,248] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20103,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:30:28,248] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20113,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:30:28,248] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20114,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:30:28,248] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20114,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:30:28,253] jepsen worker 45 - jepsen.util 3555 :ok :read [15 30] INFO [2019-12-17 12:30:28,253] jepsen worker 45 - jepsen.util 3555 :invoke :cas [15 [26 34]] INFO [2019-12-17 12:30:28,262] jepsen worker 36 - jepsen.util 3636 :ok :write [12 11] INFO [2019-12-17 12:30:28,262] jepsen worker 36 - jepsen.util 3636 :invoke :cas [12 [7 43]] INFO [2019-12-17 12:30:28,279] jepsen worker 73 - jepsen.util 3403 :ok :read [24 46] INFO [2019-12-17 12:30:28,279] jepsen worker 73 - jepsen.util 3403 :invoke :read [24 nil] INFO [2019-12-17 12:30:28,294] jepsen worker 27 - jepsen.util 3267 :ok :read [9 40] INFO [2019-12-17 12:30:28,294] jepsen worker 27 - jepsen.util 3267 :invoke :write [9 4] INFO [2019-12-17 12:30:28,299] jepsen worker 88 - jepsen.util 3598 :ok :write [29 5] INFO [2019-12-17 12:30:28,300] jepsen worker 88 - jepsen.util 3598 :invoke :write [29 47] INFO [2019-12-17 12:30:28,321] jepsen worker 45 - jepsen.util 3555 :fail :cas [15 [26 34]] :ValueNotSwapFrom INFO [2019-12-17 12:30:28,321] jepsen worker 45 - jepsen.util 3555 :invoke :cas [15 [9 14]] INFO [2019-12-17 12:30:28,341] jepsen worker 52 - jepsen.util 3562 :ok :read [17 33] INFO [2019-12-17 12:30:28,341] jepsen worker 52 - jepsen.util 3562 :invoke :read [17 nil] INFO [2019-12-17 12:30:28,352] jepsen worker 43 - jepsen.util 4273 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:30:28,352] jepsen worker 43 - jepsen.util 4273 :invoke :cas [14 [44 12]] INFO [2019-12-17 12:30:28,365] jepsen worker 64 - jepsen.util 2854 :fail :cas [21 [21 35]] :ValueNotSwapFrom INFO [2019-12-17 12:30:28,365] jepsen worker 64 - jepsen.util 2854 :invoke :cas [21 [15 22]] INFO [2019-12-17 12:30:28,380] jepsen worker 80 - jepsen.util 3410 :fail :cas [26 [11 32]] :ValueNotSwapFrom INFO [2019-12-17 12:30:28,380] jepsen worker 80 - jepsen.util 3410 :invoke :write [26 27] INFO [2019-12-17 12:30:28,386] jepsen worker 73 - jepsen.util 3403 :ok :read [24 46] INFO [2019-12-17 12:30:28,391] jepsen worker 14 - jepsen.util 3434 :fail :cas [4 [29 26]] :ValueNotSwapFrom INFO [2019-12-17 12:30:28,408] jepsen worker 89 - jepsen.util 3239 :info :write [29 40] :RequestCanceledException INFO [2019-12-17 12:30:28,408] jepsen worker 84 - jepsen.util 4404 :info :write [28 17] :RequestCanceledException INFO [2019-12-17 12:30:28,421] jepsen worker 69 - jepsen.util 3399 :fail :cas [23 [43 39]] :ValueNotSwapFrom INFO [2019-12-17 12:30:28,424] jepsen worker 66 - jepsen.util 4476 :info :cas [22 [28 12]] :RequestCanceledException INFO [2019-12-17 12:30:28,438] jepsen worker 4 - jepsen.util 2704 :ok :write [1 2] INFO [2019-12-17 12:30:28,446] jepsen worker 53 - jepsen.util 3563 :fail :cas [17 [30 15]] :ValueNotSwapFrom INFO [2019-12-17 12:30:28,461] jepsen worker 36 - jepsen.util 3636 :fail :cas [12 [7 43]] :ValueNotSwapFrom INFO [2019-12-17 12:30:28,463] jepsen worker 14 - jepsen.util 3434 :invoke :write [4 1] INFO [2019-12-17 12:30:28,463] jepsen worker 73 - jepsen.util 3403 :invoke :write [24 15] INFO [2019-12-17 12:30:28,463] jepsen worker 69 - jepsen.util 3399 :invoke :cas [23 [28 32]] INFO [2019-12-17 12:30:28,463] jepsen worker 4 - jepsen.util 2704 :invoke :cas [1 [33 11]] INFO [2019-12-17 12:30:28,464] jepsen worker 53 - jepsen.util 3563 :invoke :cas [17 [34 33]] INFO [2019-12-17 12:30:28,464] jepsen worker 36 - jepsen.util 3636 :invoke :write [12 26] INFO [2019-12-17 12:30:28,466] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20131,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:30:28,466] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20131,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:30:28,466] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20135,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:30:28,466] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20135,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:30:28,467] jepsen worker 89 - jepsen.util 3329 :invoke :write [29 11] INFO [2019-12-17 12:30:28,467] jepsen worker 84 - jepsen.util 4494 :invoke :read [28 nil] INFO [2019-12-17 12:30:28,467] jepsen worker 66 - jepsen.util 4566 :invoke :read [22 nil] INFO [2019-12-17 12:30:28,482] jepsen worker 45 - jepsen.util 3555 :fail :cas [15 [9 14]] :ValueNotSwapFrom INFO [2019-12-17 12:30:28,483] jepsen worker 45 - jepsen.util 3555 :invoke :write [15 20] INFO [2019-12-17 12:30:28,508] jepsen worker 39 - jepsen.util 4269 :info :write [13 32] :RequestCanceledException INFO [2019-12-17 12:30:28,510] jepsen worker 39 - jepsen.util 4359 :invoke :cas [13 [7 13]] INFO [2019-12-17 12:30:28,514] jepsen worker 7 - jepsen.util 4417 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:30:28,514] jepsen worker 7 - jepsen.util 4417 :invoke :read [2 nil] INFO [2019-12-17 12:30:28,523] jepsen worker 85 - jepsen.util 5125 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:30:28,523] jepsen worker 25 - jepsen.util 4885 :info :cas [8 [23 16]] :RequestCanceledException INFO [2019-12-17 12:30:28,523] jepsen worker 85 - jepsen.util 5125 :invoke :cas [28 [22 14]] INFO [2019-12-17 12:30:28,525] jepsen worker 25 - jepsen.util 4975 :invoke :write [8 42] INFO [2019-12-17 12:30:28,536] jepsen worker 50 - jepsen.util 5090 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:30:28,536] jepsen worker 6 - jepsen.util 4956 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:30:28,536] jepsen worker 50 - jepsen.util 5090 :invoke :cas [16 [24 43]] INFO [2019-12-17 12:30:28,536] jepsen worker 6 - jepsen.util 4956 :invoke :cas [2 [25 33]] INFO [2019-12-17 12:30:28,553] jepsen worker 62 - jepsen.util 4382 :info :cas [20 [15 28]] :RequestCanceledException INFO [2019-12-17 12:30:28,554] jepsen worker 58 - jepsen.util 4648 :info :cas [19 [30 4]] :RequestCanceledException INFO [2019-12-17 12:30:28,554] jepsen worker 82 - jepsen.util 5482 :info :cas [27 [19 28]] :RequestCanceledException INFO [2019-12-17 12:30:28,554] jepsen worker 87 - jepsen.util 3237 :ok :read [29 5] INFO [2019-12-17 12:30:28,554] jepsen worker 87 - jepsen.util 3237 :invoke :write [29 45] INFO [2019-12-17 12:30:28,556] jepsen worker 52 - jepsen.util 3562 :ok :read [17 33] INFO [2019-12-17 12:30:28,556] jepsen worker 52 - jepsen.util 3562 :invoke :write [17 15] INFO [2019-12-17 12:30:28,557] jepsen worker 58 - jepsen.util 4738 :invoke :read [19 nil] INFO [2019-12-17 12:30:28,557] jepsen worker 62 - jepsen.util 4472 :invoke :write [20 8] INFO [2019-12-17 12:30:28,558] jepsen worker 82 - jepsen.util 5572 :invoke :cas [27 [4 27]] INFO [2019-12-17 12:30:28,569] jepsen worker 70 - jepsen.util 4030 :ok :write [23 16] INFO [2019-12-17 12:30:28,575] jepsen worker 37 - jepsen.util 3367 :ok :write [12 11] INFO [2019-12-17 12:30:28,578] jepsen worker 47 - jepsen.util 4097 :fail :cas [15 [3 19]] :ValueNotSwapFrom INFO [2019-12-17 12:30:28,590] jepsen worker 75 - jepsen.util 4665 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:30:28,590] jepsen worker 26 - jepsen.util 4886 :info :cas [8 [29 34]] :RequestCanceledException INFO [2019-12-17 12:30:28,606] jepsen worker 67 - jepsen.util 5017 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:30:28,606] jepsen worker 31 - jepsen.util 2911 :info :write [10 18] :RequestCanceledException INFO [2019-12-17 12:30:28,608] jepsen worker 65 - jepsen.util 3035 :ok :read [21 48] INFO [2019-12-17 12:30:28,621] jepsen worker 60 - jepsen.util 4920 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:30:28,621] jepsen worker 83 - jepsen.util 4763 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:30:28,630] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:30:28,634] jepsen worker 4 - jepsen.util 2704 :fail :cas [1 [33 11]] :ValueNotSwapFrom INFO [2019-12-17 12:30:28,637] jepsen worker 69 - jepsen.util 3399 :fail :cas [23 [28 32]] :ValueNotSwapFrom INFO [2019-12-17 12:30:28,654] jepsen worker 53 - jepsen.util 3563 :fail :cas [17 [34 33]] :ValueNotSwapFrom INFO [2019-12-17 12:30:28,688] jepsen worker 42 - jepsen.util 5532 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:30:28,688] jepsen worker 2 - jepsen.util 4322 :info :write [0 31] :RequestCanceledException INFO [2019-12-17 12:30:28,688] jepsen worker 32 - jepsen.util 3902 :info :write [10 11] :RequestCanceledException INFO [2019-12-17 12:30:28,707] jepsen worker 14 - jepsen.util 3434 :ok :write [4 1] INFO [2019-12-17 12:30:28,707] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20158,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:30:28,708] jepsen worker 40 - jepsen.util 4900 :info :write [13 11] :RequestCanceledException INFO [2019-12-17 12:30:28,709] jepsen worker 46 - jepsen.util 3736 :fail :cas [15 [13 44]] :ValueNotSwapFrom INFO [2019-12-17 12:30:28,716] jepsen worker 37 - jepsen.util 3367 :invoke :write [12 29] INFO [2019-12-17 12:30:28,716] jepsen worker 70 - jepsen.util 4030 :invoke :cas [23 [28 28]] INFO [2019-12-17 12:30:28,716] jepsen worker 47 - jepsen.util 4097 :invoke :read [15 nil] INFO [2019-12-17 12:30:28,716] jepsen worker 75 - jepsen.util 4665 :invoke :cas [25 [22 43]] INFO [2019-12-17 12:30:28,716] jepsen worker 67 - jepsen.util 5017 :invoke :write [22 47] INFO [2019-12-17 12:30:28,716] jepsen worker 65 - jepsen.util 3035 :invoke :read [21 nil] INFO [2019-12-17 12:30:28,716] jepsen worker 60 - jepsen.util 4920 :invoke :cas [20 [31 8]] INFO [2019-12-17 12:30:28,716] jepsen nemesis - couchbase.util Rebalance check count: 148 INFO [2019-12-17 12:30:28,716] jepsen worker 83 - jepsen.util 4763 :invoke :cas [27 [2 21]] INFO [2019-12-17 12:30:28,716] jepsen worker 4 - jepsen.util 2704 :invoke :write [1 9] INFO [2019-12-17 12:30:28,717] jepsen worker 69 - jepsen.util 3399 :invoke :cas [23 [46 40]] INFO [2019-12-17 12:30:28,717] jepsen worker 53 - jepsen.util 3563 :invoke :write [17 29] INFO [2019-12-17 12:30:28,717] jepsen worker 42 - jepsen.util 5532 :invoke :cas [14 [44 0]] INFO [2019-12-17 12:30:28,717] jepsen worker 14 - jepsen.util 3434 :invoke :write [4 31] INFO [2019-12-17 12:30:28,717] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20158,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:30:28,717] jepsen worker 46 - jepsen.util 3736 :invoke :cas [15 [36 29]] INFO [2019-12-17 12:30:28,718] jepsen nemesis - couchbase.util Rebalance stuck count: 129 INFO [2019-12-17 12:30:28,718] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20162,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:30:28,719] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20162,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:30:28,720] jepsen worker 31 - jepsen.util 3001 :invoke :write [10 7] INFO [2019-12-17 12:30:28,721] jepsen worker 22 - jepsen.util 4792 :info :cas [7 [41 46]] :RequestCanceledException INFO [2019-12-17 12:30:28,721] jepsen worker 26 - jepsen.util 4976 :invoke :cas [8 [39 43]] INFO [2019-12-17 12:30:28,721] jepsen worker 32 - jepsen.util 3992 :invoke :read [10 nil] INFO [2019-12-17 12:30:28,722] jepsen worker 2 - jepsen.util 4412 :invoke :cas [0 [14 46]] INFO [2019-12-17 12:30:28,722] jepsen worker 40 - jepsen.util 4990 :invoke :cas [13 [24 39]] INFO [2019-12-17 12:30:28,727] jepsen worker 22 - jepsen.util 4882 :invoke :read [7 nil] INFO [2019-12-17 12:30:28,737] jepsen worker 35 - jepsen.util 4355 :info :cas [11 [9 6]] :RequestCanceledException INFO [2019-12-17 12:30:28,737] jepsen worker 33 - jepsen.util 4893 :info :cas [11 [18 47]] :RequestCanceledException INFO [2019-12-17 12:30:28,745] jepsen worker 35 - jepsen.util 4445 :invoke :write [11 17] INFO [2019-12-17 12:30:28,745] jepsen worker 33 - jepsen.util 4983 :invoke :read [11 nil] INFO [2019-12-17 12:30:28,766] jepsen worker 38 - jepsen.util 3458 :ok :read [12 11] INFO [2019-12-17 12:30:28,767] jepsen worker 38 - jepsen.util 3458 :invoke :write [12 46] INFO [2019-12-17 12:30:28,789] jepsen worker 32 - jepsen.util 3992 :ok :read [10 41] INFO [2019-12-17 12:30:28,790] jepsen worker 32 - jepsen.util 3992 :invoke :read [10 nil] INFO [2019-12-17 12:30:28,790] jepsen worker 30 - jepsen.util 3900 :fail :cas [10 [21 45]] :ValueNotSwapFrom INFO [2019-12-17 12:30:28,790] jepsen worker 30 - jepsen.util 3900 :invoke :read [10 nil] INFO [2019-12-17 12:30:28,808] jepsen worker 8 - jepsen.util 3968 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:30:28,808] jepsen worker 8 - jepsen.util 3968 :invoke :write [2 28] INFO [2019-12-17 12:30:28,816] jepsen worker 89 - jepsen.util 3329 :ok :write [29 11] INFO [2019-12-17 12:30:28,816] jepsen worker 89 - jepsen.util 3329 :invoke :cas [29 [29 41]] INFO [2019-12-17 12:30:28,821] jepsen worker 56 - jepsen.util 3566 :ok :write [18 36] INFO [2019-12-17 12:30:28,821] jepsen worker 56 - jepsen.util 3566 :invoke :write [18 35] INFO [2019-12-17 12:30:28,822] jepsen worker 17 - jepsen.util 4427 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:30:28,822] jepsen worker 41 - jepsen.util 4811 :info :cas [13 [18 15]] :RequestCanceledException INFO [2019-12-17 12:30:28,822] jepsen worker 17 - jepsen.util 4427 :invoke :write [5 47] INFO [2019-12-17 12:30:28,823] jepsen worker 32 - jepsen.util 3992 :ok :read [10 41] INFO [2019-12-17 12:30:28,824] jepsen worker 32 - jepsen.util 3992 :invoke :write [10 19] INFO [2019-12-17 12:30:28,824] jepsen worker 41 - jepsen.util 4901 :invoke :read [13 nil] INFO [2019-12-17 12:30:28,837] jepsen worker 0 - jepsen.util 5130 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:30:28,837] jepsen worker 0 - jepsen.util 5130 :invoke :write [0 14] INFO [2019-12-17 12:30:28,841] jepsen worker 79 - jepsen.util 3589 :ok :write [26 46] INFO [2019-12-17 12:30:28,841] jepsen worker 79 - jepsen.util 3589 :invoke :cas [26 [4 11]] INFO [2019-12-17 12:30:28,852] jepsen worker 57 - jepsen.util 4377 :info :write [19 17] :RequestCanceledException INFO [2019-12-17 12:30:28,852] jepsen worker 48 - jepsen.util 4008 :info :cas [16 [20 41]] :RequestCanceledException INFO [2019-12-17 12:30:28,853] jepsen worker 57 - jepsen.util 4467 :invoke :cas [19 [8 25]] INFO [2019-12-17 12:30:28,853] jepsen worker 48 - jepsen.util 4098 :invoke :write [16 40] INFO [2019-12-17 12:30:28,871] jepsen worker 3 - jepsen.util 3693 :fail :cas [1 [26 1]] :ValueNotSwapFrom INFO [2019-12-17 12:30:28,872] jepsen worker 3 - jepsen.util 3693 :invoke :write [1 0] INFO [2019-12-17 12:30:28,908] jepsen worker 54 - jepsen.util 3744 :fail :cas [18 [42 20]] :ValueNotSwapFrom INFO [2019-12-17 12:30:28,908] jepsen worker 54 - jepsen.util 3744 :invoke :read [18 nil] INFO [2019-12-17 12:30:28,922] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20198,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:30:28,922] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20198,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:30:28,922] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20201,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:30:28,922] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20201,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:30:28,922] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20202,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:30:28,922] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20202,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:30:28,929] jepsen worker 13 - jepsen.util 3343 :ok :read [4 1] INFO [2019-12-17 12:30:28,929] jepsen worker 13 - jepsen.util 3343 :invoke :cas [4 [10 40]] INFO [2019-12-17 12:30:28,947] jepsen worker 64 - jepsen.util 2854 :fail :cas [21 [15 22]] :ValueNotSwapFrom INFO [2019-12-17 12:30:28,947] jepsen worker 64 - jepsen.util 2854 :invoke :read [21 nil] INFO [2019-12-17 12:30:28,950] jepsen worker 5 - jepsen.util 3155 :ok :read [1 2] INFO [2019-12-17 12:30:28,950] jepsen worker 5 - jepsen.util 3155 :invoke :cas [1 [41 7]] INFO [2019-12-17 12:30:28,977] jepsen worker 51 - jepsen.util 3111 :fail :cas [17 [39 6]] :ValueNotSwapFrom INFO [2019-12-17 12:30:28,977] jepsen worker 51 - jepsen.util 3111 :invoke :read [17 nil] INFO [2019-12-17 12:30:28,992] jepsen worker 46 - jepsen.util 3736 :fail :cas [15 [36 29]] :ValueNotSwapFrom INFO [2019-12-17 12:30:28,992] jepsen worker 46 - jepsen.util 3736 :invoke :cas [15 [9 18]] INFO [2019-12-17 12:30:28,993] jepsen worker 47 - jepsen.util 4097 :ok :read [15 30] INFO [2019-12-17 12:30:28,993] jepsen worker 47 - jepsen.util 4097 :invoke :cas [15 [28 20]] INFO [2019-12-17 12:30:29,010] jepsen worker 65 - jepsen.util 3035 :ok :read [21 48] INFO [2019-12-17 12:30:29,011] jepsen worker 65 - jepsen.util 3035 :invoke :write [21 34] INFO [2019-12-17 12:30:29,023] jepsen worker 30 - jepsen.util 3900 :ok :read [10 41] INFO [2019-12-17 12:30:29,023] jepsen worker 30 - jepsen.util 3900 :invoke :write [10 9] INFO [2019-12-17 12:30:29,024] jepsen worker 71 - jepsen.util 3491 :ok :read [23 16] INFO [2019-12-17 12:30:29,024] jepsen worker 71 - jepsen.util 3491 :invoke :read [23 nil] INFO [2019-12-17 12:30:29,041] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20211,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:30:29,042] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20211,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:30:29,042] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20215,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:30:29,042] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20215,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:30:29,044] jepsen worker 47 - jepsen.util 4097 :fail :cas [15 [28 20]] :ValueNotSwapFrom INFO [2019-12-17 12:30:29,044] jepsen worker 47 - jepsen.util 4097 :invoke :write [15 4] INFO [2019-12-17 12:30:29,080] jepsen worker 70 - jepsen.util 4030 :fail :cas [23 [28 28]] :ValueNotSwapFrom INFO [2019-12-17 12:30:29,080] jepsen worker 70 - jepsen.util 4030 :invoke :read [23 nil] INFO [2019-12-17 12:30:29,097] jepsen worker 51 - jepsen.util 3111 :ok :read [17 33] INFO [2019-12-17 12:30:29,098] jepsen worker 51 - jepsen.util 3111 :invoke :cas [17 [32 21]] INFO [2019-12-17 12:30:29,132] jepsen worker 64 - jepsen.util 2854 :ok :read [21 48] INFO [2019-12-17 12:30:29,132] jepsen worker 64 - jepsen.util 2854 :invoke :read [21 nil] INFO [2019-12-17 12:30:29,136] jepsen worker 37 - jepsen.util 3367 :ok :write [12 29] INFO [2019-12-17 12:30:29,136] jepsen worker 37 - jepsen.util 3367 :invoke :read [12 nil] INFO [2019-12-17 12:30:29,139] jepsen worker 71 - jepsen.util 3491 :ok :read [23 16] INFO [2019-12-17 12:30:29,139] jepsen worker 71 - jepsen.util 3491 :invoke :read [23 nil] INFO [2019-12-17 12:30:29,154] jepsen worker 69 - jepsen.util 3399 :fail :cas [23 [46 40]] :ValueNotSwapFrom INFO [2019-12-17 12:30:29,154] jepsen worker 69 - jepsen.util 3399 :invoke :write [23 0] INFO [2019-12-17 12:30:29,187] jepsen worker 15 - jepsen.util 4245 :info :write [5 28] :RequestCanceledException INFO [2019-12-17 12:30:29,187] jepsen worker 76 - jepsen.util 4756 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:30:29,187] jepsen worker 76 - jepsen.util 4756 :invoke :write [25 32] INFO [2019-12-17 12:30:29,191] jepsen worker 15 - jepsen.util 4335 :invoke :read [5 nil] INFO [2019-12-17 12:30:29,222] jepsen worker 46 - jepsen.util 3736 :fail :cas [15 [9 18]] :ValueNotSwapFrom INFO [2019-12-17 12:30:29,222] jepsen worker 46 - jepsen.util 3736 :invoke :write [15 3] INFO [2019-12-17 12:30:29,223] jepsen worker 16 - jepsen.util 5146 :info :write [5 25] :RequestCanceledException INFO [2019-12-17 12:30:29,223] jepsen worker 1 - jepsen.util 4231 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:30:29,224] jepsen worker 1 - jepsen.util 4231 :invoke :write [0 33] INFO [2019-12-17 12:30:29,225] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20229,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:30:29,225] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20237,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:30:29,226] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20229,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:30:29,226] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20237,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:30:29,226] jepsen worker 16 - jepsen.util 5236 :invoke :cas [5 [26 40]] INFO [2019-12-17 12:30:29,226] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20230,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:30:29,226] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20240,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:30:29,226] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20230,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:30:29,226] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20240,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:30:29,226] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20241,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:30:29,226] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20241,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:30:29,241] jepsen worker 70 - jepsen.util 4030 :ok :read [23 16] INFO [2019-12-17 12:30:29,241] jepsen worker 70 - jepsen.util 4030 :invoke :write [23 44] INFO [2019-12-17 12:30:29,242] jepsen worker 65 - jepsen.util 3035 :ok :write [21 34] INFO [2019-12-17 12:30:29,242] jepsen worker 65 - jepsen.util 3035 :invoke :read [21 nil] INFO [2019-12-17 12:30:29,242] jepsen worker 87 - jepsen.util 3237 :ok :write [29 45] INFO [2019-12-17 12:30:29,242] jepsen worker 87 - jepsen.util 3237 :invoke :write [29 20] INFO [2019-12-17 12:30:29,290] jepsen worker 21 - jepsen.util 4701 :info :write [7 17] :RequestCanceledException INFO [2019-12-17 12:30:29,297] jepsen worker 30 - jepsen.util 3900 :ok :write [10 9] INFO [2019-12-17 12:30:29,312] jepsen worker 47 - jepsen.util 4097 :ok :write [15 4] INFO [2019-12-17 12:30:29,324] jepsen worker 61 - jepsen.util 5281 :info :write [20 3] :RequestCanceledException INFO [2019-12-17 12:30:29,337] jepsen worker 81 - jepsen.util 4671 :info :cas [27 [35 11]] :RequestCanceledException INFO [2019-12-17 12:30:29,343] jepsen worker 54 - jepsen.util 3744 :ok :read [18 36] INFO [2019-12-17 12:30:29,355] jepsen worker 9 - jepsen.util 4059 :info :write [3 32] :RequestCanceledException INFO [2019-12-17 12:30:29,401] jepsen worker 47 - jepsen.util 4097 :invoke :write [15 5] INFO [2019-12-17 12:30:29,401] jepsen worker 30 - jepsen.util 3900 :invoke :write [10 36] INFO [2019-12-17 12:30:29,401] jepsen worker 54 - jepsen.util 3744 :invoke :cas [18 [14 15]] INFO [2019-12-17 12:30:29,402] jepsen worker 61 - jepsen.util 5371 :invoke :cas [20 [12 19]] INFO [2019-12-17 12:30:29,402] jepsen worker 21 - jepsen.util 4791 :invoke :cas [7 [10 15]] INFO [2019-12-17 12:30:29,402] jepsen worker 81 - jepsen.util 4761 :invoke :read [27 nil] INFO [2019-12-17 12:30:29,402] jepsen worker 9 - jepsen.util 4149 :invoke :read [3 nil] INFO [2019-12-17 12:30:29,424] jepsen worker 63 - jepsen.util 3123 :info :write [21 17] :RequestCanceledException INFO [2019-12-17 12:30:29,424] jepsen worker 28 - jepsen.util 2458 :info :write [9 27] :RequestCanceledException INFO [2019-12-17 12:30:29,424] jepsen worker 18 - jepsen.util 3528 :info :write [6 23] :RequestCanceledException INFO [2019-12-17 12:30:29,425] jepsen worker 63 - jepsen.util 3213 :invoke :cas [21 [45 5]] INFO [2019-12-17 12:30:29,425] jepsen worker 28 - jepsen.util 2548 :invoke :write [9 6] INFO [2019-12-17 12:30:29,425] jepsen worker 18 - jepsen.util 3618 :invoke :cas [6 [8 29]] INFO [2019-12-17 12:30:29,428] jepsen worker 71 - jepsen.util 3491 :ok :read [23 16] INFO [2019-12-17 12:30:29,428] jepsen worker 71 - jepsen.util 3491 :invoke :cas [23 [17 12]] INFO [2019-12-17 12:30:29,437] jepsen worker 13 - jepsen.util 3343 :fail :cas [4 [10 40]] :ValueNotSwapFrom INFO [2019-12-17 12:30:29,437] jepsen worker 13 - jepsen.util 3343 :invoke :read [4 nil] INFO [2019-12-17 12:30:29,445] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20258,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:30:29,445] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20258,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:30:29,445] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20262,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:30:29,445] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20262,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:30:29,455] jepsen worker 20 - jepsen.util 3170 :info :write [6 40] :RequestCanceledException INFO [2019-12-17 12:30:29,456] jepsen worker 20 - jepsen.util 3260 :invoke :cas [6 [18 13]] INFO [2019-12-17 12:30:29,468] jepsen worker 9 - jepsen.util 4149 :ok :read [3 18] INFO [2019-12-17 12:30:29,468] jepsen worker 9 - jepsen.util 4149 :invoke :read [3 nil] INFO [2019-12-17 12:30:29,488] jepsen worker 71 - jepsen.util 3491 :fail :cas [23 [17 12]] :ValueNotSwapFrom INFO [2019-12-17 12:30:29,488] jepsen worker 71 - jepsen.util 3491 :invoke :cas [23 [2 32]] INFO [2019-12-17 12:30:29,491] jepsen worker 18 - jepsen.util 3618 :fail :cas [6 [8 29]] :ValueNotSwapFrom INFO [2019-12-17 12:30:29,491] jepsen worker 18 - jepsen.util 3618 :invoke :write [6 7] INFO [2019-12-17 12:30:29,495] jepsen worker 20 - jepsen.util 3260 :fail :cas [6 [18 13]] :ValueNotSwapFrom INFO [2019-12-17 12:30:29,495] jepsen worker 20 - jepsen.util 3260 :invoke :read [6 nil] INFO [2019-12-17 12:30:29,532] jepsen worker 89 - jepsen.util 3329 :fail :cas [29 [29 41]] :ValueNotSwapFrom INFO [2019-12-17 12:30:29,533] jepsen worker 89 - jepsen.util 3329 :invoke :write [29 17] INFO [2019-12-17 12:30:29,555] jepsen worker 19 - jepsen.util 3619 :info :write [6 45] :RequestCanceledException INFO [2019-12-17 12:30:29,555] jepsen worker 9 - jepsen.util 4149 :ok :read [3 18] INFO [2019-12-17 12:30:29,593] jepsen worker 20 - jepsen.util 3260 :ok :read [6 49] INFO [2019-12-17 12:30:29,597] jepsen worker 28 - jepsen.util 2548 :ok :write [9 6] INFO [2019-12-17 12:30:29,678] jepsen worker 9 - jepsen.util 4149 :invoke :write [3 15] INFO [2019-12-17 12:30:29,678] jepsen worker 20 - jepsen.util 3260 :invoke :read [6 nil] INFO [2019-12-17 12:30:29,678] jepsen worker 28 - jepsen.util 2548 :invoke :read [9 nil] INFO [2019-12-17 12:30:29,679] jepsen worker 19 - jepsen.util 3709 :invoke :write [6 43] INFO [2019-12-17 12:30:29,696] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20273,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:30:29,696] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20273,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:30:29,696] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20276,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:30:29,696] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20276,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:30:29,703] jepsen worker 79 - jepsen.util 3589 :fail :cas [26 [4 11]] :ValueNotSwapFrom INFO [2019-12-17 12:30:29,704] jepsen worker 79 - jepsen.util 3589 :invoke :read [26 nil] INFO [2019-12-17 12:30:29,707] jepsen worker 20 - jepsen.util 3260 :ok :read [6 49] INFO [2019-12-17 12:30:29,707] jepsen worker 20 - jepsen.util 3260 :invoke :read [6 nil] INFO [2019-12-17 12:30:29,724] jepsen worker 47 - jepsen.util 4097 :ok :write [15 5] INFO [2019-12-17 12:30:29,724] jepsen worker 47 - jepsen.util 4097 :invoke :write [15 22] INFO [2019-12-17 12:30:29,725] jepsen worker 28 - jepsen.util 2548 :ok :read [9 6] INFO [2019-12-17 12:30:29,725] jepsen worker 28 - jepsen.util 2548 :invoke :read [9 nil] INFO [2019-12-17 12:30:29,729] jepsen worker 20 - jepsen.util 3260 :ok :read [6 49] INFO [2019-12-17 12:30:29,729] jepsen worker 20 - jepsen.util 3260 :invoke :cas [6 [9 32]] INFO [2019-12-17 12:30:29,738] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:30:29,739] jepsen nemesis - couchbase.util Rebalance check count: 149 INFO [2019-12-17 12:30:29,739] jepsen nemesis - couchbase.util Rebalance stuck count: 130 INFO [2019-12-17 12:30:29,748] jepsen worker 51 - jepsen.util 3111 :fail :cas [17 [32 21]] :ValueNotSwapFrom INFO [2019-12-17 12:30:29,749] jepsen worker 51 - jepsen.util 3111 :invoke :write [17 46] INFO [2019-12-17 12:30:29,755] jepsen worker 86 - jepsen.util 4766 :info :write [28 19] :RequestCanceledException INFO [2019-12-17 12:30:29,757] jepsen worker 86 - jepsen.util 4856 :invoke :write [28 29] INFO [2019-12-17 12:30:29,775] jepsen worker 63 - jepsen.util 3213 :fail :cas [21 [45 5]] :ValueNotSwapFrom INFO [2019-12-17 12:30:29,775] jepsen worker 63 - jepsen.util 3213 :invoke :read [21 nil] INFO [2019-12-17 12:30:29,784] jepsen worker 28 - jepsen.util 2548 :ok :read [9 6] INFO [2019-12-17 12:30:29,785] jepsen worker 28 - jepsen.util 2548 :invoke :cas [9 [39 9]] INFO [2019-12-17 12:30:29,786] jepsen worker 68 - jepsen.util 4298 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:30:29,787] jepsen worker 68 - jepsen.util 4298 :invoke :cas [22 [17 42]] INFO [2019-12-17 12:30:29,797] jepsen worker 54 - jepsen.util 3744 :fail :cas [18 [14 15]] :ValueNotSwapFrom INFO [2019-12-17 12:30:29,797] jepsen worker 54 - jepsen.util 3744 :invoke :cas [18 [28 11]] INFO [2019-12-17 12:30:29,809] jepsen worker 24 - jepsen.util 4524 :info :cas [8 [1 36]] :RequestCanceledException INFO [2019-12-17 12:30:29,810] jepsen worker 24 - jepsen.util 4614 :invoke :write [8 9] INFO [2019-12-17 12:30:29,816] jepsen worker 13 - jepsen.util 3343 :ok :read [4 1] INFO [2019-12-17 12:30:29,816] jepsen worker 13 - jepsen.util 3343 :invoke :cas [4 [5 38]] INFO [2019-12-17 12:30:29,819] jepsen worker 20 - jepsen.util 3260 :fail :cas [6 [9 32]] :ValueNotSwapFrom INFO [2019-12-17 12:30:29,819] jepsen worker 20 - jepsen.util 3260 :invoke :cas [6 [7 38]] INFO [2019-12-17 12:30:29,823] jepsen worker 49 - jepsen.util 4099 :info :write [16 6] :RequestCanceledException INFO [2019-12-17 12:30:29,825] jepsen worker 49 - jepsen.util 4189 :invoke :write [16 39] INFO [2019-12-17 12:30:29,827] jepsen worker 28 - jepsen.util 2548 :fail :cas [9 [39 9]] :ValueNotSwapFrom INFO [2019-12-17 12:30:29,828] jepsen worker 28 - jepsen.util 2548 :invoke :cas [9 [42 6]] INFO [2019-12-17 12:30:29,838] jepsen worker 77 - jepsen.util 4577 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:30:29,838] jepsen worker 77 - jepsen.util 4577 :invoke :write [25 13] INFO [2019-12-17 12:30:29,855] jepsen worker 44 - jepsen.util 5084 :info :write [14 11] :RequestCanceledException INFO [2019-12-17 12:30:29,856] jepsen worker 71 - jepsen.util 3491 :fail :cas [23 [2 32]] :ValueNotSwapFrom INFO [2019-12-17 12:30:29,857] jepsen worker 44 - jepsen.util 5174 :invoke :read [14 nil] INFO [2019-12-17 12:30:29,857] jepsen worker 71 - jepsen.util 3491 :invoke :read [23 nil] INFO [2019-12-17 12:30:29,861] jepsen worker 20 - jepsen.util 3260 :fail :cas [6 [7 38]] :ValueNotSwapFrom INFO [2019-12-17 12:30:29,861] jepsen worker 20 - jepsen.util 3260 :invoke :read [6 nil] INFO [2019-12-17 12:30:29,870] jepsen worker 28 - jepsen.util 2548 :fail :cas [9 [42 6]] :ValueNotSwapFrom INFO [2019-12-17 12:30:29,871] jepsen worker 28 - jepsen.util 2548 :invoke :cas [9 [21 35]] INFO [2019-12-17 12:30:29,886] jepsen worker 20 - jepsen.util 3260 :ok :read [6 49] INFO [2019-12-17 12:30:29,886] jepsen worker 20 - jepsen.util 3260 :invoke :read [6 nil] INFO [2019-12-17 12:30:29,891] jepsen worker 34 - jepsen.util 4714 :info :cas [11 [19 20]] :RequestCanceledException INFO [2019-12-17 12:30:29,892] jepsen worker 34 - jepsen.util 4804 :invoke :read [11 nil] INFO [2019-12-17 12:30:29,906] jepsen worker 28 - jepsen.util 2548 :fail :cas [9 [21 35]] :ValueNotSwapFrom INFO [2019-12-17 12:30:29,906] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20302,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:30:29,907] jepsen worker 28 - jepsen.util 2548 :invoke :cas [9 [14 36]] INFO [2019-12-17 12:30:29,907] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20302,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:30:29,907] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20306,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:30:29,907] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20306,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:30:29,919] jepsen worker 20 - jepsen.util 3260 :ok :read [6 49] INFO [2019-12-17 12:30:29,919] jepsen worker 20 - jepsen.util 3260 :invoke :cas [6 [14 43]] INFO [2019-12-17 12:30:29,930] jepsen worker 28 - jepsen.util 2548 :fail :cas [9 [14 36]] :ValueNotSwapFrom INFO [2019-12-17 12:30:29,931] jepsen worker 28 - jepsen.util 2548 :invoke :cas [9 [43 20]] INFO [2019-12-17 12:30:29,937] jepsen worker 20 - jepsen.util 3260 :fail :cas [6 [14 43]] :ValueNotSwapFrom INFO [2019-12-17 12:30:29,937] jepsen worker 20 - jepsen.util 3260 :invoke :write [6 32] INFO [2019-12-17 12:30:29,938] jepsen worker 11 - jepsen.util 3701 :info :write [3 11] :RequestCanceledException INFO [2019-12-17 12:30:29,939] jepsen worker 11 - jepsen.util 3791 :invoke :cas [3 [24 32]] INFO [2019-12-17 12:30:29,945] jepsen worker 28 - jepsen.util 2548 :fail :cas [9 [43 20]] :ValueNotSwapFrom INFO [2019-12-17 12:30:29,945] jepsen worker 28 - jepsen.util 2548 :invoke :read [9 nil] INFO [2019-12-17 12:30:29,965] jepsen worker 28 - jepsen.util 2548 :ok :read [9 6] INFO [2019-12-17 12:30:29,965] jepsen worker 28 - jepsen.util 2548 :invoke :cas [9 [40 2]] INFO [2019-12-17 12:30:29,977] jepsen worker 37 - jepsen.util 3367 :ok :read [12 29] INFO [2019-12-17 12:30:29,977] jepsen worker 37 - jepsen.util 3367 :invoke :read [12 nil] INFO [2019-12-17 12:30:29,994] jepsen worker 28 - jepsen.util 2548 :fail :cas [9 [40 2]] :ValueNotSwapFrom INFO [2019-12-17 12:30:29,994] jepsen worker 28 - jepsen.util 2548 :invoke :cas [9 [48 21]] INFO [2019-12-17 12:30:30,008] jepsen worker 10 - jepsen.util 3340 :info :write [3 25] :RequestCanceledException INFO [2019-12-17 12:30:30,008] jepsen worker 28 - jepsen.util 2548 :fail :cas [9 [48 21]] :ValueNotSwapFrom INFO [2019-12-17 12:30:30,008] jepsen worker 28 - jepsen.util 2548 :invoke :write [9 33] INFO [2019-12-17 12:30:30,009] jepsen worker 10 - jepsen.util 3430 :invoke :cas [3 [7 5]] INFO [2019-12-17 12:30:30,027] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20323,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:30:30,028] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20323,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:30:30,028] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20324,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:30:30,028] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20324,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:30:30,029] jepsen worker 79 - jepsen.util 3589 :ok :read [26 46] INFO [2019-12-17 12:30:30,029] jepsen worker 79 - jepsen.util 3589 :invoke :read [26 nil] INFO [2019-12-17 12:30:30,063] jepsen worker 28 - jepsen.util 2548 :ok :write [9 33] INFO [2019-12-17 12:30:30,063] jepsen worker 28 - jepsen.util 2548 :invoke :read [9 nil] INFO [2019-12-17 12:30:30,085] jepsen worker 28 - jepsen.util 2548 :ok :read [9 33] INFO [2019-12-17 12:30:30,085] jepsen worker 28 - jepsen.util 2548 :invoke :cas [9 [15 25]] INFO [2019-12-17 12:30:30,102] jepsen worker 28 - jepsen.util 2548 :fail :cas [9 [15 25]] :ValueNotSwapFrom INFO [2019-12-17 12:30:30,103] jepsen worker 28 - jepsen.util 2548 :invoke :cas [9 [35 31]] INFO [2019-12-17 12:30:30,109] jepsen worker 29 - jepsen.util 2819 :info :cas [9 [40 24]] :RequestCanceledException INFO [2019-12-17 12:30:30,109] jepsen worker 12 - jepsen.util 3702 :info :write [4 37] :RequestCanceledException INFO [2019-12-17 12:30:30,110] jepsen worker 29 - jepsen.util 2909 :invoke :write [9 17] INFO [2019-12-17 12:30:30,110] jepsen worker 12 - jepsen.util 3792 :invoke :cas [4 [35 25]] INFO [2019-12-17 12:30:30,118] jepsen worker 28 - jepsen.util 2548 :fail :cas [9 [35 31]] :ValueNotSwapFrom INFO [2019-12-17 12:30:30,119] jepsen worker 28 - jepsen.util 2548 :invoke :write [9 35] INFO [2019-12-17 12:30:30,124] jepsen worker 74 - jepsen.util 3854 :info :write [24 13] :RequestCanceledException INFO [2019-12-17 12:30:30,125] jepsen worker 74 - jepsen.util 3944 :invoke :write [24 6] INFO [2019-12-17 12:30:30,138] jepsen worker 23 - jepsen.util 4613 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:30:30,139] jepsen worker 23 - jepsen.util 4613 :invoke :cas [7 [12 2]] INFO [2019-12-17 12:30:30,174] jepsen worker 28 - jepsen.util 2548 :ok :write [9 35] INFO [2019-12-17 12:30:30,175] jepsen worker 28 - jepsen.util 2548 :invoke :write [9 11] INFO [2019-12-17 12:30:30,192] jepsen worker 71 - jepsen.util 3491 :ok :read [23 16] INFO [2019-12-17 12:30:30,192] jepsen worker 71 - jepsen.util 3491 :invoke :cas [23 [0 47]] INFO [2019-12-17 12:30:30,209] jepsen worker 78 - jepsen.util 3858 :info :cas [26 [37 48]] :RequestCanceledException INFO [2019-12-17 12:30:30,212] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20338,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:30:30,212] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20338,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:30:30,212] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20340,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:30:30,212] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20340,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:30:30,215] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20345,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:30:30,215] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20345,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:30:30,215] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20349,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:30:30,215] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20349,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:30:30,215] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20350,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:30:30,215] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20350,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:30:30,216] jepsen worker 78 - jepsen.util 3948 :invoke :cas [26 [23 44]] INFO [2019-12-17 12:30:30,237] jepsen worker 79 - jepsen.util 3589 :ok :read [26 46] INFO [2019-12-17 12:30:30,237] jepsen worker 79 - jepsen.util 3589 :invoke :cas [26 [46 6]] INFO [2019-12-17 12:30:30,255] jepsen worker 28 - jepsen.util 2548 :ok :write [9 11] INFO [2019-12-17 12:30:30,255] jepsen worker 28 - jepsen.util 2548 :invoke :read [9 nil] INFO [2019-12-17 12:30:30,255] jepsen worker 59 - jepsen.util 4559 :info :write [19 26] :RequestCanceledException INFO [2019-12-17 12:30:30,256] jepsen worker 59 - jepsen.util 4649 :invoke :write [19 7] INFO [2019-12-17 12:30:30,275] jepsen worker 28 - jepsen.util 2548 :ok :read [9 11] INFO [2019-12-17 12:30:30,275] jepsen worker 28 - jepsen.util 2548 :invoke :read [9 nil] INFO [2019-12-17 12:30:30,289] jepsen worker 5 - jepsen.util 3155 :fail :cas [1 [41 7]] :ValueNotSwapFrom INFO [2019-12-17 12:30:30,289] jepsen worker 5 - jepsen.util 3155 :invoke :write [1 40] INFO [2019-12-17 12:30:30,293] jepsen worker 28 - jepsen.util 2548 :ok :read [9 11] INFO [2019-12-17 12:30:30,293] jepsen worker 28 - jepsen.util 2548 :invoke :write [9 24] INFO [2019-12-17 12:30:30,299] jepsen worker 71 - jepsen.util 3491 :fail :cas [23 [0 47]] :ValueNotSwapFrom INFO [2019-12-17 12:30:30,299] jepsen worker 71 - jepsen.util 3491 :invoke :read [23 nil] INFO [2019-12-17 12:30:30,324] jepsen worker 72 - jepsen.util 3492 :info :write [24 40] :RequestCanceledException INFO [2019-12-17 12:30:30,324] jepsen worker 55 - jepsen.util 4285 :info :write [18 33] :RequestCanceledException INFO [2019-12-17 12:30:30,324] jepsen worker 88 - jepsen.util 3598 :info :write [29 47] :RequestCanceledException INFO [2019-12-17 12:30:30,326] jepsen worker 72 - jepsen.util 3582 :invoke :cas [24 [43 25]] INFO [2019-12-17 12:30:30,326] jepsen worker 55 - jepsen.util 4375 :invoke :write [18 17] INFO [2019-12-17 12:30:30,326] jepsen worker 88 - jepsen.util 3688 :invoke :write [29 36] INFO [2019-12-17 12:30:30,333] jepsen worker 37 - jepsen.util 3367 :ok :read [12 29] INFO [2019-12-17 12:30:30,333] jepsen worker 37 - jepsen.util 3367 :invoke :write [12 11] INFO [2019-12-17 12:30:30,342] jepsen worker 28 - jepsen.util 2548 :ok :write [9 24] INFO [2019-12-17 12:30:30,342] jepsen worker 28 - jepsen.util 2548 :invoke :cas [9 [30 24]] INFO [2019-12-17 12:30:30,357] jepsen worker 28 - jepsen.util 2548 :fail :cas [9 [30 24]] :ValueNotSwapFrom INFO [2019-12-17 12:30:30,357] jepsen worker 28 - jepsen.util 2548 :invoke :write [9 5] INFO [2019-12-17 12:30:30,397] jepsen worker 71 - jepsen.util 3491 :ok :read [23 16] INFO [2019-12-17 12:30:30,397] jepsen worker 71 - jepsen.util 3491 :invoke :write [23 21] INFO [2019-12-17 12:30:30,404] jepsen worker 54 - jepsen.util 3744 :fail :cas [18 [28 11]] :ValueNotSwapFrom INFO [2019-12-17 12:30:30,404] jepsen worker 54 - jepsen.util 3744 :invoke :read [18 nil] INFO [2019-12-17 12:30:30,408] jepsen worker 27 - jepsen.util 3267 :info :write [9 4] :RequestCanceledException INFO [2019-12-17 12:30:30,409] jepsen worker 27 - jepsen.util 3357 :invoke :cas [9 [19 29]] INFO [2019-12-17 12:30:30,423] jepsen worker 28 - jepsen.util 2548 :ok :write [9 5] INFO [2019-12-17 12:30:30,423] jepsen worker 28 - jepsen.util 2548 :invoke :read [9 nil] INFO [2019-12-17 12:30:30,424] jepsen worker 80 - jepsen.util 3410 :info :write [26 27] :RequestCanceledException INFO [2019-12-17 12:30:30,426] jepsen worker 80 - jepsen.util 3500 :invoke :write [26 4] INFO [2019-12-17 12:30:30,429] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20370,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:30:30,429] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20370,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:30:30,429] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20373,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:30:30,429] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20373,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:30:30,429] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20374,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:30:30,429] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20374,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:30:30,439] jepsen worker 28 - jepsen.util 2548 :ok :read [9 5] INFO [2019-12-17 12:30:30,439] jepsen worker 28 - jepsen.util 2548 :invoke :read [9 nil] INFO [2019-12-17 12:30:30,439] jepsen worker 13 - jepsen.util 3343 :fail :cas [4 [5 38]] :ValueNotSwapFrom INFO [2019-12-17 12:30:30,439] jepsen worker 13 - jepsen.util 3343 :invoke :write [4 39] INFO [2019-12-17 12:30:30,484] jepsen worker 43 - jepsen.util 4273 :info :cas [14 [44 12]] :RequestCanceledException INFO [2019-12-17 12:30:30,486] jepsen worker 43 - jepsen.util 4363 :invoke :write [14 48] INFO [2019-12-17 12:30:30,490] jepsen worker 73 - jepsen.util 3403 :info :write [24 15] :RequestCanceledException INFO [2019-12-17 12:30:30,492] jepsen worker 73 - jepsen.util 3493 :invoke :cas [24 [32 19]] INFO [2019-12-17 12:30:30,492] jepsen worker 28 - jepsen.util 2548 :ok :read [9 5] INFO [2019-12-17 12:30:30,493] jepsen worker 28 - jepsen.util 2548 :invoke :cas [9 [5 44]] INFO [2019-12-17 12:30:30,500] jepsen worker 84 - jepsen.util 4494 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:30:30,501] jepsen worker 84 - jepsen.util 4494 :invoke :cas [28 [4 38]] INFO [2019-12-17 12:30:30,504] jepsen worker 54 - jepsen.util 3744 :ok :read [18 36] INFO [2019-12-17 12:30:30,505] jepsen worker 54 - jepsen.util 3744 :invoke :write [18 7] INFO [2019-12-17 12:30:30,524] jepsen worker 66 - jepsen.util 4566 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:30:30,524] jepsen worker 45 - jepsen.util 3555 :info :write [15 20] :RequestCanceledException INFO [2019-12-17 12:30:30,524] jepsen worker 36 - jepsen.util 3636 :info :write [12 26] :RequestCanceledException INFO [2019-12-17 12:30:30,524] jepsen worker 66 - jepsen.util 4566 :invoke :read [22 nil] INFO [2019-12-17 12:30:30,525] jepsen worker 36 - jepsen.util 3726 :invoke :cas [12 [37 45]] INFO [2019-12-17 12:30:30,536] jepsen worker 39 - jepsen.util 4359 :info :cas [13 [7 13]] :RequestCanceledException INFO [2019-12-17 12:30:30,537] jepsen worker 39 - jepsen.util 4449 :invoke :write [13 5] INFO [2019-12-17 12:30:30,539] jepsen worker 45 - jepsen.util 3645 :invoke :cas [15 [30 23]] INFO [2019-12-17 12:30:30,549] jepsen worker 28 - jepsen.util 2548 :ok :cas [9 [5 44]] INFO [2019-12-17 12:30:30,549] jepsen worker 28 - jepsen.util 2548 :invoke :write [9 8] INFO [2019-12-17 12:30:30,555] jepsen worker 25 - jepsen.util 4975 :info :write [8 42] :RequestCanceledException INFO [2019-12-17 12:30:30,556] jepsen worker 25 - jepsen.util 5065 :invoke :cas [8 [22 35]] INFO [2019-12-17 12:30:30,591] jepsen worker 62 - jepsen.util 4472 :info :write [20 8] :RequestCanceledException INFO [2019-12-17 12:30:30,592] jepsen worker 62 - jepsen.util 4562 :invoke :write [20 10] INFO [2019-12-17 12:30:30,597] jepsen worker 28 - jepsen.util 2548 :ok :write [9 8] INFO [2019-12-17 12:30:30,597] jepsen worker 28 - jepsen.util 2548 :invoke :read [9 nil] INFO [2019-12-17 12:30:30,608] jepsen worker 82 - jepsen.util 5572 :info :cas [27 [4 27]] :RequestCanceledException INFO [2019-12-17 12:30:30,609] jepsen worker 82 - jepsen.util 5662 :invoke :write [27 1] INFO [2019-12-17 12:30:30,618] jepsen worker 28 - jepsen.util 2548 :ok :read [9 8] INFO [2019-12-17 12:30:30,618] jepsen worker 28 - jepsen.util 2548 :invoke :read [9 nil] INFO [2019-12-17 12:30:30,623] jepsen worker 85 - jepsen.util 5125 :info :cas [28 [22 14]] :RequestCanceledException INFO [2019-12-17 12:30:30,623] jepsen worker 7 - jepsen.util 4417 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:30:30,623] jepsen worker 52 - jepsen.util 3562 :info :write [17 15] :RequestCanceledException INFO [2019-12-17 12:30:30,623] jepsen worker 58 - jepsen.util 4738 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:30:30,623] jepsen worker 7 - jepsen.util 4417 :invoke :write [2 17] INFO [2019-12-17 12:30:30,624] jepsen worker 58 - jepsen.util 4738 :invoke :write [19 23] INFO [2019-12-17 12:30:30,625] jepsen worker 52 - jepsen.util 3652 :invoke :write [17 21] INFO [2019-12-17 12:30:30,626] jepsen worker 85 - jepsen.util 5215 :invoke :write [28 26] INFO [2019-12-17 12:30:30,629] jepsen worker 50 - jepsen.util 5090 :info :cas [16 [24 43]] :RequestCanceledException INFO [2019-12-17 12:30:30,629] jepsen worker 6 - jepsen.util 4956 :info :cas [2 [25 33]] :RequestCanceledException INFO [2019-12-17 12:30:30,631] jepsen worker 50 - jepsen.util 5180 :invoke :cas [16 [12 34]] INFO [2019-12-17 12:30:30,631] jepsen worker 6 - jepsen.util 5046 :invoke :cas [2 [12 47]] INFO [2019-12-17 12:30:30,636] jepsen worker 28 - jepsen.util 2548 :ok :read [9 8] INFO [2019-12-17 12:30:30,637] jepsen worker 28 - jepsen.util 2548 :invoke :write [9 32] INFO [2019-12-17 12:30:30,687] jepsen worker 28 - jepsen.util 2548 :ok :write [9 32] INFO [2019-12-17 12:30:30,687] jepsen worker 28 - jepsen.util 2548 :invoke :read [9 nil] INFO [2019-12-17 12:30:30,697] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20399,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:30:30,697] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20399,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:30:30,698] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20403,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:30:30,698] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20403,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:30:30,702] jepsen worker 28 - jepsen.util 2548 :ok :read [9 32] INFO [2019-12-17 12:30:30,702] jepsen worker 28 - jepsen.util 2548 :invoke :cas [9 [39 4]] INFO [2019-12-17 12:30:30,704] jepsen worker 12 - jepsen.util 3792 :fail :cas [4 [35 25]] :ValueNotSwapFrom INFO [2019-12-17 12:30:30,704] jepsen worker 12 - jepsen.util 3792 :invoke :write [4 40] INFO [2019-12-17 12:30:30,717] jepsen worker 28 - jepsen.util 2548 :fail :cas [9 [39 4]] :ValueNotSwapFrom INFO [2019-12-17 12:30:30,717] jepsen worker 28 - jepsen.util 2548 :invoke :read [9 nil] INFO [2019-12-17 12:30:30,728] jepsen worker 28 - jepsen.util 2548 :ok :read [9 32] INFO [2019-12-17 12:30:30,728] jepsen worker 28 - jepsen.util 2548 :invoke :read [9 nil] INFO [2019-12-17 12:30:30,738] jepsen worker 26 - jepsen.util 4976 :info :cas [8 [39 43]] :RequestCanceledException INFO [2019-12-17 12:30:30,739] jepsen worker 28 - jepsen.util 2548 :ok :read [9 32] INFO [2019-12-17 12:30:30,739] jepsen worker 28 - jepsen.util 2548 :invoke :read [9 nil] INFO [2019-12-17 12:30:30,739] jepsen worker 26 - jepsen.util 5066 :invoke :read [8 nil] INFO [2019-12-17 12:30:30,742] jepsen worker 78 - jepsen.util 3948 :fail :cas [26 [23 44]] :ValueNotSwapFrom INFO [2019-12-17 12:30:30,743] jepsen worker 78 - jepsen.util 3948 :invoke :write [26 44] INFO [2019-12-17 12:30:30,752] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:30:30,752] jepsen nemesis - couchbase.util Rebalance check count: 150 INFO [2019-12-17 12:30:30,752] jepsen nemesis - couchbase.util Rebalance stuck count: 131 INFO [2019-12-17 12:30:30,755] jepsen worker 28 - jepsen.util 2548 :ok :read [9 32] INFO [2019-12-17 12:30:30,755] jepsen worker 28 - jepsen.util 2548 :invoke :read [9 nil] INFO [2019-12-17 12:30:30,769] jepsen worker 28 - jepsen.util 2548 :ok :read [9 32] INFO [2019-12-17 12:30:30,769] jepsen worker 28 - jepsen.util 2548 :invoke :cas [9 [38 31]] INFO [2019-12-17 12:30:30,783] jepsen worker 28 - jepsen.util 2548 :fail :cas [9 [38 31]] :ValueNotSwapFrom INFO [2019-12-17 12:30:30,784] jepsen worker 28 - jepsen.util 2548 :invoke :read [9 nil] INFO [2019-12-17 12:30:30,791] jepsen worker 42 - jepsen.util 5532 :info :cas [14 [44 0]] :RequestCanceledException INFO [2019-12-17 12:30:30,791] jepsen worker 75 - jepsen.util 4665 :info :cas [25 [22 43]] :RequestCanceledException INFO [2019-12-17 12:30:30,791] jepsen worker 4 - jepsen.util 2704 :info :write [1 9] :RequestCanceledException INFO [2019-12-17 12:30:30,791] jepsen worker 40 - jepsen.util 4990 :info :cas [13 [24 39]] :RequestCanceledException INFO [2019-12-17 12:30:30,792] jepsen worker 75 - jepsen.util 4755 :invoke :write [25 46] INFO [2019-12-17 12:30:30,792] jepsen worker 4 - jepsen.util 2794 :invoke :cas [1 [6 22]] INFO [2019-12-17 12:30:30,792] jepsen worker 40 - jepsen.util 5080 :invoke :read [13 nil] INFO [2019-12-17 12:30:30,792] jepsen worker 42 - jepsen.util 5622 :invoke :read [14 nil] INFO [2019-12-17 12:30:30,797] jepsen worker 28 - jepsen.util 2548 :ok :read [9 32] INFO [2019-12-17 12:30:30,797] jepsen worker 28 - jepsen.util 2548 :invoke :write [9 24] INFO [2019-12-17 12:30:30,806] jepsen worker 47 - jepsen.util 4097 :ok :write [15 22] INFO [2019-12-17 12:30:30,806] jepsen worker 47 - jepsen.util 4097 :invoke :read [15 nil] INFO [2019-12-17 12:30:30,806] jepsen worker 18 - jepsen.util 3618 :ok :write [6 7] INFO [2019-12-17 12:30:30,807] jepsen worker 18 - jepsen.util 3618 :invoke :write [6 28] INFO [2019-12-17 12:30:30,809] jepsen worker 22 - jepsen.util 4882 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:30:30,809] jepsen worker 38 - jepsen.util 3458 :info :write [12 46] :RequestCanceledException INFO [2019-12-17 12:30:30,809] jepsen worker 67 - jepsen.util 5017 :info :write [22 47] :RequestCanceledException INFO [2019-12-17 12:30:30,809] jepsen worker 22 - jepsen.util 4882 :invoke :cas [7 [42 48]] INFO [2019-12-17 12:30:30,810] jepsen worker 38 - jepsen.util 3548 :invoke :cas [12 [16 15]] INFO [2019-12-17 12:30:30,810] jepsen worker 67 - jepsen.util 5107 :invoke :read [22 nil] INFO [2019-12-17 12:30:30,813] jepsen worker 9 - jepsen.util 4149 :ok :write [3 15] INFO [2019-12-17 12:30:30,814] jepsen worker 9 - jepsen.util 4149 :invoke :cas [3 [4 8]] INFO [2019-12-17 12:30:30,823] jepsen worker 2 - jepsen.util 4412 :info :cas [0 [14 46]] :RequestCanceledException INFO [2019-12-17 12:30:30,823] jepsen worker 60 - jepsen.util 4920 :info :cas [20 [31 8]] :RequestCanceledException INFO [2019-12-17 12:30:30,824] jepsen worker 83 - jepsen.util 4763 :info :cas [27 [2 21]] :RequestCanceledException INFO [2019-12-17 12:30:30,824] jepsen worker 53 - jepsen.util 3563 :info :write [17 29] :RequestCanceledException INFO [2019-12-17 12:30:30,824] jepsen worker 14 - jepsen.util 3434 :info :write [4 31] :RequestCanceledException INFO [2019-12-17 12:30:30,824] jepsen worker 31 - jepsen.util 3001 :info :write [10 7] :RequestCanceledException INFO [2019-12-17 12:30:30,824] jepsen worker 35 - jepsen.util 4445 :info :write [11 17] :RequestCanceledException INFO [2019-12-17 12:30:30,824] jepsen worker 60 - jepsen.util 5010 :invoke :read [20 nil] INFO [2019-12-17 12:30:30,825] jepsen worker 14 - jepsen.util 3524 :invoke :write [4 28] INFO [2019-12-17 12:30:30,825] jepsen worker 83 - jepsen.util 4853 :invoke :cas [27 [2 45]] INFO [2019-12-17 12:30:30,825] jepsen worker 31 - jepsen.util 3091 :invoke :write [10 2] INFO [2019-12-17 12:30:30,825] jepsen worker 53 - jepsen.util 3653 :invoke :cas [17 [19 7]] INFO [2019-12-17 12:30:30,825] jepsen worker 35 - jepsen.util 4535 :invoke :cas [11 [9 31]] INFO [2019-12-17 12:30:30,825] jepsen worker 2 - jepsen.util 4502 :invoke :read [0 nil] INFO [2019-12-17 12:30:30,836] jepsen worker 33 - jepsen.util 4983 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:30:30,836] jepsen worker 33 - jepsen.util 4983 :invoke :cas [11 [38 33]] INFO [2019-12-17 12:30:30,855] jepsen worker 41 - jepsen.util 4901 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:30:30,855] jepsen worker 32 - jepsen.util 3992 :info :write [10 19] :RequestCanceledException INFO [2019-12-17 12:30:30,855] jepsen worker 56 - jepsen.util 3566 :info :write [18 35] :RequestCanceledException INFO [2019-12-17 12:30:30,855] jepsen worker 41 - jepsen.util 4901 :invoke :cas [13 [44 27]] INFO [2019-12-17 12:30:30,856] jepsen worker 32 - jepsen.util 4082 :invoke :write [10 17] INFO [2019-12-17 12:30:30,856] jepsen worker 56 - jepsen.util 3656 :invoke :cas [18 [21 29]] INFO [2019-12-17 12:30:30,864] jepsen worker 10 - jepsen.util 3430 :fail :cas [3 [7 5]] :ValueNotSwapFrom INFO [2019-12-17 12:30:30,864] jepsen worker 10 - jepsen.util 3430 :invoke :write [3 25] INFO [2019-12-17 12:30:30,874] jepsen worker 30 - jepsen.util 3900 :ok :write [10 36] INFO [2019-12-17 12:30:30,874] jepsen worker 30 - jepsen.util 3900 :invoke :write [10 31] INFO [2019-12-17 12:30:30,881] jepsen worker 63 - jepsen.util 3213 :ok :read [21 34] INFO [2019-12-17 12:30:30,881] jepsen worker 63 - jepsen.util 3213 :invoke :read [21 nil] INFO [2019-12-17 12:30:30,890] jepsen worker 48 - jepsen.util 4098 :info :write [16 40] :RequestCanceledException INFO [2019-12-17 12:30:30,892] jepsen worker 48 - jepsen.util 4188 :invoke :write [16 30] INFO [2019-12-17 12:30:30,895] jepsen worker 36 - jepsen.util 3726 :fail :cas [12 [37 45]] :ValueNotSwapFrom INFO [2019-12-17 12:30:30,895] jepsen worker 36 - jepsen.util 3726 :invoke :read [12 nil] INFO [2019-12-17 12:30:30,907] jepsen worker 72 - jepsen.util 3582 :fail :cas [24 [43 25]] :ValueNotSwapFrom INFO [2019-12-17 12:30:30,907] jepsen worker 72 - jepsen.util 3582 :invoke :cas [24 [7 45]] INFO [2019-12-17 12:30:30,908] jepsen worker 8 - jepsen.util 3968 :info :write [2 28] :RequestCanceledException INFO [2019-12-17 12:30:30,909] jepsen worker 8 - jepsen.util 4058 :invoke :write [2 18] INFO [2019-12-17 12:30:30,916] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20443,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:30:30,917] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20443,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:30:30,917] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20445,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:30:30,917] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20445,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:30:30,922] jepsen worker 3 - jepsen.util 3693 :info :write [1 0] :RequestCanceledException INFO [2019-12-17 12:30:30,922] jepsen worker 17 - jepsen.util 4427 :info :write [5 47] :RequestCanceledException INFO [2019-12-17 12:30:30,922] jepsen worker 57 - jepsen.util 4467 :info :cas [19 [8 25]] :RequestCanceledException INFO [2019-12-17 12:30:30,923] jepsen worker 3 - jepsen.util 3783 :invoke :write [1 12] INFO [2019-12-17 12:30:30,923] jepsen worker 57 - jepsen.util 4557 :invoke :write [19 14] INFO [2019-12-17 12:30:30,923] jepsen worker 17 - jepsen.util 4517 :invoke :read [5 nil] INFO [2019-12-17 12:30:30,928] jepsen worker 28 - jepsen.util 2548 :ok :write [9 24] INFO [2019-12-17 12:30:30,929] jepsen worker 28 - jepsen.util 2548 :invoke :cas [9 [9 14]] INFO [2019-12-17 12:30:30,932] jepsen worker 73 - jepsen.util 3493 :fail :cas [24 [32 19]] :ValueNotSwapFrom INFO [2019-12-17 12:30:30,932] jepsen worker 73 - jepsen.util 3493 :invoke :read [24 nil] INFO [2019-12-17 12:30:30,938] jepsen worker 0 - jepsen.util 5130 :info :write [0 14] :RequestCanceledException INFO [2019-12-17 12:30:30,939] jepsen worker 0 - jepsen.util 5220 :invoke :read [0 nil] INFO [2019-12-17 12:30:30,963] jepsen worker 9 - jepsen.util 4149 :fail :cas [3 [4 8]] :ValueNotSwapFrom INFO [2019-12-17 12:30:30,963] jepsen worker 9 - jepsen.util 4149 :invoke :read [3 nil] INFO [2019-12-17 12:30:30,978] jepsen worker 29 - jepsen.util 2909 :ok :write [9 17] INFO [2019-12-17 12:30:30,979] jepsen worker 29 - jepsen.util 2909 :invoke :read [9 nil] INFO [2019-12-17 12:30:31,025] jepsen worker 47 - jepsen.util 4097 :ok :read [15 22] INFO [2019-12-17 12:30:31,026] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20463,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:30:31,033] jepsen worker 28 - jepsen.util 2548 :fail :cas [9 [9 14]] :ValueNotSwapFrom INFO [2019-12-17 12:30:31,112] jepsen worker 47 - jepsen.util 4097 :invoke :read [15 nil] INFO [2019-12-17 12:30:31,112] jepsen worker 28 - jepsen.util 2548 :invoke :read [9 nil] INFO [2019-12-17 12:30:31,112] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20463,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:30:31,112] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20466,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:30:31,112] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20466,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:30:31,112] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20467,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:30:31,112] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20467,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:30:31,131] jepsen worker 28 - jepsen.util 2548 :ok :read [9 17] INFO [2019-12-17 12:30:31,131] jepsen worker 28 - jepsen.util 2548 :invoke :cas [9 [45 32]] INFO [2019-12-17 12:30:31,153] jepsen worker 28 - jepsen.util 2548 :fail :cas [9 [45 32]] :ValueNotSwapFrom INFO [2019-12-17 12:30:31,153] jepsen worker 28 - jepsen.util 2548 :invoke :write [9 26] INFO [2019-12-17 12:30:31,181] jepsen worker 32 - jepsen.util 4082 :ok :write [10 17] INFO [2019-12-17 12:30:31,181] jepsen worker 32 - jepsen.util 4082 :invoke :cas [10 [24 21]] INFO [2019-12-17 12:30:31,181] jepsen worker 74 - jepsen.util 3944 :ok :write [24 6] INFO [2019-12-17 12:30:31,181] jepsen worker 19 - jepsen.util 3709 :ok :write [6 43] INFO [2019-12-17 12:30:31,182] jepsen worker 74 - jepsen.util 3944 :invoke :cas [24 [8 10]] INFO [2019-12-17 12:30:31,182] jepsen worker 19 - jepsen.util 3709 :invoke :write [6 8] INFO [2019-12-17 12:30:31,193] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20476,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:30:31,193] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20476,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:30:31,193] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20477,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:30:31,193] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20477,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:30:31,201] jepsen worker 15 - jepsen.util 4335 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:30:31,201] jepsen worker 15 - jepsen.util 4335 :invoke :read [5 nil] INFO [2019-12-17 12:30:31,205] jepsen worker 4 - jepsen.util 2794 :fail :cas [1 [6 22]] :ValueNotSwapFrom INFO [2019-12-17 12:30:31,205] jepsen worker 4 - jepsen.util 2794 :invoke :read [1 nil] INFO [2019-12-17 12:30:31,224] jepsen worker 69 - jepsen.util 3399 :info :write [23 0] :RequestCanceledException INFO [2019-12-17 12:30:31,224] jepsen worker 64 - jepsen.util 2854 :fail :read [21 nil] :RequestTimeoutException INFO [2019-12-17 12:30:31,225] jepsen worker 64 - jepsen.util 2854 :invoke :cas [21 [11 36]] INFO [2019-12-17 12:30:31,226] jepsen worker 69 - jepsen.util 3489 :invoke :write [23 35] INFO [2019-12-17 12:30:31,230] jepsen worker 29 - jepsen.util 2909 :ok :read [9 17] INFO [2019-12-17 12:30:31,230] jepsen worker 29 - jepsen.util 2909 :invoke :read [9 nil] INFO [2019-12-17 12:30:31,240] jepsen worker 10 - jepsen.util 3430 :ok :write [3 25] INFO [2019-12-17 12:30:31,240] jepsen worker 10 - jepsen.util 3430 :invoke :write [3 23] INFO [2019-12-17 12:30:31,250] jepsen worker 11 - jepsen.util 3791 :fail :cas [3 [24 32]] :ValueNotSwapFrom INFO [2019-12-17 12:30:31,250] jepsen worker 11 - jepsen.util 3791 :invoke :cas [3 [46 6]] INFO [2019-12-17 12:30:31,271] jepsen worker 45 - jepsen.util 3645 :fail :cas [15 [30 23]] :ValueNotSwapFrom INFO [2019-12-17 12:30:31,272] jepsen worker 45 - jepsen.util 3645 :invoke :write [15 49] INFO [2019-12-17 12:30:31,278] jepsen worker 28 - jepsen.util 2548 :ok :write [9 26] INFO [2019-12-17 12:30:31,279] jepsen worker 28 - jepsen.util 2548 :invoke :write [9 6] INFO [2019-12-17 12:30:31,287] jepsen worker 87 - jepsen.util 3237 :info :write [29 20] :RequestCanceledException INFO [2019-12-17 12:30:31,287] jepsen worker 76 - jepsen.util 4756 :info :write [25 32] :RequestCanceledException INFO [2019-12-17 12:30:31,288] jepsen worker 76 - jepsen.util 4846 :invoke :cas [25 [31 13]] INFO [2019-12-17 12:30:31,289] jepsen worker 87 - jepsen.util 3327 :invoke :write [29 37] INFO [2019-12-17 12:30:31,290] jepsen worker 47 - jepsen.util 4097 :ok :read [15 22] INFO [2019-12-17 12:30:31,290] jepsen worker 47 - jepsen.util 4097 :invoke :write [15 14] INFO [2019-12-17 12:30:31,300] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20484,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:30:31,301] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20484,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:30:31,301] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20486,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:30:31,301] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20486,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:30:31,301] jepsen worker 65 - jepsen.util 3035 :fail :read [21 nil] :RequestTimeoutException INFO [2019-12-17 12:30:31,302] jepsen worker 65 - jepsen.util 3035 :invoke :write [21 20] INFO [2019-12-17 12:30:31,323] jepsen worker 1 - jepsen.util 4231 :info :write [0 33] :RequestCanceledException INFO [2019-12-17 12:30:31,323] jepsen worker 46 - jepsen.util 3736 :info :write [15 3] :RequestCanceledException INFO [2019-12-17 12:30:31,323] jepsen worker 16 - jepsen.util 5236 :info :cas [5 [26 40]] :RequestCanceledException INFO [2019-12-17 12:30:31,325] jepsen worker 16 - jepsen.util 5326 :invoke :cas [5 [9 2]] INFO [2019-12-17 12:30:31,325] jepsen worker 1 - jepsen.util 4321 :invoke :cas [0 [6 24]] INFO [2019-12-17 12:30:31,326] jepsen worker 46 - jepsen.util 3826 :invoke :read [15 nil] INFO [2019-12-17 12:30:31,333] jepsen worker 32 - jepsen.util 4082 :fail :cas [10 [24 21]] :ValueNotSwapFrom INFO [2019-12-17 12:30:31,333] jepsen worker 32 - jepsen.util 4082 :invoke :cas [10 [44 39]] INFO [2019-12-17 12:30:31,338] jepsen worker 70 - jepsen.util 4030 :info :write [23 44] :RequestCanceledException INFO [2019-12-17 12:30:31,339] jepsen worker 70 - jepsen.util 4120 :invoke :read [23 nil] INFO [2019-12-17 12:30:31,355] jepsen worker 29 - jepsen.util 2909 :ok :read [9 26] INFO [2019-12-17 12:30:31,355] jepsen worker 29 - jepsen.util 2909 :invoke :read [9 nil] INFO [2019-12-17 12:30:31,375] jepsen worker 38 - jepsen.util 3548 :fail :cas [12 [16 15]] :ValueNotSwapFrom INFO [2019-12-17 12:30:31,375] jepsen worker 38 - jepsen.util 3548 :invoke :read [12 nil] INFO [2019-12-17 12:30:31,393] jepsen worker 36 - jepsen.util 3726 :ok :read [12 29] INFO [2019-12-17 12:30:31,393] jepsen worker 36 - jepsen.util 3726 :invoke :read [12 nil] INFO [2019-12-17 12:30:31,403] jepsen worker 53 - jepsen.util 3653 :fail :cas [17 [19 7]] :ValueNotSwapFrom INFO [2019-12-17 12:30:31,403] jepsen worker 53 - jepsen.util 3653 :invoke :cas [17 [26 14]] INFO [2019-12-17 12:30:31,424] jepsen worker 81 - jepsen.util 4761 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:30:31,424] jepsen worker 81 - jepsen.util 4761 :invoke :write [27 35] INFO [2019-12-17 12:30:31,438] jepsen worker 61 - jepsen.util 5371 :info :cas [20 [12 19]] :RequestCanceledException INFO [2019-12-17 12:30:31,439] jepsen worker 61 - jepsen.util 5461 :invoke :read [20 nil] INFO [2019-12-17 12:30:31,442] jepsen worker 28 - jepsen.util 2548 :ok :write [9 6] INFO [2019-12-17 12:30:31,442] jepsen worker 28 - jepsen.util 2548 :invoke :write [9 23] INFO [2019-12-17 12:30:31,455] jepsen worker 21 - jepsen.util 4791 :info :cas [7 [10 15]] :RequestCanceledException INFO [2019-12-17 12:30:31,456] jepsen worker 21 - jepsen.util 4881 :invoke :read [7 nil] INFO [2019-12-17 12:30:31,487] jepsen worker 28 - jepsen.util 2548 :ok :write [9 23] INFO [2019-12-17 12:30:31,487] jepsen worker 28 - jepsen.util 2548 :invoke :write [9 1] INFO [2019-12-17 12:30:31,498] jepsen worker 70 - jepsen.util 4120 :ok :read [23 16] INFO [2019-12-17 12:30:31,498] jepsen worker 70 - jepsen.util 4120 :invoke :read [23 nil] INFO [2019-12-17 12:30:31,520] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20510,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:30:31,520] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20510,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:30:31,520] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20513,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:30:31,520] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20513,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:30:31,542] jepsen worker 28 - jepsen.util 2548 :ok :write [9 1] INFO [2019-12-17 12:30:31,542] jepsen worker 28 - jepsen.util 2548 :invoke :cas [9 [17 19]] INFO [2019-12-17 12:30:31,549] jepsen worker 70 - jepsen.util 4120 :ok :read [23 16] INFO [2019-12-17 12:30:31,549] jepsen worker 70 - jepsen.util 4120 :invoke :read [23 nil] INFO [2019-12-17 12:30:31,567] jepsen worker 30 - jepsen.util 3900 :ok :write [10 31] INFO [2019-12-17 12:30:31,568] jepsen worker 30 - jepsen.util 3900 :invoke :cas [10 [16 26]] INFO [2019-12-17 12:30:31,574] jepsen worker 28 - jepsen.util 2548 :fail :cas [9 [17 19]] :ValueNotSwapFrom INFO [2019-12-17 12:30:31,574] jepsen worker 28 - jepsen.util 2548 :invoke :cas [9 [49 43]] INFO [2019-12-17 12:30:31,590] jepsen worker 89 - jepsen.util 3329 :info :write [29 17] :RequestCanceledException INFO [2019-12-17 12:30:31,591] jepsen worker 89 - jepsen.util 3419 :invoke :cas [29 [25 1]] INFO [2019-12-17 12:30:31,603] jepsen worker 28 - jepsen.util 2548 :fail :cas [9 [49 43]] :ValueNotSwapFrom INFO [2019-12-17 12:30:31,604] jepsen worker 28 - jepsen.util 2548 :invoke :write [9 28] INFO [2019-12-17 12:30:31,637] jepsen worker 56 - jepsen.util 3656 :fail :cas [18 [21 29]] :ValueNotSwapFrom INFO [2019-12-17 12:30:31,637] jepsen worker 56 - jepsen.util 3656 :invoke :cas [18 [32 22]] INFO [2019-12-17 12:30:31,653] jepsen worker 28 - jepsen.util 2548 :ok :write [9 28] INFO [2019-12-17 12:30:31,653] jepsen worker 28 - jepsen.util 2548 :invoke :read [9 nil] INFO [2019-12-17 12:30:31,672] jepsen worker 28 - jepsen.util 2548 :ok :read [9 28] INFO [2019-12-17 12:30:31,672] jepsen worker 28 - jepsen.util 2548 :invoke :read [9 nil] INFO [2019-12-17 12:30:31,683] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20529,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:30:31,683] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20529,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:30:31,683] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20532,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:30:31,683] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20532,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:30:31,683] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20533,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:30:31,683] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20533,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:30:31,684] jepsen worker 28 - jepsen.util 2548 :ok :read [9 28] INFO [2019-12-17 12:30:31,684] jepsen worker 28 - jepsen.util 2548 :invoke :write [9 5] INFO [2019-12-17 12:30:31,726] jepsen worker 28 - jepsen.util 2548 :ok :write [9 5] INFO [2019-12-17 12:30:31,727] jepsen worker 28 - jepsen.util 2548 :invoke :cas [9 [33 25]] INFO [2019-12-17 12:30:31,739] jepsen worker 89 - jepsen.util 3419 :fail :cas [29 [25 1]] :ValueNotSwapFrom INFO [2019-12-17 12:30:31,739] jepsen worker 89 - jepsen.util 3419 :invoke :write [29 7] INFO [2019-12-17 12:30:31,746] jepsen worker 28 - jepsen.util 2548 :fail :cas [9 [33 25]] :ValueNotSwapFrom INFO [2019-12-17 12:30:31,746] jepsen worker 28 - jepsen.util 2548 :invoke :read [9 nil] INFO [2019-12-17 12:30:31,760] jepsen worker 28 - jepsen.util 2548 :ok :read [9 5] INFO [2019-12-17 12:30:31,760] jepsen worker 28 - jepsen.util 2548 :invoke :write [9 8] INFO [2019-12-17 12:30:31,768] jepsen worker 56 - jepsen.util 3656 :fail :cas [18 [32 22]] :ValueNotSwapFrom INFO [2019-12-17 12:30:31,768] jepsen worker 56 - jepsen.util 3656 :invoke :read [18 nil] INFO [2019-12-17 12:30:31,777] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:30:31,777] jepsen nemesis - couchbase.util Rebalance check count: 151 INFO [2019-12-17 12:30:31,777] jepsen nemesis - couchbase.util Rebalance stuck count: 132 INFO [2019-12-17 12:30:31,790] jepsen worker 86 - jepsen.util 4856 :info :write [28 29] :RequestCanceledException INFO [2019-12-17 12:30:31,791] jepsen worker 86 - jepsen.util 4946 :invoke :write [28 38] INFO [2019-12-17 12:30:31,803] jepsen worker 28 - jepsen.util 2548 :ok :write [9 8] INFO [2019-12-17 12:30:31,803] jepsen worker 28 - jepsen.util 2548 :invoke :write [9 37] INFO [2019-12-17 12:30:31,822] jepsen worker 56 - jepsen.util 3656 :ok :read [18 36] INFO [2019-12-17 12:30:31,822] jepsen worker 56 - jepsen.util 3656 :invoke :read [18 nil] INFO [2019-12-17 12:30:31,847] jepsen worker 28 - jepsen.util 2548 :ok :write [9 37] INFO [2019-12-17 12:30:31,847] jepsen worker 28 - jepsen.util 2548 :invoke :write [9 46] INFO [2019-12-17 12:30:31,855] jepsen worker 51 - jepsen.util 3111 :info :write [17 46] :RequestCanceledException INFO [2019-12-17 12:30:31,856] jepsen worker 51 - jepsen.util 3201 :invoke :read [17 nil] INFO [2019-12-17 12:30:31,893] jepsen worker 68 - jepsen.util 4298 :info :cas [22 [17 42]] :RequestCanceledException INFO [2019-12-17 12:30:31,894] jepsen worker 68 - jepsen.util 4388 :invoke :read [22 nil] INFO [2019-12-17 12:30:31,901] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20548,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:30:31,902] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20548,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:30:31,902] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20551,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:30:31,902] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20551,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:30:31,902] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20552,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:30:31,902] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20552,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:30:31,904] jepsen worker 28 - jepsen.util 2548 :ok :write [9 46] INFO [2019-12-17 12:30:31,904] jepsen worker 28 - jepsen.util 2548 :invoke :write [9 10] INFO [2019-12-17 12:30:31,909] jepsen worker 24 - jepsen.util 4614 :info :write [8 9] :RequestCanceledException INFO [2019-12-17 12:30:31,910] jepsen worker 24 - jepsen.util 4704 :invoke :cas [8 [45 21]] INFO [2019-12-17 12:30:31,912] jepsen worker 4 - jepsen.util 2794 :ok :read [1 2] INFO [2019-12-17 12:30:31,912] jepsen worker 4 - jepsen.util 2794 :invoke :read [1 nil] INFO [2019-12-17 12:30:31,921] jepsen worker 56 - jepsen.util 3656 :ok :read [18 36] INFO [2019-12-17 12:30:31,921] jepsen worker 56 - jepsen.util 3656 :invoke :read [18 nil] INFO [2019-12-17 12:30:31,924] jepsen worker 49 - jepsen.util 4189 :info :write [16 39] :RequestCanceledException INFO [2019-12-17 12:30:31,926] jepsen worker 49 - jepsen.util 4279 :invoke :read [16 nil] INFO [2019-12-17 12:30:31,935] jepsen worker 70 - jepsen.util 4120 :ok :read [23 16] INFO [2019-12-17 12:30:31,935] jepsen worker 70 - jepsen.util 4120 :invoke :cas [23 [38 26]] INFO [2019-12-17 12:30:31,937] jepsen worker 44 - jepsen.util 5174 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:30:31,937] jepsen worker 77 - jepsen.util 4577 :info :write [25 13] :RequestCanceledException INFO [2019-12-17 12:30:31,938] jepsen worker 44 - jepsen.util 5174 :invoke :read [14 nil] INFO [2019-12-17 12:30:31,938] jepsen worker 77 - jepsen.util 4667 :invoke :cas [25 [28 36]] INFO [2019-12-17 12:30:31,955] jepsen worker 34 - jepsen.util 4804 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:30:31,955] jepsen worker 20 - jepsen.util 3260 :info :write [6 32] :RequestCanceledException INFO [2019-12-17 12:30:31,956] jepsen worker 34 - jepsen.util 4804 :invoke :write [11 16] INFO [2019-12-17 12:30:31,956] jepsen worker 20 - jepsen.util 3350 :invoke :cas [6 [47 49]] INFO [2019-12-17 12:30:31,960] jepsen worker 28 - jepsen.util 2548 :ok :write [9 10] INFO [2019-12-17 12:30:31,961] jepsen worker 28 - jepsen.util 2548 :invoke :write [9 9] INFO [2019-12-17 12:30:31,991] jepsen worker 56 - jepsen.util 3656 :ok :read [18 36] INFO [2019-12-17 12:30:31,991] jepsen worker 56 - jepsen.util 3656 :invoke :cas [18 [5 35]] INFO [2019-12-17 12:30:32,004] jepsen worker 28 - jepsen.util 2548 :ok :write [9 9] INFO [2019-12-17 12:30:32,005] jepsen worker 28 - jepsen.util 2548 :invoke :write [9 16] INFO [2019-12-17 12:30:32,005] jepsen worker 51 - jepsen.util 3201 :ok :read [17 33] INFO [2019-12-17 12:30:32,005] jepsen worker 51 - jepsen.util 3201 :invoke :cas [17 [25 23]] INFO [2019-12-17 12:30:32,050] jepsen worker 56 - jepsen.util 3656 :fail :cas [18 [5 35]] :ValueNotSwapFrom INFO [2019-12-17 12:30:32,050] jepsen worker 56 - jepsen.util 3656 :invoke :read [18 nil] INFO [2019-12-17 12:30:32,051] jepsen worker 28 - jepsen.util 2548 :ok :write [9 16] INFO [2019-12-17 12:30:32,051] jepsen worker 28 - jepsen.util 2548 :invoke :cas [9 [17 0]] INFO [2019-12-17 12:30:32,066] jepsen worker 28 - jepsen.util 2548 :fail :cas [9 [17 0]] :ValueNotSwapFrom INFO [2019-12-17 12:30:32,066] jepsen worker 28 - jepsen.util 2548 :invoke :read [9 nil] INFO [2019-12-17 12:30:32,072] jepsen worker 51 - jepsen.util 3201 :fail :cas [17 [25 23]] :ValueNotSwapFrom INFO [2019-12-17 12:30:32,072] jepsen worker 51 - jepsen.util 3201 :invoke :read [17 nil] INFO [2019-12-17 12:30:32,079] jepsen worker 28 - jepsen.util 2548 :ok :read [9 16] INFO [2019-12-17 12:30:32,079] jepsen worker 28 - jepsen.util 2548 :invoke :cas [9 [13 20]] INFO [2019-12-17 12:30:32,093] jepsen worker 28 - jepsen.util 2548 :fail :cas [9 [13 20]] :ValueNotSwapFrom INFO [2019-12-17 12:30:32,093] jepsen worker 28 - jepsen.util 2548 :invoke :cas [9 [34 31]] INFO [2019-12-17 12:30:32,094] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20567,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:30:32,094] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20567,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:30:32,107] jepsen worker 70 - jepsen.util 4120 :fail :cas [23 [38 26]] :ValueNotSwapFrom INFO [2019-12-17 12:30:32,107] jepsen worker 28 - jepsen.util 2548 :fail :cas [9 [34 31]] :ValueNotSwapFrom INFO [2019-12-17 12:30:32,107] jepsen worker 70 - jepsen.util 4120 :invoke :read [23 nil] INFO [2019-12-17 12:30:32,108] jepsen worker 28 - jepsen.util 2548 :invoke :write [9 30] INFO [2019-12-17 12:30:32,144] jepsen worker 38 - jepsen.util 3548 :ok :read [12 29] INFO [2019-12-17 12:30:32,144] jepsen worker 38 - jepsen.util 3548 :invoke :cas [12 [23 47]] INFO [2019-12-17 12:30:32,146] jepsen worker 28 - jepsen.util 2548 :ok :write [9 30] INFO [2019-12-17 12:30:32,147] jepsen worker 28 - jepsen.util 2548 :invoke :write [9 14] INFO [2019-12-17 12:30:32,173] jepsen worker 53 - jepsen.util 3653 :fail :cas [17 [26 14]] :ValueNotSwapFrom INFO [2019-12-17 12:30:32,173] jepsen worker 53 - jepsen.util 3653 :invoke :cas [17 [2 16]] INFO [2019-12-17 12:30:32,190] jepsen worker 28 - jepsen.util 2548 :ok :write [9 14] INFO [2019-12-17 12:30:32,190] jepsen worker 28 - jepsen.util 2548 :invoke :cas [9 [24 38]] INFO [2019-12-17 12:30:32,204] jepsen worker 36 - jepsen.util 3726 :ok :read [12 29] INFO [2019-12-17 12:30:32,204] jepsen worker 36 - jepsen.util 3726 :invoke :write [12 12] INFO [2019-12-17 12:30:32,212] jepsen worker 28 - jepsen.util 2548 :fail :cas [9 [24 38]] :ValueNotSwapFrom INFO [2019-12-17 12:30:32,212] jepsen worker 28 - jepsen.util 2548 :invoke :read [9 nil] INFO [2019-12-17 12:30:32,222] jepsen worker 28 - jepsen.util 2548 :ok :read [9 14] INFO [2019-12-17 12:30:32,222] jepsen worker 28 - jepsen.util 2548 :invoke :write [9 38] INFO [2019-12-17 12:30:32,232] jepsen worker 10 - jepsen.util 3430 :ok :write [3 23] INFO [2019-12-17 12:30:32,233] jepsen worker 10 - jepsen.util 3430 :invoke :write [3 27] INFO [2019-12-17 12:30:32,236] jepsen worker 18 - jepsen.util 3618 :ok :write [6 28] INFO [2019-12-17 12:30:32,236] jepsen worker 18 - jepsen.util 3618 :invoke :write [6 35] INFO [2019-12-17 12:30:32,238] jepsen worker 23 - jepsen.util 4613 :info :cas [7 [12 2]] :RequestCanceledException INFO [2019-12-17 12:30:32,239] jepsen worker 23 - jepsen.util 4703 :invoke :write [7 37] INFO [2019-12-17 12:30:32,239] jepsen worker 47 - jepsen.util 4097 :ok :write [15 14] INFO [2019-12-17 12:30:32,239] jepsen worker 47 - jepsen.util 4097 :invoke :write [15 25] INFO [2019-12-17 12:30:32,271] jepsen worker 32 - jepsen.util 4082 :fail :cas [10 [44 39]] :ValueNotSwapFrom INFO [2019-12-17 12:30:32,271] jepsen worker 32 - jepsen.util 4082 :invoke :cas [10 [25 18]] INFO [2019-12-17 12:30:32,285] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20586,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:30:32,285] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20593,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:30:32,285] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20586,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:30:32,285] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20593,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:30:32,285] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20588,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:30:32,285] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20597,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:30:32,285] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20588,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:30:32,285] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20597,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:30:32,285] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20598,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:30:32,285] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20598,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:30:32,286] jepsen worker 29 - jepsen.util 2909 :ok :read [9 14] INFO [2019-12-17 12:30:32,286] jepsen worker 29 - jepsen.util 2909 :invoke :write [9 18] INFO [2019-12-17 12:30:32,302] jepsen worker 20 - jepsen.util 3350 :fail :cas [6 [47 49]] :ValueNotSwapFrom INFO [2019-12-17 12:30:32,302] jepsen worker 20 - jepsen.util 3350 :invoke :cas [6 [44 32]] INFO [2019-12-17 12:30:32,309] jepsen worker 5 - jepsen.util 3155 :info :write [1 40] :RequestCanceledException INFO [2019-12-17 12:30:32,310] jepsen worker 5 - jepsen.util 3245 :invoke :write [1 36] INFO [2019-12-17 12:30:32,311] jepsen worker 38 - jepsen.util 3548 :fail :cas [12 [23 47]] :ValueNotSwapFrom INFO [2019-12-17 12:30:32,311] jepsen worker 38 - jepsen.util 3548 :invoke :write [12 45] INFO [2019-12-17 12:30:32,324] jepsen worker 59 - jepsen.util 4649 :info :write [19 7] :RequestCanceledException INFO [2019-12-17 12:30:32,325] jepsen worker 59 - jepsen.util 4739 :invoke :cas [19 [19 33]] INFO [2019-12-17 12:30:32,349] jepsen worker 28 - jepsen.util 2548 :ok :write [9 38] INFO [2019-12-17 12:30:32,349] jepsen worker 28 - jepsen.util 2548 :invoke :read [9 nil] INFO [2019-12-17 12:30:32,365] jepsen worker 28 - jepsen.util 2548 :ok :read [9 38] INFO [2019-12-17 12:30:32,365] jepsen worker 28 - jepsen.util 2548 :invoke :write [9 28] INFO [2019-12-17 12:30:32,409] jepsen worker 37 - jepsen.util 3367 :info :write [12 11] :RequestCanceledException INFO [2019-12-17 12:30:32,409] jepsen worker 55 - jepsen.util 4375 :info :write [18 17] :RequestCanceledException INFO [2019-12-17 12:30:32,410] jepsen worker 37 - jepsen.util 3457 :invoke :cas [12 [43 30]] INFO [2019-12-17 12:30:32,411] jepsen worker 55 - jepsen.util 4465 :invoke :write [18 42] INFO [2019-12-17 12:30:32,415] jepsen worker 28 - jepsen.util 2548 :ok :write [9 28] INFO [2019-12-17 12:30:32,416] jepsen worker 28 - jepsen.util 2548 :invoke :read [9 nil] INFO [2019-12-17 12:30:32,422] jepsen worker 88 - jepsen.util 3688 :info :write [29 36] :RequestCanceledException INFO [2019-12-17 12:30:32,423] jepsen worker 88 - jepsen.util 3778 :invoke :read [29 nil] INFO [2019-12-17 12:30:32,434] jepsen worker 28 - jepsen.util 2548 :ok :read [9 28] INFO [2019-12-17 12:30:32,435] jepsen worker 28 - jepsen.util 2548 :invoke :cas [9 [21 11]] INFO [2019-12-17 12:30:32,438] jepsen worker 71 - jepsen.util 3491 :info :write [23 21] :RequestCanceledException INFO [2019-12-17 12:30:32,438] jepsen worker 27 - jepsen.util 3357 :info :cas [9 [19 29]] :RequestCanceledException INFO [2019-12-17 12:30:32,439] jepsen worker 71 - jepsen.util 3581 :invoke :write [23 9] INFO [2019-12-17 12:30:32,439] jepsen worker 27 - jepsen.util 3447 :invoke :read [9 nil] INFO [2019-12-17 12:30:32,453] jepsen worker 13 - jepsen.util 3343 :info :write [4 39] :RequestCanceledException INFO [2019-12-17 12:30:32,453] jepsen worker 80 - jepsen.util 3500 :info :write [26 4] :RequestCanceledException INFO [2019-12-17 12:30:32,454] jepsen worker 13 - jepsen.util 3433 :invoke :read [4 nil] INFO [2019-12-17 12:30:32,454] jepsen worker 80 - jepsen.util 3590 :invoke :write [26 15] INFO [2019-12-17 12:30:32,456] jepsen worker 53 - jepsen.util 3653 :fail :cas [17 [2 16]] :ValueNotSwapFrom INFO [2019-12-17 12:30:32,456] jepsen worker 53 - jepsen.util 3653 :invoke :cas [17 [21 35]] INFO [2019-12-17 12:30:32,457] jepsen worker 28 - jepsen.util 2548 :fail :cas [9 [21 11]] :ValueNotSwapFrom INFO [2019-12-17 12:30:32,457] jepsen worker 28 - jepsen.util 2548 :invoke :read [9 nil] INFO [2019-12-17 12:30:32,461] jepsen worker 27 - jepsen.util 3447 :ok :read [9 28] INFO [2019-12-17 12:30:32,461] jepsen worker 27 - jepsen.util 3447 :invoke :write [9 25] INFO [2019-12-17 12:30:32,473] jepsen worker 70 - jepsen.util 4120 :ok :read [23 16] INFO [2019-12-17 12:30:32,473] jepsen worker 70 - jepsen.util 4120 :invoke :write [23 24] INFO [2019-12-17 12:30:32,474] jepsen worker 28 - jepsen.util 2548 :ok :read [9 28] INFO [2019-12-17 12:30:32,474] jepsen worker 28 - jepsen.util 2548 :invoke :write [9 4] INFO [2019-12-17 12:30:32,502] jepsen worker 27 - jepsen.util 3447 :ok :write [9 25] INFO [2019-12-17 12:30:32,502] jepsen worker 27 - jepsen.util 3447 :invoke :cas [9 [26 24]] INFO [2019-12-17 12:30:32,504] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20617,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:30:32,504] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20617,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:30:32,504] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20621,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:30:32,504] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20621,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:30:32,517] jepsen worker 27 - jepsen.util 3447 :fail :cas [9 [26 24]] :ValueNotSwapFrom INFO [2019-12-17 12:30:32,517] jepsen worker 27 - jepsen.util 3447 :invoke :read [9 nil] INFO [2019-12-17 12:30:32,524] jepsen worker 43 - jepsen.util 4363 :info :write [14 48] :RequestCanceledException INFO [2019-12-17 12:30:32,524] jepsen worker 79 - jepsen.util 3589 :info :cas [26 [46 6]] :RequestCanceledException INFO [2019-12-17 12:30:32,526] jepsen worker 79 - jepsen.util 3679 :invoke :write [26 8] INFO [2019-12-17 12:30:32,526] jepsen worker 43 - jepsen.util 4453 :invoke :write [14 46] INFO [2019-12-17 12:30:32,528] jepsen worker 56 - jepsen.util 3656 :ok :read [18 36] INFO [2019-12-17 12:30:32,529] jepsen worker 56 - jepsen.util 3656 :invoke :cas [18 [31 0]] INFO [2019-12-17 12:30:32,532] jepsen worker 27 - jepsen.util 3447 :ok :read [9 25] INFO [2019-12-17 12:30:32,532] jepsen worker 27 - jepsen.util 3447 :invoke :cas [9 [0 2]] INFO [2019-12-17 12:30:32,545] jepsen worker 51 - jepsen.util 3201 :ok :read [17 33] INFO [2019-12-17 12:30:32,545] jepsen worker 51 - jepsen.util 3201 :invoke :read [17 nil] INFO [2019-12-17 12:30:32,547] jepsen worker 27 - jepsen.util 3447 :fail :cas [9 [0 2]] :ValueNotSwapFrom INFO [2019-12-17 12:30:32,547] jepsen worker 27 - jepsen.util 3447 :invoke :write [9 12] INFO [2019-12-17 12:30:32,555] jepsen worker 54 - jepsen.util 3744 :info :write [18 7] :RequestCanceledException INFO [2019-12-17 12:30:32,557] jepsen worker 54 - jepsen.util 3834 :invoke :write [18 44] INFO [2019-12-17 12:30:32,581] jepsen worker 51 - jepsen.util 3201 :ok :read [17 33] INFO [2019-12-17 12:30:32,582] jepsen worker 51 - jepsen.util 3201 :invoke :write [17 44] INFO [2019-12-17 12:30:32,587] jepsen worker 27 - jepsen.util 3447 :ok :write [9 12] INFO [2019-12-17 12:30:32,587] jepsen worker 27 - jepsen.util 3447 :invoke :read [9 nil] INFO [2019-12-17 12:30:32,604] jepsen worker 27 - jepsen.util 3447 :ok :read [9 12] INFO [2019-12-17 12:30:32,604] jepsen worker 27 - jepsen.util 3447 :invoke :read [9 nil] INFO [2019-12-17 12:30:32,608] jepsen worker 84 - jepsen.util 4494 :info :cas [28 [4 38]] :RequestCanceledException INFO [2019-12-17 12:30:32,609] jepsen worker 84 - jepsen.util 4584 :invoke :read [28 nil] INFO [2019-12-17 12:30:32,615] jepsen worker 27 - jepsen.util 3447 :ok :read [9 12] INFO [2019-12-17 12:30:32,615] jepsen worker 27 - jepsen.util 3447 :invoke :cas [9 [36 30]] INFO [2019-12-17 12:30:32,624] jepsen worker 62 - jepsen.util 4562 :info :write [20 10] :RequestCanceledException INFO [2019-12-17 12:30:32,624] jepsen worker 39 - jepsen.util 4449 :info :write [13 5] :RequestCanceledException INFO [2019-12-17 12:30:32,624] jepsen worker 66 - jepsen.util 4566 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:30:32,624] jepsen worker 66 - jepsen.util 4566 :invoke :write [22 25] INFO [2019-12-17 12:30:32,625] jepsen worker 62 - jepsen.util 4652 :invoke :read [20 nil] INFO [2019-12-17 12:30:32,625] jepsen worker 39 - jepsen.util 4539 :invoke :cas [13 [29 32]] INFO [2019-12-17 12:30:32,631] jepsen worker 27 - jepsen.util 3447 :fail :cas [9 [36 30]] :ValueNotSwapFrom INFO [2019-12-17 12:30:32,631] jepsen worker 27 - jepsen.util 3447 :invoke :write [9 47] INFO [2019-12-17 12:30:32,655] jepsen worker 6 - jepsen.util 5046 :info :cas [2 [12 47]] :RequestCanceledException INFO [2019-12-17 12:30:32,655] jepsen worker 25 - jepsen.util 5065 :info :cas [8 [22 35]] :RequestCanceledException INFO [2019-12-17 12:30:32,656] jepsen worker 25 - jepsen.util 5155 :invoke :cas [8 [23 39]] INFO [2019-12-17 12:30:32,656] jepsen worker 6 - jepsen.util 5136 :invoke :cas [2 [20 37]] INFO [2019-12-17 12:30:32,668] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20652,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:30:32,668] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20652,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:30:32,668] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20655,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:30:32,668] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20655,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:30:32,668] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20656,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:30:32,668] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20656,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:30:32,680] jepsen worker 27 - jepsen.util 3447 :ok :write [9 47] INFO [2019-12-17 12:30:32,680] jepsen worker 27 - jepsen.util 3447 :invoke :cas [9 [39 3]] INFO [2019-12-17 12:30:32,691] jepsen worker 82 - jepsen.util 5662 :info :write [27 1] :RequestCanceledException INFO [2019-12-17 12:30:32,693] jepsen worker 82 - jepsen.util 5752 :invoke :read [27 nil] INFO [2019-12-17 12:30:32,698] jepsen worker 27 - jepsen.util 3447 :fail :cas [9 [39 3]] :ValueNotSwapFrom INFO [2019-12-17 12:30:32,699] jepsen worker 27 - jepsen.util 3447 :invoke :cas [9 [43 24]] INFO [2019-12-17 12:30:32,700] jepsen worker 52 - jepsen.util 3652 :info :write [17 21] :RequestCanceledException INFO [2019-12-17 12:30:32,701] jepsen worker 52 - jepsen.util 3742 :invoke :write [17 5] INFO [2019-12-17 12:30:32,715] jepsen worker 27 - jepsen.util 3447 :fail :cas [9 [43 24]] :ValueNotSwapFrom INFO [2019-12-17 12:30:32,715] jepsen worker 27 - jepsen.util 3447 :invoke :write [9 23] INFO [2019-12-17 12:30:32,722] jepsen worker 85 - jepsen.util 5215 :info :write [28 26] :RequestCanceledException INFO [2019-12-17 12:30:32,722] jepsen worker 7 - jepsen.util 4417 :info :write [2 17] :RequestCanceledException INFO [2019-12-17 12:30:32,722] jepsen worker 58 - jepsen.util 4738 :info :write [19 23] :RequestCanceledException INFO [2019-12-17 12:30:32,722] jepsen worker 12 - jepsen.util 3792 :info :write [4 40] :RequestCanceledException INFO [2019-12-17 12:30:32,723] jepsen worker 7 - jepsen.util 4507 :invoke :write [2 10] INFO [2019-12-17 12:30:32,724] jepsen worker 85 - jepsen.util 5305 :invoke :cas [28 [45 7]] INFO [2019-12-17 12:30:32,724] jepsen worker 58 - jepsen.util 4828 :invoke :write [19 9] INFO [2019-12-17 12:30:32,724] jepsen worker 12 - jepsen.util 3882 :invoke :cas [4 [28 31]] INFO [2019-12-17 12:30:32,738] jepsen worker 50 - jepsen.util 5180 :info :cas [16 [12 34]] :RequestCanceledException INFO [2019-12-17 12:30:32,739] jepsen worker 50 - jepsen.util 5270 :invoke :write [16 30] INFO [2019-12-17 12:30:32,741] jepsen worker 4 - jepsen.util 2794 :ok :read [1 2] INFO [2019-12-17 12:30:32,742] jepsen worker 4 - jepsen.util 2794 :invoke :cas [1 [11 15]] INFO [2019-12-17 12:30:32,758] jepsen worker 27 - jepsen.util 3447 :ok :write [9 23] INFO [2019-12-17 12:30:32,758] jepsen worker 27 - jepsen.util 3447 :invoke :write [9 31] INFO [2019-12-17 12:30:32,796] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:30:32,796] jepsen nemesis - couchbase.util Rebalance check count: 152 INFO [2019-12-17 12:30:32,796] jepsen nemesis - couchbase.util Rebalance stuck count: 133 INFO [2019-12-17 12:30:32,821] jepsen worker 26 - jepsen.util 5066 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:30:32,821] jepsen worker 67 - jepsen.util 5107 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:30:32,821] jepsen worker 26 - jepsen.util 5066 :invoke :cas [8 [11 12]] INFO [2019-12-17 12:30:32,822] jepsen worker 67 - jepsen.util 5107 :invoke :cas [22 [35 1]] INFO [2019-12-17 12:30:32,822] jepsen worker 40 - jepsen.util 5080 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:30:32,822] jepsen worker 40 - jepsen.util 5080 :invoke :cas [13 [29 32]] INFO [2019-12-17 12:30:32,823] jepsen worker 27 - jepsen.util 3447 :ok :write [9 31] INFO [2019-12-17 12:30:32,823] jepsen worker 27 - jepsen.util 3447 :invoke :cas [9 [45 38]] INFO [2019-12-17 12:30:32,838] jepsen worker 42 - jepsen.util 5622 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:30:32,838] jepsen worker 13 - jepsen.util 3433 :ok :read [4 1] INFO [2019-12-17 12:30:32,838] jepsen worker 13 - jepsen.util 3433 :invoke :write [4 30] INFO [2019-12-17 12:30:32,838] jepsen worker 42 - jepsen.util 5622 :invoke :write [14 43] INFO [2019-12-17 12:30:32,841] jepsen worker 10 - jepsen.util 3430 :ok :write [3 27] INFO [2019-12-17 12:30:32,841] jepsen worker 10 - jepsen.util 3430 :invoke :cas [3 [31 4]] INFO [2019-12-17 12:30:32,848] jepsen worker 47 - jepsen.util 4097 :ok :write [15 25] INFO [2019-12-17 12:30:32,848] jepsen worker 47 - jepsen.util 4097 :invoke :cas [15 [21 16]] INFO [2019-12-17 12:30:32,850] jepsen worker 78 - jepsen.util 3948 :info :write [26 44] :RequestCanceledException INFO [2019-12-17 12:30:32,851] jepsen worker 78 - jepsen.util 4038 :invoke :read [26 nil] INFO [2019-12-17 12:30:32,857] jepsen worker 27 - jepsen.util 3447 :fail :cas [9 [45 38]] :ValueNotSwapFrom INFO [2019-12-17 12:30:32,857] jepsen worker 27 - jepsen.util 3447 :invoke :write [9 49] INFO [2019-12-17 12:30:32,874] jepsen worker 53 - jepsen.util 3653 :fail :cas [17 [21 35]] :ValueNotSwapFrom INFO [2019-12-17 12:30:32,874] jepsen worker 53 - jepsen.util 3653 :invoke :read [17 nil] INFO [2019-12-17 12:30:32,887] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20680,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:30:32,887] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20680,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:30:32,888] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20683,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:30:32,888] jepsen worker 32 - jepsen.util 4082 :fail :cas [10 [25 18]] :ValueNotSwapFrom INFO [2019-12-17 12:30:32,888] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20683,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:30:32,888] jepsen worker 32 - jepsen.util 4082 :invoke :read [10 nil] INFO [2019-12-17 12:30:32,890] jepsen worker 75 - jepsen.util 4755 :info :write [25 46] :RequestCanceledException INFO [2019-12-17 12:30:32,891] jepsen worker 60 - jepsen.util 5010 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:30:32,891] jepsen worker 2 - jepsen.util 4502 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:30:32,891] jepsen worker 60 - jepsen.util 5010 :invoke :write [20 8] INFO [2019-12-17 12:30:32,891] jepsen worker 2 - jepsen.util 4502 :invoke :write [0 46] INFO [2019-12-17 12:30:32,892] jepsen worker 75 - jepsen.util 4845 :invoke :read [25 nil] INFO [2019-12-17 12:30:32,909] jepsen worker 63 - jepsen.util 3213 :fail :read [21 nil] :RequestTimeoutException INFO [2019-12-17 12:30:32,909] jepsen worker 14 - jepsen.util 3524 :info :write [4 28] :RequestCanceledException INFO [2019-12-17 12:30:32,909] jepsen worker 22 - jepsen.util 4882 :info :cas [7 [42 48]] :RequestCanceledException INFO [2019-12-17 12:30:32,909] jepsen worker 63 - jepsen.util 3213 :invoke :read [21 nil] INFO [2019-12-17 12:30:32,910] jepsen worker 22 - jepsen.util 4972 :invoke :cas [7 [28 29]] INFO [2019-12-17 12:30:32,914] jepsen worker 14 - jepsen.util 3614 :invoke :write [4 29] INFO [2019-12-17 12:30:32,921] jepsen worker 35 - jepsen.util 4535 :info :cas [11 [9 31]] :RequestCanceledException INFO [2019-12-17 12:30:32,921] jepsen worker 83 - jepsen.util 4853 :info :cas [27 [2 45]] :RequestCanceledException INFO [2019-12-17 12:30:32,923] jepsen worker 35 - jepsen.util 4625 :invoke :write [11 32] INFO [2019-12-17 12:30:32,923] jepsen worker 83 - jepsen.util 4943 :invoke :cas [27 [29 7]] INFO [2019-12-17 12:30:32,933] jepsen worker 31 - jepsen.util 3091 :info :write [10 2] :RequestCanceledException INFO [2019-12-17 12:30:32,933] jepsen worker 33 - jepsen.util 4983 :info :cas [11 [38 33]] :RequestCanceledException INFO [2019-12-17 12:30:32,933] jepsen worker 48 - jepsen.util 4188 :info :write [16 30] :RequestCanceledException INFO [2019-12-17 12:30:32,934] jepsen worker 31 - jepsen.util 3181 :invoke :cas [10 [14 2]] INFO [2019-12-17 12:30:32,934] jepsen worker 48 - jepsen.util 4278 :invoke :cas [16 [36 41]] INFO [2019-12-17 12:30:32,934] jepsen worker 33 - jepsen.util 5073 :invoke :cas [11 [22 30]] INFO [2019-12-17 12:30:32,940] jepsen worker 10 - jepsen.util 3430 :fail :cas [3 [31 4]] :ValueNotSwapFrom INFO [2019-12-17 12:30:32,940] jepsen worker 10 - jepsen.util 3430 :invoke :read [3 nil] INFO [2019-12-17 12:30:32,952] jepsen worker 8 - jepsen.util 4058 :info :write [2 18] :RequestCanceledException INFO [2019-12-17 12:30:32,952] jepsen worker 41 - jepsen.util 4901 :info :cas [13 [44 27]] :RequestCanceledException INFO [2019-12-17 12:30:32,954] jepsen worker 41 - jepsen.util 4991 :invoke :read [13 nil] INFO [2019-12-17 12:30:32,954] jepsen worker 8 - jepsen.util 4148 :invoke :read [2 nil] INFO [2019-12-17 12:30:32,965] jepsen worker 10 - jepsen.util 3430 :ok :read [3 27] INFO [2019-12-17 12:30:32,965] jepsen worker 10 - jepsen.util 3430 :invoke :write [3 6] INFO [2019-12-17 12:30:32,968] jepsen worker 31 - jepsen.util 3181 :fail :cas [10 [14 2]] :ValueNotSwapFrom INFO [2019-12-17 12:30:32,968] jepsen worker 31 - jepsen.util 3181 :invoke :write [10 36] INFO [2019-12-17 12:30:32,968] jepsen worker 29 - jepsen.util 2909 :ok :write [9 18] INFO [2019-12-17 12:30:32,968] jepsen worker 29 - jepsen.util 2909 :invoke :cas [9 [36 29]] INFO [2019-12-17 12:30:32,980] jepsen worker 88 - jepsen.util 3778 :ok :read [29 45] INFO [2019-12-17 12:30:32,980] jepsen worker 88 - jepsen.util 3778 :invoke :read [29 nil] INFO [2019-12-17 12:30:32,989] jepsen worker 57 - jepsen.util 4557 :info :write [19 14] :RequestCanceledException INFO [2019-12-17 12:30:32,989] jepsen worker 73 - jepsen.util 3493 :fail :read [24 nil] :RequestTimeoutException INFO [2019-12-17 12:30:32,989] jepsen worker 72 - jepsen.util 3582 :info :cas [24 [7 45]] :RequestCanceledException INFO [2019-12-17 12:30:32,989] jepsen worker 9 - jepsen.util 4149 :fail :read [3 nil] :RequestTimeoutException INFO [2019-12-17 12:30:32,990] jepsen worker 73 - jepsen.util 3493 :invoke :read [24 nil] INFO [2019-12-17 12:30:32,990] jepsen worker 9 - jepsen.util 4149 :invoke :write [3 34] INFO [2019-12-17 12:30:32,990] jepsen worker 56 - jepsen.util 3656 :fail :cas [18 [31 0]] :ValueNotSwapFrom INFO [2019-12-17 12:30:32,990] jepsen worker 56 - jepsen.util 3656 :invoke :write [18 1] INFO [2019-12-17 12:30:32,991] jepsen worker 57 - jepsen.util 4647 :invoke :read [19 nil] INFO [2019-12-17 12:30:32,992] jepsen worker 72 - jepsen.util 3672 :invoke :write [24 6] INFO [2019-12-17 12:30:33,000] jepsen worker 17 - jepsen.util 4517 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:30:33,000] jepsen worker 17 - jepsen.util 4517 :invoke :write [5 23] INFO [2019-12-17 12:30:33,009] jepsen worker 47 - jepsen.util 4097 :fail :cas [15 [21 16]] :ValueNotSwapFrom INFO [2019-12-17 12:30:33,009] jepsen worker 47 - jepsen.util 4097 :invoke :write [15 5] INFO [2019-12-17 12:30:33,024] jepsen worker 3 - jepsen.util 3783 :info :write [1 12] :RequestCanceledException INFO [2019-12-17 12:30:33,024] jepsen worker 0 - jepsen.util 5220 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:30:33,025] jepsen worker 0 - jepsen.util 5220 :invoke :read [0 nil] INFO [2019-12-17 12:30:33,025] jepsen worker 3 - jepsen.util 3873 :invoke :read [1 nil] INFO [2019-12-17 12:30:33,025] jepsen worker 12 - jepsen.util 3882 :fail :cas [4 [28 31]] :ValueNotSwapFrom INFO [2019-12-17 12:30:33,025] jepsen worker 12 - jepsen.util 3882 :invoke :read [4 nil] INFO [2019-12-17 12:30:33,026] jepsen worker 32 - jepsen.util 4082 :ok :read [10 31] INFO [2019-12-17 12:30:33,026] jepsen worker 32 - jepsen.util 4082 :invoke :write [10 29] INFO [2019-12-17 12:30:33,035] jepsen worker 4 - jepsen.util 2794 :fail :cas [1 [11 15]] :ValueNotSwapFrom INFO [2019-12-17 12:30:33,035] jepsen worker 4 - jepsen.util 2794 :invoke :read [1 nil] INFO [2019-12-17 12:30:33,046] jepsen worker 63 - jepsen.util 3213 :ok :read [21 34] INFO [2019-12-17 12:30:33,047] jepsen worker 63 - jepsen.util 3213 :invoke :read [21 nil] INFO [2019-12-17 12:30:33,056] jepsen worker 29 - jepsen.util 2909 :fail :cas [9 [36 29]] :ValueNotSwapFrom INFO [2019-12-17 12:30:33,056] jepsen worker 29 - jepsen.util 2909 :invoke :read [9 nil] INFO [2019-12-17 12:30:33,069] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20710,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:30:33,069] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20710,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:30:33,069] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20713,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:30:33,069] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20713,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:30:33,069] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20714,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:30:33,069] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20714,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:30:33,085] jepsen worker 12 - jepsen.util 3882 :ok :read [4 1] INFO [2019-12-17 12:30:33,085] jepsen worker 12 - jepsen.util 3882 :invoke :read [4 nil] INFO [2019-12-17 12:30:33,108] jepsen worker 4 - jepsen.util 2794 :ok :read [1 2] INFO [2019-12-17 12:30:33,108] jepsen worker 4 - jepsen.util 2794 :invoke :cas [1 [43 0]] INFO [2019-12-17 12:30:33,123] jepsen worker 12 - jepsen.util 3882 :ok :read [4 1] INFO [2019-12-17 12:30:33,123] jepsen worker 12 - jepsen.util 3882 :invoke :write [4 35] INFO [2019-12-17 12:30:33,145] jepsen worker 4 - jepsen.util 2794 :fail :cas [1 [43 0]] :ValueNotSwapFrom INFO [2019-12-17 12:30:33,145] jepsen worker 4 - jepsen.util 2794 :invoke :write [1 8] INFO [2019-12-17 12:30:33,190] jepsen worker 88 - jepsen.util 3778 :ok :read [29 45] INFO [2019-12-17 12:30:33,190] jepsen worker 88 - jepsen.util 3778 :invoke :read [29 nil] INFO [2019-12-17 12:30:33,224] jepsen worker 19 - jepsen.util 3709 :info :write [6 8] :RequestCanceledException INFO [2019-12-17 12:30:33,225] jepsen worker 19 - jepsen.util 3799 :invoke :write [6 5] INFO [2019-12-17 12:30:33,236] jepsen worker 74 - jepsen.util 3944 :info :cas [24 [8 10]] :RequestCanceledException INFO [2019-12-17 12:30:33,237] jepsen worker 74 - jepsen.util 4034 :invoke :write [24 49] INFO [2019-12-17 12:30:33,260] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20733,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:30:33,260] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20739,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:30:33,260] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20733,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:30:33,260] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20739,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:30:33,260] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20734,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:30:33,260] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20742,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:30:33,260] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20734,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:30:33,261] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20742,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:30:33,265] jepsen worker 37 - jepsen.util 3457 :fail :cas [12 [43 30]] :ValueNotSwapFrom INFO [2019-12-17 12:30:33,265] jepsen worker 37 - jepsen.util 3457 :invoke :cas [12 [49 8]] INFO [2019-12-17 12:30:33,309] jepsen worker 15 - jepsen.util 4335 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:30:33,309] jepsen worker 15 - jepsen.util 4335 :invoke :read [5 nil] INFO [2019-12-17 12:30:33,324] jepsen worker 64 - jepsen.util 2854 :info :cas [21 [11 36]] :RequestCanceledException INFO [2019-12-17 12:30:33,324] jepsen worker 87 - jepsen.util 3327 :info :write [29 37] :RequestCanceledException INFO [2019-12-17 12:30:33,324] jepsen worker 11 - jepsen.util 3791 :info :cas [3 [46 6]] :RequestCanceledException INFO [2019-12-17 12:30:33,324] jepsen worker 64 - jepsen.util 2944 :invoke :read [21 nil] INFO [2019-12-17 12:30:33,325] jepsen worker 11 - jepsen.util 3881 :invoke :cas [3 [34 16]] INFO [2019-12-17 12:30:33,325] jepsen worker 87 - jepsen.util 3417 :invoke :write [29 29] INFO [2019-12-17 12:30:33,338] jepsen worker 69 - jepsen.util 3489 :info :write [23 35] :RequestCanceledException INFO [2019-12-17 12:30:33,338] jepsen worker 45 - jepsen.util 3645 :info :write [15 49] :RequestCanceledException INFO [2019-12-17 12:30:33,339] jepsen worker 11 - jepsen.util 3881 :fail :cas [3 [34 16]] :ValueNotSwapFrom INFO [2019-12-17 12:30:33,339] jepsen worker 11 - jepsen.util 3881 :invoke :cas [3 [17 25]] INFO [2019-12-17 12:30:33,339] jepsen worker 69 - jepsen.util 3579 :invoke :read [23 nil] INFO [2019-12-17 12:30:33,340] jepsen worker 45 - jepsen.util 3735 :invoke :read [15 nil] INFO [2019-12-17 12:30:33,355] jepsen worker 11 - jepsen.util 3881 :fail :cas [3 [17 25]] :ValueNotSwapFrom INFO [2019-12-17 12:30:33,355] jepsen worker 11 - jepsen.util 3881 :invoke :write [3 3] INFO [2019-12-17 12:30:33,356] jepsen worker 76 - jepsen.util 4846 :info :cas [25 [31 13]] :RequestCanceledException INFO [2019-12-17 12:30:33,357] jepsen worker 76 - jepsen.util 4936 :invoke :write [25 16] INFO [2019-12-17 12:30:33,357] jepsen worker 53 - jepsen.util 3653 :ok :read [17 33] INFO [2019-12-17 12:30:33,358] jepsen worker 53 - jepsen.util 3653 :invoke :write [17 44] INFO [2019-12-17 12:30:33,360] jepsen worker 37 - jepsen.util 3457 :fail :cas [12 [49 8]] :ValueNotSwapFrom INFO [2019-12-17 12:30:33,360] jepsen worker 37 - jepsen.util 3457 :invoke :cas [12 [20 0]] INFO [2019-12-17 12:30:33,378] jepsen worker 3 - jepsen.util 3873 :ok :read [1 2] INFO [2019-12-17 12:30:33,378] jepsen worker 3 - jepsen.util 3873 :invoke :read [1 nil] INFO [2019-12-17 12:30:33,391] jepsen worker 16 - jepsen.util 5326 :info :cas [5 [9 2]] :RequestCanceledException INFO [2019-12-17 12:30:33,395] jepsen worker 16 - jepsen.util 5416 :invoke :cas [5 [44 36]] INFO [2019-12-17 12:30:33,408] jepsen worker 65 - jepsen.util 3035 :info :write [21 20] :RequestCanceledException INFO [2019-12-17 12:30:33,408] jepsen worker 1 - jepsen.util 4321 :info :cas [0 [6 24]] :RequestCanceledException INFO [2019-12-17 12:30:33,409] jepsen worker 1 - jepsen.util 4411 :invoke :cas [0 [10 42]] INFO [2019-12-17 12:30:33,410] jepsen worker 65 - jepsen.util 3125 :invoke :cas [21 [19 0]] INFO [2019-12-17 12:30:33,415] jepsen worker 3 - jepsen.util 3873 :ok :read [1 2] INFO [2019-12-17 12:30:33,416] jepsen worker 3 - jepsen.util 3873 :invoke :cas [1 [5 16]] INFO [2019-12-17 12:30:33,424] jepsen worker 46 - jepsen.util 3826 :fail :read [15 nil] :RequestTimeoutException INFO [2019-12-17 12:30:33,424] jepsen worker 46 - jepsen.util 3826 :invoke :cas [15 [18 32]] INFO [2019-12-17 12:30:33,430] jepsen worker 37 - jepsen.util 3457 :fail :cas [12 [20 0]] :ValueNotSwapFrom INFO [2019-12-17 12:30:33,430] jepsen worker 37 - jepsen.util 3457 :invoke :read [12 nil] INFO [2019-12-17 12:30:33,438] jepsen worker 46 - jepsen.util 3826 :fail :cas [15 [18 32]] :ValueNotSwapFrom INFO [2019-12-17 12:30:33,438] jepsen worker 46 - jepsen.util 3826 :invoke :write [15 34] INFO [2019-12-17 12:30:33,454] jepsen worker 61 - jepsen.util 5461 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:30:33,454] jepsen worker 61 - jepsen.util 5461 :invoke :cas [20 [35 40]] INFO [2019-12-17 12:30:33,468] jepsen worker 3 - jepsen.util 3873 :fail :cas [1 [5 16]] :ValueNotSwapFrom INFO [2019-12-17 12:30:33,468] jepsen worker 3 - jepsen.util 3873 :invoke :cas [1 [43 5]] INFO [2019-12-17 12:30:33,470] jepsen worker 31 - jepsen.util 3181 :ok :write [10 36] INFO [2019-12-17 12:30:33,470] jepsen worker 10 - jepsen.util 3430 :ok :write [3 6] INFO [2019-12-17 12:30:33,470] jepsen worker 31 - jepsen.util 3181 :invoke :cas [10 [37 38]] INFO [2019-12-17 12:30:33,470] jepsen worker 10 - jepsen.util 3430 :invoke :write [3 4] INFO [2019-12-17 12:30:33,470] jepsen worker 47 - jepsen.util 4097 :ok :write [15 5] INFO [2019-12-17 12:30:33,470] jepsen worker 47 - jepsen.util 4097 :invoke :cas [15 [49 39]] INFO [2019-12-17 12:30:33,477] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20767,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:30:33,477] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20767,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:30:33,477] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20769,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:30:33,477] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20769,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:30:33,492] jepsen worker 52 - jepsen.util 3742 :ok :write [17 5] INFO [2019-12-17 12:30:33,492] jepsen worker 52 - jepsen.util 3742 :invoke :read [17 nil] INFO [2019-12-17 12:30:33,514] jepsen worker 65 - jepsen.util 3125 :fail :cas [21 [19 0]] :ValueNotSwapFrom INFO [2019-12-17 12:30:33,514] jepsen worker 65 - jepsen.util 3125 :invoke :read [21 nil] INFO [2019-12-17 12:30:33,524] jepsen worker 81 - jepsen.util 4761 :info :write [27 35] :RequestCanceledException INFO [2019-12-17 12:30:33,524] jepsen worker 21 - jepsen.util 4881 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:30:33,524] jepsen worker 21 - jepsen.util 4881 :invoke :read [7 nil] INFO [2019-12-17 12:30:33,524] jepsen worker 52 - jepsen.util 3742 :ok :read [17 5] INFO [2019-12-17 12:30:33,524] jepsen worker 63 - jepsen.util 3213 :ok :read [21 34] INFO [2019-12-17 12:30:33,525] jepsen worker 63 - jepsen.util 3213 :invoke :cas [21 [0 47]] INFO [2019-12-17 12:30:33,525] jepsen worker 81 - jepsen.util 4851 :invoke :cas [27 [38 47]] INFO [2019-12-17 12:30:33,525] jepsen worker 52 - jepsen.util 3742 :invoke :read [17 nil] INFO [2019-12-17 12:30:33,528] jepsen worker 19 - jepsen.util 3799 :ok :write [6 5] INFO [2019-12-17 12:30:33,528] jepsen worker 19 - jepsen.util 3799 :invoke :read [6 nil] INFO [2019-12-17 12:30:33,538] jepsen worker 31 - jepsen.util 3181 :fail :cas [10 [37 38]] :ValueNotSwapFrom INFO [2019-12-17 12:30:33,538] jepsen worker 31 - jepsen.util 3181 :invoke :read [10 nil] INFO [2019-12-17 12:30:33,546] jepsen worker 29 - jepsen.util 2909 :ok :read [9 18] INFO [2019-12-17 12:30:33,547] jepsen worker 29 - jepsen.util 2909 :invoke :read [9 nil] INFO [2019-12-17 12:30:33,575] jepsen worker 65 - jepsen.util 3125 :ok :read [21 34] INFO [2019-12-17 12:30:33,575] jepsen worker 65 - jepsen.util 3125 :invoke :cas [21 [34 34]] INFO [2019-12-17 12:30:33,585] jepsen worker 64 - jepsen.util 2944 :ok :read [21 34] INFO [2019-12-17 12:30:33,585] jepsen worker 64 - jepsen.util 2944 :invoke :write [21 42] INFO [2019-12-17 12:30:33,588] jepsen worker 88 - jepsen.util 3778 :ok :read [29 45] INFO [2019-12-17 12:30:33,588] jepsen worker 88 - jepsen.util 3778 :invoke :read [29 nil] INFO [2019-12-17 12:30:33,602] jepsen worker 47 - jepsen.util 4097 :fail :cas [15 [49 39]] :ValueNotSwapFrom INFO [2019-12-17 12:30:33,602] jepsen worker 47 - jepsen.util 4097 :invoke :read [15 nil] INFO [2019-12-17 12:30:33,606] jepsen worker 31 - jepsen.util 3181 :ok :read [10 36] INFO [2019-12-17 12:30:33,606] jepsen worker 31 - jepsen.util 3181 :invoke :write [10 34] INFO [2019-12-17 12:30:33,624] jepsen worker 30 - jepsen.util 3900 :info :cas [10 [16 26]] :RequestCanceledException INFO [2019-12-17 12:30:33,627] jepsen worker 30 - jepsen.util 3990 :invoke :read [10 nil] INFO [2019-12-17 12:30:33,636] jepsen worker 52 - jepsen.util 3742 :ok :read [17 5] INFO [2019-12-17 12:30:33,636] jepsen worker 52 - jepsen.util 3742 :invoke :read [17 nil] INFO [2019-12-17 12:30:33,655] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20794,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:30:33,656] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20794,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:30:33,656] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20797,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:30:33,656] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20797,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:30:33,656] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20798,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:30:33,656] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20798,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:30:33,664] jepsen worker 45 - jepsen.util 3735 :ok :read [15 5] INFO [2019-12-17 12:30:33,664] jepsen worker 45 - jepsen.util 3735 :invoke :cas [15 [38 2]] INFO [2019-12-17 12:30:33,666] jepsen worker 37 - jepsen.util 3457 :ok :read [12 29] INFO [2019-12-17 12:30:33,667] jepsen worker 37 - jepsen.util 3457 :invoke :read [12 nil] INFO [2019-12-17 12:30:33,687] jepsen worker 63 - jepsen.util 3213 :fail :cas [21 [0 47]] :ValueNotSwapFrom INFO [2019-12-17 12:30:33,687] jepsen worker 63 - jepsen.util 3213 :invoke :write [21 1] INFO [2019-12-17 12:30:33,693] jepsen worker 69 - jepsen.util 3579 :ok :read [23 16] INFO [2019-12-17 12:30:33,693] jepsen worker 69 - jepsen.util 3579 :invoke :write [23 17] INFO [2019-12-17 12:30:33,717] jepsen worker 19 - jepsen.util 3799 :ok :read [6 5] INFO [2019-12-17 12:30:33,717] jepsen worker 19 - jepsen.util 3799 :invoke :cas [6 [17 40]] INFO [2019-12-17 12:30:33,721] jepsen worker 29 - jepsen.util 2909 :ok :read [9 18] INFO [2019-12-17 12:30:33,721] jepsen worker 29 - jepsen.util 2909 :invoke :write [9 6] INFO [2019-12-17 12:30:33,722] jepsen worker 74 - jepsen.util 4034 :ok :write [24 49] INFO [2019-12-17 12:30:33,722] jepsen worker 74 - jepsen.util 4034 :invoke :write [24 42] INFO [2019-12-17 12:30:33,737] jepsen worker 30 - jepsen.util 3990 :ok :read [10 36] INFO [2019-12-17 12:30:33,737] jepsen worker 30 - jepsen.util 3990 :invoke :cas [10 [4 37]] INFO [2019-12-17 12:30:33,747] jepsen worker 87 - jepsen.util 3417 :ok :write [29 29] INFO [2019-12-17 12:30:33,747] jepsen worker 87 - jepsen.util 3417 :invoke :write [29 35] INFO [2019-12-17 12:30:33,762] jepsen worker 37 - jepsen.util 3457 :ok :read [12 29] INFO [2019-12-17 12:30:33,762] jepsen worker 37 - jepsen.util 3457 :invoke :read [12 nil] INFO [2019-12-17 12:30:33,784] jepsen worker 45 - jepsen.util 3735 :fail :cas [15 [38 2]] :ValueNotSwapFrom INFO [2019-12-17 12:30:33,784] jepsen worker 45 - jepsen.util 3735 :invoke :cas [15 [44 6]] INFO [2019-12-17 12:30:33,790] jepsen worker 89 - jepsen.util 3419 :info :write [29 7] :RequestCanceledException INFO [2019-12-17 12:30:33,791] jepsen worker 89 - jepsen.util 3509 :invoke :cas [29 [27 24]] INFO [2019-12-17 12:30:33,805] jepsen worker 19 - jepsen.util 3799 :fail :cas [6 [17 40]] :ValueNotSwapFrom INFO [2019-12-17 12:30:33,805] jepsen worker 19 - jepsen.util 3799 :invoke :write [6 4] INFO [2019-12-17 12:30:33,809] jepsen worker 86 - jepsen.util 4946 :info :write [28 38] :RequestCanceledException INFO [2019-12-17 12:30:33,810] jepsen worker 86 - jepsen.util 5036 :invoke :cas [28 [49 21]] INFO [2019-12-17 12:30:33,816] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:30:33,816] jepsen nemesis - couchbase.util Rebalance check count: 153 INFO [2019-12-17 12:30:33,816] jepsen nemesis - couchbase.util Rebalance stuck count: 134 INFO [2019-12-17 12:30:33,821] jepsen worker 30 - jepsen.util 3990 :fail :cas [10 [4 37]] :ValueNotSwapFrom INFO [2019-12-17 12:30:33,821] jepsen worker 78 - jepsen.util 4038 :ok :read [26 46] INFO [2019-12-17 12:30:33,821] jepsen worker 30 - jepsen.util 3990 :invoke :cas [10 [46 19]] INFO [2019-12-17 12:30:33,821] jepsen worker 78 - jepsen.util 4038 :invoke :cas [26 [8 30]] INFO [2019-12-17 12:30:33,875] jepsen worker 53 - jepsen.util 3653 :ok :write [17 44] INFO [2019-12-17 12:30:33,875] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20817,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:30:33,875] jepsen worker 53 - jepsen.util 3653 :invoke :write [17 1] INFO [2019-12-17 12:30:33,875] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20817,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:30:33,875] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20821,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:30:33,875] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20821,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:30:33,880] jepsen worker 64 - jepsen.util 2944 :ok :write [21 42] INFO [2019-12-17 12:30:33,880] jepsen worker 64 - jepsen.util 2944 :invoke :cas [21 [10 6]] INFO [2019-12-17 12:30:33,908] jepsen worker 47 - jepsen.util 4097 :ok :read [15 5] INFO [2019-12-17 12:30:33,908] jepsen worker 47 - jepsen.util 4097 :invoke :cas [15 [46 45]] INFO [2019-12-17 12:30:33,913] jepsen worker 45 - jepsen.util 3735 :fail :cas [15 [44 6]] :ValueNotSwapFrom INFO [2019-12-17 12:30:33,913] jepsen worker 45 - jepsen.util 3735 :invoke :read [15 nil] INFO [2019-12-17 12:30:33,932] jepsen worker 74 - jepsen.util 4034 :ok :write [24 42] INFO [2019-12-17 12:30:33,932] jepsen worker 74 - jepsen.util 4034 :invoke :read [24 nil] INFO [2019-12-17 12:30:33,938] jepsen worker 68 - jepsen.util 4388 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:30:33,938] jepsen worker 68 - jepsen.util 4388 :invoke :write [22 48] INFO [2019-12-17 12:30:33,944] jepsen worker 51 - jepsen.util 3201 :ok :write [17 44] INFO [2019-12-17 12:30:33,945] jepsen worker 51 - jepsen.util 3201 :invoke :read [17 nil] INFO [2019-12-17 12:30:33,951] jepsen worker 74 - jepsen.util 4034 :ok :read [24 42] INFO [2019-12-17 12:30:33,951] jepsen worker 74 - jepsen.util 4034 :invoke :write [24 41] INFO [2019-12-17 12:30:33,956] jepsen worker 24 - jepsen.util 4704 :info :cas [8 [45 21]] :RequestCanceledException INFO [2019-12-17 12:30:33,957] jepsen worker 24 - jepsen.util 4794 :invoke :read [8 nil] INFO [2019-12-17 12:30:33,984] jepsen worker 63 - jepsen.util 3213 :ok :write [21 1] INFO [2019-12-17 12:30:33,984] jepsen worker 63 - jepsen.util 3213 :invoke :cas [21 [1 33]] INFO [2019-12-17 12:30:33,991] jepsen worker 77 - jepsen.util 4667 :info :cas [25 [28 36]] :RequestCanceledException INFO [2019-12-17 12:30:33,992] jepsen worker 77 - jepsen.util 4757 :invoke :cas [25 [8 6]] INFO [2019-12-17 12:30:34,000] jepsen worker 37 - jepsen.util 3457 :ok :read [12 29] INFO [2019-12-17 12:30:34,000] jepsen worker 37 - jepsen.util 3457 :invoke :read [12 nil] INFO [2019-12-17 12:30:34,007] jepsen worker 74 - jepsen.util 4034 :ok :write [24 41] INFO [2019-12-17 12:30:34,007] jepsen worker 74 - jepsen.util 4034 :invoke :read [24 nil] INFO [2019-12-17 12:30:34,018] jepsen worker 31 - jepsen.util 3181 :ok :write [10 34] INFO [2019-12-17 12:30:34,018] jepsen worker 19 - jepsen.util 3799 :ok :write [6 4] INFO [2019-12-17 12:30:34,020] jepsen worker 19 - jepsen.util 3799 :invoke :cas [6 [18 37]] INFO [2019-12-17 12:30:34,020] jepsen worker 31 - jepsen.util 3181 :invoke :cas [10 [19 30]] INFO [2019-12-17 12:30:34,022] jepsen worker 49 - jepsen.util 4279 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:30:34,022] jepsen worker 49 - jepsen.util 4279 :invoke :cas [16 [29 29]] INFO [2019-12-17 12:30:34,038] jepsen worker 44 - jepsen.util 5174 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:30:34,038] jepsen worker 44 - jepsen.util 5174 :invoke :write [14 39] INFO [2019-12-17 12:30:34,048] jepsen worker 74 - jepsen.util 4034 :ok :read [24 41] INFO [2019-12-17 12:30:34,048] jepsen worker 74 - jepsen.util 4034 :invoke :write [24 21] INFO [2019-12-17 12:30:34,050] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20833,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:30:34,050] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20833,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:30:34,050] jepsen worker 34 - jepsen.util 4804 :info :write [11 16] :RequestCanceledException INFO [2019-12-17 12:30:34,050] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20834,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:30:34,050] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20834,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:30:34,062] jepsen worker 45 - jepsen.util 3735 :ok :read [15 5] INFO [2019-12-17 12:30:34,062] jepsen worker 45 - jepsen.util 3735 :invoke :cas [15 [45 8]] INFO [2019-12-17 12:30:34,070] jepsen worker 34 - jepsen.util 4894 :invoke :cas [11 [35 15]] INFO [2019-12-17 12:30:34,092] jepsen worker 31 - jepsen.util 3181 :fail :cas [10 [19 30]] :ValueNotSwapFrom INFO [2019-12-17 12:30:34,093] jepsen worker 31 - jepsen.util 3181 :invoke :cas [10 [18 25]] INFO [2019-12-17 12:30:34,137] jepsen worker 19 - jepsen.util 3799 :fail :cas [6 [18 37]] :ValueNotSwapFrom INFO [2019-12-17 12:30:34,137] jepsen worker 19 - jepsen.util 3799 :invoke :read [6 nil] INFO [2019-12-17 12:30:34,167] jepsen worker 31 - jepsen.util 3181 :fail :cas [10 [18 25]] :ValueNotSwapFrom INFO [2019-12-17 12:30:34,167] jepsen worker 31 - jepsen.util 3181 :invoke :write [10 5] INFO [2019-12-17 12:30:34,188] jepsen worker 74 - jepsen.util 4034 :ok :write [24 21] INFO [2019-12-17 12:30:34,188] jepsen worker 74 - jepsen.util 4034 :invoke :read [24 nil] INFO [2019-12-17 12:30:34,195] jepsen worker 45 - jepsen.util 3735 :fail :cas [15 [45 8]] :ValueNotSwapFrom INFO [2019-12-17 12:30:34,196] jepsen worker 45 - jepsen.util 3735 :invoke :cas [15 [46 45]] INFO [2019-12-17 12:30:34,214] jepsen worker 63 - jepsen.util 3213 :ok :cas [21 [1 33]] INFO [2019-12-17 12:30:34,214] jepsen worker 63 - jepsen.util 3213 :invoke :read [21 nil] INFO [2019-12-17 12:30:34,239] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20852,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:30:34,240] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20852,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:30:34,240] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20854,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:30:34,240] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20861,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:30:34,240] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20854,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:30:34,240] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20861,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:30:34,240] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20865,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:30:34,240] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20865,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:30:34,251] jepsen worker 19 - jepsen.util 3799 :ok :read [6 4] INFO [2019-12-17 12:30:34,251] jepsen worker 19 - jepsen.util 3799 :invoke :write [6 22] INFO [2019-12-17 12:30:34,255] jepsen worker 37 - jepsen.util 3457 :ok :read [12 29] INFO [2019-12-17 12:30:34,256] jepsen worker 37 - jepsen.util 3457 :invoke :cas [12 [18 44]] INFO [2019-12-17 12:30:34,289] jepsen worker 74 - jepsen.util 4034 :ok :read [24 21] INFO [2019-12-17 12:30:34,289] jepsen worker 74 - jepsen.util 4034 :invoke :write [24 7] INFO [2019-12-17 12:30:34,299] jepsen worker 45 - jepsen.util 3735 :fail :cas [15 [46 45]] :ValueNotSwapFrom INFO [2019-12-17 12:30:34,299] jepsen worker 45 - jepsen.util 3735 :invoke :write [15 12] INFO [2019-12-17 12:30:34,308] jepsen worker 63 - jepsen.util 3213 :ok :read [21 33] INFO [2019-12-17 12:30:34,308] jepsen worker 63 - jepsen.util 3213 :invoke :read [21 nil] INFO [2019-12-17 12:30:34,309] jepsen worker 36 - jepsen.util 3726 :info :write [12 12] :RequestCanceledException INFO [2019-12-17 12:30:34,313] jepsen worker 36 - jepsen.util 3816 :invoke :read [12 nil] INFO [2019-12-17 12:30:34,324] jepsen worker 23 - jepsen.util 4703 :info :write [7 37] :RequestCanceledException INFO [2019-12-17 12:30:34,338] jepsen worker 18 - jepsen.util 3618 :info :write [6 35] :RequestCanceledException INFO [2019-12-17 12:30:34,338] jepsen worker 5 - jepsen.util 3245 :info :write [1 36] :RequestCanceledException INFO [2019-12-17 12:30:34,347] jepsen worker 3 - jepsen.util 3873 :fail :cas [1 [43 5]] :ValueNotSwapFrom INFO [2019-12-17 12:30:34,354] jepsen worker 3 - jepsen.util 3873 :invoke :read [1 nil] INFO [2019-12-17 12:30:34,355] jepsen worker 38 - jepsen.util 3548 :info :write [12 45] :RequestCanceledException INFO [2019-12-17 12:30:34,355] jepsen worker 59 - jepsen.util 4739 :info :cas [19 [19 33]] :RequestCanceledException INFO [2019-12-17 12:30:34,356] jepsen worker 5 - jepsen.util 3335 :invoke :read [1 nil] INFO [2019-12-17 12:30:34,356] jepsen worker 38 - jepsen.util 3638 :invoke :read [12 nil] INFO [2019-12-17 12:30:34,357] jepsen worker 59 - jepsen.util 4829 :invoke :read [19 nil] INFO [2019-12-17 12:30:34,357] jepsen worker 23 - jepsen.util 4793 :invoke :write [7 4] INFO [2019-12-17 12:30:34,357] jepsen worker 18 - jepsen.util 3708 :invoke :read [6 nil] INFO [2019-12-17 12:30:34,409] jepsen worker 20 - jepsen.util 3350 :info :cas [6 [44 32]] :RequestCanceledException INFO [2019-12-17 12:30:34,412] jepsen worker 63 - jepsen.util 3213 :ok :read [21 33] INFO [2019-12-17 12:30:34,432] jepsen worker 37 - jepsen.util 3457 :fail :cas [12 [18 44]] :ValueNotSwapFrom INFO [2019-12-17 12:30:34,436] jepsen worker 52 - jepsen.util 3742 :ok :read [17 44] INFO [2019-12-17 12:30:34,441] jepsen worker 63 - jepsen.util 3213 :invoke :read [21 nil] INFO [2019-12-17 12:30:34,441] jepsen worker 37 - jepsen.util 3457 :invoke :cas [12 [41 36]] INFO [2019-12-17 12:30:34,441] jepsen worker 52 - jepsen.util 3742 :invoke :write [17 21] INFO [2019-12-17 12:30:34,442] jepsen worker 20 - jepsen.util 3440 :invoke :cas [6 [20 2]] INFO [2019-12-17 12:30:34,459] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20879,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:30:34,460] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20879,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:30:34,460] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20884,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:30:34,460] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20884,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:30:34,471] jepsen worker 18 - jepsen.util 3708 :ok :read [6 4] INFO [2019-12-17 12:30:34,472] jepsen worker 18 - jepsen.util 3708 :invoke :write [6 22] INFO [2019-12-17 12:30:34,472] jepsen worker 36 - jepsen.util 3816 :ok :read [12 29] INFO [2019-12-17 12:30:34,472] jepsen worker 36 - jepsen.util 3816 :invoke :write [12 17] INFO [2019-12-17 12:30:34,488] jepsen worker 51 - jepsen.util 3201 :ok :read [17 44] INFO [2019-12-17 12:30:34,488] jepsen worker 51 - jepsen.util 3201 :invoke :cas [17 [34 26]] INFO [2019-12-17 12:30:34,491] jepsen worker 55 - jepsen.util 4465 :info :write [18 42] :RequestCanceledException INFO [2019-12-17 12:30:34,496] jepsen worker 55 - jepsen.util 4555 :invoke :read [18 nil] INFO [2019-12-17 12:30:34,523] jepsen worker 63 - jepsen.util 3213 :ok :read [21 33] INFO [2019-12-17 12:30:34,524] jepsen worker 80 - jepsen.util 3590 :info :write [26 15] :RequestCanceledException INFO [2019-12-17 12:30:34,524] jepsen worker 71 - jepsen.util 3581 :info :write [23 9] :RequestCanceledException INFO [2019-12-17 12:30:34,524] jepsen worker 28 - jepsen.util 2548 :info :write [9 4] :RequestCanceledException INFO [2019-12-17 12:30:34,538] jepsen worker 70 - jepsen.util 4120 :info :write [23 24] :RequestCanceledException INFO [2019-12-17 12:30:34,553] jepsen worker 63 - jepsen.util 3213 :invoke :cas [21 [35 25]] INFO [2019-12-17 12:30:34,554] jepsen worker 70 - jepsen.util 4210 :invoke :cas [23 [44 19]] INFO [2019-12-17 12:30:34,554] jepsen worker 71 - jepsen.util 3671 :invoke :write [23 38] INFO [2019-12-17 12:30:34,557] jepsen worker 80 - jepsen.util 3680 :invoke :read [26 nil] INFO [2019-12-17 12:30:34,557] jepsen worker 28 - jepsen.util 2638 :invoke :write [9 47] INFO [2019-12-17 12:30:34,560] jepsen worker 89 - jepsen.util 3509 :fail :cas [29 [27 24]] :ValueNotSwapFrom INFO [2019-12-17 12:30:34,591] jepsen worker 79 - jepsen.util 3679 :info :write [26 8] :RequestCanceledException INFO [2019-12-17 12:30:34,591] jepsen worker 78 - jepsen.util 4038 :fail :cas [26 [8 30]] :ValueNotSwapFrom INFO [2019-12-17 12:30:34,598] jepsen worker 89 - jepsen.util 3509 :invoke :read [29 nil] INFO [2019-12-17 12:30:34,598] jepsen worker 78 - jepsen.util 4038 :invoke :read [26 nil] INFO [2019-12-17 12:30:34,599] jepsen worker 79 - jepsen.util 3769 :invoke :cas [26 [28 40]] INFO [2019-12-17 12:30:34,606] jepsen worker 43 - jepsen.util 4453 :info :write [14 46] :RequestCanceledException INFO [2019-12-17 12:30:34,622] jepsen worker 43 - jepsen.util 4543 :invoke :write [14 46] INFO [2019-12-17 12:30:34,623] jepsen worker 63 - jepsen.util 3213 :fail :cas [21 [35 25]] :ValueNotSwapFrom INFO [2019-12-17 12:30:34,623] jepsen worker 63 - jepsen.util 3213 :invoke :write [21 19] INFO [2019-12-17 12:30:34,624] jepsen worker 54 - jepsen.util 3834 :info :write [18 44] :RequestCanceledException INFO [2019-12-17 12:30:34,631] jepsen worker 19 - jepsen.util 3799 :ok :write [6 22] INFO [2019-12-17 12:30:34,631] jepsen worker 19 - jepsen.util 3799 :invoke :write [6 16] INFO [2019-12-17 12:30:34,633] jepsen worker 74 - jepsen.util 4034 :ok :write [24 7] INFO [2019-12-17 12:30:34,633] jepsen worker 74 - jepsen.util 4034 :invoke :write [24 1] INFO [2019-12-17 12:30:34,633] jepsen worker 54 - jepsen.util 3924 :invoke :write [18 24] INFO [2019-12-17 12:30:34,640] jepsen worker 84 - jepsen.util 4584 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:30:34,640] jepsen worker 84 - jepsen.util 4584 :invoke :read [28 nil] INFO [2019-12-17 12:30:34,642] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20901,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:30:34,643] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20901,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:30:34,643] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20906,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:30:34,643] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20906,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:30:34,652] jepsen worker 62 - jepsen.util 4652 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:30:34,652] jepsen worker 62 - jepsen.util 4652 :invoke :read [20 nil] INFO [2019-12-17 12:30:34,656] jepsen worker 31 - jepsen.util 3181 :ok :write [10 5] INFO [2019-12-17 12:30:34,656] jepsen worker 31 - jepsen.util 3181 :invoke :read [10 nil] INFO [2019-12-17 12:30:34,690] jepsen worker 25 - jepsen.util 5155 :info :cas [8 [23 39]] :RequestCanceledException INFO [2019-12-17 12:30:34,699] jepsen worker 25 - jepsen.util 5245 :invoke :read [8 nil] INFO [2019-12-17 12:30:34,706] jepsen worker 88 - jepsen.util 3778 :ok :read [29 29] INFO [2019-12-17 12:30:34,706] jepsen worker 88 - jepsen.util 3778 :invoke :cas [29 [24 9]] INFO [2019-12-17 12:30:34,707] jepsen worker 6 - jepsen.util 5136 :info :cas [2 [20 37]] :RequestCanceledException INFO [2019-12-17 12:30:34,711] jepsen worker 6 - jepsen.util 5226 :invoke :write [2 38] INFO [2019-12-17 12:30:34,722] jepsen worker 66 - jepsen.util 4566 :info :write [22 25] :RequestCanceledException INFO [2019-12-17 12:30:34,722] jepsen worker 82 - jepsen.util 5752 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:30:34,722] jepsen worker 82 - jepsen.util 5752 :invoke :write [27 28] INFO [2019-12-17 12:30:34,722] jepsen worker 39 - jepsen.util 4539 :info :cas [13 [29 32]] :RequestCanceledException INFO [2019-12-17 12:30:34,728] jepsen worker 66 - jepsen.util 4656 :invoke :cas [22 [32 49]] INFO [2019-12-17 12:30:34,731] jepsen worker 39 - jepsen.util 4629 :invoke :write [13 6] INFO [2019-12-17 12:30:34,752] jepsen worker 7 - jepsen.util 4507 :info :write [2 10] :RequestCanceledException INFO [2019-12-17 12:30:34,757] jepsen worker 7 - jepsen.util 4597 :invoke :cas [2 [32 47]] INFO [2019-12-17 12:30:34,762] jepsen worker 31 - jepsen.util 3181 :ok :read [10 5] INFO [2019-12-17 12:30:34,762] jepsen worker 31 - jepsen.util 3181 :invoke :write [10 22] INFO [2019-12-17 12:30:34,784] jepsen worker 70 - jepsen.util 4210 :fail :cas [23 [44 19]] :ValueNotSwapFrom INFO [2019-12-17 12:30:34,784] jepsen worker 70 - jepsen.util 4210 :invoke :write [23 25] INFO [2019-12-17 12:30:34,789] jepsen worker 58 - jepsen.util 4828 :info :write [19 9] :RequestCanceledException INFO [2019-12-17 12:30:34,793] jepsen worker 58 - jepsen.util 4918 :invoke :write [19 49] INFO [2019-12-17 12:30:34,799] jepsen worker 80 - jepsen.util 3680 :ok :read [26 46] INFO [2019-12-17 12:30:34,799] jepsen worker 80 - jepsen.util 3680 :invoke :read [26 nil] INFO [2019-12-17 12:30:34,808] jepsen worker 45 - jepsen.util 3735 :ok :write [15 12] INFO [2019-12-17 12:30:34,808] jepsen worker 78 - jepsen.util 4038 :ok :read [26 46] INFO [2019-12-17 12:30:34,808] jepsen worker 45 - jepsen.util 3735 :invoke :cas [15 [46 27]] INFO [2019-12-17 12:30:34,809] jepsen worker 78 - jepsen.util 4038 :invoke :cas [26 [39 6]] INFO [2019-12-17 12:30:34,811] jepsen worker 63 - jepsen.util 3213 :ok :write [21 19] INFO [2019-12-17 12:30:34,811] jepsen worker 63 - jepsen.util 3213 :invoke :read [21 nil] INFO [2019-12-17 12:30:34,824] jepsen worker 50 - jepsen.util 5270 :info :write [16 30] :RequestCanceledException INFO [2019-12-17 12:30:34,824] jepsen worker 85 - jepsen.util 5305 :info :cas [28 [45 7]] :RequestCanceledException INFO [2019-12-17 12:30:34,827] jepsen worker 85 - jepsen.util 5395 :invoke :cas [28 [16 39]] INFO [2019-12-17 12:30:34,828] jepsen worker 50 - jepsen.util 5360 :invoke :write [16 26] INFO [2019-12-17 12:30:34,834] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:30:34,834] jepsen nemesis - couchbase.util Rebalance check count: 154 INFO [2019-12-17 12:30:34,834] jepsen nemesis - couchbase.util Rebalance stuck count: 135 INFO [2019-12-17 12:30:34,856] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20933,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:30:34,857] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20933,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:30:34,857] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20936,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:30:34,857] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20936,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:30:34,880] jepsen worker 28 - jepsen.util 2638 :ok :write [9 47] INFO [2019-12-17 12:30:34,880] jepsen worker 28 - jepsen.util 2638 :invoke :cas [9 [37 6]] INFO [2019-12-17 12:30:34,911] jepsen worker 38 - jepsen.util 3638 :ok :read [12 29] INFO [2019-12-17 12:30:34,912] jepsen worker 38 - jepsen.util 3638 :invoke :write [12 43] INFO [2019-12-17 12:30:34,920] jepsen worker 26 - jepsen.util 5066 :info :cas [8 [11 12]] :RequestCanceledException INFO [2019-12-17 12:30:34,921] jepsen worker 37 - jepsen.util 3457 :fail :cas [12 [41 36]] :ValueNotSwapFrom INFO [2019-12-17 12:30:34,921] jepsen worker 37 - jepsen.util 3457 :invoke :write [12 43] INFO [2019-12-17 12:30:34,921] jepsen worker 67 - jepsen.util 5107 :info :cas [22 [35 1]] :RequestCanceledException INFO [2019-12-17 12:30:34,921] jepsen worker 22 - jepsen.util 4972 :info :cas [7 [28 29]] :RequestCanceledException INFO [2019-12-17 12:30:34,928] jepsen worker 40 - jepsen.util 5080 :info :cas [13 [29 32]] :RequestCanceledException INFO [2019-12-17 12:30:34,928] jepsen worker 26 - jepsen.util 5156 :invoke :write [8 39] INFO [2019-12-17 12:30:34,928] jepsen worker 67 - jepsen.util 5197 :invoke :read [22 nil] INFO [2019-12-17 12:30:34,929] jepsen worker 22 - jepsen.util 5062 :invoke :cas [7 [47 0]] INFO [2019-12-17 12:30:34,930] jepsen worker 63 - jepsen.util 3213 :ok :read [21 19] INFO [2019-12-17 12:30:34,933] jepsen worker 63 - jepsen.util 3213 :invoke :cas [21 [16 0]] INFO [2019-12-17 12:30:34,937] jepsen worker 42 - jepsen.util 5622 :info :write [14 43] :RequestCanceledException INFO [2019-12-17 12:30:34,938] jepsen worker 27 - jepsen.util 3447 :info :write [9 49] :RequestCanceledException INFO [2019-12-17 12:30:34,938] jepsen worker 40 - jepsen.util 5170 :invoke :write [13 10] INFO [2019-12-17 12:30:34,946] jepsen worker 42 - jepsen.util 5712 :invoke :write [14 39] INFO [2019-12-17 12:30:34,948] jepsen worker 27 - jepsen.util 3537 :invoke :cas [9 [3 43]] INFO [2019-12-17 12:30:34,950] jepsen worker 13 - jepsen.util 3433 :info :write [4 30] :RequestCanceledException INFO [2019-12-17 12:30:34,950] jepsen worker 75 - jepsen.util 4845 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:30:34,951] jepsen worker 75 - jepsen.util 4845 :invoke :write [25 31] INFO [2019-12-17 12:30:34,951] jepsen worker 13 - jepsen.util 3523 :invoke :cas [4 [20 14]] INFO [2019-12-17 12:30:34,952] jepsen worker 48 - jepsen.util 4278 :info :cas [16 [36 41]] :RequestCanceledException INFO [2019-12-17 12:30:34,958] jepsen worker 48 - jepsen.util 4368 :invoke :write [16 22] INFO [2019-12-17 12:30:34,971] jepsen worker 28 - jepsen.util 2638 :fail :cas [9 [37 6]] :ValueNotSwapFrom INFO [2019-12-17 12:30:34,971] jepsen worker 28 - jepsen.util 2638 :invoke :read [9 nil] INFO [2019-12-17 12:30:34,986] jepsen worker 60 - jepsen.util 5010 :info :write [20 8] :RequestCanceledException INFO [2019-12-17 12:30:34,986] jepsen worker 2 - jepsen.util 4502 :info :write [0 46] :RequestCanceledException INFO [2019-12-17 12:30:34,986] jepsen worker 14 - jepsen.util 3614 :info :write [4 29] :RequestCanceledException INFO [2019-12-17 12:30:34,986] jepsen worker 41 - jepsen.util 4991 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:30:34,986] jepsen worker 41 - jepsen.util 4991 :invoke :cas [13 [25 17]] INFO [2019-12-17 12:30:34,991] jepsen worker 60 - jepsen.util 5100 :invoke :cas [20 [40 1]] INFO [2019-12-17 12:30:34,991] jepsen worker 2 - jepsen.util 4592 :invoke :cas [0 [49 31]] INFO [2019-12-17 12:30:34,996] jepsen worker 14 - jepsen.util 3704 :invoke :write [4 30] INFO [2019-12-17 12:30:35,009] jepsen worker 35 - jepsen.util 4625 :info :write [11 32] :RequestCanceledException INFO [2019-12-17 12:30:35,009] jepsen worker 8 - jepsen.util 4148 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:30:35,009] jepsen worker 8 - jepsen.util 4148 :invoke :write [2 32] INFO [2019-12-17 12:30:35,017] jepsen worker 35 - jepsen.util 4715 :invoke :write [11 18] INFO [2019-12-17 12:30:35,024] jepsen worker 83 - jepsen.util 4943 :info :cas [27 [29 7]] :RequestCanceledException INFO [2019-12-17 12:30:35,024] jepsen worker 57 - jepsen.util 4647 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:30:35,025] jepsen worker 57 - jepsen.util 4647 :invoke :read [19 nil] INFO [2019-12-17 12:30:35,025] jepsen worker 33 - jepsen.util 5073 :info :cas [11 [22 30]] :RequestCanceledException INFO [2019-12-17 12:30:35,025] jepsen worker 56 - jepsen.util 3656 :info :write [18 1] :RequestCanceledException INFO [2019-12-17 12:30:35,027] jepsen worker 56 - jepsen.util 3746 :invoke :cas [18 [16 31]] INFO [2019-12-17 12:30:35,032] jepsen worker 83 - jepsen.util 5033 :invoke :read [27 nil] INFO [2019-12-17 12:30:35,033] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20947,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:30:35,034] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20947,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:30:35,034] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20951,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:30:35,034] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20951,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:30:35,034] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20953,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:30:35,035] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20953,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:30:35,035] jepsen worker 74 - jepsen.util 4034 :ok :write [24 1] INFO [2019-12-17 12:30:35,036] jepsen worker 74 - jepsen.util 4034 :invoke :read [24 nil] INFO [2019-12-17 12:30:35,036] jepsen worker 72 - jepsen.util 3672 :info :write [24 6] :RequestCanceledException INFO [2019-12-17 12:30:35,037] jepsen worker 33 - jepsen.util 5163 :invoke :read [11 nil] INFO [2019-12-17 12:30:35,038] jepsen worker 51 - jepsen.util 3201 :fail :cas [17 [34 26]] :ValueNotSwapFrom INFO [2019-12-17 12:30:35,038] jepsen worker 51 - jepsen.util 3201 :invoke :cas [17 [6 8]] INFO [2019-12-17 12:30:35,045] jepsen worker 72 - jepsen.util 3762 :invoke :read [24 nil] INFO [2019-12-17 12:30:35,077] jepsen worker 79 - jepsen.util 3769 :fail :cas [26 [28 40]] :ValueNotSwapFrom INFO [2019-12-17 12:30:35,078] jepsen worker 79 - jepsen.util 3769 :invoke :read [26 nil] INFO [2019-12-17 12:30:35,091] jepsen worker 73 - jepsen.util 3493 :fail :read [24 nil] :RequestTimeoutException INFO [2019-12-17 12:30:35,091] jepsen worker 9 - jepsen.util 4149 :info :write [3 34] :RequestCanceledException INFO [2019-12-17 12:30:35,091] jepsen worker 73 - jepsen.util 3493 :invoke :read [24 nil] INFO [2019-12-17 12:30:35,092] jepsen worker 9 - jepsen.util 4239 :invoke :write [3 44] INFO [2019-12-17 12:30:35,108] jepsen worker 17 - jepsen.util 4517 :info :write [5 23] :RequestCanceledException INFO [2019-12-17 12:30:35,108] jepsen worker 32 - jepsen.util 4082 :info :write [10 29] :RequestCanceledException INFO [2019-12-17 12:30:35,113] jepsen worker 32 - jepsen.util 4172 :invoke :write [10 8] INFO [2019-12-17 12:30:35,114] jepsen worker 17 - jepsen.util 4607 :invoke :read [5 nil] INFO [2019-12-17 12:30:35,116] jepsen worker 28 - jepsen.util 2638 :ok :read [9 47] INFO [2019-12-17 12:30:35,116] jepsen worker 28 - jepsen.util 2638 :invoke :read [9 nil] INFO [2019-12-17 12:30:35,116] jepsen worker 5 - jepsen.util 3335 :ok :read [1 2] INFO [2019-12-17 12:30:35,116] jepsen worker 5 - jepsen.util 3335 :invoke :cas [1 [1 27]] INFO [2019-12-17 12:30:35,117] jepsen worker 74 - jepsen.util 4034 :ok :read [24 1] INFO [2019-12-17 12:30:35,117] jepsen worker 74 - jepsen.util 4034 :invoke :cas [24 [39 6]] INFO [2019-12-17 12:30:35,124] jepsen worker 0 - jepsen.util 5220 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:30:35,124] jepsen worker 72 - jepsen.util 3762 :ok :read [24 1] INFO [2019-12-17 12:30:35,124] jepsen worker 0 - jepsen.util 5220 :invoke :write [0 9] INFO [2019-12-17 12:30:35,124] jepsen worker 72 - jepsen.util 3762 :invoke :cas [24 [10 47]] INFO [2019-12-17 12:30:35,136] jepsen worker 63 - jepsen.util 3213 :fail :cas [21 [16 0]] :ValueNotSwapFrom INFO [2019-12-17 12:30:35,137] jepsen worker 63 - jepsen.util 3213 :invoke :cas [21 [34 41]] INFO [2019-12-17 12:30:35,148] jepsen worker 73 - jepsen.util 3493 :ok :read [24 1] INFO [2019-12-17 12:30:35,148] jepsen worker 73 - jepsen.util 3493 :invoke :write [24 37] INFO [2019-12-17 12:30:35,187] jepsen worker 72 - jepsen.util 3762 :fail :cas [24 [10 47]] :ValueNotSwapFrom INFO [2019-12-17 12:30:35,188] jepsen worker 72 - jepsen.util 3762 :invoke :write [24 34] INFO [2019-12-17 12:30:35,197] jepsen worker 80 - jepsen.util 3680 :ok :read [26 15] INFO [2019-12-17 12:30:35,197] jepsen worker 80 - jepsen.util 3680 :invoke :cas [26 [17 39]] INFO [2019-12-17 12:30:35,205] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20982,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:30:35,205] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20982,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:30:35,205] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20983,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:30:35,206] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20983,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:30:35,208] jepsen worker 12 - jepsen.util 3882 :info :write [4 35] :RequestCanceledException INFO [2019-12-17 12:30:35,208] jepsen worker 4 - jepsen.util 2794 :info :write [1 8] :RequestCanceledException INFO [2019-12-17 12:30:35,209] jepsen worker 12 - jepsen.util 3972 :invoke :cas [4 [6 12]] INFO [2019-12-17 12:30:35,213] jepsen worker 4 - jepsen.util 2884 :invoke :read [1 nil] INFO [2019-12-17 12:30:35,225] jepsen worker 63 - jepsen.util 3213 :fail :cas [21 [34 41]] :ValueNotSwapFrom INFO [2019-12-17 12:30:35,225] jepsen worker 63 - jepsen.util 3213 :invoke :write [21 32] INFO [2019-12-17 12:30:35,231] jepsen worker 28 - jepsen.util 2638 :ok :read [9 47] INFO [2019-12-17 12:30:35,232] jepsen worker 28 - jepsen.util 2638 :invoke :write [9 40] INFO [2019-12-17 12:30:35,235] jepsen worker 74 - jepsen.util 4034 :fail :cas [24 [39 6]] :ValueNotSwapFrom INFO [2019-12-17 12:30:35,235] jepsen worker 74 - jepsen.util 4034 :invoke :cas [24 [17 27]] INFO [2019-12-17 12:30:35,257] jepsen worker 78 - jepsen.util 4038 :fail :cas [26 [39 6]] :ValueNotSwapFrom INFO [2019-12-17 12:30:35,257] jepsen worker 78 - jepsen.util 4038 :invoke :write [26 36] INFO [2019-12-17 12:30:35,286] jepsen worker 55 - jepsen.util 4555 :ok :read [18 36] INFO [2019-12-17 12:30:35,286] jepsen worker 55 - jepsen.util 4555 :invoke :read [18 nil] INFO [2019-12-17 12:30:35,301] jepsen worker 28 - jepsen.util 2638 :ok :write [9 40] INFO [2019-12-17 12:30:35,301] jepsen worker 28 - jepsen.util 2638 :invoke :read [9 nil] INFO [2019-12-17 12:30:35,315] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20995,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:30:35,315] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20995,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:30:35,315] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20996,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:30:35,315] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":20996,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:30:35,339] jepsen worker 63 - jepsen.util 3213 :ok :write [21 32] INFO [2019-12-17 12:30:35,339] jepsen worker 63 - jepsen.util 3213 :invoke :cas [21 [31 8]] INFO [2019-12-17 12:30:35,342] jepsen worker 28 - jepsen.util 2638 :ok :read [9 40] INFO [2019-12-17 12:30:35,342] jepsen worker 28 - jepsen.util 2638 :invoke :cas [9 [35 21]] INFO [2019-12-17 12:30:35,371] jepsen worker 80 - jepsen.util 3680 :fail :cas [26 [17 39]] :ValueNotSwapFrom INFO [2019-12-17 12:30:35,371] jepsen worker 80 - jepsen.util 3680 :invoke :write [26 3] INFO [2019-12-17 12:30:35,391] jepsen worker 76 - jepsen.util 4936 :info :write [25 16] :RequestCanceledException INFO [2019-12-17 12:30:35,404] jepsen worker 28 - jepsen.util 2638 :fail :cas [9 [35 21]] :ValueNotSwapFrom INFO [2019-12-17 12:30:35,404] jepsen worker 28 - jepsen.util 2638 :invoke :cas [9 [7 8]] INFO [2019-12-17 12:30:35,406] jepsen worker 15 - jepsen.util 4335 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:30:35,406] jepsen worker 16 - jepsen.util 5416 :info :cas [5 [44 36]] :RequestCanceledException INFO [2019-12-17 12:30:35,406] jepsen worker 15 - jepsen.util 4335 :invoke :write [5 15] INFO [2019-12-17 12:30:35,406] jepsen worker 76 - jepsen.util 5026 :invoke :read [25 nil] INFO [2019-12-17 12:30:35,407] jepsen worker 63 - jepsen.util 3213 :fail :cas [21 [31 8]] :ValueNotSwapFrom INFO [2019-12-17 12:30:35,408] jepsen worker 63 - jepsen.util 3213 :invoke :write [21 46] INFO [2019-12-17 12:30:35,408] jepsen worker 16 - jepsen.util 5506 :invoke :cas [5 [14 32]] INFO [2019-12-17 12:30:35,424] jepsen worker 11 - jepsen.util 3881 :info :write [3 3] :RequestCanceledException INFO [2019-12-17 12:30:35,424] jepsen worker 1 - jepsen.util 4411 :info :cas [0 [10 42]] :RequestCanceledException INFO [2019-12-17 12:30:35,426] jepsen worker 11 - jepsen.util 3971 :invoke :cas [3 [2 8]] INFO [2019-12-17 12:30:35,427] jepsen worker 1 - jepsen.util 4501 :invoke :write [0 28] INFO [2019-12-17 12:30:35,436] jepsen worker 28 - jepsen.util 2638 :fail :cas [9 [7 8]] :ValueNotSwapFrom INFO [2019-12-17 12:30:35,436] jepsen worker 28 - jepsen.util 2638 :invoke :cas [9 [40 41]] INFO [2019-12-17 12:30:35,474] jepsen worker 63 - jepsen.util 3213 :ok :write [21 46] INFO [2019-12-17 12:30:35,475] jepsen worker 63 - jepsen.util 3213 :invoke :cas [21 [15 21]] INFO [2019-12-17 12:30:35,491] jepsen worker 10 - jepsen.util 3430 :info :write [3 4] :RequestCanceledException INFO [2019-12-17 12:30:35,492] jepsen worker 10 - jepsen.util 3520 :invoke :write [3 43] INFO [2019-12-17 12:30:35,510] jepsen worker 63 - jepsen.util 3213 :fail :cas [21 [15 21]] :ValueNotSwapFrom INFO [2019-12-17 12:30:35,511] jepsen worker 63 - jepsen.util 3213 :invoke :read [21 nil] INFO [2019-12-17 12:30:35,524] jepsen worker 46 - jepsen.util 3826 :info :write [15 34] :RequestCanceledException INFO [2019-12-17 12:30:35,529] jepsen worker 46 - jepsen.util 3916 :invoke :cas [15 [48 17]] INFO [2019-12-17 12:30:35,536] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21013,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:30:35,536] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21013,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:30:35,537] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21016,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:30:35,537] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21016,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:30:35,552] jepsen worker 28 - jepsen.util 2638 :ok :cas [9 [40 41]] INFO [2019-12-17 12:30:35,553] jepsen worker 28 - jepsen.util 2638 :invoke :write [9 3] INFO [2019-12-17 12:30:35,556] jepsen worker 61 - jepsen.util 5461 :info :cas [20 [35 40]] :RequestCanceledException INFO [2019-12-17 12:30:35,556] jepsen worker 81 - jepsen.util 4851 :info :cas [27 [38 47]] :RequestCanceledException INFO [2019-12-17 12:30:35,560] jepsen worker 61 - jepsen.util 5551 :invoke :write [20 22] INFO [2019-12-17 12:30:35,560] jepsen worker 81 - jepsen.util 4941 :invoke :read [27 nil] INFO [2019-12-17 12:30:35,561] jepsen worker 63 - jepsen.util 3213 :ok :read [21 46] INFO [2019-12-17 12:30:35,561] jepsen worker 63 - jepsen.util 3213 :invoke :read [21 nil] INFO [2019-12-17 12:30:35,568] jepsen worker 79 - jepsen.util 3769 :ok :read [26 15] INFO [2019-12-17 12:30:35,568] jepsen worker 79 - jepsen.util 3769 :invoke :cas [26 [1 31]] INFO [2019-12-17 12:30:35,589] jepsen worker 89 - jepsen.util 3509 :ok :read [29 29] INFO [2019-12-17 12:30:35,589] jepsen worker 89 - jepsen.util 3509 :invoke :cas [29 [31 24]] INFO [2019-12-17 12:30:35,619] jepsen worker 63 - jepsen.util 3213 :ok :read [21 46] INFO [2019-12-17 12:30:35,620] jepsen worker 63 - jepsen.util 3213 :invoke :write [21 36] INFO [2019-12-17 12:30:35,625] jepsen worker 21 - jepsen.util 4881 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:30:35,625] jepsen worker 21 - jepsen.util 4881 :invoke :read [7 nil] INFO [2019-12-17 12:30:35,644] jepsen worker 28 - jepsen.util 2638 :ok :write [9 3] INFO [2019-12-17 12:30:35,644] jepsen worker 28 - jepsen.util 2638 :invoke :read [9 nil] INFO [2019-12-17 12:30:35,672] jepsen worker 51 - jepsen.util 3201 :fail :cas [17 [6 8]] :ValueNotSwapFrom INFO [2019-12-17 12:30:35,672] jepsen worker 51 - jepsen.util 3201 :invoke :read [17 nil] INFO [2019-12-17 12:30:35,712] jepsen worker 12 - jepsen.util 3972 :fail :cas [4 [6 12]] :ValueNotSwapFrom INFO [2019-12-17 12:30:35,712] jepsen worker 12 - jepsen.util 3972 :invoke :write [4 33] INFO [2019-12-17 12:30:35,732] jepsen worker 28 - jepsen.util 2638 :ok :read [9 3] INFO [2019-12-17 12:30:35,733] jepsen worker 28 - jepsen.util 2638 :invoke :read [9 nil] INFO [2019-12-17 12:30:35,736] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21034,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:30:35,736] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21034,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:30:35,736] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21038,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:30:35,736] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21038,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:30:35,737] jepsen worker 65 - jepsen.util 3125 :info :cas [21 [34 34]] :RequestCanceledException INFO [2019-12-17 12:30:35,738] jepsen worker 63 - jepsen.util 3213 :ok :write [21 36] INFO [2019-12-17 12:30:35,738] jepsen worker 63 - jepsen.util 3213 :invoke :read [21 nil] INFO [2019-12-17 12:30:35,741] jepsen worker 65 - jepsen.util 3215 :invoke :write [21 13] INFO [2019-12-17 12:30:35,755] jepsen worker 69 - jepsen.util 3579 :info :write [23 17] :RequestCanceledException INFO [2019-12-17 12:30:35,762] jepsen worker 69 - jepsen.util 3669 :invoke :cas [23 [2 39]] INFO [2019-12-17 12:30:35,811] jepsen worker 29 - jepsen.util 2909 :info :write [9 6] :RequestCanceledException INFO [2019-12-17 12:30:35,820] jepsen worker 29 - jepsen.util 2999 :invoke :cas [9 [24 13]] INFO [2019-12-17 12:30:35,824] jepsen worker 19 - jepsen.util 3799 :ok :write [6 16] INFO [2019-12-17 12:30:35,824] jepsen worker 9 - jepsen.util 4239 :ok :write [3 44] INFO [2019-12-17 12:30:35,824] jepsen worker 19 - jepsen.util 3799 :invoke :cas [6 [2 3]] INFO [2019-12-17 12:30:35,824] jepsen worker 9 - jepsen.util 4239 :invoke :cas [3 [30 41]] INFO [2019-12-17 12:30:35,835] jepsen worker 73 - jepsen.util 3493 :ok :write [24 37] INFO [2019-12-17 12:30:35,835] jepsen worker 73 - jepsen.util 3493 :invoke :cas [24 [20 27]] INFO [2019-12-17 12:30:35,838] jepsen worker 87 - jepsen.util 3417 :info :write [29 35] :RequestCanceledException INFO [2019-12-17 12:30:35,844] jepsen worker 31 - jepsen.util 3181 :ok :write [10 22] INFO [2019-12-17 12:30:35,845] jepsen worker 31 - jepsen.util 3181 :invoke :cas [10 [30 29]] INFO [2019-12-17 12:30:35,849] jepsen worker 87 - jepsen.util 3507 :invoke :cas [29 [12 25]] INFO [2019-12-17 12:30:35,887] jepsen worker 63 - jepsen.util 3213 :ok :read [21 36] INFO [2019-12-17 12:30:35,888] jepsen worker 63 - jepsen.util 3213 :invoke :cas [21 [19 40]] INFO [2019-12-17 12:30:35,891] jepsen worker 88 - jepsen.util 3778 :fail :cas [29 [24 9]] :ValueNotSwapFrom INFO [2019-12-17 12:30:35,891] jepsen worker 88 - jepsen.util 3778 :invoke :read [29 nil] INFO [2019-12-17 12:30:35,908] jepsen worker 86 - jepsen.util 5036 :info :cas [28 [49 21]] :RequestCanceledException INFO [2019-12-17 12:30:35,908] jepsen worker 64 - jepsen.util 2944 :info :cas [21 [10 6]] :RequestCanceledException INFO [2019-12-17 12:30:35,916] jepsen worker 64 - jepsen.util 3034 :invoke :cas [21 [4 4]] INFO [2019-12-17 12:30:35,917] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:30:35,917] jepsen nemesis - couchbase.util Rebalance check count: 155 INFO [2019-12-17 12:30:35,917] jepsen nemesis - couchbase.util Rebalance stuck count: 136 INFO [2019-12-17 12:30:35,924] jepsen worker 30 - jepsen.util 3990 :info :cas [10 [46 19]] :RequestCanceledException INFO [2019-12-17 12:30:35,941] jepsen worker 30 - jepsen.util 4080 :invoke :write [10 1] INFO [2019-12-17 12:30:35,941] jepsen worker 86 - jepsen.util 5126 :invoke :read [28 nil] INFO [2019-12-17 12:30:35,953] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21054,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:30:35,954] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21054,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:30:35,954] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21058,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:30:35,956] jepsen worker 53 - jepsen.util 3653 :info :write [17 1] :RequestCanceledException INFO [2019-12-17 12:30:35,956] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21058,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:30:35,961] jepsen worker 53 - jepsen.util 3743 :invoke :write [17 29] INFO [2019-12-17 12:30:35,977] jepsen worker 28 - jepsen.util 2638 :ok :read [9 3] INFO [2019-12-17 12:30:35,977] jepsen worker 28 - jepsen.util 2638 :invoke :read [9 nil] INFO [2019-12-17 12:30:36,009] jepsen worker 47 - jepsen.util 4097 :info :cas [15 [46 45]] :RequestCanceledException INFO [2019-12-17 12:30:36,012] jepsen worker 47 - jepsen.util 4187 :invoke :cas [15 [39 26]] INFO [2019-12-17 12:30:36,025] jepsen worker 24 - jepsen.util 4794 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:30:36,026] jepsen worker 24 - jepsen.util 4794 :invoke :read [8 nil] INFO [2019-12-17 12:30:36,026] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21068,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:30:36,030] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21068,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:30:36,038] jepsen worker 68 - jepsen.util 4388 :info :write [22 48] :RequestCanceledException INFO [2019-12-17 12:30:36,042] jepsen worker 77 - jepsen.util 4757 :info :cas [25 [8 6]] :RequestCanceledException INFO [2019-12-17 12:30:36,057] jepsen worker 77 - jepsen.util 4847 :invoke :cas [25 [26 39]] INFO [2019-12-17 12:30:36,060] jepsen worker 5 - jepsen.util 3335 :fail :cas [1 [1 27]] :ValueNotSwapFrom INFO [2019-12-17 12:30:36,061] jepsen worker 5 - jepsen.util 3335 :invoke :cas [1 [4 21]] INFO [2019-12-17 12:30:36,067] jepsen worker 68 - jepsen.util 4478 :invoke :write [22 48] INFO [2019-12-17 12:30:36,075] jepsen worker 51 - jepsen.util 3201 :ok :read [17 44] INFO [2019-12-17 12:30:36,076] jepsen worker 51 - jepsen.util 3201 :invoke :cas [17 [41 23]] INFO [2019-12-17 12:30:36,125] jepsen worker 49 - jepsen.util 4279 :info :cas [16 [29 29]] :RequestCanceledException INFO [2019-12-17 12:30:36,134] jepsen worker 49 - jepsen.util 4369 :invoke :read [16 nil] INFO [2019-12-17 12:30:36,138] jepsen worker 44 - jepsen.util 5174 :info :write [14 39] :RequestCanceledException INFO [2019-12-17 12:30:36,156] jepsen worker 34 - jepsen.util 4894 :info :cas [11 [35 15]] :RequestCanceledException INFO [2019-12-17 12:30:36,159] jepsen worker 44 - jepsen.util 5264 :invoke :write [14 27] INFO [2019-12-17 12:30:36,173] jepsen worker 34 - jepsen.util 4984 :invoke :read [11 nil] INFO [2019-12-17 12:30:36,176] jepsen worker 9 - jepsen.util 4239 :fail :cas [3 [30 41]] :ValueNotSwapFrom INFO [2019-12-17 12:30:36,176] jepsen worker 46 - jepsen.util 3916 :fail :cas [15 [48 17]] :ValueNotSwapFrom INFO [2019-12-17 12:30:36,177] jepsen worker 46 - jepsen.util 3916 :invoke :cas [15 [20 1]] INFO [2019-12-17 12:30:36,177] jepsen worker 74 - jepsen.util 4034 :fail :cas [24 [17 27]] :ValueNotSwapFrom INFO [2019-12-17 12:30:36,177] jepsen worker 74 - jepsen.util 4034 :invoke :cas [24 [3 41]] INFO [2019-12-17 12:30:36,178] jepsen worker 9 - jepsen.util 4239 :invoke :read [3 nil] INFO [2019-12-17 12:30:36,195] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21083,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:30:36,195] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21083,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:30:36,196] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21085,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:30:36,196] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21085,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:30:36,206] jepsen worker 73 - jepsen.util 3493 :fail :cas [24 [20 27]] :ValueNotSwapFrom INFO [2019-12-17 12:30:36,206] jepsen worker 73 - jepsen.util 3493 :invoke :write [24 23] INFO [2019-12-17 12:30:36,211] jepsen worker 64 - jepsen.util 3034 :fail :cas [21 [4 4]] :ValueNotSwapFrom INFO [2019-12-17 12:30:36,211] jepsen worker 64 - jepsen.util 3034 :invoke :write [21 28] INFO [2019-12-17 12:30:36,222] jepsen worker 55 - jepsen.util 4555 :ok :read [18 36] INFO [2019-12-17 12:30:36,223] jepsen worker 55 - jepsen.util 4555 :invoke :cas [18 [11 12]] INFO [2019-12-17 12:30:36,243] jepsen worker 63 - jepsen.util 3213 :fail :cas [21 [19 40]] :ValueNotSwapFrom INFO [2019-12-17 12:30:36,244] jepsen worker 63 - jepsen.util 3213 :invoke :cas [21 [28 42]] INFO [2019-12-17 12:30:36,252] jepsen worker 4 - jepsen.util 2884 :ok :read [1 2] INFO [2019-12-17 12:30:36,252] jepsen worker 4 - jepsen.util 2884 :invoke :write [1 16] INFO [2019-12-17 12:30:36,277] jepsen worker 28 - jepsen.util 2638 :ok :read [9 3] INFO [2019-12-17 12:30:36,278] jepsen worker 28 - jepsen.util 2638 :invoke :read [9 nil] INFO [2019-12-17 12:30:36,279] jepsen worker 3 - jepsen.util 3873 :ok :read [1 2] INFO [2019-12-17 12:30:36,279] jepsen worker 3 - jepsen.util 3873 :invoke :write [1 29] INFO [2019-12-17 12:30:36,306] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21092,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:30:36,306] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21092,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:30:36,307] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21095,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:30:36,307] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21095,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:30:36,307] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21096,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:30:36,307] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21096,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:30:36,330] jepsen worker 69 - jepsen.util 3669 :fail :cas [23 [2 39]] :ValueNotSwapFrom INFO [2019-12-17 12:30:36,331] jepsen worker 69 - jepsen.util 3669 :invoke :cas [23 [32 44]] INFO [2019-12-17 12:30:36,354] jepsen worker 89 - jepsen.util 3509 :fail :cas [29 [31 24]] :ValueNotSwapFrom INFO [2019-12-17 12:30:36,354] jepsen worker 89 - jepsen.util 3509 :invoke :write [29 18] INFO [2019-12-17 12:30:36,373] jepsen worker 79 - jepsen.util 3769 :fail :cas [26 [1 31]] :ValueNotSwapFrom INFO [2019-12-17 12:30:36,374] jepsen worker 79 - jepsen.util 3769 :invoke :read [26 nil] INFO [2019-12-17 12:30:36,380] jepsen worker 87 - jepsen.util 3507 :fail :cas [29 [12 25]] :ValueNotSwapFrom INFO [2019-12-17 12:30:36,381] jepsen worker 87 - jepsen.util 3507 :invoke :write [29 5] INFO [2019-12-17 12:30:36,386] jepsen worker 46 - jepsen.util 3916 :fail :cas [15 [20 1]] :ValueNotSwapFrom INFO [2019-12-17 12:30:36,386] jepsen worker 46 - jepsen.util 3916 :invoke :cas [15 [13 25]] INFO [2019-12-17 12:30:36,391] jepsen worker 74 - jepsen.util 4034 :fail :cas [24 [3 41]] :ValueNotSwapFrom INFO [2019-12-17 12:30:36,392] jepsen worker 74 - jepsen.util 4034 :invoke :read [24 nil] INFO [2019-12-17 12:30:36,397] jepsen worker 10 - jepsen.util 3520 :ok :write [3 43] INFO [2019-12-17 12:30:36,397] jepsen worker 13 - jepsen.util 3523 :fail :cas [4 [20 14]] :ValueNotSwapFrom INFO [2019-12-17 12:30:36,397] jepsen worker 10 - jepsen.util 3520 :invoke :cas [3 [23 33]] INFO [2019-12-17 12:30:36,397] jepsen worker 13 - jepsen.util 3523 :invoke :read [4 nil] INFO [2019-12-17 12:30:36,400] jepsen worker 28 - jepsen.util 2638 :ok :read [9 3] INFO [2019-12-17 12:30:36,400] jepsen worker 28 - jepsen.util 2638 :invoke :write [9 26] INFO [2019-12-17 12:30:36,423] jepsen worker 59 - jepsen.util 4829 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:30:36,423] jepsen worker 59 - jepsen.util 4829 :invoke :write [19 0] INFO [2019-12-17 12:30:36,436] jepsen worker 64 - jepsen.util 3034 :ok :write [21 28] INFO [2019-12-17 12:30:36,437] jepsen worker 64 - jepsen.util 3034 :invoke :write [21 28] INFO [2019-12-17 12:30:36,438] jepsen worker 23 - jepsen.util 4793 :info :write [7 4] :RequestCanceledException INFO [2019-12-17 12:30:36,443] jepsen worker 23 - jepsen.util 4883 :invoke :cas [7 [38 5]] INFO [2019-12-17 12:30:36,449] jepsen worker 47 - jepsen.util 4187 :fail :cas [15 [39 26]] :ValueNotSwapFrom INFO [2019-12-17 12:30:36,449] jepsen worker 47 - jepsen.util 4187 :invoke :read [15 nil] INFO [2019-12-17 12:30:36,499] jepsen worker 10 - jepsen.util 3520 :fail :cas [3 [23 33]] :ValueNotSwapFrom INFO [2019-12-17 12:30:36,499] jepsen worker 10 - jepsen.util 3520 :invoke :write [3 22] INFO [2019-12-17 12:30:36,503] jepsen worker 28 - jepsen.util 2638 :ok :write [9 26] INFO [2019-12-17 12:30:36,504] jepsen worker 28 - jepsen.util 2638 :invoke :write [9 35] INFO [2019-12-17 12:30:36,520] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21114,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:30:36,520] jepsen worker 20 - jepsen.util 3440 :info :cas [6 [20 2]] :RequestCanceledException INFO [2019-12-17 12:30:36,520] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21114,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:30:36,520] jepsen worker 36 - jepsen.util 3816 :info :write [12 17] :RequestCanceledException INFO [2019-12-17 12:30:36,520] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21118,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:30:36,520] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21118,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:30:36,521] jepsen worker 20 - jepsen.util 3530 :invoke :write [6 10] INFO [2019-12-17 12:30:36,531] jepsen worker 36 - jepsen.util 3906 :invoke :write [12 16] INFO [2019-12-17 12:30:36,540] jepsen worker 52 - jepsen.util 3742 :info :write [17 21] :RequestCanceledException INFO [2019-12-17 12:30:36,554] jepsen worker 52 - jepsen.util 3832 :invoke :write [17 37] INFO [2019-12-17 12:30:36,554] jepsen worker 18 - jepsen.util 3708 :info :write [6 22] :RequestCanceledException INFO [2019-12-17 12:30:36,568] jepsen worker 18 - jepsen.util 3798 :invoke :cas [6 [47 42]] INFO [2019-12-17 12:30:36,576] jepsen worker 5 - jepsen.util 3335 :fail :cas [1 [4 21]] :ValueNotSwapFrom INFO [2019-12-17 12:30:36,577] jepsen worker 5 - jepsen.util 3335 :invoke :cas [1 [11 48]] INFO [2019-12-17 12:30:36,619] jepsen worker 56 - jepsen.util 3746 :fail :cas [18 [16 31]] :ValueNotSwapFrom INFO [2019-12-17 12:30:36,620] jepsen worker 56 - jepsen.util 3746 :invoke :cas [18 [10 7]] INFO [2019-12-17 12:30:36,625] jepsen worker 71 - jepsen.util 3671 :info :write [23 38] :RequestCanceledException INFO [2019-12-17 12:30:36,627] jepsen worker 71 - jepsen.util 3761 :invoke :write [23 13] INFO [2019-12-17 12:30:36,634] jepsen worker 51 - jepsen.util 3201 :fail :cas [17 [41 23]] :ValueNotSwapFrom INFO [2019-12-17 12:30:36,635] jepsen worker 51 - jepsen.util 3201 :invoke :write [17 34] INFO [2019-12-17 12:30:36,667] jepsen worker 47 - jepsen.util 4187 :ok :read [15 12] INFO [2019-12-17 12:30:36,668] jepsen worker 47 - jepsen.util 4187 :invoke :read [15 nil] INFO [2019-12-17 12:30:36,679] jepsen worker 64 - jepsen.util 3034 :ok :write [21 28] INFO [2019-12-17 12:30:36,679] jepsen worker 64 - jepsen.util 3034 :invoke :read [21 nil] INFO [2019-12-17 12:30:36,683] jepsen worker 55 - jepsen.util 4555 :fail :cas [18 [11 12]] :ValueNotSwapFrom INFO [2019-12-17 12:30:36,684] jepsen worker 55 - jepsen.util 4555 :invoke :write [18 48] INFO [2019-12-17 12:30:36,691] jepsen worker 43 - jepsen.util 4543 :info :write [14 46] :RequestCanceledException INFO [2019-12-17 12:30:36,696] jepsen worker 43 - jepsen.util 4633 :invoke :cas [14 [27 34]] INFO [2019-12-17 12:30:36,709] jepsen worker 54 - jepsen.util 3924 :info :write [18 24] :RequestCanceledException INFO [2019-12-17 12:30:36,716] jepsen worker 54 - jepsen.util 4014 :invoke :cas [18 [48 26]] INFO [2019-12-17 12:30:36,717] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21135,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:30:36,717] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21135,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:30:36,717] jepsen worker 25 - jepsen.util 5245 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:30:36,717] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21136,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:30:36,717] jepsen worker 25 - jepsen.util 5245 :invoke :read [8 nil] INFO [2019-12-17 12:30:36,717] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21136,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:30:36,717] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21137,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:30:36,718] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21137,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:30:36,726] jepsen worker 63 - jepsen.util 3213 :fail :cas [21 [28 42]] :CASMismatchException INFO [2019-12-17 12:30:36,726] jepsen worker 28 - jepsen.util 2638 :ok :write [9 35] INFO [2019-12-17 12:30:36,726] jepsen worker 28 - jepsen.util 2638 :invoke :write [9 36] INFO [2019-12-17 12:30:36,726] jepsen worker 63 - jepsen.util 3213 :invoke :cas [21 [47 38]] INFO [2019-12-17 12:30:36,732] jepsen worker 84 - jepsen.util 4584 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:30:36,732] jepsen worker 84 - jepsen.util 4584 :invoke :read [28 nil] INFO [2019-12-17 12:30:36,752] jepsen worker 79 - jepsen.util 3769 :ok :read [26 15] INFO [2019-12-17 12:30:36,752] jepsen worker 79 - jepsen.util 3769 :invoke :cas [26 [33 4]] INFO [2019-12-17 12:30:36,754] jepsen worker 62 - jepsen.util 4652 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:30:36,754] jepsen worker 66 - jepsen.util 4656 :info :cas [22 [32 49]] :RequestCanceledException INFO [2019-12-17 12:30:36,755] jepsen worker 62 - jepsen.util 4652 :invoke :write [20 42] INFO [2019-12-17 12:30:36,757] jepsen worker 66 - jepsen.util 4746 :invoke :cas [22 [10 49]] INFO [2019-12-17 12:30:36,777] jepsen worker 63 - jepsen.util 3213 :fail :cas [21 [47 38]] :ValueNotSwapFrom INFO [2019-12-17 12:30:36,777] jepsen worker 63 - jepsen.util 3213 :invoke :read [21 nil] INFO [2019-12-17 12:30:36,791] jepsen worker 7 - jepsen.util 4597 :info :cas [2 [32 47]] :RequestCanceledException INFO [2019-12-17 12:30:36,801] jepsen worker 7 - jepsen.util 4687 :invoke :cas [2 [28 38]] INFO [2019-12-17 12:30:36,809] jepsen worker 70 - jepsen.util 4210 :info :write [23 25] :RequestCanceledException INFO [2019-12-17 12:30:36,818] jepsen worker 28 - jepsen.util 2638 :ok :write [9 36] INFO [2019-12-17 12:30:36,819] jepsen worker 28 - jepsen.util 2638 :invoke :write [9 1] INFO [2019-12-17 12:30:36,825] jepsen worker 82 - jepsen.util 5752 :info :write [27 28] :RequestCanceledException INFO [2019-12-17 12:30:36,825] jepsen worker 45 - jepsen.util 3735 :info :cas [15 [46 27]] :RequestCanceledException INFO [2019-12-17 12:30:36,826] jepsen worker 39 - jepsen.util 4629 :info :write [13 6] :RequestCanceledException INFO [2019-12-17 12:30:36,829] jepsen worker 70 - jepsen.util 4300 :invoke :write [23 47] INFO [2019-12-17 12:30:36,835] jepsen worker 39 - jepsen.util 4719 :invoke :read [13 nil] INFO [2019-12-17 12:30:36,841] jepsen worker 82 - jepsen.util 5842 :invoke :read [27 nil] INFO [2019-12-17 12:30:36,841] jepsen worker 6 - jepsen.util 5226 :info :write [2 38] :RequestCanceledException INFO [2019-12-17 12:30:36,841] jepsen worker 45 - jepsen.util 3825 :invoke :write [15 4] INFO [2019-12-17 12:30:36,885] jepsen worker 6 - jepsen.util 5316 :invoke :cas [2 [30 26]] INFO [2019-12-17 12:30:36,914] jepsen worker 58 - jepsen.util 4918 :info :write [19 49] :RequestCanceledException INFO [2019-12-17 12:30:36,933] jepsen worker 85 - jepsen.util 5395 :info :cas [28 [16 39]] :RequestCanceledException INFO [2019-12-17 12:30:36,933] jepsen worker 32 - jepsen.util 4172 :ok :write [10 8] INFO [2019-12-17 12:30:36,933] jepsen worker 73 - jepsen.util 3493 :ok :write [24 23] INFO [2019-12-17 12:30:36,933] jepsen worker 32 - jepsen.util 4172 :invoke :read [10 nil] INFO [2019-12-17 12:30:36,933] jepsen worker 73 - jepsen.util 3493 :invoke :read [24 nil] INFO [2019-12-17 12:30:36,940] jepsen worker 50 - jepsen.util 5360 :info :write [16 26] :RequestCanceledException INFO [2019-12-17 12:30:36,949] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21159,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:30:36,951] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21159,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:30:36,951] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21160,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:30:36,951] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21160,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:30:36,953] jepsen worker 10 - jepsen.util 3520 :ok :write [3 22] INFO [2019-12-17 12:30:36,955] jepsen worker 10 - jepsen.util 3520 :invoke :read [3 nil] INFO [2019-12-17 12:30:36,956] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:30:36,960] jepsen nemesis - couchbase.util Rebalance check count: 156 INFO [2019-12-17 12:30:36,961] jepsen nemesis - couchbase.util Rebalance stuck count: 137 INFO [2019-12-17 12:30:36,962] jepsen worker 58 - jepsen.util 5008 :invoke :write [19 35] INFO [2019-12-17 12:30:36,967] jepsen worker 50 - jepsen.util 5450 :invoke :read [16 nil] INFO [2019-12-17 12:30:36,992] jepsen worker 22 - jepsen.util 5062 :info :cas [7 [47 0]] :RequestCanceledException INFO [2019-12-17 12:30:37,001] jepsen worker 85 - jepsen.util 5485 :invoke :cas [28 [7 3]] INFO [2019-12-17 12:30:37,023] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21167,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:30:37,024] jepsen worker 65 - jepsen.util 3215 :ok :write [21 13] INFO [2019-12-17 12:30:37,024] jepsen worker 38 - jepsen.util 3638 :info :write [12 43] :RequestCanceledException INFO [2019-12-17 12:30:37,024] jepsen worker 2 - jepsen.util 4592 :info :cas [0 [49 31]] :RequestCanceledException INFO [2019-12-17 12:30:37,025] jepsen worker 37 - jepsen.util 3457 :info :write [12 43] :RequestCanceledException INFO [2019-12-17 12:30:37,027] jepsen worker 65 - jepsen.util 3215 :invoke :cas [21 [41 27]] INFO [2019-12-17 12:30:37,027] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21167,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:30:37,156] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21172,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:30:37,160] jepsen worker 67 - jepsen.util 5197 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:30:37,160] jepsen worker 40 - jepsen.util 5170 :info :write [13 10] :RequestCanceledException INFO [2019-12-17 12:30:37,160] jepsen worker 57 - jepsen.util 4647 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:30:37,160] jepsen worker 42 - jepsen.util 5712 :info :write [14 39] :RequestCanceledException INFO [2019-12-17 12:30:37,160] jepsen worker 35 - jepsen.util 4715 :info :write [11 18] :RequestCanceledException INFO [2019-12-17 12:30:37,160] jepsen worker 26 - jepsen.util 5156 :info :write [8 39] :RequestCanceledException INFO [2019-12-17 12:30:37,160] jepsen worker 27 - jepsen.util 3537 :fail :cas [9 [3 43]] :CASMismatchException INFO [2019-12-17 12:30:37,160] jepsen worker 41 - jepsen.util 4991 :info :cas [13 [25 17]] :RequestCanceledException INFO [2019-12-17 12:30:37,160] jepsen worker 8 - jepsen.util 4148 :info :write [2 32] :RequestCanceledException INFO [2019-12-17 12:30:37,161] jepsen worker 48 - jepsen.util 4368 :info :write [16 22] :RequestCanceledException INFO [2019-12-17 12:30:37,161] jepsen worker 67 - jepsen.util 5197 :invoke :cas [22 [44 26]] INFO [2019-12-17 12:30:37,161] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21172,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:30:37,161] jepsen worker 57 - jepsen.util 4647 :invoke :cas [19 [23 10]] INFO [2019-12-17 12:30:37,162] jepsen worker 75 - jepsen.util 4845 :info :write [25 31] :RequestCanceledException INFO [2019-12-17 12:30:37,167] jepsen worker 60 - jepsen.util 5100 :info :cas [20 [40 1]] :RequestCanceledException INFO [2019-12-17 12:30:37,167] jepsen worker 83 - jepsen.util 5033 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:30:37,167] jepsen worker 27 - jepsen.util 3537 :invoke :read [9 nil] INFO [2019-12-17 12:30:37,168] jepsen worker 17 - jepsen.util 4607 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:30:37,168] jepsen worker 14 - jepsen.util 3704 :info :write [4 30] :RequestCanceledException INFO [2019-12-17 12:30:37,168] jepsen worker 33 - jepsen.util 5163 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:30:37,168] jepsen worker 22 - jepsen.util 5152 :invoke :read [7 nil] INFO [2019-12-17 12:30:37,174] jepsen worker 83 - jepsen.util 5033 :invoke :cas [27 [33 42]] INFO [2019-12-17 12:30:37,174] jepsen worker 17 - jepsen.util 4607 :invoke :write [5 27] INFO [2019-12-17 12:30:37,176] jepsen worker 33 - jepsen.util 5163 :invoke :write [11 46] INFO [2019-12-17 12:30:37,199] jepsen worker 37 - jepsen.util 3547 :invoke :read [12 nil] INFO [2019-12-17 12:30:37,204] jepsen worker 2 - jepsen.util 4682 :invoke :read [0 nil] INFO [2019-12-17 12:30:37,206] jepsen worker 41 - jepsen.util 5081 :invoke :write [13 36] INFO [2019-12-17 12:30:37,211] jepsen worker 14 - jepsen.util 3794 :invoke :write [4 47] INFO [2019-12-17 12:30:37,213] jepsen worker 26 - jepsen.util 5246 :invoke :cas [8 [11 43]] INFO [2019-12-17 12:30:37,213] jepsen worker 42 - jepsen.util 5802 :invoke :cas [14 [16 47]] INFO [2019-12-17 12:30:37,215] jepsen worker 75 - jepsen.util 4935 :invoke :write [25 46] INFO [2019-12-17 12:30:37,215] jepsen worker 60 - jepsen.util 5190 :invoke :cas [20 [35 25]] INFO [2019-12-17 12:30:37,217] jepsen worker 38 - jepsen.util 3728 :invoke :cas [12 [47 4]] INFO [2019-12-17 12:30:37,220] jepsen worker 40 - jepsen.util 5260 :invoke :cas [13 [44 14]] INFO [2019-12-17 12:30:37,221] jepsen worker 48 - jepsen.util 4458 :invoke :cas [16 [21 35]] INFO [2019-12-17 12:30:37,221] jepsen worker 8 - jepsen.util 4238 :invoke :write [2 6] INFO [2019-12-17 12:30:37,224] jepsen worker 0 - jepsen.util 5220 :info :write [0 9] :RequestCanceledException INFO [2019-12-17 12:30:37,230] jepsen worker 35 - jepsen.util 4805 :invoke :cas [11 [7 10]] INFO [2019-12-17 12:30:37,238] jepsen worker 20 - jepsen.util 3530 :ok :write [6 10] INFO [2019-12-17 12:30:37,239] jepsen worker 20 - jepsen.util 3530 :invoke :write [6 15] INFO [2019-12-17 12:30:37,259] jepsen worker 0 - jepsen.util 5310 :invoke :cas [0 [1 6]] INFO [2019-12-17 12:30:37,267] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21184,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:30:37,271] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21193,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:30:37,271] jepsen worker 63 - jepsen.util 3213 :ok :read [21 28] INFO [2019-12-17 12:30:37,272] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21184,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:30:37,273] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21193,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:30:37,273] jepsen worker 63 - jepsen.util 3213 :invoke :cas [21 [24 6]] INFO [2019-12-17 12:30:37,311] jepsen worker 72 - jepsen.util 3762 :info :write [24 34] :RequestCanceledException INFO [2019-12-17 12:30:37,320] jepsen worker 72 - jepsen.util 3852 :invoke :cas [24 [38 44]] INFO [2019-12-17 12:30:37,339] jepsen worker 78 - jepsen.util 4038 :info :write [26 36] :RequestCanceledException INFO [2019-12-17 12:30:37,367] jepsen worker 78 - jepsen.util 4128 :invoke :write [26 28] INFO [2019-12-17 12:30:37,369] jepsen worker 28 - jepsen.util 2638 :ok :write [9 1] INFO [2019-12-17 12:30:37,369] jepsen worker 28 - jepsen.util 2638 :invoke :write [9 41] INFO [2019-12-17 12:30:37,427] jepsen worker 16 - jepsen.util 5506 :info :cas [5 [14 32]] :RequestCanceledException INFO [2019-12-17 12:30:37,440] jepsen worker 80 - jepsen.util 3680 :info :write [26 3] :RequestCanceledException INFO [2019-12-17 12:30:37,442] jepsen worker 11 - jepsen.util 3971 :info :cas [3 [2 8]] :RequestCanceledException INFO [2019-12-17 12:30:37,456] jepsen worker 1 - jepsen.util 4501 :info :write [0 28] :RequestCanceledException INFO [2019-12-17 12:30:37,461] jepsen worker 16 - jepsen.util 5596 :invoke :cas [5 [27 11]] INFO [2019-12-17 12:30:37,470] jepsen worker 80 - jepsen.util 3770 :invoke :cas [26 [37 22]] INFO [2019-12-17 12:30:37,470] jepsen worker 11 - jepsen.util 4061 :invoke :write [3 19] INFO [2019-12-17 12:30:37,496] jepsen worker 1 - jepsen.util 4591 :invoke :cas [0 [29 48]] INFO [2019-12-17 12:30:37,509] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21216,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:30:37,509] jepsen worker 76 - jepsen.util 5026 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:30:37,510] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21216,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:30:37,510] jepsen worker 76 - jepsen.util 5026 :invoke :cas [25 [49 45]] INFO [2019-12-17 12:30:37,510] jepsen worker 15 - jepsen.util 4335 :info :write [5 15] :RequestCanceledException INFO [2019-12-17 12:30:37,510] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21217,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:30:37,520] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21217,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:30:37,570] jepsen worker 15 - jepsen.util 4425 :invoke :cas [5 [20 33]] INFO [2019-12-17 12:30:37,610] jepsen worker 61 - jepsen.util 5551 :info :write [20 22] :RequestCanceledException INFO [2019-12-17 12:30:37,626] jepsen worker 81 - jepsen.util 4941 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:30:37,626] jepsen worker 81 - jepsen.util 4941 :invoke :cas [27 [33 18]] INFO [2019-12-17 12:30:37,654] jepsen worker 61 - jepsen.util 5641 :invoke :write [20 8] INFO [2019-12-17 12:30:37,730] jepsen worker 21 - jepsen.util 4881 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:30:37,730] jepsen worker 21 - jepsen.util 4881 :invoke :cas [7 [32 46]] INFO [2019-12-17 12:30:37,730] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21233,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:30:37,738] jepsen worker 28 - jepsen.util 2638 :ok :write [9 41] INFO [2019-12-17 12:30:37,741] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21233,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:30:37,746] jepsen worker 28 - jepsen.util 2638 :invoke :cas [9 [44 29]] INFO [2019-12-17 12:30:37,826] jepsen worker 12 - jepsen.util 3972 :info :write [4 33] :RequestCanceledException INFO [2019-12-17 12:30:37,884] jepsen worker 36 - jepsen.util 3906 :ok :write [12 16] INFO [2019-12-17 12:30:37,885] jepsen worker 36 - jepsen.util 3906 :invoke :write [12 1] INFO [2019-12-17 12:30:37,889] jepsen worker 12 - jepsen.util 4062 :invoke :read [4 nil] INFO [2019-12-17 12:30:37,918] jepsen worker 18 - jepsen.util 3798 :fail :cas [6 [47 42]] :ValueNotSwapFrom INFO [2019-12-17 12:30:37,918] jepsen worker 18 - jepsen.util 3798 :invoke :cas [6 [34 8]] INFO [2019-12-17 12:30:37,925] jepsen worker 29 - jepsen.util 2999 :info :cas [9 [24 13]] :RequestCanceledException INFO [2019-12-17 12:30:37,926] jepsen worker 19 - jepsen.util 3799 :info :cas [6 [2 3]] :RequestCanceledException INFO [2019-12-17 12:30:37,941] jepsen worker 31 - jepsen.util 3181 :info :cas [10 [30 29]] :RequestCanceledException INFO [2019-12-17 12:30:37,941] jepsen worker 47 - jepsen.util 4187 :ok :read [15 12] INFO [2019-12-17 12:30:37,942] jepsen worker 47 - jepsen.util 4187 :invoke :write [15 28] INFO [2019-12-17 12:30:37,943] jepsen worker 29 - jepsen.util 3089 :invoke :read [9 nil] INFO [2019-12-17 12:30:37,951] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21239,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:30:37,959] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21239,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:30:37,960] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21243,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:30:37,962] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21243,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:30:37,964] jepsen worker 19 - jepsen.util 3889 :invoke :cas [6 [33 12]] INFO [2019-12-17 12:30:37,982] jepsen worker 31 - jepsen.util 3271 :invoke :read [10 nil] INFO [2019-12-17 12:30:38,002] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:30:38,003] jepsen nemesis - couchbase.util Rebalance check count: 157 INFO [2019-12-17 12:30:38,003] jepsen nemesis - couchbase.util Rebalance stuck count: 138 INFO [2019-12-17 12:30:38,033] jepsen worker 79 - jepsen.util 3769 :fail :cas [26 [33 4]] :ValueNotSwapFrom INFO [2019-12-17 12:30:38,034] jepsen worker 79 - jepsen.util 3769 :invoke :write [26 22] INFO [2019-12-17 12:30:38,043] jepsen worker 86 - jepsen.util 5126 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:30:38,043] jepsen worker 86 - jepsen.util 5126 :invoke :write [28 1] INFO [2019-12-17 12:30:38,053] jepsen worker 74 - jepsen.util 4034 :ok :read [24 23] INFO [2019-12-17 12:30:38,054] jepsen worker 74 - jepsen.util 4034 :invoke :cas [24 [39 44]] INFO [2019-12-17 12:30:38,068] jepsen worker 46 - jepsen.util 3916 :fail :cas [15 [13 25]] :ValueNotSwapFrom INFO [2019-12-17 12:30:38,069] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21253,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:30:38,069] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21253,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:30:38,070] jepsen worker 46 - jepsen.util 3916 :invoke :write [15 11] INFO [2019-12-17 12:30:38,071] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21254,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:30:38,072] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21254,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:30:38,073] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21255,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:30:38,073] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21255,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:30:38,074] jepsen worker 10 - jepsen.util 3520 :ok :read [3 22] INFO [2019-12-17 12:30:38,074] jepsen worker 10 - jepsen.util 3520 :invoke :read [3 nil] INFO [2019-12-17 12:30:38,085] jepsen worker 32 - jepsen.util 4172 :ok :read [10 8] INFO [2019-12-17 12:30:38,085] jepsen worker 32 - jepsen.util 4172 :invoke :cas [10 [43 11]] INFO [2019-12-17 12:30:38,092] jepsen worker 77 - jepsen.util 4847 :info :cas [25 [26 39]] :RequestCanceledException INFO [2019-12-17 12:30:38,100] jepsen worker 89 - jepsen.util 3509 :ok :write [29 18] INFO [2019-12-17 12:30:38,101] jepsen worker 89 - jepsen.util 3509 :invoke :write [29 18] INFO [2019-12-17 12:30:38,109] jepsen worker 63 - jepsen.util 3213 :fail :cas [21 [24 6]] :ValueNotSwapFrom INFO [2019-12-17 12:30:38,110] jepsen worker 63 - jepsen.util 3213 :invoke :write [21 1] INFO [2019-12-17 12:30:38,125] jepsen worker 24 - jepsen.util 4794 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:30:38,125] jepsen worker 30 - jepsen.util 4080 :info :write [10 1] :RequestCanceledException INFO [2019-12-17 12:30:38,125] jepsen worker 24 - jepsen.util 4794 :invoke :read [8 nil] INFO [2019-12-17 12:30:38,125] jepsen worker 77 - jepsen.util 4937 :invoke :cas [25 [8 25]] INFO [2019-12-17 12:30:38,126] jepsen worker 65 - jepsen.util 3215 :fail :cas [21 [41 27]] :ValueNotSwapFrom INFO [2019-12-17 12:30:38,127] jepsen worker 65 - jepsen.util 3215 :invoke :read [21 nil] INFO [2019-12-17 12:30:38,142] jepsen worker 72 - jepsen.util 3852 :fail :cas [24 [38 44]] :ValueNotSwapFrom INFO [2019-12-17 12:30:38,142] jepsen worker 72 - jepsen.util 3852 :invoke :read [24 nil] INFO [2019-12-17 12:30:38,155] jepsen worker 53 - jepsen.util 3743 :info :write [17 29] :RequestCanceledException INFO [2019-12-17 12:30:38,236] jepsen worker 28 - jepsen.util 2638 :fail :cas [9 [44 29]] :ValueNotSwapFrom INFO [2019-12-17 12:30:38,236] jepsen worker 29 - jepsen.util 3089 :ok :read [9 41] INFO [2019-12-17 12:30:38,236] jepsen worker 28 - jepsen.util 2638 :invoke :read [9 nil] INFO [2019-12-17 12:30:38,237] jepsen worker 30 - jepsen.util 4170 :invoke :read [10 nil] INFO [2019-12-17 12:30:38,237] jepsen worker 29 - jepsen.util 3089 :invoke :read [9 nil] INFO [2019-12-17 12:30:38,239] jepsen worker 87 - jepsen.util 3507 :ok :write [29 5] INFO [2019-12-17 12:30:38,239] jepsen worker 44 - jepsen.util 5264 :info :write [14 27] :RequestCanceledException INFO [2019-12-17 12:30:38,239] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21272,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:30:38,239] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21268,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:30:38,239] jepsen worker 87 - jepsen.util 3507 :invoke :read [29 nil] INFO [2019-12-17 12:30:38,239] jepsen worker 68 - jepsen.util 4478 :info :write [22 48] :RequestCanceledException INFO [2019-12-17 12:30:38,240] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21272,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:30:38,241] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21279,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:30:38,243] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21279,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:30:38,246] jepsen worker 49 - jepsen.util 4369 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:30:38,246] jepsen worker 49 - jepsen.util 4369 :invoke :read [16 nil] INFO [2019-12-17 12:30:38,247] jepsen worker 9 - jepsen.util 4239 :fail :read [3 nil] :RequestTimeoutException INFO [2019-12-17 12:30:38,248] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21268,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:30:38,254] jepsen worker 9 - jepsen.util 4239 :invoke :write [3 17] INFO [2019-12-17 12:30:38,256] jepsen worker 34 - jepsen.util 4984 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:30:38,259] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21282,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:30:38,261] jepsen worker 34 - jepsen.util 4984 :invoke :read [11 nil] INFO [2019-12-17 12:30:38,263] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21282,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:30:38,271] jepsen worker 53 - jepsen.util 3833 :invoke :cas [17 [16 0]] INFO [2019-12-17 12:30:38,294] jepsen worker 4 - jepsen.util 2884 :info :write [1 16] :RequestCanceledException INFO [2019-12-17 12:30:38,295] jepsen worker 44 - jepsen.util 5354 :invoke :write [14 11] INFO [2019-12-17 12:30:38,298] jepsen worker 68 - jepsen.util 4568 :invoke :write [22 38] INFO [2019-12-17 12:30:38,350] jepsen worker 4 - jepsen.util 2974 :invoke :cas [1 [9 41]] INFO [2019-12-17 12:30:38,363] jepsen worker 52 - jepsen.util 3832 :ok :write [17 37] INFO [2019-12-17 12:30:38,363] jepsen worker 45 - jepsen.util 3825 :ok :write [15 4] INFO [2019-12-17 12:30:38,364] jepsen worker 52 - jepsen.util 3832 :invoke :write [17 3] INFO [2019-12-17 12:30:38,365] jepsen worker 45 - jepsen.util 3825 :invoke :read [15 nil] INFO [2019-12-17 12:30:38,459] jepsen worker 3 - jepsen.util 3873 :info :write [1 29] :RequestCanceledException INFO [2019-12-17 12:30:38,459] jepsen worker 13 - jepsen.util 3523 :fail :read [4 nil] :RequestTimeoutException INFO [2019-12-17 12:30:38,460] jepsen worker 13 - jepsen.util 3523 :invoke :cas [4 [11 16]] INFO [2019-12-17 12:30:38,487] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21289,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:30:38,489] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21289,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:30:38,489] jepsen worker 3 - jepsen.util 3963 :invoke :write [1 45] INFO [2019-12-17 12:30:38,500] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21293,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:30:38,501] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21293,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:30:38,503] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21294,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:30:38,504] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21294,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:30:38,528] jepsen worker 23 - jepsen.util 4883 :info :cas [7 [38 5]] :RequestCanceledException INFO [2019-12-17 12:30:38,537] jepsen worker 59 - jepsen.util 4829 :info :write [19 0] :RequestCanceledException INFO [2019-12-17 12:30:38,572] jepsen worker 27 - jepsen.util 3537 :ok :read [9 36] INFO [2019-12-17 12:30:38,573] jepsen worker 27 - jepsen.util 3537 :invoke :cas [9 [18 8]] INFO [2019-12-17 12:30:38,588] jepsen worker 88 - jepsen.util 3778 :ok :read [29 29] INFO [2019-12-17 12:30:38,595] jepsen worker 88 - jepsen.util 3778 :invoke :write [29 49] INFO [2019-12-17 12:30:38,595] jepsen worker 23 - jepsen.util 4973 :invoke :read [7 nil] INFO [2019-12-17 12:30:38,596] jepsen worker 36 - jepsen.util 3906 :ok :write [12 1] INFO [2019-12-17 12:30:38,597] jepsen worker 36 - jepsen.util 3906 :invoke :cas [12 [16 5]] INFO [2019-12-17 12:30:38,598] jepsen worker 59 - jepsen.util 4919 :invoke :write [19 35] INFO [2019-12-17 12:30:38,599] jepsen worker 80 - jepsen.util 3770 :fail :cas [26 [37 22]] :ValueNotSwapFrom INFO [2019-12-17 12:30:38,600] jepsen worker 80 - jepsen.util 3770 :invoke :read [26 nil] INFO [2019-12-17 12:30:38,602] jepsen worker 73 - jepsen.util 3493 :ok :read [24 23] INFO [2019-12-17 12:30:38,603] jepsen worker 73 - jepsen.util 3493 :invoke :write [24 3] INFO [2019-12-17 12:30:38,653] jepsen worker 56 - jepsen.util 3746 :info :cas [18 [10 7]] :RequestCanceledException INFO [2019-12-17 12:30:38,659] jepsen worker 56 - jepsen.util 3836 :invoke :cas [18 [7 24]] INFO [2019-12-17 12:30:38,675] jepsen worker 20 - jepsen.util 3530 :ok :write [6 15] INFO [2019-12-17 12:30:38,675] jepsen worker 20 - jepsen.util 3530 :invoke :write [6 40] INFO [2019-12-17 12:30:38,687] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21309,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:30:38,687] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21309,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:30:38,687] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21311,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:30:38,687] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21311,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:30:38,695] jepsen worker 31 - jepsen.util 3271 :ok :read [10 8] INFO [2019-12-17 12:30:38,695] jepsen worker 31 - jepsen.util 3271 :invoke :write [10 8] INFO [2019-12-17 12:30:38,701] jepsen worker 64 - jepsen.util 3034 :fail :read [21 nil] :RequestTimeoutException INFO [2019-12-17 12:30:38,701] jepsen worker 72 - jepsen.util 3852 :ok :read [24 23] INFO [2019-12-17 12:30:38,701] jepsen worker 64 - jepsen.util 3034 :invoke :read [21 nil] INFO [2019-12-17 12:30:38,701] jepsen worker 72 - jepsen.util 3852 :invoke :write [24 6] INFO [2019-12-17 12:30:38,702] jepsen worker 18 - jepsen.util 3798 :fail :cas [6 [34 8]] :ValueNotSwapFrom INFO [2019-12-17 12:30:38,702] jepsen worker 18 - jepsen.util 3798 :invoke :read [6 nil] INFO [2019-12-17 12:30:38,714] jepsen worker 47 - jepsen.util 4187 :ok :write [15 28] INFO [2019-12-17 12:30:38,715] jepsen worker 47 - jepsen.util 4187 :invoke :write [15 25] INFO [2019-12-17 12:30:38,724] jepsen worker 51 - jepsen.util 3201 :info :write [17 34] :RequestCanceledException INFO [2019-12-17 12:30:38,725] jepsen worker 43 - jepsen.util 4633 :info :cas [14 [27 34]] :RequestCanceledException INFO [2019-12-17 12:30:38,730] jepsen worker 51 - jepsen.util 3291 :invoke :read [17 nil] INFO [2019-12-17 12:30:38,732] jepsen worker 43 - jepsen.util 4723 :invoke :write [14 2] INFO [2019-12-17 12:30:38,733] jepsen worker 10 - jepsen.util 3520 :ok :read [3 22] INFO [2019-12-17 12:30:38,734] jepsen worker 84 - jepsen.util 4584 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:30:38,734] jepsen worker 10 - jepsen.util 3520 :invoke :write [3 44] INFO [2019-12-17 12:30:38,734] jepsen worker 84 - jepsen.util 4584 :invoke :cas [28 [16 31]] INFO [2019-12-17 12:30:38,747] jepsen worker 69 - jepsen.util 3669 :fail :cas [23 [32 44]] :ValueNotSwapFrom INFO [2019-12-17 12:30:38,747] jepsen worker 69 - jepsen.util 3669 :invoke :read [23 nil] INFO [2019-12-17 12:30:38,751] jepsen worker 71 - jepsen.util 3761 :info :write [23 13] :RequestCanceledException INFO [2019-12-17 12:30:38,752] jepsen worker 32 - jepsen.util 4172 :fail :cas [10 [43 11]] :ValueNotSwapFrom INFO [2019-12-17 12:30:38,752] jepsen worker 32 - jepsen.util 4172 :invoke :cas [10 [6 13]] INFO [2019-12-17 12:30:38,753] jepsen worker 55 - jepsen.util 4555 :ok :write [18 48] INFO [2019-12-17 12:30:38,753] jepsen worker 55 - jepsen.util 4555 :invoke :cas [18 [46 40]] INFO [2019-12-17 12:30:38,754] jepsen worker 71 - jepsen.util 3851 :invoke :cas [23 [11 48]] INFO [2019-12-17 12:30:38,756] jepsen worker 28 - jepsen.util 2638 :ok :read [9 41] INFO [2019-12-17 12:30:38,756] jepsen worker 28 - jepsen.util 2638 :invoke :read [9 nil] INFO [2019-12-17 12:30:38,773] jepsen worker 4 - jepsen.util 2974 :fail :cas [1 [9 41]] :ValueNotSwapFrom INFO [2019-12-17 12:30:38,773] jepsen worker 30 - jepsen.util 4170 :ok :read [10 8] INFO [2019-12-17 12:30:38,773] jepsen worker 4 - jepsen.util 2974 :invoke :read [1 nil] INFO [2019-12-17 12:30:38,773] jepsen worker 30 - jepsen.util 4170 :invoke :read [10 nil] INFO [2019-12-17 12:30:38,780] jepsen worker 5 - jepsen.util 3335 :fail :cas [1 [11 48]] :ValueNotSwapFrom INFO [2019-12-17 12:30:38,780] jepsen worker 5 - jepsen.util 3335 :invoke :write [1 22] INFO [2019-12-17 12:30:38,783] jepsen worker 36 - jepsen.util 3906 :fail :cas [12 [16 5]] :ValueNotSwapFrom INFO [2019-12-17 12:30:38,784] jepsen worker 36 - jepsen.util 3906 :invoke :read [12 nil] INFO [2019-12-17 12:30:38,784] jepsen worker 65 - jepsen.util 3215 :ok :read [21 13] INFO [2019-12-17 12:30:38,784] jepsen worker 65 - jepsen.util 3215 :invoke :write [21 37] INFO [2019-12-17 12:30:38,786] jepsen worker 45 - jepsen.util 3825 :ok :read [15 4] INFO [2019-12-17 12:30:38,786] jepsen worker 45 - jepsen.util 3825 :invoke :write [15 9] INFO [2019-12-17 12:30:38,792] jepsen worker 27 - jepsen.util 3537 :fail :cas [9 [18 8]] :ValueNotSwapFrom INFO [2019-12-17 12:30:38,792] jepsen worker 27 - jepsen.util 3537 :invoke :write [9 41] INFO [2019-12-17 12:30:38,808] jepsen worker 66 - jepsen.util 4746 :info :cas [22 [10 49]] :RequestCanceledException INFO [2019-12-17 12:30:38,811] jepsen worker 66 - jepsen.util 4836 :invoke :cas [22 [33 3]] INFO [2019-12-17 12:30:38,824] jepsen worker 7 - jepsen.util 4687 :info :cas [2 [28 38]] :RequestCanceledException INFO [2019-12-17 12:30:38,824] jepsen worker 25 - jepsen.util 5245 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:30:38,824] jepsen worker 25 - jepsen.util 5245 :invoke :read [8 nil] INFO [2019-12-17 12:30:38,826] jepsen worker 7 - jepsen.util 4777 :invoke :cas [2 [9 49]] INFO [2019-12-17 12:30:38,856] jepsen worker 62 - jepsen.util 4652 :info :write [20 42] :RequestCanceledException INFO [2019-12-17 12:30:38,856] jepsen worker 39 - jepsen.util 4719 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:30:38,856] jepsen worker 39 - jepsen.util 4719 :invoke :cas [13 [39 37]] INFO [2019-12-17 12:30:38,858] jepsen worker 37 - jepsen.util 3547 :ok :read [12 16] INFO [2019-12-17 12:30:38,858] jepsen worker 37 - jepsen.util 3547 :invoke :read [12 nil] INFO [2019-12-17 12:30:38,858] jepsen worker 62 - jepsen.util 4742 :invoke :cas [20 [24 25]] INFO [2019-12-17 12:30:38,901] jepsen worker 29 - jepsen.util 3089 :ok :read [9 41] INFO [2019-12-17 12:30:38,901] jepsen worker 29 - jepsen.util 3089 :invoke :write [9 10] INFO [2019-12-17 12:30:38,904] jepsen worker 18 - jepsen.util 3798 :ok :read [6 15] INFO [2019-12-17 12:30:38,904] jepsen worker 18 - jepsen.util 3798 :invoke :write [6 35] INFO [2019-12-17 12:30:38,906] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21347,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:30:38,907] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21347,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:30:38,907] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21349,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:30:38,907] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21349,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:30:38,909] jepsen worker 6 - jepsen.util 5316 :info :cas [2 [30 26]] :RequestCanceledException INFO [2019-12-17 12:30:38,913] jepsen worker 6 - jepsen.util 5406 :invoke :write [2 49] INFO [2019-12-17 12:30:38,924] jepsen worker 82 - jepsen.util 5842 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:30:38,924] jepsen worker 82 - jepsen.util 5842 :invoke :read [27 nil] INFO [2019-12-17 12:30:38,945] jepsen worker 32 - jepsen.util 4172 :fail :cas [10 [6 13]] :ValueNotSwapFrom INFO [2019-12-17 12:30:38,946] jepsen worker 32 - jepsen.util 4172 :invoke :read [10 nil] INFO [2019-12-17 12:30:38,963] jepsen worker 38 - jepsen.util 3728 :fail :cas [12 [47 4]] :ValueNotSwapFrom INFO [2019-12-17 12:30:38,963] jepsen worker 38 - jepsen.util 3728 :invoke :cas [12 [15 2]] INFO [2019-12-17 12:30:38,971] jepsen worker 28 - jepsen.util 2638 :ok :read [9 41] INFO [2019-12-17 12:30:38,971] jepsen worker 28 - jepsen.util 2638 :invoke :read [9 nil] INFO [2019-12-17 12:30:39,007] jepsen worker 28 - jepsen.util 2638 :ok :read [9 41] INFO [2019-12-17 12:30:39,008] jepsen worker 28 - jepsen.util 2638 :invoke :write [9 11] INFO [2019-12-17 12:30:39,015] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:30:39,016] jepsen nemesis - couchbase.util Rebalance check count: 158 INFO [2019-12-17 12:30:39,016] jepsen nemesis - couchbase.util Rebalance stuck count: 139 INFO [2019-12-17 12:30:39,021] jepsen worker 52 - jepsen.util 3832 :ok :write [17 3] INFO [2019-12-17 12:30:39,021] jepsen worker 52 - jepsen.util 3832 :invoke :cas [17 [41 45]] INFO [2019-12-17 12:30:39,028] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21360,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:30:39,028] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21360,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:30:39,028] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21363,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:30:39,028] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21363,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:30:39,028] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21364,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:30:39,028] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21364,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:30:39,038] jepsen worker 50 - jepsen.util 5450 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:30:39,039] jepsen worker 50 - jepsen.util 5450 :invoke :write [16 11] INFO [2019-12-17 12:30:39,056] jepsen worker 85 - jepsen.util 5485 :info :cas [28 [7 3]] :RequestCanceledException INFO [2019-12-17 12:30:39,058] jepsen worker 85 - jepsen.util 5575 :invoke :read [28 nil] INFO [2019-12-17 12:30:39,061] jepsen worker 88 - jepsen.util 3778 :ok :write [29 49] INFO [2019-12-17 12:30:39,061] jepsen worker 88 - jepsen.util 3778 :invoke :write [29 18] INFO [2019-12-17 12:30:39,063] jepsen worker 70 - jepsen.util 4300 :ok :write [23 47] INFO [2019-12-17 12:30:39,063] jepsen worker 70 - jepsen.util 4300 :invoke :read [23 nil] INFO [2019-12-17 12:30:39,078] jepsen worker 30 - jepsen.util 4170 :ok :read [10 8] INFO [2019-12-17 12:30:39,078] jepsen worker 30 - jepsen.util 4170 :invoke :read [10 nil] INFO [2019-12-17 12:30:39,090] jepsen worker 79 - jepsen.util 3769 :ok :write [26 22] INFO [2019-12-17 12:30:39,090] jepsen worker 79 - jepsen.util 3769 :invoke :cas [26 [17 40]] INFO [2019-12-17 12:30:39,135] jepsen worker 80 - jepsen.util 3770 :ok :read [26 22] INFO [2019-12-17 12:30:39,135] jepsen worker 80 - jepsen.util 3770 :invoke :write [26 45] INFO [2019-12-17 12:30:39,144] jepsen worker 87 - jepsen.util 3507 :ok :read [29 49] INFO [2019-12-17 12:30:39,144] jepsen worker 87 - jepsen.util 3507 :invoke :read [29 nil] INFO [2019-12-17 12:30:39,150] jepsen worker 28 - jepsen.util 2638 :ok :write [9 11] INFO [2019-12-17 12:30:39,150] jepsen worker 28 - jepsen.util 2638 :invoke :write [9 21] INFO [2019-12-17 12:30:39,160] jepsen worker 69 - jepsen.util 3669 :ok :read [23 16] INFO [2019-12-17 12:30:39,160] jepsen worker 69 - jepsen.util 3669 :invoke :write [23 6] INFO [2019-12-17 12:30:39,195] jepsen worker 63 - jepsen.util 3213 :ok :write [21 1] INFO [2019-12-17 12:30:39,195] jepsen worker 63 - jepsen.util 3213 :invoke :read [21 nil] INFO [2019-12-17 12:30:39,207] jepsen worker 56 - jepsen.util 3836 :fail :cas [18 [7 24]] :ValueNotSwapFrom INFO [2019-12-17 12:30:39,207] jepsen worker 3 - jepsen.util 3963 :ok :write [1 45] INFO [2019-12-17 12:30:39,207] jepsen worker 56 - jepsen.util 3836 :invoke :cas [18 [18 35]] INFO [2019-12-17 12:30:39,207] jepsen worker 3 - jepsen.util 3963 :invoke :cas [1 [12 21]] INFO [2019-12-17 12:30:39,224] jepsen worker 67 - jepsen.util 5197 :info :cas [22 [44 26]] :RequestCanceledException INFO [2019-12-17 12:30:39,224] jepsen worker 57 - jepsen.util 4647 :info :cas [19 [23 10]] :RequestCanceledException INFO [2019-12-17 12:30:39,224] jepsen worker 83 - jepsen.util 5033 :info :cas [27 [33 42]] :RequestCanceledException INFO [2019-12-17 12:30:39,224] jepsen worker 22 - jepsen.util 5152 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:30:39,224] jepsen worker 22 - jepsen.util 5152 :invoke :read [7 nil] INFO [2019-12-17 12:30:39,225] jepsen worker 53 - jepsen.util 3833 :fail :cas [17 [16 0]] :ValueNotSwapFrom INFO [2019-12-17 12:30:39,225] jepsen worker 53 - jepsen.util 3833 :invoke :write [17 8] INFO [2019-12-17 12:30:39,226] jepsen worker 67 - jepsen.util 5287 :invoke :read [22 nil] INFO [2019-12-17 12:30:39,228] jepsen worker 57 - jepsen.util 4737 :invoke :write [19 35] INFO [2019-12-17 12:30:39,229] jepsen worker 83 - jepsen.util 5123 :invoke :read [27 nil] INFO [2019-12-17 12:30:39,231] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21377,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:30:39,231] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21377,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:30:39,231] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21378,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:30:39,231] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21378,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:30:39,237] jepsen worker 40 - jepsen.util 5260 :info :cas [13 [44 14]] :RequestCanceledException INFO [2019-12-17 12:30:39,241] jepsen worker 40 - jepsen.util 5350 :invoke :write [13 49] INFO [2019-12-17 12:30:39,244] jepsen worker 63 - jepsen.util 3213 :ok :read [21 1] INFO [2019-12-17 12:30:39,244] jepsen worker 63 - jepsen.util 3213 :invoke :write [21 28] INFO [2019-12-17 12:30:39,245] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21388,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:30:39,245] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21388,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:30:39,245] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21391,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:30:39,245] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21391,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:30:39,253] jepsen worker 48 - jepsen.util 4458 :info :cas [16 [21 35]] :RequestCanceledException INFO [2019-12-17 12:30:39,253] jepsen worker 26 - jepsen.util 5246 :info :cas [8 [11 43]] :RequestCanceledException INFO [2019-12-17 12:30:39,257] jepsen worker 48 - jepsen.util 4548 :invoke :write [16 28] INFO [2019-12-17 12:30:39,259] jepsen worker 26 - jepsen.util 5336 :invoke :read [8 nil] INFO [2019-12-17 12:30:39,277] jepsen worker 46 - jepsen.util 3916 :ok :write [15 11] INFO [2019-12-17 12:30:39,277] jepsen worker 46 - jepsen.util 3916 :invoke :write [15 26] INFO [2019-12-17 12:30:39,277] jepsen worker 13 - jepsen.util 3523 :fail :cas [4 [11 16]] :ValueNotSwapFrom INFO [2019-12-17 12:30:39,277] jepsen worker 13 - jepsen.util 3523 :invoke :cas [4 [39 27]] INFO [2019-12-17 12:30:39,280] jepsen worker 71 - jepsen.util 3851 :fail :cas [23 [11 48]] :ValueNotSwapFrom INFO [2019-12-17 12:30:39,280] jepsen worker 71 - jepsen.util 3851 :invoke :write [23 39] INFO [2019-12-17 12:30:39,291] jepsen worker 35 - jepsen.util 4805 :info :cas [11 [7 10]] :RequestCanceledException INFO [2019-12-17 12:30:39,295] jepsen worker 35 - jepsen.util 4895 :invoke :cas [11 [36 38]] INFO [2019-12-17 12:30:39,302] jepsen worker 9 - jepsen.util 4239 :ok :write [3 17] INFO [2019-12-17 12:30:39,304] jepsen worker 9 - jepsen.util 4239 :invoke :cas [3 [14 17]] INFO [2019-12-17 12:30:39,309] jepsen worker 2 - jepsen.util 4682 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:30:39,309] jepsen worker 2 - jepsen.util 4682 :invoke :cas [0 [4 18]] INFO [2019-12-17 12:30:39,310] jepsen worker 60 - jepsen.util 5190 :info :cas [20 [35 25]] :RequestCanceledException INFO [2019-12-17 12:30:39,310] jepsen worker 0 - jepsen.util 5310 :info :cas [0 [1 6]] :RequestCanceledException INFO [2019-12-17 12:30:39,313] jepsen worker 0 - jepsen.util 5400 :invoke :cas [0 [36 13]] INFO [2019-12-17 12:30:39,314] jepsen worker 60 - jepsen.util 5280 :invoke :write [20 34] INFO [2019-12-17 12:30:39,315] jepsen worker 14 - jepsen.util 3794 :ok :write [4 47] INFO [2019-12-17 12:30:39,315] jepsen worker 14 - jepsen.util 3794 :invoke :cas [4 [35 7]] INFO [2019-12-17 12:30:39,325] jepsen worker 58 - jepsen.util 5008 :info :write [19 35] :RequestCanceledException INFO [2019-12-17 12:30:39,325] jepsen worker 42 - jepsen.util 5802 :info :cas [14 [16 47]] :RequestCanceledException INFO [2019-12-17 12:30:39,328] jepsen worker 42 - jepsen.util 5892 :invoke :write [14 42] INFO [2019-12-17 12:30:39,328] jepsen worker 58 - jepsen.util 5098 :invoke :read [19 nil] INFO [2019-12-17 12:30:39,330] jepsen worker 12 - jepsen.util 4062 :ok :read [4 47] INFO [2019-12-17 12:30:39,331] jepsen worker 12 - jepsen.util 4062 :invoke :write [4 45] INFO [2019-12-17 12:30:39,337] jepsen worker 79 - jepsen.util 3769 :fail :cas [26 [17 40]] :ValueNotSwapFrom INFO [2019-12-17 12:30:39,337] jepsen worker 79 - jepsen.util 3769 :invoke :read [26 nil] INFO [2019-12-17 12:30:39,355] jepsen worker 17 - jepsen.util 4607 :info :write [5 27] :RequestCanceledException INFO [2019-12-17 12:30:39,357] jepsen worker 17 - jepsen.util 4697 :invoke :cas [5 [24 6]] INFO [2019-12-17 12:30:39,358] jepsen worker 89 - jepsen.util 3509 :ok :write [29 18] INFO [2019-12-17 12:30:39,358] jepsen worker 89 - jepsen.util 3509 :invoke :write [29 23] INFO [2019-12-17 12:30:39,360] jepsen worker 65 - jepsen.util 3215 :ok :write [21 37] INFO [2019-12-17 12:30:39,360] jepsen worker 65 - jepsen.util 3215 :invoke :cas [21 [25 22]] INFO [2019-12-17 12:30:39,384] jepsen worker 52 - jepsen.util 3832 :fail :cas [17 [41 45]] :ValueNotSwapFrom INFO [2019-12-17 12:30:39,384] jepsen worker 52 - jepsen.util 3832 :invoke :cas [17 [14 1]] INFO [2019-12-17 12:30:39,386] jepsen worker 28 - jepsen.util 2638 :ok :write [9 21] INFO [2019-12-17 12:30:39,386] jepsen worker 28 - jepsen.util 2638 :invoke :cas [9 [5 10]] INFO [2019-12-17 12:30:39,390] jepsen worker 33 - jepsen.util 5163 :info :write [11 46] :RequestCanceledException INFO [2019-12-17 12:30:39,392] jepsen worker 33 - jepsen.util 5253 :invoke :read [11 nil] INFO [2019-12-17 12:30:39,405] jepsen worker 9 - jepsen.util 4239 :fail :cas [3 [14 17]] :ValueNotSwapFrom INFO [2019-12-17 12:30:39,406] jepsen worker 9 - jepsen.util 4239 :invoke :read [3 nil] INFO [2019-12-17 12:30:39,424] jepsen worker 65 - jepsen.util 3215 :fail :cas [21 [25 22]] :ValueNotSwapFrom INFO [2019-12-17 12:30:39,424] jepsen worker 65 - jepsen.util 3215 :invoke :cas [21 [48 28]] INFO [2019-12-17 12:30:39,432] jepsen worker 28 - jepsen.util 2638 :fail :cas [9 [5 10]] :ValueNotSwapFrom INFO [2019-12-17 12:30:39,432] jepsen worker 28 - jepsen.util 2638 :invoke :read [9 nil] INFO [2019-12-17 12:30:39,442] jepsen worker 87 - jepsen.util 3507 :ok :read [29 18] INFO [2019-12-17 12:30:39,443] jepsen worker 87 - jepsen.util 3507 :invoke :cas [29 [25 14]] INFO [2019-12-17 12:30:39,444] jepsen worker 9 - jepsen.util 4239 :ok :read [3 17] INFO [2019-12-17 12:30:39,444] jepsen worker 9 - jepsen.util 4239 :invoke :write [3 37] INFO [2019-12-17 12:30:39,485] jepsen worker 65 - jepsen.util 3215 :fail :cas [21 [48 28]] :ValueNotSwapFrom INFO [2019-12-17 12:30:39,486] jepsen worker 65 - jepsen.util 3215 :invoke :cas [21 [7 2]] INFO [2019-12-17 12:30:39,487] jepsen worker 28 - jepsen.util 2638 :ok :read [9 21] INFO [2019-12-17 12:30:39,488] jepsen worker 28 - jepsen.util 2638 :invoke :read [9 nil] INFO [2019-12-17 12:30:39,491] jepsen worker 75 - jepsen.util 4935 :info :write [25 46] :RequestCanceledException INFO [2019-12-17 12:30:39,493] jepsen worker 75 - jepsen.util 5025 :invoke :write [25 21] INFO [2019-12-17 12:30:39,494] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21424,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:30:39,494] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21424,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:30:39,509] jepsen worker 1 - jepsen.util 4591 :info :cas [0 [29 48]] :RequestCanceledException INFO [2019-12-17 12:30:39,512] jepsen worker 1 - jepsen.util 4681 :invoke :write [0 1] INFO [2019-12-17 12:30:39,517] jepsen worker 38 - jepsen.util 3728 :fail :cas [12 [15 2]] :ValueNotSwapFrom INFO [2019-12-17 12:30:39,517] jepsen worker 38 - jepsen.util 3728 :invoke :cas [12 [22 13]] INFO [2019-12-17 12:30:39,524] jepsen worker 8 - jepsen.util 4238 :info :write [2 6] :RequestCanceledException INFO [2019-12-17 12:30:39,525] jepsen worker 41 - jepsen.util 5081 :info :write [13 36] :RequestCanceledException INFO [2019-12-17 12:30:39,527] jepsen worker 8 - jepsen.util 4328 :invoke :read [2 nil] INFO [2019-12-17 12:30:39,527] jepsen worker 41 - jepsen.util 5171 :invoke :write [13 31] INFO [2019-12-17 12:30:39,530] jepsen worker 65 - jepsen.util 3215 :fail :cas [21 [7 2]] :ValueNotSwapFrom INFO [2019-12-17 12:30:39,530] jepsen worker 65 - jepsen.util 3215 :invoke :write [21 20] INFO [2019-12-17 12:30:39,556] jepsen worker 16 - jepsen.util 5596 :info :cas [5 [27 11]] :RequestCanceledException INFO [2019-12-17 12:30:39,560] jepsen worker 16 - jepsen.util 5686 :invoke :cas [5 [9 35]] INFO [2019-12-17 12:30:39,563] jepsen worker 28 - jepsen.util 2638 :ok :read [9 21] INFO [2019-12-17 12:30:39,564] jepsen worker 28 - jepsen.util 2638 :invoke :cas [9 [49 10]] INFO [2019-12-17 12:30:39,565] jepsen worker 51 - jepsen.util 3291 :ok :read [17 3] INFO [2019-12-17 12:30:39,565] jepsen worker 51 - jepsen.util 3291 :invoke :cas [17 [29 29]] INFO [2019-12-17 12:30:39,572] jepsen worker 3 - jepsen.util 3963 :fail :cas [1 [12 21]] :ValueNotSwapFrom INFO [2019-12-17 12:30:39,572] jepsen worker 3 - jepsen.util 3963 :invoke :cas [1 [36 24]] INFO [2019-12-17 12:30:39,608] jepsen worker 76 - jepsen.util 5026 :info :cas [25 [49 45]] :RequestCanceledException INFO [2019-12-17 12:30:39,611] jepsen worker 76 - jepsen.util 5116 :invoke :cas [25 [30 18]] INFO [2019-12-17 12:30:39,611] jepsen worker 28 - jepsen.util 2638 :fail :cas [9 [49 10]] :ValueNotSwapFrom INFO [2019-12-17 12:30:39,612] jepsen worker 28 - jepsen.util 2638 :invoke :read [9 nil] INFO [2019-12-17 12:30:39,624] jepsen worker 15 - jepsen.util 4425 :info :cas [5 [20 33]] :RequestCanceledException INFO [2019-12-17 12:30:39,628] jepsen worker 15 - jepsen.util 4515 :invoke :cas [5 [38 8]] INFO [2019-12-17 12:30:39,638] jepsen worker 78 - jepsen.util 4128 :info :write [26 28] :RequestCanceledException INFO [2019-12-17 12:30:39,640] jepsen worker 78 - jepsen.util 4218 :invoke :write [26 45] INFO [2019-12-17 12:30:39,652] jepsen worker 46 - jepsen.util 3916 :ok :write [15 26] INFO [2019-12-17 12:30:39,652] jepsen worker 46 - jepsen.util 3916 :invoke :write [15 47] INFO [2019-12-17 12:30:39,656] jepsen worker 65 - jepsen.util 3215 :ok :write [21 20] INFO [2019-12-17 12:30:39,656] jepsen worker 65 - jepsen.util 3215 :invoke :cas [21 [44 5]] INFO [2019-12-17 12:30:39,664] jepsen worker 55 - jepsen.util 4555 :fail :cas [18 [46 40]] :ValueNotSwapFrom INFO [2019-12-17 12:30:39,664] jepsen worker 55 - jepsen.util 4555 :invoke :read [18 nil] INFO [2019-12-17 12:30:39,667] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21450,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:30:39,668] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21450,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:30:39,669] jepsen worker 9 - jepsen.util 4239 :ok :write [3 37] INFO [2019-12-17 12:30:39,670] jepsen worker 9 - jepsen.util 4239 :invoke :write [3 22] INFO [2019-12-17 12:30:39,674] jepsen worker 28 - jepsen.util 2638 :ok :read [9 21] INFO [2019-12-17 12:30:39,674] jepsen worker 28 - jepsen.util 2638 :invoke :write [9 0] INFO [2019-12-17 12:30:39,686] jepsen worker 11 - jepsen.util 4061 :info :write [3 19] :RequestCanceledException INFO [2019-12-17 12:30:39,688] jepsen worker 11 - jepsen.util 4151 :invoke :read [3 nil] INFO [2019-12-17 12:30:39,703] jepsen worker 53 - jepsen.util 3833 :ok :write [17 8] INFO [2019-12-17 12:30:39,703] jepsen worker 53 - jepsen.util 3833 :invoke :write [17 20] INFO [2019-12-17 12:30:39,706] jepsen worker 65 - jepsen.util 3215 :fail :cas [21 [44 5]] :ValueNotSwapFrom INFO [2019-12-17 12:30:39,706] jepsen worker 65 - jepsen.util 3215 :invoke :write [21 25] INFO [2019-12-17 12:30:39,722] jepsen worker 81 - jepsen.util 4941 :info :cas [27 [33 18]] :RequestCanceledException INFO [2019-12-17 12:30:39,726] jepsen worker 81 - jepsen.util 5031 :invoke :cas [27 [26 40]] INFO [2019-12-17 12:30:39,734] jepsen worker 52 - jepsen.util 3832 :fail :cas [17 [14 1]] :ValueNotSwapFrom INFO [2019-12-17 12:30:39,734] jepsen worker 52 - jepsen.util 3832 :invoke :read [17 nil] INFO [2019-12-17 12:30:39,757] jepsen worker 11 - jepsen.util 4151 :ok :read [3 37] INFO [2019-12-17 12:30:39,758] jepsen worker 11 - jepsen.util 4151 :invoke :cas [3 [41 13]] INFO [2019-12-17 12:30:39,777] jepsen worker 38 - jepsen.util 3728 :fail :cas [12 [22 13]] :ValueNotSwapFrom INFO [2019-12-17 12:30:39,778] jepsen worker 38 - jepsen.util 3728 :invoke :read [12 nil] INFO [2019-12-17 12:30:39,801] jepsen worker 11 - jepsen.util 4151 :fail :cas [3 [41 13]] :ValueNotSwapFrom INFO [2019-12-17 12:30:39,801] jepsen worker 11 - jepsen.util 4151 :invoke :write [3 15] INFO [2019-12-17 12:30:39,809] jepsen worker 65 - jepsen.util 3215 :ok :write [21 25] INFO [2019-12-17 12:30:39,809] jepsen worker 65 - jepsen.util 3215 :invoke :read [21 nil] INFO [2019-12-17 12:30:39,824] jepsen worker 21 - jepsen.util 4881 :info :cas [7 [32 46]] :RequestCanceledException INFO [2019-12-17 12:30:39,826] jepsen worker 21 - jepsen.util 4971 :invoke :read [7 nil] INFO [2019-12-17 12:30:39,855] jepsen worker 65 - jepsen.util 3215 :ok :read [21 25] INFO [2019-12-17 12:30:39,855] jepsen worker 65 - jepsen.util 3215 :invoke :write [21 7] INFO [2019-12-17 12:30:39,862] jepsen worker 3 - jepsen.util 3963 :fail :cas [1 [36 24]] :ValueNotSwapFrom INFO [2019-12-17 12:30:39,862] jepsen worker 3 - jepsen.util 3963 :invoke :write [1 23] INFO [2019-12-17 12:30:39,880] jepsen worker 13 - jepsen.util 3523 :fail :cas [4 [39 27]] :ValueNotSwapFrom INFO [2019-12-17 12:30:39,880] jepsen worker 13 - jepsen.util 3523 :invoke :cas [4 [38 13]] INFO [2019-12-17 12:30:39,885] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21469,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:30:39,885] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21469,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:30:39,885] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21473,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:30:39,885] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21473,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:30:39,892] jepsen worker 56 - jepsen.util 3836 :fail :cas [18 [18 35]] :ValueNotSwapFrom INFO [2019-12-17 12:30:39,892] jepsen worker 56 - jepsen.util 3836 :invoke :write [18 4] INFO [2019-12-17 12:30:39,914] jepsen worker 28 - jepsen.util 2638 :ok :write [9 0] INFO [2019-12-17 12:30:39,914] jepsen worker 28 - jepsen.util 2638 :invoke :cas [9 [37 43]] INFO [2019-12-17 12:30:39,929] jepsen worker 70 - jepsen.util 4300 :ok :read [23 47] INFO [2019-12-17 12:30:39,929] jepsen worker 70 - jepsen.util 4300 :invoke :write [23 10] INFO [2019-12-17 12:30:39,940] jepsen worker 28 - jepsen.util 2638 :fail :cas [9 [37 43]] :ValueNotSwapFrom INFO [2019-12-17 12:30:39,940] jepsen worker 28 - jepsen.util 2638 :invoke :write [9 48] INFO [2019-12-17 12:30:39,993] jepsen worker 19 - jepsen.util 3889 :info :cas [6 [33 12]] :RequestCanceledException INFO [2019-12-17 12:30:39,996] jepsen worker 12 - jepsen.util 4062 :ok :write [4 45] INFO [2019-12-17 12:30:39,996] jepsen worker 12 - jepsen.util 4062 :invoke :cas [4 [35 47]] INFO [2019-12-17 12:30:39,999] jepsen worker 19 - jepsen.util 3979 :invoke :write [6 38] INFO [2019-12-17 12:30:40,011] jepsen worker 65 - jepsen.util 3215 :ok :write [21 7] INFO [2019-12-17 12:30:40,011] jepsen worker 65 - jepsen.util 3215 :invoke :write [21 25] INFO [2019-12-17 12:30:40,012] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21483,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:30:40,012] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21483,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:30:40,026] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:30:40,026] jepsen nemesis - couchbase.util Rebalance check count: 159 INFO [2019-12-17 12:30:40,026] jepsen nemesis - couchbase.util Rebalance stuck count: 140 INFO [2019-12-17 12:30:40,027] jepsen worker 28 - jepsen.util 2638 :ok :write [9 48] INFO [2019-12-17 12:30:40,027] jepsen worker 28 - jepsen.util 2638 :invoke :cas [9 [36 35]] INFO [2019-12-17 12:30:40,031] jepsen worker 37 - jepsen.util 3547 :ok :read [12 1] INFO [2019-12-17 12:30:40,031] jepsen worker 37 - jepsen.util 3547 :invoke :cas [12 [3 19]] INFO [2019-12-17 12:30:40,038] jepsen worker 61 - jepsen.util 5641 :info :write [20 8] :RequestCanceledException INFO [2019-12-17 12:30:40,039] jepsen worker 61 - jepsen.util 5731 :invoke :write [20 0] INFO [2019-12-17 12:30:40,044] jepsen worker 46 - jepsen.util 3916 :ok :write [15 47] INFO [2019-12-17 12:30:40,044] jepsen worker 46 - jepsen.util 3916 :invoke :write [15 10] INFO [2019-12-17 12:30:40,049] jepsen worker 9 - jepsen.util 4239 :ok :write [3 22] INFO [2019-12-17 12:30:40,049] jepsen worker 9 - jepsen.util 4239 :invoke :cas [3 [3 24]] INFO [2019-12-17 12:30:40,051] jepsen worker 28 - jepsen.util 2638 :fail :cas [9 [36 35]] :ValueNotSwapFrom INFO [2019-12-17 12:30:40,051] jepsen worker 28 - jepsen.util 2638 :invoke :cas [9 [16 39]] INFO [2019-12-17 12:30:40,053] jepsen worker 14 - jepsen.util 3794 :fail :cas [4 [35 7]] :ValueNotSwapFrom INFO [2019-12-17 12:30:40,053] jepsen worker 14 - jepsen.util 3794 :invoke :read [4 nil] INFO [2019-12-17 12:30:40,083] jepsen worker 28 - jepsen.util 2638 :fail :cas [9 [16 39]] :ValueNotSwapFrom INFO [2019-12-17 12:30:40,083] jepsen worker 28 - jepsen.util 2638 :invoke :write [9 49] INFO [2019-12-17 12:30:40,091] jepsen worker 74 - jepsen.util 4034 :info :cas [24 [39 44]] :RequestCanceledException INFO [2019-12-17 12:30:40,093] jepsen worker 52 - jepsen.util 3832 :ok :read [17 8] INFO [2019-12-17 12:30:40,093] jepsen worker 52 - jepsen.util 3832 :invoke :read [17 nil] INFO [2019-12-17 12:30:40,095] jepsen worker 74 - jepsen.util 4124 :invoke :write [24 25] INFO [2019-12-17 12:30:40,110] jepsen worker 65 - jepsen.util 3215 :ok :write [21 25] INFO [2019-12-17 12:30:40,110] jepsen worker 65 - jepsen.util 3215 :invoke :write [21 35] INFO [2019-12-17 12:30:40,116] jepsen worker 38 - jepsen.util 3728 :ok :read [12 1] INFO [2019-12-17 12:30:40,116] jepsen worker 38 - jepsen.util 3728 :invoke :write [12 18] INFO [2019-12-17 12:30:40,139] jepsen worker 4 - jepsen.util 2974 :ok :read [1 45] INFO [2019-12-17 12:30:40,139] jepsen worker 4 - jepsen.util 2974 :invoke :write [1 7] INFO [2019-12-17 12:30:40,140] jepsen worker 86 - jepsen.util 5126 :info :write [28 1] :RequestCanceledException INFO [2019-12-17 12:30:40,141] jepsen worker 86 - jepsen.util 5216 :invoke :write [28 11] INFO [2019-12-17 12:30:40,155] jepsen worker 79 - jepsen.util 3769 :ok :read [26 28] INFO [2019-12-17 12:30:40,155] jepsen worker 79 - jepsen.util 3769 :invoke :read [26 nil] INFO [2019-12-17 12:30:40,158] jepsen worker 14 - jepsen.util 3794 :ok :read [4 45] INFO [2019-12-17 12:30:40,158] jepsen worker 14 - jepsen.util 3794 :invoke :cas [4 [15 25]] INFO [2019-12-17 12:30:40,177] jepsen worker 65 - jepsen.util 3215 :ok :write [21 35] INFO [2019-12-17 12:30:40,177] jepsen worker 65 - jepsen.util 3215 :invoke :cas [21 [11 15]] INFO [2019-12-17 12:30:40,188] jepsen worker 28 - jepsen.util 2638 :ok :write [9 49] INFO [2019-12-17 12:30:40,188] jepsen worker 28 - jepsen.util 2638 :invoke :cas [9 [43 29]] INFO [2019-12-17 12:30:40,202] jepsen worker 65 - jepsen.util 3215 :fail :cas [21 [11 15]] :ValueNotSwapFrom INFO [2019-12-17 12:30:40,202] jepsen worker 65 - jepsen.util 3215 :invoke :read [21 nil] INFO [2019-12-17 12:30:40,215] jepsen worker 36 - jepsen.util 3906 :ok :read [12 1] INFO [2019-12-17 12:30:40,215] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21508,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:30:40,215] jepsen worker 36 - jepsen.util 3906 :invoke :write [12 37] INFO [2019-12-17 12:30:40,215] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21508,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:30:40,215] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21510,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:30:40,215] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21510,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:30:40,223] jepsen worker 28 - jepsen.util 2638 :fail :cas [9 [43 29]] :ValueNotSwapFrom INFO [2019-12-17 12:30:40,223] jepsen worker 28 - jepsen.util 2638 :invoke :write [9 37] INFO [2019-12-17 12:30:40,224] jepsen worker 24 - jepsen.util 4794 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:30:40,224] jepsen worker 24 - jepsen.util 4794 :invoke :read [8 nil] INFO [2019-12-17 12:30:40,226] jepsen worker 77 - jepsen.util 4937 :info :cas [25 [8 25]] :RequestCanceledException INFO [2019-12-17 12:30:40,227] jepsen worker 77 - jepsen.util 5027 :invoke :read [25 nil] INFO [2019-12-17 12:30:40,230] jepsen worker 87 - jepsen.util 3507 :fail :cas [29 [25 14]] :ValueNotSwapFrom INFO [2019-12-17 12:30:40,231] jepsen worker 87 - jepsen.util 3507 :invoke :write [29 0] INFO [2019-12-17 12:30:40,231] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21516,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:30:40,232] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21516,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:30:40,232] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21519,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:30:40,232] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21519,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:30:40,232] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21520,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:30:40,232] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21520,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:30:40,238] jepsen worker 65 - jepsen.util 3215 :ok :read [21 35] INFO [2019-12-17 12:30:40,238] jepsen worker 65 - jepsen.util 3215 :invoke :write [21 34] INFO [2019-12-17 12:30:40,257] jepsen worker 51 - jepsen.util 3291 :fail :cas [17 [29 29]] :ValueNotSwapFrom INFO [2019-12-17 12:30:40,257] jepsen worker 51 - jepsen.util 3291 :invoke :cas [17 [4 0]] INFO [2019-12-17 12:30:40,272] jepsen worker 19 - jepsen.util 3979 :ok :write [6 38] INFO [2019-12-17 12:30:40,273] jepsen worker 19 - jepsen.util 3979 :invoke :cas [6 [23 46]] INFO [2019-12-17 12:30:40,323] jepsen worker 19 - jepsen.util 3979 :fail :cas [6 [23 46]] :ValueNotSwapFrom INFO [2019-12-17 12:30:40,323] jepsen worker 19 - jepsen.util 3979 :invoke :write [6 14] INFO [2019-12-17 12:30:40,325] jepsen worker 49 - jepsen.util 4369 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:30:40,325] jepsen worker 49 - jepsen.util 4369 :invoke :cas [16 [14 44]] INFO [2019-12-17 12:30:40,330] jepsen worker 28 - jepsen.util 2638 :ok :write [9 37] INFO [2019-12-17 12:30:40,330] jepsen worker 28 - jepsen.util 2638 :invoke :cas [9 [15 7]] INFO [2019-12-17 12:30:40,340] jepsen worker 55 - jepsen.util 4555 :ok :read [18 48] INFO [2019-12-17 12:30:40,340] jepsen worker 55 - jepsen.util 4555 :invoke :read [18 nil] INFO [2019-12-17 12:30:40,346] jepsen worker 65 - jepsen.util 3215 :ok :write [21 34] INFO [2019-12-17 12:30:40,346] jepsen worker 65 - jepsen.util 3215 :invoke :write [21 4] INFO [2019-12-17 12:30:40,357] jepsen worker 34 - jepsen.util 4984 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:30:40,357] jepsen worker 34 - jepsen.util 4984 :invoke :read [11 nil] INFO [2019-12-17 12:30:40,381] jepsen worker 28 - jepsen.util 2638 :fail :cas [9 [15 7]] :ValueNotSwapFrom INFO [2019-12-17 12:30:40,381] jepsen worker 28 - jepsen.util 2638 :invoke :read [9 nil] INFO [2019-12-17 12:30:40,430] jepsen worker 65 - jepsen.util 3215 :ok :write [21 4] INFO [2019-12-17 12:30:40,430] jepsen worker 65 - jepsen.util 3215 :invoke :read [21 nil] INFO [2019-12-17 12:30:40,445] jepsen worker 28 - jepsen.util 2638 :ok :read [9 37] INFO [2019-12-17 12:30:40,446] jepsen worker 28 - jepsen.util 2638 :invoke :cas [9 [17 41]] INFO [2019-12-17 12:30:40,457] jepsen worker 14 - jepsen.util 3794 :fail :cas [4 [15 25]] :ValueNotSwapFrom INFO [2019-12-17 12:30:40,457] jepsen worker 14 - jepsen.util 3794 :invoke :cas [4 [38 6]] INFO [2019-12-17 12:30:40,471] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21537,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:30:40,471] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21537,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:30:40,471] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21541,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:30:40,471] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21541,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:30:40,493] jepsen worker 65 - jepsen.util 3215 :ok :read [21 4] INFO [2019-12-17 12:30:40,493] jepsen worker 65 - jepsen.util 3215 :invoke :cas [21 [47 31]] INFO [2019-12-17 12:30:40,525] jepsen worker 28 - jepsen.util 2638 :fail :cas [9 [17 41]] :ValueNotSwapFrom INFO [2019-12-17 12:30:40,525] jepsen worker 28 - jepsen.util 2638 :invoke :cas [9 [20 21]] INFO [2019-12-17 12:30:40,529] jepsen worker 55 - jepsen.util 4555 :ok :read [18 48] INFO [2019-12-17 12:30:40,529] jepsen worker 55 - jepsen.util 4555 :invoke :write [18 44] INFO [2019-12-17 12:30:40,550] jepsen worker 65 - jepsen.util 3215 :fail :cas [21 [47 31]] :ValueNotSwapFrom INFO [2019-12-17 12:30:40,551] jepsen worker 65 - jepsen.util 3215 :invoke :cas [21 [31 0]] INFO [2019-12-17 12:30:40,618] jepsen worker 28 - jepsen.util 2638 :fail :cas [9 [20 21]] :ValueNotSwapFrom INFO [2019-12-17 12:30:40,618] jepsen worker 28 - jepsen.util 2638 :invoke :write [9 14] INFO [2019-12-17 12:30:40,620] jepsen worker 52 - jepsen.util 3832 :ok :read [17 8] INFO [2019-12-17 12:30:40,620] jepsen worker 52 - jepsen.util 3832 :invoke :write [17 35] INFO [2019-12-17 12:30:40,639] jepsen worker 23 - jepsen.util 4973 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:30:40,639] jepsen worker 23 - jepsen.util 4973 :invoke :cas [7 [5 3]] INFO [2019-12-17 12:30:40,640] jepsen worker 65 - jepsen.util 3215 :fail :cas [21 [31 0]] :ValueNotSwapFrom INFO [2019-12-17 12:30:40,641] jepsen worker 65 - jepsen.util 3215 :invoke :cas [21 [7 10]] INFO [2019-12-17 12:30:40,655] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21552,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:30:40,655] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21552,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:30:40,655] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21556,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:30:40,656] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21556,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:30:40,691] jepsen worker 68 - jepsen.util 4568 :info :write [22 38] :RequestCanceledException INFO [2019-12-17 12:30:40,695] jepsen worker 37 - jepsen.util 3547 :fail :cas [12 [3 19]] :ValueNotSwapFrom INFO [2019-12-17 12:30:40,695] jepsen worker 37 - jepsen.util 3547 :invoke :read [12 nil] INFO [2019-12-17 12:30:40,695] jepsen worker 68 - jepsen.util 4658 :invoke :read [22 nil] INFO [2019-12-17 12:30:40,712] jepsen worker 13 - jepsen.util 3523 :fail :cas [4 [38 13]] :ValueNotSwapFrom INFO [2019-12-17 12:30:40,712] jepsen worker 13 - jepsen.util 3523 :invoke :cas [4 [38 30]] INFO [2019-12-17 12:30:40,721] jepsen worker 12 - jepsen.util 4062 :fail :cas [4 [35 47]] :ValueNotSwapFrom INFO [2019-12-17 12:30:40,721] jepsen worker 12 - jepsen.util 4062 :invoke :read [4 nil] INFO [2019-12-17 12:30:40,724] jepsen worker 44 - jepsen.util 5354 :info :write [14 11] :RequestCanceledException INFO [2019-12-17 12:30:40,728] jepsen worker 44 - jepsen.util 5444 :invoke :read [14 nil] INFO [2019-12-17 12:30:40,741] jepsen worker 14 - jepsen.util 3794 :fail :cas [4 [38 6]] :ValueNotSwapFrom INFO [2019-12-17 12:30:40,741] jepsen worker 14 - jepsen.util 3794 :invoke :read [4 nil] INFO [2019-12-17 12:30:40,753] jepsen worker 59 - jepsen.util 4919 :info :write [19 35] :RequestCanceledException INFO [2019-12-17 12:30:40,756] jepsen worker 59 - jepsen.util 5009 :invoke :cas [19 [6 21]] INFO [2019-12-17 12:30:40,757] jepsen worker 65 - jepsen.util 3215 :fail :cas [21 [7 10]] :ValueNotSwapFrom INFO [2019-12-17 12:30:40,757] jepsen worker 65 - jepsen.util 3215 :invoke :write [21 26] INFO [2019-12-17 12:30:40,768] jepsen worker 19 - jepsen.util 3979 :ok :write [6 14] INFO [2019-12-17 12:30:40,768] jepsen worker 19 - jepsen.util 3979 :invoke :cas [6 [20 5]] INFO [2019-12-17 12:30:40,769] jepsen worker 46 - jepsen.util 3916 :ok :write [15 10] INFO [2019-12-17 12:30:40,769] jepsen worker 46 - jepsen.util 3916 :invoke :cas [15 [0 40]] INFO [2019-12-17 12:30:40,789] jepsen worker 79 - jepsen.util 3769 :ok :read [26 28] INFO [2019-12-17 12:30:40,789] jepsen worker 79 - jepsen.util 3769 :invoke :write [26 21] INFO [2019-12-17 12:30:40,791] jepsen worker 73 - jepsen.util 3493 :info :write [24 3] :RequestCanceledException INFO [2019-12-17 12:30:40,791] jepsen worker 20 - jepsen.util 3530 :info :write [6 40] :RequestCanceledException INFO [2019-12-17 12:30:40,794] jepsen worker 73 - jepsen.util 3583 :invoke :write [24 41] INFO [2019-12-17 12:30:40,795] jepsen worker 20 - jepsen.util 3620 :invoke :cas [6 [0 33]] INFO [2019-12-17 12:30:40,808] jepsen worker 31 - jepsen.util 3271 :info :write [10 8] :RequestCanceledException INFO [2019-12-17 12:30:40,808] jepsen worker 64 - jepsen.util 3034 :fail :read [21 nil] :RequestTimeoutException INFO [2019-12-17 12:30:40,808] jepsen worker 43 - jepsen.util 4723 :info :write [14 2] :RequestCanceledException INFO [2019-12-17 12:30:40,809] jepsen worker 64 - jepsen.util 3034 :invoke :read [21 nil] INFO [2019-12-17 12:30:40,810] jepsen worker 31 - jepsen.util 3361 :invoke :cas [10 [28 29]] INFO [2019-12-17 12:30:40,811] jepsen worker 43 - jepsen.util 4813 :invoke :read [14 nil] INFO [2019-12-17 12:30:40,823] jepsen worker 72 - jepsen.util 3852 :info :write [24 6] :RequestCanceledException INFO [2019-12-17 12:30:40,825] jepsen worker 47 - jepsen.util 4187 :info :write [15 25] :RequestCanceledException INFO [2019-12-17 12:30:40,826] jepsen worker 72 - jepsen.util 3942 :invoke :write [24 48] INFO [2019-12-17 12:30:40,827] jepsen worker 10 - jepsen.util 3520 :info :write [3 44] :RequestCanceledException INFO [2019-12-17 12:30:40,830] jepsen worker 10 - jepsen.util 3610 :invoke :write [3 10] INFO [2019-12-17 12:30:40,831] jepsen worker 84 - jepsen.util 4584 :info :cas [28 [16 31]] :RequestCanceledException INFO [2019-12-17 12:30:40,831] jepsen worker 27 - jepsen.util 3537 :info :write [9 41] :RequestCanceledException INFO [2019-12-17 12:30:40,831] jepsen worker 66 - jepsen.util 4836 :info :cas [22 [33 3]] :RequestCanceledException INFO [2019-12-17 12:30:40,838] jepsen worker 27 - jepsen.util 3627 :invoke :cas [9 [21 41]] INFO [2019-12-17 12:30:40,842] jepsen worker 66 - jepsen.util 4926 :invoke :read [22 nil] INFO [2019-12-17 12:30:40,843] jepsen worker 28 - jepsen.util 2638 :ok :write [9 14] INFO [2019-12-17 12:30:40,844] jepsen worker 84 - jepsen.util 4674 :invoke :read [28 nil] INFO [2019-12-17 12:30:40,844] jepsen worker 28 - jepsen.util 2638 :invoke :read [9 nil] INFO [2019-12-17 12:30:40,849] jepsen worker 47 - jepsen.util 4277 :invoke :write [15 30] INFO [2019-12-17 12:30:40,855] jepsen worker 7 - jepsen.util 4777 :info :cas [2 [9 49]] :RequestCanceledException INFO [2019-12-17 12:30:40,857] jepsen worker 51 - jepsen.util 3291 :fail :cas [17 [4 0]] :ValueNotSwapFrom INFO [2019-12-17 12:30:40,858] jepsen worker 51 - jepsen.util 3291 :invoke :write [17 10] INFO [2019-12-17 12:30:40,867] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21584,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:30:40,868] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21584,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:30:40,869] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21585,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:30:40,869] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21585,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:30:40,891] jepsen worker 5 - jepsen.util 3335 :info :write [1 22] :RequestCanceledException INFO [2019-12-17 12:30:40,893] jepsen worker 45 - jepsen.util 3825 :info :write [15 9] :RequestCanceledException INFO [2019-12-17 12:30:40,894] jepsen worker 54 - jepsen.util 4014 :info :cas [18 [48 26]] :RequestCanceledException INFO [2019-12-17 12:30:40,899] jepsen worker 7 - jepsen.util 4867 :invoke :write [2 18] INFO [2019-12-17 12:30:40,910] jepsen worker 5 - jepsen.util 3425 :invoke :read [1 nil] INFO [2019-12-17 12:30:40,912] jepsen worker 45 - jepsen.util 3915 :invoke :cas [15 [15 10]] INFO [2019-12-17 12:30:40,918] jepsen worker 54 - jepsen.util 4104 :invoke :write [18 14] INFO [2019-12-17 12:30:40,922] jepsen worker 62 - jepsen.util 4742 :info :cas [20 [24 25]] :RequestCanceledException INFO [2019-12-17 12:30:40,923] jepsen worker 29 - jepsen.util 3089 :info :write [9 10] :RequestCanceledException INFO [2019-12-17 12:30:40,924] jepsen worker 25 - jepsen.util 5245 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:30:40,925] jepsen worker 25 - jepsen.util 5245 :invoke :cas [8 [9 6]] INFO [2019-12-17 12:30:40,926] jepsen worker 29 - jepsen.util 3179 :invoke :read [9 nil] INFO [2019-12-17 12:30:40,940] jepsen worker 18 - jepsen.util 3798 :info :write [6 35] :RequestCanceledException INFO [2019-12-17 12:30:40,949] jepsen worker 18 - jepsen.util 3888 :invoke :read [6 nil] INFO [2019-12-17 12:30:40,952] jepsen worker 39 - jepsen.util 4719 :info :cas [13 [39 37]] :RequestCanceledException INFO [2019-12-17 12:30:40,970] jepsen worker 62 - jepsen.util 4832 :invoke :read [20 nil] INFO [2019-12-17 12:30:40,992] jepsen worker 6 - jepsen.util 5406 :info :write [2 49] :RequestCanceledException INFO [2019-12-17 12:30:40,993] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21595,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:30:40,993] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21595,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:30:40,993] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21600,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:30:40,994] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21600,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:30:40,999] jepsen worker 39 - jepsen.util 4809 :invoke :read [13 nil] INFO [2019-12-17 12:30:41,007] jepsen worker 6 - jepsen.util 5496 :invoke :read [2 nil] INFO [2019-12-17 12:30:41,009] jepsen worker 20 - jepsen.util 3620 :fail :cas [6 [0 33]] :ValueNotSwapFrom INFO [2019-12-17 12:30:41,010] jepsen worker 20 - jepsen.util 3620 :invoke :write [6 3] INFO [2019-12-17 12:30:41,024] jepsen worker 82 - jepsen.util 5842 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:30:41,025] jepsen worker 82 - jepsen.util 5842 :invoke :cas [27 [11 35]] INFO [2019-12-17 12:30:41,029] jepsen worker 65 - jepsen.util 3215 :ok :write [21 26] INFO [2019-12-17 12:30:41,030] jepsen worker 65 - jepsen.util 3215 :invoke :write [21 26] INFO [2019-12-17 12:30:41,039] jepsen worker 32 - jepsen.util 4172 :fail :read [10 nil] :RequestTimeoutException INFO [2019-12-17 12:30:41,039] jepsen worker 32 - jepsen.util 4172 :invoke :write [10 12] INFO [2019-12-17 12:30:41,063] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:30:41,063] jepsen nemesis - couchbase.util Rebalance check count: 160 INFO [2019-12-17 12:30:41,064] jepsen nemesis - couchbase.util Rebalance stuck count: 141 INFO [2019-12-17 12:30:41,070] jepsen worker 19 - jepsen.util 3979 :fail :cas [6 [20 5]] :ValueNotSwapFrom INFO [2019-12-17 12:30:41,071] jepsen worker 19 - jepsen.util 3979 :invoke :cas [6 [39 4]] INFO [2019-12-17 12:30:41,107] jepsen worker 85 - jepsen.util 5575 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:30:41,107] jepsen worker 88 - jepsen.util 3778 :info :write [29 18] :RequestCanceledException INFO [2019-12-17 12:30:41,107] jepsen worker 85 - jepsen.util 5575 :invoke :cas [28 [30 29]] INFO [2019-12-17 12:30:41,112] jepsen worker 88 - jepsen.util 3868 :invoke :cas [29 [4 23]] INFO [2019-12-17 12:30:41,114] jepsen worker 64 - jepsen.util 3034 :ok :read [21 4] INFO [2019-12-17 12:30:41,115] jepsen worker 64 - jepsen.util 3034 :invoke :read [21 nil] INFO [2019-12-17 12:30:41,120] jepsen worker 46 - jepsen.util 3916 :fail :cas [15 [0 40]] :ValueNotSwapFrom INFO [2019-12-17 12:30:41,121] jepsen worker 46 - jepsen.util 3916 :invoke :cas [15 [13 27]] INFO [2019-12-17 12:30:41,122] jepsen worker 31 - jepsen.util 3361 :fail :cas [10 [28 29]] :ValueNotSwapFrom INFO [2019-12-17 12:30:41,122] jepsen worker 31 - jepsen.util 3361 :invoke :cas [10 [39 27]] INFO [2019-12-17 12:30:41,133] jepsen worker 29 - jepsen.util 3179 :ok :read [9 14] INFO [2019-12-17 12:30:41,134] jepsen worker 29 - jepsen.util 3179 :invoke :write [9 39] INFO [2019-12-17 12:30:41,138] jepsen worker 50 - jepsen.util 5450 :info :write [16 11] :RequestCanceledException INFO [2019-12-17 12:30:41,138] jepsen worker 30 - jepsen.util 4170 :fail :read [10 nil] :RequestTimeoutException INFO [2019-12-17 12:30:41,138] jepsen worker 30 - jepsen.util 4170 :invoke :cas [10 [5 5]] INFO [2019-12-17 12:30:41,140] jepsen worker 50 - jepsen.util 5540 :invoke :write [16 40] INFO [2019-12-17 12:30:41,161] jepsen worker 46 - jepsen.util 3916 :fail :cas [15 [13 27]] :ValueNotSwapFrom INFO [2019-12-17 12:30:41,161] jepsen worker 46 - jepsen.util 3916 :invoke :write [15 13] INFO [2019-12-17 12:30:41,181] jepsen worker 31 - jepsen.util 3361 :fail :cas [10 [39 27]] :ValueNotSwapFrom INFO [2019-12-17 12:30:41,181] jepsen worker 31 - jepsen.util 3361 :invoke :read [10 nil] INFO [2019-12-17 12:30:41,197] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21621,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:30:41,197] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21621,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:30:41,197] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21622,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:30:41,197] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21622,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:30:41,211] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21627,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:30:41,211] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21627,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:30:41,211] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21630,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:30:41,211] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21630,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:30:41,224] jepsen worker 80 - jepsen.util 3770 :info :write [26 45] :RequestCanceledException INFO [2019-12-17 12:30:41,226] jepsen worker 80 - jepsen.util 3860 :invoke :read [26 nil] INFO [2019-12-17 12:30:41,229] jepsen worker 29 - jepsen.util 3179 :ok :write [9 39] INFO [2019-12-17 12:30:41,230] jepsen worker 29 - jepsen.util 3179 :invoke :write [9 2] INFO [2019-12-17 12:30:41,255] jepsen worker 69 - jepsen.util 3669 :info :write [23 6] :RequestCanceledException INFO [2019-12-17 12:30:41,255] jepsen worker 83 - jepsen.util 5123 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:30:41,255] jepsen worker 83 - jepsen.util 5123 :invoke :read [27 nil] INFO [2019-12-17 12:30:41,257] jepsen worker 14 - jepsen.util 3794 :ok :read [4 45] INFO [2019-12-17 12:30:41,257] jepsen worker 14 - jepsen.util 3794 :invoke :write [4 42] INFO [2019-12-17 12:30:41,257] jepsen worker 69 - jepsen.util 3759 :invoke :read [23 nil] INFO [2019-12-17 12:30:41,291] jepsen worker 48 - jepsen.util 4548 :info :write [16 28] :RequestCanceledException INFO [2019-12-17 12:30:41,292] jepsen worker 48 - jepsen.util 4638 :invoke :read [16 nil] INFO [2019-12-17 12:30:41,309] jepsen worker 26 - jepsen.util 5336 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:30:41,309] jepsen worker 26 - jepsen.util 5336 :invoke :read [8 nil] INFO [2019-12-17 12:30:41,312] jepsen worker 63 - jepsen.util 3213 :info :write [21 28] :RequestCanceledException INFO [2019-12-17 12:30:41,316] jepsen worker 63 - jepsen.util 3303 :invoke :cas [21 [27 37]] INFO [2019-12-17 12:30:41,322] jepsen worker 29 - jepsen.util 3179 :ok :write [9 2] INFO [2019-12-17 12:30:41,323] jepsen worker 29 - jepsen.util 3179 :invoke :write [9 0] INFO [2019-12-17 12:30:41,324] jepsen worker 71 - jepsen.util 3851 :info :write [23 39] :RequestCanceledException INFO [2019-12-17 12:30:41,324] jepsen worker 67 - jepsen.util 5287 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:30:41,324] jepsen worker 35 - jepsen.util 4895 :info :cas [11 [36 38]] :RequestCanceledException INFO [2019-12-17 12:30:41,324] jepsen worker 22 - jepsen.util 5152 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:30:41,324] jepsen worker 40 - jepsen.util 5350 :info :write [13 49] :RequestCanceledException INFO [2019-12-17 12:30:41,324] jepsen worker 67 - jepsen.util 5287 :invoke :write [22 14] INFO [2019-12-17 12:30:41,324] jepsen worker 22 - jepsen.util 5152 :invoke :cas [7 [16 5]] INFO [2019-12-17 12:30:41,326] jepsen worker 71 - jepsen.util 3941 :invoke :write [23 24] INFO [2019-12-17 12:30:41,334] jepsen worker 40 - jepsen.util 5440 :invoke :cas [13 [10 4]] INFO [2019-12-17 12:30:41,334] jepsen worker 35 - jepsen.util 4985 :invoke :write [11 6] INFO [2019-12-17 12:30:41,338] jepsen worker 57 - jepsen.util 4737 :info :write [19 35] :RequestCanceledException INFO [2019-12-17 12:30:41,339] jepsen worker 0 - jepsen.util 5400 :info :cas [0 [36 13]] :RequestCanceledException INFO [2019-12-17 12:30:41,344] jepsen worker 0 - jepsen.util 5490 :invoke :write [0 21] INFO [2019-12-17 12:30:41,344] jepsen worker 73 - jepsen.util 3583 :ok :write [24 41] INFO [2019-12-17 12:30:41,345] jepsen worker 73 - jepsen.util 3583 :invoke :read [24 nil] INFO [2019-12-17 12:30:41,346] jepsen worker 57 - jepsen.util 4827 :invoke :write [19 47] INFO [2019-12-17 12:30:41,356] jepsen worker 60 - jepsen.util 5280 :info :write [20 34] :RequestCanceledException INFO [2019-12-17 12:30:41,363] jepsen worker 60 - jepsen.util 5370 :invoke :write [20 19] INFO [2019-12-17 12:30:41,391] jepsen worker 58 - jepsen.util 5098 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:30:41,391] jepsen worker 89 - jepsen.util 3509 :info :write [29 23] :RequestCanceledException INFO [2019-12-17 12:30:41,391] jepsen worker 58 - jepsen.util 5098 :invoke :read [19 nil] INFO [2019-12-17 12:30:41,393] jepsen worker 89 - jepsen.util 3599 :invoke :write [29 41] INFO [2019-12-17 12:30:41,408] jepsen worker 2 - jepsen.util 4682 :info :cas [0 [4 18]] :RequestCanceledException INFO [2019-12-17 12:30:41,408] jepsen worker 17 - jepsen.util 4697 :info :cas [5 [24 6]] :RequestCanceledException INFO [2019-12-17 12:30:41,411] jepsen worker 2 - jepsen.util 4772 :invoke :write [0 47] INFO [2019-12-17 12:30:41,412] jepsen worker 17 - jepsen.util 4787 :invoke :cas [5 [4 19]] INFO [2019-12-17 12:30:41,424] jepsen worker 33 - jepsen.util 5253 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:30:41,424] jepsen worker 33 - jepsen.util 5253 :invoke :write [11 1] INFO [2019-12-17 12:30:41,424] jepsen worker 42 - jepsen.util 5892 :info :write [14 42] :RequestCanceledException INFO [2019-12-17 12:30:41,430] jepsen worker 63 - jepsen.util 3303 :fail :cas [21 [27 37]] :ValueNotSwapFrom INFO [2019-12-17 12:30:41,430] jepsen worker 63 - jepsen.util 3303 :invoke :read [21 nil] INFO [2019-12-17 12:30:41,430] jepsen worker 42 - jepsen.util 5982 :invoke :read [14 nil] INFO [2019-12-17 12:30:41,443] jepsen worker 45 - jepsen.util 3915 :fail :cas [15 [15 10]] :ValueNotSwapFrom INFO [2019-12-17 12:30:41,444] jepsen worker 45 - jepsen.util 3915 :invoke :cas [15 [38 16]] INFO [2019-12-17 12:30:41,445] jepsen worker 73 - jepsen.util 3583 :ok :read [24 41] INFO [2019-12-17 12:30:41,445] jepsen worker 73 - jepsen.util 3583 :invoke :write [24 6] INFO [2019-12-17 12:30:41,447] jepsen worker 5 - jepsen.util 3425 :ok :read [1 45] INFO [2019-12-17 12:30:41,447] jepsen worker 5 - jepsen.util 3425 :invoke :cas [1 [3 10]] INFO [2019-12-17 12:30:41,456] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21656,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:30:41,456] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21656,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:30:41,506] jepsen worker 29 - jepsen.util 3179 :ok :write [9 0] INFO [2019-12-17 12:30:41,506] jepsen worker 29 - jepsen.util 3179 :invoke :write [9 12] INFO [2019-12-17 12:30:41,511] jepsen worker 47 - jepsen.util 4277 :ok :write [15 30] INFO [2019-12-17 12:30:41,511] jepsen worker 47 - jepsen.util 4277 :invoke :cas [15 [10 32]] INFO [2019-12-17 12:30:41,538] jepsen worker 75 - jepsen.util 5025 :info :write [25 21] :RequestCanceledException INFO [2019-12-17 12:30:41,542] jepsen worker 75 - jepsen.util 5115 :invoke :write [25 31] INFO [2019-12-17 12:30:41,547] jepsen worker 63 - jepsen.util 3303 :ok :read [21 26] INFO [2019-12-17 12:30:41,547] jepsen worker 63 - jepsen.util 3303 :invoke :read [21 nil] INFO [2019-12-17 12:30:41,556] jepsen worker 1 - jepsen.util 4681 :info :write [0 1] :RequestCanceledException INFO [2019-12-17 12:30:41,562] jepsen worker 1 - jepsen.util 4771 :invoke :cas [0 [5 11]] INFO [2019-12-17 12:30:41,582] jepsen worker 18 - jepsen.util 3888 :ok :read [6 14] INFO [2019-12-17 12:30:41,583] jepsen worker 18 - jepsen.util 3888 :invoke :write [6 32] INFO [2019-12-17 12:30:41,586] jepsen worker 45 - jepsen.util 3915 :fail :cas [15 [38 16]] :ValueNotSwapFrom INFO [2019-12-17 12:30:41,586] jepsen worker 45 - jepsen.util 3915 :invoke :write [15 41] INFO [2019-12-17 12:30:41,589] jepsen worker 41 - jepsen.util 5171 :info :write [13 31] :RequestCanceledException INFO [2019-12-17 12:30:41,592] jepsen worker 64 - jepsen.util 3034 :ok :read [21 26] INFO [2019-12-17 12:30:41,592] jepsen worker 64 - jepsen.util 3034 :invoke :write [21 27] INFO [2019-12-17 12:30:41,595] jepsen worker 41 - jepsen.util 5261 :invoke :read [13 nil] INFO [2019-12-17 12:30:41,598] jepsen worker 47 - jepsen.util 4277 :fail :cas [15 [10 32]] :ValueNotSwapFrom INFO [2019-12-17 12:30:41,598] jepsen worker 47 - jepsen.util 4277 :invoke :write [15 5] INFO [2019-12-17 12:30:41,606] jepsen worker 16 - jepsen.util 5686 :info :cas [5 [9 35]] :RequestCanceledException INFO [2019-12-17 12:30:41,612] jepsen worker 16 - jepsen.util 5776 :invoke :cas [5 [8 16]] INFO [2019-12-17 12:30:41,624] jepsen worker 29 - jepsen.util 3179 :ok :write [9 12] INFO [2019-12-17 12:30:41,624] jepsen worker 29 - jepsen.util 3179 :invoke :read [9 nil] INFO [2019-12-17 12:30:41,624] jepsen worker 8 - jepsen.util 4328 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:30:41,624] jepsen worker 76 - jepsen.util 5116 :info :cas [25 [30 18]] :RequestCanceledException INFO [2019-12-17 12:30:41,624] jepsen worker 8 - jepsen.util 4328 :invoke :read [2 nil] INFO [2019-12-17 12:30:41,626] jepsen worker 76 - jepsen.util 5206 :invoke :read [25 nil] INFO [2019-12-17 12:30:41,638] jepsen worker 15 - jepsen.util 4515 :info :cas [5 [38 8]] :RequestCanceledException INFO [2019-12-17 12:30:41,644] jepsen worker 15 - jepsen.util 4605 :invoke :cas [5 [24 18]] INFO [2019-12-17 12:30:41,658] jepsen worker 29 - jepsen.util 3179 :ok :read [9 12] INFO [2019-12-17 12:30:41,658] jepsen worker 29 - jepsen.util 3179 :invoke :cas [9 [28 33]] INFO [2019-12-17 12:30:41,700] jepsen worker 29 - jepsen.util 3179 :fail :cas [9 [28 33]] :ValueNotSwapFrom INFO [2019-12-17 12:30:41,701] jepsen worker 29 - jepsen.util 3179 :invoke :cas [9 [17 7]] INFO [2019-12-17 12:30:41,706] jepsen worker 80 - jepsen.util 3860 :ok :read [26 28] INFO [2019-12-17 12:30:41,707] jepsen worker 80 - jepsen.util 3860 :invoke :write [26 5] INFO [2019-12-17 12:30:41,709] jepsen worker 5 - jepsen.util 3425 :fail :cas [1 [3 10]] :ValueNotSwapFrom INFO [2019-12-17 12:30:41,709] jepsen worker 5 - jepsen.util 3425 :invoke :write [1 48] INFO [2019-12-17 12:30:41,737] jepsen worker 29 - jepsen.util 3179 :fail :cas [9 [17 7]] :ValueNotSwapFrom INFO [2019-12-17 12:30:41,737] jepsen worker 29 - jepsen.util 3179 :invoke :read [9 nil] INFO [2019-12-17 12:30:41,746] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21683,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:30:41,746] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21683,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:30:41,746] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21684,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:30:41,746] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21684,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:30:41,746] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21685,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:30:41,746] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21685,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:30:41,754] jepsen worker 78 - jepsen.util 4218 :info :write [26 45] :RequestCanceledException INFO [2019-12-17 12:30:41,754] jepsen worker 53 - jepsen.util 3833 :info :write [17 20] :RequestCanceledException INFO [2019-12-17 12:30:41,756] jepsen worker 78 - jepsen.util 4308 :invoke :read [26 nil] INFO [2019-12-17 12:30:41,756] jepsen worker 53 - jepsen.util 3923 :invoke :cas [17 [15 6]] INFO [2019-12-17 12:30:41,764] jepsen worker 64 - jepsen.util 3034 :ok :write [21 27] INFO [2019-12-17 12:30:41,764] jepsen worker 64 - jepsen.util 3034 :invoke :cas [21 [38 19]] INFO [2019-12-17 12:30:41,769] jepsen worker 29 - jepsen.util 3179 :ok :read [9 12] INFO [2019-12-17 12:30:41,769] jepsen worker 29 - jepsen.util 3179 :invoke :write [9 20] INFO [2019-12-17 12:30:41,791] jepsen worker 81 - jepsen.util 5031 :info :cas [27 [26 40]] :RequestCanceledException INFO [2019-12-17 12:30:41,793] jepsen worker 81 - jepsen.util 5121 :invoke :write [27 0] INFO [2019-12-17 12:30:41,811] jepsen worker 64 - jepsen.util 3034 :fail :cas [21 [38 19]] :ValueNotSwapFrom INFO [2019-12-17 12:30:41,812] jepsen worker 64 - jepsen.util 3034 :invoke :cas [21 [30 34]] INFO [2019-12-17 12:30:41,824] jepsen worker 11 - jepsen.util 4151 :info :write [3 15] :RequestCanceledException INFO [2019-12-17 12:30:41,826] jepsen worker 11 - jepsen.util 4241 :invoke :cas [3 [40 33]] INFO [2019-12-17 12:30:41,848] jepsen worker 64 - jepsen.util 3034 :fail :cas [21 [30 34]] :ValueNotSwapFrom INFO [2019-12-17 12:30:41,849] jepsen worker 64 - jepsen.util 3034 :invoke :write [21 18] INFO [2019-12-17 12:30:41,870] jepsen worker 69 - jepsen.util 3759 :ok :read [23 47] INFO [2019-12-17 12:30:41,870] jepsen worker 69 - jepsen.util 3759 :invoke :write [23 26] INFO [2019-12-17 12:30:41,890] jepsen worker 29 - jepsen.util 3179 :ok :write [9 20] INFO [2019-12-17 12:30:41,890] jepsen worker 29 - jepsen.util 3179 :invoke :write [9 2] INFO [2019-12-17 12:30:41,901] jepsen worker 21 - jepsen.util 4971 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:30:41,902] jepsen worker 21 - jepsen.util 4971 :invoke :write [7 39] INFO [2019-12-17 12:30:41,924] jepsen worker 3 - jepsen.util 3963 :info :write [1 23] :RequestCanceledException INFO [2019-12-17 12:30:41,927] jepsen worker 3 - jepsen.util 4053 :invoke :cas [1 [11 20]] INFO [2019-12-17 12:30:41,935] jepsen worker 64 - jepsen.util 3034 :ok :write [21 18] INFO [2019-12-17 12:30:41,935] jepsen worker 64 - jepsen.util 3034 :invoke :read [21 nil] INFO [2019-12-17 12:30:41,939] jepsen worker 11 - jepsen.util 4241 :fail :cas [3 [40 33]] :ValueNotSwapFrom INFO [2019-12-17 12:30:41,940] jepsen worker 11 - jepsen.util 4241 :invoke :write [3 11] INFO [2019-12-17 12:30:41,961] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21701,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:30:41,961] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21701,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:30:41,962] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21705,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:30:41,962] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21705,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:30:41,965] jepsen worker 13 - jepsen.util 3523 :fail :cas [4 [38 30]] :ValueNotSwapFrom INFO [2019-12-17 12:30:41,965] jepsen worker 13 - jepsen.util 3523 :invoke :write [4 23] INFO [2019-12-17 12:30:41,974] jepsen worker 64 - jepsen.util 3034 :ok :read [21 18] INFO [2019-12-17 12:30:41,975] jepsen worker 64 - jepsen.util 3034 :invoke :read [21 nil] INFO [2019-12-17 12:30:42,017] jepsen worker 29 - jepsen.util 3179 :ok :write [9 2] INFO [2019-12-17 12:30:42,017] jepsen worker 29 - jepsen.util 3179 :invoke :cas [9 [24 36]] INFO [2019-12-17 12:30:42,023] jepsen worker 64 - jepsen.util 3034 :ok :read [21 18] INFO [2019-12-17 12:30:42,023] jepsen worker 64 - jepsen.util 3034 :invoke :read [21 nil] INFO [2019-12-17 12:30:42,024] jepsen worker 56 - jepsen.util 3836 :info :write [18 4] :RequestCanceledException INFO [2019-12-17 12:30:42,025] jepsen worker 78 - jepsen.util 4308 :ok :read [26 28] INFO [2019-12-17 12:30:42,025] jepsen worker 78 - jepsen.util 4308 :invoke :cas [26 [8 21]] INFO [2019-12-17 12:30:42,028] jepsen worker 56 - jepsen.util 3926 :invoke :write [18 43] INFO [2019-12-17 12:30:42,038] jepsen worker 70 - jepsen.util 4300 :info :write [23 10] :RequestCanceledException INFO [2019-12-17 12:30:42,041] jepsen worker 70 - jepsen.util 4390 :invoke :read [23 nil] INFO [2019-12-17 12:30:42,044] jepsen worker 12 - jepsen.util 4062 :ok :read [4 45] INFO [2019-12-17 12:30:42,045] jepsen worker 12 - jepsen.util 4062 :invoke :read [4 nil] INFO [2019-12-17 12:30:42,077] jepsen worker 64 - jepsen.util 3034 :ok :read [21 18] INFO [2019-12-17 12:30:42,077] jepsen worker 64 - jepsen.util 3034 :invoke :write [21 15] INFO [2019-12-17 12:30:42,078] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:30:42,078] jepsen nemesis - couchbase.util Rebalance check count: 161 INFO [2019-12-17 12:30:42,078] jepsen nemesis - couchbase.util Rebalance stuck count: 142 INFO [2019-12-17 12:30:42,089] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21714,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:30:42,089] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21714,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:30:42,089] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21715,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:30:42,089] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21715,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:30:42,089] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21716,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:30:42,089] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21716,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:30:42,123] jepsen worker 53 - jepsen.util 3923 :fail :cas [17 [15 6]] :ValueNotSwapFrom INFO [2019-12-17 12:30:42,124] jepsen worker 53 - jepsen.util 3923 :invoke :read [17 nil] INFO [2019-12-17 12:30:42,124] jepsen worker 9 - jepsen.util 4239 :info :cas [3 [3 24]] :RequestCanceledException INFO [2019-12-17 12:30:42,126] jepsen worker 9 - jepsen.util 4329 :invoke :cas [3 [49 42]] INFO [2019-12-17 12:30:42,138] jepsen worker 61 - jepsen.util 5731 :info :write [20 0] :RequestCanceledException INFO [2019-12-17 12:30:42,140] jepsen worker 61 - jepsen.util 5821 :invoke :cas [20 [40 34]] INFO [2019-12-17 12:30:42,153] jepsen worker 78 - jepsen.util 4308 :fail :cas [26 [8 21]] :ValueNotSwapFrom INFO [2019-12-17 12:30:42,154] jepsen worker 78 - jepsen.util 4308 :invoke :write [26 10] INFO [2019-12-17 12:30:42,155] jepsen worker 74 - jepsen.util 4124 :info :write [24 25] :RequestCanceledException INFO [2019-12-17 12:30:42,156] jepsen worker 74 - jepsen.util 4214 :invoke :cas [24 [13 4]] INFO [2019-12-17 12:30:42,156] jepsen worker 64 - jepsen.util 3034 :ok :write [21 15] INFO [2019-12-17 12:30:42,157] jepsen worker 64 - jepsen.util 3034 :invoke :read [21 nil] INFO [2019-12-17 12:30:42,162] jepsen worker 88 - jepsen.util 3868 :fail :cas [29 [4 23]] :ValueNotSwapFrom INFO [2019-12-17 12:30:42,162] jepsen worker 88 - jepsen.util 3868 :invoke :read [29 nil] INFO [2019-12-17 12:30:42,199] jepsen worker 64 - jepsen.util 3034 :ok :read [21 15] INFO [2019-12-17 12:30:42,199] jepsen worker 64 - jepsen.util 3034 :invoke :cas [21 [15 16]] INFO [2019-12-17 12:30:42,208] jepsen worker 4 - jepsen.util 2974 :info :write [1 7] :RequestCanceledException INFO [2019-12-17 12:30:42,211] jepsen worker 4 - jepsen.util 3064 :invoke :cas [1 [48 46]] INFO [2019-12-17 12:30:42,224] jepsen worker 86 - jepsen.util 5216 :info :write [28 11] :RequestCanceledException INFO [2019-12-17 12:30:42,224] jepsen worker 38 - jepsen.util 3728 :info :write [12 18] :RequestCanceledException INFO [2019-12-17 12:30:42,227] jepsen worker 86 - jepsen.util 5306 :invoke :write [28 48] INFO [2019-12-17 12:30:42,230] jepsen worker 38 - jepsen.util 3818 :invoke :read [12 nil] INFO [2019-12-17 12:30:42,231] jepsen worker 74 - jepsen.util 4214 :fail :cas [24 [13 4]] :ValueNotSwapFrom INFO [2019-12-17 12:30:42,231] jepsen worker 74 - jepsen.util 4214 :invoke :cas [24 [40 39]] INFO [2019-12-17 12:30:42,269] jepsen worker 74 - jepsen.util 4214 :fail :cas [24 [40 39]] :ValueNotSwapFrom INFO [2019-12-17 12:30:42,270] jepsen worker 74 - jepsen.util 4214 :invoke :read [24 nil] INFO [2019-12-17 12:30:42,284] jepsen worker 53 - jepsen.util 3923 :ok :read [17 8] INFO [2019-12-17 12:30:42,284] jepsen worker 53 - jepsen.util 3923 :invoke :cas [17 [0 15]] INFO [2019-12-17 12:30:42,290] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21734,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:30:42,290] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21734,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:30:42,290] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21735,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:30:42,290] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21735,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:30:42,290] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21736,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:30:42,291] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21736,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:30:42,291] jepsen worker 77 - jepsen.util 5027 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:30:42,291] jepsen worker 87 - jepsen.util 3507 :info :write [29 0] :RequestCanceledException INFO [2019-12-17 12:30:42,291] jepsen worker 77 - jepsen.util 5027 :invoke :write [25 41] INFO [2019-12-17 12:30:42,293] jepsen worker 87 - jepsen.util 3597 :invoke :write [29 40] INFO [2019-12-17 12:30:42,303] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21744,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:30:42,303] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21744,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:30:42,303] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21746,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:30:42,303] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21746,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:30:42,307] jepsen worker 74 - jepsen.util 4214 :ok :read [24 41] INFO [2019-12-17 12:30:42,307] jepsen worker 74 - jepsen.util 4214 :invoke :read [24 nil] INFO [2019-12-17 12:30:42,309] jepsen worker 36 - jepsen.util 3906 :info :write [12 37] :RequestCanceledException INFO [2019-12-17 12:30:42,311] jepsen worker 36 - jepsen.util 3996 :invoke :write [12 13] INFO [2019-12-17 12:30:42,324] jepsen worker 24 - jepsen.util 4794 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:30:42,325] jepsen worker 24 - jepsen.util 4794 :invoke :write [8 1] INFO [2019-12-17 12:30:42,342] jepsen worker 74 - jepsen.util 4214 :ok :read [24 41] INFO [2019-12-17 12:30:42,342] jepsen worker 74 - jepsen.util 4214 :invoke :write [24 33] INFO [2019-12-17 12:30:42,345] jepsen worker 64 - jepsen.util 3034 :ok :cas [21 [15 16]] INFO [2019-12-17 12:30:42,345] jepsen worker 64 - jepsen.util 3034 :invoke :read [21 nil] INFO [2019-12-17 12:30:42,392] jepsen worker 64 - jepsen.util 3034 :ok :read [21 16] INFO [2019-12-17 12:30:42,393] jepsen worker 64 - jepsen.util 3034 :invoke :read [21 nil] INFO [2019-12-17 12:30:42,423] jepsen worker 12 - jepsen.util 4062 :ok :read [4 45] INFO [2019-12-17 12:30:42,423] jepsen worker 12 - jepsen.util 4062 :invoke :write [4 32] INFO [2019-12-17 12:30:42,424] jepsen worker 64 - jepsen.util 3034 :ok :read [21 16] INFO [2019-12-17 12:30:42,424] jepsen worker 64 - jepsen.util 3034 :invoke :write [21 28] INFO [2019-12-17 12:30:42,424] jepsen worker 49 - jepsen.util 4369 :info :cas [16 [14 44]] :RequestCanceledException INFO [2019-12-17 12:30:42,428] jepsen worker 49 - jepsen.util 4459 :invoke :write [16 31] INFO [2019-12-17 12:30:42,445] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21762,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:30:42,455] jepsen worker 34 - jepsen.util 4984 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:30:42,471] jepsen worker 70 - jepsen.util 4390 :ok :read [23 47] INFO [2019-12-17 12:30:42,488] jepsen worker 64 - jepsen.util 3034 :ok :write [21 28] INFO [2019-12-17 12:30:42,521] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21762,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:30:42,521] jepsen worker 34 - jepsen.util 4984 :invoke :read [11 nil] INFO [2019-12-17 12:30:42,521] jepsen worker 70 - jepsen.util 4390 :invoke :write [23 14] INFO [2019-12-17 12:30:42,521] jepsen worker 64 - jepsen.util 3034 :invoke :cas [21 [41 25]] INFO [2019-12-17 12:30:42,521] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21763,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:30:42,522] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21763,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:30:42,522] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21767,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:30:42,522] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21767,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:30:42,529] jepsen worker 88 - jepsen.util 3868 :ok :read [29 18] INFO [2019-12-17 12:30:42,546] jepsen worker 64 - jepsen.util 3034 :fail :cas [21 [41 25]] :ValueNotSwapFrom INFO [2019-12-17 12:30:42,587] jepsen worker 88 - jepsen.util 3868 :invoke :write [29 25] INFO [2019-12-17 12:30:42,587] jepsen worker 64 - jepsen.util 3034 :invoke :read [21 nil] INFO [2019-12-17 12:30:42,591] jepsen worker 55 - jepsen.util 4555 :info :write [18 44] :RequestCanceledException INFO [2019-12-17 12:30:42,596] jepsen worker 55 - jepsen.util 4645 :invoke :write [18 42] INFO [2019-12-17 12:30:42,615] jepsen worker 64 - jepsen.util 3034 :ok :read [21 28] INFO [2019-12-17 12:30:42,615] jepsen worker 64 - jepsen.util 3034 :invoke :cas [21 [31 6]] INFO [2019-12-17 12:30:42,635] jepsen worker 53 - jepsen.util 3923 :fail :cas [17 [0 15]] :ValueNotSwapFrom INFO [2019-12-17 12:30:42,636] jepsen worker 53 - jepsen.util 3923 :invoke :write [17 10] INFO [2019-12-17 12:30:42,647] jepsen worker 64 - jepsen.util 3034 :fail :cas [21 [31 6]] :ValueNotSwapFrom INFO [2019-12-17 12:30:42,647] jepsen worker 64 - jepsen.util 3034 :invoke :write [21 44] INFO [2019-12-17 12:30:42,707] jepsen worker 64 - jepsen.util 3034 :ok :write [21 44] INFO [2019-12-17 12:30:42,707] jepsen worker 64 - jepsen.util 3034 :invoke :cas [21 [25 36]] INFO [2019-12-17 12:30:42,709] jepsen worker 68 - jepsen.util 4658 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:30:42,709] jepsen worker 68 - jepsen.util 4658 :invoke :cas [22 [41 41]] INFO [2019-12-17 12:30:42,724] jepsen worker 52 - jepsen.util 3832 :info :write [17 35] :RequestCanceledException INFO [2019-12-17 12:30:42,727] jepsen worker 52 - jepsen.util 3922 :invoke :read [17 nil] INFO [2019-12-17 12:30:42,728] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21779,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:30:42,728] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21779,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:30:42,728] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21783,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:30:42,728] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21783,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:30:42,738] jepsen worker 23 - jepsen.util 4973 :info :cas [7 [5 3]] :RequestCanceledException INFO [2019-12-17 12:30:42,740] jepsen worker 64 - jepsen.util 3034 :fail :cas [21 [25 36]] :ValueNotSwapFrom INFO [2019-12-17 12:30:42,740] jepsen worker 64 - jepsen.util 3034 :invoke :cas [21 [28 8]] INFO [2019-12-17 12:30:42,741] jepsen worker 23 - jepsen.util 5063 :invoke :read [7 nil] INFO [2019-12-17 12:30:42,770] jepsen worker 64 - jepsen.util 3034 :fail :cas [21 [28 8]] :ValueNotSwapFrom INFO [2019-12-17 12:30:42,770] jepsen worker 64 - jepsen.util 3034 :invoke :write [21 34] INFO [2019-12-17 12:30:42,791] jepsen worker 37 - jepsen.util 3547 :fail :read [12 nil] :RequestTimeoutException INFO [2019-12-17 12:30:42,792] jepsen worker 37 - jepsen.util 3547 :invoke :read [12 nil] INFO [2019-12-17 12:30:42,793] jepsen worker 38 - jepsen.util 3818 :ok :read [12 1] INFO [2019-12-17 12:30:42,793] jepsen worker 38 - jepsen.util 3818 :invoke :read [12 nil] INFO [2019-12-17 12:30:42,824] jepsen worker 44 - jepsen.util 5444 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:30:42,824] jepsen worker 44 - jepsen.util 5444 :invoke :write [14 28] INFO [2019-12-17 12:30:42,825] jepsen worker 79 - jepsen.util 3769 :info :write [26 21] :RequestCanceledException INFO [2019-12-17 12:30:42,828] jepsen worker 79 - jepsen.util 3859 :invoke :read [26 nil] INFO [2019-12-17 12:30:42,828] jepsen worker 3 - jepsen.util 4053 :fail :cas [1 [11 20]] :ValueNotSwapFrom INFO [2019-12-17 12:30:42,829] jepsen worker 3 - jepsen.util 4053 :invoke :write [1 9] INFO [2019-12-17 12:30:42,837] jepsen worker 59 - jepsen.util 5009 :info :cas [19 [6 21]] :RequestCanceledException INFO [2019-12-17 12:30:42,846] jepsen worker 59 - jepsen.util 5099 :invoke :write [19 15] INFO [2019-12-17 12:30:42,852] jepsen worker 28 - jepsen.util 2638 :fail :read [9 nil] :RequestTimeoutException INFO [2019-12-17 12:30:42,852] jepsen worker 27 - jepsen.util 3627 :info :cas [9 [21 41]] :RequestCanceledException INFO [2019-12-17 12:30:42,852] jepsen worker 28 - jepsen.util 2638 :invoke :write [9 41] INFO [2019-12-17 12:30:42,855] jepsen worker 27 - jepsen.util 3717 :invoke :cas [9 [26 3]] INFO [2019-12-17 12:30:42,880] jepsen worker 64 - jepsen.util 3034 :ok :write [21 34] INFO [2019-12-17 12:30:42,880] jepsen worker 64 - jepsen.util 3034 :invoke :read [21 nil] INFO [2019-12-17 12:30:42,882] jepsen worker 84 - jepsen.util 4674 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:30:42,882] jepsen worker 84 - jepsen.util 4674 :invoke :write [28 25] INFO [2019-12-17 12:30:42,900] jepsen worker 43 - jepsen.util 4813 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:30:42,900] jepsen worker 43 - jepsen.util 4813 :invoke :read [14 nil] INFO [2019-12-17 12:30:42,902] jepsen worker 38 - jepsen.util 3818 :ok :read [12 1] INFO [2019-12-17 12:30:42,903] jepsen worker 38 - jepsen.util 3818 :invoke :read [12 nil] INFO [2019-12-17 12:30:42,917] jepsen worker 66 - jepsen.util 4926 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:30:42,917] jepsen worker 72 - jepsen.util 3942 :info :write [24 48] :RequestCanceledException INFO [2019-12-17 12:30:42,917] jepsen worker 66 - jepsen.util 4926 :invoke :cas [22 [6 30]] INFO [2019-12-17 12:30:42,921] jepsen worker 72 - jepsen.util 4032 :invoke :cas [24 [4 2]] INFO [2019-12-17 12:30:42,930] jepsen worker 27 - jepsen.util 3717 :fail :cas [9 [26 3]] :ValueNotSwapFrom INFO [2019-12-17 12:30:42,930] jepsen worker 27 - jepsen.util 3717 :invoke :read [9 nil] INFO [2019-12-17 12:30:42,933] jepsen worker 64 - jepsen.util 3034 :ok :read [21 34] INFO [2019-12-17 12:30:42,933] jepsen worker 64 - jepsen.util 3034 :invoke :write [21 8] INFO [2019-12-17 12:30:42,933] jepsen worker 10 - jepsen.util 3610 :info :write [3 10] :RequestCanceledException INFO [2019-12-17 12:30:42,936] jepsen worker 10 - jepsen.util 3700 :invoke :read [3 nil] INFO [2019-12-17 12:30:42,938] jepsen worker 4 - jepsen.util 3064 :fail :cas [1 [48 46]] :ValueNotSwapFrom INFO [2019-12-17 12:30:42,938] jepsen worker 4 - jepsen.util 3064 :invoke :write [1 1] INFO [2019-12-17 12:30:42,946] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21802,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:30:42,946] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21802,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:30:42,958] jepsen worker 72 - jepsen.util 4032 :fail :cas [24 [4 2]] :ValueNotSwapFrom INFO [2019-12-17 12:30:42,958] jepsen worker 72 - jepsen.util 4032 :invoke :write [24 40] INFO [2019-12-17 12:30:42,978] jepsen worker 27 - jepsen.util 3717 :ok :read [9 2] INFO [2019-12-17 12:30:42,979] jepsen worker 27 - jepsen.util 3717 :invoke :cas [9 [40 48]] INFO [2019-12-17 12:30:42,990] jepsen worker 51 - jepsen.util 3291 :info :write [17 10] :RequestCanceledException INFO [2019-12-17 12:30:42,993] jepsen worker 51 - jepsen.util 3381 :invoke :write [17 37] INFO [2019-12-17 12:30:43,004] jepsen worker 10 - jepsen.util 3700 :ok :read [3 22] INFO [2019-12-17 12:30:43,005] jepsen worker 10 - jepsen.util 3700 :invoke :read [3 nil] INFO [2019-12-17 12:30:43,007] jepsen worker 20 - jepsen.util 3620 :info :write [6 3] :SyncWriteAmbiguous INFO [2019-12-17 12:30:43,009] jepsen worker 20 - jepsen.util 3710 :invoke :cas [6 [25 10]] INFO [2019-12-17 12:30:43,024] jepsen worker 27 - jepsen.util 3717 :fail :cas [9 [40 48]] :ValueNotSwapFrom INFO [2019-12-17 12:30:43,024] jepsen worker 25 - jepsen.util 5245 :info :cas [8 [9 6]] :RequestCanceledException INFO [2019-12-17 12:30:43,024] jepsen worker 54 - jepsen.util 4104 :info :write [18 14] :RequestCanceledException INFO [2019-12-17 12:30:43,025] jepsen worker 27 - jepsen.util 3717 :invoke :read [9 nil] INFO [2019-12-17 12:30:43,025] jepsen worker 62 - jepsen.util 4832 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:30:43,025] jepsen worker 62 - jepsen.util 4832 :invoke :read [20 nil] INFO [2019-12-17 12:30:43,027] jepsen worker 54 - jepsen.util 4194 :invoke :write [18 19] INFO [2019-12-17 12:30:43,027] jepsen worker 25 - jepsen.util 5335 :invoke :read [8 nil] INFO [2019-12-17 12:30:43,038] jepsen worker 39 - jepsen.util 4809 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:30:43,038] jepsen worker 39 - jepsen.util 4809 :invoke :read [13 nil] INFO [2019-12-17 12:30:43,043] jepsen worker 10 - jepsen.util 3700 :ok :read [3 22] INFO [2019-12-17 12:30:43,044] jepsen worker 10 - jepsen.util 3700 :invoke :read [3 nil] INFO [2019-12-17 12:30:43,052] jepsen worker 6 - jepsen.util 5496 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:30:43,052] jepsen worker 6 - jepsen.util 5496 :invoke :cas [2 [48 31]] INFO [2019-12-17 12:30:43,060] jepsen worker 27 - jepsen.util 3717 :ok :read [9 2] INFO [2019-12-17 12:30:43,061] jepsen worker 27 - jepsen.util 3717 :invoke :read [9 nil] INFO [2019-12-17 12:30:43,064] jepsen worker 65 - jepsen.util 3215 :ok :write [21 26] INFO [2019-12-17 12:30:43,064] jepsen worker 65 - jepsen.util 3215 :invoke :read [21 nil] INFO [2019-12-17 12:30:43,071] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21819,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:30:43,072] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21819,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:30:43,072] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21823,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:30:43,072] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21823,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:30:43,084] jepsen worker 10 - jepsen.util 3700 :ok :read [3 22] INFO [2019-12-17 12:30:43,085] jepsen worker 10 - jepsen.util 3700 :invoke :cas [3 [32 20]] INFO [2019-12-17 12:30:43,091] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:30:43,091] jepsen nemesis - couchbase.util Rebalance check count: 162 INFO [2019-12-17 12:30:43,091] jepsen nemesis - couchbase.util Rebalance stuck count: 143 INFO [2019-12-17 12:30:43,098] jepsen worker 27 - jepsen.util 3717 :ok :read [9 2] INFO [2019-12-17 12:30:43,098] jepsen worker 27 - jepsen.util 3717 :invoke :cas [9 [4 20]] INFO [2019-12-17 12:30:43,115] jepsen worker 10 - jepsen.util 3700 :fail :cas [3 [32 20]] :ValueNotSwapFrom INFO [2019-12-17 12:30:43,115] jepsen worker 10 - jepsen.util 3700 :invoke :cas [3 [45 6]] INFO [2019-12-17 12:30:43,123] jepsen worker 82 - jepsen.util 5842 :info :cas [27 [11 35]] :RequestCanceledException INFO [2019-12-17 12:30:43,124] jepsen worker 19 - jepsen.util 3979 :info :cas [6 [39 4]] :RequestCanceledException INFO [2019-12-17 12:30:43,124] jepsen worker 7 - jepsen.util 4867 :info :write [2 18] :RequestCanceledException INFO [2019-12-17 12:30:43,127] jepsen worker 82 - jepsen.util 5932 :invoke :cas [27 [9 0]] INFO [2019-12-17 12:30:43,128] jepsen worker 7 - jepsen.util 4957 :invoke :write [2 1] INFO [2019-12-17 12:30:43,128] jepsen worker 19 - jepsen.util 4069 :invoke :cas [6 [1 10]] INFO [2019-12-17 12:30:43,139] jepsen worker 32 - jepsen.util 4172 :info :write [10 12] :RequestCanceledException INFO [2019-12-17 12:30:43,142] jepsen worker 32 - jepsen.util 4262 :invoke :cas [10 [30 48]] INFO [2019-12-17 12:30:43,148] jepsen worker 27 - jepsen.util 3717 :fail :cas [9 [4 20]] :ValueNotSwapFrom INFO [2019-12-17 12:30:43,148] jepsen worker 27 - jepsen.util 3717 :invoke :read [9 nil] INFO [2019-12-17 12:30:43,153] jepsen worker 10 - jepsen.util 3700 :fail :cas [3 [45 6]] :ValueNotSwapFrom INFO [2019-12-17 12:30:43,154] jepsen worker 10 - jepsen.util 3700 :invoke :read [3 nil] INFO [2019-12-17 12:30:43,180] jepsen worker 27 - jepsen.util 3717 :ok :read [9 2] INFO [2019-12-17 12:30:43,180] jepsen worker 27 - jepsen.util 3717 :invoke :read [9 nil] INFO [2019-12-17 12:30:43,186] jepsen worker 10 - jepsen.util 3700 :ok :read [3 22] INFO [2019-12-17 12:30:43,186] jepsen worker 10 - jepsen.util 3700 :invoke :read [3 nil] INFO [2019-12-17 12:30:43,191] jepsen worker 46 - jepsen.util 3916 :info :write [15 13] :RequestCanceledException INFO [2019-12-17 12:30:43,191] jepsen worker 50 - jepsen.util 5540 :info :write [16 40] :RequestCanceledException INFO [2019-12-17 12:30:43,194] jepsen worker 46 - jepsen.util 4006 :invoke :read [15 nil] INFO [2019-12-17 12:30:43,194] jepsen worker 50 - jepsen.util 5630 :invoke :read [16 nil] INFO [2019-12-17 12:30:43,209] jepsen worker 85 - jepsen.util 5575 :info :cas [28 [30 29]] :RequestCanceledException INFO [2019-12-17 12:30:43,212] jepsen worker 85 - jepsen.util 5665 :invoke :read [28 nil] INFO [2019-12-17 12:30:43,231] jepsen worker 27 - jepsen.util 3717 :ok :read [9 2] INFO [2019-12-17 12:30:43,232] jepsen worker 27 - jepsen.util 3717 :invoke :cas [9 [4 21]] INFO [2019-12-17 12:30:43,238] jepsen worker 10 - jepsen.util 3700 :ok :read [3 22] INFO [2019-12-17 12:30:43,238] jepsen worker 30 - jepsen.util 4170 :info :cas [10 [5 5]] :RequestCanceledException INFO [2019-12-17 12:30:43,238] jepsen worker 10 - jepsen.util 3700 :invoke :write [3 47] INFO [2019-12-17 12:30:43,241] jepsen worker 30 - jepsen.util 4260 :invoke :read [10 nil] INFO [2019-12-17 12:30:43,244] jepsen worker 46 - jepsen.util 4006 :ok :read [15 30] INFO [2019-12-17 12:30:43,244] jepsen worker 46 - jepsen.util 4006 :invoke :cas [15 [45 44]] INFO [2019-12-17 12:30:43,262] jepsen worker 27 - jepsen.util 3717 :fail :cas [9 [4 21]] :ValueNotSwapFrom INFO [2019-12-17 12:30:43,262] jepsen worker 27 - jepsen.util 3717 :invoke :read [9 nil] INFO [2019-12-17 12:30:43,269] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21847,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:30:43,270] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21847,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:30:43,270] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21848,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:30:43,270] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21848,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:30:43,287] jepsen worker 37 - jepsen.util 3547 :ok :read [12 1] INFO [2019-12-17 12:30:43,288] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21858,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:30:43,288] jepsen worker 31 - jepsen.util 3361 :fail :read [10 nil] :RequestTimeoutException INFO [2019-12-17 12:30:43,294] jepsen worker 52 - jepsen.util 3922 :ok :read [17 8] INFO [2019-12-17 12:30:43,298] jepsen worker 30 - jepsen.util 4260 :ok :read [10 8] INFO [2019-12-17 12:30:43,315] jepsen worker 46 - jepsen.util 4006 :fail :cas [15 [45 44]] :ValueNotSwapFrom INFO [2019-12-17 12:30:43,322] jepsen worker 27 - jepsen.util 3717 :ok :read [9 2] INFO [2019-12-17 12:30:43,338] jepsen worker 14 - jepsen.util 3794 :info :write [4 42] :RequestCanceledException INFO [2019-12-17 12:30:43,338] jepsen worker 48 - jepsen.util 4638 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:30:43,343] jepsen worker 73 - jepsen.util 3583 :info :write [24 6] :SyncWriteAmbiguous INFO [2019-12-17 12:30:43,345] jepsen worker 37 - jepsen.util 3547 :invoke :cas [12 [5 11]] INFO [2019-12-17 12:30:43,345] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21858,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:30:43,345] jepsen worker 31 - jepsen.util 3361 :invoke :cas [10 [6 46]] INFO [2019-12-17 12:30:43,345] jepsen worker 52 - jepsen.util 3922 :invoke :write [17 12] INFO [2019-12-17 12:30:43,345] jepsen worker 30 - jepsen.util 4260 :invoke :write [10 19] INFO [2019-12-17 12:30:43,345] jepsen worker 46 - jepsen.util 4006 :invoke :write [15 47] INFO [2019-12-17 12:30:43,345] jepsen worker 27 - jepsen.util 3717 :invoke :cas [9 [29 24]] INFO [2019-12-17 12:30:43,345] jepsen worker 48 - jepsen.util 4638 :invoke :read [16 nil] INFO [2019-12-17 12:30:43,345] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21860,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:30:43,346] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21860,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:30:43,347] jepsen worker 14 - jepsen.util 3884 :invoke :write [4 43] INFO [2019-12-17 12:30:43,348] jepsen worker 73 - jepsen.util 3673 :invoke :cas [24 [25 7]] INFO [2019-12-17 12:30:43,355] jepsen worker 83 - jepsen.util 5123 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:30:43,355] jepsen worker 83 - jepsen.util 5123 :invoke :write [27 23] INFO [2019-12-17 12:30:43,391] jepsen worker 40 - jepsen.util 5440 :info :cas [13 [10 4]] :RequestCanceledException INFO [2019-12-17 12:30:43,394] jepsen worker 40 - jepsen.util 5530 :invoke :cas [13 [2 28]] INFO [2019-12-17 12:30:43,409] jepsen worker 31 - jepsen.util 3361 :fail :cas [10 [6 46]] :ValueNotSwapFrom INFO [2019-12-17 12:30:43,409] jepsen worker 31 - jepsen.util 3361 :invoke :read [10 nil] INFO [2019-12-17 12:30:43,409] jepsen worker 26 - jepsen.util 5336 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:30:43,409] jepsen worker 35 - jepsen.util 4985 :info :write [11 6] :RequestCanceledException INFO [2019-12-17 12:30:43,409] jepsen worker 26 - jepsen.util 5336 :invoke :read [8 nil] INFO [2019-12-17 12:30:43,411] jepsen worker 35 - jepsen.util 5075 :invoke :read [11 nil] INFO [2019-12-17 12:30:43,413] jepsen worker 63 - jepsen.util 3303 :ok :read [21 26] INFO [2019-12-17 12:30:43,414] jepsen worker 63 - jepsen.util 3303 :invoke :write [21 17] INFO [2019-12-17 12:30:43,424] jepsen worker 0 - jepsen.util 5490 :info :write [0 21] :RequestCanceledException INFO [2019-12-17 12:30:43,424] jepsen worker 67 - jepsen.util 5287 :info :write [22 14] :RequestCanceledException INFO [2019-12-17 12:30:43,424] jepsen worker 22 - jepsen.util 5152 :info :cas [7 [16 5]] :RequestCanceledException INFO [2019-12-17 12:30:43,424] jepsen worker 71 - jepsen.util 3941 :info :write [23 24] :RequestCanceledException INFO [2019-12-17 12:30:43,426] jepsen worker 89 - jepsen.util 3599 :info :write [29 41] :RequestCanceledException INFO [2019-12-17 12:30:43,428] jepsen worker 0 - jepsen.util 5580 :invoke :write [0 42] INFO [2019-12-17 12:30:43,428] jepsen worker 22 - jepsen.util 5242 :invoke :read [7 nil] INFO [2019-12-17 12:30:43,429] jepsen worker 71 - jepsen.util 4031 :invoke :cas [23 [11 18]] INFO [2019-12-17 12:30:43,429] jepsen worker 67 - jepsen.util 5377 :invoke :cas [22 [41 17]] INFO [2019-12-17 12:30:43,430] jepsen worker 89 - jepsen.util 3689 :invoke :read [29 nil] INFO [2019-12-17 12:30:43,439] jepsen worker 57 - jepsen.util 4827 :info :write [19 47] :RequestCanceledException INFO [2019-12-17 12:30:43,440] jepsen worker 57 - jepsen.util 4917 :invoke :write [19 38] INFO [2019-12-17 12:30:43,456] jepsen worker 60 - jepsen.util 5370 :info :write [20 19] :RequestCanceledException INFO [2019-12-17 12:30:43,457] jepsen worker 60 - jepsen.util 5460 :invoke :write [20 43] INFO [2019-12-17 12:30:43,467] jepsen worker 9 - jepsen.util 4329 :fail :cas [3 [49 42]] :ValueNotSwapFrom INFO [2019-12-17 12:30:43,467] jepsen worker 9 - jepsen.util 4329 :invoke :cas [3 [38 45]] INFO [2019-12-17 12:30:43,488] jepsen worker 58 - jepsen.util 5098 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:30:43,488] jepsen worker 2 - jepsen.util 4772 :info :write [0 47] :RequestCanceledException INFO [2019-12-17 12:30:43,488] jepsen worker 58 - jepsen.util 5098 :invoke :cas [19 [30 18]] INFO [2019-12-17 12:30:43,490] jepsen worker 2 - jepsen.util 4862 :invoke :read [0 nil] INFO [2019-12-17 12:30:43,507] jepsen worker 79 - jepsen.util 3859 :ok :read [26 28] INFO [2019-12-17 12:30:43,507] jepsen worker 79 - jepsen.util 3859 :invoke :cas [26 [14 16]] INFO [2019-12-17 12:30:43,507] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21883,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:30:43,507] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21883,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:30:43,508] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21886,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:30:43,508] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21886,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:30:43,508] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21887,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:30:43,508] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21887,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:30:43,509] jepsen worker 17 - jepsen.util 4787 :info :cas [5 [4 19]] :RequestCanceledException INFO [2019-12-17 12:30:43,513] jepsen worker 17 - jepsen.util 4877 :invoke :cas [5 [17 4]] INFO [2019-12-17 12:30:43,524] jepsen worker 42 - jepsen.util 5982 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:30:43,524] jepsen worker 33 - jepsen.util 5253 :info :write [11 1] :RequestCanceledException INFO [2019-12-17 12:30:43,525] jepsen worker 42 - jepsen.util 5982 :invoke :read [14 nil] INFO [2019-12-17 12:30:43,528] jepsen worker 33 - jepsen.util 5343 :invoke :write [11 19] INFO [2019-12-17 12:30:43,534] jepsen worker 27 - jepsen.util 3717 :fail :cas [9 [29 24]] :ValueNotSwapFrom INFO [2019-12-17 12:30:43,534] jepsen worker 10 - jepsen.util 3700 :ok :write [3 47] INFO [2019-12-17 12:30:43,535] jepsen worker 28 - jepsen.util 2638 :ok :write [9 41] INFO [2019-12-17 12:30:43,536] jepsen worker 10 - jepsen.util 3700 :invoke :cas [3 [24 4]] INFO [2019-12-17 12:30:43,536] jepsen worker 28 - jepsen.util 2638 :invoke :cas [9 [24 4]] INFO [2019-12-17 12:30:43,536] jepsen worker 27 - jepsen.util 3717 :invoke :cas [9 [44 33]] INFO [2019-12-17 12:30:43,546] jepsen worker 65 - jepsen.util 3215 :ok :read [21 26] INFO [2019-12-17 12:30:43,546] jepsen worker 65 - jepsen.util 3215 :invoke :write [21 9] INFO [2019-12-17 12:30:43,581] jepsen worker 19 - jepsen.util 4069 :fail :cas [6 [1 10]] :ValueNotSwapFrom INFO [2019-12-17 12:30:43,582] jepsen worker 19 - jepsen.util 4069 :invoke :read [6 nil] INFO [2019-12-17 12:30:43,609] jepsen worker 18 - jepsen.util 3888 :info :write [6 32] :RequestCanceledException INFO [2019-12-17 12:30:43,615] jepsen worker 18 - jepsen.util 3978 :invoke :cas [6 [44 40]] INFO [2019-12-17 12:30:43,619] jepsen worker 32 - jepsen.util 4262 :fail :cas [10 [30 48]] :ValueNotSwapFrom INFO [2019-12-17 12:30:43,619] jepsen worker 32 - jepsen.util 4262 :invoke :write [10 38] INFO [2019-12-17 12:30:43,622] jepsen worker 31 - jepsen.util 3361 :ok :read [10 12] INFO [2019-12-17 12:30:43,622] jepsen worker 41 - jepsen.util 5261 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:30:43,622] jepsen worker 31 - jepsen.util 3361 :invoke :cas [10 [26 14]] INFO [2019-12-17 12:30:43,622] jepsen worker 41 - jepsen.util 5261 :invoke :cas [13 [24 14]] INFO [2019-12-17 12:30:43,636] jepsen worker 75 - jepsen.util 5115 :info :write [25 31] :RequestCanceledException INFO [2019-12-17 12:30:43,643] jepsen worker 75 - jepsen.util 5205 :invoke :cas [25 [17 30]] INFO [2019-12-17 12:30:43,651] jepsen worker 37 - jepsen.util 3547 :fail :cas [12 [5 11]] :ValueNotSwapFrom INFO [2019-12-17 12:30:43,651] jepsen worker 37 - jepsen.util 3547 :invoke :write [12 23] INFO [2019-12-17 12:30:43,655] jepsen worker 1 - jepsen.util 4771 :info :cas [0 [5 11]] :RequestCanceledException INFO [2019-12-17 12:30:43,655] jepsen worker 45 - jepsen.util 3915 :info :write [15 41] :RequestCanceledException INFO [2019-12-17 12:30:43,657] jepsen worker 45 - jepsen.util 4005 :invoke :read [15 nil] INFO [2019-12-17 12:30:43,659] jepsen worker 1 - jepsen.util 4861 :invoke :cas [0 [12 20]] INFO [2019-12-17 12:30:43,663] jepsen worker 73 - jepsen.util 3673 :fail :cas [24 [25 7]] :ValueNotSwapFrom INFO [2019-12-17 12:30:43,663] jepsen worker 73 - jepsen.util 3673 :invoke :write [24 45] INFO [2019-12-17 12:30:43,687] jepsen worker 16 - jepsen.util 5776 :info :cas [5 [8 16]] :RequestCanceledException INFO [2019-12-17 12:30:43,692] jepsen worker 16 - jepsen.util 5866 :invoke :read [5 nil] INFO [2019-12-17 12:30:43,696] jepsen worker 27 - jepsen.util 3717 :fail :cas [9 [44 33]] :ValueNotSwapFrom INFO [2019-12-17 12:30:43,696] jepsen worker 27 - jepsen.util 3717 :invoke :cas [9 [8 30]] INFO [2019-12-17 12:30:43,702] jepsen worker 89 - jepsen.util 3689 :ok :read [29 18] INFO [2019-12-17 12:30:43,702] jepsen worker 89 - jepsen.util 3689 :invoke :read [29 nil] INFO [2019-12-17 12:30:43,708] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21914,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:30:43,708] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21914,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:30:43,708] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21916,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:30:43,708] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21916,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:30:43,709] jepsen worker 47 - jepsen.util 4277 :info :write [15 5] :RequestCanceledException INFO [2019-12-17 12:30:43,709] jepsen worker 76 - jepsen.util 5206 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:30:43,710] jepsen worker 76 - jepsen.util 5206 :invoke :write [25 24] INFO [2019-12-17 12:30:43,712] jepsen worker 47 - jepsen.util 4367 :invoke :write [15 12] INFO [2019-12-17 12:30:43,725] jepsen worker 8 - jepsen.util 4328 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:30:43,725] jepsen worker 15 - jepsen.util 4605 :info :cas [5 [24 18]] :RequestCanceledException INFO [2019-12-17 12:30:43,725] jepsen worker 8 - jepsen.util 4328 :invoke :read [2 nil] INFO [2019-12-17 12:30:43,728] jepsen worker 15 - jepsen.util 4695 :invoke :read [5 nil] INFO [2019-12-17 12:30:43,736] jepsen worker 5 - jepsen.util 3425 :info :write [1 48] :RequestCanceledException INFO [2019-12-17 12:30:43,738] jepsen worker 5 - jepsen.util 3515 :invoke :read [1 nil] INFO [2019-12-17 12:30:43,757] jepsen worker 9 - jepsen.util 4329 :fail :cas [3 [38 45]] :ValueNotSwapFrom INFO [2019-12-17 12:30:43,757] jepsen worker 9 - jepsen.util 4329 :invoke :write [3 2] INFO [2019-12-17 12:30:43,764] jepsen worker 31 - jepsen.util 3361 :fail :cas [10 [26 14]] :ValueNotSwapFrom INFO [2019-12-17 12:30:43,764] jepsen worker 31 - jepsen.util 3361 :invoke :write [10 40] INFO [2019-12-17 12:30:43,767] jepsen worker 65 - jepsen.util 3215 :ok :write [21 9] INFO [2019-12-17 12:30:43,767] jepsen worker 65 - jepsen.util 3215 :invoke :write [21 15] INFO [2019-12-17 12:30:43,794] jepsen worker 45 - jepsen.util 4005 :ok :read [15 30] INFO [2019-12-17 12:30:43,795] jepsen worker 45 - jepsen.util 4005 :invoke :cas [15 [24 23]] INFO [2019-12-17 12:30:43,801] jepsen worker 28 - jepsen.util 2638 :fail :cas [9 [24 4]] :ValueNotSwapFrom INFO [2019-12-17 12:30:43,801] jepsen worker 28 - jepsen.util 2638 :invoke :cas [9 [49 25]] INFO [2019-12-17 12:30:43,809] jepsen worker 27 - jepsen.util 3717 :fail :cas [9 [8 30]] :ValueNotSwapFrom INFO [2019-12-17 12:30:43,809] jepsen worker 27 - jepsen.util 3717 :invoke :write [9 35] INFO [2019-12-17 12:30:43,811] jepsen worker 80 - jepsen.util 3860 :info :write [26 5] :RequestCanceledException INFO [2019-12-17 12:30:43,813] jepsen worker 80 - jepsen.util 3950 :invoke :cas [26 [46 23]] INFO [2019-12-17 12:30:43,824] jepsen worker 81 - jepsen.util 5121 :info :write [27 0] :RequestCanceledException INFO [2019-12-17 12:30:43,827] jepsen worker 81 - jepsen.util 5211 :invoke :read [27 nil] INFO [2019-12-17 12:30:43,856] jepsen worker 45 - jepsen.util 4005 :fail :cas [15 [24 23]] :ValueNotSwapFrom INFO [2019-12-17 12:30:43,856] jepsen worker 45 - jepsen.util 4005 :invoke :write [15 48] INFO [2019-12-17 12:30:43,919] jepsen worker 65 - jepsen.util 3215 :ok :write [21 15] INFO [2019-12-17 12:30:43,919] jepsen worker 65 - jepsen.util 3215 :invoke :cas [21 [45 19]] INFO [2019-12-17 12:30:43,923] jepsen worker 69 - jepsen.util 3759 :info :write [23 26] :RequestCanceledException INFO [2019-12-17 12:30:43,925] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21939,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:30:43,925] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21939,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:30:43,926] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21942,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:30:43,926] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21942,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:30:43,926] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21943,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:30:43,926] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21943,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:30:43,926] jepsen worker 69 - jepsen.util 3849 :invoke :cas [23 [10 10]] INFO [2019-12-17 12:30:43,948] jepsen worker 65 - jepsen.util 3215 :fail :cas [21 [45 19]] :ValueNotSwapFrom INFO [2019-12-17 12:30:43,948] jepsen worker 65 - jepsen.util 3215 :invoke :write [21 48] INFO [2019-12-17 12:30:43,965] jepsen worker 38 - jepsen.util 3818 :ok :read [12 1] INFO [2019-12-17 12:30:43,966] jepsen worker 38 - jepsen.util 3818 :invoke :cas [12 [20 32]] INFO [2019-12-17 12:30:43,991] jepsen worker 11 - jepsen.util 4241 :info :write [3 11] :RequestCanceledException INFO [2019-12-17 12:30:43,994] jepsen worker 11 - jepsen.util 4331 :invoke :cas [3 [26 23]] INFO [2019-12-17 12:30:44,009] jepsen worker 21 - jepsen.util 4971 :info :write [7 39] :RequestCanceledException INFO [2019-12-17 12:30:44,011] jepsen worker 21 - jepsen.util 5061 :invoke :cas [7 [35 34]] INFO [2019-12-17 12:30:44,024] jepsen worker 71 - jepsen.util 4031 :fail :cas [23 [11 18]] :ValueNotSwapFrom INFO [2019-12-17 12:30:44,025] jepsen worker 71 - jepsen.util 4031 :invoke :write [23 16] INFO [2019-12-17 12:30:44,038] jepsen worker 65 - jepsen.util 3215 :ok :write [21 48] INFO [2019-12-17 12:30:44,039] jepsen worker 65 - jepsen.util 3215 :invoke :read [21 nil] INFO [2019-12-17 12:30:44,056] jepsen worker 13 - jepsen.util 3523 :info :write [4 23] :RequestCanceledException INFO [2019-12-17 12:30:44,057] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21951,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:30:44,057] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21951,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:30:44,060] jepsen worker 13 - jepsen.util 3613 :invoke :write [4 1] INFO [2019-12-17 12:30:44,072] jepsen worker 65 - jepsen.util 3215 :ok :read [21 48] INFO [2019-12-17 12:30:44,072] jepsen worker 65 - jepsen.util 3215 :invoke :read [21 nil] INFO [2019-12-17 12:30:44,092] jepsen worker 29 - jepsen.util 3179 :info :cas [9 [24 36]] :RequestCanceledException INFO [2019-12-17 12:30:44,095] jepsen worker 29 - jepsen.util 3269 :invoke :cas [9 [3 24]] INFO [2019-12-17 12:30:44,104] jepsen worker 65 - jepsen.util 3215 :ok :read [21 48] INFO [2019-12-17 12:30:44,104] jepsen worker 65 - jepsen.util 3215 :invoke :cas [21 [25 18]] INFO [2019-12-17 12:30:44,106] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:30:44,106] jepsen nemesis - couchbase.util Rebalance check count: 163 INFO [2019-12-17 12:30:44,106] jepsen nemesis - couchbase.util Rebalance stuck count: 144 INFO [2019-12-17 12:30:44,125] jepsen worker 56 - jepsen.util 3926 :info :write [18 43] :RequestCanceledException INFO [2019-12-17 12:30:44,128] jepsen worker 56 - jepsen.util 4016 :invoke :write [18 22] INFO [2019-12-17 12:30:44,133] jepsen worker 29 - jepsen.util 3269 :fail :cas [9 [3 24]] :ValueNotSwapFrom INFO [2019-12-17 12:30:44,133] jepsen worker 29 - jepsen.util 3269 :invoke :cas [9 [44 7]] INFO [2019-12-17 12:30:44,142] jepsen worker 65 - jepsen.util 3215 :fail :cas [21 [25 18]] :ValueNotSwapFrom INFO [2019-12-17 12:30:44,143] jepsen worker 65 - jepsen.util 3215 :invoke :cas [21 [39 10]] INFO [2019-12-17 12:30:44,163] jepsen worker 29 - jepsen.util 3269 :fail :cas [9 [44 7]] :ValueNotSwapFrom INFO [2019-12-17 12:30:44,164] jepsen worker 29 - jepsen.util 3269 :invoke :cas [9 [43 44]] INFO [2019-12-17 12:30:44,164] jepsen worker 80 - jepsen.util 3950 :fail :cas [26 [46 23]] :ValueNotSwapFrom INFO [2019-12-17 12:30:44,164] jepsen worker 80 - jepsen.util 3950 :invoke :read [26 nil] INFO [2019-12-17 12:30:44,170] jepsen worker 65 - jepsen.util 3215 :fail :cas [21 [39 10]] :ValueNotSwapFrom INFO [2019-12-17 12:30:44,171] jepsen worker 65 - jepsen.util 3215 :invoke :cas [21 [18 40]] INFO [2019-12-17 12:30:44,201] jepsen worker 29 - jepsen.util 3269 :fail :cas [9 [43 44]] :ValueNotSwapFrom INFO [2019-12-17 12:30:44,201] jepsen worker 29 - jepsen.util 3269 :invoke :write [9 12] INFO [2019-12-17 12:30:44,212] jepsen worker 65 - jepsen.util 3215 :fail :cas [21 [18 40]] :ValueNotSwapFrom INFO [2019-12-17 12:30:44,212] jepsen worker 65 - jepsen.util 3215 :invoke :cas [21 [44 31]] INFO [2019-12-17 12:30:44,216] jepsen worker 5 - jepsen.util 3515 :ok :read [1 45] INFO [2019-12-17 12:30:44,217] jepsen worker 5 - jepsen.util 3515 :invoke :cas [1 [16 29]] INFO [2019-12-17 12:30:44,224] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21976,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:30:44,224] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21976,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:30:44,224] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21980,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:30:44,225] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21980,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:30:44,225] jepsen worker 61 - jepsen.util 5821 :info :cas [20 [40 34]] :RequestCanceledException INFO [2019-12-17 12:30:44,227] jepsen worker 61 - jepsen.util 5911 :invoke :cas [20 [32 5]] INFO [2019-12-17 12:30:44,239] jepsen worker 78 - jepsen.util 4308 :info :write [26 10] :RequestCanceledException INFO [2019-12-17 12:30:44,244] jepsen worker 78 - jepsen.util 4398 :invoke :cas [26 [15 44]] INFO [2019-12-17 12:30:44,244] jepsen worker 65 - jepsen.util 3215 :fail :cas [21 [44 31]] :ValueNotSwapFrom INFO [2019-12-17 12:30:44,245] jepsen worker 65 - jepsen.util 3215 :invoke :read [21 nil] INFO [2019-12-17 12:30:44,253] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21969,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:30:44,254] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21969,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:30:44,254] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21971,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:30:44,254] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21971,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:30:44,290] jepsen worker 80 - jepsen.util 3950 :ok :read [26 28] INFO [2019-12-17 12:30:44,291] jepsen worker 80 - jepsen.util 3950 :invoke :read [26 nil] INFO [2019-12-17 12:30:44,298] jepsen worker 65 - jepsen.util 3215 :ok :read [21 48] INFO [2019-12-17 12:30:44,298] jepsen worker 65 - jepsen.util 3215 :invoke :cas [21 [11 44]] INFO [2019-12-17 12:30:44,315] jepsen worker 29 - jepsen.util 3269 :ok :write [9 12] INFO [2019-12-17 12:30:44,316] jepsen worker 29 - jepsen.util 3269 :invoke :cas [9 [17 12]] INFO [2019-12-17 12:30:44,324] jepsen worker 86 - jepsen.util 5306 :info :write [28 48] :RequestCanceledException INFO [2019-12-17 12:30:44,329] jepsen worker 86 - jepsen.util 5396 :invoke :write [28 19] INFO [2019-12-17 12:30:44,347] jepsen worker 65 - jepsen.util 3215 :fail :cas [21 [11 44]] :ValueNotSwapFrom INFO [2019-12-17 12:30:44,347] jepsen worker 65 - jepsen.util 3215 :invoke :read [21 nil] INFO [2019-12-17 12:30:44,355] jepsen worker 87 - jepsen.util 3597 :info :write [29 40] :RequestCanceledException INFO [2019-12-17 12:30:44,360] jepsen worker 87 - jepsen.util 3687 :invoke :cas [29 [29 33]] INFO [2019-12-17 12:30:44,362] jepsen worker 29 - jepsen.util 3269 :fail :cas [9 [17 12]] :ValueNotSwapFrom INFO [2019-12-17 12:30:44,362] jepsen worker 29 - jepsen.util 3269 :invoke :write [9 21] INFO [2019-12-17 12:30:44,369] jepsen worker 80 - jepsen.util 3950 :ok :read [26 28] INFO [2019-12-17 12:30:44,369] jepsen worker 80 - jepsen.util 3950 :invoke :read [26 nil] INFO [2019-12-17 12:30:44,379] jepsen worker 65 - jepsen.util 3215 :ok :read [21 48] INFO [2019-12-17 12:30:44,379] jepsen worker 65 - jepsen.util 3215 :invoke :cas [21 [24 14]] INFO [2019-12-17 12:30:44,391] jepsen worker 77 - jepsen.util 5027 :info :write [25 41] :RequestCanceledException INFO [2019-12-17 12:30:44,392] jepsen worker 74 - jepsen.util 4214 :info :write [24 33] :RequestCanceledException INFO [2019-12-17 12:30:44,393] jepsen worker 77 - jepsen.util 5117 :invoke :write [25 25] INFO [2019-12-17 12:30:44,394] jepsen worker 74 - jepsen.util 4304 :invoke :cas [24 [38 13]] INFO [2019-12-17 12:30:44,425] jepsen worker 24 - jepsen.util 4794 :info :write [8 1] :RequestCanceledException INFO [2019-12-17 12:30:44,425] jepsen worker 36 - jepsen.util 3996 :info :write [12 13] :RequestCanceledException INFO [2019-12-17 12:30:44,425] jepsen worker 65 - jepsen.util 3215 :fail :cas [21 [24 14]] :ValueNotSwapFrom INFO [2019-12-17 12:30:44,426] jepsen worker 65 - jepsen.util 3215 :invoke :cas [21 [34 28]] INFO [2019-12-17 12:30:44,427] jepsen worker 24 - jepsen.util 4884 :invoke :write [8 47] INFO [2019-12-17 12:30:44,429] jepsen worker 36 - jepsen.util 4086 :invoke :read [12 nil] INFO [2019-12-17 12:30:44,448] jepsen worker 29 - jepsen.util 3269 :ok :write [9 21] INFO [2019-12-17 12:30:44,449] jepsen worker 29 - jepsen.util 3269 :invoke :cas [9 [42 7]] INFO [2019-12-17 12:30:44,454] jepsen worker 12 - jepsen.util 4062 :info :write [4 32] :RequestCanceledException INFO [2019-12-17 12:30:44,459] jepsen worker 12 - jepsen.util 4152 :invoke :write [4 2] INFO [2019-12-17 12:30:44,460] jepsen worker 65 - jepsen.util 3215 :fail :cas [21 [34 28]] :ValueNotSwapFrom INFO [2019-12-17 12:30:44,460] jepsen worker 65 - jepsen.util 3215 :invoke :write [21 27] INFO [2019-12-17 12:30:44,464] jepsen worker 80 - jepsen.util 3950 :ok :read [26 28] INFO [2019-12-17 12:30:44,465] jepsen worker 80 - jepsen.util 3950 :invoke :read [26 nil] INFO [2019-12-17 12:30:44,483] jepsen worker 29 - jepsen.util 3269 :fail :cas [9 [42 7]] :ValueNotSwapFrom INFO [2019-12-17 12:30:44,484] jepsen worker 29 - jepsen.util 3269 :invoke :cas [9 [44 47]] INFO [2019-12-17 12:30:44,491] jepsen worker 49 - jepsen.util 4459 :info :write [16 31] :RequestCanceledException INFO [2019-12-17 12:30:44,492] jepsen worker 49 - jepsen.util 4549 :invoke :read [16 nil] INFO [2019-12-17 12:30:44,494] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21999,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:30:44,494] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":21999,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:30:44,494] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":22003,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:30:44,494] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":22003,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:30:44,519] jepsen worker 29 - jepsen.util 3269 :fail :cas [9 [44 47]] :ValueNotSwapFrom INFO [2019-12-17 12:30:44,520] jepsen worker 29 - jepsen.util 3269 :invoke :write [9 1] INFO [2019-12-17 12:30:44,555] jepsen worker 87 - jepsen.util 3687 :fail :cas [29 [29 33]] :ValueNotSwapFrom INFO [2019-12-17 12:30:44,555] jepsen worker 87 - jepsen.util 3687 :invoke :cas [29 [4 19]] INFO [2019-12-17 12:30:44,555] jepsen worker 34 - jepsen.util 4984 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:30:44,555] jepsen worker 34 - jepsen.util 4984 :invoke :read [11 nil] INFO [2019-12-17 12:30:44,560] jepsen worker 65 - jepsen.util 3215 :ok :write [21 27] INFO [2019-12-17 12:30:44,560] jepsen worker 65 - jepsen.util 3215 :invoke :cas [21 [10 11]] INFO [2019-12-17 12:30:44,587] jepsen worker 89 - jepsen.util 3689 :ok :read [29 18] INFO [2019-12-17 12:30:44,587] jepsen worker 89 - jepsen.util 3689 :invoke :cas [29 [38 26]] INFO [2019-12-17 12:30:44,591] jepsen worker 29 - jepsen.util 3269 :ok :write [9 1] INFO [2019-12-17 12:30:44,592] jepsen worker 29 - jepsen.util 3269 :invoke :read [9 nil] INFO [2019-12-17 12:30:44,603] jepsen worker 65 - jepsen.util 3215 :fail :cas [21 [10 11]] :ValueNotSwapFrom INFO [2019-12-17 12:30:44,603] jepsen worker 65 - jepsen.util 3215 :invoke :cas [21 [19 7]] INFO [2019-12-17 12:30:44,620] jepsen worker 29 - jepsen.util 3269 :ok :read [9 1] INFO [2019-12-17 12:30:44,620] jepsen worker 29 - jepsen.util 3269 :invoke :write [9 12] INFO [2019-12-17 12:30:44,622] jepsen worker 88 - jepsen.util 3868 :info :write [29 25] :RequestCanceledException INFO [2019-12-17 12:30:44,625] jepsen worker 88 - jepsen.util 3958 :invoke :cas [29 [34 37]] INFO [2019-12-17 12:30:44,639] jepsen worker 70 - jepsen.util 4390 :info :write [23 14] :RequestCanceledException INFO [2019-12-17 12:30:44,643] jepsen worker 70 - jepsen.util 4480 :invoke :cas [23 [22 2]] INFO [2019-12-17 12:30:44,649] jepsen worker 46 - jepsen.util 4006 :ok :write [15 47] INFO [2019-12-17 12:30:44,649] jepsen worker 46 - jepsen.util 4006 :invoke :read [15 nil] INFO [2019-12-17 12:30:44,650] jepsen worker 9 - jepsen.util 4329 :ok :write [3 2] INFO [2019-12-17 12:30:44,651] jepsen worker 9 - jepsen.util 4329 :invoke :read [3 nil] INFO [2019-12-17 12:30:44,654] jepsen worker 30 - jepsen.util 4260 :ok :write [10 19] INFO [2019-12-17 12:30:44,654] jepsen worker 80 - jepsen.util 3950 :ok :read [26 28] INFO [2019-12-17 12:30:44,654] jepsen worker 30 - jepsen.util 4260 :invoke :write [10 0] INFO [2019-12-17 12:30:44,654] jepsen worker 80 - jepsen.util 3950 :invoke :cas [26 [1 10]] INFO [2019-12-17 12:30:44,658] jepsen worker 65 - jepsen.util 3215 :fail :cas [21 [19 7]] :ValueNotSwapFrom INFO [2019-12-17 12:30:44,659] jepsen worker 65 - jepsen.util 3215 :invoke :cas [21 [8 33]] INFO [2019-12-17 12:30:44,693] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":22025,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:30:44,693] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":22025,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:30:44,693] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":22026,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:30:44,693] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":22026,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:30:44,693] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":22027,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:30:44,693] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":22027,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:30:44,709] jepsen worker 55 - jepsen.util 4645 :info :write [18 42] :RequestCanceledException INFO [2019-12-17 12:30:44,711] jepsen worker 88 - jepsen.util 3958 :fail :cas [29 [34 37]] :ValueNotSwapFrom INFO [2019-12-17 12:30:44,711] jepsen worker 88 - jepsen.util 3958 :invoke :write [29 27] INFO [2019-12-17 12:30:44,712] jepsen worker 55 - jepsen.util 4735 :invoke :read [18 nil] INFO [2019-12-17 12:30:44,739] jepsen worker 11 - jepsen.util 4331 :fail :cas [3 [26 23]] :ValueNotSwapFrom INFO [2019-12-17 12:30:44,740] jepsen worker 11 - jepsen.util 4331 :invoke :read [3 nil] INFO [2019-12-17 12:30:44,756] jepsen worker 53 - jepsen.util 3923 :info :write [17 10] :RequestCanceledException INFO [2019-12-17 12:30:44,760] jepsen worker 53 - jepsen.util 4013 :invoke :cas [17 [29 18]] INFO [2019-12-17 12:30:44,763] jepsen worker 89 - jepsen.util 3689 :fail :cas [29 [38 26]] :ValueNotSwapFrom INFO [2019-12-17 12:30:44,764] jepsen worker 89 - jepsen.util 3689 :invoke :read [29 nil] INFO [2019-12-17 12:30:44,775] jepsen worker 74 - jepsen.util 4304 :fail :cas [24 [38 13]] :ValueNotSwapFrom INFO [2019-12-17 12:30:44,775] jepsen worker 74 - jepsen.util 4304 :invoke :read [24 nil] INFO [2019-12-17 12:30:44,780] jepsen worker 9 - jepsen.util 4329 :ok :read [3 2] INFO [2019-12-17 12:30:44,781] jepsen worker 9 - jepsen.util 4329 :invoke :read [3 nil] INFO [2019-12-17 12:30:44,783] jepsen worker 80 - jepsen.util 3950 :fail :cas [26 [1 10]] :ValueNotSwapFrom INFO [2019-12-17 12:30:44,783] jepsen worker 80 - jepsen.util 3950 :invoke :read [26 nil] INFO [2019-12-17 12:30:44,797] jepsen worker 11 - jepsen.util 4331 :ok :read [3 2] INFO [2019-12-17 12:30:44,798] jepsen worker 11 - jepsen.util 4331 :invoke :cas [3 [47 38]] INFO [2019-12-17 12:30:44,802] jepsen worker 18 - jepsen.util 3978 :fail :cas [6 [44 40]] :ValueNotSwapFrom INFO [2019-12-17 12:30:44,802] jepsen worker 18 - jepsen.util 3978 :invoke :read [6 nil] INFO [2019-12-17 12:30:44,808] jepsen worker 68 - jepsen.util 4658 :info :cas [22 [41 41]] :RequestCanceledException INFO [2019-12-17 12:30:44,810] jepsen worker 68 - jepsen.util 4748 :invoke :read [22 nil] INFO [2019-12-17 12:30:44,834] jepsen worker 65 - jepsen.util 3215 :fail :cas [21 [8 33]] :ValueNotSwapFrom INFO [2019-12-17 12:30:44,835] jepsen worker 65 - jepsen.util 3215 :invoke :read [21 nil] INFO [2019-12-17 12:30:44,840] jepsen worker 23 - jepsen.util 5063 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:30:44,840] jepsen worker 23 - jepsen.util 5063 :invoke :read [7 nil] INFO [2019-12-17 12:30:44,854] jepsen worker 9 - jepsen.util 4329 :ok :read [3 2] INFO [2019-12-17 12:30:44,854] jepsen worker 9 - jepsen.util 4329 :invoke :cas [3 [27 8]] INFO [2019-12-17 12:30:44,860] jepsen worker 11 - jepsen.util 4331 :fail :cas [3 [47 38]] :ValueNotSwapFrom INFO [2019-12-17 12:30:44,860] jepsen worker 11 - jepsen.util 4331 :invoke :read [3 nil] INFO [2019-12-17 12:30:44,861] jepsen worker 46 - jepsen.util 4006 :ok :read [15 47] INFO [2019-12-17 12:30:44,861] jepsen worker 46 - jepsen.util 4006 :invoke :write [15 21] INFO [2019-12-17 12:30:44,863] jepsen worker 29 - jepsen.util 3269 :ok :write [9 12] INFO [2019-12-17 12:30:44,863] jepsen worker 29 - jepsen.util 3269 :invoke :cas [9 [1 36]] INFO [2019-12-17 12:30:44,885] jepsen worker 18 - jepsen.util 3978 :ok :read [6 14] INFO [2019-12-17 12:30:44,885] jepsen worker 18 - jepsen.util 3978 :invoke :read [6 nil] INFO [2019-12-17 12:30:44,909] jepsen worker 3 - jepsen.util 4053 :info :write [1 9] :RequestCanceledException INFO [2019-12-17 12:30:44,909] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":22049,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:30:44,909] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":22049,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:30:44,909] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":22052,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:30:44,909] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":22052,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:30:44,911] jepsen worker 3 - jepsen.util 4143 :invoke :read [1 nil] INFO [2019-12-17 12:30:44,915] jepsen worker 30 - jepsen.util 4260 :ok :write [10 0] INFO [2019-12-17 12:30:44,915] jepsen worker 30 - jepsen.util 4260 :invoke :cas [10 [33 6]] INFO [2019-12-17 12:30:44,925] jepsen worker 59 - jepsen.util 5099 :info :write [19 15] :RequestCanceledException INFO [2019-12-17 12:30:44,925] jepsen worker 44 - jepsen.util 5444 :info :write [14 28] :RequestCanceledException INFO [2019-12-17 12:30:44,929] jepsen worker 44 - jepsen.util 5534 :invoke :write [14 34] INFO [2019-12-17 12:30:44,929] jepsen worker 59 - jepsen.util 5189 :invoke :cas [19 [44 39]] INFO [2019-12-17 12:30:44,949] jepsen worker 29 - jepsen.util 3269 :fail :cas [9 [1 36]] :ValueNotSwapFrom INFO [2019-12-17 12:30:44,950] jepsen worker 29 - jepsen.util 3269 :invoke :write [9 40] INFO [2019-12-17 12:30:44,953] jepsen worker 9 - jepsen.util 4329 :fail :cas [3 [27 8]] :ValueNotSwapFrom INFO [2019-12-17 12:30:44,953] jepsen worker 9 - jepsen.util 4329 :invoke :read [3 nil] INFO [2019-12-17 12:30:44,965] jepsen worker 65 - jepsen.util 3215 :ok :read [21 27] INFO [2019-12-17 12:30:44,966] jepsen worker 65 - jepsen.util 3215 :invoke :cas [21 [28 31]] INFO [2019-12-17 12:30:44,971] jepsen worker 11 - jepsen.util 4331 :ok :read [3 2] INFO [2019-12-17 12:30:44,971] jepsen worker 11 - jepsen.util 4331 :invoke :cas [3 [2 22]] INFO [2019-12-17 12:30:44,983] jepsen worker 74 - jepsen.util 4304 :ok :read [24 41] INFO [2019-12-17 12:30:44,984] jepsen worker 74 - jepsen.util 4304 :invoke :read [24 nil] INFO [2019-12-17 12:30:44,988] jepsen worker 84 - jepsen.util 4674 :info :write [28 25] :RequestCanceledException INFO [2019-12-17 12:30:44,991] jepsen worker 84 - jepsen.util 4764 :invoke :cas [28 [36 30]] INFO [2019-12-17 12:30:44,992] jepsen worker 30 - jepsen.util 4260 :fail :cas [10 [33 6]] :ValueNotSwapFrom INFO [2019-12-17 12:30:44,993] jepsen worker 30 - jepsen.util 4260 :invoke :write [10 12] INFO [2019-12-17 12:30:45,003] jepsen worker 9 - jepsen.util 4329 :ok :read [3 2] INFO [2019-12-17 12:30:45,003] jepsen worker 9 - jepsen.util 4329 :invoke :read [3 nil] INFO [2019-12-17 12:30:45,009] jepsen worker 64 - jepsen.util 3034 :info :write [21 8] :RequestCanceledException INFO [2019-12-17 12:30:45,009] jepsen worker 43 - jepsen.util 4813 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:30:45,009] jepsen worker 4 - jepsen.util 3064 :info :write [1 1] :RequestCanceledException INFO [2019-12-17 12:30:45,010] jepsen worker 72 - jepsen.util 4032 :info :write [24 40] :RequestCanceledException INFO [2019-12-17 12:30:45,010] jepsen worker 43 - jepsen.util 4813 :invoke :read [14 nil] INFO [2019-12-17 12:30:45,011] jepsen worker 64 - jepsen.util 3124 :invoke :cas [21 [15 26]] INFO [2019-12-17 12:30:45,012] jepsen worker 72 - jepsen.util 4122 :invoke :read [24 nil] INFO [2019-12-17 12:30:45,014] jepsen worker 4 - jepsen.util 3154 :invoke :read [1 nil] INFO [2019-12-17 12:30:45,024] jepsen worker 66 - jepsen.util 4926 :info :cas [22 [6 30]] :RequestCanceledException INFO [2019-12-17 12:30:45,030] jepsen worker 66 - jepsen.util 5016 :invoke :write [22 22] INFO [2019-12-17 12:30:45,038] jepsen worker 51 - jepsen.util 3381 :info :write [17 37] :RequestCanceledException INFO [2019-12-17 12:30:45,041] jepsen worker 51 - jepsen.util 3471 :invoke :write [17 43] INFO [2019-12-17 12:30:45,043] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":22068,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:30:45,043] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":22068,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:30:45,043] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":22070,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:30:45,043] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":22070,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:30:45,053] jepsen worker 20 - jepsen.util 3710 :info :cas [6 [25 10]] :RequestCanceledException INFO [2019-12-17 12:30:45,055] jepsen worker 20 - jepsen.util 3800 :invoke :cas [6 [44 20]] INFO [2019-12-17 12:30:45,058] jepsen worker 79 - jepsen.util 3859 :fail :cas [26 [14 16]] :ValueNotSwapFrom INFO [2019-12-17 12:30:45,058] jepsen worker 79 - jepsen.util 3859 :invoke :read [26 nil] INFO [2019-12-17 12:30:45,091] jepsen worker 25 - jepsen.util 5335 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:30:45,092] jepsen worker 25 - jepsen.util 5335 :invoke :cas [8 [49 45]] INFO [2019-12-17 12:30:45,094] jepsen worker 80 - jepsen.util 3950 :ok :read [26 28] INFO [2019-12-17 12:30:45,094] jepsen worker 80 - jepsen.util 3950 :invoke :write [26 39] INFO [2019-12-17 12:30:45,105] jepsen worker 65 - jepsen.util 3215 :fail :cas [21 [28 31]] :ValueNotSwapFrom INFO [2019-12-17 12:30:45,105] jepsen worker 65 - jepsen.util 3215 :invoke :read [21 nil] INFO [2019-12-17 12:30:45,109] jepsen worker 74 - jepsen.util 4304 :ok :read [24 41] INFO [2019-12-17 12:30:45,109] jepsen worker 74 - jepsen.util 4304 :invoke :cas [24 [36 13]] INFO [2019-12-17 12:30:45,114] jepsen worker 38 - jepsen.util 3818 :fail :cas [12 [20 32]] :ValueNotSwapFrom INFO [2019-12-17 12:30:45,115] jepsen worker 38 - jepsen.util 3818 :invoke :cas [12 [6 2]] INFO [2019-12-17 12:30:45,121] jepsen worker 9 - jepsen.util 4329 :ok :read [3 2] INFO [2019-12-17 12:30:45,121] jepsen worker 9 - jepsen.util 4329 :invoke :write [3 48] INFO [2019-12-17 12:30:45,124] jepsen worker 54 - jepsen.util 4194 :info :write [18 19] :RequestCanceledException INFO [2019-12-17 12:30:45,124] jepsen worker 62 - jepsen.util 4832 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:30:45,124] jepsen worker 62 - jepsen.util 4832 :invoke :read [20 nil] INFO [2019-12-17 12:30:45,126] jepsen worker 54 - jepsen.util 4284 :invoke :write [18 38] INFO [2019-12-17 12:30:45,138] jepsen worker 39 - jepsen.util 4809 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:30:45,138] jepsen worker 39 - jepsen.util 4809 :invoke :write [13 14] INFO [2019-12-17 12:30:45,153] jepsen worker 6 - jepsen.util 5496 :info :cas [2 [48 31]] :RequestCanceledException INFO [2019-12-17 12:30:45,157] jepsen worker 20 - jepsen.util 3800 :fail :cas [6 [44 20]] :ValueNotSwapFrom INFO [2019-12-17 12:30:45,157] jepsen worker 20 - jepsen.util 3800 :invoke :cas [6 [3 37]] INFO [2019-12-17 12:30:45,157] jepsen worker 6 - jepsen.util 5586 :invoke :write [2 5] INFO [2019-12-17 12:30:45,166] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:30:45,166] jepsen nemesis - couchbase.util Rebalance check count: 164 INFO [2019-12-17 12:30:45,166] jepsen nemesis - couchbase.util Rebalance stuck count: 145 INFO [2019-12-17 12:30:45,173] jepsen worker 78 - jepsen.util 4398 :fail :cas [26 [15 44]] :ValueNotSwapFrom INFO [2019-12-17 12:30:45,174] jepsen worker 78 - jepsen.util 4398 :invoke :cas [26 [31 21]] INFO [2019-12-17 12:30:45,188] jepsen worker 36 - jepsen.util 4086 :ok :read [12 1] INFO [2019-12-17 12:30:45,189] jepsen worker 36 - jepsen.util 4086 :invoke :cas [12 [15 10]] INFO [2019-12-17 12:30:45,193] jepsen worker 65 - jepsen.util 3215 :ok :read [21 27] INFO [2019-12-17 12:30:45,193] jepsen worker 65 - jepsen.util 3215 :invoke :write [21 12] INFO [2019-12-17 12:30:45,204] jepsen worker 29 - jepsen.util 3269 :ok :write [9 40] INFO [2019-12-17 12:30:45,204] jepsen worker 29 - jepsen.util 3269 :invoke :read [9 nil] INFO [2019-12-17 12:30:45,207] jepsen worker 20 - jepsen.util 3800 :fail :cas [6 [3 37]] :ValueNotSwapFrom INFO [2019-12-17 12:30:45,208] jepsen worker 20 - jepsen.util 3800 :invoke :read [6 nil] INFO [2019-12-17 12:30:45,209] jepsen worker 82 - jepsen.util 5932 :info :cas [27 [9 0]] :RequestCanceledException INFO [2019-12-17 12:30:45,212] jepsen worker 82 - jepsen.util 6022 :invoke :cas [27 [49 1]] INFO [2019-12-17 12:30:45,215] jepsen worker 64 - jepsen.util 3124 :fail :cas [21 [15 26]] :ValueNotSwapFrom INFO [2019-12-17 12:30:45,215] jepsen worker 64 - jepsen.util 3124 :invoke :read [21 nil] INFO [2019-12-17 12:30:45,215] jepsen worker 7 - jepsen.util 4957 :info :write [2 1] :RequestCanceledException INFO [2019-12-17 12:30:45,219] jepsen worker 7 - jepsen.util 5047 :invoke :cas [2 [36 33]] INFO [2019-12-17 12:30:45,236] jepsen worker 53 - jepsen.util 4013 :fail :cas [17 [29 18]] :ValueNotSwapFrom INFO [2019-12-17 12:30:45,237] jepsen worker 53 - jepsen.util 4013 :invoke :read [17 nil] INFO [2019-12-17 12:30:45,238] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":22099,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:30:45,238] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":22099,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:30:45,263] jepsen worker 70 - jepsen.util 4480 :fail :cas [23 [22 2]] :ValueNotSwapFrom INFO [2019-12-17 12:30:45,263] jepsen worker 70 - jepsen.util 4480 :invoke :read [23 nil] INFO [2019-12-17 12:30:45,269] jepsen worker 5 - jepsen.util 3515 :fail :cas [1 [16 29]] :ValueNotSwapFrom INFO [2019-12-17 12:30:45,269] jepsen worker 5 - jepsen.util 3515 :invoke :cas [1 [49 36]] INFO [2019-12-17 12:30:45,291] jepsen worker 50 - jepsen.util 5630 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:30:45,291] jepsen worker 20 - jepsen.util 3800 :ok :read [6 14] INFO [2019-12-17 12:30:45,291] jepsen worker 50 - jepsen.util 5630 :invoke :read [16 nil] INFO [2019-12-17 12:30:45,292] jepsen worker 20 - jepsen.util 3800 :invoke :write [6 23] INFO [2019-12-17 12:30:45,292] jepsen worker 29 - jepsen.util 3269 :ok :read [9 40] INFO [2019-12-17 12:30:45,292] jepsen worker 29 - jepsen.util 3269 :invoke :cas [9 [15 42]] INFO [2019-12-17 12:30:45,306] jepsen worker 85 - jepsen.util 5665 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:30:45,306] jepsen worker 85 - jepsen.util 5665 :invoke :cas [28 [3 6]] INFO [2019-12-17 12:30:45,314] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":22107,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:30:45,314] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":22107,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:30:45,315] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":22110,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:30:45,315] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":22110,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:30:45,324] jepsen worker 55 - jepsen.util 4735 :ok :read [18 48] INFO [2019-12-17 12:30:45,325] jepsen worker 55 - jepsen.util 4735 :invoke :cas [18 [40 38]] INFO [2019-12-17 12:30:45,335] jepsen worker 65 - jepsen.util 3215 :ok :write [21 12] INFO [2019-12-17 12:30:45,335] jepsen worker 65 - jepsen.util 3215 :invoke :write [21 39] INFO [2019-12-17 12:30:45,339] jepsen worker 29 - jepsen.util 3269 :fail :cas [9 [15 42]] :ValueNotSwapFrom INFO [2019-12-17 12:30:45,339] jepsen worker 89 - jepsen.util 3689 :ok :read [29 18] INFO [2019-12-17 12:30:45,339] jepsen worker 89 - jepsen.util 3689 :invoke :read [29 nil] INFO [2019-12-17 12:30:45,339] jepsen worker 29 - jepsen.util 3269 :invoke :write [9 38] INFO [2019-12-17 12:30:45,355] jepsen worker 38 - jepsen.util 3818 :fail :cas [12 [6 2]] :ValueNotSwapFrom INFO [2019-12-17 12:30:45,355] jepsen worker 38 - jepsen.util 3818 :invoke :read [12 nil] INFO [2019-12-17 12:30:45,386] jepsen worker 69 - jepsen.util 3849 :fail :cas [23 [10 10]] :ValueNotSwapFrom INFO [2019-12-17 12:30:45,386] jepsen worker 69 - jepsen.util 3849 :invoke :read [23 nil] INFO [2019-12-17 12:30:45,424] jepsen worker 52 - jepsen.util 3922 :info :write [17 12] :RequestCanceledException INFO [2019-12-17 12:30:45,424] jepsen worker 40 - jepsen.util 5530 :info :cas [13 [2 28]] :RequestCanceledException INFO [2019-12-17 12:30:45,427] jepsen worker 52 - jepsen.util 4012 :invoke :write [17 49] INFO [2019-12-17 12:30:45,427] jepsen worker 40 - jepsen.util 5620 :invoke :write [13 31] INFO [2019-12-17 12:30:45,429] jepsen worker 79 - jepsen.util 3859 :ok :read [26 28] INFO [2019-12-17 12:30:45,430] jepsen worker 79 - jepsen.util 3859 :invoke :cas [26 [9 18]] INFO [2019-12-17 12:30:45,439] jepsen worker 14 - jepsen.util 3884 :info :write [4 43] :RequestCanceledException INFO [2019-12-17 12:30:45,439] jepsen worker 48 - jepsen.util 4638 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:30:45,439] jepsen worker 48 - jepsen.util 4638 :invoke :write [16 43] INFO [2019-12-17 12:30:45,440] jepsen worker 29 - jepsen.util 3269 :ok :write [9 38] INFO [2019-12-17 12:30:45,440] jepsen worker 29 - jepsen.util 3269 :invoke :read [9 nil] INFO [2019-12-17 12:30:45,440] jepsen worker 14 - jepsen.util 3974 :invoke :read [4 nil] INFO [2019-12-17 12:30:45,456] jepsen worker 83 - jepsen.util 5123 :info :write [27 23] :RequestCanceledException INFO [2019-12-17 12:30:45,456] jepsen worker 67 - jepsen.util 5377 :info :cas [22 [41 17]] :RequestCanceledException INFO [2019-12-17 12:30:45,456] jepsen worker 63 - jepsen.util 3303 :info :write [21 17] :RequestCanceledException INFO [2019-12-17 12:30:45,458] jepsen worker 67 - jepsen.util 5467 :invoke :read [22 nil] INFO [2019-12-17 12:30:45,458] jepsen worker 63 - jepsen.util 3393 :invoke :read [21 nil] INFO [2019-12-17 12:30:45,458] jepsen worker 83 - jepsen.util 5213 :invoke :write [27 13] INFO [2019-12-17 12:30:45,472] jepsen worker 29 - jepsen.util 3269 :ok :read [9 38] INFO [2019-12-17 12:30:45,472] jepsen worker 29 - jepsen.util 3269 :invoke :cas [9 [20 5]] INFO [2019-12-17 12:30:45,476] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":22131,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:30:45,476] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":22131,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:30:45,476] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":22134,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:30:45,476] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":22134,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:30:45,490] jepsen worker 57 - jepsen.util 4917 :info :write [19 38] :RequestCanceledException INFO [2019-12-17 12:30:45,490] jepsen worker 35 - jepsen.util 5075 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:30:45,491] jepsen worker 35 - jepsen.util 5075 :invoke :write [11 2] INFO [2019-12-17 12:30:45,495] jepsen worker 57 - jepsen.util 5007 :invoke :read [19 nil] INFO [2019-12-17 12:30:45,505] jepsen worker 87 - jepsen.util 3687 :fail :cas [29 [4 19]] :ValueNotSwapFrom INFO [2019-12-17 12:30:45,506] jepsen worker 87 - jepsen.util 3687 :invoke :cas [29 [3 18]] INFO [2019-12-17 12:30:45,506] jepsen worker 30 - jepsen.util 4260 :ok :write [10 12] INFO [2019-12-17 12:30:45,506] jepsen worker 30 - jepsen.util 4260 :invoke :write [10 31] INFO [2019-12-17 12:30:45,508] jepsen worker 0 - jepsen.util 5580 :info :write [0 42] :RequestCanceledException INFO [2019-12-17 12:30:45,508] jepsen worker 26 - jepsen.util 5336 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:30:45,508] jepsen worker 60 - jepsen.util 5460 :info :write [20 43] :RequestCanceledException INFO [2019-12-17 12:30:45,508] jepsen worker 26 - jepsen.util 5336 :invoke :cas [8 [0 26]] INFO [2019-12-17 12:30:45,513] jepsen worker 0 - jepsen.util 5670 :invoke :cas [0 [48 6]] INFO [2019-12-17 12:30:45,513] jepsen worker 60 - jepsen.util 5550 :invoke :write [20 9] INFO [2019-12-17 12:30:45,518] jepsen worker 46 - jepsen.util 4006 :ok :write [15 21] INFO [2019-12-17 12:30:45,518] jepsen worker 46 - jepsen.util 4006 :invoke :cas [15 [26 18]] INFO [2019-12-17 12:30:45,525] jepsen worker 22 - jepsen.util 5242 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:30:45,525] jepsen worker 2 - jepsen.util 4862 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:30:45,525] jepsen worker 22 - jepsen.util 5242 :invoke :read [7 nil] INFO [2019-12-17 12:30:45,525] jepsen worker 2 - jepsen.util 4862 :invoke :read [0 nil] INFO [2019-12-17 12:30:45,539] jepsen worker 17 - jepsen.util 4877 :info :cas [5 [17 4]] :RequestCanceledException INFO [2019-12-17 12:30:45,541] jepsen worker 69 - jepsen.util 3849 :ok :read [23 47] INFO [2019-12-17 12:30:45,541] jepsen worker 69 - jepsen.util 3849 :invoke :read [23 nil] INFO [2019-12-17 12:30:45,542] jepsen worker 17 - jepsen.util 4967 :invoke :read [5 nil] INFO [2019-12-17 12:30:45,546] jepsen worker 29 - jepsen.util 3269 :fail :cas [9 [20 5]] :ValueNotSwapFrom INFO [2019-12-17 12:30:45,547] jepsen worker 29 - jepsen.util 3269 :invoke :read [9 nil] INFO [2019-12-17 12:30:45,554] jepsen worker 72 - jepsen.util 4122 :ok :read [24 41] INFO [2019-12-17 12:30:45,554] jepsen worker 72 - jepsen.util 4122 :invoke :cas [24 [1 37]] INFO [2019-12-17 12:30:45,555] jepsen worker 33 - jepsen.util 5343 :info :write [11 19] :RequestCanceledException INFO [2019-12-17 12:30:45,558] jepsen worker 33 - jepsen.util 5433 :invoke :write [11 3] INFO [2019-12-17 12:30:45,562] jepsen worker 74 - jepsen.util 4304 :fail :cas [24 [36 13]] :ValueNotSwapFrom INFO [2019-12-17 12:30:45,562] jepsen worker 74 - jepsen.util 4304 :invoke :write [24 27] INFO [2019-12-17 12:30:45,582] jepsen worker 46 - jepsen.util 4006 :fail :cas [15 [26 18]] :ValueNotSwapFrom INFO [2019-12-17 12:30:45,583] jepsen worker 46 - jepsen.util 4006 :invoke :cas [15 [13 44]] INFO [2019-12-17 12:30:45,591] jepsen worker 58 - jepsen.util 5098 :info :cas [19 [30 18]] :RequestCanceledException INFO [2019-12-17 12:30:45,596] jepsen worker 58 - jepsen.util 5188 :invoke :cas [19 [10 37]] INFO [2019-12-17 12:30:45,609] jepsen worker 55 - jepsen.util 4735 :fail :cas [18 [40 38]] :ValueNotSwapFrom INFO [2019-12-17 12:30:45,609] jepsen worker 55 - jepsen.util 4735 :invoke :write [18 36] INFO [2019-12-17 12:30:45,624] jepsen worker 42 - jepsen.util 5982 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:30:45,625] jepsen worker 42 - jepsen.util 5982 :invoke :read [14 nil] INFO [2019-12-17 12:30:45,626] jepsen worker 10 - jepsen.util 3700 :info :cas [3 [24 4]] :RequestCanceledException INFO [2019-12-17 12:30:45,631] jepsen worker 10 - jepsen.util 3790 :invoke :cas [3 [36 24]] INFO [2019-12-17 12:30:45,638] jepsen worker 19 - jepsen.util 4069 :fail :read [6 nil] :RequestTimeoutException INFO [2019-12-17 12:30:45,638] jepsen worker 19 - jepsen.util 4069 :invoke :write [6 11] INFO [2019-12-17 12:30:45,652] jepsen worker 3 - jepsen.util 4143 :ok :read [1 45] INFO [2019-12-17 12:30:45,653] jepsen worker 3 - jepsen.util 4143 :invoke :read [1 nil] INFO [2019-12-17 12:30:45,654] jepsen worker 32 - jepsen.util 4262 :info :write [10 38] :RequestCanceledException INFO [2019-12-17 12:30:45,659] jepsen worker 32 - jepsen.util 4352 :invoke :cas [10 [28 43]] INFO [2019-12-17 12:30:45,678] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":22170,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:30:45,678] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":22170,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:30:45,690] jepsen worker 37 - jepsen.util 3547 :info :write [12 23] :RequestCanceledException INFO [2019-12-17 12:30:45,690] jepsen worker 75 - jepsen.util 5205 :info :cas [25 [17 30]] :RequestCanceledException INFO [2019-12-17 12:30:45,693] jepsen worker 29 - jepsen.util 3269 :ok :read [9 38] INFO [2019-12-17 12:30:45,693] jepsen worker 29 - jepsen.util 3269 :invoke :read [9 nil] INFO [2019-12-17 12:30:45,697] jepsen worker 75 - jepsen.util 5295 :invoke :read [25 nil] INFO [2019-12-17 12:30:45,699] jepsen worker 37 - jepsen.util 3637 :invoke :write [12 33] INFO [2019-12-17 12:30:45,706] jepsen worker 38 - jepsen.util 3818 :ok :read [12 1] INFO [2019-12-17 12:30:45,706] jepsen worker 38 - jepsen.util 3818 :invoke :read [12 nil] INFO [2019-12-17 12:30:45,709] jepsen worker 72 - jepsen.util 4122 :fail :cas [24 [1 37]] :ValueNotSwapFrom INFO [2019-12-17 12:30:45,710] jepsen worker 72 - jepsen.util 4122 :invoke :cas [24 [42 8]] INFO [2019-12-17 12:30:45,723] jepsen worker 4 - jepsen.util 3154 :ok :read [1 45] INFO [2019-12-17 12:30:45,723] jepsen worker 4 - jepsen.util 3154 :invoke :write [1 12] INFO [2019-12-17 12:30:45,724] jepsen worker 46 - jepsen.util 4006 :fail :cas [15 [13 44]] :ValueNotSwapFrom INFO [2019-12-17 12:30:45,724] jepsen worker 46 - jepsen.util 4006 :invoke :write [15 36] INFO [2019-12-17 12:30:45,724] jepsen worker 41 - jepsen.util 5261 :info :cas [13 [24 14]] :RequestCanceledException INFO [2019-12-17 12:30:45,724] jepsen worker 1 - jepsen.util 4861 :info :cas [0 [12 20]] :RequestCanceledException INFO [2019-12-17 12:30:45,727] jepsen worker 41 - jepsen.util 5351 :invoke :cas [13 [38 35]] INFO [2019-12-17 12:30:45,728] jepsen worker 1 - jepsen.util 4951 :invoke :read [0 nil] INFO [2019-12-17 12:30:45,738] jepsen worker 16 - jepsen.util 5866 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:30:45,738] jepsen worker 16 - jepsen.util 5866 :invoke :write [5 49] INFO [2019-12-17 12:30:45,755] jepsen worker 63 - jepsen.util 3393 :ok :read [21 12] INFO [2019-12-17 12:30:45,755] jepsen worker 63 - jepsen.util 3393 :invoke :read [21 nil] INFO [2019-12-17 12:30:45,756] jepsen worker 73 - jepsen.util 3673 :info :write [24 45] :RequestCanceledException INFO [2019-12-17 12:30:45,756] jepsen worker 69 - jepsen.util 3849 :ok :read [23 47] INFO [2019-12-17 12:30:45,756] jepsen worker 47 - jepsen.util 4367 :info :write [15 12] :RequestCanceledException INFO [2019-12-17 12:30:45,756] jepsen worker 69 - jepsen.util 3849 :invoke :write [23 20] INFO [2019-12-17 12:30:45,759] jepsen worker 47 - jepsen.util 4457 :invoke :cas [15 [48 8]] INFO [2019-12-17 12:30:45,760] jepsen worker 73 - jepsen.util 3763 :invoke :write [24 3] INFO [2019-12-17 12:30:45,773] jepsen worker 29 - jepsen.util 3269 :ok :read [9 38] INFO [2019-12-17 12:30:45,773] jepsen worker 29 - jepsen.util 3269 :invoke :cas [9 [10 26]] INFO [2019-12-17 12:30:45,786] jepsen worker 72 - jepsen.util 4122 :fail :cas [24 [42 8]] :ValueNotSwapFrom INFO [2019-12-17 12:30:45,786] jepsen worker 72 - jepsen.util 4122 :invoke :write [24 49] INFO [2019-12-17 12:30:45,797] jepsen worker 63 - jepsen.util 3393 :ok :read [21 12] INFO [2019-12-17 12:30:45,797] jepsen worker 63 - jepsen.util 3393 :invoke :read [21 nil] INFO [2019-12-17 12:30:45,808] jepsen worker 76 - jepsen.util 5206 :info :write [25 24] :RequestCanceledException INFO [2019-12-17 12:30:45,811] jepsen worker 29 - jepsen.util 3269 :fail :cas [9 [10 26]] :ValueNotSwapFrom INFO [2019-12-17 12:30:45,811] jepsen worker 76 - jepsen.util 5296 :invoke :cas [25 [3 3]] INFO [2019-12-17 12:30:45,811] jepsen worker 29 - jepsen.util 3269 :invoke :cas [9 [21 14]] INFO [2019-12-17 12:30:45,825] jepsen worker 8 - jepsen.util 4328 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:30:45,825] jepsen worker 15 - jepsen.util 4695 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:30:45,825] jepsen worker 8 - jepsen.util 4328 :invoke :write [2 13] INFO [2019-12-17 12:30:45,825] jepsen worker 15 - jepsen.util 4695 :invoke :read [5 nil] INFO [2019-12-17 12:30:45,843] jepsen worker 63 - jepsen.util 3393 :ok :read [21 12] INFO [2019-12-17 12:30:45,843] jepsen worker 63 - jepsen.util 3393 :invoke :cas [21 [39 5]] INFO [2019-12-17 12:30:45,851] jepsen worker 29 - jepsen.util 3269 :fail :cas [9 [21 14]] :ValueNotSwapFrom INFO [2019-12-17 12:30:45,851] jepsen worker 29 - jepsen.util 3269 :invoke :read [9 nil] INFO [2019-12-17 12:30:45,854] jepsen worker 5 - jepsen.util 3515 :fail :cas [1 [49 36]] :ValueNotSwapFrom INFO [2019-12-17 12:30:45,854] jepsen worker 5 - jepsen.util 3515 :invoke :read [1 nil] INFO [2019-12-17 12:30:45,855] jepsen worker 28 - jepsen.util 2638 :info :cas [9 [49 25]] :RequestCanceledException INFO [2019-12-17 12:30:45,857] jepsen worker 28 - jepsen.util 2728 :invoke :cas [9 [3 19]] INFO [2019-12-17 12:30:45,883] jepsen worker 63 - jepsen.util 3393 :fail :cas [21 [39 5]] :ValueNotSwapFrom INFO [2019-12-17 12:30:45,883] jepsen worker 63 - jepsen.util 3393 :invoke :write [21 42] INFO [2019-12-17 12:30:45,885] jepsen worker 29 - jepsen.util 3269 :ok :read [9 38] INFO [2019-12-17 12:30:45,885] jepsen worker 29 - jepsen.util 3269 :invoke :read [9 nil] INFO [2019-12-17 12:30:45,891] jepsen worker 31 - jepsen.util 3361 :info :write [10 40] :RequestCanceledException INFO [2019-12-17 12:30:45,891] jepsen worker 27 - jepsen.util 3717 :info :write [9 35] :RequestCanceledException INFO [2019-12-17 12:30:45,892] jepsen worker 27 - jepsen.util 3807 :invoke :cas [9 [5 25]] INFO [2019-12-17 12:30:45,892] jepsen worker 31 - jepsen.util 3451 :invoke :cas [10 [8 9]] INFO [2019-12-17 12:30:45,896] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":22201,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:30:45,896] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":22201,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:30:45,896] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":22205,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:30:45,896] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":22205,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:30:45,909] jepsen worker 45 - jepsen.util 4005 :info :write [15 48] :RequestCanceledException INFO [2019-12-17 12:30:45,911] jepsen worker 45 - jepsen.util 4095 :invoke :cas [15 [8 32]] INFO [2019-12-17 12:30:45,924] jepsen worker 81 - jepsen.util 5211 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:30:45,925] jepsen worker 81 - jepsen.util 5211 :invoke :read [27 nil] INFO [2019-12-17 12:30:45,926] jepsen worker 29 - jepsen.util 3269 :ok :read [9 38] INFO [2019-12-17 12:30:45,926] jepsen worker 29 - jepsen.util 3269 :invoke :read [9 nil] INFO [2019-12-17 12:30:45,931] jepsen worker 31 - jepsen.util 3451 :fail :cas [10 [8 9]] :ValueNotSwapFrom INFO [2019-12-17 12:30:45,931] jepsen worker 31 - jepsen.util 3451 :invoke :cas [10 [13 20]] INFO [2019-12-17 12:30:45,944] jepsen worker 63 - jepsen.util 3393 :ok :write [21 42] INFO [2019-12-17 12:30:45,944] jepsen worker 63 - jepsen.util 3393 :invoke :cas [21 [37 33]] INFO [2019-12-17 12:30:45,955] jepsen worker 29 - jepsen.util 3269 :ok :read [9 38] INFO [2019-12-17 12:30:45,955] jepsen worker 29 - jepsen.util 3269 :invoke :write [9 17] INFO [2019-12-17 12:30:45,972] jepsen worker 31 - jepsen.util 3451 :fail :cas [10 [13 20]] :ValueNotSwapFrom INFO [2019-12-17 12:30:45,972] jepsen worker 31 - jepsen.util 3451 :invoke :write [10 48] INFO [2019-12-17 12:30:45,990] jepsen worker 27 - jepsen.util 3807 :fail :cas [9 [5 25]] :ValueNotSwapFrom INFO [2019-12-17 12:30:45,990] jepsen worker 27 - jepsen.util 3807 :invoke :read [9 nil] INFO [2019-12-17 12:30:46,005] jepsen worker 63 - jepsen.util 3393 :fail :cas [21 [37 33]] :ValueNotSwapFrom INFO [2019-12-17 12:30:46,005] jepsen worker 63 - jepsen.util 3393 :invoke :read [21 nil] INFO [2019-12-17 12:30:46,021] jepsen worker 27 - jepsen.util 3807 :ok :read [9 38] INFO [2019-12-17 12:30:46,021] jepsen worker 27 - jepsen.util 3807 :invoke :write [9 9] INFO [2019-12-17 12:30:46,023] jepsen worker 21 - jepsen.util 5061 :info :cas [7 [35 34]] :RequestCanceledException INFO [2019-12-17 12:30:46,026] jepsen worker 21 - jepsen.util 5151 :invoke :write [7 24] INFO [2019-12-17 12:30:46,026] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":22219,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:30:46,026] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":22219,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:30:46,045] jepsen worker 63 - jepsen.util 3393 :ok :read [21 42] INFO [2019-12-17 12:30:46,045] jepsen worker 63 - jepsen.util 3393 :invoke :write [21 10] INFO [2019-12-17 12:30:46,065] jepsen worker 78 - jepsen.util 4398 :fail :cas [26 [31 21]] :ValueNotSwapFrom INFO [2019-12-17 12:30:46,065] jepsen worker 78 - jepsen.util 4398 :invoke :read [26 nil] INFO [2019-12-17 12:30:46,070] jepsen worker 29 - jepsen.util 3269 :ok :write [9 17] INFO [2019-12-17 12:30:46,070] jepsen worker 29 - jepsen.util 3269 :invoke :write [9 7] INFO [2019-12-17 12:30:46,090] jepsen worker 13 - jepsen.util 3613 :info :write [4 1] :RequestCanceledException INFO [2019-12-17 12:30:46,092] jepsen worker 13 - jepsen.util 3703 :invoke :cas [4 [41 4]] INFO [2019-12-17 12:30:46,129] jepsen worker 87 - jepsen.util 3687 :fail :cas [29 [3 18]] :ValueNotSwapFrom INFO [2019-12-17 12:30:46,129] jepsen worker 87 - jepsen.util 3687 :invoke :cas [29 [7 30]] INFO [2019-12-17 12:30:46,138] jepsen worker 71 - jepsen.util 4031 :info :write [23 16] :RequestCanceledException INFO [2019-12-17 12:30:46,140] jepsen worker 71 - jepsen.util 4121 :invoke :write [23 42] INFO [2019-12-17 12:30:46,140] jepsen worker 89 - jepsen.util 3689 :ok :read [29 18] INFO [2019-12-17 12:30:46,141] jepsen worker 89 - jepsen.util 3689 :invoke :cas [29 [11 31]] INFO [2019-12-17 12:30:46,145] jepsen worker 63 - jepsen.util 3393 :ok :write [21 10] INFO [2019-12-17 12:30:46,146] jepsen worker 63 - jepsen.util 3393 :invoke :write [21 3] INFO [2019-12-17 12:30:46,161] jepsen worker 29 - jepsen.util 3269 :ok :write [9 7] INFO [2019-12-17 12:30:46,161] jepsen worker 29 - jepsen.util 3269 :invoke :write [9 43] INFO [2019-12-17 12:30:46,182] jepsen worker 36 - jepsen.util 4086 :fail :cas [12 [15 10]] :ValueNotSwapFrom INFO [2019-12-17 12:30:46,183] jepsen worker 36 - jepsen.util 4086 :invoke :read [12 nil] INFO [2019-12-17 12:30:46,185] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:30:46,186] jepsen nemesis - couchbase.util Rebalance check count: 165 INFO [2019-12-17 12:30:46,186] jepsen nemesis - couchbase.util Rebalance stuck count: 146 INFO [2019-12-17 12:30:46,201] jepsen worker 53 - jepsen.util 4013 :ok :read [17 8] INFO [2019-12-17 12:30:46,201] jepsen worker 53 - jepsen.util 4013 :invoke :cas [17 [34 10]] INFO [2019-12-17 12:30:46,212] jepsen worker 3 - jepsen.util 4143 :ok :read [1 45] INFO [2019-12-17 12:30:46,212] jepsen worker 3 - jepsen.util 4143 :invoke :read [1 nil] INFO [2019-12-17 12:30:46,224] jepsen worker 56 - jepsen.util 4016 :info :write [18 22] :RequestCanceledException INFO [2019-12-17 12:30:46,226] jepsen worker 56 - jepsen.util 4106 :invoke :cas [18 [21 16]] INFO [2019-12-17 12:30:46,232] jepsen worker 89 - jepsen.util 3689 :fail :cas [29 [11 31]] :ValueNotSwapFrom INFO [2019-12-17 12:30:46,232] jepsen worker 89 - jepsen.util 3689 :invoke :read [29 nil] INFO [2019-12-17 12:30:46,236] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":22237,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:30:46,236] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":22237,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:30:46,236] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":22238,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:30:46,236] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":22238,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:30:46,236] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":22239,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:30:46,236] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":22239,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:30:46,238] jepsen worker 61 - jepsen.util 5911 :info :cas [20 [32 5]] :RequestCanceledException INFO [2019-12-17 12:30:46,240] jepsen worker 61 - jepsen.util 6001 :invoke :cas [20 [29 28]] INFO [2019-12-17 12:30:46,251] jepsen worker 63 - jepsen.util 3393 :ok :write [21 3] INFO [2019-12-17 12:30:46,251] jepsen worker 63 - jepsen.util 3393 :invoke :write [21 33] INFO [2019-12-17 12:30:46,268] jepsen worker 29 - jepsen.util 3269 :ok :write [9 43] INFO [2019-12-17 12:30:46,268] jepsen worker 29 - jepsen.util 3269 :invoke :read [9 nil] INFO [2019-12-17 12:30:46,272] jepsen worker 70 - jepsen.util 4480 :ok :read [23 47] INFO [2019-12-17 12:30:46,272] jepsen worker 70 - jepsen.util 4480 :invoke :cas [23 [40 15]] INFO [2019-12-17 12:30:46,273] jepsen worker 38 - jepsen.util 3818 :ok :read [12 1] INFO [2019-12-17 12:30:46,273] jepsen worker 38 - jepsen.util 3818 :invoke :read [12 nil] INFO [2019-12-17 12:30:46,299] jepsen worker 29 - jepsen.util 3269 :ok :read [9 43] INFO [2019-12-17 12:30:46,299] jepsen worker 29 - jepsen.util 3269 :invoke :read [9 nil] INFO [2019-12-17 12:30:46,303] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":22244,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:30:46,303] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":22244,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:30:46,303] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":22248,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:30:46,303] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":22248,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:30:46,303] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":22249,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:30:46,303] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":22249,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:30:46,322] jepsen worker 79 - jepsen.util 3859 :fail :cas [26 [9 18]] :ValueNotSwapFrom INFO [2019-12-17 12:30:46,323] jepsen worker 79 - jepsen.util 3859 :invoke :cas [26 [40 1]] INFO [2019-12-17 12:30:46,335] jepsen worker 63 - jepsen.util 3393 :ok :write [21 33] INFO [2019-12-17 12:30:46,335] jepsen worker 63 - jepsen.util 3393 :invoke :read [21 nil] INFO [2019-12-17 12:30:46,340] jepsen worker 5 - jepsen.util 3515 :ok :read [1 45] INFO [2019-12-17 12:30:46,340] jepsen worker 5 - jepsen.util 3515 :invoke :read [1 nil] INFO [2019-12-17 12:30:46,348] jepsen worker 29 - jepsen.util 3269 :ok :read [9 43] INFO [2019-12-17 12:30:46,348] jepsen worker 29 - jepsen.util 3269 :invoke :read [9 nil] INFO [2019-12-17 12:30:46,366] jepsen worker 63 - jepsen.util 3393 :ok :read [21 33] INFO [2019-12-17 12:30:46,366] jepsen worker 63 - jepsen.util 3393 :invoke :read [21 nil] INFO [2019-12-17 12:30:46,377] jepsen worker 29 - jepsen.util 3269 :ok :read [9 43] INFO [2019-12-17 12:30:46,377] jepsen worker 29 - jepsen.util 3269 :invoke :cas [9 [39 42]] INFO [2019-12-17 12:30:46,401] jepsen worker 63 - jepsen.util 3393 :ok :read [21 33] INFO [2019-12-17 12:30:46,401] jepsen worker 63 - jepsen.util 3393 :invoke :read [21 nil] INFO [2019-12-17 12:30:46,401] jepsen worker 87 - jepsen.util 3687 :fail :cas [29 [7 30]] :ValueNotSwapFrom INFO [2019-12-17 12:30:46,401] jepsen worker 87 - jepsen.util 3687 :invoke :write [29 19] INFO [2019-12-17 12:30:46,417] jepsen worker 29 - jepsen.util 3269 :fail :cas [9 [39 42]] :ValueNotSwapFrom INFO [2019-12-17 12:30:46,417] jepsen worker 29 - jepsen.util 3269 :invoke :write [9 25] INFO [2019-12-17 12:30:46,425] jepsen worker 86 - jepsen.util 5396 :info :write [28 19] :RequestCanceledException INFO [2019-12-17 12:30:46,429] jepsen worker 86 - jepsen.util 5486 :invoke :cas [28 [33 10]] INFO [2019-12-17 12:30:46,434] jepsen worker 70 - jepsen.util 4480 :fail :cas [23 [40 15]] :ValueNotSwapFrom INFO [2019-12-17 12:30:46,434] jepsen worker 70 - jepsen.util 4480 :invoke :write [23 45] INFO [2019-12-17 12:30:46,442] jepsen worker 63 - jepsen.util 3393 :ok :read [21 33] INFO [2019-12-17 12:30:46,442] jepsen worker 63 - jepsen.util 3393 :invoke :read [21 nil] INFO [2019-12-17 12:30:46,464] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":22268,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:30:46,464] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":22268,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:30:46,464] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":22272,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:30:46,464] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":22272,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:30:46,474] jepsen worker 63 - jepsen.util 3393 :ok :read [21 33] INFO [2019-12-17 12:30:46,474] jepsen worker 63 - jepsen.util 3393 :invoke :read [21 nil] INFO [2019-12-17 12:30:46,483] jepsen worker 29 - jepsen.util 3269 :ok :write [9 25] INFO [2019-12-17 12:30:46,483] jepsen worker 29 - jepsen.util 3269 :invoke :read [9 nil] INFO [2019-12-17 12:30:46,497] jepsen worker 63 - jepsen.util 3393 :ok :read [21 33] INFO [2019-12-17 12:30:46,498] jepsen worker 63 - jepsen.util 3393 :invoke :write [21 36] INFO [2019-12-17 12:30:46,509] jepsen worker 77 - jepsen.util 5117 :info :write [25 25] :RequestCanceledException INFO [2019-12-17 12:30:46,514] jepsen worker 77 - jepsen.util 5207 :invoke :cas [25 [0 25]] INFO [2019-12-17 12:30:46,516] jepsen worker 29 - jepsen.util 3269 :ok :read [9 25] INFO [2019-12-17 12:30:46,516] jepsen worker 29 - jepsen.util 3269 :invoke :write [9 48] INFO [2019-12-17 12:30:46,524] jepsen worker 12 - jepsen.util 4152 :info :write [4 2] :RequestCanceledException INFO [2019-12-17 12:30:46,529] jepsen worker 12 - jepsen.util 4242 :invoke :write [4 17] INFO [2019-12-17 12:30:46,538] jepsen worker 24 - jepsen.util 4884 :info :write [8 47] :RequestCanceledException INFO [2019-12-17 12:30:46,542] jepsen worker 24 - jepsen.util 4974 :invoke :cas [8 [37 49]] INFO [2019-12-17 12:30:46,558] jepsen worker 3 - jepsen.util 4143 :ok :read [1 45] INFO [2019-12-17 12:30:46,558] jepsen worker 3 - jepsen.util 4143 :invoke :write [1 13] INFO [2019-12-17 12:30:46,558] jepsen worker 14 - jepsen.util 3974 :ok :read [4 45] INFO [2019-12-17 12:30:46,558] jepsen worker 14 - jepsen.util 3974 :invoke :write [4 49] INFO [2019-12-17 12:30:46,561] jepsen worker 30 - jepsen.util 4260 :ok :write [10 31] INFO [2019-12-17 12:30:46,561] jepsen worker 30 - jepsen.util 4260 :invoke :read [10 nil] INFO [2019-12-17 12:30:46,570] jepsen worker 63 - jepsen.util 3393 :ok :write [21 36] INFO [2019-12-17 12:30:46,570] jepsen worker 63 - jepsen.util 3393 :invoke :write [21 11] INFO [2019-12-17 12:30:46,585] jepsen worker 46 - jepsen.util 4006 :ok :write [15 36] INFO [2019-12-17 12:30:46,585] jepsen worker 46 - jepsen.util 4006 :invoke :write [15 5] INFO [2019-12-17 12:30:46,591] jepsen worker 49 - jepsen.util 4549 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:30:46,591] jepsen worker 49 - jepsen.util 4549 :invoke :cas [16 [27 11]] INFO [2019-12-17 12:30:46,592] jepsen worker 11 - jepsen.util 4331 :ok :cas [3 [2 22]] INFO [2019-12-17 12:30:46,592] jepsen worker 11 - jepsen.util 4331 :invoke :write [3 26] INFO [2019-12-17 12:30:46,596] jepsen worker 47 - jepsen.util 4457 :fail :cas [15 [48 8]] :ValueNotSwapFrom INFO [2019-12-17 12:30:46,596] jepsen worker 47 - jepsen.util 4457 :invoke :cas [15 [12 1]] INFO [2019-12-17 12:30:46,619] jepsen worker 29 - jepsen.util 3269 :ok :write [9 48] INFO [2019-12-17 12:30:46,619] jepsen worker 20 - jepsen.util 3800 :ok :write [6 23] INFO [2019-12-17 12:30:46,625] jepsen worker 45 - jepsen.util 4095 :fail :cas [15 [8 32]] :ValueNotSwapFrom INFO [2019-12-17 12:30:46,650] jepsen worker 30 - jepsen.util 4260 :ok :read [10 31] INFO [2019-12-17 12:30:46,656] jepsen worker 34 - jepsen.util 4984 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:30:46,656] jepsen worker 29 - jepsen.util 3269 :invoke :cas [9 [13 32]] INFO [2019-12-17 12:30:46,656] jepsen worker 20 - jepsen.util 3800 :invoke :read [6 nil] INFO [2019-12-17 12:30:46,656] jepsen worker 45 - jepsen.util 4095 :invoke :read [15 nil] INFO [2019-12-17 12:30:46,656] jepsen worker 30 - jepsen.util 4260 :invoke :cas [10 [41 23]] INFO [2019-12-17 12:30:46,656] jepsen worker 34 - jepsen.util 4984 :invoke :cas [11 [35 15]] INFO [2019-12-17 12:30:46,662] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":22293,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:30:46,662] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":22293,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:30:46,662] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":22297,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:30:46,662] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":22297,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:30:46,666] jepsen worker 47 - jepsen.util 4457 :fail :cas [15 [12 1]] :ValueNotSwapFrom INFO [2019-12-17 12:30:46,666] jepsen worker 47 - jepsen.util 4457 :invoke :cas [15 [9 48]] INFO [2019-12-17 12:30:46,671] jepsen worker 65 - jepsen.util 3215 :ok :write [21 39] INFO [2019-12-17 12:30:46,671] jepsen worker 65 - jepsen.util 3215 :invoke :write [21 15] INFO [2019-12-17 12:30:46,702] jepsen worker 29 - jepsen.util 3269 :fail :cas [9 [13 32]] :ValueNotSwapFrom INFO [2019-12-17 12:30:46,702] jepsen worker 29 - jepsen.util 3269 :invoke :cas [9 [4 23]] INFO [2019-12-17 12:30:46,719] jepsen worker 38 - jepsen.util 3818 :ok :read [12 1] INFO [2019-12-17 12:30:46,719] jepsen worker 38 - jepsen.util 3818 :invoke :write [12 14] INFO [2019-12-17 12:30:46,725] jepsen worker 32 - jepsen.util 4352 :fail :cas [10 [28 43]] :ValueNotSwapFrom INFO [2019-12-17 12:30:46,726] jepsen worker 32 - jepsen.util 4352 :invoke :write [10 10] INFO [2019-12-17 12:30:46,750] jepsen worker 64 - jepsen.util 3124 :ok :read [21 36] INFO [2019-12-17 12:30:46,750] jepsen worker 64 - jepsen.util 3124 :invoke :write [21 41] INFO [2019-12-17 12:30:46,756] jepsen worker 78 - jepsen.util 4398 :ok :read [26 28] INFO [2019-12-17 12:30:46,756] jepsen worker 78 - jepsen.util 4398 :invoke :cas [26 [35 14]] INFO [2019-12-17 12:30:46,762] jepsen worker 63 - jepsen.util 3393 :ok :write [21 11] INFO [2019-12-17 12:30:46,762] jepsen worker 63 - jepsen.util 3393 :invoke :cas [21 [5 14]] INFO [2019-12-17 12:30:46,766] jepsen worker 29 - jepsen.util 3269 :fail :cas [9 [4 23]] :ValueNotSwapFrom INFO [2019-12-17 12:30:46,767] jepsen worker 29 - jepsen.util 3269 :invoke :cas [9 [31 18]] INFO [2019-12-17 12:30:46,801] jepsen worker 45 - jepsen.util 4095 :ok :read [15 36] INFO [2019-12-17 12:30:46,801] jepsen worker 45 - jepsen.util 4095 :invoke :cas [15 [2 46]] INFO [2019-12-17 12:30:46,809] jepsen worker 88 - jepsen.util 3958 :info :write [29 27] :RequestCanceledException INFO [2019-12-17 12:30:46,814] jepsen worker 88 - jepsen.util 4048 :invoke :write [29 15] INFO [2019-12-17 12:30:46,824] jepsen worker 68 - jepsen.util 4748 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:30:46,825] jepsen worker 68 - jepsen.util 4748 :invoke :cas [22 [20 15]] INFO [2019-12-17 12:30:46,830] jepsen worker 29 - jepsen.util 3269 :fail :cas [9 [31 18]] :ValueNotSwapFrom INFO [2019-12-17 12:30:46,830] jepsen worker 29 - jepsen.util 3269 :invoke :read [9 nil] INFO [2019-12-17 12:30:46,830] jepsen worker 63 - jepsen.util 3393 :fail :cas [21 [5 14]] :ValueNotSwapFrom INFO [2019-12-17 12:30:46,830] jepsen worker 63 - jepsen.util 3393 :invoke :write [21 30] INFO [2019-12-17 12:30:46,833] jepsen worker 30 - jepsen.util 4260 :fail :cas [10 [41 23]] :ValueNotSwapFrom INFO [2019-12-17 12:30:46,833] jepsen worker 30 - jepsen.util 4260 :invoke :write [10 35] INFO [2019-12-17 12:30:46,834] jepsen worker 13 - jepsen.util 3703 :fail :cas [4 [41 4]] :ValueNotSwapFrom INFO [2019-12-17 12:30:46,834] jepsen worker 13 - jepsen.util 3703 :invoke :cas [4 [27 14]] INFO [2019-12-17 12:30:46,841] jepsen worker 47 - jepsen.util 4457 :fail :cas [15 [9 48]] :ValueNotSwapFrom INFO [2019-12-17 12:30:46,841] jepsen worker 47 - jepsen.util 4457 :invoke :write [15 44] INFO [2019-12-17 12:30:46,882] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":22321,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:30:46,882] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":22321,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:30:46,882] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":22324,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:30:46,882] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":22324,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:30:46,882] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":22325,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:30:46,883] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":22325,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:30:46,892] jepsen worker 29 - jepsen.util 3269 :ok :read [9 48] INFO [2019-12-17 12:30:46,892] jepsen worker 29 - jepsen.util 3269 :invoke :read [9 nil] INFO [2019-12-17 12:30:46,899] jepsen worker 79 - jepsen.util 3859 :fail :cas [26 [40 1]] :ValueNotSwapFrom INFO [2019-12-17 12:30:46,899] jepsen worker 79 - jepsen.util 3859 :invoke :write [26 1] INFO [2019-12-17 12:30:46,913] jepsen worker 89 - jepsen.util 3689 :ok :read [29 18] INFO [2019-12-17 12:30:46,914] jepsen worker 89 - jepsen.util 3689 :invoke :cas [29 [40 19]] INFO [2019-12-17 12:30:46,923] jepsen worker 72 - jepsen.util 4122 :ok :write [24 49] INFO [2019-12-17 12:30:46,924] jepsen worker 72 - jepsen.util 4122 :invoke :cas [24 [37 31]] INFO [2019-12-17 12:30:46,924] jepsen worker 18 - jepsen.util 3978 :fail :read [6 nil] :RequestTimeoutException INFO [2019-12-17 12:30:46,924] jepsen worker 18 - jepsen.util 3978 :invoke :read [6 nil] INFO [2019-12-17 12:30:46,935] jepsen worker 36 - jepsen.util 4086 :ok :read [12 1] INFO [2019-12-17 12:30:46,936] jepsen worker 36 - jepsen.util 4086 :invoke :write [12 33] INFO [2019-12-17 12:30:46,939] jepsen worker 23 - jepsen.util 5063 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:30:46,939] jepsen worker 23 - jepsen.util 5063 :invoke :cas [7 [8 0]] INFO [2019-12-17 12:30:46,961] jepsen worker 29 - jepsen.util 3269 :ok :read [9 48] INFO [2019-12-17 12:30:46,962] jepsen worker 29 - jepsen.util 3269 :invoke :write [9 7] INFO [2019-12-17 12:30:46,974] jepsen worker 56 - jepsen.util 4106 :fail :cas [18 [21 16]] :ValueNotSwapFrom INFO [2019-12-17 12:30:46,974] jepsen worker 56 - jepsen.util 4106 :invoke :cas [18 [16 19]] INFO [2019-12-17 12:30:46,977] jepsen worker 63 - jepsen.util 3393 :ok :write [21 30] INFO [2019-12-17 12:30:46,977] jepsen worker 63 - jepsen.util 3393 :invoke :cas [21 [10 32]] INFO [2019-12-17 12:30:47,007] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":22337,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:30:47,007] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":22337,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:30:47,007] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":22341,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:30:47,008] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":22341,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:30:47,008] jepsen worker 63 - jepsen.util 3393 :fail :cas [21 [10 32]] :ValueNotSwapFrom INFO [2019-12-17 12:30:47,008] jepsen worker 63 - jepsen.util 3393 :invoke :write [21 6] INFO [2019-12-17 12:30:47,008] jepsen worker 44 - jepsen.util 5534 :info :write [14 34] :RequestCanceledException INFO [2019-12-17 12:30:47,010] jepsen worker 44 - jepsen.util 5624 :invoke :cas [14 [48 4]] INFO [2019-12-17 12:30:47,024] jepsen worker 59 - jepsen.util 5189 :info :cas [19 [44 39]] :RequestCanceledException INFO [2019-12-17 12:30:47,026] jepsen worker 59 - jepsen.util 5279 :invoke :write [19 9] INFO [2019-12-17 12:30:47,038] jepsen worker 84 - jepsen.util 4764 :info :cas [28 [36 30]] :RequestCanceledException INFO [2019-12-17 12:30:47,039] jepsen worker 84 - jepsen.util 4854 :invoke :read [28 nil] INFO [2019-12-17 12:30:47,046] jepsen worker 19 - jepsen.util 4069 :ok :write [6 11] INFO [2019-12-17 12:30:47,046] jepsen worker 11 - jepsen.util 4331 :ok :write [3 26] INFO [2019-12-17 12:30:47,046] jepsen worker 19 - jepsen.util 4069 :invoke :write [6 13] INFO [2019-12-17 12:30:47,046] jepsen worker 11 - jepsen.util 4331 :invoke :write [3 22] INFO [2019-12-17 12:30:47,073] jepsen worker 46 - jepsen.util 4006 :ok :write [15 5] INFO [2019-12-17 12:30:47,073] jepsen worker 46 - jepsen.util 4006 :invoke :cas [15 [36 26]] INFO [2019-12-17 12:30:47,085] jepsen worker 20 - jepsen.util 3800 :ok :read [6 11] INFO [2019-12-17 12:30:47,085] jepsen worker 20 - jepsen.util 3800 :invoke :cas [6 [36 18]] INFO [2019-12-17 12:30:47,093] jepsen worker 5 - jepsen.util 3515 :ok :read [1 45] INFO [2019-12-17 12:30:47,093] jepsen worker 5 - jepsen.util 3515 :invoke :write [1 37] INFO [2019-12-17 12:30:47,096] jepsen worker 29 - jepsen.util 3269 :ok :write [9 7] INFO [2019-12-17 12:30:47,096] jepsen worker 29 - jepsen.util 3269 :invoke :read [9 nil] INFO [2019-12-17 12:30:47,099] jepsen worker 45 - jepsen.util 4095 :fail :cas [15 [2 46]] :ValueNotSwapFrom INFO [2019-12-17 12:30:47,099] jepsen worker 45 - jepsen.util 4095 :invoke :write [15 14] INFO [2019-12-17 12:30:47,104] jepsen worker 63 - jepsen.util 3393 :ok :write [21 6] INFO [2019-12-17 12:30:47,105] jepsen worker 63 - jepsen.util 3393 :invoke :write [21 32] INFO [2019-12-17 12:30:47,109] jepsen worker 66 - jepsen.util 5016 :info :write [22 22] :RequestCanceledException INFO [2019-12-17 12:30:47,109] jepsen worker 43 - jepsen.util 4813 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:30:47,109] jepsen worker 43 - jepsen.util 4813 :invoke :read [14 nil] INFO [2019-12-17 12:30:47,110] jepsen worker 66 - jepsen.util 5106 :invoke :write [22 19] INFO [2019-12-17 12:30:47,120] jepsen worker 53 - jepsen.util 4013 :fail :cas [17 [34 10]] :ValueNotSwapFrom INFO [2019-12-17 12:30:47,120] jepsen worker 53 - jepsen.util 4013 :invoke :cas [17 [15 36]] INFO [2019-12-17 12:30:47,124] jepsen worker 51 - jepsen.util 3471 :info :write [17 43] :RequestCanceledException INFO [2019-12-17 12:30:47,125] jepsen worker 51 - jepsen.util 3561 :invoke :cas [17 [7 32]] INFO [2019-12-17 12:30:47,156] jepsen worker 9 - jepsen.util 4329 :info :write [3 48] :RequestCanceledException INFO [2019-12-17 12:30:47,156] jepsen worker 54 - jepsen.util 4284 :info :write [18 38] :RequestCanceledException INFO [2019-12-17 12:30:47,158] jepsen worker 9 - jepsen.util 4419 :invoke :write [3 49] INFO [2019-12-17 12:30:47,158] jepsen worker 54 - jepsen.util 4374 :invoke :cas [18 [32 29]] INFO [2019-12-17 12:30:47,162] jepsen worker 72 - jepsen.util 4122 :fail :cas [24 [37 31]] :ValueNotSwapFrom INFO [2019-12-17 12:30:47,162] jepsen worker 72 - jepsen.util 4122 :invoke :write [24 41] INFO [2019-12-17 12:30:47,177] jepsen worker 18 - jepsen.util 3978 :ok :read [6 11] INFO [2019-12-17 12:30:47,177] jepsen worker 18 - jepsen.util 3978 :invoke :write [6 9] INFO [2019-12-17 12:30:47,190] jepsen worker 25 - jepsen.util 5335 :info :cas [8 [49 45]] :RequestCanceledException INFO [2019-12-17 12:30:47,190] jepsen worker 25 - jepsen.util 5425 :invoke :read [8 nil] INFO [2019-12-17 12:30:47,200] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:30:47,200] jepsen nemesis - couchbase.util Rebalance check count: 166 INFO [2019-12-17 12:30:47,200] jepsen nemesis - couchbase.util Rebalance stuck count: 147 INFO [2019-12-17 12:30:47,208] jepsen worker 80 - jepsen.util 3950 :info :write [26 39] :RequestCanceledException INFO [2019-12-17 12:30:47,210] jepsen worker 80 - jepsen.util 4040 :invoke :read [26 nil] INFO [2019-12-17 12:30:47,213] jepsen worker 63 - jepsen.util 3393 :ok :write [21 32] INFO [2019-12-17 12:30:47,213] jepsen worker 63 - jepsen.util 3393 :invoke :read [21 nil] INFO [2019-12-17 12:30:47,215] jepsen worker 56 - jepsen.util 4106 :fail :cas [18 [16 19]] :ValueNotSwapFrom INFO [2019-12-17 12:30:47,215] jepsen worker 56 - jepsen.util 4106 :invoke :cas [18 [19 16]] INFO [2019-12-17 12:30:47,217] jepsen worker 29 - jepsen.util 3269 :ok :read [9 7] INFO [2019-12-17 12:30:47,217] jepsen worker 29 - jepsen.util 3269 :invoke :read [9 nil] INFO [2019-12-17 12:30:47,219] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":22364,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:30:47,219] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":22364,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:30:47,219] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":22365,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:30:47,219] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":22365,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:30:47,224] jepsen worker 6 - jepsen.util 5586 :info :write [2 5] :RequestCanceledException INFO [2019-12-17 12:30:47,224] jepsen worker 62 - jepsen.util 4832 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:30:47,224] jepsen worker 62 - jepsen.util 4832 :invoke :cas [20 [43 25]] INFO [2019-12-17 12:30:47,226] jepsen worker 6 - jepsen.util 5676 :invoke :cas [2 [45 33]] INFO [2019-12-17 12:30:47,230] jepsen worker 46 - jepsen.util 4006 :fail :cas [15 [36 26]] :ValueNotSwapFrom INFO [2019-12-17 12:30:47,230] jepsen worker 46 - jepsen.util 4006 :invoke :read [15 nil] INFO [2019-12-17 12:30:47,239] jepsen worker 39 - jepsen.util 4809 :info :write [13 14] :RequestCanceledException INFO [2019-12-17 12:30:47,241] jepsen worker 39 - jepsen.util 4899 :invoke :read [13 nil] INFO [2019-12-17 12:30:47,247] jepsen worker 54 - jepsen.util 4374 :fail :cas [18 [32 29]] :ValueNotSwapFrom INFO [2019-12-17 12:30:47,247] jepsen worker 54 - jepsen.util 4374 :invoke :write [18 25] INFO [2019-12-17 12:30:47,270] jepsen worker 3 - jepsen.util 4143 :ok :write [1 13] INFO [2019-12-17 12:30:47,270] jepsen worker 3 - jepsen.util 4143 :invoke :write [1 47] INFO [2019-12-17 12:30:47,280] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":22373,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:30:47,280] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":22373,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:30:47,281] jepsen worker 29 - jepsen.util 3269 :ok :read [9 7] INFO [2019-12-17 12:30:47,281] jepsen worker 29 - jepsen.util 3269 :invoke :write [9 27] INFO [2019-12-17 12:30:47,291] jepsen worker 82 - jepsen.util 6022 :info :cas [27 [49 1]] :RequestCanceledException INFO [2019-12-17 12:30:47,292] jepsen worker 82 - jepsen.util 6112 :invoke :cas [27 [47 41]] INFO [2019-12-17 12:30:47,309] jepsen worker 7 - jepsen.util 5047 :info :cas [2 [36 33]] :RequestCanceledException INFO [2019-12-17 12:30:47,309] jepsen worker 7 - jepsen.util 5137 :invoke :write [2 39] INFO [2019-12-17 12:30:47,322] jepsen worker 70 - jepsen.util 4480 :ok :write [23 45] INFO [2019-12-17 12:30:47,322] jepsen worker 70 - jepsen.util 4480 :invoke :cas [23 [31 45]] INFO [2019-12-17 12:30:47,327] jepsen worker 29 - jepsen.util 3269 :ok :write [9 27] INFO [2019-12-17 12:30:47,327] jepsen worker 29 - jepsen.util 3269 :invoke :cas [9 [12 7]] INFO [2019-12-17 12:30:47,347] jepsen worker 29 - jepsen.util 3269 :fail :cas [9 [12 7]] :ValueNotSwapFrom INFO [2019-12-17 12:30:47,347] jepsen worker 29 - jepsen.util 3269 :invoke :write [9 38] INFO [2019-12-17 12:30:47,367] jepsen worker 89 - jepsen.util 3689 :fail :cas [29 [40 19]] :ValueNotSwapFrom INFO [2019-12-17 12:30:47,367] jepsen worker 89 - jepsen.util 3689 :invoke :write [29 11] INFO [2019-12-17 12:30:47,391] jepsen worker 50 - jepsen.util 5630 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:30:47,391] jepsen worker 50 - jepsen.util 5630 :invoke :cas [16 [49 11]] INFO [2019-12-17 12:30:47,396] jepsen worker 19 - jepsen.util 4069 :ok :write [6 13] INFO [2019-12-17 12:30:47,396] jepsen worker 19 - jepsen.util 4069 :invoke :cas [6 [18 6]] INFO [2019-12-17 12:30:47,409] jepsen worker 85 - jepsen.util 5665 :info :cas [28 [3 6]] :RequestCanceledException INFO [2019-12-17 12:30:47,411] jepsen worker 85 - jepsen.util 5755 :invoke :cas [28 [34 20]] INFO [2019-12-17 12:30:47,418] jepsen worker 29 - jepsen.util 3269 :ok :write [9 38] INFO [2019-12-17 12:30:47,418] jepsen worker 29 - jepsen.util 3269 :invoke :read [9 nil] INFO [2019-12-17 12:30:47,421] jepsen worker 19 - jepsen.util 4069 :fail :cas [6 [18 6]] :ValueNotSwapFrom INFO [2019-12-17 12:30:47,422] jepsen worker 19 - jepsen.util 4069 :invoke :cas [6 [16 10]] INFO [2019-12-17 12:30:47,437] jepsen worker 51 - jepsen.util 3561 :fail :cas [17 [7 32]] :ValueNotSwapFrom INFO [2019-12-17 12:30:47,437] jepsen worker 51 - jepsen.util 3561 :invoke :cas [17 [17 49]] INFO [2019-12-17 12:30:47,444] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":22400,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:30:47,444] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":22400,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:30:47,444] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":22402,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:30:47,444] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":22402,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:30:47,455] jepsen worker 29 - jepsen.util 3269 :ok :read [9 38] INFO [2019-12-17 12:30:47,455] jepsen worker 29 - jepsen.util 3269 :invoke :cas [9 [1 10]] INFO [2019-12-17 12:30:47,473] jepsen worker 19 - jepsen.util 4069 :fail :cas [6 [16 10]] :ValueNotSwapFrom INFO [2019-12-17 12:30:47,473] jepsen worker 19 - jepsen.util 4069 :invoke :read [6 nil] INFO [2019-12-17 12:30:47,478] jepsen worker 9 - jepsen.util 4419 :ok :write [3 49] INFO [2019-12-17 12:30:47,478] jepsen worker 9 - jepsen.util 4419 :invoke :read [3 nil] INFO [2019-12-17 12:30:47,482] jepsen worker 70 - jepsen.util 4480 :fail :cas [23 [31 45]] :ValueNotSwapFrom INFO [2019-12-17 12:30:47,482] jepsen worker 70 - jepsen.util 4480 :invoke :cas [23 [45 49]] INFO [2019-12-17 12:30:47,487] jepsen worker 72 - jepsen.util 4122 :ok :write [24 41] INFO [2019-12-17 12:30:47,488] jepsen worker 72 - jepsen.util 4122 :invoke :write [24 48] INFO [2019-12-17 12:30:47,498] jepsen worker 29 - jepsen.util 3269 :fail :cas [9 [1 10]] :ValueNotSwapFrom INFO [2019-12-17 12:30:47,498] jepsen worker 29 - jepsen.util 3269 :invoke :read [9 nil] INFO [2019-12-17 12:30:47,504] jepsen worker 30 - jepsen.util 4260 :ok :write [10 35] INFO [2019-12-17 12:30:47,505] jepsen worker 30 - jepsen.util 4260 :invoke :cas [10 [37 26]] INFO [2019-12-17 12:30:47,508] jepsen worker 83 - jepsen.util 5213 :info :write [27 13] :RequestCanceledException INFO [2019-12-17 12:30:47,510] jepsen worker 83 - jepsen.util 5303 :invoke :write [27 17] INFO [2019-12-17 12:30:47,511] jepsen worker 9 - jepsen.util 4419 :ok :read [3 49] INFO [2019-12-17 12:30:47,511] jepsen worker 9 - jepsen.util 4419 :invoke :cas [3 [33 43]] INFO [2019-12-17 12:30:47,524] jepsen worker 19 - jepsen.util 4069 :ok :read [6 13] INFO [2019-12-17 12:30:47,524] jepsen worker 19 - jepsen.util 4069 :invoke :cas [6 [7 5]] INFO [2019-12-17 12:30:47,525] jepsen worker 57 - jepsen.util 5007 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:30:47,525] jepsen worker 52 - jepsen.util 4012 :info :write [17 49] :RequestCanceledException INFO [2019-12-17 12:30:47,525] jepsen worker 67 - jepsen.util 5467 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:30:47,525] jepsen worker 57 - jepsen.util 5007 :invoke :write [19 30] INFO [2019-12-17 12:30:47,525] jepsen worker 67 - jepsen.util 5467 :invoke :write [22 27] INFO [2019-12-17 12:30:47,526] jepsen worker 52 - jepsen.util 4102 :invoke :cas [17 [16 35]] INFO [2019-12-17 12:30:47,538] jepsen worker 40 - jepsen.util 5620 :info :write [13 31] :RequestCanceledException INFO [2019-12-17 12:30:47,538] jepsen worker 0 - jepsen.util 5670 :info :cas [0 [48 6]] :RequestCanceledException INFO [2019-12-17 12:30:47,539] jepsen worker 48 - jepsen.util 4638 :info :write [16 43] :RequestCanceledException INFO [2019-12-17 12:30:47,539] jepsen worker 29 - jepsen.util 3269 :ok :read [9 38] INFO [2019-12-17 12:30:47,539] jepsen worker 29 - jepsen.util 3269 :invoke :write [9 24] INFO [2019-12-17 12:30:47,539] jepsen worker 40 - jepsen.util 5710 :invoke :write [13 15] INFO [2019-12-17 12:30:47,540] jepsen worker 0 - jepsen.util 5760 :invoke :write [0 14] INFO [2019-12-17 12:30:47,540] jepsen worker 48 - jepsen.util 4728 :invoke :read [16 nil] INFO [2019-12-17 12:30:47,554] jepsen worker 9 - jepsen.util 4419 :fail :cas [3 [33 43]] :ValueNotSwapFrom INFO [2019-12-17 12:30:47,554] jepsen worker 9 - jepsen.util 4419 :invoke :cas [3 [22 34]] INFO [2019-12-17 12:30:47,555] jepsen worker 60 - jepsen.util 5550 :info :write [20 9] :RequestCanceledException INFO [2019-12-17 12:30:47,556] jepsen worker 60 - jepsen.util 5640 :invoke :write [20 30] INFO [2019-12-17 12:30:47,591] jepsen worker 35 - jepsen.util 5075 :info :write [11 2] :RequestCanceledException INFO [2019-12-17 12:30:47,592] jepsen worker 35 - jepsen.util 5165 :invoke :write [11 47] INFO [2019-12-17 12:30:47,597] jepsen worker 5 - jepsen.util 3515 :ok :write [1 37] INFO [2019-12-17 12:30:47,599] jepsen worker 5 - jepsen.util 3515 :invoke :write [1 19] INFO [2019-12-17 12:30:47,600] jepsen worker 9 - jepsen.util 4419 :fail :cas [3 [22 34]] :ValueNotSwapFrom INFO [2019-12-17 12:30:47,600] jepsen worker 9 - jepsen.util 4419 :invoke :cas [3 [40 46]] INFO [2019-12-17 12:30:47,602] jepsen worker 26 - jepsen.util 5336 :info :cas [8 [0 26]] :RequestCanceledException INFO [2019-12-17 12:30:47,604] jepsen worker 26 - jepsen.util 5426 :invoke :write [8 2] INFO [2019-12-17 12:30:47,605] jepsen worker 58 - jepsen.util 5188 :info :cas [19 [10 37]] :RequestCanceledException INFO [2019-12-17 12:30:47,612] jepsen worker 58 - jepsen.util 5278 :invoke :cas [19 [1 33]] INFO [2019-12-17 12:30:47,624] jepsen worker 17 - jepsen.util 4967 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:30:47,624] jepsen worker 2 - jepsen.util 4862 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:30:47,624] jepsen worker 22 - jepsen.util 5242 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:30:47,624] jepsen worker 74 - jepsen.util 4304 :info :write [24 27] :RequestCanceledException INFO [2019-12-17 12:30:47,624] jepsen worker 17 - jepsen.util 4967 :invoke :read [5 nil] INFO [2019-12-17 12:30:47,624] jepsen worker 2 - jepsen.util 4862 :invoke :cas [0 [32 10]] INFO [2019-12-17 12:30:47,624] jepsen worker 22 - jepsen.util 5242 :invoke :read [7 nil] INFO [2019-12-17 12:30:47,625] jepsen worker 74 - jepsen.util 4394 :invoke :write [24 1] INFO [2019-12-17 12:30:47,632] jepsen worker 47 - jepsen.util 4457 :ok :write [15 44] INFO [2019-12-17 12:30:47,633] jepsen worker 47 - jepsen.util 4457 :invoke :cas [15 [46 15]] INFO [2019-12-17 12:30:47,638] jepsen worker 55 - jepsen.util 4735 :info :write [18 36] :RequestCanceledException INFO [2019-12-17 12:30:47,638] jepsen worker 14 - jepsen.util 3974 :ok :write [4 49] INFO [2019-12-17 12:30:47,638] jepsen worker 14 - jepsen.util 3974 :invoke :read [4 nil] INFO [2019-12-17 12:30:47,640] jepsen worker 55 - jepsen.util 4825 :invoke :write [18 37] INFO [2019-12-17 12:30:47,640] jepsen worker 9 - jepsen.util 4419 :fail :cas [3 [40 46]] :ValueNotSwapFrom INFO [2019-12-17 12:30:47,641] jepsen worker 9 - jepsen.util 4419 :invoke :read [3 nil] INFO [2019-12-17 12:30:47,649] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":22438,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:30:47,649] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":22438,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:30:47,649] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":22440,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:30:47,649] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":22440,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:30:47,679] jepsen worker 19 - jepsen.util 4069 :fail :cas [6 [7 5]] :ValueNotSwapFrom INFO [2019-12-17 12:30:47,679] jepsen worker 19 - jepsen.util 4069 :invoke :read [6 nil] INFO [2019-12-17 12:30:47,686] jepsen worker 47 - jepsen.util 4457 :fail :cas [15 [46 15]] :ValueNotSwapFrom INFO [2019-12-17 12:30:47,686] jepsen worker 33 - jepsen.util 5433 :info :write [11 3] :RequestCanceledException INFO [2019-12-17 12:30:47,686] jepsen worker 47 - jepsen.util 4457 :invoke :read [15 nil] INFO [2019-12-17 12:30:47,688] jepsen worker 33 - jepsen.util 5523 :invoke :read [11 nil] INFO [2019-12-17 12:30:47,699] jepsen worker 19 - jepsen.util 4069 :ok :read [6 13] INFO [2019-12-17 12:30:47,699] jepsen worker 19 - jepsen.util 4069 :invoke :write [6 3] INFO [2019-12-17 12:30:47,714] jepsen worker 47 - jepsen.util 4457 :ok :read [15 44] INFO [2019-12-17 12:30:47,715] jepsen worker 47 - jepsen.util 4457 :invoke :write [15 25] INFO [2019-12-17 12:30:47,723] jepsen worker 56 - jepsen.util 4106 :fail :cas [18 [19 16]] :ValueNotSwapFrom INFO [2019-12-17 12:30:47,723] jepsen worker 56 - jepsen.util 4106 :invoke :cas [18 [48 2]] INFO [2019-12-17 12:30:47,724] jepsen worker 37 - jepsen.util 3637 :info :write [12 33] :RequestCanceledException INFO [2019-12-17 12:30:47,725] jepsen worker 10 - jepsen.util 3790 :info :cas [3 [36 24]] :RequestCanceledException INFO [2019-12-17 12:30:47,725] jepsen worker 42 - jepsen.util 5982 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:30:47,725] jepsen worker 42 - jepsen.util 5982 :invoke :cas [14 [42 30]] INFO [2019-12-17 12:30:47,727] jepsen worker 10 - jepsen.util 3880 :invoke :write [3 24] INFO [2019-12-17 12:30:47,728] jepsen worker 37 - jepsen.util 3727 :invoke :cas [12 [4 18]] INFO [2019-12-17 12:30:47,730] jepsen worker 54 - jepsen.util 4374 :ok :write [18 25] INFO [2019-12-17 12:30:47,731] jepsen worker 54 - jepsen.util 4374 :invoke :cas [18 [6 31]] INFO [2019-12-17 12:30:47,755] jepsen worker 75 - jepsen.util 5295 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:30:47,756] jepsen worker 75 - jepsen.util 5295 :invoke :cas [25 [22 37]] INFO [2019-12-17 12:30:47,757] jepsen worker 72 - jepsen.util 4122 :ok :write [24 48] INFO [2019-12-17 12:30:47,757] jepsen worker 72 - jepsen.util 4122 :invoke :read [24 nil] INFO [2019-12-17 12:30:47,764] jepsen worker 29 - jepsen.util 3269 :ok :write [9 24] INFO [2019-12-17 12:30:47,765] jepsen worker 29 - jepsen.util 3269 :invoke :read [9 nil] INFO [2019-12-17 12:30:47,772] jepsen worker 72 - jepsen.util 4122 :ok :read [24 48] INFO [2019-12-17 12:30:47,772] jepsen worker 72 - jepsen.util 4122 :invoke :cas [24 [9 7]] INFO [2019-12-17 12:30:47,788] jepsen worker 4 - jepsen.util 3154 :info :write [1 12] :RequestCanceledException INFO [2019-12-17 12:30:47,788] jepsen worker 41 - jepsen.util 5351 :info :cas [13 [38 35]] :RequestCanceledException INFO [2019-12-17 12:30:47,789] jepsen worker 41 - jepsen.util 5441 :invoke :read [13 nil] INFO [2019-12-17 12:30:47,790] jepsen worker 4 - jepsen.util 3244 :invoke :write [1 33] INFO [2019-12-17 12:30:47,790] jepsen worker 72 - jepsen.util 4122 :fail :cas [24 [9 7]] :ValueNotSwapFrom INFO [2019-12-17 12:30:47,790] jepsen worker 72 - jepsen.util 4122 :invoke :write [24 25] INFO [2019-12-17 12:30:47,796] jepsen worker 29 - jepsen.util 3269 :ok :read [9 24] INFO [2019-12-17 12:30:47,796] jepsen worker 29 - jepsen.util 3269 :invoke :cas [9 [28 29]] INFO [2019-12-17 12:30:47,799] jepsen worker 51 - jepsen.util 3561 :fail :cas [17 [17 49]] :ValueNotSwapFrom INFO [2019-12-17 12:30:47,799] jepsen worker 51 - jepsen.util 3561 :invoke :write [17 33] INFO [2019-12-17 12:30:47,806] jepsen worker 78 - jepsen.util 4398 :fail :cas [26 [35 14]] :ValueNotSwapFrom INFO [2019-12-17 12:30:47,806] jepsen worker 78 - jepsen.util 4398 :invoke :read [26 nil] INFO [2019-12-17 12:30:47,808] jepsen worker 1 - jepsen.util 4951 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:30:47,808] jepsen worker 1 - jepsen.util 4951 :invoke :read [0 nil] INFO [2019-12-17 12:30:47,813] jepsen worker 29 - jepsen.util 3269 :fail :cas [9 [28 29]] :ValueNotSwapFrom INFO [2019-12-17 12:30:47,813] jepsen worker 29 - jepsen.util 3269 :invoke :write [9 36] INFO [2019-12-17 12:30:47,824] jepsen worker 13 - jepsen.util 3703 :fail :cas [4 [27 14]] :ValueNotSwapFrom INFO [2019-12-17 12:30:47,824] jepsen worker 13 - jepsen.util 3703 :invoke :write [4 21] INFO [2019-12-17 12:30:47,835] jepsen worker 19 - jepsen.util 4069 :ok :write [6 3] INFO [2019-12-17 12:30:47,835] jepsen worker 19 - jepsen.util 4069 :invoke :read [6 nil] INFO [2019-12-17 12:30:47,838] jepsen worker 16 - jepsen.util 5866 :info :write [5 49] :RequestCanceledException INFO [2019-12-17 12:30:47,838] jepsen worker 73 - jepsen.util 3763 :info :write [24 3] :RequestCanceledException INFO [2019-12-17 12:30:47,838] jepsen worker 69 - jepsen.util 3849 :info :write [23 20] :RequestCanceledException INFO [2019-12-17 12:30:47,839] jepsen worker 69 - jepsen.util 3939 :invoke :read [23 nil] INFO [2019-12-17 12:30:47,839] jepsen worker 16 - jepsen.util 5956 :invoke :cas [5 [42 18]] INFO [2019-12-17 12:30:47,840] jepsen worker 73 - jepsen.util 3853 :invoke :cas [24 [0 33]] INFO [2019-12-17 12:30:47,853] jepsen worker 76 - jepsen.util 5296 :info :cas [25 [3 3]] :RequestCanceledException INFO [2019-12-17 12:30:47,863] jepsen worker 76 - jepsen.util 5386 :invoke :read [25 nil] INFO [2019-12-17 12:30:47,867] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":22471,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:30:47,867] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":22471,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:30:47,867] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":22474,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:30:47,867] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":22474,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:30:47,867] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":22475,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:30:47,867] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":22475,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:30:47,876] jepsen worker 47 - jepsen.util 4457 :ok :write [15 25] INFO [2019-12-17 12:30:47,876] jepsen worker 47 - jepsen.util 4457 :invoke :write [15 11] INFO [2019-12-17 12:30:47,899] jepsen worker 29 - jepsen.util 3269 :ok :write [9 36] INFO [2019-12-17 12:30:47,899] jepsen worker 29 - jepsen.util 3269 :invoke :write [9 42] INFO [2019-12-17 12:30:47,921] jepsen worker 15 - jepsen.util 4695 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:30:47,922] jepsen worker 28 - jepsen.util 2728 :info :cas [9 [3 19]] :RequestCanceledException INFO [2019-12-17 12:30:47,922] jepsen worker 8 - jepsen.util 4328 :info :write [2 13] :RequestCanceledException INFO [2019-12-17 12:30:47,922] jepsen worker 15 - jepsen.util 4695 :invoke :cas [5 [40 47]] INFO [2019-12-17 12:30:47,922] jepsen worker 8 - jepsen.util 4418 :invoke :write [2 1] INFO [2019-12-17 12:30:47,924] jepsen worker 28 - jepsen.util 2818 :invoke :cas [9 [48 12]] INFO [2019-12-17 12:30:47,938] jepsen worker 28 - jepsen.util 2818 :fail :cas [9 [48 12]] :ValueNotSwapFrom INFO [2019-12-17 12:30:47,938] jepsen worker 28 - jepsen.util 2818 :invoke :cas [9 [38 9]] INFO [2019-12-17 12:30:47,959] jepsen worker 28 - jepsen.util 2818 :fail :cas [9 [38 9]] :ValueNotSwapFrom INFO [2019-12-17 12:30:47,959] jepsen worker 29 - jepsen.util 3269 :ok :write [9 42] INFO [2019-12-17 12:30:47,959] jepsen worker 28 - jepsen.util 2818 :invoke :cas [9 [24 31]] INFO [2019-12-17 12:30:47,959] jepsen worker 29 - jepsen.util 3269 :invoke :cas [9 [13 37]] INFO [2019-12-17 12:30:47,983] jepsen worker 29 - jepsen.util 3269 :fail :cas [9 [13 37]] :ValueNotSwapFrom INFO [2019-12-17 12:30:47,983] jepsen worker 29 - jepsen.util 3269 :invoke :read [9 nil] INFO [2019-12-17 12:30:47,991] jepsen worker 28 - jepsen.util 2818 :fail :cas [9 [24 31]] :ValueNotSwapFrom INFO [2019-12-17 12:30:47,991] jepsen worker 28 - jepsen.util 2818 :invoke :read [9 nil] INFO [2019-12-17 12:30:47,993] jepsen worker 53 - jepsen.util 4013 :fail :cas [17 [15 36]] :ValueNotSwapFrom INFO [2019-12-17 12:30:47,993] jepsen worker 69 - jepsen.util 3939 :ok :read [23 45] INFO [2019-12-17 12:30:47,993] jepsen worker 69 - jepsen.util 3939 :invoke :write [23 1] INFO [2019-12-17 12:30:47,993] jepsen worker 53 - jepsen.util 4013 :invoke :cas [17 [10 11]] INFO [2019-12-17 12:30:47,997] jepsen worker 29 - jepsen.util 3269 :ok :read [9 42] INFO [2019-12-17 12:30:47,997] jepsen worker 29 - jepsen.util 3269 :invoke :read [9 nil] INFO [2019-12-17 12:30:48,009] jepsen worker 31 - jepsen.util 3451 :info :write [10 48] :RequestCanceledException INFO [2019-12-17 12:30:48,011] jepsen worker 28 - jepsen.util 2818 :ok :read [9 42] INFO [2019-12-17 12:30:48,011] jepsen worker 28 - jepsen.util 2818 :invoke :read [9 nil] INFO [2019-12-17 12:30:48,011] jepsen worker 31 - jepsen.util 3541 :invoke :read [10 nil] INFO [2019-12-17 12:30:48,023] jepsen worker 29 - jepsen.util 3269 :ok :read [9 42] INFO [2019-12-17 12:30:48,023] jepsen worker 29 - jepsen.util 3269 :invoke :write [9 42] INFO [2019-12-17 12:30:48,024] jepsen worker 81 - jepsen.util 5211 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:30:48,025] jepsen worker 81 - jepsen.util 5211 :invoke :cas [27 [7 23]] INFO [2019-12-17 12:30:48,030] jepsen worker 28 - jepsen.util 2818 :ok :read [9 42] INFO [2019-12-17 12:30:48,030] jepsen worker 28 - jepsen.util 2818 :invoke :read [9 nil] INFO [2019-12-17 12:30:48,035] jepsen worker 31 - jepsen.util 3541 :ok :read [10 35] INFO [2019-12-17 12:30:48,035] jepsen worker 31 - jepsen.util 3541 :invoke :write [10 22] INFO [2019-12-17 12:30:48,052] jepsen worker 28 - jepsen.util 2818 :ok :read [9 42] INFO [2019-12-17 12:30:48,052] jepsen worker 28 - jepsen.util 2818 :invoke :read [9 nil] INFO [2019-12-17 12:30:48,068] jepsen worker 80 - jepsen.util 4040 :ok :read [26 28] INFO [2019-12-17 12:30:48,069] jepsen worker 80 - jepsen.util 4040 :invoke :write [26 41] INFO [2019-12-17 12:30:48,069] jepsen worker 28 - jepsen.util 2818 :ok :read [9 42] INFO [2019-12-17 12:30:48,069] jepsen worker 28 - jepsen.util 2818 :invoke :write [9 15] INFO [2019-12-17 12:30:48,091] jepsen worker 27 - jepsen.util 3807 :info :write [9 9] :RequestCanceledException INFO [2019-12-17 12:30:48,092] jepsen worker 29 - jepsen.util 3269 :ok :write [9 42] INFO [2019-12-17 12:30:48,092] jepsen worker 29 - jepsen.util 3269 :invoke :cas [9 [12 35]] INFO [2019-12-17 12:30:48,093] jepsen worker 27 - jepsen.util 3897 :invoke :cas [9 [16 15]] INFO [2019-12-17 12:30:48,098] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":22487,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:30:48,098] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":22487,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:30:48,098] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":22491,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:30:48,098] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":22491,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:30:48,134] jepsen worker 28 - jepsen.util 2818 :ok :write [9 15] INFO [2019-12-17 12:30:48,135] jepsen worker 28 - jepsen.util 2818 :invoke :write [9 12] INFO [2019-12-17 12:30:48,135] jepsen worker 29 - jepsen.util 3269 :fail :cas [9 [12 35]] :ValueNotSwapFrom INFO [2019-12-17 12:30:48,135] jepsen worker 29 - jepsen.util 3269 :invoke :read [9 nil] INFO [2019-12-17 12:30:48,139] jepsen worker 21 - jepsen.util 5151 :info :write [7 24] :RequestCanceledException INFO [2019-12-17 12:30:48,142] jepsen worker 21 - jepsen.util 5241 :invoke :write [7 37] INFO [2019-12-17 12:30:48,159] jepsen worker 29 - jepsen.util 3269 :ok :read [9 15] INFO [2019-12-17 12:30:48,159] jepsen worker 29 - jepsen.util 3269 :invoke :cas [9 [27 4]] INFO [2019-12-17 12:30:48,173] jepsen worker 29 - jepsen.util 3269 :fail :cas [9 [27 4]] :ValueNotSwapFrom INFO [2019-12-17 12:30:48,173] jepsen worker 29 - jepsen.util 3269 :invoke :read [9 nil] INFO [2019-12-17 12:30:48,179] jepsen worker 54 - jepsen.util 4374 :fail :cas [18 [6 31]] :ValueNotSwapFrom INFO [2019-12-17 12:30:48,179] jepsen worker 54 - jepsen.util 4374 :invoke :cas [18 [14 0]] INFO [2019-12-17 12:30:48,191] jepsen worker 28 - jepsen.util 2818 :ok :write [9 12] INFO [2019-12-17 12:30:48,192] jepsen worker 28 - jepsen.util 2818 :invoke :write [9 4] INFO [2019-12-17 12:30:48,200] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":22513,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:30:48,200] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":22513,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:30:48,200] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":22515,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:30:48,200] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":22515,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:30:48,207] jepsen worker 29 - jepsen.util 3269 :ok :read [9 15] INFO [2019-12-17 12:30:48,207] jepsen worker 29 - jepsen.util 3269 :invoke :write [9 0] INFO [2019-12-17 12:30:48,213] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:30:48,213] jepsen nemesis - couchbase.util Rebalance check count: 167 INFO [2019-12-17 12:30:48,213] jepsen nemesis - couchbase.util Rebalance stuck count: 148 INFO [2019-12-17 12:30:48,220] jepsen worker 72 - jepsen.util 4122 :ok :write [24 25] INFO [2019-12-17 12:30:48,221] jepsen worker 72 - jepsen.util 4122 :invoke :cas [24 [14 16]] INFO [2019-12-17 12:30:48,225] jepsen worker 71 - jepsen.util 4121 :info :write [23 42] :RequestCanceledException INFO [2019-12-17 12:30:48,226] jepsen worker 71 - jepsen.util 4211 :invoke :cas [23 [1 13]] INFO [2019-12-17 12:30:48,243] jepsen worker 28 - jepsen.util 2818 :ok :write [9 4] INFO [2019-12-17 12:30:48,244] jepsen worker 28 - jepsen.util 2818 :invoke :read [9 nil] INFO [2019-12-17 12:30:48,260] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":22520,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:30:48,260] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":22520,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:30:48,260] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":22521,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:30:48,260] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":22521,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:30:48,260] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":22523,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:30:48,260] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":22523,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:30:48,270] jepsen worker 72 - jepsen.util 4122 :fail :cas [24 [14 16]] :ValueNotSwapFrom INFO [2019-12-17 12:30:48,270] jepsen worker 28 - jepsen.util 2818 :ok :read [9 4] INFO [2019-12-17 12:30:48,270] jepsen worker 72 - jepsen.util 4122 :invoke :read [24 nil] INFO [2019-12-17 12:30:48,270] jepsen worker 28 - jepsen.util 2818 :invoke :write [9 6] INFO [2019-12-17 12:30:48,291] jepsen worker 29 - jepsen.util 3269 :ok :write [9 0] INFO [2019-12-17 12:30:48,292] jepsen worker 29 - jepsen.util 3269 :invoke :read [9 nil] INFO [2019-12-17 12:30:48,293] jepsen worker 56 - jepsen.util 4106 :fail :cas [18 [48 2]] :ValueNotSwapFrom INFO [2019-12-17 12:30:48,294] jepsen worker 56 - jepsen.util 4106 :invoke :read [18 nil] INFO [2019-12-17 12:30:48,299] jepsen worker 72 - jepsen.util 4122 :ok :read [24 25] INFO [2019-12-17 12:30:48,299] jepsen worker 72 - jepsen.util 4122 :invoke :read [24 nil] INFO [2019-12-17 12:30:48,307] jepsen worker 61 - jepsen.util 6001 :info :cas [20 [29 28]] :RequestCanceledException INFO [2019-12-17 12:30:48,308] jepsen worker 61 - jepsen.util 6091 :invoke :read [20 nil] INFO [2019-12-17 12:30:48,319] jepsen worker 29 - jepsen.util 3269 :ok :read [9 0] INFO [2019-12-17 12:30:48,319] jepsen worker 29 - jepsen.util 3269 :invoke :write [9 33] INFO [2019-12-17 12:30:48,328] jepsen worker 72 - jepsen.util 4122 :ok :read [24 25] INFO [2019-12-17 12:30:48,329] jepsen worker 72 - jepsen.util 4122 :invoke :write [24 9] INFO [2019-12-17 12:30:48,348] jepsen worker 28 - jepsen.util 2818 :ok :write [9 6] INFO [2019-12-17 12:30:48,348] jepsen worker 28 - jepsen.util 2818 :invoke :cas [9 [32 23]] INFO [2019-12-17 12:30:48,364] jepsen worker 28 - jepsen.util 2818 :fail :cas [9 [32 23]] :ValueNotSwapFrom INFO [2019-12-17 12:30:48,364] jepsen worker 28 - jepsen.util 2818 :invoke :cas [9 [25 48]] INFO [2019-12-17 12:30:48,382] jepsen worker 28 - jepsen.util 2818 :fail :cas [9 [25 48]] :ValueNotSwapFrom INFO [2019-12-17 12:30:48,384] jepsen worker 28 - jepsen.util 2818 :invoke :cas [9 [3 8]] INFO [2019-12-17 12:30:48,393] jepsen worker 29 - jepsen.util 3269 :ok :write [9 33] INFO [2019-12-17 12:30:48,393] jepsen worker 29 - jepsen.util 3269 :invoke :cas [9 [48 3]] INFO [2019-12-17 12:30:48,398] jepsen worker 28 - jepsen.util 2818 :fail :cas [9 [3 8]] :ValueNotSwapFrom INFO [2019-12-17 12:30:48,398] jepsen worker 28 - jepsen.util 2818 :invoke :cas [9 [7 38]] INFO [2019-12-17 12:30:48,413] jepsen worker 29 - jepsen.util 3269 :fail :cas [9 [48 3]] :ValueNotSwapFrom INFO [2019-12-17 12:30:48,414] jepsen worker 29 - jepsen.util 3269 :invoke :cas [9 [7 18]] INFO [2019-12-17 12:30:48,416] jepsen worker 28 - jepsen.util 2818 :fail :cas [9 [7 38]] :ValueNotSwapFrom INFO [2019-12-17 12:30:48,417] jepsen worker 28 - jepsen.util 2818 :invoke :cas [9 [35 17]] INFO [2019-12-17 12:30:48,426] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":22546,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:30:48,426] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":22546,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:30:48,426] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":22550,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:30:48,426] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":22550,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:30:48,430] jepsen worker 29 - jepsen.util 3269 :fail :cas [9 [7 18]] :ValueNotSwapFrom INFO [2019-12-17 12:30:48,431] jepsen worker 29 - jepsen.util 3269 :invoke :write [9 20] INFO [2019-12-17 12:30:48,459] jepsen worker 28 - jepsen.util 2818 :fail :cas [9 [35 17]] :ValueNotSwapFrom INFO [2019-12-17 12:30:48,480] jepsen worker 29 - jepsen.util 3269 :ok :write [9 20] INFO [2019-12-17 12:30:48,483] jepsen worker 14 - jepsen.util 3974 :ok :read [4 49] INFO [2019-12-17 12:30:48,491] jepsen worker 87 - jepsen.util 3687 :info :write [29 19] :RequestCanceledException INFO [2019-12-17 12:30:48,500] jepsen worker 37 - jepsen.util 3727 :fail :cas [12 [4 18]] :ValueNotSwapFrom INFO [2019-12-17 12:30:48,522] jepsen worker 78 - jepsen.util 4398 :ok :read [26 28] INFO [2019-12-17 12:30:48,523] jepsen worker 86 - jepsen.util 5486 :info :cas [28 [33 10]] :RequestCanceledException INFO [2019-12-17 12:30:48,536] jepsen worker 77 - jepsen.util 5207 :info :cas [25 [0 25]] :RequestCanceledException INFO [2019-12-17 12:30:48,555] jepsen worker 12 - jepsen.util 4242 :info :write [4 17] :RequestCanceledException INFO [2019-12-17 12:30:48,568] jepsen worker 53 - jepsen.util 4013 :fail :cas [17 [10 11]] :ValueNotSwapFrom INFO [2019-12-17 12:30:48,571] jepsen worker 28 - jepsen.util 2818 :invoke :read [9 nil] INFO [2019-12-17 12:30:48,571] jepsen worker 29 - jepsen.util 3269 :invoke :read [9 nil] INFO [2019-12-17 12:30:48,571] jepsen worker 14 - jepsen.util 3974 :invoke :write [4 13] INFO [2019-12-17 12:30:48,571] jepsen worker 37 - jepsen.util 3727 :invoke :cas [12 [8 1]] INFO [2019-12-17 12:30:48,571] jepsen worker 78 - jepsen.util 4398 :invoke :write [26 4] INFO [2019-12-17 12:30:48,571] jepsen worker 53 - jepsen.util 4013 :invoke :read [17 nil] INFO [2019-12-17 12:30:48,573] jepsen worker 86 - jepsen.util 5576 :invoke :read [28 nil] INFO [2019-12-17 12:30:48,573] jepsen worker 77 - jepsen.util 5297 :invoke :cas [25 [44 45]] INFO [2019-12-17 12:30:48,573] jepsen worker 12 - jepsen.util 4332 :invoke :cas [4 [19 44]] INFO [2019-12-17 12:30:48,573] jepsen worker 87 - jepsen.util 3777 :invoke :read [29 nil] INFO [2019-12-17 12:30:48,593] jepsen worker 29 - jepsen.util 3269 :ok :read [9 20] INFO [2019-12-17 12:30:48,594] jepsen worker 29 - jepsen.util 3269 :invoke :cas [9 [3 22]] INFO [2019-12-17 12:30:48,604] jepsen worker 29 - jepsen.util 3269 :fail :cas [9 [3 22]] :ValueNotSwapFrom INFO [2019-12-17 12:30:48,604] jepsen worker 29 - jepsen.util 3269 :invoke :read [9 nil] INFO [2019-12-17 12:30:48,608] jepsen worker 28 - jepsen.util 2818 :ok :read [9 20] INFO [2019-12-17 12:30:48,609] jepsen worker 28 - jepsen.util 2818 :invoke :read [9 nil] INFO [2019-12-17 12:30:48,624] jepsen worker 24 - jepsen.util 4974 :info :cas [8 [37 49]] :RequestCanceledException INFO [2019-12-17 12:30:48,625] jepsen worker 24 - jepsen.util 5064 :invoke :write [8 7] INFO [2019-12-17 12:30:48,626] jepsen worker 29 - jepsen.util 3269 :ok :read [9 20] INFO [2019-12-17 12:30:48,626] jepsen worker 29 - jepsen.util 3269 :invoke :write [9 42] INFO [2019-12-17 12:30:48,632] jepsen worker 56 - jepsen.util 4106 :ok :read [18 25] INFO [2019-12-17 12:30:48,632] jepsen worker 56 - jepsen.util 4106 :invoke :write [18 22] INFO [2019-12-17 12:30:48,634] jepsen worker 28 - jepsen.util 2818 :ok :read [9 20] INFO [2019-12-17 12:30:48,634] jepsen worker 28 - jepsen.util 2818 :invoke :read [9 nil] INFO [2019-12-17 12:30:48,645] jepsen worker 54 - jepsen.util 4374 :fail :cas [18 [14 0]] :ValueNotSwapFrom INFO [2019-12-17 12:30:48,645] jepsen worker 54 - jepsen.util 4374 :invoke :write [18 21] INFO [2019-12-17 12:30:48,671] jepsen worker 28 - jepsen.util 2818 :ok :read [9 20] INFO [2019-12-17 12:30:48,672] jepsen worker 28 - jepsen.util 2818 :invoke :cas [9 [25 33]] INFO [2019-12-17 12:30:48,687] jepsen worker 28 - jepsen.util 2818 :fail :cas [9 [25 33]] :ValueNotSwapFrom INFO [2019-12-17 12:30:48,687] jepsen worker 28 - jepsen.util 2818 :invoke :write [9 27] INFO [2019-12-17 12:30:48,691] jepsen worker 49 - jepsen.util 4549 :info :cas [16 [27 11]] :RequestCanceledException INFO [2019-12-17 12:30:48,692] jepsen worker 49 - jepsen.util 4639 :invoke :cas [16 [20 17]] INFO [2019-12-17 12:30:48,693] jepsen worker 12 - jepsen.util 4332 :fail :cas [4 [19 44]] :ValueNotSwapFrom INFO [2019-12-17 12:30:48,693] jepsen worker 12 - jepsen.util 4332 :invoke :read [4 nil] INFO [2019-12-17 12:30:48,705] jepsen worker 29 - jepsen.util 3269 :ok :write [9 42] INFO [2019-12-17 12:30:48,705] jepsen worker 29 - jepsen.util 3269 :invoke :write [9 12] INFO [2019-12-17 12:30:48,712] jepsen worker 12 - jepsen.util 4332 :ok :read [4 49] INFO [2019-12-17 12:30:48,713] jepsen worker 12 - jepsen.util 4332 :invoke :read [4 nil] INFO [2019-12-17 12:30:48,738] jepsen worker 65 - jepsen.util 3215 :info :write [21 15] :RequestCanceledException INFO [2019-12-17 12:30:48,739] jepsen worker 65 - jepsen.util 3305 :invoke :cas [21 [27 46]] INFO [2019-12-17 12:30:48,743] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":22569,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:30:48,743] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":22569,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:30:48,743] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":22570,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:30:48,743] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":22570,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:30:48,743] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":22571,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:30:48,743] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":22571,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:30:48,748] jepsen worker 28 - jepsen.util 2818 :ok :write [9 27] INFO [2019-12-17 12:30:48,748] jepsen worker 28 - jepsen.util 2818 :invoke :cas [9 [18 15]] INFO [2019-12-17 12:30:48,755] jepsen worker 34 - jepsen.util 4984 :info :cas [11 [35 15]] :RequestCanceledException INFO [2019-12-17 12:30:48,757] jepsen worker 34 - jepsen.util 5074 :invoke :read [11 nil] INFO [2019-12-17 12:30:48,760] jepsen worker 65 - jepsen.util 3305 :fail :cas [21 [27 46]] :ValueNotSwapFrom INFO [2019-12-17 12:30:48,760] jepsen worker 65 - jepsen.util 3305 :invoke :write [21 45] INFO [2019-12-17 12:30:48,772] jepsen worker 28 - jepsen.util 2818 :fail :cas [9 [18 15]] :ValueNotSwapFrom INFO [2019-12-17 12:30:48,772] jepsen worker 12 - jepsen.util 4332 :ok :read [4 49] INFO [2019-12-17 12:30:48,772] jepsen worker 12 - jepsen.util 4332 :invoke :write [4 30] INFO [2019-12-17 12:30:48,772] jepsen worker 28 - jepsen.util 2818 :invoke :cas [9 [5 37]] INFO [2019-12-17 12:30:48,798] jepsen worker 65 - jepsen.util 3305 :ok :write [21 45] INFO [2019-12-17 12:30:48,799] jepsen worker 65 - jepsen.util 3305 :invoke :write [21 3] INFO [2019-12-17 12:30:48,833] jepsen worker 71 - jepsen.util 4211 :fail :cas [23 [1 13]] :ValueNotSwapFrom INFO [2019-12-17 12:30:48,833] jepsen worker 71 - jepsen.util 4211 :invoke :cas [23 [47 24]] INFO [2019-12-17 12:30:48,839] jepsen worker 32 - jepsen.util 4352 :info :write [10 10] :RequestCanceledException INFO [2019-12-17 12:30:48,839] jepsen worker 38 - jepsen.util 3818 :info :write [12 14] :RequestCanceledException INFO [2019-12-17 12:30:48,840] jepsen worker 65 - jepsen.util 3305 :ok :write [21 3] INFO [2019-12-17 12:30:48,840] jepsen worker 65 - jepsen.util 3305 :invoke :cas [21 [41 12]] INFO [2019-12-17 12:30:48,840] jepsen worker 32 - jepsen.util 4442 :invoke :cas [10 [0 45]] INFO [2019-12-17 12:30:48,842] jepsen worker 38 - jepsen.util 3908 :invoke :write [12 4] INFO [2019-12-17 12:30:48,855] jepsen worker 65 - jepsen.util 3305 :fail :cas [21 [41 12]] :ValueNotSwapFrom INFO [2019-12-17 12:30:48,855] jepsen worker 65 - jepsen.util 3305 :invoke :cas [21 [0 36]] INFO [2019-12-17 12:30:48,856] jepsen worker 64 - jepsen.util 3124 :info :write [21 41] :RequestCanceledException INFO [2019-12-17 12:30:48,857] jepsen worker 64 - jepsen.util 3214 :invoke :read [21 nil] INFO [2019-12-17 12:30:48,885] jepsen worker 64 - jepsen.util 3214 :ok :read [21 3] INFO [2019-12-17 12:30:48,885] jepsen worker 64 - jepsen.util 3214 :invoke :read [21 nil] INFO [2019-12-17 12:30:48,888] jepsen worker 65 - jepsen.util 3305 :fail :cas [21 [0 36]] :ValueNotSwapFrom INFO [2019-12-17 12:30:48,888] jepsen worker 65 - jepsen.util 3305 :invoke :read [21 nil] INFO [2019-12-17 12:30:48,891] jepsen worker 88 - jepsen.util 4048 :info :write [29 15] :RequestCanceledException INFO [2019-12-17 12:30:48,895] jepsen worker 88 - jepsen.util 4138 :invoke :cas [29 [11 15]] INFO [2019-12-17 12:30:48,900] jepsen worker 65 - jepsen.util 3305 :ok :read [21 3] INFO [2019-12-17 12:30:48,901] jepsen worker 65 - jepsen.util 3305 :invoke :cas [21 [23 40]] INFO [2019-12-17 12:30:48,906] jepsen worker 64 - jepsen.util 3214 :ok :read [21 3] INFO [2019-12-17 12:30:48,906] jepsen worker 64 - jepsen.util 3214 :invoke :write [21 17] INFO [2019-12-17 12:30:48,915] jepsen worker 65 - jepsen.util 3305 :fail :cas [21 [23 40]] :ValueNotSwapFrom INFO [2019-12-17 12:30:48,915] jepsen worker 65 - jepsen.util 3305 :invoke :write [21 38] INFO [2019-12-17 12:30:48,924] jepsen worker 68 - jepsen.util 4748 :info :cas [22 [20 15]] :RequestCanceledException INFO [2019-12-17 12:30:48,926] jepsen worker 68 - jepsen.util 4838 :invoke :write [22 43] INFO [2019-12-17 12:30:48,928] jepsen worker 87 - jepsen.util 3777 :ok :read [29 18] INFO [2019-12-17 12:30:48,928] jepsen worker 87 - jepsen.util 3777 :invoke :read [29 nil] INFO [2019-12-17 12:30:48,952] jepsen worker 79 - jepsen.util 3859 :info :write [26 1] :RequestCanceledException INFO [2019-12-17 12:30:48,952] jepsen worker 36 - jepsen.util 4086 :info :write [12 33] :RequestCanceledException INFO [2019-12-17 12:30:48,953] jepsen worker 79 - jepsen.util 3949 :invoke :read [26 nil] INFO [2019-12-17 12:30:48,953] jepsen worker 36 - jepsen.util 4176 :invoke :cas [12 [8 36]] INFO [2019-12-17 12:30:48,957] jepsen worker 64 - jepsen.util 3214 :ok :write [21 17] INFO [2019-12-17 12:30:48,957] jepsen worker 64 - jepsen.util 3214 :invoke :cas [21 [8 9]] INFO [2019-12-17 12:30:48,962] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":22595,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:30:48,962] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":22595,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:30:48,962] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":22598,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:30:48,962] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":22598,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:30:48,974] jepsen worker 64 - jepsen.util 3214 :fail :cas [21 [8 9]] :ValueNotSwapFrom INFO [2019-12-17 12:30:48,974] jepsen worker 64 - jepsen.util 3214 :invoke :cas [21 [47 7]] INFO [2019-12-17 12:30:48,995] jepsen worker 64 - jepsen.util 3214 :fail :cas [21 [47 7]] :ValueNotSwapFrom INFO [2019-12-17 12:30:48,995] jepsen worker 64 - jepsen.util 3214 :invoke :write [21 26] INFO [2019-12-17 12:30:49,003] jepsen worker 65 - jepsen.util 3305 :ok :write [21 38] INFO [2019-12-17 12:30:49,003] jepsen worker 65 - jepsen.util 3305 :invoke :read [21 nil] INFO [2019-12-17 12:30:49,023] jepsen worker 65 - jepsen.util 3305 :ok :read [21 38] INFO [2019-12-17 12:30:49,023] jepsen worker 65 - jepsen.util 3305 :invoke :cas [21 [19 48]] INFO [2019-12-17 12:30:49,039] jepsen worker 23 - jepsen.util 5063 :info :cas [7 [8 0]] :RequestCanceledException INFO [2019-12-17 12:30:49,039] jepsen worker 23 - jepsen.util 5153 :invoke :write [7 16] INFO [2019-12-17 12:30:49,055] jepsen worker 65 - jepsen.util 3305 :fail :cas [21 [19 48]] :ValueNotSwapFrom INFO [2019-12-17 12:30:49,055] jepsen worker 64 - jepsen.util 3214 :ok :write [21 26] INFO [2019-12-17 12:30:49,055] jepsen worker 65 - jepsen.util 3305 :invoke :read [21 nil] INFO [2019-12-17 12:30:49,055] jepsen worker 64 - jepsen.util 3214 :invoke :write [21 39] INFO [2019-12-17 12:30:49,069] jepsen worker 65 - jepsen.util 3305 :ok :read [21 26] INFO [2019-12-17 12:30:49,069] jepsen worker 65 - jepsen.util 3305 :invoke :read [21 nil] INFO [2019-12-17 12:30:49,076] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":22614,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:30:49,076] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":22614,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:30:49,076] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":22618,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:30:49,076] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":22618,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:30:49,094] jepsen worker 65 - jepsen.util 3305 :ok :read [21 26] INFO [2019-12-17 12:30:49,094] jepsen worker 65 - jepsen.util 3305 :invoke :read [21 nil] INFO [2019-12-17 12:30:49,106] jepsen worker 64 - jepsen.util 3214 :ok :write [21 39] INFO [2019-12-17 12:30:49,106] jepsen worker 64 - jepsen.util 3214 :invoke :cas [21 [5 20]] INFO [2019-12-17 12:30:49,109] jepsen worker 44 - jepsen.util 5624 :info :cas [14 [48 4]] :RequestCanceledException INFO [2019-12-17 12:30:49,110] jepsen worker 65 - jepsen.util 3305 :ok :read [21 26] INFO [2019-12-17 12:30:49,110] jepsen worker 65 - jepsen.util 3305 :invoke :read [21 nil] INFO [2019-12-17 12:30:49,110] jepsen worker 44 - jepsen.util 5714 :invoke :write [14 35] INFO [2019-12-17 12:30:49,125] jepsen worker 45 - jepsen.util 4095 :info :write [15 14] :RequestCanceledException INFO [2019-12-17 12:30:49,125] jepsen worker 59 - jepsen.util 5279 :info :write [19 9] :RequestCanceledException INFO [2019-12-17 12:30:49,125] jepsen worker 64 - jepsen.util 3214 :fail :cas [21 [5 20]] :ValueNotSwapFrom INFO [2019-12-17 12:30:49,125] jepsen worker 64 - jepsen.util 3214 :invoke :read [21 nil] INFO [2019-12-17 12:30:49,126] jepsen worker 45 - jepsen.util 4185 :invoke :read [15 nil] INFO [2019-12-17 12:30:49,126] jepsen worker 59 - jepsen.util 5369 :invoke :read [19 nil] INFO [2019-12-17 12:30:49,130] jepsen worker 65 - jepsen.util 3305 :ok :read [21 39] INFO [2019-12-17 12:30:49,130] jepsen worker 65 - jepsen.util 3305 :invoke :write [21 4] INFO [2019-12-17 12:30:49,138] jepsen worker 20 - jepsen.util 3800 :info :cas [6 [36 18]] :RequestCanceledException INFO [2019-12-17 12:30:49,138] jepsen worker 84 - jepsen.util 4854 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:30:49,138] jepsen worker 10 - jepsen.util 3880 :ok :write [3 24] INFO [2019-12-17 12:30:49,138] jepsen worker 84 - jepsen.util 4854 :invoke :read [28 nil] INFO [2019-12-17 12:30:49,139] jepsen worker 10 - jepsen.util 3880 :invoke :write [3 11] INFO [2019-12-17 12:30:49,140] jepsen worker 20 - jepsen.util 3890 :invoke :read [6 nil] INFO [2019-12-17 12:30:49,140] jepsen worker 72 - jepsen.util 4122 :ok :write [24 9] INFO [2019-12-17 12:30:49,140] jepsen worker 72 - jepsen.util 4122 :invoke :cas [24 [49 38]] INFO [2019-12-17 12:30:49,151] jepsen worker 87 - jepsen.util 3777 :ok :read [29 18] INFO [2019-12-17 12:30:49,151] jepsen worker 87 - jepsen.util 3777 :invoke :read [29 nil] INFO [2019-12-17 12:30:49,156] jepsen worker 66 - jepsen.util 5106 :info :write [22 19] :RequestCanceledException INFO [2019-12-17 12:30:49,156] jepsen worker 11 - jepsen.util 4331 :info :write [3 22] :RequestCanceledException INFO [2019-12-17 12:30:49,157] jepsen worker 11 - jepsen.util 4421 :invoke :read [3 nil] INFO [2019-12-17 12:30:49,161] jepsen worker 66 - jepsen.util 5196 :invoke :cas [22 [31 38]] INFO [2019-12-17 12:30:49,181] jepsen worker 47 - jepsen.util 4457 :ok :write [15 11] INFO [2019-12-17 12:30:49,181] jepsen worker 47 - jepsen.util 4457 :invoke :read [15 nil] INFO [2019-12-17 12:30:49,187] jepsen worker 88 - jepsen.util 4138 :fail :cas [29 [11 15]] :ValueNotSwapFrom INFO [2019-12-17 12:30:49,187] jepsen worker 88 - jepsen.util 4138 :invoke :cas [29 [35 15]] INFO [2019-12-17 12:30:49,198] jepsen worker 73 - jepsen.util 3853 :fail :cas [24 [0 33]] :ValueNotSwapFrom INFO [2019-12-17 12:30:49,198] jepsen worker 73 - jepsen.util 3853 :invoke :write [24 39] INFO [2019-12-17 12:30:49,208] jepsen worker 43 - jepsen.util 4813 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:30:49,208] jepsen worker 43 - jepsen.util 4813 :invoke :cas [14 [16 38]] INFO [2019-12-17 12:30:49,217] jepsen worker 64 - jepsen.util 3214 :ok :read [21 39] INFO [2019-12-17 12:30:49,217] jepsen worker 64 - jepsen.util 3214 :invoke :write [21 44] INFO [2019-12-17 12:30:49,223] jepsen worker 20 - jepsen.util 3890 :ok :read [6 3] INFO [2019-12-17 12:30:49,223] jepsen worker 20 - jepsen.util 3890 :invoke :write [6 9] INFO [2019-12-17 12:30:49,224] jepsen worker 25 - jepsen.util 5425 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:30:49,224] jepsen worker 18 - jepsen.util 3978 :info :write [6 9] :RequestCanceledException INFO [2019-12-17 12:30:49,224] jepsen worker 25 - jepsen.util 5425 :invoke :cas [8 [43 12]] INFO [2019-12-17 12:30:49,225] jepsen worker 18 - jepsen.util 4068 :invoke :write [6 48] INFO [2019-12-17 12:30:49,244] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":22652,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:30:49,244] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":22652,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:30:49,244] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":22655,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:30:49,244] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":22655,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:30:49,255] jepsen worker 6 - jepsen.util 5676 :info :cas [2 [45 33]] :RequestCanceledException INFO [2019-12-17 12:30:49,256] jepsen worker 79 - jepsen.util 3949 :ok :read [26 28] INFO [2019-12-17 12:30:49,256] jepsen worker 79 - jepsen.util 3949 :invoke :cas [26 [19 13]] INFO [2019-12-17 12:30:49,256] jepsen worker 6 - jepsen.util 5766 :invoke :cas [2 [43 10]] INFO [2019-12-17 12:30:49,264] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:30:49,264] jepsen nemesis - couchbase.util Rebalance check count: 168 INFO [2019-12-17 12:30:49,264] jepsen worker 88 - jepsen.util 4138 :fail :cas [29 [35 15]] :ValueNotSwapFrom INFO [2019-12-17 12:30:49,264] jepsen nemesis - couchbase.util Rebalance stuck count: 149 INFO [2019-12-17 12:30:49,264] jepsen worker 88 - jepsen.util 4138 :invoke :read [29 nil] INFO [2019-12-17 12:30:49,290] jepsen worker 63 - jepsen.util 3393 :fail :read [21 nil] :RequestTimeoutException INFO [2019-12-17 12:30:49,290] jepsen worker 3 - jepsen.util 4143 :info :write [1 47] :RequestCanceledException INFO [2019-12-17 12:30:49,290] jepsen worker 63 - jepsen.util 3393 :invoke :write [21 29] INFO [2019-12-17 12:30:49,291] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":22644,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:30:49,291] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":22644,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:30:49,291] jepsen worker 3 - jepsen.util 4233 :invoke :write [1 39] INFO [2019-12-17 12:30:49,294] jepsen worker 37 - jepsen.util 3727 :fail :cas [12 [8 1]] :ValueNotSwapFrom INFO [2019-12-17 12:30:49,294] jepsen worker 37 - jepsen.util 3727 :invoke :write [12 22] INFO [2019-12-17 12:30:49,306] jepsen worker 36 - jepsen.util 4176 :fail :cas [12 [8 36]] :ValueNotSwapFrom INFO [2019-12-17 12:30:49,306] jepsen worker 36 - jepsen.util 4176 :invoke :cas [12 [41 46]] INFO [2019-12-17 12:30:49,314] jepsen worker 27 - jepsen.util 3897 :fail :cas [9 [16 15]] :ValueNotSwapFrom INFO [2019-12-17 12:30:49,315] jepsen worker 65 - jepsen.util 3305 :ok :write [21 4] INFO [2019-12-17 12:30:49,315] jepsen worker 27 - jepsen.util 3897 :invoke :read [9 nil] INFO [2019-12-17 12:30:49,315] jepsen worker 65 - jepsen.util 3305 :invoke :read [21 nil] INFO [2019-12-17 12:30:49,318] jepsen worker 45 - jepsen.util 4185 :ok :read [15 11] INFO [2019-12-17 12:30:49,318] jepsen worker 45 - jepsen.util 4185 :invoke :cas [15 [48 8]] INFO [2019-12-17 12:30:49,324] jepsen worker 39 - jepsen.util 4899 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:30:49,324] jepsen worker 46 - jepsen.util 4006 :fail :read [15 nil] :RequestTimeoutException INFO [2019-12-17 12:30:49,324] jepsen worker 62 - jepsen.util 4832 :info :cas [20 [43 25]] :RequestCanceledException INFO [2019-12-17 12:30:49,325] jepsen worker 39 - jepsen.util 4899 :invoke :cas [13 [22 49]] INFO [2019-12-17 12:30:49,325] jepsen worker 46 - jepsen.util 4006 :invoke :write [15 6] INFO [2019-12-17 12:30:49,326] jepsen worker 62 - jepsen.util 4922 :invoke :write [20 23] INFO [2019-12-17 12:30:49,358] jepsen worker 53 - jepsen.util 4013 :ok :read [17 8] INFO [2019-12-17 12:30:49,358] jepsen worker 53 - jepsen.util 4013 :invoke :cas [17 [11 11]] INFO [2019-12-17 12:30:49,363] jepsen worker 36 - jepsen.util 4176 :fail :cas [12 [41 46]] :ValueNotSwapFrom INFO [2019-12-17 12:30:49,363] jepsen worker 36 - jepsen.util 4176 :invoke :cas [12 [36 2]] INFO [2019-12-17 12:30:49,373] jepsen worker 72 - jepsen.util 4122 :fail :cas [24 [49 38]] :ValueNotSwapFrom INFO [2019-12-17 12:30:49,374] jepsen worker 72 - jepsen.util 4122 :invoke :write [24 30] INFO [2019-12-17 12:30:49,376] jepsen worker 88 - jepsen.util 4138 :ok :read [29 18] INFO [2019-12-17 12:30:49,376] jepsen worker 88 - jepsen.util 4138 :invoke :cas [29 [11 46]] INFO [2019-12-17 12:30:49,386] jepsen worker 32 - jepsen.util 4442 :fail :cas [10 [0 45]] :ValueNotSwapFrom INFO [2019-12-17 12:30:49,386] jepsen worker 32 - jepsen.util 4442 :invoke :read [10 nil] INFO [2019-12-17 12:30:49,391] jepsen worker 82 - jepsen.util 6112 :info :cas [27 [47 41]] :RequestCanceledException INFO [2019-12-17 12:30:49,392] jepsen worker 82 - jepsen.util 6202 :invoke :write [27 38] INFO [2019-12-17 12:30:49,409] jepsen worker 7 - jepsen.util 5137 :info :write [2 39] :RequestCanceledException INFO [2019-12-17 12:30:49,410] jepsen worker 11 - jepsen.util 4421 :ok :read [3 24] INFO [2019-12-17 12:30:49,410] jepsen worker 11 - jepsen.util 4421 :invoke :cas [3 [8 45]] INFO [2019-12-17 12:30:49,410] jepsen worker 7 - jepsen.util 5227 :invoke :read [2 nil] INFO [2019-12-17 12:30:49,424] jepsen worker 85 - jepsen.util 5755 :info :cas [28 [34 20]] :RequestCanceledException INFO [2019-12-17 12:30:49,424] jepsen worker 89 - jepsen.util 3689 :info :write [29 11] :RequestCanceledException INFO [2019-12-17 12:30:49,425] jepsen worker 85 - jepsen.util 5845 :invoke :write [28 24] INFO [2019-12-17 12:30:49,425] jepsen worker 89 - jepsen.util 3779 :invoke :write [29 31] INFO [2019-12-17 12:30:49,426] jepsen worker 47 - jepsen.util 4457 :ok :read [15 11] INFO [2019-12-17 12:30:49,426] jepsen worker 47 - jepsen.util 4457 :invoke :read [15 nil] INFO [2019-12-17 12:30:49,436] jepsen worker 63 - jepsen.util 3393 :ok :write [21 29] INFO [2019-12-17 12:30:49,436] jepsen worker 63 - jepsen.util 3393 :invoke :cas [21 [18 12]] INFO [2019-12-17 12:30:49,436] jepsen worker 10 - jepsen.util 3880 :ok :write [3 11] INFO [2019-12-17 12:30:49,436] jepsen worker 10 - jepsen.util 3880 :invoke :cas [3 [23 0]] INFO [2019-12-17 12:30:49,438] jepsen worker 27 - jepsen.util 3897 :ok :read [9 27] INFO [2019-12-17 12:30:49,439] jepsen worker 27 - jepsen.util 3897 :invoke :read [9 nil] INFO [2019-12-17 12:30:49,471] jepsen worker 32 - jepsen.util 4442 :ok :read [10 35] INFO [2019-12-17 12:30:49,471] jepsen worker 32 - jepsen.util 4442 :invoke :write [10 47] INFO [2019-12-17 12:30:49,487] jepsen worker 63 - jepsen.util 3393 :fail :cas [21 [18 12]] :ValueNotSwapFrom INFO [2019-12-17 12:30:49,488] jepsen worker 63 - jepsen.util 3393 :invoke :write [21 26] INFO [2019-12-17 12:30:49,491] jepsen worker 50 - jepsen.util 5630 :info :cas [16 [49 11]] :RequestCanceledException INFO [2019-12-17 12:30:49,492] jepsen worker 50 - jepsen.util 5720 :invoke :read [16 nil] INFO [2019-12-17 12:30:49,520] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":22684,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:30:49,520] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":22684,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:30:49,520] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":22687,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:30:49,520] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":22687,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:30:49,525] jepsen worker 30 - jepsen.util 4260 :info :cas [10 [37 26]] :RequestCanceledException INFO [2019-12-17 12:30:49,526] jepsen worker 30 - jepsen.util 4350 :invoke :read [10 nil] INFO [2019-12-17 12:30:49,528] jepsen worker 79 - jepsen.util 3949 :fail :cas [26 [19 13]] :ValueNotSwapFrom INFO [2019-12-17 12:30:49,529] jepsen worker 79 - jepsen.util 3949 :invoke :read [26 nil] INFO [2019-12-17 12:30:49,536] jepsen worker 11 - jepsen.util 4421 :fail :cas [3 [8 45]] :ValueNotSwapFrom INFO [2019-12-17 12:30:49,536] jepsen worker 11 - jepsen.util 4421 :invoke :read [3 nil] INFO [2019-12-17 12:30:49,538] jepsen worker 83 - jepsen.util 5303 :info :write [27 17] :RequestCanceledException INFO [2019-12-17 12:30:49,540] jepsen worker 83 - jepsen.util 5393 :invoke :write [27 27] INFO [2019-12-17 12:30:49,549] jepsen worker 87 - jepsen.util 3777 :ok :read [29 18] INFO [2019-12-17 12:30:49,550] jepsen worker 87 - jepsen.util 3777 :invoke :cas [29 [31 43]] INFO [2019-12-17 12:30:49,556] jepsen worker 52 - jepsen.util 4102 :info :cas [17 [16 35]] :RequestCanceledException INFO [2019-12-17 12:30:49,557] jepsen worker 52 - jepsen.util 4192 :invoke :cas [17 [37 40]] INFO [2019-12-17 12:30:49,559] jepsen worker 47 - jepsen.util 4457 :ok :read [15 11] INFO [2019-12-17 12:30:49,559] jepsen worker 47 - jepsen.util 4457 :invoke :cas [15 [14 13]] INFO [2019-12-17 12:30:49,575] jepsen worker 27 - jepsen.util 3897 :ok :read [9 27] INFO [2019-12-17 12:30:49,575] jepsen worker 27 - jepsen.util 3897 :invoke :cas [9 [39 9]] INFO [2019-12-17 12:30:49,587] jepsen worker 63 - jepsen.util 3393 :ok :write [21 26] INFO [2019-12-17 12:30:49,587] jepsen worker 63 - jepsen.util 3393 :invoke :write [21 22] INFO [2019-12-17 12:30:49,590] jepsen worker 0 - jepsen.util 5760 :info :write [0 14] :RequestCanceledException INFO [2019-12-17 12:30:49,592] jepsen worker 0 - jepsen.util 5850 :invoke :write [0 21] INFO [2019-12-17 12:30:49,599] jepsen worker 45 - jepsen.util 4185 :fail :cas [15 [48 8]] :ValueNotSwapFrom INFO [2019-12-17 12:30:49,600] jepsen worker 45 - jepsen.util 4185 :invoke :read [15 nil] INFO [2019-12-17 12:30:49,608] jepsen worker 48 - jepsen.util 4728 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:30:49,608] jepsen worker 48 - jepsen.util 4728 :invoke :write [16 23] INFO [2019-12-17 12:30:49,622] jepsen worker 40 - jepsen.util 5710 :info :write [13 15] :RequestCanceledException INFO [2019-12-17 12:30:49,622] jepsen worker 57 - jepsen.util 5007 :info :write [19 30] :RequestCanceledException INFO [2019-12-17 12:30:49,622] jepsen worker 60 - jepsen.util 5640 :info :write [20 30] :RequestCanceledException INFO [2019-12-17 12:30:49,622] jepsen worker 58 - jepsen.util 5278 :info :cas [19 [1 33]] :RequestCanceledException INFO [2019-12-17 12:30:49,622] jepsen worker 70 - jepsen.util 4480 :info :cas [23 [45 49]] :RequestCanceledException INFO [2019-12-17 12:30:49,622] jepsen worker 67 - jepsen.util 5467 :info :write [22 27] :RequestCanceledException INFO [2019-12-17 12:30:49,623] jepsen worker 60 - jepsen.util 5730 :invoke :write [20 38] INFO [2019-12-17 12:30:49,623] jepsen worker 40 - jepsen.util 5800 :invoke :cas [13 [48 1]] INFO [2019-12-17 12:30:49,623] jepsen worker 57 - jepsen.util 5097 :invoke :read [19 nil] INFO [2019-12-17 12:30:49,623] jepsen worker 58 - jepsen.util 5368 :invoke :cas [19 [13 42]] INFO [2019-12-17 12:30:49,623] jepsen worker 70 - jepsen.util 4570 :invoke :cas [23 [10 19]] INFO [2019-12-17 12:30:49,623] jepsen worker 67 - jepsen.util 5557 :invoke :cas [22 [2 21]] INFO [2019-12-17 12:30:49,638] jepsen worker 35 - jepsen.util 5165 :info :write [11 47] :RequestCanceledException INFO [2019-12-17 12:30:49,639] jepsen worker 35 - jepsen.util 5255 :invoke :cas [11 [49 10]] INFO [2019-12-17 12:30:49,640] jepsen worker 63 - jepsen.util 3393 :ok :write [21 22] INFO [2019-12-17 12:30:49,640] jepsen worker 63 - jepsen.util 3393 :invoke :read [21 nil] INFO [2019-12-17 12:30:49,653] jepsen worker 20 - jepsen.util 3890 :ok :write [6 9] INFO [2019-12-17 12:30:49,653] jepsen worker 20 - jepsen.util 3890 :invoke :cas [6 [15 4]] INFO [2019-12-17 12:30:49,655] jepsen worker 63 - jepsen.util 3393 :ok :read [21 22] INFO [2019-12-17 12:30:49,656] jepsen worker 26 - jepsen.util 5426 :info :write [8 2] :RequestCanceledException INFO [2019-12-17 12:30:49,656] jepsen worker 63 - jepsen.util 3393 :invoke :read [21 nil] INFO [2019-12-17 12:30:49,656] jepsen worker 36 - jepsen.util 4176 :fail :cas [12 [36 2]] :ValueNotSwapFrom INFO [2019-12-17 12:30:49,656] jepsen worker 36 - jepsen.util 4176 :invoke :cas [12 [27 16]] INFO [2019-12-17 12:30:49,656] jepsen worker 26 - jepsen.util 5516 :invoke :read [8 nil] INFO [2019-12-17 12:30:49,677] jepsen worker 63 - jepsen.util 3393 :ok :read [21 22] INFO [2019-12-17 12:30:49,677] jepsen worker 63 - jepsen.util 3393 :invoke :cas [21 [9 8]] INFO [2019-12-17 12:30:49,681] jepsen worker 30 - jepsen.util 4350 :ok :read [10 35] INFO [2019-12-17 12:30:49,681] jepsen worker 30 - jepsen.util 4350 :invoke :cas [10 [22 45]] INFO [2019-12-17 12:30:49,684] jepsen worker 20 - jepsen.util 3890 :fail :cas [6 [15 4]] :ValueNotSwapFrom INFO [2019-12-17 12:30:49,684] jepsen worker 20 - jepsen.util 3890 :invoke :cas [6 [38 20]] INFO [2019-12-17 12:30:49,687] jepsen worker 63 - jepsen.util 3393 :fail :cas [21 [9 8]] :ValueNotSwapFrom INFO [2019-12-17 12:30:49,688] jepsen worker 63 - jepsen.util 3393 :invoke :cas [21 [32 21]] INFO [2019-12-17 12:30:49,690] jepsen worker 74 - jepsen.util 4394 :info :write [24 1] :RequestCanceledException INFO [2019-12-17 12:30:49,690] jepsen worker 5 - jepsen.util 3515 :info :write [1 19] :RequestCanceledException INFO [2019-12-17 12:30:49,690] jepsen worker 9 - jepsen.util 4419 :fail :read [3 nil] :RequestTimeoutException INFO [2019-12-17 12:30:49,690] jepsen worker 9 - jepsen.util 4419 :invoke :cas [3 [26 0]] INFO [2019-12-17 12:30:49,691] jepsen worker 5 - jepsen.util 3605 :invoke :write [1 33] INFO [2019-12-17 12:30:49,691] jepsen worker 74 - jepsen.util 4484 :invoke :read [24 nil] INFO [2019-12-17 12:30:49,706] jepsen worker 30 - jepsen.util 4350 :fail :cas [10 [22 45]] :ValueNotSwapFrom INFO [2019-12-17 12:30:49,706] jepsen worker 30 - jepsen.util 4350 :invoke :cas [10 [36 43]] INFO [2019-12-17 12:30:49,709] jepsen worker 55 - jepsen.util 4825 :info :write [18 37] :RequestCanceledException INFO [2019-12-17 12:30:49,710] jepsen worker 55 - jepsen.util 4915 :invoke :write [18 7] INFO [2019-12-17 12:30:49,711] jepsen worker 63 - jepsen.util 3393 :fail :cas [21 [32 21]] :ValueNotSwapFrom INFO [2019-12-17 12:30:49,711] jepsen worker 63 - jepsen.util 3393 :invoke :read [21 nil] INFO [2019-12-17 12:30:49,718] jepsen worker 20 - jepsen.util 3890 :fail :cas [6 [38 20]] :ValueNotSwapFrom INFO [2019-12-17 12:30:49,718] jepsen worker 20 - jepsen.util 3890 :invoke :read [6 nil] INFO [2019-12-17 12:30:49,724] jepsen worker 17 - jepsen.util 4967 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:30:49,724] jepsen worker 22 - jepsen.util 5242 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:30:49,724] jepsen worker 2 - jepsen.util 4862 :info :cas [0 [32 10]] :RequestCanceledException INFO [2019-12-17 12:30:49,724] jepsen worker 33 - jepsen.util 5523 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:30:49,724] jepsen worker 17 - jepsen.util 4967 :invoke :cas [5 [27 28]] INFO [2019-12-17 12:30:49,724] jepsen worker 22 - jepsen.util 5242 :invoke :write [7 16] INFO [2019-12-17 12:30:49,724] jepsen worker 33 - jepsen.util 5523 :invoke :read [11 nil] INFO [2019-12-17 12:30:49,726] jepsen worker 2 - jepsen.util 4952 :invoke :cas [0 [41 44]] INFO [2019-12-17 12:30:49,727] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":22718,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:30:49,727] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":22718,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:30:49,727] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":22721,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:30:49,727] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":22721,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:30:49,729] jepsen worker 9 - jepsen.util 4419 :fail :cas [3 [26 0]] :ValueNotSwapFrom INFO [2019-12-17 12:30:49,729] jepsen worker 9 - jepsen.util 4419 :invoke :write [3 10] INFO [2019-12-17 12:30:49,731] jepsen worker 36 - jepsen.util 4176 :fail :cas [12 [27 16]] :ValueNotSwapFrom INFO [2019-12-17 12:30:49,731] jepsen worker 36 - jepsen.util 4176 :invoke :cas [12 [4 25]] INFO [2019-12-17 12:30:49,737] jepsen worker 30 - jepsen.util 4350 :fail :cas [10 [36 43]] :ValueNotSwapFrom INFO [2019-12-17 12:30:49,737] jepsen worker 30 - jepsen.util 4350 :invoke :read [10 nil] INFO [2019-12-17 12:30:49,743] jepsen worker 88 - jepsen.util 4138 :fail :cas [29 [11 46]] :ValueNotSwapFrom INFO [2019-12-17 12:30:49,743] jepsen worker 88 - jepsen.util 4138 :invoke :cas [29 [0 12]] INFO [2019-12-17 12:30:49,743] jepsen worker 52 - jepsen.util 4192 :fail :cas [17 [37 40]] :ValueNotSwapFrom INFO [2019-12-17 12:30:49,743] jepsen worker 52 - jepsen.util 4192 :invoke :write [17 23] INFO [2019-12-17 12:30:49,756] jepsen worker 70 - jepsen.util 4570 :fail :cas [23 [10 19]] :ValueNotSwapFrom INFO [2019-12-17 12:30:49,757] jepsen worker 70 - jepsen.util 4570 :invoke :write [23 23] INFO [2019-12-17 12:30:49,788] jepsen worker 63 - jepsen.util 3393 :ok :read [21 22] INFO [2019-12-17 12:30:49,788] jepsen worker 63 - jepsen.util 3393 :invoke :write [21 19] INFO [2019-12-17 12:30:49,789] jepsen worker 20 - jepsen.util 3890 :ok :read [6 9] INFO [2019-12-17 12:30:49,789] jepsen worker 20 - jepsen.util 3890 :invoke :cas [6 [2 14]] INFO [2019-12-17 12:30:49,793] jepsen worker 87 - jepsen.util 3777 :fail :cas [29 [31 43]] :ValueNotSwapFrom INFO [2019-12-17 12:30:49,793] jepsen worker 87 - jepsen.util 3777 :invoke :cas [29 [2 22]] INFO [2019-12-17 12:30:49,812] jepsen worker 32 - jepsen.util 4442 :ok :write [10 47] INFO [2019-12-17 12:30:49,812] jepsen worker 30 - jepsen.util 4350 :ok :read [10 35] INFO [2019-12-17 12:30:49,812] jepsen worker 32 - jepsen.util 4442 :invoke :read [10 nil] INFO [2019-12-17 12:30:49,812] jepsen worker 30 - jepsen.util 4350 :invoke :read [10 nil] INFO [2019-12-17 12:30:49,813] jepsen worker 20 - jepsen.util 3890 :fail :cas [6 [2 14]] :ValueNotSwapFrom INFO [2019-12-17 12:30:49,813] jepsen worker 20 - jepsen.util 3890 :invoke :write [6 4] INFO [2019-12-17 12:30:49,824] jepsen worker 41 - jepsen.util 5441 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:30:49,824] jepsen worker 51 - jepsen.util 3561 :info :write [17 33] :RequestCanceledException INFO [2019-12-17 12:30:49,824] jepsen worker 42 - jepsen.util 5982 :info :cas [14 [42 30]] :RequestCanceledException INFO [2019-12-17 12:30:49,825] jepsen worker 41 - jepsen.util 5441 :invoke :cas [13 [33 1]] INFO [2019-12-17 12:30:49,826] jepsen worker 42 - jepsen.util 6072 :invoke :write [14 48] INFO [2019-12-17 12:30:49,826] jepsen worker 51 - jepsen.util 3651 :invoke :write [17 6] INFO [2019-12-17 12:30:49,856] jepsen worker 75 - jepsen.util 5295 :info :cas [25 [22 37]] :RequestCanceledException INFO [2019-12-17 12:30:49,856] jepsen worker 13 - jepsen.util 3703 :info :write [4 21] :RequestCanceledException INFO [2019-12-17 12:30:49,857] jepsen worker 75 - jepsen.util 5385 :invoke :cas [25 [16 38]] INFO [2019-12-17 12:30:49,858] jepsen worker 13 - jepsen.util 3793 :invoke :read [4 nil] INFO [2019-12-17 12:30:49,875] jepsen worker 47 - jepsen.util 4457 :fail :cas [15 [14 13]] :ValueNotSwapFrom INFO [2019-12-17 12:30:49,876] jepsen worker 47 - jepsen.util 4457 :invoke :cas [15 [9 24]] INFO [2019-12-17 12:30:49,885] jepsen worker 4 - jepsen.util 3244 :ok :write [1 33] INFO [2019-12-17 12:30:49,886] jepsen worker 4 - jepsen.util 3244 :invoke :read [1 nil] INFO [2019-12-17 12:30:49,890] jepsen worker 73 - jepsen.util 3853 :ok :write [24 39] INFO [2019-12-17 12:30:49,890] jepsen worker 73 - jepsen.util 3853 :invoke :read [24 nil] INFO [2019-12-17 12:30:49,891] jepsen worker 27 - jepsen.util 3897 :fail :cas [9 [39 9]] :ValueNotSwapFrom INFO [2019-12-17 12:30:49,892] jepsen worker 27 - jepsen.util 3897 :invoke :read [9 nil] INFO [2019-12-17 12:30:49,899] jepsen worker 9 - jepsen.util 4419 :ok :write [3 10] INFO [2019-12-17 12:30:49,899] jepsen worker 30 - jepsen.util 4350 :ok :read [10 47] INFO [2019-12-17 12:30:49,899] jepsen worker 30 - jepsen.util 4350 :invoke :read [10 nil] INFO [2019-12-17 12:30:49,899] jepsen worker 9 - jepsen.util 4419 :invoke :write [3 0] INFO [2019-12-17 12:30:49,910] jepsen worker 1 - jepsen.util 4951 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:30:49,910] jepsen worker 1 - jepsen.util 4951 :invoke :write [0 23] INFO [2019-12-17 12:30:49,917] jepsen worker 74 - jepsen.util 4484 :ok :read [24 39] INFO [2019-12-17 12:30:49,917] jepsen worker 74 - jepsen.util 4484 :invoke :cas [24 [28 1]] INFO [2019-12-17 12:30:49,922] jepsen worker 36 - jepsen.util 4176 :fail :cas [12 [4 25]] :ValueNotSwapFrom INFO [2019-12-17 12:30:49,922] jepsen worker 36 - jepsen.util 4176 :invoke :write [12 13] INFO [2019-12-17 12:30:49,924] jepsen worker 16 - jepsen.util 5956 :info :cas [5 [42 18]] :RequestCanceledException INFO [2019-12-17 12:30:49,926] jepsen worker 16 - jepsen.util 6046 :invoke :write [5 3] INFO [2019-12-17 12:30:49,932] jepsen worker 13 - jepsen.util 3793 :ok :read [4 49] INFO [2019-12-17 12:30:49,932] jepsen worker 19 - jepsen.util 4069 :fail :read [6 nil] :RequestTimeoutException INFO [2019-12-17 12:30:49,932] jepsen worker 13 - jepsen.util 3793 :invoke :read [4 nil] INFO [2019-12-17 12:30:49,932] jepsen worker 19 - jepsen.util 4069 :invoke :read [6 nil] INFO [2019-12-17 12:30:49,946] jepsen worker 87 - jepsen.util 3777 :fail :cas [29 [2 22]] :ValueNotSwapFrom INFO [2019-12-17 12:30:49,946] jepsen worker 87 - jepsen.util 3777 :invoke :cas [29 [42 3]] INFO [2019-12-17 12:30:49,946] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":22760,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:30:49,947] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":22760,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:30:49,947] jepsen worker 73 - jepsen.util 3853 :ok :read [24 39] INFO [2019-12-17 12:30:49,947] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":22764,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:30:49,947] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":22764,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:30:49,947] jepsen worker 73 - jepsen.util 3853 :invoke :cas [24 [21 3]] INFO [2019-12-17 12:30:49,954] jepsen worker 76 - jepsen.util 5386 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:30:49,954] jepsen worker 76 - jepsen.util 5386 :invoke :cas [25 [22 0]] INFO [2019-12-17 12:30:49,958] jepsen worker 88 - jepsen.util 4138 :fail :cas [29 [0 12]] :ValueNotSwapFrom INFO [2019-12-17 12:30:49,958] jepsen worker 88 - jepsen.util 4138 :invoke :write [29 11] INFO [2019-12-17 12:30:49,967] jepsen worker 32 - jepsen.util 4442 :ok :read [10 47] INFO [2019-12-17 12:30:49,968] jepsen worker 32 - jepsen.util 4442 :invoke :read [10 nil] INFO [2019-12-17 12:30:49,989] jepsen worker 54 - jepsen.util 4374 :ok :write [18 21] INFO [2019-12-17 12:30:49,989] jepsen worker 54 - jepsen.util 4374 :invoke :cas [18 [31 44]] INFO [2019-12-17 12:30:50,015] jepsen worker 19 - jepsen.util 4069 :ok :read [6 9] INFO [2019-12-17 12:30:50,015] jepsen worker 19 - jepsen.util 4069 :invoke :read [6 nil] INFO [2019-12-17 12:30:50,024] jepsen worker 47 - jepsen.util 4457 :fail :cas [15 [9 24]] :ValueNotSwapFrom INFO [2019-12-17 12:30:50,024] jepsen worker 47 - jepsen.util 4457 :invoke :cas [15 [21 45]] INFO [2019-12-17 12:30:50,025] jepsen worker 15 - jepsen.util 4695 :info :cas [5 [40 47]] :RequestCanceledException INFO [2019-12-17 12:30:50,025] jepsen worker 8 - jepsen.util 4418 :info :write [2 1] :RequestCanceledException INFO [2019-12-17 12:30:50,026] jepsen worker 15 - jepsen.util 4785 :invoke :read [5 nil] INFO [2019-12-17 12:30:50,026] jepsen worker 8 - jepsen.util 4508 :invoke :write [2 48] INFO [2019-12-17 12:30:50,033] jepsen worker 71 - jepsen.util 4211 :fail :cas [23 [47 24]] :ValueNotSwapFrom INFO [2019-12-17 12:30:50,033] jepsen worker 71 - jepsen.util 4211 :invoke :cas [23 [24 44]] INFO [2019-12-17 12:30:50,048] jepsen worker 73 - jepsen.util 3853 :fail :cas [24 [21 3]] :ValueNotSwapFrom INFO [2019-12-17 12:30:50,048] jepsen worker 73 - jepsen.util 3853 :invoke :write [24 21] INFO [2019-12-17 12:30:50,062] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":22786,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:30:50,062] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":22786,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:30:50,062] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":22788,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:30:50,062] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":22788,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:30:50,075] jepsen worker 71 - jepsen.util 4211 :fail :cas [23 [24 44]] :ValueNotSwapFrom INFO [2019-12-17 12:30:50,075] jepsen worker 71 - jepsen.util 4211 :invoke :read [23 nil] INFO [2019-12-17 12:30:50,094] jepsen worker 27 - jepsen.util 3897 :ok :read [9 27] INFO [2019-12-17 12:30:50,094] jepsen worker 27 - jepsen.util 3897 :invoke :write [9 7] INFO [2019-12-17 12:30:50,107] jepsen worker 69 - jepsen.util 3939 :info :write [23 1] :RequestCanceledException INFO [2019-12-17 12:30:50,107] jepsen worker 31 - jepsen.util 3541 :info :write [10 22] :RequestCanceledException INFO [2019-12-17 12:30:50,108] jepsen worker 69 - jepsen.util 4029 :invoke :write [23 13] INFO [2019-12-17 12:30:50,108] jepsen worker 31 - jepsen.util 3631 :invoke :write [10 28] INFO [2019-12-17 12:30:50,116] jepsen worker 19 - jepsen.util 4069 :ok :read [6 9] INFO [2019-12-17 12:30:50,116] jepsen worker 19 - jepsen.util 4069 :invoke :cas [6 [27 5]] INFO [2019-12-17 12:30:50,125] jepsen worker 81 - jepsen.util 5211 :info :cas [27 [7 23]] :RequestCanceledException INFO [2019-12-17 12:30:50,126] jepsen worker 81 - jepsen.util 5301 :invoke :read [27 nil] INFO [2019-12-17 12:30:50,127] jepsen worker 63 - jepsen.util 3393 :ok :write [21 19] INFO [2019-12-17 12:30:50,127] jepsen worker 63 - jepsen.util 3393 :invoke :write [21 2] INFO [2019-12-17 12:30:50,136] jepsen worker 74 - jepsen.util 4484 :fail :cas [24 [28 1]] :ValueNotSwapFrom INFO [2019-12-17 12:30:50,136] jepsen worker 80 - jepsen.util 4040 :info :write [26 41] :RequestCanceledException INFO [2019-12-17 12:30:50,136] jepsen worker 74 - jepsen.util 4484 :invoke :cas [24 [11 25]] INFO [2019-12-17 12:30:50,137] jepsen worker 80 - jepsen.util 4130 :invoke :write [26 30] INFO [2019-12-17 12:30:50,149] jepsen worker 4 - jepsen.util 3244 :ok :read [1 33] INFO [2019-12-17 12:30:50,150] jepsen worker 4 - jepsen.util 3244 :invoke :read [1 nil] INFO [2019-12-17 12:30:50,178] jepsen worker 32 - jepsen.util 4442 :ok :read [10 47] INFO [2019-12-17 12:30:50,178] jepsen worker 32 - jepsen.util 4442 :invoke :cas [10 [3 20]] INFO [2019-12-17 12:30:50,185] jepsen worker 79 - jepsen.util 3949 :ok :read [26 28] INFO [2019-12-17 12:30:50,186] jepsen worker 79 - jepsen.util 3949 :invoke :cas [26 [16 29]] INFO [2019-12-17 12:30:50,208] jepsen worker 47 - jepsen.util 4457 :fail :cas [15 [21 45]] :ValueNotSwapFrom INFO [2019-12-17 12:30:50,208] jepsen worker 47 - jepsen.util 4457 :invoke :write [15 14] INFO [2019-12-17 12:30:50,214] jepsen worker 20 - jepsen.util 3890 :ok :write [6 4] INFO [2019-12-17 12:30:50,214] jepsen worker 20 - jepsen.util 3890 :invoke :write [6 48] INFO [2019-12-17 12:30:50,222] jepsen worker 87 - jepsen.util 3777 :fail :cas [29 [42 3]] :ValueNotSwapFrom INFO [2019-12-17 12:30:50,222] jepsen worker 87 - jepsen.util 3777 :invoke :cas [29 [12 3]] INFO [2019-12-17 12:30:50,235] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":22814,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:30:50,235] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":22814,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:30:50,235] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":22817,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:30:50,235] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":22817,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:30:50,235] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":22818,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:30:50,235] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":22818,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:30:50,236] jepsen worker 21 - jepsen.util 5241 :info :write [7 37] :RequestCanceledException INFO [2019-12-17 12:30:50,237] jepsen worker 21 - jepsen.util 5331 :invoke :cas [7 [26 40]] INFO [2019-12-17 12:30:50,243] jepsen worker 63 - jepsen.util 3393 :ok :write [21 2] INFO [2019-12-17 12:30:50,244] jepsen worker 63 - jepsen.util 3393 :invoke :read [21 nil] INFO [2019-12-17 12:30:50,248] jepsen worker 30 - jepsen.util 4350 :ok :read [10 47] INFO [2019-12-17 12:30:50,248] jepsen worker 30 - jepsen.util 4350 :invoke :write [10 26] INFO [2019-12-17 12:30:50,254] jepsen worker 27 - jepsen.util 3897 :ok :write [9 7] INFO [2019-12-17 12:30:50,254] jepsen worker 27 - jepsen.util 3897 :invoke :read [9 nil] INFO [2019-12-17 12:30:50,274] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":22807,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:30:50,274] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":22807,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:30:50,274] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":22808,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:30:50,274] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":22808,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:30:50,274] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":22809,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:30:50,274] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":22809,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:30:50,277] jepsen worker 73 - jepsen.util 3853 :ok :write [24 21] INFO [2019-12-17 12:30:50,277] jepsen worker 73 - jepsen.util 3853 :invoke :write [24 9] INFO [2019-12-17 12:30:50,279] jepsen worker 46 - jepsen.util 4006 :ok :write [15 6] INFO [2019-12-17 12:30:50,279] jepsen worker 46 - jepsen.util 4006 :invoke :write [15 2] INFO [2019-12-17 12:30:50,282] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:30:50,282] jepsen nemesis - couchbase.util Rebalance check count: 169 INFO [2019-12-17 12:30:50,282] jepsen nemesis - couchbase.util Rebalance stuck count: 150 INFO [2019-12-17 12:30:50,292] jepsen worker 63 - jepsen.util 3393 :ok :read [21 2] INFO [2019-12-17 12:30:50,292] jepsen worker 63 - jepsen.util 3393 :invoke :cas [21 [13 43]] INFO [2019-12-17 12:30:50,305] jepsen worker 32 - jepsen.util 4442 :fail :cas [10 [3 20]] :ValueNotSwapFrom INFO [2019-12-17 12:30:50,305] jepsen worker 32 - jepsen.util 4442 :invoke :cas [10 [46 5]] INFO [2019-12-17 12:30:50,324] jepsen worker 61 - jepsen.util 6091 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:30:50,324] jepsen worker 61 - jepsen.util 6091 :invoke :read [20 nil] INFO [2019-12-17 12:30:50,348] jepsen worker 63 - jepsen.util 3393 :fail :cas [21 [13 43]] :ValueNotSwapFrom INFO [2019-12-17 12:30:50,348] jepsen worker 63 - jepsen.util 3393 :invoke :write [21 38] INFO [2019-12-17 12:30:50,389] jepsen worker 63 - jepsen.util 3393 :ok :write [21 38] INFO [2019-12-17 12:30:50,389] jepsen worker 63 - jepsen.util 3393 :invoke :read [21 nil] INFO [2019-12-17 12:30:50,413] jepsen worker 63 - jepsen.util 3393 :ok :read [21 38] INFO [2019-12-17 12:30:50,416] jepsen worker 63 - jepsen.util 3393 :invoke :cas [21 [33 19]] INFO [2019-12-17 12:30:50,429] jepsen worker 53 - jepsen.util 4013 :fail :cas [17 [11 11]] :ValueNotSwapFrom INFO [2019-12-17 12:30:50,429] jepsen worker 53 - jepsen.util 4013 :invoke :read [17 nil] INFO [2019-12-17 12:30:50,431] jepsen worker 31 - jepsen.util 3631 :ok :write [10 28] INFO [2019-12-17 12:30:50,432] jepsen worker 31 - jepsen.util 3631 :invoke :cas [10 [20 48]] INFO [2019-12-17 12:30:50,432] jepsen worker 13 - jepsen.util 3793 :ok :read [4 49] INFO [2019-12-17 12:30:50,432] jepsen worker 13 - jepsen.util 3793 :invoke :cas [4 [20 0]] INFO [2019-12-17 12:30:50,433] jepsen worker 47 - jepsen.util 4457 :ok :write [15 14] INFO [2019-12-17 12:30:50,433] jepsen worker 47 - jepsen.util 4457 :invoke :write [15 10] INFO [2019-12-17 12:30:50,434] jepsen worker 63 - jepsen.util 3393 :fail :cas [21 [33 19]] :ValueNotSwapFrom INFO [2019-12-17 12:30:50,434] jepsen worker 63 - jepsen.util 3393 :invoke :write [21 45] INFO [2019-12-17 12:30:50,470] jepsen worker 32 - jepsen.util 4442 :fail :cas [10 [46 5]] :ValueNotSwapFrom INFO [2019-12-17 12:30:50,470] jepsen worker 32 - jepsen.util 4442 :invoke :cas [10 [2 16]] INFO [2019-12-17 12:30:50,487] jepsen worker 73 - jepsen.util 3853 :ok :write [24 9] INFO [2019-12-17 12:30:50,488] jepsen worker 73 - jepsen.util 3853 :invoke :read [24 nil] INFO [2019-12-17 12:30:50,503] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":22835,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:30:50,503] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":22835,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:30:50,503] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":22838,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:30:50,504] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":22838,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:30:50,504] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":22839,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:30:50,504] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":22839,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:30:50,506] jepsen worker 71 - jepsen.util 4211 :ok :read [23 45] INFO [2019-12-17 12:30:50,506] jepsen worker 71 - jepsen.util 4211 :invoke :write [23 8] INFO [2019-12-17 12:30:50,515] jepsen worker 63 - jepsen.util 3393 :ok :write [21 45] INFO [2019-12-17 12:30:50,515] jepsen worker 73 - jepsen.util 3853 :ok :read [24 9] INFO [2019-12-17 12:30:50,515] jepsen worker 63 - jepsen.util 3393 :invoke :cas [21 [8 19]] INFO [2019-12-17 12:30:50,515] jepsen worker 73 - jepsen.util 3853 :invoke :read [24 nil] INFO [2019-12-17 12:30:50,541] jepsen worker 73 - jepsen.util 3853 :ok :read [24 9] INFO [2019-12-17 12:30:50,542] jepsen worker 63 - jepsen.util 3393 :fail :cas [21 [8 19]] :ValueNotSwapFrom INFO [2019-12-17 12:30:50,543] jepsen worker 63 - jepsen.util 3393 :invoke :read [21 nil] INFO [2019-12-17 12:30:50,543] jepsen worker 73 - jepsen.util 3853 :invoke :write [24 31] INFO [2019-12-17 12:30:50,544] jepsen worker 54 - jepsen.util 4374 :fail :cas [18 [31 44]] :ValueNotSwapFrom INFO [2019-12-17 12:30:50,544] jepsen worker 54 - jepsen.util 4374 :invoke :read [18 nil] INFO [2019-12-17 12:30:50,558] jepsen worker 63 - jepsen.util 3393 :ok :read [21 45] INFO [2019-12-17 12:30:50,558] jepsen worker 63 - jepsen.util 3393 :invoke :read [21 nil] INFO [2019-12-17 12:30:50,571] jepsen worker 63 - jepsen.util 3393 :ok :read [21 45] INFO [2019-12-17 12:30:50,571] jepsen worker 63 - jepsen.util 3393 :invoke :read [21 nil] INFO [2019-12-17 12:30:50,581] jepsen worker 79 - jepsen.util 3949 :fail :cas [26 [16 29]] :ValueNotSwapFrom INFO [2019-12-17 12:30:50,581] jepsen worker 79 - jepsen.util 3949 :invoke :write [26 48] INFO [2019-12-17 12:30:50,591] jepsen worker 86 - jepsen.util 5576 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:30:50,591] jepsen worker 86 - jepsen.util 5576 :invoke :read [28 nil] INFO [2019-12-17 12:30:50,592] jepsen worker 53 - jepsen.util 4013 :ok :read [17 8] INFO [2019-12-17 12:30:50,592] jepsen worker 53 - jepsen.util 4013 :invoke :cas [17 [2 33]] INFO [2019-12-17 12:30:50,600] jepsen worker 63 - jepsen.util 3393 :ok :read [21 45] INFO [2019-12-17 12:30:50,601] jepsen worker 63 - jepsen.util 3393 :invoke :read [21 nil] INFO [2019-12-17 12:30:50,608] jepsen worker 77 - jepsen.util 5297 :info :cas [25 [44 45]] :RequestCanceledException INFO [2019-12-17 12:30:50,609] jepsen worker 77 - jepsen.util 5387 :invoke :read [25 nil] INFO [2019-12-17 12:30:50,624] jepsen worker 63 - jepsen.util 3393 :ok :read [21 45] INFO [2019-12-17 12:30:50,624] jepsen worker 63 - jepsen.util 3393 :invoke :read [21 nil] INFO [2019-12-17 12:30:50,636] jepsen worker 63 - jepsen.util 3393 :ok :read [21 45] INFO [2019-12-17 12:30:50,636] jepsen worker 63 - jepsen.util 3393 :invoke :cas [21 [28 49]] INFO [2019-12-17 12:30:50,655] jepsen worker 14 - jepsen.util 3974 :info :write [4 13] :RequestCanceledException INFO [2019-12-17 12:30:50,655] jepsen worker 24 - jepsen.util 5064 :info :write [8 7] :RequestCanceledException INFO [2019-12-17 12:30:50,655] jepsen worker 78 - jepsen.util 4398 :info :write [26 4] :RequestCanceledException INFO [2019-12-17 12:30:50,656] jepsen worker 24 - jepsen.util 5154 :invoke :write [8 34] INFO [2019-12-17 12:30:50,656] jepsen worker 14 - jepsen.util 4064 :invoke :cas [4 [14 46]] INFO [2019-12-17 12:30:50,658] jepsen worker 78 - jepsen.util 4488 :invoke :write [26 25] INFO [2019-12-17 12:30:50,667] jepsen worker 63 - jepsen.util 3393 :fail :cas [21 [28 49]] :ValueNotSwapFrom INFO [2019-12-17 12:30:50,667] jepsen worker 63 - jepsen.util 3393 :invoke :read [21 nil] INFO [2019-12-17 12:30:50,680] jepsen worker 63 - jepsen.util 3393 :ok :read [21 45] INFO [2019-12-17 12:30:50,680] jepsen worker 63 - jepsen.util 3393 :invoke :write [21 20] INFO [2019-12-17 12:30:50,691] jepsen worker 56 - jepsen.util 4106 :info :write [18 22] :RequestCanceledException INFO [2019-12-17 12:30:50,692] jepsen worker 56 - jepsen.util 4196 :invoke :write [18 27] INFO [2019-12-17 12:30:50,712] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":22863,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:30:50,712] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":22863,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:30:50,712] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":22867,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:30:50,712] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":22867,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:30:50,718] jepsen worker 63 - jepsen.util 3393 :ok :write [21 20] INFO [2019-12-17 12:30:50,718] jepsen worker 63 - jepsen.util 3393 :invoke :cas [21 [6 4]] INFO [2019-12-17 12:30:50,721] jepsen worker 13 - jepsen.util 3793 :fail :cas [4 [20 0]] :ValueNotSwapFrom INFO [2019-12-17 12:30:50,721] jepsen worker 13 - jepsen.util 3793 :invoke :read [4 nil] INFO [2019-12-17 12:30:50,725] jepsen worker 49 - jepsen.util 4639 :info :cas [16 [20 17]] :RequestCanceledException INFO [2019-12-17 12:30:50,726] jepsen worker 49 - jepsen.util 4729 :invoke :write [16 9] INFO [2019-12-17 12:30:50,738] jepsen worker 63 - jepsen.util 3393 :fail :cas [21 [6 4]] :ValueNotSwapFrom INFO [2019-12-17 12:30:50,738] jepsen worker 63 - jepsen.util 3393 :invoke :write [21 42] INFO [2019-12-17 12:30:50,777] jepsen worker 13 - jepsen.util 3793 :ok :read [4 49] INFO [2019-12-17 12:30:50,778] jepsen worker 13 - jepsen.util 3793 :invoke :cas [4 [16 1]] INFO [2019-12-17 12:30:50,784] jepsen worker 87 - jepsen.util 3777 :fail :cas [29 [12 3]] :ValueNotSwapFrom INFO [2019-12-17 12:30:50,784] jepsen worker 87 - jepsen.util 3777 :invoke :cas [29 [23 11]] INFO [2019-12-17 12:30:50,787] jepsen worker 28 - jepsen.util 2818 :info :cas [9 [5 37]] :RequestCanceledException INFO [2019-12-17 12:30:50,787] jepsen worker 63 - jepsen.util 3393 :ok :write [21 42] INFO [2019-12-17 12:30:50,788] jepsen worker 63 - jepsen.util 3393 :invoke :read [21 nil] INFO [2019-12-17 12:30:50,790] jepsen worker 28 - jepsen.util 2908 :invoke :cas [9 [12 15]] INFO [2019-12-17 12:30:50,809] jepsen worker 34 - jepsen.util 5074 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:30:50,809] jepsen worker 29 - jepsen.util 3269 :info :write [9 12] :RequestCanceledException INFO [2019-12-17 12:30:50,809] jepsen worker 63 - jepsen.util 3393 :ok :read [21 42] INFO [2019-12-17 12:30:50,809] jepsen worker 34 - jepsen.util 5074 :invoke :read [11 nil] INFO [2019-12-17 12:30:50,809] jepsen worker 63 - jepsen.util 3393 :invoke :write [21 25] INFO [2019-12-17 12:30:50,811] jepsen worker 29 - jepsen.util 3359 :invoke :cas [9 [42 48]] INFO [2019-12-17 12:30:50,816] jepsen worker 53 - jepsen.util 4013 :fail :cas [17 [2 33]] :ValueNotSwapFrom INFO [2019-12-17 12:30:50,816] jepsen worker 53 - jepsen.util 4013 :invoke :write [17 20] INFO [2019-12-17 12:30:50,820] jepsen worker 28 - jepsen.util 2908 :fail :cas [9 [12 15]] :ValueNotSwapFrom INFO [2019-12-17 12:30:50,820] jepsen worker 28 - jepsen.util 2908 :invoke :read [9 nil] INFO [2019-12-17 12:30:50,824] jepsen worker 12 - jepsen.util 4332 :info :write [4 30] :RequestCanceledException INFO [2019-12-17 12:30:50,826] jepsen worker 12 - jepsen.util 4422 :invoke :read [4 nil] INFO [2019-12-17 12:30:50,831] jepsen worker 29 - jepsen.util 3359 :fail :cas [9 [42 48]] :ValueNotSwapFrom INFO [2019-12-17 12:30:50,831] jepsen worker 29 - jepsen.util 3359 :invoke :read [9 nil] INFO [2019-12-17 12:30:50,848] jepsen worker 28 - jepsen.util 2908 :ok :read [9 7] INFO [2019-12-17 12:30:50,848] jepsen worker 28 - jepsen.util 2908 :invoke :write [9 13] INFO [2019-12-17 12:30:50,858] jepsen worker 87 - jepsen.util 3777 :fail :cas [29 [23 11]] :ValueNotSwapFrom INFO [2019-12-17 12:30:50,858] jepsen worker 87 - jepsen.util 3777 :invoke :cas [29 [10 25]] INFO [2019-12-17 12:30:50,881] jepsen worker 63 - jepsen.util 3393 :ok :write [21 25] INFO [2019-12-17 12:30:50,881] jepsen worker 63 - jepsen.util 3393 :invoke :write [21 23] INFO [2019-12-17 12:30:50,911] jepsen worker 29 - jepsen.util 3359 :ok :read [9 7] INFO [2019-12-17 12:30:50,911] jepsen worker 29 - jepsen.util 3359 :invoke :read [9 nil] INFO [2019-12-17 12:30:50,918] jepsen worker 28 - jepsen.util 2908 :ok :write [9 13] INFO [2019-12-17 12:30:50,918] jepsen worker 28 - jepsen.util 2908 :invoke :write [9 6] INFO [2019-12-17 12:30:50,932] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":22894,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:30:50,933] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":22894,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:30:50,933] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":22896,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:30:50,933] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":22896,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:30:50,938] jepsen worker 38 - jepsen.util 3908 :info :write [12 4] :RequestCanceledException INFO [2019-12-17 12:30:50,940] jepsen worker 38 - jepsen.util 3998 :invoke :cas [12 [9 11]] INFO [2019-12-17 12:30:50,941] jepsen worker 63 - jepsen.util 3393 :ok :write [21 23] INFO [2019-12-17 12:30:50,942] jepsen worker 63 - jepsen.util 3393 :invoke :write [21 43] INFO [2019-12-17 12:30:50,972] jepsen worker 73 - jepsen.util 3853 :ok :write [24 31] INFO [2019-12-17 12:30:50,973] jepsen worker 73 - jepsen.util 3853 :invoke :write [24 8] INFO [2019-12-17 12:30:50,976] jepsen worker 30 - jepsen.util 4350 :ok :write [10 26] INFO [2019-12-17 12:30:50,977] jepsen worker 30 - jepsen.util 4350 :invoke :write [10 5] INFO [2019-12-17 12:30:50,977] jepsen worker 29 - jepsen.util 3359 :ok :read [9 13] INFO [2019-12-17 12:30:50,977] jepsen worker 29 - jepsen.util 3359 :invoke :write [9 20] INFO [2019-12-17 12:30:50,978] jepsen worker 47 - jepsen.util 4457 :ok :write [15 10] INFO [2019-12-17 12:30:50,978] jepsen worker 47 - jepsen.util 4457 :invoke :cas [15 [5 3]] INFO [2019-12-17 12:30:50,989] jepsen worker 68 - jepsen.util 4838 :info :write [22 43] :RequestCanceledException INFO [2019-12-17 12:30:50,991] jepsen worker 68 - jepsen.util 4928 :invoke :write [22 0] INFO [2019-12-17 12:30:51,041] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":22905,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:30:51,041] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":22905,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:30:51,041] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":22908,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:30:51,041] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":22908,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:30:51,041] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":22910,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:30:51,041] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":22910,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:30:51,042] jepsen worker 47 - jepsen.util 4457 :fail :cas [15 [5 3]] :ValueNotSwapFrom INFO [2019-12-17 12:30:51,042] jepsen worker 47 - jepsen.util 4457 :invoke :cas [15 [24 4]] INFO [2019-12-17 12:30:51,047] jepsen worker 54 - jepsen.util 4374 :ok :read [18 21] INFO [2019-12-17 12:30:51,047] jepsen worker 54 - jepsen.util 4374 :invoke :read [18 nil] INFO [2019-12-17 12:30:51,068] jepsen worker 47 - jepsen.util 4457 :fail :cas [15 [24 4]] :ValueNotSwapFrom INFO [2019-12-17 12:30:51,068] jepsen worker 47 - jepsen.util 4457 :invoke :read [15 nil] INFO [2019-12-17 12:30:51,080] jepsen worker 28 - jepsen.util 2908 :ok :write [9 6] INFO [2019-12-17 12:30:51,082] jepsen worker 28 - jepsen.util 2908 :invoke :read [9 nil] INFO [2019-12-17 12:30:51,089] jepsen worker 63 - jepsen.util 3393 :ok :write [21 43] INFO [2019-12-17 12:30:51,089] jepsen worker 63 - jepsen.util 3393 :invoke :read [21 nil] INFO [2019-12-17 12:30:51,096] jepsen worker 14 - jepsen.util 4064 :fail :cas [4 [14 46]] :ValueNotSwapFrom INFO [2019-12-17 12:30:51,096] jepsen worker 14 - jepsen.util 4064 :invoke :cas [4 [18 19]] INFO [2019-12-17 12:30:51,116] jepsen worker 47 - jepsen.util 4457 :ok :read [15 10] INFO [2019-12-17 12:30:51,116] jepsen worker 47 - jepsen.util 4457 :invoke :read [15 nil] INFO [2019-12-17 12:30:51,125] jepsen worker 12 - jepsen.util 4422 :ok :read [4 49] INFO [2019-12-17 12:30:51,125] jepsen worker 13 - jepsen.util 3793 :fail :cas [4 [16 1]] :ValueNotSwapFrom INFO [2019-12-17 12:30:51,126] jepsen worker 13 - jepsen.util 3793 :invoke :read [4 nil] INFO [2019-12-17 12:30:51,128] jepsen worker 12 - jepsen.util 4422 :invoke :cas [4 [48 31]] INFO [2019-12-17 12:30:51,137] jepsen worker 23 - jepsen.util 5153 :info :write [7 16] :RequestCanceledException INFO [2019-12-17 12:30:51,138] jepsen worker 47 - jepsen.util 4457 :ok :read [15 10] INFO [2019-12-17 12:30:51,138] jepsen worker 47 - jepsen.util 4457 :invoke :read [15 nil] INFO [2019-12-17 12:30:51,138] jepsen worker 23 - jepsen.util 5243 :invoke :read [7 nil] INFO [2019-12-17 12:30:51,141] jepsen worker 28 - jepsen.util 2908 :ok :read [9 6] INFO [2019-12-17 12:30:51,141] jepsen worker 28 - jepsen.util 2908 :invoke :write [9 0] INFO [2019-12-17 12:30:51,154] jepsen worker 44 - jepsen.util 5714 :info :write [14 35] :RequestCanceledException INFO [2019-12-17 12:30:51,155] jepsen worker 44 - jepsen.util 5804 :invoke :read [14 nil] INFO [2019-12-17 12:30:51,161] jepsen worker 54 - jepsen.util 4374 :ok :read [18 21] INFO [2019-12-17 12:30:51,161] jepsen worker 54 - jepsen.util 4374 :invoke :read [18 nil] INFO [2019-12-17 12:30:51,175] jepsen worker 47 - jepsen.util 4457 :ok :read [15 10] INFO [2019-12-17 12:30:51,175] jepsen worker 47 - jepsen.util 4457 :invoke :read [15 nil] INFO [2019-12-17 12:30:51,185] jepsen worker 29 - jepsen.util 3359 :ok :write [9 20] INFO [2019-12-17 12:30:51,185] jepsen worker 29 - jepsen.util 3359 :invoke :write [9 8] INFO [2019-12-17 12:30:51,188] jepsen worker 59 - jepsen.util 5369 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:30:51,189] jepsen worker 59 - jepsen.util 5369 :invoke :cas [19 [46 11]] INFO [2019-12-17 12:30:51,193] jepsen worker 47 - jepsen.util 4457 :ok :read [15 10] INFO [2019-12-17 12:30:51,193] jepsen worker 47 - jepsen.util 4457 :invoke :write [15 36] INFO [2019-12-17 12:30:51,208] jepsen worker 63 - jepsen.util 3393 :ok :read [21 43] INFO [2019-12-17 12:30:51,211] jepsen worker 63 - jepsen.util 3393 :invoke :write [21 6] INFO [2019-12-17 12:30:51,217] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":22939,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:30:51,217] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":22939,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:30:51,217] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":22942,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:30:51,217] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":22942,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:30:51,222] jepsen worker 28 - jepsen.util 2908 :ok :write [9 0] INFO [2019-12-17 12:30:51,222] jepsen worker 28 - jepsen.util 2908 :invoke :cas [9 [37 6]] INFO [2019-12-17 12:30:51,238] jepsen worker 66 - jepsen.util 5196 :info :cas [22 [31 38]] :RequestCanceledException INFO [2019-12-17 12:30:51,238] jepsen worker 84 - jepsen.util 4854 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:30:51,239] jepsen worker 84 - jepsen.util 4854 :invoke :cas [28 [48 18]] INFO [2019-12-17 12:30:51,239] jepsen worker 66 - jepsen.util 5286 :invoke :read [22 nil] INFO [2019-12-17 12:30:51,253] jepsen worker 28 - jepsen.util 2908 :fail :cas [9 [37 6]] :ValueNotSwapFrom INFO [2019-12-17 12:30:51,254] jepsen worker 28 - jepsen.util 2908 :invoke :write [9 38] INFO [2019-12-17 12:30:51,256] jepsen worker 64 - jepsen.util 3214 :info :write [21 44] :RequestCanceledException INFO [2019-12-17 12:30:51,256] jepsen worker 18 - jepsen.util 4068 :info :write [6 48] :RequestCanceledException INFO [2019-12-17 12:30:51,257] jepsen worker 64 - jepsen.util 3304 :invoke :read [21 nil] INFO [2019-12-17 12:30:51,259] jepsen worker 18 - jepsen.util 4158 :invoke :write [6 42] INFO [2019-12-17 12:30:51,260] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":22930,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:30:51,260] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":22930,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:30:51,260] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":22931,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:30:51,260] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":22931,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:30:51,292] jepsen worker 14 - jepsen.util 4064 :fail :cas [4 [18 19]] :ValueNotSwapFrom INFO [2019-12-17 12:30:51,292] jepsen worker 14 - jepsen.util 4064 :invoke :cas [4 [26 47]] INFO [2019-12-17 12:30:51,295] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:30:51,295] jepsen nemesis - couchbase.util Rebalance check count: 170 INFO [2019-12-17 12:30:51,295] jepsen nemesis - couchbase.util Rebalance stuck count: 151 INFO [2019-12-17 12:30:51,305] jepsen worker 63 - jepsen.util 3393 :ok :write [21 6] INFO [2019-12-17 12:30:51,305] jepsen worker 63 - jepsen.util 3393 :invoke :read [21 nil] INFO [2019-12-17 12:30:51,309] jepsen worker 43 - jepsen.util 4813 :info :cas [14 [16 38]] :RequestCanceledException INFO [2019-12-17 12:30:51,310] jepsen worker 43 - jepsen.util 4903 :invoke :write [14 17] INFO [2019-12-17 12:30:51,325] jepsen worker 6 - jepsen.util 5766 :info :cas [2 [43 10]] :RequestCanceledException INFO [2019-12-17 12:30:51,325] jepsen worker 25 - jepsen.util 5425 :info :cas [8 [43 12]] :RequestCanceledException INFO [2019-12-17 12:30:51,326] jepsen worker 6 - jepsen.util 5856 :invoke :write [2 24] INFO [2019-12-17 12:30:51,326] jepsen worker 25 - jepsen.util 5515 :invoke :cas [8 [13 10]] INFO [2019-12-17 12:30:51,335] jepsen worker 28 - jepsen.util 2908 :ok :write [9 38] INFO [2019-12-17 12:30:51,335] jepsen worker 28 - jepsen.util 2908 :invoke :cas [9 [41 41]] INFO [2019-12-17 12:30:51,336] jepsen worker 63 - jepsen.util 3393 :ok :read [21 6] INFO [2019-12-17 12:30:51,336] jepsen worker 63 - jepsen.util 3393 :invoke :cas [21 [4 0]] INFO [2019-12-17 12:30:51,354] jepsen worker 37 - jepsen.util 3727 :info :write [12 22] :RequestCanceledException INFO [2019-12-17 12:30:51,356] jepsen worker 37 - jepsen.util 3817 :invoke :cas [12 [37 9]] INFO [2019-12-17 12:30:51,360] jepsen worker 63 - jepsen.util 3393 :fail :cas [21 [4 0]] :ValueNotSwapFrom INFO [2019-12-17 12:30:51,360] jepsen worker 63 - jepsen.util 3393 :invoke :cas [21 [42 22]] INFO [2019-12-17 12:30:51,384] jepsen worker 63 - jepsen.util 3393 :fail :cas [21 [42 22]] :ValueNotSwapFrom INFO [2019-12-17 12:30:51,384] jepsen worker 63 - jepsen.util 3393 :invoke :write [21 24] INFO [2019-12-17 12:30:51,391] jepsen worker 65 - jepsen.util 3305 :fail :read [21 nil] :RequestTimeoutException INFO [2019-12-17 12:30:51,391] jepsen worker 3 - jepsen.util 4233 :info :write [1 39] :RequestCanceledException INFO [2019-12-17 12:30:51,391] jepsen worker 65 - jepsen.util 3305 :invoke :cas [21 [26 35]] INFO [2019-12-17 12:30:51,392] jepsen worker 3 - jepsen.util 4323 :invoke :read [1 nil] INFO [2019-12-17 12:30:51,400] jepsen worker 28 - jepsen.util 2908 :fail :cas [9 [41 41]] :ValueNotSwapFrom INFO [2019-12-17 12:30:51,400] jepsen worker 28 - jepsen.util 2908 :invoke :write [9 28] INFO [2019-12-17 12:30:51,405] jepsen worker 12 - jepsen.util 4422 :fail :cas [4 [48 31]] :ValueNotSwapFrom INFO [2019-12-17 12:30:51,405] jepsen worker 12 - jepsen.util 4422 :invoke :cas [4 [41 41]] INFO [2019-12-17 12:30:51,407] jepsen worker 62 - jepsen.util 4922 :info :write [20 23] :RequestCanceledException INFO [2019-12-17 12:30:51,408] jepsen worker 62 - jepsen.util 5012 :invoke :read [20 nil] INFO [2019-12-17 12:30:51,424] jepsen worker 39 - jepsen.util 4899 :info :cas [13 [22 49]] :RequestCanceledException INFO [2019-12-17 12:30:51,424] jepsen worker 82 - jepsen.util 6202 :info :write [27 38] :RequestCanceledException INFO [2019-12-17 12:30:51,424] jepsen worker 72 - jepsen.util 4122 :info :write [24 30] :RequestCanceledException INFO [2019-12-17 12:30:51,425] jepsen worker 39 - jepsen.util 4989 :invoke :read [13 nil] INFO [2019-12-17 12:30:51,425] jepsen worker 72 - jepsen.util 4212 :invoke :read [24 nil] INFO [2019-12-17 12:30:51,425] jepsen worker 82 - jepsen.util 6292 :invoke :write [27 34] INFO [2019-12-17 12:30:51,435] jepsen worker 63 - jepsen.util 3393 :ok :write [21 24] INFO [2019-12-17 12:30:51,436] jepsen worker 63 - jepsen.util 3393 :invoke :write [21 28] INFO [2019-12-17 12:30:51,438] jepsen worker 7 - jepsen.util 5227 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:30:51,438] jepsen worker 7 - jepsen.util 5227 :invoke :read [2 nil] INFO [2019-12-17 12:30:51,448] jepsen worker 4 - jepsen.util 3244 :ok :read [1 33] INFO [2019-12-17 12:30:51,448] jepsen worker 4 - jepsen.util 3244 :invoke :write [1 11] INFO [2019-12-17 12:30:51,455] jepsen worker 85 - jepsen.util 5845 :info :write [28 24] :RequestCanceledException INFO [2019-12-17 12:30:51,455] jepsen worker 72 - jepsen.util 4212 :ok :read [24 31] INFO [2019-12-17 12:30:51,455] jepsen worker 72 - jepsen.util 4212 :invoke :cas [24 [46 20]] INFO [2019-12-17 12:30:51,456] jepsen worker 85 - jepsen.util 5935 :invoke :cas [28 [13 47]] INFO [2019-12-17 12:30:51,484] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":22969,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:30:51,484] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":22969,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:30:51,484] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":22971,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:30:51,484] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":22971,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:30:51,484] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":22972,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:30:51,484] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":22972,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:30:51,496] jepsen worker 47 - jepsen.util 4457 :ok :write [15 36] INFO [2019-12-17 12:30:51,496] jepsen worker 47 - jepsen.util 4457 :invoke :write [15 7] INFO [2019-12-17 12:30:51,518] jepsen worker 54 - jepsen.util 4374 :ok :read [18 21] INFO [2019-12-17 12:30:51,518] jepsen worker 54 - jepsen.util 4374 :invoke :write [18 12] INFO [2019-12-17 12:30:51,521] jepsen worker 30 - jepsen.util 4350 :ok :write [10 5] INFO [2019-12-17 12:30:51,521] jepsen worker 30 - jepsen.util 4350 :invoke :write [10 48] INFO [2019-12-17 12:30:51,521] jepsen worker 73 - jepsen.util 3853 :ok :write [24 8] INFO [2019-12-17 12:30:51,521] jepsen worker 73 - jepsen.util 3853 :invoke :read [24 nil] INFO [2019-12-17 12:30:51,522] jepsen worker 89 - jepsen.util 3779 :info :write [29 31] :RequestCanceledException INFO [2019-12-17 12:30:51,524] jepsen worker 89 - jepsen.util 3869 :invoke :write [29 18] INFO [2019-12-17 12:30:51,537] jepsen worker 10 - jepsen.util 3880 :info :cas [3 [23 0]] :RequestCanceledException INFO [2019-12-17 12:30:51,538] jepsen worker 72 - jepsen.util 4212 :fail :cas [24 [46 20]] :ValueNotSwapFrom INFO [2019-12-17 12:30:51,538] jepsen worker 72 - jepsen.util 4212 :invoke :write [24 20] INFO [2019-12-17 12:30:51,539] jepsen worker 10 - jepsen.util 3970 :invoke :read [3 nil] INFO [2019-12-17 12:30:51,548] jepsen worker 87 - jepsen.util 3777 :fail :cas [29 [10 25]] :ValueNotSwapFrom INFO [2019-12-17 12:30:51,548] jepsen worker 87 - jepsen.util 3777 :invoke :read [29 nil] INFO [2019-12-17 12:30:51,553] jepsen worker 3 - jepsen.util 4323 :ok :read [1 33] INFO [2019-12-17 12:30:51,553] jepsen worker 3 - jepsen.util 4323 :invoke :write [1 48] INFO [2019-12-17 12:30:51,584] jepsen worker 64 - jepsen.util 3304 :ok :read [21 24] INFO [2019-12-17 12:30:51,584] jepsen worker 64 - jepsen.util 3304 :invoke :read [21 nil] INFO [2019-12-17 12:30:51,588] jepsen worker 73 - jepsen.util 3853 :ok :read [24 8] INFO [2019-12-17 12:30:51,588] jepsen worker 73 - jepsen.util 3853 :invoke :read [24 nil] INFO [2019-12-17 12:30:51,591] jepsen worker 50 - jepsen.util 5720 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:30:51,591] jepsen worker 50 - jepsen.util 5720 :invoke :read [16 nil] INFO [2019-12-17 12:30:51,597] jepsen worker 63 - jepsen.util 3393 :ok :write [21 28] INFO [2019-12-17 12:30:51,597] jepsen worker 63 - jepsen.util 3393 :invoke :cas [21 [13 23]] INFO [2019-12-17 12:30:51,613] jepsen worker 28 - jepsen.util 2908 :ok :write [9 28] INFO [2019-12-17 12:30:51,613] jepsen worker 28 - jepsen.util 2908 :invoke :cas [9 [30 38]] INFO [2019-12-17 12:30:51,615] jepsen worker 78 - jepsen.util 4488 :ok :write [26 25] INFO [2019-12-17 12:30:51,615] jepsen worker 78 - jepsen.util 4488 :invoke :read [26 nil] INFO [2019-12-17 12:30:51,620] jepsen worker 12 - jepsen.util 4422 :fail :cas [4 [41 41]] :ValueNotSwapFrom INFO [2019-12-17 12:30:51,620] jepsen worker 12 - jepsen.util 4422 :invoke :write [4 33] INFO [2019-12-17 12:30:51,624] jepsen worker 45 - jepsen.util 4185 :fail :read [15 nil] :RequestTimeoutException INFO [2019-12-17 12:30:51,624] jepsen worker 45 - jepsen.util 4185 :invoke :read [15 nil] INFO [2019-12-17 12:30:51,624] jepsen worker 11 - jepsen.util 4421 :fail :read [3 nil] :RequestTimeoutException INFO [2019-12-17 12:30:51,624] jepsen worker 83 - jepsen.util 5393 :info :write [27 27] :RequestCanceledException INFO [2019-12-17 12:30:51,625] jepsen worker 11 - jepsen.util 4421 :invoke :write [3 7] INFO [2019-12-17 12:30:51,625] jepsen worker 83 - jepsen.util 5483 :invoke :read [27 nil] INFO [2019-12-17 12:30:51,638] jepsen worker 73 - jepsen.util 3853 :ok :read [24 8] INFO [2019-12-17 12:30:51,639] jepsen worker 73 - jepsen.util 3853 :invoke :cas [24 [25 1]] INFO [2019-12-17 12:30:51,639] jepsen worker 63 - jepsen.util 3393 :fail :cas [21 [13 23]] :ValueNotSwapFrom INFO [2019-12-17 12:30:51,639] jepsen worker 63 - jepsen.util 3393 :invoke :write [21 8] INFO [2019-12-17 12:30:51,655] jepsen worker 14 - jepsen.util 4064 :fail :cas [4 [26 47]] :ValueNotSwapFrom INFO [2019-12-17 12:30:51,655] jepsen worker 14 - jepsen.util 4064 :invoke :cas [4 [36 48]] INFO [2019-12-17 12:30:51,655] jepsen worker 0 - jepsen.util 5850 :info :write [0 21] :RequestCanceledException INFO [2019-12-17 12:30:51,656] jepsen worker 67 - jepsen.util 5557 :info :cas [22 [2 21]] :RequestCanceledException INFO [2019-12-17 12:30:51,656] jepsen worker 0 - jepsen.util 5940 :invoke :read [0 nil] INFO [2019-12-17 12:30:51,657] jepsen worker 67 - jepsen.util 5647 :invoke :read [22 nil] INFO [2019-12-17 12:30:51,683] jepsen worker 73 - jepsen.util 3853 :fail :cas [24 [25 1]] :ValueNotSwapFrom INFO [2019-12-17 12:30:51,683] jepsen worker 73 - jepsen.util 3853 :invoke :cas [24 [33 5]] INFO [2019-12-17 12:30:51,685] jepsen worker 28 - jepsen.util 2908 :fail :cas [9 [30 38]] :ValueNotSwapFrom INFO [2019-12-17 12:30:51,685] jepsen worker 28 - jepsen.util 2908 :invoke :cas [9 [2 5]] INFO [2019-12-17 12:30:51,691] jepsen worker 26 - jepsen.util 5516 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:30:51,691] jepsen worker 40 - jepsen.util 5800 :info :cas [13 [48 1]] :RequestCanceledException INFO [2019-12-17 12:30:51,691] jepsen worker 26 - jepsen.util 5516 :invoke :write [8 34] INFO [2019-12-17 12:30:51,692] jepsen worker 40 - jepsen.util 5890 :invoke :write [13 17] INFO [2019-12-17 12:30:51,694] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":23001,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:30:51,694] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":23001,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:30:51,694] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":23003,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:30:51,694] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":23003,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:30:51,709] jepsen worker 57 - jepsen.util 5097 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:30:51,709] jepsen worker 48 - jepsen.util 4728 :info :write [16 23] :RequestCanceledException INFO [2019-12-17 12:30:51,709] jepsen worker 5 - jepsen.util 3605 :info :write [1 33] :RequestCanceledException INFO [2019-12-17 12:30:51,709] jepsen worker 57 - jepsen.util 5097 :invoke :cas [19 [48 35]] INFO [2019-12-17 12:30:51,710] jepsen worker 48 - jepsen.util 4818 :invoke :cas [16 [25 9]] INFO [2019-12-17 12:30:51,710] jepsen worker 5 - jepsen.util 3695 :invoke :write [1 35] INFO [2019-12-17 12:30:51,724] jepsen worker 60 - jepsen.util 5730 :info :write [20 38] :RequestCanceledException INFO [2019-12-17 12:30:51,724] jepsen worker 35 - jepsen.util 5255 :info :cas [11 [49 10]] :RequestCanceledException INFO [2019-12-17 12:30:51,724] jepsen worker 58 - jepsen.util 5368 :info :cas [19 [13 42]] :RequestCanceledException INFO [2019-12-17 12:30:51,725] jepsen worker 35 - jepsen.util 5345 :invoke :cas [11 [32 25]] INFO [2019-12-17 12:30:51,726] jepsen worker 58 - jepsen.util 5458 :invoke :cas [19 [2 20]] INFO [2019-12-17 12:30:51,728] jepsen worker 60 - jepsen.util 5820 :invoke :write [20 38] INFO [2019-12-17 12:30:51,732] jepsen worker 55 - jepsen.util 4915 :info :write [18 7] :RequestCanceledException INFO [2019-12-17 12:30:51,737] jepsen worker 55 - jepsen.util 5005 :invoke :cas [18 [45 41]] INFO [2019-12-17 12:30:51,740] jepsen worker 73 - jepsen.util 3853 :fail :cas [24 [33 5]] :ValueNotSwapFrom INFO [2019-12-17 12:30:51,740] jepsen worker 73 - jepsen.util 3853 :invoke :read [24 nil] INFO [2019-12-17 12:30:51,740] jepsen worker 45 - jepsen.util 4185 :ok :read [15 36] INFO [2019-12-17 12:30:51,740] jepsen worker 45 - jepsen.util 4185 :invoke :cas [15 [44 6]] INFO [2019-12-17 12:30:51,741] jepsen worker 38 - jepsen.util 3998 :fail :cas [12 [9 11]] :ValueNotSwapFrom INFO [2019-12-17 12:30:51,741] jepsen worker 38 - jepsen.util 3998 :invoke :write [12 20] INFO [2019-12-17 12:30:51,754] jepsen worker 13 - jepsen.util 3793 :ok :read [4 49] INFO [2019-12-17 12:30:51,754] jepsen worker 13 - jepsen.util 3793 :invoke :write [4 27] INFO [2019-12-17 12:30:51,755] jepsen worker 2 - jepsen.util 4952 :info :cas [0 [41 44]] :RequestCanceledException INFO [2019-12-17 12:30:51,756] jepsen worker 2 - jepsen.util 5042 :invoke :write [0 43] INFO [2019-12-17 12:30:51,774] jepsen worker 18 - jepsen.util 4158 :ok :write [6 42] INFO [2019-12-17 12:30:51,775] jepsen worker 18 - jepsen.util 4158 :invoke :read [6 nil] INFO [2019-12-17 12:30:51,780] jepsen worker 73 - jepsen.util 3853 :ok :read [24 8] INFO [2019-12-17 12:30:51,780] jepsen worker 73 - jepsen.util 3853 :invoke :read [24 nil] INFO [2019-12-17 12:30:51,824] jepsen worker 17 - jepsen.util 4967 :info :cas [5 [27 28]] :RequestCanceledException INFO [2019-12-17 12:30:51,825] jepsen worker 33 - jepsen.util 5523 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:30:51,825] jepsen worker 22 - jepsen.util 5242 :info :write [7 16] :RequestCanceledException INFO [2019-12-17 12:30:51,825] jepsen worker 17 - jepsen.util 5057 :invoke :cas [5 [38 29]] INFO [2019-12-17 12:30:51,825] jepsen worker 33 - jepsen.util 5523 :invoke :read [11 nil] INFO [2019-12-17 12:30:51,826] jepsen worker 22 - jepsen.util 5332 :invoke :cas [7 [22 12]] INFO [2019-12-17 12:30:51,827] jepsen worker 37 - jepsen.util 3817 :fail :cas [12 [37 9]] :ValueNotSwapFrom INFO [2019-12-17 12:30:51,827] jepsen worker 37 - jepsen.util 3817 :invoke :read [12 nil] INFO [2019-12-17 12:30:51,838] jepsen worker 52 - jepsen.util 4192 :info :write [17 23] :RequestCanceledException INFO [2019-12-17 12:30:51,838] jepsen worker 70 - jepsen.util 4570 :info :write [23 23] :RequestCanceledException INFO [2019-12-17 12:30:51,839] jepsen worker 70 - jepsen.util 4660 :invoke :write [23 45] INFO [2019-12-17 12:30:51,839] jepsen worker 52 - jepsen.util 4282 :invoke :cas [17 [3 41]] INFO [2019-12-17 12:30:51,847] jepsen worker 28 - jepsen.util 2908 :fail :cas [9 [2 5]] :ValueNotSwapFrom INFO [2019-12-17 12:30:51,848] jepsen worker 28 - jepsen.util 2908 :invoke :cas [9 [8 18]] INFO [2019-12-17 12:30:51,854] jepsen worker 73 - jepsen.util 3853 :ok :read [24 8] INFO [2019-12-17 12:30:51,855] jepsen worker 73 - jepsen.util 3853 :invoke :cas [24 [10 12]] INFO [2019-12-17 12:30:51,884] jepsen worker 52 - jepsen.util 4282 :fail :cas [17 [3 41]] :ValueNotSwapFrom INFO [2019-12-17 12:30:51,884] jepsen worker 52 - jepsen.util 4282 :invoke :cas [17 [30 28]] INFO [2019-12-17 12:30:51,884] jepsen worker 56 - jepsen.util 4196 :ok :write [18 27] INFO [2019-12-17 12:30:51,884] jepsen worker 56 - jepsen.util 4196 :invoke :cas [18 [24 4]] INFO [2019-12-17 12:30:51,891] jepsen worker 73 - jepsen.util 3853 :fail :cas [24 [10 12]] :ValueNotSwapFrom INFO [2019-12-17 12:30:51,891] jepsen worker 51 - jepsen.util 3651 :info :write [17 6] :RequestCanceledException INFO [2019-12-17 12:30:51,891] jepsen worker 73 - jepsen.util 3853 :invoke :read [24 nil] INFO [2019-12-17 12:30:51,892] jepsen worker 51 - jepsen.util 3741 :invoke :read [17 nil] INFO [2019-12-17 12:30:51,899] jepsen worker 45 - jepsen.util 4185 :fail :cas [15 [44 6]] :ValueNotSwapFrom INFO [2019-12-17 12:30:51,899] jepsen worker 45 - jepsen.util 4185 :invoke :write [15 14] INFO [2019-12-17 12:30:51,909] jepsen worker 75 - jepsen.util 5385 :info :cas [25 [16 38]] :RequestCanceledException INFO [2019-12-17 12:30:51,910] jepsen worker 75 - jepsen.util 5475 :invoke :read [25 nil] INFO [2019-12-17 12:30:51,918] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":23036,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:30:51,918] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":23036,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:30:51,918] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":23039,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:30:51,918] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":23039,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:30:51,918] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":23040,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:30:51,918] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":23040,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:30:51,920] jepsen worker 10 - jepsen.util 3970 :ok :read [3 10] INFO [2019-12-17 12:30:51,921] jepsen worker 10 - jepsen.util 3970 :invoke :read [3 nil] INFO [2019-12-17 12:30:51,924] jepsen worker 42 - jepsen.util 6072 :info :write [14 48] :RequestCanceledException INFO [2019-12-17 12:30:51,924] jepsen worker 41 - jepsen.util 5441 :info :cas [13 [33 1]] :RequestCanceledException INFO [2019-12-17 12:30:51,934] jepsen worker 42 - jepsen.util 6162 :invoke :read [14 nil] INFO [2019-12-17 12:30:51,935] jepsen worker 37 - jepsen.util 3817 :ok :read [12 1] INFO [2019-12-17 12:30:51,935] jepsen worker 37 - jepsen.util 3817 :invoke :write [12 33] INFO [2019-12-17 12:30:51,942] jepsen worker 41 - jepsen.util 5531 :invoke :read [13 nil] INFO [2019-12-17 12:30:51,959] jepsen worker 52 - jepsen.util 4282 :fail :cas [17 [30 28]] :ValueNotSwapFrom INFO [2019-12-17 12:30:51,959] jepsen worker 52 - jepsen.util 4282 :invoke :read [17 nil] INFO [2019-12-17 12:30:51,959] jepsen worker 73 - jepsen.util 3853 :ok :read [24 8] INFO [2019-12-17 12:30:51,959] jepsen worker 73 - jepsen.util 3853 :invoke :cas [24 [34 3]] INFO [2019-12-17 12:30:51,991] jepsen worker 9 - jepsen.util 4419 :info :write [3 0] :RequestCanceledException INFO [2019-12-17 12:30:51,992] jepsen worker 9 - jepsen.util 4509 :invoke :write [3 11] INFO [2019-12-17 12:30:52,003] jepsen worker 63 - jepsen.util 3393 :ok :write [21 8] INFO [2019-12-17 12:30:52,003] jepsen worker 63 - jepsen.util 3393 :invoke :read [21 nil] INFO [2019-12-17 12:30:52,009] jepsen worker 1 - jepsen.util 4951 :info :write [0 23] :RequestCanceledException INFO [2019-12-17 12:30:52,010] jepsen worker 1 - jepsen.util 5041 :invoke :read [0 nil] INFO [2019-12-17 12:30:52,013] jepsen worker 52 - jepsen.util 4282 :ok :read [17 8] INFO [2019-12-17 12:30:52,013] jepsen worker 52 - jepsen.util 4282 :invoke :cas [17 [29 8]] INFO [2019-12-17 12:30:52,023] jepsen worker 28 - jepsen.util 2908 :fail :cas [9 [8 18]] :ValueNotSwapFrom INFO [2019-12-17 12:30:52,023] jepsen worker 28 - jepsen.util 2908 :invoke :read [9 nil] INFO [2019-12-17 12:30:52,024] jepsen worker 88 - jepsen.util 4138 :info :write [29 11] :RequestCanceledException INFO [2019-12-17 12:30:52,025] jepsen worker 36 - jepsen.util 4176 :info :write [12 13] :RequestCanceledException INFO [2019-12-17 12:30:52,025] jepsen worker 88 - jepsen.util 4228 :invoke :read [29 nil] INFO [2019-12-17 12:30:52,026] jepsen worker 36 - jepsen.util 4266 :invoke :read [12 nil] INFO [2019-12-17 12:30:52,027] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":23057,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:30:52,027] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":23057,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:30:52,075] jepsen worker 16 - jepsen.util 6046 :info :write [5 3] :RequestCanceledException INFO [2019-12-17 12:30:52,075] jepsen worker 76 - jepsen.util 5386 :info :cas [25 [22 0]] :RequestCanceledException INFO [2019-12-17 12:30:52,075] jepsen worker 15 - jepsen.util 4785 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:30:52,075] jepsen worker 15 - jepsen.util 4785 :invoke :read [5 nil] INFO [2019-12-17 12:30:52,076] jepsen worker 4 - jepsen.util 3244 :ok :write [1 11] INFO [2019-12-17 12:30:52,076] jepsen worker 4 - jepsen.util 3244 :invoke :cas [1 [20 35]] INFO [2019-12-17 12:30:52,076] jepsen worker 16 - jepsen.util 6136 :invoke :write [5 5] INFO [2019-12-17 12:30:52,076] jepsen worker 76 - jepsen.util 5476 :invoke :cas [25 [7 18]] INFO [2019-12-17 12:30:52,081] jepsen worker 63 - jepsen.util 3393 :ok :read [21 8] INFO [2019-12-17 12:30:52,081] jepsen worker 63 - jepsen.util 3393 :invoke :write [21 38] INFO [2019-12-17 12:30:52,082] jepsen worker 73 - jepsen.util 3853 :fail :cas [24 [34 3]] :ValueNotSwapFrom INFO [2019-12-17 12:30:52,083] jepsen worker 73 - jepsen.util 3853 :invoke :read [24 nil] INFO [2019-12-17 12:30:52,114] jepsen worker 73 - jepsen.util 3853 :ok :read [24 8] INFO [2019-12-17 12:30:52,114] jepsen worker 52 - jepsen.util 4282 :fail :cas [17 [29 8]] :ValueNotSwapFrom INFO [2019-12-17 12:30:52,114] jepsen worker 73 - jepsen.util 3853 :invoke :write [24 22] INFO [2019-12-17 12:30:52,114] jepsen worker 52 - jepsen.util 4282 :invoke :read [17 nil] INFO [2019-12-17 12:30:52,124] jepsen worker 8 - jepsen.util 4508 :info :write [2 48] :RequestCanceledException INFO [2019-12-17 12:30:52,125] jepsen worker 8 - jepsen.util 4598 :invoke :write [2 29] INFO [2019-12-17 12:30:52,138] jepsen worker 19 - jepsen.util 4069 :info :cas [6 [27 5]] :RequestCanceledException INFO [2019-12-17 12:30:52,139] jepsen worker 19 - jepsen.util 4159 :invoke :write [6 11] INFO [2019-12-17 12:30:52,144] jepsen worker 63 - jepsen.util 3393 :ok :write [21 38] INFO [2019-12-17 12:30:52,144] jepsen worker 63 - jepsen.util 3393 :invoke :read [21 nil] INFO [2019-12-17 12:30:52,156] jepsen worker 63 - jepsen.util 3393 :ok :read [21 38] INFO [2019-12-17 12:30:52,156] jepsen worker 63 - jepsen.util 3393 :invoke :read [21 nil] INFO [2019-12-17 12:30:52,173] jepsen worker 63 - jepsen.util 3393 :ok :read [21 38] INFO [2019-12-17 12:30:52,173] jepsen worker 63 - jepsen.util 3393 :invoke :read [21 nil] INFO [2019-12-17 12:30:52,183] jepsen worker 63 - jepsen.util 3393 :ok :read [21 38] INFO [2019-12-17 12:30:52,183] jepsen worker 63 - jepsen.util 3393 :invoke :cas [21 [22 3]] INFO [2019-12-17 12:30:52,190] jepsen worker 52 - jepsen.util 4282 :ok :read [17 8] INFO [2019-12-17 12:30:52,190] jepsen worker 52 - jepsen.util 4282 :invoke :write [17 6] INFO [2019-12-17 12:30:52,191] jepsen worker 69 - jepsen.util 4029 :info :write [23 13] :RequestCanceledException INFO [2019-12-17 12:30:52,193] jepsen worker 69 - jepsen.util 4119 :invoke :cas [23 [42 0]] INFO [2019-12-17 12:30:52,203] jepsen worker 63 - jepsen.util 3393 :fail :cas [21 [22 3]] :ValueNotSwapFrom INFO [2019-12-17 12:30:52,203] jepsen worker 63 - jepsen.util 3393 :invoke :cas [21 [46 45]] INFO [2019-12-17 12:30:52,206] jepsen worker 55 - jepsen.util 5005 :fail :cas [18 [45 41]] :ValueNotSwapFrom INFO [2019-12-17 12:30:52,207] jepsen worker 55 - jepsen.util 5005 :invoke :read [18 nil] INFO [2019-12-17 12:30:52,218] jepsen worker 63 - jepsen.util 3393 :fail :cas [21 [46 45]] :ValueNotSwapFrom INFO [2019-12-17 12:30:52,218] jepsen worker 63 - jepsen.util 3393 :invoke :cas [21 [34 44]] INFO [2019-12-17 12:30:52,225] jepsen worker 81 - jepsen.util 5301 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:30:52,225] jepsen worker 74 - jepsen.util 4484 :info :cas [24 [11 25]] :RequestCanceledException INFO [2019-12-17 12:30:52,225] jepsen worker 81 - jepsen.util 5301 :invoke :write [27 13] INFO [2019-12-17 12:30:52,226] jepsen worker 74 - jepsen.util 4574 :invoke :write [24 5] INFO [2019-12-17 12:30:52,232] jepsen worker 63 - jepsen.util 3393 :fail :cas [21 [34 44]] :ValueNotSwapFrom INFO [2019-12-17 12:30:52,232] jepsen worker 63 - jepsen.util 3393 :invoke :cas [21 [4 16]] INFO [2019-12-17 12:30:52,238] jepsen worker 80 - jepsen.util 4130 :info :write [26 30] :RequestCanceledException INFO [2019-12-17 12:30:52,239] jepsen worker 80 - jepsen.util 4220 :invoke :cas [26 [5 47]] INFO [2019-12-17 12:30:52,244] jepsen worker 63 - jepsen.util 3393 :fail :cas [21 [4 16]] :ValueNotSwapFrom INFO [2019-12-17 12:30:52,244] jepsen worker 51 - jepsen.util 3741 :ok :read [17 8] INFO [2019-12-17 12:30:52,244] jepsen worker 63 - jepsen.util 3393 :invoke :cas [21 [27 6]] INFO [2019-12-17 12:30:52,244] jepsen worker 51 - jepsen.util 3741 :invoke :read [17 nil] INFO [2019-12-17 12:30:52,247] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":23080,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:30:52,247] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":23080,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:30:52,247] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":23081,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:30:52,247] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":23081,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:30:52,247] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":23082,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:30:52,247] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":23082,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:30:52,254] jepsen worker 12 - jepsen.util 4422 :ok :write [4 33] INFO [2019-12-17 12:30:52,254] jepsen worker 12 - jepsen.util 4422 :invoke :write [4 31] INFO [2019-12-17 12:30:52,254] jepsen worker 63 - jepsen.util 3393 :fail :cas [21 [27 6]] :ValueNotSwapFrom INFO [2019-12-17 12:30:52,254] jepsen worker 63 - jepsen.util 3393 :invoke :read [21 nil] INFO [2019-12-17 12:30:52,256] jepsen worker 21 - jepsen.util 5331 :info :cas [7 [26 40]] :RequestCanceledException INFO [2019-12-17 12:30:52,257] jepsen worker 21 - jepsen.util 5421 :invoke :read [7 nil] INFO [2019-12-17 12:30:52,273] jepsen worker 63 - jepsen.util 3393 :ok :read [21 38] INFO [2019-12-17 12:30:52,273] jepsen worker 63 - jepsen.util 3393 :invoke :read [21 nil] INFO [2019-12-17 12:30:52,276] jepsen worker 14 - jepsen.util 4064 :fail :cas [4 [36 48]] :ValueNotSwapFrom INFO [2019-12-17 12:30:52,276] jepsen worker 14 - jepsen.util 4064 :invoke :write [4 34] INFO [2019-12-17 12:30:52,288] jepsen worker 63 - jepsen.util 3393 :ok :read [21 38] INFO [2019-12-17 12:30:52,289] jepsen worker 63 - jepsen.util 3393 :invoke :cas [21 [40 26]] INFO [2019-12-17 12:30:52,298] jepsen worker 87 - jepsen.util 3777 :ok :read [29 18] INFO [2019-12-17 12:30:52,298] jepsen worker 87 - jepsen.util 3777 :invoke :write [29 24] INFO [2019-12-17 12:30:52,299] jepsen worker 63 - jepsen.util 3393 :fail :cas [21 [40 26]] :ValueNotSwapFrom INFO [2019-12-17 12:30:52,299] jepsen worker 63 - jepsen.util 3393 :invoke :cas [21 [17 21]] INFO [2019-12-17 12:30:52,307] jepsen worker 20 - jepsen.util 3890 :info :write [6 48] :RequestCanceledException INFO [2019-12-17 12:30:52,308] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:30:52,308] jepsen nemesis - couchbase.util Rebalance check count: 171 INFO [2019-12-17 12:30:52,308] jepsen nemesis - couchbase.util Rebalance stuck count: 152 INFO [2019-12-17 12:30:52,308] jepsen worker 20 - jepsen.util 3980 :invoke :write [6 29] INFO [2019-12-17 12:30:52,309] jepsen worker 63 - jepsen.util 3393 :fail :cas [21 [17 21]] :ValueNotSwapFrom INFO [2019-12-17 12:30:52,309] jepsen worker 63 - jepsen.util 3393 :invoke :cas [21 [34 0]] INFO [2019-12-17 12:30:52,314] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":23091,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:30:52,314] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":23091,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:30:52,314] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":23094,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:30:52,314] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":23094,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:30:52,314] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":23095,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:30:52,315] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":23095,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:30:52,325] jepsen worker 27 - jepsen.util 3897 :fail :read [9 nil] :RequestTimeoutException INFO [2019-12-17 12:30:52,325] jepsen worker 46 - jepsen.util 4006 :info :write [15 2] :RequestCanceledException INFO [2019-12-17 12:30:52,325] jepsen worker 27 - jepsen.util 3897 :invoke :write [9 12] INFO [2019-12-17 12:30:52,325] jepsen worker 46 - jepsen.util 4096 :invoke :cas [15 [24 35]] INFO [2019-12-17 12:30:52,327] jepsen worker 63 - jepsen.util 3393 :fail :cas [21 [34 0]] :ValueNotSwapFrom INFO [2019-12-17 12:30:52,327] jepsen worker 63 - jepsen.util 3393 :invoke :read [21 nil] INFO [2019-12-17 12:30:52,339] jepsen worker 63 - jepsen.util 3393 :ok :read [21 38] INFO [2019-12-17 12:30:52,339] jepsen worker 63 - jepsen.util 3393 :invoke :write [21 20] INFO [2019-12-17 12:30:52,339] jepsen worker 56 - jepsen.util 4196 :fail :cas [18 [24 4]] :ValueNotSwapFrom INFO [2019-12-17 12:30:52,339] jepsen worker 56 - jepsen.util 4196 :invoke :write [18 36] INFO [2019-12-17 12:30:52,363] jepsen worker 72 - jepsen.util 4212 :ok :write [24 20] INFO [2019-12-17 12:30:52,363] jepsen worker 72 - jepsen.util 4212 :invoke :cas [24 [15 48]] INFO [2019-12-17 12:30:52,363] jepsen worker 30 - jepsen.util 4350 :ok :write [10 48] INFO [2019-12-17 12:30:52,363] jepsen worker 30 - jepsen.util 4350 :invoke :read [10 nil] INFO [2019-12-17 12:30:52,378] jepsen worker 11 - jepsen.util 4421 :ok :write [3 7] INFO [2019-12-17 12:30:52,378] jepsen worker 11 - jepsen.util 4421 :invoke :read [3 nil] INFO [2019-12-17 12:30:52,395] jepsen worker 45 - jepsen.util 4185 :ok :write [15 14] INFO [2019-12-17 12:30:52,395] jepsen worker 45 - jepsen.util 4185 :invoke :write [15 12] INFO [2019-12-17 12:30:52,400] jepsen worker 10 - jepsen.util 3970 :ok :read [3 7] INFO [2019-12-17 12:30:52,400] jepsen worker 10 - jepsen.util 3970 :invoke :write [3 44] INFO [2019-12-17 12:30:52,412] jepsen worker 63 - jepsen.util 3393 :ok :write [21 20] INFO [2019-12-17 12:30:52,412] jepsen worker 63 - jepsen.util 3393 :invoke :read [21 nil] INFO [2019-12-17 12:30:52,425] jepsen worker 61 - jepsen.util 6091 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:30:52,425] jepsen worker 61 - jepsen.util 6091 :invoke :write [20 24] INFO [2019-12-17 12:30:52,437] jepsen worker 51 - jepsen.util 3741 :ok :read [17 8] INFO [2019-12-17 12:30:52,438] jepsen worker 51 - jepsen.util 3741 :invoke :write [17 20] INFO [2019-12-17 12:30:52,452] jepsen worker 63 - jepsen.util 3393 :ok :read [21 20] INFO [2019-12-17 12:30:52,452] jepsen worker 63 - jepsen.util 3393 :invoke :write [21 27] INFO [2019-12-17 12:30:52,462] jepsen worker 88 - jepsen.util 4228 :ok :read [29 18] INFO [2019-12-17 12:30:52,462] jepsen worker 88 - jepsen.util 4228 :invoke :write [29 20] INFO [2019-12-17 12:30:52,494] jepsen worker 28 - jepsen.util 2908 :ok :read [9 28] INFO [2019-12-17 12:30:52,494] jepsen worker 28 - jepsen.util 2908 :invoke :write [9 33] INFO [2019-12-17 12:30:52,500] jepsen worker 27 - jepsen.util 3897 :ok :write [9 12] INFO [2019-12-17 12:30:52,501] jepsen worker 27 - jepsen.util 3897 :invoke :write [9 42] INFO [2019-12-17 12:30:52,502] jepsen worker 4 - jepsen.util 3244 :fail :cas [1 [20 35]] :ValueNotSwapFrom INFO [2019-12-17 12:30:52,502] jepsen worker 4 - jepsen.util 3244 :invoke :cas [1 [45 40]] INFO [2019-12-17 12:30:52,509] jepsen worker 31 - jepsen.util 3631 :info :cas [10 [20 48]] :RequestCanceledException INFO [2019-12-17 12:30:52,510] jepsen worker 31 - jepsen.util 3721 :invoke :cas [10 [23 37]] INFO [2019-12-17 12:30:52,513] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":23125,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:30:52,514] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":23125,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:30:52,514] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":23128,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:30:52,514] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":23128,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:30:52,514] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":23129,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:30:52,514] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":23129,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:30:52,524] jepsen worker 32 - jepsen.util 4442 :info :cas [10 [2 16]] :RequestCanceledException INFO [2019-12-17 12:30:52,529] jepsen worker 32 - jepsen.util 4532 :invoke :cas [10 [28 0]] INFO [2019-12-17 12:30:52,549] jepsen worker 69 - jepsen.util 4119 :fail :cas [23 [42 0]] :ValueNotSwapFrom INFO [2019-12-17 12:30:52,550] jepsen worker 69 - jepsen.util 4119 :invoke :read [23 nil] INFO [2019-12-17 12:30:52,555] jepsen worker 71 - jepsen.util 4211 :info :write [23 8] :RequestCanceledException INFO [2019-12-17 12:30:52,556] jepsen worker 71 - jepsen.util 4301 :invoke :write [23 31] INFO [2019-12-17 12:30:52,591] jepsen worker 55 - jepsen.util 5005 :ok :read [18 27] INFO [2019-12-17 12:30:52,592] jepsen worker 55 - jepsen.util 5005 :invoke :cas [18 [25 20]] INFO [2019-12-17 12:30:52,596] jepsen worker 63 - jepsen.util 3393 :ok :write [21 27] INFO [2019-12-17 12:30:52,596] jepsen worker 63 - jepsen.util 3393 :invoke :read [21 nil] INFO [2019-12-17 12:30:52,609] jepsen worker 79 - jepsen.util 3949 :info :write [26 48] :RequestCanceledException INFO [2019-12-17 12:30:52,610] jepsen worker 79 - jepsen.util 4039 :invoke :cas [26 [33 33]] INFO [2019-12-17 12:30:52,622] jepsen worker 77 - jepsen.util 5387 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:30:52,622] jepsen worker 77 - jepsen.util 5387 :invoke :read [25 nil] INFO [2019-12-17 12:30:52,627] jepsen worker 56 - jepsen.util 4196 :ok :write [18 36] INFO [2019-12-17 12:30:52,627] jepsen worker 56 - jepsen.util 4196 :invoke :write [18 33] INFO [2019-12-17 12:30:52,629] jepsen worker 63 - jepsen.util 3393 :ok :read [21 27] INFO [2019-12-17 12:30:52,629] jepsen worker 63 - jepsen.util 3393 :invoke :cas [21 [37 48]] INFO [2019-12-17 12:30:52,645] jepsen worker 53 - jepsen.util 4013 :ok :write [17 20] INFO [2019-12-17 12:30:52,645] jepsen worker 53 - jepsen.util 4013 :invoke :write [17 39] INFO [2019-12-17 12:30:52,658] jepsen worker 63 - jepsen.util 3393 :fail :cas [21 [37 48]] :ValueNotSwapFrom INFO [2019-12-17 12:30:52,659] jepsen worker 63 - jepsen.util 3393 :invoke :cas [21 [49 3]] INFO [2019-12-17 12:30:52,662] jepsen worker 37 - jepsen.util 3817 :ok :write [12 33] INFO [2019-12-17 12:30:52,662] jepsen worker 37 - jepsen.util 3817 :invoke :cas [12 [48 16]] INFO [2019-12-17 12:30:52,681] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":23148,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:30:52,681] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":23148,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:30:52,681] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":23151,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:30:52,681] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":23151,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:30:52,681] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":23152,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:30:52,681] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":23152,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:30:52,687] jepsen worker 63 - jepsen.util 3393 :fail :cas [21 [49 3]] :ValueNotSwapFrom INFO [2019-12-17 12:30:52,687] jepsen worker 63 - jepsen.util 3393 :invoke :write [21 8] INFO [2019-12-17 12:30:52,687] jepsen worker 24 - jepsen.util 5154 :info :write [8 34] :RequestCanceledException INFO [2019-12-17 12:30:52,687] jepsen worker 86 - jepsen.util 5576 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:30:52,688] jepsen worker 86 - jepsen.util 5576 :invoke :cas [28 [47 11]] INFO [2019-12-17 12:30:52,689] jepsen worker 24 - jepsen.util 5244 :invoke :cas [8 [11 22]] INFO [2019-12-17 12:30:52,696] jepsen worker 5 - jepsen.util 3695 :ok :write [1 35] INFO [2019-12-17 12:30:52,696] jepsen worker 5 - jepsen.util 3695 :invoke :write [1 32] INFO [2019-12-17 12:30:52,755] jepsen worker 49 - jepsen.util 4729 :info :write [16 9] :RequestCanceledException INFO [2019-12-17 12:30:52,756] jepsen worker 49 - jepsen.util 4819 :invoke :write [16 26] INFO [2019-12-17 12:30:52,759] jepsen worker 63 - jepsen.util 3393 :ok :write [21 8] INFO [2019-12-17 12:30:52,759] jepsen worker 63 - jepsen.util 3393 :invoke :read [21 nil] INFO [2019-12-17 12:30:52,761] jepsen worker 88 - jepsen.util 4228 :ok :write [29 20] INFO [2019-12-17 12:30:52,762] jepsen worker 88 - jepsen.util 4228 :invoke :cas [29 [0 37]] INFO [2019-12-17 12:30:52,778] jepsen worker 63 - jepsen.util 3393 :ok :read [21 8] INFO [2019-12-17 12:30:52,778] jepsen worker 63 - jepsen.util 3393 :invoke :read [21 nil] INFO [2019-12-17 12:30:52,779] jepsen worker 3 - jepsen.util 4323 :ok :write [1 48] INFO [2019-12-17 12:30:52,780] jepsen worker 3 - jepsen.util 4323 :invoke :read [1 nil] INFO [2019-12-17 12:30:52,789] jepsen worker 78 - jepsen.util 4488 :ok :read [26 30] INFO [2019-12-17 12:30:52,789] jepsen worker 78 - jepsen.util 4488 :invoke :cas [26 [46 45]] INFO [2019-12-17 12:30:52,796] jepsen worker 19 - jepsen.util 4159 :ok :write [6 11] INFO [2019-12-17 12:30:52,796] jepsen worker 19 - jepsen.util 4159 :invoke :cas [6 [37 29]] INFO [2019-12-17 12:30:52,796] jepsen worker 74 - jepsen.util 4574 :ok :write [24 5] INFO [2019-12-17 12:30:52,796] jepsen worker 74 - jepsen.util 4574 :invoke :write [24 21] INFO [2019-12-17 12:30:52,801] jepsen worker 63 - jepsen.util 3393 :ok :read [21 8] INFO [2019-12-17 12:30:52,801] jepsen worker 63 - jepsen.util 3393 :invoke :write [21 40] INFO [2019-12-17 12:30:52,804] jepsen worker 69 - jepsen.util 4119 :ok :read [23 45] INFO [2019-12-17 12:30:52,804] jepsen worker 69 - jepsen.util 4119 :invoke :cas [23 [43 7]] INFO [2019-12-17 12:30:52,825] jepsen worker 32 - jepsen.util 4532 :fail :cas [10 [28 0]] :ValueNotSwapFrom INFO [2019-12-17 12:30:52,825] jepsen worker 32 - jepsen.util 4532 :invoke :cas [10 [28 9]] INFO [2019-12-17 12:30:52,828] jepsen worker 10 - jepsen.util 3970 :ok :write [3 44] INFO [2019-12-17 12:30:52,828] jepsen worker 10 - jepsen.util 3970 :invoke :write [3 26] INFO [2019-12-17 12:30:52,837] jepsen worker 52 - jepsen.util 4282 :ok :write [17 6] INFO [2019-12-17 12:30:52,837] jepsen worker 52 - jepsen.util 4282 :invoke :write [17 39] INFO [2019-12-17 12:30:52,879] jepsen worker 27 - jepsen.util 3897 :ok :write [9 42] INFO [2019-12-17 12:30:52,880] jepsen worker 27 - jepsen.util 3897 :invoke :write [9 9] INFO [2019-12-17 12:30:52,903] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":23174,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:30:52,903] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":23174,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:30:52,903] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":23177,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:30:52,903] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":23177,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:30:52,908] jepsen worker 34 - jepsen.util 5074 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:30:52,909] jepsen worker 34 - jepsen.util 5074 :invoke :cas [11 [39 17]] INFO [2019-12-17 12:30:52,913] jepsen worker 63 - jepsen.util 3393 :ok :write [21 40] INFO [2019-12-17 12:30:52,913] jepsen worker 63 - jepsen.util 3393 :invoke :cas [21 [10 18]] INFO [2019-12-17 12:30:52,924] jepsen worker 63 - jepsen.util 3393 :fail :cas [21 [10 18]] :ValueNotSwapFrom INFO [2019-12-17 12:30:52,924] jepsen worker 63 - jepsen.util 3393 :invoke :write [21 11] INFO [2019-12-17 12:30:52,940] jepsen worker 4 - jepsen.util 3244 :fail :cas [1 [45 40]] :ValueNotSwapFrom INFO [2019-12-17 12:30:52,941] jepsen worker 4 - jepsen.util 3244 :invoke :write [1 25] INFO [2019-12-17 12:30:52,955] jepsen worker 36 - jepsen.util 4266 :ok :read [12 33] INFO [2019-12-17 12:30:52,955] jepsen worker 36 - jepsen.util 4266 :invoke :cas [12 [22 32]] INFO [2019-12-17 12:30:52,958] jepsen worker 88 - jepsen.util 4228 :fail :cas [29 [0 37]] :ValueNotSwapFrom INFO [2019-12-17 12:30:52,959] jepsen worker 88 - jepsen.util 4228 :invoke :cas [29 [26 4]] INFO [2019-12-17 12:30:52,973] jepsen worker 63 - jepsen.util 3393 :ok :write [21 11] INFO [2019-12-17 12:30:52,973] jepsen worker 63 - jepsen.util 3393 :invoke :read [21 nil] INFO [2019-12-17 12:30:52,989] jepsen worker 63 - jepsen.util 3393 :ok :read [21 11] INFO [2019-12-17 12:30:52,990] jepsen worker 63 - jepsen.util 3393 :invoke :read [21 nil] INFO [2019-12-17 12:30:53,001] jepsen worker 45 - jepsen.util 4185 :ok :write [15 12] INFO [2019-12-17 12:30:53,001] jepsen worker 45 - jepsen.util 4185 :invoke :write [15 29] INFO [2019-12-17 12:30:53,002] jepsen worker 74 - jepsen.util 4574 :ok :write [24 21] INFO [2019-12-17 12:30:53,002] jepsen worker 74 - jepsen.util 4574 :invoke :read [24 nil] INFO [2019-12-17 12:30:53,006] jepsen worker 63 - jepsen.util 3393 :ok :read [21 11] INFO [2019-12-17 12:30:53,006] jepsen worker 63 - jepsen.util 3393 :invoke :read [21 nil] INFO [2019-12-17 12:30:53,012] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":23189,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:30:53,012] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":23189,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:30:53,012] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":23193,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:30:53,012] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":23193,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:30:53,020] jepsen worker 55 - jepsen.util 5005 :fail :cas [18 [25 20]] :ValueNotSwapFrom INFO [2019-12-17 12:30:53,020] jepsen worker 55 - jepsen.util 5005 :invoke :cas [18 [29 23]] INFO [2019-12-17 12:30:53,021] jepsen worker 74 - jepsen.util 4574 :ok :read [24 21] INFO [2019-12-17 12:30:53,021] jepsen worker 74 - jepsen.util 4574 :invoke :cas [24 [47 31]] INFO [2019-12-17 12:30:53,038] jepsen worker 68 - jepsen.util 4928 :info :write [22 0] :RequestCanceledException INFO [2019-12-17 12:30:53,039] jepsen worker 68 - jepsen.util 5018 :invoke :read [22 nil] INFO [2019-12-17 12:30:53,043] jepsen worker 37 - jepsen.util 3817 :fail :cas [12 [48 16]] :ValueNotSwapFrom INFO [2019-12-17 12:30:53,043] jepsen worker 37 - jepsen.util 3817 :invoke :cas [12 [24 11]] INFO [2019-12-17 12:30:53,063] jepsen worker 10 - jepsen.util 3970 :ok :write [3 26] INFO [2019-12-17 12:30:53,063] jepsen worker 10 - jepsen.util 3970 :invoke :write [3 48] INFO [2019-12-17 12:30:53,068] jepsen worker 63 - jepsen.util 3393 :ok :read [21 11] INFO [2019-12-17 12:30:53,068] jepsen worker 63 - jepsen.util 3393 :invoke :cas [21 [37 24]] INFO [2019-12-17 12:30:53,074] jepsen worker 74 - jepsen.util 4574 :fail :cas [24 [47 31]] :ValueNotSwapFrom INFO [2019-12-17 12:30:53,074] jepsen worker 74 - jepsen.util 4574 :invoke :cas [24 [30 13]] INFO [2019-12-17 12:30:53,086] jepsen worker 69 - jepsen.util 4119 :fail :cas [23 [43 7]] :ValueNotSwapFrom INFO [2019-12-17 12:30:53,086] jepsen worker 69 - jepsen.util 4119 :invoke :cas [23 [45 23]] INFO [2019-12-17 12:30:53,090] jepsen worker 74 - jepsen.util 4574 :fail :cas [24 [30 13]] :ValueNotSwapFrom INFO [2019-12-17 12:30:53,090] jepsen worker 74 - jepsen.util 4574 :invoke :cas [24 [5 6]] INFO [2019-12-17 12:30:53,096] jepsen worker 63 - jepsen.util 3393 :fail :cas [21 [37 24]] :ValueNotSwapFrom INFO [2019-12-17 12:30:53,096] jepsen worker 63 - jepsen.util 3393 :invoke :write [21 26] INFO [2019-12-17 12:30:53,118] jepsen worker 74 - jepsen.util 4574 :fail :cas [24 [5 6]] :ValueNotSwapFrom INFO [2019-12-17 12:30:53,118] jepsen worker 74 - jepsen.util 4574 :invoke :write [24 38] INFO [2019-12-17 12:30:53,121] jepsen worker 79 - jepsen.util 4039 :fail :cas [26 [33 33]] :ValueNotSwapFrom INFO [2019-12-17 12:30:53,121] jepsen worker 79 - jepsen.util 4039 :invoke :read [26 nil] INFO [2019-12-17 12:30:53,156] jepsen worker 23 - jepsen.util 5243 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:30:53,156] jepsen worker 23 - jepsen.util 5243 :invoke :write [7 5] INFO [2019-12-17 12:30:53,199] jepsen worker 79 - jepsen.util 4039 :ok :read [26 30] INFO [2019-12-17 12:30:53,199] jepsen worker 79 - jepsen.util 4039 :invoke :write [26 10] INFO [2019-12-17 12:30:53,223] jepsen worker 44 - jepsen.util 5804 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:30:53,224] jepsen worker 44 - jepsen.util 5804 :invoke :read [14 nil] INFO [2019-12-17 12:30:53,232] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":23211,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:30:53,232] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":23211,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:30:53,232] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":23212,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:30:53,232] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":23212,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:30:53,233] jepsen worker 63 - jepsen.util 3393 :ok :write [21 26] INFO [2019-12-17 12:30:53,233] jepsen worker 63 - jepsen.util 3393 :invoke :cas [21 [17 27]] INFO [2019-12-17 12:30:53,260] jepsen worker 63 - jepsen.util 3393 :fail :cas [21 [17 27]] :ValueNotSwapFrom INFO [2019-12-17 12:30:53,260] jepsen worker 63 - jepsen.util 3393 :invoke :write [21 41] INFO [2019-12-17 12:30:53,290] jepsen worker 66 - jepsen.util 5286 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:30:53,290] jepsen worker 29 - jepsen.util 3359 :info :write [9 8] :RequestCanceledException INFO [2019-12-17 12:30:53,290] jepsen worker 59 - jepsen.util 5369 :info :cas [19 [46 11]] :RequestCanceledException INFO [2019-12-17 12:30:53,290] jepsen worker 66 - jepsen.util 5286 :invoke :read [22 nil] INFO [2019-12-17 12:30:53,291] jepsen worker 29 - jepsen.util 3449 :invoke :read [9 nil] INFO [2019-12-17 12:30:53,291] jepsen worker 59 - jepsen.util 5459 :invoke :read [19 nil] INFO [2019-12-17 12:30:53,298] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":23218,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:30:53,298] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":23218,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:30:53,298] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":23221,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:30:53,298] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":23221,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:30:53,304] jepsen worker 63 - jepsen.util 3393 :ok :write [21 41] INFO [2019-12-17 12:30:53,304] jepsen worker 63 - jepsen.util 3393 :invoke :cas [21 [31 10]] INFO [2019-12-17 12:30:53,317] jepsen worker 63 - jepsen.util 3393 :fail :cas [21 [31 10]] :ValueNotSwapFrom INFO [2019-12-17 12:30:53,317] jepsen worker 63 - jepsen.util 3393 :invoke :write [21 25] INFO [2019-12-17 12:30:53,322] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:30:53,322] jepsen nemesis - couchbase.util Rebalance check count: 172 INFO [2019-12-17 12:30:53,322] jepsen nemesis - couchbase.util Rebalance stuck count: 153 INFO [2019-12-17 12:30:53,338] jepsen worker 43 - jepsen.util 4903 :info :write [14 17] :RequestCanceledException INFO [2019-12-17 12:30:53,338] jepsen worker 84 - jepsen.util 4854 :info :cas [28 [48 18]] :RequestCanceledException INFO [2019-12-17 12:30:53,339] jepsen worker 84 - jepsen.util 4944 :invoke :cas [28 [15 35]] INFO [2019-12-17 12:30:53,339] jepsen worker 43 - jepsen.util 4993 :invoke :cas [14 [13 24]] INFO [2019-12-17 12:30:53,355] jepsen worker 6 - jepsen.util 5856 :info :write [2 24] :RequestCanceledException INFO [2019-12-17 12:30:53,355] jepsen worker 6 - jepsen.util 5946 :invoke :write [2 38] INFO [2019-12-17 12:30:53,357] jepsen worker 63 - jepsen.util 3393 :ok :write [21 25] INFO [2019-12-17 12:30:53,357] jepsen worker 63 - jepsen.util 3393 :invoke :read [21 nil] INFO [2019-12-17 12:30:53,373] jepsen worker 63 - jepsen.util 3393 :ok :read [21 25] INFO [2019-12-17 12:30:53,373] jepsen worker 63 - jepsen.util 3393 :invoke :write [21 8] INFO [2019-12-17 12:30:53,384] jepsen worker 78 - jepsen.util 4488 :fail :cas [26 [46 45]] :ValueNotSwapFrom INFO [2019-12-17 12:30:53,384] jepsen worker 78 - jepsen.util 4488 :invoke :cas [26 [1 48]] INFO [2019-12-17 12:30:53,419] jepsen worker 63 - jepsen.util 3393 :ok :write [21 8] INFO [2019-12-17 12:30:53,419] jepsen worker 63 - jepsen.util 3393 :invoke :cas [21 [27 25]] INFO [2019-12-17 12:30:53,424] jepsen worker 62 - jepsen.util 5012 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:30:53,424] jepsen worker 25 - jepsen.util 5515 :info :cas [8 [13 10]] :RequestCanceledException INFO [2019-12-17 12:30:53,424] jepsen worker 62 - jepsen.util 5012 :invoke :write [20 3] INFO [2019-12-17 12:30:53,425] jepsen worker 25 - jepsen.util 5605 :invoke :write [8 25] INFO [2019-12-17 12:30:53,439] jepsen worker 39 - jepsen.util 4989 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:30:53,439] jepsen worker 39 - jepsen.util 4989 :invoke :write [13 29] INFO [2019-12-17 12:30:53,446] jepsen worker 63 - jepsen.util 3393 :fail :cas [21 [27 25]] :ValueNotSwapFrom INFO [2019-12-17 12:30:53,446] jepsen worker 63 - jepsen.util 3393 :invoke :cas [21 [0 4]] INFO [2019-12-17 12:30:53,463] jepsen worker 3 - jepsen.util 4323 :ok :read [1 48] INFO [2019-12-17 12:30:53,463] jepsen worker 3 - jepsen.util 4323 :invoke :read [1 nil] INFO [2019-12-17 12:30:53,470] jepsen worker 63 - jepsen.util 3393 :fail :cas [21 [0 4]] :ValueNotSwapFrom INFO [2019-12-17 12:30:53,470] jepsen worker 63 - jepsen.util 3393 :invoke :read [21 nil] INFO [2019-12-17 12:30:53,481] jepsen worker 63 - jepsen.util 3393 :ok :read [21 8] INFO [2019-12-17 12:30:53,481] jepsen worker 63 - jepsen.util 3393 :invoke :write [21 25] INFO [2019-12-17 12:30:53,491] jepsen worker 85 - jepsen.util 5935 :info :cas [28 [13 47]] :RequestCanceledException INFO [2019-12-17 12:30:53,491] jepsen worker 65 - jepsen.util 3305 :info :cas [21 [26 35]] :RequestCanceledException INFO [2019-12-17 12:30:53,492] jepsen worker 65 - jepsen.util 3395 :invoke :read [21 nil] INFO [2019-12-17 12:30:53,492] jepsen worker 85 - jepsen.util 6025 :invoke :read [28 nil] INFO [2019-12-17 12:30:53,496] jepsen worker 36 - jepsen.util 4266 :fail :cas [12 [22 32]] :ValueNotSwapFrom INFO [2019-12-17 12:30:53,496] jepsen worker 36 - jepsen.util 4266 :invoke :cas [12 [48 48]] INFO [2019-12-17 12:30:53,498] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":23242,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:30:53,498] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":23242,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:30:53,498] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":23244,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:30:53,498] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":23244,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:30:53,507] jepsen worker 88 - jepsen.util 4228 :fail :cas [29 [26 4]] :ValueNotSwapFrom INFO [2019-12-17 12:30:53,507] jepsen worker 88 - jepsen.util 4228 :invoke :read [29 nil] INFO [2019-12-17 12:30:53,517] jepsen worker 63 - jepsen.util 3393 :ok :write [21 25] INFO [2019-12-17 12:30:53,517] jepsen worker 63 - jepsen.util 3393 :invoke :read [21 nil] INFO [2019-12-17 12:30:53,525] jepsen worker 82 - jepsen.util 6292 :info :write [27 34] :RequestCanceledException INFO [2019-12-17 12:30:53,525] jepsen worker 47 - jepsen.util 4457 :info :write [15 7] :RequestCanceledException INFO [2019-12-17 12:30:53,525] jepsen worker 82 - jepsen.util 6382 :invoke :read [27 nil] INFO [2019-12-17 12:30:53,525] jepsen worker 47 - jepsen.util 4547 :invoke :write [15 22] INFO [2019-12-17 12:30:53,529] jepsen worker 63 - jepsen.util 3393 :ok :read [21 25] INFO [2019-12-17 12:30:53,529] jepsen worker 63 - jepsen.util 3393 :invoke :read [21 nil] INFO [2019-12-17 12:30:53,539] jepsen worker 7 - jepsen.util 5227 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:30:53,539] jepsen worker 7 - jepsen.util 5227 :invoke :cas [2 [4 23]] INFO [2019-12-17 12:30:53,546] jepsen worker 63 - jepsen.util 3393 :ok :read [21 25] INFO [2019-12-17 12:30:53,547] jepsen worker 63 - jepsen.util 3393 :invoke :cas [21 [14 8]] INFO [2019-12-17 12:30:53,560] jepsen worker 55 - jepsen.util 5005 :fail :cas [18 [29 23]] :ValueNotSwapFrom INFO [2019-12-17 12:30:53,561] jepsen worker 55 - jepsen.util 5005 :invoke :write [18 33] INFO [2019-12-17 12:30:53,564] jepsen worker 36 - jepsen.util 4266 :fail :cas [12 [48 48]] :ValueNotSwapFrom INFO [2019-12-17 12:30:53,565] jepsen worker 36 - jepsen.util 4266 :invoke :read [12 nil] INFO [2019-12-17 12:30:53,573] jepsen worker 37 - jepsen.util 3817 :fail :cas [12 [24 11]] :ValueNotSwapFrom INFO [2019-12-17 12:30:53,573] jepsen worker 37 - jepsen.util 3817 :invoke :write [12 29] INFO [2019-12-17 12:30:53,578] jepsen worker 63 - jepsen.util 3393 :fail :cas [21 [14 8]] :ValueNotSwapFrom INFO [2019-12-17 12:30:53,578] jepsen worker 63 - jepsen.util 3393 :invoke :write [21 4] INFO [2019-12-17 12:30:53,609] jepsen worker 64 - jepsen.util 3304 :fail :read [21 nil] :RequestTimeoutException INFO [2019-12-17 12:30:53,609] jepsen worker 89 - jepsen.util 3869 :info :write [29 18] :RequestCanceledException INFO [2019-12-17 12:30:53,609] jepsen worker 54 - jepsen.util 4374 :info :write [18 12] :RequestCanceledException INFO [2019-12-17 12:30:53,609] jepsen worker 64 - jepsen.util 3304 :invoke :cas [21 [9 32]] INFO [2019-12-17 12:30:53,610] jepsen worker 89 - jepsen.util 3959 :invoke :read [29 nil] INFO [2019-12-17 12:30:53,610] jepsen worker 54 - jepsen.util 4464 :invoke :cas [18 [13 20]] INFO [2019-12-17 12:30:53,618] jepsen worker 63 - jepsen.util 3393 :ok :write [21 4] INFO [2019-12-17 12:30:53,618] jepsen worker 63 - jepsen.util 3393 :invoke :cas [21 [13 6]] INFO [2019-12-17 12:30:53,635] jepsen worker 63 - jepsen.util 3393 :fail :cas [21 [13 6]] :ValueNotSwapFrom INFO [2019-12-17 12:30:53,635] jepsen worker 63 - jepsen.util 3393 :invoke :cas [21 [49 34]] INFO [2019-12-17 12:30:53,638] jepsen worker 83 - jepsen.util 5483 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:30:53,638] jepsen worker 83 - jepsen.util 5483 :invoke :write [27 35] INFO [2019-12-17 12:30:53,649] jepsen worker 88 - jepsen.util 4228 :ok :read [29 20] INFO [2019-12-17 12:30:53,649] jepsen worker 88 - jepsen.util 4228 :invoke :cas [29 [6 26]] INFO [2019-12-17 12:30:53,652] jepsen worker 63 - jepsen.util 3393 :fail :cas [21 [49 34]] :ValueNotSwapFrom INFO [2019-12-17 12:30:53,652] jepsen worker 63 - jepsen.util 3393 :invoke :cas [21 [29 38]] INFO [2019-12-17 12:30:53,661] jepsen worker 63 - jepsen.util 3393 :fail :cas [21 [29 38]] :ValueNotSwapFrom INFO [2019-12-17 12:30:53,661] jepsen worker 63 - jepsen.util 3393 :invoke :read [21 nil] INFO [2019-12-17 12:30:53,665] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":23272,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:30:53,665] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":23272,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:30:53,665] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":23275,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:30:53,665] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":23275,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:30:53,673] jepsen worker 63 - jepsen.util 3393 :ok :read [21 4] INFO [2019-12-17 12:30:53,673] jepsen worker 63 - jepsen.util 3393 :invoke :read [21 nil] INFO [2019-12-17 12:30:53,688] jepsen worker 63 - jepsen.util 3393 :ok :read [21 4] INFO [2019-12-17 12:30:53,689] jepsen worker 63 - jepsen.util 3393 :invoke :read [21 nil] INFO [2019-12-17 12:30:53,691] jepsen worker 50 - jepsen.util 5720 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:30:53,691] jepsen worker 50 - jepsen.util 5720 :invoke :cas [16 [21 37]] INFO [2019-12-17 12:30:53,702] jepsen worker 63 - jepsen.util 3393 :ok :read [21 4] INFO [2019-12-17 12:30:53,703] jepsen worker 63 - jepsen.util 3393 :invoke :write [21 28] INFO [2019-12-17 12:30:53,715] jepsen worker 89 - jepsen.util 3959 :ok :read [29 20] INFO [2019-12-17 12:30:53,716] jepsen worker 89 - jepsen.util 3959 :invoke :cas [29 [18 26]] INFO [2019-12-17 12:30:53,724] jepsen worker 67 - jepsen.util 5647 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:30:53,724] jepsen worker 67 - jepsen.util 5647 :invoke :write [22 13] INFO [2019-12-17 12:30:53,739] jepsen worker 35 - jepsen.util 5345 :info :cas [11 [32 25]] :RequestCanceledException INFO [2019-12-17 12:30:53,740] jepsen worker 35 - jepsen.util 5435 :invoke :read [11 nil] INFO [2019-12-17 12:30:53,746] jepsen worker 63 - jepsen.util 3393 :ok :write [21 28] INFO [2019-12-17 12:30:53,746] jepsen worker 63 - jepsen.util 3393 :invoke :read [21 nil] INFO [2019-12-17 12:30:53,755] jepsen worker 0 - jepsen.util 5940 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:30:53,755] jepsen worker 58 - jepsen.util 5458 :info :cas [19 [2 20]] :RequestCanceledException INFO [2019-12-17 12:30:53,756] jepsen worker 0 - jepsen.util 5940 :invoke :write [0 1] INFO [2019-12-17 12:30:53,757] jepsen worker 58 - jepsen.util 5548 :invoke :write [19 20] INFO [2019-12-17 12:30:53,760] jepsen worker 63 - jepsen.util 3393 :ok :read [21 28] INFO [2019-12-17 12:30:53,760] jepsen worker 63 - jepsen.util 3393 :invoke :write [21 29] INFO [2019-12-17 12:30:53,770] jepsen worker 88 - jepsen.util 4228 :fail :cas [29 [6 26]] :ValueNotSwapFrom INFO [2019-12-17 12:30:53,770] jepsen worker 88 - jepsen.util 4228 :invoke :read [29 nil] INFO [2019-12-17 12:30:53,783] jepsen worker 89 - jepsen.util 3959 :fail :cas [29 [18 26]] :ValueNotSwapFrom INFO [2019-12-17 12:30:53,783] jepsen worker 89 - jepsen.util 3959 :invoke :cas [29 [48 42]] INFO [2019-12-17 12:30:53,789] jepsen worker 40 - jepsen.util 5890 :info :write [13 17] :RequestCanceledException INFO [2019-12-17 12:30:53,789] jepsen worker 26 - jepsen.util 5516 :info :write [8 34] :RequestCanceledException INFO [2019-12-17 12:30:53,790] jepsen worker 40 - jepsen.util 5980 :invoke :cas [13 [34 15]] INFO [2019-12-17 12:30:53,790] jepsen worker 26 - jepsen.util 5606 :invoke :read [8 nil] INFO [2019-12-17 12:30:53,803] jepsen worker 63 - jepsen.util 3393 :ok :write [21 29] INFO [2019-12-17 12:30:53,803] jepsen worker 63 - jepsen.util 3393 :invoke :read [21 nil] INFO [2019-12-17 12:30:53,809] jepsen worker 57 - jepsen.util 5097 :info :cas [19 [48 35]] :RequestCanceledException INFO [2019-12-17 12:30:53,809] jepsen worker 2 - jepsen.util 5042 :info :write [0 43] :RequestCanceledException INFO [2019-12-17 12:30:53,809] jepsen worker 48 - jepsen.util 4818 :info :cas [16 [25 9]] :RequestCanceledException INFO [2019-12-17 12:30:53,810] jepsen worker 2 - jepsen.util 5132 :invoke :cas [0 [31 1]] INFO [2019-12-17 12:30:53,810] jepsen worker 57 - jepsen.util 5187 :invoke :write [19 29] INFO [2019-12-17 12:30:53,810] jepsen worker 48 - jepsen.util 4908 :invoke :write [16 13] INFO [2019-12-17 12:30:53,816] jepsen worker 63 - jepsen.util 3393 :ok :read [21 29] INFO [2019-12-17 12:30:53,816] jepsen worker 63 - jepsen.util 3393 :invoke :read [21 nil] INFO [2019-12-17 12:30:53,824] jepsen worker 38 - jepsen.util 3998 :info :write [12 20] :RequestCanceledException INFO [2019-12-17 12:30:53,824] jepsen worker 13 - jepsen.util 3793 :info :write [4 27] :RequestCanceledException INFO [2019-12-17 12:30:53,824] jepsen worker 18 - jepsen.util 4158 :fail :read [6 nil] :RequestTimeoutException INFO [2019-12-17 12:30:53,824] jepsen worker 18 - jepsen.util 4158 :invoke :cas [6 [34 7]] INFO [2019-12-17 12:30:53,825] jepsen worker 60 - jepsen.util 5820 :info :write [20 38] :RequestCanceledException INFO [2019-12-17 12:30:53,825] jepsen worker 38 - jepsen.util 4088 :invoke :cas [12 [13 41]] INFO [2019-12-17 12:30:53,825] jepsen worker 13 - jepsen.util 3883 :invoke :cas [4 [27 5]] INFO [2019-12-17 12:30:53,826] jepsen worker 60 - jepsen.util 5910 :invoke :write [20 18] INFO [2019-12-17 12:30:53,830] jepsen worker 63 - jepsen.util 3393 :ok :read [21 29] INFO [2019-12-17 12:30:53,830] jepsen worker 63 - jepsen.util 3393 :invoke :read [21 nil] INFO [2019-12-17 12:30:53,838] jepsen worker 22 - jepsen.util 5332 :info :cas [7 [22 12]] :RequestCanceledException INFO [2019-12-17 12:30:53,839] jepsen worker 22 - jepsen.util 5422 :invoke :cas [7 [44 1]] INFO [2019-12-17 12:30:53,856] jepsen worker 70 - jepsen.util 4660 :info :write [23 45] :RequestCanceledException INFO [2019-12-17 12:30:53,856] jepsen worker 13 - jepsen.util 3883 :fail :cas [4 [27 5]] :ValueNotSwapFrom INFO [2019-12-17 12:30:53,856] jepsen worker 13 - jepsen.util 3883 :invoke :cas [4 [8 35]] INFO [2019-12-17 12:30:53,857] jepsen worker 70 - jepsen.util 4750 :invoke :write [23 4] INFO [2019-12-17 12:30:53,865] jepsen worker 63 - jepsen.util 3393 :ok :read [21 29] INFO [2019-12-17 12:30:53,865] jepsen worker 63 - jepsen.util 3393 :invoke :read [21 nil] INFO [2019-12-17 12:30:53,878] jepsen worker 63 - jepsen.util 3393 :ok :read [21 29] INFO [2019-12-17 12:30:53,878] jepsen worker 63 - jepsen.util 3393 :invoke :write [21 38] INFO [2019-12-17 12:30:53,890] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":23310,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:30:53,890] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":23310,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:30:53,891] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":23313,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:30:53,891] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":23313,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:30:53,915] jepsen worker 63 - jepsen.util 3393 :ok :write [21 38] INFO [2019-12-17 12:30:53,915] jepsen worker 63 - jepsen.util 3393 :invoke :cas [21 [11 43]] INFO [2019-12-17 12:30:53,923] jepsen worker 33 - jepsen.util 5523 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:30:53,923] jepsen worker 17 - jepsen.util 5057 :info :cas [5 [38 29]] :RequestCanceledException INFO [2019-12-17 12:30:53,924] jepsen worker 33 - jepsen.util 5523 :invoke :read [11 nil] INFO [2019-12-17 12:30:53,924] jepsen worker 17 - jepsen.util 5147 :invoke :cas [5 [34 43]] INFO [2019-12-17 12:30:53,929] jepsen worker 63 - jepsen.util 3393 :fail :cas [21 [11 43]] :ValueNotSwapFrom INFO [2019-12-17 12:30:53,929] jepsen worker 63 - jepsen.util 3393 :invoke :read [21 nil] INFO [2019-12-17 12:30:53,943] jepsen worker 63 - jepsen.util 3393 :ok :read [21 38] INFO [2019-12-17 12:30:53,943] jepsen worker 63 - jepsen.util 3393 :invoke :read [21 nil] INFO [2019-12-17 12:30:53,952] jepsen worker 63 - jepsen.util 3393 :ok :read [21 38] INFO [2019-12-17 12:30:53,952] jepsen worker 63 - jepsen.util 3393 :invoke :cas [21 [32 40]] INFO [2019-12-17 12:30:53,963] jepsen worker 63 - jepsen.util 3393 :fail :cas [21 [32 40]] :ValueNotSwapFrom INFO [2019-12-17 12:30:53,964] jepsen worker 63 - jepsen.util 3393 :invoke :cas [21 [17 17]] INFO [2019-12-17 12:30:53,976] jepsen worker 3 - jepsen.util 4323 :ok :read [1 48] INFO [2019-12-17 12:30:53,976] jepsen worker 3 - jepsen.util 4323 :invoke :cas [1 [43 33]] INFO [2019-12-17 12:30:53,978] jepsen worker 63 - jepsen.util 3393 :fail :cas [21 [17 17]] :ValueNotSwapFrom INFO [2019-12-17 12:30:53,978] jepsen worker 63 - jepsen.util 3393 :invoke :cas [21 [23 28]] INFO [2019-12-17 12:30:53,997] jepsen worker 63 - jepsen.util 3393 :fail :cas [21 [23 28]] :ValueNotSwapFrom INFO [2019-12-17 12:30:53,997] jepsen worker 63 - jepsen.util 3393 :invoke :read [21 nil] INFO [2019-12-17 12:30:53,999] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":23329,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:30:54,000] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":23329,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:30:54,000] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":23331,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:30:54,000] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":23331,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:30:54,009] jepsen worker 75 - jepsen.util 5475 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:30:54,009] jepsen worker 75 - jepsen.util 5475 :invoke :read [25 nil] INFO [2019-12-17 12:30:54,010] jepsen worker 78 - jepsen.util 4488 :fail :cas [26 [1 48]] :ValueNotSwapFrom INFO [2019-12-17 12:30:54,011] jepsen worker 78 - jepsen.util 4488 :invoke :cas [26 [2 13]] INFO [2019-12-17 12:30:54,011] jepsen worker 74 - jepsen.util 4574 :ok :write [24 38] INFO [2019-12-17 12:30:54,011] jepsen worker 74 - jepsen.util 4574 :invoke :read [24 nil] INFO [2019-12-17 12:30:54,014] jepsen worker 10 - jepsen.util 3970 :ok :write [3 48] INFO [2019-12-17 12:30:54,014] jepsen worker 10 - jepsen.util 3970 :invoke :write [3 12] INFO [2019-12-17 12:30:54,018] jepsen worker 45 - jepsen.util 4185 :ok :write [15 29] INFO [2019-12-17 12:30:54,018] jepsen worker 45 - jepsen.util 4185 :invoke :write [15 43] INFO [2019-12-17 12:30:54,022] jepsen worker 63 - jepsen.util 3393 :ok :read [21 38] INFO [2019-12-17 12:30:54,022] jepsen worker 63 - jepsen.util 3393 :invoke :read [21 nil] INFO [2019-12-17 12:30:54,022] jepsen worker 1 - jepsen.util 5041 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:30:54,023] jepsen worker 42 - jepsen.util 6162 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:30:54,023] jepsen worker 1 - jepsen.util 5041 :invoke :write [0 6] INFO [2019-12-17 12:30:54,023] jepsen worker 42 - jepsen.util 6162 :invoke :write [14 37] INFO [2019-12-17 12:30:54,023] jepsen worker 38 - jepsen.util 4088 :fail :cas [12 [13 41]] :ValueNotSwapFrom INFO [2019-12-17 12:30:54,023] jepsen worker 38 - jepsen.util 4088 :invoke :read [12 nil] INFO [2019-12-17 12:30:54,039] jepsen worker 41 - jepsen.util 5531 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:30:54,039] jepsen worker 41 - jepsen.util 5531 :invoke :cas [13 [45 11]] INFO [2019-12-17 12:30:54,048] jepsen worker 78 - jepsen.util 4488 :fail :cas [26 [2 13]] :ValueNotSwapFrom INFO [2019-12-17 12:30:54,048] jepsen worker 78 - jepsen.util 4488 :invoke :read [26 nil] INFO [2019-12-17 12:30:54,055] jepsen worker 9 - jepsen.util 4509 :info :write [3 11] :RequestCanceledException INFO [2019-12-17 12:30:54,056] jepsen worker 9 - jepsen.util 4599 :invoke :write [3 12] INFO [2019-12-17 12:30:54,077] jepsen worker 89 - jepsen.util 3959 :fail :cas [29 [48 42]] :ValueNotSwapFrom INFO [2019-12-17 12:30:54,077] jepsen worker 89 - jepsen.util 3959 :invoke :read [29 nil] INFO [2019-12-17 12:30:54,088] jepsen worker 29 - jepsen.util 3449 :ok :read [9 42] INFO [2019-12-17 12:30:54,088] jepsen worker 29 - jepsen.util 3449 :invoke :cas [9 [42 43]] INFO [2019-12-17 12:30:54,105] jepsen worker 36 - jepsen.util 4266 :ok :read [12 33] INFO [2019-12-17 12:30:54,105] jepsen worker 63 - jepsen.util 3393 :ok :read [21 38] INFO [2019-12-17 12:30:54,105] jepsen worker 36 - jepsen.util 4266 :invoke :write [12 36] INFO [2019-12-17 12:30:54,105] jepsen worker 63 - jepsen.util 3393 :invoke :cas [21 [47 21]] INFO [2019-12-17 12:30:54,105] jepsen worker 18 - jepsen.util 4158 :fail :cas [6 [34 7]] :ValueNotSwapFrom INFO [2019-12-17 12:30:54,105] jepsen worker 18 - jepsen.util 4158 :invoke :cas [6 [25 15]] INFO [2019-12-17 12:30:54,109] jepsen worker 65 - jepsen.util 3395 :ok :read [21 38] INFO [2019-12-17 12:30:54,109] jepsen worker 65 - jepsen.util 3395 :invoke :write [21 28] INFO [2019-12-17 12:30:54,124] jepsen worker 16 - jepsen.util 6136 :info :write [5 5] :RequestCanceledException INFO [2019-12-17 12:30:54,125] jepsen worker 64 - jepsen.util 3304 :fail :cas [21 [9 32]] :ValueNotSwapFrom INFO [2019-12-17 12:30:54,125] jepsen worker 64 - jepsen.util 3304 :invoke :read [21 nil] INFO [2019-12-17 12:30:54,125] jepsen worker 16 - jepsen.util 6226 :invoke :write [5 42] INFO [2019-12-17 12:30:54,136] jepsen worker 3 - jepsen.util 4323 :fail :cas [1 [43 33]] :ValueNotSwapFrom INFO [2019-12-17 12:30:54,136] jepsen worker 3 - jepsen.util 4323 :invoke :read [1 nil] INFO [2019-12-17 12:30:54,139] jepsen worker 76 - jepsen.util 5476 :info :cas [25 [7 18]] :RequestCanceledException INFO [2019-12-17 12:30:54,139] jepsen worker 73 - jepsen.util 3853 :info :write [24 22] :RequestCanceledException INFO [2019-12-17 12:30:54,140] jepsen worker 73 - jepsen.util 3943 :invoke :cas [24 [1 37]] INFO [2019-12-17 12:30:54,141] jepsen worker 76 - jepsen.util 5566 :invoke :cas [25 [37 23]] INFO [2019-12-17 12:30:54,142] jepsen worker 63 - jepsen.util 3393 :fail :cas [21 [47 21]] :ValueNotSwapFrom INFO [2019-12-17 12:30:54,143] jepsen worker 63 - jepsen.util 3393 :invoke :write [21 32] INFO [2019-12-17 12:30:54,156] jepsen worker 15 - jepsen.util 4785 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:30:54,156] jepsen worker 8 - jepsen.util 4598 :info :write [2 29] :RequestCanceledException INFO [2019-12-17 12:30:54,156] jepsen worker 15 - jepsen.util 4785 :invoke :read [5 nil] INFO [2019-12-17 12:30:54,157] jepsen worker 8 - jepsen.util 4688 :invoke :cas [2 [11 29]] INFO [2019-12-17 12:30:54,157] jepsen worker 38 - jepsen.util 4088 :ok :read [12 33] INFO [2019-12-17 12:30:54,157] jepsen worker 38 - jepsen.util 4088 :invoke :cas [12 [37 9]] INFO [2019-12-17 12:30:54,173] jepsen worker 74 - jepsen.util 4574 :ok :read [24 38] INFO [2019-12-17 12:30:54,173] jepsen worker 74 - jepsen.util 4574 :invoke :write [24 32] INFO [2019-12-17 12:30:54,177] jepsen worker 89 - jepsen.util 3959 :ok :read [29 20] INFO [2019-12-17 12:30:54,177] jepsen worker 89 - jepsen.util 3959 :invoke :cas [29 [41 45]] INFO [2019-12-17 12:30:54,182] jepsen worker 13 - jepsen.util 3883 :fail :cas [4 [8 35]] :ValueNotSwapFrom INFO [2019-12-17 12:30:54,182] jepsen worker 13 - jepsen.util 3883 :invoke :cas [4 [30 46]] INFO [2019-12-17 12:30:54,211] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":23363,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:30:54,211] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":23363,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:30:54,211] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":23364,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:30:54,211] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":23364,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:30:54,211] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":23365,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:30:54,212] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":23365,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:30:54,221] jepsen worker 73 - jepsen.util 3943 :fail :cas [24 [1 37]] :ValueNotSwapFrom INFO [2019-12-17 12:30:54,222] jepsen worker 73 - jepsen.util 3943 :invoke :read [24 nil] INFO [2019-12-17 12:30:54,237] jepsen worker 73 - jepsen.util 3943 :ok :read [24 38] INFO [2019-12-17 12:30:54,238] jepsen worker 73 - jepsen.util 3943 :invoke :read [24 nil] INFO [2019-12-17 12:30:54,251] jepsen worker 73 - jepsen.util 3943 :ok :read [24 38] INFO [2019-12-17 12:30:54,251] jepsen worker 73 - jepsen.util 3943 :invoke :cas [24 [43 1]] INFO [2019-12-17 12:30:54,255] jepsen worker 80 - jepsen.util 4220 :fail :cas [26 [5 47]] :ValueNotSwapFrom INFO [2019-12-17 12:30:54,255] jepsen worker 80 - jepsen.util 4220 :invoke :cas [26 [14 49]] INFO [2019-12-17 12:30:54,258] jepsen worker 10 - jepsen.util 3970 :ok :write [3 12] INFO [2019-12-17 12:30:54,258] jepsen worker 10 - jepsen.util 3970 :invoke :read [3 nil] INFO [2019-12-17 12:30:54,258] jepsen worker 54 - jepsen.util 4464 :fail :cas [18 [13 20]] :ValueNotSwapFrom INFO [2019-12-17 12:30:54,259] jepsen worker 54 - jepsen.util 4464 :invoke :cas [18 [1 15]] INFO [2019-12-17 12:30:54,261] jepsen worker 65 - jepsen.util 3395 :ok :write [21 28] INFO [2019-12-17 12:30:54,261] jepsen worker 65 - jepsen.util 3395 :invoke :cas [21 [18 29]] INFO [2019-12-17 12:30:54,282] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":23371,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:30:54,282] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":23371,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:30:54,282] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":23374,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:30:54,282] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":23374,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:30:54,282] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":23375,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:30:54,282] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":23375,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:30:54,291] jepsen worker 89 - jepsen.util 3959 :fail :cas [29 [41 45]] :ValueNotSwapFrom INFO [2019-12-17 12:30:54,291] jepsen worker 89 - jepsen.util 3959 :invoke :read [29 nil] INFO [2019-12-17 12:30:54,294] jepsen worker 73 - jepsen.util 3943 :fail :cas [24 [43 1]] :ValueNotSwapFrom INFO [2019-12-17 12:30:54,294] jepsen worker 73 - jepsen.util 3943 :invoke :write [24 10] INFO [2019-12-17 12:30:54,298] jepsen worker 78 - jepsen.util 4488 :ok :read [26 30] INFO [2019-12-17 12:30:54,298] jepsen worker 78 - jepsen.util 4488 :invoke :cas [26 [9 43]] INFO [2019-12-17 12:30:54,309] jepsen worker 14 - jepsen.util 4064 :info :write [4 34] :RequestCanceledException INFO [2019-12-17 12:30:54,309] jepsen worker 12 - jepsen.util 4422 :info :write [4 31] :RequestCanceledException INFO [2019-12-17 12:30:54,309] jepsen worker 14 - jepsen.util 4154 :invoke :write [4 48] INFO [2019-12-17 12:30:54,312] jepsen worker 12 - jepsen.util 4512 :invoke :write [4 32] INFO [2019-12-17 12:30:54,325] jepsen worker 81 - jepsen.util 5301 :info :write [27 13] :RequestCanceledException INFO [2019-12-17 12:30:54,326] jepsen worker 81 - jepsen.util 5391 :invoke :read [27 nil] INFO [2019-12-17 12:30:54,339] jepsen worker 87 - jepsen.util 3777 :info :write [29 24] :RequestCanceledException INFO [2019-12-17 12:30:54,339] jepsen worker 21 - jepsen.util 5421 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:30:54,339] jepsen worker 21 - jepsen.util 5421 :invoke :write [7 49] INFO [2019-12-17 12:30:54,340] jepsen worker 87 - jepsen.util 3867 :invoke :write [29 30] INFO [2019-12-17 12:30:54,341] jepsen worker 18 - jepsen.util 4158 :fail :cas [6 [25 15]] :ValueNotSwapFrom INFO [2019-12-17 12:30:54,341] jepsen worker 18 - jepsen.util 4158 :invoke :write [6 14] INFO [2019-12-17 12:30:54,355] jepsen worker 20 - jepsen.util 3980 :info :write [6 29] :RequestCanceledException INFO [2019-12-17 12:30:54,356] jepsen worker 20 - jepsen.util 4070 :invoke :read [6 nil] INFO [2019-12-17 12:30:54,363] jepsen worker 89 - jepsen.util 3959 :ok :read [29 20] INFO [2019-12-17 12:30:54,363] jepsen worker 89 - jepsen.util 3959 :invoke :read [29 nil] INFO [2019-12-17 12:30:54,364] jepsen worker 3 - jepsen.util 4323 :ok :read [1 48] INFO [2019-12-17 12:30:54,364] jepsen worker 3 - jepsen.util 4323 :invoke :read [1 nil] INFO [2019-12-17 12:30:54,365] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:30:54,365] jepsen nemesis - couchbase.util Rebalance check count: 173 INFO [2019-12-17 12:30:54,365] jepsen nemesis - couchbase.util Rebalance stuck count: 154 INFO [2019-12-17 12:30:54,394] jepsen worker 38 - jepsen.util 4088 :fail :cas [12 [37 9]] :ValueNotSwapFrom INFO [2019-12-17 12:30:54,394] jepsen worker 38 - jepsen.util 4088 :invoke :cas [12 [21 40]] INFO [2019-12-17 12:30:54,397] jepsen worker 54 - jepsen.util 4464 :fail :cas [18 [1 15]] :ValueNotSwapFrom INFO [2019-12-17 12:30:54,397] jepsen worker 54 - jepsen.util 4464 :invoke :read [18 nil] INFO [2019-12-17 12:30:54,401] jepsen worker 30 - jepsen.util 4350 :fail :read [10 nil] :RequestTimeoutException INFO [2019-12-17 12:30:54,402] jepsen worker 30 - jepsen.util 4350 :invoke :write [10 5] INFO [2019-12-17 12:30:54,414] jepsen worker 88 - jepsen.util 4228 :ok :read [29 20] INFO [2019-12-17 12:30:54,414] jepsen worker 88 - jepsen.util 4228 :invoke :read [29 nil] INFO [2019-12-17 12:30:54,417] jepsen worker 10 - jepsen.util 3970 :ok :read [3 12] INFO [2019-12-17 12:30:54,417] jepsen worker 10 - jepsen.util 3970 :invoke :cas [3 [48 17]] INFO [2019-12-17 12:30:54,424] jepsen worker 46 - jepsen.util 4096 :info :cas [15 [24 35]] :RequestCanceledException INFO [2019-12-17 12:30:54,424] jepsen worker 11 - jepsen.util 4421 :fail :read [3 nil] :RequestTimeoutException INFO [2019-12-17 12:30:54,424] jepsen worker 11 - jepsen.util 4421 :invoke :write [3 23] INFO [2019-12-17 12:30:54,425] jepsen worker 46 - jepsen.util 4186 :invoke :write [15 49] INFO [2019-12-17 12:30:54,428] jepsen worker 29 - jepsen.util 3449 :ok :cas [9 [42 43]] INFO [2019-12-17 12:30:54,428] jepsen worker 29 - jepsen.util 3449 :invoke :write [9 29] INFO [2019-12-17 12:30:54,448] jepsen worker 78 - jepsen.util 4488 :fail :cas [26 [9 43]] :ValueNotSwapFrom INFO [2019-12-17 12:30:54,448] jepsen worker 78 - jepsen.util 4488 :invoke :read [26 nil] INFO [2019-12-17 12:30:54,456] jepsen worker 72 - jepsen.util 4212 :info :cas [24 [15 48]] :RequestCanceledException INFO [2019-12-17 12:30:54,456] jepsen worker 72 - jepsen.util 4302 :invoke :read [24 nil] INFO [2019-12-17 12:30:54,485] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":23401,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:30:54,485] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":23401,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:30:54,485] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":23404,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:30:54,485] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":23404,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:30:54,485] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":23405,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:30:54,485] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":23405,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:30:54,492] jepsen worker 51 - jepsen.util 3741 :info :write [17 20] :RequestCanceledException INFO [2019-12-17 12:30:54,493] jepsen worker 51 - jepsen.util 3831 :invoke :cas [17 [35 44]] INFO [2019-12-17 12:30:54,494] jepsen worker 38 - jepsen.util 4088 :fail :cas [12 [21 40]] :ValueNotSwapFrom INFO [2019-12-17 12:30:54,494] jepsen worker 38 - jepsen.util 4088 :invoke :write [12 40] INFO [2019-12-17 12:30:54,522] jepsen worker 80 - jepsen.util 4220 :fail :cas [26 [14 49]] :ValueNotSwapFrom INFO [2019-12-17 12:30:54,522] jepsen worker 80 - jepsen.util 4220 :invoke :cas [26 [48 38]] INFO [2019-12-17 12:30:54,525] jepsen worker 61 - jepsen.util 6091 :info :write [20 24] :RequestCanceledException INFO [2019-12-17 12:30:54,525] jepsen worker 28 - jepsen.util 2908 :info :write [9 33] :RequestCanceledException INFO [2019-12-17 12:30:54,526] jepsen worker 61 - jepsen.util 6181 :invoke :read [20 nil] INFO [2019-12-17 12:30:54,526] jepsen worker 28 - jepsen.util 2998 :invoke :read [9 nil] INFO [2019-12-17 12:30:54,591] jepsen worker 31 - jepsen.util 3721 :info :cas [10 [23 37]] :RequestCanceledException INFO [2019-12-17 12:30:54,592] jepsen worker 31 - jepsen.util 3811 :invoke :write [10 35] INFO [2019-12-17 12:30:54,624] jepsen worker 71 - jepsen.util 4301 :info :write [23 31] :RequestCanceledException INFO [2019-12-17 12:30:54,625] jepsen worker 71 - jepsen.util 4391 :invoke :read [23 nil] INFO [2019-12-17 12:30:54,638] jepsen worker 56 - jepsen.util 4196 :info :write [18 33] :RequestCanceledException INFO [2019-12-17 12:30:54,639] jepsen worker 56 - jepsen.util 4286 :invoke :write [18 39] INFO [2019-12-17 12:30:54,646] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":23419,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:30:54,646] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":23419,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:30:54,646] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":23423,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:30:54,646] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":23423,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:30:54,670] jepsen worker 13 - jepsen.util 3883 :fail :cas [4 [30 46]] :ValueNotSwapFrom INFO [2019-12-17 12:30:54,670] jepsen worker 13 - jepsen.util 3883 :invoke :cas [4 [31 29]] INFO [2019-12-17 12:30:54,687] jepsen worker 89 - jepsen.util 3959 :ok :read [29 20] INFO [2019-12-17 12:30:54,687] jepsen worker 89 - jepsen.util 3959 :invoke :read [29 nil] INFO [2019-12-17 12:30:54,694] jepsen worker 3 - jepsen.util 4323 :ok :read [1 48] INFO [2019-12-17 12:30:54,694] jepsen worker 3 - jepsen.util 4323 :invoke :cas [1 [5 15]] INFO [2019-12-17 12:30:54,709] jepsen worker 13 - jepsen.util 3883 :fail :cas [4 [31 29]] :ValueNotSwapFrom INFO [2019-12-17 12:30:54,709] jepsen worker 13 - jepsen.util 3883 :invoke :read [4 nil] INFO [2019-12-17 12:30:54,724] jepsen worker 77 - jepsen.util 5387 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:30:54,724] jepsen worker 5 - jepsen.util 3695 :info :write [1 32] :RequestCanceledException INFO [2019-12-17 12:30:54,725] jepsen worker 77 - jepsen.util 5387 :invoke :cas [25 [30 20]] INFO [2019-12-17 12:30:54,725] jepsen worker 5 - jepsen.util 3785 :invoke :cas [1 [41 48]] INFO [2019-12-17 12:30:54,755] jepsen worker 53 - jepsen.util 4013 :info :write [17 39] :RequestCanceledException INFO [2019-12-17 12:30:54,755] jepsen worker 24 - jepsen.util 5244 :info :cas [8 [11 22]] :RequestCanceledException INFO [2019-12-17 12:30:54,756] jepsen worker 24 - jepsen.util 5334 :invoke :read [8 nil] INFO [2019-12-17 12:30:54,756] jepsen worker 53 - jepsen.util 4103 :invoke :write [17 24] INFO [2019-12-17 12:30:54,758] jepsen worker 80 - jepsen.util 4220 :fail :cas [26 [48 38]] :ValueNotSwapFrom INFO [2019-12-17 12:30:54,758] jepsen worker 80 - jepsen.util 4220 :invoke :write [26 0] INFO [2019-12-17 12:30:54,790] jepsen worker 86 - jepsen.util 5576 :info :cas [28 [47 11]] :RequestCanceledException INFO [2019-12-17 12:30:54,791] jepsen worker 86 - jepsen.util 5666 :invoke :cas [28 [6 16]] INFO [2019-12-17 12:30:54,803] jepsen worker 78 - jepsen.util 4488 :ok :read [26 30] INFO [2019-12-17 12:30:54,803] jepsen worker 78 - jepsen.util 4488 :invoke :write [26 45] INFO [2019-12-17 12:30:54,809] jepsen worker 13 - jepsen.util 3883 :ok :read [4 33] INFO [2019-12-17 12:30:54,809] jepsen worker 13 - jepsen.util 3883 :invoke :read [4 nil] INFO [2019-12-17 12:30:54,824] jepsen worker 19 - jepsen.util 4159 :info :cas [6 [37 29]] :RequestCanceledException INFO [2019-12-17 12:30:54,824] jepsen worker 49 - jepsen.util 4819 :info :write [16 26] :RequestCanceledException INFO [2019-12-17 12:30:54,825] jepsen worker 49 - jepsen.util 4909 :invoke :read [16 nil] INFO [2019-12-17 12:30:54,825] jepsen worker 19 - jepsen.util 4249 :invoke :write [6 36] INFO [2019-12-17 12:30:54,829] jepsen worker 89 - jepsen.util 3959 :ok :read [29 20] INFO [2019-12-17 12:30:54,829] jepsen worker 89 - jepsen.util 3959 :invoke :read [29 nil] INFO [2019-12-17 12:30:54,865] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":23443,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:30:54,865] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":23443,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:30:54,865] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":23445,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:30:54,865] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":23445,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:30:54,876] jepsen worker 13 - jepsen.util 3883 :ok :read [4 33] INFO [2019-12-17 12:30:54,876] jepsen worker 13 - jepsen.util 3883 :invoke :cas [4 [3 38]] INFO [2019-12-17 12:30:54,890] jepsen worker 51 - jepsen.util 3831 :fail :cas [17 [35 44]] :ValueNotSwapFrom INFO [2019-12-17 12:30:54,890] jepsen worker 51 - jepsen.util 3831 :invoke :cas [17 [7 9]] INFO [2019-12-17 12:30:54,900] jepsen worker 13 - jepsen.util 3883 :fail :cas [4 [3 38]] :ValueNotSwapFrom INFO [2019-12-17 12:30:54,901] jepsen worker 13 - jepsen.util 3883 :invoke :write [4 32] INFO [2019-12-17 12:30:54,909] jepsen worker 32 - jepsen.util 4532 :info :cas [10 [28 9]] :RequestCanceledException INFO [2019-12-17 12:30:54,910] jepsen worker 32 - jepsen.util 4622 :invoke :write [10 36] INFO [2019-12-17 12:30:54,925] jepsen worker 27 - jepsen.util 3897 :info :write [9 9] :RequestCanceledException INFO [2019-12-17 12:30:54,925] jepsen worker 52 - jepsen.util 4282 :info :write [17 39] :RequestCanceledException INFO [2019-12-17 12:30:54,925] jepsen worker 52 - jepsen.util 4372 :invoke :cas [17 [11 31]] INFO [2019-12-17 12:30:54,925] jepsen worker 27 - jepsen.util 3987 :invoke :write [9 29] INFO [2019-12-17 12:30:54,991] jepsen worker 4 - jepsen.util 3244 :info :write [1 25] :RequestCanceledException INFO [2019-12-17 12:30:54,991] jepsen worker 54 - jepsen.util 4464 :ok :read [18 36] INFO [2019-12-17 12:30:54,992] jepsen worker 4 - jepsen.util 3334 :invoke :cas [1 [48 8]] INFO [2019-12-17 12:30:54,993] jepsen worker 54 - jepsen.util 4464 :invoke :read [18 nil] INFO [2019-12-17 12:30:54,994] jepsen worker 73 - jepsen.util 3943 :ok :write [24 10] INFO [2019-12-17 12:30:54,994] jepsen worker 73 - jepsen.util 3943 :invoke :read [24 nil] INFO [2019-12-17 12:30:55,009] jepsen worker 34 - jepsen.util 5074 :info :cas [11 [39 17]] :RequestCanceledException INFO [2019-12-17 12:30:55,009] jepsen worker 34 - jepsen.util 5164 :invoke :read [11 nil] INFO [2019-12-17 12:30:55,022] jepsen worker 28 - jepsen.util 2998 :ok :read [9 43] INFO [2019-12-17 12:30:55,022] jepsen worker 28 - jepsen.util 2998 :invoke :write [9 35] INFO [2019-12-17 12:30:55,047] jepsen worker 27 - jepsen.util 3987 :ok :write [9 29] INFO [2019-12-17 12:30:55,047] jepsen worker 27 - jepsen.util 3987 :invoke :cas [9 [21 12]] INFO [2019-12-17 12:30:55,060] jepsen worker 3 - jepsen.util 4323 :fail :cas [1 [5 15]] :ValueNotSwapFrom INFO [2019-12-17 12:30:55,060] jepsen worker 3 - jepsen.util 4323 :invoke :read [1 nil] INFO [2019-12-17 12:30:55,078] jepsen worker 18 - jepsen.util 4158 :ok :write [6 14] INFO [2019-12-17 12:30:55,078] jepsen worker 30 - jepsen.util 4350 :ok :write [10 5] INFO [2019-12-17 12:30:55,078] jepsen worker 18 - jepsen.util 4158 :invoke :read [6 nil] INFO [2019-12-17 12:30:55,078] jepsen worker 30 - jepsen.util 4350 :invoke :cas [10 [44 14]] INFO [2019-12-17 12:30:55,078] jepsen worker 45 - jepsen.util 4185 :ok :write [15 43] INFO [2019-12-17 12:30:55,079] jepsen worker 45 - jepsen.util 4185 :invoke :cas [15 [40 12]] INFO [2019-12-17 12:30:55,079] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":23453,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:30:55,079] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":23453,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:30:55,080] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":23456,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:30:55,080] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":23456,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:30:55,080] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":23458,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:30:55,080] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":23458,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:30:55,084] jepsen worker 88 - jepsen.util 4228 :ok :read [29 20] INFO [2019-12-17 12:30:55,084] jepsen worker 88 - jepsen.util 4228 :invoke :write [29 47] INFO [2019-12-17 12:30:55,091] jepsen worker 68 - jepsen.util 5018 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:30:55,091] jepsen worker 68 - jepsen.util 5018 :invoke :read [22 nil] INFO [2019-12-17 12:30:55,113] jepsen worker 28 - jepsen.util 2998 :ok :write [9 35] INFO [2019-12-17 12:30:55,113] jepsen worker 28 - jepsen.util 2998 :invoke :cas [9 [10 33]] INFO [2019-12-17 12:30:55,139] jepsen worker 20 - jepsen.util 4070 :ok :read [6 14] INFO [2019-12-17 12:30:55,139] jepsen worker 20 - jepsen.util 4070 :invoke :cas [6 [21 24]] INFO [2019-12-17 12:30:55,140] jepsen worker 10 - jepsen.util 3970 :fail :cas [3 [48 17]] :ValueNotSwapFrom INFO [2019-12-17 12:30:55,140] jepsen worker 10 - jepsen.util 3970 :invoke :read [3 nil] INFO [2019-12-17 12:30:55,151] jepsen worker 89 - jepsen.util 3959 :ok :read [29 20] INFO [2019-12-17 12:30:55,151] jepsen worker 89 - jepsen.util 3959 :invoke :write [29 30] INFO [2019-12-17 12:30:55,155] jepsen worker 28 - jepsen.util 2998 :fail :cas [9 [10 33]] :ValueNotSwapFrom INFO [2019-12-17 12:30:55,156] jepsen worker 28 - jepsen.util 2998 :invoke :cas [9 [10 1]] INFO [2019-12-17 12:30:55,171] jepsen worker 28 - jepsen.util 2998 :fail :cas [9 [10 1]] :ValueNotSwapFrom INFO [2019-12-17 12:30:55,171] jepsen worker 28 - jepsen.util 2998 :invoke :cas [9 [7 13]] INFO [2019-12-17 12:30:55,181] jepsen worker 71 - jepsen.util 4391 :ok :read [23 45] INFO [2019-12-17 12:30:55,181] jepsen worker 71 - jepsen.util 4391 :invoke :write [23 12] INFO [2019-12-17 12:30:55,187] jepsen worker 28 - jepsen.util 2998 :fail :cas [9 [7 13]] :ValueNotSwapFrom INFO [2019-12-17 12:30:55,187] jepsen worker 28 - jepsen.util 2998 :invoke :read [9 nil] INFO [2019-12-17 12:30:55,200] jepsen worker 28 - jepsen.util 2998 :ok :read [9 35] INFO [2019-12-17 12:30:55,200] jepsen worker 28 - jepsen.util 2998 :invoke :write [9 16] INFO [2019-12-17 12:30:55,238] jepsen worker 79 - jepsen.util 4039 :info :write [26 10] :RequestCanceledException INFO [2019-12-17 12:30:55,239] jepsen worker 79 - jepsen.util 4129 :invoke :read [26 nil] INFO [2019-12-17 12:30:55,249] jepsen worker 28 - jepsen.util 2998 :ok :write [9 16] INFO [2019-12-17 12:30:55,249] jepsen worker 28 - jepsen.util 2998 :invoke :cas [9 [34 45]] INFO [2019-12-17 12:30:55,256] jepsen worker 23 - jepsen.util 5243 :info :write [7 5] :RequestCanceledException INFO [2019-12-17 12:30:55,257] jepsen worker 23 - jepsen.util 5333 :invoke :cas [7 [3 19]] INFO [2019-12-17 12:30:55,258] jepsen worker 19 - jepsen.util 4249 :ok :write [6 36] INFO [2019-12-17 12:30:55,259] jepsen worker 19 - jepsen.util 4249 :invoke :write [6 26] INFO [2019-12-17 12:30:55,268] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":23489,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:30:55,268] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":23489,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:30:55,268] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":23491,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:30:55,268] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":23491,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:30:55,283] jepsen worker 52 - jepsen.util 4372 :fail :cas [17 [11 31]] :ValueNotSwapFrom INFO [2019-12-17 12:30:55,283] jepsen worker 52 - jepsen.util 4372 :invoke :write [17 41] INFO [2019-12-17 12:30:55,298] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":23479,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:30:55,299] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":23479,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:30:55,299] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":23480,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:30:55,299] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":23480,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:30:55,320] jepsen worker 30 - jepsen.util 4350 :fail :cas [10 [44 14]] :ValueNotSwapFrom INFO [2019-12-17 12:30:55,320] jepsen worker 30 - jepsen.util 4350 :invoke :write [10 44] INFO [2019-12-17 12:30:55,325] jepsen worker 44 - jepsen.util 5804 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:30:55,325] jepsen worker 59 - jepsen.util 5459 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:30:55,325] jepsen worker 59 - jepsen.util 5459 :invoke :read [19 nil] INFO [2019-12-17 12:30:55,325] jepsen worker 44 - jepsen.util 5804 :invoke :cas [14 [48 37]] INFO [2019-12-17 12:30:55,334] jepsen worker 5 - jepsen.util 3785 :fail :cas [1 [41 48]] :ValueNotSwapFrom INFO [2019-12-17 12:30:55,334] jepsen worker 5 - jepsen.util 3785 :invoke :read [1 nil] INFO [2019-12-17 12:30:55,337] jepsen worker 11 - jepsen.util 4421 :ok :write [3 23] INFO [2019-12-17 12:30:55,337] jepsen worker 11 - jepsen.util 4421 :invoke :cas [3 [2 8]] INFO [2019-12-17 12:30:55,341] jepsen worker 28 - jepsen.util 2998 :fail :cas [9 [34 45]] :ValueNotSwapFrom INFO [2019-12-17 12:30:55,341] jepsen worker 28 - jepsen.util 2998 :invoke :write [9 29] INFO [2019-12-17 12:30:55,356] jepsen worker 43 - jepsen.util 4993 :info :cas [14 [13 24]] :RequestCanceledException INFO [2019-12-17 12:30:55,357] jepsen worker 43 - jepsen.util 5083 :invoke :read [14 nil] INFO [2019-12-17 12:30:55,371] jepsen worker 10 - jepsen.util 3970 :ok :read [3 23] INFO [2019-12-17 12:30:55,371] jepsen worker 10 - jepsen.util 3970 :invoke :read [3 nil] INFO [2019-12-17 12:30:55,391] jepsen worker 69 - jepsen.util 4119 :info :cas [23 [45 23]] :RequestCanceledException INFO [2019-12-17 12:30:55,391] jepsen worker 66 - jepsen.util 5286 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:30:55,392] jepsen worker 66 - jepsen.util 5286 :invoke :read [22 nil] INFO [2019-12-17 12:30:55,392] jepsen worker 69 - jepsen.util 4209 :invoke :write [23 46] INFO [2019-12-17 12:30:55,396] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:30:55,397] jepsen nemesis - couchbase.util Rebalance check count: 174 INFO [2019-12-17 12:30:55,397] jepsen nemesis - couchbase.util Rebalance stuck count: 155 INFO [2019-12-17 12:30:55,403] jepsen worker 11 - jepsen.util 4421 :fail :cas [3 [2 8]] :ValueNotSwapFrom INFO [2019-12-17 12:30:55,403] jepsen worker 11 - jepsen.util 4421 :invoke :read [3 nil] INFO [2019-12-17 12:30:55,411] jepsen worker 31 - jepsen.util 3811 :ok :write [10 35] INFO [2019-12-17 12:30:55,411] jepsen worker 31 - jepsen.util 3811 :invoke :write [10 28] INFO [2019-12-17 12:30:55,414] jepsen worker 10 - jepsen.util 3970 :ok :read [3 23] INFO [2019-12-17 12:30:55,414] jepsen worker 10 - jepsen.util 3970 :invoke :cas [3 [26 39]] INFO [2019-12-17 12:30:55,421] jepsen worker 3 - jepsen.util 4323 :ok :read [1 48] INFO [2019-12-17 12:30:55,421] jepsen worker 3 - jepsen.util 4323 :invoke :cas [1 [4 37]] INFO [2019-12-17 12:30:55,425] jepsen worker 6 - jepsen.util 5946 :info :write [2 38] :RequestCanceledException INFO [2019-12-17 12:30:55,426] jepsen worker 6 - jepsen.util 6036 :invoke :write [2 5] INFO [2019-12-17 12:30:55,437] jepsen worker 84 - jepsen.util 4944 :info :cas [28 [15 35]] :RequestCanceledException INFO [2019-12-17 12:30:55,438] jepsen worker 84 - jepsen.util 5034 :invoke :read [28 nil] INFO [2019-12-17 12:30:55,440] jepsen worker 11 - jepsen.util 4421 :ok :read [3 23] INFO [2019-12-17 12:30:55,441] jepsen worker 11 - jepsen.util 4421 :invoke :cas [3 [0 7]] INFO [2019-12-17 12:30:55,467] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":23512,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:30:55,467] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":23512,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:30:55,467] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":23515,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:30:55,467] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":23515,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:30:55,485] jepsen worker 45 - jepsen.util 4185 :fail :cas [15 [40 12]] :ValueNotSwapFrom INFO [2019-12-17 12:30:55,485] jepsen worker 45 - jepsen.util 4185 :invoke :write [15 5] INFO [2019-12-17 12:30:55,490] jepsen worker 10 - jepsen.util 3970 :fail :cas [3 [26 39]] :ValueNotSwapFrom INFO [2019-12-17 12:30:55,490] jepsen worker 10 - jepsen.util 3970 :invoke :cas [3 [3 19]] INFO [2019-12-17 12:30:55,491] jepsen worker 25 - jepsen.util 5605 :info :write [8 25] :RequestCanceledException INFO [2019-12-17 12:30:55,492] jepsen worker 25 - jepsen.util 5695 :invoke :read [8 nil] INFO [2019-12-17 12:30:55,501] jepsen worker 20 - jepsen.util 4070 :fail :cas [6 [21 24]] :ValueNotSwapFrom INFO [2019-12-17 12:30:55,501] jepsen worker 20 - jepsen.util 4070 :invoke :cas [6 [20 31]] INFO [2019-12-17 12:30:55,518] jepsen worker 11 - jepsen.util 4421 :fail :cas [3 [0 7]] :ValueNotSwapFrom INFO [2019-12-17 12:30:55,518] jepsen worker 11 - jepsen.util 4421 :invoke :write [3 7] INFO [2019-12-17 12:30:55,520] jepsen worker 28 - jepsen.util 2998 :ok :write [9 29] INFO [2019-12-17 12:30:55,520] jepsen worker 28 - jepsen.util 2998 :invoke :cas [9 [16 26]] INFO [2019-12-17 12:30:55,524] jepsen worker 85 - jepsen.util 6025 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:30:55,524] jepsen worker 62 - jepsen.util 5012 :info :write [20 3] :RequestCanceledException INFO [2019-12-17 12:30:55,524] jepsen worker 85 - jepsen.util 6025 :invoke :write [28 15] INFO [2019-12-17 12:30:55,526] jepsen worker 10 - jepsen.util 3970 :fail :cas [3 [3 19]] :ValueNotSwapFrom INFO [2019-12-17 12:30:55,526] jepsen worker 62 - jepsen.util 5102 :invoke :read [20 nil] INFO [2019-12-17 12:30:55,526] jepsen worker 10 - jepsen.util 3970 :invoke :write [3 42] INFO [2019-12-17 12:30:55,538] jepsen worker 39 - jepsen.util 4989 :info :write [13 29] :RequestCanceledException INFO [2019-12-17 12:30:55,538] jepsen worker 82 - jepsen.util 6382 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:30:55,538] jepsen worker 82 - jepsen.util 6382 :invoke :read [27 nil] INFO [2019-12-17 12:30:55,539] jepsen worker 39 - jepsen.util 5079 :invoke :read [13 nil] INFO [2019-12-17 12:30:55,542] jepsen worker 54 - jepsen.util 4464 :ok :read [18 36] INFO [2019-12-17 12:30:55,543] jepsen worker 54 - jepsen.util 4464 :invoke :read [18 nil] INFO [2019-12-17 12:30:55,546] jepsen worker 28 - jepsen.util 2998 :fail :cas [9 [16 26]] :ValueNotSwapFrom INFO [2019-12-17 12:30:55,546] jepsen worker 28 - jepsen.util 2998 :invoke :cas [9 [36 23]] INFO [2019-12-17 12:30:55,556] jepsen worker 47 - jepsen.util 4547 :info :write [15 22] :RequestCanceledException INFO [2019-12-17 12:30:55,557] jepsen worker 47 - jepsen.util 4637 :invoke :write [15 48] INFO [2019-12-17 12:30:55,568] jepsen worker 28 - jepsen.util 2998 :fail :cas [9 [36 23]] :ValueNotSwapFrom INFO [2019-12-17 12:30:55,569] jepsen worker 28 - jepsen.util 2998 :invoke :read [9 nil] INFO [2019-12-17 12:30:55,585] jepsen worker 19 - jepsen.util 4249 :ok :write [6 26] INFO [2019-12-17 12:30:55,585] jepsen worker 19 - jepsen.util 4249 :invoke :read [6 nil] INFO [2019-12-17 12:30:55,590] jepsen worker 37 - jepsen.util 3817 :info :write [12 29] :RequestCanceledException INFO [2019-12-17 12:30:55,590] jepsen worker 55 - jepsen.util 5005 :info :write [18 33] :RequestCanceledException INFO [2019-12-17 12:30:55,592] jepsen worker 55 - jepsen.util 5095 :invoke :cas [18 [49 40]] INFO [2019-12-17 12:30:55,592] jepsen worker 37 - jepsen.util 3907 :invoke :read [12 nil] INFO [2019-12-17 12:30:55,631] jepsen worker 55 - jepsen.util 5095 :fail :cas [18 [49 40]] :ValueNotSwapFrom INFO [2019-12-17 12:30:55,631] jepsen worker 55 - jepsen.util 5095 :invoke :cas [18 [16 35]] INFO [2019-12-17 12:30:55,638] jepsen worker 7 - jepsen.util 5227 :info :cas [2 [4 23]] :RequestCanceledException INFO [2019-12-17 12:30:55,639] jepsen worker 7 - jepsen.util 5317 :invoke :cas [2 [22 5]] INFO [2019-12-17 12:30:55,654] jepsen worker 54 - jepsen.util 4464 :ok :read [18 36] INFO [2019-12-17 12:30:55,654] jepsen worker 54 - jepsen.util 4464 :invoke :write [18 1] INFO [2019-12-17 12:30:55,727] jepsen worker 31 - jepsen.util 3811 :ok :write [10 28] INFO [2019-12-17 12:30:55,727] jepsen worker 31 - jepsen.util 3811 :invoke :cas [10 [43 3]] INFO [2019-12-17 12:30:55,728] jepsen worker 11 - jepsen.util 4421 :ok :write [3 7] INFO [2019-12-17 12:30:55,728] jepsen worker 11 - jepsen.util 4421 :invoke :read [3 nil] INFO [2019-12-17 12:30:55,738] jepsen worker 47 - jepsen.util 4637 :ok :write [15 48] INFO [2019-12-17 12:30:55,738] jepsen worker 47 - jepsen.util 4637 :invoke :cas [15 [43 46]] INFO [2019-12-17 12:30:55,738] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":23543,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:30:55,738] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":23543,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:30:55,738] jepsen worker 83 - jepsen.util 5483 :info :write [27 35] :RequestCanceledException INFO [2019-12-17 12:30:55,738] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":23545,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:30:55,738] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":23545,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:30:55,739] jepsen worker 83 - jepsen.util 5573 :invoke :write [27 47] INFO [2019-12-17 12:30:55,744] jepsen worker 11 - jepsen.util 4421 :ok :read [3 7] INFO [2019-12-17 12:30:55,744] jepsen worker 11 - jepsen.util 4421 :invoke :read [3 nil] INFO [2019-12-17 12:30:55,755] jepsen worker 78 - jepsen.util 4488 :ok :write [26 45] INFO [2019-12-17 12:30:55,755] jepsen worker 78 - jepsen.util 4488 :invoke :write [26 32] INFO [2019-12-17 12:30:55,788] jepsen worker 47 - jepsen.util 4637 :fail :cas [15 [43 46]] :ValueNotSwapFrom INFO [2019-12-17 12:30:55,791] jepsen worker 50 - jepsen.util 5720 :info :cas [16 [21 37]] :RequestCanceledException INFO [2019-12-17 12:30:55,804] jepsen worker 11 - jepsen.util 4421 :ok :read [3 7] INFO [2019-12-17 12:30:55,809] jepsen worker 35 - jepsen.util 5435 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:30:55,813] jepsen worker 45 - jepsen.util 4185 :ok :write [15 5] INFO [2019-12-17 12:30:55,814] jepsen worker 47 - jepsen.util 4637 :invoke :cas [15 [23 31]] INFO [2019-12-17 12:30:55,814] jepsen worker 11 - jepsen.util 4421 :invoke :read [3 nil] INFO [2019-12-17 12:30:55,814] jepsen worker 45 - jepsen.util 4185 :invoke :read [15 nil] INFO [2019-12-17 12:30:55,814] jepsen worker 35 - jepsen.util 5435 :invoke :write [11 2] INFO [2019-12-17 12:30:55,815] jepsen worker 50 - jepsen.util 5810 :invoke :read [16 nil] INFO [2019-12-17 12:30:55,817] jepsen worker 51 - jepsen.util 3831 :fail :cas [17 [7 9]] :ValueNotSwapFrom INFO [2019-12-17 12:30:55,817] jepsen worker 51 - jepsen.util 3831 :invoke :write [17 7] INFO [2019-12-17 12:30:55,825] jepsen worker 58 - jepsen.util 5548 :info :write [19 20] :RequestCanceledException INFO [2019-12-17 12:30:55,825] jepsen worker 67 - jepsen.util 5647 :info :write [22 13] :RequestCanceledException INFO [2019-12-17 12:30:55,825] jepsen worker 2 - jepsen.util 5132 :info :cas [0 [31 1]] :RequestCanceledException INFO [2019-12-17 12:30:55,826] jepsen worker 58 - jepsen.util 5638 :invoke :cas [19 [21 4]] INFO [2019-12-17 12:30:55,826] jepsen worker 67 - jepsen.util 5737 :invoke :read [22 nil] INFO [2019-12-17 12:30:55,828] jepsen worker 2 - jepsen.util 5222 :invoke :read [0 nil] INFO [2019-12-17 12:30:55,834] jepsen worker 31 - jepsen.util 3811 :fail :cas [10 [43 3]] :ValueNotSwapFrom INFO [2019-12-17 12:30:55,834] jepsen worker 31 - jepsen.util 3811 :invoke :write [10 28] INFO [2019-12-17 12:30:55,839] jepsen worker 40 - jepsen.util 5980 :info :cas [13 [34 15]] :RequestCanceledException INFO [2019-12-17 12:30:55,841] jepsen worker 40 - jepsen.util 6070 :invoke :cas [13 [32 28]] INFO [2019-12-17 12:30:55,842] jepsen worker 45 - jepsen.util 4185 :ok :read [15 5] INFO [2019-12-17 12:30:55,842] jepsen worker 45 - jepsen.util 4185 :invoke :write [15 9] INFO [2019-12-17 12:30:55,856] jepsen worker 26 - jepsen.util 5606 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:30:55,856] jepsen worker 0 - jepsen.util 5940 :info :write [0 1] :RequestCanceledException INFO [2019-12-17 12:30:55,856] jepsen worker 60 - jepsen.util 5910 :info :write [20 18] :RequestCanceledException INFO [2019-12-17 12:30:55,856] jepsen worker 26 - jepsen.util 5606 :invoke :write [8 11] INFO [2019-12-17 12:30:55,857] jepsen worker 0 - jepsen.util 6030 :invoke :cas [0 [3 46]] INFO [2019-12-17 12:30:55,857] jepsen worker 60 - jepsen.util 6000 :invoke :cas [20 [32 5]] INFO [2019-12-17 12:30:55,871] jepsen worker 47 - jepsen.util 4637 :fail :cas [15 [23 31]] :ValueNotSwapFrom INFO [2019-12-17 12:30:55,871] jepsen worker 47 - jepsen.util 4637 :invoke :write [15 40] INFO [2019-12-17 12:30:55,874] jepsen worker 11 - jepsen.util 4421 :ok :read [3 7] INFO [2019-12-17 12:30:55,875] jepsen worker 11 - jepsen.util 4421 :invoke :read [3 nil] INFO [2019-12-17 12:30:55,886] jepsen worker 36 - jepsen.util 4266 :ok :write [12 36] INFO [2019-12-17 12:30:55,886] jepsen worker 36 - jepsen.util 4266 :invoke :read [12 nil] INFO [2019-12-17 12:30:55,891] jepsen worker 57 - jepsen.util 5187 :info :write [19 29] :RequestCanceledException INFO [2019-12-17 12:30:55,891] jepsen worker 70 - jepsen.util 4750 :info :write [23 4] :RequestCanceledException INFO [2019-12-17 12:30:55,892] jepsen worker 57 - jepsen.util 5277 :invoke :write [19 26] INFO [2019-12-17 12:30:55,892] jepsen worker 70 - jepsen.util 4840 :invoke :write [23 20] INFO [2019-12-17 12:30:55,909] jepsen worker 48 - jepsen.util 4908 :info :write [16 13] :RequestCanceledException INFO [2019-12-17 12:30:55,911] jepsen worker 48 - jepsen.util 4998 :invoke :read [16 nil] INFO [2019-12-17 12:30:55,912] jepsen worker 11 - jepsen.util 4421 :ok :read [3 7] INFO [2019-12-17 12:30:55,912] jepsen worker 11 - jepsen.util 4421 :invoke :cas [3 [21 6]] INFO [2019-12-17 12:30:55,921] jepsen worker 13 - jepsen.util 3883 :ok :write [4 32] INFO [2019-12-17 12:30:55,922] jepsen worker 13 - jepsen.util 3883 :invoke :cas [4 [24 23]] INFO [2019-12-17 12:30:55,924] jepsen worker 22 - jepsen.util 5422 :info :cas [7 [44 1]] :RequestCanceledException INFO [2019-12-17 12:30:55,931] jepsen worker 22 - jepsen.util 5512 :invoke :write [7 5] INFO [2019-12-17 12:30:55,936] jepsen worker 11 - jepsen.util 4421 :fail :cas [3 [21 6]] :ValueNotSwapFrom INFO [2019-12-17 12:30:55,936] jepsen worker 11 - jepsen.util 4421 :invoke :write [3 13] INFO [2019-12-17 12:30:55,956] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":23568,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:30:55,956] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":23568,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:30:55,956] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":23571,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:30:55,956] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":23571,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:30:55,956] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":23572,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:30:55,956] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":23572,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:30:55,964] jepsen worker 13 - jepsen.util 3883 :fail :cas [4 [24 23]] :ValueNotSwapFrom INFO [2019-12-17 12:30:55,964] jepsen worker 13 - jepsen.util 3883 :invoke :write [4 26] INFO [2019-12-17 12:30:55,981] jepsen worker 38 - jepsen.util 4088 :ok :write [12 40] INFO [2019-12-17 12:30:55,981] jepsen worker 38 - jepsen.util 4088 :invoke :write [12 32] INFO [2019-12-17 12:30:56,007] jepsen worker 4 - jepsen.util 3334 :ok :cas [1 [48 8]] INFO [2019-12-17 12:30:56,007] jepsen worker 4 - jepsen.util 3334 :invoke :read [1 nil] INFO [2019-12-17 12:30:56,009] jepsen worker 17 - jepsen.util 5147 :info :cas [5 [34 43]] :RequestCanceledException INFO [2019-12-17 12:30:56,010] jepsen worker 17 - jepsen.util 5237 :invoke :read [5 nil] INFO [2019-12-17 12:30:56,024] jepsen worker 33 - jepsen.util 5523 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:30:56,024] jepsen worker 33 - jepsen.util 5523 :invoke :read [11 nil] INFO [2019-12-17 12:30:56,047] jepsen worker 71 - jepsen.util 4391 :ok :write [23 12] INFO [2019-12-17 12:30:56,047] jepsen worker 71 - jepsen.util 4391 :invoke :write [23 35] INFO [2019-12-17 12:30:56,065] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":23587,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:30:56,065] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":23587,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:30:56,065] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":23590,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:30:56,065] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":23590,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:30:56,074] jepsen worker 37 - jepsen.util 3907 :ok :read [12 40] INFO [2019-12-17 12:30:56,074] jepsen worker 37 - jepsen.util 3907 :invoke :read [12 nil] INFO [2019-12-17 12:30:56,086] jepsen worker 31 - jepsen.util 3811 :ok :write [10 28] INFO [2019-12-17 12:30:56,086] jepsen worker 31 - jepsen.util 3811 :invoke :write [10 17] INFO [2019-12-17 12:30:56,092] jepsen worker 56 - jepsen.util 4286 :ok :write [18 39] INFO [2019-12-17 12:30:56,092] jepsen worker 56 - jepsen.util 4286 :invoke :read [18 nil] INFO [2019-12-17 12:30:56,109] jepsen worker 75 - jepsen.util 5475 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:30:56,109] jepsen worker 75 - jepsen.util 5475 :invoke :cas [25 [44 40]] INFO [2019-12-17 12:30:56,113] jepsen worker 79 - jepsen.util 4129 :ok :read [26 45] INFO [2019-12-17 12:30:56,113] jepsen worker 79 - jepsen.util 4129 :invoke :cas [26 [48 8]] INFO [2019-12-17 12:30:56,125] jepsen worker 42 - jepsen.util 6162 :info :write [14 37] :RequestCanceledException INFO [2019-12-17 12:30:56,125] jepsen worker 9 - jepsen.util 4599 :info :write [3 12] :RequestCanceledException INFO [2019-12-17 12:30:56,125] jepsen worker 1 - jepsen.util 5041 :info :write [0 6] :RequestCanceledException INFO [2019-12-17 12:30:56,126] jepsen worker 42 - jepsen.util 6252 :invoke :read [14 nil] INFO [2019-12-17 12:30:56,126] jepsen worker 9 - jepsen.util 4689 :invoke :read [3 nil] INFO [2019-12-17 12:30:56,126] jepsen worker 1 - jepsen.util 5131 :invoke :read [0 nil] INFO [2019-12-17 12:30:56,131] jepsen worker 3 - jepsen.util 4323 :fail :cas [1 [4 37]] :ValueNotSwapFrom INFO [2019-12-17 12:30:56,131] jepsen worker 3 - jepsen.util 4323 :invoke :cas [1 [17 27]] INFO [2019-12-17 12:30:56,138] jepsen worker 41 - jepsen.util 5531 :info :cas [13 [45 11]] :RequestCanceledException INFO [2019-12-17 12:30:56,139] jepsen worker 41 - jepsen.util 5621 :invoke :cas [13 [41 3]] INFO [2019-12-17 12:30:56,150] jepsen worker 87 - jepsen.util 3867 :ok :write [29 30] INFO [2019-12-17 12:30:56,151] jepsen worker 87 - jepsen.util 3867 :invoke :cas [29 [44 38]] INFO [2019-12-17 12:30:56,155] jepsen worker 47 - jepsen.util 4637 :ok :write [15 40] INFO [2019-12-17 12:30:56,155] jepsen worker 47 - jepsen.util 4637 :invoke :write [15 13] INFO [2019-12-17 12:30:56,169] jepsen worker 89 - jepsen.util 3959 :ok :write [29 30] INFO [2019-12-17 12:30:56,169] jepsen worker 89 - jepsen.util 3959 :invoke :read [29 nil] INFO [2019-12-17 12:30:56,174] jepsen worker 9 - jepsen.util 4689 :ok :read [3 7] INFO [2019-12-17 12:30:56,174] jepsen worker 9 - jepsen.util 4689 :invoke :cas [3 [10 31]] INFO [2019-12-17 12:30:56,191] jepsen worker 63 - jepsen.util 3393 :info :write [21 32] :RequestCanceledException INFO [2019-12-17 12:30:56,191] jepsen worker 8 - jepsen.util 4688 :info :cas [2 [11 29]] :RequestCanceledException INFO [2019-12-17 12:30:56,192] jepsen worker 63 - jepsen.util 3483 :invoke :read [21 nil] INFO [2019-12-17 12:30:56,192] jepsen worker 8 - jepsen.util 4778 :invoke :write [2 49] INFO [2019-12-17 12:30:56,209] jepsen worker 64 - jepsen.util 3304 :fail :read [21 nil] :RequestTimeoutException INFO [2019-12-17 12:30:56,209] jepsen worker 74 - jepsen.util 4574 :info :write [24 32] :RequestCanceledException INFO [2019-12-17 12:30:56,209] jepsen worker 79 - jepsen.util 4129 :fail :cas [26 [48 8]] :ValueNotSwapFrom INFO [2019-12-17 12:30:56,209] jepsen worker 64 - jepsen.util 3304 :invoke :cas [21 [39 14]] INFO [2019-12-17 12:30:56,210] jepsen worker 79 - jepsen.util 4129 :invoke :cas [26 [37 5]] INFO [2019-12-17 12:30:56,211] jepsen worker 74 - jepsen.util 4664 :invoke :write [24 11] INFO [2019-12-17 12:30:56,212] jepsen worker 63 - jepsen.util 3483 :ok :read [21 28] INFO [2019-12-17 12:30:56,212] jepsen worker 63 - jepsen.util 3483 :invoke :read [21 nil] INFO [2019-12-17 12:30:56,225] jepsen worker 76 - jepsen.util 5566 :info :cas [25 [37 23]] :RequestCanceledException INFO [2019-12-17 12:30:56,227] jepsen worker 76 - jepsen.util 5656 :invoke :write [25 10] INFO [2019-12-17 12:30:56,237] jepsen worker 16 - jepsen.util 6226 :info :write [5 42] :RequestCanceledException INFO [2019-12-17 12:30:56,238] jepsen worker 16 - jepsen.util 6316 :invoke :read [5 nil] INFO [2019-12-17 12:30:56,240] jepsen worker 64 - jepsen.util 3304 :fail :cas [21 [39 14]] :ValueNotSwapFrom INFO [2019-12-17 12:30:56,240] jepsen worker 64 - jepsen.util 3304 :invoke :read [21 nil] INFO [2019-12-17 12:30:56,245] jepsen worker 55 - jepsen.util 5095 :fail :cas [18 [16 35]] :ValueNotSwapFrom INFO [2019-12-17 12:30:56,245] jepsen worker 55 - jepsen.util 5095 :invoke :read [18 nil] INFO [2019-12-17 12:30:56,251] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":23621,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:30:56,251] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":23621,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:30:56,251] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":23625,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:30:56,251] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":23625,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:30:56,256] jepsen worker 15 - jepsen.util 4785 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:30:56,256] jepsen worker 15 - jepsen.util 4785 :invoke :read [5 nil] INFO [2019-12-17 12:30:56,272] jepsen worker 63 - jepsen.util 3483 :ok :read [21 28] INFO [2019-12-17 12:30:56,272] jepsen worker 63 - jepsen.util 3483 :invoke :read [21 nil] INFO [2019-12-17 12:30:56,275] jepsen worker 3 - jepsen.util 4323 :fail :cas [1 [17 27]] :ValueNotSwapFrom INFO [2019-12-17 12:30:56,275] jepsen worker 3 - jepsen.util 4323 :invoke :write [1 9] INFO [2019-12-17 12:30:56,275] jepsen worker 52 - jepsen.util 4372 :ok :write [17 41] INFO [2019-12-17 12:30:56,275] jepsen worker 52 - jepsen.util 4372 :invoke :cas [17 [31 1]] INFO [2019-12-17 12:30:56,277] jepsen worker 12 - jepsen.util 4512 :ok :write [4 32] INFO [2019-12-17 12:30:56,277] jepsen worker 12 - jepsen.util 4512 :invoke :cas [4 [39 28]] INFO [2019-12-17 12:30:56,284] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":23610,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:30:56,284] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":23610,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:30:56,284] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":23612,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:30:56,284] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":23612,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:30:56,309] jepsen worker 65 - jepsen.util 3395 :info :cas [21 [18 29]] :RequestCanceledException INFO [2019-12-17 12:30:56,310] jepsen worker 65 - jepsen.util 3485 :invoke :cas [21 [15 16]] INFO [2019-12-17 12:30:56,312] jepsen worker 64 - jepsen.util 3304 :ok :read [21 28] INFO [2019-12-17 12:30:56,312] jepsen worker 64 - jepsen.util 3304 :invoke :read [21 nil] INFO [2019-12-17 12:30:56,328] jepsen worker 65 - jepsen.util 3485 :fail :cas [21 [15 16]] :ValueNotSwapFrom INFO [2019-12-17 12:30:56,329] jepsen worker 65 - jepsen.util 3485 :invoke :cas [21 [34 7]] INFO [2019-12-17 12:30:56,335] jepsen worker 63 - jepsen.util 3483 :ok :read [21 28] INFO [2019-12-17 12:30:56,335] jepsen worker 63 - jepsen.util 3483 :invoke :cas [21 [47 4]] INFO [2019-12-17 12:30:56,337] jepsen worker 81 - jepsen.util 5391 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:30:56,337] jepsen worker 81 - jepsen.util 5391 :invoke :cas [27 [16 47]] INFO [2019-12-17 12:30:56,352] jepsen worker 51 - jepsen.util 3831 :ok :write [17 7] INFO [2019-12-17 12:30:56,352] jepsen worker 51 - jepsen.util 3831 :invoke :write [17 22] INFO [2019-12-17 12:30:56,362] jepsen worker 65 - jepsen.util 3485 :fail :cas [21 [34 7]] :ValueNotSwapFrom INFO [2019-12-17 12:30:56,362] jepsen worker 65 - jepsen.util 3485 :invoke :read [21 nil] INFO [2019-12-17 12:30:56,369] jepsen worker 54 - jepsen.util 4464 :ok :write [18 1] INFO [2019-12-17 12:30:56,369] jepsen worker 54 - jepsen.util 4464 :invoke :cas [18 [31 41]] INFO [2019-12-17 12:30:56,378] jepsen worker 64 - jepsen.util 3304 :ok :read [21 28] INFO [2019-12-17 12:30:56,378] jepsen worker 64 - jepsen.util 3304 :invoke :read [21 nil] INFO [2019-12-17 12:30:56,392] jepsen worker 63 - jepsen.util 3483 :fail :cas [21 [47 4]] :ValueNotSwapFrom INFO [2019-12-17 12:30:56,392] jepsen worker 63 - jepsen.util 3483 :invoke :write [21 6] INFO [2019-12-17 12:30:56,403] jepsen worker 5 - jepsen.util 3785 :ok :read [1 8] INFO [2019-12-17 12:30:56,403] jepsen worker 5 - jepsen.util 3785 :invoke :write [1 40] INFO [2019-12-17 12:30:56,409] jepsen worker 14 - jepsen.util 4154 :info :write [4 48] :RequestCanceledException INFO [2019-12-17 12:30:56,410] jepsen worker 14 - jepsen.util 4244 :invoke :cas [4 [36 49]] INFO [2019-12-17 12:30:56,417] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:30:56,417] jepsen nemesis - couchbase.util Rebalance check count: 175 INFO [2019-12-17 12:30:56,417] jepsen nemesis - couchbase.util Rebalance stuck count: 156 INFO [2019-12-17 12:30:56,426] jepsen worker 64 - jepsen.util 3304 :ok :read [21 28] INFO [2019-12-17 12:30:56,426] jepsen worker 64 - jepsen.util 3304 :invoke :read [21 nil] INFO [2019-12-17 12:30:56,433] jepsen worker 65 - jepsen.util 3485 :ok :read [21 28] INFO [2019-12-17 12:30:56,434] jepsen worker 65 - jepsen.util 3485 :invoke :write [21 17] INFO [2019-12-17 12:30:56,435] jepsen worker 87 - jepsen.util 3867 :fail :cas [29 [44 38]] :ValueNotSwapFrom INFO [2019-12-17 12:30:56,435] jepsen worker 87 - jepsen.util 3867 :invoke :read [29 nil] INFO [2019-12-17 12:30:56,438] jepsen worker 21 - jepsen.util 5421 :info :write [7 49] :RequestCanceledException INFO [2019-12-17 12:30:56,439] jepsen worker 21 - jepsen.util 5511 :invoke :read [7 nil] INFO [2019-12-17 12:30:56,441] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":23651,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:30:56,441] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":23651,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:30:56,441] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":23655,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:30:56,441] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":23655,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:30:56,461] jepsen worker 63 - jepsen.util 3483 :ok :write [21 6] INFO [2019-12-17 12:30:56,461] jepsen worker 63 - jepsen.util 3483 :invoke :cas [21 [29 48]] INFO [2019-12-17 12:30:56,481] jepsen worker 63 - jepsen.util 3483 :fail :cas [21 [29 48]] :ValueNotSwapFrom INFO [2019-12-17 12:30:56,481] jepsen worker 63 - jepsen.util 3483 :invoke :read [21 nil] INFO [2019-12-17 12:30:56,491] jepsen worker 46 - jepsen.util 4186 :info :write [15 49] :RequestCanceledException INFO [2019-12-17 12:30:56,492] jepsen worker 46 - jepsen.util 4276 :invoke :cas [15 [18 35]] INFO [2019-12-17 12:30:56,496] jepsen worker 63 - jepsen.util 3483 :ok :read [21 6] INFO [2019-12-17 12:30:56,496] jepsen worker 63 - jepsen.util 3483 :invoke :read [21 nil] INFO [2019-12-17 12:30:56,498] jepsen worker 52 - jepsen.util 4372 :fail :cas [17 [31 1]] :ValueNotSwapFrom INFO [2019-12-17 12:30:56,498] jepsen worker 52 - jepsen.util 4372 :invoke :cas [17 [42 12]] INFO [2019-12-17 12:30:56,509] jepsen worker 29 - jepsen.util 3449 :info :write [9 29] :RequestCanceledException INFO [2019-12-17 12:30:56,509] jepsen worker 72 - jepsen.util 4302 :fail :read [24 nil] :RequestTimeoutException INFO [2019-12-17 12:30:56,509] jepsen worker 72 - jepsen.util 4302 :invoke :cas [24 [48 3]] INFO [2019-12-17 12:30:56,510] jepsen worker 29 - jepsen.util 3539 :invoke :write [9 33] INFO [2019-12-17 12:30:56,513] jepsen worker 11 - jepsen.util 4421 :ok :write [3 13] INFO [2019-12-17 12:30:56,513] jepsen worker 87 - jepsen.util 3867 :ok :read [29 30] INFO [2019-12-17 12:30:56,513] jepsen worker 87 - jepsen.util 3867 :invoke :read [29 nil] INFO [2019-12-17 12:30:56,513] jepsen worker 11 - jepsen.util 4421 :invoke :write [3 0] INFO [2019-12-17 12:30:56,522] jepsen worker 63 - jepsen.util 3483 :ok :read [21 6] INFO [2019-12-17 12:30:56,522] jepsen worker 63 - jepsen.util 3483 :invoke :write [21 43] INFO [2019-12-17 12:30:56,539] jepsen worker 61 - jepsen.util 6181 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:30:56,539] jepsen worker 61 - jepsen.util 6181 :invoke :cas [20 [30 34]] INFO [2019-12-17 12:30:56,561] jepsen worker 72 - jepsen.util 4302 :fail :cas [24 [48 3]] :ValueNotSwapFrom INFO [2019-12-17 12:30:56,562] jepsen worker 72 - jepsen.util 4302 :invoke :write [24 30] INFO [2019-12-17 12:30:56,562] jepsen worker 52 - jepsen.util 4372 :fail :cas [17 [42 12]] :ValueNotSwapFrom INFO [2019-12-17 12:30:56,562] jepsen worker 52 - jepsen.util 4372 :invoke :write [17 18] INFO [2019-12-17 12:30:56,597] jepsen worker 65 - jepsen.util 3485 :ok :write [21 17] INFO [2019-12-17 12:30:56,597] jepsen worker 65 - jepsen.util 3485 :invoke :read [21 nil] INFO [2019-12-17 12:30:56,605] jepsen worker 80 - jepsen.util 4220 :ok :write [26 0] INFO [2019-12-17 12:30:56,606] jepsen worker 80 - jepsen.util 4220 :invoke :cas [26 [10 47]] INFO [2019-12-17 12:30:56,606] jepsen worker 88 - jepsen.util 4228 :ok :write [29 47] INFO [2019-12-17 12:30:56,606] jepsen worker 88 - jepsen.util 4228 :invoke :read [29 nil] INFO [2019-12-17 12:30:56,623] jepsen worker 65 - jepsen.util 3485 :ok :read [21 17] INFO [2019-12-17 12:30:56,623] jepsen worker 65 - jepsen.util 3485 :invoke :write [21 0] INFO [2019-12-17 12:30:56,625] jepsen worker 74 - jepsen.util 4664 :ok :write [24 11] INFO [2019-12-17 12:30:56,625] jepsen worker 74 - jepsen.util 4664 :invoke :cas [24 [44 40]] INFO [2019-12-17 12:30:56,626] jepsen worker 31 - jepsen.util 3811 :ok :write [10 17] INFO [2019-12-17 12:30:56,626] jepsen worker 31 - jepsen.util 3811 :invoke :cas [10 [43 14]] INFO [2019-12-17 12:30:56,633] jepsen worker 36 - jepsen.util 4266 :ok :read [12 40] INFO [2019-12-17 12:30:56,633] jepsen worker 36 - jepsen.util 4266 :invoke :cas [12 [9 42]] INFO [2019-12-17 12:30:56,670] jepsen worker 37 - jepsen.util 3907 :ok :read [12 40] INFO [2019-12-17 12:30:56,670] jepsen worker 37 - jepsen.util 3907 :invoke :write [12 3] INFO [2019-12-17 12:30:56,682] jepsen worker 56 - jepsen.util 4286 :ok :read [18 1] INFO [2019-12-17 12:30:56,683] jepsen worker 56 - jepsen.util 4286 :invoke :write [18 4] INFO [2019-12-17 12:30:56,694] jepsen worker 13 - jepsen.util 3883 :ok :write [4 26] INFO [2019-12-17 12:30:56,694] jepsen worker 13 - jepsen.util 3883 :invoke :cas [4 [7 5]] INFO [2019-12-17 12:30:56,696] jepsen worker 79 - jepsen.util 4129 :fail :cas [26 [37 5]] :ValueNotSwapFrom INFO [2019-12-17 12:30:56,696] jepsen worker 79 - jepsen.util 4129 :invoke :read [26 nil] INFO [2019-12-17 12:30:56,700] jepsen worker 46 - jepsen.util 4276 :fail :cas [15 [18 35]] :ValueNotSwapFrom INFO [2019-12-17 12:30:56,700] jepsen worker 46 - jepsen.util 4276 :invoke :read [15 nil] INFO [2019-12-17 12:30:56,720] jepsen worker 74 - jepsen.util 4664 :fail :cas [24 [44 40]] :ValueNotSwapFrom INFO [2019-12-17 12:30:56,720] jepsen worker 74 - jepsen.util 4664 :invoke :read [24 nil] INFO [2019-12-17 12:30:56,720] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":23679,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:30:56,721] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":23679,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:30:56,721] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":23682,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:30:56,721] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":23682,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:30:56,721] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":23683,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:30:56,721] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":23683,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:30:56,722] jepsen worker 31 - jepsen.util 3811 :fail :cas [10 [43 14]] :ValueNotSwapFrom INFO [2019-12-17 12:30:56,722] jepsen worker 31 - jepsen.util 3811 :invoke :write [10 8] INFO [2019-12-17 12:30:56,727] jepsen worker 54 - jepsen.util 4464 :fail :cas [18 [31 41]] :ValueNotSwapFrom INFO [2019-12-17 12:30:56,728] jepsen worker 54 - jepsen.util 4464 :invoke :cas [18 [40 36]] INFO [2019-12-17 12:30:56,733] jepsen worker 46 - jepsen.util 4276 :ok :read [15 40] INFO [2019-12-17 12:30:56,734] jepsen worker 46 - jepsen.util 4276 :invoke :write [15 37] INFO [2019-12-17 12:30:56,755] jepsen worker 88 - jepsen.util 4228 :ok :read [29 47] INFO [2019-12-17 12:30:56,755] jepsen worker 88 - jepsen.util 4228 :invoke :read [29 nil] INFO [2019-12-17 12:30:56,757] jepsen worker 74 - jepsen.util 4664 :ok :read [24 11] INFO [2019-12-17 12:30:56,757] jepsen worker 74 - jepsen.util 4664 :invoke :write [24 14] INFO [2019-12-17 12:30:56,760] jepsen worker 69 - jepsen.util 4209 :ok :write [23 46] INFO [2019-12-17 12:30:56,760] jepsen worker 69 - jepsen.util 4209 :invoke :write [23 19] INFO [2019-12-17 12:30:56,774] jepsen worker 63 - jepsen.util 3483 :ok :write [21 43] INFO [2019-12-17 12:30:56,774] jepsen worker 63 - jepsen.util 3483 :invoke :write [21 41] INFO [2019-12-17 12:30:56,788] jepsen worker 47 - jepsen.util 4637 :ok :write [15 13] INFO [2019-12-17 12:30:56,819] jepsen worker 9 - jepsen.util 4689 :fail :cas [3 [10 31]] :ValueNotSwapFrom INFO [2019-12-17 12:30:56,825] jepsen worker 77 - jepsen.util 5387 :info :cas [25 [30 20]] :RequestCanceledException INFO [2019-12-17 12:30:56,825] jepsen worker 86 - jepsen.util 5666 :info :cas [28 [6 16]] :RequestCanceledException INFO [2019-12-17 12:30:56,829] jepsen worker 11 - jepsen.util 4421 :ok :write [3 0] INFO [2019-12-17 12:30:56,839] jepsen worker 24 - jepsen.util 5334 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:30:56,856] jepsen worker 53 - jepsen.util 4103 :info :write [17 24] :RequestCanceledException INFO [2019-12-17 12:30:56,858] jepsen worker 47 - jepsen.util 4637 :invoke :read [15 nil] INFO [2019-12-17 12:30:56,862] jepsen worker 9 - jepsen.util 4689 :invoke :write [3 43] INFO [2019-12-17 12:30:56,862] jepsen worker 11 - jepsen.util 4421 :invoke :read [3 nil] INFO [2019-12-17 12:30:56,862] jepsen worker 24 - jepsen.util 5334 :invoke :read [8 nil] INFO [2019-12-17 12:30:56,863] jepsen worker 86 - jepsen.util 5756 :invoke :read [28 nil] INFO [2019-12-17 12:30:56,863] jepsen worker 77 - jepsen.util 5477 :invoke :write [25 39] INFO [2019-12-17 12:30:56,863] jepsen worker 53 - jepsen.util 4193 :invoke :read [17 nil] INFO [2019-12-17 12:30:56,870] jepsen worker 70 - jepsen.util 4840 :ok :write [23 20] INFO [2019-12-17 12:30:56,870] jepsen worker 70 - jepsen.util 4840 :invoke :write [23 47] INFO [2019-12-17 12:30:56,890] jepsen worker 11 - jepsen.util 4421 :ok :read [3 0] INFO [2019-12-17 12:30:56,890] jepsen worker 11 - jepsen.util 4421 :invoke :write [3 2] INFO [2019-12-17 12:30:56,891] jepsen worker 49 - jepsen.util 4909 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:30:56,891] jepsen worker 49 - jepsen.util 4909 :invoke :read [16 nil] INFO [2019-12-17 12:30:56,918] jepsen worker 29 - jepsen.util 3539 :ok :write [9 33] INFO [2019-12-17 12:30:56,918] jepsen worker 29 - jepsen.util 3539 :invoke :cas [9 [33 3]] INFO [2019-12-17 12:30:56,925] jepsen worker 32 - jepsen.util 4622 :info :write [10 36] :RequestCanceledException INFO [2019-12-17 12:30:56,926] jepsen worker 32 - jepsen.util 4712 :invoke :cas [10 [22 40]] INFO [2019-12-17 12:30:56,927] jepsen worker 4 - jepsen.util 3334 :ok :read [1 8] INFO [2019-12-17 12:30:56,927] jepsen worker 4 - jepsen.util 3334 :invoke :read [1 nil] INFO [2019-12-17 12:30:56,939] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":23702,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:30:56,940] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":23702,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:30:56,940] jepsen worker 87 - jepsen.util 3867 :ok :read [29 47] INFO [2019-12-17 12:30:56,940] jepsen worker 87 - jepsen.util 3867 :invoke :write [29 40] INFO [2019-12-17 12:30:56,945] jepsen worker 47 - jepsen.util 4637 :ok :read [15 13] INFO [2019-12-17 12:30:56,945] jepsen worker 47 - jepsen.util 4637 :invoke :write [15 22] INFO [2019-12-17 12:30:56,967] jepsen worker 52 - jepsen.util 4372 :ok :write [17 18] INFO [2019-12-17 12:30:56,967] jepsen worker 52 - jepsen.util 4372 :invoke :read [17 nil] INFO [2019-12-17 12:30:56,972] jepsen worker 89 - jepsen.util 3959 :ok :read [29 47] INFO [2019-12-17 12:30:56,972] jepsen worker 89 - jepsen.util 3959 :invoke :cas [29 [9 6]] INFO [2019-12-17 12:30:57,010] jepsen worker 78 - jepsen.util 4488 :ok :write [26 32] INFO [2019-12-17 12:30:57,010] jepsen worker 78 - jepsen.util 4488 :invoke :cas [26 [21 39]] INFO [2019-12-17 12:30:57,018] jepsen worker 63 - jepsen.util 3483 :ok :write [21 41] INFO [2019-12-17 12:30:57,018] jepsen worker 63 - jepsen.util 3483 :invoke :write [21 1] INFO [2019-12-17 12:30:57,030] jepsen worker 55 - jepsen.util 5095 :ok :read [18 1] INFO [2019-12-17 12:30:57,031] jepsen worker 55 - jepsen.util 5095 :invoke :cas [18 [5 14]] INFO [2019-12-17 12:30:57,032] jepsen worker 3 - jepsen.util 4323 :ok :write [1 9] INFO [2019-12-17 12:30:57,032] jepsen worker 3 - jepsen.util 4323 :invoke :write [1 27] INFO [2019-12-17 12:30:57,049] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":23721,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:30:57,049] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":23721,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:30:57,049] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":23723,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:30:57,049] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":23723,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:30:57,055] jepsen worker 13 - jepsen.util 3883 :fail :cas [4 [7 5]] :ValueNotSwapFrom INFO [2019-12-17 12:30:57,055] jepsen worker 13 - jepsen.util 3883 :invoke :read [4 nil] INFO [2019-12-17 12:30:57,056] jepsen worker 73 - jepsen.util 3943 :fail :read [24 nil] :RequestTimeoutException INFO [2019-12-17 12:30:57,056] jepsen worker 73 - jepsen.util 3943 :invoke :cas [24 [27 0]] INFO [2019-12-17 12:30:57,076] jepsen worker 53 - jepsen.util 4193 :ok :read [17 18] INFO [2019-12-17 12:30:57,076] jepsen worker 53 - jepsen.util 4193 :invoke :write [17 44] INFO [2019-12-17 12:30:57,076] jepsen worker 73 - jepsen.util 3943 :fail :cas [24 [27 0]] :ValueNotSwapFrom INFO [2019-12-17 12:30:57,076] jepsen worker 73 - jepsen.util 3943 :invoke :read [24 nil] INFO [2019-12-17 12:30:57,089] jepsen worker 73 - jepsen.util 3943 :ok :read [24 11] INFO [2019-12-17 12:30:57,089] jepsen worker 73 - jepsen.util 3943 :invoke :read [24 nil] INFO [2019-12-17 12:30:57,091] jepsen worker 34 - jepsen.util 5164 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:30:57,091] jepsen worker 34 - jepsen.util 5164 :invoke :read [11 nil] INFO [2019-12-17 12:30:57,114] jepsen worker 73 - jepsen.util 3943 :ok :read [24 11] INFO [2019-12-17 12:30:57,114] jepsen worker 73 - jepsen.util 3943 :invoke :read [24 nil] INFO [2019-12-17 12:30:57,117] jepsen worker 54 - jepsen.util 4464 :fail :cas [18 [40 36]] :ValueNotSwapFrom INFO [2019-12-17 12:30:57,117] jepsen worker 54 - jepsen.util 4464 :invoke :read [18 nil] INFO [2019-12-17 12:30:57,125] jepsen worker 18 - jepsen.util 4158 :fail :read [6 nil] :RequestTimeoutException INFO [2019-12-17 12:30:57,125] jepsen worker 18 - jepsen.util 4158 :invoke :read [6 nil] INFO [2019-12-17 12:30:57,133] jepsen worker 73 - jepsen.util 3943 :ok :read [24 11] INFO [2019-12-17 12:30:57,133] jepsen worker 73 - jepsen.util 3943 :invoke :write [24 49] INFO [2019-12-17 12:30:57,134] jepsen worker 88 - jepsen.util 4228 :ok :read [29 47] INFO [2019-12-17 12:30:57,134] jepsen worker 88 - jepsen.util 4228 :invoke :write [29 39] INFO [2019-12-17 12:30:57,135] jepsen worker 63 - jepsen.util 3483 :ok :write [21 1] INFO [2019-12-17 12:30:57,135] jepsen worker 63 - jepsen.util 3483 :invoke :cas [21 [21 35]] INFO [2019-12-17 12:30:57,155] jepsen worker 27 - jepsen.util 3987 :info :cas [9 [21 12]] :RequestCanceledException INFO [2019-12-17 12:30:57,157] jepsen worker 27 - jepsen.util 4077 :invoke :write [9 16] INFO [2019-12-17 12:30:57,158] jepsen worker 14 - jepsen.util 4244 :fail :cas [4 [36 49]] :ValueNotSwapFrom INFO [2019-12-17 12:30:57,158] jepsen worker 14 - jepsen.util 4244 :invoke :cas [4 [12 7]] INFO [2019-12-17 12:30:57,160] jepsen worker 63 - jepsen.util 3483 :fail :cas [21 [21 35]] :ValueNotSwapFrom INFO [2019-12-17 12:30:57,160] jepsen worker 63 - jepsen.util 3483 :invoke :read [21 nil] INFO [2019-12-17 12:30:57,183] jepsen worker 63 - jepsen.util 3483 :ok :read [21 1] INFO [2019-12-17 12:30:57,186] jepsen worker 54 - jepsen.util 4464 :ok :read [18 1] INFO [2019-12-17 12:30:57,190] jepsen worker 63 - jepsen.util 3483 :invoke :cas [21 [31 41]] INFO [2019-12-17 12:30:57,190] jepsen worker 54 - jepsen.util 4464 :invoke :write [18 9] INFO [2019-12-17 12:30:57,192] jepsen worker 68 - jepsen.util 5018 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:30:57,192] jepsen worker 68 - jepsen.util 5018 :invoke :read [22 nil] INFO [2019-12-17 12:30:57,202] jepsen worker 46 - jepsen.util 4276 :ok :write [15 37] INFO [2019-12-17 12:30:57,202] jepsen worker 46 - jepsen.util 4276 :invoke :read [15 nil] INFO [2019-12-17 12:30:57,207] jepsen worker 63 - jepsen.util 3483 :fail :cas [21 [31 41]] :ValueNotSwapFrom INFO [2019-12-17 12:30:57,207] jepsen worker 63 - jepsen.util 3483 :invoke :cas [21 [21 24]] INFO [2019-12-17 12:30:57,217] jepsen worker 4 - jepsen.util 3334 :ok :read [1 9] INFO [2019-12-17 12:30:57,217] jepsen worker 31 - jepsen.util 3811 :ok :write [10 8] INFO [2019-12-17 12:30:57,217] jepsen worker 4 - jepsen.util 3334 :invoke :write [1 44] INFO [2019-12-17 12:30:57,217] jepsen worker 31 - jepsen.util 3811 :invoke :read [10 nil] INFO [2019-12-17 12:30:57,230] jepsen worker 52 - jepsen.util 4372 :ok :read [17 18] INFO [2019-12-17 12:30:57,230] jepsen worker 52 - jepsen.util 4372 :invoke :write [17 27] INFO [2019-12-17 12:30:57,238] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":23758,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:30:57,238] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":23758,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:30:57,238] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":23761,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:30:57,238] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":23761,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:30:57,259] jepsen worker 63 - jepsen.util 3483 :fail :cas [21 [21 24]] :ValueNotSwapFrom INFO [2019-12-17 12:30:57,259] jepsen worker 63 - jepsen.util 3483 :invoke :read [21 nil] INFO [2019-12-17 12:30:57,262] jepsen worker 11 - jepsen.util 4421 :ok :write [3 2] INFO [2019-12-17 12:30:57,262] jepsen worker 11 - jepsen.util 4421 :invoke :read [3 nil] INFO [2019-12-17 12:30:57,264] jepsen worker 89 - jepsen.util 3959 :fail :cas [29 [9 6]] :ValueNotSwapFrom INFO [2019-12-17 12:30:57,264] jepsen worker 89 - jepsen.util 3959 :invoke :write [29 44] INFO [2019-12-17 12:30:57,266] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":23749,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:30:57,267] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":23749,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:30:57,292] jepsen worker 55 - jepsen.util 5095 :fail :cas [18 [5 14]] :ValueNotSwapFrom INFO [2019-12-17 12:30:57,292] jepsen worker 32 - jepsen.util 4712 :fail :cas [10 [22 40]] :ValueNotSwapFrom INFO [2019-12-17 12:30:57,292] jepsen worker 55 - jepsen.util 5095 :invoke :cas [18 [23 8]] INFO [2019-12-17 12:30:57,292] jepsen worker 32 - jepsen.util 4712 :invoke :cas [10 [14 31]] INFO [2019-12-17 12:30:57,311] jepsen worker 73 - jepsen.util 3943 :ok :write [24 49] INFO [2019-12-17 12:30:57,312] jepsen worker 73 - jepsen.util 3943 :invoke :write [24 0] INFO [2019-12-17 12:30:57,314] jepsen worker 13 - jepsen.util 3883 :ok :read [4 26] INFO [2019-12-17 12:30:57,314] jepsen worker 13 - jepsen.util 3883 :invoke :read [4 nil] INFO [2019-12-17 12:30:57,316] jepsen worker 63 - jepsen.util 3483 :ok :read [21 1] INFO [2019-12-17 12:30:57,317] jepsen worker 63 - jepsen.util 3483 :invoke :cas [21 [27 11]] INFO [2019-12-17 12:30:57,320] jepsen worker 36 - jepsen.util 4266 :fail :cas [12 [9 42]] :ValueNotSwapFrom INFO [2019-12-17 12:30:57,320] jepsen worker 36 - jepsen.util 4266 :invoke :cas [12 [28 14]] INFO [2019-12-17 12:30:57,324] jepsen worker 31 - jepsen.util 3811 :ok :read [10 8] INFO [2019-12-17 12:30:57,324] jepsen worker 31 - jepsen.util 3811 :invoke :cas [10 [4 43]] INFO [2019-12-17 12:30:57,325] jepsen worker 23 - jepsen.util 5333 :info :cas [7 [3 19]] :RequestCanceledException INFO [2019-12-17 12:30:57,326] jepsen worker 23 - jepsen.util 5423 :invoke :read [7 nil] INFO [2019-12-17 12:30:57,341] jepsen worker 79 - jepsen.util 4129 :ok :read [26 32] INFO [2019-12-17 12:30:57,342] jepsen worker 79 - jepsen.util 4129 :invoke :read [26 nil] INFO [2019-12-17 12:30:57,373] jepsen worker 14 - jepsen.util 4244 :fail :cas [4 [12 7]] :ValueNotSwapFrom INFO [2019-12-17 12:30:57,373] jepsen worker 14 - jepsen.util 4244 :invoke :cas [4 [30 26]] INFO [2019-12-17 12:30:57,386] jepsen worker 12 - jepsen.util 4512 :fail :cas [4 [39 28]] :ValueNotSwapFrom INFO [2019-12-17 12:30:57,386] jepsen worker 12 - jepsen.util 4512 :invoke :write [4 9] INFO [2019-12-17 12:30:57,389] jepsen worker 18 - jepsen.util 4158 :ok :read [6 26] INFO [2019-12-17 12:30:57,389] jepsen worker 18 - jepsen.util 4158 :invoke :read [6 nil] INFO [2019-12-17 12:30:57,401] jepsen worker 63 - jepsen.util 3483 :fail :cas [21 [27 11]] :ValueNotSwapFrom INFO [2019-12-17 12:30:57,402] jepsen worker 63 - jepsen.util 3483 :invoke :read [21 nil] INFO [2019-12-17 12:30:57,409] jepsen worker 30 - jepsen.util 4350 :info :write [10 44] :RequestCanceledException INFO [2019-12-17 12:30:57,410] jepsen worker 30 - jepsen.util 4440 :invoke :cas [10 [12 20]] INFO [2019-12-17 12:30:57,424] jepsen worker 44 - jepsen.util 5804 :info :cas [14 [48 37]] :RequestCanceledException INFO [2019-12-17 12:30:57,425] jepsen worker 59 - jepsen.util 5459 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:30:57,425] jepsen worker 59 - jepsen.util 5459 :invoke :cas [19 [27 10]] INFO [2019-12-17 12:30:57,425] jepsen worker 44 - jepsen.util 5894 :invoke :write [14 3] INFO [2019-12-17 12:30:57,432] jepsen worker 31 - jepsen.util 3811 :fail :cas [10 [4 43]] :ValueNotSwapFrom INFO [2019-12-17 12:30:57,432] jepsen worker 31 - jepsen.util 3811 :invoke :write [10 2] INFO [2019-12-17 12:30:57,438] jepsen worker 43 - jepsen.util 5083 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:30:57,439] jepsen worker 43 - jepsen.util 5083 :invoke :read [14 nil] INFO [2019-12-17 12:30:57,463] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:30:57,463] jepsen nemesis - couchbase.util Rebalance check count: 176 INFO [2019-12-17 12:30:57,463] jepsen nemesis - couchbase.util Rebalance stuck count: 157 INFO [2019-12-17 12:30:57,464] jepsen worker 63 - jepsen.util 3483 :ok :read [21 1] INFO [2019-12-17 12:30:57,465] jepsen worker 63 - jepsen.util 3483 :invoke :cas [21 [15 13]] INFO [2019-12-17 12:30:57,472] jepsen worker 13 - jepsen.util 3883 :ok :read [4 26] INFO [2019-12-17 12:30:57,472] jepsen worker 13 - jepsen.util 3883 :invoke :cas [4 [31 0]] INFO [2019-12-17 12:30:57,486] jepsen worker 36 - jepsen.util 4266 :fail :cas [12 [28 14]] :ValueNotSwapFrom INFO [2019-12-17 12:30:57,486] jepsen worker 36 - jepsen.util 4266 :invoke :read [12 nil] INFO [2019-12-17 12:30:57,490] jepsen worker 66 - jepsen.util 5286 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:30:57,490] jepsen worker 84 - jepsen.util 5034 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:30:57,490] jepsen worker 66 - jepsen.util 5286 :invoke :read [22 nil] INFO [2019-12-17 12:30:57,490] jepsen worker 84 - jepsen.util 5034 :invoke :read [28 nil] INFO [2019-12-17 12:30:57,508] jepsen worker 11 - jepsen.util 4421 :ok :read [3 2] INFO [2019-12-17 12:30:57,508] jepsen worker 11 - jepsen.util 4421 :invoke :read [3 nil] INFO [2019-12-17 12:30:57,509] jepsen worker 25 - jepsen.util 5695 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:30:57,510] jepsen worker 25 - jepsen.util 5695 :invoke :write [8 25] INFO [2019-12-17 12:30:57,516] jepsen worker 30 - jepsen.util 4440 :fail :cas [10 [12 20]] :ValueNotSwapFrom INFO [2019-12-17 12:30:57,516] jepsen worker 30 - jepsen.util 4440 :invoke :cas [10 [41 46]] INFO [2019-12-17 12:30:57,523] jepsen worker 6 - jepsen.util 6036 :info :write [2 5] :RequestCanceledException INFO [2019-12-17 12:30:57,523] jepsen worker 20 - jepsen.util 4070 :info :cas [6 [20 31]] :RequestCanceledException INFO [2019-12-17 12:30:57,524] jepsen worker 6 - jepsen.util 6126 :invoke :read [2 nil] INFO [2019-12-17 12:30:57,525] jepsen worker 20 - jepsen.util 4160 :invoke :cas [6 [35 8]] INFO [2019-12-17 12:30:57,530] jepsen worker 32 - jepsen.util 4712 :fail :cas [10 [14 31]] :ValueNotSwapFrom INFO [2019-12-17 12:30:57,531] jepsen worker 32 - jepsen.util 4712 :invoke :cas [10 [30 19]] INFO [2019-12-17 12:30:57,534] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":23787,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:30:57,535] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":23787,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:30:57,535] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":23789,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:30:57,535] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":23789,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:30:57,545] jepsen worker 18 - jepsen.util 4158 :ok :read [6 26] INFO [2019-12-17 12:30:57,545] jepsen worker 18 - jepsen.util 4158 :invoke :read [6 nil] INFO [2019-12-17 12:30:57,549] jepsen worker 13 - jepsen.util 3883 :fail :cas [4 [31 0]] :ValueNotSwapFrom INFO [2019-12-17 12:30:57,550] jepsen worker 13 - jepsen.util 3883 :invoke :read [4 nil] INFO [2019-12-17 12:30:57,559] jepsen worker 11 - jepsen.util 4421 :ok :read [3 2] INFO [2019-12-17 12:30:57,559] jepsen worker 11 - jepsen.util 4421 :invoke :cas [3 [13 7]] INFO [2019-12-17 12:30:57,560] jepsen worker 27 - jepsen.util 4077 :ok :write [9 16] INFO [2019-12-17 12:30:57,561] jepsen worker 27 - jepsen.util 4077 :invoke :cas [9 [16 42]] INFO [2019-12-17 12:30:57,565] jepsen worker 63 - jepsen.util 3483 :fail :cas [21 [15 13]] :ValueNotSwapFrom INFO [2019-12-17 12:30:57,565] jepsen worker 63 - jepsen.util 3483 :invoke :write [21 27] INFO [2019-12-17 12:30:57,609] jepsen worker 19 - jepsen.util 4249 :fail :read [6 nil] :RequestTimeoutException INFO [2019-12-17 12:30:57,609] jepsen worker 19 - jepsen.util 4249 :invoke :write [6 46] INFO [2019-12-17 12:30:57,624] jepsen worker 62 - jepsen.util 5102 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:30:57,624] jepsen worker 85 - jepsen.util 6025 :info :write [28 15] :RequestCanceledException INFO [2019-12-17 12:30:57,624] jepsen worker 10 - jepsen.util 3970 :info :write [3 42] :RequestCanceledException INFO [2019-12-17 12:30:57,624] jepsen worker 62 - jepsen.util 5102 :invoke :cas [20 [21 39]] INFO [2019-12-17 12:30:57,625] jepsen worker 10 - jepsen.util 4060 :invoke :read [3 nil] INFO [2019-12-17 12:30:57,625] jepsen worker 85 - jepsen.util 6115 :invoke :write [28 29] INFO [2019-12-17 12:30:57,627] jepsen worker 20 - jepsen.util 4160 :fail :cas [6 [35 8]] :ValueNotSwapFrom INFO [2019-12-17 12:30:57,627] jepsen worker 20 - jepsen.util 4160 :invoke :write [6 13] INFO [2019-12-17 12:30:57,630] jepsen worker 79 - jepsen.util 4129 :ok :read [26 32] INFO [2019-12-17 12:30:57,630] jepsen worker 79 - jepsen.util 4129 :invoke :write [26 30] INFO [2019-12-17 12:30:57,638] jepsen worker 39 - jepsen.util 5079 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:30:57,638] jepsen worker 82 - jepsen.util 6382 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:30:57,639] jepsen worker 39 - jepsen.util 5079 :invoke :read [13 nil] INFO [2019-12-17 12:30:57,639] jepsen worker 82 - jepsen.util 6382 :invoke :cas [27 [35 2]] INFO [2019-12-17 12:30:57,655] jepsen worker 28 - jepsen.util 2998 :fail :read [9 nil] :RequestTimeoutException INFO [2019-12-17 12:30:57,655] jepsen worker 28 - jepsen.util 2998 :invoke :read [9 nil] INFO [2019-12-17 12:30:57,661] jepsen worker 14 - jepsen.util 4244 :fail :cas [4 [30 26]] :ValueNotSwapFrom INFO [2019-12-17 12:30:57,661] jepsen worker 14 - jepsen.util 4244 :invoke :write [4 39] INFO [2019-12-17 12:30:57,687] jepsen worker 32 - jepsen.util 4712 :fail :cas [10 [30 19]] :ValueNotSwapFrom INFO [2019-12-17 12:30:57,688] jepsen worker 32 - jepsen.util 4712 :invoke :write [10 18] INFO [2019-12-17 12:30:57,702] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":23818,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:30:57,702] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":23818,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:30:57,702] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":23821,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:30:57,702] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":23821,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:30:57,709] jepsen worker 7 - jepsen.util 5317 :info :cas [2 [22 5]] :RequestCanceledException INFO [2019-12-17 12:30:57,710] jepsen worker 7 - jepsen.util 5407 :invoke :read [2 nil] INFO [2019-12-17 12:30:57,710] jepsen worker 10 - jepsen.util 4060 :ok :read [3 2] INFO [2019-12-17 12:30:57,710] jepsen worker 10 - jepsen.util 4060 :invoke :read [3 nil] INFO [2019-12-17 12:30:57,718] jepsen worker 18 - jepsen.util 4158 :ok :read [6 26] INFO [2019-12-17 12:30:57,718] jepsen worker 18 - jepsen.util 4158 :invoke :read [6 nil] INFO [2019-12-17 12:30:57,729] jepsen worker 11 - jepsen.util 4421 :fail :cas [3 [13 7]] :ValueNotSwapFrom INFO [2019-12-17 12:30:57,729] jepsen worker 11 - jepsen.util 4421 :invoke :write [3 40] INFO [2019-12-17 12:30:57,753] jepsen worker 63 - jepsen.util 3483 :ok :write [21 27] INFO [2019-12-17 12:30:57,753] jepsen worker 63 - jepsen.util 3483 :invoke :write [21 12] INFO [2019-12-17 12:30:57,759] jepsen worker 73 - jepsen.util 3943 :ok :write [24 0] INFO [2019-12-17 12:30:57,759] jepsen worker 73 - jepsen.util 3943 :invoke :read [24 nil] INFO [2019-12-17 12:30:57,774] jepsen worker 13 - jepsen.util 3883 :ok :read [4 26] INFO [2019-12-17 12:30:57,774] jepsen worker 13 - jepsen.util 3883 :invoke :read [4 nil] INFO [2019-12-17 12:30:57,787] jepsen worker 27 - jepsen.util 4077 :ok :cas [9 [16 42]] INFO [2019-12-17 12:30:57,787] jepsen worker 27 - jepsen.util 4077 :invoke :write [9 49] INFO [2019-12-17 12:30:57,817] jepsen worker 28 - jepsen.util 2998 :ok :read [9 16] INFO [2019-12-17 12:30:57,817] jepsen worker 28 - jepsen.util 2998 :invoke :write [9 19] INFO [2019-12-17 12:30:57,825] jepsen worker 83 - jepsen.util 5573 :info :write [27 47] :RequestCanceledException INFO [2019-12-17 12:30:57,825] jepsen worker 83 - jepsen.util 5663 :invoke :write [27 10] INFO [2019-12-17 12:30:57,839] jepsen worker 50 - jepsen.util 5810 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:30:57,839] jepsen worker 50 - jepsen.util 5810 :invoke :write [16 1] INFO [2019-12-17 12:30:57,841] jepsen worker 10 - jepsen.util 4060 :ok :read [3 2] INFO [2019-12-17 12:30:57,841] jepsen worker 10 - jepsen.util 4060 :invoke :write [3 32] INFO [2019-12-17 12:30:57,855] jepsen worker 58 - jepsen.util 5638 :info :cas [19 [21 4]] :RequestCanceledException INFO [2019-12-17 12:30:57,856] jepsen worker 58 - jepsen.util 5728 :invoke :read [19 nil] INFO [2019-12-17 12:30:57,859] jepsen worker 73 - jepsen.util 3943 :ok :read [24 0] INFO [2019-12-17 12:30:57,859] jepsen worker 73 - jepsen.util 3943 :invoke :cas [24 [8 22]] INFO [2019-12-17 12:30:57,869] jepsen worker 80 - jepsen.util 4220 :fail :cas [26 [10 47]] :ValueNotSwapFrom INFO [2019-12-17 12:30:57,869] jepsen worker 80 - jepsen.util 4220 :invoke :write [26 9] INFO [2019-12-17 12:30:57,886] jepsen worker 13 - jepsen.util 3883 :ok :read [4 26] INFO [2019-12-17 12:30:57,886] jepsen worker 13 - jepsen.util 3883 :invoke :read [4 nil] INFO [2019-12-17 12:30:57,891] jepsen worker 2 - jepsen.util 5222 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:30:57,891] jepsen worker 2 - jepsen.util 5222 :invoke :cas [0 [2 11]] INFO [2019-12-17 12:30:57,907] jepsen worker 35 - jepsen.util 5435 :info :write [11 2] :RequestCanceledException INFO [2019-12-17 12:30:57,907] jepsen worker 40 - jepsen.util 6070 :info :cas [13 [32 28]] :RequestCanceledException INFO [2019-12-17 12:30:57,908] jepsen worker 40 - jepsen.util 6160 :invoke :write [13 19] INFO [2019-12-17 12:30:57,908] jepsen worker 35 - jepsen.util 5525 :invoke :cas [11 [46 22]] INFO [2019-12-17 12:30:57,921] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":23841,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:30:57,921] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":23841,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:30:57,921] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":23845,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:30:57,921] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":23845,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:30:57,922] jepsen worker 78 - jepsen.util 4488 :fail :cas [26 [21 39]] :ValueNotSwapFrom INFO [2019-12-17 12:30:57,922] jepsen worker 78 - jepsen.util 4488 :invoke :read [26 nil] INFO [2019-12-17 12:30:57,924] jepsen worker 45 - jepsen.util 4185 :info :write [15 9] :RequestCanceledException INFO [2019-12-17 12:30:57,924] jepsen worker 67 - jepsen.util 5737 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:30:57,924] jepsen worker 0 - jepsen.util 6030 :info :cas [0 [3 46]] :RequestCanceledException INFO [2019-12-17 12:30:57,924] jepsen worker 67 - jepsen.util 5737 :invoke :write [22 43] INFO [2019-12-17 12:30:57,925] jepsen worker 31 - jepsen.util 3811 :ok :write [10 2] INFO [2019-12-17 12:30:57,925] jepsen worker 31 - jepsen.util 3811 :invoke :write [10 38] INFO [2019-12-17 12:30:57,925] jepsen worker 0 - jepsen.util 6120 :invoke :cas [0 [4 31]] INFO [2019-12-17 12:30:57,925] jepsen worker 45 - jepsen.util 4275 :invoke :cas [15 [45 34]] INFO [2019-12-17 12:30:57,932] jepsen worker 73 - jepsen.util 3943 :fail :cas [24 [8 22]] :ValueNotSwapFrom INFO [2019-12-17 12:30:57,932] jepsen worker 73 - jepsen.util 3943 :invoke :write [24 1] INFO [2019-12-17 12:30:57,938] jepsen worker 60 - jepsen.util 6000 :info :cas [20 [32 5]] :RequestCanceledException INFO [2019-12-17 12:30:57,939] jepsen worker 20 - jepsen.util 4160 :ok :write [6 13] INFO [2019-12-17 12:30:57,939] jepsen worker 20 - jepsen.util 4160 :invoke :write [6 34] INFO [2019-12-17 12:30:57,939] jepsen worker 60 - jepsen.util 6090 :invoke :write [20 8] INFO [2019-12-17 12:30:57,956] jepsen worker 57 - jepsen.util 5277 :info :write [19 26] :RequestCanceledException INFO [2019-12-17 12:30:57,956] jepsen worker 26 - jepsen.util 5606 :info :write [8 11] :RequestCanceledException INFO [2019-12-17 12:30:57,957] jepsen worker 26 - jepsen.util 5696 :invoke :read [8 nil] INFO [2019-12-17 12:30:57,957] jepsen worker 57 - jepsen.util 5367 :invoke :read [19 nil] INFO [2019-12-17 12:30:57,987] jepsen worker 48 - jepsen.util 4998 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:30:57,988] jepsen worker 48 - jepsen.util 4998 :invoke :write [16 48] INFO [2019-12-17 12:30:57,994] jepsen worker 30 - jepsen.util 4440 :fail :cas [10 [41 46]] :ValueNotSwapFrom INFO [2019-12-17 12:30:57,994] jepsen worker 30 - jepsen.util 4440 :invoke :read [10 nil] INFO [2019-12-17 12:30:58,005] jepsen worker 18 - jepsen.util 4158 :ok :read [6 13] INFO [2019-12-17 12:30:58,005] jepsen worker 18 - jepsen.util 4158 :invoke :write [6 0] INFO [2019-12-17 12:30:58,008] jepsen worker 13 - jepsen.util 3883 :ok :read [4 26] INFO [2019-12-17 12:30:58,008] jepsen worker 13 - jepsen.util 3883 :invoke :cas [4 [22 43]] INFO [2019-12-17 12:30:58,009] jepsen worker 22 - jepsen.util 5512 :info :write [7 5] :RequestCanceledException INFO [2019-12-17 12:30:58,010] jepsen worker 22 - jepsen.util 5602 :invoke :write [7 1] INFO [2019-12-17 12:30:58,025] jepsen worker 38 - jepsen.util 4088 :info :write [12 32] :RequestCanceledException INFO [2019-12-17 12:30:58,025] jepsen worker 17 - jepsen.util 5237 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:30:58,025] jepsen worker 17 - jepsen.util 5237 :invoke :cas [5 [32 40]] INFO [2019-12-17 12:30:58,026] jepsen worker 38 - jepsen.util 4178 :invoke :write [12 22] INFO [2019-12-17 12:30:58,030] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":23865,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:30:58,031] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":23865,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:30:58,031] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":23867,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:30:58,031] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":23867,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:30:58,037] jepsen worker 27 - jepsen.util 4077 :ok :write [9 49] INFO [2019-12-17 12:30:58,037] jepsen worker 27 - jepsen.util 4077 :invoke :write [9 8] INFO [2019-12-17 12:30:58,038] jepsen worker 45 - jepsen.util 4275 :fail :cas [15 [45 34]] :ValueNotSwapFrom INFO [2019-12-17 12:30:58,038] jepsen worker 45 - jepsen.util 4275 :invoke :read [15 nil] INFO [2019-12-17 12:30:58,073] jepsen worker 63 - jepsen.util 3483 :ok :write [21 12] INFO [2019-12-17 12:30:58,073] jepsen worker 63 - jepsen.util 3483 :invoke :read [21 nil] INFO [2019-12-17 12:30:58,091] jepsen worker 71 - jepsen.util 4391 :info :write [23 35] :RequestCanceledException INFO [2019-12-17 12:30:58,092] jepsen worker 71 - jepsen.util 4481 :invoke :read [23 nil] INFO [2019-12-17 12:30:58,093] jepsen worker 63 - jepsen.util 3483 :ok :read [21 12] INFO [2019-12-17 12:30:58,093] jepsen worker 63 - jepsen.util 3483 :invoke :write [21 6] INFO [2019-12-17 12:30:58,124] jepsen worker 33 - jepsen.util 5523 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:30:58,125] jepsen worker 33 - jepsen.util 5523 :invoke :read [11 nil] INFO [2019-12-17 12:30:58,138] jepsen worker 42 - jepsen.util 6252 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:30:58,138] jepsen worker 42 - jepsen.util 6252 :invoke :write [14 11] INFO [2019-12-17 12:30:58,169] jepsen worker 27 - jepsen.util 4077 :ok :write [9 8] INFO [2019-12-17 12:30:58,169] jepsen worker 27 - jepsen.util 4077 :invoke :cas [9 [34 15]] INFO [2019-12-17 12:30:58,188] jepsen worker 27 - jepsen.util 4077 :fail :cas [9 [34 15]] :ValueNotSwapFrom INFO [2019-12-17 12:30:58,188] jepsen worker 27 - jepsen.util 4077 :invoke :cas [9 [23 25]] INFO [2019-12-17 12:30:58,191] jepsen worker 41 - jepsen.util 5621 :info :cas [13 [41 3]] :RequestCanceledException INFO [2019-12-17 12:30:58,193] jepsen worker 41 - jepsen.util 5711 :invoke :cas [13 [8 36]] INFO [2019-12-17 12:30:58,198] jepsen worker 63 - jepsen.util 3483 :ok :write [21 6] INFO [2019-12-17 12:30:58,199] jepsen worker 63 - jepsen.util 3483 :invoke :cas [21 [5 26]] INFO [2019-12-17 12:30:58,206] jepsen worker 27 - jepsen.util 4077 :fail :cas [9 [23 25]] :ValueNotSwapFrom INFO [2019-12-17 12:30:58,206] jepsen worker 27 - jepsen.util 4077 :invoke :write [9 12] INFO [2019-12-17 12:30:58,209] jepsen worker 75 - jepsen.util 5475 :info :cas [25 [44 40]] :RequestCanceledException INFO [2019-12-17 12:30:58,210] jepsen worker 75 - jepsen.util 5565 :invoke :write [25 10] INFO [2019-12-17 12:30:58,222] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":23893,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:30:58,222] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":23893,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:30:58,223] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":23897,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:30:58,223] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":23897,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:30:58,224] jepsen worker 63 - jepsen.util 3483 :fail :cas [21 [5 26]] :ValueNotSwapFrom INFO [2019-12-17 12:30:58,224] jepsen worker 63 - jepsen.util 3483 :invoke :read [21 nil] INFO [2019-12-17 12:30:58,224] jepsen worker 1 - jepsen.util 5131 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:30:58,225] jepsen worker 1 - jepsen.util 5131 :invoke :read [0 nil] INFO [2019-12-17 12:30:58,225] jepsen worker 8 - jepsen.util 4778 :info :write [2 49] :RequestCanceledException INFO [2019-12-17 12:30:58,226] jepsen worker 8 - jepsen.util 4868 :invoke :write [2 41] INFO [2019-12-17 12:30:58,238] jepsen worker 55 - jepsen.util 5095 :fail :cas [18 [23 8]] :ValueNotSwapFrom INFO [2019-12-17 12:30:58,238] jepsen worker 36 - jepsen.util 4266 :ok :read [12 40] INFO [2019-12-17 12:30:58,238] jepsen worker 55 - jepsen.util 5095 :invoke :write [18 32] INFO [2019-12-17 12:30:58,238] jepsen worker 36 - jepsen.util 4266 :invoke :read [12 nil] INFO [2019-12-17 12:30:58,248] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":23884,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:30:58,248] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":23884,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:30:58,248] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":23885,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:30:58,248] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":23885,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:30:58,248] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":23886,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:30:58,248] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":23886,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:30:58,256] jepsen worker 76 - jepsen.util 5656 :info :write [25 10] :RequestCanceledException INFO [2019-12-17 12:30:58,258] jepsen worker 76 - jepsen.util 5746 :invoke :read [25 nil] INFO [2019-12-17 12:30:58,261] jepsen worker 63 - jepsen.util 3483 :ok :read [21 6] INFO [2019-12-17 12:30:58,261] jepsen worker 63 - jepsen.util 3483 :invoke :cas [21 [33 9]] INFO [2019-12-17 12:30:58,279] jepsen worker 27 - jepsen.util 4077 :ok :write [9 12] INFO [2019-12-17 12:30:58,280] jepsen worker 27 - jepsen.util 4077 :invoke :write [9 19] INFO [2019-12-17 12:30:58,282] jepsen worker 63 - jepsen.util 3483 :fail :cas [21 [33 9]] :ValueNotSwapFrom INFO [2019-12-17 12:30:58,282] jepsen worker 63 - jepsen.util 3483 :invoke :read [21 nil] INFO [2019-12-17 12:30:58,305] jepsen worker 63 - jepsen.util 3483 :ok :read [21 6] INFO [2019-12-17 12:30:58,305] jepsen worker 63 - jepsen.util 3483 :invoke :write [21 48] INFO [2019-12-17 12:30:58,324] jepsen worker 16 - jepsen.util 6316 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:30:58,325] jepsen worker 16 - jepsen.util 6316 :invoke :read [5 nil] INFO [2019-12-17 12:30:58,332] jepsen worker 13 - jepsen.util 3883 :fail :cas [4 [22 43]] :ValueNotSwapFrom INFO [2019-12-17 12:30:58,332] jepsen worker 13 - jepsen.util 3883 :invoke :write [4 0] INFO [2019-12-17 12:30:58,353] jepsen worker 27 - jepsen.util 4077 :ok :write [9 19] INFO [2019-12-17 12:30:58,353] jepsen worker 27 - jepsen.util 4077 :invoke :read [9 nil] INFO [2019-12-17 12:30:58,355] jepsen worker 15 - jepsen.util 4785 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:30:58,355] jepsen worker 15 - jepsen.util 4785 :invoke :write [5 17] INFO [2019-12-17 12:30:58,367] jepsen worker 27 - jepsen.util 4077 :ok :read [9 19] INFO [2019-12-17 12:30:58,367] jepsen worker 27 - jepsen.util 4077 :invoke :write [9 4] INFO [2019-12-17 12:30:58,392] jepsen worker 63 - jepsen.util 3483 :ok :write [21 48] INFO [2019-12-17 12:30:58,392] jepsen worker 63 - jepsen.util 3483 :invoke :read [21 nil] INFO [2019-12-17 12:30:58,416] jepsen worker 63 - jepsen.util 3483 :ok :read [21 48] INFO [2019-12-17 12:30:58,416] jepsen worker 63 - jepsen.util 3483 :invoke :read [21 nil] INFO [2019-12-17 12:30:58,425] jepsen worker 27 - jepsen.util 4077 :ok :write [9 4] INFO [2019-12-17 12:30:58,425] jepsen worker 5 - jepsen.util 3785 :info :write [1 40] :RequestCanceledException INFO [2019-12-17 12:30:58,425] jepsen worker 51 - jepsen.util 3831 :info :write [17 22] :RequestCanceledException INFO [2019-12-17 12:30:58,425] jepsen worker 27 - jepsen.util 4077 :invoke :cas [9 [46 21]] INFO [2019-12-17 12:30:58,426] jepsen worker 5 - jepsen.util 3875 :invoke :read [1 nil] INFO [2019-12-17 12:30:58,426] jepsen worker 51 - jepsen.util 3921 :invoke :write [17 28] INFO [2019-12-17 12:30:58,428] jepsen worker 63 - jepsen.util 3483 :ok :read [21 48] INFO [2019-12-17 12:30:58,428] jepsen worker 63 - jepsen.util 3483 :invoke :read [21 nil] INFO [2019-12-17 12:30:58,439] jepsen worker 81 - jepsen.util 5391 :info :cas [27 [16 47]] :RequestCanceledException INFO [2019-12-17 12:30:58,440] jepsen worker 81 - jepsen.util 5481 :invoke :write [27 39] INFO [2019-12-17 12:30:58,442] jepsen worker 27 - jepsen.util 4077 :fail :cas [9 [46 21]] :ValueNotSwapFrom INFO [2019-12-17 12:30:58,442] jepsen worker 27 - jepsen.util 4077 :invoke :read [9 nil] INFO [2019-12-17 12:30:58,451] jepsen worker 63 - jepsen.util 3483 :ok :read [21 48] INFO [2019-12-17 12:30:58,452] jepsen worker 63 - jepsen.util 3483 :invoke :read [21 nil] INFO [2019-12-17 12:30:58,458] jepsen worker 27 - jepsen.util 4077 :ok :read [9 4] INFO [2019-12-17 12:30:58,458] jepsen worker 27 - jepsen.util 4077 :invoke :cas [9 [19 38]] INFO [2019-12-17 12:30:58,473] jepsen worker 63 - jepsen.util 3483 :ok :read [21 48] INFO [2019-12-17 12:30:58,473] jepsen worker 63 - jepsen.util 3483 :invoke :read [21 nil] INFO [2019-12-17 12:30:58,485] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:30:58,485] jepsen nemesis - couchbase.util Rebalance check count: 177 INFO [2019-12-17 12:30:58,486] jepsen nemesis - couchbase.util Rebalance stuck count: 158 INFO [2019-12-17 12:30:58,486] jepsen worker 78 - jepsen.util 4488 :ok :read [26 32] INFO [2019-12-17 12:30:58,486] jepsen worker 78 - jepsen.util 4488 :invoke :cas [26 [31 21]] INFO [2019-12-17 12:30:58,488] jepsen worker 27 - jepsen.util 4077 :fail :cas [9 [19 38]] :ValueNotSwapFrom INFO [2019-12-17 12:30:58,488] jepsen worker 27 - jepsen.util 4077 :invoke :write [9 10] INFO [2019-12-17 12:30:58,493] jepsen worker 63 - jepsen.util 3483 :ok :read [21 48] INFO [2019-12-17 12:30:58,493] jepsen worker 63 - jepsen.util 3483 :invoke :read [21 nil] INFO [2019-12-17 12:30:58,509] jepsen worker 64 - jepsen.util 3304 :fail :read [21 nil] :RequestTimeoutException INFO [2019-12-17 12:30:58,509] jepsen worker 64 - jepsen.util 3304 :invoke :cas [21 [17 1]] INFO [2019-12-17 12:30:58,509] jepsen worker 71 - jepsen.util 4481 :ok :read [23 20] INFO [2019-12-17 12:30:58,509] jepsen worker 71 - jepsen.util 4481 :invoke :write [23 4] INFO [2019-12-17 12:30:58,515] jepsen worker 63 - jepsen.util 3483 :ok :read [21 48] INFO [2019-12-17 12:30:58,515] jepsen worker 63 - jepsen.util 3483 :invoke :write [21 46] INFO [2019-12-17 12:30:58,523] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":23919,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:30:58,523] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":23919,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:30:58,523] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":23923,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:30:58,523] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":23923,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:30:58,525] jepsen worker 21 - jepsen.util 5511 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:30:58,525] jepsen worker 21 - jepsen.util 5511 :invoke :read [7 nil] INFO [2019-12-17 12:30:58,531] jepsen worker 64 - jepsen.util 3304 :fail :cas [21 [17 1]] :ValueNotSwapFrom INFO [2019-12-17 12:30:58,532] jepsen worker 64 - jepsen.util 3304 :invoke :write [21 38] INFO [2019-12-17 12:30:58,572] jepsen worker 27 - jepsen.util 4077 :ok :write [9 10] INFO [2019-12-17 12:30:58,572] jepsen worker 27 - jepsen.util 4077 :invoke :write [9 18] INFO [2019-12-17 12:30:58,578] jepsen worker 63 - jepsen.util 3483 :ok :write [21 46] INFO [2019-12-17 12:30:58,578] jepsen worker 63 - jepsen.util 3483 :invoke :cas [21 [20 1]] INFO [2019-12-17 12:30:58,587] jepsen worker 36 - jepsen.util 4266 :ok :read [12 40] INFO [2019-12-17 12:30:58,588] jepsen worker 36 - jepsen.util 4266 :invoke :cas [12 [28 49]] INFO [2019-12-17 12:30:58,588] jepsen worker 5 - jepsen.util 3875 :ok :read [1 9] INFO [2019-12-17 12:30:58,588] jepsen worker 5 - jepsen.util 3875 :invoke :read [1 nil] INFO [2019-12-17 12:30:58,594] jepsen worker 63 - jepsen.util 3483 :fail :cas [21 [20 1]] :ValueNotSwapFrom INFO [2019-12-17 12:30:58,594] jepsen worker 63 - jepsen.util 3483 :invoke :cas [21 [6 44]] INFO [2019-12-17 12:30:58,610] jepsen worker 72 - jepsen.util 4302 :info :write [24 30] :RequestCanceledException INFO [2019-12-17 12:30:58,611] jepsen worker 72 - jepsen.util 4392 :invoke :write [24 3] INFO [2019-12-17 12:30:58,629] jepsen worker 63 - jepsen.util 3483 :fail :cas [21 [6 44]] :ValueNotSwapFrom INFO [2019-12-17 12:30:58,629] jepsen worker 63 - jepsen.util 3483 :invoke :write [21 32] INFO [2019-12-17 12:30:58,632] jepsen worker 27 - jepsen.util 4077 :ok :write [9 18] INFO [2019-12-17 12:30:58,632] jepsen worker 27 - jepsen.util 4077 :invoke :cas [9 [18 44]] INFO [2019-12-17 12:30:58,638] jepsen worker 61 - jepsen.util 6181 :info :cas [20 [30 34]] :RequestCanceledException INFO [2019-12-17 12:30:58,639] jepsen worker 61 - jepsen.util 6271 :invoke :read [20 nil] INFO [2019-12-17 12:30:58,681] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":23950,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:30:58,681] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":23950,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:30:58,681] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":23954,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:30:58,681] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":23954,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:30:58,692] jepsen worker 65 - jepsen.util 3485 :info :write [21 0] :RequestCanceledException INFO [2019-12-17 12:30:58,692] jepsen worker 37 - jepsen.util 3907 :info :write [12 3] :RequestCanceledException INFO [2019-12-17 12:30:58,693] jepsen worker 65 - jepsen.util 3575 :invoke :read [21 nil] INFO [2019-12-17 12:30:58,693] jepsen worker 37 - jepsen.util 3997 :invoke :cas [12 [16 16]] INFO [2019-12-17 12:30:58,707] jepsen worker 56 - jepsen.util 4286 :info :write [18 4] :RequestCanceledException INFO [2019-12-17 12:30:58,709] jepsen worker 56 - jepsen.util 4376 :invoke :write [18 38] INFO [2019-12-17 12:30:58,733] jepsen worker 63 - jepsen.util 3483 :ok :write [21 32] INFO [2019-12-17 12:30:58,733] jepsen worker 63 - jepsen.util 3483 :invoke :read [21 nil] INFO [2019-12-17 12:30:58,750] jepsen worker 27 - jepsen.util 4077 :ok :cas [9 [18 44]] INFO [2019-12-17 12:30:58,750] jepsen worker 27 - jepsen.util 4077 :invoke :write [9 30] INFO [2019-12-17 12:30:58,762] jepsen worker 63 - jepsen.util 3483 :ok :read [21 32] INFO [2019-12-17 12:30:58,762] jepsen worker 63 - jepsen.util 3483 :invoke :read [21 nil] INFO [2019-12-17 12:30:58,798] jepsen worker 63 - jepsen.util 3483 :ok :read [21 32] INFO [2019-12-17 12:30:58,799] jepsen worker 63 - jepsen.util 3483 :invoke :read [21 nil] INFO [2019-12-17 12:30:58,812] jepsen worker 65 - jepsen.util 3575 :ok :read [21 32] INFO [2019-12-17 12:30:58,813] jepsen worker 65 - jepsen.util 3575 :invoke :read [21 nil] INFO [2019-12-17 12:30:58,828] jepsen worker 63 - jepsen.util 3483 :ok :read [21 32] INFO [2019-12-17 12:30:58,828] jepsen worker 63 - jepsen.util 3483 :invoke :cas [21 [28 45]] INFO [2019-12-17 12:30:58,838] jepsen worker 74 - jepsen.util 4664 :info :write [24 14] :RequestCanceledException INFO [2019-12-17 12:30:58,839] jepsen worker 74 - jepsen.util 4754 :invoke :write [24 34] INFO [2019-12-17 12:30:58,856] jepsen worker 69 - jepsen.util 4209 :info :write [23 19] :RequestCanceledException INFO [2019-12-17 12:30:58,857] jepsen worker 69 - jepsen.util 4299 :invoke :cas [23 [32 26]] INFO [2019-12-17 12:30:58,892] jepsen worker 77 - jepsen.util 5477 :info :write [25 39] :RequestCanceledException INFO [2019-12-17 12:30:58,892] jepsen worker 63 - jepsen.util 3483 :fail :cas [21 [28 45]] :ValueNotSwapFrom INFO [2019-12-17 12:30:58,892] jepsen worker 63 - jepsen.util 3483 :invoke :cas [21 [37 16]] INFO [2019-12-17 12:30:58,893] jepsen worker 27 - jepsen.util 4077 :ok :write [9 30] INFO [2019-12-17 12:30:58,893] jepsen worker 27 - jepsen.util 4077 :invoke :write [9 49] INFO [2019-12-17 12:30:58,893] jepsen worker 77 - jepsen.util 5567 :invoke :cas [25 [47 44]] INFO [2019-12-17 12:30:58,900] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":23970,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:30:58,900] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":23970,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:30:58,901] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":23972,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:30:58,901] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":23972,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:30:58,915] jepsen worker 63 - jepsen.util 3483 :fail :cas [21 [37 16]] :ValueNotSwapFrom INFO [2019-12-17 12:30:58,915] jepsen worker 63 - jepsen.util 3483 :invoke :cas [21 [25 7]] INFO [2019-12-17 12:30:58,925] jepsen worker 86 - jepsen.util 5756 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:30:58,925] jepsen worker 70 - jepsen.util 4840 :info :write [23 47] :RequestCanceledException INFO [2019-12-17 12:30:58,925] jepsen worker 86 - jepsen.util 5756 :invoke :read [28 nil] INFO [2019-12-17 12:30:58,926] jepsen worker 70 - jepsen.util 4930 :invoke :read [23 nil] INFO [2019-12-17 12:30:58,939] jepsen worker 36 - jepsen.util 4266 :fail :cas [12 [28 49]] :ValueNotSwapFrom INFO [2019-12-17 12:30:58,939] jepsen worker 24 - jepsen.util 5334 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:30:58,939] jepsen worker 36 - jepsen.util 4266 :invoke :write [12 5] INFO [2019-12-17 12:30:58,939] jepsen worker 24 - jepsen.util 5334 :invoke :write [8 12] INFO [2019-12-17 12:30:58,947] jepsen worker 63 - jepsen.util 3483 :fail :cas [21 [25 7]] :ValueNotSwapFrom INFO [2019-12-17 12:30:58,947] jepsen worker 63 - jepsen.util 3483 :invoke :read [21 nil] INFO [2019-12-17 12:30:58,956] jepsen worker 87 - jepsen.util 3867 :info :write [29 40] :RequestCanceledException INFO [2019-12-17 12:30:58,956] jepsen worker 9 - jepsen.util 4689 :info :write [3 43] :RequestCanceledException INFO [2019-12-17 12:30:58,956] jepsen worker 29 - jepsen.util 3539 :info :cas [9 [33 3]] :RequestCanceledException INFO [2019-12-17 12:30:58,957] jepsen worker 29 - jepsen.util 3629 :invoke :read [9 nil] INFO [2019-12-17 12:30:58,957] jepsen worker 87 - jepsen.util 3957 :invoke :write [29 38] INFO [2019-12-17 12:30:58,957] jepsen worker 9 - jepsen.util 4779 :invoke :write [3 13] INFO [2019-12-17 12:30:58,962] jepsen worker 27 - jepsen.util 4077 :ok :write [9 49] INFO [2019-12-17 12:30:58,963] jepsen worker 27 - jepsen.util 4077 :invoke :read [9 nil] INFO [2019-12-17 12:30:58,974] jepsen worker 63 - jepsen.util 3483 :ok :read [21 32] INFO [2019-12-17 12:30:58,974] jepsen worker 63 - jepsen.util 3483 :invoke :read [21 nil] INFO [2019-12-17 12:30:58,991] jepsen worker 49 - jepsen.util 4909 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:30:58,992] jepsen worker 49 - jepsen.util 4909 :invoke :read [16 nil] INFO [2019-12-17 12:30:59,000] jepsen worker 78 - jepsen.util 4488 :fail :cas [26 [31 21]] :ValueNotSwapFrom INFO [2019-12-17 12:30:59,000] jepsen worker 78 - jepsen.util 4488 :invoke :cas [26 [39 18]] INFO [2019-12-17 12:30:59,010] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":23988,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:30:59,010] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":23988,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:30:59,010] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":23992,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:30:59,010] jepsen worker 63 - jepsen.util 3483 :ok :read [21 32] INFO [2019-12-17 12:30:59,011] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":23992,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:30:59,011] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":23993,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:30:59,011] jepsen worker 63 - jepsen.util 3483 :invoke :cas [21 [16 37]] INFO [2019-12-17 12:30:59,011] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":23993,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:30:59,029] jepsen worker 31 - jepsen.util 3811 :ok :write [10 38] INFO [2019-12-17 12:30:59,029] jepsen worker 31 - jepsen.util 3811 :invoke :cas [10 [32 16]] INFO [2019-12-17 12:30:59,032] jepsen worker 63 - jepsen.util 3483 :fail :cas [21 [16 37]] :ValueNotSwapFrom INFO [2019-12-17 12:30:59,033] jepsen worker 63 - jepsen.util 3483 :invoke :read [21 nil] INFO [2019-12-17 12:30:59,033] jepsen worker 73 - jepsen.util 3943 :ok :write [24 1] INFO [2019-12-17 12:30:59,033] jepsen worker 73 - jepsen.util 3943 :invoke :cas [24 [15 44]] INFO [2019-12-17 12:30:59,033] jepsen worker 11 - jepsen.util 4421 :ok :write [3 40] INFO [2019-12-17 12:30:59,034] jepsen worker 11 - jepsen.util 4421 :invoke :cas [3 [20 2]] INFO [2019-12-17 12:30:59,036] jepsen worker 27 - jepsen.util 4077 :ok :read [9 49] INFO [2019-12-17 12:30:59,037] jepsen worker 27 - jepsen.util 4077 :invoke :read [9 nil] INFO [2019-12-17 12:30:59,056] jepsen worker 63 - jepsen.util 3483 :ok :read [21 32] INFO [2019-12-17 12:30:59,056] jepsen worker 63 - jepsen.util 3483 :invoke :cas [21 [18 22]] INFO [2019-12-17 12:30:59,056] jepsen worker 47 - jepsen.util 4637 :info :write [15 22] :RequestCanceledException INFO [2019-12-17 12:30:59,065] jepsen worker 45 - jepsen.util 4275 :ok :read [15 37] INFO [2019-12-17 12:30:59,065] jepsen worker 45 - jepsen.util 4275 :invoke :read [15 nil] INFO [2019-12-17 12:30:59,068] jepsen worker 47 - jepsen.util 4727 :invoke :cas [15 [2 20]] INFO [2019-12-17 12:30:59,072] jepsen worker 27 - jepsen.util 4077 :ok :read [9 49] INFO [2019-12-17 12:30:59,072] jepsen worker 27 - jepsen.util 4077 :invoke :cas [9 [6 16]] INFO [2019-12-17 12:30:59,110] jepsen worker 3 - jepsen.util 4323 :info :write [1 27] :RequestCanceledException INFO [2019-12-17 12:30:59,110] jepsen worker 53 - jepsen.util 4193 :info :write [17 44] :RequestCanceledException INFO [2019-12-17 12:30:59,111] jepsen worker 53 - jepsen.util 4283 :invoke :read [17 nil] INFO [2019-12-17 12:30:59,111] jepsen worker 3 - jepsen.util 4413 :invoke :cas [1 [5 17]] INFO [2019-12-17 12:30:59,118] jepsen worker 29 - jepsen.util 3629 :ok :read [9 49] INFO [2019-12-17 12:30:59,119] jepsen worker 29 - jepsen.util 3629 :invoke :cas [9 [7 46]] INFO [2019-12-17 12:30:59,120] jepsen worker 27 - jepsen.util 4077 :fail :cas [9 [6 16]] :ValueNotSwapFrom INFO [2019-12-17 12:30:59,120] jepsen worker 27 - jepsen.util 4077 :invoke :read [9 nil] INFO [2019-12-17 12:30:59,124] jepsen worker 63 - jepsen.util 3483 :fail :cas [21 [18 22]] :ValueNotSwapFrom INFO [2019-12-17 12:30:59,124] jepsen worker 63 - jepsen.util 3483 :invoke :write [21 1] INFO [2019-12-17 12:30:59,132] jepsen worker 31 - jepsen.util 3811 :fail :cas [10 [32 16]] :ValueNotSwapFrom INFO [2019-12-17 12:30:59,132] jepsen worker 31 - jepsen.util 3811 :invoke :read [10 nil] INFO [2019-12-17 12:30:59,173] jepsen worker 5 - jepsen.util 3875 :ok :read [1 9] INFO [2019-12-17 12:30:59,173] jepsen worker 5 - jepsen.util 3875 :invoke :cas [1 [0 44]] INFO [2019-12-17 12:30:59,176] jepsen worker 47 - jepsen.util 4727 :fail :cas [15 [2 20]] :ValueNotSwapFrom INFO [2019-12-17 12:30:59,177] jepsen worker 47 - jepsen.util 4727 :invoke :cas [15 [8 8]] INFO [2019-12-17 12:30:59,181] jepsen worker 73 - jepsen.util 3943 :fail :cas [24 [15 44]] :ValueNotSwapFrom INFO [2019-12-17 12:30:59,181] jepsen worker 73 - jepsen.util 3943 :invoke :write [24 17] INFO [2019-12-17 12:30:59,189] jepsen worker 34 - jepsen.util 5164 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:30:59,189] jepsen worker 88 - jepsen.util 4228 :info :write [29 39] :RequestCanceledException INFO [2019-12-17 12:30:59,190] jepsen worker 34 - jepsen.util 5164 :invoke :cas [11 [6 1]] INFO [2019-12-17 12:30:59,191] jepsen worker 88 - jepsen.util 4318 :invoke :write [29 4] INFO [2019-12-17 12:30:59,192] jepsen worker 37 - jepsen.util 3997 :fail :cas [12 [16 16]] :ValueNotSwapFrom INFO [2019-12-17 12:30:59,192] jepsen worker 37 - jepsen.util 3997 :invoke :cas [12 [32 4]] INFO [2019-12-17 12:30:59,200] jepsen worker 27 - jepsen.util 4077 :ok :read [9 49] INFO [2019-12-17 12:30:59,200] jepsen worker 27 - jepsen.util 4077 :invoke :read [9 nil] INFO [2019-12-17 12:30:59,230] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":24015,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:30:59,230] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":24015,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:30:59,230] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":24016,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:30:59,230] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":24016,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:30:59,230] jepsen worker 29 - jepsen.util 3629 :fail :cas [9 [7 46]] :ValueNotSwapFrom INFO [2019-12-17 12:30:59,230] jepsen worker 29 - jepsen.util 3629 :invoke :write [9 12] INFO [2019-12-17 12:30:59,238] jepsen worker 46 - jepsen.util 4276 :fail :read [15 nil] :RequestTimeoutException INFO [2019-12-17 12:30:59,238] jepsen worker 52 - jepsen.util 4372 :info :write [17 27] :RequestCanceledException INFO [2019-12-17 12:30:59,238] jepsen worker 46 - jepsen.util 4276 :invoke :read [15 nil] INFO [2019-12-17 12:30:59,240] jepsen worker 52 - jepsen.util 4462 :invoke :read [17 nil] INFO [2019-12-17 12:30:59,241] jepsen worker 45 - jepsen.util 4275 :ok :read [15 37] INFO [2019-12-17 12:30:59,242] jepsen worker 45 - jepsen.util 4275 :invoke :read [15 nil] INFO [2019-12-17 12:30:59,244] jepsen worker 37 - jepsen.util 3997 :fail :cas [12 [32 4]] :ValueNotSwapFrom INFO [2019-12-17 12:30:59,244] jepsen worker 37 - jepsen.util 3997 :invoke :write [12 42] INFO [2019-12-17 12:30:59,252] jepsen worker 27 - jepsen.util 4077 :ok :read [9 49] INFO [2019-12-17 12:30:59,252] jepsen worker 27 - jepsen.util 4077 :invoke :read [9 nil] INFO [2019-12-17 12:30:59,261] jepsen worker 31 - jepsen.util 3811 :ok :read [10 38] INFO [2019-12-17 12:30:59,262] jepsen worker 31 - jepsen.util 3811 :invoke :write [10 6] INFO [2019-12-17 12:30:59,283] jepsen worker 20 - jepsen.util 4160 :ok :write [6 34] INFO [2019-12-17 12:30:59,283] jepsen worker 20 - jepsen.util 4160 :invoke :write [6 41] INFO [2019-12-17 12:30:59,291] jepsen worker 69 - jepsen.util 4299 :fail :cas [23 [32 26]] :ValueNotSwapFrom INFO [2019-12-17 12:30:59,291] jepsen worker 69 - jepsen.util 4299 :invoke :read [23 nil] INFO [2019-12-17 12:30:59,291] jepsen worker 68 - jepsen.util 5018 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:30:59,291] jepsen worker 54 - jepsen.util 4464 :info :write [18 9] :RequestCanceledException INFO [2019-12-17 12:30:59,291] jepsen worker 68 - jepsen.util 5018 :invoke :cas [22 [13 35]] INFO [2019-12-17 12:30:59,292] jepsen worker 54 - jepsen.util 4554 :invoke :write [18 33] INFO [2019-12-17 12:30:59,305] jepsen worker 63 - jepsen.util 3483 :ok :write [21 1] INFO [2019-12-17 12:30:59,305] jepsen worker 63 - jepsen.util 3483 :invoke :cas [21 [3 29]] INFO [2019-12-17 12:30:59,311] jepsen worker 73 - jepsen.util 3943 :ok :write [24 17] INFO [2019-12-17 12:30:59,312] jepsen worker 73 - jepsen.util 3943 :invoke :write [24 12] INFO [2019-12-17 12:30:59,313] jepsen worker 11 - jepsen.util 4421 :fail :cas [3 [20 2]] :ValueNotSwapFrom INFO [2019-12-17 12:30:59,313] jepsen worker 11 - jepsen.util 4421 :invoke :write [3 33] INFO [2019-12-17 12:30:59,320] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":24027,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:30:59,320] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":24027,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:30:59,320] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":24029,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:30:59,320] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":24029,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:30:59,323] jepsen worker 45 - jepsen.util 4275 :ok :read [15 37] INFO [2019-12-17 12:30:59,323] jepsen worker 45 - jepsen.util 4275 :invoke :cas [15 [9 19]] INFO [2019-12-17 12:30:59,324] jepsen worker 89 - jepsen.util 3959 :info :write [29 44] :RequestCanceledException INFO [2019-12-17 12:30:59,325] jepsen worker 4 - jepsen.util 3334 :info :write [1 44] :RequestCanceledException INFO [2019-12-17 12:30:59,325] jepsen worker 89 - jepsen.util 4049 :invoke :write [29 37] INFO [2019-12-17 12:30:59,327] jepsen worker 4 - jepsen.util 3424 :invoke :cas [1 [32 25]] INFO [2019-12-17 12:30:59,339] jepsen worker 70 - jepsen.util 4930 :ok :read [23 20] INFO [2019-12-17 12:30:59,339] jepsen worker 70 - jepsen.util 4930 :invoke :cas [23 [41 1]] INFO [2019-12-17 12:30:59,356] jepsen worker 23 - jepsen.util 5423 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:30:59,356] jepsen worker 23 - jepsen.util 5423 :invoke :cas [7 [23 41]] INFO [2019-12-17 12:30:59,359] jepsen worker 46 - jepsen.util 4276 :ok :read [15 37] INFO [2019-12-17 12:30:59,359] jepsen worker 46 - jepsen.util 4276 :invoke :write [15 20] INFO [2019-12-17 12:30:59,364] jepsen worker 63 - jepsen.util 3483 :fail :cas [21 [3 29]] :ValueNotSwapFrom INFO [2019-12-17 12:30:59,364] jepsen worker 63 - jepsen.util 3483 :invoke :cas [21 [0 16]] INFO [2019-12-17 12:30:59,373] jepsen worker 27 - jepsen.util 4077 :ok :read [9 49] INFO [2019-12-17 12:30:59,373] jepsen worker 27 - jepsen.util 4077 :invoke :read [9 nil] INFO [2019-12-17 12:30:59,384] jepsen worker 63 - jepsen.util 3483 :fail :cas [21 [0 16]] :ValueNotSwapFrom INFO [2019-12-17 12:30:59,384] jepsen worker 63 - jepsen.util 3483 :invoke :read [21 nil] INFO [2019-12-17 12:30:59,392] jepsen worker 20 - jepsen.util 4160 :ok :write [6 41] INFO [2019-12-17 12:30:59,393] jepsen worker 20 - jepsen.util 4160 :invoke :write [6 21] INFO [2019-12-17 12:30:59,412] jepsen worker 29 - jepsen.util 3629 :ok :write [9 12] INFO [2019-12-17 12:30:59,413] jepsen worker 29 - jepsen.util 3629 :invoke :cas [9 [1 14]] INFO [2019-12-17 12:30:59,424] jepsen worker 12 - jepsen.util 4512 :info :write [4 9] :RequestCanceledException INFO [2019-12-17 12:30:59,426] jepsen worker 12 - jepsen.util 4602 :invoke :cas [4 [30 6]] INFO [2019-12-17 12:30:59,438] jepsen worker 29 - jepsen.util 3629 :fail :cas [9 [1 14]] :ValueNotSwapFrom INFO [2019-12-17 12:30:59,438] jepsen worker 29 - jepsen.util 3629 :invoke :write [9 12] INFO [2019-12-17 12:30:59,474] jepsen worker 3 - jepsen.util 4413 :fail :cas [1 [5 17]] :ValueNotSwapFrom INFO [2019-12-17 12:30:59,474] jepsen worker 3 - jepsen.util 4413 :invoke :read [1 nil] INFO [2019-12-17 12:30:59,486] jepsen worker 63 - jepsen.util 3483 :ok :read [21 1] INFO [2019-12-17 12:30:59,486] jepsen worker 63 - jepsen.util 3483 :invoke :read [21 nil] INFO [2019-12-17 12:30:59,491] jepsen worker 44 - jepsen.util 5894 :info :write [14 3] :RequestCanceledException INFO [2019-12-17 12:30:59,498] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":24057,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:30:59,498] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":24057,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:30:59,499] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":24060,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:30:59,499] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":24060,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:30:59,500] jepsen worker 63 - jepsen.util 3483 :ok :read [21 1] INFO [2019-12-17 12:30:59,500] jepsen worker 63 - jepsen.util 3483 :invoke :write [21 42] INFO [2019-12-17 12:30:59,505] jepsen worker 44 - jepsen.util 5984 :invoke :write [14 24] INFO [2019-12-17 12:30:59,513] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:30:59,513] jepsen nemesis - couchbase.util Rebalance check count: 178 INFO [2019-12-17 12:30:59,513] jepsen nemesis - couchbase.util Rebalance stuck count: 159 INFO [2019-12-17 12:30:59,523] jepsen worker 29 - jepsen.util 3629 :ok :write [9 12] INFO [2019-12-17 12:30:59,523] jepsen worker 31 - jepsen.util 3811 :ok :write [10 6] INFO [2019-12-17 12:30:59,523] jepsen worker 29 - jepsen.util 3629 :invoke :read [9 nil] INFO [2019-12-17 12:30:59,523] jepsen worker 31 - jepsen.util 3811 :invoke :cas [10 [1 28]] INFO [2019-12-17 12:30:59,525] jepsen worker 59 - jepsen.util 5459 :info :cas [19 [27 10]] :RequestCanceledException INFO [2019-12-17 12:30:59,527] jepsen worker 59 - jepsen.util 5549 :invoke :cas [19 [31 24]] INFO [2019-12-17 12:30:59,539] jepsen worker 43 - jepsen.util 5083 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:30:59,539] jepsen worker 43 - jepsen.util 5083 :invoke :write [14 20] INFO [2019-12-17 12:30:59,545] jepsen worker 70 - jepsen.util 4930 :fail :cas [23 [41 1]] :ValueNotSwapFrom INFO [2019-12-17 12:30:59,545] jepsen worker 70 - jepsen.util 4930 :invoke :write [23 41] INFO [2019-12-17 12:30:59,552] jepsen worker 29 - jepsen.util 3629 :ok :read [9 12] INFO [2019-12-17 12:30:59,552] jepsen worker 29 - jepsen.util 3629 :invoke :write [9 10] INFO [2019-12-17 12:30:59,572] jepsen worker 78 - jepsen.util 4488 :fail :cas [26 [39 18]] :ValueNotSwapFrom INFO [2019-12-17 12:30:59,572] jepsen worker 78 - jepsen.util 4488 :invoke :read [26 nil] INFO [2019-12-17 12:30:59,573] jepsen worker 31 - jepsen.util 3811 :fail :cas [10 [1 28]] :ValueNotSwapFrom INFO [2019-12-17 12:30:59,573] jepsen worker 31 - jepsen.util 3811 :invoke :read [10 nil] INFO [2019-12-17 12:30:59,582] jepsen worker 69 - jepsen.util 4299 :ok :read [23 20] INFO [2019-12-17 12:30:59,582] jepsen worker 69 - jepsen.util 4299 :invoke :read [23 nil] INFO [2019-12-17 12:30:59,584] jepsen worker 53 - jepsen.util 4283 :ok :read [17 18] INFO [2019-12-17 12:30:59,584] jepsen worker 53 - jepsen.util 4283 :invoke :read [17 nil] INFO [2019-12-17 12:30:59,586] jepsen worker 12 - jepsen.util 4602 :fail :cas [4 [30 6]] :ValueNotSwapFrom INFO [2019-12-17 12:30:59,587] jepsen worker 12 - jepsen.util 4602 :invoke :read [4 nil] INFO [2019-12-17 12:30:59,591] jepsen worker 84 - jepsen.util 5034 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:30:59,591] jepsen worker 66 - jepsen.util 5286 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:30:59,591] jepsen worker 84 - jepsen.util 5034 :invoke :cas [28 [17 46]] INFO [2019-12-17 12:30:59,591] jepsen worker 66 - jepsen.util 5286 :invoke :cas [22 [34 15]] INFO [2019-12-17 12:30:59,596] jepsen worker 31 - jepsen.util 3811 :ok :read [10 6] INFO [2019-12-17 12:30:59,596] jepsen worker 31 - jepsen.util 3811 :invoke :cas [10 [32 16]] INFO [2019-12-17 12:30:59,609] jepsen worker 25 - jepsen.util 5695 :info :write [8 25] :RequestCanceledException INFO [2019-12-17 12:30:59,609] jepsen worker 6 - jepsen.util 6126 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:30:59,609] jepsen worker 6 - jepsen.util 6126 :invoke :write [2 43] INFO [2019-12-17 12:30:59,610] jepsen worker 25 - jepsen.util 5785 :invoke :write [8 46] INFO [2019-12-17 12:30:59,626] jepsen worker 31 - jepsen.util 3811 :fail :cas [10 [32 16]] :ValueNotSwapFrom INFO [2019-12-17 12:30:59,627] jepsen worker 31 - jepsen.util 3811 :invoke :write [10 1] INFO [2019-12-17 12:30:59,656] jepsen worker 85 - jepsen.util 6115 :info :write [28 29] :RequestCanceledException INFO [2019-12-17 12:30:59,657] jepsen worker 85 - jepsen.util 6205 :invoke :write [28 24] INFO [2019-12-17 12:30:59,665] jepsen worker 3 - jepsen.util 4413 :ok :read [1 9] INFO [2019-12-17 12:30:59,665] jepsen worker 3 - jepsen.util 4413 :invoke :cas [1 [24 44]] INFO [2019-12-17 12:30:59,668] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":24088,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:30:59,668] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":24088,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:30:59,671] jepsen worker 63 - jepsen.util 3483 :ok :write [21 42] INFO [2019-12-17 12:30:59,671] jepsen worker 63 - jepsen.util 3483 :invoke :cas [21 [44 2]] INFO [2019-12-17 12:30:59,675] jepsen worker 29 - jepsen.util 3629 :ok :write [9 10] INFO [2019-12-17 12:30:59,675] jepsen worker 29 - jepsen.util 3629 :invoke :cas [9 [10 45]] INFO [2019-12-17 12:30:59,709] jepsen worker 19 - jepsen.util 4249 :info :write [6 46] :RequestCanceledException INFO [2019-12-17 12:30:59,709] jepsen worker 79 - jepsen.util 4129 :info :write [26 30] :RequestCanceledException INFO [2019-12-17 12:30:59,709] jepsen worker 14 - jepsen.util 4244 :info :write [4 39] :RequestCanceledException INFO [2019-12-17 12:30:59,711] jepsen worker 19 - jepsen.util 4339 :invoke :cas [6 [20 44]] INFO [2019-12-17 12:30:59,714] jepsen worker 79 - jepsen.util 4219 :invoke :cas [26 [40 40]] INFO [2019-12-17 12:30:59,714] jepsen worker 4 - jepsen.util 3424 :fail :cas [1 [32 25]] :ValueNotSwapFrom INFO [2019-12-17 12:30:59,715] jepsen worker 4 - jepsen.util 3424 :invoke :read [1 nil] INFO [2019-12-17 12:30:59,719] jepsen worker 14 - jepsen.util 4334 :invoke :write [4 24] INFO [2019-12-17 12:30:59,720] jepsen worker 63 - jepsen.util 3483 :fail :cas [21 [44 2]] :ValueNotSwapFrom INFO [2019-12-17 12:30:59,720] jepsen worker 63 - jepsen.util 3483 :invoke :read [21 nil] INFO [2019-12-17 12:30:59,722] jepsen worker 62 - jepsen.util 5102 :info :cas [20 [21 39]] :RequestCanceledException INFO [2019-12-17 12:30:59,722] jepsen worker 32 - jepsen.util 4712 :info :write [10 18] :RequestCanceledException INFO [2019-12-17 12:30:59,722] jepsen worker 7 - jepsen.util 5407 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:30:59,722] jepsen worker 7 - jepsen.util 5407 :invoke :write [2 5] INFO [2019-12-17 12:30:59,724] jepsen worker 32 - jepsen.util 4802 :invoke :write [10 44] INFO [2019-12-17 12:30:59,724] jepsen worker 62 - jepsen.util 5192 :invoke :cas [20 [31 49]] INFO [2019-12-17 12:30:59,738] jepsen worker 82 - jepsen.util 6382 :info :cas [27 [35 2]] :RequestCanceledException INFO [2019-12-17 12:30:59,738] jepsen worker 39 - jepsen.util 5079 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:30:59,738] jepsen worker 39 - jepsen.util 5079 :invoke :write [13 48] INFO [2019-12-17 12:30:59,739] jepsen worker 82 - jepsen.util 6472 :invoke :write [27 7] INFO [2019-12-17 12:30:59,751] jepsen worker 63 - jepsen.util 3483 :ok :read [21 42] INFO [2019-12-17 12:30:59,751] jepsen worker 63 - jepsen.util 3483 :invoke :write [21 46] INFO [2019-12-17 12:30:59,769] jepsen worker 12 - jepsen.util 4602 :ok :read [4 26] INFO [2019-12-17 12:30:59,769] jepsen worker 12 - jepsen.util 4602 :invoke :cas [4 [21 16]] INFO [2019-12-17 12:30:59,829] jepsen worker 29 - jepsen.util 3629 :ok :cas [9 [10 45]] INFO [2019-12-17 12:30:59,829] jepsen worker 29 - jepsen.util 3629 :invoke :cas [9 [39 36]] INFO [2019-12-17 12:30:59,842] jepsen worker 63 - jepsen.util 3483 :ok :write [21 46] INFO [2019-12-17 12:30:59,842] jepsen worker 63 - jepsen.util 3483 :invoke :cas [21 [0 37]] INFO [2019-12-17 12:30:59,848] jepsen worker 29 - jepsen.util 3629 :fail :cas [9 [39 36]] :ValueNotSwapFrom INFO [2019-12-17 12:30:59,848] jepsen worker 3 - jepsen.util 4413 :fail :cas [1 [24 44]] :ValueNotSwapFrom INFO [2019-12-17 12:30:59,848] jepsen worker 29 - jepsen.util 3629 :invoke :write [9 45] INFO [2019-12-17 12:30:59,848] jepsen worker 3 - jepsen.util 4413 :invoke :write [1 23] INFO [2019-12-17 12:30:59,851] jepsen worker 28 - jepsen.util 2998 :info :write [9 19] :RequestCanceledException INFO [2019-12-17 12:30:59,855] jepsen worker 28 - jepsen.util 3088 :invoke :write [9 40] INFO [2019-12-17 12:30:59,857] jepsen worker 4 - jepsen.util 3424 :ok :read [1 9] INFO [2019-12-17 12:30:59,858] jepsen worker 4 - jepsen.util 3424 :invoke :cas [1 [24 43]] INFO [2019-12-17 12:30:59,887] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":24110,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:30:59,887] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":24110,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:30:59,887] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":24113,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:30:59,887] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":24113,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:30:59,887] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":24114,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:30:59,888] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":24114,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:30:59,888] jepsen worker 83 - jepsen.util 5663 :info :write [27 10] :RequestCanceledException INFO [2019-12-17 12:30:59,889] jepsen worker 83 - jepsen.util 5753 :invoke :cas [27 [42 23]] INFO [2019-12-17 12:30:59,894] jepsen worker 11 - jepsen.util 4421 :ok :write [3 33] INFO [2019-12-17 12:30:59,894] jepsen worker 11 - jepsen.util 4421 :invoke :cas [3 [44 23]] INFO [2019-12-17 12:30:59,896] jepsen worker 31 - jepsen.util 3811 :ok :write [10 1] INFO [2019-12-17 12:30:59,896] jepsen worker 46 - jepsen.util 4276 :ok :write [15 20] INFO [2019-12-17 12:30:59,896] jepsen worker 46 - jepsen.util 4276 :invoke :read [15 nil] INFO [2019-12-17 12:30:59,896] jepsen worker 31 - jepsen.util 3811 :invoke :write [10 34] INFO [2019-12-17 12:30:59,904] jepsen worker 63 - jepsen.util 3483 :fail :cas [21 [0 37]] :ValueNotSwapFrom INFO [2019-12-17 12:30:59,904] jepsen worker 63 - jepsen.util 3483 :invoke :read [21 nil] INFO [2019-12-17 12:30:59,909] jepsen worker 79 - jepsen.util 4219 :fail :cas [26 [40 40]] :ValueNotSwapFrom INFO [2019-12-17 12:30:59,909] jepsen worker 58 - jepsen.util 5728 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:30:59,909] jepsen worker 79 - jepsen.util 4219 :invoke :cas [26 [3 42]] INFO [2019-12-17 12:30:59,909] jepsen worker 58 - jepsen.util 5728 :invoke :read [19 nil] INFO [2019-12-17 12:30:59,923] jepsen worker 40 - jepsen.util 6160 :info :write [13 19] :RequestCanceledException INFO [2019-12-17 12:30:59,923] jepsen worker 80 - jepsen.util 4220 :info :write [26 9] :RequestCanceledException INFO [2019-12-17 12:30:59,924] jepsen worker 40 - jepsen.util 6250 :invoke :write [13 9] INFO [2019-12-17 12:30:59,924] jepsen worker 80 - jepsen.util 4310 :invoke :cas [26 [9 37]] INFO [2019-12-17 12:30:59,939] jepsen worker 50 - jepsen.util 5810 :info :write [16 1] :RequestCanceledException INFO [2019-12-17 12:30:59,939] jepsen worker 10 - jepsen.util 4060 :info :write [3 32] :RequestCanceledException INFO [2019-12-17 12:30:59,939] jepsen worker 35 - jepsen.util 5525 :info :cas [11 [46 22]] :RequestCanceledException INFO [2019-12-17 12:30:59,939] jepsen worker 19 - jepsen.util 4339 :fail :cas [6 [20 44]] :ValueNotSwapFrom INFO [2019-12-17 12:30:59,939] jepsen worker 19 - jepsen.util 4339 :invoke :read [6 nil] INFO [2019-12-17 12:30:59,940] jepsen worker 35 - jepsen.util 5615 :invoke :read [11 nil] INFO [2019-12-17 12:30:59,940] jepsen worker 50 - jepsen.util 5900 :invoke :read [16 nil] INFO [2019-12-17 12:30:59,940] jepsen worker 10 - jepsen.util 4150 :invoke :write [3 30] INFO [2019-12-17 12:30:59,956] jepsen worker 0 - jepsen.util 6120 :info :cas [0 [4 31]] :RequestCanceledException INFO [2019-12-17 12:30:59,956] jepsen worker 0 - jepsen.util 6210 :invoke :cas [0 [26 2]] INFO [2019-12-17 12:30:59,975] jepsen worker 29 - jepsen.util 3629 :ok :write [9 45] INFO [2019-12-17 12:30:59,975] jepsen worker 29 - jepsen.util 3629 :invoke :cas [9 [38 40]] INFO [2019-12-17 12:30:59,983] jepsen worker 46 - jepsen.util 4276 :ok :read [15 20] INFO [2019-12-17 12:30:59,983] jepsen worker 46 - jepsen.util 4276 :invoke :cas [15 [24 33]] INFO [2019-12-17 12:30:59,985] jepsen worker 63 - jepsen.util 3483 :ok :read [21 46] INFO [2019-12-17 12:30:59,985] jepsen worker 63 - jepsen.util 3483 :invoke :write [21 16] INFO [2019-12-17 12:30:59,991] jepsen worker 60 - jepsen.util 6090 :info :write [20 8] :RequestCanceledException INFO [2019-12-17 12:30:59,991] jepsen worker 2 - jepsen.util 5222 :info :cas [0 [2 11]] :RequestCanceledException INFO [2019-12-17 12:30:59,992] jepsen worker 60 - jepsen.util 6180 :invoke :write [20 33] INFO [2019-12-17 12:30:59,992] jepsen worker 2 - jepsen.util 5312 :invoke :cas [0 [32 7]] INFO [2019-12-17 12:30:59,998] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":24139,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:30:59,998] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":24139,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:31:00,009] jepsen worker 26 - jepsen.util 5696 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:31:00,009] jepsen worker 26 - jepsen.util 5696 :invoke :read [8 nil] INFO [2019-12-17 12:31:00,013] jepsen worker 46 - jepsen.util 4276 :fail :cas [15 [24 33]] :ValueNotSwapFrom INFO [2019-12-17 12:31:00,013] jepsen worker 46 - jepsen.util 4276 :invoke :write [15 39] INFO [2019-12-17 12:31:00,021] jepsen worker 29 - jepsen.util 3629 :fail :cas [9 [38 40]] :ValueNotSwapFrom INFO [2019-12-17 12:31:00,021] jepsen worker 29 - jepsen.util 3629 :invoke :cas [9 [30 13]] INFO [2019-12-17 12:31:00,024] jepsen worker 67 - jepsen.util 5737 :info :write [22 43] :RequestCanceledException INFO [2019-12-17 12:31:00,024] jepsen worker 57 - jepsen.util 5367 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:31:00,024] jepsen worker 30 - jepsen.util 4440 :fail :read [10 nil] :RequestTimeoutException INFO [2019-12-17 12:31:00,024] jepsen worker 18 - jepsen.util 4158 :info :write [6 0] :RequestCanceledException INFO [2019-12-17 12:31:00,025] jepsen worker 57 - jepsen.util 5367 :invoke :read [19 nil] INFO [2019-12-17 12:31:00,025] jepsen worker 30 - jepsen.util 4440 :invoke :read [10 nil] INFO [2019-12-17 12:31:00,026] jepsen worker 67 - jepsen.util 5827 :invoke :read [22 nil] INFO [2019-12-17 12:31:00,026] jepsen worker 18 - jepsen.util 4248 :invoke :cas [6 [11 42]] INFO [2019-12-17 12:31:00,036] jepsen worker 5 - jepsen.util 3875 :fail :cas [1 [0 44]] :ValueNotSwapFrom INFO [2019-12-17 12:31:00,036] jepsen worker 5 - jepsen.util 3875 :invoke :write [1 6] INFO [2019-12-17 12:31:00,039] jepsen worker 22 - jepsen.util 5602 :info :write [7 1] :RequestCanceledException INFO [2019-12-17 12:31:00,040] jepsen worker 22 - jepsen.util 5692 :invoke :read [7 nil] INFO [2019-12-17 12:31:00,043] jepsen worker 78 - jepsen.util 4488 :ok :read [26 32] INFO [2019-12-17 12:31:00,043] jepsen worker 78 - jepsen.util 4488 :invoke :read [26 nil] INFO [2019-12-17 12:31:00,051] jepsen worker 29 - jepsen.util 3629 :fail :cas [9 [30 13]] :ValueNotSwapFrom INFO [2019-12-17 12:31:00,051] jepsen worker 29 - jepsen.util 3629 :invoke :read [9 nil] INFO [2019-12-17 12:31:00,055] jepsen worker 38 - jepsen.util 4178 :info :write [12 22] :RequestCanceledException INFO [2019-12-17 12:31:00,056] jepsen worker 38 - jepsen.util 4268 :invoke :write [12 37] INFO [2019-12-17 12:31:00,059] jepsen worker 28 - jepsen.util 3088 :ok :write [9 40] INFO [2019-12-17 12:31:00,059] jepsen worker 28 - jepsen.util 3088 :invoke :write [9 18] INFO [2019-12-17 12:31:00,065] jepsen worker 29 - jepsen.util 3629 :ok :read [9 45] INFO [2019-12-17 12:31:00,065] jepsen worker 29 - jepsen.util 3629 :invoke :cas [9 [39 42]] INFO [2019-12-17 12:31:00,071] jepsen worker 63 - jepsen.util 3483 :ok :write [21 16] INFO [2019-12-17 12:31:00,071] jepsen worker 63 - jepsen.util 3483 :invoke :cas [21 [40 26]] INFO [2019-12-17 12:31:00,091] jepsen worker 48 - jepsen.util 4998 :info :write [16 48] :RequestCanceledException INFO [2019-12-17 12:31:00,092] jepsen worker 48 - jepsen.util 5088 :invoke :write [16 12] INFO [2019-12-17 12:31:00,099] jepsen worker 53 - jepsen.util 4283 :ok :read [17 18] INFO [2019-12-17 12:31:00,100] jepsen worker 53 - jepsen.util 4283 :invoke :cas [17 [5 29]] INFO [2019-12-17 12:31:00,107] jepsen worker 18 - jepsen.util 4248 :fail :cas [6 [11 42]] :ValueNotSwapFrom INFO [2019-12-17 12:31:00,107] jepsen worker 18 - jepsen.util 4248 :invoke :read [6 nil] INFO [2019-12-17 12:31:00,109] jepsen worker 19 - jepsen.util 4339 :ok :read [6 41] INFO [2019-12-17 12:31:00,109] jepsen worker 19 - jepsen.util 4339 :invoke :write [6 13] INFO [2019-12-17 12:31:00,120] jepsen worker 63 - jepsen.util 3483 :fail :cas [21 [40 26]] :ValueNotSwapFrom INFO [2019-12-17 12:31:00,120] jepsen worker 63 - jepsen.util 3483 :invoke :read [21 nil] INFO [2019-12-17 12:31:00,125] jepsen worker 17 - jepsen.util 5237 :info :cas [5 [32 40]] :RequestCanceledException INFO [2019-12-17 12:31:00,126] jepsen worker 17 - jepsen.util 5327 :invoke :write [5 6] INFO [2019-12-17 12:31:00,136] jepsen worker 18 - jepsen.util 4248 :ok :read [6 41] INFO [2019-12-17 12:31:00,137] jepsen worker 18 - jepsen.util 4248 :invoke :read [6 nil] INFO [2019-12-17 12:31:00,141] jepsen worker 4 - jepsen.util 3424 :fail :cas [1 [24 43]] :ValueNotSwapFrom INFO [2019-12-17 12:31:00,141] jepsen worker 4 - jepsen.util 3424 :invoke :write [1 5] INFO [2019-12-17 12:31:00,154] jepsen worker 63 - jepsen.util 3483 :ok :read [21 16] INFO [2019-12-17 12:31:00,154] jepsen worker 63 - jepsen.util 3483 :invoke :read [21 nil] INFO [2019-12-17 12:31:00,176] jepsen worker 18 - jepsen.util 4248 :ok :read [6 41] INFO [2019-12-17 12:31:00,176] jepsen worker 18 - jepsen.util 4248 :invoke :read [6 nil] INFO [2019-12-17 12:31:00,180] jepsen worker 78 - jepsen.util 4488 :ok :read [26 32] INFO [2019-12-17 12:31:00,180] jepsen worker 78 - jepsen.util 4488 :invoke :cas [26 [30 32]] INFO [2019-12-17 12:31:00,183] jepsen worker 63 - jepsen.util 3483 :ok :read [21 16] INFO [2019-12-17 12:31:00,183] jepsen worker 63 - jepsen.util 3483 :invoke :cas [21 [36 8]] INFO [2019-12-17 12:31:00,195] jepsen worker 18 - jepsen.util 4248 :ok :read [6 41] INFO [2019-12-17 12:31:00,195] jepsen worker 18 - jepsen.util 4248 :invoke :read [6 nil] INFO [2019-12-17 12:31:00,205] jepsen worker 63 - jepsen.util 3483 :fail :cas [21 [36 8]] :ValueNotSwapFrom INFO [2019-12-17 12:31:00,205] jepsen worker 63 - jepsen.util 3483 :invoke :cas [21 [27 17]] INFO [2019-12-17 12:31:00,215] jepsen worker 18 - jepsen.util 4248 :ok :read [6 41] INFO [2019-12-17 12:31:00,215] jepsen worker 18 - jepsen.util 4248 :invoke :write [6 26] INFO [2019-12-17 12:31:00,215] jepsen worker 53 - jepsen.util 4283 :fail :cas [17 [5 29]] :ValueNotSwapFrom INFO [2019-12-17 12:31:00,216] jepsen worker 53 - jepsen.util 4283 :invoke :cas [17 [42 8]] INFO [2019-12-17 12:31:00,217] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":24171,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:31:00,217] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":24171,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:31:00,217] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":24173,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:31:00,217] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":24173,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:31:00,220] jepsen worker 63 - jepsen.util 3483 :fail :cas [21 [27 17]] :ValueNotSwapFrom INFO [2019-12-17 12:31:00,220] jepsen worker 63 - jepsen.util 3483 :invoke :cas [21 [44 40]] INFO [2019-12-17 12:31:00,225] jepsen worker 33 - jepsen.util 5523 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:31:00,225] jepsen worker 33 - jepsen.util 5523 :invoke :cas [11 [25 1]] INFO [2019-12-17 12:31:00,238] jepsen worker 29 - jepsen.util 3629 :fail :cas [9 [39 42]] :ValueNotSwapFrom INFO [2019-12-17 12:31:00,238] jepsen worker 29 - jepsen.util 3629 :invoke :read [9 nil] INFO [2019-12-17 12:31:00,238] jepsen worker 42 - jepsen.util 6252 :info :write [14 11] :RequestCanceledException INFO [2019-12-17 12:31:00,239] jepsen worker 42 - jepsen.util 6342 :invoke :write [14 44] INFO [2019-12-17 12:31:00,254] jepsen worker 29 - jepsen.util 3629 :ok :read [9 40] INFO [2019-12-17 12:31:00,254] jepsen worker 29 - jepsen.util 3629 :invoke :cas [9 [12 25]] INFO [2019-12-17 12:31:00,273] jepsen worker 29 - jepsen.util 3629 :fail :cas [9 [12 25]] :ValueNotSwapFrom INFO [2019-12-17 12:31:00,274] jepsen worker 29 - jepsen.util 3629 :invoke :read [9 nil] INFO [2019-12-17 12:31:00,290] jepsen worker 29 - jepsen.util 3629 :ok :read [9 40] INFO [2019-12-17 12:31:00,290] jepsen worker 29 - jepsen.util 3629 :invoke :cas [9 [44 2]] INFO [2019-12-17 12:31:00,291] jepsen worker 41 - jepsen.util 5711 :info :cas [13 [8 36]] :RequestCanceledException INFO [2019-12-17 12:31:00,292] jepsen worker 41 - jepsen.util 5801 :invoke :cas [13 [9 16]] INFO [2019-12-17 12:31:00,295] jepsen worker 12 - jepsen.util 4602 :fail :cas [4 [21 16]] :ValueNotSwapFrom INFO [2019-12-17 12:31:00,295] jepsen worker 12 - jepsen.util 4602 :invoke :cas [4 [7 17]] INFO [2019-12-17 12:31:00,303] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":24180,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:31:00,303] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":24180,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:31:00,303] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":24183,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:31:00,303] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":24183,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:31:00,303] jepsen worker 29 - jepsen.util 3629 :fail :cas [9 [44 2]] :ValueNotSwapFrom INFO [2019-12-17 12:31:00,303] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":24184,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:31:00,303] jepsen worker 29 - jepsen.util 3629 :invoke :cas [9 [18 46]] INFO [2019-12-17 12:31:00,304] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":24184,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:31:00,309] jepsen worker 75 - jepsen.util 5565 :info :write [25 10] :RequestCanceledException INFO [2019-12-17 12:31:00,310] jepsen worker 75 - jepsen.util 5655 :invoke :read [25 nil] INFO [2019-12-17 12:31:00,311] jepsen worker 46 - jepsen.util 4276 :ok :write [15 39] INFO [2019-12-17 12:31:00,311] jepsen worker 46 - jepsen.util 4276 :invoke :read [15 nil] INFO [2019-12-17 12:31:00,311] jepsen worker 32 - jepsen.util 4802 :ok :write [10 44] INFO [2019-12-17 12:31:00,311] jepsen worker 32 - jepsen.util 4802 :invoke :write [10 45] INFO [2019-12-17 12:31:00,316] jepsen worker 29 - jepsen.util 3629 :fail :cas [9 [18 46]] :ValueNotSwapFrom INFO [2019-12-17 12:31:00,316] jepsen worker 29 - jepsen.util 3629 :invoke :cas [9 [42 11]] INFO [2019-12-17 12:31:00,324] jepsen worker 55 - jepsen.util 5095 :info :write [18 32] :RequestCanceledException INFO [2019-12-17 12:31:00,324] jepsen worker 1 - jepsen.util 5131 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:31:00,324] jepsen worker 8 - jepsen.util 4868 :info :write [2 41] :RequestCanceledException INFO [2019-12-17 12:31:00,324] jepsen worker 1 - jepsen.util 5131 :invoke :cas [0 [49 0]] INFO [2019-12-17 12:31:00,325] jepsen worker 8 - jepsen.util 4958 :invoke :cas [2 [10 45]] INFO [2019-12-17 12:31:00,325] jepsen worker 55 - jepsen.util 5185 :invoke :write [18 21] INFO [2019-12-17 12:31:00,337] jepsen worker 29 - jepsen.util 3629 :fail :cas [9 [42 11]] :ValueNotSwapFrom INFO [2019-12-17 12:31:00,337] jepsen worker 89 - jepsen.util 4049 :ok :write [29 37] INFO [2019-12-17 12:31:00,337] jepsen worker 89 - jepsen.util 4049 :invoke :read [29 nil] INFO [2019-12-17 12:31:00,337] jepsen worker 29 - jepsen.util 3629 :invoke :cas [9 [37 46]] INFO [2019-12-17 12:31:00,338] jepsen worker 76 - jepsen.util 5746 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:31:00,338] jepsen worker 13 - jepsen.util 3883 :info :write [4 0] :RequestCanceledException INFO [2019-12-17 12:31:00,338] jepsen worker 76 - jepsen.util 5746 :invoke :read [25 nil] INFO [2019-12-17 12:31:00,339] jepsen worker 13 - jepsen.util 3973 :invoke :read [4 nil] INFO [2019-12-17 12:31:00,370] jepsen worker 30 - jepsen.util 4440 :ok :read [10 44] INFO [2019-12-17 12:31:00,371] jepsen worker 30 - jepsen.util 4440 :invoke :write [10 38] INFO [2019-12-17 12:31:00,388] jepsen worker 46 - jepsen.util 4276 :ok :read [15 39] INFO [2019-12-17 12:31:00,388] jepsen worker 46 - jepsen.util 4276 :invoke :cas [15 [38 46]] INFO [2019-12-17 12:31:00,394] jepsen worker 29 - jepsen.util 3629 :fail :cas [9 [37 46]] :ValueNotSwapFrom INFO [2019-12-17 12:31:00,394] jepsen worker 29 - jepsen.util 3629 :invoke :write [9 1] INFO [2019-12-17 12:31:00,418] jepsen worker 53 - jepsen.util 4283 :fail :cas [17 [42 8]] :ValueNotSwapFrom INFO [2019-12-17 12:31:00,418] jepsen worker 53 - jepsen.util 4283 :invoke :write [17 22] INFO [2019-12-17 12:31:00,421] jepsen worker 46 - jepsen.util 4276 :fail :cas [15 [38 46]] :ValueNotSwapFrom INFO [2019-12-17 12:31:00,421] jepsen worker 46 - jepsen.util 4276 :invoke :read [15 nil] INFO [2019-12-17 12:31:00,424] jepsen worker 16 - jepsen.util 6316 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:31:00,425] jepsen worker 16 - jepsen.util 6316 :invoke :write [5 28] INFO [2019-12-17 12:31:00,432] jepsen worker 46 - jepsen.util 4276 :ok :read [15 39] INFO [2019-12-17 12:31:00,433] jepsen worker 46 - jepsen.util 4276 :invoke :write [15 33] INFO [2019-12-17 12:31:00,434] jepsen worker 79 - jepsen.util 4219 :fail :cas [26 [3 42]] :ValueNotSwapFrom INFO [2019-12-17 12:31:00,434] jepsen worker 79 - jepsen.util 4219 :invoke :cas [26 [47 38]] INFO [2019-12-17 12:31:00,440] jepsen worker 10 - jepsen.util 4150 :ok :write [3 30] INFO [2019-12-17 12:31:00,441] jepsen worker 10 - jepsen.util 4150 :invoke :write [3 21] INFO [2019-12-17 12:31:00,447] jepsen worker 78 - jepsen.util 4488 :fail :cas [26 [30 32]] :ValueNotSwapFrom INFO [2019-12-17 12:31:00,447] jepsen worker 78 - jepsen.util 4488 :invoke :write [26 17] INFO [2019-12-17 12:31:00,456] jepsen worker 15 - jepsen.util 4785 :info :write [5 17] :RequestCanceledException INFO [2019-12-17 12:31:00,457] jepsen worker 15 - jepsen.util 4875 :invoke :read [5 nil] INFO [2019-12-17 12:31:00,466] jepsen worker 29 - jepsen.util 3629 :ok :write [9 1] INFO [2019-12-17 12:31:00,467] jepsen worker 29 - jepsen.util 3629 :invoke :write [9 28] INFO [2019-12-17 12:31:00,476] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":24213,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:31:00,477] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":24213,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:31:00,477] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":24215,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:31:00,477] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":24215,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:31:00,491] jepsen worker 81 - jepsen.util 5481 :info :write [27 39] :RequestCanceledException INFO [2019-12-17 12:31:00,492] jepsen worker 81 - jepsen.util 5571 :invoke :write [27 24] INFO [2019-12-17 12:31:00,523] jepsen worker 71 - jepsen.util 4481 :info :write [23 4] :RequestCanceledException INFO [2019-12-17 12:31:00,523] jepsen worker 51 - jepsen.util 3921 :info :write [17 28] :RequestCanceledException INFO [2019-12-17 12:31:00,524] jepsen worker 51 - jepsen.util 4011 :invoke :read [17 nil] INFO [2019-12-17 12:31:00,524] jepsen worker 71 - jepsen.util 4571 :invoke :cas [23 [24 5]] INFO [2019-12-17 12:31:00,527] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:31:00,527] jepsen nemesis - couchbase.util Rebalance check count: 179 INFO [2019-12-17 12:31:00,527] jepsen nemesis - couchbase.util Rebalance stuck count: 160 INFO [2019-12-17 12:31:00,533] jepsen worker 18 - jepsen.util 4248 :ok :write [6 26] INFO [2019-12-17 12:31:00,533] jepsen worker 18 - jepsen.util 4248 :invoke :read [6 nil] INFO [2019-12-17 12:31:00,547] jepsen worker 63 - jepsen.util 3483 :fail :cas [21 [44 40]] :ValueNotSwapFrom INFO [2019-12-17 12:31:00,547] jepsen worker 63 - jepsen.util 3483 :invoke :cas [21 [8 38]] INFO [2019-12-17 12:31:00,563] jepsen worker 18 - jepsen.util 4248 :ok :read [6 26] INFO [2019-12-17 12:31:00,563] jepsen worker 18 - jepsen.util 4248 :invoke :write [6 6] INFO [2019-12-17 12:31:00,583] jepsen worker 63 - jepsen.util 3483 :fail :cas [21 [8 38]] :ValueNotSwapFrom INFO [2019-12-17 12:31:00,584] jepsen worker 63 - jepsen.util 3483 :invoke :write [21 19] INFO [2019-12-17 12:31:00,591] jepsen worker 29 - jepsen.util 3629 :ok :write [9 28] INFO [2019-12-17 12:31:00,592] jepsen worker 29 - jepsen.util 3629 :invoke :write [9 46] INFO [2019-12-17 12:31:00,609] jepsen worker 64 - jepsen.util 3304 :info :write [21 38] :RequestCanceledException INFO [2019-12-17 12:31:00,610] jepsen worker 64 - jepsen.util 3394 :invoke :read [21 nil] INFO [2019-12-17 12:31:00,624] jepsen worker 21 - jepsen.util 5511 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:31:00,624] jepsen worker 21 - jepsen.util 5511 :invoke :write [7 49] INFO [2019-12-17 12:31:00,640] jepsen worker 29 - jepsen.util 3629 :ok :write [9 46] INFO [2019-12-17 12:31:00,640] jepsen worker 29 - jepsen.util 3629 :invoke :read [9 nil] INFO [2019-12-17 12:31:00,652] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":24238,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:31:00,653] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":24238,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:31:00,654] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":24241,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:31:00,655] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":24241,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:31:00,655] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":24242,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:31:00,655] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":24242,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:31:00,661] jepsen worker 69 - jepsen.util 4299 :ok :read [23 20] INFO [2019-12-17 12:31:00,662] jepsen worker 69 - jepsen.util 4299 :invoke :cas [23 [35 5]] INFO [2019-12-17 12:31:00,668] jepsen worker 29 - jepsen.util 3629 :ok :read [9 46] INFO [2019-12-17 12:31:00,668] jepsen worker 29 - jepsen.util 3629 :invoke :cas [9 [33 33]] INFO [2019-12-17 12:31:00,679] jepsen worker 52 - jepsen.util 4462 :ok :read [17 18] INFO [2019-12-17 12:31:00,680] jepsen worker 52 - jepsen.util 4462 :invoke :read [17 nil] INFO [2019-12-17 12:31:00,681] jepsen worker 29 - jepsen.util 3629 :fail :cas [9 [33 33]] :ValueNotSwapFrom INFO [2019-12-17 12:31:00,682] jepsen worker 29 - jepsen.util 3629 :invoke :write [9 40] INFO [2019-12-17 12:31:00,682] jepsen worker 79 - jepsen.util 4219 :fail :cas [26 [47 38]] :ValueNotSwapFrom INFO [2019-12-17 12:31:00,682] jepsen worker 79 - jepsen.util 4219 :invoke :cas [26 [34 39]] INFO [2019-12-17 12:31:00,690] jepsen worker 89 - jepsen.util 4049 :ok :read [29 37] INFO [2019-12-17 12:31:00,691] jepsen worker 89 - jepsen.util 4049 :invoke :write [29 41] INFO [2019-12-17 12:31:00,710] jepsen worker 72 - jepsen.util 4392 :info :write [24 3] :RequestCanceledException INFO [2019-12-17 12:31:00,710] jepsen worker 72 - jepsen.util 4482 :invoke :cas [24 [33 20]] INFO [2019-12-17 12:31:00,724] jepsen worker 61 - jepsen.util 6271 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:31:00,724] jepsen worker 56 - jepsen.util 4376 :info :write [18 38] :RequestCanceledException INFO [2019-12-17 12:31:00,725] jepsen worker 61 - jepsen.util 6271 :invoke :cas [20 [23 23]] INFO [2019-12-17 12:31:00,726] jepsen worker 56 - jepsen.util 4466 :invoke :read [18 nil] INFO [2019-12-17 12:31:00,728] jepsen worker 32 - jepsen.util 4802 :ok :write [10 45] INFO [2019-12-17 12:31:00,728] jepsen worker 32 - jepsen.util 4802 :invoke :write [10 33] INFO [2019-12-17 12:31:00,729] jepsen worker 10 - jepsen.util 4150 :ok :write [3 21] INFO [2019-12-17 12:31:00,729] jepsen worker 10 - jepsen.util 4150 :invoke :write [3 17] INFO [2019-12-17 12:31:00,747] jepsen worker 80 - jepsen.util 4310 :fail :cas [26 [9 37]] :ValueNotSwapFrom INFO [2019-12-17 12:31:00,747] jepsen worker 80 - jepsen.util 4310 :invoke :read [26 nil] INFO [2019-12-17 12:31:00,748] jepsen worker 29 - jepsen.util 3629 :ok :write [9 40] INFO [2019-12-17 12:31:00,748] jepsen worker 29 - jepsen.util 3629 :invoke :read [9 nil] INFO [2019-12-17 12:31:00,773] jepsen worker 18 - jepsen.util 4248 :ok :write [6 6] INFO [2019-12-17 12:31:00,773] jepsen worker 18 - jepsen.util 4248 :invoke :read [6 nil] INFO [2019-12-17 12:31:00,773] jepsen worker 46 - jepsen.util 4276 :ok :write [15 33] INFO [2019-12-17 12:31:00,773] jepsen worker 46 - jepsen.util 4276 :invoke :cas [15 [0 43]] INFO [2019-12-17 12:31:00,815] jepsen worker 51 - jepsen.util 4011 :ok :read [17 18] INFO [2019-12-17 12:31:00,816] jepsen worker 51 - jepsen.util 4011 :invoke :read [17 nil] INFO [2019-12-17 12:31:00,818] jepsen worker 46 - jepsen.util 4276 :fail :cas [15 [0 43]] :ValueNotSwapFrom INFO [2019-12-17 12:31:00,818] jepsen worker 46 - jepsen.util 4276 :invoke :cas [15 [41 43]] INFO [2019-12-17 12:31:00,840] jepsen worker 29 - jepsen.util 3629 :ok :read [9 40] INFO [2019-12-17 12:31:00,841] jepsen worker 29 - jepsen.util 3629 :invoke :cas [9 [19 33]] INFO [2019-12-17 12:31:00,841] jepsen worker 65 - jepsen.util 3575 :fail :read [21 nil] :RequestTimeoutException INFO [2019-12-17 12:31:00,841] jepsen worker 65 - jepsen.util 3575 :invoke :cas [21 [35 45]] INFO [2019-12-17 12:31:00,852] jepsen worker 46 - jepsen.util 4276 :fail :cas [15 [41 43]] :ValueNotSwapFrom INFO [2019-12-17 12:31:00,852] jepsen worker 46 - jepsen.util 4276 :invoke :cas [15 [35 24]] INFO [2019-12-17 12:31:00,867] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":24263,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:31:00,867] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":24263,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:31:00,867] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":24264,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:31:00,868] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":24264,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:31:00,872] jepsen worker 29 - jepsen.util 3629 :fail :cas [9 [19 33]] :ValueNotSwapFrom INFO [2019-12-17 12:31:00,872] jepsen worker 29 - jepsen.util 3629 :invoke :read [9 nil] INFO [2019-12-17 12:31:00,887] jepsen worker 74 - jepsen.util 4754 :info :write [24 34] :RequestCanceledException INFO [2019-12-17 12:31:00,894] jepsen worker 74 - jepsen.util 4844 :invoke :read [24 nil] INFO [2019-12-17 12:31:00,896] jepsen worker 46 - jepsen.util 4276 :fail :cas [15 [35 24]] :ValueNotSwapFrom INFO [2019-12-17 12:31:00,897] jepsen worker 46 - jepsen.util 4276 :invoke :cas [15 [35 29]] INFO [2019-12-17 12:31:00,919] jepsen worker 46 - jepsen.util 4276 :fail :cas [15 [35 29]] :ValueNotSwapFrom INFO [2019-12-17 12:31:00,920] jepsen worker 46 - jepsen.util 4276 :invoke :read [15 nil] INFO [2019-12-17 12:31:00,922] jepsen worker 77 - jepsen.util 5567 :info :cas [25 [47 44]] :RequestCanceledException INFO [2019-12-17 12:31:00,925] jepsen worker 77 - jepsen.util 5657 :invoke :cas [25 [3 30]] INFO [2019-12-17 12:31:00,931] jepsen worker 74 - jepsen.util 4844 :ok :read [24 17] INFO [2019-12-17 12:31:00,931] jepsen worker 74 - jepsen.util 4844 :invoke :write [24 27] INFO [2019-12-17 12:31:00,965] jepsen worker 63 - jepsen.util 3483 :ok :write [21 19] INFO [2019-12-17 12:31:00,965] jepsen worker 30 - jepsen.util 4440 :ok :write [10 38] INFO [2019-12-17 12:31:00,986] jepsen worker 30 - jepsen.util 4440 :invoke :read [10 nil] INFO [2019-12-17 12:31:00,987] jepsen worker 10 - jepsen.util 4150 :ok :write [3 17] INFO [2019-12-17 12:31:00,987] jepsen worker 10 - jepsen.util 4150 :invoke :read [3 nil] INFO [2019-12-17 12:31:00,991] jepsen worker 9 - jepsen.util 4779 :info :write [3 13] :RequestCanceledException INFO [2019-12-17 12:31:01,008] jepsen worker 9 - jepsen.util 4869 :invoke :read [3 nil] INFO [2019-12-17 12:31:01,010] jepsen worker 36 - jepsen.util 4266 :info :write [12 5] :RequestCanceledException INFO [2019-12-17 12:31:01,013] jepsen worker 36 - jepsen.util 4356 :invoke :read [12 nil] INFO [2019-12-17 12:31:01,018] jepsen worker 13 - jepsen.util 3973 :ok :read [4 26] INFO [2019-12-17 12:31:01,019] jepsen worker 13 - jepsen.util 3973 :invoke :cas [4 [10 0]] INFO [2019-12-17 12:31:01,021] jepsen worker 65 - jepsen.util 3575 :fail :cas [21 [35 45]] :ValueNotSwapFrom INFO [2019-12-17 12:31:01,021] jepsen worker 65 - jepsen.util 3575 :invoke :write [21 48] INFO [2019-12-17 12:31:01,022] jepsen worker 52 - jepsen.util 4462 :ok :read [17 18] INFO [2019-12-17 12:31:01,022] jepsen worker 52 - jepsen.util 4462 :invoke :cas [17 [18 6]] INFO [2019-12-17 12:31:01,024] jepsen worker 63 - jepsen.util 3483 :invoke :read [21 nil] INFO [2019-12-17 12:31:01,025] jepsen worker 86 - jepsen.util 5756 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:31:01,025] jepsen worker 87 - jepsen.util 3957 :info :write [29 38] :RequestCanceledException INFO [2019-12-17 12:31:01,025] jepsen worker 86 - jepsen.util 5756 :invoke :write [28 16] INFO [2019-12-17 12:31:01,027] jepsen worker 9 - jepsen.util 4869 :ok :read [3 17] INFO [2019-12-17 12:31:01,027] jepsen worker 9 - jepsen.util 4869 :invoke :cas [3 [36 33]] INFO [2019-12-17 12:31:01,029] jepsen worker 87 - jepsen.util 4047 :invoke :cas [29 [38 49]] INFO [2019-12-17 12:31:01,039] jepsen worker 24 - jepsen.util 5334 :info :write [8 12] :RequestCanceledException INFO [2019-12-17 12:31:01,047] jepsen worker 69 - jepsen.util 4299 :fail :cas [23 [35 5]] :ValueNotSwapFrom INFO [2019-12-17 12:31:01,047] jepsen worker 69 - jepsen.util 4299 :invoke :cas [23 [6 48]] INFO [2019-12-17 12:31:01,050] jepsen worker 72 - jepsen.util 4482 :fail :cas [24 [33 20]] :ValueNotSwapFrom INFO [2019-12-17 12:31:01,050] jepsen worker 72 - jepsen.util 4482 :invoke :cas [24 [46 17]] INFO [2019-12-17 12:31:01,061] jepsen worker 24 - jepsen.util 5424 :invoke :write [8 17] INFO [2019-12-17 12:31:01,069] jepsen worker 46 - jepsen.util 4276 :ok :read [15 33] INFO [2019-12-17 12:31:01,073] jepsen worker 80 - jepsen.util 4310 :ok :read [26 32] INFO [2019-12-17 12:31:01,073] jepsen worker 46 - jepsen.util 4276 :invoke :cas [15 [26 38]] INFO [2019-12-17 12:31:01,073] jepsen worker 12 - jepsen.util 4602 :fail :cas [4 [7 17]] :ValueNotSwapFrom INFO [2019-12-17 12:31:01,073] jepsen worker 80 - jepsen.util 4310 :invoke :cas [26 [38 15]] INFO [2019-12-17 12:31:01,074] jepsen worker 12 - jepsen.util 4602 :invoke :read [4 nil] INFO [2019-12-17 12:31:01,075] jepsen worker 10 - jepsen.util 4150 :ok :read [3 17] INFO [2019-12-17 12:31:01,076] jepsen worker 10 - jepsen.util 4150 :invoke :cas [3 [5 3]] INFO [2019-12-17 12:31:01,085] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":24280,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:31:01,086] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":24280,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:31:01,086] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":24282,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:31:01,086] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":24282,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:31:01,092] jepsen worker 49 - jepsen.util 4909 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:31:01,092] jepsen worker 49 - jepsen.util 4909 :invoke :write [16 40] INFO [2019-12-17 12:31:01,110] jepsen worker 9 - jepsen.util 4869 :fail :cas [3 [36 33]] :ValueNotSwapFrom INFO [2019-12-17 12:31:01,110] jepsen worker 9 - jepsen.util 4869 :invoke :write [3 4] INFO [2019-12-17 12:31:01,113] jepsen worker 13 - jepsen.util 3973 :fail :cas [4 [10 0]] :ValueNotSwapFrom INFO [2019-12-17 12:31:01,113] jepsen worker 13 - jepsen.util 3973 :invoke :write [4 33] INFO [2019-12-17 12:31:01,118] jepsen worker 29 - jepsen.util 3629 :ok :read [9 40] INFO [2019-12-17 12:31:01,119] jepsen worker 29 - jepsen.util 3629 :invoke :read [9 nil] INFO [2019-12-17 12:31:01,125] jepsen worker 10 - jepsen.util 4150 :fail :cas [3 [5 3]] :ValueNotSwapFrom INFO [2019-12-17 12:31:01,125] jepsen worker 10 - jepsen.util 4150 :invoke :read [3 nil] INFO [2019-12-17 12:31:01,148] jepsen worker 29 - jepsen.util 3629 :ok :read [9 40] INFO [2019-12-17 12:31:01,148] jepsen worker 29 - jepsen.util 3629 :invoke :cas [9 [33 21]] INFO [2019-12-17 12:31:01,157] jepsen worker 10 - jepsen.util 4150 :ok :read [3 17] INFO [2019-12-17 12:31:01,157] jepsen worker 10 - jepsen.util 4150 :invoke :write [3 5] INFO [2019-12-17 12:31:01,169] jepsen worker 32 - jepsen.util 4802 :ok :write [10 33] INFO [2019-12-17 12:31:01,169] jepsen worker 32 - jepsen.util 4802 :invoke :read [10 nil] INFO [2019-12-17 12:31:01,176] jepsen worker 29 - jepsen.util 3629 :fail :cas [9 [33 21]] :ValueNotSwapFrom INFO [2019-12-17 12:31:01,176] jepsen worker 29 - jepsen.util 3629 :invoke :write [9 9] INFO [2019-12-17 12:31:01,185] jepsen worker 74 - jepsen.util 4844 :ok :write [24 27] INFO [2019-12-17 12:31:01,186] jepsen worker 74 - jepsen.util 4844 :invoke :cas [24 [43 24]] INFO [2019-12-17 12:31:01,195] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":24311,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:31:01,195] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":24311,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:31:01,195] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":24312,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:31:01,195] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":24312,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:31:01,203] jepsen worker 47 - jepsen.util 4727 :info :cas [15 [8 8]] :RequestCanceledException INFO [2019-12-17 12:31:01,214] jepsen worker 87 - jepsen.util 4047 :fail :cas [29 [38 49]] :ValueNotSwapFrom INFO [2019-12-17 12:31:01,215] jepsen worker 87 - jepsen.util 4047 :invoke :write [29 21] INFO [2019-12-17 12:31:01,224] jepsen worker 47 - jepsen.util 4817 :invoke :write [15 38] INFO [2019-12-17 12:31:01,240] jepsen worker 71 - jepsen.util 4571 :fail :cas [23 [24 5]] :ValueNotSwapFrom INFO [2019-12-17 12:31:01,240] jepsen worker 71 - jepsen.util 4571 :invoke :read [23 nil] INFO [2019-12-17 12:31:01,255] jepsen worker 88 - jepsen.util 4318 :info :write [29 4] :RequestCanceledException INFO [2019-12-17 12:31:01,257] jepsen worker 88 - jepsen.util 4408 :invoke :cas [29 [38 7]] INFO [2019-12-17 12:31:01,272] jepsen worker 80 - jepsen.util 4310 :fail :cas [26 [38 15]] :ValueNotSwapFrom INFO [2019-12-17 12:31:01,272] jepsen worker 80 - jepsen.util 4310 :invoke :cas [26 [32 34]] INFO [2019-12-17 12:31:01,290] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":24317,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:31:01,290] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":24317,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:31:01,290] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":24320,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:31:01,290] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":24320,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:31:01,290] jepsen worker 34 - jepsen.util 5164 :info :cas [11 [6 1]] :RequestCanceledException INFO [2019-12-17 12:31:01,292] jepsen worker 34 - jepsen.util 5254 :invoke :read [11 nil] INFO [2019-12-17 12:31:01,292] jepsen worker 63 - jepsen.util 3483 :ok :read [21 19] INFO [2019-12-17 12:31:01,292] jepsen worker 63 - jepsen.util 3483 :invoke :read [21 nil] INFO [2019-12-17 12:31:01,292] jepsen worker 30 - jepsen.util 4440 :ok :read [10 33] INFO [2019-12-17 12:31:01,292] jepsen worker 30 - jepsen.util 4440 :invoke :read [10 nil] INFO [2019-12-17 12:31:01,304] jepsen worker 13 - jepsen.util 3973 :ok :write [4 33] INFO [2019-12-17 12:31:01,304] jepsen worker 13 - jepsen.util 3973 :invoke :write [4 14] INFO [2019-12-17 12:31:01,326] jepsen worker 72 - jepsen.util 4482 :fail :cas [24 [46 17]] :ValueNotSwapFrom INFO [2019-12-17 12:31:01,327] jepsen worker 72 - jepsen.util 4482 :invoke :write [24 26] INFO [2019-12-17 12:31:01,356] jepsen worker 73 - jepsen.util 3943 :info :write [24 12] :RequestCanceledException INFO [2019-12-17 12:31:01,356] jepsen worker 37 - jepsen.util 3997 :info :write [12 42] :RequestCanceledException INFO [2019-12-17 12:31:01,357] jepsen worker 73 - jepsen.util 4033 :invoke :cas [24 [14 4]] INFO [2019-12-17 12:31:01,357] jepsen worker 37 - jepsen.util 4087 :invoke :write [12 36] INFO [2019-12-17 12:31:01,370] jepsen worker 89 - jepsen.util 4049 :ok :write [29 41] INFO [2019-12-17 12:31:01,370] jepsen worker 89 - jepsen.util 4049 :invoke :read [29 nil] INFO [2019-12-17 12:31:01,373] jepsen worker 74 - jepsen.util 4844 :fail :cas [24 [43 24]] :ValueNotSwapFrom INFO [2019-12-17 12:31:01,373] jepsen worker 74 - jepsen.util 4844 :invoke :cas [24 [26 41]] INFO [2019-12-17 12:31:01,391] jepsen worker 68 - jepsen.util 5018 :info :cas [22 [13 35]] :RequestCanceledException INFO [2019-12-17 12:31:01,391] jepsen worker 54 - jepsen.util 4554 :info :write [18 33] :RequestCanceledException INFO [2019-12-17 12:31:01,392] jepsen worker 68 - jepsen.util 5108 :invoke :write [22 36] INFO [2019-12-17 12:31:01,392] jepsen worker 54 - jepsen.util 4644 :invoke :write [18 14] INFO [2019-12-17 12:31:01,393] jepsen worker 65 - jepsen.util 3575 :ok :write [21 48] INFO [2019-12-17 12:31:01,393] jepsen worker 65 - jepsen.util 3575 :invoke :write [21 35] INFO [2019-12-17 12:31:01,424] jepsen worker 45 - jepsen.util 4275 :info :cas [15 [9 19]] :RequestCanceledException INFO [2019-12-17 12:31:01,424] jepsen worker 20 - jepsen.util 4160 :info :write [6 21] :RequestCanceledException INFO [2019-12-17 12:31:01,426] jepsen worker 20 - jepsen.util 4250 :invoke :read [6 nil] INFO [2019-12-17 12:31:01,426] jepsen worker 45 - jepsen.util 4365 :invoke :cas [15 [27 28]] INFO [2019-12-17 12:31:01,435] jepsen worker 30 - jepsen.util 4440 :ok :read [10 33] INFO [2019-12-17 12:31:01,435] jepsen worker 30 - jepsen.util 4440 :invoke :cas [10 [7 30]] INFO [2019-12-17 12:31:01,438] jepsen worker 27 - jepsen.util 4077 :fail :read [9 nil] :RequestTimeoutException INFO [2019-12-17 12:31:01,439] jepsen worker 27 - jepsen.util 4077 :invoke :cas [9 [8 21]] INFO [2019-12-17 12:31:01,440] jepsen worker 32 - jepsen.util 4802 :ok :read [10 33] INFO [2019-12-17 12:31:01,440] jepsen worker 32 - jepsen.util 4802 :invoke :write [10 7] INFO [2019-12-17 12:31:01,455] jepsen worker 29 - jepsen.util 3629 :ok :write [9 9] INFO [2019-12-17 12:31:01,455] jepsen worker 29 - jepsen.util 3629 :invoke :read [9 nil] INFO [2019-12-17 12:31:01,456] jepsen worker 23 - jepsen.util 5423 :info :cas [7 [23 41]] :RequestCanceledException INFO [2019-12-17 12:31:01,456] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":24342,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:31:01,457] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":24342,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:31:01,457] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":24345,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:31:01,457] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":24345,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:31:01,458] jepsen worker 23 - jepsen.util 5513 :invoke :cas [7 [28 39]] INFO [2019-12-17 12:31:01,488] jepsen worker 70 - jepsen.util 4930 :ok :write [23 41] INFO [2019-12-17 12:31:01,489] jepsen worker 70 - jepsen.util 4930 :invoke :read [23 nil] INFO [2019-12-17 12:31:01,503] jepsen worker 29 - jepsen.util 3629 :ok :read [9 9] INFO [2019-12-17 12:31:01,503] jepsen worker 29 - jepsen.util 3629 :invoke :cas [9 [20 21]] INFO [2019-12-17 12:31:01,505] jepsen worker 45 - jepsen.util 4365 :fail :cas [15 [27 28]] :ValueNotSwapFrom INFO [2019-12-17 12:31:01,505] jepsen worker 45 - jepsen.util 4365 :invoke :read [15 nil] INFO [2019-12-17 12:31:01,547] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:31:01,547] jepsen nemesis - couchbase.util Rebalance check count: 180 INFO [2019-12-17 12:31:01,547] jepsen nemesis - couchbase.util Rebalance stuck count: 161 INFO [2019-12-17 12:31:01,554] jepsen worker 29 - jepsen.util 3629 :fail :cas [9 [20 21]] :ValueNotSwapFrom INFO [2019-12-17 12:31:01,554] jepsen worker 29 - jepsen.util 3629 :invoke :read [9 nil] INFO [2019-12-17 12:31:01,556] jepsen worker 44 - jepsen.util 5984 :info :write [14 24] :RequestCanceledException INFO [2019-12-17 12:31:01,556] jepsen worker 69 - jepsen.util 4299 :fail :cas [23 [6 48]] :ValueNotSwapFrom INFO [2019-12-17 12:31:01,556] jepsen worker 69 - jepsen.util 4299 :invoke :write [23 11] INFO [2019-12-17 12:31:01,557] jepsen worker 44 - jepsen.util 6074 :invoke :write [14 48] INFO [2019-12-17 12:31:01,574] jepsen worker 51 - jepsen.util 4011 :ok :read [17 18] INFO [2019-12-17 12:31:01,574] jepsen worker 51 - jepsen.util 4011 :invoke :cas [17 [4 30]] INFO [2019-12-17 12:31:01,576] jepsen worker 65 - jepsen.util 3575 :ok :write [21 35] INFO [2019-12-17 12:31:01,576] jepsen worker 65 - jepsen.util 3575 :invoke :write [21 25] INFO [2019-12-17 12:31:01,611] jepsen worker 29 - jepsen.util 3629 :ok :read [9 9] INFO [2019-12-17 12:31:01,611] jepsen worker 29 - jepsen.util 3629 :invoke :write [9 17] INFO [2019-12-17 12:31:01,619] jepsen worker 45 - jepsen.util 4365 :ok :read [15 33] INFO [2019-12-17 12:31:01,619] jepsen worker 45 - jepsen.util 4365 :invoke :cas [15 [14 10]] INFO [2019-12-17 12:31:01,625] jepsen worker 59 - jepsen.util 5549 :info :cas [19 [31 24]] :RequestCanceledException INFO [2019-12-17 12:31:01,626] jepsen worker 59 - jepsen.util 5639 :invoke :cas [19 [26 33]] INFO [2019-12-17 12:31:01,631] jepsen worker 89 - jepsen.util 4049 :ok :read [29 41] INFO [2019-12-17 12:31:01,631] jepsen worker 89 - jepsen.util 4049 :invoke :read [29 nil] INFO [2019-12-17 12:31:01,639] jepsen worker 43 - jepsen.util 5083 :info :write [14 20] :RequestCanceledException INFO [2019-12-17 12:31:01,640] jepsen worker 43 - jepsen.util 5173 :invoke :write [14 13] INFO [2019-12-17 12:31:01,665] jepsen worker 56 - jepsen.util 4466 :ok :read [18 1] INFO [2019-12-17 12:31:01,665] jepsen worker 56 - jepsen.util 4466 :invoke :read [18 nil] INFO [2019-12-17 12:31:01,691] jepsen worker 66 - jepsen.util 5286 :info :cas [22 [34 15]] :RequestCanceledException INFO [2019-12-17 12:31:01,691] jepsen worker 84 - jepsen.util 5034 :info :cas [28 [17 46]] :RequestCanceledException INFO [2019-12-17 12:31:01,691] jepsen worker 25 - jepsen.util 5785 :info :write [8 46] :RequestCanceledException INFO [2019-12-17 12:31:01,692] jepsen worker 66 - jepsen.util 5376 :invoke :write [22 49] INFO [2019-12-17 12:31:01,692] jepsen worker 25 - jepsen.util 5875 :invoke :read [8 nil] INFO [2019-12-17 12:31:01,692] jepsen worker 84 - jepsen.util 5124 :invoke :write [28 41] INFO [2019-12-17 12:31:01,704] jepsen worker 29 - jepsen.util 3629 :ok :write [9 17] INFO [2019-12-17 12:31:01,704] jepsen worker 29 - jepsen.util 3629 :invoke :write [9 13] INFO [2019-12-17 12:31:01,705] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":24366,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:31:01,705] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":24366,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:31:01,705] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":24369,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:31:01,705] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":24369,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:31:01,705] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":24370,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:31:01,705] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":24370,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:31:01,710] jepsen worker 6 - jepsen.util 6126 :info :write [2 43] :RequestCanceledException INFO [2019-12-17 12:31:01,710] jepsen worker 85 - jepsen.util 6205 :info :write [28 24] :RequestCanceledException INFO [2019-12-17 12:31:01,711] jepsen worker 6 - jepsen.util 6216 :invoke :cas [2 [5 2]] INFO [2019-12-17 12:31:01,711] jepsen worker 85 - jepsen.util 6295 :invoke :write [28 0] INFO [2019-12-17 12:31:01,715] jepsen worker 65 - jepsen.util 3575 :ok :write [21 25] INFO [2019-12-17 12:31:01,715] jepsen worker 65 - jepsen.util 3575 :invoke :read [21 nil] INFO [2019-12-17 12:31:01,722] jepsen worker 56 - jepsen.util 4466 :ok :read [18 1] INFO [2019-12-17 12:31:01,722] jepsen worker 56 - jepsen.util 4466 :invoke :write [18 2] INFO [2019-12-17 12:31:01,731] jepsen worker 79 - jepsen.util 4219 :fail :cas [26 [34 39]] :ValueNotSwapFrom INFO [2019-12-17 12:31:01,731] jepsen worker 79 - jepsen.util 4219 :invoke :read [26 nil] INFO [2019-12-17 12:31:01,736] jepsen worker 5 - jepsen.util 3875 :ok :write [1 6] INFO [2019-12-17 12:31:01,736] jepsen worker 5 - jepsen.util 3875 :invoke :write [1 37] INFO [2019-12-17 12:31:01,743] jepsen worker 72 - jepsen.util 4482 :ok :write [24 26] INFO [2019-12-17 12:31:01,743] jepsen worker 72 - jepsen.util 4482 :invoke :write [24 47] INFO [2019-12-17 12:31:01,748] jepsen worker 65 - jepsen.util 3575 :ok :read [21 25] INFO [2019-12-17 12:31:01,748] jepsen worker 65 - jepsen.util 3575 :invoke :read [21 nil] INFO [2019-12-17 12:31:01,755] jepsen worker 10 - jepsen.util 4150 :ok :write [3 5] INFO [2019-12-17 12:31:01,755] jepsen worker 10 - jepsen.util 4150 :invoke :write [3 24] INFO [2019-12-17 12:31:01,756] jepsen worker 14 - jepsen.util 4334 :info :write [4 24] :RequestCanceledException INFO [2019-12-17 12:31:01,757] jepsen worker 14 - jepsen.util 4424 :invoke :cas [4 [25 33]] INFO [2019-12-17 12:31:01,791] jepsen worker 62 - jepsen.util 5192 :info :cas [20 [31 49]] :RequestCanceledException INFO [2019-12-17 12:31:01,792] jepsen worker 62 - jepsen.util 5282 :invoke :read [20 nil] INFO [2019-12-17 12:31:01,798] jepsen worker 30 - jepsen.util 4440 :fail :cas [10 [7 30]] :ValueNotSwapFrom INFO [2019-12-17 12:31:01,798] jepsen worker 30 - jepsen.util 4440 :invoke :read [10 nil] INFO [2019-12-17 12:31:01,800] jepsen worker 20 - jepsen.util 4250 :ok :read [6 6] INFO [2019-12-17 12:31:01,801] jepsen worker 20 - jepsen.util 4250 :invoke :cas [6 [8 12]] INFO [2019-12-17 12:31:01,809] jepsen worker 82 - jepsen.util 6472 :info :write [27 7] :RequestCanceledException INFO [2019-12-17 12:31:01,813] jepsen worker 82 - jepsen.util 6562 :invoke :read [27 nil] INFO [2019-12-17 12:31:01,813] jepsen worker 47 - jepsen.util 4817 :ok :write [15 38] INFO [2019-12-17 12:31:01,814] jepsen worker 47 - jepsen.util 4817 :invoke :cas [15 [39 22]] INFO [2019-12-17 12:31:01,825] jepsen worker 7 - jepsen.util 5407 :info :write [2 5] :RequestCanceledException INFO [2019-12-17 12:31:01,828] jepsen worker 7 - jepsen.util 5497 :invoke :cas [2 [11 4]] INFO [2019-12-17 12:31:01,834] jepsen worker 3 - jepsen.util 4413 :ok :write [1 23] INFO [2019-12-17 12:31:01,834] jepsen worker 3 - jepsen.util 4413 :invoke :read [1 nil] INFO [2019-12-17 12:31:01,834] jepsen worker 65 - jepsen.util 3575 :ok :read [21 25] INFO [2019-12-17 12:31:01,835] jepsen worker 65 - jepsen.util 3575 :invoke :cas [21 [39 9]] INFO [2019-12-17 12:31:01,839] jepsen worker 39 - jepsen.util 5079 :info :write [13 48] :RequestCanceledException INFO [2019-12-17 12:31:01,840] jepsen worker 39 - jepsen.util 5169 :invoke :cas [13 [34 34]] INFO [2019-12-17 12:31:01,846] jepsen worker 47 - jepsen.util 4817 :fail :cas [15 [39 22]] :ValueNotSwapFrom INFO [2019-12-17 12:31:01,846] jepsen worker 47 - jepsen.util 4817 :invoke :read [15 nil] INFO [2019-12-17 12:31:01,856] jepsen worker 12 - jepsen.util 4602 :ok :read [4 33] INFO [2019-12-17 12:31:01,856] jepsen worker 12 - jepsen.util 4602 :invoke :read [4 nil] INFO [2019-12-17 12:31:01,860] jepsen worker 53 - jepsen.util 4283 :ok :write [17 22] INFO [2019-12-17 12:31:01,860] jepsen worker 53 - jepsen.util 4283 :invoke :cas [17 [14 48]] INFO [2019-12-17 12:31:01,865] jepsen worker 65 - jepsen.util 3575 :fail :cas [21 [39 9]] :ValueNotSwapFrom INFO [2019-12-17 12:31:01,865] jepsen worker 65 - jepsen.util 3575 :invoke :cas [21 [38 49]] INFO [2019-12-17 12:31:01,865] jepsen worker 70 - jepsen.util 4930 :ok :read [23 41] INFO [2019-12-17 12:31:01,865] jepsen worker 70 - jepsen.util 4930 :invoke :read [23 nil] INFO [2019-12-17 12:31:01,874] jepsen worker 47 - jepsen.util 4817 :ok :read [15 38] INFO [2019-12-17 12:31:01,874] jepsen worker 47 - jepsen.util 4817 :invoke :read [15 nil] INFO [2019-12-17 12:31:01,891] jepsen worker 88 - jepsen.util 4408 :fail :cas [29 [38 7]] :ValueNotSwapFrom INFO [2019-12-17 12:31:01,891] jepsen worker 88 - jepsen.util 4408 :invoke :read [29 nil] INFO [2019-12-17 12:31:01,897] jepsen worker 13 - jepsen.util 3973 :ok :write [4 14] INFO [2019-12-17 12:31:01,897] jepsen worker 13 - jepsen.util 3973 :invoke :read [4 nil] INFO [2019-12-17 12:31:01,902] jepsen worker 65 - jepsen.util 3575 :fail :cas [21 [38 49]] :ValueNotSwapFrom INFO [2019-12-17 12:31:01,902] jepsen worker 65 - jepsen.util 3575 :invoke :write [21 8] INFO [2019-12-17 12:31:01,909] jepsen worker 47 - jepsen.util 4817 :ok :read [15 38] INFO [2019-12-17 12:31:01,909] jepsen worker 47 - jepsen.util 4817 :invoke :read [15 nil] INFO [2019-12-17 12:31:01,919] jepsen worker 54 - jepsen.util 4644 :ok :write [18 14] INFO [2019-12-17 12:31:01,919] jepsen worker 54 - jepsen.util 4644 :invoke :write [18 26] INFO [2019-12-17 12:31:01,925] jepsen worker 31 - jepsen.util 3811 :info :write [10 34] :RequestCanceledException INFO [2019-12-17 12:31:01,925] jepsen worker 11 - jepsen.util 4421 :info :cas [3 [44 23]] :RequestCanceledException INFO [2019-12-17 12:31:01,927] jepsen worker 31 - jepsen.util 3901 :invoke :read [10 nil] INFO [2019-12-17 12:31:01,927] jepsen worker 11 - jepsen.util 4511 :invoke :write [3 3] INFO [2019-12-17 12:31:01,939] jepsen worker 83 - jepsen.util 5753 :info :cas [27 [42 23]] :RequestCanceledException INFO [2019-12-17 12:31:01,940] jepsen worker 83 - jepsen.util 5843 :invoke :cas [27 [17 49]] INFO [2019-12-17 12:31:01,945] jepsen worker 89 - jepsen.util 4049 :ok :read [29 41] INFO [2019-12-17 12:31:01,945] jepsen worker 89 - jepsen.util 4049 :invoke :read [29 nil] INFO [2019-12-17 12:31:01,952] jepsen worker 47 - jepsen.util 4817 :ok :read [15 38] INFO [2019-12-17 12:31:01,952] jepsen worker 47 - jepsen.util 4817 :invoke :read [15 nil] INFO [2019-12-17 12:31:01,955] jepsen worker 40 - jepsen.util 6250 :info :write [13 9] :RequestCanceledException INFO [2019-12-17 12:31:01,956] jepsen worker 40 - jepsen.util 6340 :invoke :cas [13 [46 5]] INFO [2019-12-17 12:31:01,961] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":24398,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:31:01,961] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":24398,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:31:01,961] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":24399,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:31:01,961] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":24399,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:31:01,961] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":24400,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:31:01,961] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":24400,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:31:01,983] jepsen worker 31 - jepsen.util 3901 :ok :read [10 33] INFO [2019-12-17 12:31:01,983] jepsen worker 31 - jepsen.util 3901 :invoke :read [10 nil] INFO [2019-12-17 12:31:01,990] jepsen worker 35 - jepsen.util 5615 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:31:01,990] jepsen worker 35 - jepsen.util 5615 :invoke :cas [11 [37 15]] INFO [2019-12-17 12:31:02,009] jepsen worker 58 - jepsen.util 5728 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:31:02,009] jepsen worker 50 - jepsen.util 5900 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:31:02,009] jepsen worker 58 - jepsen.util 5728 :invoke :write [19 27] INFO [2019-12-17 12:31:02,009] jepsen worker 50 - jepsen.util 5900 :invoke :read [16 nil] INFO [2019-12-17 12:31:02,026] jepsen worker 2 - jepsen.util 5312 :info :cas [0 [32 7]] :RequestCanceledException INFO [2019-12-17 12:31:02,028] jepsen worker 2 - jepsen.util 5402 :invoke :cas [0 [7 40]] INFO [2019-12-17 12:31:02,032] jepsen worker 65 - jepsen.util 3575 :ok :write [21 8] INFO [2019-12-17 12:31:02,032] jepsen worker 65 - jepsen.util 3575 :invoke :cas [21 [6 11]] INFO [2019-12-17 12:31:02,034] jepsen worker 31 - jepsen.util 3901 :ok :read [10 33] INFO [2019-12-17 12:31:02,035] jepsen worker 31 - jepsen.util 3901 :invoke :cas [10 [1 44]] INFO [2019-12-17 12:31:02,037] jepsen worker 0 - jepsen.util 6210 :info :cas [0 [26 2]] :RequestCanceledException INFO [2019-12-17 12:31:02,039] jepsen worker 0 - jepsen.util 6300 :invoke :write [0 5] INFO [2019-12-17 12:31:02,050] jepsen worker 60 - jepsen.util 6180 :info :write [20 33] :RequestCanceledException INFO [2019-12-17 12:31:02,051] jepsen worker 60 - jepsen.util 6270 :invoke :cas [20 [24 19]] INFO [2019-12-17 12:31:02,057] jepsen worker 36 - jepsen.util 4356 :ok :read [12 40] INFO [2019-12-17 12:31:02,057] jepsen worker 36 - jepsen.util 4356 :invoke :read [12 nil] INFO [2019-12-17 12:31:02,063] jepsen worker 65 - jepsen.util 3575 :fail :cas [21 [6 11]] :ValueNotSwapFrom INFO [2019-12-17 12:31:02,063] jepsen worker 65 - jepsen.util 3575 :invoke :read [21 nil] INFO [2019-12-17 12:31:02,071] jepsen worker 31 - jepsen.util 3901 :fail :cas [10 [1 44]] :ValueNotSwapFrom INFO [2019-12-17 12:31:02,072] jepsen worker 31 - jepsen.util 3901 :invoke :cas [10 [35 23]] INFO [2019-12-17 12:31:02,072] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":24424,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:31:02,072] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":24424,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:31:02,078] jepsen worker 38 - jepsen.util 4268 :ok :write [12 37] INFO [2019-12-17 12:31:02,079] jepsen worker 38 - jepsen.util 4268 :invoke :write [12 14] INFO [2019-12-17 12:31:02,085] jepsen worker 29 - jepsen.util 3629 :ok :write [9 13] INFO [2019-12-17 12:31:02,085] jepsen worker 29 - jepsen.util 3629 :invoke :read [9 nil] INFO [2019-12-17 12:31:02,088] jepsen worker 4 - jepsen.util 3424 :ok :write [1 5] INFO [2019-12-17 12:31:02,088] jepsen worker 4 - jepsen.util 3424 :invoke :write [1 39] INFO [2019-12-17 12:31:02,092] jepsen worker 67 - jepsen.util 5827 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:31:02,092] jepsen worker 67 - jepsen.util 5827 :invoke :cas [22 [4 12]] INFO [2019-12-17 12:31:02,095] jepsen worker 65 - jepsen.util 3575 :ok :read [21 8] INFO [2019-12-17 12:31:02,095] jepsen worker 65 - jepsen.util 3575 :invoke :write [21 6] INFO [2019-12-17 12:31:02,105] jepsen worker 80 - jepsen.util 4310 :ok :cas [26 [32 34]] INFO [2019-12-17 12:31:02,105] jepsen worker 80 - jepsen.util 4310 :invoke :write [26 12] INFO [2019-12-17 12:31:02,109] jepsen worker 26 - jepsen.util 5696 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:31:02,109] jepsen worker 26 - jepsen.util 5696 :invoke :cas [8 [10 12]] INFO [2019-12-17 12:31:02,109] jepsen worker 29 - jepsen.util 3629 :ok :read [9 13] INFO [2019-12-17 12:31:02,110] jepsen worker 29 - jepsen.util 3629 :invoke :read [9 nil] INFO [2019-12-17 12:31:02,125] jepsen worker 57 - jepsen.util 5367 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:31:02,125] jepsen worker 28 - jepsen.util 3088 :info :write [9 18] :RequestCanceledException INFO [2019-12-17 12:31:02,125] jepsen worker 22 - jepsen.util 5692 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:31:02,125] jepsen worker 31 - jepsen.util 3901 :fail :cas [10 [35 23]] :ValueNotSwapFrom INFO [2019-12-17 12:31:02,125] jepsen worker 19 - jepsen.util 4339 :info :write [6 13] :RequestCanceledException INFO [2019-12-17 12:31:02,125] jepsen worker 57 - jepsen.util 5367 :invoke :write [19 40] INFO [2019-12-17 12:31:02,125] jepsen worker 22 - jepsen.util 5692 :invoke :cas [7 [45 3]] INFO [2019-12-17 12:31:02,125] jepsen worker 31 - jepsen.util 3901 :invoke :read [10 nil] INFO [2019-12-17 12:31:02,126] jepsen worker 28 - jepsen.util 3178 :invoke :read [9 nil] INFO [2019-12-17 12:31:02,126] jepsen worker 19 - jepsen.util 4429 :invoke :write [6 41] INFO [2019-12-17 12:31:02,139] jepsen worker 29 - jepsen.util 3629 :ok :read [9 13] INFO [2019-12-17 12:31:02,139] jepsen worker 29 - jepsen.util 3629 :invoke :read [9 nil] INFO [2019-12-17 12:31:02,141] jepsen worker 88 - jepsen.util 4408 :ok :read [29 41] INFO [2019-12-17 12:31:02,141] jepsen worker 88 - jepsen.util 4408 :invoke :write [29 33] INFO [2019-12-17 12:31:02,153] jepsen worker 48 - jepsen.util 5088 :info :write [16 12] :RequestCanceledException INFO [2019-12-17 12:31:02,153] jepsen worker 71 - jepsen.util 4571 :ok :read [23 41] INFO [2019-12-17 12:31:02,153] jepsen worker 71 - jepsen.util 4571 :invoke :cas [23 [41 17]] INFO [2019-12-17 12:31:02,154] jepsen worker 48 - jepsen.util 5178 :invoke :cas [16 [42 28]] INFO [2019-12-17 12:31:02,159] jepsen worker 31 - jepsen.util 3901 :ok :read [10 33] INFO [2019-12-17 12:31:02,159] jepsen worker 31 - jepsen.util 3901 :invoke :read [10 nil] INFO [2019-12-17 12:31:02,172] jepsen worker 65 - jepsen.util 3575 :ok :write [21 6] INFO [2019-12-17 12:31:02,172] jepsen worker 65 - jepsen.util 3575 :invoke :write [21 11] INFO [2019-12-17 12:31:02,184] jepsen worker 29 - jepsen.util 3629 :ok :read [9 13] INFO [2019-12-17 12:31:02,184] jepsen worker 29 - jepsen.util 3629 :invoke :read [9 nil] INFO [2019-12-17 12:31:02,207] jepsen worker 5 - jepsen.util 3875 :ok :write [1 37] INFO [2019-12-17 12:31:02,207] jepsen worker 5 - jepsen.util 3875 :invoke :cas [1 [43 46]] INFO [2019-12-17 12:31:02,213] jepsen worker 31 - jepsen.util 3901 :ok :read [10 33] INFO [2019-12-17 12:31:02,213] jepsen worker 31 - jepsen.util 3901 :invoke :cas [10 [21 17]] INFO [2019-12-17 12:31:02,225] jepsen worker 17 - jepsen.util 5327 :info :write [5 6] :RequestCanceledException INFO [2019-12-17 12:31:02,226] jepsen worker 17 - jepsen.util 5417 :invoke :read [5 nil] INFO [2019-12-17 12:31:02,228] jepsen worker 56 - jepsen.util 4466 :ok :write [18 2] INFO [2019-12-17 12:31:02,228] jepsen worker 56 - jepsen.util 4466 :invoke :write [18 24] INFO [2019-12-17 12:31:02,233] jepsen worker 29 - jepsen.util 3629 :ok :read [9 13] INFO [2019-12-17 12:31:02,233] jepsen worker 29 - jepsen.util 3629 :invoke :cas [9 [32 44]] INFO [2019-12-17 12:31:02,239] jepsen worker 78 - jepsen.util 4488 :ok :write [26 17] INFO [2019-12-17 12:31:02,239] jepsen worker 78 - jepsen.util 4488 :invoke :cas [26 [26 19]] INFO [2019-12-17 12:31:02,268] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":24465,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:31:02,268] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":24465,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:31:02,268] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":24469,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:31:02,268] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":24469,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:31:02,284] jepsen worker 65 - jepsen.util 3575 :ok :write [21 11] INFO [2019-12-17 12:31:02,284] jepsen worker 65 - jepsen.util 3575 :invoke :cas [21 [23 33]] INFO [2019-12-17 12:31:02,287] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":24457,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:31:02,287] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":24457,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:31:02,287] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":24458,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:31:02,287] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":24458,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:31:02,287] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":24459,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:31:02,287] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":24459,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:31:02,288] jepsen worker 31 - jepsen.util 3901 :fail :cas [10 [21 17]] :ValueNotSwapFrom INFO [2019-12-17 12:31:02,288] jepsen worker 31 - jepsen.util 3901 :invoke :write [10 16] INFO [2019-12-17 12:31:02,288] jepsen worker 29 - jepsen.util 3629 :fail :cas [9 [32 44]] :ValueNotSwapFrom INFO [2019-12-17 12:31:02,288] jepsen worker 29 - jepsen.util 3629 :invoke :cas [9 [8 7]] INFO [2019-12-17 12:31:02,290] jepsen worker 42 - jepsen.util 6342 :info :write [14 44] :RequestCanceledException INFO [2019-12-17 12:31:02,292] jepsen worker 42 - jepsen.util 6432 :invoke :read [14 nil] INFO [2019-12-17 12:31:02,300] jepsen worker 51 - jepsen.util 4011 :fail :cas [17 [4 30]] :ValueNotSwapFrom INFO [2019-12-17 12:31:02,300] jepsen worker 51 - jepsen.util 4011 :invoke :cas [17 [10 47]] INFO [2019-12-17 12:31:02,309] jepsen worker 41 - jepsen.util 5801 :info :cas [13 [9 16]] :RequestCanceledException INFO [2019-12-17 12:31:02,310] jepsen worker 41 - jepsen.util 5891 :invoke :write [13 2] INFO [2019-12-17 12:31:02,322] jepsen worker 33 - jepsen.util 5523 :info :cas [11 [25 1]] :RequestCanceledException INFO [2019-12-17 12:31:02,322] jepsen worker 75 - jepsen.util 5655 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:31:02,322] jepsen worker 75 - jepsen.util 5655 :invoke :write [25 11] INFO [2019-12-17 12:31:02,323] jepsen worker 37 - jepsen.util 4087 :ok :write [12 36] INFO [2019-12-17 12:31:02,323] jepsen worker 37 - jepsen.util 4087 :invoke :cas [12 [24 5]] INFO [2019-12-17 12:31:02,323] jepsen worker 33 - jepsen.util 5613 :invoke :read [11 nil] INFO [2019-12-17 12:31:02,325] jepsen worker 53 - jepsen.util 4283 :fail :cas [17 [14 48]] :ValueNotSwapFrom INFO [2019-12-17 12:31:02,325] jepsen worker 53 - jepsen.util 4283 :invoke :read [17 nil] INFO [2019-12-17 12:31:02,335] jepsen worker 65 - jepsen.util 3575 :fail :cas [21 [23 33]] :ValueNotSwapFrom INFO [2019-12-17 12:31:02,335] jepsen worker 65 - jepsen.util 3575 :invoke :read [21 nil] INFO [2019-12-17 12:31:02,339] jepsen worker 8 - jepsen.util 4958 :info :cas [2 [10 45]] :RequestCanceledException INFO [2019-12-17 12:31:02,340] jepsen worker 8 - jepsen.util 5048 :invoke :read [2 nil] INFO [2019-12-17 12:31:02,354] jepsen worker 55 - jepsen.util 5185 :info :write [18 21] :RequestCanceledException INFO [2019-12-17 12:31:02,356] jepsen worker 55 - jepsen.util 5275 :invoke :write [18 27] INFO [2019-12-17 12:31:02,357] jepsen worker 3 - jepsen.util 4413 :ok :read [1 37] INFO [2019-12-17 12:31:02,357] jepsen worker 3 - jepsen.util 4413 :invoke :read [1 nil] INFO [2019-12-17 12:31:02,368] jepsen worker 29 - jepsen.util 3629 :fail :cas [9 [8 7]] :ValueNotSwapFrom INFO [2019-12-17 12:31:02,368] jepsen worker 29 - jepsen.util 3629 :invoke :cas [9 [11 31]] INFO [2019-12-17 12:31:02,376] jepsen worker 65 - jepsen.util 3575 :ok :read [21 11] INFO [2019-12-17 12:31:02,376] jepsen worker 65 - jepsen.util 3575 :invoke :read [21 nil] INFO [2019-12-17 12:31:02,401] jepsen worker 79 - jepsen.util 4219 :ok :read [26 17] INFO [2019-12-17 12:31:02,401] jepsen worker 79 - jepsen.util 4219 :invoke :read [26 nil] INFO [2019-12-17 12:31:02,411] jepsen worker 29 - jepsen.util 3629 :fail :cas [9 [11 31]] :ValueNotSwapFrom INFO [2019-12-17 12:31:02,411] jepsen worker 65 - jepsen.util 3575 :ok :read [21 11] INFO [2019-12-17 12:31:02,411] jepsen worker 29 - jepsen.util 3629 :invoke :cas [9 [28 25]] INFO [2019-12-17 12:31:02,411] jepsen worker 65 - jepsen.util 3575 :invoke :cas [21 [13 8]] INFO [2019-12-17 12:31:02,425] jepsen worker 1 - jepsen.util 5131 :info :cas [0 [49 0]] :RequestCanceledException INFO [2019-12-17 12:31:02,426] jepsen worker 1 - jepsen.util 5221 :invoke :read [0 nil] INFO [2019-12-17 12:31:02,429] jepsen worker 65 - jepsen.util 3575 :fail :cas [21 [13 8]] :ValueNotSwapFrom INFO [2019-12-17 12:31:02,430] jepsen worker 65 - jepsen.util 3575 :invoke :read [21 nil] INFO [2019-12-17 12:31:02,432] jepsen worker 29 - jepsen.util 3629 :fail :cas [9 [28 25]] :ValueNotSwapFrom INFO [2019-12-17 12:31:02,432] jepsen worker 29 - jepsen.util 3629 :invoke :cas [9 [28 10]] INFO [2019-12-17 12:31:02,439] jepsen worker 76 - jepsen.util 5746 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:31:02,439] jepsen worker 76 - jepsen.util 5746 :invoke :read [25 nil] INFO [2019-12-17 12:31:02,443] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":24496,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:31:02,444] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":24496,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:31:02,444] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":24499,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:31:02,444] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":24499,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:31:02,444] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":24500,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:31:02,444] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":24500,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:31:02,452] jepsen worker 87 - jepsen.util 4047 :ok :write [29 21] INFO [2019-12-17 12:31:02,452] jepsen worker 65 - jepsen.util 3575 :ok :read [21 11] INFO [2019-12-17 12:31:02,452] jepsen worker 87 - jepsen.util 4047 :invoke :read [29 nil] INFO [2019-12-17 12:31:02,452] jepsen worker 65 - jepsen.util 3575 :invoke :cas [21 [49 10]] INFO [2019-12-17 12:31:02,452] jepsen worker 29 - jepsen.util 3629 :fail :cas [9 [28 10]] :ValueNotSwapFrom INFO [2019-12-17 12:31:02,452] jepsen worker 29 - jepsen.util 3629 :invoke :cas [9 [35 15]] INFO [2019-12-17 12:31:02,460] jepsen worker 12 - jepsen.util 4602 :ok :read [4 14] INFO [2019-12-17 12:31:02,460] jepsen worker 12 - jepsen.util 4602 :invoke :cas [4 [4 15]] INFO [2019-12-17 12:31:02,469] jepsen worker 32 - jepsen.util 4802 :ok :write [10 7] INFO [2019-12-17 12:31:02,469] jepsen worker 32 - jepsen.util 4802 :invoke :write [10 10] INFO [2019-12-17 12:31:02,480] jepsen worker 70 - jepsen.util 4930 :ok :read [23 41] INFO [2019-12-17 12:31:02,480] jepsen worker 70 - jepsen.util 4930 :invoke :cas [23 [45 15]] INFO [2019-12-17 12:31:02,483] jepsen worker 29 - jepsen.util 3629 :fail :cas [9 [35 15]] :ValueNotSwapFrom INFO [2019-12-17 12:31:02,483] jepsen worker 29 - jepsen.util 3629 :invoke :read [9 nil] INFO [2019-12-17 12:31:02,491] jepsen worker 15 - jepsen.util 4875 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:31:02,491] jepsen worker 15 - jepsen.util 4875 :invoke :read [5 nil] INFO [2019-12-17 12:31:02,509] jepsen worker 81 - jepsen.util 5571 :info :write [27 24] :RequestCanceledException INFO [2019-12-17 12:31:02,510] jepsen worker 81 - jepsen.util 5661 :invoke :write [27 3] INFO [2019-12-17 12:31:02,518] jepsen worker 65 - jepsen.util 3575 :fail :cas [21 [49 10]] :ValueNotSwapFrom INFO [2019-12-17 12:31:02,518] jepsen worker 65 - jepsen.util 3575 :invoke :write [21 7] INFO [2019-12-17 12:31:02,523] jepsen worker 29 - jepsen.util 3629 :ok :read [9 13] INFO [2019-12-17 12:31:02,523] jepsen worker 29 - jepsen.util 3629 :invoke :cas [9 [28 31]] INFO [2019-12-17 12:31:02,524] jepsen worker 16 - jepsen.util 6316 :info :write [5 28] :RequestCanceledException INFO [2019-12-17 12:31:02,534] jepsen worker 16 - jepsen.util 6406 :invoke :write [5 37] INFO [2019-12-17 12:31:02,548] jepsen worker 38 - jepsen.util 4268 :ok :write [12 14] INFO [2019-12-17 12:31:02,548] jepsen worker 38 - jepsen.util 4268 :invoke :read [12 nil] INFO [2019-12-17 12:31:02,552] jepsen worker 52 - jepsen.util 4462 :fail :cas [17 [18 6]] :CASMismatchException INFO [2019-12-17 12:31:02,552] jepsen worker 52 - jepsen.util 4462 :invoke :read [17 nil] INFO [2019-12-17 12:31:02,552] jepsen worker 29 - jepsen.util 3629 :fail :cas [9 [28 31]] :ValueNotSwapFrom INFO [2019-12-17 12:31:02,552] jepsen worker 29 - jepsen.util 3629 :invoke :cas [9 [31 6]] INFO [2019-12-17 12:31:02,562] jepsen worker 13 - jepsen.util 3973 :ok :read [4 14] INFO [2019-12-17 12:31:02,562] jepsen worker 13 - jepsen.util 3973 :invoke :write [4 11] INFO [2019-12-17 12:31:02,569] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:31:02,569] jepsen nemesis - couchbase.util Rebalance check count: 181 INFO [2019-12-17 12:31:02,569] jepsen nemesis - couchbase.util Rebalance stuck count: 162 INFO [2019-12-17 12:31:02,569] jepsen worker 29 - jepsen.util 3629 :fail :cas [9 [31 6]] :ValueNotSwapFrom INFO [2019-12-17 12:31:02,569] jepsen worker 29 - jepsen.util 3629 :invoke :write [9 22] INFO [2019-12-17 12:31:02,588] jepsen worker 65 - jepsen.util 3575 :ok :write [21 7] INFO [2019-12-17 12:31:02,588] jepsen worker 65 - jepsen.util 3575 :invoke :write [21 18] INFO [2019-12-17 12:31:02,599] jepsen worker 4 - jepsen.util 3424 :ok :write [1 39] INFO [2019-12-17 12:31:02,599] jepsen worker 4 - jepsen.util 3424 :invoke :cas [1 [19 43]] INFO [2019-12-17 12:31:02,609] jepsen worker 11 - jepsen.util 4511 :ok :write [3 3] INFO [2019-12-17 12:31:02,610] jepsen worker 11 - jepsen.util 4511 :invoke :read [3 nil] INFO [2019-12-17 12:31:02,656] jepsen worker 64 - jepsen.util 3394 :fail :read [21 nil] :RequestTimeoutException INFO [2019-12-17 12:31:02,656] jepsen worker 64 - jepsen.util 3394 :invoke :cas [21 [47 3]] INFO [2019-12-17 12:31:02,679] jepsen worker 28 - jepsen.util 3178 :ok :read [9 13] INFO [2019-12-17 12:31:02,680] jepsen worker 28 - jepsen.util 3178 :invoke :read [9 nil] INFO [2019-12-17 12:31:02,692] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":24525,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:31:02,693] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":24525,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:31:02,693] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":24529,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:31:02,693] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":24529,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:31:02,696] jepsen worker 69 - jepsen.util 4299 :ok :write [23 11] INFO [2019-12-17 12:31:02,696] jepsen worker 69 - jepsen.util 4299 :invoke :write [23 25] INFO [2019-12-17 12:31:02,701] jepsen worker 36 - jepsen.util 4356 :ok :read [12 14] INFO [2019-12-17 12:31:02,701] jepsen worker 36 - jepsen.util 4356 :invoke :cas [12 [17 27]] INFO [2019-12-17 12:31:02,714] jepsen worker 11 - jepsen.util 4511 :ok :read [3 3] INFO [2019-12-17 12:31:02,714] jepsen worker 11 - jepsen.util 4511 :invoke :cas [3 [32 33]] INFO [2019-12-17 12:31:02,717] jepsen worker 19 - jepsen.util 4429 :ok :write [6 41] INFO [2019-12-17 12:31:02,718] jepsen worker 19 - jepsen.util 4429 :invoke :read [6 nil] INFO [2019-12-17 12:31:02,724] jepsen worker 21 - jepsen.util 5511 :info :write [7 49] :RequestCanceledException INFO [2019-12-17 12:31:02,726] jepsen worker 21 - jepsen.util 5601 :invoke :write [7 49] INFO [2019-12-17 12:31:02,734] jepsen worker 28 - jepsen.util 3178 :ok :read [9 13] INFO [2019-12-17 12:31:02,734] jepsen worker 28 - jepsen.util 3178 :invoke :write [9 18] INFO [2019-12-17 12:31:02,758] jepsen worker 88 - jepsen.util 4408 :ok :write [29 33] INFO [2019-12-17 12:31:02,758] jepsen worker 88 - jepsen.util 4408 :invoke :write [29 40] INFO [2019-12-17 12:31:02,758] jepsen worker 71 - jepsen.util 4571 :fail :cas [23 [41 17]] :ValueNotSwapFrom INFO [2019-12-17 12:31:02,758] jepsen worker 71 - jepsen.util 4571 :invoke :cas [23 [45 45]] INFO [2019-12-17 12:31:02,765] jepsen worker 72 - jepsen.util 4482 :ok :write [24 47] INFO [2019-12-17 12:31:02,765] jepsen worker 72 - jepsen.util 4482 :invoke :write [24 19] INFO [2019-12-17 12:31:02,771] jepsen worker 5 - jepsen.util 3875 :fail :cas [1 [43 46]] :ValueNotSwapFrom INFO [2019-12-17 12:31:02,771] jepsen worker 5 - jepsen.util 3875 :invoke :write [1 43] INFO [2019-12-17 12:31:02,789] jepsen worker 3 - jepsen.util 4413 :ok :read [1 39] INFO [2019-12-17 12:31:02,789] jepsen worker 3 - jepsen.util 4413 :invoke :write [1 2] INFO [2019-12-17 12:31:02,809] jepsen worker 18 - jepsen.util 4248 :fail :read [6 nil] :RequestTimeoutException INFO [2019-12-17 12:31:02,809] jepsen worker 18 - jepsen.util 4248 :invoke :cas [6 [16 24]] INFO [2019-12-17 12:31:02,817] jepsen worker 36 - jepsen.util 4356 :fail :cas [12 [17 27]] :ValueNotSwapFrom INFO [2019-12-17 12:31:02,817] jepsen worker 36 - jepsen.util 4356 :invoke :write [12 35] INFO [2019-12-17 12:31:02,819] jepsen worker 11 - jepsen.util 4511 :fail :cas [3 [32 33]] :ValueNotSwapFrom INFO [2019-12-17 12:31:02,819] jepsen worker 11 - jepsen.util 4511 :invoke :cas [3 [15 30]] INFO [2019-12-17 12:31:02,822] jepsen worker 80 - jepsen.util 4310 :ok :write [26 12] INFO [2019-12-17 12:31:02,822] jepsen worker 80 - jepsen.util 4310 :invoke :cas [26 [40 42]] INFO [2019-12-17 12:31:02,824] jepsen worker 61 - jepsen.util 6271 :info :cas [20 [23 23]] :RequestCanceledException INFO [2019-12-17 12:31:02,825] jepsen worker 61 - jepsen.util 6361 :invoke :read [20 nil] INFO [2019-12-17 12:31:02,826] jepsen worker 14 - jepsen.util 4424 :fail :cas [4 [25 33]] :ValueNotSwapFrom INFO [2019-12-17 12:31:02,826] jepsen worker 14 - jepsen.util 4424 :invoke :read [4 nil] INFO [2019-12-17 12:31:02,840] jepsen worker 19 - jepsen.util 4429 :ok :read [6 41] INFO [2019-12-17 12:31:02,840] jepsen worker 19 - jepsen.util 4429 :invoke :cas [6 [49 10]] INFO [2019-12-17 12:31:02,882] jepsen worker 89 - jepsen.util 4049 :ok :read [29 33] INFO [2019-12-17 12:31:02,882] jepsen worker 89 - jepsen.util 4049 :invoke :cas [29 [36 20]] INFO [2019-12-17 12:31:02,887] jepsen worker 29 - jepsen.util 3629 :ok :write [9 22] INFO [2019-12-17 12:31:02,888] jepsen worker 29 - jepsen.util 3629 :invoke :cas [9 [32 2]] INFO [2019-12-17 12:31:02,891] jepsen worker 19 - jepsen.util 4429 :fail :cas [6 [49 10]] :ValueNotSwapFrom INFO [2019-12-17 12:31:02,891] jepsen worker 14 - jepsen.util 4424 :ok :read [4 14] INFO [2019-12-17 12:31:02,891] jepsen worker 19 - jepsen.util 4429 :invoke :cas [6 [1 3]] INFO [2019-12-17 12:31:02,891] jepsen worker 14 - jepsen.util 4424 :invoke :read [4 nil] INFO [2019-12-17 12:31:02,904] jepsen worker 64 - jepsen.util 3394 :fail :cas [21 [47 3]] :ValueNotSwapFrom INFO [2019-12-17 12:31:02,904] jepsen worker 64 - jepsen.util 3394 :invoke :write [21 16] INFO [2019-12-17 12:31:02,916] jepsen worker 65 - jepsen.util 3575 :ok :write [21 18] INFO [2019-12-17 12:31:02,916] jepsen worker 65 - jepsen.util 3575 :invoke :cas [21 [27 25]] INFO [2019-12-17 12:31:02,918] jepsen worker 37 - jepsen.util 4087 :fail :cas [12 [24 5]] :ValueNotSwapFrom INFO [2019-12-17 12:31:02,918] jepsen worker 37 - jepsen.util 4087 :invoke :write [12 18] INFO [2019-12-17 12:31:02,933] jepsen worker 87 - jepsen.util 4047 :ok :read [29 33] INFO [2019-12-17 12:31:02,933] jepsen worker 19 - jepsen.util 4429 :fail :cas [6 [1 3]] :ValueNotSwapFrom INFO [2019-12-17 12:31:02,933] jepsen worker 87 - jepsen.util 4047 :invoke :read [29 nil] INFO [2019-12-17 12:31:02,933] jepsen worker 19 - jepsen.util 4429 :invoke :cas [6 [24 2]] INFO [2019-12-17 12:31:02,935] jepsen worker 29 - jepsen.util 3629 :fail :cas [9 [32 2]] :ValueNotSwapFrom INFO [2019-12-17 12:31:02,935] jepsen worker 29 - jepsen.util 3629 :invoke :read [9 nil] INFO [2019-12-17 12:31:02,943] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":24552,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:31:02,943] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":24552,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:31:02,943] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":24555,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:31:02,943] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":24555,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:31:02,963] jepsen worker 65 - jepsen.util 3575 :fail :cas [21 [27 25]] :ValueNotSwapFrom INFO [2019-12-17 12:31:02,963] jepsen worker 65 - jepsen.util 3575 :invoke :cas [21 [12 36]] INFO [2019-12-17 12:31:02,967] jepsen worker 89 - jepsen.util 4049 :fail :cas [29 [36 20]] :ValueNotSwapFrom INFO [2019-12-17 12:31:02,967] jepsen worker 89 - jepsen.util 4049 :invoke :read [29 nil] INFO [2019-12-17 12:31:02,978] jepsen worker 78 - jepsen.util 4488 :fail :cas [26 [26 19]] :ValueNotSwapFrom INFO [2019-12-17 12:31:02,978] jepsen worker 78 - jepsen.util 4488 :invoke :cas [26 [31 26]] INFO [2019-12-17 12:31:03,000] jepsen worker 14 - jepsen.util 4424 :ok :read [4 14] INFO [2019-12-17 12:31:03,001] jepsen worker 14 - jepsen.util 4424 :invoke :cas [4 [35 6]] INFO [2019-12-17 12:31:03,008] jepsen worker 51 - jepsen.util 4011 :fail :cas [17 [10 47]] :ValueNotSwapFrom INFO [2019-12-17 12:31:03,008] jepsen worker 51 - jepsen.util 4011 :invoke :cas [17 [2 6]] INFO [2019-12-17 12:31:03,020] jepsen worker 52 - jepsen.util 4462 :ok :read [17 22] INFO [2019-12-17 12:31:03,020] jepsen worker 52 - jepsen.util 4462 :invoke :write [17 38] INFO [2019-12-17 12:31:03,024] jepsen worker 77 - jepsen.util 5657 :info :cas [25 [3 30]] :RequestCanceledException INFO [2019-12-17 12:31:03,025] jepsen worker 77 - jepsen.util 5747 :invoke :read [25 nil] INFO [2019-12-17 12:31:03,029] jepsen worker 65 - jepsen.util 3575 :fail :cas [21 [12 36]] :ValueNotSwapFrom INFO [2019-12-17 12:31:03,029] jepsen worker 65 - jepsen.util 3575 :invoke :read [21 nil] INFO [2019-12-17 12:31:03,033] jepsen worker 19 - jepsen.util 4429 :fail :cas [6 [24 2]] :ValueNotSwapFrom INFO [2019-12-17 12:31:03,033] jepsen worker 19 - jepsen.util 4429 :invoke :write [6 34] INFO [2019-12-17 12:31:03,037] jepsen worker 53 - jepsen.util 4283 :ok :read [17 22] INFO [2019-12-17 12:31:03,037] jepsen worker 53 - jepsen.util 4283 :invoke :read [17 nil] INFO [2019-12-17 12:31:03,052] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":24570,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:31:03,053] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":24570,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:31:03,053] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":24573,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:31:03,053] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":24573,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:31:03,053] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":24574,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:31:03,053] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":24574,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:31:03,054] jepsen worker 4 - jepsen.util 3424 :fail :cas [1 [19 43]] :ValueNotSwapFrom INFO [2019-12-17 12:31:03,054] jepsen worker 4 - jepsen.util 3424 :invoke :cas [1 [7 14]] INFO [2019-12-17 12:31:03,088] jepsen worker 28 - jepsen.util 3178 :ok :write [9 18] INFO [2019-12-17 12:31:03,088] jepsen worker 28 - jepsen.util 3178 :invoke :write [9 45] INFO [2019-12-17 12:31:03,089] jepsen worker 24 - jepsen.util 5424 :info :write [8 17] :RequestCanceledException INFO [2019-12-17 12:31:03,090] jepsen worker 24 - jepsen.util 5514 :invoke :read [8 nil] INFO [2019-12-17 12:31:03,094] jepsen worker 65 - jepsen.util 3575 :ok :read [21 18] INFO [2019-12-17 12:31:03,094] jepsen worker 65 - jepsen.util 3575 :invoke :cas [21 [36 0]] INFO [2019-12-17 12:31:03,124] jepsen worker 32 - jepsen.util 4802 :ok :write [10 10] INFO [2019-12-17 12:31:03,124] jepsen worker 32 - jepsen.util 4802 :invoke :read [10 nil] INFO [2019-12-17 12:31:03,124] jepsen worker 51 - jepsen.util 4011 :fail :cas [17 [2 6]] :ValueNotSwapFrom INFO [2019-12-17 12:31:03,125] jepsen worker 51 - jepsen.util 4011 :invoke :read [17 nil] INFO [2019-12-17 12:31:03,125] jepsen worker 86 - jepsen.util 5756 :info :write [28 16] :RequestCanceledException INFO [2019-12-17 12:31:03,126] jepsen worker 86 - jepsen.util 5846 :invoke :write [28 41] INFO [2019-12-17 12:31:03,139] jepsen worker 9 - jepsen.util 4869 :info :write [3 4] :RequestCanceledException INFO [2019-12-17 12:31:03,139] jepsen worker 46 - jepsen.util 4276 :info :cas [15 [26 38]] :RequestCanceledException INFO [2019-12-17 12:31:03,140] jepsen worker 9 - jepsen.util 4959 :invoke :write [3 3] INFO [2019-12-17 12:31:03,140] jepsen worker 65 - jepsen.util 3575 :fail :cas [21 [36 0]] :ValueNotSwapFrom INFO [2019-12-17 12:31:03,141] jepsen worker 65 - jepsen.util 3575 :invoke :write [21 44] INFO [2019-12-17 12:31:03,141] jepsen worker 46 - jepsen.util 4366 :invoke :cas [15 [25 26]] INFO [2019-12-17 12:31:03,153] jepsen worker 78 - jepsen.util 4488 :fail :cas [26 [31 26]] :ValueNotSwapFrom INFO [2019-12-17 12:31:03,154] jepsen worker 78 - jepsen.util 4488 :invoke :read [26 nil] INFO [2019-12-17 12:31:03,166] jepsen worker 32 - jepsen.util 4802 :ok :read [10 10] INFO [2019-12-17 12:31:03,166] jepsen worker 32 - jepsen.util 4802 :invoke :write [10 28] INFO [2019-12-17 12:31:03,169] jepsen worker 64 - jepsen.util 3394 :ok :write [21 16] INFO [2019-12-17 12:31:03,169] jepsen worker 64 - jepsen.util 3394 :invoke :write [21 2] INFO [2019-12-17 12:31:03,191] jepsen worker 49 - jepsen.util 4909 :info :write [16 40] :RequestCanceledException INFO [2019-12-17 12:31:03,192] jepsen worker 49 - jepsen.util 4999 :invoke :write [16 23] INFO [2019-12-17 12:31:03,226] jepsen worker 19 - jepsen.util 4429 :ok :write [6 34] INFO [2019-12-17 12:31:03,227] jepsen worker 19 - jepsen.util 4429 :invoke :cas [6 [2 44]] INFO [2019-12-17 12:31:03,227] jepsen worker 28 - jepsen.util 3178 :ok :write [9 45] INFO [2019-12-17 12:31:03,227] jepsen worker 28 - jepsen.util 3178 :invoke :write [9 43] INFO [2019-12-17 12:31:03,228] jepsen worker 53 - jepsen.util 4283 :ok :read [17 22] INFO [2019-12-17 12:31:03,228] jepsen worker 53 - jepsen.util 4283 :invoke :write [17 21] INFO [2019-12-17 12:31:03,236] jepsen worker 55 - jepsen.util 5275 :ok :write [18 27] INFO [2019-12-17 12:31:03,236] jepsen worker 55 - jepsen.util 5275 :invoke :read [18 nil] INFO [2019-12-17 12:31:03,251] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":24604,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:31:03,251] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":24604,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:31:03,254] jepsen worker 80 - jepsen.util 4310 :fail :cas [26 [40 42]] :ValueNotSwapFrom INFO [2019-12-17 12:31:03,254] jepsen worker 80 - jepsen.util 4310 :invoke :write [26 27] INFO [2019-12-17 12:31:03,264] jepsen worker 19 - jepsen.util 4429 :fail :cas [6 [2 44]] :ValueNotSwapFrom INFO [2019-12-17 12:31:03,264] jepsen worker 19 - jepsen.util 4429 :invoke :read [6 nil] INFO [2019-12-17 12:31:03,270] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":24595,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:31:03,270] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":24595,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:31:03,270] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":24596,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:31:03,270] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":24596,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:31:03,270] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":24597,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:31:03,270] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":24597,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:31:03,285] jepsen worker 64 - jepsen.util 3394 :ok :write [21 2] INFO [2019-12-17 12:31:03,285] jepsen worker 64 - jepsen.util 3394 :invoke :read [21 nil] INFO [2019-12-17 12:31:03,286] jepsen worker 87 - jepsen.util 4047 :ok :read [29 33] INFO [2019-12-17 12:31:03,286] jepsen worker 87 - jepsen.util 4047 :invoke :read [29 nil] INFO [2019-12-17 12:31:03,291] jepsen worker 19 - jepsen.util 4429 :ok :read [6 34] INFO [2019-12-17 12:31:03,291] jepsen worker 28 - jepsen.util 3178 :ok :write [9 43] INFO [2019-12-17 12:31:03,292] jepsen worker 28 - jepsen.util 3178 :invoke :cas [9 [20 42]] INFO [2019-12-17 12:31:03,292] jepsen worker 19 - jepsen.util 4429 :invoke :write [6 16] INFO [2019-12-17 12:31:03,303] jepsen worker 64 - jepsen.util 3394 :ok :read [21 2] INFO [2019-12-17 12:31:03,303] jepsen worker 64 - jepsen.util 3394 :invoke :read [21 nil] INFO [2019-12-17 12:31:03,310] jepsen worker 63 - jepsen.util 3483 :fail :read [21 nil] :RequestTimeoutException INFO [2019-12-17 12:31:03,310] jepsen worker 63 - jepsen.util 3483 :invoke :read [21 nil] INFO [2019-12-17 12:31:03,313] jepsen worker 4 - jepsen.util 3424 :fail :cas [1 [7 14]] :ValueNotSwapFrom INFO [2019-12-17 12:31:03,313] jepsen worker 4 - jepsen.util 3424 :invoke :write [1 27] INFO [2019-12-17 12:31:03,330] jepsen worker 79 - jepsen.util 4219 :ok :read [26 12] INFO [2019-12-17 12:31:03,330] jepsen worker 79 - jepsen.util 4219 :invoke :read [26 nil] INFO [2019-12-17 12:31:03,339] jepsen worker 34 - jepsen.util 5254 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:31:03,339] jepsen worker 34 - jepsen.util 5254 :invoke :cas [11 [47 4]] INFO [2019-12-17 12:31:03,357] jepsen worker 28 - jepsen.util 3178 :fail :cas [9 [20 42]] :ValueNotSwapFrom INFO [2019-12-17 12:31:03,357] jepsen worker 28 - jepsen.util 3178 :invoke :write [9 4] INFO [2019-12-17 12:31:03,368] jepsen worker 51 - jepsen.util 4011 :ok :read [17 22] INFO [2019-12-17 12:31:03,368] jepsen worker 51 - jepsen.util 4011 :invoke :write [17 28] INFO [2019-12-17 12:31:03,407] jepsen worker 74 - jepsen.util 4844 :info :cas [24 [26 41]] :RequestCanceledException INFO [2019-12-17 12:31:03,408] jepsen worker 74 - jepsen.util 4934 :invoke :cas [24 [25 29]] INFO [2019-12-17 12:31:03,410] jepsen worker 64 - jepsen.util 3394 :ok :read [21 2] INFO [2019-12-17 12:31:03,410] jepsen worker 64 - jepsen.util 3394 :invoke :write [21 21] INFO [2019-12-17 12:31:03,428] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":24628,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:31:03,428] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":24628,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:31:03,428] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":24631,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:31:03,428] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":24631,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:31:03,435] jepsen worker 55 - jepsen.util 5275 :ok :read [18 27] INFO [2019-12-17 12:31:03,435] jepsen worker 55 - jepsen.util 5275 :invoke :write [18 43] INFO [2019-12-17 12:31:03,453] jepsen worker 63 - jepsen.util 3483 :ok :read [21 2] INFO [2019-12-17 12:31:03,453] jepsen worker 73 - jepsen.util 4033 :info :cas [24 [14 4]] :RequestCanceledException INFO [2019-12-17 12:31:03,453] jepsen worker 63 - jepsen.util 3483 :invoke :read [21 nil] INFO [2019-12-17 12:31:03,454] jepsen worker 73 - jepsen.util 4123 :invoke :write [24 40] INFO [2019-12-17 12:31:03,456] jepsen worker 14 - jepsen.util 4424 :fail :cas [4 [35 6]] :ValueNotSwapFrom INFO [2019-12-17 12:31:03,456] jepsen worker 14 - jepsen.util 4424 :invoke :cas [4 [44 14]] INFO [2019-12-17 12:31:03,467] jepsen worker 89 - jepsen.util 4049 :ok :read [29 33] INFO [2019-12-17 12:31:03,467] jepsen worker 89 - jepsen.util 4049 :invoke :cas [29 [42 39]] INFO [2019-12-17 12:31:03,490] jepsen worker 74 - jepsen.util 4934 :fail :cas [24 [25 29]] :ValueNotSwapFrom INFO [2019-12-17 12:31:03,490] jepsen worker 74 - jepsen.util 4934 :invoke :read [24 nil] INFO [2019-12-17 12:31:03,491] jepsen worker 68 - jepsen.util 5108 :info :write [22 36] :RequestCanceledException INFO [2019-12-17 12:31:03,492] jepsen worker 68 - jepsen.util 5198 :invoke :write [22 37] INFO [2019-12-17 12:31:03,502] jepsen worker 12 - jepsen.util 4602 :fail :cas [4 [4 15]] :ValueNotSwapFrom INFO [2019-12-17 12:31:03,502] jepsen worker 12 - jepsen.util 4602 :invoke :read [4 nil] INFO [2019-12-17 12:31:03,515] jepsen worker 74 - jepsen.util 4934 :ok :read [24 47] INFO [2019-12-17 12:31:03,515] jepsen worker 74 - jepsen.util 4934 :invoke :read [24 nil] INFO [2019-12-17 12:31:03,516] jepsen worker 14 - jepsen.util 4424 :fail :cas [4 [44 14]] :ValueNotSwapFrom INFO [2019-12-17 12:31:03,516] jepsen worker 14 - jepsen.util 4424 :invoke :cas [4 [45 31]] INFO [2019-12-17 12:31:03,517] jepsen worker 70 - jepsen.util 4930 :fail :cas [23 [45 15]] :ValueNotSwapFrom INFO [2019-12-17 12:31:03,517] jepsen worker 70 - jepsen.util 4930 :invoke :read [23 nil] INFO [2019-12-17 12:31:03,533] jepsen worker 38 - jepsen.util 4268 :ok :read [12 14] INFO [2019-12-17 12:31:03,533] jepsen worker 38 - jepsen.util 4268 :invoke :write [12 23] INFO [2019-12-17 12:31:03,539] jepsen worker 27 - jepsen.util 4077 :info :cas [9 [8 21]] :RequestCanceledException INFO [2019-12-17 12:31:03,540] jepsen worker 27 - jepsen.util 4167 :invoke :write [9 25] INFO [2019-12-17 12:31:03,550] jepsen worker 28 - jepsen.util 3178 :ok :write [9 4] INFO [2019-12-17 12:31:03,550] jepsen worker 28 - jepsen.util 3178 :invoke :read [9 nil] INFO [2019-12-17 12:31:03,552] jepsen worker 74 - jepsen.util 4934 :ok :read [24 47] INFO [2019-12-17 12:31:03,552] jepsen worker 74 - jepsen.util 4934 :invoke :cas [24 [29 10]] INFO [2019-12-17 12:31:03,556] jepsen worker 23 - jepsen.util 5513 :info :cas [7 [28 39]] :RequestCanceledException INFO [2019-12-17 12:31:03,557] jepsen worker 23 - jepsen.util 5603 :invoke :read [7 nil] INFO [2019-12-17 12:31:03,560] jepsen worker 89 - jepsen.util 4049 :fail :cas [29 [42 39]] :ValueNotSwapFrom INFO [2019-12-17 12:31:03,560] jepsen worker 89 - jepsen.util 4049 :invoke :write [29 24] INFO [2019-12-17 12:31:03,566] jepsen worker 71 - jepsen.util 4571 :fail :cas [23 [45 45]] :ValueNotSwapFrom INFO [2019-12-17 12:31:03,566] jepsen worker 71 - jepsen.util 4571 :invoke :write [23 4] INFO [2019-12-17 12:31:03,570] jepsen worker 63 - jepsen.util 3483 :ok :read [21 2] INFO [2019-12-17 12:31:03,571] jepsen worker 63 - jepsen.util 3483 :invoke :write [21 29] INFO [2019-12-17 12:31:03,580] jepsen worker 64 - jepsen.util 3394 :ok :write [21 21] INFO [2019-12-17 12:31:03,580] jepsen worker 64 - jepsen.util 3394 :invoke :cas [21 [20 41]] INFO [2019-12-17 12:31:03,583] jepsen worker 79 - jepsen.util 4219 :ok :read [26 12] INFO [2019-12-17 12:31:03,583] jepsen worker 74 - jepsen.util 4934 :fail :cas [24 [29 10]] :ValueNotSwapFrom INFO [2019-12-17 12:31:03,583] jepsen worker 79 - jepsen.util 4219 :invoke :cas [26 [9 42]] INFO [2019-12-17 12:31:03,583] jepsen worker 74 - jepsen.util 4934 :invoke :write [24 15] INFO [2019-12-17 12:31:03,596] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:31:03,596] jepsen nemesis - couchbase.util Rebalance check count: 182 INFO [2019-12-17 12:31:03,596] jepsen nemesis - couchbase.util Rebalance stuck count: 163 INFO [2019-12-17 12:31:03,617] jepsen worker 28 - jepsen.util 3178 :ok :read [9 4] INFO [2019-12-17 12:31:03,617] jepsen worker 28 - jepsen.util 3178 :invoke :cas [9 [32 24]] INFO [2019-12-17 12:31:03,623] jepsen worker 32 - jepsen.util 4802 :ok :write [10 28] INFO [2019-12-17 12:31:03,624] jepsen worker 32 - jepsen.util 4802 :invoke :cas [10 [49 39]] INFO [2019-12-17 12:31:03,624] jepsen worker 44 - jepsen.util 6074 :info :write [14 48] :RequestCanceledException INFO [2019-12-17 12:31:03,625] jepsen worker 44 - jepsen.util 6164 :invoke :cas [14 [25 41]] INFO [2019-12-17 12:31:03,638] jepsen worker 45 - jepsen.util 4365 :info :cas [15 [14 10]] :RequestCanceledException INFO [2019-12-17 12:31:03,639] jepsen worker 45 - jepsen.util 4455 :invoke :read [15 nil] INFO [2019-12-17 12:31:03,676] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":24660,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:31:03,676] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":24660,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:31:03,676] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":24663,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:31:03,676] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":24663,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:31:03,676] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":24664,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:31:03,676] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":24664,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:31:03,691] jepsen worker 59 - jepsen.util 5639 :info :cas [19 [26 33]] :RequestCanceledException INFO [2019-12-17 12:31:03,692] jepsen worker 59 - jepsen.util 5729 :invoke :write [19 3] INFO [2019-12-17 12:31:03,708] jepsen worker 43 - jepsen.util 5173 :info :write [14 13] :RequestCanceledException INFO [2019-12-17 12:31:03,709] jepsen worker 43 - jepsen.util 5263 :invoke :write [14 39] INFO [2019-12-17 12:31:03,715] jepsen worker 64 - jepsen.util 3394 :fail :cas [21 [20 41]] :ValueNotSwapFrom INFO [2019-12-17 12:31:03,715] jepsen worker 64 - jepsen.util 3394 :invoke :cas [21 [22 4]] INFO [2019-12-17 12:31:03,716] jepsen worker 78 - jepsen.util 4488 :ok :read [26 12] INFO [2019-12-17 12:31:03,717] jepsen worker 78 - jepsen.util 4488 :invoke :write [26 33] INFO [2019-12-17 12:31:03,725] jepsen worker 6 - jepsen.util 6216 :info :cas [2 [5 2]] :RequestCanceledException INFO [2019-12-17 12:31:03,725] jepsen worker 66 - jepsen.util 5376 :info :write [22 49] :RequestCanceledException INFO [2019-12-17 12:31:03,726] jepsen worker 6 - jepsen.util 6306 :invoke :read [2 nil] INFO [2019-12-17 12:31:03,726] jepsen worker 66 - jepsen.util 5466 :invoke :read [22 nil] INFO [2019-12-17 12:31:03,739] jepsen worker 25 - jepsen.util 5875 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:31:03,739] jepsen worker 25 - jepsen.util 5875 :invoke :read [8 nil] INFO [2019-12-17 12:31:03,754] jepsen worker 80 - jepsen.util 4310 :ok :write [26 27] INFO [2019-12-17 12:31:03,754] jepsen worker 80 - jepsen.util 4310 :invoke :cas [26 [32 49]] INFO [2019-12-17 12:31:03,756] jepsen worker 32 - jepsen.util 4802 :fail :cas [10 [49 39]] :ValueNotSwapFrom INFO [2019-12-17 12:31:03,756] jepsen worker 84 - jepsen.util 5124 :info :write [28 41] :RequestCanceledException INFO [2019-12-17 12:31:03,757] jepsen worker 32 - jepsen.util 4802 :invoke :write [10 28] INFO [2019-12-17 12:31:03,757] jepsen worker 64 - jepsen.util 3394 :fail :cas [21 [22 4]] :ValueNotSwapFrom INFO [2019-12-17 12:31:03,757] jepsen worker 64 - jepsen.util 3394 :invoke :read [21 nil] INFO [2019-12-17 12:31:03,758] jepsen worker 84 - jepsen.util 5214 :invoke :write [28 38] INFO [2019-12-17 12:31:03,765] jepsen worker 14 - jepsen.util 4424 :fail :cas [4 [45 31]] :ValueNotSwapFrom INFO [2019-12-17 12:31:03,765] jepsen worker 14 - jepsen.util 4424 :invoke :read [4 nil] INFO [2019-12-17 12:31:03,782] jepsen worker 64 - jepsen.util 3394 :ok :read [21 21] INFO [2019-12-17 12:31:03,782] jepsen worker 64 - jepsen.util 3394 :invoke :read [21 nil] INFO [2019-12-17 12:31:03,783] jepsen worker 63 - jepsen.util 3483 :ok :write [21 29] INFO [2019-12-17 12:31:03,783] jepsen worker 63 - jepsen.util 3483 :invoke :read [21 nil] INFO [2019-12-17 12:31:03,791] jepsen worker 85 - jepsen.util 6295 :info :write [28 0] :RequestCanceledException INFO [2019-12-17 12:31:03,794] jepsen worker 85 - jepsen.util 6385 :invoke :read [28 nil] INFO [2019-12-17 12:31:03,824] jepsen worker 9 - jepsen.util 4959 :ok :write [3 3] INFO [2019-12-17 12:31:03,824] jepsen worker 62 - jepsen.util 5282 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:31:03,824] jepsen worker 10 - jepsen.util 4150 :info :write [3 24] :RequestCanceledException INFO [2019-12-17 12:31:03,824] jepsen worker 30 - jepsen.util 4440 :fail :read [10 nil] :RequestTimeoutException INFO [2019-12-17 12:31:03,824] jepsen worker 20 - jepsen.util 4250 :info :cas [6 [8 12]] :RequestCanceledException INFO [2019-12-17 12:31:03,824] jepsen worker 9 - jepsen.util 4959 :invoke :read [3 nil] INFO [2019-12-17 12:31:03,824] jepsen worker 62 - jepsen.util 5282 :invoke :read [20 nil] INFO [2019-12-17 12:31:03,824] jepsen worker 30 - jepsen.util 4440 :invoke :write [10 20] INFO [2019-12-17 12:31:03,825] jepsen worker 10 - jepsen.util 4240 :invoke :cas [3 [23 7]] INFO [2019-12-17 12:31:03,825] jepsen worker 20 - jepsen.util 4340 :invoke :cas [6 [32 2]] INFO [2019-12-17 12:31:03,831] jepsen worker 64 - jepsen.util 3394 :ok :read [21 29] INFO [2019-12-17 12:31:03,831] jepsen worker 63 - jepsen.util 3483 :ok :read [21 29] INFO [2019-12-17 12:31:03,831] jepsen worker 64 - jepsen.util 3394 :invoke :read [21 nil] INFO [2019-12-17 12:31:03,831] jepsen worker 63 - jepsen.util 3483 :invoke :cas [21 [2 30]] INFO [2019-12-17 12:31:03,839] jepsen worker 82 - jepsen.util 6562 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:31:03,839] jepsen worker 82 - jepsen.util 6562 :invoke :write [27 33] INFO [2019-12-17 12:31:03,846] jepsen worker 10 - jepsen.util 4240 :fail :cas [3 [23 7]] :ValueNotSwapFrom INFO [2019-12-17 12:31:03,846] jepsen worker 10 - jepsen.util 4240 :invoke :read [3 nil] INFO [2019-12-17 12:31:03,857] jepsen worker 7 - jepsen.util 5497 :info :cas [2 [11 4]] :RequestCanceledException INFO [2019-12-17 12:31:03,858] jepsen worker 7 - jepsen.util 5587 :invoke :read [2 nil] INFO [2019-12-17 12:31:03,875] jepsen worker 63 - jepsen.util 3483 :fail :cas [21 [2 30]] :ValueNotSwapFrom INFO [2019-12-17 12:31:03,875] jepsen worker 63 - jepsen.util 3483 :invoke :read [21 nil] INFO [2019-12-17 12:31:03,907] jepsen worker 64 - jepsen.util 3394 :ok :read [21 29] INFO [2019-12-17 12:31:03,907] jepsen worker 64 - jepsen.util 3394 :invoke :read [21 nil] INFO [2019-12-17 12:31:03,922] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":24691,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:31:03,923] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":24691,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:31:03,923] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":24694,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:31:03,923] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":24694,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:31:03,923] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":24695,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:31:03,923] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":24695,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:31:03,925] jepsen worker 39 - jepsen.util 5169 :info :cas [13 [34 34]] :RequestCanceledException INFO [2019-12-17 12:31:03,926] jepsen worker 39 - jepsen.util 5259 :invoke :write [13 33] INFO [2019-12-17 12:31:03,959] jepsen worker 10 - jepsen.util 4240 :ok :read [3 3] INFO [2019-12-17 12:31:03,960] jepsen worker 10 - jepsen.util 4240 :invoke :read [3 nil] INFO [2019-12-17 12:31:03,973] jepsen worker 63 - jepsen.util 3483 :ok :read [21 29] INFO [2019-12-17 12:31:03,974] jepsen worker 63 - jepsen.util 3483 :invoke :cas [21 [1 42]] INFO [2019-12-17 12:31:04,003] jepsen worker 12 - jepsen.util 4602 :ok :read [4 14] INFO [2019-12-17 12:31:04,004] jepsen worker 12 - jepsen.util 4602 :invoke :cas [4 [22 5]] INFO [2019-12-17 12:31:04,009] jepsen worker 54 - jepsen.util 4644 :info :write [18 26] :RequestCanceledException INFO [2019-12-17 12:31:04,009] jepsen worker 47 - jepsen.util 4817 :fail :read [15 nil] :RequestTimeoutException INFO [2019-12-17 12:31:04,009] jepsen worker 47 - jepsen.util 4817 :invoke :write [15 2] INFO [2019-12-17 12:31:04,013] jepsen worker 37 - jepsen.util 4087 :ok :write [12 18] INFO [2019-12-17 12:31:04,013] jepsen worker 37 - jepsen.util 4087 :invoke :read [12 nil] INFO [2019-12-17 12:31:04,014] jepsen worker 54 - jepsen.util 4734 :invoke :cas [18 [22 7]] INFO [2019-12-17 12:31:04,025] jepsen worker 83 - jepsen.util 5843 :info :cas [27 [17 49]] :RequestCanceledException INFO [2019-12-17 12:31:04,026] jepsen worker 83 - jepsen.util 5933 :invoke :cas [27 [22 3]] INFO [2019-12-17 12:31:04,037] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":24703,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:31:04,037] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":24703,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:31:04,039] jepsen worker 40 - jepsen.util 6340 :info :cas [13 [46 5]] :RequestCanceledException INFO [2019-12-17 12:31:04,041] jepsen worker 64 - jepsen.util 3394 :ok :read [21 29] INFO [2019-12-17 12:31:04,041] jepsen worker 64 - jepsen.util 3394 :invoke :read [21 nil] INFO [2019-12-17 12:31:04,049] jepsen worker 40 - jepsen.util 6430 :invoke :cas [13 [19 47]] INFO [2019-12-17 12:31:04,050] jepsen worker 63 - jepsen.util 3483 :fail :cas [21 [1 42]] :ValueNotSwapFrom INFO [2019-12-17 12:31:04,050] jepsen worker 63 - jepsen.util 3483 :invoke :write [21 23] INFO [2019-12-17 12:31:04,056] jepsen worker 2 - jepsen.util 5402 :info :cas [0 [7 40]] :RequestCanceledException INFO [2019-12-17 12:31:04,057] jepsen worker 2 - jepsen.util 5492 :invoke :cas [0 [0 24]] INFO [2019-12-17 12:31:04,058] jepsen worker 70 - jepsen.util 4930 :ok :read [23 11] INFO [2019-12-17 12:31:04,059] jepsen worker 70 - jepsen.util 4930 :invoke :write [23 20] INFO [2019-12-17 12:31:04,079] jepsen worker 51 - jepsen.util 4011 :ok :write [17 28] INFO [2019-12-17 12:31:04,079] jepsen worker 51 - jepsen.util 4011 :invoke :cas [17 [49 38]] INFO [2019-12-17 12:31:04,087] jepsen worker 10 - jepsen.util 4240 :ok :read [3 3] INFO [2019-12-17 12:31:04,087] jepsen worker 10 - jepsen.util 4240 :invoke :cas [3 [33 48]] INFO [2019-12-17 12:31:04,091] jepsen worker 60 - jepsen.util 6270 :info :cas [20 [24 19]] :RequestCanceledException INFO [2019-12-17 12:31:04,091] jepsen worker 35 - jepsen.util 5615 :info :cas [11 [37 15]] :RequestCanceledException INFO [2019-12-17 12:31:04,092] jepsen worker 60 - jepsen.util 6360 :invoke :write [20 19] INFO [2019-12-17 12:31:04,092] jepsen worker 35 - jepsen.util 5705 :invoke :cas [11 [15 18]] INFO [2019-12-17 12:31:04,109] jepsen worker 50 - jepsen.util 5900 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:31:04,109] jepsen worker 58 - jepsen.util 5728 :info :write [19 27] :RequestCanceledException INFO [2019-12-17 12:31:04,109] jepsen worker 50 - jepsen.util 5900 :invoke :write [16 16] INFO [2019-12-17 12:31:04,110] jepsen worker 58 - jepsen.util 5818 :invoke :read [19 nil] INFO [2019-12-17 12:31:04,125] jepsen worker 0 - jepsen.util 6300 :info :write [0 5] :RequestCanceledException INFO [2019-12-17 12:31:04,126] jepsen worker 0 - jepsen.util 6390 :invoke :read [0 nil] INFO [2019-12-17 12:31:04,129] jepsen worker 27 - jepsen.util 4167 :ok :write [9 25] INFO [2019-12-17 12:31:04,130] jepsen worker 27 - jepsen.util 4167 :invoke :read [9 nil] INFO [2019-12-17 12:31:04,130] jepsen worker 54 - jepsen.util 4734 :fail :cas [18 [22 7]] :ValueNotSwapFrom INFO [2019-12-17 12:31:04,130] jepsen worker 54 - jepsen.util 4734 :invoke :read [18 nil] INFO [2019-12-17 12:31:04,133] jepsen worker 64 - jepsen.util 3394 :ok :read [21 29] INFO [2019-12-17 12:31:04,133] jepsen worker 64 - jepsen.util 3394 :invoke :write [21 36] INFO [2019-12-17 12:31:04,167] jepsen worker 10 - jepsen.util 4240 :fail :cas [3 [33 48]] :ValueNotSwapFrom INFO [2019-12-17 12:31:04,167] jepsen worker 10 - jepsen.util 4240 :invoke :read [3 nil] INFO [2019-12-17 12:31:04,183] jepsen worker 87 - jepsen.util 4047 :ok :read [29 33] INFO [2019-12-17 12:31:04,183] jepsen worker 87 - jepsen.util 4047 :invoke :write [29 3] INFO [2019-12-17 12:31:04,188] jepsen worker 67 - jepsen.util 5827 :info :cas [22 [4 12]] :RequestCanceledException INFO [2019-12-17 12:31:04,189] jepsen worker 67 - jepsen.util 5917 :invoke :cas [22 [38 7]] INFO [2019-12-17 12:31:04,209] jepsen worker 26 - jepsen.util 5696 :info :cas [8 [10 12]] :RequestCanceledException INFO [2019-12-17 12:31:04,210] jepsen worker 26 - jepsen.util 5786 :invoke :cas [8 [19 45]] INFO [2019-12-17 12:31:04,223] jepsen worker 57 - jepsen.util 5367 :info :write [19 40] :RequestCanceledException INFO [2019-12-17 12:31:04,223] jepsen worker 22 - jepsen.util 5692 :info :cas [7 [45 3]] :RequestCanceledException INFO [2019-12-17 12:31:04,224] jepsen worker 57 - jepsen.util 5457 :invoke :write [19 7] INFO [2019-12-17 12:31:04,224] jepsen worker 22 - jepsen.util 5782 :invoke :read [7 nil] INFO [2019-12-17 12:31:04,237] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":24737,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:31:04,238] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":24737,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:31:04,238] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":24740,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:31:04,238] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":24740,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:31:04,238] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":24741,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:31:04,238] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":24741,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:31:04,239] jepsen worker 48 - jepsen.util 5178 :info :cas [16 [42 28]] :RequestCanceledException INFO [2019-12-17 12:31:04,240] jepsen worker 48 - jepsen.util 5268 :invoke :read [16 nil] INFO [2019-12-17 12:31:04,246] jepsen worker 80 - jepsen.util 4310 :fail :cas [26 [32 49]] :ValueNotSwapFrom INFO [2019-12-17 12:31:04,246] jepsen worker 80 - jepsen.util 4310 :invoke :cas [26 [29 36]] INFO [2019-12-17 12:31:04,247] jepsen worker 27 - jepsen.util 4167 :ok :read [9 25] INFO [2019-12-17 12:31:04,247] jepsen worker 27 - jepsen.util 4167 :invoke :cas [9 [7 38]] INFO [2019-12-17 12:31:04,252] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":24730,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:31:04,252] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":24730,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:31:04,252] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":24732,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:31:04,252] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":24732,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:31:04,253] jepsen worker 17 - jepsen.util 5417 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:31:04,253] jepsen worker 17 - jepsen.util 5417 :invoke :cas [5 [30 22]] INFO [2019-12-17 12:31:04,278] jepsen worker 37 - jepsen.util 4087 :ok :read [12 18] INFO [2019-12-17 12:31:04,278] jepsen worker 37 - jepsen.util 4087 :invoke :cas [12 [47 28]] INFO [2019-12-17 12:31:04,287] jepsen worker 56 - jepsen.util 4466 :info :write [18 24] :RequestCanceledException INFO [2019-12-17 12:31:04,288] jepsen worker 56 - jepsen.util 4556 :invoke :read [18 nil] INFO [2019-12-17 12:31:04,294] jepsen worker 27 - jepsen.util 4167 :fail :cas [9 [7 38]] :ValueNotSwapFrom INFO [2019-12-17 12:31:04,294] jepsen worker 27 - jepsen.util 4167 :invoke :read [9 nil] INFO [2019-12-17 12:31:04,310] jepsen worker 64 - jepsen.util 3394 :ok :write [21 36] INFO [2019-12-17 12:31:04,310] jepsen worker 64 - jepsen.util 3394 :invoke :read [21 nil] INFO [2019-12-17 12:31:04,316] jepsen worker 27 - jepsen.util 4167 :ok :read [9 25] INFO [2019-12-17 12:31:04,316] jepsen worker 27 - jepsen.util 4167 :invoke :write [9 7] INFO [2019-12-17 12:31:04,324] jepsen worker 42 - jepsen.util 6432 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:31:04,325] jepsen worker 42 - jepsen.util 6432 :invoke :read [14 nil] INFO [2019-12-17 12:31:04,342] jepsen worker 64 - jepsen.util 3394 :ok :read [21 36] INFO [2019-12-17 12:31:04,342] jepsen worker 64 - jepsen.util 3394 :invoke :cas [21 [28 20]] INFO [2019-12-17 12:31:04,367] jepsen worker 37 - jepsen.util 4087 :fail :cas [12 [47 28]] :ValueNotSwapFrom INFO [2019-12-17 12:31:04,367] jepsen worker 37 - jepsen.util 4087 :invoke :cas [12 [38 12]] INFO [2019-12-17 12:31:04,389] jepsen worker 41 - jepsen.util 5891 :info :write [13 2] :RequestCanceledException INFO [2019-12-17 12:31:04,389] jepsen worker 31 - jepsen.util 3901 :info :write [10 16] :RequestCanceledException INFO [2019-12-17 12:31:04,390] jepsen worker 41 - jepsen.util 5981 :invoke :write [13 13] INFO [2019-12-17 12:31:04,390] jepsen worker 31 - jepsen.util 3991 :invoke :read [10 nil] INFO [2019-12-17 12:31:04,390] jepsen worker 19 - jepsen.util 4429 :ok :write [6 16] INFO [2019-12-17 12:31:04,390] jepsen worker 74 - jepsen.util 4934 :ok :write [24 15] INFO [2019-12-17 12:31:04,390] jepsen worker 32 - jepsen.util 4802 :ok :write [10 28] INFO [2019-12-17 12:31:04,390] jepsen worker 19 - jepsen.util 4429 :invoke :read [6 nil] INFO [2019-12-17 12:31:04,390] jepsen worker 74 - jepsen.util 4934 :invoke :write [24 42] INFO [2019-12-17 12:31:04,390] jepsen worker 32 - jepsen.util 4802 :invoke :cas [10 [7 17]] INFO [2019-12-17 12:31:04,391] jepsen worker 64 - jepsen.util 3394 :fail :cas [21 [28 20]] :ValueNotSwapFrom INFO [2019-12-17 12:31:04,391] jepsen worker 64 - jepsen.util 3394 :invoke :cas [21 [45 5]] INFO [2019-12-17 12:31:04,409] jepsen worker 33 - jepsen.util 5613 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:31:04,409] jepsen worker 33 - jepsen.util 5613 :invoke :write [11 46] INFO [2019-12-17 12:31:04,423] jepsen worker 8 - jepsen.util 5048 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:31:04,423] jepsen worker 75 - jepsen.util 5655 :info :write [25 11] :RequestCanceledException INFO [2019-12-17 12:31:04,423] jepsen worker 8 - jepsen.util 5048 :invoke :read [2 nil] INFO [2019-12-17 12:31:04,424] jepsen worker 75 - jepsen.util 5745 :invoke :write [25 37] INFO [2019-12-17 12:31:04,450] jepsen worker 53 - jepsen.util 4283 :ok :write [17 21] INFO [2019-12-17 12:31:04,450] jepsen worker 53 - jepsen.util 4283 :invoke :write [17 22] INFO [2019-12-17 12:31:04,454] jepsen worker 1 - jepsen.util 5221 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:31:04,454] jepsen worker 1 - jepsen.util 5221 :invoke :write [0 6] INFO [2019-12-17 12:31:04,469] jepsen worker 51 - jepsen.util 4011 :fail :cas [17 [49 38]] :ValueNotSwapFrom INFO [2019-12-17 12:31:04,469] jepsen worker 51 - jepsen.util 4011 :invoke :write [17 7] INFO [2019-12-17 12:31:04,478] jepsen worker 47 - jepsen.util 4817 :ok :write [15 2] INFO [2019-12-17 12:31:04,478] jepsen worker 47 - jepsen.util 4817 :invoke :read [15 nil] INFO [2019-12-17 12:31:04,497] jepsen worker 54 - jepsen.util 4734 :ok :read [18 27] INFO [2019-12-17 12:31:04,498] jepsen worker 54 - jepsen.util 4734 :invoke :write [18 23] INFO [2019-12-17 12:31:04,504] jepsen worker 80 - jepsen.util 4310 :fail :cas [26 [29 36]] :ValueNotSwapFrom INFO [2019-12-17 12:31:04,504] jepsen worker 80 - jepsen.util 4310 :invoke :cas [26 [9 27]] INFO [2019-12-17 12:31:04,506] jepsen worker 71 - jepsen.util 4571 :ok :write [23 4] INFO [2019-12-17 12:31:04,506] jepsen worker 71 - jepsen.util 4571 :invoke :cas [23 [2 13]] INFO [2019-12-17 12:31:04,508] jepsen worker 20 - jepsen.util 4340 :fail :cas [6 [32 2]] :ValueNotSwapFrom INFO [2019-12-17 12:31:04,509] jepsen worker 20 - jepsen.util 4340 :invoke :cas [6 [24 5]] INFO [2019-12-17 12:31:04,511] jepsen worker 64 - jepsen.util 3394 :fail :cas [21 [45 5]] :ValueNotSwapFrom INFO [2019-12-17 12:31:04,511] jepsen worker 64 - jepsen.util 3394 :invoke :write [21 15] INFO [2019-12-17 12:31:04,518] jepsen worker 19 - jepsen.util 4429 :ok :read [6 16] INFO [2019-12-17 12:31:04,518] jepsen worker 19 - jepsen.util 4429 :invoke :read [6 nil] INFO [2019-12-17 12:31:04,524] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":24765,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:31:04,524] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":24765,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:31:04,524] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":24769,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:31:04,524] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":24769,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:31:04,531] jepsen worker 79 - jepsen.util 4219 :fail :cas [26 [9 42]] :ValueNotSwapFrom INFO [2019-12-17 12:31:04,531] jepsen worker 79 - jepsen.util 4219 :invoke :cas [26 [1 35]] INFO [2019-12-17 12:31:04,539] jepsen worker 76 - jepsen.util 5746 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:31:04,539] jepsen worker 76 - jepsen.util 5746 :invoke :cas [25 [48 39]] INFO [2019-12-17 12:31:04,578] jepsen worker 80 - jepsen.util 4310 :fail :cas [26 [9 27]] :ValueNotSwapFrom INFO [2019-12-17 12:31:04,578] jepsen worker 80 - jepsen.util 4310 :invoke :cas [26 [7 39]] INFO [2019-12-17 12:31:04,585] jepsen worker 47 - jepsen.util 4817 :ok :read [15 2] INFO [2019-12-17 12:31:04,585] jepsen worker 47 - jepsen.util 4817 :invoke :cas [15 [2 44]] INFO [2019-12-17 12:31:04,590] jepsen worker 37 - jepsen.util 4087 :fail :cas [12 [38 12]] :ValueNotSwapFrom INFO [2019-12-17 12:31:04,590] jepsen worker 37 - jepsen.util 4087 :invoke :write [12 14] INFO [2019-12-17 12:31:04,590] jepsen worker 15 - jepsen.util 4875 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:31:04,590] jepsen worker 16 - jepsen.util 6406 :info :write [5 37] :RequestCanceledException INFO [2019-12-17 12:31:04,591] jepsen worker 15 - jepsen.util 4875 :invoke :read [5 nil] INFO [2019-12-17 12:31:04,592] jepsen worker 16 - jepsen.util 6496 :invoke :read [5 nil] INFO [2019-12-17 12:31:04,596] jepsen worker 27 - jepsen.util 4167 :ok :write [9 7] INFO [2019-12-17 12:31:04,596] jepsen worker 27 - jepsen.util 4167 :invoke :write [9 2] INFO [2019-12-17 12:31:04,597] jepsen worker 19 - jepsen.util 4429 :ok :read [6 16] INFO [2019-12-17 12:31:04,597] jepsen worker 19 - jepsen.util 4429 :invoke :cas [6 [23 46]] INFO [2019-12-17 12:31:04,601] jepsen worker 14 - jepsen.util 4424 :ok :read [4 14] INFO [2019-12-17 12:31:04,601] jepsen worker 14 - jepsen.util 4424 :invoke :write [4 38] INFO [2019-12-17 12:31:04,622] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:31:04,622] jepsen nemesis - couchbase.util Rebalance check count: 183 INFO [2019-12-17 12:31:04,622] jepsen nemesis - couchbase.util Rebalance stuck count: 164 INFO [2019-12-17 12:31:04,625] jepsen worker 81 - jepsen.util 5661 :info :write [27 3] :RequestCanceledException INFO [2019-12-17 12:31:04,625] jepsen worker 13 - jepsen.util 3973 :info :write [4 11] :RequestCanceledException INFO [2019-12-17 12:31:04,626] jepsen worker 13 - jepsen.util 4063 :invoke :read [4 nil] INFO [2019-12-17 12:31:04,626] jepsen worker 81 - jepsen.util 5751 :invoke :read [27 nil] INFO [2019-12-17 12:31:04,653] jepsen worker 19 - jepsen.util 4429 :fail :cas [6 [23 46]] :ValueNotSwapFrom INFO [2019-12-17 12:31:04,653] jepsen worker 19 - jepsen.util 4429 :invoke :cas [6 [2 10]] INFO [2019-12-17 12:31:04,660] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":24796,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:31:04,661] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":24796,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:31:04,661] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":24799,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:31:04,661] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":24799,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:31:04,661] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":24800,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:31:04,661] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":24800,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:31:04,695] jepsen worker 19 - jepsen.util 4429 :fail :cas [6 [2 10]] :ValueNotSwapFrom INFO [2019-12-17 12:31:04,695] jepsen worker 19 - jepsen.util 4429 :invoke :write [6 7] INFO [2019-12-17 12:31:04,705] jepsen worker 64 - jepsen.util 3394 :ok :write [21 15] INFO [2019-12-17 12:31:04,705] jepsen worker 64 - jepsen.util 3394 :invoke :read [21 nil] INFO [2019-12-17 12:31:04,707] jepsen worker 69 - jepsen.util 4299 :info :write [23 25] :RequestCanceledException INFO [2019-12-17 12:31:04,708] jepsen worker 69 - jepsen.util 4389 :invoke :read [23 nil] INFO [2019-12-17 12:31:04,732] jepsen worker 64 - jepsen.util 3394 :ok :read [21 15] INFO [2019-12-17 12:31:04,732] jepsen worker 64 - jepsen.util 3394 :invoke :cas [21 [17 9]] INFO [2019-12-17 12:31:04,782] jepsen worker 64 - jepsen.util 3394 :fail :cas [21 [17 9]] :ValueNotSwapFrom INFO [2019-12-17 12:31:04,782] jepsen worker 64 - jepsen.util 3394 :invoke :read [21 nil] INFO [2019-12-17 12:31:04,796] jepsen worker 80 - jepsen.util 4310 :fail :cas [26 [7 39]] :ValueNotSwapFrom INFO [2019-12-17 12:31:04,796] jepsen worker 27 - jepsen.util 4167 :ok :write [9 2] INFO [2019-12-17 12:31:04,796] jepsen worker 80 - jepsen.util 4310 :invoke :cas [26 [19 30]] INFO [2019-12-17 12:31:04,796] jepsen worker 27 - jepsen.util 4167 :invoke :cas [9 [11 26]] INFO [2019-12-17 12:31:04,801] jepsen worker 64 - jepsen.util 3394 :ok :read [21 15] INFO [2019-12-17 12:31:04,801] jepsen worker 64 - jepsen.util 3394 :invoke :read [21 nil] INFO [2019-12-17 12:31:04,809] jepsen worker 21 - jepsen.util 5601 :info :write [7 49] :RequestCanceledException INFO [2019-12-17 12:31:04,810] jepsen worker 21 - jepsen.util 5691 :invoke :read [7 nil] INFO [2019-12-17 12:31:04,824] jepsen worker 72 - jepsen.util 4482 :info :write [24 19] :RequestCanceledException INFO [2019-12-17 12:31:04,824] jepsen worker 88 - jepsen.util 4408 :info :write [29 40] :RequestCanceledException INFO [2019-12-17 12:31:04,826] jepsen worker 88 - jepsen.util 4498 :invoke :read [29 nil] INFO [2019-12-17 12:31:04,826] jepsen worker 72 - jepsen.util 4572 :invoke :read [24 nil] INFO [2019-12-17 12:31:04,830] jepsen worker 64 - jepsen.util 3394 :ok :read [21 15] INFO [2019-12-17 12:31:04,831] jepsen worker 64 - jepsen.util 3394 :invoke :write [21 19] INFO [2019-12-17 12:31:04,835] jepsen worker 27 - jepsen.util 4167 :fail :cas [9 [11 26]] :ValueNotSwapFrom INFO [2019-12-17 12:31:04,835] jepsen worker 27 - jepsen.util 4167 :invoke :read [9 nil] INFO [2019-12-17 12:31:04,839] jepsen worker 3 - jepsen.util 4413 :info :write [1 2] :RequestCanceledException INFO [2019-12-17 12:31:04,840] jepsen worker 3 - jepsen.util 4503 :invoke :cas [1 [1 35]] INFO [2019-12-17 12:31:04,849] jepsen worker 80 - jepsen.util 4310 :fail :cas [26 [19 30]] :ValueNotSwapFrom INFO [2019-12-17 12:31:04,849] jepsen worker 80 - jepsen.util 4310 :invoke :cas [26 [23 23]] INFO [2019-12-17 12:31:04,856] jepsen worker 36 - jepsen.util 4356 :info :write [12 35] :RequestCanceledException INFO [2019-12-17 12:31:04,856] jepsen worker 5 - jepsen.util 3875 :info :write [1 43] :RequestCanceledException INFO [2019-12-17 12:31:04,857] jepsen worker 5 - jepsen.util 3965 :invoke :cas [1 [7 43]] INFO [2019-12-17 12:31:04,857] jepsen worker 36 - jepsen.util 4446 :invoke :write [12 31] INFO [2019-12-17 12:31:04,865] jepsen worker 27 - jepsen.util 4167 :ok :read [9 2] INFO [2019-12-17 12:31:04,865] jepsen worker 27 - jepsen.util 4167 :invoke :write [9 36] INFO [2019-12-17 12:31:04,885] jepsen worker 5 - jepsen.util 3965 :fail :cas [1 [7 43]] :ValueNotSwapFrom INFO [2019-12-17 12:31:04,885] jepsen worker 5 - jepsen.util 3965 :invoke :read [1 nil] INFO [2019-12-17 12:31:04,907] jepsen worker 30 - jepsen.util 4440 :ok :write [10 20] INFO [2019-12-17 12:31:04,907] jepsen worker 30 - jepsen.util 4440 :invoke :cas [10 [2 26]] INFO [2019-12-17 12:31:04,909] jepsen worker 11 - jepsen.util 4511 :info :cas [3 [15 30]] :RequestCanceledException INFO [2019-12-17 12:31:04,909] jepsen worker 18 - jepsen.util 4248 :info :cas [6 [16 24]] :RequestCanceledException INFO [2019-12-17 12:31:04,910] jepsen worker 18 - jepsen.util 4338 :invoke :write [6 16] INFO [2019-12-17 12:31:04,910] jepsen worker 11 - jepsen.util 4601 :invoke :write [3 46] INFO [2019-12-17 12:31:04,911] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":24822,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:31:04,911] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":24822,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:31:04,911] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":24823,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:31:04,911] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":24823,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:31:04,924] jepsen worker 31 - jepsen.util 3991 :ok :read [10 20] INFO [2019-12-17 12:31:04,924] jepsen worker 31 - jepsen.util 3991 :invoke :write [10 2] INFO [2019-12-17 12:31:04,925] jepsen worker 61 - jepsen.util 6361 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:31:04,925] jepsen worker 61 - jepsen.util 6361 :invoke :read [20 nil] INFO [2019-12-17 12:31:04,946] jepsen worker 69 - jepsen.util 4389 :ok :read [23 4] INFO [2019-12-17 12:31:04,946] jepsen worker 69 - jepsen.util 4389 :invoke :write [23 45] INFO [2019-12-17 12:31:04,951] jepsen worker 13 - jepsen.util 4063 :ok :read [4 14] INFO [2019-12-17 12:31:04,952] jepsen worker 88 - jepsen.util 4498 :ok :read [29 33] INFO [2019-12-17 12:31:04,955] jepsen worker 13 - jepsen.util 4063 :invoke :write [4 15] INFO [2019-12-17 12:31:04,955] jepsen worker 88 - jepsen.util 4498 :invoke :cas [29 [44 33]] INFO [2019-12-17 12:31:04,956] jepsen worker 29 - jepsen.util 3629 :fail :read [9 nil] :RequestTimeoutException INFO [2019-12-17 12:31:04,957] jepsen worker 29 - jepsen.util 3629 :invoke :read [9 nil] INFO [2019-12-17 12:31:04,971] jepsen worker 32 - jepsen.util 4802 :fail :cas [10 [7 17]] :ValueNotSwapFrom INFO [2019-12-17 12:31:04,971] jepsen worker 32 - jepsen.util 4802 :invoke :cas [10 [32 2]] INFO [2019-12-17 12:31:04,992] jepsen worker 20 - jepsen.util 4340 :fail :cas [6 [24 5]] :ValueNotSwapFrom INFO [2019-12-17 12:31:04,992] jepsen worker 20 - jepsen.util 4340 :invoke :read [6 nil] INFO [2019-12-17 12:31:05,001] jepsen worker 30 - jepsen.util 4440 :fail :cas [10 [2 26]] :ValueNotSwapFrom INFO [2019-12-17 12:31:05,001] jepsen worker 30 - jepsen.util 4440 :invoke :cas [10 [15 31]] INFO [2019-12-17 12:31:05,010] jepsen worker 64 - jepsen.util 3394 :ok :write [21 19] INFO [2019-12-17 12:31:05,010] jepsen worker 64 - jepsen.util 3394 :invoke :read [21 nil] INFO [2019-12-17 12:31:05,018] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":24838,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:31:05,018] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":24838,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:31:05,018] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":24842,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:31:05,018] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":24842,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:31:05,022] jepsen worker 88 - jepsen.util 4498 :fail :cas [29 [44 33]] :ValueNotSwapFrom INFO [2019-12-17 12:31:05,022] jepsen worker 88 - jepsen.util 4498 :invoke :cas [29 [32 14]] INFO [2019-12-17 12:31:05,034] jepsen worker 29 - jepsen.util 3629 :ok :read [9 2] INFO [2019-12-17 12:31:05,034] jepsen worker 29 - jepsen.util 3629 :invoke :read [9 nil] INFO [2019-12-17 12:31:05,037] jepsen worker 27 - jepsen.util 4167 :ok :write [9 36] INFO [2019-12-17 12:31:05,038] jepsen worker 27 - jepsen.util 4167 :invoke :read [9 nil] INFO [2019-12-17 12:31:05,039] jepsen worker 77 - jepsen.util 5747 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:31:05,039] jepsen worker 77 - jepsen.util 5747 :invoke :write [25 35] INFO [2019-12-17 12:31:05,049] jepsen worker 5 - jepsen.util 3965 :ok :read [1 39] INFO [2019-12-17 12:31:05,049] jepsen worker 5 - jepsen.util 3965 :invoke :read [1 nil] INFO [2019-12-17 12:31:05,066] jepsen worker 64 - jepsen.util 3394 :ok :read [21 19] INFO [2019-12-17 12:31:05,066] jepsen worker 64 - jepsen.util 3394 :invoke :cas [21 [32 0]] INFO [2019-12-17 12:31:05,070] jepsen worker 12 - jepsen.util 4602 :fail :cas [4 [22 5]] :ValueNotSwapFrom INFO [2019-12-17 12:31:05,070] jepsen worker 12 - jepsen.util 4602 :invoke :cas [4 [2 11]] INFO [2019-12-17 12:31:05,103] jepsen worker 79 - jepsen.util 4219 :fail :cas [26 [1 35]] :ValueNotSwapFrom INFO [2019-12-17 12:31:05,103] jepsen worker 79 - jepsen.util 4219 :invoke :write [26 35] INFO [2019-12-17 12:31:05,117] jepsen worker 27 - jepsen.util 4167 :ok :read [9 36] INFO [2019-12-17 12:31:05,117] jepsen worker 27 - jepsen.util 4167 :invoke :cas [9 [18 36]] INFO [2019-12-17 12:31:05,123] jepsen worker 29 - jepsen.util 3629 :ok :read [9 36] INFO [2019-12-17 12:31:05,123] jepsen worker 29 - jepsen.util 3629 :invoke :read [9 nil] INFO [2019-12-17 12:31:05,125] jepsen worker 80 - jepsen.util 4310 :fail :cas [26 [23 23]] :ValueNotSwapFrom INFO [2019-12-17 12:31:05,125] jepsen worker 80 - jepsen.util 4310 :invoke :read [26 nil] INFO [2019-12-17 12:31:05,125] jepsen worker 52 - jepsen.util 4462 :info :write [17 38] :RequestCanceledException INFO [2019-12-17 12:31:05,127] jepsen worker 52 - jepsen.util 4552 :invoke :write [17 22] INFO [2019-12-17 12:31:05,156] jepsen worker 24 - jepsen.util 5514 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:31:05,156] jepsen worker 24 - jepsen.util 5514 :invoke :read [8 nil] INFO [2019-12-17 12:31:05,159] jepsen worker 56 - jepsen.util 4556 :ok :read [18 26] INFO [2019-12-17 12:31:05,160] jepsen worker 56 - jepsen.util 4556 :invoke :cas [18 [23 17]] INFO [2019-12-17 12:31:05,162] jepsen worker 64 - jepsen.util 3394 :fail :cas [21 [32 0]] :ValueNotSwapFrom INFO [2019-12-17 12:31:05,162] jepsen worker 64 - jepsen.util 3394 :invoke :write [21 40] INFO [2019-12-17 12:31:05,163] jepsen worker 32 - jepsen.util 4802 :fail :cas [10 [32 2]] :ValueNotSwapFrom INFO [2019-12-17 12:31:05,163] jepsen worker 32 - jepsen.util 4802 :invoke :write [10 33] INFO [2019-12-17 12:31:05,167] jepsen worker 27 - jepsen.util 4167 :fail :cas [9 [18 36]] :ValueNotSwapFrom INFO [2019-12-17 12:31:05,167] jepsen worker 27 - jepsen.util 4167 :invoke :cas [9 [17 9]] INFO [2019-12-17 12:31:05,183] jepsen worker 29 - jepsen.util 3629 :ok :read [9 36] INFO [2019-12-17 12:31:05,184] jepsen worker 29 - jepsen.util 3629 :invoke :write [9 16] INFO [2019-12-17 12:31:05,193] jepsen worker 74 - jepsen.util 4934 :ok :write [24 42] INFO [2019-12-17 12:31:05,193] jepsen worker 74 - jepsen.util 4934 :invoke :cas [24 [22 20]] INFO [2019-12-17 12:31:05,198] jepsen worker 47 - jepsen.util 4817 :ok :cas [15 [2 44]] INFO [2019-12-17 12:31:05,198] jepsen worker 47 - jepsen.util 4817 :invoke :read [15 nil] INFO [2019-12-17 12:31:05,204] jepsen worker 20 - jepsen.util 4340 :ok :read [6 16] INFO [2019-12-17 12:31:05,205] jepsen worker 20 - jepsen.util 4340 :invoke :cas [6 [23 5]] INFO [2019-12-17 12:31:05,205] jepsen worker 19 - jepsen.util 4429 :ok :write [6 7] INFO [2019-12-17 12:31:05,205] jepsen worker 19 - jepsen.util 4429 :invoke :write [6 22] INFO [2019-12-17 12:31:05,209] jepsen worker 46 - jepsen.util 4366 :info :cas [15 [25 26]] :RequestCanceledException INFO [2019-12-17 12:31:05,210] jepsen worker 46 - jepsen.util 4456 :invoke :cas [15 [44 49]] INFO [2019-12-17 12:31:05,220] jepsen worker 27 - jepsen.util 4167 :fail :cas [9 [17 9]] :ValueNotSwapFrom INFO [2019-12-17 12:31:05,220] jepsen worker 27 - jepsen.util 4167 :invoke :write [9 37] INFO [2019-12-17 12:31:05,222] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":24875,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:31:05,222] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":24875,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:31:05,222] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":24876,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:31:05,222] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":24876,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:31:05,225] jepsen worker 86 - jepsen.util 5846 :info :write [28 41] :RequestCanceledException INFO [2019-12-17 12:31:05,225] jepsen worker 49 - jepsen.util 4999 :info :write [16 23] :RequestCanceledException INFO [2019-12-17 12:31:05,226] jepsen worker 86 - jepsen.util 5936 :invoke :cas [28 [31 27]] INFO [2019-12-17 12:31:05,227] jepsen worker 49 - jepsen.util 5089 :invoke :read [16 nil] INFO [2019-12-17 12:31:05,228] jepsen worker 71 - jepsen.util 4571 :fail :cas [23 [2 13]] :ValueNotSwapFrom INFO [2019-12-17 12:31:05,228] jepsen worker 71 - jepsen.util 4571 :invoke :cas [23 [28 27]] INFO [2019-12-17 12:31:05,234] jepsen worker 30 - jepsen.util 4440 :fail :cas [10 [15 31]] :ValueNotSwapFrom INFO [2019-12-17 12:31:05,234] jepsen worker 30 - jepsen.util 4440 :invoke :cas [10 [43 31]] INFO [2019-12-17 12:31:05,237] jepsen worker 88 - jepsen.util 4498 :fail :cas [29 [32 14]] :ValueNotSwapFrom INFO [2019-12-17 12:31:05,237] jepsen worker 88 - jepsen.util 4498 :invoke :cas [29 [48 24]] INFO [2019-12-17 12:31:05,237] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":24864,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:31:05,237] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":24864,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:31:05,237] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":24865,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:31:05,237] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":24865,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:31:05,238] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":24866,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:31:05,238] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":24866,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:31:05,239] jepsen worker 65 - jepsen.util 3575 :info :write [21 44] :RequestCanceledException INFO [2019-12-17 12:31:05,240] jepsen worker 65 - jepsen.util 3665 :invoke :write [21 47] INFO [2019-12-17 12:31:05,279] jepsen worker 3 - jepsen.util 4503 :fail :cas [1 [1 35]] :ValueNotSwapFrom INFO [2019-12-17 12:31:05,279] jepsen worker 3 - jepsen.util 4503 :invoke :cas [1 [30 24]] INFO [2019-12-17 12:31:05,292] jepsen worker 74 - jepsen.util 4934 :fail :cas [24 [22 20]] :ValueNotSwapFrom INFO [2019-12-17 12:31:05,292] jepsen worker 74 - jepsen.util 4934 :invoke :write [24 13] INFO [2019-12-17 12:31:05,295] jepsen worker 72 - jepsen.util 4572 :ok :read [24 42] INFO [2019-12-17 12:31:05,295] jepsen worker 72 - jepsen.util 4572 :invoke :read [24 nil] INFO [2019-12-17 12:31:05,299] jepsen worker 64 - jepsen.util 3394 :ok :write [21 40] INFO [2019-12-17 12:31:05,299] jepsen worker 64 - jepsen.util 3394 :invoke :read [21 nil] INFO [2019-12-17 12:31:05,305] jepsen worker 47 - jepsen.util 4817 :ok :read [15 44] INFO [2019-12-17 12:31:05,305] jepsen worker 47 - jepsen.util 4817 :invoke :read [15 nil] INFO [2019-12-17 12:31:05,359] jepsen worker 64 - jepsen.util 3394 :ok :read [21 40] INFO [2019-12-17 12:31:05,359] jepsen worker 64 - jepsen.util 3394 :invoke :read [21 nil] INFO [2019-12-17 12:31:05,364] jepsen worker 72 - jepsen.util 4572 :ok :read [24 42] INFO [2019-12-17 12:31:05,365] jepsen worker 72 - jepsen.util 4572 :invoke :cas [24 [47 8]] INFO [2019-12-17 12:31:05,369] jepsen worker 88 - jepsen.util 4498 :fail :cas [29 [48 24]] :ValueNotSwapFrom INFO [2019-12-17 12:31:05,369] jepsen worker 88 - jepsen.util 4498 :invoke :cas [29 [22 41]] INFO [2019-12-17 12:31:05,380] jepsen worker 64 - jepsen.util 3394 :ok :read [21 40] INFO [2019-12-17 12:31:05,380] jepsen worker 64 - jepsen.util 3394 :invoke :write [21 2] INFO [2019-12-17 12:31:05,395] jepsen worker 27 - jepsen.util 4167 :ok :write [9 37] INFO [2019-12-17 12:31:05,395] jepsen worker 27 - jepsen.util 4167 :invoke :read [9 nil] INFO [2019-12-17 12:31:05,412] jepsen worker 47 - jepsen.util 4817 :ok :read [15 44] INFO [2019-12-17 12:31:05,412] jepsen worker 47 - jepsen.util 4817 :invoke :write [15 22] INFO [2019-12-17 12:31:05,425] jepsen worker 4 - jepsen.util 3424 :info :write [1 27] :RequestCanceledException INFO [2019-12-17 12:31:05,427] jepsen worker 4 - jepsen.util 3514 :invoke :write [1 45] INFO [2019-12-17 12:31:05,432] jepsen worker 27 - jepsen.util 4167 :ok :read [9 37] INFO [2019-12-17 12:31:05,432] jepsen worker 27 - jepsen.util 4167 :invoke :read [9 nil] INFO [2019-12-17 12:31:05,439] jepsen worker 34 - jepsen.util 5254 :info :cas [11 [47 4]] :RequestCanceledException INFO [2019-12-17 12:31:05,440] jepsen worker 72 - jepsen.util 4572 :fail :cas [24 [47 8]] :ValueNotSwapFrom INFO [2019-12-17 12:31:05,440] jepsen worker 72 - jepsen.util 4572 :invoke :read [24 nil] INFO [2019-12-17 12:31:05,440] jepsen worker 34 - jepsen.util 5344 :invoke :write [11 25] INFO [2019-12-17 12:31:05,456] jepsen worker 19 - jepsen.util 4429 :ok :write [6 22] INFO [2019-12-17 12:31:05,456] jepsen worker 19 - jepsen.util 4429 :invoke :write [6 47] INFO [2019-12-17 12:31:05,479] jepsen worker 27 - jepsen.util 4167 :ok :read [9 37] INFO [2019-12-17 12:31:05,479] jepsen worker 27 - jepsen.util 4167 :invoke :write [9 10] INFO [2019-12-17 12:31:05,510] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":24902,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:31:05,510] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":24902,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:31:05,510] jepsen worker 64 - jepsen.util 3394 :ok :write [21 2] INFO [2019-12-17 12:31:05,511] jepsen worker 64 - jepsen.util 3394 :invoke :cas [21 [38 31]] INFO [2019-12-17 12:31:05,524] jepsen worker 68 - jepsen.util 5198 :info :write [22 37] :RequestCanceledException INFO [2019-12-17 12:31:05,526] jepsen worker 68 - jepsen.util 5288 :invoke :cas [22 [6 12]] INFO [2019-12-17 12:31:05,537] jepsen worker 55 - jepsen.util 5275 :info :write [18 43] :RequestCanceledException INFO [2019-12-17 12:31:05,538] jepsen worker 55 - jepsen.util 5365 :invoke :write [18 37] INFO [2019-12-17 12:31:05,547] jepsen worker 5 - jepsen.util 3965 :ok :read [1 39] INFO [2019-12-17 12:31:05,547] jepsen worker 5 - jepsen.util 3965 :invoke :write [1 28] INFO [2019-12-17 12:31:05,556] jepsen worker 73 - jepsen.util 4123 :info :write [24 40] :RequestCanceledException INFO [2019-12-17 12:31:05,557] jepsen worker 73 - jepsen.util 4213 :invoke :cas [24 [28 29]] INFO [2019-12-17 12:31:05,565] jepsen worker 64 - jepsen.util 3394 :fail :cas [21 [38 31]] :ValueNotSwapFrom INFO [2019-12-17 12:31:05,565] jepsen worker 64 - jepsen.util 3394 :invoke :cas [21 [25 25]] INFO [2019-12-17 12:31:05,585] jepsen worker 51 - jepsen.util 4011 :ok :write [17 7] INFO [2019-12-17 12:31:05,585] jepsen worker 51 - jepsen.util 4011 :invoke :cas [17 [29 24]] INFO [2019-12-17 12:31:05,594] jepsen worker 73 - jepsen.util 4213 :fail :cas [24 [28 29]] :ValueNotSwapFrom INFO [2019-12-17 12:31:05,594] jepsen worker 73 - jepsen.util 4213 :invoke :cas [24 [19 3]] INFO [2019-12-17 12:31:05,607] jepsen worker 23 - jepsen.util 5603 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:31:05,607] jepsen worker 89 - jepsen.util 4049 :info :write [29 24] :RequestCanceledException INFO [2019-12-17 12:31:05,607] jepsen worker 23 - jepsen.util 5603 :invoke :write [7 49] INFO [2019-12-17 12:31:05,608] jepsen worker 89 - jepsen.util 4139 :invoke :write [29 2] INFO [2019-12-17 12:31:05,623] jepsen worker 12 - jepsen.util 4602 :fail :cas [4 [2 11]] :ValueNotSwapFrom INFO [2019-12-17 12:31:05,623] jepsen worker 12 - jepsen.util 4602 :invoke :cas [4 [40 13]] INFO [2019-12-17 12:31:05,626] jepsen worker 64 - jepsen.util 3394 :fail :cas [21 [25 25]] :ValueNotSwapFrom INFO [2019-12-17 12:31:05,626] jepsen worker 64 - jepsen.util 3394 :invoke :cas [21 [23 48]] INFO [2019-12-17 12:31:05,636] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:31:05,636] jepsen nemesis - couchbase.util Rebalance check count: 184 INFO [2019-12-17 12:31:05,636] jepsen nemesis - couchbase.util Rebalance stuck count: 165 INFO [2019-12-17 12:31:05,639] jepsen worker 38 - jepsen.util 4268 :info :write [12 23] :RequestCanceledException INFO [2019-12-17 12:31:05,639] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":24925,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:31:05,639] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":24925,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:31:05,639] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":24928,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:31:05,639] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":24928,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:31:05,641] jepsen worker 38 - jepsen.util 4358 :invoke :read [12 nil] INFO [2019-12-17 12:31:05,648] jepsen worker 73 - jepsen.util 4213 :fail :cas [24 [19 3]] :ValueNotSwapFrom INFO [2019-12-17 12:31:05,648] jepsen worker 73 - jepsen.util 4213 :invoke :read [24 nil] INFO [2019-12-17 12:31:05,655] jepsen worker 27 - jepsen.util 4167 :ok :write [9 10] INFO [2019-12-17 12:31:05,655] jepsen worker 27 - jepsen.util 4167 :invoke :cas [9 [41 27]] INFO [2019-12-17 12:31:05,662] jepsen worker 64 - jepsen.util 3394 :fail :cas [21 [23 48]] :ValueNotSwapFrom INFO [2019-12-17 12:31:05,662] jepsen worker 64 - jepsen.util 3394 :invoke :read [21 nil] INFO [2019-12-17 12:31:05,684] jepsen worker 73 - jepsen.util 4213 :ok :read [24 42] INFO [2019-12-17 12:31:05,685] jepsen worker 73 - jepsen.util 4213 :invoke :cas [24 [4 21]] INFO [2019-12-17 12:31:05,690] jepsen worker 3 - jepsen.util 4503 :fail :cas [1 [30 24]] :ValueNotSwapFrom INFO [2019-12-17 12:31:05,690] jepsen worker 3 - jepsen.util 4503 :invoke :write [1 1] INFO [2019-12-17 12:31:05,696] jepsen worker 56 - jepsen.util 4556 :fail :cas [18 [23 17]] :ValueNotSwapFrom INFO [2019-12-17 12:31:05,697] jepsen worker 56 - jepsen.util 4556 :invoke :read [18 nil] INFO [2019-12-17 12:31:05,708] jepsen worker 27 - jepsen.util 4167 :fail :cas [9 [41 27]] :ValueNotSwapFrom INFO [2019-12-17 12:31:05,708] jepsen worker 27 - jepsen.util 4167 :invoke :write [9 35] INFO [2019-12-17 12:31:05,709] jepsen worker 28 - jepsen.util 3178 :info :cas [9 [32 24]] :RequestCanceledException INFO [2019-12-17 12:31:05,711] jepsen worker 28 - jepsen.util 3268 :invoke :write [9 20] INFO [2019-12-17 12:31:05,717] jepsen worker 64 - jepsen.util 3394 :ok :read [21 2] INFO [2019-12-17 12:31:05,718] jepsen worker 64 - jepsen.util 3394 :invoke :write [21 36] INFO [2019-12-17 12:31:05,723] jepsen worker 43 - jepsen.util 5263 :info :write [14 39] :RequestCanceledException INFO [2019-12-17 12:31:05,723] jepsen worker 44 - jepsen.util 6164 :info :cas [14 [25 41]] :RequestCanceledException INFO [2019-12-17 12:31:05,725] jepsen worker 43 - jepsen.util 5353 :invoke :cas [14 [4 24]] INFO [2019-12-17 12:31:05,726] jepsen worker 44 - jepsen.util 6254 :invoke :write [14 16] INFO [2019-12-17 12:31:05,738] jepsen worker 45 - jepsen.util 4455 :fail :read [15 nil] :RequestTimeoutException INFO [2019-12-17 12:31:05,739] jepsen worker 45 - jepsen.util 4455 :invoke :cas [15 [2 44]] INFO [2019-12-17 12:31:05,740] jepsen worker 73 - jepsen.util 4213 :fail :cas [24 [4 21]] :ValueNotSwapFrom INFO [2019-12-17 12:31:05,740] jepsen worker 73 - jepsen.util 4213 :invoke :write [24 4] INFO [2019-12-17 12:31:05,755] jepsen worker 59 - jepsen.util 5729 :info :write [19 3] :RequestCanceledException INFO [2019-12-17 12:31:05,756] jepsen worker 59 - jepsen.util 5819 :invoke :write [19 49] INFO [2019-12-17 12:31:05,760] jepsen worker 19 - jepsen.util 4429 :ok :write [6 47] INFO [2019-12-17 12:31:05,761] jepsen worker 19 - jepsen.util 4429 :invoke :read [6 nil] INFO [2019-12-17 12:31:05,792] jepsen worker 6 - jepsen.util 6306 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:31:05,792] jepsen worker 6 - jepsen.util 6306 :invoke :cas [2 [5 26]] INFO [2019-12-17 12:31:05,809] jepsen worker 66 - jepsen.util 5466 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:31:05,810] jepsen worker 66 - jepsen.util 5466 :invoke :read [22 nil] INFO [2019-12-17 12:31:05,820] jepsen worker 88 - jepsen.util 4498 :fail :cas [29 [22 41]] :ValueNotSwapFrom INFO [2019-12-17 12:31:05,820] jepsen worker 88 - jepsen.util 4498 :invoke :read [29 nil] INFO [2019-12-17 12:31:05,822] jepsen worker 27 - jepsen.util 4167 :ok :write [9 35] INFO [2019-12-17 12:31:05,822] jepsen worker 27 - jepsen.util 4167 :invoke :read [9 nil] INFO [2019-12-17 12:31:05,825] jepsen worker 78 - jepsen.util 4488 :info :write [26 33] :RequestCanceledException INFO [2019-12-17 12:31:05,825] jepsen worker 84 - jepsen.util 5214 :info :write [28 38] :RequestCanceledException INFO [2019-12-17 12:31:05,826] jepsen worker 84 - jepsen.util 5304 :invoke :cas [28 [2 48]] INFO [2019-12-17 12:31:05,826] jepsen worker 78 - jepsen.util 4578 :invoke :read [26 nil] INFO [2019-12-17 12:31:05,831] jepsen worker 71 - jepsen.util 4571 :fail :cas [23 [28 27]] :ValueNotSwapFrom INFO [2019-12-17 12:31:05,831] jepsen worker 71 - jepsen.util 4571 :invoke :write [23 28] INFO [2019-12-17 12:31:05,839] jepsen worker 25 - jepsen.util 5875 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:31:05,839] jepsen worker 85 - jepsen.util 6385 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:31:05,840] jepsen worker 25 - jepsen.util 5875 :invoke :write [8 38] INFO [2019-12-17 12:31:05,840] jepsen worker 85 - jepsen.util 6385 :invoke :cas [28 [46 29]] INFO [2019-12-17 12:31:05,845] jepsen worker 64 - jepsen.util 3394 :ok :write [21 36] INFO [2019-12-17 12:31:05,845] jepsen worker 64 - jepsen.util 3394 :invoke :read [21 nil] INFO [2019-12-17 12:31:05,858] jepsen worker 27 - jepsen.util 4167 :ok :read [9 35] INFO [2019-12-17 12:31:05,858] jepsen worker 27 - jepsen.util 4167 :invoke :write [9 44] INFO [2019-12-17 12:31:05,881] jepsen worker 80 - jepsen.util 4310 :ok :read [26 27] INFO [2019-12-17 12:31:05,881] jepsen worker 80 - jepsen.util 4310 :invoke :read [26 nil] INFO [2019-12-17 12:31:05,887] jepsen worker 9 - jepsen.util 4959 :fail :read [3 nil] :RequestTimeoutException INFO [2019-12-17 12:31:05,887] jepsen worker 7 - jepsen.util 5587 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:31:05,887] jepsen worker 7 - jepsen.util 5587 :invoke :cas [2 [31 37]] INFO [2019-12-17 12:31:05,887] jepsen worker 9 - jepsen.util 4959 :invoke :write [3 15] INFO [2019-12-17 12:31:05,892] jepsen worker 64 - jepsen.util 3394 :ok :read [21 36] INFO [2019-12-17 12:31:05,892] jepsen worker 64 - jepsen.util 3394 :invoke :cas [21 [37 41]] INFO [2019-12-17 12:31:05,897] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":24956,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:31:05,897] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":24956,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:31:05,897] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":24958,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:31:05,897] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":24958,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:31:05,905] jepsen worker 74 - jepsen.util 4934 :ok :write [24 13] INFO [2019-12-17 12:31:05,905] jepsen worker 74 - jepsen.util 4934 :invoke :write [24 18] INFO [2019-12-17 12:31:05,919] jepsen worker 32 - jepsen.util 4802 :ok :write [10 33] INFO [2019-12-17 12:31:05,919] jepsen worker 32 - jepsen.util 4802 :invoke :write [10 12] INFO [2019-12-17 12:31:05,924] jepsen worker 62 - jepsen.util 5282 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:31:05,925] jepsen worker 62 - jepsen.util 5282 :invoke :cas [20 [42 14]] INFO [2019-12-17 12:31:05,933] jepsen worker 64 - jepsen.util 3394 :fail :cas [21 [37 41]] :ValueNotSwapFrom INFO [2019-12-17 12:31:05,933] jepsen worker 64 - jepsen.util 3394 :invoke :read [21 nil] INFO [2019-12-17 12:31:05,939] jepsen worker 82 - jepsen.util 6562 :info :write [27 33] :RequestCanceledException INFO [2019-12-17 12:31:05,941] jepsen worker 82 - jepsen.util 6652 :invoke :write [27 26] INFO [2019-12-17 12:31:05,961] jepsen worker 45 - jepsen.util 4455 :fail :cas [15 [2 44]] :ValueNotSwapFrom INFO [2019-12-17 12:31:05,961] jepsen worker 45 - jepsen.util 4455 :invoke :read [15 nil] INFO [2019-12-17 12:31:05,965] jepsen worker 47 - jepsen.util 4817 :ok :write [15 22] INFO [2019-12-17 12:31:05,966] jepsen worker 47 - jepsen.util 4817 :invoke :cas [15 [49 11]] INFO [2019-12-17 12:31:05,971] jepsen worker 64 - jepsen.util 3394 :ok :read [21 36] INFO [2019-12-17 12:31:05,971] jepsen worker 64 - jepsen.util 3394 :invoke :cas [21 [10 46]] INFO [2019-12-17 12:31:05,990] jepsen worker 64 - jepsen.util 3394 :fail :cas [21 [10 46]] :ValueNotSwapFrom INFO [2019-12-17 12:31:05,990] jepsen worker 64 - jepsen.util 3394 :invoke :cas [21 [40 45]] INFO [2019-12-17 12:31:05,990] jepsen worker 27 - jepsen.util 4167 :ok :write [9 44] INFO [2019-12-17 12:31:05,990] jepsen worker 27 - jepsen.util 4167 :invoke :read [9 nil] INFO [2019-12-17 12:31:06,002] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":24978,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:31:06,003] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":24978,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:31:06,003] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":24979,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:31:06,003] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":24979,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:31:06,010] jepsen worker 39 - jepsen.util 5259 :info :write [13 33] :RequestCanceledException INFO [2019-12-17 12:31:06,011] jepsen worker 39 - jepsen.util 5349 :invoke :read [13 nil] INFO [2019-12-17 12:31:06,026] jepsen worker 64 - jepsen.util 3394 :fail :cas [21 [40 45]] :ValueNotSwapFrom INFO [2019-12-17 12:31:06,026] jepsen worker 64 - jepsen.util 3394 :invoke :write [21 4] INFO [2019-12-17 12:31:06,034] jepsen worker 51 - jepsen.util 4011 :fail :cas [17 [29 24]] :ValueNotSwapFrom INFO [2019-12-17 12:31:06,034] jepsen worker 51 - jepsen.util 4011 :invoke :read [17 nil] INFO [2019-12-17 12:31:06,039] jepsen worker 27 - jepsen.util 4167 :ok :read [9 44] INFO [2019-12-17 12:31:06,039] jepsen worker 83 - jepsen.util 5933 :info :cas [27 [22 3]] :RequestCanceledException INFO [2019-12-17 12:31:06,039] jepsen worker 27 - jepsen.util 4167 :invoke :write [9 16] INFO [2019-12-17 12:31:06,040] jepsen worker 83 - jepsen.util 6023 :invoke :write [27 7] INFO [2019-12-17 12:31:06,042] jepsen worker 28 - jepsen.util 3268 :ok :write [9 20] INFO [2019-12-17 12:31:06,043] jepsen worker 28 - jepsen.util 3268 :invoke :cas [9 [44 28]] INFO [2019-12-17 12:31:06,047] jepsen worker 38 - jepsen.util 4358 :ok :read [12 18] INFO [2019-12-17 12:31:06,047] jepsen worker 38 - jepsen.util 4358 :invoke :cas [12 [25 29]] INFO [2019-12-17 12:31:06,096] jepsen worker 28 - jepsen.util 3268 :fail :cas [9 [44 28]] :ValueNotSwapFrom INFO [2019-12-17 12:31:06,096] jepsen worker 28 - jepsen.util 3268 :invoke :read [9 nil] INFO [2019-12-17 12:31:06,110] jepsen worker 35 - jepsen.util 5705 :info :cas [11 [15 18]] :RequestCanceledException INFO [2019-12-17 12:31:06,110] jepsen worker 63 - jepsen.util 3483 :info :write [21 23] :RequestCanceledException INFO [2019-12-17 12:31:06,111] jepsen worker 63 - jepsen.util 3573 :invoke :cas [21 [5 13]] INFO [2019-12-17 12:31:06,111] jepsen worker 35 - jepsen.util 5795 :invoke :cas [11 [28 32]] INFO [2019-12-17 12:31:06,125] jepsen worker 58 - jepsen.util 5818 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:31:06,125] jepsen worker 2 - jepsen.util 5492 :info :cas [0 [0 24]] :RequestCanceledException INFO [2019-12-17 12:31:06,125] jepsen worker 58 - jepsen.util 5818 :invoke :write [19 26] INFO [2019-12-17 12:31:06,126] jepsen worker 2 - jepsen.util 5582 :invoke :read [0 nil] INFO [2019-12-17 12:31:06,138] jepsen worker 70 - jepsen.util 4930 :info :write [23 20] :RequestCanceledException INFO [2019-12-17 12:31:06,138] jepsen worker 0 - jepsen.util 6390 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:31:06,138] jepsen worker 0 - jepsen.util 6390 :invoke :read [0 nil] INFO [2019-12-17 12:31:06,139] jepsen worker 70 - jepsen.util 5020 :invoke :write [23 18] INFO [2019-12-17 12:31:06,142] jepsen worker 28 - jepsen.util 3268 :ok :read [9 20] INFO [2019-12-17 12:31:06,142] jepsen worker 28 - jepsen.util 3268 :invoke :cas [9 [17 4]] INFO [2019-12-17 12:31:06,155] jepsen worker 40 - jepsen.util 6430 :info :cas [13 [19 47]] :RequestCanceledException INFO [2019-12-17 12:31:06,156] jepsen worker 40 - jepsen.util 6520 :invoke :read [13 nil] INFO [2019-12-17 12:31:06,163] jepsen worker 28 - jepsen.util 3268 :fail :cas [9 [17 4]] :ValueNotSwapFrom INFO [2019-12-17 12:31:06,163] jepsen worker 28 - jepsen.util 3268 :invoke :read [9 nil] INFO [2019-12-17 12:31:06,164] jepsen worker 74 - jepsen.util 4934 :ok :write [24 18] INFO [2019-12-17 12:31:06,164] jepsen worker 74 - jepsen.util 4934 :invoke :write [24 46] INFO [2019-12-17 12:31:06,191] jepsen worker 27 - jepsen.util 4167 :ok :write [9 16] INFO [2019-12-17 12:31:06,191] jepsen worker 27 - jepsen.util 4167 :invoke :read [9 nil] INFO [2019-12-17 12:31:06,191] jepsen worker 60 - jepsen.util 6360 :info :write [20 19] :RequestCanceledException INFO [2019-12-17 12:31:06,192] jepsen worker 60 - jepsen.util 6450 :invoke :read [20 nil] INFO [2019-12-17 12:31:06,209] jepsen worker 50 - jepsen.util 5900 :info :write [16 16] :RequestCanceledException INFO [2019-12-17 12:31:06,210] jepsen worker 50 - jepsen.util 5990 :invoke :write [16 18] INFO [2019-12-17 12:31:06,220] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25003,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:31:06,220] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25003,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:31:06,221] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25005,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:31:06,221] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25005,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:31:06,224] jepsen worker 87 - jepsen.util 4047 :info :write [29 3] :RequestCanceledException INFO [2019-12-17 12:31:06,224] jepsen worker 26 - jepsen.util 5786 :info :cas [8 [19 45]] :RequestCanceledException INFO [2019-12-17 12:31:06,225] jepsen worker 26 - jepsen.util 5876 :invoke :cas [8 [29 7]] INFO [2019-12-17 12:31:06,225] jepsen worker 87 - jepsen.util 4137 :invoke :cas [29 [36 31]] INFO [2019-12-17 12:31:06,226] jepsen worker 32 - jepsen.util 4802 :ok :write [10 12] INFO [2019-12-17 12:31:06,226] jepsen worker 32 - jepsen.util 4802 :invoke :read [10 nil] INFO [2019-12-17 12:31:06,237] jepsen worker 64 - jepsen.util 3394 :ok :write [21 4] INFO [2019-12-17 12:31:06,241] jepsen worker 64 - jepsen.util 3394 :invoke :cas [21 [49 15]] INFO [2019-12-17 12:31:06,247] jepsen worker 45 - jepsen.util 4455 :ok :read [15 22] INFO [2019-12-17 12:31:06,247] jepsen worker 45 - jepsen.util 4455 :invoke :cas [15 [31 48]] INFO [2019-12-17 12:31:06,253] jepsen worker 10 - jepsen.util 4240 :fail :read [3 nil] :RequestTimeoutException INFO [2019-12-17 12:31:06,253] jepsen worker 67 - jepsen.util 5917 :info :cas [22 [38 7]] :RequestCanceledException INFO [2019-12-17 12:31:06,253] jepsen worker 10 - jepsen.util 4240 :invoke :read [3 nil] INFO [2019-12-17 12:31:06,254] jepsen worker 67 - jepsen.util 6007 :invoke :read [22 nil] INFO [2019-12-17 12:31:06,271] jepsen worker 88 - jepsen.util 4498 :ok :read [29 33] INFO [2019-12-17 12:31:06,272] jepsen worker 88 - jepsen.util 4498 :invoke :write [29 45] INFO [2019-12-17 12:31:06,291] jepsen worker 57 - jepsen.util 5457 :info :write [19 7] :RequestCanceledException INFO [2019-12-17 12:31:06,292] jepsen worker 57 - jepsen.util 5547 :invoke :write [19 2] INFO [2019-12-17 12:31:06,294] jepsen worker 27 - jepsen.util 4167 :ok :read [9 16] INFO [2019-12-17 12:31:06,294] jepsen worker 27 - jepsen.util 4167 :invoke :cas [9 [17 19]] INFO [2019-12-17 12:31:06,306] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25009,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:31:06,306] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25009,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:31:06,306] jepsen worker 22 - jepsen.util 5782 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:31:06,307] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25011,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:31:06,307] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25011,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:31:06,307] jepsen worker 22 - jepsen.util 5782 :invoke :cas [7 [1 43]] INFO [2019-12-17 12:31:06,313] jepsen worker 28 - jepsen.util 3268 :ok :read [9 16] INFO [2019-12-17 12:31:06,313] jepsen worker 28 - jepsen.util 3268 :invoke :read [9 nil] INFO [2019-12-17 12:31:06,321] jepsen worker 64 - jepsen.util 3394 :fail :cas [21 [49 15]] :ValueNotSwapFrom INFO [2019-12-17 12:31:06,322] jepsen worker 64 - jepsen.util 3394 :invoke :read [21 nil] INFO [2019-12-17 12:31:06,322] jepsen worker 48 - jepsen.util 5268 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:31:06,323] jepsen worker 48 - jepsen.util 5268 :invoke :write [16 3] INFO [2019-12-17 12:31:06,323] jepsen worker 78 - jepsen.util 4578 :ok :read [26 27] INFO [2019-12-17 12:31:06,323] jepsen worker 78 - jepsen.util 4578 :invoke :cas [26 [7 42]] INFO [2019-12-17 12:31:06,333] jepsen worker 27 - jepsen.util 4167 :fail :cas [9 [17 19]] :ValueNotSwapFrom INFO [2019-12-17 12:31:06,333] jepsen worker 27 - jepsen.util 4167 :invoke :read [9 nil] INFO [2019-12-17 12:31:06,349] jepsen worker 80 - jepsen.util 4310 :ok :read [26 27] INFO [2019-12-17 12:31:06,349] jepsen worker 80 - jepsen.util 4310 :invoke :write [26 39] INFO [2019-12-17 12:31:06,356] jepsen worker 17 - jepsen.util 5417 :info :cas [5 [30 22]] :RequestCanceledException INFO [2019-12-17 12:31:06,357] jepsen worker 17 - jepsen.util 5507 :invoke :cas [5 [8 35]] INFO [2019-12-17 12:31:06,381] jepsen worker 28 - jepsen.util 3268 :ok :read [9 16] INFO [2019-12-17 12:31:06,382] jepsen worker 28 - jepsen.util 3268 :invoke :read [9 nil] INFO [2019-12-17 12:31:06,385] jepsen worker 51 - jepsen.util 4011 :ok :read [17 7] INFO [2019-12-17 12:31:06,385] jepsen worker 51 - jepsen.util 4011 :invoke :cas [17 [14 22]] INFO [2019-12-17 12:31:06,385] jepsen worker 87 - jepsen.util 4137 :fail :cas [29 [36 31]] :ValueNotSwapFrom INFO [2019-12-17 12:31:06,385] jepsen worker 87 - jepsen.util 4137 :invoke :read [29 nil] INFO [2019-12-17 12:31:06,390] jepsen worker 10 - jepsen.util 4240 :ok :read [3 3] INFO [2019-12-17 12:31:06,390] jepsen worker 10 - jepsen.util 4240 :invoke :write [3 36] INFO [2019-12-17 12:31:06,424] jepsen worker 42 - jepsen.util 6432 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:31:06,424] jepsen worker 42 - jepsen.util 6432 :invoke :write [14 14] INFO [2019-12-17 12:31:06,427] jepsen worker 27 - jepsen.util 4167 :ok :read [9 16] INFO [2019-12-17 12:31:06,427] jepsen worker 27 - jepsen.util 4167 :invoke :read [9 nil] INFO [2019-12-17 12:31:06,428] jepsen worker 32 - jepsen.util 4802 :ok :read [10 12] INFO [2019-12-17 12:31:06,428] jepsen worker 32 - jepsen.util 4802 :invoke :write [10 24] INFO [2019-12-17 12:31:06,452] jepsen worker 28 - jepsen.util 3268 :ok :read [9 16] INFO [2019-12-17 12:31:06,452] jepsen worker 41 - jepsen.util 5981 :info :write [13 13] :RequestCanceledException INFO [2019-12-17 12:31:06,452] jepsen worker 28 - jepsen.util 3268 :invoke :write [9 13] INFO [2019-12-17 12:31:06,453] jepsen worker 41 - jepsen.util 6071 :invoke :cas [13 [3 42]] INFO [2019-12-17 12:31:06,457] jepsen worker 64 - jepsen.util 3394 :ok :read [21 4] INFO [2019-12-17 12:31:06,457] jepsen worker 64 - jepsen.util 3394 :invoke :cas [21 [37 32]] INFO [2019-12-17 12:31:06,488] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25039,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:31:06,488] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25039,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:31:06,488] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25042,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:31:06,488] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25042,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:31:06,488] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25043,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:31:06,488] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25043,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:31:06,491] jepsen worker 75 - jepsen.util 5745 :info :write [25 37] :RequestCanceledException INFO [2019-12-17 12:31:06,492] jepsen worker 75 - jepsen.util 5835 :invoke :cas [25 [1 13]] INFO [2019-12-17 12:31:06,510] jepsen worker 33 - jepsen.util 5613 :info :write [11 46] :RequestCanceledException INFO [2019-12-17 12:31:06,510] jepsen worker 27 - jepsen.util 4167 :ok :read [9 16] INFO [2019-12-17 12:31:06,510] jepsen worker 27 - jepsen.util 4167 :invoke :write [9 35] INFO [2019-12-17 12:31:06,511] jepsen worker 33 - jepsen.util 5703 :invoke :write [11 4] INFO [2019-12-17 12:31:06,525] jepsen worker 53 - jepsen.util 4283 :info :write [17 22] :RequestCanceledException INFO [2019-12-17 12:31:06,525] jepsen worker 8 - jepsen.util 5048 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:31:06,525] jepsen worker 54 - jepsen.util 4734 :info :write [18 23] :RequestCanceledException INFO [2019-12-17 12:31:06,525] jepsen worker 8 - jepsen.util 5048 :invoke :write [2 0] INFO [2019-12-17 12:31:06,526] jepsen worker 53 - jepsen.util 4373 :invoke :read [17 nil] INFO [2019-12-17 12:31:06,526] jepsen worker 54 - jepsen.util 4824 :invoke :read [18 nil] INFO [2019-12-17 12:31:06,556] jepsen worker 1 - jepsen.util 5221 :info :write [0 6] :RequestCanceledException INFO [2019-12-17 12:31:06,557] jepsen worker 1 - jepsen.util 5311 :invoke :write [0 18] INFO [2019-12-17 12:31:06,581] jepsen worker 28 - jepsen.util 3268 :ok :write [9 13] INFO [2019-12-17 12:31:06,582] jepsen worker 28 - jepsen.util 3268 :invoke :read [9 nil] INFO [2019-12-17 12:31:06,590] jepsen worker 87 - jepsen.util 4137 :ok :read [29 33] INFO [2019-12-17 12:31:06,590] jepsen worker 87 - jepsen.util 4137 :invoke :write [29 3] INFO [2019-12-17 12:31:06,604] jepsen worker 28 - jepsen.util 3268 :ok :read [9 13] INFO [2019-12-17 12:31:06,604] jepsen worker 28 - jepsen.util 3268 :invoke :cas [9 [1 3]] INFO [2019-12-17 12:31:06,609] jepsen worker 16 - jepsen.util 6496 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:31:06,609] jepsen worker 14 - jepsen.util 4424 :info :write [4 38] :RequestCanceledException INFO [2019-12-17 12:31:06,610] jepsen worker 16 - jepsen.util 6496 :invoke :cas [5 [27 36]] INFO [2019-12-17 12:31:06,610] jepsen worker 74 - jepsen.util 4934 :ok :write [24 46] INFO [2019-12-17 12:31:06,610] jepsen worker 74 - jepsen.util 4934 :invoke :cas [24 [24 29]] INFO [2019-12-17 12:31:06,610] jepsen worker 14 - jepsen.util 4514 :invoke :cas [4 [29 1]] INFO [2019-12-17 12:31:06,612] jepsen worker 10 - jepsen.util 4240 :ok :write [3 36] INFO [2019-12-17 12:31:06,612] jepsen worker 10 - jepsen.util 4240 :invoke :read [3 nil] INFO [2019-12-17 12:31:06,625] jepsen worker 37 - jepsen.util 4087 :info :write [12 14] :RequestCanceledException INFO [2019-12-17 12:31:06,628] jepsen worker 37 - jepsen.util 4177 :invoke :write [12 45] INFO [2019-12-17 12:31:06,639] jepsen worker 81 - jepsen.util 5751 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:31:06,639] jepsen worker 76 - jepsen.util 5746 :info :cas [25 [48 39]] :RequestCanceledException INFO [2019-12-17 12:31:06,639] jepsen worker 81 - jepsen.util 5751 :invoke :write [27 19] INFO [2019-12-17 12:31:06,640] jepsen worker 76 - jepsen.util 5836 :invoke :cas [25 [42 28]] INFO [2019-12-17 12:31:06,650] jepsen worker 28 - jepsen.util 3268 :fail :cas [9 [1 3]] :ValueNotSwapFrom INFO [2019-12-17 12:31:06,650] jepsen worker 28 - jepsen.util 3268 :invoke :read [9 nil] INFO [2019-12-17 12:31:06,657] jepsen worker 89 - jepsen.util 4139 :ok :write [29 2] INFO [2019-12-17 12:31:06,657] jepsen worker 89 - jepsen.util 4139 :invoke :write [29 36] INFO [2019-12-17 12:31:06,661] jepsen worker 74 - jepsen.util 4934 :fail :cas [24 [24 29]] :ValueNotSwapFrom INFO [2019-12-17 12:31:06,661] jepsen worker 74 - jepsen.util 4934 :invoke :read [24 nil] INFO [2019-12-17 12:31:06,666] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:31:06,666] jepsen nemesis - couchbase.util Rebalance check count: 185 INFO [2019-12-17 12:31:06,666] jepsen nemesis - couchbase.util Rebalance stuck count: 166 INFO [2019-12-17 12:31:06,691] jepsen worker 15 - jepsen.util 4875 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:31:06,691] jepsen worker 15 - jepsen.util 4875 :invoke :read [5 nil] INFO [2019-12-17 12:31:06,700] jepsen worker 28 - jepsen.util 3268 :ok :read [9 13] INFO [2019-12-17 12:31:06,700] jepsen worker 28 - jepsen.util 3268 :invoke :read [9 nil] INFO [2019-12-17 12:31:06,724] jepsen worker 10 - jepsen.util 4240 :ok :read [3 36] INFO [2019-12-17 12:31:06,724] jepsen worker 10 - jepsen.util 4240 :invoke :write [3 15] INFO [2019-12-17 12:31:06,734] jepsen worker 36 - jepsen.util 4446 :ok :write [12 31] INFO [2019-12-17 12:31:06,734] jepsen worker 36 - jepsen.util 4446 :invoke :read [12 nil] INFO [2019-12-17 12:31:06,737] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25066,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:31:06,737] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25066,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:31:06,737] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25070,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:31:06,737] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25070,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:31:06,750] jepsen worker 28 - jepsen.util 3268 :ok :read [9 13] INFO [2019-12-17 12:31:06,750] jepsen worker 28 - jepsen.util 3268 :invoke :read [9 nil] INFO [2019-12-17 12:31:06,751] jepsen worker 74 - jepsen.util 4934 :ok :read [24 46] INFO [2019-12-17 12:31:06,751] jepsen worker 74 - jepsen.util 4934 :invoke :cas [24 [48 19]] INFO [2019-12-17 12:31:06,764] jepsen worker 28 - jepsen.util 3268 :ok :read [9 13] INFO [2019-12-17 12:31:06,765] jepsen worker 28 - jepsen.util 3268 :invoke :read [9 nil] INFO [2019-12-17 12:31:06,773] jepsen worker 74 - jepsen.util 4934 :fail :cas [24 [48 19]] :ValueNotSwapFrom INFO [2019-12-17 12:31:06,773] jepsen worker 74 - jepsen.util 4934 :invoke :write [24 26] INFO [2019-12-17 12:31:06,789] jepsen worker 28 - jepsen.util 3268 :ok :read [9 13] INFO [2019-12-17 12:31:06,789] jepsen worker 28 - jepsen.util 3268 :invoke :read [9 nil] INFO [2019-12-17 12:31:06,795] jepsen worker 14 - jepsen.util 4514 :fail :cas [4 [29 1]] :ValueNotSwapFrom INFO [2019-12-17 12:31:06,795] jepsen worker 14 - jepsen.util 4514 :invoke :write [4 15] INFO [2019-12-17 12:31:06,811] jepsen worker 28 - jepsen.util 3268 :ok :read [9 13] INFO [2019-12-17 12:31:06,811] jepsen worker 28 - jepsen.util 3268 :invoke :read [9 nil] INFO [2019-12-17 12:31:06,822] jepsen worker 21 - jepsen.util 5691 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:31:06,823] jepsen worker 21 - jepsen.util 5691 :invoke :read [7 nil] INFO [2019-12-17 12:31:06,835] jepsen worker 36 - jepsen.util 4446 :ok :read [12 31] INFO [2019-12-17 12:31:06,835] jepsen worker 36 - jepsen.util 4446 :invoke :cas [12 [29 13]] INFO [2019-12-17 12:31:06,840] jepsen worker 28 - jepsen.util 3268 :ok :read [9 13] INFO [2019-12-17 12:31:06,840] jepsen worker 28 - jepsen.util 3268 :invoke :read [9 nil] INFO [2019-12-17 12:31:06,860] jepsen worker 28 - jepsen.util 3268 :ok :read [9 13] INFO [2019-12-17 12:31:06,860] jepsen worker 28 - jepsen.util 3268 :invoke :write [9 30] INFO [2019-12-17 12:31:06,878] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25092,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:31:06,878] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25092,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:31:06,882] jepsen worker 36 - jepsen.util 4446 :fail :cas [12 [29 13]] :ValueNotSwapFrom INFO [2019-12-17 12:31:06,883] jepsen worker 36 - jepsen.util 4446 :invoke :cas [12 [13 32]] INFO [2019-12-17 12:31:06,884] jepsen worker 12 - jepsen.util 4602 :fail :cas [4 [40 13]] :ValueNotSwapFrom INFO [2019-12-17 12:31:06,884] jepsen worker 12 - jepsen.util 4602 :invoke :cas [4 [21 43]] INFO [2019-12-17 12:31:06,890] jepsen worker 53 - jepsen.util 4373 :ok :read [17 22] INFO [2019-12-17 12:31:06,890] jepsen worker 53 - jepsen.util 4373 :invoke :write [17 45] INFO [2019-12-17 12:31:06,904] jepsen worker 32 - jepsen.util 4802 :ok :write [10 24] INFO [2019-12-17 12:31:06,904] jepsen worker 32 - jepsen.util 4802 :invoke :read [10 nil] INFO [2019-12-17 12:31:06,907] jepsen worker 36 - jepsen.util 4446 :fail :cas [12 [13 32]] :ValueNotSwapFrom INFO [2019-12-17 12:31:06,907] jepsen worker 36 - jepsen.util 4446 :invoke :read [12 nil] INFO [2019-12-17 12:31:06,920] jepsen worker 56 - jepsen.util 4556 :ok :read [18 26] INFO [2019-12-17 12:31:06,920] jepsen worker 56 - jepsen.util 4556 :invoke :cas [18 [43 11]] INFO [2019-12-17 12:31:06,925] jepsen worker 18 - jepsen.util 4338 :info :write [6 16] :RequestCanceledException INFO [2019-12-17 12:31:06,926] jepsen worker 18 - jepsen.util 4428 :invoke :write [6 42] INFO [2019-12-17 12:31:06,931] jepsen worker 32 - jepsen.util 4802 :ok :read [10 24] INFO [2019-12-17 12:31:06,931] jepsen worker 36 - jepsen.util 4446 :ok :read [12 31] INFO [2019-12-17 12:31:06,931] jepsen worker 32 - jepsen.util 4802 :invoke :write [10 14] INFO [2019-12-17 12:31:06,931] jepsen worker 36 - jepsen.util 4446 :invoke :read [12 nil] INFO [2019-12-17 12:31:06,937] jepsen worker 28 - jepsen.util 3268 :ok :write [9 30] INFO [2019-12-17 12:31:06,937] jepsen worker 28 - jepsen.util 3268 :invoke :write [9 11] INFO [2019-12-17 12:31:06,956] jepsen worker 69 - jepsen.util 4389 :info :write [23 45] :RequestCanceledException INFO [2019-12-17 12:31:06,960] jepsen worker 69 - jepsen.util 4479 :invoke :write [23 13] INFO [2019-12-17 12:31:06,966] jepsen worker 10 - jepsen.util 4240 :ok :write [3 15] INFO [2019-12-17 12:31:06,966] jepsen worker 10 - jepsen.util 4240 :invoke :cas [3 [18 10]] INFO [2019-12-17 12:31:06,969] jepsen worker 36 - jepsen.util 4446 :ok :read [12 31] INFO [2019-12-17 12:31:06,969] jepsen worker 36 - jepsen.util 4446 :invoke :read [12 nil] INFO [2019-12-17 12:31:06,991] jepsen worker 74 - jepsen.util 4934 :ok :write [24 26] INFO [2019-12-17 12:31:06,991] jepsen worker 74 - jepsen.util 4934 :invoke :read [24 nil] INFO [2019-12-17 12:31:06,992] jepsen worker 11 - jepsen.util 4601 :info :write [3 46] :RequestCanceledException INFO [2019-12-17 12:31:06,993] jepsen worker 11 - jepsen.util 4691 :invoke :read [3 nil] INFO [2019-12-17 12:31:06,994] jepsen worker 10 - jepsen.util 4240 :fail :cas [3 [18 10]] :ValueNotSwapFrom INFO [2019-12-17 12:31:06,994] jepsen worker 10 - jepsen.util 4240 :invoke :write [3 36] INFO [2019-12-17 12:31:07,024] jepsen worker 61 - jepsen.util 6361 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:31:07,025] jepsen worker 31 - jepsen.util 3991 :info :write [10 2] :RequestCanceledException INFO [2019-12-17 12:31:07,025] jepsen worker 13 - jepsen.util 4063 :info :write [4 15] :RequestCanceledException INFO [2019-12-17 12:31:07,025] jepsen worker 61 - jepsen.util 6361 :invoke :write [20 8] INFO [2019-12-17 12:31:07,026] jepsen worker 13 - jepsen.util 4153 :invoke :read [4 nil] INFO [2019-12-17 12:31:07,026] jepsen worker 31 - jepsen.util 4081 :invoke :cas [10 [18 39]] INFO [2019-12-17 12:31:07,033] jepsen worker 74 - jepsen.util 4934 :ok :read [24 26] INFO [2019-12-17 12:31:07,034] jepsen worker 74 - jepsen.util 4934 :invoke :write [24 46] INFO [2019-12-17 12:31:07,070] jepsen worker 11 - jepsen.util 4691 :ok :read [3 15] INFO [2019-12-17 12:31:07,070] jepsen worker 11 - jepsen.util 4691 :invoke :write [3 5] INFO [2019-12-17 12:31:07,095] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25111,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:31:07,096] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25111,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:31:07,096] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25115,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:31:07,096] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25115,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:31:07,096] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25116,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:31:07,096] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25116,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:31:07,119] jepsen worker 18 - jepsen.util 4428 :ok :write [6 42] INFO [2019-12-17 12:31:07,119] jepsen worker 18 - jepsen.util 4428 :invoke :write [6 14] INFO [2019-12-17 12:31:07,138] jepsen worker 79 - jepsen.util 4219 :info :write [26 35] :RequestCanceledException INFO [2019-12-17 12:31:07,138] jepsen worker 77 - jepsen.util 5747 :info :write [25 35] :RequestCanceledException INFO [2019-12-17 12:31:07,139] jepsen worker 79 - jepsen.util 4309 :invoke :read [26 nil] INFO [2019-12-17 12:31:07,139] jepsen worker 77 - jepsen.util 5837 :invoke :write [25 45] INFO [2019-12-17 12:31:07,165] jepsen worker 13 - jepsen.util 4153 :ok :read [4 14] INFO [2019-12-17 12:31:07,166] jepsen worker 13 - jepsen.util 4153 :invoke :write [4 7] INFO [2019-12-17 12:31:07,173] jepsen worker 51 - jepsen.util 4011 :fail :cas [17 [14 22]] :ValueNotSwapFrom INFO [2019-12-17 12:31:07,173] jepsen worker 51 - jepsen.util 4011 :invoke :write [17 13] INFO [2019-12-17 12:31:07,195] jepsen worker 36 - jepsen.util 4446 :ok :read [12 31] INFO [2019-12-17 12:31:07,195] jepsen worker 36 - jepsen.util 4446 :invoke :cas [12 [11 47]] INFO [2019-12-17 12:31:07,205] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25130,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:31:07,205] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25130,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:31:07,209] jepsen worker 52 - jepsen.util 4552 :info :write [17 22] :RequestCanceledException INFO [2019-12-17 12:31:07,210] jepsen worker 52 - jepsen.util 4642 :invoke :cas [17 [26 25]] INFO [2019-12-17 12:31:07,215] jepsen worker 54 - jepsen.util 4824 :ok :read [18 26] INFO [2019-12-17 12:31:07,215] jepsen worker 54 - jepsen.util 4824 :invoke :write [18 29] INFO [2019-12-17 12:31:07,224] jepsen worker 20 - jepsen.util 4340 :info :cas [6 [23 5]] :RequestCanceledException INFO [2019-12-17 12:31:07,226] jepsen worker 20 - jepsen.util 4430 :invoke :cas [6 [44 1]] INFO [2019-12-17 12:31:07,239] jepsen worker 86 - jepsen.util 5936 :info :cas [28 [31 27]] :RequestCanceledException INFO [2019-12-17 12:31:07,240] jepsen worker 86 - jepsen.util 6026 :invoke :write [28 17] INFO [2019-12-17 12:31:07,256] jepsen worker 24 - jepsen.util 5514 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:31:07,256] jepsen worker 49 - jepsen.util 5089 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:31:07,256] jepsen worker 29 - jepsen.util 3629 :info :write [9 16] :RequestCanceledException INFO [2019-12-17 12:31:07,256] jepsen worker 24 - jepsen.util 5514 :invoke :cas [8 [0 36]] INFO [2019-12-17 12:31:07,256] jepsen worker 49 - jepsen.util 5089 :invoke :cas [16 [15 42]] INFO [2019-12-17 12:31:07,257] jepsen worker 29 - jepsen.util 3719 :invoke :cas [9 [27 23]] INFO [2019-12-17 12:31:07,286] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25140,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:31:07,287] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25140,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:31:07,287] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25142,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:31:07,287] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25142,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:31:07,300] jepsen worker 52 - jepsen.util 4642 :fail :cas [17 [26 25]] :ValueNotSwapFrom INFO [2019-12-17 12:31:07,300] jepsen worker 52 - jepsen.util 4642 :invoke :write [17 48] INFO [2019-12-17 12:31:07,325] jepsen worker 30 - jepsen.util 4440 :info :cas [10 [43 31]] :RequestCanceledException INFO [2019-12-17 12:31:07,325] jepsen worker 65 - jepsen.util 3665 :info :write [21 47] :RequestCanceledException INFO [2019-12-17 12:31:07,326] jepsen worker 30 - jepsen.util 4530 :invoke :read [10 nil] INFO [2019-12-17 12:31:07,326] jepsen worker 65 - jepsen.util 3755 :invoke :read [21 nil] INFO [2019-12-17 12:31:07,336] jepsen worker 30 - jepsen.util 4530 :ok :read [10 24] INFO [2019-12-17 12:31:07,336] jepsen worker 30 - jepsen.util 4530 :invoke :read [10 nil] INFO [2019-12-17 12:31:07,347] jepsen worker 30 - jepsen.util 4530 :ok :read [10 24] INFO [2019-12-17 12:31:07,347] jepsen worker 30 - jepsen.util 4530 :invoke :read [10 nil] INFO [2019-12-17 12:31:07,358] jepsen worker 30 - jepsen.util 4530 :ok :read [10 24] INFO [2019-12-17 12:31:07,358] jepsen worker 30 - jepsen.util 4530 :invoke :read [10 nil] INFO [2019-12-17 12:31:07,371] jepsen worker 30 - jepsen.util 4530 :ok :read [10 24] INFO [2019-12-17 12:31:07,371] jepsen worker 30 - jepsen.util 4530 :invoke :read [10 nil] INFO [2019-12-17 12:31:07,373] jepsen worker 38 - jepsen.util 4358 :fail :cas [12 [25 29]] :ValueNotSwapFrom INFO [2019-12-17 12:31:07,374] jepsen worker 38 - jepsen.util 4358 :invoke :write [12 34] INFO [2019-12-17 12:31:07,384] jepsen worker 30 - jepsen.util 4530 :ok :read [10 24] INFO [2019-12-17 12:31:07,384] jepsen worker 30 - jepsen.util 4530 :invoke :cas [10 [26 12]] INFO [2019-12-17 12:31:07,394] jepsen worker 30 - jepsen.util 4530 :fail :cas [10 [26 12]] :ValueNotSwapFrom INFO [2019-12-17 12:31:07,394] jepsen worker 30 - jepsen.util 4530 :invoke :cas [10 [44 2]] INFO [2019-12-17 12:31:07,409] jepsen worker 30 - jepsen.util 4530 :fail :cas [10 [44 2]] :ValueNotSwapFrom INFO [2019-12-17 12:31:07,409] jepsen worker 30 - jepsen.util 4530 :invoke :write [10 34] INFO [2019-12-17 12:31:07,461] jepsen worker 78 - jepsen.util 4578 :fail :cas [26 [7 42]] :ValueNotSwapFrom INFO [2019-12-17 12:31:07,461] jepsen worker 78 - jepsen.util 4578 :invoke :write [26 1] INFO [2019-12-17 12:31:07,467] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25161,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:31:07,467] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25161,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:31:07,467] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25164,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:31:07,467] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25164,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:31:07,491] jepsen worker 4 - jepsen.util 3514 :info :write [1 45] :RequestCanceledException INFO [2019-12-17 12:31:07,492] jepsen worker 4 - jepsen.util 3604 :invoke :read [1 nil] INFO [2019-12-17 12:31:07,509] jepsen worker 34 - jepsen.util 5344 :info :write [11 25] :RequestCanceledException INFO [2019-12-17 12:31:07,509] jepsen worker 72 - jepsen.util 4572 :fail :read [24 nil] :RequestTimeoutException INFO [2019-12-17 12:31:07,509] jepsen worker 72 - jepsen.util 4572 :invoke :read [24 nil] INFO [2019-12-17 12:31:07,510] jepsen worker 34 - jepsen.util 5434 :invoke :write [11 6] INFO [2019-12-17 12:31:07,525] jepsen worker 46 - jepsen.util 4456 :info :cas [15 [44 49]] :RequestCanceledException INFO [2019-12-17 12:31:07,526] jepsen worker 46 - jepsen.util 4546 :invoke :read [15 nil] INFO [2019-12-17 12:31:07,581] jepsen worker 79 - jepsen.util 4309 :ok :read [26 27] INFO [2019-12-17 12:31:07,581] jepsen worker 79 - jepsen.util 4309 :invoke :read [26 nil] INFO [2019-12-17 12:31:07,625] jepsen worker 68 - jepsen.util 5288 :info :cas [22 [6 12]] :RequestCanceledException INFO [2019-12-17 12:31:07,626] jepsen worker 68 - jepsen.util 5378 :invoke :cas [22 [2 29]] INFO [2019-12-17 12:31:07,639] jepsen worker 5 - jepsen.util 3965 :info :write [1 28] :RequestCanceledException INFO [2019-12-17 12:31:07,639] jepsen worker 55 - jepsen.util 5365 :info :write [18 37] :RequestCanceledException INFO [2019-12-17 12:31:07,640] jepsen worker 55 - jepsen.util 5455 :invoke :write [18 3] INFO [2019-12-17 12:31:07,641] jepsen worker 5 - jepsen.util 4055 :invoke :read [1 nil] INFO [2019-12-17 12:31:07,675] jepsen worker 36 - jepsen.util 4446 :fail :cas [12 [11 47]] :ValueNotSwapFrom INFO [2019-12-17 12:31:07,676] jepsen worker 36 - jepsen.util 4446 :invoke :cas [12 [16 39]] INFO [2019-12-17 12:31:07,678] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:31:07,679] jepsen nemesis - couchbase.util Rebalance check count: 186 INFO [2019-12-17 12:31:07,679] jepsen nemesis - couchbase.util Rebalance stuck count: 167 INFO [2019-12-17 12:31:07,693] jepsen worker 5 - jepsen.util 4055 :ok :read [1 39] INFO [2019-12-17 12:31:07,694] jepsen worker 5 - jepsen.util 4055 :invoke :write [1 32] INFO [2019-12-17 12:31:07,709] jepsen worker 23 - jepsen.util 5603 :info :write [7 49] :RequestCanceledException INFO [2019-12-17 12:31:07,710] jepsen worker 23 - jepsen.util 5693 :invoke :write [7 32] INFO [2019-12-17 12:31:07,717] jepsen worker 36 - jepsen.util 4446 :fail :cas [12 [16 39]] :ValueNotSwapFrom INFO [2019-12-17 12:31:07,717] jepsen worker 36 - jepsen.util 4446 :invoke :read [12 nil] INFO [2019-12-17 12:31:07,722] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25176,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:31:07,722] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25176,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:31:07,723] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25178,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:31:07,723] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25178,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:31:07,724] jepsen worker 3 - jepsen.util 4503 :info :write [1 1] :RequestCanceledException INFO [2019-12-17 12:31:07,726] jepsen worker 3 - jepsen.util 4593 :invoke :read [1 nil] INFO [2019-12-17 12:31:07,743] jepsen worker 36 - jepsen.util 4446 :ok :read [12 31] INFO [2019-12-17 12:31:07,743] jepsen worker 36 - jepsen.util 4446 :invoke :cas [12 [31 37]] INFO [2019-12-17 12:31:07,755] jepsen worker 73 - jepsen.util 4213 :info :write [24 4] :RequestCanceledException INFO [2019-12-17 12:31:07,756] jepsen worker 73 - jepsen.util 4303 :invoke :cas [24 [28 32]] INFO [2019-12-17 12:31:07,805] jepsen worker 12 - jepsen.util 4602 :fail :cas [4 [21 43]] :ValueNotSwapFrom INFO [2019-12-17 12:31:07,805] jepsen worker 12 - jepsen.util 4602 :invoke :cas [4 [47 40]] INFO [2019-12-17 12:31:07,822] jepsen worker 19 - jepsen.util 4429 :fail :read [6 nil] :RequestTimeoutException INFO [2019-12-17 12:31:07,822] jepsen worker 43 - jepsen.util 5353 :info :cas [14 [4 24]] :RequestCanceledException INFO [2019-12-17 12:31:07,823] jepsen worker 19 - jepsen.util 4429 :invoke :write [6 45] INFO [2019-12-17 12:31:07,824] jepsen worker 56 - jepsen.util 4556 :fail :cas [18 [43 11]] :ValueNotSwapFrom INFO [2019-12-17 12:31:07,824] jepsen worker 43 - jepsen.util 5443 :invoke :write [14 12] INFO [2019-12-17 12:31:07,824] jepsen worker 56 - jepsen.util 4556 :invoke :cas [18 [42 37]] INFO [2019-12-17 12:31:07,839] jepsen worker 44 - jepsen.util 6254 :info :write [14 16] :RequestCanceledException INFO [2019-12-17 12:31:07,840] jepsen worker 44 - jepsen.util 6344 :invoke :write [14 10] INFO [2019-12-17 12:31:07,846] jepsen worker 56 - jepsen.util 4556 :fail :cas [18 [42 37]] :ValueNotSwapFrom INFO [2019-12-17 12:31:07,846] jepsen worker 56 - jepsen.util 4556 :invoke :cas [18 [29 2]] INFO [2019-12-17 12:31:07,856] jepsen worker 59 - jepsen.util 5819 :info :write [19 49] :RequestCanceledException INFO [2019-12-17 12:31:07,857] jepsen worker 59 - jepsen.util 5909 :invoke :cas [19 [12 8]] INFO [2019-12-17 12:31:07,859] jepsen worker 18 - jepsen.util 4428 :ok :write [6 14] INFO [2019-12-17 12:31:07,860] jepsen worker 74 - jepsen.util 4934 :ok :write [24 46] INFO [2019-12-17 12:31:07,860] jepsen worker 18 - jepsen.util 4428 :invoke :cas [6 [26 25]] INFO [2019-12-17 12:31:07,860] jepsen worker 74 - jepsen.util 4934 :invoke :cas [24 [13 28]] INFO [2019-12-17 12:31:07,863] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25197,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:31:07,863] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25197,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:31:07,863] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25200,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:31:07,863] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25200,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:31:07,863] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25201,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:31:07,863] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25201,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:31:07,871] jepsen worker 10 - jepsen.util 4240 :ok :write [3 36] INFO [2019-12-17 12:31:07,871] jepsen worker 10 - jepsen.util 4240 :invoke :read [3 nil] INFO [2019-12-17 12:31:07,891] jepsen worker 6 - jepsen.util 6306 :info :cas [2 [5 26]] :RequestCanceledException INFO [2019-12-17 12:31:07,893] jepsen worker 6 - jepsen.util 6396 :invoke :read [2 nil] INFO [2019-12-17 12:31:07,893] jepsen worker 3 - jepsen.util 4593 :ok :read [1 39] INFO [2019-12-17 12:31:07,893] jepsen worker 3 - jepsen.util 4593 :invoke :write [1 42] INFO [2019-12-17 12:31:07,909] jepsen worker 46 - jepsen.util 4546 :ok :read [15 22] INFO [2019-12-17 12:31:07,909] jepsen worker 46 - jepsen.util 4546 :invoke :cas [15 [25 37]] INFO [2019-12-17 12:31:07,909] jepsen worker 66 - jepsen.util 5466 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:31:07,909] jepsen worker 66 - jepsen.util 5466 :invoke :cas [22 [12 13]] INFO [2019-12-17 12:31:07,923] jepsen worker 84 - jepsen.util 5304 :info :cas [28 [2 48]] :RequestCanceledException INFO [2019-12-17 12:31:07,924] jepsen worker 84 - jepsen.util 5394 :invoke :read [28 nil] INFO [2019-12-17 12:31:07,929] jepsen worker 73 - jepsen.util 4303 :fail :cas [24 [28 32]] :ValueNotSwapFrom INFO [2019-12-17 12:31:07,929] jepsen worker 73 - jepsen.util 4303 :invoke :read [24 nil] INFO [2019-12-17 12:31:07,937] jepsen worker 85 - jepsen.util 6385 :info :cas [28 [46 29]] :RequestCanceledException INFO [2019-12-17 12:31:07,937] jepsen worker 25 - jepsen.util 5875 :info :write [8 38] :RequestCanceledException INFO [2019-12-17 12:31:07,937] jepsen worker 71 - jepsen.util 4571 :info :write [23 28] :RequestCanceledException INFO [2019-12-17 12:31:07,940] jepsen worker 10 - jepsen.util 4240 :ok :read [3 36] INFO [2019-12-17 12:31:07,940] jepsen worker 10 - jepsen.util 4240 :invoke :read [3 nil] INFO [2019-12-17 12:31:07,940] jepsen worker 85 - jepsen.util 6475 :invoke :write [28 3] INFO [2019-12-17 12:31:07,941] jepsen worker 25 - jepsen.util 5965 :invoke :read [8 nil] INFO [2019-12-17 12:31:07,942] jepsen worker 71 - jepsen.util 4661 :invoke :cas [23 [1 48]] INFO [2019-12-17 12:31:07,948] jepsen worker 32 - jepsen.util 4802 :ok :write [10 14] INFO [2019-12-17 12:31:07,948] jepsen worker 32 - jepsen.util 4802 :invoke :read [10 nil] INFO [2019-12-17 12:31:07,957] jepsen worker 12 - jepsen.util 4602 :fail :cas [4 [47 40]] :ValueNotSwapFrom INFO [2019-12-17 12:31:07,957] jepsen worker 12 - jepsen.util 4602 :invoke :cas [4 [7 1]] INFO [2019-12-17 12:31:07,971] jepsen worker 46 - jepsen.util 4546 :fail :cas [15 [25 37]] :ValueNotSwapFrom INFO [2019-12-17 12:31:07,971] jepsen worker 46 - jepsen.util 4546 :invoke :write [15 46] INFO [2019-12-17 12:31:07,981] jepsen worker 65 - jepsen.util 3755 :ok :read [21 4] INFO [2019-12-17 12:31:07,981] jepsen worker 65 - jepsen.util 3755 :invoke :write [21 20] INFO [2019-12-17 12:31:07,991] jepsen worker 79 - jepsen.util 4309 :ok :read [26 27] INFO [2019-12-17 12:31:07,991] jepsen worker 79 - jepsen.util 4309 :invoke :write [26 33] INFO [2019-12-17 12:31:07,993] jepsen worker 9 - jepsen.util 4959 :info :write [3 15] :RequestCanceledException INFO [2019-12-17 12:31:07,993] jepsen worker 7 - jepsen.util 5587 :info :cas [2 [31 37]] :RequestCanceledException INFO [2019-12-17 12:31:07,994] jepsen worker 9 - jepsen.util 5049 :invoke :cas [3 [40 32]] INFO [2019-12-17 12:31:07,994] jepsen worker 7 - jepsen.util 5677 :invoke :cas [2 [49 6]] INFO [2019-12-17 12:31:08,001] jepsen worker 73 - jepsen.util 4303 :ok :read [24 46] INFO [2019-12-17 12:31:08,001] jepsen worker 73 - jepsen.util 4303 :invoke :read [24 nil] INFO [2019-12-17 12:31:08,009] jepsen worker 56 - jepsen.util 4556 :fail :cas [18 [29 2]] :ValueNotSwapFrom INFO [2019-12-17 12:31:08,009] jepsen worker 56 - jepsen.util 4556 :invoke :write [18 43] INFO [2019-12-17 12:31:08,010] jepsen worker 82 - jepsen.util 6652 :info :write [27 26] :RequestCanceledException INFO [2019-12-17 12:31:08,010] jepsen worker 47 - jepsen.util 4817 :info :cas [15 [49 11]] :RequestCanceledException INFO [2019-12-17 12:31:08,011] jepsen worker 82 - jepsen.util 6742 :invoke :read [27 nil] INFO [2019-12-17 12:31:08,011] jepsen worker 47 - jepsen.util 4907 :invoke :read [15 nil] INFO [2019-12-17 12:31:08,022] jepsen worker 72 - jepsen.util 4572 :ok :read [24 46] INFO [2019-12-17 12:31:08,022] jepsen worker 72 - jepsen.util 4572 :invoke :read [24 nil] INFO [2019-12-17 12:31:08,024] jepsen worker 39 - jepsen.util 5349 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:31:08,024] jepsen worker 62 - jepsen.util 5282 :info :cas [20 [42 14]] :RequestCanceledException INFO [2019-12-17 12:31:08,025] jepsen worker 39 - jepsen.util 5349 :invoke :read [13 nil] INFO [2019-12-17 12:31:08,025] jepsen worker 62 - jepsen.util 5372 :invoke :cas [20 [15 29]] INFO [2019-12-17 12:31:08,062] jepsen worker 31 - jepsen.util 4081 :fail :cas [10 [18 39]] :ValueNotSwapFrom INFO [2019-12-17 12:31:08,062] jepsen worker 31 - jepsen.util 4081 :invoke :cas [10 [32 12]] INFO [2019-12-17 12:31:08,067] jepsen worker 10 - jepsen.util 4240 :ok :read [3 36] INFO [2019-12-17 12:31:08,068] jepsen worker 10 - jepsen.util 4240 :invoke :cas [3 [34 8]] INFO [2019-12-17 12:31:08,069] jepsen worker 3 - jepsen.util 4593 :ok :write [1 42] INFO [2019-12-17 12:31:08,069] jepsen worker 3 - jepsen.util 4593 :invoke :write [1 7] INFO [2019-12-17 12:31:08,074] jepsen worker 13 - jepsen.util 4153 :ok :write [4 7] INFO [2019-12-17 12:31:08,074] jepsen worker 13 - jepsen.util 4153 :invoke :write [4 40] INFO [2019-12-17 12:31:08,078] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25220,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:31:08,078] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25220,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:31:08,078] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25223,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:31:08,078] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25223,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:31:08,079] jepsen worker 18 - jepsen.util 4428 :fail :cas [6 [26 25]] :ValueNotSwapFrom INFO [2019-12-17 12:31:08,080] jepsen worker 18 - jepsen.util 4428 :invoke :cas [6 [23 38]] INFO [2019-12-17 12:31:08,110] jepsen worker 29 - jepsen.util 3719 :fail :cas [9 [27 23]] :ValueNotSwapFrom INFO [2019-12-17 12:31:08,110] jepsen worker 29 - jepsen.util 3719 :invoke :read [9 nil] INFO [2019-12-17 12:31:08,121] jepsen worker 35 - jepsen.util 5795 :info :cas [11 [28 32]] :RequestCanceledException INFO [2019-12-17 12:31:08,123] jepsen worker 35 - jepsen.util 5885 :invoke :write [11 2] INFO [2019-12-17 12:31:08,129] jepsen worker 32 - jepsen.util 4802 :ok :read [10 14] INFO [2019-12-17 12:31:08,130] jepsen worker 32 - jepsen.util 4802 :invoke :read [10 nil] INFO [2019-12-17 12:31:08,133] jepsen worker 52 - jepsen.util 4642 :ok :write [17 48] INFO [2019-12-17 12:31:08,133] jepsen worker 52 - jepsen.util 4642 :invoke :write [17 26] INFO [2019-12-17 12:31:08,140] jepsen worker 83 - jepsen.util 6023 :info :write [27 7] :RequestCanceledException INFO [2019-12-17 12:31:08,143] jepsen worker 83 - jepsen.util 6113 :invoke :write [27 34] INFO [2019-12-17 12:31:08,150] jepsen worker 31 - jepsen.util 4081 :fail :cas [10 [32 12]] :ValueNotSwapFrom INFO [2019-12-17 12:31:08,150] jepsen worker 31 - jepsen.util 4081 :invoke :read [10 nil] INFO [2019-12-17 12:31:08,153] jepsen worker 65 - jepsen.util 3755 :ok :write [21 20] INFO [2019-12-17 12:31:08,154] jepsen worker 65 - jepsen.util 3755 :invoke :cas [21 [17 5]] INFO [2019-12-17 12:31:08,154] jepsen worker 29 - jepsen.util 3719 :ok :read [9 30] INFO [2019-12-17 12:31:08,155] jepsen worker 29 - jepsen.util 3719 :invoke :cas [9 [10 1]] INFO [2019-12-17 12:31:08,156] jepsen worker 63 - jepsen.util 3573 :info :cas [21 [5 13]] :RequestCanceledException INFO [2019-12-17 12:31:08,157] jepsen worker 63 - jepsen.util 3663 :invoke :cas [21 [39 38]] INFO [2019-12-17 12:31:08,191] jepsen worker 2 - jepsen.util 5582 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:31:08,191] jepsen worker 2 - jepsen.util 5582 :invoke :read [0 nil] INFO [2019-12-17 12:31:08,200] jepsen worker 38 - jepsen.util 4358 :ok :write [12 34] INFO [2019-12-17 12:31:08,201] jepsen worker 38 - jepsen.util 4358 :invoke :write [12 49] INFO [2019-12-17 12:31:08,205] jepsen worker 11 - jepsen.util 4691 :ok :write [3 5] INFO [2019-12-17 12:31:08,206] jepsen worker 11 - jepsen.util 4691 :invoke :write [3 43] INFO [2019-12-17 12:31:08,206] jepsen worker 70 - jepsen.util 5020 :info :write [23 18] :RequestCanceledException INFO [2019-12-17 12:31:08,207] jepsen worker 70 - jepsen.util 5110 :invoke :cas [23 [34 40]] INFO [2019-12-17 12:31:08,223] jepsen worker 40 - jepsen.util 6520 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:31:08,223] jepsen worker 58 - jepsen.util 5818 :info :write [19 26] :RequestCanceledException INFO [2019-12-17 12:31:08,223] jepsen worker 40 - jepsen.util 6520 :invoke :write [13 41] INFO [2019-12-17 12:31:08,224] jepsen worker 58 - jepsen.util 5908 :invoke :cas [19 [2 10]] INFO [2019-12-17 12:31:08,229] jepsen worker 4 - jepsen.util 3604 :ok :read [1 42] INFO [2019-12-17 12:31:08,229] jepsen worker 4 - jepsen.util 3604 :invoke :cas [1 [20 44]] INFO [2019-12-17 12:31:08,229] jepsen worker 0 - jepsen.util 6390 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:31:08,229] jepsen worker 60 - jepsen.util 6450 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:31:08,229] jepsen worker 0 - jepsen.util 6390 :invoke :write [0 44] INFO [2019-12-17 12:31:08,230] jepsen worker 60 - jepsen.util 6450 :invoke :read [20 nil] INFO [2019-12-17 12:31:08,231] jepsen worker 14 - jepsen.util 4514 :ok :write [4 15] INFO [2019-12-17 12:31:08,232] jepsen worker 14 - jepsen.util 4514 :invoke :read [4 nil] INFO [2019-12-17 12:31:08,237] jepsen worker 32 - jepsen.util 4802 :ok :read [10 14] INFO [2019-12-17 12:31:08,237] jepsen worker 32 - jepsen.util 4802 :invoke :read [10 nil] INFO [2019-12-17 12:31:08,256] jepsen worker 50 - jepsen.util 5990 :info :write [16 18] :RequestCanceledException INFO [2019-12-17 12:31:08,256] jepsen worker 50 - jepsen.util 6080 :invoke :read [16 nil] INFO [2019-12-17 12:31:08,264] jepsen worker 18 - jepsen.util 4428 :fail :cas [6 [23 38]] :ValueNotSwapFrom INFO [2019-12-17 12:31:08,264] jepsen worker 18 - jepsen.util 4428 :invoke :write [6 7] INFO [2019-12-17 12:31:08,270] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25258,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:31:08,270] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25258,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:31:08,270] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25261,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:31:08,270] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25261,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:31:08,270] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25262,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:31:08,270] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25262,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:31:08,291] jepsen worker 88 - jepsen.util 4498 :info :write [29 45] :RequestCanceledException INFO [2019-12-17 12:31:08,292] jepsen worker 88 - jepsen.util 4588 :invoke :cas [29 [7 44]] INFO [2019-12-17 12:31:08,297] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25250,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:31:08,297] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25250,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:31:08,297] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25252,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:31:08,297] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25252,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:31:08,308] jepsen worker 63 - jepsen.util 3663 :fail :cas [21 [39 38]] :ValueNotSwapFrom INFO [2019-12-17 12:31:08,309] jepsen worker 63 - jepsen.util 3663 :invoke :read [21 nil] INFO [2019-12-17 12:31:08,309] jepsen worker 67 - jepsen.util 6007 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:31:08,309] jepsen worker 67 - jepsen.util 6007 :invoke :write [22 20] INFO [2019-12-17 12:31:08,325] jepsen worker 57 - jepsen.util 5547 :info :write [19 2] :RequestCanceledException INFO [2019-12-17 12:31:08,325] jepsen worker 26 - jepsen.util 5876 :info :cas [8 [29 7]] :RequestCanceledException INFO [2019-12-17 12:31:08,326] jepsen worker 57 - jepsen.util 5637 :invoke :write [19 38] INFO [2019-12-17 12:31:08,326] jepsen worker 26 - jepsen.util 5966 :invoke :read [8 nil] INFO [2019-12-17 12:31:08,339] jepsen worker 45 - jepsen.util 4455 :info :cas [15 [31 48]] :RequestCanceledException INFO [2019-12-17 12:31:08,340] jepsen worker 45 - jepsen.util 4545 :invoke :write [15 19] INFO [2019-12-17 12:31:08,343] jepsen worker 74 - jepsen.util 4934 :fail :cas [24 [13 28]] :ValueNotSwapFrom INFO [2019-12-17 12:31:08,343] jepsen worker 74 - jepsen.util 4934 :invoke :read [24 nil] INFO [2019-12-17 12:31:08,355] jepsen worker 65 - jepsen.util 3755 :fail :cas [21 [17 5]] :ValueNotSwapFrom INFO [2019-12-17 12:31:08,355] jepsen worker 65 - jepsen.util 3755 :invoke :write [21 45] INFO [2019-12-17 12:31:08,367] jepsen worker 63 - jepsen.util 3663 :ok :read [21 20] INFO [2019-12-17 12:31:08,367] jepsen worker 63 - jepsen.util 3663 :invoke :cas [21 [41 22]] INFO [2019-12-17 12:31:08,375] jepsen worker 46 - jepsen.util 4546 :ok :write [15 46] INFO [2019-12-17 12:31:08,375] jepsen worker 46 - jepsen.util 4546 :invoke :cas [15 [48 30]] INFO [2019-12-17 12:31:08,380] jepsen worker 31 - jepsen.util 4081 :ok :read [10 14] INFO [2019-12-17 12:31:08,380] jepsen worker 31 - jepsen.util 4081 :invoke :cas [10 [1 26]] INFO [2019-12-17 12:31:08,394] jepsen worker 63 - jepsen.util 3663 :fail :cas [21 [41 22]] :ValueNotSwapFrom INFO [2019-12-17 12:31:08,394] jepsen worker 63 - jepsen.util 3663 :invoke :cas [21 [11 40]] INFO [2019-12-17 12:31:08,407] jepsen worker 89 - jepsen.util 4139 :ok :write [29 36] INFO [2019-12-17 12:31:08,407] jepsen worker 89 - jepsen.util 4139 :invoke :cas [29 [16 3]] INFO [2019-12-17 12:31:08,409] jepsen worker 78 - jepsen.util 4578 :ok :write [26 1] INFO [2019-12-17 12:31:08,409] jepsen worker 22 - jepsen.util 5782 :info :cas [7 [1 43]] :RequestCanceledException INFO [2019-12-17 12:31:08,409] jepsen worker 78 - jepsen.util 4578 :invoke :write [26 9] INFO [2019-12-17 12:31:08,410] jepsen worker 22 - jepsen.util 5872 :invoke :write [7 42] INFO [2019-12-17 12:31:08,425] jepsen worker 80 - jepsen.util 4310 :info :write [26 39] :RequestCanceledException INFO [2019-12-17 12:31:08,425] jepsen worker 48 - jepsen.util 5268 :info :write [16 3] :RequestCanceledException INFO [2019-12-17 12:31:08,426] jepsen worker 80 - jepsen.util 4400 :invoke :read [26 nil] INFO [2019-12-17 12:31:08,426] jepsen worker 48 - jepsen.util 5358 :invoke :read [16 nil] INFO [2019-12-17 12:31:08,436] jepsen worker 10 - jepsen.util 4240 :fail :cas [3 [34 8]] :ValueNotSwapFrom INFO [2019-12-17 12:31:08,437] jepsen worker 10 - jepsen.util 4240 :invoke :cas [3 [15 30]] INFO [2019-12-17 12:31:08,438] jepsen worker 63 - jepsen.util 3663 :fail :cas [21 [11 40]] :ValueNotSwapFrom INFO [2019-12-17 12:31:08,438] jepsen worker 63 - jepsen.util 3663 :invoke :read [21 nil] INFO [2019-12-17 12:31:08,439] jepsen worker 17 - jepsen.util 5507 :info :cas [5 [8 35]] :RequestCanceledException INFO [2019-12-17 12:31:08,440] jepsen worker 17 - jepsen.util 5597 :invoke :cas [5 [34 23]] INFO [2019-12-17 12:31:08,442] jepsen worker 89 - jepsen.util 4139 :fail :cas [29 [16 3]] :ValueNotSwapFrom INFO [2019-12-17 12:31:08,443] jepsen worker 89 - jepsen.util 4139 :invoke :read [29 nil] INFO [2019-12-17 12:31:08,447] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25290,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:31:08,447] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25290,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:31:08,447] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25296,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:31:08,447] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25296,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:31:08,461] jepsen worker 46 - jepsen.util 4546 :fail :cas [15 [48 30]] :ValueNotSwapFrom INFO [2019-12-17 12:31:08,462] jepsen worker 46 - jepsen.util 4546 :invoke :write [15 21] INFO [2019-12-17 12:31:08,465] jepsen worker 80 - jepsen.util 4400 :ok :read [26 1] INFO [2019-12-17 12:31:08,465] jepsen worker 80 - jepsen.util 4400 :invoke :read [26 nil] INFO [2019-12-17 12:31:08,489] jepsen worker 89 - jepsen.util 4139 :ok :read [29 36] INFO [2019-12-17 12:31:08,489] jepsen worker 89 - jepsen.util 4139 :invoke :read [29 nil] INFO [2019-12-17 12:31:08,498] jepsen worker 29 - jepsen.util 3719 :fail :cas [9 [10 1]] :ValueNotSwapFrom INFO [2019-12-17 12:31:08,498] jepsen worker 29 - jepsen.util 3719 :invoke :cas [9 [32 15]] INFO [2019-12-17 12:31:08,516] jepsen worker 80 - jepsen.util 4400 :ok :read [26 1] INFO [2019-12-17 12:31:08,516] jepsen worker 80 - jepsen.util 4400 :invoke :write [26 4] INFO [2019-12-17 12:31:08,525] jepsen worker 75 - jepsen.util 5835 :info :cas [25 [1 13]] :RequestCanceledException INFO [2019-12-17 12:31:08,525] jepsen worker 42 - jepsen.util 6432 :info :write [14 14] :RequestCanceledException INFO [2019-12-17 12:31:08,527] jepsen worker 42 - jepsen.util 6522 :invoke :cas [14 [43 16]] INFO [2019-12-17 12:31:08,527] jepsen worker 75 - jepsen.util 5925 :invoke :read [25 nil] INFO [2019-12-17 12:31:08,528] jepsen worker 63 - jepsen.util 3663 :ok :read [21 20] INFO [2019-12-17 12:31:08,528] jepsen worker 63 - jepsen.util 3663 :invoke :write [21 40] INFO [2019-12-17 12:31:08,547] jepsen worker 10 - jepsen.util 4240 :fail :cas [3 [15 30]] :ValueNotSwapFrom INFO [2019-12-17 12:31:08,547] jepsen worker 10 - jepsen.util 4240 :invoke :write [3 32] INFO [2019-12-17 12:31:08,555] jepsen worker 41 - jepsen.util 6071 :info :cas [13 [3 42]] :RequestCanceledException INFO [2019-12-17 12:31:08,555] jepsen worker 64 - jepsen.util 3394 :info :cas [21 [37 32]] :RequestCanceledException INFO [2019-12-17 12:31:08,556] jepsen worker 64 - jepsen.util 3484 :invoke :read [21 nil] INFO [2019-12-17 12:31:08,557] jepsen worker 41 - jepsen.util 6161 :invoke :cas [13 [5 42]] INFO [2019-12-17 12:31:08,559] jepsen worker 32 - jepsen.util 4802 :ok :read [10 14] INFO [2019-12-17 12:31:08,559] jepsen worker 32 - jepsen.util 4802 :invoke :write [10 8] INFO [2019-12-17 12:31:08,570] jepsen worker 31 - jepsen.util 4081 :fail :cas [10 [1 26]] :ValueNotSwapFrom INFO [2019-12-17 12:31:08,570] jepsen worker 31 - jepsen.util 4081 :invoke :write [10 39] INFO [2019-12-17 12:31:08,577] jepsen worker 89 - jepsen.util 4139 :ok :read [29 36] INFO [2019-12-17 12:31:08,577] jepsen worker 89 - jepsen.util 4139 :invoke :read [29 nil] INFO [2019-12-17 12:31:08,589] jepsen worker 29 - jepsen.util 3719 :fail :cas [9 [32 15]] :ValueNotSwapFrom INFO [2019-12-17 12:31:08,589] jepsen worker 29 - jepsen.util 3719 :invoke :read [9 nil] INFO [2019-12-17 12:31:08,592] jepsen worker 33 - jepsen.util 5703 :info :write [11 4] :RequestCanceledException INFO [2019-12-17 12:31:08,592] jepsen worker 27 - jepsen.util 4167 :info :write [9 35] :RequestCanceledException INFO [2019-12-17 12:31:08,592] jepsen worker 33 - jepsen.util 5793 :invoke :write [11 44] INFO [2019-12-17 12:31:08,592] jepsen worker 27 - jepsen.util 4257 :invoke :read [9 nil] INFO [2019-12-17 12:31:08,625] jepsen worker 8 - jepsen.util 5048 :info :write [2 0] :RequestCanceledException INFO [2019-12-17 12:31:08,626] jepsen worker 8 - jepsen.util 5138 :invoke :read [2 nil] INFO [2019-12-17 12:31:08,627] jepsen worker 56 - jepsen.util 4556 :ok :write [18 43] INFO [2019-12-17 12:31:08,627] jepsen worker 5 - jepsen.util 4055 :ok :write [1 32] INFO [2019-12-17 12:31:08,628] jepsen worker 56 - jepsen.util 4556 :invoke :cas [18 [12 0]] INFO [2019-12-17 12:31:08,628] jepsen worker 5 - jepsen.util 4055 :invoke :write [1 18] INFO [2019-12-17 12:31:08,639] jepsen worker 1 - jepsen.util 5311 :info :write [0 18] :RequestCanceledException INFO [2019-12-17 12:31:08,640] jepsen worker 1 - jepsen.util 5401 :invoke :write [0 9] INFO [2019-12-17 12:31:08,647] jepsen worker 12 - jepsen.util 4602 :fail :cas [4 [7 1]] :ValueNotSwapFrom INFO [2019-12-17 12:31:08,647] jepsen worker 12 - jepsen.util 4602 :invoke :write [4 26] INFO [2019-12-17 12:31:08,666] jepsen worker 13 - jepsen.util 4153 :ok :write [4 40] INFO [2019-12-17 12:31:08,666] jepsen worker 13 - jepsen.util 4153 :invoke :read [4 nil] INFO [2019-12-17 12:31:08,670] jepsen worker 63 - jepsen.util 3663 :ok :write [21 40] INFO [2019-12-17 12:31:08,670] jepsen worker 63 - jepsen.util 3663 :invoke :read [21 nil] INFO [2019-12-17 12:31:08,687] jepsen worker 80 - jepsen.util 4400 :ok :write [26 4] INFO [2019-12-17 12:31:08,687] jepsen worker 80 - jepsen.util 4400 :invoke :write [26 24] INFO [2019-12-17 12:31:08,691] jepsen worker 87 - jepsen.util 4137 :info :write [29 3] :RequestCanceledException INFO [2019-12-17 12:31:08,691] jepsen worker 76 - jepsen.util 5836 :info :cas [25 [42 28]] :RequestCanceledException INFO [2019-12-17 12:31:08,692] jepsen worker 87 - jepsen.util 4227 :invoke :read [29 nil] INFO [2019-12-17 12:31:08,692] jepsen worker 76 - jepsen.util 5926 :invoke :cas [25 [37 27]] INFO [2019-12-17 12:31:08,695] jepsen worker 14 - jepsen.util 4514 :ok :read [4 40] INFO [2019-12-17 12:31:08,695] jepsen worker 14 - jepsen.util 4514 :invoke :cas [4 [26 39]] INFO [2019-12-17 12:31:08,702] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:31:08,702] jepsen nemesis - couchbase.util Rebalance check count: 187 INFO [2019-12-17 12:31:08,702] jepsen nemesis - couchbase.util Rebalance stuck count: 168 INFO [2019-12-17 12:31:08,704] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25322,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:31:08,705] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25322,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:31:08,705] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25326,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:31:08,705] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25326,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:31:08,708] jepsen worker 63 - jepsen.util 3663 :ok :read [21 40] INFO [2019-12-17 12:31:08,708] jepsen worker 63 - jepsen.util 3663 :invoke :read [21 nil] INFO [2019-12-17 12:31:08,709] jepsen worker 16 - jepsen.util 6496 :info :cas [5 [27 36]] :RequestCanceledException INFO [2019-12-17 12:31:08,711] jepsen worker 16 - jepsen.util 6586 :invoke :write [5 5] INFO [2019-12-17 12:31:08,725] jepsen worker 37 - jepsen.util 4177 :info :write [12 45] :RequestCanceledException INFO [2019-12-17 12:31:08,727] jepsen worker 37 - jepsen.util 4267 :invoke :cas [12 [16 40]] INFO [2019-12-17 12:31:08,738] jepsen worker 63 - jepsen.util 3663 :ok :read [21 40] INFO [2019-12-17 12:31:08,738] jepsen worker 63 - jepsen.util 3663 :invoke :write [21 20] INFO [2019-12-17 12:31:08,739] jepsen worker 81 - jepsen.util 5751 :info :write [27 19] :RequestCanceledException INFO [2019-12-17 12:31:08,740] jepsen worker 81 - jepsen.util 5841 :invoke :read [27 nil] INFO [2019-12-17 12:31:08,749] jepsen worker 55 - jepsen.util 5455 :ok :write [18 3] INFO [2019-12-17 12:31:08,749] jepsen worker 55 - jepsen.util 5455 :invoke :cas [18 [22 13]] INFO [2019-12-17 12:31:08,767] jepsen worker 3 - jepsen.util 4593 :ok :write [1 7] INFO [2019-12-17 12:31:08,767] jepsen worker 3 - jepsen.util 4593 :invoke :cas [1 [15 20]] INFO [2019-12-17 12:31:08,787] jepsen worker 89 - jepsen.util 4139 :ok :read [29 36] INFO [2019-12-17 12:31:08,787] jepsen worker 15 - jepsen.util 4875 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:31:08,787] jepsen worker 89 - jepsen.util 4139 :invoke :read [29 nil] INFO [2019-12-17 12:31:08,787] jepsen worker 15 - jepsen.util 4875 :invoke :cas [5 [27 28]] INFO [2019-12-17 12:31:08,791] jepsen worker 63 - jepsen.util 3663 :ok :write [21 20] INFO [2019-12-17 12:31:08,791] jepsen worker 63 - jepsen.util 3663 :invoke :write [21 5] INFO [2019-12-17 12:31:08,798] jepsen worker 70 - jepsen.util 5110 :fail :cas [23 [34 40]] :ValueNotSwapFrom INFO [2019-12-17 12:31:08,798] jepsen worker 70 - jepsen.util 5110 :invoke :cas [23 [38 12]] INFO [2019-12-17 12:31:08,818] jepsen worker 56 - jepsen.util 4556 :fail :cas [18 [12 0]] :ValueNotSwapFrom INFO [2019-12-17 12:31:08,818] jepsen worker 56 - jepsen.util 4556 :invoke :cas [18 [27 12]] INFO [2019-12-17 12:31:08,828] jepsen worker 71 - jepsen.util 4661 :fail :cas [23 [1 48]] :ValueNotSwapFrom INFO [2019-12-17 12:31:08,828] jepsen worker 71 - jepsen.util 4661 :invoke :write [23 4] INFO [2019-12-17 12:31:08,836] jepsen worker 63 - jepsen.util 3663 :ok :write [21 5] INFO [2019-12-17 12:31:08,836] jepsen worker 11 - jepsen.util 4691 :ok :write [3 43] INFO [2019-12-17 12:31:08,836] jepsen worker 63 - jepsen.util 3663 :invoke :cas [21 [23 5]] INFO [2019-12-17 12:31:08,836] jepsen worker 11 - jepsen.util 4691 :invoke :cas [3 [29 40]] INFO [2019-12-17 12:31:08,841] jepsen worker 18 - jepsen.util 4428 :ok :write [6 7] INFO [2019-12-17 12:31:08,841] jepsen worker 18 - jepsen.util 4428 :invoke :read [6 nil] INFO [2019-12-17 12:31:08,864] jepsen worker 74 - jepsen.util 4934 :ok :read [24 46] INFO [2019-12-17 12:31:08,864] jepsen worker 74 - jepsen.util 4934 :invoke :write [24 46] INFO [2019-12-17 12:31:08,871] jepsen worker 63 - jepsen.util 3663 :fail :cas [21 [23 5]] :ValueNotSwapFrom INFO [2019-12-17 12:31:08,871] jepsen worker 63 - jepsen.util 3663 :invoke :read [21 nil] INFO [2019-12-17 12:31:08,908] jepsen worker 27 - jepsen.util 4257 :ok :read [9 30] INFO [2019-12-17 12:31:08,908] jepsen worker 53 - jepsen.util 4373 :info :write [17 45] :RequestCanceledException INFO [2019-12-17 12:31:08,908] jepsen worker 27 - jepsen.util 4257 :invoke :read [9 nil] INFO [2019-12-17 12:31:08,909] jepsen worker 53 - jepsen.util 4463 :invoke :cas [17 [42 14]] INFO [2019-12-17 12:31:08,925] jepsen worker 21 - jepsen.util 5691 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:31:08,925] jepsen worker 21 - jepsen.util 5691 :invoke :read [7 nil] INFO [2019-12-17 12:31:08,929] jepsen worker 11 - jepsen.util 4691 :fail :cas [3 [29 40]] :ValueNotSwapFrom INFO [2019-12-17 12:31:08,929] jepsen worker 11 - jepsen.util 4691 :invoke :cas [3 [45 46]] INFO [2019-12-17 12:31:08,929] jepsen worker 63 - jepsen.util 3663 :ok :read [21 5] INFO [2019-12-17 12:31:08,930] jepsen worker 63 - jepsen.util 3663 :invoke :write [21 38] INFO [2019-12-17 12:31:08,942] jepsen worker 18 - jepsen.util 4428 :ok :read [6 7] INFO [2019-12-17 12:31:08,942] jepsen worker 18 - jepsen.util 4428 :invoke :read [6 nil] INFO [2019-12-17 12:31:08,959] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25354,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:31:08,959] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25354,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:31:08,959] jepsen worker 27 - jepsen.util 4257 :ok :read [9 30] INFO [2019-12-17 12:31:08,959] jepsen worker 27 - jepsen.util 4257 :invoke :cas [9 [47 24]] INFO [2019-12-17 12:31:08,969] jepsen worker 18 - jepsen.util 4428 :ok :read [6 7] INFO [2019-12-17 12:31:08,969] jepsen worker 18 - jepsen.util 4428 :invoke :cas [6 [20 0]] INFO [2019-12-17 12:31:08,983] jepsen worker 27 - jepsen.util 4257 :fail :cas [9 [47 24]] :ValueNotSwapFrom INFO [2019-12-17 12:31:08,983] jepsen worker 27 - jepsen.util 4257 :invoke :cas [9 [0 23]] INFO [2019-12-17 12:31:09,004] jepsen worker 18 - jepsen.util 4428 :fail :cas [6 [20 0]] :ValueNotSwapFrom INFO [2019-12-17 12:31:09,005] jepsen worker 18 - jepsen.util 4428 :invoke :read [6 nil] INFO [2019-12-17 12:31:09,007] jepsen worker 70 - jepsen.util 5110 :fail :cas [23 [38 12]] :ValueNotSwapFrom INFO [2019-12-17 12:31:09,007] jepsen worker 70 - jepsen.util 5110 :invoke :cas [23 [4 18]] INFO [2019-12-17 12:31:09,009] jepsen worker 28 - jepsen.util 3268 :info :write [9 11] :RequestCanceledException INFO [2019-12-17 12:31:09,010] jepsen worker 28 - jepsen.util 3358 :invoke :write [9 26] INFO [2019-12-17 12:31:09,018] jepsen worker 27 - jepsen.util 4257 :fail :cas [9 [0 23]] :ValueNotSwapFrom INFO [2019-12-17 12:31:09,018] jepsen worker 27 - jepsen.util 4257 :invoke :cas [9 [34 18]] INFO [2019-12-17 12:31:09,023] jepsen worker 4 - jepsen.util 3604 :fail :cas [1 [20 44]] :ValueNotSwapFrom INFO [2019-12-17 12:31:09,023] jepsen worker 4 - jepsen.util 3604 :invoke :read [1 nil] INFO [2019-12-17 12:31:09,026] jepsen worker 69 - jepsen.util 4479 :info :write [23 13] :RequestCanceledException INFO [2019-12-17 12:31:09,027] jepsen worker 69 - jepsen.util 4569 :invoke :read [23 nil] INFO [2019-12-17 12:31:09,037] jepsen worker 63 - jepsen.util 3663 :ok :write [21 38] INFO [2019-12-17 12:31:09,038] jepsen worker 63 - jepsen.util 3663 :invoke :write [21 20] INFO [2019-12-17 12:31:09,041] jepsen worker 37 - jepsen.util 4267 :fail :cas [12 [16 40]] :ValueNotSwapFrom INFO [2019-12-17 12:31:09,042] jepsen worker 37 - jepsen.util 4267 :invoke :write [12 28] INFO [2019-12-17 12:31:09,046] jepsen worker 18 - jepsen.util 4428 :ok :read [6 7] INFO [2019-12-17 12:31:09,046] jepsen worker 18 - jepsen.util 4428 :invoke :read [6 nil] INFO [2019-12-17 12:31:09,051] jepsen worker 27 - jepsen.util 4257 :fail :cas [9 [34 18]] :ValueNotSwapFrom INFO [2019-12-17 12:31:09,051] jepsen worker 27 - jepsen.util 4257 :invoke :read [9 nil] INFO [2019-12-17 12:31:09,065] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25368,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:31:09,065] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25368,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:31:09,065] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25370,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:31:09,065] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25370,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:31:09,067] jepsen worker 18 - jepsen.util 4428 :ok :read [6 7] INFO [2019-12-17 12:31:09,068] jepsen worker 18 - jepsen.util 4428 :invoke :write [6 14] INFO [2019-12-17 12:31:09,075] jepsen worker 27 - jepsen.util 4257 :ok :read [9 30] INFO [2019-12-17 12:31:09,075] jepsen worker 27 - jepsen.util 4257 :invoke :write [9 25] INFO [2019-12-17 12:31:09,087] jepsen worker 63 - jepsen.util 3663 :ok :write [21 20] INFO [2019-12-17 12:31:09,088] jepsen worker 63 - jepsen.util 3663 :invoke :read [21 nil] INFO [2019-12-17 12:31:09,117] jepsen worker 63 - jepsen.util 3663 :ok :read [21 20] INFO [2019-12-17 12:31:09,117] jepsen worker 63 - jepsen.util 3663 :invoke :cas [21 [49 30]] INFO [2019-12-17 12:31:09,125] jepsen worker 56 - jepsen.util 4556 :fail :cas [18 [27 12]] :ValueNotSwapFrom INFO [2019-12-17 12:31:09,125] jepsen worker 61 - jepsen.util 6361 :info :write [20 8] :RequestCanceledException INFO [2019-12-17 12:31:09,125] jepsen worker 56 - jepsen.util 4556 :invoke :write [18 8] INFO [2019-12-17 12:31:09,126] jepsen worker 61 - jepsen.util 6451 :invoke :cas [20 [25 17]] INFO [2019-12-17 12:31:09,128] jepsen worker 63 - jepsen.util 3663 :fail :cas [21 [49 30]] :ValueNotSwapFrom INFO [2019-12-17 12:31:09,128] jepsen worker 63 - jepsen.util 3663 :invoke :read [21 nil] INFO [2019-12-17 12:31:09,147] jepsen worker 89 - jepsen.util 4139 :ok :read [29 36] INFO [2019-12-17 12:31:09,147] jepsen worker 89 - jepsen.util 4139 :invoke :cas [29 [37 11]] INFO [2019-12-17 12:31:09,172] jepsen worker 63 - jepsen.util 3663 :ok :read [21 20] INFO [2019-12-17 12:31:09,172] jepsen worker 63 - jepsen.util 3663 :invoke :cas [21 [29 0]] INFO [2019-12-17 12:31:09,172] jepsen worker 88 - jepsen.util 4588 :fail :cas [29 [7 44]] :ValueNotSwapFrom INFO [2019-12-17 12:31:09,172] jepsen worker 88 - jepsen.util 4588 :invoke :write [29 24] INFO [2019-12-17 12:31:09,176] jepsen worker 3 - jepsen.util 4593 :fail :cas [1 [15 20]] :ValueNotSwapFrom INFO [2019-12-17 12:31:09,176] jepsen worker 3 - jepsen.util 4593 :invoke :cas [1 [5 40]] INFO [2019-12-17 12:31:09,187] jepsen worker 63 - jepsen.util 3663 :fail :cas [21 [29 0]] :ValueNotSwapFrom INFO [2019-12-17 12:31:09,187] jepsen worker 63 - jepsen.util 3663 :invoke :read [21 nil] INFO [2019-12-17 12:31:09,194] jepsen worker 4 - jepsen.util 3604 :ok :read [1 7] INFO [2019-12-17 12:31:09,194] jepsen worker 4 - jepsen.util 3604 :invoke :write [1 3] INFO [2019-12-17 12:31:09,200] jepsen worker 13 - jepsen.util 4153 :ok :read [4 40] INFO [2019-12-17 12:31:09,200] jepsen worker 13 - jepsen.util 4153 :invoke :cas [4 [20 33]] INFO [2019-12-17 12:31:09,202] jepsen worker 53 - jepsen.util 4463 :fail :cas [17 [42 14]] :ValueNotSwapFrom INFO [2019-12-17 12:31:09,202] jepsen worker 53 - jepsen.util 4463 :invoke :cas [17 [27 1]] INFO [2019-12-17 12:31:09,202] jepsen worker 63 - jepsen.util 3663 :ok :read [21 20] INFO [2019-12-17 12:31:09,203] jepsen worker 63 - jepsen.util 3663 :invoke :cas [21 [29 4]] INFO [2019-12-17 12:31:09,210] jepsen worker 77 - jepsen.util 5837 :info :write [25 45] :RequestCanceledException INFO [2019-12-17 12:31:09,211] jepsen worker 77 - jepsen.util 5927 :invoke :cas [25 [15 24]] INFO [2019-12-17 12:31:09,215] jepsen worker 63 - jepsen.util 3663 :fail :cas [21 [29 4]] :ValueNotSwapFrom INFO [2019-12-17 12:31:09,215] jepsen worker 63 - jepsen.util 3663 :invoke :write [21 23] INFO [2019-12-17 12:31:09,225] jepsen worker 54 - jepsen.util 4824 :info :write [18 29] :RequestCanceledException INFO [2019-12-17 12:31:09,225] jepsen worker 51 - jepsen.util 4011 :info :write [17 13] :RequestCanceledException INFO [2019-12-17 12:31:09,226] jepsen worker 87 - jepsen.util 4227 :ok :read [29 36] INFO [2019-12-17 12:31:09,226] jepsen worker 87 - jepsen.util 4227 :invoke :cas [29 [5 38]] INFO [2019-12-17 12:31:09,226] jepsen worker 51 - jepsen.util 4101 :invoke :write [17 0] INFO [2019-12-17 12:31:09,226] jepsen worker 54 - jepsen.util 4914 :invoke :cas [18 [6 35]] INFO [2019-12-17 12:31:09,239] jepsen worker 20 - jepsen.util 4430 :info :cas [6 [44 1]] :RequestCanceledException INFO [2019-12-17 12:31:09,240] jepsen worker 20 - jepsen.util 4520 :invoke :cas [6 [9 12]] INFO [2019-12-17 12:31:09,255] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25408,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:31:09,255] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25408,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:31:09,256] jepsen worker 86 - jepsen.util 6026 :info :write [28 17] :RequestCanceledException INFO [2019-12-17 12:31:09,257] jepsen worker 86 - jepsen.util 6116 :invoke :write [28 21] INFO [2019-12-17 12:31:09,260] jepsen worker 63 - jepsen.util 3663 :ok :write [21 23] INFO [2019-12-17 12:31:09,260] jepsen worker 63 - jepsen.util 3663 :invoke :cas [21 [43 25]] INFO [2019-12-17 12:31:09,275] jepsen worker 14 - jepsen.util 4514 :fail :cas [4 [26 39]] :ValueNotSwapFrom INFO [2019-12-17 12:31:09,275] jepsen worker 14 - jepsen.util 4514 :invoke :cas [4 [41 40]] INFO [2019-12-17 12:31:09,281] jepsen worker 63 - jepsen.util 3663 :fail :cas [21 [43 25]] :ValueNotSwapFrom INFO [2019-12-17 12:31:09,282] jepsen worker 63 - jepsen.util 3663 :invoke :read [21 nil] INFO [2019-12-17 12:31:09,283] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25397,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:31:09,284] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25397,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:31:09,290] jepsen worker 89 - jepsen.util 4139 :fail :cas [29 [37 11]] :ValueNotSwapFrom INFO [2019-12-17 12:31:09,290] jepsen worker 89 - jepsen.util 4139 :invoke :read [29 nil] INFO [2019-12-17 12:31:09,296] jepsen worker 63 - jepsen.util 3663 :ok :read [21 23] INFO [2019-12-17 12:31:09,296] jepsen worker 63 - jepsen.util 3663 :invoke :write [21 48] INFO [2019-12-17 12:31:09,339] jepsen worker 63 - jepsen.util 3663 :ok :write [21 48] INFO [2019-12-17 12:31:09,339] jepsen worker 63 - jepsen.util 3663 :invoke :cas [21 [18 11]] INFO [2019-12-17 12:31:09,356] jepsen worker 49 - jepsen.util 5089 :info :cas [16 [15 42]] :RequestCanceledException INFO [2019-12-17 12:31:09,356] jepsen worker 24 - jepsen.util 5514 :info :cas [8 [0 36]] :RequestCanceledException INFO [2019-12-17 12:31:09,357] jepsen worker 49 - jepsen.util 5179 :invoke :cas [16 [44 19]] INFO [2019-12-17 12:31:09,357] jepsen worker 24 - jepsen.util 5604 :invoke :cas [8 [44 21]] INFO [2019-12-17 12:31:09,358] jepsen worker 63 - jepsen.util 3663 :fail :cas [21 [18 11]] :ValueNotSwapFrom INFO [2019-12-17 12:31:09,358] jepsen worker 63 - jepsen.util 3663 :invoke :cas [21 [41 19]] INFO [2019-12-17 12:31:09,364] jepsen worker 87 - jepsen.util 4227 :fail :cas [29 [5 38]] :ValueNotSwapFrom INFO [2019-12-17 12:31:09,364] jepsen worker 87 - jepsen.util 4227 :invoke :write [29 12] INFO [2019-12-17 12:31:09,369] jepsen worker 63 - jepsen.util 3663 :fail :cas [21 [41 19]] :ValueNotSwapFrom INFO [2019-12-17 12:31:09,369] jepsen worker 63 - jepsen.util 3663 :invoke :write [21 45] INFO [2019-12-17 12:31:09,401] jepsen worker 32 - jepsen.util 4802 :ok :write [10 8] INFO [2019-12-17 12:31:09,401] jepsen worker 74 - jepsen.util 4934 :ok :write [24 46] INFO [2019-12-17 12:31:09,401] jepsen worker 74 - jepsen.util 4934 :invoke :read [24 nil] INFO [2019-12-17 12:31:09,401] jepsen worker 32 - jepsen.util 4802 :invoke :write [10 25] INFO [2019-12-17 12:31:09,401] jepsen worker 69 - jepsen.util 4569 :ok :read [23 4] INFO [2019-12-17 12:31:09,402] jepsen worker 69 - jepsen.util 4569 :invoke :write [23 14] INFO [2019-12-17 12:31:09,403] jepsen worker 18 - jepsen.util 4428 :ok :write [6 14] INFO [2019-12-17 12:31:09,403] jepsen worker 18 - jepsen.util 4428 :invoke :read [6 nil] INFO [2019-12-17 12:31:09,405] jepsen worker 46 - jepsen.util 4546 :ok :write [15 21] INFO [2019-12-17 12:31:09,405] jepsen worker 46 - jepsen.util 4546 :invoke :cas [15 [37 6]] INFO [2019-12-17 12:31:09,423] jepsen worker 63 - jepsen.util 3663 :ok :write [21 45] INFO [2019-12-17 12:31:09,423] jepsen worker 63 - jepsen.util 3663 :invoke :read [21 nil] INFO [2019-12-17 12:31:09,432] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25440,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:31:09,432] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25440,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:31:09,432] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25442,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:31:09,432] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25442,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:31:09,457] jepsen worker 29 - jepsen.util 3719 :ok :read [9 30] INFO [2019-12-17 12:31:09,457] jepsen worker 29 - jepsen.util 3719 :invoke :read [9 nil] INFO [2019-12-17 12:31:09,480] jepsen worker 63 - jepsen.util 3663 :ok :read [21 45] INFO [2019-12-17 12:31:09,480] jepsen worker 63 - jepsen.util 3663 :invoke :cas [21 [28 32]] INFO [2019-12-17 12:31:09,487] jepsen worker 30 - jepsen.util 4530 :info :write [10 34] :RequestCanceledException INFO [2019-12-17 12:31:09,488] jepsen worker 30 - jepsen.util 4620 :invoke :cas [10 [15 23]] INFO [2019-12-17 12:31:09,490] jepsen worker 3 - jepsen.util 4593 :fail :cas [1 [5 40]] :ValueNotSwapFrom INFO [2019-12-17 12:31:09,490] jepsen worker 3 - jepsen.util 4593 :invoke :cas [1 [37 10]] INFO [2019-12-17 12:31:09,493] jepsen worker 18 - jepsen.util 4428 :ok :read [6 14] INFO [2019-12-17 12:31:09,493] jepsen worker 18 - jepsen.util 4428 :invoke :write [6 48] INFO [2019-12-17 12:31:09,495] jepsen worker 55 - jepsen.util 5455 :fail :cas [18 [22 13]] :ValueNotSwapFrom INFO [2019-12-17 12:31:09,495] jepsen worker 55 - jepsen.util 5455 :invoke :read [18 nil] INFO [2019-12-17 12:31:09,495] jepsen worker 11 - jepsen.util 4691 :fail :cas [3 [45 46]] :ValueNotSwapFrom INFO [2019-12-17 12:31:09,496] jepsen worker 11 - jepsen.util 4691 :invoke :write [3 34] INFO [2019-12-17 12:31:09,514] jepsen worker 89 - jepsen.util 4139 :ok :read [29 36] INFO [2019-12-17 12:31:09,514] jepsen worker 89 - jepsen.util 4139 :invoke :cas [29 [46 46]] INFO [2019-12-17 12:31:09,528] jepsen worker 53 - jepsen.util 4463 :fail :cas [17 [27 1]] :ValueNotSwapFrom INFO [2019-12-17 12:31:09,528] jepsen worker 53 - jepsen.util 4463 :invoke :cas [17 [14 11]] INFO [2019-12-17 12:31:09,538] jepsen worker 34 - jepsen.util 5434 :info :write [11 6] :RequestCanceledException INFO [2019-12-17 12:31:09,539] jepsen worker 34 - jepsen.util 5524 :invoke :read [11 nil] INFO [2019-12-17 12:31:09,542] jepsen worker 54 - jepsen.util 4914 :fail :cas [18 [6 35]] :ValueNotSwapFrom INFO [2019-12-17 12:31:09,543] jepsen worker 54 - jepsen.util 4914 :invoke :read [18 nil] INFO [2019-12-17 12:31:09,566] jepsen worker 74 - jepsen.util 4934 :ok :read [24 46] INFO [2019-12-17 12:31:09,566] jepsen worker 74 - jepsen.util 4934 :invoke :read [24 nil] INFO [2019-12-17 12:31:09,569] jepsen worker 63 - jepsen.util 3663 :fail :cas [21 [28 32]] :ValueNotSwapFrom INFO [2019-12-17 12:31:09,570] jepsen worker 63 - jepsen.util 3663 :invoke :cas [21 [0 49]] INFO [2019-12-17 12:31:09,576] jepsen worker 28 - jepsen.util 3358 :ok :write [9 26] INFO [2019-12-17 12:31:09,576] jepsen worker 28 - jepsen.util 3358 :invoke :write [9 11] INFO [2019-12-17 12:31:09,591] jepsen worker 63 - jepsen.util 3663 :fail :cas [21 [0 49]] :ValueNotSwapFrom INFO [2019-12-17 12:31:09,592] jepsen worker 63 - jepsen.util 3663 :invoke :write [21 24] INFO [2019-12-17 12:31:09,592] jepsen worker 30 - jepsen.util 4620 :fail :cas [10 [15 23]] :ValueNotSwapFrom INFO [2019-12-17 12:31:09,592] jepsen worker 30 - jepsen.util 4620 :invoke :write [10 13] INFO [2019-12-17 12:31:09,620] jepsen worker 13 - jepsen.util 4153 :fail :cas [4 [20 33]] :ValueNotSwapFrom INFO [2019-12-17 12:31:09,620] jepsen worker 13 - jepsen.util 4153 :invoke :read [4 nil] INFO [2019-12-17 12:31:09,633] jepsen worker 53 - jepsen.util 4463 :fail :cas [17 [14 11]] :ValueNotSwapFrom INFO [2019-12-17 12:31:09,633] jepsen worker 53 - jepsen.util 4463 :invoke :read [17 nil] INFO [2019-12-17 12:31:09,652] jepsen worker 32 - jepsen.util 4802 :ok :write [10 25] INFO [2019-12-17 12:31:09,652] jepsen worker 32 - jepsen.util 4802 :invoke :read [10 nil] INFO [2019-12-17 12:31:09,653] jepsen worker 14 - jepsen.util 4514 :fail :cas [4 [41 40]] :ValueNotSwapFrom INFO [2019-12-17 12:31:09,653] jepsen worker 14 - jepsen.util 4514 :invoke :cas [4 [10 4]] INFO [2019-12-17 12:31:09,662] jepsen worker 28 - jepsen.util 3358 :ok :write [9 11] INFO [2019-12-17 12:31:09,662] jepsen worker 28 - jepsen.util 3358 :invoke :write [9 13] INFO [2019-12-17 12:31:09,669] jepsen worker 29 - jepsen.util 3719 :ok :read [9 26] INFO [2019-12-17 12:31:09,669] jepsen worker 29 - jepsen.util 3719 :invoke :cas [9 [28 19]] INFO [2019-12-17 12:31:09,688] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25462,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:31:09,688] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25462,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:31:09,688] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25465,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:31:09,688] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25465,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:31:09,688] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25466,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:31:09,688] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25466,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:31:09,698] jepsen worker 32 - jepsen.util 4802 :ok :read [10 25] INFO [2019-12-17 12:31:09,698] jepsen worker 32 - jepsen.util 4802 :invoke :write [10 2] INFO [2019-12-17 12:31:09,725] jepsen worker 68 - jepsen.util 5378 :info :cas [22 [2 29]] :RequestCanceledException INFO [2019-12-17 12:31:09,725] jepsen worker 23 - jepsen.util 5693 :info :write [7 32] :RequestCanceledException INFO [2019-12-17 12:31:09,726] jepsen worker 68 - jepsen.util 5468 :invoke :read [22 nil] INFO [2019-12-17 12:31:09,726] jepsen worker 23 - jepsen.util 5783 :invoke :cas [7 [37 28]] INFO [2019-12-17 12:31:09,733] jepsen worker 3 - jepsen.util 4593 :fail :cas [1 [37 10]] :ValueNotSwapFrom INFO [2019-12-17 12:31:09,733] jepsen worker 3 - jepsen.util 4593 :invoke :cas [1 [45 44]] INFO [2019-12-17 12:31:09,734] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:31:09,734] jepsen nemesis - couchbase.util Rebalance check count: 188 INFO [2019-12-17 12:31:09,735] jepsen nemesis - couchbase.util Rebalance stuck count: 169 INFO [2019-12-17 12:31:09,760] jepsen worker 63 - jepsen.util 3663 :ok :write [21 24] INFO [2019-12-17 12:31:09,760] jepsen worker 63 - jepsen.util 3663 :invoke :read [21 nil] INFO [2019-12-17 12:31:09,774] jepsen worker 29 - jepsen.util 3719 :fail :cas [9 [28 19]] :ValueNotSwapFrom INFO [2019-12-17 12:31:09,774] jepsen worker 29 - jepsen.util 3719 :invoke :cas [9 [42 19]] INFO [2019-12-17 12:31:09,777] jepsen worker 18 - jepsen.util 4428 :ok :write [6 48] INFO [2019-12-17 12:31:09,777] jepsen worker 18 - jepsen.util 4428 :invoke :cas [6 [14 24]] INFO [2019-12-17 12:31:09,783] jepsen worker 3 - jepsen.util 4593 :fail :cas [1 [45 44]] :ValueNotSwapFrom INFO [2019-12-17 12:31:09,783] jepsen worker 3 - jepsen.util 4593 :invoke :write [1 13] INFO [2019-12-17 12:31:09,787] jepsen worker 28 - jepsen.util 3358 :ok :write [9 13] INFO [2019-12-17 12:31:09,787] jepsen worker 28 - jepsen.util 3358 :invoke :read [9 nil] INFO [2019-12-17 12:31:09,788] jepsen worker 89 - jepsen.util 4139 :fail :cas [29 [46 46]] :ValueNotSwapFrom INFO [2019-12-17 12:31:09,788] jepsen worker 89 - jepsen.util 4139 :invoke :cas [29 [26 19]] INFO [2019-12-17 12:31:09,797] jepsen worker 18 - jepsen.util 4428 :fail :cas [6 [14 24]] :ValueNotSwapFrom INFO [2019-12-17 12:31:09,797] jepsen worker 18 - jepsen.util 4428 :invoke :write [6 46] INFO [2019-12-17 12:31:09,833] jepsen worker 28 - jepsen.util 3358 :ok :read [9 13] INFO [2019-12-17 12:31:09,834] jepsen worker 28 - jepsen.util 3358 :invoke :write [9 17] INFO [2019-12-17 12:31:09,842] jepsen worker 32 - jepsen.util 4802 :ok :write [10 2] INFO [2019-12-17 12:31:09,842] jepsen worker 32 - jepsen.util 4802 :invoke :read [10 nil] INFO [2019-12-17 12:31:09,855] jepsen worker 54 - jepsen.util 4914 :ok :read [18 3] INFO [2019-12-17 12:31:09,855] jepsen worker 54 - jepsen.util 4914 :invoke :write [18 40] INFO [2019-12-17 12:31:09,855] jepsen worker 36 - jepsen.util 4446 :info :cas [12 [31 37]] :RequestCanceledException INFO [2019-12-17 12:31:09,863] jepsen worker 36 - jepsen.util 4536 :invoke :cas [12 [42 35]] INFO [2019-12-17 12:31:09,892] jepsen worker 44 - jepsen.util 6344 :info :write [14 10] :RequestCanceledException INFO [2019-12-17 12:31:09,894] jepsen worker 44 - jepsen.util 6434 :invoke :read [14 nil] INFO [2019-12-17 12:31:09,904] jepsen worker 14 - jepsen.util 4514 :fail :cas [4 [10 4]] :ValueNotSwapFrom INFO [2019-12-17 12:31:09,904] jepsen worker 14 - jepsen.util 4514 :invoke :read [4 nil] INFO [2019-12-17 12:31:09,909] jepsen worker 59 - jepsen.util 5909 :info :cas [19 [12 8]] :RequestCanceledException INFO [2019-12-17 12:31:09,911] jepsen worker 59 - jepsen.util 5999 :invoke :write [19 4] INFO [2019-12-17 12:31:09,913] jepsen worker 29 - jepsen.util 3719 :fail :cas [9 [42 19]] :ValueNotSwapFrom INFO [2019-12-17 12:31:09,913] jepsen worker 29 - jepsen.util 3719 :invoke :read [9 nil] INFO [2019-12-17 12:31:09,919] jepsen worker 32 - jepsen.util 4802 :ok :read [10 2] INFO [2019-12-17 12:31:09,919] jepsen worker 32 - jepsen.util 4802 :invoke :write [10 30] INFO [2019-12-17 12:31:09,925] jepsen worker 6 - jepsen.util 6396 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:31:09,925] jepsen worker 43 - jepsen.util 5443 :info :write [14 12] :RequestCanceledException INFO [2019-12-17 12:31:09,925] jepsen worker 19 - jepsen.util 4429 :info :write [6 45] :RequestCanceledException INFO [2019-12-17 12:31:09,925] jepsen worker 6 - jepsen.util 6396 :invoke :cas [2 [10 47]] INFO [2019-12-17 12:31:09,926] jepsen worker 43 - jepsen.util 5533 :invoke :cas [14 [34 41]] INFO [2019-12-17 12:31:09,926] jepsen worker 19 - jepsen.util 4519 :invoke :cas [6 [14 16]] INFO [2019-12-17 12:31:09,938] jepsen worker 84 - jepsen.util 5394 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:31:09,938] jepsen worker 84 - jepsen.util 5394 :invoke :cas [28 [20 48]] INFO [2019-12-17 12:31:09,945] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25485,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:31:09,945] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25485,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:31:09,945] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25488,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:31:09,945] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25488,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:31:09,945] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25489,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:31:09,945] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25489,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:31:09,956] jepsen worker 85 - jepsen.util 6475 :info :write [28 3] :RequestCanceledException INFO [2019-12-17 12:31:09,957] jepsen worker 85 - jepsen.util 6565 :invoke :read [28 nil] INFO [2019-12-17 12:31:09,968] jepsen worker 28 - jepsen.util 3358 :ok :write [9 17] INFO [2019-12-17 12:31:09,968] jepsen worker 28 - jepsen.util 3358 :invoke :cas [9 [18 41]] INFO [2019-12-17 12:31:09,972] jepsen worker 29 - jepsen.util 3719 :ok :read [9 13] INFO [2019-12-17 12:31:09,972] jepsen worker 29 - jepsen.util 3719 :invoke :cas [9 [14 13]] INFO [2019-12-17 12:31:09,989] jepsen worker 19 - jepsen.util 4519 :fail :cas [6 [14 16]] :ValueNotSwapFrom INFO [2019-12-17 12:31:09,990] jepsen worker 19 - jepsen.util 4519 :invoke :cas [6 [22 41]] INFO [2019-12-17 12:31:09,991] jepsen worker 53 - jepsen.util 4463 :ok :read [17 48] INFO [2019-12-17 12:31:09,991] jepsen worker 53 - jepsen.util 4463 :invoke :read [17 nil] INFO [2019-12-17 12:31:10,000] jepsen worker 36 - jepsen.util 4536 :fail :cas [12 [42 35]] :ValueNotSwapFrom INFO [2019-12-17 12:31:10,001] jepsen worker 36 - jepsen.util 4536 :invoke :read [12 nil] INFO [2019-12-17 12:31:10,009] jepsen worker 66 - jepsen.util 5466 :info :cas [22 [12 13]] :RequestCanceledException INFO [2019-12-17 12:31:10,009] jepsen worker 9 - jepsen.util 5049 :info :cas [3 [40 32]] :RequestCanceledException INFO [2019-12-17 12:31:10,010] jepsen worker 9 - jepsen.util 5139 :invoke :write [3 1] INFO [2019-12-17 12:31:10,012] jepsen worker 28 - jepsen.util 3358 :fail :cas [9 [18 41]] :ValueNotSwapFrom INFO [2019-12-17 12:31:10,013] jepsen worker 28 - jepsen.util 3358 :invoke :cas [9 [4 7]] INFO [2019-12-17 12:31:10,015] jepsen worker 66 - jepsen.util 5556 :invoke :read [22 nil] INFO [2019-12-17 12:31:10,023] jepsen worker 25 - jepsen.util 5965 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:31:10,023] jepsen worker 7 - jepsen.util 5677 :info :cas [2 [49 6]] :RequestCanceledException INFO [2019-12-17 12:31:10,023] jepsen worker 25 - jepsen.util 5965 :invoke :write [8 23] INFO [2019-12-17 12:31:10,024] jepsen worker 7 - jepsen.util 5767 :invoke :cas [2 [28 41]] INFO [2019-12-17 12:31:10,039] jepsen worker 82 - jepsen.util 6742 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:31:10,039] jepsen worker 19 - jepsen.util 4519 :fail :cas [6 [22 41]] :ValueNotSwapFrom INFO [2019-12-17 12:31:10,039] jepsen worker 82 - jepsen.util 6742 :invoke :read [27 nil] INFO [2019-12-17 12:31:10,039] jepsen worker 19 - jepsen.util 4519 :invoke :cas [6 [14 13]] INFO [2019-12-17 12:31:10,048] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25509,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:31:10,048] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25509,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:31:10,048] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25511,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:31:10,048] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25511,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:31:10,056] jepsen worker 47 - jepsen.util 4907 :fail :read [15 nil] :RequestTimeoutException INFO [2019-12-17 12:31:10,056] jepsen worker 73 - jepsen.util 4303 :fail :read [24 nil] :RequestTimeoutException INFO [2019-12-17 12:31:10,056] jepsen worker 47 - jepsen.util 4907 :invoke :write [15 42] INFO [2019-12-17 12:31:10,056] jepsen worker 73 - jepsen.util 4303 :invoke :write [24 0] INFO [2019-12-17 12:31:10,079] jepsen worker 28 - jepsen.util 3358 :fail :cas [9 [4 7]] :ValueNotSwapFrom INFO [2019-12-17 12:31:10,080] jepsen worker 28 - jepsen.util 3358 :invoke :cas [9 [14 15]] INFO [2019-12-17 12:31:10,091] jepsen worker 79 - jepsen.util 4309 :info :write [26 33] :RequestCanceledException INFO [2019-12-17 12:31:10,092] jepsen worker 79 - jepsen.util 4399 :invoke :read [26 nil] INFO [2019-12-17 12:31:10,102] jepsen worker 19 - jepsen.util 4519 :fail :cas [6 [14 13]] :ValueNotSwapFrom INFO [2019-12-17 12:31:10,103] jepsen worker 19 - jepsen.util 4519 :invoke :write [6 8] INFO [2019-12-17 12:31:10,109] jepsen worker 72 - jepsen.util 4572 :fail :read [24 nil] :RequestTimeoutException INFO [2019-12-17 12:31:10,109] jepsen worker 72 - jepsen.util 4572 :invoke :cas [24 [24 49]] INFO [2019-12-17 12:31:10,112] jepsen worker 18 - jepsen.util 4428 :ok :write [6 46] INFO [2019-12-17 12:31:10,112] jepsen worker 11 - jepsen.util 4691 :ok :write [3 34] INFO [2019-12-17 12:31:10,112] jepsen worker 18 - jepsen.util 4428 :invoke :read [6 nil] INFO [2019-12-17 12:31:10,112] jepsen worker 11 - jepsen.util 4691 :invoke :cas [3 [41 23]] INFO [2019-12-17 12:31:10,125] jepsen worker 62 - jepsen.util 5372 :info :cas [20 [15 29]] :RequestCanceledException INFO [2019-12-17 12:31:10,125] jepsen worker 39 - jepsen.util 5349 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:31:10,125] jepsen worker 39 - jepsen.util 5349 :invoke :cas [13 [29 41]] INFO [2019-12-17 12:31:10,126] jepsen worker 62 - jepsen.util 5462 :invoke :write [20 33] INFO [2019-12-17 12:31:10,128] jepsen worker 72 - jepsen.util 4572 :fail :cas [24 [24 49]] :ValueNotSwapFrom INFO [2019-12-17 12:31:10,128] jepsen worker 72 - jepsen.util 4572 :invoke :cas [24 [31 14]] INFO [2019-12-17 12:31:10,137] jepsen worker 11 - jepsen.util 4691 :fail :cas [3 [41 23]] :ValueNotSwapFrom INFO [2019-12-17 12:31:10,137] jepsen worker 11 - jepsen.util 4691 :invoke :write [3 5] INFO [2019-12-17 12:31:10,140] jepsen worker 18 - jepsen.util 4428 :ok :read [6 46] INFO [2019-12-17 12:31:10,141] jepsen worker 18 - jepsen.util 4428 :invoke :cas [6 [7 37]] INFO [2019-12-17 12:31:10,143] jepsen worker 55 - jepsen.util 5455 :ok :read [18 3] INFO [2019-12-17 12:31:10,143] jepsen worker 55 - jepsen.util 5455 :invoke :read [18 nil] INFO [2019-12-17 12:31:10,147] jepsen worker 72 - jepsen.util 4572 :fail :cas [24 [31 14]] :ValueNotSwapFrom INFO [2019-12-17 12:31:10,147] jepsen worker 72 - jepsen.util 4572 :invoke :cas [24 [2 47]] INFO [2019-12-17 12:31:10,161] jepsen worker 13 - jepsen.util 4153 :ok :read [4 40] INFO [2019-12-17 12:31:10,161] jepsen worker 13 - jepsen.util 4153 :invoke :cas [4 [17 42]] INFO [2019-12-17 12:31:10,176] jepsen worker 18 - jepsen.util 4428 :fail :cas [6 [7 37]] :ValueNotSwapFrom INFO [2019-12-17 12:31:10,176] jepsen worker 18 - jepsen.util 4428 :invoke :cas [6 [37 9]] INFO [2019-12-17 12:31:10,191] jepsen worker 35 - jepsen.util 5885 :info :write [11 2] :RequestCanceledException INFO [2019-12-17 12:31:10,192] jepsen worker 35 - jepsen.util 5975 :invoke :write [11 35] INFO [2019-12-17 12:31:10,194] jepsen worker 72 - jepsen.util 4572 :fail :cas [24 [2 47]] :ValueNotSwapFrom INFO [2019-12-17 12:31:10,194] jepsen worker 72 - jepsen.util 4572 :invoke :write [24 9] INFO [2019-12-17 12:31:10,210] jepsen worker 83 - jepsen.util 6113 :info :write [27 34] :RequestCanceledException INFO [2019-12-17 12:31:10,211] jepsen worker 83 - jepsen.util 6203 :invoke :write [27 49] INFO [2019-12-17 12:31:10,216] jepsen worker 18 - jepsen.util 4428 :fail :cas [6 [37 9]] :ValueNotSwapFrom INFO [2019-12-17 12:31:10,217] jepsen worker 18 - jepsen.util 4428 :invoke :write [6 35] INFO [2019-12-17 12:31:10,225] jepsen worker 52 - jepsen.util 4642 :info :write [17 26] :RequestCanceledException INFO [2019-12-17 12:31:10,226] jepsen worker 52 - jepsen.util 4732 :invoke :read [17 nil] INFO [2019-12-17 12:31:10,236] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25546,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:31:10,236] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25546,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:31:10,236] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25549,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:31:10,236] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25549,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:31:10,236] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25550,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:31:10,236] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25550,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:31:10,247] jepsen worker 14 - jepsen.util 4514 :ok :read [4 40] INFO [2019-12-17 12:31:10,247] jepsen worker 14 - jepsen.util 4514 :invoke :write [4 40] INFO [2019-12-17 12:31:10,256] jepsen worker 58 - jepsen.util 5908 :info :cas [19 [2 10]] :RequestCanceledException INFO [2019-12-17 12:31:10,257] jepsen worker 58 - jepsen.util 5998 :invoke :write [19 14] INFO [2019-12-17 12:31:10,267] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25537,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:31:10,268] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25537,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:31:10,268] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25538,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:31:10,268] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25538,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:31:10,268] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25539,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:31:10,268] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25539,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:31:10,278] jepsen worker 13 - jepsen.util 4153 :fail :cas [4 [17 42]] :ValueNotSwapFrom INFO [2019-12-17 12:31:10,278] jepsen worker 13 - jepsen.util 4153 :invoke :read [4 nil] INFO [2019-12-17 12:31:10,291] jepsen worker 2 - jepsen.util 5582 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:31:10,291] jepsen worker 38 - jepsen.util 4358 :info :write [12 49] :RequestCanceledException INFO [2019-12-17 12:31:10,291] jepsen worker 2 - jepsen.util 5582 :invoke :read [0 nil] INFO [2019-12-17 12:31:10,292] jepsen worker 38 - jepsen.util 4448 :invoke :write [12 5] INFO [2019-12-17 12:31:10,324] jepsen worker 50 - jepsen.util 6080 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:31:10,324] jepsen worker 40 - jepsen.util 6520 :info :write [13 41] :RequestCanceledException INFO [2019-12-17 12:31:10,325] jepsen worker 50 - jepsen.util 6080 :invoke :cas [16 [36 15]] INFO [2019-12-17 12:31:10,327] jepsen worker 40 - jepsen.util 6610 :invoke :cas [13 [5 15]] INFO [2019-12-17 12:31:10,339] jepsen worker 0 - jepsen.util 6390 :info :write [0 44] :RequestCanceledException INFO [2019-12-17 12:31:10,339] jepsen worker 60 - jepsen.util 6450 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:31:10,339] jepsen worker 60 - jepsen.util 6450 :invoke :read [20 nil] INFO [2019-12-17 12:31:10,340] jepsen worker 0 - jepsen.util 6480 :invoke :write [0 15] INFO [2019-12-17 12:31:10,404] jepsen worker 89 - jepsen.util 4139 :fail :cas [29 [26 19]] :ValueNotSwapFrom INFO [2019-12-17 12:31:10,404] jepsen worker 89 - jepsen.util 4139 :invoke :write [29 27] INFO [2019-12-17 12:31:10,409] jepsen worker 57 - jepsen.util 5637 :info :write [19 38] :RequestCanceledException INFO [2019-12-17 12:31:10,409] jepsen worker 67 - jepsen.util 6007 :info :write [22 20] :RequestCanceledException INFO [2019-12-17 12:31:10,409] jepsen worker 65 - jepsen.util 3755 :info :write [21 45] :RequestCanceledException INFO [2019-12-17 12:31:10,411] jepsen worker 57 - jepsen.util 5727 :invoke :read [19 nil] INFO [2019-12-17 12:31:10,412] jepsen worker 67 - jepsen.util 6097 :invoke :write [22 27] INFO [2019-12-17 12:31:10,412] jepsen worker 65 - jepsen.util 3845 :invoke :cas [21 [42 35]] INFO [2019-12-17 12:31:10,425] jepsen worker 26 - jepsen.util 5966 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:31:10,425] jepsen worker 26 - jepsen.util 5966 :invoke :cas [8 [28 19]] INFO [2019-12-17 12:31:10,439] jepsen worker 45 - jepsen.util 4545 :info :write [15 19] :RequestCanceledException INFO [2019-12-17 12:31:10,440] jepsen worker 45 - jepsen.util 4635 :invoke :read [15 nil] INFO [2019-12-17 12:31:10,440] jepsen worker 65 - jepsen.util 3845 :fail :cas [21 [42 35]] :ValueNotSwapFrom INFO [2019-12-17 12:31:10,441] jepsen worker 65 - jepsen.util 3845 :invoke :write [21 9] INFO [2019-12-17 12:31:10,456] jepsen worker 22 - jepsen.util 5872 :info :write [7 42] :RequestCanceledException INFO [2019-12-17 12:31:10,458] jepsen worker 22 - jepsen.util 5962 :invoke :write [7 8] INFO [2019-12-17 12:31:10,477] jepsen worker 36 - jepsen.util 4536 :ok :read [12 34] INFO [2019-12-17 12:31:10,478] jepsen worker 36 - jepsen.util 4536 :invoke :read [12 nil] INFO [2019-12-17 12:31:10,491] jepsen worker 48 - jepsen.util 5358 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:31:10,491] jepsen worker 78 - jepsen.util 4578 :info :write [26 9] :RequestCanceledException INFO [2019-12-17 12:31:10,492] jepsen worker 48 - jepsen.util 5358 :invoke :cas [16 [4 45]] INFO [2019-12-17 12:31:10,493] jepsen worker 78 - jepsen.util 4668 :invoke :read [26 nil] INFO [2019-12-17 12:31:10,507] jepsen worker 65 - jepsen.util 3845 :ok :write [21 9] INFO [2019-12-17 12:31:10,507] jepsen worker 65 - jepsen.util 3845 :invoke :read [21 nil] INFO [2019-12-17 12:31:10,508] jepsen worker 52 - jepsen.util 4732 :ok :read [17 48] INFO [2019-12-17 12:31:10,508] jepsen worker 52 - jepsen.util 4732 :invoke :write [17 18] INFO [2019-12-17 12:31:10,509] jepsen worker 17 - jepsen.util 5597 :info :cas [5 [34 23]] :RequestCanceledException INFO [2019-12-17 12:31:10,512] jepsen worker 17 - jepsen.util 5687 :invoke :cas [5 [9 40]] INFO [2019-12-17 12:31:10,517] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25569,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:31:10,517] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25569,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:31:10,517] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25572,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:31:10,517] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25572,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:31:10,529] jepsen worker 53 - jepsen.util 4463 :ok :read [17 48] INFO [2019-12-17 12:31:10,529] jepsen worker 53 - jepsen.util 4463 :invoke :read [17 nil] INFO [2019-12-17 12:31:10,538] jepsen worker 75 - jepsen.util 5925 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:31:10,538] jepsen worker 75 - jepsen.util 5925 :invoke :cas [25 [48 47]] INFO [2019-12-17 12:31:10,543] jepsen worker 65 - jepsen.util 3845 :ok :read [21 9] INFO [2019-12-17 12:31:10,543] jepsen worker 65 - jepsen.util 3845 :invoke :read [21 nil] INFO [2019-12-17 12:31:10,570] jepsen worker 65 - jepsen.util 3845 :ok :read [21 9] INFO [2019-12-17 12:31:10,570] jepsen worker 65 - jepsen.util 3845 :invoke :write [21 14] INFO [2019-12-17 12:31:10,610] jepsen worker 65 - jepsen.util 3845 :ok :write [21 14] INFO [2019-12-17 12:31:10,611] jepsen worker 65 - jepsen.util 3845 :invoke :write [21 28] INFO [2019-12-17 12:31:10,624] jepsen worker 31 - jepsen.util 4081 :info :write [10 39] :RequestCanceledException INFO [2019-12-17 12:31:10,624] jepsen worker 41 - jepsen.util 6161 :info :cas [13 [5 42]] :RequestCanceledException INFO [2019-12-17 12:31:10,625] jepsen worker 42 - jepsen.util 6522 :info :cas [14 [43 16]] :RequestCanceledException INFO [2019-12-17 12:31:10,627] jepsen worker 42 - jepsen.util 6612 :invoke :write [14 18] INFO [2019-12-17 12:31:10,627] jepsen worker 31 - jepsen.util 4171 :invoke :write [10 27] INFO [2019-12-17 12:31:10,629] jepsen worker 41 - jepsen.util 6251 :invoke :cas [13 [49 1]] INFO [2019-12-17 12:31:10,652] jepsen worker 55 - jepsen.util 5455 :ok :read [18 3] INFO [2019-12-17 12:31:10,652] jepsen worker 55 - jepsen.util 5455 :invoke :cas [18 [49 47]] INFO [2019-12-17 12:31:10,656] jepsen worker 10 - jepsen.util 4240 :info :write [3 32] :RequestCanceledException INFO [2019-12-17 12:31:10,656] jepsen worker 64 - jepsen.util 3484 :fail :read [21 nil] :RequestTimeoutException INFO [2019-12-17 12:31:10,656] jepsen worker 64 - jepsen.util 3484 :invoke :read [21 nil] INFO [2019-12-17 12:31:10,658] jepsen worker 10 - jepsen.util 4330 :invoke :read [3 nil] INFO [2019-12-17 12:31:10,662] jepsen worker 65 - jepsen.util 3845 :ok :write [21 28] INFO [2019-12-17 12:31:10,663] jepsen worker 65 - jepsen.util 3845 :invoke :write [21 24] INFO [2019-12-17 12:31:10,674] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25592,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:31:10,674] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25592,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:31:10,674] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25596,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:31:10,674] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25596,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:31:10,691] jepsen worker 33 - jepsen.util 5793 :info :write [11 44] :RequestCanceledException INFO [2019-12-17 12:31:10,693] jepsen worker 33 - jepsen.util 5883 :invoke :read [11 nil] INFO [2019-12-17 12:31:10,709] jepsen worker 5 - jepsen.util 4055 :info :write [1 18] :RequestCanceledException INFO [2019-12-17 12:31:10,712] jepsen worker 5 - jepsen.util 4145 :invoke :cas [1 [14 20]] INFO [2019-12-17 12:31:10,718] jepsen worker 65 - jepsen.util 3845 :ok :write [21 24] INFO [2019-12-17 12:31:10,719] jepsen worker 65 - jepsen.util 3845 :invoke :write [21 17] INFO [2019-12-17 12:31:10,723] jepsen worker 80 - jepsen.util 4400 :info :write [26 24] :RequestCanceledException INFO [2019-12-17 12:31:10,723] jepsen worker 76 - jepsen.util 5926 :info :cas [25 [37 27]] :RequestCanceledException INFO [2019-12-17 12:31:10,723] jepsen worker 8 - jepsen.util 5138 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:31:10,723] jepsen worker 8 - jepsen.util 5138 :invoke :write [2 44] INFO [2019-12-17 12:31:10,724] jepsen worker 80 - jepsen.util 4490 :invoke :cas [26 [31 45]] INFO [2019-12-17 12:31:10,724] jepsen worker 79 - jepsen.util 4399 :ok :read [26 4] INFO [2019-12-17 12:31:10,725] jepsen worker 79 - jepsen.util 4399 :invoke :write [26 15] INFO [2019-12-17 12:31:10,727] jepsen worker 76 - jepsen.util 6016 :invoke :write [25 32] INFO [2019-12-17 12:31:10,739] jepsen worker 1 - jepsen.util 5401 :info :write [0 9] :RequestCanceledException INFO [2019-12-17 12:31:10,739] jepsen worker 12 - jepsen.util 4602 :info :write [4 26] :RequestCanceledException INFO [2019-12-17 12:31:10,740] jepsen worker 12 - jepsen.util 4692 :invoke :read [4 nil] INFO [2019-12-17 12:31:10,741] jepsen worker 1 - jepsen.util 5491 :invoke :write [0 44] INFO [2019-12-17 12:31:10,748] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:31:10,748] jepsen nemesis - couchbase.util Rebalance check count: 189 INFO [2019-12-17 12:31:10,748] jepsen nemesis - couchbase.util Rebalance stuck count: 170 INFO [2019-12-17 12:31:10,776] jepsen worker 65 - jepsen.util 3845 :ok :write [21 17] INFO [2019-12-17 12:31:10,776] jepsen worker 65 - jepsen.util 3845 :invoke :write [21 44] INFO [2019-12-17 12:31:10,791] jepsen worker 16 - jepsen.util 6586 :info :write [5 5] :RequestCanceledException INFO [2019-12-17 12:31:10,792] jepsen worker 16 - jepsen.util 6676 :invoke :write [5 4] INFO [2019-12-17 12:31:10,819] jepsen worker 65 - jepsen.util 3845 :ok :write [21 44] INFO [2019-12-17 12:31:10,819] jepsen worker 65 - jepsen.util 3845 :invoke :read [21 nil] INFO [2019-12-17 12:31:10,824] jepsen worker 81 - jepsen.util 5841 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:31:10,824] jepsen worker 81 - jepsen.util 5841 :invoke :cas [27 [21 9]] INFO [2019-12-17 12:31:10,843] jepsen worker 13 - jepsen.util 4153 :ok :read [4 40] INFO [2019-12-17 12:31:10,843] jepsen worker 13 - jepsen.util 4153 :invoke :write [4 19] INFO [2019-12-17 12:31:10,853] jepsen worker 36 - jepsen.util 4536 :ok :read [12 34] INFO [2019-12-17 12:31:10,853] jepsen worker 36 - jepsen.util 4536 :invoke :cas [12 [0 11]] INFO [2019-12-17 12:31:10,856] jepsen worker 65 - jepsen.util 3845 :ok :read [21 44] INFO [2019-12-17 12:31:10,856] jepsen worker 65 - jepsen.util 3845 :invoke :write [21 29] INFO [2019-12-17 12:31:10,891] jepsen worker 15 - jepsen.util 4875 :info :cas [5 [27 28]] :RequestCanceledException INFO [2019-12-17 12:31:10,891] jepsen worker 71 - jepsen.util 4661 :info :write [23 4] :RequestCanceledException INFO [2019-12-17 12:31:10,893] jepsen worker 71 - jepsen.util 4751 :invoke :write [23 16] INFO [2019-12-17 12:31:10,894] jepsen worker 15 - jepsen.util 4965 :invoke :cas [5 [27 17]] INFO [2019-12-17 12:31:10,895] jepsen worker 80 - jepsen.util 4490 :fail :cas [26 [31 45]] :ValueNotSwapFrom INFO [2019-12-17 12:31:10,895] jepsen worker 80 - jepsen.util 4490 :invoke :read [26 nil] INFO [2019-12-17 12:31:10,905] jepsen worker 65 - jepsen.util 3845 :ok :write [21 29] INFO [2019-12-17 12:31:10,905] jepsen worker 65 - jepsen.util 3845 :invoke :cas [21 [11 25]] INFO [2019-12-17 12:31:10,919] jepsen worker 32 - jepsen.util 4802 :ok :write [10 30] INFO [2019-12-17 12:31:10,919] jepsen worker 32 - jepsen.util 4802 :invoke :cas [10 [5 38]] INFO [2019-12-17 12:31:10,926] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25619,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:31:10,927] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25619,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:31:10,927] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25621,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:31:10,927] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25621,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:31:10,930] jepsen worker 65 - jepsen.util 3845 :fail :cas [21 [11 25]] :ValueNotSwapFrom INFO [2019-12-17 12:31:10,930] jepsen worker 65 - jepsen.util 3845 :invoke :cas [21 [16 19]] INFO [2019-12-17 12:31:10,932] jepsen worker 47 - jepsen.util 4907 :ok :write [15 42] INFO [2019-12-17 12:31:10,932] jepsen worker 47 - jepsen.util 4907 :invoke :read [15 nil] INFO [2019-12-17 12:31:10,937] jepsen worker 11 - jepsen.util 4691 :ok :write [3 5] INFO [2019-12-17 12:31:10,937] jepsen worker 11 - jepsen.util 4691 :invoke :write [3 20] INFO [2019-12-17 12:31:10,940] jepsen worker 72 - jepsen.util 4572 :ok :write [24 9] INFO [2019-12-17 12:31:10,940] jepsen worker 72 - jepsen.util 4572 :invoke :read [24 nil] INFO [2019-12-17 12:31:10,965] jepsen worker 80 - jepsen.util 4490 :ok :read [26 4] INFO [2019-12-17 12:31:10,965] jepsen worker 80 - jepsen.util 4490 :invoke :read [26 nil] INFO [2019-12-17 12:31:10,990] jepsen worker 29 - jepsen.util 3719 :fail :cas [9 [14 13]] :ValueNotSwapFrom INFO [2019-12-17 12:31:10,990] jepsen worker 29 - jepsen.util 3719 :invoke :cas [9 [13 41]] INFO [2019-12-17 12:31:11,005] jepsen worker 80 - jepsen.util 4490 :ok :read [26 4] INFO [2019-12-17 12:31:11,005] jepsen worker 80 - jepsen.util 4490 :invoke :cas [26 [0 26]] INFO [2019-12-17 12:31:11,008] jepsen worker 45 - jepsen.util 4635 :ok :read [15 42] INFO [2019-12-17 12:31:11,008] jepsen worker 45 - jepsen.util 4635 :invoke :write [15 7] INFO [2019-12-17 12:31:11,025] jepsen worker 21 - jepsen.util 5691 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:31:11,025] jepsen worker 21 - jepsen.util 5691 :invoke :write [7 26] INFO [2019-12-17 12:31:11,032] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25634,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:31:11,033] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25634,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:31:11,033] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25636,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:31:11,033] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25636,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:31:11,082] jepsen worker 72 - jepsen.util 4572 :ok :read [24 9] INFO [2019-12-17 12:31:11,083] jepsen worker 72 - jepsen.util 4572 :invoke :write [24 30] INFO [2019-12-17 12:31:11,084] jepsen worker 65 - jepsen.util 3845 :fail :cas [21 [16 19]] :ValueNotSwapFrom INFO [2019-12-17 12:31:11,084] jepsen worker 65 - jepsen.util 3845 :invoke :cas [21 [20 3]] INFO [2019-12-17 12:31:11,098] jepsen worker 32 - jepsen.util 4802 :fail :cas [10 [5 38]] :ValueNotSwapFrom INFO [2019-12-17 12:31:11,098] jepsen worker 32 - jepsen.util 4802 :invoke :write [10 8] INFO [2019-12-17 12:31:11,100] jepsen worker 55 - jepsen.util 5455 :fail :cas [18 [49 47]] :ValueNotSwapFrom INFO [2019-12-17 12:31:11,100] jepsen worker 55 - jepsen.util 5455 :invoke :write [18 39] INFO [2019-12-17 12:31:11,110] jepsen worker 37 - jepsen.util 4267 :info :write [12 28] :RequestCanceledException INFO [2019-12-17 12:31:11,110] jepsen worker 27 - jepsen.util 4257 :info :write [9 25] :RequestCanceledException INFO [2019-12-17 12:31:11,111] jepsen worker 27 - jepsen.util 4347 :invoke :write [9 27] INFO [2019-12-17 12:31:11,112] jepsen worker 29 - jepsen.util 3719 :fail :cas [9 [13 41]] :ValueNotSwapFrom INFO [2019-12-17 12:31:11,112] jepsen worker 29 - jepsen.util 3719 :invoke :read [9 nil] INFO [2019-12-17 12:31:11,116] jepsen worker 37 - jepsen.util 4357 :invoke :write [12 29] INFO [2019-12-17 12:31:11,152] jepsen worker 80 - jepsen.util 4490 :fail :cas [26 [0 26]] :ValueNotSwapFrom INFO [2019-12-17 12:31:11,152] jepsen worker 80 - jepsen.util 4490 :invoke :write [26 24] INFO [2019-12-17 12:31:11,164] jepsen worker 65 - jepsen.util 3845 :fail :cas [21 [20 3]] :ValueNotSwapFrom INFO [2019-12-17 12:31:11,164] jepsen worker 65 - jepsen.util 3845 :invoke :write [21 2] INFO [2019-12-17 12:31:11,175] jepsen worker 29 - jepsen.util 3719 :ok :read [9 17] INFO [2019-12-17 12:31:11,175] jepsen worker 29 - jepsen.util 3719 :invoke :write [9 11] INFO [2019-12-17 12:31:11,188] jepsen worker 5 - jepsen.util 4145 :fail :cas [1 [14 20]] :ValueNotSwapFrom INFO [2019-12-17 12:31:11,188] jepsen worker 5 - jepsen.util 4145 :invoke :cas [1 [5 12]] INFO [2019-12-17 12:31:11,191] jepsen worker 88 - jepsen.util 4588 :info :write [29 24] :RequestCanceledException INFO [2019-12-17 12:31:11,191] jepsen worker 61 - jepsen.util 6451 :info :cas [20 [25 17]] :RequestCanceledException INFO [2019-12-17 12:31:11,193] jepsen worker 88 - jepsen.util 4678 :invoke :write [29 6] INFO [2019-12-17 12:31:11,195] jepsen worker 61 - jepsen.util 6541 :invoke :write [20 5] INFO [2019-12-17 12:31:11,205] jepsen worker 27 - jepsen.util 4347 :ok :write [9 27] INFO [2019-12-17 12:31:11,205] jepsen worker 27 - jepsen.util 4347 :invoke :read [9 nil] INFO [2019-12-17 12:31:11,216] jepsen worker 73 - jepsen.util 4303 :ok :write [24 0] INFO [2019-12-17 12:31:11,216] jepsen worker 73 - jepsen.util 4303 :invoke :read [24 nil] INFO [2019-12-17 12:31:11,222] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25666,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:31:11,222] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25666,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:31:11,225] jepsen worker 4 - jepsen.util 3604 :info :write [1 3] :RequestCanceledException INFO [2019-12-17 12:31:11,227] jepsen worker 4 - jepsen.util 3694 :invoke :read [1 nil] INFO [2019-12-17 12:31:11,238] jepsen worker 36 - jepsen.util 4536 :fail :cas [12 [0 11]] :ValueNotSwapFrom INFO [2019-12-17 12:31:11,238] jepsen worker 36 - jepsen.util 4536 :invoke :write [12 42] INFO [2019-12-17 12:31:11,239] jepsen worker 56 - jepsen.util 4556 :info :write [18 8] :RequestCanceledException INFO [2019-12-17 12:31:11,240] jepsen worker 56 - jepsen.util 4646 :invoke :write [18 22] INFO [2019-12-17 12:31:11,252] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25654,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:31:11,252] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25654,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:31:11,252] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25655,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:31:11,252] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25655,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:31:11,256] jepsen worker 20 - jepsen.util 4520 :info :cas [6 [9 12]] :RequestCanceledException INFO [2019-12-17 12:31:11,256] jepsen worker 20 - jepsen.util 4610 :invoke :write [6 46] INFO [2019-12-17 12:31:11,274] jepsen worker 64 - jepsen.util 3484 :ok :read [21 29] INFO [2019-12-17 12:31:11,274] jepsen worker 64 - jepsen.util 3484 :invoke :cas [21 [25 48]] INFO [2019-12-17 12:31:11,291] jepsen worker 10 - jepsen.util 4330 :ok :read [3 5] INFO [2019-12-17 12:31:11,292] jepsen worker 10 - jepsen.util 4330 :invoke :write [3 29] INFO [2019-12-17 12:31:11,295] jepsen worker 65 - jepsen.util 3845 :ok :write [21 2] INFO [2019-12-17 12:31:11,295] jepsen worker 65 - jepsen.util 3845 :invoke :write [21 10] INFO [2019-12-17 12:31:11,296] jepsen worker 78 - jepsen.util 4668 :ok :read [26 4] INFO [2019-12-17 12:31:11,296] jepsen worker 78 - jepsen.util 4668 :invoke :write [26 42] INFO [2019-12-17 12:31:11,308] jepsen worker 47 - jepsen.util 4907 :ok :read [15 42] INFO [2019-12-17 12:31:11,309] jepsen worker 47 - jepsen.util 4907 :invoke :write [15 19] INFO [2019-12-17 12:31:11,309] jepsen worker 77 - jepsen.util 5927 :info :cas [25 [15 24]] :RequestCanceledException INFO [2019-12-17 12:31:11,310] jepsen worker 77 - jepsen.util 6017 :invoke :write [25 38] INFO [2019-12-17 12:31:11,324] jepsen worker 86 - jepsen.util 6116 :info :write [28 21] :RequestCanceledException INFO [2019-12-17 12:31:11,325] jepsen worker 51 - jepsen.util 4101 :info :write [17 0] :RequestCanceledException INFO [2019-12-17 12:31:11,325] jepsen worker 86 - jepsen.util 6206 :invoke :cas [28 [47 26]] INFO [2019-12-17 12:31:11,327] jepsen worker 51 - jepsen.util 4191 :invoke :cas [17 [2 44]] INFO [2019-12-17 12:31:11,335] jepsen worker 73 - jepsen.util 4303 :ok :read [24 0] INFO [2019-12-17 12:31:11,335] jepsen worker 73 - jepsen.util 4303 :invoke :read [24 nil] INFO [2019-12-17 12:31:11,339] jepsen worker 70 - jepsen.util 5110 :info :cas [23 [4 18]] :RequestCanceledException INFO [2019-12-17 12:31:11,342] jepsen worker 70 - jepsen.util 5200 :invoke :write [23 12] INFO [2019-12-17 12:31:11,350] jepsen worker 64 - jepsen.util 3484 :fail :cas [21 [25 48]] :ValueNotSwapFrom INFO [2019-12-17 12:31:11,350] jepsen worker 64 - jepsen.util 3484 :invoke :write [21 20] INFO [2019-12-17 12:31:11,379] jepsen worker 4 - jepsen.util 3694 :ok :read [1 7] INFO [2019-12-17 12:31:11,379] jepsen worker 4 - jepsen.util 3694 :invoke :write [1 26] INFO [2019-12-17 12:31:11,388] jepsen worker 32 - jepsen.util 4802 :ok :write [10 8] INFO [2019-12-17 12:31:11,388] jepsen worker 11 - jepsen.util 4691 :ok :write [3 20] INFO [2019-12-17 12:31:11,389] jepsen worker 11 - jepsen.util 4691 :invoke :read [3 nil] INFO [2019-12-17 12:31:11,389] jepsen worker 32 - jepsen.util 4802 :invoke :cas [10 [38 1]] INFO [2019-12-17 12:31:11,392] jepsen worker 49 - jepsen.util 5179 :info :cas [16 [44 19]] :RequestCanceledException INFO [2019-12-17 12:31:11,393] jepsen worker 49 - jepsen.util 5269 :invoke :write [16 19] INFO [2019-12-17 12:31:11,407] jepsen worker 24 - jepsen.util 5604 :info :cas [8 [44 21]] :RequestCanceledException INFO [2019-12-17 12:31:11,408] jepsen worker 24 - jepsen.util 5694 :invoke :read [8 nil] INFO [2019-12-17 12:31:11,425] jepsen worker 69 - jepsen.util 4569 :info :write [23 14] :RequestCanceledException INFO [2019-12-17 12:31:11,426] jepsen worker 69 - jepsen.util 4659 :invoke :write [23 6] INFO [2019-12-17 12:31:11,436] jepsen worker 29 - jepsen.util 3719 :ok :write [9 11] INFO [2019-12-17 12:31:11,436] jepsen worker 29 - jepsen.util 3719 :invoke :cas [9 [29 3]] INFO [2019-12-17 12:31:11,448] jepsen worker 32 - jepsen.util 4802 :fail :cas [10 [38 1]] :ValueNotSwapFrom INFO [2019-12-17 12:31:11,448] jepsen worker 32 - jepsen.util 4802 :invoke :cas [10 [19 39]] INFO [2019-12-17 12:31:11,456] jepsen worker 87 - jepsen.util 4227 :info :write [29 12] :RequestCanceledException INFO [2019-12-17 12:31:11,456] jepsen worker 46 - jepsen.util 4546 :info :cas [15 [37 6]] :RequestCanceledException INFO [2019-12-17 12:31:11,457] jepsen worker 46 - jepsen.util 4636 :invoke :cas [15 [18 49]] INFO [2019-12-17 12:31:11,457] jepsen worker 87 - jepsen.util 4317 :invoke :cas [29 [0 48]] INFO [2019-12-17 12:31:11,464] jepsen worker 32 - jepsen.util 4802 :fail :cas [10 [19 39]] :ValueNotSwapFrom INFO [2019-12-17 12:31:11,464] jepsen worker 32 - jepsen.util 4802 :invoke :write [10 10] INFO [2019-12-17 12:31:11,467] jepsen worker 29 - jepsen.util 3719 :fail :cas [9 [29 3]] :ValueNotSwapFrom INFO [2019-12-17 12:31:11,467] jepsen worker 29 - jepsen.util 3719 :invoke :cas [9 [38 25]] INFO [2019-12-17 12:31:11,473] jepsen worker 46 - jepsen.util 4636 :fail :cas [15 [18 49]] :ValueNotSwapFrom INFO [2019-12-17 12:31:11,473] jepsen worker 46 - jepsen.util 4636 :invoke :cas [15 [44 21]] INFO [2019-12-17 12:31:11,496] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25693,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:31:11,496] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25693,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:31:11,501] jepsen worker 87 - jepsen.util 4317 :fail :cas [29 [0 48]] :ValueNotSwapFrom INFO [2019-12-17 12:31:11,501] jepsen worker 87 - jepsen.util 4317 :invoke :cas [29 [4 49]] INFO [2019-12-17 12:31:11,505] jepsen worker 29 - jepsen.util 3719 :fail :cas [9 [38 25]] :ValueNotSwapFrom INFO [2019-12-17 12:31:11,505] jepsen worker 29 - jepsen.util 3719 :invoke :read [9 nil] INFO [2019-12-17 12:31:11,528] jepsen worker 46 - jepsen.util 4636 :fail :cas [15 [44 21]] :ValueNotSwapFrom INFO [2019-12-17 12:31:11,529] jepsen worker 46 - jepsen.util 4636 :invoke :read [15 nil] INFO [2019-12-17 12:31:11,543] jepsen worker 46 - jepsen.util 4636 :ok :read [15 42] INFO [2019-12-17 12:31:11,543] jepsen worker 46 - jepsen.util 4636 :invoke :write [15 39] INFO [2019-12-17 12:31:11,620] jepsen worker 87 - jepsen.util 4317 :fail :cas [29 [4 49]] :ValueNotSwapFrom INFO [2019-12-17 12:31:11,625] jepsen worker 87 - jepsen.util 4317 :invoke :read [29 nil] INFO [2019-12-17 12:31:11,625] jepsen worker 30 - jepsen.util 4620 :info :write [10 13] :RequestCanceledException INFO [2019-12-17 12:31:11,626] jepsen worker 30 - jepsen.util 4710 :invoke :read [10 nil] INFO [2019-12-17 12:31:11,642] jepsen worker 30 - jepsen.util 4710 :ok :read [10 8] INFO [2019-12-17 12:31:11,642] jepsen worker 30 - jepsen.util 4710 :invoke :read [10 nil] INFO [2019-12-17 12:31:11,642] jepsen worker 34 - jepsen.util 5524 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:31:11,642] jepsen worker 74 - jepsen.util 4934 :fail :read [24 nil] :RequestTimeoutException INFO [2019-12-17 12:31:11,642] jepsen worker 34 - jepsen.util 5524 :invoke :write [11 18] INFO [2019-12-17 12:31:11,642] jepsen worker 74 - jepsen.util 4934 :invoke :write [24 13] INFO [2019-12-17 12:31:11,643] jepsen worker 5 - jepsen.util 4145 :fail :cas [1 [5 12]] :ValueNotSwapFrom INFO [2019-12-17 12:31:11,643] jepsen worker 5 - jepsen.util 4145 :invoke :read [1 nil] INFO [2019-12-17 12:31:11,659] jepsen worker 51 - jepsen.util 4191 :fail :cas [17 [2 44]] :ValueNotSwapFrom INFO [2019-12-17 12:31:11,660] jepsen worker 51 - jepsen.util 4191 :invoke :read [17 nil] INFO [2019-12-17 12:31:11,663] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25712,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:31:11,668] jepsen worker 30 - jepsen.util 4710 :ok :read [10 8] INFO [2019-12-17 12:31:11,671] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25712,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:31:11,671] jepsen worker 30 - jepsen.util 4710 :invoke :read [10 nil] INFO [2019-12-17 12:31:11,671] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25716,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:31:11,671] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25716,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:31:11,692] jepsen worker 30 - jepsen.util 4710 :ok :read [10 8] INFO [2019-12-17 12:31:11,692] jepsen worker 30 - jepsen.util 4710 :invoke :cas [10 [5 49]] INFO [2019-12-17 12:31:11,744] jepsen worker 30 - jepsen.util 4710 :fail :cas [10 [5 49]] :ValueNotSwapFrom INFO [2019-12-17 12:31:11,756] jepsen worker 68 - jepsen.util 5468 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:31:11,757] jepsen worker 30 - jepsen.util 4710 :invoke :write [10 4] INFO [2019-12-17 12:31:11,757] jepsen worker 68 - jepsen.util 5468 :invoke :read [22 nil] INFO [2019-12-17 12:31:11,763] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:31:11,763] jepsen nemesis - couchbase.util Rebalance check count: 190 INFO [2019-12-17 12:31:11,763] jepsen nemesis - couchbase.util Rebalance stuck count: 171 INFO [2019-12-17 12:31:11,773] jepsen worker 53 - jepsen.util 4463 :ok :read [17 48] INFO [2019-12-17 12:31:11,773] jepsen worker 53 - jepsen.util 4463 :invoke :cas [17 [44 34]] INFO [2019-12-17 12:31:11,820] jepsen worker 12 - jepsen.util 4692 :ok :read [4 40] INFO [2019-12-17 12:31:11,820] jepsen worker 12 - jepsen.util 4692 :invoke :cas [4 [12 47]] INFO [2019-12-17 12:31:11,824] jepsen worker 63 - jepsen.util 3663 :fail :read [21 nil] :RequestTimeoutException INFO [2019-12-17 12:31:11,824] jepsen worker 23 - jepsen.util 5783 :info :cas [7 [37 28]] :RequestCanceledException INFO [2019-12-17 12:31:11,825] jepsen worker 63 - jepsen.util 3663 :invoke :cas [21 [32 10]] INFO [2019-12-17 12:31:11,826] jepsen worker 23 - jepsen.util 5873 :invoke :cas [7 [12 28]] INFO [2019-12-17 12:31:11,838] jepsen worker 3 - jepsen.util 4593 :info :write [1 13] :RequestCanceledException INFO [2019-12-17 12:31:11,850] jepsen worker 3 - jepsen.util 4683 :invoke :write [1 31] INFO [2019-12-17 12:31:11,909] jepsen worker 44 - jepsen.util 6434 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:31:11,909] jepsen worker 44 - jepsen.util 6434 :invoke :cas [14 [5 30]] INFO [2019-12-17 12:31:11,910] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25730,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:31:11,910] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25730,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:31:11,910] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25733,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:31:11,910] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25733,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:31:11,910] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25734,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:31:11,910] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25734,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:31:11,919] jepsen worker 87 - jepsen.util 4317 :ok :read [29 36] INFO [2019-12-17 12:31:11,919] jepsen worker 87 - jepsen.util 4317 :invoke :cas [29 [43 44]] INFO [2019-12-17 12:31:11,939] jepsen worker 54 - jepsen.util 4914 :info :write [18 40] :RequestCanceledException INFO [2019-12-17 12:31:11,939] jepsen worker 43 - jepsen.util 5533 :info :cas [14 [34 41]] :RequestCanceledException INFO [2019-12-17 12:31:11,944] jepsen worker 53 - jepsen.util 4463 :fail :cas [17 [44 34]] :ValueNotSwapFrom INFO [2019-12-17 12:31:11,945] jepsen worker 53 - jepsen.util 4463 :invoke :read [17 nil] INFO [2019-12-17 12:31:11,945] jepsen worker 54 - jepsen.util 5004 :invoke :cas [18 [39 15]] INFO [2019-12-17 12:31:11,946] jepsen worker 43 - jepsen.util 5623 :invoke :cas [14 [30 39]] INFO [2019-12-17 12:31:11,954] jepsen worker 12 - jepsen.util 4692 :fail :cas [4 [12 47]] :ValueNotSwapFrom INFO [2019-12-17 12:31:11,955] jepsen worker 12 - jepsen.util 4692 :invoke :read [4 nil] INFO [2019-12-17 12:31:12,019] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25742,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:31:12,020] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25742,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:31:12,020] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25746,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:31:12,020] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25746,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:31:12,025] jepsen worker 6 - jepsen.util 6396 :info :cas [2 [10 47]] :RequestCanceledException INFO [2019-12-17 12:31:12,025] jepsen worker 85 - jepsen.util 6565 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:31:12,025] jepsen worker 59 - jepsen.util 5999 :info :write [19 4] :RequestCanceledException INFO [2019-12-17 12:31:12,025] jepsen worker 85 - jepsen.util 6565 :invoke :cas [28 [8 30]] INFO [2019-12-17 12:31:12,026] jepsen worker 6 - jepsen.util 6486 :invoke :read [2 nil] INFO [2019-12-17 12:31:12,035] jepsen worker 59 - jepsen.util 6089 :invoke :write [19 2] INFO [2019-12-17 12:31:12,039] jepsen worker 84 - jepsen.util 5394 :info :cas [28 [20 48]] :RequestCanceledException INFO [2019-12-17 12:31:12,040] jepsen worker 84 - jepsen.util 5484 :invoke :write [28 46] INFO [2019-12-17 12:31:12,044] jepsen worker 51 - jepsen.util 4191 :ok :read [17 48] INFO [2019-12-17 12:31:12,045] jepsen worker 51 - jepsen.util 4191 :invoke :write [17 0] INFO [2019-12-17 12:31:12,091] jepsen worker 9 - jepsen.util 5139 :info :write [3 1] :RequestCanceledException INFO [2019-12-17 12:31:12,092] jepsen worker 9 - jepsen.util 5229 :invoke :write [3 31] INFO [2019-12-17 12:31:12,110] jepsen worker 66 - jepsen.util 5556 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:31:12,110] jepsen worker 66 - jepsen.util 5556 :invoke :read [22 nil] INFO [2019-12-17 12:31:12,125] jepsen worker 25 - jepsen.util 5965 :info :write [8 23] :RequestCanceledException INFO [2019-12-17 12:31:12,125] jepsen worker 7 - jepsen.util 5767 :info :cas [2 [28 41]] :RequestCanceledException INFO [2019-12-17 12:31:12,126] jepsen worker 25 - jepsen.util 6055 :invoke :cas [8 [45 8]] INFO [2019-12-17 12:31:12,126] jepsen worker 7 - jepsen.util 5857 :invoke :cas [2 [3 1]] INFO [2019-12-17 12:31:12,138] jepsen worker 82 - jepsen.util 6742 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:31:12,139] jepsen worker 82 - jepsen.util 6742 :invoke :read [27 nil] INFO [2019-12-17 12:31:12,152] jepsen worker 87 - jepsen.util 4317 :fail :cas [29 [43 44]] :ValueNotSwapFrom INFO [2019-12-17 12:31:12,152] jepsen worker 87 - jepsen.util 4317 :invoke :cas [29 [22 31]] INFO [2019-12-17 12:31:12,156] jepsen worker 62 - jepsen.util 5462 :info :write [20 33] :RequestCanceledException INFO [2019-12-17 12:31:12,156] jepsen worker 19 - jepsen.util 4519 :info :write [6 8] :RequestCanceledException INFO [2019-12-17 12:31:12,156] jepsen worker 28 - jepsen.util 3358 :info :cas [9 [14 15]] :RequestCanceledException INFO [2019-12-17 12:31:12,157] jepsen worker 62 - jepsen.util 5552 :invoke :write [20 9] INFO [2019-12-17 12:31:12,157] jepsen worker 19 - jepsen.util 4609 :invoke :read [6 nil] INFO [2019-12-17 12:31:12,160] jepsen worker 28 - jepsen.util 3448 :invoke :cas [9 [16 21]] INFO [2019-12-17 12:31:12,175] jepsen worker 19 - jepsen.util 4609 :ok :read [6 46] INFO [2019-12-17 12:31:12,175] jepsen worker 19 - jepsen.util 4609 :invoke :write [6 3] INFO [2019-12-17 12:31:12,183] jepsen worker 28 - jepsen.util 3448 :fail :cas [9 [16 21]] :ValueNotSwapFrom INFO [2019-12-17 12:31:12,184] jepsen worker 28 - jepsen.util 3448 :invoke :cas [9 [44 22]] INFO [2019-12-17 12:31:12,210] jepsen worker 28 - jepsen.util 3448 :fail :cas [9 [44 22]] :ValueNotSwapFrom INFO [2019-12-17 12:31:12,210] jepsen worker 28 - jepsen.util 3448 :invoke :write [9 45] INFO [2019-12-17 12:31:12,225] jepsen worker 35 - jepsen.util 5975 :info :write [11 35] :RequestCanceledException INFO [2019-12-17 12:31:12,225] jepsen worker 39 - jepsen.util 5349 :info :cas [13 [29 41]] :RequestCanceledException INFO [2019-12-17 12:31:12,226] jepsen worker 35 - jepsen.util 6065 :invoke :cas [11 [16 43]] INFO [2019-12-17 12:31:12,226] jepsen worker 39 - jepsen.util 5439 :invoke :cas [13 [20 15]] INFO [2019-12-17 12:31:12,240] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25764,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:31:12,240] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25764,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:31:12,240] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25766,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:31:12,240] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25766,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:31:12,255] jepsen worker 28 - jepsen.util 3448 :ok :write [9 45] INFO [2019-12-17 12:31:12,255] jepsen worker 28 - jepsen.util 3448 :invoke :write [9 34] INFO [2019-12-17 12:31:12,291] jepsen worker 83 - jepsen.util 6203 :info :write [27 49] :RequestCanceledException INFO [2019-12-17 12:31:12,292] jepsen worker 83 - jepsen.util 6293 :invoke :write [27 47] INFO [2019-12-17 12:31:12,309] jepsen worker 18 - jepsen.util 4428 :info :write [6 35] :RequestCanceledException INFO [2019-12-17 12:31:12,312] jepsen worker 28 - jepsen.util 3448 :ok :write [9 34] INFO [2019-12-17 12:31:12,312] jepsen worker 28 - jepsen.util 3448 :invoke :write [9 36] INFO [2019-12-17 12:31:12,312] jepsen worker 18 - jepsen.util 4518 :invoke :read [6 nil] INFO [2019-12-17 12:31:12,313] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25771,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:31:12,313] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25771,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:31:12,313] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25775,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:31:12,313] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25775,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:31:12,325] jepsen worker 58 - jepsen.util 5998 :info :write [19 14] :RequestCanceledException INFO [2019-12-17 12:31:12,326] jepsen worker 58 - jepsen.util 6088 :invoke :read [19 nil] INFO [2019-12-17 12:31:12,335] jepsen worker 38 - jepsen.util 4448 :info :write [12 5] :RequestCanceledException INFO [2019-12-17 12:31:12,337] jepsen worker 38 - jepsen.util 4538 :invoke :cas [12 [1 15]] INFO [2019-12-17 12:31:12,356] jepsen worker 14 - jepsen.util 4514 :info :write [4 40] :RequestCanceledException INFO [2019-12-17 12:31:12,356] jepsen worker 40 - jepsen.util 6610 :info :cas [13 [5 15]] :RequestCanceledException INFO [2019-12-17 12:31:12,357] jepsen worker 14 - jepsen.util 4604 :invoke :read [4 nil] INFO [2019-12-17 12:31:12,357] jepsen worker 40 - jepsen.util 6700 :invoke :write [13 12] INFO [2019-12-17 12:31:12,358] jepsen worker 28 - jepsen.util 3448 :ok :write [9 36] INFO [2019-12-17 12:31:12,358] jepsen worker 28 - jepsen.util 3448 :invoke :cas [9 [30 29]] INFO [2019-12-17 12:31:12,381] jepsen worker 28 - jepsen.util 3448 :fail :cas [9 [30 29]] :ValueNotSwapFrom INFO [2019-12-17 12:31:12,381] jepsen worker 28 - jepsen.util 3448 :invoke :read [9 nil] INFO [2019-12-17 12:31:12,391] jepsen worker 2 - jepsen.util 5582 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:31:12,391] jepsen worker 2 - jepsen.util 5582 :invoke :read [0 nil] INFO [2019-12-17 12:31:12,399] jepsen worker 28 - jepsen.util 3448 :ok :read [9 36] INFO [2019-12-17 12:31:12,399] jepsen worker 28 - jepsen.util 3448 :invoke :cas [9 [34 9]] INFO [2019-12-17 12:31:12,411] jepsen worker 28 - jepsen.util 3448 :fail :cas [9 [34 9]] :ValueNotSwapFrom INFO [2019-12-17 12:31:12,412] jepsen worker 28 - jepsen.util 3448 :invoke :cas [9 [23 35]] INFO [2019-12-17 12:31:12,423] jepsen worker 28 - jepsen.util 3448 :fail :cas [9 [23 35]] :ValueNotSwapFrom INFO [2019-12-17 12:31:12,423] jepsen worker 28 - jepsen.util 3448 :invoke :cas [9 [38 38]] INFO [2019-12-17 12:31:12,425] jepsen worker 50 - jepsen.util 6080 :info :cas [16 [36 15]] :RequestCanceledException INFO [2019-12-17 12:31:12,425] jepsen worker 0 - jepsen.util 6480 :info :write [0 15] :RequestCanceledException INFO [2019-12-17 12:31:12,425] jepsen worker 89 - jepsen.util 4139 :info :write [29 27] :RequestCanceledException INFO [2019-12-17 12:31:12,427] jepsen worker 89 - jepsen.util 4229 :invoke :write [29 25] INFO [2019-12-17 12:31:12,429] jepsen worker 50 - jepsen.util 6170 :invoke :write [16 30] INFO [2019-12-17 12:31:12,429] jepsen worker 0 - jepsen.util 6570 :invoke :cas [0 [19 16]] INFO [2019-12-17 12:31:12,436] jepsen worker 60 - jepsen.util 6450 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:31:12,436] jepsen worker 60 - jepsen.util 6450 :invoke :read [20 nil] INFO [2019-12-17 12:31:12,443] jepsen worker 28 - jepsen.util 3448 :fail :cas [9 [38 38]] :ValueNotSwapFrom INFO [2019-12-17 12:31:12,444] jepsen worker 28 - jepsen.util 3448 :invoke :cas [9 [38 40]] INFO [2019-12-17 12:31:12,454] jepsen worker 28 - jepsen.util 3448 :fail :cas [9 [38 40]] :ValueNotSwapFrom INFO [2019-12-17 12:31:12,454] jepsen worker 28 - jepsen.util 3448 :invoke :cas [9 [14 29]] INFO [2019-12-17 12:31:12,470] jepsen worker 28 - jepsen.util 3448 :fail :cas [9 [14 29]] :ValueNotSwapFrom INFO [2019-12-17 12:31:12,470] jepsen worker 28 - jepsen.util 3448 :invoke :cas [9 [13 35]] INFO [2019-12-17 12:31:12,478] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25794,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:31:12,479] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25794,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:31:12,479] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25798,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:31:12,479] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25798,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:31:12,479] jepsen worker 28 - jepsen.util 3448 :fail :cas [9 [13 35]] :ValueNotSwapFrom INFO [2019-12-17 12:31:12,479] jepsen worker 28 - jepsen.util 3448 :invoke :cas [9 [11 33]] INFO [2019-12-17 12:31:12,490] jepsen worker 28 - jepsen.util 3448 :fail :cas [9 [11 33]] :ValueNotSwapFrom INFO [2019-12-17 12:31:12,491] jepsen worker 28 - jepsen.util 3448 :invoke :read [9 nil] INFO [2019-12-17 12:31:12,491] jepsen worker 57 - jepsen.util 5727 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:31:12,491] jepsen worker 57 - jepsen.util 5727 :invoke :write [19 19] INFO [2019-12-17 12:31:12,505] jepsen worker 28 - jepsen.util 3448 :ok :read [9 36] INFO [2019-12-17 12:31:12,505] jepsen worker 28 - jepsen.util 3448 :invoke :read [9 nil] INFO [2019-12-17 12:31:12,509] jepsen worker 67 - jepsen.util 6097 :info :write [22 27] :RequestCanceledException INFO [2019-12-17 12:31:12,514] jepsen worker 67 - jepsen.util 6187 :invoke :read [22 nil] INFO [2019-12-17 12:31:12,521] jepsen worker 28 - jepsen.util 3448 :ok :read [9 36] INFO [2019-12-17 12:31:12,521] jepsen worker 28 - jepsen.util 3448 :invoke :write [9 5] INFO [2019-12-17 12:31:12,526] jepsen worker 26 - jepsen.util 5966 :info :cas [8 [28 19]] :RequestCanceledException INFO [2019-12-17 12:31:12,527] jepsen worker 26 - jepsen.util 6056 :invoke :cas [8 [49 2]] INFO [2019-12-17 12:31:12,536] jepsen worker 5 - jepsen.util 4145 :ok :read [1 7] INFO [2019-12-17 12:31:12,536] jepsen worker 5 - jepsen.util 4145 :invoke :read [1 nil] INFO [2019-12-17 12:31:12,553] jepsen worker 28 - jepsen.util 3448 :ok :write [9 5] INFO [2019-12-17 12:31:12,554] jepsen worker 28 - jepsen.util 3448 :invoke :cas [9 [27 44]] INFO [2019-12-17 12:31:12,555] jepsen worker 87 - jepsen.util 4317 :fail :cas [29 [22 31]] :ValueNotSwapFrom INFO [2019-12-17 12:31:12,555] jepsen worker 87 - jepsen.util 4317 :invoke :read [29 nil] INFO [2019-12-17 12:31:12,556] jepsen worker 22 - jepsen.util 5962 :info :write [7 8] :RequestCanceledException INFO [2019-12-17 12:31:12,557] jepsen worker 22 - jepsen.util 6052 :invoke :read [7 nil] INFO [2019-12-17 12:31:12,574] jepsen worker 28 - jepsen.util 3448 :fail :cas [9 [27 44]] :ValueNotSwapFrom INFO [2019-12-17 12:31:12,574] jepsen worker 28 - jepsen.util 3448 :invoke :read [9 nil] INFO [2019-12-17 12:31:12,586] jepsen worker 28 - jepsen.util 3448 :ok :read [9 5] INFO [2019-12-17 12:31:12,586] jepsen worker 28 - jepsen.util 3448 :invoke :write [9 7] INFO [2019-12-17 12:31:12,592] jepsen worker 48 - jepsen.util 5358 :info :cas [16 [4 45]] :RequestCanceledException INFO [2019-12-17 12:31:12,592] jepsen worker 17 - jepsen.util 5687 :info :cas [5 [9 40]] :RequestCanceledException INFO [2019-12-17 12:31:12,593] jepsen worker 17 - jepsen.util 5777 :invoke :write [5 44] INFO [2019-12-17 12:31:12,593] jepsen worker 48 - jepsen.util 5448 :invoke :read [16 nil] INFO [2019-12-17 12:31:12,609] jepsen worker 52 - jepsen.util 4732 :info :write [17 18] :RequestCanceledException INFO [2019-12-17 12:31:12,609] jepsen worker 52 - jepsen.util 4822 :invoke :write [17 21] INFO [2019-12-17 12:31:12,618] jepsen worker 28 - jepsen.util 3448 :ok :write [9 7] INFO [2019-12-17 12:31:12,618] jepsen worker 28 - jepsen.util 3448 :invoke :read [9 nil] INFO [2019-12-17 12:31:12,635] jepsen worker 28 - jepsen.util 3448 :ok :read [9 7] INFO [2019-12-17 12:31:12,635] jepsen worker 28 - jepsen.util 3448 :invoke :read [9 nil] INFO [2019-12-17 12:31:12,639] jepsen worker 14 - jepsen.util 4604 :ok :read [4 40] INFO [2019-12-17 12:31:12,639] jepsen worker 14 - jepsen.util 4604 :invoke :write [4 30] INFO [2019-12-17 12:31:12,639] jepsen worker 75 - jepsen.util 5925 :info :cas [25 [48 47]] :RequestCanceledException INFO [2019-12-17 12:31:12,640] jepsen worker 75 - jepsen.util 6015 :invoke :read [25 nil] INFO [2019-12-17 12:31:12,657] jepsen worker 28 - jepsen.util 3448 :ok :read [9 7] INFO [2019-12-17 12:31:12,657] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25825,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:31:12,657] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25825,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:31:12,657] jepsen worker 28 - jepsen.util 3448 :invoke :write [9 34] INFO [2019-12-17 12:31:12,657] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25829,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:31:12,658] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25829,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:31:12,658] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25830,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:31:12,658] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25830,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:31:12,666] jepsen worker 38 - jepsen.util 4538 :fail :cas [12 [1 15]] :ValueNotSwapFrom INFO [2019-12-17 12:31:12,666] jepsen worker 38 - jepsen.util 4538 :invoke :cas [12 [14 27]] INFO [2019-12-17 12:31:12,692] jepsen worker 28 - jepsen.util 3448 :ok :write [9 34] INFO [2019-12-17 12:31:12,692] jepsen worker 28 - jepsen.util 3448 :invoke :write [9 5] INFO [2019-12-17 12:31:12,709] jepsen worker 42 - jepsen.util 6612 :info :write [14 18] :RequestCanceledException INFO [2019-12-17 12:31:12,711] jepsen worker 42 - jepsen.util 6702 :invoke :read [14 nil] INFO [2019-12-17 12:31:12,724] jepsen worker 33 - jepsen.util 5883 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:31:12,724] jepsen worker 31 - jepsen.util 4171 :info :write [10 27] :RequestCanceledException INFO [2019-12-17 12:31:12,724] jepsen worker 33 - jepsen.util 5883 :invoke :cas [11 [6 31]] INFO [2019-12-17 12:31:12,726] jepsen worker 31 - jepsen.util 4261 :invoke :write [10 9] INFO [2019-12-17 12:31:12,731] jepsen worker 28 - jepsen.util 3448 :ok :write [9 5] INFO [2019-12-17 12:31:12,731] jepsen worker 28 - jepsen.util 3448 :invoke :cas [9 [16 19]] INFO [2019-12-17 12:31:12,739] jepsen worker 41 - jepsen.util 6251 :info :cas [13 [49 1]] :RequestCanceledException INFO [2019-12-17 12:31:12,739] jepsen worker 79 - jepsen.util 4399 :info :write [26 15] :RequestCanceledException INFO [2019-12-17 12:31:12,740] jepsen worker 41 - jepsen.util 6341 :invoke :cas [13 [11 34]] INFO [2019-12-17 12:31:12,741] jepsen worker 79 - jepsen.util 4489 :invoke :write [26 37] INFO [2019-12-17 12:31:12,771] jepsen worker 53 - jepsen.util 4463 :ok :read [17 48] INFO [2019-12-17 12:31:12,771] jepsen worker 53 - jepsen.util 4463 :invoke :write [17 8] INFO [2019-12-17 12:31:12,773] jepsen worker 28 - jepsen.util 3448 :fail :cas [9 [16 19]] :ValueNotSwapFrom INFO [2019-12-17 12:31:12,774] jepsen worker 28 - jepsen.util 3448 :invoke :read [9 nil] INFO [2019-12-17 12:31:12,785] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:31:12,785] jepsen nemesis - couchbase.util Rebalance check count: 191 INFO [2019-12-17 12:31:12,785] jepsen nemesis - couchbase.util Rebalance stuck count: 172 INFO [2019-12-17 12:31:12,793] jepsen worker 54 - jepsen.util 5004 :fail :cas [18 [39 15]] :ValueNotSwapFrom INFO [2019-12-17 12:31:12,793] jepsen worker 54 - jepsen.util 5004 :invoke :read [18 nil] INFO [2019-12-17 12:31:12,808] jepsen worker 28 - jepsen.util 3448 :ok :read [9 5] INFO [2019-12-17 12:31:12,808] jepsen worker 28 - jepsen.util 3448 :invoke :write [9 20] INFO [2019-12-17 12:31:12,824] jepsen worker 16 - jepsen.util 6676 :info :write [5 4] :RequestCanceledException INFO [2019-12-17 12:31:12,824] jepsen worker 8 - jepsen.util 5138 :info :write [2 44] :RequestCanceledException INFO [2019-12-17 12:31:12,824] jepsen worker 76 - jepsen.util 6016 :info :write [25 32] :RequestCanceledException INFO [2019-12-17 12:31:12,825] jepsen worker 16 - jepsen.util 6766 :invoke :write [5 16] INFO [2019-12-17 12:31:12,825] jepsen worker 8 - jepsen.util 5228 :invoke :cas [2 [10 11]] INFO [2019-12-17 12:31:12,826] jepsen worker 76 - jepsen.util 6106 :invoke :read [25 nil] INFO [2019-12-17 12:31:12,834] jepsen worker 12 - jepsen.util 4692 :ok :read [4 40] INFO [2019-12-17 12:31:12,834] jepsen worker 12 - jepsen.util 4692 :invoke :write [4 33] INFO [2019-12-17 12:31:12,856] jepsen worker 28 - jepsen.util 3448 :ok :write [9 20] INFO [2019-12-17 12:31:12,856] jepsen worker 28 - jepsen.util 3448 :invoke :write [9 47] INFO [2019-12-17 12:31:12,856] jepsen worker 1 - jepsen.util 5491 :info :write [0 44] :RequestCanceledException INFO [2019-12-17 12:31:12,858] jepsen worker 20 - jepsen.util 4610 :ok :write [6 46] INFO [2019-12-17 12:31:12,858] jepsen worker 1 - jepsen.util 5581 :invoke :write [0 39] INFO [2019-12-17 12:31:12,858] jepsen worker 20 - jepsen.util 4610 :invoke :read [6 nil] INFO [2019-12-17 12:31:12,876] jepsen worker 45 - jepsen.util 4635 :ok :write [15 7] INFO [2019-12-17 12:31:12,876] jepsen worker 45 - jepsen.util 4635 :invoke :cas [15 [20 2]] INFO [2019-12-17 12:31:12,879] jepsen worker 32 - jepsen.util 4802 :ok :write [10 10] INFO [2019-12-17 12:31:12,880] jepsen worker 32 - jepsen.util 4802 :invoke :write [10 49] INFO [2019-12-17 12:31:12,883] jepsen worker 63 - jepsen.util 3663 :fail :cas [21 [32 10]] :ValueNotSwapFrom INFO [2019-12-17 12:31:12,883] jepsen worker 63 - jepsen.util 3663 :invoke :cas [21 [31 8]] INFO [2019-12-17 12:31:12,893] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25851,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:31:12,893] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25851,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:31:12,893] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25854,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:31:12,893] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25854,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:31:12,909] jepsen worker 15 - jepsen.util 4965 :info :cas [5 [27 17]] :RequestCanceledException INFO [2019-12-17 12:31:12,912] jepsen worker 15 - jepsen.util 5055 :invoke :write [5 24] INFO [2019-12-17 12:31:12,918] jepsen worker 11 - jepsen.util 4691 :ok :read [3 20] INFO [2019-12-17 12:31:12,918] jepsen worker 11 - jepsen.util 4691 :invoke :read [3 nil] INFO [2019-12-17 12:31:12,925] jepsen worker 81 - jepsen.util 5841 :info :cas [27 [21 9]] :RequestCanceledException INFO [2019-12-17 12:31:12,927] jepsen worker 81 - jepsen.util 5931 :invoke :cas [27 [12 17]] INFO [2019-12-17 12:31:12,950] jepsen worker 78 - jepsen.util 4668 :ok :write [26 42] INFO [2019-12-17 12:31:12,950] jepsen worker 78 - jepsen.util 4668 :invoke :read [26 nil] INFO [2019-12-17 12:31:12,953] jepsen worker 29 - jepsen.util 3719 :ok :read [9 20] INFO [2019-12-17 12:31:12,953] jepsen worker 29 - jepsen.util 3719 :invoke :cas [9 [23 48]] INFO [2019-12-17 12:31:12,956] jepsen worker 13 - jepsen.util 4153 :info :write [4 19] :RequestCanceledException INFO [2019-12-17 12:31:12,957] jepsen worker 13 - jepsen.util 4243 :invoke :write [4 39] INFO [2019-12-17 12:31:12,987] jepsen worker 28 - jepsen.util 3448 :ok :write [9 47] INFO [2019-12-17 12:31:12,987] jepsen worker 28 - jepsen.util 3448 :invoke :write [9 25] INFO [2019-12-17 12:31:12,989] jepsen worker 18 - jepsen.util 4518 :ok :read [6 46] INFO [2019-12-17 12:31:12,989] jepsen worker 18 - jepsen.util 4518 :invoke :cas [6 [4 4]] INFO [2019-12-17 12:31:12,991] jepsen worker 71 - jepsen.util 4751 :info :write [23 16] :RequestCanceledException INFO [2019-12-17 12:31:12,992] jepsen worker 71 - jepsen.util 4841 :invoke :write [23 28] INFO [2019-12-17 12:31:13,003] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25870,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:31:13,003] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25870,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:31:13,003] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25874,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:31:13,003] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25874,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:31:13,004] jepsen worker 9 - jepsen.util 5229 :ok :write [3 31] INFO [2019-12-17 12:31:13,005] jepsen worker 9 - jepsen.util 5229 :invoke :read [3 nil] INFO [2019-12-17 12:31:13,031] jepsen worker 20 - jepsen.util 4610 :ok :read [6 46] INFO [2019-12-17 12:31:13,031] jepsen worker 20 - jepsen.util 4610 :invoke :cas [6 [44 27]] INFO [2019-12-17 12:31:13,052] jepsen worker 63 - jepsen.util 3663 :fail :cas [21 [31 8]] :ValueNotSwapFrom INFO [2019-12-17 12:31:13,052] jepsen worker 63 - jepsen.util 3663 :invoke :read [21 nil] INFO [2019-12-17 12:31:13,056] jepsen worker 9 - jepsen.util 5229 :ok :read [3 31] INFO [2019-12-17 12:31:13,056] jepsen worker 9 - jepsen.util 5229 :invoke :cas [3 [13 38]] INFO [2019-12-17 12:31:13,060] jepsen worker 74 - jepsen.util 4934 :ok :write [24 13] INFO [2019-12-17 12:31:13,061] jepsen worker 74 - jepsen.util 4934 :invoke :write [24 45] INFO [2019-12-17 12:31:13,067] jepsen worker 20 - jepsen.util 4610 :fail :cas [6 [44 27]] :ValueNotSwapFrom INFO [2019-12-17 12:31:13,067] jepsen worker 20 - jepsen.util 4610 :invoke :cas [6 [15 36]] INFO [2019-12-17 12:31:13,089] jepsen worker 63 - jepsen.util 3663 :ok :read [21 2] INFO [2019-12-17 12:31:13,090] jepsen worker 63 - jepsen.util 3663 :invoke :write [21 20] INFO [2019-12-17 12:31:13,120] jepsen worker 11 - jepsen.util 4691 :ok :read [3 31] INFO [2019-12-17 12:31:13,120] jepsen worker 11 - jepsen.util 4691 :invoke :cas [3 [22 27]] INFO [2019-12-17 12:31:13,125] jepsen worker 21 - jepsen.util 5691 :info :write [7 26] :RequestCanceledException INFO [2019-12-17 12:31:13,128] jepsen worker 21 - jepsen.util 5781 :invoke :read [7 nil] INFO [2019-12-17 12:31:13,136] jepsen worker 9 - jepsen.util 5229 :fail :cas [3 [13 38]] :ValueNotSwapFrom INFO [2019-12-17 12:31:13,136] jepsen worker 9 - jepsen.util 5229 :invoke :read [3 nil] INFO [2019-12-17 12:31:13,143] jepsen worker 54 - jepsen.util 5004 :ok :read [18 3] INFO [2019-12-17 12:31:13,143] jepsen worker 54 - jepsen.util 5004 :invoke :write [18 44] INFO [2019-12-17 12:31:13,151] jepsen worker 28 - jepsen.util 3448 :ok :write [9 25] INFO [2019-12-17 12:31:13,152] jepsen worker 28 - jepsen.util 3448 :invoke :write [9 8] INFO [2019-12-17 12:31:13,157] jepsen worker 38 - jepsen.util 4538 :fail :cas [12 [14 27]] :ValueNotSwapFrom INFO [2019-12-17 12:31:13,157] jepsen worker 38 - jepsen.util 4538 :invoke :write [12 45] INFO [2019-12-17 12:31:13,192] jepsen worker 55 - jepsen.util 5455 :info :write [18 39] :RequestCanceledException INFO [2019-12-17 12:31:13,195] jepsen worker 55 - jepsen.util 5545 :invoke :read [18 nil] INFO [2019-12-17 12:31:13,199] jepsen worker 28 - jepsen.util 3448 :ok :write [9 8] INFO [2019-12-17 12:31:13,200] jepsen worker 28 - jepsen.util 3448 :invoke :write [9 35] INFO [2019-12-17 12:31:13,207] jepsen worker 32 - jepsen.util 4802 :ok :write [10 49] INFO [2019-12-17 12:31:13,207] jepsen worker 32 - jepsen.util 4802 :invoke :cas [10 [26 46]] INFO [2019-12-17 12:31:13,210] jepsen worker 80 - jepsen.util 4490 :info :write [26 24] :RequestCanceledException INFO [2019-12-17 12:31:13,210] jepsen worker 72 - jepsen.util 4572 :info :write [24 30] :RequestCanceledException INFO [2019-12-17 12:31:13,212] jepsen worker 80 - jepsen.util 4580 :invoke :read [26 nil] INFO [2019-12-17 12:31:13,214] jepsen worker 72 - jepsen.util 4662 :invoke :cas [24 [41 1]] INFO [2019-12-17 12:31:13,218] jepsen worker 20 - jepsen.util 4610 :fail :cas [6 [15 36]] :ValueNotSwapFrom INFO [2019-12-17 12:31:13,218] jepsen worker 20 - jepsen.util 4610 :invoke :cas [6 [39 49]] INFO [2019-12-17 12:31:13,219] jepsen worker 74 - jepsen.util 4934 :ok :write [24 45] INFO [2019-12-17 12:31:13,219] jepsen worker 74 - jepsen.util 4934 :invoke :read [24 nil] INFO [2019-12-17 12:31:13,222] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25890,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:31:13,222] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25890,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:31:13,222] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25891,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:31:13,222] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25891,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:31:13,225] jepsen worker 27 - jepsen.util 4347 :fail :read [9 nil] :RequestTimeoutException INFO [2019-12-17 12:31:13,225] jepsen worker 61 - jepsen.util 6541 :info :write [20 5] :RequestCanceledException INFO [2019-12-17 12:31:13,225] jepsen worker 27 - jepsen.util 4347 :invoke :cas [9 [1 46]] INFO [2019-12-17 12:31:13,227] jepsen worker 61 - jepsen.util 6631 :invoke :cas [20 [17 33]] INFO [2019-12-17 12:31:13,239] jepsen worker 37 - jepsen.util 4357 :info :write [12 29] :RequestCanceledException INFO [2019-12-17 12:31:13,248] jepsen worker 37 - jepsen.util 4447 :invoke :read [12 nil] INFO [2019-12-17 12:31:13,256] jepsen worker 88 - jepsen.util 4678 :info :write [29 6] :RequestCanceledException INFO [2019-12-17 12:31:13,257] jepsen worker 88 - jepsen.util 4768 :invoke :cas [29 [5 45]] INFO [2019-12-17 12:31:13,268] jepsen worker 5 - jepsen.util 4145 :ok :read [1 7] INFO [2019-12-17 12:31:13,268] jepsen worker 5 - jepsen.util 4145 :invoke :read [1 nil] INFO [2019-12-17 12:31:13,278] jepsen worker 29 - jepsen.util 3719 :fail :cas [9 [23 48]] :ValueNotSwapFrom INFO [2019-12-17 12:31:13,278] jepsen worker 29 - jepsen.util 3719 :invoke :read [9 nil] INFO [2019-12-17 12:31:13,291] jepsen worker 36 - jepsen.util 4536 :info :write [12 42] :RequestCanceledException INFO [2019-12-17 12:31:13,297] jepsen worker 36 - jepsen.util 4626 :invoke :write [12 0] INFO [2019-12-17 12:31:13,299] jepsen worker 63 - jepsen.util 3663 :ok :write [21 20] INFO [2019-12-17 12:31:13,299] jepsen worker 63 - jepsen.util 3663 :invoke :write [21 39] INFO [2019-12-17 12:31:13,300] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25899,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:31:13,300] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25899,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:31:13,301] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25902,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:31:13,301] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25902,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:31:13,309] jepsen worker 56 - jepsen.util 4646 :info :write [18 22] :RequestCanceledException INFO [2019-12-17 12:31:13,311] jepsen worker 56 - jepsen.util 4736 :invoke :write [18 28] INFO [2019-12-17 12:31:13,325] jepsen worker 65 - jepsen.util 3845 :info :write [21 10] :RequestCanceledException INFO [2019-12-17 12:31:13,327] jepsen worker 65 - jepsen.util 3935 :invoke :cas [21 [11 33]] INFO [2019-12-17 12:31:13,340] jepsen worker 77 - jepsen.util 6017 :info :write [25 38] :RequestCanceledException INFO [2019-12-17 12:31:13,357] jepsen worker 10 - jepsen.util 4330 :info :write [3 29] :RequestCanceledException INFO [2019-12-17 12:31:13,357] jepsen worker 47 - jepsen.util 4907 :info :write [15 19] :RequestCanceledException INFO [2019-12-17 12:31:13,358] jepsen worker 86 - jepsen.util 6206 :info :cas [28 [47 26]] :RequestCanceledException INFO [2019-12-17 12:31:13,358] jepsen worker 73 - jepsen.util 4303 :fail :read [24 nil] :RequestTimeoutException INFO [2019-12-17 12:31:13,369] jepsen worker 73 - jepsen.util 4303 :invoke :read [24 nil] INFO [2019-12-17 12:31:13,378] jepsen worker 10 - jepsen.util 4420 :invoke :write [3 40] INFO [2019-12-17 12:31:13,381] jepsen worker 86 - jepsen.util 6296 :invoke :read [28 nil] INFO [2019-12-17 12:31:13,383] jepsen worker 47 - jepsen.util 4997 :invoke :cas [15 [30 35]] INFO [2019-12-17 12:31:13,390] jepsen worker 77 - jepsen.util 6107 :invoke :write [25 22] INFO [2019-12-17 12:31:13,393] jepsen worker 70 - jepsen.util 5200 :info :write [23 12] :RequestCanceledException INFO [2019-12-17 12:31:13,402] jepsen worker 70 - jepsen.util 5290 :invoke :read [23 nil] INFO [2019-12-17 12:31:13,411] jepsen worker 49 - jepsen.util 5269 :info :write [16 19] :RequestCanceledException INFO [2019-12-17 12:31:13,416] jepsen worker 28 - jepsen.util 3448 :ok :write [9 35] INFO [2019-12-17 12:31:13,416] jepsen worker 28 - jepsen.util 3448 :invoke :write [9 27] INFO [2019-12-17 12:31:13,426] jepsen worker 24 - jepsen.util 5694 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:31:13,427] jepsen worker 24 - jepsen.util 5694 :invoke :read [8 nil] INFO [2019-12-17 12:31:13,465] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25923,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:31:13,468] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25923,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:31:13,469] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25925,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:31:13,469] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25925,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:31:13,469] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25926,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:31:13,469] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25926,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:31:13,469] jepsen worker 64 - jepsen.util 3484 :info :write [21 20] :RequestCanceledException INFO [2019-12-17 12:31:13,505] jepsen worker 4 - jepsen.util 3694 :info :write [1 26] :RequestCanceledException INFO [2019-12-17 12:31:13,554] jepsen worker 69 - jepsen.util 4659 :info :write [23 6] :RequestCanceledException INFO [2019-12-17 12:31:13,560] jepsen worker 49 - jepsen.util 5359 :invoke :read [16 nil] INFO [2019-12-17 12:31:13,568] jepsen worker 64 - jepsen.util 3574 :invoke :write [21 45] INFO [2019-12-17 12:31:13,630] jepsen worker 4 - jepsen.util 3784 :invoke :write [1 4] INFO [2019-12-17 12:31:13,634] jepsen worker 69 - jepsen.util 4749 :invoke :write [23 23] INFO [2019-12-17 12:31:13,682] jepsen worker 46 - jepsen.util 4636 :info :write [15 39] :RequestCanceledException INFO [2019-12-17 12:31:13,748] jepsen worker 34 - jepsen.util 5524 :info :write [11 18] :RequestCanceledException INFO [2019-12-17 12:31:13,783] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25932,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:31:13,787] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25932,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:31:13,794] jepsen worker 30 - jepsen.util 4710 :info :write [10 4] :RequestCanceledException INFO [2019-12-17 12:31:13,808] jepsen worker 46 - jepsen.util 4726 :invoke :cas [15 [11 17]] INFO [2019-12-17 12:31:13,822] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:31:13,824] jepsen nemesis - couchbase.util Rebalance check count: 192 INFO [2019-12-17 12:31:13,825] jepsen nemesis - couchbase.util Rebalance stuck count: 173 INFO [2019-12-17 12:31:13,830] jepsen worker 34 - jepsen.util 5614 :invoke :read [11 nil] INFO [2019-12-17 12:31:13,839] jepsen worker 30 - jepsen.util 4800 :invoke :write [10 0] INFO [2019-12-17 12:31:13,863] jepsen worker 68 - jepsen.util 5468 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:31:13,887] jepsen worker 68 - jepsen.util 5468 :invoke :read [22 nil] INFO [2019-12-17 12:31:13,905] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25940,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:31:13,919] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25940,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:31:13,921] jepsen worker 23 - jepsen.util 5873 :info :cas [7 [12 28]] :RequestCanceledException INFO [2019-12-17 12:31:13,927] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25945,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:31:13,927] jepsen worker 3 - jepsen.util 4683 :info :write [1 31] :RequestCanceledException INFO [2019-12-17 12:31:13,929] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25945,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:31:13,948] jepsen worker 10 - jepsen.util 4420 :ok :write [3 40] INFO [2019-12-17 12:31:13,948] jepsen worker 10 - jepsen.util 4420 :invoke :cas [3 [19 37]] INFO [2019-12-17 12:31:13,979] jepsen worker 3 - jepsen.util 4773 :invoke :read [1 nil] INFO [2019-12-17 12:31:13,980] jepsen worker 56 - jepsen.util 4736 :ok :write [18 28] INFO [2019-12-17 12:31:13,981] jepsen worker 56 - jepsen.util 4736 :invoke :write [18 7] INFO [2019-12-17 12:31:13,982] jepsen worker 23 - jepsen.util 5963 :invoke :cas [7 [39 34]] INFO [2019-12-17 12:31:14,002] jepsen worker 63 - jepsen.util 3663 :ok :write [21 39] INFO [2019-12-17 12:31:14,005] jepsen worker 63 - jepsen.util 3663 :invoke :write [21 30] INFO [2019-12-17 12:31:14,010] jepsen worker 28 - jepsen.util 3448 :ok :write [9 27] INFO [2019-12-17 12:31:14,011] jepsen worker 44 - jepsen.util 6434 :info :cas [14 [5 30]] :RequestCanceledException INFO [2019-12-17 12:31:14,013] jepsen worker 28 - jepsen.util 3448 :invoke :read [9 nil] INFO [2019-12-17 12:31:14,018] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25952,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:31:14,022] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25952,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:31:14,048] jepsen worker 89 - jepsen.util 4229 :ok :write [29 25] INFO [2019-12-17 12:31:14,049] jepsen worker 89 - jepsen.util 4229 :invoke :write [29 5] INFO [2019-12-17 12:31:14,063] jepsen worker 43 - jepsen.util 5623 :info :cas [14 [30 39]] :RequestCanceledException INFO [2019-12-17 12:31:14,096] jepsen worker 59 - jepsen.util 6089 :info :write [19 2] :RequestCanceledException INFO [2019-12-17 12:31:14,126] jepsen worker 84 - jepsen.util 5484 :info :write [28 46] :RequestCanceledException INFO [2019-12-17 12:31:14,126] jepsen worker 85 - jepsen.util 6565 :info :cas [28 [8 30]] :RequestCanceledException INFO [2019-12-17 12:31:14,139] jepsen worker 43 - jepsen.util 5713 :invoke :cas [14 [49 40]] INFO [2019-12-17 12:31:14,145] jepsen worker 44 - jepsen.util 6524 :invoke :cas [14 [7 3]] INFO [2019-12-17 12:31:14,145] jepsen worker 6 - jepsen.util 6486 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:31:14,147] jepsen worker 6 - jepsen.util 6486 :invoke :read [2 nil] INFO [2019-12-17 12:31:14,154] jepsen worker 51 - jepsen.util 4191 :info :write [17 0] :RequestCanceledException INFO [2019-12-17 12:31:14,154] jepsen worker 25 - jepsen.util 6055 :info :cas [8 [45 8]] :RequestCanceledException INFO [2019-12-17 12:31:14,158] jepsen worker 7 - jepsen.util 5857 :info :cas [2 [3 1]] :RequestCanceledException INFO [2019-12-17 12:31:14,178] jepsen worker 51 - jepsen.util 4281 :invoke :write [17 47] INFO [2019-12-17 12:31:14,184] jepsen worker 19 - jepsen.util 4609 :info :write [6 3] :RequestCanceledException INFO [2019-12-17 12:31:14,191] jepsen worker 59 - jepsen.util 6179 :invoke :read [19 nil] INFO [2019-12-17 12:31:14,192] jepsen worker 7 - jepsen.util 5947 :invoke :write [2 8] INFO [2019-12-17 12:31:14,199] jepsen worker 85 - jepsen.util 6655 :invoke :write [28 40] INFO [2019-12-17 12:31:14,207] jepsen worker 84 - jepsen.util 5574 :invoke :write [28 16] INFO [2019-12-17 12:31:14,208] jepsen worker 66 - jepsen.util 5556 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:31:14,209] jepsen worker 66 - jepsen.util 5556 :invoke :cas [22 [47 38]] INFO [2019-12-17 12:31:14,216] jepsen worker 25 - jepsen.util 6145 :invoke :write [8 20] INFO [2019-12-17 12:31:14,228] jepsen worker 62 - jepsen.util 5552 :info :write [20 9] :RequestCanceledException INFO [2019-12-17 12:31:14,230] jepsen worker 11 - jepsen.util 4691 :fail :cas [3 [22 27]] :ValueNotSwapFrom INFO [2019-12-17 12:31:14,231] jepsen worker 11 - jepsen.util 4691 :invoke :read [3 nil] INFO [2019-12-17 12:31:14,242] jepsen worker 82 - jepsen.util 6742 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:31:14,242] jepsen worker 39 - jepsen.util 5439 :info :cas [13 [20 15]] :RequestCanceledException INFO [2019-12-17 12:31:14,243] jepsen worker 82 - jepsen.util 6742 :invoke :cas [27 [4 10]] INFO [2019-12-17 12:31:14,243] jepsen worker 19 - jepsen.util 4699 :invoke :write [6 33] INFO [2019-12-17 12:31:14,262] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25970,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:31:14,263] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25962,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:31:14,263] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25970,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:31:14,272] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25975,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:31:14,274] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25962,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:31:14,274] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25975,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:31:14,275] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25963,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:31:14,282] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25963,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:31:14,282] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25964,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:31:14,296] jepsen worker 39 - jepsen.util 5529 :invoke :cas [13 [20 5]] INFO [2019-12-17 12:31:14,296] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25964,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:31:14,297] jepsen worker 62 - jepsen.util 5642 :invoke :write [20 46] INFO [2019-12-17 12:31:14,320] jepsen worker 79 - jepsen.util 4489 :ok :write [26 37] INFO [2019-12-17 12:31:14,322] jepsen worker 79 - jepsen.util 4489 :invoke :cas [26 [34 7]] INFO [2019-12-17 12:31:14,330] jepsen worker 35 - jepsen.util 6065 :info :cas [11 [16 43]] :RequestCanceledException INFO [2019-12-17 12:31:14,369] jepsen worker 83 - jepsen.util 6293 :info :write [27 47] :RequestCanceledException INFO [2019-12-17 12:31:14,388] jepsen worker 58 - jepsen.util 6088 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:31:14,391] jepsen worker 58 - jepsen.util 6088 :invoke :cas [19 [31 4]] INFO [2019-12-17 12:31:14,417] jepsen worker 35 - jepsen.util 6155 :invoke :cas [11 [47 30]] INFO [2019-12-17 12:31:14,440] jepsen worker 40 - jepsen.util 6700 :info :write [13 12] :RequestCanceledException INFO [2019-12-17 12:31:14,459] jepsen worker 83 - jepsen.util 6383 :invoke :cas [27 [39 48]] INFO [2019-12-17 12:31:14,479] jepsen worker 40 - jepsen.util 6790 :invoke :write [13 30] INFO [2019-12-17 12:31:14,491] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25988,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:31:14,492] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25988,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:31:14,492] jepsen worker 2 - jepsen.util 5582 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:31:14,492] jepsen worker 2 - jepsen.util 5582 :invoke :cas [0 [34 34]] INFO [2019-12-17 12:31:14,493] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25992,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:31:14,496] jepsen worker 0 - jepsen.util 6570 :info :cas [0 [19 16]] :RequestCanceledException INFO [2019-12-17 12:31:14,503] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25992,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:31:14,506] jepsen worker 18 - jepsen.util 4518 :fail :cas [6 [4 4]] :ValueNotSwapFrom INFO [2019-12-17 12:31:14,533] jepsen worker 50 - jepsen.util 6170 :info :write [16 30] :RequestCanceledException INFO [2019-12-17 12:31:14,547] jepsen worker 18 - jepsen.util 4518 :invoke :write [6 39] INFO [2019-12-17 12:31:14,547] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25993,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:31:14,549] jepsen worker 60 - jepsen.util 6450 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:31:14,549] jepsen worker 60 - jepsen.util 6450 :invoke :cas [20 [31 45]] INFO [2019-12-17 12:31:14,549] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":25993,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:31:14,598] jepsen worker 64 - jepsen.util 3574 :ok :write [21 45] INFO [2019-12-17 12:31:14,599] jepsen worker 64 - jepsen.util 3574 :invoke :read [21 nil] INFO [2019-12-17 12:31:14,599] jepsen worker 57 - jepsen.util 5727 :info :write [19 19] :RequestCanceledException INFO [2019-12-17 12:31:14,606] jepsen worker 0 - jepsen.util 6660 :invoke :write [0 44] INFO [2019-12-17 12:31:14,619] jepsen worker 67 - jepsen.util 6187 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:31:14,621] jepsen worker 67 - jepsen.util 6187 :invoke :write [22 4] INFO [2019-12-17 12:31:14,625] jepsen worker 26 - jepsen.util 6056 :info :cas [8 [49 2]] :RequestCanceledException INFO [2019-12-17 12:31:14,626] jepsen worker 50 - jepsen.util 6260 :invoke :cas [16 [47 36]] INFO [2019-12-17 12:31:14,627] jepsen worker 48 - jepsen.util 5448 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:31:14,627] jepsen worker 48 - jepsen.util 5448 :invoke :read [16 nil] INFO [2019-12-17 12:31:14,642] jepsen worker 22 - jepsen.util 6052 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:31:14,643] jepsen worker 22 - jepsen.util 6052 :invoke :cas [7 [3 2]] INFO [2019-12-17 12:31:14,657] jepsen worker 17 - jepsen.util 5777 :info :write [5 44] :RequestCanceledException INFO [2019-12-17 12:31:14,661] jepsen worker 57 - jepsen.util 5817 :invoke :read [19 nil] INFO [2019-12-17 12:31:14,752] jepsen worker 20 - jepsen.util 4610 :fail :cas [6 [39 49]] :ValueNotSwapFrom INFO [2019-12-17 12:31:14,752] jepsen worker 27 - jepsen.util 4347 :fail :cas [9 [1 46]] :ValueNotSwapFrom INFO [2019-12-17 12:31:14,752] jepsen worker 20 - jepsen.util 4610 :invoke :cas [6 [21 13]] INFO [2019-12-17 12:31:14,753] jepsen worker 27 - jepsen.util 4347 :invoke :read [9 nil] INFO [2019-12-17 12:31:14,753] jepsen worker 74 - jepsen.util 4934 :ok :read [24 45] INFO [2019-12-17 12:31:14,753] jepsen worker 73 - jepsen.util 4303 :ok :read [24 45] INFO [2019-12-17 12:31:14,754] jepsen worker 74 - jepsen.util 4934 :invoke :read [24 nil] INFO [2019-12-17 12:31:14,754] jepsen worker 73 - jepsen.util 4303 :invoke :cas [24 [8 49]] INFO [2019-12-17 12:31:14,755] jepsen worker 32 - jepsen.util 4802 :fail :cas [10 [26 46]] :ValueNotSwapFrom INFO [2019-12-17 12:31:14,755] jepsen worker 28 - jepsen.util 3448 :ok :read [9 27] INFO [2019-12-17 12:31:14,756] jepsen worker 72 - jepsen.util 4662 :fail :cas [24 [41 1]] :ValueNotSwapFrom INFO [2019-12-17 12:31:14,757] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26007,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:31:14,757] jepsen worker 72 - jepsen.util 4662 :invoke :cas [24 [25 24]] INFO [2019-12-17 12:31:14,757] jepsen worker 32 - jepsen.util 4802 :invoke :cas [10 [49 10]] INFO [2019-12-17 12:31:14,758] jepsen worker 28 - jepsen.util 3448 :invoke :read [9 nil] INFO [2019-12-17 12:31:14,759] jepsen worker 14 - jepsen.util 4604 :info :write [4 30] :RequestCanceledException INFO [2019-12-17 12:31:14,759] jepsen worker 42 - jepsen.util 6702 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:31:14,760] jepsen worker 52 - jepsen.util 4822 :info :write [17 21] :RequestCanceledException INFO [2019-12-17 12:31:14,760] jepsen worker 75 - jepsen.util 6015 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:31:14,760] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26007,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:31:14,760] jepsen worker 41 - jepsen.util 6341 :info :cas [13 [11 34]] :RequestCanceledException INFO [2019-12-17 12:31:14,760] jepsen worker 26 - jepsen.util 6146 :invoke :cas [8 [23 34]] INFO [2019-12-17 12:31:14,761] jepsen worker 42 - jepsen.util 6702 :invoke :cas [14 [16 23]] INFO [2019-12-17 12:31:14,762] jepsen worker 75 - jepsen.util 6015 :invoke :read [25 nil] INFO [2019-12-17 12:31:14,762] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26011,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:31:14,764] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26011,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:31:14,773] jepsen worker 13 - jepsen.util 4243 :ok :write [4 39] INFO [2019-12-17 12:31:14,775] jepsen worker 13 - jepsen.util 4243 :invoke :write [4 21] INFO [2019-12-17 12:31:14,791] jepsen worker 17 - jepsen.util 5867 :invoke :read [5 nil] INFO [2019-12-17 12:31:14,792] jepsen worker 52 - jepsen.util 4912 :invoke :cas [17 [6 44]] INFO [2019-12-17 12:31:14,818] jepsen worker 14 - jepsen.util 4694 :invoke :cas [4 [32 19]] INFO [2019-12-17 12:31:14,826] jepsen worker 33 - jepsen.util 5883 :info :cas [11 [6 31]] :RequestCanceledException INFO [2019-12-17 12:31:14,832] jepsen worker 41 - jepsen.util 6431 :invoke :cas [13 [43 37]] INFO [2019-12-17 12:31:14,840] jepsen worker 31 - jepsen.util 4261 :info :write [10 9] :RequestCanceledException INFO [2019-12-17 12:31:14,849] jepsen worker 53 - jepsen.util 4463 :info :write [17 8] :RequestCanceledException INFO [2019-12-17 12:31:14,856] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:31:14,859] jepsen nemesis - couchbase.util Rebalance check count: 193 INFO [2019-12-17 12:31:14,860] jepsen worker 10 - jepsen.util 4420 :fail :cas [3 [19 37]] :ValueNotSwapFrom INFO [2019-12-17 12:31:14,860] jepsen nemesis - couchbase.util Rebalance stuck count: 174 INFO [2019-12-17 12:31:14,861] jepsen worker 10 - jepsen.util 4420 :invoke :cas [3 [35 17]] INFO [2019-12-17 12:31:14,870] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26033,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:31:14,874] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26033,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:31:14,875] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26036,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:31:14,875] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26036,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:31:14,893] jepsen worker 33 - jepsen.util 5973 :invoke :read [11 nil] INFO [2019-12-17 12:31:14,894] jepsen worker 16 - jepsen.util 6766 :info :write [5 16] :RequestCanceledException INFO [2019-12-17 12:31:14,895] jepsen worker 53 - jepsen.util 4553 :invoke :cas [17 [48 41]] INFO [2019-12-17 12:31:14,905] jepsen worker 63 - jepsen.util 3663 :ok :write [21 30] INFO [2019-12-17 12:31:14,911] jepsen worker 63 - jepsen.util 3663 :invoke :cas [21 [18 3]] INFO [2019-12-17 12:31:14,914] jepsen worker 31 - jepsen.util 4351 :invoke :write [10 15] INFO [2019-12-17 12:31:14,919] jepsen worker 8 - jepsen.util 5228 :info :cas [2 [10 11]] :RequestCanceledException INFO [2019-12-17 12:31:14,928] jepsen worker 76 - jepsen.util 6106 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:31:14,932] jepsen worker 76 - jepsen.util 6106 :invoke :write [25 2] INFO [2019-12-17 12:31:14,950] jepsen worker 12 - jepsen.util 4692 :info :write [4 33] :RequestCanceledException INFO [2019-12-17 12:31:14,952] jepsen worker 1 - jepsen.util 5581 :info :write [0 39] :RequestCanceledException INFO [2019-12-17 12:31:14,953] jepsen worker 45 - jepsen.util 4635 :info :cas [15 [20 2]] :RequestCanceledException INFO [2019-12-17 12:31:14,963] jepsen worker 19 - jepsen.util 4699 :ok :write [6 33] INFO [2019-12-17 12:31:14,964] jepsen worker 15 - jepsen.util 5055 :info :write [5 24] :RequestCanceledException INFO [2019-12-17 12:31:14,967] jepsen worker 19 - jepsen.util 4699 :invoke :read [6 nil] INFO [2019-12-17 12:31:14,971] jepsen worker 8 - jepsen.util 5318 :invoke :write [2 3] INFO [2019-12-17 12:31:14,978] jepsen worker 12 - jepsen.util 4782 :invoke :cas [4 [2 38]] INFO [2019-12-17 12:31:14,991] jepsen worker 16 - jepsen.util 6856 :invoke :write [5 28] INFO [2019-12-17 12:31:15,006] jepsen worker 87 - jepsen.util 4317 :ok :read [29 36] INFO [2019-12-17 12:31:15,007] jepsen worker 1 - jepsen.util 5671 :invoke :write [0 35] INFO [2019-12-17 12:31:15,007] jepsen worker 15 - jepsen.util 5145 :invoke :read [5 nil] INFO [2019-12-17 12:31:15,008] jepsen worker 87 - jepsen.util 4317 :invoke :cas [29 [13 28]] INFO [2019-12-17 12:31:15,015] jepsen worker 71 - jepsen.util 4841 :info :write [23 28] :RequestCanceledException INFO [2019-12-17 12:31:15,015] jepsen worker 45 - jepsen.util 4725 :invoke :write [15 22] INFO [2019-12-17 12:31:15,028] jepsen worker 38 - jepsen.util 4538 :ok :write [12 45] INFO [2019-12-17 12:31:15,028] jepsen worker 81 - jepsen.util 5931 :info :cas [27 [12 17]] :RequestCanceledException INFO [2019-12-17 12:31:15,029] jepsen worker 38 - jepsen.util 4538 :invoke :write [12 25] INFO [2019-12-17 12:31:15,053] jepsen worker 71 - jepsen.util 4931 :invoke :cas [23 [16 24]] INFO [2019-12-17 12:31:15,073] jepsen worker 81 - jepsen.util 6021 :invoke :write [27 26] INFO [2019-12-17 12:31:15,116] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26044,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:31:15,124] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26044,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:31:15,127] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26048,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:31:15,130] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26048,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:31:15,214] jepsen worker 65 - jepsen.util 3935 :fail :cas [21 [11 33]] :ValueNotSwapFrom INFO [2019-12-17 12:31:15,215] jepsen worker 65 - jepsen.util 3935 :invoke :write [21 18] INFO [2019-12-17 12:31:15,217] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26064,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:31:15,219] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26064,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:31:15,221] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26056,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:31:15,223] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26056,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:31:15,223] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26066,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:31:15,226] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26057,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:31:15,228] jepsen worker 21 - jepsen.util 5781 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:31:15,229] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26066,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:31:15,230] jepsen worker 21 - jepsen.util 5781 :invoke :write [7 13] INFO [2019-12-17 12:31:15,230] jepsen worker 80 - jepsen.util 4580 :ok :read [26 42] INFO [2019-12-17 12:31:15,230] jepsen worker 9 - jepsen.util 5229 :fail :read [3 nil] :RequestTimeoutException INFO [2019-12-17 12:31:15,230] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26057,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:31:15,231] jepsen worker 80 - jepsen.util 4580 :invoke :read [26 nil] INFO [2019-12-17 12:31:15,232] jepsen worker 9 - jepsen.util 5229 :invoke :write [3 28] INFO [2019-12-17 12:31:15,241] jepsen worker 54 - jepsen.util 5004 :info :write [18 44] :RequestCanceledException INFO [2019-12-17 12:31:15,241] jepsen worker 55 - jepsen.util 5545 :fail :read [18 nil] :RequestTimeoutException INFO [2019-12-17 12:31:15,242] jepsen worker 55 - jepsen.util 5545 :invoke :cas [18 [37 4]] INFO [2019-12-17 12:31:15,277] jepsen worker 54 - jepsen.util 5094 :invoke :write [18 18] INFO [2019-12-17 12:31:15,278] jepsen worker 46 - jepsen.util 4726 :fail :cas [15 [11 17]] :ValueNotSwapFrom INFO [2019-12-17 12:31:15,279] jepsen worker 46 - jepsen.util 4726 :invoke :read [15 nil] INFO [2019-12-17 12:31:15,301] jepsen worker 61 - jepsen.util 6631 :info :cas [20 [17 33]] :RequestCanceledException INFO [2019-12-17 12:31:15,310] jepsen worker 37 - jepsen.util 4447 :fail :read [12 nil] :RequestTimeoutException INFO [2019-12-17 12:31:15,311] jepsen worker 29 - jepsen.util 3719 :ok :read [9 8] INFO [2019-12-17 12:31:15,319] jepsen worker 37 - jepsen.util 4447 :invoke :read [12 nil] INFO [2019-12-17 12:31:15,320] jepsen worker 29 - jepsen.util 3719 :invoke :write [9 29] INFO [2019-12-17 12:31:15,355] jepsen worker 36 - jepsen.util 4626 :info :write [12 0] :RequestCanceledException INFO [2019-12-17 12:31:15,362] jepsen worker 64 - jepsen.util 3574 :ok :read [21 45] INFO [2019-12-17 12:31:15,365] jepsen worker 64 - jepsen.util 3574 :invoke :read [21 nil] INFO [2019-12-17 12:31:15,374] jepsen worker 61 - jepsen.util 6721 :invoke :read [20 nil] INFO [2019-12-17 12:31:15,385] jepsen worker 30 - jepsen.util 4800 :ok :write [10 0] INFO [2019-12-17 12:31:15,385] jepsen worker 30 - jepsen.util 4800 :invoke :read [10 nil] INFO [2019-12-17 12:31:15,396] jepsen worker 4 - jepsen.util 3784 :ok :write [1 4] INFO [2019-12-17 12:31:15,397] jepsen worker 4 - jepsen.util 3784 :invoke :cas [1 [12 26]] INFO [2019-12-17 12:31:15,410] jepsen worker 36 - jepsen.util 4716 :invoke :read [12 nil] INFO [2019-12-17 12:31:15,412] jepsen worker 86 - jepsen.util 6296 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:31:15,412] jepsen worker 86 - jepsen.util 6296 :invoke :cas [28 [48 28]] INFO [2019-12-17 12:31:15,429] jepsen worker 47 - jepsen.util 4997 :info :cas [15 [30 35]] :RequestCanceledException INFO [2019-12-17 12:31:15,446] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26087,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:31:15,452] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26087,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:31:15,455] jepsen worker 70 - jepsen.util 5290 :fail :read [23 nil] :RequestTimeoutException INFO [2019-12-17 12:31:15,455] jepsen worker 70 - jepsen.util 5290 :invoke :cas [23 [7 44]] INFO [2019-12-17 12:31:15,455] jepsen worker 47 - jepsen.util 5087 :invoke :write [15 35] INFO [2019-12-17 12:31:15,536] jepsen worker 24 - jepsen.util 5694 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:31:15,539] jepsen worker 24 - jepsen.util 5694 :invoke :cas [8 [26 14]] INFO [2019-12-17 12:31:15,624] jepsen worker 49 - jepsen.util 5359 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:31:15,624] jepsen worker 49 - jepsen.util 5359 :invoke :read [16 nil] INFO [2019-12-17 12:31:15,664] jepsen worker 11 - jepsen.util 4691 :ok :read [3 40] INFO [2019-12-17 12:31:15,665] jepsen worker 11 - jepsen.util 4691 :invoke :write [3 31] INFO [2019-12-17 12:31:15,683] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26099,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:31:15,683] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26099,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:31:15,685] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26101,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:31:15,686] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26101,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:31:15,756] jepsen worker 69 - jepsen.util 4749 :ok :write [23 23] INFO [2019-12-17 12:31:15,756] jepsen worker 69 - jepsen.util 4749 :invoke :write [23 47] INFO [2019-12-17 12:31:15,809] jepsen worker 78 - jepsen.util 4668 :ok :read [26 37] INFO [2019-12-17 12:31:15,809] jepsen worker 78 - jepsen.util 4668 :invoke :write [26 1] INFO [2019-12-17 12:31:15,830] jepsen worker 13 - jepsen.util 4243 :ok :write [4 21] INFO [2019-12-17 12:31:15,830] jepsen worker 13 - jepsen.util 4243 :invoke :cas [4 [25 15]] INFO [2019-12-17 12:31:15,852] jepsen worker 79 - jepsen.util 4489 :fail :cas [26 [34 7]] :ValueNotSwapFrom INFO [2019-12-17 12:31:15,852] jepsen worker 79 - jepsen.util 4489 :invoke :read [26 nil] INFO [2019-12-17 12:31:15,853] jepsen worker 34 - jepsen.util 5614 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:31:15,854] jepsen worker 34 - jepsen.util 5614 :invoke :read [11 nil] INFO [2019-12-17 12:31:15,857] jepsen worker 28 - jepsen.util 3448 :ok :read [9 27] INFO [2019-12-17 12:31:15,858] jepsen worker 28 - jepsen.util 3448 :invoke :write [9 25] INFO [2019-12-17 12:31:15,893] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26109,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:31:15,896] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:31:15,896] jepsen nemesis - couchbase.util Rebalance check count: 194 INFO [2019-12-17 12:31:15,897] jepsen nemesis - couchbase.util Rebalance stuck count: 175 INFO [2019-12-17 12:31:15,899] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26109,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:31:15,901] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26113,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:31:15,902] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26113,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:31:15,903] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26114,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:31:15,905] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26114,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:31:15,933] jepsen worker 56 - jepsen.util 4736 :ok :write [18 7] INFO [2019-12-17 12:31:15,933] jepsen worker 56 - jepsen.util 4736 :invoke :read [18 nil] INFO [2019-12-17 12:31:15,957] jepsen worker 68 - jepsen.util 5468 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:31:15,957] jepsen worker 68 - jepsen.util 5468 :invoke :cas [22 [4 46]] INFO [2019-12-17 12:31:16,016] jepsen worker 23 - jepsen.util 5963 :info :cas [7 [39 34]] :RequestCanceledException INFO [2019-12-17 12:31:16,042] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26120,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:31:16,042] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26120,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:31:16,063] jepsen worker 51 - jepsen.util 4281 :ok :write [17 47] INFO [2019-12-17 12:31:16,063] jepsen worker 51 - jepsen.util 4281 :invoke :write [17 38] INFO [2019-12-17 12:31:16,077] jepsen worker 23 - jepsen.util 6053 :invoke :cas [7 [48 43]] INFO [2019-12-17 12:31:16,102] jepsen worker 54 - jepsen.util 5094 :ok :write [18 18] INFO [2019-12-17 12:31:16,102] jepsen worker 54 - jepsen.util 5094 :invoke :cas [18 [31 4]] INFO [2019-12-17 12:31:16,140] jepsen worker 3 - jepsen.util 4773 :ok :read [1 7] INFO [2019-12-17 12:31:16,141] jepsen worker 3 - jepsen.util 4773 :invoke :read [1 nil] INFO [2019-12-17 12:31:16,143] jepsen worker 74 - jepsen.util 4934 :ok :read [24 45] INFO [2019-12-17 12:31:16,143] jepsen worker 74 - jepsen.util 4934 :invoke :cas [24 [0 23]] INFO [2019-12-17 12:31:16,149] jepsen worker 63 - jepsen.util 3663 :fail :cas [21 [18 3]] :ValueNotSwapFrom INFO [2019-12-17 12:31:16,150] jepsen worker 63 - jepsen.util 3663 :invoke :write [21 38] INFO [2019-12-17 12:31:16,159] jepsen worker 88 - jepsen.util 4768 :fail :cas [29 [5 45]] :ValueNotSwapFrom INFO [2019-12-17 12:31:16,160] jepsen worker 88 - jepsen.util 4768 :invoke :cas [29 [23 4]] INFO [2019-12-17 12:31:16,206] jepsen worker 89 - jepsen.util 4229 :ok :write [29 5] INFO [2019-12-17 12:31:16,207] jepsen worker 89 - jepsen.util 4229 :invoke :read [29 nil] INFO [2019-12-17 12:31:16,227] jepsen worker 6 - jepsen.util 6486 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:31:16,227] jepsen worker 43 - jepsen.util 5713 :info :cas [14 [49 40]] :RequestCanceledException INFO [2019-12-17 12:31:16,227] jepsen worker 59 - jepsen.util 6179 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:31:16,228] jepsen worker 6 - jepsen.util 6486 :invoke :cas [2 [15 17]] INFO [2019-12-17 12:31:16,228] jepsen worker 59 - jepsen.util 6179 :invoke :read [19 nil] INFO [2019-12-17 12:31:16,242] jepsen worker 44 - jepsen.util 6524 :info :cas [14 [7 3]] :RequestCanceledException INFO [2019-12-17 12:31:16,243] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26141,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:31:16,244] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26130,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:31:16,246] jepsen worker 77 - jepsen.util 6107 :info :write [25 22] :RequestCanceledException INFO [2019-12-17 12:31:16,246] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26141,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:31:16,246] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26130,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:31:16,248] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26142,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:31:16,251] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26135,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:31:16,252] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26142,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:31:16,253] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26135,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:31:16,253] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26143,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:31:16,258] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26136,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:31:16,259] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26136,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:31:16,260] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26143,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:31:16,270] jepsen worker 10 - jepsen.util 4420 :fail :cas [3 [35 17]] :ValueNotSwapFrom INFO [2019-12-17 12:31:16,280] jepsen worker 10 - jepsen.util 4420 :invoke :write [3 22] INFO [2019-12-17 12:31:16,286] jepsen worker 87 - jepsen.util 4317 :fail :cas [29 [13 28]] :ValueNotSwapFrom INFO [2019-12-17 12:31:16,291] jepsen worker 87 - jepsen.util 4317 :invoke :read [29 nil] INFO [2019-12-17 12:31:16,294] jepsen worker 43 - jepsen.util 5803 :invoke :write [14 8] INFO [2019-12-17 12:31:16,305] jepsen worker 19 - jepsen.util 4699 :ok :read [6 33] INFO [2019-12-17 12:31:16,306] jepsen worker 19 - jepsen.util 4699 :invoke :cas [6 [0 7]] INFO [2019-12-17 12:31:16,306] jepsen worker 44 - jepsen.util 6614 :invoke :write [14 48] INFO [2019-12-17 12:31:16,308] jepsen worker 66 - jepsen.util 5556 :info :cas [22 [47 38]] :RequestCanceledException INFO [2019-12-17 12:31:16,308] jepsen worker 5 - jepsen.util 4145 :ok :read [1 7] INFO [2019-12-17 12:31:16,310] jepsen worker 5 - jepsen.util 4145 :invoke :read [1 nil] INFO [2019-12-17 12:31:16,317] jepsen worker 77 - jepsen.util 6197 :invoke :write [25 8] INFO [2019-12-17 12:31:16,326] jepsen worker 39 - jepsen.util 5529 :info :cas [13 [20 5]] :RequestCanceledException INFO [2019-12-17 12:31:16,329] jepsen worker 66 - jepsen.util 5646 :invoke :read [22 nil] INFO [2019-12-17 12:31:16,337] jepsen worker 82 - jepsen.util 6742 :info :cas [27 [4 10]] :RequestCanceledException INFO [2019-12-17 12:31:16,346] jepsen worker 39 - jepsen.util 5619 :invoke :cas [13 [18 2]] INFO [2019-12-17 12:31:16,357] jepsen worker 4 - jepsen.util 3784 :fail :cas [1 [12 26]] :ValueNotSwapFrom INFO [2019-12-17 12:31:16,357] jepsen worker 4 - jepsen.util 3784 :invoke :cas [1 [36 40]] INFO [2019-12-17 12:31:16,392] jepsen worker 82 - jepsen.util 6832 :invoke :cas [27 [16 36]] INFO [2019-12-17 12:31:16,420] jepsen worker 69 - jepsen.util 4749 :ok :write [23 47] INFO [2019-12-17 12:31:16,421] jepsen worker 69 - jepsen.util 4749 :invoke :read [23 nil] INFO [2019-12-17 12:31:16,437] jepsen worker 11 - jepsen.util 4691 :ok :write [3 31] INFO [2019-12-17 12:31:16,438] jepsen worker 11 - jepsen.util 4691 :invoke :read [3 nil] INFO [2019-12-17 12:31:16,468] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26157,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:31:16,469] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26157,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:31:16,470] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26160,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:31:16,470] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26160,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:31:16,470] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26161,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:31:16,475] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26161,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:31:16,490] jepsen worker 58 - jepsen.util 6088 :info :cas [19 [31 4]] :RequestCanceledException INFO [2019-12-17 12:31:16,511] jepsen worker 35 - jepsen.util 6155 :info :cas [11 [47 30]] :RequestCanceledException INFO [2019-12-17 12:31:16,511] jepsen worker 85 - jepsen.util 6655 :info :write [28 40] :RequestCanceledException INFO [2019-12-17 12:31:16,529] jepsen worker 84 - jepsen.util 5574 :info :write [28 16] :RequestCanceledException INFO [2019-12-17 12:31:16,529] jepsen worker 83 - jepsen.util 6383 :info :cas [27 [39 48]] :RequestCanceledException INFO [2019-12-17 12:31:16,535] jepsen worker 58 - jepsen.util 6178 :invoke :cas [19 [45 31]] INFO [2019-12-17 12:31:16,536] jepsen worker 46 - jepsen.util 4726 :ok :read [15 7] INFO [2019-12-17 12:31:16,537] jepsen worker 46 - jepsen.util 4726 :invoke :read [15 nil] INFO [2019-12-17 12:31:16,572] jepsen worker 35 - jepsen.util 6245 :invoke :read [11 nil] INFO [2019-12-17 12:31:16,578] jepsen worker 72 - jepsen.util 4662 :fail :cas [24 [25 24]] :ValueNotSwapFrom INFO [2019-12-17 12:31:16,579] jepsen worker 83 - jepsen.util 6473 :invoke :cas [27 [0 41]] INFO [2019-12-17 12:31:16,579] jepsen worker 72 - jepsen.util 4662 :invoke :read [24 nil] INFO [2019-12-17 12:31:16,579] jepsen worker 85 - jepsen.util 6745 :invoke :cas [28 [41 41]] INFO [2019-12-17 12:31:16,583] jepsen worker 64 - jepsen.util 3574 :ok :read [21 30] INFO [2019-12-17 12:31:16,586] jepsen worker 64 - jepsen.util 3574 :invoke :read [21 nil] INFO [2019-12-17 12:31:16,592] jepsen worker 2 - jepsen.util 5582 :info :cas [0 [34 34]] :RequestCanceledException INFO [2019-12-17 12:31:16,593] jepsen worker 84 - jepsen.util 5664 :invoke :cas [28 [36 41]] INFO [2019-12-17 12:31:16,642] jepsen worker 60 - jepsen.util 6450 :info :cas [20 [31 45]] :RequestCanceledException INFO [2019-12-17 12:31:16,667] jepsen worker 25 - jepsen.util 6145 :info :write [8 20] :RequestCanceledException INFO [2019-12-17 12:31:16,704] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26173,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:31:16,711] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26173,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:31:16,712] jepsen worker 62 - jepsen.util 5642 :info :write [20 46] :RequestCanceledException INFO [2019-12-17 12:31:16,712] jepsen worker 7 - jepsen.util 5947 :info :write [2 8] :RequestCanceledException INFO [2019-12-17 12:31:16,715] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26177,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:31:16,717] jepsen worker 57 - jepsen.util 5817 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:31:16,718] jepsen worker 57 - jepsen.util 5817 :invoke :write [19 27] INFO [2019-12-17 12:31:16,725] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26177,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:31:16,732] jepsen worker 48 - jepsen.util 5448 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:31:16,737] jepsen worker 2 - jepsen.util 5672 :invoke :read [0 nil] INFO [2019-12-17 12:31:16,737] jepsen worker 50 - jepsen.util 6260 :info :cas [16 [47 36]] :RequestCanceledException INFO [2019-12-17 12:31:16,738] jepsen worker 48 - jepsen.util 5448 :invoke :write [16 25] INFO [2019-12-17 12:31:16,747] jepsen worker 22 - jepsen.util 6052 :info :cas [7 [3 2]] :RequestCanceledException INFO [2019-12-17 12:31:16,788] jepsen worker 60 - jepsen.util 6540 :invoke :write [20 38] INFO [2019-12-17 12:31:16,789] jepsen worker 25 - jepsen.util 6235 :invoke :read [8 nil] INFO [2019-12-17 12:31:16,791] jepsen worker 7 - jepsen.util 6037 :invoke :cas [2 [26 12]] INFO [2019-12-17 12:31:16,801] jepsen worker 50 - jepsen.util 6350 :invoke :read [16 nil] INFO [2019-12-17 12:31:16,804] jepsen worker 62 - jepsen.util 5732 :invoke :cas [20 [2 17]] INFO [2019-12-17 12:31:16,808] jepsen worker 22 - jepsen.util 6142 :invoke :cas [7 [18 49]] INFO [2019-12-17 12:31:16,812] jepsen worker 26 - jepsen.util 6146 :info :cas [8 [23 34]] :RequestCanceledException INFO [2019-12-17 12:31:16,820] jepsen worker 28 - jepsen.util 3448 :ok :write [9 25] INFO [2019-12-17 12:31:16,821] jepsen worker 28 - jepsen.util 3448 :invoke :read [9 nil] INFO [2019-12-17 12:31:16,823] jepsen worker 75 - jepsen.util 6015 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:31:16,825] jepsen worker 75 - jepsen.util 6015 :invoke :cas [25 [40 7]] INFO [2019-12-17 12:31:16,827] jepsen worker 32 - jepsen.util 4802 :info :cas [10 [49 10]] :RequestCanceledException INFO [2019-12-17 12:31:16,827] jepsen worker 27 - jepsen.util 4347 :fail :read [9 nil] :RequestTimeoutException INFO [2019-12-17 12:31:16,827] jepsen worker 27 - jepsen.util 4347 :invoke :write [9 0] INFO [2019-12-17 12:31:16,829] jepsen worker 20 - jepsen.util 4610 :info :cas [6 [21 13]] :RequestCanceledException INFO [2019-12-17 12:31:16,832] jepsen worker 42 - jepsen.util 6702 :info :cas [14 [16 23]] :RequestCanceledException INFO [2019-12-17 12:31:16,835] jepsen worker 17 - jepsen.util 5867 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:31:16,851] jepsen worker 52 - jepsen.util 4912 :info :cas [17 [6 44]] :RequestCanceledException INFO [2019-12-17 12:31:16,851] jepsen worker 17 - jepsen.util 5867 :invoke :cas [5 [5 3]] INFO [2019-12-17 12:31:16,857] jepsen worker 73 - jepsen.util 4303 :info :cas [24 [8 49]] :RequestCanceledException INFO [2019-12-17 12:31:16,865] jepsen worker 14 - jepsen.util 4694 :info :cas [4 [32 19]] :RequestCanceledException INFO [2019-12-17 12:31:16,874] jepsen worker 26 - jepsen.util 6236 :invoke :write [8 19] INFO [2019-12-17 12:31:16,879] jepsen worker 38 - jepsen.util 4538 :ok :write [12 25] INFO [2019-12-17 12:31:16,879] jepsen worker 38 - jepsen.util 4538 :invoke :read [12 nil] INFO [2019-12-17 12:31:16,879] jepsen worker 52 - jepsen.util 5002 :invoke :read [17 nil] INFO [2019-12-17 12:31:16,895] jepsen worker 20 - jepsen.util 4700 :invoke :cas [6 [11 1]] INFO [2019-12-17 12:31:16,906] jepsen worker 14 - jepsen.util 4784 :invoke :cas [4 [28 15]] INFO [2019-12-17 12:31:16,907] jepsen worker 32 - jepsen.util 4892 :invoke :read [10 nil] INFO [2019-12-17 12:31:16,933] jepsen worker 18 - jepsen.util 4518 :info :write [6 39] :RequestCanceledException INFO [2019-12-17 12:31:16,933] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26192,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:31:16,939] jepsen worker 73 - jepsen.util 4393 :invoke :cas [24 [16 47]] INFO [2019-12-17 12:31:16,940] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26192,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:31:16,945] jepsen worker 41 - jepsen.util 6431 :info :cas [13 [43 37]] :RequestCanceledException INFO [2019-12-17 12:31:16,946] jepsen worker 40 - jepsen.util 6790 :info :write [13 30] :RequestCanceledException INFO [2019-12-17 12:31:16,980] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:31:16,984] jepsen nemesis - couchbase.util Rebalance check count: 195 INFO [2019-12-17 12:31:16,985] jepsen nemesis - couchbase.util Rebalance stuck count: 176 INFO [2019-12-17 12:31:16,998] jepsen worker 33 - jepsen.util 5973 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:31:16,998] jepsen worker 33 - jepsen.util 5973 :invoke :write [11 17] INFO [2019-12-17 12:31:17,007] jepsen worker 65 - jepsen.util 3935 :ok :write [21 18] INFO [2019-12-17 12:31:17,009] jepsen worker 65 - jepsen.util 3935 :invoke :read [21 nil] INFO [2019-12-17 12:31:17,023] jepsen worker 42 - jepsen.util 6792 :invoke :cas [14 [31 42]] INFO [2019-12-17 12:31:17,025] jepsen worker 41 - jepsen.util 6521 :invoke :read [13 nil] INFO [2019-12-17 12:31:17,036] jepsen worker 40 - jepsen.util 6880 :invoke :write [13 17] INFO [2019-12-17 12:31:17,036] jepsen worker 18 - jepsen.util 4608 :invoke :cas [6 [46 10]] INFO [2019-12-17 12:31:17,065] jepsen worker 12 - jepsen.util 4782 :info :cas [4 [2 38]] :RequestCanceledException INFO [2019-12-17 12:31:17,079] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26204,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:31:17,080] jepsen worker 29 - jepsen.util 3719 :ok :write [9 29] INFO [2019-12-17 12:31:17,094] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26204,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:31:17,094] jepsen worker 29 - jepsen.util 3719 :invoke :cas [9 [24 49]] INFO [2019-12-17 12:31:17,101] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26208,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:31:17,112] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26208,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:31:17,116] jepsen worker 67 - jepsen.util 6187 :info :write [22 4] :RequestCanceledException INFO [2019-12-17 12:31:17,116] jepsen worker 15 - jepsen.util 5145 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:31:17,119] jepsen worker 15 - jepsen.util 5145 :invoke :read [5 nil] INFO [2019-12-17 12:31:17,127] jepsen worker 12 - jepsen.util 4872 :invoke :cas [4 [24 38]] INFO [2019-12-17 12:31:17,144] jepsen worker 71 - jepsen.util 4931 :info :cas [23 [16 24]] :RequestCanceledException INFO [2019-12-17 12:31:17,145] jepsen worker 78 - jepsen.util 4668 :ok :write [26 1] INFO [2019-12-17 12:31:17,146] jepsen worker 78 - jepsen.util 4668 :invoke :write [26 32] INFO [2019-12-17 12:31:17,153] jepsen worker 13 - jepsen.util 4243 :fail :cas [4 [25 15]] :ValueNotSwapFrom INFO [2019-12-17 12:31:17,154] jepsen worker 13 - jepsen.util 4243 :invoke :read [4 nil] INFO [2019-12-17 12:31:17,167] jepsen worker 0 - jepsen.util 6660 :info :write [0 44] :RequestCanceledException INFO [2019-12-17 12:31:17,188] jepsen worker 71 - jepsen.util 5021 :invoke :read [23 nil] INFO [2019-12-17 12:31:17,203] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26219,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:31:17,211] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26219,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:31:17,241] jepsen worker 67 - jepsen.util 6277 :invoke :read [22 nil] INFO [2019-12-17 12:31:17,313] jepsen worker 0 - jepsen.util 6750 :invoke :write [0 42] INFO [2019-12-17 12:31:17,318] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26228,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:31:17,318] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26228,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:31:17,331] jepsen worker 76 - jepsen.util 6106 :info :write [25 2] :RequestCanceledException INFO [2019-12-17 12:31:17,332] jepsen worker 16 - jepsen.util 6856 :info :write [5 28] :RequestCanceledException INFO [2019-12-17 12:31:17,332] jepsen worker 31 - jepsen.util 4351 :info :write [10 15] :RequestCanceledException INFO [2019-12-17 12:31:17,338] jepsen worker 45 - jepsen.util 4725 :info :write [15 22] :RequestCanceledException INFO [2019-12-17 12:31:17,353] jepsen worker 8 - jepsen.util 5318 :info :write [2 3] :RequestCanceledException INFO [2019-12-17 12:31:17,391] jepsen worker 45 - jepsen.util 4815 :invoke :cas [15 [8 29]] INFO [2019-12-17 12:31:17,394] jepsen worker 1 - jepsen.util 5671 :info :write [0 35] :RequestCanceledException INFO [2019-12-17 12:31:17,400] jepsen worker 61 - jepsen.util 6721 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:31:17,404] jepsen worker 61 - jepsen.util 6721 :invoke :write [20 41] INFO [2019-12-17 12:31:17,407] jepsen worker 31 - jepsen.util 4441 :invoke :read [10 nil] INFO [2019-12-17 12:31:17,437] jepsen worker 30 - jepsen.util 4800 :fail :read [10 nil] :RequestTimeoutException INFO [2019-12-17 12:31:17,445] jepsen worker 30 - jepsen.util 4800 :invoke :read [10 nil] INFO [2019-12-17 12:31:17,450] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26240,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:31:17,454] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26240,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:31:17,456] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26241,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:31:17,466] jepsen worker 81 - jepsen.util 6021 :info :write [27 26] :RequestCanceledException INFO [2019-12-17 12:31:17,467] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26241,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:31:17,468] jepsen worker 76 - jepsen.util 6196 :invoke :cas [25 [13 34]] INFO [2019-12-17 12:31:17,505] jepsen worker 1 - jepsen.util 5761 :invoke :cas [0 [8 24]] INFO [2019-12-17 12:31:17,513] jepsen worker 8 - jepsen.util 5408 :invoke :write [2 23] INFO [2019-12-17 12:31:17,516] jepsen worker 86 - jepsen.util 6296 :info :cas [28 [48 28]] :RequestCanceledException INFO [2019-12-17 12:31:17,519] jepsen worker 16 - jepsen.util 6946 :invoke :cas [5 [33 9]] INFO [2019-12-17 12:31:17,592] jepsen worker 86 - jepsen.util 6386 :invoke :read [28 nil] INFO [2019-12-17 12:31:17,603] jepsen worker 81 - jepsen.util 6111 :invoke :read [27 nil] INFO [2019-12-17 12:31:17,626] jepsen worker 10 - jepsen.util 4420 :ok :write [3 22] INFO [2019-12-17 12:31:17,627] jepsen worker 10 - jepsen.util 4420 :invoke :write [3 49] INFO [2019-12-17 12:31:17,629] jepsen worker 24 - jepsen.util 5694 :info :cas [8 [26 14]] :RequestCanceledException INFO [2019-12-17 12:31:17,659] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26252,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:31:17,679] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26252,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:31:17,730] jepsen worker 9 - jepsen.util 5229 :info :write [3 28] :RequestCanceledException INFO [2019-12-17 12:31:17,736] jepsen worker 49 - jepsen.util 5359 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:31:17,738] jepsen worker 47 - jepsen.util 5087 :info :write [15 35] :RequestCanceledException INFO [2019-12-17 12:31:17,751] jepsen worker 49 - jepsen.util 5359 :invoke :cas [16 [3 46]] INFO [2019-12-17 12:31:17,979] jepsen worker 24 - jepsen.util 5784 :invoke :write [8 26] INFO [2019-12-17 12:31:17,988] jepsen worker 53 - jepsen.util 4553 :fail :cas [17 [48 41]] :ValueNotSwapFrom INFO [2019-12-17 12:31:17,988] jepsen worker 3 - jepsen.util 4773 :ok :read [1 4] INFO [2019-12-17 12:31:17,988] jepsen worker 36 - jepsen.util 4716 :ok :read [12 45] INFO [2019-12-17 12:31:17,988] jepsen worker 80 - jepsen.util 4580 :ok :read [26 37] INFO [2019-12-17 12:31:17,995] jepsen worker 79 - jepsen.util 4489 :ok :read [26 37] INFO [2019-12-17 12:31:17,996] jepsen worker 55 - jepsen.util 5545 :fail :cas [18 [37 4]] :ValueNotSwapFrom INFO [2019-12-17 12:31:18,001] jepsen worker 21 - jepsen.util 5781 :info :write [7 13] :RequestCanceledException INFO [2019-12-17 12:31:18,002] jepsen worker 80 - jepsen.util 4580 :invoke :write [26 9] INFO [2019-12-17 12:31:18,004] jepsen worker 53 - jepsen.util 4553 :invoke :cas [17 [47 26]] INFO [2019-12-17 12:31:18,021] jepsen worker 79 - jepsen.util 4489 :invoke :cas [26 [8 36]] INFO [2019-12-17 12:31:18,023] jepsen worker 36 - jepsen.util 4716 :invoke :write [12 21] INFO [2019-12-17 12:31:18,032] jepsen worker 68 - jepsen.util 5468 :info :cas [22 [4 46]] :RequestCanceledException INFO [2019-12-17 12:31:18,040] jepsen worker 34 - jepsen.util 5614 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:31:18,041] jepsen worker 55 - jepsen.util 5545 :invoke :cas [18 [32 12]] INFO [2019-12-17 12:31:18,050] jepsen worker 34 - jepsen.util 5614 :invoke :write [11 49] INFO [2019-12-17 12:31:18,061] jepsen worker 3 - jepsen.util 4773 :invoke :cas [1 [48 22]] INFO [2019-12-17 12:31:18,082] jepsen worker 47 - jepsen.util 5177 :invoke :write [15 32] INFO [2019-12-17 12:31:18,109] jepsen worker 9 - jepsen.util 5319 :invoke :cas [3 [26 1]] INFO [2019-12-17 12:31:18,109] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26263,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:31:18,122] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26263,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:31:18,134] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26268,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:31:18,134] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26274,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:31:18,141] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26274,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:31:18,142] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26268,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:31:18,166] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:31:18,171] jepsen worker 23 - jepsen.util 6053 :info :cas [7 [48 43]] :RequestCanceledException INFO [2019-12-17 12:31:18,174] jepsen nemesis - couchbase.util Rebalance check count: 196 INFO [2019-12-17 12:31:18,179] jepsen nemesis - couchbase.util Rebalance stuck count: 177 INFO [2019-12-17 12:31:18,231] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26278,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:31:18,234] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26278,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:31:18,236] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26286,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:31:18,237] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26286,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:31:18,269] jepsen worker 89 - jepsen.util 4229 :fail :read [29 nil] :RequestTimeoutException INFO [2019-12-17 12:31:18,277] jepsen worker 89 - jepsen.util 4229 :invoke :write [29 45] INFO [2019-12-17 12:31:18,280] jepsen worker 21 - jepsen.util 5871 :invoke :read [7 nil] INFO [2019-12-17 12:31:18,290] jepsen worker 68 - jepsen.util 5558 :invoke :write [22 0] INFO [2019-12-17 12:31:18,295] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26287,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:31:18,299] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26287,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:31:18,306] jepsen worker 23 - jepsen.util 6143 :invoke :cas [7 [46 17]] INFO [2019-12-17 12:31:18,307] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26289,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:31:18,313] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26289,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:31:18,327] jepsen worker 6 - jepsen.util 6486 :info :cas [2 [15 17]] :RequestCanceledException INFO [2019-12-17 12:31:18,340] jepsen worker 59 - jepsen.util 6179 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:31:18,350] jepsen worker 59 - jepsen.util 6179 :invoke :read [19 nil] INFO [2019-12-17 12:31:18,358] jepsen worker 51 - jepsen.util 4281 :info :write [17 38] :RequestCanceledException INFO [2019-12-17 12:31:18,411] jepsen worker 66 - jepsen.util 5646 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:31:18,413] jepsen worker 66 - jepsen.util 5646 :invoke :cas [22 [18 23]] INFO [2019-12-17 12:31:18,426] jepsen worker 39 - jepsen.util 5619 :info :cas [13 [18 2]] :RequestCanceledException INFO [2019-12-17 12:31:18,428] jepsen worker 6 - jepsen.util 6576 :invoke :write [2 41] INFO [2019-12-17 12:31:18,450] jepsen worker 82 - jepsen.util 6832 :info :cas [27 [16 36]] :RequestCanceledException INFO [2019-12-17 12:31:18,493] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26296,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:31:18,515] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26296,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:31:18,516] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26298,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:31:18,522] jepsen worker 51 - jepsen.util 4371 :invoke :write [17 22] INFO [2019-12-17 12:31:18,531] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26298,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:31:18,575] jepsen worker 58 - jepsen.util 6178 :info :cas [19 [45 31]] :RequestCanceledException INFO [2019-12-17 12:31:18,607] jepsen worker 83 - jepsen.util 6473 :info :cas [27 [0 41]] :RequestCanceledException INFO [2019-12-17 12:31:18,612] jepsen worker 85 - jepsen.util 6745 :info :cas [28 [41 41]] :RequestCanceledException INFO [2019-12-17 12:31:18,614] jepsen worker 82 - jepsen.util 6922 :invoke :cas [27 [1 5]] INFO [2019-12-17 12:31:18,648] jepsen worker 35 - jepsen.util 6245 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:31:18,648] jepsen worker 72 - jepsen.util 4662 :fail :read [24 nil] :RequestTimeoutException INFO [2019-12-17 12:31:18,649] jepsen worker 84 - jepsen.util 5664 :info :cas [28 [36 41]] :RequestCanceledException INFO [2019-12-17 12:31:18,651] jepsen worker 72 - jepsen.util 4662 :invoke :write [24 44] INFO [2019-12-17 12:31:18,651] jepsen worker 35 - jepsen.util 6245 :invoke :cas [11 [8 26]] INFO [2019-12-17 12:31:18,673] jepsen worker 39 - jepsen.util 5709 :invoke :write [13 21] INFO [2019-12-17 12:31:18,693] jepsen worker 58 - jepsen.util 6268 :invoke :write [19 33] INFO [2019-12-17 12:31:18,746] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26307,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:31:18,748] jepsen worker 85 - jepsen.util 6835 :invoke :write [28 3] INFO [2019-12-17 12:31:18,748] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26307,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:31:18,749] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26309,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:31:18,751] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26309,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:31:18,752] jepsen worker 83 - jepsen.util 6563 :invoke :cas [27 [14 10]] INFO [2019-12-17 12:31:18,761] jepsen worker 84 - jepsen.util 5754 :invoke :write [28 1] INFO [2019-12-17 12:31:18,767] jepsen worker 29 - jepsen.util 3719 :fail :cas [9 [24 49]] :ValueNotSwapFrom INFO [2019-12-17 12:31:18,768] jepsen worker 29 - jepsen.util 3719 :invoke :read [9 nil] INFO [2019-12-17 12:31:18,770] jepsen worker 64 - jepsen.util 3574 :ok :read [21 18] INFO [2019-12-17 12:31:18,770] jepsen worker 74 - jepsen.util 4934 :fail :cas [24 [0 23]] :ValueNotSwapFrom INFO [2019-12-17 12:31:18,770] jepsen worker 19 - jepsen.util 4699 :fail :cas [6 [0 7]] :ValueNotSwapFrom INFO [2019-12-17 12:31:18,770] jepsen worker 64 - jepsen.util 3574 :invoke :cas [21 [2 40]] INFO [2019-12-17 12:31:18,771] jepsen worker 74 - jepsen.util 4934 :invoke :cas [24 [22 1]] INFO [2019-12-17 12:31:18,771] jepsen worker 19 - jepsen.util 4699 :invoke :read [6 nil] INFO [2019-12-17 12:31:18,774] jepsen worker 88 - jepsen.util 4768 :fail :cas [29 [23 4]] :ValueNotSwapFrom INFO [2019-12-17 12:31:18,774] jepsen worker 88 - jepsen.util 4768 :invoke :read [29 nil] INFO [2019-12-17 12:31:18,774] jepsen worker 37 - jepsen.util 4447 :ok :read [12 25] INFO [2019-12-17 12:31:18,775] jepsen worker 37 - jepsen.util 4447 :invoke :write [12 4] INFO [2019-12-17 12:31:18,775] jepsen worker 54 - jepsen.util 5094 :fail :cas [18 [31 4]] :ValueNotSwapFrom INFO [2019-12-17 12:31:18,775] jepsen worker 54 - jepsen.util 5094 :invoke :read [18 nil] INFO [2019-12-17 12:31:18,782] jepsen worker 5 - jepsen.util 4145 :ok :read [1 4] INFO [2019-12-17 12:31:18,782] jepsen worker 5 - jepsen.util 4145 :invoke :cas [1 [12 22]] INFO [2019-12-17 12:31:18,824] jepsen worker 62 - jepsen.util 5732 :info :cas [20 [2 17]] :RequestCanceledException INFO [2019-12-17 12:31:18,827] jepsen worker 25 - jepsen.util 6235 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:31:18,828] jepsen worker 25 - jepsen.util 6235 :invoke :cas [8 [1 14]] INFO [2019-12-17 12:31:18,841] jepsen worker 2 - jepsen.util 5672 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:31:18,842] jepsen worker 2 - jepsen.util 5672 :invoke :cas [0 [34 11]] INFO [2019-12-17 12:31:18,842] jepsen worker 22 - jepsen.util 6142 :info :cas [7 [18 49]] :RequestCanceledException INFO [2019-12-17 12:31:18,861] jepsen worker 43 - jepsen.util 5803 :info :write [14 8] :RequestCanceledException INFO [2019-12-17 12:31:18,867] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26334,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:31:18,867] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26334,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:31:18,897] jepsen worker 77 - jepsen.util 6197 :info :write [25 8] :RequestCanceledException INFO [2019-12-17 12:31:18,910] jepsen worker 7 - jepsen.util 6037 :info :cas [2 [26 12]] :RequestCanceledException INFO [2019-12-17 12:31:18,911] jepsen worker 78 - jepsen.util 4668 :ok :write [26 32] INFO [2019-12-17 12:31:18,914] jepsen worker 50 - jepsen.util 6350 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:31:18,915] jepsen worker 50 - jepsen.util 6350 :invoke :cas [16 [15 49]] INFO [2019-12-17 12:31:18,916] jepsen worker 78 - jepsen.util 4668 :invoke :write [26 26] INFO [2019-12-17 12:31:18,921] jepsen worker 75 - jepsen.util 6015 :info :cas [25 [40 7]] :RequestCanceledException INFO [2019-12-17 12:31:18,935] jepsen worker 32 - jepsen.util 4892 :fail :read [10 nil] :RequestTimeoutException INFO [2019-12-17 12:31:18,934] jepsen worker 44 - jepsen.util 6614 :info :write [14 48] :RequestCanceledException INFO [2019-12-17 12:31:18,938] jepsen worker 63 - jepsen.util 3663 :info :write [21 38] :RequestCanceledException INFO [2019-12-17 12:31:18,938] jepsen worker 32 - jepsen.util 4892 :invoke :read [10 nil] INFO [2019-12-17 12:31:18,947] jepsen worker 17 - jepsen.util 5867 :info :cas [5 [5 3]] :RequestCanceledException INFO [2019-12-17 12:31:18,949] jepsen worker 62 - jepsen.util 5822 :invoke :read [20 nil] INFO [2019-12-17 12:31:18,954] jepsen worker 43 - jepsen.util 5893 :invoke :write [14 32] INFO [2019-12-17 12:31:18,962] jepsen worker 10 - jepsen.util 4420 :ok :write [3 49] INFO [2019-12-17 12:31:18,964] jepsen worker 10 - jepsen.util 4420 :invoke :read [3 nil] INFO [2019-12-17 12:31:19,104] jepsen worker 22 - jepsen.util 6232 :invoke :read [7 nil] INFO [2019-12-17 12:31:19,108] jepsen worker 20 - jepsen.util 4700 :fail :cas [6 [11 1]] :ValueNotSwapFrom INFO [2019-12-17 12:31:19,108] jepsen worker 73 - jepsen.util 4393 :fail :cas [24 [16 47]] :ValueNotSwapFrom INFO [2019-12-17 12:31:19,110] jepsen worker 20 - jepsen.util 4700 :invoke :cas [6 [33 28]] INFO [2019-12-17 12:31:19,110] jepsen worker 46 - jepsen.util 4726 :ok :read [15 7] INFO [2019-12-17 12:31:19,110] jepsen worker 73 - jepsen.util 4393 :invoke :read [24 nil] INFO [2019-12-17 12:31:19,112] jepsen worker 46 - jepsen.util 4726 :invoke :read [15 nil] INFO [2019-12-17 12:31:19,113] jepsen worker 11 - jepsen.util 4691 :ok :read [3 31] INFO [2019-12-17 12:31:19,113] jepsen worker 7 - jepsen.util 6127 :invoke :cas [2 [0 18]] INFO [2019-12-17 12:31:19,113] jepsen worker 87 - jepsen.util 4317 :ok :read [29 5] INFO [2019-12-17 12:31:19,113] jepsen worker 70 - jepsen.util 5290 :fail :cas [23 [7 44]] :ValueNotSwapFrom INFO [2019-12-17 12:31:19,113] jepsen worker 69 - jepsen.util 4749 :ok :read [23 47] INFO [2019-12-17 12:31:19,113] jepsen worker 11 - jepsen.util 4691 :invoke :read [3 nil] INFO [2019-12-17 12:31:19,113] jepsen worker 18 - jepsen.util 4608 :fail :cas [6 [46 10]] :ValueNotSwapFrom INFO [2019-12-17 12:31:19,114] jepsen worker 87 - jepsen.util 4317 :invoke :cas [29 [1 31]] INFO [2019-12-17 12:31:19,115] jepsen worker 70 - jepsen.util 5290 :invoke :write [23 15] INFO [2019-12-17 12:31:19,115] jepsen worker 69 - jepsen.util 4749 :invoke :read [23 nil] INFO [2019-12-17 12:31:19,115] jepsen worker 65 - jepsen.util 3935 :fail :read [21 nil] :RequestTimeoutException INFO [2019-12-17 12:31:19,115] jepsen worker 42 - jepsen.util 6792 :info :cas [14 [31 42]] :RequestCanceledException INFO [2019-12-17 12:31:19,115] jepsen worker 18 - jepsen.util 4608 :invoke :read [6 nil] INFO [2019-12-17 12:31:19,116] jepsen worker 65 - jepsen.util 3935 :invoke :cas [21 [28 7]] INFO [2019-12-17 12:31:19,118] jepsen worker 41 - jepsen.util 6521 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:31:19,118] jepsen worker 75 - jepsen.util 6105 :invoke :read [25 nil] INFO [2019-12-17 12:31:19,119] jepsen worker 41 - jepsen.util 6521 :invoke :cas [13 [35 37]] INFO [2019-12-17 12:31:19,121] jepsen worker 17 - jepsen.util 5957 :invoke :write [5 6] INFO [2019-12-17 12:31:19,123] jepsen worker 77 - jepsen.util 6287 :invoke :write [25 30] INFO [2019-12-17 12:31:19,127] jepsen worker 63 - jepsen.util 3753 :invoke :cas [21 [48 34]] INFO [2019-12-17 12:31:19,132] jepsen worker 44 - jepsen.util 6704 :invoke :write [14 7] INFO [2019-12-17 12:31:19,138] jepsen worker 42 - jepsen.util 6882 :invoke :read [14 nil] INFO [2019-12-17 12:31:19,192] jepsen worker 57 - jepsen.util 5817 :info :write [19 27] :RequestCanceledException INFO [2019-12-17 12:31:19,207] jepsen worker 15 - jepsen.util 5145 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:31:19,208] jepsen worker 15 - jepsen.util 5145 :invoke :write [5 30] INFO [2019-12-17 12:31:19,223] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26359,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:31:19,224] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26359,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:31:19,225] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26373,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:31:19,225] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26361,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:31:19,225] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26373,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:31:19,225] jepsen worker 38 - jepsen.util 4538 :ok :read [12 25] INFO [2019-12-17 12:31:19,225] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26361,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:31:19,226] jepsen worker 38 - jepsen.util 4538 :invoke :read [12 nil] INFO [2019-12-17 12:31:19,234] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26367,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:31:19,235] jepsen worker 30 - jepsen.util 4800 :ok :read [10 0] INFO [2019-12-17 12:31:19,236] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26367,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:31:19,236] jepsen worker 30 - jepsen.util 4800 :invoke :cas [10 [25 39]] INFO [2019-12-17 12:31:19,238] jepsen worker 12 - jepsen.util 4872 :fail :cas [4 [24 38]] :ValueNotSwapFrom INFO [2019-12-17 12:31:19,239] jepsen worker 12 - jepsen.util 4872 :invoke :read [4 nil] INFO [2019-12-17 12:31:19,246] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:31:19,247] jepsen nemesis - couchbase.util Rebalance check count: 197 INFO [2019-12-17 12:31:19,247] jepsen worker 28 - jepsen.util 3448 :ok :read [9 25] INFO [2019-12-17 12:31:19,248] jepsen worker 28 - jepsen.util 3448 :invoke :write [9 40] INFO [2019-12-17 12:31:19,248] jepsen nemesis - couchbase.util Rebalance stuck count: 178 INFO [2019-12-17 12:31:19,252] jepsen worker 57 - jepsen.util 5907 :invoke :read [19 nil] INFO [2019-12-17 12:31:19,257] jepsen worker 4 - jepsen.util 3784 :fail :cas [1 [36 40]] :ValueNotSwapFrom INFO [2019-12-17 12:31:19,257] jepsen worker 67 - jepsen.util 6277 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:31:19,257] jepsen worker 67 - jepsen.util 6277 :invoke :cas [22 [7 30]] INFO [2019-12-17 12:31:19,268] jepsen worker 4 - jepsen.util 3784 :invoke :cas [1 [4 47]] INFO [2019-12-17 12:31:19,269] jepsen worker 9 - jepsen.util 5319 :fail :cas [3 [26 1]] :ValueNotSwapFrom INFO [2019-12-17 12:31:19,270] jepsen worker 52 - jepsen.util 5002 :ok :read [17 47] INFO [2019-12-17 12:31:19,271] jepsen worker 9 - jepsen.util 5319 :invoke :write [3 47] INFO [2019-12-17 12:31:19,271] jepsen worker 52 - jepsen.util 5002 :invoke :cas [17 [18 36]] INFO [2019-12-17 12:31:19,272] jepsen worker 56 - jepsen.util 4736 :ok :read [18 18] INFO [2019-12-17 12:31:19,273] jepsen worker 56 - jepsen.util 4736 :invoke :cas [18 [17 49]] INFO [2019-12-17 12:31:19,287] jepsen worker 71 - jepsen.util 5021 :ok :read [23 47] INFO [2019-12-17 12:31:19,295] jepsen worker 71 - jepsen.util 5021 :invoke :cas [23 [38 8]] INFO [2019-12-17 12:31:19,298] jepsen worker 31 - jepsen.util 4441 :ok :read [10 0] INFO [2019-12-17 12:31:19,298] jepsen worker 31 - jepsen.util 4441 :invoke :cas [10 [39 8]] INFO [2019-12-17 12:31:19,312] jepsen worker 13 - jepsen.util 4243 :ok :read [4 33] INFO [2019-12-17 12:31:19,313] jepsen worker 13 - jepsen.util 4243 :invoke :write [4 44] INFO [2019-12-17 12:31:19,317] jepsen worker 51 - jepsen.util 4371 :ok :write [17 22] INFO [2019-12-17 12:31:19,318] jepsen worker 51 - jepsen.util 4371 :invoke :write [17 46] INFO [2019-12-17 12:31:19,325] jepsen worker 48 - jepsen.util 5448 :info :write [16 25] :RequestCanceledException INFO [2019-12-17 12:31:19,336] jepsen worker 14 - jepsen.util 4784 :fail :cas [4 [28 15]] :ValueNotSwapFrom INFO [2019-12-17 12:31:19,338] jepsen worker 14 - jepsen.util 4784 :invoke :write [4 41] INFO [2019-12-17 12:31:19,375] jepsen worker 47 - jepsen.util 5177 :ok :write [15 32] INFO [2019-12-17 12:31:19,378] jepsen worker 47 - jepsen.util 5177 :invoke :read [15 nil] INFO [2019-12-17 12:31:19,383] jepsen worker 48 - jepsen.util 5538 :invoke :read [16 nil] INFO [2019-12-17 12:31:19,389] jepsen worker 72 - jepsen.util 4662 :ok :write [24 44] INFO [2019-12-17 12:31:19,390] jepsen worker 72 - jepsen.util 4662 :invoke :read [24 nil] INFO [2019-12-17 12:31:19,433] jepsen worker 27 - jepsen.util 4347 :info :write [9 0] :RequestCanceledException INFO [2019-12-17 12:31:19,457] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26400,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:31:19,457] jepsen worker 60 - jepsen.util 6540 :info :write [20 38] :RequestCanceledException INFO [2019-12-17 12:31:19,457] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26400,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:31:19,482] jepsen worker 27 - jepsen.util 4437 :invoke :write [9 47] INFO [2019-12-17 12:31:19,507] jepsen worker 60 - jepsen.util 6630 :invoke :write [20 3] INFO [2019-12-17 12:31:19,525] jepsen worker 16 - jepsen.util 6946 :info :cas [5 [33 9]] :RequestCanceledException INFO [2019-12-17 12:31:19,525] jepsen worker 76 - jepsen.util 6196 :info :cas [25 [13 34]] :RequestCanceledException INFO [2019-12-17 12:31:19,529] jepsen worker 16 - jepsen.util 7036 :invoke :cas [5 [12 23]] INFO [2019-12-17 12:31:19,532] jepsen worker 76 - jepsen.util 6286 :invoke :read [25 nil] INFO [2019-12-17 12:31:19,534] jepsen worker 1 - jepsen.util 5761 :info :cas [0 [8 24]] :RequestCanceledException INFO [2019-12-17 12:31:19,539] jepsen worker 1 - jepsen.util 5851 :invoke :write [0 2] INFO [2019-12-17 12:31:19,568] jepsen worker 69 - jepsen.util 4749 :ok :read [23 47] INFO [2019-12-17 12:31:19,569] jepsen worker 69 - jepsen.util 4749 :invoke :write [23 12] INFO [2019-12-17 12:31:19,594] jepsen worker 33 - jepsen.util 5973 :info :write [11 17] :RequestCanceledException INFO [2019-12-17 12:31:19,599] jepsen worker 88 - jepsen.util 4768 :ok :read [29 5] INFO [2019-12-17 12:31:19,601] jepsen worker 88 - jepsen.util 4768 :invoke :cas [29 [3 1]] INFO [2019-12-17 12:31:19,604] jepsen worker 10 - jepsen.util 4420 :ok :read [3 49] INFO [2019-12-17 12:31:19,604] jepsen worker 54 - jepsen.util 5094 :ok :read [18 18] INFO [2019-12-17 12:31:19,604] jepsen worker 10 - jepsen.util 4420 :invoke :cas [3 [46 30]] INFO [2019-12-17 12:31:19,606] jepsen worker 54 - jepsen.util 5094 :invoke :read [18 nil] INFO [2019-12-17 12:31:19,616] jepsen worker 33 - jepsen.util 6063 :invoke :read [11 nil] INFO [2019-12-17 12:31:19,644] jepsen worker 18 - jepsen.util 4608 :ok :read [6 39] INFO [2019-12-17 12:31:19,644] jepsen worker 18 - jepsen.util 4608 :invoke :write [6 3] INFO [2019-12-17 12:31:19,651] jepsen worker 47 - jepsen.util 5177 :ok :read [15 32] INFO [2019-12-17 12:31:19,651] jepsen worker 47 - jepsen.util 5177 :invoke :write [15 6] INFO [2019-12-17 12:31:19,667] jepsen worker 19 - jepsen.util 4699 :ok :read [6 39] INFO [2019-12-17 12:31:19,668] jepsen worker 55 - jepsen.util 5545 :fail :cas [18 [32 12]] :ValueNotSwapFrom INFO [2019-12-17 12:31:19,677] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26421,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:31:19,684] jepsen worker 45 - jepsen.util 4815 :fail :cas [15 [8 29]] :ValueNotSwapFrom INFO [2019-12-17 12:31:19,690] jepsen worker 32 - jepsen.util 4892 :ok :read [10 15] INFO [2019-12-17 12:31:19,699] jepsen worker 86 - jepsen.util 6386 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:31:19,703] jepsen worker 27 - jepsen.util 4437 :ok :write [9 47] INFO [2019-12-17 12:31:19,707] jepsen worker 20 - jepsen.util 4700 :fail :cas [6 [33 28]] :ValueNotSwapFrom INFO [2019-12-17 12:31:19,709] jepsen worker 81 - jepsen.util 6111 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:31:19,713] jepsen worker 3 - jepsen.util 4773 :fail :cas [1 [48 22]] :ValueNotSwapFrom INFO [2019-12-17 12:31:19,724] jepsen worker 19 - jepsen.util 4699 :invoke :read [6 nil] INFO [2019-12-17 12:31:19,728] jepsen worker 55 - jepsen.util 5545 :invoke :read [18 nil] INFO [2019-12-17 12:31:19,729] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26421,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:31:19,729] jepsen worker 45 - jepsen.util 4815 :invoke :read [15 nil] INFO [2019-12-17 12:31:19,729] jepsen worker 86 - jepsen.util 6386 :invoke :write [28 15] INFO [2019-12-17 12:31:19,730] jepsen worker 27 - jepsen.util 4437 :invoke :read [9 nil] INFO [2019-12-17 12:31:19,731] jepsen worker 32 - jepsen.util 4892 :invoke :cas [10 [21 46]] INFO [2019-12-17 12:31:19,734] jepsen worker 20 - jepsen.util 4700 :invoke :read [6 nil] INFO [2019-12-17 12:31:19,734] jepsen worker 81 - jepsen.util 6111 :invoke :read [27 nil] INFO [2019-12-17 12:31:19,735] jepsen worker 3 - jepsen.util 4773 :invoke :cas [1 [25 42]] INFO [2019-12-17 12:31:19,752] jepsen worker 80 - jepsen.util 4580 :ok :write [26 9] INFO [2019-12-17 12:31:19,752] jepsen worker 29 - jepsen.util 3719 :ok :read [9 29] INFO [2019-12-17 12:31:19,752] jepsen worker 80 - jepsen.util 4580 :invoke :read [26 nil] INFO [2019-12-17 12:31:19,753] jepsen worker 26 - jepsen.util 6236 :info :write [8 19] :RequestCanceledException INFO [2019-12-17 12:31:19,755] jepsen worker 29 - jepsen.util 3719 :invoke :read [9 nil] INFO [2019-12-17 12:31:19,760] jepsen worker 36 - jepsen.util 4716 :ok :write [12 21] INFO [2019-12-17 12:31:19,761] jepsen worker 36 - jepsen.util 4716 :invoke :write [12 9] INFO [2019-12-17 12:31:19,766] jepsen worker 26 - jepsen.util 6326 :invoke :cas [8 [34 15]] INFO [2019-12-17 12:31:19,783] jepsen worker 5 - jepsen.util 4145 :fail :cas [1 [12 22]] :ValueNotSwapFrom INFO [2019-12-17 12:31:19,794] jepsen worker 5 - jepsen.util 4145 :invoke :write [1 45] INFO [2019-12-17 12:31:19,798] jepsen worker 74 - jepsen.util 4934 :fail :cas [24 [22 1]] :ValueNotSwapFrom INFO [2019-12-17 12:31:19,799] jepsen worker 74 - jepsen.util 4934 :invoke :write [24 19] INFO [2019-12-17 12:31:19,819] jepsen worker 49 - jepsen.util 5359 :info :cas [16 [3 46]] :RequestCanceledException INFO [2019-12-17 12:31:19,825] jepsen worker 53 - jepsen.util 4553 :fail :cas [17 [47 26]] :ValueNotSwapFrom INFO [2019-12-17 12:31:19,837] jepsen worker 53 - jepsen.util 4553 :invoke :cas [17 [28 22]] INFO [2019-12-17 12:31:19,837] jepsen worker 49 - jepsen.util 5449 :invoke :cas [16 [36 4]] INFO [2019-12-17 12:31:19,895] jepsen worker 65 - jepsen.util 3935 :fail :cas [21 [28 7]] :ValueNotSwapFrom INFO [2019-12-17 12:31:19,909] jepsen worker 65 - jepsen.util 3935 :invoke :cas [21 [36 11]] INFO [2019-12-17 12:31:19,912] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26442,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:31:19,912] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26442,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:31:19,913] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26447,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:31:19,913] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26447,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:31:19,933] jepsen worker 64 - jepsen.util 3574 :fail :cas [21 [2 40]] :ValueNotSwapFrom INFO [2019-12-17 12:31:19,935] jepsen worker 64 - jepsen.util 3574 :invoke :cas [21 [46 44]] INFO [2019-12-17 12:31:19,935] jepsen worker 79 - jepsen.util 4489 :fail :cas [26 [8 36]] :ValueNotSwapFrom INFO [2019-12-17 12:31:19,939] jepsen worker 79 - jepsen.util 4489 :invoke :cas [26 [24 41]] INFO [2019-12-17 12:31:19,949] jepsen worker 12 - jepsen.util 4872 :ok :read [4 33] INFO [2019-12-17 12:31:19,952] jepsen worker 12 - jepsen.util 4872 :invoke :write [4 31] INFO [2019-12-17 12:31:19,960] jepsen worker 13 - jepsen.util 4243 :ok :write [4 44] INFO [2019-12-17 12:31:19,960] jepsen worker 13 - jepsen.util 4243 :invoke :read [4 nil] INFO [2019-12-17 12:31:19,968] jepsen worker 78 - jepsen.util 4668 :ok :write [26 26] INFO [2019-12-17 12:31:19,968] jepsen worker 78 - jepsen.util 4668 :invoke :read [26 nil] INFO [2019-12-17 12:31:19,997] jepsen worker 40 - jepsen.util 6880 :info :write [13 17] :RequestCanceledException INFO [2019-12-17 12:31:20,012] jepsen worker 40 - jepsen.util 6970 :invoke :read [13 nil] INFO [2019-12-17 12:31:20,015] jepsen worker 30 - jepsen.util 4800 :fail :cas [10 [25 39]] :ValueNotSwapFrom INFO [2019-12-17 12:31:20,016] jepsen worker 30 - jepsen.util 4800 :invoke :write [10 19] INFO [2019-12-17 12:31:20,019] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26457,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:31:20,021] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26457,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:31:20,022] jepsen worker 0 - jepsen.util 6750 :info :write [0 42] :RequestCanceledException INFO [2019-12-17 12:31:20,031] jepsen worker 0 - jepsen.util 6840 :invoke :read [0 nil] INFO [2019-12-17 12:31:20,050] jepsen worker 10 - jepsen.util 4420 :fail :cas [3 [46 30]] :ValueNotSwapFrom INFO [2019-12-17 12:31:20,071] jepsen worker 10 - jepsen.util 4420 :invoke :write [3 48] INFO [2019-12-17 12:31:20,109] jepsen worker 71 - jepsen.util 5021 :fail :cas [23 [38 8]] :ValueNotSwapFrom INFO [2019-12-17 12:31:20,112] jepsen worker 71 - jepsen.util 5021 :invoke :cas [23 [44 49]] INFO [2019-12-17 12:31:20,153] jepsen worker 47 - jepsen.util 5177 :ok :write [15 6] INFO [2019-12-17 12:31:20,153] jepsen worker 31 - jepsen.util 4441 :fail :cas [10 [39 8]] :ValueNotSwapFrom INFO [2019-12-17 12:31:20,154] jepsen worker 47 - jepsen.util 5177 :invoke :cas [15 [16 41]] INFO [2019-12-17 12:31:20,156] jepsen worker 37 - jepsen.util 4447 :ok :write [12 4] INFO [2019-12-17 12:31:20,156] jepsen worker 63 - jepsen.util 3753 :fail :cas [21 [48 34]] :ValueNotSwapFrom INFO [2019-12-17 12:31:20,156] jepsen worker 87 - jepsen.util 4317 :fail :cas [29 [1 31]] :ValueNotSwapFrom INFO [2019-12-17 12:31:20,158] jepsen worker 37 - jepsen.util 4447 :invoke :cas [12 [30 3]] INFO [2019-12-17 12:31:20,158] jepsen worker 89 - jepsen.util 4229 :ok :write [29 45] INFO [2019-12-17 12:31:20,158] jepsen worker 28 - jepsen.util 3448 :ok :write [9 40] INFO [2019-12-17 12:31:20,158] jepsen worker 89 - jepsen.util 4229 :invoke :cas [29 [31 21]] INFO [2019-12-17 12:31:20,158] jepsen worker 28 - jepsen.util 3448 :invoke :read [9 nil] INFO [2019-12-17 12:31:20,160] jepsen worker 31 - jepsen.util 4441 :invoke :cas [10 [49 47]] INFO [2019-12-17 12:31:20,160] jepsen worker 87 - jepsen.util 4317 :invoke :write [29 46] INFO [2019-12-17 12:31:20,161] jepsen worker 27 - jepsen.util 4437 :ok :read [9 47] INFO [2019-12-17 12:31:20,163] jepsen worker 63 - jepsen.util 3753 :invoke :write [21 4] INFO [2019-12-17 12:31:20,166] jepsen worker 27 - jepsen.util 4437 :invoke :write [9 19] INFO [2019-12-17 12:31:20,170] jepsen worker 51 - jepsen.util 4371 :ok :write [17 46] INFO [2019-12-17 12:31:20,174] jepsen worker 51 - jepsen.util 4371 :invoke :write [17 42] INFO [2019-12-17 12:31:20,273] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26470,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:31:20,273] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26470,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:31:20,273] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26475,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:31:20,273] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26475,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:31:20,274] jepsen worker 18 - jepsen.util 4608 :ok :write [6 3] INFO [2019-12-17 12:31:20,274] jepsen worker 18 - jepsen.util 4608 :invoke :read [6 nil] INFO [2019-12-17 12:31:20,276] jepsen worker 36 - jepsen.util 4716 :ok :write [12 9] INFO [2019-12-17 12:31:20,277] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26479,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:31:20,278] jepsen worker 36 - jepsen.util 4716 :invoke :read [12 nil] INFO [2019-12-17 12:31:20,279] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26479,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:31:20,283] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26480,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:31:20,283] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26480,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:31:20,283] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26481,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:31:20,283] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26481,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:31:20,294] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:31:20,296] jepsen nemesis - couchbase.util Rebalance check count: 198 INFO [2019-12-17 12:31:20,296] jepsen nemesis - couchbase.util Rebalance stuck count: 179 INFO [2019-12-17 12:31:20,312] jepsen worker 23 - jepsen.util 6143 :info :cas [7 [46 17]] :RequestCanceledException INFO [2019-12-17 12:31:20,325] jepsen worker 21 - jepsen.util 5871 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:31:20,325] jepsen worker 21 - jepsen.util 5871 :invoke :read [7 nil] INFO [2019-12-17 12:31:20,334] jepsen worker 70 - jepsen.util 5290 :ok :write [23 15] INFO [2019-12-17 12:31:20,335] jepsen worker 70 - jepsen.util 5290 :invoke :write [23 36] INFO [2019-12-17 12:31:20,335] jepsen worker 23 - jepsen.util 6233 :invoke :write [7 16] INFO [2019-12-17 12:31:20,344] jepsen worker 19 - jepsen.util 4699 :ok :read [6 39] INFO [2019-12-17 12:31:20,345] jepsen worker 19 - jepsen.util 4699 :invoke :write [6 29] INFO [2019-12-17 12:31:20,348] jepsen worker 72 - jepsen.util 4662 :ok :read [24 44] INFO [2019-12-17 12:31:20,355] jepsen worker 72 - jepsen.util 4662 :invoke :read [24 nil] INFO [2019-12-17 12:31:20,397] jepsen worker 61 - jepsen.util 6721 :info :write [20 41] :RequestCanceledException INFO [2019-12-17 12:31:20,400] jepsen worker 32 - jepsen.util 4892 :fail :cas [10 [21 46]] :ValueNotSwapFrom INFO [2019-12-17 12:31:20,402] jepsen worker 32 - jepsen.util 4892 :invoke :read [10 nil] INFO [2019-12-17 12:31:20,418] jepsen worker 61 - jepsen.util 6811 :invoke :cas [20 [5 36]] INFO [2019-12-17 12:31:20,431] jepsen worker 59 - jepsen.util 6179 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:31:20,433] jepsen worker 59 - jepsen.util 6179 :invoke :cas [19 [18 8]] INFO [2019-12-17 12:31:20,458] jepsen worker 88 - jepsen.util 4768 :fail :cas [29 [3 1]] :ValueNotSwapFrom INFO [2019-12-17 12:31:20,463] jepsen worker 29 - jepsen.util 3719 :ok :read [9 47] INFO [2019-12-17 12:31:20,467] jepsen worker 52 - jepsen.util 5002 :fail :cas [17 [18 36]] :ValueNotSwapFrom INFO [2019-12-17 12:31:20,469] jepsen worker 88 - jepsen.util 4768 :invoke :cas [29 [32 22]] INFO [2019-12-17 12:31:20,473] jepsen worker 29 - jepsen.util 3719 :invoke :read [9 nil] INFO [2019-12-17 12:31:20,482] jepsen worker 52 - jepsen.util 5002 :invoke :read [17 nil] INFO [2019-12-17 12:31:20,502] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26491,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:31:20,502] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26491,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:31:20,504] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26493,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:31:20,504] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26493,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:31:20,514] jepsen worker 66 - jepsen.util 5646 :info :cas [22 [18 23]] :RequestCanceledException INFO [2019-12-17 12:31:21,947] jepsen worker 45 - jepsen.util 4815 :ok :read [15 32] INFO [2019-12-17 12:31:21,947] jepsen worker 54 - jepsen.util 5094 :ok :read [18 18] INFO [2019-12-17 12:31:21,948] jepsen worker 45 - jepsen.util 4815 :invoke :write [15 6] INFO [2019-12-17 12:31:21,950] jepsen worker 54 - jepsen.util 5094 :invoke :write [18 20] INFO [2019-12-17 12:31:21,951] jepsen worker 20 - jepsen.util 4700 :ok :read [6 39] INFO [2019-12-17 12:31:21,951] jepsen worker 14 - jepsen.util 4784 :ok :write [4 41] INFO [2019-12-17 12:31:21,951] jepsen worker 79 - jepsen.util 4489 :fail :cas [26 [24 41]] :ValueNotSwapFrom INFO [2019-12-17 12:31:21,952] jepsen worker 20 - jepsen.util 4700 :invoke :write [6 1] INFO [2019-12-17 12:31:21,953] jepsen worker 64 - jepsen.util 3574 :fail :cas [21 [46 44]] :ValueNotSwapFrom INFO [2019-12-17 12:31:21,953] jepsen worker 3 - jepsen.util 4773 :fail :cas [1 [25 42]] :ValueNotSwapFrom INFO [2019-12-17 12:31:21,955] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26508,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:31:21,955] jepsen worker 14 - jepsen.util 4784 :invoke :cas [4 [14 8]] INFO [2019-12-17 12:31:21,955] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26516,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:31:21,955] jepsen worker 79 - jepsen.util 4489 :invoke :write [26 11] INFO [2019-12-17 12:31:21,955] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26515,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:31:21,956] jepsen worker 33 - jepsen.util 6063 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:31:21,956] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26533,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:31:21,957] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26507,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:31:21,957] jepsen worker 2 - jepsen.util 5672 :info :cas [0 [34 11]] :RequestCanceledException INFO [2019-12-17 12:31:21,957] jepsen worker 48 - jepsen.util 5538 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:31:21,957] jepsen worker 66 - jepsen.util 5736 :invoke :write [22 36] INFO [2019-12-17 12:31:21,957] jepsen worker 81 - jepsen.util 6111 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:31:21,958] jepsen worker 67 - jepsen.util 6277 :info :cas [22 [7 30]] :RequestCanceledException INFO [2019-12-17 12:31:21,958] jepsen worker 44 - jepsen.util 6704 :info :write [14 7] :RequestCanceledException INFO [2019-12-17 12:31:21,958] jepsen worker 64 - jepsen.util 3574 :invoke :cas [21 [43 43]] INFO [2019-12-17 12:31:21,958] jepsen worker 77 - jepsen.util 6287 :info :write [25 30] :RequestCanceledException INFO [2019-12-17 12:31:21,958] jepsen worker 17 - jepsen.util 5957 :info :write [5 6] :RequestCanceledException INFO [2019-12-17 12:31:21,958] jepsen worker 30 - jepsen.util 4800 :ok :write [10 19] INFO [2019-12-17 12:31:21,959] jepsen worker 46 - jepsen.util 4726 :fail :read [15 nil] :RequestTimeoutException INFO [2019-12-17 12:31:21,959] jepsen worker 73 - jepsen.util 4393 :fail :read [24 nil] :RequestTimeoutException INFO [2019-12-17 12:31:21,959] jepsen worker 86 - jepsen.util 6386 :info :write [28 15] :RequestCanceledException INFO [2019-12-17 12:31:21,959] jepsen worker 5 - jepsen.util 4145 :ok :write [1 45] INFO [2019-12-17 12:31:21,959] jepsen worker 3 - jepsen.util 4773 :invoke :cas [1 [21 29]] INFO [2019-12-17 12:31:21,960] jepsen worker 84 - jepsen.util 5754 :info :write [28 1] :RequestCanceledException INFO [2019-12-17 12:31:21,960] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26508,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:31:21,960] jepsen worker 76 - jepsen.util 6286 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:31:21,960] jepsen worker 50 - jepsen.util 6350 :info :cas [16 [15 49]] :RequestCanceledException INFO [2019-12-17 12:31:21,960] jepsen worker 11 - jepsen.util 4691 :fail :read [3 nil] :RequestTimeoutException INFO [2019-12-17 12:31:21,960] jepsen worker 43 - jepsen.util 5893 :info :write [14 32] :RequestCanceledException INFO [2019-12-17 12:31:21,960] jepsen worker 41 - jepsen.util 6521 :info :cas [13 [35 37]] :RequestCanceledException INFO [2019-12-17 12:31:21,960] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26516,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:31:21,962] jepsen worker 75 - jepsen.util 6105 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:31:21,962] jepsen worker 85 - jepsen.util 6835 :info :write [28 3] :RequestCanceledException INFO [2019-12-17 12:31:21,962] jepsen worker 1 - jepsen.util 5851 :info :write [0 2] :RequestCanceledException INFO [2019-12-17 12:31:21,962] jepsen worker 57 - jepsen.util 5907 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:31:21,963] jepsen worker 49 - jepsen.util 5449 :info :cas [16 [36 4]] :RequestCanceledException INFO [2019-12-17 12:31:21,963] jepsen worker 7 - jepsen.util 6127 :info :cas [2 [0 18]] :RequestCanceledException INFO [2019-12-17 12:31:21,963] jepsen worker 35 - jepsen.util 6245 :info :cas [11 [8 26]] :RequestCanceledException INFO [2019-12-17 12:31:21,964] jepsen worker 16 - jepsen.util 7036 :info :cas [5 [12 23]] :RequestCanceledException INFO [2019-12-17 12:31:21,964] jepsen worker 22 - jepsen.util 6232 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:31:21,964] jepsen worker 68 - jepsen.util 5558 :info :write [22 0] :RequestCanceledException INFO [2019-12-17 12:31:21,964] jepsen worker 26 - jepsen.util 6326 :info :cas [8 [34 15]] :RequestCanceledException INFO [2019-12-17 12:31:21,964] jepsen worker 82 - jepsen.util 6922 :info :cas [27 [1 5]] :RequestCanceledException INFO [2019-12-17 12:31:21,964] jepsen worker 60 - jepsen.util 6630 :info :write [20 3] :RequestCanceledException INFO [2019-12-17 12:31:21,964] jepsen worker 58 - jepsen.util 6268 :info :write [19 33] :RequestCanceledException INFO [2019-12-17 12:31:21,964] jepsen worker 8 - jepsen.util 5408 :info :write [2 23] :RequestCanceledException INFO [2019-12-17 12:31:21,964] jepsen worker 25 - jepsen.util 6235 :info :cas [8 [1 14]] :RequestCanceledException INFO [2019-12-17 12:31:21,965] jepsen worker 42 - jepsen.util 6882 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:31:21,965] jepsen worker 39 - jepsen.util 5709 :info :write [13 21] :RequestCanceledException INFO [2019-12-17 12:31:21,965] jepsen worker 24 - jepsen.util 5784 :info :write [8 26] :RequestCanceledException INFO [2019-12-17 12:31:21,965] jepsen worker 6 - jepsen.util 6576 :info :write [2 41] :RequestCanceledException INFO [2019-12-17 12:31:21,965] jepsen worker 83 - jepsen.util 6563 :info :cas [27 [14 10]] :RequestCanceledException INFO [2019-12-17 12:31:21,965] jepsen worker 62 - jepsen.util 5822 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:31:21,966] jepsen worker 34 - jepsen.util 5614 :info :write [11 49] :RequestCanceledException INFO [2019-12-17 12:31:21,966] jepsen worker 69 - jepsen.util 4749 :info :write [23 12] :RequestCanceledException INFO [2019-12-17 12:31:21,966] jepsen worker 15 - jepsen.util 5145 :info :write [5 30] :RequestCanceledException INFO [2019-12-17 12:31:21,966] jepsen worker 53 - jepsen.util 4553 :info :cas [17 [28 22]] :RequestCanceledException INFO [2019-12-17 12:31:21,966] jepsen worker 9 - jepsen.util 5319 :info :write [3 47] :RequestCanceledException INFO [2019-12-17 12:31:21,967] jepsen worker 80 - jepsen.util 4580 :fail :read [26 nil] :RequestTimeoutException INFO [2019-12-17 12:31:21,968] jepsen worker 33 - jepsen.util 6063 :invoke :cas [11 [0 48]] INFO [2019-12-17 12:31:21,968] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26515,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:31:21,968] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26533,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:31:21,969] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26507,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:31:21,970] jepsen worker 48 - jepsen.util 5538 :invoke :write [16 23] INFO [2019-12-17 12:31:21,970] jepsen worker 81 - jepsen.util 6111 :invoke :read [27 nil] INFO [2019-12-17 12:31:21,971] jepsen worker 30 - jepsen.util 4800 :invoke :cas [10 [8 48]] INFO [2019-12-17 12:31:21,972] jepsen worker 5 - jepsen.util 4145 :invoke :write [1 8] INFO [2019-12-17 12:31:21,972] jepsen worker 73 - jepsen.util 4393 :invoke :cas [24 [3 12]] INFO [2019-12-17 12:31:21,972] jepsen worker 46 - jepsen.util 4726 :invoke :write [15 38] INFO [2019-12-17 12:31:21,973] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26520,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:31:21,974] jepsen worker 76 - jepsen.util 6286 :invoke :cas [25 [37 28]] INFO [2019-12-17 12:31:21,974] jepsen worker 11 - jepsen.util 4691 :invoke :cas [3 [19 19]] INFO [2019-12-17 12:31:21,976] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26521,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:31:21,976] jepsen worker 75 - jepsen.util 6105 :invoke :write [25 42] INFO [2019-12-17 12:31:21,977] jepsen worker 57 - jepsen.util 5907 :invoke :cas [19 [11 13]] INFO [2019-12-17 12:31:21,978] jepsen worker 22 - jepsen.util 6232 :invoke :write [7 17] INFO [2019-12-17 12:31:21,981] jepsen worker 42 - jepsen.util 6882 :invoke :read [14 nil] INFO [2019-12-17 12:31:21,984] jepsen worker 62 - jepsen.util 5822 :invoke :read [20 nil] INFO [2019-12-17 12:31:21,988] jepsen worker 80 - jepsen.util 4580 :invoke :cas [26 [1 14]] INFO [2019-12-17 12:31:21,988] jepsen worker 67 - jepsen.util 6367 :invoke :write [22 35] INFO [2019-12-17 12:31:21,990] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26540,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:31:21,990] jepsen worker 2 - jepsen.util 5762 :invoke :write [0 39] INFO [2019-12-17 12:31:21,996] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26520,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:31:21,997] jepsen worker 44 - jepsen.util 6794 :invoke :write [14 45] INFO [2019-12-17 12:31:21,998] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26521,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:31:21,998] jepsen worker 84 - jepsen.util 5844 :invoke :cas [28 [38 2]] INFO [2019-12-17 12:31:21,999] jepsen worker 50 - jepsen.util 6440 :invoke :write [16 38] INFO [2019-12-17 12:31:21,999] jepsen worker 77 - jepsen.util 6377 :invoke :write [25 11] INFO [2019-12-17 12:31:21,999] jepsen worker 86 - jepsen.util 6476 :invoke :write [28 4] INFO [2019-12-17 12:31:22,000] jepsen worker 43 - jepsen.util 5983 :invoke :cas [14 [12 31]] INFO [2019-12-17 12:31:22,001] jepsen worker 35 - jepsen.util 6335 :invoke :write [11 43] INFO [2019-12-17 12:31:22,002] jepsen worker 41 - jepsen.util 6611 :invoke :write [13 48] INFO [2019-12-17 12:31:22,003] jepsen worker 85 - jepsen.util 6925 :invoke :write [28 42] INFO [2019-12-17 12:31:22,003] jepsen worker 17 - jepsen.util 6047 :invoke :write [5 49] INFO [2019-12-17 12:31:22,004] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26540,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:31:22,004] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26558,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:31:22,005] jepsen worker 26 - jepsen.util 6416 :invoke :cas [8 [10 3]] INFO [2019-12-17 12:31:22,005] jepsen worker 49 - jepsen.util 5539 :invoke :cas [16 [48 10]] INFO [2019-12-17 12:31:22,005] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26524,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:31:22,006] jepsen worker 60 - jepsen.util 6720 :invoke :read [20 nil] INFO [2019-12-17 12:31:22,007] jepsen worker 1 - jepsen.util 5941 :invoke :write [0 7] INFO [2019-12-17 12:31:22,007] jepsen worker 82 - jepsen.util 7012 :invoke :read [27 nil] INFO [2019-12-17 12:31:22,007] jepsen worker 8 - jepsen.util 5498 :invoke :cas [2 [36 44]] INFO [2019-12-17 12:31:22,007] jepsen worker 68 - jepsen.util 5648 :invoke :cas [22 [17 23]] INFO [2019-12-17 12:31:22,007] jepsen worker 7 - jepsen.util 6217 :invoke :write [2 18] INFO [2019-12-17 12:31:22,008] jepsen worker 16 - jepsen.util 7126 :invoke :cas [5 [0 26]] INFO [2019-12-17 12:31:22,008] jepsen worker 58 - jepsen.util 6358 :invoke :read [19 nil] INFO [2019-12-17 12:31:22,008] jepsen worker 24 - jepsen.util 5874 :invoke :write [8 17] INFO [2019-12-17 12:31:22,008] jepsen worker 34 - jepsen.util 5704 :invoke :cas [11 [33 9]] INFO [2019-12-17 12:31:22,009] jepsen worker 9 - jepsen.util 5409 :invoke :cas [3 [47 1]] INFO [2019-12-17 12:31:22,010] jepsen worker 25 - jepsen.util 6325 :invoke :cas [8 [31 49]] INFO [2019-12-17 12:31:22,010] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26550,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:31:22,010] jepsen worker 15 - jepsen.util 5235 :invoke :write [5 0] INFO [2019-12-17 12:31:22,011] jepsen worker 39 - jepsen.util 5799 :invoke :cas [13 [42 25]] INFO [2019-12-17 12:31:22,011] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26558,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:31:22,013] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26524,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:31:22,014] jepsen worker 83 - jepsen.util 6653 :invoke :cas [27 [7 21]] INFO [2019-12-17 12:31:22,016] jepsen worker 6 - jepsen.util 6666 :invoke :read [2 nil] INFO [2019-12-17 12:31:22,016] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26550,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:31:22,016] jepsen worker 53 - jepsen.util 4643 :invoke :read [17 nil] INFO [2019-12-17 12:31:22,017] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26526,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:31:22,019] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26526,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:31:22,021] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26530,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:31:22,021] jepsen worker 69 - jepsen.util 4839 :invoke :write [23 27] INFO [2019-12-17 12:31:22,022] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26530,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:31:22,034] jepsen worker 78 - jepsen.util 4668 :ok :read [26 26] INFO [2019-12-17 12:31:22,035] jepsen worker 78 - jepsen.util 4668 :invoke :read [26 nil] INFO [2019-12-17 12:31:22,039] jepsen worker 74 - jepsen.util 4934 :info :write [24 19] :RequestCanceledException INFO [2019-12-17 12:31:22,050] jepsen worker 47 - jepsen.util 5177 :fail :cas [15 [16 41]] :ValueNotSwapFrom INFO [2019-12-17 12:31:22,051] jepsen worker 47 - jepsen.util 5177 :invoke :cas [15 [15 20]] INFO [2019-12-17 12:31:22,056] jepsen worker 65 - jepsen.util 3935 :fail :cas [21 [36 11]] :ValueNotSwapFrom INFO [2019-12-17 12:31:22,057] jepsen worker 65 - jepsen.util 3935 :invoke :read [21 nil] INFO [2019-12-17 12:31:22,060] jepsen worker 74 - jepsen.util 5024 :invoke :read [24 nil] INFO [2019-12-17 12:31:22,061] jepsen worker 55 - jepsen.util 5545 :ok :read [18 18] INFO [2019-12-17 12:31:22,062] jepsen worker 55 - jepsen.util 5545 :invoke :cas [18 [36 2]] INFO [2019-12-17 12:31:22,063] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26538,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:31:22,064] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26538,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:31:22,091] jepsen worker 40 - jepsen.util 6970 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:31:22,092] jepsen worker 40 - jepsen.util 6970 :invoke :read [13 nil] INFO [2019-12-17 12:31:22,105] jepsen worker 0 - jepsen.util 6840 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:31:22,106] jepsen worker 0 - jepsen.util 6840 :invoke :read [0 nil] INFO [2019-12-17 12:31:22,137] jepsen worker 71 - jepsen.util 5021 :fail :cas [23 [44 49]] :ValueNotSwapFrom INFO [2019-12-17 12:31:22,137] jepsen worker 71 - jepsen.util 5021 :invoke :cas [23 [31 44]] INFO [2019-12-17 12:31:22,147] jepsen worker 38 - jepsen.util 4538 :ok :read [12 9] INFO [2019-12-17 12:31:22,148] jepsen worker 38 - jepsen.util 4538 :invoke :read [12 nil] INFO [2019-12-17 12:31:22,148] jepsen worker 18 - jepsen.util 4608 :ok :read [6 3] INFO [2019-12-17 12:31:22,149] jepsen worker 18 - jepsen.util 4608 :invoke :cas [6 [44 47]] INFO [2019-12-17 12:31:22,149] jepsen worker 13 - jepsen.util 4243 :ok :read [4 44] INFO [2019-12-17 12:31:22,149] jepsen worker 13 - jepsen.util 4243 :invoke :read [4 nil] INFO [2019-12-17 12:31:22,161] jepsen worker 32 - jepsen.util 4892 :ok :read [10 15] INFO [2019-12-17 12:31:22,162] jepsen worker 32 - jepsen.util 4892 :invoke :cas [10 [15 17]] INFO [2019-12-17 12:31:22,176] jepsen worker 56 - jepsen.util 4736 :fail :cas [18 [17 49]] :ValueNotSwapFrom INFO [2019-12-17 12:31:22,176] jepsen worker 56 - jepsen.util 4736 :invoke :read [18 nil] INFO [2019-12-17 12:31:22,180] jepsen worker 72 - jepsen.util 4662 :ok :read [24 44] INFO [2019-12-17 12:31:22,180] jepsen worker 72 - jepsen.util 4662 :invoke :write [24 4] INFO [2019-12-17 12:31:22,188] jepsen worker 37 - jepsen.util 4447 :fail :cas [12 [30 3]] :ValueNotSwapFrom INFO [2019-12-17 12:31:22,188] jepsen worker 37 - jepsen.util 4447 :invoke :write [12 47] INFO [2019-12-17 12:31:22,201] jepsen worker 89 - jepsen.util 4229 :fail :cas [29 [31 21]] :ValueNotSwapFrom INFO [2019-12-17 12:31:22,202] jepsen worker 89 - jepsen.util 4229 :invoke :read [29 nil] INFO [2019-12-17 12:31:22,205] jepsen worker 28 - jepsen.util 3448 :fail :read [9 nil] :RequestTimeoutException INFO [2019-12-17 12:31:22,206] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26629,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:31:22,206] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26640,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:31:22,206] jepsen worker 31 - jepsen.util 4441 :info :cas [10 [49 47]] :RequestCanceledException INFO [2019-12-17 12:31:22,206] jepsen worker 28 - jepsen.util 3448 :invoke :write [9 49] INFO [2019-12-17 12:31:22,206] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26629,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:31:22,206] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26640,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:31:22,206] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26630,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:31:22,206] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26644,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:31:22,206] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26630,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:31:22,206] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26631,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:31:22,207] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26644,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:31:22,207] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26631,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:31:22,210] jepsen worker 31 - jepsen.util 4531 :invoke :write [10 13] INFO [2019-12-17 12:31:22,229] jepsen worker 36 - jepsen.util 4716 :ok :read [12 9] INFO [2019-12-17 12:31:22,230] jepsen worker 36 - jepsen.util 4716 :invoke :read [12 nil] INFO [2019-12-17 12:31:22,234] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:31:22,234] jepsen nemesis - couchbase.util Rebalance check count: 199 INFO [2019-12-17 12:31:22,234] jepsen nemesis - couchbase.util Rebalance stuck count: 180 INFO [2019-12-17 12:31:22,248] jepsen worker 3 - jepsen.util 4773 :fail :cas [1 [21 29]] :ValueNotSwapFrom INFO [2019-12-17 12:31:22,248] jepsen worker 3 - jepsen.util 4773 :invoke :write [1 49] INFO [2019-12-17 12:31:22,262] jepsen worker 74 - jepsen.util 5024 :ok :read [24 44] INFO [2019-12-17 12:31:22,262] jepsen worker 74 - jepsen.util 5024 :invoke :read [24 nil] INFO [2019-12-17 12:31:22,279] jepsen worker 29 - jepsen.util 3719 :ok :read [9 40] INFO [2019-12-17 12:31:22,280] jepsen worker 29 - jepsen.util 3719 :invoke :write [9 9] INFO [2019-12-17 12:31:22,299] jepsen worker 14 - jepsen.util 4784 :fail :cas [4 [14 8]] :ValueNotSwapFrom INFO [2019-12-17 12:31:22,299] jepsen worker 14 - jepsen.util 4784 :invoke :cas [4 [27 29]] INFO [2019-12-17 12:31:22,314] jepsen worker 73 - jepsen.util 4393 :fail :cas [24 [3 12]] :ValueNotSwapFrom INFO [2019-12-17 12:31:22,314] jepsen worker 73 - jepsen.util 4393 :invoke :cas [24 [49 12]] INFO [2019-12-17 12:31:22,326] jepsen worker 18 - jepsen.util 4608 :fail :cas [6 [44 47]] :ValueNotSwapFrom INFO [2019-12-17 12:31:22,326] jepsen worker 18 - jepsen.util 4608 :invoke :cas [6 [14 34]] INFO [2019-12-17 12:31:22,337] jepsen worker 51 - jepsen.util 4371 :info :write [17 42] :RequestCanceledException INFO [2019-12-17 12:31:22,341] jepsen worker 51 - jepsen.util 4461 :invoke :cas [17 [19 9]] INFO [2019-12-17 12:31:22,352] jepsen worker 38 - jepsen.util 4538 :ok :read [12 9] INFO [2019-12-17 12:31:22,352] jepsen worker 38 - jepsen.util 4538 :invoke :read [12 nil] INFO [2019-12-17 12:31:22,359] jepsen worker 11 - jepsen.util 4691 :fail :cas [3 [19 19]] :ValueNotSwapFrom INFO [2019-12-17 12:31:22,359] jepsen worker 11 - jepsen.util 4691 :invoke :cas [3 [5 41]] INFO [2019-12-17 12:31:22,368] jepsen worker 5 - jepsen.util 4145 :ok :write [1 8] INFO [2019-12-17 12:31:22,368] jepsen worker 5 - jepsen.util 4145 :invoke :cas [1 [46 19]] INFO [2019-12-17 12:31:22,370] jepsen worker 32 - jepsen.util 4892 :fail :cas [10 [15 17]] :ValueNotSwapFrom INFO [2019-12-17 12:31:22,370] jepsen worker 32 - jepsen.util 4892 :invoke :cas [10 [11 14]] INFO [2019-12-17 12:31:22,378] jepsen worker 4 - jepsen.util 3784 :fail :cas [1 [4 47]] :CASMismatchException INFO [2019-12-17 12:31:22,378] jepsen worker 4 - jepsen.util 3784 :invoke :cas [1 [0 30]] INFO [2019-12-17 12:31:22,392] jepsen worker 12 - jepsen.util 4872 :ok :write [4 31] INFO [2019-12-17 12:31:22,393] jepsen worker 12 - jepsen.util 4872 :invoke :cas [4 [9 28]] INFO [2019-12-17 12:31:22,394] jepsen worker 27 - jepsen.util 4437 :ok :write [9 19] INFO [2019-12-17 12:31:22,394] jepsen worker 27 - jepsen.util 4437 :invoke :cas [9 [31 15]] INFO [2019-12-17 12:31:22,419] jepsen worker 47 - jepsen.util 5177 :fail :cas [15 [15 20]] :ValueNotSwapFrom INFO [2019-12-17 12:31:22,420] jepsen worker 47 - jepsen.util 5177 :invoke :cas [15 [5 41]] INFO [2019-12-17 12:31:22,425] jepsen worker 21 - jepsen.util 5871 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:31:22,426] jepsen worker 21 - jepsen.util 5871 :invoke :read [7 nil] INFO [2019-12-17 12:31:22,439] jepsen worker 61 - jepsen.util 6811 :info :cas [20 [5 36]] :RequestCanceledException INFO [2019-12-17 12:31:22,447] jepsen worker 61 - jepsen.util 6901 :invoke :write [20 36] INFO [2019-12-17 12:31:22,460] jepsen worker 28 - jepsen.util 3448 :ok :write [9 49] INFO [2019-12-17 12:31:22,460] jepsen worker 28 - jepsen.util 3448 :invoke :read [9 nil] INFO [2019-12-17 12:31:22,477] jepsen worker 13 - jepsen.util 4243 :ok :read [4 31] INFO [2019-12-17 12:31:22,477] jepsen worker 13 - jepsen.util 4243 :invoke :cas [4 [0 2]] INFO [2019-12-17 12:31:22,487] jepsen worker 70 - jepsen.util 5290 :ok :write [23 36] INFO [2019-12-17 12:31:22,488] jepsen worker 70 - jepsen.util 5290 :invoke :write [23 32] INFO [2019-12-17 12:31:22,492] jepsen worker 10 - jepsen.util 4420 :info :write [3 48] :RequestCanceledException INFO [2019-12-17 12:31:22,493] jepsen worker 63 - jepsen.util 3753 :ok :write [21 4] INFO [2019-12-17 12:31:22,493] jepsen worker 63 - jepsen.util 3753 :invoke :cas [21 [13 1]] INFO [2019-12-17 12:31:22,502] jepsen worker 10 - jepsen.util 4510 :invoke :write [3 35] INFO [2019-12-17 12:31:22,502] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26663,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:31:22,503] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26663,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:31:22,504] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26667,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:31:22,505] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26667,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:31:22,505] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26668,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:31:22,506] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26668,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:31:22,526] jepsen worker 23 - jepsen.util 6233 :info :write [7 16] :RequestCanceledException INFO [2019-12-17 12:31:22,527] jepsen worker 59 - jepsen.util 6179 :info :cas [19 [18 8]] :RequestCanceledException INFO [2019-12-17 12:31:22,528] jepsen worker 87 - jepsen.util 4317 :info :write [29 46] :RequestCanceledException INFO [2019-12-17 12:31:22,529] jepsen worker 52 - jepsen.util 5002 :fail :read [17 nil] :RequestTimeoutException INFO [2019-12-17 12:31:22,530] jepsen worker 52 - jepsen.util 5002 :invoke :read [17 nil] INFO [2019-12-17 12:31:22,542] jepsen worker 87 - jepsen.util 4407 :invoke :write [29 46] INFO [2019-12-17 12:31:22,542] jepsen worker 23 - jepsen.util 6323 :invoke :cas [7 [7 44]] INFO [2019-12-17 12:31:22,543] jepsen worker 59 - jepsen.util 6269 :invoke :read [19 nil] INFO [2019-12-17 12:31:22,586] jepsen worker 28 - jepsen.util 3448 :ok :read [9 49] INFO [2019-12-17 12:31:22,586] jepsen worker 28 - jepsen.util 3448 :invoke :write [9 5] INFO [2019-12-17 12:31:22,685] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26680,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:31:22,687] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26680,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:31:22,688] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26686,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:31:22,689] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26686,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:31:22,703] jepsen worker 88 - jepsen.util 4768 :fail :cas [29 [32 22]] :ValueNotSwapFrom INFO [2019-12-17 12:31:22,703] jepsen worker 88 - jepsen.util 4768 :invoke :read [29 nil] INFO [2019-12-17 12:31:22,742] jepsen worker 30 - jepsen.util 4800 :fail :cas [10 [8 48]] :ValueNotSwapFrom INFO [2019-12-17 12:31:22,742] jepsen worker 30 - jepsen.util 4800 :invoke :read [10 nil] INFO [2019-12-17 12:31:22,768] jepsen worker 9 - jepsen.util 5409 :fail :cas [3 [47 1]] :ValueNotSwapFrom INFO [2019-12-17 12:31:22,768] jepsen worker 9 - jepsen.util 5409 :invoke :write [3 3] INFO [2019-12-17 12:31:22,775] jepsen worker 55 - jepsen.util 5545 :fail :cas [18 [36 2]] :ValueNotSwapFrom INFO [2019-12-17 12:31:22,775] jepsen worker 55 - jepsen.util 5545 :invoke :read [18 nil] INFO [2019-12-17 12:31:22,794] jepsen worker 80 - jepsen.util 4580 :fail :cas [26 [1 14]] :ValueNotSwapFrom INFO [2019-12-17 12:31:22,794] jepsen worker 80 - jepsen.util 4580 :invoke :cas [26 [39 13]] INFO [2019-12-17 12:31:22,798] jepsen worker 12 - jepsen.util 4872 :fail :cas [4 [9 28]] :ValueNotSwapFrom INFO [2019-12-17 12:31:22,798] jepsen worker 12 - jepsen.util 4872 :invoke :cas [4 [12 35]] INFO [2019-12-17 12:31:22,801] jepsen worker 71 - jepsen.util 5021 :fail :cas [23 [31 44]] :ValueNotSwapFrom INFO [2019-12-17 12:31:22,801] jepsen worker 71 - jepsen.util 5021 :invoke :read [23 nil] INFO [2019-12-17 12:31:22,801] jepsen worker 56 - jepsen.util 4736 :ok :read [18 18] INFO [2019-12-17 12:31:22,802] jepsen worker 56 - jepsen.util 4736 :invoke :write [18 46] INFO [2019-12-17 12:31:22,805] jepsen worker 51 - jepsen.util 4461 :fail :cas [17 [19 9]] :ValueNotSwapFrom INFO [2019-12-17 12:31:22,805] jepsen worker 51 - jepsen.util 4461 :invoke :read [17 nil] INFO [2019-12-17 12:31:22,806] jepsen worker 65 - jepsen.util 3935 :ok :read [21 4] INFO [2019-12-17 12:31:22,806] jepsen worker 65 - jepsen.util 3935 :invoke :write [21 36] INFO [2019-12-17 12:31:22,808] jepsen worker 89 - jepsen.util 4229 :ok :read [29 45] INFO [2019-12-17 12:31:22,808] jepsen worker 89 - jepsen.util 4229 :invoke :write [29 25] INFO [2019-12-17 12:31:22,811] jepsen worker 63 - jepsen.util 3753 :fail :cas [21 [13 1]] :ValueNotSwapFrom INFO [2019-12-17 12:31:22,811] jepsen worker 63 - jepsen.util 3753 :invoke :cas [21 [6 20]] INFO [2019-12-17 12:31:22,849] jepsen worker 3 - jepsen.util 4773 :ok :write [1 49] INFO [2019-12-17 12:31:22,849] jepsen worker 3 - jepsen.util 4773 :invoke :write [1 3] INFO [2019-12-17 12:31:22,857] jepsen worker 88 - jepsen.util 4768 :ok :read [29 45] INFO [2019-12-17 12:31:22,857] jepsen worker 88 - jepsen.util 4768 :invoke :write [29 35] INFO [2019-12-17 12:31:22,887] jepsen worker 28 - jepsen.util 3448 :ok :write [9 5] INFO [2019-12-17 12:31:22,887] jepsen worker 28 - jepsen.util 3448 :invoke :write [9 26] INFO [2019-12-17 12:31:22,897] jepsen worker 55 - jepsen.util 5545 :ok :read [18 18] INFO [2019-12-17 12:31:22,897] jepsen worker 55 - jepsen.util 5545 :invoke :read [18 nil] INFO [2019-12-17 12:31:22,904] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26701,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:31:22,904] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26701,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:31:22,904] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26704,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:31:22,904] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26704,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:31:22,945] jepsen worker 12 - jepsen.util 4872 :fail :cas [4 [12 35]] :ValueNotSwapFrom INFO [2019-12-17 12:31:22,945] jepsen worker 12 - jepsen.util 4872 :invoke :read [4 nil] INFO [2019-12-17 12:31:22,969] jepsen worker 71 - jepsen.util 5021 :ok :read [23 36] INFO [2019-12-17 12:31:22,969] jepsen worker 71 - jepsen.util 5021 :invoke :read [23 nil] INFO [2019-12-17 12:31:22,974] jepsen worker 13 - jepsen.util 4243 :fail :cas [4 [0 2]] :ValueNotSwapFrom INFO [2019-12-17 12:31:22,974] jepsen worker 13 - jepsen.util 4243 :invoke :cas [4 [44 16]] INFO [2019-12-17 12:31:22,998] jepsen worker 28 - jepsen.util 3448 :ok :write [9 26] INFO [2019-12-17 12:31:22,998] jepsen worker 28 - jepsen.util 3448 :invoke :read [9 nil] INFO [2019-12-17 12:31:22,998] jepsen worker 46 - jepsen.util 4726 :ok :write [15 38] INFO [2019-12-17 12:31:22,998] jepsen worker 46 - jepsen.util 4726 :invoke :read [15 nil] INFO [2019-12-17 12:31:23,000] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26712,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:31:23,000] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26712,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:31:23,000] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26716,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:31:23,000] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26716,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:31:23,001] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26717,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:31:23,001] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26717,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:31:23,012] jepsen worker 51 - jepsen.util 4461 :ok :read [17 42] INFO [2019-12-17 12:31:23,012] jepsen worker 51 - jepsen.util 4461 :invoke :read [17 nil] INFO [2019-12-17 12:31:23,030] jepsen worker 37 - jepsen.util 4447 :ok :write [12 47] INFO [2019-12-17 12:31:23,038] jepsen worker 53 - jepsen.util 4643 :ok :read [17 42] INFO [2019-12-17 12:31:23,056] jepsen worker 31 - jepsen.util 4531 :ok :write [10 13] INFO [2019-12-17 12:31:23,073] jepsen worker 19 - jepsen.util 4699 :ok :write [6 29] INFO [2019-12-17 12:31:23,084] jepsen worker 73 - jepsen.util 4393 :fail :cas [24 [49 12]] :ValueNotSwapFrom INFO [2019-12-17 12:31:23,085] jepsen worker 64 - jepsen.util 3574 :fail :cas [21 [43 43]] :ValueNotSwapFrom INFO [2019-12-17 12:31:23,092] jepsen worker 78 - jepsen.util 4668 :ok :read [26 26] INFO [2019-12-17 12:31:23,115] jepsen worker 28 - jepsen.util 3448 :ok :read [9 26] INFO [2019-12-17 12:31:23,130] jepsen worker 11 - jepsen.util 4691 :fail :cas [3 [5 41]] :ValueNotSwapFrom INFO [2019-12-17 12:31:23,133] jepsen worker 32 - jepsen.util 4892 :fail :cas [10 [11 14]] :ValueNotSwapFrom INFO [2019-12-17 12:31:23,142] jepsen worker 27 - jepsen.util 4437 :fail :cas [9 [31 15]] :ValueNotSwapFrom INFO [2019-12-17 12:31:23,152] jepsen worker 37 - jepsen.util 4447 :invoke :write [12 28] INFO [2019-12-17 12:31:23,152] jepsen worker 53 - jepsen.util 4643 :invoke :read [17 nil] INFO [2019-12-17 12:31:23,152] jepsen worker 31 - jepsen.util 4531 :invoke :read [10 nil] INFO [2019-12-17 12:31:23,152] jepsen worker 19 - jepsen.util 4699 :invoke :read [6 nil] INFO [2019-12-17 12:31:23,152] jepsen worker 73 - jepsen.util 4393 :invoke :write [24 9] INFO [2019-12-17 12:31:23,152] jepsen worker 64 - jepsen.util 3574 :invoke :cas [21 [15 25]] INFO [2019-12-17 12:31:23,152] jepsen worker 78 - jepsen.util 4668 :invoke :cas [26 [32 17]] INFO [2019-12-17 12:31:23,153] jepsen worker 28 - jepsen.util 3448 :invoke :write [9 46] INFO [2019-12-17 12:31:23,153] jepsen worker 11 - jepsen.util 4691 :invoke :write [3 8] INFO [2019-12-17 12:31:23,153] jepsen worker 32 - jepsen.util 4892 :invoke :cas [10 [19 47]] INFO [2019-12-17 12:31:23,154] jepsen worker 27 - jepsen.util 4437 :invoke :write [9 34] INFO [2019-12-17 12:31:23,167] jepsen worker 80 - jepsen.util 4580 :fail :cas [26 [39 13]] :ValueNotSwapFrom INFO [2019-12-17 12:31:23,169] jepsen worker 80 - jepsen.util 4580 :invoke :cas [26 [29 30]] INFO [2019-12-17 12:31:23,176] jepsen worker 47 - jepsen.util 5177 :fail :cas [15 [5 41]] :ValueNotSwapFrom INFO [2019-12-17 12:31:23,176] jepsen worker 47 - jepsen.util 5177 :invoke :read [15 nil] INFO [2019-12-17 12:31:23,192] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26734,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:31:23,192] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26734,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:31:23,192] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26735,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:31:23,192] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26735,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:31:23,200] jepsen worker 46 - jepsen.util 4726 :ok :read [15 38] INFO [2019-12-17 12:31:23,201] jepsen worker 46 - jepsen.util 4726 :invoke :cas [15 [13 14]] INFO [2019-12-17 12:31:23,214] jepsen worker 36 - jepsen.util 4716 :ok :read [12 47] INFO [2019-12-17 12:31:23,214] jepsen worker 36 - jepsen.util 4716 :invoke :write [12 38] INFO [2019-12-17 12:31:23,228] jepsen worker 88 - jepsen.util 4768 :ok :write [29 35] INFO [2019-12-17 12:31:23,229] jepsen worker 88 - jepsen.util 4768 :invoke :cas [29 [48 23]] INFO [2019-12-17 12:31:23,238] jepsen worker 31 - jepsen.util 4531 :ok :read [10 13] INFO [2019-12-17 12:31:23,238] jepsen worker 31 - jepsen.util 4531 :invoke :write [10 28] INFO [2019-12-17 12:31:23,268] jepsen worker 64 - jepsen.util 3574 :fail :cas [21 [15 25]] :ValueNotSwapFrom INFO [2019-12-17 12:31:23,268] jepsen worker 64 - jepsen.util 3574 :invoke :write [21 21] INFO [2019-12-17 12:31:23,274] jepsen worker 19 - jepsen.util 4699 :ok :read [6 29] INFO [2019-12-17 12:31:23,275] jepsen worker 19 - jepsen.util 4699 :invoke :write [6 48] INFO [2019-12-17 12:31:23,275] jepsen worker 14 - jepsen.util 4784 :fail :cas [4 [27 29]] :ValueNotSwapFrom INFO [2019-12-17 12:31:23,275] jepsen worker 14 - jepsen.util 4784 :invoke :read [4 nil] INFO [2019-12-17 12:31:23,297] jepsen worker 32 - jepsen.util 4892 :fail :cas [10 [19 47]] :ValueNotSwapFrom INFO [2019-12-17 12:31:23,298] jepsen worker 32 - jepsen.util 4892 :invoke :write [10 14] INFO [2019-12-17 12:31:23,301] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26742,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:31:23,302] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26742,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:31:23,302] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26745,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:31:23,302] jepsen worker 12 - jepsen.util 4872 :ok :read [4 31] INFO [2019-12-17 12:31:23,302] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26745,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:31:23,302] jepsen worker 12 - jepsen.util 4872 :invoke :cas [4 [0 46]] INFO [2019-12-17 12:31:23,302] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26746,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:31:23,302] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26746,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:31:23,305] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:31:23,306] jepsen nemesis - couchbase.util Rebalance check count: 200 INFO [2019-12-17 12:31:23,306] jepsen nemesis - couchbase.util Rebalance stuck count: 181 INFO [2019-12-17 12:31:23,331] jepsen worker 38 - jepsen.util 4538 :ok :read [12 47] INFO [2019-12-17 12:31:23,332] jepsen worker 38 - jepsen.util 4538 :invoke :read [12 nil] INFO [2019-12-17 12:31:23,341] jepsen worker 4 - jepsen.util 3784 :fail :cas [1 [0 30]] :ValueNotSwapFrom INFO [2019-12-17 12:31:23,341] jepsen worker 4 - jepsen.util 3784 :invoke :write [1 43] INFO [2019-12-17 12:31:23,345] jepsen worker 28 - jepsen.util 3448 :ok :write [9 46] INFO [2019-12-17 12:31:23,346] jepsen worker 28 - jepsen.util 3448 :invoke :cas [9 [6 25]] INFO [2019-12-17 12:31:23,349] jepsen worker 88 - jepsen.util 4768 :fail :cas [29 [48 23]] :ValueNotSwapFrom INFO [2019-12-17 12:31:23,349] jepsen worker 88 - jepsen.util 4768 :invoke :read [29 nil] INFO [2019-12-17 12:31:23,375] jepsen worker 12 - jepsen.util 4872 :fail :cas [4 [0 46]] :ValueNotSwapFrom INFO [2019-12-17 12:31:23,376] jepsen worker 12 - jepsen.util 4872 :invoke :write [4 49] INFO [2019-12-17 12:31:23,376] jepsen worker 55 - jepsen.util 5545 :ok :read [18 18] INFO [2019-12-17 12:31:23,376] jepsen worker 55 - jepsen.util 5545 :invoke :cas [18 [42 19]] INFO [2019-12-17 12:31:23,380] jepsen worker 28 - jepsen.util 3448 :fail :cas [9 [6 25]] :ValueNotSwapFrom INFO [2019-12-17 12:31:23,381] jepsen worker 28 - jepsen.util 3448 :invoke :read [9 nil] INFO [2019-12-17 12:31:23,396] jepsen worker 28 - jepsen.util 3448 :ok :read [9 46] INFO [2019-12-17 12:31:23,397] jepsen worker 28 - jepsen.util 3448 :invoke :write [9 1] INFO [2019-12-17 12:31:23,408] jepsen worker 88 - jepsen.util 4768 :ok :read [29 35] INFO [2019-12-17 12:31:23,408] jepsen worker 88 - jepsen.util 4768 :invoke :cas [29 [46 18]] INFO [2019-12-17 12:31:23,415] jepsen worker 71 - jepsen.util 5021 :ok :read [23 36] INFO [2019-12-17 12:31:23,415] jepsen worker 71 - jepsen.util 5021 :invoke :write [23 19] INFO [2019-12-17 12:31:23,422] jepsen worker 89 - jepsen.util 4229 :ok :write [29 25] INFO [2019-12-17 12:31:23,422] jepsen worker 89 - jepsen.util 4229 :invoke :read [29 nil] INFO [2019-12-17 12:31:23,451] jepsen worker 28 - jepsen.util 3448 :ok :write [9 1] INFO [2019-12-17 12:31:23,451] jepsen worker 28 - jepsen.util 3448 :invoke :cas [9 [4 12]] INFO [2019-12-17 12:31:23,470] jepsen worker 51 - jepsen.util 4461 :ok :read [17 42] INFO [2019-12-17 12:31:23,470] jepsen worker 51 - jepsen.util 4461 :invoke :read [17 nil] INFO [2019-12-17 12:31:23,470] jepsen worker 28 - jepsen.util 3448 :fail :cas [9 [4 12]] :ValueNotSwapFrom INFO [2019-12-17 12:31:23,470] jepsen worker 28 - jepsen.util 3448 :invoke :write [9 14] INFO [2019-12-17 12:31:23,497] jepsen worker 55 - jepsen.util 5545 :fail :cas [18 [42 19]] :ValueNotSwapFrom INFO [2019-12-17 12:31:23,497] jepsen worker 55 - jepsen.util 5545 :invoke :cas [18 [2 26]] INFO [2019-12-17 12:31:23,497] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26767,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:31:23,497] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26767,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:31:23,497] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26770,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:31:23,497] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26770,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:31:23,497] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26771,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:31:23,498] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26771,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:31:23,525] jepsen worker 37 - jepsen.util 4447 :ok :write [12 28] INFO [2019-12-17 12:31:23,525] jepsen worker 37 - jepsen.util 4447 :invoke :write [12 26] INFO [2019-12-17 12:31:23,536] jepsen worker 28 - jepsen.util 3448 :ok :write [9 14] INFO [2019-12-17 12:31:23,536] jepsen worker 28 - jepsen.util 3448 :invoke :cas [9 [47 22]] INFO [2019-12-17 12:31:23,552] jepsen worker 69 - jepsen.util 4839 :ok :write [23 27] INFO [2019-12-17 12:31:23,552] jepsen worker 69 - jepsen.util 4839 :invoke :write [23 7] INFO [2019-12-17 12:31:23,552] jepsen worker 64 - jepsen.util 3574 :ok :write [21 21] INFO [2019-12-17 12:31:23,553] jepsen worker 64 - jepsen.util 3574 :invoke :cas [21 [37 25]] INFO [2019-12-17 12:31:23,558] jepsen worker 28 - jepsen.util 3448 :fail :cas [9 [47 22]] :ValueNotSwapFrom INFO [2019-12-17 12:31:23,558] jepsen worker 28 - jepsen.util 3448 :invoke :read [9 nil] INFO [2019-12-17 12:31:23,574] jepsen worker 28 - jepsen.util 3448 :ok :read [9 14] INFO [2019-12-17 12:31:23,575] jepsen worker 28 - jepsen.util 3448 :invoke :write [9 28] INFO [2019-12-17 12:31:23,577] jepsen worker 64 - jepsen.util 3574 :fail :cas [21 [37 25]] :ValueNotSwapFrom INFO [2019-12-17 12:31:23,577] jepsen worker 64 - jepsen.util 3574 :invoke :write [21 20] INFO [2019-12-17 12:31:23,609] jepsen worker 51 - jepsen.util 4461 :ok :read [17 42] INFO [2019-12-17 12:31:23,609] jepsen worker 51 - jepsen.util 4461 :invoke :write [17 16] INFO [2019-12-17 12:31:23,624] jepsen worker 28 - jepsen.util 3448 :ok :write [9 28] INFO [2019-12-17 12:31:23,624] jepsen worker 28 - jepsen.util 3448 :invoke :write [9 27] INFO [2019-12-17 12:31:23,637] jepsen worker 5 - jepsen.util 4145 :fail :cas [1 [46 19]] :ValueNotSwapFrom INFO [2019-12-17 12:31:23,638] jepsen worker 5 - jepsen.util 4145 :invoke :write [1 14] INFO [2019-12-17 12:31:23,659] jepsen worker 64 - jepsen.util 3574 :ok :write [21 20] INFO [2019-12-17 12:31:23,659] jepsen worker 64 - jepsen.util 3574 :invoke :cas [21 [27 33]] INFO [2019-12-17 12:31:23,666] jepsen worker 55 - jepsen.util 5545 :fail :cas [18 [2 26]] :ValueNotSwapFrom INFO [2019-12-17 12:31:23,666] jepsen worker 55 - jepsen.util 5545 :invoke :write [18 24] INFO [2019-12-17 12:31:23,675] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26787,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:31:23,675] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26787,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:31:23,675] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26790,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:31:23,675] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26790,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:31:23,677] jepsen worker 11 - jepsen.util 4691 :ok :write [3 8] INFO [2019-12-17 12:31:23,677] jepsen worker 11 - jepsen.util 4691 :invoke :read [3 nil] INFO [2019-12-17 12:31:23,680] jepsen worker 64 - jepsen.util 3574 :fail :cas [21 [27 33]] :ValueNotSwapFrom INFO [2019-12-17 12:31:23,680] jepsen worker 64 - jepsen.util 3574 :invoke :write [21 49] INFO [2019-12-17 12:31:23,691] jepsen worker 12 - jepsen.util 4872 :ok :write [4 49] INFO [2019-12-17 12:31:23,691] jepsen worker 12 - jepsen.util 4872 :invoke :read [4 nil] INFO [2019-12-17 12:31:23,706] jepsen worker 31 - jepsen.util 4531 :ok :write [10 28] INFO [2019-12-17 12:31:23,706] jepsen worker 31 - jepsen.util 4531 :invoke :cas [10 [10 20]] INFO [2019-12-17 12:31:23,709] jepsen worker 88 - jepsen.util 4768 :fail :cas [29 [46 18]] :ValueNotSwapFrom INFO [2019-12-17 12:31:23,710] jepsen worker 88 - jepsen.util 4768 :invoke :cas [29 [11 21]] INFO [2019-12-17 12:31:23,718] jepsen worker 47 - jepsen.util 5177 :ok :read [15 38] INFO [2019-12-17 12:31:23,718] jepsen worker 47 - jepsen.util 5177 :invoke :write [15 17] INFO [2019-12-17 12:31:23,721] jepsen worker 12 - jepsen.util 4872 :ok :read [4 49] INFO [2019-12-17 12:31:23,721] jepsen worker 12 - jepsen.util 4872 :invoke :read [4 nil] INFO [2019-12-17 12:31:23,745] jepsen worker 54 - jepsen.util 5094 :ok :write [18 20] INFO [2019-12-17 12:31:23,745] jepsen worker 36 - jepsen.util 4716 :ok :write [12 38] INFO [2019-12-17 12:31:23,745] jepsen worker 54 - jepsen.util 5094 :invoke :write [18 46] INFO [2019-12-17 12:31:23,745] jepsen worker 36 - jepsen.util 4716 :invoke :read [12 nil] INFO [2019-12-17 12:31:23,757] jepsen worker 28 - jepsen.util 3448 :ok :write [9 27] INFO [2019-12-17 12:31:23,757] jepsen worker 28 - jepsen.util 3448 :invoke :cas [9 [45 42]] INFO [2019-12-17 12:31:23,777] jepsen worker 88 - jepsen.util 4768 :fail :cas [29 [11 21]] :ValueNotSwapFrom INFO [2019-12-17 12:31:23,777] jepsen worker 88 - jepsen.util 4768 :invoke :cas [29 [16 40]] INFO [2019-12-17 12:31:23,780] jepsen worker 14 - jepsen.util 4784 :ok :read [4 49] INFO [2019-12-17 12:31:23,780] jepsen worker 14 - jepsen.util 4784 :invoke :write [4 29] INFO [2019-12-17 12:31:23,787] jepsen worker 28 - jepsen.util 3448 :fail :cas [9 [45 42]] :ValueNotSwapFrom INFO [2019-12-17 12:31:23,788] jepsen worker 28 - jepsen.util 3448 :invoke :write [9 14] INFO [2019-12-17 12:31:23,802] jepsen worker 38 - jepsen.util 4538 :ok :read [12 38] INFO [2019-12-17 12:31:23,803] jepsen worker 38 - jepsen.util 4538 :invoke :write [12 9] INFO [2019-12-17 12:31:23,813] jepsen worker 87 - jepsen.util 4407 :ok :write [29 46] INFO [2019-12-17 12:31:23,813] jepsen worker 87 - jepsen.util 4407 :invoke :read [29 nil] INFO [2019-12-17 12:31:23,821] jepsen worker 12 - jepsen.util 4872 :ok :read [4 49] INFO [2019-12-17 12:31:23,821] jepsen worker 12 - jepsen.util 4872 :invoke :cas [4 [11 35]] INFO [2019-12-17 12:31:23,844] jepsen worker 51 - jepsen.util 4461 :ok :write [17 16] INFO [2019-12-17 12:31:23,845] jepsen worker 51 - jepsen.util 4461 :invoke :write [17 28] INFO [2019-12-17 12:31:23,867] jepsen worker 64 - jepsen.util 3574 :ok :write [21 49] INFO [2019-12-17 12:31:23,867] jepsen worker 64 - jepsen.util 3574 :invoke :cas [21 [41 48]] INFO [2019-12-17 12:31:23,886] jepsen worker 64 - jepsen.util 3574 :fail :cas [21 [41 48]] :ValueNotSwapFrom INFO [2019-12-17 12:31:23,886] jepsen worker 64 - jepsen.util 3574 :invoke :read [21 nil] INFO [2019-12-17 12:31:23,892] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26812,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:31:23,892] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26812,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:31:23,892] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26815,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:31:23,892] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26815,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:31:23,892] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26816,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:31:23,892] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26816,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:31:23,895] jepsen worker 88 - jepsen.util 4768 :fail :cas [29 [16 40]] :ValueNotSwapFrom INFO [2019-12-17 12:31:23,895] jepsen worker 88 - jepsen.util 4768 :invoke :write [29 49] INFO [2019-12-17 12:31:23,907] jepsen worker 64 - jepsen.util 3574 :ok :read [21 49] INFO [2019-12-17 12:31:23,907] jepsen worker 64 - jepsen.util 3574 :invoke :read [21 nil] INFO [2019-12-17 12:31:23,909] jepsen worker 28 - jepsen.util 3448 :ok :write [9 14] INFO [2019-12-17 12:31:23,909] jepsen worker 28 - jepsen.util 3448 :invoke :cas [9 [36 17]] INFO [2019-12-17 12:31:23,913] jepsen worker 79 - jepsen.util 4489 :ok :write [26 11] INFO [2019-12-17 12:31:23,913] jepsen worker 79 - jepsen.util 4489 :invoke :write [26 48] INFO [2019-12-17 12:31:23,918] jepsen worker 12 - jepsen.util 4872 :fail :cas [4 [11 35]] :ValueNotSwapFrom INFO [2019-12-17 12:31:23,918] jepsen worker 12 - jepsen.util 4872 :invoke :write [4 9] INFO [2019-12-17 12:31:23,937] jepsen worker 4 - jepsen.util 3784 :ok :write [1 43] INFO [2019-12-17 12:31:23,937] jepsen worker 4 - jepsen.util 3784 :invoke :read [1 nil] INFO [2019-12-17 12:31:23,947] jepsen worker 52 - jepsen.util 5002 :ok :read [17 16] INFO [2019-12-17 12:31:23,948] jepsen worker 52 - jepsen.util 5002 :invoke :write [17 18] INFO [2019-12-17 12:31:23,960] jepsen worker 28 - jepsen.util 3448 :fail :cas [9 [36 17]] :ValueNotSwapFrom INFO [2019-12-17 12:31:23,960] jepsen worker 28 - jepsen.util 3448 :invoke :cas [9 [7 41]] INFO [2019-12-17 12:31:23,965] jepsen worker 64 - jepsen.util 3574 :ok :read [21 49] INFO [2019-12-17 12:31:23,965] jepsen worker 64 - jepsen.util 3574 :invoke :read [21 nil] INFO [2019-12-17 12:31:23,966] jepsen worker 73 - jepsen.util 4393 :ok :write [24 9] INFO [2019-12-17 12:31:23,966] jepsen worker 73 - jepsen.util 4393 :invoke :read [24 nil] INFO [2019-12-17 12:31:24,000] jepsen worker 28 - jepsen.util 3448 :fail :cas [9 [7 41]] :ValueNotSwapFrom INFO [2019-12-17 12:31:24,000] jepsen worker 28 - jepsen.util 3448 :invoke :write [9 45] INFO [2019-12-17 12:31:24,005] jepsen worker 76 - jepsen.util 6286 :info :cas [25 [37 28]] :RequestCanceledException INFO [2019-12-17 12:31:24,005] jepsen worker 81 - jepsen.util 6111 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:31:24,006] jepsen worker 81 - jepsen.util 6111 :invoke :read [27 nil] INFO [2019-12-17 12:31:24,006] jepsen worker 11 - jepsen.util 4691 :ok :read [3 8] INFO [2019-12-17 12:31:24,006] jepsen worker 11 - jepsen.util 4691 :invoke :cas [3 [32 13]] INFO [2019-12-17 12:31:24,007] jepsen worker 76 - jepsen.util 6376 :invoke :write [25 35] INFO [2019-12-17 12:31:24,025] jepsen worker 57 - jepsen.util 5907 :info :cas [19 [11 13]] :RequestCanceledException INFO [2019-12-17 12:31:24,025] jepsen worker 68 - jepsen.util 5648 :info :cas [22 [17 23]] :RequestCanceledException INFO [2019-12-17 12:31:24,025] jepsen worker 26 - jepsen.util 6416 :info :cas [8 [10 3]] :RequestCanceledException INFO [2019-12-17 12:31:24,025] jepsen worker 58 - jepsen.util 6358 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:31:24,025] jepsen worker 84 - jepsen.util 5844 :info :cas [28 [38 2]] :RequestCanceledException INFO [2019-12-17 12:31:24,026] jepsen worker 58 - jepsen.util 6358 :invoke :cas [19 [12 26]] INFO [2019-12-17 12:31:24,028] jepsen worker 68 - jepsen.util 5738 :invoke :write [22 36] INFO [2019-12-17 12:31:24,029] jepsen worker 57 - jepsen.util 5997 :invoke :read [19 nil] INFO [2019-12-17 12:31:24,029] jepsen worker 84 - jepsen.util 5934 :invoke :read [28 nil] INFO [2019-12-17 12:31:24,029] jepsen worker 26 - jepsen.util 6506 :invoke :cas [8 [6 41]] INFO [2019-12-17 12:31:24,038] jepsen worker 64 - jepsen.util 3574 :ok :read [21 49] INFO [2019-12-17 12:31:24,038] jepsen worker 64 - jepsen.util 3574 :invoke :write [21 6] INFO [2019-12-17 12:31:24,039] jepsen worker 82 - jepsen.util 7012 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:31:24,040] jepsen worker 33 - jepsen.util 6063 :info :cas [11 [0 48]] :RequestCanceledException INFO [2019-12-17 12:31:24,040] jepsen worker 83 - jepsen.util 6653 :info :cas [27 [7 21]] :RequestCanceledException INFO [2019-12-17 12:31:24,040] jepsen worker 82 - jepsen.util 7012 :invoke :write [27 16] INFO [2019-12-17 12:31:24,041] jepsen worker 33 - jepsen.util 6153 :invoke :write [11 9] INFO [2019-12-17 12:31:24,041] jepsen worker 83 - jepsen.util 6743 :invoke :cas [27 [43 18]] INFO [2019-12-17 12:31:24,042] jepsen worker 88 - jepsen.util 4768 :ok :write [29 49] INFO [2019-12-17 12:31:24,042] jepsen worker 88 - jepsen.util 4768 :invoke :write [29 44] INFO [2019-12-17 12:31:24,049] jepsen worker 32 - jepsen.util 4892 :ok :write [10 14] INFO [2019-12-17 12:31:24,049] jepsen worker 32 - jepsen.util 4892 :invoke :read [10 nil] INFO [2019-12-17 12:31:24,051] jepsen worker 34 - jepsen.util 5704 :info :cas [11 [33 9]] :RequestCanceledException INFO [2019-12-17 12:31:24,051] jepsen worker 62 - jepsen.util 5822 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:31:24,051] jepsen worker 8 - jepsen.util 5498 :info :cas [2 [36 44]] :RequestCanceledException INFO [2019-12-17 12:31:24,051] jepsen worker 6 - jepsen.util 6666 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:31:24,052] jepsen worker 62 - jepsen.util 5822 :invoke :write [20 21] INFO [2019-12-17 12:31:24,052] jepsen worker 6 - jepsen.util 6666 :invoke :cas [2 [46 7]] INFO [2019-12-17 12:31:24,053] jepsen worker 34 - jepsen.util 5794 :invoke :cas [11 [46 30]] INFO [2019-12-17 12:31:24,054] jepsen worker 8 - jepsen.util 5588 :invoke :write [2 46] INFO [2019-12-17 12:31:24,058] jepsen worker 53 - jepsen.util 4643 :ok :read [17 16] INFO [2019-12-17 12:31:24,058] jepsen worker 53 - jepsen.util 4643 :invoke :read [17 nil] INFO [2019-12-17 12:31:24,084] jepsen worker 31 - jepsen.util 4531 :fail :cas [10 [10 20]] :ValueNotSwapFrom INFO [2019-12-17 12:31:24,084] jepsen worker 31 - jepsen.util 4531 :invoke :cas [10 [47 35]] INFO [2019-12-17 12:31:24,087] jepsen worker 78 - jepsen.util 4668 :fail :cas [26 [32 17]] :ValueNotSwapFrom INFO [2019-12-17 12:31:24,087] jepsen worker 78 - jepsen.util 4668 :invoke :cas [26 [1 32]] INFO [2019-12-17 12:31:24,092] jepsen worker 25 - jepsen.util 6325 :info :cas [8 [31 49]] :RequestCanceledException INFO [2019-12-17 12:31:24,092] jepsen worker 49 - jepsen.util 5539 :info :cas [16 [48 10]] :RequestCanceledException INFO [2019-12-17 12:31:24,092] jepsen worker 43 - jepsen.util 5983 :info :cas [14 [12 31]] :RequestCanceledException INFO [2019-12-17 12:31:24,092] jepsen worker 20 - jepsen.util 4700 :info :write [6 1] :RequestCanceledException INFO [2019-12-17 12:31:24,092] jepsen worker 45 - jepsen.util 4815 :info :write [15 6] :RequestCanceledException INFO [2019-12-17 12:31:24,093] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26831,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:31:24,093] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26831,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:31:24,093] jepsen worker 45 - jepsen.util 4905 :invoke :read [15 nil] INFO [2019-12-17 12:31:24,094] jepsen worker 43 - jepsen.util 6073 :invoke :read [14 nil] INFO [2019-12-17 12:31:24,094] jepsen worker 49 - jepsen.util 5629 :invoke :write [16 13] INFO [2019-12-17 12:31:24,094] jepsen worker 25 - jepsen.util 6415 :invoke :read [8 nil] INFO [2019-12-17 12:31:24,094] jepsen worker 20 - jepsen.util 4790 :invoke :cas [6 [39 32]] INFO [2019-12-17 12:31:24,106] jepsen worker 28 - jepsen.util 3448 :ok :write [9 45] INFO [2019-12-17 12:31:24,106] jepsen worker 28 - jepsen.util 3448 :invoke :cas [9 [45 11]] INFO [2019-12-17 12:31:24,107] jepsen worker 32 - jepsen.util 4892 :ok :read [10 14] INFO [2019-12-17 12:31:24,107] jepsen worker 32 - jepsen.util 4892 :invoke :read [10 nil] INFO [2019-12-17 12:31:24,109] jepsen worker 60 - jepsen.util 6720 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:31:24,109] jepsen worker 42 - jepsen.util 6882 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:31:24,109] jepsen worker 16 - jepsen.util 7126 :info :cas [5 [0 26]] :RequestCanceledException INFO [2019-12-17 12:31:24,109] jepsen worker 39 - jepsen.util 5799 :info :cas [13 [42 25]] :RequestCanceledException INFO [2019-12-17 12:31:24,109] jepsen worker 60 - jepsen.util 6720 :invoke :cas [20 [36 10]] INFO [2019-12-17 12:31:24,109] jepsen worker 42 - jepsen.util 6882 :invoke :read [14 nil] INFO [2019-12-17 12:31:24,110] jepsen worker 16 - jepsen.util 7216 :invoke :write [5 23] INFO [2019-12-17 12:31:24,110] jepsen worker 39 - jepsen.util 5889 :invoke :cas [13 [17 47]] INFO [2019-12-17 12:31:24,123] jepsen worker 15 - jepsen.util 5235 :info :write [5 0] :RequestCanceledException INFO [2019-12-17 12:31:24,124] jepsen worker 15 - jepsen.util 5325 :invoke :cas [5 [35 28]] INFO [2019-12-17 12:31:24,139] jepsen worker 24 - jepsen.util 5874 :info :write [8 17] :RequestCanceledException INFO [2019-12-17 12:31:24,139] jepsen worker 50 - jepsen.util 6440 :info :write [16 38] :RequestCanceledException INFO [2019-12-17 12:31:24,139] jepsen worker 85 - jepsen.util 6925 :info :write [28 42] :RequestCanceledException INFO [2019-12-17 12:31:24,139] jepsen worker 48 - jepsen.util 5538 :info :write [16 23] :RequestCanceledException INFO [2019-12-17 12:31:24,139] jepsen worker 70 - jepsen.util 5290 :ok :write [23 32] INFO [2019-12-17 12:31:24,140] jepsen worker 70 - jepsen.util 5290 :invoke :read [23 nil] INFO [2019-12-17 12:31:24,140] jepsen worker 24 - jepsen.util 5964 :invoke :read [8 nil] INFO [2019-12-17 12:31:24,140] jepsen worker 50 - jepsen.util 6530 :invoke :write [16 36] INFO [2019-12-17 12:31:24,140] jepsen worker 48 - jepsen.util 5628 :invoke :write [16 25] INFO [2019-12-17 12:31:24,140] jepsen worker 85 - jepsen.util 7015 :invoke :write [28 3] INFO [2019-12-17 12:31:24,154] jepsen worker 66 - jepsen.util 5736 :info :write [22 36] :RequestCanceledException INFO [2019-12-17 12:31:24,154] jepsen worker 77 - jepsen.util 6377 :info :write [25 11] :RequestCanceledException INFO [2019-12-17 12:31:24,154] jepsen worker 17 - jepsen.util 6047 :info :write [5 49] :RequestCanceledException INFO [2019-12-17 12:31:24,155] jepsen worker 66 - jepsen.util 5826 :invoke :write [22 0] INFO [2019-12-17 12:31:24,155] jepsen worker 17 - jepsen.util 6137 :invoke :write [5 20] INFO [2019-12-17 12:31:24,155] jepsen worker 77 - jepsen.util 6467 :invoke :write [25 26] INFO [2019-12-17 12:31:24,181] jepsen worker 80 - jepsen.util 4580 :fail :cas [26 [29 30]] :ValueNotSwapFrom INFO [2019-12-17 12:31:24,181] jepsen worker 80 - jepsen.util 4580 :invoke :write [26 42] INFO [2019-12-17 12:31:24,189] jepsen worker 22 - jepsen.util 6232 :info :write [7 17] :RequestCanceledException INFO [2019-12-17 12:31:24,189] jepsen worker 40 - jepsen.util 6970 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:31:24,189] jepsen worker 7 - jepsen.util 6217 :info :write [2 18] :RequestCanceledException INFO [2019-12-17 12:31:24,189] jepsen worker 2 - jepsen.util 5762 :info :write [0 39] :RequestCanceledException INFO [2019-12-17 12:31:24,189] jepsen worker 40 - jepsen.util 6970 :invoke :cas [13 [2 27]] INFO [2019-12-17 12:31:24,190] jepsen worker 55 - jepsen.util 5545 :ok :write [18 24] INFO [2019-12-17 12:31:24,190] jepsen worker 55 - jepsen.util 5545 :invoke :cas [18 [13 38]] INFO [2019-12-17 12:31:24,190] jepsen worker 22 - jepsen.util 6322 :invoke :write [7 6] INFO [2019-12-17 12:31:24,190] jepsen worker 7 - jepsen.util 6307 :invoke :write [2 11] INFO [2019-12-17 12:31:24,190] jepsen worker 2 - jepsen.util 5852 :invoke :cas [0 [28 19]] INFO [2019-12-17 12:31:24,199] jepsen worker 78 - jepsen.util 4668 :fail :cas [26 [1 32]] :ValueNotSwapFrom INFO [2019-12-17 12:31:24,199] jepsen worker 78 - jepsen.util 4668 :invoke :write [26 8] INFO [2019-12-17 12:31:24,210] jepsen worker 35 - jepsen.util 6335 :info :write [11 43] :RequestCanceledException INFO [2019-12-17 12:31:24,210] jepsen worker 0 - jepsen.util 6840 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:31:24,211] jepsen worker 44 - jepsen.util 6794 :info :write [14 45] :RequestCanceledException INFO [2019-12-17 12:31:24,211] jepsen worker 0 - jepsen.util 6840 :invoke :write [0 48] INFO [2019-12-17 12:31:24,212] jepsen worker 44 - jepsen.util 6884 :invoke :write [14 21] INFO [2019-12-17 12:31:24,212] jepsen worker 35 - jepsen.util 6425 :invoke :read [11 nil] INFO [2019-12-17 12:31:24,225] jepsen worker 86 - jepsen.util 6476 :info :write [28 4] :RequestCanceledException INFO [2019-12-17 12:31:24,225] jepsen worker 1 - jepsen.util 5941 :info :write [0 7] :RequestCanceledException INFO [2019-12-17 12:31:24,225] jepsen worker 67 - jepsen.util 6367 :info :write [22 35] :RequestCanceledException INFO [2019-12-17 12:31:24,225] jepsen worker 41 - jepsen.util 6611 :info :write [13 48] :RequestCanceledException INFO [2019-12-17 12:31:24,225] jepsen worker 72 - jepsen.util 4662 :info :write [24 4] :RequestCanceledException INFO [2019-12-17 12:31:24,225] jepsen worker 75 - jepsen.util 6105 :info :write [25 42] :RequestCanceledException INFO [2019-12-17 12:31:24,226] jepsen worker 41 - jepsen.util 6701 :invoke :write [13 37] INFO [2019-12-17 12:31:24,226] jepsen worker 67 - jepsen.util 6457 :invoke :write [22 6] INFO [2019-12-17 12:31:24,226] jepsen worker 1 - jepsen.util 6031 :invoke :write [0 38] INFO [2019-12-17 12:31:24,226] jepsen worker 86 - jepsen.util 6566 :invoke :write [28 39] INFO [2019-12-17 12:31:24,226] jepsen worker 75 - jepsen.util 6195 :invoke :cas [25 [16 27]] INFO [2019-12-17 12:31:24,227] jepsen worker 72 - jepsen.util 4752 :invoke :write [24 4] INFO [2019-12-17 12:31:24,234] jepsen worker 31 - jepsen.util 4531 :fail :cas [10 [47 35]] :ValueNotSwapFrom INFO [2019-12-17 12:31:24,234] jepsen worker 31 - jepsen.util 4531 :invoke :write [10 7] INFO [2019-12-17 12:31:24,262] jepsen worker 20 - jepsen.util 4790 :fail :cas [6 [39 32]] :ValueNotSwapFrom INFO [2019-12-17 12:31:24,263] jepsen worker 20 - jepsen.util 4790 :invoke :write [6 10] INFO [2019-12-17 12:31:24,263] jepsen worker 89 - jepsen.util 4229 :ok :read [29 49] INFO [2019-12-17 12:31:24,264] jepsen worker 89 - jepsen.util 4229 :invoke :cas [29 [35 29]] INFO [2019-12-17 12:31:24,265] jepsen worker 32 - jepsen.util 4892 :ok :read [10 14] INFO [2019-12-17 12:31:24,265] jepsen worker 32 - jepsen.util 4892 :invoke :read [10 nil] INFO [2019-12-17 12:31:24,280] jepsen worker 5 - jepsen.util 4145 :ok :write [1 14] INFO [2019-12-17 12:31:24,280] jepsen worker 64 - jepsen.util 3574 :ok :write [21 6] INFO [2019-12-17 12:31:24,280] jepsen worker 5 - jepsen.util 4145 :invoke :write [1 2] INFO [2019-12-17 12:31:24,280] jepsen worker 64 - jepsen.util 3574 :invoke :write [21 12] INFO [2019-12-17 12:31:24,280] jepsen worker 47 - jepsen.util 5177 :ok :write [15 17] INFO [2019-12-17 12:31:24,280] jepsen worker 47 - jepsen.util 5177 :invoke :cas [15 [2 16]] INFO [2019-12-17 12:31:24,292] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26882,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:31:24,292] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26892,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:31:24,292] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26882,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:31:24,292] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26892,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:31:24,292] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26884,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:31:24,292] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26895,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:31:24,292] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26884,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:31:24,292] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26895,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:31:24,292] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26896,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:31:24,292] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26896,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:31:24,305] jepsen worker 28 - jepsen.util 3448 :ok :cas [9 [45 11]] INFO [2019-12-17 12:31:24,305] jepsen worker 28 - jepsen.util 3448 :invoke :cas [9 [3 7]] INFO [2019-12-17 12:31:24,308] jepsen worker 74 - jepsen.util 5024 :fail :read [24 nil] :RequestTimeoutException INFO [2019-12-17 12:31:24,308] jepsen worker 74 - jepsen.util 5024 :invoke :write [24 27] INFO [2019-12-17 12:31:24,314] jepsen worker 73 - jepsen.util 4393 :ok :read [24 9] INFO [2019-12-17 12:31:24,314] jepsen worker 73 - jepsen.util 4393 :invoke :cas [24 [44 35]] INFO [2019-12-17 12:31:24,325] jepsen worker 29 - jepsen.util 3719 :info :write [9 9] :RequestCanceledException INFO [2019-12-17 12:31:24,326] jepsen worker 29 - jepsen.util 3809 :invoke :cas [9 [3 30]] INFO [2019-12-17 12:31:24,336] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:31:24,336] jepsen nemesis - couchbase.util Rebalance check count: 201 INFO [2019-12-17 12:31:24,336] jepsen nemesis - couchbase.util Rebalance stuck count: 182 INFO [2019-12-17 12:31:24,338] jepsen worker 70 - jepsen.util 5290 :ok :read [23 32] INFO [2019-12-17 12:31:24,338] jepsen worker 70 - jepsen.util 5290 :invoke :write [23 11] INFO [2019-12-17 12:31:24,341] jepsen worker 11 - jepsen.util 4691 :fail :cas [3 [32 13]] :ValueNotSwapFrom INFO [2019-12-17 12:31:24,341] jepsen worker 11 - jepsen.util 4691 :invoke :write [3 30] INFO [2019-12-17 12:31:24,356] jepsen worker 47 - jepsen.util 5177 :fail :cas [15 [2 16]] :ValueNotSwapFrom INFO [2019-12-17 12:31:24,357] jepsen worker 47 - jepsen.util 5177 :invoke :cas [15 [27 43]] INFO [2019-12-17 12:31:24,360] jepsen worker 87 - jepsen.util 4407 :ok :read [29 49] INFO [2019-12-17 12:31:24,360] jepsen worker 87 - jepsen.util 4407 :invoke :cas [29 [30 44]] INFO [2019-12-17 12:31:24,379] jepsen worker 55 - jepsen.util 5545 :fail :cas [18 [13 38]] :ValueNotSwapFrom INFO [2019-12-17 12:31:24,379] jepsen worker 55 - jepsen.util 5545 :invoke :cas [18 [34 39]] INFO [2019-12-17 12:31:24,388] jepsen worker 64 - jepsen.util 3574 :ok :write [21 12] INFO [2019-12-17 12:31:24,388] jepsen worker 64 - jepsen.util 3574 :invoke :read [21 nil] INFO [2019-12-17 12:31:24,390] jepsen worker 73 - jepsen.util 4393 :fail :cas [24 [44 35]] :ValueNotSwapFrom INFO [2019-12-17 12:31:24,390] jepsen worker 73 - jepsen.util 4393 :invoke :cas [24 [21 33]] INFO [2019-12-17 12:31:24,409] jepsen worker 18 - jepsen.util 4608 :info :cas [6 [14 34]] :RequestCanceledException INFO [2019-12-17 12:31:24,410] jepsen worker 18 - jepsen.util 4698 :invoke :cas [6 [26 15]] INFO [2019-12-17 12:31:24,420] jepsen worker 64 - jepsen.util 3574 :ok :read [21 12] INFO [2019-12-17 12:31:24,420] jepsen worker 64 - jepsen.util 3574 :invoke :cas [21 [0 4]] INFO [2019-12-17 12:31:24,449] jepsen worker 64 - jepsen.util 3574 :fail :cas [21 [0 4]] :ValueNotSwapFrom INFO [2019-12-17 12:31:24,449] jepsen worker 64 - jepsen.util 3574 :invoke :write [21 36] INFO [2019-12-17 12:31:24,478] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26927,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:31:24,478] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26927,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:31:24,478] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26931,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:31:24,478] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26931,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:31:24,512] jepsen worker 64 - jepsen.util 3574 :ok :write [21 36] INFO [2019-12-17 12:31:24,512] jepsen worker 64 - jepsen.util 3574 :invoke :write [21 4] INFO [2019-12-17 12:31:24,516] jepsen worker 55 - jepsen.util 5545 :fail :cas [18 [34 39]] :ValueNotSwapFrom INFO [2019-12-17 12:31:24,516] jepsen worker 55 - jepsen.util 5545 :invoke :write [18 27] INFO [2019-12-17 12:31:24,525] jepsen worker 21 - jepsen.util 5871 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:31:24,525] jepsen worker 21 - jepsen.util 5871 :invoke :write [7 15] INFO [2019-12-17 12:31:24,556] jepsen worker 61 - jepsen.util 6901 :info :write [20 36] :RequestCanceledException INFO [2019-12-17 12:31:24,557] jepsen worker 61 - jepsen.util 6991 :invoke :cas [20 [48 1]] INFO [2019-12-17 12:31:24,587] jepsen worker 64 - jepsen.util 3574 :ok :write [21 4] INFO [2019-12-17 12:31:24,587] jepsen worker 64 - jepsen.util 3574 :invoke :cas [21 [38 32]] INFO [2019-12-17 12:31:24,593] jepsen worker 36 - jepsen.util 4716 :ok :read [12 38] INFO [2019-12-17 12:31:24,594] jepsen worker 36 - jepsen.util 4716 :invoke :cas [12 [34 28]] INFO [2019-12-17 12:31:24,601] jepsen worker 4 - jepsen.util 3784 :ok :read [1 14] INFO [2019-12-17 12:31:24,601] jepsen worker 4 - jepsen.util 3784 :invoke :write [1 29] INFO [2019-12-17 12:31:24,610] jepsen worker 23 - jepsen.util 6323 :info :cas [7 [7 44]] :RequestCanceledException INFO [2019-12-17 12:31:24,610] jepsen worker 23 - jepsen.util 6413 :invoke :cas [7 [27 39]] INFO [2019-12-17 12:31:24,621] jepsen worker 64 - jepsen.util 3574 :fail :cas [21 [38 32]] :ValueNotSwapFrom INFO [2019-12-17 12:31:24,621] jepsen worker 64 - jepsen.util 3574 :invoke :read [21 nil] INFO [2019-12-17 12:31:24,625] jepsen worker 10 - jepsen.util 4510 :info :write [3 35] :RequestCanceledException INFO [2019-12-17 12:31:24,626] jepsen worker 59 - jepsen.util 6269 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:31:24,626] jepsen worker 10 - jepsen.util 4600 :invoke :write [3 29] INFO [2019-12-17 12:31:24,626] jepsen worker 59 - jepsen.util 6269 :invoke :read [19 nil] INFO [2019-12-17 12:31:24,646] jepsen worker 64 - jepsen.util 3574 :ok :read [21 4] INFO [2019-12-17 12:31:24,647] jepsen worker 64 - jepsen.util 3574 :invoke :cas [21 [27 1]] INFO [2019-12-17 12:31:24,652] jepsen worker 53 - jepsen.util 4643 :ok :read [17 16] INFO [2019-12-17 12:31:24,652] jepsen worker 53 - jepsen.util 4643 :invoke :cas [17 [15 35]] INFO [2019-12-17 12:31:24,658] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26945,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:31:24,658] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26945,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:31:24,658] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26948,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:31:24,658] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26948,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:31:24,658] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26949,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:31:24,658] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26949,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:31:24,665] jepsen worker 36 - jepsen.util 4716 :fail :cas [12 [34 28]] :ValueNotSwapFrom INFO [2019-12-17 12:31:24,665] jepsen worker 36 - jepsen.util 4716 :invoke :write [12 36] INFO [2019-12-17 12:31:24,692] jepsen worker 64 - jepsen.util 3574 :fail :cas [21 [27 1]] :ValueNotSwapFrom INFO [2019-12-17 12:31:24,692] jepsen worker 64 - jepsen.util 3574 :invoke :write [21 29] INFO [2019-12-17 12:31:24,717] jepsen worker 89 - jepsen.util 4229 :fail :cas [29 [35 29]] :ValueNotSwapFrom INFO [2019-12-17 12:31:24,717] jepsen worker 89 - jepsen.util 4229 :invoke :write [29 36] INFO [2019-12-17 12:31:24,731] jepsen worker 31 - jepsen.util 4531 :ok :write [10 7] INFO [2019-12-17 12:31:24,732] jepsen worker 31 - jepsen.util 4531 :invoke :write [10 29] INFO [2019-12-17 12:31:24,732] jepsen worker 72 - jepsen.util 4752 :ok :write [24 4] INFO [2019-12-17 12:31:24,732] jepsen worker 72 - jepsen.util 4752 :invoke :write [24 31] INFO [2019-12-17 12:31:24,781] jepsen worker 32 - jepsen.util 4892 :ok :read [10 7] INFO [2019-12-17 12:31:24,783] jepsen worker 32 - jepsen.util 4892 :invoke :cas [10 [24 12]] INFO [2019-12-17 12:31:24,795] jepsen worker 29 - jepsen.util 3809 :fail :cas [9 [3 30]] :ValueNotSwapFrom INFO [2019-12-17 12:31:24,795] jepsen worker 29 - jepsen.util 3809 :invoke :read [9 nil] INFO [2019-12-17 12:31:24,814] jepsen worker 64 - jepsen.util 3574 :ok :write [21 29] INFO [2019-12-17 12:31:24,814] jepsen worker 64 - jepsen.util 3574 :invoke :cas [21 [45 10]] INFO [2019-12-17 12:31:24,825] jepsen worker 30 - jepsen.util 4800 :fail :read [10 nil] :RequestTimeoutException INFO [2019-12-17 12:31:24,825] jepsen worker 9 - jepsen.util 5409 :info :write [3 3] :RequestCanceledException INFO [2019-12-17 12:31:24,825] jepsen worker 63 - jepsen.util 3753 :info :cas [21 [6 20]] :RequestCanceledException INFO [2019-12-17 12:31:24,825] jepsen worker 30 - jepsen.util 4800 :invoke :read [10 nil] INFO [2019-12-17 12:31:24,826] jepsen worker 63 - jepsen.util 3843 :invoke :write [21 40] INFO [2019-12-17 12:31:24,826] jepsen worker 9 - jepsen.util 5499 :invoke :write [3 40] INFO [2019-12-17 12:31:24,849] jepsen worker 64 - jepsen.util 3574 :fail :cas [21 [45 10]] :ValueNotSwapFrom INFO [2019-12-17 12:31:24,849] jepsen worker 64 - jepsen.util 3574 :invoke :write [21 5] INFO [2019-12-17 12:31:24,856] jepsen worker 56 - jepsen.util 4736 :info :write [18 46] :RequestCanceledException INFO [2019-12-17 12:31:24,857] jepsen worker 56 - jepsen.util 4826 :invoke :read [18 nil] INFO [2019-12-17 12:31:24,875] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26965,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:31:24,875] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26965,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:31:24,875] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26969,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:31:24,875] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26969,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:31:24,888] jepsen worker 30 - jepsen.util 4800 :ok :read [10 7] INFO [2019-12-17 12:31:24,888] jepsen worker 30 - jepsen.util 4800 :invoke :cas [10 [23 42]] INFO [2019-12-17 12:31:24,892] jepsen worker 3 - jepsen.util 4773 :info :write [1 3] :RequestCanceledException INFO [2019-12-17 12:31:24,895] jepsen worker 3 - jepsen.util 4863 :invoke :cas [1 [46 35]] INFO [2019-12-17 12:31:24,904] jepsen worker 30 - jepsen.util 4800 :fail :cas [10 [23 42]] :ValueNotSwapFrom INFO [2019-12-17 12:31:24,904] jepsen worker 30 - jepsen.util 4800 :invoke :write [10 5] INFO [2019-12-17 12:31:24,924] jepsen worker 53 - jepsen.util 4643 :fail :cas [17 [15 35]] :ValueNotSwapFrom INFO [2019-12-17 12:31:24,924] jepsen worker 53 - jepsen.util 4643 :invoke :read [17 nil] INFO [2019-12-17 12:31:24,925] jepsen worker 65 - jepsen.util 3935 :info :write [21 36] :RequestCanceledException INFO [2019-12-17 12:31:24,926] jepsen worker 65 - jepsen.util 4025 :invoke :write [21 28] INFO [2019-12-17 12:31:24,944] jepsen worker 64 - jepsen.util 3574 :ok :write [21 5] INFO [2019-12-17 12:31:24,944] jepsen worker 64 - jepsen.util 3574 :invoke :read [21 nil] INFO [2019-12-17 12:31:24,961] jepsen worker 64 - jepsen.util 3574 :ok :read [21 5] INFO [2019-12-17 12:31:24,961] jepsen worker 64 - jepsen.util 3574 :invoke :write [21 8] INFO [2019-12-17 12:31:24,961] jepsen worker 11 - jepsen.util 4691 :ok :write [3 30] INFO [2019-12-17 12:31:24,962] jepsen worker 11 - jepsen.util 4691 :invoke :read [3 nil] INFO [2019-12-17 12:31:24,962] jepsen worker 20 - jepsen.util 4790 :ok :write [6 10] INFO [2019-12-17 12:31:24,962] jepsen worker 20 - jepsen.util 4790 :invoke :cas [6 [19 42]] INFO [2019-12-17 12:31:24,985] jepsen worker 73 - jepsen.util 4393 :fail :cas [24 [21 33]] :ValueNotSwapFrom INFO [2019-12-17 12:31:24,986] jepsen worker 73 - jepsen.util 4393 :invoke :cas [24 [18 46]] INFO [2019-12-17 12:31:24,992] jepsen worker 13 - jepsen.util 4243 :info :cas [4 [44 16]] :RequestCanceledException INFO [2019-12-17 12:31:24,993] jepsen worker 13 - jepsen.util 4333 :invoke :cas [4 [45 9]] INFO [2019-12-17 12:31:25,008] jepsen worker 18 - jepsen.util 4698 :fail :cas [6 [26 15]] :ValueNotSwapFrom INFO [2019-12-17 12:31:25,008] jepsen worker 18 - jepsen.util 4698 :invoke :cas [6 [12 3]] INFO [2019-12-17 12:31:25,015] jepsen worker 11 - jepsen.util 4691 :ok :read [3 30] INFO [2019-12-17 12:31:25,015] jepsen worker 11 - jepsen.util 4691 :invoke :write [3 25] INFO [2019-12-17 12:31:25,023] jepsen worker 20 - jepsen.util 4790 :fail :cas [6 [19 42]] :ValueNotSwapFrom INFO [2019-12-17 12:31:25,023] jepsen worker 20 - jepsen.util 4790 :invoke :cas [6 [42 2]] INFO [2019-12-17 12:31:25,024] jepsen worker 73 - jepsen.util 4393 :fail :cas [24 [18 46]] :ValueNotSwapFrom INFO [2019-12-17 12:31:25,025] jepsen worker 73 - jepsen.util 4393 :invoke :cas [24 [44 30]] INFO [2019-12-17 12:31:25,032] jepsen worker 87 - jepsen.util 4407 :fail :cas [29 [30 44]] :ValueNotSwapFrom INFO [2019-12-17 12:31:25,032] jepsen worker 87 - jepsen.util 4407 :invoke :write [29 4] INFO [2019-12-17 12:31:25,047] jepsen worker 56 - jepsen.util 4826 :ok :read [18 24] INFO [2019-12-17 12:31:25,047] jepsen worker 56 - jepsen.util 4826 :invoke :write [18 42] INFO [2019-12-17 12:31:25,074] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26982,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:31:25,075] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26982,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:31:25,075] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26984,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:31:25,075] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":26984,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:31:25,084] jepsen worker 18 - jepsen.util 4698 :fail :cas [6 [12 3]] :ValueNotSwapFrom INFO [2019-12-17 12:31:25,084] jepsen worker 18 - jepsen.util 4698 :invoke :read [6 nil] INFO [2019-12-17 12:31:25,090] jepsen worker 64 - jepsen.util 3574 :ok :write [21 8] INFO [2019-12-17 12:31:25,090] jepsen worker 64 - jepsen.util 3574 :invoke :read [21 nil] INFO [2019-12-17 12:31:25,093] jepsen worker 20 - jepsen.util 4790 :fail :cas [6 [42 2]] :ValueNotSwapFrom INFO [2019-12-17 12:31:25,093] jepsen worker 20 - jepsen.util 4790 :invoke :read [6 nil] INFO [2019-12-17 12:31:25,118] jepsen worker 64 - jepsen.util 3574 :ok :read [21 8] INFO [2019-12-17 12:31:25,119] jepsen worker 64 - jepsen.util 3574 :invoke :cas [21 [39 1]] INFO [2019-12-17 12:31:25,122] jepsen worker 20 - jepsen.util 4790 :ok :read [6 10] INFO [2019-12-17 12:31:25,122] jepsen worker 20 - jepsen.util 4790 :invoke :write [6 27] INFO [2019-12-17 12:31:25,162] jepsen worker 64 - jepsen.util 3574 :fail :cas [21 [39 1]] :ValueNotSwapFrom INFO [2019-12-17 12:31:25,162] jepsen worker 64 - jepsen.util 3574 :invoke :read [21 nil] INFO [2019-12-17 12:31:25,186] jepsen worker 64 - jepsen.util 3574 :ok :read [21 8] INFO [2019-12-17 12:31:25,186] jepsen worker 64 - jepsen.util 3574 :invoke :write [21 37] INFO [2019-12-17 12:31:25,227] jepsen worker 53 - jepsen.util 4643 :ok :read [17 16] INFO [2019-12-17 12:31:25,227] jepsen worker 53 - jepsen.util 4643 :invoke :write [17 35] INFO [2019-12-17 12:31:25,240] jepsen worker 46 - jepsen.util 4726 :info :cas [15 [13 14]] :RequestCanceledException INFO [2019-12-17 12:31:25,241] jepsen worker 46 - jepsen.util 4816 :invoke :write [15 37] INFO [2019-12-17 12:31:25,256] jepsen worker 27 - jepsen.util 4437 :info :write [9 34] :RequestCanceledException INFO [2019-12-17 12:31:25,258] jepsen worker 27 - jepsen.util 4527 :invoke :cas [9 [25 41]] INFO [2019-12-17 12:31:25,270] jepsen worker 31 - jepsen.util 4531 :ok :write [10 29] INFO [2019-12-17 12:31:25,270] jepsen worker 72 - jepsen.util 4752 :ok :write [24 31] INFO [2019-12-17 12:31:25,270] jepsen worker 31 - jepsen.util 4531 :invoke :write [10 49] INFO [2019-12-17 12:31:25,270] jepsen worker 72 - jepsen.util 4752 :invoke :cas [24 [8 0]] INFO [2019-12-17 12:31:25,274] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":27002,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:31:25,274] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":27008,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:31:25,274] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":27002,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:31:25,274] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":27008,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:31:25,274] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":27003,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:31:25,275] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":27011,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:31:25,275] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":27003,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:31:25,275] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":27011,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:31:25,318] jepsen worker 64 - jepsen.util 3574 :ok :write [21 37] INFO [2019-12-17 12:31:25,318] jepsen worker 64 - jepsen.util 3574 :invoke :read [21 nil] INFO [2019-12-17 12:31:25,330] jepsen worker 32 - jepsen.util 4892 :fail :cas [10 [24 12]] :ValueNotSwapFrom INFO [2019-12-17 12:31:25,330] jepsen worker 32 - jepsen.util 4892 :invoke :write [10 46] INFO [2019-12-17 12:31:25,345] jepsen worker 29 - jepsen.util 3809 :ok :read [9 11] INFO [2019-12-17 12:31:25,345] jepsen worker 29 - jepsen.util 3809 :invoke :read [9 nil] INFO [2019-12-17 12:31:25,349] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:31:25,350] jepsen nemesis - couchbase.util Rebalance check count: 202 INFO [2019-12-17 12:31:25,350] jepsen nemesis - couchbase.util Rebalance stuck count: 183 INFO [2019-12-17 12:31:25,356] jepsen worker 19 - jepsen.util 4699 :info :write [6 48] :RequestCanceledException INFO [2019-12-17 12:31:25,357] jepsen worker 19 - jepsen.util 4789 :invoke :read [6 nil] INFO [2019-12-17 12:31:25,378] jepsen worker 64 - jepsen.util 3574 :ok :read [21 37] INFO [2019-12-17 12:31:25,378] jepsen worker 64 - jepsen.util 3574 :invoke :read [21 nil] INFO [2019-12-17 12:31:25,395] jepsen worker 64 - jepsen.util 3574 :ok :read [21 37] INFO [2019-12-17 12:31:25,396] jepsen worker 64 - jepsen.util 3574 :invoke :write [21 33] INFO [2019-12-17 12:31:25,428] jepsen worker 3 - jepsen.util 4863 :fail :cas [1 [46 35]] :ValueNotSwapFrom INFO [2019-12-17 12:31:25,428] jepsen worker 3 - jepsen.util 4863 :invoke :read [1 nil] INFO [2019-12-17 12:31:25,439] jepsen worker 71 - jepsen.util 5021 :info :write [23 19] :RequestCanceledException INFO [2019-12-17 12:31:25,441] jepsen worker 71 - jepsen.util 5111 :invoke :read [23 nil] INFO [2019-12-17 12:31:25,461] jepsen worker 64 - jepsen.util 3574 :ok :write [21 33] INFO [2019-12-17 12:31:25,462] jepsen worker 64 - jepsen.util 3574 :invoke :write [21 23] INFO [2019-12-17 12:31:25,463] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":27027,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:31:25,463] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":27027,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:31:25,529] jepsen worker 64 - jepsen.util 3574 :ok :write [21 23] INFO [2019-12-17 12:31:25,529] jepsen worker 64 - jepsen.util 3574 :invoke :read [21 nil] INFO [2019-12-17 12:31:25,559] jepsen worker 64 - jepsen.util 3574 :ok :read [21 23] INFO [2019-12-17 12:31:25,559] jepsen worker 64 - jepsen.util 3574 :invoke :cas [21 [16 29]] INFO [2019-12-17 12:31:25,580] jepsen worker 64 - jepsen.util 3574 :fail :cas [21 [16 29]] :ValueNotSwapFrom INFO [2019-12-17 12:31:25,581] jepsen worker 64 - jepsen.util 3574 :invoke :read [21 nil] INFO [2019-12-17 12:31:25,592] jepsen worker 69 - jepsen.util 4839 :info :write [23 7] :RequestCanceledException INFO [2019-12-17 12:31:25,615] jepsen worker 37 - jepsen.util 4447 :info :write [12 26] :RequestCanceledException INFO [2019-12-17 12:31:25,616] jepsen worker 37 - jepsen.util 4537 :invoke :cas [12 [30 23]] INFO [2019-12-17 12:31:25,617] jepsen worker 69 - jepsen.util 4929 :invoke :read [23 nil] INFO [2019-12-17 12:31:25,619] jepsen worker 64 - jepsen.util 3574 :ok :read [21 23] INFO [2019-12-17 12:31:25,619] jepsen worker 64 - jepsen.util 3574 :invoke :cas [21 [17 45]] INFO [2019-12-17 12:31:25,633] jepsen worker 64 - jepsen.util 3574 :fail :cas [21 [17 45]] :ValueNotSwapFrom INFO [2019-12-17 12:31:25,633] jepsen worker 64 - jepsen.util 3574 :invoke :read [21 nil] INFO [2019-12-17 12:31:25,644] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":27041,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:31:25,645] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":27041,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:31:25,645] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":27044,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:31:25,645] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":27044,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:31:25,645] jepsen worker 64 - jepsen.util 3574 :ok :read [21 23] INFO [2019-12-17 12:31:25,645] jepsen worker 64 - jepsen.util 3574 :invoke :read [21 nil] INFO [2019-12-17 12:31:25,661] jepsen worker 64 - jepsen.util 3574 :ok :read [21 23] INFO [2019-12-17 12:31:25,661] jepsen worker 64 - jepsen.util 3574 :invoke :write [21 39] INFO [2019-12-17 12:31:25,664] jepsen worker 13 - jepsen.util 4333 :fail :cas [4 [45 9]] :ValueNotSwapFrom INFO [2019-12-17 12:31:25,664] jepsen worker 13 - jepsen.util 4333 :invoke :cas [4 [35 28]] INFO [2019-12-17 12:31:25,700] jepsen worker 64 - jepsen.util 3574 :ok :write [21 39] INFO [2019-12-17 12:31:25,700] jepsen worker 64 - jepsen.util 3574 :invoke :read [21 nil] INFO [2019-12-17 12:31:25,718] jepsen worker 64 - jepsen.util 3574 :ok :read [21 39] INFO [2019-12-17 12:31:25,718] jepsen worker 64 - jepsen.util 3574 :invoke :read [21 nil] INFO [2019-12-17 12:31:25,732] jepsen worker 11 - jepsen.util 4691 :ok :write [3 25] INFO [2019-12-17 12:31:25,732] jepsen worker 46 - jepsen.util 4816 :ok :write [15 37] INFO [2019-12-17 12:31:25,732] jepsen worker 11 - jepsen.util 4691 :invoke :write [3 27] INFO [2019-12-17 12:31:25,732] jepsen worker 46 - jepsen.util 4816 :invoke :cas [15 [31 13]] INFO [2019-12-17 12:31:25,735] jepsen worker 32 - jepsen.util 4892 :ok :write [10 46] INFO [2019-12-17 12:31:25,736] jepsen worker 32 - jepsen.util 4892 :invoke :cas [10 [49 28]] INFO [2019-12-17 12:31:25,755] jepsen worker 18 - jepsen.util 4698 :ok :read [6 27] INFO [2019-12-17 12:31:25,756] jepsen worker 18 - jepsen.util 4698 :invoke :cas [6 [47 21]] INFO [2019-12-17 12:31:25,762] jepsen worker 20 - jepsen.util 4790 :ok :write [6 27] INFO [2019-12-17 12:31:25,763] jepsen worker 20 - jepsen.util 4790 :invoke :read [6 nil] INFO [2019-12-17 12:31:25,775] jepsen worker 64 - jepsen.util 3574 :ok :read [21 39] INFO [2019-12-17 12:31:25,776] jepsen worker 64 - jepsen.util 3574 :invoke :write [21 31] INFO [2019-12-17 12:31:25,776] jepsen worker 19 - jepsen.util 4789 :ok :read [6 27] INFO [2019-12-17 12:31:25,777] jepsen worker 19 - jepsen.util 4789 :invoke :read [6 nil] INFO [2019-12-17 12:31:25,786] jepsen worker 46 - jepsen.util 4816 :fail :cas [15 [31 13]] :ValueNotSwapFrom INFO [2019-12-17 12:31:25,786] jepsen worker 46 - jepsen.util 4816 :invoke :write [15 36] INFO [2019-12-17 12:31:25,796] jepsen worker 3 - jepsen.util 4863 :ok :read [1 14] INFO [2019-12-17 12:31:25,796] jepsen worker 3 - jepsen.util 4863 :invoke :read [1 nil] INFO [2019-12-17 12:31:25,809] jepsen worker 14 - jepsen.util 4784 :info :write [4 29] :RequestCanceledException INFO [2019-12-17 12:31:25,811] jepsen worker 14 - jepsen.util 4874 :invoke :cas [4 [1 4]] INFO [2019-12-17 12:31:25,817] jepsen worker 32 - jepsen.util 4892 :fail :cas [10 [49 28]] :ValueNotSwapFrom INFO [2019-12-17 12:31:25,817] jepsen worker 32 - jepsen.util 4892 :invoke :read [10 nil] INFO [2019-12-17 12:31:25,818] jepsen worker 20 - jepsen.util 4790 :ok :read [6 27] INFO [2019-12-17 12:31:25,818] jepsen worker 20 - jepsen.util 4790 :invoke :cas [6 [26 4]] INFO [2019-12-17 12:31:25,825] jepsen worker 54 - jepsen.util 5094 :info :write [18 46] :RequestCanceledException INFO [2019-12-17 12:31:25,827] jepsen worker 54 - jepsen.util 5184 :invoke :cas [18 [43 37]] INFO [2019-12-17 12:31:25,847] jepsen worker 32 - jepsen.util 4892 :ok :read [10 46] INFO [2019-12-17 12:31:25,847] jepsen worker 32 - jepsen.util 4892 :invoke :cas [10 [43 8]] INFO [2019-12-17 12:31:25,862] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":27066,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:31:25,862] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":27066,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:31:25,862] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":27070,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:31:25,863] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":27070,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:31:25,864] jepsen worker 20 - jepsen.util 4790 :fail :cas [6 [26 4]] :ValueNotSwapFrom INFO [2019-12-17 12:31:25,864] jepsen worker 20 - jepsen.util 4790 :invoke :cas [6 [34 24]] INFO [2019-12-17 12:31:25,879] jepsen worker 32 - jepsen.util 4892 :fail :cas [10 [43 8]] :ValueNotSwapFrom INFO [2019-12-17 12:31:25,879] jepsen worker 32 - jepsen.util 4892 :invoke :cas [10 [29 31]] INFO [2019-12-17 12:31:25,900] jepsen worker 71 - jepsen.util 5111 :ok :read [23 32] INFO [2019-12-17 12:31:25,900] jepsen worker 71 - jepsen.util 5111 :invoke :read [23 nil] INFO [2019-12-17 12:31:25,904] jepsen worker 69 - jepsen.util 4929 :ok :read [23 32] INFO [2019-12-17 12:31:25,904] jepsen worker 69 - jepsen.util 4929 :invoke :cas [23 [20 22]] INFO [2019-12-17 12:31:25,908] jepsen worker 64 - jepsen.util 3574 :ok :write [21 31] INFO [2019-12-17 12:31:25,908] jepsen worker 64 - jepsen.util 3574 :invoke :cas [21 [15 12]] INFO [2019-12-17 12:31:25,910] jepsen worker 38 - jepsen.util 4538 :info :write [12 9] :RequestCanceledException INFO [2019-12-17 12:31:25,911] jepsen worker 38 - jepsen.util 4628 :invoke :read [12 nil] INFO [2019-12-17 12:31:25,918] jepsen worker 20 - jepsen.util 4790 :fail :cas [6 [34 24]] :ValueNotSwapFrom INFO [2019-12-17 12:31:25,918] jepsen worker 20 - jepsen.util 4790 :invoke :read [6 nil] INFO [2019-12-17 12:31:25,921] jepsen worker 13 - jepsen.util 4333 :fail :cas [4 [35 28]] :ValueNotSwapFrom INFO [2019-12-17 12:31:25,921] jepsen worker 13 - jepsen.util 4333 :invoke :read [4 nil] INFO [2019-12-17 12:31:25,929] jepsen worker 32 - jepsen.util 4892 :fail :cas [10 [29 31]] :ValueNotSwapFrom INFO [2019-12-17 12:31:25,929] jepsen worker 32 - jepsen.util 4892 :invoke :cas [10 [30 6]] INFO [2019-12-17 12:31:25,939] jepsen worker 51 - jepsen.util 4461 :info :write [17 28] :RequestCanceledException INFO [2019-12-17 12:31:25,939] jepsen worker 51 - jepsen.util 4551 :invoke :write [17 24] INFO [2019-12-17 12:31:25,951] jepsen worker 64 - jepsen.util 3574 :fail :cas [21 [15 12]] :ValueNotSwapFrom INFO [2019-12-17 12:31:25,951] jepsen worker 64 - jepsen.util 3574 :invoke :write [21 48] INFO [2019-12-17 12:31:26,025] jepsen worker 52 - jepsen.util 5002 :info :write [17 18] :RequestCanceledException INFO [2019-12-17 12:31:26,025] jepsen worker 79 - jepsen.util 4489 :info :write [26 48] :RequestCanceledException INFO [2019-12-17 12:31:26,025] jepsen worker 12 - jepsen.util 4872 :info :write [4 9] :RequestCanceledException INFO [2019-12-17 12:31:26,026] jepsen worker 52 - jepsen.util 5092 :invoke :cas [17 [4 20]] INFO [2019-12-17 12:31:26,026] jepsen worker 79 - jepsen.util 4579 :invoke :write [26 48] INFO [2019-12-17 12:31:26,026] jepsen worker 12 - jepsen.util 4962 :invoke :cas [4 [9 44]] INFO [2019-12-17 12:31:26,034] jepsen worker 20 - jepsen.util 4790 :ok :read [6 27] INFO [2019-12-17 12:31:26,034] jepsen worker 20 - jepsen.util 4790 :invoke :read [6 nil] INFO [2019-12-17 12:31:26,037] jepsen worker 76 - jepsen.util 6376 :info :write [25 35] :RequestCanceledException INFO [2019-12-17 12:31:26,038] jepsen worker 76 - jepsen.util 6466 :invoke :read [25 nil] INFO [2019-12-17 12:31:26,046] jepsen worker 32 - jepsen.util 4892 :fail :cas [10 [30 6]] :ValueNotSwapFrom INFO [2019-12-17 12:31:26,046] jepsen worker 32 - jepsen.util 4892 :invoke :cas [10 [36 1]] INFO [2019-12-17 12:31:26,048] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":27082,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:31:26,048] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":27082,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:31:26,048] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":27083,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:31:26,048] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":27083,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:31:26,049] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":27084,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:31:26,049] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":27084,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:31:26,056] jepsen worker 68 - jepsen.util 5738 :info :write [22 36] :RequestCanceledException INFO [2019-12-17 12:31:26,057] jepsen worker 68 - jepsen.util 5828 :invoke :write [22 6] INFO [2019-12-17 12:31:26,092] jepsen worker 84 - jepsen.util 5934 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:31:26,092] jepsen worker 84 - jepsen.util 5934 :invoke :write [28 17] INFO [2019-12-17 12:31:26,095] jepsen worker 64 - jepsen.util 3574 :ok :write [21 48] INFO [2019-12-17 12:31:26,095] jepsen worker 64 - jepsen.util 3574 :invoke :read [21 nil] INFO [2019-12-17 12:31:26,099] jepsen worker 32 - jepsen.util 4892 :fail :cas [10 [36 1]] :ValueNotSwapFrom INFO [2019-12-17 12:31:26,099] jepsen worker 32 - jepsen.util 4892 :invoke :cas [10 [3 16]] INFO [2019-12-17 12:31:26,100] jepsen worker 20 - jepsen.util 4790 :ok :read [6 27] INFO [2019-12-17 12:31:26,100] jepsen worker 20 - jepsen.util 4790 :invoke :read [6 nil] INFO [2019-12-17 12:31:26,104] jepsen worker 37 - jepsen.util 4537 :fail :cas [12 [30 23]] :ValueNotSwapFrom INFO [2019-12-17 12:31:26,104] jepsen worker 37 - jepsen.util 4537 :invoke :write [12 47] INFO [2019-12-17 12:31:26,110] jepsen worker 43 - jepsen.util 6073 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:31:26,110] jepsen worker 26 - jepsen.util 6506 :info :cas [8 [6 41]] :RequestCanceledException INFO [2019-12-17 12:31:26,110] jepsen worker 81 - jepsen.util 6111 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:31:26,110] jepsen worker 43 - jepsen.util 6073 :invoke :read [14 nil] INFO [2019-12-17 12:31:26,110] jepsen worker 81 - jepsen.util 6111 :invoke :cas [27 [46 24]] INFO [2019-12-17 12:31:26,111] jepsen worker 26 - jepsen.util 6596 :invoke :cas [8 [1 11]] INFO [2019-12-17 12:31:26,125] jepsen worker 58 - jepsen.util 6358 :info :cas [19 [12 26]] :RequestCanceledException INFO [2019-12-17 12:31:26,125] jepsen worker 88 - jepsen.util 4768 :info :write [29 44] :RequestCanceledException INFO [2019-12-17 12:31:26,125] jepsen worker 33 - jepsen.util 6153 :info :write [11 9] :RequestCanceledException INFO [2019-12-17 12:31:26,125] jepsen worker 49 - jepsen.util 5629 :info :write [16 13] :RequestCanceledException INFO [2019-12-17 12:31:26,125] jepsen worker 57 - jepsen.util 5997 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:31:26,125] jepsen worker 8 - jepsen.util 5588 :info :write [2 46] :RequestCanceledException INFO [2019-12-17 12:31:26,125] jepsen worker 16 - jepsen.util 7216 :info :write [5 23] :RequestCanceledException INFO [2019-12-17 12:31:26,126] jepsen worker 57 - jepsen.util 5997 :invoke :write [19 11] INFO [2019-12-17 12:31:26,126] jepsen worker 58 - jepsen.util 6448 :invoke :write [19 20] INFO [2019-12-17 12:31:26,126] jepsen worker 88 - jepsen.util 4858 :invoke :read [29 nil] INFO [2019-12-17 12:31:26,126] jepsen worker 49 - jepsen.util 5719 :invoke :read [16 nil] INFO [2019-12-17 12:31:26,126] jepsen worker 8 - jepsen.util 5678 :invoke :read [2 nil] INFO [2019-12-17 12:31:26,127] jepsen worker 16 - jepsen.util 7306 :invoke :cas [5 [13 21]] INFO [2019-12-17 12:31:26,127] jepsen worker 33 - jepsen.util 6243 :invoke :read [11 nil] INFO [2019-12-17 12:31:26,139] jepsen worker 83 - jepsen.util 6743 :info :cas [27 [43 18]] :RequestCanceledException INFO [2019-12-17 12:31:26,139] jepsen worker 82 - jepsen.util 7012 :info :write [27 16] :RequestCanceledException INFO [2019-12-17 12:31:26,139] jepsen worker 25 - jepsen.util 6415 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:31:26,140] jepsen worker 25 - jepsen.util 6415 :invoke :write [8 3] INFO [2019-12-17 12:31:26,140] jepsen worker 83 - jepsen.util 6833 :invoke :read [27 nil] INFO [2019-12-17 12:31:26,140] jepsen worker 82 - jepsen.util 7102 :invoke :cas [27 [41 14]] INFO [2019-12-17 12:31:26,144] jepsen worker 64 - jepsen.util 3574 :ok :read [21 48] INFO [2019-12-17 12:31:26,144] jepsen worker 64 - jepsen.util 3574 :invoke :write [21 45] INFO [2019-12-17 12:31:26,155] jepsen worker 88 - jepsen.util 4858 :ok :read [29 49] INFO [2019-12-17 12:31:26,155] jepsen worker 88 - jepsen.util 4858 :invoke :cas [29 [4 41]] INFO [2019-12-17 12:31:26,157] jepsen worker 6 - jepsen.util 6666 :info :cas [2 [46 7]] :RequestCanceledException INFO [2019-12-17 12:31:26,157] jepsen worker 34 - jepsen.util 5794 :info :cas [11 [46 30]] :RequestCanceledException INFO [2019-12-17 12:31:26,157] jepsen worker 48 - jepsen.util 5628 :info :write [16 25] :RequestCanceledException INFO [2019-12-17 12:31:26,157] jepsen worker 62 - jepsen.util 5822 :info :write [20 21] :RequestCanceledException INFO [2019-12-17 12:31:26,158] jepsen worker 34 - jepsen.util 5884 :invoke :read [11 nil] INFO [2019-12-17 12:31:26,158] jepsen worker 6 - jepsen.util 6756 :invoke :write [2 30] INFO [2019-12-17 12:31:26,158] jepsen worker 48 - jepsen.util 5718 :invoke :write [16 15] INFO [2019-12-17 12:31:26,158] jepsen worker 62 - jepsen.util 5912 :invoke :read [20 nil] INFO [2019-12-17 12:31:26,171] jepsen worker 88 - jepsen.util 4858 :fail :cas [29 [4 41]] :ValueNotSwapFrom INFO [2019-12-17 12:31:26,171] jepsen worker 88 - jepsen.util 4858 :invoke :read [29 nil] INFO [2019-12-17 12:31:26,184] jepsen worker 32 - jepsen.util 4892 :fail :cas [10 [3 16]] :ValueNotSwapFrom INFO [2019-12-17 12:31:26,184] jepsen worker 32 - jepsen.util 4892 :invoke :write [10 22] INFO [2019-12-17 12:31:26,191] jepsen worker 88 - jepsen.util 4858 :ok :read [29 49] INFO [2019-12-17 12:31:26,191] jepsen worker 88 - jepsen.util 4858 :invoke :read [29 nil] INFO [2019-12-17 12:31:26,191] jepsen worker 45 - jepsen.util 4905 :fail :read [15 nil] :RequestTimeoutException INFO [2019-12-17 12:31:26,191] jepsen worker 39 - jepsen.util 5889 :info :cas [13 [17 47]] :RequestCanceledException INFO [2019-12-17 12:31:26,191] jepsen worker 66 - jepsen.util 5826 :info :write [22 0] :RequestCanceledException INFO [2019-12-17 12:31:26,191] jepsen worker 45 - jepsen.util 4905 :invoke :read [15 nil] INFO [2019-12-17 12:31:26,192] jepsen worker 66 - jepsen.util 5916 :invoke :read [22 nil] INFO [2019-12-17 12:31:26,192] jepsen worker 39 - jepsen.util 5979 :invoke :write [13 26] INFO [2019-12-17 12:31:26,198] jepsen worker 20 - jepsen.util 4790 :ok :read [6 27] INFO [2019-12-17 12:31:26,198] jepsen worker 20 - jepsen.util 4790 :invoke :write [6 44] INFO [2019-12-17 12:31:26,210] jepsen worker 42 - jepsen.util 6882 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:31:26,210] jepsen worker 60 - jepsen.util 6720 :info :cas [20 [36 10]] :RequestCanceledException INFO [2019-12-17 12:31:26,210] jepsen worker 17 - jepsen.util 6137 :info :write [5 20] :RequestCanceledException INFO [2019-12-17 12:31:26,210] jepsen worker 15 - jepsen.util 5325 :info :cas [5 [35 28]] :RequestCanceledException INFO [2019-12-17 12:31:26,210] jepsen worker 42 - jepsen.util 6882 :invoke :read [14 nil] INFO [2019-12-17 12:31:26,211] jepsen worker 60 - jepsen.util 6810 :invoke :write [20 16] INFO [2019-12-17 12:31:26,211] jepsen worker 17 - jepsen.util 6227 :invoke :cas [5 [6 2]] INFO [2019-12-17 12:31:26,211] jepsen worker 15 - jepsen.util 5415 :invoke :read [5 nil] INFO [2019-12-17 12:31:26,211] jepsen worker 14 - jepsen.util 4874 :fail :cas [4 [1 4]] :ValueNotSwapFrom INFO [2019-12-17 12:31:26,211] jepsen worker 14 - jepsen.util 4874 :invoke :cas [4 [49 22]] INFO [2019-12-17 12:31:26,225] jepsen worker 85 - jepsen.util 7015 :info :write [28 3] :RequestCanceledException INFO [2019-12-17 12:31:26,225] jepsen worker 77 - jepsen.util 6467 :info :write [25 26] :RequestCanceledException INFO [2019-12-17 12:31:26,225] jepsen worker 24 - jepsen.util 5964 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:31:26,225] jepsen worker 78 - jepsen.util 4668 :info :write [26 8] :RequestCanceledException INFO [2019-12-17 12:31:26,225] jepsen worker 2 - jepsen.util 5852 :info :cas [0 [28 19]] :RequestCanceledException INFO [2019-12-17 12:31:26,225] jepsen worker 24 - jepsen.util 5964 :invoke :read [8 nil] INFO [2019-12-17 12:31:26,226] jepsen worker 78 - jepsen.util 4758 :invoke :cas [26 [39 48]] INFO [2019-12-17 12:31:26,226] jepsen worker 85 - jepsen.util 7105 :invoke :cas [28 [3 45]] INFO [2019-12-17 12:31:26,226] jepsen worker 77 - jepsen.util 6557 :invoke :cas [25 [28 23]] INFO [2019-12-17 12:31:26,226] jepsen worker 2 - jepsen.util 5942 :invoke :read [0 nil] INFO [2019-12-17 12:31:26,228] jepsen worker 12 - jepsen.util 4962 :fail :cas [4 [9 44]] :ValueNotSwapFrom INFO [2019-12-17 12:31:26,228] jepsen worker 12 - jepsen.util 4962 :invoke :cas [4 [29 22]] INFO [2019-12-17 12:31:26,239] jepsen worker 22 - jepsen.util 6322 :info :write [7 6] :RequestCanceledException INFO [2019-12-17 12:31:26,239] jepsen worker 50 - jepsen.util 6530 :info :write [16 36] :RequestCanceledException INFO [2019-12-17 12:31:26,241] jepsen worker 50 - jepsen.util 6620 :invoke :cas [16 [22 41]] INFO [2019-12-17 12:31:26,241] jepsen worker 22 - jepsen.util 6412 :invoke :write [7 5] INFO [2019-12-17 12:31:26,242] jepsen worker 88 - jepsen.util 4858 :ok :read [29 49] INFO [2019-12-17 12:31:26,242] jepsen worker 88 - jepsen.util 4858 :invoke :read [29 nil] INFO [2019-12-17 12:31:26,254] jepsen worker 64 - jepsen.util 3574 :ok :write [21 45] INFO [2019-12-17 12:31:26,254] jepsen worker 64 - jepsen.util 3574 :invoke :read [21 nil] INFO [2019-12-17 12:31:26,257] jepsen worker 80 - jepsen.util 4580 :info :write [26 42] :RequestCanceledException INFO [2019-12-17 12:31:26,257] jepsen worker 1 - jepsen.util 6031 :info :write [0 38] :RequestCanceledException INFO [2019-12-17 12:31:26,257] jepsen worker 7 - jepsen.util 6307 :info :write [2 11] :RequestCanceledException INFO [2019-12-17 12:31:26,257] jepsen worker 54 - jepsen.util 5184 :fail :cas [18 [43 37]] :ValueNotSwapFrom INFO [2019-12-17 12:31:26,257] jepsen worker 54 - jepsen.util 5184 :invoke :write [18 49] INFO [2019-12-17 12:31:26,258] jepsen worker 80 - jepsen.util 4670 :invoke :write [26 10] INFO [2019-12-17 12:31:26,258] jepsen worker 7 - jepsen.util 6397 :invoke :cas [2 [36 44]] INFO [2019-12-17 12:31:26,258] jepsen worker 1 - jepsen.util 6121 :invoke :read [0 nil] INFO [2019-12-17 12:31:26,260] jepsen worker 12 - jepsen.util 4962 :fail :cas [4 [29 22]] :ValueNotSwapFrom INFO [2019-12-17 12:31:26,261] jepsen worker 12 - jepsen.util 4962 :invoke :read [4 nil] INFO [2019-12-17 12:31:26,267] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":27123,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:31:26,267] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":27136,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:31:26,267] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":27123,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:31:26,267] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":27136,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:31:26,267] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":27124,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:31:26,267] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":27141,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:31:26,267] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":27124,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:31:26,267] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":27141,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:31:26,267] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":27125,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:31:26,267] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":27125,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:31:26,292] jepsen worker 44 - jepsen.util 6884 :info :write [14 21] :RequestCanceledException INFO [2019-12-17 12:31:26,292] jepsen worker 75 - jepsen.util 6195 :info :cas [25 [16 27]] :RequestCanceledException INFO [2019-12-17 12:31:26,292] jepsen worker 40 - jepsen.util 6970 :info :cas [13 [2 27]] :RequestCanceledException INFO [2019-12-17 12:31:26,292] jepsen worker 64 - jepsen.util 3574 :ok :read [21 45] INFO [2019-12-17 12:31:26,292] jepsen worker 64 - jepsen.util 3574 :invoke :read [21 nil] INFO [2019-12-17 12:31:26,293] jepsen worker 40 - jepsen.util 7060 :invoke :cas [13 [29 44]] INFO [2019-12-17 12:31:26,293] jepsen worker 75 - jepsen.util 6285 :invoke :write [25 36] INFO [2019-12-17 12:31:26,293] jepsen worker 44 - jepsen.util 6974 :invoke :read [14 nil] INFO [2019-12-17 12:31:26,306] jepsen worker 64 - jepsen.util 3574 :ok :read [21 45] INFO [2019-12-17 12:31:26,306] jepsen worker 64 - jepsen.util 3574 :invoke :write [21 9] INFO [2019-12-17 12:31:26,309] jepsen worker 0 - jepsen.util 6840 :info :write [0 48] :RequestCanceledException INFO [2019-12-17 12:31:26,309] jepsen worker 35 - jepsen.util 6425 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:31:26,309] jepsen worker 35 - jepsen.util 6425 :invoke :cas [11 [0 37]] INFO [2019-12-17 12:31:26,310] jepsen worker 0 - jepsen.util 6930 :invoke :write [0 46] INFO [2019-12-17 12:31:26,325] jepsen worker 41 - jepsen.util 6701 :info :write [13 37] :RequestCanceledException INFO [2019-12-17 12:31:26,325] jepsen worker 86 - jepsen.util 6566 :info :write [28 39] :RequestCanceledException INFO [2019-12-17 12:31:26,327] jepsen worker 86 - jepsen.util 6656 :invoke :write [28 10] INFO [2019-12-17 12:31:26,327] jepsen worker 41 - jepsen.util 6791 :invoke :cas [13 [26 32]] INFO [2019-12-17 12:31:26,339] jepsen worker 67 - jepsen.util 6457 :info :write [22 6] :RequestCanceledException INFO [2019-12-17 12:31:26,340] jepsen worker 67 - jepsen.util 6547 :invoke :write [22 29] INFO [2019-12-17 12:31:26,346] jepsen worker 64 - jepsen.util 3574 :ok :write [21 9] INFO [2019-12-17 12:31:26,347] jepsen worker 64 - jepsen.util 3574 :invoke :read [21 nil] INFO [2019-12-17 12:31:26,357] jepsen worker 70 - jepsen.util 5290 :info :write [23 11] :RequestCanceledException INFO [2019-12-17 12:31:26,358] jepsen worker 70 - jepsen.util 5380 :invoke :write [23 4] INFO [2019-12-17 12:31:26,361] jepsen worker 52 - jepsen.util 5092 :fail :cas [17 [4 20]] :ValueNotSwapFrom INFO [2019-12-17 12:31:26,361] jepsen worker 52 - jepsen.util 5092 :invoke :read [17 nil] INFO [2019-12-17 12:31:26,362] jepsen worker 64 - jepsen.util 3574 :ok :read [21 9] INFO [2019-12-17 12:31:26,362] jepsen worker 64 - jepsen.util 3574 :invoke :read [21 nil] INFO [2019-12-17 12:31:26,365] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:31:26,365] jepsen nemesis - couchbase.util Rebalance check count: 203 INFO [2019-12-17 12:31:26,365] jepsen nemesis - couchbase.util Rebalance stuck count: 184 INFO [2019-12-17 12:31:26,376] jepsen worker 38 - jepsen.util 4628 :ok :read [12 38] INFO [2019-12-17 12:31:26,376] jepsen worker 38 - jepsen.util 4628 :invoke :cas [12 [29 48]] INFO [2019-12-17 12:31:26,380] jepsen worker 64 - jepsen.util 3574 :ok :read [21 9] INFO [2019-12-17 12:31:26,380] jepsen worker 64 - jepsen.util 3574 :invoke :cas [21 [34 20]] INFO [2019-12-17 12:31:26,390] jepsen worker 64 - jepsen.util 3574 :fail :cas [21 [34 20]] :ValueNotSwapFrom INFO [2019-12-17 12:31:26,390] jepsen worker 64 - jepsen.util 3574 :invoke :read [21 nil] INFO [2019-12-17 12:31:26,391] jepsen worker 5 - jepsen.util 4145 :info :write [1 2] :RequestCanceledException INFO [2019-12-17 12:31:26,392] jepsen worker 5 - jepsen.util 4235 :invoke :read [1 nil] INFO [2019-12-17 12:31:26,403] jepsen worker 64 - jepsen.util 3574 :ok :read [21 9] INFO [2019-12-17 12:31:26,403] jepsen worker 64 - jepsen.util 3574 :invoke :cas [21 [1 34]] INFO [2019-12-17 12:31:26,410] jepsen worker 28 - jepsen.util 3448 :info :cas [9 [3 7]] :RequestCanceledException INFO [2019-12-17 12:31:26,410] jepsen worker 74 - jepsen.util 5024 :info :write [24 27] :RequestCanceledException INFO [2019-12-17 12:31:26,411] jepsen worker 28 - jepsen.util 3538 :invoke :read [9 nil] INFO [2019-12-17 12:31:26,411] jepsen worker 74 - jepsen.util 5114 :invoke :write [24 29] INFO [2019-12-17 12:31:26,414] jepsen worker 64 - jepsen.util 3574 :fail :cas [21 [1 34]] :ValueNotSwapFrom INFO [2019-12-17 12:31:26,414] jepsen worker 64 - jepsen.util 3574 :invoke :cas [21 [33 41]] INFO [2019-12-17 12:31:26,431] jepsen worker 64 - jepsen.util 3574 :fail :cas [21 [33 41]] :ValueNotSwapFrom INFO [2019-12-17 12:31:26,432] jepsen worker 64 - jepsen.util 3574 :invoke :cas [21 [11 44]] INFO [2019-12-17 12:31:26,440] jepsen worker 47 - jepsen.util 5177 :info :cas [15 [27 43]] :RequestCanceledException INFO [2019-12-17 12:31:26,441] jepsen worker 47 - jepsen.util 5267 :invoke :write [15 12] INFO [2019-12-17 12:31:26,445] jepsen worker 64 - jepsen.util 3574 :fail :cas [21 [11 44]] :ValueNotSwapFrom INFO [2019-12-17 12:31:26,445] jepsen worker 64 - jepsen.util 3574 :invoke :write [21 3] INFO [2019-12-17 12:31:26,446] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":27182,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:31:26,446] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":27182,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:31:26,446] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":27185,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:31:26,446] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":27185,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:31:26,446] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":27186,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:31:26,446] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":27186,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:31:26,446] jepsen worker 12 - jepsen.util 4962 :ok :read [4 49] INFO [2019-12-17 12:31:26,446] jepsen worker 12 - jepsen.util 4962 :invoke :read [4 nil] INFO [2019-12-17 12:31:26,514] jepsen worker 5 - jepsen.util 4235 :ok :read [1 14] INFO [2019-12-17 12:31:26,514] jepsen worker 5 - jepsen.util 4235 :invoke :write [1 15] INFO [2019-12-17 12:31:26,539] jepsen worker 55 - jepsen.util 5545 :info :write [18 27] :RequestCanceledException INFO [2019-12-17 12:31:26,540] jepsen worker 12 - jepsen.util 4962 :ok :read [4 49] INFO [2019-12-17 12:31:26,540] jepsen worker 12 - jepsen.util 4962 :invoke :write [4 48] INFO [2019-12-17 12:31:26,540] jepsen worker 55 - jepsen.util 5635 :invoke :cas [18 [30 45]] INFO [2019-12-17 12:31:26,572] jepsen worker 69 - jepsen.util 4929 :fail :cas [23 [20 22]] :ValueNotSwapFrom INFO [2019-12-17 12:31:26,572] jepsen worker 69 - jepsen.util 4929 :invoke :write [23 47] INFO [2019-12-17 12:31:26,573] jepsen worker 3 - jepsen.util 4863 :ok :read [1 14] INFO [2019-12-17 12:31:26,573] jepsen worker 3 - jepsen.util 4863 :invoke :cas [1 [10 30]] INFO [2019-12-17 12:31:26,602] jepsen worker 13 - jepsen.util 4333 :ok :read [4 49] INFO [2019-12-17 12:31:26,602] jepsen worker 13 - jepsen.util 4333 :invoke :write [4 34] INFO [2019-12-17 12:31:26,619] jepsen worker 71 - jepsen.util 5111 :ok :read [23 32] INFO [2019-12-17 12:31:26,620] jepsen worker 71 - jepsen.util 5111 :invoke :write [23 23] INFO [2019-12-17 12:31:26,625] jepsen worker 21 - jepsen.util 5871 :info :write [7 15] :RequestCanceledException INFO [2019-12-17 12:31:26,625] jepsen worker 23 - jepsen.util 6413 :info :cas [7 [27 39]] :RequestCanceledException INFO [2019-12-17 12:31:26,627] jepsen worker 21 - jepsen.util 5961 :invoke :read [7 nil] INFO [2019-12-17 12:31:26,627] jepsen worker 23 - jepsen.util 6503 :invoke :write [7 1] INFO [2019-12-17 12:31:26,656] jepsen worker 61 - jepsen.util 6991 :info :cas [20 [48 1]] :RequestCanceledException INFO [2019-12-17 12:31:26,657] jepsen worker 61 - jepsen.util 7081 :invoke :write [20 0] INFO [2019-12-17 12:31:26,692] jepsen worker 10 - jepsen.util 4600 :info :write [3 29] :RequestCanceledException INFO [2019-12-17 12:31:26,692] jepsen worker 10 - jepsen.util 4690 :invoke :read [3 nil] INFO [2019-12-17 12:31:26,699] jepsen worker 52 - jepsen.util 5092 :ok :read [17 16] INFO [2019-12-17 12:31:26,699] jepsen worker 52 - jepsen.util 5092 :invoke :write [17 23] INFO [2019-12-17 12:31:26,710] jepsen worker 4 - jepsen.util 3784 :info :write [1 29] :RequestCanceledException INFO [2019-12-17 12:31:26,710] jepsen worker 36 - jepsen.util 4716 :info :write [12 36] :RequestCanceledException INFO [2019-12-17 12:31:26,711] jepsen worker 36 - jepsen.util 4806 :invoke :cas [12 [12 17]] INFO [2019-12-17 12:31:26,711] jepsen worker 4 - jepsen.util 3874 :invoke :cas [1 [48 14]] INFO [2019-12-17 12:31:26,719] jepsen worker 88 - jepsen.util 4858 :ok :read [29 49] INFO [2019-12-17 12:31:26,719] jepsen worker 88 - jepsen.util 4858 :invoke :write [29 49] INFO [2019-12-17 12:31:26,726] jepsen worker 59 - jepsen.util 6269 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:31:26,726] jepsen worker 59 - jepsen.util 6269 :invoke :read [19 nil] INFO [2019-12-17 12:31:26,739] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":27201,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:31:26,739] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":27201,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:31:26,739] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":27205,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:31:26,739] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":27205,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:31:26,757] jepsen worker 89 - jepsen.util 4229 :info :write [29 36] :RequestCanceledException INFO [2019-12-17 12:31:26,758] jepsen worker 89 - jepsen.util 4319 :invoke :read [29 nil] INFO [2019-12-17 12:31:26,872] jepsen worker 78 - jepsen.util 4758 :fail :cas [26 [39 48]] :ValueNotSwapFrom INFO [2019-12-17 12:31:26,872] jepsen worker 78 - jepsen.util 4758 :invoke :cas [26 [0 49]] INFO [2019-12-17 12:31:26,898] jepsen worker 38 - jepsen.util 4628 :fail :cas [12 [29 48]] :ValueNotSwapFrom INFO [2019-12-17 12:31:26,898] jepsen worker 38 - jepsen.util 4628 :invoke :write [12 14] INFO [2019-12-17 12:31:26,910] jepsen worker 63 - jepsen.util 3843 :info :write [21 40] :RequestCanceledException INFO [2019-12-17 12:31:26,911] jepsen worker 63 - jepsen.util 3933 :invoke :write [21 17] INFO [2019-12-17 12:31:26,918] jepsen worker 30 - jepsen.util 4800 :info :write [10 5] :RequestCanceledException INFO [2019-12-17 12:31:26,918] jepsen worker 9 - jepsen.util 5499 :info :write [3 40] :RequestCanceledException INFO [2019-12-17 12:31:26,919] jepsen worker 30 - jepsen.util 4890 :invoke :write [10 1] INFO [2019-12-17 12:31:26,919] jepsen worker 9 - jepsen.util 5589 :invoke :read [3 nil] INFO [2019-12-17 12:31:26,943] jepsen worker 3 - jepsen.util 4863 :fail :cas [1 [10 30]] :ValueNotSwapFrom INFO [2019-12-17 12:31:26,943] jepsen worker 3 - jepsen.util 4863 :invoke :write [1 26] INFO [2019-12-17 12:31:26,958] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":27216,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:31:26,958] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":27216,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:31:26,958] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":27219,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:31:26,958] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":27219,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:31:27,011] jepsen worker 46 - jepsen.util 4816 :ok :write [15 36] INFO [2019-12-17 12:31:27,011] jepsen worker 46 - jepsen.util 4816 :invoke :read [15 nil] INFO [2019-12-17 12:31:27,022] jepsen worker 11 - jepsen.util 4691 :ok :write [3 27] INFO [2019-12-17 12:31:27,022] jepsen worker 11 - jepsen.util 4691 :invoke :write [3 1] INFO [2019-12-17 12:31:27,025] jepsen worker 65 - jepsen.util 4025 :info :write [21 28] :RequestCanceledException INFO [2019-12-17 12:31:27,026] jepsen worker 74 - jepsen.util 5114 :ok :write [24 29] INFO [2019-12-17 12:31:27,026] jepsen worker 74 - jepsen.util 5114 :invoke :write [24 48] INFO [2019-12-17 12:31:27,027] jepsen worker 65 - jepsen.util 4115 :invoke :cas [21 [31 49]] INFO [2019-12-17 12:31:27,027] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":27226,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:31:27,027] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":27226,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:31:27,027] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":27227,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:31:27,027] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":27227,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:31:27,027] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":27228,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:31:27,027] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":27228,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:31:27,030] jepsen worker 20 - jepsen.util 4790 :ok :write [6 44] INFO [2019-12-17 12:31:27,030] jepsen worker 20 - jepsen.util 4790 :invoke :write [6 8] INFO [2019-12-17 12:31:27,030] jepsen worker 32 - jepsen.util 4892 :ok :write [10 22] INFO [2019-12-17 12:31:27,030] jepsen worker 32 - jepsen.util 4892 :invoke :cas [10 [37 12]] INFO [2019-12-17 12:31:27,036] jepsen worker 46 - jepsen.util 4816 :ok :read [15 36] INFO [2019-12-17 12:31:27,036] jepsen worker 46 - jepsen.util 4816 :invoke :cas [15 [7 31]] INFO [2019-12-17 12:31:27,039] jepsen worker 73 - jepsen.util 4393 :info :cas [24 [44 30]] :RequestCanceledException INFO [2019-12-17 12:31:27,040] jepsen worker 73 - jepsen.util 4483 :invoke :cas [24 [26 2]] INFO [2019-12-17 12:31:27,087] jepsen worker 36 - jepsen.util 4806 :fail :cas [12 [12 17]] :ValueNotSwapFrom INFO [2019-12-17 12:31:27,087] jepsen worker 36 - jepsen.util 4806 :invoke :read [12 nil] INFO [2019-12-17 12:31:27,090] jepsen worker 55 - jepsen.util 5635 :fail :cas [18 [30 45]] :ValueNotSwapFrom INFO [2019-12-17 12:31:27,090] jepsen worker 55 - jepsen.util 5635 :invoke :read [18 nil] INFO [2019-12-17 12:31:27,092] jepsen worker 87 - jepsen.util 4407 :info :write [29 4] :RequestCanceledException INFO [2019-12-17 12:31:27,093] jepsen worker 87 - jepsen.util 4497 :invoke :write [29 13] INFO [2019-12-17 12:31:27,139] jepsen worker 56 - jepsen.util 4826 :info :write [18 42] :RequestCanceledException INFO [2019-12-17 12:31:27,140] jepsen worker 56 - jepsen.util 4916 :invoke :read [18 nil] INFO [2019-12-17 12:31:27,189] jepsen worker 4 - jepsen.util 3874 :fail :cas [1 [48 14]] :ValueNotSwapFrom INFO [2019-12-17 12:31:27,189] jepsen worker 4 - jepsen.util 3874 :invoke :read [1 nil] INFO [2019-12-17 12:31:27,205] jepsen worker 89 - jepsen.util 4319 :ok :read [29 49] INFO [2019-12-17 12:31:27,205] jepsen worker 89 - jepsen.util 4319 :invoke :write [29 33] INFO [2019-12-17 12:31:27,206] jepsen worker 65 - jepsen.util 4115 :fail :cas [21 [31 49]] :ValueNotSwapFrom INFO [2019-12-17 12:31:27,206] jepsen worker 65 - jepsen.util 4115 :invoke :cas [21 [40 22]] INFO [2019-12-17 12:31:27,207] jepsen worker 46 - jepsen.util 4816 :fail :cas [15 [7 31]] :ValueNotSwapFrom INFO [2019-12-17 12:31:27,207] jepsen worker 46 - jepsen.util 4816 :invoke :write [15 29] INFO [2019-12-17 12:31:27,230] jepsen worker 10 - jepsen.util 4690 :ok :read [3 27] INFO [2019-12-17 12:31:27,230] jepsen worker 10 - jepsen.util 4690 :invoke :cas [3 [6 7]] INFO [2019-12-17 12:31:27,236] jepsen worker 55 - jepsen.util 5635 :ok :read [18 24] INFO [2019-12-17 12:31:27,236] jepsen worker 55 - jepsen.util 5635 :invoke :write [18 32] INFO [2019-12-17 12:31:27,246] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":27258,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:31:27,246] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":27258,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:31:27,246] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":27246,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:31:27,246] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":27246,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:31:27,246] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":27247,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:31:27,246] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":27247,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:31:27,268] jepsen worker 18 - jepsen.util 4698 :fail :cas [6 [47 21]] :ValueNotSwapFrom INFO [2019-12-17 12:31:27,269] jepsen worker 18 - jepsen.util 4698 :invoke :cas [6 [11 26]] INFO [2019-12-17 12:31:27,297] jepsen worker 28 - jepsen.util 3538 :ok :read [9 11] INFO [2019-12-17 12:31:27,297] jepsen worker 28 - jepsen.util 3538 :invoke :read [9 nil] INFO [2019-12-17 12:31:27,307] jepsen worker 73 - jepsen.util 4483 :fail :cas [24 [26 2]] :ValueNotSwapFrom INFO [2019-12-17 12:31:27,307] jepsen worker 73 - jepsen.util 4483 :invoke :write [24 47] INFO [2019-12-17 12:31:27,309] jepsen worker 72 - jepsen.util 4752 :info :cas [24 [8 0]] :RequestCanceledException INFO [2019-12-17 12:31:27,310] jepsen worker 72 - jepsen.util 4842 :invoke :cas [24 [37 13]] INFO [2019-12-17 12:31:27,314] jepsen worker 9 - jepsen.util 5589 :ok :read [3 27] INFO [2019-12-17 12:31:27,314] jepsen worker 9 - jepsen.util 5589 :invoke :write [3 46] INFO [2019-12-17 12:31:27,318] jepsen worker 32 - jepsen.util 4892 :fail :cas [10 [37 12]] :ValueNotSwapFrom INFO [2019-12-17 12:31:27,318] jepsen worker 32 - jepsen.util 4892 :invoke :cas [10 [29 4]] INFO [2019-12-17 12:31:27,324] jepsen worker 31 - jepsen.util 4531 :info :write [10 49] :RequestCanceledException INFO [2019-12-17 12:31:27,324] jepsen worker 53 - jepsen.util 4643 :info :write [17 35] :RequestCanceledException INFO [2019-12-17 12:31:27,324] jepsen worker 27 - jepsen.util 4527 :info :cas [9 [25 41]] :RequestCanceledException INFO [2019-12-17 12:31:27,324] jepsen worker 5 - jepsen.util 4235 :ok :write [1 15] INFO [2019-12-17 12:31:27,325] jepsen worker 5 - jepsen.util 4235 :invoke :write [1 0] INFO [2019-12-17 12:31:27,325] jepsen worker 27 - jepsen.util 4617 :invoke :write [9 19] INFO [2019-12-17 12:31:27,325] jepsen worker 53 - jepsen.util 4733 :invoke :write [17 47] INFO [2019-12-17 12:31:27,326] jepsen worker 31 - jepsen.util 4621 :invoke :cas [10 [35 25]] INFO [2019-12-17 12:31:27,326] jepsen worker 37 - jepsen.util 4537 :ok :write [12 47] INFO [2019-12-17 12:31:27,326] jepsen worker 37 - jepsen.util 4537 :invoke :read [12 nil] INFO [2019-12-17 12:31:27,341] jepsen worker 65 - jepsen.util 4115 :fail :cas [21 [40 22]] :ValueNotSwapFrom INFO [2019-12-17 12:31:27,341] jepsen worker 65 - jepsen.util 4115 :invoke :read [21 nil] INFO [2019-12-17 12:31:27,361] jepsen worker 78 - jepsen.util 4758 :fail :cas [26 [0 49]] :ValueNotSwapFrom INFO [2019-12-17 12:31:27,361] jepsen worker 78 - jepsen.util 4758 :invoke :write [26 32] INFO [2019-12-17 12:31:27,394] jepsen worker 38 - jepsen.util 4628 :ok :write [12 14] INFO [2019-12-17 12:31:27,394] jepsen worker 38 - jepsen.util 4628 :invoke :cas [12 [43 31]] INFO [2019-12-17 12:31:27,415] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:31:27,416] jepsen nemesis - couchbase.util Rebalance check count: 204 INFO [2019-12-17 12:31:27,416] jepsen nemesis - couchbase.util Rebalance stuck count: 185 INFO [2019-12-17 12:31:27,425] jepsen worker 29 - jepsen.util 3809 :fail :read [9 nil] :RequestTimeoutException INFO [2019-12-17 12:31:27,426] jepsen worker 29 - jepsen.util 3809 :invoke :read [9 nil] INFO [2019-12-17 12:31:27,426] jepsen worker 32 - jepsen.util 4892 :fail :cas [10 [29 4]] :ValueNotSwapFrom INFO [2019-12-17 12:31:27,427] jepsen worker 32 - jepsen.util 4892 :invoke :read [10 nil] INFO [2019-12-17 12:31:27,433] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":27277,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:31:27,434] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":27277,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:31:27,434] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":27280,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:31:27,434] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":27280,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:31:27,437] jepsen worker 65 - jepsen.util 4115 :ok :read [21 9] INFO [2019-12-17 12:31:27,437] jepsen worker 65 - jepsen.util 4115 :invoke :read [21 nil] INFO [2019-12-17 12:31:27,463] jepsen worker 36 - jepsen.util 4806 :ok :read [12 14] INFO [2019-12-17 12:31:27,463] jepsen worker 36 - jepsen.util 4806 :invoke :write [12 48] INFO [2019-12-17 12:31:27,469] jepsen worker 29 - jepsen.util 3809 :ok :read [9 11] INFO [2019-12-17 12:31:27,469] jepsen worker 29 - jepsen.util 3809 :invoke :read [9 nil] INFO [2019-12-17 12:31:27,478] jepsen worker 65 - jepsen.util 4115 :ok :read [21 9] INFO [2019-12-17 12:31:27,478] jepsen worker 65 - jepsen.util 4115 :invoke :write [21 4] INFO [2019-12-17 12:31:27,480] jepsen worker 32 - jepsen.util 4892 :ok :read [10 22] INFO [2019-12-17 12:31:27,480] jepsen worker 32 - jepsen.util 4892 :invoke :read [10 nil] INFO [2019-12-17 12:31:27,498] jepsen worker 19 - jepsen.util 4789 :ok :read [6 44] INFO [2019-12-17 12:31:27,498] jepsen worker 19 - jepsen.util 4789 :invoke :cas [6 [3 41]] INFO [2019-12-17 12:31:27,505] jepsen worker 54 - jepsen.util 5184 :ok :write [18 49] INFO [2019-12-17 12:31:27,505] jepsen worker 54 - jepsen.util 5184 :invoke :write [18 32] INFO [2019-12-17 12:31:27,510] jepsen worker 29 - jepsen.util 3809 :ok :read [9 11] INFO [2019-12-17 12:31:27,511] jepsen worker 29 - jepsen.util 3809 :invoke :read [9 nil] INFO [2019-12-17 12:31:27,539] jepsen worker 4 - jepsen.util 3874 :ok :read [1 15] INFO [2019-12-17 12:31:27,539] jepsen worker 4 - jepsen.util 3874 :invoke :cas [1 [21 41]] INFO [2019-12-17 12:31:27,546] jepsen worker 32 - jepsen.util 4892 :ok :read [10 22] INFO [2019-12-17 12:31:27,546] jepsen worker 32 - jepsen.util 4892 :invoke :cas [10 [17 37]] INFO [2019-12-17 12:31:27,565] jepsen worker 63 - jepsen.util 3933 :ok :write [21 17] INFO [2019-12-17 12:31:27,565] jepsen worker 63 - jepsen.util 3933 :invoke :read [21 nil] INFO [2019-12-17 12:31:27,584] jepsen worker 29 - jepsen.util 3809 :ok :read [9 11] INFO [2019-12-17 12:31:27,584] jepsen worker 29 - jepsen.util 3809 :invoke :read [9 nil] INFO [2019-12-17 12:31:27,604] jepsen worker 56 - jepsen.util 4916 :ok :read [18 49] INFO [2019-12-17 12:31:27,604] jepsen worker 56 - jepsen.util 4916 :invoke :cas [18 [37 29]] INFO [2019-12-17 12:31:27,619] jepsen worker 32 - jepsen.util 4892 :fail :cas [10 [17 37]] :ValueNotSwapFrom INFO [2019-12-17 12:31:27,619] jepsen worker 32 - jepsen.util 4892 :invoke :write [10 39] INFO [2019-12-17 12:31:27,649] jepsen worker 29 - jepsen.util 3809 :ok :read [9 11] INFO [2019-12-17 12:31:27,649] jepsen worker 29 - jepsen.util 3809 :invoke :cas [9 [26 19]] INFO [2019-12-17 12:31:27,656] jepsen worker 38 - jepsen.util 4628 :fail :cas [12 [43 31]] :ValueNotSwapFrom INFO [2019-12-17 12:31:27,656] jepsen worker 38 - jepsen.util 4628 :invoke :cas [12 [48 38]] INFO [2019-12-17 12:31:27,683] jepsen worker 56 - jepsen.util 4916 :fail :cas [18 [37 29]] :ValueNotSwapFrom INFO [2019-12-17 12:31:27,683] jepsen worker 56 - jepsen.util 4916 :invoke :cas [18 [46 22]] INFO [2019-12-17 12:31:27,694] jepsen worker 65 - jepsen.util 4115 :ok :write [21 4] INFO [2019-12-17 12:31:27,694] jepsen worker 65 - jepsen.util 4115 :invoke :cas [21 [47 12]] INFO [2019-12-17 12:31:27,711] jepsen worker 65 - jepsen.util 4115 :fail :cas [21 [47 12]] :ValueNotSwapFrom INFO [2019-12-17 12:31:27,711] jepsen worker 65 - jepsen.util 4115 :invoke :cas [21 [2 24]] INFO [2019-12-17 12:31:27,720] jepsen worker 56 - jepsen.util 4916 :fail :cas [18 [46 22]] :ValueNotSwapFrom INFO [2019-12-17 12:31:27,720] jepsen worker 56 - jepsen.util 4916 :invoke :write [18 3] INFO [2019-12-17 12:31:27,725] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":27301,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:31:27,725] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":27301,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:31:27,725] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":27303,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:31:27,725] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":27303,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:31:27,733] jepsen worker 65 - jepsen.util 4115 :fail :cas [21 [2 24]] :ValueNotSwapFrom INFO [2019-12-17 12:31:27,733] jepsen worker 65 - jepsen.util 4115 :invoke :read [21 nil] INFO [2019-12-17 12:31:27,747] jepsen worker 65 - jepsen.util 4115 :ok :read [21 4] INFO [2019-12-17 12:31:27,747] jepsen worker 65 - jepsen.util 4115 :invoke :write [21 43] INFO [2019-12-17 12:31:27,784] jepsen worker 65 - jepsen.util 4115 :ok :write [21 43] INFO [2019-12-17 12:31:27,784] jepsen worker 65 - jepsen.util 4115 :invoke :read [21 nil] INFO [2019-12-17 12:31:27,784] jepsen worker 30 - jepsen.util 4890 :ok :write [10 1] INFO [2019-12-17 12:31:27,784] jepsen worker 30 - jepsen.util 4890 :invoke :cas [10 [20 33]] INFO [2019-12-17 12:31:27,825] jepsen worker 74 - jepsen.util 5114 :ok :write [24 48] INFO [2019-12-17 12:31:27,826] jepsen worker 74 - jepsen.util 5114 :invoke :cas [24 [40 14]] INFO [2019-12-17 12:31:27,834] jepsen worker 65 - jepsen.util 4115 :ok :read [21 43] INFO [2019-12-17 12:31:27,834] jepsen worker 65 - jepsen.util 4115 :invoke :cas [21 [36 21]] INFO [2019-12-17 12:31:27,840] jepsen worker 4 - jepsen.util 3874 :fail :cas [1 [21 41]] :ValueNotSwapFrom INFO [2019-12-17 12:31:27,841] jepsen worker 4 - jepsen.util 3874 :invoke :write [1 3] INFO [2019-12-17 12:31:27,855] jepsen worker 28 - jepsen.util 3538 :ok :read [9 11] INFO [2019-12-17 12:31:27,855] jepsen worker 28 - jepsen.util 3538 :invoke :read [9 nil] INFO [2019-12-17 12:31:27,884] jepsen worker 38 - jepsen.util 4628 :fail :cas [12 [48 38]] :ValueNotSwapFrom INFO [2019-12-17 12:31:27,884] jepsen worker 38 - jepsen.util 4628 :invoke :cas [12 [37 21]] INFO [2019-12-17 12:31:27,889] jepsen worker 65 - jepsen.util 4115 :fail :cas [21 [36 21]] :ValueNotSwapFrom INFO [2019-12-17 12:31:27,889] jepsen worker 65 - jepsen.util 4115 :invoke :cas [21 [13 29]] INFO [2019-12-17 12:31:27,895] jepsen worker 74 - jepsen.util 5114 :fail :cas [24 [40 14]] :ValueNotSwapFrom INFO [2019-12-17 12:31:27,895] jepsen worker 74 - jepsen.util 5114 :invoke :cas [24 [46 5]] INFO [2019-12-17 12:31:27,919] jepsen worker 74 - jepsen.util 5114 :fail :cas [24 [46 5]] :ValueNotSwapFrom INFO [2019-12-17 12:31:27,919] jepsen worker 74 - jepsen.util 5114 :invoke :cas [24 [39 8]] INFO [2019-12-17 12:31:27,922] jepsen worker 65 - jepsen.util 4115 :fail :cas [21 [13 29]] :ValueNotSwapFrom INFO [2019-12-17 12:31:27,922] jepsen worker 65 - jepsen.util 4115 :invoke :write [21 32] INFO [2019-12-17 12:31:27,924] jepsen worker 72 - jepsen.util 4842 :fail :cas [24 [37 13]] :ValueNotSwapFrom INFO [2019-12-17 12:31:27,924] jepsen worker 72 - jepsen.util 4842 :invoke :cas [24 [2 19]] INFO [2019-12-17 12:31:27,941] jepsen worker 74 - jepsen.util 5114 :fail :cas [24 [39 8]] :ValueNotSwapFrom INFO [2019-12-17 12:31:27,941] jepsen worker 74 - jepsen.util 5114 :invoke :cas [24 [21 49]] INFO [2019-12-17 12:31:27,944] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":27319,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:31:27,944] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":27319,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:31:27,944] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":27322,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:31:27,944] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":27322,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:31:27,944] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":27323,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:31:27,944] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":27323,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:31:27,946] jepsen worker 30 - jepsen.util 4890 :fail :cas [10 [20 33]] :ValueNotSwapFrom INFO [2019-12-17 12:31:27,946] jepsen worker 30 - jepsen.util 4890 :invoke :cas [10 [13 33]] INFO [2019-12-17 12:31:27,957] jepsen worker 51 - jepsen.util 4551 :info :write [17 24] :RequestCanceledException INFO [2019-12-17 12:31:27,958] jepsen worker 51 - jepsen.util 4641 :invoke :cas [17 [20 14]] INFO [2019-12-17 12:31:27,965] jepsen worker 74 - jepsen.util 5114 :fail :cas [24 [21 49]] :ValueNotSwapFrom INFO [2019-12-17 12:31:27,965] jepsen worker 74 - jepsen.util 5114 :invoke :write [24 10] INFO [2019-12-17 12:31:27,971] jepsen worker 46 - jepsen.util 4816 :ok :write [15 29] INFO [2019-12-17 12:31:27,971] jepsen worker 46 - jepsen.util 4816 :invoke :cas [15 [9 16]] INFO [2019-12-17 12:31:27,975] jepsen worker 11 - jepsen.util 4691 :ok :write [3 1] INFO [2019-12-17 12:31:27,975] jepsen worker 11 - jepsen.util 4691 :invoke :write [3 20] INFO [2019-12-17 12:31:27,992] jepsen worker 28 - jepsen.util 3538 :ok :read [9 11] INFO [2019-12-17 12:31:27,992] jepsen worker 28 - jepsen.util 3538 :invoke :cas [9 [23 14]] INFO [2019-12-17 12:31:28,014] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":27337,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:31:28,014] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":27337,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:31:28,014] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":27338,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:31:28,014] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":27338,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:31:28,055] jepsen worker 46 - jepsen.util 4816 :fail :cas [15 [9 16]] :ValueNotSwapFrom INFO [2019-12-17 12:31:28,055] jepsen worker 46 - jepsen.util 4816 :invoke :read [15 nil] INFO [2019-12-17 12:31:28,057] jepsen worker 18 - jepsen.util 4698 :fail :cas [6 [11 26]] :ValueNotSwapFrom INFO [2019-12-17 12:31:28,057] jepsen worker 18 - jepsen.util 4698 :invoke :write [6 21] INFO [2019-12-17 12:31:28,092] jepsen worker 79 - jepsen.util 4579 :info :write [26 48] :RequestCanceledException INFO [2019-12-17 12:31:28,093] jepsen worker 79 - jepsen.util 4669 :invoke :write [26 16] INFO [2019-12-17 12:31:28,124] jepsen worker 72 - jepsen.util 4842 :fail :cas [24 [2 19]] :ValueNotSwapFrom INFO [2019-12-17 12:31:28,124] jepsen worker 72 - jepsen.util 4842 :invoke :read [24 nil] INFO [2019-12-17 12:31:28,126] jepsen worker 76 - jepsen.util 6466 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:31:28,126] jepsen worker 76 - jepsen.util 6466 :invoke :write [25 18] INFO [2019-12-17 12:31:28,139] jepsen worker 68 - jepsen.util 5828 :info :write [22 6] :RequestCanceledException INFO [2019-12-17 12:31:28,139] jepsen worker 16 - jepsen.util 7306 :info :cas [5 [13 21]] :RequestCanceledException INFO [2019-12-17 12:31:28,141] jepsen worker 30 - jepsen.util 4890 :fail :cas [10 [13 33]] :ValueNotSwapFrom INFO [2019-12-17 12:31:28,141] jepsen worker 68 - jepsen.util 5918 :invoke :write [22 25] INFO [2019-12-17 12:31:28,141] jepsen worker 16 - jepsen.util 7396 :invoke :cas [5 [41 3]] INFO [2019-12-17 12:31:28,141] jepsen worker 30 - jepsen.util 4890 :invoke :write [10 21] INFO [2019-12-17 12:31:28,143] jepsen worker 19 - jepsen.util 4789 :fail :cas [6 [3 41]] :ValueNotSwapFrom INFO [2019-12-17 12:31:28,143] jepsen worker 19 - jepsen.util 4789 :invoke :read [6 nil] INFO [2019-12-17 12:31:28,149] jepsen worker 46 - jepsen.util 4816 :ok :read [15 29] INFO [2019-12-17 12:31:28,149] jepsen worker 46 - jepsen.util 4816 :invoke :write [15 48] INFO [2019-12-17 12:31:28,150] jepsen worker 65 - jepsen.util 4115 :ok :write [21 32] INFO [2019-12-17 12:31:28,151] jepsen worker 65 - jepsen.util 4115 :invoke :write [21 39] INFO [2019-12-17 12:31:28,152] jepsen worker 26 - jepsen.util 6596 :info :cas [8 [1 11]] :RequestCanceledException INFO [2019-12-17 12:31:28,152] jepsen worker 33 - jepsen.util 6243 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:31:28,152] jepsen worker 33 - jepsen.util 6243 :invoke :write [11 38] INFO [2019-12-17 12:31:28,152] jepsen worker 28 - jepsen.util 3538 :fail :cas [9 [23 14]] :ValueNotSwapFrom INFO [2019-12-17 12:31:28,152] jepsen worker 28 - jepsen.util 3538 :invoke :read [9 nil] INFO [2019-12-17 12:31:28,153] jepsen worker 26 - jepsen.util 6686 :invoke :cas [8 [29 21]] INFO [2019-12-17 12:31:28,156] jepsen worker 38 - jepsen.util 4628 :fail :cas [12 [37 21]] :ValueNotSwapFrom INFO [2019-12-17 12:31:28,157] jepsen worker 38 - jepsen.util 4628 :invoke :read [12 nil] INFO [2019-12-17 12:31:28,168] jepsen worker 72 - jepsen.util 4842 :ok :read [24 48] INFO [2019-12-17 12:31:28,168] jepsen worker 72 - jepsen.util 4842 :invoke :read [24 nil] INFO [2019-12-17 12:31:28,184] jepsen worker 63 - jepsen.util 3933 :ok :read [21 43] INFO [2019-12-17 12:31:28,184] jepsen worker 63 - jepsen.util 3933 :invoke :read [21 nil] INFO [2019-12-17 12:31:28,191] jepsen worker 84 - jepsen.util 5934 :info :write [28 17] :RequestCanceledException INFO [2019-12-17 12:31:28,191] jepsen worker 82 - jepsen.util 7102 :info :cas [27 [41 14]] :RequestCanceledException INFO [2019-12-17 12:31:28,192] jepsen worker 82 - jepsen.util 7192 :invoke :write [27 5] INFO [2019-12-17 12:31:28,192] jepsen worker 84 - jepsen.util 6024 :invoke :write [28 39] INFO [2019-12-17 12:31:28,206] jepsen worker 65 - jepsen.util 4115 :ok :write [21 39] INFO [2019-12-17 12:31:28,206] jepsen worker 65 - jepsen.util 4115 :invoke :read [21 nil] INFO [2019-12-17 12:31:28,208] jepsen worker 49 - jepsen.util 5719 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:31:28,208] jepsen worker 34 - jepsen.util 5884 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:31:28,208] jepsen worker 81 - jepsen.util 6111 :info :cas [27 [46 24]] :RequestCanceledException INFO [2019-12-17 12:31:28,208] jepsen worker 43 - jepsen.util 6073 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:31:28,208] jepsen worker 49 - jepsen.util 5719 :invoke :read [16 nil] INFO [2019-12-17 12:31:28,208] jepsen worker 43 - jepsen.util 6073 :invoke :cas [14 [12 2]] INFO [2019-12-17 12:31:28,208] jepsen worker 34 - jepsen.util 5884 :invoke :cas [11 [42 37]] INFO [2019-12-17 12:31:28,209] jepsen worker 81 - jepsen.util 6201 :invoke :write [27 42] INFO [2019-12-17 12:31:28,225] jepsen worker 57 - jepsen.util 5997 :info :write [19 11] :RequestCanceledException INFO [2019-12-17 12:31:28,225] jepsen worker 58 - jepsen.util 6448 :info :write [19 20] :RequestCanceledException INFO [2019-12-17 12:31:28,225] jepsen worker 83 - jepsen.util 6833 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:31:28,225] jepsen worker 66 - jepsen.util 5916 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:31:28,225] jepsen worker 83 - jepsen.util 6833 :invoke :read [27 nil] INFO [2019-12-17 12:31:28,225] jepsen worker 66 - jepsen.util 5916 :invoke :read [22 nil] INFO [2019-12-17 12:31:28,226] jepsen worker 6 - jepsen.util 6756 :info :write [2 30] :RequestCanceledException INFO [2019-12-17 12:31:28,226] jepsen worker 17 - jepsen.util 6227 :info :cas [5 [6 2]] :RequestCanceledException INFO [2019-12-17 12:31:28,226] jepsen worker 8 - jepsen.util 5678 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:31:28,226] jepsen worker 57 - jepsen.util 6087 :invoke :read [19 nil] INFO [2019-12-17 12:31:28,226] jepsen worker 58 - jepsen.util 6538 :invoke :cas [19 [14 13]] INFO [2019-12-17 12:31:28,226] jepsen worker 8 - jepsen.util 5678 :invoke :cas [2 [42 29]] INFO [2019-12-17 12:31:28,227] jepsen worker 6 - jepsen.util 6846 :invoke :read [2 nil] INFO [2019-12-17 12:31:28,227] jepsen worker 17 - jepsen.util 6317 :invoke :write [5 10] INFO [2019-12-17 12:31:28,231] jepsen worker 65 - jepsen.util 4115 :ok :read [21 39] INFO [2019-12-17 12:31:28,231] jepsen worker 65 - jepsen.util 4115 :invoke :read [21 nil] INFO [2019-12-17 12:31:28,233] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":27370,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:31:28,233] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":27360,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:31:28,233] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":27370,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:31:28,233] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":27360,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:31:28,233] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":27374,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:31:28,233] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":27362,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:31:28,233] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":27374,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:31:28,233] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":27362,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:31:28,233] jepsen worker 37 - jepsen.util 4537 :ok :read [12 14] INFO [2019-12-17 12:31:28,234] jepsen worker 37 - jepsen.util 4537 :invoke :read [12 nil] INFO [2019-12-17 12:31:28,239] jepsen worker 25 - jepsen.util 6415 :info :write [8 3] :RequestCanceledException INFO [2019-12-17 12:31:28,239] jepsen worker 48 - jepsen.util 5718 :info :write [16 15] :RequestCanceledException INFO [2019-12-17 12:31:28,239] jepsen worker 15 - jepsen.util 5415 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:31:28,239] jepsen worker 15 - jepsen.util 5415 :invoke :read [5 nil] INFO [2019-12-17 12:31:28,240] jepsen worker 25 - jepsen.util 6505 :invoke :write [8 21] INFO [2019-12-17 12:31:28,240] jepsen worker 48 - jepsen.util 5808 :invoke :cas [16 [8 12]] INFO [2019-12-17 12:31:28,247] jepsen worker 65 - jepsen.util 4115 :ok :read [21 39] INFO [2019-12-17 12:31:28,248] jepsen worker 65 - jepsen.util 4115 :invoke :cas [21 [36 45]] INFO [2019-12-17 12:31:28,256] jepsen worker 62 - jepsen.util 5912 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:31:28,256] jepsen worker 62 - jepsen.util 5912 :invoke :read [20 nil] INFO [2019-12-17 12:31:28,270] jepsen worker 65 - jepsen.util 4115 :fail :cas [21 [36 45]] :ValueNotSwapFrom INFO [2019-12-17 12:31:28,270] jepsen worker 65 - jepsen.util 4115 :invoke :cas [21 [4 8]] INFO [2019-12-17 12:31:28,292] jepsen worker 39 - jepsen.util 5979 :info :write [13 26] :RequestCanceledException INFO [2019-12-17 12:31:28,292] jepsen worker 77 - jepsen.util 6557 :info :cas [25 [28 23]] :RequestCanceledException INFO [2019-12-17 12:31:28,292] jepsen worker 45 - jepsen.util 4905 :fail :read [15 nil] :RequestTimeoutException INFO [2019-12-17 12:31:28,292] jepsen worker 1 - jepsen.util 6121 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:31:28,292] jepsen worker 45 - jepsen.util 4905 :invoke :write [15 29] INFO [2019-12-17 12:31:28,292] jepsen worker 1 - jepsen.util 6121 :invoke :write [0 12] INFO [2019-12-17 12:31:28,293] jepsen worker 39 - jepsen.util 6069 :invoke :read [13 nil] INFO [2019-12-17 12:31:28,293] jepsen worker 77 - jepsen.util 6647 :invoke :cas [25 [41 16]] INFO [2019-12-17 12:31:28,300] jepsen worker 65 - jepsen.util 4115 :fail :cas [21 [4 8]] :ValueNotSwapFrom INFO [2019-12-17 12:31:28,301] jepsen worker 65 - jepsen.util 4115 :invoke :cas [21 [19 3]] INFO [2019-12-17 12:31:28,310] jepsen worker 60 - jepsen.util 6810 :info :write [20 16] :RequestCanceledException INFO [2019-12-17 12:31:28,310] jepsen worker 2 - jepsen.util 5942 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:31:28,310] jepsen worker 14 - jepsen.util 4874 :info :cas [4 [49 22]] :RequestCanceledException INFO [2019-12-17 12:31:28,310] jepsen worker 42 - jepsen.util 6882 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:31:28,310] jepsen worker 40 - jepsen.util 7060 :info :cas [13 [29 44]] :RequestCanceledException INFO [2019-12-17 12:31:28,310] jepsen worker 2 - jepsen.util 5942 :invoke :read [0 nil] INFO [2019-12-17 12:31:28,310] jepsen worker 42 - jepsen.util 6882 :invoke :read [14 nil] INFO [2019-12-17 12:31:28,311] jepsen worker 14 - jepsen.util 4964 :invoke :read [4 nil] INFO [2019-12-17 12:31:28,311] jepsen worker 60 - jepsen.util 6900 :invoke :cas [20 [1 9]] INFO [2019-12-17 12:31:28,311] jepsen worker 40 - jepsen.util 7150 :invoke :read [13 nil] INFO [2019-12-17 12:31:28,325] jepsen worker 50 - jepsen.util 6620 :info :cas [16 [22 41]] :RequestCanceledException INFO [2019-12-17 12:31:28,325] jepsen worker 85 - jepsen.util 7105 :info :cas [28 [3 45]] :RequestCanceledException INFO [2019-12-17 12:31:28,325] jepsen worker 7 - jepsen.util 6397 :info :cas [2 [36 44]] :RequestCanceledException INFO [2019-12-17 12:31:28,325] jepsen worker 24 - jepsen.util 5964 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:31:28,326] jepsen worker 75 - jepsen.util 6285 :info :write [25 36] :RequestCanceledException INFO [2019-12-17 12:31:28,326] jepsen worker 24 - jepsen.util 5964 :invoke :read [8 nil] INFO [2019-12-17 12:31:28,327] jepsen worker 85 - jepsen.util 7195 :invoke :cas [28 [4 17]] INFO [2019-12-17 12:31:28,327] jepsen worker 50 - jepsen.util 6710 :invoke :cas [16 [0 35]] INFO [2019-12-17 12:31:28,327] jepsen worker 7 - jepsen.util 6487 :invoke :cas [2 [46 8]] INFO [2019-12-17 12:31:28,327] jepsen worker 75 - jepsen.util 6375 :invoke :write [25 43] INFO [2019-12-17 12:31:28,339] jepsen worker 22 - jepsen.util 6412 :info :write [7 5] :RequestCanceledException INFO [2019-12-17 12:31:28,339] jepsen worker 44 - jepsen.util 6974 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:31:28,340] jepsen worker 44 - jepsen.util 6974 :invoke :write [14 44] INFO [2019-12-17 12:31:28,340] jepsen worker 22 - jepsen.util 6502 :invoke :read [7 nil] INFO [2019-12-17 12:31:28,353] jepsen worker 0 - jepsen.util 6930 :info :write [0 46] :RequestCanceledException INFO [2019-12-17 12:31:28,353] jepsen worker 80 - jepsen.util 4670 :info :write [26 10] :RequestCanceledException INFO [2019-12-17 12:31:28,354] jepsen worker 0 - jepsen.util 7020 :invoke :cas [0 [25 20]] INFO [2019-12-17 12:31:28,354] jepsen worker 80 - jepsen.util 4760 :invoke :write [26 44] INFO [2019-12-17 12:31:28,365] jepsen worker 5 - jepsen.util 4235 :ok :write [1 0] INFO [2019-12-17 12:31:28,365] jepsen worker 5 - jepsen.util 4235 :invoke :cas [1 [5 43]] INFO [2019-12-17 12:31:28,392] jepsen worker 41 - jepsen.util 6791 :info :cas [13 [26 32]] :RequestCanceledException INFO [2019-12-17 12:31:28,392] jepsen worker 41 - jepsen.util 6881 :invoke :write [13 5] INFO [2019-12-17 12:31:28,404] jepsen worker 30 - jepsen.util 4890 :ok :write [10 21] INFO [2019-12-17 12:31:28,404] jepsen worker 30 - jepsen.util 4890 :invoke :read [10 nil] INFO [2019-12-17 12:31:28,404] jepsen worker 11 - jepsen.util 4691 :ok :write [3 20] INFO [2019-12-17 12:31:28,404] jepsen worker 11 - jepsen.util 4691 :invoke :cas [3 [31 38]] INFO [2019-12-17 12:31:28,409] jepsen worker 35 - jepsen.util 6425 :info :cas [11 [0 37]] :RequestCanceledException INFO [2019-12-17 12:31:28,409] jepsen worker 67 - jepsen.util 6547 :info :write [22 29] :RequestCanceledException INFO [2019-12-17 12:31:28,410] jepsen worker 67 - jepsen.util 6637 :invoke :read [22 nil] INFO [2019-12-17 12:31:28,410] jepsen worker 35 - jepsen.util 6515 :invoke :write [11 30] INFO [2019-12-17 12:31:28,416] jepsen worker 30 - jepsen.util 4890 :ok :read [10 21] INFO [2019-12-17 12:31:28,416] jepsen worker 30 - jepsen.util 4890 :invoke :write [10 25] INFO [2019-12-17 12:31:28,417] jepsen worker 53 - jepsen.util 4733 :ok :write [17 47] INFO [2019-12-17 12:31:28,417] jepsen worker 53 - jepsen.util 4733 :invoke :write [17 14] INFO [2019-12-17 12:31:28,421] jepsen worker 74 - jepsen.util 5114 :ok :write [24 10] INFO [2019-12-17 12:31:28,422] jepsen worker 74 - jepsen.util 5114 :invoke :write [24 48] INFO [2019-12-17 12:31:28,422] jepsen worker 46 - jepsen.util 4816 :ok :write [15 48] INFO [2019-12-17 12:31:28,422] jepsen worker 46 - jepsen.util 4816 :invoke :write [15 32] INFO [2019-12-17 12:31:28,425] jepsen worker 86 - jepsen.util 6656 :info :write [28 10] :RequestCanceledException INFO [2019-12-17 12:31:28,425] jepsen worker 70 - jepsen.util 5380 :info :write [23 4] :RequestCanceledException INFO [2019-12-17 12:31:28,426] jepsen worker 70 - jepsen.util 5470 :invoke :read [23 nil] INFO [2019-12-17 12:31:28,426] jepsen worker 86 - jepsen.util 6746 :invoke :cas [28 [42 45]] INFO [2019-12-17 12:31:28,445] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:31:28,445] jepsen nemesis - couchbase.util Rebalance check count: 205 INFO [2019-12-17 12:31:28,445] jepsen nemesis - couchbase.util Rebalance stuck count: 186 INFO [2019-12-17 12:31:28,455] jepsen worker 12 - jepsen.util 4962 :ok :write [4 48] INFO [2019-12-17 12:31:28,455] jepsen worker 12 - jepsen.util 4962 :invoke :read [4 nil] INFO [2019-12-17 12:31:28,480] jepsen worker 18 - jepsen.util 4698 :ok :write [6 21] INFO [2019-12-17 12:31:28,481] jepsen worker 18 - jepsen.util 4698 :invoke :write [6 32] INFO [2019-12-17 12:31:28,492] jepsen worker 47 - jepsen.util 5267 :info :write [15 12] :RequestCanceledException INFO [2019-12-17 12:31:28,492] jepsen worker 64 - jepsen.util 3574 :info :write [21 3] :RequestCanceledException INFO [2019-12-17 12:31:28,493] jepsen worker 64 - jepsen.util 3664 :invoke :read [21 nil] INFO [2019-12-17 12:31:28,493] jepsen worker 47 - jepsen.util 5357 :invoke :cas [15 [7 8]] INFO [2019-12-17 12:31:28,501] jepsen worker 19 - jepsen.util 4789 :ok :read [6 21] INFO [2019-12-17 12:31:28,501] jepsen worker 19 - jepsen.util 4789 :invoke :cas [6 [21 15]] INFO [2019-12-17 12:31:28,529] jepsen worker 30 - jepsen.util 4890 :ok :write [10 25] INFO [2019-12-17 12:31:28,529] jepsen worker 30 - jepsen.util 4890 :invoke :write [10 46] INFO [2019-12-17 12:31:28,530] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":27423,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:31:28,530] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":27423,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:31:28,530] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":27427,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:31:28,530] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":27427,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:31:28,535] jepsen worker 63 - jepsen.util 3933 :ok :read [21 39] INFO [2019-12-17 12:31:28,535] jepsen worker 63 - jepsen.util 3933 :invoke :write [21 43] INFO [2019-12-17 12:31:28,578] jepsen worker 37 - jepsen.util 4537 :ok :read [12 14] INFO [2019-12-17 12:31:28,578] jepsen worker 37 - jepsen.util 4537 :invoke :read [12 nil] INFO [2019-12-17 12:31:28,590] jepsen worker 3 - jepsen.util 4863 :ok :write [1 26] INFO [2019-12-17 12:31:28,591] jepsen worker 3 - jepsen.util 4863 :invoke :write [1 13] INFO [2019-12-17 12:31:28,592] jepsen worker 69 - jepsen.util 4929 :info :write [23 47] :RequestCanceledException INFO [2019-12-17 12:31:28,593] jepsen worker 69 - jepsen.util 5019 :invoke :cas [23 [20 3]] INFO [2019-12-17 12:31:28,610] jepsen worker 47 - jepsen.util 5357 :fail :cas [15 [7 8]] :ValueNotSwapFrom INFO [2019-12-17 12:31:28,610] jepsen worker 47 - jepsen.util 5357 :invoke :read [15 nil] INFO [2019-12-17 12:31:28,611] jepsen worker 74 - jepsen.util 5114 :ok :write [24 48] INFO [2019-12-17 12:31:28,611] jepsen worker 74 - jepsen.util 5114 :invoke :read [24 nil] INFO [2019-12-17 12:31:28,617] jepsen worker 11 - jepsen.util 4691 :fail :cas [3 [31 38]] :ValueNotSwapFrom INFO [2019-12-17 12:31:28,617] jepsen worker 11 - jepsen.util 4691 :invoke :read [3 nil] INFO [2019-12-17 12:31:28,656] jepsen worker 71 - jepsen.util 5111 :info :write [23 23] :RequestCanceledException INFO [2019-12-17 12:31:28,657] jepsen worker 74 - jepsen.util 5114 :ok :read [24 48] INFO [2019-12-17 12:31:28,657] jepsen worker 71 - jepsen.util 5201 :invoke :read [23 nil] INFO [2019-12-17 12:31:28,657] jepsen worker 74 - jepsen.util 5114 :invoke :read [24 nil] INFO [2019-12-17 12:31:28,681] jepsen worker 74 - jepsen.util 5114 :ok :read [24 48] INFO [2019-12-17 12:31:28,681] jepsen worker 74 - jepsen.util 5114 :invoke :cas [24 [44 34]] INFO [2019-12-17 12:31:28,681] jepsen worker 18 - jepsen.util 4698 :ok :write [6 32] INFO [2019-12-17 12:31:28,682] jepsen worker 18 - jepsen.util 4698 :invoke :cas [6 [17 13]] INFO [2019-12-17 12:31:28,692] jepsen worker 13 - jepsen.util 4333 :info :write [4 34] :RequestCanceledException INFO [2019-12-17 12:31:28,692] jepsen worker 70 - jepsen.util 5470 :ok :read [23 32] INFO [2019-12-17 12:31:28,692] jepsen worker 70 - jepsen.util 5470 :invoke :read [23 nil] INFO [2019-12-17 12:31:28,693] jepsen worker 13 - jepsen.util 4423 :invoke :read [4 nil] INFO [2019-12-17 12:31:28,711] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":27445,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:31:28,711] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":27445,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:31:28,711] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":27448,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:31:28,712] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":27448,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:31:28,712] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":27449,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:31:28,712] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":27449,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:31:28,716] jepsen worker 12 - jepsen.util 4962 :ok :read [4 48] INFO [2019-12-17 12:31:28,716] jepsen worker 52 - jepsen.util 5092 :info :write [17 23] :RequestCanceledException INFO [2019-12-17 12:31:28,716] jepsen worker 12 - jepsen.util 4962 :invoke :read [4 nil] INFO [2019-12-17 12:31:28,716] jepsen worker 21 - jepsen.util 5961 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:31:28,716] jepsen worker 74 - jepsen.util 5114 :fail :cas [24 [44 34]] :ValueNotSwapFrom INFO [2019-12-17 12:31:28,717] jepsen worker 21 - jepsen.util 5961 :invoke :read [7 nil] INFO [2019-12-17 12:31:28,717] jepsen worker 74 - jepsen.util 5114 :invoke :cas [24 [11 6]] INFO [2019-12-17 12:31:28,717] jepsen worker 52 - jepsen.util 5182 :invoke :cas [17 [31 15]] INFO [2019-12-17 12:31:28,721] jepsen worker 19 - jepsen.util 4789 :fail :cas [6 [21 15]] :ValueNotSwapFrom INFO [2019-12-17 12:31:28,721] jepsen worker 19 - jepsen.util 4789 :invoke :read [6 nil] INFO [2019-12-17 12:31:28,739] jepsen worker 23 - jepsen.util 6503 :info :write [7 1] :RequestCanceledException INFO [2019-12-17 12:31:28,740] jepsen worker 23 - jepsen.util 6593 :invoke :read [7 nil] INFO [2019-12-17 12:31:28,744] jepsen worker 74 - jepsen.util 5114 :fail :cas [24 [11 6]] :ValueNotSwapFrom INFO [2019-12-17 12:31:28,744] jepsen worker 74 - jepsen.util 5114 :invoke :write [24 30] INFO [2019-12-17 12:31:28,756] jepsen worker 61 - jepsen.util 7081 :info :write [20 0] :RequestCanceledException INFO [2019-12-17 12:31:28,757] jepsen worker 61 - jepsen.util 7171 :invoke :cas [20 [9 42]] INFO [2019-12-17 12:31:28,776] jepsen worker 63 - jepsen.util 3933 :ok :write [21 43] INFO [2019-12-17 12:31:28,776] jepsen worker 63 - jepsen.util 3933 :invoke :write [21 30] INFO [2019-12-17 12:31:28,778] jepsen worker 51 - jepsen.util 4641 :fail :cas [17 [20 14]] :ValueNotSwapFrom INFO [2019-12-17 12:31:28,778] jepsen worker 51 - jepsen.util 4641 :invoke :write [17 8] INFO [2019-12-17 12:31:28,792] jepsen worker 88 - jepsen.util 4858 :info :write [29 49] :RequestCanceledException INFO [2019-12-17 12:31:28,793] jepsen worker 88 - jepsen.util 4948 :invoke :cas [29 [35 30]] INFO [2019-12-17 12:31:28,825] jepsen worker 59 - jepsen.util 6269 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:31:28,825] jepsen worker 59 - jepsen.util 6269 :invoke :read [19 nil] INFO [2019-12-17 12:31:28,841] jepsen worker 63 - jepsen.util 3933 :ok :write [21 30] INFO [2019-12-17 12:31:28,841] jepsen worker 63 - jepsen.util 3933 :invoke :read [21 nil] INFO [2019-12-17 12:31:28,867] jepsen worker 63 - jepsen.util 3933 :ok :read [21 30] INFO [2019-12-17 12:31:28,867] jepsen worker 63 - jepsen.util 3933 :invoke :cas [21 [11 4]] INFO [2019-12-17 12:31:28,879] jepsen worker 63 - jepsen.util 3933 :fail :cas [21 [11 4]] :ValueNotSwapFrom INFO [2019-12-17 12:31:28,879] jepsen worker 63 - jepsen.util 3933 :invoke :read [21 nil] INFO [2019-12-17 12:31:28,889] jepsen worker 63 - jepsen.util 3933 :ok :read [21 30] INFO [2019-12-17 12:31:28,889] jepsen worker 63 - jepsen.util 3933 :invoke :read [21 nil] INFO [2019-12-17 12:31:28,900] jepsen worker 63 - jepsen.util 3933 :ok :read [21 30] INFO [2019-12-17 12:31:28,900] jepsen worker 63 - jepsen.util 3933 :invoke :write [21 17] INFO [2019-12-17 12:31:28,913] jepsen worker 38 - jepsen.util 4628 :ok :read [12 14] INFO [2019-12-17 12:31:28,913] jepsen worker 38 - jepsen.util 4628 :invoke :read [12 nil] INFO [2019-12-17 12:31:28,931] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":27470,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:31:28,931] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":27470,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:31:28,931] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":27473,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:31:28,931] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":27473,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:31:28,933] jepsen worker 63 - jepsen.util 3933 :ok :write [21 17] INFO [2019-12-17 12:31:28,933] jepsen worker 63 - jepsen.util 3933 :invoke :read [21 nil] INFO [2019-12-17 12:31:28,945] jepsen worker 63 - jepsen.util 3933 :ok :read [21 17] INFO [2019-12-17 12:31:28,945] jepsen worker 63 - jepsen.util 3933 :invoke :write [21 9] INFO [2019-12-17 12:31:28,964] jepsen worker 5 - jepsen.util 4235 :fail :cas [1 [5 43]] :ValueNotSwapFrom INFO [2019-12-17 12:31:28,964] jepsen worker 5 - jepsen.util 4235 :invoke :read [1 nil] INFO [2019-12-17 12:31:28,966] jepsen worker 37 - jepsen.util 4537 :ok :read [12 14] INFO [2019-12-17 12:31:28,966] jepsen worker 37 - jepsen.util 4537 :invoke :read [12 nil] INFO [2019-12-17 12:31:28,968] jepsen worker 70 - jepsen.util 5470 :ok :read [23 32] INFO [2019-12-17 12:31:28,968] jepsen worker 70 - jepsen.util 5470 :invoke :cas [23 [12 21]] INFO [2019-12-17 12:31:28,984] jepsen worker 63 - jepsen.util 3933 :ok :write [21 9] INFO [2019-12-17 12:31:28,984] jepsen worker 63 - jepsen.util 3933 :invoke :cas [21 [17 48]] INFO [2019-12-17 12:31:28,997] jepsen worker 63 - jepsen.util 3933 :fail :cas [21 [17 48]] :ValueNotSwapFrom INFO [2019-12-17 12:31:28,997] jepsen worker 63 - jepsen.util 3933 :invoke :cas [21 [23 19]] INFO [2019-12-17 12:31:29,000] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":27487,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:31:29,000] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":27487,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:31:29,000] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":27491,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:31:29,000] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":27491,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:31:29,004] jepsen worker 13 - jepsen.util 4423 :ok :read [4 48] INFO [2019-12-17 12:31:29,004] jepsen worker 13 - jepsen.util 4423 :invoke :read [4 nil] INFO [2019-12-17 12:31:29,011] jepsen worker 63 - jepsen.util 3933 :fail :cas [21 [23 19]] :ValueNotSwapFrom INFO [2019-12-17 12:31:29,011] jepsen worker 63 - jepsen.util 3933 :invoke :cas [21 [5 41]] INFO [2019-12-17 12:31:29,020] jepsen worker 63 - jepsen.util 3933 :fail :cas [21 [5 41]] :ValueNotSwapFrom INFO [2019-12-17 12:31:29,020] jepsen worker 63 - jepsen.util 3933 :invoke :cas [21 [35 29]] INFO [2019-12-17 12:31:29,031] jepsen worker 63 - jepsen.util 3933 :fail :cas [21 [35 29]] :ValueNotSwapFrom INFO [2019-12-17 12:31:29,031] jepsen worker 63 - jepsen.util 3933 :invoke :write [21 38] INFO [2019-12-17 12:31:29,056] jepsen worker 20 - jepsen.util 4790 :info :write [6 8] :RequestCanceledException INFO [2019-12-17 12:31:29,057] jepsen worker 20 - jepsen.util 4880 :invoke :write [6 18] INFO [2019-12-17 12:31:29,064] jepsen worker 30 - jepsen.util 4890 :ok :write [10 46] INFO [2019-12-17 12:31:29,064] jepsen worker 30 - jepsen.util 4890 :invoke :cas [10 [48 17]] INFO [2019-12-17 12:31:29,071] jepsen worker 74 - jepsen.util 5114 :ok :write [24 30] INFO [2019-12-17 12:31:29,071] jepsen worker 74 - jepsen.util 5114 :invoke :write [24 8] INFO [2019-12-17 12:31:29,073] jepsen worker 46 - jepsen.util 4816 :ok :write [15 32] INFO [2019-12-17 12:31:29,073] jepsen worker 46 - jepsen.util 4816 :invoke :read [15 nil] INFO [2019-12-17 12:31:29,089] jepsen worker 69 - jepsen.util 5019 :fail :cas [23 [20 3]] :ValueNotSwapFrom INFO [2019-12-17 12:31:29,089] jepsen worker 69 - jepsen.util 5019 :invoke :read [23 nil] INFO [2019-12-17 12:31:29,096] jepsen worker 38 - jepsen.util 4628 :ok :read [12 14] INFO [2019-12-17 12:31:29,096] jepsen worker 38 - jepsen.util 4628 :invoke :cas [12 [14 13]] INFO [2019-12-17 12:31:29,103] jepsen worker 30 - jepsen.util 4890 :fail :cas [10 [48 17]] :ValueNotSwapFrom INFO [2019-12-17 12:31:29,103] jepsen worker 30 - jepsen.util 4890 :invoke :write [10 11] INFO [2019-12-17 12:31:29,103] jepsen worker 63 - jepsen.util 3933 :ok :write [21 38] INFO [2019-12-17 12:31:29,104] jepsen worker 63 - jepsen.util 3933 :invoke :write [21 19] INFO [2019-12-17 12:31:29,111] jepsen worker 46 - jepsen.util 4816 :ok :read [15 32] INFO [2019-12-17 12:31:29,111] jepsen worker 46 - jepsen.util 4816 :invoke :cas [15 [38 49]] INFO [2019-12-17 12:31:29,120] jepsen worker 20 - jepsen.util 4880 :ok :write [6 18] INFO [2019-12-17 12:31:29,121] jepsen worker 20 - jepsen.util 4880 :invoke :read [6 nil] INFO [2019-12-17 12:31:29,126] jepsen worker 87 - jepsen.util 4497 :info :write [29 13] :RequestCanceledException INFO [2019-12-17 12:31:29,127] jepsen worker 87 - jepsen.util 4587 :invoke :cas [29 [18 35]] INFO [2019-12-17 12:31:29,133] jepsen worker 46 - jepsen.util 4816 :fail :cas [15 [38 49]] :ValueNotSwapFrom INFO [2019-12-17 12:31:29,133] jepsen worker 46 - jepsen.util 4816 :invoke :write [15 17] INFO [2019-12-17 12:31:29,152] jepsen worker 14 - jepsen.util 4964 :ok :read [4 48] INFO [2019-12-17 12:31:29,152] jepsen worker 14 - jepsen.util 4964 :invoke :write [4 31] INFO [2019-12-17 12:31:29,163] jepsen worker 20 - jepsen.util 4880 :ok :read [6 18] INFO [2019-12-17 12:31:29,163] jepsen worker 20 - jepsen.util 4880 :invoke :cas [6 [5 12]] INFO [2019-12-17 12:31:29,187] jepsen worker 20 - jepsen.util 4880 :fail :cas [6 [5 12]] :ValueNotSwapFrom INFO [2019-12-17 12:31:29,187] jepsen worker 20 - jepsen.util 4880 :invoke :write [6 32] INFO [2019-12-17 12:31:29,197] jepsen worker 88 - jepsen.util 4948 :fail :cas [29 [35 30]] :ValueNotSwapFrom INFO [2019-12-17 12:31:29,198] jepsen worker 88 - jepsen.util 4948 :invoke :cas [29 [46 30]] INFO [2019-12-17 12:31:29,219] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":27513,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:31:29,219] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":27513,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:31:29,219] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":27514,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:31:29,219] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":27523,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:31:29,219] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":27514,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:31:29,219] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":27523,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:31:29,219] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":27524,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:31:29,220] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":27524,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:31:29,226] jepsen worker 89 - jepsen.util 4319 :info :write [29 33] :RequestCanceledException INFO [2019-12-17 12:31:29,228] jepsen worker 89 - jepsen.util 4409 :invoke :read [29 nil] INFO [2019-12-17 12:31:29,245] jepsen worker 37 - jepsen.util 4537 :ok :read [12 14] INFO [2019-12-17 12:31:29,246] jepsen worker 37 - jepsen.util 4537 :invoke :write [12 11] INFO [2019-12-17 12:31:29,247] jepsen worker 52 - jepsen.util 5182 :fail :cas [17 [31 15]] :ValueNotSwapFrom INFO [2019-12-17 12:31:29,247] jepsen worker 52 - jepsen.util 5182 :invoke :cas [17 [48 6]] INFO [2019-12-17 12:31:29,279] jepsen worker 63 - jepsen.util 3933 :ok :write [21 19] INFO [2019-12-17 12:31:29,279] jepsen worker 63 - jepsen.util 3933 :invoke :read [21 nil] INFO [2019-12-17 12:31:29,287] jepsen worker 69 - jepsen.util 5019 :ok :read [23 32] INFO [2019-12-17 12:31:29,287] jepsen worker 69 - jepsen.util 5019 :invoke :cas [23 [8 23]] INFO [2019-12-17 12:31:29,298] jepsen worker 71 - jepsen.util 5201 :ok :read [23 32] INFO [2019-12-17 12:31:29,298] jepsen worker 71 - jepsen.util 5201 :invoke :cas [23 [29 34]] INFO [2019-12-17 12:31:29,303] jepsen worker 30 - jepsen.util 4890 :ok :write [10 11] INFO [2019-12-17 12:31:29,303] jepsen worker 30 - jepsen.util 4890 :invoke :write [10 49] INFO [2019-12-17 12:31:29,309] jepsen worker 63 - jepsen.util 3933 :ok :read [21 19] INFO [2019-12-17 12:31:29,309] jepsen worker 63 - jepsen.util 3933 :invoke :cas [21 [10 2]] INFO [2019-12-17 12:31:29,310] jepsen worker 55 - jepsen.util 5635 :info :write [18 32] :RequestCanceledException INFO [2019-12-17 12:31:29,311] jepsen worker 55 - jepsen.util 5725 :invoke :write [18 7] INFO [2019-12-17 12:31:29,325] jepsen worker 10 - jepsen.util 4690 :info :cas [3 [6 7]] :RequestCanceledException INFO [2019-12-17 12:31:29,326] jepsen worker 10 - jepsen.util 4780 :invoke :cas [3 [14 2]] INFO [2019-12-17 12:31:29,331] jepsen worker 13 - jepsen.util 4423 :ok :read [4 48] INFO [2019-12-17 12:31:29,331] jepsen worker 13 - jepsen.util 4423 :invoke :read [4 nil] INFO [2019-12-17 12:31:29,348] jepsen worker 12 - jepsen.util 4962 :ok :read [4 48] INFO [2019-12-17 12:31:29,348] jepsen worker 12 - jepsen.util 4962 :invoke :read [4 nil] INFO [2019-12-17 12:31:29,350] jepsen worker 63 - jepsen.util 3933 :fail :cas [21 [10 2]] :ValueNotSwapFrom INFO [2019-12-17 12:31:29,350] jepsen worker 63 - jepsen.util 3933 :invoke :cas [21 [7 49]] INFO [2019-12-17 12:31:29,373] jepsen worker 71 - jepsen.util 5201 :fail :cas [23 [29 34]] :ValueNotSwapFrom INFO [2019-12-17 12:31:29,374] jepsen worker 71 - jepsen.util 5201 :invoke :write [23 8] INFO [2019-12-17 12:31:29,381] jepsen worker 63 - jepsen.util 3933 :fail :cas [21 [7 49]] :ValueNotSwapFrom INFO [2019-12-17 12:31:29,382] jepsen worker 63 - jepsen.util 3933 :invoke :cas [21 [40 2]] INFO [2019-12-17 12:31:29,388] jepsen worker 5 - jepsen.util 4235 :ok :read [1 26] INFO [2019-12-17 12:31:29,388] jepsen worker 5 - jepsen.util 4235 :invoke :write [1 8] INFO [2019-12-17 12:31:29,395] jepsen worker 63 - jepsen.util 3933 :fail :cas [21 [40 2]] :ValueNotSwapFrom INFO [2019-12-17 12:31:29,395] jepsen worker 63 - jepsen.util 3933 :invoke :write [21 13] INFO [2019-12-17 12:31:29,410] jepsen worker 73 - jepsen.util 4483 :info :write [24 47] :RequestCanceledException INFO [2019-12-17 12:31:29,410] jepsen worker 31 - jepsen.util 4621 :info :cas [10 [35 25]] :RequestCanceledException INFO [2019-12-17 12:31:29,411] jepsen worker 31 - jepsen.util 4711 :invoke :read [10 nil] INFO [2019-12-17 12:31:29,411] jepsen worker 73 - jepsen.util 4573 :invoke :cas [24 [35 47]] INFO [2019-12-17 12:31:29,423] jepsen worker 9 - jepsen.util 5589 :info :write [3 46] :RequestCanceledException INFO [2019-12-17 12:31:29,423] jepsen worker 27 - jepsen.util 4617 :info :write [9 19] :RequestCanceledException INFO [2019-12-17 12:31:29,424] jepsen worker 9 - jepsen.util 5679 :invoke :read [3 nil] INFO [2019-12-17 12:31:29,424] jepsen worker 27 - jepsen.util 4707 :invoke :cas [9 [1 49]] INFO [2019-12-17 12:31:29,453] jepsen worker 78 - jepsen.util 4758 :info :write [26 32] :RequestCanceledException INFO [2019-12-17 12:31:29,455] jepsen worker 78 - jepsen.util 4848 :invoke :write [26 29] INFO [2019-12-17 12:31:29,456] jepsen worker 27 - jepsen.util 4707 :fail :cas [9 [1 49]] :ValueNotSwapFrom INFO [2019-12-17 12:31:29,457] jepsen worker 27 - jepsen.util 4707 :invoke :read [9 nil] INFO [2019-12-17 12:31:29,458] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:31:29,458] jepsen nemesis - couchbase.util Rebalance check count: 206 INFO [2019-12-17 12:31:29,458] jepsen nemesis - couchbase.util Rebalance stuck count: 187 INFO [2019-12-17 12:31:29,459] jepsen worker 63 - jepsen.util 3933 :ok :write [21 13] INFO [2019-12-17 12:31:29,459] jepsen worker 63 - jepsen.util 3933 :invoke :cas [21 [5 28]] INFO [2019-12-17 12:31:29,487] jepsen worker 70 - jepsen.util 5470 :fail :cas [23 [12 21]] :ValueNotSwapFrom INFO [2019-12-17 12:31:29,487] jepsen worker 70 - jepsen.util 5470 :invoke :read [23 nil] INFO [2019-12-17 12:31:29,488] jepsen worker 63 - jepsen.util 3933 :fail :cas [21 [5 28]] :ValueNotSwapFrom INFO [2019-12-17 12:31:29,488] jepsen worker 63 - jepsen.util 3933 :invoke :read [21 nil] INFO [2019-12-17 12:31:29,490] jepsen worker 27 - jepsen.util 4707 :ok :read [9 11] INFO [2019-12-17 12:31:29,490] jepsen worker 27 - jepsen.util 4707 :invoke :write [9 11] INFO [2019-12-17 12:31:29,492] jepsen worker 36 - jepsen.util 4806 :info :write [12 48] :RequestCanceledException INFO [2019-12-17 12:31:29,494] jepsen worker 36 - jepsen.util 4896 :invoke :write [12 18] INFO [2019-12-17 12:31:29,508] jepsen worker 63 - jepsen.util 3933 :ok :read [21 13] INFO [2019-12-17 12:31:29,508] jepsen worker 63 - jepsen.util 3933 :invoke :read [21 nil] INFO [2019-12-17 12:31:29,512] jepsen worker 89 - jepsen.util 4409 :ok :read [29 49] INFO [2019-12-17 12:31:29,512] jepsen worker 89 - jepsen.util 4409 :invoke :cas [29 [25 1]] INFO [2019-12-17 12:31:29,516] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":27548,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:31:29,516] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":27548,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:31:29,520] jepsen worker 87 - jepsen.util 4587 :fail :cas [29 [18 35]] :ValueNotSwapFrom INFO [2019-12-17 12:31:29,520] jepsen worker 87 - jepsen.util 4587 :invoke :write [29 7] INFO [2019-12-17 12:31:29,526] jepsen worker 54 - jepsen.util 5184 :info :write [18 32] :RequestCanceledException INFO [2019-12-17 12:31:29,527] jepsen worker 54 - jepsen.util 5274 :invoke :cas [18 [48 7]] INFO [2019-12-17 12:31:29,528] jepsen worker 52 - jepsen.util 5182 :fail :cas [17 [48 6]] :ValueNotSwapFrom INFO [2019-12-17 12:31:29,528] jepsen worker 52 - jepsen.util 5182 :invoke :cas [17 [49 9]] INFO [2019-12-17 12:31:29,529] jepsen worker 63 - jepsen.util 3933 :ok :read [21 13] INFO [2019-12-17 12:31:29,529] jepsen worker 63 - jepsen.util 3933 :invoke :cas [21 [40 12]] INFO [2019-12-17 12:31:29,545] jepsen worker 63 - jepsen.util 3933 :fail :cas [21 [40 12]] :ValueNotSwapFrom INFO [2019-12-17 12:31:29,545] jepsen worker 63 - jepsen.util 3933 :invoke :write [21 19] INFO [2019-12-17 12:31:29,548] jepsen worker 27 - jepsen.util 4707 :ok :write [9 11] INFO [2019-12-17 12:31:29,548] jepsen worker 27 - jepsen.util 4707 :invoke :write [9 35] INFO [2019-12-17 12:31:29,566] jepsen worker 13 - jepsen.util 4423 :ok :read [4 48] INFO [2019-12-17 12:31:29,567] jepsen worker 13 - jepsen.util 4423 :invoke :write [4 20] INFO [2019-12-17 12:31:29,579] jepsen worker 63 - jepsen.util 3933 :ok :write [21 19] INFO [2019-12-17 12:31:29,579] jepsen worker 63 - jepsen.util 3933 :invoke :write [21 44] INFO [2019-12-17 12:31:29,598] jepsen worker 12 - jepsen.util 4962 :ok :read [4 48] INFO [2019-12-17 12:31:29,598] jepsen worker 12 - jepsen.util 4962 :invoke :read [4 nil] INFO [2019-12-17 12:31:29,618] jepsen worker 89 - jepsen.util 4409 :fail :cas [29 [25 1]] :ValueNotSwapFrom INFO [2019-12-17 12:31:29,618] jepsen worker 89 - jepsen.util 4409 :invoke :write [29 24] INFO [2019-12-17 12:31:29,623] jepsen worker 27 - jepsen.util 4707 :ok :write [9 35] INFO [2019-12-17 12:31:29,623] jepsen worker 27 - jepsen.util 4707 :invoke :read [9 nil] INFO [2019-12-17 12:31:29,633] jepsen worker 63 - jepsen.util 3933 :ok :write [21 44] INFO [2019-12-17 12:31:29,634] jepsen worker 63 - jepsen.util 3933 :invoke :read [21 nil] INFO [2019-12-17 12:31:29,636] jepsen worker 88 - jepsen.util 4948 :fail :cas [29 [46 30]] :ValueNotSwapFrom INFO [2019-12-17 12:31:29,636] jepsen worker 88 - jepsen.util 4948 :invoke :write [29 6] INFO [2019-12-17 12:31:29,639] jepsen worker 27 - jepsen.util 4707 :ok :read [9 35] INFO [2019-12-17 12:31:29,639] jepsen worker 27 - jepsen.util 4707 :invoke :cas [9 [4 49]] INFO [2019-12-17 12:31:29,655] jepsen worker 12 - jepsen.util 4962 :ok :read [4 48] INFO [2019-12-17 12:31:29,656] jepsen worker 12 - jepsen.util 4962 :invoke :read [4 nil] INFO [2019-12-17 12:31:29,665] jepsen worker 63 - jepsen.util 3933 :ok :read [21 44] INFO [2019-12-17 12:31:29,665] jepsen worker 63 - jepsen.util 3933 :invoke :write [21 45] INFO [2019-12-17 12:31:29,669] jepsen worker 27 - jepsen.util 4707 :fail :cas [9 [4 49]] :ValueNotSwapFrom INFO [2019-12-17 12:31:29,669] jepsen worker 27 - jepsen.util 4707 :invoke :cas [9 [21 9]] INFO [2019-12-17 12:31:29,691] jepsen worker 70 - jepsen.util 5470 :ok :read [23 32] INFO [2019-12-17 12:31:29,691] jepsen worker 70 - jepsen.util 5470 :invoke :write [23 17] INFO [2019-12-17 12:31:29,694] jepsen worker 27 - jepsen.util 4707 :fail :cas [9 [21 9]] :ValueNotSwapFrom INFO [2019-12-17 12:31:29,695] jepsen worker 27 - jepsen.util 4707 :invoke :write [9 6] INFO [2019-12-17 12:31:29,703] jepsen worker 52 - jepsen.util 5182 :fail :cas [17 [49 9]] :ValueNotSwapFrom INFO [2019-12-17 12:31:29,703] jepsen worker 52 - jepsen.util 5182 :invoke :read [17 nil] INFO [2019-12-17 12:31:29,711] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":27577,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:31:29,711] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":27577,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:31:29,711] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":27580,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:31:29,711] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":27580,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:31:29,725] jepsen worker 32 - jepsen.util 4892 :info :write [10 39] :RequestCanceledException INFO [2019-12-17 12:31:29,725] jepsen worker 29 - jepsen.util 3809 :info :cas [9 [26 19]] :RequestCanceledException INFO [2019-12-17 12:31:29,726] jepsen worker 32 - jepsen.util 4982 :invoke :write [10 29] INFO [2019-12-17 12:31:29,727] jepsen worker 29 - jepsen.util 3899 :invoke :cas [9 [45 46]] INFO [2019-12-17 12:31:29,738] jepsen worker 63 - jepsen.util 3933 :ok :write [21 45] INFO [2019-12-17 12:31:29,739] jepsen worker 63 - jepsen.util 3933 :invoke :write [21 29] INFO [2019-12-17 12:31:29,739] jepsen worker 56 - jepsen.util 4916 :info :write [18 3] :RequestCanceledException INFO [2019-12-17 12:31:29,740] jepsen worker 56 - jepsen.util 5006 :invoke :write [18 34] INFO [2019-12-17 12:31:29,749] jepsen worker 27 - jepsen.util 4707 :ok :write [9 6] INFO [2019-12-17 12:31:29,750] jepsen worker 27 - jepsen.util 4707 :invoke :write [9 4] INFO [2019-12-17 12:31:29,794] jepsen worker 63 - jepsen.util 3933 :ok :write [21 29] INFO [2019-12-17 12:31:29,794] jepsen worker 63 - jepsen.util 3933 :invoke :write [21 6] INFO [2019-12-17 12:31:29,834] jepsen worker 69 - jepsen.util 5019 :fail :cas [23 [8 23]] :ValueNotSwapFrom INFO [2019-12-17 12:31:29,834] jepsen worker 69 - jepsen.util 5019 :invoke :write [23 6] INFO [2019-12-17 12:31:29,862] jepsen worker 54 - jepsen.util 5274 :fail :cas [18 [48 7]] :ValueNotSwapFrom INFO [2019-12-17 12:31:29,862] jepsen worker 54 - jepsen.util 5274 :invoke :write [18 17] INFO [2019-12-17 12:31:29,880] jepsen worker 63 - jepsen.util 3933 :ok :write [21 6] INFO [2019-12-17 12:31:29,880] jepsen worker 63 - jepsen.util 3933 :invoke :cas [21 [17 7]] INFO [2019-12-17 12:31:29,889] jepsen worker 27 - jepsen.util 4707 :ok :write [9 4] INFO [2019-12-17 12:31:29,890] jepsen worker 27 - jepsen.util 4707 :invoke :cas [9 [21 39]] INFO [2019-12-17 12:31:29,910] jepsen worker 4 - jepsen.util 3874 :info :write [1 3] :RequestCanceledException INFO [2019-12-17 12:31:29,911] jepsen worker 4 - jepsen.util 3964 :invoke :read [1 nil] INFO [2019-12-17 12:31:29,920] jepsen worker 27 - jepsen.util 4707 :fail :cas [9 [21 39]] :ValueNotSwapFrom INFO [2019-12-17 12:31:29,921] jepsen worker 27 - jepsen.util 4707 :invoke :cas [9 [2 26]] INFO [2019-12-17 12:31:29,926] jepsen worker 63 - jepsen.util 3933 :fail :cas [21 [17 7]] :ValueNotSwapFrom INFO [2019-12-17 12:31:29,926] jepsen worker 63 - jepsen.util 3933 :invoke :read [21 nil] INFO [2019-12-17 12:31:29,929] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":27596,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:31:29,929] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":27596,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:31:29,929] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":27600,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:31:29,929] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":27600,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:31:29,947] jepsen worker 27 - jepsen.util 4707 :fail :cas [9 [2 26]] :ValueNotSwapFrom INFO [2019-12-17 12:31:29,947] jepsen worker 27 - jepsen.util 4707 :invoke :write [9 15] INFO [2019-12-17 12:31:29,948] jepsen worker 63 - jepsen.util 3933 :ok :read [21 6] INFO [2019-12-17 12:31:29,948] jepsen worker 63 - jepsen.util 3933 :invoke :write [21 38] INFO [2019-12-17 12:31:29,970] jepsen worker 52 - jepsen.util 5182 :ok :read [17 47] INFO [2019-12-17 12:31:29,970] jepsen worker 52 - jepsen.util 5182 :invoke :write [17 13] INFO [2019-12-17 12:31:30,009] jepsen worker 63 - jepsen.util 3933 :ok :write [21 38] INFO [2019-12-17 12:31:30,010] jepsen worker 63 - jepsen.util 3933 :invoke :cas [21 [47 7]] INFO [2019-12-17 12:31:30,026] jepsen worker 27 - jepsen.util 4707 :ok :write [9 15] INFO [2019-12-17 12:31:30,027] jepsen worker 27 - jepsen.util 4707 :invoke :read [9 nil] INFO [2019-12-17 12:31:30,042] jepsen worker 63 - jepsen.util 3933 :fail :cas [21 [47 7]] :ValueNotSwapFrom INFO [2019-12-17 12:31:30,042] jepsen worker 63 - jepsen.util 3933 :invoke :write [21 8] INFO [2019-12-17 12:31:30,043] jepsen worker 27 - jepsen.util 4707 :ok :read [9 15] INFO [2019-12-17 12:31:30,043] jepsen worker 27 - jepsen.util 4707 :invoke :write [9 13] INFO [2019-12-17 12:31:30,094] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":27611,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:31:30,094] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":27611,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:31:30,094] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":27614,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:31:30,094] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":27614,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:31:30,094] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":27615,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:31:30,094] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":27615,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:31:30,120] jepsen worker 12 - jepsen.util 4962 :ok :read [4 48] INFO [2019-12-17 12:31:30,120] jepsen worker 12 - jepsen.util 4962 :invoke :cas [4 [6 5]] INFO [2019-12-17 12:31:30,138] jepsen worker 79 - jepsen.util 4669 :info :write [26 16] :RequestCanceledException INFO [2019-12-17 12:31:30,138] jepsen worker 79 - jepsen.util 4759 :invoke :cas [26 [13 8]] INFO [2019-12-17 12:31:30,139] jepsen worker 27 - jepsen.util 4707 :ok :write [9 13] INFO [2019-12-17 12:31:30,139] jepsen worker 27 - jepsen.util 4707 :invoke :read [9 nil] INFO [2019-12-17 12:31:30,145] jepsen worker 63 - jepsen.util 3933 :ok :write [21 8] INFO [2019-12-17 12:31:30,145] jepsen worker 63 - jepsen.util 3933 :invoke :cas [21 [6 33]] INFO [2019-12-17 12:31:30,150] jepsen worker 27 - jepsen.util 4707 :ok :read [9 13] INFO [2019-12-17 12:31:30,150] jepsen worker 27 - jepsen.util 4707 :invoke :read [9 nil] INFO [2019-12-17 12:31:30,156] jepsen worker 68 - jepsen.util 5918 :info :write [22 25] :RequestCanceledException INFO [2019-12-17 12:31:30,158] jepsen worker 68 - jepsen.util 6008 :invoke :read [22 nil] INFO [2019-12-17 12:31:30,177] jepsen worker 79 - jepsen.util 4759 :fail :cas [26 [13 8]] :ValueNotSwapFrom INFO [2019-12-17 12:31:30,178] jepsen worker 79 - jepsen.util 4759 :invoke :cas [26 [41 1]] INFO [2019-12-17 12:31:30,184] jepsen worker 27 - jepsen.util 4707 :ok :read [9 13] INFO [2019-12-17 12:31:30,184] jepsen worker 63 - jepsen.util 3933 :fail :cas [21 [6 33]] :ValueNotSwapFrom INFO [2019-12-17 12:31:30,184] jepsen worker 63 - jepsen.util 3933 :invoke :cas [21 [33 47]] INFO [2019-12-17 12:31:30,184] jepsen worker 27 - jepsen.util 4707 :invoke :write [9 48] INFO [2019-12-17 12:31:30,192] jepsen worker 26 - jepsen.util 6686 :info :cas [8 [29 21]] :RequestCanceledException INFO [2019-12-17 12:31:30,193] jepsen worker 26 - jepsen.util 6776 :invoke :write [8 7] INFO [2019-12-17 12:31:30,204] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":27629,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:31:30,204] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":27629,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:31:30,204] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":27631,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:31:30,204] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":27631,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:31:30,208] jepsen worker 82 - jepsen.util 7192 :info :write [27 5] :RequestCanceledException INFO [2019-12-17 12:31:30,209] jepsen worker 82 - jepsen.util 7282 :invoke :read [27 nil] INFO [2019-12-17 12:31:30,210] jepsen worker 63 - jepsen.util 3933 :fail :cas [21 [33 47]] :ValueNotSwapFrom INFO [2019-12-17 12:31:30,210] jepsen worker 63 - jepsen.util 3933 :invoke :write [21 12] INFO [2019-12-17 12:31:30,226] jepsen worker 16 - jepsen.util 7396 :info :cas [5 [41 3]] :RequestCanceledException INFO [2019-12-17 12:31:30,226] jepsen worker 84 - jepsen.util 6024 :info :write [28 39] :RequestCanceledException INFO [2019-12-17 12:31:30,226] jepsen worker 76 - jepsen.util 6466 :info :write [25 18] :RequestCanceledException INFO [2019-12-17 12:31:30,230] jepsen worker 16 - jepsen.util 7486 :invoke :write [5 18] INFO [2019-12-17 12:31:30,230] jepsen worker 76 - jepsen.util 6556 :invoke :read [25 nil] INFO [2019-12-17 12:31:30,230] jepsen worker 84 - jepsen.util 6114 :invoke :write [28 8] INFO [2019-12-17 12:31:30,235] jepsen worker 27 - jepsen.util 4707 :ok :write [9 48] INFO [2019-12-17 12:31:30,235] jepsen worker 27 - jepsen.util 4707 :invoke :write [9 22] INFO [2019-12-17 12:31:30,239] jepsen worker 81 - jepsen.util 6201 :info :write [27 42] :RequestCanceledException INFO [2019-12-17 12:31:30,241] jepsen worker 81 - jepsen.util 6291 :invoke :cas [27 [39 20]] INFO [2019-12-17 12:31:30,256] jepsen worker 57 - jepsen.util 6087 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:31:30,256] jepsen worker 28 - jepsen.util 3538 :fail :read [9 nil] :RequestTimeoutException INFO [2019-12-17 12:31:30,256] jepsen worker 72 - jepsen.util 4842 :fail :read [24 nil] :RequestTimeoutException INFO [2019-12-17 12:31:30,257] jepsen worker 33 - jepsen.util 6243 :info :write [11 38] :RequestCanceledException INFO [2019-12-17 12:31:30,257] jepsen worker 57 - jepsen.util 6087 :invoke :read [19 nil] INFO [2019-12-17 12:31:30,257] jepsen worker 28 - jepsen.util 3538 :invoke :read [9 nil] INFO [2019-12-17 12:31:30,257] jepsen worker 72 - jepsen.util 4842 :invoke :cas [24 [13 10]] INFO [2019-12-17 12:31:30,258] jepsen worker 33 - jepsen.util 6333 :invoke :read [11 nil] INFO [2019-12-17 12:31:30,285] jepsen worker 63 - jepsen.util 3933 :ok :write [21 12] INFO [2019-12-17 12:31:30,286] jepsen worker 63 - jepsen.util 3933 :invoke :read [21 nil] INFO [2019-12-17 12:31:30,292] jepsen worker 30 - jepsen.util 4890 :ok :write [10 49] INFO [2019-12-17 12:31:30,292] jepsen worker 6 - jepsen.util 6846 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:31:30,292] jepsen worker 20 - jepsen.util 4880 :ok :write [6 32] INFO [2019-12-17 12:31:30,292] jepsen worker 30 - jepsen.util 4890 :invoke :read [10 nil] INFO [2019-12-17 12:31:30,292] jepsen worker 6 - jepsen.util 6846 :invoke :read [2 nil] INFO [2019-12-17 12:31:30,292] jepsen worker 74 - jepsen.util 5114 :ok :write [24 8] INFO [2019-12-17 12:31:30,292] jepsen worker 20 - jepsen.util 4880 :invoke :write [6 39] INFO [2019-12-17 12:31:30,292] jepsen worker 74 - jepsen.util 5114 :invoke :read [24 nil] INFO [2019-12-17 12:31:30,293] jepsen worker 12 - jepsen.util 4962 :fail :cas [4 [6 5]] :ValueNotSwapFrom INFO [2019-12-17 12:31:30,294] jepsen worker 12 - jepsen.util 4962 :invoke :write [4 2] INFO [2019-12-17 12:31:30,299] jepsen worker 63 - jepsen.util 3933 :ok :read [21 12] INFO [2019-12-17 12:31:30,299] jepsen worker 63 - jepsen.util 3933 :invoke :read [21 nil] INFO [2019-12-17 12:31:30,299] jepsen worker 27 - jepsen.util 4707 :ok :write [9 22] INFO [2019-12-17 12:31:30,299] jepsen worker 27 - jepsen.util 4707 :invoke :read [9 nil] INFO [2019-12-17 12:31:30,308] jepsen worker 46 - jepsen.util 4816 :ok :write [15 17] INFO [2019-12-17 12:31:30,308] jepsen worker 46 - jepsen.util 4816 :invoke :cas [15 [19 1]] INFO [2019-12-17 12:31:30,309] jepsen worker 49 - jepsen.util 5719 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:31:30,309] jepsen worker 43 - jepsen.util 6073 :info :cas [14 [12 2]] :RequestCanceledException INFO [2019-12-17 12:31:30,309] jepsen worker 17 - jepsen.util 6317 :info :write [5 10] :RequestCanceledException INFO [2019-12-17 12:31:30,309] jepsen worker 34 - jepsen.util 5884 :info :cas [11 [42 37]] :RequestCanceledException INFO [2019-12-17 12:31:30,310] jepsen worker 49 - jepsen.util 5719 :invoke :cas [16 [45 29]] INFO [2019-12-17 12:31:30,311] jepsen worker 17 - jepsen.util 6407 :invoke :read [5 nil] INFO [2019-12-17 12:31:30,311] jepsen worker 34 - jepsen.util 5974 :invoke :read [11 nil] INFO [2019-12-17 12:31:30,311] jepsen worker 43 - jepsen.util 6163 :invoke :read [14 nil] INFO [2019-12-17 12:31:30,313] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":27637,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:31:30,313] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":27637,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:31:30,314] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":27641,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:31:30,314] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":27641,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:31:30,325] jepsen worker 66 - jepsen.util 5916 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:31:30,325] jepsen worker 83 - jepsen.util 6833 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:31:30,325] jepsen worker 40 - jepsen.util 7150 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:31:30,325] jepsen worker 58 - jepsen.util 6538 :info :cas [19 [14 13]] :RequestCanceledException INFO [2019-12-17 12:31:30,325] jepsen worker 25 - jepsen.util 6505 :info :write [8 21] :RequestCanceledException INFO [2019-12-17 12:31:30,325] jepsen worker 8 - jepsen.util 5678 :info :cas [2 [42 29]] :RequestCanceledException INFO [2019-12-17 12:31:30,326] jepsen worker 77 - jepsen.util 6647 :info :cas [25 [41 16]] :RequestCanceledException INFO [2019-12-17 12:31:30,326] jepsen worker 66 - jepsen.util 5916 :invoke :read [22 nil] INFO [2019-12-17 12:31:30,326] jepsen worker 83 - jepsen.util 6833 :invoke :write [27 13] INFO [2019-12-17 12:31:30,326] jepsen worker 40 - jepsen.util 7150 :invoke :write [13 12] INFO [2019-12-17 12:31:30,327] jepsen worker 8 - jepsen.util 5768 :invoke :write [2 37] INFO [2019-12-17 12:31:30,327] jepsen worker 77 - jepsen.util 6737 :invoke :write [25 45] INFO [2019-12-17 12:31:30,327] jepsen worker 58 - jepsen.util 6628 :invoke :write [19 42] INFO [2019-12-17 12:31:30,327] jepsen worker 25 - jepsen.util 6595 :invoke :cas [8 [8 2]] INFO [2019-12-17 12:31:30,334] jepsen worker 28 - jepsen.util 3538 :ok :read [9 48] INFO [2019-12-17 12:31:30,334] jepsen worker 28 - jepsen.util 3538 :invoke :cas [9 [35 21]] INFO [2019-12-17 12:31:30,339] jepsen worker 15 - jepsen.util 5415 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:31:30,339] jepsen worker 48 - jepsen.util 5808 :info :cas [16 [8 12]] :RequestCanceledException INFO [2019-12-17 12:31:30,339] jepsen worker 85 - jepsen.util 7195 :info :cas [28 [4 17]] :RequestCanceledException INFO [2019-12-17 12:31:30,339] jepsen worker 15 - jepsen.util 5415 :invoke :cas [5 [6 19]] INFO [2019-12-17 12:31:30,340] jepsen worker 48 - jepsen.util 5898 :invoke :cas [16 [20 47]] INFO [2019-12-17 12:31:30,340] jepsen worker 85 - jepsen.util 7285 :invoke :cas [28 [2 34]] INFO [2019-12-17 12:31:30,343] jepsen worker 73 - jepsen.util 4573 :fail :cas [24 [35 47]] :ValueNotSwapFrom INFO [2019-12-17 12:31:30,343] jepsen worker 73 - jepsen.util 4573 :invoke :read [24 nil] INFO [2019-12-17 12:31:30,344] jepsen worker 27 - jepsen.util 4707 :ok :read [9 22] INFO [2019-12-17 12:31:30,344] jepsen worker 27 - jepsen.util 4707 :invoke :write [9 2] INFO [2019-12-17 12:31:30,357] jepsen worker 62 - jepsen.util 5912 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:31:30,357] jepsen worker 50 - jepsen.util 6710 :info :cas [16 [0 35]] :RequestCanceledException INFO [2019-12-17 12:31:30,357] jepsen worker 39 - jepsen.util 6069 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:31:30,357] jepsen worker 62 - jepsen.util 5912 :invoke :cas [20 [36 28]] INFO [2019-12-17 12:31:30,357] jepsen worker 39 - jepsen.util 6069 :invoke :read [13 nil] INFO [2019-12-17 12:31:30,358] jepsen worker 50 - jepsen.util 6800 :invoke :write [16 8] INFO [2019-12-17 12:31:30,391] jepsen worker 45 - jepsen.util 4905 :info :write [15 29] :RequestCanceledException INFO [2019-12-17 12:31:30,391] jepsen worker 1 - jepsen.util 6121 :info :write [0 12] :RequestCanceledException INFO [2019-12-17 12:31:30,391] jepsen worker 65 - jepsen.util 4115 :info :cas [21 [19 3]] :RequestCanceledException INFO [2019-12-17 12:31:30,391] jepsen worker 75 - jepsen.util 6375 :info :write [25 43] :RequestCanceledException INFO [2019-12-17 12:31:30,392] jepsen worker 65 - jepsen.util 4205 :invoke :read [21 nil] INFO [2019-12-17 12:31:30,392] jepsen worker 1 - jepsen.util 6211 :invoke :cas [0 [3 48]] INFO [2019-12-17 12:31:30,392] jepsen worker 45 - jepsen.util 4995 :invoke :cas [15 [15 6]] INFO [2019-12-17 12:31:30,393] jepsen worker 75 - jepsen.util 6465 :invoke :read [25 nil] INFO [2019-12-17 12:31:30,405] jepsen worker 72 - jepsen.util 4842 :fail :cas [24 [13 10]] :ValueNotSwapFrom INFO [2019-12-17 12:31:30,405] jepsen worker 72 - jepsen.util 4842 :invoke :write [24 43] INFO [2019-12-17 12:31:30,410] jepsen worker 60 - jepsen.util 6900 :info :cas [20 [1 9]] :RequestCanceledException INFO [2019-12-17 12:31:30,410] jepsen worker 42 - jepsen.util 6882 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:31:30,410] jepsen worker 2 - jepsen.util 5942 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:31:30,410] jepsen worker 22 - jepsen.util 6502 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:31:30,411] jepsen worker 2 - jepsen.util 5942 :invoke :read [0 nil] INFO [2019-12-17 12:31:30,411] jepsen worker 42 - jepsen.util 6882 :invoke :write [14 1] INFO [2019-12-17 12:31:30,411] jepsen worker 22 - jepsen.util 6502 :invoke :cas [7 [16 20]] INFO [2019-12-17 12:31:30,411] jepsen worker 60 - jepsen.util 6990 :invoke :cas [20 [38 12]] INFO [2019-12-17 12:31:30,424] jepsen worker 24 - jepsen.util 5964 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:31:30,424] jepsen worker 7 - jepsen.util 6487 :info :cas [2 [46 8]] :RequestCanceledException INFO [2019-12-17 12:31:30,424] jepsen worker 0 - jepsen.util 7020 :info :cas [0 [25 20]] :RequestCanceledException INFO [2019-12-17 12:31:30,424] jepsen worker 67 - jepsen.util 6637 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:31:30,425] jepsen worker 24 - jepsen.util 5964 :invoke :read [8 nil] INFO [2019-12-17 12:31:30,425] jepsen worker 67 - jepsen.util 6637 :invoke :read [22 nil] INFO [2019-12-17 12:31:30,425] jepsen worker 7 - jepsen.util 6577 :invoke :write [2 45] INFO [2019-12-17 12:31:30,425] jepsen worker 0 - jepsen.util 7110 :invoke :read [0 nil] INFO [2019-12-17 12:31:30,439] jepsen worker 80 - jepsen.util 4760 :info :write [26 44] :RequestCanceledException INFO [2019-12-17 12:31:30,439] jepsen worker 44 - jepsen.util 6974 :info :write [14 44] :RequestCanceledException INFO [2019-12-17 12:31:30,440] jepsen worker 80 - jepsen.util 4850 :invoke :cas [26 [0 21]] INFO [2019-12-17 12:31:30,440] jepsen worker 44 - jepsen.util 7064 :invoke :cas [14 [16 1]] INFO [2019-12-17 12:31:30,444] jepsen worker 31 - jepsen.util 4711 :ok :read [10 49] INFO [2019-12-17 12:31:30,444] jepsen worker 31 - jepsen.util 4711 :invoke :cas [10 [1 6]] INFO [2019-12-17 12:31:30,454] jepsen worker 9 - jepsen.util 5679 :ok :read [3 20] INFO [2019-12-17 12:31:30,455] jepsen worker 9 - jepsen.util 5679 :invoke :cas [3 [21 7]] INFO [2019-12-17 12:31:30,456] jepsen worker 41 - jepsen.util 6881 :info :write [13 5] :RequestCanceledException INFO [2019-12-17 12:31:30,457] jepsen worker 41 - jepsen.util 6971 :invoke :read [13 nil] INFO [2019-12-17 12:31:30,487] jepsen worker 35 - jepsen.util 6515 :info :write [11 30] :RequestCanceledException INFO [2019-12-17 12:31:30,487] jepsen worker 53 - jepsen.util 4733 :info :write [17 14] :RequestCanceledException INFO [2019-12-17 12:31:30,488] jepsen worker 53 - jepsen.util 4823 :invoke :read [17 nil] INFO [2019-12-17 12:31:30,488] jepsen worker 35 - jepsen.util 6605 :invoke :read [11 nil] INFO [2019-12-17 12:31:30,488] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:31:30,488] jepsen nemesis - couchbase.util Rebalance check count: 207 INFO [2019-12-17 12:31:30,488] jepsen nemesis - couchbase.util Rebalance stuck count: 188 INFO [2019-12-17 12:31:30,489] jepsen worker 63 - jepsen.util 3933 :ok :read [21 12] INFO [2019-12-17 12:31:30,489] jepsen worker 63 - jepsen.util 3933 :invoke :cas [21 [0 16]] INFO [2019-12-17 12:31:30,493] jepsen worker 30 - jepsen.util 4890 :ok :read [10 49] INFO [2019-12-17 12:31:30,493] jepsen worker 30 - jepsen.util 4890 :invoke :write [10 11] INFO [2019-12-17 12:31:30,501] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":27694,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:31:30,502] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":27694,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:31:30,502] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":27698,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:31:30,502] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":27698,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:31:30,502] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":27699,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:31:30,502] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":27699,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:31:30,519] jepsen worker 74 - jepsen.util 5114 :ok :read [24 8] INFO [2019-12-17 12:31:30,519] jepsen worker 74 - jepsen.util 5114 :invoke :cas [24 [45 38]] INFO [2019-12-17 12:31:30,525] jepsen worker 86 - jepsen.util 6746 :info :cas [28 [42 45]] :RequestCanceledException INFO [2019-12-17 12:31:30,525] jepsen worker 86 - jepsen.util 6836 :invoke :write [28 33] INFO [2019-12-17 12:31:30,539] jepsen worker 64 - jepsen.util 3664 :fail :read [21 nil] :RequestTimeoutException INFO [2019-12-17 12:31:30,539] jepsen worker 64 - jepsen.util 3664 :invoke :read [21 nil] INFO [2019-12-17 12:31:30,541] jepsen worker 29 - jepsen.util 3899 :fail :cas [9 [45 46]] :ValueNotSwapFrom INFO [2019-12-17 12:31:30,541] jepsen worker 29 - jepsen.util 3899 :invoke :read [9 nil] INFO [2019-12-17 12:31:30,543] jepsen worker 79 - jepsen.util 4759 :fail :cas [26 [41 1]] :ValueNotSwapFrom INFO [2019-12-17 12:31:30,543] jepsen worker 79 - jepsen.util 4759 :invoke :cas [26 [0 1]] INFO [2019-12-17 12:31:30,562] jepsen worker 28 - jepsen.util 3538 :fail :cas [9 [35 21]] :ValueNotSwapFrom INFO [2019-12-17 12:31:30,563] jepsen worker 28 - jepsen.util 3538 :invoke :cas [9 [2 37]] INFO [2019-12-17 12:31:30,572] jepsen worker 65 - jepsen.util 4205 :ok :read [21 12] INFO [2019-12-17 12:31:30,572] jepsen worker 65 - jepsen.util 4205 :invoke :cas [21 [30 26]] INFO [2019-12-17 12:31:30,584] jepsen worker 63 - jepsen.util 3933 :fail :cas [21 [0 16]] :ValueNotSwapFrom INFO [2019-12-17 12:31:30,584] jepsen worker 63 - jepsen.util 3933 :invoke :write [21 33] INFO [2019-12-17 12:31:30,586] jepsen worker 46 - jepsen.util 4816 :fail :cas [15 [19 1]] :ValueNotSwapFrom INFO [2019-12-17 12:31:30,586] jepsen worker 46 - jepsen.util 4816 :invoke :read [15 nil] INFO [2019-12-17 12:31:30,638] jepsen worker 45 - jepsen.util 4995 :fail :cas [15 [15 6]] :ValueNotSwapFrom INFO [2019-12-17 12:31:30,638] jepsen worker 45 - jepsen.util 4995 :invoke :write [15 35] INFO [2019-12-17 12:31:30,638] jepsen worker 31 - jepsen.util 4711 :fail :cas [10 [1 6]] :ValueNotSwapFrom INFO [2019-12-17 12:31:30,638] jepsen worker 31 - jepsen.util 4711 :invoke :cas [10 [26 23]] INFO [2019-12-17 12:31:30,638] jepsen worker 11 - jepsen.util 4691 :fail :read [3 nil] :RequestTimeoutException INFO [2019-12-17 12:31:30,638] jepsen worker 11 - jepsen.util 4691 :invoke :cas [3 [31 29]] INFO [2019-12-17 12:31:30,656] jepsen worker 3 - jepsen.util 4863 :info :write [1 13] :RequestCanceledException INFO [2019-12-17 12:31:30,656] jepsen worker 47 - jepsen.util 5357 :fail :read [15 nil] :RequestTimeoutException INFO [2019-12-17 12:31:30,656] jepsen worker 47 - jepsen.util 5357 :invoke :write [15 4] INFO [2019-12-17 12:31:30,657] jepsen worker 3 - jepsen.util 4953 :invoke :cas [1 [36 18]] INFO [2019-12-17 12:31:30,681] jepsen worker 80 - jepsen.util 4850 :fail :cas [26 [0 21]] :ValueNotSwapFrom INFO [2019-12-17 12:31:30,681] jepsen worker 80 - jepsen.util 4850 :invoke :write [26 43] INFO [2019-12-17 12:31:30,693] jepsen worker 4 - jepsen.util 3964 :ok :read [1 26] INFO [2019-12-17 12:31:30,693] jepsen worker 4 - jepsen.util 3964 :invoke :write [1 11] INFO [2019-12-17 12:31:30,694] jepsen worker 64 - jepsen.util 3664 :ok :read [21 12] INFO [2019-12-17 12:31:30,695] jepsen worker 64 - jepsen.util 3664 :invoke :read [21 nil] INFO [2019-12-17 12:31:30,698] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":27722,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:31:30,698] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":27722,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:31:30,698] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":27725,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:31:30,698] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":27725,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:31:30,698] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":27726,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:31:30,698] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":27726,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:31:30,701] jepsen worker 36 - jepsen.util 4896 :ok :write [12 18] INFO [2019-12-17 12:31:30,701] jepsen worker 36 - jepsen.util 4896 :invoke :read [12 nil] INFO [2019-12-17 12:31:30,713] jepsen worker 74 - jepsen.util 5114 :fail :cas [24 [45 38]] :ValueNotSwapFrom INFO [2019-12-17 12:31:30,713] jepsen worker 74 - jepsen.util 5114 :invoke :read [24 nil] INFO [2019-12-17 12:31:30,721] jepsen worker 9 - jepsen.util 5679 :fail :cas [3 [21 7]] :ValueNotSwapFrom INFO [2019-12-17 12:31:30,721] jepsen worker 9 - jepsen.util 5679 :invoke :cas [3 [17 10]] INFO [2019-12-17 12:31:30,727] jepsen worker 79 - jepsen.util 4759 :fail :cas [26 [0 1]] :ValueNotSwapFrom INFO [2019-12-17 12:31:30,727] jepsen worker 79 - jepsen.util 4759 :invoke :cas [26 [19 24]] INFO [2019-12-17 12:31:30,730] jepsen worker 27 - jepsen.util 4707 :ok :write [9 2] INFO [2019-12-17 12:31:30,730] jepsen worker 27 - jepsen.util 4707 :invoke :read [9 nil] INFO [2019-12-17 12:31:30,739] jepsen worker 19 - jepsen.util 4789 :fail :read [6 nil] :RequestTimeoutException INFO [2019-12-17 12:31:30,739] jepsen worker 18 - jepsen.util 4698 :info :cas [6 [17 13]] :RequestCanceledException INFO [2019-12-17 12:31:30,739] jepsen worker 19 - jepsen.util 4789 :invoke :read [6 nil] INFO [2019-12-17 12:31:30,740] jepsen worker 18 - jepsen.util 4788 :invoke :cas [6 [49 3]] INFO [2019-12-17 12:31:30,743] jepsen worker 27 - jepsen.util 4707 :ok :read [9 2] INFO [2019-12-17 12:31:30,743] jepsen worker 27 - jepsen.util 4707 :invoke :cas [9 [2 7]] INFO [2019-12-17 12:31:30,780] jepsen worker 65 - jepsen.util 4205 :fail :cas [21 [30 26]] :ValueNotSwapFrom INFO [2019-12-17 12:31:30,780] jepsen worker 65 - jepsen.util 4205 :invoke :write [21 27] INFO [2019-12-17 12:31:30,796] jepsen worker 36 - jepsen.util 4896 :ok :read [12 18] INFO [2019-12-17 12:31:30,796] jepsen worker 36 - jepsen.util 4896 :invoke :read [12 nil] INFO [2019-12-17 12:31:30,799] jepsen worker 38 - jepsen.util 4628 :fail :cas [12 [14 13]] :CASMismatchException INFO [2019-12-17 12:31:30,799] jepsen worker 38 - jepsen.util 4628 :invoke :cas [12 [31 47]] INFO [2019-12-17 12:31:30,822] jepsen worker 29 - jepsen.util 3899 :ok :read [9 22] INFO [2019-12-17 12:31:30,823] jepsen worker 29 - jepsen.util 3899 :invoke :write [9 17] INFO [2019-12-17 12:31:30,825] jepsen worker 21 - jepsen.util 5961 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:31:30,825] jepsen worker 51 - jepsen.util 4641 :info :write [17 8] :RequestCanceledException INFO [2019-12-17 12:31:30,825] jepsen worker 21 - jepsen.util 5961 :invoke :read [7 nil] INFO [2019-12-17 12:31:30,826] jepsen worker 51 - jepsen.util 4731 :invoke :read [17 nil] INFO [2019-12-17 12:31:30,839] jepsen worker 23 - jepsen.util 6593 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:31:30,839] jepsen worker 23 - jepsen.util 6593 :invoke :write [7 22] INFO [2019-12-17 12:31:30,853] jepsen worker 79 - jepsen.util 4759 :fail :cas [26 [19 24]] :ValueNotSwapFrom INFO [2019-12-17 12:31:30,853] jepsen worker 79 - jepsen.util 4759 :invoke :cas [26 [28 20]] INFO [2019-12-17 12:31:30,855] jepsen worker 61 - jepsen.util 7171 :info :cas [20 [9 42]] :RequestCanceledException INFO [2019-12-17 12:31:30,857] jepsen worker 61 - jepsen.util 7261 :invoke :write [20 34] INFO [2019-12-17 12:31:30,888] jepsen worker 31 - jepsen.util 4711 :fail :cas [10 [26 23]] :ValueNotSwapFrom INFO [2019-12-17 12:31:30,888] jepsen worker 31 - jepsen.util 4711 :invoke :read [10 nil] INFO [2019-12-17 12:31:30,898] jepsen worker 46 - jepsen.util 4816 :ok :read [15 17] INFO [2019-12-17 12:31:30,898] jepsen worker 46 - jepsen.util 4816 :invoke :write [15 32] INFO [2019-12-17 12:31:30,914] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":27751,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:31:30,914] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":27751,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:31:30,914] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":27754,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:31:30,914] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":27754,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:31:30,925] jepsen worker 59 - jepsen.util 6269 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:31:30,925] jepsen worker 59 - jepsen.util 6269 :invoke :write [19 48] INFO [2019-12-17 12:31:30,939] jepsen worker 63 - jepsen.util 3933 :ok :write [21 33] INFO [2019-12-17 12:31:30,939] jepsen worker 63 - jepsen.util 3933 :invoke :read [21 nil] INFO [2019-12-17 12:31:30,944] jepsen worker 74 - jepsen.util 5114 :ok :read [24 8] INFO [2019-12-17 12:31:30,944] jepsen worker 74 - jepsen.util 5114 :invoke :cas [24 [19 25]] INFO [2019-12-17 12:31:30,955] jepsen worker 37 - jepsen.util 4537 :ok :write [12 11] INFO [2019-12-17 12:31:30,955] jepsen worker 37 - jepsen.util 4537 :invoke :cas [12 [7 16]] INFO [2019-12-17 12:31:30,964] jepsen worker 27 - jepsen.util 4707 :ok :cas [9 [2 7]] INFO [2019-12-17 12:31:30,964] jepsen worker 27 - jepsen.util 4707 :invoke :read [9 nil] INFO [2019-12-17 12:31:30,983] jepsen worker 27 - jepsen.util 4707 :ok :read [9 7] INFO [2019-12-17 12:31:30,983] jepsen worker 27 - jepsen.util 4707 :invoke :cas [9 [28 4]] INFO [2019-12-17 12:31:30,995] jepsen worker 19 - jepsen.util 4789 :ok :read [6 32] INFO [2019-12-17 12:31:30,995] jepsen worker 19 - jepsen.util 4789 :invoke :write [6 27] INFO [2019-12-17 12:31:30,998] jepsen worker 80 - jepsen.util 4850 :ok :write [26 43] INFO [2019-12-17 12:31:30,998] jepsen worker 80 - jepsen.util 4850 :invoke :write [26 31] INFO [2019-12-17 12:31:31,001] jepsen worker 36 - jepsen.util 4896 :ok :read [12 18] INFO [2019-12-17 12:31:31,001] jepsen worker 36 - jepsen.util 4896 :invoke :cas [12 [43 45]] INFO [2019-12-17 12:31:31,020] jepsen worker 79 - jepsen.util 4759 :fail :cas [26 [28 20]] :ValueNotSwapFrom INFO [2019-12-17 12:31:31,020] jepsen worker 79 - jepsen.util 4759 :invoke :cas [26 [20 9]] INFO [2019-12-17 12:31:31,026] jepsen worker 70 - jepsen.util 5470 :ok :write [23 17] INFO [2019-12-17 12:31:31,027] jepsen worker 70 - jepsen.util 5470 :invoke :read [23 nil] INFO [2019-12-17 12:31:31,027] jepsen worker 78 - jepsen.util 4848 :ok :write [26 29] INFO [2019-12-17 12:31:31,027] jepsen worker 78 - jepsen.util 4848 :invoke :read [26 nil] INFO [2019-12-17 12:31:31,047] jepsen worker 36 - jepsen.util 4896 :fail :cas [12 [43 45]] :ValueNotSwapFrom INFO [2019-12-17 12:31:31,047] jepsen worker 36 - jepsen.util 4896 :invoke :cas [12 [34 42]] INFO [2019-12-17 12:31:31,053] jepsen worker 3 - jepsen.util 4953 :fail :cas [1 [36 18]] :ValueNotSwapFrom INFO [2019-12-17 12:31:31,053] jepsen worker 3 - jepsen.util 4953 :invoke :cas [1 [20 14]] INFO [2019-12-17 12:31:31,054] jepsen worker 52 - jepsen.util 5182 :ok :write [17 13] INFO [2019-12-17 12:31:31,054] jepsen worker 52 - jepsen.util 5182 :invoke :cas [17 [26 14]] INFO [2019-12-17 12:31:31,066] jepsen worker 79 - jepsen.util 4759 :fail :cas [26 [20 9]] :ValueNotSwapFrom INFO [2019-12-17 12:31:31,067] jepsen worker 79 - jepsen.util 4759 :invoke :write [26 45] INFO [2019-12-17 12:31:31,078] jepsen worker 63 - jepsen.util 3933 :ok :read [21 33] INFO [2019-12-17 12:31:31,079] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":27766,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:31:31,080] jepsen worker 63 - jepsen.util 3933 :invoke :read [21 nil] INFO [2019-12-17 12:31:31,080] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":27766,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:31:31,080] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":27769,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:31:31,080] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":27769,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:31:31,080] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":27770,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:31:31,080] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":27770,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:31:31,092] jepsen worker 27 - jepsen.util 4707 :fail :cas [9 [28 4]] :ValueNotSwapFrom INFO [2019-12-17 12:31:31,092] jepsen worker 27 - jepsen.util 4707 :invoke :read [9 nil] INFO [2019-12-17 12:31:31,093] jepsen worker 36 - jepsen.util 4896 :fail :cas [12 [34 42]] :ValueNotSwapFrom INFO [2019-12-17 12:31:31,093] jepsen worker 36 - jepsen.util 4896 :invoke :write [12 42] INFO [2019-12-17 12:31:31,109] jepsen worker 87 - jepsen.util 4587 :ok :write [29 7] INFO [2019-12-17 12:31:31,109] jepsen worker 87 - jepsen.util 4587 :invoke :read [29 nil] INFO [2019-12-17 12:31:31,117] jepsen worker 27 - jepsen.util 4707 :ok :read [9 7] INFO [2019-12-17 12:31:31,117] jepsen worker 27 - jepsen.util 4707 :invoke :write [9 6] INFO [2019-12-17 12:31:31,140] jepsen worker 65 - jepsen.util 4205 :ok :write [21 27] INFO [2019-12-17 12:31:31,140] jepsen worker 65 - jepsen.util 4205 :invoke :read [21 nil] INFO [2019-12-17 12:31:31,167] jepsen worker 31 - jepsen.util 4711 :ok :read [10 49] INFO [2019-12-17 12:31:31,167] jepsen worker 31 - jepsen.util 4711 :invoke :cas [10 [16 46]] INFO [2019-12-17 12:31:31,190] jepsen worker 74 - jepsen.util 5114 :fail :cas [24 [19 25]] :ValueNotSwapFrom INFO [2019-12-17 12:31:31,190] jepsen worker 74 - jepsen.util 5114 :invoke :write [24 29] INFO [2019-12-17 12:31:31,192] jepsen worker 14 - jepsen.util 4964 :info :write [4 31] :RequestCanceledException INFO [2019-12-17 12:31:31,192] jepsen worker 14 - jepsen.util 5054 :invoke :read [4 nil] INFO [2019-12-17 12:31:31,211] jepsen worker 65 - jepsen.util 4205 :ok :read [21 27] INFO [2019-12-17 12:31:31,211] jepsen worker 65 - jepsen.util 4205 :invoke :write [21 43] INFO [2019-12-17 12:31:31,227] jepsen worker 31 - jepsen.util 4711 :fail :cas [10 [16 46]] :ValueNotSwapFrom INFO [2019-12-17 12:31:31,227] jepsen worker 31 - jepsen.util 4711 :invoke :cas [10 [32 0]] INFO [2019-12-17 12:31:31,236] jepsen worker 27 - jepsen.util 4707 :ok :write [9 6] INFO [2019-12-17 12:31:31,236] jepsen worker 27 - jepsen.util 4707 :invoke :read [9 nil] INFO [2019-12-17 12:31:31,243] jepsen worker 37 - jepsen.util 4537 :fail :cas [12 [7 16]] :ValueNotSwapFrom INFO [2019-12-17 12:31:31,243] jepsen worker 37 - jepsen.util 4537 :invoke :write [12 16] INFO [2019-12-17 12:31:31,263] jepsen worker 14 - jepsen.util 5054 :ok :read [4 48] INFO [2019-12-17 12:31:31,263] jepsen worker 14 - jepsen.util 5054 :invoke :write [4 20] INFO [2019-12-17 12:31:31,284] jepsen worker 87 - jepsen.util 4587 :ok :read [29 7] INFO [2019-12-17 12:31:31,284] jepsen worker 87 - jepsen.util 4587 :invoke :read [29 nil] INFO [2019-12-17 12:31:31,297] jepsen worker 31 - jepsen.util 4711 :fail :cas [10 [32 0]] :ValueNotSwapFrom INFO [2019-12-17 12:31:31,297] jepsen worker 31 - jepsen.util 4711 :invoke :write [10 9] INFO [2019-12-17 12:31:31,298] jepsen worker 27 - jepsen.util 4707 :ok :read [9 6] INFO [2019-12-17 12:31:31,298] jepsen worker 27 - jepsen.util 4707 :invoke :cas [9 [21 42]] INFO [2019-12-17 12:31:31,299] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":27789,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:31:31,299] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":27797,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:31:31,299] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":27789,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:31:31,299] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":27790,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:31:31,299] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":27797,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:31:31,299] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":27790,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:31:31,299] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":27800,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:31:31,299] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":27800,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:31:31,318] jepsen worker 27 - jepsen.util 4707 :fail :cas [9 [21 42]] :ValueNotSwapFrom INFO [2019-12-17 12:31:31,318] jepsen worker 27 - jepsen.util 4707 :invoke :write [9 9] INFO [2019-12-17 12:31:31,339] jepsen worker 55 - jepsen.util 5725 :info :write [18 7] :RequestCanceledException INFO [2019-12-17 12:31:31,340] jepsen worker 53 - jepsen.util 4823 :ok :read [17 13] INFO [2019-12-17 12:31:31,340] jepsen worker 53 - jepsen.util 4823 :invoke :write [17 32] INFO [2019-12-17 12:31:31,340] jepsen worker 55 - jepsen.util 5815 :invoke :read [18 nil] INFO [2019-12-17 12:31:31,357] jepsen worker 10 - jepsen.util 4780 :info :cas [3 [14 2]] :RequestCanceledException INFO [2019-12-17 12:31:31,358] jepsen worker 10 - jepsen.util 4870 :invoke :read [3 nil] INFO [2019-12-17 12:31:31,358] jepsen worker 70 - jepsen.util 5470 :ok :read [23 17] INFO [2019-12-17 12:31:31,359] jepsen worker 70 - jepsen.util 5470 :invoke :read [23 nil] INFO [2019-12-17 12:31:31,387] jepsen worker 56 - jepsen.util 5006 :ok :write [18 34] INFO [2019-12-17 12:31:31,387] jepsen worker 56 - jepsen.util 5006 :invoke :read [18 nil] INFO [2019-12-17 12:31:31,392] jepsen worker 71 - jepsen.util 5201 :info :write [23 8] :RequestCanceledException INFO [2019-12-17 12:31:31,393] jepsen worker 71 - jepsen.util 5291 :invoke :read [23 nil] INFO [2019-12-17 12:31:31,397] jepsen worker 27 - jepsen.util 4707 :ok :write [9 9] INFO [2019-12-17 12:31:31,397] jepsen worker 27 - jepsen.util 4707 :invoke :read [9 nil] INFO [2019-12-17 12:31:31,413] jepsen worker 27 - jepsen.util 4707 :ok :read [9 9] INFO [2019-12-17 12:31:31,413] jepsen worker 27 - jepsen.util 4707 :invoke :read [9 nil] INFO [2019-12-17 12:31:31,433] jepsen worker 3 - jepsen.util 4953 :fail :cas [1 [20 14]] :ValueNotSwapFrom INFO [2019-12-17 12:31:31,433] jepsen worker 3 - jepsen.util 4953 :invoke :read [1 nil] INFO [2019-12-17 12:31:31,437] jepsen worker 27 - jepsen.util 4707 :ok :read [9 9] INFO [2019-12-17 12:31:31,437] jepsen worker 27 - jepsen.util 4707 :invoke :cas [9 [16 9]] INFO [2019-12-17 12:31:31,438] jepsen worker 5 - jepsen.util 4235 :info :write [1 8] :RequestCanceledException INFO [2019-12-17 12:31:31,439] jepsen worker 5 - jepsen.util 4325 :invoke :cas [1 [21 40]] INFO [2019-12-17 12:31:31,441] jepsen worker 45 - jepsen.util 4995 :ok :write [15 35] INFO [2019-12-17 12:31:31,441] jepsen worker 45 - jepsen.util 4995 :invoke :cas [15 [40 29]] INFO [2019-12-17 12:31:31,442] jepsen worker 32 - jepsen.util 4982 :ok :write [10 29] INFO [2019-12-17 12:31:31,442] jepsen worker 32 - jepsen.util 4982 :invoke :cas [10 [6 12]] INFO [2019-12-17 12:31:31,456] jepsen worker 72 - jepsen.util 4842 :ok :write [24 43] INFO [2019-12-17 12:31:31,456] jepsen worker 72 - jepsen.util 4842 :invoke :write [24 48] INFO [2019-12-17 12:31:31,477] jepsen worker 51 - jepsen.util 4731 :ok :read [17 13] INFO [2019-12-17 12:31:31,477] jepsen worker 51 - jepsen.util 4731 :invoke :read [17 nil] INFO [2019-12-17 12:31:31,482] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":27822,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:31:31,482] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":27822,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:31:31,492] jepsen worker 38 - jepsen.util 4628 :fail :cas [12 [31 47]] :ValueNotSwapFrom INFO [2019-12-17 12:31:31,492] jepsen worker 38 - jepsen.util 4628 :invoke :cas [12 [24 36]] INFO [2019-12-17 12:31:31,495] jepsen worker 27 - jepsen.util 4707 :fail :cas [9 [16 9]] :ValueNotSwapFrom INFO [2019-12-17 12:31:31,495] jepsen worker 27 - jepsen.util 4707 :invoke :cas [9 [14 39]] INFO [2019-12-17 12:31:31,513] jepsen worker 32 - jepsen.util 4982 :fail :cas [10 [6 12]] :ValueNotSwapFrom INFO [2019-12-17 12:31:31,514] jepsen worker 32 - jepsen.util 4982 :invoke :cas [10 [47 45]] INFO [2019-12-17 12:31:31,519] jepsen worker 52 - jepsen.util 5182 :fail :cas [17 [26 14]] :ValueNotSwapFrom INFO [2019-12-17 12:31:31,520] jepsen worker 52 - jepsen.util 5182 :invoke :cas [17 [19 28]] INFO [2019-12-17 12:31:31,520] jepsen worker 27 - jepsen.util 4707 :fail :cas [9 [14 39]] :ValueNotSwapFrom INFO [2019-12-17 12:31:31,520] jepsen worker 27 - jepsen.util 4707 :invoke :read [9 nil] INFO [2019-12-17 12:31:31,526] jepsen worker 29 - jepsen.util 3899 :ok :write [9 17] INFO [2019-12-17 12:31:31,526] jepsen worker 29 - jepsen.util 3899 :invoke :write [9 10] INFO [2019-12-17 12:31:31,530] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:31:31,530] jepsen nemesis - couchbase.util Rebalance check count: 208 INFO [2019-12-17 12:31:31,530] jepsen nemesis - couchbase.util Rebalance stuck count: 189 INFO [2019-12-17 12:31:31,532] jepsen worker 32 - jepsen.util 4982 :fail :cas [10 [47 45]] :ValueNotSwapFrom INFO [2019-12-17 12:31:31,533] jepsen worker 32 - jepsen.util 4982 :invoke :cas [10 [18 12]] INFO [2019-12-17 12:31:31,537] jepsen worker 27 - jepsen.util 4707 :ok :read [9 9] INFO [2019-12-17 12:31:31,537] jepsen worker 27 - jepsen.util 4707 :invoke :read [9 nil] INFO [2019-12-17 12:31:31,554] jepsen worker 3 - jepsen.util 4953 :ok :read [1 26] INFO [2019-12-17 12:31:31,554] jepsen worker 3 - jepsen.util 4953 :invoke :write [1 40] INFO [2019-12-17 12:31:31,555] jepsen worker 78 - jepsen.util 4848 :ok :read [26 29] INFO [2019-12-17 12:31:31,555] jepsen worker 78 - jepsen.util 4848 :invoke :cas [26 [16 32]] INFO [2019-12-17 12:31:31,584] jepsen worker 56 - jepsen.util 5006 :ok :read [18 34] INFO [2019-12-17 12:31:31,584] jepsen worker 56 - jepsen.util 5006 :invoke :write [18 42] INFO [2019-12-17 12:31:31,589] jepsen worker 32 - jepsen.util 4982 :fail :cas [10 [18 12]] :ValueNotSwapFrom INFO [2019-12-17 12:31:31,589] jepsen worker 32 - jepsen.util 4982 :invoke :cas [10 [39 20]] INFO [2019-12-17 12:31:31,590] jepsen worker 27 - jepsen.util 4707 :ok :read [9 17] INFO [2019-12-17 12:31:31,590] jepsen worker 27 - jepsen.util 4707 :invoke :read [9 nil] INFO [2019-12-17 12:31:31,608] jepsen worker 5 - jepsen.util 4325 :fail :cas [1 [21 40]] :ValueNotSwapFrom INFO [2019-12-17 12:31:31,609] jepsen worker 5 - jepsen.util 4325 :invoke :read [1 nil] INFO [2019-12-17 12:31:31,610] jepsen worker 13 - jepsen.util 4423 :info :write [4 20] :RequestCanceledException INFO [2019-12-17 12:31:31,612] jepsen worker 13 - jepsen.util 4513 :invoke :write [4 29] INFO [2019-12-17 12:31:31,620] jepsen worker 32 - jepsen.util 4982 :fail :cas [10 [39 20]] :ValueNotSwapFrom INFO [2019-12-17 12:31:31,623] jepsen worker 27 - jepsen.util 4707 :ok :read [9 17] INFO [2019-12-17 12:31:31,624] jepsen worker 32 - jepsen.util 4982 :invoke :write [10 41] INFO [2019-12-17 12:31:31,624] jepsen worker 27 - jepsen.util 4707 :invoke :write [9 44] INFO [2019-12-17 12:31:31,660] jepsen worker 19 - jepsen.util 4789 :ok :write [6 27] INFO [2019-12-17 12:31:31,660] jepsen worker 19 - jepsen.util 4789 :invoke :write [6 17] INFO [2019-12-17 12:31:31,666] jepsen worker 87 - jepsen.util 4587 :ok :read [29 7] INFO [2019-12-17 12:31:31,666] jepsen worker 87 - jepsen.util 4587 :invoke :cas [29 [47 5]] INFO [2019-12-17 12:31:31,679] jepsen worker 30 - jepsen.util 4890 :ok :write [10 11] INFO [2019-12-17 12:31:31,679] jepsen worker 30 - jepsen.util 4890 :invoke :write [10 10] INFO [2019-12-17 12:31:31,679] jepsen worker 51 - jepsen.util 4731 :ok :read [17 13] INFO [2019-12-17 12:31:31,680] jepsen worker 51 - jepsen.util 4731 :invoke :read [17 nil] INFO [2019-12-17 12:31:31,682] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":27852,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:31:31,682] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":27852,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:31:31,682] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":27854,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:31:31,682] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":27854,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:31:31,685] jepsen worker 63 - jepsen.util 3933 :ok :read [21 27] INFO [2019-12-17 12:31:31,685] jepsen worker 63 - jepsen.util 3933 :invoke :read [21 nil] INFO [2019-12-17 12:31:31,686] jepsen worker 29 - jepsen.util 3899 :ok :write [9 10] INFO [2019-12-17 12:31:31,687] jepsen worker 29 - jepsen.util 3899 :invoke :cas [9 [42 4]] INFO [2019-12-17 12:31:31,723] jepsen worker 89 - jepsen.util 4409 :info :write [29 24] :RequestCanceledException INFO [2019-12-17 12:31:31,723] jepsen worker 88 - jepsen.util 4948 :info :write [29 6] :RequestCanceledException INFO [2019-12-17 12:31:31,725] jepsen worker 88 - jepsen.util 5038 :invoke :cas [29 [48 15]] INFO [2019-12-17 12:31:31,725] jepsen worker 89 - jepsen.util 4499 :invoke :cas [29 [15 44]] INFO [2019-12-17 12:31:31,738] jepsen worker 52 - jepsen.util 5182 :fail :cas [17 [19 28]] :ValueNotSwapFrom INFO [2019-12-17 12:31:31,738] jepsen worker 52 - jepsen.util 5182 :invoke :read [17 nil] INFO [2019-12-17 12:31:31,739] jepsen worker 29 - jepsen.util 3899 :fail :cas [9 [42 4]] :ValueNotSwapFrom INFO [2019-12-17 12:31:31,739] jepsen worker 29 - jepsen.util 3899 :invoke :cas [9 [40 32]] INFO [2019-12-17 12:31:31,745] jepsen worker 70 - jepsen.util 5470 :ok :read [23 8] INFO [2019-12-17 12:31:31,745] jepsen worker 70 - jepsen.util 5470 :invoke :write [23 40] INFO [2019-12-17 12:31:31,765] jepsen worker 29 - jepsen.util 3899 :fail :cas [9 [40 32]] :ValueNotSwapFrom INFO [2019-12-17 12:31:31,765] jepsen worker 29 - jepsen.util 3899 :invoke :cas [9 [46 17]] INFO [2019-12-17 12:31:31,787] jepsen worker 71 - jepsen.util 5291 :ok :read [23 8] INFO [2019-12-17 12:31:31,787] jepsen worker 71 - jepsen.util 5291 :invoke :cas [23 [10 44]] INFO [2019-12-17 12:31:31,801] jepsen worker 29 - jepsen.util 3899 :fail :cas [9 [46 17]] :ValueNotSwapFrom INFO [2019-12-17 12:31:31,802] jepsen worker 29 - jepsen.util 3899 :invoke :cas [9 [40 36]] INFO [2019-12-17 12:31:31,824] jepsen worker 29 - jepsen.util 3899 :fail :cas [9 [40 36]] :ValueNotSwapFrom INFO [2019-12-17 12:31:31,824] jepsen worker 29 - jepsen.util 3899 :invoke :read [9 nil] INFO [2019-12-17 12:31:31,825] jepsen worker 87 - jepsen.util 4587 :fail :cas [29 [47 5]] :ValueNotSwapFrom INFO [2019-12-17 12:31:31,825] jepsen worker 87 - jepsen.util 4587 :invoke :cas [29 [28 4]] INFO [2019-12-17 12:31:31,836] jepsen worker 29 - jepsen.util 3899 :ok :read [9 10] INFO [2019-12-17 12:31:31,836] jepsen worker 29 - jepsen.util 3899 :invoke :cas [9 [30 40]] INFO [2019-12-17 12:31:31,852] jepsen worker 29 - jepsen.util 3899 :fail :cas [9 [30 40]] :ValueNotSwapFrom INFO [2019-12-17 12:31:31,852] jepsen worker 29 - jepsen.util 3899 :invoke :read [9 nil] INFO [2019-12-17 12:31:31,875] jepsen worker 38 - jepsen.util 4628 :fail :cas [12 [24 36]] :ValueNotSwapFrom INFO [2019-12-17 12:31:31,875] jepsen worker 38 - jepsen.util 4628 :invoke :write [12 19] INFO [2019-12-17 12:31:31,876] jepsen worker 29 - jepsen.util 3899 :ok :read [9 10] INFO [2019-12-17 12:31:31,876] jepsen worker 29 - jepsen.util 3899 :invoke :cas [9 [11 23]] INFO [2019-12-17 12:31:31,889] jepsen worker 29 - jepsen.util 3899 :fail :cas [9 [11 23]] :ValueNotSwapFrom INFO [2019-12-17 12:31:31,889] jepsen worker 29 - jepsen.util 3899 :invoke :write [9 44] INFO [2019-12-17 12:31:31,894] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":27874,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:31:31,894] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":27874,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:31:31,894] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":27877,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:31:31,894] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":27877,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:31:31,894] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":27878,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:31:31,894] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":27878,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:31:31,914] jepsen worker 78 - jepsen.util 4848 :fail :cas [26 [16 32]] :ValueNotSwapFrom INFO [2019-12-17 12:31:31,914] jepsen worker 78 - jepsen.util 4848 :invoke :write [26 34] INFO [2019-12-17 12:31:31,925] jepsen worker 69 - jepsen.util 5019 :info :write [23 6] :RequestCanceledException INFO [2019-12-17 12:31:31,925] jepsen worker 54 - jepsen.util 5274 :info :write [18 17] :RequestCanceledException INFO [2019-12-17 12:31:31,926] jepsen worker 69 - jepsen.util 5109 :invoke :write [23 43] INFO [2019-12-17 12:31:31,926] jepsen worker 54 - jepsen.util 5364 :invoke :read [18 nil] INFO [2019-12-17 12:31:31,938] jepsen worker 29 - jepsen.util 3899 :ok :write [9 44] INFO [2019-12-17 12:31:31,938] jepsen worker 29 - jepsen.util 3899 :invoke :cas [9 [30 5]] INFO [2019-12-17 12:31:31,959] jepsen worker 29 - jepsen.util 3899 :fail :cas [9 [30 5]] :ValueNotSwapFrom INFO [2019-12-17 12:31:31,959] jepsen worker 29 - jepsen.util 3899 :invoke :cas [9 [8 3]] INFO [2019-12-17 12:31:31,973] jepsen worker 29 - jepsen.util 3899 :fail :cas [9 [8 3]] :ValueNotSwapFrom INFO [2019-12-17 12:31:31,973] jepsen worker 29 - jepsen.util 3899 :invoke :cas [9 [13 1]] INFO [2019-12-17 12:31:31,985] jepsen worker 29 - jepsen.util 3899 :fail :cas [9 [13 1]] :ValueNotSwapFrom INFO [2019-12-17 12:31:31,985] jepsen worker 29 - jepsen.util 3899 :invoke :cas [9 [27 44]] INFO [2019-12-17 12:31:31,995] jepsen worker 29 - jepsen.util 3899 :fail :cas [9 [27 44]] :ValueNotSwapFrom INFO [2019-12-17 12:31:31,995] jepsen worker 29 - jepsen.util 3899 :invoke :write [9 36] INFO [2019-12-17 12:31:32,032] jepsen worker 29 - jepsen.util 3899 :ok :write [9 36] INFO [2019-12-17 12:31:32,032] jepsen worker 29 - jepsen.util 3899 :invoke :write [9 47] INFO [2019-12-17 12:31:32,067] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":27891,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:31:32,067] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":27891,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:31:32,076] jepsen worker 29 - jepsen.util 3899 :ok :write [9 47] INFO [2019-12-17 12:31:32,076] jepsen worker 29 - jepsen.util 3899 :invoke :write [9 25] INFO [2019-12-17 12:31:32,102] jepsen worker 88 - jepsen.util 5038 :fail :cas [29 [48 15]] :ValueNotSwapFrom INFO [2019-12-17 12:31:32,102] jepsen worker 88 - jepsen.util 5038 :invoke :read [29 nil] INFO [2019-12-17 12:31:32,116] jepsen worker 29 - jepsen.util 3899 :ok :write [9 25] INFO [2019-12-17 12:31:32,116] jepsen worker 29 - jepsen.util 3899 :invoke :write [9 17] INFO [2019-12-17 12:31:32,158] jepsen worker 29 - jepsen.util 3899 :ok :write [9 17] INFO [2019-12-17 12:31:32,158] jepsen worker 29 - jepsen.util 3899 :invoke :cas [9 [32 20]] INFO [2019-12-17 12:31:32,174] jepsen worker 29 - jepsen.util 3899 :fail :cas [9 [32 20]] :ValueNotSwapFrom INFO [2019-12-17 12:31:32,174] jepsen worker 29 - jepsen.util 3899 :invoke :cas [9 [20 13]] INFO [2019-12-17 12:31:32,182] jepsen worker 55 - jepsen.util 5815 :ok :read [18 34] INFO [2019-12-17 12:31:32,182] jepsen worker 55 - jepsen.util 5815 :invoke :write [18 16] INFO [2019-12-17 12:31:32,187] jepsen worker 29 - jepsen.util 3899 :fail :cas [9 [20 13]] :ValueNotSwapFrom INFO [2019-12-17 12:31:32,187] jepsen worker 29 - jepsen.util 3899 :invoke :cas [9 [2 40]] INFO [2019-12-17 12:31:32,207] jepsen worker 29 - jepsen.util 3899 :fail :cas [9 [2 40]] :ValueNotSwapFrom INFO [2019-12-17 12:31:32,207] jepsen worker 29 - jepsen.util 3899 :invoke :read [9 nil] INFO [2019-12-17 12:31:32,219] jepsen worker 29 - jepsen.util 3899 :ok :read [9 17] INFO [2019-12-17 12:31:32,219] jepsen worker 29 - jepsen.util 3899 :invoke :read [9 nil] INFO [2019-12-17 12:31:32,226] jepsen worker 68 - jepsen.util 6008 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:31:32,226] jepsen worker 68 - jepsen.util 6008 :invoke :write [22 47] INFO [2019-12-17 12:31:32,230] jepsen worker 29 - jepsen.util 3899 :ok :read [9 17] INFO [2019-12-17 12:31:32,230] jepsen worker 29 - jepsen.util 3899 :invoke :read [9 nil] INFO [2019-12-17 12:31:32,244] jepsen worker 29 - jepsen.util 3899 :ok :read [9 17] INFO [2019-12-17 12:31:32,244] jepsen worker 29 - jepsen.util 3899 :invoke :read [9 nil] INFO [2019-12-17 12:31:32,254] jepsen worker 29 - jepsen.util 3899 :ok :read [9 17] INFO [2019-12-17 12:31:32,254] jepsen worker 29 - jepsen.util 3899 :invoke :write [9 26] INFO [2019-12-17 12:31:32,257] jepsen worker 84 - jepsen.util 6114 :info :write [28 8] :RequestCanceledException INFO [2019-12-17 12:31:32,258] jepsen worker 84 - jepsen.util 6204 :invoke :write [28 36] INFO [2019-12-17 12:31:32,269] jepsen worker 71 - jepsen.util 5291 :fail :cas [23 [10 44]] :ValueNotSwapFrom INFO [2019-12-17 12:31:32,270] jepsen worker 71 - jepsen.util 5291 :invoke :cas [23 [18 7]] INFO [2019-12-17 12:31:32,286] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":27904,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:31:32,286] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":27904,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:31:32,286] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":27912,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:31:32,286] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":27906,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:31:32,286] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":27912,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:31:32,287] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":27915,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:31:32,287] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":27906,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:31:32,287] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":27915,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:31:32,287] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":27916,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:31:32,287] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":27916,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:31:32,292] jepsen worker 33 - jepsen.util 6333 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:31:32,292] jepsen worker 26 - jepsen.util 6776 :info :write [8 7] :RequestCanceledException INFO [2019-12-17 12:31:32,292] jepsen worker 33 - jepsen.util 6333 :invoke :write [11 24] INFO [2019-12-17 12:31:32,294] jepsen worker 87 - jepsen.util 4587 :fail :cas [29 [28 4]] :ValueNotSwapFrom INFO [2019-12-17 12:31:32,294] jepsen worker 87 - jepsen.util 4587 :invoke :cas [29 [6 6]] INFO [2019-12-17 12:31:32,297] jepsen worker 26 - jepsen.util 6866 :invoke :read [8 nil] INFO [2019-12-17 12:31:32,304] jepsen worker 51 - jepsen.util 4731 :ok :read [17 13] INFO [2019-12-17 12:31:32,304] jepsen worker 51 - jepsen.util 4731 :invoke :write [17 47] INFO [2019-12-17 12:31:32,307] jepsen worker 5 - jepsen.util 4325 :ok :read [1 26] INFO [2019-12-17 12:31:32,307] jepsen worker 82 - jepsen.util 7282 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:31:32,307] jepsen worker 12 - jepsen.util 4962 :info :write [4 2] :RequestCanceledException INFO [2019-12-17 12:31:32,307] jepsen worker 5 - jepsen.util 4325 :invoke :write [1 22] INFO [2019-12-17 12:31:32,307] jepsen worker 82 - jepsen.util 7282 :invoke :read [27 nil] INFO [2019-12-17 12:31:32,308] jepsen worker 12 - jepsen.util 5052 :invoke :cas [4 [48 0]] INFO [2019-12-17 12:31:32,314] jepsen worker 54 - jepsen.util 5364 :ok :read [18 34] INFO [2019-12-17 12:31:32,315] jepsen worker 54 - jepsen.util 5364 :invoke :read [18 nil] INFO [2019-12-17 12:31:32,315] jepsen worker 52 - jepsen.util 5182 :ok :read [17 13] INFO [2019-12-17 12:31:32,315] jepsen worker 52 - jepsen.util 5182 :invoke :cas [17 [47 0]] INFO [2019-12-17 12:31:32,324] jepsen worker 29 - jepsen.util 3899 :ok :write [9 26] INFO [2019-12-17 12:31:32,324] jepsen worker 29 - jepsen.util 3899 :invoke :write [9 19] INFO [2019-12-17 12:31:32,325] jepsen worker 81 - jepsen.util 6291 :info :cas [27 [39 20]] :RequestCanceledException INFO [2019-12-17 12:31:32,326] jepsen worker 81 - jepsen.util 6381 :invoke :write [27 46] INFO [2019-12-17 12:31:32,326] jepsen worker 34 - jepsen.util 5974 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:31:32,326] jepsen worker 16 - jepsen.util 7486 :info :write [5 18] :RequestCanceledException INFO [2019-12-17 12:31:32,326] jepsen worker 34 - jepsen.util 5974 :invoke :read [11 nil] INFO [2019-12-17 12:31:32,328] jepsen worker 16 - jepsen.util 7576 :invoke :write [5 30] INFO [2019-12-17 12:31:32,340] jepsen worker 76 - jepsen.util 6556 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:31:32,340] jepsen worker 43 - jepsen.util 6163 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:31:32,340] jepsen worker 76 - jepsen.util 6556 :invoke :cas [25 [13 18]] INFO [2019-12-17 12:31:32,340] jepsen worker 43 - jepsen.util 6163 :invoke :cas [14 [38 2]] INFO [2019-12-17 12:31:32,356] jepsen worker 8 - jepsen.util 5768 :info :write [2 37] :RequestCanceledException INFO [2019-12-17 12:31:32,356] jepsen worker 57 - jepsen.util 6087 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:31:32,356] jepsen worker 57 - jepsen.util 6087 :invoke :read [19 nil] INFO [2019-12-17 12:31:32,358] jepsen worker 8 - jepsen.util 5858 :invoke :write [2 39] INFO [2019-12-17 12:31:32,370] jepsen worker 29 - jepsen.util 3899 :ok :write [9 19] INFO [2019-12-17 12:31:32,370] jepsen worker 29 - jepsen.util 3899 :invoke :write [9 44] INFO [2019-12-17 12:31:32,390] jepsen worker 6 - jepsen.util 6846 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:31:32,390] jepsen worker 20 - jepsen.util 4880 :info :write [6 39] :RequestCanceledException INFO [2019-12-17 12:31:32,390] jepsen worker 58 - jepsen.util 6628 :info :write [19 42] :RequestCanceledException INFO [2019-12-17 12:31:32,390] jepsen worker 73 - jepsen.util 4573 :fail :read [24 nil] :RequestTimeoutException INFO [2019-12-17 12:31:32,390] jepsen worker 6 - jepsen.util 6846 :invoke :read [2 nil] INFO [2019-12-17 12:31:32,390] jepsen worker 73 - jepsen.util 4573 :invoke :cas [24 [20 9]] INFO [2019-12-17 12:31:32,391] jepsen worker 20 - jepsen.util 4970 :invoke :read [6 nil] INFO [2019-12-17 12:31:32,391] jepsen worker 58 - jepsen.util 6718 :invoke :write [19 15] INFO [2019-12-17 12:31:32,407] jepsen worker 17 - jepsen.util 6407 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:31:32,407] jepsen worker 49 - jepsen.util 5719 :info :cas [16 [45 29]] :RequestCanceledException INFO [2019-12-17 12:31:32,407] jepsen worker 17 - jepsen.util 6407 :invoke :write [5 4] INFO [2019-12-17 12:31:32,408] jepsen worker 25 - jepsen.util 6595 :info :cas [8 [8 2]] :RequestCanceledException INFO [2019-12-17 12:31:32,409] jepsen worker 25 - jepsen.util 6685 :invoke :write [8 30] INFO [2019-12-17 12:31:32,409] jepsen worker 49 - jepsen.util 5809 :invoke :read [16 nil] INFO [2019-12-17 12:31:32,423] jepsen worker 73 - jepsen.util 4573 :fail :cas [24 [20 9]] :ValueNotSwapFrom INFO [2019-12-17 12:31:32,423] jepsen worker 73 - jepsen.util 4573 :invoke :cas [24 [46 17]] INFO [2019-12-17 12:31:32,426] jepsen worker 20 - jepsen.util 4970 :ok :read [6 27] INFO [2019-12-17 12:31:32,426] jepsen worker 40 - jepsen.util 7150 :info :write [13 12] :RequestCanceledException INFO [2019-12-17 12:31:32,426] jepsen worker 20 - jepsen.util 4970 :invoke :read [6 nil] INFO [2019-12-17 12:31:32,426] jepsen worker 66 - jepsen.util 5916 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:31:32,426] jepsen worker 48 - jepsen.util 5898 :info :cas [16 [20 47]] :RequestCanceledException INFO [2019-12-17 12:31:32,426] jepsen worker 75 - jepsen.util 6465 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:31:32,426] jepsen worker 83 - jepsen.util 6833 :info :write [27 13] :RequestCanceledException INFO [2019-12-17 12:31:32,426] jepsen worker 66 - jepsen.util 5916 :invoke :write [22 49] INFO [2019-12-17 12:31:32,426] jepsen worker 77 - jepsen.util 6737 :info :write [25 45] :RequestCanceledException INFO [2019-12-17 12:31:32,426] jepsen worker 75 - jepsen.util 6465 :invoke :read [25 nil] INFO [2019-12-17 12:31:32,429] jepsen worker 40 - jepsen.util 7240 :invoke :read [13 nil] INFO [2019-12-17 12:31:32,430] jepsen worker 48 - jepsen.util 5988 :invoke :write [16 14] INFO [2019-12-17 12:31:32,430] jepsen worker 77 - jepsen.util 6827 :invoke :read [25 nil] INFO [2019-12-17 12:31:32,430] jepsen worker 83 - jepsen.util 6923 :invoke :write [27 40] INFO [2019-12-17 12:31:32,433] jepsen worker 85 - jepsen.util 7285 :info :cas [28 [2 34]] :RequestCanceledException INFO [2019-12-17 12:31:32,433] jepsen worker 15 - jepsen.util 5415 :info :cas [5 [6 19]] :RequestCanceledException INFO [2019-12-17 12:31:32,434] jepsen worker 60 - jepsen.util 6990 :info :cas [20 [38 12]] :RequestCanceledException INFO [2019-12-17 12:31:32,437] jepsen worker 15 - jepsen.util 5505 :invoke :cas [5 [32 10]] INFO [2019-12-17 12:31:32,437] jepsen worker 85 - jepsen.util 7375 :invoke :cas [28 [23 19]] INFO [2019-12-17 12:31:32,437] jepsen worker 60 - jepsen.util 7080 :invoke :cas [20 [7 44]] INFO [2019-12-17 12:31:32,440] jepsen worker 89 - jepsen.util 4499 :fail :cas [29 [15 44]] :ValueNotSwapFrom INFO [2019-12-17 12:31:32,440] jepsen worker 89 - jepsen.util 4499 :invoke :write [29 17] INFO [2019-12-17 12:31:32,451] jepsen worker 29 - jepsen.util 3899 :ok :write [9 44] INFO [2019-12-17 12:31:32,451] jepsen worker 29 - jepsen.util 3899 :invoke :cas [9 [9 44]] INFO [2019-12-17 12:31:32,453] jepsen worker 62 - jepsen.util 5912 :info :cas [20 [36 28]] :RequestCanceledException INFO [2019-12-17 12:31:32,453] jepsen worker 39 - jepsen.util 6069 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:31:32,453] jepsen worker 50 - jepsen.util 6800 :info :write [16 8] :RequestCanceledException INFO [2019-12-17 12:31:32,453] jepsen worker 7 - jepsen.util 6577 :info :write [2 45] :RequestCanceledException INFO [2019-12-17 12:31:32,453] jepsen worker 39 - jepsen.util 6069 :invoke :write [13 39] INFO [2019-12-17 12:31:32,458] jepsen worker 62 - jepsen.util 6002 :invoke :write [20 35] INFO [2019-12-17 12:31:32,458] jepsen worker 50 - jepsen.util 6890 :invoke :cas [16 [49 23]] INFO [2019-12-17 12:31:32,460] jepsen worker 7 - jepsen.util 6667 :invoke :read [2 nil] INFO [2019-12-17 12:31:32,465] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":27954,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:31:32,466] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":27954,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:31:32,466] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":27957,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:31:32,466] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":27957,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:31:32,482] jepsen worker 73 - jepsen.util 4573 :fail :cas [24 [46 17]] :ValueNotSwapFrom INFO [2019-12-17 12:31:32,482] jepsen worker 73 - jepsen.util 4573 :invoke :read [24 nil] INFO [2019-12-17 12:31:32,484] jepsen worker 20 - jepsen.util 4970 :ok :read [6 27] INFO [2019-12-17 12:31:32,484] jepsen worker 20 - jepsen.util 4970 :invoke :cas [6 [46 41]] INFO [2019-12-17 12:31:32,486] jepsen worker 87 - jepsen.util 4587 :fail :cas [29 [6 6]] :ValueNotSwapFrom INFO [2019-12-17 12:31:32,486] jepsen worker 87 - jepsen.util 4587 :invoke :cas [29 [2 36]] INFO [2019-12-17 12:31:32,492] jepsen worker 29 - jepsen.util 3899 :fail :cas [9 [9 44]] :ValueNotSwapFrom INFO [2019-12-17 12:31:32,492] jepsen worker 29 - jepsen.util 3899 :invoke :cas [9 [6 35]] INFO [2019-12-17 12:31:32,492] jepsen worker 1 - jepsen.util 6211 :info :cas [0 [3 48]] :RequestCanceledException INFO [2019-12-17 12:31:32,494] jepsen worker 1 - jepsen.util 6301 :invoke :cas [0 [4 2]] INFO [2019-12-17 12:31:32,510] jepsen worker 22 - jepsen.util 6502 :info :cas [7 [16 20]] :RequestCanceledException INFO [2019-12-17 12:31:32,510] jepsen worker 2 - jepsen.util 5942 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:31:32,510] jepsen worker 42 - jepsen.util 6882 :info :write [14 1] :RequestCanceledException INFO [2019-12-17 12:31:32,510] jepsen worker 44 - jepsen.util 7064 :info :cas [14 [16 1]] :RequestCanceledException INFO [2019-12-17 12:31:32,510] jepsen worker 2 - jepsen.util 5942 :invoke :cas [0 [11 6]] INFO [2019-12-17 12:31:32,514] jepsen worker 44 - jepsen.util 7154 :invoke :write [14 6] INFO [2019-12-17 12:31:32,514] jepsen worker 22 - jepsen.util 6592 :invoke :write [7 22] INFO [2019-12-17 12:31:32,516] jepsen worker 42 - jepsen.util 6972 :invoke :write [14 37] INFO [2019-12-17 12:31:32,526] jepsen worker 24 - jepsen.util 5964 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:31:32,526] jepsen worker 41 - jepsen.util 6971 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:31:32,526] jepsen worker 67 - jepsen.util 6637 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:31:32,526] jepsen worker 0 - jepsen.util 7110 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:31:32,526] jepsen worker 24 - jepsen.util 5964 :invoke :read [8 nil] INFO [2019-12-17 12:31:32,526] jepsen worker 67 - jepsen.util 6637 :invoke :read [22 nil] INFO [2019-12-17 12:31:32,526] jepsen worker 41 - jepsen.util 6971 :invoke :write [13 44] INFO [2019-12-17 12:31:32,526] jepsen worker 0 - jepsen.util 7110 :invoke :write [0 14] INFO [2019-12-17 12:31:32,546] jepsen worker 73 - jepsen.util 4573 :ok :read [24 43] INFO [2019-12-17 12:31:32,546] jepsen worker 73 - jepsen.util 4573 :invoke :cas [24 [43 44]] INFO [2019-12-17 12:31:32,548] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:31:32,549] jepsen nemesis - couchbase.util Rebalance check count: 209 INFO [2019-12-17 12:31:32,549] jepsen nemesis - couchbase.util Rebalance stuck count: 190 INFO [2019-12-17 12:31:32,554] jepsen worker 20 - jepsen.util 4970 :fail :cas [6 [46 41]] :ValueNotSwapFrom INFO [2019-12-17 12:31:32,554] jepsen worker 20 - jepsen.util 4970 :invoke :cas [6 [9 31]] INFO [2019-12-17 12:31:32,556] jepsen worker 35 - jepsen.util 6605 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:31:32,557] jepsen worker 35 - jepsen.util 6605 :invoke :cas [11 [41 26]] INFO [2019-12-17 12:31:32,557] jepsen worker 29 - jepsen.util 3899 :fail :cas [9 [6 35]] :ValueNotSwapFrom INFO [2019-12-17 12:31:32,557] jepsen worker 29 - jepsen.util 3899 :invoke :write [9 24] INFO [2019-12-17 12:31:32,600] jepsen worker 71 - jepsen.util 5291 :fail :cas [23 [18 7]] :ValueNotSwapFrom INFO [2019-12-17 12:31:32,601] jepsen worker 71 - jepsen.util 5291 :invoke :cas [23 [9 6]] INFO [2019-12-17 12:31:32,601] jepsen worker 20 - jepsen.util 4970 :fail :cas [6 [9 31]] :ValueNotSwapFrom INFO [2019-12-17 12:31:32,601] jepsen worker 20 - jepsen.util 4970 :invoke :read [6 nil] INFO [2019-12-17 12:31:32,625] jepsen worker 86 - jepsen.util 6836 :info :write [28 33] :RequestCanceledException INFO [2019-12-17 12:31:32,626] jepsen worker 87 - jepsen.util 4587 :fail :cas [29 [2 36]] :ValueNotSwapFrom INFO [2019-12-17 12:31:32,626] jepsen worker 87 - jepsen.util 4587 :invoke :read [29 nil] INFO [2019-12-17 12:31:32,628] jepsen worker 86 - jepsen.util 6926 :invoke :read [28 nil] INFO [2019-12-17 12:31:32,635] jepsen worker 29 - jepsen.util 3899 :ok :write [9 24] INFO [2019-12-17 12:31:32,636] jepsen worker 29 - jepsen.util 3899 :invoke :cas [9 [49 43]] INFO [2019-12-17 12:31:32,638] jepsen worker 20 - jepsen.util 4970 :ok :read [6 27] INFO [2019-12-17 12:31:32,638] jepsen worker 20 - jepsen.util 4970 :invoke :read [6 nil] INFO [2019-12-17 12:31:32,657] jepsen worker 28 - jepsen.util 3538 :info :cas [9 [2 37]] :RequestCanceledException INFO [2019-12-17 12:31:32,658] jepsen worker 28 - jepsen.util 3628 :invoke :read [9 nil] INFO [2019-12-17 12:31:32,665] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":27996,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:31:32,665] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":27996,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:31:32,666] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28000,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:31:32,666] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28000,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:31:32,666] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28001,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:31:32,666] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28001,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:31:32,670] jepsen worker 29 - jepsen.util 3899 :fail :cas [9 [49 43]] :ValueNotSwapFrom INFO [2019-12-17 12:31:32,670] jepsen worker 29 - jepsen.util 3899 :invoke :cas [9 [14 43]] INFO [2019-12-17 12:31:32,683] jepsen worker 20 - jepsen.util 4970 :ok :read [6 27] INFO [2019-12-17 12:31:32,683] jepsen worker 20 - jepsen.util 4970 :invoke :write [6 33] INFO [2019-12-17 12:31:32,717] jepsen worker 54 - jepsen.util 5364 :ok :read [18 34] INFO [2019-12-17 12:31:32,717] jepsen worker 54 - jepsen.util 5364 :invoke :read [18 nil] INFO [2019-12-17 12:31:32,726] jepsen worker 29 - jepsen.util 3899 :fail :cas [9 [14 43]] :ValueNotSwapFrom INFO [2019-12-17 12:31:32,727] jepsen worker 29 - jepsen.util 3899 :invoke :write [9 32] INFO [2019-12-17 12:31:32,740] jepsen worker 11 - jepsen.util 4691 :info :cas [3 [31 29]] :RequestCanceledException INFO [2019-12-17 12:31:32,740] jepsen worker 64 - jepsen.util 3664 :fail :read [21 nil] :RequestTimeoutException INFO [2019-12-17 12:31:32,740] jepsen worker 4 - jepsen.util 3964 :info :write [1 11] :RequestCanceledException INFO [2019-12-17 12:31:32,740] jepsen worker 64 - jepsen.util 3664 :invoke :cas [21 [30 47]] INFO [2019-12-17 12:31:32,741] jepsen worker 11 - jepsen.util 4781 :invoke :write [3 39] INFO [2019-12-17 12:31:32,743] jepsen worker 52 - jepsen.util 5182 :fail :cas [17 [47 0]] :ValueNotSwapFrom INFO [2019-12-17 12:31:32,743] jepsen worker 52 - jepsen.util 5182 :invoke :write [17 21] INFO [2019-12-17 12:31:32,744] jepsen worker 4 - jepsen.util 4054 :invoke :write [1 9] INFO [2019-12-17 12:31:32,756] jepsen worker 47 - jepsen.util 5357 :info :write [15 4] :RequestCanceledException INFO [2019-12-17 12:31:32,935] jepsen worker 9 - jepsen.util 5679 :info :cas [3 [17 10]] :RequestCanceledException INFO [2019-12-17 12:31:32,935] jepsen worker 61 - jepsen.util 7261 :info :write [20 34] :RequestCanceledException INFO [2019-12-17 12:31:32,935] jepsen worker 23 - jepsen.util 6593 :info :write [7 22] :RequestCanceledException INFO [2019-12-17 12:31:32,935] jepsen worker 59 - jepsen.util 6269 :info :write [19 48] :RequestCanceledException INFO [2019-12-17 12:31:32,935] jepsen worker 18 - jepsen.util 4788 :info :cas [6 [49 3]] :RequestCanceledException INFO [2019-12-17 12:31:32,935] jepsen worker 21 - jepsen.util 5961 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:31:32,935] jepsen worker 46 - jepsen.util 4816 :info :write [15 32] :RequestCanceledException INFO [2019-12-17 12:31:32,935] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28011,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:31:32,936] jepsen worker 71 - jepsen.util 5291 :fail :cas [23 [9 6]] :ValueNotSwapFrom INFO [2019-12-17 12:31:32,936] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28011,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:31:32,936] jepsen worker 21 - jepsen.util 5961 :invoke :write [7 7] INFO [2019-12-17 12:31:32,937] jepsen worker 47 - jepsen.util 5447 :invoke :write [15 27] INFO [2019-12-17 12:31:32,937] jepsen worker 61 - jepsen.util 7351 :invoke :write [20 9] INFO [2019-12-17 12:31:32,937] jepsen worker 9 - jepsen.util 5769 :invoke :read [3 nil] INFO [2019-12-17 12:31:32,937] jepsen worker 18 - jepsen.util 4878 :invoke :read [6 nil] INFO [2019-12-17 12:31:32,937] jepsen worker 59 - jepsen.util 6359 :invoke :cas [19 [40 31]] INFO [2019-12-17 12:31:32,937] jepsen worker 71 - jepsen.util 5291 :invoke :write [23 30] INFO [2019-12-17 12:31:32,937] jepsen worker 23 - jepsen.util 6683 :invoke :write [7 10] INFO [2019-12-17 12:31:32,937] jepsen worker 46 - jepsen.util 4906 :invoke :cas [15 [34 11]] INFO [2019-12-17 12:31:32,962] jepsen worker 9 - jepsen.util 5769 :ok :read [3 20] INFO [2019-12-17 12:31:32,962] jepsen worker 9 - jepsen.util 5769 :invoke :read [3 nil] INFO [2019-12-17 12:31:32,973] jepsen worker 9 - jepsen.util 5769 :ok :read [3 20] INFO [2019-12-17 12:31:32,973] jepsen worker 9 - jepsen.util 5769 :invoke :read [3 nil] INFO [2019-12-17 12:31:33,001] jepsen worker 9 - jepsen.util 5769 :ok :read [3 20] INFO [2019-12-17 12:31:33,001] jepsen worker 9 - jepsen.util 5769 :invoke :cas [3 [35 1]] INFO [2019-12-17 12:31:33,012] jepsen worker 29 - jepsen.util 3899 :ok :write [9 32] INFO [2019-12-17 12:31:33,012] jepsen worker 29 - jepsen.util 3899 :invoke :cas [9 [27 5]] INFO [2019-12-17 12:31:33,014] jepsen worker 9 - jepsen.util 5769 :fail :cas [3 [35 1]] :ValueNotSwapFrom INFO [2019-12-17 12:31:33,014] jepsen worker 9 - jepsen.util 5769 :invoke :write [3 0] INFO [2019-12-17 12:31:33,045] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28029,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:31:33,045] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28029,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:31:33,045] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28032,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:31:33,045] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28032,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:31:33,045] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28033,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:31:33,045] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28033,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:31:33,045] jepsen worker 29 - jepsen.util 3899 :fail :cas [9 [27 5]] :ValueNotSwapFrom INFO [2019-12-17 12:31:33,045] jepsen worker 29 - jepsen.util 3899 :invoke :write [9 47] INFO [2019-12-17 12:31:33,091] jepsen worker 80 - jepsen.util 4850 :info :write [26 31] :RequestCanceledException INFO [2019-12-17 12:31:33,091] jepsen worker 29 - jepsen.util 3899 :ok :write [9 47] INFO [2019-12-17 12:31:33,092] jepsen worker 29 - jepsen.util 3899 :invoke :cas [9 [40 21]] INFO [2019-12-17 12:31:33,092] jepsen worker 80 - jepsen.util 4940 :invoke :write [26 23] INFO [2019-12-17 12:31:33,106] jepsen worker 29 - jepsen.util 3899 :fail :cas [9 [40 21]] :ValueNotSwapFrom INFO [2019-12-17 12:31:33,106] jepsen worker 29 - jepsen.util 3899 :invoke :write [9 11] INFO [2019-12-17 12:31:33,132] jepsen worker 88 - jepsen.util 5038 :ok :read [29 7] INFO [2019-12-17 12:31:33,133] jepsen worker 88 - jepsen.util 5038 :invoke :read [29 nil] INFO [2019-12-17 12:31:33,152] jepsen worker 29 - jepsen.util 3899 :ok :write [9 11] INFO [2019-12-17 12:31:33,152] jepsen worker 29 - jepsen.util 3899 :invoke :read [9 nil] INFO [2019-12-17 12:31:33,153] jepsen worker 79 - jepsen.util 4759 :info :write [26 45] :RequestCanceledException INFO [2019-12-17 12:31:33,153] jepsen worker 36 - jepsen.util 4896 :info :write [12 42] :RequestCanceledException INFO [2019-12-17 12:31:33,154] jepsen worker 79 - jepsen.util 4849 :invoke :cas [26 [34 17]] INFO [2019-12-17 12:31:33,154] jepsen worker 36 - jepsen.util 4986 :invoke :cas [12 [11 35]] INFO [2019-12-17 12:31:33,171] jepsen worker 29 - jepsen.util 3899 :ok :read [9 11] INFO [2019-12-17 12:31:33,172] jepsen worker 29 - jepsen.util 3899 :invoke :write [9 29] INFO [2019-12-17 12:31:33,219] jepsen worker 29 - jepsen.util 3899 :ok :write [9 29] INFO [2019-12-17 12:31:33,219] jepsen worker 29 - jepsen.util 3899 :invoke :write [9 38] INFO [2019-12-17 12:31:33,225] jepsen worker 65 - jepsen.util 4205 :info :write [21 43] :RequestCanceledException INFO [2019-12-17 12:31:33,225] jepsen worker 74 - jepsen.util 5114 :info :write [24 29] :RequestCanceledException INFO [2019-12-17 12:31:33,227] jepsen worker 74 - jepsen.util 5204 :invoke :write [24 27] INFO [2019-12-17 12:31:33,227] jepsen worker 65 - jepsen.util 4295 :invoke :write [21 38] INFO [2019-12-17 12:31:33,263] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28048,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:31:33,263] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28054,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:31:33,263] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28048,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:31:33,263] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28054,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:31:33,263] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28049,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:31:33,263] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28057,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:31:33,263] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28049,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:31:33,263] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28057,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:31:33,264] jepsen worker 29 - jepsen.util 3899 :ok :write [9 38] INFO [2019-12-17 12:31:33,264] jepsen worker 29 - jepsen.util 3899 :invoke :write [9 12] INFO [2019-12-17 12:31:33,267] jepsen worker 88 - jepsen.util 5038 :ok :read [29 7] INFO [2019-12-17 12:31:33,267] jepsen worker 88 - jepsen.util 5038 :invoke :cas [29 [47 21]] INFO [2019-12-17 12:31:33,279] jepsen worker 87 - jepsen.util 4587 :ok :read [29 7] INFO [2019-12-17 12:31:33,279] jepsen worker 87 - jepsen.util 4587 :invoke :cas [29 [45 18]] INFO [2019-12-17 12:31:33,293] jepsen worker 54 - jepsen.util 5364 :ok :read [18 34] INFO [2019-12-17 12:31:33,294] jepsen worker 54 - jepsen.util 5364 :invoke :cas [18 [32 32]] INFO [2019-12-17 12:31:33,297] jepsen worker 65 - jepsen.util 4295 :ok :write [21 38] INFO [2019-12-17 12:31:33,297] jepsen worker 65 - jepsen.util 4295 :invoke :read [21 nil] INFO [2019-12-17 12:31:33,314] jepsen worker 65 - jepsen.util 4295 :ok :read [21 38] INFO [2019-12-17 12:31:33,314] jepsen worker 65 - jepsen.util 4295 :invoke :cas [21 [14 47]] INFO [2019-12-17 12:31:33,325] jepsen worker 37 - jepsen.util 4537 :info :write [12 16] :RequestCanceledException INFO [2019-12-17 12:31:33,325] jepsen worker 31 - jepsen.util 4711 :info :write [10 9] :RequestCanceledException INFO [2019-12-17 12:31:33,329] jepsen worker 37 - jepsen.util 4627 :invoke :read [12 nil] INFO [2019-12-17 12:31:33,329] jepsen worker 31 - jepsen.util 4801 :invoke :write [10 10] INFO [2019-12-17 12:31:33,336] jepsen worker 65 - jepsen.util 4295 :fail :cas [21 [14 47]] :ValueNotSwapFrom INFO [2019-12-17 12:31:33,336] jepsen worker 65 - jepsen.util 4295 :invoke :write [21 0] INFO [2019-12-17 12:31:33,337] jepsen worker 29 - jepsen.util 3899 :ok :write [9 12] INFO [2019-12-17 12:31:33,337] jepsen worker 29 - jepsen.util 3899 :invoke :write [9 0] INFO [2019-12-17 12:31:33,356] jepsen worker 54 - jepsen.util 5364 :fail :cas [18 [32 32]] :ValueNotSwapFrom INFO [2019-12-17 12:31:33,357] jepsen worker 54 - jepsen.util 5364 :invoke :read [18 nil] INFO [2019-12-17 12:31:33,357] jepsen worker 14 - jepsen.util 5054 :info :write [4 20] :RequestCanceledException INFO [2019-12-17 12:31:33,357] jepsen worker 14 - jepsen.util 5144 :invoke :cas [4 [16 12]] INFO [2019-12-17 12:31:33,392] jepsen worker 10 - jepsen.util 4870 :fail :read [3 nil] :RequestTimeoutException INFO [2019-12-17 12:31:33,392] jepsen worker 10 - jepsen.util 4870 :invoke :cas [3 [15 19]] INFO [2019-12-17 12:31:33,395] jepsen worker 65 - jepsen.util 4295 :ok :write [21 0] INFO [2019-12-17 12:31:33,396] jepsen worker 65 - jepsen.util 4295 :invoke :write [21 25] INFO [2019-12-17 12:31:33,440] jepsen worker 53 - jepsen.util 4823 :info :write [17 32] :RequestCanceledException INFO [2019-12-17 12:31:33,440] jepsen worker 53 - jepsen.util 4913 :invoke :cas [17 [5 7]] INFO [2019-12-17 12:31:33,442] jepsen worker 65 - jepsen.util 4295 :ok :write [21 25] INFO [2019-12-17 12:31:33,442] jepsen worker 65 - jepsen.util 4295 :invoke :write [21 15] INFO [2019-12-17 12:31:33,455] jepsen worker 88 - jepsen.util 5038 :fail :cas [29 [47 21]] :ValueNotSwapFrom INFO [2019-12-17 12:31:33,456] jepsen worker 88 - jepsen.util 5038 :invoke :write [29 26] INFO [2019-12-17 12:31:33,481] jepsen worker 65 - jepsen.util 4295 :ok :write [21 15] INFO [2019-12-17 12:31:33,481] jepsen worker 65 - jepsen.util 4295 :invoke :read [21 nil] INFO [2019-12-17 12:31:33,482] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28079,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:31:33,482] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28079,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:31:33,482] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28081,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:31:33,482] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28081,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:31:33,493] jepsen worker 65 - jepsen.util 4295 :ok :read [21 15] INFO [2019-12-17 12:31:33,493] jepsen worker 65 - jepsen.util 4295 :invoke :read [21 nil] INFO [2019-12-17 12:31:33,505] jepsen worker 65 - jepsen.util 4295 :ok :read [21 15] INFO [2019-12-17 12:31:33,505] jepsen worker 65 - jepsen.util 4295 :invoke :cas [21 [0 40]] INFO [2019-12-17 12:31:33,509] jepsen worker 45 - jepsen.util 4995 :info :cas [15 [40 29]] :RequestCanceledException INFO [2019-12-17 12:31:33,510] jepsen worker 45 - jepsen.util 5085 :invoke :write [15 43] INFO [2019-12-17 12:31:33,516] jepsen worker 65 - jepsen.util 4295 :fail :cas [21 [0 40]] :ValueNotSwapFrom INFO [2019-12-17 12:31:33,516] jepsen worker 65 - jepsen.util 4295 :invoke :write [21 27] INFO [2019-12-17 12:31:33,525] jepsen worker 19 - jepsen.util 4789 :info :write [6 17] :SyncWriteAmbiguous INFO [2019-12-17 12:31:33,526] jepsen worker 19 - jepsen.util 4879 :invoke :write [6 49] INFO [2019-12-17 12:31:33,545] jepsen worker 28 - jepsen.util 3628 :ok :read [9 12] INFO [2019-12-17 12:31:33,545] jepsen worker 28 - jepsen.util 3628 :invoke :write [9 28] INFO [2019-12-17 12:31:33,556] jepsen worker 72 - jepsen.util 4842 :info :write [24 48] :RequestCanceledException INFO [2019-12-17 12:31:33,557] jepsen worker 72 - jepsen.util 4932 :invoke :write [24 17] INFO [2019-12-17 12:31:33,569] jepsen worker 64 - jepsen.util 3664 :fail :cas [21 [30 47]] :ValueNotSwapFrom INFO [2019-12-17 12:31:33,570] jepsen worker 64 - jepsen.util 3664 :invoke :cas [21 [42 9]] INFO [2019-12-17 12:31:33,576] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:31:33,576] jepsen nemesis - couchbase.util Rebalance check count: 210 INFO [2019-12-17 12:31:33,576] jepsen nemesis - couchbase.util Rebalance stuck count: 191 INFO [2019-12-17 12:31:33,580] jepsen worker 18 - jepsen.util 4878 :ok :read [6 27] INFO [2019-12-17 12:31:33,580] jepsen worker 18 - jepsen.util 4878 :invoke :read [6 nil] INFO [2019-12-17 12:31:33,587] jepsen worker 65 - jepsen.util 4295 :ok :write [21 27] INFO [2019-12-17 12:31:33,587] jepsen worker 65 - jepsen.util 4295 :invoke :cas [21 [48 35]] INFO [2019-12-17 12:31:33,592] jepsen worker 3 - jepsen.util 4953 :info :write [1 40] :RequestCanceledException INFO [2019-12-17 12:31:33,593] jepsen worker 3 - jepsen.util 5043 :invoke :cas [1 [41 32]] INFO [2019-12-17 12:31:33,611] jepsen worker 28 - jepsen.util 3628 :ok :write [9 28] INFO [2019-12-17 12:31:33,612] jepsen worker 28 - jepsen.util 3628 :invoke :write [9 17] INFO [2019-12-17 12:31:33,625] jepsen worker 56 - jepsen.util 5006 :info :write [18 42] :RequestCanceledException INFO [2019-12-17 12:31:33,626] jepsen worker 64 - jepsen.util 3664 :fail :cas [21 [42 9]] :ValueNotSwapFrom INFO [2019-12-17 12:31:33,626] jepsen worker 64 - jepsen.util 3664 :invoke :write [21 18] INFO [2019-12-17 12:31:33,626] jepsen worker 56 - jepsen.util 5096 :invoke :cas [18 [37 8]] INFO [2019-12-17 12:31:33,639] jepsen worker 13 - jepsen.util 4513 :info :write [4 29] :RequestCanceledException INFO [2019-12-17 12:31:33,640] jepsen worker 13 - jepsen.util 4603 :invoke :write [4 42] INFO [2019-12-17 12:31:33,651] jepsen worker 18 - jepsen.util 4878 :ok :read [6 27] INFO [2019-12-17 12:31:33,651] jepsen worker 18 - jepsen.util 4878 :invoke :read [6 nil] INFO [2019-12-17 12:31:33,669] jepsen worker 37 - jepsen.util 4627 :ok :read [12 11] INFO [2019-12-17 12:31:33,669] jepsen worker 37 - jepsen.util 4627 :invoke :read [12 nil] INFO [2019-12-17 12:31:33,676] jepsen worker 28 - jepsen.util 3628 :ok :write [9 17] INFO [2019-12-17 12:31:33,677] jepsen worker 28 - jepsen.util 3628 :invoke :write [9 10] INFO [2019-12-17 12:31:33,692] jepsen worker 32 - jepsen.util 4982 :info :write [10 41] :RequestCanceledException INFO [2019-12-17 12:31:33,693] jepsen worker 32 - jepsen.util 5072 :invoke :cas [10 [27 9]] INFO [2019-12-17 12:31:33,701] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28101,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:31:33,701] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28101,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:31:33,701] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28104,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:31:33,701] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28104,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:31:33,701] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28105,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:31:33,702] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28105,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:31:33,713] jepsen worker 64 - jepsen.util 3664 :ok :write [21 18] INFO [2019-12-17 12:31:33,715] jepsen worker 64 - jepsen.util 3664 :invoke :write [21 8] INFO [2019-12-17 12:31:33,725] jepsen worker 27 - jepsen.util 4707 :info :write [9 44] :RequestCanceledException INFO [2019-12-17 12:31:33,726] jepsen worker 27 - jepsen.util 4797 :invoke :write [9 35] INFO [2019-12-17 12:31:33,739] jepsen worker 63 - jepsen.util 3933 :fail :read [21 nil] :RequestTimeoutException INFO [2019-12-17 12:31:33,739] jepsen worker 63 - jepsen.util 3933 :invoke :write [21 42] INFO [2019-12-17 12:31:33,757] jepsen worker 30 - jepsen.util 4890 :info :write [10 10] :RequestCanceledException INFO [2019-12-17 12:31:33,757] jepsen worker 30 - jepsen.util 4980 :invoke :read [10 nil] INFO [2019-12-17 12:31:33,762] jepsen worker 64 - jepsen.util 3664 :ok :write [21 8] INFO [2019-12-17 12:31:33,763] jepsen worker 64 - jepsen.util 3664 :invoke :cas [21 [1 21]] INFO [2019-12-17 12:31:33,767] jepsen worker 37 - jepsen.util 4627 :ok :read [12 11] INFO [2019-12-17 12:31:33,767] jepsen worker 37 - jepsen.util 4627 :invoke :read [12 nil] INFO [2019-12-17 12:31:33,786] jepsen worker 53 - jepsen.util 4913 :fail :cas [17 [5 7]] :ValueNotSwapFrom INFO [2019-12-17 12:31:33,786] jepsen worker 53 - jepsen.util 4913 :invoke :write [17 32] INFO [2019-12-17 12:31:33,808] jepsen worker 63 - jepsen.util 3933 :ok :write [21 42] INFO [2019-12-17 12:31:33,808] jepsen worker 63 - jepsen.util 3933 :invoke :read [21 nil] INFO [2019-12-17 12:31:33,811] jepsen worker 70 - jepsen.util 5470 :info :write [23 40] :RequestCanceledException INFO [2019-12-17 12:31:33,812] jepsen worker 70 - jepsen.util 5560 :invoke :cas [23 [14 30]] INFO [2019-12-17 12:31:33,832] jepsen worker 63 - jepsen.util 3933 :ok :read [21 42] INFO [2019-12-17 12:31:33,832] jepsen worker 63 - jepsen.util 3933 :invoke :write [21 20] INFO [2019-12-17 12:31:33,902] jepsen worker 63 - jepsen.util 3933 :ok :write [21 20] INFO [2019-12-17 12:31:33,902] jepsen worker 63 - jepsen.util 3933 :invoke :read [21 nil] INFO [2019-12-17 12:31:33,920] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28123,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:31:33,921] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28123,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:31:33,921] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28127,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:31:33,921] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28127,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:31:33,921] jepsen worker 63 - jepsen.util 3933 :ok :read [21 20] INFO [2019-12-17 12:31:33,921] jepsen worker 63 - jepsen.util 3933 :invoke :write [21 41] INFO [2019-12-17 12:31:33,949] jepsen worker 38 - jepsen.util 4628 :info :write [12 19] :RequestCanceledException INFO [2019-12-17 12:31:33,949] jepsen worker 78 - jepsen.util 4848 :info :write [26 34] :RequestCanceledException INFO [2019-12-17 12:31:33,950] jepsen worker 78 - jepsen.util 4938 :invoke :cas [26 [39 15]] INFO [2019-12-17 12:31:33,950] jepsen worker 38 - jepsen.util 4718 :invoke :read [12 nil] INFO [2019-12-17 12:31:33,992] jepsen worker 69 - jepsen.util 5109 :info :write [23 43] :RequestCanceledException INFO [2019-12-17 12:31:33,993] jepsen worker 69 - jepsen.util 5199 :invoke :cas [23 [36 14]] INFO [2019-12-17 12:31:34,000] jepsen worker 79 - jepsen.util 4849 :fail :cas [26 [34 17]] :ValueNotSwapFrom INFO [2019-12-17 12:31:34,000] jepsen worker 79 - jepsen.util 4849 :invoke :cas [26 [49 45]] INFO [2019-12-17 12:31:34,001] jepsen worker 63 - jepsen.util 3933 :ok :write [21 41] INFO [2019-12-17 12:31:34,002] jepsen worker 63 - jepsen.util 3933 :invoke :read [21 nil] INFO [2019-12-17 12:31:34,017] jepsen worker 63 - jepsen.util 3933 :ok :read [21 41] INFO [2019-12-17 12:31:34,017] jepsen worker 63 - jepsen.util 3933 :invoke :read [21 nil] INFO [2019-12-17 12:31:34,030] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28136,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:31:34,035] jepsen worker 63 - jepsen.util 3933 :ok :read [21 41] INFO [2019-12-17 12:31:34,035] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28136,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:31:34,035] jepsen worker 63 - jepsen.util 3933 :invoke :read [21 nil] INFO [2019-12-17 12:31:34,035] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28138,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:31:34,036] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28138,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:31:34,041] jepsen worker 37 - jepsen.util 4627 :ok :read [12 11] INFO [2019-12-17 12:31:34,042] jepsen worker 37 - jepsen.util 4627 :invoke :write [12 2] INFO [2019-12-17 12:31:34,056] jepsen worker 63 - jepsen.util 3933 :ok :read [21 41] INFO [2019-12-17 12:31:34,056] jepsen worker 63 - jepsen.util 3933 :invoke :write [21 23] INFO [2019-12-17 12:31:34,080] jepsen worker 14 - jepsen.util 5144 :fail :cas [4 [16 12]] :ValueNotSwapFrom INFO [2019-12-17 12:31:34,080] jepsen worker 14 - jepsen.util 5144 :invoke :read [4 nil] INFO [2019-12-17 12:31:34,094] jepsen worker 56 - jepsen.util 5096 :fail :cas [18 [37 8]] :ValueNotSwapFrom INFO [2019-12-17 12:31:34,094] jepsen worker 56 - jepsen.util 5096 :invoke :cas [18 [42 37]] INFO [2019-12-17 12:31:34,106] jepsen worker 63 - jepsen.util 3933 :ok :write [21 23] INFO [2019-12-17 12:31:34,106] jepsen worker 63 - jepsen.util 3933 :invoke :cas [21 [8 12]] INFO [2019-12-17 12:31:34,121] jepsen worker 63 - jepsen.util 3933 :fail :cas [21 [8 12]] :ValueNotSwapFrom INFO [2019-12-17 12:31:34,122] jepsen worker 63 - jepsen.util 3933 :invoke :read [21 nil] INFO [2019-12-17 12:31:34,137] jepsen worker 63 - jepsen.util 3933 :ok :read [21 23] INFO [2019-12-17 12:31:34,137] jepsen worker 63 - jepsen.util 3933 :invoke :write [21 33] INFO [2019-12-17 12:31:34,189] jepsen worker 63 - jepsen.util 3933 :ok :write [21 33] INFO [2019-12-17 12:31:34,189] jepsen worker 63 - jepsen.util 3933 :invoke :write [21 5] INFO [2019-12-17 12:31:34,194] jepsen worker 73 - jepsen.util 4573 :ok :cas [24 [43 44]] INFO [2019-12-17 12:31:34,194] jepsen worker 73 - jepsen.util 4573 :invoke :write [24 0] INFO [2019-12-17 12:31:34,225] jepsen worker 46 - jepsen.util 4906 :fail :cas [15 [34 11]] :ValueNotSwapFrom INFO [2019-12-17 12:31:34,225] jepsen worker 46 - jepsen.util 4906 :invoke :read [15 nil] INFO [2019-12-17 12:31:34,226] jepsen worker 55 - jepsen.util 5815 :info :write [18 16] :RequestCanceledException INFO [2019-12-17 12:31:34,228] jepsen worker 55 - jepsen.util 5905 :invoke :read [18 nil] INFO [2019-12-17 12:31:34,243] jepsen worker 78 - jepsen.util 4938 :fail :cas [26 [39 15]] :ValueNotSwapFrom INFO [2019-12-17 12:31:34,243] jepsen worker 78 - jepsen.util 4938 :invoke :cas [26 [21 46]] INFO [2019-12-17 12:31:34,249] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28159,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:31:34,250] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28159,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:31:34,250] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28163,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:31:34,250] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28163,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:31:34,250] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28150,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:31:34,250] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28150,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:31:34,250] jepsen worker 63 - jepsen.util 3933 :ok :write [21 5] INFO [2019-12-17 12:31:34,250] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28152,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:31:34,251] jepsen worker 63 - jepsen.util 3933 :invoke :read [21 nil] INFO [2019-12-17 12:31:34,251] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28152,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:31:34,263] jepsen worker 10 - jepsen.util 4870 :fail :cas [3 [15 19]] :ValueNotSwapFrom INFO [2019-12-17 12:31:34,264] jepsen worker 10 - jepsen.util 4870 :invoke :read [3 nil] INFO [2019-12-17 12:31:34,270] jepsen worker 69 - jepsen.util 5199 :fail :cas [23 [36 14]] :ValueNotSwapFrom INFO [2019-12-17 12:31:34,270] jepsen worker 69 - jepsen.util 5199 :invoke :write [23 2] INFO [2019-12-17 12:31:34,272] jepsen worker 63 - jepsen.util 3933 :ok :read [21 5] INFO [2019-12-17 12:31:34,272] jepsen worker 63 - jepsen.util 3933 :invoke :read [21 nil] INFO [2019-12-17 12:31:34,287] jepsen worker 63 - jepsen.util 3933 :ok :read [21 5] INFO [2019-12-17 12:31:34,287] jepsen worker 63 - jepsen.util 3933 :invoke :cas [21 [2 24]] INFO [2019-12-17 12:31:34,306] jepsen worker 63 - jepsen.util 3933 :fail :cas [21 [2 24]] :ValueNotSwapFrom INFO [2019-12-17 12:31:34,306] jepsen worker 63 - jepsen.util 3933 :invoke :write [21 49] INFO [2019-12-17 12:31:34,306] jepsen worker 87 - jepsen.util 4587 :fail :cas [29 [45 18]] :ValueNotSwapFrom INFO [2019-12-17 12:31:34,311] jepsen worker 82 - jepsen.util 7282 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:31:34,320] jepsen worker 87 - jepsen.util 4587 :invoke :cas [29 [7 43]] INFO [2019-12-17 12:31:34,320] jepsen worker 82 - jepsen.util 7282 :invoke :read [27 nil] INFO [2019-12-17 12:31:34,324] jepsen worker 68 - jepsen.util 6008 :info :write [22 47] :RequestCanceledException INFO [2019-12-17 12:31:34,324] jepsen worker 51 - jepsen.util 4731 :info :write [17 47] :RequestCanceledException INFO [2019-12-17 12:31:34,324] jepsen worker 84 - jepsen.util 6204 :info :write [28 36] :RequestCanceledException INFO [2019-12-17 12:31:34,324] jepsen worker 5 - jepsen.util 4325 :info :write [1 22] :RequestCanceledException INFO [2019-12-17 12:31:34,325] jepsen worker 51 - jepsen.util 4821 :invoke :cas [17 [15 38]] INFO [2019-12-17 12:31:34,326] jepsen worker 68 - jepsen.util 6098 :invoke :cas [22 [17 2]] INFO [2019-12-17 12:31:34,326] jepsen worker 84 - jepsen.util 6294 :invoke :write [28 20] INFO [2019-12-17 12:31:34,326] jepsen worker 5 - jepsen.util 4415 :invoke :cas [1 [11 44]] INFO [2019-12-17 12:31:34,340] jepsen worker 26 - jepsen.util 6866 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:31:34,340] jepsen worker 26 - jepsen.util 6866 :invoke :read [8 nil] INFO [2019-12-17 12:31:34,347] jepsen worker 63 - jepsen.util 3933 :ok :write [21 49] INFO [2019-12-17 12:31:34,348] jepsen worker 63 - jepsen.util 3933 :invoke :cas [21 [47 44]] INFO [2019-12-17 12:31:34,349] jepsen worker 38 - jepsen.util 4718 :ok :read [12 11] INFO [2019-12-17 12:31:34,349] jepsen worker 38 - jepsen.util 4718 :invoke :cas [12 [42 44]] INFO [2019-12-17 12:31:34,352] jepsen worker 9 - jepsen.util 5769 :ok :write [3 0] INFO [2019-12-17 12:31:34,352] jepsen worker 45 - jepsen.util 5085 :ok :write [15 43] INFO [2019-12-17 12:31:34,352] jepsen worker 9 - jepsen.util 5769 :invoke :write [3 8] INFO [2019-12-17 12:31:34,352] jepsen worker 45 - jepsen.util 5085 :invoke :cas [15 [13 15]] INFO [2019-12-17 12:31:34,356] jepsen worker 31 - jepsen.util 4801 :ok :write [10 10] INFO [2019-12-17 12:31:34,356] jepsen worker 31 - jepsen.util 4801 :invoke :read [10 nil] INFO [2019-12-17 12:31:34,370] jepsen worker 63 - jepsen.util 3933 :fail :cas [21 [47 44]] :ValueNotSwapFrom INFO [2019-12-17 12:31:34,370] jepsen worker 63 - jepsen.util 3933 :invoke :write [21 17] INFO [2019-12-17 12:31:34,392] jepsen worker 16 - jepsen.util 7576 :info :write [5 30] :RequestCanceledException INFO [2019-12-17 12:31:34,392] jepsen worker 33 - jepsen.util 6333 :info :write [11 24] :RequestCanceledException INFO [2019-12-17 12:31:34,392] jepsen worker 6 - jepsen.util 6846 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:31:34,393] jepsen worker 6 - jepsen.util 6846 :invoke :read [2 nil] INFO [2019-12-17 12:31:34,394] jepsen worker 33 - jepsen.util 6423 :invoke :cas [11 [7 42]] INFO [2019-12-17 12:31:34,400] jepsen worker 16 - jepsen.util 7666 :invoke :read [5 nil] INFO [2019-12-17 12:31:34,402] jepsen worker 32 - jepsen.util 5072 :fail :cas [10 [27 9]] :ValueNotSwapFrom INFO [2019-12-17 12:31:34,402] jepsen worker 32 - jepsen.util 5072 :invoke :read [10 nil] INFO [2019-12-17 12:31:34,403] jepsen worker 54 - jepsen.util 5364 :ok :read [18 34] INFO [2019-12-17 12:31:34,403] jepsen worker 54 - jepsen.util 5364 :invoke :cas [18 [40 10]] INFO [2019-12-17 12:31:34,410] jepsen worker 8 - jepsen.util 5858 :info :write [2 39] :RequestCanceledException INFO [2019-12-17 12:31:34,412] jepsen worker 8 - jepsen.util 5948 :invoke :write [2 41] INFO [2019-12-17 12:31:34,415] jepsen worker 31 - jepsen.util 4801 :ok :read [10 10] INFO [2019-12-17 12:31:34,416] jepsen worker 31 - jepsen.util 4801 :invoke :read [10 nil] INFO [2019-12-17 12:31:34,422] jepsen worker 45 - jepsen.util 5085 :fail :cas [15 [13 15]] :ValueNotSwapFrom INFO [2019-12-17 12:31:34,423] jepsen worker 45 - jepsen.util 5085 :invoke :write [15 34] INFO [2019-12-17 12:31:34,426] jepsen worker 49 - jepsen.util 5809 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:31:34,426] jepsen worker 81 - jepsen.util 6381 :info :write [27 46] :RequestCanceledException INFO [2019-12-17 12:31:34,426] jepsen worker 34 - jepsen.util 5974 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:31:34,426] jepsen worker 49 - jepsen.util 5809 :invoke :read [16 nil] INFO [2019-12-17 12:31:34,427] jepsen worker 34 - jepsen.util 5974 :invoke :cas [11 [43 24]] INFO [2019-12-17 12:31:34,439] jepsen worker 81 - jepsen.util 6471 :invoke :cas [27 [4 32]] INFO [2019-12-17 12:31:34,440] jepsen worker 76 - jepsen.util 6556 :info :cas [25 [13 18]] :RequestCanceledException INFO [2019-12-17 12:31:34,440] jepsen worker 58 - jepsen.util 6718 :info :write [19 15] :RequestCanceledException INFO [2019-12-17 12:31:34,440] jepsen worker 43 - jepsen.util 6163 :info :cas [14 [38 2]] :RequestCanceledException INFO [2019-12-17 12:31:34,442] jepsen worker 58 - jepsen.util 6808 :invoke :read [19 nil] INFO [2019-12-17 12:31:34,442] jepsen worker 76 - jepsen.util 6646 :invoke :write [25 37] INFO [2019-12-17 12:31:34,452] jepsen worker 43 - jepsen.util 6253 :invoke :read [14 nil] INFO [2019-12-17 12:31:34,454] jepsen worker 5 - jepsen.util 4415 :fail :cas [1 [11 44]] :ValueNotSwapFrom INFO [2019-12-17 12:31:34,454] jepsen worker 5 - jepsen.util 4415 :invoke :write [1 45] INFO [2019-12-17 12:31:34,456] jepsen worker 31 - jepsen.util 4801 :ok :read [10 10] INFO [2019-12-17 12:31:34,456] jepsen worker 79 - jepsen.util 4849 :fail :cas [26 [49 45]] :ValueNotSwapFrom INFO [2019-12-17 12:31:34,456] jepsen worker 31 - jepsen.util 4801 :invoke :write [10 43] INFO [2019-12-17 12:31:34,456] jepsen worker 79 - jepsen.util 4849 :invoke :cas [26 [37 41]] INFO [2019-12-17 12:31:34,457] jepsen worker 57 - jepsen.util 6087 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:31:34,457] jepsen worker 25 - jepsen.util 6685 :info :write [8 30] :RequestCanceledException INFO [2019-12-17 12:31:34,457] jepsen worker 15 - jepsen.util 5505 :info :cas [5 [32 10]] :RequestCanceledException INFO [2019-12-17 12:31:34,457] jepsen worker 57 - jepsen.util 6087 :invoke :read [19 nil] INFO [2019-12-17 12:31:34,459] jepsen worker 25 - jepsen.util 6775 :invoke :write [8 36] INFO [2019-12-17 12:31:34,459] jepsen worker 15 - jepsen.util 5595 :invoke :write [5 11] INFO [2019-12-17 12:31:34,469] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28194,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:31:34,470] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28194,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:31:34,470] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28197,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:31:34,470] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28197,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:31:34,470] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28198,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:31:34,470] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28198,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:31:34,483] jepsen worker 32 - jepsen.util 5072 :ok :read [10 10] INFO [2019-12-17 12:31:34,483] jepsen worker 32 - jepsen.util 5072 :invoke :cas [10 [15 39]] INFO [2019-12-17 12:31:34,492] jepsen worker 60 - jepsen.util 7080 :info :cas [20 [7 44]] :RequestCanceledException INFO [2019-12-17 12:31:34,492] jepsen worker 40 - jepsen.util 7240 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:31:34,492] jepsen worker 40 - jepsen.util 7240 :invoke :read [13 nil] INFO [2019-12-17 12:31:34,494] jepsen worker 60 - jepsen.util 7170 :invoke :write [20 14] INFO [2019-12-17 12:31:34,510] jepsen worker 48 - jepsen.util 5988 :info :write [16 14] :RequestCanceledException INFO [2019-12-17 12:31:34,510] jepsen worker 17 - jepsen.util 6407 :info :write [5 4] :RequestCanceledException INFO [2019-12-17 12:31:34,510] jepsen worker 62 - jepsen.util 6002 :info :write [20 35] :RequestCanceledException INFO [2019-12-17 12:31:34,513] jepsen worker 62 - jepsen.util 6092 :invoke :write [20 22] INFO [2019-12-17 12:31:34,513] jepsen worker 48 - jepsen.util 6078 :invoke :cas [16 [19 46]] INFO [2019-12-17 12:31:34,513] jepsen worker 17 - jepsen.util 6497 :invoke :cas [5 [39 18]] INFO [2019-12-17 12:31:34,526] jepsen worker 85 - jepsen.util 7375 :info :cas [28 [23 19]] :RequestCanceledException INFO [2019-12-17 12:31:34,526] jepsen worker 66 - jepsen.util 5916 :info :write [22 49] :RequestCanceledException INFO [2019-12-17 12:31:34,526] jepsen worker 77 - jepsen.util 6827 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:31:34,526] jepsen worker 75 - jepsen.util 6465 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:31:34,526] jepsen worker 89 - jepsen.util 4499 :info :write [29 17] :RequestCanceledException INFO [2019-12-17 12:31:34,526] jepsen worker 50 - jepsen.util 6890 :info :cas [16 [49 23]] :RequestCanceledException INFO [2019-12-17 12:31:34,526] jepsen worker 77 - jepsen.util 6827 :invoke :read [25 nil] INFO [2019-12-17 12:31:34,526] jepsen worker 75 - jepsen.util 6465 :invoke :cas [25 [34 19]] INFO [2019-12-17 12:31:34,528] jepsen worker 66 - jepsen.util 6006 :invoke :cas [22 [29 3]] INFO [2019-12-17 12:31:34,528] jepsen worker 85 - jepsen.util 7465 :invoke :cas [28 [35 6]] INFO [2019-12-17 12:31:34,528] jepsen worker 50 - jepsen.util 6980 :invoke :cas [16 [47 5]] INFO [2019-12-17 12:31:34,528] jepsen worker 89 - jepsen.util 4589 :invoke :read [29 nil] INFO [2019-12-17 12:31:34,537] jepsen worker 63 - jepsen.util 3933 :ok :write [21 17] INFO [2019-12-17 12:31:34,537] jepsen worker 63 - jepsen.util 3933 :invoke :read [21 nil] INFO [2019-12-17 12:31:34,540] jepsen worker 7 - jepsen.util 6667 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:31:34,540] jepsen worker 83 - jepsen.util 6923 :info :write [27 40] :RequestCanceledException INFO [2019-12-17 12:31:34,540] jepsen worker 7 - jepsen.util 6667 :invoke :cas [2 [14 24]] INFO [2019-12-17 12:31:34,542] jepsen worker 83 - jepsen.util 7013 :invoke :cas [27 [15 19]] INFO [2019-12-17 12:31:34,549] jepsen worker 72 - jepsen.util 4932 :ok :write [24 17] INFO [2019-12-17 12:31:34,550] jepsen worker 72 - jepsen.util 4932 :invoke :write [24 18] INFO [2019-12-17 12:31:34,551] jepsen worker 19 - jepsen.util 4879 :ok :write [6 49] INFO [2019-12-17 12:31:34,551] jepsen worker 19 - jepsen.util 4879 :invoke :write [6 48] INFO [2019-12-17 12:31:34,556] jepsen worker 39 - jepsen.util 6069 :info :write [13 39] :RequestCanceledException INFO [2019-12-17 12:31:34,556] jepsen worker 1 - jepsen.util 6301 :info :cas [0 [4 2]] :RequestCanceledException INFO [2019-12-17 12:31:34,559] jepsen worker 9 - jepsen.util 5769 :ok :write [3 8] INFO [2019-12-17 12:31:34,559] jepsen worker 9 - jepsen.util 5769 :invoke :cas [3 [27 23]] INFO [2019-12-17 12:31:34,562] jepsen worker 63 - jepsen.util 3933 :ok :read [21 17] INFO [2019-12-17 12:31:34,563] jepsen worker 63 - jepsen.util 3933 :invoke :cas [21 [35 12]] INFO [2019-12-17 12:31:34,587] jepsen worker 9 - jepsen.util 5769 :fail :cas [3 [27 23]] :ValueNotSwapFrom INFO [2019-12-17 12:31:34,587] jepsen worker 9 - jepsen.util 5769 :invoke :cas [3 [13 13]] INFO [2019-12-17 12:31:34,588] jepsen worker 1 - jepsen.util 6391 :invoke :cas [0 [49 15]] INFO [2019-12-17 12:31:34,591] jepsen worker 14 - jepsen.util 5144 :ok :read [4 48] INFO [2019-12-17 12:31:34,591] jepsen worker 14 - jepsen.util 5144 :invoke :cas [4 [3 28]] INFO [2019-12-17 12:31:34,593] jepsen worker 22 - jepsen.util 6592 :info :write [7 22] :RequestCanceledException INFO [2019-12-17 12:31:34,594] jepsen worker 39 - jepsen.util 6159 :invoke :cas [13 [38 42]] INFO [2019-12-17 12:31:34,595] jepsen worker 22 - jepsen.util 6682 :invoke :write [7 11] INFO [2019-12-17 12:31:34,597] jepsen worker 63 - jepsen.util 3933 :fail :cas [21 [35 12]] :ValueNotSwapFrom INFO [2019-12-17 12:31:34,598] jepsen worker 63 - jepsen.util 3933 :invoke :read [21 nil] INFO [2019-12-17 12:31:34,608] jepsen worker 9 - jepsen.util 5769 :fail :cas [3 [13 13]] :ValueNotSwapFrom INFO [2019-12-17 12:31:34,609] jepsen worker 9 - jepsen.util 5769 :invoke :write [3 26] INFO [2019-12-17 12:31:34,609] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:31:34,610] jepsen nemesis - couchbase.util Rebalance check count: 211 INFO [2019-12-17 12:31:34,610] jepsen nemesis - couchbase.util Rebalance stuck count: 192 INFO [2019-12-17 12:31:34,610] jepsen worker 2 - jepsen.util 5942 :info :cas [0 [11 6]] :RequestCanceledException INFO [2019-12-17 12:31:34,610] jepsen worker 42 - jepsen.util 6972 :info :write [14 37] :RequestCanceledException INFO [2019-12-17 12:31:34,612] jepsen worker 2 - jepsen.util 6032 :invoke :write [0 29] INFO [2019-12-17 12:31:34,612] jepsen worker 42 - jepsen.util 7062 :invoke :write [14 4] INFO [2019-12-17 12:31:34,614] jepsen worker 63 - jepsen.util 3933 :ok :read [21 17] INFO [2019-12-17 12:31:34,615] jepsen worker 63 - jepsen.util 3933 :invoke :read [21 nil] INFO [2019-12-17 12:31:34,626] jepsen worker 67 - jepsen.util 6637 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:31:34,626] jepsen worker 24 - jepsen.util 5964 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:31:34,626] jepsen worker 44 - jepsen.util 7154 :info :write [14 6] :RequestCanceledException INFO [2019-12-17 12:31:34,626] jepsen worker 41 - jepsen.util 6971 :info :write [13 44] :RequestCanceledException INFO [2019-12-17 12:31:34,626] jepsen worker 0 - jepsen.util 7110 :info :write [0 14] :RequestCanceledException INFO [2019-12-17 12:31:34,626] jepsen worker 67 - jepsen.util 6637 :invoke :cas [22 [12 20]] INFO [2019-12-17 12:31:34,626] jepsen worker 24 - jepsen.util 5964 :invoke :cas [8 [26 9]] INFO [2019-12-17 12:31:34,637] jepsen worker 44 - jepsen.util 7244 :invoke :write [14 45] INFO [2019-12-17 12:31:34,637] jepsen worker 41 - jepsen.util 7061 :invoke :cas [13 [15 10]] INFO [2019-12-17 12:31:34,640] jepsen worker 27 - jepsen.util 4797 :ok :write [9 35] INFO [2019-12-17 12:31:34,640] jepsen worker 27 - jepsen.util 4797 :invoke :cas [9 [12 25]] INFO [2019-12-17 12:31:34,641] jepsen worker 63 - jepsen.util 3933 :ok :read [21 17] INFO [2019-12-17 12:31:34,641] jepsen worker 63 - jepsen.util 3933 :invoke :write [21 7] INFO [2019-12-17 12:31:34,643] jepsen worker 0 - jepsen.util 7200 :invoke :write [0 3] INFO [2019-12-17 12:31:34,652] jepsen worker 35 - jepsen.util 6605 :info :cas [11 [41 26]] :RequestCanceledException INFO [2019-12-17 12:31:34,658] jepsen worker 45 - jepsen.util 5085 :ok :write [15 34] INFO [2019-12-17 12:31:34,658] jepsen worker 45 - jepsen.util 5085 :invoke :read [15 nil] INFO [2019-12-17 12:31:34,660] jepsen worker 35 - jepsen.util 6695 :invoke :read [11 nil] INFO [2019-12-17 12:31:34,689] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28245,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:31:34,689] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28245,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:31:34,689] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28249,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:31:34,689] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28249,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:31:34,690] jepsen worker 32 - jepsen.util 5072 :fail :cas [10 [15 39]] :ValueNotSwapFrom INFO [2019-12-17 12:31:34,690] jepsen worker 32 - jepsen.util 5072 :invoke :write [10 49] INFO [2019-12-17 12:31:34,710] jepsen worker 45 - jepsen.util 5085 :ok :read [15 34] INFO [2019-12-17 12:31:34,711] jepsen worker 45 - jepsen.util 5085 :invoke :write [15 48] INFO [2019-12-17 12:31:34,720] jepsen worker 3 - jepsen.util 5043 :fail :cas [1 [41 32]] :ValueNotSwapFrom INFO [2019-12-17 12:31:34,720] jepsen worker 3 - jepsen.util 5043 :invoke :write [1 20] INFO [2019-12-17 12:31:34,726] jepsen worker 20 - jepsen.util 4970 :info :write [6 33] :RequestCanceledException INFO [2019-12-17 12:31:34,726] jepsen worker 86 - jepsen.util 6926 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:31:34,726] jepsen worker 86 - jepsen.util 6926 :invoke :write [28 5] INFO [2019-12-17 12:31:34,727] jepsen worker 20 - jepsen.util 5060 :invoke :read [6 nil] INFO [2019-12-17 12:31:34,729] jepsen worker 55 - jepsen.util 5905 :ok :read [18 34] INFO [2019-12-17 12:31:34,729] jepsen worker 55 - jepsen.util 5905 :invoke :cas [18 [32 1]] INFO [2019-12-17 12:31:34,734] jepsen worker 56 - jepsen.util 5096 :fail :cas [18 [42 37]] :ValueNotSwapFrom INFO [2019-12-17 12:31:34,735] jepsen worker 56 - jepsen.util 5096 :invoke :read [18 nil] INFO [2019-12-17 12:31:34,771] jepsen worker 70 - jepsen.util 5560 :fail :cas [23 [14 30]] :ValueNotSwapFrom INFO [2019-12-17 12:31:34,771] jepsen worker 70 - jepsen.util 5560 :invoke :cas [23 [23 18]] INFO [2019-12-17 12:31:34,800] jepsen worker 63 - jepsen.util 3933 :ok :write [21 7] INFO [2019-12-17 12:31:34,800] jepsen worker 63 - jepsen.util 3933 :invoke :write [21 47] INFO [2019-12-17 12:31:34,809] jepsen worker 55 - jepsen.util 5905 :fail :cas [18 [32 1]] :ValueNotSwapFrom INFO [2019-12-17 12:31:34,809] jepsen worker 55 - jepsen.util 5905 :invoke :cas [18 [28 14]] INFO [2019-12-17 12:31:34,857] jepsen worker 54 - jepsen.util 5364 :fail :cas [18 [40 10]] :ValueNotSwapFrom INFO [2019-12-17 12:31:34,857] jepsen worker 54 - jepsen.util 5364 :invoke :cas [18 [38 49]] INFO [2019-12-17 12:31:34,878] jepsen worker 55 - jepsen.util 5905 :fail :cas [18 [28 14]] :ValueNotSwapFrom INFO [2019-12-17 12:31:34,879] jepsen worker 55 - jepsen.util 5905 :invoke :cas [18 [41 9]] INFO [2019-12-17 12:31:34,895] jepsen worker 63 - jepsen.util 3933 :ok :write [21 47] INFO [2019-12-17 12:31:34,896] jepsen worker 63 - jepsen.util 3933 :invoke :cas [21 [1 45]] INFO [2019-12-17 12:31:34,908] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28268,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:31:34,909] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28268,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:31:34,909] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28272,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:31:34,909] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28272,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:31:34,915] jepsen worker 78 - jepsen.util 4938 :fail :cas [26 [21 46]] :ValueNotSwapFrom INFO [2019-12-17 12:31:34,915] jepsen worker 78 - jepsen.util 4938 :invoke :write [26 14] INFO [2019-12-17 12:31:34,921] jepsen worker 63 - jepsen.util 3933 :fail :cas [21 [1 45]] :ValueNotSwapFrom INFO [2019-12-17 12:31:34,922] jepsen worker 63 - jepsen.util 3933 :invoke :write [21 11] INFO [2019-12-17 12:31:34,933] jepsen worker 70 - jepsen.util 5560 :fail :cas [23 [23 18]] :ValueNotSwapFrom INFO [2019-12-17 12:31:34,933] jepsen worker 70 - jepsen.util 5560 :invoke :cas [23 [43 48]] INFO [2019-12-17 12:31:34,934] jepsen worker 51 - jepsen.util 4821 :fail :cas [17 [15 38]] :ValueNotSwapFrom INFO [2019-12-17 12:31:34,934] jepsen worker 51 - jepsen.util 4821 :invoke :read [17 nil] INFO [2019-12-17 12:31:34,959] jepsen worker 11 - jepsen.util 4781 :info :write [3 39] :RequestCanceledException INFO [2019-12-17 12:31:34,959] jepsen worker 59 - jepsen.util 6359 :info :cas [19 [40 31]] :RequestCanceledException INFO [2019-12-17 12:31:34,962] jepsen worker 59 - jepsen.util 6449 :invoke :write [19 15] INFO [2019-12-17 12:31:34,964] jepsen worker 11 - jepsen.util 4871 :invoke :cas [3 [46 20]] INFO [2019-12-17 12:31:34,979] jepsen worker 79 - jepsen.util 4849 :fail :cas [26 [37 41]] :ValueNotSwapFrom INFO [2019-12-17 12:31:34,979] jepsen worker 79 - jepsen.util 4849 :invoke :read [26 nil] INFO [2019-12-17 12:31:34,988] jepsen worker 63 - jepsen.util 3933 :ok :write [21 11] INFO [2019-12-17 12:31:34,989] jepsen worker 63 - jepsen.util 3933 :invoke :cas [21 [27 0]] INFO [2019-12-17 12:31:34,993] jepsen worker 47 - jepsen.util 5447 :info :write [15 27] :RequestCanceledException INFO [2019-12-17 12:31:34,995] jepsen worker 47 - jepsen.util 5537 :invoke :read [15 nil] INFO [2019-12-17 12:31:35,004] jepsen worker 55 - jepsen.util 5905 :fail :cas [18 [41 9]] :ValueNotSwapFrom INFO [2019-12-17 12:31:35,005] jepsen worker 55 - jepsen.util 5905 :invoke :read [18 nil] INFO [2019-12-17 12:31:35,014] jepsen worker 61 - jepsen.util 7351 :info :write [20 9] :RequestCanceledException INFO [2019-12-17 12:31:35,014] jepsen worker 71 - jepsen.util 5291 :info :write [23 30] :RequestCanceledException INFO [2019-12-17 12:31:35,016] jepsen worker 71 - jepsen.util 5381 :invoke :write [23 9] INFO [2019-12-17 12:31:35,016] jepsen worker 61 - jepsen.util 7441 :invoke :write [20 40] INFO [2019-12-17 12:31:35,020] jepsen worker 11 - jepsen.util 4871 :fail :cas [3 [46 20]] :ValueNotSwapFrom INFO [2019-12-17 12:31:35,021] jepsen worker 11 - jepsen.util 4871 :invoke :cas [3 [15 49]] INFO [2019-12-17 12:31:35,021] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28284,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:31:35,021] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28284,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:31:35,021] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28289,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:31:35,021] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28289,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:31:35,023] jepsen worker 63 - jepsen.util 3933 :fail :cas [21 [27 0]] :ValueNotSwapFrom INFO [2019-12-17 12:31:35,023] jepsen worker 63 - jepsen.util 3933 :invoke :read [21 nil] INFO [2019-12-17 12:31:35,024] jepsen worker 4 - jepsen.util 4054 :info :write [1 9] :RequestCanceledException INFO [2019-12-17 12:31:35,024] jepsen worker 52 - jepsen.util 5182 :info :write [17 21] :RequestCanceledException INFO [2019-12-17 12:31:35,024] jepsen worker 23 - jepsen.util 6683 :info :write [7 10] :RequestCanceledException INFO [2019-12-17 12:31:35,024] jepsen worker 21 - jepsen.util 5961 :info :write [7 7] :RequestCanceledException INFO [2019-12-17 12:31:35,026] jepsen worker 23 - jepsen.util 6773 :invoke :cas [7 [42 17]] INFO [2019-12-17 12:31:35,026] jepsen worker 52 - jepsen.util 5272 :invoke :cas [17 [24 40]] INFO [2019-12-17 12:31:35,026] jepsen worker 4 - jepsen.util 4144 :invoke :write [1 13] INFO [2019-12-17 12:31:35,028] jepsen worker 21 - jepsen.util 6051 :invoke :read [7 nil] INFO [2019-12-17 12:31:35,047] jepsen worker 11 - jepsen.util 4871 :fail :cas [3 [15 49]] :ValueNotSwapFrom INFO [2019-12-17 12:31:35,048] jepsen worker 11 - jepsen.util 4871 :invoke :write [3 12] INFO [2019-12-17 12:31:35,051] jepsen worker 63 - jepsen.util 3933 :ok :read [21 11] INFO [2019-12-17 12:31:35,051] jepsen worker 63 - jepsen.util 3933 :invoke :write [21 30] INFO [2019-12-17 12:31:35,082] jepsen worker 89 - jepsen.util 4589 :ok :read [29 7] INFO [2019-12-17 12:31:35,082] jepsen worker 89 - jepsen.util 4589 :invoke :cas [29 [40 38]] INFO [2019-12-17 12:31:35,099] jepsen worker 51 - jepsen.util 4821 :ok :read [17 13] INFO [2019-12-17 12:31:35,100] jepsen worker 51 - jepsen.util 4821 :invoke :cas [17 [15 30]] INFO [2019-12-17 12:31:35,106] jepsen worker 80 - jepsen.util 4940 :info :write [26 23] :RequestCanceledException INFO [2019-12-17 12:31:35,108] jepsen worker 80 - jepsen.util 5030 :invoke :cas [26 [45 36]] INFO [2019-12-17 12:31:35,138] jepsen worker 55 - jepsen.util 5905 :ok :read [18 34] INFO [2019-12-17 12:31:35,138] jepsen worker 55 - jepsen.util 5905 :invoke :write [18 12] INFO [2019-12-17 12:31:35,156] jepsen worker 38 - jepsen.util 4718 :fail :cas [12 [42 44]] :ValueNotSwapFrom INFO [2019-12-17 12:31:35,156] jepsen worker 38 - jepsen.util 4718 :invoke :write [12 40] INFO [2019-12-17 12:31:35,174] jepsen worker 31 - jepsen.util 4801 :ok :write [10 43] INFO [2019-12-17 12:31:35,174] jepsen worker 31 - jepsen.util 4801 :invoke :write [10 34] INFO [2019-12-17 12:31:35,184] jepsen worker 54 - jepsen.util 5364 :fail :cas [18 [38 49]] :ValueNotSwapFrom INFO [2019-12-17 12:31:35,184] jepsen worker 54 - jepsen.util 5364 :invoke :read [18 nil] INFO [2019-12-17 12:31:35,197] jepsen worker 20 - jepsen.util 5060 :ok :read [6 49] INFO [2019-12-17 12:31:35,197] jepsen worker 20 - jepsen.util 5060 :invoke :write [6 27] INFO [2019-12-17 12:31:35,219] jepsen worker 14 - jepsen.util 5144 :fail :cas [4 [3 28]] :ValueNotSwapFrom INFO [2019-12-17 12:31:35,220] jepsen worker 14 - jepsen.util 5144 :invoke :read [4 nil] INFO [2019-12-17 12:31:35,232] jepsen worker 56 - jepsen.util 5096 :ok :read [18 34] INFO [2019-12-17 12:31:35,233] jepsen worker 56 - jepsen.util 5096 :invoke :write [18 15] INFO [2019-12-17 12:31:35,236] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28316,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:31:35,236] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28316,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:31:35,236] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28320,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:31:35,237] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28320,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:31:35,237] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28321,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:31:35,237] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28321,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:31:35,240] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28310,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:31:35,240] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28310,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:31:35,254] jepsen worker 12 - jepsen.util 5052 :info :cas [4 [48 0]] :RequestCanceledException INFO [2019-12-17 12:31:35,254] jepsen worker 74 - jepsen.util 5204 :info :write [24 27] :RequestCanceledException INFO [2019-12-17 12:31:35,255] jepsen worker 74 - jepsen.util 5294 :invoke :read [24 nil] INFO [2019-12-17 12:31:35,255] jepsen worker 12 - jepsen.util 5142 :invoke :cas [4 [40 0]] INFO [2019-12-17 12:31:35,305] jepsen worker 80 - jepsen.util 5030 :fail :cas [26 [45 36]] :ValueNotSwapFrom INFO [2019-12-17 12:31:35,305] jepsen worker 80 - jepsen.util 5030 :invoke :read [26 nil] INFO [2019-12-17 12:31:35,400] jepsen worker 70 - jepsen.util 5560 :fail :cas [23 [43 48]] :ValueNotSwapFrom INFO [2019-12-17 12:31:35,400] jepsen worker 70 - jepsen.util 5560 :invoke :cas [23 [5 41]] INFO [2019-12-17 12:31:35,410] jepsen worker 29 - jepsen.util 3899 :info :write [9 0] :RequestCanceledException INFO [2019-12-17 12:31:35,411] jepsen worker 29 - jepsen.util 3989 :invoke :write [9 8] INFO [2019-12-17 12:31:35,412] jepsen worker 51 - jepsen.util 4821 :fail :cas [17 [15 30]] :ValueNotSwapFrom INFO [2019-12-17 12:31:35,413] jepsen worker 51 - jepsen.util 4821 :invoke :read [17 nil] INFO [2019-12-17 12:31:35,452] jepsen worker 19 - jepsen.util 4879 :ok :write [6 48] INFO [2019-12-17 12:31:35,452] jepsen worker 19 - jepsen.util 4879 :invoke :write [6 40] INFO [2019-12-17 12:31:35,453] jepsen worker 31 - jepsen.util 4801 :ok :write [10 34] INFO [2019-12-17 12:31:35,453] jepsen worker 31 - jepsen.util 4801 :invoke :cas [10 [43 31]] INFO [2019-12-17 12:31:35,454] jepsen worker 9 - jepsen.util 5769 :ok :write [3 26] INFO [2019-12-17 12:31:35,454] jepsen worker 9 - jepsen.util 5769 :invoke :read [3 nil] INFO [2019-12-17 12:31:35,455] jepsen worker 73 - jepsen.util 4573 :ok :write [24 0] INFO [2019-12-17 12:31:35,455] jepsen worker 73 - jepsen.util 4573 :invoke :cas [24 [8 27]] INFO [2019-12-17 12:31:35,456] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28331,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:31:35,456] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28331,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:31:35,456] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28334,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:31:35,456] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28334,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:31:35,457] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28335,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:31:35,457] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28335,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:31:35,471] jepsen worker 14 - jepsen.util 5144 :ok :read [4 48] INFO [2019-12-17 12:31:35,471] jepsen worker 14 - jepsen.util 5144 :invoke :read [4 nil] INFO [2019-12-17 12:31:35,491] jepsen worker 79 - jepsen.util 4849 :ok :read [26 29] INFO [2019-12-17 12:31:35,492] jepsen worker 79 - jepsen.util 4849 :invoke :write [26 31] INFO [2019-12-17 12:31:35,492] jepsen worker 45 - jepsen.util 5085 :ok :write [15 48] INFO [2019-12-17 12:31:35,492] jepsen worker 45 - jepsen.util 5085 :invoke :cas [15 [27 28]] INFO [2019-12-17 12:31:35,527] jepsen worker 10 - jepsen.util 4870 :ok :read [3 26] INFO [2019-12-17 12:31:35,528] jepsen worker 10 - jepsen.util 4870 :invoke :write [3 36] INFO [2019-12-17 12:31:35,530] jepsen worker 80 - jepsen.util 5030 :ok :read [26 29] INFO [2019-12-17 12:31:35,530] jepsen worker 80 - jepsen.util 5030 :invoke :write [26 35] INFO [2019-12-17 12:31:35,543] jepsen worker 70 - jepsen.util 5560 :fail :cas [23 [5 41]] :ValueNotSwapFrom INFO [2019-12-17 12:31:35,543] jepsen worker 70 - jepsen.util 5560 :invoke :cas [23 [29 9]] INFO [2019-12-17 12:31:35,555] jepsen worker 88 - jepsen.util 5038 :info :write [29 26] :RequestCanceledException INFO [2019-12-17 12:31:35,556] jepsen worker 88 - jepsen.util 5128 :invoke :write [29 19] INFO [2019-12-17 12:31:35,563] jepsen worker 47 - jepsen.util 5537 :ok :read [15 48] INFO [2019-12-17 12:31:35,563] jepsen worker 47 - jepsen.util 5537 :invoke :cas [15 [40 9]] INFO [2019-12-17 12:31:35,565] jepsen worker 29 - jepsen.util 3989 :ok :write [9 8] INFO [2019-12-17 12:31:35,565] jepsen worker 29 - jepsen.util 3989 :invoke :write [9 22] INFO [2019-12-17 12:31:35,572] jepsen worker 45 - jepsen.util 5085 :fail :cas [15 [27 28]] :ValueNotSwapFrom INFO [2019-12-17 12:31:35,572] jepsen worker 45 - jepsen.util 5085 :invoke :read [15 nil] INFO [2019-12-17 12:31:35,582] jepsen worker 47 - jepsen.util 5537 :fail :cas [15 [40 9]] :ValueNotSwapFrom INFO [2019-12-17 12:31:35,582] jepsen worker 47 - jepsen.util 5537 :invoke :cas [15 [27 2]] INFO [2019-12-17 12:31:35,585] jepsen worker 52 - jepsen.util 5272 :fail :cas [17 [24 40]] :ValueNotSwapFrom INFO [2019-12-17 12:31:35,585] jepsen worker 52 - jepsen.util 5272 :invoke :read [17 nil] INFO [2019-12-17 12:31:35,603] jepsen worker 45 - jepsen.util 5085 :ok :read [15 48] INFO [2019-12-17 12:31:35,603] jepsen worker 45 - jepsen.util 5085 :invoke :cas [15 [3 13]] INFO [2019-12-17 12:31:35,605] jepsen worker 54 - jepsen.util 5364 :ok :read [18 34] INFO [2019-12-17 12:31:35,605] jepsen worker 54 - jepsen.util 5364 :invoke :write [18 6] INFO [2019-12-17 12:31:35,625] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:31:35,625] jepsen nemesis - couchbase.util Rebalance check count: 212 INFO [2019-12-17 12:31:35,625] jepsen nemesis - couchbase.util Rebalance stuck count: 193 INFO [2019-12-17 12:31:35,625] jepsen worker 65 - jepsen.util 4295 :info :cas [21 [48 35]] :RequestCanceledException INFO [2019-12-17 12:31:35,626] jepsen worker 65 - jepsen.util 4385 :invoke :cas [21 [19 8]] INFO [2019-12-17 12:31:35,631] jepsen worker 47 - jepsen.util 5537 :fail :cas [15 [27 2]] :ValueNotSwapFrom INFO [2019-12-17 12:31:35,631] jepsen worker 47 - jepsen.util 5537 :invoke :write [15 47] INFO [2019-12-17 12:31:35,643] jepsen worker 89 - jepsen.util 4589 :fail :cas [29 [40 38]] :ValueNotSwapFrom INFO [2019-12-17 12:31:35,643] jepsen worker 89 - jepsen.util 4589 :invoke :cas [29 [25 27]] INFO [2019-12-17 12:31:35,646] jepsen worker 12 - jepsen.util 5142 :fail :cas [4 [40 0]] :ValueNotSwapFrom INFO [2019-12-17 12:31:35,646] jepsen worker 12 - jepsen.util 5142 :invoke :write [4 43] INFO [2019-12-17 12:31:35,676] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28359,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:31:35,676] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28359,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:31:35,677] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28361,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:31:35,677] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28361,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:31:35,681] jepsen worker 51 - jepsen.util 4821 :ok :read [17 13] INFO [2019-12-17 12:31:35,682] jepsen worker 51 - jepsen.util 4821 :invoke :cas [17 [37 39]] INFO [2019-12-17 12:31:35,684] jepsen worker 45 - jepsen.util 5085 :fail :cas [15 [3 13]] :ValueNotSwapFrom INFO [2019-12-17 12:31:35,684] jepsen worker 45 - jepsen.util 5085 :invoke :cas [15 [28 40]] INFO [2019-12-17 12:31:35,704] jepsen worker 65 - jepsen.util 4385 :fail :cas [21 [19 8]] :ValueNotSwapFrom INFO [2019-12-17 12:31:35,704] jepsen worker 65 - jepsen.util 4385 :invoke :read [21 nil] INFO [2019-12-17 12:31:35,718] jepsen worker 29 - jepsen.util 3989 :ok :write [9 22] INFO [2019-12-17 12:31:35,718] jepsen worker 29 - jepsen.util 3989 :invoke :write [9 49] INFO [2019-12-17 12:31:35,719] jepsen worker 20 - jepsen.util 5060 :ok :write [6 27] INFO [2019-12-17 12:31:35,719] jepsen worker 20 - jepsen.util 5060 :invoke :read [6 nil] INFO [2019-12-17 12:31:35,745] jepsen worker 13 - jepsen.util 4603 :info :write [4 42] :RequestCanceledException INFO [2019-12-17 12:31:35,745] jepsen worker 28 - jepsen.util 3628 :info :write [9 10] :RequestCanceledException INFO [2019-12-17 12:31:35,745] jepsen worker 18 - jepsen.util 4878 :fail :read [6 nil] :RequestTimeoutException INFO [2019-12-17 12:31:35,745] jepsen worker 18 - jepsen.util 4878 :invoke :write [6 14] INFO [2019-12-17 12:31:35,747] jepsen worker 28 - jepsen.util 3718 :invoke :cas [9 [36 19]] INFO [2019-12-17 12:31:35,747] jepsen worker 13 - jepsen.util 4693 :invoke :write [4 37] INFO [2019-12-17 12:31:35,751] jepsen worker 72 - jepsen.util 4932 :ok :write [24 18] INFO [2019-12-17 12:31:35,751] jepsen worker 72 - jepsen.util 4932 :invoke :cas [24 [8 37]] INFO [2019-12-17 12:31:35,755] jepsen worker 31 - jepsen.util 4801 :fail :cas [10 [43 31]] :ValueNotSwapFrom INFO [2019-12-17 12:31:35,755] jepsen worker 31 - jepsen.util 4801 :invoke :cas [10 [48 30]] INFO [2019-12-17 12:31:35,792] jepsen worker 30 - jepsen.util 4980 :fail :read [10 nil] :RequestTimeoutException INFO [2019-12-17 12:31:35,793] jepsen worker 30 - jepsen.util 4980 :invoke :write [10 19] INFO [2019-12-17 12:31:35,832] jepsen worker 20 - jepsen.util 5060 :ok :read [6 27] INFO [2019-12-17 12:31:35,832] jepsen worker 20 - jepsen.util 5060 :invoke :cas [6 [38 46]] INFO [2019-12-17 12:31:35,840] jepsen worker 53 - jepsen.util 4913 :info :write [17 32] :RequestCanceledException INFO [2019-12-17 12:31:35,840] jepsen worker 64 - jepsen.util 3664 :info :cas [21 [1 21]] :RequestCanceledException INFO [2019-12-17 12:31:35,840] jepsen worker 36 - jepsen.util 4986 :info :cas [12 [11 35]] :RequestCanceledException INFO [2019-12-17 12:31:35,841] jepsen worker 64 - jepsen.util 3754 :invoke :read [21 nil] INFO [2019-12-17 12:31:35,841] jepsen worker 36 - jepsen.util 5076 :invoke :write [12 12] INFO [2019-12-17 12:31:35,841] jepsen worker 53 - jepsen.util 5003 :invoke :read [17 nil] INFO [2019-12-17 12:31:35,889] jepsen worker 31 - jepsen.util 4801 :fail :cas [10 [48 30]] :ValueNotSwapFrom INFO [2019-12-17 12:31:35,890] jepsen worker 31 - jepsen.util 4801 :invoke :read [10 nil] INFO [2019-12-17 12:31:35,892] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28380,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:31:35,892] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28380,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:31:35,892] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28384,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:31:35,892] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28384,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:31:35,897] jepsen worker 20 - jepsen.util 5060 :fail :cas [6 [38 46]] :ValueNotSwapFrom INFO [2019-12-17 12:31:35,898] jepsen worker 20 - jepsen.util 5060 :invoke :write [6 31] INFO [2019-12-17 12:31:35,918] jepsen worker 64 - jepsen.util 3754 :ok :read [21 11] INFO [2019-12-17 12:31:35,918] jepsen worker 64 - jepsen.util 3754 :invoke :cas [21 [20 41]] INFO [2019-12-17 12:31:35,945] jepsen worker 51 - jepsen.util 4821 :fail :cas [17 [37 39]] :ValueNotSwapFrom INFO [2019-12-17 12:31:35,946] jepsen worker 51 - jepsen.util 4821 :invoke :read [17 nil] INFO [2019-12-17 12:31:35,962] jepsen worker 74 - jepsen.util 5294 :ok :read [24 18] INFO [2019-12-17 12:31:35,962] jepsen worker 74 - jepsen.util 5294 :invoke :cas [24 [22 8]] INFO [2019-12-17 12:31:35,975] jepsen worker 14 - jepsen.util 5144 :ok :read [4 48] INFO [2019-12-17 12:31:35,976] jepsen worker 14 - jepsen.util 5144 :invoke :cas [4 [47 11]] INFO [2019-12-17 12:31:36,002] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28393,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:31:36,002] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28393,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:31:36,002] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28396,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:31:36,002] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28396,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:31:36,020] jepsen worker 31 - jepsen.util 4801 :ok :read [10 34] INFO [2019-12-17 12:31:36,021] jepsen worker 31 - jepsen.util 4801 :invoke :write [10 33] INFO [2019-12-17 12:31:36,028] jepsen worker 73 - jepsen.util 4573 :fail :cas [24 [8 27]] :ValueNotSwapFrom INFO [2019-12-17 12:31:36,029] jepsen worker 73 - jepsen.util 4573 :invoke :cas [24 [27 33]] INFO [2019-12-17 12:31:36,031] jepsen worker 52 - jepsen.util 5272 :ok :read [17 13] INFO [2019-12-17 12:31:36,031] jepsen worker 52 - jepsen.util 5272 :invoke :cas [17 [36 16]] INFO [2019-12-17 12:31:36,034] jepsen worker 70 - jepsen.util 5560 :fail :cas [23 [29 9]] :ValueNotSwapFrom INFO [2019-12-17 12:31:36,034] jepsen worker 70 - jepsen.util 5560 :invoke :read [23 nil] INFO [2019-12-17 12:31:36,051] jepsen worker 29 - jepsen.util 3989 :ok :write [9 49] INFO [2019-12-17 12:31:36,051] jepsen worker 29 - jepsen.util 3989 :invoke :write [9 13] INFO [2019-12-17 12:31:36,070] jepsen worker 51 - jepsen.util 4821 :ok :read [17 13] INFO [2019-12-17 12:31:36,071] jepsen worker 51 - jepsen.util 4821 :invoke :cas [17 [39 4]] INFO [2019-12-17 12:31:36,092] jepsen worker 37 - jepsen.util 4627 :info :write [12 2] :RequestCanceledException INFO [2019-12-17 12:31:36,093] jepsen worker 37 - jepsen.util 4717 :invoke :write [12 41] INFO [2019-12-17 12:31:36,105] jepsen worker 29 - jepsen.util 3989 :ok :write [9 13] INFO [2019-12-17 12:31:36,105] jepsen worker 29 - jepsen.util 3989 :invoke :read [9 nil] INFO [2019-12-17 12:31:36,130] jepsen worker 29 - jepsen.util 3989 :ok :read [9 13] INFO [2019-12-17 12:31:36,130] jepsen worker 29 - jepsen.util 3989 :invoke :cas [9 [34 3]] INFO [2019-12-17 12:31:36,139] jepsen worker 89 - jepsen.util 4589 :fail :cas [29 [25 27]] :ValueNotSwapFrom INFO [2019-12-17 12:31:36,139] jepsen worker 89 - jepsen.util 4589 :invoke :cas [29 [46 11]] INFO [2019-12-17 12:31:36,147] jepsen worker 70 - jepsen.util 5560 :ok :read [23 8] INFO [2019-12-17 12:31:36,148] jepsen worker 70 - jepsen.util 5560 :invoke :write [23 38] INFO [2019-12-17 12:31:36,164] jepsen worker 29 - jepsen.util 3989 :fail :cas [9 [34 3]] :ValueNotSwapFrom INFO [2019-12-17 12:31:36,164] jepsen worker 29 - jepsen.util 3989 :invoke :cas [9 [18 12]] INFO [2019-12-17 12:31:36,173] jepsen worker 29 - jepsen.util 3989 :fail :cas [9 [18 12]] :ValueNotSwapFrom INFO [2019-12-17 12:31:36,173] jepsen worker 29 - jepsen.util 3989 :invoke :cas [9 [37 43]] INFO [2019-12-17 12:31:36,190] jepsen worker 29 - jepsen.util 3989 :fail :cas [9 [37 43]] :ValueNotSwapFrom INFO [2019-12-17 12:31:36,190] jepsen worker 29 - jepsen.util 3989 :invoke :cas [9 [15 29]] INFO [2019-12-17 12:31:36,207] jepsen worker 29 - jepsen.util 3989 :fail :cas [9 [15 29]] :ValueNotSwapFrom INFO [2019-12-17 12:31:36,207] jepsen worker 29 - jepsen.util 3989 :invoke :read [9 nil] INFO [2019-12-17 12:31:36,217] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28419,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:31:36,218] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28411,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:31:36,218] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28419,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:31:36,218] jepsen worker 20 - jepsen.util 5060 :ok :write [6 31] INFO [2019-12-17 12:31:36,218] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28411,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:31:36,218] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28422,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:31:36,218] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28413,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:31:36,218] jepsen worker 20 - jepsen.util 5060 :invoke :write [6 0] INFO [2019-12-17 12:31:36,218] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28422,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:31:36,218] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28413,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:31:36,218] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28423,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:31:36,218] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28423,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:31:36,223] jepsen worker 29 - jepsen.util 3989 :ok :read [9 13] INFO [2019-12-17 12:31:36,223] jepsen worker 29 - jepsen.util 3989 :invoke :read [9 nil] INFO [2019-12-17 12:31:36,223] jepsen worker 47 - jepsen.util 5537 :ok :write [15 47] INFO [2019-12-17 12:31:36,223] jepsen worker 47 - jepsen.util 5537 :invoke :read [15 nil] INFO [2019-12-17 12:31:36,245] jepsen worker 10 - jepsen.util 4870 :ok :write [3 36] INFO [2019-12-17 12:31:36,245] jepsen worker 10 - jepsen.util 4870 :invoke :read [3 nil] INFO [2019-12-17 12:31:36,257] jepsen worker 51 - jepsen.util 4821 :fail :cas [17 [39 4]] :ValueNotSwapFrom INFO [2019-12-17 12:31:36,257] jepsen worker 51 - jepsen.util 4821 :invoke :read [17 nil] INFO [2019-12-17 12:31:36,268] jepsen worker 47 - jepsen.util 5537 :ok :read [15 47] INFO [2019-12-17 12:31:36,268] jepsen worker 47 - jepsen.util 5537 :invoke :read [15 nil] INFO [2019-12-17 12:31:36,269] jepsen worker 29 - jepsen.util 3989 :ok :read [9 13] INFO [2019-12-17 12:31:36,269] jepsen worker 29 - jepsen.util 3989 :invoke :cas [9 [31 44]] INFO [2019-12-17 12:31:36,286] jepsen worker 10 - jepsen.util 4870 :ok :read [3 36] INFO [2019-12-17 12:31:36,286] jepsen worker 10 - jepsen.util 4870 :invoke :cas [3 [17 29]] INFO [2019-12-17 12:31:36,292] jepsen worker 46 - jepsen.util 4906 :fail :read [15 nil] :RequestTimeoutException INFO [2019-12-17 12:31:36,292] jepsen worker 46 - jepsen.util 4906 :invoke :cas [15 [18 1]] INFO [2019-12-17 12:31:36,301] jepsen worker 47 - jepsen.util 5537 :ok :read [15 47] INFO [2019-12-17 12:31:36,301] jepsen worker 47 - jepsen.util 5537 :invoke :cas [15 [12 16]] INFO [2019-12-17 12:31:36,303] jepsen worker 51 - jepsen.util 4821 :ok :read [17 13] INFO [2019-12-17 12:31:36,304] jepsen worker 51 - jepsen.util 4821 :invoke :cas [17 [48 33]] INFO [2019-12-17 12:31:36,320] jepsen worker 47 - jepsen.util 5537 :fail :cas [15 [12 16]] :ValueNotSwapFrom INFO [2019-12-17 12:31:36,321] jepsen worker 47 - jepsen.util 5537 :invoke :write [15 30] INFO [2019-12-17 12:31:36,333] jepsen worker 29 - jepsen.util 3989 :fail :cas [9 [31 44]] :ValueNotSwapFrom INFO [2019-12-17 12:31:36,333] jepsen worker 29 - jepsen.util 3989 :invoke :cas [9 [31 36]] INFO [2019-12-17 12:31:36,357] jepsen worker 69 - jepsen.util 5199 :info :write [23 2] :RequestCanceledException INFO [2019-12-17 12:31:36,358] jepsen worker 69 - jepsen.util 5289 :invoke :write [23 47] INFO [2019-12-17 12:31:36,365] jepsen worker 29 - jepsen.util 3989 :fail :cas [9 [31 36]] :ValueNotSwapFrom INFO [2019-12-17 12:31:36,365] jepsen worker 29 - jepsen.util 3989 :invoke :write [9 23] INFO [2019-12-17 12:31:36,367] jepsen worker 10 - jepsen.util 4870 :fail :cas [3 [17 29]] :ValueNotSwapFrom INFO [2019-12-17 12:31:36,367] jepsen worker 10 - jepsen.util 4870 :invoke :cas [3 [18 11]] INFO [2019-12-17 12:31:36,390] jepsen worker 31 - jepsen.util 4801 :ok :write [10 33] INFO [2019-12-17 12:31:36,390] jepsen worker 31 - jepsen.util 4801 :invoke :write [10 45] INFO [2019-12-17 12:31:36,404] jepsen worker 46 - jepsen.util 4906 :fail :cas [15 [18 1]] :ValueNotSwapFrom INFO [2019-12-17 12:31:36,404] jepsen worker 46 - jepsen.util 4906 :invoke :write [15 33] INFO [2019-12-17 12:31:36,406] jepsen worker 51 - jepsen.util 4821 :fail :cas [17 [48 33]] :ValueNotSwapFrom INFO [2019-12-17 12:31:36,406] jepsen worker 51 - jepsen.util 4821 :invoke :write [17 24] INFO [2019-12-17 12:31:36,410] jepsen worker 82 - jepsen.util 7282 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:31:36,410] jepsen worker 68 - jepsen.util 6098 :info :cas [22 [17 2]] :RequestCanceledException INFO [2019-12-17 12:31:36,410] jepsen worker 82 - jepsen.util 7282 :invoke :write [27 0] INFO [2019-12-17 12:31:36,411] jepsen worker 68 - jepsen.util 6188 :invoke :read [22 nil] INFO [2019-12-17 12:31:36,423] jepsen worker 10 - jepsen.util 4870 :fail :cas [3 [18 11]] :ValueNotSwapFrom INFO [2019-12-17 12:31:36,423] jepsen worker 10 - jepsen.util 4870 :invoke :cas [3 [4 37]] INFO [2019-12-17 12:31:36,425] jepsen worker 16 - jepsen.util 7666 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:31:36,425] jepsen worker 84 - jepsen.util 6294 :info :write [28 20] :RequestCanceledException INFO [2019-12-17 12:31:36,426] jepsen worker 16 - jepsen.util 7666 :invoke :write [5 45] INFO [2019-12-17 12:31:36,427] jepsen worker 84 - jepsen.util 6384 :invoke :cas [28 [15 20]] INFO [2019-12-17 12:31:36,434] jepsen worker 80 - jepsen.util 5030 :ok :write [26 35] INFO [2019-12-17 12:31:36,434] jepsen worker 80 - jepsen.util 5030 :invoke :cas [26 [37 4]] INFO [2019-12-17 12:31:36,437] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28448,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:31:36,437] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28448,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:31:36,437] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28452,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:31:36,437] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28452,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:31:36,440] jepsen worker 26 - jepsen.util 6866 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:31:36,440] jepsen worker 26 - jepsen.util 6866 :invoke :cas [8 [19 36]] INFO [2019-12-17 12:31:36,453] jepsen worker 29 - jepsen.util 3989 :ok :write [9 23] INFO [2019-12-17 12:31:36,453] jepsen worker 29 - jepsen.util 3989 :invoke :read [9 nil] INFO [2019-12-17 12:31:36,456] jepsen worker 33 - jepsen.util 6423 :info :cas [11 [7 42]] :RequestCanceledException INFO [2019-12-17 12:31:36,457] jepsen worker 33 - jepsen.util 6513 :invoke :cas [11 [4 48]] INFO [2019-12-17 12:31:36,483] jepsen worker 29 - jepsen.util 3989 :ok :read [9 23] INFO [2019-12-17 12:31:36,484] jepsen worker 29 - jepsen.util 3989 :invoke :read [9 nil] INFO [2019-12-17 12:31:36,484] jepsen worker 20 - jepsen.util 5060 :ok :write [6 0] INFO [2019-12-17 12:31:36,484] jepsen worker 20 - jepsen.util 5060 :invoke :write [6 14] INFO [2019-12-17 12:31:36,492] jepsen worker 15 - jepsen.util 5595 :info :write [5 11] :RequestCanceledException INFO [2019-12-17 12:31:36,492] jepsen worker 8 - jepsen.util 5948 :info :write [2 41] :RequestCanceledException INFO [2019-12-17 12:31:36,492] jepsen worker 6 - jepsen.util 6846 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:31:36,493] jepsen worker 6 - jepsen.util 6846 :invoke :cas [2 [6 8]] INFO [2019-12-17 12:31:36,493] jepsen worker 8 - jepsen.util 6038 :invoke :cas [2 [14 49]] INFO [2019-12-17 12:31:36,493] jepsen worker 15 - jepsen.util 5685 :invoke :read [5 nil] INFO [2019-12-17 12:31:36,497] jepsen worker 46 - jepsen.util 4906 :ok :write [15 33] INFO [2019-12-17 12:31:36,497] jepsen worker 46 - jepsen.util 4906 :invoke :write [15 26] INFO [2019-12-17 12:31:36,502] jepsen worker 29 - jepsen.util 3989 :ok :read [9 23] INFO [2019-12-17 12:31:36,502] jepsen worker 29 - jepsen.util 3989 :invoke :read [9 nil] INFO [2019-12-17 12:31:36,510] jepsen worker 81 - jepsen.util 6471 :info :cas [27 [4 32]] :RequestCanceledException INFO [2019-12-17 12:31:36,512] jepsen worker 81 - jepsen.util 6561 :invoke :read [27 nil] INFO [2019-12-17 12:31:36,518] jepsen worker 29 - jepsen.util 3989 :ok :read [9 23] INFO [2019-12-17 12:31:36,518] jepsen worker 29 - jepsen.util 3989 :invoke :read [9 nil] INFO [2019-12-17 12:31:36,525] jepsen worker 49 - jepsen.util 5809 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:31:36,525] jepsen worker 34 - jepsen.util 5974 :info :cas [11 [43 24]] :RequestCanceledException INFO [2019-12-17 12:31:36,525] jepsen worker 58 - jepsen.util 6808 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:31:36,525] jepsen worker 25 - jepsen.util 6775 :info :write [8 36] :RequestCanceledException INFO [2019-12-17 12:31:36,525] jepsen worker 49 - jepsen.util 5809 :invoke :read [16 nil] INFO [2019-12-17 12:31:36,525] jepsen worker 58 - jepsen.util 6808 :invoke :write [19 31] INFO [2019-12-17 12:31:36,526] jepsen worker 34 - jepsen.util 6064 :invoke :cas [11 [24 20]] INFO [2019-12-17 12:31:36,526] jepsen worker 25 - jepsen.util 6865 :invoke :write [8 21] INFO [2019-12-17 12:31:36,538] jepsen worker 5 - jepsen.util 4415 :info :write [1 45] :RequestCanceledException INFO [2019-12-17 12:31:36,538] jepsen worker 76 - jepsen.util 6646 :info :write [25 37] :RequestCanceledException INFO [2019-12-17 12:31:36,538] jepsen worker 48 - jepsen.util 6078 :info :cas [16 [19 46]] :RequestCanceledException INFO [2019-12-17 12:31:36,539] jepsen worker 5 - jepsen.util 4505 :invoke :write [1 39] INFO [2019-12-17 12:31:36,539] jepsen worker 76 - jepsen.util 6736 :invoke :read [25 nil] INFO [2019-12-17 12:31:36,539] jepsen worker 48 - jepsen.util 6168 :invoke :write [16 29] INFO [2019-12-17 12:31:36,543] jepsen worker 29 - jepsen.util 3989 :ok :read [9 23] INFO [2019-12-17 12:31:36,543] jepsen worker 29 - jepsen.util 3989 :invoke :read [9 nil] INFO [2019-12-17 12:31:36,557] jepsen worker 43 - jepsen.util 6253 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:31:36,557] jepsen worker 57 - jepsen.util 6087 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:31:36,557] jepsen worker 17 - jepsen.util 6497 :info :cas [5 [39 18]] :RequestCanceledException INFO [2019-12-17 12:31:36,557] jepsen worker 43 - jepsen.util 6253 :invoke :read [14 nil] INFO [2019-12-17 12:31:36,557] jepsen worker 57 - jepsen.util 6087 :invoke :write [19 17] INFO [2019-12-17 12:31:36,558] jepsen worker 17 - jepsen.util 6587 :invoke :write [5 1] INFO [2019-12-17 12:31:36,569] jepsen worker 29 - jepsen.util 3989 :ok :read [9 23] INFO [2019-12-17 12:31:36,569] jepsen worker 29 - jepsen.util 3989 :invoke :cas [9 [18 20]] INFO [2019-12-17 12:31:36,585] jepsen worker 29 - jepsen.util 3989 :fail :cas [9 [18 20]] :ValueNotSwapFrom INFO [2019-12-17 12:31:36,587] jepsen worker 31 - jepsen.util 4801 :ok :write [10 45] INFO [2019-12-17 12:31:36,587] jepsen worker 29 - jepsen.util 3989 :invoke :write [9 0] INFO [2019-12-17 12:31:36,587] jepsen worker 31 - jepsen.util 4801 :invoke :cas [10 [23 28]] INFO [2019-12-17 12:31:36,592] jepsen worker 40 - jepsen.util 7240 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:31:36,592] jepsen worker 85 - jepsen.util 7465 :info :cas [28 [35 6]] :RequestCanceledException INFO [2019-12-17 12:31:36,592] jepsen worker 40 - jepsen.util 7240 :invoke :write [13 29] INFO [2019-12-17 12:31:36,592] jepsen worker 85 - jepsen.util 7555 :invoke :read [28 nil] INFO [2019-12-17 12:31:36,610] jepsen worker 60 - jepsen.util 7170 :info :write [20 14] :RequestCanceledException INFO [2019-12-17 12:31:36,610] jepsen worker 50 - jepsen.util 6980 :info :cas [16 [47 5]] :RequestCanceledException INFO [2019-12-17 12:31:36,611] jepsen worker 60 - jepsen.util 7260 :invoke :write [20 5] INFO [2019-12-17 12:31:36,611] jepsen worker 50 - jepsen.util 7070 :invoke :write [16 32] INFO [2019-12-17 12:31:36,621] jepsen worker 31 - jepsen.util 4801 :fail :cas [10 [23 28]] :ValueNotSwapFrom INFO [2019-12-17 12:31:36,621] jepsen worker 31 - jepsen.util 4801 :invoke :read [10 nil] INFO [2019-12-17 12:31:36,625] jepsen worker 62 - jepsen.util 6092 :info :write [20 22] :RequestCanceledException INFO [2019-12-17 12:31:36,625] jepsen worker 66 - jepsen.util 6006 :info :cas [22 [29 3]] :RequestCanceledException INFO [2019-12-17 12:31:36,625] jepsen worker 77 - jepsen.util 6827 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:31:36,625] jepsen worker 39 - jepsen.util 6159 :info :cas [13 [38 42]] :RequestCanceledException INFO [2019-12-17 12:31:36,625] jepsen worker 75 - jepsen.util 6465 :info :cas [25 [34 19]] :RequestCanceledException INFO [2019-12-17 12:31:36,626] jepsen worker 77 - jepsen.util 6827 :invoke :write [25 35] INFO [2019-12-17 12:31:36,627] jepsen worker 62 - jepsen.util 6182 :invoke :read [20 nil] INFO [2019-12-17 12:31:36,627] jepsen worker 66 - jepsen.util 6096 :invoke :write [22 6] INFO [2019-12-17 12:31:36,627] jepsen worker 75 - jepsen.util 6555 :invoke :read [25 nil] INFO [2019-12-17 12:31:36,627] jepsen worker 39 - jepsen.util 6249 :invoke :write [13 9] INFO [2019-12-17 12:31:36,635] jepsen worker 31 - jepsen.util 4801 :ok :read [10 45] INFO [2019-12-17 12:31:36,635] jepsen worker 31 - jepsen.util 4801 :invoke :read [10 nil] INFO [2019-12-17 12:31:36,638] jepsen worker 7 - jepsen.util 6667 :info :cas [2 [14 24]] :RequestCanceledException INFO [2019-12-17 12:31:36,638] jepsen worker 83 - jepsen.util 7013 :info :cas [27 [15 19]] :RequestCanceledException INFO [2019-12-17 12:31:36,639] jepsen worker 7 - jepsen.util 6757 :invoke :cas [2 [38 10]] INFO [2019-12-17 12:31:36,639] jepsen worker 83 - jepsen.util 7103 :invoke :write [27 17] INFO [2019-12-17 12:31:36,653] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:31:36,656] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28493,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:31:36,657] jepsen worker 41 - jepsen.util 7061 :info :cas [13 [15 10]] :RequestCanceledException INFO [2019-12-17 12:31:36,657] jepsen worker 1 - jepsen.util 6391 :info :cas [0 [49 15]] :RequestCanceledException INFO [2019-12-17 12:31:36,669] jepsen worker 14 - jepsen.util 5144 :fail :cas [4 [47 11]] :ValueNotSwapFrom INFO [2019-12-17 12:31:36,669] jepsen nemesis - couchbase.util Rebalance check count: 213 INFO [2019-12-17 12:31:36,670] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28493,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:31:36,671] jepsen worker 14 - jepsen.util 5144 :invoke :read [4 nil] INFO [2019-12-17 12:31:36,672] jepsen nemesis - couchbase.util Rebalance stuck count: 194 INFO [2019-12-17 12:31:36,672] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28496,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:31:36,673] jepsen worker 41 - jepsen.util 7151 :invoke :write [13 20] INFO [2019-12-17 12:31:36,673] jepsen worker 1 - jepsen.util 6481 :invoke :write [0 25] INFO [2019-12-17 12:31:36,674] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28496,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:31:36,676] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28497,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:31:36,677] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28497,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:31:36,680] jepsen worker 31 - jepsen.util 4801 :ok :read [10 45] INFO [2019-12-17 12:31:36,680] jepsen worker 31 - jepsen.util 4801 :invoke :write [10 45] INFO [2019-12-17 12:31:36,685] jepsen worker 29 - jepsen.util 3989 :ok :write [9 0] INFO [2019-12-17 12:31:36,686] jepsen worker 29 - jepsen.util 3989 :invoke :cas [9 [9 21]] INFO [2019-12-17 12:31:36,693] jepsen worker 22 - jepsen.util 6682 :info :write [7 11] :RequestCanceledException INFO [2019-12-17 12:31:36,693] jepsen worker 35 - jepsen.util 6695 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:31:36,693] jepsen worker 35 - jepsen.util 6695 :invoke :read [11 nil] INFO [2019-12-17 12:31:36,695] jepsen worker 22 - jepsen.util 6772 :invoke :write [7 43] INFO [2019-12-17 12:31:36,709] jepsen worker 2 - jepsen.util 6032 :info :write [0 29] :RequestCanceledException INFO [2019-12-17 12:31:36,712] jepsen worker 2 - jepsen.util 6122 :invoke :cas [0 [6 33]] INFO [2019-12-17 12:31:36,726] jepsen worker 42 - jepsen.util 7062 :info :write [14 4] :RequestCanceledException INFO [2019-12-17 12:31:36,726] jepsen worker 67 - jepsen.util 6637 :info :cas [22 [12 20]] :RequestCanceledException INFO [2019-12-17 12:31:36,726] jepsen worker 27 - jepsen.util 4797 :info :cas [9 [12 25]] :RequestCanceledException INFO [2019-12-17 12:31:36,726] jepsen worker 24 - jepsen.util 5964 :info :cas [8 [26 9]] :RequestCanceledException INFO [2019-12-17 12:31:36,726] jepsen worker 0 - jepsen.util 7200 :info :write [0 3] :RequestCanceledException INFO [2019-12-17 12:31:36,728] jepsen worker 27 - jepsen.util 4887 :invoke :write [9 41] INFO [2019-12-17 12:31:36,728] jepsen worker 42 - jepsen.util 7152 :invoke :cas [14 [31 26]] INFO [2019-12-17 12:31:36,729] jepsen worker 24 - jepsen.util 6054 :invoke :write [8 48] INFO [2019-12-17 12:31:36,730] jepsen worker 0 - jepsen.util 7290 :invoke :cas [0 [25 20]] INFO [2019-12-17 12:31:36,730] jepsen worker 67 - jepsen.util 6727 :invoke :write [22 26] INFO [2019-12-17 12:31:36,734] jepsen worker 44 - jepsen.util 7244 :info :write [14 45] :RequestCanceledException INFO [2019-12-17 12:31:36,736] jepsen worker 44 - jepsen.util 7334 :invoke :cas [14 [22 18]] INFO [2019-12-17 12:31:36,740] jepsen worker 29 - jepsen.util 3989 :fail :cas [9 [9 21]] :ValueNotSwapFrom INFO [2019-12-17 12:31:36,740] jepsen worker 29 - jepsen.util 3989 :invoke :read [9 nil] INFO [2019-12-17 12:31:36,757] jepsen worker 32 - jepsen.util 5072 :info :write [10 49] :RequestCanceledException INFO [2019-12-17 12:31:36,759] jepsen worker 32 - jepsen.util 5162 :invoke :read [10 nil] INFO [2019-12-17 12:31:36,760] jepsen worker 52 - jepsen.util 5272 :fail :cas [17 [36 16]] :ValueNotSwapFrom INFO [2019-12-17 12:31:36,760] jepsen worker 52 - jepsen.util 5272 :invoke :cas [17 [6 17]] INFO [2019-12-17 12:31:36,783] jepsen worker 29 - jepsen.util 3989 :ok :read [9 0] INFO [2019-12-17 12:31:36,792] jepsen worker 89 - jepsen.util 4589 :fail :cas [29 [46 11]] :ValueNotSwapFrom INFO [2019-12-17 12:31:36,792] jepsen worker 29 - jepsen.util 3989 :invoke :cas [9 [23 30]] INFO [2019-12-17 12:31:36,796] jepsen worker 89 - jepsen.util 4589 :invoke :write [29 0] INFO [2019-12-17 12:31:36,810] jepsen worker 3 - jepsen.util 5043 :info :write [1 20] :RequestCanceledException INFO [2019-12-17 12:31:36,811] jepsen worker 3 - jepsen.util 5133 :invoke :cas [1 [14 44]] INFO [2019-12-17 12:31:36,818] jepsen worker 29 - jepsen.util 3989 :fail :cas [9 [23 30]] :ValueNotSwapFrom INFO [2019-12-17 12:31:36,822] jepsen worker 29 - jepsen.util 3989 :invoke :read [9 nil] INFO [2019-12-17 12:31:36,826] jepsen worker 86 - jepsen.util 6926 :info :write [28 5] :RequestCanceledException INFO [2019-12-17 12:31:36,828] jepsen worker 86 - jepsen.util 7016 :invoke :read [28 nil] INFO [2019-12-17 12:31:36,837] jepsen worker 29 - jepsen.util 3989 :ok :read [9 0] INFO [2019-12-17 12:31:36,838] jepsen worker 29 - jepsen.util 3989 :invoke :cas [9 [2 47]] INFO [2019-12-17 12:31:36,863] jepsen worker 29 - jepsen.util 3989 :fail :cas [9 [2 47]] :ValueNotSwapFrom INFO [2019-12-17 12:31:36,863] jepsen worker 29 - jepsen.util 3989 :invoke :read [9 nil] INFO [2019-12-17 12:31:36,876] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28524,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:31:36,876] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28524,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:31:36,876] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28527,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:31:36,876] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28527,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:31:36,884] jepsen worker 29 - jepsen.util 3989 :ok :read [9 0] INFO [2019-12-17 12:31:36,884] jepsen worker 29 - jepsen.util 3989 :invoke :read [9 nil] INFO [2019-12-17 12:31:36,909] jepsen worker 29 - jepsen.util 3989 :ok :read [9 0] INFO [2019-12-17 12:31:36,910] jepsen worker 29 - jepsen.util 3989 :invoke :cas [9 [24 16]] INFO [2019-12-17 12:31:36,910] jepsen worker 87 - jepsen.util 4587 :info :cas [29 [7 43]] :RequestCanceledException INFO [2019-12-17 12:31:36,911] jepsen worker 87 - jepsen.util 4677 :invoke :read [29 nil] INFO [2019-12-17 12:31:36,924] jepsen worker 80 - jepsen.util 5030 :fail :cas [26 [37 4]] :ValueNotSwapFrom INFO [2019-12-17 12:31:36,925] jepsen worker 80 - jepsen.util 5030 :invoke :write [26 49] INFO [2019-12-17 12:31:36,931] jepsen worker 29 - jepsen.util 3989 :fail :cas [9 [24 16]] :ValueNotSwapFrom INFO [2019-12-17 12:31:36,932] jepsen worker 46 - jepsen.util 4906 :ok :write [15 26] INFO [2019-12-17 12:31:36,942] jepsen worker 29 - jepsen.util 3989 :invoke :write [9 16] INFO [2019-12-17 12:31:36,942] jepsen worker 46 - jepsen.util 4906 :invoke :cas [15 [35 22]] INFO [2019-12-17 12:31:36,951] jepsen worker 54 - jepsen.util 5364 :ok :write [18 6] INFO [2019-12-17 12:31:36,951] jepsen worker 54 - jepsen.util 5364 :invoke :cas [18 [15 1]] INFO [2019-12-17 12:31:36,951] jepsen worker 31 - jepsen.util 4801 :ok :write [10 45] INFO [2019-12-17 12:31:36,952] jepsen worker 31 - jepsen.util 4801 :invoke :cas [10 [4 14]] INFO [2019-12-17 12:31:36,968] jepsen worker 46 - jepsen.util 4906 :fail :cas [15 [35 22]] :ValueNotSwapFrom INFO [2019-12-17 12:31:36,969] jepsen worker 46 - jepsen.util 4906 :invoke :write [15 32] INFO [2019-12-17 12:31:36,979] jepsen worker 27 - jepsen.util 4887 :ok :write [9 41] INFO [2019-12-17 12:31:36,987] jepsen worker 27 - jepsen.util 4887 :invoke :write [9 28] INFO [2019-12-17 12:31:36,998] jepsen worker 31 - jepsen.util 4801 :fail :cas [10 [4 14]] :ValueNotSwapFrom INFO [2019-12-17 12:31:36,998] jepsen worker 31 - jepsen.util 4801 :invoke :read [10 nil] INFO [2019-12-17 12:31:37,048] jepsen worker 46 - jepsen.util 4906 :ok :write [15 32] INFO [2019-12-17 12:31:37,048] jepsen worker 46 - jepsen.util 4906 :invoke :cas [15 [30 40]] INFO [2019-12-17 12:31:37,057] jepsen worker 78 - jepsen.util 4938 :info :write [26 14] :RequestCanceledException INFO [2019-12-17 12:31:37,057] jepsen worker 59 - jepsen.util 6449 :info :write [19 15] :RequestCanceledException INFO [2019-12-17 12:31:37,057] jepsen worker 21 - jepsen.util 6051 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:31:37,057] jepsen worker 21 - jepsen.util 6051 :invoke :cas [7 [44 13]] INFO [2019-12-17 12:31:37,059] jepsen worker 59 - jepsen.util 6539 :invoke :write [19 30] INFO [2019-12-17 12:31:37,066] jepsen worker 78 - jepsen.util 5028 :invoke :write [26 35] INFO [2019-12-17 12:31:37,074] jepsen worker 46 - jepsen.util 4906 :fail :cas [15 [30 40]] :ValueNotSwapFrom INFO [2019-12-17 12:31:37,074] jepsen worker 46 - jepsen.util 4906 :invoke :cas [15 [4 48]] INFO [2019-12-17 12:31:37,090] jepsen worker 29 - jepsen.util 3989 :ok :write [9 16] INFO [2019-12-17 12:31:37,090] jepsen worker 29 - jepsen.util 3989 :invoke :read [9 nil] INFO [2019-12-17 12:31:37,093] jepsen worker 61 - jepsen.util 7441 :info :write [20 40] :RequestCanceledException INFO [2019-12-17 12:31:37,094] jepsen worker 61 - jepsen.util 7531 :invoke :write [20 41] INFO [2019-12-17 12:31:37,095] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28538,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:31:37,095] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28538,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:31:37,095] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28543,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:31:37,096] jepsen worker 46 - jepsen.util 4906 :fail :cas [15 [4 48]] :ValueNotSwapFrom INFO [2019-12-17 12:31:37,107] jepsen worker 29 - jepsen.util 3989 :ok :read [9 16] INFO [2019-12-17 12:31:37,111] jepsen worker 23 - jepsen.util 6773 :info :cas [7 [42 17]] :RequestCanceledException INFO [2019-12-17 12:31:37,113] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28543,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:31:37,113] jepsen worker 46 - jepsen.util 4906 :invoke :cas [15 [32 34]] INFO [2019-12-17 12:31:37,113] jepsen worker 29 - jepsen.util 3989 :invoke :cas [9 [2 6]] INFO [2019-12-17 12:31:37,114] jepsen worker 53 - jepsen.util 5003 :ok :read [17 13] INFO [2019-12-17 12:31:37,125] jepsen worker 71 - jepsen.util 5381 :info :write [23 9] :RequestCanceledException INFO [2019-12-17 12:31:37,127] jepsen worker 71 - jepsen.util 5471 :invoke :cas [23 [24 33]] INFO [2019-12-17 12:31:37,136] jepsen worker 29 - jepsen.util 3989 :fail :cas [9 [2 6]] :ValueNotSwapFrom INFO [2019-12-17 12:31:37,140] jepsen worker 63 - jepsen.util 3933 :info :write [21 30] :RequestCanceledException INFO [2019-12-17 12:31:37,140] jepsen worker 11 - jepsen.util 4871 :info :write [3 12] :RequestCanceledException INFO [2019-12-17 12:31:37,140] jepsen worker 4 - jepsen.util 4144 :info :write [1 13] :RequestCanceledException INFO [2019-12-17 12:31:37,154] jepsen worker 23 - jepsen.util 6863 :invoke :cas [7 [47 1]] INFO [2019-12-17 12:31:37,158] jepsen worker 53 - jepsen.util 5003 :invoke :read [17 nil] INFO [2019-12-17 12:31:37,162] jepsen worker 63 - jepsen.util 4023 :invoke :write [21 25] INFO [2019-12-17 12:31:37,162] jepsen worker 4 - jepsen.util 4234 :invoke :write [1 15] INFO [2019-12-17 12:31:37,162] jepsen worker 11 - jepsen.util 4961 :invoke :read [3 nil] INFO [2019-12-17 12:31:37,188] jepsen worker 31 - jepsen.util 4801 :ok :read [10 45] INFO [2019-12-17 12:31:37,188] jepsen worker 31 - jepsen.util 4801 :invoke :cas [10 [28 49]] INFO [2019-12-17 12:31:37,189] jepsen worker 29 - jepsen.util 3989 :invoke :read [9 nil] INFO [2019-12-17 12:31:37,204] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28563,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:31:37,205] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28563,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:31:37,205] jepsen worker 31 - jepsen.util 4801 :fail :cas [10 [28 49]] :ValueNotSwapFrom INFO [2019-12-17 12:31:37,206] jepsen worker 31 - jepsen.util 4801 :invoke :cas [10 [38 0]] INFO [2019-12-17 12:31:37,211] jepsen worker 46 - jepsen.util 4906 :ok :cas [15 [32 34]] INFO [2019-12-17 12:31:37,212] jepsen worker 46 - jepsen.util 4906 :invoke :write [15 15] INFO [2019-12-17 12:31:37,223] jepsen worker 55 - jepsen.util 5905 :info :write [18 12] :RequestCanceledException INFO [2019-12-17 12:31:37,227] jepsen worker 55 - jepsen.util 5995 :invoke :cas [18 [39 48]] INFO [2019-12-17 12:31:37,236] jepsen worker 31 - jepsen.util 4801 :fail :cas [10 [38 0]] :ValueNotSwapFrom INFO [2019-12-17 12:31:37,240] jepsen worker 38 - jepsen.util 4718 :info :write [12 40] :RequestCanceledException INFO [2019-12-17 12:31:37,240] jepsen worker 11 - jepsen.util 4961 :ok :read [3 36] INFO [2019-12-17 12:31:37,254] jepsen worker 31 - jepsen.util 4801 :invoke :write [10 28] INFO [2019-12-17 12:31:37,254] jepsen worker 11 - jepsen.util 4961 :invoke :write [3 38] INFO [2019-12-17 12:31:37,256] jepsen worker 38 - jepsen.util 4808 :invoke :read [12 nil] INFO [2019-12-17 12:31:37,257] jepsen worker 29 - jepsen.util 3989 :ok :read [9 16] INFO [2019-12-17 12:31:37,257] jepsen worker 29 - jepsen.util 3989 :invoke :read [9 nil] INFO [2019-12-17 12:31:37,276] jepsen worker 29 - jepsen.util 3989 :ok :read [9 16] INFO [2019-12-17 12:31:37,276] jepsen worker 29 - jepsen.util 3989 :invoke :write [9 12] INFO [2019-12-17 12:31:37,280] jepsen worker 14 - jepsen.util 5144 :ok :read [4 48] INFO [2019-12-17 12:31:37,280] jepsen worker 14 - jepsen.util 5144 :invoke :write [4 30] INFO [2019-12-17 12:31:37,314] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28570,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:31:37,314] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28570,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:31:37,314] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28573,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:31:37,315] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28573,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:31:37,325] jepsen worker 56 - jepsen.util 5096 :info :write [18 15] :RequestCanceledException INFO [2019-12-17 12:31:37,328] jepsen worker 56 - jepsen.util 5186 :invoke :cas [18 [4 25]] INFO [2019-12-17 12:31:37,335] jepsen worker 29 - jepsen.util 3989 :ok :write [9 12] INFO [2019-12-17 12:31:37,335] jepsen worker 29 - jepsen.util 3989 :invoke :write [9 48] INFO [2019-12-17 12:31:37,363] jepsen worker 3 - jepsen.util 5133 :fail :cas [1 [14 44]] :ValueNotSwapFrom INFO [2019-12-17 12:31:37,363] jepsen worker 3 - jepsen.util 5133 :invoke :cas [1 [32 41]] INFO [2019-12-17 12:31:37,378] jepsen worker 29 - jepsen.util 3989 :ok :write [9 48] INFO [2019-12-17 12:31:37,378] jepsen worker 29 - jepsen.util 3989 :invoke :write [9 39] INFO [2019-12-17 12:31:37,386] jepsen worker 54 - jepsen.util 5364 :fail :cas [18 [15 1]] :ValueNotSwapFrom INFO [2019-12-17 12:31:37,387] jepsen worker 54 - jepsen.util 5364 :invoke :cas [18 [29 49]] INFO [2019-12-17 12:31:37,440] jepsen worker 29 - jepsen.util 3989 :ok :write [9 39] INFO [2019-12-17 12:31:37,440] jepsen worker 29 - jepsen.util 3989 :invoke :write [9 12] INFO [2019-12-17 12:31:37,462] jepsen worker 56 - jepsen.util 5186 :fail :cas [18 [4 25]] :ValueNotSwapFrom INFO [2019-12-17 12:31:37,462] jepsen worker 56 - jepsen.util 5186 :invoke :cas [18 [18 21]] INFO [2019-12-17 12:31:37,483] jepsen worker 63 - jepsen.util 4023 :ok :write [21 25] INFO [2019-12-17 12:31:37,488] jepsen worker 29 - jepsen.util 3989 :ok :write [9 12] INFO [2019-12-17 12:31:37,490] jepsen worker 29 - jepsen.util 3989 :invoke :write [9 47] INFO [2019-12-17 12:31:37,490] jepsen worker 63 - jepsen.util 4023 :invoke :read [21 nil] INFO [2019-12-17 12:31:37,501] jepsen worker 52 - jepsen.util 5272 :fail :cas [17 [6 17]] :ValueNotSwapFrom INFO [2019-12-17 12:31:37,502] jepsen worker 52 - jepsen.util 5272 :invoke :read [17 nil] INFO [2019-12-17 12:31:37,513] jepsen worker 63 - jepsen.util 4023 :ok :read [21 25] INFO [2019-12-17 12:31:37,514] jepsen worker 63 - jepsen.util 4023 :invoke :cas [21 [24 40]] INFO [2019-12-17 12:31:37,526] jepsen worker 79 - jepsen.util 4849 :info :write [26 31] :RequestCanceledException INFO [2019-12-17 12:31:37,526] jepsen worker 9 - jepsen.util 5769 :fail :read [3 nil] :RequestTimeoutException INFO [2019-12-17 12:31:37,526] jepsen worker 19 - jepsen.util 4879 :info :write [6 40] :RequestCanceledException INFO [2019-12-17 12:31:37,527] jepsen worker 9 - jepsen.util 5769 :invoke :cas [3 [28 23]] INFO [2019-12-17 12:31:37,528] jepsen worker 79 - jepsen.util 4939 :invoke :write [26 33] INFO [2019-12-17 12:31:37,528] jepsen worker 19 - jepsen.util 4969 :invoke :write [6 22] INFO [2019-12-17 12:31:37,533] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28593,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:31:37,533] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28593,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:31:37,564] jepsen worker 29 - jepsen.util 3989 :ok :write [9 47] INFO [2019-12-17 12:31:37,564] jepsen worker 29 - jepsen.util 3989 :invoke :write [9 13] INFO [2019-12-17 12:31:37,632] jepsen worker 71 - jepsen.util 5471 :fail :cas [23 [24 33]] :ValueNotSwapFrom INFO [2019-12-17 12:31:37,650] jepsen worker 71 - jepsen.util 5471 :invoke :write [23 45] INFO [2019-12-17 12:31:37,673] jepsen worker 88 - jepsen.util 5128 :info :write [29 19] :RequestCanceledException INFO [2019-12-17 12:31:37,705] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28608,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:31:37,711] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28608,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:31:37,711] jepsen worker 12 - jepsen.util 5142 :info :write [4 43] :RequestCanceledException INFO [2019-12-17 12:31:38,097] jepsen worker 19 - jepsen.util 4969 :ok :write [6 22] INFO [2019-12-17 12:31:38,097] jepsen worker 19 - jepsen.util 4969 :invoke :read [6 nil] INFO [2019-12-17 12:31:38,098] jepsen worker 9 - jepsen.util 5769 :fail :cas [3 [28 23]] :ValueNotSwapFrom INFO [2019-12-17 12:31:38,098] jepsen worker 9 - jepsen.util 5769 :invoke :cas [3 [18 22]] INFO [2019-12-17 12:31:38,099] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28614,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:31:38,100] jepsen worker 13 - jepsen.util 4693 :info :write [4 37] :RequestCanceledException INFO [2019-12-17 12:31:38,102] jepsen worker 36 - jepsen.util 5076 :info :write [12 12] :RequestCanceledException INFO [2019-12-17 12:31:38,102] jepsen worker 46 - jepsen.util 4906 :ok :write [15 15] INFO [2019-12-17 12:31:38,103] jepsen worker 46 - jepsen.util 4906 :invoke :cas [15 [13 15]] INFO [2019-12-17 12:31:38,103] jepsen worker 31 - jepsen.util 4801 :ok :write [10 28] INFO [2019-12-17 12:31:38,103] jepsen worker 31 - jepsen.util 4801 :invoke :write [10 19] INFO [2019-12-17 12:31:38,104] jepsen worker 28 - jepsen.util 3718 :info :cas [9 [36 19]] :RequestCanceledException INFO [2019-12-17 12:31:38,104] jepsen worker 64 - jepsen.util 3754 :info :cas [21 [20 41]] :RequestCanceledException INFO [2019-12-17 12:31:38,104] jepsen worker 72 - jepsen.util 4932 :info :cas [24 [8 37]] :RequestCanceledException INFO [2019-12-17 12:31:38,106] jepsen worker 64 - jepsen.util 3844 :invoke :write [21 42] INFO [2019-12-17 12:31:38,107] jepsen worker 72 - jepsen.util 5022 :invoke :read [24 nil] INFO [2019-12-17 12:31:38,109] jepsen worker 28 - jepsen.util 3808 :invoke :read [9 nil] INFO [2019-12-17 12:31:38,109] jepsen worker 65 - jepsen.util 4385 :fail :read [21 nil] :RequestTimeoutException INFO [2019-12-17 12:31:38,111] jepsen worker 65 - jepsen.util 4385 :invoke :cas [21 [19 25]] INFO [2019-12-17 12:31:38,111] jepsen worker 45 - jepsen.util 5085 :info :cas [15 [28 40]] :RequestCanceledException INFO [2019-12-17 12:31:38,114] jepsen worker 45 - jepsen.util 5175 :invoke :write [15 24] INFO [2019-12-17 12:31:38,115] jepsen worker 74 - jepsen.util 5294 :info :cas [24 [22 8]] :RequestCanceledException INFO [2019-12-17 12:31:38,116] jepsen worker 73 - jepsen.util 4573 :info :cas [24 [27 33]] :RequestCanceledException INFO [2019-12-17 12:31:38,118] jepsen worker 74 - jepsen.util 5384 :invoke :cas [24 [34 18]] INFO [2019-12-17 12:31:38,119] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:31:38,119] jepsen nemesis - couchbase.util Rebalance check count: 214 INFO [2019-12-17 12:31:38,119] jepsen nemesis - couchbase.util Rebalance stuck count: 195 INFO [2019-12-17 12:31:38,120] jepsen worker 30 - jepsen.util 4980 :info :write [10 19] :RequestCanceledException INFO [2019-12-17 12:31:38,123] jepsen worker 18 - jepsen.util 4878 :info :write [6 14] :RequestCanceledException INFO [2019-12-17 12:31:38,123] jepsen worker 73 - jepsen.util 4663 :invoke :read [24 nil] INFO [2019-12-17 12:31:38,128] jepsen worker 18 - jepsen.util 4968 :invoke :write [6 7] INFO [2019-12-17 12:31:38,128] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28619,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:31:38,128] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28619,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:31:38,128] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28628,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:31:38,129] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28628,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:31:38,145] jepsen worker 30 - jepsen.util 5070 :invoke :write [10 40] INFO [2019-12-17 12:31:38,168] jepsen worker 37 - jepsen.util 4717 :info :write [12 41] :RequestCanceledException INFO [2019-12-17 12:31:38,176] jepsen worker 88 - jepsen.util 5218 :invoke :cas [29 [18 47]] INFO [2019-12-17 12:31:38,177] jepsen worker 12 - jepsen.util 5232 :invoke :write [4 8] INFO [2019-12-17 12:31:38,177] jepsen worker 36 - jepsen.util 5166 :invoke :cas [12 [11 44]] INFO [2019-12-17 12:31:38,178] jepsen worker 37 - jepsen.util 4807 :invoke :read [12 nil] INFO [2019-12-17 12:31:38,186] jepsen worker 53 - jepsen.util 5003 :ok :read [17 13] INFO [2019-12-17 12:31:38,189] jepsen worker 53 - jepsen.util 5003 :invoke :read [17 nil] INFO [2019-12-17 12:31:38,191] jepsen worker 13 - jepsen.util 4783 :invoke :cas [4 [1 6]] INFO [2019-12-17 12:31:38,213] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28614,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:31:38,213] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28641,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:31:38,214] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28623,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:31:38,215] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28649,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:31:38,215] jepsen worker 70 - jepsen.util 5560 :info :write [23 38] :RequestCanceledException INFO [2019-12-17 12:31:38,216] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28641,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:31:38,217] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28623,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:31:38,221] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28649,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:31:38,222] jepsen worker 70 - jepsen.util 5650 :invoke :cas [23 [22 27]] INFO [2019-12-17 12:31:38,223] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28643,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:31:38,231] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28652,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:31:38,231] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28643,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:31:38,231] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28652,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:31:38,232] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28653,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:31:38,232] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28653,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:31:38,253] jepsen worker 11 - jepsen.util 4961 :ok :write [3 38] INFO [2019-12-17 12:31:38,254] jepsen worker 11 - jepsen.util 4961 :invoke :cas [3 [16 27]] INFO [2019-12-17 12:31:38,263] jepsen worker 63 - jepsen.util 4023 :fail :cas [21 [24 40]] :ValueNotSwapFrom INFO [2019-12-17 12:31:38,264] jepsen worker 63 - jepsen.util 4023 :invoke :write [21 36] INFO [2019-12-17 12:31:38,354] jepsen worker 56 - jepsen.util 5186 :fail :cas [18 [18 21]] :ValueNotSwapFrom INFO [2019-12-17 12:31:38,354] jepsen worker 56 - jepsen.util 5186 :invoke :cas [18 [8 27]] INFO [2019-12-17 12:31:38,357] jepsen worker 47 - jepsen.util 5537 :info :write [15 30] :RequestCanceledException INFO [2019-12-17 12:31:38,408] jepsen worker 47 - jepsen.util 5627 :invoke :write [15 32] INFO [2019-12-17 12:31:38,430] jepsen worker 69 - jepsen.util 5289 :info :write [23 47] :RequestCanceledException INFO [2019-12-17 12:31:38,445] jepsen worker 31 - jepsen.util 4801 :ok :write [10 19] INFO [2019-12-17 12:31:38,445] jepsen worker 31 - jepsen.util 4801 :invoke :read [10 nil] INFO [2019-12-17 12:31:38,477] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28659,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:31:38,482] jepsen worker 69 - jepsen.util 5379 :invoke :write [23 46] INFO [2019-12-17 12:31:38,484] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28659,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:31:38,484] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28663,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:31:38,485] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28663,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:31:38,485] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28664,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:31:38,486] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28664,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:31:38,492] jepsen worker 51 - jepsen.util 4821 :info :write [17 24] :RequestCanceledException INFO [2019-12-17 12:31:38,492] jepsen worker 68 - jepsen.util 6188 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:31:38,493] jepsen worker 68 - jepsen.util 6188 :invoke :write [22 47] INFO [2019-12-17 12:31:38,497] jepsen worker 10 - jepsen.util 4870 :info :cas [3 [4 37]] :RequestCanceledException INFO [2019-12-17 12:31:38,506] jepsen worker 54 - jepsen.util 5364 :fail :cas [18 [29 49]] :ValueNotSwapFrom INFO [2019-12-17 12:31:38,507] jepsen worker 54 - jepsen.util 5364 :invoke :cas [18 [31 47]] INFO [2019-12-17 12:31:38,510] jepsen worker 82 - jepsen.util 7282 :info :write [27 0] :RequestCanceledException INFO [2019-12-17 12:31:38,511] jepsen worker 84 - jepsen.util 6384 :info :cas [28 [15 20]] :RequestCanceledException INFO [2019-12-17 12:31:38,512] jepsen worker 20 - jepsen.util 5060 :info :write [6 14] :RequestCanceledException INFO [2019-12-17 12:31:38,525] jepsen worker 16 - jepsen.util 7666 :info :write [5 45] :RequestCanceledException INFO [2019-12-17 12:31:38,526] jepsen worker 33 - jepsen.util 6513 :info :cas [11 [4 48]] :RequestCanceledException INFO [2019-12-17 12:31:38,527] jepsen worker 81 - jepsen.util 6561 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:31:38,528] jepsen worker 81 - jepsen.util 6561 :invoke :write [27 20] INFO [2019-12-17 12:31:38,531] jepsen worker 10 - jepsen.util 4960 :invoke :cas [3 [44 20]] INFO [2019-12-17 12:31:38,533] jepsen worker 51 - jepsen.util 4911 :invoke :write [17 9] INFO [2019-12-17 12:31:38,538] jepsen worker 82 - jepsen.util 7372 :invoke :write [27 37] INFO [2019-12-17 12:31:38,539] jepsen worker 84 - jepsen.util 6474 :invoke :read [28 nil] INFO [2019-12-17 12:31:38,540] jepsen worker 8 - jepsen.util 6038 :info :cas [2 [14 49]] :RequestCanceledException INFO [2019-12-17 12:31:38,541] jepsen worker 26 - jepsen.util 6866 :info :cas [8 [19 36]] :RequestCanceledException INFO [2019-12-17 12:31:38,548] jepsen worker 20 - jepsen.util 5150 :invoke :read [6 nil] INFO [2019-12-17 12:31:38,553] jepsen worker 16 - jepsen.util 7756 :invoke :read [5 nil] INFO [2019-12-17 12:31:38,554] jepsen worker 33 - jepsen.util 6603 :invoke :write [11 24] INFO [2019-12-17 12:31:38,557] jepsen worker 15 - jepsen.util 5685 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:31:38,558] jepsen worker 48 - jepsen.util 6168 :info :write [16 29] :RequestCanceledException INFO [2019-12-17 12:31:38,558] jepsen worker 15 - jepsen.util 5685 :invoke :write [5 28] INFO [2019-12-17 12:31:38,568] jepsen worker 8 - jepsen.util 6128 :invoke :write [2 34] INFO [2019-12-17 12:31:38,576] jepsen worker 26 - jepsen.util 6956 :invoke :write [8 14] INFO [2019-12-17 12:31:38,583] jepsen worker 14 - jepsen.util 5144 :ok :write [4 30] INFO [2019-12-17 12:31:38,584] jepsen worker 14 - jepsen.util 5144 :invoke :read [4 nil] INFO [2019-12-17 12:31:38,594] jepsen worker 6 - jepsen.util 6846 :info :cas [2 [6 8]] :RequestCanceledException INFO [2019-12-17 12:31:38,596] jepsen worker 34 - jepsen.util 6064 :info :cas [11 [24 20]] :RequestCanceledException INFO [2019-12-17 12:31:38,597] jepsen worker 48 - jepsen.util 6258 :invoke :read [16 nil] INFO [2019-12-17 12:31:38,614] jepsen worker 25 - jepsen.util 6865 :info :write [8 21] :RequestCanceledException INFO [2019-12-17 12:31:38,620] jepsen worker 34 - jepsen.util 6154 :invoke :read [11 nil] INFO [2019-12-17 12:31:38,626] jepsen worker 49 - jepsen.util 5809 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:31:38,626] jepsen worker 58 - jepsen.util 6808 :info :write [19 31] :RequestCanceledException INFO [2019-12-17 12:31:38,627] jepsen worker 17 - jepsen.util 6587 :info :write [5 1] :RequestCanceledException INFO [2019-12-17 12:31:38,628] jepsen worker 49 - jepsen.util 5809 :invoke :write [16 7] INFO [2019-12-17 12:31:38,629] jepsen worker 5 - jepsen.util 4505 :info :write [1 39] :RequestCanceledException INFO [2019-12-17 12:31:38,630] jepsen worker 75 - jepsen.util 6555 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:31:38,631] jepsen worker 75 - jepsen.util 6555 :invoke :read [25 nil] INFO [2019-12-17 12:31:38,633] jepsen worker 6 - jepsen.util 6936 :invoke :read [2 nil] INFO [2019-12-17 12:31:38,640] jepsen worker 76 - jepsen.util 6736 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:31:38,640] jepsen worker 62 - jepsen.util 6182 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:31:38,641] jepsen worker 76 - jepsen.util 6736 :invoke :write [25 43] INFO [2019-12-17 12:31:38,641] jepsen worker 62 - jepsen.util 6182 :invoke :write [20 11] INFO [2019-12-17 12:31:38,642] jepsen worker 25 - jepsen.util 6955 :invoke :write [8 17] INFO [2019-12-17 12:31:38,644] jepsen worker 5 - jepsen.util 4595 :invoke :read [1 nil] INFO [2019-12-17 12:31:38,644] jepsen worker 58 - jepsen.util 6898 :invoke :write [19 48] INFO [2019-12-17 12:31:38,645] jepsen worker 17 - jepsen.util 6677 :invoke :cas [5 [10 27]] INFO [2019-12-17 12:31:38,658] jepsen worker 43 - jepsen.util 6253 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:31:38,659] jepsen worker 57 - jepsen.util 6087 :info :write [19 17] :RequestCanceledException INFO [2019-12-17 12:31:38,660] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28679,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:31:38,660] jepsen worker 43 - jepsen.util 6253 :invoke :cas [14 [33 0]] INFO [2019-12-17 12:31:38,660] jepsen worker 66 - jepsen.util 6096 :info :write [22 6] :RequestCanceledException INFO [2019-12-17 12:31:38,662] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28679,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:31:38,663] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28684,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:31:38,664] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28684,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:31:38,692] jepsen worker 85 - jepsen.util 7555 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:31:38,693] jepsen worker 85 - jepsen.util 7555 :invoke :read [28 nil] INFO [2019-12-17 12:31:38,696] jepsen worker 40 - jepsen.util 7240 :info :write [13 29] :RequestCanceledException INFO [2019-12-17 12:31:38,697] jepsen worker 39 - jepsen.util 6249 :info :write [13 9] :RequestCanceledException INFO [2019-12-17 12:31:38,700] jepsen worker 35 - jepsen.util 6695 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:31:38,701] jepsen worker 35 - jepsen.util 6695 :invoke :cas [11 [27 2]] INFO [2019-12-17 12:31:38,706] jepsen worker 66 - jepsen.util 6186 :invoke :write [22 14] INFO [2019-12-17 12:31:38,706] jepsen worker 57 - jepsen.util 6177 :invoke :cas [19 [8 44]] INFO [2019-12-17 12:31:38,708] jepsen worker 60 - jepsen.util 7260 :info :write [20 5] :RequestCanceledException INFO [2019-12-17 12:31:38,708] jepsen worker 7 - jepsen.util 6757 :info :cas [2 [38 10]] :RequestCanceledException INFO [2019-12-17 12:31:38,716] jepsen worker 40 - jepsen.util 7330 :invoke :write [13 43] INFO [2019-12-17 12:31:38,717] jepsen worker 39 - jepsen.util 6339 :invoke :cas [13 [16 19]] INFO [2019-12-17 12:31:38,725] jepsen worker 22 - jepsen.util 6772 :info :write [7 43] :RequestCanceledException INFO [2019-12-17 12:31:38,725] jepsen worker 50 - jepsen.util 7070 :info :write [16 32] :RequestCanceledException INFO [2019-12-17 12:31:38,725] jepsen worker 77 - jepsen.util 6827 :info :write [25 35] :RequestCanceledException INFO [2019-12-17 12:31:38,726] jepsen worker 83 - jepsen.util 7103 :info :write [27 17] :RequestCanceledException INFO [2019-12-17 12:31:38,734] jepsen worker 60 - jepsen.util 7350 :invoke :cas [20 [14 33]] INFO [2019-12-17 12:31:38,734] jepsen worker 22 - jepsen.util 6862 :invoke :cas [7 [43 29]] INFO [2019-12-17 12:31:38,734] jepsen worker 50 - jepsen.util 7160 :invoke :read [16 nil] INFO [2019-12-17 12:31:38,735] jepsen worker 7 - jepsen.util 6847 :invoke :cas [2 [36 2]] INFO [2019-12-17 12:31:38,735] jepsen worker 65 - jepsen.util 4385 :fail :cas [21 [19 25]] :ValueNotSwapFrom INFO [2019-12-17 12:31:38,735] jepsen worker 77 - jepsen.util 6917 :invoke :read [25 nil] INFO [2019-12-17 12:31:38,735] jepsen worker 65 - jepsen.util 4385 :invoke :write [21 42] INFO [2019-12-17 12:31:38,736] jepsen worker 83 - jepsen.util 7193 :invoke :read [27 nil] INFO [2019-12-17 12:31:38,738] jepsen worker 38 - jepsen.util 4808 :ok :read [12 11] INFO [2019-12-17 12:31:38,739] jepsen worker 38 - jepsen.util 4808 :invoke :write [12 48] INFO [2019-12-17 12:31:38,739] jepsen worker 41 - jepsen.util 7151 :info :write [13 20] :RequestCanceledException INFO [2019-12-17 12:31:38,747] jepsen worker 41 - jepsen.util 7241 :invoke :read [13 nil] INFO [2019-12-17 12:31:38,749] jepsen worker 1 - jepsen.util 6481 :info :write [0 25] :RequestCanceledException INFO [2019-12-17 12:31:38,750] jepsen worker 0 - jepsen.util 7290 :info :cas [0 [25 20]] :RequestCanceledException INFO [2019-12-17 12:31:38,761] jepsen worker 1 - jepsen.util 6571 :invoke :read [0 nil] INFO [2019-12-17 12:31:38,762] jepsen worker 0 - jepsen.util 7380 :invoke :cas [0 [44 12]] INFO [2019-12-17 12:31:38,764] jepsen worker 9 - jepsen.util 5769 :fail :cas [3 [18 22]] :ValueNotSwapFrom INFO [2019-12-17 12:31:38,765] jepsen worker 9 - jepsen.util 5769 :invoke :read [3 nil] INFO [2019-12-17 12:31:38,773] jepsen worker 18 - jepsen.util 4968 :ok :write [6 7] INFO [2019-12-17 12:31:38,773] jepsen worker 18 - jepsen.util 4968 :invoke :read [6 nil] INFO [2019-12-17 12:31:38,787] jepsen worker 87 - jepsen.util 4677 :ok :read [29 7] INFO [2019-12-17 12:31:38,787] jepsen worker 87 - jepsen.util 4677 :invoke :read [29 nil] INFO [2019-12-17 12:31:38,792] jepsen worker 2 - jepsen.util 6122 :info :cas [0 [6 33]] :RequestCanceledException INFO [2019-12-17 12:31:38,792] jepsen worker 44 - jepsen.util 7334 :info :cas [14 [22 18]] :RequestCanceledException INFO [2019-12-17 12:31:38,796] jepsen worker 2 - jepsen.util 6212 :invoke :write [0 35] INFO [2019-12-17 12:31:38,796] jepsen worker 44 - jepsen.util 7424 :invoke :read [14 nil] INFO [2019-12-17 12:31:38,804] jepsen worker 28 - jepsen.util 3808 :ok :read [9 47] INFO [2019-12-17 12:31:38,804] jepsen worker 28 - jepsen.util 3808 :invoke :read [9 nil] INFO [2019-12-17 12:31:38,809] jepsen worker 72 - jepsen.util 5022 :ok :read [24 18] INFO [2019-12-17 12:31:38,809] jepsen worker 72 - jepsen.util 5022 :invoke :cas [24 [36 15]] INFO [2019-12-17 12:31:38,809] jepsen worker 73 - jepsen.util 4663 :ok :read [24 18] INFO [2019-12-17 12:31:38,809] jepsen worker 73 - jepsen.util 4663 :invoke :read [24 nil] INFO [2019-12-17 12:31:38,810] jepsen worker 19 - jepsen.util 4969 :ok :read [6 22] INFO [2019-12-17 12:31:38,810] jepsen worker 19 - jepsen.util 4969 :invoke :cas [6 [39 43]] INFO [2019-12-17 12:31:38,810] jepsen worker 10 - jepsen.util 4960 :fail :cas [3 [44 20]] :ValueNotSwapFrom INFO [2019-12-17 12:31:38,810] jepsen worker 10 - jepsen.util 4960 :invoke :cas [3 [42 11]] INFO [2019-12-17 12:31:38,810] jepsen worker 32 - jepsen.util 5162 :fail :read [10 nil] :RequestTimeoutException INFO [2019-12-17 12:31:38,810] jepsen worker 32 - jepsen.util 5162 :invoke :write [10 14] INFO [2019-12-17 12:31:38,810] jepsen worker 46 - jepsen.util 4906 :fail :cas [15 [13 15]] :ValueNotSwapFrom INFO [2019-12-17 12:31:38,810] jepsen worker 46 - jepsen.util 4906 :invoke :read [15 nil] INFO [2019-12-17 12:31:38,810] jepsen worker 3 - jepsen.util 5133 :fail :cas [1 [32 41]] :ValueNotSwapFrom INFO [2019-12-17 12:31:38,810] jepsen worker 3 - jepsen.util 5133 :invoke :write [1 39] INFO [2019-12-17 12:31:38,811] jepsen worker 74 - jepsen.util 5384 :fail :cas [24 [34 18]] :ValueNotSwapFrom INFO [2019-12-17 12:31:38,811] jepsen worker 74 - jepsen.util 5384 :invoke :read [24 nil] INFO [2019-12-17 12:31:38,815] jepsen worker 31 - jepsen.util 4801 :ok :read [10 19] INFO [2019-12-17 12:31:38,815] jepsen worker 31 - jepsen.util 4801 :invoke :write [10 40] INFO [2019-12-17 12:31:38,825] jepsen worker 42 - jepsen.util 7152 :info :cas [14 [31 26]] :RequestCanceledException INFO [2019-12-17 12:31:38,825] jepsen worker 67 - jepsen.util 6727 :info :write [22 26] :RequestCanceledException INFO [2019-12-17 12:31:38,825] jepsen worker 89 - jepsen.util 4589 :info :write [29 0] :RequestCanceledException INFO [2019-12-17 12:31:38,839] jepsen worker 24 - jepsen.util 6054 :info :write [8 48] :RequestCanceledException INFO [2019-12-17 12:31:38,839] jepsen worker 86 - jepsen.util 7016 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:31:38,840] jepsen worker 86 - jepsen.util 7016 :invoke :read [28 nil] INFO [2019-12-17 12:31:38,843] jepsen worker 42 - jepsen.util 7242 :invoke :cas [14 [43 15]] INFO [2019-12-17 12:31:38,843] jepsen worker 89 - jepsen.util 4679 :invoke :write [29 21] INFO [2019-12-17 12:31:38,843] jepsen worker 67 - jepsen.util 6817 :invoke :read [22 nil] INFO [2019-12-17 12:31:38,846] jepsen worker 53 - jepsen.util 5003 :ok :read [17 13] INFO [2019-12-17 12:31:38,847] jepsen worker 53 - jepsen.util 5003 :invoke :read [17 nil] INFO [2019-12-17 12:31:38,847] jepsen worker 24 - jepsen.util 6144 :invoke :write [8 2] INFO [2019-12-17 12:31:38,855] jepsen worker 56 - jepsen.util 5186 :fail :cas [18 [8 27]] :ValueNotSwapFrom INFO [2019-12-17 12:31:38,855] jepsen worker 56 - jepsen.util 5186 :invoke :cas [18 [11 4]] INFO [2019-12-17 12:31:38,861] jepsen worker 55 - jepsen.util 5995 :fail :cas [18 [39 48]] :ValueNotSwapFrom INFO [2019-12-17 12:31:38,861] jepsen worker 55 - jepsen.util 5995 :invoke :cas [18 [17 27]] INFO [2019-12-17 12:31:38,899] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28739,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:31:38,899] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28739,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:31:38,899] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28743,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:31:38,900] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28743,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:31:38,901] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28744,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:31:38,901] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28744,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:31:38,936] jepsen worker 52 - jepsen.util 5272 :ok :read [17 24] INFO [2019-12-17 12:31:38,936] jepsen worker 52 - jepsen.util 5272 :invoke :cas [17 [36 40]] INFO [2019-12-17 12:31:38,997] jepsen worker 29 - jepsen.util 3989 :ok :write [9 13] INFO [2019-12-17 12:31:38,997] jepsen worker 29 - jepsen.util 3989 :invoke :cas [9 [39 35]] INFO [2019-12-17 12:31:39,004] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28749,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:31:39,004] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28749,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:31:39,004] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28753,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:31:39,004] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28753,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:31:39,005] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28755,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:31:39,005] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28755,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:31:39,010] jepsen worker 27 - jepsen.util 4887 :info :write [9 28] :RequestCanceledException INFO [2019-12-17 12:31:39,030] jepsen worker 27 - jepsen.util 4977 :invoke :cas [9 [21 26]] INFO [2019-12-17 12:31:39,058] jepsen worker 72 - jepsen.util 5022 :fail :cas [24 [36 15]] :ValueNotSwapFrom INFO [2019-12-17 12:31:39,058] jepsen worker 72 - jepsen.util 5022 :invoke :read [24 nil] INFO [2019-12-17 12:31:39,061] jepsen worker 73 - jepsen.util 4663 :ok :read [24 18] INFO [2019-12-17 12:31:39,061] jepsen worker 73 - jepsen.util 4663 :invoke :cas [24 [3 48]] INFO [2019-12-17 12:31:39,071] jepsen worker 64 - jepsen.util 3844 :ok :write [21 42] INFO [2019-12-17 12:31:39,071] jepsen worker 64 - jepsen.util 3844 :invoke :read [21 nil] INFO [2019-12-17 12:31:39,083] jepsen worker 13 - jepsen.util 4783 :fail :cas [4 [1 6]] :ValueNotSwapFrom INFO [2019-12-17 12:31:39,084] jepsen worker 13 - jepsen.util 4783 :invoke :write [4 37] INFO [2019-12-17 12:31:39,092] jepsen worker 78 - jepsen.util 5028 :info :write [26 35] :RequestCanceledException INFO [2019-12-17 12:31:39,109] jepsen worker 19 - jepsen.util 4969 :fail :cas [6 [39 43]] :ValueNotSwapFrom INFO [2019-12-17 12:31:39,109] jepsen worker 19 - jepsen.util 4969 :invoke :read [6 nil] INFO [2019-12-17 12:31:39,110] jepsen worker 61 - jepsen.util 7531 :info :write [20 41] :RequestCanceledException INFO [2019-12-17 12:31:39,113] jepsen worker 78 - jepsen.util 5118 :invoke :read [26 nil] INFO [2019-12-17 12:31:39,126] jepsen worker 59 - jepsen.util 6539 :info :write [19 30] :RequestCanceledException INFO [2019-12-17 12:31:39,146] jepsen worker 61 - jepsen.util 7621 :invoke :write [20 22] INFO [2019-12-17 12:31:39,156] jepsen worker 21 - jepsen.util 6051 :info :cas [7 [44 13]] :RequestCanceledException INFO [2019-12-17 12:31:39,163] jepsen worker 45 - jepsen.util 5175 :ok :write [15 24] INFO [2019-12-17 12:31:39,163] jepsen worker 45 - jepsen.util 5175 :invoke :read [15 nil] INFO [2019-12-17 12:31:39,165] jepsen worker 59 - jepsen.util 6629 :invoke :read [19 nil] INFO [2019-12-17 12:31:39,166] jepsen worker 21 - jepsen.util 6141 :invoke :write [7 45] INFO [2019-12-17 12:31:39,186] jepsen worker 20 - jepsen.util 5150 :ok :read [6 7] INFO [2019-12-17 12:31:39,186] jepsen worker 20 - jepsen.util 5150 :invoke :read [6 nil] INFO [2019-12-17 12:31:39,209] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:31:39,209] jepsen nemesis - couchbase.util Rebalance check count: 215 INFO [2019-12-17 12:31:39,209] jepsen nemesis - couchbase.util Rebalance stuck count: 196 INFO [2019-12-17 12:31:39,211] jepsen worker 64 - jepsen.util 3844 :ok :read [21 42] INFO [2019-12-17 12:31:39,211] jepsen worker 64 - jepsen.util 3844 :invoke :cas [21 [25 36]] INFO [2019-12-17 12:31:39,212] jepsen worker 29 - jepsen.util 3989 :fail :cas [9 [39 35]] :ValueNotSwapFrom INFO [2019-12-17 12:31:39,212] jepsen worker 29 - jepsen.util 3989 :invoke :write [9 38] INFO [2019-12-17 12:31:39,223] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28766,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:31:39,223] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28766,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:31:39,224] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28767,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:31:39,224] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28767,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:31:39,226] jepsen worker 4 - jepsen.util 4234 :info :write [1 15] :RequestCanceledException INFO [2019-12-17 12:31:39,232] jepsen worker 4 - jepsen.util 4324 :invoke :write [1 5] INFO [2019-12-17 12:31:39,233] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28778,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:31:39,233] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28778,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:31:39,234] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28780,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:31:39,234] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28780,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:31:39,236] jepsen worker 23 - jepsen.util 6863 :info :cas [7 [47 1]] :RequestCanceledException INFO [2019-12-17 12:31:39,243] jepsen worker 23 - jepsen.util 6953 :invoke :read [7 nil] INFO [2019-12-17 12:31:39,252] jepsen worker 19 - jepsen.util 4969 :ok :read [6 7] INFO [2019-12-17 12:31:39,259] jepsen worker 19 - jepsen.util 4969 :invoke :cas [6 [13 14]] INFO [2019-12-17 12:31:39,266] jepsen worker 70 - jepsen.util 5650 :fail :cas [23 [22 27]] :ValueNotSwapFrom INFO [2019-12-17 12:31:39,266] jepsen worker 70 - jepsen.util 5650 :invoke :read [23 nil] INFO [2019-12-17 12:31:39,279] jepsen worker 27 - jepsen.util 4977 :fail :cas [9 [21 26]] :ValueNotSwapFrom INFO [2019-12-17 12:31:39,279] jepsen worker 73 - jepsen.util 4663 :fail :cas [24 [3 48]] :ValueNotSwapFrom INFO [2019-12-17 12:31:39,279] jepsen worker 27 - jepsen.util 4977 :invoke :cas [9 [12 43]] INFO [2019-12-17 12:31:39,279] jepsen worker 73 - jepsen.util 4663 :invoke :write [24 47] INFO [2019-12-17 12:31:39,328] jepsen worker 20 - jepsen.util 5150 :ok :read [6 7] INFO [2019-12-17 12:31:39,329] jepsen worker 20 - jepsen.util 5150 :invoke :cas [6 [48 10]] INFO [2019-12-17 12:31:39,331] jepsen worker 45 - jepsen.util 5175 :ok :read [15 24] INFO [2019-12-17 12:31:39,332] jepsen worker 45 - jepsen.util 5175 :invoke :write [15 19] INFO [2019-12-17 12:31:39,333] jepsen worker 54 - jepsen.util 5364 :fail :cas [18 [31 47]] :ValueNotSwapFrom INFO [2019-12-17 12:31:39,333] jepsen worker 54 - jepsen.util 5364 :invoke :cas [18 [28 47]] INFO [2019-12-17 12:31:39,337] jepsen worker 64 - jepsen.util 3844 :fail :cas [21 [25 36]] :ValueNotSwapFrom INFO [2019-12-17 12:31:39,337] jepsen worker 64 - jepsen.util 3844 :invoke :read [21 nil] INFO [2019-12-17 12:31:39,342] jepsen worker 79 - jepsen.util 4939 :ok :write [26 33] INFO [2019-12-17 12:31:39,342] jepsen worker 79 - jepsen.util 4939 :invoke :cas [26 [4 35]] INFO [2019-12-17 12:31:39,358] jepsen worker 88 - jepsen.util 5218 :fail :cas [29 [18 47]] :ValueNotSwapFrom INFO [2019-12-17 12:31:39,359] jepsen worker 88 - jepsen.util 5218 :invoke :read [29 nil] INFO [2019-12-17 12:31:39,360] jepsen worker 19 - jepsen.util 4969 :fail :cas [6 [13 14]] :ValueNotSwapFrom INFO [2019-12-17 12:31:39,360] jepsen worker 19 - jepsen.util 4969 :invoke :read [6 nil] INFO [2019-12-17 12:31:39,364] jepsen worker 27 - jepsen.util 4977 :fail :cas [9 [12 43]] :ValueNotSwapFrom INFO [2019-12-17 12:31:39,364] jepsen worker 27 - jepsen.util 4977 :invoke :read [9 nil] INFO [2019-12-17 12:31:39,410] jepsen worker 29 - jepsen.util 3989 :ok :write [9 38] INFO [2019-12-17 12:31:39,410] jepsen worker 29 - jepsen.util 3989 :invoke :cas [9 [0 47]] INFO [2019-12-17 12:31:39,421] jepsen worker 55 - jepsen.util 5995 :fail :cas [18 [17 27]] :ValueNotSwapFrom INFO [2019-12-17 12:31:39,421] jepsen worker 55 - jepsen.util 5995 :invoke :write [18 37] INFO [2019-12-17 12:31:39,424] jepsen worker 32 - jepsen.util 5162 :ok :write [10 14] INFO [2019-12-17 12:31:39,424] jepsen worker 32 - jepsen.util 5162 :invoke :write [10 36] INFO [2019-12-17 12:31:39,452] jepsen worker 64 - jepsen.util 3844 :ok :read [21 42] INFO [2019-12-17 12:31:39,452] jepsen worker 64 - jepsen.util 3844 :invoke :read [21 nil] INFO [2019-12-17 12:31:39,468] jepsen worker 20 - jepsen.util 5150 :fail :cas [6 [48 10]] :ValueNotSwapFrom INFO [2019-12-17 12:31:39,468] jepsen worker 20 - jepsen.util 5150 :invoke :cas [6 [17 14]] INFO [2019-12-17 12:31:39,488] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28799,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:31:39,488] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28799,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:31:39,496] jepsen worker 14 - jepsen.util 5144 :ok :read [4 43] INFO [2019-12-17 12:31:39,496] jepsen worker 14 - jepsen.util 5144 :invoke :write [4 35] INFO [2019-12-17 12:31:39,527] jepsen worker 80 - jepsen.util 5030 :info :write [26 49] :RequestCanceledException INFO [2019-12-17 12:31:39,532] jepsen worker 80 - jepsen.util 5120 :invoke :cas [26 [44 21]] INFO [2019-12-17 12:31:39,644] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28813,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:31:39,644] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28813,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:31:39,644] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28815,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:31:39,644] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28815,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:31:39,680] jepsen worker 87 - jepsen.util 4677 :ok :read [29 0] INFO [2019-12-17 12:31:39,681] jepsen worker 87 - jepsen.util 4677 :invoke :read [29 nil] INFO [2019-12-17 12:31:39,689] jepsen worker 72 - jepsen.util 5022 :ok :read [24 18] INFO [2019-12-17 12:31:39,689] jepsen worker 72 - jepsen.util 5022 :invoke :read [24 nil] INFO [2019-12-17 12:31:39,690] jepsen worker 88 - jepsen.util 5218 :ok :read [29 0] INFO [2019-12-17 12:31:39,690] jepsen worker 88 - jepsen.util 5218 :invoke :write [29 25] INFO [2019-12-17 12:31:39,692] jepsen worker 29 - jepsen.util 3989 :fail :cas [9 [0 47]] :ValueNotSwapFrom INFO [2019-12-17 12:31:39,692] jepsen worker 29 - jepsen.util 3989 :invoke :read [9 nil] INFO [2019-12-17 12:31:39,700] jepsen worker 78 - jepsen.util 5118 :ok :read [26 35] INFO [2019-12-17 12:31:39,701] jepsen worker 78 - jepsen.util 5118 :invoke :cas [26 [40 0]] INFO [2019-12-17 12:31:39,710] jepsen worker 64 - jepsen.util 3844 :ok :read [21 42] INFO [2019-12-17 12:31:39,710] jepsen worker 64 - jepsen.util 3844 :invoke :read [21 nil] INFO [2019-12-17 12:31:39,711] jepsen worker 20 - jepsen.util 5150 :fail :cas [6 [17 14]] :ValueNotSwapFrom INFO [2019-12-17 12:31:39,711] jepsen worker 20 - jepsen.util 5150 :invoke :cas [6 [22 39]] INFO [2019-12-17 12:31:39,727] jepsen worker 29 - jepsen.util 3989 :ok :read [9 38] INFO [2019-12-17 12:31:39,728] jepsen worker 29 - jepsen.util 3989 :invoke :read [9 nil] INFO [2019-12-17 12:31:39,770] jepsen worker 70 - jepsen.util 5650 :ok :read [23 47] INFO [2019-12-17 12:31:39,770] jepsen worker 70 - jepsen.util 5650 :invoke :cas [23 [8 23]] INFO [2019-12-17 12:31:39,817] jepsen worker 37 - jepsen.util 4807 :ok :read [12 11] INFO [2019-12-17 12:31:39,817] jepsen worker 37 - jepsen.util 4807 :invoke :write [12 28] INFO [2019-12-17 12:31:39,832] jepsen worker 29 - jepsen.util 3989 :ok :read [9 38] INFO [2019-12-17 12:31:39,835] jepsen worker 29 - jepsen.util 3989 :invoke :write [9 44] INFO [2019-12-17 12:31:39,838] jepsen worker 20 - jepsen.util 5150 :fail :cas [6 [22 39]] :ValueNotSwapFrom INFO [2019-12-17 12:31:39,838] jepsen worker 64 - jepsen.util 3844 :ok :read [21 42] INFO [2019-12-17 12:31:39,838] jepsen worker 20 - jepsen.util 5150 :invoke :write [6 32] INFO [2019-12-17 12:31:39,838] jepsen worker 64 - jepsen.util 3844 :invoke :read [21 nil] INFO [2019-12-17 12:31:39,845] jepsen worker 54 - jepsen.util 5364 :fail :cas [18 [28 47]] :ValueNotSwapFrom INFO [2019-12-17 12:31:39,845] jepsen worker 54 - jepsen.util 5364 :invoke :write [18 29] INFO [2019-12-17 12:31:39,864] jepsen worker 79 - jepsen.util 4939 :fail :cas [26 [4 35]] :ValueNotSwapFrom INFO [2019-12-17 12:31:39,865] jepsen worker 79 - jepsen.util 4939 :invoke :write [26 43] INFO [2019-12-17 12:31:39,886] jepsen worker 64 - jepsen.util 3844 :ok :read [21 42] INFO [2019-12-17 12:31:39,886] jepsen worker 64 - jepsen.util 3844 :invoke :cas [21 [10 3]] INFO [2019-12-17 12:31:39,892] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28829,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:31:39,892] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28829,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:31:39,892] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28832,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:31:39,892] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28832,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:31:39,892] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28833,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:31:39,892] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28833,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:31:39,938] jepsen worker 52 - jepsen.util 5272 :fail :cas [17 [36 40]] :ValueNotSwapFrom INFO [2019-12-17 12:31:39,938] jepsen worker 80 - jepsen.util 5120 :fail :cas [26 [44 21]] :ValueNotSwapFrom INFO [2019-12-17 12:31:39,939] jepsen worker 52 - jepsen.util 5272 :invoke :cas [17 [28 42]] INFO [2019-12-17 12:31:39,939] jepsen worker 80 - jepsen.util 5120 :invoke :read [26 nil] INFO [2019-12-17 12:31:39,944] jepsen worker 64 - jepsen.util 3844 :fail :cas [21 [10 3]] :ValueNotSwapFrom INFO [2019-12-17 12:31:39,945] jepsen worker 64 - jepsen.util 3844 :invoke :write [21 20] INFO [2019-12-17 12:31:39,945] jepsen worker 32 - jepsen.util 5162 :ok :write [10 36] INFO [2019-12-17 12:31:39,945] jepsen worker 32 - jepsen.util 5162 :invoke :cas [10 [22 37]] INFO [2019-12-17 12:31:39,945] jepsen worker 73 - jepsen.util 4663 :ok :write [24 47] INFO [2019-12-17 12:31:39,945] jepsen worker 73 - jepsen.util 4663 :invoke :cas [24 [41 17]] INFO [2019-12-17 12:31:39,982] jepsen worker 29 - jepsen.util 3989 :ok :write [9 44] INFO [2019-12-17 12:31:39,982] jepsen worker 29 - jepsen.util 3989 :invoke :write [9 20] INFO [2019-12-17 12:31:39,987] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28845,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:31:39,987] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28845,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:31:39,988] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28848,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:31:39,989] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28848,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:31:40,015] jepsen worker 5 - jepsen.util 4595 :ok :read [1 26] INFO [2019-12-17 12:31:40,015] jepsen worker 5 - jepsen.util 4595 :invoke :cas [1 [4 10]] INFO [2019-12-17 12:31:40,096] jepsen worker 72 - jepsen.util 5022 :ok :read [24 47] INFO [2019-12-17 12:31:40,096] jepsen worker 72 - jepsen.util 5022 :invoke :write [24 13] INFO [2019-12-17 12:31:40,147] jepsen worker 64 - jepsen.util 3844 :ok :write [21 20] INFO [2019-12-17 12:31:40,147] jepsen worker 64 - jepsen.util 3844 :invoke :cas [21 [48 36]] INFO [2019-12-17 12:31:40,148] jepsen worker 45 - jepsen.util 5175 :ok :write [15 19] INFO [2019-12-17 12:31:40,148] jepsen worker 45 - jepsen.util 5175 :invoke :read [15 nil] INFO [2019-12-17 12:31:40,203] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28854,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:31:40,210] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28854,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:31:40,211] jepsen worker 30 - jepsen.util 5070 :info :write [10 40] :RequestCanceledException INFO [2019-12-17 12:31:40,221] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28856,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:31:40,222] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28856,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:31:40,224] jepsen worker 30 - jepsen.util 5160 :invoke :read [10 nil] INFO [2019-12-17 12:31:40,225] jepsen worker 71 - jepsen.util 5471 :info :write [23 45] :RequestCanceledException INFO [2019-12-17 12:31:40,244] jepsen worker 71 - jepsen.util 5561 :invoke :cas [23 [44 39]] INFO [2019-12-17 12:31:40,249] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:31:40,249] jepsen nemesis - couchbase.util Rebalance check count: 216 INFO [2019-12-17 12:31:40,249] jepsen nemesis - couchbase.util Rebalance stuck count: 197 INFO [2019-12-17 12:31:40,269] jepsen worker 73 - jepsen.util 4663 :fail :cas [24 [41 17]] :ValueNotSwapFrom INFO [2019-12-17 12:31:40,269] jepsen worker 73 - jepsen.util 4663 :invoke :write [24 13] INFO [2019-12-17 12:31:40,276] jepsen worker 87 - jepsen.util 4677 :ok :read [29 0] INFO [2019-12-17 12:31:40,276] jepsen worker 87 - jepsen.util 4677 :invoke :cas [29 [42 22]] INFO [2019-12-17 12:31:40,289] jepsen worker 11 - jepsen.util 4961 :info :cas [3 [16 27]] :RequestCanceledException INFO [2019-12-17 12:31:40,291] jepsen worker 11 - jepsen.util 5051 :invoke :cas [3 [29 31]] INFO [2019-12-17 12:31:40,298] jepsen worker 80 - jepsen.util 5120 :ok :read [26 35] INFO [2019-12-17 12:31:40,298] jepsen worker 80 - jepsen.util 5120 :invoke :read [26 nil] INFO [2019-12-17 12:31:40,302] jepsen worker 19 - jepsen.util 4969 :ok :read [6 7] INFO [2019-12-17 12:31:40,303] jepsen worker 19 - jepsen.util 4969 :invoke :write [6 31] INFO [2019-12-17 12:31:40,311] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28858,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:31:40,312] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28858,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:31:40,312] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28864,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:31:40,312] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28864,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:31:40,312] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28865,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:31:40,312] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28865,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:31:40,314] jepsen worker 45 - jepsen.util 5175 :ok :read [15 19] INFO [2019-12-17 12:31:40,315] jepsen worker 45 - jepsen.util 5175 :invoke :read [15 nil] INFO [2019-12-17 12:31:40,333] jepsen worker 32 - jepsen.util 5162 :fail :cas [10 [22 37]] :ValueNotSwapFrom INFO [2019-12-17 12:31:40,333] jepsen worker 32 - jepsen.util 5162 :invoke :write [10 48] INFO [2019-12-17 12:31:40,350] jepsen worker 5 - jepsen.util 4595 :fail :cas [1 [4 10]] :ValueNotSwapFrom INFO [2019-12-17 12:31:40,350] jepsen worker 5 - jepsen.util 4595 :invoke :read [1 nil] INFO [2019-12-17 12:31:40,353] jepsen worker 20 - jepsen.util 5150 :ok :write [6 32] INFO [2019-12-17 12:31:40,353] jepsen worker 20 - jepsen.util 5150 :invoke :read [6 nil] INFO [2019-12-17 12:31:40,369] jepsen worker 30 - jepsen.util 5160 :ok :read [10 36] INFO [2019-12-17 12:31:40,369] jepsen worker 30 - jepsen.util 5160 :invoke :write [10 18] INFO [2019-12-17 12:31:40,382] jepsen worker 64 - jepsen.util 3844 :fail :cas [21 [48 36]] :ValueNotSwapFrom INFO [2019-12-17 12:31:40,382] jepsen worker 64 - jepsen.util 3844 :invoke :write [21 11] INFO [2019-12-17 12:31:40,386] jepsen worker 78 - jepsen.util 5118 :fail :cas [26 [40 0]] :ValueNotSwapFrom INFO [2019-12-17 12:31:40,386] jepsen worker 78 - jepsen.util 5118 :invoke :read [26 nil] INFO [2019-12-17 12:31:40,398] jepsen worker 20 - jepsen.util 5150 :ok :read [6 32] INFO [2019-12-17 12:31:40,398] jepsen worker 20 - jepsen.util 5150 :invoke :cas [6 [33 43]] INFO [2019-12-17 12:31:40,435] jepsen worker 70 - jepsen.util 5650 :fail :cas [23 [8 23]] :ValueNotSwapFrom INFO [2019-12-17 12:31:40,435] jepsen worker 70 - jepsen.util 5650 :invoke :read [23 nil] INFO [2019-12-17 12:31:40,451] jepsen worker 20 - jepsen.util 5150 :fail :cas [6 [33 43]] :ValueNotSwapFrom INFO [2019-12-17 12:31:40,451] jepsen worker 20 - jepsen.util 5150 :invoke :write [6 13] INFO [2019-12-17 12:31:40,473] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28882,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:31:40,473] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28882,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:31:40,473] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28886,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:31:40,473] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28886,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:31:40,483] jepsen worker 45 - jepsen.util 5175 :ok :read [15 19] INFO [2019-12-17 12:31:40,483] jepsen worker 45 - jepsen.util 5175 :invoke :write [15 28] INFO [2019-12-17 12:31:40,510] jepsen worker 71 - jepsen.util 5561 :fail :cas [23 [44 39]] :ValueNotSwapFrom INFO [2019-12-17 12:31:40,510] jepsen worker 71 - jepsen.util 5561 :invoke :write [23 7] INFO [2019-12-17 12:31:40,528] jepsen worker 79 - jepsen.util 4939 :ok :write [26 43] INFO [2019-12-17 12:31:40,528] jepsen worker 79 - jepsen.util 4939 :invoke :read [26 nil] INFO [2019-12-17 12:31:40,551] jepsen worker 72 - jepsen.util 5022 :ok :write [24 13] INFO [2019-12-17 12:31:40,551] jepsen worker 72 - jepsen.util 5022 :invoke :cas [24 [40 29]] INFO [2019-12-17 12:31:40,587] jepsen worker 20 - jepsen.util 5150 :ok :write [6 13] INFO [2019-12-17 12:31:40,587] jepsen worker 20 - jepsen.util 5150 :invoke :write [6 34] INFO [2019-12-17 12:31:40,609] jepsen worker 48 - jepsen.util 6258 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:31:40,609] jepsen worker 84 - jepsen.util 6474 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:31:40,609] jepsen worker 48 - jepsen.util 6258 :invoke :read [16 nil] INFO [2019-12-17 12:31:40,609] jepsen worker 84 - jepsen.util 6474 :invoke :read [28 nil] INFO [2019-12-17 12:31:40,633] jepsen worker 45 - jepsen.util 5175 :ok :write [15 28] INFO [2019-12-17 12:31:40,634] jepsen worker 45 - jepsen.util 5175 :invoke :cas [15 [41 22]] INFO [2019-12-17 12:31:40,640] jepsen worker 12 - jepsen.util 5232 :info :write [4 8] :RequestCanceledException INFO [2019-12-17 12:31:40,640] jepsen worker 16 - jepsen.util 7756 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:31:40,640] jepsen worker 16 - jepsen.util 7756 :invoke :write [5 4] INFO [2019-12-17 12:31:40,642] jepsen worker 12 - jepsen.util 5322 :invoke :cas [4 [21 26]] INFO [2019-12-17 12:31:40,656] jepsen worker 45 - jepsen.util 5175 :fail :cas [15 [41 22]] :ValueNotSwapFrom INFO [2019-12-17 12:31:40,657] jepsen worker 45 - jepsen.util 5175 :invoke :read [15 nil] INFO [2019-12-17 12:31:40,657] jepsen worker 15 - jepsen.util 5685 :info :write [5 28] :RequestCanceledException INFO [2019-12-17 12:31:40,659] jepsen worker 15 - jepsen.util 5775 :invoke :cas [5 [18 25]] INFO [2019-12-17 12:31:40,673] jepsen worker 5 - jepsen.util 4595 :ok :read [1 26] INFO [2019-12-17 12:31:40,673] jepsen worker 5 - jepsen.util 4595 :invoke :read [1 nil] INFO [2019-12-17 12:31:40,676] jepsen worker 64 - jepsen.util 3844 :ok :write [21 11] INFO [2019-12-17 12:31:40,677] jepsen worker 64 - jepsen.util 3844 :invoke :read [21 nil] INFO [2019-12-17 12:31:40,678] jepsen worker 45 - jepsen.util 5175 :ok :read [15 28] INFO [2019-12-17 12:31:40,678] jepsen worker 45 - jepsen.util 5175 :invoke :read [15 nil] INFO [2019-12-17 12:31:40,692] jepsen worker 68 - jepsen.util 6188 :info :write [22 47] :RequestCanceledException INFO [2019-12-17 12:31:40,692] jepsen worker 82 - jepsen.util 7372 :info :write [27 37] :RequestCanceledException INFO [2019-12-17 12:31:40,694] jepsen worker 68 - jepsen.util 6278 :invoke :write [22 34] INFO [2019-12-17 12:31:40,694] jepsen worker 82 - jepsen.util 7462 :invoke :cas [27 [39 38]] INFO [2019-12-17 12:31:40,703] jepsen worker 45 - jepsen.util 5175 :ok :read [15 28] INFO [2019-12-17 12:31:40,703] jepsen worker 17 - jepsen.util 6677 :info :cas [5 [10 27]] :RequestCanceledException INFO [2019-12-17 12:31:40,704] jepsen worker 45 - jepsen.util 5175 :invoke :write [15 8] INFO [2019-12-17 12:31:40,705] jepsen worker 17 - jepsen.util 6767 :invoke :read [5 nil] INFO [2019-12-17 12:31:40,707] jepsen worker 32 - jepsen.util 5162 :ok :write [10 48] INFO [2019-12-17 12:31:40,707] jepsen worker 32 - jepsen.util 5162 :invoke :write [10 45] INFO [2019-12-17 12:31:40,708] jepsen worker 64 - jepsen.util 3844 :ok :read [21 11] INFO [2019-12-17 12:31:40,708] jepsen worker 64 - jepsen.util 3844 :invoke :cas [21 [25 34]] INFO [2019-12-17 12:31:40,718] jepsen worker 75 - jepsen.util 6555 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:31:40,718] jepsen worker 34 - jepsen.util 6154 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:31:40,718] jepsen worker 81 - jepsen.util 6561 :info :write [27 20] :RequestCanceledException INFO [2019-12-17 12:31:40,718] jepsen worker 47 - jepsen.util 5627 :info :write [15 32] :RequestCanceledException INFO [2019-12-17 12:31:40,718] jepsen worker 51 - jepsen.util 4911 :info :write [17 9] :RequestCanceledException INFO [2019-12-17 12:31:40,718] jepsen worker 8 - jepsen.util 6128 :info :write [2 34] :RequestCanceledException INFO [2019-12-17 12:31:40,718] jepsen worker 75 - jepsen.util 6555 :invoke :write [25 45] INFO [2019-12-17 12:31:40,718] jepsen worker 34 - jepsen.util 6154 :invoke :cas [11 [44 11]] INFO [2019-12-17 12:31:40,719] jepsen worker 81 - jepsen.util 6651 :invoke :cas [27 [42 15]] INFO [2019-12-17 12:31:40,719] jepsen worker 47 - jepsen.util 5717 :invoke :read [15 nil] INFO [2019-12-17 12:31:40,719] jepsen worker 51 - jepsen.util 5001 :invoke :read [17 nil] INFO [2019-12-17 12:31:40,719] jepsen worker 8 - jepsen.util 6218 :invoke :cas [2 [36 22]] INFO [2019-12-17 12:31:40,738] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28898,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:31:40,738] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28898,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:31:40,738] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28901,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:31:40,738] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28901,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:31:40,738] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28902,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:31:40,738] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28902,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:31:40,739] jepsen worker 72 - jepsen.util 5022 :fail :cas [24 [40 29]] :ValueNotSwapFrom INFO [2019-12-17 12:31:40,739] jepsen worker 6 - jepsen.util 6936 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:31:40,739] jepsen worker 72 - jepsen.util 5022 :invoke :write [24 15] INFO [2019-12-17 12:31:40,739] jepsen worker 69 - jepsen.util 5379 :info :write [23 46] :RequestCanceledException INFO [2019-12-17 12:31:40,739] jepsen worker 57 - jepsen.util 6177 :info :cas [19 [8 44]] :RequestCanceledException INFO [2019-12-17 12:31:40,739] jepsen worker 6 - jepsen.util 6936 :invoke :cas [2 [39 0]] INFO [2019-12-17 12:31:40,740] jepsen worker 57 - jepsen.util 6267 :invoke :read [19 nil] INFO [2019-12-17 12:31:40,741] jepsen worker 69 - jepsen.util 5469 :invoke :write [23 23] INFO [2019-12-17 12:31:40,754] jepsen worker 20 - jepsen.util 5150 :ok :write [6 34] INFO [2019-12-17 12:31:40,755] jepsen worker 20 - jepsen.util 5150 :invoke :write [6 38] INFO [2019-12-17 12:31:40,757] jepsen worker 43 - jepsen.util 6253 :info :cas [14 [33 0]] :RequestCanceledException INFO [2019-12-17 12:31:40,757] jepsen worker 33 - jepsen.util 6603 :info :write [11 24] :RequestCanceledException INFO [2019-12-17 12:31:40,757] jepsen worker 63 - jepsen.util 4023 :info :write [21 36] :RequestCanceledException INFO [2019-12-17 12:31:40,757] jepsen worker 77 - jepsen.util 6917 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:31:40,757] jepsen worker 77 - jepsen.util 6917 :invoke :write [25 30] INFO [2019-12-17 12:31:40,758] jepsen worker 33 - jepsen.util 6693 :invoke :write [11 47] INFO [2019-12-17 12:31:40,758] jepsen worker 43 - jepsen.util 6343 :invoke :write [14 45] INFO [2019-12-17 12:31:40,758] jepsen worker 63 - jepsen.util 4113 :invoke :write [21 34] INFO [2019-12-17 12:31:40,763] jepsen worker 64 - jepsen.util 3844 :fail :cas [21 [25 34]] :ValueNotSwapFrom INFO [2019-12-17 12:31:40,763] jepsen worker 64 - jepsen.util 3844 :invoke :read [21 nil] INFO [2019-12-17 12:31:40,774] jepsen worker 47 - jepsen.util 5717 :ok :read [15 28] INFO [2019-12-17 12:31:40,774] jepsen worker 47 - jepsen.util 5717 :invoke :read [15 nil] INFO [2019-12-17 12:31:40,788] jepsen worker 51 - jepsen.util 5001 :ok :read [17 24] INFO [2019-12-17 12:31:40,789] jepsen worker 51 - jepsen.util 5001 :invoke :read [17 nil] INFO [2019-12-17 12:31:40,789] jepsen worker 87 - jepsen.util 4677 :fail :cas [29 [42 22]] :ValueNotSwapFrom INFO [2019-12-17 12:31:40,789] jepsen worker 87 - jepsen.util 4677 :invoke :cas [29 [10 5]] INFO [2019-12-17 12:31:40,792] jepsen worker 26 - jepsen.util 6956 :info :write [8 14] :RequestCanceledException INFO [2019-12-17 12:31:40,792] jepsen worker 85 - jepsen.util 7555 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:31:40,792] jepsen worker 35 - jepsen.util 6695 :info :cas [11 [27 2]] :RequestCanceledException INFO [2019-12-17 12:31:40,792] jepsen worker 60 - jepsen.util 7350 :info :cas [20 [14 33]] :RequestCanceledException INFO [2019-12-17 12:31:40,792] jepsen worker 41 - jepsen.util 7241 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:31:40,792] jepsen worker 58 - jepsen.util 6898 :info :write [19 48] :RequestCanceledException INFO [2019-12-17 12:31:40,792] jepsen worker 18 - jepsen.util 4968 :fail :read [6 nil] :RequestTimeoutException INFO [2019-12-17 12:31:40,792] jepsen worker 85 - jepsen.util 7555 :invoke :write [28 19] INFO [2019-12-17 12:31:40,792] jepsen worker 41 - jepsen.util 7241 :invoke :cas [13 [43 14]] INFO [2019-12-17 12:31:40,792] jepsen worker 18 - jepsen.util 4968 :invoke :cas [6 [12 41]] INFO [2019-12-17 12:31:40,795] jepsen worker 26 - jepsen.util 7046 :invoke :write [8 42] INFO [2019-12-17 12:31:40,795] jepsen worker 58 - jepsen.util 6988 :invoke :write [19 5] INFO [2019-12-17 12:31:40,796] jepsen worker 35 - jepsen.util 6785 :invoke :cas [11 [43 14]] INFO [2019-12-17 12:31:40,796] jepsen worker 60 - jepsen.util 7440 :invoke :read [20 nil] INFO [2019-12-17 12:31:40,810] jepsen worker 22 - jepsen.util 6862 :info :cas [7 [43 29]] :RequestCanceledException INFO [2019-12-17 12:31:40,810] jepsen worker 1 - jepsen.util 6571 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:31:40,811] jepsen worker 1 - jepsen.util 6571 :invoke :read [0 nil] INFO [2019-12-17 12:31:40,812] jepsen worker 22 - jepsen.util 6952 :invoke :cas [7 [5 42]] INFO [2019-12-17 12:31:40,814] jepsen worker 18 - jepsen.util 4968 :fail :cas [6 [12 41]] :ValueNotSwapFrom INFO [2019-12-17 12:31:40,814] jepsen worker 18 - jepsen.util 4968 :invoke :cas [6 [22 7]] INFO [2019-12-17 12:31:40,818] jepsen worker 64 - jepsen.util 3844 :ok :read [21 11] INFO [2019-12-17 12:31:40,818] jepsen worker 64 - jepsen.util 3844 :invoke :write [21 1] INFO [2019-12-17 12:31:40,825] jepsen worker 9 - jepsen.util 5769 :fail :read [3 nil] :RequestTimeoutException INFO [2019-12-17 12:31:40,825] jepsen worker 50 - jepsen.util 7160 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:31:40,825] jepsen worker 0 - jepsen.util 7380 :info :cas [0 [44 12]] :RequestCanceledException INFO [2019-12-17 12:31:40,825] jepsen worker 28 - jepsen.util 3808 :fail :read [9 nil] :RequestTimeoutException INFO [2019-12-17 12:31:40,825] jepsen worker 38 - jepsen.util 4808 :info :write [12 48] :RequestCanceledException INFO [2019-12-17 12:31:40,826] jepsen worker 66 - jepsen.util 6186 :info :write [22 14] :RequestCanceledException INFO [2019-12-17 12:31:40,826] jepsen worker 49 - jepsen.util 5809 :info :write [16 7] :RequestCanceledException INFO [2019-12-17 12:31:40,826] jepsen worker 39 - jepsen.util 6339 :info :cas [13 [16 19]] :RequestCanceledException INFO [2019-12-17 12:31:40,826] jepsen worker 83 - jepsen.util 7193 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:31:40,826] jepsen worker 9 - jepsen.util 5769 :invoke :read [3 nil] INFO [2019-12-17 12:31:40,826] jepsen worker 50 - jepsen.util 7160 :invoke :write [16 2] INFO [2019-12-17 12:31:40,826] jepsen worker 28 - jepsen.util 3808 :invoke :write [9 7] INFO [2019-12-17 12:31:40,826] jepsen worker 83 - jepsen.util 7193 :invoke :read [27 nil] INFO [2019-12-17 12:31:40,828] jepsen worker 0 - jepsen.util 7470 :invoke :read [0 nil] INFO [2019-12-17 12:31:40,828] jepsen worker 39 - jepsen.util 6429 :invoke :cas [13 [28 25]] INFO [2019-12-17 12:31:40,828] jepsen worker 38 - jepsen.util 4898 :invoke :write [12 4] INFO [2019-12-17 12:31:40,829] jepsen worker 49 - jepsen.util 5899 :invoke :cas [16 [8 22]] INFO [2019-12-17 12:31:40,829] jepsen worker 66 - jepsen.util 6276 :invoke :cas [22 [11 36]] INFO [2019-12-17 12:31:40,838] jepsen worker 18 - jepsen.util 4968 :fail :cas [6 [22 7]] :ValueNotSwapFrom INFO [2019-12-17 12:31:40,838] jepsen worker 18 - jepsen.util 4968 :invoke :read [6 nil] INFO [2019-12-17 12:31:40,839] jepsen worker 7 - jepsen.util 6847 :info :cas [2 [36 2]] :RequestCanceledException INFO [2019-12-17 12:31:40,839] jepsen worker 76 - jepsen.util 6736 :info :write [25 43] :RequestCanceledException INFO [2019-12-17 12:31:40,839] jepsen worker 62 - jepsen.util 6182 :info :write [20 11] :RequestCanceledException INFO [2019-12-17 12:31:40,839] jepsen worker 2 - jepsen.util 6212 :info :write [0 35] :RequestCanceledException INFO [2019-12-17 12:31:40,840] jepsen worker 2 - jepsen.util 6302 :invoke :read [0 nil] INFO [2019-12-17 12:31:40,840] jepsen worker 7 - jepsen.util 6937 :invoke :read [2 nil] INFO [2019-12-17 12:31:40,840] jepsen worker 76 - jepsen.util 6826 :invoke :read [25 nil] INFO [2019-12-17 12:31:40,841] jepsen worker 62 - jepsen.util 6272 :invoke :write [20 48] INFO [2019-12-17 12:31:40,853] jepsen worker 25 - jepsen.util 6955 :info :write [8 17] :RequestCanceledException INFO [2019-12-17 12:31:40,853] jepsen worker 40 - jepsen.util 7330 :info :write [13 43] :RequestCanceledException INFO [2019-12-17 12:31:40,853] jepsen worker 65 - jepsen.util 4385 :info :write [21 42] :RequestCanceledException INFO [2019-12-17 12:31:40,854] jepsen worker 44 - jepsen.util 7424 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:31:40,854] jepsen worker 74 - jepsen.util 5384 :fail :read [24 nil] :RequestTimeoutException INFO [2019-12-17 12:31:40,854] jepsen worker 10 - jepsen.util 4960 :info :cas [3 [42 11]] :RequestCanceledException INFO [2019-12-17 12:31:40,854] jepsen worker 74 - jepsen.util 5384 :invoke :read [24 nil] INFO [2019-12-17 12:31:40,854] jepsen worker 44 - jepsen.util 7424 :invoke :cas [14 [38 47]] INFO [2019-12-17 12:31:40,855] jepsen worker 40 - jepsen.util 7420 :invoke :read [13 nil] INFO [2019-12-17 12:31:40,855] jepsen worker 65 - jepsen.util 4475 :invoke :write [21 15] INFO [2019-12-17 12:31:40,855] jepsen worker 25 - jepsen.util 7045 :invoke :write [8 17] INFO [2019-12-17 12:31:40,855] jepsen worker 10 - jepsen.util 5050 :invoke :read [3 nil] INFO [2019-12-17 12:31:40,862] jepsen worker 53 - jepsen.util 5003 :ok :read [17 24] INFO [2019-12-17 12:31:40,862] jepsen worker 53 - jepsen.util 5003 :invoke :cas [17 [6 42]] INFO [2019-12-17 12:31:40,864] jepsen worker 80 - jepsen.util 5120 :ok :read [26 43] INFO [2019-12-17 12:31:40,864] jepsen worker 80 - jepsen.util 5120 :invoke :write [26 29] INFO [2019-12-17 12:31:40,878] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28966,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:31:40,878] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28966,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:31:40,892] jepsen worker 46 - jepsen.util 4906 :fail :read [15 nil] :RequestTimeoutException INFO [2019-12-17 12:31:40,892] jepsen worker 89 - jepsen.util 4679 :info :write [29 21] :RequestCanceledException INFO [2019-12-17 12:31:40,892] jepsen worker 46 - jepsen.util 4906 :invoke :read [15 nil] INFO [2019-12-17 12:31:40,893] jepsen worker 89 - jepsen.util 4769 :invoke :write [29 19] INFO [2019-12-17 12:31:40,897] jepsen worker 74 - jepsen.util 5384 :ok :read [24 13] INFO [2019-12-17 12:31:40,897] jepsen worker 74 - jepsen.util 5384 :invoke :write [24 28] INFO [2019-12-17 12:31:40,899] jepsen worker 78 - jepsen.util 5118 :ok :read [26 43] INFO [2019-12-17 12:31:40,899] jepsen worker 78 - jepsen.util 5118 :invoke :cas [26 [1 48]] INFO [2019-12-17 12:31:40,902] jepsen worker 64 - jepsen.util 3844 :ok :write [21 1] INFO [2019-12-17 12:31:40,902] jepsen worker 64 - jepsen.util 3844 :invoke :write [21 17] INFO [2019-12-17 12:31:40,907] jepsen worker 67 - jepsen.util 6817 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:31:40,907] jepsen worker 31 - jepsen.util 4801 :info :write [10 40] :RequestCanceledException INFO [2019-12-17 12:31:40,907] jepsen worker 67 - jepsen.util 6817 :invoke :cas [22 [27 36]] INFO [2019-12-17 12:31:40,908] jepsen worker 31 - jepsen.util 4891 :invoke :read [10 nil] INFO [2019-12-17 12:31:40,925] jepsen worker 42 - jepsen.util 7242 :info :cas [14 [43 15]] :RequestCanceledException INFO [2019-12-17 12:31:40,925] jepsen worker 3 - jepsen.util 5133 :info :write [1 39] :RequestCanceledException INFO [2019-12-17 12:31:40,925] jepsen worker 24 - jepsen.util 6144 :info :write [8 2] :RequestCanceledException INFO [2019-12-17 12:31:40,926] jepsen worker 79 - jepsen.util 4939 :ok :read [26 43] INFO [2019-12-17 12:31:40,926] jepsen worker 79 - jepsen.util 4939 :invoke :write [26 15] INFO [2019-12-17 12:31:40,927] jepsen worker 24 - jepsen.util 6234 :invoke :cas [8 [16 9]] INFO [2019-12-17 12:31:40,927] jepsen worker 42 - jepsen.util 7332 :invoke :cas [14 [27 4]] INFO [2019-12-17 12:31:40,927] jepsen worker 3 - jepsen.util 5223 :invoke :read [1 nil] INFO [2019-12-17 12:31:40,933] jepsen worker 55 - jepsen.util 5995 :ok :write [18 37] INFO [2019-12-17 12:31:40,934] jepsen worker 55 - jepsen.util 5995 :invoke :cas [18 [46 20]] INFO [2019-12-17 12:31:40,939] jepsen worker 86 - jepsen.util 7016 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:31:40,939] jepsen worker 86 - jepsen.util 7016 :invoke :write [28 21] INFO [2019-12-17 12:31:40,940] jepsen worker 56 - jepsen.util 5186 :fail :cas [18 [11 4]] :ValueNotSwapFrom INFO [2019-12-17 12:31:40,941] jepsen worker 56 - jepsen.util 5186 :invoke :read [18 nil] INFO [2019-12-17 12:31:40,963] jepsen worker 46 - jepsen.util 4906 :ok :read [15 28] INFO [2019-12-17 12:31:40,963] jepsen worker 46 - jepsen.util 4906 :invoke :read [15 nil] INFO [2019-12-17 12:31:40,968] jepsen worker 70 - jepsen.util 5650 :ok :read [23 47] INFO [2019-12-17 12:31:40,968] jepsen worker 70 - jepsen.util 5650 :invoke :write [23 40] INFO [2019-12-17 12:31:40,979] jepsen worker 51 - jepsen.util 5001 :ok :read [17 24] INFO [2019-12-17 12:31:40,979] jepsen worker 51 - jepsen.util 5001 :invoke :cas [17 [3 4]] INFO [2019-12-17 12:31:40,981] jepsen worker 31 - jepsen.util 4891 :ok :read [10 48] INFO [2019-12-17 12:31:40,981] jepsen worker 31 - jepsen.util 4891 :invoke :read [10 nil] INFO [2019-12-17 12:31:40,999] jepsen worker 31 - jepsen.util 4891 :ok :read [10 48] INFO [2019-12-17 12:31:41,000] jepsen worker 31 - jepsen.util 4891 :invoke :read [10 nil] INFO [2019-12-17 12:31:41,027] jepsen worker 53 - jepsen.util 5003 :fail :cas [17 [6 42]] :ValueNotSwapFrom INFO [2019-12-17 12:31:41,027] jepsen worker 53 - jepsen.util 5003 :invoke :cas [17 [29 34]] INFO [2019-12-17 12:31:41,054] jepsen worker 64 - jepsen.util 3844 :ok :write [21 17] INFO [2019-12-17 12:31:41,054] jepsen worker 31 - jepsen.util 4891 :ok :read [10 48] INFO [2019-12-17 12:31:41,054] jepsen worker 64 - jepsen.util 3844 :invoke :write [21 33] INFO [2019-12-17 12:31:41,055] jepsen worker 31 - jepsen.util 4891 :invoke :cas [10 [8 10]] INFO [2019-12-17 12:31:41,080] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28993,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:31:41,080] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28993,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:31:41,080] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28997,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:31:41,081] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":28997,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:31:41,082] jepsen worker 37 - jepsen.util 4807 :ok :write [12 28] INFO [2019-12-17 12:31:41,082] jepsen worker 37 - jepsen.util 4807 :invoke :read [12 nil] INFO [2019-12-17 12:31:41,096] jepsen worker 31 - jepsen.util 4891 :fail :cas [10 [8 10]] :ValueNotSwapFrom INFO [2019-12-17 12:31:41,096] jepsen worker 31 - jepsen.util 4891 :invoke :read [10 nil] INFO [2019-12-17 12:31:41,129] jepsen worker 31 - jepsen.util 4891 :ok :read [10 48] INFO [2019-12-17 12:31:41,129] jepsen worker 31 - jepsen.util 4891 :invoke :read [10 nil] INFO [2019-12-17 12:31:41,138] jepsen worker 5 - jepsen.util 4595 :ok :read [1 26] INFO [2019-12-17 12:31:41,138] jepsen worker 20 - jepsen.util 5150 :ok :write [6 38] INFO [2019-12-17 12:31:41,138] jepsen worker 5 - jepsen.util 4595 :invoke :read [1 nil] INFO [2019-12-17 12:31:41,138] jepsen worker 20 - jepsen.util 5150 :invoke :read [6 nil] INFO [2019-12-17 12:31:41,139] jepsen worker 64 - jepsen.util 3844 :ok :write [21 33] INFO [2019-12-17 12:31:41,139] jepsen worker 64 - jepsen.util 3844 :invoke :cas [21 [32 33]] INFO [2019-12-17 12:31:41,139] jepsen worker 45 - jepsen.util 5175 :ok :write [15 8] INFO [2019-12-17 12:31:41,139] jepsen worker 45 - jepsen.util 5175 :invoke :write [15 37] INFO [2019-12-17 12:31:41,154] jepsen worker 32 - jepsen.util 5162 :ok :write [10 45] INFO [2019-12-17 12:31:41,154] jepsen worker 32 - jepsen.util 5162 :invoke :cas [10 [48 37]] INFO [2019-12-17 12:31:41,165] jepsen worker 31 - jepsen.util 4891 :ok :read [10 48] INFO [2019-12-17 12:31:41,165] jepsen worker 31 - jepsen.util 4891 :invoke :read [10 nil] INFO [2019-12-17 12:31:41,193] jepsen worker 59 - jepsen.util 6629 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:31:41,194] jepsen worker 59 - jepsen.util 6629 :invoke :write [19 4] INFO [2019-12-17 12:31:41,204] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29012,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:31:41,205] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29012,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:31:41,205] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29013,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:31:41,205] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29013,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:31:41,207] jepsen worker 21 - jepsen.util 6141 :info :write [7 45] :RequestCanceledException INFO [2019-12-17 12:31:41,212] jepsen worker 74 - jepsen.util 5384 :ok :write [24 28] INFO [2019-12-17 12:31:41,212] jepsen worker 74 - jepsen.util 5384 :invoke :write [24 42] INFO [2019-12-17 12:31:41,216] jepsen worker 21 - jepsen.util 6231 :invoke :cas [7 [11 45]] INFO [2019-12-17 12:31:41,218] jepsen worker 55 - jepsen.util 5995 :fail :cas [18 [46 20]] :ValueNotSwapFrom INFO [2019-12-17 12:31:41,218] jepsen worker 55 - jepsen.util 5995 :invoke :read [18 nil] INFO [2019-12-17 12:31:41,222] jepsen worker 56 - jepsen.util 5186 :ok :read [18 37] INFO [2019-12-17 12:31:41,222] jepsen worker 56 - jepsen.util 5186 :invoke :read [18 nil] INFO [2019-12-17 12:31:41,225] jepsen worker 61 - jepsen.util 7621 :info :write [20 22] :RequestCanceledException INFO [2019-12-17 12:31:41,231] jepsen worker 61 - jepsen.util 7711 :invoke :read [20 nil] INFO [2019-12-17 12:31:41,231] jepsen worker 87 - jepsen.util 4677 :fail :cas [29 [10 5]] :ValueNotSwapFrom INFO [2019-12-17 12:31:41,231] jepsen worker 87 - jepsen.util 4677 :invoke :cas [29 [44 40]] INFO [2019-12-17 12:31:41,251] jepsen worker 64 - jepsen.util 3844 :fail :cas [21 [32 33]] :ValueNotSwapFrom INFO [2019-12-17 12:31:41,251] jepsen worker 64 - jepsen.util 3844 :invoke :cas [21 [20 37]] INFO [2019-12-17 12:31:41,257] jepsen worker 9 - jepsen.util 5769 :ok :read [3 38] INFO [2019-12-17 12:31:41,257] jepsen worker 9 - jepsen.util 5769 :invoke :cas [3 [45 30]] INFO [2019-12-17 12:31:41,266] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:31:41,266] jepsen nemesis - couchbase.util Rebalance check count: 217 INFO [2019-12-17 12:31:41,266] jepsen nemesis - couchbase.util Rebalance stuck count: 198 INFO [2019-12-17 12:31:41,270] jepsen worker 28 - jepsen.util 3808 :ok :write [9 7] INFO [2019-12-17 12:31:41,270] jepsen worker 28 - jepsen.util 3808 :invoke :read [9 nil] INFO [2019-12-17 12:31:41,274] jepsen worker 10 - jepsen.util 5050 :ok :read [3 38] INFO [2019-12-17 12:31:41,275] jepsen worker 10 - jepsen.util 5050 :invoke :write [3 32] INFO [2019-12-17 12:31:41,292] jepsen worker 13 - jepsen.util 4783 :info :write [4 37] :RequestCanceledException INFO [2019-12-17 12:31:41,295] jepsen worker 31 - jepsen.util 4891 :ok :read [10 45] INFO [2019-12-17 12:31:41,295] jepsen worker 31 - jepsen.util 4891 :invoke :read [10 nil] INFO [2019-12-17 12:31:41,299] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29020,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:31:41,299] jepsen worker 13 - jepsen.util 4873 :invoke :write [4 42] INFO [2019-12-17 12:31:41,300] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29020,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:31:41,326] jepsen worker 4 - jepsen.util 4324 :info :write [1 5] :RequestCanceledException INFO [2019-12-17 12:31:41,331] jepsen worker 4 - jepsen.util 4414 :invoke :cas [1 [39 24]] INFO [2019-12-17 12:31:41,339] jepsen worker 64 - jepsen.util 3844 :fail :cas [21 [20 37]] :ValueNotSwapFrom INFO [2019-12-17 12:31:41,339] jepsen worker 64 - jepsen.util 3844 :invoke :cas [21 [30 49]] INFO [2019-12-17 12:31:41,339] jepsen worker 23 - jepsen.util 6953 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:31:41,339] jepsen worker 23 - jepsen.util 6953 :invoke :write [7 1] INFO [2019-12-17 12:31:41,362] jepsen worker 20 - jepsen.util 5150 :ok :read [6 38] INFO [2019-12-17 12:31:41,362] jepsen worker 20 - jepsen.util 5150 :invoke :cas [6 [43 19]] INFO [2019-12-17 12:31:41,368] jepsen worker 51 - jepsen.util 5001 :fail :cas [17 [3 4]] :ValueNotSwapFrom INFO [2019-12-17 12:31:41,368] jepsen worker 51 - jepsen.util 5001 :invoke :cas [17 [43 2]] INFO [2019-12-17 12:31:41,385] jepsen worker 31 - jepsen.util 4891 :ok :read [10 45] INFO [2019-12-17 12:31:41,385] jepsen worker 31 - jepsen.util 4891 :invoke :write [10 4] INFO [2019-12-17 12:31:41,398] jepsen worker 53 - jepsen.util 5003 :fail :cas [17 [29 34]] :ValueNotSwapFrom INFO [2019-12-17 12:31:41,399] jepsen worker 53 - jepsen.util 5003 :invoke :cas [17 [47 36]] INFO [2019-12-17 12:31:41,402] jepsen worker 9 - jepsen.util 5769 :fail :cas [3 [45 30]] :ValueNotSwapFrom INFO [2019-12-17 12:31:41,403] jepsen worker 9 - jepsen.util 5769 :invoke :write [3 3] INFO [2019-12-17 12:31:41,424] jepsen worker 28 - jepsen.util 3808 :ok :read [9 7] INFO [2019-12-17 12:31:41,425] jepsen worker 28 - jepsen.util 3808 :invoke :read [9 nil] INFO [2019-12-17 12:31:41,438] jepsen worker 64 - jepsen.util 3844 :fail :cas [21 [30 49]] :ValueNotSwapFrom INFO [2019-12-17 12:31:41,438] jepsen worker 64 - jepsen.util 3844 :invoke :read [21 nil] INFO [2019-12-17 12:31:41,440] jepsen worker 27 - jepsen.util 4977 :fail :read [9 nil] :RequestTimeoutException INFO [2019-12-17 12:31:41,440] jepsen worker 27 - jepsen.util 4977 :invoke :read [9 nil] INFO [2019-12-17 12:31:41,440] jepsen worker 20 - jepsen.util 5150 :fail :cas [6 [43 19]] :ValueNotSwapFrom INFO [2019-12-17 12:31:41,440] jepsen worker 20 - jepsen.util 5150 :invoke :read [6 nil] INFO [2019-12-17 12:31:41,463] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29046,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:31:41,463] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29046,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:31:41,478] jepsen worker 64 - jepsen.util 3844 :ok :read [21 33] INFO [2019-12-17 12:31:41,478] jepsen worker 64 - jepsen.util 3844 :invoke :write [21 14] INFO [2019-12-17 12:31:41,479] jepsen worker 72 - jepsen.util 5022 :ok :write [24 15] INFO [2019-12-17 12:31:41,479] jepsen worker 72 - jepsen.util 5022 :invoke :read [24 nil] INFO [2019-12-17 12:31:41,518] jepsen worker 32 - jepsen.util 5162 :fail :cas [10 [48 37]] :ValueNotSwapFrom INFO [2019-12-17 12:31:41,518] jepsen worker 32 - jepsen.util 5162 :invoke :write [10 35] INFO [2019-12-17 12:31:41,526] jepsen worker 14 - jepsen.util 5144 :info :write [4 35] :RequestCanceledException INFO [2019-12-17 12:31:41,531] jepsen worker 14 - jepsen.util 5234 :invoke :write [4 36] INFO [2019-12-17 12:31:41,531] jepsen worker 87 - jepsen.util 4677 :fail :cas [29 [44 40]] :ValueNotSwapFrom INFO [2019-12-17 12:31:41,532] jepsen worker 87 - jepsen.util 4677 :invoke :read [29 nil] INFO [2019-12-17 12:31:41,554] jepsen worker 37 - jepsen.util 4807 :ok :read [12 28] INFO [2019-12-17 12:31:41,558] jepsen worker 56 - jepsen.util 5186 :ok :read [18 37] INFO [2019-12-17 12:31:41,577] jepsen worker 72 - jepsen.util 5022 :ok :read [24 15] INFO [2019-12-17 12:31:41,588] jepsen worker 64 - jepsen.util 3844 :ok :write [21 14] INFO [2019-12-17 12:31:41,623] jepsen worker 37 - jepsen.util 4807 :invoke :cas [12 [0 29]] INFO [2019-12-17 12:31:41,623] jepsen worker 64 - jepsen.util 3844 :invoke :cas [21 [16 43]] INFO [2019-12-17 12:31:41,623] jepsen worker 56 - jepsen.util 5186 :invoke :write [18 23] INFO [2019-12-17 12:31:41,624] jepsen worker 72 - jepsen.util 5022 :invoke :cas [24 [18 43]] INFO [2019-12-17 12:31:41,655] jepsen worker 55 - jepsen.util 5995 :ok :read [18 37] INFO [2019-12-17 12:31:41,657] jepsen worker 36 - jepsen.util 5166 :info :cas [12 [11 44]] :RequestCanceledException INFO [2019-12-17 12:31:41,663] jepsen worker 31 - jepsen.util 4891 :ok :write [10 4] INFO [2019-12-17 12:31:41,663] jepsen worker 45 - jepsen.util 5175 :ok :write [15 37] INFO [2019-12-17 12:31:41,670] jepsen worker 31 - jepsen.util 4891 :invoke :write [10 11] INFO [2019-12-17 12:31:41,670] jepsen worker 45 - jepsen.util 5175 :invoke :read [15 nil] INFO [2019-12-17 12:31:41,671] jepsen worker 55 - jepsen.util 5995 :invoke :write [18 21] INFO [2019-12-17 12:31:41,685] jepsen worker 9 - jepsen.util 5769 :ok :write [3 3] INFO [2019-12-17 12:31:41,686] jepsen worker 9 - jepsen.util 5769 :invoke :cas [3 [48 33]] INFO [2019-12-17 12:31:41,690] jepsen worker 36 - jepsen.util 5256 :invoke :write [12 43] INFO [2019-12-17 12:31:41,727] jepsen worker 88 - jepsen.util 5218 :info :write [29 25] :RequestCanceledException INFO [2019-12-17 12:31:41,731] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29066,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:31:41,736] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29066,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:31:41,740] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29067,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:31:41,743] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29067,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:31:41,773] jepsen worker 88 - jepsen.util 5308 :invoke :write [29 11] INFO [2019-12-17 12:31:41,783] jepsen worker 10 - jepsen.util 5050 :ok :write [3 32] INFO [2019-12-17 12:31:41,783] jepsen worker 10 - jepsen.util 5050 :invoke :read [3 nil] INFO [2019-12-17 12:31:41,811] jepsen worker 52 - jepsen.util 5272 :fail :cas [17 [28 42]] :ValueNotSwapFrom INFO [2019-12-17 12:31:41,811] jepsen worker 52 - jepsen.util 5272 :invoke :write [17 44] INFO [2019-12-17 12:31:41,868] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29076,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:31:41,869] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29076,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:31:41,870] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29080,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:31:41,871] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29080,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:31:41,908] jepsen worker 54 - jepsen.util 5364 :info :write [18 29] :RequestCanceledException INFO [2019-12-17 12:31:41,918] jepsen worker 72 - jepsen.util 5022 :fail :cas [24 [18 43]] :ValueNotSwapFrom INFO [2019-12-17 12:31:41,918] jepsen worker 72 - jepsen.util 5022 :invoke :read [24 nil] INFO [2019-12-17 12:31:41,918] jepsen worker 54 - jepsen.util 5454 :invoke :write [18 17] INFO [2019-12-17 12:31:41,988] jepsen worker 28 - jepsen.util 3808 :ok :read [9 7] INFO [2019-12-17 12:31:41,988] jepsen worker 28 - jepsen.util 3808 :invoke :write [9 27] INFO [2019-12-17 12:31:42,026] jepsen worker 29 - jepsen.util 3989 :info :write [9 20] :RequestCanceledException INFO [2019-12-17 12:31:42,030] jepsen worker 64 - jepsen.util 3844 :fail :cas [21 [16 43]] :ValueNotSwapFrom INFO [2019-12-17 12:31:42,031] jepsen worker 64 - jepsen.util 3844 :invoke :write [21 7] INFO [2019-12-17 12:31:42,039] jepsen worker 29 - jepsen.util 4079 :invoke :cas [9 [12 27]] INFO [2019-12-17 12:31:42,051] jepsen worker 12 - jepsen.util 5322 :fail :cas [4 [21 26]] :ValueNotSwapFrom INFO [2019-12-17 12:31:42,051] jepsen worker 12 - jepsen.util 5322 :invoke :write [4 37] INFO [2019-12-17 12:31:42,067] jepsen worker 5 - jepsen.util 4595 :ok :read [1 26] INFO [2019-12-17 12:31:42,067] jepsen worker 5 - jepsen.util 4595 :invoke :read [1 nil] INFO [2019-12-17 12:31:42,074] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29085,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:31:42,075] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29085,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:31:42,077] jepsen worker 45 - jepsen.util 5175 :ok :read [15 37] INFO [2019-12-17 12:31:42,078] jepsen worker 45 - jepsen.util 5175 :invoke :read [15 nil] INFO [2019-12-17 12:31:42,078] jepsen worker 51 - jepsen.util 5001 :fail :cas [17 [43 2]] :ValueNotSwapFrom INFO [2019-12-17 12:31:42,078] jepsen worker 51 - jepsen.util 5001 :invoke :cas [17 [7 0]] INFO [2019-12-17 12:31:42,078] jepsen worker 9 - jepsen.util 5769 :fail :cas [3 [48 33]] :ValueNotSwapFrom INFO [2019-12-17 12:31:42,078] jepsen worker 9 - jepsen.util 5769 :invoke :read [3 nil] INFO [2019-12-17 12:31:42,087] jepsen worker 10 - jepsen.util 5050 :ok :read [3 32] INFO [2019-12-17 12:31:42,087] jepsen worker 10 - jepsen.util 5050 :invoke :cas [3 [21 44]] INFO [2019-12-17 12:31:42,097] jepsen worker 72 - jepsen.util 5022 :ok :read [24 15] INFO [2019-12-17 12:31:42,097] jepsen worker 72 - jepsen.util 5022 :invoke :write [24 33] INFO [2019-12-17 12:31:42,100] jepsen worker 3 - jepsen.util 5223 :ok :read [1 26] INFO [2019-12-17 12:31:42,100] jepsen worker 3 - jepsen.util 5223 :invoke :read [1 nil] INFO [2019-12-17 12:31:42,125] jepsen worker 78 - jepsen.util 5118 :fail :cas [26 [1 48]] :ValueNotSwapFrom INFO [2019-12-17 12:31:42,126] jepsen worker 78 - jepsen.util 5118 :invoke :write [26 23] INFO [2019-12-17 12:31:42,130] jepsen worker 4 - jepsen.util 4414 :fail :cas [1 [39 24]] :ValueNotSwapFrom INFO [2019-12-17 12:31:42,130] jepsen worker 4 - jepsen.util 4414 :invoke :read [1 nil] INFO [2019-12-17 12:31:42,156] jepsen worker 53 - jepsen.util 5003 :fail :cas [17 [47 36]] :ValueNotSwapFrom INFO [2019-12-17 12:31:42,156] jepsen worker 53 - jepsen.util 5003 :invoke :write [17 10] INFO [2019-12-17 12:31:42,170] jepsen worker 45 - jepsen.util 5175 :ok :read [15 37] INFO [2019-12-17 12:31:42,171] jepsen worker 45 - jepsen.util 5175 :invoke :write [15 31] INFO [2019-12-17 12:31:42,192] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29106,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:31:42,192] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29106,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:31:42,192] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29107,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:31:42,193] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29107,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:31:42,193] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29108,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:31:42,193] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29108,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:31:42,223] jepsen worker 10 - jepsen.util 5050 :fail :cas [3 [21 44]] :ValueNotSwapFrom INFO [2019-12-17 12:31:42,223] jepsen worker 10 - jepsen.util 5050 :invoke :write [3 25] INFO [2019-12-17 12:31:42,231] jepsen worker 28 - jepsen.util 3808 :ok :write [9 27] INFO [2019-12-17 12:31:42,231] jepsen worker 28 - jepsen.util 3808 :invoke :read [9 nil] INFO [2019-12-17 12:31:42,268] jepsen worker 64 - jepsen.util 3844 :ok :write [21 7] INFO [2019-12-17 12:31:42,268] jepsen worker 64 - jepsen.util 3844 :invoke :cas [21 [6 43]] INFO [2019-12-17 12:31:42,281] jepsen worker 9 - jepsen.util 5769 :ok :read [3 32] INFO [2019-12-17 12:31:42,281] jepsen worker 9 - jepsen.util 5769 :invoke :cas [3 [0 41]] INFO [2019-12-17 12:31:42,286] jepsen worker 28 - jepsen.util 3808 :ok :read [9 27] INFO [2019-12-17 12:31:42,287] jepsen worker 28 - jepsen.util 3808 :invoke :write [9 37] INFO [2019-12-17 12:31:42,292] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29112,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:31:42,292] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29112,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:31:42,292] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29117,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:31:42,293] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29117,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:31:42,298] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:31:42,298] jepsen nemesis - couchbase.util Rebalance check count: 218 INFO [2019-12-17 12:31:42,298] jepsen nemesis - couchbase.util Rebalance stuck count: 199 INFO [2019-12-17 12:31:42,308] jepsen worker 11 - jepsen.util 5051 :info :cas [3 [29 31]] :RequestCanceledException INFO [2019-12-17 12:31:42,312] jepsen worker 11 - jepsen.util 5141 :invoke :read [3 nil] INFO [2019-12-17 12:31:42,317] jepsen worker 73 - jepsen.util 4663 :info :write [24 13] :RequestCanceledException INFO [2019-12-17 12:31:42,319] jepsen worker 73 - jepsen.util 4753 :invoke :read [24 nil] INFO [2019-12-17 12:31:42,320] jepsen worker 64 - jepsen.util 3844 :fail :cas [21 [6 43]] :ValueNotSwapFrom INFO [2019-12-17 12:31:42,320] jepsen worker 64 - jepsen.util 3844 :invoke :write [21 22] INFO [2019-12-17 12:31:42,336] jepsen worker 9 - jepsen.util 5769 :fail :cas [3 [0 41]] :ValueNotSwapFrom INFO [2019-12-17 12:31:42,336] jepsen worker 9 - jepsen.util 5769 :invoke :cas [3 [10 47]] INFO [2019-12-17 12:31:42,352] jepsen worker 11 - jepsen.util 5141 :ok :read [3 32] INFO [2019-12-17 12:31:42,352] jepsen worker 11 - jepsen.util 5141 :invoke :cas [3 [16 31]] INFO [2019-12-17 12:31:42,416] jepsen worker 28 - jepsen.util 3808 :ok :write [9 37] INFO [2019-12-17 12:31:42,416] jepsen worker 28 - jepsen.util 3808 :invoke :cas [9 [34 13]] INFO [2019-12-17 12:31:42,425] jepsen worker 19 - jepsen.util 4969 :info :write [6 31] :RequestCanceledException INFO [2019-12-17 12:31:42,425] jepsen worker 30 - jepsen.util 5160 :info :write [10 18] :RequestCanceledException INFO [2019-12-17 12:31:42,428] jepsen worker 19 - jepsen.util 5059 :invoke :cas [6 [35 12]] INFO [2019-12-17 12:31:42,428] jepsen worker 30 - jepsen.util 5250 :invoke :read [10 nil] INFO [2019-12-17 12:31:42,444] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29129,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:31:42,444] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29129,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:31:42,444] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29133,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:31:42,445] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29133,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:31:42,445] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29134,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:31:42,445] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29134,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:31:42,473] jepsen worker 64 - jepsen.util 3844 :ok :write [21 22] INFO [2019-12-17 12:31:42,473] jepsen worker 64 - jepsen.util 3844 :invoke :cas [21 [1 23]] INFO [2019-12-17 12:31:42,499] jepsen worker 31 - jepsen.util 4891 :ok :write [10 11] INFO [2019-12-17 12:31:42,499] jepsen worker 31 - jepsen.util 4891 :invoke :read [10 nil] INFO [2019-12-17 12:31:42,592] jepsen worker 71 - jepsen.util 5561 :info :write [23 7] :RequestCanceledException INFO [2019-12-17 12:31:42,599] jepsen worker 71 - jepsen.util 5651 :invoke :write [23 6] INFO [2019-12-17 12:31:42,600] jepsen worker 64 - jepsen.util 3844 :fail :cas [21 [1 23]] :ValueNotSwapFrom INFO [2019-12-17 12:31:42,600] jepsen worker 64 - jepsen.util 3844 :invoke :cas [21 [47 27]] INFO [2019-12-17 12:31:42,607] jepsen worker 28 - jepsen.util 3808 :fail :cas [9 [34 13]] :ValueNotSwapFrom INFO [2019-12-17 12:31:42,607] jepsen worker 28 - jepsen.util 3808 :invoke :cas [9 [38 16]] INFO [2019-12-17 12:31:42,609] jepsen worker 9 - jepsen.util 5769 :fail :cas [3 [10 47]] :ValueNotSwapFrom INFO [2019-12-17 12:31:42,609] jepsen worker 9 - jepsen.util 5769 :invoke :cas [3 [3 14]] INFO [2019-12-17 12:31:42,612] jepsen worker 29 - jepsen.util 4079 :fail :cas [9 [12 27]] :ValueNotSwapFrom INFO [2019-12-17 12:31:42,612] jepsen worker 29 - jepsen.util 4079 :invoke :write [9 43] INFO [2019-12-17 12:31:42,618] jepsen worker 19 - jepsen.util 5059 :fail :cas [6 [35 12]] :ValueNotSwapFrom INFO [2019-12-17 12:31:42,618] jepsen worker 19 - jepsen.util 5059 :invoke :read [6 nil] INFO [2019-12-17 12:31:42,622] jepsen worker 31 - jepsen.util 4891 :ok :read [10 11] INFO [2019-12-17 12:31:42,622] jepsen worker 31 - jepsen.util 4891 :invoke :cas [10 [43 38]] INFO [2019-12-17 12:31:42,625] jepsen worker 4 - jepsen.util 4414 :ok :read [1 26] INFO [2019-12-17 12:31:42,625] jepsen worker 4 - jepsen.util 4414 :invoke :read [1 nil] INFO [2019-12-17 12:31:42,649] jepsen worker 45 - jepsen.util 5175 :ok :write [15 31] INFO [2019-12-17 12:31:42,649] jepsen worker 72 - jepsen.util 5022 :ok :write [24 33] INFO [2019-12-17 12:31:42,649] jepsen worker 45 - jepsen.util 5175 :invoke :write [15 13] INFO [2019-12-17 12:31:42,649] jepsen worker 72 - jepsen.util 5022 :invoke :cas [24 [5 6]] INFO [2019-12-17 12:31:42,666] jepsen worker 5 - jepsen.util 4595 :ok :read [1 26] INFO [2019-12-17 12:31:42,666] jepsen worker 5 - jepsen.util 4595 :invoke :write [1 48] INFO [2019-12-17 12:31:42,710] jepsen worker 48 - jepsen.util 6258 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:31:42,710] jepsen worker 48 - jepsen.util 6258 :invoke :read [16 nil] INFO [2019-12-17 12:31:42,710] jepsen worker 84 - jepsen.util 6474 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:31:42,711] jepsen worker 84 - jepsen.util 6474 :invoke :write [28 34] INFO [2019-12-17 12:31:42,713] jepsen worker 64 - jepsen.util 3844 :fail :cas [21 [47 27]] :ValueNotSwapFrom INFO [2019-12-17 12:31:42,714] jepsen worker 64 - jepsen.util 3844 :invoke :read [21 nil] INFO [2019-12-17 12:31:42,725] jepsen worker 82 - jepsen.util 7462 :info :cas [27 [39 38]] :RequestCanceledException INFO [2019-12-17 12:31:42,726] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29147,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:31:42,732] jepsen worker 16 - jepsen.util 7756 :info :write [5 4] :RequestCanceledException INFO [2019-12-17 12:31:42,733] jepsen worker 15 - jepsen.util 5775 :info :cas [5 [18 25]] :RequestCanceledException INFO [2019-12-17 12:31:42,736] jepsen worker 19 - jepsen.util 5059 :ok :read [6 38] INFO [2019-12-17 12:31:42,743] jepsen worker 28 - jepsen.util 3808 :fail :cas [9 [38 16]] :ValueNotSwapFrom INFO [2019-12-17 12:31:42,744] jepsen worker 73 - jepsen.util 4753 :ok :read [24 33] INFO [2019-12-17 12:31:42,745] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29147,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:31:42,745] jepsen worker 19 - jepsen.util 5059 :invoke :read [6 nil] INFO [2019-12-17 12:31:42,745] jepsen worker 28 - jepsen.util 3808 :invoke :write [9 39] INFO [2019-12-17 12:31:42,745] jepsen worker 73 - jepsen.util 4753 :invoke :write [24 10] INFO [2019-12-17 12:31:42,745] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29151,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:31:42,746] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29151,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:31:42,748] jepsen worker 9 - jepsen.util 5769 :fail :cas [3 [3 14]] :ValueNotSwapFrom INFO [2019-12-17 12:31:42,748] jepsen worker 9 - jepsen.util 5769 :invoke :cas [3 [9 4]] INFO [2019-12-17 12:31:42,749] jepsen worker 16 - jepsen.util 7846 :invoke :write [5 2] INFO [2019-12-17 12:31:42,749] jepsen worker 82 - jepsen.util 7552 :invoke :write [27 21] INFO [2019-12-17 12:31:42,753] jepsen worker 15 - jepsen.util 5865 :invoke :write [5 18] INFO [2019-12-17 12:31:42,757] jepsen worker 68 - jepsen.util 6278 :info :write [22 34] :RequestCanceledException INFO [2019-12-17 12:31:42,757] jepsen worker 8 - jepsen.util 6218 :info :cas [2 [36 22]] :RequestCanceledException INFO [2019-12-17 12:31:42,765] jepsen worker 8 - jepsen.util 6308 :invoke :cas [2 [21 9]] INFO [2019-12-17 12:31:42,765] jepsen worker 68 - jepsen.util 6368 :invoke :cas [22 [49 10]] INFO [2019-12-17 12:31:42,778] jepsen worker 72 - jepsen.util 5022 :fail :cas [24 [5 6]] :ValueNotSwapFrom INFO [2019-12-17 12:31:42,783] jepsen worker 72 - jepsen.util 5022 :invoke :write [24 8] INFO [2019-12-17 12:31:42,784] jepsen worker 69 - jepsen.util 5469 :info :write [23 23] :RequestCanceledException INFO [2019-12-17 12:31:42,784] jepsen worker 17 - jepsen.util 6767 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:31:42,785] jepsen worker 17 - jepsen.util 6767 :invoke :read [5 nil] INFO [2019-12-17 12:31:42,794] jepsen worker 69 - jepsen.util 5559 :invoke :cas [23 [1 12]] INFO [2019-12-17 12:31:42,798] jepsen worker 31 - jepsen.util 4891 :fail :cas [10 [43 38]] :ValueNotSwapFrom INFO [2019-12-17 12:31:42,798] jepsen worker 31 - jepsen.util 4891 :invoke :cas [10 [25 26]] INFO [2019-12-17 12:31:42,805] jepsen worker 81 - jepsen.util 6651 :info :cas [27 [42 15]] :RequestCanceledException INFO [2019-12-17 12:31:42,806] jepsen worker 33 - jepsen.util 6693 :info :write [11 47] :RequestCanceledException INFO [2019-12-17 12:31:42,806] jepsen worker 60 - jepsen.util 7440 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:31:42,806] jepsen worker 60 - jepsen.util 7440 :invoke :cas [20 [10 15]] INFO [2019-12-17 12:31:42,814] jepsen worker 33 - jepsen.util 6783 :invoke :cas [11 [11 25]] INFO [2019-12-17 12:31:42,814] jepsen worker 81 - jepsen.util 6741 :invoke :write [27 36] INFO [2019-12-17 12:31:42,818] jepsen worker 29 - jepsen.util 4079 :ok :write [9 43] INFO [2019-12-17 12:31:42,818] jepsen worker 29 - jepsen.util 4079 :invoke :read [9 nil] INFO [2019-12-17 12:31:42,826] jepsen worker 75 - jepsen.util 6555 :info :write [25 45] :RequestCanceledException INFO [2019-12-17 12:31:42,827] jepsen worker 34 - jepsen.util 6154 :info :cas [11 [44 11]] :RequestCanceledException INFO [2019-12-17 12:31:42,827] jepsen worker 47 - jepsen.util 5717 :fail :read [15 nil] :RequestTimeoutException INFO [2019-12-17 12:31:42,827] jepsen worker 58 - jepsen.util 6988 :info :write [19 5] :RequestCanceledException INFO [2019-12-17 12:31:42,827] jepsen worker 57 - jepsen.util 6267 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:31:42,827] jepsen worker 43 - jepsen.util 6343 :info :write [14 45] :RequestCanceledException INFO [2019-12-17 12:31:42,827] jepsen worker 22 - jepsen.util 6952 :info :cas [7 [5 42]] :RequestCanceledException INFO [2019-12-17 12:31:42,827] jepsen worker 47 - jepsen.util 5717 :invoke :read [15 nil] INFO [2019-12-17 12:31:42,828] jepsen worker 57 - jepsen.util 6267 :invoke :write [19 34] INFO [2019-12-17 12:31:42,840] jepsen worker 75 - jepsen.util 6645 :invoke :cas [25 [32 3]] INFO [2019-12-17 12:31:42,841] jepsen worker 6 - jepsen.util 6936 :info :cas [2 [39 0]] :RequestCanceledException INFO [2019-12-17 12:31:42,841] jepsen worker 63 - jepsen.util 4113 :info :write [21 34] :RequestCanceledException INFO [2019-12-17 12:31:42,841] jepsen worker 0 - jepsen.util 7470 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:31:42,841] jepsen worker 22 - jepsen.util 7042 :invoke :read [7 nil] INFO [2019-12-17 12:31:42,841] jepsen worker 0 - jepsen.util 7470 :invoke :write [0 19] INFO [2019-12-17 12:31:42,844] jepsen worker 58 - jepsen.util 7078 :invoke :write [19 22] INFO [2019-12-17 12:31:42,844] jepsen worker 34 - jepsen.util 6244 :invoke :write [11 39] INFO [2019-12-17 12:31:42,845] jepsen worker 43 - jepsen.util 6433 :invoke :write [14 38] INFO [2019-12-17 12:31:42,847] jepsen worker 63 - jepsen.util 4203 :invoke :cas [21 [1 38]] INFO [2019-12-17 12:31:42,854] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29179,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:31:42,854] jepsen worker 77 - jepsen.util 6917 :info :write [25 30] :RequestCanceledException INFO [2019-12-17 12:31:42,854] jepsen worker 39 - jepsen.util 6429 :info :cas [13 [28 25]] :RequestCanceledException INFO [2019-12-17 12:31:42,856] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29179,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:31:42,856] jepsen worker 76 - jepsen.util 6826 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:31:42,856] jepsen worker 26 - jepsen.util 7046 :info :write [8 42] :RequestCanceledException INFO [2019-12-17 12:31:42,859] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29182,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:31:42,859] jepsen worker 6 - jepsen.util 7026 :invoke :cas [2 [15 5]] INFO [2019-12-17 12:31:42,859] jepsen worker 76 - jepsen.util 6826 :invoke :cas [25 [45 40]] INFO [2019-12-17 12:31:42,860] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29182,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:31:42,876] jepsen worker 77 - jepsen.util 7007 :invoke :cas [25 [11 21]] INFO [2019-12-17 12:31:42,877] jepsen worker 39 - jepsen.util 6519 :invoke :read [13 nil] INFO [2019-12-17 12:31:42,882] jepsen worker 26 - jepsen.util 7136 :invoke :write [8 17] INFO [2019-12-17 12:31:42,893] jepsen worker 35 - jepsen.util 6785 :info :cas [11 [43 14]] :RequestCanceledException INFO [2019-12-17 12:31:42,893] jepsen worker 66 - jepsen.util 6276 :info :cas [22 [11 36]] :RequestCanceledException INFO [2019-12-17 12:31:42,893] jepsen worker 41 - jepsen.util 7241 :info :cas [13 [43 14]] :RequestCanceledException INFO [2019-12-17 12:31:42,894] jepsen worker 85 - jepsen.util 7555 :info :write [28 19] :RequestCanceledException INFO [2019-12-17 12:31:42,894] jepsen worker 18 - jepsen.util 4968 :fail :read [6 nil] :RequestTimeoutException INFO [2019-12-17 12:31:42,895] jepsen worker 18 - jepsen.util 4968 :invoke :cas [6 [41 8]] INFO [2019-12-17 12:31:42,895] jepsen worker 40 - jepsen.util 7420 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:31:42,896] jepsen worker 40 - jepsen.util 7420 :invoke :cas [13 [47 14]] INFO [2019-12-17 12:31:42,913] jepsen worker 1 - jepsen.util 6571 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:31:42,915] jepsen worker 1 - jepsen.util 6571 :invoke :write [0 6] INFO [2019-12-17 12:31:42,915] jepsen worker 65 - jepsen.util 4475 :info :write [21 15] :RequestCanceledException INFO [2019-12-17 12:31:42,918] jepsen worker 65 - jepsen.util 4565 :invoke :cas [21 [12 27]] INFO [2019-12-17 12:31:42,928] jepsen worker 83 - jepsen.util 7193 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:31:42,928] jepsen worker 83 - jepsen.util 7193 :invoke :read [27 nil] INFO [2019-12-17 12:31:42,928] jepsen worker 49 - jepsen.util 5899 :info :cas [16 [8 22]] :RequestCanceledException INFO [2019-12-17 12:31:42,935] jepsen worker 50 - jepsen.util 7160 :info :write [16 2] :RequestCanceledException INFO [2019-12-17 12:31:42,940] jepsen worker 7 - jepsen.util 6937 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:31:42,944] jepsen worker 62 - jepsen.util 6272 :info :write [20 48] :RequestCanceledException INFO [2019-12-17 12:31:42,947] jepsen worker 7 - jepsen.util 6937 :invoke :write [2 43] INFO [2019-12-17 12:31:42,948] jepsen worker 2 - jepsen.util 6302 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:31:42,948] jepsen worker 38 - jepsen.util 4898 :info :write [12 4] :RequestCanceledException INFO [2019-12-17 12:31:42,950] jepsen worker 25 - jepsen.util 7045 :info :write [8 17] :RequestCanceledException INFO [2019-12-17 12:31:42,953] jepsen worker 35 - jepsen.util 6875 :invoke :write [11 45] INFO [2019-12-17 12:31:42,955] jepsen worker 2 - jepsen.util 6302 :invoke :cas [0 [48 24]] INFO [2019-12-17 12:31:42,958] jepsen worker 89 - jepsen.util 4769 :info :write [29 19] :RequestCanceledException INFO [2019-12-17 12:31:42,960] jepsen worker 41 - jepsen.util 7331 :invoke :cas [13 [36 28]] INFO [2019-12-17 12:31:42,962] jepsen worker 79 - jepsen.util 4939 :info :write [26 15] :RequestCanceledException INFO [2019-12-17 12:31:42,965] jepsen worker 44 - jepsen.util 7424 :info :cas [14 [38 47]] :RequestCanceledException INFO [2019-12-17 12:31:42,967] jepsen worker 85 - jepsen.util 7645 :invoke :read [28 nil] INFO [2019-12-17 12:31:42,980] jepsen worker 80 - jepsen.util 5120 :info :write [26 29] :RequestCanceledException INFO [2019-12-17 12:31:42,997] jepsen worker 66 - jepsen.util 6366 :invoke :read [22 nil] INFO [2019-12-17 12:31:43,008] jepsen worker 24 - jepsen.util 6234 :info :cas [8 [16 9]] :RequestCanceledException INFO [2019-12-17 12:31:43,011] jepsen worker 46 - jepsen.util 4906 :fail :read [15 nil] :RequestTimeoutException INFO [2019-12-17 12:31:43,011] jepsen worker 67 - jepsen.util 6817 :info :cas [22 [27 36]] :RequestCanceledException INFO [2019-12-17 12:31:43,011] jepsen worker 46 - jepsen.util 4906 :invoke :write [15 23] INFO [2019-12-17 12:31:43,031] jepsen worker 42 - jepsen.util 7332 :info :cas [14 [27 4]] :RequestCanceledException INFO [2019-12-17 12:31:43,042] jepsen worker 86 - jepsen.util 7016 :info :write [28 21] :RequestCanceledException INFO [2019-12-17 12:31:43,069] jepsen worker 70 - jepsen.util 5650 :info :write [23 40] :RequestCanceledException INFO [2019-12-17 12:31:43,070] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29200,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:31:43,084] jepsen worker 49 - jepsen.util 5989 :invoke :write [16 15] INFO [2019-12-17 12:31:43,091] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29200,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:31:43,123] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29202,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:31:43,131] jepsen worker 25 - jepsen.util 7135 :invoke :cas [8 [18 18]] INFO [2019-12-17 12:31:43,133] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29202,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:31:43,138] jepsen worker 79 - jepsen.util 5029 :invoke :read [26 nil] INFO [2019-12-17 12:31:43,179] jepsen worker 50 - jepsen.util 7250 :invoke :write [16 21] INFO [2019-12-17 12:31:43,186] jepsen worker 44 - jepsen.util 7514 :invoke :cas [14 [18 2]] INFO [2019-12-17 12:31:43,199] jepsen worker 89 - jepsen.util 4859 :invoke :write [29 18] INFO [2019-12-17 12:31:43,199] jepsen worker 42 - jepsen.util 7422 :invoke :write [14 6] INFO [2019-12-17 12:31:43,202] jepsen worker 38 - jepsen.util 4988 :invoke :write [12 40] INFO [2019-12-17 12:31:43,203] jepsen worker 62 - jepsen.util 6362 :invoke :cas [20 [0 48]] INFO [2019-12-17 12:31:43,205] jepsen worker 67 - jepsen.util 6907 :invoke :cas [22 [18 28]] INFO [2019-12-17 12:31:43,209] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29212,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:31:43,220] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29212,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:31:43,222] jepsen worker 24 - jepsen.util 6324 :invoke :write [8 0] INFO [2019-12-17 12:31:43,223] jepsen worker 86 - jepsen.util 7106 :invoke :cas [28 [5 33]] INFO [2019-12-17 12:31:43,225] jepsen worker 70 - jepsen.util 5740 :invoke :cas [23 [2 33]] INFO [2019-12-17 12:31:43,226] jepsen worker 80 - jepsen.util 5210 :invoke :write [26 7] INFO [2019-12-17 12:31:43,251] jepsen worker 21 - jepsen.util 6231 :info :cas [7 [11 45]] :RequestCanceledException INFO [2019-12-17 12:31:43,260] jepsen worker 61 - jepsen.util 7711 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:31:43,260] jepsen worker 74 - jepsen.util 5384 :info :write [24 42] :RequestCanceledException INFO [2019-12-17 12:31:43,260] jepsen worker 61 - jepsen.util 7711 :invoke :read [20 nil] INFO [2019-12-17 12:31:43,290] jepsen worker 10 - jepsen.util 5050 :ok :write [3 25] INFO [2019-12-17 12:31:43,291] jepsen worker 10 - jepsen.util 5050 :invoke :cas [3 [31 4]] INFO [2019-12-17 12:31:43,356] jepsen worker 59 - jepsen.util 6629 :info :write [19 4] :RequestCanceledException INFO [2019-12-17 12:31:43,368] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29218,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:31:43,398] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29218,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:31:43,401] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29222,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:31:43,401] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29222,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:31:43,439] jepsen worker 13 - jepsen.util 4873 :info :write [4 42] :RequestCanceledException INFO [2019-12-17 12:31:43,462] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:31:43,463] jepsen worker 21 - jepsen.util 6321 :invoke :read [7 nil] INFO [2019-12-17 12:31:43,479] jepsen nemesis - couchbase.util Rebalance check count: 219 INFO [2019-12-17 12:31:43,482] jepsen nemesis - couchbase.util Rebalance stuck count: 200 INFO [2019-12-17 12:31:43,483] jepsen worker 23 - jepsen.util 6953 :info :write [7 1] :RequestCanceledException INFO [2019-12-17 12:31:43,487] jepsen worker 45 - jepsen.util 5175 :ok :write [15 13] INFO [2019-12-17 12:31:43,492] jepsen worker 45 - jepsen.util 5175 :invoke :read [15 nil] INFO [2019-12-17 12:31:43,500] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29234,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:31:43,501] jepsen worker 74 - jepsen.util 5474 :invoke :read [24 nil] INFO [2019-12-17 12:31:43,508] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29234,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:31:43,548] jepsen worker 53 - jepsen.util 5003 :ok :write [17 10] INFO [2019-12-17 12:31:43,549] jepsen worker 53 - jepsen.util 5003 :invoke :write [17 0] INFO [2019-12-17 12:31:43,554] jepsen worker 27 - jepsen.util 4977 :fail :read [9 nil] :RequestTimeoutException INFO [2019-12-17 12:31:43,555] jepsen worker 27 - jepsen.util 4977 :invoke :cas [9 [37 48]] INFO [2019-12-17 12:31:43,562] jepsen worker 20 - jepsen.util 5150 :fail :read [6 nil] :RequestTimeoutException INFO [2019-12-17 12:31:43,563] jepsen worker 87 - jepsen.util 4677 :fail :read [29 nil] :RequestTimeoutException INFO [2019-12-17 12:31:43,563] jepsen worker 87 - jepsen.util 4677 :invoke :cas [29 [31 29]] INFO [2019-12-17 12:31:43,568] jepsen worker 20 - jepsen.util 5150 :invoke :write [6 41] INFO [2019-12-17 12:31:43,611] jepsen worker 59 - jepsen.util 6719 :invoke :write [19 22] INFO [2019-12-17 12:31:43,615] jepsen worker 23 - jepsen.util 7043 :invoke :write [7 11] INFO [2019-12-17 12:31:43,624] jepsen worker 14 - jepsen.util 5234 :info :write [4 36] :RequestCanceledException INFO [2019-12-17 12:31:43,624] jepsen worker 32 - jepsen.util 5162 :info :write [10 35] :RequestCanceledException INFO [2019-12-17 12:31:43,638] jepsen worker 13 - jepsen.util 4963 :invoke :cas [4 [16 49]] INFO [2019-12-17 12:31:43,685] jepsen worker 32 - jepsen.util 5252 :invoke :read [10 nil] INFO [2019-12-17 12:31:43,720] jepsen worker 14 - jepsen.util 5324 :invoke :cas [4 [23 13]] INFO [2019-12-17 12:31:43,741] jepsen worker 56 - jepsen.util 5186 :info :write [18 23] :RequestCanceledException INFO [2019-12-17 12:31:43,751] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29246,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:31:43,809] jepsen worker 55 - jepsen.util 5995 :info :write [18 21] :RequestCanceledException INFO [2019-12-17 12:31:43,827] jepsen worker 36 - jepsen.util 5256 :info :write [12 43] :RequestCanceledException INFO [2019-12-17 12:31:43,835] jepsen worker 37 - jepsen.util 4807 :fail :cas [12 [0 29]] :ValueNotSwapFrom INFO [2019-12-17 12:31:43,935] jepsen worker 88 - jepsen.util 5308 :info :write [29 11] :RequestCanceledException INFO [2019-12-17 12:31:43,935] jepsen worker 52 - jepsen.util 5272 :info :write [17 44] :RequestCanceledException INFO [2019-12-17 12:31:43,951] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29267,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:31:43,981] jepsen worker 64 - jepsen.util 3844 :ok :read [21 22] INFO [2019-12-17 12:31:44,003] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29272,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:31:44,056] jepsen worker 54 - jepsen.util 5454 :info :write [18 17] :RequestCanceledException INFO [2019-12-17 12:31:44,066] jepsen worker 9 - jepsen.util 5769 :fail :cas [3 [9 4]] :ValueNotSwapFrom INFO [2019-12-17 12:31:44,076] jepsen worker 18 - jepsen.util 4968 :fail :cas [6 [41 8]] :ValueNotSwapFrom INFO [2019-12-17 12:31:44,082] jepsen worker 19 - jepsen.util 5059 :ok :read [6 38] INFO [2019-12-17 12:31:44,105] jepsen worker 63 - jepsen.util 4203 :fail :cas [21 [1 38]] :ValueNotSwapFrom INFO [2019-12-17 12:31:44,119] jepsen worker 10 - jepsen.util 5050 :fail :cas [3 [31 4]] :ValueNotSwapFrom INFO [2019-12-17 12:31:44,126] jepsen worker 28 - jepsen.util 3808 :ok :write [9 39] INFO [2019-12-17 12:31:44,126] jepsen worker 12 - jepsen.util 5322 :info :write [4 37] :RequestCanceledException INFO [2019-12-17 12:31:44,129] jepsen worker 51 - jepsen.util 5001 :fail :cas [17 [7 0]] :ValueNotSwapFrom INFO [2019-12-17 12:31:44,136] jepsen worker 29 - jepsen.util 4079 :ok :read [9 43] INFO [2019-12-17 12:31:44,141] jepsen worker 31 - jepsen.util 4891 :fail :cas [10 [25 26]] :ValueNotSwapFrom INFO [2019-12-17 12:31:44,150] jepsen worker 4 - jepsen.util 4414 :ok :read [1 26] INFO [2019-12-17 12:31:44,157] jepsen worker 78 - jepsen.util 5118 :info :write [26 23] :RequestCanceledException INFO [2019-12-17 12:31:44,172] jepsen worker 3 - jepsen.util 5223 :ok :read [1 26] INFO [2019-12-17 12:31:44,178] jepsen worker 45 - jepsen.util 5175 :ok :read [15 13] INFO [2019-12-17 12:31:44,180] jepsen worker 65 - jepsen.util 4565 :fail :cas [21 [12 27]] :ValueNotSwapFrom INFO [2019-12-17 12:31:44,181] jepsen worker 79 - jepsen.util 5029 :ok :read [26 43] INFO [2019-12-17 12:31:44,190] jepsen worker 47 - jepsen.util 5717 :ok :read [15 13] INFO [2019-12-17 12:31:44,205] jepsen worker 32 - jepsen.util 5252 :ok :read [10 11] INFO [2019-12-17 12:31:44,205] jepsen worker 27 - jepsen.util 4977 :fail :cas [9 [37 48]] :ValueNotSwapFrom INFO [2019-12-17 12:31:44,211] jepsen worker 73 - jepsen.util 4753 :ok :write [24 10] INFO [2019-12-17 12:31:44,226] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29283,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:31:44,229] jepsen worker 69 - jepsen.util 5559 :fail :cas [23 [1 12]] :ValueNotSwapFrom INFO [2019-12-17 12:31:44,232] jepsen worker 46 - jepsen.util 4906 :ok :write [15 23] INFO [2019-12-17 12:31:44,252] jepsen worker 70 - jepsen.util 5740 :fail :cas [23 [2 33]] :ValueNotSwapFrom INFO [2019-12-17 12:31:44,260] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29277,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:31:44,288] jepsen worker 72 - jepsen.util 5022 :ok :write [24 8] INFO [2019-12-17 12:31:44,394] jepsen worker 11 - jepsen.util 5141 :info :cas [3 [16 31]] :RequestCanceledException INFO [2019-12-17 12:31:44,440] jepsen worker 30 - jepsen.util 5250 :fail :read [10 nil] :RequestTimeoutException INFO [2019-12-17 12:31:44,445] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29289,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:31:44,476] jepsen worker 13 - jepsen.util 4963 :fail :cas [4 [16 49]] :ValueNotSwapFrom INFO [2019-12-17 12:31:44,494] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:31:44,521] jepsen worker 87 - jepsen.util 4677 :fail :cas [29 [31 29]] :ValueNotSwapFrom INFO [2019-12-17 12:31:44,569] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29246,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:31:44,569] jepsen worker 37 - jepsen.util 4807 :invoke :cas [12 [25 18]] INFO [2019-12-17 12:31:44,569] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29267,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:31:44,571] jepsen worker 64 - jepsen.util 3844 :invoke :cas [21 [20 24]] INFO [2019-12-17 12:31:44,572] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29272,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:31:44,572] jepsen worker 9 - jepsen.util 5769 :invoke :write [3 21] INFO [2019-12-17 12:31:44,572] jepsen worker 18 - jepsen.util 4968 :invoke :cas [6 [35 37]] INFO [2019-12-17 12:31:44,573] jepsen worker 19 - jepsen.util 5059 :invoke :write [6 45] INFO [2019-12-17 12:31:44,573] jepsen worker 63 - jepsen.util 4203 :invoke :write [21 41] INFO [2019-12-17 12:31:44,574] jepsen worker 51 - jepsen.util 5001 :invoke :write [17 12] INFO [2019-12-17 12:31:44,574] jepsen worker 28 - jepsen.util 3808 :invoke :read [9 nil] INFO [2019-12-17 12:31:44,574] jepsen worker 29 - jepsen.util 4079 :invoke :read [9 nil] INFO [2019-12-17 12:31:44,574] jepsen worker 4 - jepsen.util 4414 :invoke :write [1 23] INFO [2019-12-17 12:31:44,574] jepsen worker 31 - jepsen.util 4891 :invoke :cas [10 [28 44]] INFO [2019-12-17 12:31:44,574] jepsen worker 3 - jepsen.util 5223 :invoke :read [1 nil] INFO [2019-12-17 12:31:44,574] jepsen worker 10 - jepsen.util 5050 :invoke :read [3 nil] INFO [2019-12-17 12:31:44,574] jepsen worker 45 - jepsen.util 5175 :invoke :write [15 17] INFO [2019-12-17 12:31:44,575] jepsen worker 65 - jepsen.util 4565 :invoke :cas [21 [30 7]] INFO [2019-12-17 12:31:44,575] jepsen worker 55 - jepsen.util 6085 :invoke :cas [18 [40 16]] INFO [2019-12-17 12:31:44,576] jepsen worker 32 - jepsen.util 5252 :invoke :cas [10 [5 20]] INFO [2019-12-17 12:31:44,576] jepsen worker 27 - jepsen.util 4977 :invoke :read [9 nil] INFO [2019-12-17 12:31:44,576] jepsen worker 47 - jepsen.util 5717 :invoke :write [15 24] INFO [2019-12-17 12:31:44,576] jepsen worker 73 - jepsen.util 4753 :invoke :write [24 3] INFO [2019-12-17 12:31:44,576] jepsen worker 79 - jepsen.util 5029 :invoke :read [26 nil] INFO [2019-12-17 12:31:44,576] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29283,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:31:44,577] jepsen worker 46 - jepsen.util 4906 :invoke :cas [15 [13 17]] INFO [2019-12-17 12:31:44,577] jepsen worker 69 - jepsen.util 5559 :invoke :write [23 11] INFO [2019-12-17 12:31:44,577] jepsen worker 70 - jepsen.util 5740 :invoke :write [23 37] INFO [2019-12-17 12:31:44,577] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29277,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:31:44,577] jepsen worker 72 - jepsen.util 5022 :invoke :write [24 15] INFO [2019-12-17 12:31:44,578] jepsen worker 30 - jepsen.util 5250 :invoke :read [10 nil] INFO [2019-12-17 12:31:44,578] jepsen worker 56 - jepsen.util 5276 :invoke :write [18 43] INFO [2019-12-17 12:31:44,578] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29289,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:31:44,578] jepsen nemesis - couchbase.util Rebalance check count: 220 INFO [2019-12-17 12:31:44,578] jepsen worker 13 - jepsen.util 4963 :invoke :write [4 3] INFO [2019-12-17 12:31:44,578] jepsen worker 87 - jepsen.util 4677 :invoke :cas [29 [35 12]] INFO [2019-12-17 12:31:44,578] jepsen worker 36 - jepsen.util 5346 :invoke :write [12 19] INFO [2019-12-17 12:31:44,578] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29270,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:31:44,579] jepsen worker 52 - jepsen.util 5362 :invoke :write [17 39] INFO [2019-12-17 12:31:44,579] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29276,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:31:44,581] jepsen worker 88 - jepsen.util 5398 :invoke :write [29 33] INFO [2019-12-17 12:31:44,583] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29287,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:31:44,583] jepsen worker 54 - jepsen.util 5544 :invoke :read [18 nil] INFO [2019-12-17 12:31:44,583] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29279,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:31:44,584] jepsen worker 12 - jepsen.util 5412 :invoke :cas [4 [9 37]] INFO [2019-12-17 12:31:44,584] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29294,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:31:44,584] jepsen nemesis - couchbase.util Rebalance stuck count: 201 INFO [2019-12-17 12:31:44,584] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29270,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:31:44,588] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29287,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:31:44,588] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29276,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:31:44,590] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29279,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:31:44,590] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29294,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:31:44,592] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29288,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:31:44,596] jepsen worker 78 - jepsen.util 5208 :invoke :cas [26 [26 48]] INFO [2019-12-17 12:31:44,597] jepsen worker 11 - jepsen.util 5231 :invoke :write [3 19] INFO [2019-12-17 12:31:44,650] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29288,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:31:44,659] jepsen worker 71 - jepsen.util 5651 :info :write [23 6] :RequestCanceledException INFO [2019-12-17 12:31:44,683] jepsen worker 71 - jepsen.util 5741 :invoke :write [23 41] INFO [2019-12-17 12:31:44,695] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29314,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:31:44,697] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29314,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:31:44,698] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29318,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:31:44,700] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29318,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:31:44,701] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29319,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:31:44,702] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29319,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:31:44,725] jepsen worker 5 - jepsen.util 4595 :info :write [1 48] :RequestCanceledException INFO [2019-12-17 12:31:44,737] jepsen worker 5 - jepsen.util 4685 :invoke :write [1 21] INFO [2019-12-17 12:31:44,810] jepsen worker 48 - jepsen.util 6258 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:31:44,810] jepsen worker 48 - jepsen.util 6258 :invoke :cas [16 [0 5]] INFO [2019-12-17 12:31:44,811] jepsen worker 84 - jepsen.util 6474 :info :write [28 34] :RequestCanceledException INFO [2019-12-17 12:31:44,811] jepsen worker 15 - jepsen.util 5865 :info :write [5 18] :RequestCanceledException INFO [2019-12-17 12:31:44,826] jepsen worker 16 - jepsen.util 7846 :info :write [5 2] :RequestCanceledException INFO [2019-12-17 12:31:44,827] jepsen worker 8 - jepsen.util 6308 :info :cas [2 [21 9]] :RequestCanceledException INFO [2019-12-17 12:31:44,827] jepsen worker 33 - jepsen.util 6783 :info :cas [11 [11 25]] :RequestCanceledException INFO [2019-12-17 12:31:44,837] jepsen worker 15 - jepsen.util 5955 :invoke :read [5 nil] INFO [2019-12-17 12:31:44,838] jepsen worker 84 - jepsen.util 6564 :invoke :read [28 nil] INFO [2019-12-17 12:31:44,840] jepsen worker 68 - jepsen.util 6368 :info :cas [22 [49 10]] :RequestCanceledException INFO [2019-12-17 12:31:44,846] jepsen worker 8 - jepsen.util 6398 :invoke :read [2 nil] INFO [2019-12-17 12:31:44,846] jepsen worker 16 - jepsen.util 7936 :invoke :cas [5 [16 8]] INFO [2019-12-17 12:31:44,848] jepsen worker 33 - jepsen.util 6873 :invoke :write [11 8] INFO [2019-12-17 12:31:44,858] jepsen worker 68 - jepsen.util 6458 :invoke :cas [22 [17 34]] INFO [2019-12-17 12:31:44,875] jepsen worker 28 - jepsen.util 3808 :ok :read [9 39] INFO [2019-12-17 12:31:44,875] jepsen worker 28 - jepsen.util 3808 :invoke :read [9 nil] INFO [2019-12-17 12:31:44,889] jepsen worker 82 - jepsen.util 7552 :info :write [27 21] :RequestCanceledException INFO [2019-12-17 12:31:44,889] jepsen worker 17 - jepsen.util 6767 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:31:44,890] jepsen worker 17 - jepsen.util 6767 :invoke :write [5 35] INFO [2019-12-17 12:31:44,899] jepsen worker 64 - jepsen.util 3844 :fail :cas [21 [20 24]] :ValueNotSwapFrom INFO [2019-12-17 12:31:44,899] jepsen worker 64 - jepsen.util 3844 :invoke :cas [21 [46 28]] INFO [2019-12-17 12:31:44,901] jepsen worker 82 - jepsen.util 7642 :invoke :read [27 nil] INFO [2019-12-17 12:31:44,902] jepsen worker 10 - jepsen.util 5050 :ok :read [3 25] INFO [2019-12-17 12:31:44,903] jepsen worker 10 - jepsen.util 5050 :invoke :read [3 nil] INFO [2019-12-17 12:31:44,911] jepsen worker 60 - jepsen.util 7440 :info :cas [20 [10 15]] :RequestCanceledException INFO [2019-12-17 12:31:44,912] jepsen worker 75 - jepsen.util 6645 :info :cas [25 [32 3]] :RequestCanceledException INFO [2019-12-17 12:31:44,912] jepsen worker 6 - jepsen.util 7026 :info :cas [2 [15 5]] :RequestCanceledException INFO [2019-12-17 12:31:44,915] jepsen worker 29 - jepsen.util 4079 :ok :read [9 39] INFO [2019-12-17 12:31:44,915] jepsen worker 31 - jepsen.util 4891 :fail :cas [10 [28 44]] :ValueNotSwapFrom INFO [2019-12-17 12:31:44,915] jepsen worker 29 - jepsen.util 4079 :invoke :write [9 4] INFO [2019-12-17 12:31:44,916] jepsen worker 31 - jepsen.util 4891 :invoke :write [10 32] INFO [2019-12-17 12:31:44,921] jepsen worker 60 - jepsen.util 7530 :invoke :write [20 5] INFO [2019-12-17 12:31:44,922] jepsen worker 75 - jepsen.util 6735 :invoke :read [25 nil] INFO [2019-12-17 12:31:44,922] jepsen worker 6 - jepsen.util 7116 :invoke :read [2 nil] INFO [2019-12-17 12:31:44,926] jepsen worker 77 - jepsen.util 7007 :info :cas [25 [11 21]] :RequestCanceledException INFO [2019-12-17 12:31:44,926] jepsen worker 22 - jepsen.util 7042 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:31:44,926] jepsen worker 22 - jepsen.util 7042 :invoke :read [7 nil] INFO [2019-12-17 12:31:44,926] jepsen worker 18 - jepsen.util 4968 :fail :cas [6 [35 37]] :ValueNotSwapFrom INFO [2019-12-17 12:31:44,926] jepsen worker 18 - jepsen.util 4968 :invoke :write [6 12] INFO [2019-12-17 12:31:44,927] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29346,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:31:44,927] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29346,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:31:44,927] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29349,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:31:44,927] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29349,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:31:44,934] jepsen worker 77 - jepsen.util 7097 :invoke :read [25 nil] INFO [2019-12-17 12:31:44,938] jepsen worker 39 - jepsen.util 6519 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:31:44,938] jepsen worker 39 - jepsen.util 6519 :invoke :write [13 35] INFO [2019-12-17 12:31:44,958] jepsen worker 76 - jepsen.util 6826 :info :cas [25 [45 40]] :RequestCanceledException INFO [2019-12-17 12:31:44,962] jepsen worker 14 - jepsen.util 5324 :fail :cas [4 [23 13]] :ValueNotSwapFrom INFO [2019-12-17 12:31:44,963] jepsen worker 14 - jepsen.util 5324 :invoke :cas [4 [19 15]] INFO [2019-12-17 12:31:44,965] jepsen worker 76 - jepsen.util 6916 :invoke :write [25 40] INFO [2019-12-17 12:31:44,967] jepsen worker 65 - jepsen.util 4565 :fail :cas [21 [30 7]] :ValueNotSwapFrom INFO [2019-12-17 12:31:44,967] jepsen worker 65 - jepsen.util 4565 :invoke :cas [21 [14 38]] INFO [2019-12-17 12:31:44,994] jepsen worker 40 - jepsen.util 7420 :info :cas [13 [47 14]] :RequestCanceledException INFO [2019-12-17 12:31:44,998] jepsen worker 40 - jepsen.util 7510 :invoke :write [13 19] INFO [2019-12-17 12:31:45,000] jepsen worker 46 - jepsen.util 4906 :fail :cas [15 [13 17]] :ValueNotSwapFrom INFO [2019-12-17 12:31:45,000] jepsen worker 46 - jepsen.util 4906 :invoke :cas [15 [1 40]] INFO [2019-12-17 12:31:45,005] jepsen worker 41 - jepsen.util 7331 :info :cas [13 [36 28]] :RequestCanceledException INFO [2019-12-17 12:31:45,010] jepsen worker 41 - jepsen.util 7421 :invoke :write [13 4] INFO [2019-12-17 12:31:45,024] jepsen worker 2 - jepsen.util 6302 :info :cas [0 [48 24]] :RequestCanceledException INFO [2019-12-17 12:31:45,024] jepsen worker 83 - jepsen.util 7193 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:31:45,024] jepsen worker 43 - jepsen.util 6433 :info :write [14 38] :RequestCanceledException INFO [2019-12-17 12:31:45,024] jepsen worker 83 - jepsen.util 7193 :invoke :cas [27 [26 11]] INFO [2019-12-17 12:31:45,025] jepsen worker 85 - jepsen.util 7645 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:31:45,025] jepsen worker 57 - jepsen.util 6267 :info :write [19 34] :RequestCanceledException INFO [2019-12-17 12:31:45,027] jepsen worker 85 - jepsen.util 7645 :invoke :write [28 44] INFO [2019-12-17 12:31:45,028] jepsen worker 57 - jepsen.util 6357 :invoke :write [19 31] INFO [2019-12-17 12:31:45,039] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29370,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:31:45,040] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29370,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:31:45,041] jepsen worker 66 - jepsen.util 6366 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:31:45,042] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29373,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:31:45,043] jepsen worker 66 - jepsen.util 6366 :invoke :write [22 17] INFO [2019-12-17 12:31:45,043] jepsen worker 58 - jepsen.util 7078 :info :write [19 22] :RequestCanceledException INFO [2019-12-17 12:31:45,045] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29373,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:31:45,045] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29374,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:31:45,047] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29374,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:31:45,062] jepsen worker 58 - jepsen.util 7168 :invoke :write [19 5] INFO [2019-12-17 12:31:45,067] jepsen worker 43 - jepsen.util 6523 :invoke :write [14 1] INFO [2019-12-17 12:31:45,072] jepsen worker 27 - jepsen.util 4977 :ok :read [9 39] INFO [2019-12-17 12:31:45,072] jepsen worker 27 - jepsen.util 4977 :invoke :read [9 nil] INFO [2019-12-17 12:31:45,085] jepsen worker 2 - jepsen.util 6392 :invoke :write [0 40] INFO [2019-12-17 12:31:45,153] jepsen worker 63 - jepsen.util 4203 :ok :write [21 41] INFO [2019-12-17 12:31:45,153] jepsen worker 63 - jepsen.util 4203 :invoke :read [21 nil] INFO [2019-12-17 12:31:45,190] jepsen worker 81 - jepsen.util 6741 :info :write [27 36] :RequestCanceledException INFO [2019-12-17 12:31:45,207] jepsen worker 81 - jepsen.util 6831 :invoke :read [27 nil] INFO [2019-12-17 12:31:45,229] jepsen worker 25 - jepsen.util 7135 :info :cas [8 [18 18]] :RequestCanceledException INFO [2019-12-17 12:31:45,229] jepsen worker 44 - jepsen.util 7514 :info :cas [14 [18 2]] :RequestCanceledException INFO [2019-12-17 12:31:45,240] jepsen worker 0 - jepsen.util 7470 :info :write [0 19] :RequestCanceledException INFO [2019-12-17 12:31:45,241] jepsen worker 86 - jepsen.util 7106 :info :cas [28 [5 33]] :RequestCanceledException INFO [2019-12-17 12:31:45,248] jepsen worker 44 - jepsen.util 7604 :invoke :read [14 nil] INFO [2019-12-17 12:31:45,250] jepsen worker 25 - jepsen.util 7225 :invoke :read [8 nil] INFO [2019-12-17 12:31:45,253] jepsen worker 0 - jepsen.util 7560 :invoke :read [0 nil] INFO [2019-12-17 12:31:45,257] jepsen worker 86 - jepsen.util 7196 :invoke :cas [28 [17 40]] INFO [2019-12-17 12:31:45,258] jepsen worker 34 - jepsen.util 6244 :info :write [11 39] :RequestCanceledException INFO [2019-12-17 12:31:45,260] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29387,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:31:45,260] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29387,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:31:45,261] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29388,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:31:45,262] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29388,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:31:45,264] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29389,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:31:45,264] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29389,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:31:45,273] jepsen worker 34 - jepsen.util 6334 :invoke :cas [11 [18 7]] INFO [2019-12-17 12:31:45,294] jepsen worker 62 - jepsen.util 6362 :info :cas [20 [0 48]] :RequestCanceledException INFO [2019-12-17 12:31:45,312] jepsen worker 67 - jepsen.util 6907 :info :cas [22 [18 28]] :RequestCanceledException INFO [2019-12-17 12:31:45,313] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29395,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:31:45,315] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29395,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:31:45,315] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29399,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:31:45,319] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29399,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:31:45,346] jepsen worker 62 - jepsen.util 6452 :invoke :write [20 6] INFO [2019-12-17 12:31:45,357] jepsen worker 61 - jepsen.util 7711 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:31:45,358] jepsen worker 61 - jepsen.util 7711 :invoke :cas [20 [3 9]] INFO [2019-12-17 12:31:45,364] jepsen worker 67 - jepsen.util 6997 :invoke :write [22 9] INFO [2019-12-17 12:31:45,413] jepsen worker 87 - jepsen.util 4677 :fail :cas [29 [35 12]] :ValueNotSwapFrom INFO [2019-12-17 12:31:45,413] jepsen worker 87 - jepsen.util 4677 :invoke :cas [29 [43 40]] INFO [2019-12-17 12:31:45,488] jepsen worker 19 - jepsen.util 5059 :ok :write [6 45] INFO [2019-12-17 12:31:45,488] jepsen worker 73 - jepsen.util 4753 :ok :write [24 3] INFO [2019-12-17 12:31:45,489] jepsen worker 73 - jepsen.util 4753 :invoke :read [24 nil] INFO [2019-12-17 12:31:45,489] jepsen worker 19 - jepsen.util 5059 :invoke :write [6 4] INFO [2019-12-17 12:31:45,492] jepsen worker 21 - jepsen.util 6321 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:31:45,498] jepsen worker 21 - jepsen.util 6321 :invoke :write [7 16] INFO [2019-12-17 12:31:45,508] jepsen worker 45 - jepsen.util 5175 :ok :write [15 17] INFO [2019-12-17 12:31:45,508] jepsen worker 45 - jepsen.util 5175 :invoke :write [15 32] INFO [2019-12-17 12:31:45,540] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29408,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:31:45,542] jepsen worker 30 - jepsen.util 5250 :ok :read [10 11] INFO [2019-12-17 12:31:45,544] jepsen worker 30 - jepsen.util 5250 :invoke :read [10 nil] INFO [2019-12-17 12:31:45,545] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29408,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:31:45,545] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29411,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:31:45,547] jepsen worker 72 - jepsen.util 5022 :ok :write [24 15] INFO [2019-12-17 12:31:45,548] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29411,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:31:45,548] jepsen worker 72 - jepsen.util 5022 :invoke :cas [24 [36 24]] INFO [2019-12-17 12:31:45,560] jepsen worker 26 - jepsen.util 7136 :info :write [8 17] :RequestCanceledException INFO [2019-12-17 12:31:45,588] jepsen worker 26 - jepsen.util 7226 :invoke :cas [8 [20 23]] INFO [2019-12-17 12:31:45,604] jepsen worker 74 - jepsen.util 5474 :fail :read [24 nil] :RequestTimeoutException INFO [2019-12-17 12:31:45,604] jepsen worker 74 - jepsen.util 5474 :invoke :cas [24 [12 28]] INFO [2019-12-17 12:31:45,610] jepsen worker 79 - jepsen.util 5029 :ok :read [26 43] INFO [2019-12-17 12:31:45,610] jepsen worker 79 - jepsen.util 5029 :invoke :cas [26 [2 11]] INFO [2019-12-17 12:31:45,612] jepsen worker 78 - jepsen.util 5208 :fail :cas [26 [26 48]] :ValueNotSwapFrom INFO [2019-12-17 12:31:45,612] jepsen worker 3 - jepsen.util 5223 :ok :read [1 26] INFO [2019-12-17 12:31:45,612] jepsen worker 78 - jepsen.util 5208 :invoke :write [26 35] INFO [2019-12-17 12:31:45,612] jepsen worker 3 - jepsen.util 5223 :invoke :cas [1 [42 9]] INFO [2019-12-17 12:31:45,614] jepsen worker 63 - jepsen.util 4203 :ok :read [21 41] INFO [2019-12-17 12:31:45,614] jepsen worker 63 - jepsen.util 4203 :invoke :cas [21 [18 48]] INFO [2019-12-17 12:31:45,653] jepsen worker 32 - jepsen.util 5252 :fail :cas [10 [5 20]] :ValueNotSwapFrom INFO [2019-12-17 12:31:45,653] jepsen worker 32 - jepsen.util 5252 :invoke :cas [10 [34 16]] INFO [2019-12-17 12:31:45,655] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:31:45,655] jepsen nemesis - couchbase.util Rebalance check count: 221 INFO [2019-12-17 12:31:45,655] jepsen nemesis - couchbase.util Rebalance stuck count: 202 INFO [2019-12-17 12:31:45,663] jepsen worker 55 - jepsen.util 6085 :fail :cas [18 [40 16]] :ValueNotSwapFrom INFO [2019-12-17 12:31:45,663] jepsen worker 55 - jepsen.util 6085 :invoke :write [18 3] INFO [2019-12-17 12:31:45,665] jepsen worker 54 - jepsen.util 5544 :ok :read [18 37] INFO [2019-12-17 12:31:45,665] jepsen worker 54 - jepsen.util 5544 :invoke :write [18 45] INFO [2019-12-17 12:31:45,683] jepsen worker 12 - jepsen.util 5412 :fail :cas [4 [9 37]] :ValueNotSwapFrom INFO [2019-12-17 12:31:45,683] jepsen worker 12 - jepsen.util 5412 :invoke :read [4 nil] INFO [2019-12-17 12:31:45,695] jepsen worker 73 - jepsen.util 4753 :ok :read [24 15] INFO [2019-12-17 12:31:45,696] jepsen worker 73 - jepsen.util 4753 :invoke :cas [24 [8 10]] INFO [2019-12-17 12:31:45,700] jepsen worker 65 - jepsen.util 4565 :fail :cas [21 [14 38]] :ValueNotSwapFrom INFO [2019-12-17 12:31:45,700] jepsen worker 65 - jepsen.util 4565 :invoke :read [21 nil] INFO [2019-12-17 12:31:45,706] jepsen worker 14 - jepsen.util 5324 :fail :cas [4 [19 15]] :ValueNotSwapFrom INFO [2019-12-17 12:31:45,706] jepsen worker 14 - jepsen.util 5324 :invoke :cas [4 [46 48]] INFO [2019-12-17 12:31:45,706] jepsen worker 72 - jepsen.util 5022 :fail :cas [24 [36 24]] :ValueNotSwapFrom INFO [2019-12-17 12:31:45,706] jepsen worker 72 - jepsen.util 5022 :invoke :read [24 nil] INFO [2019-12-17 12:31:45,709] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29428,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:31:45,709] jepsen worker 1 - jepsen.util 6571 :info :write [0 6] :RequestCanceledException INFO [2019-12-17 12:31:45,710] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29428,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:31:45,710] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29431,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:31:45,710] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29431,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:31:45,713] jepsen worker 1 - jepsen.util 6661 :invoke :cas [0 [9 6]] INFO [2019-12-17 12:31:45,739] jepsen worker 7 - jepsen.util 6937 :info :write [2 43] :RequestCanceledException INFO [2019-12-17 12:31:45,740] jepsen worker 47 - jepsen.util 5717 :ok :write [15 24] INFO [2019-12-17 12:31:45,740] jepsen worker 47 - jepsen.util 5717 :invoke :cas [15 [17 49]] INFO [2019-12-17 12:31:45,741] jepsen worker 7 - jepsen.util 7027 :invoke :cas [2 [29 0]] INFO [2019-12-17 12:31:45,759] jepsen worker 74 - jepsen.util 5474 :fail :cas [24 [12 28]] :ValueNotSwapFrom INFO [2019-12-17 12:31:45,759] jepsen worker 74 - jepsen.util 5474 :invoke :cas [24 [35 33]] INFO [2019-12-17 12:31:45,771] jepsen worker 63 - jepsen.util 4203 :fail :cas [21 [18 48]] :ValueNotSwapFrom INFO [2019-12-17 12:31:45,771] jepsen worker 63 - jepsen.util 4203 :invoke :write [21 4] INFO [2019-12-17 12:31:45,792] jepsen worker 35 - jepsen.util 6875 :info :write [11 45] :RequestCanceledException INFO [2019-12-17 12:31:45,795] jepsen worker 35 - jepsen.util 6965 :invoke :write [11 29] INFO [2019-12-17 12:31:45,810] jepsen worker 50 - jepsen.util 7250 :info :write [16 21] :RequestCanceledException INFO [2019-12-17 12:31:45,812] jepsen worker 50 - jepsen.util 7340 :invoke :read [16 nil] INFO [2019-12-17 12:31:45,826] jepsen worker 38 - jepsen.util 4988 :info :write [12 40] :RequestCanceledException INFO [2019-12-17 12:31:45,826] jepsen worker 24 - jepsen.util 6324 :info :write [8 0] :RequestCanceledException INFO [2019-12-17 12:31:45,826] jepsen worker 80 - jepsen.util 5210 :info :write [26 7] :RequestCanceledException INFO [2019-12-17 12:31:45,829] jepsen worker 80 - jepsen.util 5300 :invoke :read [26 nil] INFO [2019-12-17 12:31:45,829] jepsen worker 38 - jepsen.util 5078 :invoke :write [12 3] INFO [2019-12-17 12:31:45,829] jepsen worker 12 - jepsen.util 5412 :ok :read [4 43] INFO [2019-12-17 12:31:45,829] jepsen worker 12 - jepsen.util 5412 :invoke :read [4 nil] INFO [2019-12-17 12:31:45,833] jepsen worker 24 - jepsen.util 6414 :invoke :write [8 20] INFO [2019-12-17 12:31:45,842] jepsen worker 89 - jepsen.util 4859 :info :write [29 18] :RequestCanceledException INFO [2019-12-17 12:31:45,844] jepsen worker 37 - jepsen.util 4807 :fail :cas [12 [25 18]] :ValueNotSwapFrom INFO [2019-12-17 12:31:45,844] jepsen worker 37 - jepsen.util 4807 :invoke :cas [12 [6 32]] INFO [2019-12-17 12:31:45,845] jepsen worker 89 - jepsen.util 4949 :invoke :write [29 24] INFO [2019-12-17 12:31:45,856] jepsen worker 49 - jepsen.util 5989 :info :write [16 15] :RequestCanceledException INFO [2019-12-17 12:31:45,873] jepsen worker 42 - jepsen.util 7422 :info :write [14 6] :RequestCanceledException INFO [2019-12-17 12:31:45,877] jepsen worker 11 - jepsen.util 5231 :ok :write [3 19] INFO [2019-12-17 12:31:45,877] jepsen worker 11 - jepsen.util 5231 :invoke :write [3 45] INFO [2019-12-17 12:31:45,880] jepsen worker 49 - jepsen.util 6079 :invoke :write [16 45] INFO [2019-12-17 12:31:45,882] jepsen worker 42 - jepsen.util 7512 :invoke :cas [14 [8 33]] INFO [2019-12-17 12:31:45,904] jepsen worker 4 - jepsen.util 4414 :ok :write [1 23] INFO [2019-12-17 12:31:45,904] jepsen worker 4 - jepsen.util 4414 :invoke :cas [1 [9 28]] INFO [2019-12-17 12:31:45,918] jepsen worker 29 - jepsen.util 4079 :ok :write [9 4] INFO [2019-12-17 12:31:45,918] jepsen worker 29 - jepsen.util 4079 :invoke :cas [9 [20 17]] INFO [2019-12-17 12:31:45,926] jepsen worker 20 - jepsen.util 5150 :info :write [6 41] :RequestCanceledException INFO [2019-12-17 12:31:45,926] jepsen worker 59 - jepsen.util 6719 :info :write [19 22] :RequestCanceledException INFO [2019-12-17 12:31:45,929] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29455,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:31:45,929] jepsen worker 20 - jepsen.util 5240 :invoke :cas [6 [49 38]] INFO [2019-12-17 12:31:45,929] jepsen worker 59 - jepsen.util 6809 :invoke :write [19 5] INFO [2019-12-17 12:31:45,929] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29455,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:31:45,929] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29457,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:31:45,929] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29457,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:31:45,940] jepsen worker 23 - jepsen.util 7043 :info :write [7 11] :RequestCanceledException INFO [2019-12-17 12:31:45,941] jepsen worker 53 - jepsen.util 5003 :info :write [17 0] :RequestCanceledException INFO [2019-12-17 12:31:45,944] jepsen worker 53 - jepsen.util 5093 :invoke :write [17 12] INFO [2019-12-17 12:31:45,945] jepsen worker 23 - jepsen.util 7133 :invoke :write [7 0] INFO [2019-12-17 12:31:45,952] jepsen worker 79 - jepsen.util 5029 :fail :cas [26 [2 11]] :ValueNotSwapFrom INFO [2019-12-17 12:31:45,952] jepsen worker 79 - jepsen.util 5029 :invoke :write [26 21] INFO [2019-12-17 12:31:45,985] jepsen worker 72 - jepsen.util 5022 :ok :read [24 15] INFO [2019-12-17 12:31:45,985] jepsen worker 72 - jepsen.util 5022 :invoke :cas [24 [8 27]] INFO [2019-12-17 12:31:45,987] jepsen worker 20 - jepsen.util 5240 :fail :cas [6 [49 38]] :ValueNotSwapFrom INFO [2019-12-17 12:31:45,988] jepsen worker 20 - jepsen.util 5240 :invoke :cas [6 [21 30]] INFO [2019-12-17 12:31:45,993] jepsen worker 63 - jepsen.util 4203 :ok :write [21 4] INFO [2019-12-17 12:31:45,993] jepsen worker 63 - jepsen.util 4203 :invoke :write [21 2] INFO [2019-12-17 12:31:46,003] jepsen worker 73 - jepsen.util 4753 :fail :cas [24 [8 10]] :ValueNotSwapFrom INFO [2019-12-17 12:31:46,003] jepsen worker 73 - jepsen.util 4753 :invoke :cas [24 [34 9]] INFO [2019-12-17 12:31:46,007] jepsen worker 74 - jepsen.util 5474 :fail :cas [24 [35 33]] :ValueNotSwapFrom INFO [2019-12-17 12:31:46,007] jepsen worker 74 - jepsen.util 5474 :invoke :read [24 nil] INFO [2019-12-17 12:31:46,039] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29474,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:31:46,039] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29474,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:31:46,039] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29475,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:31:46,039] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29475,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:31:46,040] jepsen worker 72 - jepsen.util 5022 :fail :cas [24 [8 27]] :ValueNotSwapFrom INFO [2019-12-17 12:31:46,040] jepsen worker 72 - jepsen.util 5022 :invoke :read [24 nil] INFO [2019-12-17 12:31:46,050] jepsen worker 56 - jepsen.util 5276 :ok :write [18 43] INFO [2019-12-17 12:31:46,050] jepsen worker 56 - jepsen.util 5276 :invoke :write [18 20] INFO [2019-12-17 12:31:46,053] jepsen worker 20 - jepsen.util 5240 :fail :cas [6 [21 30]] :ValueNotSwapFrom INFO [2019-12-17 12:31:46,053] jepsen worker 20 - jepsen.util 5240 :invoke :read [6 nil] INFO [2019-12-17 12:31:46,129] jepsen worker 3 - jepsen.util 5223 :fail :cas [1 [42 9]] :ValueNotSwapFrom INFO [2019-12-17 12:31:46,129] jepsen worker 3 - jepsen.util 5223 :invoke :read [1 nil] INFO [2019-12-17 12:31:46,157] jepsen worker 74 - jepsen.util 5474 :ok :read [24 15] INFO [2019-12-17 12:31:46,158] jepsen worker 74 - jepsen.util 5474 :invoke :write [24 45] INFO [2019-12-17 12:31:46,168] jepsen worker 73 - jepsen.util 4753 :fail :cas [24 [34 9]] :ValueNotSwapFrom INFO [2019-12-17 12:31:46,169] jepsen worker 73 - jepsen.util 4753 :invoke :cas [24 [37 28]] INFO [2019-12-17 12:31:46,169] jepsen worker 14 - jepsen.util 5324 :fail :cas [4 [46 48]] :ValueNotSwapFrom INFO [2019-12-17 12:31:46,169] jepsen worker 14 - jepsen.util 5324 :invoke :cas [4 [29 16]] INFO [2019-12-17 12:31:46,180] jepsen worker 63 - jepsen.util 4203 :ok :write [21 2] INFO [2019-12-17 12:31:46,180] jepsen worker 63 - jepsen.util 4203 :invoke :cas [21 [22 39]] INFO [2019-12-17 12:31:46,184] jepsen worker 20 - jepsen.util 5240 :ok :read [6 45] INFO [2019-12-17 12:31:46,185] jepsen worker 72 - jepsen.util 5022 :ok :read [24 15] INFO [2019-12-17 12:31:46,185] jepsen worker 20 - jepsen.util 5240 :invoke :read [6 nil] INFO [2019-12-17 12:31:46,186] jepsen worker 72 - jepsen.util 5022 :invoke :read [24 nil] INFO [2019-12-17 12:31:46,186] jepsen worker 30 - jepsen.util 5250 :ok :read [10 11] INFO [2019-12-17 12:31:46,186] jepsen worker 30 - jepsen.util 5250 :invoke :write [10 6] INFO [2019-12-17 12:31:46,214] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29500,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:31:46,214] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29500,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:31:46,214] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29503,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:31:46,215] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29503,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:31:46,215] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29504,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:31:46,215] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29504,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:31:46,255] jepsen worker 73 - jepsen.util 4753 :fail :cas [24 [37 28]] :ValueNotSwapFrom INFO [2019-12-17 12:31:46,255] jepsen worker 73 - jepsen.util 4753 :invoke :cas [24 [49 24]] INFO [2019-12-17 12:31:46,256] jepsen worker 63 - jepsen.util 4203 :fail :cas [21 [22 39]] :ValueNotSwapFrom INFO [2019-12-17 12:31:46,256] jepsen worker 63 - jepsen.util 4203 :invoke :write [21 41] INFO [2019-12-17 12:31:46,258] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29493,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:31:46,259] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29493,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:31:46,259] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29494,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:31:46,259] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29494,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:31:46,259] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29495,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:31:46,259] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29495,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:31:46,297] jepsen worker 4 - jepsen.util 4414 :fail :cas [1 [9 28]] :ValueNotSwapFrom INFO [2019-12-17 12:31:46,297] jepsen worker 4 - jepsen.util 4414 :invoke :cas [1 [17 16]] INFO [2019-12-17 12:31:46,308] jepsen worker 80 - jepsen.util 5300 :ok :read [26 43] INFO [2019-12-17 12:31:46,309] jepsen worker 80 - jepsen.util 5300 :invoke :read [26 nil] INFO [2019-12-17 12:31:46,332] jepsen worker 63 - jepsen.util 4203 :ok :write [21 41] INFO [2019-12-17 12:31:46,336] jepsen worker 63 - jepsen.util 4203 :invoke :cas [21 [11 13]] INFO [2019-12-17 12:31:46,370] jepsen worker 63 - jepsen.util 4203 :fail :cas [21 [11 13]] :ValueNotSwapFrom INFO [2019-12-17 12:31:46,375] jepsen worker 63 - jepsen.util 4203 :invoke :read [21 nil] INFO [2019-12-17 12:31:46,383] jepsen worker 3 - jepsen.util 5223 :ok :read [1 23] INFO [2019-12-17 12:31:46,383] jepsen worker 3 - jepsen.util 5223 :invoke :cas [1 [39 44]] INFO [2019-12-17 12:31:46,405] jepsen worker 63 - jepsen.util 4203 :ok :read [21 41] INFO [2019-12-17 12:31:46,406] jepsen worker 63 - jepsen.util 4203 :invoke :cas [21 [44 6]] INFO [2019-12-17 12:31:46,420] jepsen worker 12 - jepsen.util 5412 :ok :read [4 43] INFO [2019-12-17 12:31:46,420] jepsen worker 12 - jepsen.util 5412 :invoke :write [4 20] INFO [2019-12-17 12:31:46,433] jepsen worker 63 - jepsen.util 4203 :fail :cas [21 [44 6]] :ValueNotSwapFrom INFO [2019-12-17 12:31:46,434] jepsen worker 63 - jepsen.util 4203 :invoke :write [21 14] INFO [2019-12-17 12:31:46,435] jepsen worker 31 - jepsen.util 4891 :ok :write [10 32] INFO [2019-12-17 12:31:46,436] jepsen worker 31 - jepsen.util 4891 :invoke :cas [10 [46 15]] INFO [2019-12-17 12:31:46,436] jepsen worker 9 - jepsen.util 5769 :ok :write [3 21] INFO [2019-12-17 12:31:46,436] jepsen worker 9 - jepsen.util 5769 :invoke :cas [3 [26 27]] INFO [2019-12-17 12:31:46,439] jepsen worker 45 - jepsen.util 5175 :ok :write [15 32] INFO [2019-12-17 12:31:46,439] jepsen worker 45 - jepsen.util 5175 :invoke :cas [15 [19 16]] INFO [2019-12-17 12:31:46,463] jepsen worker 74 - jepsen.util 5474 :ok :write [24 45] INFO [2019-12-17 12:31:46,463] jepsen worker 74 - jepsen.util 5474 :invoke :write [24 39] INFO [2019-12-17 12:31:46,501] jepsen worker 9 - jepsen.util 5769 :fail :cas [3 [26 27]] :ValueNotSwapFrom INFO [2019-12-17 12:31:46,502] jepsen worker 9 - jepsen.util 5769 :invoke :read [3 nil] INFO [2019-12-17 12:31:46,529] jepsen worker 31 - jepsen.util 4891 :fail :cas [10 [46 15]] :ValueNotSwapFrom INFO [2019-12-17 12:31:46,529] jepsen worker 31 - jepsen.util 4891 :invoke :read [10 nil] INFO [2019-12-17 12:31:46,531] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29514,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:31:46,531] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29514,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:31:46,531] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29517,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:31:46,532] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29517,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:31:46,532] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29518,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:31:46,532] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29518,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:31:46,536] jepsen worker 72 - jepsen.util 5022 :ok :read [24 45] INFO [2019-12-17 12:31:46,536] jepsen worker 72 - jepsen.util 5022 :invoke :cas [24 [33 49]] INFO [2019-12-17 12:31:46,602] jepsen worker 20 - jepsen.util 5240 :ok :read [6 45] INFO [2019-12-17 12:31:46,602] jepsen worker 20 - jepsen.util 5240 :invoke :write [6 19] INFO [2019-12-17 12:31:46,772] jepsen worker 52 - jepsen.util 5362 :info :write [17 39] :RequestCanceledException INFO [2019-12-17 12:31:46,772] jepsen worker 51 - jepsen.util 5001 :info :write [17 12] :RequestCanceledException INFO [2019-12-17 12:31:46,772] jepsen worker 13 - jepsen.util 4963 :info :write [4 3] :RequestCanceledException INFO [2019-12-17 12:31:46,772] jepsen worker 88 - jepsen.util 5398 :info :write [29 33] :RequestCanceledException INFO [2019-12-17 12:31:46,772] jepsen worker 70 - jepsen.util 5740 :info :write [23 37] :RequestCanceledException INFO [2019-12-17 12:31:46,772] jepsen worker 36 - jepsen.util 5346 :info :write [12 19] :RequestCanceledException INFO [2019-12-17 12:31:46,772] jepsen worker 69 - jepsen.util 5559 :info :write [23 11] :RequestCanceledException INFO [2019-12-17 12:31:46,773] jepsen worker 9 - jepsen.util 5769 :ok :read [3 21] INFO [2019-12-17 12:31:46,774] jepsen worker 51 - jepsen.util 5091 :invoke :read [17 nil] INFO [2019-12-17 12:31:46,774] jepsen worker 52 - jepsen.util 5452 :invoke :write [17 33] INFO [2019-12-17 12:31:46,774] jepsen worker 88 - jepsen.util 5488 :invoke :cas [29 [14 38]] INFO [2019-12-17 12:31:46,774] jepsen worker 13 - jepsen.util 5053 :invoke :cas [4 [36 47]] INFO [2019-12-17 12:31:46,774] jepsen worker 9 - jepsen.util 5769 :invoke :write [3 4] INFO [2019-12-17 12:31:46,775] jepsen worker 70 - jepsen.util 5830 :invoke :write [23 10] INFO [2019-12-17 12:31:46,775] jepsen worker 36 - jepsen.util 5436 :invoke :write [12 19] INFO [2019-12-17 12:31:46,775] jepsen worker 69 - jepsen.util 5649 :invoke :read [23 nil] INFO [2019-12-17 12:31:46,776] jepsen worker 63 - jepsen.util 4203 :ok :write [21 14] INFO [2019-12-17 12:31:46,776] jepsen worker 63 - jepsen.util 4203 :invoke :write [21 27] INFO [2019-12-17 12:31:46,776] jepsen worker 31 - jepsen.util 4891 :ok :read [10 32] INFO [2019-12-17 12:31:46,776] jepsen worker 31 - jepsen.util 4891 :invoke :read [10 nil] INFO [2019-12-17 12:31:46,780] jepsen worker 73 - jepsen.util 4753 :fail :cas [24 [49 24]] :ValueNotSwapFrom INFO [2019-12-17 12:31:46,780] jepsen worker 73 - jepsen.util 4753 :invoke :write [24 41] INFO [2019-12-17 12:31:46,793] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:31:46,793] jepsen nemesis - couchbase.util Rebalance check count: 222 INFO [2019-12-17 12:31:46,793] jepsen nemesis - couchbase.util Rebalance stuck count: 203 INFO [2019-12-17 12:31:46,798] jepsen worker 87 - jepsen.util 4677 :fail :cas [29 [43 40]] :ValueNotSwapFrom INFO [2019-12-17 12:31:46,798] jepsen worker 87 - jepsen.util 4677 :invoke :read [29 nil] INFO [2019-12-17 12:31:46,807] jepsen worker 54 - jepsen.util 5544 :ok :write [18 45] INFO [2019-12-17 12:31:46,807] jepsen worker 54 - jepsen.util 5544 :invoke :cas [18 [36 0]] INFO [2019-12-17 12:31:46,826] jepsen worker 5 - jepsen.util 4685 :info :write [1 21] :RequestCanceledException INFO [2019-12-17 12:31:46,827] jepsen worker 5 - jepsen.util 4775 :invoke :read [1 nil] INFO [2019-12-17 12:31:46,846] jepsen worker 74 - jepsen.util 5474 :ok :write [24 39] INFO [2019-12-17 12:31:46,846] jepsen worker 74 - jepsen.util 5474 :invoke :cas [24 [14 3]] INFO [2019-12-17 12:31:46,850] jepsen worker 31 - jepsen.util 4891 :ok :read [10 32] INFO [2019-12-17 12:31:46,850] jepsen worker 31 - jepsen.util 4891 :invoke :write [10 10] INFO [2019-12-17 12:31:46,857] jepsen worker 71 - jepsen.util 5741 :info :write [23 41] :RequestCanceledException INFO [2019-12-17 12:31:46,859] jepsen worker 71 - jepsen.util 5831 :invoke :read [23 nil] INFO [2019-12-17 12:31:46,877] jepsen worker 53 - jepsen.util 5093 :ok :write [17 12] INFO [2019-12-17 12:31:46,877] jepsen worker 53 - jepsen.util 5093 :invoke :cas [17 [1 34]] INFO [2019-12-17 12:31:46,881] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29555,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:31:46,881] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29555,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:31:46,881] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29556,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:31:46,881] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29529,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:31:46,881] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29556,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:31:46,881] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29529,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:31:46,882] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29532,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:31:46,882] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29532,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:31:46,882] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29533,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:31:46,882] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29533,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:31:46,886] jepsen worker 14 - jepsen.util 5324 :fail :cas [4 [29 16]] :ValueNotSwapFrom INFO [2019-12-17 12:31:46,887] jepsen worker 14 - jepsen.util 5324 :invoke :read [4 nil] INFO [2019-12-17 12:31:46,892] jepsen worker 72 - jepsen.util 5022 :fail :cas [24 [33 49]] :ValueNotSwapFrom INFO [2019-12-17 12:31:46,893] jepsen worker 72 - jepsen.util 5022 :invoke :cas [24 [11 39]] INFO [2019-12-17 12:31:46,893] jepsen worker 15 - jepsen.util 5955 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:31:46,893] jepsen worker 15 - jepsen.util 5955 :invoke :read [5 nil] INFO [2019-12-17 12:31:46,910] jepsen worker 84 - jepsen.util 6564 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:31:46,910] jepsen worker 48 - jepsen.util 6258 :info :cas [16 [0 5]] :RequestCanceledException INFO [2019-12-17 12:31:46,910] jepsen worker 84 - jepsen.util 6564 :invoke :cas [28 [16 0]] INFO [2019-12-17 12:31:46,911] jepsen worker 48 - jepsen.util 6348 :invoke :cas [16 [10 16]] INFO [2019-12-17 12:31:46,925] jepsen worker 28 - jepsen.util 3808 :fail :read [9 nil] :RequestTimeoutException INFO [2019-12-17 12:31:46,925] jepsen worker 8 - jepsen.util 6398 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:31:46,925] jepsen worker 28 - jepsen.util 3808 :invoke :read [9 nil] INFO [2019-12-17 12:31:46,925] jepsen worker 8 - jepsen.util 6398 :invoke :cas [2 [11 25]] INFO [2019-12-17 12:31:46,925] jepsen worker 68 - jepsen.util 6458 :info :cas [22 [17 34]] :RequestCanceledException INFO [2019-12-17 12:31:46,926] jepsen worker 68 - jepsen.util 6548 :invoke :read [22 nil] INFO [2019-12-17 12:31:46,940] jepsen worker 16 - jepsen.util 7936 :info :cas [5 [16 8]] :RequestCanceledException INFO [2019-12-17 12:31:46,940] jepsen worker 10 - jepsen.util 5050 :fail :read [3 nil] :RequestTimeoutException INFO [2019-12-17 12:31:46,940] jepsen worker 6 - jepsen.util 7116 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:31:46,940] jepsen worker 10 - jepsen.util 5050 :invoke :cas [3 [27 37]] INFO [2019-12-17 12:31:46,940] jepsen worker 6 - jepsen.util 7116 :invoke :cas [2 [16 28]] INFO [2019-12-17 12:31:46,940] jepsen worker 80 - jepsen.util 5300 :ok :read [26 43] INFO [2019-12-17 12:31:46,940] jepsen worker 80 - jepsen.util 5300 :invoke :read [26 nil] INFO [2019-12-17 12:31:46,941] jepsen worker 16 - jepsen.util 8026 :invoke :read [5 nil] INFO [2019-12-17 12:31:46,948] jepsen worker 30 - jepsen.util 5250 :ok :write [10 6] INFO [2019-12-17 12:31:46,956] jepsen worker 33 - jepsen.util 6873 :info :write [11 8] :RequestCanceledException INFO [2019-12-17 12:31:46,956] jepsen worker 77 - jepsen.util 7097 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:31:46,969] jepsen worker 9 - jepsen.util 5769 :ok :write [3 4] INFO [2019-12-17 12:31:46,973] jepsen worker 30 - jepsen.util 5250 :invoke :read [10 nil] INFO [2019-12-17 12:31:46,973] jepsen worker 77 - jepsen.util 7097 :invoke :cas [25 [38 8]] INFO [2019-12-17 12:31:46,973] jepsen worker 9 - jepsen.util 5769 :invoke :read [3 nil] INFO [2019-12-17 12:31:46,975] jepsen worker 33 - jepsen.util 6963 :invoke :cas [11 [28 27]] INFO [2019-12-17 12:31:46,977] jepsen worker 10 - jepsen.util 5050 :fail :cas [3 [27 37]] :ValueNotSwapFrom INFO [2019-12-17 12:31:46,977] jepsen worker 10 - jepsen.util 5050 :invoke :read [3 nil] INFO [2019-12-17 12:31:46,992] jepsen worker 64 - jepsen.util 3844 :info :cas [21 [46 28]] :RequestCanceledException INFO [2019-12-17 12:31:46,992] jepsen worker 82 - jepsen.util 7642 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:31:46,992] jepsen worker 17 - jepsen.util 6767 :info :write [5 35] :RequestCanceledException INFO [2019-12-17 12:31:46,992] jepsen worker 18 - jepsen.util 4968 :info :write [6 12] :RequestCanceledException INFO [2019-12-17 12:31:46,992] jepsen worker 82 - jepsen.util 7642 :invoke :read [27 nil] INFO [2019-12-17 12:31:46,993] jepsen worker 64 - jepsen.util 3934 :invoke :read [21 nil] INFO [2019-12-17 12:31:46,993] jepsen worker 18 - jepsen.util 5058 :invoke :write [6 47] INFO [2019-12-17 12:31:46,993] jepsen worker 17 - jepsen.util 6857 :invoke :write [5 44] INFO [2019-12-17 12:31:47,010] jepsen worker 28 - jepsen.util 3808 :ok :read [9 4] INFO [2019-12-17 12:31:47,010] jepsen worker 28 - jepsen.util 3808 :invoke :write [9 27] INFO [2019-12-17 12:31:47,010] jepsen worker 60 - jepsen.util 7530 :info :write [20 5] :RequestCanceledException INFO [2019-12-17 12:31:47,012] jepsen worker 60 - jepsen.util 7620 :invoke :write [20 46] INFO [2019-12-17 12:31:47,018] jepsen worker 3 - jepsen.util 5223 :fail :cas [1 [39 44]] :ValueNotSwapFrom INFO [2019-12-17 12:31:47,018] jepsen worker 3 - jepsen.util 5223 :invoke :write [1 38] INFO [2019-12-17 12:31:47,025] jepsen worker 76 - jepsen.util 6916 :info :write [25 40] :RequestCanceledException INFO [2019-12-17 12:31:47,026] jepsen worker 22 - jepsen.util 7042 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:31:47,026] jepsen worker 75 - jepsen.util 6735 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:31:47,026] jepsen worker 22 - jepsen.util 7042 :invoke :write [7 15] INFO [2019-12-17 12:31:47,026] jepsen worker 75 - jepsen.util 6735 :invoke :write [25 48] INFO [2019-12-17 12:31:47,026] jepsen worker 76 - jepsen.util 7006 :invoke :read [25 nil] INFO [2019-12-17 12:31:47,033] jepsen worker 88 - jepsen.util 5488 :fail :cas [29 [14 38]] :ValueNotSwapFrom INFO [2019-12-17 12:31:47,033] jepsen worker 88 - jepsen.util 5488 :invoke :read [29 nil] INFO [2019-12-17 12:31:47,039] jepsen worker 39 - jepsen.util 6519 :info :write [13 35] :RequestCanceledException INFO [2019-12-17 12:31:47,041] jepsen worker 39 - jepsen.util 6609 :invoke :read [13 nil] INFO [2019-12-17 12:31:47,044] jepsen worker 30 - jepsen.util 5250 :ok :read [10 6] INFO [2019-12-17 12:31:47,044] jepsen worker 30 - jepsen.util 5250 :invoke :read [10 nil] INFO [2019-12-17 12:31:47,055] jepsen worker 9 - jepsen.util 5769 :ok :read [3 4] INFO [2019-12-17 12:31:47,055] jepsen worker 9 - jepsen.util 5769 :invoke :write [3 39] INFO [2019-12-17 12:31:47,086] jepsen worker 10 - jepsen.util 5050 :ok :read [3 4] INFO [2019-12-17 12:31:47,086] jepsen worker 10 - jepsen.util 5050 :invoke :read [3 nil] INFO [2019-12-17 12:31:47,091] jepsen worker 46 - jepsen.util 4906 :info :cas [15 [1 40]] :RequestCanceledException INFO [2019-12-17 12:31:47,091] jepsen worker 40 - jepsen.util 7510 :info :write [13 19] :RequestCanceledException INFO [2019-12-17 12:31:47,093] jepsen worker 46 - jepsen.util 4996 :invoke :cas [15 [6 0]] INFO [2019-12-17 12:31:47,093] jepsen worker 40 - jepsen.util 7600 :invoke :read [13 nil] INFO [2019-12-17 12:31:47,100] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29579,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:31:47,100] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29579,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:31:47,100] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29582,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:31:47,100] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29582,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:31:47,100] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29583,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:31:47,100] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29583,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:31:47,107] jepsen worker 71 - jepsen.util 5831 :ok :read [23 47] INFO [2019-12-17 12:31:47,107] jepsen worker 71 - jepsen.util 5831 :invoke :write [23 49] INFO [2019-12-17 12:31:47,108] jepsen worker 41 - jepsen.util 7421 :info :write [13 4] :RequestCanceledException INFO [2019-12-17 12:31:47,111] jepsen worker 41 - jepsen.util 7511 :invoke :cas [13 [15 19]] INFO [2019-12-17 12:31:47,115] jepsen worker 30 - jepsen.util 5250 :ok :read [10 6] INFO [2019-12-17 12:31:47,115] jepsen worker 30 - jepsen.util 5250 :invoke :cas [10 [13 33]] INFO [2019-12-17 12:31:47,126] jepsen worker 85 - jepsen.util 7645 :info :write [28 44] :RequestCanceledException INFO [2019-12-17 12:31:47,126] jepsen worker 83 - jepsen.util 7193 :info :cas [27 [26 11]] :RequestCanceledException INFO [2019-12-17 12:31:47,126] jepsen worker 57 - jepsen.util 6357 :info :write [19 31] :RequestCanceledException INFO [2019-12-17 12:31:47,128] jepsen worker 57 - jepsen.util 6447 :invoke :read [19 nil] INFO [2019-12-17 12:31:47,128] jepsen worker 85 - jepsen.util 7735 :invoke :cas [28 [11 10]] INFO [2019-12-17 12:31:47,129] jepsen worker 83 - jepsen.util 7283 :invoke :write [27 6] INFO [2019-12-17 12:31:47,139] jepsen worker 27 - jepsen.util 4977 :fail :read [9 nil] :RequestTimeoutException INFO [2019-12-17 12:31:47,139] jepsen worker 66 - jepsen.util 6366 :info :write [22 17] :RequestCanceledException INFO [2019-12-17 12:31:47,139] jepsen worker 27 - jepsen.util 4977 :invoke :read [9 nil] INFO [2019-12-17 12:31:47,140] jepsen worker 10 - jepsen.util 5050 :ok :read [3 4] INFO [2019-12-17 12:31:47,140] jepsen worker 10 - jepsen.util 5050 :invoke :read [3 nil] INFO [2019-12-17 12:31:47,140] jepsen worker 66 - jepsen.util 6456 :invoke :cas [22 [49 33]] INFO [2019-12-17 12:31:47,144] jepsen worker 20 - jepsen.util 5240 :ok :write [6 19] INFO [2019-12-17 12:31:47,144] jepsen worker 20 - jepsen.util 5240 :invoke :write [6 46] INFO [2019-12-17 12:31:47,154] jepsen worker 30 - jepsen.util 5250 :fail :cas [10 [13 33]] :ValueNotSwapFrom INFO [2019-12-17 12:31:47,154] jepsen worker 30 - jepsen.util 5250 :invoke :read [10 nil] INFO [2019-12-17 12:31:47,156] jepsen worker 43 - jepsen.util 6523 :info :write [14 1] :RequestCanceledException INFO [2019-12-17 12:31:47,158] jepsen worker 43 - jepsen.util 6613 :invoke :read [14 nil] INFO [2019-12-17 12:31:47,164] jepsen worker 28 - jepsen.util 3808 :ok :write [9 27] INFO [2019-12-17 12:31:47,165] jepsen worker 28 - jepsen.util 3808 :invoke :write [9 28] INFO [2019-12-17 12:31:47,199] jepsen worker 72 - jepsen.util 5022 :fail :cas [24 [11 39]] :ValueNotSwapFrom INFO [2019-12-17 12:31:47,200] jepsen worker 72 - jepsen.util 5022 :invoke :read [24 nil] INFO [2019-12-17 12:31:47,210] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29612,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:31:47,211] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29612,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:31:47,211] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29613,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:31:47,211] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29613,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:31:47,218] jepsen worker 31 - jepsen.util 4891 :ok :write [10 10] INFO [2019-12-17 12:31:47,218] jepsen worker 31 - jepsen.util 4891 :invoke :read [10 nil] INFO [2019-12-17 12:31:47,238] jepsen worker 58 - jepsen.util 7168 :info :write [19 5] :RequestCanceledException INFO [2019-12-17 12:31:47,246] jepsen worker 58 - jepsen.util 7258 :invoke :write [19 31] INFO [2019-12-17 12:31:47,254] jepsen worker 10 - jepsen.util 5050 :ok :read [3 4] INFO [2019-12-17 12:31:47,256] jepsen worker 27 - jepsen.util 4977 :ok :read [9 4] INFO [2019-12-17 12:31:47,261] jepsen worker 80 - jepsen.util 5300 :ok :read [26 43] INFO [2019-12-17 12:31:47,262] jepsen worker 28 - jepsen.util 3808 :ok :write [9 28] INFO [2019-12-17 12:31:47,263] jepsen worker 31 - jepsen.util 4891 :ok :read [10 10] INFO [2019-12-17 12:31:47,274] jepsen worker 30 - jepsen.util 5250 :ok :read [10 10] INFO [2019-12-17 12:31:47,282] jepsen worker 18 - jepsen.util 5058 :ok :write [6 47] INFO [2019-12-17 12:31:47,293] jepsen worker 81 - jepsen.util 6831 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:31:47,310] jepsen worker 20 - jepsen.util 5240 :ok :write [6 46] INFO [2019-12-17 12:31:47,311] jepsen worker 44 - jepsen.util 7604 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:31:47,312] jepsen worker 27 - jepsen.util 4977 :invoke :write [9 7] INFO [2019-12-17 12:31:47,312] jepsen worker 80 - jepsen.util 5300 :invoke :read [26 nil] INFO [2019-12-17 12:31:47,312] jepsen worker 10 - jepsen.util 5050 :invoke :cas [3 [33 5]] INFO [2019-12-17 12:31:47,313] jepsen worker 28 - jepsen.util 3808 :invoke :write [9 21] INFO [2019-12-17 12:31:47,313] jepsen worker 31 - jepsen.util 4891 :invoke :write [10 38] INFO [2019-12-17 12:31:47,313] jepsen worker 30 - jepsen.util 5250 :invoke :write [10 11] INFO [2019-12-17 12:31:47,313] jepsen worker 18 - jepsen.util 5058 :invoke :write [6 12] INFO [2019-12-17 12:31:47,313] jepsen worker 81 - jepsen.util 6831 :invoke :write [27 29] INFO [2019-12-17 12:31:47,313] jepsen worker 20 - jepsen.util 5240 :invoke :read [6 nil] INFO [2019-12-17 12:31:47,313] jepsen worker 44 - jepsen.util 7604 :invoke :write [14 9] INFO [2019-12-17 12:31:47,318] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29621,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:31:47,319] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29621,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:31:47,319] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29624,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:31:47,320] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29624,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:31:47,321] jepsen worker 2 - jepsen.util 6392 :info :write [0 40] :RequestCanceledException INFO [2019-12-17 12:31:47,321] jepsen worker 25 - jepsen.util 7225 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:31:47,321] jepsen worker 34 - jepsen.util 6334 :info :cas [11 [18 7]] :RequestCanceledException INFO [2019-12-17 12:31:47,322] jepsen worker 25 - jepsen.util 7225 :invoke :write [8 48] INFO [2019-12-17 12:31:47,323] jepsen worker 2 - jepsen.util 6482 :invoke :write [0 7] INFO [2019-12-17 12:31:47,323] jepsen worker 34 - jepsen.util 6424 :invoke :write [11 24] INFO [2019-12-17 12:31:47,339] jepsen worker 0 - jepsen.util 7560 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:31:47,340] jepsen worker 0 - jepsen.util 7560 :invoke :cas [0 [39 22]] INFO [2019-12-17 12:31:47,356] jepsen worker 86 - jepsen.util 7196 :info :cas [28 [17 40]] :RequestCanceledException INFO [2019-12-17 12:31:47,357] jepsen worker 86 - jepsen.util 7286 :invoke :write [28 28] INFO [2019-12-17 12:31:47,362] jepsen worker 72 - jepsen.util 5022 :ok :read [24 39] INFO [2019-12-17 12:31:47,363] jepsen worker 72 - jepsen.util 5022 :invoke :cas [24 [28 45]] INFO [2019-12-17 12:31:47,393] jepsen worker 37 - jepsen.util 4807 :fail :cas [12 [6 32]] :ValueNotSwapFrom INFO [2019-12-17 12:31:47,393] jepsen worker 37 - jepsen.util 4807 :invoke :write [12 20] INFO [2019-12-17 12:31:47,401] jepsen worker 79 - jepsen.util 5029 :ok :write [26 21] INFO [2019-12-17 12:31:47,401] jepsen worker 79 - jepsen.util 5029 :invoke :cas [26 [5 5]] INFO [2019-12-17 12:31:47,403] jepsen worker 10 - jepsen.util 5050 :fail :cas [3 [33 5]] :ValueNotSwapFrom INFO [2019-12-17 12:31:47,404] jepsen worker 10 - jepsen.util 5050 :invoke :cas [3 [5 5]] INFO [2019-12-17 12:31:47,406] jepsen worker 20 - jepsen.util 5240 :ok :read [6 46] INFO [2019-12-17 12:31:47,406] jepsen worker 20 - jepsen.util 5240 :invoke :cas [6 [26 42]] INFO [2019-12-17 12:31:47,429] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29647,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:31:47,432] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29647,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:31:47,433] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29651,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:31:47,433] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29651,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:31:47,433] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29652,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:31:47,433] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29652,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:31:47,443] jepsen worker 4 - jepsen.util 4414 :fail :cas [1 [17 16]] :ValueNotSwapFrom INFO [2019-12-17 12:31:47,444] jepsen worker 4 - jepsen.util 4414 :invoke :cas [1 [31 23]] INFO [2019-12-17 12:31:47,446] jepsen worker 13 - jepsen.util 5053 :fail :cas [4 [36 47]] :ValueNotSwapFrom INFO [2019-12-17 12:31:47,448] jepsen worker 13 - jepsen.util 5053 :invoke :cas [4 [41 1]] INFO [2019-12-17 12:31:47,456] jepsen worker 61 - jepsen.util 7711 :info :cas [20 [3 9]] :RequestCanceledException INFO [2019-12-17 12:31:47,458] jepsen worker 61 - jepsen.util 7801 :invoke :read [20 nil] INFO [2019-12-17 12:31:47,458] jepsen worker 63 - jepsen.util 4203 :ok :write [21 27] INFO [2019-12-17 12:31:47,458] jepsen worker 63 - jepsen.util 4203 :invoke :cas [21 [12 38]] INFO [2019-12-17 12:31:47,473] jepsen worker 14 - jepsen.util 5324 :ok :read [4 20] INFO [2019-12-17 12:31:47,474] jepsen worker 14 - jepsen.util 5324 :invoke :cas [4 [34 40]] INFO [2019-12-17 12:31:47,480] jepsen worker 63 - jepsen.util 4203 :fail :cas [21 [12 38]] :ValueNotSwapFrom INFO [2019-12-17 12:31:47,480] jepsen worker 63 - jepsen.util 4203 :invoke :read [21 nil] INFO [2019-12-17 12:31:47,502] jepsen worker 51 - jepsen.util 5091 :ok :read [17 12] INFO [2019-12-17 12:31:47,503] jepsen worker 51 - jepsen.util 5091 :invoke :cas [17 [13 39]] INFO [2019-12-17 12:31:47,505] jepsen worker 28 - jepsen.util 3808 :ok :write [9 21] INFO [2019-12-17 12:31:47,505] jepsen worker 28 - jepsen.util 3808 :invoke :read [9 nil] INFO [2019-12-17 12:31:47,510] jepsen worker 63 - jepsen.util 4203 :ok :read [21 27] INFO [2019-12-17 12:31:47,510] jepsen worker 63 - jepsen.util 4203 :invoke :write [21 2] INFO [2019-12-17 12:31:47,517] jepsen worker 12 - jepsen.util 5412 :ok :write [4 20] INFO [2019-12-17 12:31:47,518] jepsen worker 12 - jepsen.util 5412 :invoke :cas [4 [37 1]] INFO [2019-12-17 12:31:47,522] jepsen worker 28 - jepsen.util 3808 :ok :read [9 21] INFO [2019-12-17 12:31:47,522] jepsen worker 28 - jepsen.util 3808 :invoke :read [9 nil] INFO [2019-12-17 12:31:47,541] jepsen worker 28 - jepsen.util 3808 :ok :read [9 21] INFO [2019-12-17 12:31:47,543] jepsen worker 28 - jepsen.util 3808 :invoke :read [9 nil] INFO [2019-12-17 12:31:47,562] jepsen worker 53 - jepsen.util 5093 :fail :cas [17 [1 34]] :ValueNotSwapFrom INFO [2019-12-17 12:31:47,563] jepsen worker 53 - jepsen.util 5093 :invoke :read [17 nil] INFO [2019-12-17 12:31:47,563] jepsen worker 69 - jepsen.util 5649 :ok :read [23 47] INFO [2019-12-17 12:31:47,564] jepsen worker 69 - jepsen.util 5649 :invoke :cas [23 [13 8]] INFO [2019-12-17 12:31:47,565] jepsen worker 28 - jepsen.util 3808 :ok :read [9 21] INFO [2019-12-17 12:31:47,565] jepsen worker 28 - jepsen.util 3808 :invoke :write [9 41] INFO [2019-12-17 12:31:47,570] jepsen worker 63 - jepsen.util 4203 :ok :write [21 2] INFO [2019-12-17 12:31:47,570] jepsen worker 63 - jepsen.util 4203 :invoke :cas [21 [43 1]] INFO [2019-12-17 12:31:47,592] jepsen worker 62 - jepsen.util 6452 :info :write [20 6] :RequestCanceledException INFO [2019-12-17 12:31:47,594] jepsen worker 62 - jepsen.util 6542 :invoke :cas [20 [21 27]] INFO [2019-12-17 12:31:47,600] jepsen worker 63 - jepsen.util 4203 :fail :cas [21 [43 1]] :ValueNotSwapFrom INFO [2019-12-17 12:31:47,600] jepsen worker 63 - jepsen.util 4203 :invoke :read [21 nil] INFO [2019-12-17 12:31:47,611] jepsen worker 67 - jepsen.util 6997 :info :write [22 9] :RequestCanceledException INFO [2019-12-17 12:31:47,612] jepsen worker 67 - jepsen.util 7087 :invoke :read [22 nil] INFO [2019-12-17 12:31:47,626] jepsen worker 26 - jepsen.util 7226 :info :cas [8 [20 23]] :RequestCanceledException INFO [2019-12-17 12:31:47,630] jepsen worker 26 - jepsen.util 7316 :invoke :cas [8 [2 20]] INFO [2019-12-17 12:31:47,640] jepsen worker 63 - jepsen.util 4203 :ok :read [21 2] INFO [2019-12-17 12:31:47,640] jepsen worker 63 - jepsen.util 4203 :invoke :cas [21 [3 33]] INFO [2019-12-17 12:31:47,640] jepsen worker 88 - jepsen.util 5488 :ok :read [29 0] INFO [2019-12-17 12:31:47,640] jepsen worker 88 - jepsen.util 5488 :invoke :write [29 14] INFO [2019-12-17 12:31:47,644] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29676,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:31:47,644] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29676,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:31:47,644] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29679,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:31:47,644] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29679,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:31:47,653] jepsen worker 18 - jepsen.util 5058 :ok :write [6 12] INFO [2019-12-17 12:31:47,653] jepsen worker 18 - jepsen.util 5058 :invoke :read [6 nil] INFO [2019-12-17 12:31:47,656] jepsen worker 28 - jepsen.util 3808 :ok :write [9 41] INFO [2019-12-17 12:31:47,656] jepsen worker 28 - jepsen.util 3808 :invoke :write [9 34] INFO [2019-12-17 12:31:47,658] jepsen worker 63 - jepsen.util 4203 :fail :cas [21 [3 33]] :ValueNotSwapFrom INFO [2019-12-17 12:31:47,658] jepsen worker 63 - jepsen.util 4203 :invoke :write [21 26] INFO [2019-12-17 12:31:47,668] jepsen worker 72 - jepsen.util 5022 :fail :cas [24 [28 45]] :ValueNotSwapFrom INFO [2019-12-17 12:31:47,669] jepsen worker 72 - jepsen.util 5022 :invoke :read [24 nil] INFO [2019-12-17 12:31:47,687] jepsen worker 21 - jepsen.util 6321 :info :write [7 16] :RequestCanceledException INFO [2019-12-17 12:31:47,689] jepsen worker 21 - jepsen.util 6411 :invoke :write [7 26] INFO [2019-12-17 12:31:47,691] jepsen worker 51 - jepsen.util 5091 :fail :cas [17 [13 39]] :ValueNotSwapFrom INFO [2019-12-17 12:31:47,691] jepsen worker 51 - jepsen.util 5091 :invoke :write [17 35] INFO [2019-12-17 12:31:47,701] jepsen worker 20 - jepsen.util 5240 :fail :cas [6 [26 42]] :ValueNotSwapFrom INFO [2019-12-17 12:31:47,701] jepsen worker 20 - jepsen.util 5240 :invoke :read [6 nil] INFO [2019-12-17 12:31:47,702] jepsen worker 55 - jepsen.util 6085 :info :write [18 3] :RequestCanceledException INFO [2019-12-17 12:31:47,703] jepsen worker 55 - jepsen.util 6175 :invoke :write [18 41] INFO [2019-12-17 12:31:47,707] jepsen worker 18 - jepsen.util 5058 :ok :read [6 12] INFO [2019-12-17 12:31:47,707] jepsen worker 18 - jepsen.util 5058 :invoke :read [6 nil] INFO [2019-12-17 12:31:47,715] jepsen worker 72 - jepsen.util 5022 :ok :read [24 39] INFO [2019-12-17 12:31:47,715] jepsen worker 72 - jepsen.util 5022 :invoke :write [24 33] INFO [2019-12-17 12:31:47,725] jepsen worker 32 - jepsen.util 5252 :info :cas [10 [34 16]] :RequestCanceledException INFO [2019-12-17 12:31:47,726] jepsen worker 65 - jepsen.util 4565 :fail :read [21 nil] :RequestTimeoutException INFO [2019-12-17 12:31:47,726] jepsen worker 65 - jepsen.util 4565 :invoke :read [21 nil] INFO [2019-12-17 12:31:47,726] jepsen worker 19 - jepsen.util 5059 :info :write [6 4] :RequestCanceledException INFO [2019-12-17 12:31:47,726] jepsen worker 78 - jepsen.util 5208 :info :write [26 35] :RequestCanceledException INFO [2019-12-17 12:31:47,727] jepsen worker 32 - jepsen.util 5342 :invoke :cas [10 [11 37]] INFO [2019-12-17 12:31:47,727] jepsen worker 19 - jepsen.util 5149 :invoke :read [6 nil] INFO [2019-12-17 12:31:47,728] jepsen worker 78 - jepsen.util 5298 :invoke :cas [26 [14 39]] INFO [2019-12-17 12:31:47,730] jepsen worker 20 - jepsen.util 5240 :ok :read [6 12] INFO [2019-12-17 12:31:47,731] jepsen worker 20 - jepsen.util 5240 :invoke :write [6 36] INFO [2019-12-17 12:31:47,740] jepsen worker 1 - jepsen.util 6661 :info :cas [0 [9 6]] :RequestCanceledException INFO [2019-12-17 12:31:47,741] jepsen worker 1 - jepsen.util 6751 :invoke :cas [0 [5 23]] INFO [2019-12-17 12:31:47,751] jepsen worker 18 - jepsen.util 5058 :ok :read [6 12] INFO [2019-12-17 12:31:47,751] jepsen worker 18 - jepsen.util 5058 :invoke :read [6 nil] INFO [2019-12-17 12:31:47,754] jepsen worker 7 - jepsen.util 7027 :info :cas [2 [29 0]] :RequestCanceledException INFO [2019-12-17 12:31:47,755] jepsen worker 7 - jepsen.util 7117 :invoke :write [2 40] INFO [2019-12-17 12:31:47,755] jepsen worker 63 - jepsen.util 4203 :ok :write [21 26] INFO [2019-12-17 12:31:47,756] jepsen worker 63 - jepsen.util 4203 :invoke :cas [21 [49 36]] INFO [2019-12-17 12:31:47,756] jepsen worker 69 - jepsen.util 5649 :fail :cas [23 [13 8]] :ValueNotSwapFrom INFO [2019-12-17 12:31:47,756] jepsen worker 69 - jepsen.util 5649 :invoke :cas [23 [19 13]] INFO [2019-12-17 12:31:47,762] jepsen worker 80 - jepsen.util 5300 :ok :read [26 21] INFO [2019-12-17 12:31:47,763] jepsen worker 80 - jepsen.util 5300 :invoke :read [26 nil] INFO [2019-12-17 12:31:47,772] jepsen worker 54 - jepsen.util 5544 :fail :cas [18 [36 0]] :ValueNotSwapFrom INFO [2019-12-17 12:31:47,772] jepsen worker 54 - jepsen.util 5544 :invoke :write [18 33] INFO [2019-12-17 12:31:47,777] jepsen worker 32 - jepsen.util 5342 :fail :cas [10 [11 37]] :ValueNotSwapFrom INFO [2019-12-17 12:31:47,778] jepsen worker 32 - jepsen.util 5342 :invoke :read [10 nil] INFO [2019-12-17 12:31:47,782] jepsen worker 28 - jepsen.util 3808 :ok :write [9 34] INFO [2019-12-17 12:31:47,782] jepsen worker 28 - jepsen.util 3808 :invoke :write [9 39] INFO [2019-12-17 12:31:47,787] jepsen worker 31 - jepsen.util 4891 :ok :write [10 38] INFO [2019-12-17 12:31:47,788] jepsen worker 31 - jepsen.util 4891 :invoke :cas [10 [18 46]] INFO [2019-12-17 12:31:47,790] jepsen worker 12 - jepsen.util 5412 :fail :cas [4 [37 1]] :ValueNotSwapFrom INFO [2019-12-17 12:31:47,790] jepsen worker 12 - jepsen.util 5412 :invoke :write [4 47] INFO [2019-12-17 12:31:47,790] jepsen worker 63 - jepsen.util 4203 :fail :cas [21 [49 36]] :ValueNotSwapFrom INFO [2019-12-17 12:31:47,791] jepsen worker 63 - jepsen.util 4203 :invoke :read [21 nil] INFO [2019-12-17 12:31:47,800] jepsen worker 5 - jepsen.util 4775 :ok :read [1 23] INFO [2019-12-17 12:31:47,800] jepsen worker 5 - jepsen.util 4775 :invoke :cas [1 [17 37]] INFO [2019-12-17 12:31:47,814] jepsen worker 87 - jepsen.util 4677 :ok :read [29 0] INFO [2019-12-17 12:31:47,814] jepsen worker 87 - jepsen.util 4677 :invoke :cas [29 [48 34]] INFO [2019-12-17 12:31:47,815] jepsen worker 65 - jepsen.util 4565 :ok :read [21 26] INFO [2019-12-17 12:31:47,816] jepsen worker 65 - jepsen.util 4565 :invoke :read [21 nil] INFO [2019-12-17 12:31:47,825] jepsen worker 79 - jepsen.util 5029 :fail :cas [26 [5 5]] :ValueNotSwapFrom INFO [2019-12-17 12:31:47,825] jepsen worker 79 - jepsen.util 5029 :invoke :write [26 48] INFO [2019-12-17 12:31:47,825] jepsen worker 35 - jepsen.util 6965 :info :write [11 29] :RequestCanceledException INFO [2019-12-17 12:31:47,826] jepsen worker 47 - jepsen.util 5717 :info :cas [15 [17 49]] :RequestCanceledException INFO [2019-12-17 12:31:47,826] jepsen worker 35 - jepsen.util 7055 :invoke :cas [11 [25 10]] INFO [2019-12-17 12:31:47,827] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:31:47,827] jepsen nemesis - couchbase.util Rebalance check count: 223 INFO [2019-12-17 12:31:47,827] jepsen nemesis - couchbase.util Rebalance stuck count: 204 INFO [2019-12-17 12:31:47,827] jepsen worker 47 - jepsen.util 5807 :invoke :write [15 14] INFO [2019-12-17 12:31:47,835] jepsen worker 4 - jepsen.util 4414 :fail :cas [1 [31 23]] :ValueNotSwapFrom INFO [2019-12-17 12:31:47,850] jepsen worker 14 - jepsen.util 5324 :fail :cas [4 [34 40]] :ValueNotSwapFrom INFO [2019-12-17 12:31:47,850] jepsen worker 32 - jepsen.util 5342 :ok :read [10 38] INFO [2019-12-17 12:31:47,862] jepsen worker 31 - jepsen.util 4891 :fail :cas [10 [18 46]] :ValueNotSwapFrom INFO [2019-12-17 12:31:47,862] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29715,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:31:47,892] jepsen worker 50 - jepsen.util 7340 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:31:47,892] jepsen worker 42 - jepsen.util 7512 :info :cas [14 [8 33]] :RequestCanceledException INFO [2019-12-17 12:31:47,902] jepsen worker 63 - jepsen.util 4203 :ok :read [21 26] INFO [2019-12-17 12:31:47,903] jepsen worker 65 - jepsen.util 4565 :ok :read [21 26] INFO [2019-12-17 12:31:47,925] jepsen worker 38 - jepsen.util 5078 :info :write [12 3] :RequestCanceledException INFO [2019-12-17 12:31:47,925] jepsen worker 49 - jepsen.util 6079 :info :write [16 45] :RequestCanceledException INFO [2019-12-17 12:31:47,940] jepsen worker 24 - jepsen.util 6414 :info :write [8 20] :RequestCanceledException INFO [2019-12-17 12:31:47,941] jepsen worker 28 - jepsen.util 3808 :ok :write [9 39] INFO [2019-12-17 12:31:47,949] jepsen worker 4 - jepsen.util 4414 :invoke :read [1 nil] INFO [2019-12-17 12:31:47,949] jepsen worker 14 - jepsen.util 5324 :invoke :cas [4 [9 7]] INFO [2019-12-17 12:31:47,950] jepsen worker 32 - jepsen.util 5342 :invoke :cas [10 [6 9]] INFO [2019-12-17 12:31:47,950] jepsen worker 31 - jepsen.util 4891 :invoke :read [10 nil] INFO [2019-12-17 12:31:47,950] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29715,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:31:47,950] jepsen worker 50 - jepsen.util 7340 :invoke :read [16 nil] INFO [2019-12-17 12:31:47,950] jepsen worker 63 - jepsen.util 4203 :invoke :write [21 14] INFO [2019-12-17 12:31:47,950] jepsen worker 65 - jepsen.util 4565 :invoke :write [21 39] INFO [2019-12-17 12:31:47,950] jepsen worker 28 - jepsen.util 3808 :invoke :write [9 43] INFO [2019-12-17 12:31:47,951] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29718,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:31:47,951] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29718,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:31:47,951] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29719,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:31:47,951] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29719,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:31:47,951] jepsen worker 42 - jepsen.util 7602 :invoke :cas [14 [44 41]] INFO [2019-12-17 12:31:47,951] jepsen worker 38 - jepsen.util 5168 :invoke :read [12 nil] INFO [2019-12-17 12:31:47,951] jepsen worker 24 - jepsen.util 6504 :invoke :cas [8 [23 17]] INFO [2019-12-17 12:31:47,951] jepsen worker 49 - jepsen.util 6169 :invoke :read [16 nil] INFO [2019-12-17 12:31:47,957] jepsen worker 89 - jepsen.util 4949 :info :write [29 24] :RequestCanceledException INFO [2019-12-17 12:31:47,958] jepsen worker 89 - jepsen.util 5039 :invoke :cas [29 [30 30]] INFO [2019-12-17 12:31:47,969] jepsen worker 80 - jepsen.util 5300 :ok :read [26 21] INFO [2019-12-17 12:31:47,970] jepsen worker 80 - jepsen.util 5300 :invoke :cas [26 [23 13]] INFO [2019-12-17 12:31:47,979] jepsen worker 31 - jepsen.util 4891 :ok :read [10 38] INFO [2019-12-17 12:31:47,980] jepsen worker 31 - jepsen.util 4891 :invoke :read [10 nil] INFO [2019-12-17 12:31:48,001] jepsen worker 72 - jepsen.util 5022 :ok :write [24 33] INFO [2019-12-17 12:31:48,001] jepsen worker 72 - jepsen.util 5022 :invoke :write [24 25] INFO [2019-12-17 12:31:48,026] jepsen worker 29 - jepsen.util 4079 :info :cas [9 [20 17]] :RequestCanceledException INFO [2019-12-17 12:31:48,026] jepsen worker 59 - jepsen.util 6809 :info :write [19 5] :RequestCanceledException INFO [2019-12-17 12:31:48,027] jepsen worker 29 - jepsen.util 4169 :invoke :write [9 38] INFO [2019-12-17 12:31:48,027] jepsen worker 59 - jepsen.util 6899 :invoke :write [19 2] INFO [2019-12-17 12:31:48,040] jepsen worker 11 - jepsen.util 5231 :info :write [3 45] :RequestCanceledException INFO [2019-12-17 12:31:48,040] jepsen worker 4 - jepsen.util 4414 :ok :read [1 23] INFO [2019-12-17 12:31:48,040] jepsen worker 4 - jepsen.util 4414 :invoke :write [1 29] INFO [2019-12-17 12:31:48,041] jepsen worker 11 - jepsen.util 5321 :invoke :write [3 48] INFO [2019-12-17 12:31:48,042] jepsen worker 18 - jepsen.util 5058 :ok :read [6 12] INFO [2019-12-17 12:31:48,042] jepsen worker 18 - jepsen.util 5058 :invoke :cas [6 [46 42]] INFO [2019-12-17 12:31:48,057] jepsen worker 23 - jepsen.util 7133 :info :write [7 0] :RequestCanceledException INFO [2019-12-17 12:31:48,058] jepsen worker 23 - jepsen.util 7223 :invoke :cas [7 [14 9]] INFO [2019-12-17 12:31:48,061] jepsen worker 14 - jepsen.util 5324 :fail :cas [4 [9 7]] :ValueNotSwapFrom INFO [2019-12-17 12:31:48,061] jepsen worker 14 - jepsen.util 5324 :invoke :cas [4 [13 47]] INFO [2019-12-17 12:31:48,081] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29738,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:31:48,081] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29738,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:31:48,081] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29740,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:31:48,081] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29740,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:31:48,089] jepsen worker 65 - jepsen.util 4565 :ok :write [21 39] INFO [2019-12-17 12:31:48,089] jepsen worker 65 - jepsen.util 4565 :invoke :cas [21 [24 35]] INFO [2019-12-17 12:31:48,097] jepsen worker 28 - jepsen.util 3808 :ok :write [9 43] INFO [2019-12-17 12:31:48,097] jepsen worker 28 - jepsen.util 3808 :invoke :cas [9 [11 5]] INFO [2019-12-17 12:31:48,118] jepsen worker 65 - jepsen.util 4565 :fail :cas [21 [24 35]] :ValueNotSwapFrom INFO [2019-12-17 12:31:48,118] jepsen worker 65 - jepsen.util 4565 :invoke :cas [21 [32 44]] INFO [2019-12-17 12:31:48,126] jepsen worker 56 - jepsen.util 5276 :info :write [18 20] :RequestCanceledException INFO [2019-12-17 12:31:48,127] jepsen worker 56 - jepsen.util 5366 :invoke :read [18 nil] INFO [2019-12-17 12:31:48,141] jepsen worker 65 - jepsen.util 4565 :fail :cas [21 [32 44]] :ValueNotSwapFrom INFO [2019-12-17 12:31:48,141] jepsen worker 65 - jepsen.util 4565 :invoke :cas [21 [46 30]] INFO [2019-12-17 12:31:48,147] jepsen worker 87 - jepsen.util 4677 :fail :cas [29 [48 34]] :ValueNotSwapFrom INFO [2019-12-17 12:31:48,147] jepsen worker 87 - jepsen.util 4677 :invoke :write [29 23] INFO [2019-12-17 12:31:48,167] jepsen worker 65 - jepsen.util 4565 :fail :cas [21 [46 30]] :ValueNotSwapFrom INFO [2019-12-17 12:31:48,167] jepsen worker 65 - jepsen.util 4565 :invoke :read [21 nil] INFO [2019-12-17 12:31:48,174] jepsen worker 56 - jepsen.util 5366 :ok :read [18 45] INFO [2019-12-17 12:31:48,174] jepsen worker 56 - jepsen.util 5366 :invoke :write [18 12] INFO [2019-12-17 12:31:48,184] jepsen worker 53 - jepsen.util 5093 :ok :read [17 12] INFO [2019-12-17 12:31:48,184] jepsen worker 53 - jepsen.util 5093 :invoke :write [17 46] INFO [2019-12-17 12:31:48,191] jepsen worker 65 - jepsen.util 4565 :ok :read [21 39] INFO [2019-12-17 12:31:48,191] jepsen worker 65 - jepsen.util 4565 :invoke :cas [21 [30 25]] INFO [2019-12-17 12:31:48,204] jepsen worker 47 - jepsen.util 5807 :ok :write [15 14] INFO [2019-12-17 12:31:48,204] jepsen worker 47 - jepsen.util 5807 :invoke :cas [15 [32 46]] INFO [2019-12-17 12:31:48,255] jepsen worker 69 - jepsen.util 5649 :fail :cas [23 [19 13]] :ValueNotSwapFrom INFO [2019-12-17 12:31:48,255] jepsen worker 69 - jepsen.util 5649 :invoke :cas [23 [16 34]] INFO [2019-12-17 12:31:48,258] jepsen worker 65 - jepsen.util 4565 :fail :cas [21 [30 25]] :ValueNotSwapFrom INFO [2019-12-17 12:31:48,258] jepsen worker 65 - jepsen.util 4565 :invoke :cas [21 [35 8]] INFO [2019-12-17 12:31:48,267] jepsen worker 5 - jepsen.util 4775 :fail :cas [1 [17 37]] :ValueNotSwapFrom INFO [2019-12-17 12:31:48,267] jepsen worker 32 - jepsen.util 5342 :fail :cas [10 [6 9]] :ValueNotSwapFrom INFO [2019-12-17 12:31:48,267] jepsen worker 5 - jepsen.util 4775 :invoke :write [1 43] INFO [2019-12-17 12:31:48,267] jepsen worker 32 - jepsen.util 5342 :invoke :read [10 nil] INFO [2019-12-17 12:31:48,283] jepsen worker 65 - jepsen.util 4565 :fail :cas [21 [35 8]] :ValueNotSwapFrom INFO [2019-12-17 12:31:48,283] jepsen worker 65 - jepsen.util 4565 :invoke :cas [21 [42 22]] INFO [2019-12-17 12:31:48,300] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29759,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:31:48,300] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29767,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:31:48,300] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29759,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:31:48,300] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29767,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:31:48,300] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29761,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:31:48,300] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29770,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:31:48,300] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29761,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:31:48,300] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29770,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:31:48,300] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29771,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:31:48,300] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29771,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:31:48,313] jepsen worker 72 - jepsen.util 5022 :ok :write [24 25] INFO [2019-12-17 12:31:48,313] jepsen worker 72 - jepsen.util 5022 :invoke :read [24 nil] INFO [2019-12-17 12:31:48,315] jepsen worker 65 - jepsen.util 4565 :fail :cas [21 [42 22]] :ValueNotSwapFrom INFO [2019-12-17 12:31:48,315] jepsen worker 65 - jepsen.util 4565 :invoke :read [21 nil] INFO [2019-12-17 12:31:48,341] jepsen worker 63 - jepsen.util 4203 :ok :write [21 14] INFO [2019-12-17 12:31:48,341] jepsen worker 63 - jepsen.util 4203 :invoke :read [21 nil] INFO [2019-12-17 12:31:48,341] jepsen worker 72 - jepsen.util 5022 :ok :read [24 25] INFO [2019-12-17 12:31:48,341] jepsen worker 72 - jepsen.util 5022 :invoke :write [24 17] INFO [2019-12-17 12:31:48,343] jepsen worker 65 - jepsen.util 4565 :ok :read [21 39] INFO [2019-12-17 12:31:48,343] jepsen worker 65 - jepsen.util 4565 :invoke :cas [21 [49 34]] INFO [2019-12-17 12:31:48,380] jepsen worker 13 - jepsen.util 5053 :fail :cas [4 [41 1]] :ValueNotSwapFrom INFO [2019-12-17 12:31:48,380] jepsen worker 13 - jepsen.util 5053 :invoke :write [4 35] INFO [2019-12-17 12:31:48,390] jepsen worker 65 - jepsen.util 4565 :fail :cas [21 [49 34]] :ValueNotSwapFrom INFO [2019-12-17 12:31:48,390] jepsen worker 65 - jepsen.util 4565 :invoke :read [21 nil] INFO [2019-12-17 12:31:48,392] jepsen worker 63 - jepsen.util 4203 :ok :read [21 14] INFO [2019-12-17 12:31:48,392] jepsen worker 63 - jepsen.util 4203 :invoke :write [21 2] INFO [2019-12-17 12:31:48,417] jepsen worker 78 - jepsen.util 5298 :fail :cas [26 [14 39]] :ValueNotSwapFrom INFO [2019-12-17 12:31:48,417] jepsen worker 78 - jepsen.util 5298 :invoke :cas [26 [30 36]] INFO [2019-12-17 12:31:48,437] jepsen worker 63 - jepsen.util 4203 :ok :write [21 2] INFO [2019-12-17 12:31:48,437] jepsen worker 63 - jepsen.util 4203 :invoke :read [21 nil] INFO [2019-12-17 12:31:48,437] jepsen worker 65 - jepsen.util 4565 :ok :read [21 14] INFO [2019-12-17 12:31:48,438] jepsen worker 65 - jepsen.util 4565 :invoke :cas [21 [30 39]] INFO [2019-12-17 12:31:48,439] jepsen worker 45 - jepsen.util 5175 :info :cas [15 [19 16]] :RequestCanceledException INFO [2019-12-17 12:31:48,441] jepsen worker 45 - jepsen.util 5265 :invoke :cas [15 [9 15]] INFO [2019-12-17 12:31:48,461] jepsen worker 63 - jepsen.util 4203 :ok :read [21 2] INFO [2019-12-17 12:31:48,461] jepsen worker 63 - jepsen.util 4203 :invoke :cas [21 [24 23]] INFO [2019-12-17 12:31:48,463] jepsen worker 65 - jepsen.util 4565 :fail :cas [21 [30 39]] :ValueNotSwapFrom INFO [2019-12-17 12:31:48,463] jepsen worker 65 - jepsen.util 4565 :invoke :write [21 30] INFO [2019-12-17 12:31:48,472] jepsen worker 80 - jepsen.util 5300 :fail :cas [26 [23 13]] :ValueNotSwapFrom INFO [2019-12-17 12:31:48,472] jepsen worker 80 - jepsen.util 5300 :invoke :read [26 nil] INFO [2019-12-17 12:31:48,480] jepsen worker 45 - jepsen.util 5265 :fail :cas [15 [9 15]] :ValueNotSwapFrom INFO [2019-12-17 12:31:48,480] jepsen worker 45 - jepsen.util 5265 :invoke :cas [15 [49 44]] INFO [2019-12-17 12:31:48,495] jepsen worker 63 - jepsen.util 4203 :fail :cas [21 [24 23]] :ValueNotSwapFrom INFO [2019-12-17 12:31:48,495] jepsen worker 63 - jepsen.util 4203 :invoke :cas [21 [40 35]] INFO [2019-12-17 12:31:48,506] jepsen worker 63 - jepsen.util 4203 :fail :cas [21 [40 35]] :ValueNotSwapFrom INFO [2019-12-17 12:31:48,506] jepsen worker 63 - jepsen.util 4203 :invoke :write [21 14] INFO [2019-12-17 12:31:48,509] jepsen worker 38 - jepsen.util 5168 :ok :read [12 28] INFO [2019-12-17 12:31:48,509] jepsen worker 38 - jepsen.util 5168 :invoke :write [12 23] INFO [2019-12-17 12:31:48,519] jepsen worker 65 - jepsen.util 4565 :ok :write [21 30] INFO [2019-12-17 12:31:48,519] jepsen worker 65 - jepsen.util 4565 :invoke :write [21 47] INFO [2019-12-17 12:31:48,524] jepsen worker 14 - jepsen.util 5324 :fail :cas [4 [13 47]] :ValueNotSwapFrom INFO [2019-12-17 12:31:48,524] jepsen worker 14 - jepsen.util 5324 :invoke :cas [4 [31 36]] INFO [2019-12-17 12:31:48,527] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29787,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:31:48,528] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29787,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:31:48,528] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29791,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:31:48,528] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29791,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:31:48,563] jepsen worker 65 - jepsen.util 4565 :ok :write [21 47] INFO [2019-12-17 12:31:48,563] jepsen worker 65 - jepsen.util 4565 :invoke :read [21 nil] INFO [2019-12-17 12:31:48,577] jepsen worker 65 - jepsen.util 4565 :ok :read [21 47] INFO [2019-12-17 12:31:48,577] jepsen worker 65 - jepsen.util 4565 :invoke :cas [21 [38 33]] INFO [2019-12-17 12:31:48,588] jepsen worker 65 - jepsen.util 4565 :fail :cas [21 [38 33]] :ValueNotSwapFrom INFO [2019-12-17 12:31:48,588] jepsen worker 65 - jepsen.util 4565 :invoke :write [21 15] INFO [2019-12-17 12:31:48,621] jepsen worker 65 - jepsen.util 4565 :ok :write [21 15] INFO [2019-12-17 12:31:48,622] jepsen worker 65 - jepsen.util 4565 :invoke :write [21 34] INFO [2019-12-17 12:31:48,662] jepsen worker 89 - jepsen.util 5039 :fail :cas [29 [30 30]] :ValueNotSwapFrom INFO [2019-12-17 12:31:48,663] jepsen worker 89 - jepsen.util 5039 :invoke :read [29 nil] INFO [2019-12-17 12:31:48,667] jepsen worker 65 - jepsen.util 4565 :ok :write [21 34] INFO [2019-12-17 12:31:48,667] jepsen worker 65 - jepsen.util 4565 :invoke :read [21 nil] INFO [2019-12-17 12:31:48,681] jepsen worker 65 - jepsen.util 4565 :ok :read [21 34] INFO [2019-12-17 12:31:48,681] jepsen worker 65 - jepsen.util 4565 :invoke :read [21 nil] INFO [2019-12-17 12:31:48,696] jepsen worker 65 - jepsen.util 4565 :ok :read [21 34] INFO [2019-12-17 12:31:48,696] jepsen worker 65 - jepsen.util 4565 :invoke :read [21 nil] INFO [2019-12-17 12:31:48,703] jepsen worker 11 - jepsen.util 5321 :ok :write [3 48] INFO [2019-12-17 12:31:48,703] jepsen worker 11 - jepsen.util 5321 :invoke :read [3 nil] INFO [2019-12-17 12:31:48,703] jepsen worker 72 - jepsen.util 5022 :ok :write [24 17] INFO [2019-12-17 12:31:48,703] jepsen worker 72 - jepsen.util 5022 :invoke :cas [24 [3 16]] INFO [2019-12-17 12:31:48,709] jepsen worker 65 - jepsen.util 4565 :ok :read [21 34] INFO [2019-12-17 12:31:48,709] jepsen worker 65 - jepsen.util 4565 :invoke :read [21 nil] INFO [2019-12-17 12:31:48,735] jepsen worker 72 - jepsen.util 5022 :fail :cas [24 [3 16]] :ValueNotSwapFrom INFO [2019-12-17 12:31:48,736] jepsen worker 72 - jepsen.util 5022 :invoke :cas [24 [25 3]] INFO [2019-12-17 12:31:48,736] jepsen worker 11 - jepsen.util 5321 :ok :read [3 48] INFO [2019-12-17 12:31:48,736] jepsen worker 11 - jepsen.util 5321 :invoke :write [3 45] INFO [2019-12-17 12:31:48,738] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29809,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:31:48,738] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29809,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:31:48,738] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29813,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:31:48,738] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29813,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:31:48,740] jepsen worker 65 - jepsen.util 4565 :ok :read [21 34] INFO [2019-12-17 12:31:48,741] jepsen worker 65 - jepsen.util 4565 :invoke :write [21 45] INFO [2019-12-17 12:31:48,754] jepsen worker 72 - jepsen.util 5022 :fail :cas [24 [25 3]] :ValueNotSwapFrom INFO [2019-12-17 12:31:48,754] jepsen worker 72 - jepsen.util 5022 :invoke :cas [24 [5 23]] INFO [2019-12-17 12:31:48,771] jepsen worker 88 - jepsen.util 5488 :ok :write [29 14] INFO [2019-12-17 12:31:48,772] jepsen worker 88 - jepsen.util 5488 :invoke :cas [29 [15 35]] INFO [2019-12-17 12:31:48,783] jepsen worker 72 - jepsen.util 5022 :fail :cas [24 [5 23]] :ValueNotSwapFrom INFO [2019-12-17 12:31:48,784] jepsen worker 72 - jepsen.util 5022 :invoke :cas [24 [1 16]] INFO [2019-12-17 12:31:48,792] jepsen worker 52 - jepsen.util 5452 :info :write [17 33] :RequestCanceledException INFO [2019-12-17 12:31:48,793] jepsen worker 52 - jepsen.util 5542 :invoke :cas [17 [37 31]] INFO [2019-12-17 12:31:48,810] jepsen worker 73 - jepsen.util 4753 :info :write [24 41] :RequestCanceledException INFO [2019-12-17 12:31:48,811] jepsen worker 72 - jepsen.util 5022 :fail :cas [24 [1 16]] :ValueNotSwapFrom INFO [2019-12-17 12:31:48,811] jepsen worker 72 - jepsen.util 5022 :invoke :read [24 nil] INFO [2019-12-17 12:31:48,812] jepsen worker 73 - jepsen.util 4843 :invoke :read [24 nil] INFO [2019-12-17 12:31:48,833] jepsen worker 11 - jepsen.util 5321 :ok :write [3 45] INFO [2019-12-17 12:31:48,833] jepsen worker 11 - jepsen.util 5321 :invoke :cas [3 [30 34]] INFO [2019-12-17 12:31:48,840] jepsen worker 70 - jepsen.util 5830 :info :write [23 10] :RequestCanceledException INFO [2019-12-17 12:31:48,841] jepsen worker 70 - jepsen.util 5920 :invoke :cas [23 [11 2]] INFO [2019-12-17 12:31:48,848] jepsen worker 72 - jepsen.util 5022 :ok :read [24 17] INFO [2019-12-17 12:31:48,848] jepsen worker 72 - jepsen.util 5022 :invoke :write [24 47] INFO [2019-12-17 12:31:48,853] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:31:48,853] jepsen nemesis - couchbase.util Rebalance check count: 224 INFO [2019-12-17 12:31:48,853] jepsen nemesis - couchbase.util Rebalance stuck count: 205 INFO [2019-12-17 12:31:48,857] jepsen worker 36 - jepsen.util 5436 :info :write [12 19] :RequestCanceledException INFO [2019-12-17 12:31:48,858] jepsen worker 36 - jepsen.util 5526 :invoke :write [12 13] INFO [2019-12-17 12:31:48,865] jepsen worker 11 - jepsen.util 5321 :fail :cas [3 [30 34]] :ValueNotSwapFrom INFO [2019-12-17 12:31:48,865] jepsen worker 11 - jepsen.util 5321 :invoke :cas [3 [9 31]] INFO [2019-12-17 12:31:48,905] jepsen worker 73 - jepsen.util 4843 :ok :read [24 17] INFO [2019-12-17 12:31:48,906] jepsen worker 73 - jepsen.util 4843 :invoke :cas [24 [20 40]] INFO [2019-12-17 12:31:48,908] jepsen worker 74 - jepsen.util 5474 :info :cas [24 [14 3]] :RequestCanceledException INFO [2019-12-17 12:31:48,908] jepsen worker 69 - jepsen.util 5649 :fail :cas [23 [16 34]] :ValueNotSwapFrom INFO [2019-12-17 12:31:48,908] jepsen worker 69 - jepsen.util 5649 :invoke :cas [23 [23 8]] INFO [2019-12-17 12:31:48,910] jepsen worker 74 - jepsen.util 5564 :invoke :write [24 48] INFO [2019-12-17 12:31:48,911] jepsen worker 11 - jepsen.util 5321 :fail :cas [3 [9 31]] :ValueNotSwapFrom INFO [2019-12-17 12:31:48,911] jepsen worker 11 - jepsen.util 5321 :invoke :read [3 nil] INFO [2019-12-17 12:31:48,918] jepsen worker 78 - jepsen.util 5298 :fail :cas [26 [30 36]] :ValueNotSwapFrom INFO [2019-12-17 12:31:48,919] jepsen worker 78 - jepsen.util 5298 :invoke :write [26 21] INFO [2019-12-17 12:31:48,925] jepsen worker 48 - jepsen.util 6348 :info :cas [16 [10 16]] :RequestCanceledException INFO [2019-12-17 12:31:48,927] jepsen worker 48 - jepsen.util 6438 :invoke :read [16 nil] INFO [2019-12-17 12:31:48,929] jepsen worker 11 - jepsen.util 5321 :ok :read [3 45] INFO [2019-12-17 12:31:48,929] jepsen worker 11 - jepsen.util 5321 :invoke :cas [3 [20 32]] INFO [2019-12-17 12:31:48,932] jepsen worker 80 - jepsen.util 5300 :ok :read [26 21] INFO [2019-12-17 12:31:48,932] jepsen worker 80 - jepsen.util 5300 :invoke :cas [26 [19 0]] INFO [2019-12-17 12:31:48,933] jepsen worker 65 - jepsen.util 4565 :ok :write [21 45] INFO [2019-12-17 12:31:48,933] jepsen worker 65 - jepsen.util 4565 :invoke :cas [21 [23 2]] INFO [2019-12-17 12:31:48,934] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29834,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:31:48,934] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29834,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:31:48,934] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29837,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:31:48,934] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29837,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:31:48,939] jepsen worker 68 - jepsen.util 6548 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:31:48,940] jepsen worker 68 - jepsen.util 6548 :invoke :read [22 nil] INFO [2019-12-17 12:31:48,971] jepsen worker 11 - jepsen.util 5321 :fail :cas [3 [20 32]] :ValueNotSwapFrom INFO [2019-12-17 12:31:48,971] jepsen worker 11 - jepsen.util 5321 :invoke :read [3 nil] INFO [2019-12-17 12:31:48,975] jepsen worker 65 - jepsen.util 4565 :fail :cas [21 [23 2]] :ValueNotSwapFrom INFO [2019-12-17 12:31:48,975] jepsen worker 65 - jepsen.util 4565 :invoke :cas [21 [18 34]] INFO [2019-12-17 12:31:48,988] jepsen worker 15 - jepsen.util 5955 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:31:48,989] jepsen worker 15 - jepsen.util 5955 :invoke :cas [5 [25 47]] INFO [2019-12-17 12:31:48,993] jepsen worker 11 - jepsen.util 5321 :ok :read [3 45] INFO [2019-12-17 12:31:48,993] jepsen worker 11 - jepsen.util 5321 :invoke :write [3 15] INFO [2019-12-17 12:31:48,996] jepsen worker 65 - jepsen.util 4565 :fail :cas [21 [18 34]] :ValueNotSwapFrom INFO [2019-12-17 12:31:48,996] jepsen worker 65 - jepsen.util 4565 :invoke :read [21 nil] INFO [2019-12-17 12:31:49,010] jepsen worker 84 - jepsen.util 6564 :info :cas [28 [16 0]] :RequestCanceledException INFO [2019-12-17 12:31:49,011] jepsen worker 84 - jepsen.util 6654 :invoke :read [28 nil] INFO [2019-12-17 12:31:49,013] jepsen worker 72 - jepsen.util 5022 :ok :write [24 47] INFO [2019-12-17 12:31:49,013] jepsen worker 72 - jepsen.util 5022 :invoke :write [24 13] INFO [2019-12-17 12:31:49,015] jepsen worker 89 - jepsen.util 5039 :ok :read [29 14] INFO [2019-12-17 12:31:49,015] jepsen worker 89 - jepsen.util 5039 :invoke :read [29 nil] INFO [2019-12-17 12:31:49,021] jepsen worker 65 - jepsen.util 4565 :ok :read [21 45] INFO [2019-12-17 12:31:49,021] jepsen worker 65 - jepsen.util 4565 :invoke :cas [21 [28 40]] INFO [2019-12-17 12:31:49,026] jepsen worker 17 - jepsen.util 6857 :info :write [5 44] :RequestCanceledException INFO [2019-12-17 12:31:49,026] jepsen worker 8 - jepsen.util 6398 :info :cas [2 [11 25]] :RequestCanceledException INFO [2019-12-17 12:31:49,026] jepsen worker 33 - jepsen.util 6963 :info :cas [11 [28 27]] :RequestCanceledException INFO [2019-12-17 12:31:49,026] jepsen worker 3 - jepsen.util 5223 :info :write [1 38] :RequestCanceledException INFO [2019-12-17 12:31:49,027] jepsen worker 17 - jepsen.util 6947 :invoke :cas [5 [1 8]] INFO [2019-12-17 12:31:49,028] jepsen worker 33 - jepsen.util 7053 :invoke :read [11 nil] INFO [2019-12-17 12:31:49,028] jepsen worker 3 - jepsen.util 5313 :invoke :cas [1 [44 21]] INFO [2019-12-17 12:31:49,031] jepsen worker 8 - jepsen.util 6488 :invoke :cas [2 [11 26]] INFO [2019-12-17 12:31:49,031] jepsen worker 73 - jepsen.util 4843 :fail :cas [24 [20 40]] :ValueNotSwapFrom INFO [2019-12-17 12:31:49,031] jepsen worker 73 - jepsen.util 4843 :invoke :read [24 nil] INFO [2019-12-17 12:31:49,040] jepsen worker 6 - jepsen.util 7116 :info :cas [2 [16 28]] :RequestCanceledException INFO [2019-12-17 12:31:49,040] jepsen worker 60 - jepsen.util 7620 :info :write [20 46] :RequestCanceledException INFO [2019-12-17 12:31:49,042] jepsen worker 6 - jepsen.util 7206 :invoke :write [2 8] INFO [2019-12-17 12:31:49,042] jepsen worker 60 - jepsen.util 7710 :invoke :write [20 21] INFO [2019-12-17 12:31:49,048] jepsen worker 65 - jepsen.util 4565 :fail :cas [21 [28 40]] :ValueNotSwapFrom INFO [2019-12-17 12:31:49,048] jepsen worker 65 - jepsen.util 4565 :invoke :read [21 nil] INFO [2019-12-17 12:31:49,057] jepsen worker 76 - jepsen.util 7006 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:31:49,057] jepsen worker 77 - jepsen.util 7097 :info :cas [25 [38 8]] :RequestCanceledException INFO [2019-12-17 12:31:49,057] jepsen worker 16 - jepsen.util 8026 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:31:49,057] jepsen worker 76 - jepsen.util 7006 :invoke :write [25 45] INFO [2019-12-17 12:31:49,057] jepsen worker 16 - jepsen.util 8026 :invoke :cas [5 [41 45]] INFO [2019-12-17 12:31:49,058] jepsen worker 77 - jepsen.util 7187 :invoke :read [25 nil] INFO [2019-12-17 12:31:49,063] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29854,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:31:49,063] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29854,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:31:49,064] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29858,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:31:49,064] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29858,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:31:49,073] jepsen worker 3 - jepsen.util 5313 :fail :cas [1 [44 21]] :ValueNotSwapFrom INFO [2019-12-17 12:31:49,073] jepsen worker 3 - jepsen.util 5313 :invoke :cas [1 [0 37]] INFO [2019-12-17 12:31:49,079] jepsen worker 65 - jepsen.util 4565 :ok :read [21 45] INFO [2019-12-17 12:31:49,079] jepsen worker 65 - jepsen.util 4565 :invoke :read [21 nil] INFO [2019-12-17 12:31:49,092] jepsen worker 64 - jepsen.util 3934 :fail :read [21 nil] :RequestTimeoutException INFO [2019-12-17 12:31:49,092] jepsen worker 82 - jepsen.util 7642 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:31:49,092] jepsen worker 64 - jepsen.util 3934 :invoke :read [21 nil] INFO [2019-12-17 12:31:49,092] jepsen worker 82 - jepsen.util 7642 :invoke :cas [27 [3 16]] INFO [2019-12-17 12:31:49,099] jepsen worker 65 - jepsen.util 4565 :ok :read [21 45] INFO [2019-12-17 12:31:49,099] jepsen worker 65 - jepsen.util 4565 :invoke :cas [21 [29 16]] INFO [2019-12-17 12:31:49,107] jepsen worker 40 - jepsen.util 7600 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:31:49,108] jepsen worker 40 - jepsen.util 7600 :invoke :read [13 nil] INFO [2019-12-17 12:31:49,126] jepsen worker 39 - jepsen.util 6609 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:31:49,126] jepsen worker 75 - jepsen.util 6735 :info :write [25 48] :RequestCanceledException INFO [2019-12-17 12:31:49,126] jepsen worker 39 - jepsen.util 6609 :invoke :write [13 30] INFO [2019-12-17 12:31:49,126] jepsen worker 22 - jepsen.util 7042 :info :write [7 15] :RequestCanceledException INFO [2019-12-17 12:31:49,126] jepsen worker 9 - jepsen.util 5769 :info :write [3 39] :RequestCanceledException INFO [2019-12-17 12:31:49,126] jepsen worker 41 - jepsen.util 7511 :info :cas [13 [15 19]] :RequestCanceledException INFO [2019-12-17 12:31:49,129] jepsen worker 9 - jepsen.util 5859 :invoke :write [3 17] INFO [2019-12-17 12:31:49,129] jepsen worker 22 - jepsen.util 7132 :invoke :read [7 nil] INFO [2019-12-17 12:31:49,129] jepsen worker 75 - jepsen.util 6825 :invoke :read [25 nil] INFO [2019-12-17 12:31:49,131] jepsen worker 41 - jepsen.util 7601 :invoke :write [13 2] INFO [2019-12-17 12:31:49,136] jepsen worker 65 - jepsen.util 4565 :fail :cas [21 [29 16]] :ValueNotSwapFrom INFO [2019-12-17 12:31:49,136] jepsen worker 65 - jepsen.util 4565 :invoke :cas [21 [30 42]] INFO [2019-12-17 12:31:49,140] jepsen worker 57 - jepsen.util 6447 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:31:49,140] jepsen worker 57 - jepsen.util 6447 :invoke :write [19 14] INFO [2019-12-17 12:31:49,143] jepsen worker 64 - jepsen.util 3934 :ok :read [21 45] INFO [2019-12-17 12:31:49,143] jepsen worker 64 - jepsen.util 3934 :invoke :read [21 nil] INFO [2019-12-17 12:31:49,156] jepsen worker 85 - jepsen.util 7735 :info :cas [28 [11 10]] :RequestCanceledException INFO [2019-12-17 12:31:49,159] jepsen worker 11 - jepsen.util 5321 :ok :write [3 15] INFO [2019-12-17 12:31:49,160] jepsen worker 11 - jepsen.util 5321 :invoke :cas [3 [7 29]] INFO [2019-12-17 12:31:49,160] jepsen worker 72 - jepsen.util 5022 :ok :write [24 13] INFO [2019-12-17 12:31:49,160] jepsen worker 72 - jepsen.util 5022 :invoke :read [24 nil] INFO [2019-12-17 12:31:49,162] jepsen worker 85 - jepsen.util 7825 :invoke :cas [28 [17 1]] INFO [2019-12-17 12:31:49,166] jepsen worker 65 - jepsen.util 4565 :fail :cas [21 [30 42]] :ValueNotSwapFrom INFO [2019-12-17 12:31:49,167] jepsen worker 65 - jepsen.util 4565 :invoke :cas [21 [27 40]] INFO [2019-12-17 12:31:49,175] jepsen worker 64 - jepsen.util 3934 :ok :read [21 45] INFO [2019-12-17 12:31:49,175] jepsen worker 64 - jepsen.util 3934 :invoke :write [21 36] INFO [2019-12-17 12:31:49,192] jepsen worker 46 - jepsen.util 4996 :info :cas [15 [6 0]] :RequestCanceledException INFO [2019-12-17 12:31:49,192] jepsen worker 66 - jepsen.util 6456 :info :cas [22 [49 33]] :RequestCanceledException INFO [2019-12-17 12:31:49,194] jepsen worker 66 - jepsen.util 6546 :invoke :read [22 nil] INFO [2019-12-17 12:31:49,194] jepsen worker 46 - jepsen.util 5086 :invoke :cas [15 [13 8]] INFO [2019-12-17 12:31:49,210] jepsen worker 43 - jepsen.util 6613 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:31:49,210] jepsen worker 71 - jepsen.util 5831 :info :write [23 49] :RequestCanceledException INFO [2019-12-17 12:31:49,210] jepsen worker 43 - jepsen.util 6613 :invoke :read [14 nil] INFO [2019-12-17 12:31:49,212] jepsen worker 71 - jepsen.util 5921 :invoke :read [23 nil] INFO [2019-12-17 12:31:49,225] jepsen worker 83 - jepsen.util 7283 :info :write [27 6] :RequestCanceledException INFO [2019-12-17 12:31:49,227] jepsen worker 83 - jepsen.util 7373 :invoke :write [27 48] INFO [2019-12-17 12:31:49,251] jepsen worker 72 - jepsen.util 5022 :ok :read [24 13] INFO [2019-12-17 12:31:49,252] jepsen worker 72 - jepsen.util 5022 :invoke :write [24 49] INFO [2019-12-17 12:31:49,262] jepsen worker 70 - jepsen.util 5920 :fail :cas [23 [11 2]] :ValueNotSwapFrom INFO [2019-12-17 12:31:49,262] jepsen worker 70 - jepsen.util 5920 :invoke :write [23 2] INFO [2019-12-17 12:31:49,273] jepsen worker 55 - jepsen.util 6175 :ok :write [18 41] INFO [2019-12-17 12:31:49,273] jepsen worker 55 - jepsen.util 6175 :invoke :write [18 30] INFO [2019-12-17 12:31:49,278] jepsen worker 88 - jepsen.util 5488 :fail :cas [29 [15 35]] :ValueNotSwapFrom INFO [2019-12-17 12:31:49,278] jepsen worker 88 - jepsen.util 5488 :invoke :write [29 37] INFO [2019-12-17 12:31:49,282] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29897,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:31:49,283] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29897,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:31:49,283] jepsen worker 65 - jepsen.util 4565 :fail :cas [21 [27 40]] :ValueNotSwapFrom INFO [2019-12-17 12:31:49,283] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29898,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:31:49,283] jepsen worker 65 - jepsen.util 4565 :invoke :read [21 nil] INFO [2019-12-17 12:31:49,283] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29898,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:31:49,283] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29908,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:31:49,283] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29908,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:31:49,299] jepsen worker 80 - jepsen.util 5300 :fail :cas [26 [19 0]] :ValueNotSwapFrom INFO [2019-12-17 12:31:49,300] jepsen worker 80 - jepsen.util 5300 :invoke :write [26 47] INFO [2019-12-17 12:31:49,315] jepsen worker 71 - jepsen.util 5921 :ok :read [23 47] INFO [2019-12-17 12:31:49,315] jepsen worker 71 - jepsen.util 5921 :invoke :write [23 4] INFO [2019-12-17 12:31:49,326] jepsen worker 58 - jepsen.util 7258 :info :write [19 31] :RequestCanceledException INFO [2019-12-17 12:31:49,328] jepsen worker 58 - jepsen.util 7348 :invoke :cas [19 [49 10]] INFO [2019-12-17 12:31:49,331] jepsen worker 65 - jepsen.util 4565 :ok :read [21 45] INFO [2019-12-17 12:31:49,331] jepsen worker 65 - jepsen.util 4565 :invoke :cas [21 [36 26]] INFO [2019-12-17 12:31:49,336] jepsen worker 69 - jepsen.util 5649 :fail :cas [23 [23 8]] :ValueNotSwapFrom INFO [2019-12-17 12:31:49,337] jepsen worker 69 - jepsen.util 5649 :invoke :write [23 28] INFO [2019-12-17 12:31:49,340] jepsen worker 27 - jepsen.util 4977 :info :write [9 7] :RequestCanceledException INFO [2019-12-17 12:31:49,340] jepsen worker 30 - jepsen.util 5250 :info :write [10 11] :RequestCanceledException INFO [2019-12-17 12:31:49,340] jepsen worker 2 - jepsen.util 6482 :info :write [0 7] :RequestCanceledException INFO [2019-12-17 12:31:49,341] jepsen worker 30 - jepsen.util 5340 :invoke :write [10 15] INFO [2019-12-17 12:31:49,342] jepsen worker 2 - jepsen.util 6572 :invoke :write [0 24] INFO [2019-12-17 12:31:49,342] jepsen worker 27 - jepsen.util 5067 :invoke :read [9 nil] INFO [2019-12-17 12:31:49,356] jepsen worker 34 - jepsen.util 6424 :info :write [11 24] :RequestCanceledException INFO [2019-12-17 12:31:49,358] jepsen worker 34 - jepsen.util 6514 :invoke :cas [11 [5 34]] INFO [2019-12-17 12:31:49,360] jepsen worker 64 - jepsen.util 3934 :ok :write [21 36] INFO [2019-12-17 12:31:49,360] jepsen worker 64 - jepsen.util 3934 :invoke :read [21 nil] INFO [2019-12-17 12:31:49,391] jepsen worker 81 - jepsen.util 6831 :info :write [27 29] :RequestCanceledException INFO [2019-12-17 12:31:49,392] jepsen worker 81 - jepsen.util 6921 :invoke :write [27 22] INFO [2019-12-17 12:31:49,410] jepsen worker 44 - jepsen.util 7604 :info :write [14 9] :RequestCanceledException INFO [2019-12-17 12:31:49,411] jepsen worker 44 - jepsen.util 7694 :invoke :read [14 nil] INFO [2019-12-17 12:31:49,420] jepsen worker 3 - jepsen.util 5313 :fail :cas [1 [0 37]] :ValueNotSwapFrom INFO [2019-12-17 12:31:49,420] jepsen worker 3 - jepsen.util 5313 :invoke :read [1 nil] INFO [2019-12-17 12:31:49,425] jepsen worker 25 - jepsen.util 7225 :info :write [8 48] :RequestCanceledException INFO [2019-12-17 12:31:49,426] jepsen worker 86 - jepsen.util 7286 :info :write [28 28] :RequestCanceledException INFO [2019-12-17 12:31:49,426] jepsen worker 37 - jepsen.util 4807 :info :write [12 20] :RequestCanceledException INFO [2019-12-17 12:31:49,427] jepsen worker 25 - jepsen.util 7315 :invoke :cas [8 [16 42]] INFO [2019-12-17 12:31:49,427] jepsen worker 37 - jepsen.util 4897 :invoke :write [12 2] INFO [2019-12-17 12:31:49,427] jepsen worker 86 - jepsen.util 7376 :invoke :read [28 nil] INFO [2019-12-17 12:31:49,431] jepsen worker 65 - jepsen.util 4565 :ok :cas [21 [36 26]] INFO [2019-12-17 12:31:49,431] jepsen worker 65 - jepsen.util 4565 :invoke :write [21 24] INFO [2019-12-17 12:31:49,440] jepsen worker 0 - jepsen.util 7560 :info :cas [0 [39 22]] :RequestCanceledException INFO [2019-12-17 12:31:49,440] jepsen worker 10 - jepsen.util 5050 :info :cas [3 [5 5]] :RequestCanceledException INFO [2019-12-17 12:31:49,441] jepsen worker 0 - jepsen.util 7650 :invoke :read [0 nil] INFO [2019-12-17 12:31:49,441] jepsen worker 10 - jepsen.util 5140 :invoke :write [3 2] INFO [2019-12-17 12:31:49,450] jepsen worker 14 - jepsen.util 5324 :fail :cas [4 [31 36]] :ValueNotSwapFrom INFO [2019-12-17 12:31:49,450] jepsen worker 14 - jepsen.util 5324 :invoke :cas [4 [11 41]] INFO [2019-12-17 12:31:49,461] jepsen worker 89 - jepsen.util 5039 :ok :read [29 14] INFO [2019-12-17 12:31:49,461] jepsen worker 89 - jepsen.util 5039 :invoke :write [29 45] INFO [2019-12-17 12:31:49,483] jepsen worker 3 - jepsen.util 5313 :ok :read [1 23] INFO [2019-12-17 12:31:49,484] jepsen worker 3 - jepsen.util 5313 :invoke :cas [1 [26 17]] INFO [2019-12-17 12:31:49,484] jepsen worker 65 - jepsen.util 4565 :ok :write [21 24] INFO [2019-12-17 12:31:49,484] jepsen worker 65 - jepsen.util 4565 :invoke :write [21 0] INFO [2019-12-17 12:31:49,492] jepsen worker 61 - jepsen.util 7801 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:31:49,492] jepsen worker 61 - jepsen.util 7801 :invoke :write [20 38] INFO [2019-12-17 12:31:49,512] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29938,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:31:49,512] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29938,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:31:49,512] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29939,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:31:49,512] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29939,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:31:49,523] jepsen worker 65 - jepsen.util 4565 :ok :write [21 0] INFO [2019-12-17 12:31:49,523] jepsen worker 65 - jepsen.util 4565 :invoke :cas [21 [23 43]] INFO [2019-12-17 12:31:49,544] jepsen worker 65 - jepsen.util 4565 :fail :cas [21 [23 43]] :ValueNotSwapFrom INFO [2019-12-17 12:31:49,544] jepsen worker 65 - jepsen.util 4565 :invoke :read [21 nil] INFO [2019-12-17 12:31:49,555] jepsen worker 65 - jepsen.util 4565 :ok :read [21 0] INFO [2019-12-17 12:31:49,555] jepsen worker 65 - jepsen.util 4565 :invoke :write [21 3] INFO [2019-12-17 12:31:49,589] jepsen worker 65 - jepsen.util 4565 :ok :write [21 3] INFO [2019-12-17 12:31:49,589] jepsen worker 65 - jepsen.util 4565 :invoke :read [21 nil] INFO [2019-12-17 12:31:49,602] jepsen worker 65 - jepsen.util 4565 :ok :read [21 3] INFO [2019-12-17 12:31:49,602] jepsen worker 65 - jepsen.util 4565 :invoke :read [21 nil] INFO [2019-12-17 12:31:49,608] jepsen worker 62 - jepsen.util 6542 :info :cas [20 [21 27]] :RequestCanceledException INFO [2019-12-17 12:31:49,609] jepsen worker 62 - jepsen.util 6632 :invoke :cas [20 [38 7]] INFO [2019-12-17 12:31:49,614] jepsen worker 65 - jepsen.util 4565 :ok :read [21 3] INFO [2019-12-17 12:31:49,614] jepsen worker 65 - jepsen.util 4565 :invoke :read [21 nil] INFO [2019-12-17 12:31:49,626] jepsen worker 67 - jepsen.util 7087 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:31:49,626] jepsen worker 67 - jepsen.util 7087 :invoke :write [22 37] INFO [2019-12-17 12:31:49,629] jepsen worker 65 - jepsen.util 4565 :ok :read [21 3] INFO [2019-12-17 12:31:49,629] jepsen worker 65 - jepsen.util 4565 :invoke :write [21 18] INFO [2019-12-17 12:31:49,652] jepsen worker 30 - jepsen.util 5340 :ok :write [10 15] INFO [2019-12-17 12:31:49,652] jepsen worker 9 - jepsen.util 5859 :ok :write [3 17] INFO [2019-12-17 12:31:49,652] jepsen worker 30 - jepsen.util 5340 :invoke :read [10 nil] INFO [2019-12-17 12:31:49,652] jepsen worker 9 - jepsen.util 5859 :invoke :read [3 nil] INFO [2019-12-17 12:31:49,655] jepsen worker 72 - jepsen.util 5022 :ok :write [24 49] INFO [2019-12-17 12:31:49,655] jepsen worker 72 - jepsen.util 5022 :invoke :cas [24 [1 30]] INFO [2019-12-17 12:31:49,679] jepsen worker 27 - jepsen.util 5067 :ok :read [9 43] INFO [2019-12-17 12:31:49,679] jepsen worker 27 - jepsen.util 5067 :invoke :cas [9 [8 38]] INFO [2019-12-17 12:31:49,683] jepsen worker 30 - jepsen.util 5340 :ok :read [10 15] INFO [2019-12-17 12:31:49,683] jepsen worker 30 - jepsen.util 5340 :invoke :cas [10 [42 39]] INFO [2019-12-17 12:31:49,689] jepsen worker 72 - jepsen.util 5022 :fail :cas [24 [1 30]] :ValueNotSwapFrom INFO [2019-12-17 12:31:49,689] jepsen worker 72 - jepsen.util 5022 :invoke :cas [24 [21 24]] INFO [2019-12-17 12:31:49,696] jepsen worker 9 - jepsen.util 5859 :ok :read [3 17] INFO [2019-12-17 12:31:49,696] jepsen worker 9 - jepsen.util 5859 :invoke :read [3 nil] INFO [2019-12-17 12:31:49,712] jepsen worker 27 - jepsen.util 5067 :fail :cas [9 [8 38]] :ValueNotSwapFrom INFO [2019-12-17 12:31:49,712] jepsen worker 27 - jepsen.util 5067 :invoke :cas [9 [21 0]] INFO [2019-12-17 12:31:49,716] jepsen worker 30 - jepsen.util 5340 :fail :cas [10 [42 39]] :ValueNotSwapFrom INFO [2019-12-17 12:31:49,716] jepsen worker 30 - jepsen.util 5340 :invoke :cas [10 [26 31]] INFO [2019-12-17 12:31:49,718] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29959,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:31:49,718] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29959,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:31:49,718] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29962,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:31:49,718] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29962,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:31:49,726] jepsen worker 51 - jepsen.util 5091 :info :write [17 35] :RequestCanceledException INFO [2019-12-17 12:31:49,728] jepsen worker 51 - jepsen.util 5181 :invoke :cas [17 [27 2]] INFO [2019-12-17 12:31:49,733] jepsen worker 72 - jepsen.util 5022 :fail :cas [24 [21 24]] :ValueNotSwapFrom INFO [2019-12-17 12:31:49,733] jepsen worker 72 - jepsen.util 5022 :invoke :write [24 27] INFO [2019-12-17 12:31:49,733] jepsen worker 9 - jepsen.util 5859 :ok :read [3 17] INFO [2019-12-17 12:31:49,733] jepsen worker 9 - jepsen.util 5859 :invoke :read [3 nil] INFO [2019-12-17 12:31:49,737] jepsen worker 65 - jepsen.util 4565 :ok :write [21 18] INFO [2019-12-17 12:31:49,737] jepsen worker 65 - jepsen.util 4565 :invoke :read [21 nil] INFO [2019-12-17 12:31:49,738] jepsen worker 26 - jepsen.util 7316 :info :cas [8 [2 20]] :RequestCanceledException INFO [2019-12-17 12:31:49,739] jepsen worker 26 - jepsen.util 7406 :invoke :cas [8 [5 34]] INFO [2019-12-17 12:31:49,743] jepsen worker 30 - jepsen.util 5340 :fail :cas [10 [26 31]] :ValueNotSwapFrom INFO [2019-12-17 12:31:49,743] jepsen worker 30 - jepsen.util 5340 :invoke :write [10 32] INFO [2019-12-17 12:31:49,753] jepsen worker 27 - jepsen.util 5067 :fail :cas [9 [21 0]] :ValueNotSwapFrom INFO [2019-12-17 12:31:49,753] jepsen worker 27 - jepsen.util 5067 :invoke :cas [9 [49 40]] INFO [2019-12-17 12:31:49,757] jepsen worker 21 - jepsen.util 6411 :info :write [7 26] :RequestCanceledException INFO [2019-12-17 12:31:49,758] jepsen worker 21 - jepsen.util 6501 :invoke :write [7 36] INFO [2019-12-17 12:31:49,790] jepsen worker 54 - jepsen.util 5544 :info :write [18 33] :RequestCanceledException INFO [2019-12-17 12:31:49,791] jepsen worker 54 - jepsen.util 5634 :invoke :cas [18 [9 10]] INFO [2019-12-17 12:31:49,793] jepsen worker 52 - jepsen.util 5542 :fail :cas [17 [37 31]] :ValueNotSwapFrom INFO [2019-12-17 12:31:49,793] jepsen worker 52 - jepsen.util 5542 :invoke :read [17 nil] INFO [2019-12-17 12:31:49,805] jepsen worker 9 - jepsen.util 5859 :ok :read [3 17] INFO [2019-12-17 12:31:49,805] jepsen worker 9 - jepsen.util 5859 :invoke :cas [3 [2 14]] INFO [2019-12-17 12:31:49,810] jepsen worker 19 - jepsen.util 5149 :fail :read [6 nil] :RequestTimeoutException INFO [2019-12-17 12:31:49,810] jepsen worker 12 - jepsen.util 5412 :info :write [4 47] :RequestCanceledException INFO [2019-12-17 12:31:49,810] jepsen worker 20 - jepsen.util 5240 :info :write [6 36] :RequestCanceledException INFO [2019-12-17 12:31:49,811] jepsen worker 19 - jepsen.util 5149 :invoke :cas [6 [24 5]] INFO [2019-12-17 12:31:49,811] jepsen worker 12 - jepsen.util 5502 :invoke :read [4 nil] INFO [2019-12-17 12:31:49,812] jepsen worker 20 - jepsen.util 5330 :invoke :cas [6 [46 1]] INFO [2019-12-17 12:31:49,839] jepsen worker 1 - jepsen.util 6751 :info :cas [0 [5 23]] :RequestCanceledException INFO [2019-12-17 12:31:49,841] jepsen worker 1 - jepsen.util 6841 :invoke :cas [0 [24 27]] INFO [2019-12-17 12:31:49,857] jepsen worker 7 - jepsen.util 7117 :info :write [2 40] :RequestCanceledException INFO [2019-12-17 12:31:49,858] jepsen worker 7 - jepsen.util 7207 :invoke :read [2 nil] INFO [2019-12-17 12:31:49,883] jepsen worker 27 - jepsen.util 5067 :fail :cas [9 [49 40]] :ValueNotSwapFrom INFO [2019-12-17 12:31:49,883] jepsen worker 27 - jepsen.util 5067 :invoke :read [9 nil] INFO [2019-12-17 12:31:49,888] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:31:49,888] jepsen nemesis - couchbase.util Rebalance check count: 225 INFO [2019-12-17 12:31:49,888] jepsen nemesis - couchbase.util Rebalance stuck count: 206 INFO [2019-12-17 12:31:49,890] jepsen worker 79 - jepsen.util 5029 :info :write [26 48] :RequestCanceledException INFO [2019-12-17 12:31:49,891] jepsen worker 79 - jepsen.util 5119 :invoke :write [26 14] INFO [2019-12-17 12:31:49,905] jepsen worker 9 - jepsen.util 5859 :fail :cas [3 [2 14]] :ValueNotSwapFrom INFO [2019-12-17 12:31:49,905] jepsen worker 9 - jepsen.util 5859 :invoke :read [3 nil] INFO [2019-12-17 12:31:49,917] jepsen worker 4 - jepsen.util 4414 :ok :write [1 29] INFO [2019-12-17 12:31:49,917] jepsen worker 4 - jepsen.util 4414 :invoke :cas [1 [17 8]] INFO [2019-12-17 12:31:49,919] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29990,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:31:49,919] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29990,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:31:49,919] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29994,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:31:49,919] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":29994,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:31:49,920] jepsen worker 27 - jepsen.util 5067 :ok :read [9 43] INFO [2019-12-17 12:31:49,920] jepsen worker 27 - jepsen.util 5067 :invoke :cas [9 [45 42]] INFO [2019-12-17 12:31:49,924] jepsen worker 80 - jepsen.util 5300 :ok :write [26 47] INFO [2019-12-17 12:31:49,924] jepsen worker 80 - jepsen.util 5300 :invoke :cas [26 [9 30]] INFO [2019-12-17 12:31:49,926] jepsen worker 35 - jepsen.util 7055 :info :cas [11 [25 10]] :RequestCanceledException INFO [2019-12-17 12:31:49,927] jepsen worker 35 - jepsen.util 7145 :invoke :write [11 6] INFO [2019-12-17 12:31:49,937] jepsen worker 9 - jepsen.util 5859 :ok :read [3 17] INFO [2019-12-17 12:31:49,937] jepsen worker 9 - jepsen.util 5859 :invoke :write [3 30] INFO [2019-12-17 12:31:49,960] jepsen worker 53 - jepsen.util 5093 :ok :write [17 46] INFO [2019-12-17 12:31:49,960] jepsen worker 53 - jepsen.util 5093 :invoke :read [17 nil] INFO [2019-12-17 12:31:49,962] jepsen worker 65 - jepsen.util 4565 :ok :read [21 18] INFO [2019-12-17 12:31:49,962] jepsen worker 65 - jepsen.util 4565 :invoke :read [21 nil] INFO [2019-12-17 12:31:49,980] jepsen worker 14 - jepsen.util 5324 :fail :cas [4 [11 41]] :ValueNotSwapFrom INFO [2019-12-17 12:31:49,980] jepsen worker 14 - jepsen.util 5324 :invoke :cas [4 [18 10]] INFO [2019-12-17 12:31:49,992] jepsen worker 50 - jepsen.util 7340 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:31:49,992] jepsen worker 50 - jepsen.util 7340 :invoke :read [16 nil] INFO [2019-12-17 12:31:50,004] jepsen worker 27 - jepsen.util 5067 :fail :cas [9 [45 42]] :ValueNotSwapFrom INFO [2019-12-17 12:31:50,004] jepsen worker 27 - jepsen.util 5067 :invoke :read [9 nil] INFO [2019-12-17 12:31:50,010] jepsen worker 42 - jepsen.util 7602 :info :cas [14 [44 41]] :RequestCanceledException INFO [2019-12-17 12:31:50,013] jepsen worker 42 - jepsen.util 7692 :invoke :cas [14 [48 42]] INFO [2019-12-17 12:31:50,017] jepsen worker 89 - jepsen.util 5039 :ok :write [29 45] INFO [2019-12-17 12:31:50,017] jepsen worker 89 - jepsen.util 5039 :invoke :write [29 6] INFO [2019-12-17 12:31:50,019] jepsen worker 20 - jepsen.util 5330 :fail :cas [6 [46 1]] :ValueNotSwapFrom INFO [2019-12-17 12:31:50,019] jepsen worker 20 - jepsen.util 5330 :invoke :cas [6 [2 41]] INFO [2019-12-17 12:31:50,025] jepsen worker 31 - jepsen.util 4891 :fail :read [10 nil] :RequestTimeoutException INFO [2019-12-17 12:31:50,026] jepsen worker 31 - jepsen.util 4891 :invoke :read [10 nil] INFO [2019-12-17 12:31:50,040] jepsen worker 24 - jepsen.util 6504 :info :cas [8 [23 17]] :RequestCanceledException INFO [2019-12-17 12:31:50,041] jepsen worker 24 - jepsen.util 6594 :invoke :read [8 nil] INFO [2019-12-17 12:31:50,046] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":30010,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:31:50,047] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":30010,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:31:50,058] jepsen worker 49 - jepsen.util 6169 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:31:50,058] jepsen worker 49 - jepsen.util 6169 :invoke :cas [16 [27 30]] INFO [2019-12-17 12:31:50,059] jepsen worker 65 - jepsen.util 4565 :ok :read [21 18] INFO [2019-12-17 12:31:50,059] jepsen worker 65 - jepsen.util 4565 :invoke :cas [21 [2 4]] INFO [2019-12-17 12:31:50,063] jepsen worker 5 - jepsen.util 4775 :ok :write [1 43] INFO [2019-12-17 12:31:50,063] jepsen worker 5 - jepsen.util 4775 :invoke :cas [1 [31 10]] INFO [2019-12-17 12:31:50,086] jepsen worker 36 - jepsen.util 5526 :ok :write [12 13] INFO [2019-12-17 12:31:50,086] jepsen worker 36 - jepsen.util 5526 :invoke :cas [12 [2 36]] INFO [2019-12-17 12:31:50,089] jepsen worker 29 - jepsen.util 4169 :info :write [9 38] :RequestCanceledException INFO [2019-12-17 12:31:50,091] jepsen worker 29 - jepsen.util 4259 :invoke :cas [9 [14 7]] INFO [2019-12-17 12:31:50,096] jepsen worker 20 - jepsen.util 5330 :fail :cas [6 [2 41]] :ValueNotSwapFrom INFO [2019-12-17 12:31:50,096] jepsen worker 20 - jepsen.util 5330 :invoke :cas [6 [14 15]] INFO [2019-12-17 12:31:50,108] jepsen worker 51 - jepsen.util 5181 :fail :cas [17 [27 2]] :ValueNotSwapFrom INFO [2019-12-17 12:31:50,108] jepsen worker 51 - jepsen.util 5181 :invoke :read [17 nil] INFO [2019-12-17 12:31:50,108] jepsen worker 59 - jepsen.util 6899 :info :write [19 2] :RequestCanceledException INFO [2019-12-17 12:31:50,108] jepsen worker 18 - jepsen.util 5058 :info :cas [6 [46 42]] :RequestCanceledException INFO [2019-12-17 12:31:50,109] jepsen worker 59 - jepsen.util 6989 :invoke :read [19 nil] INFO [2019-12-17 12:31:50,109] jepsen worker 18 - jepsen.util 5148 :invoke :read [6 nil] INFO [2019-12-17 12:31:50,122] jepsen worker 31 - jepsen.util 4891 :ok :read [10 15] INFO [2019-12-17 12:31:50,122] jepsen worker 31 - jepsen.util 4891 :invoke :cas [10 [40 22]] INFO [2019-12-17 12:31:50,123] jepsen worker 28 - jepsen.util 3808 :info :cas [9 [11 5]] :RequestCanceledException INFO [2019-12-17 12:31:50,125] jepsen worker 28 - jepsen.util 3898 :invoke :cas [9 [40 32]] INFO [2019-12-17 12:31:50,134] jepsen worker 52 - jepsen.util 5542 :ok :read [17 46] INFO [2019-12-17 12:31:50,134] jepsen worker 52 - jepsen.util 5542 :invoke :write [17 17] INFO [2019-12-17 12:31:50,139] jepsen worker 23 - jepsen.util 7223 :info :cas [7 [14 9]] :RequestCanceledException INFO [2019-12-17 12:31:50,140] jepsen worker 23 - jepsen.util 7313 :invoke :read [7 nil] INFO [2019-12-17 12:31:50,144] jepsen worker 3 - jepsen.util 5313 :fail :cas [1 [26 17]] :ValueNotSwapFrom INFO [2019-12-17 12:31:50,145] jepsen worker 3 - jepsen.util 5313 :invoke :write [1 19] INFO [2019-12-17 12:31:50,153] jepsen worker 65 - jepsen.util 4565 :fail :cas [21 [2 4]] :ValueNotSwapFrom INFO [2019-12-17 12:31:50,153] jepsen worker 65 - jepsen.util 4565 :invoke :read [21 nil] INFO [2019-12-17 12:31:50,157] jepsen worker 4 - jepsen.util 4414 :fail :cas [1 [17 8]] :ValueNotSwapFrom INFO [2019-12-17 12:31:50,157] jepsen worker 4 - jepsen.util 4414 :invoke :write [1 1] INFO [2019-12-17 12:31:50,186] jepsen worker 20 - jepsen.util 5330 :fail :cas [6 [14 15]] :ValueNotSwapFrom INFO [2019-12-17 12:31:50,186] jepsen worker 20 - jepsen.util 5330 :invoke :write [6 24] INFO [2019-12-17 12:31:50,188] jepsen worker 14 - jepsen.util 5324 :fail :cas [4 [18 10]] :ValueNotSwapFrom INFO [2019-12-17 12:31:50,188] jepsen worker 14 - jepsen.util 5324 :invoke :read [4 nil] INFO [2019-12-17 12:31:50,220] jepsen worker 5 - jepsen.util 4775 :fail :cas [1 [31 10]] :ValueNotSwapFrom INFO [2019-12-17 12:31:50,220] jepsen worker 5 - jepsen.util 4775 :invoke :read [1 nil] INFO [2019-12-17 12:31:50,228] jepsen worker 51 - jepsen.util 5181 :ok :read [17 46] INFO [2019-12-17 12:31:50,229] jepsen worker 51 - jepsen.util 5181 :invoke :read [17 nil] INFO [2019-12-17 12:31:50,230] jepsen worker 31 - jepsen.util 4891 :fail :cas [10 [40 22]] :ValueNotSwapFrom INFO [2019-12-17 12:31:50,230] jepsen worker 31 - jepsen.util 4891 :invoke :write [10 49] INFO [2019-12-17 12:31:50,257] jepsen worker 87 - jepsen.util 4677 :info :write [29 23] :RequestCanceledException INFO [2019-12-17 12:31:50,257] jepsen worker 56 - jepsen.util 5366 :info :write [18 12] :RequestCanceledException INFO [2019-12-17 12:31:50,258] jepsen worker 56 - jepsen.util 5456 :invoke :cas [18 [40 35]] INFO [2019-12-17 12:31:50,258] jepsen worker 87 - jepsen.util 4767 :invoke :cas [29 [0 18]] INFO [2019-12-17 12:31:50,263] jepsen worker 37 - jepsen.util 4897 :ok :write [12 2] INFO [2019-12-17 12:31:50,263] jepsen worker 37 - jepsen.util 4897 :invoke :cas [12 [5 43]] INFO [2019-12-17 12:31:50,266] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":30037,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:31:50,266] jepsen worker 54 - jepsen.util 5634 :fail :cas [18 [9 10]] :ValueNotSwapFrom INFO [2019-12-17 12:31:50,266] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":30044,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:31:50,266] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":30037,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:31:50,266] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":30044,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:31:50,266] jepsen worker 54 - jepsen.util 5634 :invoke :read [18 nil] INFO [2019-12-17 12:31:50,266] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":30038,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:31:50,266] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":30048,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:31:50,266] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":30038,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:31:50,266] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":30048,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:31:50,266] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":30039,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:31:50,266] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":30039,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:31:50,267] jepsen worker 28 - jepsen.util 3898 :fail :cas [9 [40 32]] :ValueNotSwapFrom INFO [2019-12-17 12:31:50,267] jepsen worker 28 - jepsen.util 3898 :invoke :write [9 39] INFO [2019-12-17 12:31:50,268] jepsen worker 65 - jepsen.util 4565 :ok :read [21 18] INFO [2019-12-17 12:31:50,268] jepsen worker 65 - jepsen.util 4565 :invoke :cas [21 [17 22]] INFO [2019-12-17 12:31:50,285] jepsen worker 52 - jepsen.util 5542 :ok :write [17 17] INFO [2019-12-17 12:31:50,285] jepsen worker 52 - jepsen.util 5542 :invoke :read [17 nil] INFO [2019-12-17 12:31:50,292] jepsen worker 47 - jepsen.util 5807 :info :cas [15 [32 46]] :RequestCanceledException INFO [2019-12-17 12:31:50,292] jepsen worker 32 - jepsen.util 5342 :fail :read [10 nil] :RequestTimeoutException INFO [2019-12-17 12:31:50,292] jepsen worker 32 - jepsen.util 5342 :invoke :write [10 8] INFO [2019-12-17 12:31:50,293] jepsen worker 47 - jepsen.util 5897 :invoke :cas [15 [17 16]] INFO [2019-12-17 12:31:50,306] jepsen worker 53 - jepsen.util 5093 :ok :read [17 46] INFO [2019-12-17 12:31:50,306] jepsen worker 53 - jepsen.util 5093 :invoke :write [17 23] INFO [2019-12-17 12:31:50,326] jepsen worker 5 - jepsen.util 4775 :ok :read [1 43] INFO [2019-12-17 12:31:50,326] jepsen worker 5 - jepsen.util 4775 :invoke :cas [1 [28 24]] INFO [2019-12-17 12:31:50,336] jepsen worker 51 - jepsen.util 5181 :ok :read [17 17] INFO [2019-12-17 12:31:50,336] jepsen worker 51 - jepsen.util 5181 :invoke :write [17 21] INFO [2019-12-17 12:31:50,361] jepsen worker 28 - jepsen.util 3898 :ok :write [9 39] INFO [2019-12-17 12:31:50,361] jepsen worker 28 - jepsen.util 3898 :invoke :read [9 nil] INFO [2019-12-17 12:31:50,380] jepsen worker 28 - jepsen.util 3898 :ok :read [9 39] INFO [2019-12-17 12:31:50,380] jepsen worker 28 - jepsen.util 3898 :invoke :write [9 34] INFO [2019-12-17 12:31:50,393] jepsen worker 87 - jepsen.util 4767 :fail :cas [29 [0 18]] :ValueNotSwapFrom INFO [2019-12-17 12:31:50,393] jepsen worker 87 - jepsen.util 4767 :invoke :cas [29 [1 25]] INFO [2019-12-17 12:31:50,424] jepsen worker 28 - jepsen.util 3898 :ok :write [9 34] INFO [2019-12-17 12:31:50,424] jepsen worker 28 - jepsen.util 3898 :invoke :cas [9 [45 43]] INFO [2019-12-17 12:31:50,425] jepsen worker 13 - jepsen.util 5053 :info :write [4 35] :RequestCanceledException INFO [2019-12-17 12:31:50,426] jepsen worker 13 - jepsen.util 5143 :invoke :write [4 44] INFO [2019-12-17 12:31:50,427] jepsen worker 52 - jepsen.util 5542 :ok :read [17 17] INFO [2019-12-17 12:31:50,427] jepsen worker 52 - jepsen.util 5542 :invoke :read [17 nil] INFO [2019-12-17 12:31:50,438] jepsen worker 28 - jepsen.util 3898 :fail :cas [9 [45 43]] :ValueNotSwapFrom INFO [2019-12-17 12:31:50,439] jepsen worker 28 - jepsen.util 3898 :invoke :cas [9 [4 39]] INFO [2019-12-17 12:31:50,453] jepsen worker 14 - jepsen.util 5324 :ok :read [4 47] INFO [2019-12-17 12:31:50,453] jepsen worker 14 - jepsen.util 5324 :invoke :cas [4 [29 19]] INFO [2019-12-17 12:31:50,454] jepsen worker 28 - jepsen.util 3898 :fail :cas [9 [4 39]] :ValueNotSwapFrom INFO [2019-12-17 12:31:50,454] jepsen worker 28 - jepsen.util 3898 :invoke :cas [9 [2 40]] INFO [2019-12-17 12:31:50,477] jepsen worker 28 - jepsen.util 3898 :fail :cas [9 [2 40]] :ValueNotSwapFrom INFO [2019-12-17 12:31:50,478] jepsen worker 28 - jepsen.util 3898 :invoke :cas [9 [35 25]] INFO [2019-12-17 12:31:50,479] jepsen worker 88 - jepsen.util 5488 :ok :write [29 37] INFO [2019-12-17 12:31:50,479] jepsen worker 88 - jepsen.util 5488 :invoke :cas [29 [29 9]] INFO [2019-12-17 12:31:50,491] jepsen worker 28 - jepsen.util 3898 :fail :cas [9 [35 25]] :ValueNotSwapFrom INFO [2019-12-17 12:31:50,491] jepsen worker 28 - jepsen.util 3898 :invoke :write [9 6] INFO [2019-12-17 12:31:50,497] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":30070,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:31:50,497] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":30070,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:31:50,497] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":30074,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:31:50,497] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":30074,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:31:50,497] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":30075,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:31:50,497] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":30075,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:31:50,499] jepsen worker 80 - jepsen.util 5300 :fail :cas [26 [9 30]] :ValueNotSwapFrom INFO [2019-12-17 12:31:50,499] jepsen worker 80 - jepsen.util 5300 :invoke :read [26 nil] INFO [2019-12-17 12:31:50,523] jepsen worker 37 - jepsen.util 4897 :fail :cas [12 [5 43]] :ValueNotSwapFrom INFO [2019-12-17 12:31:50,523] jepsen worker 37 - jepsen.util 4897 :invoke :write [12 29] INFO [2019-12-17 12:31:50,525] jepsen worker 45 - jepsen.util 5265 :info :cas [15 [49 44]] :RequestCanceledException INFO [2019-12-17 12:31:50,526] jepsen worker 38 - jepsen.util 5168 :info :write [12 23] :RequestCanceledException INFO [2019-12-17 12:31:50,526] jepsen worker 45 - jepsen.util 5355 :invoke :write [15 49] INFO [2019-12-17 12:31:50,526] jepsen worker 38 - jepsen.util 5258 :invoke :read [12 nil] INFO [2019-12-17 12:31:50,548] jepsen worker 28 - jepsen.util 3898 :ok :write [9 6] INFO [2019-12-17 12:31:50,549] jepsen worker 28 - jepsen.util 3898 :invoke :cas [9 [10 43]] INFO [2019-12-17 12:31:50,557] jepsen worker 3 - jepsen.util 5313 :ok :write [1 19] INFO [2019-12-17 12:31:50,557] jepsen worker 3 - jepsen.util 5313 :invoke :write [1 26] INFO [2019-12-17 12:31:50,564] jepsen worker 88 - jepsen.util 5488 :fail :cas [29 [29 9]] :ValueNotSwapFrom INFO [2019-12-17 12:31:50,564] jepsen worker 88 - jepsen.util 5488 :invoke :write [29 13] INFO [2019-12-17 12:31:50,593] jepsen worker 63 - jepsen.util 4203 :info :write [21 14] :RequestCanceledException INFO [2019-12-17 12:31:50,594] jepsen worker 63 - jepsen.util 4293 :invoke :write [21 25] INFO [2019-12-17 12:31:50,635] jepsen worker 70 - jepsen.util 5920 :ok :write [23 2] INFO [2019-12-17 12:31:50,635] jepsen worker 70 - jepsen.util 5920 :invoke :cas [23 [36 9]] INFO [2019-12-17 12:31:50,635] jepsen worker 51 - jepsen.util 5181 :ok :write [17 21] INFO [2019-12-17 12:31:50,635] jepsen worker 51 - jepsen.util 5181 :invoke :write [17 2] INFO [2019-12-17 12:31:50,661] jepsen worker 89 - jepsen.util 5039 :ok :write [29 6] INFO [2019-12-17 12:31:50,661] jepsen worker 89 - jepsen.util 5039 :invoke :cas [29 [27 43]] INFO [2019-12-17 12:31:50,666] jepsen worker 80 - jepsen.util 5300 :ok :read [26 47] INFO [2019-12-17 12:31:50,666] jepsen worker 80 - jepsen.util 5300 :invoke :read [26 nil] INFO [2019-12-17 12:31:50,675] jepsen worker 14 - jepsen.util 5324 :fail :cas [4 [29 19]] :ValueNotSwapFrom INFO [2019-12-17 12:31:50,676] jepsen worker 14 - jepsen.util 5324 :invoke :write [4 27] INFO [2019-12-17 12:31:50,680] jepsen worker 72 - jepsen.util 5022 :ok :write [24 27] INFO [2019-12-17 12:31:50,680] jepsen worker 72 - jepsen.util 5022 :invoke :cas [24 [43 17]] INFO [2019-12-17 12:31:50,680] jepsen worker 9 - jepsen.util 5859 :ok :write [3 30] INFO [2019-12-17 12:31:50,681] jepsen worker 9 - jepsen.util 5859 :invoke :read [3 nil] INFO [2019-12-17 12:31:50,681] jepsen worker 30 - jepsen.util 5340 :ok :write [10 32] INFO [2019-12-17 12:31:50,681] jepsen worker 30 - jepsen.util 5340 :invoke :write [10 2] INFO [2019-12-17 12:31:50,704] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":30093,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:31:50,704] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":30093,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:31:50,704] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":30097,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:31:50,704] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":30097,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:31:50,712] jepsen worker 5 - jepsen.util 4775 :fail :cas [1 [28 24]] :ValueNotSwapFrom INFO [2019-12-17 12:31:50,712] jepsen worker 5 - jepsen.util 4775 :invoke :write [1 19] INFO [2019-12-17 12:31:50,724] jepsen worker 87 - jepsen.util 4767 :fail :cas [29 [1 25]] :ValueNotSwapFrom INFO [2019-12-17 12:31:50,724] jepsen worker 87 - jepsen.util 4767 :invoke :cas [29 [38 48]] INFO [2019-12-17 12:31:50,728] jepsen worker 53 - jepsen.util 5093 :ok :write [17 23] INFO [2019-12-17 12:31:50,728] jepsen worker 53 - jepsen.util 5093 :invoke :read [17 nil] INFO [2019-12-17 12:31:50,736] jepsen worker 19 - jepsen.util 5149 :fail :cas [6 [24 5]] :ValueNotSwapFrom INFO [2019-12-17 12:31:50,736] jepsen worker 19 - jepsen.util 5149 :invoke :cas [6 [1 44]] INFO [2019-12-17 12:31:50,775] jepsen worker 52 - jepsen.util 5542 :ok :read [17 21] INFO [2019-12-17 12:31:50,775] jepsen worker 52 - jepsen.util 5542 :invoke :write [17 45] INFO [2019-12-17 12:31:50,801] jepsen worker 54 - jepsen.util 5634 :ok :read [18 12] INFO [2019-12-17 12:31:50,801] jepsen worker 54 - jepsen.util 5634 :invoke :write [18 30] INFO [2019-12-17 12:31:50,819] jepsen worker 18 - jepsen.util 5148 :ok :read [6 12] INFO [2019-12-17 12:31:50,819] jepsen worker 18 - jepsen.util 5148 :invoke :write [6 34] INFO [2019-12-17 12:31:50,834] jepsen worker 12 - jepsen.util 5502 :ok :read [4 47] INFO [2019-12-17 12:31:50,834] jepsen worker 12 - jepsen.util 5502 :invoke :read [4 nil] INFO [2019-12-17 12:31:50,841] jepsen worker 36 - jepsen.util 5526 :fail :cas [12 [2 36]] :CASMismatchException INFO [2019-12-17 12:31:50,842] jepsen worker 36 - jepsen.util 5526 :invoke :read [12 nil] INFO [2019-12-17 12:31:50,847] jepsen worker 47 - jepsen.util 5897 :fail :cas [15 [17 16]] :ValueNotSwapFrom INFO [2019-12-17 12:31:50,847] jepsen worker 47 - jepsen.util 5897 :invoke :read [15 nil] INFO [2019-12-17 12:31:50,848] jepsen worker 29 - jepsen.util 4259 :fail :cas [9 [14 7]] :ValueNotSwapFrom INFO [2019-12-17 12:31:50,848] jepsen worker 29 - jepsen.util 4259 :invoke :read [9 nil] INFO [2019-12-17 12:31:50,851] jepsen worker 37 - jepsen.util 4897 :ok :write [12 29] INFO [2019-12-17 12:31:50,851] jepsen worker 37 - jepsen.util 4897 :invoke :cas [12 [5 18]] INFO [2019-12-17 12:31:50,862] jepsen worker 72 - jepsen.util 5022 :fail :cas [24 [43 17]] :ValueNotSwapFrom INFO [2019-12-17 12:31:50,862] jepsen worker 72 - jepsen.util 5022 :invoke :cas [24 [18 37]] INFO [2019-12-17 12:31:50,876] jepsen worker 19 - jepsen.util 5149 :fail :cas [6 [1 44]] :ValueNotSwapFrom INFO [2019-12-17 12:31:50,877] jepsen worker 19 - jepsen.util 5149 :invoke :read [6 nil] INFO [2019-12-17 12:31:50,889] jepsen worker 80 - jepsen.util 5300 :ok :read [26 47] INFO [2019-12-17 12:31:50,890] jepsen worker 80 - jepsen.util 5300 :invoke :read [26 nil] INFO [2019-12-17 12:31:50,898] jepsen worker 4 - jepsen.util 4414 :ok :write [1 1] INFO [2019-12-17 12:31:50,898] jepsen worker 4 - jepsen.util 4414 :invoke :write [1 10] INFO [2019-12-17 12:31:50,903] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":30120,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:31:50,903] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":30120,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:31:50,903] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":30122,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:31:50,903] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":30122,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:31:50,944] jepsen worker 9 - jepsen.util 5859 :ok :read [3 30] INFO [2019-12-17 12:31:50,945] jepsen worker 9 - jepsen.util 5859 :invoke :read [3 nil] INFO [2019-12-17 12:31:50,945] jepsen worker 47 - jepsen.util 5897 :ok :read [15 14] INFO [2019-12-17 12:31:50,946] jepsen worker 47 - jepsen.util 5897 :invoke :read [15 nil] INFO [2019-12-17 12:31:50,951] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:31:50,951] jepsen nemesis - couchbase.util Rebalance check count: 226 INFO [2019-12-17 12:31:50,951] jepsen nemesis - couchbase.util Rebalance stuck count: 207 INFO [2019-12-17 12:31:50,957] jepsen worker 74 - jepsen.util 5564 :info :write [24 48] :RequestCanceledException INFO [2019-12-17 12:31:50,958] jepsen worker 74 - jepsen.util 5654 :invoke :read [24 nil] INFO [2019-12-17 12:31:50,973] jepsen worker 3 - jepsen.util 5313 :ok :write [1 26] INFO [2019-12-17 12:31:50,973] jepsen worker 3 - jepsen.util 5313 :invoke :read [1 nil] INFO [2019-12-17 12:31:51,003] jepsen worker 72 - jepsen.util 5022 :fail :cas [24 [18 37]] :ValueNotSwapFrom INFO [2019-12-17 12:31:51,003] jepsen worker 72 - jepsen.util 5022 :invoke :write [24 31] INFO [2019-12-17 12:31:51,016] jepsen worker 53 - jepsen.util 5093 :ok :read [17 23] INFO [2019-12-17 12:31:51,016] jepsen worker 53 - jepsen.util 5093 :invoke :write [17 27] INFO [2019-12-17 12:31:51,025] jepsen worker 48 - jepsen.util 6438 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:31:51,025] jepsen worker 78 - jepsen.util 5298 :info :write [26 21] :RequestCanceledException INFO [2019-12-17 12:31:51,025] jepsen worker 48 - jepsen.util 6438 :invoke :read [16 nil] INFO [2019-12-17 12:31:51,027] jepsen worker 78 - jepsen.util 5388 :invoke :cas [26 [33 47]] INFO [2019-12-17 12:31:51,027] jepsen worker 38 - jepsen.util 5258 :ok :read [12 29] INFO [2019-12-17 12:31:51,028] jepsen worker 38 - jepsen.util 5258 :invoke :cas [12 [8 7]] INFO [2019-12-17 12:31:51,037] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":30131,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:31:51,037] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":30131,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:31:51,037] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":30135,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:31:51,037] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":30135,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:31:51,037] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":30136,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:31:51,037] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":30136,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:31:51,037] jepsen worker 19 - jepsen.util 5149 :ok :read [6 12] INFO [2019-12-17 12:31:51,037] jepsen worker 19 - jepsen.util 5149 :invoke :cas [6 [12 23]] INFO [2019-12-17 12:31:51,039] jepsen worker 20 - jepsen.util 5330 :ok :write [6 24] INFO [2019-12-17 12:31:51,039] jepsen worker 20 - jepsen.util 5330 :invoke :write [6 23] INFO [2019-12-17 12:31:51,040] jepsen worker 68 - jepsen.util 6548 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:31:51,040] jepsen worker 68 - jepsen.util 6548 :invoke :cas [22 [47 21]] INFO [2019-12-17 12:31:51,056] jepsen worker 14 - jepsen.util 5324 :ok :write [4 27] INFO [2019-12-17 12:31:51,056] jepsen worker 14 - jepsen.util 5324 :invoke :write [4 6] INFO [2019-12-17 12:31:51,057] jepsen worker 8 - jepsen.util 6488 :info :cas [2 [11 26]] :RequestCanceledException INFO [2019-12-17 12:31:51,061] jepsen worker 8 - jepsen.util 6578 :invoke :cas [2 [12 13]] INFO [2019-12-17 12:31:51,081] jepsen worker 88 - jepsen.util 5488 :ok :write [29 13] INFO [2019-12-17 12:31:51,081] jepsen worker 88 - jepsen.util 5488 :invoke :read [29 nil] INFO [2019-12-17 12:31:51,088] jepsen worker 36 - jepsen.util 5526 :ok :read [12 29] INFO [2019-12-17 12:31:51,088] jepsen worker 36 - jepsen.util 5526 :invoke :read [12 nil] INFO [2019-12-17 12:31:51,093] jepsen worker 84 - jepsen.util 6654 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:31:51,093] jepsen worker 15 - jepsen.util 5955 :info :cas [5 [25 47]] :RequestCanceledException INFO [2019-12-17 12:31:51,093] jepsen worker 60 - jepsen.util 7710 :info :write [20 21] :RequestCanceledException INFO [2019-12-17 12:31:51,093] jepsen worker 84 - jepsen.util 6654 :invoke :cas [28 [33 24]] INFO [2019-12-17 12:31:51,094] jepsen worker 60 - jepsen.util 7800 :invoke :write [20 49] INFO [2019-12-17 12:31:51,094] jepsen worker 15 - jepsen.util 6045 :invoke :cas [5 [38 21]] INFO [2019-12-17 12:31:51,098] jepsen worker 12 - jepsen.util 5502 :ok :read [4 27] INFO [2019-12-17 12:31:51,098] jepsen worker 12 - jepsen.util 5502 :invoke :write [4 7] INFO [2019-12-17 12:31:51,110] jepsen worker 73 - jepsen.util 4843 :fail :read [24 nil] :RequestTimeoutException INFO [2019-12-17 12:31:51,110] jepsen worker 17 - jepsen.util 6947 :info :cas [5 [1 8]] :RequestCanceledException INFO [2019-12-17 12:31:51,110] jepsen worker 77 - jepsen.util 7187 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:31:51,110] jepsen worker 73 - jepsen.util 4843 :invoke :write [24 28] INFO [2019-12-17 12:31:51,110] jepsen worker 77 - jepsen.util 7187 :invoke :cas [25 [44 37]] INFO [2019-12-17 12:31:51,111] jepsen worker 17 - jepsen.util 7037 :invoke :write [5 32] INFO [2019-12-17 12:31:51,117] jepsen worker 52 - jepsen.util 5542 :ok :write [17 45] INFO [2019-12-17 12:31:51,117] jepsen worker 52 - jepsen.util 5542 :invoke :write [17 12] INFO [2019-12-17 12:31:51,120] jepsen worker 38 - jepsen.util 5258 :fail :cas [12 [8 7]] :ValueNotSwapFrom INFO [2019-12-17 12:31:51,120] jepsen worker 38 - jepsen.util 5258 :invoke :write [12 43] INFO [2019-12-17 12:31:51,126] jepsen worker 6 - jepsen.util 7206 :info :write [2 8] :RequestCanceledException INFO [2019-12-17 12:31:51,126] jepsen worker 33 - jepsen.util 7053 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:31:51,126] jepsen worker 33 - jepsen.util 7053 :invoke :cas [11 [37 15]] INFO [2019-12-17 12:31:51,127] jepsen worker 6 - jepsen.util 7296 :invoke :cas [2 [48 37]] INFO [2019-12-17 12:31:51,140] jepsen worker 47 - jepsen.util 5897 :ok :read [15 14] INFO [2019-12-17 12:31:51,140] jepsen worker 47 - jepsen.util 5897 :invoke :read [15 nil] INFO [2019-12-17 12:31:51,155] jepsen worker 65 - jepsen.util 4565 :fail :cas [21 [17 22]] :ValueNotSwapFrom INFO [2019-12-17 12:31:51,156] jepsen worker 65 - jepsen.util 4565 :invoke :cas [21 [9 34]] INFO [2019-12-17 12:31:51,157] jepsen worker 76 - jepsen.util 7006 :info :write [25 45] :RequestCanceledException INFO [2019-12-17 12:31:51,157] jepsen worker 16 - jepsen.util 8026 :info :cas [5 [41 45]] :RequestCanceledException INFO [2019-12-17 12:31:51,157] jepsen worker 75 - jepsen.util 6825 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:31:51,157] jepsen worker 75 - jepsen.util 6825 :invoke :read [25 nil] INFO [2019-12-17 12:31:51,158] jepsen worker 76 - jepsen.util 7096 :invoke :write [25 4] INFO [2019-12-17 12:31:51,158] jepsen worker 16 - jepsen.util 8116 :invoke :cas [5 [10 17]] INFO [2019-12-17 12:31:51,160] jepsen worker 45 - jepsen.util 5355 :ok :write [15 49] INFO [2019-12-17 12:31:51,160] jepsen worker 45 - jepsen.util 5355 :invoke :cas [15 [45 19]] INFO [2019-12-17 12:31:51,163] jepsen worker 32 - jepsen.util 5342 :ok :write [10 8] INFO [2019-12-17 12:31:51,163] jepsen worker 32 - jepsen.util 5342 :invoke :cas [10 [11 14]] INFO [2019-12-17 12:31:51,184] jepsen worker 36 - jepsen.util 5526 :ok :read [12 29] INFO [2019-12-17 12:31:51,184] jepsen worker 36 - jepsen.util 5526 :invoke :read [12 nil] INFO [2019-12-17 12:31:51,189] jepsen worker 45 - jepsen.util 5355 :fail :cas [15 [45 19]] :ValueNotSwapFrom INFO [2019-12-17 12:31:51,189] jepsen worker 45 - jepsen.util 5355 :invoke :write [15 11] INFO [2019-12-17 12:31:51,192] jepsen worker 85 - jepsen.util 7825 :info :cas [28 [17 1]] :RequestCanceledException INFO [2019-12-17 12:31:51,192] jepsen worker 82 - jepsen.util 7642 :info :cas [27 [3 16]] :RequestCanceledException INFO [2019-12-17 12:31:51,193] jepsen worker 82 - jepsen.util 7732 :invoke :read [27 nil] INFO [2019-12-17 12:31:51,193] jepsen worker 85 - jepsen.util 7915 :invoke :cas [28 [2 43]] INFO [2019-12-17 12:31:51,201] jepsen worker 63 - jepsen.util 4293 :ok :write [21 25] INFO [2019-12-17 12:31:51,201] jepsen worker 63 - jepsen.util 4293 :invoke :cas [21 [40 16]] INFO [2019-12-17 12:31:51,207] jepsen worker 40 - jepsen.util 7600 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:31:51,207] jepsen worker 66 - jepsen.util 6546 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:31:51,207] jepsen worker 40 - jepsen.util 7600 :invoke :read [13 nil] INFO [2019-12-17 12:31:51,207] jepsen worker 66 - jepsen.util 6546 :invoke :cas [22 [20 25]] INFO [2019-12-17 12:31:51,224] jepsen worker 39 - jepsen.util 6609 :info :write [13 30] :RequestCanceledException INFO [2019-12-17 12:31:51,224] jepsen worker 11 - jepsen.util 5321 :info :cas [3 [7 29]] :RequestCanceledException INFO [2019-12-17 12:31:51,224] jepsen worker 41 - jepsen.util 7601 :info :write [13 2] :RequestCanceledException INFO [2019-12-17 12:31:51,224] jepsen worker 46 - jepsen.util 5086 :info :cas [15 [13 8]] :RequestCanceledException INFO [2019-12-17 12:31:51,225] jepsen worker 11 - jepsen.util 5411 :invoke :cas [3 [0 13]] INFO [2019-12-17 12:31:51,225] jepsen worker 39 - jepsen.util 6699 :invoke :read [13 nil] INFO [2019-12-17 12:31:51,225] jepsen worker 41 - jepsen.util 7691 :invoke :write [13 26] INFO [2019-12-17 12:31:51,225] jepsen worker 46 - jepsen.util 5176 :invoke :cas [15 [14 6]] INFO [2019-12-17 12:31:51,226] jepsen worker 37 - jepsen.util 4897 :fail :cas [12 [5 18]] :ValueNotSwapFrom INFO [2019-12-17 12:31:51,226] jepsen worker 37 - jepsen.util 4897 :invoke :cas [12 [10 29]] INFO [2019-12-17 12:31:51,228] jepsen worker 69 - jepsen.util 5649 :ok :write [23 28] INFO [2019-12-17 12:31:51,229] jepsen worker 69 - jepsen.util 5649 :invoke :write [23 24] INFO [2019-12-17 12:31:51,239] jepsen worker 22 - jepsen.util 7132 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:31:51,239] jepsen worker 57 - jepsen.util 6447 :info :write [19 14] :RequestCanceledException INFO [2019-12-17 12:31:51,239] jepsen worker 22 - jepsen.util 7132 :invoke :write [7 34] INFO [2019-12-17 12:31:51,240] jepsen worker 57 - jepsen.util 6537 :invoke :write [19 0] INFO [2019-12-17 12:31:51,253] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":30169,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:31:51,253] jepsen worker 83 - jepsen.util 7373 :info :write [27 48] :RequestCanceledException INFO [2019-12-17 12:31:51,253] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":30183,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:31:51,254] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":30169,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:31:51,254] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":30183,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:31:51,254] jepsen worker 83 - jepsen.util 7463 :invoke :read [27 nil] INFO [2019-12-17 12:31:51,259] jepsen worker 3 - jepsen.util 5313 :ok :read [1 26] INFO [2019-12-17 12:31:51,260] jepsen worker 3 - jepsen.util 5313 :invoke :read [1 nil] INFO [2019-12-17 12:31:51,274] jepsen worker 46 - jepsen.util 5176 :fail :cas [15 [14 6]] :ValueNotSwapFrom INFO [2019-12-17 12:31:51,274] jepsen worker 46 - jepsen.util 5176 :invoke :write [15 26] INFO [2019-12-17 12:31:51,310] jepsen worker 43 - jepsen.util 6613 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:31:51,310] jepsen worker 43 - jepsen.util 6613 :invoke :write [14 39] INFO [2019-12-17 12:31:51,325] jepsen worker 55 - jepsen.util 6175 :info :write [18 30] :RequestCanceledException INFO [2019-12-17 12:31:51,326] jepsen worker 55 - jepsen.util 6265 :invoke :cas [18 [32 13]] INFO [2019-12-17 12:31:51,339] jepsen worker 71 - jepsen.util 5921 :info :write [23 4] :RequestCanceledException INFO [2019-12-17 12:31:51,341] jepsen worker 3 - jepsen.util 5313 :ok :read [1 26] INFO [2019-12-17 12:31:51,341] jepsen worker 3 - jepsen.util 5313 :invoke :cas [1 [27 25]] INFO [2019-12-17 12:31:51,341] jepsen worker 71 - jepsen.util 6011 :invoke :write [23 28] INFO [2019-12-17 12:31:51,381] jepsen worker 30 - jepsen.util 5340 :ok :write [10 2] INFO [2019-12-17 12:31:51,381] jepsen worker 30 - jepsen.util 5340 :invoke :cas [10 [13 45]] INFO [2019-12-17 12:31:51,385] jepsen worker 64 - jepsen.util 3934 :fail :read [21 nil] :RequestTimeoutException INFO [2019-12-17 12:31:51,385] jepsen worker 64 - jepsen.util 3934 :invoke :cas [21 [45 33]] INFO [2019-12-17 12:31:51,400] jepsen worker 29 - jepsen.util 4259 :ok :read [9 6] INFO [2019-12-17 12:31:51,401] jepsen worker 29 - jepsen.util 4259 :invoke :cas [9 [48 1]] INFO [2019-12-17 12:31:51,406] jepsen worker 74 - jepsen.util 5654 :ok :read [24 27] INFO [2019-12-17 12:31:51,406] jepsen worker 74 - jepsen.util 5654 :invoke :write [24 49] INFO [2019-12-17 12:31:51,411] jepsen worker 2 - jepsen.util 6572 :info :write [0 24] :RequestCanceledException INFO [2019-12-17 12:31:51,411] jepsen worker 2 - jepsen.util 6662 :invoke :read [0 nil] INFO [2019-12-17 12:31:51,426] jepsen worker 44 - jepsen.util 7694 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:31:51,426] jepsen worker 58 - jepsen.util 7348 :info :cas [19 [49 10]] :RequestCanceledException INFO [2019-12-17 12:31:51,426] jepsen worker 44 - jepsen.util 7694 :invoke :read [14 nil] INFO [2019-12-17 12:31:51,428] jepsen worker 58 - jepsen.util 7438 :invoke :write [19 20] INFO [2019-12-17 12:31:51,429] jepsen worker 32 - jepsen.util 5342 :fail :cas [10 [11 14]] :ValueNotSwapFrom INFO [2019-12-17 12:31:51,429] jepsen worker 32 - jepsen.util 5342 :invoke :write [10 41] INFO [2019-12-17 12:31:51,437] jepsen worker 36 - jepsen.util 5526 :ok :read [12 29] INFO [2019-12-17 12:31:51,437] jepsen worker 36 - jepsen.util 5526 :invoke :write [12 25] INFO [2019-12-17 12:31:51,440] jepsen worker 34 - jepsen.util 6514 :info :cas [11 [5 34]] :RequestCanceledException INFO [2019-12-17 12:31:51,441] jepsen worker 34 - jepsen.util 6604 :invoke :read [11 nil] INFO [2019-12-17 12:31:51,442] jepsen worker 30 - jepsen.util 5340 :fail :cas [10 [13 45]] :ValueNotSwapFrom INFO [2019-12-17 12:31:51,442] jepsen worker 30 - jepsen.util 5340 :invoke :read [10 nil] INFO [2019-12-17 12:31:51,451] jepsen worker 64 - jepsen.util 3934 :fail :cas [21 [45 33]] :ValueNotSwapFrom INFO [2019-12-17 12:31:51,451] jepsen worker 64 - jepsen.util 3934 :invoke :write [21 5] INFO [2019-12-17 12:31:51,452] jepsen worker 87 - jepsen.util 4767 :fail :cas [29 [38 48]] :ValueNotSwapFrom INFO [2019-12-17 12:31:51,452] jepsen worker 87 - jepsen.util 4767 :invoke :read [29 nil] INFO [2019-12-17 12:31:51,456] jepsen worker 81 - jepsen.util 6921 :info :write [27 22] :RequestCanceledException INFO [2019-12-17 12:31:51,456] jepsen worker 81 - jepsen.util 7011 :invoke :write [27 49] INFO [2019-12-17 12:31:51,465] jepsen worker 29 - jepsen.util 4259 :fail :cas [9 [48 1]] :ValueNotSwapFrom INFO [2019-12-17 12:31:51,465] jepsen worker 29 - jepsen.util 4259 :invoke :write [9 31] INFO [2019-12-17 12:31:51,477] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":30209,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:31:51,477] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":30209,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:31:51,487] jepsen worker 25 - jepsen.util 7315 :info :cas [8 [16 42]] :RequestCanceledException INFO [2019-12-17 12:31:51,488] jepsen worker 25 - jepsen.util 7405 :invoke :write [8 43] INFO [2019-12-17 12:31:51,490] jepsen worker 80 - jepsen.util 5300 :ok :read [26 47] INFO [2019-12-17 12:31:51,490] jepsen worker 80 - jepsen.util 5300 :invoke :read [26 nil] INFO [2019-12-17 12:31:51,526] jepsen worker 86 - jepsen.util 7376 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:31:51,526] jepsen worker 86 - jepsen.util 7376 :invoke :write [28 18] INFO [2019-12-17 12:31:51,529] jepsen worker 29 - jepsen.util 4259 :ok :write [9 31] INFO [2019-12-17 12:31:51,530] jepsen worker 29 - jepsen.util 4259 :invoke :write [9 1] INFO [2019-12-17 12:31:51,539] jepsen worker 56 - jepsen.util 5456 :fail :cas [18 [40 35]] :ValueNotSwapFrom INFO [2019-12-17 12:31:51,539] jepsen worker 56 - jepsen.util 5456 :invoke :write [18 32] INFO [2019-12-17 12:31:51,540] jepsen worker 0 - jepsen.util 7650 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:31:51,540] jepsen worker 0 - jepsen.util 7650 :invoke :write [0 26] INFO [2019-12-17 12:31:51,545] jepsen worker 70 - jepsen.util 5920 :fail :cas [23 [36 9]] :ValueNotSwapFrom INFO [2019-12-17 12:31:51,545] jepsen worker 70 - jepsen.util 5920 :invoke :read [23 nil] INFO [2019-12-17 12:31:51,558] jepsen worker 10 - jepsen.util 5140 :info :write [3 2] :RequestCanceledException INFO [2019-12-17 12:31:51,558] jepsen worker 89 - jepsen.util 5039 :fail :cas [29 [27 43]] :ValueNotSwapFrom INFO [2019-12-17 12:31:51,558] jepsen worker 89 - jepsen.util 5039 :invoke :write [29 47] INFO [2019-12-17 12:31:51,559] jepsen worker 10 - jepsen.util 5230 :invoke :read [3 nil] INFO [2019-12-17 12:31:51,571] jepsen worker 70 - jepsen.util 5920 :ok :read [23 28] INFO [2019-12-17 12:31:51,571] jepsen worker 70 - jepsen.util 5920 :invoke :read [23 nil] INFO [2019-12-17 12:31:51,588] jepsen worker 29 - jepsen.util 4259 :ok :write [9 1] INFO [2019-12-17 12:31:51,589] jepsen worker 29 - jepsen.util 4259 :invoke :cas [9 [18 12]] INFO [2019-12-17 12:31:51,592] jepsen worker 61 - jepsen.util 7801 :info :write [20 38] :RequestCanceledException INFO [2019-12-17 12:31:51,594] jepsen worker 61 - jepsen.util 7891 :invoke :write [20 29] INFO [2019-12-17 12:31:51,601] jepsen worker 29 - jepsen.util 4259 :fail :cas [9 [18 12]] :ValueNotSwapFrom INFO [2019-12-17 12:31:51,601] jepsen worker 29 - jepsen.util 4259 :invoke :write [9 20] INFO [2019-12-17 12:31:51,623] jepsen worker 70 - jepsen.util 5920 :ok :read [23 28] INFO [2019-12-17 12:31:51,623] jepsen worker 70 - jepsen.util 5920 :invoke :read [23 nil] INFO [2019-12-17 12:31:51,625] jepsen worker 62 - jepsen.util 6632 :info :cas [20 [38 7]] :RequestCanceledException INFO [2019-12-17 12:31:51,626] jepsen worker 62 - jepsen.util 6722 :invoke :cas [20 [8 48]] INFO [2019-12-17 12:31:51,640] jepsen worker 29 - jepsen.util 4259 :ok :write [9 20] INFO [2019-12-17 12:31:51,641] jepsen worker 29 - jepsen.util 4259 :invoke :cas [9 [13 19]] INFO [2019-12-17 12:31:51,641] jepsen worker 78 - jepsen.util 5388 :fail :cas [26 [33 47]] :ValueNotSwapFrom INFO [2019-12-17 12:31:51,642] jepsen worker 78 - jepsen.util 5388 :invoke :write [26 29] INFO [2019-12-17 12:31:51,655] jepsen worker 29 - jepsen.util 4259 :fail :cas [9 [13 19]] :ValueNotSwapFrom INFO [2019-12-17 12:31:51,655] jepsen worker 29 - jepsen.util 4259 :invoke :write [9 39] INFO [2019-12-17 12:31:51,665] jepsen worker 3 - jepsen.util 5313 :fail :cas [1 [27 25]] :ValueNotSwapFrom INFO [2019-12-17 12:31:51,665] jepsen worker 3 - jepsen.util 5313 :invoke :cas [1 [28 20]] INFO [2019-12-17 12:31:51,678] jepsen worker 70 - jepsen.util 5920 :ok :read [23 28] INFO [2019-12-17 12:31:51,678] jepsen worker 70 - jepsen.util 5920 :invoke :cas [23 [4 47]] INFO [2019-12-17 12:31:51,690] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":30241,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:31:51,690] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":30241,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:31:51,690] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":30243,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:31:51,690] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":30243,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:31:51,696] jepsen worker 29 - jepsen.util 4259 :ok :write [9 39] INFO [2019-12-17 12:31:51,696] jepsen worker 29 - jepsen.util 4259 :invoke :write [9 31] INFO [2019-12-17 12:31:51,716] jepsen worker 55 - jepsen.util 6265 :fail :cas [18 [32 13]] :ValueNotSwapFrom INFO [2019-12-17 12:31:51,716] jepsen worker 55 - jepsen.util 6265 :invoke :cas [18 [40 32]] INFO [2019-12-17 12:31:51,725] jepsen worker 67 - jepsen.util 7087 :info :write [22 37] :RequestCanceledException INFO [2019-12-17 12:31:51,726] jepsen worker 67 - jepsen.util 7177 :invoke :read [22 nil] INFO [2019-12-17 12:31:51,748] jepsen worker 88 - jepsen.util 5488 :ok :read [29 13] INFO [2019-12-17 12:31:51,748] jepsen worker 88 - jepsen.util 5488 :invoke :cas [29 [24 7]] INFO [2019-12-17 12:31:51,771] jepsen worker 29 - jepsen.util 4259 :ok :write [9 31] INFO [2019-12-17 12:31:51,772] jepsen worker 29 - jepsen.util 4259 :invoke :cas [9 [25 38]] INFO [2019-12-17 12:31:51,785] jepsen worker 29 - jepsen.util 4259 :fail :cas [9 [25 38]] :ValueNotSwapFrom INFO [2019-12-17 12:31:51,785] jepsen worker 29 - jepsen.util 4259 :invoke :read [9 nil] INFO [2019-12-17 12:31:51,800] jepsen worker 29 - jepsen.util 4259 :ok :read [9 31] INFO [2019-12-17 12:31:51,800] jepsen worker 29 - jepsen.util 4259 :invoke :cas [9 [25 33]] INFO [2019-12-17 12:31:51,810] jepsen worker 26 - jepsen.util 7406 :info :cas [8 [5 34]] :RequestCanceledException INFO [2019-12-17 12:31:51,811] jepsen worker 26 - jepsen.util 7496 :invoke :cas [8 [5 13]] INFO [2019-12-17 12:31:51,813] jepsen worker 29 - jepsen.util 4259 :fail :cas [9 [25 33]] :ValueNotSwapFrom INFO [2019-12-17 12:31:51,813] jepsen worker 29 - jepsen.util 4259 :invoke :read [9 nil] INFO [2019-12-17 12:31:51,825] jepsen worker 21 - jepsen.util 6501 :info :write [7 36] :RequestCanceledException INFO [2019-12-17 12:31:51,827] jepsen worker 21 - jepsen.util 6591 :invoke :cas [7 [8 49]] INFO [2019-12-17 12:31:51,834] jepsen worker 29 - jepsen.util 4259 :ok :read [9 31] INFO [2019-12-17 12:31:51,834] jepsen worker 29 - jepsen.util 4259 :invoke :write [9 44] INFO [2019-12-17 12:31:51,845] jepsen worker 37 - jepsen.util 4897 :fail :cas [12 [10 29]] :ValueNotSwapFrom INFO [2019-12-17 12:31:51,846] jepsen worker 37 - jepsen.util 4897 :invoke :read [12 nil] INFO [2019-12-17 12:31:51,881] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":30261,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:31:51,881] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":30261,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:31:51,881] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":30264,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:31:51,881] jepsen worker 29 - jepsen.util 4259 :ok :write [9 44] INFO [2019-12-17 12:31:51,881] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":30264,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:31:51,882] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":30265,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:31:51,882] jepsen worker 29 - jepsen.util 4259 :invoke :read [9 nil] INFO [2019-12-17 12:31:51,882] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":30265,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:31:51,884] jepsen worker 70 - jepsen.util 5920 :fail :cas [23 [4 47]] :ValueNotSwapFrom INFO [2019-12-17 12:31:51,884] jepsen worker 70 - jepsen.util 5920 :invoke :cas [23 [48 16]] INFO [2019-12-17 12:31:51,893] jepsen worker 1 - jepsen.util 6841 :info :cas [0 [24 27]] :RequestCanceledException INFO [2019-12-17 12:31:51,894] jepsen worker 1 - jepsen.util 6931 :invoke :read [0 nil] INFO [2019-12-17 12:31:51,902] jepsen worker 29 - jepsen.util 4259 :ok :read [9 44] INFO [2019-12-17 12:31:51,902] jepsen worker 29 - jepsen.util 4259 :invoke :cas [9 [18 13]] INFO [2019-12-17 12:31:51,906] jepsen worker 70 - jepsen.util 5920 :fail :cas [23 [48 16]] :ValueNotSwapFrom INFO [2019-12-17 12:31:51,906] jepsen worker 70 - jepsen.util 5920 :invoke :read [23 nil] INFO [2019-12-17 12:31:51,909] jepsen worker 7 - jepsen.util 7207 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:31:51,910] jepsen worker 7 - jepsen.util 7207 :invoke :cas [2 [24 25]] INFO [2019-12-17 12:31:51,914] jepsen worker 29 - jepsen.util 4259 :fail :cas [9 [18 13]] :ValueNotSwapFrom INFO [2019-12-17 12:31:51,915] jepsen worker 29 - jepsen.util 4259 :invoke :read [9 nil] INFO [2019-12-17 12:31:51,917] jepsen worker 80 - jepsen.util 5300 :ok :read [26 47] INFO [2019-12-17 12:31:51,917] jepsen worker 80 - jepsen.util 5300 :invoke :read [26 nil] INFO [2019-12-17 12:31:51,926] jepsen worker 79 - jepsen.util 5119 :info :write [26 14] :RequestCanceledException INFO [2019-12-17 12:31:51,927] jepsen worker 79 - jepsen.util 5209 :invoke :cas [26 [31 9]] INFO [2019-12-17 12:31:51,927] jepsen worker 29 - jepsen.util 4259 :ok :read [9 44] INFO [2019-12-17 12:31:51,927] jepsen worker 29 - jepsen.util 4259 :invoke :cas [9 [8 20]] INFO [2019-12-17 12:31:51,938] jepsen worker 29 - jepsen.util 4259 :fail :cas [9 [8 20]] :ValueNotSwapFrom INFO [2019-12-17 12:31:51,938] jepsen worker 29 - jepsen.util 4259 :invoke :cas [9 [27 43]] INFO [2019-12-17 12:31:51,943] jepsen worker 3 - jepsen.util 5313 :fail :cas [1 [28 20]] :ValueNotSwapFrom INFO [2019-12-17 12:31:51,943] jepsen worker 3 - jepsen.util 5313 :invoke :read [1 nil] INFO [2019-12-17 12:31:51,952] jepsen worker 29 - jepsen.util 4259 :fail :cas [9 [27 43]] :ValueNotSwapFrom INFO [2019-12-17 12:31:51,952] jepsen worker 29 - jepsen.util 4259 :invoke :read [9 nil] INFO [2019-12-17 12:31:51,952] jepsen worker 70 - jepsen.util 5920 :ok :read [23 28] INFO [2019-12-17 12:31:51,952] jepsen worker 70 - jepsen.util 5920 :invoke :read [23 nil] INFO [2019-12-17 12:31:51,963] jepsen worker 29 - jepsen.util 4259 :ok :read [9 44] INFO [2019-12-17 12:31:51,963] jepsen worker 29 - jepsen.util 4259 :invoke :cas [9 [38 0]] INFO [2019-12-17 12:31:51,964] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:31:51,964] jepsen nemesis - couchbase.util Rebalance check count: 227 INFO [2019-12-17 12:31:51,964] jepsen nemesis - couchbase.util Rebalance stuck count: 208 INFO [2019-12-17 12:31:51,972] jepsen worker 29 - jepsen.util 4259 :fail :cas [9 [38 0]] :ValueNotSwapFrom INFO [2019-12-17 12:31:51,972] jepsen worker 29 - jepsen.util 4259 :invoke :write [9 7] INFO [2019-12-17 12:31:51,991] jepsen worker 88 - jepsen.util 5488 :fail :cas [29 [24 7]] :ValueNotSwapFrom INFO [2019-12-17 12:31:51,991] jepsen worker 88 - jepsen.util 5488 :invoke :cas [29 [30 20]] INFO [2019-12-17 12:31:52,006] jepsen worker 70 - jepsen.util 5920 :ok :read [23 28] INFO [2019-12-17 12:31:52,006] jepsen worker 70 - jepsen.util 5920 :invoke :read [23 nil] INFO [2019-12-17 12:31:52,008] jepsen worker 37 - jepsen.util 4897 :ok :read [12 29] INFO [2019-12-17 12:31:52,008] jepsen worker 37 - jepsen.util 4897 :invoke :write [12 34] INFO [2019-12-17 12:31:52,018] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":30283,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:31:52,018] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":30283,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:31:52,018] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":30287,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:31:52,018] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":30287,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:31:52,021] jepsen worker 29 - jepsen.util 4259 :ok :write [9 7] INFO [2019-12-17 12:31:52,021] jepsen worker 29 - jepsen.util 4259 :invoke :write [9 26] INFO [2019-12-17 12:31:52,026] jepsen worker 27 - jepsen.util 5067 :fail :read [9 nil] :RequestTimeoutException INFO [2019-12-17 12:31:52,027] jepsen worker 27 - jepsen.util 5067 :invoke :write [9 44] INFO [2019-12-17 12:31:52,039] jepsen worker 35 - jepsen.util 7145 :info :write [11 6] :RequestCanceledException INFO [2019-12-17 12:31:52,040] jepsen worker 35 - jepsen.util 7235 :invoke :write [11 19] INFO [2019-12-17 12:31:52,049] jepsen worker 70 - jepsen.util 5920 :ok :read [23 28] INFO [2019-12-17 12:31:52,050] jepsen worker 70 - jepsen.util 5920 :invoke :read [23 nil] INFO [2019-12-17 12:31:52,055] jepsen worker 42 - jepsen.util 7692 :info :cas [14 [48 42]] :RequestCanceledException INFO [2019-12-17 12:31:52,056] jepsen worker 42 - jepsen.util 7782 :invoke :read [14 nil] INFO [2019-12-17 12:31:52,067] jepsen worker 29 - jepsen.util 4259 :ok :write [9 26] INFO [2019-12-17 12:31:52,067] jepsen worker 29 - jepsen.util 4259 :invoke :write [9 29] INFO [2019-12-17 12:31:52,068] jepsen worker 80 - jepsen.util 5300 :ok :read [26 47] INFO [2019-12-17 12:31:52,069] jepsen worker 80 - jepsen.util 5300 :invoke :cas [26 [5 8]] INFO [2019-12-17 12:31:52,090] jepsen worker 79 - jepsen.util 5209 :fail :cas [26 [31 9]] :ValueNotSwapFrom INFO [2019-12-17 12:31:52,090] jepsen worker 79 - jepsen.util 5209 :invoke :read [26 nil] INFO [2019-12-17 12:31:52,092] jepsen worker 50 - jepsen.util 7340 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:31:52,092] jepsen worker 50 - jepsen.util 7340 :invoke :read [16 nil] INFO [2019-12-17 12:31:52,103] jepsen worker 88 - jepsen.util 5488 :fail :cas [29 [30 20]] :ValueNotSwapFrom INFO [2019-12-17 12:31:52,103] jepsen worker 88 - jepsen.util 5488 :invoke :write [29 14] INFO [2019-12-17 12:31:52,114] jepsen worker 29 - jepsen.util 4259 :ok :write [9 29] INFO [2019-12-17 12:31:52,115] jepsen worker 29 - jepsen.util 4259 :invoke :cas [9 [27 15]] INFO [2019-12-17 12:31:52,126] jepsen worker 24 - jepsen.util 6594 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:31:52,126] jepsen worker 24 - jepsen.util 6594 :invoke :read [8 nil] INFO [2019-12-17 12:31:52,137] jepsen worker 29 - jepsen.util 4259 :fail :cas [9 [27 15]] :ValueNotSwapFrom INFO [2019-12-17 12:31:52,137] jepsen worker 29 - jepsen.util 4259 :invoke :read [9 nil] INFO [2019-12-17 12:31:52,149] jepsen worker 29 - jepsen.util 4259 :ok :read [9 29] INFO [2019-12-17 12:31:52,149] jepsen worker 29 - jepsen.util 4259 :invoke :read [9 nil] INFO [2019-12-17 12:31:52,157] jepsen worker 49 - jepsen.util 6169 :info :cas [16 [27 30]] :RequestCanceledException INFO [2019-12-17 12:31:52,157] jepsen worker 23 - jepsen.util 7313 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:31:52,157] jepsen worker 23 - jepsen.util 7313 :invoke :read [7 nil] INFO [2019-12-17 12:31:52,158] jepsen worker 49 - jepsen.util 6259 :invoke :cas [16 [3 30]] INFO [2019-12-17 12:31:52,163] jepsen worker 29 - jepsen.util 4259 :ok :read [9 29] INFO [2019-12-17 12:31:52,163] jepsen worker 29 - jepsen.util 4259 :invoke :write [9 26] INFO [2019-12-17 12:31:52,165] jepsen worker 87 - jepsen.util 4767 :ok :read [29 13] INFO [2019-12-17 12:31:52,165] jepsen worker 87 - jepsen.util 4767 :invoke :cas [29 [29 24]] INFO [2019-12-17 12:31:52,172] jepsen worker 3 - jepsen.util 5313 :ok :read [1 26] INFO [2019-12-17 12:31:52,173] jepsen worker 3 - jepsen.util 5313 :invoke :cas [1 [48 31]] INFO [2019-12-17 12:31:52,201] jepsen worker 29 - jepsen.util 4259 :ok :write [9 26] INFO [2019-12-17 12:31:52,201] jepsen worker 29 - jepsen.util 4259 :invoke :write [9 43] INFO [2019-12-17 12:31:52,210] jepsen worker 59 - jepsen.util 6989 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:31:52,210] jepsen worker 59 - jepsen.util 6989 :invoke :write [19 29] INFO [2019-12-17 12:31:52,233] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":30310,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:31:52,234] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":30317,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:31:52,234] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":30310,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:31:52,234] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":30317,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:31:52,234] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":30312,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:31:52,234] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":30318,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:31:52,234] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":30312,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:31:52,234] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":30318,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:31:52,234] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":30319,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:31:52,234] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":30319,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:31:52,245] jepsen worker 46 - jepsen.util 5176 :ok :write [15 26] INFO [2019-12-17 12:31:52,245] jepsen worker 46 - jepsen.util 5176 :invoke :cas [15 [45 20]] INFO [2019-12-17 12:31:52,260] jepsen worker 29 - jepsen.util 4259 :ok :write [9 43] INFO [2019-12-17 12:31:52,260] jepsen worker 29 - jepsen.util 4259 :invoke :read [9 nil] INFO [2019-12-17 12:31:52,261] jepsen worker 72 - jepsen.util 5022 :ok :write [24 31] INFO [2019-12-17 12:31:52,262] jepsen worker 72 - jepsen.util 5022 :invoke :read [24 nil] INFO [2019-12-17 12:31:52,262] jepsen worker 20 - jepsen.util 5330 :ok :write [6 23] INFO [2019-12-17 12:31:52,263] jepsen worker 20 - jepsen.util 5330 :invoke :write [6 36] INFO [2019-12-17 12:31:52,276] jepsen worker 70 - jepsen.util 5920 :ok :read [23 28] INFO [2019-12-17 12:31:52,276] jepsen worker 70 - jepsen.util 5920 :invoke :write [23 0] INFO [2019-12-17 12:31:52,293] jepsen worker 29 - jepsen.util 4259 :ok :read [9 43] INFO [2019-12-17 12:31:52,293] jepsen worker 29 - jepsen.util 4259 :invoke :read [9 nil] INFO [2019-12-17 12:31:52,326] jepsen worker 31 - jepsen.util 4891 :info :write [10 49] :RequestCanceledException INFO [2019-12-17 12:31:52,327] jepsen worker 31 - jepsen.util 4981 :invoke :read [10 nil] INFO [2019-12-17 12:31:52,337] jepsen worker 29 - jepsen.util 4259 :ok :read [9 43] INFO [2019-12-17 12:31:52,338] jepsen worker 29 - jepsen.util 4259 :invoke :read [9 nil] INFO [2019-12-17 12:31:52,338] jepsen worker 72 - jepsen.util 5022 :ok :read [24 31] INFO [2019-12-17 12:31:52,338] jepsen worker 72 - jepsen.util 5022 :invoke :write [24 36] INFO [2019-12-17 12:31:52,347] jepsen worker 3 - jepsen.util 5313 :fail :cas [1 [48 31]] :ValueNotSwapFrom INFO [2019-12-17 12:31:52,347] jepsen worker 3 - jepsen.util 5313 :invoke :cas [1 [30 5]] INFO [2019-12-17 12:31:52,366] jepsen worker 27 - jepsen.util 5067 :ok :write [9 44] INFO [2019-12-17 12:31:52,366] jepsen worker 27 - jepsen.util 5067 :invoke :write [9 17] INFO [2019-12-17 12:31:52,367] jepsen worker 65 - jepsen.util 4565 :fail :cas [21 [9 34]] :ValueNotSwapFrom INFO [2019-12-17 12:31:52,367] jepsen worker 65 - jepsen.util 4565 :invoke :read [21 nil] INFO [2019-12-17 12:31:52,414] jepsen worker 63 - jepsen.util 4293 :fail :cas [21 [40 16]] :ValueNotSwapFrom INFO [2019-12-17 12:31:52,414] jepsen worker 63 - jepsen.util 4293 :invoke :cas [21 [20 7]] INFO [2019-12-17 12:31:52,418] jepsen worker 29 - jepsen.util 4259 :ok :read [9 43] INFO [2019-12-17 12:31:52,418] jepsen worker 29 - jepsen.util 4259 :invoke :read [9 nil] INFO [2019-12-17 12:31:52,421] jepsen worker 3 - jepsen.util 5313 :fail :cas [1 [30 5]] :ValueNotSwapFrom INFO [2019-12-17 12:31:52,421] jepsen worker 3 - jepsen.util 5313 :invoke :cas [1 [30 28]] INFO [2019-12-17 12:31:52,425] jepsen worker 46 - jepsen.util 5176 :fail :cas [15 [45 20]] :ValueNotSwapFrom INFO [2019-12-17 12:31:52,425] jepsen worker 46 - jepsen.util 5176 :invoke :read [15 nil] INFO [2019-12-17 12:31:52,442] jepsen worker 11 - jepsen.util 5411 :fail :cas [3 [0 13]] :ValueNotSwapFrom INFO [2019-12-17 12:31:52,442] jepsen worker 11 - jepsen.util 5411 :invoke :cas [3 [37 47]] INFO [2019-12-17 12:31:52,453] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":30340,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:31:52,453] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":30340,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:31:52,453] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":30345,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:31:52,453] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":30345,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:31:52,459] jepsen worker 29 - jepsen.util 4259 :ok :read [9 44] INFO [2019-12-17 12:31:52,459] jepsen worker 29 - jepsen.util 4259 :invoke :write [9 28] INFO [2019-12-17 12:31:52,476] jepsen worker 3 - jepsen.util 5313 :fail :cas [1 [30 28]] :ValueNotSwapFrom INFO [2019-12-17 12:31:52,476] jepsen worker 3 - jepsen.util 5313 :invoke :write [1 44] INFO [2019-12-17 12:31:52,482] jepsen worker 27 - jepsen.util 5067 :ok :write [9 17] INFO [2019-12-17 12:31:52,482] jepsen worker 27 - jepsen.util 5067 :invoke :write [9 36] INFO [2019-12-17 12:31:52,526] jepsen worker 13 - jepsen.util 5143 :info :write [4 44] :RequestCanceledException INFO [2019-12-17 12:31:52,527] jepsen worker 13 - jepsen.util 5233 :invoke :read [4 nil] INFO [2019-12-17 12:31:52,543] jepsen worker 46 - jepsen.util 5176 :ok :read [15 26] INFO [2019-12-17 12:31:52,544] jepsen worker 46 - jepsen.util 5176 :invoke :write [15 7] INFO [2019-12-17 12:31:52,581] jepsen worker 29 - jepsen.util 4259 :ok :write [9 28] INFO [2019-12-17 12:31:52,581] jepsen worker 29 - jepsen.util 4259 :invoke :cas [9 [29 1]] INFO [2019-12-17 12:31:52,610] jepsen worker 20 - jepsen.util 5330 :ok :write [6 36] INFO [2019-12-17 12:31:52,610] jepsen worker 20 - jepsen.util 5330 :invoke :cas [6 [22 30]] INFO [2019-12-17 12:31:52,618] jepsen worker 29 - jepsen.util 4259 :fail :cas [9 [29 1]] :ValueNotSwapFrom INFO [2019-12-17 12:31:52,619] jepsen worker 29 - jepsen.util 4259 :invoke :write [9 1] INFO [2019-12-17 12:31:52,631] jepsen worker 63 - jepsen.util 4293 :fail :cas [21 [20 7]] :ValueNotSwapFrom INFO [2019-12-17 12:31:52,631] jepsen worker 63 - jepsen.util 4293 :invoke :write [21 24] INFO [2019-12-17 12:31:52,638] jepsen worker 28 - jepsen.util 3898 :info :cas [9 [10 43]] :RequestCanceledException INFO [2019-12-17 12:31:52,639] jepsen worker 28 - jepsen.util 3988 :invoke :read [9 nil] INFO [2019-12-17 12:31:52,646] jepsen worker 87 - jepsen.util 4767 :fail :cas [29 [29 24]] :ValueNotSwapFrom INFO [2019-12-17 12:31:52,646] jepsen worker 87 - jepsen.util 4767 :invoke :cas [29 [11 0]] INFO [2019-12-17 12:31:52,672] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":30357,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:31:52,672] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":30357,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:31:52,672] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":30361,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:31:52,672] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":30361,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:31:52,692] jepsen worker 51 - jepsen.util 5181 :info :write [17 2] :RequestCanceledException INFO [2019-12-17 12:31:52,693] jepsen worker 51 - jepsen.util 5271 :invoke :read [17 nil] INFO [2019-12-17 12:31:52,695] jepsen worker 27 - jepsen.util 5067 :ok :write [9 36] INFO [2019-12-17 12:31:52,695] jepsen worker 27 - jepsen.util 5067 :invoke :cas [9 [8 39]] INFO [2019-12-17 12:31:52,695] jepsen worker 11 - jepsen.util 5411 :fail :cas [3 [37 47]] :ValueNotSwapFrom INFO [2019-12-17 12:31:52,695] jepsen worker 11 - jepsen.util 5411 :invoke :cas [3 [17 6]] INFO [2019-12-17 12:31:52,699] jepsen worker 72 - jepsen.util 5022 :ok :write [24 36] INFO [2019-12-17 12:31:52,699] jepsen worker 72 - jepsen.util 5022 :invoke :read [24 nil] INFO [2019-12-17 12:31:52,702] jepsen worker 20 - jepsen.util 5330 :fail :cas [6 [22 30]] :ValueNotSwapFrom INFO [2019-12-17 12:31:52,702] jepsen worker 20 - jepsen.util 5330 :invoke :write [6 44] INFO [2019-12-17 12:31:52,733] jepsen worker 27 - jepsen.util 5067 :fail :cas [9 [8 39]] :ValueNotSwapFrom INFO [2019-12-17 12:31:52,733] jepsen worker 27 - jepsen.util 5067 :invoke :write [9 14] INFO [2019-12-17 12:31:52,737] jepsen worker 28 - jepsen.util 3988 :ok :read [9 36] INFO [2019-12-17 12:31:52,737] jepsen worker 28 - jepsen.util 3988 :invoke :write [9 12] INFO [2019-12-17 12:31:52,760] jepsen worker 51 - jepsen.util 5271 :ok :read [17 45] INFO [2019-12-17 12:31:52,760] jepsen worker 51 - jepsen.util 5271 :invoke :cas [17 [2 38]] INFO [2019-12-17 12:31:52,791] jepsen worker 5 - jepsen.util 4775 :info :write [1 19] :RequestCanceledException INFO [2019-12-17 12:31:52,792] jepsen worker 72 - jepsen.util 5022 :ok :read [24 36] INFO [2019-12-17 12:31:52,792] jepsen worker 72 - jepsen.util 5022 :invoke :cas [24 [32 9]] INFO [2019-12-17 12:31:52,793] jepsen worker 5 - jepsen.util 4865 :invoke :cas [1 [47 16]] INFO [2019-12-17 12:31:52,825] jepsen worker 18 - jepsen.util 5148 :info :write [6 34] :RequestCanceledException INFO [2019-12-17 12:31:52,827] jepsen worker 18 - jepsen.util 5238 :invoke :read [6 nil] INFO [2019-12-17 12:31:52,831] jepsen worker 29 - jepsen.util 4259 :ok :write [9 1] INFO [2019-12-17 12:31:52,831] jepsen worker 29 - jepsen.util 4259 :invoke :write [9 25] INFO [2019-12-17 12:31:52,840] jepsen worker 54 - jepsen.util 5634 :info :write [18 30] :RequestCanceledException INFO [2019-12-17 12:31:52,841] jepsen worker 54 - jepsen.util 5724 :invoke :cas [18 [17 37]] INFO [2019-12-17 12:31:52,867] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":30378,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:31:52,867] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":30378,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:31:52,867] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":30381,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:31:52,867] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":30381,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:31:52,880] jepsen worker 13 - jepsen.util 5233 :ok :read [4 27] INFO [2019-12-17 12:31:52,880] jepsen worker 13 - jepsen.util 5233 :invoke :cas [4 [14 36]] INFO [2019-12-17 12:31:52,896] jepsen worker 18 - jepsen.util 5238 :ok :read [6 36] INFO [2019-12-17 12:31:52,896] jepsen worker 18 - jepsen.util 5238 :invoke :cas [6 [12 33]] INFO [2019-12-17 12:31:52,908] jepsen worker 13 - jepsen.util 5233 :fail :cas [4 [14 36]] :ValueNotSwapFrom INFO [2019-12-17 12:31:52,908] jepsen worker 13 - jepsen.util 5233 :invoke :cas [4 [20 17]] INFO [2019-12-17 12:31:52,941] jepsen worker 18 - jepsen.util 5238 :fail :cas [6 [12 33]] :ValueNotSwapFrom INFO [2019-12-17 12:31:52,941] jepsen worker 18 - jepsen.util 5238 :invoke :read [6 nil] INFO [2019-12-17 12:31:52,956] jepsen worker 87 - jepsen.util 4767 :fail :cas [29 [11 0]] :ValueNotSwapFrom INFO [2019-12-17 12:31:52,956] jepsen worker 87 - jepsen.util 4767 :invoke :write [29 36] INFO [2019-12-17 12:31:52,981] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:31:52,981] jepsen nemesis - couchbase.util Rebalance check count: 228 INFO [2019-12-17 12:31:52,981] jepsen nemesis - couchbase.util Rebalance stuck count: 209 INFO [2019-12-17 12:31:52,987] jepsen worker 18 - jepsen.util 5238 :ok :read [6 36] INFO [2019-12-17 12:31:52,988] jepsen worker 18 - jepsen.util 5238 :invoke :write [6 29] INFO [2019-12-17 12:31:52,988] jepsen worker 27 - jepsen.util 5067 :ok :write [9 14] INFO [2019-12-17 12:31:52,988] jepsen worker 27 - jepsen.util 5067 :invoke :write [9 21] INFO [2019-12-17 12:31:52,989] jepsen worker 4 - jepsen.util 4414 :info :write [1 10] :RequestCanceledException INFO [2019-12-17 12:31:52,991] jepsen worker 4 - jepsen.util 4504 :invoke :cas [1 [41 18]] INFO [2019-12-17 12:31:52,998] jepsen worker 51 - jepsen.util 5271 :fail :cas [17 [2 38]] :ValueNotSwapFrom INFO [2019-12-17 12:31:52,998] jepsen worker 51 - jepsen.util 5271 :invoke :cas [17 [15 2]] INFO [2019-12-17 12:31:53,010] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":30392,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:31:53,011] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":30392,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:31:53,011] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":30396,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:31:53,011] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":30396,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:31:53,026] jepsen worker 9 - jepsen.util 5859 :fail :read [3 nil] :RequestTimeoutException INFO [2019-12-17 12:31:53,026] jepsen worker 9 - jepsen.util 5859 :invoke :cas [3 [9 40]] INFO [2019-12-17 12:31:53,027] jepsen worker 29 - jepsen.util 4259 :ok :write [9 25] INFO [2019-12-17 12:31:53,027] jepsen worker 29 - jepsen.util 4259 :invoke :write [9 49] INFO [2019-12-17 12:31:53,039] jepsen worker 5 - jepsen.util 4865 :fail :cas [1 [47 16]] :ValueNotSwapFrom INFO [2019-12-17 12:31:53,039] jepsen worker 5 - jepsen.util 4865 :invoke :write [1 30] INFO [2019-12-17 12:31:53,040] jepsen worker 53 - jepsen.util 5093 :info :write [17 27] :RequestCanceledException INFO [2019-12-17 12:31:53,041] jepsen worker 53 - jepsen.util 5183 :invoke :cas [17 [33 35]] INFO [2019-12-17 12:31:53,055] jepsen worker 51 - jepsen.util 5271 :fail :cas [17 [15 2]] :ValueNotSwapFrom INFO [2019-12-17 12:31:53,055] jepsen worker 51 - jepsen.util 5271 :invoke :read [17 nil] INFO [2019-12-17 12:31:53,087] jepsen worker 27 - jepsen.util 5067 :ok :write [9 21] INFO [2019-12-17 12:31:53,087] jepsen worker 27 - jepsen.util 5067 :invoke :cas [9 [15 30]] INFO [2019-12-17 12:31:53,093] jepsen worker 14 - jepsen.util 5324 :info :write [4 6] :RequestCanceledException INFO [2019-12-17 12:31:53,094] jepsen worker 14 - jepsen.util 5414 :invoke :write [4 29] INFO [2019-12-17 12:31:53,101] jepsen worker 9 - jepsen.util 5859 :fail :cas [3 [9 40]] :ValueNotSwapFrom INFO [2019-12-17 12:31:53,101] jepsen worker 9 - jepsen.util 5859 :invoke :write [3 8] INFO [2019-12-17 12:31:53,109] jepsen worker 19 - jepsen.util 5149 :info :cas [6 [12 23]] :RequestCanceledException INFO [2019-12-17 12:31:53,110] jepsen worker 19 - jepsen.util 5239 :invoke :read [6 nil] INFO [2019-12-17 12:31:53,121] jepsen worker 27 - jepsen.util 5067 :fail :cas [9 [15 30]] :ValueNotSwapFrom INFO [2019-12-17 12:31:53,121] jepsen worker 27 - jepsen.util 5067 :invoke :read [9 nil] INFO [2019-12-17 12:31:53,126] jepsen worker 8 - jepsen.util 6578 :info :cas [2 [12 13]] :RequestCanceledException INFO [2019-12-17 12:31:53,126] jepsen worker 48 - jepsen.util 6438 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:31:53,126] jepsen worker 48 - jepsen.util 6438 :invoke :write [16 40] INFO [2019-12-17 12:31:53,127] jepsen worker 8 - jepsen.util 6668 :invoke :cas [2 [35 1]] INFO [2019-12-17 12:31:53,140] jepsen worker 68 - jepsen.util 6548 :info :cas [22 [47 21]] :RequestCanceledException INFO [2019-12-17 12:31:53,140] jepsen worker 60 - jepsen.util 7800 :info :write [20 49] :RequestCanceledException INFO [2019-12-17 12:31:53,141] jepsen worker 60 - jepsen.util 7890 :invoke :read [20 nil] INFO [2019-12-17 12:31:53,141] jepsen worker 68 - jepsen.util 6638 :invoke :cas [22 [44 22]] INFO [2019-12-17 12:31:53,146] jepsen worker 19 - jepsen.util 5239 :ok :read [6 36] INFO [2019-12-17 12:31:53,147] jepsen worker 19 - jepsen.util 5239 :invoke :write [6 25] INFO [2019-12-17 12:31:53,157] jepsen worker 15 - jepsen.util 6045 :info :cas [5 [38 21]] :RequestCanceledException INFO [2019-12-17 12:31:53,157] jepsen worker 12 - jepsen.util 5502 :info :write [4 7] :RequestCanceledException INFO [2019-12-17 12:31:53,157] jepsen worker 38 - jepsen.util 5258 :info :write [12 43] :RequestCanceledException INFO [2019-12-17 12:31:53,158] jepsen worker 38 - jepsen.util 5348 :invoke :cas [12 [47 20]] INFO [2019-12-17 12:31:53,158] jepsen worker 15 - jepsen.util 6135 :invoke :write [5 2] INFO [2019-12-17 12:31:53,158] jepsen worker 12 - jepsen.util 5592 :invoke :write [4 2] INFO [2019-12-17 12:31:53,175] jepsen worker 29 - jepsen.util 4259 :ok :write [9 49] INFO [2019-12-17 12:31:53,175] jepsen worker 18 - jepsen.util 5238 :ok :write [6 29] INFO [2019-12-17 12:31:53,175] jepsen worker 29 - jepsen.util 4259 :invoke :read [9 nil] INFO [2019-12-17 12:31:53,175] jepsen worker 18 - jepsen.util 5238 :invoke :write [6 2] INFO [2019-12-17 12:31:53,192] jepsen worker 52 - jepsen.util 5542 :info :write [17 12] :RequestCanceledException INFO [2019-12-17 12:31:53,192] jepsen worker 84 - jepsen.util 6654 :info :cas [28 [33 24]] :RequestCanceledException INFO [2019-12-17 12:31:53,192] jepsen worker 6 - jepsen.util 7296 :info :cas [2 [48 37]] :RequestCanceledException INFO [2019-12-17 12:31:53,193] jepsen worker 52 - jepsen.util 5632 :invoke :write [17 38] INFO [2019-12-17 12:31:53,193] jepsen worker 6 - jepsen.util 7386 :invoke :write [2 31] INFO [2019-12-17 12:31:53,193] jepsen worker 84 - jepsen.util 6744 :invoke :read [28 nil] INFO [2019-12-17 12:31:53,206] jepsen worker 13 - jepsen.util 5233 :fail :cas [4 [20 17]] :ValueNotSwapFrom INFO [2019-12-17 12:31:53,206] jepsen worker 13 - jepsen.util 5233 :invoke :cas [4 [45 3]] INFO [2019-12-17 12:31:53,211] jepsen worker 77 - jepsen.util 7187 :info :cas [25 [44 37]] :RequestCanceledException INFO [2019-12-17 12:31:53,211] jepsen worker 76 - jepsen.util 7096 :info :write [25 4] :RequestCanceledException INFO [2019-12-17 12:31:53,211] jepsen worker 47 - jepsen.util 5897 :fail :read [15 nil] :RequestTimeoutException INFO [2019-12-17 12:31:53,211] jepsen worker 73 - jepsen.util 4843 :info :write [24 28] :RequestCanceledException INFO [2019-12-17 12:31:53,211] jepsen worker 47 - jepsen.util 5897 :invoke :write [15 39] INFO [2019-12-17 12:31:53,212] jepsen worker 77 - jepsen.util 7277 :invoke :read [25 nil] INFO [2019-12-17 12:31:53,212] jepsen worker 76 - jepsen.util 7186 :invoke :write [25 34] INFO [2019-12-17 12:31:53,212] jepsen worker 73 - jepsen.util 4933 :invoke :cas [24 [2 40]] INFO [2019-12-17 12:31:53,220] jepsen worker 29 - jepsen.util 4259 :ok :read [9 49] INFO [2019-12-17 12:31:53,220] jepsen worker 29 - jepsen.util 4259 :invoke :read [9 nil] INFO [2019-12-17 12:31:53,225] jepsen worker 17 - jepsen.util 7037 :info :write [5 32] :RequestCanceledException INFO [2019-12-17 12:31:53,226] jepsen worker 33 - jepsen.util 7053 :info :cas [11 [37 15]] :RequestCanceledException INFO [2019-12-17 12:31:53,227] jepsen worker 16 - jepsen.util 8116 :info :cas [5 [10 17]] :RequestCanceledException INFO [2019-12-17 12:31:53,228] jepsen worker 17 - jepsen.util 7127 :invoke :cas [5 [22 20]] INFO [2019-12-17 12:31:53,228] jepsen worker 16 - jepsen.util 8206 :invoke :write [5 36] INFO [2019-12-17 12:31:53,228] jepsen worker 33 - jepsen.util 7143 :invoke :read [11 nil] INFO [2019-12-17 12:31:53,230] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":30429,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:31:53,230] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":30418,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:31:53,230] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":30429,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:31:53,230] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":30418,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:31:53,230] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":30432,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:31:53,230] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":30420,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:31:53,230] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":30432,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:31:53,230] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":30420,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:31:53,230] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":30434,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:31:53,230] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":30421,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:31:53,230] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":30434,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:31:53,230] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":30421,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:31:53,235] jepsen worker 4 - jepsen.util 4504 :fail :cas [1 [41 18]] :ValueNotSwapFrom INFO [2019-12-17 12:31:53,235] jepsen worker 4 - jepsen.util 4504 :invoke :cas [1 [35 4]] INFO [2019-12-17 12:31:53,240] jepsen worker 82 - jepsen.util 7732 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:31:53,240] jepsen worker 45 - jepsen.util 5355 :info :write [15 11] :RequestCanceledException INFO [2019-12-17 12:31:53,240] jepsen worker 82 - jepsen.util 7732 :invoke :cas [27 [4 43]] INFO [2019-12-17 12:31:53,242] jepsen worker 45 - jepsen.util 5445 :invoke :read [15 nil] INFO [2019-12-17 12:31:53,258] jepsen worker 85 - jepsen.util 7915 :info :cas [28 [2 43]] :RequestCanceledException INFO [2019-12-17 12:31:53,258] jepsen worker 75 - jepsen.util 6825 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:31:53,258] jepsen worker 75 - jepsen.util 6825 :invoke :cas [25 [31 8]] INFO [2019-12-17 12:31:53,259] jepsen worker 85 - jepsen.util 8005 :invoke :cas [28 [0 19]] INFO [2019-12-17 12:31:53,267] jepsen worker 29 - jepsen.util 4259 :ok :read [9 49] INFO [2019-12-17 12:31:53,268] jepsen worker 29 - jepsen.util 4259 :invoke :cas [9 [5 31]] INFO [2019-12-17 12:31:53,282] jepsen worker 13 - jepsen.util 5233 :fail :cas [4 [45 3]] :ValueNotSwapFrom INFO [2019-12-17 12:31:53,282] jepsen worker 13 - jepsen.util 5233 :invoke :read [4 nil] INFO [2019-12-17 12:31:53,283] jepsen worker 51 - jepsen.util 5271 :ok :read [17 45] INFO [2019-12-17 12:31:53,283] jepsen worker 51 - jepsen.util 5271 :invoke :read [17 nil] INFO [2019-12-17 12:31:53,292] jepsen worker 39 - jepsen.util 6699 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:31:53,292] jepsen worker 39 - jepsen.util 6699 :invoke :cas [13 [43 10]] INFO [2019-12-17 12:31:53,297] jepsen worker 29 - jepsen.util 4259 :fail :cas [9 [5 31]] :ValueNotSwapFrom INFO [2019-12-17 12:31:53,297] jepsen worker 29 - jepsen.util 4259 :invoke :write [9 23] INFO [2019-12-17 12:31:53,302] jepsen worker 28 - jepsen.util 3988 :ok :write [9 12] INFO [2019-12-17 12:31:53,302] jepsen worker 28 - jepsen.util 3988 :invoke :cas [9 [45 29]] INFO [2019-12-17 12:31:53,310] jepsen worker 40 - jepsen.util 7600 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:31:53,310] jepsen worker 66 - jepsen.util 6546 :info :cas [22 [20 25]] :RequestCanceledException INFO [2019-12-17 12:31:53,310] jepsen worker 41 - jepsen.util 7691 :info :write [13 26] :RequestCanceledException INFO [2019-12-17 12:31:53,311] jepsen worker 40 - jepsen.util 7600 :invoke :read [13 nil] INFO [2019-12-17 12:31:53,312] jepsen worker 66 - jepsen.util 6636 :invoke :read [22 nil] INFO [2019-12-17 12:31:53,312] jepsen worker 41 - jepsen.util 7781 :invoke :write [13 19] INFO [2019-12-17 12:31:53,317] jepsen worker 4 - jepsen.util 4504 :fail :cas [1 [35 4]] :ValueNotSwapFrom INFO [2019-12-17 12:31:53,317] jepsen worker 4 - jepsen.util 4504 :invoke :read [1 nil] INFO [2019-12-17 12:31:53,323] jepsen worker 28 - jepsen.util 3988 :fail :cas [9 [45 29]] :ValueNotSwapFrom INFO [2019-12-17 12:31:53,323] jepsen worker 28 - jepsen.util 3988 :invoke :cas [9 [12 7]] INFO [2019-12-17 12:31:53,324] jepsen worker 69 - jepsen.util 5649 :info :write [23 24] :RequestCanceledException INFO [2019-12-17 12:31:53,325] jepsen worker 69 - jepsen.util 5739 :invoke :write [23 5] INFO [2019-12-17 12:31:53,326] jepsen worker 80 - jepsen.util 5300 :fail :cas [26 [5 8]] :ValueNotSwapFrom INFO [2019-12-17 12:31:53,327] jepsen worker 80 - jepsen.util 5300 :invoke :write [26 22] INFO [2019-12-17 12:31:53,339] jepsen worker 57 - jepsen.util 6537 :info :write [19 0] :RequestCanceledException INFO [2019-12-17 12:31:53,339] jepsen worker 22 - jepsen.util 7132 :info :write [7 34] :RequestCanceledException INFO [2019-12-17 12:31:53,340] jepsen worker 13 - jepsen.util 5233 :ok :read [4 27] INFO [2019-12-17 12:31:53,340] jepsen worker 13 - jepsen.util 5233 :invoke :read [4 nil] INFO [2019-12-17 12:31:53,340] jepsen worker 22 - jepsen.util 7222 :invoke :write [7 8] INFO [2019-12-17 12:31:53,340] jepsen worker 57 - jepsen.util 6627 :invoke :cas [19 [36 39]] INFO [2019-12-17 12:31:53,347] jepsen worker 79 - jepsen.util 5209 :ok :read [26 47] INFO [2019-12-17 12:31:53,347] jepsen worker 79 - jepsen.util 5209 :invoke :cas [26 [22 37]] INFO [2019-12-17 12:31:53,349] jepsen worker 63 - jepsen.util 4293 :ok :write [21 24] INFO [2019-12-17 12:31:53,349] jepsen worker 63 - jepsen.util 4293 :invoke :read [21 nil] INFO [2019-12-17 12:31:53,354] jepsen worker 83 - jepsen.util 7463 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:31:53,354] jepsen worker 83 - jepsen.util 7463 :invoke :read [27 nil] INFO [2019-12-17 12:31:53,373] jepsen worker 63 - jepsen.util 4293 :ok :read [21 24] INFO [2019-12-17 12:31:53,373] jepsen worker 63 - jepsen.util 4293 :invoke :cas [21 [14 8]] INFO [2019-12-17 12:31:53,393] jepsen worker 71 - jepsen.util 6011 :info :write [23 28] :RequestCanceledException INFO [2019-12-17 12:31:53,393] jepsen worker 71 - jepsen.util 6101 :invoke :write [23 20] INFO [2019-12-17 12:31:53,399] jepsen worker 63 - jepsen.util 4293 :fail :cas [21 [14 8]] :ValueNotSwapFrom INFO [2019-12-17 12:31:53,399] jepsen worker 63 - jepsen.util 4293 :invoke :cas [21 [47 42]] INFO [2019-12-17 12:31:53,411] jepsen worker 43 - jepsen.util 6613 :info :write [14 39] :RequestCanceledException INFO [2019-12-17 12:31:53,412] jepsen worker 43 - jepsen.util 6703 :invoke :read [14 nil] INFO [2019-12-17 12:31:53,415] jepsen worker 29 - jepsen.util 4259 :ok :write [9 23] INFO [2019-12-17 12:31:53,415] jepsen worker 29 - jepsen.util 4259 :invoke :cas [9 [32 47]] INFO [2019-12-17 12:31:53,422] jepsen worker 63 - jepsen.util 4293 :fail :cas [21 [47 42]] :ValueNotSwapFrom INFO [2019-12-17 12:31:53,422] jepsen worker 63 - jepsen.util 4293 :invoke :cas [21 [20 21]] INFO [2019-12-17 12:31:53,427] jepsen worker 4 - jepsen.util 4504 :ok :read [1 26] INFO [2019-12-17 12:31:53,427] jepsen worker 4 - jepsen.util 4504 :invoke :write [1 41] INFO [2019-12-17 12:31:53,449] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":30474,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:31:53,449] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":30474,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:31:53,449] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":30475,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:31:53,449] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":30475,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:31:53,452] jepsen worker 63 - jepsen.util 4293 :fail :cas [21 [20 21]] :ValueNotSwapFrom INFO [2019-12-17 12:31:53,452] jepsen worker 63 - jepsen.util 4293 :invoke :read [21 nil] INFO [2019-12-17 12:31:53,454] jepsen worker 29 - jepsen.util 4259 :fail :cas [9 [32 47]] :ValueNotSwapFrom INFO [2019-12-17 12:31:53,454] jepsen worker 29 - jepsen.util 4259 :invoke :read [9 nil] INFO [2019-12-17 12:31:53,459] jepsen worker 13 - jepsen.util 5233 :ok :read [4 27] INFO [2019-12-17 12:31:53,459] jepsen worker 13 - jepsen.util 5233 :invoke :write [4 20] INFO [2019-12-17 12:31:53,483] jepsen worker 63 - jepsen.util 4293 :ok :read [21 24] INFO [2019-12-17 12:31:53,483] jepsen worker 63 - jepsen.util 4293 :invoke :cas [21 [4 21]] INFO [2019-12-17 12:31:53,490] jepsen worker 32 - jepsen.util 5342 :info :write [10 41] :RequestCanceledException INFO [2019-12-17 12:31:53,490] jepsen worker 30 - jepsen.util 5340 :fail :read [10 nil] :RequestTimeoutException INFO [2019-12-17 12:31:53,490] jepsen worker 74 - jepsen.util 5654 :info :write [24 49] :RequestCanceledException INFO [2019-12-17 12:31:53,490] jepsen worker 64 - jepsen.util 3934 :info :write [21 5] :RequestCanceledException INFO [2019-12-17 12:31:53,490] jepsen worker 30 - jepsen.util 5340 :invoke :cas [10 [6 26]] INFO [2019-12-17 12:31:53,491] jepsen worker 32 - jepsen.util 5432 :invoke :cas [10 [12 11]] INFO [2019-12-17 12:31:53,491] jepsen worker 74 - jepsen.util 5744 :invoke :read [24 nil] INFO [2019-12-17 12:31:53,491] jepsen worker 64 - jepsen.util 4024 :invoke :cas [21 [5 22]] INFO [2019-12-17 12:31:53,511] jepsen worker 2 - jepsen.util 6662 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:31:53,512] jepsen worker 2 - jepsen.util 6662 :invoke :cas [0 [35 20]] INFO [2019-12-17 12:31:53,519] jepsen worker 63 - jepsen.util 4293 :fail :cas [21 [4 21]] :ValueNotSwapFrom INFO [2019-12-17 12:31:53,520] jepsen worker 63 - jepsen.util 4293 :invoke :write [21 19] INFO [2019-12-17 12:31:53,526] jepsen worker 44 - jepsen.util 7694 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:31:53,526] jepsen worker 25 - jepsen.util 7405 :info :write [8 43] :RequestCanceledException INFO [2019-12-17 12:31:53,526] jepsen worker 44 - jepsen.util 7694 :invoke :cas [14 [6 29]] INFO [2019-12-17 12:31:53,526] jepsen worker 58 - jepsen.util 7438 :info :write [19 20] :RequestCanceledException INFO [2019-12-17 12:31:53,527] jepsen worker 25 - jepsen.util 7495 :invoke :write [8 41] INFO [2019-12-17 12:31:53,527] jepsen worker 58 - jepsen.util 7528 :invoke :cas [19 [31 39]] INFO [2019-12-17 12:31:53,539] jepsen worker 34 - jepsen.util 6604 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:31:53,539] jepsen worker 36 - jepsen.util 5526 :info :write [12 25] :RequestCanceledException INFO [2019-12-17 12:31:53,539] jepsen worker 34 - jepsen.util 6604 :invoke :read [11 nil] INFO [2019-12-17 12:31:53,540] jepsen worker 36 - jepsen.util 5616 :invoke :cas [12 [34 27]] INFO [2019-12-17 12:31:53,557] jepsen worker 81 - jepsen.util 7011 :info :write [27 49] :RequestCanceledException INFO [2019-12-17 12:31:53,558] jepsen worker 81 - jepsen.util 7101 :invoke :cas [27 [47 36]] INFO [2019-12-17 12:31:53,565] jepsen worker 53 - jepsen.util 5183 :fail :cas [17 [33 35]] :ValueNotSwapFrom INFO [2019-12-17 12:31:53,565] jepsen worker 53 - jepsen.util 5183 :invoke :read [17 nil] INFO [2019-12-17 12:31:53,566] jepsen worker 29 - jepsen.util 4259 :ok :read [9 23] INFO [2019-12-17 12:31:53,566] jepsen worker 29 - jepsen.util 4259 :invoke :write [9 25] INFO [2019-12-17 12:31:53,578] jepsen worker 38 - jepsen.util 5348 :fail :cas [12 [47 20]] :ValueNotSwapFrom INFO [2019-12-17 12:31:53,578] jepsen worker 38 - jepsen.util 5348 :invoke :read [12 nil] INFO [2019-12-17 12:31:53,592] jepsen worker 10 - jepsen.util 5230 :fail :read [3 nil] :RequestTimeoutException INFO [2019-12-17 12:31:53,592] jepsen worker 10 - jepsen.util 5230 :invoke :write [3 31] INFO [2019-12-17 12:31:53,610] jepsen worker 61 - jepsen.util 7891 :info :write [20 29] :RequestCanceledException INFO [2019-12-17 12:31:53,612] jepsen worker 61 - jepsen.util 7981 :invoke :cas [20 [12 22]] INFO [2019-12-17 12:31:53,625] jepsen worker 89 - jepsen.util 5039 :info :write [29 47] :RequestCanceledException INFO [2019-12-17 12:31:53,625] jepsen worker 56 - jepsen.util 5456 :info :write [18 32] :RequestCanceledException INFO [2019-12-17 12:31:53,626] jepsen worker 89 - jepsen.util 5129 :invoke :read [29 nil] INFO [2019-12-17 12:31:53,626] jepsen worker 56 - jepsen.util 5546 :invoke :write [18 47] INFO [2019-12-17 12:31:53,626] jepsen worker 86 - jepsen.util 7376 :info :write [28 18] :RequestCanceledException INFO [2019-12-17 12:31:53,627] jepsen worker 86 - jepsen.util 7466 :invoke :read [28 nil] INFO [2019-12-17 12:31:53,638] jepsen worker 79 - jepsen.util 5209 :fail :cas [26 [22 37]] :ValueNotSwapFrom INFO [2019-12-17 12:31:53,638] jepsen worker 79 - jepsen.util 5209 :invoke :write [26 7] INFO [2019-12-17 12:31:53,640] jepsen worker 0 - jepsen.util 7650 :info :write [0 26] :RequestCanceledException INFO [2019-12-17 12:31:53,641] jepsen worker 0 - jepsen.util 7740 :invoke :read [0 nil] INFO [2019-12-17 12:31:53,647] jepsen worker 74 - jepsen.util 5744 :ok :read [24 36] INFO [2019-12-17 12:31:53,647] jepsen worker 74 - jepsen.util 5744 :invoke :write [24 15] INFO [2019-12-17 12:31:53,658] jepsen worker 30 - jepsen.util 5340 :fail :cas [10 [6 26]] :ValueNotSwapFrom INFO [2019-12-17 12:31:53,658] jepsen worker 30 - jepsen.util 5340 :invoke :cas [10 [16 6]] INFO [2019-12-17 12:31:53,668] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":30505,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:31:53,669] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":30505,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:31:53,669] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":30507,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:31:53,669] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":30507,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:31:53,671] jepsen worker 64 - jepsen.util 4024 :fail :cas [21 [5 22]] :ValueNotSwapFrom INFO [2019-12-17 12:31:53,671] jepsen worker 32 - jepsen.util 5432 :fail :cas [10 [12 11]] :ValueNotSwapFrom INFO [2019-12-17 12:31:53,671] jepsen worker 64 - jepsen.util 4024 :invoke :cas [21 [27 15]] INFO [2019-12-17 12:31:53,671] jepsen worker 32 - jepsen.util 5432 :invoke :read [10 nil] INFO [2019-12-17 12:31:53,677] jepsen worker 51 - jepsen.util 5271 :ok :read [17 45] INFO [2019-12-17 12:31:53,677] jepsen worker 51 - jepsen.util 5271 :invoke :write [17 3] INFO [2019-12-17 12:31:53,688] jepsen worker 9 - jepsen.util 5859 :ok :write [3 8] INFO [2019-12-17 12:31:53,689] jepsen worker 9 - jepsen.util 5859 :invoke :read [3 nil] INFO [2019-12-17 12:31:53,690] jepsen worker 32 - jepsen.util 5432 :ok :read [10 2] INFO [2019-12-17 12:31:53,691] jepsen worker 32 - jepsen.util 5432 :invoke :cas [10 [37 43]] INFO [2019-12-17 12:31:53,710] jepsen worker 78 - jepsen.util 5388 :info :write [26 29] :RequestCanceledException INFO [2019-12-17 12:31:53,711] jepsen worker 78 - jepsen.util 5478 :invoke :write [26 4] INFO [2019-12-17 12:31:53,726] jepsen worker 62 - jepsen.util 6722 :info :cas [20 [8 48]] :RequestCanceledException INFO [2019-12-17 12:31:53,729] jepsen worker 62 - jepsen.util 6812 :invoke :write [20 19] INFO [2019-12-17 12:31:53,740] jepsen worker 67 - jepsen.util 7177 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:31:53,740] jepsen worker 67 - jepsen.util 7177 :invoke :cas [22 [18 26]] INFO [2019-12-17 12:31:53,741] jepsen worker 9 - jepsen.util 5859 :ok :read [3 8] INFO [2019-12-17 12:31:53,741] jepsen worker 9 - jepsen.util 5859 :invoke :read [3 nil] INFO [2019-12-17 12:31:53,746] jepsen worker 32 - jepsen.util 5432 :fail :cas [10 [37 43]] :ValueNotSwapFrom INFO [2019-12-17 12:31:53,746] jepsen worker 32 - jepsen.util 5432 :invoke :read [10 nil] INFO [2019-12-17 12:31:53,748] jepsen worker 19 - jepsen.util 5239 :ok :write [6 25] INFO [2019-12-17 12:31:53,749] jepsen worker 19 - jepsen.util 5239 :invoke :cas [6 [22 0]] INFO [2019-12-17 12:31:53,764] jepsen worker 53 - jepsen.util 5183 :ok :read [17 45] INFO [2019-12-17 12:31:53,764] jepsen worker 53 - jepsen.util 5183 :invoke :read [17 nil] INFO [2019-12-17 12:31:53,766] jepsen worker 4 - jepsen.util 4504 :ok :write [1 41] INFO [2019-12-17 12:31:53,766] jepsen worker 4 - jepsen.util 4504 :invoke :write [1 37] INFO [2019-12-17 12:31:53,775] jepsen worker 12 - jepsen.util 5592 :ok :write [4 2] INFO [2019-12-17 12:31:53,775] jepsen worker 12 - jepsen.util 5592 :invoke :write [4 32] INFO [2019-12-17 12:31:53,781] jepsen worker 9 - jepsen.util 5859 :ok :read [3 8] INFO [2019-12-17 12:31:53,781] jepsen worker 9 - jepsen.util 5859 :invoke :cas [3 [9 26]] INFO [2019-12-17 12:31:53,782] jepsen worker 32 - jepsen.util 5432 :ok :read [10 2] INFO [2019-12-17 12:31:53,782] jepsen worker 32 - jepsen.util 5432 :invoke :write [10 32] INFO [2019-12-17 12:31:53,790] jepsen worker 73 - jepsen.util 4933 :fail :cas [24 [2 40]] :ValueNotSwapFrom INFO [2019-12-17 12:31:53,790] jepsen worker 73 - jepsen.util 4933 :invoke :read [24 nil] INFO [2019-12-17 12:31:53,795] jepsen worker 38 - jepsen.util 5348 :ok :read [12 29] INFO [2019-12-17 12:31:53,795] jepsen worker 38 - jepsen.util 5348 :invoke :write [12 38] INFO [2019-12-17 12:31:53,825] jepsen worker 55 - jepsen.util 6265 :info :cas [18 [40 32]] :RequestCanceledException INFO [2019-12-17 12:31:53,826] jepsen worker 55 - jepsen.util 6355 :invoke :write [18 26] INFO [2019-12-17 12:31:53,838] jepsen worker 9 - jepsen.util 5859 :fail :cas [3 [9 26]] :ValueNotSwapFrom INFO [2019-12-17 12:31:53,838] jepsen worker 9 - jepsen.util 5859 :invoke :cas [3 [12 41]] INFO [2019-12-17 12:31:53,845] jepsen worker 14 - jepsen.util 5414 :ok :write [4 29] INFO [2019-12-17 12:31:53,845] jepsen worker 14 - jepsen.util 5414 :invoke :write [4 21] INFO [2019-12-17 12:31:53,857] jepsen worker 26 - jepsen.util 7496 :info :cas [8 [5 13]] :RequestCanceledException INFO [2019-12-17 12:31:53,858] jepsen worker 26 - jepsen.util 7586 :invoke :read [8 nil] INFO [2019-12-17 12:31:53,866] jepsen worker 19 - jepsen.util 5239 :fail :cas [6 [22 0]] :ValueNotSwapFrom INFO [2019-12-17 12:31:53,870] jepsen worker 19 - jepsen.util 5239 :invoke :write [6 21] INFO [2019-12-17 12:31:53,891] jepsen worker 29 - jepsen.util 4259 :ok :write [9 25] INFO [2019-12-17 12:31:53,891] jepsen worker 29 - jepsen.util 4259 :invoke :read [9 nil] INFO [2019-12-17 12:31:53,919] jepsen worker 87 - jepsen.util 4767 :ok :write [29 36] INFO [2019-12-17 12:31:53,919] jepsen worker 87 - jepsen.util 4767 :invoke :read [29 nil] INFO [2019-12-17 12:31:53,926] jepsen worker 21 - jepsen.util 6591 :info :cas [7 [8 49]] :RequestCanceledException INFO [2019-12-17 12:31:53,927] jepsen worker 21 - jepsen.util 6681 :invoke :write [7 7] INFO [2019-12-17 12:31:53,936] jepsen worker 63 - jepsen.util 4293 :ok :write [21 19] INFO [2019-12-17 12:31:53,936] jepsen worker 63 - jepsen.util 4293 :invoke :write [21 27] INFO [2019-12-17 12:31:53,958] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":30537,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:31:53,958] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":30537,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:31:53,958] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":30541,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:31:53,958] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":30541,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:31:53,976] jepsen worker 9 - jepsen.util 5859 :fail :cas [3 [12 41]] :ValueNotSwapFrom INFO [2019-12-17 12:31:53,976] jepsen worker 9 - jepsen.util 5859 :invoke :cas [3 [12 15]] INFO [2019-12-17 12:31:53,976] jepsen worker 73 - jepsen.util 4933 :ok :read [24 36] INFO [2019-12-17 12:31:53,976] jepsen worker 73 - jepsen.util 4933 :invoke :write [24 2] INFO [2019-12-17 12:31:53,976] jepsen worker 30 - jepsen.util 5340 :fail :cas [10 [16 6]] :ValueNotSwapFrom INFO [2019-12-17 12:31:53,977] jepsen worker 30 - jepsen.util 5340 :invoke :cas [10 [41 39]] INFO [2019-12-17 12:31:53,977] jepsen worker 36 - jepsen.util 5616 :fail :cas [12 [34 27]] :ValueNotSwapFrom INFO [2019-12-17 12:31:53,977] jepsen worker 36 - jepsen.util 5616 :invoke :cas [12 [25 12]] INFO [2019-12-17 12:31:53,992] jepsen worker 1 - jepsen.util 6931 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:31:53,992] jepsen worker 1 - jepsen.util 6931 :invoke :write [0 39] INFO [2019-12-17 12:31:53,995] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":30553,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:31:53,996] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":30553,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:31:53,996] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":30556,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:31:53,996] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":30556,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:31:53,996] jepsen worker 64 - jepsen.util 4024 :fail :cas [21 [27 15]] :ValueNotSwapFrom INFO [2019-12-17 12:31:53,996] jepsen worker 64 - jepsen.util 4024 :invoke :cas [21 [4 27]] INFO [2019-12-17 12:31:54,009] jepsen worker 9 - jepsen.util 5859 :fail :cas [3 [12 15]] :ValueNotSwapFrom INFO [2019-12-17 12:31:54,009] jepsen worker 9 - jepsen.util 5859 :invoke :write [3 26] INFO [2019-12-17 12:31:54,010] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:31:54,010] jepsen nemesis - couchbase.util Rebalance check count: 229 INFO [2019-12-17 12:31:54,010] jepsen nemesis - couchbase.util Rebalance stuck count: 210 INFO [2019-12-17 12:31:54,010] jepsen worker 7 - jepsen.util 7207 :info :cas [2 [24 25]] :RequestCanceledException INFO [2019-12-17 12:31:54,011] jepsen worker 7 - jepsen.util 7297 :invoke :cas [2 [37 8]] INFO [2019-12-17 12:31:54,021] jepsen worker 29 - jepsen.util 4259 :ok :read [9 25] INFO [2019-12-17 12:31:54,021] jepsen worker 29 - jepsen.util 4259 :invoke :write [9 16] INFO [2019-12-17 12:31:54,038] jepsen worker 30 - jepsen.util 5340 :fail :cas [10 [41 39]] :ValueNotSwapFrom INFO [2019-12-17 12:31:54,038] jepsen worker 30 - jepsen.util 5340 :invoke :write [10 29] INFO [2019-12-17 12:31:54,048] jepsen worker 89 - jepsen.util 5129 :ok :read [29 36] INFO [2019-12-17 12:31:54,049] jepsen worker 89 - jepsen.util 5129 :invoke :read [29 nil] INFO [2019-12-17 12:31:54,055] jepsen worker 64 - jepsen.util 4024 :fail :cas [21 [4 27]] :ValueNotSwapFrom INFO [2019-12-17 12:31:54,055] jepsen worker 64 - jepsen.util 4024 :invoke :read [21 nil] INFO [2019-12-17 12:31:54,083] jepsen worker 32 - jepsen.util 5432 :ok :write [10 32] INFO [2019-12-17 12:31:54,083] jepsen worker 32 - jepsen.util 5432 :invoke :read [10 nil] INFO [2019-12-17 12:31:54,101] jepsen worker 32 - jepsen.util 5432 :ok :read [10 32] INFO [2019-12-17 12:31:54,101] jepsen worker 32 - jepsen.util 5432 :invoke :cas [10 [4 47]] INFO [2019-12-17 12:31:54,110] jepsen worker 37 - jepsen.util 4897 :info :write [12 34] :RequestCanceledException INFO [2019-12-17 12:31:54,112] jepsen worker 37 - jepsen.util 4987 :invoke :cas [12 [35 7]] INFO [2019-12-17 12:31:54,116] jepsen worker 38 - jepsen.util 5348 :ok :write [12 38] INFO [2019-12-17 12:31:54,116] jepsen worker 38 - jepsen.util 5348 :invoke :write [12 1] INFO [2019-12-17 12:31:54,125] jepsen worker 35 - jepsen.util 7235 :info :write [11 19] :RequestCanceledException INFO [2019-12-17 12:31:54,126] jepsen worker 35 - jepsen.util 7325 :invoke :write [11 40] INFO [2019-12-17 12:31:54,128] jepsen worker 32 - jepsen.util 5432 :fail :cas [10 [4 47]] :ValueNotSwapFrom INFO [2019-12-17 12:31:54,128] jepsen worker 32 - jepsen.util 5432 :invoke :cas [10 [47 3]] INFO [2019-12-17 12:31:54,139] jepsen worker 42 - jepsen.util 7782 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:31:54,140] jepsen worker 42 - jepsen.util 7782 :invoke :read [14 nil] INFO [2019-12-17 12:31:54,160] jepsen worker 51 - jepsen.util 5271 :ok :write [17 3] INFO [2019-12-17 12:31:54,160] jepsen worker 51 - jepsen.util 5271 :invoke :cas [17 [48 24]] INFO [2019-12-17 12:31:54,164] jepsen worker 32 - jepsen.util 5432 :fail :cas [10 [47 3]] :ValueNotSwapFrom INFO [2019-12-17 12:31:54,164] jepsen worker 32 - jepsen.util 5432 :invoke :read [10 nil] INFO [2019-12-17 12:31:54,173] jepsen worker 3 - jepsen.util 5313 :ok :write [1 44] INFO [2019-12-17 12:31:54,173] jepsen worker 3 - jepsen.util 5313 :invoke :read [1 nil] INFO [2019-12-17 12:31:54,182] jepsen worker 63 - jepsen.util 4293 :ok :write [21 27] INFO [2019-12-17 12:31:54,182] jepsen worker 63 - jepsen.util 4293 :invoke :read [21 nil] INFO [2019-12-17 12:31:54,193] jepsen worker 50 - jepsen.util 7340 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:31:54,193] jepsen worker 50 - jepsen.util 7340 :invoke :read [16 nil] INFO [2019-12-17 12:31:54,209] jepsen worker 32 - jepsen.util 5432 :ok :read [10 32] INFO [2019-12-17 12:31:54,209] jepsen worker 32 - jepsen.util 5432 :invoke :cas [10 [45 34]] INFO [2019-12-17 12:31:54,211] jepsen worker 88 - jepsen.util 5488 :info :write [29 14] :RequestCanceledException INFO [2019-12-17 12:31:54,212] jepsen worker 88 - jepsen.util 5578 :invoke :read [29 nil] INFO [2019-12-17 12:31:54,215] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":30577,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:31:54,215] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":30585,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:31:54,215] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":30577,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:31:54,215] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":30585,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:31:54,215] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":30579,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:31:54,215] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":30589,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:31:54,215] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":30579,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:31:54,215] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":30589,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:31:54,225] jepsen worker 24 - jepsen.util 6594 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:31:54,226] jepsen worker 24 - jepsen.util 6594 :invoke :cas [8 [13 23]] INFO [2019-12-17 12:31:54,226] jepsen worker 29 - jepsen.util 4259 :ok :write [9 16] INFO [2019-12-17 12:31:54,226] jepsen worker 29 - jepsen.util 4259 :invoke :read [9 nil] INFO [2019-12-17 12:31:54,230] jepsen worker 54 - jepsen.util 5724 :fail :cas [18 [17 37]] :ValueNotSwapFrom INFO [2019-12-17 12:31:54,230] jepsen worker 63 - jepsen.util 4293 :ok :read [21 27] INFO [2019-12-17 12:31:54,230] jepsen worker 54 - jepsen.util 5724 :invoke :cas [18 [20 47]] INFO [2019-12-17 12:31:54,230] jepsen worker 63 - jepsen.util 4293 :invoke :cas [21 [49 29]] INFO [2019-12-17 12:31:54,252] jepsen worker 32 - jepsen.util 5432 :fail :cas [10 [45 34]] :ValueNotSwapFrom INFO [2019-12-17 12:31:54,253] jepsen worker 32 - jepsen.util 5432 :invoke :read [10 nil] INFO [2019-12-17 12:31:54,257] jepsen worker 23 - jepsen.util 7313 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:31:54,257] jepsen worker 49 - jepsen.util 6259 :info :cas [16 [3 30]] :RequestCanceledException INFO [2019-12-17 12:31:54,257] jepsen worker 23 - jepsen.util 7313 :invoke :cas [7 [32 9]] INFO [2019-12-17 12:31:54,258] jepsen worker 49 - jepsen.util 6349 :invoke :cas [16 [13 19]] INFO [2019-12-17 12:31:54,270] jepsen worker 29 - jepsen.util 4259 :ok :read [9 16] INFO [2019-12-17 12:31:54,270] jepsen worker 29 - jepsen.util 4259 :invoke :write [9 5] INFO [2019-12-17 12:31:54,288] jepsen worker 63 - jepsen.util 4293 :fail :cas [21 [49 29]] :ValueNotSwapFrom INFO [2019-12-17 12:31:54,288] jepsen worker 63 - jepsen.util 4293 :invoke :cas [21 [42 6]] INFO [2019-12-17 12:31:54,292] jepsen worker 32 - jepsen.util 5432 :ok :read [10 32] INFO [2019-12-17 12:31:54,292] jepsen worker 32 - jepsen.util 5432 :invoke :read [10 nil] INFO [2019-12-17 12:31:54,300] jepsen worker 4 - jepsen.util 4504 :ok :write [1 37] INFO [2019-12-17 12:31:54,300] jepsen worker 4 - jepsen.util 4504 :invoke :read [1 nil] INFO [2019-12-17 12:31:54,310] jepsen worker 59 - jepsen.util 6989 :info :write [19 29] :RequestCanceledException INFO [2019-12-17 12:31:54,311] jepsen worker 59 - jepsen.util 7079 :invoke :write [19 36] INFO [2019-12-17 12:31:54,324] jepsen worker 19 - jepsen.util 5239 :ok :write [6 21] INFO [2019-12-17 12:31:54,324] jepsen worker 70 - jepsen.util 5920 :info :write [23 0] :RequestCanceledException INFO [2019-12-17 12:31:54,324] jepsen worker 19 - jepsen.util 5239 :invoke :cas [6 [8 17]] INFO [2019-12-17 12:31:54,326] jepsen worker 70 - jepsen.util 6010 :invoke :cas [23 [35 11]] INFO [2019-12-17 12:31:54,326] jepsen worker 88 - jepsen.util 5578 :ok :read [29 36] INFO [2019-12-17 12:31:54,326] jepsen worker 88 - jepsen.util 5578 :invoke :read [29 nil] INFO [2019-12-17 12:31:54,361] jepsen worker 9 - jepsen.util 5859 :ok :write [3 26] INFO [2019-12-17 12:31:54,361] jepsen worker 9 - jepsen.util 5859 :invoke :write [3 12] INFO [2019-12-17 12:31:54,386] jepsen worker 36 - jepsen.util 5616 :fail :cas [12 [25 12]] :ValueNotSwapFrom INFO [2019-12-17 12:31:54,386] jepsen worker 32 - jepsen.util 5432 :ok :read [10 32] INFO [2019-12-17 12:31:54,386] jepsen worker 36 - jepsen.util 5616 :invoke :read [12 nil] INFO [2019-12-17 12:31:54,387] jepsen worker 32 - jepsen.util 5432 :invoke :read [10 nil] INFO [2019-12-17 12:31:54,397] jepsen worker 63 - jepsen.util 4293 :fail :cas [21 [42 6]] :ValueNotSwapFrom INFO [2019-12-17 12:31:54,397] jepsen worker 63 - jepsen.util 4293 :invoke :cas [21 [11 20]] INFO [2019-12-17 12:31:54,415] jepsen worker 78 - jepsen.util 5478 :ok :write [26 4] INFO [2019-12-17 12:31:54,415] jepsen worker 78 - jepsen.util 5478 :invoke :cas [26 [15 49]] INFO [2019-12-17 12:31:54,424] jepsen worker 31 - jepsen.util 4981 :fail :read [10 nil] :RequestTimeoutException INFO [2019-12-17 12:31:54,424] jepsen worker 65 - jepsen.util 4565 :fail :read [21 nil] :RequestTimeoutException INFO [2019-12-17 12:31:54,424] jepsen worker 31 - jepsen.util 4981 :invoke :cas [10 [49 16]] INFO [2019-12-17 12:31:54,424] jepsen worker 65 - jepsen.util 4565 :invoke :cas [21 [18 24]] INFO [2019-12-17 12:31:54,428] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":30614,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:31:54,428] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":30614,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:31:54,428] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":30617,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:31:54,428] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":30617,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:31:54,428] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":30618,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:31:54,428] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":30618,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:31:54,433] jepsen worker 37 - jepsen.util 4987 :fail :cas [12 [35 7]] :ValueNotSwapFrom INFO [2019-12-17 12:31:54,433] jepsen worker 37 - jepsen.util 4987 :invoke :cas [12 [19 29]] INFO [2019-12-17 12:31:54,454] jepsen worker 54 - jepsen.util 5724 :fail :cas [18 [20 47]] :ValueNotSwapFrom INFO [2019-12-17 12:31:54,454] jepsen worker 54 - jepsen.util 5724 :invoke :read [18 nil] INFO [2019-12-17 12:31:54,475] jepsen worker 32 - jepsen.util 5432 :ok :read [10 32] INFO [2019-12-17 12:31:54,475] jepsen worker 32 - jepsen.util 5432 :invoke :read [10 nil] INFO [2019-12-17 12:31:54,480] jepsen worker 63 - jepsen.util 4293 :fail :cas [21 [11 20]] :ValueNotSwapFrom INFO [2019-12-17 12:31:54,480] jepsen worker 63 - jepsen.util 4293 :invoke :write [21 9] INFO [2019-12-17 12:31:54,489] jepsen worker 53 - jepsen.util 5183 :ok :read [17 3] INFO [2019-12-17 12:31:54,489] jepsen worker 53 - jepsen.util 5183 :invoke :read [17 nil] INFO [2019-12-17 12:31:54,492] jepsen worker 52 - jepsen.util 5632 :ok :write [17 38] INFO [2019-12-17 12:31:54,492] jepsen worker 52 - jepsen.util 5632 :invoke :write [17 20] INFO [2019-12-17 12:31:54,517] jepsen worker 78 - jepsen.util 5478 :fail :cas [26 [15 49]] :ValueNotSwapFrom INFO [2019-12-17 12:31:54,518] jepsen worker 78 - jepsen.util 5478 :invoke :cas [26 [23 25]] INFO [2019-12-17 12:31:54,540] jepsen worker 31 - jepsen.util 4981 :fail :cas [10 [49 16]] :ValueNotSwapFrom INFO [2019-12-17 12:31:54,540] jepsen worker 31 - jepsen.util 4981 :invoke :read [10 nil] INFO [2019-12-17 12:31:54,546] jepsen worker 53 - jepsen.util 5183 :ok :read [17 38] INFO [2019-12-17 12:31:54,546] jepsen worker 53 - jepsen.util 5183 :invoke :read [17 nil] INFO [2019-12-17 12:31:54,549] jepsen worker 32 - jepsen.util 5432 :ok :read [10 32] INFO [2019-12-17 12:31:54,549] jepsen worker 32 - jepsen.util 5432 :invoke :read [10 nil] INFO [2019-12-17 12:31:54,553] jepsen worker 37 - jepsen.util 4987 :fail :cas [12 [19 29]] :ValueNotSwapFrom INFO [2019-12-17 12:31:54,554] jepsen worker 37 - jepsen.util 4987 :invoke :cas [12 [4 11]] INFO [2019-12-17 12:31:54,555] jepsen worker 54 - jepsen.util 5724 :ok :read [18 12] INFO [2019-12-17 12:31:54,555] jepsen worker 54 - jepsen.util 5724 :invoke :cas [18 [7 15]] INFO [2019-12-17 12:31:54,556] jepsen worker 29 - jepsen.util 4259 :ok :write [9 5] INFO [2019-12-17 12:31:54,556] jepsen worker 29 - jepsen.util 4259 :invoke :cas [9 [46 37]] INFO [2019-12-17 12:31:54,574] jepsen worker 65 - jepsen.util 4565 :fail :cas [21 [18 24]] :ValueNotSwapFrom INFO [2019-12-17 12:31:54,575] jepsen worker 65 - jepsen.util 4565 :invoke :cas [21 [6 37]] INFO [2019-12-17 12:31:54,587] jepsen worker 32 - jepsen.util 5432 :ok :read [10 32] INFO [2019-12-17 12:31:54,587] jepsen worker 32 - jepsen.util 5432 :invoke :cas [10 [13 0]] INFO [2019-12-17 12:31:54,589] jepsen worker 3 - jepsen.util 5313 :ok :read [1 37] INFO [2019-12-17 12:31:54,589] jepsen worker 3 - jepsen.util 5313 :invoke :read [1 nil] INFO [2019-12-17 12:31:54,592] jepsen worker 29 - jepsen.util 4259 :fail :cas [9 [46 37]] :ValueNotSwapFrom INFO [2019-12-17 12:31:54,592] jepsen worker 29 - jepsen.util 4259 :invoke :cas [9 [8 16]] INFO [2019-12-17 12:31:54,604] jepsen worker 54 - jepsen.util 5724 :fail :cas [18 [7 15]] :ValueNotSwapFrom INFO [2019-12-17 12:31:54,604] jepsen worker 54 - jepsen.util 5724 :invoke :read [18 nil] INFO [2019-12-17 12:31:54,609] jepsen worker 78 - jepsen.util 5478 :fail :cas [26 [23 25]] :ValueNotSwapFrom INFO [2019-12-17 12:31:54,609] jepsen worker 78 - jepsen.util 5478 :invoke :read [26 nil] INFO [2019-12-17 12:31:54,624] jepsen worker 32 - jepsen.util 5432 :fail :cas [10 [13 0]] :ValueNotSwapFrom INFO [2019-12-17 12:31:54,624] jepsen worker 32 - jepsen.util 5432 :invoke :cas [10 [15 48]] INFO [2019-12-17 12:31:54,626] jepsen worker 46 - jepsen.util 5176 :info :write [15 7] :RequestCanceledException INFO [2019-12-17 12:31:54,627] jepsen worker 46 - jepsen.util 5266 :invoke :cas [15 [47 6]] INFO [2019-12-17 12:31:54,629] jepsen worker 37 - jepsen.util 4987 :fail :cas [12 [4 11]] :ValueNotSwapFrom INFO [2019-12-17 12:31:54,629] jepsen worker 37 - jepsen.util 4987 :invoke :write [12 23] INFO [2019-12-17 12:31:54,640] jepsen worker 29 - jepsen.util 4259 :fail :cas [9 [8 16]] :ValueNotSwapFrom INFO [2019-12-17 12:31:54,640] jepsen worker 29 - jepsen.util 4259 :invoke :cas [9 [21 19]] INFO [2019-12-17 12:31:54,653] jepsen worker 78 - jepsen.util 5478 :ok :read [26 4] INFO [2019-12-17 12:31:54,654] jepsen worker 78 - jepsen.util 5478 :invoke :read [26 nil] INFO [2019-12-17 12:31:54,654] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":30643,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:31:54,654] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":30643,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:31:54,654] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":30647,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:31:54,654] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":30647,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:31:54,703] jepsen worker 70 - jepsen.util 6010 :fail :cas [23 [35 11]] :ValueNotSwapFrom INFO [2019-12-17 12:31:54,703] jepsen worker 70 - jepsen.util 6010 :invoke :read [23 nil] INFO [2019-12-17 12:31:54,704] jepsen worker 32 - jepsen.util 5432 :fail :cas [10 [15 48]] :ValueNotSwapFrom INFO [2019-12-17 12:31:54,704] jepsen worker 32 - jepsen.util 5432 :invoke :cas [10 [48 41]] INFO [2019-12-17 12:31:54,720] jepsen worker 54 - jepsen.util 5724 :ok :read [18 12] INFO [2019-12-17 12:31:54,720] jepsen worker 54 - jepsen.util 5724 :invoke :write [18 18] INFO [2019-12-17 12:31:54,726] jepsen worker 20 - jepsen.util 5330 :info :write [6 44] :RequestCanceledException INFO [2019-12-17 12:31:54,726] jepsen worker 11 - jepsen.util 5411 :info :cas [3 [17 6]] :RequestCanceledException INFO [2019-12-17 12:31:54,727] jepsen worker 20 - jepsen.util 5420 :invoke :cas [6 [20 23]] INFO [2019-12-17 12:31:54,728] jepsen worker 11 - jepsen.util 5501 :invoke :write [3 26] INFO [2019-12-17 12:31:54,733] jepsen worker 36 - jepsen.util 5616 :ok :read [12 38] INFO [2019-12-17 12:31:54,733] jepsen worker 36 - jepsen.util 5616 :invoke :write [12 6] INFO [2019-12-17 12:31:54,808] jepsen worker 78 - jepsen.util 5478 :ok :read [26 4] INFO [2019-12-17 12:31:54,808] jepsen worker 78 - jepsen.util 5478 :invoke :write [26 18] INFO [2019-12-17 12:31:54,810] jepsen worker 72 - jepsen.util 5022 :info :cas [24 [32 9]] :RequestCanceledException INFO [2019-12-17 12:31:54,812] jepsen worker 72 - jepsen.util 5112 :invoke :write [24 21] INFO [2019-12-17 12:31:54,834] jepsen worker 29 - jepsen.util 4259 :fail :cas [9 [21 19]] :ValueNotSwapFrom INFO [2019-12-17 12:31:54,834] jepsen worker 29 - jepsen.util 4259 :invoke :read [9 nil] INFO [2019-12-17 12:31:54,837] jepsen worker 46 - jepsen.util 5266 :fail :cas [15 [47 6]] :ValueNotSwapFrom INFO [2019-12-17 12:31:54,837] jepsen worker 46 - jepsen.util 5266 :invoke :cas [15 [40 23]] INFO [2019-12-17 12:31:54,851] jepsen worker 63 - jepsen.util 4293 :ok :write [21 9] INFO [2019-12-17 12:31:54,852] jepsen worker 63 - jepsen.util 4293 :invoke :cas [21 [37 38]] INFO [2019-12-17 12:31:54,877] jepsen worker 12 - jepsen.util 5592 :ok :write [4 32] INFO [2019-12-17 12:31:54,877] jepsen worker 12 - jepsen.util 5592 :invoke :read [4 nil] INFO [2019-12-17 12:31:54,897] jepsen worker 65 - jepsen.util 4565 :fail :cas [21 [6 37]] :ValueNotSwapFrom INFO [2019-12-17 12:31:54,897] jepsen worker 65 - jepsen.util 4565 :invoke :write [21 23] INFO [2019-12-17 12:31:54,904] jepsen worker 55 - jepsen.util 6355 :ok :write [18 26] INFO [2019-12-17 12:31:54,904] jepsen worker 55 - jepsen.util 6355 :invoke :read [18 nil] INFO [2019-12-17 12:31:54,908] jepsen worker 32 - jepsen.util 5432 :fail :cas [10 [48 41]] :ValueNotSwapFrom INFO [2019-12-17 12:31:54,908] jepsen worker 32 - jepsen.util 5432 :invoke :read [10 nil] INFO [2019-12-17 12:31:54,927] jepsen worker 87 - jepsen.util 4767 :ok :read [29 36] INFO [2019-12-17 12:31:54,927] jepsen worker 87 - jepsen.util 4767 :invoke :read [29 nil] INFO [2019-12-17 12:31:54,934] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":30663,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:31:54,934] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":30663,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:31:54,934] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":30665,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:31:54,935] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":30665,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:31:54,976] jepsen worker 46 - jepsen.util 5266 :fail :cas [15 [40 23]] :ValueNotSwapFrom INFO [2019-12-17 12:31:54,977] jepsen worker 46 - jepsen.util 5266 :invoke :cas [15 [25 28]] INFO [2019-12-17 12:31:54,986] jepsen worker 3 - jepsen.util 5313 :ok :read [1 37] INFO [2019-12-17 12:31:54,986] jepsen worker 3 - jepsen.util 5313 :invoke :write [1 27] INFO [2019-12-17 12:31:54,991] jepsen worker 71 - jepsen.util 6101 :ok :write [23 20] INFO [2019-12-17 12:31:54,991] jepsen worker 71 - jepsen.util 6101 :invoke :cas [23 [12 20]] INFO [2019-12-17 12:31:55,017] jepsen worker 70 - jepsen.util 6010 :ok :read [23 20] INFO [2019-12-17 12:31:55,017] jepsen worker 70 - jepsen.util 6010 :invoke :read [23 nil] INFO [2019-12-17 12:31:55,026] jepsen worker 32 - jepsen.util 5432 :ok :read [10 32] INFO [2019-12-17 12:31:55,027] jepsen worker 12 - jepsen.util 5592 :ok :read [4 32] INFO [2019-12-17 12:31:55,027] jepsen worker 32 - jepsen.util 5432 :invoke :write [10 35] INFO [2019-12-17 12:31:55,027] jepsen worker 12 - jepsen.util 5592 :invoke :cas [4 [36 2]] INFO [2019-12-17 12:31:55,030] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:31:55,030] jepsen nemesis - couchbase.util Rebalance check count: 230 INFO [2019-12-17 12:31:55,030] jepsen nemesis - couchbase.util Rebalance stuck count: 211 INFO [2019-12-17 12:31:55,031] jepsen worker 46 - jepsen.util 5266 :fail :cas [15 [25 28]] :ValueNotSwapFrom INFO [2019-12-17 12:31:55,031] jepsen worker 46 - jepsen.util 5266 :invoke :read [15 nil] INFO [2019-12-17 12:31:55,067] jepsen worker 20 - jepsen.util 5420 :fail :cas [6 [20 23]] :ValueNotSwapFrom INFO [2019-12-17 12:31:55,068] jepsen worker 20 - jepsen.util 5420 :invoke :write [6 14] INFO [2019-12-17 12:31:55,077] jepsen worker 51 - jepsen.util 5271 :fail :cas [17 [48 24]] :ValueNotSwapFrom INFO [2019-12-17 12:31:55,077] jepsen worker 51 - jepsen.util 5271 :invoke :read [17 nil] INFO [2019-12-17 12:31:55,087] jepsen worker 73 - jepsen.util 4933 :ok :write [24 2] INFO [2019-12-17 12:31:55,087] jepsen worker 73 - jepsen.util 4933 :invoke :cas [24 [1 44]] INFO [2019-12-17 12:31:55,088] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":30674,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:31:55,088] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":30674,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:31:55,088] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":30676,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:31:55,088] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":30676,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:31:55,097] jepsen worker 46 - jepsen.util 5266 :ok :read [15 26] INFO [2019-12-17 12:31:55,097] jepsen worker 46 - jepsen.util 5266 :invoke :write [15 27] INFO [2019-12-17 12:31:55,103] jepsen worker 89 - jepsen.util 5129 :ok :read [29 36] INFO [2019-12-17 12:31:55,104] jepsen worker 89 - jepsen.util 5129 :invoke :read [29 nil] INFO [2019-12-17 12:31:55,139] jepsen worker 5 - jepsen.util 4865 :info :write [1 30] :RequestCanceledException INFO [2019-12-17 12:31:55,140] jepsen worker 5 - jepsen.util 4955 :invoke :write [1 34] INFO [2019-12-17 12:31:55,156] jepsen worker 8 - jepsen.util 6668 :info :cas [2 [35 1]] :RequestCanceledException INFO [2019-12-17 12:31:55,157] jepsen worker 8 - jepsen.util 6758 :invoke :read [2 nil] INFO [2019-12-17 12:31:55,161] jepsen worker 3 - jepsen.util 5313 :ok :write [1 27] INFO [2019-12-17 12:31:55,161] jepsen worker 3 - jepsen.util 5313 :invoke :write [1 19] INFO [2019-12-17 12:31:55,171] jepsen worker 53 - jepsen.util 5183 :ok :read [17 38] INFO [2019-12-17 12:31:55,171] jepsen worker 53 - jepsen.util 5183 :invoke :read [17 nil] INFO [2019-12-17 12:31:55,180] jepsen worker 13 - jepsen.util 5233 :ok :write [4 20] INFO [2019-12-17 12:31:55,181] jepsen worker 13 - jepsen.util 5233 :invoke :cas [4 [36 28]] INFO [2019-12-17 12:31:55,192] jepsen worker 68 - jepsen.util 6638 :info :cas [22 [44 22]] :RequestCanceledException INFO [2019-12-17 12:31:55,193] jepsen worker 68 - jepsen.util 6728 :invoke :write [22 13] INFO [2019-12-17 12:31:55,196] jepsen worker 14 - jepsen.util 5414 :ok :write [4 21] INFO [2019-12-17 12:31:55,196] jepsen worker 14 - jepsen.util 5414 :invoke :read [4 nil] INFO [2019-12-17 12:31:55,198] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":30694,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:31:55,198] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":30694,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:31:55,198] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":30695,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:31:55,198] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":30695,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:31:55,212] jepsen worker 63 - jepsen.util 4293 :fail :cas [21 [37 38]] :ValueNotSwapFrom INFO [2019-12-17 12:31:55,213] jepsen worker 63 - jepsen.util 4293 :invoke :read [21 nil] INFO [2019-12-17 12:31:55,225] jepsen worker 76 - jepsen.util 7186 :info :write [25 34] :RequestCanceledException INFO [2019-12-17 12:31:55,225] jepsen worker 60 - jepsen.util 7890 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:31:55,225] jepsen worker 48 - jepsen.util 6438 :info :write [16 40] :RequestCanceledException INFO [2019-12-17 12:31:55,225] jepsen worker 6 - jepsen.util 7386 :info :write [2 31] :RequestCanceledException INFO [2019-12-17 12:31:55,225] jepsen worker 15 - jepsen.util 6135 :info :write [5 2] :RequestCanceledException INFO [2019-12-17 12:31:55,225] jepsen worker 27 - jepsen.util 5067 :fail :read [9 nil] :RequestTimeoutException INFO [2019-12-17 12:31:55,225] jepsen worker 60 - jepsen.util 7890 :invoke :read [20 nil] INFO [2019-12-17 12:31:55,226] jepsen worker 27 - jepsen.util 5067 :invoke :read [9 nil] INFO [2019-12-17 12:31:55,226] jepsen worker 76 - jepsen.util 7276 :invoke :write [25 11] INFO [2019-12-17 12:31:55,226] jepsen worker 6 - jepsen.util 7476 :invoke :write [2 28] INFO [2019-12-17 12:31:55,226] jepsen worker 48 - jepsen.util 6528 :invoke :cas [16 [24 12]] INFO [2019-12-17 12:31:55,226] jepsen worker 15 - jepsen.util 6225 :invoke :read [5 nil] INFO [2019-12-17 12:31:55,238] jepsen worker 19 - jepsen.util 5239 :fail :cas [6 [8 17]] :ValueNotSwapFrom INFO [2019-12-17 12:31:55,238] jepsen worker 19 - jepsen.util 5239 :invoke :cas [6 [15 15]] INFO [2019-12-17 12:31:55,246] jepsen worker 12 - jepsen.util 5592 :fail :cas [4 [36 2]] :ValueNotSwapFrom INFO [2019-12-17 12:31:55,247] jepsen worker 12 - jepsen.util 5592 :invoke :write [4 23] INFO [2019-12-17 12:31:55,256] jepsen worker 73 - jepsen.util 4933 :fail :cas [24 [1 44]] :ValueNotSwapFrom INFO [2019-12-17 12:31:55,256] jepsen worker 73 - jepsen.util 4933 :invoke :read [24 nil] INFO [2019-12-17 12:31:55,257] jepsen worker 18 - jepsen.util 5238 :info :write [6 2] :RequestCanceledException INFO [2019-12-17 12:31:55,257] jepsen worker 17 - jepsen.util 7127 :info :cas [5 [22 20]] :RequestCanceledException INFO [2019-12-17 12:31:55,258] jepsen worker 18 - jepsen.util 5328 :invoke :write [6 30] INFO [2019-12-17 12:31:55,258] jepsen worker 17 - jepsen.util 7217 :invoke :write [5 21] INFO [2019-12-17 12:31:55,277] jepsen worker 19 - jepsen.util 5239 :fail :cas [6 [15 15]] :ValueNotSwapFrom INFO [2019-12-17 12:31:55,277] jepsen worker 19 - jepsen.util 5239 :invoke :write [6 31] INFO [2019-12-17 12:31:55,281] jepsen worker 71 - jepsen.util 6101 :fail :cas [23 [12 20]] :ValueNotSwapFrom INFO [2019-12-17 12:31:55,281] jepsen worker 71 - jepsen.util 6101 :invoke :cas [23 [28 45]] INFO [2019-12-17 12:31:55,288] jepsen worker 87 - jepsen.util 4767 :ok :read [29 36] INFO [2019-12-17 12:31:55,288] jepsen worker 87 - jepsen.util 4767 :invoke :write [29 43] INFO [2019-12-17 12:31:55,290] jepsen worker 33 - jepsen.util 7143 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:31:55,290] jepsen worker 84 - jepsen.util 6744 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:31:55,291] jepsen worker 33 - jepsen.util 7143 :invoke :cas [11 [34 45]] INFO [2019-12-17 12:31:55,291] jepsen worker 84 - jepsen.util 6744 :invoke :write [28 33] INFO [2019-12-17 12:31:55,307] jepsen worker 51 - jepsen.util 5271 :ok :read [17 38] INFO [2019-12-17 12:31:55,307] jepsen worker 51 - jepsen.util 5271 :invoke :read [17 nil] INFO [2019-12-17 12:31:55,308] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":30705,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:31:55,308] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":30705,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:31:55,308] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":30707,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:31:55,308] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":30707,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:31:55,310] jepsen worker 77 - jepsen.util 7277 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:31:55,310] jepsen worker 45 - jepsen.util 5445 :fail :read [15 nil] :RequestTimeoutException INFO [2019-12-17 12:31:55,310] jepsen worker 47 - jepsen.util 5897 :info :write [15 39] :RequestCanceledException INFO [2019-12-17 12:31:55,310] jepsen worker 77 - jepsen.util 7277 :invoke :write [25 29] INFO [2019-12-17 12:31:55,311] jepsen worker 45 - jepsen.util 5445 :invoke :cas [15 [13 38]] INFO [2019-12-17 12:31:55,312] jepsen worker 47 - jepsen.util 5987 :invoke :cas [15 [13 19]] INFO [2019-12-17 12:31:55,313] jepsen worker 4 - jepsen.util 4504 :ok :read [1 27] INFO [2019-12-17 12:31:55,313] jepsen worker 4 - jepsen.util 4504 :invoke :write [1 39] INFO [2019-12-17 12:31:55,326] jepsen worker 85 - jepsen.util 8005 :info :cas [28 [0 19]] :RequestCanceledException INFO [2019-12-17 12:31:55,326] jepsen worker 16 - jepsen.util 8206 :info :write [5 36] :RequestCanceledException INFO [2019-12-17 12:31:55,329] jepsen worker 16 - jepsen.util 8296 :invoke :cas [5 [17 16]] INFO [2019-12-17 12:31:55,329] jepsen worker 85 - jepsen.util 8095 :invoke :cas [28 [22 38]] INFO [2019-12-17 12:31:55,337] jepsen worker 66 - jepsen.util 6636 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:31:55,337] jepsen worker 82 - jepsen.util 7732 :info :cas [27 [4 43]] :RequestCanceledException INFO [2019-12-17 12:31:55,337] jepsen worker 66 - jepsen.util 6636 :invoke :write [22 48] INFO [2019-12-17 12:31:55,338] jepsen worker 73 - jepsen.util 4933 :ok :read [24 2] INFO [2019-12-17 12:31:55,338] jepsen worker 73 - jepsen.util 4933 :invoke :cas [24 [38 15]] INFO [2019-12-17 12:31:55,338] jepsen worker 82 - jepsen.util 7822 :invoke :cas [27 [33 44]] INFO [2019-12-17 12:31:55,339] jepsen worker 38 - jepsen.util 5348 :ok :write [12 1] INFO [2019-12-17 12:31:55,339] jepsen worker 38 - jepsen.util 5348 :invoke :cas [12 [21 49]] INFO [2019-12-17 12:31:55,350] jepsen worker 65 - jepsen.util 4565 :ok :write [21 23] INFO [2019-12-17 12:31:55,351] jepsen worker 65 - jepsen.util 4565 :invoke :cas [21 [4 0]] INFO [2019-12-17 12:31:55,357] jepsen worker 75 - jepsen.util 6825 :info :cas [25 [31 8]] :RequestCanceledException INFO [2019-12-17 12:31:55,357] jepsen worker 41 - jepsen.util 7781 :info :write [13 19] :RequestCanceledException INFO [2019-12-17 12:31:55,358] jepsen worker 41 - jepsen.util 7871 :invoke :read [13 nil] INFO [2019-12-17 12:31:55,358] jepsen worker 75 - jepsen.util 6915 :invoke :cas [25 [31 3]] INFO [2019-12-17 12:31:55,378] jepsen worker 36 - jepsen.util 5616 :ok :write [12 6] INFO [2019-12-17 12:31:55,378] jepsen worker 36 - jepsen.util 5616 :invoke :read [12 nil] INFO [2019-12-17 12:31:55,379] jepsen worker 63 - jepsen.util 4293 :ok :read [21 23] INFO [2019-12-17 12:31:55,379] jepsen worker 63 - jepsen.util 4293 :invoke :read [21 nil] INFO [2019-12-17 12:31:55,386] jepsen worker 47 - jepsen.util 5987 :fail :cas [15 [13 19]] :ValueNotSwapFrom INFO [2019-12-17 12:31:55,386] jepsen worker 47 - jepsen.util 5987 :invoke :write [15 23] INFO [2019-12-17 12:31:55,388] jepsen worker 39 - jepsen.util 6699 :info :cas [13 [43 10]] :RequestCanceledException INFO [2019-12-17 12:31:55,388] jepsen worker 22 - jepsen.util 7222 :info :write [7 8] :RequestCanceledException INFO [2019-12-17 12:31:55,389] jepsen worker 39 - jepsen.util 6789 :invoke :write [13 49] INFO [2019-12-17 12:31:55,389] jepsen worker 22 - jepsen.util 7312 :invoke :cas [7 [20 43]] INFO [2019-12-17 12:31:55,398] jepsen worker 70 - jepsen.util 6010 :ok :read [23 20] INFO [2019-12-17 12:31:55,398] jepsen worker 70 - jepsen.util 6010 :invoke :read [23 nil] INFO [2019-12-17 12:31:55,403] jepsen worker 65 - jepsen.util 4565 :fail :cas [21 [4 0]] :ValueNotSwapFrom INFO [2019-12-17 12:31:55,403] jepsen worker 65 - jepsen.util 4565 :invoke :write [21 11] INFO [2019-12-17 12:31:55,410] jepsen worker 80 - jepsen.util 5300 :info :write [26 22] :RequestCanceledException INFO [2019-12-17 12:31:55,410] jepsen worker 57 - jepsen.util 6627 :info :cas [19 [36 39]] :RequestCanceledException INFO [2019-12-17 12:31:55,410] jepsen worker 40 - jepsen.util 7600 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:31:55,410] jepsen worker 28 - jepsen.util 3988 :info :cas [9 [12 7]] :RequestCanceledException INFO [2019-12-17 12:31:55,410] jepsen worker 40 - jepsen.util 7600 :invoke :cas [13 [5 38]] INFO [2019-12-17 12:31:55,411] jepsen worker 80 - jepsen.util 5390 :invoke :read [26 nil] INFO [2019-12-17 12:31:55,411] jepsen worker 28 - jepsen.util 4078 :invoke :read [9 nil] INFO [2019-12-17 12:31:55,411] jepsen worker 57 - jepsen.util 6717 :invoke :cas [19 [23 49]] INFO [2019-12-17 12:31:55,422] jepsen worker 79 - jepsen.util 5209 :ok :write [26 7] INFO [2019-12-17 12:31:55,423] jepsen worker 79 - jepsen.util 5209 :invoke :read [26 nil] INFO [2019-12-17 12:31:55,426] jepsen worker 69 - jepsen.util 5739 :info :write [23 5] :RequestCanceledException INFO [2019-12-17 12:31:55,427] jepsen worker 69 - jepsen.util 5829 :invoke :read [23 nil] INFO [2019-12-17 12:31:55,440] jepsen worker 43 - jepsen.util 6703 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:31:55,440] jepsen worker 43 - jepsen.util 6703 :invoke :write [14 26] INFO [2019-12-17 12:31:55,441] jepsen worker 9 - jepsen.util 5859 :ok :write [3 12] INFO [2019-12-17 12:31:55,441] jepsen worker 9 - jepsen.util 5859 :invoke :cas [3 [18 25]] INFO [2019-12-17 12:31:55,442] jepsen worker 63 - jepsen.util 4293 :ok :read [21 23] INFO [2019-12-17 12:31:55,442] jepsen worker 63 - jepsen.util 4293 :invoke :read [21 nil] INFO [2019-12-17 12:31:55,454] jepsen worker 83 - jepsen.util 7463 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:31:55,454] jepsen worker 83 - jepsen.util 7463 :invoke :write [27 18] INFO [2019-12-17 12:31:55,457] jepsen worker 88 - jepsen.util 5578 :ok :read [29 36] INFO [2019-12-17 12:31:55,457] jepsen worker 88 - jepsen.util 5578 :invoke :read [29 nil] INFO [2019-12-17 12:31:55,458] jepsen worker 54 - jepsen.util 5724 :ok :write [18 18] INFO [2019-12-17 12:31:55,459] jepsen worker 54 - jepsen.util 5724 :invoke :write [18 28] INFO [2019-12-17 12:31:55,459] jepsen worker 27 - jepsen.util 5067 :ok :read [9 5] INFO [2019-12-17 12:31:55,459] jepsen worker 27 - jepsen.util 5067 :invoke :write [9 30] INFO [2019-12-17 12:31:55,465] jepsen worker 53 - jepsen.util 5183 :ok :read [17 38] INFO [2019-12-17 12:31:55,465] jepsen worker 53 - jepsen.util 5183 :invoke :write [17 45] INFO [2019-12-17 12:31:55,478] jepsen worker 28 - jepsen.util 4078 :ok :read [9 5] INFO [2019-12-17 12:31:55,478] jepsen worker 28 - jepsen.util 4078 :invoke :write [9 45] INFO [2019-12-17 12:31:55,491] jepsen worker 5 - jepsen.util 4955 :ok :write [1 34] INFO [2019-12-17 12:31:55,491] jepsen worker 5 - jepsen.util 4955 :invoke :cas [1 [11 3]] INFO [2019-12-17 12:31:55,495] jepsen worker 9 - jepsen.util 5859 :fail :cas [3 [18 25]] :ValueNotSwapFrom INFO [2019-12-17 12:31:55,495] jepsen worker 9 - jepsen.util 5859 :invoke :write [3 24] INFO [2019-12-17 12:31:55,495] jepsen worker 55 - jepsen.util 6355 :ok :read [18 18] INFO [2019-12-17 12:31:55,495] jepsen worker 55 - jepsen.util 6355 :invoke :cas [18 [3 48]] INFO [2019-12-17 12:31:55,515] jepsen worker 14 - jepsen.util 5414 :ok :read [4 20] INFO [2019-12-17 12:31:55,517] jepsen worker 14 - jepsen.util 5414 :invoke :cas [4 [22 26]] INFO [2019-12-17 12:31:55,521] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":30754,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:31:55,521] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":30754,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:31:55,526] jepsen worker 27 - jepsen.util 5067 :ok :write [9 30] INFO [2019-12-17 12:31:55,526] jepsen worker 27 - jepsen.util 5067 :invoke :write [9 41] INFO [2019-12-17 12:31:55,546] jepsen worker 38 - jepsen.util 5348 :fail :cas [12 [21 49]] :ValueNotSwapFrom INFO [2019-12-17 12:31:55,546] jepsen worker 38 - jepsen.util 5348 :invoke :read [12 nil] INFO [2019-12-17 12:31:55,583] jepsen worker 80 - jepsen.util 5390 :ok :read [26 7] INFO [2019-12-17 12:31:55,583] jepsen worker 80 - jepsen.util 5390 :invoke :read [26 nil] INFO [2019-12-17 12:31:55,587] jepsen worker 71 - jepsen.util 6101 :fail :cas [23 [28 45]] :ValueNotSwapFrom INFO [2019-12-17 12:31:55,587] jepsen worker 71 - jepsen.util 6101 :invoke :read [23 nil] INFO [2019-12-17 12:31:55,604] jepsen worker 51 - jepsen.util 5271 :ok :read [17 38] INFO [2019-12-17 12:31:55,604] jepsen worker 51 - jepsen.util 5271 :invoke :cas [17 [19 20]] INFO [2019-12-17 12:31:55,610] jepsen worker 25 - jepsen.util 7495 :info :write [8 41] :RequestCanceledException INFO [2019-12-17 12:31:55,610] jepsen worker 2 - jepsen.util 6662 :info :cas [0 [35 20]] :RequestCanceledException INFO [2019-12-17 12:31:55,612] jepsen worker 25 - jepsen.util 7585 :invoke :read [8 nil] INFO [2019-12-17 12:31:55,612] jepsen worker 2 - jepsen.util 6752 :invoke :cas [0 [45 17]] INFO [2019-12-17 12:31:55,612] jepsen worker 5 - jepsen.util 4955 :fail :cas [1 [11 3]] :ValueNotSwapFrom INFO [2019-12-17 12:31:55,612] jepsen worker 5 - jepsen.util 4955 :invoke :write [1 3] INFO [2019-12-17 12:31:55,619] jepsen worker 14 - jepsen.util 5414 :fail :cas [4 [22 26]] :ValueNotSwapFrom INFO [2019-12-17 12:31:55,619] jepsen worker 14 - jepsen.util 5414 :invoke :cas [4 [12 29]] INFO [2019-12-17 12:31:55,626] jepsen worker 61 - jepsen.util 7981 :info :cas [20 [12 22]] :RequestCanceledException INFO [2019-12-17 12:31:55,626] jepsen worker 44 - jepsen.util 7694 :info :cas [14 [6 29]] :RequestCanceledException INFO [2019-12-17 12:31:55,626] jepsen worker 58 - jepsen.util 7528 :info :cas [19 [31 39]] :RequestCanceledException INFO [2019-12-17 12:31:55,627] jepsen worker 58 - jepsen.util 7618 :invoke :cas [19 [41 16]] INFO [2019-12-17 12:31:55,627] jepsen worker 61 - jepsen.util 8071 :invoke :write [20 8] INFO [2019-12-17 12:31:55,627] jepsen worker 44 - jepsen.util 7784 :invoke :cas [14 [21 24]] INFO [2019-12-17 12:31:55,632] jepsen worker 71 - jepsen.util 6101 :ok :read [23 20] INFO [2019-12-17 12:31:55,632] jepsen worker 71 - jepsen.util 6101 :invoke :cas [23 [4 9]] INFO [2019-12-17 12:31:55,640] jepsen worker 51 - jepsen.util 5271 :fail :cas [17 [19 20]] :ValueNotSwapFrom INFO [2019-12-17 12:31:55,640] jepsen worker 51 - jepsen.util 5271 :invoke :write [17 17] INFO [2019-12-17 12:31:55,640] jepsen worker 34 - jepsen.util 6604 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:31:55,640] jepsen worker 34 - jepsen.util 6604 :invoke :read [11 nil] INFO [2019-12-17 12:31:55,655] jepsen worker 81 - jepsen.util 7101 :info :cas [27 [47 36]] :RequestCanceledException INFO [2019-12-17 12:31:55,656] jepsen worker 81 - jepsen.util 7191 :invoke :read [27 nil] INFO [2019-12-17 12:31:55,657] jepsen worker 36 - jepsen.util 5616 :ok :read [12 6] INFO [2019-12-17 12:31:55,657] jepsen worker 36 - jepsen.util 5616 :invoke :write [12 40] INFO [2019-12-17 12:31:55,665] jepsen worker 18 - jepsen.util 5328 :ok :write [6 30] INFO [2019-12-17 12:31:55,665] jepsen worker 18 - jepsen.util 5328 :invoke :read [6 nil] INFO [2019-12-17 12:31:55,665] jepsen worker 72 - jepsen.util 5112 :ok :write [24 21] INFO [2019-12-17 12:31:55,665] jepsen worker 72 - jepsen.util 5112 :invoke :read [24 nil] INFO [2019-12-17 12:31:55,668] jepsen worker 32 - jepsen.util 5432 :ok :write [10 35] INFO [2019-12-17 12:31:55,668] jepsen worker 32 - jepsen.util 5432 :invoke :read [10 nil] INFO [2019-12-17 12:31:55,691] jepsen worker 79 - jepsen.util 5209 :ok :read [26 7] INFO [2019-12-17 12:31:55,691] jepsen worker 79 - jepsen.util 5209 :invoke :cas [26 [31 46]] INFO [2019-12-17 12:31:55,693] jepsen worker 10 - jepsen.util 5230 :info :write [3 31] :RequestCanceledException INFO [2019-12-17 12:31:55,694] jepsen worker 10 - jepsen.util 5320 :invoke :write [3 49] INFO [2019-12-17 12:31:55,705] jepsen worker 32 - jepsen.util 5432 :ok :read [10 35] INFO [2019-12-17 12:31:55,705] jepsen worker 32 - jepsen.util 5432 :invoke :read [10 nil] INFO [2019-12-17 12:31:55,709] jepsen worker 56 - jepsen.util 5546 :info :write [18 47] :RequestCanceledException INFO [2019-12-17 12:31:55,710] jepsen worker 56 - jepsen.util 5636 :invoke :read [18 nil] INFO [2019-12-17 12:31:55,718] jepsen worker 89 - jepsen.util 5129 :ok :read [29 36] INFO [2019-12-17 12:31:55,718] jepsen worker 89 - jepsen.util 5129 :invoke :write [29 22] INFO [2019-12-17 12:31:55,721] jepsen worker 45 - jepsen.util 5445 :fail :cas [15 [13 38]] :ValueNotSwapFrom INFO [2019-12-17 12:31:55,721] jepsen worker 45 - jepsen.util 5445 :invoke :write [15 38] INFO [2019-12-17 12:31:55,726] jepsen worker 86 - jepsen.util 7466 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:31:55,726] jepsen worker 74 - jepsen.util 5744 :info :write [24 15] :RequestCanceledException INFO [2019-12-17 12:31:55,726] jepsen worker 86 - jepsen.util 7466 :invoke :cas [28 [20 30]] INFO [2019-12-17 12:31:55,727] jepsen worker 74 - jepsen.util 5834 :invoke :cas [24 [24 3]] INFO [2019-12-17 12:31:55,739] jepsen worker 0 - jepsen.util 7740 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:31:55,740] jepsen worker 0 - jepsen.util 7740 :invoke :write [0 39] INFO [2019-12-17 12:31:55,740] jepsen worker 18 - jepsen.util 5328 :ok :read [6 30] INFO [2019-12-17 12:31:55,740] jepsen worker 18 - jepsen.util 5328 :invoke :read [6 nil] INFO [2019-12-17 12:31:55,740] jepsen worker 70 - jepsen.util 6010 :ok :read [23 20] INFO [2019-12-17 12:31:55,740] jepsen worker 70 - jepsen.util 6010 :invoke :read [23 nil] INFO [2019-12-17 12:31:55,741] jepsen worker 32 - jepsen.util 5432 :ok :read [10 35] INFO [2019-12-17 12:31:55,742] jepsen worker 32 - jepsen.util 5432 :invoke :read [10 nil] INFO [2019-12-17 12:31:55,750] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":30788,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:31:55,750] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":30788,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:31:55,750] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":30790,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:31:55,750] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":30790,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:31:55,754] jepsen worker 80 - jepsen.util 5390 :ok :read [26 7] INFO [2019-12-17 12:31:55,755] jepsen worker 80 - jepsen.util 5390 :invoke :write [26 0] INFO [2019-12-17 12:31:55,770] jepsen worker 38 - jepsen.util 5348 :ok :read [12 6] INFO [2019-12-17 12:31:55,770] jepsen worker 38 - jepsen.util 5348 :invoke :write [12 45] INFO [2019-12-17 12:31:55,785] jepsen worker 79 - jepsen.util 5209 :fail :cas [26 [31 46]] :ValueNotSwapFrom INFO [2019-12-17 12:31:55,785] jepsen worker 79 - jepsen.util 5209 :invoke :read [26 nil] INFO [2019-12-17 12:31:55,810] jepsen worker 32 - jepsen.util 5432 :ok :read [10 35] INFO [2019-12-17 12:31:55,810] jepsen worker 32 - jepsen.util 5432 :invoke :write [10 3] INFO [2019-12-17 12:31:55,826] jepsen worker 62 - jepsen.util 6812 :info :write [20 19] :RequestCanceledException INFO [2019-12-17 12:31:55,828] jepsen worker 62 - jepsen.util 6902 :invoke :write [20 8] INFO [2019-12-17 12:31:55,831] jepsen worker 79 - jepsen.util 5209 :ok :read [26 7] INFO [2019-12-17 12:31:55,831] jepsen worker 79 - jepsen.util 5209 :invoke :read [26 nil] INFO [2019-12-17 12:31:55,837] jepsen worker 72 - jepsen.util 5112 :ok :read [24 21] INFO [2019-12-17 12:31:55,838] jepsen worker 72 - jepsen.util 5112 :invoke :write [24 42] INFO [2019-12-17 12:31:55,840] jepsen worker 67 - jepsen.util 7177 :info :cas [22 [18 26]] :RequestCanceledException INFO [2019-12-17 12:31:55,841] jepsen worker 67 - jepsen.util 7267 :invoke :cas [22 [13 16]] INFO [2019-12-17 12:31:55,847] jepsen worker 18 - jepsen.util 5328 :ok :read [6 30] INFO [2019-12-17 12:31:55,847] jepsen worker 18 - jepsen.util 5328 :invoke :write [6 16] INFO [2019-12-17 12:31:55,860] jepsen worker 14 - jepsen.util 5414 :fail :cas [4 [12 29]] :ValueNotSwapFrom INFO [2019-12-17 12:31:55,861] jepsen worker 14 - jepsen.util 5414 :invoke :cas [4 [45 9]] INFO [2019-12-17 12:31:55,893] jepsen worker 13 - jepsen.util 5233 :fail :cas [4 [36 28]] :ValueNotSwapFrom INFO [2019-12-17 12:31:55,893] jepsen worker 13 - jepsen.util 5233 :invoke :read [4 nil] INFO [2019-12-17 12:31:55,910] jepsen worker 26 - jepsen.util 7586 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:31:55,910] jepsen worker 26 - jepsen.util 7586 :invoke :write [8 8] INFO [2019-12-17 12:31:55,919] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":30818,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:31:55,919] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":30818,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:31:55,920] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":30822,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:31:55,920] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":30822,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:31:55,920] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":30823,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:31:55,920] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":30823,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:31:56,014] jepsen worker 71 - jepsen.util 6101 :fail :cas [23 [4 9]] :ValueNotSwapFrom INFO [2019-12-17 12:31:56,014] jepsen worker 71 - jepsen.util 6101 :invoke :write [23 19] INFO [2019-12-17 12:31:56,024] jepsen worker 21 - jepsen.util 6681 :info :write [7 7] :RequestCanceledException INFO [2019-12-17 12:31:56,026] jepsen worker 21 - jepsen.util 6771 :invoke :write [7 21] INFO [2019-12-17 12:31:56,039] jepsen worker 69 - jepsen.util 5829 :ok :read [23 20] INFO [2019-12-17 12:31:56,039] jepsen worker 69 - jepsen.util 5829 :invoke :write [23 18] INFO [2019-12-17 12:31:56,040] jepsen worker 7 - jepsen.util 7297 :info :cas [2 [37 8]] :RequestCanceledException INFO [2019-12-17 12:31:56,041] jepsen worker 7 - jepsen.util 7387 :invoke :write [2 43] INFO [2019-12-17 12:31:56,046] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:31:56,046] jepsen nemesis - couchbase.util Rebalance check count: 231 INFO [2019-12-17 12:31:56,046] jepsen nemesis - couchbase.util Rebalance stuck count: 212 INFO [2019-12-17 12:31:56,075] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":30828,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:31:56,075] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":30828,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:31:56,075] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":30831,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:31:56,076] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":30831,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:31:56,092] jepsen worker 30 - jepsen.util 5340 :info :write [10 29] :RequestCanceledException INFO [2019-12-17 12:31:56,092] jepsen worker 1 - jepsen.util 6931 :info :write [0 39] :RequestCanceledException INFO [2019-12-17 12:31:56,092] jepsen worker 64 - jepsen.util 4024 :fail :read [21 nil] :RequestTimeoutException INFO [2019-12-17 12:31:56,093] jepsen worker 64 - jepsen.util 4024 :invoke :cas [21 [26 15]] INFO [2019-12-17 12:31:56,093] jepsen worker 30 - jepsen.util 5430 :invoke :cas [10 [14 28]] INFO [2019-12-17 12:31:56,093] jepsen worker 1 - jepsen.util 7021 :invoke :write [0 33] INFO [2019-12-17 12:31:56,101] jepsen worker 79 - jepsen.util 5209 :ok :read [26 7] INFO [2019-12-17 12:31:56,101] jepsen worker 79 - jepsen.util 5209 :invoke :cas [26 [24 29]] INFO [2019-12-17 12:31:56,106] jepsen worker 64 - jepsen.util 4024 :fail :cas [21 [26 15]] :ValueNotSwapFrom INFO [2019-12-17 12:31:56,106] jepsen worker 64 - jepsen.util 4024 :invoke :write [21 6] INFO [2019-12-17 12:31:56,113] jepsen worker 30 - jepsen.util 5430 :fail :cas [10 [14 28]] :ValueNotSwapFrom INFO [2019-12-17 12:31:56,113] jepsen worker 30 - jepsen.util 5430 :invoke :read [10 nil] INFO [2019-12-17 12:31:56,144] jepsen worker 64 - jepsen.util 4024 :ok :write [21 6] INFO [2019-12-17 12:31:56,144] jepsen worker 64 - jepsen.util 4024 :invoke :read [21 nil] INFO [2019-12-17 12:31:56,165] jepsen worker 64 - jepsen.util 4024 :ok :read [21 6] INFO [2019-12-17 12:31:56,165] jepsen worker 64 - jepsen.util 4024 :invoke :cas [21 [11 22]] INFO [2019-12-17 12:31:56,178] jepsen worker 64 - jepsen.util 4024 :fail :cas [21 [11 22]] :ValueNotSwapFrom INFO [2019-12-17 12:31:56,178] jepsen worker 64 - jepsen.util 4024 :invoke :cas [21 [14 17]] INFO [2019-12-17 12:31:56,194] jepsen worker 64 - jepsen.util 4024 :fail :cas [21 [14 17]] :ValueNotSwapFrom INFO [2019-12-17 12:31:56,194] jepsen worker 64 - jepsen.util 4024 :invoke :cas [21 [2 11]] INFO [2019-12-17 12:31:56,206] jepsen worker 64 - jepsen.util 4024 :fail :cas [21 [2 11]] :ValueNotSwapFrom INFO [2019-12-17 12:31:56,206] jepsen worker 64 - jepsen.util 4024 :invoke :cas [21 [24 22]] INFO [2019-12-17 12:31:56,219] jepsen worker 64 - jepsen.util 4024 :fail :cas [21 [24 22]] :ValueNotSwapFrom INFO [2019-12-17 12:31:56,219] jepsen worker 64 - jepsen.util 4024 :invoke :read [21 nil] INFO [2019-12-17 12:31:56,225] jepsen worker 35 - jepsen.util 7325 :info :write [11 40] :RequestCanceledException INFO [2019-12-17 12:31:56,226] jepsen worker 35 - jepsen.util 7415 :invoke :read [11 nil] INFO [2019-12-17 12:31:56,235] jepsen worker 64 - jepsen.util 4024 :ok :read [21 6] INFO [2019-12-17 12:31:56,236] jepsen worker 64 - jepsen.util 4024 :invoke :cas [21 [37 8]] INFO [2019-12-17 12:31:56,239] jepsen worker 79 - jepsen.util 5209 :fail :cas [26 [24 29]] :ValueNotSwapFrom INFO [2019-12-17 12:31:56,240] jepsen worker 79 - jepsen.util 5209 :invoke :read [26 nil] INFO [2019-12-17 12:31:56,240] jepsen worker 42 - jepsen.util 7782 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:31:56,240] jepsen worker 42 - jepsen.util 7782 :invoke :read [14 nil] INFO [2019-12-17 12:31:56,249] jepsen worker 64 - jepsen.util 4024 :fail :cas [21 [37 8]] :ValueNotSwapFrom INFO [2019-12-17 12:31:56,249] jepsen worker 64 - jepsen.util 4024 :invoke :read [21 nil] INFO [2019-12-17 12:31:56,261] jepsen worker 64 - jepsen.util 4024 :ok :read [21 6] INFO [2019-12-17 12:31:56,261] jepsen worker 64 - jepsen.util 4024 :invoke :write [21 25] INFO [2019-12-17 12:31:56,292] jepsen worker 50 - jepsen.util 7340 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:31:56,293] jepsen worker 50 - jepsen.util 7340 :invoke :cas [16 [4 16]] INFO [2019-12-17 12:31:56,294] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":30846,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:31:56,294] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":30846,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:31:56,294] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":30855,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:31:56,294] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":30848,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:31:56,294] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":30855,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:31:56,294] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":30848,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:31:56,294] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":30859,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:31:56,294] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":30859,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:31:56,300] jepsen worker 70 - jepsen.util 6010 :ok :read [23 20] INFO [2019-12-17 12:31:56,300] jepsen worker 70 - jepsen.util 6010 :invoke :write [23 31] INFO [2019-12-17 12:31:56,304] jepsen worker 49 - jepsen.util 6349 :info :cas [16 [13 19]] :RequestCanceledException INFO [2019-12-17 12:31:56,311] jepsen worker 49 - jepsen.util 6439 :invoke :cas [16 [44 47]] INFO [2019-12-17 12:31:56,315] jepsen worker 88 - jepsen.util 5578 :ok :read [29 36] INFO [2019-12-17 12:31:56,315] jepsen worker 88 - jepsen.util 5578 :invoke :cas [29 [28 32]] INFO [2019-12-17 12:31:56,321] jepsen worker 64 - jepsen.util 4024 :ok :write [21 25] INFO [2019-12-17 12:31:56,321] jepsen worker 64 - jepsen.util 4024 :invoke :write [21 16] INFO [2019-12-17 12:31:56,325] jepsen worker 24 - jepsen.util 6594 :info :cas [8 [13 23]] :RequestCanceledException INFO [2019-12-17 12:31:56,326] jepsen worker 24 - jepsen.util 6684 :invoke :read [8 nil] INFO [2019-12-17 12:31:56,335] jepsen worker 55 - jepsen.util 6355 :fail :cas [18 [3 48]] :ValueNotSwapFrom INFO [2019-12-17 12:31:56,335] jepsen worker 55 - jepsen.util 6355 :invoke :read [18 nil] INFO [2019-12-17 12:31:56,343] jepsen worker 56 - jepsen.util 5636 :ok :read [18 18] INFO [2019-12-17 12:31:56,344] jepsen worker 56 - jepsen.util 5636 :invoke :cas [18 [13 1]] INFO [2019-12-17 12:31:56,357] jepsen worker 23 - jepsen.util 7313 :info :cas [7 [32 9]] :RequestCanceledException INFO [2019-12-17 12:31:56,359] jepsen worker 23 - jepsen.util 7403 :invoke :cas [7 [5 44]] INFO [2019-12-17 12:31:56,376] jepsen worker 64 - jepsen.util 4024 :ok :write [21 16] INFO [2019-12-17 12:31:56,376] jepsen worker 64 - jepsen.util 4024 :invoke :read [21 nil] INFO [2019-12-17 12:31:56,392] jepsen worker 64 - jepsen.util 4024 :ok :read [21 16] INFO [2019-12-17 12:31:56,392] jepsen worker 64 - jepsen.util 4024 :invoke :write [21 21] INFO [2019-12-17 12:31:56,419] jepsen worker 14 - jepsen.util 5414 :fail :cas [4 [45 9]] :ValueNotSwapFrom INFO [2019-12-17 12:31:56,419] jepsen worker 14 - jepsen.util 5414 :invoke :cas [4 [3 20]] INFO [2019-12-17 12:31:56,426] jepsen worker 59 - jepsen.util 7079 :info :write [19 36] :RequestCanceledException INFO [2019-12-17 12:31:56,428] jepsen worker 59 - jepsen.util 7169 :invoke :cas [19 [21 15]] INFO [2019-12-17 12:31:56,457] jepsen worker 18 - jepsen.util 5328 :ok :write [6 16] INFO [2019-12-17 12:31:56,457] jepsen worker 18 - jepsen.util 5328 :invoke :read [6 nil] INFO [2019-12-17 12:31:56,457] jepsen worker 47 - jepsen.util 5987 :ok :write [15 23] INFO [2019-12-17 12:31:56,458] jepsen worker 47 - jepsen.util 5987 :invoke :write [15 38] INFO [2019-12-17 12:31:56,462] jepsen worker 9 - jepsen.util 5859 :ok :write [3 24] INFO [2019-12-17 12:31:56,462] jepsen worker 32 - jepsen.util 5432 :ok :write [10 3] INFO [2019-12-17 12:31:56,462] jepsen worker 9 - jepsen.util 5859 :invoke :cas [3 [7 30]] INFO [2019-12-17 12:31:56,462] jepsen worker 32 - jepsen.util 5432 :invoke :read [10 nil] INFO [2019-12-17 12:31:56,507] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":30879,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:31:56,507] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":30879,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:31:56,507] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":30881,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:31:56,507] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":30881,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:31:56,518] jepsen worker 63 - jepsen.util 4293 :ok :read [21 16] INFO [2019-12-17 12:31:56,519] jepsen worker 63 - jepsen.util 4293 :invoke :write [21 6] INFO [2019-12-17 12:31:56,537] jepsen worker 64 - jepsen.util 4024 :ok :write [21 21] INFO [2019-12-17 12:31:56,537] jepsen worker 64 - jepsen.util 4024 :invoke :write [21 13] INFO [2019-12-17 12:31:56,547] jepsen worker 18 - jepsen.util 5328 :ok :read [6 16] INFO [2019-12-17 12:31:56,547] jepsen worker 18 - jepsen.util 5328 :invoke :read [6 nil] INFO [2019-12-17 12:31:56,549] jepsen worker 13 - jepsen.util 5233 :ok :read [4 20] INFO [2019-12-17 12:31:56,549] jepsen worker 32 - jepsen.util 5432 :ok :read [10 3] INFO [2019-12-17 12:31:56,549] jepsen worker 13 - jepsen.util 5233 :invoke :write [4 30] INFO [2019-12-17 12:31:56,549] jepsen worker 32 - jepsen.util 5432 :invoke :write [10 40] INFO [2019-12-17 12:31:56,556] jepsen worker 9 - jepsen.util 5859 :fail :cas [3 [7 30]] :ValueNotSwapFrom INFO [2019-12-17 12:31:56,557] jepsen worker 9 - jepsen.util 5859 :invoke :read [3 nil] INFO [2019-12-17 12:31:56,557] jepsen worker 52 - jepsen.util 5632 :info :write [17 20] :RequestCanceledException INFO [2019-12-17 12:31:56,558] jepsen worker 52 - jepsen.util 5722 :invoke :cas [17 [13 6]] INFO [2019-12-17 12:31:56,604] jepsen worker 18 - jepsen.util 5328 :ok :read [6 16] INFO [2019-12-17 12:31:56,604] jepsen worker 18 - jepsen.util 5328 :invoke :write [6 49] INFO [2019-12-17 12:31:56,619] jepsen worker 9 - jepsen.util 5859 :ok :read [3 24] INFO [2019-12-17 12:31:56,620] jepsen worker 9 - jepsen.util 5859 :invoke :write [3 0] INFO [2019-12-17 12:31:56,626] jepsen worker 31 - jepsen.util 4981 :fail :read [10 nil] :RequestTimeoutException INFO [2019-12-17 12:31:56,631] jepsen worker 31 - jepsen.util 4981 :invoke :cas [10 [45 11]] INFO [2019-12-17 12:31:56,646] jepsen worker 64 - jepsen.util 4024 :ok :write [21 13] INFO [2019-12-17 12:31:56,646] jepsen worker 64 - jepsen.util 4024 :invoke :write [21 18] INFO [2019-12-17 12:31:56,657] jepsen worker 37 - jepsen.util 4987 :info :write [12 23] :RequestCanceledException INFO [2019-12-17 12:31:56,658] jepsen worker 37 - jepsen.util 5077 :invoke :write [12 11] INFO [2019-12-17 12:31:56,672] jepsen worker 14 - jepsen.util 5414 :fail :cas [4 [3 20]] :ValueNotSwapFrom INFO [2019-12-17 12:31:56,672] jepsen worker 14 - jepsen.util 5414 :invoke :read [4 nil] INFO [2019-12-17 12:31:56,737] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":30901,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:31:56,737] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":30901,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:31:56,737] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":30904,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:31:56,737] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":30904,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:31:56,738] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":30905,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:31:56,738] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":30905,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:31:56,738] jepsen worker 64 - jepsen.util 4024 :ok :write [21 18] INFO [2019-12-17 12:31:56,738] jepsen worker 64 - jepsen.util 4024 :invoke :cas [21 [28 40]] INFO [2019-12-17 12:31:56,753] jepsen worker 64 - jepsen.util 4024 :fail :cas [21 [28 40]] :ValueNotSwapFrom INFO [2019-12-17 12:31:56,753] jepsen worker 64 - jepsen.util 4024 :invoke :cas [21 [2 20]] INFO [2019-12-17 12:31:56,763] jepsen worker 79 - jepsen.util 5209 :ok :read [26 7] INFO [2019-12-17 12:31:56,763] jepsen worker 79 - jepsen.util 5209 :invoke :cas [26 [35 39]] INFO [2019-12-17 12:31:56,772] jepsen worker 10 - jepsen.util 5320 :ok :write [3 49] INFO [2019-12-17 12:31:56,772] jepsen worker 10 - jepsen.util 5320 :invoke :write [3 0] INFO [2019-12-17 12:31:56,778] jepsen worker 64 - jepsen.util 4024 :fail :cas [21 [2 20]] :ValueNotSwapFrom INFO [2019-12-17 12:31:56,778] jepsen worker 64 - jepsen.util 4024 :invoke :write [21 15] INFO [2019-12-17 12:31:56,793] jepsen worker 11 - jepsen.util 5501 :info :write [3 26] :RequestCanceledException INFO [2019-12-17 12:31:56,794] jepsen worker 11 - jepsen.util 5591 :invoke :read [3 nil] INFO [2019-12-17 12:31:56,832] jepsen worker 32 - jepsen.util 5432 :ok :write [10 40] INFO [2019-12-17 12:31:56,832] jepsen worker 32 - jepsen.util 5432 :invoke :write [10 3] INFO [2019-12-17 12:31:56,857] jepsen worker 78 - jepsen.util 5478 :info :write [26 18] :RequestCanceledException INFO [2019-12-17 12:31:56,858] jepsen worker 78 - jepsen.util 5568 :invoke :write [26 28] INFO [2019-12-17 12:31:56,889] jepsen worker 64 - jepsen.util 4024 :ok :write [21 15] INFO [2019-12-17 12:31:56,889] jepsen worker 64 - jepsen.util 4024 :invoke :write [21 24] INFO [2019-12-17 12:31:56,891] jepsen worker 18 - jepsen.util 5328 :ok :write [6 49] INFO [2019-12-17 12:31:56,891] jepsen worker 18 - jepsen.util 5328 :invoke :read [6 nil] INFO [2019-12-17 12:31:56,892] jepsen worker 29 - jepsen.util 4259 :fail :read [9 nil] :RequestTimeoutException INFO [2019-12-17 12:31:56,892] jepsen worker 29 - jepsen.util 4259 :invoke :cas [9 [35 40]] INFO [2019-12-17 12:31:56,904] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":30918,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:31:56,904] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":30918,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:31:56,904] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":30921,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:31:56,904] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":30921,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:31:56,946] jepsen worker 11 - jepsen.util 5591 :ok :read [3 49] INFO [2019-12-17 12:31:56,947] jepsen worker 11 - jepsen.util 5591 :invoke :cas [3 [9 45]] INFO [2019-12-17 12:31:56,952] jepsen worker 29 - jepsen.util 4259 :fail :cas [9 [35 40]] :ValueNotSwapFrom INFO [2019-12-17 12:31:56,952] jepsen worker 29 - jepsen.util 4259 :invoke :write [9 4] INFO [2019-12-17 12:31:56,972] jepsen worker 18 - jepsen.util 5328 :ok :read [6 49] INFO [2019-12-17 12:31:56,972] jepsen worker 18 - jepsen.util 5328 :invoke :write [6 7] INFO [2019-12-17 12:31:56,989] jepsen worker 11 - jepsen.util 5591 :fail :cas [3 [9 45]] :ValueNotSwapFrom INFO [2019-12-17 12:31:56,989] jepsen worker 11 - jepsen.util 5591 :invoke :read [3 nil] INFO [2019-12-17 12:31:57,006] jepsen worker 5 - jepsen.util 4955 :ok :write [1 3] INFO [2019-12-17 12:31:57,006] jepsen worker 5 - jepsen.util 4955 :invoke :write [1 9] INFO [2019-12-17 12:31:57,006] jepsen worker 47 - jepsen.util 5987 :ok :write [15 38] INFO [2019-12-17 12:31:57,007] jepsen worker 47 - jepsen.util 5987 :invoke :write [15 17] INFO [2019-12-17 12:31:57,030] jepsen worker 32 - jepsen.util 5432 :ok :write [10 3] INFO [2019-12-17 12:31:57,030] jepsen worker 32 - jepsen.util 5432 :invoke :cas [10 [29 39]] INFO [2019-12-17 12:31:57,040] jepsen worker 64 - jepsen.util 4024 :ok :write [21 24] INFO [2019-12-17 12:31:57,041] jepsen worker 64 - jepsen.util 4024 :invoke :cas [21 [33 30]] INFO [2019-12-17 12:31:57,047] jepsen worker 31 - jepsen.util 4981 :fail :cas [10 [45 11]] :ValueNotSwapFrom INFO [2019-12-17 12:31:57,047] jepsen worker 31 - jepsen.util 4981 :invoke :read [10 nil] INFO [2019-12-17 12:31:57,056] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":30931,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:31:57,057] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":30931,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:31:57,057] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":30934,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:31:57,057] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":30934,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:31:57,057] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":30935,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:31:57,057] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":30935,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:31:57,060] jepsen worker 32 - jepsen.util 5432 :fail :cas [10 [29 39]] :ValueNotSwapFrom INFO [2019-12-17 12:31:57,061] jepsen worker 32 - jepsen.util 5432 :invoke :write [10 18] INFO [2019-12-17 12:31:57,066] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:31:57,066] jepsen nemesis - couchbase.util Rebalance check count: 232 INFO [2019-12-17 12:31:57,066] jepsen nemesis - couchbase.util Rebalance stuck count: 213 INFO [2019-12-17 12:31:57,125] jepsen worker 52 - jepsen.util 5722 :fail :cas [17 [13 6]] :ValueNotSwapFrom INFO [2019-12-17 12:31:57,126] jepsen worker 52 - jepsen.util 5722 :invoke :write [17 38] INFO [2019-12-17 12:31:57,126] jepsen worker 20 - jepsen.util 5420 :info :write [6 14] :RequestCanceledException INFO [2019-12-17 12:31:57,129] jepsen worker 20 - jepsen.util 5510 :invoke :read [6 nil] INFO [2019-12-17 12:31:57,134] jepsen worker 88 - jepsen.util 5578 :fail :cas [29 [28 32]] :ValueNotSwapFrom INFO [2019-12-17 12:31:57,134] jepsen worker 88 - jepsen.util 5578 :invoke :read [29 nil] INFO [2019-12-17 12:31:57,157] jepsen worker 46 - jepsen.util 5266 :info :write [15 27] :RequestCanceledException INFO [2019-12-17 12:31:57,159] jepsen worker 46 - jepsen.util 5356 :invoke :write [15 29] INFO [2019-12-17 12:31:57,162] jepsen worker 55 - jepsen.util 6355 :ok :read [18 18] INFO [2019-12-17 12:31:57,163] jepsen worker 55 - jepsen.util 6355 :invoke :write [18 0] INFO [2019-12-17 12:31:57,167] jepsen worker 29 - jepsen.util 4259 :ok :write [9 4] INFO [2019-12-17 12:31:57,167] jepsen worker 29 - jepsen.util 4259 :invoke :read [9 nil] INFO [2019-12-17 12:31:57,184] jepsen worker 70 - jepsen.util 6010 :ok :write [23 31] INFO [2019-12-17 12:31:57,184] jepsen worker 70 - jepsen.util 6010 :invoke :cas [23 [10 41]] INFO [2019-12-17 12:31:57,203] jepsen worker 56 - jepsen.util 5636 :fail :cas [18 [13 1]] :ValueNotSwapFrom INFO [2019-12-17 12:31:57,203] jepsen worker 56 - jepsen.util 5636 :invoke :write [18 21] INFO [2019-12-17 12:31:57,238] jepsen worker 3 - jepsen.util 5313 :info :write [1 19] :RequestCanceledException INFO [2019-12-17 12:31:57,238] jepsen worker 8 - jepsen.util 6758 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:31:57,238] jepsen worker 8 - jepsen.util 6758 :invoke :cas [2 [48 14]] INFO [2019-12-17 12:31:57,239] jepsen worker 3 - jepsen.util 5403 :invoke :cas [1 [6 49]] INFO [2019-12-17 12:31:57,257] jepsen worker 68 - jepsen.util 6728 :info :write [22 13] :RequestCanceledException INFO [2019-12-17 12:31:57,259] jepsen worker 68 - jepsen.util 6818 :invoke :write [22 18] INFO [2019-12-17 12:31:57,275] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":30950,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:31:57,275] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":30957,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:31:57,275] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":30950,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:31:57,275] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":30957,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:31:57,275] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":30960,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:31:57,275] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":30960,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:31:57,292] jepsen worker 6 - jepsen.util 7476 :info :write [2 28] :RequestCanceledException INFO [2019-12-17 12:31:57,293] jepsen worker 6 - jepsen.util 7566 :invoke :cas [2 [16 39]] INFO [2019-12-17 12:31:57,299] jepsen worker 79 - jepsen.util 5209 :fail :cas [26 [35 39]] :ValueNotSwapFrom INFO [2019-12-17 12:31:57,300] jepsen worker 79 - jepsen.util 5209 :invoke :cas [26 [25 29]] INFO [2019-12-17 12:31:57,309] jepsen worker 48 - jepsen.util 6528 :info :cas [16 [24 12]] :RequestCanceledException INFO [2019-12-17 12:31:57,310] jepsen worker 48 - jepsen.util 6618 :invoke :cas [16 [46 25]] INFO [2019-12-17 12:31:57,325] jepsen worker 76 - jepsen.util 7276 :info :write [25 11] :RequestCanceledException INFO [2019-12-17 12:31:57,325] jepsen worker 60 - jepsen.util 7890 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:31:57,325] jepsen worker 15 - jepsen.util 6225 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:31:57,325] jepsen worker 15 - jepsen.util 6225 :invoke :read [5 nil] INFO [2019-12-17 12:31:57,325] jepsen worker 60 - jepsen.util 7890 :invoke :cas [20 [15 41]] INFO [2019-12-17 12:31:57,326] jepsen worker 76 - jepsen.util 7366 :invoke :read [25 nil] INFO [2019-12-17 12:31:57,339] jepsen worker 12 - jepsen.util 5592 :info :write [4 23] :RequestCanceledException INFO [2019-12-17 12:31:57,339] jepsen worker 19 - jepsen.util 5239 :info :write [6 31] :RequestCanceledException INFO [2019-12-17 12:31:57,340] jepsen worker 12 - jepsen.util 5682 :invoke :write [4 27] INFO [2019-12-17 12:31:57,340] jepsen worker 19 - jepsen.util 5329 :invoke :write [6 22] INFO [2019-12-17 12:31:57,357] jepsen worker 17 - jepsen.util 7217 :info :write [5 21] :RequestCanceledException INFO [2019-12-17 12:31:57,360] jepsen worker 17 - jepsen.util 7307 :invoke :read [5 nil] INFO [2019-12-17 12:31:57,360] jepsen worker 70 - jepsen.util 6010 :fail :cas [23 [10 41]] :ValueNotSwapFrom INFO [2019-12-17 12:31:57,360] jepsen worker 70 - jepsen.util 6010 :invoke :write [23 48] INFO [2019-12-17 12:31:57,392] jepsen worker 33 - jepsen.util 7143 :info :cas [11 [34 45]] :RequestCanceledException INFO [2019-12-17 12:31:57,392] jepsen worker 84 - jepsen.util 6744 :info :write [28 33] :RequestCanceledException INFO [2019-12-17 12:31:57,392] jepsen worker 16 - jepsen.util 8296 :info :cas [5 [17 16]] :RequestCanceledException INFO [2019-12-17 12:31:57,392] jepsen worker 4 - jepsen.util 4504 :info :write [1 39] :RequestCanceledException INFO [2019-12-17 12:31:57,392] jepsen worker 87 - jepsen.util 4767 :info :write [29 43] :RequestCanceledException INFO [2019-12-17 12:31:57,393] jepsen worker 84 - jepsen.util 6834 :invoke :read [28 nil] INFO [2019-12-17 12:31:57,393] jepsen worker 16 - jepsen.util 8386 :invoke :read [5 nil] INFO [2019-12-17 12:31:57,393] jepsen worker 33 - jepsen.util 7233 :invoke :write [11 27] INFO [2019-12-17 12:31:57,393] jepsen worker 4 - jepsen.util 4594 :invoke :cas [1 [36 0]] INFO [2019-12-17 12:31:57,393] jepsen worker 87 - jepsen.util 4857 :invoke :read [29 nil] INFO [2019-12-17 12:31:57,399] jepsen worker 32 - jepsen.util 5432 :ok :write [10 18] INFO [2019-12-17 12:31:57,399] jepsen worker 32 - jepsen.util 5432 :invoke :read [10 nil] INFO [2019-12-17 12:31:57,404] jepsen worker 85 - jepsen.util 8095 :info :cas [28 [22 38]] :RequestCanceledException INFO [2019-12-17 12:31:57,404] jepsen worker 77 - jepsen.util 7277 :info :write [25 29] :RequestCanceledException INFO [2019-12-17 12:31:57,405] jepsen worker 85 - jepsen.util 8185 :invoke :read [28 nil] INFO [2019-12-17 12:31:57,407] jepsen worker 77 - jepsen.util 7367 :invoke :cas [25 [4 9]] INFO [2019-12-17 12:31:57,413] jepsen worker 18 - jepsen.util 5328 :ok :write [6 7] INFO [2019-12-17 12:31:57,413] jepsen worker 10 - jepsen.util 5320 :ok :write [3 0] INFO [2019-12-17 12:31:57,413] jepsen worker 10 - jepsen.util 5320 :invoke :cas [3 [24 48]] INFO [2019-12-17 12:31:57,413] jepsen worker 18 - jepsen.util 5328 :invoke :cas [6 [21 2]] INFO [2019-12-17 12:31:57,426] jepsen worker 65 - jepsen.util 4565 :info :write [21 11] :RequestCanceledException INFO [2019-12-17 12:31:57,426] jepsen worker 39 - jepsen.util 6789 :info :write [13 49] :RequestCanceledException INFO [2019-12-17 12:31:57,426] jepsen worker 82 - jepsen.util 7822 :info :cas [27 [33 44]] :RequestCanceledException INFO [2019-12-17 12:31:57,427] jepsen worker 39 - jepsen.util 6879 :invoke :write [13 40] INFO [2019-12-17 12:31:57,428] jepsen worker 65 - jepsen.util 4655 :invoke :read [21 nil] INFO [2019-12-17 12:31:57,428] jepsen worker 82 - jepsen.util 7912 :invoke :read [27 nil] INFO [2019-12-17 12:31:57,436] jepsen worker 46 - jepsen.util 5356 :ok :write [15 29] INFO [2019-12-17 12:31:57,436] jepsen worker 46 - jepsen.util 5356 :invoke :cas [15 [3 40]] INFO [2019-12-17 12:31:57,440] jepsen worker 66 - jepsen.util 6636 :info :write [22 48] :RequestCanceledException INFO [2019-12-17 12:31:57,440] jepsen worker 73 - jepsen.util 4933 :info :cas [24 [38 15]] :RequestCanceledException INFO [2019-12-17 12:31:57,440] jepsen worker 41 - jepsen.util 7871 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:31:57,440] jepsen worker 57 - jepsen.util 6717 :info :cas [19 [23 49]] :RequestCanceledException INFO [2019-12-17 12:31:57,440] jepsen worker 41 - jepsen.util 7871 :invoke :write [13 34] INFO [2019-12-17 12:31:57,441] jepsen worker 66 - jepsen.util 6726 :invoke :write [22 10] INFO [2019-12-17 12:31:57,441] jepsen worker 57 - jepsen.util 6807 :invoke :cas [19 [23 20]] INFO [2019-12-17 12:31:57,442] jepsen worker 73 - jepsen.util 5023 :invoke :write [24 43] INFO [2019-12-17 12:31:57,453] jepsen worker 75 - jepsen.util 6915 :info :cas [25 [31 3]] :RequestCanceledException INFO [2019-12-17 12:31:57,454] jepsen worker 75 - jepsen.util 7005 :invoke :read [25 nil] INFO [2019-12-17 12:31:57,477] jepsen worker 32 - jepsen.util 5432 :ok :read [10 18] INFO [2019-12-17 12:31:57,478] jepsen worker 32 - jepsen.util 5432 :invoke :read [10 nil] INFO [2019-12-17 12:31:57,485] jepsen worker 31 - jepsen.util 4981 :ok :read [10 18] INFO [2019-12-17 12:31:57,488] jepsen worker 31 - jepsen.util 4981 :invoke :write [10 12] INFO [2019-12-17 12:31:57,491] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":30986,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:31:57,491] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":30986,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:31:57,492] jepsen worker 22 - jepsen.util 7312 :info :cas [7 [20 43]] :RequestCanceledException INFO [2019-12-17 12:31:57,492] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":30990,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:31:57,492] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":30990,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:31:57,493] jepsen worker 22 - jepsen.util 7402 :invoke :read [7 nil] INFO [2019-12-17 12:31:57,504] jepsen worker 14 - jepsen.util 5414 :ok :read [4 20] INFO [2019-12-17 12:31:57,505] jepsen worker 14 - jepsen.util 5414 :invoke :read [4 nil] INFO [2019-12-17 12:31:57,505] jepsen worker 18 - jepsen.util 5328 :fail :cas [6 [21 2]] :ValueNotSwapFrom INFO [2019-12-17 12:31:57,505] jepsen worker 18 - jepsen.util 5328 :invoke :write [6 8] INFO [2019-12-17 12:31:57,507] jepsen worker 46 - jepsen.util 5356 :fail :cas [15 [3 40]] :ValueNotSwapFrom INFO [2019-12-17 12:31:57,507] jepsen worker 46 - jepsen.util 5356 :invoke :write [15 27] INFO [2019-12-17 12:31:57,508] jepsen worker 53 - jepsen.util 5183 :info :write [17 45] :RequestCanceledException INFO [2019-12-17 12:31:57,508] jepsen worker 40 - jepsen.util 7600 :info :cas [13 [5 38]] :RequestCanceledException INFO [2019-12-17 12:31:57,509] jepsen worker 53 - jepsen.util 5273 :invoke :cas [17 [38 28]] INFO [2019-12-17 12:31:57,509] jepsen worker 40 - jepsen.util 7690 :invoke :cas [13 [34 11]] INFO [2019-12-17 12:31:57,527] jepsen worker 28 - jepsen.util 4078 :info :write [9 45] :RequestCanceledException INFO [2019-12-17 12:31:57,527] jepsen worker 54 - jepsen.util 5724 :info :write [18 28] :RequestCanceledException INFO [2019-12-17 12:31:57,538] jepsen worker 28 - jepsen.util 4168 :invoke :write [9 14] INFO [2019-12-17 12:31:57,538] jepsen worker 54 - jepsen.util 5814 :invoke :write [18 48] INFO [2019-12-17 12:31:57,540] jepsen worker 43 - jepsen.util 6703 :info :write [14 26] :RequestCanceledException INFO [2019-12-17 12:31:57,541] jepsen worker 43 - jepsen.util 6793 :invoke :read [14 nil] INFO [2019-12-17 12:31:57,546] jepsen worker 20 - jepsen.util 5510 :ok :read [6 7] INFO [2019-12-17 12:31:57,546] jepsen worker 20 - jepsen.util 5510 :invoke :read [6 nil] INFO [2019-12-17 12:31:57,548] jepsen worker 32 - jepsen.util 5432 :ok :read [10 18] INFO [2019-12-17 12:31:57,548] jepsen worker 32 - jepsen.util 5432 :invoke :read [10 nil] INFO [2019-12-17 12:31:57,557] jepsen worker 83 - jepsen.util 7463 :info :write [27 18] :RequestCanceledException INFO [2019-12-17 12:31:57,558] jepsen worker 83 - jepsen.util 7553 :invoke :write [27 11] INFO [2019-12-17 12:31:57,573] jepsen worker 32 - jepsen.util 5432 :ok :read [10 18] INFO [2019-12-17 12:31:57,573] jepsen worker 32 - jepsen.util 5432 :invoke :cas [10 [38 3]] INFO [2019-12-17 12:31:57,581] jepsen worker 10 - jepsen.util 5320 :fail :cas [3 [24 48]] :ValueNotSwapFrom INFO [2019-12-17 12:31:57,581] jepsen worker 10 - jepsen.util 5320 :invoke :read [3 nil] INFO [2019-12-17 12:31:57,590] jepsen worker 65 - jepsen.util 4655 :ok :read [21 24] INFO [2019-12-17 12:31:57,590] jepsen worker 65 - jepsen.util 4655 :invoke :read [21 nil] INFO [2019-12-17 12:31:57,614] jepsen worker 32 - jepsen.util 5432 :fail :cas [10 [38 3]] :ValueNotSwapFrom INFO [2019-12-17 12:31:57,614] jepsen worker 32 - jepsen.util 5432 :invoke :write [10 21] INFO [2019-12-17 12:31:57,626] jepsen worker 27 - jepsen.util 5067 :info :write [9 41] :RequestCanceledException INFO [2019-12-17 12:31:57,626] jepsen worker 25 - jepsen.util 7585 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:31:57,626] jepsen worker 25 - jepsen.util 7585 :invoke :write [8 5] INFO [2019-12-17 12:31:57,628] jepsen worker 27 - jepsen.util 5157 :invoke :cas [9 [49 40]] INFO [2019-12-17 12:31:57,640] jepsen worker 44 - jepsen.util 7784 :info :cas [14 [21 24]] :RequestCanceledException INFO [2019-12-17 12:31:57,641] jepsen worker 44 - jepsen.util 7874 :invoke :write [14 20] INFO [2019-12-17 12:31:57,645] jepsen worker 28 - jepsen.util 4168 :ok :write [9 14] INFO [2019-12-17 12:31:57,645] jepsen worker 28 - jepsen.util 4168 :invoke :read [9 nil] INFO [2019-12-17 12:31:57,672] jepsen worker 73 - jepsen.util 5023 :ok :write [24 43] INFO [2019-12-17 12:31:57,672] jepsen worker 73 - jepsen.util 5023 :invoke :cas [24 [44 8]] INFO [2019-12-17 12:31:57,691] jepsen worker 28 - jepsen.util 4168 :ok :read [9 14] INFO [2019-12-17 12:31:57,691] jepsen worker 28 - jepsen.util 4168 :invoke :cas [9 [43 11]] INFO [2019-12-17 12:31:57,693] jepsen worker 2 - jepsen.util 6752 :info :cas [0 [45 17]] :RequestCanceledException INFO [2019-12-17 12:31:57,694] jepsen worker 5 - jepsen.util 4955 :ok :write [1 9] INFO [2019-12-17 12:31:57,694] jepsen worker 2 - jepsen.util 6842 :invoke :cas [0 [10 42]] INFO [2019-12-17 12:31:57,694] jepsen worker 5 - jepsen.util 4955 :invoke :read [1 nil] INFO [2019-12-17 12:31:57,696] jepsen worker 73 - jepsen.util 5023 :fail :cas [24 [44 8]] :ValueNotSwapFrom INFO [2019-12-17 12:31:57,696] jepsen worker 73 - jepsen.util 5023 :invoke :read [24 nil] INFO [2019-12-17 12:31:57,710] jepsen worker 58 - jepsen.util 7618 :info :cas [19 [41 16]] :RequestCanceledException INFO [2019-12-17 12:31:57,713] jepsen worker 58 - jepsen.util 7708 :invoke :read [19 nil] INFO [2019-12-17 12:31:57,715] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31023,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:31:57,715] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31023,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:31:57,718] jepsen worker 3 - jepsen.util 5403 :fail :cas [1 [6 49]] :ValueNotSwapFrom INFO [2019-12-17 12:31:57,718] jepsen worker 3 - jepsen.util 5403 :invoke :cas [1 [9 13]] INFO [2019-12-17 12:31:57,724] jepsen worker 28 - jepsen.util 4168 :fail :cas [9 [43 11]] :ValueNotSwapFrom INFO [2019-12-17 12:31:57,724] jepsen worker 28 - jepsen.util 4168 :invoke :cas [9 [46 22]] INFO [2019-12-17 12:31:57,726] jepsen worker 61 - jepsen.util 8071 :info :write [20 8] :RequestCanceledException INFO [2019-12-17 12:31:57,726] jepsen worker 51 - jepsen.util 5271 :info :write [17 17] :RequestCanceledException INFO [2019-12-17 12:31:57,728] jepsen worker 51 - jepsen.util 5361 :invoke :read [17 nil] INFO [2019-12-17 12:31:57,728] jepsen worker 61 - jepsen.util 8161 :invoke :write [20 7] INFO [2019-12-17 12:31:57,730] jepsen worker 73 - jepsen.util 5023 :ok :read [24 43] INFO [2019-12-17 12:31:57,732] jepsen worker 73 - jepsen.util 5023 :invoke :cas [24 [32 47]] INFO [2019-12-17 12:31:57,740] jepsen worker 34 - jepsen.util 6604 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:31:57,740] jepsen worker 36 - jepsen.util 5616 :info :write [12 40] :RequestCanceledException INFO [2019-12-17 12:31:57,740] jepsen worker 34 - jepsen.util 6604 :invoke :write [11 16] INFO [2019-12-17 12:31:57,742] jepsen worker 36 - jepsen.util 5706 :invoke :write [12 3] INFO [2019-12-17 12:31:57,755] jepsen worker 28 - jepsen.util 4168 :fail :cas [9 [46 22]] :ValueNotSwapFrom INFO [2019-12-17 12:31:57,755] jepsen worker 28 - jepsen.util 4168 :invoke :write [9 40] INFO [2019-12-17 12:31:57,757] jepsen worker 81 - jepsen.util 7191 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:31:57,757] jepsen worker 81 - jepsen.util 7191 :invoke :write [27 46] INFO [2019-12-17 12:31:57,767] jepsen worker 73 - jepsen.util 5023 :fail :cas [24 [32 47]] :ValueNotSwapFrom INFO [2019-12-17 12:31:57,767] jepsen worker 73 - jepsen.util 5023 :invoke :write [24 48] INFO [2019-12-17 12:31:57,789] jepsen worker 51 - jepsen.util 5361 :ok :read [17 38] INFO [2019-12-17 12:31:57,789] jepsen worker 51 - jepsen.util 5361 :invoke :write [17 31] INFO [2019-12-17 12:31:57,793] jepsen worker 89 - jepsen.util 5129 :info :write [29 22] :RequestCanceledException INFO [2019-12-17 12:31:57,797] jepsen worker 89 - jepsen.util 5219 :invoke :cas [29 [18 4]] INFO [2019-12-17 12:31:57,808] jepsen worker 45 - jepsen.util 5445 :info :write [15 38] :RequestCanceledException INFO [2019-12-17 12:31:57,808] jepsen worker 80 - jepsen.util 5390 :info :write [26 0] :RequestCanceledException INFO [2019-12-17 12:31:57,808] jepsen worker 38 - jepsen.util 5348 :info :write [12 45] :RequestCanceledException INFO [2019-12-17 12:31:57,808] jepsen worker 74 - jepsen.util 5834 :info :cas [24 [24 3]] :RequestCanceledException INFO [2019-12-17 12:31:57,810] jepsen worker 45 - jepsen.util 5535 :invoke :read [15 nil] INFO [2019-12-17 12:31:57,810] jepsen worker 74 - jepsen.util 5924 :invoke :cas [24 [19 14]] INFO [2019-12-17 12:31:57,810] jepsen worker 38 - jepsen.util 5438 :invoke :write [12 0] INFO [2019-12-17 12:31:57,811] jepsen worker 80 - jepsen.util 5480 :invoke :read [26 nil] INFO [2019-12-17 12:31:57,824] jepsen worker 79 - jepsen.util 5209 :fail :cas [26 [25 29]] :ValueNotSwapFrom INFO [2019-12-17 12:31:57,824] jepsen worker 79 - jepsen.util 5209 :invoke :write [26 28] INFO [2019-12-17 12:31:57,825] jepsen worker 86 - jepsen.util 7466 :info :cas [28 [20 30]] :RequestCanceledException INFO [2019-12-17 12:31:57,827] jepsen worker 86 - jepsen.util 7556 :invoke :read [28 nil] INFO [2019-12-17 12:31:57,828] jepsen worker 28 - jepsen.util 4168 :ok :write [9 40] INFO [2019-12-17 12:31:57,828] jepsen worker 28 - jepsen.util 4168 :invoke :read [9 nil] INFO [2019-12-17 12:31:57,840] jepsen worker 0 - jepsen.util 7740 :info :write [0 39] :RequestCanceledException INFO [2019-12-17 12:31:57,842] jepsen worker 0 - jepsen.util 7830 :invoke :read [0 nil] INFO [2019-12-17 12:31:57,847] jepsen worker 80 - jepsen.util 5480 :ok :read [26 7] INFO [2019-12-17 12:31:57,847] jepsen worker 80 - jepsen.util 5480 :invoke :cas [26 [17 40]] INFO [2019-12-17 12:31:57,850] jepsen worker 45 - jepsen.util 5535 :ok :read [15 29] INFO [2019-12-17 12:31:57,850] jepsen worker 45 - jepsen.util 5535 :invoke :read [15 nil] INFO [2019-12-17 12:31:57,866] jepsen worker 87 - jepsen.util 4857 :ok :read [29 36] INFO [2019-12-17 12:31:57,866] jepsen worker 87 - jepsen.util 4857 :invoke :read [29 nil] INFO [2019-12-17 12:31:57,876] jepsen worker 28 - jepsen.util 4168 :ok :read [9 40] INFO [2019-12-17 12:31:57,876] jepsen worker 28 - jepsen.util 4168 :invoke :read [9 nil] INFO [2019-12-17 12:31:57,876] jepsen worker 31 - jepsen.util 4981 :ok :write [10 12] INFO [2019-12-17 12:31:57,876] jepsen worker 31 - jepsen.util 4981 :invoke :read [10 nil] INFO [2019-12-17 12:31:57,880] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31057,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:31:57,880] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31057,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:31:57,880] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31061,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:31:57,880] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31061,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:31:57,894] jepsen worker 80 - jepsen.util 5480 :fail :cas [26 [17 40]] :ValueNotSwapFrom INFO [2019-12-17 12:31:57,894] jepsen worker 80 - jepsen.util 5480 :invoke :read [26 nil] INFO [2019-12-17 12:31:57,904] jepsen worker 72 - jepsen.util 5112 :info :write [24 42] :RequestCanceledException INFO [2019-12-17 12:31:57,905] jepsen worker 72 - jepsen.util 5202 :invoke :cas [24 [46 18]] INFO [2019-12-17 12:31:57,905] jepsen worker 20 - jepsen.util 5510 :ok :read [6 7] INFO [2019-12-17 12:31:57,905] jepsen worker 20 - jepsen.util 5510 :invoke :cas [6 [46 30]] INFO [2019-12-17 12:31:57,915] jepsen worker 10 - jepsen.util 5320 :ok :read [3 0] INFO [2019-12-17 12:31:57,915] jepsen worker 10 - jepsen.util 5320 :invoke :read [3 nil] INFO [2019-12-17 12:31:57,924] jepsen worker 62 - jepsen.util 6902 :info :write [20 8] :RequestCanceledException INFO [2019-12-17 12:31:57,925] jepsen worker 62 - jepsen.util 6992 :invoke :read [20 nil] INFO [2019-12-17 12:31:57,927] jepsen worker 65 - jepsen.util 4655 :ok :read [21 24] INFO [2019-12-17 12:31:57,927] jepsen worker 45 - jepsen.util 5535 :ok :read [15 29] INFO [2019-12-17 12:31:57,927] jepsen worker 65 - jepsen.util 4655 :invoke :read [21 nil] INFO [2019-12-17 12:31:57,928] jepsen worker 45 - jepsen.util 5535 :invoke :read [15 nil] INFO [2019-12-17 12:31:57,940] jepsen worker 67 - jepsen.util 7267 :info :cas [22 [13 16]] :RequestCanceledException INFO [2019-12-17 12:31:57,941] jepsen worker 67 - jepsen.util 7357 :invoke :read [22 nil] INFO [2019-12-17 12:31:57,962] jepsen worker 80 - jepsen.util 5480 :ok :read [26 7] INFO [2019-12-17 12:31:57,962] jepsen worker 80 - jepsen.util 5480 :invoke :write [26 6] INFO [2019-12-17 12:31:57,978] jepsen worker 31 - jepsen.util 4981 :ok :read [10 12] INFO [2019-12-17 12:31:57,979] jepsen worker 31 - jepsen.util 4981 :invoke :read [10 nil] INFO [2019-12-17 12:31:57,995] jepsen worker 28 - jepsen.util 4168 :ok :read [9 40] INFO [2019-12-17 12:31:57,995] jepsen worker 28 - jepsen.util 4168 :invoke :write [9 43] INFO [2019-12-17 12:31:58,010] jepsen worker 26 - jepsen.util 7586 :info :write [8 8] :RequestCanceledException INFO [2019-12-17 12:31:58,011] jepsen worker 26 - jepsen.util 7676 :invoke :write [8 35] INFO [2019-12-17 12:31:58,018] jepsen worker 20 - jepsen.util 5510 :fail :cas [6 [46 30]] :ValueNotSwapFrom INFO [2019-12-17 12:31:58,019] jepsen worker 20 - jepsen.util 5510 :invoke :read [6 nil] INFO [2019-12-17 12:31:58,038] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31077,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:31:58,038] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31077,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:31:58,039] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31080,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:31:58,039] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31080,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:31:58,056] jepsen worker 21 - jepsen.util 6771 :info :write [7 21] :RequestCanceledException INFO [2019-12-17 12:31:58,056] jepsen worker 69 - jepsen.util 5829 :info :write [23 18] :RequestCanceledException INFO [2019-12-17 12:31:58,057] jepsen worker 69 - jepsen.util 5919 :invoke :read [23 nil] INFO [2019-12-17 12:31:58,058] jepsen worker 21 - jepsen.util 6861 :invoke :read [7 nil] INFO [2019-12-17 12:31:58,060] jepsen worker 72 - jepsen.util 5202 :fail :cas [24 [46 18]] :ValueNotSwapFrom INFO [2019-12-17 12:31:58,060] jepsen worker 72 - jepsen.util 5202 :invoke :read [24 nil] INFO [2019-12-17 12:31:58,087] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:31:58,087] jepsen worker 10 - jepsen.util 5320 :ok :read [3 0] INFO [2019-12-17 12:31:58,087] jepsen nemesis - couchbase.util Rebalance check count: 233 INFO [2019-12-17 12:31:58,087] jepsen worker 10 - jepsen.util 5320 :invoke :read [3 nil] INFO [2019-12-17 12:31:58,087] jepsen nemesis - couchbase.util Rebalance stuck count: 214 INFO [2019-12-17 12:31:58,110] jepsen worker 45 - jepsen.util 5535 :ok :read [15 29] INFO [2019-12-17 12:31:58,110] jepsen worker 45 - jepsen.util 5535 :invoke :write [15 28] INFO [2019-12-17 12:31:58,111] jepsen worker 1 - jepsen.util 7021 :info :write [0 33] :RequestCanceledException INFO [2019-12-17 12:31:58,112] jepsen worker 1 - jepsen.util 7111 :invoke :cas [0 [41 34]] INFO [2019-12-17 12:31:58,123] jepsen worker 14 - jepsen.util 5414 :ok :read [4 20] INFO [2019-12-17 12:31:58,123] jepsen worker 14 - jepsen.util 5414 :invoke :write [4 47] INFO [2019-12-17 12:31:58,126] jepsen worker 7 - jepsen.util 7387 :info :write [2 43] :RequestCanceledException INFO [2019-12-17 12:31:58,126] jepsen worker 71 - jepsen.util 6101 :info :write [23 19] :RequestCanceledException INFO [2019-12-17 12:31:58,128] jepsen worker 7 - jepsen.util 7477 :invoke :cas [2 [12 49]] INFO [2019-12-17 12:31:58,129] jepsen worker 65 - jepsen.util 4655 :ok :read [21 24] INFO [2019-12-17 12:31:58,129] jepsen worker 65 - jepsen.util 4655 :invoke :write [21 28] INFO [2019-12-17 12:31:58,133] jepsen worker 71 - jepsen.util 6191 :invoke :cas [23 [14 3]] INFO [2019-12-17 12:31:58,134] jepsen worker 31 - jepsen.util 4981 :ok :read [10 12] INFO [2019-12-17 12:31:58,134] jepsen worker 31 - jepsen.util 4981 :invoke :read [10 nil] INFO [2019-12-17 12:31:58,166] jepsen worker 20 - jepsen.util 5510 :ok :read [6 7] INFO [2019-12-17 12:31:58,173] jepsen worker 46 - jepsen.util 5356 :ok :write [15 27] INFO [2019-12-17 12:31:58,174] jepsen worker 10 - jepsen.util 5320 :ok :read [3 0] INFO [2019-12-17 12:31:58,177] jepsen worker 28 - jepsen.util 4168 :ok :write [9 43] INFO [2019-12-17 12:31:58,177] jepsen worker 5 - jepsen.util 4955 :ok :read [1 9] INFO [2019-12-17 12:31:58,192] jepsen worker 30 - jepsen.util 5430 :fail :read [10 nil] :RequestTimeoutException INFO [2019-12-17 12:31:58,227] jepsen worker 10 - jepsen.util 5320 :invoke :read [3 nil] INFO [2019-12-17 12:31:58,227] jepsen worker 20 - jepsen.util 5510 :invoke :read [6 nil] INFO [2019-12-17 12:31:58,227] jepsen worker 46 - jepsen.util 5356 :invoke :read [15 nil] INFO [2019-12-17 12:31:58,227] jepsen worker 28 - jepsen.util 4168 :invoke :write [9 36] INFO [2019-12-17 12:31:58,227] jepsen worker 5 - jepsen.util 4955 :invoke :cas [1 [14 25]] INFO [2019-12-17 12:31:58,227] jepsen worker 30 - jepsen.util 5430 :invoke :cas [10 [16 30]] INFO [2019-12-17 12:31:58,255] jepsen worker 65 - jepsen.util 4655 :ok :write [21 28] INFO [2019-12-17 12:31:58,255] jepsen worker 65 - jepsen.util 4655 :invoke :write [21 28] INFO [2019-12-17 12:31:58,255] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31095,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:31:58,255] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31102,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:31:58,255] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31095,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:31:58,255] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31102,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:31:58,255] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31097,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:31:58,255] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31106,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:31:58,255] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31097,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:31:58,255] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31106,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:31:58,261] jepsen worker 73 - jepsen.util 5023 :ok :write [24 48] INFO [2019-12-17 12:31:58,262] jepsen worker 73 - jepsen.util 5023 :invoke :write [24 43] INFO [2019-12-17 12:31:58,263] jepsen worker 46 - jepsen.util 5356 :ok :read [15 27] INFO [2019-12-17 12:31:58,264] jepsen worker 46 - jepsen.util 5356 :invoke :cas [15 [44 47]] INFO [2019-12-17 12:31:58,287] jepsen worker 45 - jepsen.util 5535 :ok :write [15 28] INFO [2019-12-17 12:31:58,288] jepsen worker 45 - jepsen.util 5535 :invoke :read [15 nil] INFO [2019-12-17 12:31:58,292] jepsen worker 30 - jepsen.util 5430 :fail :cas [10 [16 30]] :ValueNotSwapFrom INFO [2019-12-17 12:31:58,292] jepsen worker 30 - jepsen.util 5430 :invoke :cas [10 [12 14]] INFO [2019-12-17 12:31:58,322] jepsen worker 31 - jepsen.util 4981 :ok :read [10 12] INFO [2019-12-17 12:31:58,322] jepsen worker 31 - jepsen.util 4981 :invoke :write [10 38] INFO [2019-12-17 12:31:58,327] jepsen worker 35 - jepsen.util 7415 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:31:58,327] jepsen worker 35 - jepsen.util 7415 :invoke :read [11 nil] INFO [2019-12-17 12:31:58,336] jepsen worker 89 - jepsen.util 5219 :fail :cas [29 [18 4]] :ValueNotSwapFrom INFO [2019-12-17 12:31:58,336] jepsen worker 89 - jepsen.util 5219 :invoke :read [29 nil] INFO [2019-12-17 12:31:58,340] jepsen worker 49 - jepsen.util 6439 :info :cas [16 [44 47]] :RequestCanceledException INFO [2019-12-17 12:31:58,340] jepsen worker 42 - jepsen.util 7782 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:31:58,340] jepsen worker 42 - jepsen.util 7782 :invoke :cas [14 [41 33]] INFO [2019-12-17 12:31:58,344] jepsen worker 49 - jepsen.util 6529 :invoke :cas [16 [20 15]] INFO [2019-12-17 12:31:58,357] jepsen worker 24 - jepsen.util 6684 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:31:58,357] jepsen worker 24 - jepsen.util 6684 :invoke :read [8 nil] INFO [2019-12-17 12:31:58,388] jepsen worker 10 - jepsen.util 5320 :ok :read [3 0] INFO [2019-12-17 12:31:58,388] jepsen worker 10 - jepsen.util 5320 :invoke :write [3 27] INFO [2019-12-17 12:31:58,393] jepsen worker 50 - jepsen.util 7340 :info :cas [16 [4 16]] :RequestCanceledException INFO [2019-12-17 12:31:58,394] jepsen worker 50 - jepsen.util 7430 :invoke :write [16 49] INFO [2019-12-17 12:31:58,404] jepsen worker 46 - jepsen.util 5356 :fail :cas [15 [44 47]] :ValueNotSwapFrom INFO [2019-12-17 12:31:58,404] jepsen worker 46 - jepsen.util 5356 :invoke :write [15 3] INFO [2019-12-17 12:31:58,425] jepsen worker 5 - jepsen.util 4955 :fail :cas [1 [14 25]] :ValueNotSwapFrom INFO [2019-12-17 12:31:58,425] jepsen worker 5 - jepsen.util 4955 :invoke :read [1 nil] INFO [2019-12-17 12:31:58,426] jepsen worker 23 - jepsen.util 7403 :info :cas [7 [5 44]] :RequestCanceledException INFO [2019-12-17 12:31:58,427] jepsen worker 23 - jepsen.util 7493 :invoke :read [7 nil] INFO [2019-12-17 12:31:58,445] jepsen worker 87 - jepsen.util 4857 :ok :read [29 36] INFO [2019-12-17 12:31:58,445] jepsen worker 87 - jepsen.util 4857 :invoke :write [29 38] INFO [2019-12-17 12:31:58,474] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31130,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:31:58,474] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31130,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:31:58,475] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31134,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:31:58,475] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31134,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:31:58,475] jepsen worker 71 - jepsen.util 6191 :fail :cas [23 [14 3]] :ValueNotSwapFrom INFO [2019-12-17 12:31:58,475] jepsen worker 71 - jepsen.util 6191 :invoke :cas [23 [7 38]] INFO [2019-12-17 12:31:58,486] jepsen worker 20 - jepsen.util 5510 :ok :read [6 7] INFO [2019-12-17 12:31:58,486] jepsen worker 20 - jepsen.util 5510 :invoke :cas [6 [42 31]] INFO [2019-12-17 12:31:58,493] jepsen worker 59 - jepsen.util 7169 :info :cas [19 [21 15]] :RequestCanceledException INFO [2019-12-17 12:31:58,494] jepsen worker 59 - jepsen.util 7259 :invoke :write [19 1] INFO [2019-12-17 12:31:58,497] jepsen worker 28 - jepsen.util 4168 :ok :write [9 36] INFO [2019-12-17 12:31:58,497] jepsen worker 28 - jepsen.util 4168 :invoke :read [9 nil] INFO [2019-12-17 12:31:58,502] jepsen worker 5 - jepsen.util 4955 :ok :read [1 9] INFO [2019-12-17 12:31:58,503] jepsen worker 5 - jepsen.util 4955 :invoke :cas [1 [28 2]] INFO [2019-12-17 12:31:58,513] jepsen worker 28 - jepsen.util 4168 :ok :read [9 36] INFO [2019-12-17 12:31:58,513] jepsen worker 28 - jepsen.util 4168 :invoke :write [9 0] INFO [2019-12-17 12:31:58,513] jepsen worker 65 - jepsen.util 4655 :ok :write [21 28] INFO [2019-12-17 12:31:58,513] jepsen worker 65 - jepsen.util 4655 :invoke :write [21 10] INFO [2019-12-17 12:31:58,565] jepsen worker 28 - jepsen.util 4168 :ok :write [9 0] INFO [2019-12-17 12:31:58,565] jepsen worker 28 - jepsen.util 4168 :invoke :cas [9 [24 4]] INFO [2019-12-17 12:31:58,573] jepsen worker 73 - jepsen.util 5023 :ok :write [24 43] INFO [2019-12-17 12:31:58,573] jepsen worker 73 - jepsen.util 5023 :invoke :cas [24 [24 3]] INFO [2019-12-17 12:31:58,590] jepsen worker 46 - jepsen.util 5356 :ok :write [15 3] INFO [2019-12-17 12:31:58,591] jepsen worker 46 - jepsen.util 5356 :invoke :cas [15 [22 13]] INFO [2019-12-17 12:31:58,592] jepsen worker 13 - jepsen.util 5233 :info :write [4 30] :RequestCanceledException INFO [2019-12-17 12:31:58,594] jepsen worker 13 - jepsen.util 5323 :invoke :write [4 21] INFO [2019-12-17 12:31:58,599] jepsen worker 28 - jepsen.util 4168 :fail :cas [9 [24 4]] :ValueNotSwapFrom INFO [2019-12-17 12:31:58,600] jepsen worker 28 - jepsen.util 4168 :invoke :read [9 nil] INFO [2019-12-17 12:31:58,610] jepsen worker 73 - jepsen.util 5023 :fail :cas [24 [24 3]] :ValueNotSwapFrom INFO [2019-12-17 12:31:58,610] jepsen worker 73 - jepsen.util 5023 :invoke :write [24 14] INFO [2019-12-17 12:31:58,626] jepsen worker 9 - jepsen.util 5859 :info :write [3 0] :RequestCanceledException INFO [2019-12-17 12:31:58,626] jepsen worker 63 - jepsen.util 4293 :info :write [21 6] :RequestCanceledException INFO [2019-12-17 12:31:58,627] jepsen worker 9 - jepsen.util 5949 :invoke :cas [3 [40 34]] INFO [2019-12-17 12:31:58,627] jepsen worker 63 - jepsen.util 4383 :invoke :cas [21 [14 29]] INFO [2019-12-17 12:31:58,638] jepsen worker 5 - jepsen.util 4955 :fail :cas [1 [28 2]] :ValueNotSwapFrom INFO [2019-12-17 12:31:58,638] jepsen worker 5 - jepsen.util 4955 :invoke :cas [1 [28 33]] INFO [2019-12-17 12:31:58,663] jepsen worker 28 - jepsen.util 4168 :ok :read [9 0] INFO [2019-12-17 12:31:58,663] jepsen worker 28 - jepsen.util 4168 :invoke :write [9 17] INFO [2019-12-17 12:31:58,663] jepsen worker 46 - jepsen.util 5356 :fail :cas [15 [22 13]] :ValueNotSwapFrom INFO [2019-12-17 12:31:58,664] jepsen worker 46 - jepsen.util 5356 :invoke :read [15 nil] INFO [2019-12-17 12:31:58,684] jepsen worker 9 - jepsen.util 5949 :fail :cas [3 [40 34]] :ValueNotSwapFrom INFO [2019-12-17 12:31:58,684] jepsen worker 9 - jepsen.util 5949 :invoke :write [3 1] INFO [2019-12-17 12:31:58,701] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31155,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:31:58,701] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31155,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:31:58,702] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31158,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:31:58,702] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31158,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:31:58,702] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31159,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:31:58,702] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31159,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:31:58,711] jepsen worker 5 - jepsen.util 4955 :fail :cas [1 [28 33]] :ValueNotSwapFrom INFO [2019-12-17 12:31:58,711] jepsen worker 5 - jepsen.util 4955 :invoke :cas [1 [49 18]] INFO [2019-12-17 12:31:58,726] jepsen worker 37 - jepsen.util 5077 :info :write [12 11] :RequestCanceledException INFO [2019-12-17 12:31:58,727] jepsen worker 37 - jepsen.util 5167 :invoke :write [12 10] INFO [2019-12-17 12:31:58,766] jepsen worker 5 - jepsen.util 4955 :fail :cas [1 [49 18]] :ValueNotSwapFrom INFO [2019-12-17 12:31:58,766] jepsen worker 5 - jepsen.util 4955 :invoke :write [1 20] INFO [2019-12-17 12:31:58,767] jepsen worker 46 - jepsen.util 5356 :ok :read [15 3] INFO [2019-12-17 12:31:58,768] jepsen worker 46 - jepsen.util 5356 :invoke :read [15 nil] INFO [2019-12-17 12:31:58,824] jepsen worker 28 - jepsen.util 4168 :ok :write [9 17] INFO [2019-12-17 12:31:58,824] jepsen worker 28 - jepsen.util 4168 :invoke :read [9 nil] INFO [2019-12-17 12:31:58,825] jepsen worker 71 - jepsen.util 6191 :fail :cas [23 [7 38]] :ValueNotSwapFrom INFO [2019-12-17 12:31:58,826] jepsen worker 71 - jepsen.util 6191 :invoke :cas [23 [11 36]] INFO [2019-12-17 12:31:58,845] jepsen worker 46 - jepsen.util 5356 :ok :read [15 3] INFO [2019-12-17 12:31:58,845] jepsen worker 46 - jepsen.util 5356 :invoke :write [15 0] INFO [2019-12-17 12:31:58,861] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31173,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:31:58,861] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31173,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:31:58,861] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31175,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:31:58,861] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31175,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:31:58,884] jepsen worker 28 - jepsen.util 4168 :ok :read [9 17] INFO [2019-12-17 12:31:58,884] jepsen worker 28 - jepsen.util 4168 :invoke :cas [9 [34 31]] INFO [2019-12-17 12:31:58,934] jepsen worker 28 - jepsen.util 4168 :fail :cas [9 [34 31]] :ValueNotSwapFrom INFO [2019-12-17 12:31:58,934] jepsen worker 28 - jepsen.util 4168 :invoke :cas [9 [41 31]] INFO [2019-12-17 12:31:58,953] jepsen worker 28 - jepsen.util 4168 :fail :cas [9 [41 31]] :ValueNotSwapFrom INFO [2019-12-17 12:31:58,953] jepsen worker 28 - jepsen.util 4168 :invoke :read [9 nil] INFO [2019-12-17 12:31:58,956] jepsen worker 78 - jepsen.util 5568 :info :write [26 28] :RequestCanceledException INFO [2019-12-17 12:31:58,957] jepsen worker 78 - jepsen.util 5658 :invoke :write [26 42] INFO [2019-12-17 12:31:58,972] jepsen worker 28 - jepsen.util 4168 :ok :read [9 17] INFO [2019-12-17 12:31:58,972] jepsen worker 28 - jepsen.util 4168 :invoke :read [9 nil] INFO [2019-12-17 12:31:58,987] jepsen worker 28 - jepsen.util 4168 :ok :read [9 17] INFO [2019-12-17 12:31:58,987] jepsen worker 9 - jepsen.util 5949 :ok :write [3 1] INFO [2019-12-17 12:31:58,988] jepsen worker 73 - jepsen.util 5023 :ok :write [24 14] INFO [2019-12-17 12:31:58,988] jepsen worker 9 - jepsen.util 5949 :invoke :read [3 nil] INFO [2019-12-17 12:31:58,988] jepsen worker 28 - jepsen.util 4168 :invoke :read [9 nil] INFO [2019-12-17 12:31:58,988] jepsen worker 73 - jepsen.util 5023 :invoke :write [24 36] INFO [2019-12-17 12:31:58,990] jepsen worker 31 - jepsen.util 4981 :ok :write [10 38] INFO [2019-12-17 12:31:58,990] jepsen worker 31 - jepsen.util 4981 :invoke :write [10 25] INFO [2019-12-17 12:31:59,021] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31185,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:31:59,021] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31185,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:31:59,021] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31188,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:31:59,022] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31188,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:31:59,022] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31189,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:31:59,022] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31189,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:31:59,026] jepsen worker 47 - jepsen.util 5987 :info :write [15 17] :RequestCanceledException INFO [2019-12-17 12:31:59,028] jepsen worker 47 - jepsen.util 6077 :invoke :write [15 45] INFO [2019-12-17 12:31:59,034] jepsen worker 63 - jepsen.util 4383 :fail :cas [21 [14 29]] :ValueNotSwapFrom INFO [2019-12-17 12:31:59,038] jepsen worker 71 - jepsen.util 6191 :fail :cas [23 [11 36]] :ValueNotSwapFrom INFO [2019-12-17 12:31:59,038] jepsen worker 63 - jepsen.util 4383 :invoke :read [21 nil] INFO [2019-12-17 12:31:59,038] jepsen worker 71 - jepsen.util 6191 :invoke :write [23 48] INFO [2019-12-17 12:31:59,038] jepsen worker 46 - jepsen.util 5356 :ok :write [15 0] INFO [2019-12-17 12:31:59,039] jepsen worker 46 - jepsen.util 5356 :invoke :cas [15 [30 22]] INFO [2019-12-17 12:31:59,040] jepsen worker 11 - jepsen.util 5591 :fail :read [3 nil] :RequestTimeoutException INFO [2019-12-17 12:31:59,040] jepsen worker 11 - jepsen.util 5591 :invoke :cas [3 [41 9]] INFO [2019-12-17 12:31:59,062] jepsen worker 28 - jepsen.util 4168 :ok :read [9 17] INFO [2019-12-17 12:31:59,064] jepsen worker 28 - jepsen.util 4168 :invoke :read [9 nil] INFO [2019-12-17 12:31:59,071] jepsen worker 9 - jepsen.util 5949 :ok :read [3 1] INFO [2019-12-17 12:31:59,072] jepsen worker 9 - jepsen.util 5949 :invoke :write [3 44] INFO [2019-12-17 12:31:59,087] jepsen worker 46 - jepsen.util 5356 :fail :cas [15 [30 22]] :ValueNotSwapFrom INFO [2019-12-17 12:31:59,087] jepsen worker 46 - jepsen.util 5356 :invoke :read [15 nil] INFO [2019-12-17 12:31:59,093] jepsen worker 64 - jepsen.util 4024 :info :cas [21 [33 30]] :RequestCanceledException INFO [2019-12-17 12:31:59,096] jepsen worker 64 - jepsen.util 4114 :invoke :write [21 8] INFO [2019-12-17 12:31:59,105] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:31:59,105] jepsen nemesis - couchbase.util Rebalance check count: 234 INFO [2019-12-17 12:31:59,105] jepsen nemesis - couchbase.util Rebalance stuck count: 215 INFO [2019-12-17 12:31:59,148] jepsen worker 46 - jepsen.util 5356 :ok :read [15 0] INFO [2019-12-17 12:31:59,148] jepsen worker 46 - jepsen.util 5356 :invoke :write [15 41] INFO [2019-12-17 12:31:59,189] jepsen worker 51 - jepsen.util 5361 :ok :write [17 31] INFO [2019-12-17 12:31:59,189] jepsen worker 51 - jepsen.util 5361 :invoke :write [17 5] INFO [2019-12-17 12:31:59,192] jepsen worker 55 - jepsen.util 6355 :info :write [18 0] :RequestCanceledException INFO [2019-12-17 12:31:59,192] jepsen worker 88 - jepsen.util 5578 :fail :read [29 nil] :RequestTimeoutException INFO [2019-12-17 12:31:59,192] jepsen worker 29 - jepsen.util 4259 :fail :read [9 nil] :RequestTimeoutException INFO [2019-12-17 12:31:59,192] jepsen worker 29 - jepsen.util 4259 :invoke :cas [9 [28 32]] INFO [2019-12-17 12:31:59,192] jepsen worker 88 - jepsen.util 5578 :invoke :cas [29 [42 11]] INFO [2019-12-17 12:31:59,193] jepsen worker 55 - jepsen.util 6445 :invoke :write [18 43] INFO [2019-12-17 12:31:59,198] jepsen worker 3 - jepsen.util 5403 :ok :cas [1 [9 13]] INFO [2019-12-17 12:31:59,198] jepsen worker 3 - jepsen.util 5403 :invoke :write [1 23] INFO [2019-12-17 12:31:59,211] jepsen worker 52 - jepsen.util 5722 :info :write [17 38] :RequestCanceledException INFO [2019-12-17 12:31:59,213] jepsen worker 52 - jepsen.util 5812 :invoke :cas [17 [29 45]] INFO [2019-12-17 12:31:59,233] jepsen worker 29 - jepsen.util 4259 :fail :cas [9 [28 32]] :ValueNotSwapFrom INFO [2019-12-17 12:31:59,233] jepsen worker 29 - jepsen.util 4259 :invoke :write [9 23] INFO [2019-12-17 12:31:59,237] jepsen worker 73 - jepsen.util 5023 :ok :write [24 36] INFO [2019-12-17 12:31:59,237] jepsen worker 31 - jepsen.util 4981 :ok :write [10 25] INFO [2019-12-17 12:31:59,237] jepsen worker 73 - jepsen.util 5023 :invoke :cas [24 [46 32]] INFO [2019-12-17 12:31:59,237] jepsen worker 31 - jepsen.util 4981 :invoke :write [10 18] INFO [2019-12-17 12:31:59,237] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31204,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:31:59,237] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31215,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:31:59,237] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31204,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:31:59,238] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31215,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:31:59,238] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31205,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:31:59,238] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31217,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:31:59,238] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31205,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:31:59,238] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31217,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:31:59,254] jepsen worker 89 - jepsen.util 5219 :ok :read [29 36] INFO [2019-12-17 12:31:59,254] jepsen worker 89 - jepsen.util 5219 :invoke :write [29 5] INFO [2019-12-17 12:31:59,283] jepsen worker 63 - jepsen.util 4383 :ok :read [21 28] INFO [2019-12-17 12:31:59,285] jepsen worker 63 - jepsen.util 4383 :invoke :cas [21 [23 27]] INFO [2019-12-17 12:31:59,293] jepsen worker 56 - jepsen.util 5636 :info :write [18 21] :RequestCanceledException INFO [2019-12-17 12:31:59,294] jepsen worker 56 - jepsen.util 5726 :invoke :write [18 32] INFO [2019-12-17 12:31:59,325] jepsen worker 68 - jepsen.util 6818 :info :write [22 18] :RequestCanceledException INFO [2019-12-17 12:31:59,326] jepsen worker 68 - jepsen.util 6908 :invoke :read [22 nil] INFO [2019-12-17 12:31:59,340] jepsen worker 8 - jepsen.util 6758 :info :cas [2 [48 14]] :RequestCanceledException INFO [2019-12-17 12:31:59,340] jepsen worker 6 - jepsen.util 7566 :info :cas [2 [16 39]] :RequestCanceledException INFO [2019-12-17 12:31:59,342] jepsen worker 6 - jepsen.util 7656 :invoke :cas [2 [11 31]] INFO [2019-12-17 12:31:59,342] jepsen worker 8 - jepsen.util 6848 :invoke :read [2 nil] INFO [2019-12-17 12:31:59,351] jepsen worker 64 - jepsen.util 4114 :ok :write [21 8] INFO [2019-12-17 12:31:59,351] jepsen worker 64 - jepsen.util 4114 :invoke :write [21 12] INFO [2019-12-17 12:31:59,357] jepsen worker 48 - jepsen.util 6618 :info :cas [16 [46 25]] :RequestCanceledException INFO [2019-12-17 12:31:59,360] jepsen worker 48 - jepsen.util 6708 :invoke :write [16 42] INFO [2019-12-17 12:31:59,364] jepsen worker 29 - jepsen.util 4259 :ok :write [9 23] INFO [2019-12-17 12:31:59,365] jepsen worker 29 - jepsen.util 4259 :invoke :write [9 2] INFO [2019-12-17 12:31:59,392] jepsen worker 12 - jepsen.util 5682 :info :write [4 27] :RequestCanceledException INFO [2019-12-17 12:31:59,397] jepsen worker 12 - jepsen.util 5772 :invoke :write [4 2] INFO [2019-12-17 12:31:59,405] jepsen worker 19 - jepsen.util 5329 :info :write [6 22] :RequestCanceledException INFO [2019-12-17 12:31:59,406] jepsen worker 19 - jepsen.util 5419 :invoke :cas [6 [31 46]] INFO [2019-12-17 12:31:59,423] jepsen worker 60 - jepsen.util 7890 :info :cas [20 [15 41]] :RequestCanceledException INFO [2019-12-17 12:31:59,423] jepsen worker 15 - jepsen.util 6225 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:31:59,423] jepsen worker 17 - jepsen.util 7307 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:31:59,423] jepsen worker 76 - jepsen.util 7366 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:31:59,423] jepsen worker 85 - jepsen.util 8185 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:31:59,423] jepsen worker 33 - jepsen.util 7233 :info :write [11 27] :RequestCanceledException INFO [2019-12-17 12:31:59,423] jepsen worker 15 - jepsen.util 6225 :invoke :cas [5 [19 49]] INFO [2019-12-17 12:31:59,424] jepsen worker 17 - jepsen.util 7307 :invoke :write [5 19] INFO [2019-12-17 12:31:59,424] jepsen worker 76 - jepsen.util 7366 :invoke :read [25 nil] INFO [2019-12-17 12:31:59,424] jepsen worker 85 - jepsen.util 8185 :invoke :cas [28 [16 34]] INFO [2019-12-17 12:31:59,425] jepsen worker 60 - jepsen.util 7980 :invoke :write [20 43] INFO [2019-12-17 12:31:59,425] jepsen worker 33 - jepsen.util 7323 :invoke :read [11 nil] INFO [2019-12-17 12:31:59,440] jepsen worker 70 - jepsen.util 6010 :info :write [23 48] :RequestCanceledException INFO [2019-12-17 12:31:59,440] jepsen worker 84 - jepsen.util 6834 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:31:59,440] jepsen worker 77 - jepsen.util 7367 :info :cas [25 [4 9]] :RequestCanceledException INFO [2019-12-17 12:31:59,440] jepsen worker 84 - jepsen.util 6834 :invoke :read [28 nil] INFO [2019-12-17 12:31:59,441] jepsen worker 77 - jepsen.util 7457 :invoke :read [25 nil] INFO [2019-12-17 12:31:59,442] jepsen worker 70 - jepsen.util 6100 :invoke :cas [23 [19 44]] INFO [2019-12-17 12:31:59,452] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31234,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:31:59,452] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31234,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:31:59,452] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31237,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:31:59,452] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31237,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:31:59,453] jepsen worker 16 - jepsen.util 8386 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:31:59,453] jepsen worker 39 - jepsen.util 6879 :info :write [13 40] :RequestCanceledException INFO [2019-12-17 12:31:59,453] jepsen worker 16 - jepsen.util 8386 :invoke :cas [5 [16 18]] INFO [2019-12-17 12:31:59,454] jepsen worker 39 - jepsen.util 6969 :invoke :write [13 22] INFO [2019-12-17 12:31:59,462] jepsen worker 29 - jepsen.util 4259 :ok :write [9 2] INFO [2019-12-17 12:31:59,462] jepsen worker 29 - jepsen.util 4259 :invoke :read [9 nil] INFO [2019-12-17 12:31:59,467] jepsen worker 52 - jepsen.util 5812 :fail :cas [17 [29 45]] :ValueNotSwapFrom INFO [2019-12-17 12:31:59,467] jepsen worker 52 - jepsen.util 5812 :invoke :read [17 nil] INFO [2019-12-17 12:31:59,479] jepsen worker 64 - jepsen.util 4114 :ok :write [21 12] INFO [2019-12-17 12:31:59,479] jepsen worker 64 - jepsen.util 4114 :invoke :read [21 nil] INFO [2019-12-17 12:31:59,486] jepsen worker 19 - jepsen.util 5419 :fail :cas [6 [31 46]] :ValueNotSwapFrom INFO [2019-12-17 12:31:59,486] jepsen worker 19 - jepsen.util 5419 :invoke :read [6 nil] INFO [2019-12-17 12:31:59,493] jepsen worker 4 - jepsen.util 4594 :info :cas [1 [36 0]] :RequestCanceledException INFO [2019-12-17 12:31:59,493] jepsen worker 82 - jepsen.util 7912 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:31:59,493] jepsen worker 82 - jepsen.util 7912 :invoke :read [27 nil] INFO [2019-12-17 12:31:59,494] jepsen worker 4 - jepsen.util 4684 :invoke :write [1 40] INFO [2019-12-17 12:31:59,509] jepsen worker 29 - jepsen.util 4259 :ok :read [9 2] INFO [2019-12-17 12:31:59,509] jepsen worker 29 - jepsen.util 4259 :invoke :read [9 nil] INFO [2019-12-17 12:31:59,509] jepsen worker 66 - jepsen.util 6726 :info :write [22 10] :RequestCanceledException INFO [2019-12-17 12:31:59,511] jepsen worker 66 - jepsen.util 6816 :invoke :cas [22 [2 17]] INFO [2019-12-17 12:31:59,511] jepsen worker 19 - jepsen.util 5419 :ok :read [6 7] INFO [2019-12-17 12:31:59,511] jepsen worker 19 - jepsen.util 5419 :invoke :read [6 nil] INFO [2019-12-17 12:31:59,513] jepsen worker 64 - jepsen.util 4114 :ok :read [21 12] INFO [2019-12-17 12:31:59,513] jepsen worker 64 - jepsen.util 4114 :invoke :write [21 5] INFO [2019-12-17 12:31:59,524] jepsen worker 29 - jepsen.util 4259 :ok :read [9 2] INFO [2019-12-17 12:31:59,524] jepsen worker 29 - jepsen.util 4259 :invoke :write [9 35] INFO [2019-12-17 12:31:59,526] jepsen worker 57 - jepsen.util 6807 :info :cas [19 [23 20]] :RequestCanceledException INFO [2019-12-17 12:31:59,526] jepsen worker 22 - jepsen.util 7402 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:31:59,526] jepsen worker 22 - jepsen.util 7402 :invoke :read [7 nil] INFO [2019-12-17 12:31:59,536] jepsen worker 57 - jepsen.util 6897 :invoke :read [19 nil] INFO [2019-12-17 12:31:59,540] jepsen worker 41 - jepsen.util 7871 :info :write [13 34] :RequestCanceledException INFO [2019-12-17 12:31:59,540] jepsen worker 18 - jepsen.util 5328 :info :write [6 8] :RequestCanceledException INFO [2019-12-17 12:31:59,546] jepsen worker 31 - jepsen.util 4981 :ok :write [10 18] INFO [2019-12-17 12:31:59,546] jepsen worker 31 - jepsen.util 4981 :invoke :read [10 nil] INFO [2019-12-17 12:31:59,547] jepsen worker 47 - jepsen.util 6077 :ok :write [15 45] INFO [2019-12-17 12:31:59,547] jepsen worker 41 - jepsen.util 7961 :invoke :cas [13 [10 41]] INFO [2019-12-17 12:31:59,547] jepsen worker 47 - jepsen.util 6077 :invoke :read [15 nil] INFO [2019-12-17 12:31:59,547] jepsen worker 9 - jepsen.util 5949 :ok :write [3 44] INFO [2019-12-17 12:31:59,547] jepsen worker 9 - jepsen.util 5949 :invoke :read [3 nil] INFO [2019-12-17 12:31:59,556] jepsen worker 18 - jepsen.util 5418 :invoke :read [6 nil] INFO [2019-12-17 12:31:59,557] jepsen worker 19 - jepsen.util 5419 :ok :read [6 7] INFO [2019-12-17 12:31:59,557] jepsen worker 19 - jepsen.util 5419 :invoke :cas [6 [5 2]] INFO [2019-12-17 12:31:59,557] jepsen worker 75 - jepsen.util 7005 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:31:59,557] jepsen worker 43 - jepsen.util 6793 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:31:59,557] jepsen worker 75 - jepsen.util 7005 :invoke :write [25 4] INFO [2019-12-17 12:31:59,557] jepsen worker 43 - jepsen.util 6793 :invoke :write [14 28] INFO [2019-12-17 12:31:59,592] jepsen worker 53 - jepsen.util 5273 :info :cas [17 [38 28]] :RequestCanceledException INFO [2019-12-17 12:31:59,593] jepsen worker 53 - jepsen.util 5363 :invoke :cas [17 [25 1]] INFO [2019-12-17 12:31:59,608] jepsen worker 40 - jepsen.util 7690 :info :cas [13 [34 11]] :RequestCanceledException INFO [2019-12-17 12:31:59,620] jepsen worker 40 - jepsen.util 7780 :invoke :cas [13 [20 38]] INFO [2019-12-17 12:31:59,620] jepsen worker 9 - jepsen.util 5949 :ok :read [3 44] INFO [2019-12-17 12:31:59,621] jepsen worker 9 - jepsen.util 5949 :invoke :cas [3 [27 14]] INFO [2019-12-17 12:31:59,622] jepsen worker 53 - jepsen.util 5363 :fail :cas [17 [25 1]] :ValueNotSwapFrom INFO [2019-12-17 12:31:59,622] jepsen worker 53 - jepsen.util 5363 :invoke :read [17 nil] INFO [2019-12-17 12:31:59,625] jepsen worker 83 - jepsen.util 7553 :info :write [27 11] :RequestCanceledException INFO [2019-12-17 12:31:59,627] jepsen worker 83 - jepsen.util 7643 :invoke :write [27 16] INFO [2019-12-17 12:31:59,639] jepsen worker 54 - jepsen.util 5814 :info :write [18 48] :RequestCanceledException INFO [2019-12-17 12:31:59,640] jepsen worker 54 - jepsen.util 5904 :invoke :write [18 8] INFO [2019-12-17 12:31:59,645] jepsen worker 47 - jepsen.util 6077 :ok :read [15 45] INFO [2019-12-17 12:31:59,646] jepsen worker 47 - jepsen.util 6077 :invoke :write [15 46] INFO [2019-12-17 12:31:59,653] jepsen worker 32 - jepsen.util 5432 :info :write [10 21] :RequestCanceledException INFO [2019-12-17 12:31:59,655] jepsen worker 32 - jepsen.util 5522 :invoke :read [10 nil] INFO [2019-12-17 12:31:59,660] jepsen worker 18 - jepsen.util 5418 :ok :read [6 7] INFO [2019-12-17 12:31:59,660] jepsen worker 18 - jepsen.util 5418 :invoke :read [6 nil] INFO [2019-12-17 12:31:59,664] jepsen worker 31 - jepsen.util 4981 :ok :read [10 18] INFO [2019-12-17 12:31:59,664] jepsen worker 31 - jepsen.util 4981 :invoke :read [10 nil] INFO [2019-12-17 12:31:59,670] jepsen worker 9 - jepsen.util 5949 :fail :cas [3 [27 14]] :ValueNotSwapFrom INFO [2019-12-17 12:31:59,670] jepsen worker 9 - jepsen.util 5949 :invoke :write [3 48] INFO [2019-12-17 12:31:59,684] jepsen worker 53 - jepsen.util 5363 :ok :read [17 31] INFO [2019-12-17 12:31:59,684] jepsen worker 53 - jepsen.util 5363 :invoke :cas [17 [48 23]] INFO [2019-12-17 12:31:59,687] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31277,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:31:59,687] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31277,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:31:59,687] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31280,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:31:59,687] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31280,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:31:59,688] jepsen worker 18 - jepsen.util 5418 :ok :read [6 7] INFO [2019-12-17 12:31:59,688] jepsen worker 18 - jepsen.util 5418 :invoke :read [6 nil] INFO [2019-12-17 12:31:59,691] jepsen worker 27 - jepsen.util 5157 :info :cas [9 [49 40]] :RequestCanceledException INFO [2019-12-17 12:31:59,692] jepsen worker 27 - jepsen.util 5247 :invoke :write [9 40] INFO [2019-12-17 12:31:59,706] jepsen worker 31 - jepsen.util 4981 :ok :read [10 18] INFO [2019-12-17 12:31:59,707] jepsen worker 31 - jepsen.util 4981 :invoke :write [10 29] INFO [2019-12-17 12:31:59,709] jepsen worker 44 - jepsen.util 7874 :info :write [14 20] :RequestCanceledException INFO [2019-12-17 12:31:59,710] jepsen worker 44 - jepsen.util 7964 :invoke :read [14 nil] INFO [2019-12-17 12:31:59,719] jepsen worker 64 - jepsen.util 4114 :ok :write [21 5] INFO [2019-12-17 12:31:59,719] jepsen worker 64 - jepsen.util 4114 :invoke :cas [21 [15 26]] INFO [2019-12-17 12:31:59,726] jepsen worker 25 - jepsen.util 7585 :info :write [8 5] :RequestCanceledException INFO [2019-12-17 12:31:59,726] jepsen worker 2 - jepsen.util 6842 :info :cas [0 [10 42]] :RequestCanceledException INFO [2019-12-17 12:31:59,728] jepsen worker 25 - jepsen.util 7675 :invoke :cas [8 [17 45]] INFO [2019-12-17 12:31:59,728] jepsen worker 2 - jepsen.util 6932 :invoke :read [0 nil] INFO [2019-12-17 12:31:59,740] jepsen worker 58 - jepsen.util 7708 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:31:59,741] jepsen worker 58 - jepsen.util 7708 :invoke :write [19 31] INFO [2019-12-17 12:31:59,765] jepsen worker 32 - jepsen.util 5522 :ok :read [10 18] INFO [2019-12-17 12:31:59,765] jepsen worker 32 - jepsen.util 5522 :invoke :write [10 7] INFO [2019-12-17 12:31:59,792] jepsen worker 36 - jepsen.util 5706 :info :write [12 3] :RequestCanceledException INFO [2019-12-17 12:31:59,799] jepsen worker 36 - jepsen.util 5796 :invoke :write [12 33] INFO [2019-12-17 12:31:59,808] jepsen worker 64 - jepsen.util 4114 :fail :cas [21 [15 26]] :ValueNotSwapFrom INFO [2019-12-17 12:31:59,810] jepsen worker 64 - jepsen.util 4114 :invoke :write [21 11] INFO [2019-12-17 12:31:59,824] jepsen worker 61 - jepsen.util 8161 :info :write [20 7] :RequestCanceledException INFO [2019-12-17 12:31:59,825] jepsen worker 61 - jepsen.util 8251 :invoke :write [20 32] INFO [2019-12-17 12:31:59,840] jepsen worker 34 - jepsen.util 6604 :info :write [11 16] :RequestCanceledException INFO [2019-12-17 12:31:59,840] jepsen worker 74 - jepsen.util 5924 :info :cas [24 [19 14]] :RequestCanceledException INFO [2019-12-17 12:31:59,841] jepsen worker 34 - jepsen.util 6694 :invoke :write [11 24] INFO [2019-12-17 12:31:59,841] jepsen worker 74 - jepsen.util 6014 :invoke :cas [24 [47 5]] INFO [2019-12-17 12:31:59,852] jepsen worker 29 - jepsen.util 4259 :ok :write [9 35] INFO [2019-12-17 12:31:59,852] jepsen worker 29 - jepsen.util 4259 :invoke :read [9 nil] INFO [2019-12-17 12:31:59,853] jepsen worker 52 - jepsen.util 5812 :ok :read [17 31] INFO [2019-12-17 12:31:59,853] jepsen worker 52 - jepsen.util 5812 :invoke :cas [17 [2 19]] INFO [2019-12-17 12:31:59,854] jepsen worker 38 - jepsen.util 5438 :info :write [12 0] :RequestCanceledException INFO [2019-12-17 12:31:59,854] jepsen worker 81 - jepsen.util 7191 :info :write [27 46] :RequestCanceledException INFO [2019-12-17 12:31:59,858] jepsen worker 81 - jepsen.util 7281 :invoke :cas [27 [3 23]] INFO [2019-12-17 12:31:59,858] jepsen worker 38 - jepsen.util 5528 :invoke :write [12 42] INFO [2019-12-17 12:31:59,888] jepsen worker 86 - jepsen.util 7556 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:31:59,888] jepsen worker 86 - jepsen.util 7556 :invoke :write [28 36] INFO [2019-12-17 12:31:59,893] jepsen worker 29 - jepsen.util 4259 :ok :read [9 35] INFO [2019-12-17 12:31:59,893] jepsen worker 29 - jepsen.util 4259 :invoke :cas [9 [28 14]] INFO [2019-12-17 12:31:59,908] jepsen worker 0 - jepsen.util 7830 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:31:59,908] jepsen worker 79 - jepsen.util 5209 :info :write [26 28] :RequestCanceledException INFO [2019-12-17 12:31:59,908] jepsen worker 0 - jepsen.util 7830 :invoke :read [0 nil] INFO [2019-12-17 12:31:59,909] jepsen worker 79 - jepsen.util 5299 :invoke :write [26 8] INFO [2019-12-17 12:31:59,921] jepsen worker 29 - jepsen.util 4259 :fail :cas [9 [28 14]] :ValueNotSwapFrom INFO [2019-12-17 12:31:59,921] jepsen worker 29 - jepsen.util 4259 :invoke :cas [9 [4 19]] INFO [2019-12-17 12:31:59,939] jepsen worker 62 - jepsen.util 6992 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:31:59,940] jepsen worker 62 - jepsen.util 6992 :invoke :read [20 nil] INFO [2019-12-17 12:31:59,941] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31306,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:31:59,941] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31306,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:31:59,941] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31310,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:31:59,941] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31310,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:31:59,941] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31311,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:31:59,941] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31311,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:31:59,944] jepsen worker 29 - jepsen.util 4259 :fail :cas [9 [4 19]] :ValueNotSwapFrom INFO [2019-12-17 12:31:59,944] jepsen worker 29 - jepsen.util 4259 :invoke :read [9 nil] INFO [2019-12-17 12:31:59,954] jepsen worker 67 - jepsen.util 7357 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:31:59,954] jepsen worker 67 - jepsen.util 7357 :invoke :cas [22 [19 37]] INFO [2019-12-17 12:31:59,966] jepsen worker 29 - jepsen.util 4259 :ok :read [9 35] INFO [2019-12-17 12:31:59,966] jepsen worker 29 - jepsen.util 4259 :invoke :cas [9 [33 47]] INFO [2019-12-17 12:31:59,974] jepsen worker 47 - jepsen.util 6077 :ok :write [15 46] INFO [2019-12-17 12:31:59,974] jepsen worker 47 - jepsen.util 6077 :invoke :write [15 40] INFO [2019-12-17 12:31:59,981] jepsen worker 29 - jepsen.util 4259 :fail :cas [9 [33 47]] :ValueNotSwapFrom INFO [2019-12-17 12:31:59,982] jepsen worker 29 - jepsen.util 4259 :invoke :cas [9 [33 8]] INFO [2019-12-17 12:31:59,998] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31325,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:31:59,998] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31325,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:31:59,998] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31328,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:31:59,998] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31328,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:32:00,024] jepsen worker 31 - jepsen.util 4981 :ok :write [10 29] INFO [2019-12-17 12:32:00,025] jepsen worker 31 - jepsen.util 4981 :invoke :read [10 nil] INFO [2019-12-17 12:32:00,026] jepsen worker 80 - jepsen.util 5480 :info :write [26 6] :RequestCanceledException INFO [2019-12-17 12:32:00,026] jepsen worker 26 - jepsen.util 7676 :info :write [8 35] :RequestCanceledException INFO [2019-12-17 12:32:00,027] jepsen worker 26 - jepsen.util 7766 :invoke :read [8 nil] INFO [2019-12-17 12:32:00,028] jepsen worker 80 - jepsen.util 5570 :invoke :write [26 40] INFO [2019-12-17 12:32:00,053] jepsen worker 29 - jepsen.util 4259 :fail :cas [9 [33 8]] :ValueNotSwapFrom INFO [2019-12-17 12:32:00,053] jepsen worker 29 - jepsen.util 4259 :invoke :write [9 25] INFO [2019-12-17 12:32:00,063] jepsen worker 31 - jepsen.util 4981 :ok :read [10 29] INFO [2019-12-17 12:32:00,063] jepsen worker 31 - jepsen.util 4981 :invoke :cas [10 [46 27]] INFO [2019-12-17 12:32:00,092] jepsen worker 31 - jepsen.util 4981 :fail :cas [10 [46 27]] :ValueNotSwapFrom INFO [2019-12-17 12:32:00,092] jepsen worker 31 - jepsen.util 4981 :invoke :write [10 39] INFO [2019-12-17 12:32:00,093] jepsen worker 69 - jepsen.util 5919 :fail :read [23 nil] :RequestTimeoutException INFO [2019-12-17 12:32:00,093] jepsen worker 69 - jepsen.util 5919 :invoke :read [23 nil] INFO [2019-12-17 12:32:00,108] jepsen worker 21 - jepsen.util 6861 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:32:00,108] jepsen worker 21 - jepsen.util 6861 :invoke :write [7 13] INFO [2019-12-17 12:32:00,119] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:32:00,119] jepsen nemesis - couchbase.util Rebalance check count: 235 INFO [2019-12-17 12:32:00,119] jepsen nemesis - couchbase.util Rebalance stuck count: 216 INFO [2019-12-17 12:32:00,126] jepsen worker 72 - jepsen.util 5202 :fail :read [24 nil] :RequestTimeoutException INFO [2019-12-17 12:32:00,126] jepsen worker 1 - jepsen.util 7111 :info :cas [0 [41 34]] :RequestCanceledException INFO [2019-12-17 12:32:00,126] jepsen worker 72 - jepsen.util 5202 :invoke :cas [24 [37 15]] INFO [2019-12-17 12:32:00,127] jepsen worker 1 - jepsen.util 7201 :invoke :read [0 nil] INFO [2019-12-17 12:32:00,140] jepsen worker 7 - jepsen.util 7477 :info :cas [2 [12 49]] :RequestCanceledException INFO [2019-12-17 12:32:00,142] jepsen worker 7 - jepsen.util 7567 :invoke :read [2 nil] INFO [2019-12-17 12:32:00,143] jepsen worker 29 - jepsen.util 4259 :ok :write [9 25] INFO [2019-12-17 12:32:00,143] jepsen worker 29 - jepsen.util 4259 :invoke :write [9 25] INFO [2019-12-17 12:32:00,201] jepsen worker 29 - jepsen.util 4259 :ok :write [9 25] INFO [2019-12-17 12:32:00,201] jepsen worker 29 - jepsen.util 4259 :invoke :cas [9 [15 46]] INFO [2019-12-17 12:32:00,211] jepsen worker 70 - jepsen.util 6100 :fail :cas [23 [19 44]] :ValueNotSwapFrom INFO [2019-12-17 12:32:00,211] jepsen worker 70 - jepsen.util 6100 :invoke :read [23 nil] INFO [2019-12-17 12:32:00,211] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31342,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:32:00,212] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31349,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:32:00,212] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31342,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:32:00,212] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31349,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:32:00,212] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31344,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:32:00,212] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31351,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:32:00,212] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31344,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:32:00,212] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31351,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:32:00,212] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31352,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:32:00,212] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31352,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:32:00,226] jepsen worker 14 - jepsen.util 5414 :info :write [4 47] :RequestCanceledException INFO [2019-12-17 12:32:00,227] jepsen worker 14 - jepsen.util 5504 :invoke :write [4 46] INFO [2019-12-17 12:32:00,232] jepsen worker 29 - jepsen.util 4259 :fail :cas [9 [15 46]] :ValueNotSwapFrom INFO [2019-12-17 12:32:00,232] jepsen worker 29 - jepsen.util 4259 :invoke :write [9 25] INFO [2019-12-17 12:32:00,246] jepsen worker 53 - jepsen.util 5363 :fail :cas [17 [48 23]] :ValueNotSwapFrom INFO [2019-12-17 12:32:00,246] jepsen worker 53 - jepsen.util 5363 :invoke :read [17 nil] INFO [2019-12-17 12:32:00,248] jepsen worker 52 - jepsen.util 5812 :fail :cas [17 [2 19]] :ValueNotSwapFrom INFO [2019-12-17 12:32:00,249] jepsen worker 52 - jepsen.util 5812 :invoke :cas [17 [33 31]] INFO [2019-12-17 12:32:00,280] jepsen worker 29 - jepsen.util 4259 :ok :write [9 25] INFO [2019-12-17 12:32:00,280] jepsen worker 29 - jepsen.util 4259 :invoke :cas [9 [24 8]] INFO [2019-12-17 12:32:00,300] jepsen worker 29 - jepsen.util 4259 :fail :cas [9 [24 8]] :ValueNotSwapFrom INFO [2019-12-17 12:32:00,300] jepsen worker 29 - jepsen.util 4259 :invoke :write [9 17] INFO [2019-12-17 12:32:00,327] jepsen worker 45 - jepsen.util 5535 :fail :read [15 nil] :RequestTimeoutException INFO [2019-12-17 12:32:00,327] jepsen worker 45 - jepsen.util 5535 :invoke :read [15 nil] INFO [2019-12-17 12:32:00,357] jepsen worker 29 - jepsen.util 4259 :ok :write [9 17] INFO [2019-12-17 12:32:00,357] jepsen worker 29 - jepsen.util 4259 :invoke :cas [9 [17 40]] INFO [2019-12-17 12:32:00,420] jepsen worker 29 - jepsen.util 4259 :ok :cas [9 [17 40]] INFO [2019-12-17 12:32:00,421] jepsen worker 29 - jepsen.util 4259 :invoke :cas [9 [26 41]] INFO [2019-12-17 12:32:00,426] jepsen worker 35 - jepsen.util 7415 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:32:00,426] jepsen worker 10 - jepsen.util 5320 :info :write [3 27] :RequestCanceledException INFO [2019-12-17 12:32:00,426] jepsen worker 49 - jepsen.util 6529 :info :cas [16 [20 15]] :RequestCanceledException INFO [2019-12-17 12:32:00,427] jepsen worker 35 - jepsen.util 7415 :invoke :read [11 nil] INFO [2019-12-17 12:32:00,428] jepsen worker 49 - jepsen.util 6619 :invoke :read [16 nil] INFO [2019-12-17 12:32:00,428] jepsen worker 10 - jepsen.util 5410 :invoke :write [3 13] INFO [2019-12-17 12:32:00,430] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31367,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:32:00,430] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31367,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:32:00,430] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31369,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:32:00,430] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31369,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:32:00,433] jepsen worker 29 - jepsen.util 4259 :fail :cas [9 [26 41]] :ValueNotSwapFrom INFO [2019-12-17 12:32:00,434] jepsen worker 29 - jepsen.util 4259 :invoke :cas [9 [0 45]] INFO [2019-12-17 12:32:00,441] jepsen worker 42 - jepsen.util 7782 :info :cas [14 [41 33]] :RequestCanceledException INFO [2019-12-17 12:32:00,442] jepsen worker 42 - jepsen.util 7872 :invoke :write [14 15] INFO [2019-12-17 12:32:00,453] jepsen worker 24 - jepsen.util 6684 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:32:00,453] jepsen worker 9 - jepsen.util 5949 :ok :write [3 48] INFO [2019-12-17 12:32:00,453] jepsen worker 24 - jepsen.util 6684 :invoke :write [8 37] INFO [2019-12-17 12:32:00,453] jepsen worker 9 - jepsen.util 5949 :invoke :write [3 3] INFO [2019-12-17 12:32:00,460] jepsen worker 29 - jepsen.util 4259 :fail :cas [9 [0 45]] :ValueNotSwapFrom INFO [2019-12-17 12:32:00,460] jepsen worker 29 - jepsen.util 4259 :invoke :write [9 40] INFO [2019-12-17 12:32:00,470] jepsen worker 31 - jepsen.util 4981 :ok :write [10 39] INFO [2019-12-17 12:32:00,470] jepsen worker 31 - jepsen.util 4981 :invoke :cas [10 [14 30]] INFO [2019-12-17 12:32:00,471] jepsen worker 47 - jepsen.util 6077 :ok :write [15 40] INFO [2019-12-17 12:32:00,471] jepsen worker 47 - jepsen.util 6077 :invoke :write [15 3] INFO [2019-12-17 12:32:00,476] jepsen worker 74 - jepsen.util 6014 :fail :cas [24 [47 5]] :ValueNotSwapFrom INFO [2019-12-17 12:32:00,476] jepsen worker 74 - jepsen.util 6014 :invoke :cas [24 [37 14]] INFO [2019-12-17 12:32:00,493] jepsen worker 50 - jepsen.util 7430 :info :write [16 49] :RequestCanceledException INFO [2019-12-17 12:32:00,493] jepsen worker 30 - jepsen.util 5430 :info :cas [10 [12 14]] :RequestCanceledException INFO [2019-12-17 12:32:00,494] jepsen worker 50 - jepsen.util 7520 :invoke :cas [16 [21 9]] INFO [2019-12-17 12:32:00,494] jepsen worker 30 - jepsen.util 5520 :invoke :cas [10 [49 47]] INFO [2019-12-17 12:32:00,510] jepsen worker 23 - jepsen.util 7493 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:32:00,510] jepsen worker 87 - jepsen.util 4857 :info :write [29 38] :RequestCanceledException INFO [2019-12-17 12:32:00,510] jepsen worker 23 - jepsen.util 7493 :invoke :cas [7 [5 40]] INFO [2019-12-17 12:32:00,511] jepsen worker 87 - jepsen.util 4947 :invoke :cas [29 [23 5]] INFO [2019-12-17 12:32:00,525] jepsen worker 59 - jepsen.util 7259 :info :write [19 1] :RequestCanceledException INFO [2019-12-17 12:32:00,525] jepsen worker 20 - jepsen.util 5510 :info :cas [6 [42 31]] :RequestCanceledException INFO [2019-12-17 12:32:00,526] jepsen worker 59 - jepsen.util 7349 :invoke :write [19 37] INFO [2019-12-17 12:32:00,527] jepsen worker 20 - jepsen.util 5600 :invoke :read [6 nil] INFO [2019-12-17 12:32:00,554] jepsen worker 31 - jepsen.util 4981 :fail :cas [10 [14 30]] :ValueNotSwapFrom INFO [2019-12-17 12:32:00,555] jepsen worker 31 - jepsen.util 4981 :invoke :cas [10 [15 40]] INFO [2019-12-17 12:32:00,558] jepsen worker 72 - jepsen.util 5202 :fail :cas [24 [37 15]] :ValueNotSwapFrom INFO [2019-12-17 12:32:00,558] jepsen worker 72 - jepsen.util 5202 :invoke :read [24 nil] INFO [2019-12-17 12:32:00,575] jepsen worker 31 - jepsen.util 4981 :fail :cas [10 [15 40]] :ValueNotSwapFrom INFO [2019-12-17 12:32:00,575] jepsen worker 31 - jepsen.util 4981 :invoke :cas [10 [1 23]] INFO [2019-12-17 12:32:00,577] jepsen worker 45 - jepsen.util 5535 :ok :read [15 40] INFO [2019-12-17 12:32:00,578] jepsen worker 45 - jepsen.util 5535 :invoke :cas [15 [3 35]] INFO [2019-12-17 12:32:00,578] jepsen worker 53 - jepsen.util 5363 :ok :read [17 31] INFO [2019-12-17 12:32:00,578] jepsen worker 53 - jepsen.util 5363 :invoke :read [17 nil] INFO [2019-12-17 12:32:00,598] jepsen worker 52 - jepsen.util 5812 :fail :cas [17 [33 31]] :ValueNotSwapFrom INFO [2019-12-17 12:32:00,598] jepsen worker 52 - jepsen.util 5812 :invoke :read [17 nil] INFO [2019-12-17 12:32:00,609] jepsen worker 53 - jepsen.util 5363 :ok :read [17 31] INFO [2019-12-17 12:32:00,609] jepsen worker 53 - jepsen.util 5363 :invoke :cas [17 [48 42]] INFO [2019-12-17 12:32:00,611] jepsen worker 31 - jepsen.util 4981 :fail :cas [10 [1 23]] :ValueNotSwapFrom INFO [2019-12-17 12:32:00,611] jepsen worker 31 - jepsen.util 4981 :invoke :write [10 10] INFO [2019-12-17 12:32:00,626] jepsen worker 65 - jepsen.util 4655 :info :write [21 10] :RequestCanceledException INFO [2019-12-17 12:32:00,627] jepsen worker 65 - jepsen.util 4745 :invoke :write [21 21] INFO [2019-12-17 12:32:00,640] jepsen worker 13 - jepsen.util 5323 :info :write [4 21] :RequestCanceledException INFO [2019-12-17 12:32:00,641] jepsen worker 13 - jepsen.util 5413 :invoke :write [4 31] INFO [2019-12-17 12:32:00,642] jepsen worker 52 - jepsen.util 5812 :ok :read [17 31] INFO [2019-12-17 12:32:00,642] jepsen worker 52 - jepsen.util 5812 :invoke :write [17 21] INFO [2019-12-17 12:32:00,653] jepsen worker 29 - jepsen.util 4259 :ok :write [9 40] INFO [2019-12-17 12:32:00,653] jepsen worker 29 - jepsen.util 4259 :invoke :write [9 8] INFO [2019-12-17 12:32:00,662] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31398,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:32:00,663] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31398,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:32:00,663] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31403,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:32:00,663] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31403,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:32:00,709] jepsen worker 53 - jepsen.util 5363 :fail :cas [17 [48 42]] :ValueNotSwapFrom INFO [2019-12-17 12:32:00,709] jepsen worker 53 - jepsen.util 5363 :invoke :write [17 10] INFO [2019-12-17 12:32:00,716] jepsen worker 65 - jepsen.util 4745 :ok :write [21 21] INFO [2019-12-17 12:32:00,716] jepsen worker 65 - jepsen.util 4745 :invoke :write [21 36] INFO [2019-12-17 12:32:00,758] jepsen worker 29 - jepsen.util 4259 :ok :write [9 8] INFO [2019-12-17 12:32:00,759] jepsen worker 29 - jepsen.util 4259 :invoke :write [9 2] INFO [2019-12-17 12:32:00,792] jepsen worker 37 - jepsen.util 5167 :info :write [12 10] :RequestCanceledException INFO [2019-12-17 12:32:00,798] jepsen worker 37 - jepsen.util 5257 :invoke :read [12 nil] INFO [2019-12-17 12:32:00,810] jepsen worker 65 - jepsen.util 4745 :ok :write [21 36] INFO [2019-12-17 12:32:00,810] jepsen worker 65 - jepsen.util 4745 :invoke :cas [21 [35 23]] INFO [2019-12-17 12:32:00,827] jepsen worker 5 - jepsen.util 4955 :info :write [1 20] :RequestCanceledException INFO [2019-12-17 12:32:00,828] jepsen worker 5 - jepsen.util 5045 :invoke :write [1 49] INFO [2019-12-17 12:32:00,830] jepsen worker 29 - jepsen.util 4259 :ok :write [9 2] INFO [2019-12-17 12:32:00,830] jepsen worker 29 - jepsen.util 4259 :invoke :cas [9 [39 19]] INFO [2019-12-17 12:32:00,845] jepsen worker 10 - jepsen.util 5410 :ok :write [3 13] INFO [2019-12-17 12:32:00,845] jepsen worker 10 - jepsen.util 5410 :invoke :write [3 12] INFO [2019-12-17 12:32:00,882] jepsen worker 47 - jepsen.util 6077 :ok :write [15 3] INFO [2019-12-17 12:32:00,883] jepsen worker 47 - jepsen.util 6077 :invoke :cas [15 [39 17]] INFO [2019-12-17 12:32:00,887] jepsen worker 65 - jepsen.util 4745 :fail :cas [21 [35 23]] :ValueNotSwapFrom INFO [2019-12-17 12:32:00,887] jepsen worker 65 - jepsen.util 4745 :invoke :read [21 nil] INFO [2019-12-17 12:32:00,887] jepsen worker 12 - jepsen.util 5772 :ok :write [4 2] INFO [2019-12-17 12:32:00,887] jepsen worker 12 - jepsen.util 5772 :invoke :cas [4 [47 25]] INFO [2019-12-17 12:32:00,890] jepsen worker 29 - jepsen.util 4259 :fail :cas [9 [39 19]] :ValueNotSwapFrom INFO [2019-12-17 12:32:00,890] jepsen worker 29 - jepsen.util 4259 :invoke :write [9 10] INFO [2019-12-17 12:32:00,919] jepsen worker 20 - jepsen.util 5600 :ok :read [6 7] INFO [2019-12-17 12:32:00,919] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31415,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:32:00,919] jepsen worker 20 - jepsen.util 5600 :invoke :cas [6 [23 43]] INFO [2019-12-17 12:32:00,920] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31415,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:32:00,920] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31418,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:32:00,920] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31418,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:32:00,926] jepsen worker 72 - jepsen.util 5202 :ok :read [24 36] INFO [2019-12-17 12:32:00,926] jepsen worker 72 - jepsen.util 5202 :invoke :write [24 29] INFO [2019-12-17 12:32:00,932] jepsen worker 31 - jepsen.util 4981 :ok :write [10 10] INFO [2019-12-17 12:32:00,932] jepsen worker 31 - jepsen.util 4981 :invoke :cas [10 [1 25]] INFO [2019-12-17 12:32:00,978] jepsen worker 53 - jepsen.util 5363 :ok :write [17 10] INFO [2019-12-17 12:32:00,978] jepsen worker 53 - jepsen.util 5363 :invoke :cas [17 [37 9]] INFO [2019-12-17 12:32:01,010] jepsen worker 78 - jepsen.util 5658 :info :write [26 42] :RequestCanceledException INFO [2019-12-17 12:32:01,017] jepsen worker 78 - jepsen.util 5748 :invoke :write [26 47] INFO [2019-12-17 12:32:01,030] jepsen worker 31 - jepsen.util 4981 :fail :cas [10 [1 25]] :ValueNotSwapFrom INFO [2019-12-17 12:32:01,030] jepsen worker 31 - jepsen.util 4981 :invoke :cas [10 [38 40]] INFO [2019-12-17 12:32:01,031] jepsen worker 80 - jepsen.util 5570 :ok :write [26 40] INFO [2019-12-17 12:32:01,032] jepsen worker 80 - jepsen.util 5570 :invoke :cas [26 [20 25]] INFO [2019-12-17 12:32:01,057] jepsen worker 71 - jepsen.util 6191 :info :write [23 48] :RequestCanceledException INFO [2019-12-17 12:32:01,060] jepsen worker 65 - jepsen.util 4745 :ok :read [21 36] INFO [2019-12-17 12:32:01,060] jepsen worker 65 - jepsen.util 4745 :invoke :write [21 36] INFO [2019-12-17 12:32:01,062] jepsen worker 71 - jepsen.util 6281 :invoke :read [23 nil] INFO [2019-12-17 12:32:01,078] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31430,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:32:01,078] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31430,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:32:01,078] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31433,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:32:01,078] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31433,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:32:01,079] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31434,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:32:01,080] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31434,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:32:01,098] jepsen worker 47 - jepsen.util 6077 :fail :cas [15 [39 17]] :ValueNotSwapFrom INFO [2019-12-17 12:32:01,098] jepsen worker 47 - jepsen.util 6077 :invoke :read [15 nil] INFO [2019-12-17 12:32:01,126] jepsen worker 28 - jepsen.util 4168 :fail :read [9 nil] :RequestTimeoutException INFO [2019-12-17 12:32:01,126] jepsen worker 28 - jepsen.util 4168 :invoke :write [9 39] INFO [2019-12-17 12:32:01,140] jepsen worker 11 - jepsen.util 5591 :info :cas [3 [41 9]] :RequestCanceledException INFO [2019-12-17 12:32:01,141] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:32:01,142] jepsen nemesis - couchbase.util Rebalance check count: 236 INFO [2019-12-17 12:32:01,142] jepsen nemesis - couchbase.util Rebalance stuck count: 217 INFO [2019-12-17 12:32:01,155] jepsen worker 11 - jepsen.util 5681 :invoke :read [3 nil] INFO [2019-12-17 12:32:01,200] jepsen worker 46 - jepsen.util 5356 :info :write [15 41] :RequestCanceledException INFO [2019-12-17 12:32:01,202] jepsen worker 29 - jepsen.util 4259 :ok :write [9 10] INFO [2019-12-17 12:32:01,233] jepsen worker 29 - jepsen.util 4259 :invoke :cas [9 [42 14]] INFO [2019-12-17 12:32:01,238] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31445,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:32:01,239] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31449,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:32:01,245] jepsen worker 65 - jepsen.util 4745 :ok :write [21 36] INFO [2019-12-17 12:32:01,245] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31449,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:32:01,251] jepsen worker 65 - jepsen.util 4745 :invoke :read [21 nil] INFO [2019-12-17 12:32:01,252] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31451,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:32:01,252] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31451,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:32:01,253] jepsen worker 10 - jepsen.util 5410 :ok :write [3 12] INFO [2019-12-17 12:32:01,253] jepsen worker 10 - jepsen.util 5410 :invoke :cas [3 [6 44]] INFO [2019-12-17 12:32:01,267] jepsen worker 51 - jepsen.util 5361 :info :write [17 5] :RequestCanceledException INFO [2019-12-17 12:32:01,268] jepsen worker 55 - jepsen.util 6445 :info :write [18 43] :RequestCanceledException INFO [2019-12-17 12:32:01,305] jepsen worker 88 - jepsen.util 5578 :info :cas [29 [42 11]] :RequestCanceledException INFO [2019-12-17 12:32:01,306] jepsen worker 88 - jepsen.util 5668 :invoke :write [29 15] INFO [2019-12-17 12:32:01,340] jepsen worker 63 - jepsen.util 4383 :info :cas [21 [23 27]] :RequestCanceledException INFO [2019-12-17 12:32:01,341] jepsen worker 73 - jepsen.util 5023 :info :cas [24 [46 32]] :RequestCanceledException INFO [2019-12-17 12:32:01,341] jepsen worker 3 - jepsen.util 5403 :info :write [1 23] :RequestCanceledException INFO [2019-12-17 12:32:01,343] jepsen worker 3 - jepsen.util 5493 :invoke :write [1 3] INFO [2019-12-17 12:32:01,358] jepsen worker 72 - jepsen.util 5202 :ok :write [24 29] INFO [2019-12-17 12:32:01,359] jepsen worker 72 - jepsen.util 5202 :invoke :write [24 3] INFO [2019-12-17 12:32:01,407] jepsen worker 45 - jepsen.util 5535 :ok :cas [15 [3 35]] INFO [2019-12-17 12:32:01,407] jepsen worker 45 - jepsen.util 5535 :invoke :write [15 11] INFO [2019-12-17 12:32:01,415] jepsen worker 68 - jepsen.util 6908 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:32:01,427] jepsen worker 89 - jepsen.util 5219 :info :write [29 5] :RequestCanceledException INFO [2019-12-17 12:32:01,440] jepsen worker 56 - jepsen.util 5726 :info :write [18 32] :RequestCanceledException INFO [2019-12-17 12:32:01,451] jepsen worker 8 - jepsen.util 6848 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:32:01,451] jepsen worker 8 - jepsen.util 6848 :invoke :cas [2 [13 31]] INFO [2019-12-17 12:32:01,452] jepsen worker 46 - jepsen.util 5446 :invoke :read [15 nil] INFO [2019-12-17 12:32:01,456] jepsen worker 55 - jepsen.util 6535 :invoke :cas [18 [29 7]] INFO [2019-12-17 12:32:01,468] jepsen worker 6 - jepsen.util 7656 :info :cas [2 [11 31]] :RequestCanceledException INFO [2019-12-17 12:32:01,475] jepsen worker 48 - jepsen.util 6708 :info :write [16 42] :RequestCanceledException INFO [2019-12-17 12:32:01,480] jepsen worker 63 - jepsen.util 4473 :invoke :write [21 27] INFO [2019-12-17 12:32:01,509] jepsen worker 68 - jepsen.util 6908 :invoke :read [22 nil] INFO [2019-12-17 12:32:01,517] jepsen worker 60 - jepsen.util 7980 :info :write [20 43] :RequestCanceledException INFO [2019-12-17 12:32:01,521] jepsen worker 51 - jepsen.util 5451 :invoke :read [17 nil] INFO [2019-12-17 12:32:01,532] jepsen worker 85 - jepsen.util 8185 :info :cas [28 [16 34]] :RequestCanceledException INFO [2019-12-17 12:32:01,543] jepsen worker 76 - jepsen.util 7366 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:32:01,549] jepsen worker 33 - jepsen.util 7323 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:32:01,550] jepsen worker 33 - jepsen.util 7323 :invoke :cas [11 [5 38]] INFO [2019-12-17 12:32:01,551] jepsen worker 76 - jepsen.util 7366 :invoke :cas [25 [3 16]] INFO [2019-12-17 12:32:01,552] jepsen worker 39 - jepsen.util 6969 :info :write [13 22] :RequestCanceledException INFO [2019-12-17 12:32:01,563] jepsen worker 77 - jepsen.util 7457 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:32:01,563] jepsen worker 77 - jepsen.util 7457 :invoke :write [25 21] INFO [2019-12-17 12:32:01,568] jepsen worker 15 - jepsen.util 6225 :info :cas [5 [19 49]] :RequestCanceledException INFO [2019-12-17 12:32:01,571] jepsen worker 84 - jepsen.util 6834 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:32:01,578] jepsen worker 17 - jepsen.util 7307 :info :write [5 19] :RequestCanceledException INFO [2019-12-17 12:32:01,579] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31464,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:32:01,579] jepsen worker 84 - jepsen.util 6834 :invoke :write [28 29] INFO [2019-12-17 12:32:01,580] jepsen worker 16 - jepsen.util 8386 :info :cas [5 [16 18]] :RequestCanceledException INFO [2019-12-17 12:32:01,593] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31464,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:32:01,611] jepsen worker 66 - jepsen.util 6816 :info :cas [22 [2 17]] :RequestCanceledException INFO [2019-12-17 12:32:01,612] jepsen worker 4 - jepsen.util 4684 :info :write [1 40] :RequestCanceledException INFO [2019-12-17 12:32:01,612] jepsen worker 82 - jepsen.util 7912 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:32:01,615] jepsen worker 4 - jepsen.util 4774 :invoke :cas [1 [13 40]] INFO [2019-12-17 12:32:01,616] jepsen worker 19 - jepsen.util 5419 :info :cas [6 [5 2]] :RequestCanceledException INFO [2019-12-17 12:32:01,635] jepsen worker 57 - jepsen.util 6897 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:32:01,642] jepsen worker 41 - jepsen.util 7961 :info :cas [13 [10 41]] :RequestCanceledException INFO [2019-12-17 12:32:01,642] jepsen worker 22 - jepsen.util 7402 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:32:01,643] jepsen worker 82 - jepsen.util 7912 :invoke :cas [27 [16 32]] INFO [2019-12-17 12:32:01,646] jepsen worker 73 - jepsen.util 5113 :invoke :write [24 31] INFO [2019-12-17 12:32:01,648] jepsen worker 56 - jepsen.util 5816 :invoke :cas [18 [12 3]] INFO [2019-12-17 12:32:01,649] jepsen worker 22 - jepsen.util 7402 :invoke :write [7 23] INFO [2019-12-17 12:32:01,649] jepsen worker 57 - jepsen.util 6897 :invoke :cas [19 [2 26]] INFO [2019-12-17 12:32:01,658] jepsen worker 60 - jepsen.util 8070 :invoke :cas [20 [23 0]] INFO [2019-12-17 12:32:01,660] jepsen worker 48 - jepsen.util 6798 :invoke :read [16 nil] INFO [2019-12-17 12:32:01,679] jepsen worker 85 - jepsen.util 8275 :invoke :cas [28 [5 4]] INFO [2019-12-17 12:32:01,681] jepsen worker 39 - jepsen.util 7059 :invoke :write [13 34] INFO [2019-12-17 12:32:01,681] jepsen worker 43 - jepsen.util 6793 :info :write [14 28] :RequestCanceledException INFO [2019-12-17 12:32:01,681] jepsen worker 16 - jepsen.util 8476 :invoke :read [5 nil] INFO [2019-12-17 12:32:01,682] jepsen worker 75 - jepsen.util 7005 :info :write [25 4] :RequestCanceledException INFO [2019-12-17 12:32:01,683] jepsen worker 6 - jepsen.util 7746 :invoke :write [2 44] INFO [2019-12-17 12:32:01,687] jepsen worker 66 - jepsen.util 6906 :invoke :cas [22 [31 33]] INFO [2019-12-17 12:32:01,690] jepsen worker 89 - jepsen.util 5309 :invoke :write [29 46] INFO [2019-12-17 12:32:01,693] jepsen worker 40 - jepsen.util 7780 :info :cas [13 [20 38]] :RequestCanceledException INFO [2019-12-17 12:32:01,697] jepsen worker 15 - jepsen.util 6315 :invoke :write [5 16] INFO [2019-12-17 12:32:01,699] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31476,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:32:01,702] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31476,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:32:01,702] jepsen worker 19 - jepsen.util 5509 :invoke :write [6 0] INFO [2019-12-17 12:32:01,708] jepsen worker 17 - jepsen.util 7397 :invoke :write [5 12] INFO [2019-12-17 12:32:01,712] jepsen worker 83 - jepsen.util 7643 :info :write [27 16] :RequestCanceledException INFO [2019-12-17 12:32:01,720] jepsen worker 44 - jepsen.util 7964 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:32:01,721] jepsen worker 44 - jepsen.util 7964 :invoke :cas [14 [21 24]] INFO [2019-12-17 12:32:01,723] jepsen worker 41 - jepsen.util 8051 :invoke :write [13 45] INFO [2019-12-17 12:32:01,725] jepsen worker 54 - jepsen.util 5904 :info :write [18 8] :RequestCanceledException INFO [2019-12-17 12:32:01,727] jepsen worker 54 - jepsen.util 5994 :invoke :read [18 nil] INFO [2019-12-17 12:32:01,730] jepsen worker 43 - jepsen.util 6883 :invoke :cas [14 [48 15]] INFO [2019-12-17 12:32:01,736] jepsen worker 75 - jepsen.util 7095 :invoke :read [25 nil] INFO [2019-12-17 12:32:01,738] jepsen worker 40 - jepsen.util 7870 :invoke :write [13 43] INFO [2019-12-17 12:32:01,740] jepsen worker 83 - jepsen.util 7733 :invoke :write [27 16] INFO [2019-12-17 12:32:01,759] jepsen worker 18 - jepsen.util 5418 :fail :read [6 nil] :RequestTimeoutException INFO [2019-12-17 12:32:01,760] jepsen worker 18 - jepsen.util 5418 :invoke :read [6 nil] INFO [2019-12-17 12:32:01,760] jepsen worker 27 - jepsen.util 5247 :info :write [9 40] :RequestCanceledException INFO [2019-12-17 12:32:01,762] jepsen worker 27 - jepsen.util 5337 :invoke :write [9 23] INFO [2019-12-17 12:32:01,794] jepsen worker 25 - jepsen.util 7675 :info :cas [8 [17 45]] :RequestCanceledException INFO [2019-12-17 12:32:01,809] jepsen worker 2 - jepsen.util 6932 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:32:01,813] jepsen worker 2 - jepsen.util 6932 :invoke :write [0 30] INFO [2019-12-17 12:32:01,827] jepsen worker 36 - jepsen.util 5796 :info :write [12 33] :RequestCanceledException INFO [2019-12-17 12:32:01,835] jepsen worker 58 - jepsen.util 7708 :info :write [19 31] :RequestCanceledException INFO [2019-12-17 12:32:01,837] jepsen worker 32 - jepsen.util 5522 :info :write [10 7] :RequestCanceledException INFO [2019-12-17 12:32:01,837] jepsen worker 64 - jepsen.util 4114 :info :write [21 11] :RequestCanceledException INFO [2019-12-17 12:32:01,837] jepsen worker 25 - jepsen.util 7765 :invoke :cas [8 [9 11]] INFO [2019-12-17 12:32:01,842] jepsen worker 36 - jepsen.util 5886 :invoke :cas [12 [7 28]] INFO [2019-12-17 12:32:01,851] jepsen worker 58 - jepsen.util 7798 :invoke :write [19 36] INFO [2019-12-17 12:32:01,852] jepsen worker 32 - jepsen.util 5612 :invoke :read [10 nil] INFO [2019-12-17 12:32:01,852] jepsen worker 64 - jepsen.util 4204 :invoke :read [21 nil] INFO [2019-12-17 12:32:01,885] jepsen worker 81 - jepsen.util 7281 :info :cas [27 [3 23]] :RequestCanceledException INFO [2019-12-17 12:32:01,893] jepsen worker 81 - jepsen.util 7371 :invoke :cas [27 [46 19]] INFO [2019-12-17 12:32:01,905] jepsen worker 31 - jepsen.util 4981 :fail :cas [10 [38 40]] :ValueNotSwapFrom INFO [2019-12-17 12:32:01,905] jepsen worker 31 - jepsen.util 4981 :invoke :cas [10 [11 47]] INFO [2019-12-17 12:32:01,911] jepsen worker 38 - jepsen.util 5528 :info :write [12 42] :RequestCanceledException INFO [2019-12-17 12:32:01,922] jepsen worker 38 - jepsen.util 5618 :invoke :cas [12 [45 31]] INFO [2019-12-17 12:32:01,926] jepsen worker 34 - jepsen.util 6694 :info :write [11 24] :RequestCanceledException INFO [2019-12-17 12:32:01,926] jepsen worker 79 - jepsen.util 5299 :info :write [26 8] :RequestCanceledException INFO [2019-12-17 12:32:01,934] jepsen worker 61 - jepsen.util 8251 :info :write [20 32] :RequestCanceledException INFO [2019-12-17 12:32:01,935] jepsen worker 34 - jepsen.util 6784 :invoke :read [11 nil] INFO [2019-12-17 12:32:01,935] jepsen worker 79 - jepsen.util 5389 :invoke :write [26 19] INFO [2019-12-17 12:32:01,943] jepsen worker 61 - jepsen.util 8341 :invoke :cas [20 [9 49]] INFO [2019-12-17 12:32:01,948] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31499,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:32:01,948] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31499,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:32:01,948] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31501,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:32:01,948] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31501,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:32:01,958] jepsen worker 18 - jepsen.util 5418 :ok :read [6 7] INFO [2019-12-17 12:32:01,959] jepsen worker 18 - jepsen.util 5418 :invoke :write [6 46] INFO [2019-12-17 12:32:01,965] jepsen worker 46 - jepsen.util 5446 :ok :read [15 35] INFO [2019-12-17 12:32:01,965] jepsen worker 46 - jepsen.util 5446 :invoke :read [15 nil] INFO [2019-12-17 12:32:01,972] jepsen worker 29 - jepsen.util 4259 :fail :cas [9 [42 14]] :ValueNotSwapFrom INFO [2019-12-17 12:32:01,972] jepsen worker 29 - jepsen.util 4259 :invoke :write [9 16] INFO [2019-12-17 12:32:01,991] jepsen worker 86 - jepsen.util 7556 :info :write [28 36] :RequestCanceledException INFO [2019-12-17 12:32:01,993] jepsen worker 71 - jepsen.util 6281 :ok :read [23 31] INFO [2019-12-17 12:32:01,993] jepsen worker 71 - jepsen.util 6281 :invoke :read [23 nil] INFO [2019-12-17 12:32:01,994] jepsen worker 86 - jepsen.util 7646 :invoke :cas [28 [9 43]] INFO [2019-12-17 12:32:02,002] jepsen worker 70 - jepsen.util 6100 :ok :read [23 31] INFO [2019-12-17 12:32:02,002] jepsen worker 70 - jepsen.util 6100 :invoke :cas [23 [38 12]] INFO [2019-12-17 12:32:02,002] jepsen worker 10 - jepsen.util 5410 :fail :cas [3 [6 44]] :ValueNotSwapFrom INFO [2019-12-17 12:32:02,003] jepsen worker 10 - jepsen.util 5410 :invoke :write [3 21] INFO [2019-12-17 12:32:02,003] jepsen worker 65 - jepsen.util 4745 :ok :read [21 36] INFO [2019-12-17 12:32:02,003] jepsen worker 65 - jepsen.util 4745 :invoke :read [21 nil] INFO [2019-12-17 12:32:02,003] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31531,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:32:02,003] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31531,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:32:02,003] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31534,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:32:02,004] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31534,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:32:02,004] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31535,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:32:02,004] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31535,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:32:02,008] jepsen worker 0 - jepsen.util 7830 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:32:02,008] jepsen worker 0 - jepsen.util 7830 :invoke :read [0 nil] INFO [2019-12-17 12:32:02,032] jepsen worker 69 - jepsen.util 5919 :ok :read [23 31] INFO [2019-12-17 12:32:02,033] jepsen worker 69 - jepsen.util 5919 :invoke :cas [23 [22 5]] INFO [2019-12-17 12:32:02,037] jepsen worker 62 - jepsen.util 6992 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:32:02,037] jepsen worker 62 - jepsen.util 6992 :invoke :cas [20 [30 32]] INFO [2019-12-17 12:32:02,038] jepsen worker 26 - jepsen.util 7766 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:32:02,038] jepsen worker 26 - jepsen.util 7766 :invoke :read [8 nil] INFO [2019-12-17 12:32:02,054] jepsen worker 67 - jepsen.util 7357 :info :cas [22 [19 37]] :RequestCanceledException INFO [2019-12-17 12:32:02,056] jepsen worker 67 - jepsen.util 7447 :invoke :write [22 18] INFO [2019-12-17 12:32:02,101] jepsen worker 37 - jepsen.util 5257 :ok :read [12 6] INFO [2019-12-17 12:32:02,137] jepsen worker 37 - jepsen.util 5257 :invoke :read [12 nil] INFO [2019-12-17 12:32:02,170] jepsen worker 65 - jepsen.util 4745 :ok :read [21 36] INFO [2019-12-17 12:32:02,171] jepsen worker 65 - jepsen.util 4745 :invoke :cas [21 [34 23]] INFO [2019-12-17 12:32:02,174] jepsen worker 32 - jepsen.util 5612 :ok :read [10 10] INFO [2019-12-17 12:32:02,174] jepsen worker 32 - jepsen.util 5612 :invoke :read [10 nil] INFO [2019-12-17 12:32:02,179] jepsen worker 64 - jepsen.util 4204 :ok :read [21 36] INFO [2019-12-17 12:32:02,180] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:32:02,184] jepsen worker 7 - jepsen.util 7567 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:32:02,188] jepsen nemesis - couchbase.util Rebalance check count: 237 INFO [2019-12-17 12:32:02,189] jepsen worker 64 - jepsen.util 4204 :invoke :read [21 nil] INFO [2019-12-17 12:32:02,189] jepsen worker 7 - jepsen.util 7567 :invoke :write [2 5] INFO [2019-12-17 12:32:02,189] jepsen nemesis - couchbase.util Rebalance stuck count: 218 INFO [2019-12-17 12:32:02,201] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31551,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:32:02,202] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31551,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:32:02,202] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31552,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:32:02,203] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31552,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:32:02,203] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31553,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:32:02,204] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31553,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:32:02,208] jepsen worker 21 - jepsen.util 6861 :info :write [7 13] :RequestCanceledException INFO [2019-12-17 12:32:02,214] jepsen worker 21 - jepsen.util 6951 :invoke :write [7 39] INFO [2019-12-17 12:32:02,226] jepsen worker 1 - jepsen.util 7201 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:32:02,226] jepsen worker 1 - jepsen.util 7201 :invoke :cas [0 [29 16]] INFO [2019-12-17 12:32:02,244] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31559,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:32:02,246] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31559,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:32:02,248] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31563,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:32:02,248] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31563,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:32:02,272] jepsen worker 12 - jepsen.util 5772 :fail :cas [4 [47 25]] :ValueNotSwapFrom INFO [2019-12-17 12:32:02,273] jepsen worker 12 - jepsen.util 5772 :invoke :write [4 44] INFO [2019-12-17 12:32:02,310] jepsen worker 14 - jepsen.util 5504 :info :write [4 46] :RequestCanceledException INFO [2019-12-17 12:32:02,314] jepsen worker 14 - jepsen.util 5594 :invoke :cas [4 [8 23]] INFO [2019-12-17 12:32:02,314] jepsen worker 31 - jepsen.util 4981 :fail :cas [10 [11 47]] :ValueNotSwapFrom INFO [2019-12-17 12:32:02,314] jepsen worker 31 - jepsen.util 4981 :invoke :write [10 2] INFO [2019-12-17 12:32:02,318] jepsen worker 63 - jepsen.util 4473 :ok :write [21 27] INFO [2019-12-17 12:32:02,319] jepsen worker 63 - jepsen.util 4473 :invoke :write [21 48] INFO [2019-12-17 12:32:02,351] jepsen worker 32 - jepsen.util 5612 :ok :read [10 10] INFO [2019-12-17 12:32:02,351] jepsen worker 32 - jepsen.util 5612 :invoke :read [10 nil] INFO [2019-12-17 12:32:02,353] jepsen worker 27 - jepsen.util 5337 :ok :write [9 23] INFO [2019-12-17 12:32:02,353] jepsen worker 27 - jepsen.util 5337 :invoke :cas [9 [18 47]] INFO [2019-12-17 12:32:02,372] jepsen worker 53 - jepsen.util 5363 :fail :cas [17 [37 9]] :ValueNotSwapFrom INFO [2019-12-17 12:32:02,373] jepsen worker 53 - jepsen.util 5363 :invoke :write [17 26] INFO [2019-12-17 12:32:02,399] jepsen worker 27 - jepsen.util 5337 :fail :cas [9 [18 47]] :ValueNotSwapFrom INFO [2019-12-17 12:32:02,399] jepsen worker 27 - jepsen.util 5337 :invoke :write [9 19] INFO [2019-12-17 12:32:02,426] jepsen worker 36 - jepsen.util 5886 :fail :cas [12 [7 28]] :ValueNotSwapFrom INFO [2019-12-17 12:32:02,426] jepsen worker 36 - jepsen.util 5886 :invoke :cas [12 [17 17]] INFO [2019-12-17 12:32:02,427] jepsen worker 80 - jepsen.util 5570 :fail :cas [26 [20 25]] :ValueNotSwapFrom INFO [2019-12-17 12:32:02,427] jepsen worker 80 - jepsen.util 5570 :invoke :write [26 9] INFO [2019-12-17 12:32:02,457] jepsen worker 42 - jepsen.util 7872 :info :write [14 15] :RequestCanceledException INFO [2019-12-17 12:32:02,477] jepsen worker 42 - jepsen.util 7962 :invoke :write [14 10] INFO [2019-12-17 12:32:02,488] jepsen worker 28 - jepsen.util 4168 :ok :write [9 39] INFO [2019-12-17 12:32:02,488] jepsen worker 28 - jepsen.util 4168 :invoke :read [9 nil] INFO [2019-12-17 12:32:02,489] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31575,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:32:02,489] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31575,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:32:02,489] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31576,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:32:02,489] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31576,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:32:02,490] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31577,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:32:02,490] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31577,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:32:02,510] jepsen worker 63 - jepsen.util 4473 :ok :write [21 48] INFO [2019-12-17 12:32:02,510] jepsen worker 63 - jepsen.util 4473 :invoke :read [21 nil] INFO [2019-12-17 12:32:02,526] jepsen worker 50 - jepsen.util 7520 :info :cas [16 [21 9]] :RequestCanceledException INFO [2019-12-17 12:32:02,526] jepsen worker 49 - jepsen.util 6619 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:32:02,526] jepsen worker 35 - jepsen.util 7415 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:32:02,526] jepsen worker 49 - jepsen.util 6619 :invoke :read [16 nil] INFO [2019-12-17 12:32:02,526] jepsen worker 35 - jepsen.util 7415 :invoke :cas [11 [6 44]] INFO [2019-12-17 12:32:02,534] jepsen worker 50 - jepsen.util 7610 :invoke :cas [16 [21 18]] INFO [2019-12-17 12:32:02,556] jepsen worker 51 - jepsen.util 5451 :ok :read [17 10] INFO [2019-12-17 12:32:02,556] jepsen worker 51 - jepsen.util 5451 :invoke :read [17 nil] INFO [2019-12-17 12:32:02,557] jepsen worker 9 - jepsen.util 5949 :info :write [3 3] :RequestCanceledException INFO [2019-12-17 12:32:02,557] jepsen worker 24 - jepsen.util 6684 :info :write [8 37] :RequestCanceledException INFO [2019-12-17 12:32:02,557] jepsen worker 74 - jepsen.util 6014 :info :cas [24 [37 14]] :RequestCanceledException INFO [2019-12-17 12:32:02,557] jepsen worker 70 - jepsen.util 6100 :fail :cas [23 [38 12]] :ValueNotSwapFrom INFO [2019-12-17 12:32:02,557] jepsen worker 70 - jepsen.util 6100 :invoke :cas [23 [25 10]] INFO [2019-12-17 12:32:02,558] jepsen worker 18 - jepsen.util 5418 :ok :write [6 46] INFO [2019-12-17 12:32:02,558] jepsen worker 18 - jepsen.util 5418 :invoke :read [6 nil] INFO [2019-12-17 12:32:02,559] jepsen worker 9 - jepsen.util 6039 :invoke :read [3 nil] INFO [2019-12-17 12:32:02,559] jepsen worker 24 - jepsen.util 6774 :invoke :write [8 16] INFO [2019-12-17 12:32:02,559] jepsen worker 74 - jepsen.util 6104 :invoke :cas [24 [49 7]] INFO [2019-12-17 12:32:02,578] jepsen worker 63 - jepsen.util 4473 :ok :read [21 48] INFO [2019-12-17 12:32:02,578] jepsen worker 63 - jepsen.util 4473 :invoke :read [21 nil] INFO [2019-12-17 12:32:02,589] jepsen worker 30 - jepsen.util 5520 :info :cas [10 [49 47]] :RequestCanceledException INFO [2019-12-17 12:32:02,591] jepsen worker 30 - jepsen.util 5610 :invoke :read [10 nil] INFO [2019-12-17 12:32:02,603] jepsen worker 87 - jepsen.util 4947 :info :cas [29 [23 5]] :RequestCanceledException INFO [2019-12-17 12:32:02,603] jepsen worker 23 - jepsen.util 7493 :info :cas [7 [5 40]] :RequestCanceledException INFO [2019-12-17 12:32:02,606] jepsen worker 23 - jepsen.util 7583 :invoke :cas [7 [16 18]] INFO [2019-12-17 12:32:02,606] jepsen worker 87 - jepsen.util 5037 :invoke :write [29 17] INFO [2019-12-17 12:32:02,625] jepsen worker 59 - jepsen.util 7349 :info :write [19 37] :RequestCanceledException INFO [2019-12-17 12:32:02,627] jepsen worker 65 - jepsen.util 4745 :fail :cas [21 [34 23]] :ValueNotSwapFrom INFO [2019-12-17 12:32:02,627] jepsen worker 65 - jepsen.util 4745 :invoke :write [21 19] INFO [2019-12-17 12:32:02,633] jepsen worker 59 - jepsen.util 7439 :invoke :write [19 25] INFO [2019-12-17 12:32:02,634] jepsen worker 13 - jepsen.util 5413 :ok :write [4 31] INFO [2019-12-17 12:32:02,634] jepsen worker 13 - jepsen.util 5413 :invoke :cas [4 [27 20]] INFO [2019-12-17 12:32:02,680] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31600,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:32:02,680] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31600,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:32:02,680] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31604,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:32:02,680] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31604,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:32:02,690] jepsen worker 18 - jepsen.util 5418 :ok :read [6 46] INFO [2019-12-17 12:32:02,693] jepsen worker 18 - jepsen.util 5418 :invoke :write [6 19] INFO [2019-12-17 12:32:02,703] jepsen worker 89 - jepsen.util 5309 :ok :write [29 46] INFO [2019-12-17 12:32:02,703] jepsen worker 89 - jepsen.util 5309 :invoke :write [29 14] INFO [2019-12-17 12:32:02,706] jepsen worker 63 - jepsen.util 4473 :ok :read [21 48] INFO [2019-12-17 12:32:02,706] jepsen worker 63 - jepsen.util 4473 :invoke :write [21 21] INFO [2019-12-17 12:32:02,708] jepsen worker 30 - jepsen.util 5610 :ok :read [10 10] INFO [2019-12-17 12:32:02,709] jepsen worker 30 - jepsen.util 5610 :invoke :write [10 43] INFO [2019-12-17 12:32:02,726] jepsen worker 52 - jepsen.util 5812 :info :write [17 21] :RequestCanceledException INFO [2019-12-17 12:32:02,730] jepsen worker 4 - jepsen.util 4774 :fail :cas [1 [13 40]] :ValueNotSwapFrom INFO [2019-12-17 12:32:02,730] jepsen worker 4 - jepsen.util 4774 :invoke :cas [1 [23 30]] INFO [2019-12-17 12:32:02,730] jepsen worker 52 - jepsen.util 5902 :invoke :read [17 nil] INFO [2019-12-17 12:32:02,758] jepsen worker 65 - jepsen.util 4745 :ok :write [21 19] INFO [2019-12-17 12:32:02,758] jepsen worker 65 - jepsen.util 4745 :invoke :write [21 25] INFO [2019-12-17 12:32:02,832] jepsen worker 73 - jepsen.util 5113 :ok :write [24 31] INFO [2019-12-17 12:32:02,832] jepsen worker 73 - jepsen.util 5113 :invoke :cas [24 [32 43]] INFO [2019-12-17 12:32:02,833] jepsen worker 31 - jepsen.util 4981 :ok :write [10 2] INFO [2019-12-17 12:32:02,834] jepsen worker 31 - jepsen.util 4981 :invoke :read [10 nil] INFO [2019-12-17 12:32:02,898] jepsen worker 63 - jepsen.util 4473 :ok :write [21 21] INFO [2019-12-17 12:32:02,898] jepsen worker 63 - jepsen.util 4473 :invoke :write [21 17] INFO [2019-12-17 12:32:02,904] jepsen worker 79 - jepsen.util 5389 :ok :write [26 19] INFO [2019-12-17 12:32:02,908] jepsen worker 79 - jepsen.util 5389 :invoke :cas [26 [48 12]] INFO [2019-12-17 12:32:02,908] jepsen worker 45 - jepsen.util 5535 :ok :write [15 11] INFO [2019-12-17 12:32:02,910] jepsen worker 45 - jepsen.util 5535 :invoke :write [15 28] INFO [2019-12-17 12:32:02,911] jepsen worker 5 - jepsen.util 5045 :info :write [1 49] :RequestCanceledException INFO [2019-12-17 12:32:02,930] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31620,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:32:02,943] jepsen worker 5 - jepsen.util 5135 :invoke :cas [1 [4 25]] INFO [2019-12-17 12:32:02,944] jepsen worker 20 - jepsen.util 5600 :info :cas [6 [23 43]] :RequestCanceledException INFO [2019-12-17 12:32:02,945] jepsen worker 20 - jepsen.util 5690 :invoke :cas [6 [43 13]] INFO [2019-12-17 12:32:02,951] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31620,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:32:02,981] jepsen worker 80 - jepsen.util 5570 :ok :write [26 9] INFO [2019-12-17 12:32:02,981] jepsen worker 80 - jepsen.util 5570 :invoke :write [26 15] INFO [2019-12-17 12:32:03,015] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31625,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:32:03,026] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31625,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:32:03,027] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31630,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:32:03,038] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31630,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:32:03,136] jepsen worker 47 - jepsen.util 6077 :fail :read [15 nil] :RequestTimeoutException INFO [2019-12-17 12:32:03,141] jepsen worker 78 - jepsen.util 5748 :info :write [26 47] :RequestCanceledException INFO [2019-12-17 12:32:03,142] jepsen worker 47 - jepsen.util 6077 :invoke :write [15 30] INFO [2019-12-17 12:32:03,190] jepsen worker 78 - jepsen.util 5838 :invoke :read [26 nil] INFO [2019-12-17 12:32:03,237] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31633,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:32:03,239] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31633,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:32:03,240] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31634,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:32:03,245] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31634,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:32:03,272] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31639,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:32:03,273] jepsen worker 11 - jepsen.util 5681 :fail :read [3 nil] :RequestTimeoutException INFO [2019-12-17 12:32:03,276] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31639,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:32:03,282] jepsen worker 11 - jepsen.util 5681 :invoke :write [3 48] INFO [2019-12-17 12:32:03,316] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:32:03,319] jepsen nemesis - couchbase.util Rebalance check count: 238 INFO [2019-12-17 12:32:03,344] jepsen nemesis - couchbase.util Rebalance stuck count: 219 INFO [2019-12-17 12:32:03,419] jepsen worker 18 - jepsen.util 5418 :ok :write [6 19] INFO [2019-12-17 12:32:03,422] jepsen worker 18 - jepsen.util 5418 :invoke :read [6 nil] INFO [2019-12-17 12:32:03,503] jepsen worker 65 - jepsen.util 4745 :ok :write [21 25] INFO [2019-12-17 12:32:03,504] jepsen worker 65 - jepsen.util 4745 :invoke :write [21 8] INFO [2019-12-17 12:32:03,528] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31650,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:32:03,534] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31650,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:32:03,549] jepsen worker 8 - jepsen.util 6848 :info :cas [2 [13 31]] :RequestCanceledException INFO [2019-12-17 12:32:03,620] jepsen worker 68 - jepsen.util 6908 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:32:03,627] jepsen worker 76 - jepsen.util 7366 :info :cas [25 [3 16]] :RequestCanceledException INFO [2019-12-17 12:32:03,632] jepsen worker 33 - jepsen.util 7323 :info :cas [11 [5 38]] :RequestCanceledException INFO [2019-12-17 12:32:03,635] jepsen worker 68 - jepsen.util 6908 :invoke :cas [22 [35 37]] INFO [2019-12-17 12:32:03,694] jepsen worker 82 - jepsen.util 7912 :info :cas [27 [16 32]] :RequestCanceledException INFO [2019-12-17 12:32:03,700] jepsen worker 8 - jepsen.util 6938 :invoke :write [2 15] INFO [2019-12-17 12:32:03,709] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31656,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:32:03,711] jepsen worker 76 - jepsen.util 7456 :invoke :write [25 13] INFO [2019-12-17 12:32:03,715] jepsen worker 56 - jepsen.util 5816 :info :cas [18 [12 3]] :RequestCanceledException INFO [2019-12-17 12:32:03,721] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31656,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:32:03,728] jepsen worker 57 - jepsen.util 6897 :info :cas [19 [2 26]] :RequestCanceledException INFO [2019-12-17 12:32:03,730] jepsen worker 60 - jepsen.util 8070 :info :cas [20 [23 0]] :RequestCanceledException INFO [2019-12-17 12:32:03,731] jepsen worker 66 - jepsen.util 6906 :info :cas [22 [31 33]] :RequestCanceledException INFO [2019-12-17 12:32:03,732] jepsen worker 56 - jepsen.util 5906 :invoke :write [18 0] INFO [2019-12-17 12:32:03,732] jepsen worker 33 - jepsen.util 7413 :invoke :write [11 41] INFO [2019-12-17 12:32:03,742] jepsen worker 48 - jepsen.util 6798 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:32:03,744] jepsen worker 48 - jepsen.util 6798 :invoke :cas [16 [26 5]] INFO [2019-12-17 12:32:03,758] jepsen worker 82 - jepsen.util 8002 :invoke :cas [27 [11 17]] INFO [2019-12-17 12:32:03,759] jepsen worker 85 - jepsen.util 8275 :info :cas [28 [5 4]] :RequestCanceledException INFO [2019-12-17 12:32:03,760] jepsen worker 75 - jepsen.util 7095 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:32:03,760] jepsen worker 75 - jepsen.util 7095 :invoke :read [25 nil] INFO [2019-12-17 12:32:03,763] jepsen worker 57 - jepsen.util 6987 :invoke :read [19 nil] INFO [2019-12-17 12:32:03,778] jepsen worker 60 - jepsen.util 8160 :invoke :write [20 5] INFO [2019-12-17 12:32:03,779] jepsen worker 66 - jepsen.util 6996 :invoke :read [22 nil] INFO [2019-12-17 12:32:03,794] jepsen worker 16 - jepsen.util 8476 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:32:03,795] jepsen worker 16 - jepsen.util 8476 :invoke :cas [5 [4 0]] INFO [2019-12-17 12:32:03,805] jepsen worker 85 - jepsen.util 8365 :invoke :read [28 nil] INFO [2019-12-17 12:32:03,827] jepsen worker 44 - jepsen.util 7964 :info :cas [14 [21 24]] :RequestCanceledException INFO [2019-12-17 12:32:03,828] jepsen worker 88 - jepsen.util 5668 :info :write [29 15] :RequestCanceledException INFO [2019-12-17 12:32:03,836] jepsen worker 88 - jepsen.util 5758 :invoke :read [29 nil] INFO [2019-12-17 12:32:03,839] jepsen worker 44 - jepsen.util 8054 :invoke :read [14 nil] INFO [2019-12-17 12:32:03,839] jepsen worker 43 - jepsen.util 6883 :info :cas [14 [48 15]] :RequestCanceledException INFO [2019-12-17 12:32:03,883] jepsen worker 43 - jepsen.util 6973 :invoke :cas [14 [14 48]] INFO [2019-12-17 12:32:03,885] jepsen worker 3 - jepsen.util 5493 :info :write [1 3] :RequestCanceledException INFO [2019-12-17 12:32:03,893] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31674,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:32:03,893] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31674,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:32:04,004] jepsen worker 3 - jepsen.util 5583 :invoke :cas [1 [22 29]] INFO [2019-12-17 12:32:04,005] jepsen worker 45 - jepsen.util 5535 :ok :write [15 28] INFO [2019-12-17 12:32:04,005] jepsen worker 77 - jepsen.util 7457 :info :write [25 21] :RequestCanceledException INFO [2019-12-17 12:32:04,005] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31685,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:32:04,005] jepsen worker 83 - jepsen.util 7733 :info :write [27 16] :RequestCanceledException INFO [2019-12-17 12:32:04,006] jepsen worker 64 - jepsen.util 4204 :ok :read [21 19] INFO [2019-12-17 12:32:04,006] jepsen worker 74 - jepsen.util 6104 :fail :cas [24 [49 7]] :ValueNotSwapFrom INFO [2019-12-17 12:32:04,007] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31678,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:32:04,007] jepsen worker 84 - jepsen.util 6834 :info :write [28 29] :RequestCanceledException INFO [2019-12-17 12:32:04,007] jepsen worker 25 - jepsen.util 7765 :info :cas [8 [9 11]] :RequestCanceledException INFO [2019-12-17 12:32:04,007] jepsen worker 19 - jepsen.util 5509 :info :write [6 0] :RequestCanceledException INFO [2019-12-17 12:32:04,007] jepsen worker 72 - jepsen.util 5202 :info :write [24 3] :RequestCanceledException INFO [2019-12-17 12:32:04,007] jepsen worker 17 - jepsen.util 7397 :info :write [5 12] :RequestCanceledException INFO [2019-12-17 12:32:04,007] jepsen worker 22 - jepsen.util 7402 :info :write [7 23] :RequestCanceledException INFO [2019-12-17 12:32:04,007] jepsen worker 15 - jepsen.util 6315 :info :write [5 16] :RequestCanceledException INFO [2019-12-17 12:32:04,008] jepsen worker 58 - jepsen.util 7798 :info :write [19 36] :RequestCanceledException INFO [2019-12-17 12:32:04,009] jepsen worker 34 - jepsen.util 6784 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:32:04,009] jepsen worker 89 - jepsen.util 5309 :ok :write [29 14] INFO [2019-12-17 12:32:04,009] jepsen worker 45 - jepsen.util 5535 :invoke :cas [15 [49 8]] INFO [2019-12-17 12:32:04,009] jepsen worker 2 - jepsen.util 6932 :info :write [0 30] :RequestCanceledException INFO [2019-12-17 12:32:04,010] jepsen worker 64 - jepsen.util 4204 :invoke :read [21 nil] INFO [2019-12-17 12:32:04,012] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31685,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:32:04,016] jepsen worker 81 - jepsen.util 7371 :info :cas [27 [46 19]] :RequestCanceledException INFO [2019-12-17 12:32:04,018] jepsen worker 41 - jepsen.util 8051 :info :write [13 45] :RequestCanceledException INFO [2019-12-17 12:32:04,018] jepsen worker 0 - jepsen.util 7830 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:32:04,018] jepsen worker 6 - jepsen.util 7746 :info :write [2 44] :RequestCanceledException INFO [2019-12-17 12:32:04,019] jepsen worker 46 - jepsen.util 5446 :fail :read [15 nil] :RequestTimeoutException INFO [2019-12-17 12:32:04,020] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31678,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:32:04,020] jepsen worker 74 - jepsen.util 6104 :invoke :write [24 6] INFO [2019-12-17 12:32:04,023] jepsen worker 39 - jepsen.util 7059 :info :write [13 34] :RequestCanceledException INFO [2019-12-17 12:32:04,023] jepsen worker 61 - jepsen.util 8341 :info :cas [20 [9 49]] :RequestCanceledException INFO [2019-12-17 12:32:04,023] jepsen worker 89 - jepsen.util 5309 :invoke :write [29 31] INFO [2019-12-17 12:32:04,024] jepsen worker 34 - jepsen.util 6784 :invoke :cas [11 [1 11]] INFO [2019-12-17 12:32:04,025] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31688,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:32:04,029] jepsen worker 40 - jepsen.util 7870 :info :write [13 43] :RequestCanceledException INFO [2019-12-17 12:32:04,033] jepsen worker 46 - jepsen.util 5446 :invoke :read [15 nil] INFO [2019-12-17 12:32:04,033] jepsen worker 0 - jepsen.util 7830 :invoke :cas [0 [6 33]] INFO [2019-12-17 12:32:04,034] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31688,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:32:04,050] jepsen worker 10 - jepsen.util 5410 :info :write [3 21] :RequestCanceledException INFO [2019-12-17 12:32:04,059] jepsen worker 72 - jepsen.util 5292 :invoke :cas [24 [19 22]] INFO [2019-12-17 12:32:04,060] jepsen worker 77 - jepsen.util 7547 :invoke :write [25 9] INFO [2019-12-17 12:32:04,068] jepsen worker 19 - jepsen.util 5599 :invoke :write [6 12] INFO [2019-12-17 12:32:04,070] jepsen worker 25 - jepsen.util 7855 :invoke :write [8 46] INFO [2019-12-17 12:32:04,073] jepsen worker 81 - jepsen.util 7461 :invoke :cas [27 [37 6]] INFO [2019-12-17 12:32:04,079] jepsen worker 58 - jepsen.util 7888 :invoke :read [19 nil] INFO [2019-12-17 12:32:04,079] jepsen worker 22 - jepsen.util 7492 :invoke :write [7 34] INFO [2019-12-17 12:32:04,087] jepsen worker 15 - jepsen.util 6405 :invoke :cas [5 [31 15]] INFO [2019-12-17 12:32:04,088] jepsen worker 17 - jepsen.util 7487 :invoke :write [5 12] INFO [2019-12-17 12:32:04,089] jepsen worker 84 - jepsen.util 6924 :invoke :cas [28 [30 13]] INFO [2019-12-17 12:32:04,091] jepsen worker 2 - jepsen.util 7022 :invoke :write [0 30] INFO [2019-12-17 12:32:04,091] jepsen worker 83 - jepsen.util 7823 :invoke :write [27 45] INFO [2019-12-17 12:32:04,091] jepsen worker 6 - jepsen.util 7836 :invoke :cas [2 [39 5]] INFO [2019-12-17 12:32:04,095] jepsen worker 40 - jepsen.util 7960 :invoke :write [13 32] INFO [2019-12-17 12:32:04,097] jepsen worker 29 - jepsen.util 4259 :info :write [9 16] :RequestCanceledException INFO [2019-12-17 12:32:04,098] jepsen worker 61 - jepsen.util 8431 :invoke :cas [20 [35 12]] INFO [2019-12-17 12:32:04,099] jepsen worker 41 - jepsen.util 8141 :invoke :cas [13 [25 37]] INFO [2019-12-17 12:32:04,100] jepsen worker 86 - jepsen.util 7646 :info :cas [28 [9 43]] :RequestCanceledException INFO [2019-12-17 12:32:04,109] jepsen worker 10 - jepsen.util 5500 :invoke :cas [3 [42 43]] INFO [2019-12-17 12:32:04,111] jepsen worker 39 - jepsen.util 7149 :invoke :read [13 nil] INFO [2019-12-17 12:32:04,112] jepsen worker 67 - jepsen.util 7447 :info :write [22 18] :RequestCanceledException INFO [2019-12-17 12:32:04,116] jepsen worker 29 - jepsen.util 4349 :invoke :cas [9 [32 46]] INFO [2019-12-17 12:32:04,140] jepsen worker 62 - jepsen.util 6992 :info :cas [20 [30 32]] :RequestCanceledException INFO [2019-12-17 12:32:04,141] jepsen worker 26 - jepsen.util 7766 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:32:04,141] jepsen worker 26 - jepsen.util 7766 :invoke :write [8 23] INFO [2019-12-17 12:32:04,152] jepsen worker 67 - jepsen.util 7537 :invoke :cas [22 [40 6]] INFO [2019-12-17 12:32:04,152] jepsen worker 86 - jepsen.util 7736 :invoke :read [28 nil] INFO [2019-12-17 12:32:04,199] jepsen worker 62 - jepsen.util 7082 :invoke :cas [20 [20 33]] INFO [2019-12-17 12:32:04,223] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31719,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:32:04,223] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31710,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:32:04,223] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31710,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:32:04,223] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31719,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:32:04,225] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31712,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:32:04,226] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31722,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:32:04,226] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31712,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:32:04,226] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31722,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:32:04,270] jepsen worker 9 - jepsen.util 6039 :ok :read [3 12] INFO [2019-12-17 12:32:04,270] jepsen worker 31 - jepsen.util 4981 :ok :read [10 2] INFO [2019-12-17 12:32:04,270] jepsen worker 9 - jepsen.util 6039 :invoke :read [3 nil] INFO [2019-12-17 12:32:04,271] jepsen worker 31 - jepsen.util 4981 :invoke :write [10 6] INFO [2019-12-17 12:32:04,271] jepsen worker 14 - jepsen.util 5594 :fail :cas [4 [8 23]] :ValueNotSwapFrom INFO [2019-12-17 12:32:04,273] jepsen worker 14 - jepsen.util 5594 :invoke :cas [4 [21 43]] INFO [2019-12-17 12:32:04,273] jepsen worker 20 - jepsen.util 5690 :fail :cas [6 [43 13]] :ValueNotSwapFrom INFO [2019-12-17 12:32:04,273] jepsen worker 55 - jepsen.util 6535 :fail :cas [18 [29 7]] :ValueNotSwapFrom INFO [2019-12-17 12:32:04,274] jepsen worker 20 - jepsen.util 5690 :invoke :cas [6 [11 33]] INFO [2019-12-17 12:32:04,274] jepsen worker 55 - jepsen.util 6535 :invoke :write [18 3] INFO [2019-12-17 12:32:04,285] jepsen worker 69 - jepsen.util 5919 :fail :cas [23 [22 5]] :ValueNotSwapFrom INFO [2019-12-17 12:32:04,286] jepsen worker 69 - jepsen.util 5919 :invoke :cas [23 [36 33]] INFO [2019-12-17 12:32:04,291] jepsen worker 51 - jepsen.util 5451 :ok :read [17 10] INFO [2019-12-17 12:32:04,292] jepsen worker 51 - jepsen.util 5451 :invoke :write [17 2] INFO [2019-12-17 12:32:04,293] jepsen worker 7 - jepsen.util 7567 :info :write [2 5] :RequestCanceledException INFO [2019-12-17 12:32:04,324] jepsen worker 1 - jepsen.util 7201 :info :cas [0 [29 16]] :RequestCanceledException INFO [2019-12-17 12:32:04,324] jepsen worker 21 - jepsen.util 6951 :info :write [7 39] :RequestCanceledException INFO [2019-12-17 12:32:04,324] jepsen worker 12 - jepsen.util 5772 :info :write [4 44] :RequestCanceledException INFO [2019-12-17 12:32:04,326] jepsen worker 7 - jepsen.util 7657 :invoke :write [2 14] INFO [2019-12-17 12:32:04,340] jepsen worker 63 - jepsen.util 4473 :ok :write [21 17] INFO [2019-12-17 12:32:04,340] jepsen worker 63 - jepsen.util 4473 :invoke :write [21 47] INFO [2019-12-17 12:32:04,352] jepsen worker 21 - jepsen.util 7041 :invoke :read [7 nil] INFO [2019-12-17 12:32:04,354] jepsen worker 1 - jepsen.util 7291 :invoke :cas [0 [41 8]] INFO [2019-12-17 12:32:04,355] jepsen worker 12 - jepsen.util 5862 :invoke :write [4 0] INFO [2019-12-17 12:32:04,390] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:32:04,390] jepsen nemesis - couchbase.util Rebalance check count: 239 INFO [2019-12-17 12:32:04,393] jepsen nemesis - couchbase.util Rebalance stuck count: 220 INFO [2019-12-17 12:32:04,427] jepsen worker 53 - jepsen.util 5363 :info :write [17 26] :RequestCanceledException INFO [2019-12-17 12:32:04,428] jepsen worker 32 - jepsen.util 5612 :fail :read [10 nil] :RequestTimeoutException INFO [2019-12-17 12:32:04,428] jepsen worker 32 - jepsen.util 5612 :invoke :cas [10 [26 39]] INFO [2019-12-17 12:32:04,438] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31740,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:32:04,439] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31740,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:32:04,439] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31744,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:32:04,440] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31744,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:32:04,444] jepsen worker 53 - jepsen.util 5453 :invoke :write [17 10] INFO [2019-12-17 12:32:04,508] jepsen worker 27 - jepsen.util 5337 :info :write [9 19] :RequestCanceledException INFO [2019-12-17 12:32:04,526] jepsen worker 28 - jepsen.util 4168 :fail :read [9 nil] :RequestTimeoutException INFO [2019-12-17 12:32:04,526] jepsen worker 28 - jepsen.util 4168 :invoke :write [9 17] INFO [2019-12-17 12:32:04,526] jepsen worker 27 - jepsen.util 5427 :invoke :read [9 nil] INFO [2019-12-17 12:32:04,535] jepsen worker 54 - jepsen.util 5994 :ok :read [18 32] INFO [2019-12-17 12:32:04,536] jepsen worker 54 - jepsen.util 5994 :invoke :cas [18 [18 28]] INFO [2019-12-17 12:32:04,553] jepsen worker 73 - jepsen.util 5113 :fail :cas [24 [32 43]] :ValueNotSwapFrom INFO [2019-12-17 12:32:04,554] jepsen worker 73 - jepsen.util 5113 :invoke :cas [24 [14 30]] INFO [2019-12-17 12:32:04,556] jepsen worker 42 - jepsen.util 7962 :info :write [14 10] :RequestCanceledException INFO [2019-12-17 12:32:04,578] jepsen worker 42 - jepsen.util 8052 :invoke :write [14 34] INFO [2019-12-17 12:32:04,590] jepsen worker 70 - jepsen.util 6100 :fail :cas [23 [25 10]] :ValueNotSwapFrom INFO [2019-12-17 12:32:04,591] jepsen worker 70 - jepsen.util 6100 :invoke :cas [23 [13 17]] INFO [2019-12-17 12:32:04,608] jepsen worker 18 - jepsen.util 5418 :ok :read [6 19] INFO [2019-12-17 12:32:04,608] jepsen worker 18 - jepsen.util 5418 :invoke :write [6 25] INFO [2019-12-17 12:32:04,611] jepsen worker 24 - jepsen.util 6774 :info :write [8 16] :RequestCanceledException INFO [2019-12-17 12:32:04,612] jepsen worker 5 - jepsen.util 5135 :fail :cas [1 [4 25]] :ValueNotSwapFrom INFO [2019-12-17 12:32:04,613] jepsen worker 5 - jepsen.util 5135 :invoke :write [1 4] INFO [2019-12-17 12:32:04,613] jepsen worker 52 - jepsen.util 5902 :ok :read [17 10] INFO [2019-12-17 12:32:04,613] jepsen worker 52 - jepsen.util 5902 :invoke :cas [17 [15 32]] INFO [2019-12-17 12:32:04,615] jepsen worker 24 - jepsen.util 6864 :invoke :read [8 nil] INFO [2019-12-17 12:32:04,622] jepsen worker 49 - jepsen.util 6619 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:32:04,622] jepsen worker 50 - jepsen.util 7610 :info :cas [16 [21 18]] :RequestCanceledException INFO [2019-12-17 12:32:04,623] jepsen worker 49 - jepsen.util 6619 :invoke :read [16 nil] INFO [2019-12-17 12:32:04,623] jepsen worker 35 - jepsen.util 7415 :info :cas [11 [6 44]] :RequestCanceledException INFO [2019-12-17 12:32:04,625] jepsen worker 72 - jepsen.util 5292 :fail :cas [24 [19 22]] :ValueNotSwapFrom INFO [2019-12-17 12:32:04,625] jepsen worker 72 - jepsen.util 5292 :invoke :cas [24 [38 12]] INFO [2019-12-17 12:32:04,626] jepsen worker 71 - jepsen.util 6281 :ok :read [23 31] INFO [2019-12-17 12:32:04,626] jepsen worker 71 - jepsen.util 6281 :invoke :read [23 nil] INFO [2019-12-17 12:32:04,627] jepsen worker 29 - jepsen.util 4349 :fail :cas [9 [32 46]] :ValueNotSwapFrom INFO [2019-12-17 12:32:04,627] jepsen worker 50 - jepsen.util 7700 :invoke :cas [16 [0 3]] INFO [2019-12-17 12:32:04,628] jepsen worker 35 - jepsen.util 7505 :invoke :cas [11 [49 2]] INFO [2019-12-17 12:32:04,628] jepsen worker 29 - jepsen.util 4349 :invoke :cas [9 [19 35]] INFO [2019-12-17 12:32:04,632] jepsen worker 37 - jepsen.util 5257 :ok :read [12 42] INFO [2019-12-17 12:32:04,632] jepsen worker 78 - jepsen.util 5838 :ok :read [26 9] INFO [2019-12-17 12:32:04,632] jepsen worker 37 - jepsen.util 5257 :invoke :cas [12 [45 25]] INFO [2019-12-17 12:32:04,632] jepsen worker 78 - jepsen.util 5838 :invoke :write [26 48] INFO [2019-12-17 12:32:04,633] jepsen worker 38 - jepsen.util 5618 :fail :cas [12 [45 31]] :ValueNotSwapFrom INFO [2019-12-17 12:32:04,633] jepsen worker 38 - jepsen.util 5618 :invoke :read [12 nil] INFO [2019-12-17 12:32:04,634] jepsen worker 46 - jepsen.util 5446 :ok :read [15 28] INFO [2019-12-17 12:32:04,634] jepsen worker 46 - jepsen.util 5446 :invoke :cas [15 [39 4]] INFO [2019-12-17 12:32:04,634] jepsen worker 36 - jepsen.util 5886 :fail :cas [12 [17 17]] :ValueNotSwapFrom INFO [2019-12-17 12:32:04,634] jepsen worker 36 - jepsen.util 5886 :invoke :cas [12 [31 23]] INFO [2019-12-17 12:32:04,635] jepsen worker 45 - jepsen.util 5535 :fail :cas [15 [49 8]] :ValueNotSwapFrom INFO [2019-12-17 12:32:04,635] jepsen worker 45 - jepsen.util 5535 :invoke :cas [15 [37 38]] INFO [2019-12-17 12:32:04,635] jepsen worker 64 - jepsen.util 4204 :ok :read [21 25] INFO [2019-12-17 12:32:04,635] jepsen worker 88 - jepsen.util 5758 :ok :read [29 46] INFO [2019-12-17 12:32:04,636] jepsen worker 88 - jepsen.util 5758 :invoke :read [29 nil] INFO [2019-12-17 12:32:04,636] jepsen worker 64 - jepsen.util 4204 :invoke :read [21 nil] INFO [2019-12-17 12:32:04,636] jepsen worker 79 - jepsen.util 5389 :fail :cas [26 [48 12]] :ValueNotSwapFrom INFO [2019-12-17 12:32:04,636] jepsen worker 79 - jepsen.util 5389 :invoke :write [26 27] INFO [2019-12-17 12:32:04,639] jepsen worker 9 - jepsen.util 6039 :ok :read [3 12] INFO [2019-12-17 12:32:04,639] jepsen worker 9 - jepsen.util 6039 :invoke :write [3 28] INFO [2019-12-17 12:32:04,651] jepsen worker 20 - jepsen.util 5690 :fail :cas [6 [11 33]] :ValueNotSwapFrom INFO [2019-12-17 12:32:04,652] jepsen worker 20 - jepsen.util 5690 :invoke :read [6 nil] INFO [2019-12-17 12:32:04,658] jepsen worker 32 - jepsen.util 5612 :fail :cas [10 [26 39]] :ValueNotSwapFrom INFO [2019-12-17 12:32:04,658] jepsen worker 23 - jepsen.util 7583 :info :cas [7 [16 18]] :RequestCanceledException INFO [2019-12-17 12:32:04,658] jepsen worker 32 - jepsen.util 5612 :invoke :read [10 nil] INFO [2019-12-17 12:32:04,660] jepsen worker 23 - jepsen.util 7673 :invoke :cas [7 [20 45]] INFO [2019-12-17 12:32:04,661] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31778,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:32:04,661] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31778,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:32:04,661] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31782,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:32:04,662] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31782,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:32:04,692] jepsen worker 59 - jepsen.util 7439 :info :write [19 25] :RequestCanceledException INFO [2019-12-17 12:32:04,694] jepsen worker 59 - jepsen.util 7529 :invoke :write [19 11] INFO [2019-12-17 12:32:04,726] jepsen worker 87 - jepsen.util 5037 :info :write [29 17] :RequestCanceledException INFO [2019-12-17 12:32:04,726] jepsen worker 13 - jepsen.util 5413 :info :cas [4 [27 20]] :RequestCanceledException INFO [2019-12-17 12:32:04,740] jepsen worker 87 - jepsen.util 5127 :invoke :cas [29 [15 18]] INFO [2019-12-17 12:32:04,740] jepsen worker 30 - jepsen.util 5610 :info :write [10 43] :RequestCanceledException INFO [2019-12-17 12:32:04,741] jepsen worker 13 - jepsen.util 5503 :invoke :read [4 nil] INFO [2019-12-17 12:32:04,751] jepsen worker 30 - jepsen.util 5700 :invoke :write [10 17] INFO [2019-12-17 12:32:04,759] jepsen worker 80 - jepsen.util 5570 :ok :write [26 15] INFO [2019-12-17 12:32:04,760] jepsen worker 80 - jepsen.util 5570 :invoke :cas [26 [39 2]] INFO [2019-12-17 12:32:04,796] jepsen worker 65 - jepsen.util 4745 :ok :write [21 8] INFO [2019-12-17 12:32:04,797] jepsen worker 65 - jepsen.util 4745 :invoke :write [21 8] INFO [2019-12-17 12:32:04,798] jepsen worker 69 - jepsen.util 5919 :fail :cas [23 [36 33]] :ValueNotSwapFrom INFO [2019-12-17 12:32:04,798] jepsen worker 69 - jepsen.util 5919 :invoke :read [23 nil] INFO [2019-12-17 12:32:04,826] jepsen worker 4 - jepsen.util 4774 :info :cas [1 [23 30]] :RequestCanceledException INFO [2019-12-17 12:32:04,840] jepsen worker 73 - jepsen.util 5113 :fail :cas [24 [14 30]] :ValueNotSwapFrom INFO [2019-12-17 12:32:04,840] jepsen worker 73 - jepsen.util 5113 :invoke :write [24 24] INFO [2019-12-17 12:32:04,847] jepsen worker 4 - jepsen.util 4864 :invoke :read [1 nil] INFO [2019-12-17 12:32:04,871] jepsen worker 27 - jepsen.util 5427 :ok :read [9 39] INFO [2019-12-17 12:32:04,871] jepsen worker 27 - jepsen.util 5427 :invoke :write [9 44] INFO [2019-12-17 12:32:04,872] jepsen worker 54 - jepsen.util 5994 :fail :cas [18 [18 28]] :ValueNotSwapFrom INFO [2019-12-17 12:32:04,873] jepsen worker 54 - jepsen.util 5994 :invoke :write [18 11] INFO [2019-12-17 12:32:04,874] jepsen worker 72 - jepsen.util 5292 :fail :cas [24 [38 12]] :ValueNotSwapFrom INFO [2019-12-17 12:32:04,878] jepsen worker 72 - jepsen.util 5292 :invoke :read [24 nil] INFO [2019-12-17 12:32:04,910] jepsen worker 29 - jepsen.util 4349 :fail :cas [9 [19 35]] :ValueNotSwapFrom INFO [2019-12-17 12:32:04,910] jepsen worker 29 - jepsen.util 4349 :invoke :write [9 45] INFO [2019-12-17 12:32:04,916] jepsen worker 70 - jepsen.util 6100 :fail :cas [23 [13 17]] :ValueNotSwapFrom INFO [2019-12-17 12:32:04,916] jepsen worker 70 - jepsen.util 6100 :invoke :read [23 nil] INFO [2019-12-17 12:32:04,916] jepsen worker 64 - jepsen.util 4204 :ok :read [21 17] INFO [2019-12-17 12:32:04,916] jepsen worker 64 - jepsen.util 4204 :invoke :write [21 6] INFO [2019-12-17 12:32:04,934] jepsen worker 69 - jepsen.util 5919 :ok :read [23 31] INFO [2019-12-17 12:32:04,937] jepsen worker 69 - jepsen.util 5919 :invoke :cas [23 [9 48]] INFO [2019-12-17 12:32:04,940] jepsen worker 46 - jepsen.util 5446 :fail :cas [15 [39 4]] :ValueNotSwapFrom INFO [2019-12-17 12:32:04,940] jepsen worker 46 - jepsen.util 5446 :invoke :read [15 nil] INFO [2019-12-17 12:32:04,941] jepsen worker 37 - jepsen.util 5257 :fail :cas [12 [45 25]] :ValueNotSwapFrom INFO [2019-12-17 12:32:04,941] jepsen worker 37 - jepsen.util 5257 :invoke :write [12 44] INFO [2019-12-17 12:32:04,944] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31797,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:32:04,944] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31797,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:32:04,944] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31801,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:32:04,944] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31801,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:32:04,944] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31802,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:32:04,944] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31802,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:32:04,993] jepsen worker 14 - jepsen.util 5594 :fail :cas [4 [21 43]] :ValueNotSwapFrom INFO [2019-12-17 12:32:04,993] jepsen worker 14 - jepsen.util 5594 :invoke :write [4 21] INFO [2019-12-17 12:32:04,999] jepsen worker 27 - jepsen.util 5427 :ok :write [9 44] INFO [2019-12-17 12:32:04,999] jepsen worker 27 - jepsen.util 5427 :invoke :cas [9 [49 42]] INFO [2019-12-17 12:32:05,030] jepsen worker 46 - jepsen.util 5446 :ok :read [15 28] INFO [2019-12-17 12:32:05,030] jepsen worker 31 - jepsen.util 4981 :ok :write [10 6] INFO [2019-12-17 12:32:05,030] jepsen worker 46 - jepsen.util 5446 :invoke :cas [15 [45 27]] INFO [2019-12-17 12:32:05,030] jepsen worker 31 - jepsen.util 4981 :invoke :read [10 nil] INFO [2019-12-17 12:32:05,031] jepsen worker 11 - jepsen.util 5681 :ok :write [3 48] INFO [2019-12-17 12:32:05,031] jepsen worker 11 - jepsen.util 5681 :invoke :cas [3 [23 7]] INFO [2019-12-17 12:32:05,051] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31813,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:32:05,051] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31813,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:32:05,051] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31815,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:32:05,052] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31815,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:32:05,091] jepsen worker 27 - jepsen.util 5427 :fail :cas [9 [49 42]] :ValueNotSwapFrom INFO [2019-12-17 12:32:05,091] jepsen worker 27 - jepsen.util 5427 :invoke :write [9 27] INFO [2019-12-17 12:32:05,135] jepsen worker 65 - jepsen.util 4745 :ok :write [21 8] INFO [2019-12-17 12:32:05,136] jepsen worker 65 - jepsen.util 4745 :invoke :write [21 48] INFO [2019-12-17 12:32:05,137] jepsen worker 10 - jepsen.util 5500 :fail :cas [3 [42 43]] :ValueNotSwapFrom INFO [2019-12-17 12:32:05,137] jepsen worker 10 - jepsen.util 5500 :invoke :write [3 20] INFO [2019-12-17 12:32:05,139] jepsen worker 70 - jepsen.util 6100 :ok :read [23 31] INFO [2019-12-17 12:32:05,140] jepsen worker 70 - jepsen.util 6100 :invoke :cas [23 [45 20]] INFO [2019-12-17 12:32:05,163] jepsen worker 31 - jepsen.util 4981 :ok :read [10 6] INFO [2019-12-17 12:32:05,164] jepsen worker 31 - jepsen.util 4981 :invoke :cas [10 [13 39]] INFO [2019-12-17 12:32:05,174] jepsen worker 71 - jepsen.util 6281 :ok :read [23 31] INFO [2019-12-17 12:32:05,174] jepsen worker 71 - jepsen.util 6281 :invoke :read [23 nil] INFO [2019-12-17 12:32:05,186] jepsen worker 36 - jepsen.util 5886 :fail :cas [12 [31 23]] :ValueNotSwapFrom INFO [2019-12-17 12:32:05,186] jepsen worker 36 - jepsen.util 5886 :invoke :read [12 nil] INFO [2019-12-17 12:32:05,200] jepsen worker 19 - jepsen.util 5599 :ok :write [6 12] INFO [2019-12-17 12:32:05,200] jepsen worker 19 - jepsen.util 5599 :invoke :read [6 nil] INFO [2019-12-17 12:32:05,202] jepsen worker 46 - jepsen.util 5446 :fail :cas [15 [45 27]] :ValueNotSwapFrom INFO [2019-12-17 12:32:05,202] jepsen worker 46 - jepsen.util 5446 :invoke :read [15 nil] INFO [2019-12-17 12:32:05,204] jepsen worker 11 - jepsen.util 5681 :fail :cas [3 [23 7]] :ValueNotSwapFrom INFO [2019-12-17 12:32:05,205] jepsen worker 11 - jepsen.util 5681 :invoke :cas [3 [26 31]] INFO [2019-12-17 12:32:05,209] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31832,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:32:05,209] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31832,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:32:05,209] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31833,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:32:05,209] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31833,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:32:05,220] jepsen worker 52 - jepsen.util 5902 :fail :cas [17 [15 32]] :ValueNotSwapFrom INFO [2019-12-17 12:32:05,220] jepsen worker 52 - jepsen.util 5902 :invoke :cas [17 [43 41]] INFO [2019-12-17 12:32:05,233] jepsen worker 31 - jepsen.util 4981 :fail :cas [10 [13 39]] :ValueNotSwapFrom INFO [2019-12-17 12:32:05,235] jepsen worker 27 - jepsen.util 5427 :ok :write [9 27] INFO [2019-12-17 12:32:05,237] jepsen worker 31 - jepsen.util 4981 :invoke :write [10 24] INFO [2019-12-17 12:32:05,238] jepsen worker 27 - jepsen.util 5427 :invoke :cas [9 [15 24]] INFO [2019-12-17 12:32:05,297] jepsen worker 46 - jepsen.util 5446 :ok :read [15 28] INFO [2019-12-17 12:32:05,297] jepsen worker 46 - jepsen.util 5446 :invoke :cas [15 [18 36]] INFO [2019-12-17 12:32:05,312] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31841,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:32:05,312] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31841,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:32:05,312] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31843,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:32:05,312] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31843,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:32:05,346] jepsen worker 27 - jepsen.util 5427 :fail :cas [9 [15 24]] :ValueNotSwapFrom INFO [2019-12-17 12:32:05,346] jepsen worker 27 - jepsen.util 5427 :invoke :cas [9 [7 34]] INFO [2019-12-17 12:32:05,410] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:32:05,429] jepsen worker 36 - jepsen.util 5886 :ok :read [12 42] INFO [2019-12-17 12:32:05,430] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31853,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:32:05,430] jepsen worker 46 - jepsen.util 5446 :fail :cas [15 [18 36]] :ValueNotSwapFrom INFO [2019-12-17 12:32:05,431] jepsen nemesis - couchbase.util Rebalance check count: 240 INFO [2019-12-17 12:32:05,434] jepsen worker 36 - jepsen.util 5886 :invoke :cas [12 [41 41]] INFO [2019-12-17 12:32:05,476] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31853,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:32:05,476] jepsen nemesis - couchbase.util Rebalance stuck count: 221 INFO [2019-12-17 12:32:05,476] jepsen worker 46 - jepsen.util 5446 :invoke :cas [15 [35 37]] INFO [2019-12-17 12:32:05,484] jepsen worker 80 - jepsen.util 5570 :fail :cas [26 [39 2]] :ValueNotSwapFrom INFO [2019-12-17 12:32:05,510] jepsen worker 87 - jepsen.util 5127 :fail :cas [29 [15 18]] :ValueNotSwapFrom INFO [2019-12-17 12:32:05,519] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31856,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:32:05,520] jepsen worker 80 - jepsen.util 5570 :invoke :read [26 nil] INFO [2019-12-17 12:32:05,520] jepsen worker 87 - jepsen.util 5127 :invoke :cas [29 [15 2]] INFO [2019-12-17 12:32:05,520] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31856,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:32:05,531] jepsen worker 69 - jepsen.util 5919 :fail :cas [23 [9 48]] :ValueNotSwapFrom INFO [2019-12-17 12:32:05,531] jepsen worker 69 - jepsen.util 5919 :invoke :write [23 16] INFO [2019-12-17 12:32:05,553] jepsen worker 13 - jepsen.util 5503 :ok :read [4 44] INFO [2019-12-17 12:32:05,554] jepsen worker 13 - jepsen.util 5503 :invoke :cas [4 [45 12]] INFO [2019-12-17 12:32:05,566] jepsen worker 4 - jepsen.util 4864 :ok :read [1 3] INFO [2019-12-17 12:32:05,567] jepsen worker 4 - jepsen.util 4864 :invoke :write [1 31] INFO [2019-12-17 12:32:05,583] jepsen worker 27 - jepsen.util 5427 :fail :cas [9 [7 34]] :ValueNotSwapFrom INFO [2019-12-17 12:32:05,584] jepsen worker 27 - jepsen.util 5427 :invoke :cas [9 [15 14]] INFO [2019-12-17 12:32:05,605] jepsen worker 46 - jepsen.util 5446 :fail :cas [15 [35 37]] :ValueNotSwapFrom INFO [2019-12-17 12:32:05,605] jepsen worker 46 - jepsen.util 5446 :invoke :cas [15 [40 22]] INFO [2019-12-17 12:32:05,631] jepsen worker 27 - jepsen.util 5427 :fail :cas [9 [15 14]] :ValueNotSwapFrom INFO [2019-12-17 12:32:05,631] jepsen worker 27 - jepsen.util 5427 :invoke :write [9 49] INFO [2019-12-17 12:32:05,648] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31867,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:32:05,648] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31867,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:32:05,648] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31870,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:32:05,649] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31870,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:32:05,650] jepsen worker 46 - jepsen.util 5446 :fail :cas [15 [40 22]] :ValueNotSwapFrom INFO [2019-12-17 12:32:05,650] jepsen worker 46 - jepsen.util 5446 :invoke :write [15 0] INFO [2019-12-17 12:32:05,672] jepsen worker 13 - jepsen.util 5503 :fail :cas [4 [45 12]] :ValueNotSwapFrom INFO [2019-12-17 12:32:05,673] jepsen worker 13 - jepsen.util 5503 :invoke :cas [4 [0 8]] INFO [2019-12-17 12:32:05,678] jepsen worker 88 - jepsen.util 5758 :ok :read [29 14] INFO [2019-12-17 12:32:05,679] jepsen worker 88 - jepsen.util 5758 :invoke :cas [29 [28 40]] INFO [2019-12-17 12:32:05,701] jepsen worker 68 - jepsen.util 6908 :info :cas [22 [35 37]] :RequestCanceledException INFO [2019-12-17 12:32:05,716] jepsen worker 68 - jepsen.util 6998 :invoke :read [22 nil] INFO [2019-12-17 12:32:05,717] jepsen worker 27 - jepsen.util 5427 :ok :write [9 49] INFO [2019-12-17 12:32:05,717] jepsen worker 27 - jepsen.util 5427 :invoke :cas [9 [2 18]] INFO [2019-12-17 12:32:05,724] jepsen worker 38 - jepsen.util 5618 :ok :read [12 42] INFO [2019-12-17 12:32:05,724] jepsen worker 38 - jepsen.util 5618 :invoke :read [12 nil] INFO [2019-12-17 12:32:05,793] jepsen worker 66 - jepsen.util 6996 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:32:05,794] jepsen worker 66 - jepsen.util 6996 :invoke :read [22 nil] INFO [2019-12-17 12:32:05,811] jepsen worker 85 - jepsen.util 8365 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:32:05,811] jepsen worker 85 - jepsen.util 8365 :invoke :cas [28 [25 14]] INFO [2019-12-17 12:32:05,812] jepsen worker 27 - jepsen.util 5427 :fail :cas [9 [2 18]] :ValueNotSwapFrom INFO [2019-12-17 12:32:05,813] jepsen worker 27 - jepsen.util 5427 :invoke :write [9 48] INFO [2019-12-17 12:32:05,826] jepsen worker 47 - jepsen.util 6077 :info :write [15 30] :RequestCanceledException INFO [2019-12-17 12:32:05,836] jepsen worker 47 - jepsen.util 6167 :invoke :write [15 29] INFO [2019-12-17 12:32:05,841] jepsen worker 48 - jepsen.util 6798 :info :cas [16 [26 5]] :RequestCanceledException INFO [2019-12-17 12:32:05,841] jepsen worker 82 - jepsen.util 8002 :info :cas [27 [11 17]] :RequestCanceledException INFO [2019-12-17 12:32:05,854] jepsen worker 82 - jepsen.util 8092 :invoke :read [27 nil] INFO [2019-12-17 12:32:05,854] jepsen worker 75 - jepsen.util 7095 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:32:05,855] jepsen worker 57 - jepsen.util 6987 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:32:05,855] jepsen worker 48 - jepsen.util 6888 :invoke :read [16 nil] INFO [2019-12-17 12:32:05,855] jepsen worker 75 - jepsen.util 7095 :invoke :cas [25 [27 38]] INFO [2019-12-17 12:32:05,855] jepsen worker 57 - jepsen.util 6987 :invoke :read [19 nil] INFO [2019-12-17 12:32:05,895] jepsen worker 16 - jepsen.util 8476 :info :cas [5 [4 0]] :RequestCanceledException INFO [2019-12-17 12:32:05,907] jepsen worker 13 - jepsen.util 5503 :fail :cas [4 [0 8]] :ValueNotSwapFrom INFO [2019-12-17 12:32:05,908] jepsen worker 13 - jepsen.util 5503 :invoke :read [4 nil] INFO [2019-12-17 12:32:05,910] jepsen worker 16 - jepsen.util 8566 :invoke :read [5 nil] INFO [2019-12-17 12:32:05,913] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31886,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:32:05,915] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31886,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:32:05,916] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31890,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:32:05,919] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31890,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:32:05,940] jepsen worker 44 - jepsen.util 8054 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:32:05,941] jepsen worker 44 - jepsen.util 8054 :invoke :cas [14 [1 39]] INFO [2019-12-17 12:32:05,944] jepsen worker 87 - jepsen.util 5127 :fail :cas [29 [15 2]] :ValueNotSwapFrom INFO [2019-12-17 12:32:05,945] jepsen worker 87 - jepsen.util 5127 :invoke :read [29 nil] INFO [2019-12-17 12:32:05,951] jepsen worker 52 - jepsen.util 5902 :fail :cas [17 [43 41]] :ValueNotSwapFrom INFO [2019-12-17 12:32:05,951] jepsen worker 52 - jepsen.util 5902 :invoke :write [17 30] INFO [2019-12-17 12:32:05,957] jepsen worker 43 - jepsen.util 6973 :info :cas [14 [14 48]] :RequestCanceledException INFO [2019-12-17 12:32:05,962] jepsen worker 43 - jepsen.util 7063 :invoke :cas [14 [11 19]] INFO [2019-12-17 12:32:05,966] jepsen worker 80 - jepsen.util 5570 :ok :read [26 15] INFO [2019-12-17 12:32:05,966] jepsen worker 80 - jepsen.util 5570 :invoke :cas [26 [46 0]] INFO [2019-12-17 12:32:05,979] jepsen worker 3 - jepsen.util 5583 :fail :cas [1 [22 29]] :ValueNotSwapFrom INFO [2019-12-17 12:32:05,979] jepsen worker 3 - jepsen.util 5583 :invoke :write [1 0] INFO [2019-12-17 12:32:06,004] jepsen worker 56 - jepsen.util 5906 :info :write [18 0] :RequestCanceledException INFO [2019-12-17 12:32:06,008] jepsen worker 56 - jepsen.util 5996 :invoke :write [18 35] INFO [2019-12-17 12:32:06,016] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31903,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:32:06,016] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31903,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:32:06,016] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31905,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:32:06,016] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31905,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:32:06,023] jepsen worker 27 - jepsen.util 5427 :ok :write [9 48] INFO [2019-12-17 12:32:06,024] jepsen worker 27 - jepsen.util 5427 :invoke :cas [9 [4 7]] INFO [2019-12-17 12:32:06,024] jepsen worker 8 - jepsen.util 6938 :info :write [2 15] :RequestCanceledException INFO [2019-12-17 12:32:06,024] jepsen worker 33 - jepsen.util 7413 :info :write [11 41] :RequestCanceledException INFO [2019-12-17 12:32:06,024] jepsen worker 60 - jepsen.util 8160 :info :write [20 5] :RequestCanceledException INFO [2019-12-17 12:32:06,029] jepsen worker 60 - jepsen.util 8250 :invoke :cas [20 [2 37]] INFO [2019-12-17 12:32:06,030] jepsen worker 8 - jepsen.util 7028 :invoke :cas [2 [12 46]] INFO [2019-12-17 12:32:06,030] jepsen worker 33 - jepsen.util 7503 :invoke :cas [11 [6 30]] INFO [2019-12-17 12:32:06,038] jepsen worker 71 - jepsen.util 6281 :ok :read [23 31] INFO [2019-12-17 12:32:06,038] jepsen worker 71 - jepsen.util 6281 :invoke :write [23 2] INFO [2019-12-17 12:32:06,044] jepsen worker 70 - jepsen.util 6100 :fail :cas [23 [45 20]] :ValueNotSwapFrom INFO [2019-12-17 12:32:06,044] jepsen worker 13 - jepsen.util 5503 :ok :read [4 44] INFO [2019-12-17 12:32:06,044] jepsen worker 70 - jepsen.util 6100 :invoke :read [23 nil] INFO [2019-12-17 12:32:06,044] jepsen worker 13 - jepsen.util 5503 :invoke :write [4 21] INFO [2019-12-17 12:32:06,054] jepsen worker 34 - jepsen.util 6784 :info :cas [11 [1 11]] :RequestCanceledException INFO [2019-12-17 12:32:06,069] jepsen worker 34 - jepsen.util 6874 :invoke :cas [11 [16 38]] INFO [2019-12-17 12:32:06,091] jepsen worker 31 - jepsen.util 4981 :ok :write [10 24] INFO [2019-12-17 12:32:06,092] jepsen worker 31 - jepsen.util 4981 :invoke :write [10 34] INFO [2019-12-17 12:32:06,093] jepsen worker 76 - jepsen.util 7456 :info :write [25 13] :RequestCanceledException INFO [2019-12-17 12:32:06,099] jepsen worker 46 - jepsen.util 5446 :ok :write [15 0] INFO [2019-12-17 12:32:06,100] jepsen worker 46 - jepsen.util 5446 :invoke :read [15 nil] INFO [2019-12-17 12:32:06,109] jepsen worker 74 - jepsen.util 6104 :ok :write [24 6] INFO [2019-12-17 12:32:06,109] jepsen worker 74 - jepsen.util 6104 :invoke :write [24 45] INFO [2019-12-17 12:32:06,110] jepsen worker 0 - jepsen.util 7830 :info :cas [0 [6 33]] :RequestCanceledException INFO [2019-12-17 12:32:06,111] jepsen worker 15 - jepsen.util 6405 :info :cas [5 [31 15]] :RequestCanceledException INFO [2019-12-17 12:32:06,112] jepsen worker 76 - jepsen.util 7546 :invoke :write [25 16] INFO [2019-12-17 12:32:06,114] jepsen worker 0 - jepsen.util 7920 :invoke :read [0 nil] INFO [2019-12-17 12:32:06,114] jepsen worker 10 - jepsen.util 5500 :ok :write [3 20] INFO [2019-12-17 12:32:06,115] jepsen worker 10 - jepsen.util 5500 :invoke :write [3 46] INFO [2019-12-17 12:32:06,126] jepsen worker 58 - jepsen.util 7888 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:32:06,126] jepsen worker 61 - jepsen.util 8431 :info :cas [20 [35 12]] :RequestCanceledException INFO [2019-12-17 12:32:06,126] jepsen worker 6 - jepsen.util 7836 :info :cas [2 [39 5]] :RequestCanceledException INFO [2019-12-17 12:32:06,126] jepsen worker 39 - jepsen.util 7149 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:32:06,129] jepsen worker 15 - jepsen.util 6495 :invoke :write [5 7] INFO [2019-12-17 12:32:06,130] jepsen worker 58 - jepsen.util 7888 :invoke :cas [19 [43 36]] INFO [2019-12-17 12:32:06,130] jepsen worker 39 - jepsen.util 7149 :invoke :write [13 47] INFO [2019-12-17 12:32:06,135] jepsen worker 61 - jepsen.util 8521 :invoke :write [20 6] INFO [2019-12-17 12:32:06,137] jepsen worker 6 - jepsen.util 7926 :invoke :write [2 3] INFO [2019-12-17 12:32:06,138] jepsen worker 41 - jepsen.util 8141 :info :cas [13 [25 37]] :RequestCanceledException INFO [2019-12-17 12:32:06,138] jepsen worker 84 - jepsen.util 6924 :info :cas [28 [30 13]] :RequestCanceledException INFO [2019-12-17 12:32:06,148] jepsen worker 84 - jepsen.util 7014 :invoke :write [28 27] INFO [2019-12-17 12:32:06,149] jepsen worker 41 - jepsen.util 8231 :invoke :write [13 40] INFO [2019-12-17 12:32:06,154] jepsen worker 81 - jepsen.util 7461 :info :cas [27 [37 6]] :RequestCanceledException INFO [2019-12-17 12:32:06,181] jepsen worker 81 - jepsen.util 7551 :invoke :read [27 nil] INFO [2019-12-17 12:32:06,183] jepsen worker 46 - jepsen.util 5446 :ok :read [15 0] INFO [2019-12-17 12:32:06,183] jepsen worker 46 - jepsen.util 5446 :invoke :read [15 nil] INFO [2019-12-17 12:32:06,194] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31931,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:32:06,195] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31931,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:32:06,195] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31932,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:32:06,195] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31932,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:32:06,195] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31933,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:32:06,196] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31933,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:32:06,204] jepsen worker 36 - jepsen.util 5886 :fail :cas [12 [41 41]] :ValueNotSwapFrom INFO [2019-12-17 12:32:06,205] jepsen worker 36 - jepsen.util 5886 :invoke :write [12 28] INFO [2019-12-17 12:32:06,209] jepsen worker 67 - jepsen.util 7537 :info :cas [22 [40 6]] :RequestCanceledException INFO [2019-12-17 12:32:06,214] jepsen worker 67 - jepsen.util 7627 :invoke :write [22 21] INFO [2019-12-17 12:32:06,214] jepsen worker 65 - jepsen.util 4745 :ok :write [21 48] INFO [2019-12-17 12:32:06,215] jepsen worker 65 - jepsen.util 4745 :invoke :read [21 nil] INFO [2019-12-17 12:32:06,218] jepsen worker 38 - jepsen.util 5618 :ok :read [12 42] INFO [2019-12-17 12:32:06,222] jepsen worker 38 - jepsen.util 5618 :invoke :write [12 18] INFO [2019-12-17 12:32:06,227] jepsen worker 86 - jepsen.util 7736 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:32:06,227] jepsen worker 86 - jepsen.util 7736 :invoke :cas [28 [8 29]] INFO [2019-12-17 12:32:06,252] jepsen worker 89 - jepsen.util 5309 :info :write [29 31] :RequestCanceledException INFO [2019-12-17 12:32:06,252] jepsen worker 62 - jepsen.util 7082 :info :cas [20 [20 33]] :RequestCanceledException INFO [2019-12-17 12:32:06,254] jepsen worker 89 - jepsen.util 5399 :invoke :read [29 nil] INFO [2019-12-17 12:32:06,259] jepsen worker 62 - jepsen.util 7172 :invoke :cas [20 [13 44]] INFO [2019-12-17 12:32:06,268] jepsen worker 46 - jepsen.util 5446 :ok :read [15 0] INFO [2019-12-17 12:32:06,268] jepsen worker 46 - jepsen.util 5446 :invoke :read [15 nil] INFO [2019-12-17 12:32:06,297] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31940,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:32:06,297] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31940,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:32:06,298] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31942,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:32:06,298] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31942,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:32:06,358] jepsen worker 83 - jepsen.util 7823 :info :write [27 45] :RequestCanceledException INFO [2019-12-17 12:32:06,364] jepsen worker 83 - jepsen.util 7913 :invoke :read [27 nil] INFO [2019-12-17 12:32:06,412] jepsen worker 1 - jepsen.util 7291 :info :cas [0 [41 8]] :RequestCanceledException INFO [2019-12-17 12:32:06,417] jepsen worker 2 - jepsen.util 7022 :info :write [0 30] :RequestCanceledException INFO [2019-12-17 12:32:06,421] jepsen worker 40 - jepsen.util 7960 :info :write [13 32] :RequestCanceledException INFO [2019-12-17 12:32:06,448] jepsen worker 77 - jepsen.util 7547 :info :write [25 9] :RequestCanceledException INFO [2019-12-17 12:32:06,448] jepsen worker 21 - jepsen.util 7041 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:32:06,453] jepsen worker 17 - jepsen.util 7487 :info :write [5 12] :RequestCanceledException INFO [2019-12-17 12:32:06,469] jepsen worker 21 - jepsen.util 7041 :invoke :read [7 nil] INFO [2019-12-17 12:32:06,477] jepsen worker 25 - jepsen.util 7855 :info :write [8 46] :RequestCanceledException INFO [2019-12-17 12:32:06,490] jepsen worker 26 - jepsen.util 7766 :info :write [8 23] :RequestCanceledException INFO [2019-12-17 12:32:06,500] jepsen worker 22 - jepsen.util 7492 :info :write [7 34] :RequestCanceledException INFO [2019-12-17 12:32:06,509] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31953,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:32:06,514] jepsen worker 7 - jepsen.util 7657 :info :write [2 14] :RequestCanceledException INFO [2019-12-17 12:32:06,515] jepsen worker 12 - jepsen.util 5862 :info :write [4 0] :RequestCanceledException INFO [2019-12-17 12:32:06,548] jepsen worker 63 - jepsen.util 4473 :info :write [21 47] :RequestCanceledException INFO [2019-12-17 12:32:06,562] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:32:06,565] jepsen worker 40 - jepsen.util 8050 :invoke :read [13 nil] INFO [2019-12-17 12:32:06,569] jepsen worker 1 - jepsen.util 7381 :invoke :read [0 nil] INFO [2019-12-17 12:32:06,571] jepsen worker 2 - jepsen.util 7112 :invoke :write [0 33] INFO [2019-12-17 12:32:06,585] jepsen worker 77 - jepsen.util 7637 :invoke :read [25 nil] INFO [2019-12-17 12:32:06,594] jepsen worker 22 - jepsen.util 7582 :invoke :read [7 nil] INFO [2019-12-17 12:32:06,594] jepsen worker 26 - jepsen.util 7856 :invoke :read [8 nil] INFO [2019-12-17 12:32:06,599] jepsen worker 17 - jepsen.util 7577 :invoke :cas [5 [43 24]] INFO [2019-12-17 12:32:06,601] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31953,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:32:06,601] jepsen worker 25 - jepsen.util 7945 :invoke :cas [8 [2 38]] INFO [2019-12-17 12:32:06,601] jepsen nemesis - couchbase.util Rebalance check count: 241 INFO [2019-12-17 12:32:06,613] jepsen nemesis - couchbase.util Rebalance stuck count: 222 INFO [2019-12-17 12:32:06,618] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31957,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:32:06,626] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31957,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:32:06,633] jepsen worker 24 - jepsen.util 6864 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:32:06,638] jepsen worker 63 - jepsen.util 4563 :invoke :write [21 26] INFO [2019-12-17 12:32:06,642] jepsen worker 24 - jepsen.util 6864 :invoke :cas [8 [8 37]] INFO [2019-12-17 12:32:06,642] jepsen worker 28 - jepsen.util 4168 :info :write [9 17] :RequestCanceledException INFO [2019-12-17 12:32:06,642] jepsen worker 53 - jepsen.util 5453 :info :write [17 10] :RequestCanceledException INFO [2019-12-17 12:32:06,644] jepsen worker 55 - jepsen.util 6535 :info :write [18 3] :RequestCanceledException INFO [2019-12-17 12:32:06,663] jepsen worker 51 - jepsen.util 5451 :info :write [17 2] :RequestCanceledException INFO [2019-12-17 12:32:06,663] jepsen worker 7 - jepsen.util 7747 :invoke :cas [2 [25 38]] INFO [2019-12-17 12:32:06,663] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31968,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:32:06,664] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31968,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:32:06,664] jepsen worker 42 - jepsen.util 8052 :info :write [14 34] :RequestCanceledException INFO [2019-12-17 12:32:06,665] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31970,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:32:06,668] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31970,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:32:06,674] jepsen worker 78 - jepsen.util 5838 :info :write [26 48] :RequestCanceledException INFO [2019-12-17 12:32:06,682] jepsen worker 18 - jepsen.util 5418 :info :write [6 25] :RequestCanceledException INFO [2019-12-17 12:32:06,700] jepsen worker 12 - jepsen.util 5952 :invoke :cas [4 [5 44]] INFO [2019-12-17 12:32:06,702] jepsen worker 9 - jepsen.util 6039 :info :write [3 28] :RequestCanceledException INFO [2019-12-17 12:32:06,702] jepsen worker 50 - jepsen.util 7700 :info :cas [16 [0 3]] :RequestCanceledException INFO [2019-12-17 12:32:06,706] jepsen worker 55 - jepsen.util 6625 :invoke :read [18 nil] INFO [2019-12-17 12:32:06,715] jepsen worker 28 - jepsen.util 4258 :invoke :cas [9 [30 46]] INFO [2019-12-17 12:32:06,724] jepsen worker 35 - jepsen.util 7505 :info :cas [11 [49 2]] :RequestCanceledException INFO [2019-12-17 12:32:06,730] jepsen worker 49 - jepsen.util 6619 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:32:06,732] jepsen worker 32 - jepsen.util 5612 :fail :read [10 nil] :RequestTimeoutException INFO [2019-12-17 12:32:06,733] jepsen worker 32 - jepsen.util 5612 :invoke :read [10 nil] INFO [2019-12-17 12:32:06,733] jepsen worker 49 - jepsen.util 6619 :invoke :write [16 44] INFO [2019-12-17 12:32:06,734] jepsen worker 45 - jepsen.util 5535 :info :cas [15 [37 38]] :RequestCanceledException INFO [2019-12-17 12:32:06,734] jepsen worker 23 - jepsen.util 7673 :info :cas [7 [20 45]] :RequestCanceledException INFO [2019-12-17 12:32:06,734] jepsen worker 51 - jepsen.util 5541 :invoke :read [17 nil] INFO [2019-12-17 12:32:06,735] jepsen worker 5 - jepsen.util 5135 :info :write [1 4] :RequestCanceledException INFO [2019-12-17 12:32:06,744] jepsen worker 20 - jepsen.util 5690 :fail :read [6 nil] :RequestTimeoutException INFO [2019-12-17 12:32:06,745] jepsen worker 42 - jepsen.util 8142 :invoke :read [14 nil] INFO [2019-12-17 12:32:06,745] jepsen worker 20 - jepsen.util 5690 :invoke :read [6 nil] INFO [2019-12-17 12:32:06,748] jepsen worker 18 - jepsen.util 5508 :invoke :write [6 11] INFO [2019-12-17 12:32:06,751] jepsen worker 59 - jepsen.util 7529 :info :write [19 11] :RequestCanceledException INFO [2019-12-17 12:32:06,751] jepsen worker 9 - jepsen.util 6129 :invoke :write [3 13] INFO [2019-12-17 12:32:06,751] jepsen worker 53 - jepsen.util 5543 :invoke :read [17 nil] INFO [2019-12-17 12:32:06,752] jepsen worker 79 - jepsen.util 5389 :info :write [26 27] :RequestCanceledException INFO [2019-12-17 12:32:06,766] jepsen worker 88 - jepsen.util 5758 :fail :cas [29 [28 40]] :ValueNotSwapFrom INFO [2019-12-17 12:32:06,767] jepsen worker 88 - jepsen.util 5758 :invoke :read [29 nil] INFO [2019-12-17 12:32:06,772] jepsen worker 78 - jepsen.util 5928 :invoke :read [26 nil] INFO [2019-12-17 12:32:06,795] jepsen worker 50 - jepsen.util 7790 :invoke :write [16 33] INFO [2019-12-17 12:32:06,800] jepsen worker 5 - jepsen.util 5225 :invoke :read [1 nil] INFO [2019-12-17 12:32:06,801] jepsen worker 35 - jepsen.util 7595 :invoke :cas [11 [15 17]] INFO [2019-12-17 12:32:06,809] jepsen worker 23 - jepsen.util 7763 :invoke :read [7 nil] INFO [2019-12-17 12:32:06,810] jepsen worker 45 - jepsen.util 5625 :invoke :write [15 35] INFO [2019-12-17 12:32:06,822] jepsen worker 79 - jepsen.util 5479 :invoke :write [26 36] INFO [2019-12-17 12:32:06,844] jepsen worker 59 - jepsen.util 7619 :invoke :read [19 nil] INFO [2019-12-17 12:32:06,862] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31990,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:32:06,862] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":31990,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:32:06,893] jepsen worker 72 - jepsen.util 5292 :fail :read [24 nil] :RequestTimeoutException INFO [2019-12-17 12:32:06,894] jepsen worker 30 - jepsen.util 5700 :info :write [10 17] :RequestCanceledException INFO [2019-12-17 12:32:06,898] jepsen worker 72 - jepsen.util 5292 :invoke :write [24 13] INFO [2019-12-17 12:32:06,916] jepsen worker 30 - jepsen.util 5790 :invoke :read [10 nil] INFO [2019-12-17 12:32:06,931] jepsen worker 54 - jepsen.util 5994 :info :write [18 11] :RequestCanceledException INFO [2019-12-17 12:32:06,982] jepsen worker 10 - jepsen.util 5500 :ok :write [3 46] INFO [2019-12-17 12:32:06,982] jepsen worker 10 - jepsen.util 5500 :invoke :read [3 nil] INFO [2019-12-17 12:32:06,988] jepsen worker 31 - jepsen.util 4981 :ok :write [10 34] INFO [2019-12-17 12:32:06,989] jepsen worker 74 - jepsen.util 6104 :ok :write [24 45] INFO [2019-12-17 12:32:06,993] jepsen worker 31 - jepsen.util 4981 :invoke :read [10 nil] INFO [2019-12-17 12:32:06,993] jepsen worker 74 - jepsen.util 6104 :invoke :cas [24 [14 45]] INFO [2019-12-17 12:32:06,994] jepsen worker 73 - jepsen.util 5113 :info :write [24 24] :RequestCanceledException INFO [2019-12-17 12:32:06,994] jepsen worker 46 - jepsen.util 5446 :ok :read [15 0] INFO [2019-12-17 12:32:06,995] jepsen worker 29 - jepsen.util 4349 :info :write [9 45] :RequestCanceledException INFO [2019-12-17 12:32:06,995] jepsen worker 46 - jepsen.util 5446 :invoke :cas [15 [31 29]] INFO [2019-12-17 12:32:06,996] jepsen worker 64 - jepsen.util 4204 :info :write [21 6] :RequestCanceledException INFO [2019-12-17 12:32:07,010] jepsen worker 37 - jepsen.util 5257 :info :write [12 44] :RequestCanceledException INFO [2019-12-17 12:32:07,052] jepsen worker 14 - jepsen.util 5594 :info :write [4 21] :RequestCanceledException INFO [2019-12-17 12:32:07,053] jepsen worker 54 - jepsen.util 6084 :invoke :read [18 nil] INFO [2019-12-17 12:32:07,065] jepsen worker 29 - jepsen.util 4439 :invoke :cas [9 [16 19]] INFO [2019-12-17 12:32:07,080] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32006,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:32:07,081] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32006,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:32:07,081] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32008,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:32:07,081] jepsen worker 73 - jepsen.util 5203 :invoke :cas [24 [1 44]] INFO [2019-12-17 12:32:07,085] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32008,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:32:07,088] jepsen worker 64 - jepsen.util 4294 :invoke :cas [21 [35 11]] INFO [2019-12-17 12:32:07,089] jepsen worker 37 - jepsen.util 5347 :invoke :read [12 nil] INFO [2019-12-17 12:32:07,094] jepsen worker 14 - jepsen.util 5684 :invoke :read [4 nil] INFO [2019-12-17 12:32:07,196] jepsen worker 87 - jepsen.util 5127 :ok :read [29 14] INFO [2019-12-17 12:32:07,196] jepsen worker 87 - jepsen.util 5127 :invoke :cas [29 [4 44]] INFO [2019-12-17 12:32:07,209] jepsen worker 28 - jepsen.util 4258 :fail :cas [9 [30 46]] :ValueNotSwapFrom INFO [2019-12-17 12:32:07,209] jepsen worker 28 - jepsen.util 4258 :invoke :cas [9 [46 34]] INFO [2019-12-17 12:32:07,221] jepsen worker 19 - jepsen.util 5599 :fail :read [6 nil] :RequestTimeoutException INFO [2019-12-17 12:32:07,221] jepsen worker 19 - jepsen.util 5599 :invoke :cas [6 [11 2]] INFO [2019-12-17 12:32:07,241] jepsen worker 20 - jepsen.util 5690 :ok :read [6 12] INFO [2019-12-17 12:32:07,241] jepsen worker 20 - jepsen.util 5690 :invoke :cas [6 [35 31]] INFO [2019-12-17 12:32:07,243] jepsen worker 80 - jepsen.util 5570 :fail :cas [26 [46 0]] :ValueNotSwapFrom INFO [2019-12-17 12:32:07,243] jepsen worker 80 - jepsen.util 5570 :invoke :write [26 28] INFO [2019-12-17 12:32:07,254] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32030,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:32:07,254] jepsen worker 11 - jepsen.util 5681 :info :cas [3 [26 31]] :RequestCanceledException INFO [2019-12-17 12:32:07,255] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32030,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:32:07,255] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32033,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:32:07,256] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32033,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:32:07,259] jepsen worker 70 - jepsen.util 6100 :ok :read [23 31] INFO [2019-12-17 12:32:07,260] jepsen worker 70 - jepsen.util 6100 :invoke :read [23 nil] INFO [2019-12-17 12:32:07,262] jepsen worker 11 - jepsen.util 5771 :invoke :read [3 nil] INFO [2019-12-17 12:32:07,268] jepsen worker 74 - jepsen.util 6104 :fail :cas [24 [14 45]] :ValueNotSwapFrom INFO [2019-12-17 12:32:07,268] jepsen worker 74 - jepsen.util 6104 :invoke :cas [24 [16 28]] INFO [2019-12-17 12:32:07,270] jepsen worker 32 - jepsen.util 5612 :ok :read [10 34] INFO [2019-12-17 12:32:07,270] jepsen worker 32 - jepsen.util 5612 :invoke :read [10 nil] INFO [2019-12-17 12:32:07,272] jepsen worker 10 - jepsen.util 5500 :ok :read [3 46] INFO [2019-12-17 12:32:07,272] jepsen worker 10 - jepsen.util 5500 :invoke :read [3 nil] INFO [2019-12-17 12:32:07,272] jepsen worker 46 - jepsen.util 5446 :fail :cas [15 [31 29]] :ValueNotSwapFrom INFO [2019-12-17 12:32:07,272] jepsen worker 46 - jepsen.util 5446 :invoke :read [15 nil] INFO [2019-12-17 12:32:07,283] jepsen worker 89 - jepsen.util 5399 :ok :read [29 14] INFO [2019-12-17 12:32:07,283] jepsen worker 89 - jepsen.util 5399 :invoke :cas [29 [20 33]] INFO [2019-12-17 12:32:07,285] jepsen worker 64 - jepsen.util 4294 :fail :cas [21 [35 11]] :ValueNotSwapFrom INFO [2019-12-17 12:32:07,285] jepsen worker 64 - jepsen.util 4294 :invoke :write [21 38] INFO [2019-12-17 12:32:07,290] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32020,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:32:07,291] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32020,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:32:07,291] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32021,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:32:07,291] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32021,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:32:07,293] jepsen worker 30 - jepsen.util 5790 :ok :read [10 34] INFO [2019-12-17 12:32:07,293] jepsen worker 30 - jepsen.util 5790 :invoke :write [10 11] INFO [2019-12-17 12:32:07,335] jepsen worker 19 - jepsen.util 5599 :fail :cas [6 [11 2]] :ValueNotSwapFrom INFO [2019-12-17 12:32:07,336] jepsen worker 19 - jepsen.util 5599 :invoke :cas [6 [27 47]] INFO [2019-12-17 12:32:07,349] jepsen worker 54 - jepsen.util 6084 :ok :read [18 32] INFO [2019-12-17 12:32:07,349] jepsen worker 54 - jepsen.util 6084 :invoke :write [18 3] INFO [2019-12-17 12:32:07,383] jepsen worker 74 - jepsen.util 6104 :fail :cas [24 [16 28]] :ValueNotSwapFrom INFO [2019-12-17 12:32:07,384] jepsen worker 74 - jepsen.util 6104 :invoke :cas [24 [47 38]] INFO [2019-12-17 12:32:07,390] jepsen worker 31 - jepsen.util 4981 :ok :read [10 34] INFO [2019-12-17 12:32:07,390] jepsen worker 31 - jepsen.util 4981 :invoke :read [10 nil] INFO [2019-12-17 12:32:07,391] jepsen worker 28 - jepsen.util 4258 :fail :cas [9 [46 34]] :ValueNotSwapFrom INFO [2019-12-17 12:32:07,392] jepsen worker 28 - jepsen.util 4258 :invoke :write [9 28] INFO [2019-12-17 12:32:07,431] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32052,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:32:07,433] jepsen worker 29 - jepsen.util 4439 :fail :cas [9 [16 19]] :ValueNotSwapFrom INFO [2019-12-17 12:32:07,437] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32052,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:32:07,438] jepsen worker 29 - jepsen.util 4439 :invoke :write [9 48] INFO [2019-12-17 12:32:07,438] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32053,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:32:07,438] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32053,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:32:07,444] jepsen worker 63 - jepsen.util 4563 :ok :write [21 26] INFO [2019-12-17 12:32:07,444] jepsen worker 63 - jepsen.util 4563 :invoke :write [21 9] INFO [2019-12-17 12:32:07,459] jepsen worker 74 - jepsen.util 6104 :fail :cas [24 [47 38]] :ValueNotSwapFrom INFO [2019-12-17 12:32:07,460] jepsen worker 74 - jepsen.util 6104 :invoke :read [24 nil] INFO [2019-12-17 12:32:07,536] jepsen worker 46 - jepsen.util 5446 :ok :read [15 0] INFO [2019-12-17 12:32:07,536] jepsen worker 46 - jepsen.util 5446 :invoke :cas [15 [10 21]] INFO [2019-12-17 12:32:07,588] jepsen worker 20 - jepsen.util 5690 :fail :cas [6 [35 31]] :ValueNotSwapFrom INFO [2019-12-17 12:32:07,588] jepsen worker 20 - jepsen.util 5690 :invoke :read [6 nil] INFO [2019-12-17 12:32:07,588] jepsen worker 69 - jepsen.util 5919 :info :write [23 16] :RequestCanceledException INFO [2019-12-17 12:32:07,589] jepsen worker 87 - jepsen.util 5127 :fail :cas [29 [4 44]] :ValueNotSwapFrom INFO [2019-12-17 12:32:07,589] jepsen worker 87 - jepsen.util 5127 :invoke :cas [29 [47 28]] INFO [2019-12-17 12:32:07,592] jepsen worker 69 - jepsen.util 6009 :invoke :cas [23 [36 30]] INFO [2019-12-17 12:32:07,606] jepsen worker 4 - jepsen.util 4864 :info :write [1 31] :RequestCanceledException INFO [2019-12-17 12:32:07,614] jepsen worker 4 - jepsen.util 4954 :invoke :cas [1 [11 18]] INFO [2019-12-17 12:32:07,615] jepsen worker 70 - jepsen.util 6100 :ok :read [23 31] INFO [2019-12-17 12:32:07,616] jepsen worker 70 - jepsen.util 6100 :invoke :read [23 nil] INFO [2019-12-17 12:32:07,624] jepsen worker 74 - jepsen.util 6104 :ok :read [24 45] INFO [2019-12-17 12:32:07,624] jepsen worker 74 - jepsen.util 6104 :invoke :cas [24 [36 14]] INFO [2019-12-17 12:32:07,638] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:32:07,638] jepsen nemesis - couchbase.util Rebalance check count: 242 INFO [2019-12-17 12:32:07,639] jepsen nemesis - couchbase.util Rebalance stuck count: 223 INFO [2019-12-17 12:32:07,662] jepsen worker 28 - jepsen.util 4258 :ok :write [9 28] INFO [2019-12-17 12:32:07,663] jepsen worker 28 - jepsen.util 4258 :invoke :cas [9 [16 46]] INFO [2019-12-17 12:32:07,681] jepsen worker 74 - jepsen.util 6104 :fail :cas [24 [36 14]] :ValueNotSwapFrom INFO [2019-12-17 12:32:07,681] jepsen worker 74 - jepsen.util 6104 :invoke :write [24 40] INFO [2019-12-17 12:32:07,693] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32070,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:32:07,693] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32070,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:32:07,693] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32072,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:32:07,694] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32072,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:32:07,705] jepsen worker 3 - jepsen.util 5583 :ok :write [1 0] INFO [2019-12-17 12:32:07,705] jepsen worker 3 - jepsen.util 5583 :invoke :read [1 nil] INFO [2019-12-17 12:32:07,756] jepsen worker 12 - jepsen.util 5952 :fail :cas [4 [5 44]] :ValueNotSwapFrom INFO [2019-12-17 12:32:07,756] jepsen worker 12 - jepsen.util 5952 :invoke :cas [4 [43 16]] INFO [2019-12-17 12:32:07,768] jepsen worker 46 - jepsen.util 5446 :fail :cas [15 [10 21]] :ValueNotSwapFrom INFO [2019-12-17 12:32:07,768] jepsen worker 46 - jepsen.util 5446 :invoke :cas [15 [20 29]] INFO [2019-12-17 12:32:07,796] jepsen worker 13 - jepsen.util 5503 :ok :write [4 21] INFO [2019-12-17 12:32:07,796] jepsen worker 13 - jepsen.util 5503 :invoke :write [4 5] INFO [2019-12-17 12:32:07,825] jepsen worker 28 - jepsen.util 4258 :fail :cas [9 [16 46]] :ValueNotSwapFrom INFO [2019-12-17 12:32:07,825] jepsen worker 28 - jepsen.util 4258 :invoke :cas [9 [35 42]] INFO [2019-12-17 12:32:07,826] jepsen worker 68 - jepsen.util 6998 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:32:07,826] jepsen worker 68 - jepsen.util 6998 :invoke :cas [22 [47 41]] INFO [2019-12-17 12:32:07,840] jepsen worker 46 - jepsen.util 5446 :fail :cas [15 [20 29]] :ValueNotSwapFrom INFO [2019-12-17 12:32:07,840] jepsen worker 46 - jepsen.util 5446 :invoke :read [15 nil] INFO [2019-12-17 12:32:07,863] jepsen worker 69 - jepsen.util 6009 :fail :cas [23 [36 30]] :ValueNotSwapFrom INFO [2019-12-17 12:32:07,863] jepsen worker 69 - jepsen.util 6009 :invoke :cas [23 [1 12]] INFO [2019-12-17 12:32:07,865] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32083,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:32:07,865] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32083,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:32:07,865] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32087,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:32:07,865] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32087,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:32:07,893] jepsen worker 66 - jepsen.util 6996 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:32:07,893] jepsen worker 66 - jepsen.util 6996 :invoke :read [22 nil] INFO [2019-12-17 12:32:07,894] jepsen worker 3 - jepsen.util 5583 :ok :read [1 0] INFO [2019-12-17 12:32:07,894] jepsen worker 3 - jepsen.util 5583 :invoke :cas [1 [45 46]] INFO [2019-12-17 12:32:07,906] jepsen worker 85 - jepsen.util 8365 :info :cas [28 [25 14]] :RequestCanceledException INFO [2019-12-17 12:32:07,907] jepsen worker 28 - jepsen.util 4258 :fail :cas [9 [35 42]] :ValueNotSwapFrom INFO [2019-12-17 12:32:07,907] jepsen worker 28 - jepsen.util 4258 :invoke :read [9 nil] INFO [2019-12-17 12:32:07,911] jepsen worker 85 - jepsen.util 8455 :invoke :write [28 10] INFO [2019-12-17 12:32:07,924] jepsen worker 48 - jepsen.util 6888 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:32:07,924] jepsen worker 48 - jepsen.util 6888 :invoke :read [16 nil] INFO [2019-12-17 12:32:07,929] jepsen worker 89 - jepsen.util 5399 :fail :cas [29 [20 33]] :ValueNotSwapFrom INFO [2019-12-17 12:32:07,929] jepsen worker 89 - jepsen.util 5399 :invoke :write [29 39] INFO [2019-12-17 12:32:07,941] jepsen worker 47 - jepsen.util 6167 :info :write [15 29] :RequestCanceledException INFO [2019-12-17 12:32:07,942] jepsen worker 47 - jepsen.util 6257 :invoke :write [15 32] INFO [2019-12-17 12:32:07,945] jepsen worker 45 - jepsen.util 5625 :ok :write [15 35] INFO [2019-12-17 12:32:07,945] jepsen worker 45 - jepsen.util 5625 :invoke :read [15 nil] INFO [2019-12-17 12:32:07,945] jepsen worker 9 - jepsen.util 6129 :ok :write [3 13] INFO [2019-12-17 12:32:07,945] jepsen worker 9 - jepsen.util 6129 :invoke :read [3 nil] INFO [2019-12-17 12:32:07,954] jepsen worker 28 - jepsen.util 4258 :ok :read [9 28] INFO [2019-12-17 12:32:07,954] jepsen worker 28 - jepsen.util 4258 :invoke :cas [9 [24 46]] INFO [2019-12-17 12:32:07,955] jepsen worker 82 - jepsen.util 8092 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:32:07,955] jepsen worker 75 - jepsen.util 7095 :info :cas [25 [27 38]] :RequestCanceledException INFO [2019-12-17 12:32:07,956] jepsen worker 82 - jepsen.util 8092 :invoke :cas [27 [11 40]] INFO [2019-12-17 12:32:07,956] jepsen worker 57 - jepsen.util 6987 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:32:07,956] jepsen worker 57 - jepsen.util 6987 :invoke :cas [19 [9 20]] INFO [2019-12-17 12:32:07,981] jepsen worker 75 - jepsen.util 7185 :invoke :read [25 nil] INFO [2019-12-17 12:32:07,990] jepsen worker 73 - jepsen.util 5203 :fail :cas [24 [1 44]] :ValueNotSwapFrom INFO [2019-12-17 12:32:07,990] jepsen worker 73 - jepsen.util 5203 :invoke :cas [24 [11 7]] INFO [2019-12-17 12:32:07,993] jepsen worker 16 - jepsen.util 8566 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:32:07,993] jepsen worker 16 - jepsen.util 8566 :invoke :read [5 nil] INFO [2019-12-17 12:32:08,007] jepsen worker 52 - jepsen.util 5902 :info :write [17 30] :RequestCanceledException INFO [2019-12-17 12:32:08,007] jepsen worker 43 - jepsen.util 7063 :info :cas [14 [11 19]] :RequestCanceledException INFO [2019-12-17 12:32:08,011] jepsen worker 52 - jepsen.util 5992 :invoke :cas [17 [34 22]] INFO [2019-12-17 12:32:08,012] jepsen worker 43 - jepsen.util 7153 :invoke :read [14 nil] INFO [2019-12-17 12:32:08,019] jepsen worker 46 - jepsen.util 5446 :ok :read [15 0] INFO [2019-12-17 12:32:08,019] jepsen worker 46 - jepsen.util 5446 :invoke :write [15 43] INFO [2019-12-17 12:32:08,035] jepsen worker 12 - jepsen.util 5952 :fail :cas [4 [43 16]] :ValueNotSwapFrom INFO [2019-12-17 12:32:08,036] jepsen worker 12 - jepsen.util 5952 :invoke :read [4 nil] INFO [2019-12-17 12:32:08,040] jepsen worker 44 - jepsen.util 8054 :info :cas [14 [1 39]] :RequestCanceledException INFO [2019-12-17 12:32:08,040] jepsen worker 27 - jepsen.util 5427 :info :cas [9 [4 7]] :RequestCanceledException INFO [2019-12-17 12:32:08,041] jepsen worker 60 - jepsen.util 8250 :info :cas [20 [2 37]] :RequestCanceledException INFO [2019-12-17 12:32:08,042] jepsen worker 27 - jepsen.util 5517 :invoke :write [9 39] INFO [2019-12-17 12:32:08,045] jepsen worker 44 - jepsen.util 8144 :invoke :cas [14 [21 13]] INFO [2019-12-17 12:32:08,047] jepsen worker 60 - jepsen.util 8340 :invoke :cas [20 [29 24]] INFO [2019-12-17 12:32:08,054] jepsen worker 8 - jepsen.util 7028 :info :cas [2 [12 46]] :RequestCanceledException INFO [2019-12-17 12:32:08,060] jepsen worker 8 - jepsen.util 7118 :invoke :read [2 nil] INFO [2019-12-17 12:32:08,063] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32105,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:32:08,063] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32105,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:32:08,091] jepsen worker 31 - jepsen.util 4981 :ok :read [10 34] INFO [2019-12-17 12:32:08,091] jepsen worker 31 - jepsen.util 4981 :invoke :read [10 nil] INFO [2019-12-17 12:32:08,091] jepsen worker 73 - jepsen.util 5203 :fail :cas [24 [11 7]] :ValueNotSwapFrom INFO [2019-12-17 12:32:08,091] jepsen worker 73 - jepsen.util 5203 :invoke :cas [24 [27 45]] INFO [2019-12-17 12:32:08,092] jepsen worker 34 - jepsen.util 6874 :info :cas [11 [16 38]] :RequestCanceledException INFO [2019-12-17 12:32:08,094] jepsen worker 34 - jepsen.util 6964 :invoke :write [11 42] INFO [2019-12-17 12:32:08,097] jepsen worker 69 - jepsen.util 6009 :fail :cas [23 [1 12]] :ValueNotSwapFrom INFO [2019-12-17 12:32:08,097] jepsen worker 69 - jepsen.util 6009 :invoke :read [23 nil] INFO [2019-12-17 12:32:08,097] jepsen worker 45 - jepsen.util 5625 :ok :read [15 35] INFO [2019-12-17 12:32:08,098] jepsen worker 45 - jepsen.util 5625 :invoke :read [15 nil] INFO [2019-12-17 12:32:08,103] jepsen worker 28 - jepsen.util 4258 :fail :cas [9 [24 46]] :ValueNotSwapFrom INFO [2019-12-17 12:32:08,104] jepsen worker 28 - jepsen.util 4258 :invoke :write [9 26] INFO [2019-12-17 12:32:08,125] jepsen worker 56 - jepsen.util 5996 :info :write [18 35] :RequestCanceledException INFO [2019-12-17 12:32:08,125] jepsen worker 0 - jepsen.util 7920 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:32:08,125] jepsen worker 33 - jepsen.util 7503 :info :cas [11 [6 30]] :RequestCanceledException INFO [2019-12-17 12:32:08,126] jepsen worker 0 - jepsen.util 7920 :invoke :read [0 nil] INFO [2019-12-17 12:32:08,139] jepsen worker 56 - jepsen.util 6086 :invoke :write [18 36] INFO [2019-12-17 12:32:08,139] jepsen worker 71 - jepsen.util 6281 :info :write [23 2] :RequestCanceledException INFO [2019-12-17 12:32:08,139] jepsen worker 33 - jepsen.util 7593 :invoke :read [11 nil] INFO [2019-12-17 12:32:08,160] jepsen worker 71 - jepsen.util 6371 :invoke :cas [23 [1 46]] INFO [2019-12-17 12:32:08,187] jepsen worker 29 - jepsen.util 4439 :ok :write [9 48] INFO [2019-12-17 12:32:08,187] jepsen worker 29 - jepsen.util 4439 :invoke :read [9 nil] INFO [2019-12-17 12:32:08,212] jepsen worker 76 - jepsen.util 7546 :info :write [25 16] :RequestCanceledException INFO [2019-12-17 12:32:08,214] jepsen worker 41 - jepsen.util 8231 :info :write [13 40] :RequestCanceledException INFO [2019-12-17 12:32:08,228] jepsen worker 81 - jepsen.util 7551 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:32:08,230] jepsen worker 27 - jepsen.util 5517 :ok :write [9 39] INFO [2019-12-17 12:32:08,230] jepsen worker 27 - jepsen.util 5517 :invoke :read [9 nil] INFO [2019-12-17 12:32:08,233] jepsen worker 81 - jepsen.util 7551 :invoke :read [27 nil] INFO [2019-12-17 12:32:08,239] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32135,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:32:08,246] jepsen worker 15 - jepsen.util 6495 :info :write [5 7] :RequestCanceledException INFO [2019-12-17 12:32:08,246] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32135,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:32:08,246] jepsen worker 36 - jepsen.util 5886 :info :write [12 28] :RequestCanceledException INFO [2019-12-17 12:32:08,247] jepsen worker 67 - jepsen.util 7627 :info :write [22 21] :RequestCanceledException INFO [2019-12-17 12:32:08,247] jepsen worker 58 - jepsen.util 7888 :info :cas [19 [43 36]] :RequestCanceledException INFO [2019-12-17 12:32:08,248] jepsen worker 39 - jepsen.util 7149 :info :write [13 47] :RequestCanceledException INFO [2019-12-17 12:32:08,248] jepsen worker 76 - jepsen.util 7636 :invoke :read [25 nil] INFO [2019-12-17 12:32:08,249] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32138,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:32:08,249] jepsen worker 41 - jepsen.util 8321 :invoke :cas [13 [16 47]] INFO [2019-12-17 12:32:08,249] jepsen worker 6 - jepsen.util 7926 :info :write [2 3] :RequestCanceledException INFO [2019-12-17 12:32:08,249] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32138,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:32:08,256] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32139,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:32:08,259] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32139,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:32:08,261] jepsen worker 61 - jepsen.util 8521 :info :write [20 6] :RequestCanceledException INFO [2019-12-17 12:32:08,262] jepsen worker 65 - jepsen.util 4745 :fail :read [21 nil] :RequestTimeoutException INFO [2019-12-17 12:32:08,262] jepsen worker 65 - jepsen.util 4745 :invoke :read [21 nil] INFO [2019-12-17 12:32:08,263] jepsen worker 36 - jepsen.util 5976 :invoke :cas [12 [42 20]] INFO [2019-12-17 12:32:08,273] jepsen worker 15 - jepsen.util 6585 :invoke :write [5 42] INFO [2019-12-17 12:32:08,280] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32131,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:32:08,286] jepsen worker 39 - jepsen.util 7239 :invoke :cas [13 [44 15]] INFO [2019-12-17 12:32:08,287] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32131,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:32:08,288] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32133,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:32:08,288] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32133,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:32:08,295] jepsen worker 84 - jepsen.util 7014 :info :write [28 27] :RequestCanceledException INFO [2019-12-17 12:32:08,296] jepsen worker 58 - jepsen.util 7978 :invoke :cas [19 [49 37]] INFO [2019-12-17 12:32:08,302] jepsen worker 61 - jepsen.util 8611 :invoke :write [20 36] INFO [2019-12-17 12:32:08,308] jepsen worker 6 - jepsen.util 8016 :invoke :read [2 nil] INFO [2019-12-17 12:32:08,326] jepsen worker 67 - jepsen.util 7717 :invoke :write [22 37] INFO [2019-12-17 12:32:08,327] jepsen worker 86 - jepsen.util 7736 :info :cas [28 [8 29]] :RequestCanceledException INFO [2019-12-17 12:32:08,329] jepsen worker 38 - jepsen.util 5618 :info :write [12 18] :RequestCanceledException INFO [2019-12-17 12:32:08,335] jepsen worker 30 - jepsen.util 5790 :ok :write [10 11] INFO [2019-12-17 12:32:08,336] jepsen worker 30 - jepsen.util 5790 :invoke :read [10 nil] INFO [2019-12-17 12:32:08,341] jepsen worker 62 - jepsen.util 7172 :info :cas [20 [13 44]] :RequestCanceledException INFO [2019-12-17 12:32:08,348] jepsen worker 18 - jepsen.util 5508 :ok :write [6 11] INFO [2019-12-17 12:32:08,349] jepsen worker 18 - jepsen.util 5508 :invoke :read [6 nil] INFO [2019-12-17 12:32:08,405] jepsen worker 83 - jepsen.util 7913 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:32:08,420] jepsen worker 83 - jepsen.util 7913 :invoke :cas [27 [15 2]] INFO [2019-12-17 12:32:08,454] jepsen worker 84 - jepsen.util 7104 :invoke :read [28 nil] INFO [2019-12-17 12:32:08,468] jepsen worker 62 - jepsen.util 7262 :invoke :read [20 nil] INFO [2019-12-17 12:32:08,486] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32153,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:32:08,488] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32153,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:32:08,503] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32157,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:32:08,510] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32157,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:32:08,513] jepsen worker 86 - jepsen.util 7826 :invoke :cas [28 [11 27]] INFO [2019-12-17 12:32:08,531] jepsen worker 21 - jepsen.util 7041 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:32:08,532] jepsen worker 38 - jepsen.util 5708 :invoke :read [12 nil] INFO [2019-12-17 12:32:08,535] jepsen worker 21 - jepsen.util 7041 :invoke :write [7 42] INFO [2019-12-17 12:32:08,619] jepsen worker 74 - jepsen.util 6104 :ok :write [24 40] INFO [2019-12-17 12:32:08,620] jepsen worker 74 - jepsen.util 6104 :invoke :cas [24 [23 16]] INFO [2019-12-17 12:32:08,627] jepsen worker 1 - jepsen.util 7381 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:32:08,627] jepsen worker 22 - jepsen.util 7582 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:32:08,627] jepsen worker 25 - jepsen.util 7945 :info :cas [8 [2 38]] :RequestCanceledException INFO [2019-12-17 12:32:08,627] jepsen worker 1 - jepsen.util 7381 :invoke :read [0 nil] INFO [2019-12-17 12:32:08,627] jepsen worker 22 - jepsen.util 7582 :invoke :cas [7 [47 1]] INFO [2019-12-17 12:32:08,649] jepsen worker 25 - jepsen.util 8035 :invoke :read [8 nil] INFO [2019-12-17 12:32:08,656] jepsen worker 77 - jepsen.util 7637 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:32:08,657] jepsen worker 77 - jepsen.util 7637 :invoke :cas [25 [16 49]] INFO [2019-12-17 12:32:08,691] jepsen worker 26 - jepsen.util 7856 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:32:08,692] jepsen worker 26 - jepsen.util 7856 :invoke :cas [8 [29 36]] INFO [2019-12-17 12:32:08,696] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:32:08,697] jepsen nemesis - couchbase.util Rebalance check count: 243 INFO [2019-12-17 12:32:08,697] jepsen nemesis - couchbase.util Rebalance stuck count: 224 INFO [2019-12-17 12:32:08,708] jepsen worker 40 - jepsen.util 8050 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:32:08,709] jepsen worker 40 - jepsen.util 8050 :invoke :write [13 7] INFO [2019-12-17 12:32:08,709] jepsen worker 17 - jepsen.util 7577 :info :cas [5 [43 24]] :RequestCanceledException INFO [2019-12-17 12:32:08,726] jepsen worker 24 - jepsen.util 6864 :info :cas [8 [8 37]] :RequestCanceledException INFO [2019-12-17 12:32:08,727] jepsen worker 17 - jepsen.util 7667 :invoke :read [5 nil] INFO [2019-12-17 12:32:08,729] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32165,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:32:08,730] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32165,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:32:08,730] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32170,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:32:08,731] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32170,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:32:08,736] jepsen worker 24 - jepsen.util 6954 :invoke :read [8 nil] INFO [2019-12-17 12:32:08,758] jepsen worker 7 - jepsen.util 7747 :info :cas [2 [25 38]] :RequestCanceledException INFO [2019-12-17 12:32:08,786] jepsen worker 7 - jepsen.util 7837 :invoke :cas [2 [9 44]] INFO [2019-12-17 12:32:08,797] jepsen worker 46 - jepsen.util 5446 :ok :write [15 43] INFO [2019-12-17 12:32:08,799] jepsen worker 46 - jepsen.util 5446 :invoke :write [15 28] INFO [2019-12-17 12:32:08,818] jepsen worker 78 - jepsen.util 5928 :fail :read [26 nil] :RequestTimeoutException INFO [2019-12-17 12:32:08,820] jepsen worker 78 - jepsen.util 5928 :invoke :write [26 16] INFO [2019-12-17 12:32:08,829] jepsen worker 23 - jepsen.util 7763 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:32:08,830] jepsen worker 23 - jepsen.util 7763 :invoke :read [7 nil] INFO [2019-12-17 12:32:08,832] jepsen worker 55 - jepsen.util 6625 :ok :read [18 3] INFO [2019-12-17 12:32:08,839] jepsen worker 55 - jepsen.util 6625 :invoke :write [18 20] INFO [2019-12-17 12:32:08,847] jepsen worker 42 - jepsen.util 8142 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:32:08,847] jepsen worker 42 - jepsen.util 8142 :invoke :read [14 nil] INFO [2019-12-17 12:32:08,859] jepsen worker 35 - jepsen.util 7595 :info :cas [11 [15 17]] :RequestCanceledException INFO [2019-12-17 12:32:08,884] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32188,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:32:08,886] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32188,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:32:08,886] jepsen worker 35 - jepsen.util 7685 :invoke :cas [11 [13 15]] INFO [2019-12-17 12:32:08,886] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32189,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:32:08,887] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32189,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:32:08,928] jepsen worker 59 - jepsen.util 7619 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:32:08,928] jepsen worker 59 - jepsen.util 7619 :invoke :write [19 34] INFO [2019-12-17 12:32:08,937] jepsen worker 5 - jepsen.util 5225 :ok :read [1 0] INFO [2019-12-17 12:32:08,942] jepsen worker 2 - jepsen.util 7112 :info :write [0 33] :RequestCanceledException INFO [2019-12-17 12:32:08,942] jepsen worker 5 - jepsen.util 5225 :invoke :write [1 12] INFO [2019-12-17 12:32:08,948] jepsen worker 27 - jepsen.util 5517 :ok :read [9 39] INFO [2019-12-17 12:32:08,948] jepsen worker 51 - jepsen.util 5541 :ok :read [17 10] INFO [2019-12-17 12:32:08,949] jepsen worker 27 - jepsen.util 5517 :invoke :read [9 nil] INFO [2019-12-17 12:32:08,949] jepsen worker 51 - jepsen.util 5541 :invoke :cas [17 [45 15]] INFO [2019-12-17 12:32:08,951] jepsen worker 3 - jepsen.util 5583 :fail :cas [1 [45 46]] :ValueNotSwapFrom INFO [2019-12-17 12:32:08,951] jepsen worker 3 - jepsen.util 5583 :invoke :read [1 nil] INFO [2019-12-17 12:32:08,958] jepsen worker 2 - jepsen.util 7202 :invoke :read [0 nil] INFO [2019-12-17 12:32:08,969] jepsen worker 53 - jepsen.util 5543 :ok :read [17 10] INFO [2019-12-17 12:32:08,970] jepsen worker 53 - jepsen.util 5543 :invoke :cas [17 [4 2]] INFO [2019-12-17 12:32:08,970] jepsen worker 29 - jepsen.util 4439 :ok :read [9 48] INFO [2019-12-17 12:32:08,970] jepsen worker 29 - jepsen.util 4439 :invoke :read [9 nil] INFO [2019-12-17 12:32:08,975] jepsen worker 65 - jepsen.util 4745 :ok :read [21 26] INFO [2019-12-17 12:32:08,976] jepsen worker 65 - jepsen.util 4745 :invoke :write [21 3] INFO [2019-12-17 12:32:08,977] jepsen worker 14 - jepsen.util 5684 :ok :read [4 21] INFO [2019-12-17 12:32:08,978] jepsen worker 14 - jepsen.util 5684 :invoke :cas [4 [44 41]] INFO [2019-12-17 12:32:08,978] jepsen worker 9 - jepsen.util 6129 :ok :read [3 13] INFO [2019-12-17 12:32:08,978] jepsen worker 9 - jepsen.util 6129 :invoke :cas [3 [49 32]] INFO [2019-12-17 12:32:08,996] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32203,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:32:08,996] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32203,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:32:08,996] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32206,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:32:08,997] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32206,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:32:09,015] jepsen worker 30 - jepsen.util 5790 :ok :read [10 11] INFO [2019-12-17 12:32:09,015] jepsen worker 30 - jepsen.util 5790 :invoke :cas [10 [11 8]] INFO [2019-12-17 12:32:09,017] jepsen worker 87 - jepsen.util 5127 :fail :cas [29 [47 28]] :ValueNotSwapFrom INFO [2019-12-17 12:32:09,017] jepsen worker 87 - jepsen.util 5127 :invoke :read [29 nil] INFO [2019-12-17 12:32:09,019] jepsen worker 49 - jepsen.util 6619 :info :write [16 44] :RequestCanceledException INFO [2019-12-17 12:32:09,020] jepsen worker 74 - jepsen.util 6104 :fail :cas [24 [23 16]] :ValueNotSwapFrom INFO [2019-12-17 12:32:09,021] jepsen worker 74 - jepsen.util 6104 :invoke :read [24 nil] INFO [2019-12-17 12:32:09,024] jepsen worker 31 - jepsen.util 4981 :ok :read [10 34] INFO [2019-12-17 12:32:09,024] jepsen worker 31 - jepsen.util 4981 :invoke :read [10 nil] INFO [2019-12-17 12:32:09,027] jepsen worker 4 - jepsen.util 4954 :fail :cas [1 [11 18]] :ValueNotSwapFrom INFO [2019-12-17 12:32:09,028] jepsen worker 4 - jepsen.util 4954 :invoke :write [1 8] INFO [2019-12-17 12:32:09,028] jepsen worker 12 - jepsen.util 5952 :ok :read [4 21] INFO [2019-12-17 12:32:09,028] jepsen worker 49 - jepsen.util 6709 :invoke :read [16 nil] INFO [2019-12-17 12:32:09,028] jepsen worker 12 - jepsen.util 5952 :invoke :read [4 nil] INFO [2019-12-17 12:32:09,031] jepsen worker 18 - jepsen.util 5508 :ok :read [6 11] INFO [2019-12-17 12:32:09,031] jepsen worker 18 - jepsen.util 5508 :invoke :read [6 nil] INFO [2019-12-17 12:32:09,035] jepsen worker 88 - jepsen.util 5758 :ok :read [29 14] INFO [2019-12-17 12:32:09,035] jepsen worker 88 - jepsen.util 5758 :invoke :read [29 nil] INFO [2019-12-17 12:32:09,040] jepsen worker 38 - jepsen.util 5708 :ok :read [12 42] INFO [2019-12-17 12:32:09,040] jepsen worker 38 - jepsen.util 5708 :invoke :cas [12 [1 19]] INFO [2019-12-17 12:32:09,044] jepsen worker 71 - jepsen.util 6371 :fail :cas [23 [1 46]] :ValueNotSwapFrom INFO [2019-12-17 12:32:09,044] jepsen worker 71 - jepsen.util 6371 :invoke :cas [23 [2 29]] INFO [2019-12-17 12:32:09,053] jepsen worker 70 - jepsen.util 6100 :ok :read [23 31] INFO [2019-12-17 12:32:09,054] jepsen worker 70 - jepsen.util 6100 :invoke :read [23 nil] INFO [2019-12-17 12:32:09,059] jepsen worker 69 - jepsen.util 6009 :ok :read [23 31] INFO [2019-12-17 12:32:09,059] jepsen worker 69 - jepsen.util 6009 :invoke :cas [23 [40 35]] INFO [2019-12-17 12:32:09,111] jepsen worker 79 - jepsen.util 5479 :info :write [26 36] :RequestCanceledException INFO [2019-12-17 12:32:09,115] jepsen worker 37 - jepsen.util 5347 :fail :read [12 nil] :RequestTimeoutException INFO [2019-12-17 12:32:09,115] jepsen worker 37 - jepsen.util 5347 :invoke :read [12 nil] INFO [2019-12-17 12:32:09,117] jepsen worker 79 - jepsen.util 5569 :invoke :read [26 nil] INFO [2019-12-17 12:32:09,148] jepsen worker 29 - jepsen.util 4439 :ok :read [9 39] INFO [2019-12-17 12:32:09,148] jepsen worker 29 - jepsen.util 4439 :invoke :write [9 30] INFO [2019-12-17 12:32:09,166] jepsen worker 51 - jepsen.util 5541 :fail :cas [17 [45 15]] :ValueNotSwapFrom INFO [2019-12-17 12:32:09,167] jepsen worker 51 - jepsen.util 5541 :invoke :write [17 9] INFO [2019-12-17 12:32:09,184] jepsen worker 27 - jepsen.util 5517 :ok :read [9 39] INFO [2019-12-17 12:32:09,185] jepsen worker 27 - jepsen.util 5517 :invoke :cas [9 [16 10]] INFO [2019-12-17 12:32:09,195] jepsen worker 72 - jepsen.util 5292 :info :write [24 13] :RequestCanceledException INFO [2019-12-17 12:32:09,200] jepsen worker 72 - jepsen.util 5382 :invoke :cas [24 [49 22]] INFO [2019-12-17 12:32:09,203] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32231,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:32:09,204] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32231,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:32:09,204] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32232,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:32:09,204] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32232,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:32:09,204] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32233,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:32:09,205] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32233,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:32:09,205] jepsen worker 74 - jepsen.util 6104 :ok :read [24 40] INFO [2019-12-17 12:32:09,205] jepsen worker 74 - jepsen.util 6104 :invoke :write [24 41] INFO [2019-12-17 12:32:09,214] jepsen worker 9 - jepsen.util 6129 :fail :cas [3 [49 32]] :ValueNotSwapFrom INFO [2019-12-17 12:32:09,215] jepsen worker 9 - jepsen.util 6129 :invoke :cas [3 [1 19]] INFO [2019-12-17 12:32:09,225] jepsen worker 50 - jepsen.util 7790 :info :write [16 33] :RequestCanceledException INFO [2019-12-17 12:32:09,229] jepsen worker 50 - jepsen.util 7880 :invoke :read [16 nil] INFO [2019-12-17 12:32:09,232] jepsen worker 31 - jepsen.util 4981 :ok :read [10 11] INFO [2019-12-17 12:32:09,232] jepsen worker 31 - jepsen.util 4981 :invoke :write [10 6] INFO [2019-12-17 12:32:09,250] jepsen worker 88 - jepsen.util 5758 :ok :read [29 14] INFO [2019-12-17 12:32:09,250] jepsen worker 88 - jepsen.util 5758 :invoke :cas [29 [43 30]] INFO [2019-12-17 12:32:09,253] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32239,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:32:09,253] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32239,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:32:09,253] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32242,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:32:09,253] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32242,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:32:09,257] jepsen worker 27 - jepsen.util 5517 :fail :cas [9 [16 10]] :ValueNotSwapFrom INFO [2019-12-17 12:32:09,257] jepsen worker 27 - jepsen.util 5517 :invoke :cas [9 [8 43]] INFO [2019-12-17 12:32:09,262] jepsen worker 14 - jepsen.util 5684 :fail :cas [4 [44 41]] :ValueNotSwapFrom INFO [2019-12-17 12:32:09,262] jepsen worker 14 - jepsen.util 5684 :invoke :write [4 25] INFO [2019-12-17 12:32:09,288] jepsen worker 53 - jepsen.util 5543 :fail :cas [17 [4 2]] :ValueNotSwapFrom INFO [2019-12-17 12:32:09,289] jepsen worker 53 - jepsen.util 5543 :invoke :read [17 nil] INFO [2019-12-17 12:32:09,294] jepsen worker 65 - jepsen.util 4745 :ok :write [21 3] INFO [2019-12-17 12:32:09,294] jepsen worker 65 - jepsen.util 4745 :invoke :write [21 10] INFO [2019-12-17 12:32:09,323] jepsen worker 32 - jepsen.util 5612 :fail :read [10 nil] :RequestTimeoutException INFO [2019-12-17 12:32:09,323] jepsen worker 32 - jepsen.util 5612 :invoke :write [10 3] INFO [2019-12-17 12:32:09,325] jepsen worker 64 - jepsen.util 4294 :info :write [21 38] :RequestCanceledException INFO [2019-12-17 12:32:09,329] jepsen worker 64 - jepsen.util 4384 :invoke :write [21 48] INFO [2019-12-17 12:32:09,333] jepsen worker 27 - jepsen.util 5517 :fail :cas [9 [8 43]] :ValueNotSwapFrom INFO [2019-12-17 12:32:09,333] jepsen worker 27 - jepsen.util 5517 :invoke :cas [9 [7 46]] INFO [2019-12-17 12:32:09,353] jepsen worker 11 - jepsen.util 5771 :fail :read [3 nil] :RequestTimeoutException INFO [2019-12-17 12:32:09,353] jepsen worker 80 - jepsen.util 5570 :info :write [26 28] :RequestCanceledException INFO [2019-12-17 12:32:09,353] jepsen worker 10 - jepsen.util 5500 :fail :read [3 nil] :RequestTimeoutException INFO [2019-12-17 12:32:09,353] jepsen worker 11 - jepsen.util 5771 :invoke :cas [3 [21 31]] INFO [2019-12-17 12:32:09,353] jepsen worker 10 - jepsen.util 5500 :invoke :read [3 nil] INFO [2019-12-17 12:32:09,354] jepsen worker 18 - jepsen.util 5508 :ok :read [6 11] INFO [2019-12-17 12:32:09,354] jepsen worker 18 - jepsen.util 5508 :invoke :cas [6 [17 14]] INFO [2019-12-17 12:32:09,357] jepsen worker 80 - jepsen.util 5660 :invoke :cas [26 [37 42]] INFO [2019-12-17 12:32:09,377] jepsen worker 27 - jepsen.util 5517 :fail :cas [9 [7 46]] :ValueNotSwapFrom INFO [2019-12-17 12:32:09,377] jepsen worker 27 - jepsen.util 5517 :invoke :cas [9 [28 8]] INFO [2019-12-17 12:32:09,426] jepsen worker 27 - jepsen.util 5517 :fail :cas [9 [28 8]] :ValueNotSwapFrom INFO [2019-12-17 12:32:09,426] jepsen worker 27 - jepsen.util 5517 :invoke :read [9 nil] INFO [2019-12-17 12:32:09,426] jepsen worker 19 - jepsen.util 5599 :info :cas [6 [27 47]] :RequestCanceledException INFO [2019-12-17 12:32:09,429] jepsen worker 19 - jepsen.util 5689 :invoke :cas [6 [5 24]] INFO [2019-12-17 12:32:09,451] jepsen worker 46 - jepsen.util 5446 :ok :write [15 28] INFO [2019-12-17 12:32:09,451] jepsen worker 46 - jepsen.util 5446 :invoke :cas [15 [18 39]] INFO [2019-12-17 12:32:09,456] jepsen worker 54 - jepsen.util 6084 :info :write [18 3] :RequestCanceledException INFO [2019-12-17 12:32:09,459] jepsen worker 54 - jepsen.util 6174 :invoke :cas [18 [40 42]] INFO [2019-12-17 12:32:09,462] jepsen worker 38 - jepsen.util 5708 :fail :cas [12 [1 19]] :ValueNotSwapFrom INFO [2019-12-17 12:32:09,462] jepsen worker 38 - jepsen.util 5708 :invoke :read [12 nil] INFO [2019-12-17 12:32:09,472] jepsen worker 65 - jepsen.util 4745 :ok :write [21 10] INFO [2019-12-17 12:32:09,473] jepsen worker 65 - jepsen.util 4745 :invoke :cas [21 [39 5]] INFO [2019-12-17 12:32:09,480] jepsen worker 13 - jepsen.util 5503 :ok :write [4 5] INFO [2019-12-17 12:32:09,480] jepsen worker 13 - jepsen.util 5503 :invoke :read [4 nil] INFO [2019-12-17 12:32:09,493] jepsen worker 27 - jepsen.util 5517 :ok :read [9 39] INFO [2019-12-17 12:32:09,493] jepsen worker 27 - jepsen.util 5517 :invoke :write [9 35] INFO [2019-12-17 12:32:09,508] jepsen worker 5 - jepsen.util 5225 :ok :write [1 12] INFO [2019-12-17 12:32:09,508] jepsen worker 5 - jepsen.util 5225 :invoke :cas [1 [41 25]] INFO [2019-12-17 12:32:09,513] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32265,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:32:09,514] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32265,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:32:09,523] jepsen worker 29 - jepsen.util 4439 :ok :write [9 30] INFO [2019-12-17 12:32:09,523] jepsen worker 29 - jepsen.util 4439 :invoke :read [9 nil] INFO [2019-12-17 12:32:09,540] jepsen worker 63 - jepsen.util 4563 :info :write [21 9] :RequestCanceledException INFO [2019-12-17 12:32:09,543] jepsen worker 63 - jepsen.util 4653 :invoke :cas [21 [35 31]] INFO [2019-12-17 12:32:09,561] jepsen worker 53 - jepsen.util 5543 :ok :read [17 10] INFO [2019-12-17 12:32:09,561] jepsen worker 53 - jepsen.util 5543 :invoke :read [17 nil] INFO [2019-12-17 12:32:09,567] jepsen worker 65 - jepsen.util 4745 :fail :cas [21 [39 5]] :ValueNotSwapFrom INFO [2019-12-17 12:32:09,567] jepsen worker 65 - jepsen.util 4745 :invoke :read [21 nil] INFO [2019-12-17 12:32:09,572] jepsen worker 88 - jepsen.util 5758 :fail :cas [29 [43 30]] :ValueNotSwapFrom INFO [2019-12-17 12:32:09,573] jepsen worker 88 - jepsen.util 5758 :invoke :write [29 3] INFO [2019-12-17 12:32:09,622] jepsen worker 80 - jepsen.util 5660 :fail :cas [26 [37 42]] :ValueNotSwapFrom INFO [2019-12-17 12:32:09,622] jepsen worker 80 - jepsen.util 5660 :invoke :read [26 nil] INFO [2019-12-17 12:32:09,623] jepsen worker 64 - jepsen.util 4384 :ok :write [21 48] INFO [2019-12-17 12:32:09,623] jepsen worker 64 - jepsen.util 4384 :invoke :read [21 nil] INFO [2019-12-17 12:32:09,625] jepsen worker 29 - jepsen.util 4439 :ok :read [9 30] INFO [2019-12-17 12:32:09,625] jepsen worker 29 - jepsen.util 4439 :invoke :read [9 nil] INFO [2019-12-17 12:32:09,639] jepsen worker 20 - jepsen.util 5690 :fail :read [6 nil] :RequestTimeoutException INFO [2019-12-17 12:32:09,639] jepsen worker 20 - jepsen.util 5690 :invoke :write [6 4] INFO [2019-12-17 12:32:09,643] jepsen worker 63 - jepsen.util 4653 :fail :cas [21 [35 31]] :ValueNotSwapFrom INFO [2019-12-17 12:32:09,644] jepsen worker 63 - jepsen.util 4653 :invoke :cas [21 [15 32]] INFO [2019-12-17 12:32:09,646] jepsen worker 65 - jepsen.util 4745 :ok :read [21 10] INFO [2019-12-17 12:32:09,646] jepsen worker 65 - jepsen.util 4745 :invoke :cas [21 [42 30]] INFO [2019-12-17 12:32:09,651] jepsen worker 27 - jepsen.util 5517 :ok :write [9 35] INFO [2019-12-17 12:32:09,652] jepsen worker 27 - jepsen.util 5517 :invoke :write [9 36] INFO [2019-12-17 12:32:09,658] jepsen worker 38 - jepsen.util 5708 :ok :read [12 42] INFO [2019-12-17 12:32:09,658] jepsen worker 38 - jepsen.util 5708 :invoke :read [12 nil] INFO [2019-12-17 12:32:09,686] jepsen worker 65 - jepsen.util 4745 :fail :cas [21 [42 30]] :ValueNotSwapFrom INFO [2019-12-17 12:32:09,686] jepsen worker 65 - jepsen.util 4745 :invoke :read [21 nil] INFO [2019-12-17 12:32:09,704] jepsen worker 29 - jepsen.util 4439 :ok :read [9 30] INFO [2019-12-17 12:32:09,704] jepsen worker 29 - jepsen.util 4439 :invoke :read [9 nil] INFO [2019-12-17 12:32:09,705] jepsen worker 64 - jepsen.util 4384 :ok :read [21 48] INFO [2019-12-17 12:32:09,706] jepsen worker 64 - jepsen.util 4384 :invoke :write [21 11] INFO [2019-12-17 12:32:09,721] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32288,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:32:09,721] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32288,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:32:09,730] jepsen worker 5 - jepsen.util 5225 :fail :cas [1 [41 25]] :ValueNotSwapFrom INFO [2019-12-17 12:32:09,731] jepsen worker 5 - jepsen.util 5225 :invoke :cas [1 [10 48]] INFO [2019-12-17 12:32:09,743] jepsen worker 63 - jepsen.util 4653 :fail :cas [21 [15 32]] :ValueNotSwapFrom INFO [2019-12-17 12:32:09,743] jepsen worker 63 - jepsen.util 4653 :invoke :read [21 nil] INFO [2019-12-17 12:32:09,801] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:32:09,801] jepsen nemesis - couchbase.util Rebalance check count: 244 INFO [2019-12-17 12:32:09,801] jepsen nemesis - couchbase.util Rebalance stuck count: 225 INFO [2019-12-17 12:32:09,805] jepsen worker 88 - jepsen.util 5758 :ok :write [29 3] INFO [2019-12-17 12:32:09,805] jepsen worker 88 - jepsen.util 5758 :invoke :cas [29 [41 18]] INFO [2019-12-17 12:32:09,805] jepsen worker 89 - jepsen.util 5399 :ok :write [29 39] INFO [2019-12-17 12:32:09,805] jepsen worker 89 - jepsen.util 5399 :invoke :write [29 46] INFO [2019-12-17 12:32:09,839] jepsen worker 78 - jepsen.util 5928 :ok :write [26 16] INFO [2019-12-17 12:32:09,839] jepsen worker 78 - jepsen.util 5928 :invoke :cas [26 [24 46]] INFO [2019-12-17 12:32:09,844] jepsen worker 5 - jepsen.util 5225 :fail :cas [1 [10 48]] :ValueNotSwapFrom INFO [2019-12-17 12:32:09,844] jepsen worker 5 - jepsen.util 5225 :invoke :write [1 41] INFO [2019-12-17 12:32:09,868] jepsen worker 65 - jepsen.util 4745 :ok :read [21 48] INFO [2019-12-17 12:32:09,868] jepsen worker 65 - jepsen.util 4745 :invoke :read [21 nil] INFO [2019-12-17 12:32:09,869] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32305,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:32:09,869] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32305,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:32:09,870] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32309,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:32:09,870] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32309,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:32:09,888] jepsen worker 29 - jepsen.util 4439 :ok :read [9 35] INFO [2019-12-17 12:32:09,889] jepsen worker 29 - jepsen.util 4439 :invoke :write [9 38] INFO [2019-12-17 12:32:09,918] jepsen worker 68 - jepsen.util 6998 :info :cas [22 [47 41]] :RequestCanceledException INFO [2019-12-17 12:32:09,923] jepsen worker 68 - jepsen.util 7088 :invoke :read [22 nil] INFO [2019-12-17 12:32:09,925] jepsen worker 56 - jepsen.util 6086 :ok :write [18 36] INFO [2019-12-17 12:32:09,925] jepsen worker 56 - jepsen.util 6086 :invoke :read [18 nil] INFO [2019-12-17 12:32:09,954] jepsen worker 80 - jepsen.util 5660 :ok :read [26 16] INFO [2019-12-17 12:32:09,954] jepsen worker 80 - jepsen.util 5660 :invoke :write [26 47] INFO [2019-12-17 12:32:09,954] jepsen worker 65 - jepsen.util 4745 :ok :read [21 48] INFO [2019-12-17 12:32:09,954] jepsen worker 65 - jepsen.util 4745 :invoke :read [21 nil] INFO [2019-12-17 12:32:09,975] jepsen worker 63 - jepsen.util 4653 :ok :read [21 48] INFO [2019-12-17 12:32:09,975] jepsen worker 63 - jepsen.util 4653 :invoke :write [21 43] INFO [2019-12-17 12:32:09,977] jepsen worker 27 - jepsen.util 5517 :ok :write [9 36] INFO [2019-12-17 12:32:09,978] jepsen worker 27 - jepsen.util 5517 :invoke :read [9 nil] INFO [2019-12-17 12:32:09,984] jepsen worker 64 - jepsen.util 4384 :ok :write [21 11] INFO [2019-12-17 12:32:09,984] jepsen worker 64 - jepsen.util 4384 :invoke :read [21 nil] INFO [2019-12-17 12:32:09,988] jepsen worker 66 - jepsen.util 6996 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:32:09,989] jepsen worker 66 - jepsen.util 6996 :invoke :cas [22 [5 46]] INFO [2019-12-17 12:32:10,006] jepsen worker 52 - jepsen.util 5992 :fail :cas [17 [34 22]] :ValueNotSwapFrom INFO [2019-12-17 12:32:10,007] jepsen worker 52 - jepsen.util 5992 :invoke :write [17 42] INFO [2019-12-17 12:32:10,007] jepsen worker 71 - jepsen.util 6371 :fail :cas [23 [2 29]] :ValueNotSwapFrom INFO [2019-12-17 12:32:10,008] jepsen worker 71 - jepsen.util 6371 :invoke :read [23 nil] INFO [2019-12-17 12:32:10,010] jepsen worker 85 - jepsen.util 8455 :info :write [28 10] :RequestCanceledException INFO [2019-12-17 12:32:10,014] jepsen worker 85 - jepsen.util 8545 :invoke :read [28 nil] INFO [2019-12-17 12:32:10,017] jepsen worker 43 - jepsen.util 7153 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:32:10,017] jepsen worker 48 - jepsen.util 6888 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:32:10,017] jepsen worker 43 - jepsen.util 7153 :invoke :read [14 nil] INFO [2019-12-17 12:32:10,017] jepsen worker 48 - jepsen.util 6888 :invoke :read [16 nil] INFO [2019-12-17 12:32:10,017] jepsen worker 3 - jepsen.util 5583 :ok :read [1 12] INFO [2019-12-17 12:32:10,017] jepsen worker 3 - jepsen.util 5583 :invoke :read [1 nil] INFO [2019-12-17 12:32:10,026] jepsen worker 47 - jepsen.util 6257 :info :write [15 32] :RequestCanceledException INFO [2019-12-17 12:32:10,027] jepsen worker 37 - jepsen.util 5347 :ok :read [12 42] INFO [2019-12-17 12:32:10,027] jepsen worker 37 - jepsen.util 5347 :invoke :cas [12 [5 13]] INFO [2019-12-17 12:32:10,032] jepsen worker 47 - jepsen.util 6347 :invoke :cas [15 [28 4]] INFO [2019-12-17 12:32:10,040] jepsen worker 75 - jepsen.util 7185 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:32:10,040] jepsen worker 75 - jepsen.util 7185 :invoke :cas [25 [24 24]] INFO [2019-12-17 12:32:10,056] jepsen worker 74 - jepsen.util 6104 :ok :write [24 41] INFO [2019-12-17 12:32:10,057] jepsen worker 74 - jepsen.util 6104 :invoke :write [24 12] INFO [2019-12-17 12:32:10,058] jepsen worker 82 - jepsen.util 8092 :info :cas [27 [11 40]] :RequestCanceledException INFO [2019-12-17 12:32:10,063] jepsen worker 12 - jepsen.util 5952 :ok :read [4 5] INFO [2019-12-17 12:32:10,063] jepsen worker 12 - jepsen.util 5952 :invoke :read [4 nil] INFO [2019-12-17 12:32:10,069] jepsen worker 30 - jepsen.util 5790 :ok :cas [10 [11 8]] INFO [2019-12-17 12:32:10,069] jepsen worker 30 - jepsen.util 5790 :invoke :read [10 nil] INFO [2019-12-17 12:32:10,084] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32320,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:32:10,085] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32320,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:32:10,085] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32323,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:32:10,085] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32323,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:32:10,085] jepsen worker 57 - jepsen.util 6987 :info :cas [19 [9 20]] :RequestCanceledException INFO [2019-12-17 12:32:10,085] jepsen worker 82 - jepsen.util 8182 :invoke :read [27 nil] INFO [2019-12-17 12:32:10,089] jepsen worker 57 - jepsen.util 7077 :invoke :cas [19 [14 42]] INFO [2019-12-17 12:32:10,091] jepsen worker 16 - jepsen.util 8566 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:32:10,091] jepsen worker 16 - jepsen.util 8566 :invoke :read [5 nil] INFO [2019-12-17 12:32:10,110] jepsen worker 44 - jepsen.util 8144 :info :cas [14 [21 13]] :RequestCanceledException INFO [2019-12-17 12:32:10,114] jepsen worker 44 - jepsen.util 8234 :invoke :cas [14 [22 24]] INFO [2019-12-17 12:32:10,124] jepsen worker 60 - jepsen.util 8340 :info :cas [20 [29 24]] :RequestCanceledException INFO [2019-12-17 12:32:10,129] jepsen worker 60 - jepsen.util 8430 :invoke :read [20 nil] INFO [2019-12-17 12:32:10,129] jepsen worker 64 - jepsen.util 4384 :ok :read [21 11] INFO [2019-12-17 12:32:10,129] jepsen worker 64 - jepsen.util 4384 :invoke :write [21 3] INFO [2019-12-17 12:32:10,132] jepsen worker 27 - jepsen.util 5517 :ok :read [9 36] INFO [2019-12-17 12:32:10,132] jepsen worker 27 - jepsen.util 5517 :invoke :cas [9 [9 0]] INFO [2019-12-17 12:32:10,137] jepsen worker 65 - jepsen.util 4745 :ok :read [21 48] INFO [2019-12-17 12:32:10,137] jepsen worker 65 - jepsen.util 4745 :invoke :read [21 nil] INFO [2019-12-17 12:32:10,140] jepsen worker 8 - jepsen.util 7118 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:32:10,140] jepsen worker 8 - jepsen.util 7118 :invoke :read [2 nil] INFO [2019-12-17 12:32:10,147] jepsen worker 19 - jepsen.util 5689 :fail :cas [6 [5 24]] :ValueNotSwapFrom INFO [2019-12-17 12:32:10,147] jepsen worker 19 - jepsen.util 5689 :invoke :write [6 12] INFO [2019-12-17 12:32:10,155] jepsen worker 34 - jepsen.util 6964 :info :write [11 42] :RequestCanceledException INFO [2019-12-17 12:32:10,161] jepsen worker 34 - jepsen.util 7054 :invoke :write [11 31] INFO [2019-12-17 12:32:10,178] jepsen worker 78 - jepsen.util 5928 :fail :cas [26 [24 46]] :ValueNotSwapFrom INFO [2019-12-17 12:32:10,178] jepsen worker 78 - jepsen.util 5928 :invoke :write [26 1] INFO [2019-12-17 12:32:10,193] jepsen worker 73 - jepsen.util 5203 :info :cas [24 [27 45]] :RequestCanceledException INFO [2019-12-17 12:32:10,193] jepsen worker 33 - jepsen.util 7593 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:32:10,193] jepsen worker 33 - jepsen.util 7593 :invoke :write [11 23] INFO [2019-12-17 12:32:10,194] jepsen worker 45 - jepsen.util 5625 :fail :read [15 nil] :RequestTimeoutException INFO [2019-12-17 12:32:10,194] jepsen worker 45 - jepsen.util 5625 :invoke :cas [15 [29 4]] INFO [2019-12-17 12:32:10,196] jepsen worker 73 - jepsen.util 5293 :invoke :cas [24 [6 3]] INFO [2019-12-17 12:32:10,206] jepsen worker 69 - jepsen.util 6009 :fail :cas [23 [40 35]] :ValueNotSwapFrom INFO [2019-12-17 12:32:10,206] jepsen worker 69 - jepsen.util 6009 :invoke :cas [23 [28 22]] INFO [2019-12-17 12:32:10,209] jepsen worker 28 - jepsen.util 4258 :info :write [9 26] :RequestCanceledException INFO [2019-12-17 12:32:10,214] jepsen worker 28 - jepsen.util 4348 :invoke :write [9 21] INFO [2019-12-17 12:32:10,226] jepsen worker 0 - jepsen.util 7920 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:32:10,226] jepsen worker 0 - jepsen.util 7920 :invoke :read [0 nil] INFO [2019-12-17 12:32:10,231] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32361,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:32:10,231] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32361,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:32:10,232] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32364,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:32:10,232] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32364,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:32:10,232] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32365,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:32:10,233] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32365,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:32:10,260] jepsen worker 70 - jepsen.util 6100 :ok :read [23 31] INFO [2019-12-17 12:32:10,260] jepsen worker 70 - jepsen.util 6100 :invoke :cas [23 [47 32]] INFO [2019-12-17 12:32:10,268] jepsen worker 53 - jepsen.util 5543 :ok :read [17 10] INFO [2019-12-17 12:32:10,268] jepsen worker 53 - jepsen.util 5543 :invoke :cas [17 [46 35]] INFO [2019-12-17 12:32:10,281] jepsen worker 27 - jepsen.util 5517 :fail :cas [9 [9 0]] :ValueNotSwapFrom INFO [2019-12-17 12:32:10,281] jepsen worker 27 - jepsen.util 5517 :invoke :read [9 nil] INFO [2019-12-17 12:32:10,295] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32350,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:32:10,296] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32350,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:32:10,296] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32352,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:32:10,296] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32352,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:32:10,298] jepsen worker 29 - jepsen.util 4439 :ok :write [9 38] INFO [2019-12-17 12:32:10,299] jepsen worker 45 - jepsen.util 5625 :fail :cas [15 [29 4]] :ValueNotSwapFrom INFO [2019-12-17 12:32:10,299] jepsen worker 29 - jepsen.util 4439 :invoke :write [9 35] INFO [2019-12-17 12:32:10,299] jepsen worker 45 - jepsen.util 5625 :invoke :read [15 nil] INFO [2019-12-17 12:32:10,311] jepsen worker 58 - jepsen.util 7978 :info :cas [19 [49 37]] :RequestCanceledException INFO [2019-12-17 12:32:10,316] jepsen worker 58 - jepsen.util 8068 :invoke :read [19 nil] INFO [2019-12-17 12:32:10,326] jepsen worker 81 - jepsen.util 7551 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:32:10,326] jepsen worker 79 - jepsen.util 5569 :ok :read [26 16] INFO [2019-12-17 12:32:10,326] jepsen worker 81 - jepsen.util 7551 :invoke :cas [27 [9 46]] INFO [2019-12-17 12:32:10,327] jepsen worker 79 - jepsen.util 5569 :invoke :cas [26 [11 47]] INFO [2019-12-17 12:32:10,327] jepsen worker 76 - jepsen.util 7636 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:32:10,327] jepsen worker 76 - jepsen.util 7636 :invoke :read [25 nil] INFO [2019-12-17 12:32:10,339] jepsen worker 41 - jepsen.util 8321 :info :cas [13 [16 47]] :RequestCanceledException INFO [2019-12-17 12:32:10,339] jepsen worker 6 - jepsen.util 8016 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:32:10,339] jepsen worker 6 - jepsen.util 8016 :invoke :write [2 2] INFO [2019-12-17 12:32:10,341] jepsen worker 41 - jepsen.util 8411 :invoke :read [13 nil] INFO [2019-12-17 12:32:10,345] jepsen worker 65 - jepsen.util 4745 :ok :read [21 11] INFO [2019-12-17 12:32:10,346] jepsen worker 65 - jepsen.util 4745 :invoke :write [21 20] INFO [2019-12-17 12:32:10,350] jepsen worker 27 - jepsen.util 5517 :ok :read [9 38] INFO [2019-12-17 12:32:10,350] jepsen worker 27 - jepsen.util 5517 :invoke :cas [9 [42 15]] INFO [2019-12-17 12:32:10,355] jepsen worker 87 - jepsen.util 5127 :ok :read [29 3] INFO [2019-12-17 12:32:10,355] jepsen worker 87 - jepsen.util 5127 :invoke :cas [29 [25 29]] INFO [2019-12-17 12:32:10,387] jepsen worker 64 - jepsen.util 4384 :ok :write [21 3] INFO [2019-12-17 12:32:10,387] jepsen worker 64 - jepsen.util 4384 :invoke :read [21 nil] INFO [2019-12-17 12:32:10,390] jepsen worker 39 - jepsen.util 7239 :info :cas [13 [44 15]] :RequestCanceledException INFO [2019-12-17 12:32:10,392] jepsen worker 39 - jepsen.util 7329 :invoke :write [13 7] INFO [2019-12-17 12:32:10,444] jepsen worker 29 - jepsen.util 4439 :ok :write [9 35] INFO [2019-12-17 12:32:10,444] jepsen worker 29 - jepsen.util 4439 :invoke :read [9 nil] INFO [2019-12-17 12:32:10,476] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32385,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:32:10,493] jepsen worker 83 - jepsen.util 7913 :info :cas [27 [15 2]] :RequestCanceledException INFO [2019-12-17 12:32:10,493] jepsen worker 62 - jepsen.util 7262 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:32:10,500] jepsen worker 71 - jepsen.util 6371 :ok :read [23 31] INFO [2019-12-17 12:32:10,513] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32385,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:32:10,513] jepsen worker 65 - jepsen.util 4745 :ok :write [21 20] INFO [2019-12-17 12:32:10,513] jepsen worker 62 - jepsen.util 7262 :invoke :write [20 43] INFO [2019-12-17 12:32:10,513] jepsen worker 71 - jepsen.util 6371 :invoke :cas [23 [4 31]] INFO [2019-12-17 12:32:10,514] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32386,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:32:10,515] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32386,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:32:10,515] jepsen worker 65 - jepsen.util 4745 :invoke :cas [21 [41 2]] INFO [2019-12-17 12:32:10,516] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32388,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:32:10,517] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32388,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:32:10,526] jepsen worker 84 - jepsen.util 7104 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:32:10,527] jepsen worker 84 - jepsen.util 7104 :invoke :cas [28 [10 6]] INFO [2019-12-17 12:32:10,527] jepsen worker 83 - jepsen.util 8003 :invoke :cas [27 [7 20]] INFO [2019-12-17 12:32:10,531] jepsen worker 29 - jepsen.util 4439 :ok :read [9 35] INFO [2019-12-17 12:32:10,531] jepsen worker 29 - jepsen.util 4439 :invoke :read [9 nil] INFO [2019-12-17 12:32:10,532] jepsen worker 27 - jepsen.util 5517 :fail :cas [9 [42 15]] :ValueNotSwapFrom INFO [2019-12-17 12:32:10,533] jepsen worker 27 - jepsen.util 5517 :invoke :cas [9 [9 25]] INFO [2019-12-17 12:32:10,548] jepsen worker 45 - jepsen.util 5625 :ok :read [15 28] INFO [2019-12-17 12:32:10,549] jepsen worker 45 - jepsen.util 5625 :invoke :read [15 nil] INFO [2019-12-17 12:32:10,612] jepsen worker 86 - jepsen.util 7826 :info :cas [28 [11 27]] :RequestCanceledException INFO [2019-12-17 12:32:10,636] jepsen worker 86 - jepsen.util 7916 :invoke :cas [28 [23 40]] INFO [2019-12-17 12:32:10,640] jepsen worker 64 - jepsen.util 4384 :ok :read [21 3] INFO [2019-12-17 12:32:10,640] jepsen worker 64 - jepsen.util 4384 :invoke :read [21 nil] INFO [2019-12-17 12:32:10,696] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32404,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:32:10,696] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32404,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:32:10,697] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32407,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:32:10,699] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32407,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:32:10,700] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32408,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:32:10,701] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32408,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:32:10,705] jepsen worker 54 - jepsen.util 6174 :fail :cas [18 [40 42]] :ValueNotSwapFrom INFO [2019-12-17 12:32:10,705] jepsen worker 54 - jepsen.util 6174 :invoke :read [18 nil] INFO [2019-12-17 12:32:10,715] jepsen worker 56 - jepsen.util 6086 :ok :read [18 36] INFO [2019-12-17 12:32:10,715] jepsen worker 56 - jepsen.util 6086 :invoke :cas [18 [21 49]] INFO [2019-12-17 12:32:10,720] jepsen worker 13 - jepsen.util 5503 :ok :read [4 5] INFO [2019-12-17 12:32:10,721] jepsen worker 13 - jepsen.util 5503 :invoke :read [4 nil] INFO [2019-12-17 12:32:10,721] jepsen worker 1 - jepsen.util 7381 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:32:10,722] jepsen worker 61 - jepsen.util 8611 :info :write [20 36] :RequestCanceledException INFO [2019-12-17 12:32:10,722] jepsen worker 1 - jepsen.util 7381 :invoke :read [0 nil] INFO [2019-12-17 12:32:10,722] jepsen worker 22 - jepsen.util 7582 :info :cas [7 [47 1]] :RequestCanceledException INFO [2019-12-17 12:32:10,722] jepsen worker 15 - jepsen.util 6585 :info :write [5 42] :RequestCanceledException INFO [2019-12-17 12:32:10,722] jepsen worker 21 - jepsen.util 7041 :info :write [7 42] :RequestCanceledException INFO [2019-12-17 12:32:10,729] jepsen worker 15 - jepsen.util 6675 :invoke :write [5 21] INFO [2019-12-17 12:32:10,729] jepsen worker 22 - jepsen.util 7672 :invoke :read [7 nil] INFO [2019-12-17 12:32:10,729] jepsen worker 61 - jepsen.util 8701 :invoke :cas [20 [24 45]] INFO [2019-12-17 12:32:10,730] jepsen worker 21 - jepsen.util 7131 :invoke :cas [7 [38 36]] INFO [2019-12-17 12:32:10,734] jepsen worker 74 - jepsen.util 6104 :ok :write [24 12] INFO [2019-12-17 12:32:10,735] jepsen worker 74 - jepsen.util 6104 :invoke :write [24 40] INFO [2019-12-17 12:32:10,737] jepsen worker 25 - jepsen.util 8035 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:32:10,738] jepsen worker 25 - jepsen.util 8035 :invoke :cas [8 [40 15]] INFO [2019-12-17 12:32:10,754] jepsen worker 77 - jepsen.util 7637 :info :cas [25 [16 49]] :RequestCanceledException INFO [2019-12-17 12:32:10,754] jepsen worker 67 - jepsen.util 7717 :info :write [22 37] :RequestCanceledException INFO [2019-12-17 12:32:10,758] jepsen worker 17 - jepsen.util 7667 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:32:10,758] jepsen worker 17 - jepsen.util 7667 :invoke :cas [5 [10 21]] INFO [2019-12-17 12:32:10,762] jepsen worker 77 - jepsen.util 7727 :invoke :write [25 3] INFO [2019-12-17 12:32:10,762] jepsen worker 67 - jepsen.util 7807 :invoke :cas [22 [13 23]] INFO [2019-12-17 12:32:10,762] jepsen worker 3 - jepsen.util 5583 :ok :read [1 12] INFO [2019-12-17 12:32:10,763] jepsen worker 29 - jepsen.util 4439 :ok :read [9 35] INFO [2019-12-17 12:32:10,763] jepsen worker 3 - jepsen.util 5583 :invoke :read [1 nil] INFO [2019-12-17 12:32:10,763] jepsen worker 29 - jepsen.util 4439 :invoke :read [9 nil] INFO [2019-12-17 12:32:10,770] jepsen worker 12 - jepsen.util 5952 :ok :read [4 5] INFO [2019-12-17 12:32:10,770] jepsen worker 12 - jepsen.util 5952 :invoke :read [4 nil] INFO [2019-12-17 12:32:10,778] jepsen worker 27 - jepsen.util 5517 :fail :cas [9 [9 25]] :ValueNotSwapFrom INFO [2019-12-17 12:32:10,778] jepsen worker 65 - jepsen.util 4745 :fail :cas [21 [41 2]] :ValueNotSwapFrom INFO [2019-12-17 12:32:10,778] jepsen worker 65 - jepsen.util 4745 :invoke :read [21 nil] INFO [2019-12-17 12:32:10,780] jepsen worker 45 - jepsen.util 5625 :ok :read [15 28] INFO [2019-12-17 12:32:10,780] jepsen worker 45 - jepsen.util 5625 :invoke :read [15 nil] INFO [2019-12-17 12:32:10,781] jepsen worker 37 - jepsen.util 5347 :fail :cas [12 [5 13]] :ValueNotSwapFrom INFO [2019-12-17 12:32:10,782] jepsen worker 37 - jepsen.util 5347 :invoke :write [12 19] INFO [2019-12-17 12:32:10,783] jepsen worker 27 - jepsen.util 5517 :invoke :write [9 21] INFO [2019-12-17 12:32:10,792] jepsen worker 26 - jepsen.util 7856 :info :cas [8 [29 36]] :RequestCanceledException INFO [2019-12-17 12:32:10,798] jepsen worker 26 - jepsen.util 7946 :invoke :read [8 nil] INFO [2019-12-17 12:32:10,806] jepsen worker 40 - jepsen.util 8050 :info :write [13 7] :RequestCanceledException INFO [2019-12-17 12:32:10,808] jepsen worker 64 - jepsen.util 4384 :ok :read [21 20] INFO [2019-12-17 12:32:10,808] jepsen worker 64 - jepsen.util 4384 :invoke :read [21 nil] INFO [2019-12-17 12:32:10,812] jepsen worker 40 - jepsen.util 8140 :invoke :read [13 nil] INFO [2019-12-17 12:32:10,821] jepsen worker 73 - jepsen.util 5293 :fail :cas [24 [6 3]] :ValueNotSwapFrom INFO [2019-12-17 12:32:10,821] jepsen worker 73 - jepsen.util 5293 :invoke :read [24 nil] INFO [2019-12-17 12:32:10,821] jepsen worker 69 - jepsen.util 6009 :fail :cas [23 [28 22]] :ValueNotSwapFrom INFO [2019-12-17 12:32:10,821] jepsen worker 69 - jepsen.util 6009 :invoke :read [23 nil] INFO [2019-12-17 12:32:10,824] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:32:10,825] jepsen nemesis - couchbase.util Rebalance check count: 245 INFO [2019-12-17 12:32:10,825] jepsen nemesis - couchbase.util Rebalance stuck count: 226 INFO [2019-12-17 12:32:10,826] jepsen worker 24 - jepsen.util 6954 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:32:10,826] jepsen worker 24 - jepsen.util 6954 :invoke :read [8 nil] INFO [2019-12-17 12:32:10,835] jepsen worker 29 - jepsen.util 4439 :ok :read [9 35] INFO [2019-12-17 12:32:10,835] jepsen worker 29 - jepsen.util 4439 :invoke :write [9 35] INFO [2019-12-17 12:32:10,880] jepsen worker 65 - jepsen.util 4745 :ok :read [21 20] INFO [2019-12-17 12:32:10,883] jepsen worker 65 - jepsen.util 4745 :invoke :read [21 nil] INFO [2019-12-17 12:32:10,893] jepsen worker 7 - jepsen.util 7837 :info :cas [2 [9 44]] :RequestCanceledException INFO [2019-12-17 12:32:10,897] jepsen worker 7 - jepsen.util 7927 :invoke :write [2 24] INFO [2019-12-17 12:32:10,911] jepsen worker 35 - jepsen.util 7685 :info :cas [11 [13 15]] :RequestCanceledException INFO [2019-12-17 12:32:10,916] jepsen worker 35 - jepsen.util 7775 :invoke :write [11 20] INFO [2019-12-17 12:32:10,917] jepsen worker 56 - jepsen.util 6086 :fail :cas [18 [21 49]] :ValueNotSwapFrom INFO [2019-12-17 12:32:10,917] jepsen worker 56 - jepsen.util 6086 :invoke :read [18 nil] INFO [2019-12-17 12:32:10,926] jepsen worker 23 - jepsen.util 7763 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:32:10,926] jepsen worker 23 - jepsen.util 7763 :invoke :read [7 nil] INFO [2019-12-17 12:32:10,929] jepsen worker 45 - jepsen.util 5625 :ok :read [15 28] INFO [2019-12-17 12:32:10,930] jepsen worker 45 - jepsen.util 5625 :invoke :write [15 46] INFO [2019-12-17 12:32:10,940] jepsen worker 42 - jepsen.util 8142 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:32:10,940] jepsen worker 42 - jepsen.util 8142 :invoke :read [14 nil] INFO [2019-12-17 12:32:10,953] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32439,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:32:10,953] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32439,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:32:10,953] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32441,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:32:10,953] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32441,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:32:10,963] jepsen worker 13 - jepsen.util 5503 :ok :read [4 5] INFO [2019-12-17 12:32:10,964] jepsen worker 13 - jepsen.util 5503 :invoke :write [4 31] INFO [2019-12-17 12:32:10,984] jepsen worker 53 - jepsen.util 5543 :fail :cas [17 [46 35]] :ValueNotSwapFrom INFO [2019-12-17 12:32:10,984] jepsen worker 53 - jepsen.util 5543 :invoke :cas [17 [8 37]] INFO [2019-12-17 12:32:11,027] jepsen worker 59 - jepsen.util 7619 :info :write [19 34] :RequestCanceledException INFO [2019-12-17 12:32:11,027] jepsen worker 2 - jepsen.util 7202 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:32:11,028] jepsen worker 2 - jepsen.util 7202 :invoke :cas [0 [28 29]] INFO [2019-12-17 12:32:11,038] jepsen worker 49 - jepsen.util 6709 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:32:11,039] jepsen worker 49 - jepsen.util 6709 :invoke :read [16 nil] INFO [2019-12-17 12:32:11,043] jepsen worker 59 - jepsen.util 7709 :invoke :write [19 43] INFO [2019-12-17 12:32:11,053] jepsen worker 70 - jepsen.util 6100 :fail :cas [23 [47 32]] :ValueNotSwapFrom INFO [2019-12-17 12:32:11,053] jepsen worker 70 - jepsen.util 6100 :invoke :cas [23 [29 48]] INFO [2019-12-17 12:32:11,068] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32450,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:32:11,068] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32450,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:32:11,068] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32451,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:32:11,068] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32451,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:32:11,069] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32455,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:32:11,069] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32455,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:32:11,078] jepsen worker 87 - jepsen.util 5127 :fail :cas [29 [25 29]] :ValueNotSwapFrom INFO [2019-12-17 12:32:11,079] jepsen worker 87 - jepsen.util 5127 :invoke :read [29 nil] INFO [2019-12-17 12:32:11,083] jepsen worker 38 - jepsen.util 5708 :ok :read [12 42] INFO [2019-12-17 12:32:11,083] jepsen worker 38 - jepsen.util 5708 :invoke :cas [12 [30 26]] INFO [2019-12-17 12:32:11,084] jepsen worker 64 - jepsen.util 4384 :ok :read [21 20] INFO [2019-12-17 12:32:11,084] jepsen worker 64 - jepsen.util 4384 :invoke :cas [21 [19 39]] INFO [2019-12-17 12:32:11,085] jepsen worker 3 - jepsen.util 5583 :ok :read [1 12] INFO [2019-12-17 12:32:11,086] jepsen worker 3 - jepsen.util 5583 :invoke :write [1 44] INFO [2019-12-17 12:32:11,093] jepsen worker 55 - jepsen.util 6625 :info :write [18 20] :RequestCanceledException INFO [2019-12-17 12:32:11,093] jepsen worker 4 - jepsen.util 4954 :info :write [1 8] :RequestCanceledException INFO [2019-12-17 12:32:11,098] jepsen worker 55 - jepsen.util 6715 :invoke :cas [18 [1 21]] INFO [2019-12-17 12:32:11,098] jepsen worker 4 - jepsen.util 5044 :invoke :write [1 30] INFO [2019-12-17 12:32:11,105] jepsen worker 29 - jepsen.util 4439 :ok :write [9 35] INFO [2019-12-17 12:32:11,105] jepsen worker 29 - jepsen.util 4439 :invoke :write [9 24] INFO [2019-12-17 12:32:11,125] jepsen worker 65 - jepsen.util 4745 :ok :read [21 20] INFO [2019-12-17 12:32:11,126] jepsen worker 65 - jepsen.util 4745 :invoke :cas [21 [36 18]] INFO [2019-12-17 12:32:11,129] jepsen worker 12 - jepsen.util 5952 :ok :read [4 5] INFO [2019-12-17 12:32:11,130] jepsen worker 12 - jepsen.util 5952 :invoke :read [4 nil] INFO [2019-12-17 12:32:11,132] jepsen worker 79 - jepsen.util 5569 :fail :cas [26 [11 47]] :ValueNotSwapFrom INFO [2019-12-17 12:32:11,132] jepsen worker 79 - jepsen.util 5569 :invoke :read [26 nil] INFO [2019-12-17 12:32:11,152] jepsen worker 69 - jepsen.util 6009 :ok :read [23 31] INFO [2019-12-17 12:32:11,153] jepsen worker 69 - jepsen.util 6009 :invoke :write [23 31] INFO [2019-12-17 12:32:11,165] jepsen worker 64 - jepsen.util 4384 :fail :cas [21 [19 39]] :ValueNotSwapFrom INFO [2019-12-17 12:32:11,165] jepsen worker 64 - jepsen.util 4384 :invoke :cas [21 [42 25]] INFO [2019-12-17 12:32:11,171] jepsen worker 65 - jepsen.util 4745 :fail :cas [21 [36 18]] :ValueNotSwapFrom INFO [2019-12-17 12:32:11,172] jepsen worker 65 - jepsen.util 4745 :invoke :cas [21 [9 9]] INFO [2019-12-17 12:32:11,182] jepsen worker 20 - jepsen.util 5690 :ok :write [6 4] INFO [2019-12-17 12:32:11,183] jepsen worker 20 - jepsen.util 5690 :invoke :write [6 25] INFO [2019-12-17 12:32:11,202] jepsen worker 47 - jepsen.util 6347 :ok :cas [15 [28 4]] INFO [2019-12-17 12:32:11,202] jepsen worker 47 - jepsen.util 6347 :invoke :read [15 nil] INFO [2019-12-17 12:32:11,227] jepsen worker 51 - jepsen.util 5541 :info :write [17 9] :RequestCanceledException INFO [2019-12-17 12:32:11,227] jepsen worker 72 - jepsen.util 5382 :info :cas [24 [49 22]] :RequestCanceledException INFO [2019-12-17 12:32:11,227] jepsen worker 9 - jepsen.util 6129 :info :cas [3 [1 19]] :RequestCanceledException INFO [2019-12-17 12:32:11,230] jepsen worker 72 - jepsen.util 5472 :invoke :write [24 39] INFO [2019-12-17 12:32:11,231] jepsen worker 51 - jepsen.util 5631 :invoke :cas [17 [28 16]] INFO [2019-12-17 12:32:11,233] jepsen worker 9 - jepsen.util 6219 :invoke :read [3 nil] INFO [2019-12-17 12:32:11,241] jepsen worker 65 - jepsen.util 4745 :fail :cas [21 [9 9]] :ValueNotSwapFrom INFO [2019-12-17 12:32:11,241] jepsen worker 65 - jepsen.util 4745 :invoke :write [21 11] INFO [2019-12-17 12:32:11,242] jepsen worker 64 - jepsen.util 4384 :fail :cas [21 [42 25]] :ValueNotSwapFrom INFO [2019-12-17 12:32:11,243] jepsen worker 64 - jepsen.util 4384 :invoke :read [21 nil] INFO [2019-12-17 12:32:11,285] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32476,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:32:11,288] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32476,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:32:11,293] jepsen worker 50 - jepsen.util 7880 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:32:11,294] jepsen worker 50 - jepsen.util 7880 :invoke :write [16 8] INFO [2019-12-17 12:32:11,312] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32485,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:32:11,312] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32485,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:32:11,313] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32487,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:32:11,313] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32487,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:32:11,319] jepsen worker 88 - jepsen.util 5758 :fail :cas [29 [41 18]] :ValueNotSwapFrom INFO [2019-12-17 12:32:11,320] jepsen worker 88 - jepsen.util 5758 :invoke :write [29 9] INFO [2019-12-17 12:32:11,328] jepsen worker 31 - jepsen.util 4981 :info :write [10 6] :RequestCanceledException INFO [2019-12-17 12:32:11,332] jepsen worker 29 - jepsen.util 4439 :ok :write [9 24] INFO [2019-12-17 12:32:11,333] jepsen worker 29 - jepsen.util 4439 :invoke :cas [9 [1 30]] INFO [2019-12-17 12:32:11,341] jepsen worker 14 - jepsen.util 5684 :info :write [4 25] :RequestCanceledException INFO [2019-12-17 12:32:11,345] jepsen worker 31 - jepsen.util 5071 :invoke :write [10 19] INFO [2019-12-17 12:32:11,351] jepsen worker 14 - jepsen.util 5774 :invoke :write [4 44] INFO [2019-12-17 12:32:11,359] jepsen worker 11 - jepsen.util 5771 :info :cas [3 [21 31]] :RequestCanceledException INFO [2019-12-17 12:32:11,360] jepsen worker 10 - jepsen.util 5500 :fail :read [3 nil] :RequestTimeoutException INFO [2019-12-17 12:32:11,360] jepsen worker 10 - jepsen.util 5500 :invoke :read [3 nil] INFO [2019-12-17 12:32:11,367] jepsen worker 11 - jepsen.util 5861 :invoke :read [3 nil] INFO [2019-12-17 12:32:11,426] jepsen worker 32 - jepsen.util 5612 :info :write [10 3] :RequestCanceledException INFO [2019-12-17 12:32:11,427] jepsen worker 56 - jepsen.util 6086 :ok :read [18 36] INFO [2019-12-17 12:32:11,427] jepsen worker 56 - jepsen.util 6086 :invoke :write [18 19] INFO [2019-12-17 12:32:11,436] jepsen worker 32 - jepsen.util 5702 :invoke :write [10 1] INFO [2019-12-17 12:32:11,443] jepsen worker 54 - jepsen.util 6174 :ok :read [18 36] INFO [2019-12-17 12:32:11,443] jepsen worker 54 - jepsen.util 6174 :invoke :cas [18 [1 3]] INFO [2019-12-17 12:32:11,453] jepsen worker 18 - jepsen.util 5508 :info :cas [6 [17 14]] :RequestCanceledException INFO [2019-12-17 12:32:11,455] jepsen worker 29 - jepsen.util 4439 :fail :cas [9 [1 30]] :ValueNotSwapFrom INFO [2019-12-17 12:32:11,456] jepsen worker 29 - jepsen.util 4439 :invoke :read [9 nil] INFO [2019-12-17 12:32:11,457] jepsen worker 18 - jepsen.util 5598 :invoke :cas [6 [36 40]] INFO [2019-12-17 12:32:11,458] jepsen worker 9 - jepsen.util 6219 :ok :read [3 13] INFO [2019-12-17 12:32:11,458] jepsen worker 9 - jepsen.util 6219 :invoke :write [3 11] INFO [2019-12-17 12:32:11,460] jepsen worker 80 - jepsen.util 5660 :ok :write [26 47] INFO [2019-12-17 12:32:11,461] jepsen worker 80 - jepsen.util 5660 :invoke :cas [26 [17 48]] INFO [2019-12-17 12:32:11,474] jepsen worker 64 - jepsen.util 4384 :ok :read [21 20] INFO [2019-12-17 12:32:11,474] jepsen worker 64 - jepsen.util 4384 :invoke :cas [21 [1 13]] INFO [2019-12-17 12:32:11,506] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32502,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:32:11,506] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32502,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:32:11,511] jepsen worker 46 - jepsen.util 5446 :info :cas [15 [18 39]] :RequestCanceledException INFO [2019-12-17 12:32:11,514] jepsen worker 46 - jepsen.util 5536 :invoke :write [15 46] INFO [2019-12-17 12:32:11,599] jepsen worker 29 - jepsen.util 4439 :ok :read [9 24] INFO [2019-12-17 12:32:11,599] jepsen worker 29 - jepsen.util 4439 :invoke :read [9 nil] INFO [2019-12-17 12:32:11,602] jepsen worker 18 - jepsen.util 5598 :fail :cas [6 [36 40]] :ValueNotSwapFrom INFO [2019-12-17 12:32:11,602] jepsen worker 18 - jepsen.util 5598 :invoke :write [6 6] INFO [2019-12-17 12:32:11,605] jepsen worker 38 - jepsen.util 5708 :fail :cas [12 [30 26]] :ValueNotSwapFrom INFO [2019-12-17 12:32:11,605] jepsen worker 38 - jepsen.util 5708 :invoke :write [12 39] INFO [2019-12-17 12:32:11,628] jepsen worker 70 - jepsen.util 6100 :fail :cas [23 [29 48]] :ValueNotSwapFrom INFO [2019-12-17 12:32:11,628] jepsen worker 70 - jepsen.util 6100 :invoke :read [23 nil] INFO [2019-12-17 12:32:11,640] jepsen worker 29 - jepsen.util 4439 :ok :read [9 24] INFO [2019-12-17 12:32:11,640] jepsen worker 29 - jepsen.util 4439 :invoke :read [9 nil] INFO [2019-12-17 12:32:11,680] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32523,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:32:11,680] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32523,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:32:11,680] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32525,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:32:11,681] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32525,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:32:11,691] jepsen worker 53 - jepsen.util 5543 :fail :cas [17 [8 37]] :ValueNotSwapFrom INFO [2019-12-17 12:32:11,692] jepsen worker 53 - jepsen.util 5543 :invoke :cas [17 [39 26]] INFO [2019-12-17 12:32:11,862] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:32:11,862] jepsen nemesis - couchbase.util Rebalance check count: 246 INFO [2019-12-17 12:32:11,862] jepsen nemesis - couchbase.util Rebalance stuck count: 227 INFO [2019-12-17 12:32:11,867] jepsen worker 55 - jepsen.util 6715 :fail :cas [18 [1 21]] :ValueNotSwapFrom INFO [2019-12-17 12:32:11,867] jepsen worker 55 - jepsen.util 6715 :invoke :read [18 nil] INFO [2019-12-17 12:32:11,871] jepsen worker 74 - jepsen.util 6104 :ok :write [24 40] INFO [2019-12-17 12:32:11,871] jepsen worker 74 - jepsen.util 6104 :invoke :cas [24 [46 26]] INFO [2019-12-17 12:32:11,879] jepsen worker 87 - jepsen.util 5127 :ok :read [29 3] INFO [2019-12-17 12:32:11,880] jepsen worker 87 - jepsen.util 5127 :invoke :write [29 11] INFO [2019-12-17 12:32:11,891] jepsen worker 20 - jepsen.util 5690 :ok :write [6 25] INFO [2019-12-17 12:32:11,891] jepsen worker 20 - jepsen.util 5690 :invoke :read [6 nil] INFO [2019-12-17 12:32:11,902] jepsen worker 79 - jepsen.util 5569 :ok :read [26 47] INFO [2019-12-17 12:32:11,902] jepsen worker 79 - jepsen.util 5569 :invoke :cas [26 [37 27]] INFO [2019-12-17 12:32:11,932] jepsen worker 74 - jepsen.util 6104 :fail :cas [24 [46 26]] :ValueNotSwapFrom INFO [2019-12-17 12:32:11,933] jepsen worker 74 - jepsen.util 6104 :invoke :cas [24 [33 1]] INFO [2019-12-17 12:32:11,941] jepsen worker 5 - jepsen.util 5225 :info :write [1 41] :RequestCanceledException INFO [2019-12-17 12:32:11,947] jepsen worker 5 - jepsen.util 5315 :invoke :read [1 nil] INFO [2019-12-17 12:32:11,953] jepsen worker 71 - jepsen.util 6371 :fail :cas [23 [4 31]] :ValueNotSwapFrom INFO [2019-12-17 12:32:11,954] jepsen worker 71 - jepsen.util 6371 :invoke :read [23 nil] INFO [2019-12-17 12:32:11,957] jepsen worker 11 - jepsen.util 5861 :ok :read [3 13] INFO [2019-12-17 12:32:11,957] jepsen worker 89 - jepsen.util 5399 :info :write [29 46] :RequestCanceledException INFO [2019-12-17 12:32:11,957] jepsen worker 11 - jepsen.util 5861 :invoke :cas [3 [40 18]] INFO [2019-12-17 12:32:11,962] jepsen worker 89 - jepsen.util 5489 :invoke :read [29 nil] INFO [2019-12-17 12:32:11,962] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32528,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:32:11,962] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32528,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:32:11,962] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32531,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:32:11,963] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32531,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:32:11,963] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32532,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:32:11,963] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32532,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:32:11,965] jepsen worker 20 - jepsen.util 5690 :ok :read [6 25] INFO [2019-12-17 12:32:11,966] jepsen worker 20 - jepsen.util 5690 :invoke :read [6 nil] INFO [2019-12-17 12:32:11,983] jepsen worker 74 - jepsen.util 6104 :fail :cas [24 [33 1]] :ValueNotSwapFrom INFO [2019-12-17 12:32:11,983] jepsen worker 74 - jepsen.util 6104 :invoke :read [24 nil] INFO [2019-12-17 12:32:11,984] jepsen worker 53 - jepsen.util 5543 :fail :cas [17 [39 26]] :ValueNotSwapFrom INFO [2019-12-17 12:32:11,985] jepsen worker 53 - jepsen.util 5543 :invoke :write [17 11] INFO [2019-12-17 12:32:11,993] jepsen worker 68 - jepsen.util 7088 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:32:11,993] jepsen worker 68 - jepsen.util 7088 :invoke :cas [22 [0 45]] INFO [2019-12-17 12:32:11,998] jepsen worker 11 - jepsen.util 5861 :fail :cas [3 [40 18]] :ValueNotSwapFrom INFO [2019-12-17 12:32:11,998] jepsen worker 11 - jepsen.util 5861 :invoke :write [3 5] INFO [2019-12-17 12:32:12,026] jepsen worker 63 - jepsen.util 4653 :info :write [21 43] :RequestCanceledException INFO [2019-12-17 12:32:12,027] jepsen worker 74 - jepsen.util 6104 :ok :read [24 40] INFO [2019-12-17 12:32:12,028] jepsen worker 74 - jepsen.util 6104 :invoke :write [24 28] INFO [2019-12-17 12:32:12,029] jepsen worker 20 - jepsen.util 5690 :ok :read [6 25] INFO [2019-12-17 12:32:12,029] jepsen worker 63 - jepsen.util 4743 :invoke :read [21 nil] INFO [2019-12-17 12:32:12,029] jepsen worker 20 - jepsen.util 5690 :invoke :write [6 46] INFO [2019-12-17 12:32:12,058] jepsen worker 52 - jepsen.util 5992 :info :write [17 42] :RequestCanceledException INFO [2019-12-17 12:32:12,062] jepsen worker 52 - jepsen.util 6082 :invoke :read [17 nil] INFO [2019-12-17 12:32:12,073] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32549,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:32:12,074] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32549,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:32:12,094] jepsen worker 66 - jepsen.util 6996 :info :cas [22 [5 46]] :RequestCanceledException INFO [2019-12-17 12:32:12,099] jepsen worker 66 - jepsen.util 7086 :invoke :write [22 8] INFO [2019-12-17 12:32:12,103] jepsen worker 12 - jepsen.util 5952 :ok :read [4 5] INFO [2019-12-17 12:32:12,103] jepsen worker 12 - jepsen.util 5952 :invoke :read [4 nil] INFO [2019-12-17 12:32:12,111] jepsen worker 85 - jepsen.util 8545 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:32:12,111] jepsen worker 85 - jepsen.util 8545 :invoke :cas [28 [24 1]] INFO [2019-12-17 12:32:12,114] jepsen worker 31 - jepsen.util 5071 :ok :write [10 19] INFO [2019-12-17 12:32:12,114] jepsen worker 31 - jepsen.util 5071 :invoke :cas [10 [40 17]] INFO [2019-12-17 12:32:12,127] jepsen worker 43 - jepsen.util 7153 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:32:12,128] jepsen worker 48 - jepsen.util 6888 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:32:12,128] jepsen worker 43 - jepsen.util 7153 :invoke :write [14 18] INFO [2019-12-17 12:32:12,129] jepsen worker 48 - jepsen.util 6888 :invoke :write [16 25] INFO [2019-12-17 12:32:12,129] jepsen worker 44 - jepsen.util 8234 :info :cas [14 [22 24]] :RequestCanceledException INFO [2019-12-17 12:32:12,131] jepsen worker 9 - jepsen.util 6219 :ok :write [3 11] INFO [2019-12-17 12:32:12,133] jepsen worker 9 - jepsen.util 6219 :invoke :read [3 nil] INFO [2019-12-17 12:32:12,143] jepsen worker 75 - jepsen.util 7185 :info :cas [25 [24 24]] :RequestCanceledException INFO [2019-12-17 12:32:12,147] jepsen worker 82 - jepsen.util 8182 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:32:12,149] jepsen worker 30 - jepsen.util 5790 :fail :read [10 nil] :RequestTimeoutException INFO [2019-12-17 12:32:12,149] jepsen worker 82 - jepsen.util 8182 :invoke :read [27 nil] INFO [2019-12-17 12:32:12,150] jepsen worker 30 - jepsen.util 5790 :invoke :cas [10 [10 38]] INFO [2019-12-17 12:32:12,161] jepsen worker 57 - jepsen.util 7077 :info :cas [19 [14 42]] :RequestCanceledException INFO [2019-12-17 12:32:12,163] jepsen worker 44 - jepsen.util 8324 :invoke :cas [14 [36 39]] INFO [2019-12-17 12:32:12,193] jepsen worker 75 - jepsen.util 7275 :invoke :cas [25 [24 31]] INFO [2019-12-17 12:32:12,196] jepsen worker 16 - jepsen.util 8566 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:32:12,197] jepsen worker 60 - jepsen.util 8430 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:32:12,198] jepsen worker 16 - jepsen.util 8566 :invoke :write [5 49] INFO [2019-12-17 12:32:12,198] jepsen worker 60 - jepsen.util 8430 :invoke :write [20 12] INFO [2019-12-17 12:32:12,200] jepsen worker 57 - jepsen.util 7167 :invoke :write [19 19] INFO [2019-12-17 12:32:12,211] jepsen worker 78 - jepsen.util 5928 :info :write [26 1] :RequestCanceledException INFO [2019-12-17 12:32:12,211] jepsen worker 34 - jepsen.util 7054 :info :write [11 31] :RequestCanceledException INFO [2019-12-17 12:32:12,241] jepsen worker 78 - jepsen.util 6018 :invoke :read [26 nil] INFO [2019-12-17 12:32:12,242] jepsen worker 8 - jepsen.util 7118 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:32:12,244] jepsen worker 19 - jepsen.util 5689 :info :write [6 12] :RequestCanceledException INFO [2019-12-17 12:32:12,245] jepsen worker 8 - jepsen.util 7118 :invoke :cas [2 [19 20]] INFO [2019-12-17 12:32:12,246] jepsen worker 28 - jepsen.util 4348 :info :write [9 21] :RequestCanceledException INFO [2019-12-17 12:32:12,247] jepsen worker 34 - jepsen.util 7144 :invoke :write [11 5] INFO [2019-12-17 12:32:12,258] jepsen worker 36 - jepsen.util 5976 :info :cas [12 [42 20]] :RequestCanceledException INFO [2019-12-17 12:32:12,260] jepsen worker 19 - jepsen.util 5779 :invoke :read [6 nil] INFO [2019-12-17 12:32:12,261] jepsen worker 28 - jepsen.util 4438 :invoke :cas [9 [41 16]] INFO [2019-12-17 12:32:12,272] jepsen worker 36 - jepsen.util 6066 :invoke :read [12 nil] INFO [2019-12-17 12:32:12,288] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32574,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:32:12,289] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32574,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:32:12,289] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32578,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:32:12,289] jepsen worker 33 - jepsen.util 7593 :info :write [11 23] :RequestCanceledException INFO [2019-12-17 12:32:12,289] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32578,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:32:12,295] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32567,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:32:12,295] jepsen worker 33 - jepsen.util 7683 :invoke :cas [11 [3 17]] INFO [2019-12-17 12:32:12,295] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32567,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:32:12,295] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32569,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:32:12,295] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32569,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:32:12,324] jepsen worker 9 - jepsen.util 6219 :ok :read [3 11] INFO [2019-12-17 12:32:12,324] jepsen worker 9 - jepsen.util 6219 :invoke :write [3 18] INFO [2019-12-17 12:32:12,327] jepsen worker 0 - jepsen.util 7920 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:32:12,327] jepsen worker 0 - jepsen.util 7920 :invoke :read [0 nil] INFO [2019-12-17 12:32:12,330] jepsen worker 31 - jepsen.util 5071 :fail :cas [10 [40 17]] :ValueNotSwapFrom INFO [2019-12-17 12:32:12,330] jepsen worker 31 - jepsen.util 5071 :invoke :cas [10 [15 16]] INFO [2019-12-17 12:32:12,337] jepsen worker 5 - jepsen.util 5315 :ok :read [1 12] INFO [2019-12-17 12:32:12,338] jepsen worker 5 - jepsen.util 5315 :invoke :read [1 nil] INFO [2019-12-17 12:32:12,340] jepsen worker 64 - jepsen.util 4384 :fail :cas [21 [1 13]] :ValueNotSwapFrom INFO [2019-12-17 12:32:12,340] jepsen worker 64 - jepsen.util 4384 :invoke :write [21 47] INFO [2019-12-17 12:32:12,352] jepsen worker 54 - jepsen.util 6174 :fail :cas [18 [1 3]] :ValueNotSwapFrom INFO [2019-12-17 12:32:12,353] jepsen worker 54 - jepsen.util 6174 :invoke :cas [18 [37 33]] INFO [2019-12-17 12:32:12,358] jepsen worker 58 - jepsen.util 8068 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:32:12,358] jepsen worker 58 - jepsen.util 8068 :invoke :write [19 17] INFO [2019-12-17 12:32:12,361] jepsen worker 74 - jepsen.util 6104 :ok :write [24 28] INFO [2019-12-17 12:32:12,361] jepsen worker 74 - jepsen.util 6104 :invoke :cas [24 [40 9]] INFO [2019-12-17 12:32:12,388] jepsen worker 71 - jepsen.util 6371 :ok :read [23 31] INFO [2019-12-17 12:32:12,389] jepsen worker 71 - jepsen.util 6371 :invoke :cas [23 [21 27]] INFO [2019-12-17 12:32:12,389] jepsen worker 19 - jepsen.util 5779 :ok :read [6 25] INFO [2019-12-17 12:32:12,389] jepsen worker 19 - jepsen.util 5779 :invoke :write [6 8] INFO [2019-12-17 12:32:12,391] jepsen worker 28 - jepsen.util 4438 :fail :cas [9 [41 16]] :ValueNotSwapFrom INFO [2019-12-17 12:32:12,392] jepsen worker 28 - jepsen.util 4438 :invoke :read [9 nil] INFO [2019-12-17 12:32:12,417] jepsen worker 31 - jepsen.util 5071 :fail :cas [10 [15 16]] :ValueNotSwapFrom INFO [2019-12-17 12:32:12,418] jepsen worker 31 - jepsen.util 5071 :invoke :cas [10 [49 24]] INFO [2019-12-17 12:32:12,424] jepsen worker 74 - jepsen.util 6104 :fail :cas [24 [40 9]] :ValueNotSwapFrom INFO [2019-12-17 12:32:12,425] jepsen worker 74 - jepsen.util 6104 :invoke :read [24 nil] INFO [2019-12-17 12:32:12,427] jepsen worker 41 - jepsen.util 8411 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:32:12,427] jepsen worker 81 - jepsen.util 7551 :info :cas [27 [9 46]] :RequestCanceledException INFO [2019-12-17 12:32:12,427] jepsen worker 76 - jepsen.util 7636 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:32:12,427] jepsen worker 41 - jepsen.util 8411 :invoke :cas [13 [40 8]] INFO [2019-12-17 12:32:12,427] jepsen worker 76 - jepsen.util 7636 :invoke :write [25 36] INFO [2019-12-17 12:32:12,432] jepsen worker 81 - jepsen.util 7641 :invoke :read [27 nil] INFO [2019-12-17 12:32:12,441] jepsen worker 6 - jepsen.util 8016 :info :write [2 2] :RequestCanceledException INFO [2019-12-17 12:32:12,446] jepsen worker 6 - jepsen.util 8106 :invoke :read [2 nil] INFO [2019-12-17 12:32:12,473] jepsen worker 51 - jepsen.util 5631 :fail :cas [17 [28 16]] :ValueNotSwapFrom INFO [2019-12-17 12:32:12,473] jepsen worker 51 - jepsen.util 5631 :invoke :cas [17 [23 23]] INFO [2019-12-17 12:32:12,477] jepsen worker 37 - jepsen.util 5347 :ok :write [12 19] INFO [2019-12-17 12:32:12,478] jepsen worker 20 - jepsen.util 5690 :ok :write [6 46] INFO [2019-12-17 12:32:12,478] jepsen worker 37 - jepsen.util 5347 :invoke :cas [12 [48 8]] INFO [2019-12-17 12:32:12,478] jepsen worker 20 - jepsen.util 5690 :invoke :write [6 30] INFO [2019-12-17 12:32:12,507] jepsen worker 53 - jepsen.util 5543 :ok :write [17 11] INFO [2019-12-17 12:32:12,508] jepsen worker 53 - jepsen.util 5543 :invoke :read [17 nil] INFO [2019-12-17 12:32:12,512] jepsen worker 74 - jepsen.util 6104 :ok :read [24 28] INFO [2019-12-17 12:32:12,512] jepsen worker 74 - jepsen.util 6104 :invoke :write [24 47] INFO [2019-12-17 12:32:12,513] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32605,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:32:12,514] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32605,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:32:12,515] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32610,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:32:12,515] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32610,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:32:12,530] jepsen worker 64 - jepsen.util 4384 :ok :write [21 47] INFO [2019-12-17 12:32:12,531] jepsen worker 64 - jepsen.util 4384 :invoke :cas [21 [37 0]] INFO [2019-12-17 12:32:12,540] jepsen worker 30 - jepsen.util 5790 :fail :cas [10 [10 38]] :ValueNotSwapFrom INFO [2019-12-17 12:32:12,540] jepsen worker 30 - jepsen.util 5790 :invoke :read [10 nil] INFO [2019-12-17 12:32:12,556] jepsen worker 36 - jepsen.util 6066 :ok :read [12 19] INFO [2019-12-17 12:32:12,556] jepsen worker 36 - jepsen.util 6066 :invoke :write [12 44] INFO [2019-12-17 12:32:12,593] jepsen worker 39 - jepsen.util 7329 :info :write [13 7] :RequestCanceledException INFO [2019-12-17 12:32:12,593] jepsen worker 62 - jepsen.util 7262 :info :write [20 43] :RequestCanceledException INFO [2019-12-17 12:32:12,595] jepsen worker 62 - jepsen.util 7352 :invoke :write [20 12] INFO [2019-12-17 12:32:12,595] jepsen worker 39 - jepsen.util 7419 :invoke :cas [13 [15 25]] INFO [2019-12-17 12:32:12,607] jepsen worker 83 - jepsen.util 8003 :info :cas [27 [7 20]] :RequestCanceledException INFO [2019-12-17 12:32:12,611] jepsen worker 83 - jepsen.util 8093 :invoke :cas [27 [19 26]] INFO [2019-12-17 12:32:12,616] jepsen worker 64 - jepsen.util 4384 :fail :cas [21 [37 0]] :ValueNotSwapFrom INFO [2019-12-17 12:32:12,617] jepsen worker 64 - jepsen.util 4384 :invoke :read [21 nil] INFO [2019-12-17 12:32:12,626] jepsen worker 84 - jepsen.util 7104 :info :cas [28 [10 6]] :RequestCanceledException INFO [2019-12-17 12:32:12,630] jepsen worker 84 - jepsen.util 7194 :invoke :write [28 7] INFO [2019-12-17 12:32:12,631] jepsen worker 5 - jepsen.util 5315 :ok :read [1 12] INFO [2019-12-17 12:32:12,631] jepsen worker 5 - jepsen.util 5315 :invoke :cas [1 [6 35]] INFO [2019-12-17 12:32:12,669] jepsen worker 78 - jepsen.util 6018 :ok :read [26 47] INFO [2019-12-17 12:32:12,669] jepsen worker 78 - jepsen.util 6018 :invoke :write [26 48] INFO [2019-12-17 12:32:12,672] jepsen worker 70 - jepsen.util 6100 :ok :read [23 31] INFO [2019-12-17 12:32:12,672] jepsen worker 70 - jepsen.util 6100 :invoke :write [23 13] INFO [2019-12-17 12:32:12,677] jepsen worker 64 - jepsen.util 4384 :ok :read [21 47] INFO [2019-12-17 12:32:12,677] jepsen worker 64 - jepsen.util 4384 :invoke :read [21 nil] INFO [2019-12-17 12:32:12,704] jepsen worker 64 - jepsen.util 4384 :ok :read [21 47] INFO [2019-12-17 12:32:12,705] jepsen worker 64 - jepsen.util 4384 :invoke :write [21 7] INFO [2019-12-17 12:32:12,726] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32629,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:32:12,727] jepsen worker 86 - jepsen.util 7916 :info :cas [28 [23 40]] :RequestCanceledException INFO [2019-12-17 12:32:12,727] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32629,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:32:12,727] jepsen worker 1 - jepsen.util 7381 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:32:12,727] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32633,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:32:12,727] jepsen worker 1 - jepsen.util 7381 :invoke :cas [0 [19 24]] INFO [2019-12-17 12:32:12,727] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32633,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:32:12,729] jepsen worker 86 - jepsen.util 8006 :invoke :cas [28 [14 3]] INFO [2019-12-17 12:32:12,729] jepsen worker 79 - jepsen.util 5569 :fail :cas [26 [37 27]] :ValueNotSwapFrom INFO [2019-12-17 12:32:12,730] jepsen worker 79 - jepsen.util 5569 :invoke :write [26 45] INFO [2019-12-17 12:32:12,739] jepsen worker 25 - jepsen.util 8035 :info :cas [8 [40 15]] :RequestCanceledException INFO [2019-12-17 12:32:12,742] jepsen worker 25 - jepsen.util 8125 :invoke :write [8 9] INFO [2019-12-17 12:32:12,754] jepsen worker 22 - jepsen.util 7672 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:32:12,754] jepsen worker 22 - jepsen.util 7672 :invoke :cas [7 [29 37]] INFO [2019-12-17 12:32:12,763] jepsen worker 53 - jepsen.util 5543 :ok :read [17 11] INFO [2019-12-17 12:32:12,763] jepsen worker 53 - jepsen.util 5543 :invoke :write [17 49] INFO [2019-12-17 12:32:12,790] jepsen worker 9 - jepsen.util 6219 :ok :write [3 18] INFO [2019-12-17 12:32:12,790] jepsen worker 19 - jepsen.util 5779 :ok :write [6 8] INFO [2019-12-17 12:32:12,790] jepsen worker 9 - jepsen.util 6219 :invoke :write [3 38] INFO [2019-12-17 12:32:12,790] jepsen worker 19 - jepsen.util 5779 :invoke :write [6 24] INFO [2019-12-17 12:32:12,793] jepsen worker 21 - jepsen.util 7131 :info :cas [7 [38 36]] :RequestCanceledException INFO [2019-12-17 12:32:12,795] jepsen worker 64 - jepsen.util 4384 :ok :write [21 7] INFO [2019-12-17 12:32:12,795] jepsen worker 64 - jepsen.util 4384 :invoke :cas [21 [1 36]] INFO [2019-12-17 12:32:12,795] jepsen worker 21 - jepsen.util 7221 :invoke :read [7 nil] INFO [2019-12-17 12:32:12,811] jepsen worker 77 - jepsen.util 7727 :info :write [25 3] :RequestCanceledException INFO [2019-12-17 12:32:12,813] jepsen worker 77 - jepsen.util 7817 :invoke :cas [25 [2 6]] INFO [2019-12-17 12:32:12,826] jepsen worker 67 - jepsen.util 7807 :info :cas [22 [13 23]] :RequestCanceledException INFO [2019-12-17 12:32:12,826] jepsen worker 61 - jepsen.util 8701 :info :cas [20 [24 45]] :RequestCanceledException INFO [2019-12-17 12:32:12,829] jepsen worker 61 - jepsen.util 8791 :invoke :write [20 48] INFO [2019-12-17 12:32:12,829] jepsen worker 67 - jepsen.util 7897 :invoke :cas [22 [26 38]] INFO [2019-12-17 12:32:12,840] jepsen worker 15 - jepsen.util 6675 :info :write [5 21] :RequestCanceledException INFO [2019-12-17 12:32:12,840] jepsen worker 26 - jepsen.util 7946 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:32:12,840] jepsen worker 26 - jepsen.util 7946 :invoke :read [8 nil] INFO [2019-12-17 12:32:12,844] jepsen worker 15 - jepsen.util 6765 :invoke :read [5 nil] INFO [2019-12-17 12:32:12,858] jepsen worker 40 - jepsen.util 8140 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:32:12,858] jepsen worker 17 - jepsen.util 7667 :info :cas [5 [10 21]] :RequestCanceledException INFO [2019-12-17 12:32:12,858] jepsen worker 40 - jepsen.util 8140 :invoke :read [13 nil] INFO [2019-12-17 12:32:12,862] jepsen worker 71 - jepsen.util 6371 :fail :cas [23 [21 27]] :ValueNotSwapFrom INFO [2019-12-17 12:32:12,862] jepsen worker 71 - jepsen.util 6371 :invoke :read [23 nil] INFO [2019-12-17 12:32:12,863] jepsen worker 17 - jepsen.util 7757 :invoke :cas [5 [6 28]] INFO [2019-12-17 12:32:12,878] jepsen worker 64 - jepsen.util 4384 :fail :cas [21 [1 36]] :ValueNotSwapFrom INFO [2019-12-17 12:32:12,878] jepsen worker 64 - jepsen.util 4384 :invoke :write [21 42] INFO [2019-12-17 12:32:12,881] jepsen worker 74 - jepsen.util 6104 :ok :write [24 47] INFO [2019-12-17 12:32:12,881] jepsen worker 74 - jepsen.util 6104 :invoke :read [24 nil] INFO [2019-12-17 12:32:12,893] jepsen worker 27 - jepsen.util 5517 :info :write [9 21] :RequestCanceledException INFO [2019-12-17 12:32:12,894] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:32:12,894] jepsen nemesis - couchbase.util Rebalance check count: 247 INFO [2019-12-17 12:32:12,894] jepsen nemesis - couchbase.util Rebalance stuck count: 228 INFO [2019-12-17 12:32:12,896] jepsen worker 27 - jepsen.util 5607 :invoke :read [9 nil] INFO [2019-12-17 12:32:12,915] jepsen worker 74 - jepsen.util 6104 :ok :read [24 47] INFO [2019-12-17 12:32:12,915] jepsen worker 74 - jepsen.util 6104 :invoke :read [24 nil] INFO [2019-12-17 12:32:12,924] jepsen worker 37 - jepsen.util 5347 :fail :cas [12 [48 8]] :ValueNotSwapFrom INFO [2019-12-17 12:32:12,925] jepsen worker 37 - jepsen.util 5347 :invoke :read [12 nil] INFO [2019-12-17 12:32:12,926] jepsen worker 73 - jepsen.util 5293 :fail :read [24 nil] :RequestTimeoutException INFO [2019-12-17 12:32:12,926] jepsen worker 24 - jepsen.util 6954 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:32:12,926] jepsen worker 7 - jepsen.util 7927 :info :write [2 24] :RequestCanceledException INFO [2019-12-17 12:32:12,926] jepsen worker 73 - jepsen.util 5293 :invoke :write [24 30] INFO [2019-12-17 12:32:12,926] jepsen worker 24 - jepsen.util 6954 :invoke :cas [8 [35 32]] INFO [2019-12-17 12:32:12,929] jepsen worker 7 - jepsen.util 8017 :invoke :cas [2 [4 14]] INFO [2019-12-17 12:32:12,943] jepsen worker 52 - jepsen.util 6082 :ok :read [17 11] INFO [2019-12-17 12:32:12,944] jepsen worker 52 - jepsen.util 6082 :invoke :read [17 nil] INFO [2019-12-17 12:32:12,948] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32655,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:32:12,948] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32655,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:32:12,948] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32658,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:32:12,948] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32658,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:32:12,966] jepsen worker 74 - jepsen.util 6104 :ok :read [24 47] INFO [2019-12-17 12:32:12,967] jepsen worker 74 - jepsen.util 6104 :invoke :read [24 nil] INFO [2019-12-17 12:32:12,981] jepsen worker 64 - jepsen.util 4384 :ok :write [21 42] INFO [2019-12-17 12:32:12,981] jepsen worker 64 - jepsen.util 4384 :invoke :read [21 nil] INFO [2019-12-17 12:32:12,992] jepsen worker 45 - jepsen.util 5625 :info :write [15 46] :RequestCanceledException INFO [2019-12-17 12:32:12,992] jepsen worker 35 - jepsen.util 7775 :info :write [11 20] :RequestCanceledException INFO [2019-12-17 12:32:12,995] jepsen worker 45 - jepsen.util 5715 :invoke :cas [15 [0 16]] INFO [2019-12-17 12:32:12,995] jepsen worker 35 - jepsen.util 7865 :invoke :write [11 32] INFO [2019-12-17 12:32:13,021] jepsen worker 64 - jepsen.util 4384 :ok :read [21 42] INFO [2019-12-17 12:32:13,021] jepsen worker 64 - jepsen.util 4384 :invoke :write [21 18] INFO [2019-12-17 12:32:13,026] jepsen worker 23 - jepsen.util 7763 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:32:13,026] jepsen worker 13 - jepsen.util 5503 :info :write [4 31] :RequestCanceledException INFO [2019-12-17 12:32:13,027] jepsen worker 23 - jepsen.util 7763 :invoke :read [7 nil] INFO [2019-12-17 12:32:13,028] jepsen worker 13 - jepsen.util 5593 :invoke :cas [4 [48 7]] INFO [2019-12-17 12:32:13,039] jepsen worker 42 - jepsen.util 8142 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:32:13,039] jepsen worker 42 - jepsen.util 8142 :invoke :cas [14 [25 39]] INFO [2019-12-17 12:32:13,061] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32674,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:32:13,061] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32674,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:32:13,061] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32678,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:32:13,061] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32678,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:32:13,086] jepsen worker 64 - jepsen.util 4384 :ok :write [21 18] INFO [2019-12-17 12:32:13,086] jepsen worker 64 - jepsen.util 4384 :invoke :read [21 nil] INFO [2019-12-17 12:32:13,111] jepsen worker 59 - jepsen.util 7709 :info :write [19 43] :RequestCanceledException INFO [2019-12-17 12:32:13,114] jepsen worker 64 - jepsen.util 4384 :ok :read [21 18] INFO [2019-12-17 12:32:13,114] jepsen worker 59 - jepsen.util 7799 :invoke :read [19 nil] INFO [2019-12-17 12:32:13,114] jepsen worker 64 - jepsen.util 4384 :invoke :cas [21 [35 16]] INFO [2019-12-17 12:32:13,126] jepsen worker 2 - jepsen.util 7202 :info :cas [0 [28 29]] :RequestCanceledException INFO [2019-12-17 12:32:13,130] jepsen worker 2 - jepsen.util 7292 :invoke :write [0 48] INFO [2019-12-17 12:32:13,134] jepsen worker 54 - jepsen.util 6174 :fail :cas [18 [37 33]] :ValueNotSwapFrom INFO [2019-12-17 12:32:13,135] jepsen worker 54 - jepsen.util 6174 :invoke :read [18 nil] INFO [2019-12-17 12:32:13,141] jepsen worker 49 - jepsen.util 6709 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:32:13,141] jepsen worker 4 - jepsen.util 5044 :info :write [1 30] :RequestCanceledException INFO [2019-12-17 12:32:13,141] jepsen worker 49 - jepsen.util 6709 :invoke :cas [16 [11 14]] INFO [2019-12-17 12:32:13,143] jepsen worker 4 - jepsen.util 5134 :invoke :read [1 nil] INFO [2019-12-17 12:32:13,153] jepsen worker 64 - jepsen.util 4384 :fail :cas [21 [35 16]] :ValueNotSwapFrom INFO [2019-12-17 12:32:13,154] jepsen worker 64 - jepsen.util 4384 :invoke :cas [21 [1 36]] INFO [2019-12-17 12:32:13,155] jepsen worker 5 - jepsen.util 5315 :fail :cas [1 [6 35]] :ValueNotSwapFrom INFO [2019-12-17 12:32:13,155] jepsen worker 5 - jepsen.util 5315 :invoke :cas [1 [34 20]] INFO [2019-12-17 12:32:13,183] jepsen worker 64 - jepsen.util 4384 :fail :cas [21 [1 36]] :ValueNotSwapFrom INFO [2019-12-17 12:32:13,183] jepsen worker 64 - jepsen.util 4384 :invoke :cas [21 [45 4]] INFO [2019-12-17 12:32:13,196] jepsen worker 71 - jepsen.util 6371 :ok :read [23 31] INFO [2019-12-17 12:32:13,196] jepsen worker 71 - jepsen.util 6371 :invoke :read [23 nil] INFO [2019-12-17 12:32:13,209] jepsen worker 64 - jepsen.util 4384 :fail :cas [21 [45 4]] :ValueNotSwapFrom INFO [2019-12-17 12:32:13,209] jepsen worker 64 - jepsen.util 4384 :invoke :write [21 9] INFO [2019-12-17 12:32:13,226] jepsen worker 3 - jepsen.util 5583 :info :write [1 44] :RequestCanceledException INFO [2019-12-17 12:32:13,226] jepsen worker 69 - jepsen.util 6009 :info :write [23 31] :RequestCanceledException INFO [2019-12-17 12:32:13,226] jepsen worker 47 - jepsen.util 6347 :fail :read [15 nil] :RequestTimeoutException INFO [2019-12-17 12:32:13,226] jepsen worker 47 - jepsen.util 6347 :invoke :cas [15 [19 9]] INFO [2019-12-17 12:32:13,228] jepsen worker 69 - jepsen.util 6099 :invoke :cas [23 [19 11]] INFO [2019-12-17 12:32:13,228] jepsen worker 3 - jepsen.util 5673 :invoke :read [1 nil] INFO [2019-12-17 12:32:13,258] jepsen worker 72 - jepsen.util 5472 :info :write [24 39] :RequestCanceledException INFO [2019-12-17 12:32:13,262] jepsen worker 89 - jepsen.util 5489 :ok :read [29 3] INFO [2019-12-17 12:32:13,263] jepsen worker 89 - jepsen.util 5489 :invoke :cas [29 [20 21]] INFO [2019-12-17 12:32:13,263] jepsen worker 72 - jepsen.util 5562 :invoke :cas [24 [29 31]] INFO [2019-12-17 12:32:13,272] jepsen worker 64 - jepsen.util 4384 :ok :write [21 9] INFO [2019-12-17 12:32:13,272] jepsen worker 64 - jepsen.util 4384 :invoke :write [21 31] INFO [2019-12-17 12:32:13,276] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32704,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:32:13,276] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32704,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:32:13,278] jepsen worker 37 - jepsen.util 5347 :ok :read [12 19] INFO [2019-12-17 12:32:13,278] jepsen worker 37 - jepsen.util 5347 :invoke :write [12 46] INFO [2019-12-17 12:32:13,281] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32696,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:32:13,281] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32696,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:32:13,281] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32697,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:32:13,281] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32697,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:32:13,298] jepsen worker 72 - jepsen.util 5562 :fail :cas [24 [29 31]] :ValueNotSwapFrom INFO [2019-12-17 12:32:13,298] jepsen worker 72 - jepsen.util 5562 :invoke :read [24 nil] INFO [2019-12-17 12:32:13,323] jepsen worker 12 - jepsen.util 5952 :ok :read [4 5] INFO [2019-12-17 12:32:13,324] jepsen worker 12 - jepsen.util 5952 :invoke :read [4 nil] INFO [2019-12-17 12:32:13,343] jepsen worker 64 - jepsen.util 4384 :ok :write [21 31] INFO [2019-12-17 12:32:13,343] jepsen worker 64 - jepsen.util 4384 :invoke :write [21 26] INFO [2019-12-17 12:32:13,345] jepsen worker 51 - jepsen.util 5631 :fail :cas [17 [23 23]] :ValueNotSwapFrom INFO [2019-12-17 12:32:13,345] jepsen worker 51 - jepsen.util 5631 :invoke :read [17 nil] INFO [2019-12-17 12:32:13,357] jepsen worker 72 - jepsen.util 5562 :ok :read [24 47] INFO [2019-12-17 12:32:13,357] jepsen worker 72 - jepsen.util 5562 :invoke :cas [24 [23 40]] INFO [2019-12-17 12:32:13,357] jepsen worker 52 - jepsen.util 6082 :ok :read [17 11] INFO [2019-12-17 12:32:13,357] jepsen worker 65 - jepsen.util 4745 :info :write [21 11] :RequestCanceledException INFO [2019-12-17 12:32:13,357] jepsen worker 52 - jepsen.util 6082 :invoke :read [17 nil] INFO [2019-12-17 12:32:13,360] jepsen worker 65 - jepsen.util 4835 :invoke :cas [21 [46 18]] INFO [2019-12-17 12:32:13,382] jepsen worker 72 - jepsen.util 5562 :fail :cas [24 [23 40]] :ValueNotSwapFrom INFO [2019-12-17 12:32:13,382] jepsen worker 72 - jepsen.util 5562 :invoke :read [24 nil] INFO [2019-12-17 12:32:13,394] jepsen worker 50 - jepsen.util 7880 :info :write [16 8] :RequestCanceledException INFO [2019-12-17 12:32:13,394] jepsen worker 65 - jepsen.util 4835 :fail :cas [21 [46 18]] :ValueNotSwapFrom INFO [2019-12-17 12:32:13,394] jepsen worker 65 - jepsen.util 4835 :invoke :read [21 nil] INFO [2019-12-17 12:32:13,397] jepsen worker 50 - jepsen.util 7970 :invoke :cas [16 [39 7]] INFO [2019-12-17 12:32:13,426] jepsen worker 88 - jepsen.util 5758 :info :write [29 9] :RequestCanceledException INFO [2019-12-17 12:32:13,426] jepsen worker 14 - jepsen.util 5774 :info :write [4 44] :RequestCanceledException INFO [2019-12-17 12:32:13,427] jepsen worker 12 - jepsen.util 5952 :ok :read [4 5] INFO [2019-12-17 12:32:13,427] jepsen worker 12 - jepsen.util 5952 :invoke :write [4 7] INFO [2019-12-17 12:32:13,432] jepsen worker 14 - jepsen.util 5864 :invoke :read [4 nil] INFO [2019-12-17 12:32:13,433] jepsen worker 88 - jepsen.util 5848 :invoke :write [29 19] INFO [2019-12-17 12:32:13,445] jepsen worker 13 - jepsen.util 5593 :fail :cas [4 [48 7]] :ValueNotSwapFrom INFO [2019-12-17 12:32:13,446] jepsen worker 13 - jepsen.util 5593 :invoke :write [4 6] INFO [2019-12-17 12:32:13,453] jepsen worker 65 - jepsen.util 4835 :ok :read [21 31] INFO [2019-12-17 12:32:13,453] jepsen worker 65 - jepsen.util 4835 :invoke :cas [21 [25 43]] INFO [2019-12-17 12:32:13,457] jepsen worker 10 - jepsen.util 5500 :fail :read [3 nil] :RequestTimeoutException INFO [2019-12-17 12:32:13,457] jepsen worker 10 - jepsen.util 5500 :invoke :write [3 27] INFO [2019-12-17 12:32:13,484] jepsen worker 72 - jepsen.util 5562 :ok :read [24 47] INFO [2019-12-17 12:32:13,485] jepsen worker 72 - jepsen.util 5562 :invoke :write [24 20] INFO [2019-12-17 12:32:13,487] jepsen worker 64 - jepsen.util 4384 :ok :write [21 26] INFO [2019-12-17 12:32:13,487] jepsen worker 64 - jepsen.util 4384 :invoke :cas [21 [37 14]] INFO [2019-12-17 12:32:13,490] jepsen worker 54 - jepsen.util 6174 :ok :read [18 36] INFO [2019-12-17 12:32:13,490] jepsen worker 54 - jepsen.util 6174 :invoke :read [18 nil] INFO [2019-12-17 12:32:13,500] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32730,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:32:13,500] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32730,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:32:13,501] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32731,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:32:13,501] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32731,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:32:13,507] jepsen worker 65 - jepsen.util 4835 :fail :cas [21 [25 43]] :ValueNotSwapFrom INFO [2019-12-17 12:32:13,507] jepsen worker 65 - jepsen.util 4835 :invoke :read [21 nil] INFO [2019-12-17 12:32:13,525] jepsen worker 56 - jepsen.util 6086 :info :write [18 19] :RequestCanceledException INFO [2019-12-17 12:32:13,525] jepsen worker 80 - jepsen.util 5660 :info :cas [26 [17 48]] :RequestCanceledException INFO [2019-12-17 12:32:13,530] jepsen worker 80 - jepsen.util 5750 :invoke :cas [26 [28 38]] INFO [2019-12-17 12:32:13,530] jepsen worker 56 - jepsen.util 6176 :invoke :read [18 nil] INFO [2019-12-17 12:32:13,533] jepsen worker 4 - jepsen.util 5134 :ok :read [1 12] INFO [2019-12-17 12:32:13,533] jepsen worker 4 - jepsen.util 5134 :invoke :cas [1 [47 13]] INFO [2019-12-17 12:32:13,537] jepsen worker 64 - jepsen.util 4384 :fail :cas [21 [37 14]] :ValueNotSwapFrom INFO [2019-12-17 12:32:13,537] jepsen worker 64 - jepsen.util 4384 :invoke :read [21 nil] INFO [2019-12-17 12:32:13,553] jepsen worker 71 - jepsen.util 6371 :ok :read [23 31] INFO [2019-12-17 12:32:13,553] jepsen worker 71 - jepsen.util 6371 :invoke :write [23 29] INFO [2019-12-17 12:32:13,558] jepsen worker 32 - jepsen.util 5702 :info :write [10 1] :RequestCanceledException INFO [2019-12-17 12:32:13,562] jepsen worker 51 - jepsen.util 5631 :ok :read [17 11] INFO [2019-12-17 12:32:13,562] jepsen worker 51 - jepsen.util 5631 :invoke :read [17 nil] INFO [2019-12-17 12:32:13,562] jepsen worker 32 - jepsen.util 5792 :invoke :cas [10 [48 13]] INFO [2019-12-17 12:32:13,573] jepsen worker 64 - jepsen.util 4384 :ok :read [21 26] INFO [2019-12-17 12:32:13,573] jepsen worker 64 - jepsen.util 4384 :invoke :read [21 nil] INFO [2019-12-17 12:32:13,592] jepsen worker 32 - jepsen.util 5792 :fail :cas [10 [48 13]] :ValueNotSwapFrom INFO [2019-12-17 12:32:13,592] jepsen worker 32 - jepsen.util 5792 :invoke :cas [10 [47 24]] INFO [2019-12-17 12:32:13,593] jepsen worker 46 - jepsen.util 5536 :info :write [15 46] :RequestCanceledException INFO [2019-12-17 12:32:13,597] jepsen worker 46 - jepsen.util 5626 :invoke :read [15 nil] INFO [2019-12-17 12:32:13,607] jepsen worker 3 - jepsen.util 5673 :ok :read [1 12] INFO [2019-12-17 12:32:13,607] jepsen worker 3 - jepsen.util 5673 :invoke :cas [1 [43 8]] INFO [2019-12-17 12:32:13,607] jepsen worker 64 - jepsen.util 4384 :ok :read [21 26] INFO [2019-12-17 12:32:13,607] jepsen worker 64 - jepsen.util 4384 :invoke :cas [21 [34 35]] INFO [2019-12-17 12:32:13,620] jepsen worker 32 - jepsen.util 5792 :fail :cas [10 [47 24]] :ValueNotSwapFrom INFO [2019-12-17 12:32:13,620] jepsen worker 32 - jepsen.util 5792 :invoke :cas [10 [47 43]] INFO [2019-12-17 12:32:13,626] jepsen worker 18 - jepsen.util 5598 :info :write [6 6] :RequestCanceledException INFO [2019-12-17 12:32:13,629] jepsen worker 18 - jepsen.util 5688 :invoke :write [6 2] INFO [2019-12-17 12:32:13,654] jepsen worker 64 - jepsen.util 4384 :fail :cas [21 [34 35]] :ValueNotSwapFrom INFO [2019-12-17 12:32:13,654] jepsen worker 64 - jepsen.util 4384 :invoke :cas [21 [47 30]] INFO [2019-12-17 12:32:13,665] jepsen worker 19 - jepsen.util 5779 :ok :write [6 24] INFO [2019-12-17 12:32:13,665] jepsen worker 19 - jepsen.util 5779 :invoke :read [6 nil] INFO [2019-12-17 12:32:13,667] jepsen worker 32 - jepsen.util 5792 :fail :cas [10 [47 43]] :ValueNotSwapFrom INFO [2019-12-17 12:32:13,667] jepsen worker 32 - jepsen.util 5792 :invoke :cas [10 [6 48]] INFO [2019-12-17 12:32:13,668] jepsen worker 56 - jepsen.util 6176 :ok :read [18 36] INFO [2019-12-17 12:32:13,668] jepsen worker 56 - jepsen.util 6176 :invoke :cas [18 [49 14]] INFO [2019-12-17 12:32:13,683] jepsen worker 73 - jepsen.util 5293 :ok :write [24 30] INFO [2019-12-17 12:32:13,684] jepsen worker 9 - jepsen.util 6219 :ok :write [3 38] INFO [2019-12-17 12:32:13,684] jepsen worker 9 - jepsen.util 6219 :invoke :write [3 42] INFO [2019-12-17 12:32:13,684] jepsen worker 73 - jepsen.util 5293 :invoke :cas [24 [9 9]] INFO [2019-12-17 12:32:13,704] jepsen worker 4 - jepsen.util 5134 :fail :cas [1 [47 13]] :ValueNotSwapFrom INFO [2019-12-17 12:32:13,704] jepsen worker 4 - jepsen.util 5134 :invoke :write [1 49] INFO [2019-12-17 12:32:13,706] jepsen worker 64 - jepsen.util 4384 :fail :cas [21 [47 30]] :ValueNotSwapFrom INFO [2019-12-17 12:32:13,706] jepsen worker 64 - jepsen.util 4384 :invoke :write [21 38] INFO [2019-12-17 12:32:13,715] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32757,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:32:13,715] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32757,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:32:13,716] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32760,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:32:13,716] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32760,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:32:13,726] jepsen worker 29 - jepsen.util 4439 :fail :read [9 nil] :RequestTimeoutException INFO [2019-12-17 12:32:13,726] jepsen worker 38 - jepsen.util 5708 :info :write [12 39] :RequestCanceledException INFO [2019-12-17 12:32:13,726] jepsen worker 29 - jepsen.util 4439 :invoke :cas [9 [42 17]] INFO [2019-12-17 12:32:13,735] jepsen worker 38 - jepsen.util 5798 :invoke :write [12 17] INFO [2019-12-17 12:32:13,757] jepsen worker 27 - jepsen.util 5607 :ok :read [9 24] INFO [2019-12-17 12:32:13,757] jepsen worker 27 - jepsen.util 5607 :invoke :write [9 19] INFO [2019-12-17 12:32:13,759] jepsen worker 5 - jepsen.util 5315 :fail :cas [1 [34 20]] :ValueNotSwapFrom INFO [2019-12-17 12:32:13,759] jepsen worker 5 - jepsen.util 5315 :invoke :write [1 2] INFO [2019-12-17 12:32:13,814] jepsen worker 46 - jepsen.util 5626 :ok :read [15 4] INFO [2019-12-17 12:32:13,815] jepsen worker 89 - jepsen.util 5489 :fail :cas [29 [20 21]] :ValueNotSwapFrom INFO [2019-12-17 12:32:13,815] jepsen worker 46 - jepsen.util 5626 :invoke :cas [15 [38 37]] INFO [2019-12-17 12:32:13,815] jepsen worker 89 - jepsen.util 5489 :invoke :write [29 48] INFO [2019-12-17 12:32:13,817] jepsen worker 32 - jepsen.util 5792 :fail :cas [10 [6 48]] :ValueNotSwapFrom INFO [2019-12-17 12:32:13,817] jepsen worker 32 - jepsen.util 5792 :invoke :write [10 7] INFO [2019-12-17 12:32:13,820] jepsen worker 74 - jepsen.util 6104 :ok :read [24 30] INFO [2019-12-17 12:32:13,820] jepsen worker 74 - jepsen.util 6104 :invoke :read [24 nil] INFO [2019-12-17 12:32:13,826] jepsen worker 80 - jepsen.util 5750 :fail :cas [26 [28 38]] :ValueNotSwapFrom INFO [2019-12-17 12:32:13,827] jepsen worker 80 - jepsen.util 5750 :invoke :write [26 42] INFO [2019-12-17 12:32:13,872] jepsen worker 19 - jepsen.util 5779 :ok :read [6 24] INFO [2019-12-17 12:32:13,872] jepsen worker 19 - jepsen.util 5779 :invoke :read [6 nil] INFO [2019-12-17 12:32:13,888] jepsen worker 69 - jepsen.util 6099 :fail :cas [23 [19 11]] :ValueNotSwapFrom INFO [2019-12-17 12:32:13,888] jepsen worker 69 - jepsen.util 6099 :invoke :read [23 nil] INFO [2019-12-17 12:32:13,919] jepsen worker 3 - jepsen.util 5673 :fail :cas [1 [43 8]] :ValueNotSwapFrom INFO [2019-12-17 12:32:13,919] jepsen worker 3 - jepsen.util 5673 :invoke :cas [1 [43 29]] INFO [2019-12-17 12:32:13,927] jepsen worker 55 - jepsen.util 6715 :fail :read [18 nil] :RequestTimeoutException INFO [2019-12-17 12:32:13,927] jepsen worker 55 - jepsen.util 6715 :invoke :read [18 nil] INFO [2019-12-17 12:32:13,931] jepsen worker 14 - jepsen.util 5864 :ok :read [4 5] INFO [2019-12-17 12:32:13,931] jepsen worker 14 - jepsen.util 5864 :invoke :cas [4 [15 7]] INFO [2019-12-17 12:32:13,934] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32780,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:32:13,934] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32780,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:32:13,934] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32783,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:32:13,934] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32783,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:32:13,934] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32784,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:32:13,934] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32784,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:32:13,936] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:32:13,936] jepsen nemesis - couchbase.util Rebalance check count: 248 INFO [2019-12-17 12:32:13,936] jepsen nemesis - couchbase.util Rebalance stuck count: 229 INFO [2019-12-17 12:32:13,950] jepsen worker 45 - jepsen.util 5715 :fail :cas [15 [0 16]] :ValueNotSwapFrom INFO [2019-12-17 12:32:13,950] jepsen worker 45 - jepsen.util 5715 :invoke :write [15 32] INFO [2019-12-17 12:32:13,958] jepsen worker 87 - jepsen.util 5127 :info :write [29 11] :RequestCanceledException INFO [2019-12-17 12:32:13,962] jepsen worker 87 - jepsen.util 5217 :invoke :write [29 31] INFO [2019-12-17 12:32:13,985] jepsen worker 47 - jepsen.util 6347 :fail :cas [15 [19 9]] :ValueNotSwapFrom INFO [2019-12-17 12:32:13,986] jepsen worker 47 - jepsen.util 6347 :invoke :cas [15 [36 6]] INFO [2019-12-17 12:32:13,998] jepsen worker 73 - jepsen.util 5293 :fail :cas [24 [9 9]] :ValueNotSwapFrom INFO [2019-12-17 12:32:13,998] jepsen worker 73 - jepsen.util 5293 :invoke :cas [24 [36 0]] INFO [2019-12-17 12:32:14,036] jepsen worker 56 - jepsen.util 6176 :fail :cas [18 [49 14]] :ValueNotSwapFrom INFO [2019-12-17 12:32:14,036] jepsen worker 56 - jepsen.util 6176 :invoke :read [18 nil] INFO [2019-12-17 12:32:14,040] jepsen worker 29 - jepsen.util 4439 :fail :cas [9 [42 17]] :ValueNotSwapFrom INFO [2019-12-17 12:32:14,040] jepsen worker 29 - jepsen.util 4439 :invoke :write [9 2] INFO [2019-12-17 12:32:14,041] jepsen worker 11 - jepsen.util 5861 :info :write [3 5] :RequestCanceledException INFO [2019-12-17 12:32:14,041] jepsen worker 63 - jepsen.util 4743 :fail :read [21 nil] :RequestTimeoutException INFO [2019-12-17 12:32:14,041] jepsen worker 63 - jepsen.util 4743 :invoke :read [21 nil] INFO [2019-12-17 12:32:14,043] jepsen worker 11 - jepsen.util 5951 :invoke :read [3 nil] INFO [2019-12-17 12:32:14,045] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32794,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:32:14,045] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32794,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:32:14,045] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32798,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:32:14,045] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32798,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:32:14,064] jepsen worker 74 - jepsen.util 6104 :ok :read [24 30] INFO [2019-12-17 12:32:14,064] jepsen worker 74 - jepsen.util 6104 :invoke :read [24 nil] INFO [2019-12-17 12:32:14,093] jepsen worker 68 - jepsen.util 7088 :info :cas [22 [0 45]] :RequestCanceledException INFO [2019-12-17 12:32:14,094] jepsen worker 63 - jepsen.util 4743 :ok :read [21 26] INFO [2019-12-17 12:32:14,095] jepsen worker 63 - jepsen.util 4743 :invoke :cas [21 [44 47]] INFO [2019-12-17 12:32:14,096] jepsen worker 68 - jepsen.util 7178 :invoke :cas [22 [48 42]] INFO [2019-12-17 12:32:14,124] jepsen worker 66 - jepsen.util 7086 :info :write [22 8] :RequestCanceledException INFO [2019-12-17 12:32:14,128] jepsen worker 66 - jepsen.util 7176 :invoke :read [22 nil] INFO [2019-12-17 12:32:14,134] jepsen worker 11 - jepsen.util 5951 :ok :read [3 38] INFO [2019-12-17 12:32:14,134] jepsen worker 11 - jepsen.util 5951 :invoke :cas [3 [19 15]] INFO [2019-12-17 12:32:14,147] jepsen worker 63 - jepsen.util 4743 :fail :cas [21 [44 47]] :ValueNotSwapFrom INFO [2019-12-17 12:32:14,147] jepsen worker 63 - jepsen.util 4743 :invoke :write [21 31] INFO [2019-12-17 12:32:14,181] jepsen worker 11 - jepsen.util 5951 :fail :cas [3 [19 15]] :ValueNotSwapFrom INFO [2019-12-17 12:32:14,182] jepsen worker 11 - jepsen.util 5951 :invoke :cas [3 [21 46]] INFO [2019-12-17 12:32:14,192] jepsen worker 64 - jepsen.util 4384 :ok :write [21 38] INFO [2019-12-17 12:32:14,192] jepsen worker 64 - jepsen.util 4384 :invoke :write [21 5] INFO [2019-12-17 12:32:14,193] jepsen worker 44 - jepsen.util 8324 :info :cas [14 [36 39]] :RequestCanceledException INFO [2019-12-17 12:32:14,195] jepsen worker 44 - jepsen.util 8414 :invoke :cas [14 [30 12]] INFO [2019-12-17 12:32:14,204] jepsen worker 85 - jepsen.util 8545 :info :cas [28 [24 1]] :RequestCanceledException INFO [2019-12-17 12:32:14,204] jepsen worker 75 - jepsen.util 7275 :info :cas [25 [24 31]] :RequestCanceledException INFO [2019-12-17 12:32:14,207] jepsen worker 85 - jepsen.util 8635 :invoke :read [28 nil] INFO [2019-12-17 12:32:14,207] jepsen worker 75 - jepsen.util 7365 :invoke :write [25 48] INFO [2019-12-17 12:32:14,233] jepsen worker 27 - jepsen.util 5607 :ok :write [9 19] INFO [2019-12-17 12:32:14,233] jepsen worker 27 - jepsen.util 5607 :invoke :write [9 15] INFO [2019-12-17 12:32:14,239] jepsen worker 82 - jepsen.util 8182 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:32:14,240] jepsen worker 82 - jepsen.util 8182 :invoke :read [27 nil] INFO [2019-12-17 12:32:14,249] jepsen worker 53 - jepsen.util 5543 :ok :write [17 49] INFO [2019-12-17 12:32:14,250] jepsen worker 53 - jepsen.util 5543 :invoke :cas [17 [36 13]] INFO [2019-12-17 12:32:14,258] jepsen worker 32 - jepsen.util 5792 :ok :write [10 7] INFO [2019-12-17 12:32:14,258] jepsen worker 32 - jepsen.util 5792 :invoke :read [10 nil] INFO [2019-12-17 12:32:14,258] jepsen worker 18 - jepsen.util 5688 :ok :write [6 2] INFO [2019-12-17 12:32:14,258] jepsen worker 18 - jepsen.util 5688 :invoke :cas [6 [3 32]] INFO [2019-12-17 12:32:14,262] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32819,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:32:14,262] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32819,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:32:14,262] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32821,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:32:14,262] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32821,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:32:14,263] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32811,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:32:14,263] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32811,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:32:14,263] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32813,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:32:14,263] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32813,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:32:14,282] jepsen worker 63 - jepsen.util 4743 :ok :write [21 31] INFO [2019-12-17 12:32:14,282] jepsen worker 63 - jepsen.util 4743 :invoke :cas [21 [23 4]] INFO [2019-12-17 12:32:14,285] jepsen worker 36 - jepsen.util 6066 :ok :write [12 44] INFO [2019-12-17 12:32:14,285] jepsen worker 36 - jepsen.util 6066 :invoke :write [12 35] INFO [2019-12-17 12:32:14,293] jepsen worker 16 - jepsen.util 8566 :info :write [5 49] :RequestCanceledException INFO [2019-12-17 12:32:14,293] jepsen worker 60 - jepsen.util 8430 :info :write [20 12] :RequestCanceledException INFO [2019-12-17 12:32:14,293] jepsen worker 13 - jepsen.util 5593 :ok :write [4 6] INFO [2019-12-17 12:32:14,293] jepsen worker 13 - jepsen.util 5593 :invoke :cas [4 [22 48]] INFO [2019-12-17 12:32:14,295] jepsen worker 60 - jepsen.util 8520 :invoke :cas [20 [42 25]] INFO [2019-12-17 12:32:14,295] jepsen worker 16 - jepsen.util 8656 :invoke :write [5 0] INFO [2019-12-17 12:32:14,324] jepsen worker 33 - jepsen.util 7683 :info :cas [11 [3 17]] :RequestCanceledException INFO [2019-12-17 12:32:14,324] jepsen worker 57 - jepsen.util 7167 :info :write [19 19] :RequestCanceledException INFO [2019-12-17 12:32:14,324] jepsen worker 48 - jepsen.util 6888 :info :write [16 25] :RequestCanceledException INFO [2019-12-17 12:32:14,324] jepsen worker 43 - jepsen.util 7153 :info :write [14 18] :RequestCanceledException INFO [2019-12-17 12:32:14,329] jepsen worker 33 - jepsen.util 7773 :invoke :cas [11 [40 30]] INFO [2019-12-17 12:32:14,329] jepsen worker 48 - jepsen.util 6978 :invoke :read [16 nil] INFO [2019-12-17 12:32:14,329] jepsen worker 43 - jepsen.util 7243 :invoke :write [14 2] INFO [2019-12-17 12:32:14,329] jepsen worker 57 - jepsen.util 7257 :invoke :cas [19 [31 4]] INFO [2019-12-17 12:32:14,331] jepsen worker 8 - jepsen.util 7118 :info :cas [2 [19 20]] :RequestCanceledException INFO [2019-12-17 12:32:14,336] jepsen worker 8 - jepsen.util 7208 :invoke :cas [2 [30 30]] INFO [2019-12-17 12:32:14,349] jepsen worker 45 - jepsen.util 5715 :ok :write [15 32] INFO [2019-12-17 12:32:14,350] jepsen worker 45 - jepsen.util 5715 :invoke :write [15 36] INFO [2019-12-17 12:32:14,352] jepsen worker 51 - jepsen.util 5631 :ok :read [17 49] INFO [2019-12-17 12:32:14,352] jepsen worker 51 - jepsen.util 5631 :invoke :cas [17 [24 47]] INFO [2019-12-17 12:32:14,353] jepsen worker 34 - jepsen.util 7144 :info :write [11 5] :RequestCanceledException INFO [2019-12-17 12:32:14,359] jepsen worker 34 - jepsen.util 7234 :invoke :read [11 nil] INFO [2019-12-17 12:32:14,365] jepsen worker 63 - jepsen.util 4743 :fail :cas [21 [23 4]] :ValueNotSwapFrom INFO [2019-12-17 12:32:14,365] jepsen worker 63 - jepsen.util 4743 :invoke :write [21 12] INFO [2019-12-17 12:32:14,399] jepsen worker 32 - jepsen.util 5792 :ok :read [10 7] INFO [2019-12-17 12:32:14,399] jepsen worker 32 - jepsen.util 5792 :invoke :read [10 nil] INFO [2019-12-17 12:32:14,401] jepsen worker 18 - jepsen.util 5688 :fail :cas [6 [3 32]] :ValueNotSwapFrom INFO [2019-12-17 12:32:14,401] jepsen worker 18 - jepsen.util 5688 :invoke :write [6 29] INFO [2019-12-17 12:32:14,416] jepsen worker 74 - jepsen.util 6104 :ok :read [24 30] INFO [2019-12-17 12:32:14,416] jepsen worker 74 - jepsen.util 6104 :invoke :write [24 24] INFO [2019-12-17 12:32:14,427] jepsen worker 0 - jepsen.util 7920 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:32:14,427] jepsen worker 0 - jepsen.util 7920 :invoke :write [0 18] INFO [2019-12-17 12:32:14,443] jepsen worker 56 - jepsen.util 6176 :ok :read [18 36] INFO [2019-12-17 12:32:14,444] jepsen worker 56 - jepsen.util 6176 :invoke :write [18 31] INFO [2019-12-17 12:32:14,457] jepsen worker 58 - jepsen.util 8068 :info :write [19 17] :RequestCanceledException INFO [2019-12-17 12:32:14,457] jepsen worker 6 - jepsen.util 8106 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:32:14,457] jepsen worker 6 - jepsen.util 8106 :invoke :write [2 25] INFO [2019-12-17 12:32:14,459] jepsen worker 58 - jepsen.util 8158 :invoke :write [19 31] INFO [2019-12-17 12:32:14,476] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32851,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:32:14,477] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32851,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:32:14,477] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32853,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:32:14,477] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32853,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:32:14,477] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32854,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:32:14,477] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32854,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:32:14,478] jepsen worker 32 - jepsen.util 5792 :ok :read [10 7] INFO [2019-12-17 12:32:14,478] jepsen worker 32 - jepsen.util 5792 :invoke :write [10 16] INFO [2019-12-17 12:32:14,484] jepsen worker 28 - jepsen.util 4438 :fail :read [9 nil] :RequestTimeoutException INFO [2019-12-17 12:32:14,484] jepsen worker 28 - jepsen.util 4438 :invoke :cas [9 [11 37]] INFO [2019-12-17 12:32:14,506] jepsen worker 27 - jepsen.util 5607 :ok :write [9 15] INFO [2019-12-17 12:32:14,506] jepsen worker 27 - jepsen.util 5607 :invoke :write [9 19] INFO [2019-12-17 12:32:14,509] jepsen worker 3 - jepsen.util 5673 :fail :cas [1 [43 29]] :ValueNotSwapFrom INFO [2019-12-17 12:32:14,509] jepsen worker 3 - jepsen.util 5673 :invoke :write [1 24] INFO [2019-12-17 12:32:14,510] jepsen worker 31 - jepsen.util 5071 :info :cas [10 [49 24]] :RequestCanceledException INFO [2019-12-17 12:32:14,513] jepsen worker 31 - jepsen.util 5161 :invoke :cas [10 [22 27]] INFO [2019-12-17 12:32:14,526] jepsen worker 41 - jepsen.util 8411 :info :cas [13 [40 8]] :RequestCanceledException INFO [2019-12-17 12:32:14,526] jepsen worker 76 - jepsen.util 7636 :info :write [25 36] :RequestCanceledException INFO [2019-12-17 12:32:14,529] jepsen worker 41 - jepsen.util 8501 :invoke :write [13 24] INFO [2019-12-17 12:32:14,529] jepsen worker 76 - jepsen.util 7726 :invoke :write [25 11] INFO [2019-12-17 12:32:14,531] jepsen worker 52 - jepsen.util 6082 :ok :read [17 49] INFO [2019-12-17 12:32:14,531] jepsen worker 52 - jepsen.util 6082 :invoke :write [17 48] INFO [2019-12-17 12:32:14,538] jepsen worker 81 - jepsen.util 7641 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:32:14,538] jepsen worker 20 - jepsen.util 5690 :info :write [6 30] :RequestCanceledException INFO [2019-12-17 12:32:14,538] jepsen worker 81 - jepsen.util 7641 :invoke :read [27 nil] INFO [2019-12-17 12:32:14,540] jepsen worker 20 - jepsen.util 5780 :invoke :read [6 nil] INFO [2019-12-17 12:32:14,626] jepsen worker 62 - jepsen.util 7352 :info :write [20 12] :RequestCanceledException INFO [2019-12-17 12:32:14,629] jepsen worker 62 - jepsen.util 7442 :invoke :cas [20 [12 33]] INFO [2019-12-17 12:32:14,632] jepsen worker 54 - jepsen.util 6174 :ok :read [18 36] INFO [2019-12-17 12:32:14,632] jepsen worker 54 - jepsen.util 6174 :invoke :read [18 nil] INFO [2019-12-17 12:32:14,634] jepsen worker 63 - jepsen.util 4743 :ok :write [21 12] INFO [2019-12-17 12:32:14,634] jepsen worker 63 - jepsen.util 4743 :invoke :cas [21 [28 20]] INFO [2019-12-17 12:32:14,638] jepsen worker 13 - jepsen.util 5593 :fail :cas [4 [22 48]] :ValueNotSwapFrom INFO [2019-12-17 12:32:14,639] jepsen worker 13 - jepsen.util 5593 :invoke :cas [4 [39 9]] INFO [2019-12-17 12:32:14,640] jepsen worker 30 - jepsen.util 5790 :fail :read [10 nil] :RequestTimeoutException INFO [2019-12-17 12:32:14,640] jepsen worker 30 - jepsen.util 5790 :invoke :write [10 27] INFO [2019-12-17 12:32:14,675] jepsen worker 20 - jepsen.util 5780 :ok :read [6 2] INFO [2019-12-17 12:32:14,676] jepsen worker 20 - jepsen.util 5780 :invoke :read [6 nil] INFO [2019-12-17 12:32:14,692] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32874,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:32:14,693] jepsen worker 39 - jepsen.util 7419 :info :cas [13 [15 25]] :RequestCanceledException INFO [2019-12-17 12:32:14,693] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32874,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:32:14,693] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32878,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:32:14,693] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32878,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:32:14,695] jepsen worker 39 - jepsen.util 7509 :invoke :write [13 2] INFO [2019-12-17 12:32:14,698] jepsen worker 27 - jepsen.util 5607 :ok :write [9 19] INFO [2019-12-17 12:32:14,698] jepsen worker 27 - jepsen.util 5607 :invoke :write [9 36] INFO [2019-12-17 12:32:14,710] jepsen worker 83 - jepsen.util 8093 :info :cas [27 [19 26]] :RequestCanceledException INFO [2019-12-17 12:32:14,712] jepsen worker 83 - jepsen.util 8183 :invoke :cas [27 [49 35]] INFO [2019-12-17 12:32:14,712] jepsen worker 63 - jepsen.util 4743 :fail :cas [21 [28 20]] :ValueNotSwapFrom INFO [2019-12-17 12:32:14,712] jepsen worker 63 - jepsen.util 4743 :invoke :write [21 0] INFO [2019-12-17 12:32:14,727] jepsen worker 84 - jepsen.util 7194 :info :write [28 7] :RequestCanceledException INFO [2019-12-17 12:32:14,730] jepsen worker 84 - jepsen.util 7284 :invoke :cas [28 [38 33]] INFO [2019-12-17 12:32:14,732] jepsen worker 55 - jepsen.util 6715 :ok :read [18 36] INFO [2019-12-17 12:32:14,732] jepsen worker 55 - jepsen.util 6715 :invoke :cas [18 [25 8]] INFO [2019-12-17 12:32:14,740] jepsen worker 78 - jepsen.util 6018 :info :write [26 48] :RequestCanceledException INFO [2019-12-17 12:32:14,740] jepsen worker 86 - jepsen.util 8006 :info :cas [28 [14 3]] :RequestCanceledException INFO [2019-12-17 12:32:14,742] jepsen worker 78 - jepsen.util 6108 :invoke :read [26 nil] INFO [2019-12-17 12:32:14,742] jepsen worker 86 - jepsen.util 8096 :invoke :read [28 nil] INFO [2019-12-17 12:32:14,757] jepsen worker 70 - jepsen.util 6100 :info :write [23 13] :RequestCanceledException INFO [2019-12-17 12:32:14,757] jepsen worker 79 - jepsen.util 5569 :info :write [26 45] :RequestCanceledException INFO [2019-12-17 12:32:14,759] jepsen worker 70 - jepsen.util 6190 :invoke :read [23 nil] INFO [2019-12-17 12:32:14,759] jepsen worker 79 - jepsen.util 5659 :invoke :read [26 nil] INFO [2019-12-17 12:32:14,775] jepsen worker 13 - jepsen.util 5593 :fail :cas [4 [39 9]] :ValueNotSwapFrom INFO [2019-12-17 12:32:14,775] jepsen worker 13 - jepsen.util 5593 :invoke :write [4 8] INFO [2019-12-17 12:32:14,826] jepsen worker 1 - jepsen.util 7381 :info :cas [0 [19 24]] :RequestCanceledException INFO [2019-12-17 12:32:14,826] jepsen worker 21 - jepsen.util 7221 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:32:14,826] jepsen worker 21 - jepsen.util 7221 :invoke :read [7 nil] INFO [2019-12-17 12:32:14,829] jepsen worker 1 - jepsen.util 7471 :invoke :write [0 19] INFO [2019-12-17 12:32:14,841] jepsen worker 27 - jepsen.util 5607 :ok :write [9 36] INFO [2019-12-17 12:32:14,841] jepsen worker 27 - jepsen.util 5607 :invoke :read [9 nil] INFO [2019-12-17 12:32:14,857] jepsen worker 22 - jepsen.util 7672 :info :cas [7 [29 37]] :RequestCanceledException INFO [2019-12-17 12:32:14,857] jepsen worker 25 - jepsen.util 8125 :info :write [8 9] :RequestCanceledException INFO [2019-12-17 12:32:14,859] jepsen worker 25 - jepsen.util 8215 :invoke :cas [8 [20 35]] INFO [2019-12-17 12:32:14,859] jepsen worker 22 - jepsen.util 7762 :invoke :read [7 nil] INFO [2019-12-17 12:32:14,889] jepsen worker 63 - jepsen.util 4743 :ok :write [21 0] INFO [2019-12-17 12:32:14,889] jepsen worker 63 - jepsen.util 4743 :invoke :read [21 nil] INFO [2019-12-17 12:32:14,890] jepsen worker 77 - jepsen.util 7817 :info :cas [25 [2 6]] :RequestCanceledException INFO [2019-12-17 12:32:14,894] jepsen worker 77 - jepsen.util 7907 :invoke :read [25 nil] INFO [2019-12-17 12:32:14,903] jepsen worker 27 - jepsen.util 5607 :ok :read [9 36] INFO [2019-12-17 12:32:14,903] jepsen worker 27 - jepsen.util 5607 :invoke :cas [9 [33 17]] INFO [2019-12-17 12:32:14,909] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32900,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:32:14,910] jepsen worker 61 - jepsen.util 8791 :info :write [20 48] :RequestCanceledException INFO [2019-12-17 12:32:14,910] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32900,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:32:14,912] jepsen worker 61 - jepsen.util 8881 :invoke :write [20 0] INFO [2019-12-17 12:32:14,926] jepsen worker 67 - jepsen.util 7897 :info :cas [22 [26 38]] :RequestCanceledException INFO [2019-12-17 12:32:14,929] jepsen worker 67 - jepsen.util 7987 :invoke :read [22 nil] INFO [2019-12-17 12:32:14,941] jepsen worker 15 - jepsen.util 6765 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:32:14,941] jepsen worker 26 - jepsen.util 7946 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:32:14,941] jepsen worker 15 - jepsen.util 6765 :invoke :write [5 29] INFO [2019-12-17 12:32:14,941] jepsen worker 26 - jepsen.util 7946 :invoke :write [8 41] INFO [2019-12-17 12:32:14,948] jepsen worker 63 - jepsen.util 4743 :ok :read [21 0] INFO [2019-12-17 12:32:14,948] jepsen worker 63 - jepsen.util 4743 :invoke :read [21 nil] INFO [2019-12-17 12:32:14,957] jepsen worker 40 - jepsen.util 8140 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:32:14,957] jepsen worker 17 - jepsen.util 7757 :info :cas [5 [6 28]] :RequestCanceledException INFO [2019-12-17 12:32:14,957] jepsen worker 40 - jepsen.util 8140 :invoke :read [13 nil] INFO [2019-12-17 12:32:14,960] jepsen worker 17 - jepsen.util 7847 :invoke :cas [5 [39 38]] INFO [2019-12-17 12:32:14,975] jepsen worker 27 - jepsen.util 5607 :fail :cas [9 [33 17]] :ValueNotSwapFrom INFO [2019-12-17 12:32:14,975] jepsen worker 27 - jepsen.util 5607 :invoke :write [9 17] INFO [2019-12-17 12:32:14,981] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:32:14,981] jepsen nemesis - couchbase.util Rebalance check count: 249 INFO [2019-12-17 12:32:14,981] jepsen nemesis - couchbase.util Rebalance stuck count: 230 INFO [2019-12-17 12:32:14,984] jepsen worker 7 - jepsen.util 8017 :info :cas [2 [4 14]] :RequestCanceledException INFO [2019-12-17 12:32:14,987] jepsen worker 7 - jepsen.util 8107 :invoke :cas [2 [7 24]] INFO [2019-12-17 12:32:15,010] jepsen worker 63 - jepsen.util 4743 :ok :read [21 0] INFO [2019-12-17 12:32:15,010] jepsen worker 63 - jepsen.util 4743 :invoke :write [21 11] INFO [2019-12-17 12:32:15,020] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32915,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:32:15,020] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32915,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:32:15,020] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32918,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:32:15,020] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32918,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:32:15,020] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32919,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:32:15,020] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32919,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:32:15,026] jepsen worker 24 - jepsen.util 6954 :info :cas [8 [35 32]] :RequestCanceledException INFO [2019-12-17 12:32:15,029] jepsen worker 24 - jepsen.util 7044 :invoke :write [8 34] INFO [2019-12-17 12:32:15,042] jepsen worker 18 - jepsen.util 5688 :ok :write [6 29] INFO [2019-12-17 12:32:15,042] jepsen worker 18 - jepsen.util 5688 :invoke :read [6 nil] INFO [2019-12-17 12:32:15,070] jepsen worker 28 - jepsen.util 4438 :fail :cas [9 [11 37]] :ValueNotSwapFrom INFO [2019-12-17 12:32:15,070] jepsen worker 28 - jepsen.util 4438 :invoke :cas [9 [29 35]] INFO [2019-12-17 12:32:15,096] jepsen worker 31 - jepsen.util 5161 :fail :cas [10 [22 27]] :ValueNotSwapFrom INFO [2019-12-17 12:32:15,096] jepsen worker 31 - jepsen.util 5161 :invoke :read [10 nil] INFO [2019-12-17 12:32:15,108] jepsen worker 18 - jepsen.util 5688 :ok :read [6 29] INFO [2019-12-17 12:32:15,109] jepsen worker 18 - jepsen.util 5688 :invoke :cas [6 [3 29]] INFO [2019-12-17 12:32:15,110] jepsen worker 35 - jepsen.util 7865 :info :write [11 32] :RequestCanceledException INFO [2019-12-17 12:32:15,111] jepsen worker 35 - jepsen.util 7955 :invoke :read [11 nil] INFO [2019-12-17 12:32:15,126] jepsen worker 23 - jepsen.util 7763 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:32:15,127] jepsen worker 23 - jepsen.util 7763 :invoke :read [7 nil] INFO [2019-12-17 12:32:15,140] jepsen worker 14 - jepsen.util 5864 :fail :cas [4 [15 7]] :ValueNotSwapFrom INFO [2019-12-17 12:32:15,140] jepsen worker 42 - jepsen.util 8142 :info :cas [14 [25 39]] :RequestCanceledException INFO [2019-12-17 12:32:15,140] jepsen worker 28 - jepsen.util 4438 :fail :cas [9 [29 35]] :ValueNotSwapFrom INFO [2019-12-17 12:32:15,140] jepsen worker 14 - jepsen.util 5864 :invoke :read [4 nil] INFO [2019-12-17 12:32:15,140] jepsen worker 28 - jepsen.util 4438 :invoke :read [9 nil] INFO [2019-12-17 12:32:15,141] jepsen worker 42 - jepsen.util 8232 :invoke :write [14 7] INFO [2019-12-17 12:32:15,148] jepsen worker 53 - jepsen.util 5543 :fail :cas [17 [36 13]] :ValueNotSwapFrom INFO [2019-12-17 12:32:15,149] jepsen worker 53 - jepsen.util 5543 :invoke :read [17 nil] INFO [2019-12-17 12:32:15,155] jepsen worker 59 - jepsen.util 7799 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:32:15,155] jepsen worker 59 - jepsen.util 7799 :invoke :cas [19 [41 11]] INFO [2019-12-17 12:32:15,164] jepsen worker 31 - jepsen.util 5161 :ok :read [10 7] INFO [2019-12-17 12:32:15,164] jepsen worker 31 - jepsen.util 5161 :invoke :cas [10 [28 25]] INFO [2019-12-17 12:32:15,188] jepsen worker 27 - jepsen.util 5607 :ok :write [9 17] INFO [2019-12-17 12:32:15,188] jepsen worker 27 - jepsen.util 5607 :invoke :read [9 nil] INFO [2019-12-17 12:32:15,210] jepsen worker 18 - jepsen.util 5688 :fail :cas [6 [3 29]] :ValueNotSwapFrom INFO [2019-12-17 12:32:15,210] jepsen worker 18 - jepsen.util 5688 :invoke :cas [6 [4 8]] INFO [2019-12-17 12:32:15,214] jepsen worker 14 - jepsen.util 5864 :ok :read [4 6] INFO [2019-12-17 12:32:15,214] jepsen worker 14 - jepsen.util 5864 :invoke :read [4 nil] INFO [2019-12-17 12:32:15,220] jepsen worker 63 - jepsen.util 4743 :ok :write [21 11] INFO [2019-12-17 12:32:15,220] jepsen worker 63 - jepsen.util 4743 :invoke :write [21 42] INFO [2019-12-17 12:32:15,226] jepsen worker 2 - jepsen.util 7292 :info :write [0 48] :RequestCanceledException INFO [2019-12-17 12:32:15,230] jepsen worker 2 - jepsen.util 7382 :invoke :cas [0 [1 24]] INFO [2019-12-17 12:32:15,237] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32937,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:32:15,238] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32937,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:32:15,238] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32946,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:32:15,238] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32946,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:32:15,238] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32948,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:32:15,238] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32948,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:32:15,239] jepsen worker 49 - jepsen.util 6709 :info :cas [16 [11 14]] :RequestCanceledException INFO [2019-12-17 12:32:15,244] jepsen worker 49 - jepsen.util 6799 :invoke :write [16 5] INFO [2019-12-17 12:32:15,258] jepsen worker 27 - jepsen.util 5607 :ok :read [9 17] INFO [2019-12-17 12:32:15,258] jepsen worker 27 - jepsen.util 5607 :invoke :write [9 2] INFO [2019-12-17 12:32:15,261] jepsen worker 28 - jepsen.util 4438 :ok :read [9 17] INFO [2019-12-17 12:32:15,261] jepsen worker 28 - jepsen.util 4438 :invoke :cas [9 [38 5]] INFO [2019-12-17 12:32:15,272] jepsen worker 31 - jepsen.util 5161 :fail :cas [10 [28 25]] :ValueNotSwapFrom INFO [2019-12-17 12:32:15,272] jepsen worker 31 - jepsen.util 5161 :invoke :read [10 nil] INFO [2019-12-17 12:32:15,299] jepsen worker 18 - jepsen.util 5688 :fail :cas [6 [4 8]] :ValueNotSwapFrom INFO [2019-12-17 12:32:15,299] jepsen worker 18 - jepsen.util 5688 :invoke :cas [6 [26 10]] INFO [2019-12-17 12:32:15,308] jepsen worker 37 - jepsen.util 5347 :info :write [12 46] :RequestCanceledException INFO [2019-12-17 12:32:15,309] jepsen worker 28 - jepsen.util 4438 :fail :cas [9 [38 5]] :ValueNotSwapFrom INFO [2019-12-17 12:32:15,309] jepsen worker 28 - jepsen.util 4438 :invoke :write [9 3] INFO [2019-12-17 12:32:15,312] jepsen worker 37 - jepsen.util 5437 :invoke :read [12 nil] INFO [2019-12-17 12:32:15,319] jepsen worker 31 - jepsen.util 5161 :ok :read [10 7] INFO [2019-12-17 12:32:15,319] jepsen worker 31 - jepsen.util 5161 :invoke :read [10 nil] INFO [2019-12-17 12:32:15,348] jepsen worker 18 - jepsen.util 5688 :fail :cas [6 [26 10]] :ValueNotSwapFrom INFO [2019-12-17 12:32:15,348] jepsen worker 18 - jepsen.util 5688 :invoke :cas [6 [14 43]] INFO [2019-12-17 12:32:15,357] jepsen worker 63 - jepsen.util 4743 :ok :write [21 42] INFO [2019-12-17 12:32:15,357] jepsen worker 63 - jepsen.util 4743 :invoke :read [21 nil] INFO [2019-12-17 12:32:15,376] jepsen worker 51 - jepsen.util 5631 :fail :cas [17 [24 47]] :ValueNotSwapFrom INFO [2019-12-17 12:32:15,376] jepsen worker 51 - jepsen.util 5631 :invoke :read [17 nil] INFO [2019-12-17 12:32:15,382] jepsen worker 53 - jepsen.util 5543 :ok :read [17 49] INFO [2019-12-17 12:32:15,382] jepsen worker 53 - jepsen.util 5543 :invoke :read [17 nil] INFO [2019-12-17 12:32:15,398] jepsen worker 63 - jepsen.util 4743 :ok :read [21 42] INFO [2019-12-17 12:32:15,398] jepsen worker 63 - jepsen.util 4743 :invoke :cas [21 [40 40]] INFO [2019-12-17 12:32:15,412] jepsen worker 31 - jepsen.util 5161 :ok :read [10 7] INFO [2019-12-17 12:32:15,412] jepsen worker 31 - jepsen.util 5161 :invoke :read [10 nil] INFO [2019-12-17 12:32:15,422] jepsen worker 54 - jepsen.util 6174 :ok :read [18 36] INFO [2019-12-17 12:32:15,422] jepsen worker 54 - jepsen.util 6174 :invoke :cas [18 [35 48]] INFO [2019-12-17 12:32:15,426] jepsen worker 50 - jepsen.util 7970 :info :cas [16 [39 7]] :RequestCanceledException INFO [2019-12-17 12:32:15,428] jepsen worker 50 - jepsen.util 8060 :invoke :write [16 27] INFO [2019-12-17 12:32:15,431] jepsen worker 63 - jepsen.util 4743 :fail :cas [21 [40 40]] :ValueNotSwapFrom INFO [2019-12-17 12:32:15,431] jepsen worker 63 - jepsen.util 4743 :invoke :write [21 5] INFO [2019-12-17 12:32:15,444] jepsen worker 70 - jepsen.util 6190 :ok :read [23 31] INFO [2019-12-17 12:32:15,444] jepsen worker 70 - jepsen.util 6190 :invoke :cas [23 [35 13]] INFO [2019-12-17 12:32:15,445] jepsen worker 18 - jepsen.util 5688 :fail :cas [6 [14 43]] :ValueNotSwapFrom INFO [2019-12-17 12:32:15,445] jepsen worker 18 - jepsen.util 5688 :invoke :read [6 nil] INFO [2019-12-17 12:32:15,453] jepsen worker 27 - jepsen.util 5607 :ok :write [9 2] INFO [2019-12-17 12:32:15,454] jepsen worker 27 - jepsen.util 5607 :invoke :cas [9 [22 33]] INFO [2019-12-17 12:32:15,456] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32968,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:32:15,457] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32968,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:32:15,457] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32971,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:32:15,457] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32971,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:32:15,511] jepsen worker 88 - jepsen.util 5848 :info :write [29 19] :RequestCanceledException INFO [2019-12-17 12:32:15,511] jepsen worker 31 - jepsen.util 5161 :ok :read [10 7] INFO [2019-12-17 12:32:15,511] jepsen worker 31 - jepsen.util 5161 :invoke :read [10 nil] INFO [2019-12-17 12:32:15,513] jepsen worker 88 - jepsen.util 5938 :invoke :write [29 30] INFO [2019-12-17 12:32:15,516] jepsen worker 27 - jepsen.util 5607 :fail :cas [9 [22 33]] :ValueNotSwapFrom INFO [2019-12-17 12:32:15,516] jepsen worker 27 - jepsen.util 5607 :invoke :write [9 25] INFO [2019-12-17 12:32:15,521] jepsen worker 18 - jepsen.util 5688 :ok :read [6 29] INFO [2019-12-17 12:32:15,521] jepsen worker 18 - jepsen.util 5688 :invoke :read [6 nil] INFO [2019-12-17 12:32:15,526] jepsen worker 12 - jepsen.util 5952 :info :write [4 7] :RequestCanceledException INFO [2019-12-17 12:32:15,527] jepsen worker 12 - jepsen.util 6042 :invoke :write [4 16] INFO [2019-12-17 12:32:15,536] jepsen worker 31 - jepsen.util 5161 :ok :read [10 7] INFO [2019-12-17 12:32:15,536] jepsen worker 31 - jepsen.util 5161 :invoke :read [10 nil] INFO [2019-12-17 12:32:15,539] jepsen worker 18 - jepsen.util 5688 :ok :read [6 29] INFO [2019-12-17 12:32:15,540] jepsen worker 18 - jepsen.util 5688 :invoke :write [6 21] INFO [2019-12-17 12:32:15,540] jepsen worker 72 - jepsen.util 5562 :info :write [24 20] :RequestCanceledException INFO [2019-12-17 12:32:15,542] jepsen worker 72 - jepsen.util 5652 :invoke :write [24 10] INFO [2019-12-17 12:32:15,558] jepsen worker 65 - jepsen.util 4835 :fail :read [21 nil] :RequestTimeoutException INFO [2019-12-17 12:32:15,558] jepsen worker 10 - jepsen.util 5500 :info :write [3 27] :RequestCanceledException INFO [2019-12-17 12:32:15,558] jepsen worker 65 - jepsen.util 4835 :invoke :write [21 28] INFO [2019-12-17 12:32:15,561] jepsen worker 10 - jepsen.util 5590 :invoke :write [3 45] INFO [2019-12-17 12:32:15,590] jepsen worker 79 - jepsen.util 5659 :ok :read [26 47] INFO [2019-12-17 12:32:15,590] jepsen worker 79 - jepsen.util 5659 :invoke :write [26 2] INFO [2019-12-17 12:32:15,592] jepsen worker 31 - jepsen.util 5161 :ok :read [10 7] INFO [2019-12-17 12:32:15,593] jepsen worker 31 - jepsen.util 5161 :invoke :read [10 nil] INFO [2019-12-17 12:32:15,594] jepsen worker 63 - jepsen.util 4743 :ok :write [21 5] INFO [2019-12-17 12:32:15,594] jepsen worker 63 - jepsen.util 4743 :invoke :read [21 nil] INFO [2019-12-17 12:32:15,603] jepsen worker 71 - jepsen.util 6371 :info :write [23 29] :RequestCanceledException INFO [2019-12-17 12:32:15,604] jepsen worker 71 - jepsen.util 6461 :invoke :cas [23 [15 12]] INFO [2019-12-17 12:32:15,615] jepsen worker 63 - jepsen.util 4743 :ok :read [21 5] INFO [2019-12-17 12:32:15,615] jepsen worker 63 - jepsen.util 4743 :invoke :write [21 4] INFO [2019-12-17 12:32:15,637] jepsen worker 32 - jepsen.util 5792 :ok :write [10 16] INFO [2019-12-17 12:32:15,637] jepsen worker 32 - jepsen.util 5792 :invoke :read [10 nil] INFO [2019-12-17 12:32:15,637] jepsen worker 45 - jepsen.util 5715 :ok :write [15 36] INFO [2019-12-17 12:32:15,637] jepsen worker 45 - jepsen.util 5715 :invoke :read [15 nil] INFO [2019-12-17 12:32:15,639] jepsen worker 27 - jepsen.util 5607 :ok :write [9 25] INFO [2019-12-17 12:32:15,639] jepsen worker 27 - jepsen.util 5607 :invoke :cas [9 [7 5]] INFO [2019-12-17 12:32:15,665] jepsen worker 45 - jepsen.util 5715 :ok :read [15 36] INFO [2019-12-17 12:32:15,665] jepsen worker 45 - jepsen.util 5715 :invoke :write [15 0] INFO [2019-12-17 12:32:15,668] jepsen worker 14 - jepsen.util 5864 :ok :read [4 6] INFO [2019-12-17 12:32:15,669] jepsen worker 14 - jepsen.util 5864 :invoke :write [4 48] INFO [2019-12-17 12:32:15,676] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32997,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:32:15,676] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":32997,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:32:15,676] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":33000,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:32:15,677] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":33000,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:32:15,695] jepsen worker 63 - jepsen.util 4743 :ok :write [21 4] INFO [2019-12-17 12:32:15,696] jepsen worker 63 - jepsen.util 4743 :invoke :write [21 11] INFO [2019-12-17 12:32:15,708] jepsen worker 53 - jepsen.util 5543 :ok :read [17 49] INFO [2019-12-17 12:32:15,708] jepsen worker 53 - jepsen.util 5543 :invoke :write [17 42] INFO [2019-12-17 12:32:15,735] jepsen worker 51 - jepsen.util 5631 :ok :read [17 49] INFO [2019-12-17 12:32:15,735] jepsen worker 51 - jepsen.util 5631 :invoke :cas [17 [40 6]] INFO [2019-12-17 12:32:15,781] jepsen worker 65 - jepsen.util 4835 :ok :write [21 28] INFO [2019-12-17 12:32:15,782] jepsen worker 65 - jepsen.util 4835 :invoke :write [21 48] INFO [2019-12-17 12:32:15,793] jepsen worker 70 - jepsen.util 6190 :fail :cas [23 [35 13]] :ValueNotSwapFrom INFO [2019-12-17 12:32:15,793] jepsen worker 70 - jepsen.util 6190 :invoke :write [23 40] INFO [2019-12-17 12:32:15,793] jepsen worker 4 - jepsen.util 5134 :info :write [1 49] :RequestCanceledException INFO [2019-12-17 12:32:15,793] jepsen worker 9 - jepsen.util 6219 :info :write [3 42] :RequestCanceledException INFO [2019-12-17 12:32:15,795] jepsen worker 9 - jepsen.util 6309 :invoke :write [3 10] INFO [2019-12-17 12:32:15,802] jepsen worker 4 - jepsen.util 5224 :invoke :cas [1 [15 38]] INFO [2019-12-17 12:32:15,804] jepsen worker 38 - jepsen.util 5798 :info :write [12 17] :RequestCanceledException INFO [2019-12-17 12:32:15,804] jepsen worker 5 - jepsen.util 5315 :info :write [1 2] :RequestCanceledException INFO [2019-12-17 12:32:15,806] jepsen worker 38 - jepsen.util 5888 :invoke :read [12 nil] INFO [2019-12-17 12:32:15,806] jepsen worker 5 - jepsen.util 5405 :invoke :read [1 nil] INFO [2019-12-17 12:32:15,823] jepsen worker 51 - jepsen.util 5631 :fail :cas [17 [40 6]] :ValueNotSwapFrom INFO [2019-12-17 12:32:15,823] jepsen worker 51 - jepsen.util 5631 :invoke :read [17 nil] INFO [2019-12-17 12:32:15,827] jepsen worker 46 - jepsen.util 5626 :info :cas [15 [38 37]] :RequestCanceledException INFO [2019-12-17 12:32:15,829] jepsen worker 46 - jepsen.util 5716 :invoke :write [15 48] INFO [2019-12-17 12:32:15,846] jepsen worker 65 - jepsen.util 4835 :ok :write [21 48] INFO [2019-12-17 12:32:15,847] jepsen worker 65 - jepsen.util 4835 :invoke :cas [21 [4 7]] INFO [2019-12-17 12:32:15,862] jepsen worker 65 - jepsen.util 4835 :fail :cas [21 [4 7]] :ValueNotSwapFrom INFO [2019-12-17 12:32:15,863] jepsen worker 65 - jepsen.util 4835 :invoke :write [21 43] INFO [2019-12-17 12:32:15,895] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":33017,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:32:15,895] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":33017,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:32:15,896] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":33020,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:32:15,896] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":33020,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:32:15,896] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":33021,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:32:15,896] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":33021,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:32:15,911] jepsen worker 69 - jepsen.util 6099 :fail :read [23 nil] :RequestTimeoutException INFO [2019-12-17 12:32:15,911] jepsen worker 69 - jepsen.util 6099 :invoke :cas [23 [15 9]] INFO [2019-12-17 12:32:15,924] jepsen worker 65 - jepsen.util 4835 :ok :write [21 43] INFO [2019-12-17 12:32:15,924] jepsen worker 65 - jepsen.util 4835 :invoke :cas [21 [23 20]] INFO [2019-12-17 12:32:15,926] jepsen worker 80 - jepsen.util 5750 :info :write [26 42] :RequestCanceledException INFO [2019-12-17 12:32:15,926] jepsen worker 89 - jepsen.util 5489 :info :write [29 48] :RequestCanceledException INFO [2019-12-17 12:32:15,926] jepsen worker 19 - jepsen.util 5779 :fail :read [6 nil] :RequestTimeoutException INFO [2019-12-17 12:32:15,926] jepsen worker 19 - jepsen.util 5779 :invoke :read [6 nil] INFO [2019-12-17 12:32:15,927] jepsen worker 89 - jepsen.util 5579 :invoke :read [29 nil] INFO [2019-12-17 12:32:15,928] jepsen worker 80 - jepsen.util 5840 :invoke :cas [26 [41 38]] INFO [2019-12-17 12:32:15,944] jepsen worker 65 - jepsen.util 4835 :fail :cas [21 [23 20]] :ValueNotSwapFrom INFO [2019-12-17 12:32:15,944] jepsen worker 65 - jepsen.util 4835 :invoke :cas [21 [26 49]] INFO [2019-12-17 12:32:15,950] jepsen worker 54 - jepsen.util 6174 :fail :cas [18 [35 48]] :ValueNotSwapFrom INFO [2019-12-17 12:32:15,951] jepsen worker 54 - jepsen.util 6174 :invoke :read [18 nil] INFO [2019-12-17 12:32:15,952] jepsen worker 55 - jepsen.util 6715 :fail :cas [18 [25 8]] :ValueNotSwapFrom INFO [2019-12-17 12:32:15,952] jepsen worker 55 - jepsen.util 6715 :invoke :read [18 nil] INFO [2019-12-17 12:32:15,967] jepsen worker 65 - jepsen.util 4835 :fail :cas [21 [26 49]] :ValueNotSwapFrom INFO [2019-12-17 12:32:15,967] jepsen worker 65 - jepsen.util 4835 :invoke :cas [21 [1 15]] INFO [2019-12-17 12:32:15,986] jepsen worker 65 - jepsen.util 4835 :fail :cas [21 [1 15]] :ValueNotSwapFrom INFO [2019-12-17 12:32:15,986] jepsen worker 65 - jepsen.util 4835 :invoke :cas [21 [33 40]] INFO [2019-12-17 12:32:15,997] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:32:15,997] jepsen nemesis - couchbase.util Rebalance check count: 250 INFO [2019-12-17 12:32:15,998] jepsen nemesis - couchbase.util Rebalance stuck count: 231 INFO [2019-12-17 12:32:16,005] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":33034,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:32:16,005] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":33034,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:32:16,005] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":33037,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:32:16,005] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":33037,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:32:16,005] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":33038,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:32:16,005] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":33038,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:32:16,009] jepsen worker 65 - jepsen.util 4835 :fail :cas [21 [33 40]] :ValueNotSwapFrom INFO [2019-12-17 12:32:16,009] jepsen worker 65 - jepsen.util 4835 :invoke :read [21 nil] INFO [2019-12-17 12:32:16,011] jepsen worker 78 - jepsen.util 6108 :ok :read [26 47] INFO [2019-12-17 12:32:16,011] jepsen worker 78 - jepsen.util 6108 :invoke :cas [26 [47 0]] INFO [2019-12-17 12:32:16,027] jepsen worker 47 - jepsen.util 6347 :info :cas [15 [36 6]] :RequestCanceledException INFO [2019-12-17 12:32:16,027] jepsen worker 87 - jepsen.util 5217 :info :write [29 31] :RequestCanceledException INFO [2019-12-17 12:32:16,027] jepsen worker 73 - jepsen.util 5293 :info :cas [24 [36 0]] :RequestCanceledException INFO [2019-12-17 12:32:16,028] jepsen worker 47 - jepsen.util 6437 :invoke :read [15 nil] INFO [2019-12-17 12:32:16,028] jepsen worker 65 - jepsen.util 4835 :ok :read [21 43] INFO [2019-12-17 12:32:16,028] jepsen worker 65 - jepsen.util 4835 :invoke :cas [21 [16 32]] INFO [2019-12-17 12:32:16,029] jepsen worker 87 - jepsen.util 5307 :invoke :cas [29 [13 4]] INFO [2019-12-17 12:32:16,029] jepsen worker 73 - jepsen.util 5383 :invoke :cas [24 [14 28]] INFO [2019-12-17 12:32:16,056] jepsen worker 47 - jepsen.util 6437 :ok :read [15 36] INFO [2019-12-17 12:32:16,056] jepsen worker 47 - jepsen.util 6437 :invoke :cas [15 [40 37]] INFO [2019-12-17 12:32:16,059] jepsen worker 65 - jepsen.util 4835 :fail :cas [21 [16 32]] :ValueNotSwapFrom INFO [2019-12-17 12:32:16,059] jepsen worker 65 - jepsen.util 4835 :invoke :cas [21 [15 48]] INFO [2019-12-17 12:32:16,071] jepsen worker 47 - jepsen.util 6437 :fail :cas [15 [40 37]] :ValueNotSwapFrom INFO [2019-12-17 12:32:16,071] jepsen worker 47 - jepsen.util 6437 :invoke :cas [15 [6 9]] INFO [2019-12-17 12:32:16,075] jepsen worker 65 - jepsen.util 4835 :fail :cas [21 [15 48]] :ValueNotSwapFrom INFO [2019-12-17 12:32:16,076] jepsen worker 65 - jepsen.util 4835 :invoke :write [21 20] INFO [2019-12-17 12:32:16,086] jepsen worker 47 - jepsen.util 6437 :fail :cas [15 [6 9]] :ValueNotSwapFrom INFO [2019-12-17 12:32:16,087] jepsen worker 47 - jepsen.util 6437 :invoke :read [15 nil] INFO [2019-12-17 12:32:16,111] jepsen worker 47 - jepsen.util 6437 :ok :read [15 36] INFO [2019-12-17 12:32:16,111] jepsen worker 47 - jepsen.util 6437 :invoke :cas [15 [2 27]] INFO [2019-12-17 12:32:16,114] jepsen worker 54 - jepsen.util 6174 :ok :read [18 36] INFO [2019-12-17 12:32:16,114] jepsen worker 54 - jepsen.util 6174 :invoke :cas [18 [28 24]] INFO [2019-12-17 12:32:16,125] jepsen worker 29 - jepsen.util 4439 :info :write [9 2] :RequestCanceledException INFO [2019-12-17 12:32:16,126] jepsen worker 29 - jepsen.util 4529 :invoke :read [9 nil] INFO [2019-12-17 12:32:16,137] jepsen worker 47 - jepsen.util 6437 :fail :cas [15 [2 27]] :ValueNotSwapFrom INFO [2019-12-17 12:32:16,138] jepsen worker 47 - jepsen.util 6437 :invoke :write [15 41] INFO [2019-12-17 12:32:16,150] jepsen worker 65 - jepsen.util 4835 :ok :write [21 20] INFO [2019-12-17 12:32:16,150] jepsen worker 65 - jepsen.util 4835 :invoke :write [21 26] INFO [2019-12-17 12:32:16,157] jepsen worker 68 - jepsen.util 7178 :info :cas [22 [48 42]] :RequestCanceledException INFO [2019-12-17 12:32:16,159] jepsen worker 68 - jepsen.util 7268 :invoke :cas [22 [27 32]] INFO [2019-12-17 12:32:16,187] jepsen worker 51 - jepsen.util 5631 :ok :read [17 49] INFO [2019-12-17 12:32:16,187] jepsen worker 51 - jepsen.util 5631 :invoke :write [17 10] INFO [2019-12-17 12:32:16,205] jepsen worker 65 - jepsen.util 4835 :ok :write [21 26] INFO [2019-12-17 12:32:16,205] jepsen worker 65 - jepsen.util 4835 :invoke :write [21 30] INFO [2019-12-17 12:32:16,214] jepsen worker 5 - jepsen.util 5405 :ok :read [1 12] INFO [2019-12-17 12:32:16,214] jepsen worker 5 - jepsen.util 5405 :invoke :write [1 46] INFO [2019-12-17 12:32:16,216] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":33065,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:32:16,216] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":33057,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:32:16,216] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":33065,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:32:16,216] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":33057,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:32:16,216] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":33069,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:32:16,217] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":33059,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:32:16,217] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":33069,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:32:16,217] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":33059,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:32:16,226] jepsen worker 66 - jepsen.util 7176 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:32:16,226] jepsen worker 75 - jepsen.util 7365 :info :write [25 48] :RequestCanceledException INFO [2019-12-17 12:32:16,226] jepsen worker 66 - jepsen.util 7176 :invoke :write [22 29] INFO [2019-12-17 12:32:16,228] jepsen worker 75 - jepsen.util 7455 :invoke :write [25 23] INFO [2019-12-17 12:32:16,240] jepsen worker 11 - jepsen.util 5951 :info :cas [3 [21 46]] :RequestCanceledException INFO [2019-12-17 12:32:16,243] jepsen worker 11 - jepsen.util 6041 :invoke :write [3 10] INFO [2019-12-17 12:32:16,268] jepsen worker 65 - jepsen.util 4835 :ok :write [21 30] INFO [2019-12-17 12:32:16,268] jepsen worker 65 - jepsen.util 4835 :invoke :write [21 13] INFO [2019-12-17 12:32:16,293] jepsen worker 44 - jepsen.util 8414 :info :cas [14 [30 12]] :RequestCanceledException INFO [2019-12-17 12:32:16,295] jepsen worker 44 - jepsen.util 8504 :invoke :read [14 nil] INFO [2019-12-17 12:32:16,311] jepsen worker 85 - jepsen.util 8635 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:32:16,311] jepsen worker 64 - jepsen.util 4384 :info :write [21 5] :RequestCanceledException INFO [2019-12-17 12:32:16,311] jepsen worker 36 - jepsen.util 6066 :info :write [12 35] :RequestCanceledException INFO [2019-12-17 12:32:16,311] jepsen worker 85 - jepsen.util 8635 :invoke :cas [28 [40 39]] INFO [2019-12-17 12:32:16,313] jepsen worker 64 - jepsen.util 4474 :invoke :write [21 25] INFO [2019-12-17 12:32:16,314] jepsen worker 36 - jepsen.util 6156 :invoke :read [12 nil] INFO [2019-12-17 12:32:16,339] jepsen worker 82 - jepsen.util 8182 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:32:16,339] jepsen worker 60 - jepsen.util 8520 :info :cas [20 [42 25]] :RequestCanceledException INFO [2019-12-17 12:32:16,340] jepsen worker 82 - jepsen.util 8182 :invoke :write [27 40] INFO [2019-12-17 12:32:16,342] jepsen worker 60 - jepsen.util 8610 :invoke :read [20 nil] INFO [2019-12-17 12:32:16,357] jepsen worker 71 - jepsen.util 6461 :fail :cas [23 [15 12]] :ValueNotSwapFrom INFO [2019-12-17 12:32:16,357] jepsen worker 71 - jepsen.util 6461 :invoke :write [23 21] INFO [2019-12-17 12:32:16,358] jepsen worker 16 - jepsen.util 8656 :info :write [5 0] :RequestCanceledException INFO [2019-12-17 12:32:16,361] jepsen worker 16 - jepsen.util 8746 :invoke :write [5 7] INFO [2019-12-17 12:32:16,369] jepsen worker 65 - jepsen.util 4835 :ok :write [21 13] INFO [2019-12-17 12:32:16,369] jepsen worker 65 - jepsen.util 4835 :invoke :write [21 1] INFO [2019-12-17 12:32:16,393] jepsen worker 48 - jepsen.util 6978 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:32:16,394] jepsen worker 48 - jepsen.util 6978 :invoke :write [16 13] INFO [2019-12-17 12:32:16,399] jepsen worker 69 - jepsen.util 6099 :fail :cas [23 [15 9]] :ValueNotSwapFrom INFO [2019-12-17 12:32:16,399] jepsen worker 69 - jepsen.util 6099 :invoke :read [23 nil] INFO [2019-12-17 12:32:16,411] jepsen worker 43 - jepsen.util 7243 :info :write [14 2] :RequestCanceledException INFO [2019-12-17 12:32:16,413] jepsen worker 43 - jepsen.util 7333 :invoke :read [14 nil] INFO [2019-12-17 12:32:16,427] jepsen worker 33 - jepsen.util 7773 :info :cas [11 [40 30]] :RequestCanceledException INFO [2019-12-17 12:32:16,427] jepsen worker 57 - jepsen.util 7257 :info :cas [19 [31 4]] :RequestCanceledException INFO [2019-12-17 12:32:16,429] jepsen worker 57 - jepsen.util 7347 :invoke :cas [19 [7 48]] INFO [2019-12-17 12:32:16,430] jepsen worker 33 - jepsen.util 7863 :invoke :read [11 nil] INFO [2019-12-17 12:32:16,431] jepsen worker 65 - jepsen.util 4835 :ok :write [21 1] INFO [2019-12-17 12:32:16,431] jepsen worker 65 - jepsen.util 4835 :invoke :read [21 nil] INFO [2019-12-17 12:32:16,436] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":33088,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:32:16,436] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":33088,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:32:16,436] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":33092,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:32:16,436] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":33092,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:32:16,440] jepsen worker 8 - jepsen.util 7208 :info :cas [2 [30 30]] :RequestCanceledException INFO [2019-12-17 12:32:16,442] jepsen worker 8 - jepsen.util 7298 :invoke :write [2 33] INFO [2019-12-17 12:32:16,454] jepsen worker 34 - jepsen.util 7234 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:32:16,461] jepsen worker 65 - jepsen.util 4835 :ok :read [21 1] INFO [2019-12-17 12:32:16,484] jepsen worker 38 - jepsen.util 5888 :ok :read [12 44] INFO [2019-12-17 12:32:16,497] jepsen worker 4 - jepsen.util 5224 :fail :cas [1 [15 38]] :ValueNotSwapFrom INFO [2019-12-17 12:32:16,526] jepsen worker 74 - jepsen.util 6104 :info :write [24 24] :RequestCanceledException INFO [2019-12-17 12:32:16,526] jepsen worker 58 - jepsen.util 8158 :info :write [19 31] :RequestCanceledException INFO [2019-12-17 12:32:16,527] jepsen worker 0 - jepsen.util 7920 :info :write [0 18] :RequestCanceledException INFO [2019-12-17 12:32:16,527] jepsen worker 3 - jepsen.util 5673 :info :write [1 24] :RequestCanceledException INFO [2019-12-17 12:32:16,529] jepsen worker 80 - jepsen.util 5840 :fail :cas [26 [41 38]] :ValueNotSwapFrom INFO [2019-12-17 12:32:16,540] jepsen worker 56 - jepsen.util 6176 :info :write [18 31] :RequestCanceledException INFO [2019-12-17 12:32:16,557] jepsen worker 6 - jepsen.util 8106 :info :write [2 25] :RequestCanceledException INFO [2019-12-17 12:32:16,558] jepsen worker 52 - jepsen.util 6082 :info :write [17 48] :RequestCanceledException INFO [2019-12-17 12:32:16,559] jepsen worker 55 - jepsen.util 6715 :ok :read [18 36] INFO [2019-12-17 12:32:16,606] jepsen worker 34 - jepsen.util 7234 :invoke :cas [11 [0 31]] INFO [2019-12-17 12:32:16,611] jepsen worker 41 - jepsen.util 8501 :info :write [13 24] :RequestCanceledException INFO [2019-12-17 12:32:16,618] jepsen worker 36 - jepsen.util 6156 :ok :read [12 44] INFO [2019-12-17 12:32:16,622] jepsen worker 18 - jepsen.util 5688 :ok :write [6 21] INFO [2019-12-17 12:32:16,622] jepsen worker 45 - jepsen.util 5715 :ok :write [15 0] INFO [2019-12-17 12:32:16,625] jepsen worker 72 - jepsen.util 5652 :ok :write [24 10] INFO [2019-12-17 12:32:16,625] jepsen worker 11 - jepsen.util 6041 :ok :write [3 10] INFO [2019-12-17 12:32:16,626] jepsen worker 76 - jepsen.util 7726 :info :write [25 11] :RequestCanceledException INFO [2019-12-17 12:32:16,636] jepsen worker 65 - jepsen.util 4835 :invoke :read [21 nil] INFO [2019-12-17 12:32:16,636] jepsen worker 38 - jepsen.util 5888 :invoke :write [12 39] INFO [2019-12-17 12:32:16,637] jepsen worker 4 - jepsen.util 5224 :invoke :cas [1 [49 37]] INFO [2019-12-17 12:32:16,638] jepsen worker 80 - jepsen.util 5840 :invoke :read [26 nil] INFO [2019-12-17 12:32:16,639] jepsen worker 81 - jepsen.util 7641 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:32:16,639] jepsen worker 55 - jepsen.util 6715 :invoke :cas [18 [2 17]] INFO [2019-12-17 12:32:16,640] jepsen worker 74 - jepsen.util 6194 :invoke :write [24 4] INFO [2019-12-17 12:32:16,641] jepsen worker 36 - jepsen.util 6156 :invoke :cas [12 [25 38]] INFO [2019-12-17 12:32:16,641] jepsen worker 18 - jepsen.util 5688 :invoke :cas [6 [40 7]] INFO [2019-12-17 12:32:16,641] jepsen worker 45 - jepsen.util 5715 :invoke :cas [15 [18 6]] INFO [2019-12-17 12:32:16,641] jepsen worker 72 - jepsen.util 5652 :invoke :read [24 nil] INFO [2019-12-17 12:32:16,642] jepsen worker 0 - jepsen.util 8010 :invoke :write [0 48] INFO [2019-12-17 12:32:16,642] jepsen worker 11 - jepsen.util 6041 :invoke :cas [3 [42 35]] INFO [2019-12-17 12:32:16,642] jepsen worker 3 - jepsen.util 5763 :invoke :cas [1 [46 35]] INFO [2019-12-17 12:32:16,643] jepsen worker 56 - jepsen.util 6266 :invoke :read [18 nil] INFO [2019-12-17 12:32:16,643] jepsen worker 81 - jepsen.util 7641 :invoke :read [27 nil] INFO [2019-12-17 12:32:16,643] jepsen worker 58 - jepsen.util 8248 :invoke :write [19 18] INFO [2019-12-17 12:32:16,645] jepsen worker 41 - jepsen.util 8591 :invoke :cas [13 [1 10]] INFO [2019-12-17 12:32:16,645] jepsen worker 6 - jepsen.util 8196 :invoke :cas [2 [5 30]] INFO [2019-12-17 12:32:16,647] jepsen worker 52 - jepsen.util 6172 :invoke :write [17 20] INFO [2019-12-17 12:32:16,647] jepsen worker 76 - jepsen.util 7816 :invoke :cas [25 [1 3]] INFO [2019-12-17 12:32:16,653] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":33098,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:32:16,653] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":33098,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:32:16,653] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":33102,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:32:16,654] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":33102,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:32:16,654] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":33104,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:32:16,654] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":33104,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:32:16,656] jepsen worker 62 - jepsen.util 7442 :info :cas [20 [12 33]] :RequestCanceledException INFO [2019-12-17 12:32:16,657] jepsen worker 89 - jepsen.util 5579 :ok :read [29 3] INFO [2019-12-17 12:32:16,657] jepsen worker 89 - jepsen.util 5579 :invoke :cas [29 [39 22]] INFO [2019-12-17 12:32:16,662] jepsen worker 62 - jepsen.util 7532 :invoke :cas [20 [25 19]] INFO [2019-12-17 12:32:16,680] jepsen worker 19 - jepsen.util 5779 :ok :read [6 21] INFO [2019-12-17 12:32:16,680] jepsen worker 19 - jepsen.util 5779 :invoke :read [6 nil] INFO [2019-12-17 12:32:16,697] jepsen worker 73 - jepsen.util 5383 :fail :cas [24 [14 28]] :ValueNotSwapFrom INFO [2019-12-17 12:32:16,697] jepsen worker 73 - jepsen.util 5383 :invoke :write [24 4] INFO [2019-12-17 12:32:16,717] jepsen worker 31 - jepsen.util 5161 :ok :read [10 16] INFO [2019-12-17 12:32:16,717] jepsen worker 31 - jepsen.util 5161 :invoke :read [10 nil] INFO [2019-12-17 12:32:16,725] jepsen worker 39 - jepsen.util 7509 :info :write [13 2] :RequestCanceledException INFO [2019-12-17 12:32:16,730] jepsen worker 39 - jepsen.util 7599 :invoke :read [13 nil] INFO [2019-12-17 12:32:16,738] jepsen worker 37 - jepsen.util 5437 :ok :read [12 44] INFO [2019-12-17 12:32:16,738] jepsen worker 37 - jepsen.util 5437 :invoke :read [12 nil] INFO [2019-12-17 12:32:16,740] jepsen worker 20 - jepsen.util 5780 :fail :read [6 nil] :RequestTimeoutException INFO [2019-12-17 12:32:16,740] jepsen worker 30 - jepsen.util 5790 :info :write [10 27] :RequestCanceledException INFO [2019-12-17 12:32:16,740] jepsen worker 20 - jepsen.util 5780 :invoke :write [6 44] INFO [2019-12-17 12:32:16,750] jepsen worker 30 - jepsen.util 5880 :invoke :read [10 nil] INFO [2019-12-17 12:32:16,751] jepsen worker 65 - jepsen.util 4835 :ok :read [21 1] INFO [2019-12-17 12:32:16,752] jepsen worker 65 - jepsen.util 4835 :invoke :cas [21 [7 2]] INFO [2019-12-17 12:32:16,756] jepsen worker 87 - jepsen.util 5307 :fail :cas [29 [13 4]] :ValueNotSwapFrom INFO [2019-12-17 12:32:16,756] jepsen worker 87 - jepsen.util 5307 :invoke :read [29 nil] INFO [2019-12-17 12:32:16,776] jepsen worker 54 - jepsen.util 6174 :fail :cas [18 [28 24]] :ValueNotSwapFrom INFO [2019-12-17 12:32:16,777] jepsen worker 54 - jepsen.util 6174 :invoke :write [18 44] INFO [2019-12-17 12:32:16,792] jepsen worker 83 - jepsen.util 8183 :info :cas [27 [49 35]] :RequestCanceledException INFO [2019-12-17 12:32:16,794] jepsen worker 83 - jepsen.util 8273 :invoke :read [27 nil] INFO [2019-12-17 12:32:16,803] jepsen worker 84 - jepsen.util 7284 :info :cas [28 [38 33]] :RequestCanceledException INFO [2019-12-17 12:32:16,807] jepsen worker 84 - jepsen.util 7374 :invoke :cas [28 [31 45]] INFO [2019-12-17 12:32:16,821] jepsen worker 11 - jepsen.util 6041 :fail :cas [3 [42 35]] :ValueNotSwapFrom INFO [2019-12-17 12:32:16,822] jepsen worker 11 - jepsen.util 6041 :invoke :read [3 nil] INFO [2019-12-17 12:32:16,832] jepsen worker 65 - jepsen.util 4835 :fail :cas [21 [7 2]] :ValueNotSwapFrom INFO [2019-12-17 12:32:16,833] jepsen worker 65 - jepsen.util 4835 :invoke :cas [21 [19 36]] INFO [2019-12-17 12:32:16,839] jepsen worker 64 - jepsen.util 4474 :ok :write [21 25] INFO [2019-12-17 12:32:16,839] jepsen worker 64 - jepsen.util 4474 :invoke :cas [21 [28 0]] INFO [2019-12-17 12:32:16,841] jepsen worker 86 - jepsen.util 8096 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:32:16,841] jepsen worker 13 - jepsen.util 5593 :info :write [4 8] :RequestCanceledException INFO [2019-12-17 12:32:16,841] jepsen worker 86 - jepsen.util 8096 :invoke :read [28 nil] INFO [2019-12-17 12:32:16,844] jepsen worker 13 - jepsen.util 5683 :invoke :read [4 nil] INFO [2019-12-17 12:32:16,870] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":33144,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:32:16,870] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":33144,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:32:16,870] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":33148,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:32:16,870] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":33148,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:32:16,887] jepsen worker 89 - jepsen.util 5579 :fail :cas [29 [39 22]] :ValueNotSwapFrom INFO [2019-12-17 12:32:16,889] jepsen worker 89 - jepsen.util 5579 :invoke :cas [29 [47 30]] INFO [2019-12-17 12:32:16,892] jepsen worker 1 - jepsen.util 7471 :info :write [0 19] :RequestCanceledException INFO [2019-12-17 12:32:16,898] jepsen worker 1 - jepsen.util 7561 :invoke :read [0 nil] INFO [2019-12-17 12:32:16,903] jepsen worker 25 - jepsen.util 8215 :info :cas [8 [20 35]] :RequestCanceledException INFO [2019-12-17 12:32:16,904] jepsen worker 72 - jepsen.util 5652 :ok :read [24 10] INFO [2019-12-17 12:32:16,905] jepsen worker 72 - jepsen.util 5652 :invoke :write [24 15] INFO [2019-12-17 12:32:16,907] jepsen worker 25 - jepsen.util 8305 :invoke :write [8 31] INFO [2019-12-17 12:32:16,918] jepsen worker 22 - jepsen.util 7762 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:32:16,918] jepsen worker 21 - jepsen.util 7221 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:32:16,919] jepsen worker 22 - jepsen.util 7762 :invoke :write [7 46] INFO [2019-12-17 12:32:16,919] jepsen worker 21 - jepsen.util 7221 :invoke :write [7 30] INFO [2019-12-17 12:32:16,934] jepsen worker 77 - jepsen.util 7907 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:32:16,934] jepsen worker 77 - jepsen.util 7907 :invoke :write [25 49] INFO [2019-12-17 12:32:16,936] jepsen worker 65 - jepsen.util 4835 :fail :cas [21 [19 36]] :ValueNotSwapFrom INFO [2019-12-17 12:32:16,936] jepsen worker 65 - jepsen.util 4835 :invoke :cas [21 [9 31]] INFO [2019-12-17 12:32:16,949] jepsen worker 19 - jepsen.util 5779 :ok :read [6 21] INFO [2019-12-17 12:32:16,951] jepsen worker 19 - jepsen.util 5779 :invoke :read [6 nil] INFO [2019-12-17 12:32:16,951] jepsen worker 56 - jepsen.util 6266 :ok :read [18 36] INFO [2019-12-17 12:32:16,951] jepsen worker 56 - jepsen.util 6266 :invoke :read [18 nil] INFO [2019-12-17 12:32:16,957] jepsen worker 61 - jepsen.util 8881 :info :write [20 0] :RequestCanceledException INFO [2019-12-17 12:32:16,960] jepsen worker 61 - jepsen.util 8971 :invoke :read [20 nil] INFO [2019-12-17 12:32:16,975] jepsen worker 65 - jepsen.util 4835 :fail :cas [21 [9 31]] :ValueNotSwapFrom INFO [2019-12-17 12:32:16,976] jepsen worker 65 - jepsen.util 4835 :invoke :write [21 28] INFO [2019-12-17 12:32:16,981] jepsen worker 69 - jepsen.util 6099 :ok :read [23 31] INFO [2019-12-17 12:32:16,981] jepsen worker 69 - jepsen.util 6099 :invoke :read [23 nil] INFO [2019-12-17 12:32:16,993] jepsen worker 17 - jepsen.util 7847 :info :cas [5 [39 38]] :RequestCanceledException INFO [2019-12-17 12:32:16,994] jepsen worker 17 - jepsen.util 7937 :invoke :read [5 nil] INFO [2019-12-17 12:32:17,011] jepsen worker 7 - jepsen.util 8107 :info :cas [2 [7 24]] :RequestCanceledException INFO [2019-12-17 12:32:17,012] jepsen worker 7 - jepsen.util 8197 :invoke :read [2 nil] INFO [2019-12-17 12:32:17,014] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:32:17,014] jepsen nemesis - couchbase.util Rebalance check count: 251 INFO [2019-12-17 12:32:17,015] jepsen nemesis - couchbase.util Rebalance stuck count: 232 INFO [2019-12-17 12:32:17,026] jepsen worker 67 - jepsen.util 7987 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:32:17,027] jepsen worker 67 - jepsen.util 7987 :invoke :write [22 0] INFO [2019-12-17 12:32:17,041] jepsen worker 15 - jepsen.util 6765 :info :write [5 29] :RequestCanceledException INFO [2019-12-17 12:32:17,041] jepsen worker 26 - jepsen.util 7946 :info :write [8 41] :RequestCanceledException INFO [2019-12-17 12:32:17,044] jepsen worker 80 - jepsen.util 5840 :ok :read [26 47] INFO [2019-12-17 12:32:17,044] jepsen worker 26 - jepsen.util 8036 :invoke :cas [8 [42 27]] INFO [2019-12-17 12:32:17,044] jepsen worker 80 - jepsen.util 5840 :invoke :cas [26 [13 4]] INFO [2019-12-17 12:32:17,044] jepsen worker 15 - jepsen.util 6855 :invoke :read [5 nil] INFO [2019-12-17 12:32:17,049] jepsen worker 65 - jepsen.util 4835 :ok :write [21 28] INFO [2019-12-17 12:32:17,049] jepsen worker 65 - jepsen.util 4835 :invoke :cas [21 [48 30]] INFO [2019-12-17 12:32:17,057] jepsen worker 40 - jepsen.util 8140 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:32:17,057] jepsen worker 40 - jepsen.util 8140 :invoke :cas [13 [10 28]] INFO [2019-12-17 12:32:17,074] jepsen worker 37 - jepsen.util 5437 :ok :read [12 44] INFO [2019-12-17 12:32:17,075] jepsen worker 37 - jepsen.util 5437 :invoke :write [12 34] INFO [2019-12-17 12:32:17,079] jepsen worker 65 - jepsen.util 4835 :fail :cas [21 [48 30]] :ValueNotSwapFrom INFO [2019-12-17 12:32:17,079] jepsen worker 65 - jepsen.util 4835 :invoke :write [21 44] INFO [2019-12-17 12:32:17,080] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":33163,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:32:17,083] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":33163,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:32:17,083] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":33168,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:32:17,083] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":33168,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:32:17,104] jepsen worker 55 - jepsen.util 6715 :fail :cas [18 [2 17]] :ValueNotSwapFrom INFO [2019-12-17 12:32:17,104] jepsen worker 55 - jepsen.util 6715 :invoke :read [18 nil] INFO [2019-12-17 12:32:17,127] jepsen worker 24 - jepsen.util 7044 :info :write [8 34] :RequestCanceledException INFO [2019-12-17 12:32:17,129] jepsen worker 24 - jepsen.util 7134 :invoke :read [8 nil] INFO [2019-12-17 12:32:17,140] jepsen worker 35 - jepsen.util 7955 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:32:17,141] jepsen worker 35 - jepsen.util 7955 :invoke :cas [11 [23 24]] INFO [2019-12-17 12:32:17,144] jepsen worker 9 - jepsen.util 6309 :ok :write [3 10] INFO [2019-12-17 12:32:17,144] jepsen worker 9 - jepsen.util 6309 :invoke :cas [3 [5 13]] INFO [2019-12-17 12:32:17,144] jepsen worker 46 - jepsen.util 5716 :ok :write [15 48] INFO [2019-12-17 12:32:17,144] jepsen worker 46 - jepsen.util 5716 :invoke :write [15 16] INFO [2019-12-17 12:32:17,146] jepsen worker 74 - jepsen.util 6194 :ok :write [24 4] INFO [2019-12-17 12:32:17,146] jepsen worker 74 - jepsen.util 6194 :invoke :write [24 32] INFO [2019-12-17 12:32:17,180] jepsen worker 65 - jepsen.util 4835 :ok :write [21 44] INFO [2019-12-17 12:32:17,180] jepsen worker 65 - jepsen.util 4835 :invoke :cas [21 [11 32]] INFO [2019-12-17 12:32:17,194] jepsen worker 89 - jepsen.util 5579 :fail :cas [29 [47 30]] :ValueNotSwapFrom INFO [2019-12-17 12:32:17,194] jepsen worker 89 - jepsen.util 5579 :invoke :read [29 nil] INFO [2019-12-17 12:32:17,200] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":33187,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:32:17,201] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":33187,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:32:17,201] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":33188,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:32:17,201] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":33188,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:32:17,216] jepsen worker 45 - jepsen.util 5715 :fail :cas [15 [18 6]] :ValueNotSwapFrom INFO [2019-12-17 12:32:17,216] jepsen worker 45 - jepsen.util 5715 :invoke :write [15 7] INFO [2019-12-17 12:32:17,227] jepsen worker 23 - jepsen.util 7763 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:32:17,227] jepsen worker 23 - jepsen.util 7763 :invoke :cas [7 [8 43]] INFO [2019-12-17 12:32:17,240] jepsen worker 29 - jepsen.util 4529 :ok :read [9 25] INFO [2019-12-17 12:32:17,240] jepsen worker 29 - jepsen.util 4529 :invoke :write [9 20] INFO [2019-12-17 12:32:17,242] jepsen worker 69 - jepsen.util 6099 :ok :read [23 31] INFO [2019-12-17 12:32:17,243] jepsen worker 69 - jepsen.util 6099 :invoke :read [23 nil] INFO [2019-12-17 12:32:17,253] jepsen worker 42 - jepsen.util 8232 :info :write [14 7] :RequestCanceledException INFO [2019-12-17 12:32:17,253] jepsen worker 59 - jepsen.util 7799 :info :cas [19 [41 11]] :RequestCanceledException INFO [2019-12-17 12:32:17,259] jepsen worker 59 - jepsen.util 7889 :invoke :read [19 nil] INFO [2019-12-17 12:32:17,259] jepsen worker 42 - jepsen.util 8322 :invoke :write [14 44] INFO [2019-12-17 12:32:17,289] jepsen worker 18 - jepsen.util 5688 :fail :cas [6 [40 7]] :ValueNotSwapFrom INFO [2019-12-17 12:32:17,290] jepsen worker 18 - jepsen.util 5688 :invoke :read [6 nil] INFO [2019-12-17 12:32:17,291] jepsen worker 9 - jepsen.util 6309 :fail :cas [3 [5 13]] :ValueNotSwapFrom INFO [2019-12-17 12:32:17,292] jepsen worker 9 - jepsen.util 6309 :invoke :read [3 nil] INFO [2019-12-17 12:32:17,292] jepsen worker 65 - jepsen.util 4835 :fail :cas [21 [11 32]] :ValueNotSwapFrom INFO [2019-12-17 12:32:17,292] jepsen worker 36 - jepsen.util 6156 :fail :cas [12 [25 38]] :ValueNotSwapFrom INFO [2019-12-17 12:32:17,292] jepsen worker 65 - jepsen.util 4835 :invoke :read [21 nil] INFO [2019-12-17 12:32:17,292] jepsen worker 36 - jepsen.util 6156 :invoke :read [12 nil] INFO [2019-12-17 12:32:17,292] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":33195,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:32:17,292] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":33195,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:32:17,292] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":33198,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:32:17,292] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":33198,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:32:17,293] jepsen worker 49 - jepsen.util 6799 :info :write [16 5] :RequestCanceledException INFO [2019-12-17 12:32:17,296] jepsen worker 49 - jepsen.util 6889 :invoke :write [16 37] INFO [2019-12-17 12:32:17,323] jepsen worker 31 - jepsen.util 5161 :ok :read [10 16] INFO [2019-12-17 12:32:17,323] jepsen worker 31 - jepsen.util 5161 :invoke :write [10 4] INFO [2019-12-17 12:32:17,327] jepsen worker 2 - jepsen.util 7382 :info :cas [0 [1 24]] :RequestCanceledException INFO [2019-12-17 12:32:17,331] jepsen worker 2 - jepsen.util 7472 :invoke :cas [0 [13 32]] INFO [2019-12-17 12:32:17,361] jepsen worker 30 - jepsen.util 5880 :ok :read [10 16] INFO [2019-12-17 12:32:17,362] jepsen worker 30 - jepsen.util 5880 :invoke :write [10 43] INFO [2019-12-17 12:32:17,367] jepsen worker 36 - jepsen.util 6156 :ok :read [12 44] INFO [2019-12-17 12:32:17,367] jepsen worker 36 - jepsen.util 6156 :invoke :write [12 5] INFO [2019-12-17 12:32:17,383] jepsen worker 80 - jepsen.util 5840 :fail :cas [26 [13 4]] :ValueNotSwapFrom INFO [2019-12-17 12:32:17,383] jepsen worker 80 - jepsen.util 5840 :invoke :read [26 nil] INFO [2019-12-17 12:32:17,393] jepsen worker 28 - jepsen.util 4438 :info :write [9 3] :RequestCanceledException INFO [2019-12-17 12:32:17,407] jepsen worker 28 - jepsen.util 4528 :invoke :read [9 nil] INFO [2019-12-17 12:32:17,417] jepsen worker 65 - jepsen.util 4835 :ok :read [21 44] INFO [2019-12-17 12:32:17,417] jepsen worker 65 - jepsen.util 4835 :invoke :read [21 nil] INFO [2019-12-17 12:32:17,422] jepsen worker 55 - jepsen.util 6715 :ok :read [18 36] INFO [2019-12-17 12:32:17,422] jepsen worker 55 - jepsen.util 6715 :invoke :cas [18 [4 41]] INFO [2019-12-17 12:32:17,446] jepsen worker 4 - jepsen.util 5224 :fail :cas [1 [49 37]] :ValueNotSwapFrom INFO [2019-12-17 12:32:17,446] jepsen worker 4 - jepsen.util 5224 :invoke :read [1 nil] INFO [2019-12-17 12:32:17,468] jepsen worker 28 - jepsen.util 4528 :ok :read [9 25] INFO [2019-12-17 12:32:17,468] jepsen worker 28 - jepsen.util 4528 :invoke :read [9 nil] INFO [2019-12-17 12:32:17,474] jepsen worker 3 - jepsen.util 5763 :fail :cas [1 [46 35]] :ValueNotSwapFrom INFO [2019-12-17 12:32:17,475] jepsen worker 3 - jepsen.util 5763 :invoke :cas [1 [17 33]] INFO [2019-12-17 12:32:17,484] jepsen worker 65 - jepsen.util 4835 :ok :read [21 44] INFO [2019-12-17 12:32:17,484] jepsen worker 65 - jepsen.util 4835 :invoke :write [21 34] INFO [2019-12-17 12:32:17,495] jepsen worker 13 - jepsen.util 5683 :ok :read [4 6] INFO [2019-12-17 12:32:17,496] jepsen worker 13 - jepsen.util 5683 :invoke :cas [4 [27 27]] INFO [2019-12-17 12:32:17,506] jepsen worker 28 - jepsen.util 4528 :ok :read [9 25] INFO [2019-12-17 12:32:17,506] jepsen worker 28 - jepsen.util 4528 :invoke :read [9 nil] INFO [2019-12-17 12:32:17,519] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":33219,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:32:17,519] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":33219,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:32:17,526] jepsen worker 50 - jepsen.util 8060 :info :write [16 27] :RequestCanceledException INFO [2019-12-17 12:32:17,533] jepsen worker 50 - jepsen.util 8150 :invoke :write [16 10] INFO [2019-12-17 12:32:17,540] jepsen worker 88 - jepsen.util 5938 :info :write [29 30] :RequestCanceledException INFO [2019-12-17 12:32:17,543] jepsen worker 65 - jepsen.util 4835 :ok :write [21 34] INFO [2019-12-17 12:32:17,544] jepsen worker 65 - jepsen.util 4835 :invoke :read [21 nil] INFO [2019-12-17 12:32:17,546] jepsen worker 88 - jepsen.util 6028 :invoke :read [29 nil] INFO [2019-12-17 12:32:17,558] jepsen worker 12 - jepsen.util 6042 :info :write [4 16] :RequestCanceledException INFO [2019-12-17 12:32:17,560] jepsen worker 12 - jepsen.util 6132 :invoke :cas [4 [27 1]] INFO [2019-12-17 12:32:17,562] jepsen worker 89 - jepsen.util 5579 :ok :read [29 3] INFO [2019-12-17 12:32:17,562] jepsen worker 89 - jepsen.util 5579 :invoke :read [29 nil] INFO [2019-12-17 12:32:17,575] jepsen worker 65 - jepsen.util 4835 :ok :read [21 34] INFO [2019-12-17 12:32:17,575] jepsen worker 65 - jepsen.util 4835 :invoke :cas [21 [38 30]] INFO [2019-12-17 12:32:17,593] jepsen worker 10 - jepsen.util 5590 :info :write [3 45] :RequestCanceledException INFO [2019-12-17 12:32:17,596] jepsen worker 10 - jepsen.util 5680 :invoke :write [3 2] INFO [2019-12-17 12:32:17,597] jepsen worker 80 - jepsen.util 5840 :ok :read [26 47] INFO [2019-12-17 12:32:17,597] jepsen worker 80 - jepsen.util 5840 :invoke :write [26 41] INFO [2019-12-17 12:32:17,605] jepsen worker 65 - jepsen.util 4835 :fail :cas [21 [38 30]] :ValueNotSwapFrom INFO [2019-12-17 12:32:17,605] jepsen worker 65 - jepsen.util 4835 :invoke :write [21 49] INFO [2019-12-17 12:32:17,630] jepsen worker 12 - jepsen.util 6132 :fail :cas [4 [27 1]] :ValueNotSwapFrom INFO [2019-12-17 12:32:17,631] jepsen worker 12 - jepsen.util 6132 :invoke :write [4 35] INFO [2019-12-17 12:32:17,643] jepsen worker 55 - jepsen.util 6715 :fail :cas [18 [4 41]] :ValueNotSwapFrom INFO [2019-12-17 12:32:17,643] jepsen worker 55 - jepsen.util 6715 :invoke :cas [18 [42 46]] INFO [2019-12-17 12:32:17,657] jepsen worker 32 - jepsen.util 5792 :fail :read [10 nil] :RequestTimeoutException INFO [2019-12-17 12:32:17,657] jepsen worker 79 - jepsen.util 5659 :info :write [26 2] :RequestCanceledException INFO [2019-12-17 12:32:17,657] jepsen worker 32 - jepsen.util 5792 :invoke :read [10 nil] INFO [2019-12-17 12:32:17,658] jepsen worker 79 - jepsen.util 5749 :invoke :read [26 nil] INFO [2019-12-17 12:32:17,665] jepsen worker 65 - jepsen.util 4835 :ok :write [21 49] INFO [2019-12-17 12:32:17,668] jepsen worker 65 - jepsen.util 4835 :invoke :write [21 28] INFO [2019-12-17 12:32:17,682] jepsen worker 89 - jepsen.util 5579 :ok :read [29 3] INFO [2019-12-17 12:32:17,683] jepsen worker 89 - jepsen.util 5579 :invoke :write [29 29] INFO [2019-12-17 12:32:17,688] jepsen worker 32 - jepsen.util 5792 :ok :read [10 16] INFO [2019-12-17 12:32:17,688] jepsen worker 32 - jepsen.util 5792 :invoke :write [10 27] INFO [2019-12-17 12:32:17,692] jepsen worker 14 - jepsen.util 5864 :info :write [4 48] :RequestCanceledException INFO [2019-12-17 12:32:17,695] jepsen worker 14 - jepsen.util 5954 :invoke :cas [4 [38 19]] INFO [2019-12-17 12:32:17,705] jepsen worker 3 - jepsen.util 5763 :fail :cas [1 [17 33]] :ValueNotSwapFrom INFO [2019-12-17 12:32:17,705] jepsen worker 3 - jepsen.util 5763 :invoke :read [1 nil] INFO [2019-12-17 12:32:17,721] jepsen worker 13 - jepsen.util 5683 :fail :cas [4 [27 27]] :ValueNotSwapFrom INFO [2019-12-17 12:32:17,721] jepsen worker 13 - jepsen.util 5683 :invoke :cas [4 [35 47]] INFO [2019-12-17 12:32:17,727] jepsen worker 27 - jepsen.util 5607 :info :cas [9 [7 5]] :RequestCanceledException INFO [2019-12-17 12:32:17,730] jepsen worker 27 - jepsen.util 5697 :invoke :cas [9 [14 11]] INFO [2019-12-17 12:32:17,740] jepsen worker 63 - jepsen.util 4743 :info :write [21 11] :RequestCanceledException INFO [2019-12-17 12:32:17,743] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":33243,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:32:17,744] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":33243,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:32:17,744] jepsen worker 63 - jepsen.util 4833 :invoke :cas [21 [25 36]] INFO [2019-12-17 12:32:17,767] jepsen worker 4 - jepsen.util 5224 :ok :read [1 12] INFO [2019-12-17 12:32:17,767] jepsen worker 4 - jepsen.util 5224 :invoke :read [1 nil] INFO [2019-12-17 12:32:17,768] jepsen worker 27 - jepsen.util 5697 :fail :cas [9 [14 11]] :ValueNotSwapFrom INFO [2019-12-17 12:32:17,768] jepsen worker 27 - jepsen.util 5697 :invoke :write [9 32] INFO [2019-12-17 12:32:17,771] jepsen worker 65 - jepsen.util 4835 :ok :write [21 28] INFO [2019-12-17 12:32:17,771] jepsen worker 65 - jepsen.util 4835 :invoke :write [21 26] INFO [2019-12-17 12:32:17,788] jepsen worker 53 - jepsen.util 5543 :info :write [17 42] :RequestCanceledException INFO [2019-12-17 12:32:17,791] jepsen worker 53 - jepsen.util 5633 :invoke :cas [17 [26 10]] INFO [2019-12-17 12:32:17,826] jepsen worker 70 - jepsen.util 6190 :info :write [23 40] :RequestCanceledException INFO [2019-12-17 12:32:17,827] jepsen worker 70 - jepsen.util 6280 :invoke :cas [23 [1 45]] INFO [2019-12-17 12:32:17,841] jepsen worker 27 - jepsen.util 5697 :ok :write [9 32] INFO [2019-12-17 12:32:17,842] jepsen worker 27 - jepsen.util 5697 :invoke :cas [9 [8 19]] INFO [2019-12-17 12:32:17,855] jepsen worker 79 - jepsen.util 5749 :ok :read [26 47] INFO [2019-12-17 12:32:17,855] jepsen worker 79 - jepsen.util 5749 :invoke :write [26 20] INFO [2019-12-17 12:32:17,882] jepsen worker 27 - jepsen.util 5697 :fail :cas [9 [8 19]] :ValueNotSwapFrom INFO [2019-12-17 12:32:17,882] jepsen worker 27 - jepsen.util 5697 :invoke :read [9 nil] INFO [2019-12-17 12:32:17,913] jepsen worker 27 - jepsen.util 5697 :ok :read [9 32] INFO [2019-12-17 12:32:17,913] jepsen worker 27 - jepsen.util 5697 :invoke :write [9 29] INFO [2019-12-17 12:32:17,935] jepsen worker 3 - jepsen.util 5763 :ok :read [1 12] INFO [2019-12-17 12:32:17,935] jepsen worker 3 - jepsen.util 5763 :invoke :write [1 4] INFO [2019-12-17 12:32:17,942] jepsen worker 65 - jepsen.util 4835 :ok :write [21 26] INFO [2019-12-17 12:32:17,943] jepsen worker 65 - jepsen.util 4835 :invoke :cas [21 [15 34]] INFO [2019-12-17 12:32:17,959] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":33264,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:32:17,959] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":33264,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:32:17,960] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":33268,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:32:17,960] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":33268,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:32:17,971] jepsen worker 13 - jepsen.util 5683 :fail :cas [4 [35 47]] :ValueNotSwapFrom INFO [2019-12-17 12:32:17,972] jepsen worker 13 - jepsen.util 5683 :invoke :read [4 nil] INFO [2019-12-17 12:32:17,977] jepsen worker 65 - jepsen.util 4835 :fail :cas [21 [15 34]] :ValueNotSwapFrom INFO [2019-12-17 12:32:17,977] jepsen worker 65 - jepsen.util 4835 :invoke :write [21 8] INFO [2019-12-17 12:32:17,991] jepsen worker 27 - jepsen.util 5697 :ok :write [9 29] INFO [2019-12-17 12:32:17,992] jepsen worker 27 - jepsen.util 5697 :invoke :read [9 nil] INFO [2019-12-17 12:32:18,031] jepsen worker 4 - jepsen.util 5224 :ok :read [1 12] INFO [2019-12-17 12:32:18,031] jepsen worker 4 - jepsen.util 5224 :invoke :read [1 nil] INFO [2019-12-17 12:32:18,037] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:32:18,037] jepsen nemesis - couchbase.util Rebalance check count: 252 INFO [2019-12-17 12:32:18,038] jepsen nemesis - couchbase.util Rebalance stuck count: 233 INFO [2019-12-17 12:32:18,039] jepsen worker 27 - jepsen.util 5697 :ok :read [9 29] INFO [2019-12-17 12:32:18,039] jepsen worker 27 - jepsen.util 5697 :invoke :read [9 nil] INFO [2019-12-17 12:32:18,045] jepsen worker 69 - jepsen.util 6099 :ok :read [23 31] INFO [2019-12-17 12:32:18,045] jepsen worker 69 - jepsen.util 6099 :invoke :read [23 nil] INFO [2019-12-17 12:32:18,050] jepsen worker 14 - jepsen.util 5954 :fail :cas [4 [38 19]] :ValueNotSwapFrom INFO [2019-12-17 12:32:18,050] jepsen worker 14 - jepsen.util 5954 :invoke :write [4 0] INFO [2019-12-17 12:32:18,060] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":33281,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:32:18,060] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":33281,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:32:18,060] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":33282,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:32:18,060] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":33282,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:32:18,069] jepsen worker 65 - jepsen.util 4835 :ok :write [21 8] INFO [2019-12-17 12:32:18,070] jepsen worker 65 - jepsen.util 4835 :invoke :write [21 11] INFO [2019-12-17 12:32:18,075] jepsen worker 27 - jepsen.util 5697 :ok :read [9 29] INFO [2019-12-17 12:32:18,075] jepsen worker 27 - jepsen.util 5697 :invoke :cas [9 [26 38]] INFO [2019-12-17 12:32:18,117] jepsen worker 27 - jepsen.util 5697 :fail :cas [9 [26 38]] :ValueNotSwapFrom INFO [2019-12-17 12:32:18,121] jepsen worker 27 - jepsen.util 5697 :invoke :read [9 nil] INFO [2019-12-17 12:32:18,143] jepsen worker 55 - jepsen.util 6715 :fail :cas [18 [42 46]] :ValueNotSwapFrom INFO [2019-12-17 12:32:18,144] jepsen worker 55 - jepsen.util 6715 :invoke :write [18 29] INFO [2019-12-17 12:32:18,155] jepsen worker 27 - jepsen.util 5697 :ok :read [9 29] INFO [2019-12-17 12:32:18,156] jepsen worker 65 - jepsen.util 4835 :ok :write [21 11] INFO [2019-12-17 12:32:18,156] jepsen worker 27 - jepsen.util 5697 :invoke :read [9 nil] INFO [2019-12-17 12:32:18,157] jepsen worker 65 - jepsen.util 4835 :invoke :read [21 nil] INFO [2019-12-17 12:32:18,191] jepsen worker 65 - jepsen.util 4835 :ok :read [21 11] INFO [2019-12-17 12:32:18,191] jepsen worker 65 - jepsen.util 4835 :invoke :read [21 nil] INFO [2019-12-17 12:32:18,194] jepsen worker 27 - jepsen.util 5697 :ok :read [9 29] INFO [2019-12-17 12:32:18,194] jepsen worker 27 - jepsen.util 5697 :invoke :write [9 42] INFO [2019-12-17 12:32:18,211] jepsen worker 68 - jepsen.util 7268 :info :cas [22 [27 32]] :RequestCanceledException INFO [2019-12-17 12:32:18,214] jepsen worker 68 - jepsen.util 7358 :invoke :cas [22 [22 11]] INFO [2019-12-17 12:32:18,226] jepsen worker 65 - jepsen.util 4835 :ok :read [21 11] INFO [2019-12-17 12:32:18,226] jepsen worker 65 - jepsen.util 4835 :invoke :write [21 47] INFO [2019-12-17 12:32:18,226] jepsen worker 51 - jepsen.util 5631 :info :write [17 10] :RequestCanceledException INFO [2019-12-17 12:32:18,228] jepsen worker 51 - jepsen.util 5721 :invoke :cas [17 [44 28]] INFO [2019-12-17 12:32:18,240] jepsen worker 47 - jepsen.util 6437 :info :write [15 41] :RequestCanceledException INFO [2019-12-17 12:32:18,242] jepsen worker 47 - jepsen.util 6527 :invoke :read [15 nil] INFO [2019-12-17 12:32:18,273] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":33302,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:32:18,274] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":33302,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:32:18,278] jepsen worker 53 - jepsen.util 5633 :fail :cas [17 [26 10]] :ValueNotSwapFrom INFO [2019-12-17 12:32:18,278] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":33306,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:32:18,278] jepsen worker 53 - jepsen.util 5633 :invoke :cas [17 [17 12]] INFO [2019-12-17 12:32:18,278] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":33306,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:32:18,291] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":33294,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:32:18,291] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":33294,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:32:18,291] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":33295,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:32:18,291] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":33295,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:32:18,291] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":33296,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:32:18,291] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":33296,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:32:18,301] jepsen worker 27 - jepsen.util 5697 :ok :write [9 42] INFO [2019-12-17 12:32:18,301] jepsen worker 27 - jepsen.util 5697 :invoke :cas [9 [2 27]] INFO [2019-12-17 12:32:18,320] jepsen worker 5 - jepsen.util 5405 :info :write [1 46] :RequestCanceledException INFO [2019-12-17 12:32:18,320] jepsen worker 75 - jepsen.util 7455 :info :write [25 23] :RequestCanceledException INFO [2019-12-17 12:32:18,320] jepsen worker 66 - jepsen.util 7176 :info :write [22 29] :RequestCanceledException INFO [2019-12-17 12:32:18,321] jepsen worker 65 - jepsen.util 4835 :ok :write [21 47] INFO [2019-12-17 12:32:18,321] jepsen worker 65 - jepsen.util 4835 :invoke :write [21 7] INFO [2019-12-17 12:32:18,325] jepsen worker 5 - jepsen.util 5495 :invoke :write [1 28] INFO [2019-12-17 12:32:18,325] jepsen worker 75 - jepsen.util 7545 :invoke :write [25 47] INFO [2019-12-17 12:32:18,326] jepsen worker 66 - jepsen.util 7266 :invoke :write [22 35] INFO [2019-12-17 12:32:18,356] jepsen worker 47 - jepsen.util 6527 :ok :read [15 48] INFO [2019-12-17 12:32:18,357] jepsen worker 47 - jepsen.util 6527 :invoke :write [15 43] INFO [2019-12-17 12:32:18,357] jepsen worker 44 - jepsen.util 8504 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:32:18,357] jepsen worker 44 - jepsen.util 8504 :invoke :cas [14 [46 42]] INFO [2019-12-17 12:32:18,365] jepsen worker 87 - jepsen.util 5307 :ok :read [29 3] INFO [2019-12-17 12:32:18,365] jepsen worker 87 - jepsen.util 5307 :invoke :read [29 nil] INFO [2019-12-17 12:32:18,378] jepsen worker 27 - jepsen.util 5697 :fail :cas [9 [2 27]] :ValueNotSwapFrom INFO [2019-12-17 12:32:18,378] jepsen worker 27 - jepsen.util 5697 :invoke :cas [9 [23 42]] INFO [2019-12-17 12:32:18,411] jepsen worker 71 - jepsen.util 6461 :info :write [23 21] :RequestCanceledException INFO [2019-12-17 12:32:18,411] jepsen worker 60 - jepsen.util 8610 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:32:18,411] jepsen worker 85 - jepsen.util 8635 :info :cas [28 [40 39]] :RequestCanceledException INFO [2019-12-17 12:32:18,418] jepsen worker 65 - jepsen.util 4835 :ok :write [21 7] INFO [2019-12-17 12:32:18,426] jepsen worker 16 - jepsen.util 8746 :info :write [5 7] :RequestCanceledException INFO [2019-12-17 12:32:18,430] jepsen worker 70 - jepsen.util 6280 :fail :cas [23 [1 45]] :ValueNotSwapFrom INFO [2019-12-17 12:32:18,440] jepsen worker 43 - jepsen.util 7333 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:32:18,440] jepsen worker 82 - jepsen.util 8182 :info :write [27 40] :RequestCanceledException INFO [2019-12-17 12:32:18,454] jepsen worker 57 - jepsen.util 7347 :info :cas [19 [7 48]] :RequestCanceledException INFO [2019-12-17 12:32:18,457] jepsen worker 69 - jepsen.util 6099 :ok :read [23 31] INFO [2019-12-17 12:32:18,469] jepsen worker 20 - jepsen.util 5780 :ok :write [6 44] INFO [2019-12-17 12:32:18,469] jepsen worker 10 - jepsen.util 5680 :ok :write [3 2] INFO [2019-12-17 12:32:18,469] jepsen worker 46 - jepsen.util 5716 :ok :write [15 16] INFO [2019-12-17 12:32:18,470] jepsen worker 74 - jepsen.util 6194 :ok :write [24 32] INFO [2019-12-17 12:32:18,492] jepsen worker 48 - jepsen.util 6978 :info :write [16 13] :RequestCanceledException INFO [2019-12-17 12:32:18,492] jepsen worker 8 - jepsen.util 7298 :info :write [2 33] :RequestCanceledException INFO [2019-12-17 12:32:18,496] jepsen worker 11 - jepsen.util 6041 :ok :read [3 2] INFO [2019-12-17 12:32:18,499] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":33322,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:32:18,511] jepsen worker 60 - jepsen.util 8610 :invoke :cas [20 [1 17]] INFO [2019-12-17 12:32:18,511] jepsen worker 65 - jepsen.util 4835 :invoke :write [21 23] INFO [2019-12-17 12:32:18,511] jepsen worker 70 - jepsen.util 6280 :invoke :read [23 nil] INFO [2019-12-17 12:32:18,511] jepsen worker 43 - jepsen.util 7333 :invoke :read [14 nil] INFO [2019-12-17 12:32:18,511] jepsen worker 69 - jepsen.util 6099 :invoke :read [23 nil] INFO [2019-12-17 12:32:18,511] jepsen worker 20 - jepsen.util 5780 :invoke :cas [6 [7 15]] INFO [2019-12-17 12:32:18,511] jepsen worker 10 - jepsen.util 5680 :invoke :cas [3 [35 25]] INFO [2019-12-17 12:32:18,511] jepsen worker 46 - jepsen.util 5716 :invoke :read [15 nil] INFO [2019-12-17 12:32:18,511] jepsen worker 74 - jepsen.util 6194 :invoke :cas [24 [5 0]] INFO [2019-12-17 12:32:18,511] jepsen worker 11 - jepsen.util 6041 :invoke :read [3 nil] INFO [2019-12-17 12:32:18,512] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":33322,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:32:18,513] jepsen worker 71 - jepsen.util 6551 :invoke :cas [23 [1 49]] INFO [2019-12-17 12:32:18,514] jepsen worker 16 - jepsen.util 8836 :invoke :write [5 47] INFO [2019-12-17 12:32:18,514] jepsen worker 85 - jepsen.util 8725 :invoke :cas [28 [25 5]] INFO [2019-12-17 12:32:18,514] jepsen worker 82 - jepsen.util 8272 :invoke :write [27 10] INFO [2019-12-17 12:32:18,514] jepsen worker 57 - jepsen.util 7437 :invoke :cas [19 [36 8]] INFO [2019-12-17 12:32:18,514] jepsen worker 8 - jepsen.util 7388 :invoke :read [2 nil] INFO [2019-12-17 12:32:18,514] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":33325,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:32:18,515] jepsen worker 48 - jepsen.util 7068 :invoke :write [16 45] INFO [2019-12-17 12:32:18,515] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":33325,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:32:18,515] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":33326,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:32:18,515] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":33326,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:32:18,526] jepsen worker 33 - jepsen.util 7863 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:32:18,527] jepsen worker 33 - jepsen.util 7863 :invoke :read [11 nil] INFO [2019-12-17 12:32:18,540] jepsen worker 29 - jepsen.util 4529 :ok :write [9 20] INFO [2019-12-17 12:32:18,540] jepsen worker 29 - jepsen.util 4529 :invoke :cas [9 [0 5]] INFO [2019-12-17 12:32:18,574] jepsen worker 4 - jepsen.util 5224 :ok :read [1 12] INFO [2019-12-17 12:32:18,575] jepsen worker 4 - jepsen.util 5224 :invoke :write [1 17] INFO [2019-12-17 12:32:18,594] jepsen worker 10 - jepsen.util 5680 :fail :cas [3 [35 25]] :ValueNotSwapFrom INFO [2019-12-17 12:32:18,594] jepsen worker 10 - jepsen.util 5680 :invoke :cas [3 [1 25]] INFO [2019-12-17 12:32:18,604] jepsen worker 20 - jepsen.util 5780 :fail :cas [6 [7 15]] :ValueNotSwapFrom INFO [2019-12-17 12:32:18,605] jepsen worker 20 - jepsen.util 5780 :invoke :read [6 nil] INFO [2019-12-17 12:32:18,624] jepsen worker 78 - jepsen.util 6108 :info :cas [26 [47 0]] :RequestCanceledException INFO [2019-12-17 12:32:18,627] jepsen worker 11 - jepsen.util 6041 :ok :read [3 2] INFO [2019-12-17 12:32:18,627] jepsen worker 11 - jepsen.util 6041 :invoke :read [3 nil] INFO [2019-12-17 12:32:18,628] jepsen worker 78 - jepsen.util 6198 :invoke :cas [26 [7 16]] INFO [2019-12-17 12:32:18,630] jepsen worker 74 - jepsen.util 6194 :fail :cas [24 [5 0]] :ValueNotSwapFrom INFO [2019-12-17 12:32:18,630] jepsen worker 74 - jepsen.util 6194 :invoke :read [24 nil] INFO [2019-12-17 12:32:18,643] jepsen worker 20 - jepsen.util 5780 :ok :read [6 44] INFO [2019-12-17 12:32:18,643] jepsen worker 20 - jepsen.util 5780 :invoke :write [6 7] INFO [2019-12-17 12:32:18,654] jepsen worker 34 - jepsen.util 7234 :info :cas [11 [0 31]] :RequestCanceledException INFO [2019-12-17 12:32:18,658] jepsen worker 34 - jepsen.util 7324 :invoke :write [11 47] INFO [2019-12-17 12:32:18,672] jepsen worker 10 - jepsen.util 5680 :fail :cas [3 [1 25]] :ValueNotSwapFrom INFO [2019-12-17 12:32:18,673] jepsen worker 10 - jepsen.util 5680 :invoke :write [3 31] INFO [2019-12-17 12:32:18,676] jepsen worker 88 - jepsen.util 6028 :ok :read [29 3] INFO [2019-12-17 12:32:18,676] jepsen worker 88 - jepsen.util 6028 :invoke :read [29 nil] INFO [2019-12-17 12:32:18,684] jepsen worker 65 - jepsen.util 4835 :ok :write [21 23] INFO [2019-12-17 12:32:18,684] jepsen worker 65 - jepsen.util 4835 :invoke :read [21 nil] INFO [2019-12-17 12:32:18,693] jepsen worker 41 - jepsen.util 8591 :info :cas [13 [1 10]] :RequestCanceledException INFO [2019-12-17 12:32:18,696] jepsen worker 41 - jepsen.util 8681 :invoke :read [13 nil] INFO [2019-12-17 12:32:18,699] jepsen worker 13 - jepsen.util 5683 :ok :read [4 6] INFO [2019-12-17 12:32:18,699] jepsen worker 13 - jepsen.util 5683 :invoke :write [4 5] INFO [2019-12-17 12:32:18,711] jepsen worker 6 - jepsen.util 8196 :info :cas [2 [5 30]] :RequestCanceledException INFO [2019-12-17 12:32:18,713] jepsen worker 6 - jepsen.util 8286 :invoke :cas [2 [41 18]] INFO [2019-12-17 12:32:18,713] jepsen worker 74 - jepsen.util 6194 :ok :read [24 32] INFO [2019-12-17 12:32:18,714] jepsen worker 74 - jepsen.util 6194 :invoke :cas [24 [4 10]] INFO [2019-12-17 12:32:18,727] jepsen worker 58 - jepsen.util 8248 :info :write [19 18] :RequestCanceledException INFO [2019-12-17 12:32:18,727] jepsen worker 73 - jepsen.util 5383 :info :write [24 4] :RequestCanceledException INFO [2019-12-17 12:32:18,727] jepsen worker 52 - jepsen.util 6172 :info :write [17 20] :RequestCanceledException INFO [2019-12-17 12:32:18,727] jepsen worker 0 - jepsen.util 8010 :info :write [0 48] :RequestCanceledException INFO [2019-12-17 12:32:18,728] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":33352,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:32:18,728] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":33352,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:32:18,728] jepsen worker 52 - jepsen.util 6262 :invoke :read [17 nil] INFO [2019-12-17 12:32:18,728] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":33357,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:32:18,728] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":33357,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:32:18,729] jepsen worker 73 - jepsen.util 5473 :invoke :read [24 nil] INFO [2019-12-17 12:32:18,730] jepsen worker 58 - jepsen.util 8338 :invoke :write [19 30] INFO [2019-12-17 12:32:18,731] jepsen worker 0 - jepsen.util 8100 :invoke :read [0 nil] INFO [2019-12-17 12:32:18,737] jepsen worker 11 - jepsen.util 6041 :ok :read [3 2] INFO [2019-12-17 12:32:18,738] jepsen worker 11 - jepsen.util 6041 :invoke :cas [3 [11 16]] INFO [2019-12-17 12:32:18,740] jepsen worker 81 - jepsen.util 7641 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:32:18,740] jepsen worker 76 - jepsen.util 7816 :info :cas [25 [1 3]] :RequestCanceledException INFO [2019-12-17 12:32:18,740] jepsen worker 81 - jepsen.util 7641 :invoke :read [27 nil] INFO [2019-12-17 12:32:18,742] jepsen worker 76 - jepsen.util 7906 :invoke :cas [25 [12 23]] INFO [2019-12-17 12:32:18,757] jepsen worker 38 - jepsen.util 5888 :info :write [12 39] :RequestCanceledException INFO [2019-12-17 12:32:18,757] jepsen worker 62 - jepsen.util 7532 :info :cas [20 [25 19]] :RequestCanceledException INFO [2019-12-17 12:32:18,759] jepsen worker 38 - jepsen.util 5978 :invoke :cas [12 [6 46]] INFO [2019-12-17 12:32:18,759] jepsen worker 62 - jepsen.util 7622 :invoke :read [20 nil] INFO [2019-12-17 12:32:18,764] jepsen worker 31 - jepsen.util 5161 :ok :write [10 4] INFO [2019-12-17 12:32:18,764] jepsen worker 31 - jepsen.util 5161 :invoke :read [10 nil] INFO [2019-12-17 12:32:18,774] jepsen worker 65 - jepsen.util 4835 :ok :read [21 23] INFO [2019-12-17 12:32:18,774] jepsen worker 65 - jepsen.util 4835 :invoke :cas [21 [31 23]] INFO [2019-12-17 12:32:18,789] jepsen worker 74 - jepsen.util 6194 :fail :cas [24 [4 10]] :ValueNotSwapFrom INFO [2019-12-17 12:32:18,789] jepsen worker 74 - jepsen.util 6194 :invoke :read [24 nil] INFO [2019-12-17 12:32:18,806] jepsen worker 63 - jepsen.util 4833 :fail :cas [21 [25 36]] :ValueNotSwapFrom INFO [2019-12-17 12:32:18,806] jepsen worker 63 - jepsen.util 4833 :invoke :write [21 28] INFO [2019-12-17 12:32:18,810] jepsen worker 83 - jepsen.util 8273 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:32:18,811] jepsen worker 83 - jepsen.util 8273 :invoke :read [27 nil] INFO [2019-12-17 12:32:18,826] jepsen worker 84 - jepsen.util 7374 :info :cas [28 [31 45]] :RequestCanceledException INFO [2019-12-17 12:32:18,827] jepsen worker 39 - jepsen.util 7599 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:32:18,827] jepsen worker 39 - jepsen.util 7599 :invoke :read [13 nil] INFO [2019-12-17 12:32:18,831] jepsen worker 84 - jepsen.util 7464 :invoke :read [28 nil] INFO [2019-12-17 12:32:18,859] jepsen worker 54 - jepsen.util 6174 :info :write [18 44] :RequestCanceledException INFO [2019-12-17 12:32:18,863] jepsen worker 46 - jepsen.util 5716 :ok :read [15 16] INFO [2019-12-17 12:32:18,863] jepsen worker 46 - jepsen.util 5716 :invoke :write [15 35] INFO [2019-12-17 12:32:18,868] jepsen worker 54 - jepsen.util 6264 :invoke :read [18 nil] INFO [2019-12-17 12:32:18,875] jepsen worker 65 - jepsen.util 4835 :fail :cas [21 [31 23]] :ValueNotSwapFrom INFO [2019-12-17 12:32:18,875] jepsen worker 65 - jepsen.util 4835 :invoke :write [21 19] INFO [2019-12-17 12:32:18,887] jepsen worker 29 - jepsen.util 4529 :fail :cas [9 [0 5]] :ValueNotSwapFrom INFO [2019-12-17 12:32:18,887] jepsen worker 29 - jepsen.util 4529 :invoke :write [9 20] INFO [2019-12-17 12:32:18,913] jepsen worker 31 - jepsen.util 5161 :ok :read [10 4] INFO [2019-12-17 12:32:18,913] jepsen worker 71 - jepsen.util 6551 :fail :cas [23 [1 49]] :ValueNotSwapFrom INFO [2019-12-17 12:32:18,913] jepsen worker 31 - jepsen.util 5161 :invoke :write [10 22] INFO [2019-12-17 12:32:18,913] jepsen worker 71 - jepsen.util 6551 :invoke :cas [23 [11 7]] INFO [2019-12-17 12:32:18,924] jepsen worker 74 - jepsen.util 6194 :ok :read [24 32] INFO [2019-12-17 12:32:18,924] jepsen worker 74 - jepsen.util 6194 :invoke :cas [24 [14 21]] INFO [2019-12-17 12:32:18,927] jepsen worker 64 - jepsen.util 4474 :info :cas [21 [28 0]] :RequestCanceledException INFO [2019-12-17 12:32:18,929] jepsen worker 64 - jepsen.util 4564 :invoke :write [21 0] INFO [2019-12-17 12:32:18,940] jepsen worker 86 - jepsen.util 8096 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:32:18,941] jepsen worker 86 - jepsen.util 8096 :invoke :write [28 42] INFO [2019-12-17 12:32:18,945] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":33385,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:32:18,946] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":33385,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:32:18,946] jepsen worker 53 - jepsen.util 5633 :fail :cas [17 [17 12]] :ValueNotSwapFrom INFO [2019-12-17 12:32:18,946] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":33388,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:32:18,946] jepsen worker 53 - jepsen.util 5633 :invoke :cas [17 [48 32]] INFO [2019-12-17 12:32:18,946] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":33388,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:32:18,954] jepsen worker 1 - jepsen.util 7561 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:32:18,955] jepsen worker 1 - jepsen.util 7561 :invoke :write [0 42] INFO [2019-12-17 12:32:18,977] jepsen worker 38 - jepsen.util 5978 :fail :cas [12 [6 46]] :ValueNotSwapFrom INFO [2019-12-17 12:32:18,978] jepsen worker 38 - jepsen.util 5978 :invoke :read [12 nil] INFO [2019-12-17 12:32:18,982] jepsen worker 74 - jepsen.util 6194 :fail :cas [24 [14 21]] :ValueNotSwapFrom INFO [2019-12-17 12:32:18,982] jepsen worker 74 - jepsen.util 6194 :invoke :write [24 45] INFO [2019-12-17 12:32:18,993] jepsen worker 61 - jepsen.util 8971 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:32:18,993] jepsen worker 61 - jepsen.util 8971 :invoke :write [20 4] INFO [2019-12-17 12:32:19,011] jepsen worker 17 - jepsen.util 7937 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:32:19,011] jepsen worker 17 - jepsen.util 7937 :invoke :read [5 nil] INFO [2019-12-17 12:32:19,021] jepsen worker 63 - jepsen.util 4833 :ok :write [21 28] INFO [2019-12-17 12:32:19,021] jepsen worker 63 - jepsen.util 4833 :invoke :cas [21 [37 0]] INFO [2019-12-17 12:32:19,026] jepsen worker 22 - jepsen.util 7762 :info :write [7 46] :RequestCanceledException INFO [2019-12-17 12:32:19,026] jepsen worker 7 - jepsen.util 8197 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:32:19,026] jepsen worker 7 - jepsen.util 8197 :invoke :write [2 49] INFO [2019-12-17 12:32:19,026] jepsen worker 72 - jepsen.util 5652 :info :write [24 15] :RequestCanceledException INFO [2019-12-17 12:32:19,026] jepsen worker 25 - jepsen.util 8305 :info :write [8 31] :RequestCanceledException INFO [2019-12-17 12:32:19,026] jepsen worker 21 - jepsen.util 7221 :info :write [7 30] :RequestCanceledException INFO [2019-12-17 12:32:19,026] jepsen worker 19 - jepsen.util 5779 :fail :read [6 nil] :RequestTimeoutException INFO [2019-12-17 12:32:19,027] jepsen worker 19 - jepsen.util 5779 :invoke :cas [6 [29 23]] INFO [2019-12-17 12:32:19,028] jepsen worker 22 - jepsen.util 7852 :invoke :read [7 nil] INFO [2019-12-17 12:32:19,028] jepsen worker 72 - jepsen.util 5742 :invoke :read [24 nil] INFO [2019-12-17 12:32:19,029] jepsen worker 25 - jepsen.util 8395 :invoke :write [8 44] INFO [2019-12-17 12:32:19,029] jepsen worker 21 - jepsen.util 7311 :invoke :read [7 nil] INFO [2019-12-17 12:32:19,040] jepsen worker 77 - jepsen.util 7907 :info :write [25 49] :RequestCanceledException INFO [2019-12-17 12:32:19,042] jepsen worker 77 - jepsen.util 7997 :invoke :write [25 42] INFO [2019-12-17 12:32:19,047] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":33402,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:32:19,047] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":33402,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:32:19,047] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":33406,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:32:19,047] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":33406,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:32:19,047] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":33407,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:32:19,047] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":33407,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:32:19,055] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:32:19,055] jepsen nemesis - couchbase.util Rebalance check count: 253 INFO [2019-12-17 12:32:19,055] jepsen nemesis - couchbase.util Rebalance stuck count: 234 INFO [2019-12-17 12:32:19,057] jepsen worker 15 - jepsen.util 6855 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:32:19,057] jepsen worker 56 - jepsen.util 6266 :fail :read [18 nil] :RequestTimeoutException INFO [2019-12-17 12:32:19,058] jepsen worker 15 - jepsen.util 6855 :invoke :read [5 nil] INFO [2019-12-17 12:32:19,058] jepsen worker 56 - jepsen.util 6266 :invoke :read [18 nil] INFO [2019-12-17 12:32:19,067] jepsen worker 37 - jepsen.util 5437 :ok :write [12 34] INFO [2019-12-17 12:32:19,068] jepsen worker 37 - jepsen.util 5437 :invoke :write [12 43] INFO [2019-12-17 12:32:19,069] jepsen worker 88 - jepsen.util 6028 :ok :read [29 3] INFO [2019-12-17 12:32:19,070] jepsen worker 88 - jepsen.util 6028 :invoke :cas [29 [17 21]] INFO [2019-12-17 12:32:19,100] jepsen worker 64 - jepsen.util 4564 :ok :write [21 0] INFO [2019-12-17 12:32:19,100] jepsen worker 20 - jepsen.util 5780 :ok :write [6 7] INFO [2019-12-17 12:32:19,100] jepsen worker 64 - jepsen.util 4564 :invoke :read [21 nil] INFO [2019-12-17 12:32:19,100] jepsen worker 20 - jepsen.util 5780 :invoke :read [6 nil] INFO [2019-12-17 12:32:19,126] jepsen worker 67 - jepsen.util 7987 :info :write [22 0] :RequestCanceledException INFO [2019-12-17 12:32:19,128] jepsen worker 67 - jepsen.util 8077 :invoke :read [22 nil] INFO [2019-12-17 12:32:19,140] jepsen worker 26 - jepsen.util 8036 :info :cas [8 [42 27]] :RequestCanceledException INFO [2019-12-17 12:32:19,141] jepsen worker 79 - jepsen.util 5749 :ok :write [26 20] INFO [2019-12-17 12:32:19,141] jepsen worker 79 - jepsen.util 5749 :invoke :read [26 nil] INFO [2019-12-17 12:32:19,143] jepsen worker 26 - jepsen.util 8126 :invoke :cas [8 [37 21]] INFO [2019-12-17 12:32:19,150] jepsen worker 73 - jepsen.util 5473 :ok :read [24 32] INFO [2019-12-17 12:32:19,150] jepsen worker 73 - jepsen.util 5473 :invoke :write [24 49] INFO [2019-12-17 12:32:19,157] jepsen worker 40 - jepsen.util 8140 :info :cas [13 [10 28]] :RequestCanceledException INFO [2019-12-17 12:32:19,161] jepsen worker 40 - jepsen.util 8230 :invoke :cas [13 [9 41]] INFO [2019-12-17 12:32:19,177] jepsen worker 19 - jepsen.util 5779 :fail :cas [6 [29 23]] :ValueNotSwapFrom INFO [2019-12-17 12:32:19,178] jepsen worker 19 - jepsen.util 5779 :invoke :read [6 nil] INFO [2019-12-17 12:32:19,195] jepsen worker 69 - jepsen.util 6099 :ok :read [23 31] INFO [2019-12-17 12:32:19,195] jepsen worker 69 - jepsen.util 6099 :invoke :read [23 nil] INFO [2019-12-17 12:32:19,198] jepsen worker 64 - jepsen.util 4564 :ok :read [21 0] INFO [2019-12-17 12:32:19,198] jepsen worker 64 - jepsen.util 4564 :invoke :read [21 nil] INFO [2019-12-17 12:32:19,208] jepsen worker 11 - jepsen.util 6041 :fail :cas [3 [11 16]] :ValueNotSwapFrom INFO [2019-12-17 12:32:19,209] jepsen worker 11 - jepsen.util 6041 :invoke :read [3 nil] INFO [2019-12-17 12:32:19,227] jepsen worker 24 - jepsen.util 7134 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:32:19,227] jepsen worker 24 - jepsen.util 7134 :invoke :read [8 nil] INFO [2019-12-17 12:32:19,240] jepsen worker 19 - jepsen.util 5779 :ok :read [6 7] INFO [2019-12-17 12:32:19,240] jepsen worker 19 - jepsen.util 5779 :invoke :read [6 nil] INFO [2019-12-17 12:32:19,241] jepsen worker 35 - jepsen.util 7955 :info :cas [11 [23 24]] :RequestCanceledException INFO [2019-12-17 12:32:19,242] jepsen worker 20 - jepsen.util 5780 :ok :read [6 7] INFO [2019-12-17 12:32:19,243] jepsen worker 20 - jepsen.util 5780 :invoke :write [6 46] INFO [2019-12-17 12:32:19,243] jepsen worker 35 - jepsen.util 8045 :invoke :read [11 nil] INFO [2019-12-17 12:32:19,267] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":33443,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:32:19,267] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":33443,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:32:19,267] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":33444,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:32:19,267] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":33444,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:32:19,273] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":33431,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:32:19,273] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":33431,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:32:19,293] jepsen worker 59 - jepsen.util 7889 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:32:19,293] jepsen worker 59 - jepsen.util 7889 :invoke :write [19 21] INFO [2019-12-17 12:32:19,311] jepsen worker 42 - jepsen.util 8322 :info :write [14 44] :RequestCanceledException INFO [2019-12-17 12:32:19,314] jepsen worker 42 - jepsen.util 8412 :invoke :write [14 27] INFO [2019-12-17 12:32:19,318] jepsen worker 64 - jepsen.util 4564 :ok :read [21 0] INFO [2019-12-17 12:32:19,318] jepsen worker 64 - jepsen.util 4564 :invoke :cas [21 [21 17]] INFO [2019-12-17 12:32:19,326] jepsen worker 45 - jepsen.util 5715 :info :write [15 7] :RequestCanceledException INFO [2019-12-17 12:32:19,326] jepsen worker 9 - jepsen.util 6309 :fail :read [3 nil] :RequestTimeoutException INFO [2019-12-17 12:32:19,327] jepsen worker 23 - jepsen.util 7763 :info :cas [7 [8 43]] :RequestCanceledException INFO [2019-12-17 12:32:19,327] jepsen worker 49 - jepsen.util 6889 :info :write [16 37] :RequestCanceledException INFO [2019-12-17 12:32:19,327] jepsen worker 9 - jepsen.util 6309 :invoke :read [3 nil] INFO [2019-12-17 12:32:19,330] jepsen worker 45 - jepsen.util 5805 :invoke :read [15 nil] INFO [2019-12-17 12:32:19,330] jepsen worker 23 - jepsen.util 7853 :invoke :read [7 nil] INFO [2019-12-17 12:32:19,331] jepsen worker 49 - jepsen.util 6979 :invoke :write [16 26] INFO [2019-12-17 12:32:19,336] jepsen worker 11 - jepsen.util 6041 :ok :read [3 2] INFO [2019-12-17 12:32:19,337] jepsen worker 11 - jepsen.util 6041 :invoke :cas [3 [42 47]] INFO [2019-12-17 12:32:19,345] jepsen worker 5 - jepsen.util 5495 :ok :write [1 28] INFO [2019-12-17 12:32:19,345] jepsen worker 5 - jepsen.util 5495 :invoke :cas [1 [30 33]] INFO [2019-12-17 12:32:19,362] jepsen worker 19 - jepsen.util 5779 :ok :read [6 7] INFO [2019-12-17 12:32:19,362] jepsen worker 19 - jepsen.util 5779 :invoke :cas [6 [2 33]] INFO [2019-12-17 12:32:19,376] jepsen worker 70 - jepsen.util 6280 :ok :read [23 31] INFO [2019-12-17 12:32:19,376] jepsen worker 70 - jepsen.util 6280 :invoke :cas [23 [41 32]] INFO [2019-12-17 12:32:19,382] jepsen worker 45 - jepsen.util 5805 :ok :read [15 16] INFO [2019-12-17 12:32:19,382] jepsen worker 45 - jepsen.util 5805 :invoke :cas [15 [38 41]] INFO [2019-12-17 12:32:19,393] jepsen worker 30 - jepsen.util 5880 :info :write [10 43] :RequestCanceledException INFO [2019-12-17 12:32:19,393] jepsen worker 18 - jepsen.util 5688 :fail :read [6 nil] :RequestTimeoutException INFO [2019-12-17 12:32:19,393] jepsen worker 36 - jepsen.util 6156 :info :write [12 5] :RequestCanceledException INFO [2019-12-17 12:32:19,393] jepsen worker 64 - jepsen.util 4564 :fail :cas [21 [21 17]] :ValueNotSwapFrom INFO [2019-12-17 12:32:19,393] jepsen worker 18 - jepsen.util 5688 :invoke :read [6 nil] INFO [2019-12-17 12:32:19,393] jepsen worker 64 - jepsen.util 4564 :invoke :write [21 14] INFO [2019-12-17 12:32:19,396] jepsen worker 36 - jepsen.util 6246 :invoke :read [12 nil] INFO [2019-12-17 12:32:19,396] jepsen worker 30 - jepsen.util 5970 :invoke :write [10 36] INFO [2019-12-17 12:32:19,410] jepsen worker 19 - jepsen.util 5779 :fail :cas [6 [2 33]] :ValueNotSwapFrom INFO [2019-12-17 12:32:19,411] jepsen worker 19 - jepsen.util 5779 :invoke :cas [6 [30 41]] INFO [2019-12-17 12:32:19,425] jepsen worker 10 - jepsen.util 5680 :ok :write [3 31] INFO [2019-12-17 12:32:19,425] jepsen worker 10 - jepsen.util 5680 :invoke :write [3 37] INFO [2019-12-17 12:32:19,425] jepsen worker 46 - jepsen.util 5716 :ok :write [15 35] INFO [2019-12-17 12:32:19,425] jepsen worker 46 - jepsen.util 5716 :invoke :read [15 nil] INFO [2019-12-17 12:32:19,426] jepsen worker 45 - jepsen.util 5805 :fail :cas [15 [38 41]] :ValueNotSwapFrom INFO [2019-12-17 12:32:19,426] jepsen worker 45 - jepsen.util 5805 :invoke :read [15 nil] INFO [2019-12-17 12:32:19,440] jepsen worker 2 - jepsen.util 7472 :info :cas [0 [13 32]] :RequestCanceledException INFO [2019-12-17 12:32:19,443] jepsen worker 2 - jepsen.util 7562 :invoke :write [0 5] INFO [2019-12-17 12:32:19,458] jepsen worker 52 - jepsen.util 6262 :ok :read [17 49] INFO [2019-12-17 12:32:19,459] jepsen worker 52 - jepsen.util 6262 :invoke :write [17 0] INFO [2019-12-17 12:32:19,497] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":33471,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:32:19,497] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":33471,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:32:19,497] jepsen worker 9 - jepsen.util 6309 :ok :read [3 31] INFO [2019-12-17 12:32:19,497] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":33472,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:32:19,497] jepsen worker 9 - jepsen.util 6309 :invoke :cas [3 [45 25]] INFO [2019-12-17 12:32:19,498] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":33472,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:32:19,498] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":33473,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:32:19,498] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":33473,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:32:19,499] jepsen worker 45 - jepsen.util 5805 :ok :read [15 35] INFO [2019-12-17 12:32:19,499] jepsen worker 45 - jepsen.util 5805 :invoke :cas [15 [45 11]] INFO [2019-12-17 12:32:19,504] jepsen worker 13 - jepsen.util 5683 :ok :write [4 5] INFO [2019-12-17 12:32:19,504] jepsen worker 13 - jepsen.util 5683 :invoke :write [4 48] INFO [2019-12-17 12:32:19,519] jepsen worker 29 - jepsen.util 4529 :ok :write [9 20] INFO [2019-12-17 12:32:19,520] jepsen worker 29 - jepsen.util 4529 :invoke :write [9 36] INFO [2019-12-17 12:32:19,543] jepsen worker 45 - jepsen.util 5805 :fail :cas [15 [45 11]] :ValueNotSwapFrom INFO [2019-12-17 12:32:19,544] jepsen worker 45 - jepsen.util 5805 :invoke :write [15 35] INFO [2019-12-17 12:32:19,549] jepsen worker 64 - jepsen.util 4564 :ok :write [21 14] INFO [2019-12-17 12:32:19,550] jepsen worker 64 - jepsen.util 4564 :invoke :write [21 25] INFO [2019-12-17 12:32:19,550] jepsen worker 28 - jepsen.util 4528 :fail :read [9 nil] :RequestTimeoutException INFO [2019-12-17 12:32:19,550] jepsen worker 28 - jepsen.util 4528 :invoke :read [9 nil] INFO [2019-12-17 12:32:19,582] jepsen worker 18 - jepsen.util 5688 :ok :read [6 7] INFO [2019-12-17 12:32:19,582] jepsen worker 18 - jepsen.util 5688 :invoke :read [6 nil] INFO [2019-12-17 12:32:19,622] jepsen worker 71 - jepsen.util 6551 :fail :cas [23 [11 7]] :ValueNotSwapFrom INFO [2019-12-17 12:32:19,622] jepsen worker 71 - jepsen.util 6551 :invoke :read [23 nil] INFO [2019-12-17 12:32:19,625] jepsen worker 28 - jepsen.util 4528 :ok :read [9 20] INFO [2019-12-17 12:32:19,625] jepsen worker 28 - jepsen.util 4528 :invoke :cas [9 [9 11]] INFO [2019-12-17 12:32:19,627] jepsen worker 80 - jepsen.util 5840 :info :write [26 41] :RequestCanceledException INFO [2019-12-17 12:32:19,627] jepsen worker 50 - jepsen.util 8150 :info :write [16 10] :RequestCanceledException INFO [2019-12-17 12:32:19,629] jepsen worker 80 - jepsen.util 5930 :invoke :write [26 40] INFO [2019-12-17 12:32:19,631] jepsen worker 50 - jepsen.util 8240 :invoke :read [16 nil] INFO [2019-12-17 12:32:19,652] jepsen worker 19 - jepsen.util 5779 :fail :cas [6 [30 41]] :ValueNotSwapFrom INFO [2019-12-17 12:32:19,653] jepsen worker 19 - jepsen.util 5779 :invoke :cas [6 [49 14]] INFO [2019-12-17 12:32:19,662] jepsen worker 46 - jepsen.util 5716 :ok :read [15 35] INFO [2019-12-17 12:32:19,662] jepsen worker 46 - jepsen.util 5716 :invoke :read [15 nil] INFO [2019-12-17 12:32:19,708] jepsen worker 79 - jepsen.util 5749 :ok :read [26 20] INFO [2019-12-17 12:32:19,708] jepsen worker 79 - jepsen.util 5749 :invoke :write [26 44] INFO [2019-12-17 12:32:19,711] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":33490,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:32:19,711] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":33490,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:32:19,711] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":33493,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:32:19,711] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":33493,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:32:19,711] jepsen worker 89 - jepsen.util 5579 :info :write [29 29] :RequestCanceledException INFO [2019-12-17 12:32:19,711] jepsen worker 12 - jepsen.util 6132 :info :write [4 35] :RequestCanceledException INFO [2019-12-17 12:32:19,713] jepsen worker 89 - jepsen.util 5669 :invoke :write [29 13] INFO [2019-12-17 12:32:19,713] jepsen worker 12 - jepsen.util 6222 :invoke :write [4 40] INFO [2019-12-17 12:32:19,727] jepsen worker 29 - jepsen.util 4529 :ok :write [9 36] INFO [2019-12-17 12:32:19,727] jepsen worker 29 - jepsen.util 4529 :invoke :read [9 nil] INFO [2019-12-17 12:32:19,751] jepsen worker 64 - jepsen.util 4564 :ok :write [21 25] INFO [2019-12-17 12:32:19,751] jepsen worker 64 - jepsen.util 4564 :invoke :read [21 nil] INFO [2019-12-17 12:32:19,756] jepsen worker 32 - jepsen.util 5792 :info :write [10 27] :RequestCanceledException INFO [2019-12-17 12:32:19,761] jepsen worker 32 - jepsen.util 5882 :invoke :write [10 32] INFO [2019-12-17 12:32:19,762] jepsen worker 46 - jepsen.util 5716 :ok :read [15 35] INFO [2019-12-17 12:32:19,762] jepsen worker 46 - jepsen.util 5716 :invoke :read [15 nil] INFO [2019-12-17 12:32:19,791] jepsen worker 70 - jepsen.util 6280 :fail :cas [23 [41 32]] :ValueNotSwapFrom INFO [2019-12-17 12:32:19,791] jepsen worker 70 - jepsen.util 6280 :invoke :cas [23 [21 5]] INFO [2019-12-17 12:32:19,812] jepsen worker 51 - jepsen.util 5721 :fail :cas [17 [44 28]] :ValueNotSwapFrom INFO [2019-12-17 12:32:19,812] jepsen worker 51 - jepsen.util 5721 :invoke :write [17 18] INFO [2019-12-17 12:32:19,813] jepsen worker 88 - jepsen.util 6028 :fail :cas [29 [17 21]] :ValueNotSwapFrom INFO [2019-12-17 12:32:19,813] jepsen worker 88 - jepsen.util 6028 :invoke :read [29 nil] INFO [2019-12-17 12:32:19,817] jepsen worker 29 - jepsen.util 4529 :ok :read [9 36] INFO [2019-12-17 12:32:19,817] jepsen worker 29 - jepsen.util 4529 :invoke :cas [9 [39 47]] INFO [2019-12-17 12:32:19,849] jepsen worker 19 - jepsen.util 5779 :fail :cas [6 [49 14]] :ValueNotSwapFrom INFO [2019-12-17 12:32:19,849] jepsen worker 19 - jepsen.util 5779 :invoke :write [6 49] INFO [2019-12-17 12:32:19,866] jepsen worker 46 - jepsen.util 5716 :ok :read [15 35] INFO [2019-12-17 12:32:19,866] jepsen worker 46 - jepsen.util 5716 :invoke :write [15 14] INFO [2019-12-17 12:32:19,893] jepsen worker 53 - jepsen.util 5633 :fail :cas [17 [48 32]] :ValueNotSwapFrom INFO [2019-12-17 12:32:19,894] jepsen worker 53 - jepsen.util 5633 :invoke :cas [17 [40 17]] INFO [2019-12-17 12:32:19,913] jepsen worker 87 - jepsen.util 5307 :ok :read [29 3] INFO [2019-12-17 12:32:19,914] jepsen worker 87 - jepsen.util 5307 :invoke :write [29 10] INFO [2019-12-17 12:32:19,931] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":33509,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:32:19,931] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":33509,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:32:19,931] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":33512,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:32:19,931] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":33512,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:32:19,931] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":33513,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:32:19,932] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":33513,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:32:19,952] jepsen worker 29 - jepsen.util 4529 :fail :cas [9 [39 47]] :ValueNotSwapFrom INFO [2019-12-17 12:32:19,952] jepsen worker 29 - jepsen.util 4529 :invoke :write [9 36] INFO [2019-12-17 12:32:19,969] jepsen worker 38 - jepsen.util 5978 :ok :read [12 34] INFO [2019-12-17 12:32:19,969] jepsen worker 38 - jepsen.util 5978 :invoke :cas [12 [47 18]] INFO [2019-12-17 12:32:19,979] jepsen worker 64 - jepsen.util 4564 :ok :read [21 25] INFO [2019-12-17 12:32:19,979] jepsen worker 64 - jepsen.util 4564 :invoke :read [21 nil] INFO [2019-12-17 12:32:19,982] jepsen worker 78 - jepsen.util 6198 :fail :cas [26 [7 16]] :ValueNotSwapFrom INFO [2019-12-17 12:32:19,982] jepsen worker 78 - jepsen.util 6198 :invoke :cas [26 [13 8]] INFO [2019-12-17 12:32:20,021] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":33523,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:32:20,021] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":33523,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:32:20,021] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":33526,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:32:20,021] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":33526,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:32:20,021] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":33527,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:32:20,022] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":33527,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:32:20,038] jepsen worker 3 - jepsen.util 5763 :info :write [1 4] :RequestCanceledException INFO [2019-12-17 12:32:20,042] jepsen worker 64 - jepsen.util 4564 :ok :read [21 25] INFO [2019-12-17 12:32:20,042] jepsen worker 64 - jepsen.util 4564 :invoke :read [21 nil] INFO [2019-12-17 12:32:20,044] jepsen worker 3 - jepsen.util 5853 :invoke :read [1 nil] INFO [2019-12-17 12:32:20,044] jepsen worker 37 - jepsen.util 5437 :ok :write [12 43] INFO [2019-12-17 12:32:20,044] jepsen worker 37 - jepsen.util 5437 :invoke :write [12 17] INFO [2019-12-17 12:32:20,063] jepsen worker 5 - jepsen.util 5495 :fail :cas [1 [30 33]] :ValueNotSwapFrom INFO [2019-12-17 12:32:20,063] jepsen worker 5 - jepsen.util 5495 :invoke :read [1 nil] INFO [2019-12-17 12:32:20,066] jepsen worker 74 - jepsen.util 6194 :ok :write [24 45] INFO [2019-12-17 12:32:20,066] jepsen worker 74 - jepsen.util 6194 :invoke :cas [24 [26 26]] INFO [2019-12-17 12:32:20,067] jepsen worker 20 - jepsen.util 5780 :ok :write [6 46] INFO [2019-12-17 12:32:20,068] jepsen worker 20 - jepsen.util 5780 :invoke :read [6 nil] INFO [2019-12-17 12:32:20,072] jepsen worker 30 - jepsen.util 5970 :ok :write [10 36] INFO [2019-12-17 12:32:20,072] jepsen worker 30 - jepsen.util 5970 :invoke :cas [10 [11 22]] INFO [2019-12-17 12:32:20,076] jepsen worker 64 - jepsen.util 4564 :ok :read [21 25] INFO [2019-12-17 12:32:20,076] jepsen worker 64 - jepsen.util 4564 :invoke :write [21 49] INFO [2019-12-17 12:32:20,082] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:32:20,082] jepsen nemesis - couchbase.util Rebalance check count: 254 INFO [2019-12-17 12:32:20,082] jepsen nemesis - couchbase.util Rebalance stuck count: 235 INFO [2019-12-17 12:32:20,115] jepsen worker 69 - jepsen.util 6099 :ok :read [23 31] INFO [2019-12-17 12:32:20,115] jepsen worker 69 - jepsen.util 6099 :invoke :cas [23 [11 24]] INFO [2019-12-17 12:32:20,151] jepsen worker 14 - jepsen.util 5954 :info :write [4 0] :RequestCanceledException INFO [2019-12-17 12:32:20,155] jepsen worker 14 - jepsen.util 6044 :invoke :read [4 nil] INFO [2019-12-17 12:32:20,176] jepsen worker 74 - jepsen.util 6194 :fail :cas [24 [26 26]] :ValueNotSwapFrom INFO [2019-12-17 12:32:20,176] jepsen worker 74 - jepsen.util 6194 :invoke :write [24 38] INFO [2019-12-17 12:32:20,177] jepsen worker 70 - jepsen.util 6280 :fail :cas [23 [21 5]] :ValueNotSwapFrom INFO [2019-12-17 12:32:20,177] jepsen worker 70 - jepsen.util 6280 :invoke :read [23 nil] INFO [2019-12-17 12:32:20,184] jepsen worker 11 - jepsen.util 6041 :fail :cas [3 [42 47]] :ValueNotSwapFrom INFO [2019-12-17 12:32:20,184] jepsen worker 11 - jepsen.util 6041 :invoke :cas [3 [47 48]] INFO [2019-12-17 12:32:20,184] jepsen worker 9 - jepsen.util 6309 :fail :cas [3 [45 25]] :ValueNotSwapFrom INFO [2019-12-17 12:32:20,185] jepsen worker 9 - jepsen.util 6309 :invoke :cas [3 [22 29]] INFO [2019-12-17 12:32:20,185] jepsen worker 13 - jepsen.util 5683 :ok :write [4 48] INFO [2019-12-17 12:32:20,185] jepsen worker 13 - jepsen.util 5683 :invoke :cas [4 [45 15]] INFO [2019-12-17 12:32:20,193] jepsen worker 20 - jepsen.util 5780 :ok :read [6 46] INFO [2019-12-17 12:32:20,194] jepsen worker 20 - jepsen.util 5780 :invoke :read [6 nil] INFO [2019-12-17 12:32:20,226] jepsen worker 55 - jepsen.util 6715 :info :write [18 29] :RequestCanceledException INFO [2019-12-17 12:32:20,231] jepsen worker 55 - jepsen.util 6805 :invoke :read [18 nil] INFO [2019-12-17 12:32:20,234] jepsen worker 11 - jepsen.util 6041 :fail :cas [3 [47 48]] :ValueNotSwapFrom INFO [2019-12-17 12:32:20,234] jepsen worker 11 - jepsen.util 6041 :invoke :read [3 nil] INFO [2019-12-17 12:32:20,240] jepsen worker 68 - jepsen.util 7358 :info :cas [22 [22 11]] :RequestCanceledException INFO [2019-12-17 12:32:20,242] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":33551,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:32:20,242] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":33551,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:32:20,242] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":33555,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:32:20,242] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":33555,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:32:20,243] jepsen worker 71 - jepsen.util 6551 :ok :read [23 31] INFO [2019-12-17 12:32:20,243] jepsen worker 71 - jepsen.util 6551 :invoke :cas [23 [48 26]] INFO [2019-12-17 12:32:20,243] jepsen worker 68 - jepsen.util 7448 :invoke :cas [22 [19 24]] INFO [2019-12-17 12:32:20,246] jepsen worker 29 - jepsen.util 4529 :ok :write [9 36] INFO [2019-12-17 12:32:20,246] jepsen worker 29 - jepsen.util 4529 :invoke :write [9 24] INFO [2019-12-17 12:32:20,247] jepsen worker 45 - jepsen.util 5805 :ok :write [15 35] INFO [2019-12-17 12:32:20,247] jepsen worker 45 - jepsen.util 5805 :invoke :read [15 nil] INFO [2019-12-17 12:32:20,250] jepsen worker 36 - jepsen.util 6246 :ok :read [12 43] INFO [2019-12-17 12:32:20,250] jepsen worker 36 - jepsen.util 6246 :invoke :cas [12 [40 26]] INFO [2019-12-17 12:32:20,253] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":33543,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:32:20,253] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":33543,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:32:20,253] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":33544,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:32:20,253] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":33544,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:32:20,253] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":33545,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:32:20,254] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":33545,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:32:20,281] jepsen worker 28 - jepsen.util 4528 :fail :cas [9 [9 11]] :ValueNotSwapFrom INFO [2019-12-17 12:32:20,281] jepsen worker 28 - jepsen.util 4528 :invoke :cas [9 [49 32]] INFO [2019-12-17 12:32:20,287] jepsen worker 20 - jepsen.util 5780 :ok :read [6 46] INFO [2019-12-17 12:32:20,288] jepsen worker 20 - jepsen.util 5780 :invoke :write [6 31] INFO [2019-12-17 12:32:20,311] jepsen worker 11 - jepsen.util 6041 :ok :read [3 31] INFO [2019-12-17 12:32:20,311] jepsen worker 11 - jepsen.util 6041 :invoke :cas [3 [27 45]] INFO [2019-12-17 12:32:20,321] jepsen worker 45 - jepsen.util 5805 :ok :read [15 35] INFO [2019-12-17 12:32:20,321] jepsen worker 45 - jepsen.util 5805 :invoke :read [15 nil] INFO [2019-12-17 12:32:20,343] jepsen worker 28 - jepsen.util 4528 :fail :cas [9 [49 32]] :ValueNotSwapFrom INFO [2019-12-17 12:32:20,344] jepsen worker 28 - jepsen.util 4528 :invoke :read [9 nil] INFO [2019-12-17 12:32:20,357] jepsen worker 3 - jepsen.util 5853 :ok :read [1 28] INFO [2019-12-17 12:32:20,357] jepsen worker 3 - jepsen.util 5853 :invoke :write [1 2] INFO [2019-12-17 12:32:20,363] jepsen worker 45 - jepsen.util 5805 :ok :read [15 35] INFO [2019-12-17 12:32:20,363] jepsen worker 45 - jepsen.util 5805 :invoke :write [15 30] INFO [2019-12-17 12:32:20,383] jepsen worker 55 - jepsen.util 6805 :ok :read [18 36] INFO [2019-12-17 12:32:20,384] jepsen worker 55 - jepsen.util 6805 :invoke :write [18 21] INFO [2019-12-17 12:32:20,401] jepsen worker 70 - jepsen.util 6280 :ok :read [23 31] INFO [2019-12-17 12:32:20,401] jepsen worker 70 - jepsen.util 6280 :invoke :write [23 33] INFO [2019-12-17 12:32:20,404] jepsen worker 75 - jepsen.util 7545 :info :write [25 47] :RequestCanceledException INFO [2019-12-17 12:32:20,407] jepsen worker 75 - jepsen.util 7635 :invoke :read [25 nil] INFO [2019-12-17 12:32:20,422] jepsen worker 53 - jepsen.util 5633 :fail :cas [17 [40 17]] :ValueNotSwapFrom INFO [2019-12-17 12:32:20,422] jepsen worker 53 - jepsen.util 5633 :invoke :write [17 39] INFO [2019-12-17 12:32:20,423] jepsen worker 64 - jepsen.util 4564 :ok :write [21 49] INFO [2019-12-17 12:32:20,423] jepsen worker 64 - jepsen.util 4564 :invoke :read [21 nil] INFO [2019-12-17 12:32:20,426] jepsen worker 66 - jepsen.util 7266 :info :write [22 35] :RequestCanceledException INFO [2019-12-17 12:32:20,426] jepsen worker 47 - jepsen.util 6527 :info :write [15 43] :RequestCanceledException INFO [2019-12-17 12:32:20,426] jepsen worker 27 - jepsen.util 5697 :info :cas [9 [23 42]] :RequestCanceledException INFO [2019-12-17 12:32:20,429] jepsen worker 27 - jepsen.util 5787 :invoke :read [9 nil] INFO [2019-12-17 12:32:20,429] jepsen worker 47 - jepsen.util 6617 :invoke :write [15 37] INFO [2019-12-17 12:32:20,430] jepsen worker 66 - jepsen.util 7356 :invoke :cas [22 [7 21]] INFO [2019-12-17 12:32:20,442] jepsen worker 28 - jepsen.util 4528 :ok :read [9 36] INFO [2019-12-17 12:32:20,442] jepsen worker 28 - jepsen.util 4528 :invoke :write [9 28] INFO [2019-12-17 12:32:20,457] jepsen worker 44 - jepsen.util 8504 :info :cas [14 [46 42]] :RequestCanceledException INFO [2019-12-17 12:32:20,460] jepsen worker 44 - jepsen.util 8594 :invoke :write [14 28] INFO [2019-12-17 12:32:20,466] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":33575,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:32:20,466] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":33575,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:32:20,466] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":33579,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:32:20,467] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":33579,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:32:20,469] jepsen worker 29 - jepsen.util 4529 :ok :write [9 24] INFO [2019-12-17 12:32:20,469] jepsen worker 29 - jepsen.util 4529 :invoke :read [9 nil] INFO [2019-12-17 12:32:20,471] jepsen worker 32 - jepsen.util 5882 :ok :write [10 32] INFO [2019-12-17 12:32:20,471] jepsen worker 32 - jepsen.util 5882 :invoke :read [10 nil] INFO [2019-12-17 12:32:20,478] jepsen worker 64 - jepsen.util 4564 :ok :read [21 49] INFO [2019-12-17 12:32:20,479] jepsen worker 64 - jepsen.util 4564 :invoke :write [21 11] INFO [2019-12-17 12:32:20,491] jepsen worker 54 - jepsen.util 6264 :ok :read [18 36] INFO [2019-12-17 12:32:20,491] jepsen worker 54 - jepsen.util 6264 :invoke :cas [18 [33 22]] INFO [2019-12-17 12:32:20,510] jepsen worker 30 - jepsen.util 5970 :fail :cas [10 [11 22]] :ValueNotSwapFrom INFO [2019-12-17 12:32:20,510] jepsen worker 30 - jepsen.util 5970 :invoke :cas [10 [21 27]] INFO [2019-12-17 12:32:20,519] jepsen worker 29 - jepsen.util 4529 :ok :read [9 24] INFO [2019-12-17 12:32:20,519] jepsen worker 29 - jepsen.util 4529 :invoke :cas [9 [30 0]] INFO [2019-12-17 12:32:20,525] jepsen worker 8 - jepsen.util 7388 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:32:20,525] jepsen worker 85 - jepsen.util 8725 :info :cas [28 [25 5]] :RequestCanceledException INFO [2019-12-17 12:32:20,525] jepsen worker 8 - jepsen.util 7388 :invoke :write [2 3] INFO [2019-12-17 12:32:20,528] jepsen worker 85 - jepsen.util 8815 :invoke :read [28 nil] INFO [2019-12-17 12:32:20,536] jepsen worker 13 - jepsen.util 5683 :fail :cas [4 [45 15]] :ValueNotSwapFrom INFO [2019-12-17 12:32:20,536] jepsen worker 13 - jepsen.util 5683 :invoke :read [4 nil] INFO [2019-12-17 12:32:20,540] jepsen worker 43 - jepsen.util 7333 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:32:20,540] jepsen worker 48 - jepsen.util 7068 :info :write [16 45] :RequestCanceledException INFO [2019-12-17 12:32:20,540] jepsen worker 43 - jepsen.util 7333 :invoke :cas [14 [17 24]] INFO [2019-12-17 12:32:20,543] jepsen worker 48 - jepsen.util 7158 :invoke :read [16 nil] INFO [2019-12-17 12:32:20,554] jepsen worker 9 - jepsen.util 6309 :fail :cas [3 [22 29]] :ValueNotSwapFrom INFO [2019-12-17 12:32:20,555] jepsen worker 9 - jepsen.util 6309 :invoke :read [3 nil] INFO [2019-12-17 12:32:20,558] jepsen worker 16 - jepsen.util 8836 :info :write [5 47] :RequestCanceledException INFO [2019-12-17 12:32:20,560] jepsen worker 16 - jepsen.util 8926 :invoke :cas [5 [14 31]] INFO [2019-12-17 12:32:20,565] jepsen worker 52 - jepsen.util 6262 :ok :write [17 0] INFO [2019-12-17 12:32:20,565] jepsen worker 52 - jepsen.util 6262 :invoke :cas [17 [29 4]] INFO [2019-12-17 12:32:20,574] jepsen worker 71 - jepsen.util 6551 :fail :cas [23 [48 26]] :ValueNotSwapFrom INFO [2019-12-17 12:32:20,574] jepsen worker 71 - jepsen.util 6551 :invoke :cas [23 [12 2]] INFO [2019-12-17 12:32:20,592] jepsen worker 82 - jepsen.util 8272 :info :write [27 10] :RequestCanceledException INFO [2019-12-17 12:32:20,595] jepsen worker 27 - jepsen.util 5787 :ok :read [9 24] INFO [2019-12-17 12:32:20,595] jepsen worker 27 - jepsen.util 5787 :invoke :cas [9 [49 43]] INFO [2019-12-17 12:32:20,596] jepsen worker 82 - jepsen.util 8362 :invoke :write [27 15] INFO [2019-12-17 12:32:20,600] jepsen worker 29 - jepsen.util 4529 :fail :cas [9 [30 0]] :ValueNotSwapFrom INFO [2019-12-17 12:32:20,600] jepsen worker 29 - jepsen.util 4529 :invoke :write [9 2] INFO [2019-12-17 12:32:20,608] jepsen worker 60 - jepsen.util 8610 :info :cas [20 [1 17]] :RequestCanceledException INFO [2019-12-17 12:32:20,608] jepsen worker 57 - jepsen.util 7437 :info :cas [19 [36 8]] :RequestCanceledException INFO [2019-12-17 12:32:20,611] jepsen worker 60 - jepsen.util 8700 :invoke :cas [20 [41 39]] INFO [2019-12-17 12:32:20,612] jepsen worker 57 - jepsen.util 7527 :invoke :cas [19 [44 48]] INFO [2019-12-17 12:32:20,612] jepsen worker 32 - jepsen.util 5882 :ok :read [10 32] INFO [2019-12-17 12:32:20,612] jepsen worker 32 - jepsen.util 5882 :invoke :cas [10 [21 6]] INFO [2019-12-17 12:32:20,618] jepsen worker 33 - jepsen.util 7863 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:32:20,618] jepsen worker 33 - jepsen.util 7863 :invoke :cas [11 [4 7]] INFO [2019-12-17 12:32:20,619] jepsen worker 64 - jepsen.util 4564 :ok :write [21 11] INFO [2019-12-17 12:32:20,619] jepsen worker 64 - jepsen.util 4564 :invoke :write [21 35] INFO [2019-12-17 12:32:20,637] jepsen worker 30 - jepsen.util 5970 :fail :cas [10 [21 27]] :ValueNotSwapFrom INFO [2019-12-17 12:32:20,637] jepsen worker 30 - jepsen.util 5970 :invoke :read [10 nil] INFO [2019-12-17 12:32:20,640] jepsen worker 4 - jepsen.util 5224 :info :write [1 17] :RequestCanceledException INFO [2019-12-17 12:32:20,643] jepsen worker 4 - jepsen.util 5314 :invoke :read [1 nil] INFO [2019-12-17 12:32:20,647] jepsen worker 56 - jepsen.util 6266 :ok :read [18 36] INFO [2019-12-17 12:32:20,647] jepsen worker 56 - jepsen.util 6266 :invoke :write [18 29] INFO [2019-12-17 12:32:20,656] jepsen worker 9 - jepsen.util 6309 :ok :read [3 31] INFO [2019-12-17 12:32:20,656] jepsen worker 9 - jepsen.util 6309 :invoke :read [3 nil] INFO [2019-12-17 12:32:20,658] jepsen worker 12 - jepsen.util 6222 :ok :write [4 40] INFO [2019-12-17 12:32:20,659] jepsen worker 12 - jepsen.util 6222 :invoke :cas [4 [5 23]] INFO [2019-12-17 12:32:20,683] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":33611,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:32:20,683] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":33611,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:32:20,683] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":33614,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:32:20,683] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":33614,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:32:20,683] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":33615,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:32:20,683] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":33615,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:32:20,692] jepsen worker 27 - jepsen.util 5787 :fail :cas [9 [49 43]] :ValueNotSwapFrom INFO [2019-12-17 12:32:20,693] jepsen worker 27 - jepsen.util 5787 :invoke :cas [9 [46 49]] INFO [2019-12-17 12:32:20,695] jepsen worker 38 - jepsen.util 5978 :fail :cas [12 [47 18]] :ValueNotSwapFrom INFO [2019-12-17 12:32:20,695] jepsen worker 38 - jepsen.util 5978 :invoke :cas [12 [41 0]] INFO [2019-12-17 12:32:20,713] jepsen worker 79 - jepsen.util 5749 :ok :write [26 44] INFO [2019-12-17 12:32:20,714] jepsen worker 79 - jepsen.util 5749 :invoke :write [26 31] INFO [2019-12-17 12:32:20,724] jepsen worker 34 - jepsen.util 7324 :info :write [11 47] :RequestCanceledException INFO [2019-12-17 12:32:20,727] jepsen worker 34 - jepsen.util 7414 :invoke :write [11 9] INFO [2019-12-17 12:32:20,752] jepsen worker 13 - jepsen.util 5683 :ok :read [4 40] INFO [2019-12-17 12:32:20,753] jepsen worker 13 - jepsen.util 5683 :invoke :cas [4 [4 2]] INFO [2019-12-17 12:32:20,754] jepsen worker 58 - jepsen.util 8338 :info :write [19 30] :RequestCanceledException INFO [2019-12-17 12:32:20,756] jepsen worker 58 - jepsen.util 8428 :invoke :write [19 30] INFO [2019-12-17 12:32:20,785] jepsen worker 37 - jepsen.util 5437 :ok :write [12 17] INFO [2019-12-17 12:32:20,785] jepsen worker 37 - jepsen.util 5437 :invoke :write [12 19] INFO [2019-12-17 12:32:20,788] jepsen worker 32 - jepsen.util 5882 :fail :cas [10 [21 6]] :ValueNotSwapFrom INFO [2019-12-17 12:32:20,789] jepsen worker 32 - jepsen.util 5882 :invoke :read [10 nil] INFO [2019-12-17 12:32:20,793] jepsen worker 76 - jepsen.util 7906 :info :cas [25 [12 23]] :RequestCanceledException INFO [2019-12-17 12:32:20,793] jepsen worker 41 - jepsen.util 8681 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:32:20,794] jepsen worker 41 - jepsen.util 8681 :invoke :cas [13 [33 9]] INFO [2019-12-17 12:32:20,795] jepsen worker 76 - jepsen.util 7996 :invoke :write [25 28] INFO [2019-12-17 12:32:20,811] jepsen worker 6 - jepsen.util 8286 :info :cas [2 [41 18]] :RequestCanceledException INFO [2019-12-17 12:32:20,811] jepsen worker 30 - jepsen.util 5970 :ok :read [10 32] INFO [2019-12-17 12:32:20,812] jepsen worker 30 - jepsen.util 5970 :invoke :cas [10 [4 20]] INFO [2019-12-17 12:32:20,815] jepsen worker 6 - jepsen.util 8376 :invoke :write [2 40] INFO [2019-12-17 12:32:20,823] jepsen worker 89 - jepsen.util 5669 :ok :write [29 13] INFO [2019-12-17 12:32:20,823] jepsen worker 89 - jepsen.util 5669 :invoke :write [29 9] INFO [2019-12-17 12:32:20,825] jepsen worker 0 - jepsen.util 8100 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:32:20,825] jepsen worker 62 - jepsen.util 7622 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:32:20,826] jepsen worker 0 - jepsen.util 8100 :invoke :write [0 3] INFO [2019-12-17 12:32:20,826] jepsen worker 62 - jepsen.util 7622 :invoke :read [20 nil] INFO [2019-12-17 12:32:20,829] jepsen worker 9 - jepsen.util 6309 :ok :read [3 31] INFO [2019-12-17 12:32:20,830] jepsen worker 9 - jepsen.util 6309 :invoke :read [3 nil] INFO [2019-12-17 12:32:20,839] jepsen worker 27 - jepsen.util 5787 :fail :cas [9 [46 49]] :ValueNotSwapFrom INFO [2019-12-17 12:32:20,840] jepsen worker 27 - jepsen.util 5787 :invoke :write [9 0] INFO [2019-12-17 12:32:20,840] jepsen worker 81 - jepsen.util 7641 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:32:20,840] jepsen worker 81 - jepsen.util 7641 :invoke :read [27 nil] INFO [2019-12-17 12:32:20,855] jepsen worker 32 - jepsen.util 5882 :ok :read [10 32] INFO [2019-12-17 12:32:20,855] jepsen worker 32 - jepsen.util 5882 :invoke :cas [10 [13 1]] INFO [2019-12-17 12:32:20,871] jepsen worker 52 - jepsen.util 6262 :fail :cas [17 [29 4]] :ValueNotSwapFrom INFO [2019-12-17 12:32:20,871] jepsen worker 52 - jepsen.util 6262 :invoke :read [17 nil] INFO [2019-12-17 12:32:20,889] jepsen worker 69 - jepsen.util 6099 :fail :cas [23 [11 24]] :ValueNotSwapFrom INFO [2019-12-17 12:32:20,890] jepsen worker 69 - jepsen.util 6099 :invoke :write [23 24] INFO [2019-12-17 12:32:20,902] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":33640,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:32:20,902] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":33640,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:32:20,903] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":33643,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:32:20,903] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":33643,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:32:20,912] jepsen worker 83 - jepsen.util 8273 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:32:20,912] jepsen worker 83 - jepsen.util 8273 :invoke :read [27 nil] INFO [2019-12-17 12:32:20,912] jepsen worker 30 - jepsen.util 5970 :fail :cas [10 [4 20]] :ValueNotSwapFrom INFO [2019-12-17 12:32:20,912] jepsen worker 30 - jepsen.util 5970 :invoke :read [10 nil] INFO [2019-12-17 12:32:20,918] jepsen worker 29 - jepsen.util 4529 :ok :write [9 2] INFO [2019-12-17 12:32:20,918] jepsen worker 29 - jepsen.util 4529 :invoke :write [9 29] INFO [2019-12-17 12:32:20,926] jepsen worker 39 - jepsen.util 7599 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:32:20,926] jepsen worker 39 - jepsen.util 7599 :invoke :read [13 nil] INFO [2019-12-17 12:32:20,940] jepsen worker 84 - jepsen.util 7464 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:32:20,940] jepsen worker 84 - jepsen.util 7464 :invoke :write [28 49] INFO [2019-12-17 12:32:20,958] jepsen worker 88 - jepsen.util 6028 :ok :read [29 13] INFO [2019-12-17 12:32:20,958] jepsen worker 65 - jepsen.util 4835 :info :write [21 19] :RequestCanceledException INFO [2019-12-17 12:32:20,958] jepsen worker 88 - jepsen.util 6028 :invoke :write [29 20] INFO [2019-12-17 12:32:20,960] jepsen worker 65 - jepsen.util 4925 :invoke :cas [21 [47 41]] INFO [2019-12-17 12:32:20,965] jepsen worker 9 - jepsen.util 6309 :ok :read [3 31] INFO [2019-12-17 12:32:20,965] jepsen worker 9 - jepsen.util 6309 :invoke :read [3 nil] INFO [2019-12-17 12:32:20,967] jepsen worker 64 - jepsen.util 4564 :ok :write [21 35] INFO [2019-12-17 12:32:20,967] jepsen worker 64 - jepsen.util 4564 :invoke :write [21 33] INFO [2019-12-17 12:32:20,968] jepsen worker 36 - jepsen.util 6246 :fail :cas [12 [40 26]] :ValueNotSwapFrom INFO [2019-12-17 12:32:20,968] jepsen worker 36 - jepsen.util 6246 :invoke :read [12 nil] INFO [2019-12-17 12:32:20,990] jepsen worker 31 - jepsen.util 5161 :info :write [10 22] :RequestCanceledException INFO [2019-12-17 12:32:20,991] jepsen worker 31 - jepsen.util 5251 :invoke :write [10 37] INFO [2019-12-17 12:32:20,994] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":33658,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:32:20,994] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":33658,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:32:20,994] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":33661,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:32:20,994] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":33661,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:32:20,994] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":33662,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:32:20,994] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":33662,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:32:21,023] jepsen worker 65 - jepsen.util 4925 :fail :cas [21 [47 41]] :ValueNotSwapFrom INFO [2019-12-17 12:32:21,023] jepsen worker 65 - jepsen.util 4925 :invoke :cas [21 [17 39]] INFO [2019-12-17 12:32:21,029] jepsen worker 27 - jepsen.util 5787 :ok :write [9 0] INFO [2019-12-17 12:32:21,029] jepsen worker 27 - jepsen.util 5787 :invoke :cas [9 [46 4]] INFO [2019-12-17 12:32:21,037] jepsen worker 70 - jepsen.util 6280 :ok :write [23 33] INFO [2019-12-17 12:32:21,037] jepsen worker 70 - jepsen.util 6280 :invoke :write [23 45] INFO [2019-12-17 12:32:21,039] jepsen worker 86 - jepsen.util 8096 :info :write [28 42] :RequestCanceledException INFO [2019-12-17 12:32:21,039] jepsen worker 21 - jepsen.util 7311 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:32:21,039] jepsen worker 21 - jepsen.util 7311 :invoke :write [7 4] INFO [2019-12-17 12:32:21,041] jepsen worker 86 - jepsen.util 8186 :invoke :write [28 36] INFO [2019-12-17 12:32:21,058] jepsen worker 1 - jepsen.util 7561 :info :write [0 42] :RequestCanceledException INFO [2019-12-17 12:32:21,058] jepsen worker 20 - jepsen.util 5780 :ok :write [6 31] INFO [2019-12-17 12:32:21,058] jepsen worker 20 - jepsen.util 5780 :invoke :write [6 16] INFO [2019-12-17 12:32:21,059] jepsen worker 1 - jepsen.util 7651 :invoke :read [0 nil] INFO [2019-12-17 12:32:21,059] jepsen worker 74 - jepsen.util 6194 :ok :write [24 38] INFO [2019-12-17 12:32:21,059] jepsen worker 74 - jepsen.util 6194 :invoke :write [24 34] INFO [2019-12-17 12:32:21,067] jepsen worker 3 - jepsen.util 5853 :ok :write [1 2] INFO [2019-12-17 12:32:21,067] jepsen worker 3 - jepsen.util 5853 :invoke :cas [1 [46 23]] INFO [2019-12-17 12:32:21,069] jepsen worker 9 - jepsen.util 6309 :ok :read [3 31] INFO [2019-12-17 12:32:21,069] jepsen worker 9 - jepsen.util 6309 :invoke :read [3 nil] INFO [2019-12-17 12:32:21,093] jepsen worker 22 - jepsen.util 7852 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:32:21,093] jepsen worker 63 - jepsen.util 4833 :info :cas [21 [37 0]] :RequestCanceledException INFO [2019-12-17 12:32:21,093] jepsen worker 61 - jepsen.util 8971 :info :write [20 4] :RequestCanceledException INFO [2019-12-17 12:32:21,093] jepsen worker 22 - jepsen.util 7852 :invoke :read [7 nil] INFO [2019-12-17 12:32:21,094] jepsen worker 61 - jepsen.util 9061 :invoke :cas [20 [47 8]] INFO [2019-12-17 12:32:21,095] jepsen worker 63 - jepsen.util 4923 :invoke :cas [21 [21 2]] INFO [2019-12-17 12:32:21,106] jepsen worker 27 - jepsen.util 5787 :fail :cas [9 [46 4]] :ValueNotSwapFrom INFO [2019-12-17 12:32:21,107] jepsen worker 27 - jepsen.util 5787 :invoke :read [9 nil] INFO [2019-12-17 12:32:21,111] jepsen worker 17 - jepsen.util 7937 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:32:21,111] jepsen worker 72 - jepsen.util 5742 :fail :read [24 nil] :RequestTimeoutException INFO [2019-12-17 12:32:21,111] jepsen worker 17 - jepsen.util 7937 :invoke :write [5 4] INFO [2019-12-17 12:32:21,111] jepsen worker 72 - jepsen.util 5742 :invoke :cas [24 [13 45]] INFO [2019-12-17 12:32:21,112] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:32:21,112] jepsen nemesis - couchbase.util Rebalance check count: 255 INFO [2019-12-17 12:32:21,112] jepsen nemesis - couchbase.util Rebalance stuck count: 236 INFO [2019-12-17 12:32:21,125] jepsen worker 25 - jepsen.util 8395 :info :write [8 44] :RequestCanceledException INFO [2019-12-17 12:32:21,125] jepsen worker 7 - jepsen.util 8197 :info :write [2 49] :RequestCanceledException INFO [2019-12-17 12:32:21,126] jepsen worker 25 - jepsen.util 8485 :invoke :read [8 nil] INFO [2019-12-17 12:32:21,126] jepsen worker 7 - jepsen.util 8287 :invoke :read [2 nil] INFO [2019-12-17 12:32:21,149] jepsen worker 52 - jepsen.util 6262 :ok :read [17 0] INFO [2019-12-17 12:32:21,149] jepsen worker 52 - jepsen.util 6262 :invoke :write [17 45] INFO [2019-12-17 12:32:21,152] jepsen worker 5 - jepsen.util 5495 :ok :read [1 2] INFO [2019-12-17 12:32:21,152] jepsen worker 5 - jepsen.util 5495 :invoke :read [1 nil] INFO [2019-12-17 12:32:21,157] jepsen worker 77 - jepsen.util 7997 :info :write [25 42] :RequestCanceledException INFO [2019-12-17 12:32:21,157] jepsen worker 15 - jepsen.util 6855 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:32:21,157] jepsen worker 15 - jepsen.util 6855 :invoke :cas [5 [48 46]] INFO [2019-12-17 12:32:21,158] jepsen worker 77 - jepsen.util 8087 :invoke :cas [25 [27 34]] INFO [2019-12-17 12:32:21,170] jepsen worker 9 - jepsen.util 6309 :ok :read [3 31] INFO [2019-12-17 12:32:21,170] jepsen worker 9 - jepsen.util 6309 :invoke :read [3 nil] INFO [2019-12-17 12:32:21,177] jepsen worker 53 - jepsen.util 5633 :ok :write [17 39] INFO [2019-12-17 12:32:21,177] jepsen worker 53 - jepsen.util 5633 :invoke :read [17 nil] INFO [2019-12-17 12:32:21,187] jepsen worker 71 - jepsen.util 6551 :fail :cas [23 [12 2]] :ValueNotSwapFrom INFO [2019-12-17 12:32:21,187] jepsen worker 71 - jepsen.util 6551 :invoke :write [23 24] INFO [2019-12-17 12:32:21,193] jepsen worker 26 - jepsen.util 8126 :info :cas [8 [37 21]] :RequestCanceledException INFO [2019-12-17 12:32:21,194] jepsen worker 26 - jepsen.util 8216 :invoke :read [8 nil] INFO [2019-12-17 12:32:21,195] jepsen worker 45 - jepsen.util 5805 :ok :write [15 30] INFO [2019-12-17 12:32:21,195] jepsen worker 45 - jepsen.util 5805 :invoke :write [15 26] INFO [2019-12-17 12:32:21,205] jepsen worker 27 - jepsen.util 5787 :ok :read [9 0] INFO [2019-12-17 12:32:21,205] jepsen worker 27 - jepsen.util 5787 :invoke :cas [9 [10 11]] INFO [2019-12-17 12:32:21,209] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":33701,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:32:21,209] jepsen worker 40 - jepsen.util 8230 :info :cas [13 [9 41]] :RequestCanceledException INFO [2019-12-17 12:32:21,209] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":33701,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:32:21,209] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":33703,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:32:21,209] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":33703,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:32:21,210] jepsen worker 40 - jepsen.util 8320 :invoke :write [13 21] INFO [2019-12-17 12:32:21,227] jepsen worker 67 - jepsen.util 8077 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:32:21,227] jepsen worker 67 - jepsen.util 8077 :invoke :cas [22 [35 12]] INFO [2019-12-17 12:32:21,231] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":33691,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:32:21,231] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":33691,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:32:21,233] jepsen worker 4 - jepsen.util 5314 :ok :read [1 2] INFO [2019-12-17 12:32:21,233] jepsen worker 4 - jepsen.util 5314 :invoke :write [1 14] INFO [2019-12-17 12:32:21,240] jepsen worker 73 - jepsen.util 5473 :info :write [24 49] :RequestCanceledException INFO [2019-12-17 12:32:21,242] jepsen worker 73 - jepsen.util 5563 :invoke :read [24 nil] INFO [2019-12-17 12:32:21,244] jepsen worker 9 - jepsen.util 6309 :ok :read [3 31] INFO [2019-12-17 12:32:21,244] jepsen worker 9 - jepsen.util 6309 :invoke :write [3 7] INFO [2019-12-17 12:32:21,244] jepsen worker 80 - jepsen.util 5930 :ok :write [26 40] INFO [2019-12-17 12:32:21,244] jepsen worker 80 - jepsen.util 5930 :invoke :read [26 nil] INFO [2019-12-17 12:32:21,245] jepsen worker 13 - jepsen.util 5683 :fail :cas [4 [4 2]] :ValueNotSwapFrom INFO [2019-12-17 12:32:21,245] jepsen worker 13 - jepsen.util 5683 :invoke :write [4 47] INFO [2019-12-17 12:32:21,277] jepsen worker 27 - jepsen.util 5787 :fail :cas [9 [10 11]] :ValueNotSwapFrom INFO [2019-12-17 12:32:21,277] jepsen worker 27 - jepsen.util 5787 :invoke :write [9 33] INFO [2019-12-17 12:32:21,284] jepsen worker 12 - jepsen.util 6222 :fail :cas [4 [5 23]] :ValueNotSwapFrom INFO [2019-12-17 12:32:21,284] jepsen worker 12 - jepsen.util 6222 :invoke :read [4 nil] INFO [2019-12-17 12:32:21,290] jepsen worker 3 - jepsen.util 5853 :fail :cas [1 [46 23]] :ValueNotSwapFrom INFO [2019-12-17 12:32:21,291] jepsen worker 3 - jepsen.util 5853 :invoke :cas [1 [37 28]] INFO [2019-12-17 12:32:21,307] jepsen worker 28 - jepsen.util 4528 :ok :write [9 28] INFO [2019-12-17 12:32:21,307] jepsen worker 28 - jepsen.util 4528 :invoke :write [9 14] INFO [2019-12-17 12:32:21,319] jepsen worker 64 - jepsen.util 4564 :ok :write [21 33] INFO [2019-12-17 12:32:21,320] jepsen worker 64 - jepsen.util 4564 :invoke :write [21 0] INFO [2019-12-17 12:32:21,326] jepsen worker 24 - jepsen.util 7134 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:32:21,326] jepsen worker 24 - jepsen.util 7134 :invoke :cas [8 [11 30]] INFO [2019-12-17 12:32:21,326] jepsen worker 35 - jepsen.util 8045 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:32:21,327] jepsen worker 35 - jepsen.util 8045 :invoke :write [11 7] INFO [2019-12-17 12:32:21,340] jepsen worker 38 - jepsen.util 5978 :fail :cas [12 [41 0]] :ValueNotSwapFrom INFO [2019-12-17 12:32:21,340] jepsen worker 38 - jepsen.util 5978 :invoke :write [12 15] INFO [2019-12-17 12:32:21,341] jepsen worker 42 - jepsen.util 8412 :info :write [14 27] :RequestCanceledException INFO [2019-12-17 12:32:21,342] jepsen worker 42 - jepsen.util 8502 :invoke :cas [14 [5 7]] INFO [2019-12-17 12:32:21,346] jepsen worker 63 - jepsen.util 4923 :fail :cas [21 [21 2]] :ValueNotSwapFrom INFO [2019-12-17 12:32:21,346] jepsen worker 63 - jepsen.util 4923 :invoke :read [21 nil] INFO [2019-12-17 12:32:21,382] jepsen worker 65 - jepsen.util 4925 :fail :cas [21 [17 39]] :ValueNotSwapFrom INFO [2019-12-17 12:32:21,383] jepsen worker 65 - jepsen.util 4925 :invoke :write [21 48] INFO [2019-12-17 12:32:21,387] jepsen worker 79 - jepsen.util 5749 :ok :write [26 31] INFO [2019-12-17 12:32:21,387] jepsen worker 79 - jepsen.util 5749 :invoke :read [26 nil] INFO [2019-12-17 12:32:21,393] jepsen worker 49 - jepsen.util 6979 :info :write [16 26] :RequestCanceledException INFO [2019-12-17 12:32:21,393] jepsen worker 59 - jepsen.util 7889 :info :write [19 21] :RequestCanceledException INFO [2019-12-17 12:32:21,395] jepsen worker 49 - jepsen.util 7069 :invoke :cas [16 [45 40]] INFO [2019-12-17 12:32:21,404] jepsen worker 59 - jepsen.util 7979 :invoke :write [19 26] INFO [2019-12-17 12:32:21,426] jepsen worker 23 - jepsen.util 7853 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:32:21,426] jepsen worker 55 - jepsen.util 6805 :ok :write [18 21] INFO [2019-12-17 12:32:21,426] jepsen worker 23 - jepsen.util 7853 :invoke :write [7 2] INFO [2019-12-17 12:32:21,426] jepsen worker 55 - jepsen.util 6805 :invoke :write [18 5] INFO [2019-12-17 12:32:21,437] jepsen worker 37 - jepsen.util 5437 :ok :write [12 19] INFO [2019-12-17 12:32:21,438] jepsen worker 37 - jepsen.util 5437 :invoke :write [12 28] INFO [2019-12-17 12:32:21,450] jepsen worker 31 - jepsen.util 5251 :ok :write [10 37] INFO [2019-12-17 12:32:21,450] jepsen worker 31 - jepsen.util 5251 :invoke :write [10 19] INFO [2019-12-17 12:32:21,450] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":33728,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:32:21,452] jepsen worker 74 - jepsen.util 6194 :ok :write [24 34] INFO [2019-12-17 12:32:21,453] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":33728,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:32:21,453] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":33730,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:32:21,453] jepsen worker 74 - jepsen.util 6194 :invoke :read [24 nil] INFO [2019-12-17 12:32:21,453] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":33730,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:32:21,457] jepsen worker 89 - jepsen.util 5669 :ok :write [29 9] INFO [2019-12-17 12:32:21,457] jepsen worker 89 - jepsen.util 5669 :invoke :cas [29 [11 16]] INFO [2019-12-17 12:32:21,458] jepsen worker 70 - jepsen.util 6280 :ok :write [23 45] INFO [2019-12-17 12:32:21,459] jepsen worker 70 - jepsen.util 6280 :invoke :cas [23 [43 21]] INFO [2019-12-17 12:32:21,465] jepsen worker 64 - jepsen.util 4564 :ok :write [21 0] INFO [2019-12-17 12:32:21,465] jepsen worker 64 - jepsen.util 4564 :invoke :cas [21 [6 28]] INFO [2019-12-17 12:32:21,503] jepsen worker 74 - jepsen.util 6194 :ok :read [24 34] INFO [2019-12-17 12:32:21,503] jepsen worker 74 - jepsen.util 6194 :invoke :cas [24 [12 41]] INFO [2019-12-17 12:32:21,526] jepsen worker 10 - jepsen.util 5680 :info :write [3 37] :RequestCanceledException INFO [2019-12-17 12:32:21,527] jepsen worker 10 - jepsen.util 5770 :invoke :cas [3 [6 44]] INFO [2019-12-17 12:32:21,538] jepsen worker 74 - jepsen.util 6194 :fail :cas [24 [12 41]] :ValueNotSwapFrom INFO [2019-12-17 12:32:21,538] jepsen worker 74 - jepsen.util 6194 :invoke :cas [24 [45 13]] INFO [2019-12-17 12:32:21,541] jepsen worker 2 - jepsen.util 7562 :info :write [0 5] :RequestCanceledException INFO [2019-12-17 12:32:21,542] jepsen worker 2 - jepsen.util 7652 :invoke :write [0 33] INFO [2019-12-17 12:32:21,558] jepsen worker 65 - jepsen.util 4925 :ok :write [21 48] INFO [2019-12-17 12:32:21,558] jepsen worker 65 - jepsen.util 4925 :invoke :read [21 nil] INFO [2019-12-17 12:32:21,566] jepsen worker 74 - jepsen.util 6194 :fail :cas [24 [45 13]] :ValueNotSwapFrom INFO [2019-12-17 12:32:21,566] jepsen worker 74 - jepsen.util 6194 :invoke :write [24 19] INFO [2019-12-17 12:32:21,580] jepsen worker 36 - jepsen.util 6246 :ok :read [12 19] INFO [2019-12-17 12:32:21,581] jepsen worker 36 - jepsen.util 6246 :invoke :write [12 40] INFO [2019-12-17 12:32:21,593] jepsen worker 18 - jepsen.util 5688 :fail :read [6 nil] :RequestTimeoutException INFO [2019-12-17 12:32:21,594] jepsen worker 18 - jepsen.util 5688 :invoke :read [6 nil] INFO [2019-12-17 12:32:21,602] jepsen worker 9 - jepsen.util 6309 :ok :write [3 7] INFO [2019-12-17 12:32:21,602] jepsen worker 9 - jepsen.util 6309 :invoke :read [3 nil] INFO [2019-12-17 12:32:21,607] jepsen worker 65 - jepsen.util 4925 :ok :read [21 48] INFO [2019-12-17 12:32:21,607] jepsen worker 65 - jepsen.util 4925 :invoke :read [21 nil] INFO [2019-12-17 12:32:21,620] jepsen worker 10 - jepsen.util 5770 :fail :cas [3 [6 44]] :ValueNotSwapFrom INFO [2019-12-17 12:32:21,620] jepsen worker 10 - jepsen.util 5770 :invoke :read [3 nil] INFO [2019-12-17 12:32:21,642] jepsen worker 78 - jepsen.util 6198 :fail :cas [26 [13 8]] :ValueNotSwapFrom INFO [2019-12-17 12:32:21,642] jepsen worker 78 - jepsen.util 6198 :invoke :read [26 nil] INFO [2019-12-17 12:32:21,645] jepsen worker 3 - jepsen.util 5853 :fail :cas [1 [37 28]] :ValueNotSwapFrom INFO [2019-12-17 12:32:21,645] jepsen worker 3 - jepsen.util 5853 :invoke :cas [1 [6 13]] INFO [2019-12-17 12:32:21,646] jepsen worker 9 - jepsen.util 6309 :ok :read [3 7] INFO [2019-12-17 12:32:21,646] jepsen worker 9 - jepsen.util 6309 :invoke :cas [3 [35 22]] INFO [2019-12-17 12:32:21,647] jepsen worker 10 - jepsen.util 5770 :ok :read [3 7] INFO [2019-12-17 12:32:21,647] jepsen worker 10 - jepsen.util 5770 :invoke :cas [3 [21 46]] INFO [2019-12-17 12:32:21,652] jepsen worker 65 - jepsen.util 4925 :ok :read [21 48] INFO [2019-12-17 12:32:21,652] jepsen worker 65 - jepsen.util 4925 :invoke :read [21 nil] INFO [2019-12-17 12:32:21,662] jepsen worker 79 - jepsen.util 5749 :ok :read [26 31] INFO [2019-12-17 12:32:21,662] jepsen worker 79 - jepsen.util 5749 :invoke :read [26 nil] INFO [2019-12-17 12:32:21,670] jepsen worker 9 - jepsen.util 6309 :fail :cas [3 [35 22]] :ValueNotSwapFrom INFO [2019-12-17 12:32:21,670] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":33751,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:32:21,670] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":33751,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:32:21,670] jepsen worker 9 - jepsen.util 6309 :invoke :cas [3 [16 1]] INFO [2019-12-17 12:32:21,670] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":33754,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:32:21,670] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":33754,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:32:21,688] jepsen worker 53 - jepsen.util 5633 :ok :read [17 39] INFO [2019-12-17 12:32:21,688] jepsen worker 53 - jepsen.util 5633 :invoke :read [17 nil] INFO [2019-12-17 12:32:21,707] jepsen worker 10 - jepsen.util 5770 :fail :cas [3 [21 46]] :ValueNotSwapFrom INFO [2019-12-17 12:32:21,707] jepsen worker 10 - jepsen.util 5770 :invoke :write [3 5] INFO [2019-12-17 12:32:21,713] jepsen worker 65 - jepsen.util 4925 :ok :read [21 48] INFO [2019-12-17 12:32:21,714] jepsen worker 65 - jepsen.util 4925 :invoke :read [21 nil] INFO [2019-12-17 12:32:21,726] jepsen worker 50 - jepsen.util 8240 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:32:21,726] jepsen worker 50 - jepsen.util 8240 :invoke :cas [16 [47 33]] INFO [2019-12-17 12:32:21,740] jepsen worker 9 - jepsen.util 6309 :fail :cas [3 [16 1]] :ValueNotSwapFrom INFO [2019-12-17 12:32:21,741] jepsen worker 9 - jepsen.util 6309 :invoke :read [3 nil] INFO [2019-12-17 12:32:21,744] jepsen worker 70 - jepsen.util 6280 :fail :cas [23 [43 21]] :ValueNotSwapFrom INFO [2019-12-17 12:32:21,744] jepsen worker 70 - jepsen.util 6280 :invoke :read [23 nil] INFO [2019-12-17 12:32:21,744] jepsen worker 5 - jepsen.util 5495 :ok :read [1 2] INFO [2019-12-17 12:32:21,744] jepsen worker 5 - jepsen.util 5495 :invoke :read [1 nil] INFO [2019-12-17 12:32:21,751] jepsen worker 12 - jepsen.util 6222 :ok :read [4 40] INFO [2019-12-17 12:32:21,751] jepsen worker 12 - jepsen.util 6222 :invoke :cas [4 [28 10]] INFO [2019-12-17 12:32:21,779] jepsen worker 65 - jepsen.util 4925 :ok :read [21 48] INFO [2019-12-17 12:32:21,779] jepsen worker 65 - jepsen.util 4925 :invoke :write [21 15] INFO [2019-12-17 12:32:21,801] jepsen worker 38 - jepsen.util 5978 :ok :write [12 15] INFO [2019-12-17 12:32:21,801] jepsen worker 38 - jepsen.util 5978 :invoke :cas [12 [12 21]] INFO [2019-12-17 12:32:21,849] jepsen worker 65 - jepsen.util 4925 :ok :write [21 15] INFO [2019-12-17 12:32:21,849] jepsen worker 65 - jepsen.util 4925 :invoke :read [21 nil] INFO [2019-12-17 12:32:21,854] jepsen worker 89 - jepsen.util 5669 :fail :cas [29 [11 16]] :ValueNotSwapFrom INFO [2019-12-17 12:32:21,854] jepsen worker 89 - jepsen.util 5669 :invoke :cas [29 [22 15]] INFO [2019-12-17 12:32:21,857] jepsen worker 51 - jepsen.util 5721 :info :write [17 18] :RequestCanceledException INFO [2019-12-17 12:32:21,859] jepsen worker 51 - jepsen.util 5811 :invoke :read [17 nil] INFO [2019-12-17 12:32:21,865] jepsen worker 45 - jepsen.util 5805 :ok :write [15 26] INFO [2019-12-17 12:32:21,865] jepsen worker 45 - jepsen.util 5805 :invoke :write [15 40] INFO [2019-12-17 12:32:21,867] jepsen worker 31 - jepsen.util 5251 :ok :write [10 19] INFO [2019-12-17 12:32:21,867] jepsen worker 31 - jepsen.util 5251 :invoke :read [10 nil] INFO [2019-12-17 12:32:21,883] jepsen worker 20 - jepsen.util 5780 :ok :write [6 16] INFO [2019-12-17 12:32:21,883] jepsen worker 74 - jepsen.util 6194 :ok :write [24 19] INFO [2019-12-17 12:32:21,883] jepsen worker 20 - jepsen.util 5780 :invoke :write [6 9] INFO [2019-12-17 12:32:21,884] jepsen worker 74 - jepsen.util 6194 :invoke :write [24 0] INFO [2019-12-17 12:32:21,887] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":33776,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:32:21,887] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":33776,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:32:21,887] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":33781,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:32:21,891] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":33781,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:32:21,896] jepsen worker 46 - jepsen.util 5716 :info :write [15 14] :RequestCanceledException INFO [2019-12-17 12:32:21,898] jepsen worker 46 - jepsen.util 5806 :invoke :write [15 20] INFO [2019-12-17 12:32:21,903] jepsen worker 65 - jepsen.util 4925 :ok :read [21 15] INFO [2019-12-17 12:32:21,903] jepsen worker 65 - jepsen.util 4925 :invoke :read [21 nil] INFO [2019-12-17 12:32:21,903] jepsen worker 18 - jepsen.util 5688 :ok :read [6 16] INFO [2019-12-17 12:32:21,904] jepsen worker 18 - jepsen.util 5688 :invoke :write [6 9] INFO [2019-12-17 12:32:21,926] jepsen worker 19 - jepsen.util 5779 :info :write [6 49] :RequestCanceledException INFO [2019-12-17 12:32:21,932] jepsen worker 19 - jepsen.util 5869 :invoke :write [6 17] INFO [2019-12-17 12:32:21,932] jepsen worker 12 - jepsen.util 6222 :fail :cas [4 [28 10]] :ValueNotSwapFrom INFO [2019-12-17 12:32:21,932] jepsen worker 12 - jepsen.util 6222 :invoke :cas [4 [48 6]] INFO [2019-12-17 12:32:21,951] jepsen worker 31 - jepsen.util 5251 :ok :read [10 19] INFO [2019-12-17 12:32:21,951] jepsen worker 31 - jepsen.util 5251 :invoke :read [10 nil] INFO [2019-12-17 12:32:21,958] jepsen worker 87 - jepsen.util 5307 :info :write [29 10] :RequestCanceledException INFO [2019-12-17 12:32:21,960] jepsen worker 87 - jepsen.util 5397 :invoke :write [29 29] INFO [2019-12-17 12:32:21,960] jepsen worker 38 - jepsen.util 5978 :fail :cas [12 [12 21]] :ValueNotSwapFrom INFO [2019-12-17 12:32:21,960] jepsen worker 38 - jepsen.util 5978 :invoke :read [12 nil] INFO [2019-12-17 12:32:21,967] jepsen worker 80 - jepsen.util 5930 :ok :read [26 31] INFO [2019-12-17 12:32:21,967] jepsen worker 80 - jepsen.util 5930 :invoke :cas [26 [40 37]] INFO [2019-12-17 12:32:21,970] jepsen worker 65 - jepsen.util 4925 :ok :read [21 15] INFO [2019-12-17 12:32:21,970] jepsen worker 65 - jepsen.util 4925 :invoke :write [21 24] INFO [2019-12-17 12:32:22,028] jepsen worker 5 - jepsen.util 5495 :ok :read [1 2] INFO [2019-12-17 12:32:22,029] jepsen worker 5 - jepsen.util 5495 :invoke :write [1 8] INFO [2019-12-17 12:32:22,074] jepsen worker 65 - jepsen.util 4925 :ok :write [21 24] INFO [2019-12-17 12:32:22,074] jepsen worker 65 - jepsen.util 4925 :invoke :read [21 nil] INFO [2019-12-17 12:32:22,079] jepsen worker 78 - jepsen.util 6198 :ok :read [26 31] INFO [2019-12-17 12:32:22,079] jepsen worker 78 - jepsen.util 6198 :invoke :cas [26 [26 42]] INFO [2019-12-17 12:32:22,086] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":33797,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:32:22,086] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":33797,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:32:22,099] jepsen worker 65 - jepsen.util 4925 :ok :read [21 24] INFO [2019-12-17 12:32:22,099] jepsen worker 65 - jepsen.util 4925 :invoke :write [21 0] INFO [2019-12-17 12:32:22,123] jepsen worker 53 - jepsen.util 5633 :ok :read [17 39] INFO [2019-12-17 12:32:22,123] jepsen worker 53 - jepsen.util 5633 :invoke :cas [17 [20 12]] INFO [2019-12-17 12:32:22,132] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:32:22,132] jepsen nemesis - couchbase.util Rebalance check count: 256 INFO [2019-12-17 12:32:22,132] jepsen nemesis - couchbase.util Rebalance stuck count: 237 INFO [2019-12-17 12:32:22,162] jepsen worker 65 - jepsen.util 4925 :ok :write [21 0] INFO [2019-12-17 12:32:22,162] jepsen worker 65 - jepsen.util 4925 :invoke :write [21 31] INFO [2019-12-17 12:32:22,180] jepsen worker 89 - jepsen.util 5669 :fail :cas [29 [22 15]] :ValueNotSwapFrom INFO [2019-12-17 12:32:22,180] jepsen worker 89 - jepsen.util 5669 :invoke :read [29 nil] INFO [2019-12-17 12:32:22,216] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":33809,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:32:22,216] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":33809,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:32:22,217] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":33811,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:32:22,217] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":33811,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:32:22,224] jepsen worker 51 - jepsen.util 5811 :ok :read [17 39] INFO [2019-12-17 12:32:22,225] jepsen worker 51 - jepsen.util 5811 :invoke :write [17 36] INFO [2019-12-17 12:32:22,230] jepsen worker 65 - jepsen.util 4925 :ok :write [21 31] INFO [2019-12-17 12:32:22,230] jepsen worker 65 - jepsen.util 4925 :invoke :cas [21 [36 45]] INFO [2019-12-17 12:32:22,238] jepsen worker 3 - jepsen.util 5853 :fail :cas [1 [6 13]] :ValueNotSwapFrom INFO [2019-12-17 12:32:22,239] jepsen worker 3 - jepsen.util 5853 :invoke :cas [1 [15 47]] INFO [2019-12-17 12:32:22,256] jepsen worker 14 - jepsen.util 6044 :ok :read [4 40] INFO [2019-12-17 12:32:22,256] jepsen worker 14 - jepsen.util 6044 :invoke :write [4 37] INFO [2019-12-17 12:32:22,257] jepsen worker 65 - jepsen.util 4925 :fail :cas [21 [36 45]] :ValueNotSwapFrom INFO [2019-12-17 12:32:22,257] jepsen worker 65 - jepsen.util 4925 :invoke :cas [21 [24 5]] INFO [2019-12-17 12:32:22,266] jepsen worker 79 - jepsen.util 5749 :ok :read [26 31] INFO [2019-12-17 12:32:22,266] jepsen worker 79 - jepsen.util 5749 :invoke :read [26 nil] INFO [2019-12-17 12:32:22,270] jepsen worker 65 - jepsen.util 4925 :fail :cas [21 [24 5]] :ValueNotSwapFrom INFO [2019-12-17 12:32:22,270] jepsen worker 65 - jepsen.util 4925 :invoke :cas [21 [24 27]] INFO [2019-12-17 12:32:22,283] jepsen worker 65 - jepsen.util 4925 :fail :cas [21 [24 27]] :ValueNotSwapFrom INFO [2019-12-17 12:32:22,283] jepsen worker 65 - jepsen.util 4925 :invoke :write [21 24] INFO [2019-12-17 12:32:22,292] jepsen worker 68 - jepsen.util 7448 :info :cas [22 [19 24]] :RequestCanceledException INFO [2019-12-17 12:32:22,304] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":33816,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:32:22,305] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":33816,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:32:22,305] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":33820,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:32:22,305] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":33820,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:32:22,317] jepsen worker 68 - jepsen.util 7538 :invoke :cas [22 [37 28]] INFO [2019-12-17 12:32:22,328] jepsen worker 3 - jepsen.util 5853 :fail :cas [1 [15 47]] :ValueNotSwapFrom INFO [2019-12-17 12:32:22,329] jepsen worker 3 - jepsen.util 5853 :invoke :cas [1 [30 28]] INFO [2019-12-17 12:32:22,340] jepsen worker 11 - jepsen.util 6041 :info :cas [3 [27 45]] :RequestCanceledException INFO [2019-12-17 12:32:22,342] jepsen worker 11 - jepsen.util 6131 :invoke :cas [3 [15 9]] INFO [2019-12-17 12:32:22,342] jepsen worker 70 - jepsen.util 6280 :ok :read [23 45] INFO [2019-12-17 12:32:22,342] jepsen worker 70 - jepsen.util 6280 :invoke :cas [23 [12 28]] INFO [2019-12-17 12:32:22,342] jepsen worker 80 - jepsen.util 5930 :fail :cas [26 [40 37]] :ValueNotSwapFrom INFO [2019-12-17 12:32:22,342] jepsen worker 80 - jepsen.util 5930 :invoke :write [26 15] INFO [2019-12-17 12:32:22,350] jepsen worker 65 - jepsen.util 4925 :ok :write [21 24] INFO [2019-12-17 12:32:22,350] jepsen worker 65 - jepsen.util 4925 :invoke :cas [21 [48 4]] INFO [2019-12-17 12:32:22,366] jepsen worker 11 - jepsen.util 6131 :fail :cas [3 [15 9]] :ValueNotSwapFrom INFO [2019-12-17 12:32:22,366] jepsen worker 11 - jepsen.util 6131 :invoke :write [3 8] INFO [2019-12-17 12:32:22,371] jepsen worker 65 - jepsen.util 4925 :fail :cas [21 [48 4]] :ValueNotSwapFrom INFO [2019-12-17 12:32:22,371] jepsen worker 65 - jepsen.util 4925 :invoke :read [21 nil] INFO [2019-12-17 12:32:22,380] jepsen worker 78 - jepsen.util 6198 :fail :cas [26 [26 42]] :ValueNotSwapFrom INFO [2019-12-17 12:32:22,380] jepsen worker 78 - jepsen.util 6198 :invoke :write [26 46] INFO [2019-12-17 12:32:22,404] jepsen worker 65 - jepsen.util 4925 :ok :read [21 24] INFO [2019-12-17 12:32:22,404] jepsen worker 65 - jepsen.util 4925 :invoke :read [21 nil] INFO [2019-12-17 12:32:22,418] jepsen worker 3 - jepsen.util 5853 :fail :cas [1 [30 28]] :ValueNotSwapFrom INFO [2019-12-17 12:32:22,418] jepsen worker 3 - jepsen.util 5853 :invoke :write [1 42] INFO [2019-12-17 12:32:22,427] jepsen worker 79 - jepsen.util 5749 :ok :read [26 31] INFO [2019-12-17 12:32:22,428] jepsen worker 79 - jepsen.util 5749 :invoke :cas [26 [35 11]] INFO [2019-12-17 12:32:22,431] jepsen worker 65 - jepsen.util 4925 :ok :read [21 24] INFO [2019-12-17 12:32:22,432] jepsen worker 65 - jepsen.util 4925 :invoke :cas [21 [19 10]] INFO [2019-12-17 12:32:22,435] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":33842,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:32:22,435] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":33842,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:32:22,435] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":33846,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:32:22,435] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":33846,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:32:22,443] jepsen worker 65 - jepsen.util 4925 :fail :cas [21 [19 10]] :ValueNotSwapFrom INFO [2019-12-17 12:32:22,443] jepsen worker 65 - jepsen.util 4925 :invoke :cas [21 [23 5]] INFO [2019-12-17 12:32:22,458] jepsen worker 65 - jepsen.util 4925 :fail :cas [21 [23 5]] :ValueNotSwapFrom INFO [2019-12-17 12:32:22,458] jepsen worker 65 - jepsen.util 4925 :invoke :cas [21 [20 7]] INFO [2019-12-17 12:32:22,458] jepsen worker 66 - jepsen.util 7356 :info :cas [22 [7 21]] :RequestCanceledException INFO [2019-12-17 12:32:22,459] jepsen worker 66 - jepsen.util 7446 :invoke :cas [22 [27 25]] INFO [2019-12-17 12:32:22,475] jepsen worker 65 - jepsen.util 4925 :fail :cas [21 [20 7]] :ValueNotSwapFrom INFO [2019-12-17 12:32:22,475] jepsen worker 65 - jepsen.util 4925 :invoke :write [21 46] INFO [2019-12-17 12:32:22,504] jepsen worker 75 - jepsen.util 7635 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:32:22,504] jepsen worker 75 - jepsen.util 7635 :invoke :cas [25 [44 8]] INFO [2019-12-17 12:32:22,513] jepsen worker 70 - jepsen.util 6280 :fail :cas [23 [12 28]] :ValueNotSwapFrom INFO [2019-12-17 12:32:22,513] jepsen worker 70 - jepsen.util 6280 :invoke :write [23 19] INFO [2019-12-17 12:32:22,521] jepsen worker 65 - jepsen.util 4925 :ok :write [21 46] INFO [2019-12-17 12:32:22,521] jepsen worker 65 - jepsen.util 4925 :invoke :cas [21 [47 12]] INFO [2019-12-17 12:32:22,527] jepsen worker 44 - jepsen.util 8594 :info :write [14 28] :RequestCanceledException INFO [2019-12-17 12:32:22,529] jepsen worker 44 - jepsen.util 8684 :invoke :cas [14 [42 31]] INFO [2019-12-17 12:32:22,539] jepsen worker 47 - jepsen.util 6617 :info :write [15 37] :RequestCanceledException INFO [2019-12-17 12:32:22,541] jepsen worker 10 - jepsen.util 5770 :ok :write [3 5] INFO [2019-12-17 12:32:22,541] jepsen worker 10 - jepsen.util 5770 :invoke :cas [3 [13 32]] INFO [2019-12-17 12:32:22,541] jepsen worker 18 - jepsen.util 5688 :ok :write [6 9] INFO [2019-12-17 12:32:22,542] jepsen worker 45 - jepsen.util 5805 :ok :write [15 40] INFO [2019-12-17 12:32:22,542] jepsen worker 18 - jepsen.util 5688 :invoke :write [6 33] INFO [2019-12-17 12:32:22,542] jepsen worker 45 - jepsen.util 5805 :invoke :cas [15 [9 38]] INFO [2019-12-17 12:32:22,544] jepsen worker 65 - jepsen.util 4925 :fail :cas [21 [47 12]] :ValueNotSwapFrom INFO [2019-12-17 12:32:22,544] jepsen worker 65 - jepsen.util 4925 :invoke :cas [21 [48 2]] INFO [2019-12-17 12:32:22,546] jepsen worker 47 - jepsen.util 6707 :invoke :read [15 nil] INFO [2019-12-17 12:32:22,558] jepsen worker 54 - jepsen.util 6264 :info :cas [18 [33 22]] :RequestCanceledException INFO [2019-12-17 12:32:22,559] jepsen worker 54 - jepsen.util 6354 :invoke :write [18 14] INFO [2019-12-17 12:32:22,574] jepsen worker 74 - jepsen.util 6194 :ok :write [24 0] INFO [2019-12-17 12:32:22,574] jepsen worker 74 - jepsen.util 6194 :invoke :cas [24 [1 0]] INFO [2019-12-17 12:32:22,586] jepsen worker 31 - jepsen.util 5251 :ok :read [10 19] INFO [2019-12-17 12:32:22,587] jepsen worker 31 - jepsen.util 5251 :invoke :write [10 38] INFO [2019-12-17 12:32:22,592] jepsen worker 85 - jepsen.util 8815 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:32:22,592] jepsen worker 85 - jepsen.util 8815 :invoke :cas [28 [3 36]] INFO [2019-12-17 12:32:22,611] jepsen worker 48 - jepsen.util 7158 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:32:22,611] jepsen worker 48 - jepsen.util 7158 :invoke :write [16 37] INFO [2019-12-17 12:32:22,624] jepsen worker 16 - jepsen.util 8926 :info :cas [5 [14 31]] :RequestCanceledException INFO [2019-12-17 12:32:22,624] jepsen worker 57 - jepsen.util 7527 :info :cas [19 [44 48]] :RequestCanceledException INFO [2019-12-17 12:32:22,624] jepsen worker 8 - jepsen.util 7388 :info :write [2 3] :RequestCanceledException INFO [2019-12-17 12:32:22,626] jepsen worker 57 - jepsen.util 7617 :invoke :read [19 nil] INFO [2019-12-17 12:32:22,626] jepsen worker 16 - jepsen.util 9016 :invoke :read [5 nil] INFO [2019-12-17 12:32:22,626] jepsen worker 8 - jepsen.util 7478 :invoke :write [2 2] INFO [2019-12-17 12:32:22,636] jepsen worker 43 - jepsen.util 7333 :info :cas [14 [17 24]] :RequestCanceledException INFO [2019-12-17 12:32:22,636] jepsen worker 82 - jepsen.util 8362 :info :write [27 15] :RequestCanceledException INFO [2019-12-17 12:32:22,638] jepsen worker 43 - jepsen.util 7423 :invoke :cas [14 [14 20]] INFO [2019-12-17 12:32:22,639] jepsen worker 10 - jepsen.util 5770 :fail :cas [3 [13 32]] :ValueNotSwapFrom INFO [2019-12-17 12:32:22,639] jepsen worker 82 - jepsen.util 8452 :invoke :cas [27 [42 6]] INFO [2019-12-17 12:32:22,639] jepsen worker 10 - jepsen.util 5770 :invoke :cas [3 [8 1]] INFO [2019-12-17 12:32:22,643] jepsen worker 74 - jepsen.util 6194 :fail :cas [24 [1 0]] :ValueNotSwapFrom INFO [2019-12-17 12:32:22,643] jepsen worker 74 - jepsen.util 6194 :invoke :write [24 42] INFO [2019-12-17 12:32:22,645] jepsen worker 45 - jepsen.util 5805 :fail :cas [15 [9 38]] :ValueNotSwapFrom INFO [2019-12-17 12:32:22,645] jepsen worker 45 - jepsen.util 5805 :invoke :read [15 nil] INFO [2019-12-17 12:32:22,645] jepsen worker 12 - jepsen.util 6222 :fail :cas [4 [48 6]] :ValueNotSwapFrom INFO [2019-12-17 12:32:22,645] jepsen worker 12 - jepsen.util 6222 :invoke :write [4 42] INFO [2019-12-17 12:32:22,645] jepsen worker 65 - jepsen.util 4925 :fail :cas [21 [48 2]] :ValueNotSwapFrom INFO [2019-12-17 12:32:22,645] jepsen worker 65 - jepsen.util 4925 :invoke :read [21 nil] INFO [2019-12-17 12:32:22,649] jepsen worker 89 - jepsen.util 5669 :ok :read [29 9] INFO [2019-12-17 12:32:22,649] jepsen worker 89 - jepsen.util 5669 :invoke :write [29 6] INFO [2019-12-17 12:32:22,654] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":33873,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:32:22,654] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":33873,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:32:22,654] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":33877,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:32:22,654] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":33877,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:32:22,657] jepsen worker 60 - jepsen.util 8700 :info :cas [20 [41 39]] :RequestCanceledException INFO [2019-12-17 12:32:22,660] jepsen worker 60 - jepsen.util 8790 :invoke :write [20 47] INFO [2019-12-17 12:32:22,678] jepsen worker 65 - jepsen.util 4925 :ok :read [21 46] INFO [2019-12-17 12:32:22,678] jepsen worker 65 - jepsen.util 4925 :invoke :cas [21 [42 2]] INFO [2019-12-17 12:32:22,685] jepsen worker 79 - jepsen.util 5749 :fail :cas [26 [35 11]] :ValueNotSwapFrom INFO [2019-12-17 12:32:22,685] jepsen worker 79 - jepsen.util 5749 :invoke :cas [26 [27 0]] INFO [2019-12-17 12:32:22,711] jepsen worker 65 - jepsen.util 4925 :fail :cas [21 [42 2]] :ValueNotSwapFrom INFO [2019-12-17 12:32:22,712] jepsen worker 65 - jepsen.util 4925 :invoke :cas [21 [22 33]] INFO [2019-12-17 12:32:22,716] jepsen worker 45 - jepsen.util 5805 :ok :read [15 40] INFO [2019-12-17 12:32:22,717] jepsen worker 45 - jepsen.util 5805 :invoke :cas [15 [15 26]] INFO [2019-12-17 12:32:22,717] jepsen worker 10 - jepsen.util 5770 :fail :cas [3 [8 1]] :ValueNotSwapFrom INFO [2019-12-17 12:32:22,719] jepsen worker 10 - jepsen.util 5770 :invoke :read [3 nil] INFO [2019-12-17 12:32:22,726] jepsen worker 33 - jepsen.util 7863 :info :cas [11 [4 7]] :RequestCanceledException INFO [2019-12-17 12:32:22,727] jepsen worker 33 - jepsen.util 7953 :invoke :cas [11 [22 32]] INFO [2019-12-17 12:32:22,736] jepsen worker 38 - jepsen.util 5978 :ok :read [12 15] INFO [2019-12-17 12:32:22,736] jepsen worker 38 - jepsen.util 5978 :invoke :read [12 nil] INFO [2019-12-17 12:32:22,749] jepsen worker 45 - jepsen.util 5805 :fail :cas [15 [15 26]] :ValueNotSwapFrom INFO [2019-12-17 12:32:22,750] jepsen worker 45 - jepsen.util 5805 :invoke :write [15 26] INFO [2019-12-17 12:32:22,752] jepsen worker 10 - jepsen.util 5770 :ok :read [3 5] INFO [2019-12-17 12:32:22,752] jepsen worker 10 - jepsen.util 5770 :invoke :read [3 nil] INFO [2019-12-17 12:32:22,757] jepsen worker 56 - jepsen.util 6266 :info :write [18 29] :RequestCanceledException INFO [2019-12-17 12:32:22,758] jepsen worker 56 - jepsen.util 6356 :invoke :cas [18 [26 26]] INFO [2019-12-17 12:32:22,801] jepsen worker 10 - jepsen.util 5770 :ok :read [3 5] INFO [2019-12-17 12:32:22,801] jepsen worker 10 - jepsen.util 5770 :invoke :read [3 nil] INFO [2019-12-17 12:32:22,811] jepsen worker 34 - jepsen.util 7414 :info :write [11 9] :RequestCanceledException INFO [2019-12-17 12:32:22,813] jepsen worker 34 - jepsen.util 7504 :invoke :cas [11 [29 7]] INFO [2019-12-17 12:32:22,817] jepsen worker 65 - jepsen.util 4925 :fail :cas [21 [22 33]] :ValueNotSwapFrom INFO [2019-12-17 12:32:22,818] jepsen worker 65 - jepsen.util 4925 :invoke :read [21 nil] INFO [2019-12-17 12:32:22,823] jepsen worker 18 - jepsen.util 5688 :ok :write [6 33] INFO [2019-12-17 12:32:22,824] jepsen worker 18 - jepsen.util 5688 :invoke :write [6 28] INFO [2019-12-17 12:32:22,827] jepsen worker 58 - jepsen.util 8428 :info :write [19 30] :RequestCanceledException INFO [2019-12-17 12:32:22,828] jepsen worker 58 - jepsen.util 8518 :invoke :write [19 40] INFO [2019-12-17 12:32:22,841] jepsen worker 76 - jepsen.util 7996 :info :write [25 28] :RequestCanceledException INFO [2019-12-17 12:32:22,842] jepsen worker 76 - jepsen.util 8086 :invoke :cas [25 [21 23]] INFO [2019-12-17 12:32:22,857] jepsen worker 6 - jepsen.util 8376 :info :write [2 40] :RequestCanceledException INFO [2019-12-17 12:32:22,858] jepsen worker 6 - jepsen.util 8466 :invoke :cas [2 [6 33]] INFO [2019-12-17 12:32:22,876] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":33904,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:32:22,876] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":33904,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:32:22,876] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":33906,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:32:22,876] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":33906,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:32:22,893] jepsen worker 41 - jepsen.util 8681 :info :cas [13 [33 9]] :RequestCanceledException INFO [2019-12-17 12:32:22,894] jepsen worker 41 - jepsen.util 8771 :invoke :cas [13 [8 11]] INFO [2019-12-17 12:32:22,904] jepsen worker 10 - jepsen.util 5770 :ok :read [3 5] INFO [2019-12-17 12:32:22,904] jepsen worker 10 - jepsen.util 5770 :invoke :read [3 nil] INFO [2019-12-17 12:32:22,910] jepsen worker 65 - jepsen.util 4925 :ok :read [21 46] INFO [2019-12-17 12:32:22,910] jepsen worker 65 - jepsen.util 4925 :invoke :read [21 nil] INFO [2019-12-17 12:32:22,911] jepsen worker 69 - jepsen.util 6099 :info :write [23 24] :RequestCanceledException INFO [2019-12-17 12:32:22,912] jepsen worker 69 - jepsen.util 6189 :invoke :cas [23 [5 36]] INFO [2019-12-17 12:32:22,915] jepsen worker 47 - jepsen.util 6707 :ok :read [15 40] INFO [2019-12-17 12:32:22,916] jepsen worker 47 - jepsen.util 6707 :invoke :write [15 12] INFO [2019-12-17 12:32:22,926] jepsen worker 0 - jepsen.util 8100 :info :write [0 3] :RequestCanceledException INFO [2019-12-17 12:32:22,926] jepsen worker 32 - jepsen.util 5882 :info :cas [10 [13 1]] :RequestCanceledException INFO [2019-12-17 12:32:22,926] jepsen worker 62 - jepsen.util 7622 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:32:22,926] jepsen worker 30 - jepsen.util 5970 :fail :read [10 nil] :RequestTimeoutException INFO [2019-12-17 12:32:22,927] jepsen worker 62 - jepsen.util 7622 :invoke :read [20 nil] INFO [2019-12-17 12:32:22,927] jepsen worker 30 - jepsen.util 5970 :invoke :cas [10 [41 42]] INFO [2019-12-17 12:32:22,928] jepsen worker 0 - jepsen.util 8190 :invoke :cas [0 [8 45]] INFO [2019-12-17 12:32:22,928] jepsen worker 32 - jepsen.util 5972 :invoke :write [10 40] INFO [2019-12-17 12:32:22,929] jepsen worker 10 - jepsen.util 5770 :ok :read [3 5] INFO [2019-12-17 12:32:22,929] jepsen worker 10 - jepsen.util 5770 :invoke :read [3 nil] INFO [2019-12-17 12:32:22,940] jepsen worker 81 - jepsen.util 7641 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:32:22,940] jepsen worker 81 - jepsen.util 7641 :invoke :cas [27 [8 48]] INFO [2019-12-17 12:32:22,979] jepsen worker 65 - jepsen.util 4925 :ok :read [21 46] INFO [2019-12-17 12:32:22,979] jepsen worker 65 - jepsen.util 4925 :invoke :write [21 1] INFO [2019-12-17 12:32:22,991] jepsen worker 79 - jepsen.util 5749 :fail :cas [26 [27 0]] :ValueNotSwapFrom INFO [2019-12-17 12:32:22,992] jepsen worker 79 - jepsen.util 5749 :invoke :read [26 nil] INFO [2019-12-17 12:32:22,994] jepsen worker 29 - jepsen.util 4529 :info :write [9 29] :RequestCanceledException INFO [2019-12-17 12:32:22,994] jepsen worker 88 - jepsen.util 6028 :info :write [29 20] :RequestCanceledException INFO [2019-12-17 12:32:22,995] jepsen worker 29 - jepsen.util 4619 :invoke :write [9 11] INFO [2019-12-17 12:32:22,996] jepsen worker 88 - jepsen.util 6118 :invoke :read [29 nil] INFO [2019-12-17 12:32:22,996] jepsen worker 30 - jepsen.util 5970 :fail :cas [10 [41 42]] :ValueNotSwapFrom INFO [2019-12-17 12:32:22,997] jepsen worker 30 - jepsen.util 5970 :invoke :cas [10 [8 38]] INFO [2019-12-17 12:32:23,011] jepsen worker 83 - jepsen.util 8273 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:32:23,011] jepsen worker 83 - jepsen.util 8273 :invoke :cas [27 [47 36]] INFO [2019-12-17 12:32:23,015] jepsen worker 10 - jepsen.util 5770 :ok :read [3 5] INFO [2019-12-17 12:32:23,015] jepsen worker 10 - jepsen.util 5770 :invoke :read [3 nil] INFO [2019-12-17 12:32:23,021] jepsen worker 56 - jepsen.util 6356 :fail :cas [18 [26 26]] :ValueNotSwapFrom INFO [2019-12-17 12:32:23,022] jepsen worker 56 - jepsen.util 6356 :invoke :write [18 3] INFO [2019-12-17 12:32:23,026] jepsen worker 39 - jepsen.util 7599 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:32:23,026] jepsen worker 39 - jepsen.util 7599 :invoke :read [13 nil] INFO [2019-12-17 12:32:23,031] jepsen worker 30 - jepsen.util 5970 :fail :cas [10 [8 38]] :ValueNotSwapFrom INFO [2019-12-17 12:32:23,032] jepsen worker 30 - jepsen.util 5970 :invoke :cas [10 [40 0]] INFO [2019-12-17 12:32:23,041] jepsen worker 84 - jepsen.util 7464 :info :write [28 49] :RequestCanceledException INFO [2019-12-17 12:32:23,047] jepsen worker 10 - jepsen.util 5770 :ok :read [3 5] INFO [2019-12-17 12:32:23,047] jepsen worker 10 - jepsen.util 5770 :invoke :cas [3 [46 38]] INFO [2019-12-17 12:32:23,047] jepsen worker 84 - jepsen.util 7554 :invoke :write [28 31] INFO [2019-12-17 12:32:23,064] jepsen worker 30 - jepsen.util 5970 :fail :cas [10 [40 0]] :ValueNotSwapFrom INFO [2019-12-17 12:32:23,064] jepsen worker 30 - jepsen.util 5970 :invoke :cas [10 [14 15]] INFO [2019-12-17 12:32:23,064] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":33921,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:32:23,064] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":33921,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:32:23,064] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":33924,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:32:23,064] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":33924,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:32:23,064] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":33925,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:32:23,064] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":33925,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:32:23,074] jepsen worker 10 - jepsen.util 5770 :fail :cas [3 [46 38]] :ValueNotSwapFrom INFO [2019-12-17 12:32:23,075] jepsen worker 10 - jepsen.util 5770 :invoke :read [3 nil] INFO [2019-12-17 12:32:23,084] jepsen worker 74 - jepsen.util 6194 :ok :write [24 42] INFO [2019-12-17 12:32:23,084] jepsen worker 74 - jepsen.util 6194 :invoke :read [24 nil] INFO [2019-12-17 12:32:23,084] jepsen worker 30 - jepsen.util 5970 :fail :cas [10 [14 15]] :ValueNotSwapFrom INFO [2019-12-17 12:32:23,084] jepsen worker 30 - jepsen.util 5970 :invoke :read [10 nil] INFO [2019-12-17 12:32:23,089] jepsen worker 10 - jepsen.util 5770 :ok :read [3 5] INFO [2019-12-17 12:32:23,090] jepsen worker 10 - jepsen.util 5770 :invoke :write [3 26] INFO [2019-12-17 12:32:23,098] jepsen worker 79 - jepsen.util 5749 :ok :read [26 31] INFO [2019-12-17 12:32:23,098] jepsen worker 79 - jepsen.util 5749 :invoke :write [26 41] INFO [2019-12-17 12:32:23,100] jepsen worker 70 - jepsen.util 6280 :ok :write [23 19] INFO [2019-12-17 12:32:23,100] jepsen worker 70 - jepsen.util 6280 :invoke :read [23 nil] INFO [2019-12-17 12:32:23,111] jepsen worker 86 - jepsen.util 8186 :info :write [28 36] :RequestCanceledException INFO [2019-12-17 12:32:23,112] jepsen worker 86 - jepsen.util 8276 :invoke :write [28 47] INFO [2019-12-17 12:32:23,126] jepsen worker 1 - jepsen.util 7651 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:32:23,127] jepsen worker 1 - jepsen.util 7651 :invoke :cas [0 [48 4]] INFO [2019-12-17 12:32:23,133] jepsen worker 74 - jepsen.util 6194 :ok :read [24 42] INFO [2019-12-17 12:32:23,133] jepsen worker 74 - jepsen.util 6194 :invoke :read [24 nil] INFO [2019-12-17 12:32:23,138] jepsen worker 30 - jepsen.util 5970 :ok :read [10 19] INFO [2019-12-17 12:32:23,138] jepsen worker 30 - jepsen.util 5970 :invoke :cas [10 [29 47]] INFO [2019-12-17 12:32:23,140] jepsen worker 21 - jepsen.util 7311 :info :write [7 4] :RequestCanceledException INFO [2019-12-17 12:32:23,140] jepsen worker 61 - jepsen.util 9061 :info :cas [20 [47 8]] :RequestCanceledException INFO [2019-12-17 12:32:23,142] jepsen worker 61 - jepsen.util 9151 :invoke :read [20 nil] INFO [2019-12-17 12:32:23,142] jepsen worker 21 - jepsen.util 7401 :invoke :cas [7 [31 48]] INFO [2019-12-17 12:32:23,163] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:32:23,171] jepsen worker 53 - jepsen.util 5633 :fail :cas [17 [20 12]] :ValueNotSwapFrom INFO [2019-12-17 12:32:23,176] jepsen nemesis - couchbase.util Rebalance check count: 257 INFO [2019-12-17 12:32:23,176] jepsen nemesis - couchbase.util Rebalance stuck count: 238 INFO [2019-12-17 12:32:23,176] jepsen worker 53 - jepsen.util 5633 :invoke :cas [17 [5 36]] INFO [2019-12-17 12:32:23,179] jepsen worker 74 - jepsen.util 6194 :ok :read [24 42] INFO [2019-12-17 12:32:23,183] jepsen worker 74 - jepsen.util 6194 :invoke :cas [24 [23 5]] INFO [2019-12-17 12:32:23,193] jepsen worker 22 - jepsen.util 7852 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:32:23,193] jepsen worker 7 - jepsen.util 8287 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:32:23,193] jepsen worker 22 - jepsen.util 7852 :invoke :cas [7 [16 43]] INFO [2019-12-17 12:32:23,193] jepsen worker 7 - jepsen.util 8287 :invoke :read [2 nil] INFO [2019-12-17 12:32:23,197] jepsen worker 32 - jepsen.util 5972 :ok :write [10 40] INFO [2019-12-17 12:32:23,197] jepsen worker 32 - jepsen.util 5972 :invoke :cas [10 [22 31]] INFO [2019-12-17 12:32:23,199] jepsen worker 74 - jepsen.util 6194 :fail :cas [24 [23 5]] :ValueNotSwapFrom INFO [2019-12-17 12:32:23,199] jepsen worker 45 - jepsen.util 5805 :ok :write [15 26] INFO [2019-12-17 12:32:23,199] jepsen worker 74 - jepsen.util 6194 :invoke :write [24 10] INFO [2019-12-17 12:32:23,199] jepsen worker 45 - jepsen.util 5805 :invoke :write [15 9] INFO [2019-12-17 12:32:23,202] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":33954,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:32:23,202] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":33954,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:32:23,208] jepsen worker 77 - jepsen.util 8087 :info :cas [25 [27 34]] :RequestCanceledException INFO [2019-12-17 12:32:23,208] jepsen worker 17 - jepsen.util 7937 :info :write [5 4] :RequestCanceledException INFO [2019-12-17 12:32:23,208] jepsen worker 72 - jepsen.util 5742 :info :cas [24 [13 45]] :RequestCanceledException INFO [2019-12-17 12:32:23,210] jepsen worker 77 - jepsen.util 8177 :invoke :write [25 22] INFO [2019-12-17 12:32:23,210] jepsen worker 17 - jepsen.util 8027 :invoke :read [5 nil] INFO [2019-12-17 12:32:23,211] jepsen worker 72 - jepsen.util 5832 :invoke :write [24 43] INFO [2019-12-17 12:32:23,214] jepsen worker 70 - jepsen.util 6280 :ok :read [23 19] INFO [2019-12-17 12:32:23,214] jepsen worker 70 - jepsen.util 6280 :invoke :cas [23 [22 1]] INFO [2019-12-17 12:32:23,227] jepsen worker 52 - jepsen.util 6262 :info :write [17 45] :RequestCanceledException INFO [2019-12-17 12:32:23,227] jepsen worker 25 - jepsen.util 8485 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:32:23,227] jepsen worker 26 - jepsen.util 8216 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:32:23,227] jepsen worker 25 - jepsen.util 8485 :invoke :cas [8 [44 25]] INFO [2019-12-17 12:32:23,227] jepsen worker 26 - jepsen.util 8216 :invoke :cas [8 [25 11]] INFO [2019-12-17 12:32:23,228] jepsen worker 52 - jepsen.util 6352 :invoke :cas [17 [25 41]] INFO [2019-12-17 12:32:23,236] jepsen worker 32 - jepsen.util 5972 :fail :cas [10 [22 31]] :ValueNotSwapFrom INFO [2019-12-17 12:32:23,236] jepsen worker 32 - jepsen.util 5972 :invoke :write [10 14] INFO [2019-12-17 12:32:23,241] jepsen worker 40 - jepsen.util 8320 :info :write [13 21] :RequestCanceledException INFO [2019-12-17 12:32:23,241] jepsen worker 71 - jepsen.util 6551 :info :write [23 24] :RequestCanceledException INFO [2019-12-17 12:32:23,242] jepsen worker 40 - jepsen.util 8410 :invoke :write [13 6] INFO [2019-12-17 12:32:23,242] jepsen worker 71 - jepsen.util 6641 :invoke :write [23 32] INFO [2019-12-17 12:32:23,255] jepsen worker 70 - jepsen.util 6280 :fail :cas [23 [22 1]] :ValueNotSwapFrom INFO [2019-12-17 12:32:23,255] jepsen worker 70 - jepsen.util 6280 :invoke :write [23 36] INFO [2019-12-17 12:32:23,257] jepsen worker 73 - jepsen.util 5563 :fail :read [24 nil] :RequestTimeoutException INFO [2019-12-17 12:32:23,257] jepsen worker 15 - jepsen.util 6855 :info :cas [5 [48 46]] :RequestCanceledException INFO [2019-12-17 12:32:23,258] jepsen worker 73 - jepsen.util 5563 :invoke :cas [24 [5 30]] INFO [2019-12-17 12:32:23,259] jepsen worker 15 - jepsen.util 6945 :invoke :write [5 38] INFO [2019-12-17 12:32:23,288] jepsen worker 52 - jepsen.util 6352 :fail :cas [17 [25 41]] :ValueNotSwapFrom INFO [2019-12-17 12:32:23,288] jepsen worker 52 - jepsen.util 6352 :invoke :cas [17 [2 1]] INFO [2019-12-17 12:32:23,293] jepsen worker 4 - jepsen.util 5314 :info :write [1 14] :RequestCanceledException INFO [2019-12-17 12:32:23,293] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":33965,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:32:23,294] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":33965,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:32:23,294] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":33968,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:32:23,294] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":33968,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:32:23,294] jepsen worker 4 - jepsen.util 5404 :invoke :read [1 nil] INFO [2019-12-17 12:32:23,326] jepsen worker 27 - jepsen.util 5787 :info :write [9 33] :RequestCanceledException INFO [2019-12-17 12:32:23,326] jepsen worker 67 - jepsen.util 8077 :info :cas [22 [35 12]] :RequestCanceledException INFO [2019-12-17 12:32:23,328] jepsen worker 27 - jepsen.util 5877 :invoke :read [9 nil] INFO [2019-12-17 12:32:23,328] jepsen worker 67 - jepsen.util 8167 :invoke :write [22 15] INFO [2019-12-17 12:32:23,337] jepsen worker 80 - jepsen.util 5930 :ok :write [26 15] INFO [2019-12-17 12:32:23,337] jepsen worker 80 - jepsen.util 5930 :invoke :write [26 2] INFO [2019-12-17 12:32:23,340] jepsen worker 13 - jepsen.util 5683 :info :write [4 47] :RequestCanceledException INFO [2019-12-17 12:32:23,342] jepsen worker 13 - jepsen.util 5773 :invoke :read [4 nil] INFO [2019-12-17 12:32:23,358] jepsen worker 63 - jepsen.util 4923 :fail :read [21 nil] :RequestTimeoutException INFO [2019-12-17 12:32:23,358] jepsen worker 63 - jepsen.util 4923 :invoke :write [21 10] INFO [2019-12-17 12:32:23,361] jepsen worker 28 - jepsen.util 4528 :info :write [9 14] :RequestCanceledException INFO [2019-12-17 12:32:23,366] jepsen worker 28 - jepsen.util 4618 :invoke :write [9 4] INFO [2019-12-17 12:32:23,411] jepsen worker 12 - jepsen.util 6222 :ok :write [4 42] INFO [2019-12-17 12:32:23,411] jepsen worker 12 - jepsen.util 6222 :invoke :write [4 15] INFO [2019-12-17 12:32:23,426] jepsen worker 35 - jepsen.util 8045 :info :write [11 7] :RequestCanceledException INFO [2019-12-17 12:32:23,426] jepsen worker 24 - jepsen.util 7134 :info :cas [8 [11 30]] :RequestCanceledException INFO [2019-12-17 12:32:23,427] jepsen worker 42 - jepsen.util 8502 :info :cas [14 [5 7]] :RequestCanceledException INFO [2019-12-17 12:32:23,427] jepsen worker 24 - jepsen.util 7224 :invoke :write [8 28] INFO [2019-12-17 12:32:23,427] jepsen worker 35 - jepsen.util 8135 :invoke :read [11 nil] INFO [2019-12-17 12:32:23,428] jepsen worker 42 - jepsen.util 8592 :invoke :write [14 5] INFO [2019-12-17 12:32:23,466] jepsen worker 38 - jepsen.util 5978 :ok :read [12 15] INFO [2019-12-17 12:32:23,466] jepsen worker 38 - jepsen.util 5978 :invoke :write [12 35] INFO [2019-12-17 12:32:23,493] jepsen worker 49 - jepsen.util 7069 :info :cas [16 [45 40]] :RequestCanceledException INFO [2019-12-17 12:32:23,494] jepsen worker 49 - jepsen.util 7159 :invoke :cas [16 [42 1]] INFO [2019-12-17 12:32:23,505] jepsen worker 18 - jepsen.util 5688 :ok :write [6 28] INFO [2019-12-17 12:32:23,506] jepsen worker 18 - jepsen.util 5688 :invoke :write [6 0] INFO [2019-12-17 12:32:23,511] jepsen worker 59 - jepsen.util 7979 :info :write [19 26] :RequestCanceledException INFO [2019-12-17 12:32:23,511] jepsen worker 37 - jepsen.util 5437 :info :write [12 28] :RequestCanceledException INFO [2019-12-17 12:32:23,512] jepsen worker 37 - jepsen.util 5527 :invoke :read [12 nil] INFO [2019-12-17 12:32:23,513] jepsen worker 59 - jepsen.util 8069 :invoke :cas [19 [8 20]] INFO [2019-12-17 12:32:23,527] jepsen worker 55 - jepsen.util 6805 :info :write [18 5] :RequestCanceledException INFO [2019-12-17 12:32:23,527] jepsen worker 23 - jepsen.util 7853 :info :write [7 2] :RequestCanceledException INFO [2019-12-17 12:32:23,527] jepsen worker 64 - jepsen.util 4564 :info :cas [21 [6 28]] :RequestCanceledException INFO [2019-12-17 12:32:23,528] jepsen worker 23 - jepsen.util 7943 :invoke :cas [7 [31 36]] INFO [2019-12-17 12:32:23,528] jepsen worker 55 - jepsen.util 6895 :invoke :cas [18 [17 45]] INFO [2019-12-17 12:32:23,529] jepsen worker 64 - jepsen.util 4654 :invoke :read [21 nil] INFO [2019-12-17 12:32:23,531] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":33991,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:32:23,531] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":33991,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:32:23,531] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":33995,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:32:23,532] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":33995,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:32:23,532] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":33996,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:32:23,532] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":33996,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:32:23,611] jepsen worker 36 - jepsen.util 6246 :info :write [12 40] :RequestCanceledException INFO [2019-12-17 12:32:23,613] jepsen worker 36 - jepsen.util 6336 :invoke :write [12 41] INFO [2019-12-17 12:32:23,640] jepsen worker 2 - jepsen.util 7652 :info :write [0 33] :RequestCanceledException INFO [2019-12-17 12:32:23,641] jepsen worker 2 - jepsen.util 7742 :invoke :cas [0 [37 46]] INFO [2019-12-17 12:32:23,677] jepsen worker 71 - jepsen.util 6641 :ok :write [23 32] INFO [2019-12-17 12:32:23,678] jepsen worker 71 - jepsen.util 6641 :invoke :write [23 11] INFO [2019-12-17 12:32:23,684] jepsen worker 69 - jepsen.util 6189 :fail :cas [23 [5 36]] :ValueNotSwapFrom INFO [2019-12-17 12:32:23,684] jepsen worker 69 - jepsen.util 6189 :invoke :read [23 nil] INFO [2019-12-17 12:32:23,748] jepsen worker 45 - jepsen.util 5805 :ok :write [15 9] INFO [2019-12-17 12:32:23,748] jepsen worker 10 - jepsen.util 5770 :ok :write [3 26] INFO [2019-12-17 12:32:23,748] jepsen worker 45 - jepsen.util 5805 :invoke :cas [15 [12 42]] INFO [2019-12-17 12:32:23,748] jepsen worker 10 - jepsen.util 5770 :invoke :write [3 23] INFO [2019-12-17 12:32:23,749] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":34012,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:32:23,749] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":34012,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:32:23,749] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":34014,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:32:23,749] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":34014,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:32:23,753] jepsen worker 32 - jepsen.util 5972 :ok :write [10 14] INFO [2019-12-17 12:32:23,754] jepsen worker 32 - jepsen.util 5972 :invoke :write [10 37] INFO [2019-12-17 12:32:23,754] jepsen worker 74 - jepsen.util 6194 :ok :write [24 10] INFO [2019-12-17 12:32:23,754] jepsen worker 74 - jepsen.util 6194 :invoke :cas [24 [0 29]] INFO [2019-12-17 12:32:23,770] jepsen worker 18 - jepsen.util 5688 :ok :write [6 0] INFO [2019-12-17 12:32:23,770] jepsen worker 18 - jepsen.util 5688 :invoke :cas [6 [15 13]] INFO [2019-12-17 12:32:23,784] jepsen worker 88 - jepsen.util 6118 :ok :read [29 9] INFO [2019-12-17 12:32:23,784] jepsen worker 88 - jepsen.util 6118 :invoke :cas [29 [33 24]] INFO [2019-12-17 12:32:23,826] jepsen worker 50 - jepsen.util 8240 :info :cas [16 [47 33]] :RequestCanceledException INFO [2019-12-17 12:32:23,826] jepsen worker 9 - jepsen.util 6309 :fail :read [3 nil] :RequestTimeoutException INFO [2019-12-17 12:32:23,827] jepsen worker 9 - jepsen.util 6309 :invoke :cas [3 [31 22]] INFO [2019-12-17 12:32:23,829] jepsen worker 50 - jepsen.util 8330 :invoke :cas [16 [20 15]] INFO [2019-12-17 12:32:23,832] jepsen worker 74 - jepsen.util 6194 :fail :cas [24 [0 29]] :ValueNotSwapFrom INFO [2019-12-17 12:32:23,832] jepsen worker 74 - jepsen.util 6194 :invoke :read [24 nil] INFO [2019-12-17 12:32:23,858] jepsen worker 18 - jepsen.util 5688 :fail :cas [6 [15 13]] :ValueNotSwapFrom INFO [2019-12-17 12:32:23,858] jepsen worker 18 - jepsen.util 5688 :invoke :write [6 42] INFO [2019-12-17 12:32:23,861] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":34027,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:32:23,861] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":34027,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:32:23,861] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":34030,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:32:23,861] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":34030,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:32:23,861] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":34031,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:32:23,862] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":34031,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:32:23,926] jepsen worker 46 - jepsen.util 5806 :info :write [15 20] :RequestCanceledException INFO [2019-12-17 12:32:23,927] jepsen worker 46 - jepsen.util 5896 :invoke :cas [15 [45 5]] INFO [2019-12-17 12:32:23,941] jepsen worker 20 - jepsen.util 5780 :info :write [6 9] :RequestCanceledException INFO [2019-12-17 12:32:23,943] jepsen worker 20 - jepsen.util 5870 :invoke :cas [6 [5 5]] INFO [2019-12-17 12:32:23,953] jepsen worker 13 - jepsen.util 5773 :ok :read [4 42] INFO [2019-12-17 12:32:23,953] jepsen worker 13 - jepsen.util 5773 :invoke :cas [4 [8 20]] INFO [2019-12-17 12:32:23,956] jepsen worker 74 - jepsen.util 6194 :ok :read [24 10] INFO [2019-12-17 12:32:23,957] jepsen worker 74 - jepsen.util 6194 :invoke :write [24 12] INFO [2019-12-17 12:32:23,989] jepsen worker 19 - jepsen.util 5869 :info :write [6 17] :RequestCanceledException INFO [2019-12-17 12:32:23,990] jepsen worker 19 - jepsen.util 5959 :invoke :read [6 nil] INFO [2019-12-17 12:32:23,999] jepsen worker 53 - jepsen.util 5633 :fail :cas [17 [5 36]] :ValueNotSwapFrom INFO [2019-12-17 12:32:23,999] jepsen worker 53 - jepsen.util 5633 :invoke :cas [17 [16 1]] INFO [2019-12-17 12:32:24,001] jepsen worker 20 - jepsen.util 5870 :fail :cas [6 [5 5]] :ValueNotSwapFrom INFO [2019-12-17 12:32:24,001] jepsen worker 20 - jepsen.util 5870 :invoke :cas [6 [20 13]] INFO [2019-12-17 12:32:24,005] jepsen worker 28 - jepsen.util 4618 :ok :write [9 4] INFO [2019-12-17 12:32:24,005] jepsen worker 28 - jepsen.util 4618 :invoke :write [9 41] INFO [2019-12-17 12:32:24,011] jepsen worker 87 - jepsen.util 5397 :info :write [29 29] :RequestCanceledException INFO [2019-12-17 12:32:24,012] jepsen worker 87 - jepsen.util 5487 :invoke :write [29 39] INFO [2019-12-17 12:32:24,015] jepsen worker 63 - jepsen.util 4923 :ok :write [21 10] INFO [2019-12-17 12:32:24,015] jepsen worker 63 - jepsen.util 4923 :invoke :write [21 23] INFO [2019-12-17 12:32:24,016] jepsen worker 72 - jepsen.util 5832 :ok :write [24 43] INFO [2019-12-17 12:32:24,016] jepsen worker 72 - jepsen.util 5832 :invoke :read [24 nil] INFO [2019-12-17 12:32:24,046] jepsen worker 4 - jepsen.util 5404 :ok :read [1 2] INFO [2019-12-17 12:32:24,046] jepsen worker 4 - jepsen.util 5404 :invoke :cas [1 [12 30]] INFO [2019-12-17 12:32:24,062] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":34038,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:32:24,062] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":34038,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:32:24,062] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":34039,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:32:24,062] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":34039,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:32:24,062] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":34040,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:32:24,063] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":34040,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:32:24,075] jepsen worker 64 - jepsen.util 4654 :ok :read [21 10] INFO [2019-12-17 12:32:24,075] jepsen worker 64 - jepsen.util 4654 :invoke :read [21 nil] INFO [2019-12-17 12:32:24,092] jepsen worker 37 - jepsen.util 5527 :ok :read [12 15] INFO [2019-12-17 12:32:24,092] jepsen worker 37 - jepsen.util 5527 :invoke :write [12 3] INFO [2019-12-17 12:32:24,093] jepsen worker 5 - jepsen.util 5495 :info :write [1 8] :RequestCanceledException INFO [2019-12-17 12:32:24,094] jepsen worker 5 - jepsen.util 5585 :invoke :cas [1 [40 18]] INFO [2019-12-17 12:32:24,130] jepsen worker 63 - jepsen.util 4923 :ok :write [21 23] INFO [2019-12-17 12:32:24,131] jepsen worker 63 - jepsen.util 4923 :invoke :read [21 nil] INFO [2019-12-17 12:32:24,147] jepsen worker 13 - jepsen.util 5773 :fail :cas [4 [8 20]] :ValueNotSwapFrom INFO [2019-12-17 12:32:24,147] jepsen worker 13 - jepsen.util 5773 :invoke :cas [4 [26 49]] INFO [2019-12-17 12:32:24,159] jepsen worker 63 - jepsen.util 4923 :ok :read [21 23] INFO [2019-12-17 12:32:24,159] jepsen worker 63 - jepsen.util 4923 :invoke :read [21 nil] INFO [2019-12-17 12:32:24,179] jepsen worker 63 - jepsen.util 4923 :ok :read [21 23] INFO [2019-12-17 12:32:24,180] jepsen worker 63 - jepsen.util 4923 :invoke :cas [21 [11 32]] INFO [2019-12-17 12:32:24,196] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:32:24,196] jepsen nemesis - couchbase.util Rebalance check count: 258 INFO [2019-12-17 12:32:24,196] jepsen nemesis - couchbase.util Rebalance stuck count: 239 INFO [2019-12-17 12:32:24,204] jepsen worker 63 - jepsen.util 4923 :fail :cas [21 [11 32]] :ValueNotSwapFrom INFO [2019-12-17 12:32:24,205] jepsen worker 63 - jepsen.util 4923 :invoke :write [21 7] INFO [2019-12-17 12:32:24,257] jepsen worker 51 - jepsen.util 5811 :info :write [17 36] :RequestCanceledException INFO [2019-12-17 12:32:24,258] jepsen worker 51 - jepsen.util 5901 :invoke :read [17 nil] INFO [2019-12-17 12:32:24,266] jepsen worker 32 - jepsen.util 5972 :ok :write [10 37] INFO [2019-12-17 12:32:24,268] jepsen worker 32 - jepsen.util 5972 :invoke :write [10 3] INFO [2019-12-17 12:32:24,278] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":34065,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:32:24,278] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":34065,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:32:24,278] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":34068,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:32:24,278] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":34068,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:32:24,278] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":34069,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:32:24,278] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":34069,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:32:24,295] jepsen worker 63 - jepsen.util 4923 :ok :write [21 7] INFO [2019-12-17 12:32:24,295] jepsen worker 63 - jepsen.util 4923 :invoke :read [21 nil] INFO [2019-12-17 12:32:24,297] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":34058,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:32:24,297] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":34058,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:32:24,297] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":34060,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:32:24,297] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":34060,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:32:24,305] jepsen worker 28 - jepsen.util 4618 :ok :write [9 41] INFO [2019-12-17 12:32:24,305] jepsen worker 28 - jepsen.util 4618 :invoke :read [9 nil] INFO [2019-12-17 12:32:24,334] jepsen worker 55 - jepsen.util 6895 :fail :cas [18 [17 45]] :ValueNotSwapFrom INFO [2019-12-17 12:32:24,335] jepsen worker 55 - jepsen.util 6895 :invoke :cas [18 [20 28]] INFO [2019-12-17 12:32:24,336] jepsen worker 63 - jepsen.util 4923 :ok :read [21 7] INFO [2019-12-17 12:32:24,336] jepsen worker 63 - jepsen.util 4923 :invoke :cas [21 [0 0]] INFO [2019-12-17 12:32:24,354] jepsen worker 63 - jepsen.util 4923 :fail :cas [21 [0 0]] :ValueNotSwapFrom INFO [2019-12-17 12:32:24,354] jepsen worker 63 - jepsen.util 4923 :invoke :read [21 nil] INFO [2019-12-17 12:32:24,355] jepsen worker 88 - jepsen.util 6118 :fail :cas [29 [33 24]] :ValueNotSwapFrom INFO [2019-12-17 12:32:24,355] jepsen worker 88 - jepsen.util 6118 :invoke :read [29 nil] INFO [2019-12-17 12:32:24,358] jepsen worker 14 - jepsen.util 6044 :info :write [4 37] :RequestCanceledException INFO [2019-12-17 12:32:24,359] jepsen worker 14 - jepsen.util 6134 :invoke :cas [4 [13 21]] INFO [2019-12-17 12:32:24,374] jepsen worker 51 - jepsen.util 5901 :ok :read [17 39] INFO [2019-12-17 12:32:24,374] jepsen worker 51 - jepsen.util 5901 :invoke :write [17 16] INFO [2019-12-17 12:32:24,382] jepsen worker 63 - jepsen.util 4923 :ok :read [21 7] INFO [2019-12-17 12:32:24,382] jepsen worker 63 - jepsen.util 4923 :invoke :read [21 nil] INFO [2019-12-17 12:32:24,405] jepsen worker 63 - jepsen.util 4923 :ok :read [21 7] INFO [2019-12-17 12:32:24,405] jepsen worker 63 - jepsen.util 4923 :invoke :read [21 nil] INFO [2019-12-17 12:32:24,411] jepsen worker 14 - jepsen.util 6134 :fail :cas [4 [13 21]] :ValueNotSwapFrom INFO [2019-12-17 12:32:24,411] jepsen worker 14 - jepsen.util 6134 :invoke :read [4 nil] INFO [2019-12-17 12:32:24,413] jepsen worker 5 - jepsen.util 5585 :fail :cas [1 [40 18]] :ValueNotSwapFrom INFO [2019-12-17 12:32:24,413] jepsen worker 5 - jepsen.util 5585 :invoke :write [1 24] INFO [2019-12-17 12:32:24,429] jepsen worker 68 - jepsen.util 7538 :info :cas [22 [37 28]] :RequestCanceledException INFO [2019-12-17 12:32:24,430] jepsen worker 68 - jepsen.util 7628 :invoke :cas [22 [38 9]] INFO [2019-12-17 12:32:24,441] jepsen worker 11 - jepsen.util 6131 :info :write [3 8] :RequestCanceledException INFO [2019-12-17 12:32:24,441] jepsen worker 3 - jepsen.util 5853 :info :write [1 42] :RequestCanceledException INFO [2019-12-17 12:32:24,442] jepsen worker 11 - jepsen.util 6221 :invoke :write [3 44] INFO [2019-12-17 12:32:24,442] jepsen worker 3 - jepsen.util 5943 :invoke :read [1 nil] INFO [2019-12-17 12:32:24,450] jepsen worker 52 - jepsen.util 6352 :fail :cas [17 [2 1]] :ValueNotSwapFrom INFO [2019-12-17 12:32:24,450] jepsen worker 52 - jepsen.util 6352 :invoke :cas [17 [20 7]] INFO [2019-12-17 12:32:24,456] jepsen worker 78 - jepsen.util 6198 :info :write [26 46] :RequestCanceledException INFO [2019-12-17 12:32:24,457] jepsen worker 4 - jepsen.util 5404 :fail :cas [1 [12 30]] :ValueNotSwapFrom INFO [2019-12-17 12:32:24,457] jepsen worker 4 - jepsen.util 5404 :invoke :write [1 4] INFO [2019-12-17 12:32:24,458] jepsen worker 78 - jepsen.util 6288 :invoke :read [26 nil] INFO [2019-12-17 12:32:24,459] jepsen worker 69 - jepsen.util 6189 :ok :read [23 32] INFO [2019-12-17 12:32:24,459] jepsen worker 69 - jepsen.util 6189 :invoke :write [23 28] INFO [2019-12-17 12:32:24,464] jepsen worker 63 - jepsen.util 4923 :ok :read [21 7] INFO [2019-12-17 12:32:24,464] jepsen worker 63 - jepsen.util 4923 :invoke :write [21 42] INFO [2019-12-17 12:32:24,505] jepsen worker 14 - jepsen.util 6134 :ok :read [4 42] INFO [2019-12-17 12:32:24,505] jepsen worker 14 - jepsen.util 6134 :invoke :cas [4 [4 18]] INFO [2019-12-17 12:32:24,512] jepsen worker 63 - jepsen.util 4923 :ok :write [21 42] INFO [2019-12-17 12:32:24,512] jepsen worker 63 - jepsen.util 4923 :invoke :cas [21 [13 20]] INFO [2019-12-17 12:32:24,516] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":34085,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:32:24,516] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":34085,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:32:24,516] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":34090,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:32:24,517] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":34090,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:32:24,538] jepsen worker 63 - jepsen.util 4923 :fail :cas [21 [13 20]] :ValueNotSwapFrom INFO [2019-12-17 12:32:24,538] jepsen worker 63 - jepsen.util 4923 :invoke :read [21 nil] INFO [2019-12-17 12:32:24,541] jepsen worker 14 - jepsen.util 6134 :fail :cas [4 [4 18]] :ValueNotSwapFrom INFO [2019-12-17 12:32:24,542] jepsen worker 14 - jepsen.util 6134 :invoke :write [4 34] INFO [2019-12-17 12:32:24,554] jepsen worker 53 - jepsen.util 5633 :fail :cas [17 [16 1]] :ValueNotSwapFrom INFO [2019-12-17 12:32:24,554] jepsen worker 18 - jepsen.util 5688 :ok :write [6 42] INFO [2019-12-17 12:32:24,554] jepsen worker 63 - jepsen.util 4923 :ok :read [21 42] INFO [2019-12-17 12:32:24,555] jepsen worker 53 - jepsen.util 5633 :invoke :read [17 nil] INFO [2019-12-17 12:32:24,555] jepsen worker 18 - jepsen.util 5688 :invoke :read [6 nil] INFO [2019-12-17 12:32:24,555] jepsen worker 63 - jepsen.util 4923 :invoke :cas [21 [21 11]] INFO [2019-12-17 12:32:24,555] jepsen worker 74 - jepsen.util 6194 :ok :write [24 12] INFO [2019-12-17 12:32:24,555] jepsen worker 74 - jepsen.util 6194 :invoke :write [24 48] INFO [2019-12-17 12:32:24,555] jepsen worker 32 - jepsen.util 5972 :ok :write [10 3] INFO [2019-12-17 12:32:24,555] jepsen worker 32 - jepsen.util 5972 :invoke :read [10 nil] INFO [2019-12-17 12:32:24,558] jepsen worker 66 - jepsen.util 7446 :info :cas [22 [27 25]] :RequestCanceledException INFO [2019-12-17 12:32:24,560] jepsen worker 66 - jepsen.util 7536 :invoke :read [22 nil] INFO [2019-12-17 12:32:24,576] jepsen worker 32 - jepsen.util 5972 :ok :read [10 3] INFO [2019-12-17 12:32:24,576] jepsen worker 32 - jepsen.util 5972 :invoke :read [10 nil] INFO [2019-12-17 12:32:24,580] jepsen worker 13 - jepsen.util 5773 :fail :cas [4 [26 49]] :ValueNotSwapFrom INFO [2019-12-17 12:32:24,580] jepsen worker 13 - jepsen.util 5773 :invoke :cas [4 [35 39]] INFO [2019-12-17 12:32:24,594] jepsen worker 10 - jepsen.util 5770 :ok :write [3 23] INFO [2019-12-17 12:32:24,595] jepsen worker 10 - jepsen.util 5770 :invoke :write [3 4] INFO [2019-12-17 12:32:24,611] jepsen worker 54 - jepsen.util 6354 :info :write [18 14] :RequestCanceledException INFO [2019-12-17 12:32:24,611] jepsen worker 75 - jepsen.util 7635 :info :cas [25 [44 8]] :RequestCanceledException INFO [2019-12-17 12:32:24,623] jepsen worker 44 - jepsen.util 8684 :info :cas [14 [42 31]] :RequestCanceledException INFO [2019-12-17 12:32:24,630] jepsen worker 19 - jepsen.util 5959 :ok :read [6 42] INFO [2019-12-17 12:32:24,638] jepsen worker 16 - jepsen.util 9016 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:32:24,649] jepsen worker 32 - jepsen.util 5972 :ok :read [10 3] INFO [2019-12-17 12:32:24,651] jepsen worker 18 - jepsen.util 5688 :ok :read [6 42] INFO [2019-12-17 12:32:24,651] jepsen worker 8 - jepsen.util 7478 :info :write [2 2] :RequestCanceledException INFO [2019-12-17 12:32:24,652] jepsen worker 72 - jepsen.util 5832 :ok :read [24 12] INFO [2019-12-17 12:32:24,660] jepsen worker 64 - jepsen.util 4654 :ok :read [21 42] INFO [2019-12-17 12:32:24,663] jepsen worker 63 - jepsen.util 4923 :fail :cas [21 [21 11]] :ValueNotSwapFrom INFO [2019-12-17 12:32:24,689] jepsen worker 74 - jepsen.util 6194 :ok :write [24 48] INFO [2019-12-17 12:32:24,689] jepsen worker 31 - jepsen.util 5251 :info :write [10 38] :RequestCanceledException INFO [2019-12-17 12:32:24,689] jepsen worker 82 - jepsen.util 8452 :info :cas [27 [42 6]] :RequestCanceledException INFO [2019-12-17 12:32:24,689] jepsen worker 85 - jepsen.util 8815 :info :cas [28 [3 36]] :RequestCanceledException INFO [2019-12-17 12:32:24,689] jepsen worker 89 - jepsen.util 5669 :info :write [29 6] :RequestCanceledException INFO [2019-12-17 12:32:24,711] jepsen worker 48 - jepsen.util 7158 :info :write [16 37] :RequestCanceledException INFO [2019-12-17 12:32:24,711] jepsen worker 60 - jepsen.util 8790 :info :write [20 47] :RequestCanceledException INFO [2019-12-17 12:32:24,716] jepsen worker 11 - jepsen.util 6221 :ok :write [3 44] INFO [2019-12-17 12:32:24,726] jepsen worker 57 - jepsen.util 7617 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:32:24,726] jepsen worker 43 - jepsen.util 7423 :info :cas [14 [14 20]] :RequestCanceledException INFO [2019-12-17 12:32:24,730] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":34113,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:32:24,741] jepsen worker 19 - jepsen.util 5959 :invoke :write [6 18] INFO [2019-12-17 12:32:24,741] jepsen worker 16 - jepsen.util 9016 :invoke :read [5 nil] INFO [2019-12-17 12:32:24,741] jepsen worker 32 - jepsen.util 5972 :invoke :cas [10 [49 29]] INFO [2019-12-17 12:32:24,741] jepsen worker 54 - jepsen.util 6444 :invoke :cas [18 [26 41]] INFO [2019-12-17 12:32:24,741] jepsen worker 75 - jepsen.util 7725 :invoke :cas [25 [15 4]] INFO [2019-12-17 12:32:24,741] jepsen worker 44 - jepsen.util 8774 :invoke :read [14 nil] INFO [2019-12-17 12:32:24,742] jepsen worker 18 - jepsen.util 5688 :invoke :write [6 34] INFO [2019-12-17 12:32:24,742] jepsen worker 72 - jepsen.util 5832 :invoke :read [24 nil] INFO [2019-12-17 12:32:24,742] jepsen worker 64 - jepsen.util 4654 :invoke :cas [21 [36 8]] INFO [2019-12-17 12:32:24,742] jepsen worker 74 - jepsen.util 6194 :invoke :read [24 nil] INFO [2019-12-17 12:32:24,742] jepsen worker 63 - jepsen.util 4923 :invoke :cas [21 [42 11]] INFO [2019-12-17 12:32:24,743] jepsen worker 8 - jepsen.util 7568 :invoke :write [2 19] INFO [2019-12-17 12:32:24,743] jepsen worker 82 - jepsen.util 8542 :invoke :write [27 31] INFO [2019-12-17 12:32:24,743] jepsen worker 31 - jepsen.util 5341 :invoke :read [10 nil] INFO [2019-12-17 12:32:24,743] jepsen worker 85 - jepsen.util 8905 :invoke :write [28 5] INFO [2019-12-17 12:32:24,744] jepsen worker 11 - jepsen.util 6221 :invoke :cas [3 [18 41]] INFO [2019-12-17 12:32:24,744] jepsen worker 57 - jepsen.util 7617 :invoke :write [19 13] INFO [2019-12-17 12:32:24,744] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":34113,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:32:24,744] jepsen worker 60 - jepsen.util 8880 :invoke :cas [20 [36 25]] INFO [2019-12-17 12:32:24,745] jepsen worker 43 - jepsen.util 7513 :invoke :write [14 7] INFO [2019-12-17 12:32:24,749] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":34117,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:32:24,750] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":34117,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:32:24,751] jepsen worker 48 - jepsen.util 7248 :invoke :write [16 2] INFO [2019-12-17 12:32:24,771] jepsen worker 88 - jepsen.util 6118 :ok :read [29 9] INFO [2019-12-17 12:32:24,772] jepsen worker 89 - jepsen.util 5759 :invoke :write [29 30] INFO [2019-12-17 12:32:24,778] jepsen worker 88 - jepsen.util 6118 :invoke :cas [29 [11 20]] INFO [2019-12-17 12:32:24,804] jepsen worker 78 - jepsen.util 6288 :ok :read [26 15] INFO [2019-12-17 12:32:24,804] jepsen worker 78 - jepsen.util 6288 :invoke :write [26 37] INFO [2019-12-17 12:32:24,827] jepsen worker 33 - jepsen.util 7953 :info :cas [11 [22 32]] :RequestCanceledException INFO [2019-12-17 12:32:24,830] jepsen worker 33 - jepsen.util 8043 :invoke :read [11 nil] INFO [2019-12-17 12:32:24,838] jepsen worker 55 - jepsen.util 6895 :fail :cas [18 [20 28]] :ValueNotSwapFrom INFO [2019-12-17 12:32:24,838] jepsen worker 55 - jepsen.util 6895 :invoke :cas [18 [20 17]] INFO [2019-12-17 12:32:24,855] jepsen worker 32 - jepsen.util 5972 :fail :cas [10 [49 29]] :ValueNotSwapFrom INFO [2019-12-17 12:32:24,856] jepsen worker 32 - jepsen.util 5972 :invoke :write [10 37] INFO [2019-12-17 12:32:24,857] jepsen worker 14 - jepsen.util 6134 :ok :write [4 34] INFO [2019-12-17 12:32:24,857] jepsen worker 34 - jepsen.util 7504 :info :cas [11 [29 7]] :RequestCanceledException INFO [2019-12-17 12:32:24,859] jepsen worker 34 - jepsen.util 7594 :invoke :read [11 nil] INFO [2019-12-17 12:32:24,861] jepsen worker 14 - jepsen.util 6134 :invoke :write [4 44] INFO [2019-12-17 12:32:24,882] jepsen worker 74 - jepsen.util 6194 :ok :read [24 48] INFO [2019-12-17 12:32:24,882] jepsen worker 74 - jepsen.util 6194 :invoke :write [24 48] INFO [2019-12-17 12:32:24,888] jepsen worker 72 - jepsen.util 5832 :ok :read [24 48] INFO [2019-12-17 12:32:24,888] jepsen worker 72 - jepsen.util 5832 :invoke :cas [24 [11 20]] INFO [2019-12-17 12:32:24,894] jepsen worker 76 - jepsen.util 8086 :info :cas [25 [21 23]] :RequestCanceledException INFO [2019-12-17 12:32:24,895] jepsen worker 76 - jepsen.util 8176 :invoke :cas [25 [21 42]] INFO [2019-12-17 12:32:24,901] jepsen worker 31 - jepsen.util 5341 :ok :read [10 3] INFO [2019-12-17 12:32:24,901] jepsen worker 31 - jepsen.util 5341 :invoke :cas [10 [29 18]] INFO [2019-12-17 12:32:24,908] jepsen worker 6 - jepsen.util 8466 :info :cas [2 [6 33]] :RequestCanceledException INFO [2019-12-17 12:32:24,909] jepsen worker 6 - jepsen.util 8556 :invoke :cas [2 [20 37]] INFO [2019-12-17 12:32:24,921] jepsen worker 64 - jepsen.util 4654 :fail :cas [21 [36 8]] :ValueNotSwapFrom INFO [2019-12-17 12:32:24,922] jepsen worker 64 - jepsen.util 4654 :invoke :read [21 nil] INFO [2019-12-17 12:32:24,927] jepsen worker 58 - jepsen.util 8518 :info :write [19 40] :RequestCanceledException INFO [2019-12-17 12:32:24,927] jepsen worker 41 - jepsen.util 8771 :info :cas [13 [8 11]] :RequestCanceledException INFO [2019-12-17 12:32:24,928] jepsen worker 58 - jepsen.util 8608 :invoke :read [19 nil] INFO [2019-12-17 12:32:24,928] jepsen worker 41 - jepsen.util 8861 :invoke :read [13 nil] INFO [2019-12-17 12:32:24,946] jepsen worker 11 - jepsen.util 6221 :fail :cas [3 [18 41]] :ValueNotSwapFrom INFO [2019-12-17 12:32:24,946] jepsen worker 11 - jepsen.util 6221 :invoke :write [3 29] INFO [2019-12-17 12:32:24,949] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":34146,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:32:24,949] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":34146,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:32:24,949] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":34149,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:32:24,949] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":34149,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:32:24,954] jepsen worker 0 - jepsen.util 8190 :info :cas [0 [8 45]] :RequestCanceledException INFO [2019-12-17 12:32:24,955] jepsen worker 0 - jepsen.util 8280 :invoke :cas [0 [34 7]] INFO [2019-12-17 12:32:24,955] jepsen worker 13 - jepsen.util 5773 :fail :cas [4 [35 39]] :ValueNotSwapFrom INFO [2019-12-17 12:32:24,956] jepsen worker 13 - jepsen.util 5773 :invoke :cas [4 [28 17]] INFO [2019-12-17 12:32:24,956] jepsen worker 54 - jepsen.util 6444 :fail :cas [18 [26 41]] :ValueNotSwapFrom INFO [2019-12-17 12:32:24,956] jepsen worker 54 - jepsen.util 6444 :invoke :write [18 47] INFO [2019-12-17 12:32:24,971] jepsen worker 53 - jepsen.util 5633 :ok :read [17 39] INFO [2019-12-17 12:32:24,971] jepsen worker 53 - jepsen.util 5633 :invoke :read [17 nil] INFO [2019-12-17 12:32:24,993] jepsen worker 47 - jepsen.util 6707 :info :write [15 12] :RequestCanceledException INFO [2019-12-17 12:32:24,998] jepsen worker 47 - jepsen.util 6797 :invoke :read [15 nil] INFO [2019-12-17 12:32:25,026] jepsen worker 65 - jepsen.util 4925 :info :write [21 1] :RequestCanceledException INFO [2019-12-17 12:32:25,026] jepsen worker 62 - jepsen.util 7622 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:32:25,027] jepsen worker 62 - jepsen.util 7622 :invoke :write [20 29] INFO [2019-12-17 12:32:25,028] jepsen worker 65 - jepsen.util 5015 :invoke :write [21 15] INFO [2019-12-17 12:32:25,028] jepsen worker 12 - jepsen.util 6222 :ok :write [4 15] INFO [2019-12-17 12:32:25,028] jepsen worker 12 - jepsen.util 6222 :invoke :read [4 nil] INFO [2019-12-17 12:32:25,031] jepsen worker 47 - jepsen.util 6797 :ok :read [15 9] INFO [2019-12-17 12:32:25,031] jepsen worker 47 - jepsen.util 6797 :invoke :write [15 23] INFO [2019-12-17 12:32:25,040] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":34167,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:32:25,040] jepsen worker 56 - jepsen.util 6356 :info :write [18 3] :RequestCanceledException INFO [2019-12-17 12:32:25,040] jepsen worker 81 - jepsen.util 7641 :info :cas [27 [8 48]] :RequestCanceledException INFO [2019-12-17 12:32:25,040] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":34167,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:32:25,040] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":34171,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:32:25,040] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":34171,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:32:25,041] jepsen worker 56 - jepsen.util 6446 :invoke :cas [18 [30 23]] INFO [2019-12-17 12:32:25,041] jepsen worker 81 - jepsen.util 7731 :invoke :cas [27 [45 44]] INFO [2019-12-17 12:32:25,079] jepsen worker 38 - jepsen.util 5978 :ok :write [12 35] INFO [2019-12-17 12:32:25,079] jepsen worker 38 - jepsen.util 5978 :invoke :write [12 3] INFO [2019-12-17 12:32:25,093] jepsen worker 29 - jepsen.util 4619 :info :write [9 11] :RequestCanceledException INFO [2019-12-17 12:32:25,094] jepsen worker 29 - jepsen.util 4709 :invoke :cas [9 [22 43]] INFO [2019-12-17 12:32:25,111] jepsen worker 83 - jepsen.util 8273 :info :cas [27 [47 36]] :RequestCanceledException INFO [2019-12-17 12:32:25,112] jepsen worker 83 - jepsen.util 8363 :invoke :write [27 9] INFO [2019-12-17 12:32:25,122] jepsen worker 71 - jepsen.util 6641 :ok :write [23 11] INFO [2019-12-17 12:32:25,122] jepsen worker 71 - jepsen.util 6641 :invoke :read [23 nil] INFO [2019-12-17 12:32:25,125] jepsen worker 39 - jepsen.util 7599 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:32:25,125] jepsen worker 84 - jepsen.util 7554 :info :write [28 31] :RequestCanceledException INFO [2019-12-17 12:32:25,125] jepsen worker 39 - jepsen.util 7599 :invoke :read [13 nil] INFO [2019-12-17 12:32:25,126] jepsen worker 84 - jepsen.util 7644 :invoke :cas [28 [23 5]] INFO [2019-12-17 12:32:25,129] jepsen worker 88 - jepsen.util 6118 :fail :cas [29 [11 20]] :ValueNotSwapFrom INFO [2019-12-17 12:32:25,130] jepsen worker 88 - jepsen.util 6118 :invoke :write [29 46] INFO [2019-12-17 12:32:25,136] jepsen worker 14 - jepsen.util 6134 :ok :write [4 44] INFO [2019-12-17 12:32:25,137] jepsen worker 14 - jepsen.util 6134 :invoke :cas [4 [4 20]] INFO [2019-12-17 12:32:25,142] jepsen worker 86 - jepsen.util 8276 :info :write [28 47] :RequestCanceledException INFO [2019-12-17 12:32:25,142] jepsen worker 79 - jepsen.util 5749 :info :write [26 41] :RequestCanceledException INFO [2019-12-17 12:32:25,143] jepsen worker 79 - jepsen.util 5839 :invoke :read [26 nil] INFO [2019-12-17 12:32:25,146] jepsen worker 86 - jepsen.util 8366 :invoke :write [28 14] INFO [2019-12-17 12:32:25,150] jepsen worker 63 - jepsen.util 4923 :ok :cas [21 [42 11]] INFO [2019-12-17 12:32:25,151] jepsen worker 55 - jepsen.util 6895 :fail :cas [18 [20 17]] :ValueNotSwapFrom INFO [2019-12-17 12:32:25,151] jepsen worker 55 - jepsen.util 6895 :invoke :read [18 nil] INFO [2019-12-17 12:32:25,151] jepsen worker 63 - jepsen.util 4923 :invoke :write [21 42] INFO [2019-12-17 12:32:25,159] jepsen worker 61 - jepsen.util 9151 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:32:25,159] jepsen worker 61 - jepsen.util 9151 :invoke :cas [20 [38 20]] INFO [2019-12-17 12:32:25,174] jepsen worker 4 - jepsen.util 5404 :ok :write [1 4] INFO [2019-12-17 12:32:25,174] jepsen worker 4 - jepsen.util 5404 :invoke :read [1 nil] INFO [2019-12-17 12:32:25,195] jepsen worker 29 - jepsen.util 4709 :fail :cas [9 [22 43]] :ValueNotSwapFrom INFO [2019-12-17 12:32:25,196] jepsen worker 29 - jepsen.util 4709 :invoke :read [9 nil] INFO [2019-12-17 12:32:25,212] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:32:25,212] jepsen nemesis - couchbase.util Rebalance check count: 259 INFO [2019-12-17 12:32:25,212] jepsen nemesis - couchbase.util Rebalance stuck count: 240 INFO [2019-12-17 12:32:25,227] jepsen worker 21 - jepsen.util 7401 :info :cas [7 [31 48]] :RequestCanceledException INFO [2019-12-17 12:32:25,227] jepsen worker 1 - jepsen.util 7651 :info :cas [0 [48 4]] :RequestCanceledException INFO [2019-12-17 12:32:25,227] jepsen worker 30 - jepsen.util 5970 :info :cas [10 [29 47]] :RequestCanceledException INFO [2019-12-17 12:32:25,228] jepsen worker 21 - jepsen.util 7491 :invoke :cas [7 [31 12]] INFO [2019-12-17 12:32:25,228] jepsen worker 1 - jepsen.util 7741 :invoke :read [0 nil] INFO [2019-12-17 12:32:25,228] jepsen worker 30 - jepsen.util 6060 :invoke :write [10 47] INFO [2019-12-17 12:32:25,237] jepsen worker 53 - jepsen.util 5633 :ok :read [17 39] INFO [2019-12-17 12:32:25,237] jepsen worker 53 - jepsen.util 5633 :invoke :cas [17 [18 21]] INFO [2019-12-17 12:32:25,249] jepsen worker 51 - jepsen.util 5901 :ok :write [17 16] INFO [2019-12-17 12:32:25,249] jepsen worker 51 - jepsen.util 5901 :invoke :read [17 nil] INFO [2019-12-17 12:32:25,256] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":34202,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:32:25,256] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":34202,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:32:25,257] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":34205,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:32:25,257] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":34205,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:32:25,257] jepsen worker 40 - jepsen.util 8410 :info :write [13 6] :RequestCanceledException INFO [2019-12-17 12:32:25,260] jepsen worker 40 - jepsen.util 8500 :invoke :read [13 nil] INFO [2019-12-17 12:32:25,266] jepsen worker 63 - jepsen.util 4923 :ok :write [21 42] INFO [2019-12-17 12:32:25,266] jepsen worker 63 - jepsen.util 4923 :invoke :cas [21 [48 1]] INFO [2019-12-17 12:32:25,278] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":34194,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:32:25,278] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":34194,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:32:25,293] jepsen worker 7 - jepsen.util 8287 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:32:25,293] jepsen worker 22 - jepsen.util 7852 :info :cas [7 [16 43]] :RequestCanceledException INFO [2019-12-17 12:32:25,293] jepsen worker 77 - jepsen.util 8177 :info :write [25 22] :RequestCanceledException INFO [2019-12-17 12:32:25,293] jepsen worker 7 - jepsen.util 8287 :invoke :cas [2 [20 40]] INFO [2019-12-17 12:32:25,294] jepsen worker 15 - jepsen.util 6945 :info :write [5 38] :RequestCanceledException INFO [2019-12-17 12:32:25,294] jepsen worker 22 - jepsen.util 7942 :invoke :write [7 39] INFO [2019-12-17 12:32:25,294] jepsen worker 77 - jepsen.util 8267 :invoke :cas [25 [44 35]] INFO [2019-12-17 12:32:25,295] jepsen worker 15 - jepsen.util 7035 :invoke :write [5 13] INFO [2019-12-17 12:32:25,304] jepsen worker 71 - jepsen.util 6641 :ok :read [23 11] INFO [2019-12-17 12:32:25,304] jepsen worker 71 - jepsen.util 6641 :invoke :write [23 33] INFO [2019-12-17 12:32:25,311] jepsen worker 17 - jepsen.util 8027 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:32:25,311] jepsen worker 17 - jepsen.util 8027 :invoke :write [5 42] INFO [2019-12-17 12:32:25,313] jepsen worker 12 - jepsen.util 6222 :ok :read [4 44] INFO [2019-12-17 12:32:25,313] jepsen worker 12 - jepsen.util 6222 :invoke :read [4 nil] INFO [2019-12-17 12:32:25,321] jepsen worker 63 - jepsen.util 4923 :fail :cas [21 [48 1]] :ValueNotSwapFrom INFO [2019-12-17 12:32:25,321] jepsen worker 63 - jepsen.util 4923 :invoke :read [21 nil] INFO [2019-12-17 12:32:25,326] jepsen worker 26 - jepsen.util 8216 :info :cas [8 [25 11]] :RequestCanceledException INFO [2019-12-17 12:32:25,326] jepsen worker 25 - jepsen.util 8485 :info :cas [8 [44 25]] :RequestCanceledException INFO [2019-12-17 12:32:25,326] jepsen worker 70 - jepsen.util 6280 :info :write [23 36] :RequestCanceledException INFO [2019-12-17 12:32:25,327] jepsen worker 25 - jepsen.util 8575 :invoke :write [8 20] INFO [2019-12-17 12:32:25,327] jepsen worker 26 - jepsen.util 8306 :invoke :cas [8 [32 24]] INFO [2019-12-17 12:32:25,327] jepsen worker 70 - jepsen.util 6370 :invoke :cas [23 [10 36]] INFO [2019-12-17 12:32:25,348] jepsen worker 51 - jepsen.util 5901 :ok :read [17 16] INFO [2019-12-17 12:32:25,348] jepsen worker 51 - jepsen.util 5901 :invoke :read [17 nil] INFO [2019-12-17 12:32:25,358] jepsen worker 73 - jepsen.util 5563 :info :cas [24 [5 30]] :RequestCanceledException INFO [2019-12-17 12:32:25,358] jepsen worker 80 - jepsen.util 5930 :info :write [26 2] :RequestCanceledException INFO [2019-12-17 12:32:25,359] jepsen worker 80 - jepsen.util 6020 :invoke :write [26 7] INFO [2019-12-17 12:32:25,360] jepsen worker 73 - jepsen.util 5653 :invoke :read [24 nil] INFO [2019-12-17 12:32:25,376] jepsen worker 63 - jepsen.util 4923 :ok :read [21 42] INFO [2019-12-17 12:32:25,376] jepsen worker 63 - jepsen.util 4923 :invoke :cas [21 [45 2]] INFO [2019-12-17 12:32:25,389] jepsen worker 54 - jepsen.util 6444 :ok :write [18 47] INFO [2019-12-17 12:32:25,389] jepsen worker 54 - jepsen.util 6444 :invoke :read [18 nil] INFO [2019-12-17 12:32:25,404] jepsen worker 63 - jepsen.util 4923 :fail :cas [21 [45 2]] :ValueNotSwapFrom INFO [2019-12-17 12:32:25,404] jepsen worker 63 - jepsen.util 4923 :invoke :cas [21 [19 22]] INFO [2019-12-17 12:32:25,416] jepsen worker 14 - jepsen.util 6134 :fail :cas [4 [4 20]] :ValueNotSwapFrom INFO [2019-12-17 12:32:25,416] jepsen worker 14 - jepsen.util 6134 :invoke :write [4 32] INFO [2019-12-17 12:32:25,422] jepsen worker 38 - jepsen.util 5978 :ok :write [12 3] INFO [2019-12-17 12:32:25,422] jepsen worker 38 - jepsen.util 5978 :invoke :cas [12 [9 37]] INFO [2019-12-17 12:32:25,427] jepsen worker 27 - jepsen.util 5877 :fail :read [9 nil] :RequestTimeoutException INFO [2019-12-17 12:32:25,427] jepsen worker 27 - jepsen.util 5877 :invoke :write [9 28] INFO [2019-12-17 12:32:25,440] jepsen worker 67 - jepsen.util 8167 :info :write [22 15] :RequestCanceledException INFO [2019-12-17 12:32:25,440] jepsen worker 63 - jepsen.util 4923 :fail :cas [21 [19 22]] :ValueNotSwapFrom INFO [2019-12-17 12:32:25,441] jepsen worker 63 - jepsen.util 4923 :invoke :write [21 23] INFO [2019-12-17 12:32:25,441] jepsen worker 67 - jepsen.util 8257 :invoke :read [22 nil] INFO [2019-12-17 12:32:25,474] jepsen worker 70 - jepsen.util 6370 :fail :cas [23 [10 36]] :ValueNotSwapFrom INFO [2019-12-17 12:32:25,474] jepsen worker 70 - jepsen.util 6370 :invoke :cas [23 [49 31]] INFO [2019-12-17 12:32:25,489] jepsen worker 63 - jepsen.util 4923 :ok :write [21 23] INFO [2019-12-17 12:32:25,489] jepsen worker 63 - jepsen.util 4923 :invoke :read [21 nil] INFO [2019-12-17 12:32:25,493] jepsen worker 24 - jepsen.util 7224 :info :write [8 28] :RequestCanceledException INFO [2019-12-17 12:32:25,494] jepsen worker 24 - jepsen.util 7314 :invoke :read [8 nil] INFO [2019-12-17 12:32:25,494] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":34235,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:32:25,494] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":34235,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:32:25,494] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":34237,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:32:25,494] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":34237,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:32:25,504] jepsen worker 35 - jepsen.util 8135 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:32:25,504] jepsen worker 35 - jepsen.util 8135 :invoke :read [11 nil] INFO [2019-12-17 12:32:25,509] jepsen worker 51 - jepsen.util 5901 :ok :read [17 16] INFO [2019-12-17 12:32:25,510] jepsen worker 51 - jepsen.util 5901 :invoke :cas [17 [19 14]] INFO [2019-12-17 12:32:25,513] jepsen worker 63 - jepsen.util 4923 :ok :read [21 23] INFO [2019-12-17 12:32:25,513] jepsen worker 63 - jepsen.util 4923 :invoke :cas [21 [14 45]] INFO [2019-12-17 12:32:25,525] jepsen worker 42 - jepsen.util 8592 :info :write [14 5] :RequestCanceledException INFO [2019-12-17 12:32:25,525] jepsen worker 59 - jepsen.util 8069 :info :cas [19 [8 20]] :RequestCanceledException INFO [2019-12-17 12:32:25,526] jepsen worker 54 - jepsen.util 6444 :ok :read [18 47] INFO [2019-12-17 12:32:25,526] jepsen worker 54 - jepsen.util 6444 :invoke :read [18 nil] INFO [2019-12-17 12:32:25,527] jepsen worker 59 - jepsen.util 8159 :invoke :cas [19 [28 18]] INFO [2019-12-17 12:32:25,527] jepsen worker 42 - jepsen.util 8682 :invoke :read [14 nil] INFO [2019-12-17 12:32:25,533] jepsen worker 63 - jepsen.util 4923 :fail :cas [21 [14 45]] :ValueNotSwapFrom INFO [2019-12-17 12:32:25,533] jepsen worker 63 - jepsen.util 4923 :invoke :write [21 30] INFO [2019-12-17 12:32:25,540] jepsen worker 49 - jepsen.util 7159 :info :cas [16 [42 1]] :RequestCanceledException INFO [2019-12-17 12:32:25,541] jepsen worker 49 - jepsen.util 7249 :invoke :read [16 nil] INFO [2019-12-17 12:32:25,581] jepsen worker 63 - jepsen.util 4923 :ok :write [21 30] INFO [2019-12-17 12:32:25,581] jepsen worker 63 - jepsen.util 4923 :invoke :cas [21 [47 22]] INFO [2019-12-17 12:32:25,588] jepsen worker 79 - jepsen.util 5839 :ok :read [26 2] INFO [2019-12-17 12:32:25,588] jepsen worker 79 - jepsen.util 5839 :invoke :cas [26 [38 21]] INFO [2019-12-17 12:32:25,591] jepsen worker 55 - jepsen.util 6895 :ok :read [18 47] INFO [2019-12-17 12:32:25,591] jepsen worker 55 - jepsen.util 6895 :invoke :write [18 3] INFO [2019-12-17 12:32:25,593] jepsen worker 23 - jepsen.util 7943 :info :cas [7 [31 36]] :RequestCanceledException INFO [2019-12-17 12:32:25,594] jepsen worker 23 - jepsen.util 8033 :invoke :cas [7 [48 43]] INFO [2019-12-17 12:32:25,603] jepsen worker 87 - jepsen.util 5487 :ok :write [29 39] INFO [2019-12-17 12:32:25,603] jepsen worker 87 - jepsen.util 5487 :invoke :cas [29 [24 18]] INFO [2019-12-17 12:32:25,613] jepsen worker 63 - jepsen.util 4923 :fail :cas [21 [47 22]] :ValueNotSwapFrom INFO [2019-12-17 12:32:25,616] jepsen worker 63 - jepsen.util 4923 :invoke :cas [21 [3 15]] INFO [2019-12-17 12:32:25,642] jepsen worker 36 - jepsen.util 6336 :info :write [12 41] :RequestCanceledException INFO [2019-12-17 12:32:25,644] jepsen worker 3 - jepsen.util 5943 :ok :read [1 4] INFO [2019-12-17 12:32:25,645] jepsen worker 3 - jepsen.util 5943 :invoke :cas [1 [31 38]] INFO [2019-12-17 12:32:25,649] jepsen worker 36 - jepsen.util 6426 :invoke :write [12 21] INFO [2019-12-17 12:32:25,654] jepsen worker 63 - jepsen.util 4923 :fail :cas [21 [3 15]] :ValueNotSwapFrom INFO [2019-12-17 12:32:25,655] jepsen worker 63 - jepsen.util 4923 :invoke :write [21 44] INFO [2019-12-17 12:32:25,660] jepsen worker 2 - jepsen.util 7742 :info :cas [0 [37 46]] :RequestCanceledException INFO [2019-12-17 12:32:25,664] jepsen worker 2 - jepsen.util 7832 :invoke :read [0 nil] INFO [2019-12-17 12:32:25,683] jepsen worker 56 - jepsen.util 6446 :fail :cas [18 [30 23]] :ValueNotSwapFrom INFO [2019-12-17 12:32:25,683] jepsen worker 56 - jepsen.util 6446 :invoke :read [18 nil] INFO [2019-12-17 12:32:25,703] jepsen worker 32 - jepsen.util 5972 :ok :write [10 37] INFO [2019-12-17 12:32:25,703] jepsen worker 32 - jepsen.util 5972 :invoke :cas [10 [31 38]] INFO [2019-12-17 12:32:25,705] jepsen worker 74 - jepsen.util 6194 :ok :write [24 48] INFO [2019-12-17 12:32:25,705] jepsen worker 74 - jepsen.util 6194 :invoke :cas [24 [40 6]] INFO [2019-12-17 12:32:25,707] jepsen worker 88 - jepsen.util 6118 :ok :write [29 46] INFO [2019-12-17 12:32:25,707] jepsen worker 88 - jepsen.util 6118 :invoke :write [29 16] INFO [2019-12-17 12:32:25,714] jepsen worker 52 - jepsen.util 6352 :fail :cas [17 [20 7]] :ValueNotSwapFrom INFO [2019-12-17 12:32:25,714] jepsen worker 52 - jepsen.util 6352 :invoke :read [17 nil] INFO [2019-12-17 12:32:25,718] jepsen worker 19 - jepsen.util 5959 :ok :write [6 18] INFO [2019-12-17 12:32:25,718] jepsen worker 19 - jepsen.util 5959 :invoke :read [6 nil] INFO [2019-12-17 12:32:25,719] jepsen worker 14 - jepsen.util 6134 :ok :write [4 32] INFO [2019-12-17 12:32:25,719] jepsen worker 14 - jepsen.util 6134 :invoke :read [4 nil] INFO [2019-12-17 12:32:25,722] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":34259,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:32:25,722] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":34259,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:32:25,722] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":34262,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:32:25,722] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":34262,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:32:25,722] jepsen worker 47 - jepsen.util 6797 :ok :write [15 23] INFO [2019-12-17 12:32:25,722] jepsen worker 47 - jepsen.util 6797 :invoke :write [15 32] INFO [2019-12-17 12:32:25,724] jepsen worker 4 - jepsen.util 5404 :ok :read [1 4] INFO [2019-12-17 12:32:25,725] jepsen worker 4 - jepsen.util 5404 :invoke :read [1 nil] INFO [2019-12-17 12:32:25,750] jepsen worker 70 - jepsen.util 6370 :fail :cas [23 [49 31]] :ValueNotSwapFrom INFO [2019-12-17 12:32:25,750] jepsen worker 70 - jepsen.util 6370 :invoke :cas [23 [5 4]] INFO [2019-12-17 12:32:25,754] jepsen worker 53 - jepsen.util 5633 :fail :cas [17 [18 21]] :ValueNotSwapFrom INFO [2019-12-17 12:32:25,754] jepsen worker 53 - jepsen.util 5633 :invoke :cas [17 [42 45]] INFO [2019-12-17 12:32:25,760] jepsen worker 56 - jepsen.util 6446 :ok :read [18 47] INFO [2019-12-17 12:32:25,760] jepsen worker 56 - jepsen.util 6446 :invoke :read [18 nil] INFO [2019-12-17 12:32:25,768] jepsen worker 64 - jepsen.util 4654 :ok :read [21 30] INFO [2019-12-17 12:32:25,768] jepsen worker 64 - jepsen.util 4654 :invoke :read [21 nil] INFO [2019-12-17 12:32:25,775] jepsen worker 31 - jepsen.util 5341 :fail :cas [10 [29 18]] :ValueNotSwapFrom INFO [2019-12-17 12:32:25,775] jepsen worker 31 - jepsen.util 5341 :invoke :cas [10 [41 26]] INFO [2019-12-17 12:32:25,775] jepsen worker 74 - jepsen.util 6194 :fail :cas [24 [40 6]] :ValueNotSwapFrom INFO [2019-12-17 12:32:25,775] jepsen worker 74 - jepsen.util 6194 :invoke :cas [24 [13 12]] INFO [2019-12-17 12:32:25,778] jepsen worker 54 - jepsen.util 6444 :ok :read [18 47] INFO [2019-12-17 12:32:25,778] jepsen worker 54 - jepsen.util 6444 :invoke :read [18 nil] INFO [2019-12-17 12:32:25,779] jepsen worker 12 - jepsen.util 6222 :ok :read [4 32] INFO [2019-12-17 12:32:25,780] jepsen worker 12 - jepsen.util 6222 :invoke :write [4 43] INFO [2019-12-17 12:32:25,817] jepsen worker 63 - jepsen.util 4923 :ok :write [21 44] INFO [2019-12-17 12:32:25,817] jepsen worker 63 - jepsen.util 4923 :invoke :read [21 nil] INFO [2019-12-17 12:32:25,824] jepsen worker 74 - jepsen.util 6194 :fail :cas [24 [13 12]] :ValueNotSwapFrom INFO [2019-12-17 12:32:25,825] jepsen worker 74 - jepsen.util 6194 :invoke :read [24 nil] INFO [2019-12-17 12:32:25,838] jepsen worker 63 - jepsen.util 4923 :ok :read [21 44] INFO [2019-12-17 12:32:25,839] jepsen worker 63 - jepsen.util 4923 :invoke :cas [21 [1 49]] INFO [2019-12-17 12:32:25,842] jepsen worker 74 - jepsen.util 6194 :ok :read [24 48] INFO [2019-12-17 12:32:25,842] jepsen worker 74 - jepsen.util 6194 :invoke :cas [24 [37 47]] INFO [2019-12-17 12:32:25,858] jepsen worker 45 - jepsen.util 5805 :info :cas [15 [12 42]] :RequestCanceledException INFO [2019-12-17 12:32:25,859] jepsen worker 45 - jepsen.util 5895 :invoke :read [15 nil] INFO [2019-12-17 12:32:25,870] jepsen worker 56 - jepsen.util 6446 :ok :read [18 47] INFO [2019-12-17 12:32:25,870] jepsen worker 56 - jepsen.util 6446 :invoke :cas [18 [7 10]] INFO [2019-12-17 12:32:25,882] jepsen worker 14 - jepsen.util 6134 :ok :read [4 32] INFO [2019-12-17 12:32:25,882] jepsen worker 14 - jepsen.util 6134 :invoke :write [4 41] INFO [2019-12-17 12:32:25,889] jepsen worker 74 - jepsen.util 6194 :fail :cas [24 [37 47]] :ValueNotSwapFrom INFO [2019-12-17 12:32:25,890] jepsen worker 74 - jepsen.util 6194 :invoke :write [24 45] INFO [2019-12-17 12:32:25,925] jepsen worker 9 - jepsen.util 6309 :info :cas [3 [31 22]] :RequestCanceledException INFO [2019-12-17 12:32:25,925] jepsen worker 50 - jepsen.util 8330 :info :cas [16 [20 15]] :RequestCanceledException INFO [2019-12-17 12:32:25,926] jepsen worker 63 - jepsen.util 4923 :fail :cas [21 [1 49]] :ValueNotSwapFrom INFO [2019-12-17 12:32:25,926] jepsen worker 63 - jepsen.util 4923 :invoke :write [21 38] INFO [2019-12-17 12:32:25,926] jepsen worker 50 - jepsen.util 8420 :invoke :cas [16 [26 19]] INFO [2019-12-17 12:32:25,929] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":34291,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:32:25,929] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":34291,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:32:25,930] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":34295,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:32:25,930] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":34295,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:32:25,930] jepsen worker 9 - jepsen.util 6399 :invoke :cas [3 [33 39]] INFO [2019-12-17 12:32:25,960] jepsen worker 80 - jepsen.util 6020 :ok :write [26 7] INFO [2019-12-17 12:32:25,960] jepsen worker 80 - jepsen.util 6020 :invoke :cas [26 [37 48]] INFO [2019-12-17 12:32:25,965] jepsen worker 56 - jepsen.util 6446 :fail :cas [18 [7 10]] :ValueNotSwapFrom INFO [2019-12-17 12:32:25,965] jepsen worker 56 - jepsen.util 6446 :invoke :cas [18 [4 22]] INFO [2019-12-17 12:32:25,968] jepsen worker 70 - jepsen.util 6370 :fail :cas [23 [5 4]] :ValueNotSwapFrom INFO [2019-12-17 12:32:25,969] jepsen worker 70 - jepsen.util 6370 :invoke :write [23 6] INFO [2019-12-17 12:32:25,988] jepsen worker 63 - jepsen.util 4923 :ok :write [21 38] INFO [2019-12-17 12:32:25,988] jepsen worker 63 - jepsen.util 4923 :invoke :read [21 nil] INFO [2019-12-17 12:32:25,992] jepsen worker 46 - jepsen.util 5896 :info :cas [15 [45 5]] :RequestCanceledException INFO [2019-12-17 12:32:25,993] jepsen worker 46 - jepsen.util 5986 :invoke :read [15 nil] INFO [2019-12-17 12:32:26,016] jepsen worker 69 - jepsen.util 6189 :ok :write [23 28] INFO [2019-12-17 12:32:26,016] jepsen worker 69 - jepsen.util 6189 :invoke :cas [23 [34 32]] INFO [2019-12-17 12:32:26,024] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":34309,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:32:26,024] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":34309,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:32:26,024] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":34312,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:32:26,024] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":34312,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:32:26,043] jepsen worker 54 - jepsen.util 6444 :ok :read [18 47] INFO [2019-12-17 12:32:26,043] jepsen worker 63 - jepsen.util 4923 :ok :read [21 38] INFO [2019-12-17 12:32:26,043] jepsen worker 54 - jepsen.util 6444 :invoke :cas [18 [23 24]] INFO [2019-12-17 12:32:26,044] jepsen worker 63 - jepsen.util 4923 :invoke :cas [21 [1 36]] INFO [2019-12-17 12:32:26,058] jepsen worker 63 - jepsen.util 4923 :fail :cas [21 [1 36]] :ValueNotSwapFrom INFO [2019-12-17 12:32:26,062] jepsen worker 63 - jepsen.util 4923 :invoke :read [21 nil] INFO [2019-12-17 12:32:26,086] jepsen worker 4 - jepsen.util 5404 :ok :read [1 4] INFO [2019-12-17 12:32:26,086] jepsen worker 4 - jepsen.util 5404 :invoke :write [1 9] INFO [2019-12-17 12:32:26,093] jepsen worker 51 - jepsen.util 5901 :fail :cas [17 [19 14]] :ValueNotSwapFrom INFO [2019-12-17 12:32:26,093] jepsen worker 51 - jepsen.util 5901 :invoke :write [17 21] INFO [2019-12-17 12:32:26,093] jepsen worker 63 - jepsen.util 4923 :ok :read [21 38] INFO [2019-12-17 12:32:26,093] jepsen worker 63 - jepsen.util 4923 :invoke :read [21 nil] INFO [2019-12-17 12:32:26,106] jepsen worker 63 - jepsen.util 4923 :ok :read [21 38] INFO [2019-12-17 12:32:26,108] jepsen worker 63 - jepsen.util 4923 :invoke :write [21 20] INFO [2019-12-17 12:32:26,111] jepsen worker 20 - jepsen.util 5870 :info :cas [6 [20 13]] :RequestCanceledException INFO [2019-12-17 12:32:26,112] jepsen worker 20 - jepsen.util 5960 :invoke :write [6 0] INFO [2019-12-17 12:32:26,157] jepsen worker 37 - jepsen.util 5527 :info :write [12 3] :RequestCanceledException INFO [2019-12-17 12:32:26,158] jepsen worker 37 - jepsen.util 5617 :invoke :write [12 6] INFO [2019-12-17 12:32:26,175] jepsen worker 53 - jepsen.util 5633 :fail :cas [17 [42 45]] :ValueNotSwapFrom INFO [2019-12-17 12:32:26,175] jepsen worker 53 - jepsen.util 5633 :invoke :cas [17 [30 26]] INFO [2019-12-17 12:32:26,184] jepsen worker 63 - jepsen.util 4923 :ok :write [21 20] INFO [2019-12-17 12:32:26,184] jepsen worker 63 - jepsen.util 4923 :invoke :cas [21 [25 29]] INFO [2019-12-17 12:32:26,206] jepsen worker 63 - jepsen.util 4923 :fail :cas [21 [25 29]] :ValueNotSwapFrom INFO [2019-12-17 12:32:26,206] jepsen worker 63 - jepsen.util 4923 :invoke :read [21 nil] INFO [2019-12-17 12:32:26,218] jepsen worker 63 - jepsen.util 4923 :ok :read [21 20] INFO [2019-12-17 12:32:26,218] jepsen worker 63 - jepsen.util 4923 :invoke :cas [21 [9 5]] INFO [2019-12-17 12:32:26,230] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:32:26,230] jepsen nemesis - couchbase.util Rebalance check count: 260 INFO [2019-12-17 12:32:26,230] jepsen nemesis - couchbase.util Rebalance stuck count: 241 INFO [2019-12-17 12:32:26,244] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":34333,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:32:26,246] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":34333,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:32:26,246] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":34337,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:32:26,246] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":34337,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:32:26,251] jepsen worker 56 - jepsen.util 6446 :fail :cas [18 [4 22]] :ValueNotSwapFrom INFO [2019-12-17 12:32:26,251] jepsen worker 56 - jepsen.util 6446 :invoke :write [18 30] INFO [2019-12-17 12:32:26,269] jepsen worker 63 - jepsen.util 4923 :fail :cas [21 [9 5]] :ValueNotSwapFrom INFO [2019-12-17 12:32:26,269] jepsen worker 63 - jepsen.util 4923 :invoke :read [21 nil] INFO [2019-12-17 12:32:26,281] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":34325,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:32:26,283] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":34325,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:32:26,284] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":34326,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:32:26,284] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":34326,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:32:26,284] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":34327,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:32:26,284] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":34327,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:32:26,289] jepsen worker 63 - jepsen.util 4923 :ok :read [21 20] INFO [2019-12-17 12:32:26,289] jepsen worker 63 - jepsen.util 4923 :invoke :read [21 nil] INFO [2019-12-17 12:32:26,295] jepsen worker 3 - jepsen.util 5943 :fail :cas [1 [31 38]] :ValueNotSwapFrom INFO [2019-12-17 12:32:26,295] jepsen worker 3 - jepsen.util 5943 :invoke :write [1 25] INFO [2019-12-17 12:32:26,298] jepsen worker 87 - jepsen.util 5487 :fail :cas [29 [24 18]] :ValueNotSwapFrom INFO [2019-12-17 12:32:26,298] jepsen worker 87 - jepsen.util 5487 :invoke :read [29 nil] INFO [2019-12-17 12:32:26,304] jepsen worker 63 - jepsen.util 4923 :ok :read [21 20] INFO [2019-12-17 12:32:26,304] jepsen worker 63 - jepsen.util 4923 :invoke :read [21 nil] INFO [2019-12-17 12:32:26,317] jepsen worker 30 - jepsen.util 6060 :ok :write [10 47] INFO [2019-12-17 12:32:26,318] jepsen worker 30 - jepsen.util 6060 :invoke :write [10 49] INFO [2019-12-17 12:32:26,320] jepsen worker 11 - jepsen.util 6221 :ok :write [3 29] INFO [2019-12-17 12:32:26,320] jepsen worker 11 - jepsen.util 6221 :invoke :write [3 36] INFO [2019-12-17 12:32:26,322] jepsen worker 47 - jepsen.util 6797 :ok :write [15 32] INFO [2019-12-17 12:32:26,322] jepsen worker 47 - jepsen.util 6797 :invoke :read [15 nil] INFO [2019-12-17 12:32:26,326] jepsen worker 28 - jepsen.util 4618 :fail :read [9 nil] :RequestTimeoutException INFO [2019-12-17 12:32:26,327] jepsen worker 28 - jepsen.util 4618 :invoke :write [9 8] INFO [2019-12-17 12:32:26,331] jepsen worker 63 - jepsen.util 4923 :ok :read [21 20] INFO [2019-12-17 12:32:26,331] jepsen worker 63 - jepsen.util 4923 :invoke :read [21 nil] INFO [2019-12-17 12:32:26,345] jepsen worker 36 - jepsen.util 6426 :ok :write [12 21] INFO [2019-12-17 12:32:26,345] jepsen worker 36 - jepsen.util 6426 :invoke :read [12 nil] INFO [2019-12-17 12:32:26,350] jepsen worker 12 - jepsen.util 6222 :ok :write [4 43] INFO [2019-12-17 12:32:26,350] jepsen worker 12 - jepsen.util 6222 :invoke :write [4 6] INFO [2019-12-17 12:32:26,359] jepsen worker 31 - jepsen.util 5341 :fail :cas [10 [41 26]] :ValueNotSwapFrom INFO [2019-12-17 12:32:26,359] jepsen worker 31 - jepsen.util 5341 :invoke :write [10 13] INFO [2019-12-17 12:32:26,393] jepsen worker 52 - jepsen.util 6352 :ok :read [17 16] INFO [2019-12-17 12:32:26,393] jepsen worker 52 - jepsen.util 6352 :invoke :cas [17 [6 19]] INFO [2019-12-17 12:32:26,402] jepsen worker 45 - jepsen.util 5895 :ok :read [15 32] INFO [2019-12-17 12:32:26,402] jepsen worker 45 - jepsen.util 5895 :invoke :cas [15 [8 26]] INFO [2019-12-17 12:32:26,404] jepsen worker 73 - jepsen.util 5653 :ok :read [24 48] INFO [2019-12-17 12:32:26,405] jepsen worker 73 - jepsen.util 5653 :invoke :write [24 47] INFO [2019-12-17 12:32:26,407] jepsen worker 38 - jepsen.util 5978 :fail :cas [12 [9 37]] :ValueNotSwapFrom INFO [2019-12-17 12:32:26,407] jepsen worker 38 - jepsen.util 5978 :invoke :cas [12 [5 25]] INFO [2019-12-17 12:32:26,410] jepsen worker 63 - jepsen.util 4923 :ok :read [21 20] INFO [2019-12-17 12:32:26,410] jepsen worker 63 - jepsen.util 4923 :invoke :cas [21 [6 11]] INFO [2019-12-17 12:32:26,420] jepsen worker 5 - jepsen.util 5585 :ok :write [1 24] INFO [2019-12-17 12:32:26,420] jepsen worker 5 - jepsen.util 5585 :invoke :cas [1 [27 8]] INFO [2019-12-17 12:32:26,431] jepsen worker 87 - jepsen.util 5487 :ok :read [29 46] INFO [2019-12-17 12:32:26,437] jepsen worker 87 - jepsen.util 5487 :invoke :cas [29 [6 26]] INFO [2019-12-17 12:32:26,451] jepsen worker 47 - jepsen.util 6797 :ok :read [15 32] INFO [2019-12-17 12:32:26,451] jepsen worker 47 - jepsen.util 6797 :invoke :cas [15 [36 33]] INFO [2019-12-17 12:32:26,475] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":34360,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:32:26,475] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":34360,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:32:26,475] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":34364,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:32:26,476] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":34364,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:32:26,479] jepsen worker 63 - jepsen.util 4923 :fail :cas [21 [6 11]] :ValueNotSwapFrom INFO [2019-12-17 12:32:26,479] jepsen worker 63 - jepsen.util 4923 :invoke :write [21 45] INFO [2019-12-17 12:32:26,482] jepsen worker 79 - jepsen.util 5839 :fail :cas [26 [38 21]] :ValueNotSwapFrom INFO [2019-12-17 12:32:26,482] jepsen worker 79 - jepsen.util 5839 :invoke :write [26 13] INFO [2019-12-17 12:32:26,483] jepsen worker 54 - jepsen.util 6444 :fail :cas [18 [23 24]] :ValueNotSwapFrom INFO [2019-12-17 12:32:26,483] jepsen worker 54 - jepsen.util 6444 :invoke :read [18 nil] INFO [2019-12-17 12:32:26,500] jepsen worker 52 - jepsen.util 6352 :fail :cas [17 [6 19]] :ValueNotSwapFrom INFO [2019-12-17 12:32:26,501] jepsen worker 47 - jepsen.util 6797 :fail :cas [15 [36 33]] :ValueNotSwapFrom INFO [2019-12-17 12:32:26,501] jepsen worker 52 - jepsen.util 6352 :invoke :cas [17 [36 30]] INFO [2019-12-17 12:32:26,501] jepsen worker 47 - jepsen.util 6797 :invoke :read [15 nil] INFO [2019-12-17 12:32:26,511] jepsen worker 68 - jepsen.util 7628 :info :cas [22 [38 9]] :RequestCanceledException INFO [2019-12-17 12:32:26,512] jepsen worker 68 - jepsen.util 7718 :invoke :cas [22 [5 39]] INFO [2019-12-17 12:32:26,535] jepsen worker 87 - jepsen.util 5487 :fail :cas [29 [6 26]] :ValueNotSwapFrom INFO [2019-12-17 12:32:26,537] jepsen worker 27 - jepsen.util 5877 :ok :write [9 28] INFO [2019-12-17 12:32:26,537] jepsen worker 27 - jepsen.util 5877 :invoke :write [9 43] INFO [2019-12-17 12:32:26,539] jepsen worker 87 - jepsen.util 5487 :invoke :cas [29 [30 44]] INFO [2019-12-17 12:32:26,541] jepsen worker 74 - jepsen.util 6194 :ok :write [24 45] INFO [2019-12-17 12:32:26,541] jepsen worker 74 - jepsen.util 6194 :invoke :cas [24 [17 11]] INFO [2019-12-17 12:32:26,544] jepsen worker 47 - jepsen.util 6797 :ok :read [15 32] INFO [2019-12-17 12:32:26,544] jepsen worker 47 - jepsen.util 6797 :invoke :cas [15 [2 32]] INFO [2019-12-17 12:32:26,561] jepsen worker 69 - jepsen.util 6189 :fail :cas [23 [34 32]] :ValueNotSwapFrom INFO [2019-12-17 12:32:26,562] jepsen worker 69 - jepsen.util 6189 :invoke :read [23 nil] INFO [2019-12-17 12:32:26,594] jepsen worker 36 - jepsen.util 6426 :ok :read [12 21] INFO [2019-12-17 12:32:26,594] jepsen worker 36 - jepsen.util 6426 :invoke :write [12 15] INFO [2019-12-17 12:32:26,626] jepsen worker 66 - jepsen.util 7536 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:32:26,626] jepsen worker 10 - jepsen.util 5770 :info :write [3 4] :RequestCanceledException INFO [2019-12-17 12:32:26,627] jepsen worker 66 - jepsen.util 7536 :invoke :write [22 31] INFO [2019-12-17 12:32:26,628] jepsen worker 10 - jepsen.util 5860 :invoke :read [3 nil] INFO [2019-12-17 12:32:26,631] jepsen worker 19 - jepsen.util 5959 :ok :read [6 18] INFO [2019-12-17 12:32:26,631] jepsen worker 19 - jepsen.util 5959 :invoke :write [6 46] INFO [2019-12-17 12:32:26,633] jepsen worker 52 - jepsen.util 6352 :fail :cas [17 [36 30]] :ValueNotSwapFrom INFO [2019-12-17 12:32:26,633] jepsen worker 52 - jepsen.util 6352 :invoke :cas [17 [25 10]] INFO [2019-12-17 12:32:26,643] jepsen worker 9 - jepsen.util 6399 :fail :cas [3 [33 39]] :ValueNotSwapFrom INFO [2019-12-17 12:32:26,643] jepsen worker 9 - jepsen.util 6399 :invoke :read [3 nil] INFO [2019-12-17 12:32:26,678] jepsen worker 47 - jepsen.util 6797 :fail :cas [15 [2 32]] :ValueNotSwapFrom INFO [2019-12-17 12:32:26,678] jepsen worker 47 - jepsen.util 6797 :invoke :write [15 45] INFO [2019-12-17 12:32:26,689] jepsen worker 80 - jepsen.util 6020 :fail :cas [26 [37 48]] :ValueNotSwapFrom INFO [2019-12-17 12:32:26,689] jepsen worker 80 - jepsen.util 6020 :invoke :read [26 nil] INFO [2019-12-17 12:32:26,691] jepsen worker 64 - jepsen.util 4654 :ok :read [21 20] INFO [2019-12-17 12:32:26,691] jepsen worker 64 - jepsen.util 4654 :invoke :cas [21 [41 17]] INFO [2019-12-17 12:32:26,694] jepsen worker 74 - jepsen.util 6194 :fail :cas [24 [17 11]] :ValueNotSwapFrom INFO [2019-12-17 12:32:26,694] jepsen worker 74 - jepsen.util 6194 :invoke :write [24 27] INFO [2019-12-17 12:32:26,728] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":34384,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:32:26,728] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":34384,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:32:26,728] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":34388,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:32:26,729] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":34388,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:32:26,736] jepsen worker 87 - jepsen.util 5487 :fail :cas [29 [30 44]] :ValueNotSwapFrom INFO [2019-12-17 12:32:26,736] jepsen worker 87 - jepsen.util 5487 :invoke :read [29 nil] INFO [2019-12-17 12:32:26,737] jepsen worker 10 - jepsen.util 5860 :ok :read [3 29] INFO [2019-12-17 12:32:26,737] jepsen worker 10 - jepsen.util 5860 :invoke :write [3 4] INFO [2019-12-17 12:32:26,759] jepsen worker 54 - jepsen.util 6444 :ok :read [18 47] INFO [2019-12-17 12:32:26,759] jepsen worker 54 - jepsen.util 6444 :invoke :cas [18 [37 27]] INFO [2019-12-17 12:32:26,773] jepsen worker 9 - jepsen.util 6399 :ok :read [3 29] INFO [2019-12-17 12:32:26,773] jepsen worker 9 - jepsen.util 6399 :invoke :cas [3 [23 46]] INFO [2019-12-17 12:32:26,775] jepsen worker 46 - jepsen.util 5986 :ok :read [15 32] INFO [2019-12-17 12:32:26,775] jepsen worker 46 - jepsen.util 5986 :invoke :write [15 27] INFO [2019-12-17 12:32:26,777] jepsen worker 71 - jepsen.util 6641 :ok :write [23 33] INFO [2019-12-17 12:32:26,777] jepsen worker 71 - jepsen.util 6641 :invoke :write [23 13] INFO [2019-12-17 12:32:26,791] jepsen worker 18 - jepsen.util 5688 :info :write [6 34] :RequestCanceledException INFO [2019-12-17 12:32:26,791] jepsen worker 85 - jepsen.util 8905 :info :write [28 5] :RequestCanceledException INFO [2019-12-17 12:32:26,792] jepsen worker 63 - jepsen.util 4923 :ok :write [21 45] INFO [2019-12-17 12:32:26,792] jepsen worker 18 - jepsen.util 5778 :invoke :read [6 nil] INFO [2019-12-17 12:32:26,792] jepsen worker 85 - jepsen.util 8995 :invoke :read [28 nil] INFO [2019-12-17 12:32:26,792] jepsen worker 63 - jepsen.util 4923 :invoke :cas [21 [42 36]] INFO [2019-12-17 12:32:26,811] jepsen worker 60 - jepsen.util 8880 :info :cas [20 [36 25]] :RequestCanceledException INFO [2019-12-17 12:32:26,811] jepsen worker 75 - jepsen.util 7725 :info :cas [25 [15 4]] :RequestCanceledException INFO [2019-12-17 12:32:26,812] jepsen worker 75 - jepsen.util 7815 :invoke :read [25 nil] INFO [2019-12-17 12:32:26,812] jepsen worker 60 - jepsen.util 8970 :invoke :cas [20 [37 42]] INFO [2019-12-17 12:32:26,820] jepsen worker 63 - jepsen.util 4923 :fail :cas [21 [42 36]] :ValueNotSwapFrom INFO [2019-12-17 12:32:26,821] jepsen worker 63 - jepsen.util 4923 :invoke :read [21 nil] INFO [2019-12-17 12:32:26,827] jepsen worker 57 - jepsen.util 7617 :info :write [19 13] :RequestCanceledException INFO [2019-12-17 12:32:26,827] jepsen worker 44 - jepsen.util 8774 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:32:26,827] jepsen worker 43 - jepsen.util 7513 :info :write [14 7] :RequestCanceledException INFO [2019-12-17 12:32:26,827] jepsen worker 44 - jepsen.util 8774 :invoke :cas [14 [48 39]] INFO [2019-12-17 12:32:26,828] jepsen worker 57 - jepsen.util 7707 :invoke :cas [19 [39 28]] INFO [2019-12-17 12:32:26,828] jepsen worker 43 - jepsen.util 7603 :invoke :cas [14 [22 17]] INFO [2019-12-17 12:32:26,839] jepsen worker 16 - jepsen.util 9016 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:32:26,839] jepsen worker 48 - jepsen.util 7248 :info :write [16 2] :RequestCanceledException INFO [2019-12-17 12:32:26,839] jepsen worker 8 - jepsen.util 7568 :info :write [2 19] :RequestCanceledException INFO [2019-12-17 12:32:26,839] jepsen worker 16 - jepsen.util 9016 :invoke :write [5 23] INFO [2019-12-17 12:32:26,840] jepsen worker 8 - jepsen.util 7658 :invoke :read [2 nil] INFO [2019-12-17 12:32:26,840] jepsen worker 48 - jepsen.util 7338 :invoke :write [16 42] INFO [2019-12-17 12:32:26,842] jepsen worker 14 - jepsen.util 6134 :ok :write [4 41] INFO [2019-12-17 12:32:26,842] jepsen worker 14 - jepsen.util 6134 :invoke :read [4 nil] INFO [2019-12-17 12:32:26,843] jepsen worker 56 - jepsen.util 6446 :ok :write [18 30] INFO [2019-12-17 12:32:26,843] jepsen worker 56 - jepsen.util 6446 :invoke :write [18 40] INFO [2019-12-17 12:32:26,857] jepsen worker 89 - jepsen.util 5759 :info :write [29 30] :RequestCanceledException INFO [2019-12-17 12:32:26,857] jepsen worker 82 - jepsen.util 8542 :info :write [27 31] :RequestCanceledException INFO [2019-12-17 12:32:26,857] jepsen worker 78 - jepsen.util 6288 :info :write [26 37] :RequestCanceledException INFO [2019-12-17 12:32:26,858] jepsen worker 82 - jepsen.util 8632 :invoke :cas [27 [48 23]] INFO [2019-12-17 12:32:26,858] jepsen worker 78 - jepsen.util 6378 :invoke :read [26 nil] INFO [2019-12-17 12:32:26,858] jepsen worker 89 - jepsen.util 5849 :invoke :cas [29 [20 27]] INFO [2019-12-17 12:32:26,865] jepsen worker 63 - jepsen.util 4923 :ok :read [21 45] INFO [2019-12-17 12:32:26,865] jepsen worker 63 - jepsen.util 4923 :invoke :write [21 24] INFO [2019-12-17 12:32:26,886] jepsen worker 3 - jepsen.util 5943 :ok :write [1 25] INFO [2019-12-17 12:32:26,886] jepsen worker 3 - jepsen.util 5943 :invoke :cas [1 [7 23]] INFO [2019-12-17 12:32:26,893] jepsen worker 34 - jepsen.util 7594 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:32:26,893] jepsen worker 34 - jepsen.util 7594 :invoke :read [11 nil] INFO [2019-12-17 12:32:26,893] jepsen worker 54 - jepsen.util 6444 :fail :cas [18 [37 27]] :ValueNotSwapFrom INFO [2019-12-17 12:32:26,893] jepsen worker 54 - jepsen.util 6444 :invoke :read [18 nil] INFO [2019-12-17 12:32:26,911] jepsen worker 76 - jepsen.util 8176 :info :cas [25 [21 42]] :RequestCanceledException INFO [2019-12-17 12:32:26,912] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":34420,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:32:26,912] jepsen worker 76 - jepsen.util 8266 :invoke :cas [25 [27 36]] INFO [2019-12-17 12:32:26,912] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":34420,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:32:26,912] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":34422,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:32:26,912] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":34422,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:32:26,926] jepsen worker 33 - jepsen.util 8043 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:32:26,926] jepsen worker 6 - jepsen.util 8556 :info :cas [2 [20 37]] :RequestCanceledException INFO [2019-12-17 12:32:26,926] jepsen worker 72 - jepsen.util 5832 :info :cas [24 [11 20]] :RequestCanceledException INFO [2019-12-17 12:32:26,926] jepsen worker 33 - jepsen.util 8043 :invoke :write [11 31] INFO [2019-12-17 12:32:26,927] jepsen worker 6 - jepsen.util 8646 :invoke :cas [2 [23 34]] INFO [2019-12-17 12:32:26,927] jepsen worker 72 - jepsen.util 5922 :invoke :write [24 46] INFO [2019-12-17 12:32:26,928] jepsen worker 63 - jepsen.util 4923 :ok :write [21 24] INFO [2019-12-17 12:32:26,928] jepsen worker 63 - jepsen.util 4923 :invoke :read [21 nil] INFO [2019-12-17 12:32:26,931] jepsen worker 38 - jepsen.util 5978 :fail :cas [12 [5 25]] :ValueNotSwapFrom INFO [2019-12-17 12:32:26,931] jepsen worker 38 - jepsen.util 5978 :invoke :read [12 nil] INFO [2019-12-17 12:32:26,940] jepsen worker 58 - jepsen.util 8608 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:32:26,941] jepsen worker 58 - jepsen.util 8608 :invoke :read [19 nil] INFO [2019-12-17 12:32:26,949] jepsen worker 63 - jepsen.util 4923 :ok :read [21 24] INFO [2019-12-17 12:32:26,949] jepsen worker 63 - jepsen.util 4923 :invoke :cas [21 [40 32]] INFO [2019-12-17 12:32:26,955] jepsen worker 41 - jepsen.util 8861 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:32:26,955] jepsen worker 41 - jepsen.util 8861 :invoke :write [13 42] INFO [2019-12-17 12:32:26,971] jepsen worker 30 - jepsen.util 6060 :ok :write [10 49] INFO [2019-12-17 12:32:26,971] jepsen worker 20 - jepsen.util 5960 :ok :write [6 0] INFO [2019-12-17 12:32:26,971] jepsen worker 30 - jepsen.util 6060 :invoke :read [10 nil] INFO [2019-12-17 12:32:26,971] jepsen worker 20 - jepsen.util 5960 :invoke :write [6 12] INFO [2019-12-17 12:32:26,975] jepsen worker 63 - jepsen.util 4923 :fail :cas [21 [40 32]] :ValueNotSwapFrom INFO [2019-12-17 12:32:26,975] jepsen worker 63 - jepsen.util 4923 :invoke :read [21 nil] INFO [2019-12-17 12:32:26,976] jepsen worker 11 - jepsen.util 6221 :ok :write [3 36] INFO [2019-12-17 12:32:26,976] jepsen worker 11 - jepsen.util 6221 :invoke :read [3 nil] INFO [2019-12-17 12:32:26,990] jepsen worker 47 - jepsen.util 6797 :ok :write [15 45] INFO [2019-12-17 12:32:26,990] jepsen worker 47 - jepsen.util 6797 :invoke :cas [15 [15 30]] INFO [2019-12-17 12:32:27,006] jepsen worker 53 - jepsen.util 5633 :fail :cas [17 [30 26]] :ValueNotSwapFrom INFO [2019-12-17 12:32:27,006] jepsen worker 53 - jepsen.util 5633 :invoke :cas [17 [34 22]] INFO [2019-12-17 12:32:27,010] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":34442,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:32:27,010] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":34442,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:32:27,010] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":34445,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:32:27,010] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":34445,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:32:27,010] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":34446,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:32:27,010] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":34446,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:32:27,027] jepsen worker 0 - jepsen.util 8280 :info :cas [0 [34 7]] :RequestCanceledException INFO [2019-12-17 12:32:27,028] jepsen worker 0 - jepsen.util 8370 :invoke :write [0 21] INFO [2019-12-17 12:32:27,031] jepsen worker 45 - jepsen.util 5895 :fail :cas [15 [8 26]] :ValueNotSwapFrom INFO [2019-12-17 12:32:27,032] jepsen worker 45 - jepsen.util 5895 :invoke :write [15 49] INFO [2019-12-17 12:32:27,058] jepsen worker 13 - jepsen.util 5773 :info :cas [4 [28 17]] :RequestCanceledException INFO [2019-12-17 12:32:27,059] jepsen worker 13 - jepsen.util 5863 :invoke :cas [4 [5 3]] INFO [2019-12-17 12:32:27,074] jepsen worker 63 - jepsen.util 4923 :ok :read [21 24] INFO [2019-12-17 12:32:27,074] jepsen worker 63 - jepsen.util 4923 :invoke :read [21 nil] INFO [2019-12-17 12:32:27,083] jepsen worker 5 - jepsen.util 5585 :fail :cas [1 [27 8]] :ValueNotSwapFrom INFO [2019-12-17 12:32:27,083] jepsen worker 5 - jepsen.util 5585 :invoke :read [1 nil] INFO [2019-12-17 12:32:27,111] jepsen worker 65 - jepsen.util 5015 :info :write [21 15] :RequestCanceledException INFO [2019-12-17 12:32:27,112] jepsen worker 65 - jepsen.util 5105 :invoke :write [21 39] INFO [2019-12-17 12:32:27,126] jepsen worker 64 - jepsen.util 4654 :fail :cas [21 [41 17]] :ValueNotSwapFrom INFO [2019-12-17 12:32:27,126] jepsen worker 64 - jepsen.util 4654 :invoke :cas [21 [0 32]] INFO [2019-12-17 12:32:27,127] jepsen worker 62 - jepsen.util 7622 :info :write [20 29] :RequestCanceledException INFO [2019-12-17 12:32:27,128] jepsen worker 62 - jepsen.util 7712 :invoke :cas [20 [20 23]] INFO [2019-12-17 12:32:27,140] jepsen worker 47 - jepsen.util 6797 :fail :cas [15 [15 30]] :ValueNotSwapFrom INFO [2019-12-17 12:32:27,140] jepsen worker 47 - jepsen.util 6797 :invoke :cas [15 [10 37]] INFO [2019-12-17 12:32:27,141] jepsen worker 30 - jepsen.util 6060 :ok :read [10 49] INFO [2019-12-17 12:32:27,141] jepsen worker 81 - jepsen.util 7731 :info :cas [27 [45 44]] :RequestCanceledException INFO [2019-12-17 12:32:27,141] jepsen worker 30 - jepsen.util 6060 :invoke :cas [10 [34 45]] INFO [2019-12-17 12:32:27,142] jepsen worker 81 - jepsen.util 7821 :invoke :read [27 nil] INFO [2019-12-17 12:32:27,148] jepsen worker 63 - jepsen.util 4923 :ok :read [21 24] INFO [2019-12-17 12:32:27,148] jepsen worker 63 - jepsen.util 4923 :invoke :cas [21 [38 42]] INFO [2019-12-17 12:32:27,175] jepsen worker 47 - jepsen.util 6797 :fail :cas [15 [10 37]] :ValueNotSwapFrom INFO [2019-12-17 12:32:27,175] jepsen worker 47 - jepsen.util 6797 :invoke :read [15 nil] INFO [2019-12-17 12:32:27,194] jepsen worker 84 - jepsen.util 7644 :info :cas [28 [23 5]] :RequestCanceledException INFO [2019-12-17 12:32:27,195] jepsen worker 84 - jepsen.util 7734 :invoke :cas [28 [15 7]] INFO [2019-12-17 12:32:27,196] jepsen worker 30 - jepsen.util 6060 :fail :cas [10 [34 45]] :ValueNotSwapFrom INFO [2019-12-17 12:32:27,196] jepsen worker 30 - jepsen.util 6060 :invoke :cas [10 [43 42]] INFO [2019-12-17 12:32:27,212] jepsen worker 63 - jepsen.util 4923 :fail :cas [21 [38 42]] :ValueNotSwapFrom INFO [2019-12-17 12:32:27,212] jepsen worker 63 - jepsen.util 4923 :invoke :read [21 nil] INFO [2019-12-17 12:32:27,222] jepsen worker 14 - jepsen.util 6134 :ok :read [4 41] INFO [2019-12-17 12:32:27,222] jepsen worker 14 - jepsen.util 6134 :invoke :read [4 nil] INFO [2019-12-17 12:32:27,226] jepsen worker 39 - jepsen.util 7599 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:32:27,226] jepsen worker 83 - jepsen.util 8363 :info :write [27 9] :RequestCanceledException INFO [2019-12-17 12:32:27,226] jepsen worker 39 - jepsen.util 7599 :invoke :read [13 nil] INFO [2019-12-17 12:32:27,226] jepsen worker 86 - jepsen.util 8366 :info :write [28 14] :RequestCanceledException INFO [2019-12-17 12:32:27,227] jepsen worker 83 - jepsen.util 8453 :invoke :write [27 24] INFO [2019-12-17 12:32:27,227] jepsen worker 86 - jepsen.util 8456 :invoke :cas [28 [6 30]] INFO [2019-12-17 12:32:27,229] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":34473,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:32:27,229] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":34473,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:32:27,229] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":34475,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:32:27,229] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":34475,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:32:27,232] jepsen worker 30 - jepsen.util 6060 :fail :cas [10 [43 42]] :ValueNotSwapFrom INFO [2019-12-17 12:32:27,232] jepsen worker 30 - jepsen.util 6060 :invoke :read [10 nil] INFO [2019-12-17 12:32:27,234] jepsen worker 21 - jepsen.util 7491 :info :cas [7 [31 12]] :RequestCanceledException INFO [2019-12-17 12:32:27,235] jepsen worker 21 - jepsen.util 7581 :invoke :write [7 19] INFO [2019-12-17 12:32:27,255] jepsen worker 54 - jepsen.util 6444 :ok :read [18 30] INFO [2019-12-17 12:32:27,255] jepsen worker 54 - jepsen.util 6444 :invoke :read [18 nil] INFO [2019-12-17 12:32:27,257] jepsen worker 61 - jepsen.util 9151 :info :cas [20 [38 20]] :RequestCanceledException INFO [2019-12-17 12:32:27,257] jepsen worker 1 - jepsen.util 7741 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:32:27,257] jepsen worker 29 - jepsen.util 4709 :fail :read [9 nil] :RequestTimeoutException INFO [2019-12-17 12:32:27,257] jepsen worker 1 - jepsen.util 7741 :invoke :cas [0 [31 6]] INFO [2019-12-17 12:32:27,257] jepsen worker 29 - jepsen.util 4709 :invoke :write [9 48] INFO [2019-12-17 12:32:27,257] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:32:27,257] jepsen nemesis - couchbase.util Rebalance check count: 261 INFO [2019-12-17 12:32:27,257] jepsen nemesis - couchbase.util Rebalance stuck count: 242 INFO [2019-12-17 12:32:27,258] jepsen worker 61 - jepsen.util 9241 :invoke :read [20 nil] INFO [2019-12-17 12:32:27,260] jepsen worker 80 - jepsen.util 6020 :ok :read [26 41] INFO [2019-12-17 12:32:27,260] jepsen worker 80 - jepsen.util 6020 :invoke :write [26 7] INFO [2019-12-17 12:32:27,276] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":34461,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:32:27,276] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":34461,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:32:27,276] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":34462,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:32:27,276] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":34462,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:32:27,293] jepsen worker 40 - jepsen.util 8500 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:32:27,293] jepsen worker 40 - jepsen.util 8500 :invoke :read [13 nil] INFO [2019-12-17 12:32:27,297] jepsen worker 87 - jepsen.util 5487 :ok :read [29 46] INFO [2019-12-17 12:32:27,297] jepsen worker 87 - jepsen.util 5487 :invoke :cas [29 [33 45]] INFO [2019-12-17 12:32:27,298] jepsen worker 30 - jepsen.util 6060 :ok :read [10 49] INFO [2019-12-17 12:32:27,298] jepsen worker 30 - jepsen.util 6060 :invoke :cas [10 [33 21]] INFO [2019-12-17 12:32:27,301] jepsen worker 63 - jepsen.util 4923 :ok :read [21 24] INFO [2019-12-17 12:32:27,301] jepsen worker 63 - jepsen.util 4923 :invoke :write [21 0] INFO [2019-12-17 12:32:27,310] jepsen worker 47 - jepsen.util 6797 :ok :read [15 45] INFO [2019-12-17 12:32:27,310] jepsen worker 22 - jepsen.util 7942 :info :write [7 39] :RequestCanceledException INFO [2019-12-17 12:32:27,311] jepsen worker 47 - jepsen.util 6797 :invoke :write [15 1] INFO [2019-12-17 12:32:27,311] jepsen worker 22 - jepsen.util 8032 :invoke :read [7 nil] INFO [2019-12-17 12:32:27,327] jepsen worker 77 - jepsen.util 8267 :info :cas [25 [44 35]] :RequestCanceledException INFO [2019-12-17 12:32:27,328] jepsen worker 78 - jepsen.util 6378 :ok :read [26 41] INFO [2019-12-17 12:32:27,328] jepsen worker 78 - jepsen.util 6378 :invoke :read [26 nil] INFO [2019-12-17 12:32:27,331] jepsen worker 30 - jepsen.util 6060 :fail :cas [10 [33 21]] :ValueNotSwapFrom INFO [2019-12-17 12:32:27,332] jepsen worker 30 - jepsen.util 6060 :invoke :write [10 20] INFO [2019-12-17 12:32:27,333] jepsen worker 77 - jepsen.util 8357 :invoke :write [25 30] INFO [2019-12-17 12:32:27,341] jepsen worker 15 - jepsen.util 7035 :info :write [5 13] :RequestCanceledException INFO [2019-12-17 12:32:27,342] jepsen worker 15 - jepsen.util 7125 :invoke :read [5 nil] INFO [2019-12-17 12:32:27,355] jepsen worker 25 - jepsen.util 8575 :info :write [8 20] :RequestCanceledException INFO [2019-12-17 12:32:27,356] jepsen worker 25 - jepsen.util 8665 :invoke :write [8 5] INFO [2019-12-17 12:32:27,357] jepsen worker 13 - jepsen.util 5863 :fail :cas [4 [5 3]] :ValueNotSwapFrom INFO [2019-12-17 12:32:27,357] jepsen worker 13 - jepsen.util 5863 :invoke :write [4 7] INFO [2019-12-17 12:32:27,370] jepsen worker 29 - jepsen.util 4709 :ok :write [9 48] INFO [2019-12-17 12:32:27,370] jepsen worker 29 - jepsen.util 4709 :invoke :read [9 nil] INFO [2019-12-17 12:32:27,392] jepsen worker 29 - jepsen.util 4709 :ok :read [9 48] INFO [2019-12-17 12:32:27,393] jepsen worker 29 - jepsen.util 4709 :invoke :write [9 21] INFO [2019-12-17 12:32:27,393] jepsen worker 52 - jepsen.util 6352 :fail :cas [17 [25 10]] :ValueNotSwapFrom INFO [2019-12-17 12:32:27,393] jepsen worker 7 - jepsen.util 8287 :info :cas [2 [20 40]] :RequestCanceledException INFO [2019-12-17 12:32:27,393] jepsen worker 52 - jepsen.util 6352 :invoke :cas [17 [14 26]] INFO [2019-12-17 12:32:27,396] jepsen worker 7 - jepsen.util 8377 :invoke :cas [2 [4 6]] INFO [2019-12-17 12:32:27,410] jepsen worker 17 - jepsen.util 8027 :info :write [5 42] :RequestCanceledException INFO [2019-12-17 12:32:27,411] jepsen worker 54 - jepsen.util 6444 :ok :read [18 30] INFO [2019-12-17 12:32:27,411] jepsen worker 69 - jepsen.util 6189 :ok :read [23 33] INFO [2019-12-17 12:32:27,411] jepsen worker 54 - jepsen.util 6444 :invoke :write [18 37] INFO [2019-12-17 12:32:27,411] jepsen worker 69 - jepsen.util 6189 :invoke :cas [23 [9 31]] INFO [2019-12-17 12:32:27,411] jepsen worker 17 - jepsen.util 8117 :invoke :read [5 nil] INFO [2019-12-17 12:32:27,422] jepsen worker 3 - jepsen.util 5943 :fail :cas [1 [7 23]] :ValueNotSwapFrom INFO [2019-12-17 12:32:27,422] jepsen worker 3 - jepsen.util 5943 :invoke :write [1 47] INFO [2019-12-17 12:32:27,426] jepsen worker 26 - jepsen.util 8306 :info :cas [8 [32 24]] :RequestCanceledException INFO [2019-12-17 12:32:27,427] jepsen worker 26 - jepsen.util 8396 :invoke :write [8 39] INFO [2019-12-17 12:32:27,434] jepsen worker 53 - jepsen.util 5633 :fail :cas [17 [34 22]] :ValueNotSwapFrom INFO [2019-12-17 12:32:27,435] jepsen worker 53 - jepsen.util 5633 :invoke :read [17 nil] INFO [2019-12-17 12:32:27,445] jepsen worker 29 - jepsen.util 4709 :ok :write [9 21] INFO [2019-12-17 12:32:27,446] jepsen worker 29 - jepsen.util 4709 :invoke :cas [9 [47 49]] INFO [2019-12-17 12:32:27,451] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":34508,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:32:27,451] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":34508,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:32:27,451] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":34511,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:32:27,451] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":34511,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:32:27,464] jepsen worker 5 - jepsen.util 5585 :ok :read [1 25] INFO [2019-12-17 12:32:27,464] jepsen worker 5 - jepsen.util 5585 :invoke :cas [1 [4 48]] INFO [2019-12-17 12:32:27,465] jepsen worker 29 - jepsen.util 4709 :fail :cas [9 [47 49]] :ValueNotSwapFrom INFO [2019-12-17 12:32:27,465] jepsen worker 29 - jepsen.util 4709 :invoke :write [9 2] INFO [2019-12-17 12:32:27,494] jepsen worker 78 - jepsen.util 6378 :ok :read [26 41] INFO [2019-12-17 12:32:27,494] jepsen worker 78 - jepsen.util 6378 :invoke :cas [26 [44 7]] INFO [2019-12-17 12:32:27,504] jepsen worker 38 - jepsen.util 5978 :ok :read [12 21] INFO [2019-12-17 12:32:27,505] jepsen worker 38 - jepsen.util 5978 :invoke :cas [12 [45 26]] INFO [2019-12-17 12:32:27,506] jepsen worker 89 - jepsen.util 5849 :fail :cas [29 [20 27]] :ValueNotSwapFrom INFO [2019-12-17 12:32:27,506] jepsen worker 89 - jepsen.util 5849 :invoke :cas [29 [34 47]] INFO [2019-12-17 12:32:27,519] jepsen worker 29 - jepsen.util 4709 :ok :write [9 2] INFO [2019-12-17 12:32:27,519] jepsen worker 29 - jepsen.util 4709 :invoke :write [9 27] INFO [2019-12-17 12:32:27,540] jepsen worker 67 - jepsen.util 8257 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:32:27,540] jepsen worker 67 - jepsen.util 8257 :invoke :cas [22 [40 30]] INFO [2019-12-17 12:32:27,558] jepsen worker 24 - jepsen.util 7314 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:32:27,559] jepsen worker 24 - jepsen.util 7314 :invoke :write [8 8] INFO [2019-12-17 12:32:27,562] jepsen worker 29 - jepsen.util 4709 :ok :write [9 27] INFO [2019-12-17 12:32:27,562] jepsen worker 29 - jepsen.util 4709 :invoke :cas [9 [7 38]] INFO [2019-12-17 12:32:27,566] jepsen worker 87 - jepsen.util 5487 :fail :cas [29 [33 45]] :ValueNotSwapFrom INFO [2019-12-17 12:32:27,566] jepsen worker 87 - jepsen.util 5487 :invoke :cas [29 [20 16]] INFO [2019-12-17 12:32:27,577] jepsen worker 14 - jepsen.util 6134 :ok :read [4 41] INFO [2019-12-17 12:32:27,577] jepsen worker 14 - jepsen.util 6134 :invoke :read [4 nil] INFO [2019-12-17 12:32:27,580] jepsen worker 29 - jepsen.util 4709 :fail :cas [9 [7 38]] :ValueNotSwapFrom INFO [2019-12-17 12:32:27,581] jepsen worker 29 - jepsen.util 4709 :invoke :cas [9 [40 24]] INFO [2019-12-17 12:32:27,589] jepsen worker 29 - jepsen.util 4709 :fail :cas [9 [40 24]] :ValueNotSwapFrom INFO [2019-12-17 12:32:27,589] jepsen worker 29 - jepsen.util 4709 :invoke :cas [9 [45 29]] INFO [2019-12-17 12:32:27,593] jepsen worker 42 - jepsen.util 8682 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:32:27,593] jepsen worker 42 - jepsen.util 8682 :invoke :write [14 2] INFO [2019-12-17 12:32:27,601] jepsen worker 29 - jepsen.util 4709 :fail :cas [9 [45 29]] :ValueNotSwapFrom INFO [2019-12-17 12:32:27,601] jepsen worker 29 - jepsen.util 4709 :invoke :cas [9 [35 24]] INFO [2019-12-17 12:32:27,609] jepsen worker 35 - jepsen.util 8135 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:32:27,609] jepsen worker 49 - jepsen.util 7249 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:32:27,609] jepsen worker 55 - jepsen.util 6895 :info :write [18 3] :RequestCanceledException INFO [2019-12-17 12:32:27,610] jepsen worker 35 - jepsen.util 8135 :invoke :cas [11 [39 13]] INFO [2019-12-17 12:32:27,610] jepsen worker 49 - jepsen.util 7249 :invoke :write [16 12] INFO [2019-12-17 12:32:27,610] jepsen worker 55 - jepsen.util 6985 :invoke :write [18 49] INFO [2019-12-17 12:32:27,614] jepsen worker 29 - jepsen.util 4709 :fail :cas [9 [35 24]] :ValueNotSwapFrom INFO [2019-12-17 12:32:27,614] jepsen worker 29 - jepsen.util 4709 :invoke :cas [9 [47 7]] INFO [2019-12-17 12:32:27,624] jepsen worker 29 - jepsen.util 4709 :fail :cas [9 [47 7]] :ValueNotSwapFrom INFO [2019-12-17 12:32:27,624] jepsen worker 29 - jepsen.util 4709 :invoke :cas [9 [27 29]] INFO [2019-12-17 12:32:27,626] jepsen worker 23 - jepsen.util 8033 :info :cas [7 [48 43]] :RequestCanceledException INFO [2019-12-17 12:32:27,626] jepsen worker 59 - jepsen.util 8159 :info :cas [19 [28 18]] :RequestCanceledException INFO [2019-12-17 12:32:27,627] jepsen worker 59 - jepsen.util 8249 :invoke :read [19 nil] INFO [2019-12-17 12:32:27,628] jepsen worker 23 - jepsen.util 8123 :invoke :write [7 30] INFO [2019-12-17 12:32:27,638] jepsen worker 52 - jepsen.util 6352 :fail :cas [17 [14 26]] :ValueNotSwapFrom INFO [2019-12-17 12:32:27,638] jepsen worker 52 - jepsen.util 6352 :invoke :write [17 19] INFO [2019-12-17 12:32:27,674] jepsen worker 14 - jepsen.util 6134 :ok :read [4 41] INFO [2019-12-17 12:32:27,674] jepsen worker 14 - jepsen.util 6134 :invoke :write [4 18] INFO [2019-12-17 12:32:27,685] jepsen worker 29 - jepsen.util 4709 :ok :cas [9 [27 29]] INFO [2019-12-17 12:32:27,685] jepsen worker 29 - jepsen.util 4709 :invoke :read [9 nil] INFO [2019-12-17 12:32:27,695] jepsen worker 69 - jepsen.util 6189 :fail :cas [23 [9 31]] :ValueNotSwapFrom INFO [2019-12-17 12:32:27,695] jepsen worker 69 - jepsen.util 6189 :invoke :read [23 nil] INFO [2019-12-17 12:32:27,709] jepsen worker 29 - jepsen.util 4709 :ok :read [9 29] INFO [2019-12-17 12:32:27,709] jepsen worker 29 - jepsen.util 4709 :invoke :cas [9 [22 42]] INFO [2019-12-17 12:32:27,712] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":34541,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:32:27,713] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":34541,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:32:27,721] jepsen worker 29 - jepsen.util 4709 :fail :cas [9 [22 42]] :ValueNotSwapFrom INFO [2019-12-17 12:32:27,722] jepsen worker 29 - jepsen.util 4709 :invoke :read [9 nil] INFO [2019-12-17 12:32:27,727] jepsen worker 32 - jepsen.util 5972 :info :cas [10 [31 38]] :RequestCanceledException INFO [2019-12-17 12:32:27,728] jepsen worker 32 - jepsen.util 6062 :invoke :write [10 41] INFO [2019-12-17 12:32:27,734] jepsen worker 29 - jepsen.util 4709 :ok :read [9 29] INFO [2019-12-17 12:32:27,735] jepsen worker 29 - jepsen.util 4709 :invoke :cas [9 [1 6]] INFO [2019-12-17 12:32:27,750] jepsen worker 29 - jepsen.util 4709 :fail :cas [9 [1 6]] :ValueNotSwapFrom INFO [2019-12-17 12:32:27,750] jepsen worker 29 - jepsen.util 4709 :invoke :read [9 nil] INFO [2019-12-17 12:32:27,757] jepsen worker 2 - jepsen.util 7832 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:32:27,758] jepsen worker 2 - jepsen.util 7832 :invoke :write [0 24] INFO [2019-12-17 12:32:27,762] jepsen worker 29 - jepsen.util 4709 :ok :read [9 29] INFO [2019-12-17 12:32:27,763] jepsen worker 29 - jepsen.util 4709 :invoke :write [9 32] INFO [2019-12-17 12:32:27,778] jepsen worker 87 - jepsen.util 5487 :fail :cas [29 [20 16]] :ValueNotSwapFrom INFO [2019-12-17 12:32:27,778] jepsen worker 87 - jepsen.util 5487 :invoke :write [29 47] INFO [2019-12-17 12:32:27,810] jepsen worker 88 - jepsen.util 6118 :info :write [29 16] :RequestCanceledException INFO [2019-12-17 12:32:27,813] jepsen worker 29 - jepsen.util 4709 :ok :write [9 32] INFO [2019-12-17 12:32:27,813] jepsen worker 29 - jepsen.util 4709 :invoke :write [9 8] INFO [2019-12-17 12:32:27,813] jepsen worker 88 - jepsen.util 6208 :invoke :write [29 49] INFO [2019-12-17 12:32:27,855] jepsen worker 5 - jepsen.util 5585 :fail :cas [1 [4 48]] :ValueNotSwapFrom INFO [2019-12-17 12:32:27,855] jepsen worker 5 - jepsen.util 5585 :invoke :write [1 41] INFO [2019-12-17 12:32:27,882] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":34561,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:32:27,882] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":34561,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:32:27,883] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":34564,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:32:27,883] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":34564,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:32:27,883] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":34565,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:32:27,883] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":34565,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:32:27,889] jepsen worker 29 - jepsen.util 4709 :ok :write [9 8] INFO [2019-12-17 12:32:27,889] jepsen worker 29 - jepsen.util 4709 :invoke :cas [9 [22 31]] INFO [2019-12-17 12:32:27,899] jepsen worker 78 - jepsen.util 6378 :fail :cas [26 [44 7]] :ValueNotSwapFrom INFO [2019-12-17 12:32:27,899] jepsen worker 78 - jepsen.util 6378 :invoke :read [26 nil] INFO [2019-12-17 12:32:27,908] jepsen worker 29 - jepsen.util 4709 :fail :cas [9 [22 31]] :ValueNotSwapFrom INFO [2019-12-17 12:32:27,908] jepsen worker 29 - jepsen.util 4709 :invoke :cas [9 [9 4]] INFO [2019-12-17 12:32:27,923] jepsen worker 29 - jepsen.util 4709 :fail :cas [9 [9 4]] :ValueNotSwapFrom INFO [2019-12-17 12:32:27,923] jepsen worker 29 - jepsen.util 4709 :invoke :write [9 20] INFO [2019-12-17 12:32:27,925] jepsen worker 69 - jepsen.util 6189 :ok :read [23 33] INFO [2019-12-17 12:32:27,925] jepsen worker 69 - jepsen.util 6189 :invoke :read [23 nil] INFO [2019-12-17 12:32:27,977] jepsen worker 29 - jepsen.util 4709 :ok :write [9 20] INFO [2019-12-17 12:32:27,978] jepsen worker 29 - jepsen.util 4709 :invoke :cas [9 [6 33]] INFO [2019-12-17 12:32:27,980] jepsen worker 38 - jepsen.util 5978 :fail :cas [12 [45 26]] :ValueNotSwapFrom INFO [2019-12-17 12:32:27,980] jepsen worker 38 - jepsen.util 5978 :invoke :cas [12 [40 13]] INFO [2019-12-17 12:32:27,990] jepsen worker 29 - jepsen.util 4709 :fail :cas [9 [6 33]] :ValueNotSwapFrom INFO [2019-12-17 12:32:27,990] jepsen worker 29 - jepsen.util 4709 :invoke :write [9 12] INFO [2019-12-17 12:32:27,992] jepsen worker 70 - jepsen.util 6370 :info :write [23 6] :RequestCanceledException INFO [2019-12-17 12:32:27,993] jepsen worker 50 - jepsen.util 8420 :info :cas [16 [26 19]] :RequestCanceledException INFO [2019-12-17 12:32:27,993] jepsen worker 50 - jepsen.util 8510 :invoke :read [16 nil] INFO [2019-12-17 12:32:27,993] jepsen worker 70 - jepsen.util 6460 :invoke :cas [23 [21 7]] INFO [2019-12-17 12:32:28,017] jepsen worker 30 - jepsen.util 6060 :ok :write [10 20] INFO [2019-12-17 12:32:28,017] jepsen worker 30 - jepsen.util 6060 :invoke :write [10 46] INFO [2019-12-17 12:32:28,017] jepsen worker 46 - jepsen.util 5986 :ok :write [15 27] INFO [2019-12-17 12:32:28,017] jepsen worker 46 - jepsen.util 5986 :invoke :cas [15 [36 49]] INFO [2019-12-17 12:32:28,043] jepsen worker 64 - jepsen.util 4654 :fail :cas [21 [0 32]] :ValueNotSwapFrom INFO [2019-12-17 12:32:28,044] jepsen worker 64 - jepsen.util 4654 :invoke :write [21 18] INFO [2019-12-17 12:32:28,060] jepsen worker 20 - jepsen.util 5960 :ok :write [6 12] INFO [2019-12-17 12:32:28,060] jepsen worker 74 - jepsen.util 6194 :ok :write [24 27] INFO [2019-12-17 12:32:28,060] jepsen worker 20 - jepsen.util 5960 :invoke :write [6 13] INFO [2019-12-17 12:32:28,060] jepsen worker 74 - jepsen.util 6194 :invoke :read [24 nil] INFO [2019-12-17 12:32:28,071] jepsen worker 70 - jepsen.util 6460 :fail :cas [23 [21 7]] :ValueNotSwapFrom INFO [2019-12-17 12:32:28,072] jepsen worker 70 - jepsen.util 6460 :invoke :cas [23 [18 3]] INFO [2019-12-17 12:32:28,075] jepsen worker 46 - jepsen.util 5986 :fail :cas [15 [36 49]] :ValueNotSwapFrom INFO [2019-12-17 12:32:28,075] jepsen worker 46 - jepsen.util 5986 :invoke :cas [15 [47 14]] INFO [2019-12-17 12:32:28,098] jepsen worker 89 - jepsen.util 5849 :fail :cas [29 [34 47]] :ValueNotSwapFrom INFO [2019-12-17 12:32:28,098] jepsen worker 89 - jepsen.util 5849 :invoke :cas [29 [3 13]] INFO [2019-12-17 12:32:28,100] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":34576,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:32:28,100] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":34576,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:32:28,100] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":34579,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:32:28,100] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":34579,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:32:28,101] jepsen worker 4 - jepsen.util 5404 :info :write [1 9] :RequestCanceledException INFO [2019-12-17 12:32:28,102] jepsen worker 4 - jepsen.util 5494 :invoke :write [1 43] INFO [2019-12-17 12:32:28,114] jepsen worker 29 - jepsen.util 4709 :ok :write [9 12] INFO [2019-12-17 12:32:28,114] jepsen worker 29 - jepsen.util 4709 :invoke :cas [9 [38 14]] INFO [2019-12-17 12:32:28,124] jepsen worker 51 - jepsen.util 5901 :info :write [17 21] :RequestCanceledException INFO [2019-12-17 12:32:28,125] jepsen worker 51 - jepsen.util 5991 :invoke :cas [17 [9 28]] INFO [2019-12-17 12:32:28,132] jepsen worker 18 - jepsen.util 5778 :ok :read [6 12] INFO [2019-12-17 12:32:28,132] jepsen worker 18 - jepsen.util 5778 :invoke :write [6 37] INFO [2019-12-17 12:32:28,161] jepsen worker 69 - jepsen.util 6189 :ok :read [23 33] INFO [2019-12-17 12:32:28,161] jepsen worker 69 - jepsen.util 6189 :invoke :cas [23 [8 27]] INFO [2019-12-17 12:32:28,168] jepsen worker 74 - jepsen.util 6194 :ok :read [24 27] INFO [2019-12-17 12:32:28,168] jepsen worker 74 - jepsen.util 6194 :invoke :read [24 nil] INFO [2019-12-17 12:32:28,172] jepsen worker 29 - jepsen.util 4709 :fail :cas [9 [38 14]] :ValueNotSwapFrom INFO [2019-12-17 12:32:28,173] jepsen worker 64 - jepsen.util 4654 :ok :write [21 18] INFO [2019-12-17 12:32:28,173] jepsen worker 29 - jepsen.util 4709 :invoke :read [9 nil] INFO [2019-12-17 12:32:28,173] jepsen worker 64 - jepsen.util 4654 :invoke :write [21 19] INFO [2019-12-17 12:32:28,190] jepsen worker 29 - jepsen.util 4709 :ok :read [9 12] INFO [2019-12-17 12:32:28,190] jepsen worker 29 - jepsen.util 4709 :invoke :cas [9 [9 47]] INFO [2019-12-17 12:32:28,217] jepsen worker 29 - jepsen.util 4709 :fail :cas [9 [9 47]] :ValueNotSwapFrom INFO [2019-12-17 12:32:28,218] jepsen worker 29 - jepsen.util 4709 :invoke :read [9 nil] INFO [2019-12-17 12:32:28,225] jepsen worker 53 - jepsen.util 5633 :ok :read [17 16] INFO [2019-12-17 12:32:28,225] jepsen worker 53 - jepsen.util 5633 :invoke :read [17 nil] INFO [2019-12-17 12:32:28,258] jepsen worker 37 - jepsen.util 5617 :info :write [12 6] :RequestCanceledException INFO [2019-12-17 12:32:28,258] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":34599,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:32:28,258] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":34599,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:32:28,259] jepsen worker 32 - jepsen.util 6062 :ok :write [10 41] INFO [2019-12-17 12:32:28,259] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":34601,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:32:28,259] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":34601,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:32:28,259] jepsen worker 32 - jepsen.util 6062 :invoke :read [10 nil] INFO [2019-12-17 12:32:28,259] jepsen worker 37 - jepsen.util 5707 :invoke :write [12 22] INFO [2019-12-17 12:32:28,261] jepsen worker 64 - jepsen.util 4654 :ok :write [21 19] INFO [2019-12-17 12:32:28,262] jepsen worker 64 - jepsen.util 4654 :invoke :cas [21 [17 30]] INFO [2019-12-17 12:32:28,264] jepsen worker 29 - jepsen.util 4709 :ok :read [9 12] INFO [2019-12-17 12:32:28,265] jepsen worker 29 - jepsen.util 4709 :invoke :write [9 28] INFO [2019-12-17 12:32:28,272] jepsen worker 20 - jepsen.util 5960 :ok :write [6 13] INFO [2019-12-17 12:32:28,272] jepsen worker 20 - jepsen.util 5960 :invoke :read [6 nil] INFO [2019-12-17 12:32:28,273] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:32:28,273] jepsen nemesis - couchbase.util Rebalance check count: 262 INFO [2019-12-17 12:32:28,273] jepsen nemesis - couchbase.util Rebalance stuck count: 243 INFO [2019-12-17 12:32:28,278] jepsen worker 64 - jepsen.util 4654 :fail :cas [21 [17 30]] :ValueNotSwapFrom INFO [2019-12-17 12:32:28,278] jepsen worker 64 - jepsen.util 4654 :invoke :write [21 5] INFO [2019-12-17 12:32:28,298] jepsen worker 20 - jepsen.util 5960 :ok :read [6 13] INFO [2019-12-17 12:32:28,299] jepsen worker 20 - jepsen.util 5960 :invoke :read [6 nil] INFO [2019-12-17 12:32:28,313] jepsen worker 53 - jepsen.util 5633 :ok :read [17 16] INFO [2019-12-17 12:32:28,313] jepsen worker 53 - jepsen.util 5633 :invoke :write [17 33] INFO [2019-12-17 12:32:28,316] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":34606,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:32:28,317] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":34606,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:32:28,317] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":34610,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:32:28,317] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":34610,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:32:28,338] jepsen worker 20 - jepsen.util 5960 :ok :read [6 13] INFO [2019-12-17 12:32:28,339] jepsen worker 20 - jepsen.util 5960 :invoke :write [6 49] INFO [2019-12-17 12:32:28,340] jepsen worker 89 - jepsen.util 5849 :fail :cas [29 [3 13]] :ValueNotSwapFrom INFO [2019-12-17 12:32:28,340] jepsen worker 89 - jepsen.util 5849 :invoke :read [29 nil] INFO [2019-12-17 12:32:28,360] jepsen worker 29 - jepsen.util 4709 :ok :write [9 28] INFO [2019-12-17 12:32:28,361] jepsen worker 29 - jepsen.util 4709 :invoke :cas [9 [6 35]] INFO [2019-12-17 12:32:28,382] jepsen worker 78 - jepsen.util 6378 :ok :read [26 41] INFO [2019-12-17 12:32:28,382] jepsen worker 78 - jepsen.util 6378 :invoke :cas [26 [46 48]] INFO [2019-12-17 12:32:28,382] jepsen worker 29 - jepsen.util 4709 :fail :cas [9 [6 35]] :ValueNotSwapFrom INFO [2019-12-17 12:32:28,382] jepsen worker 29 - jepsen.util 4709 :invoke :write [9 37] INFO [2019-12-17 12:32:28,409] jepsen worker 12 - jepsen.util 6222 :info :write [4 6] :RequestCanceledException INFO [2019-12-17 12:32:28,410] jepsen worker 12 - jepsen.util 6312 :invoke :cas [4 [2 39]] INFO [2019-12-17 12:32:28,426] jepsen worker 73 - jepsen.util 5653 :info :write [24 47] :RequestCanceledException INFO [2019-12-17 12:32:28,427] jepsen worker 31 - jepsen.util 5341 :info :write [10 13] :RequestCanceledException INFO [2019-12-17 12:32:28,427] jepsen worker 28 - jepsen.util 4618 :info :write [9 8] :RequestCanceledException INFO [2019-12-17 12:32:28,427] jepsen worker 73 - jepsen.util 5743 :invoke :cas [24 [13 28]] INFO [2019-12-17 12:32:28,428] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":34627,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:32:28,428] jepsen worker 28 - jepsen.util 4708 :invoke :read [9 nil] INFO [2019-12-17 12:32:28,428] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":34627,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:32:28,428] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":34632,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:32:28,429] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":34632,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:32:28,429] jepsen worker 64 - jepsen.util 4654 :ok :write [21 5] INFO [2019-12-17 12:32:28,429] jepsen worker 64 - jepsen.util 4654 :invoke :read [21 nil] INFO [2019-12-17 12:32:28,430] jepsen worker 31 - jepsen.util 5431 :invoke :read [10 nil] INFO [2019-12-17 12:32:28,442] jepsen worker 38 - jepsen.util 5978 :fail :cas [12 [40 13]] :ValueNotSwapFrom INFO [2019-12-17 12:32:28,442] jepsen worker 38 - jepsen.util 5978 :invoke :read [12 nil] INFO [2019-12-17 12:32:28,455] jepsen worker 78 - jepsen.util 6378 :fail :cas [26 [46 48]] :ValueNotSwapFrom INFO [2019-12-17 12:32:28,455] jepsen worker 78 - jepsen.util 6378 :invoke :read [26 nil] INFO [2019-12-17 12:32:28,470] jepsen worker 73 - jepsen.util 5743 :fail :cas [24 [13 28]] :ValueNotSwapFrom INFO [2019-12-17 12:32:28,470] jepsen worker 73 - jepsen.util 5743 :invoke :cas [24 [41 36]] INFO [2019-12-17 12:32:28,500] jepsen worker 30 - jepsen.util 6060 :ok :write [10 46] INFO [2019-12-17 12:32:28,500] jepsen worker 30 - jepsen.util 6060 :invoke :write [10 29] INFO [2019-12-17 12:32:28,505] jepsen worker 28 - jepsen.util 4708 :ok :read [9 28] INFO [2019-12-17 12:32:28,505] jepsen worker 28 - jepsen.util 4708 :invoke :write [9 38] INFO [2019-12-17 12:32:28,508] jepsen worker 73 - jepsen.util 5743 :fail :cas [24 [41 36]] :ValueNotSwapFrom INFO [2019-12-17 12:32:28,508] jepsen worker 73 - jepsen.util 5743 :invoke :write [24 49] INFO [2019-12-17 12:32:28,509] jepsen worker 79 - jepsen.util 5839 :info :write [26 13] :RequestCanceledException INFO [2019-12-17 12:32:28,511] jepsen worker 79 - jepsen.util 5929 :invoke :write [26 17] INFO [2019-12-17 12:32:28,528] jepsen worker 68 - jepsen.util 7718 :info :cas [22 [5 39]] :RequestCanceledException INFO [2019-12-17 12:32:28,529] jepsen worker 68 - jepsen.util 7808 :invoke :write [22 46] INFO [2019-12-17 12:32:28,541] jepsen worker 32 - jepsen.util 6062 :ok :read [10 46] INFO [2019-12-17 12:32:28,541] jepsen worker 32 - jepsen.util 6062 :invoke :read [10 nil] INFO [2019-12-17 12:32:28,582] jepsen worker 64 - jepsen.util 4654 :ok :read [21 5] INFO [2019-12-17 12:32:28,582] jepsen worker 64 - jepsen.util 4654 :invoke :write [21 43] INFO [2019-12-17 12:32:28,627] jepsen worker 27 - jepsen.util 5877 :info :write [9 43] :RequestCanceledException INFO [2019-12-17 12:32:28,628] jepsen worker 27 - jepsen.util 5967 :invoke :write [9 5] INFO [2019-12-17 12:32:28,635] jepsen worker 29 - jepsen.util 4709 :ok :write [9 37] INFO [2019-12-17 12:32:28,635] jepsen worker 29 - jepsen.util 4709 :invoke :cas [9 [43 2]] INFO [2019-12-17 12:32:28,640] jepsen worker 36 - jepsen.util 6426 :info :write [12 15] :RequestCanceledException INFO [2019-12-17 12:32:28,641] jepsen worker 36 - jepsen.util 6516 :invoke :write [12 29] INFO [2019-12-17 12:32:28,643] jepsen worker 78 - jepsen.util 6378 :ok :read [26 41] INFO [2019-12-17 12:32:28,643] jepsen worker 78 - jepsen.util 6378 :invoke :cas [26 [12 44]] INFO [2019-12-17 12:32:28,674] jepsen worker 29 - jepsen.util 4709 :fail :cas [9 [43 2]] :ValueNotSwapFrom INFO [2019-12-17 12:32:28,675] jepsen worker 29 - jepsen.util 4709 :invoke :read [9 nil] INFO [2019-12-17 12:32:28,695] jepsen worker 51 - jepsen.util 5991 :fail :cas [17 [9 28]] :ValueNotSwapFrom INFO [2019-12-17 12:32:28,695] jepsen worker 51 - jepsen.util 5991 :invoke :cas [17 [12 47]] INFO [2019-12-17 12:32:28,697] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":34649,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:32:28,697] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":34649,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:32:28,697] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":34653,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:32:28,698] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":34653,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:32:28,698] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":34654,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:32:28,698] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":34654,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:32:28,712] jepsen worker 70 - jepsen.util 6460 :fail :cas [23 [18 3]] :ValueNotSwapFrom INFO [2019-12-17 12:32:28,713] jepsen worker 70 - jepsen.util 6460 :invoke :read [23 nil] INFO [2019-12-17 12:32:28,724] jepsen worker 64 - jepsen.util 4654 :ok :write [21 43] INFO [2019-12-17 12:32:28,724] jepsen worker 64 - jepsen.util 4654 :invoke :cas [21 [32 44]] INFO [2019-12-17 12:32:28,727] jepsen worker 66 - jepsen.util 7536 :info :write [22 31] :RequestCanceledException INFO [2019-12-17 12:32:28,727] jepsen worker 19 - jepsen.util 5959 :info :write [6 46] :RequestCanceledException INFO [2019-12-17 12:32:28,728] jepsen worker 66 - jepsen.util 7626 :invoke :write [22 17] INFO [2019-12-17 12:32:28,728] jepsen worker 19 - jepsen.util 6049 :invoke :read [6 nil] INFO [2019-12-17 12:32:28,733] jepsen worker 29 - jepsen.util 4709 :ok :read [9 37] INFO [2019-12-17 12:32:28,733] jepsen worker 29 - jepsen.util 4709 :invoke :cas [9 [18 32]] INFO [2019-12-17 12:32:28,751] jepsen worker 64 - jepsen.util 4654 :fail :cas [21 [32 44]] :ValueNotSwapFrom INFO [2019-12-17 12:32:28,752] jepsen worker 64 - jepsen.util 4654 :invoke :cas [21 [42 29]] INFO [2019-12-17 12:32:28,761] jepsen worker 89 - jepsen.util 5849 :ok :read [29 46] INFO [2019-12-17 12:32:28,761] jepsen worker 89 - jepsen.util 5849 :invoke :cas [29 [19 41]] INFO [2019-12-17 12:32:28,765] jepsen worker 27 - jepsen.util 5967 :ok :write [9 5] INFO [2019-12-17 12:32:28,765] jepsen worker 27 - jepsen.util 5967 :invoke :read [9 nil] INFO [2019-12-17 12:32:28,765] jepsen worker 20 - jepsen.util 5960 :ok :write [6 49] INFO [2019-12-17 12:32:28,765] jepsen worker 20 - jepsen.util 5960 :invoke :read [6 nil] INFO [2019-12-17 12:32:28,765] jepsen worker 30 - jepsen.util 6060 :ok :write [10 29] INFO [2019-12-17 12:32:28,765] jepsen worker 73 - jepsen.util 5743 :ok :write [24 49] INFO [2019-12-17 12:32:28,766] jepsen worker 30 - jepsen.util 6060 :invoke :cas [10 [25 30]] INFO [2019-12-17 12:32:28,766] jepsen worker 73 - jepsen.util 5743 :invoke :write [24 47] INFO [2019-12-17 12:32:28,771] jepsen worker 29 - jepsen.util 4709 :fail :cas [9 [18 32]] :ValueNotSwapFrom INFO [2019-12-17 12:32:28,771] jepsen worker 29 - jepsen.util 4709 :invoke :read [9 nil] INFO [2019-12-17 12:32:28,771] jepsen worker 38 - jepsen.util 5978 :ok :read [12 21] INFO [2019-12-17 12:32:28,771] jepsen worker 38 - jepsen.util 5978 :invoke :cas [12 [3 49]] INFO [2019-12-17 12:32:28,790] jepsen worker 10 - jepsen.util 5860 :info :write [3 4] :RequestCanceledException INFO [2019-12-17 12:32:28,791] jepsen worker 10 - jepsen.util 5950 :invoke :read [3 nil] INFO [2019-12-17 12:32:28,808] jepsen worker 38 - jepsen.util 5978 :fail :cas [12 [3 49]] :ValueNotSwapFrom INFO [2019-12-17 12:32:28,808] jepsen worker 38 - jepsen.util 5978 :invoke :cas [12 [40 26]] INFO [2019-12-17 12:32:28,812] jepsen worker 9 - jepsen.util 6399 :info :cas [3 [23 46]] :RequestCanceledException INFO [2019-12-17 12:32:28,813] jepsen worker 9 - jepsen.util 6489 :invoke :write [3 24] INFO [2019-12-17 12:32:28,826] jepsen worker 85 - jepsen.util 8995 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:32:28,826] jepsen worker 71 - jepsen.util 6641 :info :write [23 13] :RequestCanceledException INFO [2019-12-17 12:32:28,826] jepsen worker 85 - jepsen.util 8995 :invoke :cas [28 [33 31]] INFO [2019-12-17 12:32:28,827] jepsen worker 71 - jepsen.util 6731 :invoke :cas [23 [49 23]] INFO [2019-12-17 12:32:28,832] jepsen worker 38 - jepsen.util 5978 :fail :cas [12 [40 26]] :ValueNotSwapFrom INFO [2019-12-17 12:32:28,832] jepsen worker 38 - jepsen.util 5978 :invoke :read [12 nil] INFO [2019-12-17 12:32:28,841] jepsen worker 75 - jepsen.util 7815 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:32:28,841] jepsen worker 75 - jepsen.util 7815 :invoke :read [25 nil] INFO [2019-12-17 12:32:28,858] jepsen worker 60 - jepsen.util 8970 :info :cas [20 [37 42]] :RequestCanceledException INFO [2019-12-17 12:32:28,858] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":34680,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:32:28,858] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":34680,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:32:28,858] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":34683,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:32:28,859] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":34683,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:32:28,863] jepsen worker 60 - jepsen.util 9060 :invoke :cas [20 [13 17]] INFO [2019-12-17 12:32:28,873] jepsen worker 32 - jepsen.util 6062 :ok :read [10 29] INFO [2019-12-17 12:32:28,874] jepsen worker 32 - jepsen.util 6062 :invoke :cas [10 [36 37]] INFO [2019-12-17 12:32:28,893] jepsen worker 43 - jepsen.util 7603 :info :cas [14 [22 17]] :RequestCanceledException INFO [2019-12-17 12:32:28,895] jepsen worker 43 - jepsen.util 7693 :invoke :read [14 nil] INFO [2019-12-17 12:32:28,897] jepsen worker 31 - jepsen.util 5431 :ok :read [10 29] INFO [2019-12-17 12:32:28,897] jepsen worker 31 - jepsen.util 5431 :invoke :write [10 2] INFO [2019-12-17 12:32:28,910] jepsen worker 64 - jepsen.util 4654 :fail :cas [21 [42 29]] :ValueNotSwapFrom INFO [2019-12-17 12:32:28,910] jepsen worker 64 - jepsen.util 4654 :invoke :read [21 nil] INFO [2019-12-17 12:32:28,911] jepsen worker 8 - jepsen.util 7658 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:32:28,911] jepsen worker 8 - jepsen.util 7658 :invoke :write [2 9] INFO [2019-12-17 12:32:28,914] jepsen worker 70 - jepsen.util 6460 :ok :read [23 33] INFO [2019-12-17 12:32:28,914] jepsen worker 70 - jepsen.util 6460 :invoke :read [23 nil] INFO [2019-12-17 12:32:28,917] jepsen worker 29 - jepsen.util 4709 :ok :read [9 5] INFO [2019-12-17 12:32:28,918] jepsen worker 29 - jepsen.util 4709 :invoke :cas [9 [8 35]] INFO [2019-12-17 12:32:28,927] jepsen worker 44 - jepsen.util 8774 :info :cas [14 [48 39]] :RequestCanceledException INFO [2019-12-17 12:32:28,927] jepsen worker 57 - jepsen.util 7707 :info :cas [19 [39 28]] :RequestCanceledException INFO [2019-12-17 12:32:28,927] jepsen worker 48 - jepsen.util 7338 :info :write [16 42] :RequestCanceledException INFO [2019-12-17 12:32:28,927] jepsen worker 56 - jepsen.util 6446 :info :write [18 40] :RequestCanceledException INFO [2019-12-17 12:32:28,928] jepsen worker 48 - jepsen.util 7428 :invoke :write [16 37] INFO [2019-12-17 12:32:28,928] jepsen worker 44 - jepsen.util 8864 :invoke :cas [14 [49 28]] INFO [2019-12-17 12:32:28,928] jepsen worker 57 - jepsen.util 7797 :invoke :read [19 nil] INFO [2019-12-17 12:32:28,928] jepsen worker 56 - jepsen.util 6536 :invoke :read [18 nil] INFO [2019-12-17 12:32:28,940] jepsen worker 16 - jepsen.util 9016 :info :write [5 23] :RequestCanceledException INFO [2019-12-17 12:32:28,940] jepsen worker 82 - jepsen.util 8632 :info :cas [27 [48 23]] :RequestCanceledException INFO [2019-12-17 12:32:28,942] jepsen worker 82 - jepsen.util 8722 :invoke :read [27 nil] INFO [2019-12-17 12:32:28,942] jepsen worker 16 - jepsen.util 9106 :invoke :cas [5 [24 8]] INFO [2019-12-17 12:32:28,985] jepsen worker 30 - jepsen.util 6060 :fail :cas [10 [25 30]] :ValueNotSwapFrom INFO [2019-12-17 12:32:28,985] jepsen worker 30 - jepsen.util 6060 :invoke :read [10 nil] INFO [2019-12-17 12:32:28,987] jepsen worker 27 - jepsen.util 5967 :ok :read [9 5] INFO [2019-12-17 12:32:28,987] jepsen worker 27 - jepsen.util 5967 :invoke :write [9 11] INFO [2019-12-17 12:32:28,989] jepsen worker 10 - jepsen.util 5950 :ok :read [3 36] INFO [2019-12-17 12:32:28,989] jepsen worker 10 - jepsen.util 5950 :invoke :cas [3 [43 46]] INFO [2019-12-17 12:32:28,993] jepsen worker 34 - jepsen.util 7594 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:32:28,993] jepsen worker 76 - jepsen.util 8266 :info :cas [25 [27 36]] :RequestCanceledException INFO [2019-12-17 12:32:28,994] jepsen worker 34 - jepsen.util 7594 :invoke :write [11 38] INFO [2019-12-17 12:32:28,994] jepsen worker 76 - jepsen.util 8356 :invoke :cas [25 [28 5]] INFO [2019-12-17 12:32:29,001] jepsen worker 64 - jepsen.util 4654 :ok :read [21 43] INFO [2019-12-17 12:32:29,001] jepsen worker 64 - jepsen.util 4654 :invoke :read [21 nil] INFO [2019-12-17 12:32:29,006] jepsen worker 19 - jepsen.util 6049 :ok :read [6 49] INFO [2019-12-17 12:32:29,006] jepsen worker 19 - jepsen.util 6049 :invoke :write [6 2] INFO [2019-12-17 12:32:29,011] jepsen worker 6 - jepsen.util 8646 :info :cas [2 [23 34]] :RequestCanceledException INFO [2019-12-17 12:32:29,012] jepsen worker 6 - jepsen.util 8736 :invoke :write [2 30] INFO [2019-12-17 12:32:29,026] jepsen worker 72 - jepsen.util 5922 :info :write [24 46] :RequestCanceledException INFO [2019-12-17 12:32:29,026] jepsen worker 33 - jepsen.util 8043 :info :write [11 31] :RequestCanceledException INFO [2019-12-17 12:32:29,026] jepsen worker 11 - jepsen.util 6221 :fail :read [3 nil] :RequestTimeoutException INFO [2019-12-17 12:32:29,026] jepsen worker 11 - jepsen.util 6221 :invoke :write [3 5] INFO [2019-12-17 12:32:29,027] jepsen worker 33 - jepsen.util 8133 :invoke :write [11 41] INFO [2019-12-17 12:32:29,027] jepsen worker 72 - jepsen.util 6012 :invoke :write [24 46] INFO [2019-12-17 12:32:29,041] jepsen worker 58 - jepsen.util 8608 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:32:29,041] jepsen worker 58 - jepsen.util 8608 :invoke :cas [19 [28 34]] INFO [2019-12-17 12:32:29,058] jepsen worker 29 - jepsen.util 4709 :fail :cas [9 [8 35]] :ValueNotSwapFrom INFO [2019-12-17 12:32:29,058] jepsen worker 29 - jepsen.util 4709 :invoke :cas [9 [8 18]] INFO [2019-12-17 12:32:29,058] jepsen worker 41 - jepsen.util 8861 :info :write [13 42] :RequestCanceledException INFO [2019-12-17 12:32:29,059] jepsen worker 41 - jepsen.util 8951 :invoke :write [13 1] INFO [2019-12-17 12:32:29,066] jepsen worker 38 - jepsen.util 5978 :ok :read [12 21] INFO [2019-12-17 12:32:29,066] jepsen worker 38 - jepsen.util 5978 :invoke :read [12 nil] INFO [2019-12-17 12:32:29,071] jepsen worker 20 - jepsen.util 5960 :ok :read [6 49] INFO [2019-12-17 12:32:29,071] jepsen worker 20 - jepsen.util 5960 :invoke :cas [6 [37 37]] INFO [2019-12-17 12:32:29,077] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":34703,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:32:29,077] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":34703,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:32:29,078] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":34707,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:32:29,078] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":34707,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:32:29,082] jepsen worker 64 - jepsen.util 4654 :ok :read [21 43] INFO [2019-12-17 12:32:29,082] jepsen worker 64 - jepsen.util 4654 :invoke :cas [21 [24 39]] INFO [2019-12-17 12:32:29,087] jepsen worker 10 - jepsen.util 5950 :fail :cas [3 [43 46]] :ValueNotSwapFrom INFO [2019-12-17 12:32:29,088] jepsen worker 10 - jepsen.util 5950 :invoke :write [3 48] INFO [2019-12-17 12:32:29,103] jepsen worker 12 - jepsen.util 6312 :fail :cas [4 [2 39]] :ValueNotSwapFrom INFO [2019-12-17 12:32:29,103] jepsen worker 12 - jepsen.util 6312 :invoke :write [4 1] INFO [2019-12-17 12:32:29,120] jepsen worker 71 - jepsen.util 6731 :fail :cas [23 [49 23]] :ValueNotSwapFrom INFO [2019-12-17 12:32:29,120] jepsen worker 71 - jepsen.util 6731 :invoke :cas [23 [48 47]] INFO [2019-12-17 12:32:29,126] jepsen worker 45 - jepsen.util 5895 :info :write [15 49] :RequestCanceledException INFO [2019-12-17 12:32:29,127] jepsen worker 45 - jepsen.util 5985 :invoke :write [15 38] INFO [2019-12-17 12:32:29,140] jepsen worker 0 - jepsen.util 8370 :info :write [0 21] :RequestCanceledException INFO [2019-12-17 12:32:29,141] jepsen worker 0 - jepsen.util 8460 :invoke :write [0 48] INFO [2019-12-17 12:32:29,153] jepsen worker 29 - jepsen.util 4709 :fail :cas [9 [8 18]] :ValueNotSwapFrom INFO [2019-12-17 12:32:29,154] jepsen worker 29 - jepsen.util 4709 :invoke :read [9 nil] INFO [2019-12-17 12:32:29,167] jepsen worker 69 - jepsen.util 6189 :fail :cas [23 [8 27]] :ValueNotSwapFrom INFO [2019-12-17 12:32:29,168] jepsen worker 70 - jepsen.util 6460 :ok :read [23 33] INFO [2019-12-17 12:32:29,168] jepsen worker 69 - jepsen.util 6189 :invoke :cas [23 [9 3]] INFO [2019-12-17 12:32:29,168] jepsen worker 27 - jepsen.util 5967 :ok :write [9 11] INFO [2019-12-17 12:32:29,168] jepsen worker 70 - jepsen.util 6460 :invoke :cas [23 [17 5]] INFO [2019-12-17 12:32:29,168] jepsen worker 27 - jepsen.util 5967 :invoke :write [9 35] INFO [2019-12-17 12:32:29,187] jepsen worker 56 - jepsen.util 6536 :ok :read [18 30] INFO [2019-12-17 12:32:29,187] jepsen worker 56 - jepsen.util 6536 :invoke :read [18 nil] INFO [2019-12-17 12:32:29,188] jepsen worker 38 - jepsen.util 5978 :ok :read [12 21] INFO [2019-12-17 12:32:29,189] jepsen worker 38 - jepsen.util 5978 :invoke :write [12 33] INFO [2019-12-17 12:32:29,194] jepsen worker 62 - jepsen.util 7712 :info :cas [20 [20 23]] :RequestCanceledException INFO [2019-12-17 12:32:29,195] jepsen worker 62 - jepsen.util 7802 :invoke :read [20 nil] INFO [2019-12-17 12:32:29,211] jepsen worker 81 - jepsen.util 7821 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:32:29,211] jepsen worker 81 - jepsen.util 7821 :invoke :cas [27 [19 46]] INFO [2019-12-17 12:32:29,226] jepsen worker 65 - jepsen.util 5105 :info :write [21 39] :RequestCanceledException INFO [2019-12-17 12:32:29,227] jepsen worker 84 - jepsen.util 7734 :info :cas [28 [15 7]] :RequestCanceledException INFO [2019-12-17 12:32:29,227] jepsen worker 65 - jepsen.util 5195 :invoke :cas [21 [29 12]] INFO [2019-12-17 12:32:29,229] jepsen worker 84 - jepsen.util 7824 :invoke :cas [28 [14 22]] INFO [2019-12-17 12:32:29,237] jepsen worker 27 - jepsen.util 5967 :ok :write [9 35] INFO [2019-12-17 12:32:29,238] jepsen worker 27 - jepsen.util 5967 :invoke :read [9 nil] INFO [2019-12-17 12:32:29,241] jepsen worker 78 - jepsen.util 6378 :fail :cas [26 [12 44]] :ValueNotSwapFrom INFO [2019-12-17 12:32:29,241] jepsen worker 78 - jepsen.util 6378 :invoke :read [26 nil] INFO [2019-12-17 12:32:29,245] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":34734,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:32:29,246] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":34734,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:32:29,246] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":34735,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:32:29,246] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":34735,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:32:29,257] jepsen worker 27 - jepsen.util 5967 :ok :read [9 35] INFO [2019-12-17 12:32:29,257] jepsen worker 27 - jepsen.util 5967 :invoke :write [9 23] INFO [2019-12-17 12:32:29,258] jepsen worker 86 - jepsen.util 8456 :info :cas [28 [6 30]] :RequestCanceledException INFO [2019-12-17 12:32:29,259] jepsen worker 86 - jepsen.util 8546 :invoke :read [28 nil] INFO [2019-12-17 12:32:29,288] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:32:29,288] jepsen nemesis - couchbase.util Rebalance check count: 263 INFO [2019-12-17 12:32:29,288] jepsen nemesis - couchbase.util Rebalance stuck count: 244 INFO [2019-12-17 12:32:29,294] jepsen worker 61 - jepsen.util 9241 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:32:29,294] jepsen worker 61 - jepsen.util 9241 :invoke :cas [20 [33 8]] INFO [2019-12-17 12:32:29,297] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":34742,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:32:29,297] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":34742,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:32:29,297] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":34745,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:32:29,297] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":34745,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:32:29,297] jepsen worker 27 - jepsen.util 5967 :ok :write [9 23] INFO [2019-12-17 12:32:29,298] jepsen worker 27 - jepsen.util 5967 :invoke :read [9 nil] INFO [2019-12-17 12:32:29,310] jepsen worker 80 - jepsen.util 6020 :info :write [26 7] :RequestCanceledException INFO [2019-12-17 12:32:29,311] jepsen worker 80 - jepsen.util 6110 :invoke :read [26 nil] INFO [2019-12-17 12:32:29,313] jepsen worker 27 - jepsen.util 5967 :ok :read [9 23] INFO [2019-12-17 12:32:29,313] jepsen worker 27 - jepsen.util 5967 :invoke :write [9 48] INFO [2019-12-17 12:32:29,327] jepsen worker 39 - jepsen.util 7599 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:32:29,327] jepsen worker 21 - jepsen.util 7581 :info :write [7 19] :RequestCanceledException INFO [2019-12-17 12:32:29,327] jepsen worker 39 - jepsen.util 7599 :invoke :cas [13 [45 19]] INFO [2019-12-17 12:32:29,328] jepsen worker 21 - jepsen.util 7671 :invoke :cas [7 [33 49]] INFO [2019-12-17 12:32:29,329] jepsen worker 56 - jepsen.util 6536 :ok :read [18 30] INFO [2019-12-17 12:32:29,329] jepsen worker 56 - jepsen.util 6536 :invoke :write [18 15] INFO [2019-12-17 12:32:29,330] jepsen worker 89 - jepsen.util 5849 :fail :cas [29 [19 41]] :ValueNotSwapFrom INFO [2019-12-17 12:32:29,330] jepsen worker 89 - jepsen.util 5849 :invoke :read [29 nil] INFO [2019-12-17 12:32:29,340] jepsen worker 83 - jepsen.util 8453 :info :write [27 24] :RequestCanceledException INFO [2019-12-17 12:32:29,341] jepsen worker 83 - jepsen.util 8543 :invoke :read [27 nil] INFO [2019-12-17 12:32:29,344] jepsen worker 70 - jepsen.util 6460 :fail :cas [23 [17 5]] :ValueNotSwapFrom INFO [2019-12-17 12:32:29,345] jepsen worker 70 - jepsen.util 6460 :invoke :cas [23 [6 31]] INFO [2019-12-17 12:32:29,357] jepsen worker 27 - jepsen.util 5967 :ok :write [9 48] INFO [2019-12-17 12:32:29,357] jepsen worker 27 - jepsen.util 5967 :invoke :write [9 10] INFO [2019-12-17 12:32:29,357] jepsen worker 1 - jepsen.util 7741 :info :cas [0 [31 6]] :RequestCanceledException INFO [2019-12-17 12:32:29,357] jepsen worker 63 - jepsen.util 4923 :info :write [21 0] :RequestCanceledException INFO [2019-12-17 12:32:29,358] jepsen worker 63 - jepsen.util 5013 :invoke :write [21 28] INFO [2019-12-17 12:32:29,362] jepsen worker 1 - jepsen.util 7831 :invoke :cas [0 [48 41]] INFO [2019-12-17 12:32:29,377] jepsen worker 51 - jepsen.util 5991 :fail :cas [17 [12 47]] :ValueNotSwapFrom INFO [2019-12-17 12:32:29,377] jepsen worker 51 - jepsen.util 5991 :invoke :cas [17 [27 18]] INFO [2019-12-17 12:32:29,393] jepsen worker 47 - jepsen.util 6797 :info :write [15 1] :RequestCanceledException INFO [2019-12-17 12:32:29,393] jepsen worker 40 - jepsen.util 8500 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:32:29,394] jepsen worker 40 - jepsen.util 8500 :invoke :write [13 23] INFO [2019-12-17 12:32:29,395] jepsen worker 47 - jepsen.util 6887 :invoke :write [15 6] INFO [2019-12-17 12:32:29,405] jepsen worker 89 - jepsen.util 5849 :ok :read [29 46] INFO [2019-12-17 12:32:29,405] jepsen worker 89 - jepsen.util 5849 :invoke :write [29 25] INFO [2019-12-17 12:32:29,411] jepsen worker 22 - jepsen.util 8032 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:32:29,411] jepsen worker 22 - jepsen.util 8032 :invoke :write [7 27] INFO [2019-12-17 12:32:29,418] jepsen worker 27 - jepsen.util 5967 :ok :write [9 10] INFO [2019-12-17 12:32:29,418] jepsen worker 27 - jepsen.util 5967 :invoke :read [9 nil] INFO [2019-12-17 12:32:29,426] jepsen worker 7 - jepsen.util 8377 :info :cas [2 [4 6]] :RequestCanceledException INFO [2019-12-17 12:32:29,427] jepsen worker 77 - jepsen.util 8357 :info :write [25 30] :RequestCanceledException INFO [2019-12-17 12:32:29,427] jepsen worker 7 - jepsen.util 8467 :invoke :read [2 nil] INFO [2019-12-17 12:32:29,428] jepsen worker 77 - jepsen.util 8447 :invoke :cas [25 [27 29]] INFO [2019-12-17 12:32:29,431] jepsen worker 70 - jepsen.util 6460 :fail :cas [23 [6 31]] :ValueNotSwapFrom INFO [2019-12-17 12:32:29,431] jepsen worker 70 - jepsen.util 6460 :invoke :cas [23 [7 12]] INFO [2019-12-17 12:32:29,441] jepsen worker 15 - jepsen.util 7125 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:32:29,441] jepsen worker 3 - jepsen.util 5943 :info :write [1 47] :RequestCanceledException INFO [2019-12-17 12:32:29,442] jepsen worker 15 - jepsen.util 7125 :invoke :cas [5 [14 27]] INFO [2019-12-17 12:32:29,445] jepsen worker 3 - jepsen.util 6033 :invoke :cas [1 [17 18]] INFO [2019-12-17 12:32:29,451] jepsen worker 27 - jepsen.util 5967 :ok :read [9 10] INFO [2019-12-17 12:32:29,451] jepsen worker 27 - jepsen.util 5967 :invoke :cas [9 [37 45]] INFO [2019-12-17 12:32:29,458] jepsen worker 25 - jepsen.util 8665 :info :write [8 5] :RequestCanceledException INFO [2019-12-17 12:32:29,458] jepsen worker 13 - jepsen.util 5863 :info :write [4 7] :RequestCanceledException INFO [2019-12-17 12:32:29,459] jepsen worker 25 - jepsen.util 8755 :invoke :read [8 nil] INFO [2019-12-17 12:32:29,459] jepsen worker 13 - jepsen.util 5953 :invoke :cas [4 [38 9]] INFO [2019-12-17 12:32:29,460] jepsen worker 27 - jepsen.util 5967 :fail :cas [9 [37 45]] :ValueNotSwapFrom INFO [2019-12-17 12:32:29,460] jepsen worker 27 - jepsen.util 5967 :invoke :cas [9 [32 49]] INFO [2019-12-17 12:32:29,471] jepsen worker 27 - jepsen.util 5967 :fail :cas [9 [32 49]] :ValueNotSwapFrom INFO [2019-12-17 12:32:29,471] jepsen worker 27 - jepsen.util 5967 :invoke :cas [9 [49 1]] INFO [2019-12-17 12:32:29,482] jepsen worker 27 - jepsen.util 5967 :fail :cas [9 [49 1]] :ValueNotSwapFrom INFO [2019-12-17 12:32:29,482] jepsen worker 27 - jepsen.util 5967 :invoke :read [9 nil] INFO [2019-12-17 12:32:29,493] jepsen worker 17 - jepsen.util 8117 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:32:29,493] jepsen worker 54 - jepsen.util 6444 :info :write [18 37] :RequestCanceledException INFO [2019-12-17 12:32:29,494] jepsen worker 17 - jepsen.util 8117 :invoke :cas [5 [33 26]] INFO [2019-12-17 12:32:29,494] jepsen worker 27 - jepsen.util 5967 :ok :read [9 10] INFO [2019-12-17 12:32:29,494] jepsen worker 27 - jepsen.util 5967 :invoke :cas [9 [23 11]] INFO [2019-12-17 12:32:29,495] jepsen worker 54 - jepsen.util 6534 :invoke :write [18 44] INFO [2019-12-17 12:32:29,507] jepsen worker 27 - jepsen.util 5967 :fail :cas [9 [23 11]] :ValueNotSwapFrom INFO [2019-12-17 12:32:29,507] jepsen worker 27 - jepsen.util 5967 :invoke :cas [9 [2 8]] INFO [2019-12-17 12:32:29,511] jepsen worker 26 - jepsen.util 8396 :info :write [8 39] :RequestCanceledException INFO [2019-12-17 12:32:29,513] jepsen worker 26 - jepsen.util 8486 :invoke :write [8 15] INFO [2019-12-17 12:32:29,516] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":34777,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:32:29,517] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":34777,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:32:29,517] jepsen worker 80 - jepsen.util 6110 :ok :read [26 41] INFO [2019-12-17 12:32:29,517] jepsen worker 80 - jepsen.util 6110 :invoke :write [26 29] INFO [2019-12-17 12:32:29,520] jepsen worker 27 - jepsen.util 5967 :fail :cas [9 [2 8]] :ValueNotSwapFrom INFO [2019-12-17 12:32:29,520] jepsen worker 27 - jepsen.util 5967 :invoke :cas [9 [42 1]] INFO [2019-12-17 12:32:29,533] jepsen worker 27 - jepsen.util 5967 :fail :cas [9 [42 1]] :ValueNotSwapFrom INFO [2019-12-17 12:32:29,533] jepsen worker 27 - jepsen.util 5967 :invoke :write [9 31] INFO [2019-12-17 12:32:29,574] jepsen worker 27 - jepsen.util 5967 :ok :write [9 31] INFO [2019-12-17 12:32:29,574] jepsen worker 27 - jepsen.util 5967 :invoke :read [9 nil] INFO [2019-12-17 12:32:29,586] jepsen worker 71 - jepsen.util 6731 :fail :cas [23 [48 47]] :ValueNotSwapFrom INFO [2019-12-17 12:32:29,586] jepsen worker 71 - jepsen.util 6731 :invoke :write [23 8] INFO [2019-12-17 12:32:29,592] jepsen worker 27 - jepsen.util 5967 :ok :read [9 31] INFO [2019-12-17 12:32:29,592] jepsen worker 27 - jepsen.util 5967 :invoke :read [9 nil] INFO [2019-12-17 12:32:29,609] jepsen worker 27 - jepsen.util 5967 :ok :read [9 31] INFO [2019-12-17 12:32:29,609] jepsen worker 27 - jepsen.util 5967 :invoke :cas [9 [23 33]] INFO [2019-12-17 12:32:29,627] jepsen worker 55 - jepsen.util 6985 :info :write [18 49] :RequestCanceledException INFO [2019-12-17 12:32:29,628] jepsen worker 27 - jepsen.util 5967 :fail :cas [9 [23 33]] :ValueNotSwapFrom INFO [2019-12-17 12:32:29,628] jepsen worker 27 - jepsen.util 5967 :invoke :cas [9 [3 12]] INFO [2019-12-17 12:32:29,631] jepsen worker 55 - jepsen.util 7075 :invoke :cas [18 [18 31]] INFO [2019-12-17 12:32:29,641] jepsen worker 67 - jepsen.util 8257 :info :cas [22 [40 30]] :RequestCanceledException INFO [2019-12-17 12:32:29,641] jepsen worker 59 - jepsen.util 8249 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:32:29,641] jepsen worker 59 - jepsen.util 8249 :invoke :write [19 20] INFO [2019-12-17 12:32:29,642] jepsen worker 67 - jepsen.util 8347 :invoke :read [22 nil] INFO [2019-12-17 12:32:29,647] jepsen worker 27 - jepsen.util 5967 :fail :cas [9 [3 12]] :ValueNotSwapFrom INFO [2019-12-17 12:32:29,648] jepsen worker 27 - jepsen.util 5967 :invoke :write [9 26] INFO [2019-12-17 12:32:29,656] jepsen worker 13 - jepsen.util 5953 :fail :cas [4 [38 9]] :ValueNotSwapFrom INFO [2019-12-17 12:32:29,656] jepsen worker 13 - jepsen.util 5953 :invoke :read [4 nil] INFO [2019-12-17 12:32:29,657] jepsen worker 24 - jepsen.util 7314 :info :write [8 8] :RequestCanceledException INFO [2019-12-17 12:32:29,657] jepsen worker 23 - jepsen.util 8123 :info :write [7 30] :RequestCanceledException INFO [2019-12-17 12:32:29,658] jepsen worker 24 - jepsen.util 7404 :invoke :read [8 nil] INFO [2019-12-17 12:32:29,658] jepsen worker 23 - jepsen.util 8213 :invoke :cas [7 [23 5]] INFO [2019-12-17 12:32:29,684] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":34807,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:32:29,684] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":34807,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:32:29,694] jepsen worker 42 - jepsen.util 8682 :info :write [14 2] :RequestCanceledException INFO [2019-12-17 12:32:29,694] jepsen worker 14 - jepsen.util 6134 :info :write [4 18] :RequestCanceledException INFO [2019-12-17 12:32:29,695] jepsen worker 42 - jepsen.util 8772 :invoke :read [14 nil] INFO [2019-12-17 12:32:29,695] jepsen worker 14 - jepsen.util 6224 :invoke :write [4 10] INFO [2019-12-17 12:32:29,699] jepsen worker 27 - jepsen.util 5967 :ok :write [9 26] INFO [2019-12-17 12:32:29,699] jepsen worker 27 - jepsen.util 5967 :invoke :read [9 nil] INFO [2019-12-17 12:32:29,707] jepsen worker 70 - jepsen.util 6460 :fail :cas [23 [7 12]] :ValueNotSwapFrom INFO [2019-12-17 12:32:29,707] jepsen worker 70 - jepsen.util 6460 :invoke :cas [23 [49 24]] INFO [2019-12-17 12:32:29,711] jepsen worker 35 - jepsen.util 8135 :info :cas [11 [39 13]] :RequestCanceledException INFO [2019-12-17 12:32:29,711] jepsen worker 49 - jepsen.util 7249 :info :write [16 12] :RequestCanceledException INFO [2019-12-17 12:32:29,712] jepsen worker 35 - jepsen.util 8225 :invoke :read [11 nil] INFO [2019-12-17 12:32:29,712] jepsen worker 49 - jepsen.util 7339 :invoke :read [16 nil] INFO [2019-12-17 12:32:29,713] jepsen worker 73 - jepsen.util 5743 :ok :write [24 47] INFO [2019-12-17 12:32:29,714] jepsen worker 73 - jepsen.util 5743 :invoke :write [24 42] INFO [2019-12-17 12:32:29,715] jepsen worker 9 - jepsen.util 6489 :ok :write [3 24] INFO [2019-12-17 12:32:29,715] jepsen worker 9 - jepsen.util 6489 :invoke :write [3 26] INFO [2019-12-17 12:32:29,717] jepsen worker 31 - jepsen.util 5431 :ok :write [10 2] INFO [2019-12-17 12:32:29,718] jepsen worker 31 - jepsen.util 5431 :invoke :read [10 nil] INFO [2019-12-17 12:32:29,723] jepsen worker 27 - jepsen.util 5967 :ok :read [9 26] INFO [2019-12-17 12:32:29,723] jepsen worker 27 - jepsen.util 5967 :invoke :read [9 nil] INFO [2019-12-17 12:32:29,740] jepsen worker 52 - jepsen.util 6352 :info :write [17 19] :RequestCanceledException INFO [2019-12-17 12:32:29,741] jepsen worker 27 - jepsen.util 5967 :ok :read [9 26] INFO [2019-12-17 12:32:29,741] jepsen worker 27 - jepsen.util 5967 :invoke :read [9 nil] INFO [2019-12-17 12:32:29,741] jepsen worker 52 - jepsen.util 6442 :invoke :write [17 16] INFO [2019-12-17 12:32:29,784] jepsen worker 27 - jepsen.util 5967 :ok :read [9 26] INFO [2019-12-17 12:32:29,784] jepsen worker 27 - jepsen.util 5967 :invoke :write [9 46] INFO [2019-12-17 12:32:29,792] jepsen worker 19 - jepsen.util 6049 :ok :write [6 2] INFO [2019-12-17 12:32:29,792] jepsen worker 19 - jepsen.util 6049 :invoke :cas [6 [30 42]] INFO [2019-12-17 12:32:29,815] jepsen worker 55 - jepsen.util 7075 :fail :cas [18 [18 31]] :ValueNotSwapFrom INFO [2019-12-17 12:32:29,815] jepsen worker 55 - jepsen.util 7075 :invoke :read [18 nil] INFO [2019-12-17 12:32:29,821] jepsen worker 30 - jepsen.util 6060 :ok :read [10 2] INFO [2019-12-17 12:32:29,821] jepsen worker 30 - jepsen.util 6060 :invoke :cas [10 [4 32]] INFO [2019-12-17 12:32:29,840] jepsen worker 87 - jepsen.util 5487 :info :write [29 47] :RequestCanceledException INFO [2019-12-17 12:32:29,841] jepsen worker 87 - jepsen.util 5577 :invoke :cas [29 [45 1]] INFO [2019-12-17 12:32:29,853] jepsen worker 65 - jepsen.util 5195 :fail :cas [21 [29 12]] :ValueNotSwapFrom INFO [2019-12-17 12:32:29,854] jepsen worker 65 - jepsen.util 5195 :invoke :cas [21 [39 42]] INFO [2019-12-17 12:32:29,858] jepsen worker 2 - jepsen.util 7832 :info :write [0 24] :RequestCanceledException INFO [2019-12-17 12:32:29,862] jepsen worker 2 - jepsen.util 7922 :invoke :write [0 27] INFO [2019-12-17 12:32:29,876] jepsen worker 31 - jepsen.util 5431 :ok :read [10 2] INFO [2019-12-17 12:32:29,876] jepsen worker 31 - jepsen.util 5431 :invoke :write [10 2] INFO [2019-12-17 12:32:29,891] jepsen worker 27 - jepsen.util 5967 :ok :write [9 46] INFO [2019-12-17 12:32:29,891] jepsen worker 27 - jepsen.util 5967 :invoke :cas [9 [29 28]] INFO [2019-12-17 12:32:29,893] jepsen worker 88 - jepsen.util 6208 :info :write [29 49] :RequestCanceledException INFO [2019-12-17 12:32:29,894] jepsen worker 88 - jepsen.util 6298 :invoke :write [29 10] INFO [2019-12-17 12:32:29,897] jepsen worker 69 - jepsen.util 6189 :fail :cas [23 [9 3]] :ValueNotSwapFrom INFO [2019-12-17 12:32:29,897] jepsen worker 69 - jepsen.util 6189 :invoke :cas [23 [35 10]] INFO [2019-12-17 12:32:29,920] jepsen worker 55 - jepsen.util 7075 :ok :read [18 30] INFO [2019-12-17 12:32:29,920] jepsen worker 78 - jepsen.util 6378 :ok :read [26 41] INFO [2019-12-17 12:32:29,920] jepsen worker 55 - jepsen.util 7075 :invoke :read [18 nil] INFO [2019-12-17 12:32:29,920] jepsen worker 78 - jepsen.util 6378 :invoke :cas [26 [21 22]] INFO [2019-12-17 12:32:29,941] jepsen worker 27 - jepsen.util 5967 :fail :cas [9 [29 28]] :ValueNotSwapFrom INFO [2019-12-17 12:32:29,941] jepsen worker 27 - jepsen.util 5967 :invoke :read [9 nil] INFO [2019-12-17 12:32:29,943] jepsen worker 19 - jepsen.util 6049 :fail :cas [6 [30 42]] :ValueNotSwapFrom INFO [2019-12-17 12:32:29,944] jepsen worker 19 - jepsen.util 6049 :invoke :read [6 nil] INFO [2019-12-17 12:32:29,955] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":34834,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:32:29,955] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":34834,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:32:29,956] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":34838,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:32:29,956] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":34838,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:32:29,957] jepsen worker 5 - jepsen.util 5585 :info :write [1 41] :RequestCanceledException INFO [2019-12-17 12:32:29,958] jepsen worker 5 - jepsen.util 5675 :invoke :cas [1 [41 31]] INFO [2019-12-17 12:32:29,964] jepsen worker 51 - jepsen.util 5991 :fail :cas [17 [27 18]] :ValueNotSwapFrom INFO [2019-12-17 12:32:29,964] jepsen worker 51 - jepsen.util 5991 :invoke :cas [17 [14 43]] INFO [2019-12-17 12:32:29,983] jepsen worker 27 - jepsen.util 5967 :ok :read [9 46] INFO [2019-12-17 12:32:29,984] jepsen worker 27 - jepsen.util 5967 :invoke :cas [9 [49 48]] INFO [2019-12-17 12:32:29,984] jepsen worker 69 - jepsen.util 6189 :fail :cas [23 [35 10]] :ValueNotSwapFrom INFO [2019-12-17 12:32:29,984] jepsen worker 69 - jepsen.util 6189 :invoke :read [23 nil] INFO [2019-12-17 12:32:29,994] jepsen worker 47 - jepsen.util 6887 :ok :write [15 6] INFO [2019-12-17 12:32:29,994] jepsen worker 47 - jepsen.util 6887 :invoke :cas [15 [18 7]] INFO [2019-12-17 12:32:30,002] jepsen worker 27 - jepsen.util 5967 :fail :cas [9 [49 48]] :ValueNotSwapFrom INFO [2019-12-17 12:32:30,002] jepsen worker 27 - jepsen.util 5967 :invoke :cas [9 [22 13]] INFO [2019-12-17 12:32:30,010] jepsen worker 50 - jepsen.util 8510 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:32:30,011] jepsen worker 50 - jepsen.util 8510 :invoke :write [16 29] INFO [2019-12-17 12:32:30,018] jepsen worker 63 - jepsen.util 5013 :ok :write [21 28] INFO [2019-12-17 12:32:30,018] jepsen worker 63 - jepsen.util 5013 :invoke :write [21 22] INFO [2019-12-17 12:32:30,025] jepsen worker 27 - jepsen.util 5967 :fail :cas [9 [22 13]] :ValueNotSwapFrom INFO [2019-12-17 12:32:30,025] jepsen worker 27 - jepsen.util 5967 :invoke :cas [9 [42 5]] INFO [2019-12-17 12:32:30,037] jepsen worker 3 - jepsen.util 6033 :fail :cas [1 [17 18]] :ValueNotSwapFrom INFO [2019-12-17 12:32:30,037] jepsen worker 3 - jepsen.util 6033 :invoke :cas [1 [34 45]] INFO [2019-12-17 12:32:30,046] jepsen worker 65 - jepsen.util 5195 :fail :cas [21 [39 42]] :ValueNotSwapFrom INFO [2019-12-17 12:32:30,047] jepsen worker 65 - jepsen.util 5195 :invoke :read [21 nil] INFO [2019-12-17 12:32:30,065] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":34855,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:32:30,065] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":34855,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:32:30,065] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":34857,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:32:30,065] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":34857,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:32:30,078] jepsen worker 27 - jepsen.util 5967 :fail :cas [9 [42 5]] :ValueNotSwapFrom INFO [2019-12-17 12:32:30,078] jepsen worker 27 - jepsen.util 5967 :invoke :cas [9 [9 49]] INFO [2019-12-17 12:32:30,092] jepsen worker 78 - jepsen.util 6378 :fail :cas [26 [21 22]] :ValueNotSwapFrom INFO [2019-12-17 12:32:30,092] jepsen worker 27 - jepsen.util 5967 :fail :cas [9 [9 49]] :ValueNotSwapFrom INFO [2019-12-17 12:32:30,092] jepsen worker 78 - jepsen.util 6378 :invoke :read [26 nil] INFO [2019-12-17 12:32:30,092] jepsen worker 27 - jepsen.util 5967 :invoke :write [9 5] INFO [2019-12-17 12:32:30,127] jepsen worker 46 - jepsen.util 5986 :info :cas [15 [47 14]] :RequestCanceledException INFO [2019-12-17 12:32:30,128] jepsen worker 46 - jepsen.util 6076 :invoke :cas [15 [11 28]] INFO [2019-12-17 12:32:30,137] jepsen worker 63 - jepsen.util 5013 :ok :write [21 22] INFO [2019-12-17 12:32:30,138] jepsen worker 63 - jepsen.util 5013 :invoke :cas [21 [44 7]] INFO [2019-12-17 12:32:30,140] jepsen worker 4 - jepsen.util 5494 :info :write [1 43] :RequestCanceledException INFO [2019-12-17 12:32:30,142] jepsen worker 4 - jepsen.util 5584 :invoke :read [1 nil] INFO [2019-12-17 12:32:30,146] jepsen worker 55 - jepsen.util 7075 :ok :read [18 30] INFO [2019-12-17 12:32:30,146] jepsen worker 55 - jepsen.util 7075 :invoke :write [18 20] INFO [2019-12-17 12:32:30,193] jepsen worker 46 - jepsen.util 6076 :fail :cas [15 [11 28]] :ValueNotSwapFrom INFO [2019-12-17 12:32:30,193] jepsen worker 46 - jepsen.util 6076 :invoke :cas [15 [33 26]] INFO [2019-12-17 12:32:30,202] jepsen worker 63 - jepsen.util 5013 :fail :cas [21 [44 7]] :ValueNotSwapFrom INFO [2019-12-17 12:32:30,202] jepsen worker 63 - jepsen.util 5013 :invoke :write [21 47] INFO [2019-12-17 12:32:30,203] jepsen worker 31 - jepsen.util 5431 :ok :write [10 2] INFO [2019-12-17 12:32:30,204] jepsen worker 31 - jepsen.util 5431 :invoke :write [10 24] INFO [2019-12-17 12:32:30,210] jepsen worker 51 - jepsen.util 5991 :fail :cas [17 [14 43]] :ValueNotSwapFrom INFO [2019-12-17 12:32:30,210] jepsen worker 51 - jepsen.util 5991 :invoke :write [17 43] INFO [2019-12-17 12:32:30,211] jepsen worker 18 - jepsen.util 5778 :info :write [6 37] :RequestCanceledException INFO [2019-12-17 12:32:30,211] jepsen worker 74 - jepsen.util 6194 :fail :read [24 nil] :RequestTimeoutException INFO [2019-12-17 12:32:30,212] jepsen worker 74 - jepsen.util 6194 :invoke :write [24 9] INFO [2019-12-17 12:32:30,213] jepsen worker 18 - jepsen.util 5868 :invoke :read [6 nil] INFO [2019-12-17 12:32:30,225] jepsen worker 46 - jepsen.util 6076 :fail :cas [15 [33 26]] :ValueNotSwapFrom INFO [2019-12-17 12:32:30,225] jepsen worker 46 - jepsen.util 6076 :invoke :write [15 6] INFO [2019-12-17 12:32:30,231] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":34873,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:32:30,231] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":34873,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:32:30,231] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":34874,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:32:30,231] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":34874,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:32:30,231] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":34875,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:32:30,231] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":34875,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:32:30,247] jepsen worker 27 - jepsen.util 5967 :ok :write [9 5] INFO [2019-12-17 12:32:30,248] jepsen worker 27 - jepsen.util 5967 :invoke :write [9 44] INFO [2019-12-17 12:32:30,258] jepsen worker 3 - jepsen.util 6033 :fail :cas [1 [34 45]] :ValueNotSwapFrom INFO [2019-12-17 12:32:30,258] jepsen worker 3 - jepsen.util 6033 :invoke :cas [1 [48 21]] INFO [2019-12-17 12:32:30,284] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":34882,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:32:30,285] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":34882,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:32:30,285] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":34886,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:32:30,285] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":34886,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:32:30,294] jepsen worker 18 - jepsen.util 5868 :ok :read [6 2] INFO [2019-12-17 12:32:30,294] jepsen worker 18 - jepsen.util 5868 :invoke :cas [6 [41 24]] INFO [2019-12-17 12:32:30,301] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:32:30,301] jepsen nemesis - couchbase.util Rebalance check count: 264 INFO [2019-12-17 12:32:30,301] jepsen nemesis - couchbase.util Rebalance stuck count: 245 INFO [2019-12-17 12:32:30,327] jepsen worker 37 - jepsen.util 5707 :info :write [12 22] :RequestCanceledException INFO [2019-12-17 12:32:30,328] jepsen worker 37 - jepsen.util 5797 :invoke :read [12 nil] INFO [2019-12-17 12:32:30,338] jepsen worker 78 - jepsen.util 6378 :ok :read [26 41] INFO [2019-12-17 12:32:30,338] jepsen worker 78 - jepsen.util 6378 :invoke :cas [26 [19 9]] INFO [2019-12-17 12:32:30,340] jepsen worker 63 - jepsen.util 5013 :ok :write [21 47] INFO [2019-12-17 12:32:30,340] jepsen worker 63 - jepsen.util 5013 :invoke :write [21 45] INFO [2019-12-17 12:32:30,349] jepsen worker 27 - jepsen.util 5967 :ok :write [9 44] INFO [2019-12-17 12:32:30,349] jepsen worker 27 - jepsen.util 5967 :invoke :cas [9 [26 9]] INFO [2019-12-17 12:32:30,367] jepsen worker 18 - jepsen.util 5868 :fail :cas [6 [41 24]] :ValueNotSwapFrom INFO [2019-12-17 12:32:30,367] jepsen worker 18 - jepsen.util 5868 :invoke :read [6 nil] INFO [2019-12-17 12:32:30,378] jepsen worker 5 - jepsen.util 5675 :fail :cas [1 [41 31]] :ValueNotSwapFrom INFO [2019-12-17 12:32:30,378] jepsen worker 5 - jepsen.util 5675 :invoke :write [1 37] INFO [2019-12-17 12:32:30,391] jepsen worker 27 - jepsen.util 5967 :fail :cas [9 [26 9]] :ValueNotSwapFrom INFO [2019-12-17 12:32:30,391] jepsen worker 27 - jepsen.util 5967 :invoke :read [9 nil] INFO [2019-12-17 12:32:30,412] jepsen worker 53 - jepsen.util 5633 :info :write [17 33] :RequestCanceledException INFO [2019-12-17 12:32:30,413] jepsen worker 53 - jepsen.util 5723 :invoke :cas [17 [1 4]] INFO [2019-12-17 12:32:30,414] jepsen worker 69 - jepsen.util 6189 :ok :read [23 33] INFO [2019-12-17 12:32:30,415] jepsen worker 69 - jepsen.util 6189 :invoke :read [23 nil] INFO [2019-12-17 12:32:30,418] jepsen worker 18 - jepsen.util 5868 :ok :read [6 2] INFO [2019-12-17 12:32:30,418] jepsen worker 18 - jepsen.util 5868 :invoke :write [6 18] INFO [2019-12-17 12:32:30,450] jepsen worker 27 - jepsen.util 5967 :ok :read [9 44] INFO [2019-12-17 12:32:30,451] jepsen worker 27 - jepsen.util 5967 :invoke :read [9 nil] INFO [2019-12-17 12:32:30,470] jepsen worker 46 - jepsen.util 6076 :ok :write [15 6] INFO [2019-12-17 12:32:30,470] jepsen worker 46 - jepsen.util 6076 :invoke :cas [15 [45 23]] INFO [2019-12-17 12:32:30,476] jepsen worker 73 - jepsen.util 5743 :ok :write [24 42] INFO [2019-12-17 12:32:30,476] jepsen worker 73 - jepsen.util 5743 :invoke :cas [24 [31 42]] INFO [2019-12-17 12:32:30,500] jepsen worker 47 - jepsen.util 6887 :fail :cas [15 [18 7]] :ValueNotSwapFrom INFO [2019-12-17 12:32:30,500] jepsen worker 47 - jepsen.util 6887 :invoke :write [15 7] INFO [2019-12-17 12:32:30,503] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":34905,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:32:30,503] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":34905,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:32:30,503] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":34908,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:32:30,504] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":34908,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:32:30,504] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":34909,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:32:30,504] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":34909,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:32:30,521] jepsen worker 46 - jepsen.util 6076 :fail :cas [15 [45 23]] :ValueNotSwapFrom INFO [2019-12-17 12:32:30,522] jepsen worker 46 - jepsen.util 6076 :invoke :read [15 nil] INFO [2019-12-17 12:32:30,526] jepsen worker 28 - jepsen.util 4708 :info :write [9 38] :RequestCanceledException INFO [2019-12-17 12:32:30,527] jepsen worker 28 - jepsen.util 4798 :invoke :read [9 nil] INFO [2019-12-17 12:32:30,531] jepsen worker 37 - jepsen.util 5797 :ok :read [12 21] INFO [2019-12-17 12:32:30,531] jepsen worker 37 - jepsen.util 5797 :invoke :cas [12 [16 41]] INFO [2019-12-17 12:32:30,538] jepsen worker 13 - jepsen.util 5953 :ok :read [4 41] INFO [2019-12-17 12:32:30,539] jepsen worker 13 - jepsen.util 5953 :invoke :write [4 11] INFO [2019-12-17 12:32:30,539] jepsen worker 70 - jepsen.util 6460 :fail :cas [23 [49 24]] :ValueNotSwapFrom INFO [2019-12-17 12:32:30,539] jepsen worker 70 - jepsen.util 6460 :invoke :write [23 7] INFO [2019-12-17 12:32:30,557] jepsen worker 79 - jepsen.util 5929 :info :write [26 17] :RequestCanceledException INFO [2019-12-17 12:32:30,559] jepsen worker 79 - jepsen.util 6019 :invoke :write [26 1] INFO [2019-12-17 12:32:30,559] jepsen worker 63 - jepsen.util 5013 :ok :write [21 45] INFO [2019-12-17 12:32:30,559] jepsen worker 63 - jepsen.util 5013 :invoke :read [21 nil] INFO [2019-12-17 12:32:30,561] jepsen worker 46 - jepsen.util 6076 :ok :read [15 6] INFO [2019-12-17 12:32:30,561] jepsen worker 46 - jepsen.util 6076 :invoke :read [15 nil] INFO [2019-12-17 12:32:30,581] jepsen worker 63 - jepsen.util 5013 :ok :read [21 45] INFO [2019-12-17 12:32:30,581] jepsen worker 63 - jepsen.util 5013 :invoke :cas [21 [40 33]] INFO [2019-12-17 12:32:30,589] jepsen worker 87 - jepsen.util 5577 :fail :cas [29 [45 1]] :ValueNotSwapFrom INFO [2019-12-17 12:32:30,589] jepsen worker 87 - jepsen.util 5577 :invoke :read [29 nil] INFO [2019-12-17 12:32:30,598] jepsen worker 46 - jepsen.util 6076 :ok :read [15 6] INFO [2019-12-17 12:32:30,598] jepsen worker 46 - jepsen.util 6076 :invoke :cas [15 [5 40]] INFO [2019-12-17 12:32:30,618] jepsen worker 63 - jepsen.util 5013 :fail :cas [21 [40 33]] :ValueNotSwapFrom INFO [2019-12-17 12:32:30,618] jepsen worker 63 - jepsen.util 5013 :invoke :cas [21 [9 32]] INFO [2019-12-17 12:32:30,623] jepsen worker 87 - jepsen.util 5577 :ok :read [29 46] INFO [2019-12-17 12:32:30,623] jepsen worker 87 - jepsen.util 5577 :invoke :cas [29 [32 11]] INFO [2019-12-17 12:32:30,627] jepsen worker 68 - jepsen.util 7808 :info :write [22 46] :RequestCanceledException INFO [2019-12-17 12:32:30,628] jepsen worker 68 - jepsen.util 7898 :invoke :read [22 nil] INFO [2019-12-17 12:32:30,628] jepsen worker 46 - jepsen.util 6076 :fail :cas [15 [5 40]] :ValueNotSwapFrom INFO [2019-12-17 12:32:30,628] jepsen worker 46 - jepsen.util 6076 :invoke :cas [15 [44 44]] INFO [2019-12-17 12:32:30,637] jepsen worker 63 - jepsen.util 5013 :fail :cas [21 [9 32]] :ValueNotSwapFrom INFO [2019-12-17 12:32:30,638] jepsen worker 63 - jepsen.util 5013 :invoke :write [21 37] INFO [2019-12-17 12:32:30,652] jepsen worker 46 - jepsen.util 6076 :fail :cas [15 [44 44]] :ValueNotSwapFrom INFO [2019-12-17 12:32:30,653] jepsen worker 46 - jepsen.util 6076 :invoke :write [15 23] INFO [2019-12-17 12:32:30,680] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":34931,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:32:30,680] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":34931,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:32:30,680] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":34935,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:32:30,680] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":34935,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:32:30,702] jepsen worker 3 - jepsen.util 6033 :fail :cas [1 [48 21]] :ValueNotSwapFrom INFO [2019-12-17 12:32:30,702] jepsen worker 3 - jepsen.util 6033 :invoke :cas [1 [7 11]] INFO [2019-12-17 12:32:30,705] jepsen worker 63 - jepsen.util 5013 :ok :write [21 37] INFO [2019-12-17 12:32:30,706] jepsen worker 63 - jepsen.util 5013 :invoke :cas [21 [35 35]] INFO [2019-12-17 12:32:30,710] jepsen worker 36 - jepsen.util 6516 :info :write [12 29] :RequestCanceledException INFO [2019-12-17 12:32:30,711] jepsen worker 36 - jepsen.util 6606 :invoke :write [12 35] INFO [2019-12-17 12:32:30,729] jepsen worker 63 - jepsen.util 5013 :fail :cas [21 [35 35]] :ValueNotSwapFrom INFO [2019-12-17 12:32:30,729] jepsen worker 63 - jepsen.util 5013 :invoke :write [21 27] INFO [2019-12-17 12:32:30,734] jepsen worker 78 - jepsen.util 6378 :fail :cas [26 [19 9]] :ValueNotSwapFrom INFO [2019-12-17 12:32:30,734] jepsen worker 78 - jepsen.util 6378 :invoke :write [26 2] INFO [2019-12-17 12:32:30,785] jepsen worker 63 - jepsen.util 5013 :ok :write [21 27] INFO [2019-12-17 12:32:30,785] jepsen worker 63 - jepsen.util 5013 :invoke :cas [21 [48 3]] INFO [2019-12-17 12:32:30,802] jepsen worker 63 - jepsen.util 5013 :fail :cas [21 [48 3]] :ValueNotSwapFrom INFO [2019-12-17 12:32:30,802] jepsen worker 63 - jepsen.util 5013 :invoke :cas [21 [40 47]] INFO [2019-12-17 12:32:30,802] jepsen worker 4 - jepsen.util 5584 :ok :read [1 25] INFO [2019-12-17 12:32:30,802] jepsen worker 4 - jepsen.util 5584 :invoke :cas [1 [38 7]] INFO [2019-12-17 12:32:30,813] jepsen worker 69 - jepsen.util 6189 :ok :read [23 33] INFO [2019-12-17 12:32:30,813] jepsen worker 69 - jepsen.util 6189 :invoke :read [23 nil] INFO [2019-12-17 12:32:30,822] jepsen worker 63 - jepsen.util 5013 :fail :cas [21 [40 47]] :ValueNotSwapFrom INFO [2019-12-17 12:32:30,822] jepsen worker 63 - jepsen.util 5013 :invoke :cas [21 [9 25]] INFO [2019-12-17 12:32:30,823] jepsen worker 31 - jepsen.util 5431 :ok :write [10 24] INFO [2019-12-17 12:32:30,823] jepsen worker 31 - jepsen.util 5431 :invoke :cas [10 [7 31]] INFO [2019-12-17 12:32:30,825] jepsen worker 37 - jepsen.util 5797 :fail :cas [12 [16 41]] :ValueNotSwapFrom INFO [2019-12-17 12:32:30,825] jepsen worker 37 - jepsen.util 5797 :invoke :cas [12 [38 0]] INFO [2019-12-17 12:32:30,826] jepsen worker 66 - jepsen.util 7626 :info :write [22 17] :RequestCanceledException INFO [2019-12-17 12:32:30,827] jepsen worker 66 - jepsen.util 7716 :invoke :write [22 38] INFO [2019-12-17 12:32:30,853] jepsen worker 63 - jepsen.util 5013 :fail :cas [21 [9 25]] :ValueNotSwapFrom INFO [2019-12-17 12:32:30,853] jepsen worker 63 - jepsen.util 5013 :invoke :write [21 39] INFO [2019-12-17 12:32:30,859] jepsen worker 28 - jepsen.util 4798 :ok :read [9 44] INFO [2019-12-17 12:32:30,859] jepsen worker 28 - jepsen.util 4798 :invoke :write [9 28] INFO [2019-12-17 12:32:30,884] jepsen worker 31 - jepsen.util 5431 :fail :cas [10 [7 31]] :ValueNotSwapFrom INFO [2019-12-17 12:32:30,884] jepsen worker 31 - jepsen.util 5431 :invoke :write [10 8] INFO [2019-12-17 12:32:30,911] jepsen worker 60 - jepsen.util 9060 :info :cas [20 [13 17]] :RequestCanceledException INFO [2019-12-17 12:32:30,913] jepsen worker 60 - jepsen.util 9150 :invoke :read [20 nil] INFO [2019-12-17 12:32:30,919] jepsen worker 18 - jepsen.util 5868 :ok :write [6 18] INFO [2019-12-17 12:32:30,919] jepsen worker 18 - jepsen.util 5868 :invoke :read [6 nil] INFO [2019-12-17 12:32:30,926] jepsen worker 85 - jepsen.util 8995 :info :cas [28 [33 31]] :RequestCanceledException INFO [2019-12-17 12:32:30,926] jepsen worker 43 - jepsen.util 7693 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:32:30,926] jepsen worker 32 - jepsen.util 6062 :info :cas [10 [36 37]] :RequestCanceledException INFO [2019-12-17 12:32:30,926] jepsen worker 43 - jepsen.util 7693 :invoke :read [14 nil] INFO [2019-12-17 12:32:30,927] jepsen worker 32 - jepsen.util 6152 :invoke :read [10 nil] INFO [2019-12-17 12:32:30,927] jepsen worker 85 - jepsen.util 9085 :invoke :write [28 49] INFO [2019-12-17 12:32:30,938] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":34953,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:32:30,938] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":34953,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:32:30,938] jepsen worker 75 - jepsen.util 7815 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:32:30,938] jepsen worker 75 - jepsen.util 7815 :invoke :cas [25 [25 45]] INFO [2019-12-17 12:32:30,939] jepsen worker 27 - jepsen.util 5967 :ok :read [9 44] INFO [2019-12-17 12:32:30,939] jepsen worker 27 - jepsen.util 5967 :invoke :cas [9 [40 6]] INFO [2019-12-17 12:32:30,953] jepsen worker 18 - jepsen.util 5868 :ok :read [6 18] INFO [2019-12-17 12:32:30,953] jepsen worker 18 - jepsen.util 5868 :invoke :write [6 20] INFO [2019-12-17 12:32:30,953] jepsen worker 46 - jepsen.util 6076 :ok :write [15 23] INFO [2019-12-17 12:32:30,953] jepsen worker 46 - jepsen.util 6076 :invoke :write [15 43] INFO [2019-12-17 12:32:30,958] jepsen worker 44 - jepsen.util 8864 :info :cas [14 [49 28]] :RequestCanceledException INFO [2019-12-17 12:32:30,960] jepsen worker 44 - jepsen.util 8954 :invoke :write [14 15] INFO [2019-12-17 12:32:30,973] jepsen worker 74 - jepsen.util 6194 :ok :write [24 9] INFO [2019-12-17 12:32:30,973] jepsen worker 63 - jepsen.util 5013 :ok :write [21 39] INFO [2019-12-17 12:32:30,973] jepsen worker 74 - jepsen.util 6194 :invoke :write [24 39] INFO [2019-12-17 12:32:30,973] jepsen worker 63 - jepsen.util 5013 :invoke :cas [21 [16 8]] INFO [2019-12-17 12:32:30,986] jepsen worker 27 - jepsen.util 5967 :fail :cas [9 [40 6]] :ValueNotSwapFrom INFO [2019-12-17 12:32:30,986] jepsen worker 27 - jepsen.util 5967 :invoke :cas [9 [27 2]] INFO [2019-12-17 12:32:31,011] jepsen worker 8 - jepsen.util 7658 :info :write [2 9] :RequestCanceledException INFO [2019-12-17 12:32:31,011] jepsen worker 82 - jepsen.util 8722 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:32:31,011] jepsen worker 82 - jepsen.util 8722 :invoke :write [27 23] INFO [2019-12-17 12:32:31,012] jepsen worker 8 - jepsen.util 7748 :invoke :cas [2 [37 38]] INFO [2019-12-17 12:32:31,023] jepsen worker 28 - jepsen.util 4798 :ok :write [9 28] INFO [2019-12-17 12:32:31,025] jepsen worker 53 - jepsen.util 5723 :fail :cas [17 [1 4]] :ValueNotSwapFrom INFO [2019-12-17 12:32:31,025] jepsen worker 28 - jepsen.util 4798 :invoke :read [9 nil] INFO [2019-12-17 12:32:31,025] jepsen worker 53 - jepsen.util 5723 :invoke :read [17 nil] INFO [2019-12-17 12:32:31,026] jepsen worker 16 - jepsen.util 9106 :info :cas [5 [24 8]] :RequestCanceledException INFO [2019-12-17 12:32:31,026] jepsen worker 57 - jepsen.util 7797 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:32:31,027] jepsen worker 57 - jepsen.util 7797 :invoke :read [19 nil] INFO [2019-12-17 12:32:31,027] jepsen worker 16 - jepsen.util 9196 :invoke :write [5 35] INFO [2019-12-17 12:32:31,041] jepsen worker 48 - jepsen.util 7428 :info :write [16 37] :RequestCanceledException INFO [2019-12-17 12:32:31,041] jepsen worker 76 - jepsen.util 8356 :info :cas [25 [28 5]] :RequestCanceledException INFO [2019-12-17 12:32:31,042] jepsen worker 76 - jepsen.util 8446 :invoke :write [25 16] INFO [2019-12-17 12:32:31,042] jepsen worker 48 - jepsen.util 7518 :invoke :write [16 42] INFO [2019-12-17 12:32:31,044] jepsen worker 63 - jepsen.util 5013 :fail :cas [21 [16 8]] :ValueNotSwapFrom INFO [2019-12-17 12:32:31,045] jepsen worker 63 - jepsen.util 5013 :invoke :cas [21 [37 19]] INFO [2019-12-17 12:32:31,045] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":34973,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:32:31,045] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":34973,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:32:31,045] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":34976,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:32:31,045] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":34976,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:32:31,045] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":34977,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:32:31,046] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":34977,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:32:31,054] jepsen worker 6 - jepsen.util 8736 :info :write [2 30] :RequestCanceledException INFO [2019-12-17 12:32:31,056] jepsen worker 6 - jepsen.util 8826 :invoke :write [2 12] INFO [2019-12-17 12:32:31,073] jepsen worker 32 - jepsen.util 6152 :ok :read [10 24] INFO [2019-12-17 12:32:31,074] jepsen worker 32 - jepsen.util 6152 :invoke :read [10 nil] INFO [2019-12-17 12:32:31,090] jepsen worker 3 - jepsen.util 6033 :fail :cas [1 [7 11]] :ValueNotSwapFrom INFO [2019-12-17 12:32:31,090] jepsen worker 3 - jepsen.util 6033 :invoke :cas [1 [29 20]] INFO [2019-12-17 12:32:31,092] jepsen worker 63 - jepsen.util 5013 :fail :cas [21 [37 19]] :ValueNotSwapFrom INFO [2019-12-17 12:32:31,092] jepsen worker 63 - jepsen.util 5013 :invoke :write [21 17] INFO [2019-12-17 12:32:31,093] jepsen worker 72 - jepsen.util 6012 :info :write [24 46] :RequestCanceledException INFO [2019-12-17 12:32:31,093] jepsen worker 34 - jepsen.util 7594 :info :write [11 38] :RequestCanceledException INFO [2019-12-17 12:32:31,094] jepsen worker 34 - jepsen.util 7684 :invoke :cas [11 [49 39]] INFO [2019-12-17 12:32:31,096] jepsen worker 72 - jepsen.util 6102 :invoke :cas [24 [40 5]] INFO [2019-12-17 12:32:31,105] jepsen worker 28 - jepsen.util 4798 :ok :read [9 28] INFO [2019-12-17 12:32:31,108] jepsen worker 28 - jepsen.util 4798 :invoke :read [9 nil] INFO [2019-12-17 12:32:31,111] jepsen worker 41 - jepsen.util 8951 :info :write [13 1] :RequestCanceledException INFO [2019-12-17 12:32:31,112] jepsen worker 41 - jepsen.util 9041 :invoke :read [13 nil] INFO [2019-12-17 12:32:31,126] jepsen worker 33 - jepsen.util 8133 :info :write [11 41] :RequestCanceledException INFO [2019-12-17 12:32:31,127] jepsen worker 11 - jepsen.util 6221 :info :write [3 5] :RequestCanceledException INFO [2019-12-17 12:32:31,127] jepsen worker 64 - jepsen.util 4654 :info :cas [21 [24 39]] :RequestCanceledException INFO [2019-12-17 12:32:31,127] jepsen worker 33 - jepsen.util 8223 :invoke :write [11 43] INFO [2019-12-17 12:32:31,128] jepsen worker 64 - jepsen.util 4744 :invoke :read [21 nil] INFO [2019-12-17 12:32:31,129] jepsen worker 11 - jepsen.util 6311 :invoke :read [3 nil] INFO [2019-12-17 12:32:31,135] jepsen worker 32 - jepsen.util 6152 :ok :read [10 24] INFO [2019-12-17 12:32:31,135] jepsen worker 32 - jepsen.util 6152 :invoke :write [10 42] INFO [2019-12-17 12:32:31,140] jepsen worker 58 - jepsen.util 8608 :info :cas [19 [28 34]] :RequestCanceledException INFO [2019-12-17 12:32:31,140] jepsen worker 20 - jepsen.util 5960 :info :cas [6 [37 37]] :RequestCanceledException INFO [2019-12-17 12:32:31,141] jepsen worker 20 - jepsen.util 6050 :invoke :read [6 nil] INFO [2019-12-17 12:32:31,141] jepsen worker 58 - jepsen.util 8698 :invoke :cas [19 [49 38]] INFO [2019-12-17 12:32:31,155] jepsen worker 11 - jepsen.util 6311 :ok :read [3 24] INFO [2019-12-17 12:32:31,155] jepsen worker 11 - jepsen.util 6311 :invoke :write [3 49] INFO [2019-12-17 12:32:31,158] jepsen worker 10 - jepsen.util 5950 :info :write [3 48] :RequestCanceledException INFO [2019-12-17 12:32:31,163] jepsen worker 10 - jepsen.util 6040 :invoke :read [3 nil] INFO [2019-12-17 12:32:31,190] jepsen worker 28 - jepsen.util 4798 :ok :read [9 28] INFO [2019-12-17 12:32:31,190] jepsen worker 28 - jepsen.util 4798 :invoke :read [9 nil] INFO [2019-12-17 12:32:31,193] jepsen worker 12 - jepsen.util 6312 :info :write [4 1] :RequestCanceledException INFO [2019-12-17 12:32:31,194] jepsen worker 12 - jepsen.util 6402 :invoke :cas [4 [1 10]] INFO [2019-12-17 12:32:31,208] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":35004,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:32:31,208] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":35004,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:32:31,208] jepsen worker 38 - jepsen.util 5978 :info :write [12 33] :RequestCanceledException INFO [2019-12-17 12:32:31,208] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":35005,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:32:31,209] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":35005,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:32:31,209] jepsen worker 38 - jepsen.util 6068 :invoke :write [12 24] INFO [2019-12-17 12:32:31,211] jepsen worker 64 - jepsen.util 4744 :ok :read [21 39] INFO [2019-12-17 12:32:31,211] jepsen worker 64 - jepsen.util 4744 :invoke :write [21 29] INFO [2019-12-17 12:32:31,223] jepsen worker 63 - jepsen.util 5013 :ok :write [21 17] INFO [2019-12-17 12:32:31,223] jepsen worker 63 - jepsen.util 5013 :invoke :cas [21 [11 16]] INFO [2019-12-17 12:32:31,226] jepsen worker 45 - jepsen.util 5985 :info :write [15 38] :RequestCanceledException INFO [2019-12-17 12:32:31,227] jepsen worker 45 - jepsen.util 6075 :invoke :read [15 nil] INFO [2019-12-17 12:32:31,241] jepsen worker 0 - jepsen.util 8460 :info :write [0 48] :RequestCanceledException INFO [2019-12-17 12:32:31,241] jepsen worker 20 - jepsen.util 6050 :ok :read [6 18] INFO [2019-12-17 12:32:31,241] jepsen worker 20 - jepsen.util 6050 :invoke :read [6 nil] INFO [2019-12-17 12:32:31,242] jepsen worker 0 - jepsen.util 8550 :invoke :write [0 35] INFO [2019-12-17 12:32:31,248] jepsen worker 37 - jepsen.util 5797 :fail :cas [12 [38 0]] :ValueNotSwapFrom INFO [2019-12-17 12:32:31,248] jepsen worker 37 - jepsen.util 5797 :invoke :read [12 nil] INFO [2019-12-17 12:32:31,257] jepsen worker 63 - jepsen.util 5013 :fail :cas [21 [11 16]] :ValueNotSwapFrom INFO [2019-12-17 12:32:31,257] jepsen worker 63 - jepsen.util 5013 :invoke :cas [21 [39 35]] INFO [2019-12-17 12:32:31,257] jepsen worker 29 - jepsen.util 4709 :fail :read [9 nil] :RequestTimeoutException INFO [2019-12-17 12:32:31,257] jepsen worker 62 - jepsen.util 7802 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:32:31,258] jepsen worker 29 - jepsen.util 4709 :invoke :read [9 nil] INFO [2019-12-17 12:32:31,258] jepsen worker 62 - jepsen.util 7802 :invoke :cas [20 [27 22]] INFO [2019-12-17 12:32:31,264] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":35012,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:32:31,264] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":35012,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:32:31,264] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":35015,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:32:31,264] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":35015,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:32:31,264] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":35016,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:32:31,264] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":35016,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:32:31,283] jepsen worker 20 - jepsen.util 6050 :ok :read [6 18] INFO [2019-12-17 12:32:31,283] jepsen worker 20 - jepsen.util 6050 :invoke :cas [6 [27 2]] INFO [2019-12-17 12:32:31,293] jepsen worker 63 - jepsen.util 5013 :fail :cas [21 [39 35]] :ValueNotSwapFrom INFO [2019-12-17 12:32:31,293] jepsen worker 84 - jepsen.util 7824 :info :cas [28 [14 22]] :RequestCanceledException INFO [2019-12-17 12:32:31,294] jepsen worker 63 - jepsen.util 5013 :invoke :read [21 nil] INFO [2019-12-17 12:32:31,294] jepsen worker 84 - jepsen.util 7914 :invoke :read [28 nil] INFO [2019-12-17 12:32:31,306] jepsen worker 20 - jepsen.util 6050 :fail :cas [6 [27 2]] :ValueNotSwapFrom INFO [2019-12-17 12:32:31,306] jepsen worker 20 - jepsen.util 6050 :invoke :read [6 nil] INFO [2019-12-17 12:32:31,311] jepsen worker 86 - jepsen.util 8546 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:32:31,311] jepsen worker 81 - jepsen.util 7821 :info :cas [27 [19 46]] :RequestCanceledException INFO [2019-12-17 12:32:31,311] jepsen worker 86 - jepsen.util 8546 :invoke :write [28 8] INFO [2019-12-17 12:32:31,312] jepsen worker 81 - jepsen.util 7911 :invoke :cas [27 [18 31]] INFO [2019-12-17 12:32:31,321] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:32:31,321] jepsen nemesis - couchbase.util Rebalance check count: 265 INFO [2019-12-17 12:32:31,321] jepsen nemesis - couchbase.util Rebalance stuck count: 246 INFO [2019-12-17 12:32:31,325] jepsen worker 63 - jepsen.util 5013 :ok :read [21 17] INFO [2019-12-17 12:32:31,325] jepsen worker 63 - jepsen.util 5013 :invoke :write [21 1] INFO [2019-12-17 12:32:31,334] jepsen worker 20 - jepsen.util 6050 :ok :read [6 18] INFO [2019-12-17 12:32:31,334] jepsen worker 20 - jepsen.util 6050 :invoke :read [6 nil] INFO [2019-12-17 12:32:31,341] jepsen worker 21 - jepsen.util 7671 :info :cas [7 [33 49]] :RequestCanceledException INFO [2019-12-17 12:32:31,343] jepsen worker 21 - jepsen.util 7761 :invoke :cas [7 [22 20]] INFO [2019-12-17 12:32:31,358] jepsen worker 69 - jepsen.util 6189 :ok :read [23 33] INFO [2019-12-17 12:32:31,358] jepsen worker 69 - jepsen.util 6189 :invoke :read [23 nil] INFO [2019-12-17 12:32:31,358] jepsen worker 83 - jepsen.util 8543 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:32:31,358] jepsen worker 83 - jepsen.util 8543 :invoke :cas [27 [15 1]] INFO [2019-12-17 12:32:31,374] jepsen worker 63 - jepsen.util 5013 :ok :write [21 1] INFO [2019-12-17 12:32:31,375] jepsen worker 63 - jepsen.util 5013 :invoke :read [21 nil] INFO [2019-12-17 12:32:31,393] jepsen worker 61 - jepsen.util 9241 :info :cas [20 [33 8]] :RequestCanceledException INFO [2019-12-17 12:32:31,394] jepsen worker 1 - jepsen.util 7831 :info :cas [0 [48 41]] :RequestCanceledException INFO [2019-12-17 12:32:31,394] jepsen worker 63 - jepsen.util 5013 :ok :read [21 1] INFO [2019-12-17 12:32:31,395] jepsen worker 63 - jepsen.util 5013 :invoke :write [21 49] INFO [2019-12-17 12:32:31,395] jepsen worker 61 - jepsen.util 9331 :invoke :write [20 1] INFO [2019-12-17 12:32:31,395] jepsen worker 1 - jepsen.util 7921 :invoke :write [0 3] INFO [2019-12-17 12:32:31,398] jepsen worker 56 - jepsen.util 6536 :info :write [18 15] :RequestCanceledException INFO [2019-12-17 12:32:31,399] jepsen worker 56 - jepsen.util 6626 :invoke :read [18 nil] INFO [2019-12-17 12:32:31,426] jepsen worker 39 - jepsen.util 7599 :info :cas [13 [45 19]] :RequestCanceledException INFO [2019-12-17 12:32:31,426] jepsen worker 89 - jepsen.util 5849 :info :write [29 25] :RequestCanceledException INFO [2019-12-17 12:32:31,427] jepsen worker 89 - jepsen.util 5939 :invoke :cas [29 [48 40]] INFO [2019-12-17 12:32:31,428] jepsen worker 39 - jepsen.util 7689 :invoke :cas [13 [42 2]] INFO [2019-12-17 12:32:31,440] jepsen worker 77 - jepsen.util 8447 :info :cas [25 [27 29]] :RequestCanceledException INFO [2019-12-17 12:32:31,441] jepsen worker 77 - jepsen.util 8537 :invoke :write [25 32] INFO [2019-12-17 12:32:31,457] jepsen worker 63 - jepsen.util 5013 :ok :write [21 49] INFO [2019-12-17 12:32:31,457] jepsen worker 63 - jepsen.util 5013 :invoke :cas [21 [21 31]] INFO [2019-12-17 12:32:31,472] jepsen worker 63 - jepsen.util 5013 :fail :cas [21 [21 31]] :ValueNotSwapFrom INFO [2019-12-17 12:32:31,473] jepsen worker 63 - jepsen.util 5013 :invoke :write [21 34] INFO [2019-12-17 12:32:31,483] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":35046,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:32:31,483] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":35046,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:32:31,483] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":35049,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:32:31,483] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":35049,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:32:31,493] jepsen worker 40 - jepsen.util 8500 :info :write [13 23] :RequestCanceledException INFO [2019-12-17 12:32:31,493] jepsen worker 3 - jepsen.util 6033 :fail :cas [1 [29 20]] :ValueNotSwapFrom INFO [2019-12-17 12:32:31,496] jepsen worker 3 - jepsen.util 6033 :invoke :write [1 6] INFO [2019-12-17 12:32:31,497] jepsen worker 40 - jepsen.util 8590 :invoke :cas [13 [42 33]] INFO [2019-12-17 12:32:31,511] jepsen worker 4 - jepsen.util 5584 :fail :cas [1 [38 7]] :ValueNotSwapFrom INFO [2019-12-17 12:32:31,512] jepsen worker 54 - jepsen.util 6534 :info :write [18 44] :RequestCanceledException INFO [2019-12-17 12:32:31,512] jepsen worker 22 - jepsen.util 8032 :info :write [7 27] :RequestCanceledException INFO [2019-12-17 12:32:31,512] jepsen worker 4 - jepsen.util 5584 :invoke :write [1 19] INFO [2019-12-17 12:32:31,513] jepsen worker 22 - jepsen.util 8122 :invoke :cas [7 [3 14]] INFO [2019-12-17 12:32:31,513] jepsen worker 54 - jepsen.util 6624 :invoke :cas [18 [1 9]] INFO [2019-12-17 12:32:31,526] jepsen worker 63 - jepsen.util 5013 :ok :write [21 34] INFO [2019-12-17 12:32:31,526] jepsen worker 63 - jepsen.util 5013 :invoke :write [21 8] INFO [2019-12-17 12:32:31,526] jepsen worker 25 - jepsen.util 8755 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:32:31,526] jepsen worker 25 - jepsen.util 8755 :invoke :read [8 nil] INFO [2019-12-17 12:32:31,527] jepsen worker 7 - jepsen.util 8467 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:32:31,527] jepsen worker 7 - jepsen.util 8467 :invoke :write [2 6] INFO [2019-12-17 12:32:31,538] jepsen worker 47 - jepsen.util 6887 :ok :write [15 7] INFO [2019-12-17 12:32:31,538] jepsen worker 47 - jepsen.util 6887 :invoke :read [15 nil] INFO [2019-12-17 12:32:31,540] jepsen worker 18 - jepsen.util 5868 :ok :write [6 20] INFO [2019-12-17 12:32:31,540] jepsen worker 18 - jepsen.util 5868 :invoke :write [6 26] INFO [2019-12-17 12:32:31,540] jepsen worker 15 - jepsen.util 7125 :info :cas [5 [14 27]] :RequestCanceledException INFO [2019-12-17 12:32:31,541] jepsen worker 15 - jepsen.util 7215 :invoke :cas [5 [34 44]] INFO [2019-12-17 12:32:31,550] jepsen worker 32 - jepsen.util 6152 :ok :write [10 42] INFO [2019-12-17 12:32:31,551] jepsen worker 32 - jepsen.util 6152 :invoke :write [10 47] INFO [2019-12-17 12:32:31,567] jepsen worker 87 - jepsen.util 5577 :fail :cas [29 [32 11]] :ValueNotSwapFrom INFO [2019-12-17 12:32:31,567] jepsen worker 87 - jepsen.util 5577 :invoke :cas [29 [27 3]] INFO [2019-12-17 12:32:31,571] jepsen worker 27 - jepsen.util 5967 :fail :cas [9 [27 2]] :ValueNotSwapFrom INFO [2019-12-17 12:32:31,572] jepsen worker 27 - jepsen.util 5967 :invoke :read [9 nil] INFO [2019-12-17 12:32:31,574] jepsen worker 72 - jepsen.util 6102 :fail :cas [24 [40 5]] :ValueNotSwapFrom INFO [2019-12-17 12:32:31,574] jepsen worker 72 - jepsen.util 6102 :invoke :write [24 41] INFO [2019-12-17 12:32:31,591] jepsen worker 64 - jepsen.util 4744 :ok :write [21 29] INFO [2019-12-17 12:32:31,591] jepsen worker 64 - jepsen.util 4744 :invoke :cas [21 [8 35]] INFO [2019-12-17 12:32:31,593] jepsen worker 17 - jepsen.util 8117 :info :cas [5 [33 26]] :RequestCanceledException INFO [2019-12-17 12:32:31,594] jepsen worker 71 - jepsen.util 6731 :info :write [23 8] :RequestCanceledException INFO [2019-12-17 12:32:31,595] jepsen worker 71 - jepsen.util 6821 :invoke :write [23 35] INFO [2019-12-17 12:32:31,598] jepsen worker 17 - jepsen.util 8207 :invoke :cas [5 [1 23]] INFO [2019-12-17 12:32:31,627] jepsen worker 26 - jepsen.util 8486 :info :write [8 15] :RequestCanceledException INFO [2019-12-17 12:32:31,627] jepsen worker 80 - jepsen.util 6110 :info :write [26 29] :RequestCanceledException INFO [2019-12-17 12:32:31,628] jepsen worker 26 - jepsen.util 8576 :invoke :write [8 5] INFO [2019-12-17 12:32:31,628] jepsen worker 80 - jepsen.util 6200 :invoke :cas [26 [40 26]] INFO [2019-12-17 12:32:31,642] jepsen worker 37 - jepsen.util 5797 :ok :read [12 21] INFO [2019-12-17 12:32:31,642] jepsen worker 37 - jepsen.util 5797 :invoke :write [12 23] INFO [2019-12-17 12:32:31,658] jepsen worker 67 - jepsen.util 8347 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:32:31,658] jepsen worker 67 - jepsen.util 8347 :invoke :cas [22 [1 48]] INFO [2019-12-17 12:32:31,659] jepsen worker 11 - jepsen.util 6311 :ok :write [3 49] INFO [2019-12-17 12:32:31,659] jepsen worker 11 - jepsen.util 6311 :invoke :read [3 nil] INFO [2019-12-17 12:32:31,661] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":35075,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:32:31,662] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":35075,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:32:31,666] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":35079,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:32:31,666] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":35079,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:32:31,676] jepsen worker 45 - jepsen.util 6075 :ok :read [15 7] INFO [2019-12-17 12:32:31,676] jepsen worker 45 - jepsen.util 6075 :invoke :read [15 nil] INFO [2019-12-17 12:32:31,687] jepsen worker 29 - jepsen.util 4709 :ok :read [9 28] INFO [2019-12-17 12:32:31,687] jepsen worker 29 - jepsen.util 4709 :invoke :write [9 19] INFO [2019-12-17 12:32:31,694] jepsen worker 23 - jepsen.util 8213 :info :cas [7 [23 5]] :RequestCanceledException INFO [2019-12-17 12:32:31,695] jepsen worker 23 - jepsen.util 8303 :invoke :cas [7 [10 29]] INFO [2019-12-17 12:32:31,702] jepsen worker 11 - jepsen.util 6311 :ok :read [3 49] INFO [2019-12-17 12:32:31,702] jepsen worker 11 - jepsen.util 6311 :invoke :read [3 nil] INFO [2019-12-17 12:32:31,711] jepsen worker 45 - jepsen.util 6075 :ok :read [15 7] INFO [2019-12-17 12:32:31,711] jepsen worker 45 - jepsen.util 6075 :invoke :read [15 nil] INFO [2019-12-17 12:32:31,711] jepsen worker 42 - jepsen.util 8772 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:32:31,712] jepsen worker 42 - jepsen.util 8772 :invoke :cas [14 [5 22]] INFO [2019-12-17 12:32:31,726] jepsen worker 24 - jepsen.util 7404 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:32:31,726] jepsen worker 14 - jepsen.util 6224 :info :write [4 10] :RequestCanceledException INFO [2019-12-17 12:32:31,732] jepsen worker 24 - jepsen.util 7404 :invoke :read [8 nil] INFO [2019-12-17 12:32:31,734] jepsen worker 14 - jepsen.util 6314 :invoke :cas [4 [44 44]] INFO [2019-12-17 12:32:31,737] jepsen worker 89 - jepsen.util 5939 :fail :cas [29 [48 40]] :ValueNotSwapFrom INFO [2019-12-17 12:32:31,739] jepsen worker 89 - jepsen.util 5939 :invoke :read [29 nil] INFO [2019-12-17 12:32:31,739] jepsen worker 53 - jepsen.util 5723 :ok :read [17 16] INFO [2019-12-17 12:32:31,741] jepsen worker 80 - jepsen.util 6200 :fail :cas [26 [40 26]] :ValueNotSwapFrom INFO [2019-12-17 12:32:31,741] jepsen worker 35 - jepsen.util 8225 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:32:31,741] jepsen worker 59 - jepsen.util 8249 :info :write [19 20] :RequestCanceledException INFO [2019-12-17 12:32:31,742] jepsen worker 80 - jepsen.util 6200 :invoke :write [26 25] INFO [2019-12-17 12:32:31,742] jepsen worker 53 - jepsen.util 5723 :invoke :cas [17 [7 34]] INFO [2019-12-17 12:32:31,744] jepsen worker 35 - jepsen.util 8225 :invoke :write [11 37] INFO [2019-12-17 12:32:31,745] jepsen worker 59 - jepsen.util 8339 :invoke :cas [19 [4 6]] INFO [2019-12-17 12:32:31,756] jepsen worker 29 - jepsen.util 4709 :ok :write [9 19] INFO [2019-12-17 12:32:31,757] jepsen worker 49 - jepsen.util 7339 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:32:31,759] jepsen worker 29 - jepsen.util 4709 :invoke :write [9 7] INFO [2019-12-17 12:32:31,761] jepsen worker 49 - jepsen.util 7339 :invoke :read [16 nil] INFO [2019-12-17 12:32:31,791] jepsen worker 56 - jepsen.util 6626 :ok :read [18 30] INFO [2019-12-17 12:32:31,792] jepsen worker 56 - jepsen.util 6626 :invoke :read [18 nil] INFO [2019-12-17 12:32:31,809] jepsen worker 54 - jepsen.util 6624 :fail :cas [18 [1 9]] :ValueNotSwapFrom INFO [2019-12-17 12:32:31,810] jepsen worker 54 - jepsen.util 6624 :invoke :cas [18 [39 11]] INFO [2019-12-17 12:32:31,811] jepsen worker 29 - jepsen.util 4709 :ok :write [9 7] INFO [2019-12-17 12:32:31,811] jepsen worker 29 - jepsen.util 4709 :invoke :cas [9 [7 21]] INFO [2019-12-17 12:32:31,827] jepsen worker 9 - jepsen.util 6489 :info :write [3 26] :RequestCanceledException INFO [2019-12-17 12:32:31,827] jepsen worker 52 - jepsen.util 6442 :info :write [17 16] :RequestCanceledException INFO [2019-12-17 12:32:31,829] jepsen worker 9 - jepsen.util 6579 :invoke :cas [3 [2 3]] INFO [2019-12-17 12:32:31,829] jepsen worker 52 - jepsen.util 6532 :invoke :read [17 nil] INFO [2019-12-17 12:32:31,832] jepsen worker 32 - jepsen.util 6152 :ok :write [10 47] INFO [2019-12-17 12:32:31,832] jepsen worker 74 - jepsen.util 6194 :ok :write [24 39] INFO [2019-12-17 12:32:31,832] jepsen worker 32 - jepsen.util 6152 :invoke :write [10 21] INFO [2019-12-17 12:32:31,832] jepsen worker 74 - jepsen.util 6194 :invoke :read [24 nil] INFO [2019-12-17 12:32:31,832] jepsen worker 12 - jepsen.util 6402 :fail :cas [4 [1 10]] :ValueNotSwapFrom INFO [2019-12-17 12:32:31,832] jepsen worker 12 - jepsen.util 6402 :invoke :cas [4 [21 41]] INFO [2019-12-17 12:32:31,833] jepsen worker 18 - jepsen.util 5868 :ok :write [6 26] INFO [2019-12-17 12:32:31,833] jepsen worker 18 - jepsen.util 5868 :invoke :read [6 nil] INFO [2019-12-17 12:32:31,890] jepsen worker 9 - jepsen.util 6579 :fail :cas [3 [2 3]] :ValueNotSwapFrom INFO [2019-12-17 12:32:31,891] jepsen worker 9 - jepsen.util 6579 :invoke :cas [3 [11 17]] INFO [2019-12-17 12:32:31,892] jepsen worker 27 - jepsen.util 5967 :ok :read [9 7] INFO [2019-12-17 12:32:31,893] jepsen worker 27 - jepsen.util 5967 :invoke :write [9 36] INFO [2019-12-17 12:32:31,898] jepsen worker 47 - jepsen.util 6887 :ok :read [15 7] INFO [2019-12-17 12:32:31,898] jepsen worker 47 - jepsen.util 6887 :invoke :write [15 7] INFO [2019-12-17 12:32:31,911] jepsen worker 2 - jepsen.util 7922 :info :write [0 27] :RequestCanceledException INFO [2019-12-17 12:32:31,916] jepsen worker 2 - jepsen.util 8012 :invoke :read [0 nil] INFO [2019-12-17 12:32:31,921] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":35110,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:32:31,921] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":35110,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:32:31,922] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":35112,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:32:31,922] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":35112,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:32:31,922] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":35113,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:32:31,922] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":35113,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:32:31,927] jepsen worker 88 - jepsen.util 6298 :info :write [29 10] :RequestCanceledException INFO [2019-12-17 12:32:31,927] jepsen worker 30 - jepsen.util 6060 :info :cas [10 [4 32]] :RequestCanceledException INFO [2019-12-17 12:32:31,928] jepsen worker 88 - jepsen.util 6388 :invoke :cas [29 [40 4]] INFO [2019-12-17 12:32:31,928] jepsen worker 30 - jepsen.util 6150 :invoke :read [10 nil] INFO [2019-12-17 12:32:31,963] jepsen worker 87 - jepsen.util 5577 :fail :cas [29 [27 3]] :ValueNotSwapFrom INFO [2019-12-17 12:32:31,964] jepsen worker 87 - jepsen.util 5577 :invoke :cas [29 [45 24]] INFO [2019-12-17 12:32:31,986] jepsen worker 9 - jepsen.util 6579 :fail :cas [3 [11 17]] :ValueNotSwapFrom INFO [2019-12-17 12:32:31,987] jepsen worker 9 - jepsen.util 6579 :invoke :read [3 nil] INFO [2019-12-17 12:32:31,988] jepsen worker 74 - jepsen.util 6194 :ok :read [24 39] INFO [2019-12-17 12:32:31,988] jepsen worker 74 - jepsen.util 6194 :invoke :write [24 3] INFO [2019-12-17 12:32:32,016] jepsen worker 9 - jepsen.util 6579 :ok :read [3 49] INFO [2019-12-17 12:32:32,017] jepsen worker 9 - jepsen.util 6579 :invoke :read [3 nil] INFO [2019-12-17 12:32:32,031] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":35123,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:32:32,031] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":35123,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:32:32,031] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":35124,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:32:32,031] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":35124,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:32:32,031] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":35125,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:32:32,031] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":35125,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:32:32,038] jepsen worker 9 - jepsen.util 6579 :ok :read [3 49] INFO [2019-12-17 12:32:32,038] jepsen worker 29 - jepsen.util 4709 :ok :cas [9 [7 21]] INFO [2019-12-17 12:32:32,038] jepsen worker 9 - jepsen.util 6579 :invoke :write [3 5] INFO [2019-12-17 12:32:32,038] jepsen worker 29 - jepsen.util 4709 :invoke :read [9 nil] INFO [2019-12-17 12:32:32,040] jepsen worker 19 - jepsen.util 6049 :fail :read [6 nil] :RequestTimeoutException INFO [2019-12-17 12:32:32,040] jepsen worker 19 - jepsen.util 6049 :invoke :write [6 2] INFO [2019-12-17 12:32:32,057] jepsen worker 69 - jepsen.util 6189 :ok :read [23 33] INFO [2019-12-17 12:32:32,057] jepsen worker 69 - jepsen.util 6189 :invoke :cas [23 [23 39]] INFO [2019-12-17 12:32:32,059] jepsen worker 29 - jepsen.util 4709 :ok :read [9 21] INFO [2019-12-17 12:32:32,060] jepsen worker 29 - jepsen.util 4709 :invoke :cas [9 [24 41]] INFO [2019-12-17 12:32:32,082] jepsen worker 29 - jepsen.util 4709 :fail :cas [9 [24 41]] :ValueNotSwapFrom INFO [2019-12-17 12:32:32,083] jepsen worker 29 - jepsen.util 4709 :invoke :write [9 28] INFO [2019-12-17 12:32:32,111] jepsen worker 50 - jepsen.util 8510 :info :write [16 29] :RequestCanceledException INFO [2019-12-17 12:32:32,112] jepsen worker 50 - jepsen.util 8600 :invoke :cas [16 [29 37]] INFO [2019-12-17 12:32:32,122] jepsen worker 56 - jepsen.util 6626 :ok :read [18 30] INFO [2019-12-17 12:32:32,122] jepsen worker 56 - jepsen.util 6626 :invoke :write [18 19] INFO [2019-12-17 12:32:32,127] jepsen worker 65 - jepsen.util 5195 :fail :read [21 nil] :RequestTimeoutException INFO [2019-12-17 12:32:32,127] jepsen worker 65 - jepsen.util 5195 :invoke :cas [21 [0 3]] INFO [2019-12-17 12:32:32,143] jepsen worker 54 - jepsen.util 6624 :fail :cas [18 [39 11]] :ValueNotSwapFrom INFO [2019-12-17 12:32:32,143] jepsen worker 54 - jepsen.util 6624 :invoke :read [18 nil] INFO [2019-12-17 12:32:32,152] jepsen worker 65 - jepsen.util 5195 :fail :cas [21 [0 3]] :ValueNotSwapFrom INFO [2019-12-17 12:32:32,152] jepsen worker 65 - jepsen.util 5195 :invoke :write [21 32] INFO [2019-12-17 12:32:32,173] jepsen worker 72 - jepsen.util 6102 :ok :write [24 41] INFO [2019-12-17 12:32:32,174] jepsen worker 72 - jepsen.util 6102 :invoke :read [24 nil] INFO [2019-12-17 12:32:32,186] jepsen worker 29 - jepsen.util 4709 :ok :write [9 28] INFO [2019-12-17 12:32:32,187] jepsen worker 29 - jepsen.util 4709 :invoke :cas [9 [3 45]] INFO [2019-12-17 12:32:32,193] jepsen worker 18 - jepsen.util 5868 :ok :read [6 26] INFO [2019-12-17 12:32:32,194] jepsen worker 18 - jepsen.util 5868 :invoke :cas [6 [37 33]] INFO [2019-12-17 12:32:32,194] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":35143,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:32:32,194] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":35143,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:32:32,194] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":35146,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:32:32,194] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":35146,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:32:32,196] jepsen worker 32 - jepsen.util 6152 :ok :write [10 21] INFO [2019-12-17 12:32:32,196] jepsen worker 32 - jepsen.util 6152 :invoke :cas [10 [31 29]] INFO [2019-12-17 12:32:32,215] jepsen worker 29 - jepsen.util 4709 :fail :cas [9 [3 45]] :ValueNotSwapFrom INFO [2019-12-17 12:32:32,215] jepsen worker 29 - jepsen.util 4709 :invoke :write [9 0] INFO [2019-12-17 12:32:32,236] jepsen worker 14 - jepsen.util 6314 :fail :cas [4 [44 44]] :ValueNotSwapFrom INFO [2019-12-17 12:32:32,236] jepsen worker 55 - jepsen.util 7075 :info :write [18 20] :RequestCanceledException INFO [2019-12-17 12:32:32,236] jepsen worker 14 - jepsen.util 6314 :invoke :write [4 47] INFO [2019-12-17 12:32:32,237] jepsen worker 65 - jepsen.util 5195 :ok :write [21 32] INFO [2019-12-17 12:32:32,237] jepsen worker 55 - jepsen.util 7165 :invoke :cas [18 [16 20]] INFO [2019-12-17 12:32:32,237] jepsen worker 65 - jepsen.util 5195 :invoke :cas [21 [14 29]] INFO [2019-12-17 12:32:32,251] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":35152,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:32:32,251] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":35152,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:32:32,251] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":35155,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:32:32,251] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":35155,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:32:32,251] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":35156,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:32:32,251] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":35156,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:32:32,256] jepsen worker 89 - jepsen.util 5939 :ok :read [29 46] INFO [2019-12-17 12:32:32,256] jepsen worker 89 - jepsen.util 5939 :invoke :write [29 26] INFO [2019-12-17 12:32:32,258] jepsen worker 51 - jepsen.util 5991 :info :write [17 43] :RequestCanceledException INFO [2019-12-17 12:32:32,259] jepsen worker 51 - jepsen.util 6081 :invoke :cas [17 [3 46]] INFO [2019-12-17 12:32:32,280] jepsen worker 88 - jepsen.util 6388 :fail :cas [29 [40 4]] :ValueNotSwapFrom INFO [2019-12-17 12:32:32,280] jepsen worker 88 - jepsen.util 6388 :invoke :cas [29 [40 25]] INFO [2019-12-17 12:32:32,290] jepsen worker 65 - jepsen.util 5195 :fail :cas [21 [14 29]] :ValueNotSwapFrom INFO [2019-12-17 12:32:32,290] jepsen worker 65 - jepsen.util 5195 :invoke :write [21 49] INFO [2019-12-17 12:32:32,300] jepsen worker 27 - jepsen.util 5967 :ok :write [9 36] INFO [2019-12-17 12:32:32,301] jepsen worker 27 - jepsen.util 5967 :invoke :cas [9 [14 32]] INFO [2019-12-17 12:32:32,302] jepsen worker 47 - jepsen.util 6887 :ok :write [15 7] INFO [2019-12-17 12:32:32,302] jepsen worker 47 - jepsen.util 6887 :invoke :cas [15 [17 49]] INFO [2019-12-17 12:32:32,329] jepsen worker 27 - jepsen.util 5967 :fail :cas [9 [14 32]] :ValueNotSwapFrom INFO [2019-12-17 12:32:32,330] jepsen worker 27 - jepsen.util 5967 :invoke :read [9 nil] INFO [2019-12-17 12:32:32,366] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:32:32,366] jepsen nemesis - couchbase.util Rebalance check count: 266 INFO [2019-12-17 12:32:32,366] jepsen nemesis - couchbase.util Rebalance stuck count: 247 INFO [2019-12-17 12:32:32,379] jepsen worker 27 - jepsen.util 5967 :ok :read [9 36] INFO [2019-12-17 12:32:32,379] jepsen worker 27 - jepsen.util 5967 :invoke :read [9 nil] INFO [2019-12-17 12:32:32,385] jepsen worker 87 - jepsen.util 5577 :fail :cas [29 [45 24]] :ValueNotSwapFrom INFO [2019-12-17 12:32:32,386] jepsen worker 87 - jepsen.util 5577 :invoke :cas [29 [38 46]] INFO [2019-12-17 12:32:32,396] jepsen worker 3 - jepsen.util 6033 :ok :write [1 6] INFO [2019-12-17 12:32:32,396] jepsen worker 3 - jepsen.util 6033 :invoke :read [1 nil] INFO [2019-12-17 12:32:32,408] jepsen worker 27 - jepsen.util 5967 :ok :read [9 36] INFO [2019-12-17 12:32:32,408] jepsen worker 27 - jepsen.util 5967 :invoke :read [9 nil] INFO [2019-12-17 12:32:32,416] jepsen worker 53 - jepsen.util 5723 :fail :cas [17 [7 34]] :ValueNotSwapFrom INFO [2019-12-17 12:32:32,416] jepsen worker 53 - jepsen.util 5723 :invoke :read [17 nil] INFO [2019-12-17 12:32:32,428] jepsen worker 29 - jepsen.util 4709 :ok :write [9 0] INFO [2019-12-17 12:32:32,428] jepsen worker 29 - jepsen.util 4709 :invoke :write [9 21] INFO [2019-12-17 12:32:32,430] jepsen worker 9 - jepsen.util 6579 :ok :write [3 5] INFO [2019-12-17 12:32:32,430] jepsen worker 9 - jepsen.util 6579 :invoke :read [3 nil] INFO [2019-12-17 12:32:32,436] jepsen worker 27 - jepsen.util 5967 :ok :read [9 36] INFO [2019-12-17 12:32:32,436] jepsen worker 27 - jepsen.util 5967 :invoke :write [9 27] INFO [2019-12-17 12:32:32,439] jepsen worker 5 - jepsen.util 5675 :info :write [1 37] :RequestCanceledException INFO [2019-12-17 12:32:32,440] jepsen worker 5 - jepsen.util 5765 :invoke :read [1 nil] INFO [2019-12-17 12:32:32,462] jepsen worker 9 - jepsen.util 6579 :ok :read [3 5] INFO [2019-12-17 12:32:32,462] jepsen worker 9 - jepsen.util 6579 :invoke :cas [3 [36 18]] INFO [2019-12-17 12:32:32,470] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":35173,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:32:32,470] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":35173,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:32:32,470] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":35175,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:32:32,470] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":35175,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:32:32,471] jepsen worker 88 - jepsen.util 6388 :fail :cas [29 [40 25]] :ValueNotSwapFrom INFO [2019-12-17 12:32:32,471] jepsen worker 88 - jepsen.util 6388 :invoke :write [29 6] INFO [2019-12-17 12:32:32,493] jepsen worker 65 - jepsen.util 5195 :ok :write [21 49] INFO [2019-12-17 12:32:32,493] jepsen worker 65 - jepsen.util 5195 :invoke :read [21 nil] INFO [2019-12-17 12:32:32,512] jepsen worker 73 - jepsen.util 5743 :info :cas [24 [31 42]] :RequestCanceledException INFO [2019-12-17 12:32:32,515] jepsen worker 73 - jepsen.util 5833 :invoke :read [24 nil] INFO [2019-12-17 12:32:32,518] jepsen worker 65 - jepsen.util 5195 :ok :read [21 49] INFO [2019-12-17 12:32:32,518] jepsen worker 65 - jepsen.util 5195 :invoke :read [21 nil] INFO [2019-12-17 12:32:32,518] jepsen worker 9 - jepsen.util 6579 :fail :cas [3 [36 18]] :ValueNotSwapFrom INFO [2019-12-17 12:32:32,518] jepsen worker 9 - jepsen.util 6579 :invoke :write [3 44] INFO [2019-12-17 12:32:32,529] jepsen worker 54 - jepsen.util 6624 :ok :read [18 30] INFO [2019-12-17 12:32:32,529] jepsen worker 54 - jepsen.util 6624 :invoke :write [18 17] INFO [2019-12-17 12:32:32,540] jepsen worker 12 - jepsen.util 6402 :fail :cas [4 [21 41]] :ValueNotSwapFrom INFO [2019-12-17 12:32:32,540] jepsen worker 12 - jepsen.util 6402 :invoke :cas [4 [43 11]] INFO [2019-12-17 12:32:32,550] jepsen worker 19 - jepsen.util 6049 :ok :write [6 2] INFO [2019-12-17 12:32:32,550] jepsen worker 19 - jepsen.util 6049 :invoke :write [6 19] INFO [2019-12-17 12:32:32,551] jepsen worker 73 - jepsen.util 5833 :ok :read [24 41] INFO [2019-12-17 12:32:32,552] jepsen worker 73 - jepsen.util 5833 :invoke :write [24 39] INFO [2019-12-17 12:32:32,580] jepsen worker 65 - jepsen.util 5195 :ok :read [21 49] INFO [2019-12-17 12:32:32,580] jepsen worker 65 - jepsen.util 5195 :invoke :read [21 nil] INFO [2019-12-17 12:32:32,593] jepsen worker 13 - jepsen.util 5953 :info :write [4 11] :RequestCanceledException INFO [2019-12-17 12:32:32,594] jepsen worker 13 - jepsen.util 6043 :invoke :read [4 nil] INFO [2019-12-17 12:32:32,606] jepsen worker 29 - jepsen.util 4709 :ok :write [9 21] INFO [2019-12-17 12:32:32,606] jepsen worker 29 - jepsen.util 4709 :invoke :write [9 12] INFO [2019-12-17 12:32:32,627] jepsen worker 70 - jepsen.util 6460 :info :write [23 7] :RequestCanceledException INFO [2019-12-17 12:32:32,629] jepsen worker 70 - jepsen.util 6550 :invoke :cas [23 [37 32]] INFO [2019-12-17 12:32:32,630] jepsen worker 74 - jepsen.util 6194 :ok :write [24 3] INFO [2019-12-17 12:32:32,630] jepsen worker 74 - jepsen.util 6194 :invoke :cas [24 [6 47]] INFO [2019-12-17 12:32:32,644] jepsen worker 53 - jepsen.util 5723 :ok :read [17 16] INFO [2019-12-17 12:32:32,644] jepsen worker 53 - jepsen.util 5723 :invoke :cas [17 [24 4]] INFO [2019-12-17 12:32:32,653] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":35198,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:32:32,653] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":35198,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:32:32,653] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":35202,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:32:32,653] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":35202,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:32:32,654] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":35203,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:32:32,654] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":35203,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:32:32,657] jepsen worker 79 - jepsen.util 6019 :info :write [26 1] :RequestCanceledException INFO [2019-12-17 12:32:32,658] jepsen worker 79 - jepsen.util 6109 :invoke :read [26 nil] INFO [2019-12-17 12:32:32,661] jepsen worker 47 - jepsen.util 6887 :fail :cas [15 [17 49]] :ValueNotSwapFrom INFO [2019-12-17 12:32:32,661] jepsen worker 47 - jepsen.util 6887 :invoke :read [15 nil] INFO [2019-12-17 12:32:32,665] jepsen worker 52 - jepsen.util 6532 :ok :read [17 16] INFO [2019-12-17 12:32:32,665] jepsen worker 52 - jepsen.util 6532 :invoke :cas [17 [40 26]] INFO [2019-12-17 12:32:32,679] jepsen worker 74 - jepsen.util 6194 :fail :cas [24 [6 47]] :ValueNotSwapFrom INFO [2019-12-17 12:32:32,679] jepsen worker 74 - jepsen.util 6194 :invoke :read [24 nil] INFO [2019-12-17 12:32:32,701] jepsen worker 29 - jepsen.util 4709 :ok :write [9 12] INFO [2019-12-17 12:32:32,701] jepsen worker 29 - jepsen.util 4709 :invoke :read [9 nil] INFO [2019-12-17 12:32:32,707] jepsen worker 47 - jepsen.util 6887 :ok :read [15 7] INFO [2019-12-17 12:32:32,707] jepsen worker 47 - jepsen.util 6887 :invoke :cas [15 [10 42]] INFO [2019-12-17 12:32:32,712] jepsen worker 70 - jepsen.util 6550 :fail :cas [23 [37 32]] :ValueNotSwapFrom INFO [2019-12-17 12:32:32,712] jepsen worker 70 - jepsen.util 6550 :invoke :write [23 14] INFO [2019-12-17 12:32:32,723] jepsen worker 52 - jepsen.util 6532 :fail :cas [17 [40 26]] :ValueNotSwapFrom INFO [2019-12-17 12:32:32,723] jepsen worker 52 - jepsen.util 6532 :invoke :cas [17 [21 23]] INFO [2019-12-17 12:32:32,726] jepsen worker 68 - jepsen.util 7898 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:32:32,727] jepsen worker 68 - jepsen.util 7898 :invoke :read [22 nil] INFO [2019-12-17 12:32:32,754] jepsen worker 78 - jepsen.util 6378 :info :write [26 2] :RequestCanceledException INFO [2019-12-17 12:32:32,756] jepsen worker 78 - jepsen.util 6468 :invoke :read [26 nil] INFO [2019-12-17 12:32:32,760] jepsen worker 52 - jepsen.util 6532 :fail :cas [17 [21 23]] :ValueNotSwapFrom INFO [2019-12-17 12:32:32,760] jepsen worker 52 - jepsen.util 6532 :invoke :cas [17 [18 25]] INFO [2019-12-17 12:32:32,761] jepsen worker 29 - jepsen.util 4709 :ok :read [9 12] INFO [2019-12-17 12:32:32,761] jepsen worker 29 - jepsen.util 4709 :invoke :cas [9 [25 48]] INFO [2019-12-17 12:32:32,763] jepsen worker 74 - jepsen.util 6194 :ok :read [24 3] INFO [2019-12-17 12:32:32,763] jepsen worker 74 - jepsen.util 6194 :invoke :write [24 33] INFO [2019-12-17 12:32:32,793] jepsen worker 36 - jepsen.util 6606 :info :write [12 35] :RequestCanceledException INFO [2019-12-17 12:32:32,794] jepsen worker 36 - jepsen.util 6696 :invoke :write [12 7] INFO [2019-12-17 12:32:32,796] jepsen worker 55 - jepsen.util 7165 :fail :cas [18 [16 20]] :ValueNotSwapFrom INFO [2019-12-17 12:32:32,796] jepsen worker 47 - jepsen.util 6887 :fail :cas [15 [10 42]] :ValueNotSwapFrom INFO [2019-12-17 12:32:32,797] jepsen worker 47 - jepsen.util 6887 :invoke :read [15 nil] INFO [2019-12-17 12:32:32,797] jepsen worker 55 - jepsen.util 7165 :invoke :read [18 nil] INFO [2019-12-17 12:32:32,801] jepsen worker 69 - jepsen.util 6189 :fail :cas [23 [23 39]] :ValueNotSwapFrom INFO [2019-12-17 12:32:32,801] jepsen worker 69 - jepsen.util 6189 :invoke :read [23 nil] INFO [2019-12-17 12:32:32,837] jepsen worker 29 - jepsen.util 4709 :fail :cas [9 [25 48]] :ValueNotSwapFrom INFO [2019-12-17 12:32:32,837] jepsen worker 29 - jepsen.util 4709 :invoke :read [9 nil] INFO [2019-12-17 12:32:32,839] jepsen worker 51 - jepsen.util 6081 :fail :cas [17 [3 46]] :ValueNotSwapFrom INFO [2019-12-17 12:32:32,839] jepsen worker 51 - jepsen.util 6081 :invoke :read [17 nil] INFO [2019-12-17 12:32:32,878] jepsen worker 29 - jepsen.util 4709 :ok :read [9 12] INFO [2019-12-17 12:32:32,878] jepsen worker 29 - jepsen.util 4709 :invoke :read [9 nil] INFO [2019-12-17 12:32:32,897] jepsen worker 29 - jepsen.util 4709 :ok :read [9 12] INFO [2019-12-17 12:32:32,898] jepsen worker 29 - jepsen.util 4709 :invoke :cas [9 [3 19]] INFO [2019-12-17 12:32:32,909] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":35225,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:32:32,909] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":35225,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:32:32,909] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":35228,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:32:32,910] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":35228,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:32:32,911] jepsen worker 66 - jepsen.util 7716 :info :write [22 38] :RequestCanceledException INFO [2019-12-17 12:32:32,912] jepsen worker 66 - jepsen.util 7806 :invoke :write [22 2] INFO [2019-12-17 12:32:32,918] jepsen worker 29 - jepsen.util 4709 :fail :cas [9 [3 19]] :ValueNotSwapFrom INFO [2019-12-17 12:32:32,918] jepsen worker 29 - jepsen.util 4709 :invoke :cas [9 [16 20]] INFO [2019-12-17 12:32:32,927] jepsen worker 60 - jepsen.util 9150 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:32:32,927] jepsen worker 60 - jepsen.util 9150 :invoke :read [20 nil] INFO [2019-12-17 12:32:32,936] jepsen worker 29 - jepsen.util 4709 :fail :cas [9 [16 20]] :ValueNotSwapFrom INFO [2019-12-17 12:32:32,936] jepsen worker 29 - jepsen.util 4709 :invoke :write [9 9] INFO [2019-12-17 12:32:32,941] jepsen worker 31 - jepsen.util 5431 :info :write [10 8] :RequestCanceledException INFO [2019-12-17 12:32:32,942] jepsen worker 31 - jepsen.util 5521 :invoke :write [10 45] INFO [2019-12-17 12:32:32,958] jepsen worker 85 - jepsen.util 9085 :info :write [28 49] :RequestCanceledException INFO [2019-12-17 12:32:32,959] jepsen worker 85 - jepsen.util 9175 :invoke :read [28 nil] INFO [2019-12-17 12:32:32,980] jepsen worker 79 - jepsen.util 6109 :ok :read [26 41] INFO [2019-12-17 12:32:32,980] jepsen worker 79 - jepsen.util 6109 :invoke :write [26 28] INFO [2019-12-17 12:32:32,986] jepsen worker 87 - jepsen.util 5577 :fail :cas [29 [38 46]] :ValueNotSwapFrom INFO [2019-12-17 12:32:32,986] jepsen worker 87 - jepsen.util 5577 :invoke :read [29 nil] INFO [2019-12-17 12:32:32,991] jepsen worker 78 - jepsen.util 6468 :ok :read [26 41] INFO [2019-12-17 12:32:32,991] jepsen worker 78 - jepsen.util 6468 :invoke :write [26 47] INFO [2019-12-17 12:32:33,014] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":35241,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:32:33,015] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":35241,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:32:33,015] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":35244,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:32:33,015] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":35244,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:32:33,015] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":35245,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:32:33,015] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":35245,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:32:33,017] jepsen worker 52 - jepsen.util 6532 :fail :cas [17 [18 25]] :ValueNotSwapFrom INFO [2019-12-17 12:32:33,017] jepsen worker 52 - jepsen.util 6532 :invoke :read [17 nil] INFO [2019-12-17 12:32:33,024] jepsen worker 29 - jepsen.util 4709 :ok :write [9 9] INFO [2019-12-17 12:32:33,024] jepsen worker 29 - jepsen.util 4709 :invoke :cas [9 [44 42]] INFO [2019-12-17 12:32:33,026] jepsen worker 44 - jepsen.util 8954 :info :write [14 15] :RequestCanceledException INFO [2019-12-17 12:32:33,026] jepsen worker 43 - jepsen.util 7693 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:32:33,027] jepsen worker 43 - jepsen.util 7693 :invoke :read [14 nil] INFO [2019-12-17 12:32:33,027] jepsen worker 44 - jepsen.util 9044 :invoke :read [14 nil] INFO [2019-12-17 12:32:33,041] jepsen worker 75 - jepsen.util 7815 :info :cas [25 [25 45]] :RequestCanceledException INFO [2019-12-17 12:32:33,042] jepsen worker 75 - jepsen.util 7905 :invoke :read [25 nil] INFO [2019-12-17 12:32:33,059] jepsen worker 46 - jepsen.util 6076 :info :write [15 43] :RequestCanceledException INFO [2019-12-17 12:32:33,060] jepsen worker 46 - jepsen.util 6166 :invoke :write [15 19] INFO [2019-12-17 12:32:33,060] jepsen worker 29 - jepsen.util 4709 :fail :cas [9 [44 42]] :ValueNotSwapFrom INFO [2019-12-17 12:32:33,060] jepsen worker 29 - jepsen.util 4709 :invoke :read [9 nil] INFO [2019-12-17 12:32:33,065] jepsen worker 12 - jepsen.util 6402 :fail :cas [4 [43 11]] :ValueNotSwapFrom INFO [2019-12-17 12:32:33,065] jepsen worker 12 - jepsen.util 6402 :invoke :cas [4 [26 17]] INFO [2019-12-17 12:32:33,069] jepsen worker 69 - jepsen.util 6189 :ok :read [23 33] INFO [2019-12-17 12:32:33,069] jepsen worker 69 - jepsen.util 6189 :invoke :cas [23 [27 18]] INFO [2019-12-17 12:32:33,085] jepsen worker 29 - jepsen.util 4709 :ok :read [9 9] INFO [2019-12-17 12:32:33,085] jepsen worker 29 - jepsen.util 4709 :invoke :cas [9 [46 26]] INFO [2019-12-17 12:32:33,093] jepsen worker 53 - jepsen.util 5723 :fail :cas [17 [24 4]] :ValueNotSwapFrom INFO [2019-12-17 12:32:33,093] jepsen worker 53 - jepsen.util 5723 :invoke :read [17 nil] INFO [2019-12-17 12:32:33,094] jepsen worker 8 - jepsen.util 7748 :info :cas [2 [37 38]] :RequestCanceledException INFO [2019-12-17 12:32:33,095] jepsen worker 8 - jepsen.util 7838 :invoke :write [2 32] INFO [2019-12-17 12:32:33,104] jepsen worker 51 - jepsen.util 6081 :ok :read [17 16] INFO [2019-12-17 12:32:33,104] jepsen worker 51 - jepsen.util 6081 :invoke :cas [17 [40 16]] INFO [2019-12-17 12:32:33,111] jepsen worker 16 - jepsen.util 9196 :info :write [5 35] :RequestCanceledException INFO [2019-12-17 12:32:33,111] jepsen worker 82 - jepsen.util 8722 :info :write [27 23] :RequestCanceledException INFO [2019-12-17 12:32:33,112] jepsen worker 16 - jepsen.util 9286 :invoke :cas [5 [4 15]] INFO [2019-12-17 12:32:33,113] jepsen worker 82 - jepsen.util 8812 :invoke :write [27 33] INFO [2019-12-17 12:32:33,120] jepsen worker 55 - jepsen.util 7165 :ok :read [18 30] INFO [2019-12-17 12:32:33,120] jepsen worker 55 - jepsen.util 7165 :invoke :cas [18 [19 20]] INFO [2019-12-17 12:32:33,127] jepsen worker 57 - jepsen.util 7797 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:32:33,127] jepsen worker 76 - jepsen.util 8446 :info :write [25 16] :RequestCanceledException INFO [2019-12-17 12:32:33,127] jepsen worker 34 - jepsen.util 7684 :info :cas [11 [49 39]] :RequestCanceledException INFO [2019-12-17 12:32:33,127] jepsen worker 57 - jepsen.util 7797 :invoke :read [19 nil] INFO [2019-12-17 12:32:33,128] jepsen worker 76 - jepsen.util 8536 :invoke :cas [25 [22 5]] INFO [2019-12-17 12:32:33,128] jepsen worker 34 - jepsen.util 7774 :invoke :cas [11 [12 29]] INFO [2019-12-17 12:32:33,140] jepsen worker 48 - jepsen.util 7518 :info :write [16 42] :RequestCanceledException INFO [2019-12-17 12:32:33,141] jepsen worker 48 - jepsen.util 7608 :invoke :read [16 nil] INFO [2019-12-17 12:32:33,158] jepsen worker 6 - jepsen.util 8826 :info :write [2 12] :RequestCanceledException INFO [2019-12-17 12:32:33,159] jepsen worker 6 - jepsen.util 8916 :invoke :read [2 nil] INFO [2019-12-17 12:32:33,167] jepsen worker 14 - jepsen.util 6314 :ok :write [4 47] INFO [2019-12-17 12:32:33,167] jepsen worker 14 - jepsen.util 6314 :invoke :read [4 nil] INFO [2019-12-17 12:32:33,169] jepsen worker 80 - jepsen.util 6200 :ok :write [26 25] INFO [2019-12-17 12:32:33,169] jepsen worker 80 - jepsen.util 6200 :invoke :write [26 42] INFO [2019-12-17 12:32:33,193] jepsen worker 58 - jepsen.util 8698 :info :cas [19 [49 38]] :RequestCanceledException INFO [2019-12-17 12:32:33,194] jepsen worker 58 - jepsen.util 8788 :invoke :cas [19 [27 13]] INFO [2019-12-17 12:32:33,210] jepsen worker 52 - jepsen.util 6532 :ok :read [17 16] INFO [2019-12-17 12:32:33,210] jepsen worker 52 - jepsen.util 6532 :invoke :cas [17 [1 28]] INFO [2019-12-17 12:32:33,211] jepsen worker 10 - jepsen.util 6040 :fail :read [3 nil] :RequestTimeoutException INFO [2019-12-17 12:32:33,211] jepsen worker 41 - jepsen.util 9041 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:32:33,211] jepsen worker 10 - jepsen.util 6040 :invoke :read [3 nil] INFO [2019-12-17 12:32:33,211] jepsen worker 41 - jepsen.util 9041 :invoke :write [13 21] INFO [2019-12-17 12:32:33,225] jepsen worker 79 - jepsen.util 6109 :ok :write [26 28] INFO [2019-12-17 12:32:33,225] jepsen worker 79 - jepsen.util 6109 :invoke :read [26 nil] INFO [2019-12-17 12:32:33,227] jepsen worker 33 - jepsen.util 8223 :info :write [11 43] :RequestCanceledException INFO [2019-12-17 12:32:33,228] jepsen worker 33 - jepsen.util 8313 :invoke :read [11 nil] INFO [2019-12-17 12:32:33,230] jepsen worker 10 - jepsen.util 6040 :ok :read [3 5] INFO [2019-12-17 12:32:33,230] jepsen worker 10 - jepsen.util 6040 :invoke :cas [3 [2 14]] INFO [2019-12-17 12:32:33,238] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":35280,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:32:33,238] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":35280,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:32:33,238] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":35283,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:32:33,238] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":35283,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:32:33,241] jepsen worker 38 - jepsen.util 6068 :info :write [12 24] :RequestCanceledException INFO [2019-12-17 12:32:33,241] jepsen worker 28 - jepsen.util 4798 :fail :read [9 nil] :RequestTimeoutException INFO [2019-12-17 12:32:33,241] jepsen worker 28 - jepsen.util 4798 :invoke :read [9 nil] INFO [2019-12-17 12:32:33,242] jepsen worker 38 - jepsen.util 6158 :invoke :write [12 13] INFO [2019-12-17 12:32:33,245] jepsen worker 10 - jepsen.util 6040 :fail :cas [3 [2 14]] :ValueNotSwapFrom INFO [2019-12-17 12:32:33,245] jepsen worker 10 - jepsen.util 6040 :invoke :cas [3 [33 41]] INFO [2019-12-17 12:32:33,261] jepsen worker 53 - jepsen.util 5723 :ok :read [17 16] INFO [2019-12-17 12:32:33,261] jepsen worker 10 - jepsen.util 6040 :fail :cas [3 [33 41]] :ValueNotSwapFrom INFO [2019-12-17 12:32:33,262] jepsen worker 53 - jepsen.util 5723 :invoke :cas [17 [36 29]] INFO [2019-12-17 12:32:33,262] jepsen worker 10 - jepsen.util 6040 :invoke :cas [3 [20 15]] INFO [2019-12-17 12:32:33,265] jepsen worker 56 - jepsen.util 6626 :ok :write [18 19] INFO [2019-12-17 12:32:33,265] jepsen worker 56 - jepsen.util 6626 :invoke :cas [18 [44 16]] INFO [2019-12-17 12:32:33,276] jepsen worker 71 - jepsen.util 6821 :ok :write [23 35] INFO [2019-12-17 12:32:33,277] jepsen worker 71 - jepsen.util 6821 :invoke :write [23 35] INFO [2019-12-17 12:32:33,279] jepsen worker 10 - jepsen.util 6040 :fail :cas [3 [20 15]] :ValueNotSwapFrom INFO [2019-12-17 12:32:33,279] jepsen worker 10 - jepsen.util 6040 :invoke :write [3 46] INFO [2019-12-17 12:32:33,285] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":35271,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:32:33,285] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":35271,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:32:33,285] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":35272,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:32:33,285] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":35272,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:32:33,288] jepsen worker 69 - jepsen.util 6189 :fail :cas [23 [27 18]] :ValueNotSwapFrom INFO [2019-12-17 12:32:33,288] jepsen worker 69 - jepsen.util 6189 :invoke :write [23 37] INFO [2019-12-17 12:32:33,288] jepsen worker 9 - jepsen.util 6579 :ok :write [3 44] INFO [2019-12-17 12:32:33,289] jepsen worker 9 - jepsen.util 6579 :invoke :read [3 nil] INFO [2019-12-17 12:32:33,292] jepsen worker 19 - jepsen.util 6049 :ok :write [6 19] INFO [2019-12-17 12:32:33,292] jepsen worker 19 - jepsen.util 6049 :invoke :cas [6 [21 24]] INFO [2019-12-17 12:32:33,324] jepsen worker 9 - jepsen.util 6579 :ok :read [3 44] INFO [2019-12-17 12:32:33,324] jepsen worker 9 - jepsen.util 6579 :invoke :read [3 nil] INFO [2019-12-17 12:32:33,326] jepsen worker 0 - jepsen.util 8550 :info :write [0 35] :RequestCanceledException INFO [2019-12-17 12:32:33,327] jepsen worker 0 - jepsen.util 8640 :invoke :cas [0 [3 49]] INFO [2019-12-17 12:32:33,355] jepsen worker 62 - jepsen.util 7802 :info :cas [20 [27 22]] :RequestCanceledException INFO [2019-12-17 12:32:33,356] jepsen worker 62 - jepsen.util 7892 :invoke :cas [20 [42 9]] INFO [2019-12-17 12:32:33,365] jepsen worker 9 - jepsen.util 6579 :ok :read [3 44] INFO [2019-12-17 12:32:33,365] jepsen worker 9 - jepsen.util 6579 :invoke :cas [3 [17 36]] INFO [2019-12-17 12:32:33,384] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:32:33,384] jepsen nemesis - couchbase.util Rebalance check count: 267 INFO [2019-12-17 12:32:33,384] jepsen nemesis - couchbase.util Rebalance stuck count: 248 INFO [2019-12-17 12:32:33,392] jepsen worker 9 - jepsen.util 6579 :fail :cas [3 [17 36]] :ValueNotSwapFrom INFO [2019-12-17 12:32:33,392] jepsen worker 9 - jepsen.util 6579 :invoke :cas [3 [9 6]] INFO [2019-12-17 12:32:33,393] jepsen worker 84 - jepsen.util 7914 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:32:33,393] jepsen worker 84 - jepsen.util 7914 :invoke :read [28 nil] INFO [2019-12-17 12:32:33,395] jepsen worker 13 - jepsen.util 6043 :ok :read [4 47] INFO [2019-12-17 12:32:33,395] jepsen worker 13 - jepsen.util 6043 :invoke :read [4 nil] INFO [2019-12-17 12:32:33,396] jepsen worker 28 - jepsen.util 4798 :ok :read [9 9] INFO [2019-12-17 12:32:33,396] jepsen worker 28 - jepsen.util 4798 :invoke :cas [9 [16 19]] INFO [2019-12-17 12:32:33,407] jepsen worker 12 - jepsen.util 6402 :fail :cas [4 [26 17]] :ValueNotSwapFrom INFO [2019-12-17 12:32:33,407] jepsen worker 12 - jepsen.util 6402 :invoke :write [4 45] INFO [2019-12-17 12:32:33,409] jepsen worker 81 - jepsen.util 7911 :info :cas [27 [18 31]] :RequestCanceledException INFO [2019-12-17 12:32:33,409] jepsen worker 86 - jepsen.util 8546 :info :write [28 8] :RequestCanceledException INFO [2019-12-17 12:32:33,410] jepsen worker 81 - jepsen.util 8001 :invoke :read [27 nil] INFO [2019-12-17 12:32:33,410] jepsen worker 86 - jepsen.util 8636 :invoke :cas [28 [3 22]] INFO [2019-12-17 12:32:33,415] jepsen worker 9 - jepsen.util 6579 :fail :cas [3 [9 6]] :ValueNotSwapFrom INFO [2019-12-17 12:32:33,415] jepsen worker 9 - jepsen.util 6579 :invoke :write [3 3] INFO [2019-12-17 12:32:33,415] jepsen worker 19 - jepsen.util 6049 :fail :cas [6 [21 24]] :ValueNotSwapFrom INFO [2019-12-17 12:32:33,415] jepsen worker 19 - jepsen.util 6049 :invoke :write [6 23] INFO [2019-12-17 12:32:33,424] jepsen worker 28 - jepsen.util 4798 :fail :cas [9 [16 19]] :ValueNotSwapFrom INFO [2019-12-17 12:32:33,424] jepsen worker 28 - jepsen.util 4798 :invoke :read [9 nil] INFO [2019-12-17 12:32:33,427] jepsen worker 21 - jepsen.util 7761 :info :cas [7 [22 20]] :RequestCanceledException INFO [2019-12-17 12:32:33,427] jepsen worker 20 - jepsen.util 6050 :fail :read [6 nil] :RequestTimeoutException INFO [2019-12-17 12:32:33,427] jepsen worker 20 - jepsen.util 6050 :invoke :write [6 20] INFO [2019-12-17 12:32:33,429] jepsen worker 21 - jepsen.util 7851 :invoke :write [7 1] INFO [2019-12-17 12:32:33,437] jepsen worker 79 - jepsen.util 6109 :ok :read [26 2] INFO [2019-12-17 12:32:33,438] jepsen worker 79 - jepsen.util 6109 :invoke :write [26 22] INFO [2019-12-17 12:32:33,441] jepsen worker 61 - jepsen.util 9331 :info :write [20 1] :RequestCanceledException INFO [2019-12-17 12:32:33,442] jepsen worker 61 - jepsen.util 9421 :invoke :read [20 nil] INFO [2019-12-17 12:32:33,453] jepsen worker 51 - jepsen.util 6081 :fail :cas [17 [40 16]] :ValueNotSwapFrom INFO [2019-12-17 12:32:33,453] jepsen worker 51 - jepsen.util 6081 :invoke :read [17 nil] INFO [2019-12-17 12:32:33,457] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":35316,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:32:33,457] jepsen worker 83 - jepsen.util 8543 :info :cas [27 [15 1]] :RequestCanceledException INFO [2019-12-17 12:32:33,457] jepsen worker 1 - jepsen.util 7921 :info :write [0 3] :RequestCanceledException INFO [2019-12-17 12:32:33,457] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":35316,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:32:33,458] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":35319,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:32:33,458] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":35319,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:32:33,458] jepsen worker 1 - jepsen.util 8011 :invoke :write [0 33] INFO [2019-12-17 12:32:33,458] jepsen worker 83 - jepsen.util 8633 :invoke :cas [27 [12 3]] INFO [2019-12-17 12:32:33,494] jepsen worker 52 - jepsen.util 6532 :fail :cas [17 [1 28]] :ValueNotSwapFrom INFO [2019-12-17 12:32:33,494] jepsen worker 52 - jepsen.util 6532 :invoke :read [17 nil] INFO [2019-12-17 12:32:33,512] jepsen worker 39 - jepsen.util 7689 :info :cas [13 [42 2]] :RequestCanceledException INFO [2019-12-17 12:32:33,513] jepsen worker 39 - jepsen.util 7779 :invoke :write [13 31] INFO [2019-12-17 12:32:33,526] jepsen worker 77 - jepsen.util 8537 :info :write [25 32] :RequestCanceledException INFO [2019-12-17 12:32:33,526] jepsen worker 4 - jepsen.util 5584 :info :write [1 19] :RequestCanceledException INFO [2019-12-17 12:32:33,528] jepsen worker 4 - jepsen.util 5674 :invoke :read [1 nil] INFO [2019-12-17 12:32:33,528] jepsen worker 77 - jepsen.util 8627 :invoke :write [25 43] INFO [2019-12-17 12:32:33,533] jepsen worker 53 - jepsen.util 5723 :fail :cas [17 [36 29]] :ValueNotSwapFrom INFO [2019-12-17 12:32:33,534] jepsen worker 53 - jepsen.util 5723 :invoke :write [17 40] INFO [2019-12-17 12:32:33,540] jepsen worker 40 - jepsen.util 8590 :info :cas [13 [42 33]] :RequestCanceledException INFO [2019-12-17 12:32:33,541] jepsen worker 40 - jepsen.util 8680 :invoke :cas [13 [13 43]] INFO [2019-12-17 12:32:33,556] jepsen worker 31 - jepsen.util 5521 :ok :write [10 45] INFO [2019-12-17 12:32:33,556] jepsen worker 31 - jepsen.util 5521 :invoke :write [10 15] INFO [2019-12-17 12:32:33,557] jepsen worker 22 - jepsen.util 8122 :info :cas [7 [3 14]] :RequestCanceledException INFO [2019-12-17 12:32:33,558] jepsen worker 73 - jepsen.util 5833 :ok :write [24 39] INFO [2019-12-17 12:32:33,558] jepsen worker 73 - jepsen.util 5833 :invoke :read [24 nil] INFO [2019-12-17 12:32:33,558] jepsen worker 22 - jepsen.util 8212 :invoke :read [7 nil] INFO [2019-12-17 12:32:33,573] jepsen worker 87 - jepsen.util 5577 :ok :read [29 46] INFO [2019-12-17 12:32:33,573] jepsen worker 87 - jepsen.util 5577 :invoke :cas [29 [2 6]] INFO [2019-12-17 12:32:33,593] jepsen worker 15 - jepsen.util 7215 :info :cas [5 [34 44]] :RequestCanceledException INFO [2019-12-17 12:32:33,594] jepsen worker 15 - jepsen.util 7305 :invoke :write [5 25] INFO [2019-12-17 12:32:33,605] jepsen worker 52 - jepsen.util 6532 :ok :read [17 16] INFO [2019-12-17 12:32:33,605] jepsen worker 52 - jepsen.util 6532 :invoke :write [17 40] INFO [2019-12-17 12:32:33,607] jepsen worker 14 - jepsen.util 6314 :ok :read [4 47] INFO [2019-12-17 12:32:33,607] jepsen worker 14 - jepsen.util 6314 :invoke :read [4 nil] INFO [2019-12-17 12:32:33,627] jepsen worker 25 - jepsen.util 8755 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:32:33,627] jepsen worker 7 - jepsen.util 8467 :info :write [2 6] :RequestCanceledException INFO [2019-12-17 12:32:33,627] jepsen worker 17 - jepsen.util 8207 :info :cas [5 [1 23]] :RequestCanceledException INFO [2019-12-17 12:32:33,627] jepsen worker 63 - jepsen.util 5013 :info :write [21 8] :RequestCanceledException INFO [2019-12-17 12:32:33,627] jepsen worker 25 - jepsen.util 8755 :invoke :write [8 42] INFO [2019-12-17 12:32:33,628] jepsen worker 63 - jepsen.util 5103 :invoke :read [21 nil] INFO [2019-12-17 12:32:33,628] jepsen worker 7 - jepsen.util 8557 :invoke :read [2 nil] INFO [2019-12-17 12:32:33,628] jepsen worker 17 - jepsen.util 8297 :invoke :write [5 44] INFO [2019-12-17 12:32:33,641] jepsen worker 64 - jepsen.util 4744 :info :cas [21 [8 35]] :RequestCanceledException INFO [2019-12-17 12:32:33,642] jepsen worker 64 - jepsen.util 4834 :invoke :write [21 27] INFO [2019-12-17 12:32:33,650] jepsen worker 14 - jepsen.util 6314 :ok :read [4 47] INFO [2019-12-17 12:32:33,651] jepsen worker 14 - jepsen.util 6314 :invoke :write [4 15] INFO [2019-12-17 12:32:33,653] jepsen worker 63 - jepsen.util 5103 :ok :read [21 49] INFO [2019-12-17 12:32:33,654] jepsen worker 63 - jepsen.util 5103 :invoke :write [21 34] INFO [2019-12-17 12:32:33,678] jepsen worker 46 - jepsen.util 6166 :ok :write [15 19] INFO [2019-12-17 12:32:33,678] jepsen worker 46 - jepsen.util 6166 :invoke :read [15 nil] INFO [2019-12-17 12:32:33,698] jepsen worker 29 - jepsen.util 4709 :fail :cas [9 [46 26]] :ValueNotSwapFrom INFO [2019-12-17 12:32:33,698] jepsen worker 29 - jepsen.util 4709 :invoke :read [9 nil] INFO [2019-12-17 12:32:33,712] jepsen worker 37 - jepsen.util 5797 :info :write [12 23] :RequestCanceledException INFO [2019-12-17 12:32:33,713] jepsen worker 37 - jepsen.util 5887 :invoke :write [12 22] INFO [2019-12-17 12:32:33,722] jepsen worker 19 - jepsen.util 6049 :ok :write [6 23] INFO [2019-12-17 12:32:33,722] jepsen worker 19 - jepsen.util 6049 :invoke :cas [6 [20 29]] INFO [2019-12-17 12:32:33,727] jepsen worker 23 - jepsen.util 8303 :info :cas [7 [10 29]] :RequestCanceledException INFO [2019-12-17 12:32:33,728] jepsen worker 23 - jepsen.util 8393 :invoke :write [7 41] INFO [2019-12-17 12:32:33,735] jepsen worker 46 - jepsen.util 6166 :ok :read [15 19] INFO [2019-12-17 12:32:33,735] jepsen worker 46 - jepsen.util 6166 :invoke :cas [15 [4 25]] INFO [2019-12-17 12:32:33,739] jepsen worker 26 - jepsen.util 8576 :info :write [8 5] :RequestCanceledException INFO [2019-12-17 12:32:33,742] jepsen worker 26 - jepsen.util 8666 :invoke :read [8 nil] INFO [2019-12-17 12:32:33,747] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":35345,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:32:33,747] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":35345,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:32:33,747] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":35347,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:32:33,747] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":35347,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:32:33,754] jepsen worker 29 - jepsen.util 4709 :ok :read [9 9] INFO [2019-12-17 12:32:33,754] jepsen worker 29 - jepsen.util 4709 :invoke :read [9 nil] INFO [2019-12-17 12:32:33,758] jepsen worker 45 - jepsen.util 6075 :fail :read [15 nil] :RequestTimeoutException INFO [2019-12-17 12:32:33,758] jepsen worker 11 - jepsen.util 6311 :fail :read [3 nil] :RequestTimeoutException INFO [2019-12-17 12:32:33,758] jepsen worker 19 - jepsen.util 6049 :fail :cas [6 [20 29]] :ValueNotSwapFrom INFO [2019-12-17 12:32:33,758] jepsen worker 45 - jepsen.util 6075 :invoke :cas [15 [48 39]] INFO [2019-12-17 12:32:33,758] jepsen worker 11 - jepsen.util 6311 :invoke :write [3 33] INFO [2019-12-17 12:32:33,758] jepsen worker 19 - jepsen.util 6049 :invoke :read [6 nil] INFO [2019-12-17 12:32:33,759] jepsen worker 67 - jepsen.util 8347 :info :cas [22 [1 48]] :RequestCanceledException INFO [2019-12-17 12:32:33,760] jepsen worker 67 - jepsen.util 8437 :invoke :cas [22 [48 20]] INFO [2019-12-17 12:32:33,796] jepsen worker 64 - jepsen.util 4834 :ok :write [21 27] INFO [2019-12-17 12:32:33,796] jepsen worker 64 - jepsen.util 4834 :invoke :write [21 2] INFO [2019-12-17 12:32:33,807] jepsen worker 46 - jepsen.util 6166 :fail :cas [15 [4 25]] :ValueNotSwapFrom INFO [2019-12-17 12:32:33,807] jepsen worker 46 - jepsen.util 6166 :invoke :cas [15 [19 8]] INFO [2019-12-17 12:32:33,809] jepsen worker 51 - jepsen.util 6081 :ok :read [17 16] INFO [2019-12-17 12:32:33,810] jepsen worker 51 - jepsen.util 6081 :invoke :read [17 nil] INFO [2019-12-17 12:32:33,811] jepsen worker 59 - jepsen.util 8339 :info :cas [19 [4 6]] :RequestCanceledException INFO [2019-12-17 12:32:33,811] jepsen worker 42 - jepsen.util 8772 :info :cas [14 [5 22]] :RequestCanceledException INFO [2019-12-17 12:32:33,812] jepsen worker 59 - jepsen.util 8429 :invoke :read [19 nil] INFO [2019-12-17 12:32:33,812] jepsen worker 42 - jepsen.util 8862 :invoke :cas [14 [24 18]] INFO [2019-12-17 12:32:33,824] jepsen worker 19 - jepsen.util 6049 :ok :read [6 23] INFO [2019-12-17 12:32:33,825] jepsen worker 19 - jepsen.util 6049 :invoke :cas [6 [0 24]] INFO [2019-12-17 12:32:33,826] jepsen worker 24 - jepsen.util 7404 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:32:33,826] jepsen worker 24 - jepsen.util 7404 :invoke :cas [8 [3 22]] INFO [2019-12-17 12:32:33,841] jepsen worker 35 - jepsen.util 8225 :info :write [11 37] :RequestCanceledException INFO [2019-12-17 12:32:33,842] jepsen worker 35 - jepsen.util 8315 :invoke :write [11 35] INFO [2019-12-17 12:32:33,859] jepsen worker 19 - jepsen.util 6049 :fail :cas [6 [0 24]] :ValueNotSwapFrom INFO [2019-12-17 12:32:33,859] jepsen worker 19 - jepsen.util 6049 :invoke :cas [6 [34 2]] INFO [2019-12-17 12:32:33,860] jepsen worker 49 - jepsen.util 7339 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:32:33,860] jepsen worker 49 - jepsen.util 7339 :invoke :read [16 nil] INFO [2019-12-17 12:32:33,870] jepsen worker 10 - jepsen.util 6040 :ok :write [3 46] INFO [2019-12-17 12:32:33,870] jepsen worker 29 - jepsen.util 4709 :ok :read [9 9] INFO [2019-12-17 12:32:33,870] jepsen worker 10 - jepsen.util 6040 :invoke :read [3 nil] INFO [2019-12-17 12:32:33,870] jepsen worker 29 - jepsen.util 4709 :invoke :cas [9 [0 28]] INFO [2019-12-17 12:32:33,896] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":35375,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:32:33,896] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":35375,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:32:33,896] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":35379,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:32:33,897] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":35379,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:32:33,913] jepsen worker 56 - jepsen.util 6626 :fail :cas [18 [44 16]] :ValueNotSwapFrom INFO [2019-12-17 12:32:33,913] jepsen worker 56 - jepsen.util 6626 :invoke :read [18 nil] INFO [2019-12-17 12:32:33,915] jepsen worker 31 - jepsen.util 5521 :ok :write [10 15] INFO [2019-12-17 12:32:33,915] jepsen worker 31 - jepsen.util 5521 :invoke :cas [10 [6 38]] INFO [2019-12-17 12:32:33,923] jepsen worker 14 - jepsen.util 6314 :ok :write [4 15] INFO [2019-12-17 12:32:33,923] jepsen worker 14 - jepsen.util 6314 :invoke :read [4 nil] INFO [2019-12-17 12:32:33,933] jepsen worker 45 - jepsen.util 6075 :fail :cas [15 [48 39]] :ValueNotSwapFrom INFO [2019-12-17 12:32:33,933] jepsen worker 45 - jepsen.util 6075 :invoke :cas [15 [34 36]] INFO [2019-12-17 12:32:33,933] jepsen worker 19 - jepsen.util 6049 :fail :cas [6 [34 2]] :ValueNotSwapFrom INFO [2019-12-17 12:32:33,933] jepsen worker 19 - jepsen.util 6049 :invoke :cas [6 [39 8]] INFO [2019-12-17 12:32:33,946] jepsen worker 10 - jepsen.util 6040 :ok :read [3 46] INFO [2019-12-17 12:32:33,947] jepsen worker 10 - jepsen.util 6040 :invoke :cas [3 [46 1]] INFO [2019-12-17 12:32:33,958] jepsen worker 80 - jepsen.util 6200 :ok :write [26 42] INFO [2019-12-17 12:32:33,958] jepsen worker 80 - jepsen.util 6200 :invoke :cas [26 [29 31]] INFO [2019-12-17 12:32:33,960] jepsen worker 2 - jepsen.util 8012 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:32:33,960] jepsen worker 2 - jepsen.util 8012 :invoke :cas [0 [22 47]] INFO [2019-12-17 12:32:33,974] jepsen worker 13 - jepsen.util 6043 :ok :read [4 15] INFO [2019-12-17 12:32:33,974] jepsen worker 13 - jepsen.util 6043 :invoke :read [4 nil] INFO [2019-12-17 12:32:33,982] jepsen worker 19 - jepsen.util 6049 :fail :cas [6 [39 8]] :ValueNotSwapFrom INFO [2019-12-17 12:32:33,982] jepsen worker 19 - jepsen.util 6049 :invoke :read [6 nil] INFO [2019-12-17 12:32:33,993] jepsen worker 30 - jepsen.util 6150 :fail :read [10 nil] :RequestTimeoutException INFO [2019-12-17 12:32:33,994] jepsen worker 30 - jepsen.util 6150 :invoke :read [10 nil] INFO [2019-12-17 12:32:33,997] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":35397,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:32:33,997] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":35397,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:32:33,998] jepsen worker 4 - jepsen.util 5674 :ok :read [1 6] INFO [2019-12-17 12:32:33,998] jepsen worker 4 - jepsen.util 5674 :invoke :write [1 1] INFO [2019-12-17 12:32:34,003] jepsen worker 80 - jepsen.util 6200 :fail :cas [26 [29 31]] :ValueNotSwapFrom INFO [2019-12-17 12:32:34,003] jepsen worker 80 - jepsen.util 6200 :invoke :read [26 nil] INFO [2019-12-17 12:32:34,033] jepsen worker 31 - jepsen.util 5521 :fail :cas [10 [6 38]] :ValueNotSwapFrom INFO [2019-12-17 12:32:34,033] jepsen worker 31 - jepsen.util 5521 :invoke :read [10 nil] INFO [2019-12-17 12:32:34,045] jepsen worker 19 - jepsen.util 6049 :ok :read [6 23] INFO [2019-12-17 12:32:34,045] jepsen worker 19 - jepsen.util 6049 :invoke :cas [6 [10 9]] INFO [2019-12-17 12:32:34,059] jepsen worker 87 - jepsen.util 5577 :fail :cas [29 [2 6]] :ValueNotSwapFrom INFO [2019-12-17 12:32:34,059] jepsen worker 87 - jepsen.util 5577 :invoke :read [29 nil] INFO [2019-12-17 12:32:34,083] jepsen worker 80 - jepsen.util 6200 :ok :read [26 42] INFO [2019-12-17 12:32:34,084] jepsen worker 80 - jepsen.util 6200 :invoke :write [26 10] INFO [2019-12-17 12:32:34,086] jepsen worker 13 - jepsen.util 6043 :ok :read [4 15] INFO [2019-12-17 12:32:34,087] jepsen worker 13 - jepsen.util 6043 :invoke :cas [4 [31 21]] INFO [2019-12-17 12:32:34,101] jepsen worker 14 - jepsen.util 6314 :ok :read [4 15] INFO [2019-12-17 12:32:34,101] jepsen worker 14 - jepsen.util 6314 :invoke :read [4 nil] INFO [2019-12-17 12:32:34,123] jepsen worker 30 - jepsen.util 6150 :ok :read [10 15] INFO [2019-12-17 12:32:34,123] jepsen worker 30 - jepsen.util 6150 :invoke :write [10 22] INFO [2019-12-17 12:32:34,160] jepsen worker 56 - jepsen.util 6626 :ok :read [18 19] INFO [2019-12-17 12:32:34,160] jepsen worker 37 - jepsen.util 5887 :ok :write [12 22] INFO [2019-12-17 12:32:34,160] jepsen worker 56 - jepsen.util 6626 :invoke :read [18 nil] INFO [2019-12-17 12:32:34,160] jepsen worker 37 - jepsen.util 5887 :invoke :write [12 37] INFO [2019-12-17 12:32:34,167] jepsen worker 19 - jepsen.util 6049 :fail :cas [6 [10 9]] :ValueNotSwapFrom INFO [2019-12-17 12:32:34,167] jepsen worker 19 - jepsen.util 6049 :invoke :write [6 2] INFO [2019-12-17 12:32:34,176] jepsen worker 31 - jepsen.util 5521 :ok :read [10 15] INFO [2019-12-17 12:32:34,176] jepsen worker 31 - jepsen.util 5521 :invoke :write [10 3] INFO [2019-12-17 12:32:34,189] jepsen worker 4 - jepsen.util 5674 :ok :write [1 1] INFO [2019-12-17 12:32:34,189] jepsen worker 4 - jepsen.util 5674 :invoke :read [1 nil] INFO [2019-12-17 12:32:34,189] jepsen worker 56 - jepsen.util 6626 :ok :read [18 19] INFO [2019-12-17 12:32:34,190] jepsen worker 56 - jepsen.util 6626 :invoke :write [18 21] INFO [2019-12-17 12:32:34,190] jepsen worker 72 - jepsen.util 6102 :fail :read [24 nil] :RequestTimeoutException INFO [2019-12-17 12:32:34,190] jepsen worker 72 - jepsen.util 6102 :invoke :read [24 nil] INFO [2019-12-17 12:32:34,204] jepsen worker 13 - jepsen.util 6043 :fail :cas [4 [31 21]] :ValueNotSwapFrom INFO [2019-12-17 12:32:34,204] jepsen worker 13 - jepsen.util 6043 :invoke :read [4 nil] INFO [2019-12-17 12:32:34,211] jepsen worker 50 - jepsen.util 8600 :info :cas [16 [29 37]] :RequestCanceledException INFO [2019-12-17 12:32:34,212] jepsen worker 50 - jepsen.util 8690 :invoke :write [16 38] INFO [2019-12-17 12:32:34,216] jepsen worker 64 - jepsen.util 4834 :ok :write [21 2] INFO [2019-12-17 12:32:34,217] jepsen worker 64 - jepsen.util 4834 :invoke :read [21 nil] INFO [2019-12-17 12:32:34,223] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":35426,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:32:34,223] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":35426,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:32:34,223] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":35430,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:32:34,223] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":35430,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:32:34,224] jepsen worker 87 - jepsen.util 5577 :ok :read [29 46] INFO [2019-12-17 12:32:34,224] jepsen worker 87 - jepsen.util 5577 :invoke :read [29 nil] INFO [2019-12-17 12:32:34,232] jepsen worker 64 - jepsen.util 4834 :ok :read [21 2] INFO [2019-12-17 12:32:34,232] jepsen worker 64 - jepsen.util 4834 :invoke :read [21 nil] INFO [2019-12-17 12:32:34,241] jepsen worker 32 - jepsen.util 6152 :info :cas [10 [31 29]] :RequestCanceledException INFO [2019-12-17 12:32:34,241] jepsen worker 14 - jepsen.util 6314 :ok :read [4 15] INFO [2019-12-17 12:32:34,242] jepsen worker 14 - jepsen.util 6314 :invoke :write [4 37] INFO [2019-12-17 12:32:34,242] jepsen worker 32 - jepsen.util 6242 :invoke :write [10 42] INFO [2019-12-17 12:32:34,264] jepsen worker 4 - jepsen.util 5674 :ok :read [1 1] INFO [2019-12-17 12:32:34,264] jepsen worker 4 - jepsen.util 5674 :invoke :write [1 40] INFO [2019-12-17 12:32:34,264] jepsen worker 80 - jepsen.util 6200 :ok :write [26 10] INFO [2019-12-17 12:32:34,264] jepsen worker 80 - jepsen.util 6200 :invoke :cas [26 [16 29]] INFO [2019-12-17 12:32:34,270] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":35418,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:32:34,270] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":35418,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:32:34,270] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":35420,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:32:34,270] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":35420,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:32:34,281] jepsen worker 87 - jepsen.util 5577 :ok :read [29 46] INFO [2019-12-17 12:32:34,282] jepsen worker 87 - jepsen.util 5577 :invoke :read [29 nil] INFO [2019-12-17 12:32:34,284] jepsen worker 64 - jepsen.util 4834 :ok :read [21 2] INFO [2019-12-17 12:32:34,284] jepsen worker 64 - jepsen.util 4834 :invoke :write [21 28] INFO [2019-12-17 12:32:34,294] jepsen worker 18 - jepsen.util 5868 :info :cas [6 [37 33]] :RequestCanceledException INFO [2019-12-17 12:32:34,294] jepsen worker 89 - jepsen.util 5939 :info :write [29 26] :RequestCanceledException INFO [2019-12-17 12:32:34,295] jepsen worker 18 - jepsen.util 5958 :invoke :write [6 18] INFO [2019-12-17 12:32:34,295] jepsen worker 89 - jepsen.util 6029 :invoke :write [29 48] INFO [2019-12-17 12:32:34,304] jepsen worker 51 - jepsen.util 6081 :ok :read [17 16] INFO [2019-12-17 12:32:34,304] jepsen worker 51 - jepsen.util 6081 :invoke :read [17 nil] INFO [2019-12-17 12:32:34,318] jepsen worker 71 - jepsen.util 6821 :ok :write [23 35] INFO [2019-12-17 12:32:34,318] jepsen worker 71 - jepsen.util 6821 :invoke :read [23 nil] INFO [2019-12-17 12:32:34,362] jepsen worker 64 - jepsen.util 4834 :ok :write [21 28] INFO [2019-12-17 12:32:34,362] jepsen worker 64 - jepsen.util 4834 :invoke :cas [21 [43 35]] INFO [2019-12-17 12:32:34,368] jepsen worker 55 - jepsen.util 7165 :ok :cas [18 [19 20]] INFO [2019-12-17 12:32:34,368] jepsen worker 55 - jepsen.util 7165 :invoke :cas [18 [37 43]] INFO [2019-12-17 12:32:34,375] jepsen worker 11 - jepsen.util 6311 :ok :write [3 33] INFO [2019-12-17 12:32:34,375] jepsen worker 11 - jepsen.util 6311 :invoke :read [3 nil] INFO [2019-12-17 12:32:34,376] jepsen worker 30 - jepsen.util 6150 :ok :write [10 22] INFO [2019-12-17 12:32:34,376] jepsen worker 30 - jepsen.util 6150 :invoke :read [10 nil] INFO [2019-12-17 12:32:34,378] jepsen worker 64 - jepsen.util 4834 :fail :cas [21 [43 35]] :ValueNotSwapFrom INFO [2019-12-17 12:32:34,378] jepsen worker 64 - jepsen.util 4834 :invoke :write [21 10] INFO [2019-12-17 12:32:34,416] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:32:34,416] jepsen nemesis - couchbase.util Rebalance check count: 268 INFO [2019-12-17 12:32:34,416] jepsen nemesis - couchbase.util Rebalance stuck count: 249 INFO [2019-12-17 12:32:34,427] jepsen worker 72 - jepsen.util 6102 :ok :read [24 39] INFO [2019-12-17 12:32:34,427] jepsen worker 72 - jepsen.util 6102 :invoke :cas [24 [41 13]] INFO [2019-12-17 12:32:34,439] jepsen worker 19 - jepsen.util 6049 :ok :write [6 2] INFO [2019-12-17 12:32:34,440] jepsen worker 19 - jepsen.util 6049 :invoke :cas [6 [0 30]] INFO [2019-12-17 12:32:34,442] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":35451,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:32:34,442] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":35451,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:32:34,443] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":35454,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:32:34,443] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":35454,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:32:34,443] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":35455,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:32:34,443] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":35455,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:32:34,445] jepsen worker 29 - jepsen.util 4709 :fail :cas [9 [0 28]] :ValueNotSwapFrom INFO [2019-12-17 12:32:34,445] jepsen worker 29 - jepsen.util 4709 :invoke :write [9 9] INFO [2019-12-17 12:32:34,453] jepsen worker 13 - jepsen.util 6043 :ok :read [4 15] INFO [2019-12-17 12:32:34,453] jepsen worker 13 - jepsen.util 6043 :invoke :cas [4 [4 36]] INFO [2019-12-17 12:32:34,458] jepsen worker 5 - jepsen.util 5765 :fail :read [1 nil] :RequestTimeoutException INFO [2019-12-17 12:32:34,458] jepsen worker 3 - jepsen.util 6033 :fail :read [1 nil] :RequestTimeoutException INFO [2019-12-17 12:32:34,458] jepsen worker 5 - jepsen.util 5765 :invoke :cas [1 [39 26]] INFO [2019-12-17 12:32:34,458] jepsen worker 3 - jepsen.util 6033 :invoke :write [1 44] INFO [2019-12-17 12:32:34,477] jepsen worker 19 - jepsen.util 6049 :fail :cas [6 [0 30]] :ValueNotSwapFrom INFO [2019-12-17 12:32:34,477] jepsen worker 19 - jepsen.util 6049 :invoke :read [6 nil] INFO [2019-12-17 12:32:34,487] jepsen worker 72 - jepsen.util 6102 :fail :cas [24 [41 13]] :ValueNotSwapFrom INFO [2019-12-17 12:32:34,487] jepsen worker 72 - jepsen.util 6102 :invoke :read [24 nil] INFO [2019-12-17 12:32:34,493] jepsen worker 27 - jepsen.util 5967 :info :write [9 27] :RequestCanceledException INFO [2019-12-17 12:32:34,494] jepsen worker 27 - jepsen.util 6057 :invoke :write [9 38] INFO [2019-12-17 12:32:34,496] jepsen worker 45 - jepsen.util 6075 :fail :cas [15 [34 36]] :ValueNotSwapFrom INFO [2019-12-17 12:32:34,501] jepsen worker 45 - jepsen.util 6075 :invoke :write [15 38] INFO [2019-12-17 12:32:34,524] jepsen worker 13 - jepsen.util 6043 :fail :cas [4 [4 36]] :ValueNotSwapFrom INFO [2019-12-17 12:32:34,524] jepsen worker 13 - jepsen.util 6043 :invoke :read [4 nil] INFO [2019-12-17 12:32:34,526] jepsen worker 88 - jepsen.util 6388 :info :write [29 6] :RequestCanceledException INFO [2019-12-17 12:32:34,527] jepsen worker 88 - jepsen.util 6478 :invoke :read [29 nil] INFO [2019-12-17 12:32:34,532] jepsen worker 87 - jepsen.util 5577 :ok :read [29 46] INFO [2019-12-17 12:32:34,532] jepsen worker 87 - jepsen.util 5577 :invoke :read [29 nil] INFO [2019-12-17 12:32:34,533] jepsen worker 19 - jepsen.util 6049 :ok :read [6 2] INFO [2019-12-17 12:32:34,533] jepsen worker 19 - jepsen.util 6049 :invoke :read [6 nil] INFO [2019-12-17 12:32:34,542] jepsen worker 72 - jepsen.util 6102 :ok :read [24 39] INFO [2019-12-17 12:32:34,542] jepsen worker 72 - jepsen.util 6102 :invoke :write [24 2] INFO [2019-12-17 12:32:34,548] jepsen worker 11 - jepsen.util 6311 :ok :read [3 33] INFO [2019-12-17 12:32:34,548] jepsen worker 11 - jepsen.util 6311 :invoke :cas [3 [17 46]] INFO [2019-12-17 12:32:34,569] jepsen worker 19 - jepsen.util 6049 :ok :read [6 2] INFO [2019-12-17 12:32:34,569] jepsen worker 71 - jepsen.util 6821 :ok :read [23 35] INFO [2019-12-17 12:32:34,569] jepsen worker 19 - jepsen.util 6049 :invoke :write [6 24] INFO [2019-12-17 12:32:34,569] jepsen worker 71 - jepsen.util 6821 :invoke :read [23 nil] INFO [2019-12-17 12:32:34,593] jepsen worker 64 - jepsen.util 4834 :ok :write [21 10] INFO [2019-12-17 12:32:34,593] jepsen worker 64 - jepsen.util 4834 :invoke :read [21 nil] INFO [2019-12-17 12:32:34,622] jepsen worker 29 - jepsen.util 4709 :ok :write [9 9] INFO [2019-12-17 12:32:34,623] jepsen worker 29 - jepsen.util 4709 :invoke :cas [9 [29 26]] INFO [2019-12-17 12:32:34,626] jepsen worker 65 - jepsen.util 5195 :fail :read [21 nil] :RequestTimeoutException INFO [2019-12-17 12:32:34,626] jepsen worker 65 - jepsen.util 5195 :invoke :write [21 8] INFO [2019-12-17 12:32:34,633] jepsen worker 54 - jepsen.util 6624 :info :write [18 17] :RequestCanceledException INFO [2019-12-17 12:32:34,633] jepsen worker 54 - jepsen.util 6714 :invoke :write [18 48] INFO [2019-12-17 12:32:34,678] jepsen worker 29 - jepsen.util 4709 :fail :cas [9 [29 26]] :ValueNotSwapFrom INFO [2019-12-17 12:32:34,678] jepsen worker 29 - jepsen.util 4709 :invoke :cas [9 [28 8]] INFO [2019-12-17 12:32:34,683] jepsen worker 64 - jepsen.util 4834 :ok :read [21 10] INFO [2019-12-17 12:32:34,683] jepsen worker 64 - jepsen.util 4834 :invoke :cas [21 [4 32]] INFO [2019-12-17 12:32:34,715] jepsen worker 29 - jepsen.util 4709 :fail :cas [9 [28 8]] :ValueNotSwapFrom INFO [2019-12-17 12:32:34,715] jepsen worker 29 - jepsen.util 4709 :invoke :cas [9 [49 39]] INFO [2019-12-17 12:32:34,715] jepsen worker 51 - jepsen.util 6081 :ok :read [17 16] INFO [2019-12-17 12:32:34,715] jepsen worker 51 - jepsen.util 6081 :invoke :read [17 nil] INFO [2019-12-17 12:32:34,728] jepsen worker 64 - jepsen.util 4834 :fail :cas [21 [4 32]] :ValueNotSwapFrom INFO [2019-12-17 12:32:34,728] jepsen worker 64 - jepsen.util 4834 :invoke :cas [21 [47 23]] INFO [2019-12-17 12:32:34,736] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":35477,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:32:34,736] jepsen worker 29 - jepsen.util 4709 :fail :cas [9 [49 39]] :ValueNotSwapFrom INFO [2019-12-17 12:32:34,736] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":35477,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:32:34,736] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":35479,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:32:34,736] jepsen worker 29 - jepsen.util 4709 :invoke :write [9 41] INFO [2019-12-17 12:32:34,736] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":35479,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:32:34,739] jepsen worker 65 - jepsen.util 5195 :ok :write [21 8] INFO [2019-12-17 12:32:34,739] jepsen worker 65 - jepsen.util 5195 :invoke :cas [21 [41 35]] INFO [2019-12-17 12:32:34,748] jepsen worker 64 - jepsen.util 4834 :fail :cas [21 [47 23]] :ValueNotSwapFrom INFO [2019-12-17 12:32:34,748] jepsen worker 64 - jepsen.util 4834 :invoke :write [21 20] INFO [2019-12-17 12:32:34,756] jepsen worker 65 - jepsen.util 5195 :fail :cas [21 [41 35]] :ValueNotSwapFrom INFO [2019-12-17 12:32:34,757] jepsen worker 65 - jepsen.util 5195 :invoke :write [21 24] INFO [2019-12-17 12:32:34,773] jepsen worker 51 - jepsen.util 6081 :ok :read [17 16] INFO [2019-12-17 12:32:34,773] jepsen worker 51 - jepsen.util 6081 :invoke :cas [17 [44 26]] INFO [2019-12-17 12:32:34,802] jepsen worker 88 - jepsen.util 6478 :ok :read [29 46] INFO [2019-12-17 12:32:34,802] jepsen worker 88 - jepsen.util 6478 :invoke :cas [29 [30 49]] INFO [2019-12-17 12:32:34,806] jepsen worker 19 - jepsen.util 6049 :ok :write [6 24] INFO [2019-12-17 12:32:34,806] jepsen worker 19 - jepsen.util 6049 :invoke :write [6 22] INFO [2019-12-17 12:32:34,811] jepsen worker 47 - jepsen.util 6887 :fail :read [15 nil] :RequestTimeoutException INFO [2019-12-17 12:32:34,811] jepsen worker 74 - jepsen.util 6194 :info :write [24 33] :RequestCanceledException INFO [2019-12-17 12:32:34,811] jepsen worker 70 - jepsen.util 6550 :info :write [23 14] :RequestCanceledException INFO [2019-12-17 12:32:34,813] jepsen worker 47 - jepsen.util 6887 :invoke :read [15 nil] INFO [2019-12-17 12:32:34,813] jepsen worker 74 - jepsen.util 6284 :invoke :cas [24 [14 40]] INFO [2019-12-17 12:32:34,817] jepsen worker 70 - jepsen.util 6640 :invoke :cas [23 [14 33]] INFO [2019-12-17 12:32:34,829] jepsen worker 68 - jepsen.util 7898 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:32:34,835] jepsen worker 68 - jepsen.util 7898 :invoke :read [22 nil] INFO [2019-12-17 12:32:34,837] jepsen worker 72 - jepsen.util 6102 :ok :write [24 2] INFO [2019-12-17 12:32:34,837] jepsen worker 72 - jepsen.util 6102 :invoke :read [24 nil] INFO [2019-12-17 12:32:34,858] jepsen worker 36 - jepsen.util 6696 :info :write [12 7] :RequestCanceledException INFO [2019-12-17 12:32:34,862] jepsen worker 36 - jepsen.util 6786 :invoke :write [12 49] INFO [2019-12-17 12:32:34,881] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":35503,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:32:34,882] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":35503,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:32:34,882] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":35506,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:32:34,882] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":35506,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:32:34,897] jepsen worker 72 - jepsen.util 6102 :ok :read [24 2] INFO [2019-12-17 12:32:34,898] jepsen worker 72 - jepsen.util 6102 :invoke :read [24 nil] INFO [2019-12-17 12:32:34,898] jepsen worker 47 - jepsen.util 6887 :ok :read [15 19] INFO [2019-12-17 12:32:34,900] jepsen worker 47 - jepsen.util 6887 :invoke :read [15 nil] INFO [2019-12-17 12:32:34,913] jepsen worker 29 - jepsen.util 4709 :ok :write [9 41] INFO [2019-12-17 12:32:34,913] jepsen worker 29 - jepsen.util 4709 :invoke :read [9 nil] INFO [2019-12-17 12:32:34,923] jepsen worker 65 - jepsen.util 5195 :ok :write [21 24] INFO [2019-12-17 12:32:34,924] jepsen worker 65 - jepsen.util 5195 :invoke :cas [21 [16 22]] INFO [2019-12-17 12:32:34,927] jepsen worker 66 - jepsen.util 7806 :info :write [22 2] :RequestCanceledException INFO [2019-12-17 12:32:34,928] jepsen worker 66 - jepsen.util 7896 :invoke :cas [22 [28 21]] INFO [2019-12-17 12:32:34,940] jepsen worker 72 - jepsen.util 6102 :ok :read [24 2] INFO [2019-12-17 12:32:34,941] jepsen worker 72 - jepsen.util 6102 :invoke :read [24 nil] INFO [2019-12-17 12:32:34,985] jepsen worker 74 - jepsen.util 6284 :fail :cas [24 [14 40]] :ValueNotSwapFrom INFO [2019-12-17 12:32:34,985] jepsen worker 74 - jepsen.util 6284 :invoke :read [24 nil] INFO [2019-12-17 12:32:34,993] jepsen worker 55 - jepsen.util 7165 :fail :cas [18 [37 43]] :ValueNotSwapFrom INFO [2019-12-17 12:32:34,993] jepsen worker 55 - jepsen.util 7165 :invoke :cas [18 [48 41]] INFO [2019-12-17 12:32:35,012] jepsen worker 85 - jepsen.util 9175 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:32:35,012] jepsen worker 85 - jepsen.util 9175 :invoke :cas [28 [11 36]] INFO [2019-12-17 12:32:35,025] jepsen worker 60 - jepsen.util 9150 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:32:35,026] jepsen worker 60 - jepsen.util 9150 :invoke :write [20 36] INFO [2019-12-17 12:32:35,027] jepsen worker 71 - jepsen.util 6821 :ok :read [23 35] INFO [2019-12-17 12:32:35,028] jepsen worker 71 - jepsen.util 6821 :invoke :write [23 16] INFO [2019-12-17 12:32:35,028] jepsen worker 47 - jepsen.util 6887 :ok :read [15 19] INFO [2019-12-17 12:32:35,028] jepsen worker 47 - jepsen.util 6887 :invoke :cas [15 [0 2]] INFO [2019-12-17 12:32:35,039] jepsen worker 29 - jepsen.util 4709 :ok :read [9 41] INFO [2019-12-17 12:32:35,040] jepsen worker 29 - jepsen.util 4709 :invoke :cas [9 [17 29]] INFO [2019-12-17 12:32:35,044] jepsen worker 78 - jepsen.util 6468 :info :write [26 47] :RequestCanceledException INFO [2019-12-17 12:32:35,047] jepsen worker 78 - jepsen.util 6558 :invoke :read [26 nil] INFO [2019-12-17 12:32:35,096] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":35517,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:32:35,096] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":35517,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:32:35,096] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":35520,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:32:35,096] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":35520,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:32:35,096] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":35521,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:32:35,096] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":35521,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:32:35,104] jepsen worker 13 - jepsen.util 6043 :ok :read [4 15] INFO [2019-12-17 12:32:35,104] jepsen worker 13 - jepsen.util 6043 :invoke :read [4 nil] INFO [2019-12-17 12:32:35,108] jepsen worker 65 - jepsen.util 5195 :fail :cas [21 [16 22]] :ValueNotSwapFrom INFO [2019-12-17 12:32:35,108] jepsen worker 80 - jepsen.util 6200 :fail :cas [26 [16 29]] :ValueNotSwapFrom INFO [2019-12-17 12:32:35,110] jepsen worker 65 - jepsen.util 5195 :invoke :read [21 nil] INFO [2019-12-17 12:32:35,110] jepsen worker 80 - jepsen.util 6200 :invoke :cas [26 [20 22]] INFO [2019-12-17 12:32:35,115] jepsen worker 72 - jepsen.util 6102 :ok :read [24 2] INFO [2019-12-17 12:32:35,115] jepsen worker 72 - jepsen.util 6102 :invoke :cas [24 [14 13]] INFO [2019-12-17 12:32:35,127] jepsen worker 8 - jepsen.util 7838 :info :write [2 32] :RequestCanceledException INFO [2019-12-17 12:32:35,127] jepsen worker 43 - jepsen.util 7693 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:32:35,127] jepsen worker 44 - jepsen.util 9044 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:32:35,128] jepsen worker 43 - jepsen.util 7693 :invoke :cas [14 [13 21]] INFO [2019-12-17 12:32:35,128] jepsen worker 44 - jepsen.util 9044 :invoke :cas [14 [3 25]] INFO [2019-12-17 12:32:35,128] jepsen worker 8 - jepsen.util 7928 :invoke :cas [2 [13 27]] INFO [2019-12-17 12:32:35,139] jepsen worker 5 - jepsen.util 5765 :fail :cas [1 [39 26]] :ValueNotSwapFrom INFO [2019-12-17 12:32:35,139] jepsen worker 5 - jepsen.util 5765 :invoke :write [1 10] INFO [2019-12-17 12:32:35,139] jepsen worker 87 - jepsen.util 5577 :ok :read [29 46] INFO [2019-12-17 12:32:35,140] jepsen worker 87 - jepsen.util 5577 :invoke :write [29 45] INFO [2019-12-17 12:32:35,141] jepsen worker 75 - jepsen.util 7905 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:32:35,141] jepsen worker 34 - jepsen.util 7774 :info :cas [11 [12 29]] :RequestCanceledException INFO [2019-12-17 12:32:35,141] jepsen worker 75 - jepsen.util 7905 :invoke :read [25 nil] INFO [2019-12-17 12:32:35,142] jepsen worker 34 - jepsen.util 7864 :invoke :read [11 nil] INFO [2019-12-17 12:32:35,158] jepsen worker 48 - jepsen.util 7608 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:32:35,158] jepsen worker 48 - jepsen.util 7608 :invoke :read [16 nil] INFO [2019-12-17 12:32:35,167] jepsen worker 47 - jepsen.util 6887 :fail :cas [15 [0 2]] :ValueNotSwapFrom INFO [2019-12-17 12:32:35,167] jepsen worker 47 - jepsen.util 6887 :invoke :cas [15 [24 6]] INFO [2019-12-17 12:32:35,179] jepsen worker 74 - jepsen.util 6284 :ok :read [24 2] INFO [2019-12-17 12:32:35,179] jepsen worker 74 - jepsen.util 6284 :invoke :cas [24 [43 11]] INFO [2019-12-17 12:32:35,193] jepsen worker 72 - jepsen.util 6102 :fail :cas [24 [14 13]] :ValueNotSwapFrom INFO [2019-12-17 12:32:35,193] jepsen worker 72 - jepsen.util 6102 :invoke :read [24 nil] INFO [2019-12-17 12:32:35,193] jepsen worker 16 - jepsen.util 9286 :info :cas [5 [4 15]] :RequestCanceledException INFO [2019-12-17 12:32:35,194] jepsen worker 16 - jepsen.util 9376 :invoke :read [5 nil] INFO [2019-12-17 12:32:35,210] jepsen worker 70 - jepsen.util 6640 :fail :cas [23 [14 33]] :ValueNotSwapFrom INFO [2019-12-17 12:32:35,211] jepsen worker 70 - jepsen.util 6640 :invoke :write [23 42] INFO [2019-12-17 12:32:35,212] jepsen worker 82 - jepsen.util 8812 :info :write [27 33] :RequestCanceledException INFO [2019-12-17 12:32:35,213] jepsen worker 82 - jepsen.util 8902 :invoke :read [27 nil] INFO [2019-12-17 12:32:35,225] jepsen worker 57 - jepsen.util 7797 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:32:35,225] jepsen worker 76 - jepsen.util 8536 :info :cas [25 [22 5]] :RequestCanceledException INFO [2019-12-17 12:32:35,227] jepsen worker 57 - jepsen.util 7797 :invoke :cas [19 [36 42]] INFO [2019-12-17 12:32:35,228] jepsen worker 6 - jepsen.util 8916 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:32:35,228] jepsen worker 6 - jepsen.util 8916 :invoke :write [2 27] INFO [2019-12-17 12:32:35,230] jepsen worker 76 - jepsen.util 8626 :invoke :cas [25 [49 25]] INFO [2019-12-17 12:32:35,239] jepsen worker 29 - jepsen.util 4709 :fail :cas [9 [17 29]] :ValueNotSwapFrom INFO [2019-12-17 12:32:35,240] jepsen worker 29 - jepsen.util 4709 :invoke :cas [9 [19 18]] INFO [2019-12-17 12:32:35,251] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":35545,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:32:35,251] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":35545,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:32:35,289] jepsen worker 58 - jepsen.util 8788 :info :cas [19 [27 13]] :RequestCanceledException INFO [2019-12-17 12:32:35,290] jepsen worker 58 - jepsen.util 8878 :invoke :cas [19 [26 26]] INFO [2019-12-17 12:32:35,298] jepsen worker 65 - jepsen.util 5195 :ok :read [21 24] INFO [2019-12-17 12:32:35,301] jepsen worker 65 - jepsen.util 5195 :invoke :cas [21 [40 29]] INFO [2019-12-17 12:32:35,307] jepsen worker 72 - jepsen.util 6102 :ok :read [24 2] INFO [2019-12-17 12:32:35,307] jepsen worker 72 - jepsen.util 6102 :invoke :cas [24 [4 47]] INFO [2019-12-17 12:32:35,310] jepsen worker 47 - jepsen.util 6887 :fail :cas [15 [24 6]] :ValueNotSwapFrom INFO [2019-12-17 12:32:35,312] jepsen worker 41 - jepsen.util 9041 :info :write [13 21] :RequestCanceledException INFO [2019-12-17 12:32:35,312] jepsen worker 33 - jepsen.util 8313 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:32:35,312] jepsen worker 33 - jepsen.util 8313 :invoke :read [11 nil] INFO [2019-12-17 12:32:35,313] jepsen worker 47 - jepsen.util 6887 :invoke :cas [15 [21 29]] INFO [2019-12-17 12:32:35,313] jepsen worker 41 - jepsen.util 9131 :invoke :cas [13 [34 40]] INFO [2019-12-17 12:32:35,317] jepsen worker 87 - jepsen.util 5577 :ok :write [29 45] INFO [2019-12-17 12:32:35,317] jepsen worker 87 - jepsen.util 5577 :invoke :write [29 14] INFO [2019-12-17 12:32:35,319] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":35552,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:32:35,319] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":35552,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:32:35,319] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":35556,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:32:35,319] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":35556,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:32:35,327] jepsen worker 38 - jepsen.util 6158 :info :write [12 13] :RequestCanceledException INFO [2019-12-17 12:32:35,328] jepsen worker 38 - jepsen.util 6248 :invoke :read [12 nil] INFO [2019-12-17 12:32:35,340] jepsen worker 0 - jepsen.util 8640 :info :cas [0 [3 49]] :RequestCanceledException INFO [2019-12-17 12:32:35,341] jepsen worker 0 - jepsen.util 8730 :invoke :write [0 37] INFO [2019-12-17 12:32:35,347] jepsen worker 69 - jepsen.util 6189 :info :write [23 37] :RequestCanceledException INFO [2019-12-17 12:32:35,348] jepsen worker 74 - jepsen.util 6284 :fail :cas [24 [43 11]] :ValueNotSwapFrom INFO [2019-12-17 12:32:35,348] jepsen worker 74 - jepsen.util 6284 :invoke :cas [24 [24 11]] INFO [2019-12-17 12:32:35,356] jepsen worker 69 - jepsen.util 6279 :invoke :cas [23 [20 37]] INFO [2019-12-17 12:32:35,358] jepsen worker 89 - jepsen.util 6029 :ok :write [29 48] INFO [2019-12-17 12:32:35,358] jepsen worker 89 - jepsen.util 6029 :invoke :cas [29 [39 38]] INFO [2019-12-17 12:32:35,370] jepsen worker 64 - jepsen.util 4834 :ok :write [21 20] INFO [2019-12-17 12:32:35,370] jepsen worker 64 - jepsen.util 4834 :invoke :read [21 nil] INFO [2019-12-17 12:32:35,396] jepsen worker 29 - jepsen.util 4709 :fail :cas [9 [19 18]] :ValueNotSwapFrom INFO [2019-12-17 12:32:35,396] jepsen worker 29 - jepsen.util 4709 :invoke :read [9 nil] INFO [2019-12-17 12:32:35,423] jepsen worker 65 - jepsen.util 5195 :fail :cas [21 [40 29]] :ValueNotSwapFrom INFO [2019-12-17 12:32:35,424] jepsen worker 65 - jepsen.util 5195 :invoke :write [21 46] INFO [2019-12-17 12:32:35,427] jepsen worker 12 - jepsen.util 6402 :info :write [4 45] :RequestCanceledException INFO [2019-12-17 12:32:35,428] jepsen worker 12 - jepsen.util 6492 :invoke :cas [4 [44 28]] INFO [2019-12-17 12:32:35,429] jepsen worker 81 - jepsen.util 8001 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:32:35,429] jepsen worker 81 - jepsen.util 8001 :invoke :read [27 nil] INFO [2019-12-17 12:32:35,429] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":35578,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:32:35,429] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":35578,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:32:35,429] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":35581,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:32:35,429] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":35581,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:32:35,440] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:32:35,440] jepsen nemesis - couchbase.util Rebalance check count: 269 INFO [2019-12-17 12:32:35,440] jepsen nemesis - couchbase.util Rebalance stuck count: 250 INFO [2019-12-17 12:32:35,441] jepsen worker 28 - jepsen.util 4798 :fail :read [9 nil] :RequestTimeoutException INFO [2019-12-17 12:32:35,441] jepsen worker 28 - jepsen.util 4798 :invoke :cas [9 [20 22]] INFO [2019-12-17 12:32:35,450] jepsen worker 47 - jepsen.util 6887 :fail :cas [15 [21 29]] :ValueNotSwapFrom INFO [2019-12-17 12:32:35,450] jepsen worker 47 - jepsen.util 6887 :invoke :read [15 nil] INFO [2019-12-17 12:32:35,457] jepsen worker 72 - jepsen.util 6102 :fail :cas [24 [4 47]] :ValueNotSwapFrom INFO [2019-12-17 12:32:35,457] jepsen worker 72 - jepsen.util 6102 :invoke :cas [24 [17 8]] INFO [2019-12-17 12:32:35,457] jepsen worker 62 - jepsen.util 7892 :info :cas [20 [42 9]] :RequestCanceledException INFO [2019-12-17 12:32:35,459] jepsen worker 62 - jepsen.util 7982 :invoke :read [20 nil] INFO [2019-12-17 12:32:35,467] jepsen worker 74 - jepsen.util 6284 :fail :cas [24 [24 11]] :ValueNotSwapFrom INFO [2019-12-17 12:32:35,467] jepsen worker 74 - jepsen.util 6284 :invoke :write [24 27] INFO [2019-12-17 12:32:35,474] jepsen worker 64 - jepsen.util 4834 :ok :read [21 20] INFO [2019-12-17 12:32:35,474] jepsen worker 64 - jepsen.util 4834 :invoke :write [21 35] INFO [2019-12-17 12:32:35,486] jepsen worker 36 - jepsen.util 6786 :ok :write [12 49] INFO [2019-12-17 12:32:35,486] jepsen worker 36 - jepsen.util 6786 :invoke :read [12 nil] INFO [2019-12-17 12:32:35,494] jepsen worker 86 - jepsen.util 8636 :info :cas [28 [3 22]] :RequestCanceledException INFO [2019-12-17 12:32:35,494] jepsen worker 84 - jepsen.util 7914 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:32:35,494] jepsen worker 84 - jepsen.util 7914 :invoke :read [28 nil] INFO [2019-12-17 12:32:35,502] jepsen worker 86 - jepsen.util 8726 :invoke :write [28 2] INFO [2019-12-17 12:32:35,508] jepsen worker 80 - jepsen.util 6200 :fail :cas [26 [20 22]] :ValueNotSwapFrom INFO [2019-12-17 12:32:35,508] jepsen worker 80 - jepsen.util 6200 :invoke :cas [26 [38 20]] INFO [2019-12-17 12:32:35,509] jepsen worker 29 - jepsen.util 4709 :ok :read [9 41] INFO [2019-12-17 12:32:35,510] jepsen worker 29 - jepsen.util 4709 :invoke :read [9 nil] INFO [2019-12-17 12:32:35,511] jepsen worker 21 - jepsen.util 7851 :info :write [7 1] :RequestCanceledException INFO [2019-12-17 12:32:35,512] jepsen worker 69 - jepsen.util 6279 :fail :cas [23 [20 37]] :ValueNotSwapFrom INFO [2019-12-17 12:32:35,512] jepsen worker 69 - jepsen.util 6279 :invoke :read [23 nil] INFO [2019-12-17 12:32:35,512] jepsen worker 21 - jepsen.util 7941 :invoke :cas [7 [22 34]] INFO [2019-12-17 12:32:35,526] jepsen worker 89 - jepsen.util 6029 :fail :cas [29 [39 38]] :ValueNotSwapFrom INFO [2019-12-17 12:32:35,526] jepsen worker 89 - jepsen.util 6029 :invoke :write [29 35] INFO [2019-12-17 12:32:35,527] jepsen worker 9 - jepsen.util 6579 :info :write [3 3] :RequestCanceledException INFO [2019-12-17 12:32:35,527] jepsen worker 79 - jepsen.util 6109 :info :write [26 22] :RequestCanceledException INFO [2019-12-17 12:32:35,527] jepsen worker 20 - jepsen.util 6050 :info :write [6 20] :RequestCanceledException INFO [2019-12-17 12:32:35,527] jepsen worker 61 - jepsen.util 9421 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:32:35,527] jepsen worker 61 - jepsen.util 9421 :invoke :write [20 17] INFO [2019-12-17 12:32:35,528] jepsen worker 9 - jepsen.util 6669 :invoke :write [3 49] INFO [2019-12-17 12:32:35,531] jepsen worker 20 - jepsen.util 6140 :invoke :cas [6 [13 7]] INFO [2019-12-17 12:32:35,535] jepsen worker 79 - jepsen.util 6199 :invoke :cas [26 [37 39]] INFO [2019-12-17 12:32:35,540] jepsen worker 1 - jepsen.util 8011 :info :write [0 33] :RequestCanceledException INFO [2019-12-17 12:32:35,541] jepsen worker 1 - jepsen.util 8101 :invoke :cas [0 [25 42]] INFO [2019-12-17 12:32:35,556] jepsen worker 72 - jepsen.util 6102 :fail :cas [24 [17 8]] :ValueNotSwapFrom INFO [2019-12-17 12:32:35,556] jepsen worker 72 - jepsen.util 6102 :invoke :cas [24 [19 23]] INFO [2019-12-17 12:32:35,558] jepsen worker 83 - jepsen.util 8633 :info :cas [27 [12 3]] :RequestCanceledException INFO [2019-12-17 12:32:35,559] jepsen worker 83 - jepsen.util 8723 :invoke :write [27 30] INFO [2019-12-17 12:32:35,566] jepsen worker 29 - jepsen.util 4709 :ok :read [9 41] INFO [2019-12-17 12:32:35,566] jepsen worker 29 - jepsen.util 4709 :invoke :read [9 nil] INFO [2019-12-17 12:32:35,577] jepsen worker 47 - jepsen.util 6887 :ok :read [15 19] INFO [2019-12-17 12:32:35,577] jepsen worker 47 - jepsen.util 6887 :invoke :cas [15 [33 16]] INFO [2019-12-17 12:32:35,581] jepsen worker 19 - jepsen.util 6049 :ok :write [6 22] INFO [2019-12-17 12:32:35,581] jepsen worker 19 - jepsen.util 6049 :invoke :cas [6 [38 7]] INFO [2019-12-17 12:32:35,591] jepsen worker 36 - jepsen.util 6786 :ok :read [12 49] INFO [2019-12-17 12:32:35,592] jepsen worker 36 - jepsen.util 6786 :invoke :read [12 nil] INFO [2019-12-17 12:32:35,611] jepsen worker 69 - jepsen.util 6279 :ok :read [23 35] INFO [2019-12-17 12:32:35,612] jepsen worker 69 - jepsen.util 6279 :invoke :cas [23 [23 8]] INFO [2019-12-17 12:32:35,612] jepsen worker 77 - jepsen.util 8627 :info :write [25 43] :RequestCanceledException INFO [2019-12-17 12:32:35,613] jepsen worker 77 - jepsen.util 8717 :invoke :cas [25 [9 33]] INFO [2019-12-17 12:32:35,626] jepsen worker 29 - jepsen.util 4709 :ok :read [9 41] INFO [2019-12-17 12:32:35,626] jepsen worker 39 - jepsen.util 7779 :info :write [13 31] :RequestCanceledException INFO [2019-12-17 12:32:35,626] jepsen worker 29 - jepsen.util 4709 :invoke :write [9 1] INFO [2019-12-17 12:32:35,626] jepsen worker 73 - jepsen.util 5833 :fail :read [24 nil] :RequestTimeoutException INFO [2019-12-17 12:32:35,627] jepsen worker 73 - jepsen.util 5833 :invoke :cas [24 [45 34]] INFO [2019-12-17 12:32:35,627] jepsen worker 53 - jepsen.util 5723 :info :write [17 40] :RequestCanceledException INFO [2019-12-17 12:32:35,627] jepsen worker 40 - jepsen.util 8680 :info :cas [13 [13 43]] :RequestCanceledException INFO [2019-12-17 12:32:35,628] jepsen worker 39 - jepsen.util 7869 :invoke :write [13 37] INFO [2019-12-17 12:32:35,628] jepsen worker 53 - jepsen.util 5813 :invoke :cas [17 [36 41]] INFO [2019-12-17 12:32:35,628] jepsen worker 40 - jepsen.util 8770 :invoke :read [13 nil] INFO [2019-12-17 12:32:35,629] jepsen worker 13 - jepsen.util 6043 :ok :read [4 15] INFO [2019-12-17 12:32:35,629] jepsen worker 13 - jepsen.util 6043 :invoke :cas [4 [0 19]] INFO [2019-12-17 12:32:35,641] jepsen worker 22 - jepsen.util 8212 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:32:35,641] jepsen worker 22 - jepsen.util 8212 :invoke :write [7 1] INFO [2019-12-17 12:32:35,646] jepsen worker 52 - jepsen.util 6532 :info :write [17 40] :RequestCanceledException INFO [2019-12-17 12:32:35,654] jepsen worker 52 - jepsen.util 6622 :invoke :write [17 20] INFO [2019-12-17 12:32:35,655] jepsen worker 15 - jepsen.util 7305 :info :write [5 25] :RequestCanceledException INFO [2019-12-17 12:32:35,656] jepsen worker 15 - jepsen.util 7395 :invoke :read [5 nil] INFO [2019-12-17 12:32:35,679] jepsen worker 20 - jepsen.util 6140 :fail :cas [6 [13 7]] :ValueNotSwapFrom INFO [2019-12-17 12:32:35,679] jepsen worker 20 - jepsen.util 6140 :invoke :cas [6 [35 9]] INFO [2019-12-17 12:32:35,693] jepsen worker 7 - jepsen.util 8557 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:32:35,694] jepsen worker 7 - jepsen.util 8557 :invoke :read [2 nil] INFO [2019-12-17 12:32:35,697] jepsen worker 28 - jepsen.util 4798 :fail :cas [9 [20 22]] :ValueNotSwapFrom INFO [2019-12-17 12:32:35,697] jepsen worker 28 - jepsen.util 4798 :invoke :read [9 nil] INFO [2019-12-17 12:32:35,701] jepsen worker 55 - jepsen.util 7165 :fail :cas [18 [48 41]] :ValueNotSwapFrom INFO [2019-12-17 12:32:35,701] jepsen worker 55 - jepsen.util 7165 :invoke :write [18 3] INFO [2019-12-17 12:32:35,705] jepsen worker 53 - jepsen.util 5813 :fail :cas [17 [36 41]] :ValueNotSwapFrom INFO [2019-12-17 12:32:35,705] jepsen worker 53 - jepsen.util 5813 :invoke :write [17 26] INFO [2019-12-17 12:32:35,711] jepsen worker 17 - jepsen.util 8297 :info :write [5 44] :RequestCanceledException INFO [2019-12-17 12:32:35,712] jepsen worker 17 - jepsen.util 8387 :invoke :write [5 6] INFO [2019-12-17 12:32:35,719] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":35620,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:32:35,719] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":35620,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:32:35,727] jepsen worker 25 - jepsen.util 8755 :info :write [8 42] :RequestCanceledException INFO [2019-12-17 12:32:35,727] jepsen worker 63 - jepsen.util 5103 :info :write [21 34] :RequestCanceledException INFO [2019-12-17 12:32:35,734] jepsen worker 65 - jepsen.util 5195 :ok :write [21 46] INFO [2019-12-17 12:32:35,734] jepsen worker 65 - jepsen.util 5195 :invoke :read [21 nil] INFO [2019-12-17 12:32:35,736] jepsen worker 25 - jepsen.util 8845 :invoke :read [8 nil] INFO [2019-12-17 12:32:35,741] jepsen worker 19 - jepsen.util 6049 :fail :cas [6 [38 7]] :ValueNotSwapFrom INFO [2019-12-17 12:32:35,741] jepsen worker 19 - jepsen.util 6049 :invoke :cas [6 [33 14]] INFO [2019-12-17 12:32:35,742] jepsen worker 47 - jepsen.util 6887 :fail :cas [15 [33 16]] :ValueNotSwapFrom INFO [2019-12-17 12:32:35,742] jepsen worker 47 - jepsen.util 6887 :invoke :write [15 37] INFO [2019-12-17 12:32:35,742] jepsen worker 63 - jepsen.util 5193 :invoke :write [21 13] INFO [2019-12-17 12:32:35,758] jepsen worker 23 - jepsen.util 8393 :info :write [7 41] :RequestCanceledException INFO [2019-12-17 12:32:35,759] jepsen worker 23 - jepsen.util 8483 :invoke :read [7 nil] INFO [2019-12-17 12:32:35,761] jepsen worker 28 - jepsen.util 4798 :ok :read [9 41] INFO [2019-12-17 12:32:35,761] jepsen worker 28 - jepsen.util 4798 :invoke :cas [9 [6 28]] INFO [2019-12-17 12:32:35,771] jepsen worker 20 - jepsen.util 6140 :fail :cas [6 [35 9]] :ValueNotSwapFrom INFO [2019-12-17 12:32:35,771] jepsen worker 20 - jepsen.util 6140 :invoke :read [6 nil] INFO [2019-12-17 12:32:35,772] jepsen worker 14 - jepsen.util 6314 :ok :write [4 37] INFO [2019-12-17 12:32:35,772] jepsen worker 14 - jepsen.util 6314 :invoke :cas [4 [49 26]] INFO [2019-12-17 12:32:35,778] jepsen worker 73 - jepsen.util 5833 :fail :cas [24 [45 34]] :ValueNotSwapFrom INFO [2019-12-17 12:32:35,778] jepsen worker 73 - jepsen.util 5833 :invoke :read [24 nil] INFO [2019-12-17 12:32:35,793] jepsen worker 67 - jepsen.util 8437 :info :cas [22 [48 20]] :RequestCanceledException INFO [2019-12-17 12:32:35,794] jepsen worker 67 - jepsen.util 8527 :invoke :write [22 6] INFO [2019-12-17 12:32:35,819] jepsen worker 65 - jepsen.util 5195 :ok :read [21 46] INFO [2019-12-17 12:32:35,819] jepsen worker 65 - jepsen.util 5195 :invoke :read [21 nil] INFO [2019-12-17 12:32:35,827] jepsen worker 26 - jepsen.util 8666 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:32:35,827] jepsen worker 42 - jepsen.util 8862 :info :cas [14 [24 18]] :RequestCanceledException INFO [2019-12-17 12:32:35,827] jepsen worker 26 - jepsen.util 8666 :invoke :read [8 nil] INFO [2019-12-17 12:32:35,828] jepsen worker 42 - jepsen.util 8952 :invoke :read [14 nil] INFO [2019-12-17 12:32:35,842] jepsen worker 73 - jepsen.util 5833 :ok :read [24 2] INFO [2019-12-17 12:32:35,842] jepsen worker 73 - jepsen.util 5833 :invoke :cas [24 [5 24]] INFO [2019-12-17 12:32:35,846] jepsen worker 29 - jepsen.util 4709 :ok :write [9 1] INFO [2019-12-17 12:32:35,846] jepsen worker 29 - jepsen.util 4709 :invoke :read [9 nil] INFO [2019-12-17 12:32:35,853] jepsen worker 19 - jepsen.util 6049 :fail :cas [6 [33 14]] :ValueNotSwapFrom INFO [2019-12-17 12:32:35,853] jepsen worker 19 - jepsen.util 6049 :invoke :write [6 32] INFO [2019-12-17 12:32:35,855] jepsen worker 46 - jepsen.util 6166 :info :cas [15 [19 8]] :RequestCanceledException INFO [2019-12-17 12:32:35,855] jepsen worker 64 - jepsen.util 4834 :ok :write [21 35] INFO [2019-12-17 12:32:35,855] jepsen worker 64 - jepsen.util 4834 :invoke :write [21 41] INFO [2019-12-17 12:32:35,858] jepsen worker 46 - jepsen.util 6256 :invoke :cas [15 [29 45]] INFO [2019-12-17 12:32:35,859] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":35652,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:32:35,859] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":35652,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:32:35,859] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":35655,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:32:35,859] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":35655,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:32:35,859] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":35656,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:32:35,859] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":35656,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:32:35,889] jepsen worker 29 - jepsen.util 4709 :ok :read [9 1] INFO [2019-12-17 12:32:35,889] jepsen worker 29 - jepsen.util 4709 :invoke :cas [9 [11 3]] INFO [2019-12-17 12:32:35,899] jepsen worker 69 - jepsen.util 6279 :fail :cas [23 [23 8]] :ValueNotSwapFrom INFO [2019-12-17 12:32:35,899] jepsen worker 69 - jepsen.util 6279 :invoke :read [23 nil] INFO [2019-12-17 12:32:35,911] jepsen worker 59 - jepsen.util 8429 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:32:35,911] jepsen worker 59 - jepsen.util 8429 :invoke :cas [19 [28 48]] INFO [2019-12-17 12:32:35,927] jepsen worker 24 - jepsen.util 7404 :info :cas [8 [3 22]] :RequestCanceledException INFO [2019-12-17 12:32:35,928] jepsen worker 24 - jepsen.util 7494 :invoke :cas [8 [12 24]] INFO [2019-12-17 12:32:35,934] jepsen worker 80 - jepsen.util 6200 :fail :cas [26 [38 20]] :ValueNotSwapFrom INFO [2019-12-17 12:32:35,935] jepsen worker 80 - jepsen.util 6200 :invoke :cas [26 [18 22]] INFO [2019-12-17 12:32:35,941] jepsen worker 35 - jepsen.util 8315 :info :write [11 35] :RequestCanceledException INFO [2019-12-17 12:32:35,942] jepsen worker 35 - jepsen.util 8405 :invoke :read [11 nil] INFO [2019-12-17 12:32:35,948] jepsen worker 28 - jepsen.util 4798 :fail :cas [9 [6 28]] :ValueNotSwapFrom INFO [2019-12-17 12:32:35,948] jepsen worker 28 - jepsen.util 4798 :invoke :cas [9 [24 43]] INFO [2019-12-17 12:32:35,956] jepsen worker 65 - jepsen.util 5195 :ok :read [21 46] INFO [2019-12-17 12:32:35,957] jepsen worker 65 - jepsen.util 5195 :invoke :read [21 nil] INFO [2019-12-17 12:32:35,958] jepsen worker 49 - jepsen.util 7339 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:32:35,958] jepsen worker 49 - jepsen.util 7339 :invoke :cas [16 [3 25]] INFO [2019-12-17 12:32:35,961] jepsen worker 29 - jepsen.util 4709 :fail :cas [9 [11 3]] :ValueNotSwapFrom INFO [2019-12-17 12:32:35,961] jepsen worker 29 - jepsen.util 4709 :invoke :write [9 19] INFO [2019-12-17 12:32:35,965] jepsen worker 36 - jepsen.util 6786 :ok :read [12 49] INFO [2019-12-17 12:32:35,965] jepsen worker 36 - jepsen.util 6786 :invoke :read [12 nil] INFO [2019-12-17 12:32:35,997] jepsen worker 56 - jepsen.util 6626 :ok :write [18 21] INFO [2019-12-17 12:32:35,997] jepsen worker 56 - jepsen.util 6626 :invoke :cas [18 [26 44]] INFO [2019-12-17 12:32:36,018] jepsen worker 73 - jepsen.util 5833 :fail :cas [24 [5 24]] :ValueNotSwapFrom INFO [2019-12-17 12:32:36,018] jepsen worker 73 - jepsen.util 5833 :invoke :read [24 nil] INFO [2019-12-17 12:32:36,025] jepsen worker 89 - jepsen.util 6029 :ok :write [29 35] INFO [2019-12-17 12:32:36,025] jepsen worker 89 - jepsen.util 6029 :invoke :write [29 2] INFO [2019-12-17 12:32:36,029] jepsen worker 46 - jepsen.util 6256 :fail :cas [15 [29 45]] :ValueNotSwapFrom INFO [2019-12-17 12:32:36,029] jepsen worker 46 - jepsen.util 6256 :invoke :cas [15 [1 43]] INFO [2019-12-17 12:32:36,044] jepsen worker 5 - jepsen.util 5765 :ok :write [1 10] INFO [2019-12-17 12:32:36,044] jepsen worker 5 - jepsen.util 5765 :invoke :cas [1 [35 22]] INFO [2019-12-17 12:32:36,054] jepsen worker 9 - jepsen.util 6669 :ok :write [3 49] INFO [2019-12-17 12:32:36,055] jepsen worker 9 - jepsen.util 6669 :invoke :read [3 nil] INFO [2019-12-17 12:32:36,058] jepsen worker 2 - jepsen.util 8012 :info :cas [0 [22 47]] :RequestCanceledException INFO [2019-12-17 12:32:36,060] jepsen worker 2 - jepsen.util 8102 :invoke :read [0 nil] INFO [2019-12-17 12:32:36,068] jepsen worker 80 - jepsen.util 6200 :fail :cas [26 [18 22]] :ValueNotSwapFrom INFO [2019-12-17 12:32:36,068] jepsen worker 80 - jepsen.util 6200 :invoke :write [26 44] INFO [2019-12-17 12:32:36,077] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":35670,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:32:36,078] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":35670,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:32:36,078] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":35673,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:32:36,078] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":35673,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:32:36,078] jepsen worker 65 - jepsen.util 5195 :ok :read [21 35] INFO [2019-12-17 12:32:36,079] jepsen worker 65 - jepsen.util 5195 :invoke :write [21 29] INFO [2019-12-17 12:32:36,097] jepsen worker 38 - jepsen.util 6248 :ok :read [12 49] INFO [2019-12-17 12:32:36,097] jepsen worker 38 - jepsen.util 6248 :invoke :write [12 16] INFO [2019-12-17 12:32:36,100] jepsen worker 36 - jepsen.util 6786 :ok :read [12 49] INFO [2019-12-17 12:32:36,100] jepsen worker 36 - jepsen.util 6786 :invoke :cas [12 [34 41]] INFO [2019-12-17 12:32:36,111] jepsen worker 10 - jepsen.util 6040 :info :cas [3 [46 1]] :RequestCanceledException INFO [2019-12-17 12:32:36,112] jepsen worker 10 - jepsen.util 6130 :invoke :read [3 nil] INFO [2019-12-17 12:32:36,117] jepsen worker 4 - jepsen.util 5674 :ok :write [1 40] INFO [2019-12-17 12:32:36,118] jepsen worker 4 - jepsen.util 5674 :invoke :cas [1 [49 48]] INFO [2019-12-17 12:32:36,137] jepsen worker 73 - jepsen.util 5833 :ok :read [24 2] INFO [2019-12-17 12:32:36,137] jepsen worker 73 - jepsen.util 5833 :invoke :read [24 nil] INFO [2019-12-17 12:32:36,142] jepsen worker 56 - jepsen.util 6626 :fail :cas [18 [26 44]] :ValueNotSwapFrom INFO [2019-12-17 12:32:36,142] jepsen worker 56 - jepsen.util 6626 :invoke :read [18 nil] INFO [2019-12-17 12:32:36,155] jepsen worker 46 - jepsen.util 6256 :fail :cas [15 [1 43]] :ValueNotSwapFrom INFO [2019-12-17 12:32:36,156] jepsen worker 46 - jepsen.util 6256 :invoke :cas [15 [20 22]] INFO [2019-12-17 12:32:36,157] jepsen worker 9 - jepsen.util 6669 :ok :read [3 49] INFO [2019-12-17 12:32:36,157] jepsen worker 9 - jepsen.util 6669 :invoke :write [3 18] INFO [2019-12-17 12:32:36,175] jepsen worker 69 - jepsen.util 6279 :ok :read [23 37] INFO [2019-12-17 12:32:36,175] jepsen worker 69 - jepsen.util 6279 :invoke :cas [23 [10 13]] INFO [2019-12-17 12:32:36,189] jepsen worker 64 - jepsen.util 4834 :ok :write [21 41] INFO [2019-12-17 12:32:36,190] jepsen worker 64 - jepsen.util 4834 :invoke :read [21 nil] INFO [2019-12-17 12:32:36,194] jepsen worker 31 - jepsen.util 5521 :info :write [10 3] :RequestCanceledException INFO [2019-12-17 12:32:36,195] jepsen worker 31 - jepsen.util 5611 :invoke :read [10 nil] INFO [2019-12-17 12:32:36,195] jepsen worker 51 - jepsen.util 6081 :fail :cas [17 [44 26]] :ValueNotSwapFrom INFO [2019-12-17 12:32:36,195] jepsen worker 51 - jepsen.util 6081 :invoke :read [17 nil] INFO [2019-12-17 12:32:36,216] jepsen worker 64 - jepsen.util 4834 :ok :read [21 41] INFO [2019-12-17 12:32:36,216] jepsen worker 64 - jepsen.util 4834 :invoke :read [21 nil] INFO [2019-12-17 12:32:36,229] jepsen worker 88 - jepsen.util 6478 :fail :cas [29 [30 49]] :ValueNotSwapFrom INFO [2019-12-17 12:32:36,229] jepsen worker 88 - jepsen.util 6478 :invoke :cas [29 [29 8]] INFO [2019-12-17 12:32:36,238] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":35694,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:32:36,238] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":35694,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:32:36,238] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":35696,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:32:36,238] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":35696,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:32:36,239] jepsen worker 29 - jepsen.util 4709 :ok :write [9 19] INFO [2019-12-17 12:32:36,239] jepsen worker 29 - jepsen.util 4709 :invoke :write [9 48] INFO [2019-12-17 12:32:36,239] jepsen worker 37 - jepsen.util 5887 :info :write [12 37] :RequestCanceledException INFO [2019-12-17 12:32:36,240] jepsen worker 37 - jepsen.util 5977 :invoke :write [12 37] INFO [2019-12-17 12:32:36,242] jepsen worker 64 - jepsen.util 4834 :ok :read [21 41] INFO [2019-12-17 12:32:36,242] jepsen worker 64 - jepsen.util 4834 :invoke :cas [21 [36 27]] INFO [2019-12-17 12:32:36,257] jepsen worker 50 - jepsen.util 8690 :info :write [16 38] :RequestCanceledException INFO [2019-12-17 12:32:36,258] jepsen worker 50 - jepsen.util 8780 :invoke :write [16 0] INFO [2019-12-17 12:32:36,278] jepsen worker 65 - jepsen.util 5195 :ok :write [21 29] INFO [2019-12-17 12:32:36,279] jepsen worker 65 - jepsen.util 5195 :invoke :cas [21 [27 13]] INFO [2019-12-17 12:32:36,288] jepsen worker 31 - jepsen.util 5611 :ok :read [10 22] INFO [2019-12-17 12:32:36,288] jepsen worker 31 - jepsen.util 5611 :invoke :write [10 38] INFO [2019-12-17 12:32:36,289] jepsen worker 5 - jepsen.util 5765 :fail :cas [1 [35 22]] :ValueNotSwapFrom INFO [2019-12-17 12:32:36,289] jepsen worker 5 - jepsen.util 5765 :invoke :write [1 30] INFO [2019-12-17 12:32:36,295] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":35703,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:32:36,295] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":35703,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:32:36,295] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":35706,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:32:36,295] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":35706,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:32:36,295] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":35707,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:32:36,295] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":35707,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:32:36,304] jepsen worker 64 - jepsen.util 4834 :fail :cas [21 [36 27]] :ValueNotSwapFrom INFO [2019-12-17 12:32:36,304] jepsen worker 64 - jepsen.util 4834 :invoke :write [21 9] INFO [2019-12-17 12:32:36,326] jepsen worker 46 - jepsen.util 6256 :fail :cas [15 [20 22]] :ValueNotSwapFrom INFO [2019-12-17 12:32:36,327] jepsen worker 32 - jepsen.util 6242 :info :write [10 42] :RequestCanceledException INFO [2019-12-17 12:32:36,327] jepsen worker 46 - jepsen.util 6256 :invoke :read [15 nil] INFO [2019-12-17 12:32:36,328] jepsen worker 32 - jepsen.util 6332 :invoke :cas [10 [35 23]] INFO [2019-12-17 12:32:36,355] jepsen worker 78 - jepsen.util 6558 :ok :read [26 10] INFO [2019-12-17 12:32:36,355] jepsen worker 78 - jepsen.util 6558 :invoke :cas [26 [15 7]] INFO [2019-12-17 12:32:36,365] jepsen worker 65 - jepsen.util 5195 :fail :cas [21 [27 13]] :ValueNotSwapFrom INFO [2019-12-17 12:32:36,365] jepsen worker 65 - jepsen.util 5195 :invoke :read [21 nil] INFO [2019-12-17 12:32:36,393] jepsen worker 18 - jepsen.util 5958 :info :write [6 18] :RequestCanceledException INFO [2019-12-17 12:32:36,393] jepsen worker 30 - jepsen.util 6150 :fail :read [10 nil] :RequestTimeoutException INFO [2019-12-17 12:32:36,393] jepsen worker 30 - jepsen.util 6150 :invoke :read [10 nil] INFO [2019-12-17 12:32:36,394] jepsen worker 18 - jepsen.util 6048 :invoke :cas [6 [26 28]] INFO [2019-12-17 12:32:36,407] jepsen worker 47 - jepsen.util 6887 :ok :write [15 37] INFO [2019-12-17 12:32:36,408] jepsen worker 29 - jepsen.util 4709 :ok :write [9 48] INFO [2019-12-17 12:32:36,408] jepsen worker 47 - jepsen.util 6887 :invoke :read [15 nil] INFO [2019-12-17 12:32:36,408] jepsen worker 29 - jepsen.util 4709 :invoke :read [9 nil] INFO [2019-12-17 12:32:36,408] jepsen worker 46 - jepsen.util 6256 :ok :read [15 19] INFO [2019-12-17 12:32:36,409] jepsen worker 46 - jepsen.util 6256 :invoke :read [15 nil] INFO [2019-12-17 12:32:36,410] jepsen worker 52 - jepsen.util 6622 :ok :write [17 20] INFO [2019-12-17 12:32:36,411] jepsen worker 52 - jepsen.util 6622 :invoke :write [17 37] INFO [2019-12-17 12:32:36,428] jepsen worker 51 - jepsen.util 6081 :ok :read [17 20] INFO [2019-12-17 12:32:36,428] jepsen worker 51 - jepsen.util 6081 :invoke :write [17 30] INFO [2019-12-17 12:32:36,437] jepsen worker 65 - jepsen.util 5195 :ok :read [21 29] INFO [2019-12-17 12:32:36,437] jepsen worker 65 - jepsen.util 5195 :invoke :write [21 39] INFO [2019-12-17 12:32:36,473] jepsen worker 89 - jepsen.util 6029 :ok :write [29 2] INFO [2019-12-17 12:32:36,473] jepsen worker 89 - jepsen.util 6029 :invoke :write [29 43] INFO [2019-12-17 12:32:36,479] jepsen worker 88 - jepsen.util 6478 :fail :cas [29 [29 8]] :ValueNotSwapFrom INFO [2019-12-17 12:32:36,479] jepsen worker 88 - jepsen.util 6478 :invoke :cas [29 [20 33]] INFO [2019-12-17 12:32:36,495] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:32:36,496] jepsen nemesis - couchbase.util Rebalance check count: 270 INFO [2019-12-17 12:32:36,496] jepsen nemesis - couchbase.util Rebalance stuck count: 251 INFO [2019-12-17 12:32:36,508] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":35729,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:32:36,508] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":35729,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:32:36,508] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":35733,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:32:36,508] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":35733,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:32:36,520] jepsen worker 56 - jepsen.util 6626 :ok :read [18 21] INFO [2019-12-17 12:32:36,520] jepsen worker 56 - jepsen.util 6626 :invoke :write [18 27] INFO [2019-12-17 12:32:36,538] jepsen worker 36 - jepsen.util 6786 :fail :cas [12 [34 41]] :ValueNotSwapFrom INFO [2019-12-17 12:32:36,538] jepsen worker 36 - jepsen.util 6786 :invoke :cas [12 [48 17]] INFO [2019-12-17 12:32:36,546] jepsen worker 46 - jepsen.util 6256 :ok :read [15 37] INFO [2019-12-17 12:32:36,547] jepsen worker 46 - jepsen.util 6256 :invoke :write [15 2] INFO [2019-12-17 12:32:36,549] jepsen worker 32 - jepsen.util 6332 :fail :cas [10 [35 23]] :ValueNotSwapFrom INFO [2019-12-17 12:32:36,549] jepsen worker 32 - jepsen.util 6332 :invoke :read [10 nil] INFO [2019-12-17 12:32:36,556] jepsen worker 29 - jepsen.util 4709 :ok :read [9 48] INFO [2019-12-17 12:32:36,556] jepsen worker 29 - jepsen.util 4709 :invoke :read [9 nil] INFO [2019-12-17 12:32:36,557] jepsen worker 3 - jepsen.util 6033 :info :write [1 44] :RequestCanceledException INFO [2019-12-17 12:32:36,557] jepsen worker 45 - jepsen.util 6075 :info :write [15 38] :RequestCanceledException INFO [2019-12-17 12:32:36,558] jepsen worker 45 - jepsen.util 6165 :invoke :write [15 48] INFO [2019-12-17 12:32:36,558] jepsen worker 3 - jepsen.util 6123 :invoke :read [1 nil] INFO [2019-12-17 12:32:36,567] jepsen worker 78 - jepsen.util 6558 :fail :cas [26 [15 7]] :ValueNotSwapFrom INFO [2019-12-17 12:32:36,567] jepsen worker 78 - jepsen.util 6558 :invoke :write [26 16] INFO [2019-12-17 12:32:36,573] jepsen worker 64 - jepsen.util 4834 :ok :write [21 9] INFO [2019-12-17 12:32:36,574] jepsen worker 64 - jepsen.util 4834 :invoke :read [21 nil] INFO [2019-12-17 12:32:36,576] jepsen worker 29 - jepsen.util 4709 :ok :read [9 48] INFO [2019-12-17 12:32:36,576] jepsen worker 29 - jepsen.util 4709 :invoke :write [9 46] INFO [2019-12-17 12:32:36,627] jepsen worker 27 - jepsen.util 6057 :info :write [9 38] :RequestCanceledException INFO [2019-12-17 12:32:36,631] jepsen worker 27 - jepsen.util 6147 :invoke :write [9 46] INFO [2019-12-17 12:32:36,649] jepsen worker 47 - jepsen.util 6887 :ok :read [15 37] INFO [2019-12-17 12:32:36,649] jepsen worker 47 - jepsen.util 6887 :invoke :cas [15 [31 12]] INFO [2019-12-17 12:32:36,655] jepsen worker 11 - jepsen.util 6311 :info :cas [3 [17 46]] :RequestCanceledException INFO [2019-12-17 12:32:36,656] jepsen worker 11 - jepsen.util 6401 :invoke :read [3 nil] INFO [2019-12-17 12:32:36,675] jepsen worker 12 - jepsen.util 6492 :fail :cas [4 [44 28]] :ValueNotSwapFrom INFO [2019-12-17 12:32:36,676] jepsen worker 12 - jepsen.util 6492 :invoke :read [4 nil] INFO [2019-12-17 12:32:36,703] jepsen worker 18 - jepsen.util 6048 :fail :cas [6 [26 28]] :ValueNotSwapFrom INFO [2019-12-17 12:32:36,703] jepsen worker 18 - jepsen.util 6048 :invoke :write [6 5] INFO [2019-12-17 12:32:36,705] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":35749,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:32:36,705] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":35749,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:32:36,705] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":35754,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:32:36,705] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":35754,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:32:36,714] jepsen worker 32 - jepsen.util 6332 :ok :read [10 22] INFO [2019-12-17 12:32:36,715] jepsen worker 32 - jepsen.util 6332 :invoke :read [10 nil] INFO [2019-12-17 12:32:36,734] jepsen worker 65 - jepsen.util 5195 :ok :write [21 39] INFO [2019-12-17 12:32:36,735] jepsen worker 65 - jepsen.util 5195 :invoke :cas [21 [8 3]] INFO [2019-12-17 12:32:36,742] jepsen worker 69 - jepsen.util 6279 :fail :cas [23 [10 13]] :ValueNotSwapFrom INFO [2019-12-17 12:32:36,742] jepsen worker 69 - jepsen.util 6279 :invoke :cas [23 [24 10]] INFO [2019-12-17 12:32:36,755] jepsen worker 54 - jepsen.util 6714 :info :write [18 48] :RequestCanceledException INFO [2019-12-17 12:32:36,756] jepsen worker 54 - jepsen.util 6804 :invoke :write [18 23] INFO [2019-12-17 12:32:36,775] jepsen worker 79 - jepsen.util 6199 :fail :cas [26 [37 39]] :ValueNotSwapFrom INFO [2019-12-17 12:32:36,775] jepsen worker 79 - jepsen.util 6199 :invoke :write [26 16] INFO [2019-12-17 12:32:36,778] jepsen worker 47 - jepsen.util 6887 :fail :cas [15 [31 12]] :ValueNotSwapFrom INFO [2019-12-17 12:32:36,778] jepsen worker 47 - jepsen.util 6887 :invoke :read [15 nil] INFO [2019-12-17 12:32:36,809] jepsen worker 4 - jepsen.util 5674 :fail :cas [1 [49 48]] :ValueNotSwapFrom INFO [2019-12-17 12:32:36,809] jepsen worker 4 - jepsen.util 5674 :invoke :cas [1 [37 16]] INFO [2019-12-17 12:32:36,818] jepsen worker 32 - jepsen.util 6332 :ok :read [10 22] INFO [2019-12-17 12:32:36,818] jepsen worker 32 - jepsen.util 6332 :invoke :write [10 38] INFO [2019-12-17 12:32:36,835] jepsen worker 12 - jepsen.util 6492 :ok :read [4 37] INFO [2019-12-17 12:32:36,835] jepsen worker 12 - jepsen.util 6492 :invoke :write [4 13] INFO [2019-12-17 12:32:36,880] jepsen worker 64 - jepsen.util 4834 :ok :read [21 39] INFO [2019-12-17 12:32:36,881] jepsen worker 64 - jepsen.util 4834 :invoke :cas [21 [1 42]] INFO [2019-12-17 12:32:36,889] jepsen worker 19 - jepsen.util 6049 :ok :write [6 32] INFO [2019-12-17 12:32:36,890] jepsen worker 19 - jepsen.util 6049 :invoke :read [6 nil] INFO [2019-12-17 12:32:36,890] jepsen worker 47 - jepsen.util 6887 :ok :read [15 37] INFO [2019-12-17 12:32:36,891] jepsen worker 47 - jepsen.util 6887 :invoke :read [15 nil] INFO [2019-12-17 12:32:36,909] jepsen worker 29 - jepsen.util 4709 :ok :write [9 46] INFO [2019-12-17 12:32:36,909] jepsen worker 29 - jepsen.util 4709 :invoke :read [9 nil] INFO [2019-12-17 12:32:36,926] jepsen worker 68 - jepsen.util 7898 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:32:36,926] jepsen worker 68 - jepsen.util 7898 :invoke :write [22 28] INFO [2019-12-17 12:32:36,926] jepsen worker 3 - jepsen.util 6123 :ok :read [1 40] INFO [2019-12-17 12:32:36,927] jepsen worker 3 - jepsen.util 6123 :invoke :read [1 nil] INFO [2019-12-17 12:32:36,928] jepsen worker 88 - jepsen.util 6478 :fail :cas [29 [20 33]] :ValueNotSwapFrom INFO [2019-12-17 12:32:36,928] jepsen worker 88 - jepsen.util 6478 :invoke :write [29 6] INFO [2019-12-17 12:32:36,945] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":35769,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:32:36,945] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":35769,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:32:36,945] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":35772,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:32:36,945] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":35772,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:32:36,947] jepsen worker 10 - jepsen.util 6130 :ok :read [3 49] INFO [2019-12-17 12:32:36,948] jepsen worker 10 - jepsen.util 6130 :invoke :cas [3 [39 45]] INFO [2019-12-17 12:32:36,975] jepsen worker 29 - jepsen.util 4709 :ok :read [9 46] INFO [2019-12-17 12:32:36,976] jepsen worker 29 - jepsen.util 4709 :invoke :cas [9 [0 17]] INFO [2019-12-17 12:32:36,983] jepsen worker 19 - jepsen.util 6049 :ok :read [6 32] INFO [2019-12-17 12:32:36,983] jepsen worker 19 - jepsen.util 6049 :invoke :write [6 49] INFO [2019-12-17 12:32:37,027] jepsen worker 66 - jepsen.util 7896 :info :cas [22 [28 21]] :RequestCanceledException INFO [2019-12-17 12:32:37,028] jepsen worker 66 - jepsen.util 7986 :invoke :cas [22 [20 13]] INFO [2019-12-17 12:32:37,032] jepsen worker 29 - jepsen.util 4709 :fail :cas [9 [0 17]] :ValueNotSwapFrom INFO [2019-12-17 12:32:37,032] jepsen worker 29 - jepsen.util 4709 :invoke :read [9 nil] INFO [2019-12-17 12:32:37,054] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":35784,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:32:37,054] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":35784,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:32:37,054] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":35786,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:32:37,054] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":35786,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:32:37,061] jepsen worker 36 - jepsen.util 6786 :fail :cas [12 [48 17]] :ValueNotSwapFrom INFO [2019-12-17 12:32:37,061] jepsen worker 36 - jepsen.util 6786 :invoke :write [12 31] INFO [2019-12-17 12:32:37,083] jepsen worker 10 - jepsen.util 6130 :fail :cas [3 [39 45]] :ValueNotSwapFrom INFO [2019-12-17 12:32:37,090] jepsen worker 9 - jepsen.util 6669 :ok :write [3 18] INFO [2019-12-17 12:32:37,091] jepsen worker 29 - jepsen.util 4709 :ok :read [9 46] INFO [2019-12-17 12:32:37,093] jepsen worker 9 - jepsen.util 6669 :invoke :read [3 nil] INFO [2019-12-17 12:32:37,093] jepsen worker 10 - jepsen.util 6130 :invoke :write [3 14] INFO [2019-12-17 12:32:37,093] jepsen worker 29 - jepsen.util 4709 :invoke :cas [9 [1 28]] INFO [2019-12-17 12:32:37,103] jepsen worker 30 - jepsen.util 6150 :ok :read [10 22] INFO [2019-12-17 12:32:37,103] jepsen worker 30 - jepsen.util 6150 :invoke :cas [10 [19 46]] INFO [2019-12-17 12:32:37,111] jepsen worker 71 - jepsen.util 6821 :info :write [23 16] :RequestCanceledException INFO [2019-12-17 12:32:37,111] jepsen worker 85 - jepsen.util 9175 :info :cas [28 [11 36]] :RequestCanceledException INFO [2019-12-17 12:32:37,112] jepsen worker 71 - jepsen.util 6911 :invoke :cas [23 [31 2]] INFO [2019-12-17 12:32:37,112] jepsen worker 85 - jepsen.util 9265 :invoke :cas [28 [5 26]] INFO [2019-12-17 12:32:37,123] jepsen worker 14 - jepsen.util 6314 :fail :cas [4 [49 26]] :ValueNotSwapFrom INFO [2019-12-17 12:32:37,123] jepsen worker 14 - jepsen.util 6314 :invoke :write [4 23] INFO [2019-12-17 12:32:37,126] jepsen worker 60 - jepsen.util 9150 :info :write [20 36] :RequestCanceledException INFO [2019-12-17 12:32:37,127] jepsen worker 60 - jepsen.util 9240 :invoke :cas [20 [23 1]] INFO [2019-12-17 12:32:37,158] jepsen worker 8 - jepsen.util 7928 :info :cas [2 [13 27]] :RequestCanceledException INFO [2019-12-17 12:32:37,160] jepsen worker 8 - jepsen.util 8018 :invoke :write [2 34] INFO [2019-12-17 12:32:37,163] jepsen worker 11 - jepsen.util 6401 :ok :read [3 18] INFO [2019-12-17 12:32:37,163] jepsen worker 11 - jepsen.util 6401 :invoke :read [3 nil] INFO [2019-12-17 12:32:37,183] jepsen worker 29 - jepsen.util 4709 :fail :cas [9 [1 28]] :ValueNotSwapFrom INFO [2019-12-17 12:32:37,184] jepsen worker 29 - jepsen.util 4709 :invoke :write [9 29] INFO [2019-12-17 12:32:37,205] jepsen worker 69 - jepsen.util 6279 :fail :cas [23 [24 10]] :ValueNotSwapFrom INFO [2019-12-17 12:32:37,205] jepsen worker 69 - jepsen.util 6279 :invoke :read [23 nil] INFO [2019-12-17 12:32:37,212] jepsen worker 9 - jepsen.util 6669 :ok :read [3 18] INFO [2019-12-17 12:32:37,213] jepsen worker 9 - jepsen.util 6669 :invoke :write [3 25] INFO [2019-12-17 12:32:37,220] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":35805,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:32:37,220] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":35805,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:32:37,220] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":35806,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:32:37,220] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":35806,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:32:37,222] jepsen worker 30 - jepsen.util 6150 :fail :cas [10 [19 46]] :ValueNotSwapFrom INFO [2019-12-17 12:32:37,222] jepsen worker 30 - jepsen.util 6150 :invoke :write [10 46] INFO [2019-12-17 12:32:37,227] jepsen worker 43 - jepsen.util 7693 :info :cas [14 [13 21]] :RequestCanceledException INFO [2019-12-17 12:32:37,227] jepsen worker 44 - jepsen.util 9044 :info :cas [14 [3 25]] :RequestCanceledException INFO [2019-12-17 12:32:37,227] jepsen worker 34 - jepsen.util 7864 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:32:37,227] jepsen worker 34 - jepsen.util 7864 :invoke :cas [11 [5 35]] INFO [2019-12-17 12:32:37,228] jepsen worker 43 - jepsen.util 7783 :invoke :read [14 nil] INFO [2019-12-17 12:32:37,228] jepsen worker 44 - jepsen.util 9134 :invoke :read [14 nil] INFO [2019-12-17 12:32:37,241] jepsen worker 75 - jepsen.util 7905 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:32:37,241] jepsen worker 75 - jepsen.util 7905 :invoke :cas [25 [10 9]] INFO [2019-12-17 12:32:37,253] jepsen worker 11 - jepsen.util 6401 :ok :read [3 18] INFO [2019-12-17 12:32:37,253] jepsen worker 11 - jepsen.util 6401 :invoke :cas [3 [10 9]] INFO [2019-12-17 12:32:37,258] jepsen worker 48 - jepsen.util 7608 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:32:37,258] jepsen worker 48 - jepsen.util 7608 :invoke :write [16 33] INFO [2019-12-17 12:32:37,269] jepsen worker 4 - jepsen.util 5674 :fail :cas [1 [37 16]] :ValueNotSwapFrom INFO [2019-12-17 12:32:37,269] jepsen worker 4 - jepsen.util 5674 :invoke :read [1 nil] INFO [2019-12-17 12:32:37,273] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":35813,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:32:37,273] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":35813,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:32:37,284] jepsen worker 69 - jepsen.util 6279 :ok :read [23 37] INFO [2019-12-17 12:32:37,284] jepsen worker 69 - jepsen.util 6279 :invoke :cas [23 [42 39]] INFO [2019-12-17 12:32:37,293] jepsen worker 16 - jepsen.util 9376 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:32:37,293] jepsen worker 70 - jepsen.util 6640 :info :write [23 42] :RequestCanceledException INFO [2019-12-17 12:32:37,293] jepsen worker 16 - jepsen.util 9376 :invoke :write [5 11] INFO [2019-12-17 12:32:37,294] jepsen worker 70 - jepsen.util 6730 :invoke :read [23 nil] INFO [2019-12-17 12:32:37,311] jepsen worker 82 - jepsen.util 8902 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:32:37,312] jepsen worker 82 - jepsen.util 8902 :invoke :cas [27 [41 14]] INFO [2019-12-17 12:32:37,317] jepsen worker 29 - jepsen.util 4709 :ok :write [9 29] INFO [2019-12-17 12:32:37,317] jepsen worker 29 - jepsen.util 4709 :invoke :read [9 nil] INFO [2019-12-17 12:32:37,327] jepsen worker 6 - jepsen.util 8916 :info :write [2 27] :RequestCanceledException INFO [2019-12-17 12:32:37,327] jepsen worker 76 - jepsen.util 8626 :info :cas [25 [49 25]] :RequestCanceledException INFO [2019-12-17 12:32:37,327] jepsen worker 57 - jepsen.util 7797 :info :cas [19 [36 42]] :RequestCanceledException INFO [2019-12-17 12:32:37,328] jepsen worker 6 - jepsen.util 9006 :invoke :cas [2 [7 6]] INFO [2019-12-17 12:32:37,328] jepsen worker 57 - jepsen.util 7887 :invoke :cas [19 [2 10]] INFO [2019-12-17 12:32:37,328] jepsen worker 76 - jepsen.util 8716 :invoke :cas [25 [19 8]] INFO [2019-12-17 12:32:37,336] jepsen worker 32 - jepsen.util 6332 :ok :write [10 38] INFO [2019-12-17 12:32:37,336] jepsen worker 46 - jepsen.util 6256 :ok :write [15 2] INFO [2019-12-17 12:32:37,336] jepsen worker 32 - jepsen.util 6332 :invoke :read [10 nil] INFO [2019-12-17 12:32:37,336] jepsen worker 46 - jepsen.util 6256 :invoke :write [15 42] INFO [2019-12-17 12:32:37,340] jepsen worker 58 - jepsen.util 8878 :info :cas [19 [26 26]] :RequestCanceledException INFO [2019-12-17 12:32:37,341] jepsen worker 58 - jepsen.util 8968 :invoke :write [19 23] INFO [2019-12-17 12:32:37,344] jepsen worker 3 - jepsen.util 6123 :ok :read [1 40] INFO [2019-12-17 12:32:37,344] jepsen worker 3 - jepsen.util 6123 :invoke :read [1 nil] INFO [2019-12-17 12:32:37,358] jepsen worker 41 - jepsen.util 9131 :info :cas [13 [34 40]] :RequestCanceledException INFO [2019-12-17 12:32:37,359] jepsen worker 41 - jepsen.util 9221 :invoke :cas [13 [36 1]] INFO [2019-12-17 12:32:37,374] jepsen worker 29 - jepsen.util 4709 :ok :read [9 29] INFO [2019-12-17 12:32:37,375] jepsen worker 29 - jepsen.util 4709 :invoke :read [9 nil] INFO [2019-12-17 12:32:37,392] jepsen worker 0 - jepsen.util 8730 :info :write [0 37] :RequestCanceledException INFO [2019-12-17 12:32:37,392] jepsen worker 87 - jepsen.util 5577 :info :write [29 14] :RequestCanceledException INFO [2019-12-17 12:32:37,393] jepsen worker 87 - jepsen.util 5667 :invoke :read [29 nil] INFO [2019-12-17 12:32:37,398] jepsen worker 0 - jepsen.util 8820 :invoke :write [0 35] INFO [2019-12-17 12:32:37,411] jepsen worker 33 - jepsen.util 8313 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:32:37,411] jepsen worker 33 - jepsen.util 8313 :invoke :write [11 27] INFO [2019-12-17 12:32:37,434] jepsen worker 65 - jepsen.util 5195 :fail :cas [21 [8 3]] :ValueNotSwapFrom INFO [2019-12-17 12:32:37,434] jepsen worker 65 - jepsen.util 5195 :invoke :cas [21 [38 49]] INFO [2019-12-17 12:32:37,442] jepsen worker 32 - jepsen.util 6332 :ok :read [10 38] INFO [2019-12-17 12:32:37,442] jepsen worker 32 - jepsen.util 6332 :invoke :cas [10 [18 17]] INFO [2019-12-17 12:32:37,447] jepsen worker 29 - jepsen.util 4709 :ok :read [9 29] INFO [2019-12-17 12:32:37,447] jepsen worker 29 - jepsen.util 4709 :invoke :cas [9 [8 43]] INFO [2019-12-17 12:32:37,459] jepsen worker 12 - jepsen.util 6492 :ok :write [4 13] INFO [2019-12-17 12:32:37,459] jepsen worker 12 - jepsen.util 6492 :invoke :cas [4 [14 34]] INFO [2019-12-17 12:32:37,470] jepsen worker 32 - jepsen.util 6332 :fail :cas [10 [18 17]] :ValueNotSwapFrom INFO [2019-12-17 12:32:37,470] jepsen worker 32 - jepsen.util 6332 :invoke :cas [10 [48 45]] INFO [2019-12-17 12:32:37,474] jepsen worker 29 - jepsen.util 4709 :fail :cas [9 [8 43]] :ValueNotSwapFrom INFO [2019-12-17 12:32:37,474] jepsen worker 29 - jepsen.util 4709 :invoke :read [9 nil] INFO [2019-12-17 12:32:37,487] jepsen worker 32 - jepsen.util 6332 :fail :cas [10 [48 45]] :ValueNotSwapFrom INFO [2019-12-17 12:32:37,488] jepsen worker 32 - jepsen.util 6332 :invoke :cas [10 [48 45]] INFO [2019-12-17 12:32:37,490] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":35846,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:32:37,490] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":35846,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:32:37,494] jepsen worker 29 - jepsen.util 4709 :ok :read [9 29] INFO [2019-12-17 12:32:37,494] jepsen worker 29 - jepsen.util 4709 :invoke :read [9 nil] INFO [2019-12-17 12:32:37,499] jepsen worker 32 - jepsen.util 6332 :fail :cas [10 [48 45]] :ValueNotSwapFrom INFO [2019-12-17 12:32:37,499] jepsen worker 32 - jepsen.util 6332 :invoke :write [10 48] INFO [2019-12-17 12:32:37,507] jepsen worker 29 - jepsen.util 4709 :ok :read [9 29] INFO [2019-12-17 12:32:37,507] jepsen worker 29 - jepsen.util 4709 :invoke :read [9 nil] INFO [2019-12-17 12:32:37,527] jepsen worker 81 - jepsen.util 8001 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:32:37,527] jepsen worker 74 - jepsen.util 6284 :info :write [24 27] :RequestCanceledException INFO [2019-12-17 12:32:37,527] jepsen worker 21 - jepsen.util 7941 :info :cas [7 [22 34]] :RequestCanceledException INFO [2019-12-17 12:32:37,527] jepsen worker 81 - jepsen.util 8001 :invoke :cas [27 [26 39]] INFO [2019-12-17 12:32:37,528] jepsen worker 21 - jepsen.util 8031 :invoke :cas [7 [1 36]] INFO [2019-12-17 12:32:37,530] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:32:37,530] jepsen nemesis - couchbase.util Rebalance check count: 271 INFO [2019-12-17 12:32:37,530] jepsen nemesis - couchbase.util Rebalance stuck count: 252 INFO [2019-12-17 12:32:37,531] jepsen worker 13 - jepsen.util 6043 :fail :cas [4 [0 19]] :ValueNotSwapFrom INFO [2019-12-17 12:32:37,531] jepsen worker 13 - jepsen.util 6043 :invoke :read [4 nil] INFO [2019-12-17 12:32:37,540] jepsen worker 62 - jepsen.util 7982 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:32:37,541] jepsen worker 62 - jepsen.util 7982 :invoke :cas [20 [27 36]] INFO [2019-12-17 12:32:37,542] jepsen worker 29 - jepsen.util 4709 :ok :read [9 29] INFO [2019-12-17 12:32:37,542] jepsen worker 29 - jepsen.util 4709 :invoke :cas [9 [21 48]] INFO [2019-12-17 12:32:37,547] jepsen worker 74 - jepsen.util 6374 :invoke :read [24 nil] INFO [2019-12-17 12:32:37,558] jepsen worker 86 - jepsen.util 8726 :info :write [28 2] :RequestCanceledException INFO [2019-12-17 12:32:37,558] jepsen worker 86 - jepsen.util 8816 :invoke :write [28 32] INFO [2019-12-17 12:32:37,570] jepsen worker 29 - jepsen.util 4709 :fail :cas [9 [21 48]] :ValueNotSwapFrom INFO [2019-12-17 12:32:37,570] jepsen worker 29 - jepsen.util 4709 :invoke :write [9 34] INFO [2019-12-17 12:32:37,584] jepsen worker 74 - jepsen.util 6374 :ok :read [24 2] INFO [2019-12-17 12:32:37,584] jepsen worker 74 - jepsen.util 6374 :invoke :read [24 nil] INFO [2019-12-17 12:32:37,593] jepsen worker 89 - jepsen.util 6029 :ok :write [29 43] INFO [2019-12-17 12:32:37,593] jepsen worker 89 - jepsen.util 6029 :invoke :read [29 nil] INFO [2019-12-17 12:32:37,594] jepsen worker 84 - jepsen.util 7914 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:32:37,594] jepsen worker 72 - jepsen.util 6102 :info :cas [24 [19 23]] :RequestCanceledException INFO [2019-12-17 12:32:37,594] jepsen worker 84 - jepsen.util 7914 :invoke :cas [28 [5 27]] INFO [2019-12-17 12:32:37,595] jepsen worker 72 - jepsen.util 6192 :invoke :write [24 30] INFO [2019-12-17 12:32:37,621] jepsen worker 3 - jepsen.util 6123 :ok :read [1 40] INFO [2019-12-17 12:32:37,621] jepsen worker 3 - jepsen.util 6123 :invoke :read [1 nil] INFO [2019-12-17 12:32:37,627] jepsen worker 61 - jepsen.util 9421 :info :write [20 17] :RequestCanceledException INFO [2019-12-17 12:32:37,627] jepsen worker 77 - jepsen.util 8717 :info :cas [25 [9 33]] :RequestCanceledException INFO [2019-12-17 12:32:37,628] jepsen worker 61 - jepsen.util 9511 :invoke :read [20 nil] INFO [2019-12-17 12:32:37,628] jepsen worker 77 - jepsen.util 8807 :invoke :write [25 26] INFO [2019-12-17 12:32:37,641] jepsen worker 1 - jepsen.util 8101 :info :cas [0 [25 42]] :RequestCanceledException INFO [2019-12-17 12:32:37,642] jepsen worker 1 - jepsen.util 8191 :invoke :cas [0 [44 25]] INFO [2019-12-17 12:32:37,643] jepsen worker 69 - jepsen.util 6279 :fail :cas [23 [42 39]] :ValueNotSwapFrom INFO [2019-12-17 12:32:37,644] jepsen worker 69 - jepsen.util 6279 :invoke :cas [23 [49 20]] INFO [2019-12-17 12:32:37,646] jepsen worker 74 - jepsen.util 6374 :ok :read [24 2] INFO [2019-12-17 12:32:37,646] jepsen worker 74 - jepsen.util 6374 :invoke :cas [24 [2 21]] INFO [2019-12-17 12:32:37,658] jepsen worker 83 - jepsen.util 8723 :info :write [27 30] :RequestCanceledException INFO [2019-12-17 12:32:37,661] jepsen worker 83 - jepsen.util 8813 :invoke :write [27 14] INFO [2019-12-17 12:32:37,690] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":35875,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:32:37,690] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":35875,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:32:37,690] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":35878,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:32:37,691] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":35878,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:32:37,693] jepsen worker 39 - jepsen.util 7869 :info :write [13 37] :RequestCanceledException INFO [2019-12-17 12:32:37,694] jepsen worker 39 - jepsen.util 7959 :invoke :write [13 5] INFO [2019-12-17 12:32:37,695] jepsen worker 29 - jepsen.util 4709 :ok :write [9 34] INFO [2019-12-17 12:32:37,695] jepsen worker 29 - jepsen.util 4709 :invoke :cas [9 [13 2]] INFO [2019-12-17 12:32:37,726] jepsen worker 29 - jepsen.util 4709 :fail :cas [9 [13 2]] :ValueNotSwapFrom INFO [2019-12-17 12:32:37,726] jepsen worker 29 - jepsen.util 4709 :invoke :write [9 11] INFO [2019-12-17 12:32:37,727] jepsen worker 55 - jepsen.util 7165 :info :write [18 3] :RequestCanceledException INFO [2019-12-17 12:32:37,727] jepsen worker 40 - jepsen.util 8770 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:32:37,727] jepsen worker 40 - jepsen.util 8770 :invoke :read [13 nil] INFO [2019-12-17 12:32:37,728] jepsen worker 55 - jepsen.util 7255 :invoke :read [18 nil] INFO [2019-12-17 12:32:37,741] jepsen worker 22 - jepsen.util 8212 :info :write [7 1] :RequestCanceledException INFO [2019-12-17 12:32:37,742] jepsen worker 22 - jepsen.util 8302 :invoke :cas [7 [14 10]] INFO [2019-12-17 12:32:37,755] jepsen worker 78 - jepsen.util 6558 :ok :write [26 16] INFO [2019-12-17 12:32:37,755] jepsen worker 78 - jepsen.util 6558 :invoke :cas [26 [36 29]] INFO [2019-12-17 12:32:37,757] jepsen worker 15 - jepsen.util 7395 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:32:37,758] jepsen worker 15 - jepsen.util 7395 :invoke :write [5 15] INFO [2019-12-17 12:32:37,767] jepsen worker 12 - jepsen.util 6492 :fail :cas [4 [14 34]] :ValueNotSwapFrom INFO [2019-12-17 12:32:37,767] jepsen worker 3 - jepsen.util 6123 :ok :read [1 40] INFO [2019-12-17 12:32:37,767] jepsen worker 12 - jepsen.util 6492 :invoke :write [4 43] INFO [2019-12-17 12:32:37,767] jepsen worker 3 - jepsen.util 6123 :invoke :write [1 24] INFO [2019-12-17 12:32:37,780] jepsen worker 29 - jepsen.util 4709 :ok :write [9 11] INFO [2019-12-17 12:32:37,780] jepsen worker 29 - jepsen.util 4709 :invoke :cas [9 [12 41]] INFO [2019-12-17 12:32:37,793] jepsen worker 25 - jepsen.util 8845 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:32:37,793] jepsen worker 7 - jepsen.util 8557 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:32:37,793] jepsen worker 25 - jepsen.util 8845 :invoke :write [8 7] INFO [2019-12-17 12:32:37,793] jepsen worker 7 - jepsen.util 8557 :invoke :write [2 41] INFO [2019-12-17 12:32:37,794] jepsen worker 29 - jepsen.util 4709 :fail :cas [9 [12 41]] :ValueNotSwapFrom INFO [2019-12-17 12:32:37,794] jepsen worker 29 - jepsen.util 4709 :invoke :cas [9 [43 23]] INFO [2019-12-17 12:32:37,810] jepsen worker 63 - jepsen.util 5193 :info :write [21 13] :RequestCanceledException INFO [2019-12-17 12:32:37,810] jepsen worker 53 - jepsen.util 5813 :info :write [17 26] :RequestCanceledException INFO [2019-12-17 12:32:37,810] jepsen worker 20 - jepsen.util 6140 :fail :read [6 nil] :RequestTimeoutException INFO [2019-12-17 12:32:37,810] jepsen worker 20 - jepsen.util 6140 :invoke :write [6 22] INFO [2019-12-17 12:32:37,811] jepsen worker 63 - jepsen.util 5283 :invoke :read [21 nil] INFO [2019-12-17 12:32:37,812] jepsen worker 29 - jepsen.util 4709 :fail :cas [9 [43 23]] :ValueNotSwapFrom INFO [2019-12-17 12:32:37,812] jepsen worker 29 - jepsen.util 4709 :invoke :write [9 42] INFO [2019-12-17 12:32:37,815] jepsen worker 53 - jepsen.util 5903 :invoke :write [17 25] INFO [2019-12-17 12:32:37,827] jepsen worker 17 - jepsen.util 8387 :info :write [5 6] :RequestCanceledException INFO [2019-12-17 12:32:37,827] jepsen worker 23 - jepsen.util 8483 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:32:37,827] jepsen worker 23 - jepsen.util 8483 :invoke :read [7 nil] INFO [2019-12-17 12:32:37,828] jepsen worker 17 - jepsen.util 8477 :invoke :read [5 nil] INFO [2019-12-17 12:32:37,833] jepsen worker 71 - jepsen.util 6911 :fail :cas [23 [31 2]] :ValueNotSwapFrom INFO [2019-12-17 12:32:37,834] jepsen worker 71 - jepsen.util 6911 :invoke :read [23 nil] INFO [2019-12-17 12:32:37,836] jepsen worker 63 - jepsen.util 5283 :ok :read [21 39] INFO [2019-12-17 12:32:37,836] jepsen worker 63 - jepsen.util 5283 :invoke :read [21 nil] INFO [2019-12-17 12:32:37,841] jepsen worker 67 - jepsen.util 8527 :info :write [22 6] :RequestCanceledException INFO [2019-12-17 12:32:37,842] jepsen worker 67 - jepsen.util 8617 :invoke :cas [22 [42 32]] INFO [2019-12-17 12:32:37,858] jepsen worker 42 - jepsen.util 8952 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:32:37,858] jepsen worker 42 - jepsen.util 8952 :invoke :write [14 29] INFO [2019-12-17 12:32:37,861] jepsen worker 70 - jepsen.util 6730 :ok :read [23 37] INFO [2019-12-17 12:32:37,861] jepsen worker 70 - jepsen.util 6730 :invoke :write [23 5] INFO [2019-12-17 12:32:37,869] jepsen worker 29 - jepsen.util 4709 :ok :write [9 42] INFO [2019-12-17 12:32:37,870] jepsen worker 29 - jepsen.util 4709 :invoke :cas [9 [2 18]] INFO [2019-12-17 12:32:37,887] jepsen worker 63 - jepsen.util 5283 :ok :read [21 39] INFO [2019-12-17 12:32:37,887] jepsen worker 63 - jepsen.util 5283 :invoke :read [21 nil] INFO [2019-12-17 12:32:37,897] jepsen worker 87 - jepsen.util 5667 :ok :read [29 43] INFO [2019-12-17 12:32:37,897] jepsen worker 87 - jepsen.util 5667 :invoke :read [29 nil] INFO [2019-12-17 12:32:37,900] jepsen worker 29 - jepsen.util 4709 :fail :cas [9 [2 18]] :ValueNotSwapFrom INFO [2019-12-17 12:32:37,901] jepsen worker 29 - jepsen.util 4709 :invoke :cas [9 [15 23]] INFO [2019-12-17 12:32:37,901] jepsen worker 10 - jepsen.util 6130 :ok :write [3 14] INFO [2019-12-17 12:32:37,901] jepsen worker 10 - jepsen.util 6130 :invoke :write [3 46] INFO [2019-12-17 12:32:37,903] jepsen worker 18 - jepsen.util 6048 :ok :write [6 5] INFO [2019-12-17 12:32:37,903] jepsen worker 18 - jepsen.util 6048 :invoke :write [6 3] INFO [2019-12-17 12:32:37,903] jepsen worker 32 - jepsen.util 6332 :ok :write [10 48] INFO [2019-12-17 12:32:37,903] jepsen worker 32 - jepsen.util 6332 :invoke :write [10 37] INFO [2019-12-17 12:32:37,904] jepsen worker 46 - jepsen.util 6256 :ok :write [15 42] INFO [2019-12-17 12:32:37,904] jepsen worker 46 - jepsen.util 6256 :invoke :read [15 nil] INFO [2019-12-17 12:32:37,927] jepsen worker 26 - jepsen.util 8666 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:32:37,927] jepsen worker 26 - jepsen.util 8666 :invoke :write [8 48] INFO [2019-12-17 12:32:37,928] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":35909,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:32:37,928] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":35909,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:32:37,928] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":35913,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:32:37,928] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":35913,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:32:37,939] jepsen worker 65 - jepsen.util 5195 :fail :cas [21 [38 49]] :ValueNotSwapFrom INFO [2019-12-17 12:32:37,939] jepsen worker 65 - jepsen.util 5195 :invoke :write [21 7] INFO [2019-12-17 12:32:37,944] jepsen worker 63 - jepsen.util 5283 :ok :read [21 39] INFO [2019-12-17 12:32:37,944] jepsen worker 63 - jepsen.util 5283 :invoke :write [21 9] INFO [2019-12-17 12:32:37,953] jepsen worker 74 - jepsen.util 6374 :ok :cas [24 [2 21]] INFO [2019-12-17 12:32:37,953] jepsen worker 74 - jepsen.util 6374 :invoke :cas [24 [13 33]] INFO [2019-12-17 12:32:37,971] jepsen worker 46 - jepsen.util 6256 :ok :read [15 42] INFO [2019-12-17 12:32:37,971] jepsen worker 46 - jepsen.util 6256 :invoke :read [15 nil] INFO [2019-12-17 12:32:37,976] jepsen worker 29 - jepsen.util 4709 :fail :cas [9 [15 23]] :ValueNotSwapFrom INFO [2019-12-17 12:32:37,976] jepsen worker 29 - jepsen.util 4709 :invoke :write [9 5] INFO [2019-12-17 12:32:37,983] jepsen worker 74 - jepsen.util 6374 :fail :cas [24 [13 33]] :ValueNotSwapFrom INFO [2019-12-17 12:32:37,984] jepsen worker 74 - jepsen.util 6374 :invoke :write [24 38] INFO [2019-12-17 12:32:37,985] jepsen worker 64 - jepsen.util 4834 :fail :cas [21 [1 42]] :ValueNotSwapFrom INFO [2019-12-17 12:32:37,985] jepsen worker 64 - jepsen.util 4834 :invoke :cas [21 [8 29]] INFO [2019-12-17 12:32:37,993] jepsen worker 24 - jepsen.util 7494 :info :cas [8 [12 24]] :RequestCanceledException INFO [2019-12-17 12:32:37,994] jepsen worker 24 - jepsen.util 7584 :invoke :read [8 nil] INFO [2019-12-17 12:32:38,004] jepsen worker 65 - jepsen.util 5195 :ok :write [21 7] INFO [2019-12-17 12:32:38,004] jepsen worker 65 - jepsen.util 5195 :invoke :cas [21 [36 45]] INFO [2019-12-17 12:32:38,009] jepsen worker 59 - jepsen.util 8429 :info :cas [19 [28 48]] :RequestCanceledException INFO [2019-12-17 12:32:38,009] jepsen worker 35 - jepsen.util 8405 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:32:38,010] jepsen worker 35 - jepsen.util 8405 :invoke :read [11 nil] INFO [2019-12-17 12:32:38,011] jepsen worker 59 - jepsen.util 8519 :invoke :write [19 47] INFO [2019-12-17 12:32:38,023] jepsen worker 4 - jepsen.util 5674 :ok :read [1 40] INFO [2019-12-17 12:32:38,023] jepsen worker 4 - jepsen.util 5674 :invoke :read [1 nil] INFO [2019-12-17 12:32:38,028] jepsen worker 46 - jepsen.util 6256 :ok :read [15 42] INFO [2019-12-17 12:32:38,028] jepsen worker 46 - jepsen.util 6256 :invoke :write [15 35] INFO [2019-12-17 12:32:38,036] jepsen worker 38 - jepsen.util 6248 :ok :write [12 16] INFO [2019-12-17 12:32:38,036] jepsen worker 38 - jepsen.util 6248 :invoke :read [12 nil] INFO [2019-12-17 12:32:38,038] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":35934,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:32:38,039] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":35934,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:32:38,039] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":35937,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:32:38,039] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":35937,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:32:38,039] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":35938,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:32:38,039] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":35938,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:32:38,041] jepsen worker 28 - jepsen.util 4798 :info :cas [9 [24 43]] :RequestCanceledException INFO [2019-12-17 12:32:38,042] jepsen worker 28 - jepsen.util 4888 :invoke :write [9 13] INFO [2019-12-17 12:32:38,058] jepsen worker 49 - jepsen.util 7339 :info :cas [16 [3 25]] :RequestCanceledException INFO [2019-12-17 12:32:38,059] jepsen worker 49 - jepsen.util 7429 :invoke :cas [16 [26 39]] INFO [2019-12-17 12:32:38,067] jepsen worker 87 - jepsen.util 5667 :ok :read [29 43] INFO [2019-12-17 12:32:38,067] jepsen worker 87 - jepsen.util 5667 :invoke :cas [29 [34 34]] INFO [2019-12-17 12:32:38,088] jepsen worker 80 - jepsen.util 6200 :ok :write [26 44] INFO [2019-12-17 12:32:38,088] jepsen worker 80 - jepsen.util 6200 :invoke :read [26 nil] INFO [2019-12-17 12:32:38,126] jepsen worker 65 - jepsen.util 5195 :fail :cas [21 [36 45]] :ValueNotSwapFrom INFO [2019-12-17 12:32:38,126] jepsen worker 65 - jepsen.util 5195 :invoke :write [21 19] INFO [2019-12-17 12:32:38,127] jepsen worker 2 - jepsen.util 8102 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:32:38,127] jepsen worker 2 - jepsen.util 8102 :invoke :write [0 33] INFO [2019-12-17 12:32:38,139] jepsen worker 55 - jepsen.util 7255 :ok :read [18 21] INFO [2019-12-17 12:32:38,139] jepsen worker 55 - jepsen.util 7255 :invoke :cas [18 [34 32]] INFO [2019-12-17 12:32:38,151] jepsen worker 20 - jepsen.util 6140 :ok :write [6 22] INFO [2019-12-17 12:32:38,151] jepsen worker 20 - jepsen.util 6140 :invoke :read [6 nil] INFO [2019-12-17 12:32:38,151] jepsen worker 63 - jepsen.util 5283 :ok :write [21 9] INFO [2019-12-17 12:32:38,151] jepsen worker 63 - jepsen.util 5283 :invoke :read [21 nil] INFO [2019-12-17 12:32:38,171] jepsen worker 20 - jepsen.util 6140 :ok :read [6 22] INFO [2019-12-17 12:32:38,171] jepsen worker 20 - jepsen.util 6140 :invoke :write [6 13] INFO [2019-12-17 12:32:38,189] jepsen worker 89 - jepsen.util 6029 :ok :read [29 43] INFO [2019-12-17 12:32:38,189] jepsen worker 89 - jepsen.util 6029 :invoke :write [29 48] INFO [2019-12-17 12:32:38,206] jepsen worker 29 - jepsen.util 4709 :ok :write [9 5] INFO [2019-12-17 12:32:38,206] jepsen worker 29 - jepsen.util 4709 :invoke :cas [9 [36 14]] INFO [2019-12-17 12:32:38,207] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":35956,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:32:38,207] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":35956,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:32:38,207] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":35958,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:32:38,207] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":35958,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:32:38,222] jepsen worker 80 - jepsen.util 6200 :ok :read [26 44] INFO [2019-12-17 12:32:38,222] jepsen worker 80 - jepsen.util 6200 :invoke :write [26 36] INFO [2019-12-17 12:32:38,227] jepsen worker 73 - jepsen.util 5833 :fail :read [24 nil] :RequestTimeoutException INFO [2019-12-17 12:32:38,227] jepsen worker 73 - jepsen.util 5833 :invoke :read [24 nil] INFO [2019-12-17 12:32:38,233] jepsen worker 63 - jepsen.util 5283 :ok :read [21 9] INFO [2019-12-17 12:32:38,234] jepsen worker 63 - jepsen.util 5283 :invoke :read [21 nil] INFO [2019-12-17 12:32:38,249] jepsen worker 29 - jepsen.util 4709 :fail :cas [9 [36 14]] :ValueNotSwapFrom INFO [2019-12-17 12:32:38,249] jepsen worker 29 - jepsen.util 4709 :invoke :write [9 27] INFO [2019-12-17 12:32:38,254] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":35964,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:32:38,254] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":35964,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:32:38,254] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":35968,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:32:38,254] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":35968,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:32:38,278] jepsen worker 63 - jepsen.util 5283 :ok :read [21 9] INFO [2019-12-17 12:32:38,278] jepsen worker 63 - jepsen.util 5283 :invoke :read [21 nil] INFO [2019-12-17 12:32:38,279] jepsen worker 88 - jepsen.util 6478 :ok :write [29 6] INFO [2019-12-17 12:32:38,279] jepsen worker 88 - jepsen.util 6478 :invoke :read [29 nil] INFO [2019-12-17 12:32:38,283] jepsen worker 71 - jepsen.util 6911 :ok :read [23 37] INFO [2019-12-17 12:32:38,283] jepsen worker 71 - jepsen.util 6911 :invoke :write [23 47] INFO [2019-12-17 12:32:38,294] jepsen worker 50 - jepsen.util 8780 :info :write [16 0] :RequestCanceledException INFO [2019-12-17 12:32:38,295] jepsen worker 50 - jepsen.util 8870 :invoke :cas [16 [37 47]] INFO [2019-12-17 12:32:38,316] jepsen worker 78 - jepsen.util 6558 :fail :cas [26 [36 29]] :ValueNotSwapFrom INFO [2019-12-17 12:32:38,316] jepsen worker 78 - jepsen.util 6558 :invoke :read [26 nil] INFO [2019-12-17 12:32:38,327] jepsen worker 37 - jepsen.util 5977 :info :write [12 37] :RequestCanceledException INFO [2019-12-17 12:32:38,328] jepsen worker 37 - jepsen.util 6067 :invoke :read [12 nil] INFO [2019-12-17 12:32:38,329] jepsen worker 29 - jepsen.util 4709 :ok :write [9 27] INFO [2019-12-17 12:32:38,330] jepsen worker 29 - jepsen.util 4709 :invoke :read [9 nil] INFO [2019-12-17 12:32:38,354] jepsen worker 29 - jepsen.util 4709 :ok :read [9 27] INFO [2019-12-17 12:32:38,354] jepsen worker 29 - jepsen.util 4709 :invoke :write [9 48] INFO [2019-12-17 12:32:38,358] jepsen worker 31 - jepsen.util 5611 :info :write [10 38] :RequestCanceledException INFO [2019-12-17 12:32:38,358] jepsen worker 5 - jepsen.util 5765 :info :write [1 30] :RequestCanceledException INFO [2019-12-17 12:32:38,359] jepsen worker 31 - jepsen.util 5701 :invoke :cas [10 [15 30]] INFO [2019-12-17 12:32:38,359] jepsen worker 5 - jepsen.util 5855 :invoke :cas [1 [42 33]] INFO [2019-12-17 12:32:38,387] jepsen worker 38 - jepsen.util 6248 :ok :read [12 16] INFO [2019-12-17 12:32:38,387] jepsen worker 38 - jepsen.util 6248 :invoke :read [12 nil] INFO [2019-12-17 12:32:38,400] jepsen worker 10 - jepsen.util 6130 :ok :write [3 46] INFO [2019-12-17 12:32:38,400] jepsen worker 10 - jepsen.util 6130 :invoke :write [3 33] INFO [2019-12-17 12:32:38,401] jepsen worker 46 - jepsen.util 6256 :ok :write [15 35] INFO [2019-12-17 12:32:38,401] jepsen worker 46 - jepsen.util 6256 :invoke :write [15 43] INFO [2019-12-17 12:32:38,420] jepsen worker 72 - jepsen.util 6192 :ok :write [24 30] INFO [2019-12-17 12:32:38,420] jepsen worker 72 - jepsen.util 6192 :invoke :write [24 10] INFO [2019-12-17 12:32:38,441] jepsen worker 51 - jepsen.util 6081 :info :write [17 30] :RequestCanceledException INFO [2019-12-17 12:32:38,441] jepsen worker 52 - jepsen.util 6622 :info :write [17 37] :RequestCanceledException INFO [2019-12-17 12:32:38,442] jepsen worker 51 - jepsen.util 6171 :invoke :cas [17 [30 10]] INFO [2019-12-17 12:32:38,443] jepsen worker 52 - jepsen.util 6712 :invoke :cas [17 [36 3]] INFO [2019-12-17 12:32:38,446] jepsen worker 29 - jepsen.util 4709 :ok :write [9 48] INFO [2019-12-17 12:32:38,446] jepsen worker 29 - jepsen.util 4709 :invoke :write [9 21] INFO [2019-12-17 12:32:38,454] jepsen worker 31 - jepsen.util 5701 :fail :cas [10 [15 30]] :ValueNotSwapFrom INFO [2019-12-17 12:32:38,454] jepsen worker 31 - jepsen.util 5701 :invoke :read [10 nil] INFO [2019-12-17 12:32:38,468] jepsen worker 65 - jepsen.util 5195 :ok :write [21 19] INFO [2019-12-17 12:32:38,469] jepsen worker 65 - jepsen.util 5195 :invoke :write [21 43] INFO [2019-12-17 12:32:38,473] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":35988,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:32:38,473] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":35988,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:32:38,474] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":35991,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:32:38,474] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":35991,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:32:38,474] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":35992,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:32:38,474] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":35992,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:32:38,480] jepsen worker 38 - jepsen.util 6248 :ok :read [12 16] INFO [2019-12-17 12:32:38,480] jepsen worker 38 - jepsen.util 6248 :invoke :cas [12 [19 35]] INFO [2019-12-17 12:32:38,496] jepsen worker 20 - jepsen.util 6140 :ok :write [6 13] INFO [2019-12-17 12:32:38,496] jepsen worker 20 - jepsen.util 6140 :invoke :cas [6 [39 30]] INFO [2019-12-17 12:32:38,500] jepsen worker 29 - jepsen.util 4709 :ok :write [9 21] INFO [2019-12-17 12:32:38,500] jepsen worker 29 - jepsen.util 4709 :invoke :write [9 18] INFO [2019-12-17 12:32:38,526] jepsen worker 56 - jepsen.util 6626 :info :write [18 27] :RequestCanceledException INFO [2019-12-17 12:32:38,527] jepsen worker 56 - jepsen.util 6716 :invoke :read [18 nil] INFO [2019-12-17 12:32:38,547] jepsen worker 52 - jepsen.util 6712 :fail :cas [17 [36 3]] :ValueNotSwapFrom INFO [2019-12-17 12:32:38,547] jepsen worker 52 - jepsen.util 6712 :invoke :cas [17 [33 13]] INFO [2019-12-17 12:32:38,548] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:32:38,548] jepsen nemesis - couchbase.util Rebalance check count: 272 INFO [2019-12-17 12:32:38,549] jepsen nemesis - couchbase.util Rebalance stuck count: 253 INFO [2019-12-17 12:32:38,553] jepsen worker 20 - jepsen.util 6140 :fail :cas [6 [39 30]] :ValueNotSwapFrom INFO [2019-12-17 12:32:38,553] jepsen worker 20 - jepsen.util 6140 :invoke :cas [6 [30 31]] INFO [2019-12-17 12:32:38,563] jepsen worker 13 - jepsen.util 6043 :ok :read [4 13] INFO [2019-12-17 12:32:38,563] jepsen worker 13 - jepsen.util 6043 :invoke :write [4 6] INFO [2019-12-17 12:32:38,571] jepsen worker 20 - jepsen.util 6140 :fail :cas [6 [30 31]] :ValueNotSwapFrom INFO [2019-12-17 12:32:38,572] jepsen worker 20 - jepsen.util 6140 :invoke :read [6 nil] INFO [2019-12-17 12:32:38,585] jepsen worker 4 - jepsen.util 5674 :ok :read [1 40] INFO [2019-12-17 12:32:38,585] jepsen worker 4 - jepsen.util 5674 :invoke :write [1 17] INFO [2019-12-17 12:32:38,591] jepsen worker 20 - jepsen.util 6140 :ok :read [6 13] INFO [2019-12-17 12:32:38,591] jepsen worker 20 - jepsen.util 6140 :invoke :write [6 19] INFO [2019-12-17 12:32:38,606] jepsen worker 51 - jepsen.util 6171 :fail :cas [17 [30 10]] :ValueNotSwapFrom INFO [2019-12-17 12:32:38,606] jepsen worker 51 - jepsen.util 6171 :invoke :write [17 30] INFO [2019-12-17 12:32:38,615] jepsen worker 69 - jepsen.util 6279 :fail :cas [23 [49 20]] :ValueNotSwapFrom INFO [2019-12-17 12:32:38,615] jepsen worker 69 - jepsen.util 6279 :invoke :read [23 nil] INFO [2019-12-17 12:32:38,632] jepsen worker 10 - jepsen.util 6130 :ok :write [3 33] INFO [2019-12-17 12:32:38,632] jepsen worker 38 - jepsen.util 6248 :fail :cas [12 [19 35]] :ValueNotSwapFrom INFO [2019-12-17 12:32:38,632] jepsen worker 10 - jepsen.util 6130 :invoke :read [3 nil] INFO [2019-12-17 12:32:38,632] jepsen worker 38 - jepsen.util 6248 :invoke :write [12 6] INFO [2019-12-17 12:32:38,639] jepsen worker 29 - jepsen.util 4709 :ok :write [9 18] INFO [2019-12-17 12:32:38,639] jepsen worker 29 - jepsen.util 4709 :invoke :write [9 27] INFO [2019-12-17 12:32:38,641] jepsen worker 45 - jepsen.util 6165 :info :write [15 48] :RequestCanceledException INFO [2019-12-17 12:32:38,642] jepsen worker 45 - jepsen.util 6255 :invoke :cas [15 [41 16]] INFO [2019-12-17 12:32:38,669] jepsen worker 10 - jepsen.util 6130 :ok :read [3 33] INFO [2019-12-17 12:32:38,669] jepsen worker 10 - jepsen.util 6130 :invoke :write [3 43] INFO [2019-12-17 12:32:38,675] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36014,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:32:38,675] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36014,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:32:38,675] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36018,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:32:38,675] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36018,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:32:38,699] jepsen worker 55 - jepsen.util 7255 :fail :cas [18 [34 32]] :ValueNotSwapFrom INFO [2019-12-17 12:32:38,699] jepsen worker 55 - jepsen.util 7255 :invoke :write [18 48] INFO [2019-12-17 12:32:38,724] jepsen worker 29 - jepsen.util 4709 :ok :write [9 27] INFO [2019-12-17 12:32:38,725] jepsen worker 29 - jepsen.util 4709 :invoke :write [9 37] INFO [2019-12-17 12:32:38,727] jepsen worker 27 - jepsen.util 6147 :info :write [9 46] :RequestCanceledException INFO [2019-12-17 12:32:38,728] jepsen worker 27 - jepsen.util 6237 :invoke :read [9 nil] INFO [2019-12-17 12:32:38,733] jepsen worker 52 - jepsen.util 6712 :fail :cas [17 [33 13]] :ValueNotSwapFrom INFO [2019-12-17 12:32:38,733] jepsen worker 52 - jepsen.util 6712 :invoke :cas [17 [9 35]] INFO [2019-12-17 12:32:38,781] jepsen worker 29 - jepsen.util 4709 :ok :write [9 37] INFO [2019-12-17 12:32:38,781] jepsen worker 29 - jepsen.util 4709 :invoke :cas [9 [32 46]] INFO [2019-12-17 12:32:38,802] jepsen worker 29 - jepsen.util 4709 :fail :cas [9 [32 46]] :ValueNotSwapFrom INFO [2019-12-17 12:32:38,802] jepsen worker 29 - jepsen.util 4709 :invoke :cas [9 [47 44]] INFO [2019-12-17 12:32:38,811] jepsen worker 87 - jepsen.util 5667 :fail :cas [29 [34 34]] :ValueNotSwapFrom INFO [2019-12-17 12:32:38,811] jepsen worker 87 - jepsen.util 5667 :invoke :write [29 42] INFO [2019-12-17 12:32:38,811] jepsen worker 54 - jepsen.util 6804 :info :write [18 23] :RequestCanceledException INFO [2019-12-17 12:32:38,813] jepsen worker 54 - jepsen.util 6894 :invoke :write [18 11] INFO [2019-12-17 12:32:38,821] jepsen worker 29 - jepsen.util 4709 :fail :cas [9 [47 44]] :ValueNotSwapFrom INFO [2019-12-17 12:32:38,821] jepsen worker 29 - jepsen.util 4709 :invoke :read [9 nil] INFO [2019-12-17 12:32:38,825] jepsen worker 79 - jepsen.util 6199 :info :write [26 16] :RequestCanceledException INFO [2019-12-17 12:32:38,826] jepsen worker 79 - jepsen.util 6289 :invoke :write [26 1] INFO [2019-12-17 12:32:38,848] jepsen worker 88 - jepsen.util 6478 :ok :read [29 6] INFO [2019-12-17 12:32:38,848] jepsen worker 88 - jepsen.util 6478 :invoke :read [29 nil] INFO [2019-12-17 12:32:38,849] jepsen worker 29 - jepsen.util 4709 :ok :read [9 37] INFO [2019-12-17 12:32:38,849] jepsen worker 29 - jepsen.util 4709 :invoke :read [9 nil] INFO [2019-12-17 12:32:38,876] jepsen worker 29 - jepsen.util 4709 :ok :read [9 37] INFO [2019-12-17 12:32:38,876] jepsen worker 29 - jepsen.util 4709 :invoke :cas [9 [38 38]] INFO [2019-12-17 12:32:38,896] jepsen worker 29 - jepsen.util 4709 :fail :cas [9 [38 38]] :ValueNotSwapFrom INFO [2019-12-17 12:32:38,897] jepsen worker 29 - jepsen.util 4709 :invoke :cas [9 [25 48]] INFO [2019-12-17 12:32:38,902] jepsen worker 78 - jepsen.util 6558 :ok :read [26 44] INFO [2019-12-17 12:32:38,902] jepsen worker 78 - jepsen.util 6558 :invoke :cas [26 [7 26]] INFO [2019-12-17 12:32:38,911] jepsen worker 29 - jepsen.util 4709 :fail :cas [9 [25 48]] :ValueNotSwapFrom INFO [2019-12-17 12:32:38,911] jepsen worker 29 - jepsen.util 4709 :invoke :write [9 2] INFO [2019-12-17 12:32:38,911] jepsen worker 47 - jepsen.util 6887 :fail :read [15 nil] :RequestTimeoutException INFO [2019-12-17 12:32:38,911] jepsen worker 47 - jepsen.util 6887 :invoke :read [15 nil] INFO [2019-12-17 12:32:38,912] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36036,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:32:38,912] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36036,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:32:38,912] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36038,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:32:38,912] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36038,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:32:38,914] jepsen worker 5 - jepsen.util 5855 :fail :cas [1 [42 33]] :ValueNotSwapFrom INFO [2019-12-17 12:32:38,914] jepsen worker 5 - jepsen.util 5855 :invoke :read [1 nil] INFO [2019-12-17 12:32:38,950] jepsen worker 29 - jepsen.util 4709 :ok :write [9 2] INFO [2019-12-17 12:32:38,950] jepsen worker 29 - jepsen.util 4709 :invoke :read [9 nil] INFO [2019-12-17 12:32:38,960] jepsen worker 29 - jepsen.util 4709 :ok :read [9 2] INFO [2019-12-17 12:32:38,960] jepsen worker 29 - jepsen.util 4709 :invoke :write [9 46] INFO [2019-12-17 12:32:38,979] jepsen worker 78 - jepsen.util 6558 :fail :cas [26 [7 26]] :ValueNotSwapFrom INFO [2019-12-17 12:32:38,980] jepsen worker 78 - jepsen.util 6558 :invoke :cas [26 [10 19]] INFO [2019-12-17 12:32:38,995] jepsen worker 5 - jepsen.util 5855 :ok :read [1 40] INFO [2019-12-17 12:32:38,995] jepsen worker 5 - jepsen.util 5855 :invoke :read [1 nil] INFO [2019-12-17 12:32:39,008] jepsen worker 29 - jepsen.util 4709 :ok :write [9 46] INFO [2019-12-17 12:32:39,008] jepsen worker 29 - jepsen.util 4709 :invoke :cas [9 [7 48]] INFO [2019-12-17 12:32:39,020] jepsen worker 29 - jepsen.util 4709 :fail :cas [9 [7 48]] :ValueNotSwapFrom INFO [2019-12-17 12:32:39,020] jepsen worker 29 - jepsen.util 4709 :invoke :read [9 nil] INFO [2019-12-17 12:32:39,022] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36048,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:32:39,022] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36048,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:32:39,022] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36049,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:32:39,022] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36049,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:32:39,022] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36050,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:32:39,022] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36050,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:32:39,027] jepsen worker 68 - jepsen.util 7898 :info :write [22 28] :RequestCanceledException INFO [2019-12-17 12:32:39,028] jepsen worker 68 - jepsen.util 7988 :invoke :cas [22 [27 22]] INFO [2019-12-17 12:32:39,032] jepsen worker 29 - jepsen.util 4709 :ok :read [9 46] INFO [2019-12-17 12:32:39,032] jepsen worker 29 - jepsen.util 4709 :invoke :cas [9 [2 21]] INFO [2019-12-17 12:32:39,040] jepsen worker 19 - jepsen.util 6049 :info :write [6 49] :RequestCanceledException INFO [2019-12-17 12:32:39,041] jepsen worker 19 - jepsen.util 6139 :invoke :write [6 32] INFO [2019-12-17 12:32:39,043] jepsen worker 29 - jepsen.util 4709 :fail :cas [9 [2 21]] :ValueNotSwapFrom INFO [2019-12-17 12:32:39,044] jepsen worker 29 - jepsen.util 4709 :invoke :read [9 nil] INFO [2019-12-17 12:32:39,057] jepsen worker 29 - jepsen.util 4709 :ok :read [9 46] INFO [2019-12-17 12:32:39,057] jepsen worker 29 - jepsen.util 4709 :invoke :write [9 12] INFO [2019-12-17 12:32:39,074] jepsen worker 88 - jepsen.util 6478 :ok :read [29 6] INFO [2019-12-17 12:32:39,074] jepsen worker 88 - jepsen.util 6478 :invoke :read [29 nil] INFO [2019-12-17 12:32:39,097] jepsen worker 37 - jepsen.util 6067 :ok :read [12 16] INFO [2019-12-17 12:32:39,097] jepsen worker 37 - jepsen.util 6067 :invoke :read [12 nil] INFO [2019-12-17 12:32:39,101] jepsen worker 29 - jepsen.util 4709 :ok :write [9 12] INFO [2019-12-17 12:32:39,102] jepsen worker 29 - jepsen.util 4709 :invoke :cas [9 [21 38]] INFO [2019-12-17 12:32:39,112] jepsen worker 36 - jepsen.util 6786 :info :write [12 31] :RequestCanceledException INFO [2019-12-17 12:32:39,113] jepsen worker 36 - jepsen.util 6876 :invoke :cas [12 [44 40]] INFO [2019-12-17 12:32:39,118] jepsen worker 29 - jepsen.util 4709 :fail :cas [9 [21 38]] :ValueNotSwapFrom INFO [2019-12-17 12:32:39,119] jepsen worker 29 - jepsen.util 4709 :invoke :write [9 1] INFO [2019-12-17 12:32:39,121] jepsen worker 37 - jepsen.util 6067 :ok :read [12 16] INFO [2019-12-17 12:32:39,121] jepsen worker 37 - jepsen.util 6067 :invoke :read [12 nil] INFO [2019-12-17 12:32:39,127] jepsen worker 66 - jepsen.util 7986 :info :cas [22 [20 13]] :RequestCanceledException INFO [2019-12-17 12:32:39,128] jepsen worker 66 - jepsen.util 8076 :invoke :write [22 21] INFO [2019-12-17 12:32:39,156] jepsen worker 85 - jepsen.util 9265 :info :cas [28 [5 26]] :RequestCanceledException INFO [2019-12-17 12:32:39,157] jepsen worker 85 - jepsen.util 9355 :invoke :write [28 20] INFO [2019-12-17 12:32:39,165] jepsen worker 29 - jepsen.util 4709 :ok :write [9 1] INFO [2019-12-17 12:32:39,166] jepsen worker 29 - jepsen.util 4709 :invoke :read [9 nil] INFO [2019-12-17 12:32:39,183] jepsen worker 29 - jepsen.util 4709 :ok :read [9 1] INFO [2019-12-17 12:32:39,183] jepsen worker 29 - jepsen.util 4709 :invoke :write [9 33] INFO [2019-12-17 12:32:39,184] jepsen worker 69 - jepsen.util 6279 :ok :read [23 37] INFO [2019-12-17 12:32:39,185] jepsen worker 69 - jepsen.util 6279 :invoke :write [23 49] INFO [2019-12-17 12:32:39,193] jepsen worker 8 - jepsen.util 8018 :info :write [2 34] :RequestCanceledException INFO [2019-12-17 12:32:39,193] jepsen worker 14 - jepsen.util 6314 :info :write [4 23] :RequestCanceledException INFO [2019-12-17 12:32:39,194] jepsen worker 14 - jepsen.util 6404 :invoke :write [4 46] INFO [2019-12-17 12:32:39,194] jepsen worker 8 - jepsen.util 8108 :invoke :cas [2 [11 16]] INFO [2019-12-17 12:32:39,199] jepsen worker 88 - jepsen.util 6478 :ok :read [29 6] INFO [2019-12-17 12:32:39,199] jepsen worker 88 - jepsen.util 6478 :invoke :cas [29 [40 16]] INFO [2019-12-17 12:32:39,227] jepsen worker 60 - jepsen.util 9240 :info :cas [20 [23 1]] :RequestCanceledException INFO [2019-12-17 12:32:39,228] jepsen worker 60 - jepsen.util 9330 :invoke :read [20 nil] INFO [2019-12-17 12:32:39,229] jepsen worker 29 - jepsen.util 4709 :ok :write [9 33] INFO [2019-12-17 12:32:39,229] jepsen worker 29 - jepsen.util 4709 :invoke :read [9 nil] INFO [2019-12-17 12:32:39,233] jepsen worker 36 - jepsen.util 6876 :fail :cas [12 [44 40]] :ValueNotSwapFrom INFO [2019-12-17 12:32:39,233] jepsen worker 36 - jepsen.util 6876 :invoke :read [12 nil] INFO [2019-12-17 12:32:39,241] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36084,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:32:39,241] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36084,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:32:39,241] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36087,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:32:39,241] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36087,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:32:39,250] jepsen worker 29 - jepsen.util 4709 :ok :read [9 33] INFO [2019-12-17 12:32:39,250] jepsen worker 29 - jepsen.util 4709 :invoke :read [9 nil] INFO [2019-12-17 12:32:39,264] jepsen worker 29 - jepsen.util 4709 :ok :read [9 33] INFO [2019-12-17 12:32:39,264] jepsen worker 29 - jepsen.util 4709 :invoke :cas [9 [9 2]] INFO [2019-12-17 12:32:39,276] jepsen worker 36 - jepsen.util 6876 :ok :read [12 16] INFO [2019-12-17 12:32:39,276] jepsen worker 36 - jepsen.util 6876 :invoke :cas [12 [34 0]] INFO [2019-12-17 12:32:39,278] jepsen worker 29 - jepsen.util 4709 :fail :cas [9 [9 2]] :ValueNotSwapFrom INFO [2019-12-17 12:32:39,278] jepsen worker 29 - jepsen.util 4709 :invoke :write [9 9] INFO [2019-12-17 12:32:39,282] jepsen worker 56 - jepsen.util 6716 :ok :read [18 21] INFO [2019-12-17 12:32:39,282] jepsen worker 56 - jepsen.util 6716 :invoke :write [18 43] INFO [2019-12-17 12:32:39,293] jepsen worker 30 - jepsen.util 6150 :info :write [10 46] :RequestCanceledException INFO [2019-12-17 12:32:39,293] jepsen worker 9 - jepsen.util 6669 :info :write [3 25] :RequestCanceledException INFO [2019-12-17 12:32:39,293] jepsen worker 11 - jepsen.util 6401 :info :cas [3 [10 9]] :RequestCanceledException INFO [2019-12-17 12:32:39,293] jepsen worker 9 - jepsen.util 6759 :invoke :read [3 nil] INFO [2019-12-17 12:32:39,294] jepsen worker 11 - jepsen.util 6491 :invoke :cas [3 [30 42]] INFO [2019-12-17 12:32:39,294] jepsen worker 30 - jepsen.util 6240 :invoke :cas [10 [37 43]] INFO [2019-12-17 12:32:39,303] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36074,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:32:39,303] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36074,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:32:39,303] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36075,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:32:39,303] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36075,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:32:39,312] jepsen worker 43 - jepsen.util 7783 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:32:39,312] jepsen worker 43 - jepsen.util 7783 :invoke :cas [14 [2 0]] INFO [2019-12-17 12:32:39,325] jepsen worker 29 - jepsen.util 4709 :ok :write [9 9] INFO [2019-12-17 12:32:39,325] jepsen worker 29 - jepsen.util 4709 :invoke :write [9 18] INFO [2019-12-17 12:32:39,326] jepsen worker 34 - jepsen.util 7864 :info :cas [11 [5 35]] :RequestCanceledException INFO [2019-12-17 12:32:39,327] jepsen worker 34 - jepsen.util 7954 :invoke :read [11 nil] INFO [2019-12-17 12:32:39,328] jepsen worker 44 - jepsen.util 9134 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:32:39,328] jepsen worker 44 - jepsen.util 9134 :invoke :cas [14 [49 26]] INFO [2019-12-17 12:32:39,341] jepsen worker 75 - jepsen.util 7905 :info :cas [25 [10 9]] :RequestCanceledException INFO [2019-12-17 12:32:39,342] jepsen worker 75 - jepsen.util 7995 :invoke :cas [25 [32 28]] INFO [2019-12-17 12:32:39,351] jepsen worker 36 - jepsen.util 6876 :fail :cas [12 [34 0]] :ValueNotSwapFrom INFO [2019-12-17 12:32:39,351] jepsen worker 36 - jepsen.util 6876 :invoke :write [12 29] INFO [2019-12-17 12:32:39,356] jepsen worker 52 - jepsen.util 6712 :fail :cas [17 [9 35]] :ValueNotSwapFrom INFO [2019-12-17 12:32:39,356] jepsen worker 52 - jepsen.util 6712 :invoke :read [17 nil] INFO [2019-12-17 12:32:39,357] jepsen worker 6 - jepsen.util 9006 :info :cas [2 [7 6]] :RequestCanceledException INFO [2019-12-17 12:32:39,357] jepsen worker 48 - jepsen.util 7608 :info :write [16 33] :RequestCanceledException INFO [2019-12-17 12:32:39,358] jepsen worker 48 - jepsen.util 7698 :invoke :write [16 18] INFO [2019-12-17 12:32:39,358] jepsen worker 6 - jepsen.util 9096 :invoke :read [2 nil] INFO [2019-12-17 12:32:39,379] jepsen worker 29 - jepsen.util 4709 :ok :write [9 18] INFO [2019-12-17 12:32:39,379] jepsen worker 29 - jepsen.util 4709 :invoke :read [9 nil] INFO [2019-12-17 12:32:39,393] jepsen worker 76 - jepsen.util 8716 :info :cas [25 [19 8]] :RequestCanceledException INFO [2019-12-17 12:32:39,393] jepsen worker 16 - jepsen.util 9376 :info :write [5 11] :RequestCanceledException INFO [2019-12-17 12:32:39,394] jepsen worker 76 - jepsen.util 8806 :invoke :read [25 nil] INFO [2019-12-17 12:32:39,394] jepsen worker 16 - jepsen.util 9466 :invoke :write [5 2] INFO [2019-12-17 12:32:39,395] jepsen worker 29 - jepsen.util 4709 :ok :read [9 18] INFO [2019-12-17 12:32:39,395] jepsen worker 29 - jepsen.util 4709 :invoke :read [9 nil] INFO [2019-12-17 12:32:39,406] jepsen worker 29 - jepsen.util 4709 :ok :read [9 18] INFO [2019-12-17 12:32:39,406] jepsen worker 29 - jepsen.util 4709 :invoke :write [9 0] INFO [2019-12-17 12:32:39,412] jepsen worker 58 - jepsen.util 8968 :info :write [19 23] :RequestCanceledException INFO [2019-12-17 12:32:39,412] jepsen worker 82 - jepsen.util 8902 :info :cas [27 [41 14]] :RequestCanceledException INFO [2019-12-17 12:32:39,413] jepsen worker 82 - jepsen.util 8992 :invoke :read [27 nil] INFO [2019-12-17 12:32:39,413] jepsen worker 58 - jepsen.util 9058 :invoke :write [19 37] INFO [2019-12-17 12:32:39,426] jepsen worker 57 - jepsen.util 7887 :info :cas [19 [2 10]] :RequestCanceledException INFO [2019-12-17 12:32:39,427] jepsen worker 57 - jepsen.util 7977 :invoke :write [19 14] INFO [2019-12-17 12:32:39,427] jepsen worker 41 - jepsen.util 9221 :info :cas [13 [36 1]] :RequestCanceledException INFO [2019-12-17 12:32:39,428] jepsen worker 41 - jepsen.util 9311 :invoke :cas [13 [37 0]] INFO [2019-12-17 12:32:39,456] jepsen worker 37 - jepsen.util 6067 :ok :read [12 16] INFO [2019-12-17 12:32:39,456] jepsen worker 37 - jepsen.util 6067 :invoke :write [12 18] INFO [2019-12-17 12:32:39,458] jepsen worker 0 - jepsen.util 8820 :info :write [0 35] :RequestCanceledException INFO [2019-12-17 12:32:39,459] jepsen worker 0 - jepsen.util 8910 :invoke :read [0 nil] INFO [2019-12-17 12:32:39,460] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36117,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:32:39,460] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36117,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:32:39,460] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36120,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:32:39,460] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36120,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:32:39,465] jepsen worker 29 - jepsen.util 4709 :ok :write [9 0] INFO [2019-12-17 12:32:39,465] jepsen worker 29 - jepsen.util 4709 :invoke :cas [9 [13 25]] INFO [2019-12-17 12:32:39,479] jepsen worker 29 - jepsen.util 4709 :fail :cas [9 [13 25]] :ValueNotSwapFrom INFO [2019-12-17 12:32:39,479] jepsen worker 29 - jepsen.util 4709 :invoke :cas [9 [4 22]] INFO [2019-12-17 12:32:39,485] jepsen worker 52 - jepsen.util 6712 :ok :read [17 20] INFO [2019-12-17 12:32:39,485] jepsen worker 52 - jepsen.util 6712 :invoke :write [17 20] INFO [2019-12-17 12:32:39,497] jepsen worker 29 - jepsen.util 4709 :fail :cas [9 [4 22]] :ValueNotSwapFrom INFO [2019-12-17 12:32:39,497] jepsen worker 29 - jepsen.util 4709 :invoke :cas [9 [42 45]] INFO [2019-12-17 12:32:39,512] jepsen worker 33 - jepsen.util 8313 :info :write [11 27] :RequestCanceledException INFO [2019-12-17 12:32:39,513] jepsen worker 33 - jepsen.util 8403 :invoke :read [11 nil] INFO [2019-12-17 12:32:39,517] jepsen worker 29 - jepsen.util 4709 :fail :cas [9 [42 45]] :ValueNotSwapFrom INFO [2019-12-17 12:32:39,517] jepsen worker 29 - jepsen.util 4709 :invoke :write [9 4] INFO [2019-12-17 12:32:39,532] jepsen worker 78 - jepsen.util 6558 :fail :cas [26 [10 19]] :ValueNotSwapFrom INFO [2019-12-17 12:32:39,532] jepsen worker 78 - jepsen.util 6558 :invoke :cas [26 [47 20]] INFO [2019-12-17 12:32:39,555] jepsen worker 5 - jepsen.util 5855 :ok :read [1 40] INFO [2019-12-17 12:32:39,555] jepsen worker 5 - jepsen.util 5855 :invoke :read [1 nil] INFO [2019-12-17 12:32:39,558] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:32:39,558] jepsen nemesis - couchbase.util Rebalance check count: 273 INFO [2019-12-17 12:32:39,558] jepsen nemesis - couchbase.util Rebalance stuck count: 254 INFO [2019-12-17 12:32:39,581] jepsen worker 29 - jepsen.util 4709 :ok :write [9 4] INFO [2019-12-17 12:32:39,582] jepsen worker 29 - jepsen.util 4709 :invoke :write [9 26] INFO [2019-12-17 12:32:39,611] jepsen worker 86 - jepsen.util 8816 :info :write [28 32] :RequestCanceledException INFO [2019-12-17 12:32:39,615] jepsen worker 86 - jepsen.util 8906 :invoke :write [28 44] INFO [2019-12-17 12:32:39,626] jepsen worker 81 - jepsen.util 8001 :info :cas [27 [26 39]] :RequestCanceledException INFO [2019-12-17 12:32:39,626] jepsen worker 21 - jepsen.util 8031 :info :cas [7 [1 36]] :RequestCanceledException INFO [2019-12-17 12:32:39,627] jepsen worker 81 - jepsen.util 8091 :invoke :write [27 39] INFO [2019-12-17 12:32:39,627] jepsen worker 21 - jepsen.util 8121 :invoke :read [7 nil] INFO [2019-12-17 12:32:39,638] jepsen worker 29 - jepsen.util 4709 :ok :write [9 26] INFO [2019-12-17 12:32:39,638] jepsen worker 29 - jepsen.util 4709 :invoke :write [9 30] INFO [2019-12-17 12:32:39,641] jepsen worker 62 - jepsen.util 7982 :info :cas [20 [27 36]] :RequestCanceledException INFO [2019-12-17 12:32:39,641] jepsen worker 61 - jepsen.util 9511 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:32:39,641] jepsen worker 61 - jepsen.util 9511 :invoke :read [20 nil] INFO [2019-12-17 12:32:39,643] jepsen worker 62 - jepsen.util 8072 :invoke :cas [20 [5 19]] INFO [2019-12-17 12:32:39,656] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36140,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:32:39,656] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36140,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:32:39,656] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36143,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:32:39,657] jepsen worker 77 - jepsen.util 8807 :info :write [25 26] :RequestCanceledException INFO [2019-12-17 12:32:39,657] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36143,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:32:39,657] jepsen worker 77 - jepsen.util 8897 :invoke :cas [25 [40 32]] INFO [2019-12-17 12:32:39,688] jepsen worker 29 - jepsen.util 4709 :ok :write [9 30] INFO [2019-12-17 12:32:39,689] jepsen worker 29 - jepsen.util 4709 :invoke :write [9 13] INFO [2019-12-17 12:32:39,693] jepsen worker 84 - jepsen.util 7914 :info :cas [28 [5 27]] :RequestCanceledException INFO [2019-12-17 12:32:39,693] jepsen worker 83 - jepsen.util 8813 :info :write [27 14] :RequestCanceledException INFO [2019-12-17 12:32:39,694] jepsen worker 84 - jepsen.util 8004 :invoke :write [28 45] INFO [2019-12-17 12:32:39,696] jepsen worker 83 - jepsen.util 8903 :invoke :write [27 28] INFO [2019-12-17 12:32:39,711] jepsen worker 39 - jepsen.util 7959 :info :write [13 5] :RequestCanceledException INFO [2019-12-17 12:32:39,712] jepsen worker 39 - jepsen.util 8049 :invoke :cas [13 [22 30]] INFO [2019-12-17 12:32:39,726] jepsen worker 1 - jepsen.util 8191 :info :cas [0 [44 25]] :RequestCanceledException INFO [2019-12-17 12:32:39,726] jepsen worker 1 - jepsen.util 8281 :invoke :cas [0 [20 39]] INFO [2019-12-17 12:32:39,749] jepsen worker 29 - jepsen.util 4709 :ok :write [9 13] INFO [2019-12-17 12:32:39,749] jepsen worker 29 - jepsen.util 4709 :invoke :read [9 nil] INFO [2019-12-17 12:32:39,768] jepsen worker 29 - jepsen.util 4709 :ok :read [9 13] INFO [2019-12-17 12:32:39,768] jepsen worker 29 - jepsen.util 4709 :invoke :write [9 46] INFO [2019-12-17 12:32:39,805] jepsen worker 78 - jepsen.util 6558 :fail :cas [26 [47 20]] :ValueNotSwapFrom INFO [2019-12-17 12:32:39,805] jepsen worker 78 - jepsen.util 6558 :invoke :write [26 44] INFO [2019-12-17 12:32:39,822] jepsen worker 29 - jepsen.util 4709 :ok :write [9 46] INFO [2019-12-17 12:32:39,822] jepsen worker 29 - jepsen.util 4709 :invoke :cas [9 [14 18]] INFO [2019-12-17 12:32:39,825] jepsen worker 53 - jepsen.util 5903 :info :write [17 25] :RequestCanceledException INFO [2019-12-17 12:32:39,825] jepsen worker 40 - jepsen.util 8770 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:32:39,825] jepsen worker 12 - jepsen.util 6492 :info :write [4 43] :RequestCanceledException INFO [2019-12-17 12:32:39,825] jepsen worker 40 - jepsen.util 8770 :invoke :cas [13 [23 49]] INFO [2019-12-17 12:32:39,826] jepsen worker 53 - jepsen.util 5993 :invoke :write [17 22] INFO [2019-12-17 12:32:39,826] jepsen worker 12 - jepsen.util 6582 :invoke :cas [4 [0 48]] INFO [2019-12-17 12:32:39,835] jepsen worker 88 - jepsen.util 6478 :fail :cas [29 [40 16]] :ValueNotSwapFrom INFO [2019-12-17 12:32:39,835] jepsen worker 88 - jepsen.util 6478 :invoke :cas [29 [46 14]] INFO [2019-12-17 12:32:39,837] jepsen worker 29 - jepsen.util 4709 :fail :cas [9 [14 18]] :ValueNotSwapFrom INFO [2019-12-17 12:32:39,837] jepsen worker 29 - jepsen.util 4709 :invoke :write [9 30] INFO [2019-12-17 12:32:39,841] jepsen worker 22 - jepsen.util 8302 :info :cas [7 [14 10]] :RequestCanceledException INFO [2019-12-17 12:32:39,841] jepsen worker 22 - jepsen.util 8392 :invoke :cas [7 [23 6]] INFO [2019-12-17 12:32:39,858] jepsen worker 15 - jepsen.util 7395 :info :write [5 15] :RequestCanceledException INFO [2019-12-17 12:32:39,858] jepsen worker 3 - jepsen.util 6123 :info :write [1 24] :RequestCanceledException INFO [2019-12-17 12:32:39,859] jepsen worker 3 - jepsen.util 6213 :invoke :write [1 12] INFO [2019-12-17 12:32:39,859] jepsen worker 15 - jepsen.util 7485 :invoke :read [5 nil] INFO [2019-12-17 12:32:39,892] jepsen worker 29 - jepsen.util 4709 :ok :write [9 30] INFO [2019-12-17 12:32:39,892] jepsen worker 29 - jepsen.util 4709 :invoke :read [9 nil] INFO [2019-12-17 12:32:39,893] jepsen worker 17 - jepsen.util 8477 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:32:39,893] jepsen worker 7 - jepsen.util 8557 :info :write [2 41] :RequestCanceledException INFO [2019-12-17 12:32:39,893] jepsen worker 25 - jepsen.util 8845 :info :write [8 7] :RequestCanceledException INFO [2019-12-17 12:32:39,893] jepsen worker 17 - jepsen.util 8477 :invoke :cas [5 [28 48]] INFO [2019-12-17 12:32:39,894] jepsen worker 25 - jepsen.util 8935 :invoke :cas [8 [16 27]] INFO [2019-12-17 12:32:39,894] jepsen worker 7 - jepsen.util 8647 :invoke :cas [2 [16 44]] INFO [2019-12-17 12:32:39,899] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36163,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:32:39,899] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36163,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:32:39,899] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36164,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:32:39,899] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36164,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:32:39,899] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36165,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:32:39,899] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36165,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:32:39,907] jepsen worker 29 - jepsen.util 4709 :ok :read [9 30] INFO [2019-12-17 12:32:39,908] jepsen worker 29 - jepsen.util 4709 :invoke :write [9 40] INFO [2019-12-17 12:32:39,910] jepsen worker 67 - jepsen.util 8617 :info :cas [22 [42 32]] :RequestCanceledException INFO [2019-12-17 12:32:39,910] jepsen worker 12 - jepsen.util 6582 :fail :cas [4 [0 48]] :ValueNotSwapFrom INFO [2019-12-17 12:32:39,911] jepsen worker 12 - jepsen.util 6582 :invoke :write [4 26] INFO [2019-12-17 12:32:39,911] jepsen worker 67 - jepsen.util 8707 :invoke :cas [22 [1 31]] INFO [2019-12-17 12:32:39,926] jepsen worker 18 - jepsen.util 6048 :info :write [6 3] :RequestCanceledException INFO [2019-12-17 12:32:39,926] jepsen worker 23 - jepsen.util 8483 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:32:39,926] jepsen worker 23 - jepsen.util 8483 :invoke :write [7 38] INFO [2019-12-17 12:32:39,927] jepsen worker 18 - jepsen.util 6138 :invoke :cas [6 [47 8]] INFO [2019-12-17 12:32:39,940] jepsen worker 70 - jepsen.util 6730 :info :write [23 5] :RequestCanceledException INFO [2019-12-17 12:32:39,941] jepsen worker 70 - jepsen.util 6820 :invoke :cas [23 [3 38]] INFO [2019-12-17 12:32:39,949] jepsen worker 29 - jepsen.util 4709 :ok :write [9 40] INFO [2019-12-17 12:32:39,949] jepsen worker 29 - jepsen.util 4709 :invoke :read [9 nil] INFO [2019-12-17 12:32:39,958] jepsen worker 42 - jepsen.util 8952 :info :write [14 29] :RequestCanceledException INFO [2019-12-17 12:32:39,958] jepsen worker 42 - jepsen.util 9042 :invoke :read [14 nil] INFO [2019-12-17 12:32:39,963] jepsen worker 29 - jepsen.util 4709 :ok :read [9 40] INFO [2019-12-17 12:32:39,963] jepsen worker 29 - jepsen.util 4709 :invoke :read [9 nil] INFO [2019-12-17 12:32:39,976] jepsen worker 29 - jepsen.util 4709 :ok :read [9 40] INFO [2019-12-17 12:32:39,976] jepsen worker 29 - jepsen.util 4709 :invoke :read [9 nil] INFO [2019-12-17 12:32:39,985] jepsen worker 29 - jepsen.util 4709 :ok :read [9 40] INFO [2019-12-17 12:32:39,985] jepsen worker 29 - jepsen.util 4709 :invoke :cas [9 [7 35]] INFO [2019-12-17 12:32:39,994] jepsen worker 74 - jepsen.util 6374 :info :write [24 38] :RequestCanceledException INFO [2019-12-17 12:32:39,995] jepsen worker 74 - jepsen.util 6464 :invoke :cas [24 [45 28]] INFO [2019-12-17 12:32:40,002] jepsen worker 29 - jepsen.util 4709 :fail :cas [9 [7 35]] :ValueNotSwapFrom INFO [2019-12-17 12:32:40,002] jepsen worker 29 - jepsen.util 4709 :invoke :cas [9 [14 47]] INFO [2019-12-17 12:32:40,008] jepsen worker 74 - jepsen.util 6464 :fail :cas [24 [45 28]] :ValueNotSwapFrom INFO [2019-12-17 12:32:40,008] jepsen worker 74 - jepsen.util 6464 :invoke :cas [24 [49 32]] INFO [2019-12-17 12:32:40,009] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36187,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:32:40,009] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36187,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:32:40,011] jepsen worker 32 - jepsen.util 6332 :info :write [10 37] :RequestCanceledException INFO [2019-12-17 12:32:40,012] jepsen worker 32 - jepsen.util 6422 :invoke :cas [10 [30 34]] INFO [2019-12-17 12:32:40,014] jepsen worker 29 - jepsen.util 4709 :fail :cas [9 [14 47]] :ValueNotSwapFrom INFO [2019-12-17 12:32:40,014] jepsen worker 29 - jepsen.util 4709 :invoke :cas [9 [35 27]] INFO [2019-12-17 12:32:40,025] jepsen worker 74 - jepsen.util 6464 :fail :cas [24 [49 32]] :ValueNotSwapFrom INFO [2019-12-17 12:32:40,025] jepsen worker 74 - jepsen.util 6464 :invoke :cas [24 [44 6]] INFO [2019-12-17 12:32:40,027] jepsen worker 26 - jepsen.util 8666 :info :write [8 48] :RequestCanceledException INFO [2019-12-17 12:32:40,027] jepsen worker 64 - jepsen.util 4834 :info :cas [21 [8 29]] :RequestCanceledException INFO [2019-12-17 12:32:40,027] jepsen worker 24 - jepsen.util 7584 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:32:40,027] jepsen worker 24 - jepsen.util 7584 :invoke :read [8 nil] INFO [2019-12-17 12:32:40,028] jepsen worker 64 - jepsen.util 4924 :invoke :cas [21 [15 44]] INFO [2019-12-17 12:32:40,032] jepsen worker 26 - jepsen.util 8756 :invoke :write [8 22] INFO [2019-12-17 12:32:40,035] jepsen worker 29 - jepsen.util 4709 :fail :cas [9 [35 27]] :ValueNotSwapFrom INFO [2019-12-17 12:32:40,035] jepsen worker 29 - jepsen.util 4709 :invoke :write [9 23] INFO [2019-12-17 12:32:40,042] jepsen worker 59 - jepsen.util 8519 :info :write [19 47] :RequestCanceledException INFO [2019-12-17 12:32:40,043] jepsen worker 59 - jepsen.util 8609 :invoke :write [19 1] INFO [2019-12-17 12:32:40,054] jepsen worker 74 - jepsen.util 6464 :fail :cas [24 [44 6]] :ValueNotSwapFrom INFO [2019-12-17 12:32:40,054] jepsen worker 74 - jepsen.util 6464 :invoke :write [24 1] INFO [2019-12-17 12:32:40,058] jepsen worker 28 - jepsen.util 4888 :info :write [9 13] :RequestCanceledException INFO [2019-12-17 12:32:40,059] jepsen worker 28 - jepsen.util 4978 :invoke :write [9 31] INFO [2019-12-17 12:32:40,108] jepsen worker 29 - jepsen.util 4709 :ok :write [9 23] INFO [2019-12-17 12:32:40,109] jepsen worker 29 - jepsen.util 4709 :invoke :write [9 6] INFO [2019-12-17 12:32:40,111] jepsen worker 35 - jepsen.util 8405 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:32:40,111] jepsen worker 35 - jepsen.util 8405 :invoke :write [11 11] INFO [2019-12-17 12:32:40,127] jepsen worker 49 - jepsen.util 7429 :info :cas [16 [26 39]] :RequestCanceledException INFO [2019-12-17 12:32:40,130] jepsen worker 49 - jepsen.util 7519 :invoke :cas [16 [22 8]] INFO [2019-12-17 12:32:40,167] jepsen worker 29 - jepsen.util 4709 :ok :write [9 6] INFO [2019-12-17 12:32:40,167] jepsen worker 29 - jepsen.util 4709 :invoke :read [9 nil] INFO [2019-12-17 12:32:40,209] jepsen worker 89 - jepsen.util 6029 :info :write [29 48] :RequestCanceledException INFO [2019-12-17 12:32:40,210] jepsen worker 89 - jepsen.util 6119 :invoke :write [29 5] INFO [2019-12-17 12:32:40,214] jepsen worker 29 - jepsen.util 4709 :ok :read [9 6] INFO [2019-12-17 12:32:40,214] jepsen worker 29 - jepsen.util 4709 :invoke :cas [9 [11 20]] INFO [2019-12-17 12:32:40,227] jepsen worker 10 - jepsen.util 6130 :ok :write [3 43] INFO [2019-12-17 12:32:40,227] jepsen worker 10 - jepsen.util 6130 :invoke :write [3 9] INFO [2019-12-17 12:32:40,227] jepsen worker 2 - jepsen.util 8102 :info :write [0 33] :RequestCanceledException INFO [2019-12-17 12:32:40,228] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36215,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:32:40,228] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36215,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:32:40,229] jepsen worker 20 - jepsen.util 6140 :ok :write [6 19] INFO [2019-12-17 12:32:40,229] jepsen worker 20 - jepsen.util 6140 :invoke :read [6 nil] INFO [2019-12-17 12:32:40,230] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36218,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:32:40,231] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36218,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:32:40,231] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36219,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:32:40,231] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36219,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:32:40,235] jepsen worker 2 - jepsen.util 8192 :invoke :write [0 7] INFO [2019-12-17 12:32:40,252] jepsen worker 72 - jepsen.util 6192 :ok :write [24 10] INFO [2019-12-17 12:32:40,252] jepsen worker 72 - jepsen.util 6192 :invoke :read [24 nil] INFO [2019-12-17 12:32:40,257] jepsen worker 80 - jepsen.util 6200 :info :write [26 36] :RequestCanceledException INFO [2019-12-17 12:32:40,261] jepsen worker 80 - jepsen.util 6290 :invoke :read [26 nil] INFO [2019-12-17 12:32:40,276] jepsen worker 5 - jepsen.util 5855 :ok :read [1 40] INFO [2019-12-17 12:32:40,277] jepsen worker 5 - jepsen.util 5855 :invoke :read [1 nil] INFO [2019-12-17 12:32:40,281] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36208,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:32:40,281] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36208,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:32:40,281] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36209,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:32:40,281] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36209,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:32:40,282] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36210,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:32:40,282] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36210,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:32:40,291] jepsen worker 29 - jepsen.util 4709 :fail :cas [9 [11 20]] :ValueNotSwapFrom INFO [2019-12-17 12:32:40,292] jepsen worker 29 - jepsen.util 4709 :invoke :write [9 25] INFO [2019-12-17 12:32:40,293] jepsen worker 46 - jepsen.util 6256 :ok :write [15 43] INFO [2019-12-17 12:32:40,293] jepsen worker 46 - jepsen.util 6256 :invoke :cas [15 [7 29]] INFO [2019-12-17 12:32:40,325] jepsen worker 73 - jepsen.util 5833 :fail :read [24 nil] :RequestTimeoutException INFO [2019-12-17 12:32:40,325] jepsen worker 73 - jepsen.util 5833 :invoke :write [24 26] INFO [2019-12-17 12:32:40,327] jepsen worker 30 - jepsen.util 6240 :fail :cas [10 [37 43]] :ValueNotSwapFrom INFO [2019-12-17 12:32:40,327] jepsen worker 30 - jepsen.util 6240 :invoke :cas [10 [0 11]] INFO [2019-12-17 12:32:40,342] jepsen worker 71 - jepsen.util 6911 :info :write [23 47] :RequestCanceledException INFO [2019-12-17 12:32:40,351] jepsen worker 18 - jepsen.util 6138 :fail :cas [6 [47 8]] :ValueNotSwapFrom INFO [2019-12-17 12:32:40,351] jepsen worker 18 - jepsen.util 6138 :invoke :write [6 33] INFO [2019-12-17 12:32:40,353] jepsen worker 71 - jepsen.util 7001 :invoke :read [23 nil] INFO [2019-12-17 12:32:40,355] jepsen worker 63 - jepsen.util 5283 :fail :read [21 nil] :RequestTimeoutException INFO [2019-12-17 12:32:40,355] jepsen worker 63 - jepsen.util 5283 :invoke :read [21 nil] INFO [2019-12-17 12:32:40,357] jepsen worker 27 - jepsen.util 6237 :ok :read [9 6] INFO [2019-12-17 12:32:40,357] jepsen worker 27 - jepsen.util 6237 :invoke :read [9 nil] INFO [2019-12-17 12:32:40,372] jepsen worker 45 - jepsen.util 6255 :fail :cas [15 [41 16]] :ValueNotSwapFrom INFO [2019-12-17 12:32:40,373] jepsen worker 45 - jepsen.util 6255 :invoke :write [15 21] INFO [2019-12-17 12:32:40,381] jepsen worker 11 - jepsen.util 6491 :fail :cas [3 [30 42]] :ValueNotSwapFrom INFO [2019-12-17 12:32:40,381] jepsen worker 11 - jepsen.util 6491 :invoke :write [3 24] INFO [2019-12-17 12:32:40,394] jepsen worker 50 - jepsen.util 8870 :info :cas [16 [37 47]] :RequestCanceledException INFO [2019-12-17 12:32:40,398] jepsen worker 50 - jepsen.util 8960 :invoke :cas [16 [0 26]] INFO [2019-12-17 12:32:40,420] jepsen worker 64 - jepsen.util 4924 :fail :cas [21 [15 44]] :ValueNotSwapFrom INFO [2019-12-17 12:32:40,420] jepsen worker 47 - jepsen.util 6887 :ok :read [15 43] INFO [2019-12-17 12:32:40,420] jepsen worker 64 - jepsen.util 4924 :invoke :read [21 nil] INFO [2019-12-17 12:32:40,420] jepsen worker 47 - jepsen.util 6887 :invoke :read [15 nil] INFO [2019-12-17 12:32:40,445] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36241,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:32:40,445] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36241,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:32:40,445] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36245,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:32:40,445] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36245,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:32:40,445] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36246,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:32:40,445] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36246,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:32:40,471] jepsen worker 9 - jepsen.util 6759 :ok :read [3 43] INFO [2019-12-17 12:32:40,471] jepsen worker 9 - jepsen.util 6759 :invoke :cas [3 [0 16]] INFO [2019-12-17 12:32:40,490] jepsen worker 72 - jepsen.util 6192 :ok :read [24 10] INFO [2019-12-17 12:32:40,490] jepsen worker 72 - jepsen.util 6192 :invoke :read [24 nil] INFO [2019-12-17 12:32:40,490] jepsen worker 88 - jepsen.util 6478 :fail :cas [29 [46 14]] :ValueNotSwapFrom INFO [2019-12-17 12:32:40,491] jepsen worker 88 - jepsen.util 6478 :invoke :read [29 nil] INFO [2019-12-17 12:32:40,495] jepsen worker 32 - jepsen.util 6422 :fail :cas [10 [30 34]] :ValueNotSwapFrom INFO [2019-12-17 12:32:40,495] jepsen worker 32 - jepsen.util 6422 :invoke :cas [10 [29 16]] INFO [2019-12-17 12:32:40,526] jepsen worker 65 - jepsen.util 5195 :info :write [21 43] :RequestCanceledException INFO [2019-12-17 12:32:40,526] jepsen worker 31 - jepsen.util 5701 :fail :read [10 nil] :RequestTimeoutException INFO [2019-12-17 12:32:40,526] jepsen worker 31 - jepsen.util 5701 :invoke :cas [10 [38 20]] INFO [2019-12-17 12:32:40,532] jepsen worker 65 - jepsen.util 5285 :invoke :read [21 nil] INFO [2019-12-17 12:32:40,538] jepsen worker 27 - jepsen.util 6237 :ok :read [9 6] INFO [2019-12-17 12:32:40,538] jepsen worker 27 - jepsen.util 6237 :invoke :read [9 nil] INFO [2019-12-17 12:32:40,555] jepsen worker 20 - jepsen.util 6140 :ok :read [6 19] INFO [2019-12-17 12:32:40,555] jepsen worker 20 - jepsen.util 6140 :invoke :read [6 nil] INFO [2019-12-17 12:32:40,567] jepsen worker 28 - jepsen.util 4978 :ok :write [9 31] INFO [2019-12-17 12:32:40,567] jepsen worker 28 - jepsen.util 4978 :invoke :write [9 23] INFO [2019-12-17 12:32:40,580] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:32:40,580] jepsen nemesis - couchbase.util Rebalance check count: 274 INFO [2019-12-17 12:32:40,581] jepsen nemesis - couchbase.util Rebalance stuck count: 255 INFO [2019-12-17 12:32:40,594] jepsen worker 13 - jepsen.util 6043 :info :write [4 6] :RequestCanceledException INFO [2019-12-17 12:32:40,595] jepsen worker 13 - jepsen.util 6133 :invoke :read [4 nil] INFO [2019-12-17 12:32:40,625] jepsen worker 64 - jepsen.util 4924 :ok :read [21 19] INFO [2019-12-17 12:32:40,625] jepsen worker 64 - jepsen.util 4924 :invoke :cas [21 [8 37]] INFO [2019-12-17 12:32:40,632] jepsen worker 80 - jepsen.util 6290 :ok :read [26 44] INFO [2019-12-17 12:32:40,632] jepsen worker 80 - jepsen.util 6290 :invoke :write [26 6] INFO [2019-12-17 12:32:40,644] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36259,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:32:40,644] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36259,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:32:40,644] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36263,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:32:40,644] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36263,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:32:40,644] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36264,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:32:40,645] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36264,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:32:40,658] jepsen worker 51 - jepsen.util 6171 :info :write [17 30] :RequestCanceledException INFO [2019-12-17 12:32:40,659] jepsen worker 51 - jepsen.util 6261 :invoke :cas [17 [28 33]] INFO [2019-12-17 12:32:40,672] jepsen worker 70 - jepsen.util 6820 :fail :cas [23 [3 38]] :ValueNotSwapFrom INFO [2019-12-17 12:32:40,673] jepsen worker 70 - jepsen.util 6820 :invoke :cas [23 [26 33]] INFO [2019-12-17 12:32:40,681] jepsen worker 65 - jepsen.util 5285 :ok :read [21 19] INFO [2019-12-17 12:32:40,681] jepsen worker 65 - jepsen.util 5285 :invoke :read [21 nil] INFO [2019-12-17 12:32:40,694] jepsen worker 4 - jepsen.util 5674 :info :write [1 17] :RequestCanceledException INFO [2019-12-17 12:32:40,695] jepsen worker 4 - jepsen.util 5764 :invoke :read [1 nil] INFO [2019-12-17 12:32:40,703] jepsen worker 28 - jepsen.util 4978 :ok :write [9 23] INFO [2019-12-17 12:32:40,703] jepsen worker 28 - jepsen.util 4978 :invoke :write [9 47] INFO [2019-12-17 12:32:40,718] jepsen worker 31 - jepsen.util 5701 :fail :cas [10 [38 20]] :ValueNotSwapFrom INFO [2019-12-17 12:32:40,718] jepsen worker 31 - jepsen.util 5701 :invoke :read [10 nil] INFO [2019-12-17 12:32:40,727] jepsen worker 55 - jepsen.util 7255 :info :write [18 48] :RequestCanceledException INFO [2019-12-17 12:32:40,728] jepsen worker 38 - jepsen.util 6248 :info :write [12 6] :RequestCanceledException INFO [2019-12-17 12:32:40,729] jepsen worker 38 - jepsen.util 6338 :invoke :write [12 41] INFO [2019-12-17 12:32:40,730] jepsen worker 5 - jepsen.util 5855 :ok :read [1 40] INFO [2019-12-17 12:32:40,730] jepsen worker 5 - jepsen.util 5855 :invoke :read [1 nil] INFO [2019-12-17 12:32:40,740] jepsen worker 27 - jepsen.util 6237 :ok :read [9 31] INFO [2019-12-17 12:32:40,740] jepsen worker 55 - jepsen.util 7345 :invoke :read [18 nil] INFO [2019-12-17 12:32:40,740] jepsen worker 27 - jepsen.util 6237 :invoke :write [9 12] INFO [2019-12-17 12:32:40,742] jepsen worker 65 - jepsen.util 5285 :ok :read [21 19] INFO [2019-12-17 12:32:40,742] jepsen worker 65 - jepsen.util 5285 :invoke :read [21 nil] INFO [2019-12-17 12:32:40,764] jepsen worker 64 - jepsen.util 4924 :fail :cas [21 [8 37]] :ValueNotSwapFrom INFO [2019-12-17 12:32:40,764] jepsen worker 64 - jepsen.util 4924 :invoke :cas [21 [41 28]] INFO [2019-12-17 12:32:40,776] jepsen worker 31 - jepsen.util 5701 :ok :read [10 48] INFO [2019-12-17 12:32:40,776] jepsen worker 31 - jepsen.util 5701 :invoke :read [10 nil] INFO [2019-12-17 12:32:40,781] jepsen worker 71 - jepsen.util 7001 :ok :read [23 37] INFO [2019-12-17 12:32:40,781] jepsen worker 71 - jepsen.util 7001 :invoke :read [23 nil] INFO [2019-12-17 12:32:40,799] jepsen worker 65 - jepsen.util 5285 :ok :read [21 19] INFO [2019-12-17 12:32:40,799] jepsen worker 65 - jepsen.util 5285 :invoke :write [21 19] INFO [2019-12-17 12:32:40,821] jepsen worker 28 - jepsen.util 4978 :ok :write [9 47] INFO [2019-12-17 12:32:40,821] jepsen worker 28 - jepsen.util 4978 :invoke :write [9 15] INFO [2019-12-17 12:32:40,834] jepsen worker 64 - jepsen.util 4924 :fail :cas [21 [41 28]] :ValueNotSwapFrom INFO [2019-12-17 12:32:40,834] jepsen worker 64 - jepsen.util 4924 :invoke :read [21 nil] INFO [2019-12-17 12:32:40,838] jepsen worker 87 - jepsen.util 5667 :info :write [29 42] :RequestCanceledException INFO [2019-12-17 12:32:40,838] jepsen worker 54 - jepsen.util 6894 :info :write [18 11] :RequestCanceledException INFO [2019-12-17 12:32:40,839] jepsen worker 54 - jepsen.util 6984 :invoke :write [18 1] INFO [2019-12-17 12:32:40,842] jepsen worker 45 - jepsen.util 6255 :ok :write [15 21] INFO [2019-12-17 12:32:40,842] jepsen worker 45 - jepsen.util 6255 :invoke :read [15 nil] INFO [2019-12-17 12:32:40,843] jepsen worker 87 - jepsen.util 5757 :invoke :cas [29 [47 11]] INFO [2019-12-17 12:32:40,847] jepsen worker 10 - jepsen.util 6130 :ok :write [3 9] INFO [2019-12-17 12:32:40,847] jepsen worker 10 - jepsen.util 6130 :invoke :write [3 30] INFO [2019-12-17 12:32:40,858] jepsen worker 79 - jepsen.util 6289 :info :write [26 1] :RequestCanceledException INFO [2019-12-17 12:32:40,859] jepsen worker 79 - jepsen.util 6379 :invoke :cas [26 [33 48]] INFO [2019-12-17 12:32:40,884] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36286,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:32:40,885] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36286,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:32:40,885] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36289,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:32:40,885] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36289,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:32:40,918] jepsen worker 27 - jepsen.util 6237 :ok :write [9 12] INFO [2019-12-17 12:32:40,918] jepsen worker 27 - jepsen.util 6237 :invoke :read [9 nil] INFO [2019-12-17 12:32:40,954] jepsen worker 65 - jepsen.util 5285 :ok :write [21 19] INFO [2019-12-17 12:32:40,955] jepsen worker 65 - jepsen.util 5285 :invoke :read [21 nil] INFO [2019-12-17 12:32:40,963] jepsen worker 20 - jepsen.util 6140 :ok :read [6 19] INFO [2019-12-17 12:32:40,964] jepsen worker 20 - jepsen.util 6140 :invoke :read [6 nil] INFO [2019-12-17 12:32:40,970] jepsen worker 30 - jepsen.util 6240 :fail :cas [10 [0 11]] :ValueNotSwapFrom INFO [2019-12-17 12:32:40,970] jepsen worker 30 - jepsen.util 6240 :invoke :read [10 nil] INFO [2019-12-17 12:32:40,998] jepsen worker 31 - jepsen.util 5701 :ok :read [10 48] INFO [2019-12-17 12:32:40,998] jepsen worker 31 - jepsen.util 5701 :invoke :cas [10 [23 1]] INFO [2019-12-17 12:32:41,074] jepsen worker 45 - jepsen.util 6255 :ok :read [15 21] INFO [2019-12-17 12:32:41,075] jepsen worker 45 - jepsen.util 6255 :invoke :cas [15 [23 48]] INFO [2019-12-17 12:32:41,080] jepsen worker 64 - jepsen.util 4924 :ok :read [21 19] INFO [2019-12-17 12:32:41,080] jepsen worker 64 - jepsen.util 4924 :invoke :write [21 9] INFO [2019-12-17 12:32:41,081] jepsen worker 88 - jepsen.util 6478 :ok :read [29 42] INFO [2019-12-17 12:32:41,081] jepsen worker 88 - jepsen.util 6478 :invoke :cas [29 [29 3]] INFO [2019-12-17 12:32:41,094] jepsen worker 19 - jepsen.util 6139 :info :write [6 32] :RequestCanceledException INFO [2019-12-17 12:32:41,094] jepsen worker 5 - jepsen.util 5855 :ok :read [1 40] INFO [2019-12-17 12:32:41,094] jepsen worker 5 - jepsen.util 5855 :invoke :cas [1 [44 3]] INFO [2019-12-17 12:32:41,097] jepsen worker 19 - jepsen.util 6229 :invoke :read [6 nil] INFO [2019-12-17 12:32:41,103] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36298,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:32:41,103] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36298,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:32:41,104] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36302,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:32:41,104] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36302,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:32:41,109] jepsen worker 28 - jepsen.util 4978 :ok :write [9 15] INFO [2019-12-17 12:32:41,109] jepsen worker 28 - jepsen.util 4978 :invoke :write [9 8] INFO [2019-12-17 12:32:41,118] jepsen worker 65 - jepsen.util 5285 :ok :read [21 19] INFO [2019-12-17 12:32:41,119] jepsen worker 65 - jepsen.util 5285 :invoke :cas [21 [41 48]] INFO [2019-12-17 12:32:41,127] jepsen worker 68 - jepsen.util 7988 :info :cas [22 [27 22]] :RequestCanceledException INFO [2019-12-17 12:32:41,130] jepsen worker 20 - jepsen.util 6140 :ok :read [6 19] INFO [2019-12-17 12:32:41,130] jepsen worker 20 - jepsen.util 6140 :invoke :cas [6 [26 38]] INFO [2019-12-17 12:32:41,132] jepsen worker 68 - jepsen.util 8078 :invoke :write [22 46] INFO [2019-12-17 12:32:41,159] jepsen worker 27 - jepsen.util 6237 :ok :read [9 12] INFO [2019-12-17 12:32:41,159] jepsen worker 27 - jepsen.util 6237 :invoke :cas [9 [7 30]] INFO [2019-12-17 12:32:41,211] jepsen worker 69 - jepsen.util 6279 :info :write [23 49] :RequestCanceledException INFO [2019-12-17 12:32:41,214] jepsen worker 69 - jepsen.util 6369 :invoke :read [23 nil] INFO [2019-12-17 12:32:41,227] jepsen worker 8 - jepsen.util 8108 :info :cas [2 [11 16]] :RequestCanceledException INFO [2019-12-17 12:32:41,228] jepsen worker 66 - jepsen.util 8076 :info :write [22 21] :RequestCanceledException INFO [2019-12-17 12:32:41,233] jepsen worker 8 - jepsen.util 8198 :invoke :cas [2 [24 34]] INFO [2019-12-17 12:32:41,234] jepsen worker 66 - jepsen.util 8166 :invoke :write [22 48] INFO [2019-12-17 12:32:41,241] jepsen worker 85 - jepsen.util 9355 :info :write [28 20] :RequestCanceledException INFO [2019-12-17 12:32:41,247] jepsen worker 85 - jepsen.util 9445 :invoke :write [28 27] INFO [2019-12-17 12:32:41,255] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36314,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:32:41,255] jepsen worker 14 - jepsen.util 6404 :info :write [4 46] :RequestCanceledException INFO [2019-12-17 12:32:41,255] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36314,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:32:41,256] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36315,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:32:41,256] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36315,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:32:41,256] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36316,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:32:41,257] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36316,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:32:41,261] jepsen worker 13 - jepsen.util 6133 :ok :read [4 13] INFO [2019-12-17 12:32:41,261] jepsen worker 13 - jepsen.util 6133 :invoke :write [4 9] INFO [2019-12-17 12:32:41,264] jepsen worker 14 - jepsen.util 6494 :invoke :write [4 26] INFO [2019-12-17 12:32:41,276] jepsen worker 4 - jepsen.util 5764 :ok :read [1 40] INFO [2019-12-17 12:32:41,276] jepsen worker 4 - jepsen.util 5764 :invoke :write [1 34] INFO [2019-12-17 12:32:41,293] jepsen worker 60 - jepsen.util 9330 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:32:41,293] jepsen worker 60 - jepsen.util 9330 :invoke :cas [20 [33 11]] INFO [2019-12-17 12:32:41,294] jepsen worker 56 - jepsen.util 6716 :info :write [18 43] :RequestCanceledException INFO [2019-12-17 12:32:41,302] jepsen worker 56 - jepsen.util 6806 :invoke :write [18 16] INFO [2019-12-17 12:32:41,305] jepsen worker 88 - jepsen.util 6478 :fail :cas [29 [29 3]] :ValueNotSwapFrom INFO [2019-12-17 12:32:41,305] jepsen worker 88 - jepsen.util 6478 :invoke :read [29 nil] INFO [2019-12-17 12:32:41,322] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36321,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:32:41,322] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36321,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:32:41,330] jepsen worker 45 - jepsen.util 6255 :fail :cas [15 [23 48]] :ValueNotSwapFrom INFO [2019-12-17 12:32:41,330] jepsen worker 45 - jepsen.util 6255 :invoke :write [15 28] INFO [2019-12-17 12:32:41,334] jepsen worker 31 - jepsen.util 5701 :fail :cas [10 [23 1]] :ValueNotSwapFrom INFO [2019-12-17 12:32:41,334] jepsen worker 31 - jepsen.util 5701 :invoke :cas [10 [31 38]] INFO [2019-12-17 12:32:41,343] jepsen worker 65 - jepsen.util 5285 :fail :cas [21 [41 48]] :ValueNotSwapFrom INFO [2019-12-17 12:32:41,343] jepsen worker 65 - jepsen.util 5285 :invoke :cas [21 [7 8]] INFO [2019-12-17 12:32:41,358] jepsen worker 34 - jepsen.util 7954 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:32:41,358] jepsen worker 34 - jepsen.util 7954 :invoke :read [11 nil] INFO [2019-12-17 12:32:41,379] jepsen worker 20 - jepsen.util 6140 :fail :cas [6 [26 38]] :ValueNotSwapFrom INFO [2019-12-17 12:32:41,380] jepsen worker 20 - jepsen.util 6140 :invoke :write [6 7] INFO [2019-12-17 12:32:41,389] jepsen worker 18 - jepsen.util 6138 :ok :write [6 33] INFO [2019-12-17 12:32:41,389] jepsen worker 18 - jepsen.util 6138 :invoke :write [6 21] INFO [2019-12-17 12:32:41,390] jepsen worker 28 - jepsen.util 4978 :ok :write [9 8] INFO [2019-12-17 12:32:41,390] jepsen worker 28 - jepsen.util 4978 :invoke :cas [9 [24 27]] INFO [2019-12-17 12:32:41,390] jepsen worker 48 - jepsen.util 7698 :info :write [16 18] :RequestCanceledException INFO [2019-12-17 12:32:41,394] jepsen worker 64 - jepsen.util 4924 :ok :write [21 9] INFO [2019-12-17 12:32:41,395] jepsen worker 64 - jepsen.util 4924 :invoke :read [21 nil] INFO [2019-12-17 12:32:41,397] jepsen worker 48 - jepsen.util 7788 :invoke :cas [16 [5 12]] INFO [2019-12-17 12:32:41,411] jepsen worker 43 - jepsen.util 7783 :info :cas [14 [2 0]] :RequestCanceledException INFO [2019-12-17 12:32:41,411] jepsen worker 36 - jepsen.util 6876 :info :write [12 29] :RequestCanceledException INFO [2019-12-17 12:32:41,411] jepsen worker 6 - jepsen.util 9096 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:32:41,411] jepsen worker 6 - jepsen.util 9096 :invoke :read [2 nil] INFO [2019-12-17 12:32:41,412] jepsen worker 36 - jepsen.util 6966 :invoke :read [12 nil] INFO [2019-12-17 12:32:41,415] jepsen worker 51 - jepsen.util 6261 :fail :cas [17 [28 33]] :ValueNotSwapFrom INFO [2019-12-17 12:32:41,415] jepsen worker 51 - jepsen.util 6261 :invoke :read [17 nil] INFO [2019-12-17 12:32:41,418] jepsen worker 43 - jepsen.util 7873 :invoke :read [14 nil] INFO [2019-12-17 12:32:41,418] jepsen worker 5 - jepsen.util 5855 :fail :cas [1 [44 3]] :ValueNotSwapFrom INFO [2019-12-17 12:32:41,418] jepsen worker 5 - jepsen.util 5855 :invoke :cas [1 [6 29]] INFO [2019-12-17 12:32:41,419] jepsen worker 55 - jepsen.util 7345 :ok :read [18 21] INFO [2019-12-17 12:32:41,419] jepsen worker 55 - jepsen.util 7345 :invoke :write [18 3] INFO [2019-12-17 12:32:41,427] jepsen worker 44 - jepsen.util 9134 :info :cas [14 [49 26]] :RequestCanceledException INFO [2019-12-17 12:32:41,427] jepsen worker 75 - jepsen.util 7995 :info :cas [25 [32 28]] :RequestCanceledException INFO [2019-12-17 12:32:41,427] jepsen worker 76 - jepsen.util 8806 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:32:41,427] jepsen worker 76 - jepsen.util 8806 :invoke :write [25 49] INFO [2019-12-17 12:32:41,435] jepsen worker 44 - jepsen.util 9224 :invoke :cas [14 [21 32]] INFO [2019-12-17 12:32:41,435] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36352,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:32:41,436] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36352,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:32:41,436] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36353,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:32:41,437] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36353,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:32:41,438] jepsen worker 75 - jepsen.util 8085 :invoke :read [25 nil] INFO [2019-12-17 12:32:41,441] jepsen worker 16 - jepsen.util 9466 :info :write [5 2] :RequestCanceledException INFO [2019-12-17 12:32:41,450] jepsen worker 16 - jepsen.util 9556 :invoke :cas [5 [22 23]] INFO [2019-12-17 12:32:41,455] jepsen worker 65 - jepsen.util 5285 :fail :cas [21 [7 8]] :ValueNotSwapFrom INFO [2019-12-17 12:32:41,455] jepsen worker 65 - jepsen.util 5285 :invoke :cas [21 [8 45]] INFO [2019-12-17 12:32:41,456] jepsen worker 82 - jepsen.util 8992 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:32:41,456] jepsen worker 82 - jepsen.util 8992 :invoke :cas [27 [20 49]] INFO [2019-12-17 12:32:41,479] jepsen worker 69 - jepsen.util 6369 :ok :read [23 37] INFO [2019-12-17 12:32:41,480] jepsen worker 69 - jepsen.util 6369 :invoke :cas [23 [49 8]] INFO [2019-12-17 12:32:41,489] jepsen worker 28 - jepsen.util 4978 :fail :cas [9 [24 27]] :ValueNotSwapFrom INFO [2019-12-17 12:32:41,489] jepsen worker 28 - jepsen.util 4978 :invoke :write [9 6] INFO [2019-12-17 12:32:41,494] jepsen worker 57 - jepsen.util 7977 :info :write [19 14] :RequestCanceledException INFO [2019-12-17 12:32:41,502] jepsen worker 57 - jepsen.util 8067 :invoke :read [19 nil] INFO [2019-12-17 12:32:41,506] jepsen worker 70 - jepsen.util 6820 :fail :cas [23 [26 33]] :ValueNotSwapFrom INFO [2019-12-17 12:32:41,507] jepsen worker 70 - jepsen.util 6820 :invoke :cas [23 [22 23]] INFO [2019-12-17 12:32:41,511] jepsen worker 41 - jepsen.util 9311 :info :cas [13 [37 0]] :RequestCanceledException INFO [2019-12-17 12:32:41,511] jepsen worker 37 - jepsen.util 6067 :info :write [12 18] :RequestCanceledException INFO [2019-12-17 12:32:41,518] jepsen worker 41 - jepsen.util 9401 :invoke :read [13 nil] INFO [2019-12-17 12:32:41,519] jepsen worker 37 - jepsen.util 6157 :invoke :cas [12 [31 18]] INFO [2019-12-17 12:32:41,527] jepsen worker 0 - jepsen.util 8910 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:32:41,527] jepsen worker 0 - jepsen.util 8910 :invoke :read [0 nil] INFO [2019-12-17 12:32:41,527] jepsen worker 58 - jepsen.util 9058 :info :write [19 37] :RequestCanceledException INFO [2019-12-17 12:32:41,527] jepsen worker 52 - jepsen.util 6712 :info :write [17 20] :RequestCanceledException INFO [2019-12-17 12:32:41,528] jepsen worker 79 - jepsen.util 6379 :fail :cas [26 [33 48]] :ValueNotSwapFrom INFO [2019-12-17 12:32:41,528] jepsen worker 52 - jepsen.util 6802 :invoke :read [17 nil] INFO [2019-12-17 12:32:41,529] jepsen worker 79 - jepsen.util 6379 :invoke :write [26 34] INFO [2019-12-17 12:32:41,538] jepsen worker 58 - jepsen.util 9148 :invoke :read [19 nil] INFO [2019-12-17 12:32:41,540] jepsen worker 45 - jepsen.util 6255 :ok :write [15 28] INFO [2019-12-17 12:32:41,541] jepsen worker 45 - jepsen.util 6255 :invoke :read [15 nil] INFO [2019-12-17 12:32:41,541] jepsen worker 33 - jepsen.util 8403 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:32:41,541] jepsen worker 33 - jepsen.util 8403 :invoke :cas [11 [9 45]] INFO [2019-12-17 12:32:41,544] jepsen worker 71 - jepsen.util 7001 :ok :read [23 37] INFO [2019-12-17 12:32:41,544] jepsen worker 71 - jepsen.util 7001 :invoke :read [23 nil] INFO [2019-12-17 12:32:41,545] jepsen worker 31 - jepsen.util 5701 :fail :cas [10 [31 38]] :ValueNotSwapFrom INFO [2019-12-17 12:32:41,545] jepsen worker 31 - jepsen.util 5701 :invoke :read [10 nil] INFO [2019-12-17 12:32:41,556] jepsen worker 64 - jepsen.util 4924 :ok :read [21 9] INFO [2019-12-17 12:32:41,557] jepsen worker 64 - jepsen.util 4924 :invoke :read [21 nil] INFO [2019-12-17 12:32:41,568] jepsen worker 65 - jepsen.util 5285 :fail :cas [21 [8 45]] :ValueNotSwapFrom INFO [2019-12-17 12:32:41,568] jepsen worker 65 - jepsen.util 5285 :invoke :cas [21 [19 30]] INFO [2019-12-17 12:32:41,603] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:32:41,603] jepsen nemesis - couchbase.util Rebalance check count: 275 INFO [2019-12-17 12:32:41,603] jepsen nemesis - couchbase.util Rebalance stuck count: 256 INFO [2019-12-17 12:32:41,612] jepsen worker 10 - jepsen.util 6130 :ok :write [3 30] INFO [2019-12-17 12:32:41,612] jepsen worker 10 - jepsen.util 6130 :invoke :cas [3 [35 35]] INFO [2019-12-17 12:32:41,618] jepsen worker 31 - jepsen.util 5701 :ok :read [10 48] INFO [2019-12-17 12:32:41,618] jepsen worker 31 - jepsen.util 5701 :invoke :write [10 48] INFO [2019-12-17 12:32:41,641] jepsen worker 87 - jepsen.util 5757 :fail :cas [29 [47 11]] :ValueNotSwapFrom INFO [2019-12-17 12:32:41,641] jepsen worker 87 - jepsen.util 5757 :invoke :cas [29 [19 10]] INFO [2019-12-17 12:32:41,658] jepsen worker 86 - jepsen.util 8906 :info :write [28 44] :RequestCanceledException INFO [2019-12-17 12:32:41,662] jepsen worker 64 - jepsen.util 4924 :ok :read [21 9] INFO [2019-12-17 12:32:41,662] jepsen worker 64 - jepsen.util 4924 :invoke :write [21 4] INFO [2019-12-17 12:32:41,664] jepsen worker 86 - jepsen.util 8996 :invoke :read [28 nil] INFO [2019-12-17 12:32:41,670] jepsen worker 65 - jepsen.util 5285 :fail :cas [21 [19 30]] :ValueNotSwapFrom INFO [2019-12-17 12:32:41,670] jepsen worker 65 - jepsen.util 5285 :invoke :write [21 1] INFO [2019-12-17 12:32:41,679] jepsen worker 20 - jepsen.util 6140 :ok :write [6 7] INFO [2019-12-17 12:32:41,679] jepsen worker 20 - jepsen.util 6140 :invoke :cas [6 [26 36]] INFO [2019-12-17 12:32:41,685] jepsen worker 10 - jepsen.util 6130 :fail :cas [3 [35 35]] :ValueNotSwapFrom INFO [2019-12-17 12:32:41,685] jepsen worker 10 - jepsen.util 6130 :invoke :read [3 nil] INFO [2019-12-17 12:32:41,693] jepsen worker 21 - jepsen.util 8121 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:32:41,693] jepsen worker 21 - jepsen.util 8121 :invoke :read [7 nil] INFO [2019-12-17 12:32:41,711] jepsen worker 62 - jepsen.util 8072 :info :cas [20 [5 19]] :RequestCanceledException INFO [2019-12-17 12:32:41,713] jepsen worker 62 - jepsen.util 8162 :invoke :cas [20 [14 8]] INFO [2019-12-17 12:32:41,725] jepsen worker 77 - jepsen.util 8897 :info :cas [25 [40 32]] :RequestCanceledException INFO [2019-12-17 12:32:41,725] jepsen worker 81 - jepsen.util 8091 :info :write [27 39] :RequestCanceledException INFO [2019-12-17 12:32:41,725] jepsen worker 39 - jepsen.util 8049 :info :cas [13 [22 30]] :RequestCanceledException INFO [2019-12-17 12:32:41,728] jepsen worker 39 - jepsen.util 8139 :invoke :cas [13 [27 8]] INFO [2019-12-17 12:32:41,728] jepsen worker 77 - jepsen.util 8987 :invoke :write [25 33] INFO [2019-12-17 12:32:41,730] jepsen worker 81 - jepsen.util 8181 :invoke :read [27 nil] INFO [2019-12-17 12:32:41,731] jepsen worker 28 - jepsen.util 4978 :ok :write [9 6] INFO [2019-12-17 12:32:41,731] jepsen worker 28 - jepsen.util 4978 :invoke :write [9 25] INFO [2019-12-17 12:32:41,740] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36382,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:32:41,740] jepsen worker 84 - jepsen.util 8004 :info :write [28 45] :RequestCanceledException INFO [2019-12-17 12:32:41,740] jepsen worker 61 - jepsen.util 9511 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:32:41,740] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36382,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:32:41,740] jepsen worker 61 - jepsen.util 9511 :invoke :read [20 nil] INFO [2019-12-17 12:32:41,740] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36386,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:32:41,740] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36386,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:32:41,742] jepsen worker 84 - jepsen.util 8094 :invoke :read [28 nil] INFO [2019-12-17 12:32:41,758] jepsen worker 83 - jepsen.util 8903 :info :write [27 28] :RequestCanceledException INFO [2019-12-17 12:32:41,764] jepsen worker 83 - jepsen.util 8993 :invoke :read [27 nil] INFO [2019-12-17 12:32:41,788] jepsen worker 10 - jepsen.util 6130 :ok :read [3 30] INFO [2019-12-17 12:32:41,788] jepsen worker 10 - jepsen.util 6130 :invoke :write [3 17] INFO [2019-12-17 12:32:41,789] jepsen worker 20 - jepsen.util 6140 :fail :cas [6 [26 36]] :ValueNotSwapFrom INFO [2019-12-17 12:32:41,789] jepsen worker 20 - jepsen.util 6140 :invoke :cas [6 [25 29]] INFO [2019-12-17 12:32:41,792] jepsen worker 37 - jepsen.util 6157 :fail :cas [12 [31 18]] :ValueNotSwapFrom INFO [2019-12-17 12:32:41,792] jepsen worker 37 - jepsen.util 6157 :invoke :read [12 nil] INFO [2019-12-17 12:32:41,794] jepsen worker 1 - jepsen.util 8281 :info :cas [0 [20 39]] :RequestCanceledException INFO [2019-12-17 12:32:41,797] jepsen worker 1 - jepsen.util 8371 :invoke :cas [0 [15 18]] INFO [2019-12-17 12:32:41,803] jepsen worker 65 - jepsen.util 5285 :ok :write [21 1] INFO [2019-12-17 12:32:41,803] jepsen worker 65 - jepsen.util 5285 :invoke :read [21 nil] INFO [2019-12-17 12:32:41,817] jepsen worker 31 - jepsen.util 5701 :ok :write [10 48] INFO [2019-12-17 12:32:41,817] jepsen worker 31 - jepsen.util 5701 :invoke :cas [10 [48 25]] INFO [2019-12-17 12:32:41,840] jepsen worker 78 - jepsen.util 6558 :info :write [26 44] :RequestCanceledException INFO [2019-12-17 12:32:41,842] jepsen worker 78 - jepsen.util 6648 :invoke :write [26 22] INFO [2019-12-17 12:32:41,867] jepsen worker 65 - jepsen.util 5285 :ok :read [21 1] INFO [2019-12-17 12:32:41,867] jepsen worker 65 - jepsen.util 5285 :invoke :read [21 nil] INFO [2019-12-17 12:32:41,868] jepsen worker 36 - jepsen.util 6966 :ok :read [12 16] INFO [2019-12-17 12:32:41,868] jepsen worker 36 - jepsen.util 6966 :invoke :cas [12 [16 19]] INFO [2019-12-17 12:32:41,875] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36410,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:32:41,875] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36410,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:32:41,875] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36414,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:32:41,875] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36414,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:32:41,884] jepsen worker 51 - jepsen.util 6261 :ok :read [17 20] INFO [2019-12-17 12:32:41,884] jepsen worker 51 - jepsen.util 6261 :invoke :write [17 37] INFO [2019-12-17 12:32:41,902] jepsen worker 20 - jepsen.util 6140 :fail :cas [6 [25 29]] :ValueNotSwapFrom INFO [2019-12-17 12:32:41,902] jepsen worker 20 - jepsen.util 6140 :invoke :cas [6 [37 38]] INFO [2019-12-17 12:32:41,911] jepsen worker 53 - jepsen.util 5993 :info :write [17 22] :RequestCanceledException INFO [2019-12-17 12:32:41,911] jepsen worker 7 - jepsen.util 8647 :info :cas [2 [16 44]] :RequestCanceledException INFO [2019-12-17 12:32:41,913] jepsen worker 53 - jepsen.util 6083 :invoke :read [17 nil] INFO [2019-12-17 12:32:41,915] jepsen worker 7 - jepsen.util 8737 :invoke :write [2 38] INFO [2019-12-17 12:32:41,927] jepsen worker 15 - jepsen.util 7485 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:32:41,927] jepsen worker 15 - jepsen.util 7485 :invoke :write [5 41] INFO [2019-12-17 12:32:41,927] jepsen worker 40 - jepsen.util 8770 :info :cas [13 [23 49]] :RequestCanceledException INFO [2019-12-17 12:32:41,927] jepsen worker 67 - jepsen.util 8707 :info :cas [22 [1 31]] :RequestCanceledException INFO [2019-12-17 12:32:41,931] jepsen worker 40 - jepsen.util 8860 :invoke :write [13 19] INFO [2019-12-17 12:32:41,933] jepsen worker 67 - jepsen.util 8797 :invoke :cas [22 [2 39]] INFO [2019-12-17 12:32:41,941] jepsen worker 22 - jepsen.util 8392 :info :cas [7 [23 6]] :RequestCanceledException INFO [2019-12-17 12:32:41,949] jepsen worker 22 - jepsen.util 8482 :invoke :write [7 29] INFO [2019-12-17 12:32:41,952] jepsen worker 13 - jepsen.util 6133 :ok :write [4 9] INFO [2019-12-17 12:32:41,952] jepsen worker 13 - jepsen.util 6133 :invoke :read [4 nil] INFO [2019-12-17 12:32:41,957] jepsen worker 65 - jepsen.util 5285 :ok :read [21 1] INFO [2019-12-17 12:32:41,957] jepsen worker 65 - jepsen.util 5285 :invoke :cas [21 [25 35]] INFO [2019-12-17 12:32:41,958] jepsen worker 3 - jepsen.util 6213 :info :write [1 12] :RequestCanceledException INFO [2019-12-17 12:32:41,965] jepsen worker 3 - jepsen.util 6303 :invoke :read [1 nil] INFO [2019-12-17 12:32:41,994] jepsen worker 17 - jepsen.util 8477 :info :cas [5 [28 48]] :RequestCanceledException INFO [2019-12-17 12:32:41,994] jepsen worker 25 - jepsen.util 8935 :info :cas [8 [16 27]] :RequestCanceledException INFO [2019-12-17 12:32:41,996] jepsen worker 25 - jepsen.util 9025 :invoke :cas [8 [42 30]] INFO [2019-12-17 12:32:41,997] jepsen worker 17 - jepsen.util 8567 :invoke :cas [5 [16 9]] INFO [2019-12-17 12:32:42,002] jepsen worker 52 - jepsen.util 6802 :ok :read [17 20] INFO [2019-12-17 12:32:42,002] jepsen worker 52 - jepsen.util 6802 :invoke :write [17 32] INFO [2019-12-17 12:32:42,008] jepsen worker 80 - jepsen.util 6290 :ok :write [26 6] INFO [2019-12-17 12:32:42,008] jepsen worker 80 - jepsen.util 6290 :invoke :write [26 6] INFO [2019-12-17 12:32:42,010] jepsen worker 64 - jepsen.util 4924 :ok :write [21 4] INFO [2019-12-17 12:32:42,010] jepsen worker 64 - jepsen.util 4924 :invoke :read [21 nil] INFO [2019-12-17 12:32:42,027] jepsen worker 23 - jepsen.util 8483 :info :write [7 38] :RequestCanceledException INFO [2019-12-17 12:32:42,027] jepsen worker 12 - jepsen.util 6582 :info :write [4 26] :RequestCanceledException INFO [2019-12-17 12:32:42,027] jepsen worker 42 - jepsen.util 9042 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:32:42,027] jepsen worker 42 - jepsen.util 9042 :invoke :read [14 nil] INFO [2019-12-17 12:32:42,033] jepsen worker 12 - jepsen.util 6672 :invoke :write [4 19] INFO [2019-12-17 12:32:42,035] jepsen worker 23 - jepsen.util 8573 :invoke :write [7 13] INFO [2019-12-17 12:32:42,036] jepsen worker 65 - jepsen.util 5285 :fail :cas [21 [25 35]] :ValueNotSwapFrom INFO [2019-12-17 12:32:42,037] jepsen worker 65 - jepsen.util 5285 :invoke :cas [21 [49 28]] INFO [2019-12-17 12:32:42,041] jepsen worker 56 - jepsen.util 6806 :ok :write [18 16] INFO [2019-12-17 12:32:42,041] jepsen worker 56 - jepsen.util 6806 :invoke :read [18 nil] INFO [2019-12-17 12:32:42,048] jepsen worker 10 - jepsen.util 6130 :ok :write [3 17] INFO [2019-12-17 12:32:42,048] jepsen worker 10 - jepsen.util 6130 :invoke :write [3 49] INFO [2019-12-17 12:32:42,054] jepsen worker 28 - jepsen.util 4978 :ok :write [9 25] INFO [2019-12-17 12:32:42,054] jepsen worker 28 - jepsen.util 4978 :invoke :write [9 25] INFO [2019-12-17 12:32:42,059] jepsen worker 20 - jepsen.util 6140 :fail :cas [6 [37 38]] :ValueNotSwapFrom INFO [2019-12-17 12:32:42,060] jepsen worker 20 - jepsen.util 6140 :invoke :write [6 3] INFO [2019-12-17 12:32:42,086] jepsen worker 64 - jepsen.util 4924 :ok :read [21 4] INFO [2019-12-17 12:32:42,086] jepsen worker 64 - jepsen.util 4924 :invoke :cas [21 [3 37]] INFO [2019-12-17 12:32:42,087] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36432,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:32:42,087] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36432,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:32:42,087] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36433,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:32:42,087] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36433,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:32:42,093] jepsen worker 74 - jepsen.util 6464 :info :write [24 1] :RequestCanceledException INFO [2019-12-17 12:32:42,097] jepsen worker 74 - jepsen.util 6554 :invoke :read [24 nil] INFO [2019-12-17 12:32:42,122] jepsen worker 65 - jepsen.util 5285 :fail :cas [21 [49 28]] :ValueNotSwapFrom INFO [2019-12-17 12:32:42,122] jepsen worker 65 - jepsen.util 5285 :invoke :read [21 nil] INFO [2019-12-17 12:32:42,125] jepsen worker 24 - jepsen.util 7584 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:32:42,126] jepsen worker 24 - jepsen.util 7584 :invoke :write [8 41] INFO [2019-12-17 12:32:42,141] jepsen worker 26 - jepsen.util 8756 :info :write [8 22] :RequestCanceledException INFO [2019-12-17 12:32:42,146] jepsen worker 26 - jepsen.util 8846 :invoke :cas [8 [36 27]] INFO [2019-12-17 12:32:42,158] jepsen worker 64 - jepsen.util 4924 :fail :cas [21 [3 37]] :ValueNotSwapFrom INFO [2019-12-17 12:32:42,158] jepsen worker 59 - jepsen.util 8609 :info :write [19 1] :RequestCanceledException INFO [2019-12-17 12:32:42,158] jepsen worker 64 - jepsen.util 4924 :invoke :cas [21 [25 15]] INFO [2019-12-17 12:32:42,160] jepsen worker 59 - jepsen.util 8699 :invoke :cas [19 [38 16]] INFO [2019-12-17 12:32:42,171] jepsen worker 89 - jepsen.util 6119 :ok :write [29 5] INFO [2019-12-17 12:32:42,171] jepsen worker 89 - jepsen.util 6119 :invoke :cas [29 [12 6]] INFO [2019-12-17 12:32:42,184] jepsen worker 70 - jepsen.util 6820 :fail :cas [23 [22 23]] :ValueNotSwapFrom INFO [2019-12-17 12:32:42,184] jepsen worker 70 - jepsen.util 6820 :invoke :cas [23 [5 27]] INFO [2019-12-17 12:32:42,193] jepsen worker 49 - jepsen.util 7519 :info :cas [16 [22 8]] :RequestCanceledException INFO [2019-12-17 12:32:42,199] jepsen worker 49 - jepsen.util 7609 :invoke :cas [16 [25 20]] INFO [2019-12-17 12:32:42,204] jepsen worker 65 - jepsen.util 5285 :ok :read [21 4] INFO [2019-12-17 12:32:42,204] jepsen worker 65 - jepsen.util 5285 :invoke :write [21 19] INFO [2019-12-17 12:32:42,208] jepsen worker 10 - jepsen.util 6130 :ok :write [3 49] INFO [2019-12-17 12:32:42,208] jepsen worker 10 - jepsen.util 6130 :invoke :write [3 7] INFO [2019-12-17 12:32:42,211] jepsen worker 35 - jepsen.util 8405 :info :write [11 11] :RequestCanceledException INFO [2019-12-17 12:32:42,218] jepsen worker 35 - jepsen.util 8495 :invoke :read [11 nil] INFO [2019-12-17 12:32:42,236] jepsen worker 64 - jepsen.util 4924 :fail :cas [21 [25 15]] :ValueNotSwapFrom INFO [2019-12-17 12:32:42,236] jepsen worker 64 - jepsen.util 4924 :invoke :cas [21 [25 5]] INFO [2019-12-17 12:32:42,242] jepsen worker 71 - jepsen.util 7001 :ok :read [23 37] INFO [2019-12-17 12:32:42,242] jepsen worker 71 - jepsen.util 7001 :invoke :read [23 nil] INFO [2019-12-17 12:32:42,245] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36456,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:32:42,245] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36456,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:32:42,245] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36458,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:32:42,245] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36458,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:32:42,257] jepsen worker 87 - jepsen.util 5757 :fail :cas [29 [19 10]] :ValueNotSwapFrom INFO [2019-12-17 12:32:42,257] jepsen worker 87 - jepsen.util 5757 :invoke :read [29 nil] INFO [2019-12-17 12:32:42,262] jepsen worker 28 - jepsen.util 4978 :ok :write [9 25] INFO [2019-12-17 12:32:42,262] jepsen worker 28 - jepsen.util 4978 :invoke :cas [9 [33 2]] INFO [2019-12-17 12:32:42,284] jepsen worker 37 - jepsen.util 6157 :ok :read [12 16] INFO [2019-12-17 12:32:42,284] jepsen worker 37 - jepsen.util 6157 :invoke :read [12 nil] INFO [2019-12-17 12:32:42,299] jepsen worker 55 - jepsen.util 7345 :ok :write [18 3] INFO [2019-12-17 12:32:42,300] jepsen worker 55 - jepsen.util 7345 :invoke :read [18 nil] INFO [2019-12-17 12:32:42,304] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36464,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:32:42,304] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36464,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:32:42,304] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36467,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:32:42,304] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36467,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:32:42,304] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36468,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:32:42,305] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36468,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:32:42,311] jepsen worker 64 - jepsen.util 4924 :fail :cas [21 [25 5]] :ValueNotSwapFrom INFO [2019-12-17 12:32:42,312] jepsen worker 64 - jepsen.util 4924 :invoke :write [21 39] INFO [2019-12-17 12:32:42,316] jepsen worker 56 - jepsen.util 6806 :ok :read [18 3] INFO [2019-12-17 12:32:42,316] jepsen worker 56 - jepsen.util 6806 :invoke :write [18 29] INFO [2019-12-17 12:32:42,327] jepsen worker 46 - jepsen.util 6256 :info :cas [15 [7 29]] :RequestCanceledException INFO [2019-12-17 12:32:42,327] jepsen worker 2 - jepsen.util 8192 :info :write [0 7] :RequestCanceledException INFO [2019-12-17 12:32:42,328] jepsen worker 46 - jepsen.util 6346 :invoke :write [15 29] INFO [2019-12-17 12:32:42,330] jepsen worker 65 - jepsen.util 5285 :ok :write [21 19] INFO [2019-12-17 12:32:42,330] jepsen worker 65 - jepsen.util 5285 :invoke :cas [21 [37 43]] INFO [2019-12-17 12:32:42,332] jepsen worker 2 - jepsen.util 8282 :invoke :write [0 11] INFO [2019-12-17 12:32:42,355] jepsen worker 29 - jepsen.util 4709 :info :write [9 25] :RequestCanceledException INFO [2019-12-17 12:32:42,356] jepsen worker 28 - jepsen.util 4978 :fail :cas [9 [33 2]] :ValueNotSwapFrom INFO [2019-12-17 12:32:42,356] jepsen worker 28 - jepsen.util 4978 :invoke :cas [9 [32 11]] INFO [2019-12-17 12:32:42,359] jepsen worker 29 - jepsen.util 4799 :invoke :read [9 nil] INFO [2019-12-17 12:32:42,363] jepsen worker 88 - jepsen.util 6478 :ok :read [29 5] INFO [2019-12-17 12:32:42,363] jepsen worker 88 - jepsen.util 6478 :invoke :cas [29 [9 24]] INFO [2019-12-17 12:32:42,383] jepsen worker 53 - jepsen.util 6083 :ok :read [17 20] INFO [2019-12-17 12:32:42,383] jepsen worker 53 - jepsen.util 6083 :invoke :write [17 35] INFO [2019-12-17 12:32:42,396] jepsen worker 65 - jepsen.util 5285 :fail :cas [21 [37 43]] :ValueNotSwapFrom INFO [2019-12-17 12:32:42,396] jepsen worker 65 - jepsen.util 5285 :invoke :read [21 nil] INFO [2019-12-17 12:32:42,423] jepsen worker 28 - jepsen.util 4978 :fail :cas [9 [32 11]] :ValueNotSwapFrom INFO [2019-12-17 12:32:42,423] jepsen worker 28 - jepsen.util 4978 :invoke :write [9 17] INFO [2019-12-17 12:32:42,425] jepsen worker 11 - jepsen.util 6491 :info :write [3 24] :RequestCanceledException INFO [2019-12-17 12:32:42,426] jepsen worker 73 - jepsen.util 5833 :info :write [24 26] :RequestCanceledException INFO [2019-12-17 12:32:42,426] jepsen worker 50 - jepsen.util 8960 :info :cas [16 [0 26]] :RequestCanceledException INFO [2019-12-17 12:32:42,435] jepsen worker 73 - jepsen.util 5923 :invoke :write [24 6] INFO [2019-12-17 12:32:42,438] jepsen worker 11 - jepsen.util 6581 :invoke :cas [3 [3 27]] INFO [2019-12-17 12:32:42,439] jepsen worker 50 - jepsen.util 9050 :invoke :cas [16 [40 24]] INFO [2019-12-17 12:32:42,441] jepsen worker 64 - jepsen.util 4924 :ok :write [21 39] INFO [2019-12-17 12:32:42,441] jepsen worker 64 - jepsen.util 4924 :invoke :write [21 25] INFO [2019-12-17 12:32:42,456] jepsen worker 69 - jepsen.util 6369 :fail :cas [23 [49 8]] :ValueNotSwapFrom INFO [2019-12-17 12:32:42,458] jepsen worker 63 - jepsen.util 5283 :fail :read [21 nil] :RequestTimeoutException INFO [2019-12-17 12:32:42,488] jepsen worker 63 - jepsen.util 5283 :invoke :write [21 19] INFO [2019-12-17 12:32:42,488] jepsen worker 69 - jepsen.util 6369 :invoke :cas [23 [19 10]] INFO [2019-12-17 12:32:42,509] jepsen worker 29 - jepsen.util 4799 :ok :read [9 25] INFO [2019-12-17 12:32:42,510] jepsen worker 29 - jepsen.util 4799 :invoke :write [9 40] INFO [2019-12-17 12:32:42,510] jepsen worker 47 - jepsen.util 6887 :fail :read [15 nil] :RequestTimeoutException INFO [2019-12-17 12:32:42,511] jepsen worker 9 - jepsen.util 6759 :info :cas [3 [0 16]] :RequestCanceledException INFO [2019-12-17 12:32:42,511] jepsen worker 47 - jepsen.util 6887 :invoke :cas [15 [3 39]] INFO [2019-12-17 12:32:42,511] jepsen worker 32 - jepsen.util 6422 :info :cas [10 [29 16]] :RequestCanceledException INFO [2019-12-17 12:32:42,516] jepsen worker 13 - jepsen.util 6133 :ok :read [4 9] INFO [2019-12-17 12:32:42,516] jepsen worker 13 - jepsen.util 6133 :invoke :read [4 nil] INFO [2019-12-17 12:32:42,517] jepsen worker 9 - jepsen.util 6849 :invoke :cas [3 [49 6]] INFO [2019-12-17 12:32:42,518] jepsen worker 32 - jepsen.util 6512 :invoke :cas [10 [49 33]] INFO [2019-12-17 12:32:42,527] jepsen worker 72 - jepsen.util 6192 :fail :read [24 nil] :RequestTimeoutException INFO [2019-12-17 12:32:42,528] jepsen worker 72 - jepsen.util 6192 :invoke :write [24 17] INFO [2019-12-17 12:32:42,533] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36490,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:32:42,533] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36490,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:32:42,534] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36492,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:32:42,535] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36492,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:32:42,535] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36493,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:32:42,535] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36493,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:32:42,542] jepsen worker 5 - jepsen.util 5855 :fail :cas [1 [6 29]] :ValueNotSwapFrom INFO [2019-12-17 12:32:42,542] jepsen worker 5 - jepsen.util 5855 :invoke :read [1 nil] INFO [2019-12-17 12:32:42,547] jepsen worker 65 - jepsen.util 5285 :ok :read [21 19] INFO [2019-12-17 12:32:42,547] jepsen worker 65 - jepsen.util 5285 :invoke :cas [21 [5 37]] INFO [2019-12-17 12:32:42,552] jepsen worker 11 - jepsen.util 6581 :fail :cas [3 [3 27]] :ValueNotSwapFrom INFO [2019-12-17 12:32:42,552] jepsen worker 11 - jepsen.util 6581 :invoke :read [3 nil] INFO [2019-12-17 12:32:42,565] jepsen worker 89 - jepsen.util 6119 :fail :cas [29 [12 6]] :ValueNotSwapFrom INFO [2019-12-17 12:32:42,565] jepsen worker 89 - jepsen.util 6119 :invoke :cas [29 [41 30]] INFO [2019-12-17 12:32:42,575] jepsen worker 70 - jepsen.util 6820 :fail :cas [23 [5 27]] :ValueNotSwapFrom INFO [2019-12-17 12:32:42,575] jepsen worker 70 - jepsen.util 6820 :invoke :write [23 27] INFO [2019-12-17 12:32:42,608] jepsen worker 55 - jepsen.util 7345 :ok :read [18 3] INFO [2019-12-17 12:32:42,608] jepsen worker 55 - jepsen.util 7345 :invoke :read [18 nil] INFO [2019-12-17 12:32:42,616] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:32:42,616] jepsen nemesis - couchbase.util Rebalance check count: 276 INFO [2019-12-17 12:32:42,616] jepsen nemesis - couchbase.util Rebalance stuck count: 257 INFO [2019-12-17 12:32:42,623] jepsen worker 28 - jepsen.util 4978 :ok :write [9 17] INFO [2019-12-17 12:32:42,624] jepsen worker 28 - jepsen.util 4978 :invoke :write [9 18] INFO [2019-12-17 12:32:42,637] jepsen worker 64 - jepsen.util 4924 :ok :write [21 25] INFO [2019-12-17 12:32:42,637] jepsen worker 64 - jepsen.util 4924 :invoke :write [21 14] INFO [2019-12-17 12:32:42,677] jepsen worker 65 - jepsen.util 5285 :fail :cas [21 [5 37]] :ValueNotSwapFrom INFO [2019-12-17 12:32:42,677] jepsen worker 65 - jepsen.util 5285 :invoke :read [21 nil] INFO [2019-12-17 12:32:42,709] jepsen worker 69 - jepsen.util 6369 :fail :cas [23 [19 10]] :ValueNotSwapFrom INFO [2019-12-17 12:32:42,709] jepsen worker 69 - jepsen.util 6369 :invoke :write [23 14] INFO [2019-12-17 12:32:42,726] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36516,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:32:42,727] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36516,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:32:42,727] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36520,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:32:42,727] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36520,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:32:42,728] jepsen worker 11 - jepsen.util 6581 :ok :read [3 49] INFO [2019-12-17 12:32:42,728] jepsen worker 11 - jepsen.util 6581 :invoke :write [3 9] INFO [2019-12-17 12:32:42,738] jepsen worker 28 - jepsen.util 4978 :ok :write [9 18] INFO [2019-12-17 12:32:42,739] jepsen worker 28 - jepsen.util 4978 :invoke :write [9 1] INFO [2019-12-17 12:32:42,747] jepsen worker 87 - jepsen.util 5757 :ok :read [29 5] INFO [2019-12-17 12:32:42,748] jepsen worker 87 - jepsen.util 5757 :invoke :write [29 10] INFO [2019-12-17 12:32:42,766] jepsen worker 20 - jepsen.util 6140 :ok :write [6 3] INFO [2019-12-17 12:32:42,766] jepsen worker 20 - jepsen.util 6140 :invoke :read [6 nil] INFO [2019-12-17 12:32:42,785] jepsen worker 37 - jepsen.util 6157 :ok :read [12 16] INFO [2019-12-17 12:32:42,785] jepsen worker 37 - jepsen.util 6157 :invoke :read [12 nil] INFO [2019-12-17 12:32:42,787] jepsen worker 3 - jepsen.util 6303 :ok :read [1 40] INFO [2019-12-17 12:32:42,788] jepsen worker 3 - jepsen.util 6303 :invoke :write [1 8] INFO [2019-12-17 12:32:42,800] jepsen worker 65 - jepsen.util 5285 :ok :read [21 25] INFO [2019-12-17 12:32:42,800] jepsen worker 65 - jepsen.util 5285 :invoke :cas [21 [34 21]] INFO [2019-12-17 12:32:42,818] jepsen worker 89 - jepsen.util 6119 :fail :cas [29 [41 30]] :ValueNotSwapFrom INFO [2019-12-17 12:32:42,819] jepsen worker 89 - jepsen.util 6119 :invoke :write [29 37] INFO [2019-12-17 12:32:42,842] jepsen worker 13 - jepsen.util 6133 :ok :read [4 9] INFO [2019-12-17 12:32:42,843] jepsen worker 13 - jepsen.util 6133 :invoke :cas [4 [14 7]] INFO [2019-12-17 12:32:42,852] jepsen worker 46 - jepsen.util 6346 :ok :write [15 29] INFO [2019-12-17 12:32:42,852] jepsen worker 46 - jepsen.util 6346 :invoke :read [15 nil] INFO [2019-12-17 12:32:42,857] jepsen worker 38 - jepsen.util 6338 :info :write [12 41] :RequestCanceledException INFO [2019-12-17 12:32:42,862] jepsen worker 55 - jepsen.util 7345 :ok :read [18 3] INFO [2019-12-17 12:32:42,862] jepsen worker 55 - jepsen.util 7345 :invoke :cas [18 [29 45]] INFO [2019-12-17 12:32:42,862] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36537,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:32:42,862] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36537,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:32:42,862] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36539,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:32:42,862] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36539,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:32:42,866] jepsen worker 38 - jepsen.util 6428 :invoke :read [12 nil] INFO [2019-12-17 12:32:42,872] jepsen worker 10 - jepsen.util 6130 :ok :write [3 7] INFO [2019-12-17 12:32:42,872] jepsen worker 10 - jepsen.util 6130 :invoke :write [3 43] INFO [2019-12-17 12:32:42,894] jepsen worker 54 - jepsen.util 6984 :info :write [18 1] :RequestCanceledException INFO [2019-12-17 12:32:42,895] jepsen worker 65 - jepsen.util 5285 :fail :cas [21 [34 21]] :ValueNotSwapFrom INFO [2019-12-17 12:32:42,895] jepsen worker 65 - jepsen.util 5285 :invoke :write [21 1] INFO [2019-12-17 12:32:42,902] jepsen worker 54 - jepsen.util 7074 :invoke :read [18 nil] INFO [2019-12-17 12:32:42,909] jepsen worker 28 - jepsen.util 4978 :ok :write [9 1] INFO [2019-12-17 12:32:42,910] jepsen worker 28 - jepsen.util 4978 :invoke :write [9 35] INFO [2019-12-17 12:32:42,914] jepsen worker 63 - jepsen.util 5283 :ok :write [21 19] INFO [2019-12-17 12:32:42,914] jepsen worker 63 - jepsen.util 5283 :invoke :write [21 28] INFO [2019-12-17 12:32:42,930] jepsen worker 71 - jepsen.util 7001 :ok :read [23 37] INFO [2019-12-17 12:32:42,930] jepsen worker 71 - jepsen.util 7001 :invoke :read [23 nil] INFO [2019-12-17 12:32:42,963] jepsen worker 72 - jepsen.util 6192 :ok :write [24 17] INFO [2019-12-17 12:32:42,963] jepsen worker 72 - jepsen.util 6192 :invoke :read [24 nil] INFO [2019-12-17 12:32:42,976] jepsen worker 9 - jepsen.util 6849 :fail :cas [3 [49 6]] :ValueNotSwapFrom INFO [2019-12-17 12:32:42,976] jepsen worker 9 - jepsen.util 6849 :invoke :cas [3 [40 22]] INFO [2019-12-17 12:32:43,001] jepsen worker 37 - jepsen.util 6157 :ok :read [12 16] INFO [2019-12-17 12:32:43,002] jepsen worker 37 - jepsen.util 6157 :invoke :write [12 46] INFO [2019-12-17 12:32:43,033] jepsen worker 46 - jepsen.util 6346 :ok :read [15 29] INFO [2019-12-17 12:32:43,034] jepsen worker 46 - jepsen.util 6346 :invoke :cas [15 [7 38]] INFO [2019-12-17 12:32:43,041] jepsen worker 30 - jepsen.util 6240 :fail :read [10 nil] :RequestTimeoutException INFO [2019-12-17 12:32:43,041] jepsen worker 30 - jepsen.util 6240 :invoke :cas [10 [46 10]] INFO [2019-12-17 12:32:43,067] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36551,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:32:43,068] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36551,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:32:43,068] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36554,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:32:43,068] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36554,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:32:43,069] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36555,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:32:43,069] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36555,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:32:43,080] jepsen worker 63 - jepsen.util 5283 :ok :write [21 28] INFO [2019-12-17 12:32:43,080] jepsen worker 63 - jepsen.util 5283 :invoke :read [21 nil] INFO [2019-12-17 12:32:43,106] jepsen worker 13 - jepsen.util 6133 :fail :cas [4 [14 7]] :ValueNotSwapFrom INFO [2019-12-17 12:32:43,106] jepsen worker 13 - jepsen.util 6133 :invoke :cas [4 [8 6]] INFO [2019-12-17 12:32:43,110] jepsen worker 55 - jepsen.util 7345 :fail :cas [18 [29 45]] :ValueNotSwapFrom INFO [2019-12-17 12:32:43,111] jepsen worker 55 - jepsen.util 7345 :invoke :read [18 nil] INFO [2019-12-17 12:32:43,127] jepsen worker 28 - jepsen.util 4978 :ok :write [9 35] INFO [2019-12-17 12:32:43,127] jepsen worker 28 - jepsen.util 4978 :invoke :read [9 nil] INFO [2019-12-17 12:32:43,134] jepsen worker 46 - jepsen.util 6346 :fail :cas [15 [7 38]] :ValueNotSwapFrom INFO [2019-12-17 12:32:43,134] jepsen worker 46 - jepsen.util 6346 :invoke :read [15 nil] INFO [2019-12-17 12:32:43,141] jepsen worker 19 - jepsen.util 6229 :fail :read [6 nil] :RequestTimeoutException INFO [2019-12-17 12:32:43,141] jepsen worker 19 - jepsen.util 6229 :invoke :cas [6 [11 47]] INFO [2019-12-17 12:32:43,147] jepsen worker 10 - jepsen.util 6130 :ok :write [3 43] INFO [2019-12-17 12:32:43,148] jepsen worker 10 - jepsen.util 6130 :invoke :read [3 nil] INFO [2019-12-17 12:32:43,153] jepsen worker 65 - jepsen.util 5285 :ok :write [21 1] INFO [2019-12-17 12:32:43,154] jepsen worker 65 - jepsen.util 5285 :invoke :cas [21 [43 44]] INFO [2019-12-17 12:32:43,179] jepsen worker 63 - jepsen.util 5283 :ok :read [21 28] INFO [2019-12-17 12:32:43,179] jepsen worker 63 - jepsen.util 5283 :invoke :write [21 33] INFO [2019-12-17 12:32:43,185] jepsen worker 28 - jepsen.util 4978 :ok :read [9 35] INFO [2019-12-17 12:32:43,185] jepsen worker 28 - jepsen.util 4978 :invoke :write [9 40] INFO [2019-12-17 12:32:43,224] jepsen worker 30 - jepsen.util 6240 :fail :cas [10 [46 10]] :ValueNotSwapFrom INFO [2019-12-17 12:32:43,224] jepsen worker 30 - jepsen.util 6240 :invoke :write [10 15] INFO [2019-12-17 12:32:43,227] jepsen worker 27 - jepsen.util 6237 :info :cas [9 [7 30]] :RequestCanceledException INFO [2019-12-17 12:32:43,229] jepsen worker 27 - jepsen.util 6327 :invoke :write [9 29] INFO [2019-12-17 12:32:43,232] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36567,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:32:43,232] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36567,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:32:43,232] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36568,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:32:43,232] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36568,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:32:43,258] jepsen worker 68 - jepsen.util 8078 :info :write [22 46] :RequestCanceledException INFO [2019-12-17 12:32:43,260] jepsen worker 65 - jepsen.util 5285 :fail :cas [21 [43 44]] :ValueNotSwapFrom INFO [2019-12-17 12:32:43,261] jepsen worker 65 - jepsen.util 5285 :invoke :read [21 nil] INFO [2019-12-17 12:32:43,263] jepsen worker 68 - jepsen.util 8168 :invoke :read [22 nil] INFO [2019-12-17 12:32:43,278] jepsen worker 46 - jepsen.util 6346 :ok :read [15 29] INFO [2019-12-17 12:32:43,278] jepsen worker 46 - jepsen.util 6346 :invoke :cas [15 [23 46]] INFO [2019-12-17 12:32:43,279] jepsen worker 10 - jepsen.util 6130 :ok :read [3 43] INFO [2019-12-17 12:32:43,279] jepsen worker 10 - jepsen.util 6130 :invoke :cas [3 [18 49]] INFO [2019-12-17 12:32:43,280] jepsen worker 19 - jepsen.util 6229 :fail :cas [6 [11 47]] :ValueNotSwapFrom INFO [2019-12-17 12:32:43,280] jepsen worker 19 - jepsen.util 6229 :invoke :write [6 22] INFO [2019-12-17 12:32:43,287] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36579,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:32:43,287] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36579,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:32:43,288] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36580,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:32:43,288] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36580,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:32:43,289] jepsen worker 55 - jepsen.util 7345 :ok :read [18 3] INFO [2019-12-17 12:32:43,289] jepsen worker 55 - jepsen.util 7345 :invoke :write [18 24] INFO [2019-12-17 12:32:43,293] jepsen worker 8 - jepsen.util 8198 :info :cas [2 [24 34]] :RequestCanceledException INFO [2019-12-17 12:32:43,296] jepsen worker 8 - jepsen.util 8288 :invoke :read [2 nil] INFO [2019-12-17 12:32:43,311] jepsen worker 66 - jepsen.util 8166 :info :write [22 48] :RequestCanceledException INFO [2019-12-17 12:32:43,312] jepsen worker 54 - jepsen.util 7074 :ok :read [18 3] INFO [2019-12-17 12:32:43,313] jepsen worker 54 - jepsen.util 7074 :invoke :write [18 12] INFO [2019-12-17 12:32:43,315] jepsen worker 66 - jepsen.util 8256 :invoke :cas [22 [49 40]] INFO [2019-12-17 12:32:43,327] jepsen worker 85 - jepsen.util 9445 :info :write [28 27] :RequestCanceledException INFO [2019-12-17 12:32:43,331] jepsen worker 65 - jepsen.util 5285 :ok :read [21 1] INFO [2019-12-17 12:32:43,331] jepsen worker 65 - jepsen.util 5285 :invoke :read [21 nil] INFO [2019-12-17 12:32:43,333] jepsen worker 85 - jepsen.util 9535 :invoke :cas [28 [6 49]] INFO [2019-12-17 12:32:43,341] jepsen worker 4 - jepsen.util 5764 :info :write [1 34] :RequestCanceledException INFO [2019-12-17 12:32:43,341] jepsen worker 14 - jepsen.util 6494 :info :write [4 26] :RequestCanceledException INFO [2019-12-17 12:32:43,345] jepsen worker 14 - jepsen.util 6584 :invoke :read [4 nil] INFO [2019-12-17 12:32:43,347] jepsen worker 4 - jepsen.util 5854 :invoke :write [1 9] INFO [2019-12-17 12:32:43,375] jepsen worker 63 - jepsen.util 5283 :ok :write [21 33] INFO [2019-12-17 12:32:43,375] jepsen worker 63 - jepsen.util 5283 :invoke :read [21 nil] INFO [2019-12-17 12:32:43,390] jepsen worker 60 - jepsen.util 9330 :info :cas [20 [33 11]] :RequestCanceledException INFO [2019-12-17 12:32:43,391] jepsen worker 28 - jepsen.util 4978 :ok :write [9 40] INFO [2019-12-17 12:32:43,391] jepsen worker 65 - jepsen.util 5285 :ok :read [21 1] INFO [2019-12-17 12:32:43,391] jepsen worker 28 - jepsen.util 4978 :invoke :cas [9 [3 24]] INFO [2019-12-17 12:32:43,391] jepsen worker 65 - jepsen.util 5285 :invoke :write [21 23] INFO [2019-12-17 12:32:43,395] jepsen worker 60 - jepsen.util 9420 :invoke :cas [20 [28 21]] INFO [2019-12-17 12:32:43,426] jepsen worker 63 - jepsen.util 5283 :ok :read [21 33] INFO [2019-12-17 12:32:43,426] jepsen worker 63 - jepsen.util 5283 :invoke :read [21 nil] INFO [2019-12-17 12:32:43,427] jepsen worker 48 - jepsen.util 7788 :info :cas [16 [5 12]] :RequestCanceledException INFO [2019-12-17 12:32:43,431] jepsen worker 48 - jepsen.util 7878 :invoke :write [16 13] INFO [2019-12-17 12:32:43,441] jepsen worker 18 - jepsen.util 6138 :info :write [6 21] :RequestCanceledException INFO [2019-12-17 12:32:43,444] jepsen worker 18 - jepsen.util 6228 :invoke :write [6 12] INFO [2019-12-17 12:32:43,458] jepsen worker 34 - jepsen.util 7954 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:32:43,458] jepsen worker 34 - jepsen.util 7954 :invoke :read [11 nil] INFO [2019-12-17 12:32:43,477] jepsen worker 28 - jepsen.util 4978 :fail :cas [9 [3 24]] :ValueNotSwapFrom INFO [2019-12-17 12:32:43,478] jepsen worker 28 - jepsen.util 4978 :invoke :cas [9 [10 25]] INFO [2019-12-17 12:32:43,485] jepsen worker 38 - jepsen.util 6428 :ok :read [12 16] INFO [2019-12-17 12:32:43,486] jepsen worker 38 - jepsen.util 6428 :invoke :write [12 9] INFO [2019-12-17 12:32:43,508] jepsen worker 63 - jepsen.util 5283 :ok :read [21 33] INFO [2019-12-17 12:32:43,508] jepsen worker 63 - jepsen.util 5283 :invoke :cas [21 [46 13]] INFO [2019-12-17 12:32:43,512] jepsen worker 6 - jepsen.util 9096 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:32:43,512] jepsen worker 43 - jepsen.util 7873 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:32:43,512] jepsen worker 6 - jepsen.util 9096 :invoke :write [2 25] INFO [2019-12-17 12:32:43,512] jepsen worker 43 - jepsen.util 7873 :invoke :cas [14 [14 43]] INFO [2019-12-17 12:32:43,521] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36603,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:32:43,521] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36603,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:32:43,521] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36605,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:32:43,521] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36605,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:32:43,522] jepsen worker 65 - jepsen.util 5285 :ok :write [21 23] INFO [2019-12-17 12:32:43,522] jepsen worker 65 - jepsen.util 5285 :invoke :read [21 nil] INFO [2019-12-17 12:32:43,524] jepsen worker 57 - jepsen.util 8067 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:32:43,524] jepsen worker 44 - jepsen.util 9224 :info :cas [14 [21 32]] :RequestCanceledException INFO [2019-12-17 12:32:43,524] jepsen worker 76 - jepsen.util 8806 :info :write [25 49] :RequestCanceledException INFO [2019-12-17 12:32:43,524] jepsen worker 57 - jepsen.util 8067 :invoke :write [19 9] INFO [2019-12-17 12:32:43,530] jepsen worker 44 - jepsen.util 9314 :invoke :write [14 32] INFO [2019-12-17 12:32:43,530] jepsen worker 76 - jepsen.util 8896 :invoke :write [25 13] INFO [2019-12-17 12:32:43,532] jepsen worker 30 - jepsen.util 6240 :ok :write [10 15] INFO [2019-12-17 12:32:43,532] jepsen worker 30 - jepsen.util 6240 :invoke :write [10 37] INFO [2019-12-17 12:32:43,542] jepsen worker 75 - jepsen.util 8085 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:32:43,543] jepsen worker 75 - jepsen.util 8085 :invoke :cas [25 [40 25]] INFO [2019-12-17 12:32:43,558] jepsen worker 16 - jepsen.util 9556 :info :cas [5 [22 23]] :RequestCanceledException INFO [2019-12-17 12:32:43,558] jepsen worker 82 - jepsen.util 8992 :info :cas [27 [20 49]] :RequestCanceledException INFO [2019-12-17 12:32:43,566] jepsen worker 16 - jepsen.util 9646 :invoke :read [5 nil] INFO [2019-12-17 12:32:43,567] jepsen worker 82 - jepsen.util 9082 :invoke :write [27 24] INFO [2019-12-17 12:32:43,587] jepsen worker 63 - jepsen.util 5283 :fail :cas [21 [46 13]] :ValueNotSwapFrom INFO [2019-12-17 12:32:43,588] jepsen worker 63 - jepsen.util 5283 :invoke :write [21 15] INFO [2019-12-17 12:32:43,593] jepsen worker 41 - jepsen.util 9401 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:32:43,593] jepsen worker 41 - jepsen.util 9401 :invoke :read [13 nil] INFO [2019-12-17 12:32:43,609] jepsen worker 65 - jepsen.util 5285 :ok :read [21 23] INFO [2019-12-17 12:32:43,609] jepsen worker 65 - jepsen.util 5285 :invoke :cas [21 [14 49]] INFO [2019-12-17 12:32:43,611] jepsen worker 45 - jepsen.util 6255 :fail :read [15 nil] :RequestTimeoutException INFO [2019-12-17 12:32:43,611] jepsen worker 45 - jepsen.util 6255 :invoke :write [15 8] INFO [2019-12-17 12:32:43,612] jepsen worker 10 - jepsen.util 6130 :fail :cas [3 [18 49]] :ValueNotSwapFrom INFO [2019-12-17 12:32:43,612] jepsen worker 10 - jepsen.util 6130 :invoke :read [3 nil] INFO [2019-12-17 12:32:43,627] jepsen worker 0 - jepsen.util 8910 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:32:43,627] jepsen worker 0 - jepsen.util 8910 :invoke :read [0 nil] INFO [2019-12-17 12:32:43,628] jepsen worker 79 - jepsen.util 6379 :info :write [26 34] :RequestCanceledException INFO [2019-12-17 12:32:43,634] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:32:43,634] jepsen worker 79 - jepsen.util 6469 :invoke :cas [26 [37 33]] INFO [2019-12-17 12:32:43,634] jepsen nemesis - couchbase.util Rebalance check count: 277 INFO [2019-12-17 12:32:43,635] jepsen nemesis - couchbase.util Rebalance stuck count: 258 INFO [2019-12-17 12:32:43,641] jepsen worker 58 - jepsen.util 9148 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:32:43,641] jepsen worker 33 - jepsen.util 8403 :info :cas [11 [9 45]] :RequestCanceledException INFO [2019-12-17 12:32:43,641] jepsen worker 58 - jepsen.util 9148 :invoke :cas [19 [11 36]] INFO [2019-12-17 12:32:43,648] jepsen worker 33 - jepsen.util 8493 :invoke :read [11 nil] INFO [2019-12-17 12:32:43,672] jepsen worker 65 - jepsen.util 5285 :fail :cas [21 [14 49]] :ValueNotSwapFrom INFO [2019-12-17 12:32:43,673] jepsen worker 65 - jepsen.util 5285 :invoke :read [21 nil] INFO [2019-12-17 12:32:43,689] jepsen worker 87 - jepsen.util 5757 :ok :write [29 10] INFO [2019-12-17 12:32:43,689] jepsen worker 87 - jepsen.util 5757 :invoke :read [29 nil] INFO [2019-12-17 12:32:43,712] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36635,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:32:43,712] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36635,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:32:43,712] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36636,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:32:43,712] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36636,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:32:43,727] jepsen worker 62 - jepsen.util 8162 :info :cas [20 [14 8]] :RequestCanceledException INFO [2019-12-17 12:32:43,732] jepsen worker 62 - jepsen.util 8252 :invoke :cas [20 [21 49]] INFO [2019-12-17 12:32:43,736] jepsen worker 56 - jepsen.util 6806 :ok :write [18 29] INFO [2019-12-17 12:32:43,736] jepsen worker 56 - jepsen.util 6806 :invoke :cas [18 [24 40]] INFO [2019-12-17 12:32:43,741] jepsen worker 5 - jepsen.util 5855 :ok :read [1 40] INFO [2019-12-17 12:32:43,741] jepsen worker 5 - jepsen.util 5855 :invoke :cas [1 [45 15]] INFO [2019-12-17 12:32:43,745] jepsen worker 65 - jepsen.util 5285 :ok :read [21 23] INFO [2019-12-17 12:32:43,745] jepsen worker 65 - jepsen.util 5285 :invoke :cas [21 [39 45]] INFO [2019-12-17 12:32:43,758] jepsen worker 86 - jepsen.util 8996 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:32:43,758] jepsen worker 86 - jepsen.util 8996 :invoke :read [28 nil] INFO [2019-12-17 12:32:43,761] jepsen worker 63 - jepsen.util 5283 :ok :write [21 15] INFO [2019-12-17 12:32:43,761] jepsen worker 63 - jepsen.util 5283 :invoke :cas [21 [14 41]] INFO [2019-12-17 12:32:43,792] jepsen worker 21 - jepsen.util 8121 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:32:43,792] jepsen worker 39 - jepsen.util 8139 :info :cas [13 [27 8]] :RequestCanceledException INFO [2019-12-17 12:32:43,792] jepsen worker 21 - jepsen.util 8121 :invoke :write [7 24] INFO [2019-12-17 12:32:43,796] jepsen worker 39 - jepsen.util 8229 :invoke :cas [13 [16 43]] INFO [2019-12-17 12:32:43,811] jepsen worker 77 - jepsen.util 8987 :info :write [25 33] :RequestCanceledException INFO [2019-12-17 12:32:43,812] jepsen worker 3 - jepsen.util 6303 :ok :write [1 8] INFO [2019-12-17 12:32:43,812] jepsen worker 3 - jepsen.util 6303 :invoke :write [1 43] INFO [2019-12-17 12:32:43,814] jepsen worker 71 - jepsen.util 7001 :ok :read [23 37] INFO [2019-12-17 12:32:43,814] jepsen worker 71 - jepsen.util 7001 :invoke :write [23 42] INFO [2019-12-17 12:32:43,814] jepsen worker 77 - jepsen.util 9077 :invoke :read [25 nil] INFO [2019-12-17 12:32:43,816] jepsen worker 65 - jepsen.util 5285 :fail :cas [21 [39 45]] :ValueNotSwapFrom INFO [2019-12-17 12:32:43,816] jepsen worker 65 - jepsen.util 5285 :invoke :write [21 42] INFO [2019-12-17 12:32:43,827] jepsen worker 81 - jepsen.util 8181 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:32:43,827] jepsen worker 1 - jepsen.util 8371 :info :cas [0 [15 18]] :RequestCanceledException INFO [2019-12-17 12:32:43,828] jepsen worker 81 - jepsen.util 8181 :invoke :write [27 37] INFO [2019-12-17 12:32:43,828] jepsen worker 30 - jepsen.util 6240 :ok :write [10 37] INFO [2019-12-17 12:32:43,828] jepsen worker 18 - jepsen.util 6228 :ok :write [6 12] INFO [2019-12-17 12:32:43,828] jepsen worker 30 - jepsen.util 6240 :invoke :read [10 nil] INFO [2019-12-17 12:32:43,828] jepsen worker 18 - jepsen.util 6228 :invoke :write [6 47] INFO [2019-12-17 12:32:43,836] jepsen worker 1 - jepsen.util 8461 :invoke :read [0 nil] INFO [2019-12-17 12:32:43,841] jepsen worker 84 - jepsen.util 8094 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:32:43,841] jepsen worker 61 - jepsen.util 9511 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:32:43,841] jepsen worker 84 - jepsen.util 8094 :invoke :read [28 nil] INFO [2019-12-17 12:32:43,841] jepsen worker 61 - jepsen.util 9511 :invoke :read [20 nil] INFO [2019-12-17 12:32:43,854] jepsen worker 83 - jepsen.util 8993 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:32:43,854] jepsen worker 83 - jepsen.util 8993 :invoke :read [27 nil] INFO [2019-12-17 12:32:43,876] jepsen worker 63 - jepsen.util 5283 :fail :cas [21 [14 41]] :ValueNotSwapFrom INFO [2019-12-17 12:32:43,876] jepsen worker 63 - jepsen.util 5283 :invoke :cas [21 [25 30]] INFO [2019-12-17 12:32:43,887] jepsen worker 56 - jepsen.util 6806 :fail :cas [18 [24 40]] :ValueNotSwapFrom INFO [2019-12-17 12:32:43,887] jepsen worker 56 - jepsen.util 6806 :invoke :write [18 2] INFO [2019-12-17 12:32:43,894] jepsen worker 78 - jepsen.util 6648 :info :write [26 22] :RequestCanceledException INFO [2019-12-17 12:32:43,896] jepsen worker 78 - jepsen.util 6738 :invoke :read [26 nil] INFO [2019-12-17 12:32:43,902] jepsen worker 13 - jepsen.util 6133 :fail :cas [4 [8 6]] :ValueNotSwapFrom INFO [2019-12-17 12:32:43,902] jepsen worker 13 - jepsen.util 6133 :invoke :read [4 nil] INFO [2019-12-17 12:32:43,908] jepsen worker 28 - jepsen.util 4978 :fail :cas [9 [10 25]] :ValueNotSwapFrom INFO [2019-12-17 12:32:43,908] jepsen worker 28 - jepsen.util 4978 :invoke :read [9 nil] INFO [2019-12-17 12:32:43,915] jepsen worker 45 - jepsen.util 6255 :ok :write [15 8] INFO [2019-12-17 12:32:43,915] jepsen worker 45 - jepsen.util 6255 :invoke :write [15 36] INFO [2019-12-17 12:32:43,928] jepsen worker 31 - jepsen.util 5701 :info :cas [10 [48 25]] :RequestCanceledException INFO [2019-12-17 12:32:43,928] jepsen worker 51 - jepsen.util 6261 :info :write [17 37] :RequestCanceledException INFO [2019-12-17 12:32:43,930] jepsen worker 31 - jepsen.util 5791 :invoke :write [10 13] INFO [2019-12-17 12:32:43,930] jepsen worker 30 - jepsen.util 6240 :ok :read [10 37] INFO [2019-12-17 12:32:43,930] jepsen worker 30 - jepsen.util 6240 :invoke :cas [10 [3 21]] INFO [2019-12-17 12:32:43,935] jepsen worker 63 - jepsen.util 5283 :fail :cas [21 [25 30]] :ValueNotSwapFrom INFO [2019-12-17 12:32:43,935] jepsen worker 63 - jepsen.util 5283 :invoke :write [21 30] INFO [2019-12-17 12:32:43,935] jepsen worker 51 - jepsen.util 6351 :invoke :read [17 nil] INFO [2019-12-17 12:32:43,947] jepsen worker 65 - jepsen.util 5285 :ok :write [21 42] INFO [2019-12-17 12:32:43,947] jepsen worker 65 - jepsen.util 5285 :invoke :read [21 nil] INFO [2019-12-17 12:32:43,957] jepsen worker 67 - jepsen.util 8797 :info :cas [22 [2 39]] :RequestCanceledException INFO [2019-12-17 12:32:43,960] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36658,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:32:43,960] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36658,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:32:43,960] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36664,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:32:43,960] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36664,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:32:43,966] jepsen worker 67 - jepsen.util 8887 :invoke :write [22 24] INFO [2019-12-17 12:32:43,976] jepsen worker 79 - jepsen.util 6469 :fail :cas [26 [37 33]] :ValueNotSwapFrom INFO [2019-12-17 12:32:43,977] jepsen worker 79 - jepsen.util 6469 :invoke :write [26 36] INFO [2019-12-17 12:32:44,006] jepsen worker 25 - jepsen.util 9025 :info :cas [8 [42 30]] :RequestCanceledException INFO [2019-12-17 12:32:44,008] jepsen worker 25 - jepsen.util 9115 :invoke :read [8 nil] INFO [2019-12-17 12:32:44,021] jepsen worker 18 - jepsen.util 6228 :ok :write [6 47] INFO [2019-12-17 12:32:44,022] jepsen worker 18 - jepsen.util 6228 :invoke :write [6 41] INFO [2019-12-17 12:32:44,027] jepsen worker 15 - jepsen.util 7485 :info :write [5 41] :RequestCanceledException INFO [2019-12-17 12:32:44,027] jepsen worker 7 - jepsen.util 8737 :info :write [2 38] :RequestCanceledException INFO [2019-12-17 12:32:44,027] jepsen worker 22 - jepsen.util 8482 :info :write [7 29] :RequestCanceledException INFO [2019-12-17 12:32:44,027] jepsen worker 17 - jepsen.util 8567 :info :cas [5 [16 9]] :RequestCanceledException INFO [2019-12-17 12:32:44,032] jepsen worker 7 - jepsen.util 8827 :invoke :write [2 43] INFO [2019-12-17 12:32:44,032] jepsen worker 17 - jepsen.util 8657 :invoke :cas [5 [24 34]] INFO [2019-12-17 12:32:44,034] jepsen worker 15 - jepsen.util 7575 :invoke :cas [5 [30 44]] INFO [2019-12-17 12:32:44,034] jepsen worker 22 - jepsen.util 8572 :invoke :write [7 25] INFO [2019-12-17 12:32:44,041] jepsen worker 80 - jepsen.util 6290 :info :write [26 6] :RequestCanceledException INFO [2019-12-17 12:32:44,042] jepsen worker 40 - jepsen.util 8860 :info :write [13 19] :RequestCanceledException INFO [2019-12-17 12:32:44,042] jepsen worker 65 - jepsen.util 5285 :ok :read [21 42] INFO [2019-12-17 12:32:44,042] jepsen worker 65 - jepsen.util 5285 :invoke :read [21 nil] INFO [2019-12-17 12:32:44,046] jepsen worker 80 - jepsen.util 6380 :invoke :read [26 nil] INFO [2019-12-17 12:32:44,047] jepsen worker 40 - jepsen.util 8950 :invoke :cas [13 [34 16]] INFO [2019-12-17 12:32:44,048] jepsen worker 30 - jepsen.util 6240 :fail :cas [10 [3 21]] :ValueNotSwapFrom INFO [2019-12-17 12:32:44,048] jepsen worker 30 - jepsen.util 6240 :invoke :cas [10 [43 3]] INFO [2019-12-17 12:32:44,053] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36677,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:32:44,053] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36677,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:32:44,053] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36678,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:32:44,053] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36678,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:32:44,062] jepsen worker 5 - jepsen.util 5855 :fail :cas [1 [45 15]] :ValueNotSwapFrom INFO [2019-12-17 12:32:44,062] jepsen worker 5 - jepsen.util 5855 :invoke :cas [1 [11 28]] INFO [2019-12-17 12:32:44,120] jepsen worker 28 - jepsen.util 4978 :ok :read [9 40] INFO [2019-12-17 12:32:44,120] jepsen worker 28 - jepsen.util 4978 :invoke :write [9 7] INFO [2019-12-17 12:32:44,127] jepsen worker 42 - jepsen.util 9042 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:32:44,127] jepsen worker 52 - jepsen.util 6802 :info :write [17 32] :RequestCanceledException INFO [2019-12-17 12:32:44,127] jepsen worker 42 - jepsen.util 9042 :invoke :read [14 nil] INFO [2019-12-17 12:32:44,128] jepsen worker 74 - jepsen.util 6554 :fail :read [24 nil] :RequestTimeoutException INFO [2019-12-17 12:32:44,128] jepsen worker 74 - jepsen.util 6554 :invoke :write [24 25] INFO [2019-12-17 12:32:44,129] jepsen worker 52 - jepsen.util 6892 :invoke :cas [17 [27 47]] INFO [2019-12-17 12:32:44,138] jepsen worker 65 - jepsen.util 5285 :ok :read [21 42] INFO [2019-12-17 12:32:44,138] jepsen worker 65 - jepsen.util 5285 :invoke :cas [21 [36 0]] INFO [2019-12-17 12:32:44,139] jepsen worker 12 - jepsen.util 6672 :info :write [4 19] :RequestCanceledException INFO [2019-12-17 12:32:44,140] jepsen worker 78 - jepsen.util 6738 :ok :read [26 34] INFO [2019-12-17 12:32:44,156] jepsen worker 23 - jepsen.util 8573 :info :write [7 13] :RequestCanceledException INFO [2019-12-17 12:32:44,165] jepsen worker 63 - jepsen.util 5283 :ok :write [21 30] INFO [2019-12-17 12:32:44,170] jepsen worker 78 - jepsen.util 6738 :invoke :cas [26 [45 2]] INFO [2019-12-17 12:32:44,171] jepsen worker 63 - jepsen.util 5283 :invoke :cas [21 [16 8]] INFO [2019-12-17 12:32:44,172] jepsen worker 12 - jepsen.util 6762 :invoke :write [4 19] INFO [2019-12-17 12:32:44,177] jepsen worker 23 - jepsen.util 8663 :invoke :write [7 18] INFO [2019-12-17 12:32:44,189] jepsen worker 5 - jepsen.util 5855 :fail :cas [1 [11 28]] :ValueNotSwapFrom INFO [2019-12-17 12:32:44,190] jepsen worker 5 - jepsen.util 5855 :invoke :cas [1 [28 38]] INFO [2019-12-17 12:32:44,194] jepsen worker 31 - jepsen.util 5791 :ok :write [10 13] INFO [2019-12-17 12:32:44,194] jepsen worker 26 - jepsen.util 8846 :info :cas [8 [36 27]] :RequestCanceledException INFO [2019-12-17 12:32:44,194] jepsen worker 31 - jepsen.util 5791 :invoke :read [10 nil] INFO [2019-12-17 12:32:44,201] jepsen worker 26 - jepsen.util 8936 :invoke :cas [8 [15 31]] INFO [2019-12-17 12:32:44,208] jepsen worker 30 - jepsen.util 6240 :fail :cas [10 [43 3]] :ValueNotSwapFrom INFO [2019-12-17 12:32:44,208] jepsen worker 30 - jepsen.util 6240 :invoke :write [10 24] INFO [2019-12-17 12:32:44,212] jepsen worker 59 - jepsen.util 8699 :info :cas [19 [38 16]] :RequestCanceledException INFO [2019-12-17 12:32:44,217] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36704,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:32:44,217] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36704,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:32:44,217] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36705,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:32:44,217] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36705,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:32:44,218] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36706,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:32:44,218] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36706,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:32:44,218] jepsen worker 59 - jepsen.util 8789 :invoke :read [19 nil] INFO [2019-12-17 12:32:44,225] jepsen worker 65 - jepsen.util 5285 :fail :cas [21 [36 0]] :ValueNotSwapFrom INFO [2019-12-17 12:32:44,225] jepsen worker 65 - jepsen.util 5285 :invoke :cas [21 [45 14]] INFO [2019-12-17 12:32:44,227] jepsen worker 24 - jepsen.util 7584 :info :write [8 41] :RequestCanceledException INFO [2019-12-17 12:32:44,228] jepsen worker 49 - jepsen.util 7609 :info :cas [16 [25 20]] :RequestCanceledException INFO [2019-12-17 12:32:44,231] jepsen worker 24 - jepsen.util 7674 :invoke :read [8 nil] INFO [2019-12-17 12:32:44,232] jepsen worker 49 - jepsen.util 7699 :invoke :write [16 9] INFO [2019-12-17 12:32:44,233] jepsen worker 63 - jepsen.util 5283 :fail :cas [21 [16 8]] :ValueNotSwapFrom INFO [2019-12-17 12:32:44,234] jepsen worker 63 - jepsen.util 5283 :invoke :read [21 nil] INFO [2019-12-17 12:32:44,241] jepsen worker 35 - jepsen.util 8495 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:32:44,241] jepsen worker 35 - jepsen.util 8495 :invoke :cas [11 [43 27]] INFO [2019-12-17 12:32:44,251] jepsen worker 55 - jepsen.util 7345 :ok :write [18 24] INFO [2019-12-17 12:32:44,251] jepsen worker 55 - jepsen.util 7345 :invoke :read [18 nil] INFO [2019-12-17 12:32:44,271] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36714,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:32:44,271] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36714,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:32:44,271] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36717,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:32:44,271] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36717,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:32:44,271] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36718,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:32:44,271] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36718,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:32:44,277] jepsen worker 52 - jepsen.util 6892 :fail :cas [17 [27 47]] :ValueNotSwapFrom INFO [2019-12-17 12:32:44,277] jepsen worker 52 - jepsen.util 6892 :invoke :read [17 nil] INFO [2019-12-17 12:32:44,284] jepsen worker 28 - jepsen.util 4978 :ok :write [9 7] INFO [2019-12-17 12:32:44,284] jepsen worker 28 - jepsen.util 4978 :invoke :read [9 nil] INFO [2019-12-17 12:32:44,312] jepsen worker 65 - jepsen.util 5285 :fail :cas [21 [45 14]] :ValueNotSwapFrom INFO [2019-12-17 12:32:44,312] jepsen worker 65 - jepsen.util 5285 :invoke :read [21 nil] INFO [2019-12-17 12:32:44,322] jepsen worker 5 - jepsen.util 5855 :fail :cas [1 [28 38]] :ValueNotSwapFrom INFO [2019-12-17 12:32:44,323] jepsen worker 5 - jepsen.util 5855 :invoke :cas [1 [16 0]] INFO [2019-12-17 12:32:44,328] jepsen worker 36 - jepsen.util 6966 :ok :cas [12 [16 19]] INFO [2019-12-17 12:32:44,328] jepsen worker 36 - jepsen.util 6966 :invoke :write [12 48] INFO [2019-12-17 12:32:44,338] jepsen worker 63 - jepsen.util 5283 :ok :read [21 30] INFO [2019-12-17 12:32:44,338] jepsen worker 63 - jepsen.util 5283 :invoke :cas [21 [38 40]] INFO [2019-12-17 12:32:44,366] jepsen worker 88 - jepsen.util 6478 :fail :cas [29 [9 24]] :ValueNotSwapFrom INFO [2019-12-17 12:32:44,366] jepsen worker 88 - jepsen.util 6478 :invoke :cas [29 [29 11]] INFO [2019-12-17 12:32:44,397] jepsen worker 52 - jepsen.util 6892 :ok :read [17 20] INFO [2019-12-17 12:32:44,397] jepsen worker 52 - jepsen.util 6892 :invoke :cas [17 [21 41]] INFO [2019-12-17 12:32:44,402] jepsen worker 79 - jepsen.util 6469 :ok :write [26 36] INFO [2019-12-17 12:32:44,402] jepsen worker 79 - jepsen.util 6469 :invoke :cas [26 [20 7]] INFO [2019-12-17 12:32:44,411] jepsen worker 53 - jepsen.util 6083 :info :write [17 35] :RequestCanceledException INFO [2019-12-17 12:32:44,416] jepsen worker 53 - jepsen.util 6173 :invoke :read [17 nil] INFO [2019-12-17 12:32:44,424] jepsen worker 5 - jepsen.util 5855 :fail :cas [1 [16 0]] :ValueNotSwapFrom INFO [2019-12-17 12:32:44,424] jepsen worker 5 - jepsen.util 5855 :invoke :read [1 nil] INFO [2019-12-17 12:32:44,427] jepsen worker 2 - jepsen.util 8282 :info :write [0 11] :RequestCanceledException INFO [2019-12-17 12:32:44,431] jepsen worker 2 - jepsen.util 8372 :invoke :cas [0 [33 48]] INFO [2019-12-17 12:32:44,434] jepsen worker 54 - jepsen.util 7074 :ok :write [18 12] INFO [2019-12-17 12:32:44,435] jepsen worker 54 - jepsen.util 7074 :invoke :write [18 19] INFO [2019-12-17 12:32:44,440] jepsen worker 63 - jepsen.util 5283 :fail :cas [21 [38 40]] :ValueNotSwapFrom INFO [2019-12-17 12:32:44,440] jepsen worker 63 - jepsen.util 5283 :invoke :cas [21 [3 26]] INFO [2019-12-17 12:32:44,445] jepsen worker 65 - jepsen.util 5285 :ok :read [21 30] INFO [2019-12-17 12:32:44,445] jepsen worker 65 - jepsen.util 5285 :invoke :write [21 2] INFO [2019-12-17 12:32:44,463] jepsen worker 88 - jepsen.util 6478 :fail :cas [29 [29 11]] :ValueNotSwapFrom INFO [2019-12-17 12:32:44,464] jepsen worker 88 - jepsen.util 6478 :invoke :read [29 nil] INFO [2019-12-17 12:32:44,499] jepsen worker 63 - jepsen.util 5283 :fail :cas [21 [3 26]] :ValueNotSwapFrom INFO [2019-12-17 12:32:44,499] jepsen worker 63 - jepsen.util 5283 :invoke :write [21 43] INFO [2019-12-17 12:32:44,500] jepsen worker 4 - jepsen.util 5854 :ok :write [1 9] INFO [2019-12-17 12:32:44,500] jepsen worker 4 - jepsen.util 5854 :invoke :cas [1 [34 2]] INFO [2019-12-17 12:32:44,509] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36739,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:32:44,511] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36739,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:32:44,512] jepsen worker 73 - jepsen.util 5923 :info :write [24 6] :RequestCanceledException INFO [2019-12-17 12:32:44,512] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36743,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:32:44,514] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36743,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:32:44,514] jepsen worker 65 - jepsen.util 5285 :ok :write [21 2] INFO [2019-12-17 12:32:44,515] jepsen worker 65 - jepsen.util 5285 :invoke :write [21 5] INFO [2019-12-17 12:32:44,520] jepsen worker 73 - jepsen.util 6013 :invoke :cas [24 [5 33]] INFO [2019-12-17 12:32:44,541] jepsen worker 50 - jepsen.util 9050 :info :cas [16 [40 24]] :RequestCanceledException INFO [2019-12-17 12:32:44,546] jepsen worker 50 - jepsen.util 9140 :invoke :write [16 36] INFO [2019-12-17 12:32:44,579] jepsen worker 73 - jepsen.util 6013 :fail :cas [24 [5 33]] :ValueNotSwapFrom INFO [2019-12-17 12:32:44,580] jepsen worker 73 - jepsen.util 6013 :invoke :write [24 36] INFO [2019-12-17 12:32:44,585] jepsen worker 55 - jepsen.util 7345 :ok :read [18 12] INFO [2019-12-17 12:32:44,585] jepsen worker 55 - jepsen.util 7345 :invoke :cas [18 [12 12]] INFO [2019-12-17 12:32:44,594] jepsen worker 29 - jepsen.util 4799 :info :write [9 40] :RequestCanceledException INFO [2019-12-17 12:32:44,599] jepsen worker 29 - jepsen.util 4889 :invoke :cas [9 [23 47]] INFO [2019-12-17 12:32:44,601] jepsen worker 71 - jepsen.util 7001 :ok :write [23 42] INFO [2019-12-17 12:32:44,601] jepsen worker 71 - jepsen.util 7001 :invoke :write [23 44] INFO [2019-12-17 12:32:44,612] jepsen worker 47 - jepsen.util 6887 :info :cas [15 [3 39]] :RequestCanceledException INFO [2019-12-17 12:32:44,621] jepsen worker 52 - jepsen.util 6892 :fail :cas [17 [21 41]] :ValueNotSwapFrom INFO [2019-12-17 12:32:44,622] jepsen worker 52 - jepsen.util 6892 :invoke :read [17 nil] INFO [2019-12-17 12:32:44,624] jepsen worker 47 - jepsen.util 6977 :invoke :write [15 47] INFO [2019-12-17 12:32:44,625] jepsen worker 32 - jepsen.util 6512 :info :cas [10 [49 33]] :RequestCanceledException INFO [2019-12-17 12:32:44,628] jepsen worker 32 - jepsen.util 6602 :invoke :write [10 26] INFO [2019-12-17 12:32:44,635] jepsen worker 5 - jepsen.util 5855 :ok :read [1 9] INFO [2019-12-17 12:32:44,635] jepsen worker 5 - jepsen.util 5855 :invoke :write [1 30] INFO [2019-12-17 12:32:44,636] jepsen worker 65 - jepsen.util 5285 :ok :write [21 5] INFO [2019-12-17 12:32:44,636] jepsen worker 65 - jepsen.util 5285 :invoke :read [21 nil] INFO [2019-12-17 12:32:44,653] jepsen worker 74 - jepsen.util 6554 :ok :write [24 25] INFO [2019-12-17 12:32:44,653] jepsen worker 74 - jepsen.util 6554 :invoke :read [24 nil] INFO [2019-12-17 12:32:44,658] jepsen worker 70 - jepsen.util 6820 :info :write [23 27] :RequestCanceledException INFO [2019-12-17 12:32:44,661] jepsen worker 70 - jepsen.util 6910 :invoke :read [23 nil] INFO [2019-12-17 12:32:44,661] jepsen worker 88 - jepsen.util 6478 :ok :read [29 10] INFO [2019-12-17 12:32:44,661] jepsen worker 88 - jepsen.util 6478 :invoke :write [29 41] INFO [2019-12-17 12:32:44,666] jepsen worker 30 - jepsen.util 6240 :ok :write [10 24] INFO [2019-12-17 12:32:44,666] jepsen worker 30 - jepsen.util 6240 :invoke :cas [10 [14 7]] INFO [2019-12-17 12:32:44,672] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:32:44,672] jepsen nemesis - couchbase.util Rebalance check count: 278 INFO [2019-12-17 12:32:44,672] jepsen nemesis - couchbase.util Rebalance stuck count: 259 INFO [2019-12-17 12:32:44,691] jepsen worker 45 - jepsen.util 6255 :ok :write [15 36] INFO [2019-12-17 12:32:44,692] jepsen worker 45 - jepsen.util 6255 :invoke :read [15 nil] INFO [2019-12-17 12:32:44,694] jepsen worker 29 - jepsen.util 4889 :fail :cas [9 [23 47]] :ValueNotSwapFrom INFO [2019-12-17 12:32:44,695] jepsen worker 29 - jepsen.util 4889 :invoke :write [9 49] INFO [2019-12-17 12:32:44,701] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36762,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:32:44,701] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36762,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:32:44,701] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36766,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:32:44,701] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36766,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:32:44,707] jepsen worker 14 - jepsen.util 6584 :ok :read [4 9] INFO [2019-12-17 12:32:44,707] jepsen worker 14 - jepsen.util 6584 :invoke :cas [4 [21 25]] INFO [2019-12-17 12:32:44,716] jepsen worker 63 - jepsen.util 5283 :ok :write [21 43] INFO [2019-12-17 12:32:44,716] jepsen worker 63 - jepsen.util 5283 :invoke :cas [21 [3 3]] INFO [2019-12-17 12:32:44,725] jepsen worker 74 - jepsen.util 6554 :ok :read [24 25] INFO [2019-12-17 12:32:44,725] jepsen worker 74 - jepsen.util 6554 :invoke :cas [24 [3 13]] INFO [2019-12-17 12:32:44,728] jepsen worker 64 - jepsen.util 4924 :info :write [21 14] :RequestCanceledException INFO [2019-12-17 12:32:44,732] jepsen worker 64 - jepsen.util 5014 :invoke :read [21 nil] INFO [2019-12-17 12:32:44,772] jepsen worker 89 - jepsen.util 6119 :ok :write [29 37] INFO [2019-12-17 12:32:44,772] jepsen worker 89 - jepsen.util 6119 :invoke :read [29 nil] INFO [2019-12-17 12:32:44,774] jepsen worker 52 - jepsen.util 6892 :ok :read [17 35] INFO [2019-12-17 12:32:44,775] jepsen worker 52 - jepsen.util 6892 :invoke :write [17 19] INFO [2019-12-17 12:32:44,787] jepsen worker 73 - jepsen.util 6013 :ok :write [24 36] INFO [2019-12-17 12:32:44,787] jepsen worker 73 - jepsen.util 6013 :invoke :write [24 33] INFO [2019-12-17 12:32:44,795] jepsen worker 45 - jepsen.util 6255 :ok :read [15 36] INFO [2019-12-17 12:32:44,795] jepsen worker 45 - jepsen.util 6255 :invoke :cas [15 [20 30]] INFO [2019-12-17 12:32:44,809] jepsen worker 20 - jepsen.util 6140 :fail :read [6 nil] :RequestTimeoutException INFO [2019-12-17 12:32:44,809] jepsen worker 20 - jepsen.util 6140 :invoke :write [6 8] INFO [2019-12-17 12:32:44,821] jepsen worker 74 - jepsen.util 6554 :fail :cas [24 [3 13]] :ValueNotSwapFrom INFO [2019-12-17 12:32:44,821] jepsen worker 5 - jepsen.util 5855 :ok :write [1 30] INFO [2019-12-17 12:32:44,821] jepsen worker 74 - jepsen.util 6554 :invoke :write [24 25] INFO [2019-12-17 12:32:44,821] jepsen worker 5 - jepsen.util 5855 :invoke :write [1 1] INFO [2019-12-17 12:32:44,826] jepsen worker 69 - jepsen.util 6369 :info :write [23 14] :RequestCanceledException INFO [2019-12-17 12:32:44,827] jepsen worker 11 - jepsen.util 6581 :info :write [3 9] :RequestCanceledException INFO [2019-12-17 12:32:44,830] jepsen worker 11 - jepsen.util 6671 :invoke :read [3 nil] INFO [2019-12-17 12:32:44,832] jepsen worker 69 - jepsen.util 6459 :invoke :cas [23 [40 10]] INFO [2019-12-17 12:32:44,866] jepsen worker 45 - jepsen.util 6255 :fail :cas [15 [20 30]] :ValueNotSwapFrom INFO [2019-12-17 12:32:44,866] jepsen worker 45 - jepsen.util 6255 :invoke :cas [15 [1 5]] INFO [2019-12-17 12:32:44,894] jepsen worker 64 - jepsen.util 5014 :ok :read [21 43] INFO [2019-12-17 12:32:44,894] jepsen worker 64 - jepsen.util 5014 :invoke :read [21 nil] INFO [2019-12-17 12:32:44,896] jepsen worker 29 - jepsen.util 4889 :ok :write [9 49] INFO [2019-12-17 12:32:44,896] jepsen worker 29 - jepsen.util 4889 :invoke :write [9 31] INFO [2019-12-17 12:32:44,904] jepsen worker 14 - jepsen.util 6584 :fail :cas [4 [21 25]] :ValueNotSwapFrom INFO [2019-12-17 12:32:44,904] jepsen worker 14 - jepsen.util 6584 :invoke :read [4 nil] INFO [2019-12-17 12:32:44,934] jepsen worker 13 - jepsen.util 6133 :ok :read [4 9] INFO [2019-12-17 12:32:44,934] jepsen worker 13 - jepsen.util 6133 :invoke :write [4 32] INFO [2019-12-17 12:32:44,943] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36789,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:32:44,943] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36789,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:32:44,943] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36792,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:32:44,943] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36792,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:32:44,943] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36793,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:32:44,943] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36793,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:32:44,976] jepsen worker 80 - jepsen.util 6380 :ok :read [26 36] INFO [2019-12-17 12:32:44,976] jepsen worker 80 - jepsen.util 6380 :invoke :cas [26 [48 22]] INFO [2019-12-17 12:32:45,015] jepsen worker 29 - jepsen.util 4889 :ok :write [9 31] INFO [2019-12-17 12:32:45,016] jepsen worker 29 - jepsen.util 4889 :invoke :cas [9 [22 6]] INFO [2019-12-17 12:32:45,026] jepsen worker 78 - jepsen.util 6738 :fail :cas [26 [45 2]] :ValueNotSwapFrom INFO [2019-12-17 12:32:45,026] jepsen worker 78 - jepsen.util 6738 :invoke :read [26 nil] INFO [2019-12-17 12:32:45,027] jepsen worker 72 - jepsen.util 6192 :fail :read [24 nil] :RequestTimeoutException INFO [2019-12-17 12:32:45,027] jepsen worker 72 - jepsen.util 6192 :invoke :write [24 29] INFO [2019-12-17 12:32:45,044] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36800,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:32:45,045] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36800,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:32:45,045] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36805,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:32:45,045] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36805,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:32:45,052] jepsen worker 3 - jepsen.util 6303 :ok :write [1 43] INFO [2019-12-17 12:32:45,052] jepsen worker 3 - jepsen.util 6303 :invoke :read [1 nil] INFO [2019-12-17 12:32:45,058] jepsen worker 9 - jepsen.util 6849 :info :cas [3 [40 22]] :RequestCanceledException INFO [2019-12-17 12:32:45,060] jepsen worker 9 - jepsen.util 6939 :invoke :cas [3 [49 33]] INFO [2019-12-17 12:32:45,076] jepsen worker 29 - jepsen.util 4889 :fail :cas [9 [22 6]] :ValueNotSwapFrom INFO [2019-12-17 12:32:45,076] jepsen worker 29 - jepsen.util 4889 :invoke :cas [9 [49 49]] INFO [2019-12-17 12:32:45,087] jepsen worker 4 - jepsen.util 5854 :fail :cas [1 [34 2]] :ValueNotSwapFrom INFO [2019-12-17 12:32:45,087] jepsen worker 4 - jepsen.util 5854 :invoke :write [1 20] INFO [2019-12-17 12:32:45,111] jepsen worker 37 - jepsen.util 6157 :info :write [12 46] :RequestCanceledException INFO [2019-12-17 12:32:45,113] jepsen worker 37 - jepsen.util 6247 :invoke :cas [12 [2 38]] INFO [2019-12-17 12:32:45,124] jepsen worker 69 - jepsen.util 6459 :fail :cas [23 [40 10]] :ValueNotSwapFrom INFO [2019-12-17 12:32:45,124] jepsen worker 69 - jepsen.util 6459 :invoke :cas [23 [7 30]] INFO [2019-12-17 12:32:45,147] jepsen worker 29 - jepsen.util 4889 :fail :cas [9 [49 49]] :ValueNotSwapFrom INFO [2019-12-17 12:32:45,147] jepsen worker 29 - jepsen.util 4889 :invoke :write [9 42] INFO [2019-12-17 12:32:45,149] jepsen worker 51 - jepsen.util 6351 :ok :read [17 35] INFO [2019-12-17 12:32:45,149] jepsen worker 51 - jepsen.util 6351 :invoke :read [17 nil] INFO [2019-12-17 12:32:45,179] jepsen worker 54 - jepsen.util 7074 :ok :write [18 19] INFO [2019-12-17 12:32:45,180] jepsen worker 54 - jepsen.util 7074 :invoke :read [18 nil] INFO [2019-12-17 12:32:45,204] jepsen worker 88 - jepsen.util 6478 :ok :write [29 41] INFO [2019-12-17 12:32:45,204] jepsen worker 88 - jepsen.util 6478 :invoke :read [29 nil] INFO [2019-12-17 12:32:45,207] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36820,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:32:45,207] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36820,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:32:45,207] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36821,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:32:45,207] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36821,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:32:45,222] jepsen worker 70 - jepsen.util 6910 :ok :read [23 14] INFO [2019-12-17 12:32:45,223] jepsen worker 70 - jepsen.util 6910 :invoke :cas [23 [19 15]] INFO [2019-12-17 12:32:45,223] jepsen worker 12 - jepsen.util 6762 :ok :write [4 19] INFO [2019-12-17 12:32:45,223] jepsen worker 12 - jepsen.util 6762 :invoke :read [4 nil] INFO [2019-12-17 12:32:45,231] jepsen worker 29 - jepsen.util 4889 :ok :write [9 42] INFO [2019-12-17 12:32:45,232] jepsen worker 29 - jepsen.util 4889 :invoke :read [9 nil] INFO [2019-12-17 12:32:45,254] jepsen worker 20 - jepsen.util 6140 :ok :write [6 8] INFO [2019-12-17 12:32:45,254] jepsen worker 20 - jepsen.util 6140 :invoke :write [6 9] INFO [2019-12-17 12:32:45,254] jepsen worker 47 - jepsen.util 6977 :ok :write [15 47] INFO [2019-12-17 12:32:45,254] jepsen worker 47 - jepsen.util 6977 :invoke :read [15 nil] INFO [2019-12-17 12:32:45,259] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36827,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:32:45,259] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36827,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:32:45,259] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36830,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:32:45,259] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36830,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:32:45,264] jepsen worker 14 - jepsen.util 6584 :ok :read [4 19] INFO [2019-12-17 12:32:45,264] jepsen worker 14 - jepsen.util 6584 :invoke :read [4 nil] INFO [2019-12-17 12:32:45,279] jepsen worker 87 - jepsen.util 5757 :ok :read [29 41] INFO [2019-12-17 12:32:45,279] jepsen worker 87 - jepsen.util 5757 :invoke :read [29 nil] INFO [2019-12-17 12:32:45,282] jepsen worker 32 - jepsen.util 6602 :ok :write [10 26] INFO [2019-12-17 12:32:45,282] jepsen worker 32 - jepsen.util 6602 :invoke :read [10 nil] INFO [2019-12-17 12:32:45,283] jepsen worker 29 - jepsen.util 4889 :ok :read [9 42] INFO [2019-12-17 12:32:45,283] jepsen worker 29 - jepsen.util 4889 :invoke :read [9 nil] INFO [2019-12-17 12:32:45,313] jepsen worker 9 - jepsen.util 6939 :fail :cas [3 [49 33]] :ValueNotSwapFrom INFO [2019-12-17 12:32:45,313] jepsen worker 9 - jepsen.util 6939 :invoke :write [3 37] INFO [2019-12-17 12:32:45,320] jepsen worker 11 - jepsen.util 6671 :ok :read [3 43] INFO [2019-12-17 12:32:45,320] jepsen worker 11 - jepsen.util 6671 :invoke :write [3 8] INFO [2019-12-17 12:32:45,327] jepsen worker 66 - jepsen.util 8256 :info :cas [22 [49 40]] :RequestCanceledException INFO [2019-12-17 12:32:45,327] jepsen worker 27 - jepsen.util 6327 :info :write [9 29] :RequestCanceledException INFO [2019-12-17 12:32:45,330] jepsen worker 66 - jepsen.util 8346 :invoke :write [22 27] INFO [2019-12-17 12:32:45,334] jepsen worker 73 - jepsen.util 6013 :ok :write [24 33] INFO [2019-12-17 12:32:45,334] jepsen worker 73 - jepsen.util 6013 :invoke :cas [24 [17 25]] INFO [2019-12-17 12:32:45,335] jepsen worker 27 - jepsen.util 6417 :invoke :cas [9 [17 29]] INFO [2019-12-17 12:32:45,341] jepsen worker 19 - jepsen.util 6229 :info :write [6 22] :RequestCanceledException INFO [2019-12-17 12:32:45,341] jepsen worker 68 - jepsen.util 8168 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:32:45,341] jepsen worker 68 - jepsen.util 8168 :invoke :write [22 15] INFO [2019-12-17 12:32:45,348] jepsen worker 19 - jepsen.util 6319 :invoke :cas [6 [46 45]] INFO [2019-12-17 12:32:45,350] jepsen worker 79 - jepsen.util 6469 :fail :cas [26 [20 7]] :ValueNotSwapFrom INFO [2019-12-17 12:32:45,350] jepsen worker 79 - jepsen.util 6469 :invoke :write [26 21] INFO [2019-12-17 12:32:45,355] jepsen worker 46 - jepsen.util 6346 :info :cas [15 [23 46]] :RequestCanceledException INFO [2019-12-17 12:32:45,355] jepsen worker 8 - jepsen.util 8288 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:32:45,356] jepsen worker 8 - jepsen.util 8288 :invoke :cas [2 [19 28]] INFO [2019-12-17 12:32:45,361] jepsen worker 47 - jepsen.util 6977 :ok :read [15 47] INFO [2019-12-17 12:32:45,361] jepsen worker 47 - jepsen.util 6977 :invoke :read [15 nil] INFO [2019-12-17 12:32:45,368] jepsen worker 46 - jepsen.util 6436 :invoke :write [15 24] INFO [2019-12-17 12:32:45,393] jepsen worker 85 - jepsen.util 9535 :info :cas [28 [6 49]] :RequestCanceledException INFO [2019-12-17 12:32:45,399] jepsen worker 85 - jepsen.util 9625 :invoke :write [28 0] INFO [2019-12-17 12:32:45,413] jepsen worker 89 - jepsen.util 6119 :ok :read [29 41] INFO [2019-12-17 12:32:45,414] jepsen worker 89 - jepsen.util 6119 :invoke :cas [29 [41 47]] INFO [2019-12-17 12:32:45,437] jepsen worker 29 - jepsen.util 4889 :ok :read [9 42] INFO [2019-12-17 12:32:45,437] jepsen worker 29 - jepsen.util 4889 :invoke :read [9 nil] INFO [2019-12-17 12:32:45,441] jepsen worker 60 - jepsen.util 9420 :info :cas [20 [28 21]] :RequestCanceledException INFO [2019-12-17 12:32:45,443] jepsen worker 73 - jepsen.util 6013 :fail :cas [24 [17 25]] :ValueNotSwapFrom INFO [2019-12-17 12:32:45,443] jepsen worker 73 - jepsen.util 6013 :invoke :write [24 18] INFO [2019-12-17 12:32:45,443] jepsen worker 32 - jepsen.util 6602 :ok :read [10 26] INFO [2019-12-17 12:32:45,443] jepsen worker 32 - jepsen.util 6602 :invoke :cas [10 [15 48]] INFO [2019-12-17 12:32:45,445] jepsen worker 60 - jepsen.util 9510 :invoke :write [20 36] INFO [2019-12-17 12:32:45,446] jepsen worker 87 - jepsen.util 5757 :ok :read [29 41] INFO [2019-12-17 12:32:45,446] jepsen worker 87 - jepsen.util 5757 :invoke :cas [29 [3 29]] INFO [2019-12-17 12:32:45,458] jepsen worker 48 - jepsen.util 7878 :info :write [16 13] :RequestCanceledException INFO [2019-12-17 12:32:45,460] jepsen worker 27 - jepsen.util 6417 :fail :cas [9 [17 29]] :ValueNotSwapFrom INFO [2019-12-17 12:32:45,462] jepsen worker 27 - jepsen.util 6417 :invoke :write [9 4] INFO [2019-12-17 12:32:45,463] jepsen worker 48 - jepsen.util 7968 :invoke :write [16 32] INFO [2019-12-17 12:32:45,494] jepsen worker 3 - jepsen.util 6303 :ok :read [1 43] INFO [2019-12-17 12:32:45,494] jepsen worker 3 - jepsen.util 6303 :invoke :cas [1 [39 22]] INFO [2019-12-17 12:32:45,497] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36854,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:32:45,497] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36854,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:32:45,497] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36857,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:32:45,497] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36857,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:32:45,525] jepsen worker 32 - jepsen.util 6602 :fail :cas [10 [15 48]] :ValueNotSwapFrom INFO [2019-12-17 12:32:45,525] jepsen worker 32 - jepsen.util 6602 :invoke :read [10 nil] INFO [2019-12-17 12:32:45,551] jepsen worker 53 - jepsen.util 6173 :ok :read [17 35] INFO [2019-12-17 12:32:45,551] jepsen worker 53 - jepsen.util 6173 :invoke :read [17 nil] INFO [2019-12-17 12:32:45,552] jepsen worker 69 - jepsen.util 6459 :fail :cas [23 [7 30]] :ValueNotSwapFrom INFO [2019-12-17 12:32:45,552] jepsen worker 69 - jepsen.util 6459 :invoke :cas [23 [20 20]] INFO [2019-12-17 12:32:45,557] jepsen worker 34 - jepsen.util 7954 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:32:45,558] jepsen worker 34 - jepsen.util 7954 :invoke :write [11 48] INFO [2019-12-17 12:32:45,585] jepsen worker 32 - jepsen.util 6602 :ok :read [10 26] INFO [2019-12-17 12:32:45,585] jepsen worker 32 - jepsen.util 6602 :invoke :read [10 nil] INFO [2019-12-17 12:32:45,594] jepsen worker 38 - jepsen.util 6428 :info :write [12 9] :RequestCanceledException INFO [2019-12-17 12:32:45,594] jepsen worker 44 - jepsen.util 9314 :info :write [14 32] :RequestCanceledException INFO [2019-12-17 12:32:45,596] jepsen worker 38 - jepsen.util 6518 :invoke :read [12 nil] INFO [2019-12-17 12:32:45,599] jepsen worker 44 - jepsen.util 9404 :invoke :write [14 3] INFO [2019-12-17 12:32:45,611] jepsen worker 20 - jepsen.util 6140 :ok :write [6 9] INFO [2019-12-17 12:32:45,611] jepsen worker 43 - jepsen.util 7873 :info :cas [14 [14 43]] :RequestCanceledException INFO [2019-12-17 12:32:45,611] jepsen worker 6 - jepsen.util 9096 :info :write [2 25] :RequestCanceledException INFO [2019-12-17 12:32:45,611] jepsen worker 76 - jepsen.util 8896 :info :write [25 13] :RequestCanceledException INFO [2019-12-17 12:32:45,612] jepsen worker 20 - jepsen.util 6140 :invoke :cas [6 [42 29]] INFO [2019-12-17 12:32:45,614] jepsen worker 80 - jepsen.util 6380 :fail :cas [26 [48 22]] :ValueNotSwapFrom INFO [2019-12-17 12:32:45,614] jepsen worker 80 - jepsen.util 6380 :invoke :read [26 nil] INFO [2019-12-17 12:32:45,615] jepsen worker 43 - jepsen.util 7963 :invoke :write [14 37] INFO [2019-12-17 12:32:45,616] jepsen worker 76 - jepsen.util 8986 :invoke :read [25 nil] INFO [2019-12-17 12:32:45,616] jepsen worker 6 - jepsen.util 9186 :invoke :cas [2 [35 43]] INFO [2019-12-17 12:32:45,616] jepsen worker 46 - jepsen.util 6436 :ok :write [15 24] INFO [2019-12-17 12:32:45,617] jepsen worker 46 - jepsen.util 6436 :invoke :read [15 nil] INFO [2019-12-17 12:32:45,618] jepsen worker 52 - jepsen.util 6892 :ok :write [17 19] INFO [2019-12-17 12:32:45,619] jepsen worker 52 - jepsen.util 6892 :invoke :cas [17 [41 32]] INFO [2019-12-17 12:32:45,627] jepsen worker 57 - jepsen.util 8067 :info :write [19 9] :RequestCanceledException INFO [2019-12-17 12:32:45,627] jepsen worker 16 - jepsen.util 9646 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:32:45,627] jepsen worker 16 - jepsen.util 9646 :invoke :read [5 nil] INFO [2019-12-17 12:32:45,628] jepsen worker 51 - jepsen.util 6351 :ok :read [17 35] INFO [2019-12-17 12:32:45,628] jepsen worker 51 - jepsen.util 6351 :invoke :read [17 nil] INFO [2019-12-17 12:32:45,633] jepsen worker 57 - jepsen.util 8157 :invoke :cas [19 [0 20]] INFO [2019-12-17 12:32:45,641] jepsen worker 82 - jepsen.util 9082 :info :write [27 24] :RequestCanceledException INFO [2019-12-17 12:32:45,641] jepsen worker 75 - jepsen.util 8085 :info :cas [25 [40 25]] :RequestCanceledException INFO [2019-12-17 12:32:45,641] jepsen worker 10 - jepsen.util 6130 :fail :read [3 nil] :RequestTimeoutException INFO [2019-12-17 12:32:45,642] jepsen worker 10 - jepsen.util 6130 :invoke :read [3 nil] INFO [2019-12-17 12:32:45,644] jepsen worker 75 - jepsen.util 8175 :invoke :cas [25 [25 39]] INFO [2019-12-17 12:32:45,645] jepsen worker 82 - jepsen.util 9172 :invoke :cas [27 [11 1]] INFO [2019-12-17 12:32:45,649] jepsen worker 32 - jepsen.util 6602 :ok :read [10 26] INFO [2019-12-17 12:32:45,649] jepsen worker 32 - jepsen.util 6602 :invoke :cas [10 [38 29]] INFO [2019-12-17 12:32:45,684] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36887,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:32:45,684] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36887,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:32:45,684] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36890,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:32:45,684] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36890,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:32:45,691] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:32:45,691] jepsen nemesis - couchbase.util Rebalance check count: 279 INFO [2019-12-17 12:32:45,691] jepsen nemesis - couchbase.util Rebalance stuck count: 260 INFO [2019-12-17 12:32:45,692] jepsen worker 29 - jepsen.util 4889 :ok :read [9 42] INFO [2019-12-17 12:32:45,692] jepsen worker 29 - jepsen.util 4889 :invoke :read [9 nil] INFO [2019-12-17 12:32:45,693] jepsen worker 41 - jepsen.util 9401 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:32:45,693] jepsen worker 41 - jepsen.util 9401 :invoke :read [13 nil] INFO [2019-12-17 12:32:45,696] jepsen worker 54 - jepsen.util 7074 :ok :read [18 19] INFO [2019-12-17 12:32:45,697] jepsen worker 54 - jepsen.util 7074 :invoke :write [18 19] INFO [2019-12-17 12:32:45,709] jepsen worker 14 - jepsen.util 6584 :ok :read [4 19] INFO [2019-12-17 12:32:45,709] jepsen worker 14 - jepsen.util 6584 :invoke :read [4 nil] INFO [2019-12-17 12:32:45,714] jepsen worker 19 - jepsen.util 6319 :fail :cas [6 [46 45]] :ValueNotSwapFrom INFO [2019-12-17 12:32:45,714] jepsen worker 19 - jepsen.util 6319 :invoke :read [6 nil] INFO [2019-12-17 12:32:45,727] jepsen worker 46 - jepsen.util 6436 :ok :read [15 24] INFO [2019-12-17 12:32:45,727] jepsen worker 46 - jepsen.util 6436 :invoke :read [15 nil] INFO [2019-12-17 12:32:45,727] jepsen worker 0 - jepsen.util 8910 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:32:45,727] jepsen worker 33 - jepsen.util 8493 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:32:45,727] jepsen worker 0 - jepsen.util 8910 :invoke :cas [0 [24 5]] INFO [2019-12-17 12:32:45,727] jepsen worker 33 - jepsen.util 8493 :invoke :read [11 nil] INFO [2019-12-17 12:32:45,741] jepsen worker 58 - jepsen.util 9148 :info :cas [19 [11 36]] :RequestCanceledException INFO [2019-12-17 12:32:45,749] jepsen worker 58 - jepsen.util 9238 :invoke :write [19 16] INFO [2019-12-17 12:32:45,753] jepsen worker 32 - jepsen.util 6602 :fail :cas [10 [38 29]] :ValueNotSwapFrom INFO [2019-12-17 12:32:45,753] jepsen worker 32 - jepsen.util 6602 :invoke :cas [10 [25 7]] INFO [2019-12-17 12:32:45,772] jepsen worker 29 - jepsen.util 4889 :ok :read [9 42] INFO [2019-12-17 12:32:45,772] jepsen worker 29 - jepsen.util 4889 :invoke :read [9 nil] INFO [2019-12-17 12:32:45,794] jepsen worker 62 - jepsen.util 8252 :info :cas [20 [21 49]] :RequestCanceledException INFO [2019-12-17 12:32:45,799] jepsen worker 62 - jepsen.util 8342 :invoke :cas [20 [18 3]] INFO [2019-12-17 12:32:45,812] jepsen worker 39 - jepsen.util 8229 :info :cas [13 [16 43]] :RequestCanceledException INFO [2019-12-17 12:32:45,819] jepsen worker 39 - jepsen.util 8319 :invoke :cas [13 [44 27]] INFO [2019-12-17 12:32:45,821] jepsen worker 69 - jepsen.util 6459 :fail :cas [23 [20 20]] :ValueNotSwapFrom INFO [2019-12-17 12:32:45,823] jepsen worker 69 - jepsen.util 6459 :invoke :write [23 34] INFO [2019-12-17 12:32:45,827] jepsen worker 77 - jepsen.util 9077 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:32:45,827] jepsen worker 77 - jepsen.util 9077 :invoke :read [25 nil] INFO [2019-12-17 12:32:45,831] jepsen worker 19 - jepsen.util 6319 :ok :read [6 9] INFO [2019-12-17 12:32:45,831] jepsen worker 19 - jepsen.util 6319 :invoke :read [6 nil] INFO [2019-12-17 12:32:45,854] jepsen worker 46 - jepsen.util 6436 :ok :read [15 24] INFO [2019-12-17 12:32:45,854] jepsen worker 46 - jepsen.util 6436 :invoke :read [15 nil] INFO [2019-12-17 12:32:45,858] jepsen worker 86 - jepsen.util 8996 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:32:45,859] jepsen worker 86 - jepsen.util 8996 :invoke :cas [28 [45 11]] INFO [2019-12-17 12:32:45,894] jepsen worker 21 - jepsen.util 8121 :info :write [7 24] :RequestCanceledException INFO [2019-12-17 12:32:45,898] jepsen worker 3 - jepsen.util 6303 :fail :cas [1 [39 22]] :ValueNotSwapFrom INFO [2019-12-17 12:32:45,898] jepsen worker 3 - jepsen.util 6303 :invoke :read [1 nil] INFO [2019-12-17 12:32:45,901] jepsen worker 21 - jepsen.util 8211 :invoke :write [7 30] INFO [2019-12-17 12:32:45,901] jepsen worker 70 - jepsen.util 6910 :fail :cas [23 [19 15]] :ValueNotSwapFrom INFO [2019-12-17 12:32:45,901] jepsen worker 70 - jepsen.util 6910 :invoke :cas [23 [5 36]] INFO [2019-12-17 12:32:45,903] jepsen worker 87 - jepsen.util 5757 :fail :cas [29 [3 29]] :ValueNotSwapFrom INFO [2019-12-17 12:32:45,903] jepsen worker 87 - jepsen.util 5757 :invoke :read [29 nil] INFO [2019-12-17 12:32:45,905] jepsen worker 29 - jepsen.util 4889 :ok :read [9 42] INFO [2019-12-17 12:32:45,905] jepsen worker 29 - jepsen.util 4889 :invoke :cas [9 [12 23]] INFO [2019-12-17 12:32:45,911] jepsen worker 32 - jepsen.util 6602 :fail :cas [10 [25 7]] :ValueNotSwapFrom INFO [2019-12-17 12:32:45,912] jepsen worker 32 - jepsen.util 6602 :invoke :cas [10 [48 31]] INFO [2019-12-17 12:32:45,928] jepsen worker 81 - jepsen.util 8181 :info :write [27 37] :RequestCanceledException INFO [2019-12-17 12:32:45,930] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36914,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:32:45,931] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36914,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:32:45,931] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36918,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:32:45,931] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36918,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:32:45,933] jepsen worker 81 - jepsen.util 8271 :invoke :cas [27 [34 12]] INFO [2019-12-17 12:32:45,936] jepsen worker 19 - jepsen.util 6319 :ok :read [6 9] INFO [2019-12-17 12:32:45,936] jepsen worker 19 - jepsen.util 6319 :invoke :read [6 nil] INFO [2019-12-17 12:32:45,939] jepsen worker 46 - jepsen.util 6436 :ok :read [15 24] INFO [2019-12-17 12:32:45,939] jepsen worker 46 - jepsen.util 6436 :invoke :read [15 nil] INFO [2019-12-17 12:32:45,941] jepsen worker 1 - jepsen.util 8461 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:32:45,941] jepsen worker 84 - jepsen.util 8094 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:32:45,941] jepsen worker 61 - jepsen.util 9511 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:32:45,941] jepsen worker 1 - jepsen.util 8461 :invoke :read [0 nil] INFO [2019-12-17 12:32:45,941] jepsen worker 84 - jepsen.util 8094 :invoke :read [28 nil] INFO [2019-12-17 12:32:45,941] jepsen worker 61 - jepsen.util 9511 :invoke :read [20 nil] INFO [2019-12-17 12:32:45,958] jepsen worker 83 - jepsen.util 8993 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:32:45,958] jepsen worker 83 - jepsen.util 8993 :invoke :write [27 5] INFO [2019-12-17 12:32:45,958] jepsen worker 56 - jepsen.util 6806 :info :write [18 2] :RequestCanceledException INFO [2019-12-17 12:32:45,961] jepsen worker 56 - jepsen.util 6896 :invoke :read [18 nil] INFO [2019-12-17 12:32:45,974] jepsen worker 32 - jepsen.util 6602 :fail :cas [10 [48 31]] :ValueNotSwapFrom INFO [2019-12-17 12:32:45,974] jepsen worker 32 - jepsen.util 6602 :invoke :cas [10 [27 35]] INFO [2019-12-17 12:32:45,975] jepsen worker 29 - jepsen.util 4889 :fail :cas [9 [12 23]] :ValueNotSwapFrom INFO [2019-12-17 12:32:45,975] jepsen worker 29 - jepsen.util 4889 :invoke :write [9 27] INFO [2019-12-17 12:32:46,011] jepsen worker 67 - jepsen.util 8887 :info :write [22 24] :RequestCanceledException INFO [2019-12-17 12:32:46,017] jepsen worker 67 - jepsen.util 8977 :invoke :write [22 4] INFO [2019-12-17 12:32:46,020] jepsen worker 46 - jepsen.util 6436 :ok :read [15 24] INFO [2019-12-17 12:32:46,020] jepsen worker 46 - jepsen.util 6436 :invoke :read [15 nil] INFO [2019-12-17 12:32:46,026] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36937,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:32:46,026] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36937,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:32:46,026] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36938,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:32:46,027] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36938,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:32:46,027] jepsen worker 25 - jepsen.util 9115 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:32:46,027] jepsen worker 25 - jepsen.util 9115 :invoke :write [8 28] INFO [2019-12-17 12:32:46,048] jepsen worker 19 - jepsen.util 6319 :ok :read [6 9] INFO [2019-12-17 12:32:46,049] jepsen worker 29 - jepsen.util 4889 :ok :write [9 27] INFO [2019-12-17 12:32:46,058] jepsen worker 17 - jepsen.util 8657 :info :cas [5 [24 34]] :RequestCanceledException INFO [2019-12-17 12:32:46,069] jepsen worker 32 - jepsen.util 6602 :fail :cas [10 [27 35]] :ValueNotSwapFrom INFO [2019-12-17 12:32:46,093] jepsen worker 22 - jepsen.util 8572 :info :write [7 25] :RequestCanceledException INFO [2019-12-17 12:32:46,097] jepsen worker 19 - jepsen.util 6319 :invoke :read [6 nil] INFO [2019-12-17 12:32:46,098] jepsen worker 32 - jepsen.util 6602 :invoke :read [10 nil] INFO [2019-12-17 12:32:46,098] jepsen worker 29 - jepsen.util 4889 :invoke :cas [9 [42 33]] INFO [2019-12-17 12:32:46,110] jepsen worker 17 - jepsen.util 8747 :invoke :write [5 12] INFO [2019-12-17 12:32:46,117] jepsen worker 22 - jepsen.util 8662 :invoke :cas [7 [27 5]] INFO [2019-12-17 12:32:46,127] jepsen worker 15 - jepsen.util 7575 :info :cas [5 [30 44]] :RequestCanceledException INFO [2019-12-17 12:32:46,127] jepsen worker 18 - jepsen.util 6228 :info :write [6 41] :RequestCanceledException INFO [2019-12-17 12:32:46,127] jepsen worker 40 - jepsen.util 8950 :info :cas [13 [34 16]] :RequestCanceledException INFO [2019-12-17 12:32:46,130] jepsen worker 18 - jepsen.util 6318 :invoke :read [6 nil] INFO [2019-12-17 12:32:46,138] jepsen worker 40 - jepsen.util 9040 :invoke :cas [13 [25 8]] INFO [2019-12-17 12:32:46,142] jepsen worker 7 - jepsen.util 8827 :info :write [2 43] :RequestCanceledException INFO [2019-12-17 12:32:46,142] jepsen worker 15 - jepsen.util 7665 :invoke :read [5 nil] INFO [2019-12-17 12:32:46,153] jepsen worker 7 - jepsen.util 8917 :invoke :cas [2 [19 32]] INFO [2019-12-17 12:32:46,157] jepsen worker 80 - jepsen.util 6380 :ok :read [26 36] INFO [2019-12-17 12:32:46,158] jepsen worker 80 - jepsen.util 6380 :invoke :read [26 nil] INFO [2019-12-17 12:32:46,159] jepsen worker 12 - jepsen.util 6762 :ok :read [4 19] INFO [2019-12-17 12:32:46,159] jepsen worker 12 - jepsen.util 6762 :invoke :write [4 39] INFO [2019-12-17 12:32:46,166] jepsen worker 38 - jepsen.util 6518 :ok :read [12 19] INFO [2019-12-17 12:32:46,166] jepsen worker 38 - jepsen.util 6518 :invoke :cas [12 [33 23]] INFO [2019-12-17 12:32:46,166] jepsen worker 78 - jepsen.util 6738 :ok :read [26 36] INFO [2019-12-17 12:32:46,166] jepsen worker 78 - jepsen.util 6738 :invoke :cas [26 [25 30]] INFO [2019-12-17 12:32:46,186] jepsen worker 37 - jepsen.util 6247 :fail :cas [12 [2 38]] :ValueNotSwapFrom INFO [2019-12-17 12:32:46,186] jepsen worker 37 - jepsen.util 6247 :invoke :cas [12 [5 42]] INFO [2019-12-17 12:32:46,194] jepsen worker 88 - jepsen.util 6478 :ok :read [29 41] INFO [2019-12-17 12:32:46,194] jepsen worker 88 - jepsen.util 6478 :invoke :read [29 nil] INFO [2019-12-17 12:32:46,204] jepsen worker 14 - jepsen.util 6584 :ok :read [4 19] INFO [2019-12-17 12:32:46,205] jepsen worker 14 - jepsen.util 6584 :invoke :cas [4 [15 39]] INFO [2019-12-17 12:32:46,223] jepsen worker 18 - jepsen.util 6318 :ok :read [6 9] INFO [2019-12-17 12:32:46,223] jepsen worker 18 - jepsen.util 6318 :invoke :cas [6 [49 9]] INFO [2019-12-17 12:32:46,224] jepsen worker 46 - jepsen.util 6436 :ok :read [15 24] INFO [2019-12-17 12:32:46,224] jepsen worker 46 - jepsen.util 6436 :invoke :write [15 43] INFO [2019-12-17 12:32:46,224] jepsen worker 19 - jepsen.util 6319 :ok :read [6 9] INFO [2019-12-17 12:32:46,224] jepsen worker 19 - jepsen.util 6319 :invoke :write [6 25] INFO [2019-12-17 12:32:46,227] jepsen worker 42 - jepsen.util 9042 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:32:46,227] jepsen worker 42 - jepsen.util 9042 :invoke :write [14 9] INFO [2019-12-17 12:32:46,227] jepsen worker 31 - jepsen.util 5791 :fail :read [10 nil] :RequestTimeoutException INFO [2019-12-17 12:32:46,228] jepsen worker 31 - jepsen.util 5791 :invoke :read [10 nil] INFO [2019-12-17 12:32:46,228] jepsen worker 23 - jepsen.util 8663 :info :write [7 18] :RequestCanceledException INFO [2019-12-17 12:32:46,237] jepsen worker 23 - jepsen.util 8753 :invoke :write [7 38] INFO [2019-12-17 12:32:46,238] jepsen worker 29 - jepsen.util 4889 :fail :cas [9 [42 33]] :ValueNotSwapFrom INFO [2019-12-17 12:32:46,239] jepsen worker 29 - jepsen.util 4889 :invoke :read [9 nil] INFO [2019-12-17 12:32:46,249] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36968,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:32:46,249] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36968,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:32:46,249] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36970,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:32:46,249] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36970,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:32:46,275] jepsen worker 32 - jepsen.util 6602 :ok :read [10 26] INFO [2019-12-17 12:32:46,275] jepsen worker 32 - jepsen.util 6602 :invoke :read [10 nil] INFO [2019-12-17 12:32:46,288] jepsen worker 53 - jepsen.util 6173 :ok :read [17 19] INFO [2019-12-17 12:32:46,289] jepsen worker 53 - jepsen.util 6173 :invoke :read [17 nil] INFO [2019-12-17 12:32:46,293] jepsen worker 26 - jepsen.util 8936 :info :cas [8 [15 31]] :RequestCanceledException INFO [2019-12-17 12:32:46,301] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36960,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:32:46,301] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36960,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:32:46,301] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36962,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:32:46,302] jepsen worker 26 - jepsen.util 9026 :invoke :write [8 9] INFO [2019-12-17 12:32:46,302] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36962,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:32:46,313] jepsen worker 59 - jepsen.util 8789 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:32:46,315] jepsen worker 59 - jepsen.util 8789 :invoke :write [19 30] INFO [2019-12-17 12:32:46,320] jepsen worker 51 - jepsen.util 6351 :ok :read [17 19] INFO [2019-12-17 12:32:46,320] jepsen worker 51 - jepsen.util 6351 :invoke :write [17 3] INFO [2019-12-17 12:32:46,327] jepsen worker 28 - jepsen.util 4978 :fail :read [9 nil] :RequestTimeoutException INFO [2019-12-17 12:32:46,327] jepsen worker 24 - jepsen.util 7674 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:32:46,327] jepsen worker 28 - jepsen.util 4978 :invoke :read [9 nil] INFO [2019-12-17 12:32:46,327] jepsen worker 24 - jepsen.util 7674 :invoke :write [8 35] INFO [2019-12-17 12:32:46,341] jepsen worker 35 - jepsen.util 8495 :info :cas [11 [43 27]] :RequestCanceledException INFO [2019-12-17 12:32:46,341] jepsen worker 49 - jepsen.util 7699 :info :write [16 9] :RequestCanceledException INFO [2019-12-17 12:32:46,347] jepsen worker 35 - jepsen.util 8585 :invoke :read [11 nil] INFO [2019-12-17 12:32:46,349] jepsen worker 31 - jepsen.util 5791 :ok :read [10 26] INFO [2019-12-17 12:32:46,349] jepsen worker 31 - jepsen.util 5791 :invoke :read [10 nil] INFO [2019-12-17 12:32:46,349] jepsen worker 49 - jepsen.util 7789 :invoke :cas [16 [5 43]] INFO [2019-12-17 12:32:46,357] jepsen worker 18 - jepsen.util 6318 :fail :cas [6 [49 9]] :ValueNotSwapFrom INFO [2019-12-17 12:32:46,357] jepsen worker 18 - jepsen.util 6318 :invoke :cas [6 [27 30]] INFO [2019-12-17 12:32:46,357] jepsen worker 52 - jepsen.util 6892 :fail :cas [17 [41 32]] :ValueNotSwapFrom INFO [2019-12-17 12:32:46,357] jepsen worker 52 - jepsen.util 6892 :invoke :cas [17 [8 9]] INFO [2019-12-17 12:32:46,384] jepsen worker 29 - jepsen.util 4889 :ok :read [9 27] INFO [2019-12-17 12:32:46,384] jepsen worker 29 - jepsen.util 4889 :invoke :write [9 39] INFO [2019-12-17 12:32:46,392] jepsen worker 32 - jepsen.util 6602 :ok :read [10 26] INFO [2019-12-17 12:32:46,392] jepsen worker 32 - jepsen.util 6602 :invoke :cas [10 [45 41]] INFO [2019-12-17 12:32:46,394] jepsen worker 36 - jepsen.util 6966 :info :write [12 48] :RequestCanceledException INFO [2019-12-17 12:32:46,396] jepsen worker 36 - jepsen.util 7056 :invoke :read [12 nil] INFO [2019-12-17 12:32:46,418] jepsen worker 9 - jepsen.util 6939 :ok :write [3 37] INFO [2019-12-17 12:32:46,418] jepsen worker 9 - jepsen.util 6939 :invoke :read [3 nil] INFO [2019-12-17 12:32:46,427] jepsen worker 73 - jepsen.util 6013 :ok :write [24 18] INFO [2019-12-17 12:32:46,427] jepsen worker 73 - jepsen.util 6013 :invoke :read [24 nil] INFO [2019-12-17 12:32:46,433] jepsen worker 28 - jepsen.util 4978 :ok :read [9 27] INFO [2019-12-17 12:32:46,434] jepsen worker 28 - jepsen.util 4978 :invoke :cas [9 [2 25]] INFO [2019-12-17 12:32:46,438] jepsen worker 31 - jepsen.util 5791 :ok :read [10 26] INFO [2019-12-17 12:32:46,438] jepsen worker 31 - jepsen.util 5791 :invoke :write [10 38] INFO [2019-12-17 12:32:46,480] jepsen worker 70 - jepsen.util 6910 :fail :cas [23 [5 36]] :ValueNotSwapFrom INFO [2019-12-17 12:32:46,491] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36997,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:32:46,494] jepsen worker 32 - jepsen.util 6602 :fail :cas [10 [45 41]] :ValueNotSwapFrom INFO [2019-12-17 12:32:46,503] jepsen worker 18 - jepsen.util 6318 :fail :cas [6 [27 30]] :ValueNotSwapFrom INFO [2019-12-17 12:32:46,505] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":36997,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:32:46,505] jepsen worker 70 - jepsen.util 6910 :invoke :read [23 nil] INFO [2019-12-17 12:32:46,505] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37000,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:32:46,505] jepsen worker 18 - jepsen.util 6318 :invoke :cas [6 [12 16]] INFO [2019-12-17 12:32:46,505] jepsen worker 32 - jepsen.util 6602 :invoke :cas [10 [23 21]] INFO [2019-12-17 12:32:46,507] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37000,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:32:46,507] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37001,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:32:46,507] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37001,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:32:46,527] jepsen worker 2 - jepsen.util 8372 :info :cas [0 [33 48]] :RequestCanceledException INFO [2019-12-17 12:32:46,531] jepsen worker 2 - jepsen.util 8462 :invoke :read [0 nil] INFO [2019-12-17 12:32:46,533] jepsen worker 9 - jepsen.util 6939 :ok :read [3 37] INFO [2019-12-17 12:32:46,534] jepsen worker 9 - jepsen.util 6939 :invoke :cas [3 [17 16]] INFO [2019-12-17 12:32:46,538] jepsen worker 10 - jepsen.util 6130 :ok :read [3 37] INFO [2019-12-17 12:32:46,538] jepsen worker 10 - jepsen.util 6130 :invoke :write [3 2] INFO [2019-12-17 12:32:46,574] jepsen worker 52 - jepsen.util 6892 :fail :cas [17 [8 9]] :ValueNotSwapFrom INFO [2019-12-17 12:32:46,574] jepsen worker 52 - jepsen.util 6892 :invoke :cas [17 [13 8]] INFO [2019-12-17 12:32:46,582] jepsen worker 73 - jepsen.util 6013 :ok :read [24 18] INFO [2019-12-17 12:32:46,582] jepsen worker 73 - jepsen.util 6013 :invoke :write [24 33] INFO [2019-12-17 12:32:46,582] jepsen worker 88 - jepsen.util 6478 :ok :read [29 41] INFO [2019-12-17 12:32:46,583] jepsen worker 88 - jepsen.util 6478 :invoke :write [29 24] INFO [2019-12-17 12:32:46,602] jepsen worker 50 - jepsen.util 9140 :info :write [16 36] :RequestCanceledException INFO [2019-12-17 12:32:46,609] jepsen worker 50 - jepsen.util 9230 :invoke :write [16 7] INFO [2019-12-17 12:32:46,615] jepsen worker 28 - jepsen.util 4978 :fail :cas [9 [2 25]] :ValueNotSwapFrom INFO [2019-12-17 12:32:46,616] jepsen worker 28 - jepsen.util 4978 :invoke :write [9 48] INFO [2019-12-17 12:32:46,617] jepsen worker 3 - jepsen.util 6303 :ok :read [1 43] INFO [2019-12-17 12:32:46,618] jepsen worker 3 - jepsen.util 6303 :invoke :write [1 6] INFO [2019-12-17 12:32:46,656] jepsen worker 71 - jepsen.util 7001 :info :write [23 44] :RequestCanceledException INFO [2019-12-17 12:32:46,659] jepsen worker 71 - jepsen.util 7091 :invoke :read [23 nil] INFO [2019-12-17 12:32:46,662] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37015,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:32:46,662] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37015,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:32:46,662] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37020,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:32:46,662] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37020,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:32:46,681] jepsen worker 32 - jepsen.util 6602 :fail :cas [10 [23 21]] :ValueNotSwapFrom INFO [2019-12-17 12:32:46,681] jepsen worker 32 - jepsen.util 6602 :invoke :write [10 42] INFO [2019-12-17 12:32:46,684] jepsen worker 65 - jepsen.util 5285 :fail :read [21 nil] :RequestTimeoutException INFO [2019-12-17 12:32:46,684] jepsen worker 65 - jepsen.util 5285 :invoke :cas [21 [25 43]] INFO [2019-12-17 12:32:46,711] jepsen worker 46 - jepsen.util 6436 :ok :write [15 43] INFO [2019-12-17 12:32:46,712] jepsen worker 46 - jepsen.util 6436 :invoke :read [15 nil] INFO [2019-12-17 12:32:46,714] jepsen worker 19 - jepsen.util 6319 :ok :write [6 25] INFO [2019-12-17 12:32:46,716] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:32:46,720] jepsen worker 36 - jepsen.util 7056 :ok :read [12 19] INFO [2019-12-17 12:32:46,737] jepsen worker 9 - jepsen.util 6939 :fail :cas [3 [17 16]] :ValueNotSwapFrom INFO [2019-12-17 12:32:46,741] jepsen worker 30 - jepsen.util 6240 :info :cas [10 [14 7]] :RequestCanceledException INFO [2019-12-17 12:32:46,756] jepsen worker 63 - jepsen.util 5283 :info :cas [21 [3 3]] :RequestCanceledException INFO [2019-12-17 12:32:46,787] jepsen worker 19 - jepsen.util 6319 :invoke :cas [6 [20 30]] INFO [2019-12-17 12:32:46,787] jepsen nemesis - couchbase.util Rebalance check count: 280 INFO [2019-12-17 12:32:46,787] jepsen worker 36 - jepsen.util 7056 :invoke :write [12 16] INFO [2019-12-17 12:32:46,788] jepsen worker 9 - jepsen.util 6939 :invoke :write [3 10] INFO [2019-12-17 12:32:46,788] jepsen nemesis - couchbase.util Rebalance stuck count: 261 INFO [2019-12-17 12:32:46,791] jepsen worker 78 - jepsen.util 6738 :fail :cas [26 [25 30]] :ValueNotSwapFrom INFO [2019-12-17 12:32:46,791] jepsen worker 78 - jepsen.util 6738 :invoke :write [26 20] INFO [2019-12-17 12:32:46,791] jepsen worker 63 - jepsen.util 5373 :invoke :write [21 42] INFO [2019-12-17 12:32:46,799] jepsen worker 29 - jepsen.util 4889 :ok :write [9 39] INFO [2019-12-17 12:32:46,800] jepsen worker 29 - jepsen.util 4889 :invoke :read [9 nil] INFO [2019-12-17 12:32:46,855] jepsen worker 30 - jepsen.util 6330 :invoke :read [10 nil] INFO [2019-12-17 12:32:46,897] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37033,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:32:46,898] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37033,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:32:46,912] jepsen worker 45 - jepsen.util 6255 :info :cas [15 [1 5]] :RequestCanceledException INFO [2019-12-17 12:32:46,926] jepsen worker 38 - jepsen.util 6518 :fail :cas [12 [33 23]] :ValueNotSwapFrom INFO [2019-12-17 12:32:46,927] jepsen worker 38 - jepsen.util 6518 :invoke :read [12 nil] INFO [2019-12-17 12:32:46,927] jepsen worker 74 - jepsen.util 6554 :info :write [24 25] :RequestCanceledException INFO [2019-12-17 12:32:46,942] jepsen worker 5 - jepsen.util 5855 :info :write [1 1] :RequestCanceledException INFO [2019-12-17 12:32:47,012] jepsen worker 13 - jepsen.util 6133 :info :write [4 32] :RequestCanceledException INFO [2019-12-17 12:32:47,012] jepsen worker 64 - jepsen.util 5014 :fail :read [21 nil] :RequestTimeoutException INFO [2019-12-17 12:32:47,013] jepsen worker 64 - jepsen.util 5014 :invoke :write [21 21] INFO [2019-12-17 12:32:47,013] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37042,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:32:47,014] jepsen worker 31 - jepsen.util 5791 :ok :write [10 38] INFO [2019-12-17 12:32:47,014] jepsen worker 31 - jepsen.util 5791 :invoke :read [10 nil] INFO [2019-12-17 12:32:47,015] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37042,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:32:47,016] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37045,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:32:47,020] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37045,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:32:47,024] jepsen worker 5 - jepsen.util 5945 :invoke :read [1 nil] INFO [2019-12-17 12:32:47,025] jepsen worker 74 - jepsen.util 6644 :invoke :cas [24 [1 25]] INFO [2019-12-17 12:32:47,034] jepsen worker 45 - jepsen.util 6345 :invoke :write [15 17] INFO [2019-12-17 12:32:47,038] jepsen worker 28 - jepsen.util 4978 :ok :write [9 48] INFO [2019-12-17 12:32:47,039] jepsen worker 28 - jepsen.util 4978 :invoke :cas [9 [36 9]] INFO [2019-12-17 12:32:47,107] jepsen worker 13 - jepsen.util 6223 :invoke :cas [4 [9 14]] INFO [2019-12-17 12:32:47,128] jepsen worker 4 - jepsen.util 5854 :info :write [1 20] :RequestCanceledException INFO [2019-12-17 12:32:47,128] jepsen worker 72 - jepsen.util 6192 :info :write [24 29] :RequestCanceledException INFO [2019-12-17 12:32:47,137] jepsen worker 72 - jepsen.util 6282 :invoke :cas [24 [36 38]] INFO [2019-12-17 12:32:47,145] jepsen worker 73 - jepsen.util 6013 :ok :write [24 33] INFO [2019-12-17 12:32:47,146] jepsen worker 73 - jepsen.util 6013 :invoke :cas [24 [33 4]] INFO [2019-12-17 12:32:47,185] jepsen worker 4 - jepsen.util 5944 :invoke :read [1 nil] INFO [2019-12-17 12:32:47,187] jepsen worker 10 - jepsen.util 6130 :ok :write [3 2] INFO [2019-12-17 12:32:47,187] jepsen worker 10 - jepsen.util 6130 :invoke :read [3 nil] INFO [2019-12-17 12:32:47,228] jepsen worker 55 - jepsen.util 7345 :info :cas [18 [12 12]] :RequestCanceledException INFO [2019-12-17 12:32:47,238] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37057,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:32:47,239] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37065,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:32:47,241] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37057,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:32:47,241] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37065,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:32:47,244] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37067,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:32:47,261] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37067,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:32:47,262] jepsen worker 55 - jepsen.util 7435 :invoke :write [18 36] INFO [2019-12-17 12:32:47,375] jepsen worker 14 - jepsen.util 6584 :fail :cas [4 [15 39]] :ValueNotSwapFrom INFO [2019-12-17 12:32:47,376] jepsen worker 14 - jepsen.util 6584 :invoke :cas [4 [10 29]] INFO [2019-12-17 12:32:47,413] jepsen worker 11 - jepsen.util 6671 :info :write [3 8] :RequestCanceledException INFO [2019-12-17 12:32:47,429] jepsen worker 66 - jepsen.util 8346 :info :write [22 27] :RequestCanceledException INFO [2019-12-17 12:32:47,451] jepsen worker 47 - jepsen.util 6977 :fail :read [15 nil] :RequestTimeoutException INFO [2019-12-17 12:32:47,452] jepsen worker 47 - jepsen.util 6977 :invoke :read [15 nil] INFO [2019-12-17 12:32:47,452] jepsen worker 68 - jepsen.util 8168 :info :write [22 15] :RequestCanceledException INFO [2019-12-17 12:32:47,464] jepsen worker 8 - jepsen.util 8288 :info :cas [2 [19 28]] :RequestCanceledException INFO [2019-12-17 12:32:47,468] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37071,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:32:47,469] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37071,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:32:47,470] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37072,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:32:47,475] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37072,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:32:47,482] jepsen worker 11 - jepsen.util 6761 :invoke :cas [3 [12 23]] INFO [2019-12-17 12:32:47,483] jepsen worker 79 - jepsen.util 6469 :info :write [26 21] :RequestCanceledException INFO [2019-12-17 12:32:47,517] jepsen worker 27 - jepsen.util 6417 :info :write [9 4] :RequestCanceledException INFO [2019-12-17 12:32:47,529] jepsen worker 48 - jepsen.util 7968 :info :write [16 32] :RequestCanceledException INFO [2019-12-17 12:32:47,553] jepsen worker 85 - jepsen.util 9625 :info :write [28 0] :RequestCanceledException INFO [2019-12-17 12:32:47,563] jepsen worker 60 - jepsen.util 9510 :info :write [20 36] :RequestCanceledException INFO [2019-12-17 12:32:47,577] jepsen worker 79 - jepsen.util 6559 :invoke :cas [26 [13 22]] INFO [2019-12-17 12:32:47,582] jepsen worker 8 - jepsen.util 8378 :invoke :cas [2 [32 30]] INFO [2019-12-17 12:32:47,583] jepsen worker 66 - jepsen.util 8436 :invoke :write [22 11] INFO [2019-12-17 12:32:47,605] jepsen worker 68 - jepsen.util 8258 :invoke :cas [22 [37 39]] INFO [2019-12-17 12:32:47,606] jepsen worker 27 - jepsen.util 6507 :invoke :cas [9 [42 49]] INFO [2019-12-17 12:32:47,640] jepsen worker 76 - jepsen.util 8986 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:32:47,641] jepsen worker 76 - jepsen.util 8986 :invoke :write [25 38] INFO [2019-12-17 12:32:47,656] jepsen worker 34 - jepsen.util 7954 :info :write [11 48] :RequestCanceledException INFO [2019-12-17 12:32:47,658] jepsen worker 6 - jepsen.util 9186 :info :cas [2 [35 43]] :RequestCanceledException INFO [2019-12-17 12:32:47,679] jepsen worker 9 - jepsen.util 6939 :ok :write [3 10] INFO [2019-12-17 12:32:47,686] jepsen worker 9 - jepsen.util 6939 :invoke :read [3 nil] INFO [2019-12-17 12:32:47,695] jepsen worker 75 - jepsen.util 8175 :info :cas [25 [25 39]] :RequestCanceledException INFO [2019-12-17 12:32:47,695] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37087,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:32:47,704] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37087,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:32:47,714] jepsen worker 60 - jepsen.util 9600 :invoke :write [20 0] INFO [2019-12-17 12:32:47,716] jepsen worker 44 - jepsen.util 9404 :info :write [14 3] :RequestCanceledException INFO [2019-12-17 12:32:47,716] jepsen worker 20 - jepsen.util 6140 :info :cas [6 [42 29]] :RequestCanceledException INFO [2019-12-17 12:32:47,716] jepsen worker 82 - jepsen.util 9172 :info :cas [27 [11 1]] :RequestCanceledException INFO [2019-12-17 12:32:47,730] jepsen worker 85 - jepsen.util 9715 :invoke :cas [28 [44 2]] INFO [2019-12-17 12:32:47,732] jepsen worker 57 - jepsen.util 8157 :info :cas [19 [0 20]] :RequestCanceledException INFO [2019-12-17 12:32:47,733] jepsen worker 48 - jepsen.util 8058 :invoke :write [16 16] INFO [2019-12-17 12:32:47,734] jepsen worker 16 - jepsen.util 9646 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:32:47,737] jepsen worker 43 - jepsen.util 7963 :info :write [14 37] :RequestCanceledException INFO [2019-12-17 12:32:47,740] jepsen worker 16 - jepsen.util 9646 :invoke :write [5 22] INFO [2019-12-17 12:32:47,779] jepsen worker 34 - jepsen.util 8044 :invoke :read [11 nil] INFO [2019-12-17 12:32:47,796] jepsen worker 6 - jepsen.util 9276 :invoke :write [2 5] INFO [2019-12-17 12:32:47,797] jepsen worker 20 - jepsen.util 6230 :invoke :cas [6 [29 25]] INFO [2019-12-17 12:32:47,797] jepsen worker 41 - jepsen.util 9401 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:32:47,797] jepsen worker 41 - jepsen.util 9401 :invoke :read [13 nil] INFO [2019-12-17 12:32:47,820] jepsen worker 54 - jepsen.util 7074 :info :write [18 19] :RequestCanceledException INFO [2019-12-17 12:32:47,835] jepsen worker 0 - jepsen.util 8910 :info :cas [0 [24 5]] :RequestCanceledException INFO [2019-12-17 12:32:47,835] jepsen worker 58 - jepsen.util 9238 :info :write [19 16] :RequestCanceledException INFO [2019-12-17 12:32:47,846] jepsen worker 62 - jepsen.util 8342 :info :cas [20 [18 3]] :RequestCanceledException INFO [2019-12-17 12:32:47,846] jepsen worker 33 - jepsen.util 8493 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:32:47,851] jepsen worker 75 - jepsen.util 8265 :invoke :read [25 nil] INFO [2019-12-17 12:32:47,851] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:32:47,851] jepsen worker 33 - jepsen.util 8493 :invoke :write [11 49] INFO [2019-12-17 12:32:47,855] jepsen worker 54 - jepsen.util 7164 :invoke :write [18 26] INFO [2019-12-17 12:32:47,855] jepsen nemesis - couchbase.util Rebalance check count: 281 INFO [2019-12-17 12:32:47,856] jepsen nemesis - couchbase.util Rebalance stuck count: 262 INFO [2019-12-17 12:32:47,862] jepsen worker 39 - jepsen.util 8319 :info :cas [13 [44 27]] :RequestCanceledException INFO [2019-12-17 12:32:47,864] jepsen worker 57 - jepsen.util 8247 :invoke :write [19 30] INFO [2019-12-17 12:32:47,864] jepsen worker 82 - jepsen.util 9262 :invoke :cas [27 [9 23]] INFO [2019-12-17 12:32:47,876] jepsen worker 44 - jepsen.util 9494 :invoke :write [14 39] INFO [2019-12-17 12:32:47,888] jepsen worker 43 - jepsen.util 8053 :invoke :read [14 nil] INFO [2019-12-17 12:32:47,907] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37096,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:32:47,908] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37096,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:32:47,909] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37100,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:32:47,913] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37100,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:32:47,919] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37102,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:32:47,921] jepsen worker 89 - jepsen.util 6119 :info :cas [29 [41 47]] :RequestCanceledException INFO [2019-12-17 12:32:47,922] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37102,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:32:47,923] jepsen worker 58 - jepsen.util 9328 :invoke :read [19 nil] INFO [2019-12-17 12:32:47,926] jepsen worker 77 - jepsen.util 9077 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:32:47,927] jepsen worker 77 - jepsen.util 9077 :invoke :read [25 nil] INFO [2019-12-17 12:32:47,927] jepsen worker 0 - jepsen.util 9000 :invoke :read [0 nil] INFO [2019-12-17 12:32:47,930] jepsen worker 69 - jepsen.util 6459 :info :write [23 34] :RequestCanceledException INFO [2019-12-17 12:32:47,950] jepsen worker 62 - jepsen.util 8432 :invoke :cas [20 [13 39]] INFO [2019-12-17 12:32:47,965] jepsen worker 86 - jepsen.util 8996 :info :cas [28 [45 11]] :RequestCanceledException INFO [2019-12-17 12:32:47,966] jepsen worker 39 - jepsen.util 8409 :invoke :cas [13 [15 26]] INFO [2019-12-17 12:32:47,996] jepsen worker 81 - jepsen.util 8271 :info :cas [27 [34 12]] :RequestCanceledException INFO [2019-12-17 12:32:48,022] jepsen worker 69 - jepsen.util 6549 :invoke :cas [23 [35 41]] INFO [2019-12-17 12:32:48,030] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37110,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:32:48,031] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37110,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:32:48,032] jepsen worker 21 - jepsen.util 8211 :info :write [7 30] :RequestCanceledException INFO [2019-12-17 12:32:48,040] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37113,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:32:48,055] jepsen worker 1 - jepsen.util 8461 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:32:48,068] jepsen worker 86 - jepsen.util 9086 :invoke :cas [28 [33 26]] INFO [2019-12-17 12:32:48,068] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37113,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:32:48,068] jepsen worker 1 - jepsen.util 8461 :invoke :read [0 nil] INFO [2019-12-17 12:32:48,076] jepsen worker 83 - jepsen.util 8993 :info :write [27 5] :RequestCanceledException INFO [2019-12-17 12:32:48,076] jepsen worker 89 - jepsen.util 6209 :invoke :cas [29 [44 32]] INFO [2019-12-17 12:32:48,082] jepsen worker 84 - jepsen.util 8094 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:32:48,083] jepsen worker 61 - jepsen.util 9511 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:32:48,084] jepsen worker 84 - jepsen.util 8094 :invoke :write [28 37] INFO [2019-12-17 12:32:48,084] jepsen worker 61 - jepsen.util 9511 :invoke :read [20 nil] INFO [2019-12-17 12:32:48,084] jepsen worker 87 - jepsen.util 5757 :ok :read [29 41] INFO [2019-12-17 12:32:48,085] jepsen worker 87 - jepsen.util 5757 :invoke :read [29 nil] INFO [2019-12-17 12:32:48,115] jepsen worker 46 - jepsen.util 6436 :ok :read [15 43] INFO [2019-12-17 12:32:48,115] jepsen worker 46 - jepsen.util 6436 :invoke :write [15 33] INFO [2019-12-17 12:32:48,132] jepsen worker 67 - jepsen.util 8977 :info :write [22 4] :RequestCanceledException INFO [2019-12-17 12:32:48,135] jepsen worker 25 - jepsen.util 9115 :info :write [8 28] :RequestCanceledException INFO [2019-12-17 12:32:48,140] jepsen worker 81 - jepsen.util 8361 :invoke :write [27 44] INFO [2019-12-17 12:32:48,141] jepsen worker 21 - jepsen.util 8301 :invoke :read [7 nil] INFO [2019-12-17 12:32:48,156] jepsen worker 83 - jepsen.util 9083 :invoke :cas [27 [26 18]] INFO [2019-12-17 12:32:48,163] jepsen worker 22 - jepsen.util 8662 :info :cas [7 [27 5]] :RequestCanceledException INFO [2019-12-17 12:32:48,199] jepsen worker 40 - jepsen.util 9040 :info :cas [13 [25 8]] :RequestCanceledException INFO [2019-12-17 12:32:48,208] jepsen worker 67 - jepsen.util 9067 :invoke :read [22 nil] INFO [2019-12-17 12:32:48,211] jepsen worker 25 - jepsen.util 9205 :invoke :cas [8 [0 46]] INFO [2019-12-17 12:32:48,216] jepsen worker 15 - jepsen.util 7665 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:32:48,216] jepsen worker 15 - jepsen.util 7665 :invoke :read [5 nil] INFO [2019-12-17 12:32:48,218] jepsen worker 22 - jepsen.util 8752 :invoke :write [7 22] INFO [2019-12-17 12:32:48,227] jepsen worker 29 - jepsen.util 4889 :ok :read [9 39] INFO [2019-12-17 12:32:48,228] jepsen worker 29 - jepsen.util 4889 :invoke :read [9 nil] INFO [2019-12-17 12:32:48,229] jepsen worker 7 - jepsen.util 8917 :info :cas [2 [19 32]] :RequestCanceledException INFO [2019-12-17 12:32:48,233] jepsen worker 37 - jepsen.util 6247 :fail :cas [12 [5 42]] :ValueNotSwapFrom INFO [2019-12-17 12:32:48,249] jepsen worker 37 - jepsen.util 6247 :invoke :write [12 42] INFO [2019-12-17 12:32:48,250] jepsen worker 17 - jepsen.util 8747 :info :write [5 12] :RequestCanceledException INFO [2019-12-17 12:32:48,251] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37124,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:32:48,252] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37132,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:32:48,257] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37124,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:32:48,259] jepsen worker 12 - jepsen.util 6762 :info :write [4 39] :RequestCanceledException INFO [2019-12-17 12:32:48,260] jepsen worker 19 - jepsen.util 6319 :fail :cas [6 [20 30]] :ValueNotSwapFrom INFO [2019-12-17 12:32:48,261] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37128,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:32:48,262] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37128,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:32:48,262] jepsen worker 19 - jepsen.util 6319 :invoke :cas [6 [46 3]] INFO [2019-12-17 12:32:48,263] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37132,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:32:48,267] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37137,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:32:48,268] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37137,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:32:48,268] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37138,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:32:48,268] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37138,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:32:48,285] jepsen worker 7 - jepsen.util 9007 :invoke :cas [2 [5 38]] INFO [2019-12-17 12:32:48,293] jepsen worker 12 - jepsen.util 6852 :invoke :read [4 nil] INFO [2019-12-17 12:32:48,302] jepsen worker 40 - jepsen.util 9130 :invoke :cas [13 [1 17]] INFO [2019-12-17 12:32:48,316] jepsen worker 17 - jepsen.util 8837 :invoke :cas [5 [5 47]] INFO [2019-12-17 12:32:48,321] jepsen worker 74 - jepsen.util 6644 :fail :cas [24 [1 25]] :ValueNotSwapFrom INFO [2019-12-17 12:32:48,334] jepsen worker 74 - jepsen.util 6644 :invoke :read [24 nil] INFO [2019-12-17 12:32:48,335] jepsen worker 42 - jepsen.util 9042 :info :write [14 9] :RequestCanceledException INFO [2019-12-17 12:32:48,345] jepsen worker 23 - jepsen.util 8753 :info :write [7 38] :RequestCanceledException INFO [2019-12-17 12:32:48,360] jepsen worker 53 - jepsen.util 6173 :fail :read [17 nil] :RequestTimeoutException INFO [2019-12-17 12:32:48,360] jepsen worker 26 - jepsen.util 9026 :info :write [8 9] :RequestCanceledException INFO [2019-12-17 12:32:48,360] jepsen worker 53 - jepsen.util 6173 :invoke :write [17 46] INFO [2019-12-17 12:32:48,389] jepsen worker 42 - jepsen.util 9132 :invoke :write [14 35] INFO [2019-12-17 12:32:48,396] jepsen worker 49 - jepsen.util 7789 :info :cas [16 [5 43]] :RequestCanceledException INFO [2019-12-17 12:32:48,396] jepsen worker 51 - jepsen.util 6351 :info :write [17 3] :RequestCanceledException INFO [2019-12-17 12:32:48,397] jepsen worker 23 - jepsen.util 8843 :invoke :read [7 nil] INFO [2019-12-17 12:32:48,412] jepsen worker 59 - jepsen.util 8789 :info :write [19 30] :RequestCanceledException INFO [2019-12-17 12:32:48,413] jepsen worker 51 - jepsen.util 6441 :invoke :read [17 nil] INFO [2019-12-17 12:32:48,415] jepsen worker 26 - jepsen.util 9116 :invoke :read [8 nil] INFO [2019-12-17 12:32:48,426] jepsen worker 24 - jepsen.util 7674 :info :write [8 35] :RequestCanceledException INFO [2019-12-17 12:32:48,427] jepsen worker 35 - jepsen.util 8585 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:32:48,430] jepsen worker 35 - jepsen.util 8585 :invoke :write [11 18] INFO [2019-12-17 12:32:48,454] jepsen worker 49 - jepsen.util 7879 :invoke :cas [16 [25 2]] INFO [2019-12-17 12:32:48,482] jepsen worker 24 - jepsen.util 7764 :invoke :read [8 nil] INFO [2019-12-17 12:32:48,482] jepsen worker 59 - jepsen.util 8879 :invoke :write [19 22] INFO [2019-12-17 12:32:48,524] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37152,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:32:48,525] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37152,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:32:48,526] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37157,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:32:48,535] jepsen worker 18 - jepsen.util 6318 :info :cas [6 [12 16]] :RequestCanceledException INFO [2019-12-17 12:32:48,535] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37157,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:32:48,570] jepsen worker 18 - jepsen.util 6408 :invoke :read [6 nil] INFO [2019-12-17 12:32:48,631] jepsen worker 2 - jepsen.util 8462 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:32:48,631] jepsen worker 56 - jepsen.util 6896 :ok :read [18 19] INFO [2019-12-17 12:32:48,631] jepsen worker 2 - jepsen.util 8462 :invoke :read [0 nil] INFO [2019-12-17 12:32:48,632] jepsen worker 56 - jepsen.util 6896 :invoke :write [18 40] INFO [2019-12-17 12:32:48,645] jepsen worker 88 - jepsen.util 6478 :info :write [29 24] :RequestCanceledException INFO [2019-12-17 12:32:48,677] jepsen worker 88 - jepsen.util 6568 :invoke :write [29 7] INFO [2019-12-17 12:32:48,695] jepsen worker 3 - jepsen.util 6303 :info :write [1 6] :RequestCanceledException INFO [2019-12-17 12:32:48,700] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37171,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:32:48,700] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37171,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:32:48,704] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37174,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:32:48,706] jepsen worker 64 - jepsen.util 5014 :ok :write [21 21] INFO [2019-12-17 12:32:48,713] jepsen worker 64 - jepsen.util 5014 :invoke :read [21 nil] INFO [2019-12-17 12:32:48,717] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37174,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:32:48,721] jepsen worker 3 - jepsen.util 6393 :invoke :read [1 nil] INFO [2019-12-17 12:32:48,732] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37175,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:32:48,734] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37175,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:32:48,738] jepsen worker 80 - jepsen.util 6380 :ok :read [26 36] INFO [2019-12-17 12:32:48,738] jepsen worker 80 - jepsen.util 6380 :invoke :read [26 nil] INFO [2019-12-17 12:32:48,739] jepsen worker 70 - jepsen.util 6910 :ok :read [23 14] INFO [2019-12-17 12:32:48,740] jepsen worker 70 - jepsen.util 6910 :invoke :read [23 nil] INFO [2019-12-17 12:32:48,745] jepsen worker 50 - jepsen.util 9230 :info :write [16 7] :RequestCanceledException INFO [2019-12-17 12:32:48,761] jepsen worker 32 - jepsen.util 6602 :info :write [10 42] :RequestCanceledException INFO [2019-12-17 12:32:48,769] jepsen worker 31 - jepsen.util 5791 :ok :read [10 38] INFO [2019-12-17 12:32:48,769] jepsen worker 65 - jepsen.util 5285 :fail :cas [21 [25 43]] :ValueNotSwapFrom INFO [2019-12-17 12:32:48,777] jepsen worker 65 - jepsen.util 5285 :invoke :cas [21 [45 42]] INFO [2019-12-17 12:32:48,777] jepsen worker 31 - jepsen.util 5791 :invoke :write [10 15] INFO [2019-12-17 12:32:48,788] jepsen worker 32 - jepsen.util 6692 :invoke :cas [10 [46 42]] INFO [2019-12-17 12:32:48,789] jepsen worker 50 - jepsen.util 9320 :invoke :read [16 nil] INFO [2019-12-17 12:32:48,792] jepsen worker 30 - jepsen.util 6330 :ok :read [10 38] INFO [2019-12-17 12:32:48,800] jepsen worker 30 - jepsen.util 6330 :invoke :cas [10 [7 18]] INFO [2019-12-17 12:32:48,931] jepsen worker 63 - jepsen.util 5373 :info :write [21 42] :RequestCanceledException INFO [2019-12-17 12:32:48,932] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:32:48,933] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37195,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:32:48,936] jepsen nemesis - couchbase.util Rebalance check count: 282 INFO [2019-12-17 12:32:48,937] jepsen worker 45 - jepsen.util 6345 :ok :write [15 17] INFO [2019-12-17 12:32:48,939] jepsen nemesis - couchbase.util Rebalance stuck count: 263 INFO [2019-12-17 12:32:48,939] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37195,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:32:48,940] jepsen worker 45 - jepsen.util 6345 :invoke :cas [15 [27 26]] INFO [2019-12-17 12:32:48,951] jepsen worker 63 - jepsen.util 5463 :invoke :read [21 nil] INFO [2019-12-17 12:32:48,967] jepsen worker 78 - jepsen.util 6738 :info :write [26 20] :RequestCanceledException INFO [2019-12-17 12:32:49,011] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37201,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:32:49,012] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37201,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:32:49,060] jepsen worker 78 - jepsen.util 6828 :invoke :cas [26 [23 4]] INFO [2019-12-17 12:32:49,114] jepsen worker 36 - jepsen.util 7056 :info :write [12 16] :RequestCanceledException INFO [2019-12-17 12:32:49,116] jepsen worker 5 - jepsen.util 5945 :fail :read [1 nil] :RequestTimeoutException INFO [2019-12-17 12:32:49,117] jepsen worker 5 - jepsen.util 5945 :invoke :write [1 20] INFO [2019-12-17 12:32:49,122] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37204,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:32:49,129] jepsen worker 46 - jepsen.util 6436 :ok :write [15 33] INFO [2019-12-17 12:32:49,131] jepsen worker 46 - jepsen.util 6436 :invoke :read [15 nil] INFO [2019-12-17 12:32:49,131] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37204,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:32:49,162] jepsen worker 13 - jepsen.util 6223 :info :cas [4 [9 14]] :RequestCanceledException INFO [2019-12-17 12:32:49,223] jepsen worker 36 - jepsen.util 7146 :invoke :cas [12 [17 23]] INFO [2019-12-17 12:32:49,234] jepsen worker 13 - jepsen.util 6313 :invoke :cas [4 [23 11]] INFO [2019-12-17 12:32:49,241] jepsen worker 10 - jepsen.util 6130 :fail :read [3 nil] :RequestTimeoutException INFO [2019-12-17 12:32:49,244] jepsen worker 10 - jepsen.util 6130 :invoke :write [3 49] INFO [2019-12-17 12:32:49,291] jepsen worker 4 - jepsen.util 5944 :fail :read [1 nil] :RequestTimeoutException INFO [2019-12-17 12:32:49,291] jepsen worker 4 - jepsen.util 5944 :invoke :read [1 nil] INFO [2019-12-17 12:32:49,297] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37210,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:32:49,299] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37210,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:32:49,299] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37218,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:32:49,301] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37218,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:32:49,301] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37219,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:32:49,301] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37219,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:32:49,362] jepsen worker 27 - jepsen.util 6507 :fail :cas [9 [42 49]] :ValueNotSwapFrom INFO [2019-12-17 12:32:49,364] jepsen worker 27 - jepsen.util 6507 :invoke :read [9 nil] INFO [2019-12-17 12:32:49,417] jepsen worker 14 - jepsen.util 6584 :info :cas [4 [10 29]] :RequestCanceledException INFO [2019-12-17 12:32:49,470] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37228,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:32:49,470] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37228,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:32:49,497] jepsen worker 11 - jepsen.util 6761 :fail :cas [3 [12 23]] :ValueNotSwapFrom INFO [2019-12-17 12:32:49,505] jepsen worker 11 - jepsen.util 6761 :invoke :write [3 19] INFO [2019-12-17 12:32:49,537] jepsen worker 52 - jepsen.util 6892 :fail :cas [17 [13 8]] :ValueNotSwapFrom INFO [2019-12-17 12:32:49,542] jepsen worker 52 - jepsen.util 6892 :invoke :read [17 nil] INFO [2019-12-17 12:32:49,559] jepsen worker 14 - jepsen.util 6674 :invoke :read [4 nil] INFO [2019-12-17 12:32:49,560] jepsen worker 31 - jepsen.util 5791 :ok :write [10 15] INFO [2019-12-17 12:32:49,560] jepsen worker 31 - jepsen.util 5791 :invoke :read [10 nil] INFO [2019-12-17 12:32:49,583] jepsen worker 28 - jepsen.util 4978 :fail :cas [9 [36 9]] :ValueNotSwapFrom INFO [2019-12-17 12:32:49,592] jepsen worker 28 - jepsen.util 4978 :invoke :write [9 19] INFO [2019-12-17 12:32:49,652] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37237,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:32:49,653] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37237,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:32:49,653] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37241,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:32:49,653] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37241,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:32:49,657] jepsen worker 8 - jepsen.util 8378 :info :cas [2 [32 30]] :RequestCanceledException INFO [2019-12-17 12:32:49,663] jepsen worker 20 - jepsen.util 6230 :fail :cas [6 [29 25]] :ValueNotSwapFrom INFO [2019-12-17 12:32:49,664] jepsen worker 20 - jepsen.util 6230 :invoke :write [6 0] INFO [2019-12-17 12:32:49,666] jepsen worker 72 - jepsen.util 6282 :fail :cas [24 [36 38]] :ValueNotSwapFrom INFO [2019-12-17 12:32:49,666] jepsen worker 72 - jepsen.util 6282 :invoke :read [24 nil] INFO [2019-12-17 12:32:49,677] jepsen worker 8 - jepsen.util 8468 :invoke :cas [2 [42 21]] INFO [2019-12-17 12:32:49,694] jepsen worker 68 - jepsen.util 8258 :info :cas [22 [37 39]] :RequestCanceledException INFO [2019-12-17 12:32:49,706] jepsen worker 68 - jepsen.util 8348 :invoke :write [22 25] INFO [2019-12-17 12:32:49,712] jepsen worker 18 - jepsen.util 6408 :ok :read [6 25] INFO [2019-12-17 12:32:49,712] jepsen worker 18 - jepsen.util 6408 :invoke :cas [6 [31 22]] INFO [2019-12-17 12:32:49,713] jepsen worker 55 - jepsen.util 7435 :info :write [18 36] :RequestCanceledException INFO [2019-12-17 12:32:49,723] jepsen worker 55 - jepsen.util 7525 :invoke :cas [18 [19 3]] INFO [2019-12-17 12:32:49,728] jepsen worker 74 - jepsen.util 6644 :ok :read [24 33] INFO [2019-12-17 12:32:49,729] jepsen worker 74 - jepsen.util 6644 :invoke :write [24 29] INFO [2019-12-17 12:32:49,731] jepsen worker 29 - jepsen.util 4889 :ok :read [9 48] INFO [2019-12-17 12:32:49,732] jepsen worker 29 - jepsen.util 4889 :invoke :read [9 nil] INFO [2019-12-17 12:32:49,758] jepsen worker 19 - jepsen.util 6319 :fail :cas [6 [46 3]] :ValueNotSwapFrom INFO [2019-12-17 12:32:49,765] jepsen worker 19 - jepsen.util 6319 :invoke :cas [6 [41 40]] INFO [2019-12-17 12:32:49,774] jepsen worker 71 - jepsen.util 7091 :ok :read [23 14] INFO [2019-12-17 12:32:49,775] jepsen worker 71 - jepsen.util 7091 :invoke :cas [23 [41 39]] INFO [2019-12-17 12:32:49,775] jepsen worker 89 - jepsen.util 6209 :fail :cas [29 [44 32]] :ValueNotSwapFrom INFO [2019-12-17 12:32:49,776] jepsen worker 89 - jepsen.util 6209 :invoke :read [29 nil] INFO [2019-12-17 12:32:49,831] jepsen worker 34 - jepsen.util 8044 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:32:49,837] jepsen worker 34 - jepsen.util 8044 :invoke :read [11 nil] INFO [2019-12-17 12:32:49,841] jepsen worker 85 - jepsen.util 9715 :info :cas [28 [44 2]] :RequestCanceledException INFO [2019-12-17 12:32:49,885] jepsen worker 85 - jepsen.util 9805 :invoke :write [28 37] INFO [2019-12-17 12:32:49,899] jepsen worker 41 - jepsen.util 9401 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:32:49,901] jepsen worker 41 - jepsen.util 9401 :invoke :write [13 14] INFO [2019-12-17 12:32:49,915] jepsen worker 43 - jepsen.util 8053 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:32:49,916] jepsen worker 43 - jepsen.util 8053 :invoke :read [14 nil] INFO [2019-12-17 12:32:49,932] jepsen worker 75 - jepsen.util 8265 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:32:49,934] jepsen worker 75 - jepsen.util 8265 :invoke :cas [25 [23 19]] INFO [2019-12-17 12:32:49,934] jepsen worker 32 - jepsen.util 6692 :fail :cas [10 [46 42]] :ValueNotSwapFrom INFO [2019-12-17 12:32:49,935] jepsen worker 32 - jepsen.util 6692 :invoke :read [10 nil] INFO [2019-12-17 12:32:49,937] jepsen worker 82 - jepsen.util 9262 :info :cas [27 [9 23]] :RequestCanceledException INFO [2019-12-17 12:32:49,962] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37257,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:32:49,969] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37257,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:32:49,970] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37259,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:32:49,973] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37259,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:32:49,983] jepsen worker 10 - jepsen.util 6130 :ok :write [3 49] INFO [2019-12-17 12:32:49,988] jepsen worker 10 - jepsen.util 6130 :invoke :write [3 6] INFO [2019-12-17 12:32:50,010] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:32:50,011] jepsen nemesis - couchbase.util Rebalance check count: 283 INFO [2019-12-17 12:32:50,012] jepsen nemesis - couchbase.util Rebalance stuck count: 264 INFO [2019-12-17 12:32:50,014] jepsen worker 82 - jepsen.util 9352 :invoke :write [27 45] INFO [2019-12-17 12:32:50,030] jepsen worker 39 - jepsen.util 8409 :info :cas [13 [15 26]] :RequestCanceledException INFO [2019-12-17 12:32:50,030] jepsen worker 58 - jepsen.util 9328 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:32:50,030] jepsen worker 54 - jepsen.util 7164 :ok :write [18 26] INFO [2019-12-17 12:32:50,031] jepsen worker 58 - jepsen.util 9328 :invoke :read [19 nil] INFO [2019-12-17 12:32:50,032] jepsen worker 77 - jepsen.util 9077 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:32:50,034] jepsen worker 54 - jepsen.util 7164 :invoke :cas [18 [35 4]] INFO [2019-12-17 12:32:50,034] jepsen worker 77 - jepsen.util 9077 :invoke :cas [25 [7 19]] INFO [2019-12-17 12:32:50,036] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37267,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:32:50,036] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37267,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:32:50,047] jepsen worker 63 - jepsen.util 5463 :ok :read [21 21] INFO [2019-12-17 12:32:50,049] jepsen worker 63 - jepsen.util 5463 :invoke :write [21 21] INFO [2019-12-17 12:32:50,050] jepsen worker 0 - jepsen.util 9000 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:32:50,052] jepsen worker 0 - jepsen.util 9000 :invoke :cas [0 [4 0]] INFO [2019-12-17 12:32:50,065] jepsen worker 62 - jepsen.util 8432 :info :cas [20 [13 39]] :RequestCanceledException INFO [2019-12-17 12:32:50,080] jepsen worker 39 - jepsen.util 8499 :invoke :cas [13 [15 33]] INFO [2019-12-17 12:32:50,104] jepsen worker 62 - jepsen.util 8522 :invoke :cas [20 [18 45]] INFO [2019-12-17 12:32:50,122] jepsen worker 86 - jepsen.util 9086 :info :cas [28 [33 26]] :RequestCanceledException INFO [2019-12-17 12:32:50,141] jepsen worker 87 - jepsen.util 5757 :fail :read [29 nil] :RequestTimeoutException INFO [2019-12-17 12:32:50,142] jepsen worker 1 - jepsen.util 8461 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:32:50,143] jepsen worker 87 - jepsen.util 5757 :invoke :read [29 nil] INFO [2019-12-17 12:32:50,145] jepsen worker 1 - jepsen.util 8461 :invoke :write [0 21] INFO [2019-12-17 12:32:50,162] jepsen worker 61 - jepsen.util 9511 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:32:50,162] jepsen worker 61 - jepsen.util 9511 :invoke :cas [20 [46 6]] INFO [2019-12-17 12:32:50,165] jepsen worker 21 - jepsen.util 8301 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:32:50,166] jepsen worker 21 - jepsen.util 8301 :invoke :read [7 nil] INFO [2019-12-17 12:32:50,166] jepsen worker 76 - jepsen.util 8986 :info :write [25 38] :RequestCanceledException INFO [2019-12-17 12:32:50,196] jepsen worker 86 - jepsen.util 9176 :invoke :read [28 nil] INFO [2019-12-17 12:32:50,209] jepsen worker 38 - jepsen.util 6518 :ok :read [12 19] INFO [2019-12-17 12:32:50,210] jepsen worker 38 - jepsen.util 6518 :invoke :write [12 47] INFO [2019-12-17 12:32:50,210] jepsen worker 64 - jepsen.util 5014 :ok :read [21 21] INFO [2019-12-17 12:32:50,211] jepsen worker 64 - jepsen.util 5014 :invoke :read [21 nil] INFO [2019-12-17 12:32:50,220] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37290,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:32:50,221] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37290,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:32:50,224] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37292,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:32:50,229] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37292,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:32:50,230] jepsen worker 76 - jepsen.util 9076 :invoke :read [25 nil] INFO [2019-12-17 12:32:50,232] jepsen worker 83 - jepsen.util 9083 :info :cas [27 [26 18]] :RequestCanceledException INFO [2019-12-17 12:32:50,259] jepsen worker 83 - jepsen.util 9173 :invoke :write [27 18] INFO [2019-12-17 12:32:50,286] jepsen worker 79 - jepsen.util 6559 :fail :cas [26 [13 22]] :ValueNotSwapFrom INFO [2019-12-17 12:32:50,286] jepsen worker 79 - jepsen.util 6559 :invoke :write [26 49] INFO [2019-12-17 12:32:50,289] jepsen worker 78 - jepsen.util 6828 :fail :cas [26 [23 4]] :ValueNotSwapFrom INFO [2019-12-17 12:32:50,291] jepsen worker 78 - jepsen.util 6828 :invoke :cas [26 [40 24]] INFO [2019-12-17 12:32:50,296] jepsen worker 67 - jepsen.util 9067 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:32:50,296] jepsen worker 67 - jepsen.util 9067 :invoke :read [22 nil] INFO [2019-12-17 12:32:50,297] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37280,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:32:50,305] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37280,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:32:50,307] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37284,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:32:50,310] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37284,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:32:50,311] jepsen worker 69 - jepsen.util 6549 :fail :cas [23 [35 41]] :ValueNotSwapFrom INFO [2019-12-17 12:32:50,312] jepsen worker 25 - jepsen.util 9205 :info :cas [8 [0 46]] :RequestCanceledException INFO [2019-12-17 12:32:50,312] jepsen worker 69 - jepsen.util 6549 :invoke :read [23 nil] INFO [2019-12-17 12:32:50,316] jepsen worker 15 - jepsen.util 7665 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:32:50,316] jepsen worker 15 - jepsen.util 7665 :invoke :read [5 nil] INFO [2019-12-17 12:32:50,329] jepsen worker 66 - jepsen.util 8436 :info :write [22 11] :RequestCanceledException INFO [2019-12-17 12:32:50,329] jepsen worker 40 - jepsen.util 9130 :info :cas [13 [1 17]] :RequestCanceledException INFO [2019-12-17 12:32:50,337] jepsen worker 9 - jepsen.util 6939 :ok :read [3 10] INFO [2019-12-17 12:32:50,338] jepsen worker 9 - jepsen.util 6939 :invoke :write [3 35] INFO [2019-12-17 12:32:50,343] jepsen worker 37 - jepsen.util 6247 :ok :write [12 42] INFO [2019-12-17 12:32:50,344] jepsen worker 37 - jepsen.util 6247 :invoke :read [12 nil] INFO [2019-12-17 12:32:50,346] jepsen worker 7 - jepsen.util 9007 :info :cas [2 [5 38]] :RequestCanceledException INFO [2019-12-17 12:32:50,348] jepsen worker 47 - jepsen.util 6977 :ok :read [15 17] INFO [2019-12-17 12:32:50,349] jepsen worker 47 - jepsen.util 6977 :invoke :write [15 29] INFO [2019-12-17 12:32:50,360] jepsen worker 65 - jepsen.util 5285 :fail :cas [21 [45 42]] :ValueNotSwapFrom INFO [2019-12-17 12:32:50,360] jepsen worker 65 - jepsen.util 5285 :invoke :write [21 2] INFO [2019-12-17 12:32:50,362] jepsen worker 25 - jepsen.util 9295 :invoke :read [8 nil] INFO [2019-12-17 12:32:50,369] jepsen worker 40 - jepsen.util 9220 :invoke :read [13 nil] INFO [2019-12-17 12:32:50,392] jepsen worker 17 - jepsen.util 8837 :info :cas [5 [5 47]] :RequestCanceledException INFO [2019-12-17 12:32:50,393] jepsen worker 66 - jepsen.util 8526 :invoke :read [22 nil] INFO [2019-12-17 12:32:50,402] jepsen worker 53 - jepsen.util 6173 :ok :write [17 46] INFO [2019-12-17 12:32:50,403] jepsen worker 53 - jepsen.util 6173 :invoke :write [17 32] INFO [2019-12-17 12:32:50,404] jepsen worker 70 - jepsen.util 6910 :ok :read [23 14] INFO [2019-12-17 12:32:50,405] jepsen worker 70 - jepsen.util 6910 :invoke :read [23 nil] INFO [2019-12-17 12:32:50,432] jepsen worker 23 - jepsen.util 8843 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:32:50,433] jepsen worker 31 - jepsen.util 5791 :ok :read [10 15] INFO [2019-12-17 12:32:50,434] jepsen worker 23 - jepsen.util 8843 :invoke :read [7 nil] INFO [2019-12-17 12:32:50,434] jepsen worker 31 - jepsen.util 5791 :invoke :cas [10 [37 44]] INFO [2019-12-17 12:32:50,448] jepsen worker 17 - jepsen.util 8927 :invoke :write [5 42] INFO [2019-12-17 12:32:50,460] jepsen worker 26 - jepsen.util 9116 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:32:50,461] jepsen worker 26 - jepsen.util 9116 :invoke :cas [8 [7 27]] INFO [2019-12-17 12:32:50,462] jepsen worker 7 - jepsen.util 9097 :invoke :cas [2 [16 9]] INFO [2019-12-17 12:32:50,465] jepsen worker 46 - jepsen.util 6436 :ok :read [15 33] INFO [2019-12-17 12:32:50,466] jepsen worker 46 - jepsen.util 6436 :invoke :read [15 nil] INFO [2019-12-17 12:32:50,495] jepsen worker 6 - jepsen.util 9276 :info :write [2 5] :RequestCanceledException INFO [2019-12-17 12:32:50,496] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37311,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:32:50,512] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37311,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:32:50,515] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37317,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:32:50,516] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37317,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:32:50,529] jepsen worker 49 - jepsen.util 7879 :info :cas [16 [25 2]] :RequestCanceledException INFO [2019-12-17 12:32:50,530] jepsen worker 16 - jepsen.util 9646 :info :write [5 22] :RequestCanceledException INFO [2019-12-17 12:32:50,531] jepsen worker 33 - jepsen.util 8493 :info :write [11 49] :RequestCanceledException INFO [2019-12-17 12:32:50,559] jepsen worker 6 - jepsen.util 9366 :invoke :read [2 nil] INFO [2019-12-17 12:32:50,563] jepsen worker 48 - jepsen.util 8058 :info :write [16 16] :RequestCanceledException INFO [2019-12-17 12:32:50,572] jepsen worker 49 - jepsen.util 7969 :invoke :cas [16 [20 19]] INFO [2019-12-17 12:32:50,580] jepsen worker 16 - jepsen.util 9736 :invoke :write [5 13] INFO [2019-12-17 12:32:50,597] jepsen worker 24 - jepsen.util 7764 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:32:50,597] jepsen worker 24 - jepsen.util 7764 :invoke :cas [8 [16 0]] INFO [2019-12-17 12:32:50,597] jepsen worker 33 - jepsen.util 8583 :invoke :read [11 nil] INFO [2019-12-17 12:32:50,599] jepsen worker 44 - jepsen.util 9494 :info :write [14 39] :RequestCanceledException INFO [2019-12-17 12:32:50,609] jepsen worker 48 - jepsen.util 8148 :invoke :read [16 nil] INFO [2019-12-17 12:32:50,628] jepsen worker 28 - jepsen.util 4978 :ok :write [9 19] INFO [2019-12-17 12:32:50,629] jepsen worker 28 - jepsen.util 4978 :invoke :write [9 17] INFO [2019-12-17 12:32:50,631] jepsen worker 60 - jepsen.util 9600 :info :write [20 0] :RequestCanceledException INFO [2019-12-17 12:32:50,646] jepsen worker 84 - jepsen.util 8094 :info :write [28 37] :RequestCanceledException INFO [2019-12-17 12:32:50,660] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37329,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:32:50,662] jepsen worker 57 - jepsen.util 8247 :info :write [19 30] :RequestCanceledException INFO [2019-12-17 12:32:50,662] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37329,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:32:50,662] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37331,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:32:50,662] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37331,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:32:50,686] jepsen worker 44 - jepsen.util 9584 :invoke :write [14 20] INFO [2019-12-17 12:32:50,689] jepsen worker 45 - jepsen.util 6345 :fail :cas [15 [27 26]] :ValueNotSwapFrom INFO [2019-12-17 12:32:50,689] jepsen worker 45 - jepsen.util 6345 :invoke :read [15 nil] INFO [2019-12-17 12:32:50,691] jepsen worker 57 - jepsen.util 8337 :invoke :cas [19 [27 8]] INFO [2019-12-17 12:32:50,694] jepsen worker 60 - jepsen.util 9690 :invoke :read [20 nil] INFO [2019-12-17 12:32:50,711] jepsen worker 42 - jepsen.util 9132 :info :write [14 35] :RequestCanceledException INFO [2019-12-17 12:32:50,714] jepsen worker 59 - jepsen.util 8879 :info :write [19 22] :RequestCanceledException INFO [2019-12-17 12:32:50,724] jepsen worker 84 - jepsen.util 8184 :invoke :read [28 nil] INFO [2019-12-17 12:32:50,725] jepsen worker 27 - jepsen.util 6507 :ok :read [9 48] INFO [2019-12-17 12:32:50,728] jepsen worker 27 - jepsen.util 6507 :invoke :cas [9 [13 44]] INFO [2019-12-17 12:32:50,731] jepsen worker 35 - jepsen.util 8585 :info :write [11 18] :RequestCanceledException INFO [2019-12-17 12:32:50,732] jepsen worker 2 - jepsen.util 8462 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:32:50,734] jepsen worker 2 - jepsen.util 8462 :invoke :cas [0 [30 37]] INFO [2019-12-17 12:32:50,755] jepsen worker 81 - jepsen.util 8361 :info :write [27 44] :RequestCanceledException INFO [2019-12-17 12:32:50,778] jepsen worker 59 - jepsen.util 8969 :invoke :read [19 nil] INFO [2019-12-17 12:32:50,786] jepsen worker 42 - jepsen.util 9222 :invoke :cas [14 [35 23]] INFO [2019-12-17 12:32:50,797] jepsen worker 35 - jepsen.util 8675 :invoke :write [11 49] INFO [2019-12-17 12:32:50,817] jepsen worker 80 - jepsen.util 6380 :fail :read [26 nil] :RequestTimeoutException INFO [2019-12-17 12:32:50,817] jepsen worker 80 - jepsen.util 6380 :invoke :write [26 23] INFO [2019-12-17 12:32:50,834] jepsen worker 22 - jepsen.util 8752 :info :write [7 22] :RequestCanceledException INFO [2019-12-17 12:32:50,848] jepsen worker 81 - jepsen.util 8451 :invoke :read [27 nil] INFO [2019-12-17 12:32:50,899] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37353,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:32:50,899] jepsen worker 50 - jepsen.util 9320 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:32:50,900] jepsen worker 50 - jepsen.util 9320 :invoke :read [16 nil] INFO [2019-12-17 12:32:50,938] jepsen worker 30 - jepsen.util 6330 :fail :cas [10 [7 18]] :ValueNotSwapFrom INFO [2019-12-17 12:32:50,938] jepsen worker 30 - jepsen.util 6330 :invoke :cas [10 [12 22]] INFO [2019-12-17 12:32:50,944] jepsen worker 74 - jepsen.util 6644 :ok :write [24 29] INFO [2019-12-17 12:32:50,945] jepsen worker 74 - jepsen.util 6644 :invoke :cas [24 [7 20]] INFO [2019-12-17 12:32:50,953] jepsen worker 4 - jepsen.util 5944 :ok :read [1 43] INFO [2019-12-17 12:32:50,958] jepsen worker 38 - jepsen.util 6518 :ok :write [12 47] INFO [2019-12-17 12:32:50,960] jepsen worker 4 - jepsen.util 5944 :invoke :read [1 nil] INFO [2019-12-17 12:32:50,975] jepsen worker 38 - jepsen.util 6518 :invoke :cas [12 [2 19]] INFO [2019-12-17 12:32:50,987] jepsen worker 22 - jepsen.util 8842 :invoke :read [7 nil] INFO [2019-12-17 12:32:51,013] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37353,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:32:51,070] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37364,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:32:51,079] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37364,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:32:51,080] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:32:51,083] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37369,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:32:51,085] jepsen nemesis - couchbase.util Rebalance check count: 284 INFO [2019-12-17 12:32:51,105] jepsen nemesis - couchbase.util Rebalance stuck count: 265 INFO [2019-12-17 12:32:51,109] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37369,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:32:51,119] jepsen worker 56 - jepsen.util 6896 :info :write [18 40] :RequestCanceledException INFO [2019-12-17 12:32:51,150] jepsen worker 88 - jepsen.util 6568 :info :write [29 7] :RequestCanceledException INFO [2019-12-17 12:32:51,170] jepsen worker 56 - jepsen.util 6986 :invoke :cas [18 [44 4]] INFO [2019-12-17 12:32:51,188] jepsen worker 88 - jepsen.util 6658 :invoke :cas [29 [23 15]] INFO [2019-12-17 12:32:51,233] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37374,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:32:51,238] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37374,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:32:51,266] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37382,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:32:51,276] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37382,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:32:51,278] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37383,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:32:51,279] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37383,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:32:51,286] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37384,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:32:51,289] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37384,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:32:51,295] jepsen worker 3 - jepsen.util 6393 :ok :read [1 43] INFO [2019-12-17 12:32:51,296] jepsen worker 3 - jepsen.util 6393 :invoke :cas [1 [3 33]] INFO [2019-12-17 12:32:51,313] jepsen worker 5 - jepsen.util 5945 :info :write [1 20] :RequestCanceledException INFO [2019-12-17 12:32:51,346] jepsen worker 12 - jepsen.util 6852 :ok :read [4 19] INFO [2019-12-17 12:32:51,347] jepsen worker 12 - jepsen.util 6852 :invoke :write [4 34] INFO [2019-12-17 12:32:51,348] jepsen worker 5 - jepsen.util 6035 :invoke :write [1 0] INFO [2019-12-17 12:32:51,355] jepsen worker 14 - jepsen.util 6674 :ok :read [4 19] INFO [2019-12-17 12:32:51,357] jepsen worker 14 - jepsen.util 6674 :invoke :read [4 nil] INFO [2019-12-17 12:32:51,402] jepsen worker 51 - jepsen.util 6441 :ok :read [17 19] INFO [2019-12-17 12:32:51,407] jepsen worker 51 - jepsen.util 6441 :invoke :cas [17 [49 33]] INFO [2019-12-17 12:32:51,471] jepsen worker 19 - jepsen.util 6319 :fail :cas [6 [41 40]] :ValueNotSwapFrom INFO [2019-12-17 12:32:51,472] jepsen worker 19 - jepsen.util 6319 :invoke :read [6 nil] INFO [2019-12-17 12:32:51,480] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37393,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:32:51,484] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37393,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:32:51,491] jepsen worker 18 - jepsen.util 6408 :fail :cas [6 [31 22]] :ValueNotSwapFrom INFO [2019-12-17 12:32:51,491] jepsen worker 18 - jepsen.util 6408 :invoke :cas [6 [30 13]] INFO [2019-12-17 12:32:51,534] jepsen worker 72 - jepsen.util 6282 :ok :read [24 33] INFO [2019-12-17 12:32:51,535] jepsen worker 72 - jepsen.util 6282 :invoke :write [24 37] INFO [2019-12-17 12:32:51,550] jepsen worker 10 - jepsen.util 6130 :ok :write [3 6] INFO [2019-12-17 12:32:51,553] jepsen worker 10 - jepsen.util 6130 :invoke :cas [3 [3 13]] INFO [2019-12-17 12:32:51,564] jepsen worker 64 - jepsen.util 5014 :ok :read [21 21] INFO [2019-12-17 12:32:51,575] jepsen worker 64 - jepsen.util 5014 :invoke :cas [21 [17 47]] INFO [2019-12-17 12:32:51,624] jepsen worker 20 - jepsen.util 6230 :ok :write [6 0] INFO [2019-12-17 12:32:51,625] jepsen worker 20 - jepsen.util 6230 :invoke :read [6 nil] INFO [2019-12-17 12:32:51,625] jepsen worker 65 - jepsen.util 5285 :ok :write [21 2] INFO [2019-12-17 12:32:51,628] jepsen worker 65 - jepsen.util 5285 :invoke :read [21 nil] INFO [2019-12-17 12:32:51,698] jepsen worker 8 - jepsen.util 8468 :info :cas [2 [42 21]] :RequestCanceledException INFO [2019-12-17 12:32:51,713] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37403,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:32:51,713] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37403,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:32:51,713] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37407,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:32:51,713] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37407,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:32:51,728] jepsen worker 11 - jepsen.util 6761 :info :write [3 19] :RequestCanceledException INFO [2019-12-17 12:32:51,760] jepsen worker 11 - jepsen.util 6851 :invoke :read [3 nil] INFO [2019-12-17 12:32:51,789] jepsen worker 63 - jepsen.util 5463 :ok :write [21 21] INFO [2019-12-17 12:32:51,790] jepsen worker 63 - jepsen.util 5463 :invoke :cas [21 [46 19]] INFO [2019-12-17 12:32:51,790] jepsen worker 8 - jepsen.util 8558 :invoke :write [2 25] INFO [2019-12-17 12:32:51,927] jepsen worker 13 - jepsen.util 6313 :fail :cas [4 [23 11]] :ValueNotSwapFrom INFO [2019-12-17 12:32:51,928] jepsen worker 34 - jepsen.util 8044 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:32:51,928] jepsen worker 13 - jepsen.util 6313 :invoke :read [4 nil] INFO [2019-12-17 12:32:51,928] jepsen worker 34 - jepsen.util 8044 :invoke :write [11 43] INFO [2019-12-17 12:32:51,961] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37414,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:32:51,961] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37414,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:32:51,961] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37417,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:32:51,962] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37417,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:32:51,966] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37418,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:32:51,981] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37418,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:32:51,993] jepsen worker 36 - jepsen.util 7146 :fail :cas [12 [17 23]] :ValueNotSwapFrom INFO [2019-12-17 12:32:51,993] jepsen worker 36 - jepsen.util 7146 :invoke :cas [12 [24 34]] INFO [2019-12-17 12:32:52,026] jepsen worker 43 - jepsen.util 8053 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:32:52,026] jepsen worker 43 - jepsen.util 8053 :invoke :read [14 nil] INFO [2019-12-17 12:32:52,032] jepsen worker 28 - jepsen.util 4978 :ok :write [9 17] INFO [2019-12-17 12:32:52,033] jepsen worker 28 - jepsen.util 4978 :invoke :read [9 nil] INFO [2019-12-17 12:32:52,033] jepsen worker 75 - jepsen.util 8265 :info :cas [25 [23 19]] :RequestCanceledException INFO [2019-12-17 12:32:52,356] jepsen worker 69 - jepsen.util 6549 :ok :read [23 14] INFO [2019-12-17 12:32:52,356] jepsen worker 70 - jepsen.util 6910 :ok :read [23 14] INFO [2019-12-17 12:32:52,356] jepsen worker 69 - jepsen.util 6549 :invoke :read [23 nil] INFO [2019-12-17 12:32:52,360] jepsen worker 32 - jepsen.util 6692 :ok :read [10 15] INFO [2019-12-17 12:32:52,360] jepsen worker 32 - jepsen.util 6692 :invoke :read [10 nil] INFO [2019-12-17 12:32:52,361] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37431,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:32:52,362] jepsen worker 0 - jepsen.util 9000 :info :cas [0 [4 0]] :RequestCanceledException INFO [2019-12-17 12:32:52,362] jepsen worker 70 - jepsen.util 6910 :invoke :write [23 7] INFO [2019-12-17 12:32:52,362] jepsen worker 29 - jepsen.util 4889 :ok :read [9 48] INFO [2019-12-17 12:32:52,363] jepsen worker 29 - jepsen.util 4889 :invoke :read [9 nil] INFO [2019-12-17 12:32:52,363] jepsen worker 39 - jepsen.util 8499 :info :cas [13 [15 33]] :RequestCanceledException INFO [2019-12-17 12:32:52,364] jepsen worker 68 - jepsen.util 8348 :info :write [22 25] :RequestCanceledException INFO [2019-12-17 12:32:52,365] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37431,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:32:52,365] jepsen worker 61 - jepsen.util 9511 :info :cas [20 [46 6]] :RequestCanceledException INFO [2019-12-17 12:32:52,366] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37425,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:32:52,372] jepsen worker 58 - jepsen.util 9328 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:32:52,372] jepsen worker 15 - jepsen.util 7665 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:32:52,372] jepsen worker 67 - jepsen.util 9067 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:32:52,372] jepsen worker 41 - jepsen.util 9401 :info :write [13 14] :RequestCanceledException INFO [2019-12-17 12:32:52,374] jepsen worker 62 - jepsen.util 8522 :info :cas [20 [18 45]] :RequestCanceledException INFO [2019-12-17 12:32:52,377] jepsen worker 21 - jepsen.util 8301 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:32:52,378] jepsen worker 77 - jepsen.util 9077 :info :cas [25 [7 19]] :RequestCanceledException INFO [2019-12-17 12:32:52,378] jepsen worker 82 - jepsen.util 9352 :info :write [27 45] :RequestCanceledException INFO [2019-12-17 12:32:52,378] jepsen worker 86 - jepsen.util 9176 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:32:52,379] jepsen worker 25 - jepsen.util 9295 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:32:52,380] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37425,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:32:52,380] jepsen worker 58 - jepsen.util 9328 :invoke :read [19 nil] INFO [2019-12-17 12:32:52,381] jepsen worker 15 - jepsen.util 7665 :invoke :write [5 44] INFO [2019-12-17 12:32:52,380] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37437,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:32:52,382] jepsen worker 21 - jepsen.util 8301 :invoke :read [7 nil] INFO [2019-12-17 12:32:52,383] jepsen worker 76 - jepsen.util 9076 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:32:52,383] jepsen worker 67 - jepsen.util 9067 :invoke :write [22 32] INFO [2019-12-17 12:32:52,390] jepsen worker 25 - jepsen.util 9295 :invoke :write [8 37] INFO [2019-12-17 12:32:52,390] jepsen worker 86 - jepsen.util 9176 :invoke :write [28 14] INFO [2019-12-17 12:32:52,407] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37437,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:32:52,416] jepsen worker 76 - jepsen.util 9076 :invoke :cas [25 [34 39]] INFO [2019-12-17 12:32:52,416] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37439,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:32:52,424] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37439,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:32:52,438] jepsen worker 40 - jepsen.util 9220 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:32:52,441] jepsen worker 40 - jepsen.util 9220 :invoke :cas [13 [7 47]] INFO [2019-12-17 12:32:52,448] jepsen worker 1 - jepsen.util 8461 :info :write [0 21] :RequestCanceledException INFO [2019-12-17 12:32:52,468] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:32:52,472] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37442,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:32:52,474] jepsen worker 47 - jepsen.util 6977 :ok :write [15 29] INFO [2019-12-17 12:32:52,476] jepsen worker 75 - jepsen.util 8355 :invoke :write [25 7] INFO [2019-12-17 12:32:52,476] jepsen nemesis - couchbase.util Rebalance check count: 285 INFO [2019-12-17 12:32:52,477] jepsen nemesis - couchbase.util Rebalance stuck count: 266 INFO [2019-12-17 12:32:52,482] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37442,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:32:52,482] jepsen worker 47 - jepsen.util 6977 :invoke :cas [15 [19 47]] INFO [2019-12-17 12:32:52,484] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37447,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:32:52,485] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37451,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:32:52,486] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37447,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:32:52,491] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37451,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:32:52,502] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37453,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:32:52,510] jepsen worker 66 - jepsen.util 8526 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:32:52,512] jepsen worker 66 - jepsen.util 8526 :invoke :write [22 23] INFO [2019-12-17 12:32:52,514] jepsen worker 61 - jepsen.util 9601 :invoke :cas [20 [43 14]] INFO [2019-12-17 12:32:52,515] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37453,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:32:52,530] jepsen worker 23 - jepsen.util 8843 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:32:52,532] jepsen worker 0 - jepsen.util 9090 :invoke :cas [0 [1 2]] INFO [2019-12-17 12:32:52,533] jepsen worker 7 - jepsen.util 9097 :info :cas [2 [16 9]] :RequestCanceledException INFO [2019-12-17 12:32:52,534] jepsen worker 41 - jepsen.util 9491 :invoke :read [13 nil] INFO [2019-12-17 12:32:52,541] jepsen worker 62 - jepsen.util 8612 :invoke :write [20 9] INFO [2019-12-17 12:32:52,549] jepsen worker 85 - jepsen.util 9805 :info :write [28 37] :RequestCanceledException INFO [2019-12-17 12:32:52,559] jepsen worker 23 - jepsen.util 8843 :invoke :write [7 24] INFO [2019-12-17 12:32:52,560] jepsen worker 26 - jepsen.util 9116 :info :cas [8 [7 27]] :RequestCanceledException INFO [2019-12-17 12:32:52,595] jepsen worker 73 - jepsen.util 6013 :info :cas [24 [33 4]] :RequestCanceledException INFO [2019-12-17 12:32:52,599] jepsen worker 82 - jepsen.util 9442 :invoke :write [27 14] INFO [2019-12-17 12:32:52,599] jepsen worker 77 - jepsen.util 9167 :invoke :cas [25 [46 41]] INFO [2019-12-17 12:32:52,601] jepsen worker 39 - jepsen.util 8589 :invoke :write [13 10] INFO [2019-12-17 12:32:52,608] jepsen worker 68 - jepsen.util 8438 :invoke :read [22 nil] INFO [2019-12-17 12:32:52,635] jepsen worker 1 - jepsen.util 8551 :invoke :write [0 48] INFO [2019-12-17 12:32:52,644] jepsen worker 6 - jepsen.util 9366 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:32:52,645] jepsen worker 6 - jepsen.util 9366 :invoke :cas [2 [8 39]] INFO [2019-12-17 12:32:52,660] jepsen worker 49 - jepsen.util 7969 :info :cas [16 [20 19]] :RequestCanceledException INFO [2019-12-17 12:32:52,660] jepsen worker 7 - jepsen.util 9187 :invoke :read [2 nil] INFO [2019-12-17 12:32:52,665] jepsen worker 26 - jepsen.util 9206 :invoke :read [8 nil] INFO [2019-12-17 12:32:52,686] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37464,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:32:52,688] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37464,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:32:52,688] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37469,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:32:52,696] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37469,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:32:52,696] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37470,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:32:52,698] jepsen worker 73 - jepsen.util 6103 :invoke :write [24 24] INFO [2019-12-17 12:32:52,702] jepsen worker 33 - jepsen.util 8583 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:32:52,707] jepsen worker 24 - jepsen.util 7764 :info :cas [8 [16 0]] :RequestCanceledException INFO [2019-12-17 12:32:52,707] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37470,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:32:52,711] jepsen worker 33 - jepsen.util 8583 :invoke :read [11 nil] INFO [2019-12-17 12:32:52,713] jepsen worker 48 - jepsen.util 8148 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:32:52,715] jepsen worker 48 - jepsen.util 8148 :invoke :read [16 nil] INFO [2019-12-17 12:32:52,716] jepsen worker 85 - jepsen.util 9895 :invoke :read [28 nil] INFO [2019-12-17 12:32:52,741] jepsen worker 79 - jepsen.util 6559 :info :write [26 49] :RequestCanceledException INFO [2019-12-17 12:32:52,741] jepsen worker 57 - jepsen.util 8337 :info :cas [19 [27 8]] :RequestCanceledException INFO [2019-12-17 12:32:52,741] jepsen worker 9 - jepsen.util 6939 :info :write [3 35] :RequestCanceledException INFO [2019-12-17 12:32:52,759] jepsen worker 24 - jepsen.util 7854 :invoke :write [8 20] INFO [2019-12-17 12:32:52,761] jepsen worker 60 - jepsen.util 9690 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:32:52,761] jepsen worker 60 - jepsen.util 9690 :invoke :write [20 43] INFO [2019-12-17 12:32:52,765] jepsen worker 49 - jepsen.util 8059 :invoke :read [16 nil] INFO [2019-12-17 12:32:52,766] jepsen worker 79 - jepsen.util 6649 :invoke :read [26 nil] INFO [2019-12-17 12:32:52,776] jepsen worker 57 - jepsen.util 8427 :invoke :write [19 38] INFO [2019-12-17 12:32:52,778] jepsen worker 9 - jepsen.util 7029 :invoke :write [3 44] INFO [2019-12-17 12:32:52,797] jepsen worker 59 - jepsen.util 8969 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:32:52,799] jepsen worker 59 - jepsen.util 8969 :invoke :read [19 nil] INFO [2019-12-17 12:32:52,813] jepsen worker 42 - jepsen.util 9222 :info :cas [14 [35 23]] :RequestCanceledException INFO [2019-12-17 12:32:52,828] jepsen worker 84 - jepsen.util 8184 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:32:52,828] jepsen worker 2 - jepsen.util 8462 :info :cas [0 [30 37]] :RequestCanceledException INFO [2019-12-17 12:32:52,828] jepsen worker 16 - jepsen.util 9736 :info :write [5 13] :RequestCanceledException INFO [2019-12-17 12:32:52,829] jepsen worker 84 - jepsen.util 8184 :invoke :read [28 nil] INFO [2019-12-17 12:32:52,841] jepsen worker 83 - jepsen.util 9173 :info :write [27 18] :RequestCanceledException INFO [2019-12-17 12:32:52,841] jepsen worker 42 - jepsen.util 9312 :invoke :cas [14 [49 37]] INFO [2019-12-17 12:32:52,845] jepsen worker 2 - jepsen.util 8552 :invoke :read [0 nil] INFO [2019-12-17 12:32:52,847] jepsen worker 16 - jepsen.util 9826 :invoke :read [5 nil] INFO [2019-12-17 12:32:52,849] jepsen worker 88 - jepsen.util 6658 :fail :cas [29 [23 15]] :ValueNotSwapFrom INFO [2019-12-17 12:32:52,850] jepsen worker 88 - jepsen.util 6658 :invoke :write [29 47] INFO [2019-12-17 12:32:52,855] jepsen worker 38 - jepsen.util 6518 :fail :cas [12 [2 19]] :ValueNotSwapFrom INFO [2019-12-17 12:32:52,856] jepsen worker 38 - jepsen.util 6518 :invoke :write [12 16] INFO [2019-12-17 12:32:52,859] jepsen worker 53 - jepsen.util 6173 :info :write [17 32] :RequestCanceledException INFO [2019-12-17 12:32:52,860] jepsen worker 52 - jepsen.util 6892 :ok :read [17 19] INFO [2019-12-17 12:32:52,860] jepsen worker 52 - jepsen.util 6892 :invoke :write [17 14] INFO [2019-12-17 12:32:52,861] jepsen worker 56 - jepsen.util 6986 :fail :cas [18 [44 4]] :ValueNotSwapFrom INFO [2019-12-17 12:32:52,862] jepsen worker 83 - jepsen.util 9263 :invoke :read [27 nil] INFO [2019-12-17 12:32:52,862] jepsen worker 56 - jepsen.util 6986 :invoke :cas [18 [22 24]] INFO [2019-12-17 12:32:52,864] jepsen worker 4 - jepsen.util 5944 :ok :read [1 43] INFO [2019-12-17 12:32:52,865] jepsen worker 4 - jepsen.util 5944 :invoke :write [1 6] INFO [2019-12-17 12:32:52,865] jepsen worker 89 - jepsen.util 6209 :ok :read [29 24] INFO [2019-12-17 12:32:52,866] jepsen worker 89 - jepsen.util 6209 :invoke :write [29 36] INFO [2019-12-17 12:32:52,870] jepsen worker 46 - jepsen.util 6436 :ok :read [15 33] INFO [2019-12-17 12:32:52,871] jepsen worker 46 - jepsen.util 6436 :invoke :cas [15 [35 16]] INFO [2019-12-17 12:32:52,872] jepsen worker 53 - jepsen.util 6263 :invoke :cas [17 [36 30]] INFO [2019-12-17 12:32:52,874] jepsen worker 71 - jepsen.util 7091 :fail :cas [23 [41 39]] :ValueNotSwapFrom INFO [2019-12-17 12:32:52,876] jepsen worker 71 - jepsen.util 7091 :invoke :write [23 48] INFO [2019-12-17 12:32:52,888] jepsen worker 54 - jepsen.util 7164 :fail :cas [18 [35 4]] :ValueNotSwapFrom INFO [2019-12-17 12:32:52,889] jepsen worker 54 - jepsen.util 7164 :invoke :write [18 35] INFO [2019-12-17 12:32:52,941] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37486,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:32:52,941] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37486,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:32:52,941] jepsen worker 81 - jepsen.util 8451 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:32:52,952] jepsen worker 81 - jepsen.util 8451 :invoke :cas [27 [17 8]] INFO [2019-12-17 12:32:52,995] jepsen worker 50 - jepsen.util 9320 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:32:52,996] jepsen worker 50 - jepsen.util 9320 :invoke :write [16 3] INFO [2019-12-17 12:32:53,170] jepsen worker 74 - jepsen.util 6644 :fail :cas [24 [7 20]] :ValueNotSwapFrom INFO [2019-12-17 12:32:53,172] jepsen worker 55 - jepsen.util 7525 :fail :cas [18 [19 3]] :CASMismatchException INFO [2019-12-17 12:32:53,172] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37498,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:32:53,172] jepsen worker 14 - jepsen.util 6674 :ok :read [4 19] INFO [2019-12-17 12:32:53,172] jepsen worker 28 - jepsen.util 4978 :ok :read [9 17] INFO [2019-12-17 12:32:53,172] jepsen worker 55 - jepsen.util 7525 :invoke :write [18 9] INFO [2019-12-17 12:32:53,173] jepsen worker 31 - jepsen.util 5791 :fail :cas [10 [37 44]] :ValueNotSwapFrom INFO [2019-12-17 12:32:53,174] jepsen worker 78 - jepsen.util 6828 :fail :cas [26 [40 24]] :ValueNotSwapFrom INFO [2019-12-17 12:32:53,174] jepsen worker 22 - jepsen.util 8842 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:32:53,174] jepsen worker 74 - jepsen.util 6644 :invoke :write [24 39] INFO [2019-12-17 12:32:53,181] jepsen worker 14 - jepsen.util 6674 :invoke :write [4 34] INFO [2019-12-17 12:32:53,183] jepsen worker 22 - jepsen.util 8842 :invoke :read [7 nil] INFO [2019-12-17 12:32:53,183] jepsen worker 28 - jepsen.util 4978 :invoke :cas [9 [46 46]] INFO [2019-12-17 12:32:53,187] jepsen worker 31 - jepsen.util 5791 :invoke :write [10 37] INFO [2019-12-17 12:32:53,192] jepsen worker 78 - jepsen.util 6828 :invoke :write [26 30] INFO [2019-12-17 12:32:53,204] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37498,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:32:53,204] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37502,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:32:53,213] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37502,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:32:53,225] jepsen worker 17 - jepsen.util 8927 :info :write [5 42] :RequestCanceledException INFO [2019-12-17 12:32:53,236] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37530,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:32:53,248] jepsen worker 12 - jepsen.util 6852 :ok :write [4 34] INFO [2019-12-17 12:32:53,250] jepsen worker 12 - jepsen.util 6852 :invoke :write [4 25] INFO [2019-12-17 12:32:53,262] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37530,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:32:53,268] jepsen worker 72 - jepsen.util 6282 :ok :write [24 37] INFO [2019-12-17 12:32:53,270] jepsen worker 72 - jepsen.util 6282 :invoke :cas [24 [31 22]] INFO [2019-12-17 12:32:53,270] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37533,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:32:53,272] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37533,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:32:53,291] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37527,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:32:53,291] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37527,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:32:53,297] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37528,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:32:53,309] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37528,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:32:53,310] jepsen worker 80 - jepsen.util 6380 :info :write [26 23] :RequestCanceledException INFO [2019-12-17 12:32:53,335] jepsen worker 35 - jepsen.util 8675 :info :write [11 49] :RequestCanceledException INFO [2019-12-17 12:32:53,350] jepsen worker 80 - jepsen.util 6470 :invoke :write [26 44] INFO [2019-12-17 12:32:53,414] jepsen worker 17 - jepsen.util 9017 :invoke :read [5 nil] INFO [2019-12-17 12:32:53,420] jepsen worker 35 - jepsen.util 8765 :invoke :read [11 nil] INFO [2019-12-17 12:32:53,430] jepsen worker 44 - jepsen.util 9584 :info :write [14 20] :RequestCanceledException INFO [2019-12-17 12:32:53,503] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37542,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:32:53,516] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37542,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:32:53,564] jepsen worker 19 - jepsen.util 6319 :fail :read [6 nil] :RequestTimeoutException INFO [2019-12-17 12:32:53,565] jepsen worker 19 - jepsen.util 6319 :invoke :write [6 32] INFO [2019-12-17 12:32:53,612] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:32:53,622] jepsen nemesis - couchbase.util Rebalance check count: 286 INFO [2019-12-17 12:32:53,628] jepsen nemesis - couchbase.util Rebalance stuck count: 267 INFO [2019-12-17 12:32:53,664] jepsen worker 44 - jepsen.util 9674 :invoke :write [14 3] INFO [2019-12-17 12:32:53,664] jepsen worker 10 - jepsen.util 6130 :info :cas [3 [3 13]] :RequestCanceledException INFO [2019-12-17 12:32:53,675] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37550,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:32:53,682] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37550,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:32:53,723] jepsen worker 20 - jepsen.util 6230 :fail :read [6 nil] :RequestTimeoutException INFO [2019-12-17 12:32:53,736] jepsen worker 20 - jepsen.util 6230 :invoke :write [6 5] INFO [2019-12-17 12:32:53,786] jepsen worker 10 - jepsen.util 6220 :invoke :write [3 27] INFO [2019-12-17 12:32:53,817] jepsen worker 5 - jepsen.util 6035 :info :write [1 0] :RequestCanceledException INFO [2019-12-17 12:32:53,875] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37558,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:32:53,886] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37558,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:32:53,895] jepsen worker 37 - jepsen.util 6247 :ok :read [12 42] INFO [2019-12-17 12:32:53,897] jepsen worker 37 - jepsen.util 6247 :invoke :write [12 1] INFO [2019-12-17 12:32:53,897] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37560,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:32:53,898] jepsen worker 5 - jepsen.util 6125 :invoke :write [1 37] INFO [2019-12-17 12:32:53,899] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37560,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:32:53,900] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37561,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:32:53,933] jepsen worker 70 - jepsen.util 6910 :ok :write [23 7] INFO [2019-12-17 12:32:53,934] jepsen worker 70 - jepsen.util 6910 :invoke :read [23 nil] INFO [2019-12-17 12:32:53,937] jepsen worker 87 - jepsen.util 5757 :ok :read [29 24] INFO [2019-12-17 12:32:53,938] jepsen worker 87 - jepsen.util 5757 :invoke :read [29 nil] INFO [2019-12-17 12:32:53,942] jepsen worker 30 - jepsen.util 6330 :fail :cas [10 [12 22]] :ValueNotSwapFrom INFO [2019-12-17 12:32:53,943] jepsen worker 30 - jepsen.util 6330 :invoke :write [10 27] INFO [2019-12-17 12:32:53,948] jepsen worker 11 - jepsen.util 6851 :ok :read [3 6] INFO [2019-12-17 12:32:53,949] jepsen worker 11 - jepsen.util 6851 :invoke :read [3 nil] INFO [2019-12-17 12:32:53,951] jepsen worker 27 - jepsen.util 6507 :fail :cas [9 [13 44]] :ValueNotSwapFrom INFO [2019-12-17 12:32:53,951] jepsen worker 65 - jepsen.util 5285 :ok :read [21 2] INFO [2019-12-17 12:32:53,951] jepsen worker 27 - jepsen.util 6507 :invoke :read [9 nil] INFO [2019-12-17 12:32:53,951] jepsen worker 47 - jepsen.util 6977 :fail :cas [15 [19 47]] :ValueNotSwapFrom INFO [2019-12-17 12:32:53,951] jepsen worker 65 - jepsen.util 5285 :invoke :write [21 7] INFO [2019-12-17 12:32:53,951] jepsen worker 47 - jepsen.util 6977 :invoke :write [15 41] INFO [2019-12-17 12:32:53,954] jepsen worker 18 - jepsen.util 6408 :fail :cas [6 [30 13]] :ValueNotSwapFrom INFO [2019-12-17 12:32:53,954] jepsen worker 18 - jepsen.util 6408 :invoke :write [6 28] INFO [2019-12-17 12:32:53,954] jepsen worker 29 - jepsen.util 4889 :ok :read [9 17] INFO [2019-12-17 12:32:53,955] jepsen worker 29 - jepsen.util 4889 :invoke :read [9 nil] INFO [2019-12-17 12:32:53,955] jepsen worker 64 - jepsen.util 5014 :fail :cas [21 [17 47]] :ValueNotSwapFrom INFO [2019-12-17 12:32:53,955] jepsen worker 63 - jepsen.util 5463 :fail :cas [21 [46 19]] :ValueNotSwapFrom INFO [2019-12-17 12:32:53,955] jepsen worker 64 - jepsen.util 5014 :invoke :cas [21 [23 7]] INFO [2019-12-17 12:32:53,955] jepsen worker 63 - jepsen.util 5463 :invoke :cas [21 [12 46]] INFO [2019-12-17 12:32:53,971] jepsen worker 45 - jepsen.util 6345 :ok :read [15 33] INFO [2019-12-17 12:32:53,972] jepsen worker 45 - jepsen.util 6345 :invoke :write [15 39] INFO [2019-12-17 12:32:54,005] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37561,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:32:54,107] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37574,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:32:54,109] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37574,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:32:54,110] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37578,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:32:54,131] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37578,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:32:54,134] jepsen worker 43 - jepsen.util 8053 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:32:54,135] jepsen worker 43 - jepsen.util 8053 :invoke :write [14 40] INFO [2019-12-17 12:32:54,190] jepsen worker 69 - jepsen.util 6549 :ok :read [23 14] INFO [2019-12-17 12:32:54,190] jepsen worker 69 - jepsen.util 6549 :invoke :write [23 41] INFO [2019-12-17 12:32:54,239] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37594,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:32:54,248] jepsen worker 9 - jepsen.util 7029 :ok :write [3 44] INFO [2019-12-17 12:32:54,248] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37594,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:32:54,249] jepsen worker 9 - jepsen.util 7029 :invoke :read [3 nil] INFO [2019-12-17 12:32:54,255] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37598,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:32:54,257] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37598,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:32:54,270] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37586,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:32:54,270] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37586,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:32:54,271] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37591,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:32:54,272] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37591,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:32:54,297] jepsen worker 73 - jepsen.util 6103 :ok :write [24 24] INFO [2019-12-17 12:32:54,298] jepsen worker 73 - jepsen.util 6103 :invoke :write [24 27] INFO [2019-12-17 12:32:54,367] jepsen worker 3 - jepsen.util 6393 :fail :cas [1 [3 33]] :ValueNotSwapFrom INFO [2019-12-17 12:32:54,368] jepsen worker 3 - jepsen.util 6393 :invoke :write [1 11] INFO [2019-12-17 12:32:54,411] jepsen worker 51 - jepsen.util 6441 :fail :cas [17 [49 33]] :ValueNotSwapFrom INFO [2019-12-17 12:32:54,411] jepsen worker 51 - jepsen.util 6441 :invoke :write [17 39] INFO [2019-12-17 12:32:54,427] jepsen worker 76 - jepsen.util 9076 :info :cas [25 [34 39]] :RequestCanceledException INFO [2019-12-17 12:32:54,430] jepsen worker 58 - jepsen.util 9328 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:32:54,431] jepsen worker 58 - jepsen.util 9328 :invoke :cas [19 [41 18]] INFO [2019-12-17 12:32:54,443] jepsen worker 8 - jepsen.util 8558 :info :write [2 25] :RequestCanceledException INFO [2019-12-17 12:32:54,443] jepsen worker 34 - jepsen.util 8044 :info :write [11 43] :RequestCanceledException INFO [2019-12-17 12:32:54,454] jepsen worker 76 - jepsen.util 9166 :invoke :read [25 nil] INFO [2019-12-17 12:32:54,457] jepsen worker 21 - jepsen.util 8301 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:32:54,457] jepsen worker 21 - jepsen.util 8301 :invoke :write [7 42] INFO [2019-12-17 12:32:54,474] jepsen worker 88 - jepsen.util 6658 :ok :write [29 47] INFO [2019-12-17 12:32:54,474] jepsen worker 88 - jepsen.util 6658 :invoke :read [29 nil] INFO [2019-12-17 12:32:54,474] jepsen worker 8 - jepsen.util 8648 :invoke :write [2 34] INFO [2019-12-17 12:32:54,481] jepsen worker 34 - jepsen.util 8134 :invoke :cas [11 [28 16]] INFO [2019-12-17 12:32:54,488] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37605,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:32:54,489] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37605,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:32:54,489] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37610,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:32:54,490] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37610,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:32:54,534] jepsen worker 40 - jepsen.util 9220 :info :cas [13 [7 47]] :RequestCanceledException INFO [2019-12-17 12:32:54,540] jepsen worker 61 - jepsen.util 9601 :info :cas [20 [43 14]] :RequestCanceledException INFO [2019-12-17 12:32:54,570] jepsen worker 38 - jepsen.util 6518 :ok :write [12 16] INFO [2019-12-17 12:32:54,570] jepsen worker 38 - jepsen.util 6518 :invoke :write [12 25] INFO [2019-12-17 12:32:54,586] jepsen worker 46 - jepsen.util 6436 :fail :cas [15 [35 16]] :ValueNotSwapFrom INFO [2019-12-17 12:32:54,587] jepsen worker 46 - jepsen.util 6436 :invoke :cas [15 [22 29]] INFO [2019-12-17 12:32:54,587] jepsen worker 61 - jepsen.util 9691 :invoke :cas [20 [38 10]] INFO [2019-12-17 12:32:54,597] jepsen worker 0 - jepsen.util 9090 :info :cas [0 [1 2]] :RequestCanceledException INFO [2019-12-17 12:32:54,621] jepsen worker 74 - jepsen.util 6644 :ok :write [24 39] INFO [2019-12-17 12:32:54,621] jepsen worker 74 - jepsen.util 6644 :invoke :read [24 nil] INFO [2019-12-17 12:32:54,650] jepsen worker 31 - jepsen.util 5791 :ok :write [10 37] INFO [2019-12-17 12:32:54,650] jepsen worker 31 - jepsen.util 5791 :invoke :write [10 42] INFO [2019-12-17 12:32:54,651] jepsen worker 14 - jepsen.util 6674 :ok :write [4 34] INFO [2019-12-17 12:32:54,652] jepsen worker 68 - jepsen.util 8438 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:32:54,654] jepsen worker 41 - jepsen.util 9491 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:32:54,660] jepsen worker 68 - jepsen.util 8438 :invoke :write [22 0] INFO [2019-12-17 12:32:54,667] jepsen worker 14 - jepsen.util 6674 :invoke :write [4 6] INFO [2019-12-17 12:32:54,668] jepsen worker 40 - jepsen.util 9310 :invoke :read [13 nil] INFO [2019-12-17 12:32:54,668] jepsen worker 77 - jepsen.util 9167 :info :cas [25 [46 41]] :RequestCanceledException INFO [2019-12-17 12:32:54,670] jepsen worker 41 - jepsen.util 9491 :invoke :read [13 nil] INFO [2019-12-17 12:32:54,699] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37620,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:32:54,732] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37620,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:32:54,760] jepsen worker 7 - jepsen.util 9187 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:32:54,770] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37624,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:32:54,772] jepsen worker 85 - jepsen.util 9895 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:32:54,776] jepsen worker 7 - jepsen.util 9187 :invoke :write [2 7] INFO [2019-12-17 12:32:54,777] jepsen worker 6 - jepsen.util 9366 :info :cas [2 [8 39]] :RequestCanceledException INFO [2019-12-17 12:32:54,807] jepsen worker 26 - jepsen.util 9206 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:32:54,843] jepsen worker 33 - jepsen.util 8583 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:32:54,843] jepsen worker 33 - jepsen.util 8583 :invoke :cas [11 [26 41]] INFO [2019-12-17 12:32:54,864] jepsen worker 48 - jepsen.util 8148 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:32:54,888] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37624,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:32:54,896] jepsen worker 59 - jepsen.util 8969 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:32:54,899] jepsen worker 85 - jepsen.util 9895 :invoke :write [28 45] INFO [2019-12-17 12:32:54,905] jepsen worker 49 - jepsen.util 8059 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:32:54,906] jepsen worker 26 - jepsen.util 9206 :invoke :cas [8 [36 14]] INFO [2019-12-17 12:32:54,907] jepsen worker 2 - jepsen.util 8552 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:32:54,912] jepsen worker 67 - jepsen.util 9067 :info :write [22 32] :RequestCanceledException INFO [2019-12-17 12:32:54,915] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37630,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:32:54,917] jepsen worker 48 - jepsen.util 8148 :invoke :read [16 nil] INFO [2019-12-17 12:32:54,917] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:32:54,917] jepsen worker 16 - jepsen.util 9826 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:32:54,914] jepsen worker 0 - jepsen.util 9180 :invoke :read [0 nil] INFO [2019-12-17 12:32:54,921] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37625,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:32:54,922] jepsen worker 2 - jepsen.util 8552 :invoke :read [0 nil] INFO [2019-12-17 12:32:54,935] jepsen worker 59 - jepsen.util 8969 :invoke :cas [19 [39 33]] INFO [2019-12-17 12:32:54,943] jepsen worker 49 - jepsen.util 8059 :invoke :read [16 nil] INFO [2019-12-17 12:32:54,944] jepsen worker 83 - jepsen.util 9263 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:32:54,945] jepsen worker 84 - jepsen.util 8184 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:32:54,946] jepsen worker 53 - jepsen.util 6263 :info :cas [17 [36 30]] :RequestCanceledException INFO [2019-12-17 12:32:54,947] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37630,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:32:54,949] jepsen nemesis - couchbase.util Rebalance check count: 287 INFO [2019-12-17 12:32:54,951] jepsen worker 42 - jepsen.util 9312 :info :cas [14 [49 37]] :RequestCanceledException INFO [2019-12-17 12:32:54,958] jepsen worker 16 - jepsen.util 9826 :invoke :read [5 nil] INFO [2019-12-17 12:32:54,960] jepsen worker 25 - jepsen.util 9295 :info :write [8 37] :RequestCanceledException INFO [2019-12-17 12:32:54,962] jepsen worker 84 - jepsen.util 8184 :invoke :cas [28 [1 9]] INFO [2019-12-17 12:32:54,962] jepsen worker 83 - jepsen.util 9263 :invoke :cas [27 [43 17]] INFO [2019-12-17 12:32:54,963] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37634,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:32:54,964] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37625,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:32:54,964] jepsen worker 77 - jepsen.util 9257 :invoke :read [25 nil] INFO [2019-12-17 12:32:54,964] jepsen nemesis - couchbase.util Rebalance stuck count: 268 INFO [2019-12-17 12:32:54,967] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37634,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:32:55,029] jepsen worker 30 - jepsen.util 6330 :ok :write [10 27] INFO [2019-12-17 12:32:55,029] jepsen worker 30 - jepsen.util 6330 :invoke :read [10 nil] INFO [2019-12-17 12:32:55,060] jepsen worker 65 - jepsen.util 5285 :ok :write [21 7] INFO [2019-12-17 12:32:55,060] jepsen worker 65 - jepsen.util 5285 :invoke :read [21 nil] INFO [2019-12-17 12:32:55,074] jepsen worker 60 - jepsen.util 9690 :info :write [20 43] :RequestCanceledException INFO [2019-12-17 12:32:55,088] jepsen worker 81 - jepsen.util 8451 :info :cas [27 [17 8]] :RequestCanceledException INFO [2019-12-17 12:32:55,108] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37645,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:32:55,146] jepsen worker 6 - jepsen.util 9456 :invoke :read [2 nil] INFO [2019-12-17 12:32:55,170] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37645,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:32:55,171] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37651,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:32:55,192] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37651,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:32:55,224] jepsen worker 53 - jepsen.util 6353 :invoke :read [17 nil] INFO [2019-12-17 12:32:55,225] jepsen worker 66 - jepsen.util 8526 :info :write [22 23] :RequestCanceledException INFO [2019-12-17 12:32:55,231] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37661,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:32:55,232] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37661,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:32:55,232] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37659,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:32:55,236] jepsen worker 15 - jepsen.util 7665 :info :write [5 44] :RequestCanceledException INFO [2019-12-17 12:32:55,239] jepsen worker 1 - jepsen.util 8551 :info :write [0 48] :RequestCanceledException INFO [2019-12-17 12:32:55,254] jepsen worker 89 - jepsen.util 6209 :info :write [29 36] :RequestCanceledException INFO [2019-12-17 12:32:55,255] jepsen worker 75 - jepsen.util 8355 :info :write [25 7] :RequestCanceledException INFO [2019-12-17 12:32:55,270] jepsen worker 62 - jepsen.util 8612 :info :write [20 9] :RequestCanceledException INFO [2019-12-17 12:32:55,273] jepsen worker 39 - jepsen.util 8589 :info :write [13 10] :RequestCanceledException INFO [2019-12-17 12:32:55,276] jepsen worker 79 - jepsen.util 6649 :ok :read [26 49] INFO [2019-12-17 12:32:55,291] jepsen worker 22 - jepsen.util 8842 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:32:55,292] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37659,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:32:55,292] jepsen worker 86 - jepsen.util 9176 :info :write [28 14] :RequestCanceledException INFO [2019-12-17 12:32:55,292] jepsen worker 42 - jepsen.util 9402 :invoke :write [14 12] INFO [2019-12-17 12:32:55,292] jepsen worker 24 - jepsen.util 7854 :info :write [8 20] :RequestCanceledException INFO [2019-12-17 12:32:55,294] jepsen worker 4 - jepsen.util 5944 :info :write [1 6] :RequestCanceledException INFO [2019-12-17 12:32:55,294] jepsen worker 23 - jepsen.util 8843 :info :write [7 24] :RequestCanceledException INFO [2019-12-17 12:32:55,335] jepsen worker 57 - jepsen.util 8427 :info :write [19 38] :RequestCanceledException INFO [2019-12-17 12:32:55,346] jepsen worker 82 - jepsen.util 9442 :info :write [27 14] :RequestCanceledException INFO [2019-12-17 12:32:55,350] jepsen worker 52 - jepsen.util 6892 :ok :write [17 14] INFO [2019-12-17 12:32:55,359] jepsen worker 52 - jepsen.util 6892 :invoke :cas [17 [43 25]] INFO [2019-12-17 12:32:55,366] jepsen worker 22 - jepsen.util 8842 :invoke :cas [7 [4 46]] INFO [2019-12-17 12:32:55,372] jepsen worker 79 - jepsen.util 6649 :invoke :cas [26 [7 10]] INFO [2019-12-17 12:32:55,399] jepsen worker 54 - jepsen.util 7164 :ok :write [18 35] INFO [2019-12-17 12:32:55,419] jepsen worker 54 - jepsen.util 7164 :invoke :write [18 25] INFO [2019-12-17 12:32:55,421] jepsen worker 67 - jepsen.util 9157 :invoke :write [22 4] INFO [2019-12-17 12:32:55,435] jepsen worker 25 - jepsen.util 9385 :invoke :cas [8 [2 47]] INFO [2019-12-17 12:32:55,443] jepsen worker 17 - jepsen.util 9017 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:32:55,443] jepsen worker 17 - jepsen.util 9017 :invoke :cas [5 [14 15]] INFO [2019-12-17 12:32:55,463] jepsen worker 71 - jepsen.util 7091 :info :write [23 48] :RequestCanceledException INFO [2019-12-17 12:32:55,478] jepsen worker 71 - jepsen.util 7181 :invoke :cas [23 [33 35]] INFO [2019-12-17 12:32:55,487] jepsen worker 60 - jepsen.util 9780 :invoke :read [20 nil] INFO [2019-12-17 12:32:55,489] jepsen worker 39 - jepsen.util 8679 :invoke :write [13 12] INFO [2019-12-17 12:32:55,491] jepsen worker 81 - jepsen.util 8541 :invoke :cas [27 [26 41]] INFO [2019-12-17 12:32:55,499] jepsen worker 35 - jepsen.util 8765 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:32:55,500] jepsen worker 35 - jepsen.util 8765 :invoke :read [11 nil] INFO [2019-12-17 12:32:55,510] jepsen worker 66 - jepsen.util 8616 :invoke :cas [22 [1 42]] INFO [2019-12-17 12:32:55,516] jepsen worker 57 - jepsen.util 8517 :invoke :write [19 29] INFO [2019-12-17 12:32:55,519] jepsen worker 62 - jepsen.util 8702 :invoke :cas [20 [38 27]] INFO [2019-12-17 12:32:55,520] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37672,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:32:55,521] jepsen worker 75 - jepsen.util 8445 :invoke :write [25 10] INFO [2019-12-17 12:32:55,521] jepsen worker 86 - jepsen.util 9266 :invoke :cas [28 [5 5]] INFO [2019-12-17 12:32:55,533] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37672,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:32:55,534] jepsen worker 1 - jepsen.util 8641 :invoke :write [0 29] INFO [2019-12-17 12:32:55,537] jepsen worker 4 - jepsen.util 6034 :invoke :read [1 nil] INFO [2019-12-17 12:32:55,538] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37677,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:32:55,548] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37677,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:32:55,552] jepsen worker 24 - jepsen.util 7944 :invoke :write [8 10] INFO [2019-12-17 12:32:55,555] jepsen worker 82 - jepsen.util 9532 :invoke :write [27 17] INFO [2019-12-17 12:32:55,565] jepsen worker 89 - jepsen.util 6299 :invoke :write [29 33] INFO [2019-12-17 12:32:55,566] jepsen worker 80 - jepsen.util 6470 :info :write [26 44] :RequestCanceledException INFO [2019-12-17 12:32:55,572] jepsen worker 23 - jepsen.util 8933 :invoke :read [7 nil] INFO [2019-12-17 12:32:55,589] jepsen worker 15 - jepsen.util 7755 :invoke :write [5 47] INFO [2019-12-17 12:32:55,624] jepsen worker 36 - jepsen.util 7146 :fail :cas [12 [24 34]] :ValueNotSwapFrom INFO [2019-12-17 12:32:55,625] jepsen worker 36 - jepsen.util 7146 :invoke :read [12 nil] INFO [2019-12-17 12:32:55,629] jepsen worker 28 - jepsen.util 4978 :fail :cas [9 [46 46]] :ValueNotSwapFrom INFO [2019-12-17 12:32:55,630] jepsen worker 28 - jepsen.util 4978 :invoke :cas [9 [48 3]] INFO [2019-12-17 12:32:55,650] jepsen worker 80 - jepsen.util 6560 :invoke :cas [26 [21 27]] INFO [2019-12-17 12:32:55,655] jepsen worker 56 - jepsen.util 6986 :fail :cas [18 [22 24]] :ValueNotSwapFrom INFO [2019-12-17 12:32:55,656] jepsen worker 56 - jepsen.util 6986 :invoke :cas [18 [45 17]] INFO [2019-12-17 12:32:55,659] jepsen worker 12 - jepsen.util 6852 :info :write [4 25] :RequestCanceledException INFO [2019-12-17 12:32:55,672] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37694,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:32:55,690] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37694,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:32:55,696] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37698,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:32:55,701] jepsen worker 50 - jepsen.util 9320 :info :write [16 3] :RequestCanceledException INFO [2019-12-17 12:32:55,702] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37698,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:32:55,755] jepsen worker 12 - jepsen.util 6942 :invoke :cas [4 [17 16]] INFO [2019-12-17 12:32:55,764] jepsen worker 69 - jepsen.util 6549 :ok :write [23 41] INFO [2019-12-17 12:32:55,769] jepsen worker 69 - jepsen.util 6549 :invoke :cas [23 [31 2]] INFO [2019-12-17 12:32:55,837] jepsen worker 50 - jepsen.util 9410 :invoke :write [16 3] INFO [2019-12-17 12:32:55,845] jepsen worker 10 - jepsen.util 6220 :ok :write [3 27] INFO [2019-12-17 12:32:55,846] jepsen worker 10 - jepsen.util 6220 :invoke :write [3 33] INFO [2019-12-17 12:32:55,969] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37710,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:32:55,969] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37710,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:32:55,969] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37712,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:32:55,970] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37712,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:32:55,970] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37713,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:32:55,970] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37713,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:32:55,976] jepsen worker 32 - jepsen.util 6692 :ok :read [10 15] INFO [2019-12-17 12:32:55,976] jepsen worker 32 - jepsen.util 6692 :invoke :read [10 nil] INFO [2019-12-17 12:32:55,981] jepsen worker 27 - jepsen.util 6507 :ok :read [9 17] INFO [2019-12-17 12:32:55,981] jepsen worker 27 - jepsen.util 6507 :invoke :cas [9 [39 29]] INFO [2019-12-17 12:32:55,981] jepsen worker 64 - jepsen.util 5014 :fail :cas [21 [23 7]] :ValueNotSwapFrom INFO [2019-12-17 12:32:55,982] jepsen worker 64 - jepsen.util 5014 :invoke :read [21 nil] INFO [2019-12-17 12:32:55,987] jepsen worker 13 - jepsen.util 6313 :ok :read [4 34] INFO [2019-12-17 12:32:55,988] jepsen worker 13 - jepsen.util 6313 :invoke :write [4 1] INFO [2019-12-17 12:32:56,009] jepsen worker 70 - jepsen.util 6910 :ok :read [23 7] INFO [2019-12-17 12:32:56,010] jepsen worker 70 - jepsen.util 6910 :invoke :read [23 nil] INFO [2019-12-17 12:32:56,018] jepsen worker 55 - jepsen.util 7525 :info :write [18 9] :RequestCanceledException INFO [2019-12-17 12:32:56,018] jepsen worker 78 - jepsen.util 6828 :info :write [26 30] :RequestCanceledException INFO [2019-12-17 12:32:56,021] jepsen worker 55 - jepsen.util 7615 :invoke :cas [18 [43 3]] INFO [2019-12-17 12:32:56,021] jepsen worker 78 - jepsen.util 6918 :invoke :write [26 17] INFO [2019-12-17 12:32:56,045] jepsen worker 72 - jepsen.util 6282 :fail :cas [24 [31 22]] :ValueNotSwapFrom INFO [2019-12-17 12:32:56,047] jepsen worker 72 - jepsen.util 6282 :invoke :write [24 32] INFO [2019-12-17 12:32:56,057] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37732,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:32:56,057] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37732,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:32:56,060] jepsen worker 87 - jepsen.util 5757 :ok :read [29 36] INFO [2019-12-17 12:32:56,061] jepsen worker 9 - jepsen.util 7029 :ok :read [3 44] INFO [2019-12-17 12:32:56,061] jepsen worker 87 - jepsen.util 5757 :invoke :read [29 nil] INFO [2019-12-17 12:32:56,061] jepsen worker 9 - jepsen.util 7029 :invoke :cas [3 [20 19]] INFO [2019-12-17 12:32:56,087] jepsen worker 11 - jepsen.util 6851 :ok :read [3 44] INFO [2019-12-17 12:32:56,088] jepsen worker 11 - jepsen.util 6851 :invoke :write [3 40] INFO [2019-12-17 12:32:56,089] jepsen worker 63 - jepsen.util 5463 :fail :cas [21 [12 46]] :ValueNotSwapFrom INFO [2019-12-17 12:32:56,090] jepsen worker 63 - jepsen.util 5463 :invoke :write [21 24] INFO [2019-12-17 12:32:56,094] jepsen worker 74 - jepsen.util 6644 :ok :read [24 39] INFO [2019-12-17 12:32:56,094] jepsen worker 74 - jepsen.util 6644 :invoke :cas [24 [40 0]] INFO [2019-12-17 12:32:56,094] jepsen worker 65 - jepsen.util 5285 :ok :read [21 7] INFO [2019-12-17 12:32:56,095] jepsen worker 65 - jepsen.util 5285 :invoke :read [21 nil] INFO [2019-12-17 12:32:56,097] jepsen worker 52 - jepsen.util 6892 :fail :cas [17 [43 25]] :ValueNotSwapFrom INFO [2019-12-17 12:32:56,098] jepsen worker 52 - jepsen.util 6892 :invoke :cas [17 [2 37]] INFO [2019-12-17 12:32:56,099] jepsen worker 30 - jepsen.util 6330 :ok :read [10 27] INFO [2019-12-17 12:32:56,099] jepsen worker 30 - jepsen.util 6330 :invoke :read [10 nil] INFO [2019-12-17 12:32:56,101] jepsen worker 29 - jepsen.util 4889 :ok :read [9 17] INFO [2019-12-17 12:32:56,101] jepsen worker 29 - jepsen.util 4889 :invoke :cas [9 [38 44]] INFO [2019-12-17 12:32:56,111] jepsen worker 44 - jepsen.util 9674 :info :write [14 3] :RequestCanceledException INFO [2019-12-17 12:32:56,112] jepsen worker 20 - jepsen.util 6230 :info :write [6 5] :RequestCanceledException INFO [2019-12-17 12:32:56,116] jepsen worker 20 - jepsen.util 6320 :invoke :read [6 nil] INFO [2019-12-17 12:32:56,123] jepsen worker 44 - jepsen.util 9764 :invoke :read [14 nil] INFO [2019-12-17 12:32:56,142] jepsen worker 5 - jepsen.util 6125 :info :write [1 37] :RequestCanceledException INFO [2019-12-17 12:32:56,146] jepsen worker 88 - jepsen.util 6658 :ok :read [29 47] INFO [2019-12-17 12:32:56,147] jepsen worker 88 - jepsen.util 6658 :invoke :read [29 nil] INFO [2019-12-17 12:32:56,153] jepsen worker 56 - jepsen.util 6986 :fail :cas [18 [45 17]] :ValueNotSwapFrom INFO [2019-12-17 12:32:56,154] jepsen worker 56 - jepsen.util 6986 :invoke :write [18 0] INFO [2019-12-17 12:32:56,156] jepsen worker 37 - jepsen.util 6247 :info :write [12 1] :RequestCanceledException INFO [2019-12-17 12:32:56,159] jepsen worker 37 - jepsen.util 6337 :invoke :cas [12 [26 0]] INFO [2019-12-17 12:32:56,163] jepsen worker 53 - jepsen.util 6353 :ok :read [17 14] INFO [2019-12-17 12:32:56,163] jepsen worker 53 - jepsen.util 6353 :invoke :write [17 26] INFO [2019-12-17 12:32:56,164] jepsen worker 5 - jepsen.util 6215 :invoke :cas [1 [25 30]] INFO [2019-12-17 12:32:56,204] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:32:56,205] jepsen nemesis - couchbase.util Rebalance check count: 288 INFO [2019-12-17 12:32:56,206] jepsen nemesis - couchbase.util Rebalance stuck count: 269 INFO [2019-12-17 12:32:56,217] jepsen worker 19 - jepsen.util 6319 :info :write [6 32] :RequestCanceledException INFO [2019-12-17 12:32:56,245] jepsen worker 47 - jepsen.util 6977 :info :write [15 41] :RequestCanceledException INFO [2019-12-17 12:32:56,256] jepsen worker 19 - jepsen.util 6409 :invoke :write [6 33] INFO [2019-12-17 12:32:56,259] jepsen worker 14 - jepsen.util 6674 :ok :write [4 6] INFO [2019-12-17 12:32:56,260] jepsen worker 14 - jepsen.util 6674 :invoke :write [4 24] INFO [2019-12-17 12:32:56,277] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37756,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:32:56,283] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37756,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:32:56,290] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37757,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:32:56,290] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37757,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:32:56,290] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37758,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:32:56,290] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37758,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:32:56,309] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37764,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:32:56,310] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37764,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:32:56,310] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37768,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:32:56,310] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37768,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:32:56,321] jepsen worker 47 - jepsen.util 7067 :invoke :read [15 nil] INFO [2019-12-17 12:32:56,344] jepsen worker 45 - jepsen.util 6345 :info :write [15 39] :RequestCanceledException INFO [2019-12-17 12:32:56,373] jepsen worker 45 - jepsen.util 6435 :invoke :write [15 12] INFO [2019-12-17 12:32:56,433] jepsen worker 18 - jepsen.util 6408 :info :write [6 28] :RequestCanceledException INFO [2019-12-17 12:32:56,453] jepsen worker 18 - jepsen.util 6498 :invoke :cas [6 [3 3]] INFO [2019-12-17 12:32:56,454] jepsen worker 73 - jepsen.util 6103 :ok :write [24 27] INFO [2019-12-17 12:32:56,454] jepsen worker 73 - jepsen.util 6103 :invoke :cas [24 [25 23]] INFO [2019-12-17 12:32:56,460] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37772,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:32:56,460] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37772,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:32:56,466] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37776,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:32:56,470] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37776,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:32:56,475] jepsen worker 51 - jepsen.util 6441 :ok :write [17 39] INFO [2019-12-17 12:32:56,475] jepsen worker 51 - jepsen.util 6441 :invoke :read [17 nil] INFO [2019-12-17 12:32:56,494] jepsen worker 34 - jepsen.util 8134 :info :cas [11 [28 16]] :RequestCanceledException INFO [2019-12-17 12:32:56,499] jepsen worker 34 - jepsen.util 8224 :invoke :cas [11 [1 47]] INFO [2019-12-17 12:32:56,512] jepsen worker 43 - jepsen.util 8053 :info :write [14 40] :RequestCanceledException INFO [2019-12-17 12:32:56,519] jepsen worker 43 - jepsen.util 8143 :invoke :write [14 26] INFO [2019-12-17 12:32:56,528] jepsen worker 58 - jepsen.util 9328 :info :cas [19 [41 18]] :RequestCanceledException INFO [2019-12-17 12:32:56,531] jepsen worker 58 - jepsen.util 9418 :invoke :read [19 nil] INFO [2019-12-17 12:32:56,555] jepsen worker 76 - jepsen.util 9166 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:32:56,593] jepsen worker 31 - jepsen.util 5791 :ok :write [10 42] INFO [2019-12-17 12:32:56,596] jepsen worker 76 - jepsen.util 9166 :invoke :write [25 1] INFO [2019-12-17 12:32:56,608] jepsen worker 31 - jepsen.util 5791 :invoke :cas [10 [39 19]] INFO [2019-12-17 12:32:56,623] jepsen worker 61 - jepsen.util 9691 :info :cas [20 [38 10]] :RequestCanceledException INFO [2019-12-17 12:32:56,631] jepsen worker 46 - jepsen.util 6436 :info :cas [15 [22 29]] :RequestCanceledException INFO [2019-12-17 12:32:56,678] jepsen worker 38 - jepsen.util 6518 :ok :write [12 25] INFO [2019-12-17 12:32:56,680] jepsen worker 38 - jepsen.util 6518 :invoke :cas [12 [22 9]] INFO [2019-12-17 12:32:56,710] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37788,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:32:56,731] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37788,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:32:56,731] jepsen worker 78 - jepsen.util 6918 :ok :write [26 17] INFO [2019-12-17 12:32:56,732] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37791,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:32:56,732] jepsen worker 40 - jepsen.util 9310 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:32:56,733] jepsen worker 78 - jepsen.util 6918 :invoke :read [26 nil] INFO [2019-12-17 12:32:56,737] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37791,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:32:56,738] jepsen worker 40 - jepsen.util 9310 :invoke :write [13 32] INFO [2019-12-17 12:32:56,739] jepsen worker 61 - jepsen.util 9781 :invoke :cas [20 [27 0]] INFO [2019-12-17 12:32:56,740] jepsen worker 41 - jepsen.util 9491 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:32:56,741] jepsen worker 41 - jepsen.util 9491 :invoke :read [13 nil] INFO [2019-12-17 12:32:56,748] jepsen worker 69 - jepsen.util 6549 :fail :cas [23 [31 2]] :ValueNotSwapFrom INFO [2019-12-17 12:32:56,752] jepsen worker 69 - jepsen.util 6549 :invoke :write [23 15] INFO [2019-12-17 12:32:56,899] jepsen worker 46 - jepsen.util 6526 :invoke :write [15 20] INFO [2019-12-17 12:32:56,912] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37801,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:32:56,928] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37801,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:32:56,945] jepsen worker 26 - jepsen.util 9206 :info :cas [8 [36 14]] :RequestCanceledException INFO [2019-12-17 12:32:57,006] jepsen worker 33 - jepsen.util 8583 :info :cas [11 [26 41]] :RequestCanceledException INFO [2019-12-17 12:32:57,008] jepsen worker 59 - jepsen.util 8969 :info :cas [19 [39 33]] :RequestCanceledException INFO [2019-12-17 12:32:57,030] jepsen worker 48 - jepsen.util 8148 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:32:57,039] jepsen worker 49 - jepsen.util 8059 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:32:57,043] jepsen worker 48 - jepsen.util 8148 :invoke :write [16 26] INFO [2019-12-17 12:32:57,045] jepsen worker 49 - jepsen.util 8059 :invoke :write [16 6] INFO [2019-12-17 12:32:57,056] jepsen worker 2 - jepsen.util 8552 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:32:57,084] jepsen worker 83 - jepsen.util 9263 :info :cas [27 [43 17]] :RequestCanceledException INFO [2019-12-17 12:32:57,090] jepsen worker 77 - jepsen.util 9257 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:32:57,090] jepsen worker 77 - jepsen.util 9257 :invoke :read [25 nil] INFO [2019-12-17 12:32:57,099] jepsen worker 16 - jepsen.util 9826 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:32:57,104] jepsen worker 16 - jepsen.util 9826 :invoke :cas [5 [6 34]] INFO [2019-12-17 12:32:57,106] jepsen worker 2 - jepsen.util 8552 :invoke :write [0 17] INFO [2019-12-17 12:32:57,110] jepsen worker 0 - jepsen.util 9180 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:32:57,121] jepsen worker 84 - jepsen.util 8184 :info :cas [28 [1 9]] :RequestCanceledException INFO [2019-12-17 12:32:57,213] jepsen worker 0 - jepsen.util 9180 :invoke :cas [0 [46 45]] INFO [2019-12-17 12:32:57,233] jepsen worker 87 - jepsen.util 5757 :ok :read [29 47] INFO [2019-12-17 12:32:57,233] jepsen worker 87 - jepsen.util 5757 :invoke :cas [29 [27 48]] INFO [2019-12-17 12:32:57,235] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37804,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:32:57,244] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37816,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:32:57,274] jepsen worker 6 - jepsen.util 9456 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:32:57,331] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37814,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:32:57,413] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:32:57,447] jepsen worker 79 - jepsen.util 6649 :info :cas [26 [7 10]] :RequestCanceledException INFO [2019-12-17 12:32:57,482] jepsen worker 22 - jepsen.util 8842 :info :cas [7 [4 46]] :RequestCanceledException INFO [2019-12-17 12:32:57,489] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37826,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:32:57,559] jepsen worker 17 - jepsen.util 9017 :info :cas [5 [14 15]] :RequestCanceledException INFO [2019-12-17 12:32:57,565] jepsen worker 60 - jepsen.util 9780 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:32:57,626] jepsen worker 81 - jepsen.util 8541 :info :cas [27 [26 41]] :RequestCanceledException INFO [2019-12-17 12:32:57,627] jepsen worker 66 - jepsen.util 8616 :info :cas [22 [1 42]] :RequestCanceledException INFO [2019-12-17 12:32:57,627] jepsen worker 35 - jepsen.util 8765 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:32:57,630] jepsen worker 3 - jepsen.util 6393 :info :write [1 11] :RequestCanceledException INFO [2019-12-17 12:32:57,631] jepsen worker 71 - jepsen.util 7181 :info :cas [23 [33 35]] :RequestCanceledException INFO [2019-12-17 12:32:57,635] jepsen worker 4 - jepsen.util 6034 :fail :read [1 nil] :RequestTimeoutException INFO [2019-12-17 12:32:57,675] jepsen worker 62 - jepsen.util 8702 :info :cas [20 [38 27]] :RequestCanceledException INFO [2019-12-17 12:32:57,678] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37832,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:32:57,704] jepsen worker 21 - jepsen.util 8301 :info :write [7 42] :RequestCanceledException INFO [2019-12-17 12:32:57,738] jepsen worker 25 - jepsen.util 9385 :info :cas [8 [2 47]] :RequestCanceledException INFO [2019-12-17 12:32:57,740] jepsen worker 86 - jepsen.util 9266 :info :cas [28 [5 5]] :RequestCanceledException INFO [2019-12-17 12:32:57,742] jepsen worker 84 - jepsen.util 8274 :invoke :write [28 40] INFO [2019-12-17 12:32:57,743] jepsen worker 8 - jepsen.util 8648 :info :write [2 34] :RequestCanceledException INFO [2019-12-17 12:32:57,746] jepsen worker 23 - jepsen.util 8933 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:32:57,774] jepsen worker 80 - jepsen.util 6560 :info :cas [26 [21 27]] :RequestCanceledException INFO [2019-12-17 12:32:57,815] jepsen worker 70 - jepsen.util 6910 :ok :read [23 41] INFO [2019-12-17 12:32:57,863] jepsen worker 26 - jepsen.util 9296 :invoke :cas [8 [45 34]] INFO [2019-12-17 12:32:57,891] jepsen worker 68 - jepsen.util 8438 :info :write [22 0] :RequestCanceledException INFO [2019-12-17 12:32:57,896] jepsen worker 59 - jepsen.util 9059 :invoke :cas [19 [23 4]] INFO [2019-12-17 12:32:57,905] jepsen worker 12 - jepsen.util 6942 :info :cas [4 [17 16]] :RequestCanceledException INFO [2019-12-17 12:32:57,966] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37838,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:32:57,966] jepsen worker 7 - jepsen.util 9187 :info :write [2 7] :RequestCanceledException INFO [2019-12-17 12:32:57,975] jepsen worker 83 - jepsen.util 9353 :invoke :read [27 nil] INFO [2019-12-17 12:32:57,988] jepsen worker 15 - jepsen.util 7755 :info :write [5 47] :RequestCanceledException INFO [2019-12-17 12:32:57,990] jepsen worker 42 - jepsen.util 9402 :info :write [14 12] :RequestCanceledException INFO [2019-12-17 12:32:57,994] jepsen worker 10 - jepsen.util 6220 :info :write [3 33] :RequestCanceledException INFO [2019-12-17 12:32:58,012] jepsen worker 24 - jepsen.util 7944 :info :write [8 10] :RequestCanceledException INFO [2019-12-17 12:32:58,015] jepsen worker 82 - jepsen.util 9532 :info :write [27 17] :RequestCanceledException INFO [2019-12-17 12:32:58,024] jepsen worker 39 - jepsen.util 8679 :info :write [13 12] :RequestCanceledException INFO [2019-12-17 12:32:58,026] jepsen worker 1 - jepsen.util 8641 :info :write [0 29] :RequestCanceledException INFO [2019-12-17 12:32:58,046] jepsen worker 54 - jepsen.util 7164 :info :write [18 25] :RequestCanceledException INFO [2019-12-17 12:32:58,047] jepsen worker 27 - jepsen.util 6507 :info :cas [9 [39 29]] :RequestCanceledException INFO [2019-12-17 12:32:58,054] jepsen worker 67 - jepsen.util 9157 :info :write [22 4] :RequestCanceledException INFO [2019-12-17 12:32:58,055] jepsen worker 75 - jepsen.util 8445 :info :write [25 10] :RequestCanceledException INFO [2019-12-17 12:32:58,069] jepsen worker 32 - jepsen.util 6692 :fail :read [10 nil] :RequestTimeoutException INFO [2019-12-17 12:32:58,070] jepsen worker 33 - jepsen.util 8673 :invoke :read [11 nil] INFO [2019-12-17 12:32:58,071] jepsen worker 85 - jepsen.util 9895 :info :write [28 45] :RequestCanceledException INFO [2019-12-17 12:32:58,072] jepsen worker 57 - jepsen.util 8517 :info :write [19 29] :RequestCanceledException INFO [2019-12-17 12:32:58,076] jepsen worker 50 - jepsen.util 9410 :info :write [16 3] :RequestCanceledException INFO [2019-12-17 12:32:58,097] jepsen worker 89 - jepsen.util 6299 :info :write [29 33] :RequestCanceledException INFO [2019-12-17 12:32:58,139] jepsen worker 11 - jepsen.util 6851 :info :write [3 40] :RequestCanceledException INFO [2019-12-17 12:32:58,140] jepsen worker 74 - jepsen.util 6644 :info :cas [24 [40 0]] :RequestCanceledException INFO [2019-12-17 12:32:58,140] jepsen worker 72 - jepsen.util 6282 :info :write [24 32] :RequestCanceledException INFO [2019-12-17 12:32:58,140] jepsen worker 29 - jepsen.util 4889 :info :cas [9 [38 44]] :RequestCanceledException INFO [2019-12-17 12:32:58,149] jepsen worker 55 - jepsen.util 7615 :info :cas [18 [43 3]] :RequestCanceledException INFO [2019-12-17 12:32:58,150] jepsen worker 52 - jepsen.util 6892 :info :cas [17 [2 37]] :RequestCanceledException INFO [2019-12-17 12:32:58,161] jepsen worker 13 - jepsen.util 6313 :info :write [4 1] :RequestCanceledException INFO [2019-12-17 12:32:58,203] jepsen worker 88 - jepsen.util 6658 :fail :read [29 nil] :RequestTimeoutException INFO [2019-12-17 12:32:58,224] jepsen worker 20 - jepsen.util 6320 :fail :read [6 nil] :RequestTimeoutException INFO [2019-12-17 12:32:58,243] jepsen worker 63 - jepsen.util 5463 :info :write [21 24] :RequestCanceledException INFO [2019-12-17 12:32:58,244] jepsen worker 37 - jepsen.util 6337 :info :cas [12 [26 0]] :RequestCanceledException INFO [2019-12-17 12:32:58,254] jepsen worker 44 - jepsen.util 9764 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:32:58,254] jepsen worker 56 - jepsen.util 6986 :info :write [18 0] :RequestCanceledException INFO [2019-12-17 12:32:58,264] jepsen worker 5 - jepsen.util 6215 :info :cas [1 [25 30]] :RequestCanceledException INFO [2019-12-17 12:32:58,346] jepsen worker 53 - jepsen.util 6353 :ok :write [17 26] INFO [2019-12-17 12:32:58,378] jepsen worker 9 - jepsen.util 7029 :fail :cas [3 [20 19]] :ValueNotSwapFrom INFO [2019-12-17 12:32:58,441] jepsen worker 65 - jepsen.util 5285 :ok :read [21 7] INFO [2019-12-17 12:32:58,510] jepsen worker 45 - jepsen.util 6435 :info :write [15 12] :RequestCanceledException INFO [2019-12-17 12:32:58,536] jepsen worker 19 - jepsen.util 6409 :info :write [6 33] :RequestCanceledException INFO [2019-12-17 12:32:58,538] jepsen worker 14 - jepsen.util 6674 :info :write [4 24] :RequestCanceledException INFO [2019-12-17 12:32:58,544] jepsen worker 51 - jepsen.util 6441 :fail :read [17 nil] :RequestTimeoutException INFO [2019-12-17 12:32:58,544] jepsen worker 34 - jepsen.util 8224 :info :cas [11 [1 47]] :RequestCanceledException INFO [2019-12-17 12:32:58,544] jepsen worker 58 - jepsen.util 9418 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:32:58,810] jepsen worker 78 - jepsen.util 6918 :fail :read [26 nil] :RequestTimeoutException INFO [2019-12-17 12:32:58,811] jepsen worker 61 - jepsen.util 9781 :info :cas [20 [27 0]] :RequestCanceledException INFO [2019-12-17 12:32:58,813] jepsen worker 64 - jepsen.util 5014 :ok :read [21 7] INFO [2019-12-17 12:32:58,838] jepsen worker 41 - jepsen.util 9491 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:32:58,873] jepsen worker 43 - jepsen.util 8143 :info :write [14 26] :RequestCanceledException INFO [2019-12-17 12:32:58,914] jepsen worker 18 - jepsen.util 6498 :fail :cas [6 [3 3]] :ValueNotSwapFrom INFO [2019-12-17 12:32:59,125] jepsen worker 73 - jepsen.util 6103 :fail :cas [24 [25 23]] :ValueNotSwapFrom INFO [2019-12-17 12:32:59,160] jepsen worker 77 - jepsen.util 9257 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:32:59,215] jepsen worker 16 - jepsen.util 9826 :info :cas [5 [6 34]] :RequestCanceledException INFO [2019-12-17 12:32:59,244] jepsen worker 0 - jepsen.util 9180 :info :cas [0 [46 45]] :RequestCanceledException INFO [2019-12-17 12:32:59,245] jepsen worker 28 - jepsen.util 4978 :fail :cas [9 [48 3]] :ValueNotSwapFrom INFO [2019-12-17 12:32:59,246] jepsen worker 47 - jepsen.util 7067 :ok :read [15 41] INFO [2019-12-17 12:32:59,262] jepsen worker 31 - jepsen.util 5791 :fail :cas [10 [39 19]] :ValueNotSwapFrom INFO [2019-12-17 12:32:59,275] jepsen worker 36 - jepsen.util 7146 :ok :read [12 25] INFO [2019-12-17 12:32:59,369] jepsen worker 30 - jepsen.util 6330 :ok :read [10 27] INFO [2019-12-17 12:32:59,400] jepsen worker 38 - jepsen.util 6518 :fail :cas [12 [22 9]] :ValueNotSwapFrom INFO [2019-12-17 12:32:59,545] jepsen worker 87 - jepsen.util 5757 :fail :cas [29 [27 48]] :ValueNotSwapFrom INFO [2019-12-17 12:32:59,944] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37804,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:32:59,945] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37816,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:32:59,945] jepsen worker 6 - jepsen.util 9456 :invoke :write [2 42] INFO [2019-12-17 12:32:59,945] jepsen nemesis - couchbase.util Rebalance check count: 289 INFO [2019-12-17 12:32:59,945] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37814,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:32:59,949] jepsen worker 22 - jepsen.util 8932 :invoke :read [7 nil] INFO [2019-12-17 12:32:59,957] jepsen worker 79 - jepsen.util 6739 :invoke :read [26 nil] INFO [2019-12-17 12:32:59,985] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37826,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:33:00,108] jepsen worker 17 - jepsen.util 9107 :invoke :write [5 28] INFO [2019-12-17 12:33:00,113] jepsen worker 60 - jepsen.util 9780 :invoke :read [20 nil] INFO [2019-12-17 12:33:00,130] jepsen worker 81 - jepsen.util 8631 :invoke :write [27 48] INFO [2019-12-17 12:33:00,153] jepsen worker 35 - jepsen.util 8765 :invoke :write [11 22] INFO [2019-12-17 12:33:00,155] jepsen worker 4 - jepsen.util 6034 :invoke :cas [1 [0 39]] INFO [2019-12-17 12:33:00,155] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37832,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:33:00,157] jepsen worker 23 - jepsen.util 8933 :invoke :read [7 nil] INFO [2019-12-17 12:33:00,157] jepsen worker 70 - jepsen.util 6910 :invoke :read [23 nil] INFO [2019-12-17 12:33:00,158] jepsen worker 25 - jepsen.util 9475 :invoke :read [8 nil] INFO [2019-12-17 12:33:00,159] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37838,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:33:00,159] jepsen worker 8 - jepsen.util 8738 :invoke :write [2 43] INFO [2019-12-17 12:33:00,159] jepsen worker 62 - jepsen.util 8792 :invoke :cas [20 [41 17]] INFO [2019-12-17 12:33:00,160] jepsen worker 68 - jepsen.util 8528 :invoke :read [22 nil] INFO [2019-12-17 12:33:00,162] jepsen worker 32 - jepsen.util 6692 :invoke :read [10 nil] INFO [2019-12-17 12:33:00,164] jepsen worker 39 - jepsen.util 8769 :invoke :write [13 15] INFO [2019-12-17 12:33:00,164] jepsen worker 10 - jepsen.util 6310 :invoke :write [3 48] INFO [2019-12-17 12:33:00,164] jepsen worker 86 - jepsen.util 9356 :invoke :read [28 nil] INFO [2019-12-17 12:33:00,165] jepsen worker 66 - jepsen.util 8706 :invoke :cas [22 [36 43]] INFO [2019-12-17 12:33:00,165] jepsen worker 3 - jepsen.util 6483 :invoke :cas [1 [42 8]] INFO [2019-12-17 12:33:00,166] jepsen worker 72 - jepsen.util 6372 :invoke :cas [24 [15 18]] INFO [2019-12-17 12:33:00,166] jepsen worker 57 - jepsen.util 8607 :invoke :write [19 28] INFO [2019-12-17 12:33:00,166] jepsen worker 55 - jepsen.util 7705 :invoke :read [18 nil] INFO [2019-12-17 12:33:00,167] jepsen worker 52 - jepsen.util 6982 :invoke :read [17 nil] INFO [2019-12-17 12:33:00,167] jepsen worker 20 - jepsen.util 6320 :invoke :cas [6 [1 12]] INFO [2019-12-17 12:33:00,167] jepsen worker 44 - jepsen.util 9764 :invoke :write [14 18] INFO [2019-12-17 12:33:00,167] jepsen worker 53 - jepsen.util 6353 :invoke :read [17 nil] INFO [2019-12-17 12:33:00,167] jepsen worker 9 - jepsen.util 7029 :invoke :write [3 15] INFO [2019-12-17 12:33:00,168] jepsen worker 51 - jepsen.util 6441 :invoke :read [17 nil] INFO [2019-12-17 12:33:00,168] jepsen worker 58 - jepsen.util 9418 :invoke :read [19 nil] INFO [2019-12-17 12:33:00,168] jepsen worker 78 - jepsen.util 6918 :invoke :write [26 11] INFO [2019-12-17 12:33:00,168] jepsen worker 64 - jepsen.util 5014 :invoke :cas [21 [16 27]] INFO [2019-12-17 12:33:00,168] jepsen worker 41 - jepsen.util 9491 :invoke :read [13 nil] INFO [2019-12-17 12:33:00,168] jepsen worker 18 - jepsen.util 6498 :invoke :read [6 nil] INFO [2019-12-17 12:33:00,169] jepsen worker 15 - jepsen.util 7845 :invoke :read [5 nil] INFO [2019-12-17 12:33:00,172] jepsen worker 12 - jepsen.util 7032 :invoke :read [4 nil] INFO [2019-12-17 12:33:00,172] jepsen worker 80 - jepsen.util 6650 :invoke :cas [26 [12 5]] INFO [2019-12-17 12:33:00,172] jepsen worker 65 - jepsen.util 5285 :invoke :read [21 nil] INFO [2019-12-17 12:33:00,173] jepsen worker 88 - jepsen.util 6658 :invoke :cas [29 [25 47]] INFO [2019-12-17 12:33:00,173] jepsen worker 77 - jepsen.util 9257 :invoke :cas [25 [25 13]] INFO [2019-12-17 12:33:00,174] jepsen worker 28 - jepsen.util 4978 :invoke :read [9 nil] INFO [2019-12-17 12:33:00,174] jepsen worker 47 - jepsen.util 7067 :invoke :cas [15 [27 16]] INFO [2019-12-17 12:33:00,174] jepsen worker 31 - jepsen.util 5791 :invoke :write [10 38] INFO [2019-12-17 12:33:00,174] jepsen worker 19 - jepsen.util 6499 :invoke :read [6 nil] INFO [2019-12-17 12:33:00,174] jepsen worker 30 - jepsen.util 6330 :invoke :cas [10 [16 10]] INFO [2019-12-17 12:33:00,174] jepsen worker 87 - jepsen.util 5757 :invoke :cas [29 [11 17]] INFO [2019-12-17 12:33:00,174] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37810,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:33:00,174] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37824,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:33:00,174] jepsen nemesis - couchbase.util Rebalance stuck count: 270 INFO [2019-12-17 12:33:00,174] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37830,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:33:00,175] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37854,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:33:00,176] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37875,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:33:00,176] jepsen worker 73 - jepsen.util 6103 :invoke :write [24 35] INFO [2019-12-17 12:33:00,176] jepsen worker 45 - jepsen.util 6525 :invoke :read [15 nil] INFO [2019-12-17 12:33:00,176] jepsen worker 67 - jepsen.util 9247 :invoke :read [22 nil] INFO [2019-12-17 12:33:00,177] jepsen worker 13 - jepsen.util 6403 :invoke :cas [4 [47 33]] INFO [2019-12-17 12:33:00,178] jepsen worker 36 - jepsen.util 7146 :invoke :write [12 47] INFO [2019-12-17 12:33:00,179] jepsen worker 74 - jepsen.util 6734 :invoke :read [24 nil] INFO [2019-12-17 12:33:00,180] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37842,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:33:00,180] jepsen worker 1 - jepsen.util 8731 :invoke :cas [0 [23 32]] INFO [2019-12-17 12:33:00,180] jepsen worker 61 - jepsen.util 9871 :invoke :write [20 34] INFO [2019-12-17 12:33:00,182] jepsen worker 38 - jepsen.util 6518 :invoke :write [12 18] INFO [2019-12-17 12:33:00,182] jepsen worker 85 - jepsen.util 9985 :invoke :write [28 13] INFO [2019-12-17 12:33:00,183] jepsen worker 43 - jepsen.util 8233 :invoke :read [14 nil] INFO [2019-12-17 12:33:00,183] jepsen worker 56 - jepsen.util 7076 :invoke :cas [18 [13 28]] INFO [2019-12-17 12:33:00,185] jepsen worker 16 - jepsen.util 9916 :invoke :write [5 28] INFO [2019-12-17 12:33:00,186] jepsen worker 63 - jepsen.util 5553 :invoke :cas [21 [49 40]] INFO [2019-12-17 12:33:00,188] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37810,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:33:00,188] jepsen worker 42 - jepsen.util 9492 :invoke :read [14 nil] INFO [2019-12-17 12:33:00,190] jepsen worker 54 - jepsen.util 7254 :invoke :cas [18 [46 13]] INFO [2019-12-17 12:33:00,190] jepsen worker 37 - jepsen.util 6427 :invoke :write [12 13] INFO [2019-12-17 12:33:00,190] jepsen worker 14 - jepsen.util 6764 :invoke :write [4 4] INFO [2019-12-17 12:33:00,190] jepsen worker 71 - jepsen.util 7271 :invoke :write [23 37] INFO [2019-12-17 12:33:00,191] jepsen worker 11 - jepsen.util 6941 :invoke :read [3 nil] INFO [2019-12-17 12:33:00,191] jepsen worker 0 - jepsen.util 9270 :invoke :read [0 nil] INFO [2019-12-17 12:33:00,192] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37830,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:33:00,192] jepsen worker 82 - jepsen.util 9622 :invoke :read [27 nil] INFO [2019-12-17 12:33:00,193] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37824,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:33:00,193] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37854,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:33:00,194] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37875,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:33:00,195] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37842,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:33:00,196] jepsen worker 7 - jepsen.util 9277 :invoke :write [2 17] INFO [2019-12-17 12:33:00,197] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37847,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:33:00,199] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37865,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:33:00,199] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37877,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:33:00,199] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37882,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:33:00,201] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37865,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:33:00,201] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37877,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:33:00,202] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37882,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:33:00,202] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37867,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:33:00,202] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37912,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:33:00,203] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37850,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:33:00,209] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37885,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:33:00,209] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37867,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:33:00,210] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37912,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:33:00,210] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37847,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:33:00,213] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37856,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:33:00,215] jepsen worker 24 - jepsen.util 8034 :invoke :cas [8 [9 40]] INFO [2019-12-17 12:33:00,216] jepsen worker 50 - jepsen.util 9500 :invoke :cas [16 [4 21]] INFO [2019-12-17 12:33:00,218] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37885,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:33:00,221] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37850,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:33:00,221] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37868,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:33:00,221] jepsen worker 27 - jepsen.util 6597 :invoke :read [9 nil] INFO [2019-12-17 12:33:00,225] jepsen worker 5 - jepsen.util 6305 :invoke :write [1 9] INFO [2019-12-17 12:33:00,232] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37856,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:33:00,233] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37887,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:33:00,233] jepsen worker 29 - jepsen.util 4979 :invoke :cas [9 [24 17]] INFO [2019-12-17 12:33:00,242] jepsen worker 75 - jepsen.util 8535 :invoke :read [25 nil] INFO [2019-12-17 12:33:00,244] jepsen worker 21 - jepsen.util 8391 :invoke :read [7 nil] INFO [2019-12-17 12:33:00,248] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37857,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:33:00,250] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37868,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:33:00,256] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37917,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:33:00,258] jepsen worker 34 - jepsen.util 8314 :invoke :read [11 nil] INFO [2019-12-17 12:33:00,269] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37887,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:33:00,282] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37857,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:33:00,290] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37917,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:33:00,293] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37891,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:33:00,311] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37906,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:33:00,312] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37858,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:33:00,314] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37891,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:33:00,316] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37906,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:33:00,322] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37921,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:33:00,323] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37900,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:33:00,341] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37858,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:33:00,350] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37921,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:33:00,354] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37900,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:33:00,364] jepsen worker 76 - jepsen.util 9166 :info :write [25 1] :RequestCanceledException INFO [2019-12-17 12:33:00,378] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37925,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:33:00,381] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37925,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:33:00,386] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37893,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:33:00,406] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37893,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:33:00,418] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37986,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:33:00,424] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37986,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:33:00,428] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37991,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:33:00,438] jepsen worker 89 - jepsen.util 6389 :invoke :write [29 21] INFO [2019-12-17 12:33:00,438] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37966,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:33:00,443] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37991,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:33:00,455] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37966,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:33:00,468] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37968,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:33:00,474] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37968,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:33:00,495] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37999,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:33:00,497] jepsen worker 69 - jepsen.util 6549 :info :write [23 15] :RequestCanceledException INFO [2019-12-17 12:33:00,507] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":37999,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:33:00,520] jepsen worker 46 - jepsen.util 6526 :info :write [15 20] :RequestCanceledException INFO [2019-12-17 12:33:00,520] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38004,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:33:00,529] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38004,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:33:00,547] jepsen worker 76 - jepsen.util 9256 :invoke :read [25 nil] INFO [2019-12-17 12:33:00,592] jepsen worker 46 - jepsen.util 6616 :invoke :cas [15 [19 12]] INFO [2019-12-17 12:33:00,593] jepsen worker 69 - jepsen.util 6639 :invoke :read [23 nil] INFO [2019-12-17 12:33:00,697] jepsen worker 2 - jepsen.util 8552 :info :write [0 17] :RequestCanceledException INFO [2019-12-17 12:33:00,704] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38010,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:33:00,709] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38010,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:33:00,713] jepsen worker 48 - jepsen.util 8148 :info :write [16 26] :RequestCanceledException INFO [2019-12-17 12:33:00,713] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38013,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:33:00,720] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38013,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:33:00,738] jepsen worker 49 - jepsen.util 8059 :info :write [16 6] :RequestCanceledException INFO [2019-12-17 12:33:00,809] jepsen worker 2 - jepsen.util 8642 :invoke :write [0 11] INFO [2019-12-17 12:33:00,837] jepsen worker 40 - jepsen.util 9310 :info :write [13 32] :RequestCanceledException INFO [2019-12-17 12:33:00,964] jepsen worker 5 - jepsen.util 6305 :ok :write [1 9] INFO [2019-12-17 12:33:00,964] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38023,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:33:00,964] jepsen worker 5 - jepsen.util 6305 :invoke :read [1 nil] INFO [2019-12-17 12:33:00,964] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38023,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:33:00,965] jepsen worker 40 - jepsen.util 9400 :invoke :read [13 nil] INFO [2019-12-17 12:33:00,978] jepsen worker 48 - jepsen.util 8238 :invoke :cas [16 [21 3]] INFO [2019-12-17 12:33:01,014] jepsen worker 49 - jepsen.util 8149 :invoke :cas [16 [21 39]] INFO [2019-12-17 12:33:01,054] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38032,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:33:01,064] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38032,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:33:01,071] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38037,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:33:01,074] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38037,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:33:01,224] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38053,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:33:01,230] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38053,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:33:01,293] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38060,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:33:01,296] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38060,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:33:01,317] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:33:01,318] jepsen nemesis - couchbase.util Rebalance check count: 290 INFO [2019-12-17 12:33:01,324] jepsen nemesis - couchbase.util Rebalance stuck count: 271 INFO [2019-12-17 12:33:01,485] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38067,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:33:01,490] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38067,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:33:01,589] jepsen worker 89 - jepsen.util 6389 :ok :write [29 21] INFO [2019-12-17 12:33:01,591] jepsen worker 89 - jepsen.util 6389 :invoke :cas [29 [22 39]] INFO [2019-12-17 12:33:01,662] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38077,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:33:01,662] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38077,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:33:01,677] jepsen worker 45 - jepsen.util 6525 :ok :read [15 41] INFO [2019-12-17 12:33:01,678] jepsen worker 45 - jepsen.util 6525 :invoke :cas [15 [31 27]] INFO [2019-12-17 12:33:01,788] jepsen worker 29 - jepsen.util 4979 :fail :cas [9 [24 17]] :ValueNotSwapFrom INFO [2019-12-17 12:33:01,789] jepsen worker 29 - jepsen.util 4979 :invoke :write [9 17] INFO [2019-12-17 12:33:01,799] jepsen worker 18 - jepsen.util 6498 :ok :read [6 5] INFO [2019-12-17 12:33:01,800] jepsen worker 18 - jepsen.util 6498 :invoke :write [6 49] INFO [2019-12-17 12:33:01,809] jepsen worker 11 - jepsen.util 6941 :ok :read [3 27] INFO [2019-12-17 12:33:01,810] jepsen worker 11 - jepsen.util 6941 :invoke :write [3 29] INFO [2019-12-17 12:33:01,816] jepsen worker 54 - jepsen.util 7254 :fail :cas [18 [46 13]] :ValueNotSwapFrom INFO [2019-12-17 12:33:01,817] jepsen worker 54 - jepsen.util 7254 :invoke :cas [18 [3 5]] INFO [2019-12-17 12:33:01,818] jepsen worker 55 - jepsen.util 7705 :ok :read [18 25] INFO [2019-12-17 12:33:01,819] jepsen worker 55 - jepsen.util 7705 :invoke :cas [18 [3 5]] INFO [2019-12-17 12:33:01,834] jepsen worker 20 - jepsen.util 6320 :fail :cas [6 [1 12]] :ValueNotSwapFrom INFO [2019-12-17 12:33:01,834] jepsen worker 20 - jepsen.util 6320 :invoke :read [6 nil] INFO [2019-12-17 12:33:01,875] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38086,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:33:01,886] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38086,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:33:01,887] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38088,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:33:01,889] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38088,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:33:01,889] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38089,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:33:01,889] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38089,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:33:01,890] jepsen worker 19 - jepsen.util 6499 :ok :read [6 5] INFO [2019-12-17 12:33:01,890] jepsen worker 19 - jepsen.util 6499 :invoke :write [6 23] INFO [2019-12-17 12:33:01,913] jepsen worker 4 - jepsen.util 6034 :fail :cas [1 [0 39]] :ValueNotSwapFrom INFO [2019-12-17 12:33:01,914] jepsen worker 4 - jepsen.util 6034 :invoke :read [1 nil] INFO [2019-12-17 12:33:01,917] jepsen worker 51 - jepsen.util 6441 :ok :read [17 26] INFO [2019-12-17 12:33:01,918] jepsen worker 3 - jepsen.util 6483 :fail :cas [1 [42 8]] :ValueNotSwapFrom INFO [2019-12-17 12:33:01,918] jepsen worker 51 - jepsen.util 6441 :invoke :read [17 nil] INFO [2019-12-17 12:33:01,919] jepsen worker 3 - jepsen.util 6483 :invoke :write [1 26] INFO [2019-12-17 12:33:01,925] jepsen worker 64 - jepsen.util 5014 :fail :cas [21 [16 27]] :ValueNotSwapFrom INFO [2019-12-17 12:33:01,925] jepsen worker 80 - jepsen.util 6650 :fail :cas [26 [12 5]] :ValueNotSwapFrom INFO [2019-12-17 12:33:01,925] jepsen worker 64 - jepsen.util 5014 :invoke :write [21 17] INFO [2019-12-17 12:33:01,925] jepsen worker 80 - jepsen.util 6650 :invoke :cas [26 [17 6]] INFO [2019-12-17 12:33:01,925] jepsen worker 12 - jepsen.util 7032 :ok :read [4 1] INFO [2019-12-17 12:33:01,926] jepsen worker 12 - jepsen.util 7032 :invoke :write [4 43] INFO [2019-12-17 12:33:01,933] jepsen worker 72 - jepsen.util 6372 :fail :cas [24 [15 18]] :ValueNotSwapFrom INFO [2019-12-17 12:33:01,934] jepsen worker 72 - jepsen.util 6372 :invoke :write [24 36] INFO [2019-12-17 12:33:01,959] jepsen worker 87 - jepsen.util 5757 :fail :cas [29 [11 17]] :ValueNotSwapFrom INFO [2019-12-17 12:33:01,959] jepsen worker 87 - jepsen.util 5757 :invoke :cas [29 [4 5]] INFO [2019-12-17 12:33:01,969] jepsen worker 63 - jepsen.util 5553 :fail :cas [21 [49 40]] :ValueNotSwapFrom INFO [2019-12-17 12:33:01,970] jepsen worker 63 - jepsen.util 5553 :invoke :write [21 33] INFO [2019-12-17 12:33:01,988] jepsen worker 32 - jepsen.util 6692 :ok :read [10 42] INFO [2019-12-17 12:33:01,989] jepsen worker 32 - jepsen.util 6692 :invoke :write [10 4] INFO [2019-12-17 12:33:02,005] jepsen worker 27 - jepsen.util 6597 :ok :read [9 17] INFO [2019-12-17 12:33:02,007] jepsen worker 27 - jepsen.util 6597 :invoke :write [9 17] INFO [2019-12-17 12:33:02,008] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38096,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:33:02,008] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38096,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:33:02,008] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38099,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:33:02,009] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38099,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:33:02,022] jepsen worker 79 - jepsen.util 6739 :ok :read [26 17] INFO [2019-12-17 12:33:02,023] jepsen worker 79 - jepsen.util 6739 :invoke :write [26 28] INFO [2019-12-17 12:33:02,042] jepsen worker 65 - jepsen.util 5285 :ok :read [21 7] INFO [2019-12-17 12:33:02,044] jepsen worker 65 - jepsen.util 5285 :invoke :write [21 38] INFO [2019-12-17 12:33:02,057] jepsen worker 74 - jepsen.util 6734 :ok :read [24 27] INFO [2019-12-17 12:33:02,057] jepsen worker 5 - jepsen.util 6305 :ok :read [1 9] INFO [2019-12-17 12:33:02,057] jepsen worker 74 - jepsen.util 6734 :invoke :read [24 nil] INFO [2019-12-17 12:33:02,058] jepsen worker 5 - jepsen.util 6305 :invoke :write [1 47] INFO [2019-12-17 12:33:02,065] jepsen worker 28 - jepsen.util 4978 :ok :read [9 17] INFO [2019-12-17 12:33:02,065] jepsen worker 28 - jepsen.util 4978 :invoke :write [9 39] INFO [2019-12-17 12:33:02,068] jepsen worker 70 - jepsen.util 6910 :ok :read [23 48] INFO [2019-12-17 12:33:02,069] jepsen worker 70 - jepsen.util 6910 :invoke :write [23 7] INFO [2019-12-17 12:33:02,073] jepsen worker 30 - jepsen.util 6330 :fail :cas [10 [16 10]] :ValueNotSwapFrom INFO [2019-12-17 12:33:02,074] jepsen worker 30 - jepsen.util 6330 :invoke :write [10 7] INFO [2019-12-17 12:33:02,076] jepsen worker 47 - jepsen.util 7067 :fail :cas [15 [27 16]] :ValueNotSwapFrom INFO [2019-12-17 12:33:02,076] jepsen worker 47 - jepsen.util 7067 :invoke :cas [15 [0 32]] INFO [2019-12-17 12:33:02,194] jepsen worker 23 - jepsen.util 8933 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:33:02,194] jepsen worker 23 - jepsen.util 8933 :invoke :cas [7 [43 32]] INFO [2019-12-17 12:33:02,195] jepsen worker 33 - jepsen.util 8673 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:33:02,199] jepsen worker 33 - jepsen.util 8673 :invoke :write [11 29] INFO [2019-12-17 12:33:02,213] jepsen worker 22 - jepsen.util 8932 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:33:02,214] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38118,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:33:02,215] jepsen worker 22 - jepsen.util 8932 :invoke :read [7 nil] INFO [2019-12-17 12:33:02,215] jepsen worker 62 - jepsen.util 8792 :info :cas [20 [41 17]] :RequestCanceledException INFO [2019-12-17 12:33:02,215] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38118,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:33:02,215] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38120,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:33:02,215] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38120,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:33:02,224] jepsen worker 62 - jepsen.util 8882 :invoke :write [20 41] INFO [2019-12-17 12:33:02,228] jepsen worker 60 - jepsen.util 9780 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:33:02,228] jepsen worker 83 - jepsen.util 9353 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:33:02,228] jepsen worker 68 - jepsen.util 8528 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:33:02,228] jepsen worker 41 - jepsen.util 9491 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:33:02,228] jepsen worker 25 - jepsen.util 9475 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:33:02,228] jepsen worker 1 - jepsen.util 8731 :info :cas [0 [23 32]] :RequestCanceledException INFO [2019-12-17 12:33:02,228] jepsen worker 86 - jepsen.util 9356 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:33:02,228] jepsen worker 58 - jepsen.util 9418 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:33:02,228] jepsen worker 60 - jepsen.util 9780 :invoke :write [20 43] INFO [2019-12-17 12:33:02,228] jepsen worker 83 - jepsen.util 9353 :invoke :write [27 17] INFO [2019-12-17 12:33:02,228] jepsen worker 68 - jepsen.util 8528 :invoke :cas [22 [27 28]] INFO [2019-12-17 12:33:02,228] jepsen worker 67 - jepsen.util 9247 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:33:02,228] jepsen worker 41 - jepsen.util 9491 :invoke :cas [13 [20 46]] INFO [2019-12-17 12:33:02,228] jepsen worker 25 - jepsen.util 9475 :invoke :read [8 nil] INFO [2019-12-17 12:33:02,228] jepsen worker 86 - jepsen.util 9356 :invoke :cas [28 [2 10]] INFO [2019-12-17 12:33:02,228] jepsen worker 58 - jepsen.util 9418 :invoke :cas [19 [5 36]] INFO [2019-12-17 12:33:02,228] jepsen worker 43 - jepsen.util 8233 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:33:02,229] jepsen worker 67 - jepsen.util 9247 :invoke :read [22 nil] INFO [2019-12-17 12:33:02,229] jepsen worker 43 - jepsen.util 8233 :invoke :cas [14 [37 28]] INFO [2019-12-17 12:33:02,237] jepsen worker 1 - jepsen.util 8821 :invoke :cas [0 [29 11]] INFO [2019-12-17 12:33:02,241] jepsen worker 26 - jepsen.util 9296 :info :cas [8 [45 34]] :RequestCanceledException INFO [2019-12-17 12:33:02,241] jepsen worker 42 - jepsen.util 9492 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:33:02,241] jepsen worker 0 - jepsen.util 9270 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:33:02,242] jepsen worker 42 - jepsen.util 9492 :invoke :read [14 nil] INFO [2019-12-17 12:33:02,242] jepsen worker 0 - jepsen.util 9270 :invoke :cas [0 [28 17]] INFO [2019-12-17 12:33:02,250] jepsen worker 26 - jepsen.util 9386 :invoke :read [8 nil] INFO [2019-12-17 12:33:02,254] jepsen worker 52 - jepsen.util 6982 :ok :read [17 26] INFO [2019-12-17 12:33:02,255] jepsen worker 52 - jepsen.util 6982 :invoke :cas [17 [6 44]] INFO [2019-12-17 12:33:02,259] jepsen worker 77 - jepsen.util 9257 :info :cas [25 [25 13]] :RequestCanceledException INFO [2019-12-17 12:33:02,259] jepsen worker 59 - jepsen.util 9059 :info :cas [19 [23 4]] :RequestCanceledException INFO [2019-12-17 12:33:02,259] jepsen worker 82 - jepsen.util 9622 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:33:02,259] jepsen worker 10 - jepsen.util 6310 :info :write [3 48] :RequestCanceledException INFO [2019-12-17 12:33:02,259] jepsen worker 82 - jepsen.util 9622 :invoke :write [27 23] INFO [2019-12-17 12:33:02,263] jepsen worker 10 - jepsen.util 6400 :invoke :cas [3 [1 2]] INFO [2019-12-17 12:33:02,265] jepsen worker 59 - jepsen.util 9149 :invoke :read [19 nil] INFO [2019-12-17 12:33:02,267] jepsen worker 77 - jepsen.util 9347 :invoke :write [25 6] INFO [2019-12-17 12:33:02,292] jepsen worker 53 - jepsen.util 6353 :ok :read [17 26] INFO [2019-12-17 12:33:02,292] jepsen worker 53 - jepsen.util 6353 :invoke :write [17 35] INFO [2019-12-17 12:33:02,294] jepsen worker 15 - jepsen.util 7845 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:33:02,294] jepsen worker 66 - jepsen.util 8706 :info :cas [22 [36 43]] :RequestCanceledException INFO [2019-12-17 12:33:02,294] jepsen worker 56 - jepsen.util 7076 :info :cas [18 [13 28]] :RequestCanceledException INFO [2019-12-17 12:33:02,294] jepsen worker 24 - jepsen.util 8034 :info :cas [8 [9 40]] :RequestCanceledException INFO [2019-12-17 12:33:02,294] jepsen worker 21 - jepsen.util 8391 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:33:02,294] jepsen worker 15 - jepsen.util 7845 :invoke :write [5 20] INFO [2019-12-17 12:33:02,296] jepsen worker 21 - jepsen.util 8391 :invoke :cas [7 [16 48]] INFO [2019-12-17 12:33:02,297] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38126,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:33:02,298] jepsen worker 56 - jepsen.util 7166 :invoke :cas [18 [9 24]] INFO [2019-12-17 12:33:02,298] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38126,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:33:02,298] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38130,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:33:02,298] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38130,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:33:02,301] jepsen worker 24 - jepsen.util 8124 :invoke :read [8 nil] INFO [2019-12-17 12:33:02,305] jepsen worker 66 - jepsen.util 8796 :invoke :cas [22 [40 28]] INFO [2019-12-17 12:33:02,310] jepsen worker 69 - jepsen.util 6639 :ok :read [23 48] INFO [2019-12-17 12:33:02,311] jepsen worker 69 - jepsen.util 6639 :invoke :write [23 12] INFO [2019-12-17 12:33:02,312] jepsen worker 50 - jepsen.util 9500 :info :cas [16 [4 21]] :RequestCanceledException INFO [2019-12-17 12:33:02,313] jepsen worker 34 - jepsen.util 8314 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:33:02,313] jepsen worker 34 - jepsen.util 8314 :invoke :write [11 40] INFO [2019-12-17 12:33:02,324] jepsen worker 54 - jepsen.util 7254 :fail :cas [18 [3 5]] :ValueNotSwapFrom INFO [2019-12-17 12:33:02,324] jepsen worker 54 - jepsen.util 7254 :invoke :write [18 43] INFO [2019-12-17 12:33:02,327] jepsen worker 75 - jepsen.util 8535 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:33:02,327] jepsen worker 75 - jepsen.util 8535 :invoke :read [25 nil] INFO [2019-12-17 12:33:02,329] jepsen worker 50 - jepsen.util 9590 :invoke :read [16 nil] INFO [2019-12-17 12:33:02,331] jepsen worker 88 - jepsen.util 6658 :fail :cas [29 [25 47]] :ValueNotSwapFrom INFO [2019-12-17 12:33:02,331] jepsen worker 88 - jepsen.util 6658 :invoke :cas [29 [7 12]] INFO [2019-12-17 12:33:02,365] jepsen worker 89 - jepsen.util 6389 :fail :cas [29 [22 39]] :ValueNotSwapFrom INFO [2019-12-17 12:33:02,365] jepsen worker 89 - jepsen.util 6389 :invoke :cas [29 [19 3]] INFO [2019-12-17 12:33:02,387] jepsen worker 4 - jepsen.util 6034 :ok :read [1 9] INFO [2019-12-17 12:33:02,388] jepsen worker 4 - jepsen.util 6034 :invoke :cas [1 [22 7]] INFO [2019-12-17 12:33:02,392] jepsen worker 13 - jepsen.util 6403 :fail :cas [4 [47 33]] :ValueNotSwapFrom INFO [2019-12-17 12:33:02,392] jepsen worker 13 - jepsen.util 6403 :invoke :read [4 nil] INFO [2019-12-17 12:33:02,410] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:33:02,410] jepsen nemesis - couchbase.util Rebalance check count: 291 INFO [2019-12-17 12:33:02,410] jepsen nemesis - couchbase.util Rebalance stuck count: 272 INFO [2019-12-17 12:33:02,479] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38168,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:33:02,479] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38168,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:33:02,479] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38171,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:33:02,479] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38171,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:33:02,479] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38172,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:33:02,479] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38172,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:33:02,489] jepsen worker 55 - jepsen.util 7705 :fail :cas [18 [3 5]] :ValueNotSwapFrom INFO [2019-12-17 12:33:02,491] jepsen worker 55 - jepsen.util 7705 :invoke :read [18 nil] INFO [2019-12-17 12:33:02,550] jepsen worker 37 - jepsen.util 6427 :ok :write [12 13] INFO [2019-12-17 12:33:02,550] jepsen worker 37 - jepsen.util 6427 :invoke :read [12 nil] INFO [2019-12-17 12:33:02,555] jepsen worker 56 - jepsen.util 7166 :fail :cas [18 [9 24]] :ValueNotSwapFrom INFO [2019-12-17 12:33:02,555] jepsen worker 56 - jepsen.util 7166 :invoke :write [18 39] INFO [2019-12-17 12:33:02,567] jepsen worker 20 - jepsen.util 6320 :ok :read [6 5] INFO [2019-12-17 12:33:02,567] jepsen worker 20 - jepsen.util 6320 :invoke :write [6 0] INFO [2019-12-17 12:33:02,627] jepsen worker 84 - jepsen.util 8274 :info :write [28 40] :RequestCanceledException INFO [2019-12-17 12:33:02,628] jepsen worker 74 - jepsen.util 6734 :ok :read [24 27] INFO [2019-12-17 12:33:02,629] jepsen worker 74 - jepsen.util 6734 :invoke :write [24 5] INFO [2019-12-17 12:33:02,637] jepsen worker 84 - jepsen.util 8364 :invoke :write [28 6] INFO [2019-12-17 12:33:02,642] jepsen worker 76 - jepsen.util 9256 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:33:02,642] jepsen worker 76 - jepsen.util 9256 :invoke :cas [25 [33 28]] INFO [2019-12-17 12:33:02,654] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38180,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:33:02,654] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38180,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:33:02,654] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38184,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:33:02,654] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38184,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:33:02,657] jepsen worker 46 - jepsen.util 6616 :info :cas [15 [19 12]] :RequestCanceledException INFO [2019-12-17 12:33:02,665] jepsen worker 46 - jepsen.util 6706 :invoke :write [15 41] INFO [2019-12-17 12:33:02,666] jepsen worker 10 - jepsen.util 6400 :fail :cas [3 [1 2]] :ValueNotSwapFrom INFO [2019-12-17 12:33:02,666] jepsen worker 10 - jepsen.util 6400 :invoke :cas [3 [27 11]] INFO [2019-12-17 12:33:02,683] jepsen worker 36 - jepsen.util 7146 :ok :write [12 47] INFO [2019-12-17 12:33:02,683] jepsen worker 36 - jepsen.util 7146 :invoke :read [12 nil] INFO [2019-12-17 12:33:02,768] jepsen worker 47 - jepsen.util 7067 :fail :cas [15 [0 32]] :ValueNotSwapFrom INFO [2019-12-17 12:33:02,768] jepsen worker 47 - jepsen.util 7067 :invoke :write [15 36] INFO [2019-12-17 12:33:02,784] jepsen worker 78 - jepsen.util 6918 :ok :write [26 11] INFO [2019-12-17 12:33:02,784] jepsen worker 78 - jepsen.util 6918 :invoke :cas [26 [15 9]] INFO [2019-12-17 12:33:02,800] jepsen worker 51 - jepsen.util 6441 :ok :read [17 26] INFO [2019-12-17 12:33:02,800] jepsen worker 51 - jepsen.util 6441 :invoke :write [17 47] INFO [2019-12-17 12:33:02,827] jepsen worker 44 - jepsen.util 9764 :info :write [14 18] :RequestCanceledException INFO [2019-12-17 12:33:02,829] jepsen worker 44 - jepsen.util 9854 :invoke :write [14 25] INFO [2019-12-17 12:33:02,829] jepsen worker 6 - jepsen.util 9456 :info :write [2 42] :RequestCanceledException INFO [2019-12-17 12:33:02,832] jepsen worker 6 - jepsen.util 9546 :invoke :write [2 26] INFO [2019-12-17 12:33:02,835] jepsen worker 39 - jepsen.util 8769 :info :write [13 15] :RequestCanceledException INFO [2019-12-17 12:33:02,837] jepsen worker 39 - jepsen.util 8859 :invoke :read [13 nil] INFO [2019-12-17 12:33:02,857] jepsen worker 81 - jepsen.util 8631 :info :write [27 48] :RequestCanceledException INFO [2019-12-17 12:33:02,860] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38198,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:33:02,861] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38198,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:33:02,861] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38202,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:33:02,861] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38202,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:33:02,878] jepsen worker 79 - jepsen.util 6739 :ok :write [26 28] INFO [2019-12-17 12:33:02,878] jepsen worker 79 - jepsen.util 6739 :invoke :write [26 45] INFO [2019-12-17 12:33:02,887] jepsen worker 81 - jepsen.util 8721 :invoke :read [27 nil] INFO [2019-12-17 12:33:02,961] jepsen worker 5 - jepsen.util 6305 :ok :write [1 47] INFO [2019-12-17 12:33:02,961] jepsen worker 5 - jepsen.util 6305 :invoke :read [1 nil] INFO [2019-12-17 12:33:03,080] jepsen worker 80 - jepsen.util 6650 :fail :cas [26 [17 6]] :CASMismatchException INFO [2019-12-17 12:33:03,081] jepsen worker 17 - jepsen.util 9107 :info :write [5 28] :RequestCanceledException INFO [2019-12-17 12:33:03,081] jepsen worker 9 - jepsen.util 7029 :info :write [3 15] :RequestCanceledException INFO [2019-12-17 12:33:03,081] jepsen worker 80 - jepsen.util 6650 :invoke :read [26 nil] INFO [2019-12-17 12:33:03,081] jepsen worker 48 - jepsen.util 8238 :info :cas [16 [21 3]] :RequestCanceledException INFO [2019-12-17 12:33:03,081] jepsen worker 85 - jepsen.util 9985 :info :write [28 13] :RequestCanceledException INFO [2019-12-17 12:33:03,082] jepsen worker 40 - jepsen.util 9400 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:33:03,082] jepsen worker 40 - jepsen.util 9400 :invoke :cas [13 [44 39]] INFO [2019-12-17 12:33:03,083] jepsen worker 14 - jepsen.util 6764 :info :write [4 4] :RequestCanceledException INFO [2019-12-17 12:33:03,083] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38211,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:33:03,084] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38211,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:33:03,084] jepsen worker 9 - jepsen.util 7119 :invoke :cas [3 [19 19]] INFO [2019-12-17 12:33:03,084] jepsen worker 48 - jepsen.util 8328 :invoke :write [16 17] INFO [2019-12-17 12:33:03,084] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38213,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:33:03,085] jepsen worker 85 - jepsen.util 10075 :invoke :read [28 nil] INFO [2019-12-17 12:33:03,085] jepsen worker 8 - jepsen.util 8738 :info :write [2 43] :RequestCanceledException INFO [2019-12-17 12:33:03,085] jepsen worker 38 - jepsen.util 6518 :info :write [12 18] :RequestCanceledException INFO [2019-12-17 12:33:03,085] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38213,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:33:03,087] jepsen worker 14 - jepsen.util 6854 :invoke :write [4 42] INFO [2019-12-17 12:33:03,089] jepsen worker 38 - jepsen.util 6608 :invoke :cas [12 [48 23]] INFO [2019-12-17 12:33:03,091] jepsen worker 17 - jepsen.util 9197 :invoke :cas [5 [45 4]] INFO [2019-12-17 12:33:03,093] jepsen worker 8 - jepsen.util 8828 :invoke :cas [2 [33 13]] INFO [2019-12-17 12:33:03,097] jepsen worker 89 - jepsen.util 6389 :fail :cas [29 [19 3]] :ValueNotSwapFrom INFO [2019-12-17 12:33:03,098] jepsen worker 89 - jepsen.util 6389 :invoke :write [29 20] INFO [2019-12-17 12:33:03,104] jepsen worker 52 - jepsen.util 6982 :fail :cas [17 [6 44]] :ValueNotSwapFrom INFO [2019-12-17 12:33:03,104] jepsen worker 52 - jepsen.util 6982 :invoke :read [17 nil] INFO [2019-12-17 12:33:03,125] jepsen worker 7 - jepsen.util 9277 :info :write [2 17] :RequestCanceledException INFO [2019-12-17 12:33:03,127] jepsen worker 7 - jepsen.util 9367 :invoke :read [2 nil] INFO [2019-12-17 12:33:03,141] jepsen worker 57 - jepsen.util 8607 :info :write [19 28] :RequestCanceledException INFO [2019-12-17 12:33:03,141] jepsen worker 49 - jepsen.util 8149 :info :cas [16 [21 39]] :RequestCanceledException INFO [2019-12-17 12:33:03,141] jepsen worker 61 - jepsen.util 9871 :info :write [20 34] :RequestCanceledException INFO [2019-12-17 12:33:03,148] jepsen worker 57 - jepsen.util 8697 :invoke :read [19 nil] INFO [2019-12-17 12:33:03,148] jepsen worker 49 - jepsen.util 8239 :invoke :cas [16 [46 31]] INFO [2019-12-17 12:33:03,149] jepsen worker 61 - jepsen.util 9961 :invoke :write [20 28] INFO [2019-12-17 12:33:03,151] jepsen worker 4 - jepsen.util 6034 :fail :cas [1 [22 7]] :ValueNotSwapFrom INFO [2019-12-17 12:33:03,151] jepsen worker 4 - jepsen.util 6034 :invoke :cas [1 [14 30]] INFO [2019-12-17 12:33:03,169] jepsen worker 36 - jepsen.util 7146 :ok :read [12 47] INFO [2019-12-17 12:33:03,171] jepsen worker 36 - jepsen.util 7146 :invoke :write [12 0] INFO [2019-12-17 12:33:03,174] jepsen worker 9 - jepsen.util 7119 :fail :cas [3 [19 19]] :ValueNotSwapFrom INFO [2019-12-17 12:33:03,174] jepsen worker 9 - jepsen.util 7119 :invoke :read [3 nil] INFO [2019-12-17 12:33:03,195] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38231,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:33:03,195] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38231,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:33:03,195] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38232,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:33:03,195] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38232,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:33:03,210] jepsen worker 16 - jepsen.util 9916 :info :write [5 28] :RequestCanceledException INFO [2019-12-17 12:33:03,210] jepsen worker 71 - jepsen.util 7271 :info :write [23 37] :RequestCanceledException INFO [2019-12-17 12:33:03,222] jepsen worker 16 - jepsen.util 10006 :invoke :write [5 26] INFO [2019-12-17 12:33:03,226] jepsen worker 71 - jepsen.util 7361 :invoke :cas [23 [5 14]] INFO [2019-12-17 12:33:03,232] jepsen worker 31 - jepsen.util 5791 :info :write [10 38] :RequestCanceledException INFO [2019-12-17 12:33:03,237] jepsen worker 31 - jepsen.util 5881 :invoke :cas [10 [5 31]] INFO [2019-12-17 12:33:03,240] jepsen worker 79 - jepsen.util 6739 :ok :write [26 45] INFO [2019-12-17 12:33:03,240] jepsen worker 79 - jepsen.util 6739 :invoke :read [26 nil] INFO [2019-12-17 12:33:03,246] jepsen worker 78 - jepsen.util 6918 :fail :cas [26 [15 9]] :ValueNotSwapFrom INFO [2019-12-17 12:33:03,247] jepsen worker 78 - jepsen.util 6918 :invoke :write [26 19] INFO [2019-12-17 12:33:03,254] jepsen worker 13 - jepsen.util 6403 :ok :read [4 4] INFO [2019-12-17 12:33:03,256] jepsen worker 13 - jepsen.util 6403 :invoke :cas [4 [46 46]] INFO [2019-12-17 12:33:03,260] jepsen worker 73 - jepsen.util 6103 :info :write [24 35] :RequestCanceledException INFO [2019-12-17 12:33:03,263] jepsen worker 73 - jepsen.util 6193 :invoke :write [24 5] INFO [2019-12-17 12:33:03,267] jepsen worker 9 - jepsen.util 7119 :ok :read [3 27] INFO [2019-12-17 12:33:03,268] jepsen worker 9 - jepsen.util 7119 :invoke :read [3 nil] INFO [2019-12-17 12:33:03,283] jepsen worker 31 - jepsen.util 5881 :fail :cas [10 [5 31]] :ValueNotSwapFrom INFO [2019-12-17 12:33:03,283] jepsen worker 31 - jepsen.util 5881 :invoke :read [10 nil] INFO [2019-12-17 12:33:03,292] jepsen worker 35 - jepsen.util 8765 :info :write [11 22] :RequestCanceledException INFO [2019-12-17 12:33:03,297] jepsen worker 35 - jepsen.util 8855 :invoke :cas [11 [34 12]] INFO [2019-12-17 12:33:03,299] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38240,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:33:03,299] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38240,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:33:03,300] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38242,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:33:03,300] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38242,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:33:03,319] jepsen worker 37 - jepsen.util 6427 :ok :read [12 18] INFO [2019-12-17 12:33:03,319] jepsen worker 37 - jepsen.util 6427 :invoke :cas [12 [7 28]] INFO [2019-12-17 12:33:03,332] jepsen worker 53 - jepsen.util 6353 :ok :write [17 35] INFO [2019-12-17 12:33:03,332] jepsen worker 53 - jepsen.util 6353 :invoke :read [17 nil] INFO [2019-12-17 12:33:03,383] jepsen worker 31 - jepsen.util 5881 :ok :read [10 42] INFO [2019-12-17 12:33:03,383] jepsen worker 31 - jepsen.util 5881 :invoke :read [10 nil] INFO [2019-12-17 12:33:03,435] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:33:03,436] jepsen nemesis - couchbase.util Rebalance check count: 292 INFO [2019-12-17 12:33:03,437] jepsen nemesis - couchbase.util Rebalance stuck count: 273 INFO [2019-12-17 12:33:03,519] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38258,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:33:03,520] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38258,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:33:03,521] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38260,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:33:03,521] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38260,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:33:03,587] jepsen worker 9 - jepsen.util 7119 :ok :read [3 27] INFO [2019-12-17 12:33:03,588] jepsen worker 9 - jepsen.util 7119 :invoke :read [3 nil] INFO [2019-12-17 12:33:03,593] jepsen worker 2 - jepsen.util 8642 :info :write [0 11] :RequestCanceledException INFO [2019-12-17 12:33:03,601] jepsen worker 2 - jepsen.util 8732 :invoke :write [0 41] INFO [2019-12-17 12:33:03,604] jepsen worker 3 - jepsen.util 6483 :ok :write [1 26] INFO [2019-12-17 12:33:03,604] jepsen worker 3 - jepsen.util 6483 :invoke :write [1 13] INFO [2019-12-17 12:33:03,625] jepsen worker 80 - jepsen.util 6650 :ok :read [26 45] INFO [2019-12-17 12:33:03,625] jepsen worker 80 - jepsen.util 6650 :invoke :cas [26 [43 27]] INFO [2019-12-17 12:33:03,627] jepsen worker 89 - jepsen.util 6389 :ok :write [29 20] INFO [2019-12-17 12:33:03,628] jepsen worker 89 - jepsen.util 6389 :invoke :read [29 nil] INFO [2019-12-17 12:33:03,632] jepsen worker 71 - jepsen.util 7361 :fail :cas [23 [5 14]] :ValueNotSwapFrom INFO [2019-12-17 12:33:03,632] jepsen worker 71 - jepsen.util 7361 :invoke :read [23 nil] INFO [2019-12-17 12:33:03,648] jepsen worker 87 - jepsen.util 5757 :fail :cas [29 [4 5]] :ValueNotSwapFrom INFO [2019-12-17 12:33:03,649] jepsen worker 87 - jepsen.util 5757 :invoke :cas [29 [19 0]] INFO [2019-12-17 12:33:03,649] jepsen worker 88 - jepsen.util 6658 :fail :cas [29 [7 12]] :ValueNotSwapFrom INFO [2019-12-17 12:33:03,650] jepsen worker 88 - jepsen.util 6658 :invoke :read [29 nil] INFO [2019-12-17 12:33:04,463] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38277,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:33:04,463] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38281,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:33:04,464] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38289,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:33:04,464] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38270,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:33:04,464] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38276,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:33:04,464] jepsen worker 42 - jepsen.util 9492 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:33:04,465] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38281,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:33:04,465] jepsen worker 75 - jepsen.util 8535 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:33:04,465] jepsen worker 0 - jepsen.util 9270 :info :cas [0 [28 17]] :RequestCanceledException INFO [2019-12-17 12:33:04,465] jepsen worker 45 - jepsen.util 6525 :info :cas [15 [31 27]] :RequestCanceledException INFO [2019-12-17 12:33:04,465] jepsen worker 31 - jepsen.util 5881 :ok :read [10 42] INFO [2019-12-17 12:33:04,465] jepsen worker 77 - jepsen.util 9347 :info :write [25 6] :RequestCanceledException INFO [2019-12-17 12:33:04,465] jepsen worker 21 - jepsen.util 8391 :info :cas [7 [16 48]] :RequestCanceledException INFO [2019-12-17 12:33:04,465] jepsen worker 32 - jepsen.util 6692 :info :write [10 4] :RequestCanceledException INFO [2019-12-17 12:33:04,465] jepsen worker 50 - jepsen.util 9590 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:33:04,465] jepsen worker 19 - jepsen.util 6499 :info :write [6 23] :RequestCanceledException INFO [2019-12-17 12:33:04,465] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38289,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:33:04,465] jepsen worker 38 - jepsen.util 6608 :fail :cas [12 [48 23]] :ValueNotSwapFrom INFO [2019-12-17 12:33:04,465] jepsen worker 70 - jepsen.util 6910 :info :write [23 7] :RequestCanceledException INFO [2019-12-17 12:33:04,465] jepsen worker 66 - jepsen.util 8796 :info :cas [22 [40 28]] :RequestCanceledException INFO [2019-12-17 12:33:04,466] jepsen worker 25 - jepsen.util 9475 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:33:04,466] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38277,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:33:04,466] jepsen worker 29 - jepsen.util 4979 :info :write [9 17] :RequestCanceledException INFO [2019-12-17 12:33:04,466] jepsen worker 34 - jepsen.util 8314 :info :write [11 40] :RequestCanceledException INFO [2019-12-17 12:33:04,466] jepsen worker 12 - jepsen.util 7032 :info :write [4 43] :RequestCanceledException INFO [2019-12-17 12:33:04,466] jepsen worker 67 - jepsen.util 9247 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:33:04,466] jepsen worker 22 - jepsen.util 8932 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:33:04,466] jepsen worker 64 - jepsen.util 5014 :info :write [21 17] :RequestCanceledException INFO [2019-12-17 12:33:04,466] jepsen worker 27 - jepsen.util 6597 :info :write [9 17] :RequestCanceledException INFO [2019-12-17 12:33:04,466] jepsen worker 28 - jepsen.util 4978 :info :write [9 39] :RequestCanceledException INFO [2019-12-17 12:33:04,466] jepsen worker 58 - jepsen.util 9418 :info :cas [19 [5 36]] :RequestCanceledException INFO [2019-12-17 12:33:04,467] jepsen worker 63 - jepsen.util 5553 :info :write [21 33] :RequestCanceledException INFO [2019-12-17 12:33:04,467] jepsen worker 15 - jepsen.util 7845 :info :write [5 20] :RequestCanceledException INFO [2019-12-17 12:33:04,467] jepsen worker 30 - jepsen.util 6330 :info :write [10 7] :RequestCanceledException INFO [2019-12-17 12:33:04,467] jepsen worker 43 - jepsen.util 8233 :info :cas [14 [37 28]] :RequestCanceledException INFO [2019-12-17 12:33:04,467] jepsen worker 62 - jepsen.util 8882 :info :write [20 41] :RequestCanceledException INFO [2019-12-17 12:33:04,467] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38270,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:33:04,467] jepsen worker 86 - jepsen.util 9356 :info :cas [28 [2 10]] :RequestCanceledException INFO [2019-12-17 12:33:04,467] jepsen worker 72 - jepsen.util 6372 :info :write [24 36] :RequestCanceledException INFO [2019-12-17 12:33:04,467] jepsen worker 65 - jepsen.util 5285 :info :write [21 38] :RequestCanceledException INFO [2019-12-17 12:33:04,467] jepsen worker 51 - jepsen.util 6441 :ok :write [17 47] INFO [2019-12-17 12:33:04,468] jepsen worker 24 - jepsen.util 8124 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:33:04,468] jepsen worker 26 - jepsen.util 9386 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:33:04,468] jepsen worker 54 - jepsen.util 7254 :info :write [18 43] :RequestCanceledException INFO [2019-12-17 12:33:04,468] jepsen worker 18 - jepsen.util 6498 :info :write [6 49] :RequestCanceledException INFO [2019-12-17 12:33:04,468] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38285,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:33:04,468] jepsen worker 83 - jepsen.util 9353 :info :write [27 17] :RequestCanceledException INFO [2019-12-17 12:33:04,468] jepsen worker 11 - jepsen.util 6941 :info :write [3 29] :RequestCanceledException INFO [2019-12-17 12:33:04,468] jepsen worker 1 - jepsen.util 8821 :info :cas [0 [29 11]] :RequestCanceledException INFO [2019-12-17 12:33:04,468] jepsen worker 82 - jepsen.util 9622 :info :write [27 23] :RequestCanceledException INFO [2019-12-17 12:33:04,468] jepsen worker 33 - jepsen.util 8673 :info :write [11 29] :RequestCanceledException INFO [2019-12-17 12:33:04,468] jepsen worker 60 - jepsen.util 9780 :info :write [20 43] :RequestCanceledException INFO [2019-12-17 12:33:04,468] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38276,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:33:04,468] jepsen worker 23 - jepsen.util 8933 :info :cas [7 [43 32]] :RequestCanceledException INFO [2019-12-17 12:33:04,468] jepsen worker 68 - jepsen.util 8528 :info :cas [22 [27 28]] :RequestCanceledException INFO [2019-12-17 12:33:04,468] jepsen worker 69 - jepsen.util 6639 :info :write [23 12] :RequestCanceledException INFO [2019-12-17 12:33:04,468] jepsen worker 41 - jepsen.util 9491 :info :cas [13 [20 46]] :RequestCanceledException INFO [2019-12-17 12:33:04,468] jepsen worker 42 - jepsen.util 9492 :invoke :cas [14 [22 7]] INFO [2019-12-17 12:33:04,468] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38291,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:33:04,468] jepsen worker 75 - jepsen.util 8535 :invoke :write [25 21] INFO [2019-12-17 12:33:04,469] jepsen worker 59 - jepsen.util 9149 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:33:04,469] jepsen worker 31 - jepsen.util 5881 :invoke :read [10 nil] INFO [2019-12-17 12:33:04,469] jepsen worker 50 - jepsen.util 9590 :invoke :read [16 nil] INFO [2019-12-17 12:33:04,469] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38305,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:33:04,470] jepsen worker 38 - jepsen.util 6608 :invoke :cas [12 [32 1]] INFO [2019-12-17 12:33:04,470] jepsen worker 25 - jepsen.util 9475 :invoke :read [8 nil] INFO [2019-12-17 12:33:04,470] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38287,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:33:04,472] jepsen worker 67 - jepsen.util 9247 :invoke :read [22 nil] INFO [2019-12-17 12:33:04,472] jepsen worker 22 - jepsen.util 8932 :invoke :write [7 46] INFO [2019-12-17 12:33:04,473] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38273,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:33:04,474] jepsen worker 51 - jepsen.util 6441 :invoke :cas [17 [25 16]] INFO [2019-12-17 12:33:04,474] jepsen worker 24 - jepsen.util 8124 :invoke :read [8 nil] INFO [2019-12-17 12:33:04,475] jepsen worker 26 - jepsen.util 9386 :invoke :read [8 nil] INFO [2019-12-17 12:33:04,475] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38285,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:33:04,479] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38288,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:33:04,481] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38291,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:33:04,482] jepsen worker 59 - jepsen.util 9149 :invoke :read [19 nil] INFO [2019-12-17 12:33:04,483] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38305,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:33:04,484] jepsen worker 45 - jepsen.util 6615 :invoke :cas [15 [11 16]] INFO [2019-12-17 12:33:04,485] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38287,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:33:04,486] jepsen worker 77 - jepsen.util 9437 :invoke :cas [25 [10 21]] INFO [2019-12-17 12:33:04,487] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38273,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:33:04,487] jepsen worker 32 - jepsen.util 6782 :invoke :write [10 45] INFO [2019-12-17 12:33:04,490] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38295,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:33:04,490] jepsen worker 12 - jepsen.util 7122 :invoke :write [4 37] INFO [2019-12-17 12:33:04,490] jepsen worker 34 - jepsen.util 8404 :invoke :read [11 nil] INFO [2019-12-17 12:33:04,491] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38288,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:33:04,491] jepsen worker 27 - jepsen.util 6687 :invoke :write [9 24] INFO [2019-12-17 12:33:04,491] jepsen worker 29 - jepsen.util 5069 :invoke :read [9 nil] INFO [2019-12-17 12:33:04,491] jepsen worker 0 - jepsen.util 9360 :invoke :write [0 11] INFO [2019-12-17 12:33:04,492] jepsen worker 21 - jepsen.util 8481 :invoke :cas [7 [29 49]] INFO [2019-12-17 12:33:04,492] jepsen worker 70 - jepsen.util 7000 :invoke :write [23 20] INFO [2019-12-17 12:33:04,492] jepsen worker 58 - jepsen.util 9508 :invoke :cas [19 [8 19]] INFO [2019-12-17 12:33:04,493] jepsen worker 28 - jepsen.util 5068 :invoke :write [9 39] INFO [2019-12-17 12:33:04,493] jepsen worker 19 - jepsen.util 6589 :invoke :cas [6 [22 26]] INFO [2019-12-17 12:33:04,493] jepsen worker 64 - jepsen.util 5104 :invoke :write [21 45] INFO [2019-12-17 12:33:04,493] jepsen worker 66 - jepsen.util 8886 :invoke :cas [22 [18 39]] INFO [2019-12-17 12:33:04,493] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38296,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:33:04,494] jepsen worker 63 - jepsen.util 5643 :invoke :write [21 38] INFO [2019-12-17 12:33:04,494] jepsen worker 15 - jepsen.util 7935 :invoke :write [5 10] INFO [2019-12-17 12:33:04,495] jepsen worker 86 - jepsen.util 9446 :invoke :write [28 15] INFO [2019-12-17 12:33:04,495] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38295,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:33:04,495] jepsen worker 11 - jepsen.util 7031 :invoke :read [3 nil] INFO [2019-12-17 12:33:04,495] jepsen worker 62 - jepsen.util 8972 :invoke :cas [20 [34 2]] INFO [2019-12-17 12:33:04,496] jepsen worker 54 - jepsen.util 7344 :invoke :cas [18 [32 37]] INFO [2019-12-17 12:33:04,496] jepsen worker 72 - jepsen.util 6462 :invoke :write [24 36] INFO [2019-12-17 12:33:04,496] jepsen worker 43 - jepsen.util 8323 :invoke :cas [14 [22 12]] INFO [2019-12-17 12:33:04,496] jepsen worker 18 - jepsen.util 6588 :invoke :write [6 22] INFO [2019-12-17 12:33:04,497] jepsen worker 30 - jepsen.util 6420 :invoke :write [10 1] INFO [2019-12-17 12:33:04,497] jepsen worker 33 - jepsen.util 8763 :invoke :cas [11 [24 46]] INFO [2019-12-17 12:33:04,497] jepsen worker 69 - jepsen.util 6729 :invoke :write [23 48] INFO [2019-12-17 12:33:04,497] jepsen worker 65 - jepsen.util 5375 :invoke :write [21 32] INFO [2019-12-17 12:33:04,497] jepsen worker 41 - jepsen.util 9581 :invoke :write [13 48] INFO [2019-12-17 12:33:04,498] jepsen worker 60 - jepsen.util 9870 :invoke :write [20 0] INFO [2019-12-17 12:33:04,498] jepsen worker 1 - jepsen.util 8911 :invoke :cas [0 [7 18]] INFO [2019-12-17 12:33:04,499] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38296,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:33:04,500] jepsen worker 23 - jepsen.util 9023 :invoke :read [7 nil] INFO [2019-12-17 12:33:04,500] jepsen worker 82 - jepsen.util 9712 :invoke :write [27 16] INFO [2019-12-17 12:33:04,500] jepsen worker 83 - jepsen.util 9443 :invoke :cas [27 [1 44]] INFO [2019-12-17 12:33:04,502] jepsen worker 68 - jepsen.util 8618 :invoke :read [22 nil] INFO [2019-12-17 12:33:04,546] jepsen worker 37 - jepsen.util 6427 :fail :cas [12 [7 28]] :ValueNotSwapFrom INFO [2019-12-17 12:33:04,548] jepsen worker 37 - jepsen.util 6427 :invoke :cas [12 [0 38]] INFO [2019-12-17 12:33:04,590] jepsen worker 4 - jepsen.util 6034 :fail :cas [1 [14 30]] :ValueNotSwapFrom INFO [2019-12-17 12:33:04,590] jepsen worker 9 - jepsen.util 7119 :ok :read [3 27] INFO [2019-12-17 12:33:04,590] jepsen worker 4 - jepsen.util 6034 :invoke :read [1 nil] INFO [2019-12-17 12:33:04,591] jepsen worker 9 - jepsen.util 7119 :invoke :cas [3 [10 16]] INFO [2019-12-17 12:33:04,610] jepsen worker 20 - jepsen.util 6320 :info :write [6 0] :RequestCanceledException INFO [2019-12-17 12:33:04,613] jepsen worker 20 - jepsen.util 6410 :invoke :read [6 nil] INFO [2019-12-17 12:33:04,645] jepsen worker 5 - jepsen.util 6305 :ok :read [1 26] INFO [2019-12-17 12:33:04,645] jepsen worker 5 - jepsen.util 6305 :invoke :read [1 nil] INFO [2019-12-17 12:33:04,645] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:33:04,645] jepsen nemesis - couchbase.util Rebalance check count: 293 INFO [2019-12-17 12:33:04,645] jepsen nemesis - couchbase.util Rebalance stuck count: 274 INFO [2019-12-17 12:33:04,649] jepsen worker 13 - jepsen.util 6403 :fail :cas [4 [46 46]] :ValueNotSwapFrom INFO [2019-12-17 12:33:04,649] jepsen worker 13 - jepsen.util 6403 :invoke :read [4 nil] INFO [2019-12-17 12:33:04,650] jepsen worker 55 - jepsen.util 7705 :ok :read [18 25] INFO [2019-12-17 12:33:04,650] jepsen worker 55 - jepsen.util 7705 :invoke :cas [18 [0 33]] INFO [2019-12-17 12:33:04,651] jepsen worker 79 - jepsen.util 6739 :ok :read [26 45] INFO [2019-12-17 12:33:04,652] jepsen worker 79 - jepsen.util 6739 :invoke :cas [26 [6 11]] INFO [2019-12-17 12:33:04,658] jepsen worker 56 - jepsen.util 7166 :info :write [18 39] :RequestCanceledException INFO [2019-12-17 12:33:04,661] jepsen worker 56 - jepsen.util 7256 :invoke :cas [18 [44 20]] INFO [2019-12-17 12:33:04,692] jepsen worker 31 - jepsen.util 5881 :ok :read [10 42] INFO [2019-12-17 12:33:04,692] jepsen worker 31 - jepsen.util 5881 :invoke :cas [10 [0 21]] INFO [2019-12-17 12:33:04,706] jepsen worker 45 - jepsen.util 6615 :fail :cas [15 [11 16]] :ValueNotSwapFrom INFO [2019-12-17 12:33:04,706] jepsen worker 45 - jepsen.util 6615 :invoke :cas [15 [3 11]] INFO [2019-12-17 12:33:04,707] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38358,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:33:04,707] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38358,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:33:04,707] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38361,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:33:04,708] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38361,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:33:04,708] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38362,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:33:04,708] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38362,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:33:04,712] jepsen worker 74 - jepsen.util 6734 :info :write [24 5] :RequestCanceledException INFO [2019-12-17 12:33:04,715] jepsen worker 74 - jepsen.util 6824 :invoke :cas [24 [12 9]] INFO [2019-12-17 12:33:04,724] jepsen worker 52 - jepsen.util 6982 :ok :read [17 47] INFO [2019-12-17 12:33:04,724] jepsen worker 52 - jepsen.util 6982 :invoke :read [17 nil] INFO [2019-12-17 12:33:04,728] jepsen worker 84 - jepsen.util 8364 :info :write [28 6] :RequestCanceledException INFO [2019-12-17 12:33:04,730] jepsen worker 84 - jepsen.util 8454 :invoke :cas [28 [9 40]] INFO [2019-12-17 12:33:04,742] jepsen worker 76 - jepsen.util 9256 :info :cas [25 [33 28]] :RequestCanceledException INFO [2019-12-17 12:33:04,745] jepsen worker 76 - jepsen.util 9346 :invoke :cas [25 [29 45]] INFO [2019-12-17 12:33:04,769] jepsen worker 31 - jepsen.util 5881 :fail :cas [10 [0 21]] :ValueNotSwapFrom INFO [2019-12-17 12:33:04,769] jepsen worker 31 - jepsen.util 5881 :invoke :write [10 48] INFO [2019-12-17 12:33:04,772] jepsen worker 29 - jepsen.util 5069 :ok :read [9 17] INFO [2019-12-17 12:33:04,772] jepsen worker 29 - jepsen.util 5069 :invoke :read [9 nil] INFO [2019-12-17 12:33:04,789] jepsen worker 80 - jepsen.util 6650 :fail :cas [26 [43 27]] :ValueNotSwapFrom INFO [2019-12-17 12:33:04,794] jepsen worker 47 - jepsen.util 7067 :info :write [15 36] :RequestCanceledException INFO [2019-12-17 12:33:04,808] jepsen worker 89 - jepsen.util 6389 :ok :read [29 20] INFO [2019-12-17 12:33:04,840] jepsen worker 46 - jepsen.util 6706 :info :write [15 41] :RequestCanceledException INFO [2019-12-17 12:33:04,852] jepsen worker 71 - jepsen.util 7361 :ok :read [23 48] INFO [2019-12-17 12:33:04,860] jepsen worker 19 - jepsen.util 6589 :fail :cas [6 [22 26]] :ValueNotSwapFrom INFO [2019-12-17 12:33:04,884] jepsen worker 4 - jepsen.util 6034 :ok :read [1 26] INFO [2019-12-17 12:33:04,891] jepsen worker 39 - jepsen.util 8859 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:33:04,906] jepsen worker 11 - jepsen.util 7031 :ok :read [3 27] INFO [2019-12-17 12:33:04,912] jepsen worker 81 - jepsen.util 8721 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:33:04,918] jepsen worker 80 - jepsen.util 6650 :invoke :cas [26 [44 42]] INFO [2019-12-17 12:33:04,918] jepsen worker 89 - jepsen.util 6389 :invoke :cas [29 [27 20]] INFO [2019-12-17 12:33:04,918] jepsen worker 78 - jepsen.util 6918 :ok :write [26 19] INFO [2019-12-17 12:33:04,918] jepsen worker 19 - jepsen.util 6589 :invoke :write [6 45] INFO [2019-12-17 12:33:04,918] jepsen worker 71 - jepsen.util 7361 :invoke :cas [23 [28 39]] INFO [2019-12-17 12:33:04,921] jepsen worker 47 - jepsen.util 7157 :invoke :write [15 18] INFO [2019-12-17 12:33:04,921] jepsen worker 46 - jepsen.util 6796 :invoke :read [15 nil] INFO [2019-12-17 12:33:04,921] jepsen worker 79 - jepsen.util 6739 :fail :cas [26 [6 11]] :ValueNotSwapFrom INFO [2019-12-17 12:33:04,923] jepsen worker 4 - jepsen.util 6034 :invoke :read [1 nil] INFO [2019-12-17 12:33:04,923] jepsen worker 39 - jepsen.util 8859 :invoke :write [13 9] INFO [2019-12-17 12:33:04,923] jepsen worker 81 - jepsen.util 8721 :invoke :read [27 nil] INFO [2019-12-17 12:33:04,923] jepsen worker 11 - jepsen.util 7031 :invoke :write [3 36] INFO [2019-12-17 12:33:04,923] jepsen worker 78 - jepsen.util 6918 :invoke :write [26 15] INFO [2019-12-17 12:33:04,924] jepsen worker 79 - jepsen.util 6739 :invoke :cas [26 [5 41]] INFO [2019-12-17 12:33:04,928] jepsen worker 6 - jepsen.util 9546 :info :write [2 26] :RequestCanceledException INFO [2019-12-17 12:33:04,931] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38377,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:33:04,932] jepsen worker 6 - jepsen.util 9636 :invoke :cas [2 [43 39]] INFO [2019-12-17 12:33:04,959] jepsen worker 44 - jepsen.util 9854 :info :write [14 25] :RequestCanceledException INFO [2019-12-17 12:33:04,974] jepsen worker 9 - jepsen.util 7119 :fail :cas [3 [10 16]] :ValueNotSwapFrom INFO [2019-12-17 12:33:04,988] jepsen worker 36 - jepsen.util 7146 :ok :write [12 0] INFO [2019-12-17 12:33:04,997] jepsen worker 80 - jepsen.util 6650 :fail :cas [26 [44 42]] :ValueNotSwapFrom INFO [2019-12-17 12:33:05,024] jepsen worker 51 - jepsen.util 6441 :fail :cas [17 [25 16]] :ValueNotSwapFrom INFO [2019-12-17 12:33:05,031] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38377,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:33:05,032] jepsen worker 9 - jepsen.util 7119 :invoke :read [3 nil] INFO [2019-12-17 12:33:05,032] jepsen worker 36 - jepsen.util 7146 :invoke :cas [12 [29 48]] INFO [2019-12-17 12:33:05,032] jepsen worker 80 - jepsen.util 6650 :invoke :read [26 nil] INFO [2019-12-17 12:33:05,032] jepsen worker 51 - jepsen.util 6441 :invoke :read [17 nil] INFO [2019-12-17 12:33:05,032] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38380,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:33:05,032] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38380,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:33:05,033] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38381,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:33:05,033] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38381,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:33:05,040] jepsen worker 44 - jepsen.util 9944 :invoke :read [14 nil] INFO [2019-12-17 12:33:05,047] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38394,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:33:05,047] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38394,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:33:05,047] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38395,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:33:05,047] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38395,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:33:05,047] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38396,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:33:05,047] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38396,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:33:05,069] jepsen worker 53 - jepsen.util 6353 :ok :read [17 47] INFO [2019-12-17 12:33:05,069] jepsen worker 53 - jepsen.util 6353 :invoke :write [17 23] INFO [2019-12-17 12:33:05,078] jepsen worker 14 - jepsen.util 6854 :ok :write [4 42] INFO [2019-12-17 12:33:05,079] jepsen worker 14 - jepsen.util 6854 :invoke :write [4 33] INFO [2019-12-17 12:33:05,102] jepsen worker 89 - jepsen.util 6389 :fail :cas [29 [27 20]] :ValueNotSwapFrom INFO [2019-12-17 12:33:05,102] jepsen worker 89 - jepsen.util 6389 :invoke :read [29 nil] INFO [2019-12-17 12:33:05,112] jepsen worker 54 - jepsen.util 7344 :fail :cas [18 [32 37]] :ValueNotSwapFrom INFO [2019-12-17 12:33:05,112] jepsen worker 40 - jepsen.util 9400 :info :cas [13 [44 39]] :RequestCanceledException INFO [2019-12-17 12:33:05,112] jepsen worker 17 - jepsen.util 9197 :info :cas [5 [45 4]] :RequestCanceledException INFO [2019-12-17 12:33:05,112] jepsen worker 54 - jepsen.util 7344 :invoke :read [18 nil] INFO [2019-12-17 12:33:05,115] jepsen worker 17 - jepsen.util 9287 :invoke :cas [5 [13 2]] INFO [2019-12-17 12:33:05,115] jepsen worker 40 - jepsen.util 9490 :invoke :cas [13 [34 49]] INFO [2019-12-17 12:33:05,128] jepsen worker 8 - jepsen.util 8828 :info :cas [2 [33 13]] :RequestCanceledException INFO [2019-12-17 12:33:05,130] jepsen worker 8 - jepsen.util 8918 :invoke :read [2 nil] INFO [2019-12-17 12:33:05,142] jepsen worker 48 - jepsen.util 8328 :info :write [16 17] :RequestCanceledException INFO [2019-12-17 12:33:05,142] jepsen worker 7 - jepsen.util 9367 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:33:05,143] jepsen worker 7 - jepsen.util 9367 :invoke :write [2 30] INFO [2019-12-17 12:33:05,145] jepsen worker 48 - jepsen.util 8418 :invoke :read [16 nil] INFO [2019-12-17 12:33:05,155] jepsen worker 85 - jepsen.util 10075 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:33:05,155] jepsen worker 57 - jepsen.util 8697 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:33:05,156] jepsen worker 85 - jepsen.util 10075 :invoke :write [28 36] INFO [2019-12-17 12:33:05,156] jepsen worker 57 - jepsen.util 8697 :invoke :cas [19 [4 42]] INFO [2019-12-17 12:33:05,171] jepsen worker 71 - jepsen.util 7361 :fail :cas [23 [28 39]] :ValueNotSwapFrom INFO [2019-12-17 12:33:05,171] jepsen worker 71 - jepsen.util 7361 :invoke :cas [23 [5 29]] INFO [2019-12-17 12:33:05,174] jepsen worker 78 - jepsen.util 6918 :ok :write [26 15] INFO [2019-12-17 12:33:05,174] jepsen worker 78 - jepsen.util 6918 :invoke :cas [26 [20 19]] INFO [2019-12-17 12:33:05,190] jepsen worker 4 - jepsen.util 6034 :ok :read [1 26] INFO [2019-12-17 12:33:05,190] jepsen worker 4 - jepsen.util 6034 :invoke :cas [1 [32 2]] INFO [2019-12-17 12:33:05,191] jepsen worker 9 - jepsen.util 7119 :ok :read [3 27] INFO [2019-12-17 12:33:05,191] jepsen worker 9 - jepsen.util 7119 :invoke :cas [3 [32 13]] INFO [2019-12-17 12:33:05,194] jepsen worker 61 - jepsen.util 9961 :info :write [20 28] :RequestCanceledException INFO [2019-12-17 12:33:05,197] jepsen worker 61 - jepsen.util 10051 :invoke :cas [20 [42 25]] INFO [2019-12-17 12:33:05,201] jepsen worker 79 - jepsen.util 6739 :fail :cas [26 [5 41]] :ValueNotSwapFrom INFO [2019-12-17 12:33:05,201] jepsen worker 79 - jepsen.util 6739 :invoke :read [26 nil] INFO [2019-12-17 12:33:05,212] jepsen worker 10 - jepsen.util 6400 :info :cas [3 [27 11]] :RequestCanceledException INFO [2019-12-17 12:33:05,214] jepsen worker 10 - jepsen.util 6490 :invoke :write [3 41] INFO [2019-12-17 12:33:05,228] jepsen worker 49 - jepsen.util 8239 :info :cas [16 [46 31]] :RequestCanceledException INFO [2019-12-17 12:33:05,231] jepsen worker 49 - jepsen.util 8329 :invoke :read [16 nil] INFO [2019-12-17 12:33:05,245] jepsen worker 80 - jepsen.util 6650 :ok :read [26 15] INFO [2019-12-17 12:33:05,245] jepsen worker 80 - jepsen.util 6650 :invoke :cas [26 [13 49]] INFO [2019-12-17 12:33:05,257] jepsen worker 89 - jepsen.util 6389 :ok :read [29 20] INFO [2019-12-17 12:33:05,257] jepsen worker 89 - jepsen.util 6389 :invoke :write [29 23] INFO [2019-12-17 12:33:05,264] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38422,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:33:05,264] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38422,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:33:05,264] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38430,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:33:05,264] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38423,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:33:05,265] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38430,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:33:05,265] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38423,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:33:05,265] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38433,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:33:05,265] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38433,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:33:05,265] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38434,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:33:05,265] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38434,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:33:05,270] jepsen worker 53 - jepsen.util 6353 :ok :write [17 23] INFO [2019-12-17 12:33:05,270] jepsen worker 53 - jepsen.util 6353 :invoke :read [17 nil] INFO [2019-12-17 12:33:05,278] jepsen worker 71 - jepsen.util 7361 :fail :cas [23 [5 29]] :ValueNotSwapFrom INFO [2019-12-17 12:33:05,278] jepsen worker 71 - jepsen.util 7361 :invoke :write [23 9] INFO [2019-12-17 12:33:05,306] jepsen worker 9 - jepsen.util 7119 :fail :cas [3 [32 13]] :ValueNotSwapFrom INFO [2019-12-17 12:33:05,311] jepsen worker 16 - jepsen.util 10006 :info :write [5 26] :RequestCanceledException INFO [2019-12-17 12:33:05,325] jepsen worker 38 - jepsen.util 6608 :fail :cas [12 [32 1]] :ValueNotSwapFrom INFO [2019-12-17 12:33:05,328] jepsen worker 35 - jepsen.util 8855 :info :cas [11 [34 12]] :RequestCanceledException INFO [2019-12-17 12:33:05,341] jepsen worker 4 - jepsen.util 6034 :fail :cas [1 [32 2]] :ValueNotSwapFrom INFO [2019-12-17 12:33:05,343] jepsen worker 13 - jepsen.util 6403 :ok :read [4 42] INFO [2019-12-17 12:33:05,358] jepsen worker 73 - jepsen.util 6193 :info :write [24 5] :RequestCanceledException INFO [2019-12-17 12:33:05,358] jepsen worker 5 - jepsen.util 6305 :ok :read [1 26] INFO [2019-12-17 12:33:05,372] jepsen worker 79 - jepsen.util 6739 :ok :read [26 15] INFO [2019-12-17 12:33:05,375] jepsen worker 78 - jepsen.util 6918 :fail :cas [26 [20 19]] :ValueNotSwapFrom INFO [2019-12-17 12:33:05,385] jepsen worker 53 - jepsen.util 6353 :ok :read [17 23] INFO [2019-12-17 12:33:05,394] jepsen worker 9 - jepsen.util 7119 :invoke :read [3 nil] INFO [2019-12-17 12:33:05,394] jepsen worker 38 - jepsen.util 6608 :invoke :read [12 nil] INFO [2019-12-17 12:33:05,394] jepsen worker 4 - jepsen.util 6034 :invoke :write [1 39] INFO [2019-12-17 12:33:05,394] jepsen worker 13 - jepsen.util 6403 :invoke :write [4 21] INFO [2019-12-17 12:33:05,394] jepsen worker 5 - jepsen.util 6305 :invoke :write [1 37] INFO [2019-12-17 12:33:05,394] jepsen worker 79 - jepsen.util 6739 :invoke :write [26 14] INFO [2019-12-17 12:33:05,394] jepsen worker 78 - jepsen.util 6918 :invoke :write [26 48] INFO [2019-12-17 12:33:05,395] jepsen worker 53 - jepsen.util 6353 :invoke :write [17 44] INFO [2019-12-17 12:33:05,398] jepsen worker 35 - jepsen.util 8945 :invoke :write [11 28] INFO [2019-12-17 12:33:05,398] jepsen worker 16 - jepsen.util 10096 :invoke :cas [5 [10 19]] INFO [2019-12-17 12:33:05,399] jepsen worker 73 - jepsen.util 6283 :invoke :read [24 nil] INFO [2019-12-17 12:33:05,475] jepsen worker 80 - jepsen.util 6650 :fail :cas [26 [13 49]] :ValueNotSwapFrom INFO [2019-12-17 12:33:05,475] jepsen worker 80 - jepsen.util 6650 :invoke :write [26 5] INFO [2019-12-17 12:33:05,482] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38455,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:33:05,483] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38455,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:33:05,500] jepsen worker 88 - jepsen.util 6658 :ok :read [29 20] INFO [2019-12-17 12:33:05,501] jepsen worker 88 - jepsen.util 6658 :invoke :write [29 17] INFO [2019-12-17 12:33:05,540] jepsen worker 9 - jepsen.util 7119 :ok :read [3 27] INFO [2019-12-17 12:33:05,540] jepsen worker 9 - jepsen.util 7119 :invoke :cas [3 [36 29]] INFO [2019-12-17 12:33:05,583] jepsen worker 38 - jepsen.util 6608 :ok :read [12 0] INFO [2019-12-17 12:33:05,583] jepsen worker 38 - jepsen.util 6608 :invoke :write [12 35] INFO [2019-12-17 12:33:05,612] jepsen worker 3 - jepsen.util 6483 :ok :write [1 13] INFO [2019-12-17 12:33:05,613] jepsen worker 3 - jepsen.util 6483 :invoke :read [1 nil] INFO [2019-12-17 12:33:05,619] jepsen worker 36 - jepsen.util 7146 :fail :cas [12 [29 48]] :ValueNotSwapFrom INFO [2019-12-17 12:33:05,619] jepsen worker 36 - jepsen.util 7146 :invoke :write [12 33] INFO [2019-12-17 12:33:05,636] jepsen worker 73 - jepsen.util 6283 :ok :read [24 27] INFO [2019-12-17 12:33:05,636] jepsen worker 73 - jepsen.util 6283 :invoke :read [24 nil] INFO [2019-12-17 12:33:05,637] jepsen worker 51 - jepsen.util 6441 :ok :read [17 23] INFO [2019-12-17 12:33:05,637] jepsen worker 51 - jepsen.util 6441 :invoke :cas [17 [2 30]] INFO [2019-12-17 12:33:05,638] jepsen worker 9 - jepsen.util 7119 :fail :cas [3 [36 29]] :ValueNotSwapFrom INFO [2019-12-17 12:33:05,639] jepsen worker 9 - jepsen.util 7119 :invoke :write [3 41] INFO [2019-12-17 12:33:05,642] jepsen worker 89 - jepsen.util 6389 :ok :write [29 23] INFO [2019-12-17 12:33:05,642] jepsen worker 89 - jepsen.util 6389 :invoke :read [29 nil] INFO [2019-12-17 12:33:05,668] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:33:05,668] jepsen nemesis - couchbase.util Rebalance check count: 294 INFO [2019-12-17 12:33:05,668] jepsen nemesis - couchbase.util Rebalance stuck count: 275 INFO [2019-12-17 12:33:05,669] jepsen worker 54 - jepsen.util 7344 :ok :read [18 25] INFO [2019-12-17 12:33:05,670] jepsen worker 54 - jepsen.util 7344 :invoke :write [18 5] INFO [2019-12-17 12:33:05,674] jepsen worker 73 - jepsen.util 6283 :ok :read [24 27] INFO [2019-12-17 12:33:05,674] jepsen worker 73 - jepsen.util 6283 :invoke :read [24 nil] INFO [2019-12-17 12:33:05,690] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38465,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:33:05,691] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38465,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:33:05,691] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38469,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:33:05,691] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38469,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:33:05,693] jepsen worker 2 - jepsen.util 8732 :info :write [0 41] :RequestCanceledException INFO [2019-12-17 12:33:05,696] jepsen worker 2 - jepsen.util 8822 :invoke :read [0 nil] INFO [2019-12-17 12:33:05,705] jepsen worker 78 - jepsen.util 6918 :ok :write [26 48] INFO [2019-12-17 12:33:05,705] jepsen worker 87 - jepsen.util 5757 :info :cas [29 [19 0]] :RequestCanceledException INFO [2019-12-17 12:33:05,705] jepsen worker 78 - jepsen.util 6918 :invoke :cas [26 [35 20]] INFO [2019-12-17 12:33:05,707] jepsen worker 87 - jepsen.util 5847 :invoke :cas [29 [28 23]] INFO [2019-12-17 12:33:05,722] jepsen worker 73 - jepsen.util 6283 :ok :read [24 27] INFO [2019-12-17 12:33:05,723] jepsen worker 73 - jepsen.util 6283 :invoke :read [24 nil] INFO [2019-12-17 12:33:05,748] jepsen worker 73 - jepsen.util 6283 :ok :read [24 27] INFO [2019-12-17 12:33:05,816] jepsen worker 73 - jepsen.util 6283 :invoke :read [24 nil] INFO [2019-12-17 12:33:05,847] jepsen worker 73 - jepsen.util 6283 :ok :read [24 27] INFO [2019-12-17 12:33:05,847] jepsen worker 73 - jepsen.util 6283 :invoke :read [24 nil] INFO [2019-12-17 12:33:05,877] jepsen worker 73 - jepsen.util 6283 :ok :read [24 27] INFO [2019-12-17 12:33:05,877] jepsen worker 73 - jepsen.util 6283 :invoke :cas [24 [40 6]] INFO [2019-12-17 12:33:05,880] jepsen worker 4 - jepsen.util 6034 :ok :write [1 39] INFO [2019-12-17 12:33:05,881] jepsen worker 4 - jepsen.util 6034 :invoke :write [1 20] INFO [2019-12-17 12:33:05,908] jepsen worker 73 - jepsen.util 6283 :fail :cas [24 [40 6]] :ValueNotSwapFrom INFO [2019-12-17 12:33:05,908] jepsen worker 73 - jepsen.util 6283 :invoke :write [24 49] INFO [2019-12-17 12:33:05,909] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38483,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:33:05,910] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38483,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:33:05,910] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38487,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:33:05,910] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38487,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:33:05,977] jepsen worker 53 - jepsen.util 6353 :ok :write [17 44] INFO [2019-12-17 12:33:05,977] jepsen worker 53 - jepsen.util 6353 :invoke :read [17 nil] INFO [2019-12-17 12:33:06,003] jepsen worker 56 - jepsen.util 7256 :fail :cas [18 [44 20]] :ValueNotSwapFrom INFO [2019-12-17 12:33:06,003] jepsen worker 56 - jepsen.util 7256 :invoke :write [18 23] INFO [2019-12-17 12:33:06,021] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38495,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:33:06,021] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38495,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:33:06,021] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38497,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:33:06,021] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38497,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:33:06,021] jepsen worker 3 - jepsen.util 6483 :ok :read [1 39] INFO [2019-12-17 12:33:06,021] jepsen worker 3 - jepsen.util 6483 :invoke :cas [1 [25 21]] INFO [2019-12-17 12:33:06,047] jepsen worker 5 - jepsen.util 6305 :ok :write [1 37] INFO [2019-12-17 12:33:06,047] jepsen worker 5 - jepsen.util 6305 :invoke :read [1 nil] INFO [2019-12-17 12:33:06,099] jepsen worker 80 - jepsen.util 6650 :ok :write [26 5] INFO [2019-12-17 12:33:06,100] jepsen worker 80 - jepsen.util 6650 :invoke :read [26 nil] INFO [2019-12-17 12:33:06,148] jepsen worker 89 - jepsen.util 6389 :ok :read [29 23] INFO [2019-12-17 12:33:06,149] jepsen worker 89 - jepsen.util 6389 :invoke :read [29 nil] INFO [2019-12-17 12:33:06,175] jepsen worker 52 - jepsen.util 6982 :ok :read [17 44] INFO [2019-12-17 12:33:06,239] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38503,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:33:06,243] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38510,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:33:06,243] jepsen worker 13 - jepsen.util 6403 :ok :write [4 21] INFO [2019-12-17 12:33:06,258] jepsen worker 56 - jepsen.util 7256 :ok :write [18 23] INFO [2019-12-17 12:33:06,274] jepsen worker 12 - jepsen.util 7122 :ok :write [4 37] INFO [2019-12-17 12:33:06,284] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38503,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:33:06,284] jepsen worker 52 - jepsen.util 6982 :invoke :read [17 nil] INFO [2019-12-17 12:33:06,284] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38510,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:33:06,284] jepsen worker 13 - jepsen.util 6403 :invoke :write [4 18] INFO [2019-12-17 12:33:06,284] jepsen worker 56 - jepsen.util 7256 :invoke :cas [18 [35 20]] INFO [2019-12-17 12:33:06,284] jepsen worker 12 - jepsen.util 7122 :invoke :write [4 15] INFO [2019-12-17 12:33:06,285] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38504,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:33:06,285] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38514,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:33:06,285] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38504,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:33:06,285] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38514,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:33:06,285] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38505,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:33:06,285] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38505,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:33:06,304] jepsen worker 3 - jepsen.util 6483 :fail :cas [1 [25 21]] :ValueNotSwapFrom INFO [2019-12-17 12:33:06,304] jepsen worker 3 - jepsen.util 6483 :invoke :read [1 nil] INFO [2019-12-17 12:33:06,324] jepsen worker 56 - jepsen.util 7256 :fail :cas [18 [35 20]] :ValueNotSwapFrom INFO [2019-12-17 12:33:06,324] jepsen worker 56 - jepsen.util 7256 :invoke :read [18 nil] INFO [2019-12-17 12:33:06,341] jepsen worker 69 - jepsen.util 6729 :ok :write [23 48] INFO [2019-12-17 12:33:06,341] jepsen worker 69 - jepsen.util 6729 :invoke :cas [23 [45 45]] INFO [2019-12-17 12:33:06,387] jepsen worker 3 - jepsen.util 6483 :ok :read [1 37] INFO [2019-12-17 12:33:06,387] jepsen worker 3 - jepsen.util 6483 :invoke :read [1 nil] INFO [2019-12-17 12:33:06,430] jepsen worker 14 - jepsen.util 6854 :ok :write [4 33] INFO [2019-12-17 12:33:06,431] jepsen worker 14 - jepsen.util 6854 :invoke :write [4 19] INFO [2019-12-17 12:33:06,457] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38525,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:33:06,457] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38525,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:33:06,458] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38529,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:33:06,458] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38529,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:33:06,460] jepsen worker 56 - jepsen.util 7256 :ok :read [18 23] INFO [2019-12-17 12:33:06,461] jepsen worker 56 - jepsen.util 7256 :invoke :write [18 21] INFO [2019-12-17 12:33:06,464] jepsen worker 79 - jepsen.util 6739 :ok :write [26 14] INFO [2019-12-17 12:33:06,466] jepsen worker 79 - jepsen.util 6739 :invoke :cas [26 [10 4]] INFO [2019-12-17 12:33:06,504] jepsen worker 87 - jepsen.util 5847 :fail :cas [29 [28 23]] :ValueNotSwapFrom INFO [2019-12-17 12:33:06,505] jepsen worker 87 - jepsen.util 5847 :invoke :write [29 5] INFO [2019-12-17 12:33:06,511] jepsen worker 70 - jepsen.util 7000 :ok :write [23 20] INFO [2019-12-17 12:33:06,511] jepsen worker 70 - jepsen.util 7000 :invoke :cas [23 [1 7]] INFO [2019-12-17 12:33:06,512] jepsen worker 50 - jepsen.util 9590 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:33:06,512] jepsen worker 50 - jepsen.util 9590 :invoke :read [16 nil] INFO [2019-12-17 12:33:06,527] jepsen worker 25 - jepsen.util 9475 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:33:06,527] jepsen worker 67 - jepsen.util 9247 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:33:06,527] jepsen worker 24 - jepsen.util 8124 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:33:06,527] jepsen worker 58 - jepsen.util 9508 :info :cas [19 [8 19]] :RequestCanceledException INFO [2019-12-17 12:33:06,530] jepsen worker 34 - jepsen.util 8404 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:33:06,530] jepsen worker 67 - jepsen.util 9247 :invoke :cas [22 [49 2]] INFO [2019-12-17 12:33:06,530] jepsen worker 25 - jepsen.util 9475 :invoke :write [8 17] INFO [2019-12-17 12:33:06,530] jepsen worker 24 - jepsen.util 8124 :invoke :read [8 nil] INFO [2019-12-17 12:33:06,530] jepsen worker 34 - jepsen.util 8404 :invoke :cas [11 [11 26]] INFO [2019-12-17 12:33:06,530] jepsen worker 59 - jepsen.util 9149 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:33:06,531] jepsen worker 62 - jepsen.util 8972 :info :cas [20 [34 2]] :RequestCanceledException INFO [2019-12-17 12:33:06,531] jepsen worker 59 - jepsen.util 9149 :invoke :read [19 nil] INFO [2019-12-17 12:33:06,531] jepsen worker 68 - jepsen.util 8618 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:33:06,531] jepsen worker 33 - jepsen.util 8763 :info :cas [11 [24 46]] :RequestCanceledException INFO [2019-12-17 12:33:06,531] jepsen worker 23 - jepsen.util 9023 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:33:06,531] jepsen worker 68 - jepsen.util 8618 :invoke :read [22 nil] INFO [2019-12-17 12:33:06,531] jepsen worker 23 - jepsen.util 9023 :invoke :write [7 16] INFO [2019-12-17 12:33:06,531] jepsen worker 77 - jepsen.util 9437 :info :cas [25 [10 21]] :RequestCanceledException INFO [2019-12-17 12:33:06,532] jepsen worker 43 - jepsen.util 8323 :info :cas [14 [22 12]] :RequestCanceledException INFO [2019-12-17 12:33:06,535] jepsen worker 58 - jepsen.util 9598 :invoke :write [19 35] INFO [2019-12-17 12:33:06,538] jepsen worker 62 - jepsen.util 9062 :invoke :write [20 41] INFO [2019-12-17 12:33:06,539] jepsen worker 33 - jepsen.util 8853 :invoke :read [11 nil] INFO [2019-12-17 12:33:06,539] jepsen worker 77 - jepsen.util 9527 :invoke :read [25 nil] INFO [2019-12-17 12:33:06,539] jepsen worker 43 - jepsen.util 8413 :invoke :write [14 45] INFO [2019-12-17 12:33:06,541] jepsen worker 42 - jepsen.util 9492 :info :cas [14 [22 7]] :RequestCanceledException INFO [2019-12-17 12:33:06,541] jepsen worker 21 - jepsen.util 8481 :info :cas [7 [29 49]] :RequestCanceledException INFO [2019-12-17 12:33:06,542] jepsen worker 66 - jepsen.util 8886 :info :cas [22 [18 39]] :RequestCanceledException INFO [2019-12-17 12:33:06,548] jepsen worker 21 - jepsen.util 8571 :invoke :read [7 nil] INFO [2019-12-17 12:33:06,548] jepsen worker 66 - jepsen.util 8976 :invoke :read [22 nil] INFO [2019-12-17 12:33:06,549] jepsen worker 42 - jepsen.util 9582 :invoke :read [14 nil] INFO [2019-12-17 12:33:06,559] jepsen worker 83 - jepsen.util 9443 :info :cas [27 [1 44]] :RequestCanceledException INFO [2019-12-17 12:33:06,566] jepsen worker 83 - jepsen.util 9533 :invoke :cas [27 [11 25]] INFO [2019-12-17 12:33:06,569] jepsen worker 3 - jepsen.util 6483 :ok :read [1 37] INFO [2019-12-17 12:33:06,569] jepsen worker 3 - jepsen.util 6483 :invoke :cas [1 [40 4]] INFO [2019-12-17 12:33:06,594] jepsen worker 26 - jepsen.util 9386 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:33:06,594] jepsen worker 27 - jepsen.util 6687 :info :write [9 24] :RequestCanceledException INFO [2019-12-17 12:33:06,594] jepsen worker 28 - jepsen.util 5068 :info :write [9 39] :RequestCanceledException INFO [2019-12-17 12:33:06,594] jepsen worker 86 - jepsen.util 9446 :info :write [28 15] :RequestCanceledException INFO [2019-12-17 12:33:06,594] jepsen worker 18 - jepsen.util 6588 :info :write [6 22] :RequestCanceledException INFO [2019-12-17 12:33:06,594] jepsen worker 26 - jepsen.util 9386 :invoke :cas [8 [30 45]] INFO [2019-12-17 12:33:06,594] jepsen worker 60 - jepsen.util 9870 :info :write [20 0] :RequestCanceledException INFO [2019-12-17 12:33:06,596] jepsen worker 28 - jepsen.util 5158 :invoke :write [9 25] INFO [2019-12-17 12:33:06,598] jepsen worker 27 - jepsen.util 6777 :invoke :cas [9 [10 48]] INFO [2019-12-17 12:33:06,599] jepsen worker 18 - jepsen.util 6678 :invoke :read [6 nil] INFO [2019-12-17 12:33:06,600] jepsen worker 86 - jepsen.util 9536 :invoke :cas [28 [33 44]] INFO [2019-12-17 12:33:06,600] jepsen worker 60 - jepsen.util 9960 :invoke :read [20 nil] INFO [2019-12-17 12:33:06,612] jepsen worker 1 - jepsen.util 8911 :info :cas [0 [7 18]] :RequestCanceledException INFO [2019-12-17 12:33:06,612] jepsen worker 22 - jepsen.util 8932 :info :write [7 46] :RequestCanceledException INFO [2019-12-17 12:33:06,612] jepsen worker 30 - jepsen.util 6420 :info :write [10 1] :RequestCanceledException INFO [2019-12-17 12:33:06,616] jepsen worker 30 - jepsen.util 6510 :invoke :cas [10 [32 37]] INFO [2019-12-17 12:33:06,616] jepsen worker 22 - jepsen.util 9022 :invoke :read [7 nil] INFO [2019-12-17 12:33:06,617] jepsen worker 1 - jepsen.util 9001 :invoke :write [0 25] INFO [2019-12-17 12:33:06,625] jepsen worker 64 - jepsen.util 5104 :info :write [21 45] :RequestCanceledException INFO [2019-12-17 12:33:06,625] jepsen worker 75 - jepsen.util 8535 :info :write [25 21] :RequestCanceledException INFO [2019-12-17 12:33:06,625] jepsen worker 15 - jepsen.util 7935 :info :write [5 10] :RequestCanceledException INFO [2019-12-17 12:33:06,625] jepsen worker 0 - jepsen.util 9360 :info :write [0 11] :RequestCanceledException INFO [2019-12-17 12:33:06,625] jepsen worker 41 - jepsen.util 9581 :info :write [13 48] :RequestCanceledException INFO [2019-12-17 12:33:06,629] jepsen worker 41 - jepsen.util 9671 :invoke :read [13 nil] INFO [2019-12-17 12:33:06,629] jepsen worker 0 - jepsen.util 9450 :invoke :cas [0 [34 9]] INFO [2019-12-17 12:33:06,629] jepsen worker 75 - jepsen.util 8625 :invoke :read [25 nil] INFO [2019-12-17 12:33:06,630] jepsen worker 64 - jepsen.util 5194 :invoke :read [21 nil] INFO [2019-12-17 12:33:06,630] jepsen worker 15 - jepsen.util 8025 :invoke :cas [5 [43 38]] INFO [2019-12-17 12:33:06,641] jepsen worker 32 - jepsen.util 6782 :info :write [10 45] :RequestCanceledException INFO [2019-12-17 12:33:06,641] jepsen worker 82 - jepsen.util 9712 :info :write [27 16] :RequestCanceledException INFO [2019-12-17 12:33:06,645] jepsen worker 32 - jepsen.util 6872 :invoke :write [10 19] INFO [2019-12-17 12:33:06,646] jepsen worker 82 - jepsen.util 9802 :invoke :read [27 nil] INFO [2019-12-17 12:33:06,656] jepsen worker 63 - jepsen.util 5643 :info :write [21 38] :RequestCanceledException INFO [2019-12-17 12:33:06,656] jepsen worker 72 - jepsen.util 6462 :info :write [24 36] :RequestCanceledException INFO [2019-12-17 12:33:06,656] jepsen worker 65 - jepsen.util 5375 :info :write [21 32] :RequestCanceledException INFO [2019-12-17 12:33:06,661] jepsen worker 63 - jepsen.util 5733 :invoke :write [21 45] INFO [2019-12-17 12:33:06,662] jepsen worker 72 - jepsen.util 6552 :invoke :cas [24 [36 28]] INFO [2019-12-17 12:33:06,662] jepsen worker 65 - jepsen.util 5465 :invoke :cas [21 [26 27]] INFO [2019-12-17 12:33:06,667] jepsen worker 27 - jepsen.util 6777 :fail :cas [9 [10 48]] :ValueNotSwapFrom INFO [2019-12-17 12:33:06,668] jepsen worker 27 - jepsen.util 6777 :invoke :write [9 17] INFO [2019-12-17 12:33:06,669] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38567,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:33:06,669] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38567,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:33:06,669] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38572,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:33:06,670] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38572,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:33:06,690] jepsen worker 20 - jepsen.util 6410 :fail :read [6 nil] :RequestTimeoutException INFO [2019-12-17 12:33:06,690] jepsen worker 20 - jepsen.util 6410 :invoke :read [6 nil] INFO [2019-12-17 12:33:06,705] jepsen worker 30 - jepsen.util 6510 :fail :cas [10 [32 37]] :ValueNotSwapFrom INFO [2019-12-17 12:33:06,705] jepsen worker 30 - jepsen.util 6510 :invoke :read [10 nil] INFO [2019-12-17 12:33:06,719] jepsen worker 18 - jepsen.util 6678 :ok :read [6 5] INFO [2019-12-17 12:33:06,719] jepsen worker 55 - jepsen.util 7705 :fail :cas [18 [0 33]] :ValueNotSwapFrom INFO [2019-12-17 12:33:06,720] jepsen worker 55 - jepsen.util 7705 :invoke :cas [18 [24 40]] INFO [2019-12-17 12:33:06,720] jepsen worker 18 - jepsen.util 6678 :invoke :cas [6 [15 3]] INFO [2019-12-17 12:33:06,727] jepsen worker 64 - jepsen.util 5194 :ok :read [21 7] INFO [2019-12-17 12:33:06,728] jepsen worker 64 - jepsen.util 5194 :invoke :cas [21 [26 2]] INFO [2019-12-17 12:33:06,728] jepsen worker 74 - jepsen.util 6824 :info :cas [24 [12 9]] :RequestCanceledException INFO [2019-12-17 12:33:06,730] jepsen worker 74 - jepsen.util 6914 :invoke :cas [24 [25 24]] INFO [2019-12-17 12:33:06,741] jepsen worker 88 - jepsen.util 6658 :ok :write [29 17] INFO [2019-12-17 12:33:06,741] jepsen worker 88 - jepsen.util 6658 :invoke :cas [29 [34 7]] INFO [2019-12-17 12:33:06,741] jepsen worker 84 - jepsen.util 8454 :info :cas [28 [9 40]] :RequestCanceledException INFO [2019-12-17 12:33:06,743] jepsen worker 84 - jepsen.util 8544 :invoke :read [28 nil] INFO [2019-12-17 12:33:06,744] jepsen worker 4 - jepsen.util 6034 :ok :write [1 20] INFO [2019-12-17 12:33:06,744] jepsen worker 4 - jepsen.util 6034 :invoke :cas [1 [31 45]] INFO [2019-12-17 12:33:06,753] jepsen worker 65 - jepsen.util 5465 :fail :cas [21 [26 27]] :ValueNotSwapFrom INFO [2019-12-17 12:33:06,754] jepsen worker 65 - jepsen.util 5465 :invoke :cas [21 [24 10]] INFO [2019-12-17 12:33:06,757] jepsen worker 76 - jepsen.util 9346 :info :cas [25 [29 45]] :RequestCanceledException INFO [2019-12-17 12:33:06,759] jepsen worker 76 - jepsen.util 9436 :invoke :cas [25 [33 25]] INFO [2019-12-17 12:33:06,765] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:33:06,784] jepsen worker 30 - jepsen.util 6510 :ok :read [10 42] INFO [2019-12-17 12:33:06,789] jepsen worker 52 - jepsen.util 6982 :ok :read [17 44] INFO [2019-12-17 12:33:06,805] jepsen worker 36 - jepsen.util 7146 :ok :write [12 33] INFO [2019-12-17 12:33:06,812] jepsen worker 45 - jepsen.util 6615 :info :cas [15 [3 11]] :RequestCanceledException INFO [2019-12-17 12:33:06,812] jepsen worker 29 - jepsen.util 5069 :fail :read [9 nil] :RequestTimeoutException INFO [2019-12-17 12:33:06,836] jepsen worker 28 - jepsen.util 5158 :ok :write [9 25] INFO [2019-12-17 12:33:06,840] jepsen worker 31 - jepsen.util 5881 :info :write [10 48] :RequestCanceledException INFO [2019-12-17 12:33:06,847] jepsen worker 72 - jepsen.util 6552 :fail :cas [24 [36 28]] :ValueNotSwapFrom INFO [2019-12-17 12:33:06,847] jepsen worker 18 - jepsen.util 6678 :fail :cas [6 [15 3]] :ValueNotSwapFrom INFO [2019-12-17 12:33:06,862] jepsen worker 51 - jepsen.util 6441 :fail :cas [17 [2 30]] :ValueNotSwapFrom INFO [2019-12-17 12:33:06,886] jepsen worker 20 - jepsen.util 6410 :ok :read [6 5] INFO [2019-12-17 12:33:06,888] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38594,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:33:06,898] jepsen nemesis - couchbase.util Rebalance check count: 295 INFO [2019-12-17 12:33:06,898] jepsen worker 30 - jepsen.util 6510 :invoke :write [10 8] INFO [2019-12-17 12:33:06,898] jepsen worker 52 - jepsen.util 6982 :invoke :write [17 15] INFO [2019-12-17 12:33:06,898] jepsen worker 36 - jepsen.util 7146 :invoke :cas [12 [41 3]] INFO [2019-12-17 12:33:06,898] jepsen worker 29 - jepsen.util 5069 :invoke :read [9 nil] INFO [2019-12-17 12:33:06,898] jepsen worker 28 - jepsen.util 5158 :invoke :write [9 11] INFO [2019-12-17 12:33:06,899] jepsen worker 72 - jepsen.util 6552 :invoke :cas [24 [37 43]] INFO [2019-12-17 12:33:06,899] jepsen worker 18 - jepsen.util 6678 :invoke :read [6 nil] INFO [2019-12-17 12:33:06,899] jepsen worker 51 - jepsen.util 6441 :invoke :write [17 46] INFO [2019-12-17 12:33:06,899] jepsen worker 20 - jepsen.util 6410 :invoke :read [6 nil] INFO [2019-12-17 12:33:06,899] jepsen nemesis - couchbase.util Rebalance stuck count: 276 INFO [2019-12-17 12:33:06,899] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38594,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:33:06,906] jepsen worker 45 - jepsen.util 6705 :invoke :read [15 nil] INFO [2019-12-17 12:33:06,907] jepsen worker 31 - jepsen.util 5971 :invoke :read [10 nil] INFO [2019-12-17 12:33:07,036] jepsen worker 46 - jepsen.util 6796 :fail :read [15 nil] :RequestTimeoutException INFO [2019-12-17 12:33:07,036] jepsen worker 39 - jepsen.util 8859 :info :write [13 9] :RequestCanceledException INFO [2019-12-17 12:33:07,036] jepsen worker 11 - jepsen.util 7031 :info :write [3 36] :RequestCanceledException INFO [2019-12-17 12:33:07,036] jepsen worker 81 - jepsen.util 8721 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:33:07,036] jepsen worker 19 - jepsen.util 6589 :info :write [6 45] :RequestCanceledException INFO [2019-12-17 12:33:07,036] jepsen worker 12 - jepsen.util 7122 :ok :write [4 15] INFO [2019-12-17 12:33:07,036] jepsen worker 46 - jepsen.util 6796 :invoke :cas [15 [23 4]] INFO [2019-12-17 12:33:07,036] jepsen worker 81 - jepsen.util 8721 :invoke :write [27 37] INFO [2019-12-17 12:33:07,037] jepsen worker 47 - jepsen.util 7157 :info :write [15 18] :RequestCanceledException INFO [2019-12-17 12:33:07,037] jepsen worker 12 - jepsen.util 7122 :invoke :cas [4 [26 22]] INFO [2019-12-17 12:33:07,037] jepsen worker 11 - jepsen.util 7121 :invoke :read [3 nil] INFO [2019-12-17 12:33:07,038] jepsen worker 47 - jepsen.util 7247 :invoke :write [15 42] INFO [2019-12-17 12:33:07,038] jepsen worker 39 - jepsen.util 8949 :invoke :cas [13 [22 23]] INFO [2019-12-17 12:33:07,038] jepsen worker 19 - jepsen.util 6679 :invoke :cas [6 [11 38]] INFO [2019-12-17 12:33:07,049] jepsen worker 74 - jepsen.util 6914 :fail :cas [24 [25 24]] :ValueNotSwapFrom INFO [2019-12-17 12:33:07,049] jepsen worker 74 - jepsen.util 6914 :invoke :read [24 nil] INFO [2019-12-17 12:33:07,051] jepsen worker 65 - jepsen.util 5465 :fail :cas [21 [24 10]] :ValueNotSwapFrom INFO [2019-12-17 12:33:07,051] jepsen worker 65 - jepsen.util 5465 :invoke :write [21 11] INFO [2019-12-17 12:33:07,051] jepsen worker 64 - jepsen.util 5194 :fail :cas [21 [26 2]] :ValueNotSwapFrom INFO [2019-12-17 12:33:07,051] jepsen worker 64 - jepsen.util 5194 :invoke :write [21 49] INFO [2019-12-17 12:33:07,054] jepsen worker 18 - jepsen.util 6678 :ok :read [6 5] INFO [2019-12-17 12:33:07,054] jepsen worker 18 - jepsen.util 6678 :invoke :write [6 25] INFO [2019-12-17 12:33:07,055] jepsen worker 72 - jepsen.util 6552 :fail :cas [24 [37 43]] :ValueNotSwapFrom INFO [2019-12-17 12:33:07,055] jepsen worker 72 - jepsen.util 6552 :invoke :write [24 27] INFO [2019-12-17 12:33:07,056] jepsen worker 80 - jepsen.util 6650 :ok :read [26 14] INFO [2019-12-17 12:33:07,056] jepsen worker 80 - jepsen.util 6650 :invoke :cas [26 [19 2]] INFO [2019-12-17 12:33:07,061] jepsen worker 20 - jepsen.util 6410 :ok :read [6 5] INFO [2019-12-17 12:33:07,061] jepsen worker 20 - jepsen.util 6410 :invoke :cas [6 [17 32]] INFO [2019-12-17 12:33:07,070] jepsen worker 89 - jepsen.util 6389 :ok :read [29 17] INFO [2019-12-17 12:33:07,071] jepsen worker 89 - jepsen.util 6389 :invoke :write [29 4] INFO [2019-12-17 12:33:07,072] jepsen worker 56 - jepsen.util 7256 :ok :write [18 21] INFO [2019-12-17 12:33:07,072] jepsen worker 56 - jepsen.util 7256 :invoke :write [18 40] INFO [2019-12-17 12:33:07,072] jepsen worker 14 - jepsen.util 6854 :ok :write [4 19] INFO [2019-12-17 12:33:07,072] jepsen worker 14 - jepsen.util 6854 :invoke :write [4 2] INFO [2019-12-17 12:33:07,096] jepsen worker 70 - jepsen.util 7000 :fail :cas [23 [1 7]] :ValueNotSwapFrom INFO [2019-12-17 12:33:07,096] jepsen worker 70 - jepsen.util 7000 :invoke :read [23 nil] INFO [2019-12-17 12:33:07,112] jepsen worker 6 - jepsen.util 9636 :info :cas [2 [43 39]] :RequestCanceledException INFO [2019-12-17 12:33:07,113] jepsen worker 6 - jepsen.util 9726 :invoke :cas [2 [29 13]] INFO [2019-12-17 12:33:07,128] jepsen worker 44 - jepsen.util 9944 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:33:07,128] jepsen worker 44 - jepsen.util 9944 :invoke :write [14 17] INFO [2019-12-17 12:33:07,140] jepsen worker 17 - jepsen.util 9287 :info :cas [5 [13 2]] :RequestCanceledException INFO [2019-12-17 12:33:07,141] jepsen worker 17 - jepsen.util 9377 :invoke :write [5 46] INFO [2019-12-17 12:33:07,145] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38604,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:33:07,146] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38604,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:33:07,146] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38607,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:33:07,146] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38607,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:33:07,146] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38608,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:33:07,146] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38608,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:33:07,152] jepsen worker 63 - jepsen.util 5733 :ok :write [21 45] INFO [2019-12-17 12:33:07,153] jepsen worker 63 - jepsen.util 5733 :invoke :cas [21 [37 29]] INFO [2019-12-17 12:33:07,158] jepsen worker 40 - jepsen.util 9490 :info :cas [13 [34 49]] :RequestCanceledException INFO [2019-12-17 12:33:07,160] jepsen worker 40 - jepsen.util 9580 :invoke :write [13 12] INFO [2019-12-17 12:33:07,193] jepsen worker 55 - jepsen.util 7705 :fail :cas [18 [24 40]] :ValueNotSwapFrom INFO [2019-12-17 12:33:07,193] jepsen worker 55 - jepsen.util 7705 :invoke :read [18 nil] INFO [2019-12-17 12:33:07,194] jepsen worker 48 - jepsen.util 8418 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:33:07,194] jepsen worker 48 - jepsen.util 8418 :invoke :read [16 nil] INFO [2019-12-17 12:33:07,196] jepsen worker 31 - jepsen.util 5971 :ok :read [10 42] INFO [2019-12-17 12:33:07,196] jepsen worker 31 - jepsen.util 5971 :invoke :read [10 nil] INFO [2019-12-17 12:33:07,212] jepsen worker 61 - jepsen.util 10051 :info :cas [20 [42 25]] :RequestCanceledException INFO [2019-12-17 12:33:07,214] jepsen worker 61 - jepsen.util 10141 :invoke :write [20 6] INFO [2019-12-17 12:33:07,217] jepsen worker 12 - jepsen.util 7122 :fail :cas [4 [26 22]] :ValueNotSwapFrom INFO [2019-12-17 12:33:07,217] jepsen worker 12 - jepsen.util 7122 :invoke :read [4 nil] INFO [2019-12-17 12:33:07,221] jepsen worker 37 - jepsen.util 6427 :fail :cas [12 [0 38]] :CASMismatchException INFO [2019-12-17 12:33:07,221] jepsen worker 37 - jepsen.util 6427 :invoke :cas [12 [16 13]] INFO [2019-12-17 12:33:07,225] jepsen worker 70 - jepsen.util 7000 :ok :read [23 20] INFO [2019-12-17 12:33:07,225] jepsen worker 70 - jepsen.util 7000 :invoke :write [23 13] INFO [2019-12-17 12:33:07,227] jepsen worker 8 - jepsen.util 8918 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:33:07,227] jepsen worker 8 - jepsen.util 8918 :invoke :cas [2 [17 27]] INFO [2019-12-17 12:33:07,233] jepsen worker 38 - jepsen.util 6608 :ok :write [12 35] INFO [2019-12-17 12:33:07,233] jepsen worker 38 - jepsen.util 6608 :invoke :cas [12 [48 21]] INFO [2019-12-17 12:33:07,241] jepsen worker 3 - jepsen.util 6483 :fail :cas [1 [40 4]] :ValueNotSwapFrom INFO [2019-12-17 12:33:07,241] jepsen worker 3 - jepsen.util 6483 :invoke :cas [1 [29 22]] INFO [2019-12-17 12:33:07,241] jepsen worker 7 - jepsen.util 9367 :info :write [2 30] :RequestCanceledException INFO [2019-12-17 12:33:07,242] jepsen worker 7 - jepsen.util 9457 :invoke :read [2 nil] INFO [2019-12-17 12:33:07,246] jepsen worker 4 - jepsen.util 6034 :fail :cas [1 [31 45]] :ValueNotSwapFrom INFO [2019-12-17 12:33:07,246] jepsen worker 4 - jepsen.util 6034 :invoke :read [1 nil] INFO [2019-12-17 12:33:07,252] jepsen worker 11 - jepsen.util 7121 :ok :read [3 27] INFO [2019-12-17 12:33:07,252] jepsen worker 11 - jepsen.util 7121 :invoke :read [3 nil] INFO [2019-12-17 12:33:07,255] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38638,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:33:07,255] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38648,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:33:07,255] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38638,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:33:07,255] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38648,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:33:07,255] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38639,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:33:07,255] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38639,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:33:07,255] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38640,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:33:07,256] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38640,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:33:07,258] jepsen worker 57 - jepsen.util 8697 :info :cas [19 [4 42]] :RequestCanceledException INFO [2019-12-17 12:33:07,258] jepsen worker 85 - jepsen.util 10075 :info :write [28 36] :RequestCanceledException INFO [2019-12-17 12:33:07,260] jepsen worker 57 - jepsen.util 8787 :invoke :write [19 30] INFO [2019-12-17 12:33:07,260] jepsen worker 85 - jepsen.util 10165 :invoke :write [28 10] INFO [2019-12-17 12:33:07,263] jepsen worker 78 - jepsen.util 6918 :fail :cas [26 [35 20]] :ValueNotSwapFrom INFO [2019-12-17 12:33:07,263] jepsen worker 78 - jepsen.util 6918 :invoke :read [26 nil] INFO [2019-12-17 12:33:07,269] jepsen worker 53 - jepsen.util 6353 :ok :read [17 44] INFO [2019-12-17 12:33:07,269] jepsen worker 53 - jepsen.util 6353 :invoke :cas [17 [20 49]] INFO [2019-12-17 12:33:07,284] jepsen worker 3 - jepsen.util 6483 :fail :cas [1 [29 22]] :ValueNotSwapFrom INFO [2019-12-17 12:33:07,284] jepsen worker 3 - jepsen.util 6483 :invoke :write [1 36] INFO [2019-12-17 12:33:07,321] jepsen worker 79 - jepsen.util 6739 :fail :cas [26 [10 4]] :ValueNotSwapFrom INFO [2019-12-17 12:33:07,322] jepsen worker 79 - jepsen.util 6739 :invoke :cas [26 [31 46]] INFO [2019-12-17 12:33:07,327] jepsen worker 10 - jepsen.util 6490 :info :write [3 41] :RequestCanceledException INFO [2019-12-17 12:33:07,327] jepsen worker 71 - jepsen.util 7361 :info :write [23 9] :RequestCanceledException INFO [2019-12-17 12:33:07,328] jepsen worker 10 - jepsen.util 6580 :invoke :cas [3 [33 47]] INFO [2019-12-17 12:33:07,329] jepsen worker 71 - jepsen.util 7451 :invoke :write [23 35] INFO [2019-12-17 12:33:07,340] jepsen worker 49 - jepsen.util 8329 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:33:07,341] jepsen worker 49 - jepsen.util 8329 :invoke :write [16 47] INFO [2019-12-17 12:33:07,352] jepsen worker 55 - jepsen.util 7705 :ok :read [18 21] INFO [2019-12-17 12:33:07,352] jepsen worker 55 - jepsen.util 7705 :invoke :cas [18 [28 48]] INFO [2019-12-17 12:33:07,359] jepsen worker 45 - jepsen.util 6705 :ok :read [15 41] INFO [2019-12-17 12:33:07,359] jepsen worker 45 - jepsen.util 6705 :invoke :read [15 nil] INFO [2019-12-17 12:33:07,365] jepsen worker 12 - jepsen.util 7122 :ok :read [4 18] INFO [2019-12-17 12:33:07,365] jepsen worker 12 - jepsen.util 7122 :invoke :read [4 nil] INFO [2019-12-17 12:33:07,377] jepsen worker 54 - jepsen.util 7344 :ok :write [18 5] INFO [2019-12-17 12:33:07,377] jepsen worker 54 - jepsen.util 7344 :invoke :write [18 12] INFO [2019-12-17 12:33:07,400] jepsen worker 63 - jepsen.util 5733 :fail :cas [21 [37 29]] :ValueNotSwapFrom INFO [2019-12-17 12:33:07,400] jepsen worker 63 - jepsen.util 5733 :invoke :write [21 38] INFO [2019-12-17 12:33:07,401] jepsen worker 37 - jepsen.util 6427 :fail :cas [12 [16 13]] :ValueNotSwapFrom INFO [2019-12-17 12:33:07,402] jepsen worker 37 - jepsen.util 6427 :invoke :read [12 nil] INFO [2019-12-17 12:33:07,422] jepsen worker 45 - jepsen.util 6705 :ok :read [15 41] INFO [2019-12-17 12:33:07,422] jepsen worker 45 - jepsen.util 6705 :invoke :write [15 31] INFO [2019-12-17 12:33:07,428] jepsen worker 16 - jepsen.util 10096 :info :cas [5 [10 19]] :RequestCanceledException INFO [2019-12-17 12:33:07,429] jepsen worker 13 - jepsen.util 6403 :ok :write [4 18] INFO [2019-12-17 12:33:07,429] jepsen worker 13 - jepsen.util 6403 :invoke :read [4 nil] INFO [2019-12-17 12:33:07,430] jepsen worker 16 - jepsen.util 10186 :invoke :write [5 36] INFO [2019-12-17 12:33:07,436] jepsen worker 19 - jepsen.util 6679 :fail :cas [6 [11 38]] :ValueNotSwapFrom INFO [2019-12-17 12:33:07,436] jepsen worker 19 - jepsen.util 6679 :invoke :read [6 nil] INFO [2019-12-17 12:33:07,436] jepsen worker 55 - jepsen.util 7705 :fail :cas [18 [28 48]] :ValueNotSwapFrom INFO [2019-12-17 12:33:07,436] jepsen worker 55 - jepsen.util 7705 :invoke :write [18 49] INFO [2019-12-17 12:33:07,438] jepsen worker 46 - jepsen.util 6796 :fail :cas [15 [23 4]] :ValueNotSwapFrom INFO [2019-12-17 12:33:07,439] jepsen worker 46 - jepsen.util 6796 :invoke :cas [15 [16 38]] INFO [2019-12-17 12:33:07,458] jepsen worker 35 - jepsen.util 8945 :info :write [11 28] :RequestCanceledException INFO [2019-12-17 12:33:07,459] jepsen worker 88 - jepsen.util 6658 :fail :cas [29 [34 7]] :ValueNotSwapFrom INFO [2019-12-17 12:33:07,459] jepsen worker 88 - jepsen.util 6658 :invoke :read [29 nil] INFO [2019-12-17 12:33:07,459] jepsen worker 35 - jepsen.util 9035 :invoke :read [11 nil] INFO [2019-12-17 12:33:07,469] jepsen worker 80 - jepsen.util 6650 :fail :cas [26 [19 2]] :ValueNotSwapFrom INFO [2019-12-17 12:33:07,469] jepsen worker 80 - jepsen.util 6650 :invoke :read [26 nil] INFO [2019-12-17 12:33:07,474] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38678,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:33:07,474] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38678,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:33:07,475] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38682,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:33:07,475] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38682,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:33:07,475] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38683,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:33:07,475] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38683,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:33:07,496] jepsen worker 12 - jepsen.util 7122 :ok :read [4 18] INFO [2019-12-17 12:33:07,497] jepsen worker 12 - jepsen.util 7122 :invoke :read [4 nil] INFO [2019-12-17 12:33:07,504] jepsen worker 5 - jepsen.util 6305 :ok :read [1 20] INFO [2019-12-17 12:33:07,504] jepsen worker 5 - jepsen.util 6305 :invoke :read [1 nil] INFO [2019-12-17 12:33:07,505] jepsen worker 63 - jepsen.util 5733 :ok :write [21 38] INFO [2019-12-17 12:33:07,506] jepsen worker 63 - jepsen.util 5733 :invoke :cas [21 [32 27]] INFO [2019-12-17 12:33:07,526] jepsen worker 63 - jepsen.util 5733 :fail :cas [21 [32 27]] :ValueNotSwapFrom INFO [2019-12-17 12:33:07,526] jepsen worker 63 - jepsen.util 5733 :invoke :write [21 47] INFO [2019-12-17 12:33:07,534] jepsen worker 12 - jepsen.util 7122 :ok :read [4 18] INFO [2019-12-17 12:33:07,534] jepsen worker 12 - jepsen.util 7122 :invoke :cas [4 [19 18]] INFO [2019-12-17 12:33:07,551] jepsen worker 3 - jepsen.util 6483 :ok :write [1 36] INFO [2019-12-17 12:33:07,551] jepsen worker 3 - jepsen.util 6483 :invoke :write [1 10] INFO [2019-12-17 12:33:07,552] jepsen worker 38 - jepsen.util 6608 :fail :cas [12 [48 21]] :ValueNotSwapFrom INFO [2019-12-17 12:33:07,553] jepsen worker 38 - jepsen.util 6608 :invoke :cas [12 [23 15]] INFO [2019-12-17 12:33:07,579] jepsen worker 72 - jepsen.util 6552 :ok :write [24 27] INFO [2019-12-17 12:33:07,579] jepsen worker 72 - jepsen.util 6552 :invoke :read [24 nil] INFO [2019-12-17 12:33:07,580] jepsen worker 12 - jepsen.util 7122 :fail :cas [4 [19 18]] :ValueNotSwapFrom INFO [2019-12-17 12:33:07,580] jepsen worker 12 - jepsen.util 7122 :invoke :write [4 21] INFO [2019-12-17 12:33:07,590] jepsen worker 78 - jepsen.util 6918 :ok :read [26 14] INFO [2019-12-17 12:33:07,590] jepsen worker 78 - jepsen.util 6918 :invoke :read [26 nil] INFO [2019-12-17 12:33:07,592] jepsen worker 32 - jepsen.util 6872 :ok :write [10 19] INFO [2019-12-17 12:33:07,592] jepsen worker 32 - jepsen.util 6872 :invoke :cas [10 [24 20]] INFO [2019-12-17 12:33:07,593] jepsen worker 28 - jepsen.util 5158 :ok :write [9 11] INFO [2019-12-17 12:33:07,593] jepsen worker 28 - jepsen.util 5158 :invoke :write [9 16] INFO [2019-12-17 12:33:07,602] jepsen worker 18 - jepsen.util 6678 :ok :write [6 25] INFO [2019-12-17 12:33:07,602] jepsen worker 18 - jepsen.util 6678 :invoke :write [6 44] INFO [2019-12-17 12:33:07,611] jepsen worker 10 - jepsen.util 6580 :fail :cas [3 [33 47]] :ValueNotSwapFrom INFO [2019-12-17 12:33:07,611] jepsen worker 10 - jepsen.util 6580 :invoke :write [3 8] INFO [2019-12-17 12:33:07,625] jepsen worker 53 - jepsen.util 6353 :fail :cas [17 [20 49]] :ValueNotSwapFrom INFO [2019-12-17 12:33:07,625] jepsen worker 53 - jepsen.util 6353 :invoke :cas [17 [1 7]] INFO [2019-12-17 12:33:07,630] jepsen worker 13 - jepsen.util 6403 :ok :read [4 18] INFO [2019-12-17 12:33:07,631] jepsen worker 13 - jepsen.util 6403 :invoke :write [4 48] INFO [2019-12-17 12:33:07,639] jepsen worker 69 - jepsen.util 6729 :fail :cas [23 [45 45]] :ValueNotSwapFrom INFO [2019-12-17 12:33:07,639] jepsen worker 69 - jepsen.util 6729 :invoke :read [23 nil] INFO [2019-12-17 12:33:07,659] jepsen worker 36 - jepsen.util 7146 :fail :cas [12 [41 3]] :ValueNotSwapFrom INFO [2019-12-17 12:33:07,659] jepsen worker 36 - jepsen.util 7146 :invoke :read [12 nil] INFO [2019-12-17 12:33:07,678] jepsen worker 69 - jepsen.util 6729 :ok :read [23 20] INFO [2019-12-17 12:33:07,678] jepsen worker 69 - jepsen.util 6729 :invoke :cas [23 [24 14]] INFO [2019-12-17 12:33:07,680] jepsen worker 37 - jepsen.util 6427 :ok :read [12 35] INFO [2019-12-17 12:33:07,680] jepsen worker 37 - jepsen.util 6427 :invoke :read [12 nil] INFO [2019-12-17 12:33:07,691] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38711,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:33:07,692] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38711,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:33:07,692] jepsen worker 29 - jepsen.util 5069 :ok :read [9 25] INFO [2019-12-17 12:33:07,692] jepsen worker 29 - jepsen.util 5069 :invoke :cas [9 [33 19]] INFO [2019-12-17 12:33:07,693] jepsen worker 72 - jepsen.util 6552 :ok :read [24 27] INFO [2019-12-17 12:33:07,693] jepsen worker 72 - jepsen.util 6552 :invoke :read [24 nil] INFO [2019-12-17 12:33:07,710] jepsen worker 2 - jepsen.util 8822 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:33:07,710] jepsen worker 2 - jepsen.util 8822 :invoke :write [0 45] INFO [2019-12-17 12:33:07,716] jepsen worker 32 - jepsen.util 6872 :fail :cas [10 [24 20]] :ValueNotSwapFrom INFO [2019-12-17 12:33:07,716] jepsen worker 32 - jepsen.util 6872 :invoke :cas [10 [26 34]] INFO [2019-12-17 12:33:07,727] jepsen worker 9 - jepsen.util 7119 :info :write [3 41] :RequestCanceledException INFO [2019-12-17 12:33:07,728] jepsen worker 9 - jepsen.util 7209 :invoke :write [3 9] INFO [2019-12-17 12:33:07,751] jepsen worker 88 - jepsen.util 6658 :ok :read [29 17] INFO [2019-12-17 12:33:07,751] jepsen worker 88 - jepsen.util 6658 :invoke :cas [29 [36 14]] INFO [2019-12-17 12:33:07,760] jepsen worker 29 - jepsen.util 5069 :fail :cas [9 [33 19]] :ValueNotSwapFrom INFO [2019-12-17 12:33:07,760] jepsen worker 29 - jepsen.util 5069 :invoke :read [9 nil] INFO [2019-12-17 12:33:07,773] jepsen worker 63 - jepsen.util 5733 :ok :write [21 47] INFO [2019-12-17 12:33:07,774] jepsen worker 63 - jepsen.util 5733 :invoke :cas [21 [22 35]] INFO [2019-12-17 12:33:07,781] jepsen worker 38 - jepsen.util 6608 :fail :cas [12 [23 15]] :ValueNotSwapFrom INFO [2019-12-17 12:33:07,781] jepsen worker 38 - jepsen.util 6608 :invoke :write [12 43] INFO [2019-12-17 12:33:07,786] jepsen worker 5 - jepsen.util 6305 :ok :read [1 36] INFO [2019-12-17 12:33:07,786] jepsen worker 5 - jepsen.util 6305 :invoke :read [1 nil] INFO [2019-12-17 12:33:07,789] jepsen worker 32 - jepsen.util 6872 :fail :cas [10 [26 34]] :ValueNotSwapFrom INFO [2019-12-17 12:33:07,789] jepsen worker 32 - jepsen.util 6872 :invoke :read [10 nil] INFO [2019-12-17 12:33:07,799] jepsen worker 79 - jepsen.util 6739 :fail :cas [26 [31 46]] :ValueNotSwapFrom INFO [2019-12-17 12:33:07,799] jepsen worker 79 - jepsen.util 6739 :invoke :read [26 nil] INFO [2019-12-17 12:33:07,801] jepsen worker 29 - jepsen.util 5069 :ok :read [9 11] INFO [2019-12-17 12:33:07,801] jepsen worker 29 - jepsen.util 5069 :invoke :write [9 8] INFO [2019-12-17 12:33:07,826] jepsen worker 63 - jepsen.util 5733 :fail :cas [21 [22 35]] :ValueNotSwapFrom INFO [2019-12-17 12:33:07,827] jepsen worker 63 - jepsen.util 5733 :invoke :write [21 44] INFO [2019-12-17 12:33:07,828] jepsen worker 78 - jepsen.util 6918 :ok :read [26 14] INFO [2019-12-17 12:33:07,828] jepsen worker 78 - jepsen.util 6918 :invoke :cas [26 [1 2]] INFO [2019-12-17 12:33:07,840] jepsen worker 36 - jepsen.util 7146 :ok :read [12 35] INFO [2019-12-17 12:33:07,840] jepsen worker 36 - jepsen.util 7146 :invoke :read [12 nil] INFO [2019-12-17 12:33:07,855] jepsen worker 4 - jepsen.util 6034 :ok :read [1 36] INFO [2019-12-17 12:33:07,855] jepsen worker 4 - jepsen.util 6034 :invoke :read [1 nil] INFO [2019-12-17 12:33:07,864] jepsen worker 47 - jepsen.util 7247 :ok :write [15 42] INFO [2019-12-17 12:33:07,864] jepsen worker 47 - jepsen.util 7247 :invoke :read [15 nil] INFO [2019-12-17 12:33:07,868] jepsen worker 32 - jepsen.util 6872 :ok :read [10 19] INFO [2019-12-17 12:33:07,868] jepsen worker 32 - jepsen.util 6872 :invoke :cas [10 [31 47]] INFO [2019-12-17 12:33:07,884] jepsen worker 74 - jepsen.util 6914 :ok :read [24 27] INFO [2019-12-17 12:33:07,884] jepsen worker 74 - jepsen.util 6914 :invoke :write [24 33] INFO [2019-12-17 12:33:07,894] jepsen worker 47 - jepsen.util 7247 :ok :read [15 42] INFO [2019-12-17 12:33:07,894] jepsen worker 47 - jepsen.util 7247 :invoke :write [15 7] INFO [2019-12-17 12:33:07,905] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38734,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:33:07,905] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38734,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:33:07,905] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38737,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:33:07,905] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38737,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:33:07,905] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38738,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:33:07,905] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38738,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:33:07,913] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:33:07,913] jepsen nemesis - couchbase.util Rebalance check count: 296 INFO [2019-12-17 12:33:07,913] jepsen nemesis - couchbase.util Rebalance stuck count: 277 INFO [2019-12-17 12:33:07,933] jepsen worker 53 - jepsen.util 6353 :fail :cas [17 [1 7]] :ValueNotSwapFrom INFO [2019-12-17 12:33:07,934] jepsen worker 53 - jepsen.util 6353 :invoke :read [17 nil] INFO [2019-12-17 12:33:07,964] jepsen worker 29 - jepsen.util 5069 :ok :write [9 8] INFO [2019-12-17 12:33:07,964] jepsen worker 29 - jepsen.util 5069 :invoke :write [9 27] INFO [2019-12-17 12:33:07,994] jepsen worker 73 - jepsen.util 6283 :info :write [24 49] :RequestCanceledException INFO [2019-12-17 12:33:07,995] jepsen worker 73 - jepsen.util 6373 :invoke :cas [24 [30 27]] INFO [2019-12-17 12:33:08,014] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38750,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:33:08,014] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38750,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:33:08,040] jepsen worker 32 - jepsen.util 6872 :fail :cas [10 [31 47]] :ValueNotSwapFrom INFO [2019-12-17 12:33:08,040] jepsen worker 32 - jepsen.util 6872 :invoke :cas [10 [22 40]] INFO [2019-12-17 12:33:08,055] jepsen worker 63 - jepsen.util 5733 :ok :write [21 44] INFO [2019-12-17 12:33:08,055] jepsen worker 63 - jepsen.util 5733 :invoke :write [21 0] INFO [2019-12-17 12:33:08,070] jepsen worker 69 - jepsen.util 6729 :fail :cas [23 [24 14]] :ValueNotSwapFrom INFO [2019-12-17 12:33:08,070] jepsen worker 69 - jepsen.util 6729 :invoke :read [23 nil] INFO [2019-12-17 12:33:08,097] jepsen worker 73 - jepsen.util 6373 :fail :cas [24 [30 27]] :ValueNotSwapFrom INFO [2019-12-17 12:33:08,098] jepsen worker 73 - jepsen.util 6373 :invoke :read [24 nil] INFO [2019-12-17 12:33:08,112] jepsen worker 36 - jepsen.util 7146 :ok :read [12 35] INFO [2019-12-17 12:33:08,112] jepsen worker 36 - jepsen.util 7146 :invoke :write [12 6] INFO [2019-12-17 12:33:08,151] jepsen worker 29 - jepsen.util 5069 :ok :write [9 27] INFO [2019-12-17 12:33:08,151] jepsen worker 29 - jepsen.util 5069 :invoke :cas [9 [21 30]] INFO [2019-12-17 12:33:08,189] jepsen worker 32 - jepsen.util 6872 :fail :cas [10 [22 40]] :ValueNotSwapFrom INFO [2019-12-17 12:33:08,189] jepsen worker 32 - jepsen.util 6872 :invoke :write [10 28] INFO [2019-12-17 12:33:08,230] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38766,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:33:08,230] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38766,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:33:08,230] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38770,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:33:08,230] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38770,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:33:08,230] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38771,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:33:08,230] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38771,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:33:08,237] jepsen worker 37 - jepsen.util 6427 :ok :read [12 35] INFO [2019-12-17 12:33:08,237] jepsen worker 37 - jepsen.util 6427 :invoke :read [12 nil] INFO [2019-12-17 12:33:08,238] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38759,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:33:08,238] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38759,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:33:08,238] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38761,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:33:08,238] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38761,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:33:08,271] jepsen worker 63 - jepsen.util 5733 :ok :write [21 0] INFO [2019-12-17 12:33:08,271] jepsen worker 63 - jepsen.util 5733 :invoke :read [21 nil] INFO [2019-12-17 12:33:08,278] jepsen worker 29 - jepsen.util 5069 :fail :cas [9 [21 30]] :ValueNotSwapFrom INFO [2019-12-17 12:33:08,278] jepsen worker 29 - jepsen.util 5069 :invoke :cas [9 [3 35]] INFO [2019-12-17 12:33:08,322] jepsen worker 63 - jepsen.util 5733 :ok :read [21 0] INFO [2019-12-17 12:33:08,378] jepsen worker 63 - jepsen.util 5733 :invoke :write [21 38] INFO [2019-12-17 12:33:08,444] jepsen worker 63 - jepsen.util 5733 :ok :write [21 38] INFO [2019-12-17 12:33:08,444] jepsen worker 63 - jepsen.util 5733 :invoke :cas [21 [7 7]] INFO [2019-12-17 12:33:08,453] jepsen worker 80 - jepsen.util 6650 :ok :read [26 14] INFO [2019-12-17 12:33:08,453] jepsen worker 80 - jepsen.util 6650 :invoke :read [26 nil] INFO [2019-12-17 12:33:08,457] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38777,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:33:08,457] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38777,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:33:08,457] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38781,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:33:08,457] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38781,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:33:08,481] jepsen worker 63 - jepsen.util 5733 :fail :cas [21 [7 7]] :ValueNotSwapFrom INFO [2019-12-17 12:33:08,481] jepsen worker 63 - jepsen.util 5733 :invoke :cas [21 [37 36]] INFO [2019-12-17 12:33:08,499] jepsen worker 63 - jepsen.util 5733 :fail :cas [21 [37 36]] :ValueNotSwapFrom INFO [2019-12-17 12:33:08,499] jepsen worker 63 - jepsen.util 5733 :invoke :read [21 nil] INFO [2019-12-17 12:33:08,506] jepsen worker 88 - jepsen.util 6658 :fail :cas [29 [36 14]] :ValueNotSwapFrom INFO [2019-12-17 12:33:08,506] jepsen worker 88 - jepsen.util 6658 :invoke :read [29 nil] INFO [2019-12-17 12:33:08,528] jepsen worker 87 - jepsen.util 5847 :info :write [29 5] :RequestCanceledException INFO [2019-12-17 12:33:08,529] jepsen worker 63 - jepsen.util 5733 :ok :read [21 38] INFO [2019-12-17 12:33:08,529] jepsen worker 87 - jepsen.util 5937 :invoke :write [29 41] INFO [2019-12-17 12:33:08,530] jepsen worker 63 - jepsen.util 5733 :invoke :cas [21 [5 34]] INFO [2019-12-17 12:33:08,531] jepsen worker 78 - jepsen.util 6918 :fail :cas [26 [1 2]] :ValueNotSwapFrom INFO [2019-12-17 12:33:08,531] jepsen worker 78 - jepsen.util 6918 :invoke :write [26 39] INFO [2019-12-17 12:33:08,564] jepsen worker 53 - jepsen.util 6353 :ok :read [17 44] INFO [2019-12-17 12:33:08,565] jepsen worker 53 - jepsen.util 6353 :invoke :cas [17 [11 27]] INFO [2019-12-17 12:33:08,569] jepsen worker 63 - jepsen.util 5733 :fail :cas [21 [5 34]] :ValueNotSwapFrom INFO [2019-12-17 12:33:08,569] jepsen worker 63 - jepsen.util 5733 :invoke :read [21 nil] INFO [2019-12-17 12:33:08,587] jepsen worker 63 - jepsen.util 5733 :ok :read [21 38] INFO [2019-12-17 12:33:08,587] jepsen worker 63 - jepsen.util 5733 :invoke :write [21 19] INFO [2019-12-17 12:33:08,592] jepsen worker 5 - jepsen.util 6305 :ok :read [1 36] INFO [2019-12-17 12:33:08,592] jepsen worker 5 - jepsen.util 6305 :invoke :cas [1 [30 13]] INFO [2019-12-17 12:33:08,595] jepsen worker 77 - jepsen.util 9527 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:33:08,595] jepsen worker 77 - jepsen.util 9527 :invoke :write [25 48] INFO [2019-12-17 12:33:08,612] jepsen worker 43 - jepsen.util 8413 :info :write [14 45] :RequestCanceledException INFO [2019-12-17 12:33:08,612] jepsen worker 50 - jepsen.util 9590 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:33:08,627] jepsen worker 34 - jepsen.util 8404 :info :cas [11 [11 26]] :RequestCanceledException INFO [2019-12-17 12:33:08,627] jepsen worker 67 - jepsen.util 9247 :info :cas [22 [49 2]] :RequestCanceledException INFO [2019-12-17 12:33:08,628] jepsen worker 24 - jepsen.util 8124 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:33:08,628] jepsen worker 68 - jepsen.util 8618 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:33:08,628] jepsen worker 25 - jepsen.util 9475 :info :write [8 17] :RequestCanceledException INFO [2019-12-17 12:33:08,628] jepsen worker 33 - jepsen.util 8853 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:33:08,628] jepsen worker 83 - jepsen.util 9533 :info :cas [27 [11 25]] :RequestCanceledException INFO [2019-12-17 12:33:08,629] jepsen worker 59 - jepsen.util 9149 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:33:08,629] jepsen worker 21 - jepsen.util 8571 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:33:08,629] jepsen worker 23 - jepsen.util 9023 :info :write [7 16] :RequestCanceledException INFO [2019-12-17 12:33:08,634] jepsen worker 63 - jepsen.util 5733 :ok :write [21 19] INFO [2019-12-17 12:33:08,641] jepsen worker 66 - jepsen.util 8976 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:33:08,641] jepsen worker 0 - jepsen.util 9450 :info :cas [0 [34 9]] :RequestCanceledException INFO [2019-12-17 12:33:08,641] jepsen worker 58 - jepsen.util 9598 :info :write [19 35] :RequestCanceledException INFO [2019-12-17 12:33:08,641] jepsen worker 86 - jepsen.util 9536 :info :cas [28 [33 44]] :RequestCanceledException INFO [2019-12-17 12:33:08,653] jepsen worker 79 - jepsen.util 6739 :ok :read [26 14] INFO [2019-12-17 12:33:08,658] jepsen worker 42 - jepsen.util 9582 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:33:08,659] jepsen worker 62 - jepsen.util 9062 :info :write [20 41] :RequestCanceledException INFO [2019-12-17 12:33:08,659] jepsen worker 60 - jepsen.util 9960 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:33:08,659] jepsen worker 75 - jepsen.util 8625 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:33:08,660] jepsen worker 37 - jepsen.util 6427 :ok :read [12 35] INFO [2019-12-17 12:33:08,663] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38796,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:33:08,674] jepsen worker 4 - jepsen.util 6034 :ok :read [1 36] INFO [2019-12-17 12:33:08,676] jepsen worker 88 - jepsen.util 6658 :ok :read [29 17] INFO [2019-12-17 12:33:08,692] jepsen worker 26 - jepsen.util 9386 :info :cas [8 [30 45]] :RequestCanceledException INFO [2019-12-17 12:33:08,692] jepsen worker 22 - jepsen.util 9022 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:33:08,693] jepsen worker 15 - jepsen.util 8025 :info :cas [5 [43 38]] :RequestCanceledException INFO [2019-12-17 12:33:08,707] jepsen worker 50 - jepsen.util 9590 :invoke :write [16 9] INFO [2019-12-17 12:33:08,707] jepsen worker 24 - jepsen.util 8124 :invoke :read [8 nil] INFO [2019-12-17 12:33:08,707] jepsen worker 68 - jepsen.util 8618 :invoke :read [22 nil] INFO [2019-12-17 12:33:08,707] jepsen worker 33 - jepsen.util 8853 :invoke :write [11 20] INFO [2019-12-17 12:33:08,707] jepsen worker 59 - jepsen.util 9149 :invoke :read [19 nil] INFO [2019-12-17 12:33:08,707] jepsen worker 21 - jepsen.util 8571 :invoke :write [7 19] INFO [2019-12-17 12:33:08,708] jepsen worker 63 - jepsen.util 5733 :invoke :read [21 nil] INFO [2019-12-17 12:33:08,708] jepsen worker 66 - jepsen.util 8976 :invoke :read [22 nil] INFO [2019-12-17 12:33:08,708] jepsen worker 79 - jepsen.util 6739 :invoke :read [26 nil] INFO [2019-12-17 12:33:08,708] jepsen worker 42 - jepsen.util 9582 :invoke :cas [14 [31 35]] INFO [2019-12-17 12:33:08,708] jepsen worker 60 - jepsen.util 9960 :invoke :cas [20 [4 29]] INFO [2019-12-17 12:33:08,708] jepsen worker 34 - jepsen.util 8494 :invoke :cas [11 [45 44]] INFO [2019-12-17 12:33:08,708] jepsen worker 75 - jepsen.util 8625 :invoke :cas [25 [29 23]] INFO [2019-12-17 12:33:08,708] jepsen worker 83 - jepsen.util 9623 :invoke :read [27 nil] INFO [2019-12-17 12:33:08,708] jepsen worker 37 - jepsen.util 6427 :invoke :cas [12 [32 26]] INFO [2019-12-17 12:33:08,708] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38796,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:33:08,708] jepsen worker 4 - jepsen.util 6034 :invoke :read [1 nil] INFO [2019-12-17 12:33:08,709] jepsen worker 67 - jepsen.util 9337 :invoke :write [22 41] INFO [2019-12-17 12:33:08,709] jepsen worker 88 - jepsen.util 6658 :invoke :write [29 1] INFO [2019-12-17 12:33:08,709] jepsen worker 23 - jepsen.util 9113 :invoke :cas [7 [29 6]] INFO [2019-12-17 12:33:08,709] jepsen worker 43 - jepsen.util 8503 :invoke :cas [14 [42 44]] INFO [2019-12-17 12:33:08,709] jepsen worker 25 - jepsen.util 9565 :invoke :read [8 nil] INFO [2019-12-17 12:33:08,709] jepsen worker 58 - jepsen.util 9688 :invoke :cas [19 [19 30]] INFO [2019-12-17 12:33:08,709] jepsen worker 0 - jepsen.util 9540 :invoke :read [0 nil] INFO [2019-12-17 12:33:08,709] jepsen worker 22 - jepsen.util 9022 :invoke :write [7 15] INFO [2019-12-17 12:33:08,709] jepsen worker 86 - jepsen.util 9626 :invoke :write [28 38] INFO [2019-12-17 12:33:08,709] jepsen worker 1 - jepsen.util 9001 :info :write [0 25] :RequestCanceledException INFO [2019-12-17 12:33:08,710] jepsen worker 27 - jepsen.util 6777 :info :write [9 17] :RequestCanceledException INFO [2019-12-17 12:33:08,710] jepsen worker 26 - jepsen.util 9476 :invoke :read [8 nil] INFO [2019-12-17 12:33:08,710] jepsen worker 62 - jepsen.util 9152 :invoke :cas [20 [47 8]] INFO [2019-12-17 12:33:08,710] jepsen worker 15 - jepsen.util 8115 :invoke :write [5 15] INFO [2019-12-17 12:33:08,710] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38799,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:33:08,712] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38799,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:33:08,713] jepsen worker 1 - jepsen.util 9091 :invoke :write [0 1] INFO [2019-12-17 12:33:08,713] jepsen worker 27 - jepsen.util 6867 :invoke :cas [9 [35 35]] INFO [2019-12-17 12:33:08,713] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38800,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:33:08,713] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38800,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:33:08,727] jepsen worker 41 - jepsen.util 9671 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:33:08,728] jepsen worker 82 - jepsen.util 9802 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:33:08,728] jepsen worker 41 - jepsen.util 9671 :invoke :cas [13 [22 40]] INFO [2019-12-17 12:33:08,728] jepsen worker 82 - jepsen.util 9802 :invoke :write [27 41] INFO [2019-12-17 12:33:08,767] jepsen worker 80 - jepsen.util 6650 :ok :read [26 14] INFO [2019-12-17 12:33:08,767] jepsen worker 80 - jepsen.util 6650 :invoke :write [26 35] INFO [2019-12-17 12:33:08,769] jepsen worker 63 - jepsen.util 5733 :ok :read [21 19] INFO [2019-12-17 12:33:08,769] jepsen worker 63 - jepsen.util 5733 :invoke :cas [21 [47 0]] INFO [2019-12-17 12:33:08,798] jepsen worker 63 - jepsen.util 5733 :fail :cas [21 [47 0]] :ValueNotSwapFrom INFO [2019-12-17 12:33:08,798] jepsen worker 63 - jepsen.util 5733 :invoke :write [21 21] INFO [2019-12-17 12:33:09,097] jepsen worker 18 - jepsen.util 6678 :ok :write [6 44] INFO [2019-12-17 12:33:09,098] jepsen worker 74 - jepsen.util 6914 :ok :write [24 33] INFO [2019-12-17 12:33:09,098] jepsen worker 32 - jepsen.util 6872 :ok :write [10 28] INFO [2019-12-17 12:33:09,098] jepsen worker 10 - jepsen.util 6580 :ok :write [3 8] INFO [2019-12-17 12:33:09,098] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38837,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:33:09,099] jepsen worker 65 - jepsen.util 5465 :info :write [21 11] :RequestCanceledException INFO [2019-12-17 12:33:09,099] jepsen worker 51 - jepsen.util 6441 :info :write [17 46] :RequestCanceledException INFO [2019-12-17 12:33:09,099] jepsen worker 14 - jepsen.util 6854 :info :write [4 2] :RequestCanceledException INFO [2019-12-17 12:33:09,099] jepsen worker 76 - jepsen.util 9436 :info :cas [25 [33 25]] :RequestCanceledException INFO [2019-12-17 12:33:09,099] jepsen worker 64 - jepsen.util 5194 :info :write [21 49] :RequestCanceledException INFO [2019-12-17 12:33:09,099] jepsen worker 84 - jepsen.util 8544 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:33:09,099] jepsen worker 56 - jepsen.util 7256 :info :write [18 40] :RequestCanceledException INFO [2019-12-17 12:33:09,099] jepsen worker 81 - jepsen.util 8721 :info :write [27 37] :RequestCanceledException INFO [2019-12-17 12:33:09,099] jepsen worker 39 - jepsen.util 8949 :info :cas [13 [22 23]] :RequestCanceledException INFO [2019-12-17 12:33:09,099] jepsen worker 20 - jepsen.util 6410 :info :cas [6 [17 32]] :RequestCanceledException INFO [2019-12-17 12:33:09,100] jepsen worker 18 - jepsen.util 6678 :invoke :write [6 33] INFO [2019-12-17 12:33:09,100] jepsen worker 74 - jepsen.util 6914 :invoke :write [24 3] INFO [2019-12-17 12:33:09,100] jepsen worker 32 - jepsen.util 6872 :invoke :write [10 34] INFO [2019-12-17 12:33:09,101] jepsen worker 10 - jepsen.util 6580 :invoke :cas [3 [16 14]] INFO [2019-12-17 12:33:09,102] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38837,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:33:09,103] jepsen worker 51 - jepsen.util 6531 :invoke :write [17 15] INFO [2019-12-17 12:33:09,103] jepsen worker 14 - jepsen.util 6944 :invoke :read [4 nil] INFO [2019-12-17 12:33:09,104] jepsen worker 84 - jepsen.util 8544 :invoke :cas [28 [6 38]] INFO [2019-12-17 12:33:09,104] jepsen worker 76 - jepsen.util 9526 :invoke :read [25 nil] INFO [2019-12-17 12:33:09,104] jepsen worker 64 - jepsen.util 5284 :invoke :read [21 nil] INFO [2019-12-17 12:33:09,105] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38841,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:33:09,105] jepsen worker 20 - jepsen.util 6500 :invoke :write [6 14] INFO [2019-12-17 12:33:09,106] jepsen worker 65 - jepsen.util 5555 :invoke :write [21 45] INFO [2019-12-17 12:33:09,108] jepsen worker 39 - jepsen.util 9039 :invoke :cas [13 [28 42]] INFO [2019-12-17 12:33:09,109] jepsen worker 81 - jepsen.util 8811 :invoke :write [27 30] INFO [2019-12-17 12:33:09,109] jepsen worker 56 - jepsen.util 7346 :invoke :read [18 nil] INFO [2019-12-17 12:33:09,109] jepsen worker 69 - jepsen.util 6729 :ok :read [23 20] INFO [2019-12-17 12:33:09,109] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38841,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:33:09,115] jepsen worker 27 - jepsen.util 6867 :fail :cas [9 [35 35]] :ValueNotSwapFrom INFO [2019-12-17 12:33:09,116] jepsen worker 29 - jepsen.util 5069 :fail :cas [9 [3 35]] :ValueNotSwapFrom INFO [2019-12-17 12:33:09,116] jepsen worker 79 - jepsen.util 6739 :ok :read [26 14] INFO [2019-12-17 12:33:09,117] jepsen worker 37 - jepsen.util 6427 :fail :cas [12 [32 26]] :ValueNotSwapFrom INFO [2019-12-17 12:33:09,119] jepsen worker 69 - jepsen.util 6729 :invoke :read [23 nil] INFO [2019-12-17 12:33:09,120] jepsen worker 27 - jepsen.util 6867 :invoke :read [9 nil] INFO [2019-12-17 12:33:09,120] jepsen worker 29 - jepsen.util 5069 :invoke :read [9 nil] INFO [2019-12-17 12:33:09,120] jepsen worker 79 - jepsen.util 6739 :invoke :cas [26 [7 47]] INFO [2019-12-17 12:33:09,120] jepsen worker 37 - jepsen.util 6427 :invoke :cas [12 [40 39]] INFO [2019-12-17 12:33:09,125] jepsen worker 89 - jepsen.util 6389 :info :write [29 4] :RequestCanceledException INFO [2019-12-17 12:33:09,125] jepsen worker 52 - jepsen.util 6982 :info :write [17 15] :RequestCanceledException INFO [2019-12-17 12:33:09,125] jepsen worker 30 - jepsen.util 6510 :info :write [10 8] :RequestCanceledException INFO [2019-12-17 12:33:09,125] jepsen worker 4 - jepsen.util 6034 :ok :read [1 36] INFO [2019-12-17 12:33:09,125] jepsen worker 4 - jepsen.util 6034 :invoke :read [1 nil] INFO [2019-12-17 12:33:09,126] jepsen worker 52 - jepsen.util 7072 :invoke :read [17 nil] INFO [2019-12-17 12:33:09,126] jepsen worker 89 - jepsen.util 6479 :invoke :write [29 18] INFO [2019-12-17 12:33:09,126] jepsen worker 30 - jepsen.util 6600 :invoke :write [10 5] INFO [2019-12-17 12:33:09,140] jepsen worker 6 - jepsen.util 9726 :info :cas [2 [29 13]] :RequestCanceledException INFO [2019-12-17 12:33:09,141] jepsen worker 6 - jepsen.util 9816 :invoke :read [2 nil] INFO [2019-12-17 12:33:09,158] jepsen worker 17 - jepsen.util 9377 :info :write [5 46] :RequestCanceledException INFO [2019-12-17 12:33:09,159] jepsen worker 17 - jepsen.util 9467 :invoke :cas [5 [44 47]] INFO [2019-12-17 12:33:09,163] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:33:09,163] jepsen nemesis - couchbase.util Rebalance check count: 297 INFO [2019-12-17 12:33:09,163] jepsen nemesis - couchbase.util Rebalance stuck count: 278 INFO [2019-12-17 12:33:09,208] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38844,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:33:09,208] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38873,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:33:09,208] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38873,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:33:09,208] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38844,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:33:09,208] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38874,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:33:09,208] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38846,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:33:09,208] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38874,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:33:09,208] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38846,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:33:09,208] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38875,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:33:09,212] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38875,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:33:09,220] jepsen worker 56 - jepsen.util 7346 :ok :read [18 5] INFO [2019-12-17 12:33:09,220] jepsen worker 56 - jepsen.util 7346 :invoke :write [18 5] INFO [2019-12-17 12:33:09,227] jepsen worker 44 - jepsen.util 9944 :info :write [14 17] :RequestCanceledException INFO [2019-12-17 12:33:09,227] jepsen worker 40 - jepsen.util 9580 :info :write [13 12] :RequestCanceledException INFO [2019-12-17 12:33:09,229] jepsen worker 44 - jepsen.util 10034 :invoke :read [14 nil] INFO [2019-12-17 12:33:09,229] jepsen worker 40 - jepsen.util 9670 :invoke :cas [13 [16 48]] INFO [2019-12-17 12:33:09,230] jepsen worker 10 - jepsen.util 6580 :fail :cas [3 [16 14]] :ValueNotSwapFrom INFO [2019-12-17 12:33:09,230] jepsen worker 10 - jepsen.util 6580 :invoke :read [3 nil] INFO [2019-12-17 12:33:09,244] jepsen worker 64 - jepsen.util 5284 :ok :read [21 19] INFO [2019-12-17 12:33:09,256] jepsen worker 69 - jepsen.util 6729 :ok :read [23 20] INFO [2019-12-17 12:33:09,257] jepsen worker 31 - jepsen.util 5971 :fail :read [10 nil] :RequestTimeoutException INFO [2019-12-17 12:33:09,257] jepsen worker 70 - jepsen.util 7000 :info :write [23 13] :RequestCanceledException INFO [2019-12-17 12:33:09,289] jepsen worker 69 - jepsen.util 6729 :invoke :read [23 nil] INFO [2019-12-17 12:33:09,289] jepsen worker 64 - jepsen.util 5284 :invoke :read [21 nil] INFO [2019-12-17 12:33:09,289] jepsen worker 31 - jepsen.util 5971 :invoke :write [10 48] INFO [2019-12-17 12:33:09,289] jepsen worker 48 - jepsen.util 8418 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:33:09,290] jepsen worker 61 - jepsen.util 10141 :info :write [20 6] :RequestCanceledException INFO [2019-12-17 12:33:09,290] jepsen worker 48 - jepsen.util 8418 :invoke :write [16 25] INFO [2019-12-17 12:33:09,290] jepsen worker 70 - jepsen.util 7090 :invoke :read [23 nil] INFO [2019-12-17 12:33:09,291] jepsen worker 10 - jepsen.util 6580 :ok :read [3 8] INFO [2019-12-17 12:33:09,291] jepsen worker 10 - jepsen.util 6580 :invoke :cas [3 [33 6]] INFO [2019-12-17 12:33:09,291] jepsen worker 61 - jepsen.util 10231 :invoke :read [20 nil] INFO [2019-12-17 12:33:09,303] jepsen worker 7 - jepsen.util 9457 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:33:09,303] jepsen worker 7 - jepsen.util 9457 :invoke :cas [2 [49 28]] INFO [2019-12-17 12:33:09,315] jepsen worker 5 - jepsen.util 6305 :fail :cas [1 [30 13]] :ValueNotSwapFrom INFO [2019-12-17 12:33:09,315] jepsen worker 5 - jepsen.util 6305 :invoke :read [1 nil] INFO [2019-12-17 12:33:09,317] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38880,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:33:09,317] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38880,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:33:09,318] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38883,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:33:09,318] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38883,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:33:09,318] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38884,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:33:09,318] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38884,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:33:09,324] jepsen worker 57 - jepsen.util 8787 :info :write [19 30] :RequestCanceledException INFO [2019-12-17 12:33:09,324] jepsen worker 8 - jepsen.util 8918 :info :cas [2 [17 27]] :RequestCanceledException INFO [2019-12-17 12:33:09,324] jepsen worker 11 - jepsen.util 7121 :fail :read [3 nil] :RequestTimeoutException INFO [2019-12-17 12:33:09,324] jepsen worker 11 - jepsen.util 7121 :invoke :cas [3 [3 1]] INFO [2019-12-17 12:33:09,325] jepsen worker 8 - jepsen.util 9008 :invoke :read [2 nil] INFO [2019-12-17 12:33:09,325] jepsen worker 57 - jepsen.util 8877 :invoke :cas [19 [21 3]] INFO [2019-12-17 12:33:09,338] jepsen worker 85 - jepsen.util 10165 :info :write [28 10] :RequestCanceledException INFO [2019-12-17 12:33:09,339] jepsen worker 85 - jepsen.util 10255 :invoke :read [28 nil] INFO [2019-12-17 12:33:09,383] jepsen worker 64 - jepsen.util 5284 :ok :read [21 19] INFO [2019-12-17 12:33:09,383] jepsen worker 64 - jepsen.util 5284 :invoke :cas [21 [23 31]] INFO [2019-12-17 12:33:09,409] jepsen worker 10 - jepsen.util 6580 :fail :cas [3 [33 6]] :ValueNotSwapFrom INFO [2019-12-17 12:33:09,409] jepsen worker 10 - jepsen.util 6580 :invoke :read [3 nil] INFO [2019-12-17 12:33:09,418] jepsen worker 71 - jepsen.util 7451 :info :write [23 35] :RequestCanceledException INFO [2019-12-17 12:33:09,419] jepsen worker 71 - jepsen.util 7541 :invoke :read [23 nil] INFO [2019-12-17 12:33:09,424] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38906,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:33:09,424] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38906,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:33:09,425] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38909,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:33:09,425] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38909,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:33:09,425] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38910,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:33:09,425] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38910,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:33:09,441] jepsen worker 49 - jepsen.util 8329 :info :write [16 47] :RequestCanceledException INFO [2019-12-17 12:33:09,441] jepsen worker 54 - jepsen.util 7344 :info :write [18 12] :RequestCanceledException INFO [2019-12-17 12:33:09,441] jepsen worker 45 - jepsen.util 6705 :info :write [15 31] :RequestCanceledException INFO [2019-12-17 12:33:09,442] jepsen worker 54 - jepsen.util 7434 :invoke :write [18 18] INFO [2019-12-17 12:33:09,442] jepsen worker 49 - jepsen.util 8419 :invoke :cas [16 [10 48]] INFO [2019-12-17 12:33:09,442] jepsen worker 45 - jepsen.util 6795 :invoke :cas [15 [36 10]] INFO [2019-12-17 12:33:09,456] jepsen worker 55 - jepsen.util 7705 :info :write [18 49] :RequestCanceledException INFO [2019-12-17 12:33:09,457] jepsen worker 55 - jepsen.util 7795 :invoke :read [18 nil] INFO [2019-12-17 12:33:09,461] jepsen worker 63 - jepsen.util 5733 :ok :write [21 21] INFO [2019-12-17 12:33:09,461] jepsen worker 63 - jepsen.util 5733 :invoke :read [21 nil] INFO [2019-12-17 12:33:09,471] jepsen worker 64 - jepsen.util 5284 :fail :cas [21 [23 31]] :ValueNotSwapFrom INFO [2019-12-17 12:33:09,471] jepsen worker 64 - jepsen.util 5284 :invoke :read [21 nil] INFO [2019-12-17 12:33:09,490] jepsen worker 16 - jepsen.util 10186 :info :write [5 36] :RequestCanceledException INFO [2019-12-17 12:33:09,490] jepsen worker 46 - jepsen.util 6796 :info :cas [15 [16 38]] :RequestCanceledException INFO [2019-12-17 12:33:09,491] jepsen worker 46 - jepsen.util 6886 :invoke :write [15 3] INFO [2019-12-17 12:33:09,491] jepsen worker 16 - jepsen.util 10276 :invoke :write [5 16] INFO [2019-12-17 12:33:09,504] jepsen worker 35 - jepsen.util 9035 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:33:09,504] jepsen worker 35 - jepsen.util 9035 :invoke :read [11 nil] INFO [2019-12-17 12:33:09,506] jepsen worker 63 - jepsen.util 5733 :ok :read [21 21] INFO [2019-12-17 12:33:09,506] jepsen worker 63 - jepsen.util 5733 :invoke :write [21 16] INFO [2019-12-17 12:33:09,515] jepsen worker 10 - jepsen.util 6580 :ok :read [3 8] INFO [2019-12-17 12:33:09,516] jepsen worker 10 - jepsen.util 6580 :invoke :cas [3 [3 44]] INFO [2019-12-17 12:33:09,528] jepsen worker 19 - jepsen.util 6679 :fail :read [6 nil] :RequestTimeoutException INFO [2019-12-17 12:33:09,528] jepsen worker 19 - jepsen.util 6679 :invoke :read [6 nil] INFO [2019-12-17 12:33:09,540] jepsen worker 64 - jepsen.util 5284 :ok :read [21 21] INFO [2019-12-17 12:33:09,540] jepsen worker 64 - jepsen.util 5284 :invoke :write [21 11] INFO [2019-12-17 12:33:09,542] jepsen worker 79 - jepsen.util 6739 :fail :cas [26 [7 47]] :ValueNotSwapFrom INFO [2019-12-17 12:33:09,543] jepsen worker 79 - jepsen.util 6739 :invoke :cas [26 [30 23]] INFO [2019-12-17 12:33:09,544] jepsen worker 53 - jepsen.util 6353 :fail :cas [17 [11 27]] :ValueNotSwapFrom INFO [2019-12-17 12:33:09,544] jepsen worker 53 - jepsen.util 6353 :invoke :cas [17 [12 48]] INFO [2019-12-17 12:33:09,599] jepsen worker 10 - jepsen.util 6580 :fail :cas [3 [3 44]] :ValueNotSwapFrom INFO [2019-12-17 12:33:09,599] jepsen worker 10 - jepsen.util 6580 :invoke :read [3 nil] INFO [2019-12-17 12:33:09,608] jepsen worker 3 - jepsen.util 6483 :info :write [1 10] :RequestCanceledException INFO [2019-12-17 12:33:09,609] jepsen worker 3 - jepsen.util 6573 :invoke :write [1 21] INFO [2019-12-17 12:33:09,659] jepsen worker 12 - jepsen.util 7122 :info :write [4 21] :RequestCanceledException INFO [2019-12-17 12:33:09,661] jepsen worker 12 - jepsen.util 7212 :invoke :cas [4 [0 27]] INFO [2019-12-17 12:33:09,674] jepsen worker 64 - jepsen.util 5284 :ok :write [21 11] INFO [2019-12-17 12:33:09,675] jepsen worker 64 - jepsen.util 5284 :invoke :read [21 nil] INFO [2019-12-17 12:33:09,696] jepsen worker 28 - jepsen.util 5158 :info :write [9 16] :RequestCanceledException INFO [2019-12-17 12:33:09,697] jepsen worker 13 - jepsen.util 6403 :info :write [4 48] :RequestCanceledException INFO [2019-12-17 12:33:09,708] jepsen worker 28 - jepsen.util 5248 :invoke :read [9 nil] INFO [2019-12-17 12:33:09,710] jepsen worker 13 - jepsen.util 6493 :invoke :cas [4 [36 38]] INFO [2019-12-17 12:33:09,713] jepsen worker 64 - jepsen.util 5284 :ok :read [21 11] INFO [2019-12-17 12:33:09,713] jepsen worker 64 - jepsen.util 5284 :invoke :cas [21 [11 38]] INFO [2019-12-17 12:33:09,731] jepsen worker 52 - jepsen.util 7072 :ok :read [17 44] INFO [2019-12-17 12:33:09,731] jepsen worker 52 - jepsen.util 7072 :invoke :cas [17 [29 24]] INFO [2019-12-17 12:33:09,735] jepsen worker 71 - jepsen.util 7541 :ok :read [23 20] INFO [2019-12-17 12:33:09,735] jepsen worker 71 - jepsen.util 7541 :invoke :cas [23 [1 6]] INFO [2019-12-17 12:33:09,744] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38929,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:33:09,744] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38929,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:33:09,744] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38932,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:33:09,744] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38932,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:33:09,756] jepsen worker 72 - jepsen.util 6552 :fail :read [24 nil] :RequestTimeoutException INFO [2019-12-17 12:33:09,757] jepsen worker 72 - jepsen.util 6552 :invoke :cas [24 [45 17]] INFO [2019-12-17 12:33:09,794] jepsen worker 38 - jepsen.util 6608 :info :write [12 43] :RequestCanceledException INFO [2019-12-17 12:33:09,796] jepsen worker 38 - jepsen.util 6698 :invoke :cas [12 [29 7]] INFO [2019-12-17 12:33:09,811] jepsen worker 72 - jepsen.util 6552 :fail :cas [24 [45 17]] :ValueNotSwapFrom INFO [2019-12-17 12:33:09,811] jepsen worker 72 - jepsen.util 6552 :invoke :write [24 19] INFO [2019-12-17 12:33:09,812] jepsen worker 2 - jepsen.util 8822 :info :write [0 45] :RequestCanceledException INFO [2019-12-17 12:33:09,814] jepsen worker 2 - jepsen.util 8912 :invoke :read [0 nil] INFO [2019-12-17 12:33:09,826] jepsen worker 9 - jepsen.util 7209 :info :write [3 9] :RequestCanceledException INFO [2019-12-17 12:33:09,830] jepsen worker 9 - jepsen.util 7299 :invoke :cas [3 [7 13]] INFO [2019-12-17 12:33:09,839] jepsen worker 51 - jepsen.util 6531 :ok :write [17 15] INFO [2019-12-17 12:33:09,839] jepsen worker 51 - jepsen.util 6531 :invoke :read [17 nil] INFO [2019-12-17 12:33:09,865] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38949,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:33:09,865] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38949,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:33:09,866] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38952,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:33:09,866] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38952,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:33:09,866] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38953,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:33:09,866] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38953,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:33:09,883] jepsen worker 37 - jepsen.util 6427 :fail :cas [12 [40 39]] :ValueNotSwapFrom INFO [2019-12-17 12:33:09,887] jepsen worker 37 - jepsen.util 6427 :invoke :write [12 40] INFO [2019-12-17 12:33:09,901] jepsen worker 64 - jepsen.util 5284 :ok :cas [21 [11 38]] INFO [2019-12-17 12:33:09,902] jepsen worker 64 - jepsen.util 5284 :invoke :write [21 33] INFO [2019-12-17 12:33:09,991] jepsen worker 70 - jepsen.util 7090 :ok :read [23 20] INFO [2019-12-17 12:33:09,991] jepsen worker 70 - jepsen.util 7090 :invoke :read [23 nil] INFO [2019-12-17 12:33:09,991] jepsen worker 47 - jepsen.util 7247 :info :write [15 7] :RequestCanceledException INFO [2019-12-17 12:33:09,994] jepsen worker 47 - jepsen.util 7337 :invoke :read [15 nil] INFO [2019-12-17 12:33:10,040] jepsen worker 51 - jepsen.util 6531 :ok :read [17 15] INFO [2019-12-17 12:33:10,040] jepsen worker 51 - jepsen.util 6531 :invoke :write [17 29] INFO [2019-12-17 12:33:10,087] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38957,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:33:10,087] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38957,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:33:10,087] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38960,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:33:10,087] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38960,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:33:10,087] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38962,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:33:10,088] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38962,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:33:10,097] jepsen worker 46 - jepsen.util 6886 :ok :write [15 3] INFO [2019-12-17 12:33:10,097] jepsen worker 46 - jepsen.util 6886 :invoke :read [15 nil] INFO [2019-12-17 12:33:10,100] jepsen worker 32 - jepsen.util 6872 :ok :write [10 34] INFO [2019-12-17 12:33:10,100] jepsen worker 20 - jepsen.util 6500 :ok :write [6 14] INFO [2019-12-17 12:33:10,100] jepsen worker 32 - jepsen.util 6872 :invoke :read [10 nil] INFO [2019-12-17 12:33:10,100] jepsen worker 20 - jepsen.util 6500 :invoke :read [6 nil] INFO [2019-12-17 12:33:10,106] jepsen worker 14 - jepsen.util 6944 :ok :read [4 18] INFO [2019-12-17 12:33:10,106] jepsen worker 14 - jepsen.util 6944 :invoke :cas [4 [18 41]] INFO [2019-12-17 12:33:10,141] jepsen worker 73 - jepsen.util 6373 :fail :read [24 nil] :RequestTimeoutException INFO [2019-12-17 12:33:10,141] jepsen worker 73 - jepsen.util 6373 :invoke :write [24 46] INFO [2019-12-17 12:33:10,142] jepsen worker 36 - jepsen.util 7146 :info :write [12 6] :RequestCanceledException INFO [2019-12-17 12:33:10,144] jepsen worker 36 - jepsen.util 7236 :invoke :read [12 nil] INFO [2019-12-17 12:33:10,148] jepsen worker 4 - jepsen.util 6034 :ok :read [1 36] INFO [2019-12-17 12:33:10,148] jepsen worker 4 - jepsen.util 6034 :invoke :write [1 14] INFO [2019-12-17 12:33:10,188] jepsen worker 74 - jepsen.util 6914 :ok :write [24 3] INFO [2019-12-17 12:33:10,188] jepsen worker 74 - jepsen.util 6914 :invoke :cas [24 [36 30]] INFO [2019-12-17 12:33:10,189] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:33:10,189] jepsen nemesis - couchbase.util Rebalance check count: 298 INFO [2019-12-17 12:33:10,189] jepsen nemesis - couchbase.util Rebalance stuck count: 279 INFO [2019-12-17 12:33:10,197] jepsen worker 46 - jepsen.util 6886 :ok :read [15 3] INFO [2019-12-17 12:33:10,198] jepsen worker 46 - jepsen.util 6886 :invoke :cas [15 [0 8]] INFO [2019-12-17 12:33:10,207] jepsen worker 9 - jepsen.util 7299 :fail :cas [3 [7 13]] :ValueNotSwapFrom INFO [2019-12-17 12:33:10,207] jepsen worker 9 - jepsen.util 7299 :invoke :write [3 23] INFO [2019-12-17 12:33:10,222] jepsen worker 20 - jepsen.util 6500 :ok :read [6 14] INFO [2019-12-17 12:33:10,222] jepsen worker 20 - jepsen.util 6500 :invoke :write [6 32] INFO [2019-12-17 12:33:10,229] jepsen worker 45 - jepsen.util 6795 :fail :cas [15 [36 10]] :ValueNotSwapFrom INFO [2019-12-17 12:33:10,229] jepsen worker 45 - jepsen.util 6795 :invoke :read [15 nil] INFO [2019-12-17 12:33:10,247] jepsen worker 28 - jepsen.util 5248 :ok :read [9 27] INFO [2019-12-17 12:33:10,247] jepsen worker 28 - jepsen.util 5248 :invoke :write [9 37] INFO [2019-12-17 12:33:10,264] jepsen worker 32 - jepsen.util 6872 :ok :read [10 34] INFO [2019-12-17 12:33:10,265] jepsen worker 32 - jepsen.util 6872 :invoke :read [10 nil] INFO [2019-12-17 12:33:10,265] jepsen worker 65 - jepsen.util 5555 :ok :write [21 45] INFO [2019-12-17 12:33:10,265] jepsen worker 65 - jepsen.util 5555 :invoke :cas [21 [28 24]] INFO [2019-12-17 12:33:10,268] jepsen worker 55 - jepsen.util 7795 :ok :read [18 5] INFO [2019-12-17 12:33:10,268] jepsen worker 55 - jepsen.util 7795 :invoke :read [18 nil] INFO [2019-12-17 12:33:10,289] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38981,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:33:10,289] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38981,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:33:10,289] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38985,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:33:10,289] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38985,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:33:10,297] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38973,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:33:10,297] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38973,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:33:10,297] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38975,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:33:10,298] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":38975,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:33:10,314] jepsen worker 69 - jepsen.util 6729 :ok :read [23 20] INFO [2019-12-17 12:33:10,314] jepsen worker 69 - jepsen.util 6729 :invoke :write [23 7] INFO [2019-12-17 12:33:10,332] jepsen worker 45 - jepsen.util 6795 :ok :read [15 3] INFO [2019-12-17 12:33:10,332] jepsen worker 45 - jepsen.util 6795 :invoke :read [15 nil] INFO [2019-12-17 12:33:10,345] jepsen worker 5 - jepsen.util 6305 :ok :read [1 36] INFO [2019-12-17 12:33:10,345] jepsen worker 5 - jepsen.util 6305 :invoke :write [1 19] INFO [2019-12-17 12:33:10,353] jepsen worker 74 - jepsen.util 6914 :fail :cas [24 [36 30]] :ValueNotSwapFrom INFO [2019-12-17 12:33:10,353] jepsen worker 74 - jepsen.util 6914 :invoke :read [24 nil] INFO [2019-12-17 12:33:10,387] jepsen worker 52 - jepsen.util 7072 :fail :cas [17 [29 24]] :ValueNotSwapFrom INFO [2019-12-17 12:33:10,387] jepsen worker 52 - jepsen.util 7072 :invoke :read [17 nil] INFO [2019-12-17 12:33:10,400] jepsen worker 65 - jepsen.util 5555 :fail :cas [21 [28 24]] :ValueNotSwapFrom INFO [2019-12-17 12:33:10,400] jepsen worker 65 - jepsen.util 5555 :invoke :read [21 nil] INFO [2019-12-17 12:33:10,405] jepsen worker 71 - jepsen.util 7541 :fail :cas [23 [1 6]] :ValueNotSwapFrom INFO [2019-12-17 12:33:10,405] jepsen worker 71 - jepsen.util 7541 :invoke :cas [23 [27 23]] INFO [2019-12-17 12:33:10,420] jepsen worker 74 - jepsen.util 6914 :ok :read [24 3] INFO [2019-12-17 12:33:10,421] jepsen worker 74 - jepsen.util 6914 :invoke :cas [24 [44 11]] INFO [2019-12-17 12:33:10,422] jepsen worker 79 - jepsen.util 6739 :fail :cas [26 [30 23]] :ValueNotSwapFrom INFO [2019-12-17 12:33:10,422] jepsen worker 79 - jepsen.util 6739 :invoke :write [26 40] INFO [2019-12-17 12:33:10,431] jepsen worker 65 - jepsen.util 5555 :ok :read [21 45] INFO [2019-12-17 12:33:10,432] jepsen worker 65 - jepsen.util 5555 :invoke :read [21 nil] INFO [2019-12-17 12:33:10,440] jepsen worker 74 - jepsen.util 6914 :fail :cas [24 [44 11]] :ValueNotSwapFrom INFO [2019-12-17 12:33:10,441] jepsen worker 74 - jepsen.util 6914 :invoke :write [24 16] INFO [2019-12-17 12:33:10,468] jepsen worker 65 - jepsen.util 5555 :ok :read [21 45] INFO [2019-12-17 12:33:10,468] jepsen worker 65 - jepsen.util 5555 :invoke :read [21 nil] INFO [2019-12-17 12:33:10,481] jepsen worker 13 - jepsen.util 6493 :fail :cas [4 [36 38]] :ValueNotSwapFrom INFO [2019-12-17 12:33:10,481] jepsen worker 13 - jepsen.util 6493 :invoke :read [4 nil] INFO [2019-12-17 12:33:10,481] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":39003,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:33:10,482] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":39003,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:33:10,482] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":39006,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:33:10,482] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":39006,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:33:10,482] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":39007,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:33:10,482] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":39007,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:33:10,505] jepsen worker 65 - jepsen.util 5555 :ok :read [21 45] INFO [2019-12-17 12:33:10,505] jepsen worker 70 - jepsen.util 7090 :ok :read [23 20] INFO [2019-12-17 12:33:10,505] jepsen worker 65 - jepsen.util 5555 :invoke :read [21 nil] INFO [2019-12-17 12:33:10,505] jepsen worker 70 - jepsen.util 7090 :invoke :cas [23 [37 14]] INFO [2019-12-17 12:33:10,533] jepsen worker 65 - jepsen.util 5555 :ok :read [21 45] INFO [2019-12-17 12:33:10,533] jepsen worker 65 - jepsen.util 5555 :invoke :cas [21 [16 35]] INFO [2019-12-17 12:33:10,543] jepsen worker 28 - jepsen.util 5248 :ok :write [9 37] INFO [2019-12-17 12:33:10,544] jepsen worker 28 - jepsen.util 5248 :invoke :write [9 33] INFO [2019-12-17 12:33:10,550] jepsen worker 65 - jepsen.util 5555 :fail :cas [21 [16 35]] :ValueNotSwapFrom INFO [2019-12-17 12:33:10,550] jepsen worker 65 - jepsen.util 5555 :invoke :cas [21 [15 14]] INFO [2019-12-17 12:33:10,557] jepsen worker 87 - jepsen.util 5937 :info :write [29 41] :RequestCanceledException INFO [2019-12-17 12:33:10,558] jepsen worker 87 - jepsen.util 6027 :invoke :cas [29 [8 7]] INFO [2019-12-17 12:33:10,577] jepsen worker 65 - jepsen.util 5555 :fail :cas [21 [15 14]] :ValueNotSwapFrom INFO [2019-12-17 12:33:10,577] jepsen worker 65 - jepsen.util 5555 :invoke :write [21 28] INFO [2019-12-17 12:33:10,594] jepsen worker 78 - jepsen.util 6918 :info :write [26 39] :RequestCanceledException INFO [2019-12-17 12:33:10,595] jepsen worker 78 - jepsen.util 7008 :invoke :write [26 38] INFO [2019-12-17 12:33:10,608] jepsen worker 38 - jepsen.util 6698 :fail :cas [12 [29 7]] :ValueNotSwapFrom INFO [2019-12-17 12:33:10,609] jepsen worker 38 - jepsen.util 6698 :invoke :cas [12 [48 47]] INFO [2019-12-17 12:33:10,695] jepsen worker 77 - jepsen.util 9527 :info :write [25 48] :RequestCanceledException INFO [2019-12-17 12:33:10,696] jepsen worker 77 - jepsen.util 9617 :invoke :read [25 nil] INFO [2019-12-17 12:33:10,700] jepsen worker 28 - jepsen.util 5248 :ok :write [9 33] INFO [2019-12-17 12:33:10,701] jepsen worker 28 - jepsen.util 5248 :invoke :write [9 45] INFO [2019-12-17 12:33:10,701] jepsen worker 52 - jepsen.util 7072 :ok :read [17 15] INFO [2019-12-17 12:33:10,702] jepsen worker 52 - jepsen.util 7072 :invoke :cas [17 [35 36]] INFO [2019-12-17 12:33:10,707] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":39022,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:33:10,707] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":39022,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:33:10,708] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":39026,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:33:10,708] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":39026,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:33:10,710] jepsen worker 65 - jepsen.util 5555 :ok :write [21 28] INFO [2019-12-17 12:33:10,710] jepsen worker 65 - jepsen.util 5555 :invoke :cas [21 [8 4]] INFO [2019-12-17 12:33:10,720] jepsen worker 71 - jepsen.util 7541 :fail :cas [23 [27 23]] :ValueNotSwapFrom INFO [2019-12-17 12:33:10,720] jepsen worker 71 - jepsen.util 7541 :invoke :read [23 nil] INFO [2019-12-17 12:33:10,721] jepsen worker 59 - jepsen.util 9149 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:33:10,721] jepsen worker 24 - jepsen.util 8124 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:33:10,721] jepsen worker 23 - jepsen.util 9113 :info :cas [7 [29 6]] :RequestCanceledException INFO [2019-12-17 12:33:10,721] jepsen worker 26 - jepsen.util 9476 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:33:10,721] jepsen worker 59 - jepsen.util 9149 :invoke :write [19 24] INFO [2019-12-17 12:33:10,721] jepsen worker 21 - jepsen.util 8571 :info :write [7 19] :RequestCanceledException INFO [2019-12-17 12:33:10,721] jepsen worker 58 - jepsen.util 9688 :info :cas [19 [19 30]] :RequestCanceledException INFO [2019-12-17 12:33:10,721] jepsen worker 34 - jepsen.util 8494 :info :cas [11 [45 44]] :RequestCanceledException INFO [2019-12-17 12:33:10,721] jepsen worker 33 - jepsen.util 8853 :info :write [11 20] :RequestCanceledException INFO [2019-12-17 12:33:10,721] jepsen worker 68 - jepsen.util 8618 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:33:10,721] jepsen worker 24 - jepsen.util 8124 :invoke :cas [8 [25 16]] INFO [2019-12-17 12:33:10,722] jepsen worker 26 - jepsen.util 9476 :invoke :read [8 nil] INFO [2019-12-17 12:33:10,722] jepsen worker 68 - jepsen.util 8618 :invoke :write [22 19] INFO [2019-12-17 12:33:10,722] jepsen worker 23 - jepsen.util 9203 :invoke :cas [7 [36 20]] INFO [2019-12-17 12:33:10,723] jepsen worker 21 - jepsen.util 8661 :invoke :write [7 4] INFO [2019-12-17 12:33:10,723] jepsen worker 58 - jepsen.util 9778 :invoke :cas [19 [27 31]] INFO [2019-12-17 12:33:10,723] jepsen worker 33 - jepsen.util 8943 :invoke :cas [11 [47 6]] INFO [2019-12-17 12:33:10,723] jepsen worker 34 - jepsen.util 8584 :invoke :cas [11 [42 37]] INFO [2019-12-17 12:33:10,736] jepsen worker 53 - jepsen.util 6353 :fail :cas [17 [12 48]] :ValueNotSwapFrom INFO [2019-12-17 12:33:10,736] jepsen worker 53 - jepsen.util 6353 :invoke :write [17 46] INFO [2019-12-17 12:33:10,740] jepsen worker 43 - jepsen.util 8503 :info :cas [14 [42 44]] :RequestCanceledException INFO [2019-12-17 12:33:10,740] jepsen worker 66 - jepsen.util 8976 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:33:10,740] jepsen worker 62 - jepsen.util 9152 :info :cas [20 [47 8]] :RequestCanceledException INFO [2019-12-17 12:33:10,740] jepsen worker 66 - jepsen.util 8976 :invoke :read [22 nil] INFO [2019-12-17 12:33:10,741] jepsen worker 43 - jepsen.util 8593 :invoke :read [14 nil] INFO [2019-12-17 12:33:10,741] jepsen worker 62 - jepsen.util 9242 :invoke :read [20 nil] INFO [2019-12-17 12:33:10,757] jepsen worker 42 - jepsen.util 9582 :info :cas [14 [31 35]] :RequestCanceledException INFO [2019-12-17 12:33:10,757] jepsen worker 75 - jepsen.util 8625 :info :cas [25 [29 23]] :RequestCanceledException INFO [2019-12-17 12:33:10,757] jepsen worker 60 - jepsen.util 9960 :info :cas [20 [4 29]] :RequestCanceledException INFO [2019-12-17 12:33:10,757] jepsen worker 15 - jepsen.util 8115 :info :write [5 15] :RequestCanceledException INFO [2019-12-17 12:33:10,757] jepsen worker 25 - jepsen.util 9565 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:33:10,758] jepsen worker 65 - jepsen.util 5555 :fail :cas [21 [8 4]] :ValueNotSwapFrom INFO [2019-12-17 12:33:10,758] jepsen worker 25 - jepsen.util 9565 :invoke :write [8 15] INFO [2019-12-17 12:33:10,758] jepsen worker 65 - jepsen.util 5555 :invoke :cas [21 [13 35]] INFO [2019-12-17 12:33:10,758] jepsen worker 75 - jepsen.util 8715 :invoke :cas [25 [30 28]] INFO [2019-12-17 12:33:10,758] jepsen worker 42 - jepsen.util 9672 :invoke :write [14 45] INFO [2019-12-17 12:33:10,759] jepsen worker 60 - jepsen.util 10050 :invoke :write [20 8] INFO [2019-12-17 12:33:10,759] jepsen worker 15 - jepsen.util 8205 :invoke :read [5 nil] INFO [2019-12-17 12:33:10,781] jepsen worker 36 - jepsen.util 7236 :ok :read [12 35] INFO [2019-12-17 12:33:10,781] jepsen worker 36 - jepsen.util 7236 :invoke :cas [12 [28 8]] INFO [2019-12-17 12:33:10,794] jepsen worker 0 - jepsen.util 9540 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:33:10,794] jepsen worker 83 - jepsen.util 9623 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:33:10,794] jepsen worker 22 - jepsen.util 9022 :info :write [7 15] :RequestCanceledException INFO [2019-12-17 12:33:10,794] jepsen worker 0 - jepsen.util 9540 :invoke :write [0 34] INFO [2019-12-17 12:33:10,794] jepsen worker 88 - jepsen.util 6658 :info :write [29 1] :RequestCanceledException INFO [2019-12-17 12:33:10,794] jepsen worker 83 - jepsen.util 9623 :invoke :write [27 23] INFO [2019-12-17 12:33:10,795] jepsen worker 22 - jepsen.util 9112 :invoke :cas [7 [24 33]] INFO [2019-12-17 12:33:10,796] jepsen worker 88 - jepsen.util 6748 :invoke :write [29 37] INFO [2019-12-17 12:33:10,797] jepsen worker 71 - jepsen.util 7541 :ok :read [23 20] INFO [2019-12-17 12:33:10,797] jepsen worker 71 - jepsen.util 7541 :invoke :write [23 46] INFO [2019-12-17 12:33:10,811] jepsen worker 86 - jepsen.util 9626 :info :write [28 38] :RequestCanceledException INFO [2019-12-17 12:33:10,811] jepsen worker 50 - jepsen.util 9590 :info :write [16 9] :RequestCanceledException INFO [2019-12-17 12:33:10,811] jepsen worker 67 - jepsen.util 9337 :info :write [22 41] :RequestCanceledException INFO [2019-12-17 12:33:10,812] jepsen worker 65 - jepsen.util 5555 :fail :cas [21 [13 35]] :ValueNotSwapFrom INFO [2019-12-17 12:33:10,812] jepsen worker 65 - jepsen.util 5555 :invoke :read [21 nil] INFO [2019-12-17 12:33:10,813] jepsen worker 50 - jepsen.util 9680 :invoke :read [16 nil] INFO [2019-12-17 12:33:10,813] jepsen worker 86 - jepsen.util 9716 :invoke :cas [28 [17 26]] INFO [2019-12-17 12:33:10,813] jepsen worker 67 - jepsen.util 9427 :invoke :read [22 nil] INFO [2019-12-17 12:33:10,827] jepsen worker 82 - jepsen.util 9802 :info :write [27 41] :RequestCanceledException INFO [2019-12-17 12:33:10,827] jepsen worker 41 - jepsen.util 9671 :info :cas [13 [22 40]] :RequestCanceledException INFO [2019-12-17 12:33:10,827] jepsen worker 80 - jepsen.util 6650 :info :write [26 35] :RequestCanceledException INFO [2019-12-17 12:33:10,828] jepsen worker 1 - jepsen.util 9091 :info :write [0 1] :RequestCanceledException INFO [2019-12-17 12:33:10,829] jepsen worker 41 - jepsen.util 9761 :invoke :read [13 nil] INFO [2019-12-17 12:33:10,829] jepsen worker 82 - jepsen.util 9892 :invoke :read [27 nil] INFO [2019-12-17 12:33:10,830] jepsen worker 1 - jepsen.util 9181 :invoke :write [0 37] INFO [2019-12-17 12:33:10,831] jepsen worker 80 - jepsen.util 6740 :invoke :write [26 39] INFO [2019-12-17 12:33:10,835] jepsen worker 65 - jepsen.util 5555 :ok :read [21 28] INFO [2019-12-17 12:33:10,835] jepsen worker 65 - jepsen.util 5555 :invoke :read [21 nil] INFO [2019-12-17 12:33:10,853] jepsen worker 65 - jepsen.util 5555 :ok :read [21 28] INFO [2019-12-17 12:33:10,853] jepsen worker 65 - jepsen.util 5555 :invoke :write [21 17] INFO [2019-12-17 12:33:10,856] jepsen worker 28 - jepsen.util 5248 :ok :write [9 45] INFO [2019-12-17 12:33:10,856] jepsen worker 28 - jepsen.util 5248 :invoke :cas [9 [6 3]] INFO [2019-12-17 12:33:10,877] jepsen worker 52 - jepsen.util 7072 :fail :cas [17 [35 36]] :ValueNotSwapFrom INFO [2019-12-17 12:33:10,877] jepsen worker 52 - jepsen.util 7072 :invoke :read [17 nil] INFO [2019-12-17 12:33:10,879] jepsen worker 28 - jepsen.util 5248 :fail :cas [9 [6 3]] :ValueNotSwapFrom INFO [2019-12-17 12:33:10,880] jepsen worker 28 - jepsen.util 5248 :invoke :write [9 24] INFO [2019-12-17 12:33:10,942] jepsen worker 28 - jepsen.util 5248 :ok :write [9 24] INFO [2019-12-17 12:33:10,942] jepsen worker 28 - jepsen.util 5248 :invoke :write [9 21] INFO [2019-12-17 12:33:10,947] jepsen worker 65 - jepsen.util 5555 :ok :write [21 17] INFO [2019-12-17 12:33:10,948] jepsen worker 65 - jepsen.util 5555 :invoke :read [21 nil] INFO [2019-12-17 12:33:10,954] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":39070,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:33:10,954] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":39070,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:33:10,954] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":39071,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:33:10,954] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":39071,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:33:10,993] jepsen worker 65 - jepsen.util 5555 :ok :read [21 17] INFO [2019-12-17 12:33:10,994] jepsen worker 65 - jepsen.util 5555 :invoke :read [21 nil] INFO [2019-12-17 12:33:11,005] jepsen worker 55 - jepsen.util 7795 :ok :read [18 5] INFO [2019-12-17 12:33:11,005] jepsen worker 55 - jepsen.util 7795 :invoke :read [18 nil] INFO [2019-12-17 12:33:11,013] jepsen worker 65 - jepsen.util 5555 :ok :read [21 17] INFO [2019-12-17 12:33:11,013] jepsen worker 65 - jepsen.util 5555 :invoke :read [21 nil] INFO [2019-12-17 12:33:11,025] jepsen worker 28 - jepsen.util 5248 :ok :write [9 21] INFO [2019-12-17 12:33:11,025] jepsen worker 28 - jepsen.util 5248 :invoke :write [9 47] INFO [2019-12-17 12:33:11,036] jepsen worker 65 - jepsen.util 5555 :ok :read [21 17] INFO [2019-12-17 12:33:11,037] jepsen worker 65 - jepsen.util 5555 :invoke :write [21 46] INFO [2019-12-17 12:33:11,058] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":39081,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:33:11,059] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":39081,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:33:11,059] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":39084,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:33:11,059] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":39084,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:33:11,059] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":39085,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:33:11,059] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":39085,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:33:11,100] jepsen worker 65 - jepsen.util 5555 :ok :write [21 46] INFO [2019-12-17 12:33:11,100] jepsen worker 65 - jepsen.util 5555 :invoke :write [21 12] INFO [2019-12-17 12:33:11,104] jepsen worker 28 - jepsen.util 5248 :ok :write [9 47] INFO [2019-12-17 12:33:11,105] jepsen worker 28 - jepsen.util 5248 :invoke :cas [9 [4 37]] INFO [2019-12-17 12:33:11,119] jepsen worker 55 - jepsen.util 7795 :ok :read [18 5] INFO [2019-12-17 12:33:11,120] jepsen worker 52 - jepsen.util 7072 :ok :read [17 15] INFO [2019-12-17 12:33:11,120] jepsen worker 55 - jepsen.util 7795 :invoke :cas [18 [37 34]] INFO [2019-12-17 12:33:11,120] jepsen worker 52 - jepsen.util 7072 :invoke :cas [17 [23 22]] INFO [2019-12-17 12:33:11,128] jepsen worker 18 - jepsen.util 6678 :info :write [6 33] :RequestCanceledException INFO [2019-12-17 12:33:11,129] jepsen worker 18 - jepsen.util 6768 :invoke :read [6 nil] INFO [2019-12-17 12:33:11,141] jepsen worker 76 - jepsen.util 9526 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:33:11,142] jepsen worker 76 - jepsen.util 9526 :invoke :write [25 49] INFO [2019-12-17 12:33:11,147] jepsen worker 28 - jepsen.util 5248 :fail :cas [9 [4 37]] :ValueNotSwapFrom INFO [2019-12-17 12:33:11,147] jepsen worker 28 - jepsen.util 5248 :invoke :cas [9 [12 16]] INFO [2019-12-17 12:33:11,154] jepsen worker 9 - jepsen.util 7299 :ok :write [3 23] INFO [2019-12-17 12:33:11,154] jepsen worker 73 - jepsen.util 6373 :ok :write [24 46] INFO [2019-12-17 12:33:11,154] jepsen worker 9 - jepsen.util 7299 :invoke :cas [3 [41 21]] INFO [2019-12-17 12:33:11,156] jepsen worker 73 - jepsen.util 6373 :invoke :cas [24 [4 49]] INFO [2019-12-17 12:33:11,157] jepsen worker 38 - jepsen.util 6698 :fail :cas [12 [48 47]] :ValueNotSwapFrom INFO [2019-12-17 12:33:11,157] jepsen worker 38 - jepsen.util 6698 :invoke :read [12 nil] INFO [2019-12-17 12:33:11,167] jepsen worker 18 - jepsen.util 6768 :ok :read [6 14] INFO [2019-12-17 12:33:11,167] jepsen worker 18 - jepsen.util 6768 :invoke :cas [6 [24 46]] INFO [2019-12-17 12:33:11,194] jepsen worker 27 - jepsen.util 6867 :fail :read [9 nil] :RequestTimeoutException INFO [2019-12-17 12:33:11,194] jepsen worker 30 - jepsen.util 6600 :info :write [10 5] :RequestCanceledException INFO [2019-12-17 12:33:11,194] jepsen worker 27 - jepsen.util 6867 :invoke :write [9 25] INFO [2019-12-17 12:33:11,195] jepsen worker 28 - jepsen.util 5248 :fail :cas [9 [12 16]] :ValueNotSwapFrom INFO [2019-12-17 12:33:11,195] jepsen worker 28 - jepsen.util 5248 :invoke :read [9 nil] INFO [2019-12-17 12:33:11,195] jepsen worker 30 - jepsen.util 6690 :invoke :cas [10 [48 27]] INFO [2019-12-17 12:33:11,204] jepsen worker 9 - jepsen.util 7299 :fail :cas [3 [41 21]] :ValueNotSwapFrom INFO [2019-12-17 12:33:11,204] jepsen worker 9 - jepsen.util 7299 :invoke :read [3 nil] INFO [2019-12-17 12:33:11,204] jepsen worker 73 - jepsen.util 6373 :fail :cas [24 [4 49]] :ValueNotSwapFrom INFO [2019-12-17 12:33:11,205] jepsen worker 73 - jepsen.util 6373 :invoke :cas [24 [19 13]] INFO [2019-12-17 12:33:11,209] jepsen worker 29 - jepsen.util 5069 :fail :read [9 nil] :RequestTimeoutException INFO [2019-12-17 12:33:11,209] jepsen worker 84 - jepsen.util 8544 :info :cas [28 [6 38]] :RequestCanceledException INFO [2019-12-17 12:33:11,209] jepsen worker 39 - jepsen.util 9039 :info :cas [13 [28 42]] :RequestCanceledException INFO [2019-12-17 12:33:11,209] jepsen worker 6 - jepsen.util 9816 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:33:11,209] jepsen worker 29 - jepsen.util 5069 :invoke :cas [9 [24 13]] INFO [2019-12-17 12:33:11,209] jepsen worker 6 - jepsen.util 9816 :invoke :write [2 26] INFO [2019-12-17 12:33:11,210] jepsen worker 84 - jepsen.util 8634 :invoke :cas [28 [9 36]] INFO [2019-12-17 12:33:11,210] jepsen worker 39 - jepsen.util 9129 :invoke :write [13 15] INFO [2019-12-17 12:33:11,214] jepsen worker 18 - jepsen.util 6768 :fail :cas [6 [24 46]] :ValueNotSwapFrom INFO [2019-12-17 12:33:11,214] jepsen worker 18 - jepsen.util 6768 :invoke :write [6 32] INFO [2019-12-17 12:33:11,228] jepsen worker 89 - jepsen.util 6479 :info :write [29 18] :RequestCanceledException INFO [2019-12-17 12:33:11,228] jepsen worker 81 - jepsen.util 8811 :info :write [27 30] :RequestCanceledException INFO [2019-12-17 12:33:11,228] jepsen worker 17 - jepsen.util 9467 :info :cas [5 [44 47]] :RequestCanceledException INFO [2019-12-17 12:33:11,229] jepsen worker 89 - jepsen.util 6569 :invoke :read [29 nil] INFO [2019-12-17 12:33:11,229] jepsen worker 17 - jepsen.util 9557 :invoke :cas [5 [19 29]] INFO [2019-12-17 12:33:11,229] jepsen worker 81 - jepsen.util 8901 :invoke :cas [27 [9 36]] INFO [2019-12-17 12:33:11,234] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:33:11,234] jepsen nemesis - couchbase.util Rebalance check count: 299 INFO [2019-12-17 12:33:11,234] jepsen nemesis - couchbase.util Rebalance stuck count: 280 INFO [2019-12-17 12:33:11,242] jepsen worker 56 - jepsen.util 7346 :info :write [18 5] :RequestCanceledException INFO [2019-12-17 12:33:11,242] jepsen worker 44 - jepsen.util 10034 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:33:11,242] jepsen worker 44 - jepsen.util 10034 :invoke :cas [14 [1 0]] INFO [2019-12-17 12:33:11,243] jepsen worker 56 - jepsen.util 7436 :invoke :write [18 10] INFO [2019-12-17 12:33:11,244] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":39114,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:33:11,244] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":39114,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:33:11,253] jepsen worker 40 - jepsen.util 9670 :info :cas [13 [16 48]] :RequestCanceledException INFO [2019-12-17 12:33:11,254] jepsen worker 40 - jepsen.util 9760 :invoke :cas [13 [42 18]] INFO [2019-12-17 12:33:11,298] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":39101,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:33:11,298] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":39101,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:33:11,299] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":39102,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:33:11,299] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":39102,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:33:11,299] jepsen worker 28 - jepsen.util 5248 :ok :read [9 47] INFO [2019-12-17 12:33:11,299] jepsen worker 28 - jepsen.util 5248 :invoke :cas [9 [20 35]] INFO [2019-12-17 12:33:11,328] jepsen worker 73 - jepsen.util 6373 :fail :cas [24 [19 13]] :ValueNotSwapFrom INFO [2019-12-17 12:33:11,329] jepsen worker 73 - jepsen.util 6373 :invoke :cas [24 [37 7]] INFO [2019-12-17 12:33:11,332] jepsen worker 9 - jepsen.util 7299 :ok :read [3 23] INFO [2019-12-17 12:33:11,332] jepsen worker 9 - jepsen.util 7299 :invoke :write [3 49] INFO [2019-12-17 12:33:11,354] jepsen worker 29 - jepsen.util 5069 :fail :cas [9 [24 13]] :ValueNotSwapFrom INFO [2019-12-17 12:33:11,355] jepsen worker 29 - jepsen.util 5069 :invoke :read [9 nil] INFO [2019-12-17 12:33:11,358] jepsen worker 30 - jepsen.util 6690 :fail :cas [10 [48 27]] :ValueNotSwapFrom INFO [2019-12-17 12:33:11,358] jepsen worker 30 - jepsen.util 6690 :invoke :cas [10 [33 4]] INFO [2019-12-17 12:33:11,358] jepsen worker 31 - jepsen.util 5971 :info :write [10 48] :RequestCanceledException INFO [2019-12-17 12:33:11,359] jepsen worker 31 - jepsen.util 6061 :invoke :write [10 3] INFO [2019-12-17 12:33:11,369] jepsen worker 38 - jepsen.util 6698 :ok :read [12 35] INFO [2019-12-17 12:33:11,369] jepsen worker 38 - jepsen.util 6698 :invoke :read [12 nil] INFO [2019-12-17 12:33:11,376] jepsen worker 70 - jepsen.util 7090 :fail :cas [23 [37 14]] :ValueNotSwapFrom INFO [2019-12-17 12:33:11,376] jepsen worker 70 - jepsen.util 7090 :invoke :cas [23 [28 17]] INFO [2019-12-17 12:33:11,389] jepsen worker 36 - jepsen.util 7236 :fail :cas [12 [28 8]] :ValueNotSwapFrom INFO [2019-12-17 12:33:11,389] jepsen worker 36 - jepsen.util 7236 :invoke :cas [12 [47 29]] INFO [2019-12-17 12:33:11,391] jepsen worker 73 - jepsen.util 6373 :fail :cas [24 [37 7]] :ValueNotSwapFrom INFO [2019-12-17 12:33:11,391] jepsen worker 73 - jepsen.util 6373 :invoke :write [24 8] INFO [2019-12-17 12:33:11,391] jepsen worker 61 - jepsen.util 10231 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:33:11,391] jepsen worker 48 - jepsen.util 8418 :info :write [16 25] :RequestCanceledException INFO [2019-12-17 12:33:11,391] jepsen worker 61 - jepsen.util 10231 :invoke :read [20 nil] INFO [2019-12-17 12:33:11,393] jepsen worker 65 - jepsen.util 5555 :ok :write [21 12] INFO [2019-12-17 12:33:11,393] jepsen worker 65 - jepsen.util 5555 :invoke :read [21 nil] INFO [2019-12-17 12:33:11,393] jepsen worker 48 - jepsen.util 8508 :invoke :cas [16 [48 45]] INFO [2019-12-17 12:33:11,398] jepsen worker 30 - jepsen.util 6690 :fail :cas [10 [33 4]] :ValueNotSwapFrom INFO [2019-12-17 12:33:11,398] jepsen worker 30 - jepsen.util 6690 :invoke :write [10 28] INFO [2019-12-17 12:33:11,412] jepsen worker 8 - jepsen.util 9008 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:33:11,412] jepsen worker 7 - jepsen.util 9457 :info :cas [2 [49 28]] :RequestCanceledException INFO [2019-12-17 12:33:11,412] jepsen worker 8 - jepsen.util 9008 :invoke :write [2 18] INFO [2019-12-17 12:33:11,413] jepsen worker 7 - jepsen.util 9547 :invoke :cas [2 [32 24]] INFO [2019-12-17 12:33:11,428] jepsen worker 57 - jepsen.util 8877 :info :cas [19 [21 3]] :RequestCanceledException INFO [2019-12-17 12:33:11,428] jepsen worker 11 - jepsen.util 7121 :info :cas [3 [3 1]] :RequestCanceledException INFO [2019-12-17 12:33:11,429] jepsen worker 11 - jepsen.util 7211 :invoke :read [3 nil] INFO [2019-12-17 12:33:11,429] jepsen worker 57 - jepsen.util 8967 :invoke :cas [19 [42 15]] INFO [2019-12-17 12:33:11,441] jepsen worker 85 - jepsen.util 10255 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:33:11,442] jepsen worker 85 - jepsen.util 10255 :invoke :read [28 nil] INFO [2019-12-17 12:33:11,461] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":39146,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:33:11,461] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":39146,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:33:11,461] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":39149,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:33:11,461] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":39149,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:33:11,461] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":39150,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:33:11,461] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":39150,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:33:11,492] jepsen worker 52 - jepsen.util 7072 :fail :cas [17 [23 22]] :ValueNotSwapFrom INFO [2019-12-17 12:33:11,492] jepsen worker 52 - jepsen.util 7072 :invoke :cas [17 [12 49]] INFO [2019-12-17 12:33:11,494] jepsen worker 49 - jepsen.util 8419 :info :cas [16 [10 48]] :RequestCanceledException INFO [2019-12-17 12:33:11,495] jepsen worker 49 - jepsen.util 8509 :invoke :read [16 nil] INFO [2019-12-17 12:33:11,508] jepsen worker 27 - jepsen.util 6867 :ok :write [9 25] INFO [2019-12-17 12:33:11,508] jepsen worker 27 - jepsen.util 6867 :invoke :write [9 30] INFO [2019-12-17 12:33:11,516] jepsen worker 13 - jepsen.util 6493 :ok :read [4 18] INFO [2019-12-17 12:33:11,516] jepsen worker 13 - jepsen.util 6493 :invoke :read [4 nil] INFO [2019-12-17 12:33:11,529] jepsen worker 54 - jepsen.util 7434 :info :write [18 18] :RequestCanceledException INFO [2019-12-17 12:33:11,530] jepsen worker 54 - jepsen.util 7524 :invoke :cas [18 [29 36]] INFO [2019-12-17 12:33:11,540] jepsen worker 63 - jepsen.util 5733 :info :write [21 16] :RequestCanceledException INFO [2019-12-17 12:33:11,542] jepsen worker 63 - jepsen.util 5823 :invoke :write [21 16] INFO [2019-12-17 12:33:11,555] jepsen worker 16 - jepsen.util 10276 :info :write [5 16] :RequestCanceledException INFO [2019-12-17 12:33:11,556] jepsen worker 16 - jepsen.util 10366 :invoke :read [5 nil] INFO [2019-12-17 12:33:11,565] jepsen worker 87 - jepsen.util 6027 :fail :cas [29 [8 7]] :ValueNotSwapFrom INFO [2019-12-17 12:33:11,565] jepsen worker 87 - jepsen.util 6027 :invoke :cas [29 [32 43]] INFO [2019-12-17 12:33:11,584] jepsen worker 27 - jepsen.util 6867 :ok :write [9 30] INFO [2019-12-17 12:33:11,585] jepsen worker 27 - jepsen.util 6867 :invoke :write [9 9] INFO [2019-12-17 12:33:11,609] jepsen worker 35 - jepsen.util 9035 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:33:11,609] jepsen worker 35 - jepsen.util 9035 :invoke :read [11 nil] INFO [2019-12-17 12:33:11,614] jepsen worker 38 - jepsen.util 6698 :ok :read [12 35] INFO [2019-12-17 12:33:11,614] jepsen worker 38 - jepsen.util 6698 :invoke :write [12 25] INFO [2019-12-17 12:33:11,627] jepsen worker 3 - jepsen.util 6573 :info :write [1 21] :RequestCanceledException INFO [2019-12-17 12:33:11,627] jepsen worker 19 - jepsen.util 6679 :fail :read [6 nil] :RequestTimeoutException INFO [2019-12-17 12:33:11,628] jepsen worker 19 - jepsen.util 6679 :invoke :read [6 nil] INFO [2019-12-17 12:33:11,629] jepsen worker 3 - jepsen.util 6663 :invoke :write [1 14] INFO [2019-12-17 12:33:11,656] jepsen worker 10 - jepsen.util 6580 :fail :read [3 nil] :RequestTimeoutException INFO [2019-12-17 12:33:11,657] jepsen worker 10 - jepsen.util 6580 :invoke :write [3 49] INFO [2019-12-17 12:33:11,658] jepsen worker 19 - jepsen.util 6679 :ok :read [6 33] INFO [2019-12-17 12:33:11,658] jepsen worker 19 - jepsen.util 6679 :invoke :cas [6 [25 4]] INFO [2019-12-17 12:33:11,671] jepsen worker 12 - jepsen.util 7212 :fail :cas [4 [0 27]] :ValueNotSwapFrom INFO [2019-12-17 12:33:11,671] jepsen worker 12 - jepsen.util 7212 :invoke :cas [4 [37 29]] INFO [2019-12-17 12:33:11,677] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":39170,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:33:11,677] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":39170,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:33:11,677] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":39172,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:33:11,677] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":39172,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:33:11,689] jepsen worker 27 - jepsen.util 6867 :ok :write [9 9] INFO [2019-12-17 12:33:11,690] jepsen worker 27 - jepsen.util 6867 :invoke :write [9 26] INFO [2019-12-17 12:33:11,694] jepsen worker 19 - jepsen.util 6679 :fail :cas [6 [25 4]] :ValueNotSwapFrom INFO [2019-12-17 12:33:11,694] jepsen worker 19 - jepsen.util 6679 :invoke :read [6 nil] INFO [2019-12-17 12:33:11,739] jepsen worker 19 - jepsen.util 6679 :ok :read [6 33] INFO [2019-12-17 12:33:11,739] jepsen worker 19 - jepsen.util 6679 :invoke :write [6 44] INFO [2019-12-17 12:33:11,820] jepsen worker 27 - jepsen.util 6867 :ok :write [9 26] INFO [2019-12-17 12:33:11,820] jepsen worker 27 - jepsen.util 6867 :invoke :cas [9 [47 7]] INFO [2019-12-17 12:33:11,825] jepsen worker 36 - jepsen.util 7236 :fail :cas [12 [47 29]] :ValueNotSwapFrom INFO [2019-12-17 12:33:11,825] jepsen worker 36 - jepsen.util 7236 :invoke :cas [12 [25 36]] INFO [2019-12-17 12:33:11,835] jepsen worker 18 - jepsen.util 6768 :ok :write [6 32] INFO [2019-12-17 12:33:11,835] jepsen worker 18 - jepsen.util 6768 :invoke :cas [6 [45 27]] INFO [2019-12-17 12:33:11,839] jepsen worker 9 - jepsen.util 7299 :ok :write [3 49] INFO [2019-12-17 12:33:11,839] jepsen worker 9 - jepsen.util 7299 :invoke :cas [3 [34 38]] INFO [2019-12-17 12:33:11,840] jepsen worker 27 - jepsen.util 6867 :fail :cas [9 [47 7]] :ValueNotSwapFrom INFO [2019-12-17 12:33:11,841] jepsen worker 27 - jepsen.util 6867 :invoke :cas [9 [44 40]] INFO [2019-12-17 12:33:11,841] jepsen worker 31 - jepsen.util 6061 :ok :write [10 3] INFO [2019-12-17 12:33:11,841] jepsen worker 31 - jepsen.util 6061 :invoke :write [10 33] INFO [2019-12-17 12:33:11,855] jepsen worker 72 - jepsen.util 6552 :info :write [24 19] :RequestCanceledException INFO [2019-12-17 12:33:11,855] jepsen worker 2 - jepsen.util 8912 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:33:11,856] jepsen worker 2 - jepsen.util 8912 :invoke :read [0 nil] INFO [2019-12-17 12:33:11,857] jepsen worker 72 - jepsen.util 6642 :invoke :write [24 15] INFO [2019-12-17 12:33:11,911] jepsen worker 29 - jepsen.util 5069 :ok :read [9 26] INFO [2019-12-17 12:33:11,911] jepsen worker 29 - jepsen.util 5069 :invoke :read [9 nil] INFO [2019-12-17 12:33:11,920] jepsen worker 37 - jepsen.util 6427 :info :write [12 40] :RequestCanceledException INFO [2019-12-17 12:33:11,921] jepsen worker 37 - jepsen.util 6517 :invoke :cas [12 [32 32]] INFO [2019-12-17 12:33:11,939] jepsen worker 27 - jepsen.util 6867 :fail :cas [9 [44 40]] :ValueNotSwapFrom INFO [2019-12-17 12:33:11,939] jepsen worker 27 - jepsen.util 6867 :invoke :write [9 47] INFO [2019-12-17 12:33:11,940] jepsen worker 52 - jepsen.util 7072 :fail :cas [17 [12 49]] :ValueNotSwapFrom INFO [2019-12-17 12:33:11,941] jepsen worker 52 - jepsen.util 7072 :invoke :cas [17 [49 43]] INFO [2019-12-17 12:33:11,945] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":39187,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:33:11,945] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":39187,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:33:11,946] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":39191,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:33:11,946] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":39191,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:33:11,955] jepsen worker 64 - jepsen.util 5284 :info :write [21 33] :RequestCanceledException INFO [2019-12-17 12:33:11,956] jepsen worker 64 - jepsen.util 5374 :invoke :read [21 nil] INFO [2019-12-17 12:33:11,972] jepsen worker 73 - jepsen.util 6373 :ok :write [24 8] INFO [2019-12-17 12:33:11,972] jepsen worker 73 - jepsen.util 6373 :invoke :write [24 46] INFO [2019-12-17 12:33:11,981] jepsen worker 11 - jepsen.util 7211 :ok :read [3 49] INFO [2019-12-17 12:33:11,981] jepsen worker 11 - jepsen.util 7211 :invoke :cas [3 [8 36]] INFO [2019-12-17 12:33:11,982] jepsen worker 78 - jepsen.util 7008 :ok :write [26 38] INFO [2019-12-17 12:33:11,982] jepsen worker 78 - jepsen.util 7008 :invoke :read [26 nil] INFO [2019-12-17 12:33:12,007] jepsen worker 54 - jepsen.util 7524 :fail :cas [18 [29 36]] :ValueNotSwapFrom INFO [2019-12-17 12:33:12,007] jepsen worker 54 - jepsen.util 7524 :invoke :write [18 33] INFO [2019-12-17 12:33:12,039] jepsen worker 18 - jepsen.util 6768 :fail :cas [6 [45 27]] :ValueNotSwapFrom INFO [2019-12-17 12:33:12,039] jepsen worker 18 - jepsen.util 6768 :invoke :cas [6 [33 14]] INFO [2019-12-17 12:33:12,052] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":39202,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:33:12,052] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":39202,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:33:12,052] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":39205,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:33:12,052] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":39205,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:33:12,052] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":39206,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:33:12,052] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":39206,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:33:12,055] jepsen worker 52 - jepsen.util 7072 :fail :cas [17 [49 43]] :ValueNotSwapFrom INFO [2019-12-17 12:33:12,055] jepsen worker 52 - jepsen.util 7072 :invoke :write [17 24] INFO [2019-12-17 12:33:12,071] jepsen worker 29 - jepsen.util 5069 :ok :read [9 26] INFO [2019-12-17 12:33:12,071] jepsen worker 29 - jepsen.util 5069 :invoke :cas [9 [45 48]] INFO [2019-12-17 12:33:12,079] jepsen worker 11 - jepsen.util 7211 :fail :cas [3 [8 36]] :ValueNotSwapFrom INFO [2019-12-17 12:33:12,079] jepsen worker 11 - jepsen.util 7211 :invoke :read [3 nil] INFO [2019-12-17 12:33:12,084] jepsen worker 63 - jepsen.util 5823 :ok :write [21 16] INFO [2019-12-17 12:33:12,084] jepsen worker 63 - jepsen.util 5823 :invoke :cas [21 [20 13]] INFO [2019-12-17 12:33:12,094] jepsen worker 47 - jepsen.util 7337 :fail :read [15 nil] :RequestTimeoutException INFO [2019-12-17 12:33:12,094] jepsen worker 47 - jepsen.util 7337 :invoke :cas [15 [18 26]] INFO [2019-12-17 12:33:12,106] jepsen worker 27 - jepsen.util 6867 :ok :write [9 47] INFO [2019-12-17 12:33:12,106] jepsen worker 27 - jepsen.util 6867 :invoke :cas [9 [33 34]] INFO [2019-12-17 12:33:12,109] jepsen worker 51 - jepsen.util 6531 :info :write [17 29] :RequestCanceledException INFO [2019-12-17 12:33:12,110] jepsen worker 51 - jepsen.util 6621 :invoke :read [17 nil] INFO [2019-12-17 12:33:12,113] jepsen worker 3 - jepsen.util 6663 :ok :write [1 14] INFO [2019-12-17 12:33:12,114] jepsen worker 3 - jepsen.util 6663 :invoke :write [1 38] INFO [2019-12-17 12:33:12,118] jepsen worker 18 - jepsen.util 6768 :fail :cas [6 [33 14]] :ValueNotSwapFrom INFO [2019-12-17 12:33:12,118] jepsen worker 18 - jepsen.util 6768 :invoke :read [6 nil] INFO [2019-12-17 12:33:12,161] jepsen worker 80 - jepsen.util 6740 :ok :write [26 39] INFO [2019-12-17 12:33:12,161] jepsen worker 80 - jepsen.util 6740 :invoke :write [26 45] INFO [2019-12-17 12:33:12,199] jepsen worker 89 - jepsen.util 6569 :ok :read [29 17] INFO [2019-12-17 12:33:12,199] jepsen worker 89 - jepsen.util 6569 :invoke :read [29 nil] INFO [2019-12-17 12:33:12,218] jepsen worker 4 - jepsen.util 6034 :info :write [1 14] :RequestCanceledException INFO [2019-12-17 12:33:12,220] jepsen worker 4 - jepsen.util 6124 :invoke :cas [1 [47 44]] INFO [2019-12-17 12:33:12,223] jepsen worker 29 - jepsen.util 5069 :fail :cas [9 [45 48]] :ValueNotSwapFrom INFO [2019-12-17 12:33:12,223] jepsen worker 29 - jepsen.util 5069 :invoke :read [9 nil] INFO [2019-12-17 12:33:12,226] jepsen worker 47 - jepsen.util 7337 :fail :cas [15 [18 26]] :ValueNotSwapFrom INFO [2019-12-17 12:33:12,226] jepsen worker 47 - jepsen.util 7337 :invoke :cas [15 [16 45]] INFO [2019-12-17 12:33:12,235] jepsen worker 46 - jepsen.util 6886 :info :cas [15 [0 8]] :RequestCanceledException INFO [2019-12-17 12:33:12,236] jepsen worker 46 - jepsen.util 6976 :invoke :cas [15 [27 18]] INFO [2019-12-17 12:33:12,250] jepsen worker 55 - jepsen.util 7795 :fail :cas [18 [37 34]] :ValueNotSwapFrom INFO [2019-12-17 12:33:12,250] jepsen worker 55 - jepsen.util 7795 :invoke :write [18 17] INFO [2019-12-17 12:33:12,270] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":39219,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:33:12,270] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":39219,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:33:12,271] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":39221,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:33:12,271] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":39221,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:33:12,271] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:33:12,272] jepsen nemesis - couchbase.util Rebalance check count: 300 INFO [2019-12-17 12:33:12,272] jepsen nemesis - couchbase.util Rebalance stuck count: 281 INFO [2019-12-17 12:33:12,289] jepsen worker 70 - jepsen.util 7090 :fail :cas [23 [28 17]] :ValueNotSwapFrom INFO [2019-12-17 12:33:12,290] jepsen worker 70 - jepsen.util 7090 :invoke :cas [23 [32 8]] INFO [2019-12-17 12:33:12,291] jepsen worker 27 - jepsen.util 6867 :fail :cas [9 [33 34]] :ValueNotSwapFrom INFO [2019-12-17 12:33:12,291] jepsen worker 27 - jepsen.util 6867 :invoke :read [9 nil] INFO [2019-12-17 12:33:12,308] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":39227,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:33:12,308] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":39227,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:33:12,308] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":39230,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:33:12,308] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":39230,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:33:12,308] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":39231,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:33:12,308] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":39231,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:33:12,311] jepsen worker 32 - jepsen.util 6872 :fail :read [10 nil] :RequestTimeoutException INFO [2019-12-17 12:33:12,311] jepsen worker 32 - jepsen.util 6872 :invoke :cas [10 [26 36]] INFO [2019-12-17 12:33:12,328] jepsen worker 20 - jepsen.util 6500 :info :write [6 32] :RequestCanceledException INFO [2019-12-17 12:33:12,329] jepsen worker 20 - jepsen.util 6590 :invoke :write [6 18] INFO [2019-12-17 12:33:12,331] jepsen worker 47 - jepsen.util 7337 :fail :cas [15 [16 45]] :ValueNotSwapFrom INFO [2019-12-17 12:33:12,331] jepsen worker 47 - jepsen.util 7337 :invoke :read [15 nil] INFO [2019-12-17 12:33:12,342] jepsen worker 69 - jepsen.util 6729 :info :write [23 7] :RequestCanceledException INFO [2019-12-17 12:33:12,343] jepsen worker 69 - jepsen.util 6819 :invoke :read [23 nil] INFO [2019-12-17 12:33:12,352] jepsen worker 63 - jepsen.util 5823 :fail :cas [21 [20 13]] :ValueNotSwapFrom INFO [2019-12-17 12:33:12,353] jepsen worker 63 - jepsen.util 5823 :invoke :cas [21 [8 9]] INFO [2019-12-17 12:33:12,393] jepsen worker 12 - jepsen.util 7212 :fail :cas [4 [37 29]] :ValueNotSwapFrom INFO [2019-12-17 12:33:12,393] jepsen worker 12 - jepsen.util 7212 :invoke :cas [4 [28 7]] INFO [2019-12-17 12:33:12,394] jepsen worker 11 - jepsen.util 7211 :ok :read [3 49] INFO [2019-12-17 12:33:12,394] jepsen worker 11 - jepsen.util 7211 :invoke :write [3 17] INFO [2019-12-17 12:33:12,412] jepsen worker 45 - jepsen.util 6795 :fail :read [15 nil] :RequestTimeoutException INFO [2019-12-17 12:33:12,412] jepsen worker 45 - jepsen.util 6795 :invoke :read [15 nil] INFO [2019-12-17 12:33:12,431] jepsen worker 27 - jepsen.util 6867 :ok :read [9 47] INFO [2019-12-17 12:33:12,432] jepsen worker 27 - jepsen.util 6867 :invoke :cas [9 [10 38]] INFO [2019-12-17 12:33:12,439] jepsen worker 5 - jepsen.util 6305 :info :write [1 19] :RequestCanceledException INFO [2019-12-17 12:33:12,440] jepsen worker 5 - jepsen.util 6395 :invoke :read [1 nil] INFO [2019-12-17 12:33:12,442] jepsen worker 36 - jepsen.util 7236 :fail :cas [12 [25 36]] :ValueNotSwapFrom INFO [2019-12-17 12:33:12,442] jepsen worker 36 - jepsen.util 7236 :invoke :write [12 31] INFO [2019-12-17 12:33:12,444] jepsen worker 79 - jepsen.util 6739 :ok :write [26 40] INFO [2019-12-17 12:33:12,444] jepsen worker 79 - jepsen.util 6739 :invoke :write [26 27] INFO [2019-12-17 12:33:12,445] jepsen worker 63 - jepsen.util 5823 :fail :cas [21 [8 9]] :ValueNotSwapFrom INFO [2019-12-17 12:33:12,445] jepsen worker 63 - jepsen.util 5823 :invoke :write [21 28] INFO [2019-12-17 12:33:12,455] jepsen worker 18 - jepsen.util 6768 :ok :read [6 32] INFO [2019-12-17 12:33:12,455] jepsen worker 18 - jepsen.util 6768 :invoke :write [6 2] INFO [2019-12-17 12:33:12,477] jepsen worker 4 - jepsen.util 6124 :fail :cas [1 [47 44]] :ValueNotSwapFrom INFO [2019-12-17 12:33:12,477] jepsen worker 4 - jepsen.util 6124 :invoke :write [1 47] INFO [2019-12-17 12:33:12,493] jepsen worker 88 - jepsen.util 6748 :ok :write [29 37] INFO [2019-12-17 12:33:12,493] jepsen worker 88 - jepsen.util 6748 :invoke :read [29 nil] INFO [2019-12-17 12:33:12,494] jepsen worker 74 - jepsen.util 6914 :info :write [24 16] :RequestCanceledException INFO [2019-12-17 12:33:12,495] jepsen worker 74 - jepsen.util 7004 :invoke :write [24 41] INFO [2019-12-17 12:33:12,505] jepsen worker 47 - jepsen.util 7337 :ok :read [15 3] INFO [2019-12-17 12:33:12,505] jepsen worker 47 - jepsen.util 7337 :invoke :write [15 46] INFO [2019-12-17 12:33:12,527] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":39248,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:33:12,527] jepsen worker 27 - jepsen.util 6867 :fail :cas [9 [10 38]] :ValueNotSwapFrom INFO [2019-12-17 12:33:12,527] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":39248,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:33:12,528] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":39251,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:33:12,528] jepsen worker 27 - jepsen.util 6867 :invoke :cas [9 [12 13]] INFO [2019-12-17 12:33:12,528] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":39251,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:33:12,528] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":39252,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:33:12,528] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":39252,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:33:12,529] jepsen worker 29 - jepsen.util 5069 :ok :read [9 47] INFO [2019-12-17 12:33:12,529] jepsen worker 29 - jepsen.util 5069 :invoke :write [9 19] INFO [2019-12-17 12:33:12,555] jepsen worker 38 - jepsen.util 6698 :ok :write [12 25] INFO [2019-12-17 12:33:12,555] jepsen worker 38 - jepsen.util 6698 :invoke :write [12 30] INFO [2019-12-17 12:33:12,564] jepsen worker 10 - jepsen.util 6580 :ok :write [3 49] INFO [2019-12-17 12:33:12,565] jepsen worker 10 - jepsen.util 6580 :invoke :write [3 18] INFO [2019-12-17 12:33:12,575] jepsen worker 46 - jepsen.util 6976 :fail :cas [15 [27 18]] :ValueNotSwapFrom INFO [2019-12-17 12:33:12,575] jepsen worker 46 - jepsen.util 6976 :invoke :cas [15 [3 1]] INFO [2019-12-17 12:33:12,607] jepsen worker 13 - jepsen.util 6493 :ok :read [4 18] INFO [2019-12-17 12:33:12,607] jepsen worker 13 - jepsen.util 6493 :invoke :read [4 nil] INFO [2019-12-17 12:33:12,616] jepsen worker 37 - jepsen.util 6517 :fail :cas [12 [32 32]] :ValueNotSwapFrom INFO [2019-12-17 12:33:12,616] jepsen worker 37 - jepsen.util 6517 :invoke :cas [12 [17 49]] INFO [2019-12-17 12:33:12,642] jepsen worker 87 - jepsen.util 6027 :fail :cas [29 [32 43]] :ValueNotSwapFrom INFO [2019-12-17 12:33:12,642] jepsen worker 87 - jepsen.util 6027 :invoke :cas [29 [30 47]] INFO [2019-12-17 12:33:12,645] jepsen worker 51 - jepsen.util 6621 :ok :read [17 29] INFO [2019-12-17 12:33:12,645] jepsen worker 51 - jepsen.util 6621 :invoke :write [17 33] INFO [2019-12-17 12:33:12,655] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":39271,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:33:12,656] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":39271,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:33:12,656] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":39276,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:33:12,656] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":39276,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:33:12,663] jepsen worker 32 - jepsen.util 6872 :fail :cas [10 [26 36]] :ValueNotSwapFrom INFO [2019-12-17 12:33:12,663] jepsen worker 32 - jepsen.util 6872 :invoke :read [10 nil] INFO [2019-12-17 12:33:12,680] jepsen worker 63 - jepsen.util 5823 :ok :write [21 28] INFO [2019-12-17 12:33:12,680] jepsen worker 63 - jepsen.util 5823 :invoke :read [21 nil] INFO [2019-12-17 12:33:12,698] jepsen worker 89 - jepsen.util 6569 :ok :read [29 37] INFO [2019-12-17 12:33:12,699] jepsen worker 89 - jepsen.util 6569 :invoke :write [29 36] INFO [2019-12-17 12:33:12,705] jepsen worker 72 - jepsen.util 6642 :ok :write [24 15] INFO [2019-12-17 12:33:12,705] jepsen worker 72 - jepsen.util 6642 :invoke :read [24 nil] INFO [2019-12-17 12:33:12,720] jepsen worker 63 - jepsen.util 5823 :ok :read [21 28] INFO [2019-12-17 12:33:12,721] jepsen worker 63 - jepsen.util 5823 :invoke :cas [21 [28 34]] INFO [2019-12-17 12:33:12,737] jepsen worker 78 - jepsen.util 7008 :ok :read [26 40] INFO [2019-12-17 12:33:12,737] jepsen worker 78 - jepsen.util 7008 :invoke :cas [26 [19 1]] INFO [2019-12-17 12:33:12,742] jepsen worker 34 - jepsen.util 8584 :info :cas [11 [42 37]] :RequestCanceledException INFO [2019-12-17 12:33:12,743] jepsen worker 34 - jepsen.util 8674 :invoke :write [11 14] INFO [2019-12-17 12:33:12,758] jepsen worker 43 - jepsen.util 8593 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:33:12,758] jepsen worker 77 - jepsen.util 9617 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:33:12,759] jepsen worker 43 - jepsen.util 8593 :invoke :read [14 nil] INFO [2019-12-17 12:33:12,759] jepsen worker 77 - jepsen.util 9617 :invoke :read [25 nil] INFO [2019-12-17 12:33:12,793] jepsen worker 21 - jepsen.util 8661 :info :write [7 4] :RequestCanceledException INFO [2019-12-17 12:33:12,793] jepsen worker 53 - jepsen.util 6353 :info :write [17 46] :RequestCanceledException INFO [2019-12-17 12:33:12,793] jepsen worker 75 - jepsen.util 8715 :info :cas [25 [30 28]] :RequestCanceledException INFO [2019-12-17 12:33:12,794] jepsen worker 53 - jepsen.util 6443 :invoke :write [17 16] INFO [2019-12-17 12:33:12,794] jepsen worker 75 - jepsen.util 8805 :invoke :read [25 nil] INFO [2019-12-17 12:33:12,794] jepsen worker 21 - jepsen.util 8751 :invoke :cas [7 [13 38]] INFO [2019-12-17 12:33:12,802] jepsen worker 63 - jepsen.util 5823 :ok :cas [21 [28 34]] INFO [2019-12-17 12:33:12,802] jepsen worker 63 - jepsen.util 5823 :invoke :write [21 41] INFO [2019-12-17 12:33:12,810] jepsen worker 42 - jepsen.util 9672 :info :write [14 45] :RequestCanceledException INFO [2019-12-17 12:33:12,810] jepsen worker 58 - jepsen.util 9778 :info :cas [19 [27 31]] :RequestCanceledException INFO [2019-12-17 12:33:12,812] jepsen worker 42 - jepsen.util 9762 :invoke :write [14 11] INFO [2019-12-17 12:33:12,812] jepsen worker 58 - jepsen.util 9868 :invoke :cas [19 [48 19]] INFO [2019-12-17 12:33:12,815] jepsen worker 37 - jepsen.util 6517 :fail :cas [12 [17 49]] :ValueNotSwapFrom INFO [2019-12-17 12:33:12,815] jepsen worker 37 - jepsen.util 6517 :invoke :read [12 nil] INFO [2019-12-17 12:33:12,817] jepsen worker 71 - jepsen.util 7541 :ok :write [23 46] INFO [2019-12-17 12:33:12,817] jepsen worker 71 - jepsen.util 7541 :invoke :write [23 24] INFO [2019-12-17 12:33:12,825] jepsen worker 11 - jepsen.util 7211 :ok :write [3 17] INFO [2019-12-17 12:33:12,825] jepsen worker 11 - jepsen.util 7211 :invoke :read [3 nil] INFO [2019-12-17 12:33:12,826] jepsen worker 26 - jepsen.util 9476 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:33:12,826] jepsen worker 23 - jepsen.util 9203 :info :cas [7 [36 20]] :RequestCanceledException INFO [2019-12-17 12:33:12,826] jepsen worker 24 - jepsen.util 8124 :info :cas [8 [25 16]] :RequestCanceledException INFO [2019-12-17 12:33:12,826] jepsen worker 33 - jepsen.util 8943 :info :cas [11 [47 6]] :RequestCanceledException INFO [2019-12-17 12:33:12,826] jepsen worker 59 - jepsen.util 9149 :info :write [19 24] :RequestCanceledException INFO [2019-12-17 12:33:12,826] jepsen worker 68 - jepsen.util 8618 :info :write [22 19] :RequestCanceledException INFO [2019-12-17 12:33:12,826] jepsen worker 62 - jepsen.util 9242 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:33:12,826] jepsen worker 60 - jepsen.util 10050 :info :write [20 8] :RequestCanceledException INFO [2019-12-17 12:33:12,826] jepsen worker 67 - jepsen.util 9427 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:33:12,827] jepsen worker 26 - jepsen.util 9476 :invoke :write [8 45] INFO [2019-12-17 12:33:12,827] jepsen worker 62 - jepsen.util 9242 :invoke :write [20 41] INFO [2019-12-17 12:33:12,827] jepsen worker 67 - jepsen.util 9427 :invoke :read [22 nil] INFO [2019-12-17 12:33:12,828] jepsen worker 23 - jepsen.util 9293 :invoke :write [7 42] INFO [2019-12-17 12:33:12,831] jepsen worker 33 - jepsen.util 9033 :invoke :write [11 38] INFO [2019-12-17 12:33:12,831] jepsen worker 59 - jepsen.util 9239 :invoke :read [19 nil] INFO [2019-12-17 12:33:12,831] jepsen worker 68 - jepsen.util 8708 :invoke :write [22 31] INFO [2019-12-17 12:33:12,831] jepsen worker 24 - jepsen.util 8214 :invoke :read [8 nil] INFO [2019-12-17 12:33:12,832] jepsen worker 60 - jepsen.util 10140 :invoke :read [20 nil] INFO [2019-12-17 12:33:12,841] jepsen worker 15 - jepsen.util 8205 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:33:12,841] jepsen worker 66 - jepsen.util 8976 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:33:12,841] jepsen worker 41 - jepsen.util 9761 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:33:12,841] jepsen worker 15 - jepsen.util 8205 :invoke :cas [5 [48 3]] INFO [2019-12-17 12:33:12,841] jepsen worker 66 - jepsen.util 8976 :invoke :cas [22 [44 44]] INFO [2019-12-17 12:33:12,841] jepsen worker 41 - jepsen.util 9761 :invoke :write [13 40] INFO [2019-12-17 12:33:12,859] jepsen worker 25 - jepsen.util 9565 :info :write [8 15] :RequestCanceledException INFO [2019-12-17 12:33:12,859] jepsen worker 82 - jepsen.util 9892 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:33:12,859] jepsen worker 22 - jepsen.util 9112 :info :cas [7 [24 33]] :RequestCanceledException INFO [2019-12-17 12:33:12,859] jepsen worker 82 - jepsen.util 9892 :invoke :cas [27 [34 40]] INFO [2019-12-17 12:33:12,860] jepsen worker 25 - jepsen.util 9655 :invoke :cas [8 [0 15]] INFO [2019-12-17 12:33:12,860] jepsen worker 22 - jepsen.util 9202 :invoke :cas [7 [25 12]] INFO [2019-12-17 12:33:12,894] jepsen worker 0 - jepsen.util 9540 :info :write [0 34] :RequestCanceledException INFO [2019-12-17 12:33:12,894] jepsen worker 83 - jepsen.util 9623 :info :write [27 23] :RequestCanceledException INFO [2019-12-17 12:33:12,894] jepsen worker 50 - jepsen.util 9680 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:33:12,894] jepsen worker 50 - jepsen.util 9680 :invoke :read [16 nil] INFO [2019-12-17 12:33:12,895] jepsen worker 0 - jepsen.util 9630 :invoke :write [0 25] INFO [2019-12-17 12:33:12,895] jepsen worker 83 - jepsen.util 9713 :invoke :cas [27 [11 25]] INFO [2019-12-17 12:33:12,901] jepsen worker 70 - jepsen.util 7090 :fail :cas [23 [32 8]] :ValueNotSwapFrom INFO [2019-12-17 12:33:12,901] jepsen worker 70 - jepsen.util 7090 :invoke :cas [23 [24 1]] INFO [2019-12-17 12:33:12,904] jepsen worker 86 - jepsen.util 9716 :info :cas [28 [17 26]] :RequestCanceledException INFO [2019-12-17 12:33:12,905] jepsen worker 86 - jepsen.util 9806 :invoke :read [28 nil] INFO [2019-12-17 12:33:12,914] jepsen worker 63 - jepsen.util 5823 :ok :write [21 41] INFO [2019-12-17 12:33:12,914] jepsen worker 63 - jepsen.util 5823 :invoke :write [21 29] INFO [2019-12-17 12:33:12,927] jepsen worker 1 - jepsen.util 9181 :info :write [0 37] :RequestCanceledException INFO [2019-12-17 12:33:12,928] jepsen worker 14 - jepsen.util 6944 :info :cas [4 [18 41]] :RequestCanceledException INFO [2019-12-17 12:33:12,929] jepsen worker 1 - jepsen.util 9271 :invoke :cas [0 [14 45]] INFO [2019-12-17 12:33:12,930] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":39314,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:33:12,930] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":39314,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:33:12,930] jepsen worker 14 - jepsen.util 7034 :invoke :read [4 nil] INFO [2019-12-17 12:33:12,930] jepsen worker 69 - jepsen.util 6819 :ok :read [23 46] INFO [2019-12-17 12:33:12,931] jepsen worker 69 - jepsen.util 6819 :invoke :write [23 46] INFO [2019-12-17 12:33:13,009] jepsen worker 29 - jepsen.util 5069 :ok :write [9 19] INFO [2019-12-17 12:33:13,009] jepsen worker 29 - jepsen.util 5069 :invoke :cas [9 [28 2]] INFO [2019-12-17 12:33:13,038] jepsen worker 12 - jepsen.util 7212 :fail :cas [4 [28 7]] :ValueNotSwapFrom INFO [2019-12-17 12:33:13,038] jepsen worker 12 - jepsen.util 7212 :invoke :read [4 nil] INFO [2019-12-17 12:33:13,039] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":39328,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:33:13,039] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":39328,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:33:13,039] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":39332,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:33:13,040] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":39332,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:33:13,045] jepsen worker 63 - jepsen.util 5823 :ok :write [21 29] INFO [2019-12-17 12:33:13,046] jepsen worker 63 - jepsen.util 5823 :invoke :write [21 45] INFO [2019-12-17 12:33:13,051] jepsen worker 29 - jepsen.util 5069 :fail :cas [9 [28 2]] :ValueNotSwapFrom INFO [2019-12-17 12:33:13,052] jepsen worker 29 - jepsen.util 5069 :invoke :cas [9 [3 8]] INFO [2019-12-17 12:33:13,085] jepsen worker 88 - jepsen.util 6748 :ok :read [29 37] INFO [2019-12-17 12:33:13,085] jepsen worker 88 - jepsen.util 6748 :invoke :read [29 nil] INFO [2019-12-17 12:33:13,087] jepsen worker 20 - jepsen.util 6590 :ok :write [6 18] INFO [2019-12-17 12:33:13,087] jepsen worker 20 - jepsen.util 6590 :invoke :write [6 36] INFO [2019-12-17 12:33:13,090] jepsen worker 31 - jepsen.util 6061 :ok :write [10 33] INFO [2019-12-17 12:33:13,090] jepsen worker 31 - jepsen.util 6061 :invoke :write [10 28] INFO [2019-12-17 12:33:13,095] jepsen worker 47 - jepsen.util 7337 :ok :write [15 46] INFO [2019-12-17 12:33:13,096] jepsen worker 47 - jepsen.util 7337 :invoke :write [15 47] INFO [2019-12-17 12:33:13,119] jepsen worker 29 - jepsen.util 5069 :fail :cas [9 [3 8]] :ValueNotSwapFrom INFO [2019-12-17 12:33:13,119] jepsen worker 29 - jepsen.util 5069 :invoke :cas [9 [3 33]] INFO [2019-12-17 12:33:13,135] jepsen worker 29 - jepsen.util 5069 :fail :cas [9 [3 33]] :ValueNotSwapFrom INFO [2019-12-17 12:33:13,135] jepsen worker 29 - jepsen.util 5069 :invoke :read [9 nil] INFO [2019-12-17 12:33:13,138] jepsen worker 37 - jepsen.util 6517 :ok :read [12 25] INFO [2019-12-17 12:33:13,138] jepsen worker 37 - jepsen.util 6517 :invoke :cas [12 [28 40]] INFO [2019-12-17 12:33:13,167] jepsen worker 12 - jepsen.util 7212 :ok :read [4 18] INFO [2019-12-17 12:33:13,168] jepsen worker 12 - jepsen.util 7212 :invoke :cas [4 [0 35]] INFO [2019-12-17 12:33:13,176] jepsen worker 29 - jepsen.util 5069 :ok :read [9 19] INFO [2019-12-17 12:33:13,177] jepsen worker 29 - jepsen.util 5069 :invoke :read [9 nil] INFO [2019-12-17 12:33:13,191] jepsen worker 63 - jepsen.util 5823 :ok :write [21 45] INFO [2019-12-17 12:33:13,191] jepsen worker 63 - jepsen.util 5823 :invoke :read [21 nil] INFO [2019-12-17 12:33:13,197] jepsen worker 29 - jepsen.util 5069 :ok :read [9 19] INFO [2019-12-17 12:33:13,197] jepsen worker 29 - jepsen.util 5069 :invoke :read [9 nil] INFO [2019-12-17 12:33:13,239] jepsen worker 78 - jepsen.util 7008 :fail :cas [26 [19 1]] :ValueNotSwapFrom INFO [2019-12-17 12:33:13,239] jepsen worker 63 - jepsen.util 5823 :ok :read [21 45] INFO [2019-12-17 12:33:13,239] jepsen worker 78 - jepsen.util 7008 :invoke :read [26 nil] INFO [2019-12-17 12:33:13,239] jepsen worker 63 - jepsen.util 5823 :invoke :write [21 43] INFO [2019-12-17 12:33:13,241] jepsen worker 76 - jepsen.util 9526 :info :write [25 49] :RequestCanceledException INFO [2019-12-17 12:33:13,241] jepsen worker 84 - jepsen.util 8634 :info :cas [28 [9 36]] :RequestCanceledException INFO [2019-12-17 12:33:13,241] jepsen worker 29 - jepsen.util 5069 :ok :read [9 19] INFO [2019-12-17 12:33:13,242] jepsen worker 29 - jepsen.util 5069 :invoke :cas [9 [44 48]] INFO [2019-12-17 12:33:13,242] jepsen worker 76 - jepsen.util 9616 :invoke :write [25 19] INFO [2019-12-17 12:33:13,242] jepsen worker 84 - jepsen.util 8724 :invoke :write [28 17] INFO [2019-12-17 12:33:13,258] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":39348,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:33:13,258] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":39348,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:33:13,258] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":39349,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:33:13,258] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":39349,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:33:13,258] jepsen worker 39 - jepsen.util 9129 :info :write [13 15] :RequestCanceledException INFO [2019-12-17 12:33:13,259] jepsen worker 39 - jepsen.util 9219 :invoke :read [13 nil] INFO [2019-12-17 12:33:13,260] jepsen worker 29 - jepsen.util 5069 :fail :cas [9 [44 48]] :ValueNotSwapFrom INFO [2019-12-17 12:33:13,260] jepsen worker 29 - jepsen.util 5069 :invoke :cas [9 [17 17]] INFO [2019-12-17 12:33:13,286] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:33:13,286] jepsen nemesis - couchbase.util Rebalance check count: 301 INFO [2019-12-17 12:33:13,286] jepsen nemesis - couchbase.util Rebalance stuck count: 282 INFO [2019-12-17 12:33:13,287] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":39355,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:33:13,287] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":39355,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:33:13,287] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":39358,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:33:13,287] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":39358,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:33:13,295] jepsen worker 29 - jepsen.util 5069 :fail :cas [9 [17 17]] :ValueNotSwapFrom INFO [2019-12-17 12:33:13,295] jepsen worker 29 - jepsen.util 5069 :invoke :cas [9 [29 41]] INFO [2019-12-17 12:33:13,295] jepsen worker 17 - jepsen.util 9557 :info :cas [5 [19 29]] :RequestCanceledException INFO [2019-12-17 12:33:13,296] jepsen worker 17 - jepsen.util 9647 :invoke :read [5 nil] INFO [2019-12-17 12:33:13,306] jepsen worker 88 - jepsen.util 6748 :ok :read [29 37] INFO [2019-12-17 12:33:13,307] jepsen worker 88 - jepsen.util 6748 :invoke :write [29 33] INFO [2019-12-17 12:33:13,309] jepsen worker 6 - jepsen.util 9816 :info :write [2 26] :RequestCanceledException INFO [2019-12-17 12:33:13,309] jepsen worker 81 - jepsen.util 8901 :info :cas [27 [9 36]] :RequestCanceledException INFO [2019-12-17 12:33:13,309] jepsen worker 28 - jepsen.util 5248 :info :cas [9 [20 35]] :RequestCanceledException INFO [2019-12-17 12:33:13,310] jepsen worker 6 - jepsen.util 9906 :invoke :read [2 nil] INFO [2019-12-17 12:33:13,310] jepsen worker 28 - jepsen.util 5338 :invoke :read [9 nil] INFO [2019-12-17 12:33:13,310] jepsen worker 81 - jepsen.util 8991 :invoke :cas [27 [19 21]] INFO [2019-12-17 12:33:13,311] jepsen worker 14 - jepsen.util 7034 :ok :read [4 18] INFO [2019-12-17 12:33:13,311] jepsen worker 14 - jepsen.util 7034 :invoke :cas [4 [47 6]] INFO [2019-12-17 12:33:13,325] jepsen worker 29 - jepsen.util 5069 :fail :cas [9 [29 41]] :ValueNotSwapFrom INFO [2019-12-17 12:33:13,325] jepsen worker 29 - jepsen.util 5069 :invoke :write [9 44] INFO [2019-12-17 12:33:13,328] jepsen worker 56 - jepsen.util 7436 :info :write [18 10] :RequestCanceledException INFO [2019-12-17 12:33:13,329] jepsen worker 56 - jepsen.util 7526 :invoke :write [18 13] INFO [2019-12-17 12:33:13,331] jepsen worker 63 - jepsen.util 5823 :ok :write [21 43] INFO [2019-12-17 12:33:13,331] jepsen worker 63 - jepsen.util 5823 :invoke :read [21 nil] INFO [2019-12-17 12:33:13,334] jepsen worker 12 - jepsen.util 7212 :fail :cas [4 [0 35]] :ValueNotSwapFrom INFO [2019-12-17 12:33:13,334] jepsen worker 12 - jepsen.util 7212 :invoke :write [4 42] INFO [2019-12-17 12:33:13,341] jepsen worker 44 - jepsen.util 10034 :info :cas [14 [1 0]] :RequestCanceledException INFO [2019-12-17 12:33:13,341] jepsen worker 40 - jepsen.util 9760 :info :cas [13 [42 18]] :RequestCanceledException INFO [2019-12-17 12:33:13,342] jepsen worker 44 - jepsen.util 10124 :invoke :read [14 nil] INFO [2019-12-17 12:33:13,342] jepsen worker 40 - jepsen.util 9850 :invoke :read [13 nil] INFO [2019-12-17 12:33:13,352] jepsen worker 63 - jepsen.util 5823 :ok :read [21 43] INFO [2019-12-17 12:33:13,352] jepsen worker 63 - jepsen.util 5823 :invoke :write [21 29] INFO [2019-12-17 12:33:13,425] jepsen worker 29 - jepsen.util 5069 :ok :write [9 44] INFO [2019-12-17 12:33:13,425] jepsen worker 29 - jepsen.util 5069 :invoke :cas [9 [12 14]] INFO [2019-12-17 12:33:13,427] jepsen worker 48 - jepsen.util 8508 :info :cas [16 [48 45]] :RequestCanceledException INFO [2019-12-17 12:33:13,428] jepsen worker 30 - jepsen.util 6690 :info :write [10 28] :RequestCanceledException INFO [2019-12-17 12:33:13,428] jepsen worker 48 - jepsen.util 8598 :invoke :read [16 nil] INFO [2019-12-17 12:33:13,429] jepsen worker 30 - jepsen.util 6780 :invoke :write [10 20] INFO [2019-12-17 12:33:13,437] jepsen worker 70 - jepsen.util 7090 :fail :cas [23 [24 1]] :ValueNotSwapFrom INFO [2019-12-17 12:33:13,438] jepsen worker 70 - jepsen.util 7090 :invoke :write [23 20] INFO [2019-12-17 12:33:13,446] jepsen worker 29 - jepsen.util 5069 :fail :cas [9 [12 14]] :ValueNotSwapFrom INFO [2019-12-17 12:33:13,446] jepsen worker 29 - jepsen.util 5069 :invoke :read [9 nil] INFO [2019-12-17 12:33:13,450] jepsen worker 14 - jepsen.util 7034 :fail :cas [4 [47 6]] :ValueNotSwapFrom INFO [2019-12-17 12:33:13,450] jepsen worker 14 - jepsen.util 7034 :invoke :read [4 nil] INFO [2019-12-17 12:33:13,460] jepsen worker 78 - jepsen.util 7008 :ok :read [26 40] INFO [2019-12-17 12:33:13,460] jepsen worker 78 - jepsen.util 7008 :invoke :write [26 45] INFO [2019-12-17 12:33:13,462] jepsen worker 63 - jepsen.util 5823 :ok :write [21 29] INFO [2019-12-17 12:33:13,462] jepsen worker 63 - jepsen.util 5823 :invoke :read [21 nil] INFO [2019-12-17 12:33:13,464] jepsen worker 29 - jepsen.util 5069 :ok :read [9 44] INFO [2019-12-17 12:33:13,465] jepsen worker 29 - jepsen.util 5069 :invoke :write [9 31] INFO [2019-12-17 12:33:13,468] jepsen worker 31 - jepsen.util 6061 :ok :write [10 28] INFO [2019-12-17 12:33:13,468] jepsen worker 20 - jepsen.util 6590 :ok :write [6 36] INFO [2019-12-17 12:33:13,468] jepsen worker 20 - jepsen.util 6590 :invoke :write [6 28] INFO [2019-12-17 12:33:13,468] jepsen worker 31 - jepsen.util 6061 :invoke :cas [10 [27 21]] INFO [2019-12-17 12:33:13,515] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":39386,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:33:13,516] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":39386,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:33:13,516] jepsen worker 8 - jepsen.util 9008 :info :write [2 18] :RequestCanceledException INFO [2019-12-17 12:33:13,516] jepsen worker 61 - jepsen.util 10231 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:33:13,516] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":39388,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:33:13,516] jepsen worker 65 - jepsen.util 5555 :fail :read [21 nil] :RequestTimeoutException INFO [2019-12-17 12:33:13,516] jepsen worker 7 - jepsen.util 9547 :info :cas [2 [32 24]] :RequestCanceledException INFO [2019-12-17 12:33:13,516] jepsen worker 61 - jepsen.util 10231 :invoke :read [20 nil] INFO [2019-12-17 12:33:13,516] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":39388,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:33:13,516] jepsen worker 65 - jepsen.util 5555 :invoke :cas [21 [7 32]] INFO [2019-12-17 12:33:13,517] jepsen worker 7 - jepsen.util 9637 :invoke :read [2 nil] INFO [2019-12-17 12:33:13,517] jepsen worker 8 - jepsen.util 9098 :invoke :write [2 14] INFO [2019-12-17 12:33:13,525] jepsen worker 37 - jepsen.util 6517 :fail :cas [12 [28 40]] :ValueNotSwapFrom INFO [2019-12-17 12:33:13,525] jepsen worker 5 - jepsen.util 6395 :ok :read [1 14] INFO [2019-12-17 12:33:13,525] jepsen worker 37 - jepsen.util 6517 :invoke :cas [12 [15 12]] INFO [2019-12-17 12:33:13,525] jepsen worker 63 - jepsen.util 5823 :ok :read [21 29] INFO [2019-12-17 12:33:13,526] jepsen worker 5 - jepsen.util 6395 :invoke :write [1 32] INFO [2019-12-17 12:33:13,527] jepsen worker 31 - jepsen.util 6061 :fail :cas [10 [27 21]] :ValueNotSwapFrom INFO [2019-12-17 12:33:13,527] jepsen worker 63 - jepsen.util 5823 :invoke :read [21 nil] INFO [2019-12-17 12:33:13,527] jepsen worker 31 - jepsen.util 6061 :invoke :cas [10 [6 46]] INFO [2019-12-17 12:33:13,528] jepsen worker 57 - jepsen.util 8967 :info :cas [19 [42 15]] :RequestCanceledException INFO [2019-12-17 12:33:13,529] jepsen worker 57 - jepsen.util 9057 :invoke :read [19 nil] INFO [2019-12-17 12:33:13,540] jepsen worker 47 - jepsen.util 7337 :ok :write [15 47] INFO [2019-12-17 12:33:13,540] jepsen worker 47 - jepsen.util 7337 :invoke :read [15 nil] INFO [2019-12-17 12:33:13,541] jepsen worker 85 - jepsen.util 10255 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:33:13,541] jepsen worker 49 - jepsen.util 8509 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:33:13,541] jepsen worker 85 - jepsen.util 10255 :invoke :write [28 4] INFO [2019-12-17 12:33:13,541] jepsen worker 49 - jepsen.util 8509 :invoke :cas [16 [15 23]] INFO [2019-12-17 12:33:13,595] jepsen worker 16 - jepsen.util 10366 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:33:13,595] jepsen worker 16 - jepsen.util 10366 :invoke :cas [5 [9 30]] INFO [2019-12-17 12:33:13,603] jepsen worker 31 - jepsen.util 6061 :fail :cas [10 [6 46]] :ValueNotSwapFrom INFO [2019-12-17 12:33:13,603] jepsen worker 31 - jepsen.util 6061 :invoke :write [10 4] INFO [2019-12-17 12:33:13,613] jepsen worker 63 - jepsen.util 5823 :ok :read [21 29] INFO [2019-12-17 12:33:13,613] jepsen worker 63 - jepsen.util 5823 :invoke :write [21 28] INFO [2019-12-17 12:33:13,647] jepsen worker 28 - jepsen.util 5338 :ok :read [9 44] INFO [2019-12-17 12:33:13,647] jepsen worker 28 - jepsen.util 5338 :invoke :write [9 5] INFO [2019-12-17 12:33:13,684] jepsen worker 29 - jepsen.util 5069 :ok :write [9 31] INFO [2019-12-17 12:33:13,684] jepsen worker 29 - jepsen.util 5069 :invoke :write [9 20] INFO [2019-12-17 12:33:13,689] jepsen worker 14 - jepsen.util 7034 :ok :read [4 18] INFO [2019-12-17 12:33:13,689] jepsen worker 14 - jepsen.util 7034 :invoke :write [4 11] INFO [2019-12-17 12:33:13,712] jepsen worker 35 - jepsen.util 9035 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:33:13,712] jepsen worker 35 - jepsen.util 9035 :invoke :read [11 nil] INFO [2019-12-17 12:33:13,735] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":39417,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:33:13,736] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":39417,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:33:13,736] jepsen worker 63 - jepsen.util 5823 :ok :write [21 28] INFO [2019-12-17 12:33:13,736] jepsen worker 63 - jepsen.util 5823 :invoke :cas [21 [38 3]] INFO [2019-12-17 12:33:13,760] jepsen worker 63 - jepsen.util 5823 :fail :cas [21 [38 3]] :ValueNotSwapFrom INFO [2019-12-17 12:33:13,760] jepsen worker 63 - jepsen.util 5823 :invoke :read [21 nil] INFO [2019-12-17 12:33:13,788] jepsen worker 63 - jepsen.util 5823 :ok :read [21 28] INFO [2019-12-17 12:33:13,788] jepsen worker 63 - jepsen.util 5823 :invoke :cas [21 [20 38]] INFO [2019-12-17 12:33:13,798] jepsen worker 29 - jepsen.util 5069 :ok :write [9 20] INFO [2019-12-17 12:33:13,798] jepsen worker 29 - jepsen.util 5069 :invoke :write [9 35] INFO [2019-12-17 12:33:13,821] jepsen worker 63 - jepsen.util 5823 :fail :cas [21 [20 38]] :ValueNotSwapFrom INFO [2019-12-17 12:33:13,821] jepsen worker 63 - jepsen.util 5823 :invoke :write [21 16] INFO [2019-12-17 12:33:13,826] jepsen worker 19 - jepsen.util 6679 :info :write [6 44] :RequestCanceledException INFO [2019-12-17 12:33:13,827] jepsen worker 19 - jepsen.util 6769 :invoke :read [6 nil] INFO [2019-12-17 12:33:13,842] jepsen worker 13 - jepsen.util 6493 :ok :read [4 18] INFO [2019-12-17 12:33:13,842] jepsen worker 13 - jepsen.util 6493 :invoke :cas [4 [37 45]] INFO [2019-12-17 12:33:13,849] jepsen worker 19 - jepsen.util 6769 :ok :read [6 36] INFO [2019-12-17 12:33:13,849] jepsen worker 19 - jepsen.util 6769 :invoke :read [6 nil] INFO [2019-12-17 12:33:13,878] jepsen worker 19 - jepsen.util 6769 :ok :read [6 36] INFO [2019-12-17 12:33:13,878] jepsen worker 19 - jepsen.util 6769 :invoke :cas [6 [36 34]] INFO [2019-12-17 12:33:13,903] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":39432,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:33:13,903] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":39432,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:33:13,903] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":39436,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:33:13,903] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":39436,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:33:13,908] jepsen worker 87 - jepsen.util 6027 :fail :cas [29 [30 47]] :ValueNotSwapFrom INFO [2019-12-17 12:33:13,908] jepsen worker 87 - jepsen.util 6027 :invoke :write [29 10] INFO [2019-12-17 12:33:13,922] jepsen worker 29 - jepsen.util 5069 :ok :write [9 35] INFO [2019-12-17 12:33:13,922] jepsen worker 29 - jepsen.util 5069 :invoke :read [9 nil] INFO [2019-12-17 12:33:13,926] jepsen worker 9 - jepsen.util 7299 :info :cas [3 [34 38]] :RequestCanceledException INFO [2019-12-17 12:33:13,927] jepsen worker 9 - jepsen.util 7389 :invoke :write [3 21] INFO [2019-12-17 12:33:13,940] jepsen worker 63 - jepsen.util 5823 :ok :write [21 16] INFO [2019-12-17 12:33:13,940] jepsen worker 63 - jepsen.util 5823 :invoke :write [21 6] INFO [2019-12-17 12:33:13,947] jepsen worker 29 - jepsen.util 5069 :ok :read [9 35] INFO [2019-12-17 12:33:13,948] jepsen worker 29 - jepsen.util 5069 :invoke :read [9 nil] INFO [2019-12-17 12:33:13,954] jepsen worker 2 - jepsen.util 8912 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:33:13,954] jepsen worker 2 - jepsen.util 8912 :invoke :write [0 7] INFO [2019-12-17 12:33:13,983] jepsen worker 29 - jepsen.util 5069 :ok :read [9 35] INFO [2019-12-17 12:33:13,984] jepsen worker 29 - jepsen.util 5069 :invoke :write [9 4] INFO [2019-12-17 12:33:14,010] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":39445,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:33:14,010] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":39445,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:33:14,010] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":39449,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:33:14,010] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":39449,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:33:14,010] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":39450,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:33:14,010] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":39450,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:33:14,041] jepsen worker 73 - jepsen.util 6373 :info :write [24 46] :RequestCanceledException INFO [2019-12-17 12:33:14,042] jepsen worker 73 - jepsen.util 6463 :invoke :cas [24 [18 18]] INFO [2019-12-17 12:33:14,059] jepsen worker 64 - jepsen.util 5374 :fail :read [21 nil] :RequestTimeoutException INFO [2019-12-17 12:33:14,059] jepsen worker 54 - jepsen.util 7524 :info :write [18 33] :RequestCanceledException INFO [2019-12-17 12:33:14,059] jepsen worker 64 - jepsen.util 5374 :invoke :read [21 nil] INFO [2019-12-17 12:33:14,060] jepsen worker 54 - jepsen.util 7614 :invoke :read [18 nil] INFO [2019-12-17 12:33:14,066] jepsen worker 29 - jepsen.util 5069 :ok :write [9 4] INFO [2019-12-17 12:33:14,067] jepsen worker 29 - jepsen.util 5069 :invoke :read [9 nil] INFO [2019-12-17 12:33:14,072] jepsen worker 14 - jepsen.util 7034 :ok :write [4 11] INFO [2019-12-17 12:33:14,072] jepsen worker 14 - jepsen.util 7034 :invoke :read [4 nil] INFO [2019-12-17 12:33:14,074] jepsen worker 64 - jepsen.util 5374 :ok :read [21 16] INFO [2019-12-17 12:33:14,074] jepsen worker 64 - jepsen.util 5374 :invoke :cas [21 [47 2]] INFO [2019-12-17 12:33:14,080] jepsen worker 29 - jepsen.util 5069 :ok :read [9 4] INFO [2019-12-17 12:33:14,081] jepsen worker 29 - jepsen.util 5069 :invoke :read [9 nil] INFO [2019-12-17 12:33:14,103] jepsen worker 64 - jepsen.util 5374 :fail :cas [21 [47 2]] :ValueNotSwapFrom INFO [2019-12-17 12:33:14,103] jepsen worker 64 - jepsen.util 5374 :invoke :write [21 24] INFO [2019-12-17 12:33:14,111] jepsen worker 20 - jepsen.util 6590 :ok :write [6 28] INFO [2019-12-17 12:33:14,112] jepsen worker 29 - jepsen.util 5069 :ok :read [9 4] INFO [2019-12-17 12:33:14,116] jepsen worker 29 - jepsen.util 5069 :invoke :cas [9 [1 26]] INFO [2019-12-17 12:33:14,116] jepsen worker 20 - jepsen.util 6590 :invoke :cas [6 [25 30]] INFO [2019-12-17 12:33:14,159] jepsen worker 52 - jepsen.util 7072 :info :write [17 24] :RequestCanceledException INFO [2019-12-17 12:33:14,160] jepsen worker 52 - jepsen.util 7162 :invoke :cas [17 [11 30]] INFO [2019-12-17 12:33:14,165] jepsen worker 20 - jepsen.util 6590 :fail :cas [6 [25 30]] :ValueNotSwapFrom INFO [2019-12-17 12:33:14,166] jepsen worker 29 - jepsen.util 5069 :fail :cas [9 [1 26]] :ValueNotSwapFrom INFO [2019-12-17 12:33:14,166] jepsen worker 20 - jepsen.util 6590 :invoke :read [6 nil] INFO [2019-12-17 12:33:14,166] jepsen worker 29 - jepsen.util 5069 :invoke :write [9 38] INFO [2019-12-17 12:33:14,176] jepsen worker 14 - jepsen.util 7034 :ok :read [4 11] INFO [2019-12-17 12:33:14,177] jepsen worker 14 - jepsen.util 7034 :invoke :read [4 nil] INFO [2019-12-17 12:33:14,186] jepsen worker 30 - jepsen.util 6780 :ok :write [10 20] INFO [2019-12-17 12:33:14,186] jepsen worker 30 - jepsen.util 6780 :invoke :cas [10 [11 47]] INFO [2019-12-17 12:33:14,195] jepsen worker 80 - jepsen.util 6740 :info :write [26 45] :RequestCanceledException INFO [2019-12-17 12:33:14,196] jepsen worker 80 - jepsen.util 6830 :invoke :write [26 46] INFO [2019-12-17 12:33:14,203] jepsen worker 20 - jepsen.util 6590 :ok :read [6 28] INFO [2019-12-17 12:33:14,203] jepsen worker 20 - jepsen.util 6590 :invoke :write [6 49] INFO [2019-12-17 12:33:14,204] jepsen worker 3 - jepsen.util 6663 :info :write [1 38] :RequestCanceledException INFO [2019-12-17 12:33:14,205] jepsen worker 3 - jepsen.util 6753 :invoke :cas [1 [5 44]] INFO [2019-12-17 12:33:14,207] jepsen worker 9 - jepsen.util 7389 :ok :write [3 21] INFO [2019-12-17 12:33:14,207] jepsen worker 9 - jepsen.util 7389 :invoke :cas [3 [32 4]] INFO [2019-12-17 12:33:14,222] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":39467,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:33:14,222] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":39467,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:33:14,222] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":39468,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:33:14,222] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":39468,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:33:14,222] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":39469,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:33:14,222] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":39469,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:33:14,242] jepsen worker 14 - jepsen.util 7034 :ok :read [4 11] INFO [2019-12-17 12:33:14,242] jepsen worker 14 - jepsen.util 7034 :invoke :cas [4 [45 22]] INFO [2019-12-17 12:33:14,254] jepsen worker 9 - jepsen.util 7389 :fail :cas [3 [32 4]] :ValueNotSwapFrom INFO [2019-12-17 12:33:14,254] jepsen worker 9 - jepsen.util 7389 :invoke :write [3 0] INFO [2019-12-17 12:33:14,261] jepsen worker 37 - jepsen.util 6517 :fail :cas [12 [15 12]] :ValueNotSwapFrom INFO [2019-12-17 12:33:14,261] jepsen worker 37 - jepsen.util 6517 :invoke :cas [12 [23 15]] INFO [2019-12-17 12:33:14,262] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":39478,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:33:14,262] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":39478,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:33:14,262] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":39482,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:33:14,262] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":39482,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:33:14,277] jepsen worker 28 - jepsen.util 5338 :ok :write [9 5] INFO [2019-12-17 12:33:14,277] jepsen worker 64 - jepsen.util 5374 :ok :write [21 24] INFO [2019-12-17 12:33:14,277] jepsen worker 28 - jepsen.util 5338 :invoke :read [9 nil] INFO [2019-12-17 12:33:14,277] jepsen worker 64 - jepsen.util 5374 :invoke :write [21 10] INFO [2019-12-17 12:33:14,321] jepsen worker 30 - jepsen.util 6780 :fail :cas [10 [11 47]] :ValueNotSwapFrom INFO [2019-12-17 12:33:14,321] jepsen worker 30 - jepsen.util 6780 :invoke :cas [10 [18 35]] INFO [2019-12-17 12:33:14,321] jepsen worker 14 - jepsen.util 7034 :fail :cas [4 [45 22]] :ValueNotSwapFrom INFO [2019-12-17 12:33:14,322] jepsen worker 14 - jepsen.util 7034 :invoke :read [4 nil] INFO [2019-12-17 12:33:14,327] jepsen worker 55 - jepsen.util 7795 :info :write [18 17] :RequestCanceledException INFO [2019-12-17 12:33:14,329] jepsen worker 55 - jepsen.util 7885 :invoke :write [18 15] INFO [2019-12-17 12:33:14,330] jepsen worker 28 - jepsen.util 5338 :ok :read [9 5] INFO [2019-12-17 12:33:14,330] jepsen worker 28 - jepsen.util 5338 :invoke :read [9 nil] INFO [2019-12-17 12:33:14,339] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:33:14,340] jepsen nemesis - couchbase.util Rebalance check count: 302 INFO [2019-12-17 12:33:14,340] jepsen nemesis - couchbase.util Rebalance stuck count: 283 INFO [2019-12-17 12:33:14,346] jepsen worker 30 - jepsen.util 6780 :fail :cas [10 [18 35]] :ValueNotSwapFrom INFO [2019-12-17 12:33:14,347] jepsen worker 30 - jepsen.util 6780 :invoke :cas [10 [18 32]] INFO [2019-12-17 12:33:14,362] jepsen worker 28 - jepsen.util 5338 :ok :read [9 5] INFO [2019-12-17 12:33:14,362] jepsen worker 28 - jepsen.util 5338 :invoke :read [9 nil] INFO [2019-12-17 12:33:14,369] jepsen worker 20 - jepsen.util 6590 :ok :write [6 49] INFO [2019-12-17 12:33:14,369] jepsen worker 20 - jepsen.util 6590 :invoke :cas [6 [6 2]] INFO [2019-12-17 12:33:14,405] jepsen worker 30 - jepsen.util 6780 :fail :cas [10 [18 32]] :ValueNotSwapFrom INFO [2019-12-17 12:33:14,407] jepsen worker 30 - jepsen.util 6780 :invoke :cas [10 [15 48]] INFO [2019-12-17 12:33:14,419] jepsen worker 20 - jepsen.util 6590 :fail :cas [6 [6 2]] :ValueNotSwapFrom INFO [2019-12-17 12:33:14,419] jepsen worker 20 - jepsen.util 6590 :invoke :read [6 nil] INFO [2019-12-17 12:33:14,426] jepsen worker 28 - jepsen.util 5338 :ok :read [9 5] INFO [2019-12-17 12:33:14,426] jepsen worker 28 - jepsen.util 5338 :invoke :read [9 nil] INFO [2019-12-17 12:33:14,441] jepsen worker 30 - jepsen.util 6780 :fail :cas [10 [15 48]] :ValueNotSwapFrom INFO [2019-12-17 12:33:14,441] jepsen worker 30 - jepsen.util 6780 :invoke :write [10 25] INFO [2019-12-17 12:33:14,457] jepsen worker 20 - jepsen.util 6590 :ok :read [6 49] INFO [2019-12-17 12:33:14,457] jepsen worker 20 - jepsen.util 6590 :invoke :cas [6 [47 11]] INFO [2019-12-17 12:33:14,481] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":39500,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:33:14,481] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":39500,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:33:14,481] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":39503,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:33:14,481] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":39503,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:33:14,485] jepsen worker 20 - jepsen.util 6590 :fail :cas [6 [47 11]] :ValueNotSwapFrom INFO [2019-12-17 12:33:14,485] jepsen worker 20 - jepsen.util 6590 :invoke :write [6 48] INFO [2019-12-17 12:33:14,495] jepsen worker 28 - jepsen.util 5338 :ok :read [9 5] INFO [2019-12-17 12:33:14,495] jepsen worker 4 - jepsen.util 6124 :info :write [1 47] :RequestCanceledException INFO [2019-12-17 12:33:14,495] jepsen worker 28 - jepsen.util 5338 :invoke :write [9 3] INFO [2019-12-17 12:33:14,496] jepsen worker 4 - jepsen.util 6214 :invoke :read [1 nil] INFO [2019-12-17 12:33:14,504] jepsen worker 45 - jepsen.util 6795 :fail :read [15 nil] :RequestTimeoutException INFO [2019-12-17 12:33:14,504] jepsen worker 36 - jepsen.util 7236 :info :write [12 31] :RequestCanceledException INFO [2019-12-17 12:33:14,504] jepsen worker 18 - jepsen.util 6768 :info :write [6 2] :RequestCanceledException INFO [2019-12-17 12:33:14,504] jepsen worker 45 - jepsen.util 6795 :invoke :cas [15 [41 21]] INFO [2019-12-17 12:33:14,505] jepsen worker 18 - jepsen.util 6858 :invoke :cas [6 [17 49]] INFO [2019-12-17 12:33:14,505] jepsen worker 36 - jepsen.util 7326 :invoke :read [12 nil] INFO [2019-12-17 12:33:14,519] jepsen worker 13 - jepsen.util 6493 :fail :cas [4 [37 45]] :ValueNotSwapFrom INFO [2019-12-17 12:33:14,519] jepsen worker 13 - jepsen.util 6493 :invoke :cas [4 [17 29]] INFO [2019-12-17 12:33:14,527] jepsen worker 79 - jepsen.util 6739 :info :write [26 27] :RequestCanceledException INFO [2019-12-17 12:33:14,527] jepsen worker 74 - jepsen.util 7004 :info :write [24 41] :RequestCanceledException INFO [2019-12-17 12:33:14,529] jepsen worker 79 - jepsen.util 6829 :invoke :write [26 15] INFO [2019-12-17 12:33:14,529] jepsen worker 74 - jepsen.util 7094 :invoke :read [24 nil] INFO [2019-12-17 12:33:14,546] jepsen worker 54 - jepsen.util 7614 :ok :read [18 5] INFO [2019-12-17 12:33:14,546] jepsen worker 51 - jepsen.util 6621 :ok :write [17 33] INFO [2019-12-17 12:33:14,546] jepsen worker 54 - jepsen.util 7614 :invoke :write [18 37] INFO [2019-12-17 12:33:14,546] jepsen worker 51 - jepsen.util 6621 :invoke :cas [17 [15 48]] INFO [2019-12-17 12:33:14,559] jepsen worker 45 - jepsen.util 6795 :fail :cas [15 [41 21]] :ValueNotSwapFrom INFO [2019-12-17 12:33:14,559] jepsen worker 45 - jepsen.util 6795 :invoke :write [15 18] INFO [2019-12-17 12:33:14,570] jepsen worker 18 - jepsen.util 6858 :fail :cas [6 [17 49]] :ValueNotSwapFrom INFO [2019-12-17 12:33:14,570] jepsen worker 18 - jepsen.util 6858 :invoke :read [6 nil] INFO [2019-12-17 12:33:14,584] jepsen worker 80 - jepsen.util 6830 :ok :write [26 46] INFO [2019-12-17 12:33:14,584] jepsen worker 80 - jepsen.util 6830 :invoke :cas [26 [17 10]] INFO [2019-12-17 12:33:14,594] jepsen worker 27 - jepsen.util 6867 :info :cas [9 [12 13]] :RequestCanceledException INFO [2019-12-17 12:33:14,595] jepsen worker 27 - jepsen.util 6957 :invoke :cas [9 [26 49]] INFO [2019-12-17 12:33:14,605] jepsen worker 9 - jepsen.util 7389 :ok :write [3 0] INFO [2019-12-17 12:33:14,606] jepsen worker 9 - jepsen.util 7389 :invoke :write [3 1] INFO [2019-12-17 12:33:14,606] jepsen worker 12 - jepsen.util 7212 :ok :write [4 42] INFO [2019-12-17 12:33:14,607] jepsen worker 12 - jepsen.util 7212 :invoke :write [4 44] INFO [2019-12-17 12:33:14,612] jepsen worker 46 - jepsen.util 6976 :info :cas [15 [3 1]] :RequestCanceledException INFO [2019-12-17 12:33:14,614] jepsen worker 46 - jepsen.util 7066 :invoke :read [15 nil] INFO [2019-12-17 12:33:14,619] jepsen worker 64 - jepsen.util 5374 :ok :write [21 10] INFO [2019-12-17 12:33:14,620] jepsen worker 64 - jepsen.util 5374 :invoke :cas [21 [14 42]] INFO [2019-12-17 12:33:14,624] jepsen worker 27 - jepsen.util 6957 :fail :cas [9 [26 49]] :ValueNotSwapFrom INFO [2019-12-17 12:33:14,625] jepsen worker 27 - jepsen.util 6957 :invoke :cas [9 [28 30]] INFO [2019-12-17 12:33:14,642] jepsen worker 38 - jepsen.util 6698 :info :write [12 30] :RequestCanceledException INFO [2019-12-17 12:33:14,643] jepsen worker 38 - jepsen.util 6788 :invoke :cas [12 [32 31]] INFO [2019-12-17 12:33:14,644] jepsen worker 64 - jepsen.util 5374 :fail :cas [21 [14 42]] :ValueNotSwapFrom INFO [2019-12-17 12:33:14,644] jepsen worker 64 - jepsen.util 5374 :invoke :write [21 4] INFO [2019-12-17 12:33:14,654] jepsen worker 10 - jepsen.util 6580 :info :write [3 18] :RequestCanceledException INFO [2019-12-17 12:33:14,655] jepsen worker 10 - jepsen.util 6670 :invoke :cas [3 [9 38]] INFO [2019-12-17 12:33:14,683] jepsen worker 52 - jepsen.util 7162 :fail :cas [17 [11 30]] :ValueNotSwapFrom INFO [2019-12-17 12:33:14,683] jepsen worker 52 - jepsen.util 7162 :invoke :cas [17 [37 36]] INFO [2019-12-17 12:33:14,690] jepsen worker 14 - jepsen.util 7034 :ok :read [4 42] INFO [2019-12-17 12:33:14,690] jepsen worker 14 - jepsen.util 7034 :invoke :cas [4 [19 13]] INFO [2019-12-17 12:33:14,699] jepsen worker 64 - jepsen.util 5374 :ok :write [21 4] INFO [2019-12-17 12:33:14,699] jepsen worker 64 - jepsen.util 5374 :invoke :cas [21 [20 5]] INFO [2019-12-17 12:33:14,702] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":39528,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:33:14,702] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":39528,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:33:14,702] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":39532,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:33:14,702] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":39532,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:33:14,710] jepsen worker 32 - jepsen.util 6872 :fail :read [10 nil] :RequestTimeoutException INFO [2019-12-17 12:33:14,710] jepsen worker 32 - jepsen.util 6872 :invoke :write [10 1] INFO [2019-12-17 12:33:14,729] jepsen worker 89 - jepsen.util 6569 :info :write [29 36] :RequestCanceledException INFO [2019-12-17 12:33:14,729] jepsen worker 72 - jepsen.util 6642 :fail :read [24 nil] :RequestTimeoutException INFO [2019-12-17 12:33:14,729] jepsen worker 72 - jepsen.util 6642 :invoke :read [24 nil] INFO [2019-12-17 12:33:14,730] jepsen worker 89 - jepsen.util 6659 :invoke :read [29 nil] INFO [2019-12-17 12:33:14,731] jepsen worker 64 - jepsen.util 5374 :fail :cas [21 [20 5]] :ValueNotSwapFrom INFO [2019-12-17 12:33:14,732] jepsen worker 64 - jepsen.util 5374 :invoke :read [21 nil] INFO [2019-12-17 12:33:14,739] jepsen worker 4 - jepsen.util 6214 :ok :read [1 14] INFO [2019-12-17 12:33:14,739] jepsen worker 4 - jepsen.util 6214 :invoke :read [1 nil] INFO [2019-12-17 12:33:14,752] jepsen worker 64 - jepsen.util 5374 :ok :read [21 4] INFO [2019-12-17 12:33:14,753] jepsen worker 64 - jepsen.util 5374 :invoke :write [21 39] INFO [2019-12-17 12:33:14,790] jepsen worker 20 - jepsen.util 6590 :ok :write [6 48] INFO [2019-12-17 12:33:14,790] jepsen worker 20 - jepsen.util 6590 :invoke :read [6 nil] INFO [2019-12-17 12:33:14,794] jepsen worker 34 - jepsen.util 8674 :info :write [11 14] :RequestCanceledException INFO [2019-12-17 12:33:14,796] jepsen worker 34 - jepsen.util 8764 :invoke :cas [11 [26 34]] INFO [2019-12-17 12:33:14,796] jepsen worker 30 - jepsen.util 6780 :ok :write [10 25] INFO [2019-12-17 12:33:14,796] jepsen worker 30 - jepsen.util 6780 :invoke :read [10 nil] INFO [2019-12-17 12:33:14,805] jepsen worker 14 - jepsen.util 7034 :fail :cas [4 [19 13]] :ValueNotSwapFrom INFO [2019-12-17 12:33:14,805] jepsen worker 14 - jepsen.util 7034 :invoke :read [4 nil] INFO [2019-12-17 12:33:14,810] jepsen worker 53 - jepsen.util 6443 :info :write [17 16] :RequestCanceledException INFO [2019-12-17 12:33:14,811] jepsen worker 53 - jepsen.util 6533 :invoke :cas [17 [6 35]] INFO [2019-12-17 12:33:14,818] jepsen worker 58 - jepsen.util 9868 :info :cas [19 [48 19]] :RequestCanceledException INFO [2019-12-17 12:33:14,820] jepsen worker 58 - jepsen.util 9958 :invoke :write [19 16] INFO [2019-12-17 12:33:14,826] jepsen worker 75 - jepsen.util 8805 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:33:14,826] jepsen worker 75 - jepsen.util 8805 :invoke :write [25 19] INFO [2019-12-17 12:33:14,831] jepsen worker 10 - jepsen.util 6670 :fail :cas [3 [9 38]] :ValueNotSwapFrom INFO [2019-12-17 12:33:14,831] jepsen worker 10 - jepsen.util 6670 :invoke :read [3 nil] INFO [2019-12-17 12:33:14,839] jepsen worker 21 - jepsen.util 8751 :info :cas [7 [13 38]] :RequestCanceledException INFO [2019-12-17 12:33:14,840] jepsen worker 21 - jepsen.util 8841 :invoke :read [7 nil] INFO [2019-12-17 12:33:14,841] jepsen worker 72 - jepsen.util 6642 :ok :read [24 15] INFO [2019-12-17 12:33:14,842] jepsen worker 72 - jepsen.util 6642 :invoke :read [24 nil] INFO [2019-12-17 12:33:14,842] jepsen worker 64 - jepsen.util 5374 :ok :write [21 39] INFO [2019-12-17 12:33:14,842] jepsen worker 64 - jepsen.util 5374 :invoke :cas [21 [48 13]] INFO [2019-12-17 12:33:14,851] jepsen worker 20 - jepsen.util 6590 :ok :read [6 48] INFO [2019-12-17 12:33:14,851] jepsen worker 20 - jepsen.util 6590 :invoke :cas [6 [36 0]] INFO [2019-12-17 12:33:14,859] jepsen worker 24 - jepsen.util 8214 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:33:14,859] jepsen worker 77 - jepsen.util 9617 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:33:14,859] jepsen worker 71 - jepsen.util 7541 :info :write [23 24] :RequestCanceledException INFO [2019-12-17 12:33:14,859] jepsen worker 43 - jepsen.util 8593 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:33:14,859] jepsen worker 42 - jepsen.util 9762 :info :write [14 11] :RequestCanceledException INFO [2019-12-17 12:33:14,859] jepsen worker 24 - jepsen.util 8214 :invoke :read [8 nil] INFO [2019-12-17 12:33:14,859] jepsen worker 77 - jepsen.util 9617 :invoke :read [25 nil] INFO [2019-12-17 12:33:14,860] jepsen worker 43 - jepsen.util 8593 :invoke :write [14 17] INFO [2019-12-17 12:33:14,860] jepsen worker 42 - jepsen.util 9852 :invoke :cas [14 [7 49]] INFO [2019-12-17 12:33:14,860] jepsen worker 71 - jepsen.util 7631 :invoke :write [23 12] INFO [2019-12-17 12:33:14,867] jepsen worker 64 - jepsen.util 5374 :fail :cas [21 [48 13]] :ValueNotSwapFrom INFO [2019-12-17 12:33:14,867] jepsen worker 64 - jepsen.util 5374 :invoke :read [21 nil] INFO [2019-12-17 12:33:14,871] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":39560,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:33:14,871] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":39560,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:33:14,871] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":39564,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:33:14,871] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":39564,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:33:14,887] jepsen worker 25 - jepsen.util 9655 :info :cas [8 [0 15]] :RequestCanceledException INFO [2019-12-17 12:33:14,887] jepsen worker 23 - jepsen.util 9293 :info :write [7 42] :RequestCanceledException INFO [2019-12-17 12:33:14,888] jepsen worker 25 - jepsen.util 9745 :invoke :cas [8 [37 27]] INFO [2019-12-17 12:33:14,888] jepsen worker 23 - jepsen.util 9383 :invoke :cas [7 [9 43]] INFO [2019-12-17 12:33:14,894] jepsen worker 10 - jepsen.util 6670 :ok :read [3 0] INFO [2019-12-17 12:33:14,894] jepsen worker 10 - jepsen.util 6670 :invoke :read [3 nil] INFO [2019-12-17 12:33:14,906] jepsen worker 64 - jepsen.util 5374 :ok :read [21 39] INFO [2019-12-17 12:33:14,906] jepsen worker 64 - jepsen.util 5374 :invoke :write [21 18] INFO [2019-12-17 12:33:14,908] jepsen worker 4 - jepsen.util 6214 :ok :read [1 14] INFO [2019-12-17 12:33:14,908] jepsen worker 4 - jepsen.util 6214 :invoke :write [1 35] INFO [2019-12-17 12:33:14,912] jepsen worker 33 - jepsen.util 9033 :info :write [11 38] :RequestCanceledException INFO [2019-12-17 12:33:14,912] jepsen worker 22 - jepsen.util 9202 :info :cas [7 [25 12]] :RequestCanceledException INFO [2019-12-17 12:33:14,912] jepsen worker 11 - jepsen.util 7211 :fail :read [3 nil] :RequestTimeoutException INFO [2019-12-17 12:33:14,912] jepsen worker 11 - jepsen.util 7211 :invoke :read [3 nil] INFO [2019-12-17 12:33:14,913] jepsen worker 22 - jepsen.util 9292 :invoke :read [7 nil] INFO [2019-12-17 12:33:14,914] jepsen worker 33 - jepsen.util 9123 :invoke :write [11 27] INFO [2019-12-17 12:33:14,917] jepsen worker 72 - jepsen.util 6642 :ok :read [24 15] INFO [2019-12-17 12:33:14,917] jepsen worker 72 - jepsen.util 6642 :invoke :read [24 nil] INFO [2019-12-17 12:33:14,927] jepsen worker 59 - jepsen.util 9239 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:33:14,927] jepsen worker 67 - jepsen.util 9427 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:33:14,927] jepsen worker 0 - jepsen.util 9630 :info :write [0 25] :RequestCanceledException INFO [2019-12-17 12:33:14,927] jepsen worker 26 - jepsen.util 9476 :info :write [8 45] :RequestCanceledException INFO [2019-12-17 12:33:14,927] jepsen worker 59 - jepsen.util 9239 :invoke :read [19 nil] INFO [2019-12-17 12:33:14,927] jepsen worker 67 - jepsen.util 9427 :invoke :write [22 19] INFO [2019-12-17 12:33:14,928] jepsen worker 60 - jepsen.util 10140 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:33:14,928] jepsen worker 62 - jepsen.util 9242 :info :write [20 41] :RequestCanceledException INFO [2019-12-17 12:33:14,928] jepsen worker 60 - jepsen.util 10140 :invoke :cas [20 [8 39]] INFO [2019-12-17 12:33:14,928] jepsen worker 26 - jepsen.util 9566 :invoke :read [8 nil] INFO [2019-12-17 12:33:14,928] jepsen worker 0 - jepsen.util 9720 :invoke :read [0 nil] INFO [2019-12-17 12:33:14,929] jepsen worker 62 - jepsen.util 9332 :invoke :write [20 3] INFO [2019-12-17 12:33:14,935] jepsen worker 20 - jepsen.util 6590 :fail :cas [6 [36 0]] :ValueNotSwapFrom INFO [2019-12-17 12:33:14,935] jepsen worker 20 - jepsen.util 6590 :invoke :write [6 8] INFO [2019-12-17 12:33:14,941] jepsen worker 68 - jepsen.util 8708 :info :write [22 31] :RequestCanceledException INFO [2019-12-17 12:33:14,941] jepsen worker 15 - jepsen.util 8205 :info :cas [5 [48 3]] :RequestCanceledException INFO [2019-12-17 12:33:14,941] jepsen worker 66 - jepsen.util 8976 :info :cas [22 [44 44]] :RequestCanceledException INFO [2019-12-17 12:33:14,941] jepsen worker 41 - jepsen.util 9761 :info :write [13 40] :RequestCanceledException INFO [2019-12-17 12:33:14,941] jepsen worker 83 - jepsen.util 9713 :info :cas [27 [11 25]] :RequestCanceledException INFO [2019-12-17 12:33:14,942] jepsen worker 15 - jepsen.util 8295 :invoke :write [5 31] INFO [2019-12-17 12:33:14,942] jepsen worker 68 - jepsen.util 8798 :invoke :write [22 3] INFO [2019-12-17 12:33:14,942] jepsen worker 66 - jepsen.util 9066 :invoke :write [22 19] INFO [2019-12-17 12:33:14,943] jepsen worker 41 - jepsen.util 9851 :invoke :cas [13 [27 24]] INFO [2019-12-17 12:33:14,943] jepsen worker 83 - jepsen.util 9803 :invoke :read [27 nil] INFO [2019-12-17 12:33:14,951] jepsen worker 10 - jepsen.util 6670 :ok :read [3 0] INFO [2019-12-17 12:33:14,951] jepsen worker 10 - jepsen.util 6670 :invoke :write [3 36] INFO [2019-12-17 12:33:14,959] jepsen worker 86 - jepsen.util 9806 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:33:14,959] jepsen worker 82 - jepsen.util 9892 :info :cas [27 [34 40]] :RequestCanceledException INFO [2019-12-17 12:33:14,960] jepsen worker 86 - jepsen.util 9806 :invoke :read [28 nil] INFO [2019-12-17 12:33:14,961] jepsen worker 82 - jepsen.util 9982 :invoke :read [27 nil] INFO [2019-12-17 12:33:14,973] jepsen worker 72 - jepsen.util 6642 :ok :read [24 15] INFO [2019-12-17 12:33:14,973] jepsen worker 72 - jepsen.util 6642 :invoke :read [24 nil] INFO [2019-12-17 12:33:14,978] jepsen worker 13 - jepsen.util 6493 :fail :cas [4 [17 29]] :ValueNotSwapFrom INFO [2019-12-17 12:33:14,978] jepsen worker 13 - jepsen.util 6493 :invoke :cas [4 [36 10]] INFO [2019-12-17 12:33:14,994] jepsen worker 50 - jepsen.util 9680 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:33:14,994] jepsen worker 50 - jepsen.util 9680 :invoke :cas [16 [31 46]] INFO [2019-12-17 12:33:15,005] jepsen worker 51 - jepsen.util 6621 :fail :cas [17 [15 48]] :ValueNotSwapFrom INFO [2019-12-17 12:33:15,005] jepsen worker 51 - jepsen.util 6621 :invoke :read [17 nil] INFO [2019-12-17 12:33:15,018] jepsen worker 28 - jepsen.util 5338 :ok :write [9 3] INFO [2019-12-17 12:33:15,019] jepsen worker 28 - jepsen.util 5338 :invoke :write [9 17] INFO [2019-12-17 12:33:15,025] jepsen worker 1 - jepsen.util 9271 :info :cas [0 [14 45]] :RequestCanceledException INFO [2019-12-17 12:33:15,026] jepsen worker 1 - jepsen.util 9361 :invoke :cas [0 [15 25]] INFO [2019-12-17 12:33:15,040] jepsen worker 69 - jepsen.util 6819 :info :write [23 46] :RequestCanceledException INFO [2019-12-17 12:33:15,041] jepsen worker 69 - jepsen.util 6909 :invoke :write [23 38] INFO [2019-12-17 12:33:15,061] jepsen worker 3 - jepsen.util 6753 :fail :cas [1 [5 44]] :ValueNotSwapFrom INFO [2019-12-17 12:33:15,061] jepsen worker 3 - jepsen.util 6753 :invoke :write [1 26] INFO [2019-12-17 12:33:15,066] jepsen worker 64 - jepsen.util 5374 :ok :write [21 18] INFO [2019-12-17 12:33:15,067] jepsen worker 64 - jepsen.util 5374 :invoke :write [21 24] INFO [2019-12-17 12:33:15,084] jepsen worker 89 - jepsen.util 6659 :ok :read [29 37] INFO [2019-12-17 12:33:15,084] jepsen worker 89 - jepsen.util 6659 :invoke :write [29 28] INFO [2019-12-17 12:33:15,090] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":39596,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:33:15,090] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":39596,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:33:15,090] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":39599,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:33:15,090] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":39599,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:33:15,090] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":39600,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:33:15,090] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":39600,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:33:15,109] jepsen worker 36 - jepsen.util 7326 :ok :read [12 25] INFO [2019-12-17 12:33:15,109] jepsen worker 36 - jepsen.util 7326 :invoke :read [12 nil] INFO [2019-12-17 12:33:15,115] jepsen worker 28 - jepsen.util 5338 :ok :write [9 17] INFO [2019-12-17 12:33:15,116] jepsen worker 28 - jepsen.util 5338 :invoke :write [9 29] INFO [2019-12-17 12:33:15,135] jepsen worker 52 - jepsen.util 7162 :fail :cas [17 [37 36]] :ValueNotSwapFrom INFO [2019-12-17 12:33:15,135] jepsen worker 52 - jepsen.util 7162 :invoke :write [17 43] INFO [2019-12-17 12:33:15,142] jepsen worker 38 - jepsen.util 6788 :fail :cas [12 [32 31]] :ValueNotSwapFrom INFO [2019-12-17 12:33:15,143] jepsen worker 38 - jepsen.util 6788 :invoke :write [12 39] INFO [2019-12-17 12:33:15,160] jepsen worker 64 - jepsen.util 5374 :ok :write [21 24] INFO [2019-12-17 12:33:15,160] jepsen worker 64 - jepsen.util 5374 :invoke :read [21 nil] INFO [2019-12-17 12:33:15,181] jepsen worker 64 - jepsen.util 5374 :ok :read [21 24] INFO [2019-12-17 12:33:15,181] jepsen worker 64 - jepsen.util 5374 :invoke :read [21 nil] INFO [2019-12-17 12:33:15,192] jepsen worker 45 - jepsen.util 6795 :ok :write [15 18] INFO [2019-12-17 12:33:15,192] jepsen worker 45 - jepsen.util 6795 :invoke :cas [15 [35 30]] INFO [2019-12-17 12:33:15,196] jepsen worker 53 - jepsen.util 6533 :fail :cas [17 [6 35]] :ValueNotSwapFrom INFO [2019-12-17 12:33:15,196] jepsen worker 53 - jepsen.util 6533 :invoke :read [17 nil] INFO [2019-12-17 12:33:15,200] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":39615,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:33:15,200] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":39615,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:33:15,200] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":39616,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:33:15,200] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":39616,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:33:15,203] jepsen worker 28 - jepsen.util 5338 :ok :write [9 29] INFO [2019-12-17 12:33:15,203] jepsen worker 28 - jepsen.util 5338 :invoke :write [9 8] INFO [2019-12-17 12:33:15,206] jepsen worker 20 - jepsen.util 6590 :ok :write [6 8] INFO [2019-12-17 12:33:15,206] jepsen worker 20 - jepsen.util 6590 :invoke :write [6 15] INFO [2019-12-17 12:33:15,213] jepsen worker 64 - jepsen.util 5374 :ok :read [21 24] INFO [2019-12-17 12:33:15,213] jepsen worker 9 - jepsen.util 7389 :ok :write [3 1] INFO [2019-12-17 12:33:15,214] jepsen worker 64 - jepsen.util 5374 :invoke :write [21 30] INFO [2019-12-17 12:33:15,214] jepsen worker 9 - jepsen.util 7389 :invoke :write [3 4] INFO [2019-12-17 12:33:15,242] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":39627,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:33:15,243] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":39627,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:33:15,243] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":39629,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:33:15,243] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":39629,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:33:15,309] jepsen worker 76 - jepsen.util 9616 :info :write [25 19] :RequestCanceledException INFO [2019-12-17 12:33:15,310] jepsen worker 76 - jepsen.util 9706 :invoke :read [25 nil] INFO [2019-12-17 12:33:15,317] jepsen worker 64 - jepsen.util 5374 :ok :write [21 30] INFO [2019-12-17 12:33:15,317] jepsen worker 64 - jepsen.util 5374 :invoke :write [21 2] INFO [2019-12-17 12:33:15,318] jepsen worker 13 - jepsen.util 6493 :fail :cas [4 [36 10]] :ValueNotSwapFrom INFO [2019-12-17 12:33:15,318] jepsen worker 13 - jepsen.util 6493 :invoke :read [4 nil] INFO [2019-12-17 12:33:15,328] jepsen worker 84 - jepsen.util 8724 :info :write [28 17] :RequestCanceledException INFO [2019-12-17 12:33:15,328] jepsen worker 6 - jepsen.util 9906 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:33:15,328] jepsen worker 88 - jepsen.util 6748 :info :write [29 33] :RequestCanceledException INFO [2019-12-17 12:33:15,328] jepsen worker 6 - jepsen.util 9906 :invoke :write [2 37] INFO [2019-12-17 12:33:15,329] jepsen worker 84 - jepsen.util 8814 :invoke :cas [28 [12 29]] INFO [2019-12-17 12:33:15,329] jepsen worker 88 - jepsen.util 6838 :invoke :write [29 29] INFO [2019-12-17 12:33:15,342] jepsen worker 39 - jepsen.util 9219 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:33:15,342] jepsen worker 39 - jepsen.util 9219 :invoke :cas [13 [45 3]] INFO [2019-12-17 12:33:15,347] jepsen worker 14 - jepsen.util 7034 :ok :read [4 42] INFO [2019-12-17 12:33:15,347] jepsen worker 14 - jepsen.util 7034 :invoke :write [4 12] INFO [2019-12-17 12:33:15,352] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:33:15,352] jepsen nemesis - couchbase.util Rebalance check count: 303 INFO [2019-12-17 12:33:15,352] jepsen nemesis - couchbase.util Rebalance stuck count: 284 INFO [2019-12-17 12:33:15,355] jepsen worker 28 - jepsen.util 5338 :ok :write [9 8] INFO [2019-12-17 12:33:15,355] jepsen worker 28 - jepsen.util 5338 :invoke :write [9 37] INFO [2019-12-17 12:33:15,358] jepsen worker 40 - jepsen.util 9850 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:33:15,358] jepsen worker 17 - jepsen.util 9647 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:33:15,358] jepsen worker 40 - jepsen.util 9850 :invoke :write [13 12] INFO [2019-12-17 12:33:15,358] jepsen worker 17 - jepsen.util 9647 :invoke :cas [5 [36 33]] INFO [2019-12-17 12:33:15,391] jepsen worker 64 - jepsen.util 5374 :ok :write [21 2] INFO [2019-12-17 12:33:15,391] jepsen worker 64 - jepsen.util 5374 :invoke :cas [21 [39 11]] INFO [2019-12-17 12:33:15,404] jepsen worker 51 - jepsen.util 6621 :ok :read [17 33] INFO [2019-12-17 12:33:15,404] jepsen worker 81 - jepsen.util 8991 :info :cas [27 [19 21]] :RequestCanceledException INFO [2019-12-17 12:33:15,404] jepsen worker 51 - jepsen.util 6621 :invoke :write [17 14] INFO [2019-12-17 12:33:15,405] jepsen worker 81 - jepsen.util 9081 :invoke :write [27 12] INFO [2019-12-17 12:33:15,411] jepsen worker 64 - jepsen.util 5374 :fail :cas [21 [39 11]] :ValueNotSwapFrom INFO [2019-12-17 12:33:15,411] jepsen worker 64 - jepsen.util 5374 :invoke :read [21 nil] INFO [2019-12-17 12:33:15,422] jepsen worker 28 - jepsen.util 5338 :ok :write [9 37] INFO [2019-12-17 12:33:15,422] jepsen worker 28 - jepsen.util 5338 :invoke :write [9 10] INFO [2019-12-17 12:33:15,424] jepsen worker 64 - jepsen.util 5374 :ok :read [21 2] INFO [2019-12-17 12:33:15,424] jepsen worker 64 - jepsen.util 5374 :invoke :read [21 nil] INFO [2019-12-17 12:33:15,428] jepsen worker 56 - jepsen.util 7526 :info :write [18 13] :RequestCanceledException INFO [2019-12-17 12:33:15,429] jepsen worker 56 - jepsen.util 7616 :invoke :write [18 35] INFO [2019-12-17 12:33:15,442] jepsen worker 44 - jepsen.util 10124 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:33:15,442] jepsen worker 44 - jepsen.util 10124 :invoke :read [14 nil] INFO [2019-12-17 12:33:15,443] jepsen worker 64 - jepsen.util 5374 :ok :read [21 2] INFO [2019-12-17 12:33:15,444] jepsen worker 64 - jepsen.util 5374 :invoke :read [21 nil] INFO [2019-12-17 12:33:15,459] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":39650,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:33:15,459] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":39650,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:33:15,459] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":39653,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:33:15,459] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":39653,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:33:15,460] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":39654,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:33:15,460] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":39654,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:33:15,472] jepsen worker 64 - jepsen.util 5374 :ok :read [21 2] INFO [2019-12-17 12:33:15,472] jepsen worker 64 - jepsen.util 5374 :invoke :read [21 nil] INFO [2019-12-17 12:33:15,486] jepsen worker 28 - jepsen.util 5338 :ok :write [9 10] INFO [2019-12-17 12:33:15,486] jepsen worker 28 - jepsen.util 5338 :invoke :cas [9 [2 35]] INFO [2019-12-17 12:33:15,489] jepsen worker 64 - jepsen.util 5374 :ok :read [21 2] INFO [2019-12-17 12:33:15,489] jepsen worker 64 - jepsen.util 5374 :invoke :write [21 1] INFO [2019-12-17 12:33:15,501] jepsen worker 36 - jepsen.util 7326 :ok :read [12 25] INFO [2019-12-17 12:33:15,501] jepsen worker 36 - jepsen.util 7326 :invoke :cas [12 [2 23]] INFO [2019-12-17 12:33:15,512] jepsen worker 28 - jepsen.util 5338 :fail :cas [9 [2 35]] :ValueNotSwapFrom INFO [2019-12-17 12:33:15,512] jepsen worker 28 - jepsen.util 5338 :invoke :cas [9 [49 42]] INFO [2019-12-17 12:33:15,529] jepsen worker 70 - jepsen.util 7090 :info :write [23 20] :RequestCanceledException INFO [2019-12-17 12:33:15,529] jepsen worker 48 - jepsen.util 8598 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:33:15,530] jepsen worker 48 - jepsen.util 8598 :invoke :read [16 nil] INFO [2019-12-17 12:33:15,531] jepsen worker 70 - jepsen.util 7180 :invoke :read [23 nil] INFO [2019-12-17 12:33:15,542] jepsen worker 78 - jepsen.util 7008 :info :write [26 45] :RequestCanceledException INFO [2019-12-17 12:33:15,542] jepsen worker 57 - jepsen.util 9057 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:33:15,542] jepsen worker 57 - jepsen.util 9057 :invoke :cas [19 [36 48]] INFO [2019-12-17 12:33:15,545] jepsen worker 78 - jepsen.util 7098 :invoke :read [26 nil] INFO [2019-12-17 12:33:15,552] jepsen worker 53 - jepsen.util 6533 :ok :read [17 33] INFO [2019-12-17 12:33:15,552] jepsen worker 28 - jepsen.util 5338 :fail :cas [9 [49 42]] :ValueNotSwapFrom INFO [2019-12-17 12:33:15,552] jepsen worker 53 - jepsen.util 6533 :invoke :cas [17 [2 26]] INFO [2019-12-17 12:33:15,552] jepsen worker 28 - jepsen.util 5338 :invoke :read [9 nil] INFO [2019-12-17 12:33:15,554] jepsen worker 5 - jepsen.util 6395 :info :write [1 32] :RequestCanceledException INFO [2019-12-17 12:33:15,555] jepsen worker 5 - jepsen.util 6485 :invoke :write [1 27] INFO [2019-12-17 12:33:15,565] jepsen worker 64 - jepsen.util 5374 :ok :write [21 1] INFO [2019-12-17 12:33:15,565] jepsen worker 64 - jepsen.util 5374 :invoke :cas [21 [0 28]] INFO [2019-12-17 12:33:15,571] jepsen worker 28 - jepsen.util 5338 :ok :read [9 10] INFO [2019-12-17 12:33:15,571] jepsen worker 28 - jepsen.util 5338 :invoke :read [9 nil] INFO [2019-12-17 12:33:15,591] jepsen worker 47 - jepsen.util 7337 :fail :read [15 nil] :RequestTimeoutException INFO [2019-12-17 12:33:15,591] jepsen worker 61 - jepsen.util 10231 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:33:15,591] jepsen worker 65 - jepsen.util 5555 :info :cas [21 [7 32]] :RequestCanceledException INFO [2019-12-17 12:33:15,591] jepsen worker 47 - jepsen.util 7337 :invoke :read [15 nil] INFO [2019-12-17 12:33:15,591] jepsen worker 61 - jepsen.util 10231 :invoke :write [20 33] INFO [2019-12-17 12:33:15,592] jepsen worker 64 - jepsen.util 5374 :fail :cas [21 [0 28]] :ValueNotSwapFrom INFO [2019-12-17 12:33:15,592] jepsen worker 64 - jepsen.util 5374 :invoke :read [21 nil] INFO [2019-12-17 12:33:15,593] jepsen worker 65 - jepsen.util 5645 :invoke :cas [21 [31 34]] INFO [2019-12-17 12:33:15,612] jepsen worker 7 - jepsen.util 9637 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:33:15,612] jepsen worker 7 - jepsen.util 9637 :invoke :read [2 nil] INFO [2019-12-17 12:33:15,616] jepsen worker 28 - jepsen.util 5338 :ok :read [9 10] INFO [2019-12-17 12:33:15,616] jepsen worker 28 - jepsen.util 5338 :invoke :cas [9 [1 44]] INFO [2019-12-17 12:33:15,628] jepsen worker 8 - jepsen.util 9098 :info :write [2 14] :RequestCanceledException INFO [2019-12-17 12:33:15,631] jepsen worker 8 - jepsen.util 9188 :invoke :write [2 37] INFO [2019-12-17 12:33:15,642] jepsen worker 49 - jepsen.util 8509 :info :cas [16 [15 23]] :RequestCanceledException INFO [2019-12-17 12:33:15,642] jepsen worker 85 - jepsen.util 10255 :info :write [28 4] :RequestCanceledException INFO [2019-12-17 12:33:15,649] jepsen worker 49 - jepsen.util 8599 :invoke :cas [16 [31 37]] INFO [2019-12-17 12:33:15,650] jepsen worker 85 - jepsen.util 10345 :invoke :write [28 46] INFO [2019-12-17 12:33:15,656] jepsen worker 31 - jepsen.util 6061 :info :write [10 4] :RequestCanceledException INFO [2019-12-17 12:33:15,664] jepsen worker 31 - jepsen.util 6151 :invoke :cas [10 [37 14]] INFO [2019-12-17 12:33:15,669] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":39680,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:33:15,670] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":39680,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:33:15,672] jepsen worker 64 - jepsen.util 5374 :ok :read [21 1] INFO [2019-12-17 12:33:15,672] jepsen worker 64 - jepsen.util 5374 :invoke :write [21 26] INFO [2019-12-17 12:33:15,684] jepsen worker 65 - jepsen.util 5645 :fail :cas [21 [31 34]] :ValueNotSwapFrom INFO [2019-12-17 12:33:15,684] jepsen worker 65 - jepsen.util 5645 :invoke :cas [21 [29 17]] INFO [2019-12-17 12:33:15,686] jepsen worker 47 - jepsen.util 7337 :ok :read [15 18] INFO [2019-12-17 12:33:15,686] jepsen worker 47 - jepsen.util 7337 :invoke :cas [15 [19 3]] INFO [2019-12-17 12:33:15,692] jepsen worker 16 - jepsen.util 10366 :info :cas [5 [9 30]] :RequestCanceledException INFO [2019-12-17 12:33:15,697] jepsen worker 16 - jepsen.util 10456 :invoke :cas [5 [1 11]] INFO [2019-12-17 12:33:15,701] jepsen worker 28 - jepsen.util 5338 :fail :cas [9 [1 44]] :ValueNotSwapFrom INFO [2019-12-17 12:33:15,701] jepsen worker 28 - jepsen.util 5338 :invoke :read [9 nil] INFO [2019-12-17 12:33:15,743] jepsen worker 65 - jepsen.util 5645 :fail :cas [21 [29 17]] :ValueNotSwapFrom INFO [2019-12-17 12:33:15,743] jepsen worker 65 - jepsen.util 5645 :invoke :read [21 nil] INFO [2019-12-17 12:33:15,760] jepsen worker 47 - jepsen.util 7337 :fail :cas [15 [19 3]] :ValueNotSwapFrom INFO [2019-12-17 12:33:15,760] jepsen worker 47 - jepsen.util 7337 :invoke :cas [15 [14 8]] INFO [2019-12-17 12:33:15,773] jepsen worker 65 - jepsen.util 5645 :ok :read [21 1] INFO [2019-12-17 12:33:15,774] jepsen worker 65 - jepsen.util 5645 :invoke :cas [21 [8 2]] INFO [2019-12-17 12:33:15,793] jepsen worker 78 - jepsen.util 7098 :ok :read [26 46] INFO [2019-12-17 12:33:15,793] jepsen worker 78 - jepsen.util 7098 :invoke :cas [26 [45 4]] INFO [2019-12-17 12:33:15,801] jepsen worker 64 - jepsen.util 5374 :ok :write [21 26] INFO [2019-12-17 12:33:15,802] jepsen worker 64 - jepsen.util 5374 :invoke :write [21 2] INFO [2019-12-17 12:33:15,914] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":39698,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:33:15,914] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":39698,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:33:15,914] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":39701,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:33:15,914] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":39701,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:33:15,915] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":39702,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:33:15,915] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":39702,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:33:15,915] jepsen worker 28 - jepsen.util 5338 :ok :read [9 10] INFO [2019-12-17 12:33:15,915] jepsen worker 35 - jepsen.util 9035 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:33:15,915] jepsen worker 28 - jepsen.util 5338 :invoke :write [9 24] INFO [2019-12-17 12:33:15,915] jepsen worker 35 - jepsen.util 9035 :invoke :cas [11 [48 15]] INFO [2019-12-17 12:33:15,919] jepsen worker 87 - jepsen.util 6027 :info :write [29 10] :RequestCanceledException INFO [2019-12-17 12:33:15,921] jepsen worker 47 - jepsen.util 7337 :fail :cas [15 [14 8]] :ValueNotSwapFrom INFO [2019-12-17 12:33:15,922] jepsen worker 47 - jepsen.util 7337 :invoke :read [15 nil] INFO [2019-12-17 12:33:15,922] jepsen worker 87 - jepsen.util 6117 :invoke :read [29 nil] INFO [2019-12-17 12:33:15,925] jepsen worker 65 - jepsen.util 5645 :fail :cas [21 [8 2]] :ValueNotSwapFrom INFO [2019-12-17 12:33:15,925] jepsen worker 65 - jepsen.util 5645 :invoke :write [21 2] INFO [2019-12-17 12:33:15,925] jepsen worker 13 - jepsen.util 6493 :ok :read [4 42] INFO [2019-12-17 12:33:15,925] jepsen worker 13 - jepsen.util 6493 :invoke :read [4 nil] INFO [2019-12-17 12:33:15,926] jepsen worker 37 - jepsen.util 6517 :fail :cas [12 [23 15]] :ValueNotSwapFrom INFO [2019-12-17 12:33:15,926] jepsen worker 37 - jepsen.util 6517 :invoke :write [12 11] INFO [2019-12-17 12:33:15,931] jepsen worker 80 - jepsen.util 6830 :fail :cas [26 [17 10]] :ValueNotSwapFrom INFO [2019-12-17 12:33:15,931] jepsen worker 80 - jepsen.util 6830 :invoke :write [26 29] INFO [2019-12-17 12:33:15,954] jepsen worker 19 - jepsen.util 6769 :info :cas [6 [36 34]] :RequestCanceledException INFO [2019-12-17 12:33:15,955] jepsen worker 19 - jepsen.util 6859 :invoke :read [6 nil] INFO [2019-12-17 12:33:16,000] jepsen worker 47 - jepsen.util 7337 :ok :read [15 18] INFO [2019-12-17 12:33:16,001] jepsen worker 47 - jepsen.util 7337 :invoke :read [15 nil] INFO [2019-12-17 12:33:16,013] jepsen worker 64 - jepsen.util 5374 :ok :write [21 2] INFO [2019-12-17 12:33:16,013] jepsen worker 64 - jepsen.util 5374 :invoke :write [21 11] INFO [2019-12-17 12:33:16,024] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":39713,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:33:16,024] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":39713,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:33:16,024] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":39714,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:33:16,024] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":39714,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:33:16,028] jepsen worker 63 - jepsen.util 5823 :info :write [21 6] :RequestCanceledException INFO [2019-12-17 12:33:16,030] jepsen worker 63 - jepsen.util 5913 :invoke :cas [21 [11 24]] INFO [2019-12-17 12:33:16,048] jepsen worker 19 - jepsen.util 6859 :ok :read [6 8] INFO [2019-12-17 12:33:16,048] jepsen worker 19 - jepsen.util 6859 :invoke :write [6 17] INFO [2019-12-17 12:33:16,056] jepsen worker 2 - jepsen.util 8912 :info :write [0 7] :RequestCanceledException INFO [2019-12-17 12:33:16,057] jepsen worker 2 - jepsen.util 9002 :invoke :write [0 35] INFO [2019-12-17 12:33:16,059] jepsen worker 28 - jepsen.util 5338 :ok :write [9 24] INFO [2019-12-17 12:33:16,059] jepsen worker 28 - jepsen.util 5338 :invoke :read [9 nil] INFO [2019-12-17 12:33:16,082] jepsen worker 47 - jepsen.util 7337 :ok :read [15 18] INFO [2019-12-17 12:33:16,082] jepsen worker 47 - jepsen.util 7337 :invoke :cas [15 [33 38]] INFO [2019-12-17 12:33:16,089] jepsen worker 28 - jepsen.util 5338 :ok :read [9 24] INFO [2019-12-17 12:33:16,089] jepsen worker 28 - jepsen.util 5338 :invoke :cas [9 [39 31]] INFO [2019-12-17 12:33:16,094] jepsen worker 73 - jepsen.util 6463 :info :cas [24 [18 18]] :RequestCanceledException INFO [2019-12-17 12:33:16,095] jepsen worker 73 - jepsen.util 6553 :invoke :read [24 nil] INFO [2019-12-17 12:33:16,098] jepsen worker 47 - jepsen.util 7337 :fail :cas [15 [33 38]] :ValueNotSwapFrom INFO [2019-12-17 12:33:16,098] jepsen worker 47 - jepsen.util 7337 :invoke :read [15 nil] INFO [2019-12-17 12:33:16,101] jepsen worker 64 - jepsen.util 5374 :ok :write [21 11] INFO [2019-12-17 12:33:16,101] jepsen worker 64 - jepsen.util 5374 :invoke :read [21 nil] INFO [2019-12-17 12:33:16,113] jepsen worker 32 - jepsen.util 6872 :ok :write [10 1] INFO [2019-12-17 12:33:16,113] jepsen worker 9 - jepsen.util 7389 :ok :write [3 4] INFO [2019-12-17 12:33:16,113] jepsen worker 32 - jepsen.util 6872 :invoke :cas [10 [47 28]] INFO [2019-12-17 12:33:16,113] jepsen worker 9 - jepsen.util 7389 :invoke :read [3 nil] INFO [2019-12-17 12:33:16,114] jepsen worker 20 - jepsen.util 6590 :ok :write [6 15] INFO [2019-12-17 12:33:16,115] jepsen worker 20 - jepsen.util 6590 :invoke :cas [6 [17 24]] INFO [2019-12-17 12:33:16,125] jepsen worker 28 - jepsen.util 5338 :fail :cas [9 [39 31]] :ValueNotSwapFrom INFO [2019-12-17 12:33:16,125] jepsen worker 28 - jepsen.util 5338 :invoke :write [9 45] INFO [2019-12-17 12:33:16,167] jepsen worker 36 - jepsen.util 7326 :fail :cas [12 [2 23]] :ValueNotSwapFrom INFO [2019-12-17 12:33:16,167] jepsen worker 36 - jepsen.util 7326 :invoke :cas [12 [13 42]] INFO [2019-12-17 12:33:16,211] jepsen worker 45 - jepsen.util 6795 :fail :cas [15 [35 30]] :ValueNotSwapFrom INFO [2019-12-17 12:33:16,211] jepsen worker 45 - jepsen.util 6795 :invoke :write [15 6] INFO [2019-12-17 12:33:16,213] jepsen worker 29 - jepsen.util 5069 :info :write [9 38] :RequestCanceledException INFO [2019-12-17 12:33:16,214] jepsen worker 29 - jepsen.util 5159 :invoke :write [9 39] INFO [2019-12-17 12:33:16,243] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":39735,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:33:16,243] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":39743,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:33:16,243] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":39735,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:33:16,243] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":39743,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:33:16,243] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":39737,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:33:16,243] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":39747,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:33:16,243] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":39737,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:33:16,243] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":39747,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:33:16,255] jepsen worker 20 - jepsen.util 6590 :fail :cas [6 [17 24]] :ValueNotSwapFrom INFO [2019-12-17 12:33:16,255] jepsen worker 20 - jepsen.util 6590 :invoke :cas [6 [14 9]] INFO [2019-12-17 12:33:16,258] jepsen worker 47 - jepsen.util 7337 :ok :read [15 18] INFO [2019-12-17 12:33:16,258] jepsen worker 47 - jepsen.util 7337 :invoke :write [15 31] INFO [2019-12-17 12:33:16,274] jepsen worker 31 - jepsen.util 6151 :fail :cas [10 [37 14]] :ValueNotSwapFrom INFO [2019-12-17 12:33:16,274] jepsen worker 31 - jepsen.util 6151 :invoke :cas [10 [21 11]] INFO [2019-12-17 12:33:16,278] jepsen worker 9 - jepsen.util 7389 :ok :read [3 4] INFO [2019-12-17 12:33:16,279] jepsen worker 9 - jepsen.util 7389 :invoke :cas [3 [8 1]] INFO [2019-12-17 12:33:16,286] jepsen worker 12 - jepsen.util 7212 :ok :write [4 44] INFO [2019-12-17 12:33:16,286] jepsen worker 12 - jepsen.util 7212 :invoke :write [4 40] INFO [2019-12-17 12:33:16,299] jepsen worker 73 - jepsen.util 6553 :ok :read [24 15] INFO [2019-12-17 12:33:16,299] jepsen worker 73 - jepsen.util 6553 :invoke :read [24 nil] INFO [2019-12-17 12:33:16,320] jepsen worker 32 - jepsen.util 6872 :fail :cas [10 [47 28]] :ValueNotSwapFrom INFO [2019-12-17 12:33:16,320] jepsen worker 32 - jepsen.util 6872 :invoke :cas [10 [40 20]] INFO [2019-12-17 12:33:16,326] jepsen worker 70 - jepsen.util 7180 :ok :read [23 46] INFO [2019-12-17 12:33:16,326] jepsen worker 70 - jepsen.util 7180 :invoke :cas [23 [15 49]] INFO [2019-12-17 12:33:16,327] jepsen worker 78 - jepsen.util 7098 :fail :cas [26 [45 4]] :ValueNotSwapFrom INFO [2019-12-17 12:33:16,328] jepsen worker 78 - jepsen.util 7098 :invoke :write [26 1] INFO [2019-12-17 12:33:16,350] jepsen worker 28 - jepsen.util 5338 :ok :write [9 45] INFO [2019-12-17 12:33:16,350] jepsen worker 28 - jepsen.util 5338 :invoke :cas [9 [42 35]] INFO [2019-12-17 12:33:16,363] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:33:16,364] jepsen nemesis - couchbase.util Rebalance check count: 304 INFO [2019-12-17 12:33:16,364] jepsen nemesis - couchbase.util Rebalance stuck count: 285 INFO [2019-12-17 12:33:16,366] jepsen worker 63 - jepsen.util 5913 :ok :cas [21 [11 24]] INFO [2019-12-17 12:33:16,366] jepsen worker 63 - jepsen.util 5913 :invoke :cas [21 [34 10]] INFO [2019-12-17 12:33:16,369] jepsen worker 13 - jepsen.util 6493 :ok :read [4 44] INFO [2019-12-17 12:33:16,369] jepsen worker 13 - jepsen.util 6493 :invoke :write [4 44] INFO [2019-12-17 12:33:16,398] jepsen worker 9 - jepsen.util 7389 :fail :cas [3 [8 1]] :ValueNotSwapFrom INFO [2019-12-17 12:33:16,398] jepsen worker 9 - jepsen.util 7389 :invoke :cas [3 [15 39]] INFO [2019-12-17 12:33:16,414] jepsen worker 19 - jepsen.util 6859 :ok :write [6 17] INFO [2019-12-17 12:33:16,414] jepsen worker 19 - jepsen.util 6859 :invoke :write [6 3] INFO [2019-12-17 12:33:16,417] jepsen worker 63 - jepsen.util 5913 :fail :cas [21 [34 10]] :ValueNotSwapFrom INFO [2019-12-17 12:33:16,417] jepsen worker 63 - jepsen.util 5913 :invoke :read [21 nil] INFO [2019-12-17 12:33:16,418] jepsen worker 53 - jepsen.util 6533 :fail :cas [17 [2 26]] :ValueNotSwapFrom INFO [2019-12-17 12:33:16,418] jepsen worker 53 - jepsen.util 6533 :invoke :read [17 nil] INFO [2019-12-17 12:33:16,428] jepsen worker 55 - jepsen.util 7885 :info :write [18 15] :RequestCanceledException INFO [2019-12-17 12:33:16,429] jepsen worker 55 - jepsen.util 7975 :invoke :read [18 nil] INFO [2019-12-17 12:33:16,440] jepsen worker 53 - jepsen.util 6533 :ok :read [17 33] INFO [2019-12-17 12:33:16,440] jepsen worker 53 - jepsen.util 6533 :invoke :read [17 nil] INFO [2019-12-17 12:33:16,453] jepsen worker 9 - jepsen.util 7389 :fail :cas [3 [15 39]] :ValueNotSwapFrom INFO [2019-12-17 12:33:16,457] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":39767,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:33:16,461] jepsen worker 63 - jepsen.util 5913 :ok :read [21 24] INFO [2019-12-17 12:33:16,466] jepsen worker 47 - jepsen.util 7337 :ok :write [15 31] INFO [2019-12-17 12:33:16,468] jepsen worker 71 - jepsen.util 7631 :ok :write [23 12] INFO [2019-12-17 12:33:16,480] jepsen worker 19 - jepsen.util 6859 :ok :write [6 3] INFO [2019-12-17 12:33:16,515] jepsen worker 9 - jepsen.util 7389 :invoke :read [3 nil] INFO [2019-12-17 12:33:16,515] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":39767,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:33:16,515] jepsen worker 63 - jepsen.util 5913 :invoke :read [21 nil] INFO [2019-12-17 12:33:16,515] jepsen worker 47 - jepsen.util 7337 :invoke :cas [15 [47 20]] INFO [2019-12-17 12:33:16,515] jepsen worker 71 - jepsen.util 7631 :invoke :read [23 nil] INFO [2019-12-17 12:33:16,515] jepsen worker 19 - jepsen.util 6859 :invoke :read [6 nil] INFO [2019-12-17 12:33:16,515] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":39771,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:33:16,522] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":39771,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:33:16,540] jepsen worker 9 - jepsen.util 7389 :ok :read [3 4] INFO [2019-12-17 12:33:16,540] jepsen worker 9 - jepsen.util 7389 :invoke :write [3 17] INFO [2019-12-17 12:33:16,549] jepsen worker 53 - jepsen.util 6533 :ok :read [17 33] INFO [2019-12-17 12:33:16,549] jepsen worker 53 - jepsen.util 6533 :invoke :write [17 46] INFO [2019-12-17 12:33:16,558] jepsen worker 74 - jepsen.util 7094 :fail :read [24 nil] :RequestTimeoutException INFO [2019-12-17 12:33:16,558] jepsen worker 74 - jepsen.util 7094 :invoke :cas [24 [3 30]] INFO [2019-12-17 12:33:16,568] jepsen worker 19 - jepsen.util 6859 :ok :read [6 3] INFO [2019-12-17 12:33:16,568] jepsen worker 19 - jepsen.util 6859 :invoke :cas [6 [49 47]] INFO [2019-12-17 12:33:16,570] jepsen worker 63 - jepsen.util 5913 :ok :read [21 24] INFO [2019-12-17 12:33:16,570] jepsen worker 63 - jepsen.util 5913 :invoke :write [21 47] INFO [2019-12-17 12:33:16,573] jepsen worker 47 - jepsen.util 7337 :fail :cas [15 [47 20]] :ValueNotSwapFrom INFO [2019-12-17 12:33:16,573] jepsen worker 47 - jepsen.util 7337 :invoke :read [15 nil] INFO [2019-12-17 12:33:16,594] jepsen worker 74 - jepsen.util 7094 :fail :cas [24 [3 30]] :ValueNotSwapFrom INFO [2019-12-17 12:33:16,594] jepsen worker 74 - jepsen.util 7094 :invoke :read [24 nil] INFO [2019-12-17 12:33:16,600] jepsen worker 47 - jepsen.util 7337 :ok :read [15 31] INFO [2019-12-17 12:33:16,600] jepsen worker 47 - jepsen.util 7337 :invoke :cas [15 [31 1]] INFO [2019-12-17 12:33:16,608] jepsen worker 19 - jepsen.util 6859 :fail :cas [6 [49 47]] :ValueNotSwapFrom INFO [2019-12-17 12:33:16,608] jepsen worker 19 - jepsen.util 6859 :invoke :write [6 34] INFO [2019-12-17 12:33:16,611] jepsen worker 54 - jepsen.util 7614 :info :write [18 37] :RequestCanceledException INFO [2019-12-17 12:33:16,611] jepsen worker 18 - jepsen.util 6858 :fail :read [6 nil] :RequestTimeoutException INFO [2019-12-17 12:33:16,611] jepsen worker 18 - jepsen.util 6858 :invoke :read [6 nil] INFO [2019-12-17 12:33:16,612] jepsen worker 54 - jepsen.util 7704 :invoke :write [18 18] INFO [2019-12-17 12:33:16,619] jepsen worker 74 - jepsen.util 7094 :ok :read [24 15] INFO [2019-12-17 12:33:16,619] jepsen worker 74 - jepsen.util 7094 :invoke :read [24 nil] INFO [2019-12-17 12:33:16,641] jepsen worker 79 - jepsen.util 6829 :info :write [26 15] :RequestCanceledException INFO [2019-12-17 12:33:16,642] jepsen worker 79 - jepsen.util 6919 :invoke :read [26 nil] INFO [2019-12-17 12:33:16,659] jepsen worker 18 - jepsen.util 6858 :ok :read [6 3] INFO [2019-12-17 12:33:16,660] jepsen worker 18 - jepsen.util 6858 :invoke :write [6 49] INFO [2019-12-17 12:33:16,676] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":39793,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:33:16,676] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":39793,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:33:16,676] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":39796,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:33:16,676] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":39796,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:33:16,676] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":39797,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:33:16,676] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":39797,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:33:16,694] jepsen worker 46 - jepsen.util 7066 :fail :read [15 nil] :RequestTimeoutException INFO [2019-12-17 12:33:16,694] jepsen worker 46 - jepsen.util 7066 :invoke :write [15 1] INFO [2019-12-17 12:33:16,714] jepsen worker 53 - jepsen.util 6533 :ok :write [17 46] INFO [2019-12-17 12:33:16,714] jepsen worker 53 - jepsen.util 6533 :invoke :cas [17 [26 15]] INFO [2019-12-17 12:33:16,719] jepsen worker 70 - jepsen.util 7180 :fail :cas [23 [15 49]] :ValueNotSwapFrom INFO [2019-12-17 12:33:16,719] jepsen worker 70 - jepsen.util 7180 :invoke :cas [23 [28 10]] INFO [2019-12-17 12:33:16,728] jepsen worker 27 - jepsen.util 6957 :info :cas [9 [28 30]] :RequestCanceledException INFO [2019-12-17 12:33:16,729] jepsen worker 27 - jepsen.util 7047 :invoke :write [9 11] INFO [2019-12-17 12:33:16,741] jepsen worker 79 - jepsen.util 6919 :ok :read [26 46] INFO [2019-12-17 12:33:16,741] jepsen worker 79 - jepsen.util 6919 :invoke :cas [26 [36 18]] INFO [2019-12-17 12:33:16,772] jepsen worker 53 - jepsen.util 6533 :fail :cas [17 [26 15]] :ValueNotSwapFrom INFO [2019-12-17 12:33:16,772] jepsen worker 53 - jepsen.util 6533 :invoke :write [17 0] INFO [2019-12-17 12:33:16,838] jepsen worker 37 - jepsen.util 6517 :ok :write [12 11] INFO [2019-12-17 12:33:16,838] jepsen worker 37 - jepsen.util 6517 :invoke :read [12 nil] INFO [2019-12-17 12:33:16,839] jepsen worker 78 - jepsen.util 7098 :ok :write [26 1] INFO [2019-12-17 12:33:16,839] jepsen worker 78 - jepsen.util 7098 :invoke :cas [26 [3 35]] INFO [2019-12-17 12:33:16,850] jepsen worker 18 - jepsen.util 6858 :ok :write [6 49] INFO [2019-12-17 12:33:16,850] jepsen worker 9 - jepsen.util 7389 :ok :write [3 17] INFO [2019-12-17 12:33:16,850] jepsen worker 18 - jepsen.util 6858 :invoke :read [6 nil] INFO [2019-12-17 12:33:16,850] jepsen worker 9 - jepsen.util 7389 :invoke :write [3 36] INFO [2019-12-17 12:33:16,853] jepsen worker 36 - jepsen.util 7326 :fail :cas [12 [13 42]] :ValueNotSwapFrom INFO [2019-12-17 12:33:16,853] jepsen worker 36 - jepsen.util 7326 :invoke :read [12 nil] INFO [2019-12-17 12:33:16,859] jepsen worker 34 - jepsen.util 8764 :info :cas [11 [26 34]] :RequestCanceledException INFO [2019-12-17 12:33:16,860] jepsen worker 34 - jepsen.util 8854 :invoke :write [11 22] INFO [2019-12-17 12:33:16,868] jepsen worker 47 - jepsen.util 7337 :ok :cas [15 [31 1]] INFO [2019-12-17 12:33:16,868] jepsen worker 47 - jepsen.util 7337 :invoke :cas [15 [47 38]] INFO [2019-12-17 12:33:16,873] jepsen worker 18 - jepsen.util 6858 :ok :read [6 49] INFO [2019-12-17 12:33:16,873] jepsen worker 18 - jepsen.util 6858 :invoke :write [6 19] INFO [2019-12-17 12:33:16,882] jepsen worker 4 - jepsen.util 6214 :ok :write [1 35] INFO [2019-12-17 12:33:16,882] jepsen worker 4 - jepsen.util 6214 :invoke :cas [1 [33 30]] INFO [2019-12-17 12:33:16,894] jepsen worker 58 - jepsen.util 9958 :info :write [19 16] :RequestCanceledException INFO [2019-12-17 12:33:16,894] jepsen worker 30 - jepsen.util 6780 :fail :read [10 nil] :RequestTimeoutException INFO [2019-12-17 12:33:16,894] jepsen worker 30 - jepsen.util 6780 :invoke :read [10 nil] INFO [2019-12-17 12:33:16,895] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":39814,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:33:16,895] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":39814,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:33:16,895] jepsen worker 58 - jepsen.util 10048 :invoke :read [19 nil] INFO [2019-12-17 12:33:16,912] jepsen worker 21 - jepsen.util 8841 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:33:16,912] jepsen worker 21 - jepsen.util 8841 :invoke :cas [7 [37 25]] INFO [2019-12-17 12:33:16,921] jepsen worker 27 - jepsen.util 7047 :ok :write [9 11] INFO [2019-12-17 12:33:16,921] jepsen worker 27 - jepsen.util 7047 :invoke :cas [9 [24 5]] INFO [2019-12-17 12:33:16,927] jepsen worker 23 - jepsen.util 9383 :info :cas [7 [9 43]] :RequestCanceledException INFO [2019-12-17 12:33:16,928] jepsen worker 42 - jepsen.util 9852 :info :cas [14 [7 49]] :RequestCanceledException INFO [2019-12-17 12:33:16,928] jepsen worker 75 - jepsen.util 8805 :info :write [25 19] :RequestCanceledException INFO [2019-12-17 12:33:16,928] jepsen worker 23 - jepsen.util 9473 :invoke :read [7 nil] INFO [2019-12-17 12:33:16,929] jepsen worker 75 - jepsen.util 8895 :invoke :cas [25 [31 19]] INFO [2019-12-17 12:33:16,929] jepsen worker 42 - jepsen.util 9942 :invoke :cas [14 [13 21]] INFO [2019-12-17 12:33:16,941] jepsen worker 0 - jepsen.util 9720 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:33:16,941] jepsen worker 0 - jepsen.util 9720 :invoke :cas [0 [42 9]] INFO [2019-12-17 12:33:16,941] jepsen worker 30 - jepsen.util 6780 :ok :read [10 1] INFO [2019-12-17 12:33:16,942] jepsen worker 30 - jepsen.util 6780 :invoke :cas [10 [12 6]] INFO [2019-12-17 12:33:16,959] jepsen worker 24 - jepsen.util 8214 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:33:16,959] jepsen worker 77 - jepsen.util 9617 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:33:16,959] jepsen worker 43 - jepsen.util 8593 :info :write [14 17] :RequestCanceledException INFO [2019-12-17 12:33:16,959] jepsen worker 62 - jepsen.util 9332 :info :write [20 3] :RequestCanceledException INFO [2019-12-17 12:33:16,959] jepsen worker 25 - jepsen.util 9745 :info :cas [8 [37 27]] :RequestCanceledException INFO [2019-12-17 12:33:16,959] jepsen worker 77 - jepsen.util 9617 :invoke :write [25 45] INFO [2019-12-17 12:33:16,959] jepsen worker 24 - jepsen.util 8214 :invoke :cas [8 [23 7]] INFO [2019-12-17 12:33:16,960] jepsen worker 62 - jepsen.util 9422 :invoke :read [20 nil] INFO [2019-12-17 12:33:16,960] jepsen worker 43 - jepsen.util 8683 :invoke :cas [14 [26 15]] INFO [2019-12-17 12:33:16,960] jepsen worker 25 - jepsen.util 9835 :invoke :write [8 33] INFO [2019-12-17 12:33:16,962] jepsen worker 27 - jepsen.util 7047 :fail :cas [9 [24 5]] :ValueNotSwapFrom INFO [2019-12-17 12:33:16,962] jepsen worker 27 - jepsen.util 7047 :invoke :write [9 3] INFO [2019-12-17 12:33:16,970] jepsen worker 36 - jepsen.util 7326 :ok :read [12 11] INFO [2019-12-17 12:33:16,970] jepsen worker 36 - jepsen.util 7326 :invoke :cas [12 [22 40]] INFO [2019-12-17 12:33:16,982] jepsen worker 87 - jepsen.util 6117 :ok :read [29 37] INFO [2019-12-17 12:33:16,982] jepsen worker 87 - jepsen.util 6117 :invoke :read [29 nil] INFO [2019-12-17 12:33:16,986] jepsen worker 30 - jepsen.util 6780 :fail :cas [10 [12 6]] :ValueNotSwapFrom INFO [2019-12-17 12:33:16,986] jepsen worker 30 - jepsen.util 6780 :invoke :cas [10 [17 32]] INFO [2019-12-17 12:33:16,994] jepsen worker 22 - jepsen.util 9292 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:33:16,994] jepsen worker 68 - jepsen.util 8798 :info :write [22 3] :RequestCanceledException INFO [2019-12-17 12:33:16,994] jepsen worker 22 - jepsen.util 9292 :invoke :cas [7 [27 15]] INFO [2019-12-17 12:33:16,995] jepsen worker 68 - jepsen.util 8888 :invoke :read [22 nil] INFO [2019-12-17 12:33:17,005] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":39842,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:33:17,005] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":39842,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:33:17,005] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":39846,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:33:17,005] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":39846,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:33:17,012] jepsen worker 33 - jepsen.util 9123 :info :write [11 27] :RequestCanceledException INFO [2019-12-17 12:33:17,012] jepsen worker 11 - jepsen.util 7211 :fail :read [3 nil] :RequestTimeoutException INFO [2019-12-17 12:33:17,012] jepsen worker 66 - jepsen.util 9066 :info :write [22 19] :RequestCanceledException INFO [2019-12-17 12:33:17,012] jepsen worker 11 - jepsen.util 7211 :invoke :write [3 40] INFO [2019-12-17 12:33:17,013] jepsen worker 66 - jepsen.util 9156 :invoke :read [22 nil] INFO [2019-12-17 12:33:17,013] jepsen worker 33 - jepsen.util 9213 :invoke :read [11 nil] INFO [2019-12-17 12:33:17,015] jepsen worker 55 - jepsen.util 7975 :ok :read [18 33] INFO [2019-12-17 12:33:17,015] jepsen worker 55 - jepsen.util 7975 :invoke :read [18 nil] INFO [2019-12-17 12:33:17,027] jepsen worker 71 - jepsen.util 7631 :ok :read [23 12] INFO [2019-12-17 12:33:17,027] jepsen worker 71 - jepsen.util 7631 :invoke :read [23 nil] INFO [2019-12-17 12:33:17,028] jepsen worker 60 - jepsen.util 10140 :info :cas [20 [8 39]] :RequestCanceledException INFO [2019-12-17 12:33:17,028] jepsen worker 59 - jepsen.util 9239 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:33:17,028] jepsen worker 15 - jepsen.util 8295 :info :write [5 31] :RequestCanceledException INFO [2019-12-17 12:33:17,028] jepsen worker 26 - jepsen.util 9566 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:33:17,028] jepsen worker 67 - jepsen.util 9427 :info :write [22 19] :RequestCanceledException INFO [2019-12-17 12:33:17,028] jepsen worker 41 - jepsen.util 9851 :info :cas [13 [27 24]] :RequestCanceledException INFO [2019-12-17 12:33:17,028] jepsen worker 1 - jepsen.util 9361 :info :cas [0 [15 25]] :RequestCanceledException INFO [2019-12-17 12:33:17,028] jepsen worker 10 - jepsen.util 6670 :info :write [3 36] :RequestCanceledException INFO [2019-12-17 12:33:17,028] jepsen worker 72 - jepsen.util 6642 :fail :read [24 nil] :RequestTimeoutException INFO [2019-12-17 12:33:17,028] jepsen worker 59 - jepsen.util 9239 :invoke :write [19 30] INFO [2019-12-17 12:33:17,028] jepsen worker 26 - jepsen.util 9566 :invoke :cas [8 [43 37]] INFO [2019-12-17 12:33:17,028] jepsen worker 72 - jepsen.util 6642 :invoke :read [24 nil] INFO [2019-12-17 12:33:17,028] jepsen worker 27 - jepsen.util 7047 :ok :write [9 3] INFO [2019-12-17 12:33:17,029] jepsen worker 27 - jepsen.util 7047 :invoke :cas [9 [12 17]] INFO [2019-12-17 12:33:17,029] jepsen worker 60 - jepsen.util 10230 :invoke :read [20 nil] INFO [2019-12-17 12:33:17,029] jepsen worker 15 - jepsen.util 8385 :invoke :write [5 1] INFO [2019-12-17 12:33:17,029] jepsen worker 67 - jepsen.util 9517 :invoke :write [22 46] INFO [2019-12-17 12:33:17,029] jepsen worker 1 - jepsen.util 9451 :invoke :read [0 nil] INFO [2019-12-17 12:33:17,029] jepsen worker 41 - jepsen.util 9941 :invoke :read [13 nil] INFO [2019-12-17 12:33:17,029] jepsen worker 10 - jepsen.util 6760 :invoke :cas [3 [34 46]] INFO [2019-12-17 12:33:17,042] jepsen worker 83 - jepsen.util 9803 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:33:17,042] jepsen worker 83 - jepsen.util 9803 :invoke :cas [27 [18 37]] INFO [2019-12-17 12:33:17,042] jepsen worker 30 - jepsen.util 6780 :fail :cas [10 [17 32]] :ValueNotSwapFrom INFO [2019-12-17 12:33:17,042] jepsen worker 30 - jepsen.util 6780 :invoke :read [10 nil] INFO [2019-12-17 12:33:17,050] jepsen worker 27 - jepsen.util 7047 :fail :cas [9 [12 17]] :ValueNotSwapFrom INFO [2019-12-17 12:33:17,050] jepsen worker 27 - jepsen.util 7047 :invoke :write [9 11] INFO [2019-12-17 12:33:17,059] jepsen worker 82 - jepsen.util 9982 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:33:17,059] jepsen worker 86 - jepsen.util 9806 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:33:17,059] jepsen worker 82 - jepsen.util 9982 :invoke :write [27 26] INFO [2019-12-17 12:33:17,059] jepsen worker 86 - jepsen.util 9806 :invoke :cas [28 [10 43]] INFO [2019-12-17 12:33:17,061] jepsen worker 30 - jepsen.util 6780 :ok :read [10 1] INFO [2019-12-17 12:33:17,061] jepsen worker 30 - jepsen.util 6780 :invoke :read [10 nil] INFO [2019-12-17 12:33:17,066] jepsen worker 72 - jepsen.util 6642 :ok :read [24 15] INFO [2019-12-17 12:33:17,066] jepsen worker 72 - jepsen.util 6642 :invoke :write [24 48] INFO [2019-12-17 12:33:17,077] jepsen worker 79 - jepsen.util 6919 :fail :cas [26 [36 18]] :ValueNotSwapFrom INFO [2019-12-17 12:33:17,077] jepsen worker 79 - jepsen.util 6919 :invoke :read [26 nil] INFO [2019-12-17 12:33:17,095] jepsen worker 50 - jepsen.util 9680 :info :cas [16 [31 46]] :RequestCanceledException INFO [2019-12-17 12:33:17,095] jepsen worker 69 - jepsen.util 6909 :info :write [23 38] :RequestCanceledException INFO [2019-12-17 12:33:17,096] jepsen worker 69 - jepsen.util 6999 :invoke :cas [23 [26 36]] INFO [2019-12-17 12:33:17,096] jepsen worker 50 - jepsen.util 9770 :invoke :cas [16 [40 49]] INFO [2019-12-17 12:33:17,101] jepsen worker 27 - jepsen.util 7047 :ok :write [9 11] INFO [2019-12-17 12:33:17,102] jepsen worker 27 - jepsen.util 7047 :invoke :cas [9 [38 44]] INFO [2019-12-17 12:33:17,117] jepsen worker 27 - jepsen.util 7047 :fail :cas [9 [38 44]] :ValueNotSwapFrom INFO [2019-12-17 12:33:17,117] jepsen worker 27 - jepsen.util 7047 :invoke :read [9 nil] INFO [2019-12-17 12:33:17,126] jepsen worker 27 - jepsen.util 7047 :ok :read [9 11] INFO [2019-12-17 12:33:17,126] jepsen worker 27 - jepsen.util 7047 :invoke :cas [9 [49 27]] INFO [2019-12-17 12:33:17,139] jepsen worker 27 - jepsen.util 7047 :fail :cas [9 [49 27]] :ValueNotSwapFrom INFO [2019-12-17 12:33:17,139] jepsen worker 27 - jepsen.util 7047 :invoke :write [9 10] INFO [2019-12-17 12:33:17,141] jepsen worker 89 - jepsen.util 6659 :info :write [29 28] :RequestCanceledException INFO [2019-12-17 12:33:17,142] jepsen worker 89 - jepsen.util 6749 :invoke :cas [29 [46 4]] INFO [2019-12-17 12:33:17,146] jepsen worker 79 - jepsen.util 6919 :ok :read [26 1] INFO [2019-12-17 12:33:17,146] jepsen worker 79 - jepsen.util 6919 :invoke :read [26 nil] INFO [2019-12-17 12:33:17,157] jepsen worker 37 - jepsen.util 6517 :ok :read [12 11] INFO [2019-12-17 12:33:17,158] jepsen worker 37 - jepsen.util 6517 :invoke :write [12 20] INFO [2019-12-17 12:33:17,159] jepsen worker 3 - jepsen.util 6753 :info :write [1 26] :RequestCanceledException INFO [2019-12-17 12:33:17,160] jepsen worker 3 - jepsen.util 6843 :invoke :write [1 15] INFO [2019-12-17 12:33:17,208] jepsen worker 27 - jepsen.util 7047 :ok :write [9 10] INFO [2019-12-17 12:33:17,208] jepsen worker 27 - jepsen.util 7047 :invoke :read [9 nil] INFO [2019-12-17 12:33:17,212] jepsen worker 38 - jepsen.util 6788 :info :write [12 39] :RequestCanceledException INFO [2019-12-17 12:33:17,213] jepsen worker 38 - jepsen.util 6878 :invoke :write [12 30] INFO [2019-12-17 12:33:17,218] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":39881,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:33:17,218] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":39888,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:33:17,218] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":39881,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:33:17,218] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":39888,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:33:17,218] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":39891,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:33:17,219] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":39891,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:33:17,219] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":39892,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:33:17,219] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":39892,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:33:17,222] jepsen worker 27 - jepsen.util 7047 :ok :read [9 10] INFO [2019-12-17 12:33:17,222] jepsen worker 27 - jepsen.util 7047 :invoke :read [9 nil] INFO [2019-12-17 12:33:17,224] jepsen worker 79 - jepsen.util 6919 :ok :read [26 1] INFO [2019-12-17 12:33:17,225] jepsen worker 79 - jepsen.util 6919 :invoke :cas [26 [30 3]] INFO [2019-12-17 12:33:17,225] jepsen worker 70 - jepsen.util 7180 :fail :cas [23 [28 10]] :ValueNotSwapFrom INFO [2019-12-17 12:33:17,225] jepsen worker 70 - jepsen.util 7180 :invoke :cas [23 [43 42]] INFO [2019-12-17 12:33:17,228] jepsen worker 52 - jepsen.util 7162 :info :write [17 43] :RequestCanceledException INFO [2019-12-17 12:33:17,231] jepsen worker 87 - jepsen.util 6117 :ok :read [29 37] INFO [2019-12-17 12:33:17,231] jepsen worker 87 - jepsen.util 6117 :invoke :read [29 nil] INFO [2019-12-17 12:33:17,232] jepsen worker 52 - jepsen.util 7252 :invoke :cas [17 [15 7]] INFO [2019-12-17 12:33:17,239] jepsen worker 27 - jepsen.util 7047 :ok :read [9 10] INFO [2019-12-17 12:33:17,239] jepsen worker 27 - jepsen.util 7047 :invoke :cas [9 [20 13]] INFO [2019-12-17 12:33:17,253] jepsen worker 27 - jepsen.util 7047 :fail :cas [9 [20 13]] :ValueNotSwapFrom INFO [2019-12-17 12:33:17,253] jepsen worker 27 - jepsen.util 7047 :invoke :cas [9 [4 35]] INFO [2019-12-17 12:33:17,268] jepsen worker 27 - jepsen.util 7047 :fail :cas [9 [4 35]] :ValueNotSwapFrom INFO [2019-12-17 12:33:17,268] jepsen worker 27 - jepsen.util 7047 :invoke :read [9 nil] INFO [2019-12-17 12:33:17,282] jepsen worker 79 - jepsen.util 6919 :fail :cas [26 [30 3]] :ValueNotSwapFrom INFO [2019-12-17 12:33:17,282] jepsen worker 79 - jepsen.util 6919 :invoke :cas [26 [45 0]] INFO [2019-12-17 12:33:17,283] jepsen worker 27 - jepsen.util 7047 :ok :read [9 10] INFO [2019-12-17 12:33:17,283] jepsen worker 27 - jepsen.util 7047 :invoke :read [9 nil] INFO [2019-12-17 12:33:17,284] jepsen worker 55 - jepsen.util 7975 :ok :read [18 33] INFO [2019-12-17 12:33:17,284] jepsen worker 55 - jepsen.util 7975 :invoke :cas [18 [40 40]] INFO [2019-12-17 12:33:17,288] jepsen worker 4 - jepsen.util 6214 :fail :cas [1 [33 30]] :ValueNotSwapFrom INFO [2019-12-17 12:33:17,288] jepsen worker 4 - jepsen.util 6214 :invoke :cas [1 [6 27]] INFO [2019-12-17 12:33:17,296] jepsen worker 27 - jepsen.util 7047 :ok :read [9 10] INFO [2019-12-17 12:33:17,298] jepsen worker 27 - jepsen.util 7047 :invoke :read [9 nil] INFO [2019-12-17 12:33:17,310] jepsen worker 27 - jepsen.util 7047 :ok :read [9 10] INFO [2019-12-17 12:33:17,310] jepsen worker 27 - jepsen.util 7047 :invoke :cas [9 [40 12]] INFO [2019-12-17 12:33:17,312] jepsen worker 71 - jepsen.util 7631 :ok :read [23 12] INFO [2019-12-17 12:33:17,312] jepsen worker 76 - jepsen.util 9706 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:33:17,312] jepsen worker 71 - jepsen.util 7631 :invoke :cas [23 [48 39]] INFO [2019-12-17 12:33:17,312] jepsen worker 76 - jepsen.util 9706 :invoke :cas [25 [26 37]] INFO [2019-12-17 12:33:17,322] jepsen worker 27 - jepsen.util 7047 :fail :cas [9 [40 12]] :ValueNotSwapFrom INFO [2019-12-17 12:33:17,322] jepsen worker 27 - jepsen.util 7047 :invoke :read [9 nil] INFO [2019-12-17 12:33:17,333] jepsen worker 27 - jepsen.util 7047 :ok :read [9 10] INFO [2019-12-17 12:33:17,333] jepsen worker 27 - jepsen.util 7047 :invoke :read [9 nil] INFO [2019-12-17 12:33:17,345] jepsen worker 27 - jepsen.util 7047 :ok :read [9 10] INFO [2019-12-17 12:33:17,345] jepsen worker 27 - jepsen.util 7047 :invoke :cas [9 [41 10]] INFO [2019-12-17 12:33:17,358] jepsen worker 9 - jepsen.util 7389 :ok :write [3 36] INFO [2019-12-17 12:33:17,358] jepsen worker 9 - jepsen.util 7389 :invoke :write [3 37] INFO [2019-12-17 12:33:17,359] jepsen worker 18 - jepsen.util 6858 :ok :write [6 19] INFO [2019-12-17 12:33:17,360] jepsen worker 18 - jepsen.util 6858 :invoke :cas [6 [42 16]] INFO [2019-12-17 12:33:17,364] jepsen worker 27 - jepsen.util 7047 :fail :cas [9 [41 10]] :ValueNotSwapFrom INFO [2019-12-17 12:33:17,364] jepsen worker 27 - jepsen.util 7047 :invoke :cas [9 [6 16]] INFO [2019-12-17 12:33:17,385] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:33:17,385] jepsen nemesis - couchbase.util Rebalance check count: 305 INFO [2019-12-17 12:33:17,385] jepsen nemesis - couchbase.util Rebalance stuck count: 286 INFO [2019-12-17 12:33:17,388] jepsen worker 53 - jepsen.util 6533 :ok :write [17 0] INFO [2019-12-17 12:33:17,389] jepsen worker 53 - jepsen.util 6533 :invoke :cas [17 [25 7]] INFO [2019-12-17 12:33:17,389] jepsen worker 72 - jepsen.util 6642 :ok :write [24 48] INFO [2019-12-17 12:33:17,389] jepsen worker 72 - jepsen.util 6642 :invoke :write [24 4] INFO [2019-12-17 12:33:17,394] jepsen worker 14 - jepsen.util 7034 :info :write [4 12] :RequestCanceledException INFO [2019-12-17 12:33:17,396] jepsen worker 14 - jepsen.util 7124 :invoke :read [4 nil] INFO [2019-12-17 12:33:17,399] jepsen worker 27 - jepsen.util 7047 :fail :cas [9 [6 16]] :ValueNotSwapFrom INFO [2019-12-17 12:33:17,399] jepsen worker 27 - jepsen.util 7047 :invoke :write [9 40] INFO [2019-12-17 12:33:17,400] jepsen worker 10 - jepsen.util 6760 :fail :cas [3 [34 46]] :ValueNotSwapFrom INFO [2019-12-17 12:33:17,400] jepsen worker 10 - jepsen.util 6760 :invoke :read [3 nil] INFO [2019-12-17 12:33:17,427] jepsen worker 51 - jepsen.util 6621 :info :write [17 14] :RequestCanceledException INFO [2019-12-17 12:33:17,427] jepsen worker 6 - jepsen.util 9906 :info :write [2 37] :RequestCanceledException INFO [2019-12-17 12:33:17,427] jepsen worker 84 - jepsen.util 8814 :info :cas [28 [12 29]] :RequestCanceledException INFO [2019-12-17 12:33:17,428] jepsen worker 84 - jepsen.util 8904 :invoke :cas [28 [33 16]] INFO [2019-12-17 12:33:17,428] jepsen worker 51 - jepsen.util 6711 :invoke :read [17 nil] INFO [2019-12-17 12:33:17,428] jepsen worker 6 - jepsen.util 9996 :invoke :write [2 35] INFO [2019-12-17 12:33:17,434] jepsen worker 69 - jepsen.util 6999 :fail :cas [23 [26 36]] :ValueNotSwapFrom INFO [2019-12-17 12:33:17,434] jepsen worker 69 - jepsen.util 6999 :invoke :read [23 nil] INFO [2019-12-17 12:33:17,439] jepsen worker 88 - jepsen.util 6838 :info :write [29 29] :RequestCanceledException INFO [2019-12-17 12:33:17,439] jepsen worker 39 - jepsen.util 9219 :info :cas [13 [45 3]] :RequestCanceledException INFO [2019-12-17 12:33:17,440] jepsen worker 39 - jepsen.util 9309 :invoke :read [13 nil] INFO [2019-12-17 12:33:17,440] jepsen worker 88 - jepsen.util 6928 :invoke :write [29 2] INFO [2019-12-17 12:33:17,445] jepsen worker 10 - jepsen.util 6760 :ok :read [3 36] INFO [2019-12-17 12:33:17,445] jepsen worker 10 - jepsen.util 6760 :invoke :write [3 44] INFO [2019-12-17 12:33:17,456] jepsen worker 40 - jepsen.util 9850 :info :write [13 12] :RequestCanceledException INFO [2019-12-17 12:33:17,456] jepsen worker 17 - jepsen.util 9647 :info :cas [5 [36 33]] :RequestCanceledException INFO [2019-12-17 12:33:17,456] jepsen worker 81 - jepsen.util 9081 :info :write [27 12] :RequestCanceledException INFO [2019-12-17 12:33:17,457] jepsen worker 81 - jepsen.util 9171 :invoke :cas [27 [0 1]] INFO [2019-12-17 12:33:17,457] jepsen worker 4 - jepsen.util 6214 :fail :cas [1 [6 27]] :ValueNotSwapFrom INFO [2019-12-17 12:33:17,457] jepsen worker 17 - jepsen.util 9737 :invoke :read [5 nil] INFO [2019-12-17 12:33:17,458] jepsen worker 40 - jepsen.util 9940 :invoke :read [13 nil] INFO [2019-12-17 12:33:17,458] jepsen worker 4 - jepsen.util 6214 :invoke :cas [1 [43 42]] INFO [2019-12-17 12:33:17,465] jepsen worker 27 - jepsen.util 7047 :ok :write [9 40] INFO [2019-12-17 12:33:17,465] jepsen worker 27 - jepsen.util 7047 :invoke :read [9 nil] INFO [2019-12-17 12:33:17,478] jepsen worker 27 - jepsen.util 7047 :ok :read [9 40] INFO [2019-12-17 12:33:17,478] jepsen worker 27 - jepsen.util 7047 :invoke :read [9 nil] INFO [2019-12-17 12:33:17,492] jepsen worker 78 - jepsen.util 7098 :fail :cas [26 [3 35]] :ValueNotSwapFrom INFO [2019-12-17 12:33:17,495] jepsen worker 27 - jepsen.util 7047 :ok :read [9 40] INFO [2019-12-17 12:33:17,506] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":39923,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:33:17,508] jepsen worker 11 - jepsen.util 7211 :ok :write [3 40] INFO [2019-12-17 12:33:17,521] jepsen worker 54 - jepsen.util 7704 :ok :write [18 18] INFO [2019-12-17 12:33:17,523] jepsen worker 18 - jepsen.util 6858 :fail :cas [6 [42 16]] :ValueNotSwapFrom INFO [2019-12-17 12:33:17,528] jepsen worker 56 - jepsen.util 7616 :info :write [18 35] :RequestCanceledException INFO [2019-12-17 12:33:17,530] jepsen worker 72 - jepsen.util 6642 :ok :write [24 4] INFO [2019-12-17 12:33:17,531] jepsen worker 52 - jepsen.util 7252 :fail :cas [17 [15 7]] :ValueNotSwapFrom INFO [2019-12-17 12:33:17,542] jepsen worker 44 - jepsen.util 10124 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:33:17,557] jepsen worker 79 - jepsen.util 6919 :fail :cas [26 [45 0]] :ValueNotSwapFrom INFO [2019-12-17 12:33:17,569] jepsen worker 70 - jepsen.util 7180 :fail :cas [23 [43 42]] :ValueNotSwapFrom INFO [2019-12-17 12:33:17,581] jepsen worker 14 - jepsen.util 7124 :ok :read [4 44] INFO [2019-12-17 12:33:17,586] jepsen worker 37 - jepsen.util 6517 :ok :write [12 20] INFO [2019-12-17 12:33:17,592] jepsen worker 53 - jepsen.util 6533 :fail :cas [17 [25 7]] :ValueNotSwapFrom INFO [2019-12-17 12:33:17,624] jepsen worker 27 - jepsen.util 7047 :invoke :read [9 nil] INFO [2019-12-17 12:33:17,624] jepsen worker 78 - jepsen.util 7098 :invoke :cas [26 [16 4]] INFO [2019-12-17 12:33:17,624] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":39923,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:33:17,624] jepsen worker 11 - jepsen.util 7211 :invoke :read [3 nil] INFO [2019-12-17 12:33:17,624] jepsen worker 54 - jepsen.util 7704 :invoke :write [18 48] INFO [2019-12-17 12:33:17,624] jepsen worker 72 - jepsen.util 6642 :invoke :read [24 nil] INFO [2019-12-17 12:33:17,624] jepsen worker 52 - jepsen.util 7252 :invoke :cas [17 [35 37]] INFO [2019-12-17 12:33:17,624] jepsen worker 44 - jepsen.util 10124 :invoke :read [14 nil] INFO [2019-12-17 12:33:17,624] jepsen worker 79 - jepsen.util 6919 :invoke :cas [26 [17 17]] INFO [2019-12-17 12:33:17,624] jepsen worker 70 - jepsen.util 7180 :invoke :read [23 nil] INFO [2019-12-17 12:33:17,624] jepsen worker 14 - jepsen.util 7124 :invoke :read [4 nil] INFO [2019-12-17 12:33:17,625] jepsen worker 37 - jepsen.util 6517 :invoke :read [12 nil] INFO [2019-12-17 12:33:17,625] jepsen worker 53 - jepsen.util 6533 :invoke :write [17 33] INFO [2019-12-17 12:33:17,625] jepsen worker 18 - jepsen.util 6858 :invoke :write [6 48] INFO [2019-12-17 12:33:17,625] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":39926,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:33:17,625] jepsen worker 56 - jepsen.util 7706 :invoke :write [18 27] INFO [2019-12-17 12:33:17,628] jepsen worker 4 - jepsen.util 6214 :fail :cas [1 [43 42]] :ValueNotSwapFrom INFO [2019-12-17 12:33:17,628] jepsen worker 5 - jepsen.util 6485 :info :write [1 27] :RequestCanceledException INFO [2019-12-17 12:33:17,628] jepsen worker 48 - jepsen.util 8598 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:33:17,628] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":39926,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:33:17,628] jepsen worker 4 - jepsen.util 6214 :invoke :cas [1 [8 11]] INFO [2019-12-17 12:33:17,628] jepsen worker 48 - jepsen.util 8598 :invoke :cas [16 [11 45]] INFO [2019-12-17 12:33:17,629] jepsen worker 5 - jepsen.util 6575 :invoke :cas [1 [29 2]] INFO [2019-12-17 12:33:17,642] jepsen worker 57 - jepsen.util 9057 :info :cas [19 [36 48]] :RequestCanceledException INFO [2019-12-17 12:33:17,643] jepsen worker 57 - jepsen.util 9147 :invoke :read [19 nil] INFO [2019-12-17 12:33:17,647] jepsen worker 87 - jepsen.util 6117 :ok :read [29 37] INFO [2019-12-17 12:33:17,647] jepsen worker 87 - jepsen.util 6117 :invoke :write [29 12] INFO [2019-12-17 12:33:17,653] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":39959,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:33:17,653] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":39959,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:33:17,653] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":39962,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:33:17,653] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":39962,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:33:17,664] jepsen worker 11 - jepsen.util 7211 :ok :read [3 40] INFO [2019-12-17 12:33:17,664] jepsen worker 27 - jepsen.util 7047 :ok :read [9 40] INFO [2019-12-17 12:33:17,664] jepsen worker 11 - jepsen.util 7211 :invoke :read [3 nil] INFO [2019-12-17 12:33:17,664] jepsen worker 27 - jepsen.util 7047 :invoke :write [9 33] INFO [2019-12-17 12:33:17,677] jepsen worker 72 - jepsen.util 6642 :ok :read [24 4] INFO [2019-12-17 12:33:17,677] jepsen worker 72 - jepsen.util 6642 :invoke :write [24 26] INFO [2019-12-17 12:33:17,693] jepsen worker 61 - jepsen.util 10231 :info :write [20 33] :RequestCanceledException INFO [2019-12-17 12:33:17,693] jepsen worker 85 - jepsen.util 10345 :info :write [28 46] :RequestCanceledException INFO [2019-12-17 12:33:17,694] jepsen worker 85 - jepsen.util 10435 :invoke :read [28 nil] INFO [2019-12-17 12:33:17,694] jepsen worker 61 - jepsen.util 10321 :invoke :read [20 nil] INFO [2019-12-17 12:33:17,699] jepsen worker 78 - jepsen.util 7098 :fail :cas [26 [16 4]] :ValueNotSwapFrom INFO [2019-12-17 12:33:17,699] jepsen worker 78 - jepsen.util 7098 :invoke :cas [26 [14 21]] INFO [2019-12-17 12:33:17,710] jepsen worker 7 - jepsen.util 9637 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:33:17,710] jepsen worker 7 - jepsen.util 9637 :invoke :cas [2 [49 25]] INFO [2019-12-17 12:33:17,712] jepsen worker 69 - jepsen.util 6999 :ok :read [23 12] INFO [2019-12-17 12:33:17,712] jepsen worker 69 - jepsen.util 6999 :invoke :read [23 nil] INFO [2019-12-17 12:33:17,719] jepsen worker 27 - jepsen.util 7047 :ok :write [9 33] INFO [2019-12-17 12:33:17,719] jepsen worker 27 - jepsen.util 7047 :invoke :read [9 nil] INFO [2019-12-17 12:33:17,727] jepsen worker 49 - jepsen.util 8599 :info :cas [16 [31 37]] :RequestCanceledException INFO [2019-12-17 12:33:17,727] jepsen worker 16 - jepsen.util 10456 :info :cas [5 [1 11]] :RequestCanceledException INFO [2019-12-17 12:33:17,728] jepsen worker 16 - jepsen.util 10546 :invoke :write [5 21] INFO [2019-12-17 12:33:17,728] jepsen worker 49 - jepsen.util 8689 :invoke :write [16 44] INFO [2019-12-17 12:33:17,736] jepsen worker 27 - jepsen.util 7047 :ok :read [9 33] INFO [2019-12-17 12:33:17,737] jepsen worker 27 - jepsen.util 7047 :invoke :write [9 9] INFO [2019-12-17 12:33:17,746] jepsen worker 36 - jepsen.util 7326 :fail :cas [12 [22 40]] :ValueNotSwapFrom INFO [2019-12-17 12:33:17,746] jepsen worker 36 - jepsen.util 7326 :invoke :write [12 28] INFO [2019-12-17 12:33:17,759] jepsen worker 8 - jepsen.util 9188 :info :write [2 37] :RequestCanceledException INFO [2019-12-17 12:33:17,760] jepsen worker 8 - jepsen.util 9278 :invoke :cas [2 [31 36]] INFO [2019-12-17 12:33:17,778] jepsen worker 27 - jepsen.util 7047 :ok :write [9 9] INFO [2019-12-17 12:33:17,778] jepsen worker 27 - jepsen.util 7047 :invoke :cas [9 [24 1]] INFO [2019-12-17 12:33:17,798] jepsen worker 27 - jepsen.util 7047 :fail :cas [9 [24 1]] :ValueNotSwapFrom INFO [2019-12-17 12:33:17,798] jepsen worker 27 - jepsen.util 7047 :invoke :cas [9 [16 38]] INFO [2019-12-17 12:33:17,815] jepsen worker 27 - jepsen.util 7047 :fail :cas [9 [16 38]] :ValueNotSwapFrom INFO [2019-12-17 12:33:17,815] jepsen worker 27 - jepsen.util 7047 :invoke :cas [9 [25 24]] INFO [2019-12-17 12:33:17,836] jepsen worker 89 - jepsen.util 6749 :fail :cas [29 [46 4]] :ValueNotSwapFrom INFO [2019-12-17 12:33:17,836] jepsen worker 89 - jepsen.util 6749 :invoke :cas [29 [3 12]] INFO [2019-12-17 12:33:17,843] jepsen worker 27 - jepsen.util 7047 :fail :cas [9 [25 24]] :ValueNotSwapFrom INFO [2019-12-17 12:33:17,843] jepsen worker 27 - jepsen.util 7047 :invoke :cas [9 [48 14]] INFO [2019-12-17 12:33:17,856] jepsen worker 69 - jepsen.util 6999 :ok :read [23 12] INFO [2019-12-17 12:33:17,857] jepsen worker 69 - jepsen.util 6999 :invoke :write [23 41] INFO [2019-12-17 12:33:17,871] jepsen worker 27 - jepsen.util 7047 :fail :cas [9 [48 14]] :ValueNotSwapFrom INFO [2019-12-17 12:33:17,871] jepsen worker 18 - jepsen.util 6858 :ok :write [6 48] INFO [2019-12-17 12:33:17,871] jepsen worker 27 - jepsen.util 7047 :invoke :read [9 nil] INFO [2019-12-17 12:33:17,872] jepsen worker 18 - jepsen.util 6858 :invoke :write [6 32] INFO [2019-12-17 12:33:17,872] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":39986,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:33:17,872] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":39986,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:33:17,873] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":39989,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:33:17,873] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":39989,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:33:17,878] jepsen worker 51 - jepsen.util 6711 :ok :read [17 0] INFO [2019-12-17 12:33:17,878] jepsen worker 51 - jepsen.util 6711 :invoke :cas [17 [28 10]] INFO [2019-12-17 12:33:17,895] jepsen worker 27 - jepsen.util 7047 :ok :read [9 9] INFO [2019-12-17 12:33:17,895] jepsen worker 27 - jepsen.util 7047 :invoke :write [9 44] INFO [2019-12-17 12:33:17,911] jepsen worker 78 - jepsen.util 7098 :fail :cas [26 [14 21]] :ValueNotSwapFrom INFO [2019-12-17 12:33:17,917] jepsen worker 78 - jepsen.util 7098 :invoke :cas [26 [31 32]] INFO [2019-12-17 12:33:17,938] jepsen worker 27 - jepsen.util 7047 :ok :write [9 44] INFO [2019-12-17 12:33:17,938] jepsen worker 27 - jepsen.util 7047 :invoke :read [9 nil] INFO [2019-12-17 12:33:17,942] jepsen worker 65 - jepsen.util 5645 :info :write [21 2] :RequestCanceledException INFO [2019-12-17 12:33:17,942] jepsen worker 65 - jepsen.util 5735 :invoke :write [21 39] INFO [2019-12-17 12:33:17,957] jepsen worker 70 - jepsen.util 7180 :ok :read [23 12] INFO [2019-12-17 12:33:17,957] jepsen worker 70 - jepsen.util 7180 :invoke :cas [23 [23 31]] INFO [2019-12-17 12:33:17,967] jepsen worker 27 - jepsen.util 7047 :ok :read [9 44] INFO [2019-12-17 12:33:17,967] jepsen worker 27 - jepsen.util 7047 :invoke :write [9 18] INFO [2019-12-17 12:33:17,986] jepsen worker 65 - jepsen.util 5735 :ok :write [21 39] INFO [2019-12-17 12:33:17,986] jepsen worker 65 - jepsen.util 5735 :invoke :cas [21 [40 34]] INFO [2019-12-17 12:33:17,988] jepsen worker 72 - jepsen.util 6642 :ok :write [24 26] INFO [2019-12-17 12:33:17,988] jepsen worker 72 - jepsen.util 6642 :invoke :cas [24 [44 25]] INFO [2019-12-17 12:33:18,006] jepsen worker 14 - jepsen.util 7124 :ok :read [4 44] INFO [2019-12-17 12:33:18,006] jepsen worker 14 - jepsen.util 7124 :invoke :cas [4 [7 37]] INFO [2019-12-17 12:33:18,008] jepsen worker 35 - jepsen.util 9035 :info :cas [11 [48 15]] :RequestCanceledException INFO [2019-12-17 12:33:18,009] jepsen worker 35 - jepsen.util 9125 :invoke :write [11 27] INFO [2019-12-17 12:33:18,015] jepsen worker 65 - jepsen.util 5735 :fail :cas [21 [40 34]] :ValueNotSwapFrom INFO [2019-12-17 12:33:18,015] jepsen worker 65 - jepsen.util 5735 :invoke :write [21 6] INFO [2019-12-17 12:33:18,018] jepsen worker 55 - jepsen.util 7975 :fail :cas [18 [40 40]] :ValueNotSwapFrom INFO [2019-12-17 12:33:18,019] jepsen worker 55 - jepsen.util 7975 :invoke :cas [18 [40 49]] INFO [2019-12-17 12:33:18,028] jepsen worker 78 - jepsen.util 7098 :fail :cas [26 [31 32]] :ValueNotSwapFrom INFO [2019-12-17 12:33:18,028] jepsen worker 78 - jepsen.util 7098 :invoke :cas [26 [27 45]] INFO [2019-12-17 12:33:18,029] jepsen worker 72 - jepsen.util 6642 :fail :cas [24 [44 25]] :ValueNotSwapFrom INFO [2019-12-17 12:33:18,029] jepsen worker 72 - jepsen.util 6642 :invoke :cas [24 [0 39]] INFO [2019-12-17 12:33:18,041] jepsen worker 80 - jepsen.util 6830 :info :write [26 29] :RequestCanceledException INFO [2019-12-17 12:33:18,042] jepsen worker 80 - jepsen.util 6920 :invoke :read [26 nil] INFO [2019-12-17 12:33:18,050] jepsen worker 4 - jepsen.util 6214 :fail :cas [1 [8 11]] :ValueNotSwapFrom INFO [2019-12-17 12:33:18,050] jepsen worker 4 - jepsen.util 6214 :invoke :cas [1 [8 13]] INFO [2019-12-17 12:33:18,076] jepsen worker 72 - jepsen.util 6642 :fail :cas [24 [0 39]] :ValueNotSwapFrom INFO [2019-12-17 12:33:18,076] jepsen worker 72 - jepsen.util 6642 :invoke :write [24 47] INFO [2019-12-17 12:33:18,091] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40003,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:33:18,091] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40003,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:33:18,094] jepsen worker 2 - jepsen.util 9002 :info :write [0 35] :RequestCanceledException INFO [2019-12-17 12:33:18,095] jepsen worker 2 - jepsen.util 9092 :invoke :read [0 nil] INFO [2019-12-17 12:33:18,126] jepsen worker 89 - jepsen.util 6749 :fail :cas [29 [3 12]] :ValueNotSwapFrom INFO [2019-12-17 12:33:18,126] jepsen worker 27 - jepsen.util 7047 :ok :write [9 18] INFO [2019-12-17 12:33:18,126] jepsen worker 89 - jepsen.util 6749 :invoke :cas [29 [37 35]] INFO [2019-12-17 12:33:18,126] jepsen worker 27 - jepsen.util 7047 :invoke :write [9 13] INFO [2019-12-17 12:33:18,126] jepsen worker 18 - jepsen.util 6858 :ok :write [6 32] INFO [2019-12-17 12:33:18,126] jepsen worker 18 - jepsen.util 6858 :invoke :read [6 nil] INFO [2019-12-17 12:33:18,131] jepsen worker 65 - jepsen.util 5735 :ok :write [21 6] INFO [2019-12-17 12:33:18,131] jepsen worker 65 - jepsen.util 5735 :invoke :read [21 nil] INFO [2019-12-17 12:33:18,157] jepsen worker 64 - jepsen.util 5374 :fail :read [21 nil] :RequestTimeoutException INFO [2019-12-17 12:33:18,158] jepsen worker 64 - jepsen.util 5374 :invoke :write [21 3] INFO [2019-12-17 12:33:18,165] jepsen worker 52 - jepsen.util 7252 :fail :cas [17 [35 37]] :ValueNotSwapFrom INFO [2019-12-17 12:33:18,165] jepsen worker 52 - jepsen.util 7252 :invoke :cas [17 [30 12]] INFO [2019-12-17 12:33:18,188] jepsen worker 18 - jepsen.util 6858 :ok :read [6 32] INFO [2019-12-17 12:33:18,188] jepsen worker 18 - jepsen.util 6858 :invoke :read [6 nil] INFO [2019-12-17 12:33:18,199] jepsen worker 65 - jepsen.util 5735 :ok :read [21 6] INFO [2019-12-17 12:33:18,200] jepsen worker 65 - jepsen.util 5735 :invoke :read [21 nil] INFO [2019-12-17 12:33:18,201] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40025,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:33:18,201] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40025,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:33:18,201] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40026,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:33:18,202] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40026,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:33:18,202] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40027,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:33:18,202] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40027,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:33:18,203] jepsen worker 70 - jepsen.util 7180 :fail :cas [23 [23 31]] :ValueNotSwapFrom INFO [2019-12-17 12:33:18,203] jepsen worker 70 - jepsen.util 7180 :invoke :cas [23 [19 18]] INFO [2019-12-17 12:33:18,208] jepsen worker 51 - jepsen.util 6711 :fail :cas [17 [28 10]] :ValueNotSwapFrom INFO [2019-12-17 12:33:18,208] jepsen worker 51 - jepsen.util 6711 :invoke :write [17 15] INFO [2019-12-17 12:33:18,228] jepsen worker 29 - jepsen.util 5159 :info :write [9 39] :RequestCanceledException INFO [2019-12-17 12:33:18,229] jepsen worker 29 - jepsen.util 5249 :invoke :write [9 24] INFO [2019-12-17 12:33:18,240] jepsen worker 79 - jepsen.util 6919 :fail :cas [26 [17 17]] :ValueNotSwapFrom INFO [2019-12-17 12:33:18,240] jepsen worker 79 - jepsen.util 6919 :invoke :read [26 nil] INFO [2019-12-17 12:33:18,252] jepsen worker 18 - jepsen.util 6858 :ok :read [6 32] INFO [2019-12-17 12:33:18,252] jepsen worker 18 - jepsen.util 6858 :invoke :read [6 nil] INFO [2019-12-17 12:33:18,256] jepsen worker 65 - jepsen.util 5735 :ok :read [21 6] INFO [2019-12-17 12:33:18,256] jepsen worker 65 - jepsen.util 5735 :invoke :write [21 9] INFO [2019-12-17 12:33:18,281] jepsen worker 14 - jepsen.util 7124 :fail :cas [4 [7 37]] :ValueNotSwapFrom INFO [2019-12-17 12:33:18,282] jepsen worker 14 - jepsen.util 7124 :invoke :cas [4 [40 15]] INFO [2019-12-17 12:33:18,306] jepsen worker 4 - jepsen.util 6214 :fail :cas [1 [8 13]] :ValueNotSwapFrom INFO [2019-12-17 12:33:18,307] jepsen worker 4 - jepsen.util 6214 :invoke :read [1 nil] INFO [2019-12-17 12:33:18,311] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40034,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:33:18,311] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40034,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:33:18,311] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40038,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:33:18,311] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40038,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:33:18,313] jepsen worker 45 - jepsen.util 6795 :info :write [15 6] :RequestCanceledException INFO [2019-12-17 12:33:18,313] jepsen worker 31 - jepsen.util 6151 :info :cas [10 [21 11]] :RequestCanceledException INFO [2019-12-17 12:33:18,313] jepsen worker 73 - jepsen.util 6553 :fail :read [24 nil] :RequestTimeoutException INFO [2019-12-17 12:33:18,313] jepsen worker 73 - jepsen.util 6553 :invoke :write [24 9] INFO [2019-12-17 12:33:18,315] jepsen worker 31 - jepsen.util 6241 :invoke :read [10 nil] INFO [2019-12-17 12:33:18,315] jepsen worker 45 - jepsen.util 6885 :invoke :write [15 43] INFO [2019-12-17 12:33:18,328] jepsen worker 20 - jepsen.util 6590 :info :cas [6 [14 9]] :RequestCanceledException INFO [2019-12-17 12:33:18,332] jepsen worker 20 - jepsen.util 6680 :invoke :write [6 21] INFO [2019-12-17 12:33:18,366] jepsen worker 78 - jepsen.util 7098 :fail :cas [26 [27 45]] :ValueNotSwapFrom INFO [2019-12-17 12:33:18,366] jepsen worker 78 - jepsen.util 7098 :invoke :read [26 nil] INFO [2019-12-17 12:33:18,376] jepsen worker 71 - jepsen.util 7631 :fail :cas [23 [48 39]] :ValueNotSwapFrom INFO [2019-12-17 12:33:18,376] jepsen worker 71 - jepsen.util 7631 :invoke :write [23 49] INFO [2019-12-17 12:33:18,377] jepsen worker 27 - jepsen.util 7047 :ok :write [9 13] INFO [2019-12-17 12:33:18,377] jepsen worker 27 - jepsen.util 7047 :invoke :read [9 nil] INFO [2019-12-17 12:33:18,384] jepsen worker 72 - jepsen.util 6642 :ok :write [24 47] INFO [2019-12-17 12:33:18,384] jepsen worker 72 - jepsen.util 6642 :invoke :write [24 49] INFO [2019-12-17 12:33:18,389] jepsen worker 80 - jepsen.util 6920 :ok :read [26 1] INFO [2019-12-17 12:33:18,389] jepsen worker 80 - jepsen.util 6920 :invoke :cas [26 [43 4]] INFO [2019-12-17 12:33:18,389] jepsen worker 12 - jepsen.util 7212 :info :write [4 40] :RequestCanceledException INFO [2019-12-17 12:33:18,389] jepsen worker 28 - jepsen.util 5338 :info :cas [9 [42 35]] :RequestCanceledException INFO [2019-12-17 12:33:18,391] jepsen worker 18 - jepsen.util 6858 :ok :read [6 32] INFO [2019-12-17 12:33:18,391] jepsen worker 18 - jepsen.util 6858 :invoke :cas [6 [39 7]] INFO [2019-12-17 12:33:18,392] jepsen worker 12 - jepsen.util 7302 :invoke :write [4 1] INFO [2019-12-17 12:33:18,392] jepsen worker 28 - jepsen.util 5428 :invoke :write [9 43] INFO [2019-12-17 12:33:18,402] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:33:18,402] jepsen nemesis - couchbase.util Rebalance check count: 306 INFO [2019-12-17 12:33:18,403] jepsen nemesis - couchbase.util Rebalance stuck count: 287 INFO [2019-12-17 12:33:18,412] jepsen worker 32 - jepsen.util 6872 :info :cas [10 [40 20]] :RequestCanceledException INFO [2019-12-17 12:33:18,413] jepsen worker 32 - jepsen.util 6962 :invoke :cas [10 [16 28]] INFO [2019-12-17 12:33:18,421] jepsen worker 64 - jepsen.util 5374 :ok :write [21 3] INFO [2019-12-17 12:33:18,421] jepsen worker 64 - jepsen.util 5374 :invoke :cas [21 [32 45]] INFO [2019-12-17 12:33:18,427] jepsen worker 13 - jepsen.util 6493 :info :write [4 44] :RequestCanceledException INFO [2019-12-17 12:33:18,429] jepsen worker 13 - jepsen.util 6583 :invoke :write [4 37] INFO [2019-12-17 12:33:18,442] jepsen worker 78 - jepsen.util 7098 :ok :read [26 1] INFO [2019-12-17 12:33:18,442] jepsen worker 78 - jepsen.util 7098 :invoke :read [26 nil] INFO [2019-12-17 12:33:18,470] jepsen worker 80 - jepsen.util 6920 :fail :cas [26 [43 4]] :ValueNotSwapFrom INFO [2019-12-17 12:33:18,470] jepsen worker 80 - jepsen.util 6920 :invoke :cas [26 [32 47]] INFO [2019-12-17 12:33:18,492] jepsen worker 32 - jepsen.util 6962 :fail :cas [10 [16 28]] :ValueNotSwapFrom INFO [2019-12-17 12:33:18,492] jepsen worker 32 - jepsen.util 6962 :invoke :write [10 21] INFO [2019-12-17 12:33:18,496] jepsen worker 36 - jepsen.util 7326 :ok :write [12 28] INFO [2019-12-17 12:33:18,496] jepsen worker 36 - jepsen.util 7326 :invoke :write [12 12] INFO [2019-12-17 12:33:18,507] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40061,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:33:18,507] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40061,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:33:18,507] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40064,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:33:18,508] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40064,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:33:18,508] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40065,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:33:18,508] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40065,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:33:18,530] jepsen worker 31 - jepsen.util 6241 :ok :read [10 1] INFO [2019-12-17 12:33:18,530] jepsen worker 31 - jepsen.util 6241 :invoke :write [10 6] INFO [2019-12-17 12:33:18,557] jepsen worker 5 - jepsen.util 6575 :fail :cas [1 [29 2]] :ValueNotSwapFrom INFO [2019-12-17 12:33:18,557] jepsen worker 5 - jepsen.util 6575 :invoke :cas [1 [18 4]] INFO [2019-12-17 12:33:18,570] jepsen worker 88 - jepsen.util 6928 :ok :write [29 2] INFO [2019-12-17 12:33:18,570] jepsen worker 88 - jepsen.util 6928 :invoke :cas [29 [31 14]] INFO [2019-12-17 12:33:18,605] jepsen worker 64 - jepsen.util 5374 :fail :cas [21 [32 45]] :ValueNotSwapFrom INFO [2019-12-17 12:33:18,605] jepsen worker 64 - jepsen.util 5374 :invoke :write [21 21] INFO [2019-12-17 12:33:18,628] jepsen worker 63 - jepsen.util 5913 :info :write [21 47] :RequestCanceledException INFO [2019-12-17 12:33:18,628] jepsen worker 63 - jepsen.util 6003 :invoke :cas [21 [30 27]] INFO [2019-12-17 12:33:18,641] jepsen worker 56 - jepsen.util 7706 :ok :write [18 27] INFO [2019-12-17 12:33:18,641] jepsen worker 56 - jepsen.util 7706 :invoke :write [18 11] INFO [2019-12-17 12:33:18,642] jepsen worker 69 - jepsen.util 6999 :ok :write [23 41] INFO [2019-12-17 12:33:18,642] jepsen worker 69 - jepsen.util 6999 :invoke :write [23 14] INFO [2019-12-17 12:33:18,655] jepsen worker 19 - jepsen.util 6859 :info :write [6 34] :RequestCanceledException INFO [2019-12-17 12:33:18,655] jepsen worker 74 - jepsen.util 7094 :fail :read [24 nil] :RequestTimeoutException INFO [2019-12-17 12:33:18,655] jepsen worker 74 - jepsen.util 7094 :invoke :cas [24 [42 6]] INFO [2019-12-17 12:33:18,658] jepsen worker 19 - jepsen.util 6949 :invoke :write [6 43] INFO [2019-12-17 12:33:18,659] jepsen worker 63 - jepsen.util 6003 :fail :cas [21 [30 27]] :ValueNotSwapFrom INFO [2019-12-17 12:33:18,659] jepsen worker 63 - jepsen.util 6003 :invoke :cas [21 [41 8]] INFO [2019-12-17 12:33:18,664] jepsen worker 12 - jepsen.util 7302 :ok :write [4 1] INFO [2019-12-17 12:33:18,664] jepsen worker 12 - jepsen.util 7302 :invoke :write [4 25] INFO [2019-12-17 12:33:18,693] jepsen worker 64 - jepsen.util 5374 :ok :write [21 21] INFO [2019-12-17 12:33:18,693] jepsen worker 64 - jepsen.util 5374 :invoke :write [21 36] INFO [2019-12-17 12:33:18,701] jepsen worker 70 - jepsen.util 7180 :fail :cas [23 [19 18]] :ValueNotSwapFrom INFO [2019-12-17 12:33:18,701] jepsen worker 70 - jepsen.util 7180 :invoke :write [23 14] INFO [2019-12-17 12:33:18,739] jepsen worker 74 - jepsen.util 7094 :fail :cas [24 [42 6]] :ValueNotSwapFrom INFO [2019-12-17 12:33:18,739] jepsen worker 74 - jepsen.util 7094 :invoke :write [24 28] INFO [2019-12-17 12:33:18,747] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40077,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:33:18,747] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40077,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:33:18,748] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40081,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:33:18,748] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40081,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:33:18,759] jepsen worker 14 - jepsen.util 7124 :fail :cas [4 [40 15]] :ValueNotSwapFrom INFO [2019-12-17 12:33:18,760] jepsen worker 14 - jepsen.util 7124 :invoke :cas [4 [1 32]] INFO [2019-12-17 12:33:18,760] jepsen worker 63 - jepsen.util 6003 :fail :cas [21 [41 8]] :ValueNotSwapFrom INFO [2019-12-17 12:33:18,760] jepsen worker 63 - jepsen.util 6003 :invoke :write [21 49] INFO [2019-12-17 12:33:18,762] jepsen worker 73 - jepsen.util 6553 :ok :write [24 9] INFO [2019-12-17 12:33:18,762] jepsen worker 73 - jepsen.util 6553 :invoke :write [24 30] INFO [2019-12-17 12:33:18,764] jepsen worker 20 - jepsen.util 6680 :ok :write [6 21] INFO [2019-12-17 12:33:18,764] jepsen worker 20 - jepsen.util 6680 :invoke :cas [6 [44 41]] INFO [2019-12-17 12:33:18,790] jepsen worker 46 - jepsen.util 7066 :info :write [15 1] :RequestCanceledException INFO [2019-12-17 12:33:18,791] jepsen worker 52 - jepsen.util 7252 :fail :cas [17 [30 12]] :ValueNotSwapFrom INFO [2019-12-17 12:33:18,791] jepsen worker 52 - jepsen.util 7252 :invoke :read [17 nil] INFO [2019-12-17 12:33:18,792] jepsen worker 46 - jepsen.util 7156 :invoke :write [15 8] INFO [2019-12-17 12:33:18,814] jepsen worker 18 - jepsen.util 6858 :fail :cas [6 [39 7]] :ValueNotSwapFrom INFO [2019-12-17 12:33:18,814] jepsen worker 18 - jepsen.util 6858 :invoke :write [6 12] INFO [2019-12-17 12:33:18,825] jepsen worker 89 - jepsen.util 6749 :fail :cas [29 [37 35]] :ValueNotSwapFrom INFO [2019-12-17 12:33:18,825] jepsen worker 89 - jepsen.util 6749 :invoke :write [29 4] INFO [2019-12-17 12:33:18,839] jepsen worker 78 - jepsen.util 7098 :ok :read [26 1] INFO [2019-12-17 12:33:18,839] jepsen worker 78 - jepsen.util 7098 :invoke :read [26 nil] INFO [2019-12-17 12:33:18,856] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40102,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:33:18,856] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40102,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:33:18,857] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40104,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:33:18,857] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40104,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:33:18,857] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40105,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:33:18,857] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40105,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:33:18,876] jepsen worker 4 - jepsen.util 6214 :ok :read [1 35] INFO [2019-12-17 12:33:18,876] jepsen worker 4 - jepsen.util 6214 :invoke :write [1 16] INFO [2019-12-17 12:33:18,883] jepsen worker 64 - jepsen.util 5374 :ok :write [21 36] INFO [2019-12-17 12:33:18,884] jepsen worker 64 - jepsen.util 5374 :invoke :write [21 33] INFO [2019-12-17 12:33:18,892] jepsen worker 47 - jepsen.util 7337 :info :cas [15 [47 38]] :RequestCanceledException INFO [2019-12-17 12:33:18,893] jepsen worker 47 - jepsen.util 7427 :invoke :cas [15 [24 14]] INFO [2019-12-17 12:33:18,896] jepsen worker 79 - jepsen.util 6919 :ok :read [26 1] INFO [2019-12-17 12:33:18,896] jepsen worker 79 - jepsen.util 6919 :invoke :read [26 nil] INFO [2019-12-17 12:33:18,912] jepsen worker 34 - jepsen.util 8854 :info :write [11 22] :RequestCanceledException INFO [2019-12-17 12:33:18,912] jepsen worker 5 - jepsen.util 6575 :fail :cas [1 [18 4]] :ValueNotSwapFrom INFO [2019-12-17 12:33:18,912] jepsen worker 5 - jepsen.util 6575 :invoke :read [1 nil] INFO [2019-12-17 12:33:18,914] jepsen worker 34 - jepsen.util 8944 :invoke :write [11 41] INFO [2019-12-17 12:33:18,928] jepsen worker 58 - jepsen.util 10048 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:33:18,928] jepsen worker 58 - jepsen.util 10048 :invoke :cas [19 [7 48]] INFO [2019-12-17 12:33:18,937] jepsen worker 47 - jepsen.util 7427 :fail :cas [15 [24 14]] :ValueNotSwapFrom INFO [2019-12-17 12:33:18,937] jepsen worker 47 - jepsen.util 7427 :invoke :write [15 21] INFO [2019-12-17 12:33:18,941] jepsen worker 23 - jepsen.util 9473 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:33:18,942] jepsen worker 23 - jepsen.util 9473 :invoke :write [7 3] INFO [2019-12-17 12:33:18,958] jepsen worker 75 - jepsen.util 8895 :info :cas [25 [31 19]] :RequestCanceledException INFO [2019-12-17 12:33:18,961] jepsen worker 75 - jepsen.util 8985 :invoke :cas [25 [5 47]] INFO [2019-12-17 12:33:18,969] jepsen worker 88 - jepsen.util 6928 :fail :cas [29 [31 14]] :ValueNotSwapFrom INFO [2019-12-17 12:33:18,969] jepsen worker 88 - jepsen.util 6928 :invoke :write [29 0] INFO [2019-12-17 12:33:18,986] jepsen worker 37 - jepsen.util 6517 :ok :read [12 28] INFO [2019-12-17 12:33:18,987] jepsen worker 37 - jepsen.util 6517 :invoke :write [12 3] INFO [2019-12-17 12:33:18,994] jepsen worker 62 - jepsen.util 9422 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:33:18,995] jepsen worker 62 - jepsen.util 9422 :invoke :read [20 nil] INFO [2019-12-17 12:33:19,002] jepsen worker 63 - jepsen.util 6003 :ok :write [21 49] INFO [2019-12-17 12:33:19,003] jepsen worker 63 - jepsen.util 6003 :invoke :read [21 nil] INFO [2019-12-17 12:33:19,013] jepsen worker 21 - jepsen.util 8841 :info :cas [7 [37 25]] :RequestCanceledException INFO [2019-12-17 12:33:19,013] jepsen worker 43 - jepsen.util 8683 :info :cas [14 [26 15]] :RequestCanceledException INFO [2019-12-17 12:33:19,015] jepsen worker 43 - jepsen.util 8773 :invoke :cas [14 [6 26]] INFO [2019-12-17 12:33:19,015] jepsen worker 21 - jepsen.util 8931 :invoke :cas [7 [37 34]] INFO [2019-12-17 12:33:19,027] jepsen worker 42 - jepsen.util 9942 :info :cas [14 [13 21]] :RequestCanceledException INFO [2019-12-17 12:33:19,028] jepsen worker 25 - jepsen.util 9835 :info :write [8 33] :RequestCanceledException INFO [2019-12-17 12:33:19,028] jepsen worker 33 - jepsen.util 9213 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:33:19,028] jepsen worker 33 - jepsen.util 9213 :invoke :cas [11 [46 21]] INFO [2019-12-17 12:33:19,030] jepsen worker 42 - jepsen.util 10032 :invoke :cas [14 [20 20]] INFO [2019-12-17 12:33:19,030] jepsen worker 25 - jepsen.util 9925 :invoke :read [8 nil] INFO [2019-12-17 12:33:19,042] jepsen worker 0 - jepsen.util 9720 :info :cas [0 [42 9]] :RequestCanceledException INFO [2019-12-17 12:33:19,042] jepsen worker 68 - jepsen.util 8888 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:33:19,042] jepsen worker 1 - jepsen.util 9451 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:33:19,042] jepsen worker 1 - jepsen.util 9451 :invoke :write [0 13] INFO [2019-12-17 12:33:19,042] jepsen worker 68 - jepsen.util 8888 :invoke :write [22 4] INFO [2019-12-17 12:33:19,045] jepsen worker 0 - jepsen.util 9810 :invoke :cas [0 [3 33]] INFO [2019-12-17 12:33:19,047] jepsen worker 63 - jepsen.util 6003 :ok :read [21 49] INFO [2019-12-17 12:33:19,047] jepsen worker 63 - jepsen.util 6003 :invoke :cas [21 [18 38]] INFO [2019-12-17 12:33:19,059] jepsen worker 24 - jepsen.util 8214 :info :cas [8 [23 7]] :RequestCanceledException INFO [2019-12-17 12:33:19,059] jepsen worker 77 - jepsen.util 9617 :info :write [25 45] :RequestCanceledException INFO [2019-12-17 12:33:19,059] jepsen worker 41 - jepsen.util 9941 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:33:19,059] jepsen worker 66 - jepsen.util 9156 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:33:19,060] jepsen worker 41 - jepsen.util 9941 :invoke :write [13 7] INFO [2019-12-17 12:33:19,060] jepsen worker 66 - jepsen.util 9156 :invoke :write [22 44] INFO [2019-12-17 12:33:19,062] jepsen worker 24 - jepsen.util 8304 :invoke :read [8 nil] INFO [2019-12-17 12:33:19,063] jepsen worker 77 - jepsen.util 9707 :invoke :write [25 37] INFO [2019-12-17 12:33:19,075] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40119,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:33:19,076] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40119,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:33:19,076] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40121,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:33:19,076] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40121,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:33:19,076] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40124,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:33:19,076] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40124,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:33:19,082] jepsen worker 63 - jepsen.util 6003 :fail :cas [21 [18 38]] :ValueNotSwapFrom INFO [2019-12-17 12:33:19,082] jepsen worker 63 - jepsen.util 6003 :invoke :write [21 44] INFO [2019-12-17 12:33:19,092] jepsen worker 80 - jepsen.util 6920 :fail :cas [26 [32 47]] :ValueNotSwapFrom INFO [2019-12-17 12:33:19,092] jepsen worker 80 - jepsen.util 6920 :invoke :cas [26 [12 19]] INFO [2019-12-17 12:33:19,093] jepsen worker 32 - jepsen.util 6962 :ok :write [10 21] INFO [2019-12-17 12:33:19,093] jepsen worker 32 - jepsen.util 6962 :invoke :cas [10 [5 13]] INFO [2019-12-17 12:33:19,093] jepsen worker 19 - jepsen.util 6949 :ok :write [6 43] INFO [2019-12-17 12:33:19,093] jepsen worker 19 - jepsen.util 6949 :invoke :write [6 1] INFO [2019-12-17 12:33:19,095] jepsen worker 60 - jepsen.util 10230 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:33:19,095] jepsen worker 22 - jepsen.util 9292 :info :cas [7 [27 15]] :RequestCanceledException INFO [2019-12-17 12:33:19,095] jepsen worker 30 - jepsen.util 6780 :fail :read [10 nil] :RequestTimeoutException INFO [2019-12-17 12:33:19,095] jepsen worker 30 - jepsen.util 6780 :invoke :read [10 nil] INFO [2019-12-17 12:33:19,095] jepsen worker 60 - jepsen.util 10230 :invoke :write [20 19] INFO [2019-12-17 12:33:19,099] jepsen worker 22 - jepsen.util 9382 :invoke :cas [7 [11 45]] INFO [2019-12-17 12:33:19,113] jepsen worker 15 - jepsen.util 8385 :info :write [5 1] :RequestCanceledException INFO [2019-12-17 12:33:19,113] jepsen worker 50 - jepsen.util 9770 :info :cas [16 [40 49]] :RequestCanceledException INFO [2019-12-17 12:33:19,115] jepsen worker 50 - jepsen.util 9860 :invoke :write [16 28] INFO [2019-12-17 12:33:19,116] jepsen worker 15 - jepsen.util 8475 :invoke :write [5 1] INFO [2019-12-17 12:33:19,128] jepsen worker 26 - jepsen.util 9566 :info :cas [8 [43 37]] :RequestCanceledException INFO [2019-12-17 12:33:19,128] jepsen worker 59 - jepsen.util 9239 :info :write [19 30] :RequestCanceledException INFO [2019-12-17 12:33:19,128] jepsen worker 67 - jepsen.util 9517 :info :write [22 46] :RequestCanceledException INFO [2019-12-17 12:33:19,131] jepsen worker 26 - jepsen.util 9656 :invoke :read [8 nil] INFO [2019-12-17 12:33:19,131] jepsen worker 67 - jepsen.util 9607 :invoke :write [22 48] INFO [2019-12-17 12:33:19,132] jepsen worker 59 - jepsen.util 9329 :invoke :cas [19 [9 43]] INFO [2019-12-17 12:33:19,142] jepsen worker 83 - jepsen.util 9803 :info :cas [27 [18 37]] :RequestCanceledException INFO [2019-12-17 12:33:19,144] jepsen worker 83 - jepsen.util 9893 :invoke :cas [27 [12 5]] INFO [2019-12-17 12:33:19,154] jepsen worker 52 - jepsen.util 7252 :ok :read [17 0] INFO [2019-12-17 12:33:19,154] jepsen worker 52 - jepsen.util 7252 :invoke :cas [17 [41 0]] INFO [2019-12-17 12:33:19,159] jepsen worker 86 - jepsen.util 9806 :info :cas [28 [10 43]] :RequestCanceledException INFO [2019-12-17 12:33:19,159] jepsen worker 82 - jepsen.util 9982 :info :write [27 26] :RequestCanceledException INFO [2019-12-17 12:33:19,165] jepsen worker 86 - jepsen.util 9896 :invoke :read [28 nil] INFO [2019-12-17 12:33:19,165] jepsen worker 82 - jepsen.util 10072 :invoke :cas [27 [13 20]] INFO [2019-12-17 12:33:19,191] jepsen worker 78 - jepsen.util 7098 :ok :read [26 1] INFO [2019-12-17 12:33:19,191] jepsen worker 78 - jepsen.util 7098 :invoke :read [26 nil] INFO [2019-12-17 12:33:19,195] jepsen worker 30 - jepsen.util 6780 :ok :read [10 21] INFO [2019-12-17 12:33:19,195] jepsen worker 30 - jepsen.util 6780 :invoke :write [10 8] INFO [2019-12-17 12:33:19,206] jepsen worker 32 - jepsen.util 6962 :fail :cas [10 [5 13]] :ValueNotSwapFrom INFO [2019-12-17 12:33:19,206] jepsen worker 32 - jepsen.util 6962 :invoke :read [10 nil] INFO [2019-12-17 12:33:19,242] jepsen worker 3 - jepsen.util 6843 :info :write [1 15] :RequestCanceledException INFO [2019-12-17 12:33:19,246] jepsen worker 3 - jepsen.util 6933 :invoke :read [1 nil] INFO [2019-12-17 12:33:19,247] jepsen worker 79 - jepsen.util 6919 :ok :read [26 1] INFO [2019-12-17 12:33:19,247] jepsen worker 63 - jepsen.util 6003 :ok :write [21 44] INFO [2019-12-17 12:33:19,247] jepsen worker 79 - jepsen.util 6919 :invoke :read [26 nil] INFO [2019-12-17 12:33:19,248] jepsen worker 63 - jepsen.util 6003 :invoke :cas [21 [36 11]] INFO [2019-12-17 12:33:19,248] jepsen worker 32 - jepsen.util 6962 :ok :read [10 21] INFO [2019-12-17 12:33:19,248] jepsen worker 32 - jepsen.util 6962 :invoke :cas [10 [34 47]] INFO [2019-12-17 12:33:19,259] jepsen worker 38 - jepsen.util 6878 :info :write [12 30] :RequestCanceledException INFO [2019-12-17 12:33:19,261] jepsen worker 38 - jepsen.util 6968 :invoke :cas [12 [24 25]] INFO [2019-12-17 12:33:19,290] jepsen worker 32 - jepsen.util 6962 :fail :cas [10 [34 47]] :ValueNotSwapFrom INFO [2019-12-17 12:33:19,290] jepsen worker 32 - jepsen.util 6962 :invoke :write [10 17] INFO [2019-12-17 12:33:19,295] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40157,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:33:19,295] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40157,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:33:19,295] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40158,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:33:19,295] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40167,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:33:19,295] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40158,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:33:19,295] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40167,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:33:19,307] jepsen worker 63 - jepsen.util 6003 :fail :cas [21 [36 11]] :ValueNotSwapFrom INFO [2019-12-17 12:33:19,307] jepsen worker 63 - jepsen.util 6003 :invoke :read [21 nil] INFO [2019-12-17 12:33:19,344] jepsen worker 80 - jepsen.util 6920 :fail :cas [26 [12 19]] :ValueNotSwapFrom INFO [2019-12-17 12:33:19,345] jepsen worker 80 - jepsen.util 6920 :invoke :read [26 nil] INFO [2019-12-17 12:33:19,357] jepsen worker 63 - jepsen.util 6003 :ok :read [21 44] INFO [2019-12-17 12:33:19,357] jepsen worker 63 - jepsen.util 6003 :invoke :read [21 nil] INFO [2019-12-17 12:33:19,361] jepsen worker 52 - jepsen.util 7252 :fail :cas [17 [41 0]] :ValueNotSwapFrom INFO [2019-12-17 12:33:19,361] jepsen worker 52 - jepsen.util 7252 :invoke :cas [17 [11 11]] INFO [2019-12-17 12:33:19,391] jepsen worker 63 - jepsen.util 6003 :ok :read [21 44] INFO [2019-12-17 12:33:19,392] jepsen worker 63 - jepsen.util 6003 :invoke :write [21 1] INFO [2019-12-17 12:33:19,406] jepsen worker 5 - jepsen.util 6575 :ok :read [1 35] INFO [2019-12-17 12:33:19,406] jepsen worker 5 - jepsen.util 6575 :invoke :read [1 nil] INFO [2019-12-17 12:33:19,411] jepsen worker 76 - jepsen.util 9706 :info :cas [25 [26 37]] :RequestCanceledException INFO [2019-12-17 12:33:19,414] jepsen worker 78 - jepsen.util 7098 :ok :read [26 1] INFO [2019-12-17 12:33:19,414] jepsen worker 78 - jepsen.util 7098 :invoke :cas [26 [36 28]] INFO [2019-12-17 12:33:19,414] jepsen worker 76 - jepsen.util 9796 :invoke :read [25 nil] INFO [2019-12-17 12:33:19,415] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:33:19,415] jepsen nemesis - couchbase.util Rebalance check count: 307 INFO [2019-12-17 12:33:19,416] jepsen nemesis - couchbase.util Rebalance stuck count: 288 INFO [2019-12-17 12:33:19,427] jepsen worker 9 - jepsen.util 7389 :info :write [3 37] :RequestCanceledException INFO [2019-12-17 12:33:19,429] jepsen worker 9 - jepsen.util 7479 :invoke :write [3 29] INFO [2019-12-17 12:33:19,459] jepsen worker 63 - jepsen.util 6003 :ok :write [21 1] INFO [2019-12-17 12:33:19,459] jepsen worker 63 - jepsen.util 6003 :invoke :cas [21 [9 22]] INFO [2019-12-17 12:33:19,565] jepsen worker 40 - jepsen.util 9940 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:33:19,565] jepsen worker 10 - jepsen.util 6760 :info :write [3 44] :RequestCanceledException INFO [2019-12-17 12:33:19,565] jepsen worker 6 - jepsen.util 9996 :info :write [2 35] :RequestCanceledException INFO [2019-12-17 12:33:19,565] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40186,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:33:19,565] jepsen worker 84 - jepsen.util 8904 :info :cas [28 [33 16]] :RequestCanceledException INFO [2019-12-17 12:33:19,565] jepsen worker 81 - jepsen.util 9171 :info :cas [27 [0 1]] :RequestCanceledException INFO [2019-12-17 12:33:19,565] jepsen worker 17 - jepsen.util 9737 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:33:19,565] jepsen worker 40 - jepsen.util 9940 :invoke :cas [13 [45 37]] INFO [2019-12-17 12:33:19,565] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40186,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:33:19,566] jepsen worker 17 - jepsen.util 9737 :invoke :write [5 4] INFO [2019-12-17 12:33:19,566] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40189,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:33:19,566] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40189,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:33:19,460] jepsen worker 39 - jepsen.util 9309 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:33:19,566] jepsen worker 39 - jepsen.util 9309 :invoke :cas [13 [18 47]] INFO [2019-12-17 12:33:19,566] jepsen worker 6 - jepsen.util 10086 :invoke :cas [2 [28 11]] INFO [2019-12-17 12:33:19,566] jepsen worker 10 - jepsen.util 6850 :invoke :write [3 48] INFO [2019-12-17 12:33:19,566] jepsen worker 84 - jepsen.util 8994 :invoke :cas [28 [0 8]] INFO [2019-12-17 12:33:19,566] jepsen worker 81 - jepsen.util 9261 :invoke :read [27 nil] INFO [2019-12-17 12:33:19,571] jepsen worker 5 - jepsen.util 6575 :ok :read [1 35] INFO [2019-12-17 12:33:19,572] jepsen worker 5 - jepsen.util 6575 :invoke :write [1 40] INFO [2019-12-17 12:33:19,575] jepsen worker 79 - jepsen.util 6919 :ok :read [26 1] INFO [2019-12-17 12:33:19,575] jepsen worker 63 - jepsen.util 6003 :fail :cas [21 [9 22]] :ValueNotSwapFrom INFO [2019-12-17 12:33:19,575] jepsen worker 79 - jepsen.util 6919 :invoke :write [26 48] INFO [2019-12-17 12:33:19,576] jepsen worker 63 - jepsen.util 6003 :invoke :write [21 41] INFO [2019-12-17 12:33:19,612] jepsen worker 63 - jepsen.util 6003 :ok :write [21 41] INFO [2019-12-17 12:33:19,612] jepsen worker 63 - jepsen.util 6003 :invoke :cas [21 [38 11]] INFO [2019-12-17 12:33:19,637] jepsen worker 63 - jepsen.util 6003 :fail :cas [21 [38 11]] :ValueNotSwapFrom INFO [2019-12-17 12:33:19,637] jepsen worker 63 - jepsen.util 6003 :invoke :write [21 35] INFO [2019-12-17 12:33:19,641] jepsen worker 44 - jepsen.util 10124 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:33:19,642] jepsen worker 44 - jepsen.util 10124 :invoke :cas [14 [7 39]] INFO [2019-12-17 12:33:19,644] jepsen worker 80 - jepsen.util 6920 :ok :read [26 1] INFO [2019-12-17 12:33:19,644] jepsen worker 80 - jepsen.util 6920 :invoke :read [26 nil] INFO [2019-12-17 12:33:19,659] jepsen worker 54 - jepsen.util 7704 :info :write [18 48] :RequestCanceledException INFO [2019-12-17 12:33:19,660] jepsen worker 54 - jepsen.util 7794 :invoke :write [18 21] INFO [2019-12-17 12:33:19,675] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40209,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:33:19,675] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40209,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:33:19,675] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40211,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:33:19,675] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40211,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:33:19,679] jepsen worker 63 - jepsen.util 6003 :ok :write [21 35] INFO [2019-12-17 12:33:19,679] jepsen worker 63 - jepsen.util 6003 :invoke :read [21 nil] INFO [2019-12-17 12:33:19,690] jepsen worker 63 - jepsen.util 6003 :ok :read [21 35] INFO [2019-12-17 12:33:19,690] jepsen worker 63 - jepsen.util 6003 :invoke :write [21 6] INFO [2019-12-17 12:33:19,710] jepsen worker 61 - jepsen.util 10321 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:33:19,710] jepsen worker 11 - jepsen.util 7211 :fail :read [3 nil] :RequestTimeoutException INFO [2019-12-17 12:33:19,711] jepsen worker 61 - jepsen.util 10321 :invoke :read [20 nil] INFO [2019-12-17 12:33:19,711] jepsen worker 11 - jepsen.util 7211 :invoke :write [3 42] INFO [2019-12-17 12:33:19,727] jepsen worker 53 - jepsen.util 6533 :info :write [17 33] :RequestCanceledException INFO [2019-12-17 12:33:19,727] jepsen worker 48 - jepsen.util 8598 :info :cas [16 [11 45]] :RequestCanceledException INFO [2019-12-17 12:33:19,727] jepsen worker 57 - jepsen.util 9147 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:33:19,728] jepsen worker 57 - jepsen.util 9147 :invoke :write [19 24] INFO [2019-12-17 12:33:19,728] jepsen worker 48 - jepsen.util 8688 :invoke :read [16 nil] INFO [2019-12-17 12:33:19,728] jepsen worker 53 - jepsen.util 6623 :invoke :cas [17 [48 2]] INFO [2019-12-17 12:33:19,729] jepsen worker 78 - jepsen.util 7098 :fail :cas [26 [36 28]] :ValueNotSwapFrom INFO [2019-12-17 12:33:19,729] jepsen worker 78 - jepsen.util 7098 :invoke :read [26 nil] INFO [2019-12-17 12:33:19,742] jepsen worker 87 - jepsen.util 6117 :info :write [29 12] :RequestCanceledException INFO [2019-12-17 12:33:19,743] jepsen worker 87 - jepsen.util 6207 :invoke :read [29 nil] INFO [2019-12-17 12:33:19,743] jepsen worker 63 - jepsen.util 6003 :ok :write [21 6] INFO [2019-12-17 12:33:19,744] jepsen worker 63 - jepsen.util 6003 :invoke :cas [21 [41 45]] INFO [2019-12-17 12:33:19,756] jepsen worker 63 - jepsen.util 6003 :fail :cas [21 [41 45]] :ValueNotSwapFrom INFO [2019-12-17 12:33:19,756] jepsen worker 63 - jepsen.util 6003 :invoke :cas [21 [35 10]] INFO [2019-12-17 12:33:19,772] jepsen worker 63 - jepsen.util 6003 :fail :cas [21 [35 10]] :ValueNotSwapFrom INFO [2019-12-17 12:33:19,772] jepsen worker 63 - jepsen.util 6003 :invoke :write [21 25] INFO [2019-12-17 12:33:19,794] jepsen worker 85 - jepsen.util 10435 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:33:19,794] jepsen worker 85 - jepsen.util 10435 :invoke :read [28 nil] INFO [2019-12-17 12:33:19,812] jepsen worker 7 - jepsen.util 9637 :info :cas [2 [49 25]] :RequestCanceledException INFO [2019-12-17 12:33:19,813] jepsen worker 7 - jepsen.util 9727 :invoke :write [2 31] INFO [2019-12-17 12:33:19,820] jepsen worker 63 - jepsen.util 6003 :ok :write [21 25] INFO [2019-12-17 12:33:19,820] jepsen worker 63 - jepsen.util 6003 :invoke :cas [21 [5 46]] INFO [2019-12-17 12:33:19,825] jepsen worker 80 - jepsen.util 6920 :ok :read [26 1] INFO [2019-12-17 12:33:19,825] jepsen worker 80 - jepsen.util 6920 :invoke :write [26 29] INFO [2019-12-17 12:33:19,828] jepsen worker 16 - jepsen.util 10546 :info :write [5 21] :RequestCanceledException INFO [2019-12-17 12:33:19,829] jepsen worker 16 - jepsen.util 10636 :invoke :cas [5 [22 20]] INFO [2019-12-17 12:33:19,835] jepsen worker 63 - jepsen.util 6003 :fail :cas [21 [5 46]] :ValueNotSwapFrom INFO [2019-12-17 12:33:19,835] jepsen worker 63 - jepsen.util 6003 :invoke :write [21 24] INFO [2019-12-17 12:33:19,841] jepsen worker 49 - jepsen.util 8689 :info :write [16 44] :RequestCanceledException INFO [2019-12-17 12:33:19,842] jepsen worker 49 - jepsen.util 8779 :invoke :cas [16 [33 7]] INFO [2019-12-17 12:33:19,856] jepsen worker 53 - jepsen.util 6623 :fail :cas [17 [48 2]] :ValueNotSwapFrom INFO [2019-12-17 12:33:19,856] jepsen worker 53 - jepsen.util 6623 :invoke :read [17 nil] INFO [2019-12-17 12:33:19,860] jepsen worker 8 - jepsen.util 9278 :info :cas [2 [31 36]] :RequestCanceledException INFO [2019-12-17 12:33:19,861] jepsen worker 8 - jepsen.util 9368 :invoke :cas [2 [6 23]] INFO [2019-12-17 12:33:19,892] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40236,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:33:19,892] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40236,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:33:19,892] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40239,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:33:19,892] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40239,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:33:19,892] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40240,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:33:19,892] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40240,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:33:19,893] jepsen worker 52 - jepsen.util 7252 :fail :cas [17 [11 11]] :ValueNotSwapFrom INFO [2019-12-17 12:33:19,893] jepsen worker 52 - jepsen.util 7252 :invoke :cas [17 [48 6]] INFO [2019-12-17 12:33:19,894] jepsen worker 63 - jepsen.util 6003 :ok :write [21 24] INFO [2019-12-17 12:33:19,894] jepsen worker 63 - jepsen.util 6003 :invoke :write [21 10] INFO [2019-12-17 12:33:19,903] jepsen worker 38 - jepsen.util 6968 :fail :cas [12 [24 25]] :ValueNotSwapFrom INFO [2019-12-17 12:33:19,903] jepsen worker 38 - jepsen.util 6968 :invoke :write [12 17] INFO [2019-12-17 12:33:19,925] jepsen worker 78 - jepsen.util 7098 :ok :read [26 1] INFO [2019-12-17 12:33:19,926] jepsen worker 78 - jepsen.util 7098 :invoke :write [26 13] INFO [2019-12-17 12:33:19,949] jepsen worker 63 - jepsen.util 6003 :ok :write [21 10] INFO [2019-12-17 12:33:19,949] jepsen worker 63 - jepsen.util 6003 :invoke :read [21 nil] INFO [2019-12-17 12:33:19,963] jepsen worker 63 - jepsen.util 6003 :ok :read [21 10] INFO [2019-12-17 12:33:19,964] jepsen worker 63 - jepsen.util 6003 :invoke :cas [21 [13 31]] INFO [2019-12-17 12:33:19,976] jepsen worker 63 - jepsen.util 6003 :fail :cas [21 [13 31]] :ValueNotSwapFrom INFO [2019-12-17 12:33:19,976] jepsen worker 63 - jepsen.util 6003 :invoke :cas [21 [19 49]] INFO [2019-12-17 12:33:19,987] jepsen worker 63 - jepsen.util 6003 :fail :cas [21 [19 49]] :ValueNotSwapFrom INFO [2019-12-17 12:33:19,987] jepsen worker 63 - jepsen.util 6003 :invoke :cas [21 [42 43]] INFO [2019-12-17 12:33:19,998] jepsen worker 63 - jepsen.util 6003 :fail :cas [21 [42 43]] :ValueNotSwapFrom INFO [2019-12-17 12:33:19,998] jepsen worker 63 - jepsen.util 6003 :invoke :read [21 nil] INFO [2019-12-17 12:33:20,001] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40250,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:33:20,002] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40250,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:33:20,002] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40253,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:33:20,002] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40253,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:33:20,002] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40254,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:33:20,002] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40254,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:33:20,010] jepsen worker 63 - jepsen.util 6003 :ok :read [21 10] INFO [2019-12-17 12:33:20,010] jepsen worker 63 - jepsen.util 6003 :invoke :write [21 38] INFO [2019-12-17 12:33:20,042] jepsen worker 55 - jepsen.util 7975 :info :cas [18 [40 49]] :RequestCanceledException INFO [2019-12-17 12:33:20,043] jepsen worker 55 - jepsen.util 8065 :invoke :write [18 5] INFO [2019-12-17 12:33:20,046] jepsen worker 63 - jepsen.util 6003 :ok :write [21 38] INFO [2019-12-17 12:33:20,046] jepsen worker 63 - jepsen.util 6003 :invoke :write [21 47] INFO [2019-12-17 12:33:20,094] jepsen worker 35 - jepsen.util 9125 :info :write [11 27] :RequestCanceledException INFO [2019-12-17 12:33:20,094] jepsen worker 53 - jepsen.util 6623 :ok :read [17 0] INFO [2019-12-17 12:33:20,094] jepsen worker 53 - jepsen.util 6623 :invoke :read [17 nil] INFO [2019-12-17 12:33:20,095] jepsen worker 35 - jepsen.util 9215 :invoke :cas [11 [5 23]] INFO [2019-12-17 12:33:20,107] jepsen worker 63 - jepsen.util 6003 :ok :write [21 47] INFO [2019-12-17 12:33:20,107] jepsen worker 63 - jepsen.util 6003 :invoke :write [21 8] INFO [2019-12-17 12:33:20,128] jepsen worker 2 - jepsen.util 9092 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:33:20,128] jepsen worker 2 - jepsen.util 9092 :invoke :read [0 nil] INFO [2019-12-17 12:33:20,174] jepsen worker 63 - jepsen.util 6003 :ok :write [21 8] INFO [2019-12-17 12:33:20,174] jepsen worker 63 - jepsen.util 6003 :invoke :read [21 nil] INFO [2019-12-17 12:33:20,197] jepsen worker 63 - jepsen.util 6003 :ok :read [21 8] INFO [2019-12-17 12:33:20,197] jepsen worker 63 - jepsen.util 6003 :invoke :write [21 30] INFO [2019-12-17 12:33:20,220] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40264,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:33:20,220] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40272,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:33:20,220] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40264,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:33:20,220] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40272,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:33:20,220] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40266,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:33:20,220] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40275,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:33:20,220] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40266,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:33:20,220] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40275,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:33:20,220] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40276,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:33:20,220] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40276,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:33:20,228] jepsen worker 51 - jepsen.util 6711 :info :write [17 15] :RequestCanceledException INFO [2019-12-17 12:33:20,229] jepsen worker 51 - jepsen.util 6801 :invoke :write [17 11] INFO [2019-12-17 12:33:20,232] jepsen worker 53 - jepsen.util 6623 :ok :read [17 0] INFO [2019-12-17 12:33:20,232] jepsen worker 53 - jepsen.util 6623 :invoke :cas [17 [40 22]] INFO [2019-12-17 12:33:20,234] jepsen worker 63 - jepsen.util 6003 :ok :write [21 30] INFO [2019-12-17 12:33:20,234] jepsen worker 63 - jepsen.util 6003 :invoke :read [21 nil] INFO [2019-12-17 12:33:20,241] jepsen worker 29 - jepsen.util 5249 :info :write [9 24] :RequestCanceledException INFO [2019-12-17 12:33:20,242] jepsen worker 29 - jepsen.util 5339 :invoke :cas [9 [21 16]] INFO [2019-12-17 12:33:20,245] jepsen worker 63 - jepsen.util 6003 :ok :read [21 30] INFO [2019-12-17 12:33:20,246] jepsen worker 63 - jepsen.util 6003 :invoke :read [21 nil] INFO [2019-12-17 12:33:20,263] jepsen worker 63 - jepsen.util 6003 :ok :read [21 30] INFO [2019-12-17 12:33:20,263] jepsen worker 63 - jepsen.util 6003 :invoke :read [21 nil] INFO [2019-12-17 12:33:20,273] jepsen worker 63 - jepsen.util 6003 :ok :read [21 30] INFO [2019-12-17 12:33:20,273] jepsen worker 63 - jepsen.util 6003 :invoke :cas [21 [21 13]] INFO [2019-12-17 12:33:20,285] jepsen worker 63 - jepsen.util 6003 :fail :cas [21 [21 13]] :ValueNotSwapFrom INFO [2019-12-17 12:33:20,285] jepsen worker 63 - jepsen.util 6003 :invoke :write [21 25] INFO [2019-12-17 12:33:20,318] jepsen worker 63 - jepsen.util 6003 :ok :write [21 25] INFO [2019-12-17 12:33:20,318] jepsen worker 63 - jepsen.util 6003 :invoke :cas [21 [21 16]] INFO [2019-12-17 12:33:20,328] jepsen worker 65 - jepsen.util 5735 :info :write [21 9] :RequestCanceledException INFO [2019-12-17 12:33:20,329] jepsen worker 65 - jepsen.util 5825 :invoke :write [21 49] INFO [2019-12-17 12:33:20,335] jepsen worker 63 - jepsen.util 6003 :fail :cas [21 [21 16]] :ValueNotSwapFrom INFO [2019-12-17 12:33:20,335] jepsen worker 63 - jepsen.util 6003 :invoke :cas [21 [29 30]] INFO [2019-12-17 12:33:20,340] jepsen worker 87 - jepsen.util 6207 :ok :read [29 2] INFO [2019-12-17 12:33:20,340] jepsen worker 87 - jepsen.util 6207 :invoke :read [29 nil] INFO [2019-12-17 12:33:20,365] jepsen worker 65 - jepsen.util 5825 :ok :write [21 49] INFO [2019-12-17 12:33:20,365] jepsen worker 65 - jepsen.util 5825 :invoke :cas [21 [36 18]] INFO [2019-12-17 12:33:20,385] jepsen worker 63 - jepsen.util 6003 :fail :cas [21 [29 30]] :ValueNotSwapFrom INFO [2019-12-17 12:33:20,385] jepsen worker 63 - jepsen.util 6003 :invoke :cas [21 [35 45]] INFO [2019-12-17 12:33:20,395] jepsen worker 27 - jepsen.util 7047 :fail :read [9 nil] :RequestTimeoutException INFO [2019-12-17 12:33:20,395] jepsen worker 27 - jepsen.util 7047 :invoke :cas [9 [23 47]] INFO [2019-12-17 12:33:20,403] jepsen worker 65 - jepsen.util 5825 :fail :cas [21 [36 18]] :ValueNotSwapFrom INFO [2019-12-17 12:33:20,403] jepsen worker 65 - jepsen.util 5825 :invoke :cas [21 [1 25]] INFO [2019-12-17 12:33:20,408] jepsen worker 63 - jepsen.util 6003 :fail :cas [21 [35 45]] :ValueNotSwapFrom INFO [2019-12-17 12:33:20,408] jepsen worker 63 - jepsen.util 6003 :invoke :write [21 42] INFO [2019-12-17 12:33:20,415] jepsen worker 65 - jepsen.util 5825 :fail :cas [21 [1 25]] :ValueNotSwapFrom INFO [2019-12-17 12:33:20,415] jepsen worker 65 - jepsen.util 5825 :invoke :write [21 48] INFO [2019-12-17 12:33:20,428] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:33:20,428] jepsen worker 45 - jepsen.util 6885 :info :write [15 43] :RequestCanceledException INFO [2019-12-17 12:33:20,428] jepsen worker 72 - jepsen.util 6642 :info :write [24 49] :RequestCanceledException INFO [2019-12-17 12:33:20,428] jepsen worker 28 - jepsen.util 5428 :info :write [9 43] :RequestCanceledException INFO [2019-12-17 12:33:20,428] jepsen nemesis - couchbase.util Rebalance check count: 308 INFO [2019-12-17 12:33:20,429] jepsen nemesis - couchbase.util Rebalance stuck count: 289 INFO [2019-12-17 12:33:20,429] jepsen worker 45 - jepsen.util 6975 :invoke :cas [15 [19 22]] INFO [2019-12-17 12:33:20,429] jepsen worker 72 - jepsen.util 6732 :invoke :read [24 nil] INFO [2019-12-17 12:33:20,430] jepsen worker 28 - jepsen.util 5518 :invoke :read [9 nil] INFO [2019-12-17 12:33:20,440] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40297,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:33:20,440] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40297,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:33:20,440] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40302,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:33:20,440] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40302,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:33:20,442] jepsen worker 71 - jepsen.util 7631 :info :write [23 49] :RequestCanceledException INFO [2019-12-17 12:33:20,443] jepsen worker 71 - jepsen.util 7721 :invoke :write [23 17] INFO [2019-12-17 12:33:20,445] jepsen worker 63 - jepsen.util 6003 :ok :write [21 42] INFO [2019-12-17 12:33:20,445] jepsen worker 63 - jepsen.util 6003 :invoke :write [21 36] INFO [2019-12-17 12:33:20,459] jepsen worker 13 - jepsen.util 6583 :info :write [4 37] :RequestCanceledException INFO [2019-12-17 12:33:20,460] jepsen worker 13 - jepsen.util 6673 :invoke :read [4 nil] INFO [2019-12-17 12:33:20,460] jepsen worker 87 - jepsen.util 6207 :ok :read [29 2] INFO [2019-12-17 12:33:20,460] jepsen worker 87 - jepsen.util 6207 :invoke :cas [29 [1 40]] INFO [2019-12-17 12:33:20,478] jepsen worker 65 - jepsen.util 5825 :ok :write [21 48] INFO [2019-12-17 12:33:20,478] jepsen worker 65 - jepsen.util 5825 :invoke :cas [21 [37 41]] INFO [2019-12-17 12:33:20,525] jepsen worker 63 - jepsen.util 6003 :ok :write [21 36] INFO [2019-12-17 12:33:20,525] jepsen worker 63 - jepsen.util 6003 :invoke :write [21 13] INFO [2019-12-17 12:33:20,528] jepsen worker 36 - jepsen.util 7326 :info :write [12 12] :RequestCanceledException INFO [2019-12-17 12:33:20,530] jepsen worker 36 - jepsen.util 7416 :invoke :read [12 nil] INFO [2019-12-17 12:33:20,558] jepsen worker 31 - jepsen.util 6241 :info :write [10 6] :RequestCanceledException INFO [2019-12-17 12:33:20,559] jepsen worker 31 - jepsen.util 6331 :invoke :cas [10 [10 22]] INFO [2019-12-17 12:33:20,562] jepsen worker 3 - jepsen.util 6933 :ok :read [1 35] INFO [2019-12-17 12:33:20,562] jepsen worker 3 - jepsen.util 6933 :invoke :write [1 17] INFO [2019-12-17 12:33:20,571] jepsen worker 87 - jepsen.util 6207 :fail :cas [29 [1 40]] :ValueNotSwapFrom INFO [2019-12-17 12:33:20,571] jepsen worker 87 - jepsen.util 6207 :invoke :write [29 13] INFO [2019-12-17 12:33:20,577] jepsen worker 63 - jepsen.util 6003 :ok :write [21 13] INFO [2019-12-17 12:33:20,577] jepsen worker 63 - jepsen.util 6003 :invoke :write [21 25] INFO [2019-12-17 12:33:20,622] jepsen worker 63 - jepsen.util 6003 :ok :write [21 25] INFO [2019-12-17 12:33:20,623] jepsen worker 63 - jepsen.util 6003 :invoke :cas [21 [21 11]] INFO [2019-12-17 12:33:20,635] jepsen worker 63 - jepsen.util 6003 :fail :cas [21 [21 11]] :ValueNotSwapFrom INFO [2019-12-17 12:33:20,635] jepsen worker 63 - jepsen.util 6003 :invoke :write [21 10] INFO [2019-12-17 12:33:20,654] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40318,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:33:20,654] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40318,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:33:20,654] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40321,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:33:20,654] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40321,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:33:20,654] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40322,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:33:20,654] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40322,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:33:20,668] jepsen worker 53 - jepsen.util 6623 :fail :cas [17 [40 22]] :ValueNotSwapFrom INFO [2019-12-17 12:33:20,669] jepsen worker 53 - jepsen.util 6623 :invoke :write [17 26] INFO [2019-12-17 12:33:20,676] jepsen worker 63 - jepsen.util 6003 :ok :write [21 10] INFO [2019-12-17 12:33:20,676] jepsen worker 63 - jepsen.util 6003 :invoke :write [21 36] INFO [2019-12-17 12:33:20,694] jepsen worker 69 - jepsen.util 6999 :info :write [23 14] :RequestCanceledException INFO [2019-12-17 12:33:20,695] jepsen worker 69 - jepsen.util 7089 :invoke :write [23 44] INFO [2019-12-17 12:33:20,705] jepsen worker 52 - jepsen.util 7252 :fail :cas [17 [48 6]] :ValueNotSwapFrom INFO [2019-12-17 12:33:20,705] jepsen worker 52 - jepsen.util 7252 :invoke :read [17 nil] INFO [2019-12-17 12:33:20,710] jepsen worker 12 - jepsen.util 7302 :info :write [4 25] :RequestCanceledException INFO [2019-12-17 12:33:20,711] jepsen worker 12 - jepsen.util 7392 :invoke :read [4 nil] INFO [2019-12-17 12:33:20,714] jepsen worker 74 - jepsen.util 7094 :info :write [24 28] :SyncWriteAmbiguous INFO [2019-12-17 12:33:20,714] jepsen worker 74 - jepsen.util 7184 :invoke :cas [24 [48 36]] INFO [2019-12-17 12:33:20,737] jepsen worker 64 - jepsen.util 5374 :ok :write [21 33] INFO [2019-12-17 12:33:20,737] jepsen worker 64 - jepsen.util 5374 :invoke :read [21 nil] INFO [2019-12-17 12:33:20,742] jepsen worker 56 - jepsen.util 7706 :info :write [18 11] :RequestCanceledException INFO [2019-12-17 12:33:20,743] jepsen worker 56 - jepsen.util 7796 :invoke :cas [18 [25 28]] INFO [2019-12-17 12:33:20,748] jepsen worker 29 - jepsen.util 5339 :fail :cas [9 [21 16]] :ValueNotSwapFrom INFO [2019-12-17 12:33:20,748] jepsen worker 29 - jepsen.util 5339 :invoke :cas [9 [47 7]] INFO [2019-12-17 12:33:20,768] jepsen worker 64 - jepsen.util 5374 :ok :read [21 33] INFO [2019-12-17 12:33:20,768] jepsen worker 64 - jepsen.util 5374 :invoke :read [21 nil] INFO [2019-12-17 12:33:20,780] jepsen worker 29 - jepsen.util 5339 :fail :cas [9 [47 7]] :ValueNotSwapFrom INFO [2019-12-17 12:33:20,780] jepsen worker 29 - jepsen.util 5339 :invoke :cas [9 [5 13]] INFO [2019-12-17 12:33:20,787] jepsen worker 64 - jepsen.util 5374 :ok :read [21 33] INFO [2019-12-17 12:33:20,787] jepsen worker 64 - jepsen.util 5374 :invoke :write [21 14] INFO [2019-12-17 12:33:20,794] jepsen worker 70 - jepsen.util 7180 :info :write [23 14] :RequestCanceledException INFO [2019-12-17 12:33:20,794] jepsen worker 20 - jepsen.util 6680 :info :cas [6 [44 41]] :RequestCanceledException INFO [2019-12-17 12:33:20,796] jepsen worker 70 - jepsen.util 7270 :invoke :cas [23 [10 49]] INFO [2019-12-17 12:33:20,796] jepsen worker 20 - jepsen.util 6770 :invoke :cas [6 [6 10]] INFO [2019-12-17 12:33:20,796] jepsen worker 29 - jepsen.util 5339 :fail :cas [9 [5 13]] :ValueNotSwapFrom INFO [2019-12-17 12:33:20,796] jepsen worker 29 - jepsen.util 5339 :invoke :cas [9 [19 16]] INFO [2019-12-17 12:33:20,812] jepsen worker 73 - jepsen.util 6553 :info :write [24 30] :RequestCanceledException INFO [2019-12-17 12:33:20,812] jepsen worker 46 - jepsen.util 7156 :info :write [15 8] :RequestCanceledException INFO [2019-12-17 12:33:20,813] jepsen worker 46 - jepsen.util 7246 :invoke :write [15 22] INFO [2019-12-17 12:33:20,813] jepsen worker 73 - jepsen.util 6643 :invoke :read [24 nil] INFO [2019-12-17 12:33:20,836] jepsen worker 64 - jepsen.util 5374 :ok :write [21 14] INFO [2019-12-17 12:33:20,836] jepsen worker 64 - jepsen.util 5374 :invoke :read [21 nil] INFO [2019-12-17 12:33:20,858] jepsen worker 64 - jepsen.util 5374 :ok :read [21 14] INFO [2019-12-17 12:33:20,858] jepsen worker 64 - jepsen.util 5374 :invoke :write [21 5] INFO [2019-12-17 12:33:20,867] jepsen worker 47 - jepsen.util 7427 :info :write [15 21] :SyncWriteAmbiguous INFO [2019-12-17 12:33:20,868] jepsen worker 47 - jepsen.util 7517 :invoke :cas [15 [45 27]] INFO [2019-12-17 12:33:20,873] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40341,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:33:20,873] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40341,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:33:20,873] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40342,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:33:20,873] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40342,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:33:20,896] jepsen worker 28 - jepsen.util 5518 :ok :read [9 13] INFO [2019-12-17 12:33:20,896] jepsen worker 28 - jepsen.util 5518 :invoke :write [9 36] INFO [2019-12-17 12:33:20,910] jepsen worker 18 - jepsen.util 6858 :info :write [6 12] :RequestCanceledException INFO [2019-12-17 12:33:20,911] jepsen worker 18 - jepsen.util 6948 :invoke :write [6 7] INFO [2019-12-17 12:33:20,928] jepsen worker 89 - jepsen.util 6749 :info :write [29 4] :RequestCanceledException INFO [2019-12-17 12:33:20,929] jepsen worker 89 - jepsen.util 6839 :invoke :cas [29 [12 27]] INFO [2019-12-17 12:33:20,932] jepsen worker 73 - jepsen.util 6643 :ok :read [24 9] INFO [2019-12-17 12:33:20,933] jepsen worker 73 - jepsen.util 6643 :invoke :cas [24 [15 34]] INFO [2019-12-17 12:33:20,942] jepsen worker 34 - jepsen.util 8944 :info :write [11 41] :RequestCanceledException INFO [2019-12-17 12:33:20,944] jepsen worker 34 - jepsen.util 9034 :invoke :read [11 nil] INFO [2019-12-17 12:33:20,952] jepsen worker 12 - jepsen.util 7392 :ok :read [4 1] INFO [2019-12-17 12:33:20,952] jepsen worker 12 - jepsen.util 7392 :invoke :cas [4 [30 36]] INFO [2019-12-17 12:33:20,953] jepsen worker 13 - jepsen.util 6673 :ok :read [4 1] INFO [2019-12-17 12:33:20,953] jepsen worker 13 - jepsen.util 6673 :invoke :cas [4 [36 37]] INFO [2019-12-17 12:33:20,967] jepsen worker 64 - jepsen.util 5374 :ok :write [21 5] INFO [2019-12-17 12:33:20,968] jepsen worker 64 - jepsen.util 5374 :invoke :write [21 29] INFO [2019-12-17 12:33:20,978] jepsen worker 28 - jepsen.util 5518 :ok :write [9 36] INFO [2019-12-17 12:33:20,978] jepsen worker 28 - jepsen.util 5518 :invoke :write [9 31] INFO [2019-12-17 12:33:20,994] jepsen worker 4 - jepsen.util 6214 :info :write [1 16] :RequestCanceledException INFO [2019-12-17 12:33:20,995] jepsen worker 4 - jepsen.util 6304 :invoke :write [1 35] INFO [2019-12-17 12:33:21,010] jepsen worker 13 - jepsen.util 6673 :fail :cas [4 [36 37]] :ValueNotSwapFrom INFO [2019-12-17 12:33:21,010] jepsen worker 13 - jepsen.util 6673 :invoke :write [4 48] INFO [2019-12-17 12:33:21,027] jepsen worker 58 - jepsen.util 10048 :info :cas [19 [7 48]] :RequestCanceledException INFO [2019-12-17 12:33:21,027] jepsen worker 43 - jepsen.util 8773 :info :cas [14 [6 26]] :RequestCanceledException INFO [2019-12-17 12:33:21,027] jepsen worker 88 - jepsen.util 6928 :info :write [29 0] :RequestCanceledException INFO [2019-12-17 12:33:21,028] jepsen worker 73 - jepsen.util 6643 :fail :cas [24 [15 34]] :ValueNotSwapFrom INFO [2019-12-17 12:33:21,028] jepsen worker 73 - jepsen.util 6643 :invoke :cas [24 [29 17]] INFO [2019-12-17 12:33:21,028] jepsen worker 88 - jepsen.util 7018 :invoke :cas [29 [0 9]] INFO [2019-12-17 12:33:21,028] jepsen worker 58 - jepsen.util 10138 :invoke :write [19 1] INFO [2019-12-17 12:33:21,029] jepsen worker 43 - jepsen.util 8863 :invoke :cas [14 [35 18]] INFO [2019-12-17 12:33:21,042] jepsen worker 37 - jepsen.util 6517 :info :write [12 3] :RequestCanceledException INFO [2019-12-17 12:33:21,042] jepsen worker 23 - jepsen.util 9473 :info :write [7 3] :RequestCanceledException INFO [2019-12-17 12:33:21,043] jepsen worker 37 - jepsen.util 6607 :invoke :write [12 10] INFO [2019-12-17 12:33:21,043] jepsen worker 23 - jepsen.util 9563 :invoke :cas [7 [38 38]] INFO [2019-12-17 12:33:21,054] jepsen worker 36 - jepsen.util 7416 :ok :read [12 28] INFO [2019-12-17 12:33:21,054] jepsen worker 36 - jepsen.util 7416 :invoke :cas [12 [26 18]] INFO [2019-12-17 12:33:21,059] jepsen worker 75 - jepsen.util 8985 :info :cas [25 [5 47]] :RequestCanceledException INFO [2019-12-17 12:33:21,060] jepsen worker 75 - jepsen.util 9075 :invoke :read [25 nil] INFO [2019-12-17 12:33:21,066] jepsen worker 19 - jepsen.util 6949 :info :write [6 1] :SyncWriteAmbiguous INFO [2019-12-17 12:33:21,067] jepsen worker 19 - jepsen.util 7039 :invoke :write [6 41] INFO [2019-12-17 12:33:21,080] jepsen worker 28 - jepsen.util 5518 :ok :write [9 31] INFO [2019-12-17 12:33:21,080] jepsen worker 28 - jepsen.util 5518 :invoke :read [9 nil] INFO [2019-12-17 12:33:21,093] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40358,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:33:21,093] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40358,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:33:21,093] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40360,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:33:21,093] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40360,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:33:21,094] jepsen worker 62 - jepsen.util 9422 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:33:21,094] jepsen worker 21 - jepsen.util 8931 :info :cas [7 [37 34]] :RequestCanceledException INFO [2019-12-17 12:33:21,094] jepsen worker 62 - jepsen.util 9422 :invoke :write [20 43] INFO [2019-12-17 12:33:21,095] jepsen worker 21 - jepsen.util 9021 :invoke :write [7 26] INFO [2019-12-17 12:33:21,110] jepsen worker 42 - jepsen.util 10032 :info :cas [14 [20 20]] :RequestCanceledException INFO [2019-12-17 12:33:21,112] jepsen worker 42 - jepsen.util 10122 :invoke :write [14 44] INFO [2019-12-17 12:33:21,124] jepsen worker 27 - jepsen.util 7047 :fail :cas [9 [23 47]] :ValueNotSwapFrom INFO [2019-12-17 12:33:21,124] jepsen worker 27 - jepsen.util 7047 :invoke :read [9 nil] INFO [2019-12-17 12:33:21,128] jepsen worker 25 - jepsen.util 9925 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:33:21,128] jepsen worker 33 - jepsen.util 9213 :info :cas [11 [46 21]] :RequestCanceledException INFO [2019-12-17 12:33:21,128] jepsen worker 77 - jepsen.util 9707 :info :write [25 37] :RequestCanceledException INFO [2019-12-17 12:33:21,128] jepsen worker 25 - jepsen.util 9925 :invoke :cas [8 [46 15]] INFO [2019-12-17 12:33:21,129] jepsen worker 77 - jepsen.util 9797 :invoke :read [25 nil] INFO [2019-12-17 12:33:21,129] jepsen worker 33 - jepsen.util 9303 :invoke :write [11 35] INFO [2019-12-17 12:33:21,142] jepsen worker 0 - jepsen.util 9810 :info :cas [0 [3 33]] :RequestCanceledException INFO [2019-12-17 12:33:21,142] jepsen worker 68 - jepsen.util 8888 :info :write [22 4] :RequestCanceledException INFO [2019-12-17 12:33:21,142] jepsen worker 1 - jepsen.util 9451 :info :write [0 13] :RequestCanceledException INFO [2019-12-17 12:33:21,142] jepsen worker 26 - jepsen.util 9656 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:33:21,142] jepsen worker 26 - jepsen.util 9656 :invoke :read [8 nil] INFO [2019-12-17 12:33:21,143] jepsen worker 68 - jepsen.util 8978 :invoke :write [22 42] INFO [2019-12-17 12:33:21,143] jepsen worker 0 - jepsen.util 9900 :invoke :write [0 41] INFO [2019-12-17 12:33:21,143] jepsen worker 1 - jepsen.util 9541 :invoke :cas [0 [36 5]] INFO [2019-12-17 12:33:21,145] jepsen worker 32 - jepsen.util 6962 :info :write [10 17] :SyncWriteAmbiguous INFO [2019-12-17 12:33:21,146] jepsen worker 32 - jepsen.util 7052 :invoke :cas [10 [16 10]] INFO [2019-12-17 12:33:21,149] jepsen worker 45 - jepsen.util 6975 :fail :cas [15 [19 22]] :ValueNotSwapFrom INFO [2019-12-17 12:33:21,149] jepsen worker 45 - jepsen.util 6975 :invoke :cas [15 [1 36]] INFO [2019-12-17 12:33:21,153] jepsen worker 52 - jepsen.util 7252 :ok :read [17 0] INFO [2019-12-17 12:33:21,153] jepsen worker 52 - jepsen.util 7252 :invoke :write [17 20] INFO [2019-12-17 12:33:21,159] jepsen worker 24 - jepsen.util 8304 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:33:21,159] jepsen worker 66 - jepsen.util 9156 :info :write [22 44] :RequestCanceledException INFO [2019-12-17 12:33:21,159] jepsen worker 41 - jepsen.util 9941 :info :write [13 7] :RequestCanceledException INFO [2019-12-17 12:33:21,159] jepsen worker 67 - jepsen.util 9607 :info :write [22 48] :RequestCanceledException INFO [2019-12-17 12:33:21,159] jepsen worker 24 - jepsen.util 8304 :invoke :read [8 nil] INFO [2019-12-17 12:33:21,160] jepsen worker 41 - jepsen.util 10031 :invoke :read [13 nil] INFO [2019-12-17 12:33:21,160] jepsen worker 66 - jepsen.util 9246 :invoke :cas [22 [37 22]] INFO [2019-12-17 12:33:21,160] jepsen worker 67 - jepsen.util 9697 :invoke :write [22 4] INFO [2019-12-17 12:33:21,174] jepsen worker 31 - jepsen.util 6331 :fail :cas [10 [10 22]] :ValueNotSwapFrom INFO [2019-12-17 12:33:21,174] jepsen worker 31 - jepsen.util 6331 :invoke :write [10 14] INFO [2019-12-17 12:33:21,187] jepsen worker 32 - jepsen.util 7052 :fail :cas [10 [16 10]] :ValueNotSwapFrom INFO [2019-12-17 12:33:21,187] jepsen worker 32 - jepsen.util 7052 :invoke :write [10 4] INFO [2019-12-17 12:33:21,192] jepsen worker 60 - jepsen.util 10230 :info :write [20 19] :RequestCanceledException INFO [2019-12-17 12:33:21,192] jepsen worker 83 - jepsen.util 9893 :info :cas [27 [12 5]] :RequestCanceledException INFO [2019-12-17 12:33:21,192] jepsen worker 22 - jepsen.util 9382 :info :cas [7 [11 45]] :RequestCanceledException INFO [2019-12-17 12:33:21,193] jepsen worker 60 - jepsen.util 10320 :invoke :read [20 nil] INFO [2019-12-17 12:33:21,193] jepsen worker 83 - jepsen.util 9983 :invoke :read [27 nil] INFO [2019-12-17 12:33:21,193] jepsen worker 22 - jepsen.util 9472 :invoke :read [7 nil] INFO [2019-12-17 12:33:21,195] jepsen worker 74 - jepsen.util 7184 :fail :cas [24 [48 36]] :ValueNotSwapFrom INFO [2019-12-17 12:33:21,196] jepsen worker 74 - jepsen.util 7184 :invoke :read [24 nil] INFO [2019-12-17 12:33:21,202] jepsen worker 56 - jepsen.util 7796 :fail :cas [18 [25 28]] :ValueNotSwapFrom INFO [2019-12-17 12:33:21,202] jepsen worker 56 - jepsen.util 7796 :invoke :cas [18 [35 12]] INFO [2019-12-17 12:33:21,203] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40400,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:33:21,203] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40400,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:33:21,203] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40401,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:33:21,203] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40401,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:33:21,212] jepsen worker 50 - jepsen.util 9860 :info :write [16 28] :RequestCanceledException INFO [2019-12-17 12:33:21,212] jepsen worker 86 - jepsen.util 9896 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:33:21,212] jepsen worker 86 - jepsen.util 9896 :invoke :write [28 12] INFO [2019-12-17 12:33:21,212] jepsen worker 50 - jepsen.util 9950 :invoke :cas [16 [22 18]] INFO [2019-12-17 12:33:21,221] jepsen worker 20 - jepsen.util 6770 :fail :cas [6 [6 10]] :ValueNotSwapFrom INFO [2019-12-17 12:33:21,221] jepsen worker 20 - jepsen.util 6770 :invoke :write [6 5] INFO [2019-12-17 12:33:21,227] jepsen worker 82 - jepsen.util 10072 :info :cas [27 [13 20]] :RequestCanceledException INFO [2019-12-17 12:33:21,227] jepsen worker 15 - jepsen.util 8475 :info :write [5 1] :RequestCanceledException INFO [2019-12-17 12:33:21,227] jepsen worker 59 - jepsen.util 9329 :info :cas [19 [9 43]] :RequestCanceledException INFO [2019-12-17 12:33:21,228] jepsen worker 59 - jepsen.util 9419 :invoke :write [19 38] INFO [2019-12-17 12:33:21,228] jepsen worker 15 - jepsen.util 8565 :invoke :cas [5 [0 2]] INFO [2019-12-17 12:33:21,228] jepsen worker 82 - jepsen.util 10162 :invoke :read [27 nil] INFO [2019-12-17 12:33:21,234] jepsen worker 70 - jepsen.util 7270 :fail :cas [23 [10 49]] :ValueNotSwapFrom INFO [2019-12-17 12:33:21,234] jepsen worker 70 - jepsen.util 7270 :invoke :write [23 4] INFO [2019-12-17 12:33:21,234] jepsen worker 47 - jepsen.util 7517 :fail :cas [15 [45 27]] :ValueNotSwapFrom INFO [2019-12-17 12:33:21,234] jepsen worker 47 - jepsen.util 7517 :invoke :read [15 nil] INFO [2019-12-17 12:33:21,244] jepsen worker 74 - jepsen.util 7184 :ok :read [24 9] INFO [2019-12-17 12:33:21,244] jepsen worker 74 - jepsen.util 7184 :invoke :read [24 nil] INFO [2019-12-17 12:33:21,256] jepsen worker 12 - jepsen.util 7392 :fail :cas [4 [30 36]] :ValueNotSwapFrom INFO [2019-12-17 12:33:21,256] jepsen worker 12 - jepsen.util 7392 :invoke :read [4 nil] INFO [2019-12-17 12:33:21,265] jepsen worker 27 - jepsen.util 7047 :ok :read [9 31] INFO [2019-12-17 12:33:21,265] jepsen worker 27 - jepsen.util 7047 :invoke :read [9 nil] INFO [2019-12-17 12:33:21,275] jepsen worker 88 - jepsen.util 7018 :fail :cas [29 [0 9]] :ValueNotSwapFrom INFO [2019-12-17 12:33:21,275] jepsen worker 88 - jepsen.util 7018 :invoke :read [29 nil] INFO [2019-12-17 12:33:21,289] jepsen worker 47 - jepsen.util 7517 :ok :read [15 1] INFO [2019-12-17 12:33:21,289] jepsen worker 47 - jepsen.util 7517 :invoke :write [15 6] INFO [2019-12-17 12:33:21,294] jepsen worker 30 - jepsen.util 6780 :info :write [10 8] :RequestCanceledException INFO [2019-12-17 12:33:21,295] jepsen worker 30 - jepsen.util 6870 :invoke :cas [10 [19 14]] INFO [2019-12-17 12:33:21,303] jepsen worker 74 - jepsen.util 7184 :ok :read [24 9] INFO [2019-12-17 12:33:21,303] jepsen worker 74 - jepsen.util 7184 :invoke :read [24 nil] INFO [2019-12-17 12:33:21,313] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40409,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:33:21,313] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40409,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:33:21,313] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40412,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:33:21,313] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40412,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:33:21,314] jepsen worker 12 - jepsen.util 7392 :ok :read [4 1] INFO [2019-12-17 12:33:21,314] jepsen worker 12 - jepsen.util 7392 :invoke :read [4 nil] INFO [2019-12-17 12:33:21,326] jepsen worker 89 - jepsen.util 6839 :fail :cas [29 [12 27]] :ValueNotSwapFrom INFO [2019-12-17 12:33:21,326] jepsen worker 89 - jepsen.util 6839 :invoke :read [29 nil] INFO [2019-12-17 12:33:21,337] jepsen worker 27 - jepsen.util 7047 :ok :read [9 31] INFO [2019-12-17 12:33:21,337] jepsen worker 27 - jepsen.util 7047 :invoke :cas [9 [32 5]] INFO [2019-12-17 12:33:21,373] jepsen worker 88 - jepsen.util 7018 :ok :read [29 2] INFO [2019-12-17 12:33:21,373] jepsen worker 88 - jepsen.util 7018 :invoke :read [29 nil] INFO [2019-12-17 12:33:21,393] jepsen worker 14 - jepsen.util 7124 :info :cas [4 [1 32]] :RequestCanceledException INFO [2019-12-17 12:33:21,394] jepsen worker 14 - jepsen.util 7214 :invoke :read [4 nil] INFO [2019-12-17 12:33:21,395] jepsen worker 19 - jepsen.util 7039 :ok :write [6 41] INFO [2019-12-17 12:33:21,395] jepsen worker 19 - jepsen.util 7039 :invoke :cas [6 [35 23]] INFO [2019-12-17 12:33:21,395] jepsen worker 11 - jepsen.util 7211 :ok :write [3 42] INFO [2019-12-17 12:33:21,395] jepsen worker 11 - jepsen.util 7211 :invoke :cas [3 [13 23]] INFO [2019-12-17 12:33:21,413] jepsen worker 12 - jepsen.util 7392 :ok :read [4 1] INFO [2019-12-17 12:33:21,413] jepsen worker 12 - jepsen.util 7392 :invoke :write [4 45] INFO [2019-12-17 12:33:21,419] jepsen worker 19 - jepsen.util 7039 :fail :cas [6 [35 23]] :ValueNotSwapFrom INFO [2019-12-17 12:33:21,419] jepsen worker 19 - jepsen.util 7039 :invoke :read [6 nil] INFO [2019-12-17 12:33:21,420] jepsen worker 55 - jepsen.util 8065 :ok :write [18 5] INFO [2019-12-17 12:33:21,420] jepsen worker 55 - jepsen.util 8065 :invoke :read [18 nil] INFO [2019-12-17 12:33:21,428] jepsen worker 76 - jepsen.util 9796 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:33:21,428] jepsen worker 76 - jepsen.util 9796 :invoke :read [25 nil] INFO [2019-12-17 12:33:21,436] jepsen worker 72 - jepsen.util 6732 :ok :read [24 9] INFO [2019-12-17 12:33:21,436] jepsen worker 72 - jepsen.util 6732 :invoke :write [24 45] INFO [2019-12-17 12:33:21,444] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:33:21,444] jepsen nemesis - couchbase.util Rebalance check count: 309 INFO [2019-12-17 12:33:21,444] jepsen nemesis - couchbase.util Rebalance stuck count: 290 INFO [2019-12-17 12:33:21,448] jepsen worker 19 - jepsen.util 7039 :ok :read [6 41] INFO [2019-12-17 12:33:21,448] jepsen worker 19 - jepsen.util 7039 :invoke :write [6 24] INFO [2019-12-17 12:33:21,454] jepsen worker 56 - jepsen.util 7796 :fail :cas [18 [35 12]] :ValueNotSwapFrom INFO [2019-12-17 12:33:21,454] jepsen worker 56 - jepsen.util 7796 :invoke :read [18 nil] INFO [2019-12-17 12:33:21,474] jepsen worker 14 - jepsen.util 7214 :ok :read [4 1] INFO [2019-12-17 12:33:21,474] jepsen worker 14 - jepsen.util 7214 :invoke :write [4 33] INFO [2019-12-17 12:33:21,495] jepsen worker 9 - jepsen.util 7479 :info :write [3 29] :RequestCanceledException INFO [2019-12-17 12:33:21,496] jepsen worker 9 - jepsen.util 7569 :invoke :read [3 nil] INFO [2019-12-17 12:33:21,497] jepsen worker 11 - jepsen.util 7211 :fail :cas [3 [13 23]] :ValueNotSwapFrom INFO [2019-12-17 12:33:21,497] jepsen worker 11 - jepsen.util 7211 :invoke :write [3 32] INFO [2019-12-17 12:33:21,514] jepsen worker 9 - jepsen.util 7569 :ok :read [3 42] INFO [2019-12-17 12:33:21,514] jepsen worker 9 - jepsen.util 7569 :invoke :cas [3 [47 28]] INFO [2019-12-17 12:33:21,532] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40443,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:33:21,532] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40443,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:33:21,532] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40445,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:33:21,532] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40445,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:33:21,534] jepsen worker 9 - jepsen.util 7569 :fail :cas [3 [47 28]] :ValueNotSwapFrom INFO [2019-12-17 12:33:21,534] jepsen worker 9 - jepsen.util 7569 :invoke :read [3 nil] INFO [2019-12-17 12:33:21,535] jepsen worker 47 - jepsen.util 7517 :ok :write [15 6] INFO [2019-12-17 12:33:21,535] jepsen worker 47 - jepsen.util 7517 :invoke :cas [15 [17 17]] INFO [2019-12-17 12:33:21,559] jepsen worker 74 - jepsen.util 7184 :ok :read [24 9] INFO [2019-12-17 12:33:21,559] jepsen worker 74 - jepsen.util 7184 :invoke :cas [24 [33 10]] INFO [2019-12-17 12:33:21,571] jepsen worker 56 - jepsen.util 7796 :ok :read [18 5] INFO [2019-12-17 12:33:21,572] jepsen worker 56 - jepsen.util 7796 :invoke :cas [18 [32 48]] INFO [2019-12-17 12:33:21,576] jepsen worker 47 - jepsen.util 7517 :fail :cas [15 [17 17]] :ValueNotSwapFrom INFO [2019-12-17 12:33:21,577] jepsen worker 47 - jepsen.util 7517 :invoke :write [15 24] INFO [2019-12-17 12:33:21,592] jepsen worker 9 - jepsen.util 7569 :ok :read [3 42] INFO [2019-12-17 12:33:21,592] jepsen worker 9 - jepsen.util 7569 :invoke :read [3 nil] INFO [2019-12-17 12:33:21,595] jepsen worker 74 - jepsen.util 7184 :fail :cas [24 [33 10]] :ValueNotSwapFrom INFO [2019-12-17 12:33:21,595] jepsen worker 74 - jepsen.util 7184 :invoke :write [24 46] INFO [2019-12-17 12:33:21,605] jepsen worker 9 - jepsen.util 7569 :ok :read [3 42] INFO [2019-12-17 12:33:21,605] jepsen worker 9 - jepsen.util 7569 :invoke :read [3 nil] INFO [2019-12-17 12:33:21,612] jepsen worker 6 - jepsen.util 10086 :info :cas [2 [28 11]] :RequestCanceledException INFO [2019-12-17 12:33:21,612] jepsen worker 17 - jepsen.util 9737 :info :write [5 4] :RequestCanceledException INFO [2019-12-17 12:33:21,613] jepsen worker 17 - jepsen.util 9827 :invoke :cas [5 [14 33]] INFO [2019-12-17 12:33:21,613] jepsen worker 6 - jepsen.util 10176 :invoke :cas [2 [28 17]] INFO [2019-12-17 12:33:21,619] jepsen worker 32 - jepsen.util 7052 :ok :write [10 4] INFO [2019-12-17 12:33:21,619] jepsen worker 32 - jepsen.util 7052 :invoke :write [10 24] INFO [2019-12-17 12:33:21,621] jepsen worker 72 - jepsen.util 6732 :ok :write [24 45] INFO [2019-12-17 12:33:21,621] jepsen worker 72 - jepsen.util 6732 :invoke :write [24 6] INFO [2019-12-17 12:33:21,621] jepsen worker 12 - jepsen.util 7392 :ok :write [4 45] INFO [2019-12-17 12:33:21,622] jepsen worker 12 - jepsen.util 7392 :invoke :write [4 24] INFO [2019-12-17 12:33:21,627] jepsen worker 79 - jepsen.util 6919 :info :write [26 48] :RequestCanceledException INFO [2019-12-17 12:33:21,627] jepsen worker 10 - jepsen.util 6850 :info :write [3 48] :RequestCanceledException INFO [2019-12-17 12:33:21,627] jepsen worker 40 - jepsen.util 9940 :info :cas [13 [45 37]] :RequestCanceledException INFO [2019-12-17 12:33:21,628] jepsen worker 10 - jepsen.util 6940 :invoke :read [3 nil] INFO [2019-12-17 12:33:21,628] jepsen worker 40 - jepsen.util 10030 :invoke :write [13 33] INFO [2019-12-17 12:33:21,628] jepsen worker 79 - jepsen.util 7009 :invoke :write [26 31] INFO [2019-12-17 12:33:21,642] jepsen worker 84 - jepsen.util 8994 :info :cas [28 [0 8]] :RequestCanceledException INFO [2019-12-17 12:33:21,643] jepsen worker 84 - jepsen.util 9084 :invoke :read [28 nil] INFO [2019-12-17 12:33:21,646] jepsen worker 52 - jepsen.util 7252 :ok :write [17 20] INFO [2019-12-17 12:33:21,646] jepsen worker 9 - jepsen.util 7569 :ok :read [3 42] INFO [2019-12-17 12:33:21,646] jepsen worker 52 - jepsen.util 7252 :invoke :read [17 nil] INFO [2019-12-17 12:33:21,646] jepsen worker 9 - jepsen.util 7569 :invoke :cas [3 [1 43]] INFO [2019-12-17 12:33:21,658] jepsen worker 5 - jepsen.util 6575 :info :write [1 40] :RequestCanceledException INFO [2019-12-17 12:33:21,658] jepsen worker 81 - jepsen.util 9261 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:33:21,658] jepsen worker 39 - jepsen.util 9309 :info :cas [13 [18 47]] :RequestCanceledException INFO [2019-12-17 12:33:21,658] jepsen worker 81 - jepsen.util 9261 :invoke :cas [27 [17 19]] INFO [2019-12-17 12:33:21,659] jepsen worker 5 - jepsen.util 6665 :invoke :cas [1 [14 2]] INFO [2019-12-17 12:33:21,662] jepsen worker 39 - jepsen.util 9399 :invoke :write [13 33] INFO [2019-12-17 12:33:21,680] jepsen worker 89 - jepsen.util 6839 :ok :read [29 2] INFO [2019-12-17 12:33:21,680] jepsen worker 89 - jepsen.util 6839 :invoke :read [29 nil] INFO [2019-12-17 12:33:21,683] jepsen worker 3 - jepsen.util 6933 :ok :write [1 17] INFO [2019-12-17 12:33:21,683] jepsen worker 3 - jepsen.util 6933 :invoke :write [1 46] INFO [2019-12-17 12:33:21,696] jepsen worker 9 - jepsen.util 7569 :fail :cas [3 [1 43]] :ValueNotSwapFrom INFO [2019-12-17 12:33:21,696] jepsen worker 9 - jepsen.util 7569 :invoke :cas [3 [16 44]] INFO [2019-12-17 12:33:21,696] jepsen worker 19 - jepsen.util 7039 :ok :write [6 24] INFO [2019-12-17 12:33:21,697] jepsen worker 19 - jepsen.util 7039 :invoke :cas [6 [27 13]] INFO [2019-12-17 12:33:21,721] jepsen worker 5 - jepsen.util 6665 :fail :cas [1 [14 2]] :ValueNotSwapFrom INFO [2019-12-17 12:33:21,721] jepsen worker 5 - jepsen.util 6665 :invoke :read [1 nil] INFO [2019-12-17 12:33:21,724] jepsen worker 10 - jepsen.util 6940 :ok :read [3 42] INFO [2019-12-17 12:33:21,724] jepsen worker 10 - jepsen.util 6940 :invoke :read [3 nil] INFO [2019-12-17 12:33:21,728] jepsen worker 54 - jepsen.util 7794 :info :write [18 21] :RequestCanceledException INFO [2019-12-17 12:33:21,729] jepsen worker 54 - jepsen.util 7884 :invoke :cas [18 [4 19]] INFO [2019-12-17 12:33:21,742] jepsen worker 44 - jepsen.util 10124 :info :cas [14 [7 39]] :RequestCanceledException INFO [2019-12-17 12:33:21,743] jepsen worker 44 - jepsen.util 10214 :invoke :write [14 47] INFO [2019-12-17 12:33:21,748] jepsen worker 19 - jepsen.util 7039 :fail :cas [6 [27 13]] :ValueNotSwapFrom INFO [2019-12-17 12:33:21,748] jepsen worker 19 - jepsen.util 7039 :invoke :read [6 nil] INFO [2019-12-17 12:33:21,751] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40470,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:33:21,751] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40470,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:33:21,751] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40473,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:33:21,751] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40473,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:33:21,752] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40475,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:33:21,752] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40475,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:33:21,755] jepsen worker 9 - jepsen.util 7569 :fail :cas [3 [16 44]] :ValueNotSwapFrom INFO [2019-12-17 12:33:21,755] jepsen worker 9 - jepsen.util 7569 :invoke :cas [3 [44 12]] INFO [2019-12-17 12:33:21,790] jepsen worker 9 - jepsen.util 7569 :fail :cas [3 [44 12]] :ValueNotSwapFrom INFO [2019-12-17 12:33:21,790] jepsen worker 9 - jepsen.util 7569 :invoke :read [3 nil] INFO [2019-12-17 12:33:21,790] jepsen worker 10 - jepsen.util 6940 :ok :read [3 42] INFO [2019-12-17 12:33:21,791] jepsen worker 10 - jepsen.util 6940 :invoke :write [3 28] INFO [2019-12-17 12:33:21,793] jepsen worker 5 - jepsen.util 6665 :ok :read [1 17] INFO [2019-12-17 12:33:21,793] jepsen worker 5 - jepsen.util 6665 :invoke :write [1 47] INFO [2019-12-17 12:33:21,794] jepsen worker 48 - jepsen.util 8688 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:33:21,795] jepsen worker 48 - jepsen.util 8688 :invoke :read [16 nil] INFO [2019-12-17 12:33:21,812] jepsen worker 9 - jepsen.util 7569 :ok :read [3 42] INFO [2019-12-17 12:33:21,812] jepsen worker 61 - jepsen.util 10321 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:33:21,812] jepsen worker 61 - jepsen.util 10321 :invoke :read [20 nil] INFO [2019-12-17 12:33:21,812] jepsen worker 9 - jepsen.util 7569 :invoke :cas [3 [12 27]] INFO [2019-12-17 12:33:21,828] jepsen worker 57 - jepsen.util 9147 :info :write [19 24] :RequestCanceledException INFO [2019-12-17 12:33:21,829] jepsen worker 57 - jepsen.util 9237 :invoke :cas [19 [33 37]] INFO [2019-12-17 12:33:21,835] jepsen worker 19 - jepsen.util 7039 :ok :read [6 24] INFO [2019-12-17 12:33:21,835] jepsen worker 19 - jepsen.util 7039 :invoke :read [6 nil] INFO [2019-12-17 12:33:21,840] jepsen worker 9 - jepsen.util 7569 :fail :cas [3 [12 27]] :ValueNotSwapFrom INFO [2019-12-17 12:33:21,840] jepsen worker 9 - jepsen.util 7569 :invoke :read [3 nil] INFO [2019-12-17 12:33:21,842] jepsen worker 7 - jepsen.util 9727 :info :write [2 31] :RequestCanceledException INFO [2019-12-17 12:33:21,843] jepsen worker 7 - jepsen.util 9817 :invoke :read [2 nil] INFO [2019-12-17 12:33:21,859] jepsen worker 16 - jepsen.util 10636 :info :cas [5 [22 20]] :RequestCanceledException INFO [2019-12-17 12:33:21,862] jepsen worker 16 - jepsen.util 10726 :invoke :write [5 46] INFO [2019-12-17 12:33:21,884] jepsen worker 47 - jepsen.util 7517 :ok :write [15 24] INFO [2019-12-17 12:33:21,884] jepsen worker 47 - jepsen.util 7517 :invoke :cas [15 [20 39]] INFO [2019-12-17 12:33:21,887] jepsen worker 88 - jepsen.util 7018 :ok :read [29 2] INFO [2019-12-17 12:33:21,887] jepsen worker 88 - jepsen.util 7018 :invoke :write [29 24] INFO [2019-12-17 12:33:21,888] jepsen worker 9 - jepsen.util 7569 :ok :read [3 42] INFO [2019-12-17 12:33:21,888] jepsen worker 9 - jepsen.util 7569 :invoke :cas [3 [20 22]] INFO [2019-12-17 12:33:21,894] jepsen worker 11 - jepsen.util 7211 :ok :write [3 32] INFO [2019-12-17 12:33:21,894] jepsen worker 11 - jepsen.util 7211 :invoke :cas [3 [44 43]] INFO [2019-12-17 12:33:21,894] jepsen worker 85 - jepsen.util 10435 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:33:21,894] jepsen worker 8 - jepsen.util 9368 :info :cas [2 [6 23]] :RequestCanceledException INFO [2019-12-17 12:33:21,894] jepsen worker 85 - jepsen.util 10435 :invoke :read [28 nil] INFO [2019-12-17 12:33:21,895] jepsen worker 8 - jepsen.util 9458 :invoke :write [2 47] INFO [2019-12-17 12:33:21,912] jepsen worker 80 - jepsen.util 6920 :info :write [26 29] :RequestCanceledException INFO [2019-12-17 12:33:21,913] jepsen worker 80 - jepsen.util 7010 :invoke :write [26 26] INFO [2019-12-17 12:33:21,926] jepsen worker 49 - jepsen.util 8779 :info :cas [16 [33 7]] :RequestCanceledException INFO [2019-12-17 12:33:21,927] jepsen worker 49 - jepsen.util 8869 :invoke :write [16 46] INFO [2019-12-17 12:33:21,928] jepsen worker 9 - jepsen.util 7569 :fail :cas [3 [20 22]] :ValueNotSwapFrom INFO [2019-12-17 12:33:21,928] jepsen worker 9 - jepsen.util 7569 :invoke :write [3 43] INFO [2019-12-17 12:33:21,931] jepsen worker 11 - jepsen.util 7211 :fail :cas [3 [44 43]] :ValueNotSwapFrom INFO [2019-12-17 12:33:21,931] jepsen worker 11 - jepsen.util 7211 :invoke :write [3 44] INFO [2019-12-17 12:33:21,941] jepsen worker 36 - jepsen.util 7416 :fail :cas [12 [26 18]] :ValueNotSwapFrom INFO [2019-12-17 12:33:21,941] jepsen worker 36 - jepsen.util 7416 :invoke :read [12 nil] INFO [2019-12-17 12:33:21,958] jepsen worker 38 - jepsen.util 6968 :info :write [12 17] :RequestCanceledException INFO [2019-12-17 12:33:21,959] jepsen worker 38 - jepsen.util 7058 :invoke :write [12 24] INFO [2019-12-17 12:33:21,969] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40502,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:33:21,969] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40502,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:33:21,969] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40506,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:33:21,969] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40506,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:33:21,975] jepsen worker 32 - jepsen.util 7052 :ok :write [10 24] INFO [2019-12-17 12:33:21,975] jepsen worker 32 - jepsen.util 7052 :invoke :write [10 9] INFO [2019-12-17 12:33:21,988] jepsen worker 74 - jepsen.util 7184 :ok :write [24 46] INFO [2019-12-17 12:33:21,988] jepsen worker 74 - jepsen.util 7184 :invoke :write [24 48] INFO [2019-12-17 12:33:22,000] jepsen worker 47 - jepsen.util 7517 :fail :cas [15 [20 39]] :ValueNotSwapFrom INFO [2019-12-17 12:33:22,000] jepsen worker 47 - jepsen.util 7517 :invoke :read [15 nil] INFO [2019-12-17 12:33:22,012] jepsen worker 78 - jepsen.util 7098 :info :write [26 13] :RequestCanceledException INFO [2019-12-17 12:33:22,014] jepsen worker 78 - jepsen.util 7188 :invoke :cas [26 [39 15]] INFO [2019-12-17 12:33:22,076] jepsen worker 55 - jepsen.util 8065 :ok :read [18 5] INFO [2019-12-17 12:33:22,076] jepsen worker 55 - jepsen.util 8065 :invoke :read [18 nil] INFO [2019-12-17 12:33:22,078] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40525,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:33:22,078] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40525,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:33:22,078] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40527,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:33:22,078] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40527,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:33:22,085] jepsen worker 11 - jepsen.util 7211 :ok :write [3 44] INFO [2019-12-17 12:33:22,085] jepsen worker 11 - jepsen.util 7211 :invoke :cas [3 [22 42]] INFO [2019-12-17 12:33:22,097] jepsen worker 11 - jepsen.util 7211 :fail :cas [3 [22 42]] :ValueNotSwapFrom INFO [2019-12-17 12:33:22,097] jepsen worker 11 - jepsen.util 7211 :invoke :read [3 nil] INFO [2019-12-17 12:33:22,109] jepsen worker 32 - jepsen.util 7052 :ok :write [10 9] INFO [2019-12-17 12:33:22,110] jepsen worker 11 - jepsen.util 7211 :ok :read [3 44] INFO [2019-12-17 12:33:22,110] jepsen worker 32 - jepsen.util 7052 :invoke :read [10 nil] INFO [2019-12-17 12:33:22,110] jepsen worker 11 - jepsen.util 7211 :invoke :write [3 5] INFO [2019-12-17 12:33:22,125] jepsen worker 74 - jepsen.util 7184 :ok :write [24 48] INFO [2019-12-17 12:33:22,125] jepsen worker 74 - jepsen.util 7184 :invoke :write [24 2] INFO [2019-12-17 12:33:22,128] jepsen worker 35 - jepsen.util 9215 :info :cas [11 [5 23]] :RequestCanceledException INFO [2019-12-17 12:33:22,132] jepsen worker 35 - jepsen.util 9305 :invoke :cas [11 [49 34]] INFO [2019-12-17 12:33:22,158] jepsen worker 56 - jepsen.util 7796 :fail :cas [18 [32 48]] :ValueNotSwapFrom INFO [2019-12-17 12:33:22,158] jepsen worker 56 - jepsen.util 7796 :invoke :write [18 22] INFO [2019-12-17 12:33:22,161] jepsen worker 32 - jepsen.util 7052 :ok :read [10 9] INFO [2019-12-17 12:33:22,162] jepsen worker 32 - jepsen.util 7052 :invoke :cas [10 [0 6]] INFO [2019-12-17 12:33:22,170] jepsen worker 52 - jepsen.util 7252 :ok :read [17 20] INFO [2019-12-17 12:33:22,170] jepsen worker 52 - jepsen.util 7252 :invoke :write [17 44] INFO [2019-12-17 12:33:22,186] jepsen worker 89 - jepsen.util 6839 :ok :read [29 2] INFO [2019-12-17 12:33:22,187] jepsen worker 89 - jepsen.util 6839 :invoke :read [29 nil] INFO [2019-12-17 12:33:22,200] jepsen worker 32 - jepsen.util 7052 :fail :cas [10 [0 6]] :ValueNotSwapFrom INFO [2019-12-17 12:33:22,200] jepsen worker 32 - jepsen.util 7052 :invoke :read [10 nil] INFO [2019-12-17 12:33:22,227] jepsen worker 2 - jepsen.util 9092 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:33:22,228] jepsen worker 2 - jepsen.util 9092 :invoke :read [0 nil] INFO [2019-12-17 12:33:22,233] jepsen worker 32 - jepsen.util 7052 :ok :read [10 9] INFO [2019-12-17 12:33:22,233] jepsen worker 32 - jepsen.util 7052 :invoke :write [10 22] INFO [2019-12-17 12:33:22,240] jepsen worker 74 - jepsen.util 7184 :ok :write [24 2] INFO [2019-12-17 12:33:22,240] jepsen worker 74 - jepsen.util 7184 :invoke :read [24 nil] INFO [2019-12-17 12:33:22,242] jepsen worker 51 - jepsen.util 6801 :info :write [17 11] :RequestCanceledException INFO [2019-12-17 12:33:22,243] jepsen worker 51 - jepsen.util 6891 :invoke :cas [17 [48 38]] INFO [2019-12-17 12:33:22,265] jepsen worker 11 - jepsen.util 7211 :ok :write [3 5] INFO [2019-12-17 12:33:22,266] jepsen worker 11 - jepsen.util 7211 :invoke :cas [3 [36 46]] INFO [2019-12-17 12:33:22,273] jepsen worker 36 - jepsen.util 7416 :ok :read [12 28] INFO [2019-12-17 12:33:22,274] jepsen worker 36 - jepsen.util 7416 :invoke :read [12 nil] INFO [2019-12-17 12:33:22,275] jepsen worker 74 - jepsen.util 7184 :ok :read [24 2] INFO [2019-12-17 12:33:22,275] jepsen worker 74 - jepsen.util 7184 :invoke :read [24 nil] INFO [2019-12-17 12:33:22,283] jepsen worker 11 - jepsen.util 7211 :fail :cas [3 [36 46]] :ValueNotSwapFrom INFO [2019-12-17 12:33:22,283] jepsen worker 11 - jepsen.util 7211 :invoke :read [3 nil] INFO [2019-12-17 12:33:22,298] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40541,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:33:22,298] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40549,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:33:22,298] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40541,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:33:22,298] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40549,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:33:22,298] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40542,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:33:22,298] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40553,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:33:22,298] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40542,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:33:22,299] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40553,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:33:22,299] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40543,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:33:22,299] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40543,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:33:22,300] jepsen worker 74 - jepsen.util 7184 :ok :read [24 2] INFO [2019-12-17 12:33:22,300] jepsen worker 74 - jepsen.util 7184 :invoke :cas [24 [37 10]] INFO [2019-12-17 12:33:22,310] jepsen worker 11 - jepsen.util 7211 :ok :read [3 5] INFO [2019-12-17 12:33:22,310] jepsen worker 11 - jepsen.util 7211 :invoke :write [3 0] INFO [2019-12-17 12:33:22,310] jepsen worker 51 - jepsen.util 6891 :fail :cas [17 [48 38]] :ValueNotSwapFrom INFO [2019-12-17 12:33:22,310] jepsen worker 51 - jepsen.util 6891 :invoke :cas [17 [36 18]] INFO [2019-12-17 12:33:22,323] jepsen worker 74 - jepsen.util 7184 :fail :cas [24 [37 10]] :ValueNotSwapFrom INFO [2019-12-17 12:33:22,323] jepsen worker 74 - jepsen.util 7184 :invoke :cas [24 [15 31]] INFO [2019-12-17 12:33:22,324] jepsen worker 78 - jepsen.util 7188 :fail :cas [26 [39 15]] :ValueNotSwapFrom INFO [2019-12-17 12:33:22,324] jepsen worker 78 - jepsen.util 7188 :invoke :write [26 45] INFO [2019-12-17 12:33:22,335] jepsen worker 74 - jepsen.util 7184 :fail :cas [24 [15 31]] :ValueNotSwapFrom INFO [2019-12-17 12:33:22,335] jepsen worker 74 - jepsen.util 7184 :invoke :cas [24 [0 11]] INFO [2019-12-17 12:33:22,356] jepsen worker 74 - jepsen.util 7184 :fail :cas [24 [0 11]] :ValueNotSwapFrom INFO [2019-12-17 12:33:22,356] jepsen worker 74 - jepsen.util 7184 :invoke :read [24 nil] INFO [2019-12-17 12:33:22,358] jepsen worker 54 - jepsen.util 7884 :fail :cas [18 [4 19]] :ValueNotSwapFrom INFO [2019-12-17 12:33:22,358] jepsen worker 54 - jepsen.util 7884 :invoke :cas [18 [16 8]] INFO [2019-12-17 12:33:22,377] jepsen worker 74 - jepsen.util 7184 :ok :read [24 2] INFO [2019-12-17 12:33:22,377] jepsen worker 74 - jepsen.util 7184 :invoke :cas [24 [18 10]] INFO [2019-12-17 12:33:22,389] jepsen worker 74 - jepsen.util 7184 :fail :cas [24 [18 10]] :ValueNotSwapFrom INFO [2019-12-17 12:33:22,389] jepsen worker 74 - jepsen.util 7184 :invoke :cas [24 [2 13]] INFO [2019-12-17 12:33:22,393] jepsen worker 51 - jepsen.util 6891 :fail :cas [17 [36 18]] :ValueNotSwapFrom INFO [2019-12-17 12:33:22,393] jepsen worker 51 - jepsen.util 6891 :invoke :cas [17 [42 0]] INFO [2019-12-17 12:33:22,420] jepsen worker 55 - jepsen.util 8065 :ok :read [18 5] INFO [2019-12-17 12:33:22,420] jepsen worker 55 - jepsen.util 8065 :invoke :write [18 40] INFO [2019-12-17 12:33:22,432] jepsen worker 32 - jepsen.util 7052 :ok :write [10 22] INFO [2019-12-17 12:33:22,432] jepsen worker 32 - jepsen.util 7052 :invoke :read [10 nil] INFO [2019-12-17 12:33:22,444] jepsen worker 32 - jepsen.util 7052 :ok :read [10 22] INFO [2019-12-17 12:33:22,444] jepsen worker 32 - jepsen.util 7052 :invoke :read [10 nil] INFO [2019-12-17 12:33:22,458] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:33:22,458] jepsen nemesis - couchbase.util Rebalance check count: 310 INFO [2019-12-17 12:33:22,458] jepsen nemesis - couchbase.util Rebalance stuck count: 291 INFO [2019-12-17 12:33:22,465] jepsen worker 32 - jepsen.util 7052 :ok :read [10 22] INFO [2019-12-17 12:33:22,466] jepsen worker 32 - jepsen.util 7052 :invoke :read [10 nil] INFO [2019-12-17 12:33:22,476] jepsen worker 3 - jepsen.util 6933 :ok :write [1 46] INFO [2019-12-17 12:33:22,476] jepsen worker 3 - jepsen.util 6933 :invoke :read [1 nil] INFO [2019-12-17 12:33:22,479] jepsen worker 11 - jepsen.util 7211 :ok :write [3 0] INFO [2019-12-17 12:33:22,479] jepsen worker 11 - jepsen.util 7211 :invoke :read [3 nil] INFO [2019-12-17 12:33:22,483] jepsen worker 32 - jepsen.util 7052 :ok :read [10 22] INFO [2019-12-17 12:33:22,483] jepsen worker 32 - jepsen.util 7052 :invoke :write [10 16] INFO [2019-12-17 12:33:22,491] jepsen worker 74 - jepsen.util 7184 :ok :cas [24 [2 13]] INFO [2019-12-17 12:33:22,491] jepsen worker 74 - jepsen.util 7184 :invoke :read [24 nil] INFO [2019-12-17 12:33:22,497] jepsen worker 11 - jepsen.util 7211 :ok :read [3 0] INFO [2019-12-17 12:33:22,497] jepsen worker 11 - jepsen.util 7211 :invoke :write [3 26] INFO [2019-12-17 12:33:22,513] jepsen worker 74 - jepsen.util 7184 :ok :read [24 13] INFO [2019-12-17 12:33:22,513] jepsen worker 74 - jepsen.util 7184 :invoke :write [24 26] INFO [2019-12-17 12:33:22,515] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40576,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:33:22,516] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40576,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:33:22,516] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40579,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:33:22,516] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40579,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:33:22,516] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40580,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:33:22,516] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40580,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:33:22,516] jepsen worker 3 - jepsen.util 6933 :ok :read [1 46] INFO [2019-12-17 12:33:22,516] jepsen worker 3 - jepsen.util 6933 :invoke :write [1 39] INFO [2019-12-17 12:33:22,528] jepsen worker 65 - jepsen.util 5825 :info :cas [21 [37 41]] :RequestCanceledException INFO [2019-12-17 12:33:22,529] jepsen worker 65 - jepsen.util 5915 :invoke :read [21 nil] INFO [2019-12-17 12:33:22,534] jepsen worker 71 - jepsen.util 7721 :info :write [23 17] :RequestCanceledException INFO [2019-12-17 12:33:22,534] jepsen worker 71 - jepsen.util 7811 :invoke :write [23 47] INFO [2019-12-17 12:33:22,548] jepsen worker 88 - jepsen.util 7018 :ok :write [29 24] INFO [2019-12-17 12:33:22,548] jepsen worker 88 - jepsen.util 7018 :invoke :read [29 nil] INFO [2019-12-17 12:33:22,562] jepsen worker 65 - jepsen.util 5915 :ok :read [21 5] INFO [2019-12-17 12:33:22,562] jepsen worker 65 - jepsen.util 5915 :invoke :write [21 3] INFO [2019-12-17 12:33:22,573] jepsen worker 53 - jepsen.util 6623 :ok :write [17 26] INFO [2019-12-17 12:33:22,573] jepsen worker 53 - jepsen.util 6623 :invoke :cas [17 [42 6]] INFO [2019-12-17 12:33:22,586] jepsen worker 53 - jepsen.util 6623 :fail :cas [17 [42 6]] :ValueNotSwapFrom INFO [2019-12-17 12:33:22,586] jepsen worker 53 - jepsen.util 6623 :invoke :read [17 nil] INFO [2019-12-17 12:33:22,620] jepsen worker 53 - jepsen.util 6623 :ok :read [17 26] INFO [2019-12-17 12:33:22,620] jepsen worker 53 - jepsen.util 6623 :invoke :write [17 47] INFO [2019-12-17 12:33:22,628] jepsen worker 87 - jepsen.util 6207 :info :write [29 13] :RequestCanceledException INFO [2019-12-17 12:33:22,629] jepsen worker 87 - jepsen.util 6297 :invoke :cas [29 [30 21]] INFO [2019-12-17 12:33:22,638] jepsen worker 65 - jepsen.util 5915 :ok :write [21 3] INFO [2019-12-17 12:33:22,638] jepsen worker 65 - jepsen.util 5915 :invoke :read [21 nil] INFO [2019-12-17 12:33:22,651] jepsen worker 51 - jepsen.util 6891 :fail :cas [17 [42 0]] :ValueNotSwapFrom INFO [2019-12-17 12:33:22,651] jepsen worker 51 - jepsen.util 6891 :invoke :write [17 38] INFO [2019-12-17 12:33:22,660] jepsen worker 65 - jepsen.util 5915 :ok :read [21 3] INFO [2019-12-17 12:33:22,660] jepsen worker 65 - jepsen.util 5915 :invoke :cas [21 [22 44]] INFO [2019-12-17 12:33:22,682] jepsen worker 65 - jepsen.util 5915 :fail :cas [21 [22 44]] :ValueNotSwapFrom INFO [2019-12-17 12:33:22,683] jepsen worker 65 - jepsen.util 5915 :invoke :write [21 4] INFO [2019-12-17 12:33:22,708] jepsen worker 11 - jepsen.util 7211 :ok :write [3 26] INFO [2019-12-17 12:33:22,708] jepsen worker 11 - jepsen.util 7211 :invoke :write [3 20] INFO [2019-12-17 12:33:22,712] jepsen worker 69 - jepsen.util 7089 :info :write [23 44] :RequestCanceledException INFO [2019-12-17 12:33:22,713] jepsen worker 69 - jepsen.util 7179 :invoke :read [23 nil] INFO [2019-12-17 12:33:22,728] jepsen worker 63 - jepsen.util 6003 :info :write [21 36] :RequestCanceledException INFO [2019-12-17 12:33:22,729] jepsen worker 63 - jepsen.util 6093 :invoke :read [21 nil] INFO [2019-12-17 12:33:22,731] jepsen worker 65 - jepsen.util 5915 :ok :write [21 4] INFO [2019-12-17 12:33:22,731] jepsen worker 65 - jepsen.util 5915 :invoke :cas [21 [41 11]] INFO [2019-12-17 12:33:22,734] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40601,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:33:22,734] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40601,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:33:22,734] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40605,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:33:22,734] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40605,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:33:22,749] jepsen worker 54 - jepsen.util 7884 :fail :cas [18 [16 8]] :ValueNotSwapFrom INFO [2019-12-17 12:33:22,749] jepsen worker 54 - jepsen.util 7884 :invoke :read [18 nil] INFO [2019-12-17 12:33:22,749] jepsen worker 65 - jepsen.util 5915 :fail :cas [21 [41 11]] :ValueNotSwapFrom INFO [2019-12-17 12:33:22,750] jepsen worker 65 - jepsen.util 5915 :invoke :read [21 nil] INFO [2019-12-17 12:33:22,750] jepsen worker 63 - jepsen.util 6093 :ok :read [21 3] INFO [2019-12-17 12:33:22,751] jepsen worker 63 - jepsen.util 6093 :invoke :read [21 nil] INFO [2019-12-17 12:33:22,769] jepsen worker 65 - jepsen.util 5915 :ok :read [21 4] INFO [2019-12-17 12:33:22,769] jepsen worker 65 - jepsen.util 5915 :invoke :write [21 12] INFO [2019-12-17 12:33:22,789] jepsen worker 63 - jepsen.util 6093 :ok :read [21 4] INFO [2019-12-17 12:33:22,789] jepsen worker 63 - jepsen.util 6093 :invoke :read [21 nil] INFO [2019-12-17 12:33:22,826] jepsen worker 63 - jepsen.util 6093 :ok :read [21 4] INFO [2019-12-17 12:33:22,826] jepsen worker 63 - jepsen.util 6093 :invoke :write [21 42] INFO [2019-12-17 12:33:22,830] jepsen worker 65 - jepsen.util 5915 :ok :write [21 12] INFO [2019-12-17 12:33:22,831] jepsen worker 65 - jepsen.util 5915 :invoke :write [21 40] INFO [2019-12-17 12:33:22,831] jepsen worker 32 - jepsen.util 7052 :ok :write [10 16] INFO [2019-12-17 12:33:22,832] jepsen worker 32 - jepsen.util 7052 :invoke :read [10 nil] INFO [2019-12-17 12:33:22,832] jepsen worker 74 - jepsen.util 7184 :ok :write [24 26] INFO [2019-12-17 12:33:22,832] jepsen worker 74 - jepsen.util 7184 :invoke :read [24 nil] INFO [2019-12-17 12:33:22,851] jepsen worker 11 - jepsen.util 7211 :ok :write [3 20] INFO [2019-12-17 12:33:22,851] jepsen worker 11 - jepsen.util 7211 :invoke :cas [3 [49 38]] INFO [2019-12-17 12:33:22,852] jepsen worker 74 - jepsen.util 7184 :ok :read [24 26] INFO [2019-12-17 12:33:22,852] jepsen worker 74 - jepsen.util 7184 :invoke :cas [24 [19 6]] INFO [2019-12-17 12:33:22,858] jepsen worker 29 - jepsen.util 5339 :info :cas [9 [19 16]] :RequestCanceledException INFO [2019-12-17 12:33:22,859] jepsen worker 29 - jepsen.util 5429 :invoke :cas [9 [22 6]] INFO [2019-12-17 12:33:22,885] jepsen worker 32 - jepsen.util 7052 :ok :read [10 16] INFO [2019-12-17 12:33:22,885] jepsen worker 32 - jepsen.util 7052 :invoke :read [10 nil] INFO [2019-12-17 12:33:22,889] jepsen worker 88 - jepsen.util 7018 :ok :read [29 24] INFO [2019-12-17 12:33:22,889] jepsen worker 88 - jepsen.util 7018 :invoke :cas [29 [49 0]] INFO [2019-12-17 12:33:22,900] jepsen worker 53 - jepsen.util 6623 :ok :write [17 47] INFO [2019-12-17 12:33:22,900] jepsen worker 53 - jepsen.util 6623 :invoke :write [17 44] INFO [2019-12-17 12:33:22,912] jepsen worker 74 - jepsen.util 7184 :fail :cas [24 [19 6]] :ValueNotSwapFrom INFO [2019-12-17 12:33:22,912] jepsen worker 74 - jepsen.util 7184 :invoke :cas [24 [40 16]] INFO [2019-12-17 12:33:22,913] jepsen worker 46 - jepsen.util 7246 :info :write [15 22] :RequestCanceledException INFO [2019-12-17 12:33:22,913] jepsen worker 11 - jepsen.util 7211 :fail :cas [3 [49 38]] :ValueNotSwapFrom INFO [2019-12-17 12:33:22,913] jepsen worker 11 - jepsen.util 7211 :invoke :write [3 24] INFO [2019-12-17 12:33:22,914] jepsen worker 46 - jepsen.util 7336 :invoke :cas [15 [28 2]] INFO [2019-12-17 12:33:22,920] jepsen worker 29 - jepsen.util 5429 :fail :cas [9 [22 6]] :ValueNotSwapFrom INFO [2019-12-17 12:33:22,920] jepsen worker 29 - jepsen.util 5429 :invoke :read [9 nil] INFO [2019-12-17 12:33:22,925] jepsen worker 74 - jepsen.util 7184 :fail :cas [24 [40 16]] :ValueNotSwapFrom INFO [2019-12-17 12:33:22,926] jepsen worker 74 - jepsen.util 7184 :invoke :read [24 nil] INFO [2019-12-17 12:33:22,940] jepsen worker 32 - jepsen.util 7052 :ok :read [10 16] INFO [2019-12-17 12:33:22,940] jepsen worker 32 - jepsen.util 7052 :invoke :write [10 23] INFO [2019-12-17 12:33:22,953] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40628,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:33:22,953] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40628,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:33:22,953] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40630,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:33:22,953] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40630,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:33:22,959] jepsen worker 63 - jepsen.util 6093 :ok :write [21 42] INFO [2019-12-17 12:33:22,959] jepsen worker 63 - jepsen.util 6093 :invoke :cas [21 [24 44]] INFO [2019-12-17 12:33:22,959] jepsen worker 18 - jepsen.util 6948 :info :write [6 7] :RequestCanceledException INFO [2019-12-17 12:33:22,960] jepsen worker 18 - jepsen.util 7038 :invoke :cas [6 [38 16]] INFO [2019-12-17 12:33:22,973] jepsen worker 74 - jepsen.util 7184 :ok :read [24 26] INFO [2019-12-17 12:33:22,973] jepsen worker 74 - jepsen.util 7184 :invoke :read [24 nil] INFO [2019-12-17 12:33:22,990] jepsen worker 18 - jepsen.util 7038 :fail :cas [6 [38 16]] :ValueNotSwapFrom INFO [2019-12-17 12:33:22,991] jepsen worker 18 - jepsen.util 7038 :invoke :write [6 36] INFO [2019-12-17 12:33:22,991] jepsen worker 89 - jepsen.util 6839 :ok :read [29 24] INFO [2019-12-17 12:33:22,991] jepsen worker 89 - jepsen.util 6839 :invoke :read [29 nil] INFO [2019-12-17 12:33:22,994] jepsen worker 34 - jepsen.util 9034 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:33:22,994] jepsen worker 34 - jepsen.util 9034 :invoke :cas [11 [6 24]] INFO [2019-12-17 12:33:23,013] jepsen worker 4 - jepsen.util 6304 :info :write [1 35] :RequestCanceledException INFO [2019-12-17 12:33:23,014] jepsen worker 4 - jepsen.util 6394 :invoke :write [1 27] INFO [2019-12-17 12:33:23,015] jepsen worker 36 - jepsen.util 7416 :ok :read [12 28] INFO [2019-12-17 12:33:23,015] jepsen worker 36 - jepsen.util 7416 :invoke :read [12 nil] INFO [2019-12-17 12:33:23,027] jepsen worker 74 - jepsen.util 7184 :ok :read [24 26] INFO [2019-12-17 12:33:23,044] jepsen worker 63 - jepsen.util 6093 :fail :cas [21 [24 44]] :ValueNotSwapFrom INFO [2019-12-17 12:33:23,050] jepsen worker 65 - jepsen.util 5915 :ok :write [21 40] INFO [2019-12-17 12:33:23,058] jepsen worker 74 - jepsen.util 7184 :invoke :read [24 nil] INFO [2019-12-17 12:33:23,058] jepsen worker 63 - jepsen.util 6093 :invoke :cas [21 [21 20]] INFO [2019-12-17 12:33:23,058] jepsen worker 65 - jepsen.util 5915 :invoke :read [21 nil] INFO [2019-12-17 12:33:23,059] jepsen worker 13 - jepsen.util 6673 :info :write [4 48] :RequestCanceledException INFO [2019-12-17 12:33:23,059] jepsen worker 64 - jepsen.util 5374 :info :write [21 29] :RequestCanceledException INFO [2019-12-17 12:33:23,059] jepsen worker 43 - jepsen.util 8863 :info :cas [14 [35 18]] :RequestCanceledException INFO [2019-12-17 12:33:23,059] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40647,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:33:23,059] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40647,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:33:23,059] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40650,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:33:23,060] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40650,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:33:23,060] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40651,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:33:23,060] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40651,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:33:23,060] jepsen worker 43 - jepsen.util 8953 :invoke :write [14 11] INFO [2019-12-17 12:33:23,060] jepsen worker 64 - jepsen.util 5464 :invoke :cas [21 [24 8]] INFO [2019-12-17 12:33:23,060] jepsen worker 13 - jepsen.util 6763 :invoke :cas [4 [0 45]] INFO [2019-12-17 12:33:23,086] jepsen worker 65 - jepsen.util 5915 :ok :read [21 40] INFO [2019-12-17 12:33:23,086] jepsen worker 63 - jepsen.util 6093 :fail :cas [21 [21 20]] :ValueNotSwapFrom INFO [2019-12-17 12:33:23,094] jepsen worker 63 - jepsen.util 6093 :invoke :write [21 24] INFO [2019-12-17 12:33:23,094] jepsen worker 65 - jepsen.util 5915 :invoke :cas [21 [22 14]] INFO [2019-12-17 12:33:23,095] jepsen worker 23 - jepsen.util 9563 :info :cas [7 [38 38]] :RequestCanceledException INFO [2019-12-17 12:33:23,098] jepsen worker 23 - jepsen.util 9653 :invoke :write [7 42] INFO [2019-12-17 12:33:23,100] jepsen worker 64 - jepsen.util 5464 :fail :cas [21 [24 8]] :ValueNotSwapFrom INFO [2019-12-17 12:33:23,104] jepsen worker 54 - jepsen.util 7884 :ok :read [18 5] INFO [2019-12-17 12:33:23,112] jepsen worker 75 - jepsen.util 9075 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:33:23,121] jepsen worker 65 - jepsen.util 5915 :fail :cas [21 [22 14]] :ValueNotSwapFrom INFO [2019-12-17 12:33:23,128] jepsen worker 37 - jepsen.util 6607 :info :write [12 10] :RequestCanceledException INFO [2019-12-17 12:33:23,128] jepsen worker 73 - jepsen.util 6643 :info :cas [24 [29 17]] :RequestCanceledException INFO [2019-12-17 12:33:23,128] jepsen worker 28 - jepsen.util 5518 :fail :read [9 nil] :RequestTimeoutException INFO [2019-12-17 12:33:23,128] jepsen worker 64 - jepsen.util 5464 :invoke :read [21 nil] INFO [2019-12-17 12:33:23,128] jepsen worker 54 - jepsen.util 7884 :invoke :read [18 nil] INFO [2019-12-17 12:33:23,129] jepsen worker 75 - jepsen.util 9075 :invoke :write [25 27] INFO [2019-12-17 12:33:23,129] jepsen worker 65 - jepsen.util 5915 :invoke :cas [21 [34 21]] INFO [2019-12-17 12:33:23,129] jepsen worker 28 - jepsen.util 5518 :invoke :read [9 nil] INFO [2019-12-17 12:33:23,129] jepsen worker 37 - jepsen.util 6697 :invoke :cas [12 [31 19]] INFO [2019-12-17 12:33:23,129] jepsen worker 73 - jepsen.util 6733 :invoke :read [24 nil] INFO [2019-12-17 12:33:23,142] jepsen worker 58 - jepsen.util 10138 :info :write [19 1] :RequestCanceledException INFO [2019-12-17 12:33:23,142] jepsen worker 21 - jepsen.util 9021 :info :write [7 26] :RequestCanceledException INFO [2019-12-17 12:33:23,145] jepsen worker 88 - jepsen.util 7018 :fail :cas [29 [49 0]] :ValueNotSwapFrom INFO [2019-12-17 12:33:23,147] jepsen worker 88 - jepsen.util 7018 :invoke :read [29 nil] INFO [2019-12-17 12:33:23,148] jepsen worker 58 - jepsen.util 10228 :invoke :cas [19 [3 35]] INFO [2019-12-17 12:33:23,148] jepsen worker 21 - jepsen.util 9111 :invoke :read [7 nil] INFO [2019-12-17 12:33:23,160] jepsen worker 42 - jepsen.util 10122 :info :write [14 44] :RequestCanceledException INFO [2019-12-17 12:33:23,161] jepsen worker 42 - jepsen.util 10212 :invoke :write [14 9] INFO [2019-12-17 12:33:23,165] jepsen worker 65 - jepsen.util 5915 :fail :cas [21 [34 21]] :ValueNotSwapFrom INFO [2019-12-17 12:33:23,165] jepsen worker 65 - jepsen.util 5915 :invoke :cas [21 [11 14]] INFO [2019-12-17 12:33:23,179] jepsen worker 87 - jepsen.util 6297 :fail :cas [29 [30 21]] :ValueNotSwapFrom INFO [2019-12-17 12:33:23,179] jepsen worker 28 - jepsen.util 5518 :ok :read [9 31] INFO [2019-12-17 12:33:23,179] jepsen worker 87 - jepsen.util 6297 :invoke :read [29 nil] INFO [2019-12-17 12:33:23,179] jepsen worker 28 - jepsen.util 5518 :invoke :cas [9 [19 40]] INFO [2019-12-17 12:33:23,194] jepsen worker 62 - jepsen.util 9422 :info :write [20 43] :RequestCanceledException INFO [2019-12-17 12:33:23,194] jepsen worker 33 - jepsen.util 9303 :info :write [11 35] :RequestCanceledException INFO [2019-12-17 12:33:23,194] jepsen worker 66 - jepsen.util 9246 :info :cas [22 [37 22]] :RequestCanceledException INFO [2019-12-17 12:33:23,194] jepsen worker 31 - jepsen.util 6331 :info :write [10 14] :RequestCanceledException INFO [2019-12-17 12:33:23,196] jepsen worker 31 - jepsen.util 6421 :invoke :write [10 15] INFO [2019-12-17 12:33:23,196] jepsen worker 33 - jepsen.util 9393 :invoke :read [11 nil] INFO [2019-12-17 12:33:23,196] jepsen worker 62 - jepsen.util 9512 :invoke :read [20 nil] INFO [2019-12-17 12:33:23,196] jepsen worker 66 - jepsen.util 9336 :invoke :write [22 15] INFO [2019-12-17 12:33:23,210] jepsen worker 68 - jepsen.util 8978 :info :write [22 42] :RequestCanceledException INFO [2019-12-17 12:33:23,210] jepsen worker 67 - jepsen.util 9697 :info :write [22 4] :RequestCanceledException INFO [2019-12-17 12:33:23,211] jepsen worker 68 - jepsen.util 9068 :invoke :read [22 nil] INFO [2019-12-17 12:33:23,211] jepsen worker 67 - jepsen.util 9787 :invoke :read [22 nil] INFO [2019-12-17 12:33:23,214] jepsen worker 65 - jepsen.util 5915 :fail :cas [21 [11 14]] :ValueNotSwapFrom INFO [2019-12-17 12:33:23,215] jepsen worker 65 - jepsen.util 5915 :invoke :read [21 nil] INFO [2019-12-17 12:33:23,227] jepsen worker 41 - jepsen.util 10031 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:33:23,227] jepsen worker 77 - jepsen.util 9797 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:33:23,227] jepsen worker 50 - jepsen.util 9950 :info :cas [16 [22 18]] :RequestCanceledException INFO [2019-12-17 12:33:23,227] jepsen worker 77 - jepsen.util 9797 :invoke :read [25 nil] INFO [2019-12-17 12:33:23,227] jepsen worker 41 - jepsen.util 10031 :invoke :write [13 26] INFO [2019-12-17 12:33:23,228] jepsen worker 25 - jepsen.util 9925 :info :cas [8 [46 15]] :RequestCanceledException INFO [2019-12-17 12:33:23,228] jepsen worker 0 - jepsen.util 9900 :info :write [0 41] :RequestCanceledException INFO [2019-12-17 12:33:23,228] jepsen worker 60 - jepsen.util 10320 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:33:23,228] jepsen worker 60 - jepsen.util 10320 :invoke :cas [20 [41 40]] INFO [2019-12-17 12:33:23,228] jepsen worker 50 - jepsen.util 10040 :invoke :cas [16 [18 43]] INFO [2019-12-17 12:33:23,229] jepsen worker 25 - jepsen.util 10015 :invoke :cas [8 [44 42]] INFO [2019-12-17 12:33:23,229] jepsen worker 0 - jepsen.util 9990 :invoke :cas [0 [42 21]] INFO [2019-12-17 12:33:23,242] jepsen worker 26 - jepsen.util 9656 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:33:23,242] jepsen worker 83 - jepsen.util 9983 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:33:23,242] jepsen worker 1 - jepsen.util 9541 :info :cas [0 [36 5]] :RequestCanceledException INFO [2019-12-17 12:33:23,242] jepsen worker 26 - jepsen.util 9656 :invoke :write [8 9] INFO [2019-12-17 12:33:23,243] jepsen worker 83 - jepsen.util 9983 :invoke :cas [27 [46 46]] INFO [2019-12-17 12:33:23,245] jepsen worker 1 - jepsen.util 9631 :invoke :cas [0 [39 28]] INFO [2019-12-17 12:33:23,255] jepsen worker 28 - jepsen.util 5518 :fail :cas [9 [19 40]] :ValueNotSwapFrom INFO [2019-12-17 12:33:23,255] jepsen worker 28 - jepsen.util 5518 :invoke :cas [9 [18 3]] INFO [2019-12-17 12:33:23,258] jepsen worker 22 - jepsen.util 9472 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:33:23,258] jepsen worker 24 - jepsen.util 8304 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:33:23,259] jepsen worker 24 - jepsen.util 8304 :invoke :write [8 27] INFO [2019-12-17 12:33:23,259] jepsen worker 22 - jepsen.util 9472 :invoke :cas [7 [49 39]] INFO [2019-12-17 12:33:23,272] jepsen worker 63 - jepsen.util 6093 :ok :write [21 24] INFO [2019-12-17 12:33:23,272] jepsen worker 63 - jepsen.util 6093 :invoke :cas [21 [18 35]] INFO [2019-12-17 12:33:23,279] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40677,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:33:23,279] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40677,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:33:23,279] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40692,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:33:23,279] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40678,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:33:23,279] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40692,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:33:23,279] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40678,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:33:23,279] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40694,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:33:23,279] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40694,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:33:23,294] jepsen worker 20 - jepsen.util 6770 :info :write [6 5] :RequestCanceledException INFO [2019-12-17 12:33:23,294] jepsen worker 59 - jepsen.util 9419 :info :write [19 38] :RequestCanceledException INFO [2019-12-17 12:33:23,296] jepsen worker 59 - jepsen.util 9509 :invoke :read [19 nil] INFO [2019-12-17 12:33:23,296] jepsen worker 20 - jepsen.util 6860 :invoke :write [6 20] INFO [2019-12-17 12:33:23,312] jepsen worker 86 - jepsen.util 9896 :info :write [28 12] :RequestCanceledException INFO [2019-12-17 12:33:23,313] jepsen worker 15 - jepsen.util 8565 :info :cas [5 [0 2]] :RequestCanceledException INFO [2019-12-17 12:33:23,318] jepsen worker 15 - jepsen.util 8655 :invoke :read [5 nil] INFO [2019-12-17 12:33:23,318] jepsen worker 86 - jepsen.util 9986 :invoke :cas [28 [32 35]] INFO [2019-12-17 12:33:23,326] jepsen worker 28 - jepsen.util 5518 :fail :cas [9 [18 3]] :ValueNotSwapFrom INFO [2019-12-17 12:33:23,326] jepsen worker 28 - jepsen.util 5518 :invoke :cas [9 [48 22]] INFO [2019-12-17 12:33:23,328] jepsen worker 70 - jepsen.util 7270 :info :write [23 4] :RequestCanceledException INFO [2019-12-17 12:33:23,328] jepsen worker 82 - jepsen.util 10162 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:33:23,328] jepsen worker 82 - jepsen.util 10162 :invoke :cas [27 [28 26]] INFO [2019-12-17 12:33:23,333] jepsen worker 70 - jepsen.util 7360 :invoke :read [23 nil] INFO [2019-12-17 12:33:23,342] jepsen worker 30 - jepsen.util 6870 :info :cas [10 [19 14]] :RequestCanceledException INFO [2019-12-17 12:33:23,343] jepsen worker 30 - jepsen.util 6960 :invoke :read [10 nil] INFO [2019-12-17 12:33:23,360] jepsen worker 64 - jepsen.util 5464 :ok :read [21 24] INFO [2019-12-17 12:33:23,361] jepsen worker 64 - jepsen.util 5464 :invoke :cas [21 [22 25]] INFO [2019-12-17 12:33:23,367] jepsen worker 63 - jepsen.util 6093 :fail :cas [21 [18 35]] :ValueNotSwapFrom INFO [2019-12-17 12:33:23,368] jepsen worker 63 - jepsen.util 6093 :invoke :write [21 24] INFO [2019-12-17 12:33:23,378] jepsen worker 37 - jepsen.util 6697 :fail :cas [12 [31 19]] :ValueNotSwapFrom INFO [2019-12-17 12:33:23,378] jepsen worker 37 - jepsen.util 6697 :invoke :read [12 nil] INFO [2019-12-17 12:33:23,395] jepsen worker 45 - jepsen.util 6975 :info :cas [15 [1 36]] :RequestCanceledException INFO [2019-12-17 12:33:23,395] jepsen worker 27 - jepsen.util 7047 :info :cas [9 [32 5]] :RequestCanceledException INFO [2019-12-17 12:33:23,398] jepsen worker 45 - jepsen.util 7065 :invoke :cas [15 [35 25]] INFO [2019-12-17 12:33:23,398] jepsen worker 27 - jepsen.util 7137 :invoke :read [9 nil] INFO [2019-12-17 12:33:23,406] jepsen worker 28 - jepsen.util 5518 :fail :cas [9 [48 22]] :ValueNotSwapFrom INFO [2019-12-17 12:33:23,406] jepsen worker 28 - jepsen.util 5518 :invoke :read [9 nil] INFO [2019-12-17 12:33:23,414] jepsen worker 30 - jepsen.util 6960 :ok :read [10 16] INFO [2019-12-17 12:33:23,414] jepsen worker 30 - jepsen.util 6960 :invoke :cas [10 [41 46]] INFO [2019-12-17 12:33:23,415] jepsen worker 11 - jepsen.util 7211 :ok :write [3 24] INFO [2019-12-17 12:33:23,415] jepsen worker 11 - jepsen.util 7211 :invoke :write [3 12] INFO [2019-12-17 12:33:23,429] jepsen worker 64 - jepsen.util 5464 :fail :cas [21 [22 25]] :ValueNotSwapFrom INFO [2019-12-17 12:33:23,429] jepsen worker 64 - jepsen.util 5464 :invoke :read [21 nil] INFO [2019-12-17 12:33:23,430] jepsen worker 79 - jepsen.util 7009 :ok :write [26 31] INFO [2019-12-17 12:33:23,431] jepsen worker 79 - jepsen.util 7009 :invoke :read [26 nil] INFO [2019-12-17 12:33:23,446] jepsen worker 32 - jepsen.util 7052 :ok :write [10 23] INFO [2019-12-17 12:33:23,447] jepsen worker 32 - jepsen.util 7052 :invoke :write [10 16] INFO [2019-12-17 12:33:23,465] jepsen worker 30 - jepsen.util 6960 :fail :cas [10 [41 46]] :ValueNotSwapFrom INFO [2019-12-17 12:33:23,465] jepsen worker 30 - jepsen.util 6960 :invoke :read [10 nil] INFO [2019-12-17 12:33:23,470] jepsen worker 54 - jepsen.util 7884 :ok :read [18 5] INFO [2019-12-17 12:33:23,470] jepsen worker 54 - jepsen.util 7884 :invoke :cas [18 [2 39]] INFO [2019-12-17 12:33:23,486] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:33:23,486] jepsen nemesis - couchbase.util Rebalance check count: 311 INFO [2019-12-17 12:33:23,486] jepsen nemesis - couchbase.util Rebalance stuck count: 292 INFO [2019-12-17 12:33:23,488] jepsen worker 28 - jepsen.util 5518 :ok :read [9 31] INFO [2019-12-17 12:33:23,488] jepsen worker 28 - jepsen.util 5518 :invoke :write [9 48] INFO [2019-12-17 12:33:23,488] jepsen worker 18 - jepsen.util 7038 :ok :write [6 36] INFO [2019-12-17 12:33:23,489] jepsen worker 18 - jepsen.util 7038 :invoke :read [6 nil] INFO [2019-12-17 12:33:23,497] jepsen worker 5 - jepsen.util 6665 :ok :write [1 47] INFO [2019-12-17 12:33:23,497] jepsen worker 5 - jepsen.util 6665 :invoke :read [1 nil] INFO [2019-12-17 12:33:23,498] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40726,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:33:23,498] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40726,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:33:23,498] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40729,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:33:23,499] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40729,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:33:23,504] jepsen worker 69 - jepsen.util 7179 :ok :read [23 41] INFO [2019-12-17 12:33:23,505] jepsen worker 69 - jepsen.util 7179 :invoke :read [23 nil] INFO [2019-12-17 12:33:23,528] jepsen worker 76 - jepsen.util 9796 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:33:23,528] jepsen worker 76 - jepsen.util 9796 :invoke :cas [25 [10 7]] INFO [2019-12-17 12:33:23,530] jepsen worker 88 - jepsen.util 7018 :ok :read [29 24] INFO [2019-12-17 12:33:23,530] jepsen worker 88 - jepsen.util 7018 :invoke :read [29 nil] INFO [2019-12-17 12:33:23,536] jepsen worker 87 - jepsen.util 6297 :ok :read [29 24] INFO [2019-12-17 12:33:23,536] jepsen worker 87 - jepsen.util 6297 :invoke :read [29 nil] INFO [2019-12-17 12:33:23,559] jepsen worker 73 - jepsen.util 6733 :ok :read [24 26] INFO [2019-12-17 12:33:23,559] jepsen worker 73 - jepsen.util 6733 :invoke :write [24 37] INFO [2019-12-17 12:33:23,559] jepsen worker 14 - jepsen.util 7214 :info :write [4 33] :RequestCanceledException INFO [2019-12-17 12:33:23,565] jepsen worker 14 - jepsen.util 7304 :invoke :cas [4 [45 25]] INFO [2019-12-17 12:33:23,586] jepsen worker 30 - jepsen.util 6960 :ok :read [10 23] INFO [2019-12-17 12:33:23,586] jepsen worker 74 - jepsen.util 7184 :ok :read [24 26] INFO [2019-12-17 12:33:23,586] jepsen worker 30 - jepsen.util 6960 :invoke :cas [10 [42 28]] INFO [2019-12-17 12:33:23,586] jepsen worker 74 - jepsen.util 7184 :invoke :write [24 46] INFO [2019-12-17 12:33:23,589] jepsen worker 63 - jepsen.util 6093 :ok :write [21 24] INFO [2019-12-17 12:33:23,589] jepsen worker 63 - jepsen.util 6093 :invoke :write [21 13] INFO [2019-12-17 12:33:23,601] jepsen worker 18 - jepsen.util 7038 :ok :read [6 36] INFO [2019-12-17 12:33:23,602] jepsen worker 18 - jepsen.util 7038 :invoke :read [6 nil] INFO [2019-12-17 12:33:23,629] jepsen worker 5 - jepsen.util 6665 :ok :read [1 47] INFO [2019-12-17 12:33:23,630] jepsen worker 5 - jepsen.util 6665 :invoke :write [1 1] INFO [2019-12-17 12:33:23,672] jepsen worker 70 - jepsen.util 7360 :ok :read [23 41] INFO [2019-12-17 12:33:23,672] jepsen worker 70 - jepsen.util 7360 :invoke :write [23 22] INFO [2019-12-17 12:33:23,691] jepsen worker 17 - jepsen.util 9827 :info :cas [5 [14 33]] :RequestCanceledException INFO [2019-12-17 12:33:23,697] jepsen worker 17 - jepsen.util 9917 :invoke :write [5 17] INFO [2019-12-17 12:33:23,712] jepsen worker 72 - jepsen.util 6732 :info :write [24 6] :RequestCanceledException INFO [2019-12-17 12:33:23,712] jepsen worker 6 - jepsen.util 10176 :info :cas [2 [28 17]] :RequestCanceledException INFO [2019-12-17 12:33:23,712] jepsen worker 39 - jepsen.util 9399 :info :write [13 33] :RequestCanceledException INFO [2019-12-17 12:33:23,718] jepsen worker 6 - jepsen.util 10266 :invoke :write [2 31] INFO [2019-12-17 12:33:23,718] jepsen worker 72 - jepsen.util 6822 :invoke :read [24 nil] INFO [2019-12-17 12:33:23,718] jepsen worker 79 - jepsen.util 7009 :ok :read [26 31] INFO [2019-12-17 12:33:23,718] jepsen worker 39 - jepsen.util 9489 :invoke :cas [13 [1 42]] INFO [2019-12-17 12:33:23,718] jepsen worker 79 - jepsen.util 7009 :invoke :write [26 30] INFO [2019-12-17 12:33:23,718] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40750,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:33:23,718] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40750,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:33:23,719] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40754,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:33:23,719] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40754,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:33:23,719] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40755,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:33:23,719] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40755,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:33:23,724] jepsen worker 64 - jepsen.util 5464 :ok :read [21 24] INFO [2019-12-17 12:33:23,724] jepsen worker 64 - jepsen.util 5464 :invoke :write [21 12] INFO [2019-12-17 12:33:23,728] jepsen worker 84 - jepsen.util 9084 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:33:23,728] jepsen worker 84 - jepsen.util 9084 :invoke :write [28 13] INFO [2019-12-17 12:33:23,728] jepsen worker 12 - jepsen.util 7392 :info :write [4 24] :RequestCanceledException INFO [2019-12-17 12:33:23,734] jepsen worker 12 - jepsen.util 7482 :invoke :write [4 0] INFO [2019-12-17 12:33:23,742] jepsen worker 40 - jepsen.util 10030 :info :write [13 33] :RequestCanceledException INFO [2019-12-17 12:33:23,747] jepsen worker 40 - jepsen.util 10120 :invoke :read [13 nil] INFO [2019-12-17 12:33:23,750] jepsen worker 87 - jepsen.util 6297 :ok :read [29 24] INFO [2019-12-17 12:33:23,750] jepsen worker 87 - jepsen.util 6297 :invoke :write [29 26] INFO [2019-12-17 12:33:23,755] jepsen worker 30 - jepsen.util 6960 :fail :cas [10 [42 28]] :ValueNotSwapFrom INFO [2019-12-17 12:33:23,755] jepsen worker 30 - jepsen.util 6960 :invoke :cas [10 [36 38]] INFO [2019-12-17 12:33:23,759] jepsen worker 81 - jepsen.util 9261 :info :cas [27 [17 19]] :RequestCanceledException INFO [2019-12-17 12:33:23,761] jepsen worker 81 - jepsen.util 9351 :invoke :write [27 5] INFO [2019-12-17 12:33:23,784] jepsen worker 63 - jepsen.util 6093 :ok :write [21 13] INFO [2019-12-17 12:33:23,784] jepsen worker 63 - jepsen.util 6093 :invoke :read [21 nil] INFO [2019-12-17 12:33:23,815] jepsen worker 89 - jepsen.util 6839 :ok :read [29 24] INFO [2019-12-17 12:33:23,815] jepsen worker 89 - jepsen.util 6839 :invoke :read [29 nil] INFO [2019-12-17 12:33:23,825] jepsen worker 72 - jepsen.util 6822 :ok :read [24 26] INFO [2019-12-17 12:33:23,825] jepsen worker 72 - jepsen.util 6822 :invoke :write [24 10] INFO [2019-12-17 12:33:23,828] jepsen worker 10 - jepsen.util 6940 :info :write [3 28] :RequestCanceledException INFO [2019-12-17 12:33:23,832] jepsen worker 10 - jepsen.util 7030 :invoke :write [3 5] INFO [2019-12-17 12:33:23,842] jepsen worker 44 - jepsen.util 10214 :info :write [14 47] :RequestCanceledException INFO [2019-12-17 12:33:23,846] jepsen worker 44 - jepsen.util 10304 :invoke :read [14 nil] INFO [2019-12-17 12:33:23,859] jepsen worker 57 - jepsen.util 9237 :info :cas [19 [33 37]] :RequestCanceledException INFO [2019-12-17 12:33:23,863] jepsen worker 57 - jepsen.util 9327 :invoke :cas [19 [18 5]] INFO [2019-12-17 12:33:23,866] jepsen worker 36 - jepsen.util 7416 :ok :read [12 28] INFO [2019-12-17 12:33:23,867] jepsen worker 36 - jepsen.util 7416 :invoke :read [12 nil] INFO [2019-12-17 12:33:23,894] jepsen worker 48 - jepsen.util 8688 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:33:23,894] jepsen worker 16 - jepsen.util 10726 :info :write [5 46] :RequestCanceledException INFO [2019-12-17 12:33:23,895] jepsen worker 48 - jepsen.util 8688 :invoke :write [16 16] INFO [2019-12-17 12:33:23,896] jepsen worker 30 - jepsen.util 6960 :fail :cas [10 [36 38]] :ValueNotSwapFrom INFO [2019-12-17 12:33:23,896] jepsen worker 30 - jepsen.util 6960 :invoke :write [10 6] INFO [2019-12-17 12:33:23,900] jepsen worker 16 - jepsen.util 10816 :invoke :write [5 20] INFO [2019-12-17 12:33:23,911] jepsen worker 51 - jepsen.util 6891 :ok :write [17 38] INFO [2019-12-17 12:33:23,911] jepsen worker 51 - jepsen.util 6891 :invoke :cas [17 [34 48]] INFO [2019-12-17 12:33:23,912] jepsen worker 61 - jepsen.util 10321 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:33:23,913] jepsen worker 8 - jepsen.util 9458 :info :write [2 47] :RequestCanceledException INFO [2019-12-17 12:33:23,913] jepsen worker 61 - jepsen.util 10321 :invoke :write [20 19] INFO [2019-12-17 12:33:23,915] jepsen worker 8 - jepsen.util 9548 :invoke :write [2 29] INFO [2019-12-17 12:33:23,927] jepsen worker 7 - jepsen.util 9817 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:33:23,927] jepsen worker 7 - jepsen.util 9817 :invoke :read [2 nil] INFO [2019-12-17 12:33:23,929] jepsen worker 19 - jepsen.util 7039 :fail :read [6 nil] :RequestTimeoutException INFO [2019-12-17 12:33:23,929] jepsen worker 80 - jepsen.util 7010 :info :write [26 26] :RequestCanceledException INFO [2019-12-17 12:33:23,929] jepsen worker 19 - jepsen.util 7039 :invoke :cas [6 [43 36]] INFO [2019-12-17 12:33:23,930] jepsen worker 80 - jepsen.util 7100 :invoke :cas [26 [20 39]] INFO [2019-12-17 12:33:23,939] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40777,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:33:23,939] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40777,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:33:23,939] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40779,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:33:23,939] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40779,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:33:23,941] jepsen worker 28 - jepsen.util 5518 :ok :write [9 48] INFO [2019-12-17 12:33:23,941] jepsen worker 49 - jepsen.util 8869 :info :write [16 46] :RequestCanceledException INFO [2019-12-17 12:33:23,941] jepsen worker 28 - jepsen.util 5518 :invoke :write [9 28] INFO [2019-12-17 12:33:23,944] jepsen worker 49 - jepsen.util 8959 :invoke :write [16 4] INFO [2019-12-17 12:33:23,960] jepsen worker 64 - jepsen.util 5464 :ok :write [21 12] INFO [2019-12-17 12:33:23,960] jepsen worker 64 - jepsen.util 5464 :invoke :cas [21 [10 44]] INFO [2019-12-17 12:33:23,979] jepsen worker 19 - jepsen.util 7039 :fail :cas [6 [43 36]] :ValueNotSwapFrom INFO [2019-12-17 12:33:23,979] jepsen worker 19 - jepsen.util 7039 :invoke :read [6 nil] INFO [2019-12-17 12:33:23,990] jepsen worker 85 - jepsen.util 10435 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:33:23,991] jepsen worker 85 - jepsen.util 10435 :invoke :write [28 44] INFO [2019-12-17 12:33:24,002] jepsen worker 64 - jepsen.util 5464 :fail :cas [21 [10 44]] :ValueNotSwapFrom INFO [2019-12-17 12:33:24,002] jepsen worker 64 - jepsen.util 5464 :invoke :write [21 2] INFO [2019-12-17 12:33:24,017] jepsen worker 89 - jepsen.util 6839 :ok :read [29 24] INFO [2019-12-17 12:33:24,017] jepsen worker 89 - jepsen.util 6839 :invoke :read [29 nil] INFO [2019-12-17 12:33:24,027] jepsen worker 9 - jepsen.util 7569 :info :write [3 43] :RequestCanceledException INFO [2019-12-17 12:33:24,030] jepsen worker 9 - jepsen.util 7659 :invoke :cas [3 [25 0]] INFO [2019-12-17 12:33:24,031] jepsen worker 13 - jepsen.util 6763 :fail :cas [4 [0 45]] :ValueNotSwapFrom INFO [2019-12-17 12:33:24,031] jepsen worker 13 - jepsen.util 6763 :invoke :read [4 nil] INFO [2019-12-17 12:33:24,042] jepsen worker 47 - jepsen.util 7517 :fail :read [15 nil] :RequestTimeoutException INFO [2019-12-17 12:33:24,042] jepsen worker 47 - jepsen.util 7517 :invoke :read [15 nil] INFO [2019-12-17 12:33:24,046] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40798,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:33:24,046] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40798,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:33:24,059] jepsen worker 38 - jepsen.util 7058 :info :write [12 24] :RequestCanceledException INFO [2019-12-17 12:33:24,061] jepsen worker 38 - jepsen.util 7148 :invoke :cas [12 [18 15]] INFO [2019-12-17 12:33:24,080] jepsen worker 9 - jepsen.util 7659 :fail :cas [3 [25 0]] :ValueNotSwapFrom INFO [2019-12-17 12:33:24,080] jepsen worker 9 - jepsen.util 7659 :invoke :cas [3 [44 14]] INFO [2019-12-17 12:33:24,085] jepsen worker 80 - jepsen.util 7100 :fail :cas [26 [20 39]] :ValueNotSwapFrom INFO [2019-12-17 12:33:24,085] jepsen worker 80 - jepsen.util 7100 :invoke :cas [26 [47 47]] INFO [2019-12-17 12:33:24,107] jepsen worker 20 - jepsen.util 6860 :ok :write [6 20] INFO [2019-12-17 12:33:24,108] jepsen worker 20 - jepsen.util 6860 :invoke :cas [6 [28 31]] INFO [2019-12-17 12:33:24,108] jepsen worker 32 - jepsen.util 7052 :ok :write [10 16] INFO [2019-12-17 12:33:24,108] jepsen worker 32 - jepsen.util 7052 :invoke :write [10 18] INFO [2019-12-17 12:33:24,109] jepsen worker 9 - jepsen.util 7659 :fail :cas [3 [44 14]] :ValueNotSwapFrom INFO [2019-12-17 12:33:24,109] jepsen worker 9 - jepsen.util 7659 :invoke :write [3 1] INFO [2019-12-17 12:33:24,147] jepsen worker 20 - jepsen.util 6860 :fail :cas [6 [28 31]] :ValueNotSwapFrom INFO [2019-12-17 12:33:24,147] jepsen worker 28 - jepsen.util 5518 :ok :write [9 28] INFO [2019-12-17 12:33:24,147] jepsen worker 20 - jepsen.util 6860 :invoke :read [6 nil] INFO [2019-12-17 12:33:24,147] jepsen worker 28 - jepsen.util 5518 :invoke :read [9 nil] INFO [2019-12-17 12:33:24,155] jepsen worker 74 - jepsen.util 7184 :ok :write [24 46] INFO [2019-12-17 12:33:24,156] jepsen worker 74 - jepsen.util 7184 :invoke :write [24 38] INFO [2019-12-17 12:33:24,165] jepsen worker 54 - jepsen.util 7884 :fail :cas [18 [2 39]] :ValueNotSwapFrom INFO [2019-12-17 12:33:24,166] jepsen worker 54 - jepsen.util 7884 :invoke :cas [18 [34 34]] INFO [2019-12-17 12:33:24,170] jepsen worker 51 - jepsen.util 6891 :fail :cas [17 [34 48]] :ValueNotSwapFrom INFO [2019-12-17 12:33:24,171] jepsen worker 51 - jepsen.util 6891 :invoke :write [17 23] INFO [2019-12-17 12:33:24,195] jepsen worker 35 - jepsen.util 9305 :info :cas [11 [49 34]] :RequestCanceledException INFO [2019-12-17 12:33:24,199] jepsen worker 35 - jepsen.util 9395 :invoke :cas [11 [24 0]] INFO [2019-12-17 12:33:24,213] jepsen worker 88 - jepsen.util 7018 :ok :read [29 24] INFO [2019-12-17 12:33:24,213] jepsen worker 88 - jepsen.util 7018 :invoke :write [29 0] INFO [2019-12-17 12:33:24,227] jepsen worker 52 - jepsen.util 7252 :info :write [17 44] :RequestCanceledException INFO [2019-12-17 12:33:24,231] jepsen worker 52 - jepsen.util 7342 :invoke :read [17 nil] INFO [2019-12-17 12:33:24,259] jepsen worker 56 - jepsen.util 7796 :info :write [18 22] :RequestCanceledException INFO [2019-12-17 12:33:24,265] jepsen worker 56 - jepsen.util 7886 :invoke :cas [18 [15 13]] INFO [2019-12-17 12:33:24,266] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40826,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:33:24,266] jepsen worker 20 - jepsen.util 6860 :ok :read [6 20] INFO [2019-12-17 12:33:24,266] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40818,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:33:24,266] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40826,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:33:24,266] jepsen worker 20 - jepsen.util 6860 :invoke :read [6 nil] INFO [2019-12-17 12:33:24,266] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40818,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:33:24,267] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40829,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:33:24,267] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40820,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:33:24,267] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40829,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:33:24,267] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40820,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:33:24,267] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40830,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:33:24,268] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40830,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:33:24,268] jepsen worker 28 - jepsen.util 5518 :ok :read [9 28] INFO [2019-12-17 12:33:24,268] jepsen worker 28 - jepsen.util 5518 :invoke :write [9 6] INFO [2019-12-17 12:33:24,283] jepsen worker 18 - jepsen.util 7038 :ok :read [6 20] INFO [2019-12-17 12:33:24,283] jepsen worker 18 - jepsen.util 7038 :invoke :read [6 nil] INFO [2019-12-17 12:33:24,285] jepsen worker 37 - jepsen.util 6697 :ok :read [12 28] INFO [2019-12-17 12:33:24,285] jepsen worker 37 - jepsen.util 6697 :invoke :write [12 15] INFO [2019-12-17 12:33:24,303] jepsen worker 38 - jepsen.util 7148 :fail :cas [12 [18 15]] :ValueNotSwapFrom INFO [2019-12-17 12:33:24,303] jepsen worker 38 - jepsen.util 7148 :invoke :write [12 47] INFO [2019-12-17 12:33:24,317] jepsen worker 64 - jepsen.util 5464 :ok :write [21 2] INFO [2019-12-17 12:33:24,317] jepsen worker 64 - jepsen.util 5464 :invoke :write [21 47] INFO [2019-12-17 12:33:24,328] jepsen worker 2 - jepsen.util 9092 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:33:24,329] jepsen worker 2 - jepsen.util 9092 :invoke :cas [0 [6 27]] INFO [2019-12-17 12:33:24,334] jepsen worker 47 - jepsen.util 7517 :ok :read [15 24] INFO [2019-12-17 12:33:24,334] jepsen worker 47 - jepsen.util 7517 :invoke :read [15 nil] INFO [2019-12-17 12:33:24,365] jepsen worker 20 - jepsen.util 6860 :ok :read [6 20] INFO [2019-12-17 12:33:24,365] jepsen worker 20 - jepsen.util 6860 :invoke :write [6 35] INFO [2019-12-17 12:33:24,367] jepsen worker 18 - jepsen.util 7038 :ok :read [6 20] INFO [2019-12-17 12:33:24,367] jepsen worker 18 - jepsen.util 7038 :invoke :cas [6 [20 19]] INFO [2019-12-17 12:33:24,391] jepsen worker 47 - jepsen.util 7517 :ok :read [15 24] INFO [2019-12-17 12:33:24,391] jepsen worker 47 - jepsen.util 7517 :invoke :read [15 nil] INFO [2019-12-17 12:33:24,426] jepsen worker 78 - jepsen.util 7188 :info :write [26 45] :RequestCanceledException INFO [2019-12-17 12:33:24,430] jepsen worker 78 - jepsen.util 7278 :invoke :read [26 nil] INFO [2019-12-17 12:33:24,454] jepsen worker 69 - jepsen.util 7179 :ok :read [23 41] INFO [2019-12-17 12:33:24,454] jepsen worker 69 - jepsen.util 7179 :invoke :write [23 46] INFO [2019-12-17 12:33:24,486] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40846,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:33:24,486] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40846,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:33:24,486] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40851,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:33:24,486] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40851,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:33:24,500] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:33:24,500] jepsen nemesis - couchbase.util Rebalance check count: 312 INFO [2019-12-17 12:33:24,500] jepsen nemesis - couchbase.util Rebalance stuck count: 293 INFO [2019-12-17 12:33:24,513] jepsen worker 55 - jepsen.util 8065 :info :write [18 40] :RequestCanceledException INFO [2019-12-17 12:33:24,515] jepsen worker 55 - jepsen.util 8155 :invoke :read [18 nil] INFO [2019-12-17 12:33:24,539] jepsen worker 28 - jepsen.util 5518 :ok :write [9 6] INFO [2019-12-17 12:33:24,539] jepsen worker 28 - jepsen.util 5518 :invoke :write [9 37] INFO [2019-12-17 12:33:24,541] jepsen worker 3 - jepsen.util 6933 :info :write [1 39] :RequestCanceledException INFO [2019-12-17 12:33:24,542] jepsen worker 71 - jepsen.util 7811 :info :write [23 47] :RequestCanceledException INFO [2019-12-17 12:33:24,545] jepsen worker 3 - jepsen.util 7023 :invoke :write [1 16] INFO [2019-12-17 12:33:24,546] jepsen worker 71 - jepsen.util 7901 :invoke :write [23 3] INFO [2019-12-17 12:33:24,552] jepsen worker 47 - jepsen.util 7517 :ok :read [15 24] INFO [2019-12-17 12:33:24,552] jepsen worker 47 - jepsen.util 7517 :invoke :read [15 nil] INFO [2019-12-17 12:33:24,562] jepsen worker 80 - jepsen.util 7100 :fail :cas [26 [47 47]] :ValueNotSwapFrom INFO [2019-12-17 12:33:24,563] jepsen worker 80 - jepsen.util 7100 :invoke :cas [26 [17 20]] INFO [2019-12-17 12:33:24,572] jepsen worker 36 - jepsen.util 7416 :ok :read [12 28] INFO [2019-12-17 12:33:24,572] jepsen worker 36 - jepsen.util 7416 :invoke :read [12 nil] INFO [2019-12-17 12:33:24,576] jepsen worker 64 - jepsen.util 5464 :ok :write [21 47] INFO [2019-12-17 12:33:24,576] jepsen worker 64 - jepsen.util 5464 :invoke :cas [21 [33 13]] INFO [2019-12-17 12:33:24,589] jepsen worker 47 - jepsen.util 7517 :ok :read [15 24] INFO [2019-12-17 12:33:24,589] jepsen worker 47 - jepsen.util 7517 :invoke :read [15 nil] INFO [2019-12-17 12:33:24,604] jepsen worker 14 - jepsen.util 7304 :ok :cas [4 [45 25]] INFO [2019-12-17 12:33:24,604] jepsen worker 14 - jepsen.util 7304 :invoke :cas [4 [0 14]] INFO [2019-12-17 12:33:24,672] jepsen worker 13 - jepsen.util 6763 :ok :read [4 25] INFO [2019-12-17 12:33:24,672] jepsen worker 13 - jepsen.util 6763 :invoke :cas [4 [49 47]] INFO [2019-12-17 12:33:24,689] jepsen worker 56 - jepsen.util 7886 :fail :cas [18 [15 13]] :ValueNotSwapFrom INFO [2019-12-17 12:33:24,689] jepsen worker 56 - jepsen.util 7886 :invoke :read [18 nil] INFO [2019-12-17 12:33:24,704] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40865,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:33:24,705] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40865,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:33:24,705] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40870,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:33:24,705] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40870,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:33:24,712] jepsen worker 64 - jepsen.util 5464 :fail :cas [21 [33 13]] :ValueNotSwapFrom INFO [2019-12-17 12:33:24,712] jepsen worker 64 - jepsen.util 5464 :invoke :write [21 27] INFO [2019-12-17 12:33:24,729] jepsen worker 5 - jepsen.util 6665 :ok :write [1 1] INFO [2019-12-17 12:33:24,729] jepsen worker 5 - jepsen.util 6665 :invoke :write [1 44] INFO [2019-12-17 12:33:24,729] jepsen worker 47 - jepsen.util 7517 :ok :read [15 24] INFO [2019-12-17 12:33:24,729] jepsen worker 47 - jepsen.util 7517 :invoke :read [15 nil] INFO [2019-12-17 12:33:24,741] jepsen worker 73 - jepsen.util 6733 :ok :write [24 37] INFO [2019-12-17 12:33:24,741] jepsen worker 73 - jepsen.util 6733 :invoke :write [24 0] INFO [2019-12-17 12:33:24,753] jepsen worker 9 - jepsen.util 7659 :ok :write [3 1] INFO [2019-12-17 12:33:24,753] jepsen worker 9 - jepsen.util 7659 :invoke :cas [3 [41 0]] INFO [2019-12-17 12:33:24,771] jepsen worker 20 - jepsen.util 6860 :ok :write [6 35] INFO [2019-12-17 12:33:24,771] jepsen worker 20 - jepsen.util 6860 :invoke :cas [6 [9 16]] INFO [2019-12-17 12:33:24,775] jepsen worker 47 - jepsen.util 7517 :ok :read [15 24] INFO [2019-12-17 12:33:24,776] jepsen worker 47 - jepsen.util 7517 :invoke :read [15 nil] INFO [2019-12-17 12:33:24,795] jepsen worker 9 - jepsen.util 7659 :fail :cas [3 [41 0]] :ValueNotSwapFrom INFO [2019-12-17 12:33:24,796] jepsen worker 9 - jepsen.util 7659 :invoke :read [3 nil] INFO [2019-12-17 12:33:24,822] jepsen worker 37 - jepsen.util 6697 :ok :write [12 15] INFO [2019-12-17 12:33:24,822] jepsen worker 37 - jepsen.util 6697 :invoke :cas [12 [19 4]] INFO [2019-12-17 12:33:24,824] jepsen worker 20 - jepsen.util 6860 :fail :cas [6 [9 16]] :ValueNotSwapFrom INFO [2019-12-17 12:33:24,825] jepsen worker 20 - jepsen.util 6860 :invoke :write [6 21] INFO [2019-12-17 12:33:24,826] jepsen worker 47 - jepsen.util 7517 :ok :read [15 24] INFO [2019-12-17 12:33:24,826] jepsen worker 47 - jepsen.util 7517 :invoke :write [15 15] INFO [2019-12-17 12:33:24,847] jepsen worker 28 - jepsen.util 5518 :ok :write [9 37] INFO [2019-12-17 12:33:24,848] jepsen worker 28 - jepsen.util 5518 :invoke :write [9 40] INFO [2019-12-17 12:33:24,861] jepsen worker 9 - jepsen.util 7659 :ok :read [3 1] INFO [2019-12-17 12:33:24,862] jepsen worker 9 - jepsen.util 7659 :invoke :write [3 24] INFO [2019-12-17 12:33:24,868] jepsen worker 64 - jepsen.util 5464 :ok :write [21 27] INFO [2019-12-17 12:33:24,869] jepsen worker 64 - jepsen.util 5464 :invoke :read [21 nil] INFO [2019-12-17 12:33:24,886] jepsen worker 80 - jepsen.util 7100 :fail :cas [26 [17 20]] :ValueNotSwapFrom INFO [2019-12-17 12:33:24,886] jepsen worker 80 - jepsen.util 7100 :invoke :read [26 nil] INFO [2019-12-17 12:33:24,903] jepsen worker 89 - jepsen.util 6839 :ok :read [29 24] INFO [2019-12-17 12:33:24,903] jepsen worker 89 - jepsen.util 6839 :invoke :read [29 nil] INFO [2019-12-17 12:33:24,923] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40885,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:33:24,924] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40885,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:33:24,924] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40888,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:33:24,924] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40888,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:33:24,926] jepsen worker 46 - jepsen.util 7336 :info :cas [15 [28 2]] :RequestCanceledException INFO [2019-12-17 12:33:24,927] jepsen worker 53 - jepsen.util 6623 :ok :write [17 44] INFO [2019-12-17 12:33:24,927] jepsen worker 53 - jepsen.util 6623 :invoke :write [17 14] INFO [2019-12-17 12:33:24,933] jepsen worker 46 - jepsen.util 7426 :invoke :write [15 45] INFO [2019-12-17 12:33:24,940] jepsen worker 56 - jepsen.util 7886 :ok :read [18 5] INFO [2019-12-17 12:33:24,941] jepsen worker 56 - jepsen.util 7886 :invoke :cas [18 [10 8]] INFO [2019-12-17 12:33:24,963] jepsen worker 64 - jepsen.util 5464 :ok :read [21 27] INFO [2019-12-17 12:33:24,964] jepsen worker 64 - jepsen.util 5464 :invoke :read [21 nil] INFO [2019-12-17 12:33:24,967] jepsen worker 54 - jepsen.util 7884 :fail :cas [18 [34 34]] :ValueNotSwapFrom INFO [2019-12-17 12:33:24,967] jepsen worker 54 - jepsen.util 7884 :invoke :write [18 43] INFO [2019-12-17 12:33:25,096] jepsen worker 29 - jepsen.util 5429 :fail :read [9 nil] :RequestTimeoutException INFO [2019-12-17 12:33:25,096] jepsen worker 4 - jepsen.util 6394 :info :write [1 27] :RequestCanceledException INFO [2019-12-17 12:33:25,096] jepsen worker 29 - jepsen.util 5429 :invoke :cas [9 [14 34]] INFO [2019-12-17 12:33:25,096] jepsen worker 34 - jepsen.util 9034 :info :cas [11 [6 24]] :RequestCanceledException INFO [2019-12-17 12:33:25,097] jepsen worker 4 - jepsen.util 6484 :invoke :write [1 16] INFO [2019-12-17 12:33:25,097] jepsen worker 34 - jepsen.util 9124 :invoke :read [11 nil] INFO [2019-12-17 12:33:25,104] jepsen worker 64 - jepsen.util 5464 :ok :read [21 27] INFO [2019-12-17 12:33:25,104] jepsen worker 64 - jepsen.util 5464 :invoke :cas [21 [28 45]] INFO [2019-12-17 12:33:25,127] jepsen worker 43 - jepsen.util 8953 :info :write [14 11] :RequestCanceledException INFO [2019-12-17 12:33:25,130] jepsen worker 43 - jepsen.util 9043 :invoke :read [14 nil] INFO [2019-12-17 12:33:25,154] jepsen worker 37 - jepsen.util 6697 :fail :cas [12 [19 4]] :ValueNotSwapFrom INFO [2019-12-17 12:33:25,154] jepsen worker 37 - jepsen.util 6697 :invoke :cas [12 [1 14]] INFO [2019-12-17 12:33:25,156] jepsen worker 64 - jepsen.util 5464 :fail :cas [21 [28 45]] :ValueNotSwapFrom INFO [2019-12-17 12:33:25,156] jepsen worker 64 - jepsen.util 5464 :invoke :write [21 25] INFO [2019-12-17 12:33:25,159] jepsen worker 28 - jepsen.util 5518 :ok :write [9 40] INFO [2019-12-17 12:33:25,159] jepsen worker 28 - jepsen.util 5518 :invoke :write [9 23] INFO [2019-12-17 12:33:25,164] jepsen worker 70 - jepsen.util 7360 :ok :write [23 22] INFO [2019-12-17 12:33:25,164] jepsen worker 70 - jepsen.util 7360 :invoke :write [23 48] INFO [2019-12-17 12:33:25,177] jepsen worker 52 - jepsen.util 7342 :ok :read [17 38] INFO [2019-12-17 12:33:25,177] jepsen worker 52 - jepsen.util 7342 :invoke :read [17 nil] INFO [2019-12-17 12:33:25,195] jepsen worker 21 - jepsen.util 9111 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:33:25,195] jepsen worker 21 - jepsen.util 9111 :invoke :write [7 18] INFO [2019-12-17 12:33:25,205] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40900,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:33:25,205] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40917,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:33:25,205] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40900,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:33:25,205] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40917,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:33:25,205] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40904,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:33:25,205] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40904,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:33:25,212] jepsen worker 75 - jepsen.util 9075 :info :write [25 27] :RequestCanceledException INFO [2019-12-17 12:33:25,212] jepsen worker 42 - jepsen.util 10212 :info :write [14 9] :RequestCanceledException INFO [2019-12-17 12:33:25,213] jepsen worker 42 - jepsen.util 10302 :invoke :write [14 42] INFO [2019-12-17 12:33:25,213] jepsen worker 75 - jepsen.util 9165 :invoke :write [25 7] INFO [2019-12-17 12:33:25,224] jepsen worker 78 - jepsen.util 7278 :ok :read [26 31] INFO [2019-12-17 12:33:25,224] jepsen worker 78 - jepsen.util 7278 :invoke :cas [26 [47 31]] INFO [2019-12-17 12:33:25,228] jepsen worker 31 - jepsen.util 6421 :info :write [10 15] :RequestCanceledException INFO [2019-12-17 12:33:25,228] jepsen worker 58 - jepsen.util 10228 :info :cas [19 [3 35]] :RequestCanceledException INFO [2019-12-17 12:33:25,228] jepsen worker 65 - jepsen.util 5915 :fail :read [21 nil] :RequestTimeoutException INFO [2019-12-17 12:33:25,228] jepsen worker 23 - jepsen.util 9653 :info :write [7 42] :RequestCanceledException INFO [2019-12-17 12:33:25,228] jepsen worker 66 - jepsen.util 9336 :info :write [22 15] :RequestCanceledException INFO [2019-12-17 12:33:25,228] jepsen worker 64 - jepsen.util 5464 :ok :write [21 25] INFO [2019-12-17 12:33:25,228] jepsen worker 65 - jepsen.util 5915 :invoke :write [21 25] INFO [2019-12-17 12:33:25,228] jepsen worker 64 - jepsen.util 5464 :invoke :read [21 nil] INFO [2019-12-17 12:33:25,229] jepsen worker 31 - jepsen.util 6511 :invoke :cas [10 [18 40]] INFO [2019-12-17 12:33:25,229] jepsen worker 58 - jepsen.util 10318 :invoke :write [19 18] INFO [2019-12-17 12:33:25,229] jepsen worker 23 - jepsen.util 9743 :invoke :write [7 32] INFO [2019-12-17 12:33:25,229] jepsen worker 66 - jepsen.util 9426 :invoke :cas [22 [37 32]] INFO [2019-12-17 12:33:25,241] jepsen worker 33 - jepsen.util 9393 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:33:25,241] jepsen worker 25 - jepsen.util 10015 :info :cas [8 [44 42]] :RequestCanceledException INFO [2019-12-17 12:33:25,241] jepsen worker 33 - jepsen.util 9393 :invoke :read [11 nil] INFO [2019-12-17 12:33:25,242] jepsen worker 25 - jepsen.util 10105 :invoke :read [8 nil] INFO [2019-12-17 12:33:25,249] jepsen worker 28 - jepsen.util 5518 :ok :write [9 23] INFO [2019-12-17 12:33:25,249] jepsen worker 28 - jepsen.util 5518 :invoke :cas [9 [20 2]] INFO [2019-12-17 12:33:25,254] jepsen worker 64 - jepsen.util 5464 :ok :read [21 25] INFO [2019-12-17 12:33:25,254] jepsen worker 64 - jepsen.util 5464 :invoke :cas [21 [30 13]] INFO [2019-12-17 12:33:25,259] jepsen worker 62 - jepsen.util 9512 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:33:25,259] jepsen worker 0 - jepsen.util 9990 :info :cas [0 [42 21]] :RequestCanceledException INFO [2019-12-17 12:33:25,260] jepsen worker 62 - jepsen.util 9512 :invoke :write [20 40] INFO [2019-12-17 12:33:25,261] jepsen worker 0 - jepsen.util 10080 :invoke :cas [0 [3 1]] INFO [2019-12-17 12:33:25,267] jepsen worker 14 - jepsen.util 7304 :fail :cas [4 [0 14]] :ValueNotSwapFrom INFO [2019-12-17 12:33:25,267] jepsen worker 14 - jepsen.util 7304 :invoke :read [4 nil] INFO [2019-12-17 12:33:25,293] jepsen worker 55 - jepsen.util 8155 :ok :read [18 5] INFO [2019-12-17 12:33:25,293] jepsen worker 68 - jepsen.util 9068 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:33:25,293] jepsen worker 55 - jepsen.util 8155 :invoke :write [18 5] INFO [2019-12-17 12:33:25,293] jepsen worker 68 - jepsen.util 9068 :invoke :write [22 32] INFO [2019-12-17 12:33:25,313] jepsen worker 67 - jepsen.util 9787 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:33:25,313] jepsen worker 67 - jepsen.util 9787 :invoke :read [22 nil] INFO [2019-12-17 12:33:25,314] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40924,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:33:25,314] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40924,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:33:25,315] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40927,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:33:25,315] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40927,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:33:25,319] jepsen worker 80 - jepsen.util 7100 :ok :read [26 31] INFO [2019-12-17 12:33:25,319] jepsen worker 80 - jepsen.util 7100 :invoke :cas [26 [1 10]] INFO [2019-12-17 12:33:25,323] jepsen worker 31 - jepsen.util 6511 :fail :cas [10 [18 40]] :ValueNotSwapFrom INFO [2019-12-17 12:33:25,323] jepsen worker 31 - jepsen.util 6511 :invoke :read [10 nil] INFO [2019-12-17 12:33:25,324] jepsen worker 64 - jepsen.util 5464 :fail :cas [21 [30 13]] :ValueNotSwapFrom INFO [2019-12-17 12:33:25,324] jepsen worker 64 - jepsen.util 5464 :invoke :write [21 19] INFO [2019-12-17 12:33:25,327] jepsen worker 77 - jepsen.util 9797 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:33:25,327] jepsen worker 1 - jepsen.util 9631 :info :cas [0 [39 28]] :RequestCanceledException INFO [2019-12-17 12:33:25,327] jepsen worker 60 - jepsen.util 10320 :info :cas [20 [41 40]] :RequestCanceledException INFO [2019-12-17 12:33:25,327] jepsen worker 41 - jepsen.util 10031 :info :write [13 26] :RequestCanceledException INFO [2019-12-17 12:33:25,327] jepsen worker 50 - jepsen.util 10040 :info :cas [16 [18 43]] :RequestCanceledException INFO [2019-12-17 12:33:25,328] jepsen worker 77 - jepsen.util 9797 :invoke :write [25 2] INFO [2019-12-17 12:33:25,329] jepsen worker 41 - jepsen.util 10121 :invoke :read [13 nil] INFO [2019-12-17 12:33:25,329] jepsen worker 50 - jepsen.util 10130 :invoke :write [16 38] INFO [2019-12-17 12:33:25,329] jepsen worker 60 - jepsen.util 10410 :invoke :read [20 nil] INFO [2019-12-17 12:33:25,329] jepsen worker 1 - jepsen.util 9721 :invoke :read [0 nil] INFO [2019-12-17 12:33:25,342] jepsen worker 26 - jepsen.util 9656 :info :write [8 9] :RequestCanceledException INFO [2019-12-17 12:33:25,342] jepsen worker 86 - jepsen.util 9986 :info :cas [28 [32 35]] :RequestCanceledException INFO [2019-12-17 12:33:25,342] jepsen worker 83 - jepsen.util 9983 :info :cas [27 [46 46]] :RequestCanceledException INFO [2019-12-17 12:33:25,343] jepsen worker 83 - jepsen.util 10073 :invoke :read [27 nil] INFO [2019-12-17 12:33:25,343] jepsen worker 86 - jepsen.util 10076 :invoke :cas [28 [40 13]] INFO [2019-12-17 12:33:25,343] jepsen worker 26 - jepsen.util 9746 :invoke :cas [8 [1 26]] INFO [2019-12-17 12:33:25,348] jepsen worker 56 - jepsen.util 7886 :fail :cas [18 [10 8]] :ValueNotSwapFrom INFO [2019-12-17 12:33:25,348] jepsen worker 56 - jepsen.util 7886 :invoke :cas [18 [43 2]] INFO [2019-12-17 12:33:25,356] jepsen worker 22 - jepsen.util 9472 :info :cas [7 [49 39]] :RequestCanceledException INFO [2019-12-17 12:33:25,356] jepsen worker 24 - jepsen.util 8304 :info :write [8 27] :RequestCanceledException INFO [2019-12-17 12:33:25,357] jepsen worker 22 - jepsen.util 9562 :invoke :read [7 nil] INFO [2019-12-17 12:33:25,357] jepsen worker 24 - jepsen.util 8394 :invoke :read [8 nil] INFO [2019-12-17 12:33:25,363] jepsen worker 20 - jepsen.util 6860 :ok :write [6 21] INFO [2019-12-17 12:33:25,364] jepsen worker 20 - jepsen.util 6860 :invoke :cas [6 [17 46]] INFO [2019-12-17 12:33:25,392] jepsen worker 29 - jepsen.util 5429 :fail :cas [9 [14 34]] :ValueNotSwapFrom INFO [2019-12-17 12:33:25,392] jepsen worker 29 - jepsen.util 5429 :invoke :cas [9 [22 43]] INFO [2019-12-17 12:33:25,395] jepsen worker 59 - jepsen.util 9509 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:33:25,395] jepsen worker 59 - jepsen.util 9509 :invoke :cas [19 [15 37]] INFO [2019-12-17 12:33:25,396] jepsen worker 65 - jepsen.util 5915 :ok :write [21 25] INFO [2019-12-17 12:33:25,396] jepsen worker 65 - jepsen.util 5915 :invoke :write [21 8] INFO [2019-12-17 12:33:25,408] jepsen worker 36 - jepsen.util 7416 :ok :read [12 15] INFO [2019-12-17 12:33:25,409] jepsen worker 36 - jepsen.util 7416 :invoke :read [12 nil] INFO [2019-12-17 12:33:25,412] jepsen worker 27 - jepsen.util 7137 :fail :read [9 nil] :RequestTimeoutException INFO [2019-12-17 12:33:25,413] jepsen worker 27 - jepsen.util 7137 :invoke :write [9 28] INFO [2019-12-17 12:33:25,421] jepsen worker 20 - jepsen.util 6860 :fail :cas [6 [17 46]] :ValueNotSwapFrom INFO [2019-12-17 12:33:25,421] jepsen worker 20 - jepsen.util 6860 :invoke :write [6 14] INFO [2019-12-17 12:33:25,428] jepsen worker 82 - jepsen.util 10162 :info :cas [27 [28 26]] :RequestCanceledException INFO [2019-12-17 12:33:25,428] jepsen worker 15 - jepsen.util 8655 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:33:25,428] jepsen worker 15 - jepsen.util 8655 :invoke :cas [5 [19 37]] INFO [2019-12-17 12:33:25,429] jepsen worker 82 - jepsen.util 10252 :invoke :write [27 0] INFO [2019-12-17 12:33:25,436] jepsen worker 31 - jepsen.util 6511 :ok :read [10 16] INFO [2019-12-17 12:33:25,436] jepsen worker 31 - jepsen.util 6511 :invoke :read [10 nil] INFO [2019-12-17 12:33:25,455] jepsen worker 38 - jepsen.util 7148 :ok :write [12 47] INFO [2019-12-17 12:33:25,455] jepsen worker 38 - jepsen.util 7148 :invoke :write [12 35] INFO [2019-12-17 12:33:25,456] jepsen worker 29 - jepsen.util 5429 :fail :cas [9 [22 43]] :ValueNotSwapFrom INFO [2019-12-17 12:33:25,456] jepsen worker 29 - jepsen.util 5429 :invoke :write [9 1] INFO [2019-12-17 12:33:25,468] jepsen worker 89 - jepsen.util 6839 :ok :read [29 24] INFO [2019-12-17 12:33:25,468] jepsen worker 89 - jepsen.util 6839 :invoke :cas [29 [26 48]] INFO [2019-12-17 12:33:25,469] jepsen worker 87 - jepsen.util 6297 :ok :write [29 26] INFO [2019-12-17 12:33:25,469] jepsen worker 87 - jepsen.util 6297 :invoke :cas [29 [31 14]] INFO [2019-12-17 12:33:25,482] jepsen worker 4 - jepsen.util 6484 :ok :write [1 16] INFO [2019-12-17 12:33:25,482] jepsen worker 4 - jepsen.util 6484 :invoke :cas [1 [49 8]] INFO [2019-12-17 12:33:25,495] jepsen worker 45 - jepsen.util 7065 :info :cas [15 [35 25]] :RequestCanceledException INFO [2019-12-17 12:33:25,496] jepsen worker 45 - jepsen.util 7155 :invoke :write [15 41] INFO [2019-12-17 12:33:25,512] jepsen worker 11 - jepsen.util 7211 :info :write [3 12] :RequestCanceledException INFO [2019-12-17 12:33:25,513] jepsen worker 11 - jepsen.util 7301 :invoke :cas [3 [42 11]] INFO [2019-12-17 12:33:25,514] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:33:25,514] jepsen nemesis - couchbase.util Rebalance check count: 313 INFO [2019-12-17 12:33:25,514] jepsen nemesis - couchbase.util Rebalance stuck count: 294 INFO [2019-12-17 12:33:25,527] jepsen worker 80 - jepsen.util 7100 :fail :cas [26 [1 10]] :ValueNotSwapFrom INFO [2019-12-17 12:33:25,527] jepsen worker 80 - jepsen.util 7100 :invoke :cas [26 [21 28]] INFO [2019-12-17 12:33:25,528] jepsen worker 27 - jepsen.util 7137 :ok :write [9 28] INFO [2019-12-17 12:33:25,528] jepsen worker 27 - jepsen.util 7137 :invoke :read [9 nil] INFO [2019-12-17 12:33:25,533] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40971,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:33:25,533] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40971,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:33:25,533] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40974,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:33:25,533] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40974,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:33:25,533] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40975,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:33:25,533] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40975,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:33:25,539] jepsen worker 65 - jepsen.util 5915 :ok :write [21 8] INFO [2019-12-17 12:33:25,539] jepsen worker 65 - jepsen.util 5915 :invoke :read [21 nil] INFO [2019-12-17 12:33:25,554] jepsen worker 65 - jepsen.util 5915 :ok :read [21 8] INFO [2019-12-17 12:33:25,554] jepsen worker 65 - jepsen.util 5915 :invoke :read [21 nil] INFO [2019-12-17 12:33:25,560] jepsen worker 53 - jepsen.util 6623 :ok :write [17 14] INFO [2019-12-17 12:33:25,560] jepsen worker 53 - jepsen.util 6623 :invoke :cas [17 [17 17]] INFO [2019-12-17 12:33:25,565] jepsen worker 37 - jepsen.util 6697 :fail :cas [12 [1 14]] :ValueNotSwapFrom INFO [2019-12-17 12:33:25,565] jepsen worker 37 - jepsen.util 6697 :invoke :write [12 32] INFO [2019-12-17 12:33:25,578] jepsen worker 56 - jepsen.util 7886 :fail :cas [18 [43 2]] :ValueNotSwapFrom INFO [2019-12-17 12:33:25,578] jepsen worker 56 - jepsen.util 7886 :invoke :read [18 nil] INFO [2019-12-17 12:33:25,583] jepsen worker 65 - jepsen.util 5915 :ok :read [21 8] INFO [2019-12-17 12:33:25,583] jepsen worker 65 - jepsen.util 5915 :invoke :cas [21 [8 19]] INFO [2019-12-17 12:33:25,628] jepsen worker 76 - jepsen.util 9796 :info :cas [25 [10 7]] :RequestCanceledException INFO [2019-12-17 12:33:25,629] jepsen worker 76 - jepsen.util 9886 :invoke :read [25 nil] INFO [2019-12-17 12:33:25,634] jepsen worker 29 - jepsen.util 5429 :ok :write [9 1] INFO [2019-12-17 12:33:25,634] jepsen worker 29 - jepsen.util 5429 :invoke :read [9 nil] INFO [2019-12-17 12:33:25,649] jepsen worker 4 - jepsen.util 6484 :fail :cas [1 [49 8]] :ValueNotSwapFrom INFO [2019-12-17 12:33:25,649] jepsen worker 4 - jepsen.util 6484 :invoke :read [1 nil] INFO [2019-12-17 12:33:25,677] jepsen worker 80 - jepsen.util 7100 :fail :cas [26 [21 28]] :ValueNotSwapFrom INFO [2019-12-17 12:33:25,677] jepsen worker 80 - jepsen.util 7100 :invoke :cas [26 [19 47]] INFO [2019-12-17 12:33:25,690] jepsen worker 29 - jepsen.util 5429 :ok :read [9 1] INFO [2019-12-17 12:33:25,690] jepsen worker 29 - jepsen.util 5429 :invoke :write [9 49] INFO [2019-12-17 12:33:25,711] jepsen worker 55 - jepsen.util 8155 :ok :write [18 5] INFO [2019-12-17 12:33:25,711] jepsen worker 55 - jepsen.util 8155 :invoke :cas [18 [37 1]] INFO [2019-12-17 12:33:25,712] jepsen worker 73 - jepsen.util 6733 :ok :write [24 0] INFO [2019-12-17 12:33:25,712] jepsen worker 73 - jepsen.util 6733 :invoke :read [24 nil] INFO [2019-12-17 12:33:25,713] jepsen worker 12 - jepsen.util 7482 :ok :write [4 0] INFO [2019-12-17 12:33:25,713] jepsen worker 12 - jepsen.util 7482 :invoke :write [4 37] INFO [2019-12-17 12:33:25,716] jepsen worker 47 - jepsen.util 7517 :ok :write [15 15] INFO [2019-12-17 12:33:25,716] jepsen worker 47 - jepsen.util 7517 :invoke :read [15 nil] INFO [2019-12-17 12:33:25,732] jepsen worker 80 - jepsen.util 7100 :fail :cas [26 [19 47]] :ValueNotSwapFrom INFO [2019-12-17 12:33:25,732] jepsen worker 80 - jepsen.util 7100 :invoke :write [26 16] INFO [2019-12-17 12:33:25,737] jepsen worker 9 - jepsen.util 7659 :ok :write [3 24] INFO [2019-12-17 12:33:25,738] jepsen worker 9 - jepsen.util 7659 :invoke :cas [3 [49 39]] INFO [2019-12-17 12:33:25,741] jepsen worker 47 - jepsen.util 7517 :ok :read [15 15] INFO [2019-12-17 12:33:25,741] jepsen worker 47 - jepsen.util 7517 :invoke :cas [15 [43 49]] INFO [2019-12-17 12:33:25,742] jepsen worker 17 - jepsen.util 9917 :info :write [5 17] :RequestCanceledException INFO [2019-12-17 12:33:25,743] jepsen worker 17 - jepsen.util 10007 :invoke :cas [5 [17 33]] INFO [2019-12-17 12:33:25,743] jepsen worker 73 - jepsen.util 6733 :ok :read [24 0] INFO [2019-12-17 12:33:25,743] jepsen worker 73 - jepsen.util 6733 :invoke :write [24 22] INFO [2019-12-17 12:33:25,752] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40997,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:33:25,752] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":40997,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:33:25,752] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41000,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:33:25,752] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41000,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:33:25,758] jepsen worker 52 - jepsen.util 7342 :ok :read [17 14] INFO [2019-12-17 12:33:25,758] jepsen worker 52 - jepsen.util 7342 :invoke :write [17 35] INFO [2019-12-17 12:33:25,758] jepsen worker 79 - jepsen.util 7009 :info :write [26 30] :RequestCanceledException INFO [2019-12-17 12:33:25,758] jepsen worker 6 - jepsen.util 10266 :info :write [2 31] :RequestCanceledException INFO [2019-12-17 12:33:25,759] jepsen worker 79 - jepsen.util 7099 :invoke :cas [26 [37 21]] INFO [2019-12-17 12:33:25,760] jepsen worker 6 - jepsen.util 10356 :invoke :read [2 nil] INFO [2019-12-17 12:33:25,774] jepsen worker 5 - jepsen.util 6665 :ok :write [1 44] INFO [2019-12-17 12:33:25,774] jepsen worker 5 - jepsen.util 6665 :invoke :cas [1 [49 3]] INFO [2019-12-17 12:33:25,786] jepsen worker 14 - jepsen.util 7304 :ok :read [4 0] INFO [2019-12-17 12:33:25,786] jepsen worker 14 - jepsen.util 7304 :invoke :cas [4 [28 39]] INFO [2019-12-17 12:33:25,794] jepsen worker 39 - jepsen.util 9489 :info :cas [13 [1 42]] :RequestCanceledException INFO [2019-12-17 12:33:25,794] jepsen worker 63 - jepsen.util 6093 :fail :read [21 nil] :RequestTimeoutException INFO [2019-12-17 12:33:25,795] jepsen worker 63 - jepsen.util 6093 :invoke :read [21 nil] INFO [2019-12-17 12:33:25,795] jepsen worker 39 - jepsen.util 9579 :invoke :read [13 nil] INFO [2019-12-17 12:33:25,805] jepsen worker 47 - jepsen.util 7517 :fail :cas [15 [43 49]] :ValueNotSwapFrom INFO [2019-12-17 12:33:25,805] jepsen worker 47 - jepsen.util 7517 :invoke :cas [15 [29 47]] INFO [2019-12-17 12:33:25,811] jepsen worker 9 - jepsen.util 7659 :fail :cas [3 [49 39]] :ValueNotSwapFrom INFO [2019-12-17 12:33:25,811] jepsen worker 9 - jepsen.util 7659 :invoke :write [3 20] INFO [2019-12-17 12:33:25,822] jepsen worker 78 - jepsen.util 7278 :fail :cas [26 [47 31]] :ValueNotSwapFrom INFO [2019-12-17 12:33:25,822] jepsen worker 78 - jepsen.util 7278 :invoke :write [26 19] INFO [2019-12-17 12:33:25,826] jepsen worker 40 - jepsen.util 10120 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:33:25,826] jepsen worker 84 - jepsen.util 9084 :info :write [28 13] :RequestCanceledException INFO [2019-12-17 12:33:25,826] jepsen worker 40 - jepsen.util 10120 :invoke :cas [13 [14 26]] INFO [2019-12-17 12:33:25,828] jepsen worker 13 - jepsen.util 6763 :fail :cas [4 [49 47]] :ValueNotSwapFrom INFO [2019-12-17 12:33:25,838] jepsen worker 13 - jepsen.util 6763 :invoke :write [4 15] INFO [2019-12-17 12:33:25,839] jepsen worker 84 - jepsen.util 9174 :invoke :cas [28 [20 13]] INFO [2019-12-17 12:33:25,841] jepsen worker 81 - jepsen.util 9351 :info :write [27 5] :RequestCanceledException INFO [2019-12-17 12:33:25,843] jepsen worker 81 - jepsen.util 9441 :invoke :write [27 26] INFO [2019-12-17 12:33:25,851] jepsen worker 53 - jepsen.util 6623 :fail :cas [17 [17 17]] :ValueNotSwapFrom INFO [2019-12-17 12:33:25,852] jepsen worker 53 - jepsen.util 6623 :invoke :write [17 10] INFO [2019-12-17 12:33:25,858] jepsen worker 10 - jepsen.util 7030 :info :write [3 5] :RequestCanceledException INFO [2019-12-17 12:33:25,859] jepsen worker 10 - jepsen.util 7120 :invoke :cas [3 [9 26]] INFO [2019-12-17 12:33:25,861] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41031,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:33:25,862] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41031,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:33:25,862] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41035,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:33:25,862] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41035,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:33:25,872] jepsen worker 47 - jepsen.util 7517 :fail :cas [15 [29 47]] :ValueNotSwapFrom INFO [2019-12-17 12:33:25,872] jepsen worker 47 - jepsen.util 7517 :invoke :cas [15 [1 7]] INFO [2019-12-17 12:33:25,879] jepsen worker 29 - jepsen.util 5429 :ok :write [9 49] INFO [2019-12-17 12:33:25,879] jepsen worker 29 - jepsen.util 5429 :invoke :write [9 23] INFO [2019-12-17 12:33:25,883] jepsen worker 10 - jepsen.util 7120 :fail :cas [3 [9 26]] :ValueNotSwapFrom INFO [2019-12-17 12:33:25,884] jepsen worker 10 - jepsen.util 7120 :invoke :read [3 nil] INFO [2019-12-17 12:33:25,890] jepsen worker 47 - jepsen.util 7517 :fail :cas [15 [1 7]] :ValueNotSwapFrom INFO [2019-12-17 12:33:25,890] jepsen worker 47 - jepsen.util 7517 :invoke :write [15 5] INFO [2019-12-17 12:33:25,895] jepsen worker 57 - jepsen.util 9327 :info :cas [19 [18 5]] :RequestCanceledException INFO [2019-12-17 12:33:25,896] jepsen worker 57 - jepsen.util 9417 :invoke :cas [19 [37 16]] INFO [2019-12-17 12:33:25,898] jepsen worker 65 - jepsen.util 5915 :ok :cas [21 [8 19]] INFO [2019-12-17 12:33:25,898] jepsen worker 65 - jepsen.util 5915 :invoke :write [21 2] INFO [2019-12-17 12:33:25,905] jepsen worker 10 - jepsen.util 7120 :ok :read [3 24] INFO [2019-12-17 12:33:25,905] jepsen worker 10 - jepsen.util 7120 :invoke :read [3 nil] INFO [2019-12-17 12:33:25,906] jepsen worker 63 - jepsen.util 6093 :ok :read [21 8] INFO [2019-12-17 12:33:25,906] jepsen worker 63 - jepsen.util 6093 :invoke :read [21 nil] INFO [2019-12-17 12:33:25,924] jepsen worker 56 - jepsen.util 7886 :ok :read [18 5] INFO [2019-12-17 12:33:25,924] jepsen worker 56 - jepsen.util 7886 :invoke :cas [18 [21 38]] INFO [2019-12-17 12:33:25,928] jepsen worker 72 - jepsen.util 6822 :info :write [24 10] :RequestCanceledException INFO [2019-12-17 12:33:25,928] jepsen worker 44 - jepsen.util 10304 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:33:25,928] jepsen worker 30 - jepsen.util 6960 :info :write [10 6] :RequestCanceledException INFO [2019-12-17 12:33:25,928] jepsen worker 44 - jepsen.util 10304 :invoke :read [14 nil] INFO [2019-12-17 12:33:25,929] jepsen worker 72 - jepsen.util 6912 :invoke :cas [24 [47 36]] INFO [2019-12-17 12:33:25,929] jepsen worker 30 - jepsen.util 7050 :invoke :write [10 35] INFO [2019-12-17 12:33:25,938] jepsen worker 63 - jepsen.util 6093 :ok :read [21 19] INFO [2019-12-17 12:33:25,938] jepsen worker 63 - jepsen.util 6093 :invoke :read [21 nil] INFO [2019-12-17 12:33:25,993] jepsen worker 48 - jepsen.util 8688 :info :write [16 16] :RequestCanceledException INFO [2019-12-17 12:33:25,995] jepsen worker 48 - jepsen.util 8778 :invoke :read [16 nil] INFO [2019-12-17 12:33:25,997] jepsen worker 36 - jepsen.util 7416 :ok :read [12 47] INFO [2019-12-17 12:33:25,997] jepsen worker 36 - jepsen.util 7416 :invoke :read [12 nil] INFO [2019-12-17 12:33:26,000] jepsen worker 4 - jepsen.util 6484 :ok :read [1 44] INFO [2019-12-17 12:33:26,000] jepsen worker 4 - jepsen.util 6484 :invoke :write [1 10] INFO [2019-12-17 12:33:26,013] jepsen worker 61 - jepsen.util 10321 :info :write [20 19] :RequestCanceledException INFO [2019-12-17 12:33:26,013] jepsen worker 16 - jepsen.util 10816 :info :write [5 20] :RequestCanceledException INFO [2019-12-17 12:33:26,014] jepsen worker 61 - jepsen.util 10411 :invoke :read [20 nil] INFO [2019-12-17 12:33:26,014] jepsen worker 16 - jepsen.util 10906 :invoke :read [5 nil] INFO [2019-12-17 12:33:26,015] jepsen worker 29 - jepsen.util 5429 :ok :write [9 23] INFO [2019-12-17 12:33:26,016] jepsen worker 29 - jepsen.util 5429 :invoke :read [9 nil] INFO [2019-12-17 12:33:26,027] jepsen worker 7 - jepsen.util 9817 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:33:26,027] jepsen worker 7 - jepsen.util 9817 :invoke :read [2 nil] INFO [2019-12-17 12:33:26,029] jepsen worker 8 - jepsen.util 9548 :info :write [2 29] :RequestCanceledException INFO [2019-12-17 12:33:26,029] jepsen worker 19 - jepsen.util 7039 :fail :read [6 nil] :RequestTimeoutException INFO [2019-12-17 12:33:26,029] jepsen worker 19 - jepsen.util 7039 :invoke :write [6 24] INFO [2019-12-17 12:33:26,030] jepsen worker 8 - jepsen.util 9638 :invoke :cas [2 [24 42]] INFO [2019-12-17 12:33:26,050] jepsen worker 29 - jepsen.util 5429 :ok :read [9 23] INFO [2019-12-17 12:33:26,050] jepsen worker 29 - jepsen.util 5429 :invoke :read [9 nil] INFO [2019-12-17 12:33:26,059] jepsen worker 49 - jepsen.util 8959 :info :write [16 4] :RequestCanceledException INFO [2019-12-17 12:33:26,060] jepsen worker 49 - jepsen.util 9049 :invoke :write [16 45] INFO [2019-12-17 12:33:26,073] jepsen worker 5 - jepsen.util 6665 :fail :cas [1 [49 3]] :ValueNotSwapFrom INFO [2019-12-17 12:33:26,073] jepsen worker 5 - jepsen.util 6665 :invoke :cas [1 [27 3]] INFO [2019-12-17 12:33:26,077] jepsen worker 65 - jepsen.util 5915 :ok :write [21 2] INFO [2019-12-17 12:33:26,077] jepsen worker 65 - jepsen.util 5915 :invoke :read [21 nil] INFO [2019-12-17 12:33:26,083] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41051,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:33:26,083] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41051,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:33:26,085] jepsen worker 29 - jepsen.util 5429 :ok :read [9 23] INFO [2019-12-17 12:33:26,086] jepsen worker 29 - jepsen.util 5429 :invoke :write [9 12] INFO [2019-12-17 12:33:26,094] jepsen worker 85 - jepsen.util 10435 :info :write [28 44] :RequestCanceledException INFO [2019-12-17 12:33:26,095] jepsen worker 85 - jepsen.util 10525 :invoke :cas [28 [33 47]] INFO [2019-12-17 12:33:26,103] jepsen worker 87 - jepsen.util 6297 :fail :cas [29 [31 14]] :ValueNotSwapFrom INFO [2019-12-17 12:33:26,104] jepsen worker 87 - jepsen.util 6297 :invoke :write [29 9] INFO [2019-12-17 12:33:26,109] jepsen worker 55 - jepsen.util 8155 :fail :cas [18 [37 1]] :ValueNotSwapFrom INFO [2019-12-17 12:33:26,109] jepsen worker 55 - jepsen.util 8155 :invoke :cas [18 [9 22]] INFO [2019-12-17 12:33:26,127] jepsen worker 79 - jepsen.util 7099 :fail :cas [26 [37 21]] :ValueNotSwapFrom INFO [2019-12-17 12:33:26,127] jepsen worker 79 - jepsen.util 7099 :invoke :read [26 nil] INFO [2019-12-17 12:33:26,145] jepsen worker 73 - jepsen.util 6733 :ok :write [24 22] INFO [2019-12-17 12:33:26,145] jepsen worker 73 - jepsen.util 6733 :invoke :read [24 nil] INFO [2019-12-17 12:33:26,152] jepsen worker 36 - jepsen.util 7416 :ok :read [12 47] INFO [2019-12-17 12:33:26,152] jepsen worker 36 - jepsen.util 7416 :invoke :write [12 43] INFO [2019-12-17 12:33:26,159] jepsen worker 32 - jepsen.util 7052 :info :write [10 18] :RequestCanceledException INFO [2019-12-17 12:33:26,160] jepsen worker 32 - jepsen.util 7142 :invoke :read [10 nil] INFO [2019-12-17 12:33:26,163] jepsen worker 47 - jepsen.util 7517 :ok :write [15 5] INFO [2019-12-17 12:33:26,163] jepsen worker 47 - jepsen.util 7517 :invoke :write [15 14] INFO [2019-12-17 12:33:26,187] jepsen worker 73 - jepsen.util 6733 :ok :read [24 22] INFO [2019-12-17 12:33:26,187] jepsen worker 73 - jepsen.util 6733 :invoke :read [24 nil] INFO [2019-12-17 12:33:26,212] jepsen worker 35 - jepsen.util 9395 :info :cas [11 [24 0]] :RequestCanceledException INFO [2019-12-17 12:33:26,212] jepsen worker 51 - jepsen.util 6891 :info :write [17 23] :RequestCanceledException INFO [2019-12-17 12:33:26,213] jepsen worker 51 - jepsen.util 6981 :invoke :cas [17 [40 22]] INFO [2019-12-17 12:33:26,213] jepsen worker 35 - jepsen.util 9485 :invoke :cas [11 [19 35]] INFO [2019-12-17 12:33:26,231] jepsen worker 14 - jepsen.util 7304 :fail :cas [4 [28 39]] :ValueNotSwapFrom INFO [2019-12-17 12:33:26,232] jepsen worker 14 - jepsen.util 7304 :invoke :write [4 9] INFO [2019-12-17 12:33:26,241] jepsen worker 73 - jepsen.util 6733 :ok :read [24 22] INFO [2019-12-17 12:33:26,241] jepsen worker 73 - jepsen.util 6733 :invoke :read [24 nil] INFO [2019-12-17 12:33:26,242] jepsen worker 74 - jepsen.util 7184 :info :write [24 38] :RequestCanceledException INFO [2019-12-17 12:33:26,243] jepsen worker 74 - jepsen.util 7274 :invoke :write [24 6] INFO [2019-12-17 12:33:26,257] jepsen worker 73 - jepsen.util 6733 :ok :read [24 22] INFO [2019-12-17 12:33:26,257] jepsen worker 73 - jepsen.util 6733 :invoke :cas [24 [37 29]] INFO [2019-12-17 12:33:26,264] jepsen worker 51 - jepsen.util 6981 :fail :cas [17 [40 22]] :ValueNotSwapFrom INFO [2019-12-17 12:33:26,264] jepsen worker 51 - jepsen.util 6981 :invoke :read [17 nil] INFO [2019-12-17 12:33:26,264] jepsen worker 29 - jepsen.util 5429 :ok :write [9 12] INFO [2019-12-17 12:33:26,264] jepsen worker 29 - jepsen.util 5429 :invoke :cas [9 [19 17]] INFO [2019-12-17 12:33:26,267] jepsen worker 70 - jepsen.util 7360 :ok :write [23 48] INFO [2019-12-17 12:33:26,267] jepsen worker 70 - jepsen.util 7360 :invoke :cas [23 [36 25]] INFO [2019-12-17 12:33:26,284] jepsen worker 32 - jepsen.util 7142 :ok :read [10 16] INFO [2019-12-17 12:33:26,285] jepsen worker 32 - jepsen.util 7142 :invoke :cas [10 [47 1]] INFO [2019-12-17 12:33:26,287] jepsen worker 30 - jepsen.util 7050 :ok :write [10 35] INFO [2019-12-17 12:33:26,287] jepsen worker 30 - jepsen.util 7050 :invoke :write [10 2] INFO [2019-12-17 12:33:26,289] jepsen worker 9 - jepsen.util 7659 :ok :write [3 20] INFO [2019-12-17 12:33:26,289] jepsen worker 9 - jepsen.util 7659 :invoke :write [3 38] INFO [2019-12-17 12:33:26,290] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41079,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:33:26,290] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41088,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:33:26,290] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41079,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:33:26,290] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41088,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:33:26,290] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41080,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:33:26,291] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41092,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:33:26,291] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41080,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:33:26,291] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41092,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:33:26,291] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41081,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:33:26,291] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41081,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:33:26,296] jepsen worker 29 - jepsen.util 5429 :fail :cas [9 [19 17]] :ValueNotSwapFrom INFO [2019-12-17 12:33:26,296] jepsen worker 29 - jepsen.util 5429 :invoke :read [9 nil] INFO [2019-12-17 12:33:26,319] jepsen worker 32 - jepsen.util 7142 :fail :cas [10 [47 1]] :ValueNotSwapFrom INFO [2019-12-17 12:33:26,319] jepsen worker 32 - jepsen.util 7142 :invoke :cas [10 [7 37]] INFO [2019-12-17 12:33:26,327] jepsen worker 88 - jepsen.util 7018 :info :write [29 0] :RequestCanceledException INFO [2019-12-17 12:33:26,328] jepsen worker 88 - jepsen.util 7108 :invoke :read [29 nil] INFO [2019-12-17 12:33:26,329] jepsen worker 29 - jepsen.util 5429 :ok :read [9 12] INFO [2019-12-17 12:33:26,330] jepsen worker 29 - jepsen.util 5429 :invoke :read [9 nil] INFO [2019-12-17 12:33:26,359] jepsen worker 32 - jepsen.util 7142 :fail :cas [10 [7 37]] :ValueNotSwapFrom INFO [2019-12-17 12:33:26,359] jepsen worker 32 - jepsen.util 7142 :invoke :write [10 30] INFO [2019-12-17 12:33:26,402] jepsen worker 19 - jepsen.util 7039 :ok :write [6 24] INFO [2019-12-17 12:33:26,402] jepsen worker 74 - jepsen.util 7274 :ok :write [24 6] INFO [2019-12-17 12:33:26,402] jepsen worker 19 - jepsen.util 7039 :invoke :write [6 36] INFO [2019-12-17 12:33:26,402] jepsen worker 74 - jepsen.util 7274 :invoke :write [24 37] INFO [2019-12-17 12:33:26,411] jepsen worker 5 - jepsen.util 6665 :fail :cas [1 [27 3]] :ValueNotSwapFrom INFO [2019-12-17 12:33:26,411] jepsen worker 5 - jepsen.util 6665 :invoke :write [1 49] INFO [2019-12-17 12:33:26,425] jepsen worker 2 - jepsen.util 9092 :info :cas [0 [6 27]] :RequestCanceledException INFO [2019-12-17 12:33:26,426] jepsen worker 2 - jepsen.util 9182 :invoke :cas [0 [35 10]] INFO [2019-12-17 12:33:26,428] jepsen worker 29 - jepsen.util 5429 :ok :read [9 12] INFO [2019-12-17 12:33:26,428] jepsen worker 29 - jepsen.util 5429 :invoke :write [9 19] INFO [2019-12-17 12:33:26,479] jepsen worker 73 - jepsen.util 6733 :fail :cas [24 [37 29]] :ValueNotSwapFrom INFO [2019-12-17 12:33:26,479] jepsen worker 73 - jepsen.util 6733 :invoke :write [24 40] INFO [2019-12-17 12:33:26,507] jepsen worker 56 - jepsen.util 7886 :fail :cas [18 [21 38]] :ValueNotSwapFrom INFO [2019-12-17 12:33:26,507] jepsen worker 56 - jepsen.util 7886 :invoke :read [18 nil] INFO [2019-12-17 12:33:26,509] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41112,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:33:26,509] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41112,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:33:26,509] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41116,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:33:26,509] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41116,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:33:26,527] jepsen worker 69 - jepsen.util 7179 :info :write [23 46] :RequestCanceledException INFO [2019-12-17 12:33:26,528] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:33:26,528] jepsen nemesis - couchbase.util Rebalance check count: 314 INFO [2019-12-17 12:33:26,528] jepsen nemesis - couchbase.util Rebalance stuck count: 295 INFO [2019-12-17 12:33:26,529] jepsen worker 69 - jepsen.util 7269 :invoke :read [23 nil] INFO [2019-12-17 12:33:26,542] jepsen worker 18 - jepsen.util 7038 :info :cas [6 [20 19]] :RequestCanceledException INFO [2019-12-17 12:33:26,543] jepsen worker 18 - jepsen.util 7128 :invoke :read [6 nil] INFO [2019-12-17 12:33:26,566] jepsen worker 51 - jepsen.util 6981 :ok :read [17 14] INFO [2019-12-17 12:33:26,566] jepsen worker 51 - jepsen.util 6981 :invoke :read [17 nil] INFO [2019-12-17 12:33:26,595] jepsen worker 3 - jepsen.util 7023 :info :write [1 16] :RequestCanceledException INFO [2019-12-17 12:33:26,595] jepsen worker 3 - jepsen.util 7113 :invoke :cas [1 [11 34]] INFO [2019-12-17 12:33:26,612] jepsen worker 71 - jepsen.util 7901 :info :write [23 3] :RequestCanceledException INFO [2019-12-17 12:33:26,612] jepsen worker 71 - jepsen.util 7991 :invoke :write [23 4] INFO [2019-12-17 12:33:26,651] jepsen worker 79 - jepsen.util 7099 :ok :read [26 31] INFO [2019-12-17 12:33:26,651] jepsen worker 79 - jepsen.util 7099 :invoke :write [26 22] INFO [2019-12-17 12:33:26,686] jepsen worker 70 - jepsen.util 7360 :fail :cas [23 [36 25]] :ValueNotSwapFrom INFO [2019-12-17 12:33:26,686] jepsen worker 70 - jepsen.util 7360 :invoke :write [23 22] INFO [2019-12-17 12:33:26,728] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41127,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:33:26,728] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41127,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:33:26,729] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41131,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:33:26,729] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41131,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:33:26,747] jepsen worker 55 - jepsen.util 8155 :fail :cas [18 [9 22]] :ValueNotSwapFrom INFO [2019-12-17 12:33:26,748] jepsen worker 55 - jepsen.util 8155 :invoke :cas [18 [8 3]] INFO [2019-12-17 12:33:26,783] jepsen worker 69 - jepsen.util 7269 :ok :read [23 48] INFO [2019-12-17 12:33:26,783] jepsen worker 69 - jepsen.util 7269 :invoke :read [23 nil] INFO [2019-12-17 12:33:26,836] jepsen worker 69 - jepsen.util 7269 :ok :read [23 48] INFO [2019-12-17 12:33:26,836] jepsen worker 69 - jepsen.util 7269 :invoke :cas [23 [19 12]] INFO [2019-12-17 12:33:26,856] jepsen worker 88 - jepsen.util 7108 :ok :read [29 26] INFO [2019-12-17 12:33:26,858] jepsen worker 88 - jepsen.util 7108 :invoke :read [29 nil] INFO [2019-12-17 12:33:26,914] jepsen worker 51 - jepsen.util 6981 :ok :read [17 14] INFO [2019-12-17 12:33:26,915] jepsen worker 51 - jepsen.util 6981 :invoke :cas [17 [45 39]] INFO [2019-12-17 12:33:26,924] jepsen worker 56 - jepsen.util 7886 :ok :read [18 5] INFO [2019-12-17 12:33:26,924] jepsen worker 56 - jepsen.util 7886 :invoke :write [18 5] INFO [2019-12-17 12:33:26,944] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41139,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:33:26,944] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41139,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:33:26,944] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41142,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:33:26,944] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41142,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:33:26,978] jepsen worker 51 - jepsen.util 6981 :fail :cas [17 [45 39]] :ValueNotSwapFrom INFO [2019-12-17 12:33:26,978] jepsen worker 51 - jepsen.util 6981 :invoke :write [17 16] INFO [2019-12-17 12:33:27,027] jepsen worker 46 - jepsen.util 7426 :info :write [15 45] :RequestCanceledException INFO [2019-12-17 12:33:27,028] jepsen worker 46 - jepsen.util 7516 :invoke :cas [15 [29 41]] INFO [2019-12-17 12:33:27,052] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41148,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:33:27,052] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41148,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:33:27,052] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41151,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:33:27,052] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41151,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:33:27,052] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41152,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:33:27,052] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41152,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:33:27,075] jepsen worker 55 - jepsen.util 8155 :fail :cas [18 [8 3]] :ValueNotSwapFrom INFO [2019-12-17 12:33:27,075] jepsen worker 55 - jepsen.util 8155 :invoke :write [18 30] INFO [2019-12-17 12:33:27,119] jepsen worker 88 - jepsen.util 7108 :ok :read [29 26] INFO [2019-12-17 12:33:27,119] jepsen worker 88 - jepsen.util 7108 :invoke :read [29 nil] INFO [2019-12-17 12:33:27,128] jepsen worker 54 - jepsen.util 7884 :info :write [18 43] :RequestCanceledException INFO [2019-12-17 12:33:27,129] jepsen worker 54 - jepsen.util 7974 :invoke :read [18 nil] INFO [2019-12-17 12:33:27,159] jepsen worker 34 - jepsen.util 9124 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:33:27,159] jepsen worker 34 - jepsen.util 9124 :invoke :read [11 nil] INFO [2019-12-17 12:33:27,166] jepsen worker 3 - jepsen.util 7113 :fail :cas [1 [11 34]] :ValueNotSwapFrom INFO [2019-12-17 12:33:27,166] jepsen worker 3 - jepsen.util 7113 :invoke :write [1 28] INFO [2019-12-17 12:33:27,228] jepsen worker 43 - jepsen.util 9043 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:33:27,228] jepsen worker 43 - jepsen.util 9043 :invoke :cas [14 [7 1]] INFO [2019-12-17 12:33:27,259] jepsen worker 23 - jepsen.util 9743 :info :write [7 32] :RequestCanceledException INFO [2019-12-17 12:33:27,260] jepsen worker 23 - jepsen.util 9833 :invoke :write [7 32] INFO [2019-12-17 12:33:27,264] jepsen worker 54 - jepsen.util 7974 :ok :read [18 5] INFO [2019-12-17 12:33:27,265] jepsen worker 54 - jepsen.util 7974 :invoke :cas [18 [32 17]] INFO [2019-12-17 12:33:27,271] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41159,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:33:27,271] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41167,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:33:27,271] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41159,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:33:27,271] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41167,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:33:27,271] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41160,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:33:27,271] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41170,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:33:27,271] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41160,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:33:27,271] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41170,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:33:27,285] jepsen worker 54 - jepsen.util 7974 :fail :cas [18 [32 17]] :ValueNotSwapFrom INFO [2019-12-17 12:33:27,285] jepsen worker 54 - jepsen.util 7974 :invoke :write [18 12] INFO [2019-12-17 12:33:27,295] jepsen worker 42 - jepsen.util 10302 :info :write [14 42] :RequestCanceledException INFO [2019-12-17 12:33:27,295] jepsen worker 21 - jepsen.util 9111 :info :write [7 18] :RequestCanceledException INFO [2019-12-17 12:33:27,295] jepsen worker 25 - jepsen.util 10105 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:33:27,295] jepsen worker 25 - jepsen.util 10105 :invoke :cas [8 [38 46]] INFO [2019-12-17 12:33:27,295] jepsen worker 21 - jepsen.util 9201 :invoke :write [7 2] INFO [2019-12-17 12:33:27,295] jepsen worker 42 - jepsen.util 10392 :invoke :cas [14 [24 4]] INFO [2019-12-17 12:33:27,312] jepsen worker 0 - jepsen.util 10080 :info :cas [0 [3 1]] :RequestCanceledException INFO [2019-12-17 12:33:27,312] jepsen worker 75 - jepsen.util 9165 :info :write [25 7] :RequestCanceledException INFO [2019-12-17 12:33:27,313] jepsen worker 0 - jepsen.util 10170 :invoke :read [0 nil] INFO [2019-12-17 12:33:27,313] jepsen worker 75 - jepsen.util 9255 :invoke :read [25 nil] INFO [2019-12-17 12:33:27,328] jepsen worker 66 - jepsen.util 9426 :info :cas [22 [37 32]] :RequestCanceledException INFO [2019-12-17 12:33:27,328] jepsen worker 28 - jepsen.util 5518 :info :cas [9 [20 2]] :RequestCanceledException INFO [2019-12-17 12:33:27,329] jepsen worker 66 - jepsen.util 9516 :invoke :cas [22 [11 17]] INFO [2019-12-17 12:33:27,329] jepsen worker 28 - jepsen.util 5608 :invoke :write [9 33] INFO [2019-12-17 12:33:27,341] jepsen worker 58 - jepsen.util 10318 :info :write [19 18] :RequestCanceledException INFO [2019-12-17 12:33:27,341] jepsen worker 33 - jepsen.util 9393 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:33:27,342] jepsen worker 33 - jepsen.util 9393 :invoke :write [11 24] INFO [2019-12-17 12:33:27,342] jepsen worker 58 - jepsen.util 10408 :invoke :read [19 nil] INFO [2019-12-17 12:33:27,360] jepsen worker 60 - jepsen.util 10410 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:33:27,360] jepsen worker 62 - jepsen.util 9512 :info :write [20 40] :RequestCanceledException INFO [2019-12-17 12:33:27,360] jepsen worker 60 - jepsen.util 10410 :invoke :write [20 47] INFO [2019-12-17 12:33:27,361] jepsen worker 62 - jepsen.util 9602 :invoke :write [20 43] INFO [2019-12-17 12:33:27,373] jepsen worker 28 - jepsen.util 5608 :ok :write [9 33] INFO [2019-12-17 12:33:27,373] jepsen worker 28 - jepsen.util 5608 :invoke :cas [9 [9 25]] INFO [2019-12-17 12:33:27,394] jepsen worker 83 - jepsen.util 10073 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:33:27,394] jepsen worker 64 - jepsen.util 5464 :info :write [21 19] :RequestCanceledException INFO [2019-12-17 12:33:27,394] jepsen worker 68 - jepsen.util 9068 :info :write [22 32] :RequestCanceledException INFO [2019-12-17 12:33:27,394] jepsen worker 83 - jepsen.util 10073 :invoke :write [27 46] INFO [2019-12-17 12:33:27,395] jepsen worker 68 - jepsen.util 9158 :invoke :write [22 26] INFO [2019-12-17 12:33:27,395] jepsen worker 64 - jepsen.util 5554 :invoke :write [21 25] INFO [2019-12-17 12:33:27,396] jepsen worker 28 - jepsen.util 5608 :fail :cas [9 [9 25]] :ValueNotSwapFrom INFO [2019-12-17 12:33:27,396] jepsen worker 28 - jepsen.util 5608 :invoke :read [9 nil] INFO [2019-12-17 12:33:27,410] jepsen worker 28 - jepsen.util 5608 :ok :read [9 33] INFO [2019-12-17 12:33:27,410] jepsen worker 28 - jepsen.util 5608 :invoke :write [9 29] INFO [2019-12-17 12:33:27,413] jepsen worker 67 - jepsen.util 9787 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:33:27,413] jepsen worker 86 - jepsen.util 10076 :info :cas [28 [40 13]] :RequestCanceledException INFO [2019-12-17 12:33:27,413] jepsen worker 67 - jepsen.util 9787 :invoke :read [22 nil] INFO [2019-12-17 12:33:27,415] jepsen worker 86 - jepsen.util 10166 :invoke :write [28 7] INFO [2019-12-17 12:33:27,428] jepsen worker 41 - jepsen.util 10121 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:33:27,428] jepsen worker 1 - jepsen.util 9721 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:33:27,428] jepsen worker 26 - jepsen.util 9746 :info :cas [8 [1 26]] :RequestCanceledException INFO [2019-12-17 12:33:27,428] jepsen worker 77 - jepsen.util 9797 :info :write [25 2] :RequestCanceledException INFO [2019-12-17 12:33:27,428] jepsen worker 1 - jepsen.util 9721 :invoke :read [0 nil] INFO [2019-12-17 12:33:27,428] jepsen worker 41 - jepsen.util 10121 :invoke :write [13 1] INFO [2019-12-17 12:33:27,429] jepsen worker 26 - jepsen.util 9836 :invoke :cas [8 [33 38]] INFO [2019-12-17 12:33:27,429] jepsen worker 77 - jepsen.util 9887 :invoke :cas [25 [12 41]] INFO [2019-12-17 12:33:27,442] jepsen worker 50 - jepsen.util 10130 :info :write [16 38] :RequestCanceledException INFO [2019-12-17 12:33:27,442] jepsen worker 22 - jepsen.util 9562 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:33:27,442] jepsen worker 22 - jepsen.util 9562 :invoke :cas [7 [28 43]] INFO [2019-12-17 12:33:27,443] jepsen worker 50 - jepsen.util 10220 :invoke :write [16 48] INFO [2019-12-17 12:33:27,453] jepsen worker 28 - jepsen.util 5608 :ok :write [9 29] INFO [2019-12-17 12:33:27,453] jepsen worker 28 - jepsen.util 5608 :invoke :cas [9 [48 16]] INFO [2019-12-17 12:33:27,459] jepsen worker 24 - jepsen.util 8394 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:33:27,459] jepsen worker 24 - jepsen.util 8394 :invoke :read [8 nil] INFO [2019-12-17 12:33:27,467] jepsen worker 28 - jepsen.util 5608 :fail :cas [9 [48 16]] :ValueNotSwapFrom INFO [2019-12-17 12:33:27,467] jepsen worker 28 - jepsen.util 5608 :invoke :cas [9 [41 32]] INFO [2019-12-17 12:33:27,478] jepsen worker 28 - jepsen.util 5608 :fail :cas [9 [41 32]] :ValueNotSwapFrom INFO [2019-12-17 12:33:27,478] jepsen worker 28 - jepsen.util 5608 :invoke :read [9 nil] INFO [2019-12-17 12:33:27,489] jepsen worker 28 - jepsen.util 5608 :ok :read [9 29] INFO [2019-12-17 12:33:27,490] jepsen worker 28 - jepsen.util 5608 :invoke :write [9 10] INFO [2019-12-17 12:33:27,491] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41197,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:33:27,491] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41197,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:33:27,494] jepsen worker 59 - jepsen.util 9509 :info :cas [19 [15 37]] :RequestCanceledException INFO [2019-12-17 12:33:27,494] jepsen worker 38 - jepsen.util 7148 :info :write [12 35] :RequestCanceledException INFO [2019-12-17 12:33:27,495] jepsen worker 59 - jepsen.util 9599 :invoke :write [19 39] INFO [2019-12-17 12:33:27,495] jepsen worker 38 - jepsen.util 7238 :invoke :cas [12 [24 20]] INFO [2019-12-17 12:33:27,502] jepsen worker 69 - jepsen.util 7269 :fail :cas [23 [19 12]] :ValueNotSwapFrom INFO [2019-12-17 12:33:27,502] jepsen worker 69 - jepsen.util 7269 :invoke :cas [23 [31 13]] INFO [2019-12-17 12:33:27,513] jepsen worker 20 - jepsen.util 6860 :info :write [6 14] :RequestCanceledException INFO [2019-12-17 12:33:27,514] jepsen worker 20 - jepsen.util 6950 :invoke :read [6 nil] INFO [2019-12-17 12:33:27,524] jepsen worker 28 - jepsen.util 5608 :ok :write [9 10] INFO [2019-12-17 12:33:27,524] jepsen worker 28 - jepsen.util 5608 :invoke :read [9 nil] INFO [2019-12-17 12:33:27,528] jepsen worker 15 - jepsen.util 8655 :info :cas [5 [19 37]] :RequestCanceledException INFO [2019-12-17 12:33:27,528] jepsen worker 31 - jepsen.util 6511 :fail :read [10 nil] :RequestTimeoutException INFO [2019-12-17 12:33:27,528] jepsen worker 82 - jepsen.util 10252 :info :write [27 0] :RequestCanceledException INFO [2019-12-17 12:33:27,528] jepsen worker 31 - jepsen.util 6511 :invoke :write [10 5] INFO [2019-12-17 12:33:27,529] jepsen worker 82 - jepsen.util 10342 :invoke :write [27 27] INFO [2019-12-17 12:33:27,529] jepsen worker 15 - jepsen.util 8745 :invoke :read [5 nil] INFO [2019-12-17 12:33:27,542] jepsen worker 69 - jepsen.util 7269 :fail :cas [23 [31 13]] :ValueNotSwapFrom INFO [2019-12-17 12:33:27,542] jepsen worker 69 - jepsen.util 7269 :invoke :cas [23 [37 21]] INFO [2019-12-17 12:33:27,547] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:33:27,547] jepsen nemesis - couchbase.util Rebalance check count: 315 INFO [2019-12-17 12:33:27,547] jepsen nemesis - couchbase.util Rebalance stuck count: 296 INFO [2019-12-17 12:33:27,549] jepsen worker 28 - jepsen.util 5608 :ok :read [9 10] INFO [2019-12-17 12:33:27,549] jepsen worker 28 - jepsen.util 5608 :invoke :read [9 nil] INFO [2019-12-17 12:33:27,559] jepsen worker 28 - jepsen.util 5608 :ok :read [9 10] INFO [2019-12-17 12:33:27,559] jepsen worker 28 - jepsen.util 5608 :invoke :cas [9 [45 24]] INFO [2019-12-17 12:33:27,569] jepsen worker 69 - jepsen.util 7269 :fail :cas [23 [37 21]] :ValueNotSwapFrom INFO [2019-12-17 12:33:27,569] jepsen worker 69 - jepsen.util 7269 :invoke :read [23 nil] INFO [2019-12-17 12:33:27,571] jepsen worker 28 - jepsen.util 5608 :fail :cas [9 [45 24]] :ValueNotSwapFrom INFO [2019-12-17 12:33:27,571] jepsen worker 28 - jepsen.util 5608 :invoke :read [9 nil] INFO [2019-12-17 12:33:27,582] jepsen worker 28 - jepsen.util 5608 :ok :read [9 10] INFO [2019-12-17 12:33:27,583] jepsen worker 28 - jepsen.util 5608 :invoke :read [9 nil] INFO [2019-12-17 12:33:27,595] jepsen worker 28 - jepsen.util 5608 :ok :read [9 10] INFO [2019-12-17 12:33:27,595] jepsen worker 28 - jepsen.util 5608 :invoke :cas [9 [10 11]] INFO [2019-12-17 12:33:27,595] jepsen worker 45 - jepsen.util 7155 :info :write [15 41] :RequestCanceledException INFO [2019-12-17 12:33:27,596] jepsen worker 45 - jepsen.util 7245 :invoke :cas [15 [8 35]] INFO [2019-12-17 12:33:27,604] jepsen worker 69 - jepsen.util 7269 :ok :read [23 48] INFO [2019-12-17 12:33:27,604] jepsen worker 69 - jepsen.util 7269 :invoke :cas [23 [26 5]] INFO [2019-12-17 12:33:27,612] jepsen worker 27 - jepsen.util 7137 :fail :read [9 nil] :RequestTimeoutException INFO [2019-12-17 12:33:27,612] jepsen worker 11 - jepsen.util 7301 :info :cas [3 [42 11]] :RequestCanceledException INFO [2019-12-17 12:33:27,613] jepsen worker 27 - jepsen.util 7137 :invoke :read [9 nil] INFO [2019-12-17 12:33:27,614] jepsen worker 11 - jepsen.util 7391 :invoke :write [3 14] INFO [2019-12-17 12:33:27,641] jepsen worker 28 - jepsen.util 5608 :ok :cas [9 [10 11]] INFO [2019-12-17 12:33:27,641] jepsen worker 28 - jepsen.util 5608 :invoke :write [9 44] INFO [2019-12-17 12:33:27,642] jepsen worker 37 - jepsen.util 6697 :info :write [12 32] :RequestCanceledException INFO [2019-12-17 12:33:27,643] jepsen worker 37 - jepsen.util 6787 :invoke :read [12 nil] INFO [2019-12-17 12:33:27,669] jepsen worker 88 - jepsen.util 7108 :ok :read [29 26] INFO [2019-12-17 12:33:27,669] jepsen worker 88 - jepsen.util 7108 :invoke :cas [29 [1 40]] INFO [2019-12-17 12:33:27,673] jepsen worker 69 - jepsen.util 7269 :fail :cas [23 [26 5]] :ValueNotSwapFrom INFO [2019-12-17 12:33:27,674] jepsen worker 69 - jepsen.util 7269 :invoke :cas [23 [0 42]] INFO [2019-12-17 12:33:27,683] jepsen worker 28 - jepsen.util 5608 :ok :write [9 44] INFO [2019-12-17 12:33:27,683] jepsen worker 28 - jepsen.util 5608 :invoke :write [9 5] INFO [2019-12-17 12:33:27,708] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41235,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:33:27,708] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41235,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:33:27,708] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41237,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:33:27,708] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41237,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:33:27,719] jepsen worker 28 - jepsen.util 5608 :ok :write [9 5] INFO [2019-12-17 12:33:27,719] jepsen worker 28 - jepsen.util 5608 :invoke :write [9 38] INFO [2019-12-17 12:33:27,728] jepsen worker 76 - jepsen.util 9886 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:33:27,728] jepsen worker 76 - jepsen.util 9886 :invoke :read [25 nil] INFO [2019-12-17 12:33:27,756] jepsen worker 28 - jepsen.util 5608 :ok :write [9 38] INFO [2019-12-17 12:33:27,756] jepsen worker 28 - jepsen.util 5608 :invoke :read [9 nil] INFO [2019-12-17 12:33:27,780] jepsen worker 28 - jepsen.util 5608 :ok :read [9 38] INFO [2019-12-17 12:33:27,780] jepsen worker 28 - jepsen.util 5608 :invoke :read [9 nil] INFO [2019-12-17 12:33:27,791] jepsen worker 28 - jepsen.util 5608 :ok :read [9 38] INFO [2019-12-17 12:33:27,791] jepsen worker 28 - jepsen.util 5608 :invoke :read [9 nil] INFO [2019-12-17 12:33:27,803] jepsen worker 28 - jepsen.util 5608 :ok :read [9 38] INFO [2019-12-17 12:33:27,804] jepsen worker 28 - jepsen.util 5608 :invoke :read [9 nil] INFO [2019-12-17 12:33:27,812] jepsen worker 12 - jepsen.util 7482 :info :write [4 37] :RequestCanceledException INFO [2019-12-17 12:33:27,813] jepsen worker 12 - jepsen.util 7572 :invoke :write [4 46] INFO [2019-12-17 12:33:27,815] jepsen worker 28 - jepsen.util 5608 :ok :read [9 38] INFO [2019-12-17 12:33:27,815] jepsen worker 28 - jepsen.util 5608 :invoke :cas [9 [11 34]] INFO [2019-12-17 12:33:27,827] jepsen worker 6 - jepsen.util 10356 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:33:27,827] jepsen worker 52 - jepsen.util 7342 :info :write [17 35] :RequestCanceledException INFO [2019-12-17 12:33:27,828] jepsen worker 6 - jepsen.util 10356 :invoke :write [2 19] INFO [2019-12-17 12:33:27,829] jepsen worker 52 - jepsen.util 7432 :invoke :write [17 31] INFO [2019-12-17 12:33:27,832] jepsen worker 28 - jepsen.util 5608 :fail :cas [9 [11 34]] :ValueNotSwapFrom INFO [2019-12-17 12:33:27,832] jepsen worker 28 - jepsen.util 5608 :invoke :cas [9 [42 37]] INFO [2019-12-17 12:33:27,842] jepsen worker 80 - jepsen.util 7100 :info :write [26 16] :RequestCanceledException INFO [2019-12-17 12:33:27,842] jepsen worker 28 - jepsen.util 5608 :fail :cas [9 [42 37]] :ValueNotSwapFrom INFO [2019-12-17 12:33:27,842] jepsen worker 17 - jepsen.util 10007 :info :cas [5 [17 33]] :RequestCanceledException INFO [2019-12-17 12:33:27,842] jepsen worker 28 - jepsen.util 5608 :invoke :cas [9 [36 16]] INFO [2019-12-17 12:33:27,843] jepsen worker 80 - jepsen.util 7190 :invoke :cas [26 [3 31]] INFO [2019-12-17 12:33:27,843] jepsen worker 17 - jepsen.util 10097 :invoke :read [5 nil] INFO [2019-12-17 12:33:27,857] jepsen worker 28 - jepsen.util 5608 :fail :cas [9 [36 16]] :ValueNotSwapFrom INFO [2019-12-17 12:33:27,857] jepsen worker 28 - jepsen.util 5608 :invoke :write [9 3] INFO [2019-12-17 12:33:27,861] jepsen worker 78 - jepsen.util 7278 :info :write [26 19] :RequestCanceledException INFO [2019-12-17 12:33:27,862] jepsen worker 78 - jepsen.util 7368 :invoke :cas [26 [22 35]] INFO [2019-12-17 12:33:27,894] jepsen worker 39 - jepsen.util 9579 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:33:27,894] jepsen worker 39 - jepsen.util 9579 :invoke :read [13 nil] INFO [2019-12-17 12:33:27,901] jepsen worker 28 - jepsen.util 5608 :ok :write [9 3] INFO [2019-12-17 12:33:27,901] jepsen worker 28 - jepsen.util 5608 :invoke :cas [9 [12 8]] INFO [2019-12-17 12:33:27,910] jepsen worker 13 - jepsen.util 6763 :info :write [4 15] :RequestCanceledException INFO [2019-12-17 12:33:27,910] jepsen worker 84 - jepsen.util 9174 :info :cas [28 [20 13]] :RequestCanceledException INFO [2019-12-17 12:33:27,910] jepsen worker 53 - jepsen.util 6623 :info :write [17 10] :RequestCanceledException INFO [2019-12-17 12:33:27,911] jepsen worker 13 - jepsen.util 6853 :invoke :read [4 nil] INFO [2019-12-17 12:33:27,911] jepsen worker 84 - jepsen.util 9264 :invoke :write [28 28] INFO [2019-12-17 12:33:27,911] jepsen worker 53 - jepsen.util 6713 :invoke :cas [17 [31 37]] INFO [2019-12-17 12:33:27,915] jepsen worker 28 - jepsen.util 5608 :fail :cas [9 [12 8]] :ValueNotSwapFrom INFO [2019-12-17 12:33:27,915] jepsen worker 28 - jepsen.util 5608 :invoke :read [9 nil] INFO [2019-12-17 12:33:27,928] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41257,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:33:27,928] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41257,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:33:27,928] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41260,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:33:27,928] jepsen worker 40 - jepsen.util 10120 :info :cas [13 [14 26]] :RequestCanceledException INFO [2019-12-17 12:33:27,928] jepsen worker 81 - jepsen.util 9441 :info :write [27 26] :RequestCanceledException INFO [2019-12-17 12:33:27,928] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41260,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:33:27,928] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41261,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:33:27,928] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41261,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:33:27,929] jepsen worker 40 - jepsen.util 10210 :invoke :read [13 nil] INFO [2019-12-17 12:33:27,929] jepsen worker 81 - jepsen.util 9531 :invoke :read [27 nil] INFO [2019-12-17 12:33:27,931] jepsen worker 28 - jepsen.util 5608 :ok :read [9 3] INFO [2019-12-17 12:33:27,931] jepsen worker 28 - jepsen.util 5608 :invoke :write [9 12] INFO [2019-12-17 12:33:27,935] jepsen worker 38 - jepsen.util 7238 :fail :cas [12 [24 20]] :ValueNotSwapFrom INFO [2019-12-17 12:33:27,936] jepsen worker 38 - jepsen.util 7238 :invoke :write [12 18] INFO [2019-12-17 12:33:27,942] jepsen worker 10 - jepsen.util 7120 :fail :read [3 nil] :RequestTimeoutException INFO [2019-12-17 12:33:27,942] jepsen worker 10 - jepsen.util 7120 :invoke :cas [3 [36 37]] INFO [2019-12-17 12:33:27,957] jepsen worker 57 - jepsen.util 9417 :info :cas [19 [37 16]] :RequestCanceledException INFO [2019-12-17 12:33:27,958] jepsen worker 57 - jepsen.util 9507 :invoke :cas [19 [48 49]] INFO [2019-12-17 12:33:27,974] jepsen worker 28 - jepsen.util 5608 :ok :write [9 12] INFO [2019-12-17 12:33:27,974] jepsen worker 28 - jepsen.util 5608 :invoke :cas [9 [41 27]] INFO [2019-12-17 12:33:27,995] jepsen worker 63 - jepsen.util 6093 :fail :read [21 nil] :RequestTimeoutException INFO [2019-12-17 12:33:27,995] jepsen worker 63 - jepsen.util 6093 :invoke :cas [21 [19 6]] INFO [2019-12-17 12:33:27,997] jepsen worker 28 - jepsen.util 5608 :fail :cas [9 [41 27]] :ValueNotSwapFrom INFO [2019-12-17 12:33:27,997] jepsen worker 28 - jepsen.util 5608 :invoke :write [9 17] INFO [2019-12-17 12:33:28,012] jepsen worker 48 - jepsen.util 8778 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:33:28,012] jepsen worker 48 - jepsen.util 8778 :invoke :write [16 14] INFO [2019-12-17 12:33:28,028] jepsen worker 61 - jepsen.util 10411 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:33:28,028] jepsen worker 72 - jepsen.util 6912 :info :cas [24 [47 36]] :RequestCanceledException INFO [2019-12-17 12:33:28,028] jepsen worker 44 - jepsen.util 10304 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:33:28,028] jepsen worker 61 - jepsen.util 10411 :invoke :read [20 nil] INFO [2019-12-17 12:33:28,028] jepsen worker 44 - jepsen.util 10304 :invoke :write [14 34] INFO [2019-12-17 12:33:28,029] jepsen worker 72 - jepsen.util 7002 :invoke :write [24 47] INFO [2019-12-17 12:33:28,038] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41279,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:33:28,038] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41279,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:33:28,042] jepsen worker 4 - jepsen.util 6484 :info :write [1 10] :RequestCanceledException INFO [2019-12-17 12:33:28,042] jepsen worker 16 - jepsen.util 10906 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:33:28,043] jepsen worker 16 - jepsen.util 10906 :invoke :read [5 nil] INFO [2019-12-17 12:33:28,043] jepsen worker 4 - jepsen.util 6574 :invoke :write [1 6] INFO [2019-12-17 12:33:28,048] jepsen worker 28 - jepsen.util 5608 :ok :write [9 17] INFO [2019-12-17 12:33:28,049] jepsen worker 28 - jepsen.util 5608 :invoke :read [9 nil] INFO [2019-12-17 12:33:28,058] jepsen worker 8 - jepsen.util 9638 :info :cas [2 [24 42]] :RequestCanceledException INFO [2019-12-17 12:33:28,059] jepsen worker 8 - jepsen.util 9728 :invoke :read [2 nil] INFO [2019-12-17 12:33:28,069] jepsen worker 28 - jepsen.util 5608 :ok :read [9 17] INFO [2019-12-17 12:33:28,069] jepsen worker 47 - jepsen.util 7517 :ok :write [15 14] INFO [2019-12-17 12:33:28,069] jepsen worker 28 - jepsen.util 5608 :invoke :cas [9 [29 0]] INFO [2019-12-17 12:33:28,069] jepsen worker 47 - jepsen.util 7517 :invoke :write [15 45] INFO [2019-12-17 12:33:28,069] jepsen worker 9 - jepsen.util 7659 :ok :write [3 38] INFO [2019-12-17 12:33:28,069] jepsen worker 9 - jepsen.util 7659 :invoke :cas [3 [38 32]] INFO [2019-12-17 12:33:28,069] jepsen worker 74 - jepsen.util 7274 :ok :write [24 37] INFO [2019-12-17 12:33:28,069] jepsen worker 74 - jepsen.util 7274 :invoke :write [24 49] INFO [2019-12-17 12:33:28,072] jepsen worker 37 - jepsen.util 6787 :ok :read [12 47] INFO [2019-12-17 12:33:28,073] jepsen worker 37 - jepsen.util 6787 :invoke :write [12 15] INFO [2019-12-17 12:33:28,078] jepsen worker 30 - jepsen.util 7050 :ok :write [10 2] INFO [2019-12-17 12:33:28,079] jepsen worker 30 - jepsen.util 7050 :invoke :write [10 48] INFO [2019-12-17 12:33:28,079] jepsen worker 19 - jepsen.util 7039 :ok :write [6 36] INFO [2019-12-17 12:33:28,079] jepsen worker 19 - jepsen.util 7039 :invoke :cas [6 [13 42]] INFO [2019-12-17 12:33:28,093] jepsen worker 88 - jepsen.util 7108 :fail :cas [29 [1 40]] :ValueNotSwapFrom INFO [2019-12-17 12:33:28,094] jepsen worker 88 - jepsen.util 7108 :invoke :cas [29 [26 29]] INFO [2019-12-17 12:33:28,126] jepsen worker 65 - jepsen.util 5915 :fail :read [21 nil] :RequestTimeoutException INFO [2019-12-17 12:33:28,126] jepsen worker 7 - jepsen.util 9817 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:33:28,126] jepsen worker 65 - jepsen.util 5915 :invoke :cas [21 [31 6]] INFO [2019-12-17 12:33:28,126] jepsen worker 7 - jepsen.util 9817 :invoke :write [2 47] INFO [2019-12-17 12:33:28,127] jepsen worker 89 - jepsen.util 6839 :info :cas [29 [26 48]] :RequestCanceledException INFO [2019-12-17 12:33:28,127] jepsen worker 49 - jepsen.util 9049 :info :write [16 45] :RequestCanceledException INFO [2019-12-17 12:33:28,128] jepsen worker 89 - jepsen.util 6929 :invoke :read [29 nil] INFO [2019-12-17 12:33:28,128] jepsen worker 49 - jepsen.util 9139 :invoke :read [16 nil] INFO [2019-12-17 12:33:28,137] jepsen worker 45 - jepsen.util 7245 :fail :cas [15 [8 35]] :ValueNotSwapFrom INFO [2019-12-17 12:33:28,137] jepsen worker 45 - jepsen.util 7245 :invoke :cas [15 [42 18]] INFO [2019-12-17 12:33:28,149] jepsen worker 46 - jepsen.util 7516 :fail :cas [15 [29 41]] :ValueNotSwapFrom INFO [2019-12-17 12:33:28,149] jepsen worker 46 - jepsen.util 7516 :invoke :cas [15 [11 2]] INFO [2019-12-17 12:33:28,159] jepsen worker 87 - jepsen.util 6297 :info :write [29 9] :RequestCanceledException INFO [2019-12-17 12:33:28,160] jepsen worker 18 - jepsen.util 7128 :ok :read [6 36] INFO [2019-12-17 12:33:28,160] jepsen worker 18 - jepsen.util 7128 :invoke :write [6 49] INFO [2019-12-17 12:33:28,160] jepsen worker 87 - jepsen.util 6387 :invoke :cas [29 [17 24]] INFO [2019-12-17 12:33:28,165] jepsen worker 28 - jepsen.util 5608 :fail :cas [9 [29 0]] :ValueNotSwapFrom INFO [2019-12-17 12:33:28,165] jepsen worker 28 - jepsen.util 5608 :invoke :read [9 nil] INFO [2019-12-17 12:33:28,171] jepsen worker 20 - jepsen.util 6950 :ok :read [6 36] INFO [2019-12-17 12:33:28,172] jepsen worker 20 - jepsen.util 6950 :invoke :cas [6 [43 44]] INFO [2019-12-17 12:33:28,184] jepsen worker 10 - jepsen.util 7120 :fail :cas [3 [36 37]] :ValueNotSwapFrom INFO [2019-12-17 12:33:28,184] jepsen worker 10 - jepsen.util 7120 :invoke :write [3 40] INFO [2019-12-17 12:33:28,195] jepsen worker 85 - jepsen.util 10525 :info :cas [28 [33 47]] :RequestCanceledException INFO [2019-12-17 12:33:28,196] jepsen worker 63 - jepsen.util 6093 :fail :cas [21 [19 6]] :ValueNotSwapFrom INFO [2019-12-17 12:33:28,196] jepsen worker 63 - jepsen.util 6093 :invoke :read [21 nil] INFO [2019-12-17 12:33:28,196] jepsen worker 85 - jepsen.util 10615 :invoke :cas [28 [38 37]] INFO [2019-12-17 12:33:28,222] jepsen worker 28 - jepsen.util 5608 :ok :read [9 17] INFO [2019-12-17 12:33:28,222] jepsen worker 28 - jepsen.util 5608 :invoke :write [9 41] INFO [2019-12-17 12:33:28,224] jepsen worker 36 - jepsen.util 7416 :info :write [12 43] :RequestCanceledException INFO [2019-12-17 12:33:28,225] jepsen worker 36 - jepsen.util 7506 :invoke :write [12 39] INFO [2019-12-17 12:33:28,230] jepsen worker 20 - jepsen.util 6950 :fail :cas [6 [43 44]] :ValueNotSwapFrom INFO [2019-12-17 12:33:28,231] jepsen worker 20 - jepsen.util 6950 :invoke :write [6 27] INFO [2019-12-17 12:33:28,240] jepsen worker 35 - jepsen.util 9485 :info :cas [11 [19 35]] :RequestCanceledException INFO [2019-12-17 12:33:28,241] jepsen worker 35 - jepsen.util 9575 :invoke :cas [11 [5 18]] INFO [2019-12-17 12:33:28,256] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41312,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:33:28,256] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41320,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:33:28,256] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41312,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:33:28,256] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41320,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:33:28,256] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41315,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:33:28,256] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41321,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:33:28,256] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41315,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:33:28,256] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41321,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:33:28,257] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41323,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:33:28,257] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41323,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:33:28,260] jepsen worker 63 - jepsen.util 6093 :ok :read [21 2] INFO [2019-12-17 12:33:28,260] jepsen worker 63 - jepsen.util 6093 :invoke :cas [21 [12 24]] INFO [2019-12-17 12:33:28,269] jepsen worker 64 - jepsen.util 5554 :ok :write [21 25] INFO [2019-12-17 12:33:28,269] jepsen worker 64 - jepsen.util 5554 :invoke :read [21 nil] INFO [2019-12-17 12:33:28,311] jepsen worker 46 - jepsen.util 7516 :fail :cas [15 [11 2]] :ValueNotSwapFrom INFO [2019-12-17 12:33:28,311] jepsen worker 46 - jepsen.util 7516 :invoke :write [15 1] INFO [2019-12-17 12:33:28,328] jepsen worker 14 - jepsen.util 7304 :info :write [4 9] :RequestCanceledException INFO [2019-12-17 12:33:28,330] jepsen worker 14 - jepsen.util 7394 :invoke :cas [4 [38 34]] INFO [2019-12-17 12:33:28,368] jepsen worker 64 - jepsen.util 5554 :ok :read [21 25] INFO [2019-12-17 12:33:28,368] jepsen worker 64 - jepsen.util 5554 :invoke :read [21 nil] INFO [2019-12-17 12:33:28,374] jepsen worker 73 - jepsen.util 6733 :ok :write [24 40] INFO [2019-12-17 12:33:28,374] jepsen worker 73 - jepsen.util 6733 :invoke :cas [24 [10 29]] INFO [2019-12-17 12:33:28,382] jepsen worker 51 - jepsen.util 6981 :ok :write [17 16] INFO [2019-12-17 12:33:28,382] jepsen worker 51 - jepsen.util 6981 :invoke :cas [17 [39 47]] INFO [2019-12-17 12:33:28,402] jepsen worker 28 - jepsen.util 5608 :ok :write [9 41] INFO [2019-12-17 12:33:28,404] jepsen worker 28 - jepsen.util 5608 :invoke :read [9 nil] INFO [2019-12-17 12:33:28,408] jepsen worker 80 - jepsen.util 7190 :fail :cas [26 [3 31]] :ValueNotSwapFrom INFO [2019-12-17 12:33:28,408] jepsen worker 80 - jepsen.util 7190 :invoke :read [26 nil] INFO [2019-12-17 12:33:28,409] jepsen worker 32 - jepsen.util 7142 :info :write [10 30] :RequestCanceledException INFO [2019-12-17 12:33:28,411] jepsen worker 32 - jepsen.util 7232 :invoke :read [10 nil] INFO [2019-12-17 12:33:28,414] jepsen worker 31 - jepsen.util 6511 :ok :write [10 5] INFO [2019-12-17 12:33:28,414] jepsen worker 31 - jepsen.util 6511 :invoke :read [10 nil] INFO [2019-12-17 12:33:28,420] jepsen worker 71 - jepsen.util 7991 :ok :write [23 4] INFO [2019-12-17 12:33:28,421] jepsen worker 71 - jepsen.util 7991 :invoke :read [23 nil] INFO [2019-12-17 12:33:28,442] jepsen worker 89 - jepsen.util 6929 :ok :read [29 26] INFO [2019-12-17 12:33:28,442] jepsen worker 89 - jepsen.util 6929 :invoke :write [29 8] INFO [2019-12-17 12:33:28,452] jepsen worker 28 - jepsen.util 5608 :ok :read [9 41] INFO [2019-12-17 12:33:28,453] jepsen worker 28 - jepsen.util 5608 :invoke :read [9 nil] INFO [2019-12-17 12:33:28,472] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41343,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:33:28,473] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41343,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:33:28,473] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41346,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:33:28,473] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41346,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:33:28,473] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41347,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:33:28,473] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41347,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:33:28,482] jepsen worker 65 - jepsen.util 5915 :fail :cas [21 [31 6]] :ValueNotSwapFrom INFO [2019-12-17 12:33:28,482] jepsen worker 65 - jepsen.util 5915 :invoke :read [21 nil] INFO [2019-12-17 12:33:28,484] jepsen worker 14 - jepsen.util 7394 :fail :cas [4 [38 34]] :ValueNotSwapFrom INFO [2019-12-17 12:33:28,484] jepsen worker 14 - jepsen.util 7394 :invoke :cas [4 [41 28]] INFO [2019-12-17 12:33:28,493] jepsen worker 5 - jepsen.util 6665 :info :write [1 49] :RequestCanceledException INFO [2019-12-17 12:33:28,493] jepsen worker 2 - jepsen.util 9182 :info :cas [0 [35 10]] :RequestCanceledException INFO [2019-12-17 12:33:28,495] jepsen worker 5 - jepsen.util 6755 :invoke :cas [1 [3 13]] INFO [2019-12-17 12:33:28,495] jepsen worker 2 - jepsen.util 9272 :invoke :write [0 34] INFO [2019-12-17 12:33:28,501] jepsen worker 28 - jepsen.util 5608 :ok :read [9 41] INFO [2019-12-17 12:33:28,501] jepsen worker 28 - jepsen.util 5608 :invoke :write [9 2] INFO [2019-12-17 12:33:28,507] jepsen worker 55 - jepsen.util 8155 :ok :write [18 30] INFO [2019-12-17 12:33:28,507] jepsen worker 55 - jepsen.util 8155 :invoke :cas [18 [41 6]] INFO [2019-12-17 12:33:28,512] jepsen worker 29 - jepsen.util 5429 :info :write [9 19] :RequestCanceledException INFO [2019-12-17 12:33:28,513] jepsen worker 29 - jepsen.util 5519 :invoke :write [9 42] INFO [2019-12-17 12:33:28,516] jepsen worker 78 - jepsen.util 7368 :fail :cas [26 [22 35]] :ValueNotSwapFrom INFO [2019-12-17 12:33:28,516] jepsen worker 78 - jepsen.util 7368 :invoke :cas [26 [36 30]] INFO [2019-12-17 12:33:28,533] jepsen worker 13 - jepsen.util 6853 :ok :read [4 0] INFO [2019-12-17 12:33:28,533] jepsen worker 13 - jepsen.util 6853 :invoke :cas [4 [28 4]] INFO [2019-12-17 12:33:28,540] jepsen worker 65 - jepsen.util 5915 :ok :read [21 25] INFO [2019-12-17 12:33:28,540] jepsen worker 65 - jepsen.util 5915 :invoke :read [21 nil] INFO [2019-12-17 12:33:28,564] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:33:28,564] jepsen nemesis - couchbase.util Rebalance check count: 316 INFO [2019-12-17 12:33:28,564] jepsen nemesis - couchbase.util Rebalance stuck count: 297 INFO [2019-12-17 12:33:28,564] jepsen worker 65 - jepsen.util 5915 :ok :read [21 25] INFO [2019-12-17 12:33:28,565] jepsen worker 65 - jepsen.util 5915 :invoke :read [21 nil] INFO [2019-12-17 12:33:28,569] jepsen worker 51 - jepsen.util 6981 :fail :cas [17 [39 47]] :ValueNotSwapFrom INFO [2019-12-17 12:33:28,569] jepsen worker 51 - jepsen.util 6981 :invoke :cas [17 [3 24]] INFO [2019-12-17 12:33:28,591] jepsen worker 65 - jepsen.util 5915 :ok :read [21 25] INFO [2019-12-17 12:33:28,592] jepsen worker 65 - jepsen.util 5915 :invoke :cas [21 [10 41]] INFO [2019-12-17 12:33:28,594] jepsen worker 28 - jepsen.util 5608 :ok :write [9 2] INFO [2019-12-17 12:33:28,594] jepsen worker 28 - jepsen.util 5608 :invoke :cas [9 [49 28]] INFO [2019-12-17 12:33:28,620] jepsen worker 65 - jepsen.util 5915 :fail :cas [21 [10 41]] :ValueNotSwapFrom INFO [2019-12-17 12:33:28,620] jepsen worker 65 - jepsen.util 5915 :invoke :read [21 nil] INFO [2019-12-17 12:33:28,621] jepsen worker 47 - jepsen.util 7517 :ok :write [15 45] INFO [2019-12-17 12:33:28,622] jepsen worker 47 - jepsen.util 7517 :invoke :read [15 nil] INFO [2019-12-17 12:33:28,625] jepsen worker 28 - jepsen.util 5608 :fail :cas [9 [49 28]] :ValueNotSwapFrom INFO [2019-12-17 12:33:28,625] jepsen worker 28 - jepsen.util 5608 :invoke :write [9 8] INFO [2019-12-17 12:33:28,633] jepsen worker 87 - jepsen.util 6387 :fail :cas [29 [17 24]] :ValueNotSwapFrom INFO [2019-12-17 12:33:28,633] jepsen worker 87 - jepsen.util 6387 :invoke :write [29 4] INFO [2019-12-17 12:33:28,662] jepsen worker 53 - jepsen.util 6713 :fail :cas [17 [31 37]] :ValueNotSwapFrom INFO [2019-12-17 12:33:28,662] jepsen worker 53 - jepsen.util 6713 :invoke :cas [17 [19 10]] INFO [2019-12-17 12:33:28,691] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41369,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:33:28,691] jepsen worker 71 - jepsen.util 7991 :ok :read [23 4] INFO [2019-12-17 12:33:28,691] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41369,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:33:28,691] jepsen worker 71 - jepsen.util 7991 :invoke :write [23 19] INFO [2019-12-17 12:33:28,691] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41372,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:33:28,692] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41372,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:33:28,692] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41373,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:33:28,692] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41373,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:33:28,714] jepsen worker 69 - jepsen.util 7269 :fail :cas [23 [0 42]] :ValueNotSwapFrom INFO [2019-12-17 12:33:28,714] jepsen worker 69 - jepsen.util 7269 :invoke :read [23 nil] INFO [2019-12-17 12:33:28,723] jepsen worker 65 - jepsen.util 5915 :ok :read [21 25] INFO [2019-12-17 12:33:28,723] jepsen worker 65 - jepsen.util 5915 :invoke :read [21 nil] INFO [2019-12-17 12:33:28,733] jepsen worker 45 - jepsen.util 7245 :fail :cas [15 [42 18]] :ValueNotSwapFrom INFO [2019-12-17 12:33:28,733] jepsen worker 45 - jepsen.util 7245 :invoke :write [15 1] INFO [2019-12-17 12:33:28,744] jepsen worker 53 - jepsen.util 6713 :fail :cas [17 [19 10]] :ValueNotSwapFrom INFO [2019-12-17 12:33:28,744] jepsen worker 53 - jepsen.util 6713 :invoke :read [17 nil] INFO [2019-12-17 12:33:28,749] jepsen worker 80 - jepsen.util 7190 :ok :read [26 31] INFO [2019-12-17 12:33:28,749] jepsen worker 80 - jepsen.util 7190 :invoke :read [26 nil] INFO [2019-12-17 12:33:28,751] jepsen worker 65 - jepsen.util 5915 :ok :read [21 25] INFO [2019-12-17 12:33:28,751] jepsen worker 65 - jepsen.util 5915 :invoke :cas [21 [36 41]] INFO [2019-12-17 12:33:28,759] jepsen worker 79 - jepsen.util 7099 :info :write [26 22] :RequestCanceledException INFO [2019-12-17 12:33:28,759] jepsen worker 70 - jepsen.util 7360 :info :write [23 22] :RequestCanceledException INFO [2019-12-17 12:33:28,760] jepsen worker 79 - jepsen.util 7189 :invoke :read [26 nil] INFO [2019-12-17 12:33:28,760] jepsen worker 70 - jepsen.util 7450 :invoke :cas [23 [21 32]] INFO [2019-12-17 12:33:28,774] jepsen worker 20 - jepsen.util 6950 :ok :write [6 27] INFO [2019-12-17 12:33:28,775] jepsen worker 20 - jepsen.util 6950 :invoke :write [6 17] INFO [2019-12-17 12:33:28,780] jepsen worker 65 - jepsen.util 5915 :fail :cas [21 [36 41]] :ValueNotSwapFrom INFO [2019-12-17 12:33:28,781] jepsen worker 65 - jepsen.util 5915 :invoke :read [21 nil] INFO [2019-12-17 12:33:28,799] jepsen worker 11 - jepsen.util 7391 :ok :write [3 14] INFO [2019-12-17 12:33:28,800] jepsen worker 11 - jepsen.util 7391 :invoke :cas [3 [28 31]] INFO [2019-12-17 12:33:28,833] jepsen worker 69 - jepsen.util 7269 :ok :read [23 4] INFO [2019-12-17 12:33:28,833] jepsen worker 69 - jepsen.util 7269 :invoke :read [23 nil] INFO [2019-12-17 12:33:28,842] jepsen worker 89 - jepsen.util 6929 :ok :write [29 8] INFO [2019-12-17 12:33:28,842] jepsen worker 89 - jepsen.util 6929 :invoke :write [29 25] INFO [2019-12-17 12:33:28,865] jepsen worker 88 - jepsen.util 7108 :fail :cas [29 [26 29]] :ValueNotSwapFrom INFO [2019-12-17 12:33:28,866] jepsen worker 88 - jepsen.util 7108 :invoke :write [29 2] INFO [2019-12-17 12:33:28,869] jepsen worker 53 - jepsen.util 6713 :ok :read [17 16] INFO [2019-12-17 12:33:28,869] jepsen worker 53 - jepsen.util 6713 :invoke :read [17 nil] INFO [2019-12-17 12:33:28,884] jepsen worker 65 - jepsen.util 5915 :ok :read [21 25] INFO [2019-12-17 12:33:28,884] jepsen worker 65 - jepsen.util 5915 :invoke :read [21 nil] INFO [2019-12-17 12:33:28,886] jepsen worker 63 - jepsen.util 6093 :fail :cas [21 [12 24]] :ValueNotSwapFrom INFO [2019-12-17 12:33:28,886] jepsen worker 63 - jepsen.util 6093 :invoke :cas [21 [12 42]] INFO [2019-12-17 12:33:28,901] jepsen worker 28 - jepsen.util 5608 :ok :write [9 8] INFO [2019-12-17 12:33:28,901] jepsen worker 28 - jepsen.util 5608 :invoke :write [9 36] INFO [2019-12-17 12:33:28,908] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41390,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:33:28,909] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41390,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:33:28,909] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41393,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:33:28,909] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41393,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:33:28,909] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41394,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:33:28,909] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41394,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:33:28,933] jepsen worker 65 - jepsen.util 5915 :ok :read [21 25] INFO [2019-12-17 12:33:28,933] jepsen worker 65 - jepsen.util 5915 :invoke :read [21 nil] INFO [2019-12-17 12:33:28,941] jepsen worker 56 - jepsen.util 7886 :info :write [18 5] :RequestCanceledException INFO [2019-12-17 12:33:28,943] jepsen worker 56 - jepsen.util 7976 :invoke :read [18 nil] INFO [2019-12-17 12:33:28,971] jepsen worker 51 - jepsen.util 6981 :fail :cas [17 [3 24]] :ValueNotSwapFrom INFO [2019-12-17 12:33:28,971] jepsen worker 51 - jepsen.util 6981 :invoke :write [17 2] INFO [2019-12-17 12:33:28,985] jepsen worker 55 - jepsen.util 8155 :fail :cas [18 [41 6]] :ValueNotSwapFrom INFO [2019-12-17 12:33:28,985] jepsen worker 55 - jepsen.util 8155 :invoke :write [18 12] INFO [2019-12-17 12:33:28,988] jepsen worker 65 - jepsen.util 5915 :ok :read [21 25] INFO [2019-12-17 12:33:28,988] jepsen worker 65 - jepsen.util 5915 :invoke :write [21 43] INFO [2019-12-17 12:33:29,010] jepsen worker 14 - jepsen.util 7394 :fail :cas [4 [41 28]] :ValueNotSwapFrom INFO [2019-12-17 12:33:29,010] jepsen worker 14 - jepsen.util 7394 :invoke :write [4 37] INFO [2019-12-17 12:33:29,013] jepsen worker 28 - jepsen.util 5608 :ok :write [9 36] INFO [2019-12-17 12:33:29,014] jepsen worker 28 - jepsen.util 5608 :invoke :write [9 1] INFO [2019-12-17 12:33:29,018] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41405,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:33:29,019] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41405,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:33:29,019] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41410,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:33:29,019] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41410,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:33:29,062] jepsen worker 79 - jepsen.util 7189 :ok :read [26 31] INFO [2019-12-17 12:33:29,062] jepsen worker 79 - jepsen.util 7189 :invoke :read [26 nil] INFO [2019-12-17 12:33:29,069] jepsen worker 65 - jepsen.util 5915 :ok :write [21 43] INFO [2019-12-17 12:33:29,070] jepsen worker 65 - jepsen.util 5915 :invoke :read [21 nil] INFO [2019-12-17 12:33:29,070] jepsen worker 80 - jepsen.util 7190 :ok :read [26 31] INFO [2019-12-17 12:33:29,071] jepsen worker 80 - jepsen.util 7190 :invoke :read [26 nil] INFO [2019-12-17 12:33:29,075] jepsen worker 28 - jepsen.util 5608 :ok :write [9 1] INFO [2019-12-17 12:33:29,075] jepsen worker 28 - jepsen.util 5608 :invoke :write [9 2] INFO [2019-12-17 12:33:29,087] jepsen worker 13 - jepsen.util 6853 :fail :cas [4 [28 4]] :ValueNotSwapFrom INFO [2019-12-17 12:33:29,088] jepsen worker 13 - jepsen.util 6853 :invoke :write [4 41] INFO [2019-12-17 12:33:29,097] jepsen worker 65 - jepsen.util 5915 :ok :read [21 43] INFO [2019-12-17 12:33:29,098] jepsen worker 65 - jepsen.util 5915 :invoke :read [21 nil] INFO [2019-12-17 12:33:29,139] jepsen worker 65 - jepsen.util 5915 :ok :read [21 43] INFO [2019-12-17 12:33:29,139] jepsen worker 65 - jepsen.util 5915 :invoke :read [21 nil] INFO [2019-12-17 12:33:29,150] jepsen worker 28 - jepsen.util 5608 :ok :write [9 2] INFO [2019-12-17 12:33:29,150] jepsen worker 28 - jepsen.util 5608 :invoke :write [9 2] INFO [2019-12-17 12:33:29,158] jepsen worker 65 - jepsen.util 5915 :ok :read [21 43] INFO [2019-12-17 12:33:29,158] jepsen worker 65 - jepsen.util 5915 :invoke :read [21 nil] INFO [2019-12-17 12:33:29,205] jepsen worker 65 - jepsen.util 5915 :ok :read [21 43] INFO [2019-12-17 12:33:29,205] jepsen worker 65 - jepsen.util 5915 :invoke :write [21 34] INFO [2019-12-17 12:33:29,206] jepsen worker 70 - jepsen.util 7450 :fail :cas [23 [21 32]] :ValueNotSwapFrom INFO [2019-12-17 12:33:29,206] jepsen worker 70 - jepsen.util 7450 :invoke :write [23 5] INFO [2019-12-17 12:33:29,218] jepsen worker 69 - jepsen.util 7269 :ok :read [23 4] INFO [2019-12-17 12:33:29,219] jepsen worker 69 - jepsen.util 7269 :invoke :read [23 nil] INFO [2019-12-17 12:33:29,238] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41423,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:33:29,238] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41431,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:33:29,238] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41423,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:33:29,238] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41431,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:33:29,238] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41424,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:33:29,238] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41434,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:33:29,238] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41424,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:33:29,238] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41434,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:33:29,242] jepsen worker 3 - jepsen.util 7113 :info :write [1 28] :RequestCanceledException INFO [2019-12-17 12:33:29,243] jepsen worker 56 - jepsen.util 7976 :ok :read [18 30] INFO [2019-12-17 12:33:29,243] jepsen worker 56 - jepsen.util 7976 :invoke :cas [18 [44 5]] INFO [2019-12-17 12:33:29,245] jepsen worker 3 - jepsen.util 7203 :invoke :cas [1 [40 34]] INFO [2019-12-17 12:33:29,252] jepsen worker 53 - jepsen.util 6713 :ok :read [17 16] INFO [2019-12-17 12:33:29,252] jepsen worker 53 - jepsen.util 6713 :invoke :read [17 nil] INFO [2019-12-17 12:33:29,260] jepsen worker 34 - jepsen.util 9124 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:33:29,260] jepsen worker 34 - jepsen.util 9124 :invoke :cas [11 [45 15]] INFO [2019-12-17 12:33:29,281] jepsen worker 28 - jepsen.util 5608 :ok :write [9 2] INFO [2019-12-17 12:33:29,281] jepsen worker 28 - jepsen.util 5608 :invoke :write [9 36] INFO [2019-12-17 12:33:29,291] jepsen worker 53 - jepsen.util 6713 :ok :read [17 16] INFO [2019-12-17 12:33:29,291] jepsen worker 53 - jepsen.util 6713 :invoke :cas [17 [10 4]] INFO [2019-12-17 12:33:29,291] jepsen worker 20 - jepsen.util 6950 :ok :write [6 17] INFO [2019-12-17 12:33:29,291] jepsen worker 45 - jepsen.util 7245 :ok :write [15 1] INFO [2019-12-17 12:33:29,292] jepsen worker 20 - jepsen.util 6950 :invoke :cas [6 [37 11]] INFO [2019-12-17 12:33:29,292] jepsen worker 45 - jepsen.util 7245 :invoke :cas [15 [47 22]] INFO [2019-12-17 12:33:29,301] jepsen worker 36 - jepsen.util 7506 :ok :write [12 39] INFO [2019-12-17 12:33:29,302] jepsen worker 36 - jepsen.util 7506 :invoke :read [12 nil] INFO [2019-12-17 12:33:29,307] jepsen worker 23 - jepsen.util 9833 :info :write [7 32] :RequestCanceledException INFO [2019-12-17 12:33:29,309] jepsen worker 23 - jepsen.util 9923 :invoke :read [7 nil] INFO [2019-12-17 12:33:29,313] jepsen worker 65 - jepsen.util 5915 :ok :write [21 34] INFO [2019-12-17 12:33:29,313] jepsen worker 65 - jepsen.util 5915 :invoke :write [21 24] INFO [2019-12-17 12:33:29,326] jepsen worker 63 - jepsen.util 6093 :fail :cas [21 [12 42]] :ValueNotSwapFrom INFO [2019-12-17 12:33:29,326] jepsen worker 63 - jepsen.util 6093 :invoke :cas [21 [48 14]] INFO [2019-12-17 12:33:29,328] jepsen worker 43 - jepsen.util 9043 :info :cas [14 [7 1]] :RequestCanceledException INFO [2019-12-17 12:33:29,328] jepsen worker 21 - jepsen.util 9201 :info :write [7 2] :RequestCanceledException INFO [2019-12-17 12:33:29,328] jepsen worker 75 - jepsen.util 9255 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:33:29,329] jepsen worker 75 - jepsen.util 9255 :invoke :write [25 26] INFO [2019-12-17 12:33:29,330] jepsen worker 43 - jepsen.util 9133 :invoke :cas [14 [5 16]] INFO [2019-12-17 12:33:29,330] jepsen worker 21 - jepsen.util 9291 :invoke :read [7 nil] INFO [2019-12-17 12:33:29,340] jepsen worker 80 - jepsen.util 7190 :ok :read [26 31] INFO [2019-12-17 12:33:29,340] jepsen worker 80 - jepsen.util 7190 :invoke :write [26 32] INFO [2019-12-17 12:33:29,342] jepsen worker 42 - jepsen.util 10392 :info :cas [14 [24 4]] :RequestCanceledException INFO [2019-12-17 12:33:29,343] jepsen worker 45 - jepsen.util 7245 :fail :cas [15 [47 22]] :ValueNotSwapFrom INFO [2019-12-17 12:33:29,343] jepsen worker 45 - jepsen.util 7245 :invoke :write [15 32] INFO [2019-12-17 12:33:29,344] jepsen worker 42 - jepsen.util 10482 :invoke :cas [14 [9 5]] INFO [2019-12-17 12:33:29,359] jepsen worker 0 - jepsen.util 10170 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:33:29,360] jepsen worker 0 - jepsen.util 10170 :invoke :cas [0 [45 31]] INFO [2019-12-17 12:33:29,367] jepsen worker 56 - jepsen.util 7976 :fail :cas [18 [44 5]] :ValueNotSwapFrom INFO [2019-12-17 12:33:29,375] jepsen worker 56 - jepsen.util 7976 :invoke :cas [18 [37 19]] INFO [2019-12-17 12:33:29,394] jepsen worker 54 - jepsen.util 7974 :info :write [18 12] :RequestCanceledException INFO [2019-12-17 12:33:29,397] jepsen worker 28 - jepsen.util 5608 :ok :write [9 36] INFO [2019-12-17 12:33:29,398] jepsen worker 28 - jepsen.util 5608 :invoke :cas [9 [42 17]] INFO [2019-12-17 12:33:29,404] jepsen worker 25 - jepsen.util 10105 :info :cas [8 [38 46]] :RequestCanceledException INFO [2019-12-17 12:33:29,412] jepsen worker 66 - jepsen.util 9516 :info :cas [22 [11 17]] :RequestCanceledException INFO [2019-12-17 12:33:29,419] jepsen worker 65 - jepsen.util 5915 :ok :write [21 24] INFO [2019-12-17 12:33:29,420] jepsen worker 65 - jepsen.util 5915 :invoke :write [21 14] INFO [2019-12-17 12:33:29,426] jepsen worker 58 - jepsen.util 10408 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:33:29,426] jepsen worker 58 - jepsen.util 10408 :invoke :write [19 24] INFO [2019-12-17 12:33:29,426] jepsen worker 25 - jepsen.util 10195 :invoke :write [8 30] INFO [2019-12-17 12:33:29,427] jepsen worker 66 - jepsen.util 9606 :invoke :write [22 29] INFO [2019-12-17 12:33:29,434] jepsen worker 54 - jepsen.util 8064 :invoke :cas [18 [30 11]] INFO [2019-12-17 12:33:29,441] jepsen worker 53 - jepsen.util 6713 :fail :cas [17 [10 4]] :ValueNotSwapFrom INFO [2019-12-17 12:33:29,441] jepsen worker 53 - jepsen.util 6713 :invoke :cas [17 [49 30]] INFO [2019-12-17 12:33:29,442] jepsen worker 33 - jepsen.util 9393 :info :write [11 24] :RequestCanceledException INFO [2019-12-17 12:33:29,442] jepsen worker 62 - jepsen.util 9602 :info :write [20 43] :RequestCanceledException INFO [2019-12-17 12:33:29,448] jepsen worker 62 - jepsen.util 9692 :invoke :write [20 38] INFO [2019-12-17 12:33:29,449] jepsen worker 33 - jepsen.util 9483 :invoke :read [11 nil] INFO [2019-12-17 12:33:29,455] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41460,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:33:29,455] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41460,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:33:29,456] jepsen worker 60 - jepsen.util 10410 :info :write [20 47] :RequestCanceledException INFO [2019-12-17 12:33:29,456] jepsen worker 68 - jepsen.util 9158 :info :write [22 26] :RequestCanceledException INFO [2019-12-17 12:33:29,456] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41462,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:33:29,456] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41462,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:33:29,457] jepsen worker 68 - jepsen.util 9248 :invoke :read [22 nil] INFO [2019-12-17 12:33:29,458] jepsen worker 60 - jepsen.util 10500 :invoke :write [20 32] INFO [2019-12-17 12:33:29,461] jepsen worker 20 - jepsen.util 6950 :fail :cas [6 [37 11]] :ValueNotSwapFrom INFO [2019-12-17 12:33:29,462] jepsen worker 20 - jepsen.util 6950 :invoke :read [6 nil] INFO [2019-12-17 12:33:29,467] jepsen worker 28 - jepsen.util 5608 :fail :cas [9 [42 17]] :ValueNotSwapFrom INFO [2019-12-17 12:33:29,467] jepsen worker 28 - jepsen.util 5608 :invoke :cas [9 [21 19]] INFO [2019-12-17 12:33:29,468] jepsen worker 63 - jepsen.util 6093 :fail :cas [21 [48 14]] :ValueNotSwapFrom INFO [2019-12-17 12:33:29,468] jepsen worker 63 - jepsen.util 6093 :invoke :read [21 nil] INFO [2019-12-17 12:33:29,495] jepsen worker 83 - jepsen.util 10073 :info :write [27 46] :RequestCanceledException INFO [2019-12-17 12:33:29,495] jepsen worker 86 - jepsen.util 10166 :info :write [28 7] :RequestCanceledException INFO [2019-12-17 12:33:29,497] jepsen worker 56 - jepsen.util 7976 :fail :cas [18 [37 19]] :ValueNotSwapFrom INFO [2019-12-17 12:33:29,497] jepsen worker 56 - jepsen.util 7976 :invoke :cas [18 [49 8]] INFO [2019-12-17 12:33:29,503] jepsen worker 86 - jepsen.util 10256 :invoke :cas [28 [36 35]] INFO [2019-12-17 12:33:29,503] jepsen worker 83 - jepsen.util 10163 :invoke :cas [27 [17 12]] INFO [2019-12-17 12:33:29,507] jepsen worker 67 - jepsen.util 9787 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:33:29,507] jepsen worker 67 - jepsen.util 9787 :invoke :read [22 nil] INFO [2019-12-17 12:33:29,507] jepsen worker 26 - jepsen.util 9836 :info :cas [8 [33 38]] :RequestCanceledException INFO [2019-12-17 12:33:29,514] jepsen worker 26 - jepsen.util 9926 :invoke :write [8 35] INFO [2019-12-17 12:33:29,528] jepsen worker 77 - jepsen.util 9887 :info :cas [25 [12 41]] :RequestCanceledException INFO [2019-12-17 12:33:29,528] jepsen worker 1 - jepsen.util 9721 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:33:29,528] jepsen worker 41 - jepsen.util 10121 :info :write [13 1] :RequestCanceledException INFO [2019-12-17 12:33:29,528] jepsen worker 1 - jepsen.util 9721 :invoke :write [0 48] INFO [2019-12-17 12:33:29,529] jepsen worker 28 - jepsen.util 5608 :fail :cas [9 [21 19]] :ValueNotSwapFrom INFO [2019-12-17 12:33:29,529] jepsen worker 28 - jepsen.util 5608 :invoke :read [9 nil] INFO [2019-12-17 12:33:29,529] jepsen worker 77 - jepsen.util 9977 :invoke :write [25 11] INFO [2019-12-17 12:33:29,529] jepsen worker 41 - jepsen.util 10211 :invoke :write [13 20] INFO [2019-12-17 12:33:29,535] jepsen worker 20 - jepsen.util 6950 :ok :read [6 17] INFO [2019-12-17 12:33:29,535] jepsen worker 20 - jepsen.util 6950 :invoke :write [6 46] INFO [2019-12-17 12:33:29,537] jepsen worker 63 - jepsen.util 6093 :ok :read [21 24] INFO [2019-12-17 12:33:29,537] jepsen worker 63 - jepsen.util 6093 :invoke :cas [21 [7 44]] INFO [2019-12-17 12:33:29,539] jepsen worker 50 - jepsen.util 10220 :info :write [16 48] :RequestCanceledException INFO [2019-12-17 12:33:29,539] jepsen worker 22 - jepsen.util 9562 :info :cas [7 [28 43]] :RequestCanceledException INFO [2019-12-17 12:33:29,541] jepsen worker 50 - jepsen.util 10310 :invoke :cas [16 [10 47]] INFO [2019-12-17 12:33:29,541] jepsen worker 22 - jepsen.util 9652 :invoke :read [7 nil] INFO [2019-12-17 12:33:29,544] jepsen worker 79 - jepsen.util 7189 :ok :read [26 31] INFO [2019-12-17 12:33:29,545] jepsen worker 79 - jepsen.util 7189 :invoke :write [26 11] INFO [2019-12-17 12:33:29,559] jepsen worker 24 - jepsen.util 8394 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:33:29,559] jepsen worker 59 - jepsen.util 9599 :info :write [19 39] :RequestCanceledException INFO [2019-12-17 12:33:29,560] jepsen worker 24 - jepsen.util 8394 :invoke :cas [8 [45 4]] INFO [2019-12-17 12:33:29,561] jepsen worker 59 - jepsen.util 9689 :invoke :write [19 15] INFO [2019-12-17 12:33:29,563] jepsen worker 65 - jepsen.util 5915 :ok :write [21 14] INFO [2019-12-17 12:33:29,563] jepsen worker 65 - jepsen.util 5915 :invoke :read [21 nil] INFO [2019-12-17 12:33:29,579] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:33:29,579] jepsen nemesis - couchbase.util Rebalance check count: 317 INFO [2019-12-17 12:33:29,579] jepsen nemesis - couchbase.util Rebalance stuck count: 298 INFO [2019-12-17 12:33:29,583] jepsen worker 28 - jepsen.util 5608 :ok :read [9 36] INFO [2019-12-17 12:33:29,583] jepsen worker 28 - jepsen.util 5608 :invoke :cas [9 [11 10]] INFO [2019-12-17 12:33:29,585] jepsen worker 63 - jepsen.util 6093 :fail :cas [21 [7 44]] :ValueNotSwapFrom INFO [2019-12-17 12:33:29,586] jepsen worker 63 - jepsen.util 6093 :invoke :write [21 35] INFO [2019-12-17 12:33:29,605] jepsen worker 65 - jepsen.util 5915 :ok :read [21 14] INFO [2019-12-17 12:33:29,605] jepsen worker 65 - jepsen.util 5915 :invoke :cas [21 [40 9]] INFO [2019-12-17 12:33:29,607] jepsen worker 69 - jepsen.util 7269 :ok :read [23 4] INFO [2019-12-17 12:33:29,608] jepsen worker 69 - jepsen.util 7269 :invoke :read [23 nil] INFO [2019-12-17 12:33:29,611] jepsen worker 28 - jepsen.util 5608 :fail :cas [9 [11 10]] :ValueNotSwapFrom INFO [2019-12-17 12:33:29,611] jepsen worker 28 - jepsen.util 5608 :invoke :cas [9 [24 19]] INFO [2019-12-17 12:33:29,612] jepsen worker 82 - jepsen.util 10342 :info :write [27 27] :RequestCanceledException INFO [2019-12-17 12:33:29,614] jepsen worker 82 - jepsen.util 10432 :invoke :write [27 32] INFO [2019-12-17 12:33:29,617] jepsen worker 56 - jepsen.util 7976 :fail :cas [18 [49 8]] :ValueNotSwapFrom INFO [2019-12-17 12:33:29,617] jepsen worker 56 - jepsen.util 7976 :invoke :cas [18 [6 12]] INFO [2019-12-17 12:33:29,628] jepsen worker 15 - jepsen.util 8745 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:33:29,628] jepsen worker 15 - jepsen.util 8745 :invoke :read [5 nil] INFO [2019-12-17 12:33:29,646] jepsen worker 28 - jepsen.util 5608 :fail :cas [9 [24 19]] :ValueNotSwapFrom INFO [2019-12-17 12:33:29,647] jepsen worker 28 - jepsen.util 5608 :invoke :write [9 11] INFO [2019-12-17 12:33:29,648] jepsen worker 53 - jepsen.util 6713 :fail :cas [17 [49 30]] :ValueNotSwapFrom INFO [2019-12-17 12:33:29,648] jepsen worker 53 - jepsen.util 6713 :invoke :read [17 nil] INFO [2019-12-17 12:33:29,650] jepsen worker 65 - jepsen.util 5915 :fail :cas [21 [40 9]] :ValueNotSwapFrom INFO [2019-12-17 12:33:29,650] jepsen worker 65 - jepsen.util 5915 :invoke :cas [21 [20 31]] INFO [2019-12-17 12:33:29,674] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41505,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:33:29,674] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41505,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:33:29,685] jepsen worker 56 - jepsen.util 7976 :fail :cas [18 [6 12]] :ValueNotSwapFrom INFO [2019-12-17 12:33:29,685] jepsen worker 56 - jepsen.util 7976 :invoke :cas [18 [13 10]] INFO [2019-12-17 12:33:29,688] jepsen worker 65 - jepsen.util 5915 :fail :cas [21 [20 31]] :ValueNotSwapFrom INFO [2019-12-17 12:33:29,689] jepsen worker 65 - jepsen.util 5915 :invoke :write [21 3] INFO [2019-12-17 12:33:29,710] jepsen worker 27 - jepsen.util 7137 :fail :read [9 nil] :RequestTimeoutException INFO [2019-12-17 12:33:29,710] jepsen worker 27 - jepsen.util 7137 :invoke :read [9 nil] INFO [2019-12-17 12:33:29,735] jepsen worker 28 - jepsen.util 5608 :ok :write [9 11] INFO [2019-12-17 12:33:29,735] jepsen worker 28 - jepsen.util 5608 :invoke :write [9 48] INFO [2019-12-17 12:33:29,749] jepsen worker 65 - jepsen.util 5915 :ok :write [21 3] INFO [2019-12-17 12:33:29,749] jepsen worker 65 - jepsen.util 5915 :invoke :cas [21 [25 39]] INFO [2019-12-17 12:33:29,754] jepsen worker 27 - jepsen.util 7137 :ok :read [9 11] INFO [2019-12-17 12:33:29,754] jepsen worker 27 - jepsen.util 7137 :invoke :cas [9 [28 2]] INFO [2019-12-17 12:33:29,767] jepsen worker 36 - jepsen.util 7506 :ok :read [12 39] INFO [2019-12-17 12:33:29,767] jepsen worker 36 - jepsen.util 7506 :invoke :write [12 17] INFO [2019-12-17 12:33:29,786] jepsen worker 65 - jepsen.util 5915 :fail :cas [21 [25 39]] :ValueNotSwapFrom INFO [2019-12-17 12:33:29,786] jepsen worker 65 - jepsen.util 5915 :invoke :read [21 nil] INFO [2019-12-17 12:33:29,803] jepsen worker 65 - jepsen.util 5915 :ok :read [21 3] INFO [2019-12-17 12:33:29,803] jepsen worker 65 - jepsen.util 5915 :invoke :read [21 nil] INFO [2019-12-17 12:33:29,808] jepsen worker 56 - jepsen.util 7976 :fail :cas [18 [13 10]] :ValueNotSwapFrom INFO [2019-12-17 12:33:29,808] jepsen worker 56 - jepsen.util 7976 :invoke :cas [18 [13 40]] INFO [2019-12-17 12:33:29,828] jepsen worker 76 - jepsen.util 9886 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:33:29,828] jepsen worker 76 - jepsen.util 9886 :invoke :read [25 nil] INFO [2019-12-17 12:33:29,829] jepsen worker 65 - jepsen.util 5915 :ok :read [21 3] INFO [2019-12-17 12:33:29,829] jepsen worker 65 - jepsen.util 5915 :invoke :read [21 nil] INFO [2019-12-17 12:33:29,832] jepsen worker 28 - jepsen.util 5608 :ok :write [9 48] INFO [2019-12-17 12:33:29,832] jepsen worker 28 - jepsen.util 5608 :invoke :cas [9 [11 2]] INFO [2019-12-17 12:33:29,856] jepsen worker 28 - jepsen.util 5608 :fail :cas [9 [11 2]] :ValueNotSwapFrom INFO [2019-12-17 12:33:29,856] jepsen worker 28 - jepsen.util 5608 :invoke :cas [9 [12 10]] INFO [2019-12-17 12:33:29,863] jepsen worker 65 - jepsen.util 5915 :ok :read [21 3] INFO [2019-12-17 12:33:29,863] jepsen worker 65 - jepsen.util 5915 :invoke :write [21 27] INFO [2019-12-17 12:33:29,892] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41525,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:33:29,892] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41525,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:33:29,893] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41529,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:33:29,893] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41529,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:33:29,893] jepsen worker 28 - jepsen.util 5608 :fail :cas [9 [12 10]] :ValueNotSwapFrom INFO [2019-12-17 12:33:29,893] jepsen worker 28 - jepsen.util 5608 :invoke :cas [9 [9 41]] INFO [2019-12-17 12:33:29,894] jepsen worker 12 - jepsen.util 7572 :info :write [4 46] :RequestCanceledException INFO [2019-12-17 12:33:29,896] jepsen worker 12 - jepsen.util 7662 :invoke :write [4 24] INFO [2019-12-17 12:33:29,905] jepsen worker 56 - jepsen.util 7976 :fail :cas [18 [13 40]] :ValueNotSwapFrom INFO [2019-12-17 12:33:29,906] jepsen worker 56 - jepsen.util 7976 :invoke :cas [18 [39 8]] INFO [2019-12-17 12:33:29,908] jepsen worker 52 - jepsen.util 7432 :info :write [17 31] :RequestCanceledException INFO [2019-12-17 12:33:29,909] jepsen worker 52 - jepsen.util 7522 :invoke :read [17 nil] INFO [2019-12-17 12:33:29,919] jepsen worker 28 - jepsen.util 5608 :fail :cas [9 [9 41]] :ValueNotSwapFrom INFO [2019-12-17 12:33:29,919] jepsen worker 28 - jepsen.util 5608 :invoke :read [9 nil] INFO [2019-12-17 12:33:29,927] jepsen worker 6 - jepsen.util 10356 :info :write [2 19] :RequestCanceledException INFO [2019-12-17 12:33:29,928] jepsen worker 6 - jepsen.util 10446 :invoke :read [2 nil] INFO [2019-12-17 12:33:29,937] jepsen worker 28 - jepsen.util 5608 :ok :read [9 48] INFO [2019-12-17 12:33:29,937] jepsen worker 28 - jepsen.util 5608 :invoke :cas [9 [0 22]] INFO [2019-12-17 12:33:29,941] jepsen worker 81 - jepsen.util 9531 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:33:29,941] jepsen worker 17 - jepsen.util 10097 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:33:29,942] jepsen worker 81 - jepsen.util 9531 :invoke :write [27 45] INFO [2019-12-17 12:33:29,942] jepsen worker 17 - jepsen.util 10097 :invoke :write [5 44] INFO [2019-12-17 12:33:29,943] jepsen worker 65 - jepsen.util 5915 :ok :write [21 27] INFO [2019-12-17 12:33:29,944] jepsen worker 65 - jepsen.util 5915 :invoke :read [21 nil] INFO [2019-12-17 12:33:29,956] jepsen worker 28 - jepsen.util 5608 :fail :cas [9 [0 22]] :ValueNotSwapFrom INFO [2019-12-17 12:33:29,956] jepsen worker 28 - jepsen.util 5608 :invoke :write [9 20] INFO [2019-12-17 12:33:29,967] jepsen worker 65 - jepsen.util 5915 :ok :read [21 27] INFO [2019-12-17 12:33:29,968] jepsen worker 65 - jepsen.util 5915 :invoke :read [21 nil] INFO [2019-12-17 12:33:29,972] jepsen worker 69 - jepsen.util 7269 :ok :read [23 4] INFO [2019-12-17 12:33:29,973] jepsen worker 69 - jepsen.util 7269 :invoke :cas [23 [41 42]] INFO [2019-12-17 12:33:29,994] jepsen worker 84 - jepsen.util 9264 :info :write [28 28] :RequestCanceledException INFO [2019-12-17 12:33:29,994] jepsen worker 39 - jepsen.util 9579 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:33:29,994] jepsen worker 39 - jepsen.util 9579 :invoke :write [13 34] INFO [2019-12-17 12:33:29,995] jepsen worker 84 - jepsen.util 9354 :invoke :write [28 39] INFO [2019-12-17 12:33:29,997] jepsen worker 65 - jepsen.util 5915 :ok :read [21 27] INFO [2019-12-17 12:33:29,997] jepsen worker 65 - jepsen.util 5915 :invoke :read [21 nil] INFO [2019-12-17 12:33:30,002] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41547,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:33:30,002] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41547,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:33:30,002] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41550,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:33:30,002] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41550,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:33:30,015] jepsen worker 65 - jepsen.util 5915 :ok :read [21 27] INFO [2019-12-17 12:33:30,015] jepsen worker 65 - jepsen.util 5915 :invoke :read [21 nil] INFO [2019-12-17 12:33:30,019] jepsen worker 28 - jepsen.util 5608 :ok :write [9 20] INFO [2019-12-17 12:33:30,020] jepsen worker 28 - jepsen.util 5608 :invoke :read [9 nil] INFO [2019-12-17 12:33:30,028] jepsen worker 40 - jepsen.util 10210 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:33:30,028] jepsen worker 40 - jepsen.util 10210 :invoke :read [13 nil] INFO [2019-12-17 12:33:30,028] jepsen worker 38 - jepsen.util 7238 :info :write [12 18] :RequestCanceledException INFO [2019-12-17 12:33:30,029] jepsen worker 38 - jepsen.util 7328 :invoke :read [12 nil] INFO [2019-12-17 12:33:30,036] jepsen worker 65 - jepsen.util 5915 :ok :read [21 27] INFO [2019-12-17 12:33:30,036] jepsen worker 65 - jepsen.util 5915 :invoke :cas [21 [33 9]] INFO [2019-12-17 12:33:30,040] jepsen worker 28 - jepsen.util 5608 :ok :read [9 20] INFO [2019-12-17 12:33:30,040] jepsen worker 28 - jepsen.util 5608 :invoke :read [9 nil] INFO [2019-12-17 12:33:30,058] jepsen worker 57 - jepsen.util 9507 :info :cas [19 [48 49]] :RequestCanceledException INFO [2019-12-17 12:33:30,059] jepsen worker 57 - jepsen.util 9597 :invoke :write [19 34] INFO [2019-12-17 12:33:30,064] jepsen worker 65 - jepsen.util 5915 :fail :cas [21 [33 9]] :ValueNotSwapFrom INFO [2019-12-17 12:33:30,065] jepsen worker 65 - jepsen.util 5915 :invoke :write [21 11] INFO [2019-12-17 12:33:30,069] jepsen worker 28 - jepsen.util 5608 :ok :read [9 20] INFO [2019-12-17 12:33:30,069] jepsen worker 28 - jepsen.util 5608 :invoke :write [9 20] INFO [2019-12-17 12:33:30,078] jepsen worker 53 - jepsen.util 6713 :ok :read [17 16] INFO [2019-12-17 12:33:30,078] jepsen worker 53 - jepsen.util 6713 :invoke :read [17 nil] INFO [2019-12-17 12:33:30,080] jepsen worker 56 - jepsen.util 7976 :fail :cas [18 [39 8]] :ValueNotSwapFrom INFO [2019-12-17 12:33:30,080] jepsen worker 56 - jepsen.util 7976 :invoke :read [18 nil] INFO [2019-12-17 12:33:30,092] jepsen worker 4 - jepsen.util 6574 :info :write [1 6] :RequestCanceledException INFO [2019-12-17 12:33:30,092] jepsen worker 30 - jepsen.util 7050 :info :write [10 48] :RequestCanceledException INFO [2019-12-17 12:33:30,093] jepsen worker 4 - jepsen.util 6664 :invoke :read [1 nil] INFO [2019-12-17 12:33:30,093] jepsen worker 30 - jepsen.util 7140 :invoke :write [10 2] INFO [2019-12-17 12:33:30,110] jepsen worker 48 - jepsen.util 8778 :info :write [16 14] :RequestCanceledException INFO [2019-12-17 12:33:30,110] jepsen worker 8 - jepsen.util 9728 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:33:30,110] jepsen worker 8 - jepsen.util 9728 :invoke :write [2 2] INFO [2019-12-17 12:33:30,111] jepsen worker 48 - jepsen.util 8868 :invoke :write [16 38] INFO [2019-12-17 12:33:30,128] jepsen worker 44 - jepsen.util 10304 :info :write [14 34] :RequestCanceledException INFO [2019-12-17 12:33:30,128] jepsen worker 72 - jepsen.util 7002 :info :write [24 47] :RequestCanceledException INFO [2019-12-17 12:33:30,128] jepsen worker 9 - jepsen.util 7659 :info :cas [3 [38 32]] :RequestCanceledException INFO [2019-12-17 12:33:30,128] jepsen worker 37 - jepsen.util 6787 :info :write [12 15] :RequestCanceledException INFO [2019-12-17 12:33:30,129] jepsen worker 61 - jepsen.util 10411 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:33:30,129] jepsen worker 19 - jepsen.util 7039 :info :cas [6 [13 42]] :RequestCanceledException INFO [2019-12-17 12:33:30,129] jepsen worker 61 - jepsen.util 10411 :invoke :write [20 20] INFO [2019-12-17 12:33:30,129] jepsen worker 44 - jepsen.util 10394 :invoke :write [14 29] INFO [2019-12-17 12:33:30,129] jepsen worker 72 - jepsen.util 7092 :invoke :cas [24 [16 20]] INFO [2019-12-17 12:33:30,130] jepsen worker 9 - jepsen.util 7749 :invoke :cas [3 [44 32]] INFO [2019-12-17 12:33:30,130] jepsen worker 19 - jepsen.util 7129 :invoke :cas [6 [29 32]] INFO [2019-12-17 12:33:30,130] jepsen worker 37 - jepsen.util 6877 :invoke :cas [12 [7 17]] INFO [2019-12-17 12:33:30,142] jepsen worker 16 - jepsen.util 10906 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:33:30,142] jepsen worker 49 - jepsen.util 9139 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:33:30,142] jepsen worker 16 - jepsen.util 10906 :invoke :write [5 18] INFO [2019-12-17 12:33:30,142] jepsen worker 49 - jepsen.util 9139 :invoke :write [16 39] INFO [2019-12-17 12:33:30,149] jepsen worker 72 - jepsen.util 7092 :fail :cas [24 [16 20]] :ValueNotSwapFrom INFO [2019-12-17 12:33:30,149] jepsen worker 72 - jepsen.util 7092 :invoke :write [24 20] INFO [2019-12-17 12:33:30,156] jepsen worker 74 - jepsen.util 7274 :info :write [24 49] :RequestCanceledException INFO [2019-12-17 12:33:30,158] jepsen worker 65 - jepsen.util 5915 :ok :write [21 11] INFO [2019-12-17 12:33:30,158] jepsen worker 74 - jepsen.util 7364 :invoke :write [24 13] INFO [2019-12-17 12:33:30,158] jepsen worker 65 - jepsen.util 5915 :invoke :cas [21 [38 29]] INFO [2019-12-17 12:33:30,164] jepsen worker 28 - jepsen.util 5608 :ok :write [9 20] INFO [2019-12-17 12:33:30,164] jepsen worker 28 - jepsen.util 5608 :invoke :read [9 nil] INFO [2019-12-17 12:33:30,203] jepsen worker 19 - jepsen.util 7129 :fail :cas [6 [29 32]] :ValueNotSwapFrom INFO [2019-12-17 12:33:30,203] jepsen worker 19 - jepsen.util 7129 :invoke :read [6 nil] INFO [2019-12-17 12:33:30,211] jepsen worker 28 - jepsen.util 5608 :ok :read [9 20] INFO [2019-12-17 12:33:30,211] jepsen worker 28 - jepsen.util 5608 :invoke :cas [9 [9 9]] INFO [2019-12-17 12:33:30,218] jepsen worker 19 - jepsen.util 7129 :ok :read [6 17] INFO [2019-12-17 12:33:30,218] jepsen worker 19 - jepsen.util 7129 :invoke :cas [6 [43 42]] INFO [2019-12-17 12:33:30,219] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41581,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:33:30,219] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41589,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:33:30,220] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41581,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:33:30,220] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41589,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:33:30,220] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41582,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:33:30,220] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41593,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:33:30,220] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41582,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:33:30,220] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41593,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:33:30,220] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41583,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:33:30,220] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41583,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:33:30,225] jepsen worker 28 - jepsen.util 5608 :fail :cas [9 [9 9]] :ValueNotSwapFrom INFO [2019-12-17 12:33:30,225] jepsen worker 28 - jepsen.util 5608 :invoke :cas [9 [30 25]] INFO [2019-12-17 12:33:30,228] jepsen worker 85 - jepsen.util 10615 :info :cas [28 [38 37]] :RequestCanceledException INFO [2019-12-17 12:33:30,228] jepsen worker 18 - jepsen.util 7128 :info :write [6 49] :RequestCanceledException INFO [2019-12-17 12:33:30,228] jepsen worker 7 - jepsen.util 9817 :info :write [2 47] :RequestCanceledException INFO [2019-12-17 12:33:30,229] jepsen worker 85 - jepsen.util 10705 :invoke :write [28 28] INFO [2019-12-17 12:33:30,229] jepsen worker 18 - jepsen.util 7218 :invoke :cas [6 [25 7]] INFO [2019-12-17 12:33:30,230] jepsen worker 7 - jepsen.util 9907 :invoke :read [2 nil] INFO [2019-12-17 12:33:30,234] jepsen worker 19 - jepsen.util 7129 :fail :cas [6 [43 42]] :ValueNotSwapFrom INFO [2019-12-17 12:33:30,234] jepsen worker 19 - jepsen.util 7129 :invoke :write [6 41] INFO [2019-12-17 12:33:30,242] jepsen worker 10 - jepsen.util 7120 :info :write [3 40] :RequestCanceledException INFO [2019-12-17 12:33:30,243] jepsen worker 10 - jepsen.util 7210 :invoke :read [3 nil] INFO [2019-12-17 12:33:30,258] jepsen worker 28 - jepsen.util 5608 :fail :cas [9 [30 25]] :ValueNotSwapFrom INFO [2019-12-17 12:33:30,259] jepsen worker 28 - jepsen.util 5608 :invoke :read [9 nil] INFO [2019-12-17 12:33:30,271] jepsen worker 56 - jepsen.util 7976 :ok :read [18 30] INFO [2019-12-17 12:33:30,271] jepsen worker 56 - jepsen.util 7976 :invoke :write [18 8] INFO [2019-12-17 12:33:30,277] jepsen worker 18 - jepsen.util 7218 :fail :cas [6 [25 7]] :ValueNotSwapFrom INFO [2019-12-17 12:33:30,277] jepsen worker 18 - jepsen.util 7218 :invoke :write [6 8] INFO [2019-12-17 12:33:30,313] jepsen worker 35 - jepsen.util 9575 :info :cas [11 [5 18]] :RequestCanceledException INFO [2019-12-17 12:33:30,314] jepsen worker 35 - jepsen.util 9665 :invoke :cas [11 [7 43]] INFO [2019-12-17 12:33:30,327] jepsen worker 3 - jepsen.util 7203 :fail :cas [1 [40 34]] :ValueNotSwapFrom INFO [2019-12-17 12:33:30,327] jepsen worker 3 - jepsen.util 7203 :invoke :read [1 nil] INFO [2019-12-17 12:33:30,328] jepsen worker 46 - jepsen.util 7516 :info :write [15 1] :RequestCanceledException INFO [2019-12-17 12:33:30,329] jepsen worker 46 - jepsen.util 7606 :invoke :read [15 nil] INFO [2019-12-17 12:33:30,342] jepsen worker 3 - jepsen.util 7203 :ok :read [1 44] INFO [2019-12-17 12:33:30,343] jepsen worker 3 - jepsen.util 7203 :invoke :read [1 nil] INFO [2019-12-17 12:33:30,344] jepsen worker 52 - jepsen.util 7522 :ok :read [17 16] INFO [2019-12-17 12:33:30,344] jepsen worker 52 - jepsen.util 7522 :invoke :read [17 nil] INFO [2019-12-17 12:33:30,373] jepsen worker 69 - jepsen.util 7269 :fail :cas [23 [41 42]] :ValueNotSwapFrom INFO [2019-12-17 12:33:30,373] jepsen worker 69 - jepsen.util 7269 :invoke :write [23 29] INFO [2019-12-17 12:33:30,403] jepsen worker 53 - jepsen.util 6713 :ok :read [17 16] INFO [2019-12-17 12:33:30,403] jepsen worker 53 - jepsen.util 6713 :invoke :cas [17 [47 22]] INFO [2019-12-17 12:33:30,406] jepsen worker 3 - jepsen.util 7203 :ok :read [1 44] INFO [2019-12-17 12:33:30,406] jepsen worker 3 - jepsen.util 7203 :invoke :write [1 5] INFO [2019-12-17 12:33:30,416] jepsen worker 30 - jepsen.util 7140 :ok :write [10 2] INFO [2019-12-17 12:33:30,417] jepsen worker 30 - jepsen.util 7140 :invoke :read [10 nil] INFO [2019-12-17 12:33:30,421] jepsen worker 5 - jepsen.util 6755 :fail :cas [1 [3 13]] :ValueNotSwapFrom INFO [2019-12-17 12:33:30,421] jepsen worker 5 - jepsen.util 6755 :invoke :cas [1 [8 17]] INFO [2019-12-17 12:33:30,426] jepsen worker 20 - jepsen.util 6950 :ok :write [6 46] INFO [2019-12-17 12:33:30,426] jepsen worker 20 - jepsen.util 6950 :invoke :write [6 46] INFO [2019-12-17 12:33:30,428] jepsen worker 31 - jepsen.util 6511 :fail :read [10 nil] :RequestTimeoutException INFO [2019-12-17 12:33:30,428] jepsen worker 64 - jepsen.util 5554 :fail :read [21 nil] :RequestTimeoutException INFO [2019-12-17 12:33:30,428] jepsen worker 31 - jepsen.util 6511 :invoke :read [10 nil] INFO [2019-12-17 12:33:30,428] jepsen worker 64 - jepsen.util 5554 :invoke :cas [21 [28 41]] INFO [2019-12-17 12:33:30,439] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41615,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:33:30,439] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41615,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:33:30,439] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41619,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:33:30,439] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41619,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:33:30,439] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41620,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:33:30,439] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41620,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:33:30,442] jepsen worker 78 - jepsen.util 7368 :fail :cas [26 [36 30]] :ValueNotSwapFrom INFO [2019-12-17 12:33:30,442] jepsen worker 32 - jepsen.util 7232 :fail :read [10 nil] :RequestTimeoutException INFO [2019-12-17 12:33:30,442] jepsen worker 32 - jepsen.util 7232 :invoke :read [10 nil] INFO [2019-12-17 12:33:30,442] jepsen worker 78 - jepsen.util 7368 :invoke :write [26 30] INFO [2019-12-17 12:33:30,452] jepsen worker 30 - jepsen.util 7140 :ok :read [10 2] INFO [2019-12-17 12:33:30,452] jepsen worker 30 - jepsen.util 7140 :invoke :write [10 34] INFO [2019-12-17 12:33:30,455] jepsen worker 45 - jepsen.util 7245 :ok :write [15 32] INFO [2019-12-17 12:33:30,455] jepsen worker 45 - jepsen.util 7245 :invoke :cas [15 [47 42]] INFO [2019-12-17 12:33:30,455] jepsen worker 4 - jepsen.util 6664 :ok :read [1 44] INFO [2019-12-17 12:33:30,456] jepsen worker 4 - jepsen.util 6664 :invoke :read [1 nil] INFO [2019-12-17 12:33:30,456] jepsen worker 73 - jepsen.util 6733 :info :cas [24 [10 29]] :RequestCanceledException INFO [2019-12-17 12:33:30,458] jepsen worker 73 - jepsen.util 6823 :invoke :read [24 nil] INFO [2019-12-17 12:33:30,461] jepsen worker 64 - jepsen.util 5554 :fail :cas [21 [28 41]] :ValueNotSwapFrom INFO [2019-12-17 12:33:30,461] jepsen worker 64 - jepsen.util 5554 :invoke :read [21 nil] INFO [2019-12-17 12:33:30,463] jepsen worker 31 - jepsen.util 6511 :ok :read [10 2] INFO [2019-12-17 12:33:30,463] jepsen worker 31 - jepsen.util 6511 :invoke :write [10 48] INFO [2019-12-17 12:33:30,493] jepsen worker 73 - jepsen.util 6823 :ok :read [24 49] INFO [2019-12-17 12:33:30,494] jepsen worker 73 - jepsen.util 6823 :invoke :cas [24 [36 17]] INFO [2019-12-17 12:33:30,511] jepsen worker 63 - jepsen.util 6093 :ok :write [21 35] INFO [2019-12-17 12:33:30,511] jepsen worker 63 - jepsen.util 6093 :invoke :write [21 46] INFO [2019-12-17 12:33:30,511] jepsen worker 64 - jepsen.util 5554 :ok :read [21 11] INFO [2019-12-17 12:33:30,512] jepsen worker 64 - jepsen.util 5554 :invoke :read [21 nil] INFO [2019-12-17 12:33:30,527] jepsen worker 2 - jepsen.util 9272 :info :write [0 34] :RequestCanceledException INFO [2019-12-17 12:33:30,528] jepsen worker 2 - jepsen.util 9362 :invoke :write [0 13] INFO [2019-12-17 12:33:30,530] jepsen worker 53 - jepsen.util 6713 :fail :cas [17 [47 22]] :ValueNotSwapFrom INFO [2019-12-17 12:33:30,530] jepsen worker 53 - jepsen.util 6713 :invoke :read [17 nil] INFO [2019-12-17 12:33:30,556] jepsen worker 64 - jepsen.util 5554 :ok :read [21 35] INFO [2019-12-17 12:33:30,556] jepsen worker 64 - jepsen.util 5554 :invoke :write [21 10] INFO [2019-12-17 12:33:30,561] jepsen worker 73 - jepsen.util 6823 :fail :cas [24 [36 17]] :ValueNotSwapFrom INFO [2019-12-17 12:33:30,561] jepsen worker 73 - jepsen.util 6823 :invoke :write [24 29] INFO [2019-12-17 12:33:30,568] jepsen worker 9 - jepsen.util 7749 :fail :cas [3 [44 32]] :ValueNotSwapFrom INFO [2019-12-17 12:33:30,568] jepsen worker 9 - jepsen.util 7749 :invoke :read [3 nil] INFO [2019-12-17 12:33:30,594] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:33:30,594] jepsen nemesis - couchbase.util Rebalance check count: 318 INFO [2019-12-17 12:33:30,594] jepsen nemesis - couchbase.util Rebalance stuck count: 299 INFO [2019-12-17 12:33:30,595] jepsen worker 29 - jepsen.util 5519 :info :write [9 42] :RequestCanceledException INFO [2019-12-17 12:33:30,596] jepsen worker 29 - jepsen.util 5609 :invoke :write [9 28] INFO [2019-12-17 12:33:30,612] jepsen worker 64 - jepsen.util 5554 :ok :write [21 10] INFO [2019-12-17 12:33:30,612] jepsen worker 64 - jepsen.util 5554 :invoke :read [21 nil] INFO [2019-12-17 12:33:30,629] jepsen worker 5 - jepsen.util 6755 :fail :cas [1 [8 17]] :ValueNotSwapFrom INFO [2019-12-17 12:33:30,629] jepsen worker 5 - jepsen.util 6755 :invoke :read [1 nil] INFO [2019-12-17 12:33:30,642] jepsen worker 47 - jepsen.util 7517 :fail :read [15 nil] :RequestTimeoutException INFO [2019-12-17 12:33:30,642] jepsen worker 47 - jepsen.util 7517 :invoke :write [15 18] INFO [2019-12-17 12:33:30,653] jepsen worker 64 - jepsen.util 5554 :ok :read [21 10] INFO [2019-12-17 12:33:30,653] jepsen worker 64 - jepsen.util 5554 :invoke :write [21 45] INFO [2019-12-17 12:33:30,657] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41645,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:33:30,658] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41645,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:33:30,658] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41649,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:33:30,658] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41649,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:33:30,659] jepsen worker 87 - jepsen.util 6387 :info :write [29 4] :RequestCanceledException INFO [2019-12-17 12:33:30,660] jepsen worker 87 - jepsen.util 6477 :invoke :write [29 47] INFO [2019-12-17 12:33:30,672] jepsen worker 74 - jepsen.util 7364 :ok :write [24 13] INFO [2019-12-17 12:33:30,672] jepsen worker 74 - jepsen.util 7364 :invoke :write [24 44] INFO [2019-12-17 12:33:30,697] jepsen worker 55 - jepsen.util 8155 :ok :write [18 12] INFO [2019-12-17 12:33:30,697] jepsen worker 55 - jepsen.util 8155 :invoke :write [18 42] INFO [2019-12-17 12:33:30,704] jepsen worker 32 - jepsen.util 7232 :ok :read [10 2] INFO [2019-12-17 12:33:30,704] jepsen worker 32 - jepsen.util 7232 :invoke :cas [10 [22 1]] INFO [2019-12-17 12:33:30,719] jepsen worker 30 - jepsen.util 7140 :ok :write [10 34] INFO [2019-12-17 12:33:30,720] jepsen worker 30 - jepsen.util 7140 :invoke :write [10 39] INFO [2019-12-17 12:33:30,727] jepsen worker 45 - jepsen.util 7245 :fail :cas [15 [47 42]] :ValueNotSwapFrom INFO [2019-12-17 12:33:30,727] jepsen worker 45 - jepsen.util 7245 :invoke :write [15 42] INFO [2019-12-17 12:33:30,733] jepsen worker 64 - jepsen.util 5554 :ok :write [21 45] INFO [2019-12-17 12:33:30,733] jepsen worker 64 - jepsen.util 5554 :invoke :read [21 nil] INFO [2019-12-17 12:33:30,736] jepsen worker 38 - jepsen.util 7328 :ok :read [12 39] INFO [2019-12-17 12:33:30,736] jepsen worker 38 - jepsen.util 7328 :invoke :cas [12 [30 18]] INFO [2019-12-17 12:33:30,749] jepsen worker 64 - jepsen.util 5554 :ok :read [21 45] INFO [2019-12-17 12:33:30,749] jepsen worker 64 - jepsen.util 5554 :invoke :cas [21 [2 3]] INFO [2019-12-17 12:33:30,760] jepsen worker 71 - jepsen.util 7991 :info :write [23 19] :RequestCanceledException INFO [2019-12-17 12:33:30,761] jepsen worker 71 - jepsen.util 8081 :invoke :cas [23 [49 24]] INFO [2019-12-17 12:33:30,768] jepsen worker 64 - jepsen.util 5554 :fail :cas [21 [2 3]] :ValueNotSwapFrom INFO [2019-12-17 12:33:30,769] jepsen worker 64 - jepsen.util 5554 :invoke :read [21 nil] INFO [2019-12-17 12:33:30,784] jepsen worker 37 - jepsen.util 6877 :fail :cas [12 [7 17]] :ValueNotSwapFrom INFO [2019-12-17 12:33:30,784] jepsen worker 37 - jepsen.util 6877 :invoke :read [12 nil] INFO [2019-12-17 12:33:30,808] jepsen worker 64 - jepsen.util 5554 :ok :read [21 45] INFO [2019-12-17 12:33:30,808] jepsen worker 64 - jepsen.util 5554 :invoke :write [21 21] INFO [2019-12-17 12:33:30,827] jepsen worker 52 - jepsen.util 7522 :ok :read [17 16] INFO [2019-12-17 12:33:30,827] jepsen worker 52 - jepsen.util 7522 :invoke :write [17 39] INFO [2019-12-17 12:33:30,856] jepsen worker 11 - jepsen.util 7391 :info :cas [3 [28 31]] :RequestCanceledException INFO [2019-12-17 12:33:30,859] jepsen worker 11 - jepsen.util 7481 :invoke :read [3 nil] INFO [2019-12-17 12:33:30,874] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41670,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:33:30,874] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41670,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:33:30,876] jepsen worker 64 - jepsen.util 5554 :ok :write [21 21] INFO [2019-12-17 12:33:30,876] jepsen worker 64 - jepsen.util 5554 :invoke :write [21 33] INFO [2019-12-17 12:33:30,924] jepsen worker 64 - jepsen.util 5554 :ok :write [21 33] INFO [2019-12-17 12:33:30,925] jepsen worker 64 - jepsen.util 5554 :invoke :cas [21 [13 7]] INFO [2019-12-17 12:33:30,926] jepsen worker 88 - jepsen.util 7108 :info :write [29 2] :RequestCanceledException INFO [2019-12-17 12:33:30,926] jepsen worker 89 - jepsen.util 6929 :info :write [29 25] :RequestCanceledException INFO [2019-12-17 12:33:30,928] jepsen worker 89 - jepsen.util 7019 :invoke :read [29 nil] INFO [2019-12-17 12:33:30,928] jepsen worker 88 - jepsen.util 7198 :invoke :read [29 nil] INFO [2019-12-17 12:33:30,958] jepsen worker 64 - jepsen.util 5554 :fail :cas [21 [13 7]] :ValueNotSwapFrom INFO [2019-12-17 12:33:30,958] jepsen worker 64 - jepsen.util 5554 :invoke :cas [21 [18 35]] INFO [2019-12-17 12:33:30,983] jepsen worker 64 - jepsen.util 5554 :fail :cas [21 [18 35]] :ValueNotSwapFrom INFO [2019-12-17 12:33:30,983] jepsen worker 64 - jepsen.util 5554 :invoke :read [21 nil] INFO [2019-12-17 12:33:31,011] jepsen worker 64 - jepsen.util 5554 :ok :read [21 33] INFO [2019-12-17 12:33:31,011] jepsen worker 64 - jepsen.util 5554 :invoke :write [21 1] INFO [2019-12-17 12:33:31,012] jepsen worker 4 - jepsen.util 6664 :ok :read [1 44] INFO [2019-12-17 12:33:31,012] jepsen worker 4 - jepsen.util 6664 :invoke :write [1 26] INFO [2019-12-17 12:33:31,027] jepsen worker 51 - jepsen.util 6981 :info :write [17 2] :RequestCanceledException INFO [2019-12-17 12:33:31,027] jepsen worker 14 - jepsen.util 7394 :info :write [4 37] :RequestCanceledException INFO [2019-12-17 12:33:31,030] jepsen worker 14 - jepsen.util 7484 :invoke :write [4 4] INFO [2019-12-17 12:33:31,030] jepsen worker 51 - jepsen.util 7071 :invoke :cas [17 [46 5]] INFO [2019-12-17 12:33:31,052] jepsen worker 5 - jepsen.util 6755 :ok :read [1 44] INFO [2019-12-17 12:33:31,053] jepsen worker 5 - jepsen.util 6755 :invoke :cas [1 [22 20]] INFO [2019-12-17 12:33:31,061] jepsen worker 20 - jepsen.util 6950 :ok :write [6 46] INFO [2019-12-17 12:33:31,061] jepsen worker 20 - jepsen.util 6950 :invoke :cas [6 [3 31]] INFO [2019-12-17 12:33:31,063] jepsen worker 47 - jepsen.util 7517 :ok :write [15 18] INFO [2019-12-17 12:33:31,064] jepsen worker 47 - jepsen.util 7517 :invoke :cas [15 [23 12]] INFO [2019-12-17 12:33:31,093] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41680,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:33:31,093] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41680,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:33:31,093] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41683,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:33:31,093] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41683,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:33:31,093] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41684,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:33:31,093] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41684,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:33:31,128] jepsen worker 13 - jepsen.util 6853 :info :write [4 41] :RequestCanceledException INFO [2019-12-17 12:33:31,132] jepsen worker 13 - jepsen.util 6943 :invoke :cas [4 [26 2]] INFO [2019-12-17 12:33:31,138] jepsen worker 5 - jepsen.util 6755 :fail :cas [1 [22 20]] :ValueNotSwapFrom INFO [2019-12-17 12:33:31,138] jepsen worker 5 - jepsen.util 6755 :invoke :read [1 nil] INFO [2019-12-17 12:33:31,147] jepsen worker 64 - jepsen.util 5554 :ok :write [21 1] INFO [2019-12-17 12:33:31,147] jepsen worker 64 - jepsen.util 5554 :invoke :read [21 nil] INFO [2019-12-17 12:33:31,151] jepsen worker 32 - jepsen.util 7232 :fail :cas [10 [22 1]] :ValueNotSwapFrom INFO [2019-12-17 12:33:31,151] jepsen worker 32 - jepsen.util 7232 :invoke :read [10 nil] INFO [2019-12-17 12:33:31,168] jepsen worker 9 - jepsen.util 7749 :ok :read [3 14] INFO [2019-12-17 12:33:31,168] jepsen worker 9 - jepsen.util 7749 :invoke :read [3 nil] INFO [2019-12-17 12:33:31,174] jepsen worker 53 - jepsen.util 6713 :ok :read [17 16] INFO [2019-12-17 12:33:31,174] jepsen worker 53 - jepsen.util 6713 :invoke :read [17 nil] INFO [2019-12-17 12:33:31,175] jepsen worker 29 - jepsen.util 5609 :ok :write [9 28] INFO [2019-12-17 12:33:31,176] jepsen worker 29 - jepsen.util 5609 :invoke :read [9 nil] INFO [2019-12-17 12:33:31,187] jepsen worker 71 - jepsen.util 8081 :fail :cas [23 [49 24]] :ValueNotSwapFrom INFO [2019-12-17 12:33:31,187] jepsen worker 71 - jepsen.util 8081 :invoke :read [23 nil] INFO [2019-12-17 12:33:31,196] jepsen worker 30 - jepsen.util 7140 :ok :write [10 39] INFO [2019-12-17 12:33:31,196] jepsen worker 30 - jepsen.util 7140 :invoke :cas [10 [16 6]] INFO [2019-12-17 12:33:31,202] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41701,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:33:31,202] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41701,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:33:31,220] jepsen worker 38 - jepsen.util 7328 :fail :cas [12 [30 18]] :ValueNotSwapFrom INFO [2019-12-17 12:33:31,220] jepsen worker 38 - jepsen.util 7328 :invoke :cas [12 [29 48]] INFO [2019-12-17 12:33:31,225] jepsen worker 64 - jepsen.util 5554 :ok :read [21 1] INFO [2019-12-17 12:33:31,225] jepsen worker 64 - jepsen.util 5554 :invoke :write [21 6] INFO [2019-12-17 12:33:31,229] jepsen worker 70 - jepsen.util 7450 :info :write [23 5] :RequestCanceledException INFO [2019-12-17 12:33:31,230] jepsen worker 20 - jepsen.util 6950 :fail :cas [6 [3 31]] :ValueNotSwapFrom INFO [2019-12-17 12:33:31,230] jepsen worker 20 - jepsen.util 6950 :invoke :write [6 37] INFO [2019-12-17 12:33:31,233] jepsen worker 70 - jepsen.util 7540 :invoke :read [23 nil] INFO [2019-12-17 12:33:31,236] jepsen worker 45 - jepsen.util 7245 :ok :write [15 42] INFO [2019-12-17 12:33:31,236] jepsen worker 45 - jepsen.util 7245 :invoke :write [15 47] INFO [2019-12-17 12:33:31,269] jepsen worker 9 - jepsen.util 7749 :ok :read [3 14] INFO [2019-12-17 12:33:31,269] jepsen worker 9 - jepsen.util 7749 :invoke :write [3 6] INFO [2019-12-17 12:33:31,311] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41710,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:33:31,312] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41710,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:33:31,312] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41712,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:33:31,312] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41712,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:33:31,320] jepsen worker 23 - jepsen.util 9923 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:33:31,321] jepsen worker 23 - jepsen.util 9923 :invoke :write [7 21] INFO [2019-12-17 12:33:31,329] jepsen worker 29 - jepsen.util 5609 :ok :read [9 28] INFO [2019-12-17 12:33:31,329] jepsen worker 29 - jepsen.util 5609 :invoke :read [9 nil] INFO [2019-12-17 12:33:31,340] jepsen worker 47 - jepsen.util 7517 :fail :cas [15 [23 12]] :ValueNotSwapFrom INFO [2019-12-17 12:33:31,340] jepsen worker 47 - jepsen.util 7517 :invoke :read [15 nil] INFO [2019-12-17 12:33:31,343] jepsen worker 30 - jepsen.util 7140 :fail :cas [10 [16 6]] :ValueNotSwapFrom INFO [2019-12-17 12:33:31,343] jepsen worker 30 - jepsen.util 7140 :invoke :write [10 36] INFO [2019-12-17 12:33:31,356] jepsen worker 34 - jepsen.util 9124 :info :cas [11 [45 15]] :RequestCanceledException INFO [2019-12-17 12:33:31,367] jepsen worker 34 - jepsen.util 9214 :invoke :cas [11 [48 36]] INFO [2019-12-17 12:33:31,395] jepsen worker 43 - jepsen.util 9133 :info :cas [14 [5 16]] :RequestCanceledException INFO [2019-12-17 12:33:31,401] jepsen worker 43 - jepsen.util 9223 :invoke :cas [14 [41 24]] INFO [2019-12-17 12:33:31,412] jepsen worker 21 - jepsen.util 9291 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:33:31,413] jepsen worker 21 - jepsen.util 9291 :invoke :write [7 38] INFO [2019-12-17 12:33:31,428] jepsen worker 75 - jepsen.util 9255 :info :write [25 26] :RequestCanceledException INFO [2019-12-17 12:33:31,429] jepsen worker 42 - jepsen.util 10482 :info :cas [14 [9 5]] :RequestCanceledException INFO [2019-12-17 12:33:31,429] jepsen worker 80 - jepsen.util 7190 :info :write [26 32] :RequestCanceledException INFO [2019-12-17 12:33:31,435] jepsen worker 42 - jepsen.util 10572 :invoke :read [14 nil] INFO [2019-12-17 12:33:31,435] jepsen worker 80 - jepsen.util 7280 :invoke :cas [26 [34 42]] INFO [2019-12-17 12:33:31,435] jepsen worker 75 - jepsen.util 9345 :invoke :read [25 nil] INFO [2019-12-17 12:33:31,438] jepsen worker 32 - jepsen.util 7232 :ok :read [10 39] INFO [2019-12-17 12:33:31,438] jepsen worker 32 - jepsen.util 7232 :invoke :read [10 nil] INFO [2019-12-17 12:33:31,450] jepsen worker 37 - jepsen.util 6877 :ok :read [12 39] INFO [2019-12-17 12:33:31,450] jepsen worker 37 - jepsen.util 6877 :invoke :cas [12 [11 30]] INFO [2019-12-17 12:33:31,450] jepsen worker 88 - jepsen.util 7198 :ok :read [29 8] INFO [2019-12-17 12:33:31,450] jepsen worker 88 - jepsen.util 7198 :invoke :write [29 14] INFO [2019-12-17 12:33:31,456] jepsen worker 66 - jepsen.util 9606 :info :write [22 29] :RequestCanceledException INFO [2019-12-17 12:33:31,456] jepsen worker 0 - jepsen.util 10170 :info :cas [0 [45 31]] :RequestCanceledException INFO [2019-12-17 12:33:31,461] jepsen worker 66 - jepsen.util 9696 :invoke :write [22 32] INFO [2019-12-17 12:33:31,461] jepsen worker 0 - jepsen.util 10260 :invoke :cas [0 [3 26]] INFO [2019-12-17 12:33:31,490] jepsen worker 89 - jepsen.util 7019 :ok :read [29 8] INFO [2019-12-17 12:33:31,490] jepsen worker 89 - jepsen.util 7019 :invoke :cas [29 [19 19]] INFO [2019-12-17 12:33:31,491] jepsen worker 38 - jepsen.util 7328 :fail :cas [12 [29 48]] :ValueNotSwapFrom INFO [2019-12-17 12:33:31,491] jepsen worker 38 - jepsen.util 7328 :invoke :write [12 32] INFO [2019-12-17 12:33:31,495] jepsen worker 62 - jepsen.util 9692 :info :write [20 38] :RequestCanceledException INFO [2019-12-17 12:33:31,498] jepsen worker 62 - jepsen.util 9782 :invoke :write [20 39] INFO [2019-12-17 12:33:31,513] jepsen worker 33 - jepsen.util 9483 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:33:31,513] jepsen worker 33 - jepsen.util 9483 :invoke :cas [11 [1 13]] INFO [2019-12-17 12:33:31,518] jepsen worker 5 - jepsen.util 6755 :ok :read [1 44] INFO [2019-12-17 12:33:31,519] jepsen worker 5 - jepsen.util 6755 :invoke :cas [1 [39 3]] INFO [2019-12-17 12:33:31,527] jepsen worker 83 - jepsen.util 10163 :info :cas [27 [17 12]] :RequestCanceledException INFO [2019-12-17 12:33:31,528] jepsen worker 58 - jepsen.util 10408 :info :write [19 24] :RequestCanceledException INFO [2019-12-17 12:33:31,528] jepsen worker 68 - jepsen.util 9248 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:33:31,528] jepsen worker 68 - jepsen.util 9248 :invoke :read [22 nil] INFO [2019-12-17 12:33:31,531] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41727,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:33:31,532] jepsen worker 83 - jepsen.util 10253 :invoke :cas [27 [2 1]] INFO [2019-12-17 12:33:31,533] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41727,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:33:31,533] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41730,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:33:31,533] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41730,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:33:31,534] jepsen worker 58 - jepsen.util 10498 :invoke :write [19 41] INFO [2019-12-17 12:33:31,543] jepsen worker 25 - jepsen.util 10195 :info :write [8 30] :RequestCanceledException INFO [2019-12-17 12:33:31,543] jepsen worker 60 - jepsen.util 10500 :info :write [20 32] :RequestCanceledException INFO [2019-12-17 12:33:31,544] jepsen worker 25 - jepsen.util 10285 :invoke :cas [8 [33 11]] INFO [2019-12-17 12:33:31,545] jepsen worker 60 - jepsen.util 10590 :invoke :write [20 23] INFO [2019-12-17 12:33:31,560] jepsen worker 22 - jepsen.util 9652 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:33:31,560] jepsen worker 86 - jepsen.util 10256 :info :cas [28 [36 35]] :RequestCanceledException INFO [2019-12-17 12:33:31,560] jepsen worker 22 - jepsen.util 9652 :invoke :read [7 nil] INFO [2019-12-17 12:33:31,564] jepsen worker 86 - jepsen.util 10346 :invoke :write [28 41] INFO [2019-12-17 12:33:31,583] jepsen worker 45 - jepsen.util 7245 :ok :write [15 47] INFO [2019-12-17 12:33:31,583] jepsen worker 45 - jepsen.util 7245 :invoke :write [15 3] INFO [2019-12-17 12:33:31,594] jepsen worker 20 - jepsen.util 6950 :ok :write [6 37] INFO [2019-12-17 12:33:31,594] jepsen worker 20 - jepsen.util 6950 :invoke :read [6 nil] INFO [2019-12-17 12:33:31,594] jepsen worker 26 - jepsen.util 9926 :info :write [8 35] :RequestCanceledException INFO [2019-12-17 12:33:31,604] jepsen worker 26 - jepsen.util 10016 :invoke :cas [8 [47 27]] INFO [2019-12-17 12:33:31,606] jepsen worker 67 - jepsen.util 9787 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:33:31,607] jepsen worker 77 - jepsen.util 9977 :info :write [25 11] :RequestCanceledException INFO [2019-12-17 12:33:31,607] jepsen worker 67 - jepsen.util 9787 :invoke :read [22 nil] INFO [2019-12-17 12:33:31,607] jepsen worker 79 - jepsen.util 7189 :info :write [26 11] :RequestCanceledException INFO [2019-12-17 12:33:31,613] jepsen worker 79 - jepsen.util 7279 :invoke :cas [26 [49 9]] INFO [2019-12-17 12:33:31,613] jepsen worker 77 - jepsen.util 10067 :invoke :read [25 nil] INFO [2019-12-17 12:33:31,617] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:33:31,617] jepsen nemesis - couchbase.util Rebalance check count: 319 INFO [2019-12-17 12:33:31,618] jepsen nemesis - couchbase.util Rebalance stuck count: 300 INFO [2019-12-17 12:33:31,627] jepsen worker 41 - jepsen.util 10211 :info :write [13 20] :RequestCanceledException INFO [2019-12-17 12:33:31,627] jepsen worker 1 - jepsen.util 9721 :info :write [0 48] :RequestCanceledException INFO [2019-12-17 12:33:31,628] jepsen worker 59 - jepsen.util 9689 :info :write [19 15] :RequestCanceledException INFO [2019-12-17 12:33:31,636] jepsen worker 1 - jepsen.util 9811 :invoke :write [0 24] INFO [2019-12-17 12:33:31,636] jepsen worker 41 - jepsen.util 10301 :invoke :cas [13 [43 46]] INFO [2019-12-17 12:33:31,636] jepsen worker 59 - jepsen.util 9779 :invoke :read [19 nil] INFO [2019-12-17 12:33:31,643] jepsen worker 50 - jepsen.util 10310 :info :cas [16 [10 47]] :RequestCanceledException INFO [2019-12-17 12:33:31,643] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41763,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:33:31,644] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41763,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:33:31,644] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41765,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:33:31,644] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41765,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:33:31,650] jepsen worker 50 - jepsen.util 10400 :invoke :write [16 26] INFO [2019-12-17 12:33:31,659] jepsen worker 24 - jepsen.util 8394 :info :cas [8 [45 4]] :RequestCanceledException INFO [2019-12-17 12:33:31,686] jepsen worker 32 - jepsen.util 7232 :ok :read [10 39] INFO [2019-12-17 12:33:31,686] jepsen worker 32 - jepsen.util 7232 :invoke :write [10 42] INFO [2019-12-17 12:33:31,695] jepsen worker 82 - jepsen.util 10432 :info :write [27 32] :RequestCanceledException INFO [2019-12-17 12:33:31,701] jepsen worker 24 - jepsen.util 8484 :invoke :cas [8 [11 37]] INFO [2019-12-17 12:33:31,728] jepsen worker 15 - jepsen.util 8745 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:33:31,729] jepsen worker 15 - jepsen.util 8745 :invoke :read [5 nil] INFO [2019-12-17 12:33:31,735] jepsen worker 82 - jepsen.util 10522 :invoke :cas [27 [33 2]] INFO [2019-12-17 12:33:31,796] jepsen worker 36 - jepsen.util 7506 :info :write [12 17] :RequestCanceledException INFO [2019-12-17 12:33:31,812] jepsen worker 27 - jepsen.util 7137 :info :cas [9 [28 2]] :RequestCanceledException INFO [2019-12-17 12:33:31,818] jepsen worker 36 - jepsen.util 7596 :invoke :cas [12 [4 10]] INFO [2019-12-17 12:33:31,852] jepsen worker 27 - jepsen.util 7227 :invoke :write [9 42] INFO [2019-12-17 12:33:31,862] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41774,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:33:31,863] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41774,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:33:31,864] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41775,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:33:31,866] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41775,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:33:31,868] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41776,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:33:31,871] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41776,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:33:31,913] jepsen worker 12 - jepsen.util 7662 :info :write [4 24] :RequestCanceledException INFO [2019-12-17 12:33:31,937] jepsen worker 76 - jepsen.util 9886 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:33:31,938] jepsen worker 76 - jepsen.util 9886 :invoke :write [25 16] INFO [2019-12-17 12:33:31,950] jepsen worker 6 - jepsen.util 10446 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:33:31,950] jepsen worker 6 - jepsen.util 10446 :invoke :write [2 44] INFO [2019-12-17 12:33:31,974] jepsen worker 12 - jepsen.util 7752 :invoke :read [4 nil] INFO [2019-12-17 12:33:31,990] jepsen worker 20 - jepsen.util 6950 :ok :read [6 37] INFO [2019-12-17 12:33:31,990] jepsen worker 20 - jepsen.util 6950 :invoke :read [6 nil] INFO [2019-12-17 12:33:32,021] jepsen worker 13 - jepsen.util 6943 :fail :cas [4 [26 2]] :ValueNotSwapFrom INFO [2019-12-17 12:33:32,021] jepsen worker 13 - jepsen.util 6943 :invoke :read [4 nil] INFO [2019-12-17 12:33:32,029] jepsen worker 71 - jepsen.util 8081 :ok :read [23 4] INFO [2019-12-17 12:33:32,029] jepsen worker 71 - jepsen.util 8081 :invoke :read [23 nil] INFO [2019-12-17 12:33:32,042] jepsen worker 17 - jepsen.util 10097 :info :write [5 44] :RequestCanceledException INFO [2019-12-17 12:33:32,042] jepsen worker 81 - jepsen.util 9531 :info :write [27 45] :RequestCanceledException INFO [2019-12-17 12:33:32,044] jepsen worker 70 - jepsen.util 7540 :ok :read [23 4] INFO [2019-12-17 12:33:32,045] jepsen worker 70 - jepsen.util 7540 :invoke :read [23 nil] INFO [2019-12-17 12:33:32,045] jepsen worker 81 - jepsen.util 9621 :invoke :cas [27 [30 0]] INFO [2019-12-17 12:33:32,046] jepsen worker 17 - jepsen.util 10187 :invoke :read [5 nil] INFO [2019-12-17 12:33:32,051] jepsen worker 37 - jepsen.util 6877 :fail :cas [12 [11 30]] :ValueNotSwapFrom INFO [2019-12-17 12:33:32,051] jepsen worker 37 - jepsen.util 6877 :invoke :cas [12 [5 23]] INFO [2019-12-17 12:33:32,059] jepsen worker 84 - jepsen.util 9354 :info :write [28 39] :RequestCanceledException INFO [2019-12-17 12:33:32,063] jepsen worker 84 - jepsen.util 9444 :invoke :cas [28 [39 8]] INFO [2019-12-17 12:33:32,067] jepsen worker 20 - jepsen.util 6950 :ok :read [6 37] INFO [2019-12-17 12:33:32,067] jepsen worker 20 - jepsen.util 6950 :invoke :write [6 17] INFO [2019-12-17 12:33:32,068] jepsen worker 89 - jepsen.util 7019 :fail :cas [29 [19 19]] :ValueNotSwapFrom INFO [2019-12-17 12:33:32,068] jepsen worker 89 - jepsen.util 7019 :invoke :write [29 23] INFO [2019-12-17 12:33:32,083] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41782,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:33:32,083] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41782,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:33:32,083] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41785,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:33:32,084] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41785,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:33:32,092] jepsen worker 39 - jepsen.util 9579 :info :write [13 34] :RequestCanceledException INFO [2019-12-17 12:33:32,092] jepsen worker 57 - jepsen.util 9597 :info :write [19 34] :RequestCanceledException INFO [2019-12-17 12:33:32,095] jepsen worker 39 - jepsen.util 9669 :invoke :write [13 49] INFO [2019-12-17 12:33:32,095] jepsen worker 57 - jepsen.util 9687 :invoke :cas [19 [49 43]] INFO [2019-12-17 12:33:32,129] jepsen worker 40 - jepsen.util 10210 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:33:32,129] jepsen worker 40 - jepsen.util 10210 :invoke :cas [13 [42 16]] INFO [2019-12-17 12:33:32,141] jepsen worker 48 - jepsen.util 8868 :info :write [16 38] :RequestCanceledException INFO [2019-12-17 12:33:32,146] jepsen worker 48 - jepsen.util 8958 :invoke :read [16 nil] INFO [2019-12-17 12:33:32,151] jepsen worker 79 - jepsen.util 7279 :fail :cas [26 [49 9]] :ValueNotSwapFrom INFO [2019-12-17 12:33:32,151] jepsen worker 79 - jepsen.util 7279 :invoke :write [26 0] INFO [2019-12-17 12:33:32,159] jepsen worker 44 - jepsen.util 10394 :info :write [14 29] :RequestCanceledException INFO [2019-12-17 12:33:32,162] jepsen worker 44 - jepsen.util 10484 :invoke :cas [14 [39 16]] INFO [2019-12-17 12:33:32,169] jepsen worker 5 - jepsen.util 6755 :fail :cas [1 [39 3]] :ValueNotSwapFrom INFO [2019-12-17 12:33:32,169] jepsen worker 5 - jepsen.util 6755 :invoke :read [1 nil] INFO [2019-12-17 12:33:32,204] jepsen worker 45 - jepsen.util 7245 :ok :write [15 3] INFO [2019-12-17 12:33:32,204] jepsen worker 45 - jepsen.util 7245 :invoke :read [15 nil] INFO [2019-12-17 12:33:32,205] jepsen worker 30 - jepsen.util 7140 :ok :write [10 36] INFO [2019-12-17 12:33:32,205] jepsen worker 30 - jepsen.util 7140 :invoke :read [10 nil] INFO [2019-12-17 12:33:32,205] jepsen worker 20 - jepsen.util 6950 :ok :write [6 17] INFO [2019-12-17 12:33:32,206] jepsen worker 20 - jepsen.util 6950 :invoke :cas [6 [34 26]] INFO [2019-12-17 12:33:32,213] jepsen worker 8 - jepsen.util 9728 :info :write [2 2] :RequestCanceledException INFO [2019-12-17 12:33:32,214] jepsen worker 8 - jepsen.util 9818 :invoke :cas [2 [41 29]] INFO [2019-12-17 12:33:32,227] jepsen worker 72 - jepsen.util 7092 :info :write [24 20] :RequestCanceledException INFO [2019-12-17 12:33:32,228] jepsen worker 61 - jepsen.util 10411 :info :write [20 20] :RequestCanceledException INFO [2019-12-17 12:33:32,228] jepsen worker 65 - jepsen.util 5915 :info :cas [21 [38 29]] :RequestCanceledException INFO [2019-12-17 12:33:32,229] jepsen worker 65 - jepsen.util 6005 :invoke :read [21 nil] INFO [2019-12-17 12:33:32,229] jepsen worker 72 - jepsen.util 7182 :invoke :write [24 18] INFO [2019-12-17 12:33:32,230] jepsen worker 61 - jepsen.util 10501 :invoke :read [20 nil] INFO [2019-12-17 12:33:32,241] jepsen worker 49 - jepsen.util 9139 :info :write [16 39] :RequestCanceledException INFO [2019-12-17 12:33:32,241] jepsen worker 16 - jepsen.util 10906 :info :write [5 18] :RequestCanceledException INFO [2019-12-17 12:33:32,246] jepsen worker 16 - jepsen.util 10996 :invoke :write [5 9] INFO [2019-12-17 12:33:32,246] jepsen worker 49 - jepsen.util 9229 :invoke :write [16 42] INFO [2019-12-17 12:33:32,256] jepsen worker 85 - jepsen.util 10705 :info :write [28 28] :RequestCanceledException INFO [2019-12-17 12:33:32,261] jepsen worker 85 - jepsen.util 10795 :invoke :read [28 nil] INFO [2019-12-17 12:33:32,276] jepsen worker 30 - jepsen.util 7140 :ok :read [10 36] INFO [2019-12-17 12:33:32,277] jepsen worker 30 - jepsen.util 7140 :invoke :read [10 nil] INFO [2019-12-17 12:33:32,281] jepsen worker 65 - jepsen.util 6005 :ok :read [21 1] INFO [2019-12-17 12:33:32,282] jepsen worker 65 - jepsen.util 6005 :invoke :cas [21 [4 0]] INFO [2019-12-17 12:33:32,290] jepsen worker 20 - jepsen.util 6950 :fail :cas [6 [34 26]] :ValueNotSwapFrom INFO [2019-12-17 12:33:32,290] jepsen worker 20 - jepsen.util 6950 :invoke :write [6 7] INFO [2019-12-17 12:33:32,292] jepsen worker 7 - jepsen.util 9907 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:33:32,292] jepsen worker 7 - jepsen.util 9907 :invoke :cas [2 [36 3]] INFO [2019-12-17 12:33:32,302] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41806,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:33:32,302] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41806,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:33:32,302] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41808,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:33:32,302] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41808,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:33:32,305] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41816,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:33:32,305] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41816,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:33:32,306] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41819,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:33:32,306] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41819,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:33:32,306] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41820,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:33:32,306] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41820,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:33:32,312] jepsen worker 65 - jepsen.util 6005 :fail :cas [21 [4 0]] :ValueNotSwapFrom INFO [2019-12-17 12:33:32,312] jepsen worker 65 - jepsen.util 6005 :invoke :read [21 nil] INFO [2019-12-17 12:33:32,313] jepsen worker 19 - jepsen.util 7129 :info :write [6 41] :RequestCanceledException INFO [2019-12-17 12:33:32,313] jepsen worker 10 - jepsen.util 7210 :fail :read [3 nil] :RequestTimeoutException INFO [2019-12-17 12:33:32,313] jepsen worker 28 - jepsen.util 5608 :fail :read [9 nil] :RequestTimeoutException INFO [2019-12-17 12:33:32,313] jepsen worker 10 - jepsen.util 7210 :invoke :write [3 11] INFO [2019-12-17 12:33:32,313] jepsen worker 28 - jepsen.util 5608 :invoke :cas [9 [23 45]] INFO [2019-12-17 12:33:32,314] jepsen worker 19 - jepsen.util 7219 :invoke :read [6 nil] INFO [2019-12-17 12:33:32,327] jepsen worker 18 - jepsen.util 7218 :info :write [6 8] :RequestCanceledException INFO [2019-12-17 12:33:32,327] jepsen worker 35 - jepsen.util 9665 :info :cas [11 [7 43]] :RequestCanceledException INFO [2019-12-17 12:33:32,330] jepsen worker 18 - jepsen.util 7308 :invoke :write [6 21] INFO [2019-12-17 12:33:32,330] jepsen worker 35 - jepsen.util 9755 :invoke :write [11 34] INFO [2019-12-17 12:33:32,333] jepsen worker 56 - jepsen.util 7976 :info :write [18 8] :RequestCanceledException INFO [2019-12-17 12:33:32,334] jepsen worker 46 - jepsen.util 7606 :fail :read [15 nil] :RequestTimeoutException INFO [2019-12-17 12:33:32,334] jepsen worker 46 - jepsen.util 7606 :invoke :cas [15 [0 41]] INFO [2019-12-17 12:33:32,337] jepsen worker 56 - jepsen.util 8066 :invoke :read [18 nil] INFO [2019-12-17 12:33:32,357] jepsen worker 65 - jepsen.util 6005 :ok :read [21 1] INFO [2019-12-17 12:33:32,357] jepsen worker 65 - jepsen.util 6005 :invoke :cas [21 [31 8]] INFO [2019-12-17 12:33:32,379] jepsen worker 28 - jepsen.util 5608 :fail :cas [9 [23 45]] :ValueNotSwapFrom INFO [2019-12-17 12:33:32,379] jepsen worker 28 - jepsen.util 5608 :invoke :write [9 22] INFO [2019-12-17 12:33:32,405] jepsen worker 19 - jepsen.util 7219 :ok :read [6 17] INFO [2019-12-17 12:33:32,405] jepsen worker 19 - jepsen.util 7219 :invoke :cas [6 [6 12]] INFO [2019-12-17 12:33:32,412] jepsen worker 69 - jepsen.util 7269 :info :write [23 29] :RequestCanceledException INFO [2019-12-17 12:33:32,419] jepsen worker 69 - jepsen.util 7359 :invoke :read [23 nil] INFO [2019-12-17 12:33:32,453] jepsen worker 70 - jepsen.util 7540 :ok :read [23 4] INFO [2019-12-17 12:33:32,453] jepsen worker 70 - jepsen.util 7540 :invoke :cas [23 [23 35]] INFO [2019-12-17 12:33:32,458] jepsen worker 3 - jepsen.util 7203 :info :write [1 5] :RequestCanceledException INFO [2019-12-17 12:33:32,458] jepsen worker 78 - jepsen.util 7368 :info :write [26 30] :RequestCanceledException INFO [2019-12-17 12:33:32,460] jepsen worker 78 - jepsen.util 7458 :invoke :read [26 nil] INFO [2019-12-17 12:33:32,460] jepsen worker 3 - jepsen.util 7293 :invoke :write [1 6] INFO [2019-12-17 12:33:32,475] jepsen worker 69 - jepsen.util 7359 :ok :read [23 4] INFO [2019-12-17 12:33:32,476] jepsen worker 69 - jepsen.util 7359 :invoke :write [23 45] INFO [2019-12-17 12:33:32,495] jepsen worker 32 - jepsen.util 7232 :ok :write [10 42] INFO [2019-12-17 12:33:32,495] jepsen worker 32 - jepsen.util 7232 :invoke :write [10 5] INFO [2019-12-17 12:33:32,500] jepsen worker 72 - jepsen.util 7182 :ok :write [24 18] INFO [2019-12-17 12:33:32,500] jepsen worker 72 - jepsen.util 7182 :invoke :cas [24 [27 19]] INFO [2019-12-17 12:33:32,524] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41845,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:33:32,524] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41845,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:33:32,525] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41848,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:33:32,525] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41848,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:33:32,525] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41849,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:33:32,525] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41849,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:33:32,528] jepsen worker 31 - jepsen.util 6511 :info :write [10 48] :RequestCanceledException INFO [2019-12-17 12:33:32,531] jepsen worker 31 - jepsen.util 6601 :invoke :cas [10 [9 42]] INFO [2019-12-17 12:33:32,547] jepsen worker 30 - jepsen.util 7140 :ok :read [10 42] INFO [2019-12-17 12:33:32,547] jepsen worker 30 - jepsen.util 7140 :invoke :read [10 nil] INFO [2019-12-17 12:33:32,548] jepsen worker 27 - jepsen.util 7227 :ok :write [9 42] INFO [2019-12-17 12:33:32,548] jepsen worker 27 - jepsen.util 7227 :invoke :cas [9 [33 41]] INFO [2019-12-17 12:33:32,551] jepsen worker 70 - jepsen.util 7540 :fail :cas [23 [23 35]] :ValueNotSwapFrom INFO [2019-12-17 12:33:32,552] jepsen worker 70 - jepsen.util 7540 :invoke :write [23 7] INFO [2019-12-17 12:33:32,595] jepsen worker 72 - jepsen.util 7182 :fail :cas [24 [27 19]] :ValueNotSwapFrom INFO [2019-12-17 12:33:32,595] jepsen worker 72 - jepsen.util 7182 :invoke :read [24 nil] INFO [2019-12-17 12:33:32,596] jepsen worker 63 - jepsen.util 6093 :info :write [21 46] :RequestCanceledException INFO [2019-12-17 12:33:32,603] jepsen worker 63 - jepsen.util 6183 :invoke :write [21 3] INFO [2019-12-17 12:33:32,629] jepsen worker 2 - jepsen.util 9362 :info :write [0 13] :RequestCanceledException INFO [2019-12-17 12:33:32,629] jepsen worker 54 - jepsen.util 8064 :info :cas [18 [30 11]] :RequestCanceledException INFO [2019-12-17 12:33:32,643] jepsen worker 54 - jepsen.util 8154 :invoke :read [18 nil] INFO [2019-12-17 12:33:32,645] jepsen worker 2 - jepsen.util 9452 :invoke :cas [0 [35 5]] INFO [2019-12-17 12:33:32,646] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:33:32,648] jepsen worker 51 - jepsen.util 7071 :fail :cas [17 [46 5]] :ValueNotSwapFrom INFO [2019-12-17 12:33:32,659] jepsen worker 73 - jepsen.util 6823 :info :write [24 29] :RequestCanceledException INFO [2019-12-17 12:33:32,666] jepsen worker 31 - jepsen.util 6601 :fail :cas [10 [9 42]] :ValueNotSwapFrom INFO [2019-12-17 12:33:32,682] jepsen nemesis - couchbase.util Rebalance check count: 320 INFO [2019-12-17 12:33:32,682] jepsen worker 51 - jepsen.util 7071 :invoke :read [17 nil] INFO [2019-12-17 12:33:32,682] jepsen worker 31 - jepsen.util 6601 :invoke :read [10 nil] INFO [2019-12-17 12:33:32,682] jepsen nemesis - couchbase.util Rebalance stuck count: 301 INFO [2019-12-17 12:33:32,687] jepsen worker 73 - jepsen.util 6913 :invoke :cas [24 [21 28]] INFO [2019-12-17 12:33:32,711] jepsen worker 30 - jepsen.util 7140 :ok :read [10 42] INFO [2019-12-17 12:33:32,711] jepsen worker 30 - jepsen.util 7140 :invoke :cas [10 [17 18]] INFO [2019-12-17 12:33:32,713] jepsen worker 87 - jepsen.util 6477 :info :write [29 47] :RequestCanceledException INFO [2019-12-17 12:33:32,718] jepsen worker 87 - jepsen.util 6567 :invoke :read [29 nil] INFO [2019-12-17 12:33:32,729] jepsen worker 55 - jepsen.util 8155 :info :write [18 42] :RequestCanceledException INFO [2019-12-17 12:33:32,742] jepsen worker 74 - jepsen.util 7364 :info :write [24 44] :RequestCanceledException INFO [2019-12-17 12:33:32,744] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41861,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:33:32,745] jepsen worker 36 - jepsen.util 7596 :fail :cas [12 [4 10]] :ValueNotSwapFrom INFO [2019-12-17 12:33:32,747] jepsen worker 78 - jepsen.util 7458 :ok :read [26 31] INFO [2019-12-17 12:33:32,748] jepsen worker 55 - jepsen.util 8245 :invoke :read [18 nil] INFO [2019-12-17 12:33:32,776] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41861,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:33:32,776] jepsen worker 36 - jepsen.util 7596 :invoke :write [12 1] INFO [2019-12-17 12:33:32,776] jepsen worker 78 - jepsen.util 7458 :invoke :cas [26 [31 30]] INFO [2019-12-17 12:33:32,777] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41865,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:33:32,778] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41865,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:33:32,779] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41866,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:33:32,780] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41866,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:33:32,789] jepsen worker 72 - jepsen.util 7182 :ok :read [24 18] INFO [2019-12-17 12:33:32,789] jepsen worker 72 - jepsen.util 7182 :invoke :write [24 10] INFO [2019-12-17 12:33:32,789] jepsen worker 74 - jepsen.util 7454 :invoke :read [24 nil] INFO [2019-12-17 12:33:32,800] jepsen worker 56 - jepsen.util 8066 :ok :read [18 12] INFO [2019-12-17 12:33:32,801] jepsen worker 56 - jepsen.util 8066 :invoke :write [18 42] INFO [2019-12-17 12:33:32,906] jepsen worker 31 - jepsen.util 6601 :ok :read [10 42] INFO [2019-12-17 12:33:32,907] jepsen worker 31 - jepsen.util 6601 :invoke :cas [10 [26 41]] INFO [2019-12-17 12:33:32,926] jepsen worker 52 - jepsen.util 7522 :info :write [17 39] :RequestCanceledException INFO [2019-12-17 12:33:32,941] jepsen worker 11 - jepsen.util 7481 :fail :read [3 nil] :RequestTimeoutException INFO [2019-12-17 12:33:32,941] jepsen worker 11 - jepsen.util 7481 :invoke :read [3 nil] INFO [2019-12-17 12:33:32,942] jepsen worker 52 - jepsen.util 7612 :invoke :read [17 nil] INFO [2019-12-17 12:33:32,959] jepsen worker 12 - jepsen.util 7752 :ok :read [4 0] INFO [2019-12-17 12:33:32,959] jepsen worker 12 - jepsen.util 7752 :invoke :write [4 15] INFO [2019-12-17 12:33:32,964] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41883,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:33:32,966] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41883,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:33:32,967] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41884,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:33:32,967] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41884,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:33:32,977] jepsen worker 30 - jepsen.util 7140 :fail :cas [10 [17 18]] :ValueNotSwapFrom INFO [2019-12-17 12:33:32,977] jepsen worker 30 - jepsen.util 7140 :invoke :write [10 15] INFO [2019-12-17 12:33:32,987] jepsen worker 53 - jepsen.util 6713 :ok :read [17 16] INFO [2019-12-17 12:33:32,987] jepsen worker 53 - jepsen.util 6713 :invoke :write [17 42] INFO [2019-12-17 12:33:32,995] jepsen worker 80 - jepsen.util 7280 :fail :cas [26 [34 42]] :ValueNotSwapFrom INFO [2019-12-17 12:33:32,996] jepsen worker 80 - jepsen.util 7280 :invoke :write [26 39] INFO [2019-12-17 12:33:33,020] jepsen worker 64 - jepsen.util 5554 :ok :write [21 6] INFO [2019-12-17 12:33:33,020] jepsen worker 64 - jepsen.util 5554 :invoke :write [21 42] INFO [2019-12-17 12:33:33,028] jepsen worker 5 - jepsen.util 6755 :ok :read [1 44] INFO [2019-12-17 12:33:33,028] jepsen worker 5 - jepsen.util 6755 :invoke :write [1 29] INFO [2019-12-17 12:33:33,040] jepsen worker 13 - jepsen.util 6943 :ok :read [4 0] INFO [2019-12-17 12:33:33,040] jepsen worker 13 - jepsen.util 6943 :invoke :read [4 nil] INFO [2019-12-17 12:33:33,075] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41891,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:33:33,075] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41891,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:33:33,075] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41895,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:33:33,076] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41895,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:33:33,096] jepsen worker 14 - jepsen.util 7484 :info :write [4 4] :RequestCanceledException INFO [2019-12-17 12:33:33,102] jepsen worker 14 - jepsen.util 7574 :invoke :cas [4 [37 13]] INFO [2019-12-17 12:33:33,112] jepsen worker 4 - jepsen.util 6664 :info :write [1 26] :RequestCanceledException INFO [2019-12-17 12:33:33,116] jepsen worker 4 - jepsen.util 6754 :invoke :read [1 nil] INFO [2019-12-17 12:33:33,129] jepsen worker 64 - jepsen.util 5554 :ok :write [21 42] INFO [2019-12-17 12:33:33,130] jepsen worker 64 - jepsen.util 5554 :invoke :write [21 12] INFO [2019-12-17 12:33:33,275] jepsen worker 71 - jepsen.util 8081 :ok :read [23 4] INFO [2019-12-17 12:33:33,276] jepsen worker 71 - jepsen.util 8081 :invoke :cas [23 [5 11]] INFO [2019-12-17 12:33:33,298] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41912,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:33:33,299] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41912,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:33:33,301] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41906,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:33:33,301] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41915,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:33:33,301] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41906,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:33:33,302] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41915,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:33:33,396] jepsen worker 9 - jepsen.util 7749 :info :write [3 6] :RequestCanceledException INFO [2019-12-17 12:33:33,396] jepsen worker 29 - jepsen.util 5609 :fail :read [9 nil] :RequestTimeoutException INFO [2019-12-17 12:33:33,397] jepsen worker 29 - jepsen.util 5609 :invoke :cas [9 [25 46]] INFO [2019-12-17 12:33:33,428] jepsen worker 64 - jepsen.util 5554 :ok :write [21 12] INFO [2019-12-17 12:33:33,428] jepsen worker 64 - jepsen.util 5554 :invoke :read [21 nil] INFO [2019-12-17 12:33:33,429] jepsen worker 23 - jepsen.util 9923 :info :write [7 21] :RequestCanceledException INFO [2019-12-17 12:33:33,432] jepsen worker 43 - jepsen.util 9223 :info :cas [14 [41 24]] :RequestCanceledException INFO [2019-12-17 12:33:33,448] jepsen worker 47 - jepsen.util 7517 :fail :read [15 nil] :RequestTimeoutException INFO [2019-12-17 12:33:33,452] jepsen worker 47 - jepsen.util 7517 :invoke :cas [15 [26 23]] INFO [2019-12-17 12:33:33,460] jepsen worker 34 - jepsen.util 9214 :info :cas [11 [48 36]] :RequestCanceledException INFO [2019-12-17 12:33:33,480] jepsen worker 9 - jepsen.util 7839 :invoke :write [3 21] INFO [2019-12-17 12:33:33,489] jepsen worker 23 - jepsen.util 10013 :invoke :write [7 25] INFO [2019-12-17 12:33:33,496] jepsen worker 42 - jepsen.util 10572 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:33:33,500] jepsen worker 42 - jepsen.util 10572 :invoke :cas [14 [0 33]] INFO [2019-12-17 12:33:33,504] jepsen worker 43 - jepsen.util 9313 :invoke :write [14 36] INFO [2019-12-17 12:33:33,507] jepsen worker 34 - jepsen.util 9304 :invoke :read [11 nil] INFO [2019-12-17 12:33:33,513] jepsen worker 21 - jepsen.util 9291 :info :write [7 38] :RequestCanceledException INFO [2019-12-17 12:33:33,529] jepsen worker 75 - jepsen.util 9345 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:33:33,529] jepsen worker 88 - jepsen.util 7198 :info :write [29 14] :RequestCanceledException INFO [2019-12-17 12:33:33,531] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41922,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:33:33,531] jepsen worker 38 - jepsen.util 7328 :info :write [12 32] :RequestCanceledException INFO [2019-12-17 12:33:33,531] jepsen worker 62 - jepsen.util 9782 :info :write [20 39] :RequestCanceledException INFO [2019-12-17 12:33:33,541] jepsen worker 66 - jepsen.util 9696 :info :write [22 32] :RequestCanceledException INFO [2019-12-17 12:33:33,544] jepsen worker 21 - jepsen.util 9381 :invoke :read [7 nil] INFO [2019-12-17 12:33:33,554] jepsen worker 75 - jepsen.util 9345 :invoke :read [25 nil] INFO [2019-12-17 12:33:33,556] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41922,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:33:33,561] jepsen worker 0 - jepsen.util 10260 :info :cas [0 [3 26]] :RequestCanceledException INFO [2019-12-17 12:33:33,581] jepsen worker 38 - jepsen.util 7418 :invoke :write [12 10] INFO [2019-12-17 12:33:33,584] jepsen worker 88 - jepsen.util 7288 :invoke :write [29 43] INFO [2019-12-17 12:33:33,584] jepsen worker 62 - jepsen.util 9872 :invoke :write [20 29] INFO [2019-12-17 12:33:33,589] jepsen worker 66 - jepsen.util 9786 :invoke :read [22 nil] INFO [2019-12-17 12:33:33,591] jepsen worker 0 - jepsen.util 10350 :invoke :write [0 27] INFO [2019-12-17 12:33:33,595] jepsen worker 83 - jepsen.util 10253 :info :cas [27 [2 1]] :RequestCanceledException INFO [2019-12-17 12:33:33,618] jepsen worker 33 - jepsen.util 9483 :info :cas [11 [1 13]] :RequestCanceledException INFO [2019-12-17 12:33:33,621] jepsen worker 83 - jepsen.util 10343 :invoke :cas [27 [4 13]] INFO [2019-12-17 12:33:33,622] jepsen worker 58 - jepsen.util 10498 :info :write [19 41] :RequestCanceledException INFO [2019-12-17 12:33:33,629] jepsen worker 68 - jepsen.util 9248 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:33:33,629] jepsen worker 26 - jepsen.util 10016 :info :cas [8 [47 27]] :RequestCanceledException INFO [2019-12-17 12:33:33,629] jepsen worker 25 - jepsen.util 10285 :info :cas [8 [33 11]] :RequestCanceledException INFO [2019-12-17 12:33:33,630] jepsen worker 68 - jepsen.util 9248 :invoke :write [22 17] INFO [2019-12-17 12:33:33,643] jepsen worker 60 - jepsen.util 10590 :info :write [20 23] :RequestCanceledException INFO [2019-12-17 12:33:33,656] jepsen worker 52 - jepsen.util 7612 :ok :read [17 16] INFO [2019-12-17 12:33:33,657] jepsen worker 52 - jepsen.util 7612 :invoke :cas [17 [33 2]] INFO [2019-12-17 12:33:33,659] jepsen worker 37 - jepsen.util 6877 :fail :cas [12 [5 23]] :ValueNotSwapFrom INFO [2019-12-17 12:33:33,659] jepsen worker 37 - jepsen.util 6877 :invoke :read [12 nil] INFO [2019-12-17 12:33:33,661] jepsen worker 22 - jepsen.util 9652 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:33:33,662] jepsen worker 86 - jepsen.util 10346 :info :write [28 41] :RequestCanceledException INFO [2019-12-17 12:33:33,663] jepsen worker 59 - jepsen.util 9779 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:33:33,664] jepsen worker 22 - jepsen.util 9652 :invoke :cas [7 [20 23]] INFO [2019-12-17 12:33:33,665] jepsen worker 59 - jepsen.util 9779 :invoke :read [19 nil] INFO [2019-12-17 12:33:33,670] jepsen worker 33 - jepsen.util 9573 :invoke :write [11 1] INFO [2019-12-17 12:33:33,681] jepsen worker 58 - jepsen.util 10588 :invoke :write [19 23] INFO [2019-12-17 12:33:33,683] jepsen worker 26 - jepsen.util 10106 :invoke :cas [8 [33 14]] INFO [2019-12-17 12:33:33,684] jepsen worker 25 - jepsen.util 10375 :invoke :write [8 26] INFO [2019-12-17 12:33:33,685] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41940,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:33:33,686] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41940,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:33:33,689] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41941,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:33:33,690] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41941,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:33:33,694] jepsen worker 60 - jepsen.util 10680 :invoke :cas [20 [43 32]] INFO [2019-12-17 12:33:33,711] jepsen worker 67 - jepsen.util 9787 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:33:33,712] jepsen worker 67 - jepsen.util 9787 :invoke :write [22 9] INFO [2019-12-17 12:33:33,713] jepsen worker 77 - jepsen.util 10067 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:33:33,713] jepsen worker 86 - jepsen.util 10436 :invoke :read [28 nil] INFO [2019-12-17 12:33:33,714] jepsen worker 77 - jepsen.util 10067 :invoke :read [25 nil] INFO [2019-12-17 12:33:33,715] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:33:33,715] jepsen nemesis - couchbase.util Rebalance check count: 321 INFO [2019-12-17 12:33:33,716] jepsen nemesis - couchbase.util Rebalance stuck count: 302 INFO [2019-12-17 12:33:33,743] jepsen worker 41 - jepsen.util 10301 :info :cas [13 [43 46]] :RequestCanceledException INFO [2019-12-17 12:33:33,759] jepsen worker 41 - jepsen.util 10391 :invoke :read [13 nil] INFO [2019-12-17 12:33:33,775] jepsen worker 54 - jepsen.util 8154 :ok :read [18 12] INFO [2019-12-17 12:33:33,776] jepsen worker 54 - jepsen.util 8154 :invoke :cas [18 [41 45]] INFO [2019-12-17 12:33:33,789] jepsen worker 55 - jepsen.util 8245 :ok :read [18 12] INFO [2019-12-17 12:33:33,789] jepsen worker 55 - jepsen.util 8245 :invoke :read [18 nil] INFO [2019-12-17 12:33:33,795] jepsen worker 24 - jepsen.util 8484 :info :cas [8 [11 37]] :RequestCanceledException INFO [2019-12-17 12:33:33,795] jepsen worker 51 - jepsen.util 7071 :ok :read [17 16] INFO [2019-12-17 12:33:33,795] jepsen worker 51 - jepsen.util 7071 :invoke :read [17 nil] INFO [2019-12-17 12:33:33,804] jepsen worker 87 - jepsen.util 6567 :ok :read [29 8] INFO [2019-12-17 12:33:33,805] jepsen worker 87 - jepsen.util 6567 :invoke :read [29 nil] INFO [2019-12-17 12:33:33,813] jepsen worker 82 - jepsen.util 10522 :info :cas [27 [33 2]] :RequestCanceledException INFO [2019-12-17 12:33:33,816] jepsen worker 24 - jepsen.util 8574 :invoke :write [8 5] INFO [2019-12-17 12:33:33,823] jepsen worker 82 - jepsen.util 10612 :invoke :read [27 nil] INFO [2019-12-17 12:33:33,828] jepsen worker 50 - jepsen.util 10400 :info :write [16 26] :RequestCanceledException INFO [2019-12-17 12:33:33,828] jepsen worker 1 - jepsen.util 9811 :info :write [0 24] :RequestCanceledException INFO [2019-12-17 12:33:33,828] jepsen worker 15 - jepsen.util 8745 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:33:33,829] jepsen worker 15 - jepsen.util 8745 :invoke :cas [5 [48 2]] INFO [2019-12-17 12:33:33,837] jepsen worker 1 - jepsen.util 9901 :invoke :write [0 3] INFO [2019-12-17 12:33:33,838] jepsen worker 50 - jepsen.util 10490 :invoke :write [16 5] INFO [2019-12-17 12:33:33,856] jepsen worker 64 - jepsen.util 5554 :ok :read [21 12] INFO [2019-12-17 12:33:33,856] jepsen worker 64 - jepsen.util 5554 :invoke :write [21 47] INFO [2019-12-17 12:33:33,865] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41969,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:33:33,866] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41969,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:33:33,867] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41972,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:33:33,867] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41972,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:33:33,921] jepsen worker 13 - jepsen.util 6943 :ok :read [4 0] INFO [2019-12-17 12:33:33,921] jepsen worker 13 - jepsen.util 6943 :invoke :read [4 nil] INFO [2019-12-17 12:33:33,926] jepsen worker 71 - jepsen.util 8081 :fail :cas [23 [5 11]] :ValueNotSwapFrom INFO [2019-12-17 12:33:33,926] jepsen worker 14 - jepsen.util 7574 :fail :cas [4 [37 13]] :ValueNotSwapFrom INFO [2019-12-17 12:33:33,927] jepsen worker 71 - jepsen.util 8081 :invoke :read [23 nil] INFO [2019-12-17 12:33:33,927] jepsen worker 14 - jepsen.util 7574 :invoke :read [4 nil] INFO [2019-12-17 12:33:33,933] jepsen worker 4 - jepsen.util 6754 :ok :read [1 44] INFO [2019-12-17 12:33:33,933] jepsen worker 4 - jepsen.util 6754 :invoke :read [1 nil] INFO [2019-12-17 12:33:33,974] jepsen worker 64 - jepsen.util 5554 :ok :write [21 47] INFO [2019-12-17 12:33:33,975] jepsen worker 64 - jepsen.util 5554 :invoke :read [21 nil] INFO [2019-12-17 12:33:34,023] jepsen worker 64 - jepsen.util 5554 :ok :read [21 47] INFO [2019-12-17 12:33:34,024] jepsen worker 64 - jepsen.util 5554 :invoke :read [21 nil] INFO [2019-12-17 12:33:34,027] jepsen worker 76 - jepsen.util 9886 :info :write [25 16] :RequestCanceledException INFO [2019-12-17 12:33:34,030] jepsen worker 76 - jepsen.util 9976 :invoke :cas [25 [2 36]] INFO [2019-12-17 12:33:34,041] jepsen worker 6 - jepsen.util 10446 :info :write [2 44] :RequestCanceledException INFO [2019-12-17 12:33:34,045] jepsen worker 6 - jepsen.util 10536 :invoke :cas [2 [18 37]] INFO [2019-12-17 12:33:34,054] jepsen worker 64 - jepsen.util 5554 :ok :read [21 47] INFO [2019-12-17 12:33:34,054] jepsen worker 64 - jepsen.util 5554 :invoke :write [21 11] INFO [2019-12-17 12:33:34,073] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41984,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:33:34,073] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41984,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:33:34,074] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41987,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:33:34,074] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":41987,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:33:34,092] jepsen worker 17 - jepsen.util 10187 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:33:34,092] jepsen worker 17 - jepsen.util 10187 :invoke :write [5 14] INFO [2019-12-17 12:33:34,107] jepsen worker 64 - jepsen.util 5554 :ok :write [21 11] INFO [2019-12-17 12:33:34,107] jepsen worker 64 - jepsen.util 5554 :invoke :read [21 nil] INFO [2019-12-17 12:33:34,112] jepsen worker 84 - jepsen.util 9444 :info :cas [28 [39 8]] :RequestCanceledException INFO [2019-12-17 12:33:34,115] jepsen worker 84 - jepsen.util 9534 :invoke :cas [28 [16 24]] INFO [2019-12-17 12:33:34,128] jepsen worker 81 - jepsen.util 9621 :info :cas [27 [30 0]] :RequestCanceledException INFO [2019-12-17 12:33:34,128] jepsen worker 39 - jepsen.util 9669 :info :write [13 49] :RequestCanceledException INFO [2019-12-17 12:33:34,132] jepsen worker 39 - jepsen.util 9759 :invoke :cas [13 [25 25]] INFO [2019-12-17 12:33:34,132] jepsen worker 81 - jepsen.util 9711 :invoke :cas [27 [8 9]] INFO [2019-12-17 12:33:34,142] jepsen worker 57 - jepsen.util 9687 :info :cas [19 [49 43]] :RequestCanceledException INFO [2019-12-17 12:33:34,158] jepsen worker 14 - jepsen.util 7574 :ok :read [4 0] INFO [2019-12-17 12:33:34,158] jepsen worker 37 - jepsen.util 6877 :ok :read [12 39] INFO [2019-12-17 12:33:34,159] jepsen worker 89 - jepsen.util 7019 :info :write [29 23] :RequestCanceledException INFO [2019-12-17 12:33:34,159] jepsen worker 48 - jepsen.util 8958 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:33:34,159] jepsen worker 20 - jepsen.util 6950 :info :write [6 7] :SyncWriteAmbiguous INFO [2019-12-17 12:33:34,159] jepsen worker 37 - jepsen.util 6877 :invoke :read [12 nil] INFO [2019-12-17 12:33:34,159] jepsen worker 14 - jepsen.util 7574 :invoke :write [4 17] INFO [2019-12-17 12:33:34,159] jepsen worker 57 - jepsen.util 9777 :invoke :write [19 26] INFO [2019-12-17 12:33:34,159] jepsen worker 48 - jepsen.util 8958 :invoke :write [16 2] INFO [2019-12-17 12:33:34,163] jepsen worker 89 - jepsen.util 7109 :invoke :cas [29 [45 30]] INFO [2019-12-17 12:33:34,163] jepsen worker 20 - jepsen.util 7040 :invoke :write [6 4] INFO [2019-12-17 12:33:34,171] jepsen worker 51 - jepsen.util 7071 :ok :read [17 16] INFO [2019-12-17 12:33:34,171] jepsen worker 51 - jepsen.util 7071 :invoke :write [17 29] INFO [2019-12-17 12:33:34,188] jepsen worker 64 - jepsen.util 5554 :ok :read [21 11] INFO [2019-12-17 12:33:34,188] jepsen worker 64 - jepsen.util 5554 :invoke :write [21 10] INFO [2019-12-17 12:33:34,373] jepsen worker 72 - jepsen.util 7182 :ok :write [24 10] INFO [2019-12-17 12:33:34,373] jepsen worker 32 - jepsen.util 7232 :ok :write [10 5] INFO [2019-12-17 12:33:34,373] jepsen worker 16 - jepsen.util 10996 :info :write [5 9] :RequestCanceledException INFO [2019-12-17 12:33:34,373] jepsen worker 44 - jepsen.util 10484 :info :cas [14 [39 16]] :RequestCanceledException INFO [2019-12-17 12:33:34,373] jepsen worker 61 - jepsen.util 10501 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:33:34,373] jepsen worker 40 - jepsen.util 10210 :info :cas [13 [42 16]] :RequestCanceledException INFO [2019-12-17 12:33:34,373] jepsen worker 45 - jepsen.util 7245 :fail :read [15 nil] :RequestTimeoutException INFO [2019-12-17 12:33:34,373] jepsen worker 49 - jepsen.util 9229 :info :write [16 42] :RequestCanceledException INFO [2019-12-17 12:33:34,373] jepsen worker 72 - jepsen.util 7182 :invoke :write [24 34] INFO [2019-12-17 12:33:34,373] jepsen worker 46 - jepsen.util 7606 :info :cas [15 [0 41]] :RequestCanceledException INFO [2019-12-17 12:33:34,373] jepsen worker 35 - jepsen.util 9755 :info :write [11 34] :RequestCanceledException INFO [2019-12-17 12:33:34,373] jepsen worker 32 - jepsen.util 7232 :invoke :write [10 1] INFO [2019-12-17 12:33:34,374] jepsen worker 61 - jepsen.util 10501 :invoke :write [20 9] INFO [2019-12-17 12:33:34,374] jepsen worker 45 - jepsen.util 7245 :invoke :read [15 nil] INFO [2019-12-17 12:33:34,378] jepsen worker 10 - jepsen.util 7210 :info :write [3 11] :SyncWriteAmbiguous INFO [2019-12-17 12:33:34,378] jepsen worker 79 - jepsen.util 7279 :info :write [26 0] :RequestCanceledException INFO [2019-12-17 12:33:34,378] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42005,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:33:34,378] jepsen worker 7 - jepsen.util 9907 :info :cas [2 [36 3]] :RequestCanceledException INFO [2019-12-17 12:33:34,378] jepsen worker 8 - jepsen.util 9818 :info :cas [2 [41 29]] :RequestCanceledException INFO [2019-12-17 12:33:34,378] jepsen worker 85 - jepsen.util 10795 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:33:34,379] jepsen worker 44 - jepsen.util 10574 :invoke :read [14 nil] INFO [2019-12-17 12:33:34,379] jepsen worker 16 - jepsen.util 11086 :invoke :write [5 32] INFO [2019-12-17 12:33:34,379] jepsen worker 40 - jepsen.util 10300 :invoke :read [13 nil] INFO [2019-12-17 12:33:34,379] jepsen worker 49 - jepsen.util 9319 :invoke :cas [16 [4 4]] INFO [2019-12-17 12:33:34,379] jepsen worker 46 - jepsen.util 7696 :invoke :cas [15 [4 39]] INFO [2019-12-17 12:33:34,381] jepsen worker 35 - jepsen.util 9845 :invoke :read [11 nil] INFO [2019-12-17 12:33:34,381] jepsen worker 10 - jepsen.util 7300 :invoke :read [3 nil] INFO [2019-12-17 12:33:34,382] jepsen worker 79 - jepsen.util 7369 :invoke :write [26 41] INFO [2019-12-17 12:33:34,382] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42005,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:33:34,382] jepsen worker 85 - jepsen.util 10795 :invoke :cas [28 [47 44]] INFO [2019-12-17 12:33:34,383] jepsen worker 73 - jepsen.util 6913 :fail :cas [24 [21 28]] :ValueNotSwapFrom INFO [2019-12-17 12:33:34,383] jepsen worker 7 - jepsen.util 9997 :invoke :cas [2 [18 27]] INFO [2019-12-17 12:33:34,383] jepsen worker 8 - jepsen.util 9908 :invoke :cas [2 [29 38]] INFO [2019-12-17 12:33:34,383] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42012,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:33:34,384] jepsen worker 73 - jepsen.util 6913 :invoke :write [24 25] INFO [2019-12-17 12:33:34,384] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42012,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:33:34,385] jepsen worker 28 - jepsen.util 5608 :ok :write [9 22] INFO [2019-12-17 12:33:34,385] jepsen worker 28 - jepsen.util 5608 :invoke :cas [9 [11 8]] INFO [2019-12-17 12:33:34,387] jepsen worker 29 - jepsen.util 5609 :fail :cas [9 [25 46]] :ValueNotSwapFrom INFO [2019-12-17 12:33:34,388] jepsen worker 29 - jepsen.util 5609 :invoke :cas [9 [11 18]] INFO [2019-12-17 12:33:34,391] jepsen worker 55 - jepsen.util 8245 :ok :read [18 12] INFO [2019-12-17 12:33:34,391] jepsen worker 55 - jepsen.util 8245 :invoke :read [18 nil] INFO [2019-12-17 12:33:34,395] jepsen worker 65 - jepsen.util 6005 :fail :cas [21 [31 8]] :ValueNotSwapFrom INFO [2019-12-17 12:33:34,396] jepsen worker 65 - jepsen.util 6005 :invoke :read [21 nil] INFO [2019-12-17 12:33:34,403] jepsen worker 87 - jepsen.util 6567 :ok :read [29 8] INFO [2019-12-17 12:33:34,403] jepsen worker 87 - jepsen.util 6567 :invoke :cas [29 [12 28]] INFO [2019-12-17 12:33:34,403] jepsen worker 89 - jepsen.util 7109 :fail :cas [29 [45 30]] :ValueNotSwapFrom INFO [2019-12-17 12:33:34,404] jepsen worker 89 - jepsen.util 7109 :invoke :cas [29 [8 28]] INFO [2019-12-17 12:33:34,407] jepsen worker 31 - jepsen.util 6601 :fail :cas [10 [26 41]] :ValueNotSwapFrom INFO [2019-12-17 12:33:34,407] jepsen worker 31 - jepsen.util 6601 :invoke :read [10 nil] INFO [2019-12-17 12:33:34,411] jepsen worker 13 - jepsen.util 6943 :ok :read [4 0] INFO [2019-12-17 12:33:34,411] jepsen worker 13 - jepsen.util 6943 :invoke :write [4 42] INFO [2019-12-17 12:33:34,428] jepsen worker 18 - jepsen.util 7308 :info :write [6 21] :RequestCanceledException INFO [2019-12-17 12:33:34,429] jepsen worker 18 - jepsen.util 7398 :invoke :write [6 2] INFO [2019-12-17 12:33:34,456] jepsen worker 46 - jepsen.util 7696 :fail :cas [15 [4 39]] :ValueNotSwapFrom INFO [2019-12-17 12:33:34,456] jepsen worker 46 - jepsen.util 7696 :invoke :write [15 14] INFO [2019-12-17 12:33:34,466] jepsen worker 63 - jepsen.util 6183 :ok :write [21 3] INFO [2019-12-17 12:33:34,466] jepsen worker 63 - jepsen.util 6183 :invoke :cas [21 [38 32]] INFO [2019-12-17 12:33:34,482] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42013,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:33:34,482] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42013,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:33:34,482] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42015,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:33:34,482] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42015,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:33:34,483] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42043,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:33:34,483] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42043,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:33:34,483] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42046,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:33:34,483] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42046,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:33:34,483] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42047,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:33:34,483] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42047,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:33:34,513] jepsen worker 19 - jepsen.util 7219 :info :cas [6 [6 12]] :RequestCanceledException INFO [2019-12-17 12:33:34,513] jepsen worker 3 - jepsen.util 7293 :info :write [1 6] :RequestCanceledException INFO [2019-12-17 12:33:34,514] jepsen worker 19 - jepsen.util 7309 :invoke :cas [6 [27 10]] INFO [2019-12-17 12:33:34,514] jepsen worker 3 - jepsen.util 7383 :invoke :write [1 14] INFO [2019-12-17 12:33:34,517] jepsen worker 11 - jepsen.util 7481 :ok :read [3 14] INFO [2019-12-17 12:33:34,517] jepsen worker 11 - jepsen.util 7481 :invoke :read [3 nil] INFO [2019-12-17 12:33:34,528] jepsen worker 69 - jepsen.util 7359 :info :write [23 45] :RequestCanceledException INFO [2019-12-17 12:33:34,529] jepsen worker 69 - jepsen.util 7449 :invoke :cas [23 [43 1]] INFO [2019-12-17 12:33:34,535] jepsen worker 47 - jepsen.util 7517 :fail :cas [15 [26 23]] :ValueNotSwapFrom INFO [2019-12-17 12:33:34,535] jepsen worker 47 - jepsen.util 7517 :invoke :read [15 nil] INFO [2019-12-17 12:33:34,550] jepsen worker 45 - jepsen.util 7245 :ok :read [15 3] INFO [2019-12-17 12:33:34,550] jepsen worker 45 - jepsen.util 7245 :invoke :read [15 nil] INFO [2019-12-17 12:33:34,551] jepsen worker 87 - jepsen.util 6567 :fail :cas [29 [12 28]] :ValueNotSwapFrom INFO [2019-12-17 12:33:34,551] jepsen worker 87 - jepsen.util 6567 :invoke :cas [29 [12 0]] INFO [2019-12-17 12:33:34,580] jepsen worker 71 - jepsen.util 8081 :ok :read [23 4] INFO [2019-12-17 12:33:34,580] jepsen worker 71 - jepsen.util 8081 :invoke :read [23 nil] INFO [2019-12-17 12:33:34,610] jepsen worker 30 - jepsen.util 7140 :ok :write [10 15] INFO [2019-12-17 12:33:34,611] jepsen worker 30 - jepsen.util 7140 :invoke :cas [10 [14 9]] INFO [2019-12-17 12:33:34,611] jepsen worker 46 - jepsen.util 7696 :ok :write [15 14] INFO [2019-12-17 12:33:34,611] jepsen worker 9 - jepsen.util 7839 :ok :write [3 21] INFO [2019-12-17 12:33:34,611] jepsen worker 46 - jepsen.util 7696 :invoke :read [15 nil] INFO [2019-12-17 12:33:34,611] jepsen worker 9 - jepsen.util 7839 :invoke :write [3 7] INFO [2019-12-17 12:33:34,624] jepsen worker 20 - jepsen.util 7040 :ok :write [6 4] INFO [2019-12-17 12:33:34,624] jepsen worker 20 - jepsen.util 7040 :invoke :read [6 nil] INFO [2019-12-17 12:33:34,641] jepsen worker 27 - jepsen.util 7227 :info :cas [9 [33 41]] :RequestCanceledException INFO [2019-12-17 12:33:34,642] jepsen worker 70 - jepsen.util 7540 :info :write [23 7] :RequestCanceledException INFO [2019-12-17 12:33:34,643] jepsen worker 27 - jepsen.util 7317 :invoke :cas [9 [32 10]] INFO [2019-12-17 12:33:34,643] jepsen worker 70 - jepsen.util 7630 :invoke :write [23 19] INFO [2019-12-17 12:33:34,662] jepsen worker 20 - jepsen.util 7040 :ok :read [6 4] INFO [2019-12-17 12:33:34,662] jepsen worker 20 - jepsen.util 7040 :invoke :write [6 14] INFO [2019-12-17 12:33:34,668] jepsen worker 46 - jepsen.util 7696 :ok :read [15 14] INFO [2019-12-17 12:33:34,668] jepsen worker 46 - jepsen.util 7696 :invoke :read [15 nil] INFO [2019-12-17 12:33:34,693] jepsen worker 10 - jepsen.util 7300 :ok :read [3 21] INFO [2019-12-17 12:33:34,694] jepsen worker 10 - jepsen.util 7300 :invoke :cas [3 [15 9]] INFO [2019-12-17 12:33:34,702] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42064,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:33:34,702] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42064,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:33:34,702] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42068,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:33:34,702] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42068,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:33:34,722] jepsen worker 27 - jepsen.util 7317 :fail :cas [9 [32 10]] :ValueNotSwapFrom INFO [2019-12-17 12:33:34,722] jepsen worker 27 - jepsen.util 7317 :invoke :read [9 nil] INFO [2019-12-17 12:33:34,728] jepsen worker 2 - jepsen.util 9452 :info :cas [0 [35 5]] :RequestCanceledException INFO [2019-12-17 12:33:34,729] jepsen worker 2 - jepsen.util 9542 :invoke :write [0 9] INFO [2019-12-17 12:33:34,744] jepsen worker 71 - jepsen.util 8081 :ok :read [23 4] INFO [2019-12-17 12:33:34,744] jepsen worker 71 - jepsen.util 8081 :invoke :write [23 3] INFO [2019-12-17 12:33:34,749] jepsen worker 46 - jepsen.util 7696 :ok :read [15 14] INFO [2019-12-17 12:33:34,749] jepsen worker 46 - jepsen.util 7696 :invoke :write [15 19] INFO [2019-12-17 12:33:34,751] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:33:34,751] jepsen nemesis - couchbase.util Rebalance check count: 322 INFO [2019-12-17 12:33:34,751] jepsen nemesis - couchbase.util Rebalance stuck count: 303 INFO [2019-12-17 12:33:34,753] jepsen worker 52 - jepsen.util 7612 :fail :cas [17 [33 2]] :ValueNotSwapFrom INFO [2019-12-17 12:33:34,753] jepsen worker 52 - jepsen.util 7612 :invoke :cas [17 [18 48]] INFO [2019-12-17 12:33:34,767] jepsen worker 37 - jepsen.util 6877 :ok :read [12 39] INFO [2019-12-17 12:33:34,767] jepsen worker 37 - jepsen.util 6877 :invoke :read [12 nil] INFO [2019-12-17 12:33:34,782] jepsen worker 28 - jepsen.util 5608 :fail :cas [9 [11 8]] :ValueNotSwapFrom INFO [2019-12-17 12:33:34,782] jepsen worker 28 - jepsen.util 5608 :invoke :write [9 7] INFO [2019-12-17 12:33:34,810] jepsen worker 27 - jepsen.util 7317 :ok :read [9 22] INFO [2019-12-17 12:33:34,810] jepsen worker 27 - jepsen.util 7317 :invoke :write [9 11] INFO [2019-12-17 12:33:34,824] jepsen worker 65 - jepsen.util 6005 :ok :read [21 3] INFO [2019-12-17 12:33:34,824] jepsen worker 65 - jepsen.util 6005 :invoke :write [21 33] INFO [2019-12-17 12:33:34,829] jepsen worker 54 - jepsen.util 8154 :fail :cas [18 [41 45]] :ValueNotSwapFrom INFO [2019-12-17 12:33:34,829] jepsen worker 54 - jepsen.util 8154 :invoke :read [18 nil] INFO [2019-12-17 12:33:34,841] jepsen worker 74 - jepsen.util 7454 :fail :read [24 nil] :RequestTimeoutException INFO [2019-12-17 12:33:34,842] jepsen worker 74 - jepsen.util 7454 :invoke :write [24 7] INFO [2019-12-17 12:33:34,848] jepsen worker 88 - jepsen.util 7288 :ok :write [29 43] INFO [2019-12-17 12:33:34,848] jepsen worker 88 - jepsen.util 7288 :invoke :read [29 nil] INFO [2019-12-17 12:33:34,906] jepsen worker 27 - jepsen.util 7317 :ok :write [9 11] INFO [2019-12-17 12:33:34,906] jepsen worker 27 - jepsen.util 7317 :invoke :read [9 nil] INFO [2019-12-17 12:33:34,922] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42088,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:33:34,922] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42088,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:33:34,922] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42092,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:33:34,922] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42092,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:33:34,928] jepsen worker 36 - jepsen.util 7596 :info :write [12 1] :RequestCanceledException INFO [2019-12-17 12:33:34,931] jepsen worker 36 - jepsen.util 7686 :invoke :cas [12 [41 10]] INFO [2019-12-17 12:33:34,934] jepsen worker 27 - jepsen.util 7317 :ok :read [9 11] INFO [2019-12-17 12:33:34,934] jepsen worker 27 - jepsen.util 7317 :invoke :cas [9 [9 15]] INFO [2019-12-17 12:33:34,959] jepsen worker 56 - jepsen.util 8066 :info :write [18 42] :RequestCanceledException INFO [2019-12-17 12:33:34,960] jepsen worker 56 - jepsen.util 8156 :invoke :cas [18 [20 31]] INFO [2019-12-17 12:33:34,971] jepsen worker 27 - jepsen.util 7317 :fail :cas [9 [9 15]] :ValueNotSwapFrom INFO [2019-12-17 12:33:34,971] jepsen worker 27 - jepsen.util 7317 :invoke :read [9 nil] INFO [2019-12-17 12:33:34,996] jepsen worker 27 - jepsen.util 7317 :ok :read [9 11] INFO [2019-12-17 12:33:34,996] jepsen worker 27 - jepsen.util 7317 :invoke :read [9 nil] INFO [2019-12-17 12:33:34,999] jepsen worker 72 - jepsen.util 7182 :ok :write [24 34] INFO [2019-12-17 12:33:34,999] jepsen worker 72 - jepsen.util 7182 :invoke :write [24 29] INFO [2019-12-17 12:33:35,026] jepsen worker 9 - jepsen.util 7839 :ok :write [3 7] INFO [2019-12-17 12:33:35,026] jepsen worker 9 - jepsen.util 7839 :invoke :read [3 nil] INFO [2019-12-17 12:33:35,031] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42099,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:33:35,031] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42099,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:33:35,031] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42102,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:33:35,031] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42102,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:33:35,031] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42103,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:33:35,031] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42103,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:33:35,048] jepsen worker 79 - jepsen.util 7369 :ok :write [26 41] INFO [2019-12-17 12:33:35,048] jepsen worker 79 - jepsen.util 7369 :invoke :read [26 nil] INFO [2019-12-17 12:33:35,048] jepsen worker 27 - jepsen.util 7317 :ok :read [9 11] INFO [2019-12-17 12:33:35,048] jepsen worker 27 - jepsen.util 7317 :invoke :write [9 27] INFO [2019-12-17 12:33:35,059] jepsen worker 12 - jepsen.util 7752 :info :write [4 15] :RequestCanceledException INFO [2019-12-17 12:33:35,060] jepsen worker 5 - jepsen.util 6755 :info :write [1 29] :RequestCanceledException INFO [2019-12-17 12:33:35,061] jepsen worker 12 - jepsen.util 7842 :invoke :read [4 nil] INFO [2019-12-17 12:33:35,061] jepsen worker 5 - jepsen.util 6845 :invoke :read [1 nil] INFO [2019-12-17 12:33:35,068] jepsen worker 10 - jepsen.util 7300 :fail :cas [3 [15 9]] :ValueNotSwapFrom INFO [2019-12-17 12:33:35,069] jepsen worker 10 - jepsen.util 7300 :invoke :cas [3 [5 5]] INFO [2019-12-17 12:33:35,102] jepsen worker 89 - jepsen.util 7109 :fail :cas [29 [8 28]] :CASMismatchException INFO [2019-12-17 12:33:35,102] jepsen worker 89 - jepsen.util 7109 :invoke :cas [29 [13 35]] INFO [2019-12-17 12:33:35,106] jepsen worker 55 - jepsen.util 8245 :ok :read [18 12] INFO [2019-12-17 12:33:35,106] jepsen worker 55 - jepsen.util 8245 :invoke :cas [18 [34 9]] INFO [2019-12-17 12:33:35,113] jepsen worker 53 - jepsen.util 6713 :info :write [17 42] :RequestCanceledException INFO [2019-12-17 12:33:35,113] jepsen worker 80 - jepsen.util 7280 :info :write [26 39] :RequestCanceledException INFO [2019-12-17 12:33:35,114] jepsen worker 53 - jepsen.util 6803 :invoke :write [17 15] INFO [2019-12-17 12:33:35,116] jepsen worker 80 - jepsen.util 7370 :invoke :write [26 32] INFO [2019-12-17 12:33:35,120] jepsen worker 20 - jepsen.util 7040 :ok :write [6 14] INFO [2019-12-17 12:33:35,120] jepsen worker 20 - jepsen.util 7040 :invoke :read [6 nil] INFO [2019-12-17 12:33:35,142] jepsen worker 20 - jepsen.util 7040 :ok :read [6 14] INFO [2019-12-17 12:33:35,142] jepsen worker 20 - jepsen.util 7040 :invoke :read [6 nil] INFO [2019-12-17 12:33:35,152] jepsen worker 27 - jepsen.util 7317 :ok :write [9 27] INFO [2019-12-17 12:33:35,152] jepsen worker 27 - jepsen.util 7317 :invoke :cas [9 [1 40]] INFO [2019-12-17 12:33:35,157] jepsen worker 20 - jepsen.util 7040 :ok :read [6 14] INFO [2019-12-17 12:33:35,158] jepsen worker 20 - jepsen.util 7040 :invoke :write [6 6] INFO [2019-12-17 12:33:35,168] jepsen worker 27 - jepsen.util 7317 :fail :cas [9 [1 40]] :ValueNotSwapFrom INFO [2019-12-17 12:33:35,168] jepsen worker 27 - jepsen.util 7317 :invoke :cas [9 [18 44]] INFO [2019-12-17 12:33:35,221] jepsen worker 27 - jepsen.util 7317 :fail :cas [9 [18 44]] :ValueNotSwapFrom INFO [2019-12-17 12:33:35,221] jepsen worker 27 - jepsen.util 7317 :invoke :read [9 nil] INFO [2019-12-17 12:33:35,245] jepsen worker 27 - jepsen.util 7317 :ok :read [9 27] INFO [2019-12-17 12:33:35,245] jepsen worker 27 - jepsen.util 7317 :invoke :write [9 15] INFO [2019-12-17 12:33:35,246] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42122,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:33:35,246] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42122,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:33:35,247] jepsen worker 36 - jepsen.util 7686 :fail :cas [12 [41 10]] :ValueNotSwapFrom INFO [2019-12-17 12:33:35,248] jepsen worker 36 - jepsen.util 7686 :invoke :cas [12 [23 3]] INFO [2019-12-17 12:33:35,252] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42129,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:33:35,252] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42129,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:33:35,252] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42132,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:33:35,252] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42132,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:33:35,260] jepsen worker 52 - jepsen.util 7612 :fail :cas [17 [18 48]] :ValueNotSwapFrom INFO [2019-12-17 12:33:35,260] jepsen worker 52 - jepsen.util 7612 :invoke :cas [17 [41 17]] INFO [2019-12-17 12:33:35,285] jepsen worker 46 - jepsen.util 7696 :ok :write [15 19] INFO [2019-12-17 12:33:35,286] jepsen worker 46 - jepsen.util 7696 :invoke :cas [15 [31 14]] INFO [2019-12-17 12:33:35,288] jepsen worker 13 - jepsen.util 6943 :ok :write [4 42] INFO [2019-12-17 12:33:35,288] jepsen worker 13 - jepsen.util 6943 :invoke :read [4 nil] INFO [2019-12-17 12:33:35,315] jepsen worker 69 - jepsen.util 7449 :fail :cas [23 [43 1]] :ValueNotSwapFrom INFO [2019-12-17 12:33:35,316] jepsen worker 69 - jepsen.util 7449 :invoke :write [23 9] INFO [2019-12-17 12:33:35,332] jepsen worker 46 - jepsen.util 7696 :fail :cas [15 [31 14]] :ValueNotSwapFrom INFO [2019-12-17 12:33:35,332] jepsen worker 46 - jepsen.util 7696 :invoke :write [15 14] INFO [2019-12-17 12:33:35,339] jepsen worker 72 - jepsen.util 7182 :ok :write [24 29] INFO [2019-12-17 12:33:35,339] jepsen worker 72 - jepsen.util 7182 :invoke :write [24 38] INFO [2019-12-17 12:33:35,352] jepsen worker 20 - jepsen.util 7040 :ok :write [6 6] INFO [2019-12-17 12:33:35,353] jepsen worker 20 - jepsen.util 7040 :invoke :cas [6 [43 1]] INFO [2019-12-17 12:33:35,369] jepsen worker 27 - jepsen.util 7317 :ok :write [9 15] INFO [2019-12-17 12:33:35,369] jepsen worker 27 - jepsen.util 7317 :invoke :write [9 19] INFO [2019-12-17 12:33:35,373] jepsen worker 20 - jepsen.util 7040 :fail :cas [6 [43 1]] :ValueNotSwapFrom INFO [2019-12-17 12:33:35,373] jepsen worker 20 - jepsen.util 7040 :invoke :read [6 nil] INFO [2019-12-17 12:33:35,403] jepsen worker 37 - jepsen.util 6877 :ok :read [12 39] INFO [2019-12-17 12:33:35,403] jepsen worker 37 - jepsen.util 6877 :invoke :write [12 12] INFO [2019-12-17 12:33:35,411] jepsen worker 80 - jepsen.util 7370 :ok :write [26 32] INFO [2019-12-17 12:33:35,411] jepsen worker 80 - jepsen.util 7370 :invoke :cas [26 [16 5]] INFO [2019-12-17 12:33:35,420] jepsen worker 20 - jepsen.util 7040 :ok :read [6 6] INFO [2019-12-17 12:33:35,420] jepsen worker 20 - jepsen.util 7040 :invoke :read [6 nil] INFO [2019-12-17 12:33:35,438] jepsen worker 20 - jepsen.util 7040 :ok :read [6 6] INFO [2019-12-17 12:33:35,438] jepsen worker 20 - jepsen.util 7040 :invoke :cas [6 [47 41]] INFO [2019-12-17 12:33:35,440] jepsen worker 27 - jepsen.util 7317 :ok :write [9 19] INFO [2019-12-17 12:33:35,440] jepsen worker 27 - jepsen.util 7317 :invoke :write [9 32] INFO [2019-12-17 12:33:35,468] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42149,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:33:35,468] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42149,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:33:35,468] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42152,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:33:35,468] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42152,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:33:35,474] jepsen worker 20 - jepsen.util 7040 :fail :cas [6 [47 41]] :ValueNotSwapFrom INFO [2019-12-17 12:33:35,474] jepsen worker 20 - jepsen.util 7040 :invoke :cas [6 [28 19]] INFO [2019-12-17 12:33:35,498] jepsen worker 20 - jepsen.util 7040 :fail :cas [6 [28 19]] :ValueNotSwapFrom INFO [2019-12-17 12:33:35,498] jepsen worker 20 - jepsen.util 7040 :invoke :read [6 nil] INFO [2019-12-17 12:33:35,515] jepsen worker 79 - jepsen.util 7369 :ok :read [26 32] INFO [2019-12-17 12:33:35,516] jepsen worker 79 - jepsen.util 7369 :invoke :write [26 47] INFO [2019-12-17 12:33:35,532] jepsen worker 12 - jepsen.util 7842 :ok :read [4 42] INFO [2019-12-17 12:33:35,532] jepsen worker 12 - jepsen.util 7842 :invoke :read [4 nil] INFO [2019-12-17 12:33:35,543] jepsen worker 20 - jepsen.util 7040 :ok :read [6 6] INFO [2019-12-17 12:33:35,544] jepsen worker 20 - jepsen.util 7040 :invoke :cas [6 [12 46]] INFO [2019-12-17 12:33:35,559] jepsen worker 34 - jepsen.util 9304 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:33:35,559] jepsen worker 34 - jepsen.util 9304 :invoke :cas [11 [48 5]] INFO [2019-12-17 12:33:35,562] jepsen worker 27 - jepsen.util 7317 :ok :write [9 32] INFO [2019-12-17 12:33:35,562] jepsen worker 27 - jepsen.util 7317 :invoke :read [9 nil] INFO [2019-12-17 12:33:35,570] jepsen worker 55 - jepsen.util 8245 :fail :cas [18 [34 9]] :ValueNotSwapFrom INFO [2019-12-17 12:33:35,570] jepsen worker 55 - jepsen.util 8245 :invoke :write [18 38] INFO [2019-12-17 12:33:35,580] jepsen worker 20 - jepsen.util 7040 :fail :cas [6 [12 46]] :ValueNotSwapFrom INFO [2019-12-17 12:33:35,580] jepsen worker 20 - jepsen.util 7040 :invoke :cas [6 [22 22]] INFO [2019-12-17 12:33:35,585] jepsen worker 89 - jepsen.util 7109 :fail :cas [29 [13 35]] :ValueNotSwapFrom INFO [2019-12-17 12:33:35,585] jepsen worker 89 - jepsen.util 7109 :invoke :read [29 nil] INFO [2019-12-17 12:33:35,594] jepsen worker 4 - jepsen.util 6754 :ok :read [1 44] INFO [2019-12-17 12:33:35,594] jepsen worker 4 - jepsen.util 6754 :invoke :cas [1 [7 9]] INFO [2019-12-17 12:33:35,595] jepsen worker 42 - jepsen.util 10572 :info :cas [14 [0 33]] :RequestCanceledException INFO [2019-12-17 12:33:35,595] jepsen worker 42 - jepsen.util 10662 :invoke :read [14 nil] INFO [2019-12-17 12:33:35,605] jepsen worker 3 - jepsen.util 7383 :ok :write [1 14] INFO [2019-12-17 12:33:35,605] jepsen worker 3 - jepsen.util 7383 :invoke :write [1 47] INFO [2019-12-17 12:33:35,621] jepsen worker 78 - jepsen.util 7458 :fail :cas [26 [31 30]] :CASMismatchException INFO [2019-12-17 12:33:35,621] jepsen worker 78 - jepsen.util 7458 :invoke :cas [26 [0 37]] INFO [2019-12-17 12:33:35,626] jepsen worker 75 - jepsen.util 9345 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:33:35,626] jepsen worker 83 - jepsen.util 10343 :info :cas [27 [4 13]] :RequestCanceledException INFO [2019-12-17 12:33:35,626] jepsen worker 21 - jepsen.util 9381 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:33:35,627] jepsen worker 75 - jepsen.util 9345 :invoke :cas [25 [24 19]] INFO [2019-12-17 12:33:35,627] jepsen worker 21 - jepsen.util 9381 :invoke :cas [7 [12 11]] INFO [2019-12-17 12:33:35,628] jepsen worker 83 - jepsen.util 10433 :invoke :read [27 nil] INFO [2019-12-17 12:33:35,632] jepsen worker 27 - jepsen.util 7317 :ok :read [9 32] INFO [2019-12-17 12:33:35,632] jepsen worker 27 - jepsen.util 7317 :invoke :read [9 nil] INFO [2019-12-17 12:33:35,642] jepsen worker 66 - jepsen.util 9786 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:33:35,642] jepsen worker 43 - jepsen.util 9313 :info :write [14 36] :RequestCanceledException INFO [2019-12-17 12:33:35,642] jepsen worker 66 - jepsen.util 9786 :invoke :cas [22 [1 18]] INFO [2019-12-17 12:33:35,643] jepsen worker 43 - jepsen.util 9403 :invoke :cas [14 [27 8]] INFO [2019-12-17 12:33:35,657] jepsen worker 36 - jepsen.util 7686 :fail :cas [12 [23 3]] :ValueNotSwapFrom INFO [2019-12-17 12:33:35,658] jepsen worker 36 - jepsen.util 7686 :invoke :cas [12 [6 14]] INFO [2019-12-17 12:33:35,665] jepsen worker 14 - jepsen.util 7574 :ok :write [4 17] INFO [2019-12-17 12:33:35,665] jepsen worker 14 - jepsen.util 7574 :invoke :read [4 nil] INFO [2019-12-17 12:33:35,675] jepsen worker 20 - jepsen.util 7040 :fail :cas [6 [22 22]] :ValueNotSwapFrom INFO [2019-12-17 12:33:35,675] jepsen worker 20 - jepsen.util 7040 :invoke :read [6 nil] INFO [2019-12-17 12:33:35,682] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42179,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:33:35,682] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42179,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:33:35,683] jepsen worker 27 - jepsen.util 7317 :ok :read [9 32] INFO [2019-12-17 12:33:35,683] jepsen worker 27 - jepsen.util 7317 :invoke :write [9 3] INFO [2019-12-17 12:33:35,708] jepsen worker 20 - jepsen.util 7040 :ok :read [6 6] INFO [2019-12-17 12:33:35,709] jepsen worker 20 - jepsen.util 7040 :invoke :write [6 24] INFO [2019-12-17 12:33:35,711] jepsen worker 4 - jepsen.util 6754 :fail :cas [1 [7 9]] :ValueNotSwapFrom INFO [2019-12-17 12:33:35,711] jepsen worker 4 - jepsen.util 6754 :invoke :cas [1 [9 29]] INFO [2019-12-17 12:33:35,728] jepsen worker 23 - jepsen.util 10013 :info :write [7 25] :RequestCanceledException INFO [2019-12-17 12:33:35,728] jepsen worker 86 - jepsen.util 10436 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:33:35,728] jepsen worker 26 - jepsen.util 10106 :info :cas [8 [33 14]] :RequestCanceledException INFO [2019-12-17 12:33:35,728] jepsen worker 62 - jepsen.util 9872 :info :write [20 29] :RequestCanceledException INFO [2019-12-17 12:33:35,728] jepsen worker 86 - jepsen.util 10436 :invoke :cas [28 [12 30]] INFO [2019-12-17 12:33:35,729] jepsen worker 23 - jepsen.util 10103 :invoke :read [7 nil] INFO [2019-12-17 12:33:35,729] jepsen worker 26 - jepsen.util 10196 :invoke :cas [8 [47 45]] INFO [2019-12-17 12:33:35,729] jepsen worker 62 - jepsen.util 9962 :invoke :write [20 26] INFO [2019-12-17 12:33:35,758] jepsen worker 87 - jepsen.util 6567 :fail :cas [29 [12 0]] :ValueNotSwapFrom INFO [2019-12-17 12:33:35,759] jepsen worker 87 - jepsen.util 6567 :invoke :write [29 32] INFO [2019-12-17 12:33:35,759] jepsen worker 22 - jepsen.util 9652 :info :cas [7 [20 23]] :RequestCanceledException INFO [2019-12-17 12:33:35,759] jepsen worker 59 - jepsen.util 9779 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:33:35,759] jepsen worker 60 - jepsen.util 10680 :info :cas [20 [43 32]] :RequestCanceledException INFO [2019-12-17 12:33:35,759] jepsen worker 0 - jepsen.util 10350 :info :write [0 27] :RequestCanceledException INFO [2019-12-17 12:33:35,759] jepsen worker 59 - jepsen.util 9779 :invoke :read [19 nil] INFO [2019-12-17 12:33:35,760] jepsen worker 22 - jepsen.util 9742 :invoke :cas [7 [1 2]] INFO [2019-12-17 12:33:35,760] jepsen worker 60 - jepsen.util 10770 :invoke :write [20 43] INFO [2019-12-17 12:33:35,760] jepsen worker 0 - jepsen.util 10440 :invoke :read [0 nil] INFO [2019-12-17 12:33:35,763] jepsen worker 27 - jepsen.util 7317 :ok :write [9 3] INFO [2019-12-17 12:33:35,763] jepsen worker 27 - jepsen.util 7317 :invoke :cas [9 [44 47]] INFO [2019-12-17 12:33:35,765] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:33:35,766] jepsen nemesis - couchbase.util Rebalance check count: 323 INFO [2019-12-17 12:33:35,766] jepsen nemesis - couchbase.util Rebalance stuck count: 304 INFO [2019-12-17 12:33:35,780] jepsen worker 27 - jepsen.util 7317 :fail :cas [9 [44 47]] :ValueNotSwapFrom INFO [2019-12-17 12:33:35,780] jepsen worker 27 - jepsen.util 7317 :invoke :read [9 nil] INFO [2019-12-17 12:33:35,789] jepsen worker 5 - jepsen.util 6845 :ok :read [1 14] INFO [2019-12-17 12:33:35,790] jepsen worker 5 - jepsen.util 6845 :invoke :write [1 39] INFO [2019-12-17 12:33:35,794] jepsen worker 89 - jepsen.util 7109 :ok :read [29 43] INFO [2019-12-17 12:33:35,794] jepsen worker 89 - jepsen.util 7109 :invoke :write [29 14] INFO [2019-12-17 12:33:35,795] jepsen worker 38 - jepsen.util 7418 :info :write [12 10] :RequestCanceledException INFO [2019-12-17 12:33:35,795] jepsen worker 41 - jepsen.util 10391 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:33:35,795] jepsen worker 41 - jepsen.util 10391 :invoke :cas [13 [33 4]] INFO [2019-12-17 12:33:35,796] jepsen worker 38 - jepsen.util 7508 :invoke :read [12 nil] INFO [2019-12-17 12:33:35,803] jepsen worker 27 - jepsen.util 7317 :ok :read [9 3] INFO [2019-12-17 12:33:35,803] jepsen worker 27 - jepsen.util 7317 :invoke :read [9 nil] INFO [2019-12-17 12:33:35,813] jepsen worker 67 - jepsen.util 9787 :info :write [22 9] :RequestCanceledException INFO [2019-12-17 12:33:35,813] jepsen worker 58 - jepsen.util 10588 :info :write [19 23] :RequestCanceledException INFO [2019-12-17 12:33:35,813] jepsen worker 77 - jepsen.util 10067 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:33:35,813] jepsen worker 77 - jepsen.util 10067 :invoke :cas [25 [27 22]] INFO [2019-12-17 12:33:35,814] jepsen worker 67 - jepsen.util 9877 :invoke :cas [22 [13 49]] INFO [2019-12-17 12:33:35,814] jepsen worker 58 - jepsen.util 10678 :invoke :cas [19 [3 3]] INFO [2019-12-17 12:33:35,820] jepsen worker 27 - jepsen.util 7317 :ok :read [9 3] INFO [2019-12-17 12:33:35,821] jepsen worker 27 - jepsen.util 7317 :invoke :read [9 nil] INFO [2019-12-17 12:33:35,821] jepsen worker 78 - jepsen.util 7458 :fail :cas [26 [0 37]] :ValueNotSwapFrom INFO [2019-12-17 12:33:35,821] jepsen worker 78 - jepsen.util 7458 :invoke :cas [26 [33 18]] INFO [2019-12-17 12:33:35,827] jepsen worker 68 - jepsen.util 9248 :info :write [22 17] :RequestCanceledException INFO [2019-12-17 12:33:35,827] jepsen worker 82 - jepsen.util 10612 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:33:35,827] jepsen worker 82 - jepsen.util 10612 :invoke :read [27 nil] INFO [2019-12-17 12:33:35,828] jepsen worker 68 - jepsen.util 9338 :invoke :cas [22 [27 39]] INFO [2019-12-17 12:33:35,835] jepsen worker 54 - jepsen.util 8154 :ok :read [18 12] INFO [2019-12-17 12:33:35,835] jepsen worker 54 - jepsen.util 8154 :invoke :write [18 25] INFO [2019-12-17 12:33:35,840] jepsen worker 27 - jepsen.util 7317 :ok :read [9 3] INFO [2019-12-17 12:33:35,840] jepsen worker 27 - jepsen.util 7317 :invoke :read [9 nil] INFO [2019-12-17 12:33:35,841] jepsen worker 25 - jepsen.util 10375 :info :write [8 26] :RequestCanceledException INFO [2019-12-17 12:33:35,842] jepsen worker 25 - jepsen.util 10465 :invoke :write [8 48] INFO [2019-12-17 12:33:35,844] jepsen worker 80 - jepsen.util 7370 :fail :cas [26 [16 5]] :ValueNotSwapFrom INFO [2019-12-17 12:33:35,845] jepsen worker 80 - jepsen.util 7370 :invoke :read [26 nil] INFO [2019-12-17 12:33:35,854] jepsen worker 36 - jepsen.util 7686 :fail :cas [12 [6 14]] :ValueNotSwapFrom INFO [2019-12-17 12:33:35,854] jepsen worker 36 - jepsen.util 7686 :invoke :write [12 10] INFO [2019-12-17 12:33:35,857] jepsen worker 27 - jepsen.util 7317 :ok :read [9 3] INFO [2019-12-17 12:33:35,857] jepsen worker 27 - jepsen.util 7317 :invoke :write [9 20] INFO [2019-12-17 12:33:35,888] jepsen worker 14 - jepsen.util 7574 :ok :read [4 17] INFO [2019-12-17 12:33:35,888] jepsen worker 14 - jepsen.util 7574 :invoke :cas [4 [11 4]] INFO [2019-12-17 12:33:35,891] jepsen worker 88 - jepsen.util 7288 :ok :read [29 43] INFO [2019-12-17 12:33:35,891] jepsen worker 88 - jepsen.util 7288 :invoke :read [29 nil] INFO [2019-12-17 12:33:35,894] jepsen worker 33 - jepsen.util 9573 :info :write [11 1] :RequestCanceledException INFO [2019-12-17 12:33:35,895] jepsen worker 72 - jepsen.util 7182 :ok :write [24 38] INFO [2019-12-17 12:33:35,895] jepsen worker 72 - jepsen.util 7182 :invoke :write [24 35] INFO [2019-12-17 12:33:35,896] jepsen worker 33 - jepsen.util 9663 :invoke :cas [11 [34 3]] INFO [2019-12-17 12:33:35,899] jepsen worker 46 - jepsen.util 7696 :ok :write [15 14] INFO [2019-12-17 12:33:35,899] jepsen worker 46 - jepsen.util 7696 :invoke :cas [15 [18 23]] INFO [2019-12-17 12:33:35,904] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42217,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:33:35,904] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42217,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:33:35,904] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42220,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:33:35,904] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42220,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:33:35,904] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42222,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:33:35,904] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42222,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:33:35,911] jepsen worker 69 - jepsen.util 7449 :ok :write [23 9] INFO [2019-12-17 12:33:35,912] jepsen worker 69 - jepsen.util 7449 :invoke :read [23 nil] INFO [2019-12-17 12:33:35,912] jepsen worker 24 - jepsen.util 8574 :info :write [8 5] :RequestCanceledException INFO [2019-12-17 12:33:35,913] jepsen worker 24 - jepsen.util 8664 :invoke :read [8 nil] INFO [2019-12-17 12:33:35,918] jepsen worker 56 - jepsen.util 8156 :fail :cas [18 [20 31]] :ValueNotSwapFrom INFO [2019-12-17 12:33:35,918] jepsen worker 56 - jepsen.util 8156 :invoke :cas [18 [49 10]] INFO [2019-12-17 12:33:35,928] jepsen worker 15 - jepsen.util 8745 :info :cas [5 [48 2]] :RequestCanceledException INFO [2019-12-17 12:33:35,930] jepsen worker 15 - jepsen.util 8835 :invoke :write [5 44] INFO [2019-12-17 12:33:35,942] jepsen worker 1 - jepsen.util 9901 :info :write [0 3] :RequestCanceledException INFO [2019-12-17 12:33:35,943] jepsen worker 1 - jepsen.util 9991 :invoke :read [0 nil] INFO [2019-12-17 12:33:35,954] jepsen worker 46 - jepsen.util 7696 :fail :cas [15 [18 23]] :ValueNotSwapFrom INFO [2019-12-17 12:33:35,954] jepsen worker 46 - jepsen.util 7696 :invoke :cas [15 [33 47]] INFO [2019-12-17 12:33:35,955] jepsen worker 20 - jepsen.util 7040 :ok :write [6 24] INFO [2019-12-17 12:33:35,955] jepsen worker 20 - jepsen.util 7040 :invoke :cas [6 [21 41]] INFO [2019-12-17 12:33:35,959] jepsen worker 50 - jepsen.util 10490 :info :write [16 5] :RequestCanceledException INFO [2019-12-17 12:33:35,961] jepsen worker 50 - jepsen.util 10580 :invoke :cas [16 [37 39]] INFO [2019-12-17 12:33:35,994] jepsen worker 46 - jepsen.util 7696 :fail :cas [15 [33 47]] :ValueNotSwapFrom INFO [2019-12-17 12:33:35,994] jepsen worker 46 - jepsen.util 7696 :invoke :read [15 nil] INFO [2019-12-17 12:33:36,011] jepsen worker 51 - jepsen.util 7071 :ok :write [17 29] INFO [2019-12-17 12:33:36,011] jepsen worker 51 - jepsen.util 7071 :invoke :cas [17 [20 9]] INFO [2019-12-17 12:33:36,011] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42238,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:33:36,011] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42238,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:33:36,021] jepsen worker 46 - jepsen.util 7696 :ok :read [15 14] INFO [2019-12-17 12:33:36,021] jepsen worker 46 - jepsen.util 7696 :invoke :cas [15 [5 26]] INFO [2019-12-17 12:33:36,026] jepsen worker 4 - jepsen.util 6754 :fail :cas [1 [9 29]] :ValueNotSwapFrom INFO [2019-12-17 12:33:36,026] jepsen worker 4 - jepsen.util 6754 :invoke :cas [1 [17 45]] INFO [2019-12-17 12:33:36,030] jepsen worker 79 - jepsen.util 7369 :ok :write [26 47] INFO [2019-12-17 12:33:36,031] jepsen worker 79 - jepsen.util 7369 :invoke :cas [26 [45 30]] INFO [2019-12-17 12:33:36,031] jepsen worker 20 - jepsen.util 7040 :fail :cas [6 [21 41]] :ValueNotSwapFrom INFO [2019-12-17 12:33:36,031] jepsen worker 20 - jepsen.util 7040 :invoke :write [6 18] INFO [2019-12-17 12:33:36,045] jepsen worker 27 - jepsen.util 7317 :ok :write [9 20] INFO [2019-12-17 12:33:36,045] jepsen worker 27 - jepsen.util 7317 :invoke :write [9 4] INFO [2019-12-17 12:33:36,045] jepsen worker 46 - jepsen.util 7696 :fail :cas [15 [5 26]] :ValueNotSwapFrom INFO [2019-12-17 12:33:36,045] jepsen worker 46 - jepsen.util 7696 :invoke :cas [15 [33 28]] INFO [2019-12-17 12:33:36,063] jepsen worker 13 - jepsen.util 6943 :ok :read [4 17] INFO [2019-12-17 12:33:36,064] jepsen worker 13 - jepsen.util 6943 :invoke :cas [4 [18 33]] INFO [2019-12-17 12:33:36,085] jepsen worker 46 - jepsen.util 7696 :fail :cas [15 [33 28]] :ValueNotSwapFrom INFO [2019-12-17 12:33:36,085] jepsen worker 46 - jepsen.util 7696 :invoke :cas [15 [37 1]] INFO [2019-12-17 12:33:36,095] jepsen worker 6 - jepsen.util 10536 :info :cas [2 [18 37]] :RequestCanceledException INFO [2019-12-17 12:33:36,096] jepsen worker 6 - jepsen.util 10626 :invoke :read [2 nil] INFO [2019-12-17 12:33:36,109] jepsen worker 80 - jepsen.util 7370 :ok :read [26 47] INFO [2019-12-17 12:33:36,109] jepsen worker 80 - jepsen.util 7370 :invoke :write [26 15] INFO [2019-12-17 12:33:36,127] jepsen worker 46 - jepsen.util 7696 :fail :cas [15 [37 1]] :ValueNotSwapFrom INFO [2019-12-17 12:33:36,128] jepsen worker 46 - jepsen.util 7696 :invoke :read [15 nil] INFO [2019-12-17 12:33:36,128] jepsen worker 76 - jepsen.util 9976 :info :cas [25 [2 36]] :RequestCanceledException INFO [2019-12-17 12:33:36,129] jepsen worker 76 - jepsen.util 10066 :invoke :read [25 nil] INFO [2019-12-17 12:33:36,142] jepsen worker 81 - jepsen.util 9711 :info :cas [27 [8 9]] :RequestCanceledException INFO [2019-12-17 12:33:36,143] jepsen worker 81 - jepsen.util 9801 :invoke :write [27 10] INFO [2019-12-17 12:33:36,148] jepsen worker 27 - jepsen.util 7317 :ok :write [9 4] INFO [2019-12-17 12:33:36,148] jepsen worker 27 - jepsen.util 7317 :invoke :write [9 18] INFO [2019-12-17 12:33:36,155] jepsen worker 46 - jepsen.util 7696 :ok :read [15 14] INFO [2019-12-17 12:33:36,156] jepsen worker 46 - jepsen.util 7696 :invoke :read [15 nil] INFO [2019-12-17 12:33:36,160] jepsen worker 78 - jepsen.util 7458 :fail :cas [26 [33 18]] :ValueNotSwapFrom INFO [2019-12-17 12:33:36,160] jepsen worker 78 - jepsen.util 7458 :invoke :cas [26 [42 20]] INFO [2019-12-17 12:33:36,180] jepsen worker 14 - jepsen.util 7574 :fail :cas [4 [11 4]] :ValueNotSwapFrom INFO [2019-12-17 12:33:36,181] jepsen worker 14 - jepsen.util 7574 :invoke :write [4 33] INFO [2019-12-17 12:33:36,186] jepsen worker 46 - jepsen.util 7696 :ok :read [15 14] INFO [2019-12-17 12:33:36,186] jepsen worker 46 - jepsen.util 7696 :invoke :cas [15 [42 33]] INFO [2019-12-17 12:33:36,195] jepsen worker 17 - jepsen.util 10187 :info :write [5 14] :RequestCanceledException INFO [2019-12-17 12:33:36,196] jepsen worker 17 - jepsen.util 10277 :invoke :read [5 nil] INFO [2019-12-17 12:33:36,213] jepsen worker 84 - jepsen.util 9534 :info :cas [28 [16 24]] :RequestCanceledException INFO [2019-12-17 12:33:36,214] jepsen worker 84 - jepsen.util 9624 :invoke :cas [28 [36 3]] INFO [2019-12-17 12:33:36,217] jepsen worker 46 - jepsen.util 7696 :fail :cas [15 [42 33]] :ValueNotSwapFrom INFO [2019-12-17 12:33:36,217] jepsen worker 46 - jepsen.util 7696 :invoke :read [15 nil] INFO [2019-12-17 12:33:36,227] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42262,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:33:36,227] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42262,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:33:36,227] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42265,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:33:36,227] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42265,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:33:36,229] jepsen worker 39 - jepsen.util 9759 :info :cas [13 [25 25]] :RequestCanceledException INFO [2019-12-17 12:33:36,230] jepsen worker 39 - jepsen.util 9849 :invoke :read [13 nil] INFO [2019-12-17 12:33:36,235] jepsen worker 27 - jepsen.util 7317 :ok :write [9 18] INFO [2019-12-17 12:33:36,236] jepsen worker 27 - jepsen.util 7317 :invoke :write [9 16] INFO [2019-12-17 12:33:36,237] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42269,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:33:36,237] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42269,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:33:36,237] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42274,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:33:36,237] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42274,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:33:36,237] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42275,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:33:36,238] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42275,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:33:36,243] jepsen worker 72 - jepsen.util 7182 :ok :write [24 35] INFO [2019-12-17 12:33:36,243] jepsen worker 72 - jepsen.util 7182 :invoke :cas [24 [10 41]] INFO [2019-12-17 12:33:36,244] jepsen worker 46 - jepsen.util 7696 :ok :read [15 14] INFO [2019-12-17 12:33:36,244] jepsen worker 46 - jepsen.util 7696 :invoke :read [15 nil] INFO [2019-12-17 12:33:36,260] jepsen worker 48 - jepsen.util 8958 :info :write [16 2] :RequestCanceledException INFO [2019-12-17 12:33:36,260] jepsen worker 57 - jepsen.util 9777 :info :write [19 26] :RequestCanceledException INFO [2019-12-17 12:33:36,264] jepsen worker 57 - jepsen.util 9867 :invoke :cas [19 [33 10]] INFO [2019-12-17 12:33:36,264] jepsen worker 48 - jepsen.util 9048 :invoke :cas [16 [22 4]] INFO [2019-12-17 12:33:36,302] jepsen worker 20 - jepsen.util 7040 :ok :write [6 18] INFO [2019-12-17 12:33:36,302] jepsen worker 20 - jepsen.util 7040 :invoke :write [6 13] INFO [2019-12-17 12:33:36,303] jepsen worker 72 - jepsen.util 7182 :fail :cas [24 [10 41]] :ValueNotSwapFrom INFO [2019-12-17 12:33:36,303] jepsen worker 72 - jepsen.util 7182 :invoke :read [24 nil] INFO [2019-12-17 12:33:36,318] jepsen worker 12 - jepsen.util 7842 :ok :read [4 17] INFO [2019-12-17 12:33:36,318] jepsen worker 12 - jepsen.util 7842 :invoke :cas [4 [29 46]] INFO [2019-12-17 12:33:36,328] jepsen worker 4 - jepsen.util 6754 :fail :cas [1 [17 45]] :ValueNotSwapFrom INFO [2019-12-17 12:33:36,328] jepsen worker 4 - jepsen.util 6754 :invoke :read [1 nil] INFO [2019-12-17 12:33:36,329] jepsen worker 87 - jepsen.util 6567 :ok :write [29 32] INFO [2019-12-17 12:33:36,330] jepsen worker 87 - jepsen.util 6567 :invoke :write [29 0] INFO [2019-12-17 12:33:36,334] jepsen worker 52 - jepsen.util 7612 :fail :cas [17 [41 17]] :ValueNotSwapFrom INFO [2019-12-17 12:33:36,334] jepsen worker 52 - jepsen.util 7612 :invoke :read [17 nil] INFO [2019-12-17 12:33:36,343] jepsen worker 38 - jepsen.util 7508 :ok :read [12 39] INFO [2019-12-17 12:33:36,343] jepsen worker 38 - jepsen.util 7508 :invoke :write [12 30] INFO [2019-12-17 12:33:36,353] jepsen worker 46 - jepsen.util 7696 :ok :read [15 14] INFO [2019-12-17 12:33:36,353] jepsen worker 46 - jepsen.util 7696 :invoke :write [15 20] INFO [2019-12-17 12:33:36,365] jepsen worker 51 - jepsen.util 7071 :fail :cas [17 [20 9]] :ValueNotSwapFrom INFO [2019-12-17 12:33:36,365] jepsen worker 51 - jepsen.util 7071 :invoke :write [17 10] INFO [2019-12-17 12:33:36,383] jepsen worker 88 - jepsen.util 7288 :ok :read [29 32] INFO [2019-12-17 12:33:36,383] jepsen worker 88 - jepsen.util 7288 :invoke :read [29 nil] INFO [2019-12-17 12:33:36,393] jepsen worker 35 - jepsen.util 9845 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:33:36,393] jepsen worker 85 - jepsen.util 10795 :info :cas [28 [47 44]] :RequestCanceledException INFO [2019-12-17 12:33:36,393] jepsen worker 35 - jepsen.util 9845 :invoke :write [11 44] INFO [2019-12-17 12:33:36,395] jepsen worker 85 - jepsen.util 10885 :invoke :read [28 nil] INFO [2019-12-17 12:33:36,396] jepsen worker 72 - jepsen.util 7182 :ok :read [24 35] INFO [2019-12-17 12:33:36,396] jepsen worker 72 - jepsen.util 7182 :invoke :write [24 3] INFO [2019-12-17 12:33:36,400] jepsen worker 56 - jepsen.util 8156 :fail :cas [18 [49 10]] :ValueNotSwapFrom INFO [2019-12-17 12:33:36,400] jepsen worker 56 - jepsen.util 8156 :invoke :write [18 15] INFO [2019-12-17 12:33:36,411] jepsen worker 14 - jepsen.util 7574 :ok :write [4 33] INFO [2019-12-17 12:33:36,411] jepsen worker 14 - jepsen.util 7574 :invoke :read [4 nil] INFO [2019-12-17 12:33:36,412] jepsen worker 44 - jepsen.util 10574 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:33:36,413] jepsen worker 44 - jepsen.util 10574 :invoke :write [14 42] INFO [2019-12-17 12:33:36,427] jepsen worker 64 - jepsen.util 5554 :info :write [21 10] :RequestCanceledException INFO [2019-12-17 12:33:36,428] jepsen worker 64 - jepsen.util 5644 :invoke :read [21 nil] INFO [2019-12-17 12:33:36,428] jepsen worker 16 - jepsen.util 11086 :info :write [5 32] :RequestCanceledException INFO [2019-12-17 12:33:36,428] jepsen worker 31 - jepsen.util 6601 :fail :read [10 nil] :RequestTimeoutException INFO [2019-12-17 12:33:36,429] jepsen worker 31 - jepsen.util 6601 :invoke :cas [10 [43 9]] INFO [2019-12-17 12:33:36,430] jepsen worker 16 - jepsen.util 11176 :invoke :write [5 18] INFO [2019-12-17 12:33:36,442] jepsen worker 40 - jepsen.util 10300 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:33:36,442] jepsen worker 7 - jepsen.util 9997 :info :cas [2 [18 27]] :RequestCanceledException INFO [2019-12-17 12:33:36,442] jepsen worker 32 - jepsen.util 7232 :info :write [10 1] :RequestCanceledException INFO [2019-12-17 12:33:36,442] jepsen worker 61 - jepsen.util 10501 :info :write [20 9] :RequestCanceledException INFO [2019-12-17 12:33:36,442] jepsen worker 40 - jepsen.util 10300 :invoke :cas [13 [26 43]] INFO [2019-12-17 12:33:36,443] jepsen worker 32 - jepsen.util 7322 :invoke :write [10 10] INFO [2019-12-17 12:33:36,443] jepsen worker 7 - jepsen.util 10087 :invoke :cas [2 [10 7]] INFO [2019-12-17 12:33:36,443] jepsen worker 61 - jepsen.util 10591 :invoke :write [20 44] INFO [2019-12-17 12:33:36,443] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42302,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:33:36,443] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42302,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:33:36,443] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42306,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:33:36,443] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42306,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:33:36,453] jepsen worker 27 - jepsen.util 7317 :ok :write [9 16] INFO [2019-12-17 12:33:36,454] jepsen worker 27 - jepsen.util 7317 :invoke :cas [9 [28 10]] INFO [2019-12-17 12:33:36,459] jepsen worker 49 - jepsen.util 9319 :info :cas [16 [4 4]] :RequestCanceledException INFO [2019-12-17 12:33:36,459] jepsen worker 8 - jepsen.util 9908 :info :cas [2 [29 38]] :RequestCanceledException INFO [2019-12-17 12:33:36,460] jepsen worker 8 - jepsen.util 9998 :invoke :write [2 30] INFO [2019-12-17 12:33:36,460] jepsen worker 49 - jepsen.util 9409 :invoke :cas [16 [34 13]] INFO [2019-12-17 12:33:36,465] jepsen worker 64 - jepsen.util 5644 :ok :read [21 3] INFO [2019-12-17 12:33:36,465] jepsen worker 64 - jepsen.util 5644 :invoke :write [21 38] INFO [2019-12-17 12:33:36,494] jepsen worker 27 - jepsen.util 7317 :fail :cas [9 [28 10]] :ValueNotSwapFrom INFO [2019-12-17 12:33:36,494] jepsen worker 27 - jepsen.util 7317 :invoke :read [9 nil] INFO [2019-12-17 12:33:36,494] jepsen worker 73 - jepsen.util 6913 :info :write [24 25] :RequestCanceledException INFO [2019-12-17 12:33:36,494] jepsen worker 29 - jepsen.util 5609 :info :cas [9 [11 18]] :RequestCanceledException INFO [2019-12-17 12:33:36,496] jepsen worker 29 - jepsen.util 5699 :invoke :write [9 0] INFO [2019-12-17 12:33:36,496] jepsen worker 73 - jepsen.util 7003 :invoke :cas [24 [34 34]] INFO [2019-12-17 12:33:36,499] jepsen worker 78 - jepsen.util 7458 :fail :cas [26 [42 20]] :ValueNotSwapFrom INFO [2019-12-17 12:33:36,499] jepsen worker 78 - jepsen.util 7458 :invoke :write [26 33] INFO [2019-12-17 12:33:36,514] jepsen worker 12 - jepsen.util 7842 :fail :cas [4 [29 46]] :ValueNotSwapFrom INFO [2019-12-17 12:33:36,514] jepsen worker 12 - jepsen.util 7842 :invoke :cas [4 [16 40]] INFO [2019-12-17 12:33:36,521] jepsen worker 27 - jepsen.util 7317 :ok :read [9 16] INFO [2019-12-17 12:33:36,521] jepsen worker 27 - jepsen.util 7317 :invoke :cas [9 [43 41]] INFO [2019-12-17 12:33:36,528] jepsen worker 18 - jepsen.util 7398 :info :write [6 2] :RequestCanceledException INFO [2019-12-17 12:33:36,529] jepsen worker 18 - jepsen.util 7488 :invoke :cas [6 [23 41]] INFO [2019-12-17 12:33:36,531] jepsen worker 36 - jepsen.util 7686 :ok :write [12 10] INFO [2019-12-17 12:33:36,531] jepsen worker 36 - jepsen.util 7686 :invoke :read [12 nil] INFO [2019-12-17 12:33:36,543] jepsen worker 11 - jepsen.util 7481 :fail :read [3 nil] :RequestTimeoutException INFO [2019-12-17 12:33:36,543] jepsen worker 63 - jepsen.util 6183 :info :cas [21 [38 32]] :RequestCanceledException INFO [2019-12-17 12:33:36,543] jepsen worker 11 - jepsen.util 7481 :invoke :read [3 nil] INFO [2019-12-17 12:33:36,544] jepsen worker 63 - jepsen.util 6273 :invoke :cas [21 [16 17]] INFO [2019-12-17 12:33:36,545] jepsen worker 27 - jepsen.util 7317 :fail :cas [9 [43 41]] :ValueNotSwapFrom INFO [2019-12-17 12:33:36,545] jepsen worker 27 - jepsen.util 7317 :invoke :read [9 nil] INFO [2019-12-17 12:33:36,566] jepsen worker 69 - jepsen.util 7449 :ok :read [23 9] INFO [2019-12-17 12:33:36,566] jepsen worker 69 - jepsen.util 7449 :invoke :write [23 34] INFO [2019-12-17 12:33:36,592] jepsen worker 19 - jepsen.util 7309 :info :cas [6 [27 10]] :RequestCanceledException INFO [2019-12-17 12:33:36,593] jepsen worker 19 - jepsen.util 7399 :invoke :cas [6 [45 49]] INFO [2019-12-17 12:33:36,597] jepsen worker 4 - jepsen.util 6754 :ok :read [1 14] INFO [2019-12-17 12:33:36,597] jepsen worker 4 - jepsen.util 6754 :invoke :read [1 nil] INFO [2019-12-17 12:33:36,600] jepsen worker 11 - jepsen.util 7481 :ok :read [3 7] INFO [2019-12-17 12:33:36,600] jepsen worker 11 - jepsen.util 7481 :invoke :write [3 12] INFO [2019-12-17 12:33:36,601] jepsen worker 18 - jepsen.util 7488 :fail :cas [6 [23 41]] :ValueNotSwapFrom INFO [2019-12-17 12:33:36,601] jepsen worker 18 - jepsen.util 7488 :invoke :read [6 nil] INFO [2019-12-17 12:33:36,604] jepsen worker 88 - jepsen.util 7288 :ok :read [29 32] INFO [2019-12-17 12:33:36,605] jepsen worker 88 - jepsen.util 7288 :invoke :write [29 8] INFO [2019-12-17 12:33:36,617] jepsen worker 79 - jepsen.util 7369 :fail :cas [26 [45 30]] :ValueNotSwapFrom INFO [2019-12-17 12:33:36,617] jepsen worker 79 - jepsen.util 7369 :invoke :write [26 6] INFO [2019-12-17 12:33:36,628] jepsen worker 30 - jepsen.util 7140 :info :cas [10 [14 9]] :RequestCanceledException INFO [2019-12-17 12:33:36,629] jepsen worker 30 - jepsen.util 7230 :invoke :write [10 34] INFO [2019-12-17 12:33:36,636] jepsen worker 13 - jepsen.util 6943 :fail :cas [4 [18 33]] :ValueNotSwapFrom INFO [2019-12-17 12:33:36,637] jepsen worker 13 - jepsen.util 6943 :invoke :write [4 49] INFO [2019-12-17 12:33:36,640] jepsen worker 27 - jepsen.util 7317 :ok :read [9 16] INFO [2019-12-17 12:33:36,640] jepsen worker 47 - jepsen.util 7517 :fail :read [15 nil] :RequestTimeoutException INFO [2019-12-17 12:33:36,640] jepsen worker 27 - jepsen.util 7317 :invoke :cas [9 [34 16]] INFO [2019-12-17 12:33:36,640] jepsen worker 45 - jepsen.util 7245 :fail :read [15 nil] :RequestTimeoutException INFO [2019-12-17 12:33:36,640] jepsen worker 47 - jepsen.util 7517 :invoke :write [15 33] INFO [2019-12-17 12:33:36,640] jepsen worker 45 - jepsen.util 7245 :invoke :read [15 nil] INFO [2019-12-17 12:33:36,659] jepsen worker 70 - jepsen.util 7630 :info :write [23 19] :RequestCanceledException INFO [2019-12-17 12:33:36,660] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42336,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:33:36,660] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42336,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:33:36,660] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42339,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:33:36,660] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42339,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:33:36,660] jepsen worker 70 - jepsen.util 7720 :invoke :read [23 nil] INFO [2019-12-17 12:33:36,660] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42340,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:33:36,660] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42340,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:33:36,667] jepsen worker 18 - jepsen.util 7488 :ok :read [6 18] INFO [2019-12-17 12:33:36,667] jepsen worker 18 - jepsen.util 7488 :invoke :write [6 0] INFO [2019-12-17 12:33:36,668] jepsen worker 12 - jepsen.util 7842 :fail :cas [4 [16 40]] :ValueNotSwapFrom INFO [2019-12-17 12:33:36,668] jepsen worker 12 - jepsen.util 7842 :invoke :write [4 22] INFO [2019-12-17 12:33:36,677] jepsen worker 19 - jepsen.util 7399 :fail :cas [6 [45 49]] :ValueNotSwapFrom INFO [2019-12-17 12:33:36,677] jepsen worker 19 - jepsen.util 7399 :invoke :cas [6 [46 34]] INFO [2019-12-17 12:33:36,683] jepsen worker 64 - jepsen.util 5644 :ok :write [21 38] INFO [2019-12-17 12:33:36,683] jepsen worker 64 - jepsen.util 5644 :invoke :cas [21 [30 25]] INFO [2019-12-17 12:33:36,725] jepsen worker 72 - jepsen.util 7182 :ok :write [24 3] INFO [2019-12-17 12:33:36,725] jepsen worker 20 - jepsen.util 7040 :ok :write [6 13] INFO [2019-12-17 12:33:36,725] jepsen worker 72 - jepsen.util 7182 :invoke :write [24 35] INFO [2019-12-17 12:33:36,725] jepsen worker 20 - jepsen.util 7040 :invoke :cas [6 [49 45]] INFO [2019-12-17 12:33:36,753] jepsen worker 36 - jepsen.util 7686 :ok :read [12 10] INFO [2019-12-17 12:33:36,753] jepsen worker 36 - jepsen.util 7686 :invoke :write [12 36] INFO [2019-12-17 12:33:36,765] jepsen worker 4 - jepsen.util 6754 :ok :read [1 14] INFO [2019-12-17 12:33:36,765] jepsen worker 4 - jepsen.util 6754 :invoke :write [1 31] INFO [2019-12-17 12:33:36,785] jepsen worker 14 - jepsen.util 7574 :ok :read [4 33] INFO [2019-12-17 12:33:36,785] jepsen worker 14 - jepsen.util 7574 :invoke :read [4 nil] INFO [2019-12-17 12:33:36,794] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:33:36,794] jepsen nemesis - couchbase.util Rebalance check count: 324 INFO [2019-12-17 12:33:36,794] jepsen nemesis - couchbase.util Rebalance stuck count: 305 INFO [2019-12-17 12:33:36,811] jepsen worker 70 - jepsen.util 7720 :ok :read [23 9] INFO [2019-12-17 12:33:36,812] jepsen worker 70 - jepsen.util 7720 :invoke :cas [23 [37 25]] INFO [2019-12-17 12:33:36,813] jepsen worker 20 - jepsen.util 7040 :fail :cas [6 [49 45]] :ValueNotSwapFrom INFO [2019-12-17 12:33:36,813] jepsen worker 28 - jepsen.util 5608 :info :write [9 7] :RequestCanceledException INFO [2019-12-17 12:33:36,813] jepsen worker 20 - jepsen.util 7040 :invoke :read [6 nil] INFO [2019-12-17 12:33:36,813] jepsen worker 31 - jepsen.util 6601 :fail :cas [10 [43 9]] :ValueNotSwapFrom INFO [2019-12-17 12:33:36,813] jepsen worker 31 - jepsen.util 6601 :invoke :read [10 nil] INFO [2019-12-17 12:33:36,814] jepsen worker 28 - jepsen.util 5698 :invoke :read [9 nil] INFO [2019-12-17 12:33:36,828] jepsen worker 2 - jepsen.util 9542 :info :write [0 9] :RequestCanceledException INFO [2019-12-17 12:33:36,829] jepsen worker 71 - jepsen.util 8081 :info :write [23 3] :RequestCanceledException INFO [2019-12-17 12:33:36,833] jepsen worker 71 - jepsen.util 8171 :invoke :read [23 nil] INFO [2019-12-17 12:33:36,833] jepsen worker 63 - jepsen.util 6273 :fail :cas [21 [16 17]] :ValueNotSwapFrom INFO [2019-12-17 12:33:36,833] jepsen worker 63 - jepsen.util 6273 :invoke :read [21 nil] INFO [2019-12-17 12:33:36,835] jepsen worker 2 - jepsen.util 9632 :invoke :write [0 31] INFO [2019-12-17 12:33:36,844] jepsen worker 46 - jepsen.util 7696 :ok :write [15 20] INFO [2019-12-17 12:33:36,844] jepsen worker 30 - jepsen.util 7230 :ok :write [10 34] INFO [2019-12-17 12:33:36,844] jepsen worker 46 - jepsen.util 7696 :invoke :cas [15 [9 15]] INFO [2019-12-17 12:33:36,844] jepsen worker 30 - jepsen.util 7230 :invoke :write [10 15] INFO [2019-12-17 12:33:36,850] jepsen worker 20 - jepsen.util 7040 :ok :read [6 13] INFO [2019-12-17 12:33:36,850] jepsen worker 20 - jepsen.util 7040 :invoke :read [6 nil] INFO [2019-12-17 12:33:36,879] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42366,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:33:36,880] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42366,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:33:36,880] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42369,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:33:36,880] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42369,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:33:36,880] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42370,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:33:36,880] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42370,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:33:36,885] jepsen worker 70 - jepsen.util 7720 :fail :cas [23 [37 25]] :ValueNotSwapFrom INFO [2019-12-17 12:33:36,885] jepsen worker 70 - jepsen.util 7720 :invoke :cas [23 [20 32]] INFO [2019-12-17 12:33:36,887] jepsen worker 28 - jepsen.util 5698 :ok :read [9 16] INFO [2019-12-17 12:33:36,888] jepsen worker 28 - jepsen.util 5698 :invoke :cas [9 [10 4]] INFO [2019-12-17 12:33:36,902] jepsen worker 20 - jepsen.util 7040 :ok :read [6 13] INFO [2019-12-17 12:33:36,903] jepsen worker 20 - jepsen.util 7040 :invoke :cas [6 [32 22]] INFO [2019-12-17 12:33:36,913] jepsen worker 65 - jepsen.util 6005 :info :write [21 33] :RequestCanceledException INFO [2019-12-17 12:33:36,923] jepsen worker 28 - jepsen.util 5698 :fail :cas [9 [10 4]] :ValueNotSwapFrom INFO [2019-12-17 12:33:36,923] jepsen worker 28 - jepsen.util 5698 :invoke :write [9 31] INFO [2019-12-17 12:33:36,925] jepsen worker 65 - jepsen.util 6095 :invoke :read [21 nil] INFO [2019-12-17 12:33:36,931] jepsen worker 29 - jepsen.util 5699 :ok :write [9 0] INFO [2019-12-17 12:33:36,931] jepsen worker 29 - jepsen.util 5699 :invoke :read [9 nil] INFO [2019-12-17 12:33:36,933] jepsen worker 74 - jepsen.util 7454 :info :write [24 7] :RequestCanceledException INFO [2019-12-17 12:33:36,934] jepsen worker 74 - jepsen.util 7544 :invoke :write [24 8] INFO [2019-12-17 12:33:37,003] jepsen worker 46 - jepsen.util 7696 :fail :cas [15 [9 15]] :ValueNotSwapFrom INFO [2019-12-17 12:33:37,003] jepsen worker 46 - jepsen.util 7696 :invoke :read [15 nil] INFO [2019-12-17 12:33:37,012] jepsen worker 14 - jepsen.util 7574 :ok :read [4 33] INFO [2019-12-17 12:33:37,012] jepsen worker 14 - jepsen.util 7574 :invoke :write [4 14] INFO [2019-12-17 12:33:37,025] jepsen worker 29 - jepsen.util 5699 :ok :read [9 0] INFO [2019-12-17 12:33:37,025] jepsen worker 29 - jepsen.util 5699 :invoke :cas [9 [22 49]] INFO [2019-12-17 12:33:37,029] jepsen worker 20 - jepsen.util 7040 :fail :cas [6 [32 22]] :ValueNotSwapFrom INFO [2019-12-17 12:33:37,029] jepsen worker 20 - jepsen.util 7040 :invoke :write [6 13] INFO [2019-12-17 12:33:37,060] jepsen worker 46 - jepsen.util 7696 :ok :read [15 20] INFO [2019-12-17 12:33:37,060] jepsen worker 46 - jepsen.util 7696 :invoke :cas [15 [42 23]] INFO [2019-12-17 12:33:37,095] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42378,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:33:37,095] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42378,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:33:37,095] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42382,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:33:37,095] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42382,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:33:37,095] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42383,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:33:37,095] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42383,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:33:37,104] jepsen worker 29 - jepsen.util 5699 :fail :cas [9 [22 49]] :ValueNotSwapFrom INFO [2019-12-17 12:33:37,104] jepsen worker 29 - jepsen.util 5699 :invoke :cas [9 [38 19]] INFO [2019-12-17 12:33:37,105] jepsen worker 56 - jepsen.util 8156 :ok :write [18 15] INFO [2019-12-17 12:33:37,105] jepsen worker 56 - jepsen.util 8156 :invoke :write [18 26] INFO [2019-12-17 12:33:37,112] jepsen worker 9 - jepsen.util 7839 :fail :read [3 nil] :RequestTimeoutException INFO [2019-12-17 12:33:37,112] jepsen worker 10 - jepsen.util 7300 :info :cas [3 [5 5]] :RequestCanceledException INFO [2019-12-17 12:33:37,113] jepsen worker 9 - jepsen.util 7839 :invoke :cas [3 [40 11]] INFO [2019-12-17 12:33:37,113] jepsen worker 10 - jepsen.util 7390 :invoke :cas [3 [8 26]] INFO [2019-12-17 12:33:37,121] jepsen worker 46 - jepsen.util 7696 :fail :cas [15 [42 23]] :ValueNotSwapFrom INFO [2019-12-17 12:33:37,121] jepsen worker 46 - jepsen.util 7696 :invoke :cas [15 [35 17]] INFO [2019-12-17 12:33:37,154] jepsen worker 29 - jepsen.util 5699 :fail :cas [9 [38 19]] :ValueNotSwapFrom INFO [2019-12-17 12:33:37,155] jepsen worker 29 - jepsen.util 5699 :invoke :write [9 44] INFO [2019-12-17 12:33:37,159] jepsen worker 53 - jepsen.util 6803 :info :write [17 15] :RequestCanceledException INFO [2019-12-17 12:33:37,160] jepsen worker 53 - jepsen.util 6893 :invoke :write [17 27] INFO [2019-12-17 12:33:37,194] jepsen worker 10 - jepsen.util 7390 :fail :cas [3 [8 26]] :ValueNotSwapFrom INFO [2019-12-17 12:33:37,194] jepsen worker 10 - jepsen.util 7390 :invoke :read [3 nil] INFO [2019-12-17 12:33:37,203] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42397,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:33:37,203] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42397,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:33:37,203] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42398,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:33:37,203] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42398,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:33:37,203] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42399,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:33:37,203] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42399,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:33:37,216] jepsen worker 28 - jepsen.util 5698 :ok :write [9 31] INFO [2019-12-17 12:33:37,216] jepsen worker 28 - jepsen.util 5698 :invoke :write [9 31] INFO [2019-12-17 12:33:37,222] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42404,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:33:37,222] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42404,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:33:37,222] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42407,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:33:37,222] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42407,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:33:37,242] jepsen worker 46 - jepsen.util 7696 :fail :cas [15 [35 17]] :ValueNotSwapFrom INFO [2019-12-17 12:33:37,242] jepsen worker 46 - jepsen.util 7696 :invoke :write [15 49] INFO [2019-12-17 12:33:37,285] jepsen worker 29 - jepsen.util 5699 :ok :write [9 44] INFO [2019-12-17 12:33:37,285] jepsen worker 29 - jepsen.util 5699 :invoke :write [9 42] INFO [2019-12-17 12:33:37,329] jepsen worker 52 - jepsen.util 7612 :ok :read [17 29] INFO [2019-12-17 12:33:37,329] jepsen worker 52 - jepsen.util 7612 :invoke :write [17 39] INFO [2019-12-17 12:33:37,428] jepsen worker 37 - jepsen.util 6877 :info :write [12 12] :RequestCanceledException INFO [2019-12-17 12:33:37,429] jepsen worker 37 - jepsen.util 6967 :invoke :read [12 nil] INFO [2019-12-17 12:33:37,471] jepsen worker 71 - jepsen.util 8171 :ok :read [23 19] INFO [2019-12-17 12:33:37,471] jepsen worker 71 - jepsen.util 8171 :invoke :read [23 nil] INFO [2019-12-17 12:33:37,531] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42415,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:33:37,531] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42415,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:33:37,531] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42419,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:33:37,531] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42419,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:33:37,554] jepsen worker 70 - jepsen.util 7720 :fail :cas [23 [20 32]] :ValueNotSwapFrom INFO [2019-12-17 12:33:37,554] jepsen worker 70 - jepsen.util 7720 :invoke :write [23 24] INFO [2019-12-17 12:33:37,613] jepsen worker 3 - jepsen.util 7383 :info :write [1 47] :RequestCanceledException INFO [2019-12-17 12:33:37,614] jepsen worker 3 - jepsen.util 7473 :invoke :read [1 nil] INFO [2019-12-17 12:33:37,629] jepsen worker 55 - jepsen.util 8245 :info :write [18 38] :RequestCanceledException INFO [2019-12-17 12:33:37,630] jepsen worker 55 - jepsen.util 8335 :invoke :read [18 nil] INFO [2019-12-17 12:33:37,660] jepsen worker 34 - jepsen.util 9304 :info :cas [11 [48 5]] :RequestCanceledException INFO [2019-12-17 12:33:37,660] jepsen worker 34 - jepsen.util 9394 :invoke :cas [11 [20 26]] INFO [2019-12-17 12:33:37,695] jepsen worker 42 - jepsen.util 10662 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:33:37,696] jepsen worker 42 - jepsen.util 10662 :invoke :write [14 39] INFO [2019-12-17 12:33:37,712] jepsen worker 83 - jepsen.util 10433 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:33:37,713] jepsen worker 83 - jepsen.util 10433 :invoke :cas [27 [29 2]] INFO [2019-12-17 12:33:37,727] jepsen worker 43 - jepsen.util 9403 :info :cas [14 [27 8]] :RequestCanceledException INFO [2019-12-17 12:33:37,727] jepsen worker 21 - jepsen.util 9381 :info :cas [7 [12 11]] :RequestCanceledException INFO [2019-12-17 12:33:37,727] jepsen worker 75 - jepsen.util 9345 :info :cas [25 [24 19]] :RequestCanceledException INFO [2019-12-17 12:33:37,728] jepsen worker 75 - jepsen.util 9435 :invoke :write [25 11] INFO [2019-12-17 12:33:37,728] jepsen worker 43 - jepsen.util 9493 :invoke :cas [14 [37 48]] INFO [2019-12-17 12:33:37,728] jepsen worker 21 - jepsen.util 9471 :invoke :read [7 nil] INFO [2019-12-17 12:33:37,742] jepsen worker 23 - jepsen.util 10103 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:33:37,742] jepsen worker 66 - jepsen.util 9786 :info :cas [22 [1 18]] :RequestCanceledException INFO [2019-12-17 12:33:37,742] jepsen worker 23 - jepsen.util 10103 :invoke :write [7 13] INFO [2019-12-17 12:33:37,743] jepsen worker 66 - jepsen.util 9876 :invoke :write [22 39] INFO [2019-12-17 12:33:37,750] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42428,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:33:37,750] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42428,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:33:37,760] jepsen worker 26 - jepsen.util 10196 :info :cas [8 [47 45]] :RequestCanceledException INFO [2019-12-17 12:33:37,760] jepsen worker 26 - jepsen.util 10286 :invoke :write [8 43] INFO [2019-12-17 12:33:37,773] jepsen worker 37 - jepsen.util 6967 :ok :read [12 10] INFO [2019-12-17 12:33:37,773] jepsen worker 37 - jepsen.util 6967 :invoke :read [12 nil] INFO [2019-12-17 12:33:37,785] jepsen worker 71 - jepsen.util 8171 :ok :read [23 19] INFO [2019-12-17 12:33:37,786] jepsen worker 71 - jepsen.util 8171 :invoke :cas [23 [31 31]] INFO [2019-12-17 12:33:37,795] jepsen worker 22 - jepsen.util 9742 :info :cas [7 [1 2]] :RequestCanceledException INFO [2019-12-17 12:33:37,796] jepsen worker 22 - jepsen.util 9832 :invoke :cas [7 [21 2]] INFO [2019-12-17 12:33:37,807] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:33:37,807] jepsen nemesis - couchbase.util Rebalance check count: 325 INFO [2019-12-17 12:33:37,807] jepsen nemesis - couchbase.util Rebalance stuck count: 306 INFO [2019-12-17 12:33:37,813] jepsen worker 60 - jepsen.util 10770 :info :write [20 43] :RequestCanceledException INFO [2019-12-17 12:33:37,814] jepsen worker 60 - jepsen.util 10860 :invoke :write [20 16] INFO [2019-12-17 12:33:37,828] jepsen worker 0 - jepsen.util 10440 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:33:37,828] jepsen worker 86 - jepsen.util 10436 :info :cas [28 [12 30]] :RequestCanceledException INFO [2019-12-17 12:33:37,828] jepsen worker 89 - jepsen.util 7109 :info :write [29 14] :RequestCanceledException INFO [2019-12-17 12:33:37,828] jepsen worker 58 - jepsen.util 10678 :info :cas [19 [3 3]] :RequestCanceledException INFO [2019-12-17 12:33:37,828] jepsen worker 62 - jepsen.util 9962 :info :write [20 26] :RequestCanceledException INFO [2019-12-17 12:33:37,828] jepsen worker 0 - jepsen.util 10440 :invoke :read [0 nil] INFO [2019-12-17 12:33:37,829] jepsen worker 89 - jepsen.util 7199 :invoke :cas [29 [1 25]] INFO [2019-12-17 12:33:37,829] jepsen worker 86 - jepsen.util 10526 :invoke :write [28 40] INFO [2019-12-17 12:33:37,829] jepsen worker 58 - jepsen.util 10768 :invoke :read [19 nil] INFO [2019-12-17 12:33:37,829] jepsen worker 62 - jepsen.util 10052 :invoke :cas [20 [39 32]] INFO [2019-12-17 12:33:37,837] jepsen worker 55 - jepsen.util 8335 :ok :read [18 15] INFO [2019-12-17 12:33:37,837] jepsen worker 55 - jepsen.util 8335 :invoke :cas [18 [44 0]] INFO [2019-12-17 12:33:37,842] jepsen worker 5 - jepsen.util 6845 :info :write [1 39] :RequestCanceledException INFO [2019-12-17 12:33:37,843] jepsen worker 5 - jepsen.util 6935 :invoke :write [1 42] INFO [2019-12-17 12:33:37,861] jepsen worker 59 - jepsen.util 9779 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:33:37,862] jepsen worker 54 - jepsen.util 8154 :info :write [18 25] :RequestCanceledException INFO [2019-12-17 12:33:37,862] jepsen worker 59 - jepsen.util 9779 :invoke :cas [19 [38 0]] INFO [2019-12-17 12:33:37,862] jepsen worker 67 - jepsen.util 9877 :info :cas [22 [13 49]] :RequestCanceledException INFO [2019-12-17 12:33:37,863] jepsen worker 37 - jepsen.util 6967 :ok :read [12 10] INFO [2019-12-17 12:33:37,863] jepsen worker 37 - jepsen.util 6967 :invoke :read [12 nil] INFO [2019-12-17 12:33:37,863] jepsen worker 54 - jepsen.util 8244 :invoke :write [18 22] INFO [2019-12-17 12:33:37,863] jepsen worker 67 - jepsen.util 9967 :invoke :write [22 8] INFO [2019-12-17 12:33:37,895] jepsen worker 41 - jepsen.util 10391 :info :cas [13 [33 4]] :RequestCanceledException INFO [2019-12-17 12:33:37,895] jepsen worker 68 - jepsen.util 9338 :info :cas [22 [27 39]] :RequestCanceledException INFO [2019-12-17 12:33:37,896] jepsen worker 68 - jepsen.util 9428 :invoke :write [22 4] INFO [2019-12-17 12:33:37,896] jepsen worker 41 - jepsen.util 10481 :invoke :cas [13 [24 7]] INFO [2019-12-17 12:33:37,902] jepsen worker 71 - jepsen.util 8171 :fail :cas [23 [31 31]] :ValueNotSwapFrom INFO [2019-12-17 12:33:37,902] jepsen worker 71 - jepsen.util 8171 :invoke :cas [23 [29 43]] INFO [2019-12-17 12:33:37,911] jepsen worker 77 - jepsen.util 10067 :info :cas [25 [27 22]] :RequestCanceledException INFO [2019-12-17 12:33:37,911] jepsen worker 25 - jepsen.util 10465 :info :write [8 48] :RequestCanceledException INFO [2019-12-17 12:33:37,913] jepsen worker 77 - jepsen.util 10157 :invoke :write [25 15] INFO [2019-12-17 12:33:37,913] jepsen worker 25 - jepsen.util 10555 :invoke :read [8 nil] INFO [2019-12-17 12:33:37,928] jepsen worker 82 - jepsen.util 10612 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:33:37,928] jepsen worker 33 - jepsen.util 9663 :info :cas [11 [34 3]] :RequestCanceledException INFO [2019-12-17 12:33:37,928] jepsen worker 82 - jepsen.util 10612 :invoke :write [27 24] INFO [2019-12-17 12:33:37,929] jepsen worker 33 - jepsen.util 9753 :invoke :write [11 49] INFO [2019-12-17 12:33:37,943] jepsen worker 24 - jepsen.util 8664 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:33:37,943] jepsen worker 24 - jepsen.util 8664 :invoke :write [8 28] INFO [2019-12-17 12:33:37,960] jepsen worker 15 - jepsen.util 8835 :info :write [5 44] :RequestCanceledException INFO [2019-12-17 12:33:37,961] jepsen worker 15 - jepsen.util 8925 :invoke :cas [5 [33 29]] INFO [2019-12-17 12:33:37,970] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42450,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:33:37,970] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42450,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:33:37,971] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42455,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:33:37,971] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42455,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:33:37,990] jepsen worker 50 - jepsen.util 10580 :info :cas [16 [37 39]] :RequestCanceledException INFO [2019-12-17 12:33:37,992] jepsen worker 50 - jepsen.util 10670 :invoke :cas [16 [26 21]] INFO [2019-12-17 12:33:38,001] jepsen worker 55 - jepsen.util 8335 :fail :cas [18 [44 0]] :ValueNotSwapFrom INFO [2019-12-17 12:33:38,001] jepsen worker 55 - jepsen.util 8335 :invoke :write [18 27] INFO [2019-12-17 12:33:38,028] jepsen worker 1 - jepsen.util 9991 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:33:38,028] jepsen worker 1 - jepsen.util 9991 :invoke :write [0 27] INFO [2019-12-17 12:33:38,079] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42474,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:33:38,079] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42474,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:33:38,079] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42475,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:33:38,079] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42475,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:33:38,082] jepsen worker 3 - jepsen.util 7473 :ok :read [1 14] INFO [2019-12-17 12:33:38,082] jepsen worker 3 - jepsen.util 7473 :invoke :read [1 nil] INFO [2019-12-17 12:33:38,092] jepsen worker 72 - jepsen.util 7182 :ok :write [24 35] INFO [2019-12-17 12:33:38,092] jepsen worker 30 - jepsen.util 7230 :ok :write [10 15] INFO [2019-12-17 12:33:38,093] jepsen worker 46 - jepsen.util 7696 :ok :write [15 49] INFO [2019-12-17 12:33:38,093] jepsen worker 72 - jepsen.util 7182 :invoke :cas [24 [40 29]] INFO [2019-12-17 12:33:38,093] jepsen worker 30 - jepsen.util 7230 :invoke :cas [10 [29 25]] INFO [2019-12-17 12:33:38,093] jepsen worker 11 - jepsen.util 7481 :ok :write [3 12] INFO [2019-12-17 12:33:38,093] jepsen worker 46 - jepsen.util 7696 :invoke :write [15 10] INFO [2019-12-17 12:33:38,093] jepsen worker 18 - jepsen.util 7488 :ok :write [6 0] INFO [2019-12-17 12:33:38,093] jepsen worker 18 - jepsen.util 7488 :invoke :read [6 nil] INFO [2019-12-17 12:33:38,093] jepsen worker 11 - jepsen.util 7481 :invoke :read [3 nil] INFO [2019-12-17 12:33:38,110] jepsen worker 6 - jepsen.util 10626 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:33:38,110] jepsen worker 6 - jepsen.util 10626 :invoke :write [2 4] INFO [2019-12-17 12:33:38,128] jepsen worker 80 - jepsen.util 7370 :info :write [26 15] :RequestCanceledException INFO [2019-12-17 12:33:38,129] jepsen worker 80 - jepsen.util 7460 :invoke :write [26 16] INFO [2019-12-17 12:33:38,173] jepsen worker 11 - jepsen.util 7481 :ok :read [3 12] INFO [2019-12-17 12:33:38,174] jepsen worker 11 - jepsen.util 7481 :invoke :cas [3 [2 45]] INFO [2019-12-17 12:33:38,174] jepsen worker 9 - jepsen.util 7839 :fail :cas [3 [40 11]] :ValueNotSwapFrom INFO [2019-12-17 12:33:38,175] jepsen worker 9 - jepsen.util 7839 :invoke :write [3 23] INFO [2019-12-17 12:33:38,192] jepsen worker 31 - jepsen.util 6601 :ok :read [10 15] INFO [2019-12-17 12:33:38,192] jepsen worker 31 - jepsen.util 6601 :invoke :cas [10 [3 28]] INFO [2019-12-17 12:33:38,228] jepsen worker 76 - jepsen.util 10066 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:33:38,228] jepsen worker 84 - jepsen.util 9624 :info :cas [28 [36 3]] :RequestCanceledException INFO [2019-12-17 12:33:38,228] jepsen worker 76 - jepsen.util 10066 :invoke :read [25 nil] INFO [2019-12-17 12:33:38,230] jepsen worker 84 - jepsen.util 9714 :invoke :read [28 nil] INFO [2019-12-17 12:33:38,240] jepsen worker 63 - jepsen.util 6273 :ok :read [21 38] INFO [2019-12-17 12:33:38,240] jepsen worker 63 - jepsen.util 6273 :invoke :write [21 9] INFO [2019-12-17 12:33:38,243] jepsen worker 81 - jepsen.util 9801 :info :write [27 10] :RequestCanceledException INFO [2019-12-17 12:33:38,244] jepsen worker 81 - jepsen.util 9891 :invoke :write [27 9] INFO [2019-12-17 12:33:38,260] jepsen worker 17 - jepsen.util 10277 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:33:38,260] jepsen worker 17 - jepsen.util 10277 :invoke :cas [5 [6 2]] INFO [2019-12-17 12:33:38,265] jepsen worker 72 - jepsen.util 7182 :fail :cas [24 [40 29]] :ValueNotSwapFrom INFO [2019-12-17 12:33:38,265] jepsen worker 72 - jepsen.util 7182 :invoke :cas [24 [45 48]] INFO [2019-12-17 12:33:38,295] jepsen worker 39 - jepsen.util 9849 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:33:38,295] jepsen worker 39 - jepsen.util 9849 :invoke :read [13 nil] INFO [2019-12-17 12:33:38,296] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42489,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:33:38,296] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42489,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:33:38,296] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42490,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:33:38,296] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42490,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:33:38,296] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42491,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:33:38,296] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42491,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:33:38,306] jepsen worker 11 - jepsen.util 7481 :fail :cas [3 [2 45]] :ValueNotSwapFrom INFO [2019-12-17 12:33:38,306] jepsen worker 11 - jepsen.util 7481 :invoke :read [3 nil] INFO [2019-12-17 12:33:38,310] jepsen worker 31 - jepsen.util 6601 :fail :cas [10 [3 28]] :ValueNotSwapFrom INFO [2019-12-17 12:33:38,310] jepsen worker 31 - jepsen.util 6601 :invoke :read [10 nil] INFO [2019-12-17 12:33:38,312] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42498,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:33:38,312] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42498,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:33:38,312] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42499,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:33:38,313] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42499,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:33:38,313] jepsen worker 18 - jepsen.util 7488 :ok :read [6 0] INFO [2019-12-17 12:33:38,313] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42500,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:33:38,313] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42500,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:33:38,313] jepsen worker 18 - jepsen.util 7488 :invoke :cas [6 [29 30]] INFO [2019-12-17 12:33:38,313] jepsen worker 57 - jepsen.util 9867 :info :cas [19 [33 10]] :RequestCanceledException INFO [2019-12-17 12:33:38,314] jepsen worker 57 - jepsen.util 9957 :invoke :cas [19 [44 17]] INFO [2019-12-17 12:33:38,327] jepsen worker 11 - jepsen.util 7481 :ok :read [3 12] INFO [2019-12-17 12:33:38,327] jepsen worker 11 - jepsen.util 7481 :invoke :write [3 43] INFO [2019-12-17 12:33:38,328] jepsen worker 48 - jepsen.util 9048 :info :cas [16 [22 4]] :RequestCanceledException INFO [2019-12-17 12:33:38,330] jepsen worker 48 - jepsen.util 9138 :invoke :write [16 24] INFO [2019-12-17 12:33:38,389] jepsen worker 4 - jepsen.util 6754 :ok :write [1 31] INFO [2019-12-17 12:33:38,389] jepsen worker 4 - jepsen.util 6754 :invoke :write [1 9] INFO [2019-12-17 12:33:38,396] jepsen worker 13 - jepsen.util 6943 :ok :write [4 49] INFO [2019-12-17 12:33:38,397] jepsen worker 13 - jepsen.util 6943 :invoke :cas [4 [43 42]] INFO [2019-12-17 12:33:38,402] jepsen worker 63 - jepsen.util 6273 :ok :write [21 9] INFO [2019-12-17 12:33:38,402] jepsen worker 63 - jepsen.util 6273 :invoke :write [21 9] INFO [2019-12-17 12:33:38,410] jepsen worker 87 - jepsen.util 6567 :info :write [29 0] :RequestCanceledException INFO [2019-12-17 12:33:38,410] jepsen worker 51 - jepsen.util 7071 :info :write [17 10] :RequestCanceledException INFO [2019-12-17 12:33:38,411] jepsen worker 51 - jepsen.util 7161 :invoke :write [17 30] INFO [2019-12-17 12:33:38,411] jepsen worker 87 - jepsen.util 6657 :invoke :cas [29 [8 46]] INFO [2019-12-17 12:33:38,431] jepsen worker 3 - jepsen.util 7473 :ok :read [1 31] INFO [2019-12-17 12:33:38,431] jepsen worker 3 - jepsen.util 7473 :invoke :write [1 36] INFO [2019-12-17 12:33:38,442] jepsen worker 85 - jepsen.util 10885 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:33:38,442] jepsen worker 38 - jepsen.util 7508 :info :write [12 30] :RequestCanceledException INFO [2019-12-17 12:33:38,442] jepsen worker 85 - jepsen.util 10885 :invoke :cas [28 [17 5]] INFO [2019-12-17 12:33:38,443] jepsen worker 38 - jepsen.util 7598 :invoke :cas [12 [28 24]] INFO [2019-12-17 12:33:38,456] jepsen worker 37 - jepsen.util 6967 :ok :read [12 10] INFO [2019-12-17 12:33:38,456] jepsen worker 37 - jepsen.util 6967 :invoke :cas [12 [38 6]] INFO [2019-12-17 12:33:38,483] jepsen worker 71 - jepsen.util 8171 :fail :cas [23 [29 43]] :ValueNotSwapFrom INFO [2019-12-17 12:33:38,483] jepsen worker 71 - jepsen.util 8171 :invoke :cas [23 [3 25]] INFO [2019-12-17 12:33:38,495] jepsen worker 35 - jepsen.util 9845 :info :write [11 44] :RequestCanceledException INFO [2019-12-17 12:33:38,495] jepsen worker 32 - jepsen.util 7322 :info :write [10 10] :RequestCanceledException INFO [2019-12-17 12:33:38,496] jepsen worker 35 - jepsen.util 9935 :invoke :read [11 nil] INFO [2019-12-17 12:33:38,496] jepsen worker 32 - jepsen.util 7412 :invoke :read [10 nil] INFO [2019-12-17 12:33:38,498] jepsen worker 63 - jepsen.util 6273 :ok :write [21 9] INFO [2019-12-17 12:33:38,498] jepsen worker 63 - jepsen.util 6273 :invoke :cas [21 [30 6]] INFO [2019-12-17 12:33:38,507] jepsen worker 89 - jepsen.util 7199 :fail :cas [29 [1 25]] :ValueNotSwapFrom INFO [2019-12-17 12:33:38,507] jepsen worker 89 - jepsen.util 7199 :invoke :read [29 nil] INFO [2019-12-17 12:33:38,513] jepsen worker 44 - jepsen.util 10574 :info :write [14 42] :RequestCanceledException INFO [2019-12-17 12:33:38,513] jepsen worker 7 - jepsen.util 10087 :info :cas [2 [10 7]] :RequestCanceledException INFO [2019-12-17 12:33:38,514] jepsen worker 7 - jepsen.util 10177 :invoke :cas [2 [33 48]] INFO [2019-12-17 12:33:38,514] jepsen worker 44 - jepsen.util 10664 :invoke :cas [14 [28 32]] INFO [2019-12-17 12:33:38,515] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42520,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:33:38,515] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42520,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:33:38,515] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42523,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:33:38,515] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42523,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:33:38,515] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42524,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:33:38,515] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42524,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:33:38,524] jepsen worker 63 - jepsen.util 6273 :fail :cas [21 [30 6]] :ValueNotSwapFrom INFO [2019-12-17 12:33:38,524] jepsen worker 63 - jepsen.util 6273 :invoke :write [21 20] INFO [2019-12-17 12:33:38,526] jepsen worker 61 - jepsen.util 10591 :info :write [20 44] :RequestCanceledException INFO [2019-12-17 12:33:38,526] jepsen worker 16 - jepsen.util 11176 :info :write [5 18] :RequestCanceledException INFO [2019-12-17 12:33:38,527] jepsen worker 61 - jepsen.util 10681 :invoke :write [20 12] INFO [2019-12-17 12:33:38,527] jepsen worker 16 - jepsen.util 11266 :invoke :read [5 nil] INFO [2019-12-17 12:33:38,541] jepsen worker 40 - jepsen.util 10300 :info :cas [13 [26 43]] :RequestCanceledException INFO [2019-12-17 12:33:38,542] jepsen worker 8 - jepsen.util 9998 :info :write [2 30] :RequestCanceledException INFO [2019-12-17 12:33:38,543] jepsen worker 40 - jepsen.util 10390 :invoke :cas [13 [44 3]] INFO [2019-12-17 12:33:38,543] jepsen worker 8 - jepsen.util 10088 :invoke :read [2 nil] INFO [2019-12-17 12:33:38,560] jepsen worker 49 - jepsen.util 9409 :info :cas [16 [34 13]] :RequestCanceledException INFO [2019-12-17 12:33:38,562] jepsen worker 49 - jepsen.util 9499 :invoke :read [16 nil] INFO [2019-12-17 12:33:38,578] jepsen worker 63 - jepsen.util 6273 :ok :write [21 20] INFO [2019-12-17 12:33:38,578] jepsen worker 63 - jepsen.util 6273 :invoke :read [21 nil] INFO [2019-12-17 12:33:38,593] jepsen worker 74 - jepsen.util 7544 :ok :write [24 8] INFO [2019-12-17 12:33:38,593] jepsen worker 46 - jepsen.util 7696 :ok :write [15 10] INFO [2019-12-17 12:33:38,593] jepsen worker 74 - jepsen.util 7544 :invoke :read [24 nil] INFO [2019-12-17 12:33:38,593] jepsen worker 46 - jepsen.util 7696 :invoke :read [15 nil] INFO [2019-12-17 12:33:38,593] jepsen worker 73 - jepsen.util 7003 :info :cas [24 [34 34]] :RequestCanceledException INFO [2019-12-17 12:33:38,594] jepsen worker 78 - jepsen.util 7458 :info :write [26 33] :RequestCanceledException INFO [2019-12-17 12:33:38,596] jepsen worker 73 - jepsen.util 7093 :invoke :read [24 nil] INFO [2019-12-17 12:33:38,596] jepsen worker 78 - jepsen.util 7548 :invoke :cas [26 [8 46]] INFO [2019-12-17 12:33:38,599] jepsen worker 63 - jepsen.util 6273 :ok :read [21 20] INFO [2019-12-17 12:33:38,600] jepsen worker 63 - jepsen.util 6273 :invoke :write [21 16] INFO [2019-12-17 12:33:38,614] jepsen worker 5 - jepsen.util 6935 :ok :write [1 42] INFO [2019-12-17 12:33:38,614] jepsen worker 5 - jepsen.util 6935 :invoke :read [1 nil] INFO [2019-12-17 12:33:38,629] jepsen worker 69 - jepsen.util 7449 :info :write [23 34] :RequestCanceledException INFO [2019-12-17 12:33:38,631] jepsen worker 69 - jepsen.util 7539 :invoke :write [23 42] INFO [2019-12-17 12:33:38,670] jepsen worker 32 - jepsen.util 7412 :ok :read [10 15] INFO [2019-12-17 12:33:38,670] jepsen worker 32 - jepsen.util 7412 :invoke :cas [10 [19 20]] INFO [2019-12-17 12:33:38,682] jepsen worker 70 - jepsen.util 7720 :ok :write [23 24] INFO [2019-12-17 12:33:38,682] jepsen worker 70 - jepsen.util 7720 :invoke :write [23 2] INFO [2019-12-17 12:33:38,692] jepsen worker 30 - jepsen.util 7230 :fail :cas [10 [29 25]] :ValueNotSwapFrom INFO [2019-12-17 12:33:38,692] jepsen worker 30 - jepsen.util 7230 :invoke :cas [10 [2 26]] INFO [2019-12-17 12:33:38,707] jepsen worker 46 - jepsen.util 7696 :ok :read [15 10] INFO [2019-12-17 12:33:38,707] jepsen worker 46 - jepsen.util 7696 :invoke :cas [15 [20 21]] INFO [2019-12-17 12:33:38,711] jepsen worker 88 - jepsen.util 7288 :info :write [29 8] :RequestCanceledException INFO [2019-12-17 12:33:38,712] jepsen worker 88 - jepsen.util 7378 :invoke :read [29 nil] INFO [2019-12-17 12:33:38,720] jepsen worker 32 - jepsen.util 7412 :fail :cas [10 [19 20]] :ValueNotSwapFrom INFO [2019-12-17 12:33:38,720] jepsen worker 32 - jepsen.util 7412 :invoke :write [10 31] INFO [2019-12-17 12:33:38,720] jepsen worker 9 - jepsen.util 7839 :ok :write [3 23] INFO [2019-12-17 12:33:38,721] jepsen worker 9 - jepsen.util 7839 :invoke :read [3 nil] INFO [2019-12-17 12:33:38,726] jepsen worker 46 - jepsen.util 7696 :fail :cas [15 [20 21]] :ValueNotSwapFrom INFO [2019-12-17 12:33:38,726] jepsen worker 46 - jepsen.util 7696 :invoke :write [15 1] INFO [2019-12-17 12:33:38,727] jepsen worker 79 - jepsen.util 7369 :info :write [26 6] :RequestCanceledException INFO [2019-12-17 12:33:38,727] jepsen worker 12 - jepsen.util 7842 :info :write [4 22] :RequestCanceledException INFO [2019-12-17 12:33:38,728] jepsen worker 79 - jepsen.util 7459 :invoke :read [26 nil] INFO [2019-12-17 12:33:38,729] jepsen worker 12 - jepsen.util 7932 :invoke :cas [4 [18 18]] INFO [2019-12-17 12:33:38,735] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42551,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:33:38,735] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42551,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:33:38,735] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42555,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:33:38,735] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42555,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:33:38,742] jepsen worker 27 - jepsen.util 7317 :info :cas [9 [34 16]] :RequestCanceledException INFO [2019-12-17 12:33:38,742] jepsen worker 45 - jepsen.util 7245 :fail :read [15 nil] :RequestTimeoutException INFO [2019-12-17 12:33:38,743] jepsen worker 19 - jepsen.util 7399 :info :cas [6 [46 34]] :RequestCanceledException INFO [2019-12-17 12:33:38,743] jepsen worker 45 - jepsen.util 7245 :invoke :cas [15 [44 5]] INFO [2019-12-17 12:33:38,744] jepsen worker 27 - jepsen.util 7407 :invoke :write [9 27] INFO [2019-12-17 12:33:38,744] jepsen worker 19 - jepsen.util 7489 :invoke :cas [6 [32 36]] INFO [2019-12-17 12:33:38,745] jepsen worker 47 - jepsen.util 7517 :info :write [15 33] :RequestCanceledException INFO [2019-12-17 12:33:38,748] jepsen worker 47 - jepsen.util 7607 :invoke :write [15 40] INFO [2019-12-17 12:33:38,759] jepsen worker 64 - jepsen.util 5644 :info :cas [21 [30 25]] :RequestCanceledException INFO [2019-12-17 12:33:38,760] jepsen worker 64 - jepsen.util 5734 :invoke :read [21 nil] INFO [2019-12-17 12:33:38,771] jepsen worker 37 - jepsen.util 6967 :fail :cas [12 [38 6]] :ValueNotSwapFrom INFO [2019-12-17 12:33:38,772] jepsen worker 37 - jepsen.util 6967 :invoke :write [12 30] INFO [2019-12-17 12:33:38,788] jepsen worker 4 - jepsen.util 6754 :ok :write [1 9] INFO [2019-12-17 12:33:38,788] jepsen worker 4 - jepsen.util 6754 :invoke :write [1 36] INFO [2019-12-17 12:33:38,795] jepsen worker 36 - jepsen.util 7686 :info :write [12 36] :RequestCanceledException INFO [2019-12-17 12:33:38,797] jepsen worker 36 - jepsen.util 7776 :invoke :write [12 32] INFO [2019-12-17 12:33:38,825] jepsen worker 71 - jepsen.util 8171 :fail :cas [23 [3 25]] :ValueNotSwapFrom INFO [2019-12-17 12:33:38,825] jepsen worker 71 - jepsen.util 8171 :invoke :cas [23 [48 13]] INFO [2019-12-17 12:33:38,838] jepsen worker 74 - jepsen.util 7544 :ok :read [24 8] INFO [2019-12-17 12:33:38,838] jepsen worker 74 - jepsen.util 7544 :invoke :write [24 7] INFO [2019-12-17 12:33:38,842] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:33:38,843] jepsen nemesis - couchbase.util Rebalance check count: 326 INFO [2019-12-17 12:33:38,843] jepsen nemesis - couchbase.util Rebalance stuck count: 307 INFO [2019-12-17 12:33:38,853] jepsen worker 13 - jepsen.util 6943 :fail :cas [4 [43 42]] :ValueNotSwapFrom INFO [2019-12-17 12:33:38,853] jepsen worker 13 - jepsen.util 6943 :invoke :write [4 1] INFO [2019-12-17 12:33:38,885] jepsen worker 18 - jepsen.util 7488 :fail :cas [6 [29 30]] :ValueNotSwapFrom INFO [2019-12-17 12:33:38,885] jepsen worker 18 - jepsen.util 7488 :invoke :write [6 35] INFO [2019-12-17 12:33:38,891] jepsen worker 45 - jepsen.util 7245 :fail :cas [15 [44 5]] :ValueNotSwapFrom INFO [2019-12-17 12:33:38,891] jepsen worker 45 - jepsen.util 7245 :invoke :cas [15 [12 35]] INFO [2019-12-17 12:33:38,901] jepsen worker 87 - jepsen.util 6657 :fail :cas [29 [8 46]] :ValueNotSwapFrom INFO [2019-12-17 12:33:38,902] jepsen worker 87 - jepsen.util 6657 :invoke :write [29 26] INFO [2019-12-17 12:33:38,928] jepsen worker 2 - jepsen.util 9632 :info :write [0 31] :RequestCanceledException INFO [2019-12-17 12:33:38,929] jepsen worker 2 - jepsen.util 9722 :invoke :cas [0 [37 13]] INFO [2019-12-17 12:33:38,942] jepsen worker 65 - jepsen.util 6095 :fail :read [21 nil] :RequestTimeoutException INFO [2019-12-17 12:33:38,942] jepsen worker 65 - jepsen.util 6095 :invoke :cas [21 [7 47]] INFO [2019-12-17 12:33:38,950] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42578,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:33:38,950] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42578,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:33:38,950] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42581,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:33:38,950] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42581,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:33:38,950] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42582,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:33:38,950] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42582,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:33:38,950] jepsen worker 3 - jepsen.util 7473 :ok :write [1 36] INFO [2019-12-17 12:33:38,950] jepsen worker 3 - jepsen.util 7473 :invoke :cas [1 [20 8]] INFO [2019-12-17 12:33:38,971] jepsen worker 30 - jepsen.util 7230 :fail :cas [10 [2 26]] :ValueNotSwapFrom INFO [2019-12-17 12:33:38,971] jepsen worker 30 - jepsen.util 7230 :invoke :cas [10 [33 20]] INFO [2019-12-17 12:33:38,978] jepsen worker 73 - jepsen.util 7093 :ok :read [24 8] INFO [2019-12-17 12:33:38,978] jepsen worker 73 - jepsen.util 7093 :invoke :cas [24 [13 10]] INFO [2019-12-17 12:33:38,988] jepsen worker 78 - jepsen.util 7548 :fail :cas [26 [8 46]] :ValueNotSwapFrom INFO [2019-12-17 12:33:38,988] jepsen worker 78 - jepsen.util 7548 :invoke :cas [26 [17 2]] INFO [2019-12-17 12:33:39,003] jepsen worker 64 - jepsen.util 5734 :ok :read [21 20] INFO [2019-12-17 12:33:39,003] jepsen worker 64 - jepsen.util 5734 :invoke :read [21 nil] INFO [2019-12-17 12:33:39,028] jepsen worker 14 - jepsen.util 7574 :info :write [4 14] :RequestCanceledException INFO [2019-12-17 12:33:39,030] jepsen worker 14 - jepsen.util 7664 :invoke :cas [4 [43 12]] INFO [2019-12-17 12:33:39,033] jepsen worker 63 - jepsen.util 6273 :ok :write [21 16] INFO [2019-12-17 12:33:39,033] jepsen worker 63 - jepsen.util 6273 :invoke :cas [21 [49 10]] INFO [2019-12-17 12:33:39,036] jepsen worker 9 - jepsen.util 7839 :ok :read [3 23] INFO [2019-12-17 12:33:39,036] jepsen worker 9 - jepsen.util 7839 :invoke :cas [3 [23 30]] INFO [2019-12-17 12:33:39,045] jepsen worker 5 - jepsen.util 6935 :ok :read [1 36] INFO [2019-12-17 12:33:39,045] jepsen worker 5 - jepsen.util 6935 :invoke :read [1 nil] INFO [2019-12-17 12:33:39,056] jepsen worker 19 - jepsen.util 7489 :fail :cas [6 [32 36]] :ValueNotSwapFrom INFO [2019-12-17 12:33:39,056] jepsen worker 19 - jepsen.util 7489 :invoke :write [6 46] INFO [2019-12-17 12:33:39,059] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42593,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:33:39,059] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42593,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:33:39,060] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42596,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:33:39,060] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42596,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:33:39,060] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42597,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:33:39,060] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42597,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:33:39,066] jepsen worker 80 - jepsen.util 7460 :ok :write [26 16] INFO [2019-12-17 12:33:39,066] jepsen worker 80 - jepsen.util 7460 :invoke :write [26 20] INFO [2019-12-17 12:33:39,091] jepsen worker 63 - jepsen.util 6273 :fail :cas [21 [49 10]] :ValueNotSwapFrom INFO [2019-12-17 12:33:39,091] jepsen worker 63 - jepsen.util 6273 :invoke :write [21 18] INFO [2019-12-17 12:33:39,095] jepsen worker 20 - jepsen.util 7040 :info :write [6 13] :RequestCanceledException INFO [2019-12-17 12:33:39,096] jepsen worker 20 - jepsen.util 7130 :invoke :write [6 48] INFO [2019-12-17 12:33:39,112] jepsen worker 56 - jepsen.util 8156 :info :write [18 26] :RequestCanceledException INFO [2019-12-17 12:33:39,114] jepsen worker 56 - jepsen.util 8246 :invoke :cas [18 [38 26]] INFO [2019-12-17 12:33:39,114] jepsen worker 27 - jepsen.util 7407 :ok :write [9 27] INFO [2019-12-17 12:33:39,114] jepsen worker 27 - jepsen.util 7407 :invoke :cas [9 [10 0]] INFO [2019-12-17 12:33:39,119] jepsen worker 55 - jepsen.util 8335 :ok :write [18 27] INFO [2019-12-17 12:33:39,119] jepsen worker 55 - jepsen.util 8335 :invoke :cas [18 [11 36]] INFO [2019-12-17 12:33:39,135] jepsen worker 73 - jepsen.util 7093 :fail :cas [24 [13 10]] :ValueNotSwapFrom INFO [2019-12-17 12:33:39,136] jepsen worker 73 - jepsen.util 7093 :invoke :write [24 33] INFO [2019-12-17 12:33:39,176] jepsen worker 27 - jepsen.util 7407 :fail :cas [9 [10 0]] :ValueNotSwapFrom INFO [2019-12-17 12:33:39,176] jepsen worker 27 - jepsen.util 7407 :invoke :read [9 nil] INFO [2019-12-17 12:33:39,187] jepsen worker 88 - jepsen.util 7378 :ok :read [29 32] INFO [2019-12-17 12:33:39,187] jepsen worker 88 - jepsen.util 7378 :invoke :cas [29 [2 30]] INFO [2019-12-17 12:33:39,195] jepsen worker 53 - jepsen.util 6893 :info :write [17 27] :RequestCanceledException INFO [2019-12-17 12:33:39,196] jepsen worker 53 - jepsen.util 6983 :invoke :cas [17 [32 42]] INFO [2019-12-17 12:33:39,204] jepsen worker 32 - jepsen.util 7412 :ok :write [10 31] INFO [2019-12-17 12:33:39,204] jepsen worker 32 - jepsen.util 7412 :invoke :read [10 nil] INFO [2019-12-17 12:33:39,209] jepsen worker 78 - jepsen.util 7548 :fail :cas [26 [17 2]] :ValueNotSwapFrom INFO [2019-12-17 12:33:39,209] jepsen worker 78 - jepsen.util 7548 :invoke :read [26 nil] INFO [2019-12-17 12:33:39,219] jepsen worker 27 - jepsen.util 7407 :ok :read [9 27] INFO [2019-12-17 12:33:39,219] jepsen worker 27 - jepsen.util 7407 :invoke :write [9 17] INFO [2019-12-17 12:33:39,219] jepsen worker 51 - jepsen.util 7161 :ok :write [17 30] INFO [2019-12-17 12:33:39,220] jepsen worker 51 - jepsen.util 7161 :invoke :cas [17 [38 41]] INFO [2019-12-17 12:33:39,221] jepsen worker 70 - jepsen.util 7720 :ok :write [23 2] INFO [2019-12-17 12:33:39,222] jepsen worker 70 - jepsen.util 7720 :invoke :cas [23 [26 30]] INFO [2019-12-17 12:33:39,228] jepsen worker 10 - jepsen.util 7390 :fail :read [3 nil] :RequestTimeoutException INFO [2019-12-17 12:33:39,228] jepsen worker 10 - jepsen.util 7390 :invoke :read [3 nil] INFO [2019-12-17 12:33:39,236] jepsen worker 32 - jepsen.util 7412 :ok :read [10 31] INFO [2019-12-17 12:33:39,236] jepsen worker 32 - jepsen.util 7412 :invoke :write [10 21] INFO [2019-12-17 12:33:39,241] jepsen worker 74 - jepsen.util 7544 :ok :write [24 7] INFO [2019-12-17 12:33:39,242] jepsen worker 74 - jepsen.util 7544 :invoke :write [24 17] INFO [2019-12-17 12:33:39,242] jepsen worker 38 - jepsen.util 7598 :fail :cas [12 [28 24]] :ValueNotSwapFrom INFO [2019-12-17 12:33:39,242] jepsen worker 38 - jepsen.util 7598 :invoke :read [12 nil] INFO [2019-12-17 12:33:39,244] jepsen worker 10 - jepsen.util 7390 :ok :read [3 23] INFO [2019-12-17 12:33:39,244] jepsen worker 10 - jepsen.util 7390 :invoke :read [3 nil] INFO [2019-12-17 12:33:39,257] jepsen worker 89 - jepsen.util 7199 :ok :read [29 32] INFO [2019-12-17 12:33:39,258] jepsen worker 89 - jepsen.util 7199 :invoke :cas [29 [12 16]] INFO [2019-12-17 12:33:39,261] jepsen worker 5 - jepsen.util 6935 :ok :read [1 36] INFO [2019-12-17 12:33:39,261] jepsen worker 5 - jepsen.util 6935 :invoke :write [1 39] INFO [2019-12-17 12:33:39,276] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42614,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:33:39,276] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42614,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:33:39,276] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42615,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:33:39,276] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42615,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:33:39,287] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42626,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:33:39,287] jepsen worker 28 - jepsen.util 5698 :info :write [9 31] :RequestCanceledException INFO [2019-12-17 12:33:39,287] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42626,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:33:39,287] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42628,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:33:39,287] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42628,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:33:39,288] jepsen worker 28 - jepsen.util 5788 :invoke :cas [9 [20 15]] INFO [2019-12-17 12:33:39,299] jepsen worker 30 - jepsen.util 7230 :fail :cas [10 [33 20]] :ValueNotSwapFrom INFO [2019-12-17 12:33:39,299] jepsen worker 30 - jepsen.util 7230 :invoke :write [10 46] INFO [2019-12-17 12:33:39,311] jepsen worker 56 - jepsen.util 8246 :fail :cas [18 [38 26]] :ValueNotSwapFrom INFO [2019-12-17 12:33:39,312] jepsen worker 56 - jepsen.util 8246 :invoke :write [18 21] INFO [2019-12-17 12:33:39,314] jepsen worker 64 - jepsen.util 5734 :ok :read [21 16] INFO [2019-12-17 12:33:39,314] jepsen worker 64 - jepsen.util 5734 :invoke :write [21 15] INFO [2019-12-17 12:33:39,315] jepsen worker 19 - jepsen.util 7489 :ok :write [6 46] INFO [2019-12-17 12:33:39,315] jepsen worker 19 - jepsen.util 7489 :invoke :cas [6 [47 1]] INFO [2019-12-17 12:33:39,322] jepsen worker 79 - jepsen.util 7459 :ok :read [26 16] INFO [2019-12-17 12:33:39,322] jepsen worker 79 - jepsen.util 7459 :invoke :read [26 nil] INFO [2019-12-17 12:33:39,328] jepsen worker 29 - jepsen.util 5699 :info :write [9 42] :RequestCanceledException INFO [2019-12-17 12:33:39,332] jepsen worker 29 - jepsen.util 5789 :invoke :write [9 21] INFO [2019-12-17 12:33:39,359] jepsen worker 52 - jepsen.util 7612 :info :write [17 39] :RequestCanceledException INFO [2019-12-17 12:33:39,361] jepsen worker 52 - jepsen.util 7702 :invoke :write [17 30] INFO [2019-12-17 12:33:39,404] jepsen worker 63 - jepsen.util 6273 :ok :write [21 18] INFO [2019-12-17 12:33:39,405] jepsen worker 63 - jepsen.util 6273 :invoke :write [21 36] INFO [2019-12-17 12:33:39,407] jepsen worker 27 - jepsen.util 7407 :ok :write [9 17] INFO [2019-12-17 12:33:39,407] jepsen worker 27 - jepsen.util 7407 :invoke :cas [9 [48 2]] INFO [2019-12-17 12:33:39,455] jepsen worker 12 - jepsen.util 7932 :fail :cas [4 [18 18]] :ValueNotSwapFrom INFO [2019-12-17 12:33:39,455] jepsen worker 12 - jepsen.util 7932 :invoke :cas [4 [14 27]] INFO [2019-12-17 12:33:39,456] jepsen worker 3 - jepsen.util 7473 :fail :cas [1 [20 8]] :ValueNotSwapFrom INFO [2019-12-17 12:33:39,456] jepsen worker 3 - jepsen.util 7473 :invoke :read [1 nil] INFO [2019-12-17 12:33:39,459] jepsen worker 88 - jepsen.util 7378 :fail :cas [29 [2 30]] :ValueNotSwapFrom INFO [2019-12-17 12:33:39,459] jepsen worker 88 - jepsen.util 7378 :invoke :read [29 nil] INFO [2019-12-17 12:33:39,461] jepsen worker 27 - jepsen.util 7407 :fail :cas [9 [48 2]] :ValueNotSwapFrom INFO [2019-12-17 12:33:39,462] jepsen worker 27 - jepsen.util 7407 :invoke :read [9 nil] INFO [2019-12-17 12:33:39,479] jepsen worker 19 - jepsen.util 7489 :fail :cas [6 [47 1]] :ValueNotSwapFrom INFO [2019-12-17 12:33:39,479] jepsen worker 19 - jepsen.util 7489 :invoke :cas [6 [35 22]] INFO [2019-12-17 12:33:39,495] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42650,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:33:39,495] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42650,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:33:39,495] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42653,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:33:39,495] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42653,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:33:39,495] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42654,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:33:39,495] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42654,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:33:39,495] jepsen worker 27 - jepsen.util 7407 :ok :read [9 17] INFO [2019-12-17 12:33:39,495] jepsen worker 27 - jepsen.util 7407 :invoke :cas [9 [31 30]] INFO [2019-12-17 12:33:39,504] jepsen worker 53 - jepsen.util 6983 :fail :cas [17 [32 42]] :ValueNotSwapFrom INFO [2019-12-17 12:33:39,504] jepsen worker 53 - jepsen.util 6983 :invoke :write [17 23] INFO [2019-12-17 12:33:39,526] jepsen worker 27 - jepsen.util 7407 :fail :cas [9 [31 30]] :ValueNotSwapFrom INFO [2019-12-17 12:33:39,527] jepsen worker 27 - jepsen.util 7407 :invoke :cas [9 [39 33]] INFO [2019-12-17 12:33:39,531] jepsen worker 55 - jepsen.util 8335 :fail :cas [18 [11 36]] :ValueNotSwapFrom INFO [2019-12-17 12:33:39,532] jepsen worker 55 - jepsen.util 8335 :invoke :read [18 nil] INFO [2019-12-17 12:33:39,540] jepsen worker 89 - jepsen.util 7199 :fail :cas [29 [12 16]] :ValueNotSwapFrom INFO [2019-12-17 12:33:39,540] jepsen worker 89 - jepsen.util 7199 :invoke :read [29 nil] INFO [2019-12-17 12:33:39,550] jepsen worker 29 - jepsen.util 5789 :ok :write [9 21] INFO [2019-12-17 12:33:39,550] jepsen worker 29 - jepsen.util 5789 :invoke :read [9 nil] INFO [2019-12-17 12:33:39,551] jepsen worker 27 - jepsen.util 7407 :fail :cas [9 [39 33]] :ValueNotSwapFrom INFO [2019-12-17 12:33:39,551] jepsen worker 27 - jepsen.util 7407 :invoke :cas [9 [6 40]] INFO [2019-12-17 12:33:39,573] jepsen worker 63 - jepsen.util 6273 :ok :write [21 36] INFO [2019-12-17 12:33:39,573] jepsen worker 63 - jepsen.util 6273 :invoke :read [21 nil] INFO [2019-12-17 12:33:39,627] jepsen worker 29 - jepsen.util 5789 :ok :read [9 21] INFO [2019-12-17 12:33:39,628] jepsen worker 29 - jepsen.util 5789 :invoke :cas [9 [27 25]] INFO [2019-12-17 12:33:39,628] jepsen worker 88 - jepsen.util 7378 :ok :read [29 32] INFO [2019-12-17 12:33:39,628] jepsen worker 88 - jepsen.util 7378 :invoke :cas [29 [29 11]] INFO [2019-12-17 12:33:39,634] jepsen worker 38 - jepsen.util 7598 :ok :read [12 30] INFO [2019-12-17 12:33:39,634] jepsen worker 38 - jepsen.util 7598 :invoke :write [12 18] INFO [2019-12-17 12:33:39,645] jepsen worker 12 - jepsen.util 7932 :fail :cas [4 [14 27]] :ValueNotSwapFrom INFO [2019-12-17 12:33:39,645] jepsen worker 12 - jepsen.util 7932 :invoke :write [4 34] INFO [2019-12-17 12:33:39,664] jepsen worker 27 - jepsen.util 7407 :fail :cas [9 [6 40]] :ValueNotSwapFrom INFO [2019-12-17 12:33:39,665] jepsen worker 27 - jepsen.util 7407 :invoke :write [9 2] INFO [2019-12-17 12:33:39,682] jepsen worker 19 - jepsen.util 7489 :fail :cas [6 [35 22]] :ValueNotSwapFrom INFO [2019-12-17 12:33:39,682] jepsen worker 19 - jepsen.util 7489 :invoke :write [6 33] INFO [2019-12-17 12:33:39,708] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42674,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:33:39,717] jepsen worker 29 - jepsen.util 5789 :fail :cas [9 [27 25]] :ValueNotSwapFrom INFO [2019-12-17 12:33:39,731] jepsen worker 89 - jepsen.util 7199 :ok :read [29 32] INFO [2019-12-17 12:33:39,750] jepsen worker 27 - jepsen.util 7407 :ok :write [9 2] INFO [2019-12-17 12:33:39,759] jepsen worker 34 - jepsen.util 9394 :info :cas [11 [20 26]] :RequestCanceledException INFO [2019-12-17 12:33:39,770] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42674,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:33:39,770] jepsen worker 89 - jepsen.util 7199 :invoke :cas [29 [15 30]] INFO [2019-12-17 12:33:39,770] jepsen worker 29 - jepsen.util 5789 :invoke :write [9 34] INFO [2019-12-17 12:33:39,770] jepsen worker 27 - jepsen.util 7407 :invoke :read [9 nil] INFO [2019-12-17 12:33:39,771] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42676,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:33:39,771] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42676,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:33:39,771] jepsen worker 34 - jepsen.util 9484 :invoke :cas [11 [10 6]] INFO [2019-12-17 12:33:39,790] jepsen worker 42 - jepsen.util 10662 :info :write [14 39] :RequestCanceledException INFO [2019-12-17 12:33:39,790] jepsen worker 43 - jepsen.util 9493 :info :cas [14 [37 48]] :RequestCanceledException INFO [2019-12-17 12:33:39,791] jepsen worker 42 - jepsen.util 10752 :invoke :read [14 nil] INFO [2019-12-17 12:33:39,791] jepsen worker 43 - jepsen.util 9583 :invoke :cas [14 [2 8]] INFO [2019-12-17 12:33:39,809] jepsen worker 27 - jepsen.util 7407 :ok :read [9 2] INFO [2019-12-17 12:33:39,809] jepsen worker 27 - jepsen.util 7407 :invoke :read [9 nil] INFO [2019-12-17 12:33:39,810] jepsen worker 83 - jepsen.util 10433 :info :cas [27 [29 2]] :RequestCanceledException INFO [2019-12-17 12:33:39,810] jepsen worker 21 - jepsen.util 9471 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:33:39,810] jepsen worker 21 - jepsen.util 9471 :invoke :cas [7 [47 17]] INFO [2019-12-17 12:33:39,812] jepsen worker 83 - jepsen.util 10523 :invoke :read [27 nil] INFO [2019-12-17 12:33:39,814] jepsen worker 78 - jepsen.util 7548 :ok :read [26 16] INFO [2019-12-17 12:33:39,814] jepsen worker 78 - jepsen.util 7548 :invoke :write [26 44] INFO [2019-12-17 12:33:39,828] jepsen worker 75 - jepsen.util 9435 :info :write [25 11] :RequestCanceledException INFO [2019-12-17 12:33:39,828] jepsen worker 66 - jepsen.util 9876 :info :write [22 39] :RequestCanceledException INFO [2019-12-17 12:33:39,829] jepsen worker 66 - jepsen.util 9966 :invoke :read [22 nil] INFO [2019-12-17 12:33:39,829] jepsen worker 75 - jepsen.util 9525 :invoke :write [25 7] INFO [2019-12-17 12:33:39,831] jepsen worker 89 - jepsen.util 7199 :fail :cas [29 [15 30]] :ValueNotSwapFrom INFO [2019-12-17 12:33:39,832] jepsen worker 89 - jepsen.util 7199 :invoke :write [29 43] INFO [2019-12-17 12:33:39,839] jepsen worker 27 - jepsen.util 7407 :ok :read [9 2] INFO [2019-12-17 12:33:39,839] jepsen worker 27 - jepsen.util 7407 :invoke :write [9 11] INFO [2019-12-17 12:33:39,842] jepsen worker 26 - jepsen.util 10286 :info :write [8 43] :RequestCanceledException INFO [2019-12-17 12:33:39,842] jepsen worker 62 - jepsen.util 10052 :info :cas [20 [39 32]] :RequestCanceledException INFO [2019-12-17 12:33:39,842] jepsen worker 23 - jepsen.util 10103 :info :write [7 13] :RequestCanceledException INFO [2019-12-17 12:33:39,843] jepsen worker 23 - jepsen.util 10193 :invoke :write [7 43] INFO [2019-12-17 12:33:39,843] jepsen worker 62 - jepsen.util 10142 :invoke :cas [20 [43 27]] INFO [2019-12-17 12:33:39,843] jepsen worker 26 - jepsen.util 10376 :invoke :cas [8 [20 32]] INFO [2019-12-17 12:33:39,859] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:33:39,859] jepsen nemesis - couchbase.util Rebalance check count: 327 INFO [2019-12-17 12:33:39,859] jepsen nemesis - couchbase.util Rebalance stuck count: 308 INFO [2019-12-17 12:33:39,859] jepsen worker 22 - jepsen.util 9832 :info :cas [7 [21 2]] :RequestCanceledException INFO [2019-12-17 12:33:39,860] jepsen worker 22 - jepsen.util 9922 :invoke :read [7 nil] INFO [2019-12-17 12:33:39,877] jepsen worker 29 - jepsen.util 5789 :ok :write [9 34] INFO [2019-12-17 12:33:39,878] jepsen worker 29 - jepsen.util 5789 :invoke :cas [9 [2 20]] INFO [2019-12-17 12:33:39,893] jepsen worker 29 - jepsen.util 5789 :fail :cas [9 [2 20]] :ValueNotSwapFrom INFO [2019-12-17 12:33:39,893] jepsen worker 29 - jepsen.util 5789 :invoke :write [9 46] INFO [2019-12-17 12:33:39,895] jepsen worker 67 - jepsen.util 9967 :info :write [22 8] :RequestCanceledException INFO [2019-12-17 12:33:39,896] jepsen worker 67 - jepsen.util 10057 :invoke :read [22 nil] INFO [2019-12-17 12:33:39,913] jepsen worker 71 - jepsen.util 8171 :fail :cas [23 [48 13]] :ValueNotSwapFrom INFO [2019-12-17 12:33:39,913] jepsen worker 86 - jepsen.util 10526 :info :write [28 40] :RequestCanceledException INFO [2019-12-17 12:33:39,913] jepsen worker 68 - jepsen.util 9428 :info :write [22 4] :RequestCanceledException INFO [2019-12-17 12:33:39,913] jepsen worker 71 - jepsen.util 8171 :invoke :read [23 nil] INFO [2019-12-17 12:33:39,914] jepsen worker 86 - jepsen.util 10616 :invoke :write [28 46] INFO [2019-12-17 12:33:39,916] jepsen worker 68 - jepsen.util 9518 :invoke :write [22 8] INFO [2019-12-17 12:33:39,927] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42698,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:33:39,928] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42698,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:33:39,928] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42701,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:33:39,928] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42701,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:33:39,928] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42702,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:33:39,928] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42702,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:33:39,928] jepsen worker 0 - jepsen.util 10440 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:33:39,928] jepsen worker 58 - jepsen.util 10768 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:33:39,928] jepsen worker 60 - jepsen.util 10860 :info :write [20 16] :RequestCanceledException INFO [2019-12-17 12:33:39,928] jepsen worker 54 - jepsen.util 8244 :info :write [18 22] :RequestCanceledException INFO [2019-12-17 12:33:39,928] jepsen worker 41 - jepsen.util 10481 :info :cas [13 [24 7]] :RequestCanceledException INFO [2019-12-17 12:33:39,928] jepsen worker 0 - jepsen.util 10440 :invoke :cas [0 [13 14]] INFO [2019-12-17 12:33:39,928] jepsen worker 58 - jepsen.util 10768 :invoke :write [19 13] INFO [2019-12-17 12:33:39,929] jepsen worker 60 - jepsen.util 10950 :invoke :write [20 43] INFO [2019-12-17 12:33:39,929] jepsen worker 41 - jepsen.util 10571 :invoke :read [13 nil] INFO [2019-12-17 12:33:39,929] jepsen worker 54 - jepsen.util 8334 :invoke :cas [18 [33 14]] INFO [2019-12-17 12:33:39,939] jepsen worker 27 - jepsen.util 7407 :ok :write [9 11] INFO [2019-12-17 12:33:39,939] jepsen worker 27 - jepsen.util 7407 :invoke :write [9 6] INFO [2019-12-17 12:33:39,942] jepsen worker 63 - jepsen.util 6273 :ok :read [21 36] INFO [2019-12-17 12:33:39,942] jepsen worker 63 - jepsen.util 6273 :invoke :read [21 nil] INFO [2019-12-17 12:33:39,942] jepsen worker 77 - jepsen.util 10157 :info :write [25 15] :RequestCanceledException INFO [2019-12-17 12:33:39,944] jepsen worker 77 - jepsen.util 10247 :invoke :cas [25 [8 16]] INFO [2019-12-17 12:33:39,956] jepsen worker 25 - jepsen.util 10555 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:33:39,956] jepsen worker 59 - jepsen.util 9779 :info :cas [19 [38 0]] :RequestCanceledException INFO [2019-12-17 12:33:39,956] jepsen worker 25 - jepsen.util 10555 :invoke :read [8 nil] INFO [2019-12-17 12:33:39,957] jepsen worker 59 - jepsen.util 9869 :invoke :cas [19 [35 13]] INFO [2019-12-17 12:33:39,975] jepsen worker 29 - jepsen.util 5789 :ok :write [9 46] INFO [2019-12-17 12:33:39,975] jepsen worker 29 - jepsen.util 5789 :invoke :cas [9 [10 27]] INFO [2019-12-17 12:33:39,993] jepsen worker 29 - jepsen.util 5789 :fail :cas [9 [10 27]] :ValueNotSwapFrom INFO [2019-12-17 12:33:39,993] jepsen worker 29 - jepsen.util 5789 :invoke :write [9 3] INFO [2019-12-17 12:33:39,995] jepsen worker 15 - jepsen.util 8925 :info :cas [5 [33 29]] :RequestCanceledException INFO [2019-12-17 12:33:39,996] jepsen worker 15 - jepsen.util 9015 :invoke :cas [5 [2 45]] INFO [2019-12-17 12:33:40,007] jepsen worker 14 - jepsen.util 7664 :fail :cas [4 [43 12]] :ValueNotSwapFrom INFO [2019-12-17 12:33:40,007] jepsen worker 14 - jepsen.util 7664 :invoke :cas [4 [38 25]] INFO [2019-12-17 12:33:40,013] jepsen worker 50 - jepsen.util 10670 :info :cas [16 [26 21]] :RequestCanceledException INFO [2019-12-17 12:33:40,014] jepsen worker 50 - jepsen.util 10760 :invoke :write [16 10] INFO [2019-12-17 12:33:40,029] jepsen worker 33 - jepsen.util 9753 :info :write [11 49] :RequestCanceledException INFO [2019-12-17 12:33:40,029] jepsen worker 82 - jepsen.util 10612 :info :write [27 24] :RequestCanceledException INFO [2019-12-17 12:33:40,030] jepsen worker 33 - jepsen.util 9843 :invoke :cas [11 [27 49]] INFO [2019-12-17 12:33:40,030] jepsen worker 82 - jepsen.util 10702 :invoke :read [27 nil] INFO [2019-12-17 12:33:40,035] jepsen worker 27 - jepsen.util 7407 :ok :write [9 6] INFO [2019-12-17 12:33:40,035] jepsen worker 27 - jepsen.util 7407 :invoke :read [9 nil] INFO [2019-12-17 12:33:40,037] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42723,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:33:40,038] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42723,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:33:40,042] jepsen worker 24 - jepsen.util 8664 :info :write [8 28] :RequestCanceledException INFO [2019-12-17 12:33:40,043] jepsen worker 24 - jepsen.util 8754 :invoke :write [8 6] INFO [2019-12-17 12:33:40,050] jepsen worker 27 - jepsen.util 7407 :ok :read [9 6] INFO [2019-12-17 12:33:40,050] jepsen worker 27 - jepsen.util 7407 :invoke :cas [9 [16 25]] INFO [2019-12-17 12:33:40,076] jepsen worker 27 - jepsen.util 7407 :fail :cas [9 [16 25]] :ValueNotSwapFrom INFO [2019-12-17 12:33:40,076] jepsen worker 27 - jepsen.util 7407 :invoke :cas [9 [7 44]] INFO [2019-12-17 12:33:40,077] jepsen worker 3 - jepsen.util 7473 :ok :read [1 36] INFO [2019-12-17 12:33:40,077] jepsen worker 3 - jepsen.util 7473 :invoke :read [1 nil] INFO [2019-12-17 12:33:40,092] jepsen worker 29 - jepsen.util 5789 :ok :write [9 3] INFO [2019-12-17 12:33:40,092] jepsen worker 29 - jepsen.util 5789 :invoke :read [9 nil] INFO [2019-12-17 12:33:40,094] jepsen worker 51 - jepsen.util 7161 :fail :cas [17 [38 41]] :ValueNotSwapFrom INFO [2019-12-17 12:33:40,094] jepsen worker 51 - jepsen.util 7161 :invoke :write [17 11] INFO [2019-12-17 12:33:40,099] jepsen worker 79 - jepsen.util 7459 :ok :read [26 16] INFO [2019-12-17 12:33:40,099] jepsen worker 79 - jepsen.util 7459 :invoke :read [26 nil] INFO [2019-12-17 12:33:40,105] jepsen worker 27 - jepsen.util 7407 :fail :cas [9 [7 44]] :ValueNotSwapFrom INFO [2019-12-17 12:33:40,105] jepsen worker 27 - jepsen.util 7407 :invoke :write [9 27] INFO [2019-12-17 12:33:40,113] jepsen worker 29 - jepsen.util 5789 :ok :read [9 3] INFO [2019-12-17 12:33:40,113] jepsen worker 29 - jepsen.util 5789 :invoke :cas [9 [0 0]] INFO [2019-12-17 12:33:40,123] jepsen worker 70 - jepsen.util 7720 :fail :cas [23 [26 30]] :ValueNotSwapFrom INFO [2019-12-17 12:33:40,123] jepsen worker 70 - jepsen.util 7720 :invoke :read [23 nil] INFO [2019-12-17 12:33:40,128] jepsen worker 1 - jepsen.util 9991 :info :write [0 27] :RequestCanceledException INFO [2019-12-17 12:33:40,129] jepsen worker 1 - jepsen.util 10081 :invoke :cas [0 [7 43]] INFO [2019-12-17 12:33:40,136] jepsen worker 88 - jepsen.util 7378 :fail :cas [29 [29 11]] :ValueNotSwapFrom INFO [2019-12-17 12:33:40,137] jepsen worker 88 - jepsen.util 7378 :invoke :cas [29 [13 15]] INFO [2019-12-17 12:33:40,144] jepsen worker 29 - jepsen.util 5789 :fail :cas [9 [0 0]] :ValueNotSwapFrom INFO [2019-12-17 12:33:40,144] jepsen worker 29 - jepsen.util 5789 :invoke :read [9 nil] INFO [2019-12-17 12:33:40,158] jepsen worker 29 - jepsen.util 5789 :ok :read [9 3] INFO [2019-12-17 12:33:40,159] jepsen worker 29 - jepsen.util 5789 :invoke :read [9 nil] INFO [2019-12-17 12:33:40,162] jepsen worker 55 - jepsen.util 8335 :ok :read [18 27] INFO [2019-12-17 12:33:40,162] jepsen worker 55 - jepsen.util 8335 :invoke :read [18 nil] INFO [2019-12-17 12:33:40,191] jepsen worker 29 - jepsen.util 5789 :ok :read [9 3] INFO [2019-12-17 12:33:40,191] jepsen worker 29 - jepsen.util 5789 :invoke :cas [9 [28 27]] INFO [2019-12-17 12:33:40,198] jepsen worker 27 - jepsen.util 7407 :ok :write [9 27] INFO [2019-12-17 12:33:40,198] jepsen worker 27 - jepsen.util 7407 :invoke :cas [9 [45 30]] INFO [2019-12-17 12:33:40,212] jepsen worker 6 - jepsen.util 10626 :info :write [2 4] :RequestCanceledException INFO [2019-12-17 12:33:40,213] jepsen worker 6 - jepsen.util 10716 :invoke :cas [2 [3 49]] INFO [2019-12-17 12:33:40,217] jepsen worker 29 - jepsen.util 5789 :fail :cas [9 [28 27]] :ValueNotSwapFrom INFO [2019-12-17 12:33:40,217] jepsen worker 29 - jepsen.util 5789 :invoke :cas [9 [19 19]] INFO [2019-12-17 12:33:40,234] jepsen worker 27 - jepsen.util 7407 :fail :cas [9 [45 30]] :ValueNotSwapFrom INFO [2019-12-17 12:33:40,234] jepsen worker 27 - jepsen.util 7407 :invoke :write [9 1] INFO [2019-12-17 12:33:40,242] jepsen worker 84 - jepsen.util 9714 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:33:40,242] jepsen worker 84 - jepsen.util 9714 :invoke :read [28 nil] INFO [2019-12-17 12:33:40,252] jepsen worker 29 - jepsen.util 5789 :fail :cas [9 [19 19]] :ValueNotSwapFrom INFO [2019-12-17 12:33:40,252] jepsen worker 29 - jepsen.util 5789 :invoke :read [9 nil] INFO [2019-12-17 12:33:40,252] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42750,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:33:40,252] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42750,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:33:40,252] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42752,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:33:40,252] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42752,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:33:40,259] jepsen worker 81 - jepsen.util 9891 :info :write [27 9] :RequestCanceledException INFO [2019-12-17 12:33:40,260] jepsen worker 81 - jepsen.util 9981 :invoke :read [27 nil] INFO [2019-12-17 12:33:40,268] jepsen worker 74 - jepsen.util 7544 :ok :write [24 17] INFO [2019-12-17 12:33:40,268] jepsen worker 19 - jepsen.util 7489 :ok :write [6 33] INFO [2019-12-17 12:33:40,268] jepsen worker 74 - jepsen.util 7544 :invoke :read [24 nil] INFO [2019-12-17 12:33:40,268] jepsen worker 19 - jepsen.util 7489 :invoke :read [6 nil] INFO [2019-12-17 12:33:40,272] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42761,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:33:40,272] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42761,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:33:40,272] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42762,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:33:40,272] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42762,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:33:40,272] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42763,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:33:40,272] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42763,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:33:40,285] jepsen worker 29 - jepsen.util 5789 :ok :read [9 27] INFO [2019-12-17 12:33:40,285] jepsen worker 29 - jepsen.util 5789 :invoke :write [9 0] INFO [2019-12-17 12:33:40,326] jepsen worker 32 - jepsen.util 7412 :ok :write [10 21] INFO [2019-12-17 12:33:40,326] jepsen worker 32 - jepsen.util 7412 :invoke :read [10 nil] INFO [2019-12-17 12:33:40,327] jepsen worker 47 - jepsen.util 7607 :ok :write [15 40] INFO [2019-12-17 12:33:40,327] jepsen worker 47 - jepsen.util 7607 :invoke :cas [15 [11 3]] INFO [2019-12-17 12:33:40,327] jepsen worker 76 - jepsen.util 10066 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:33:40,327] jepsen worker 72 - jepsen.util 7182 :info :cas [24 [45 48]] :RequestCanceledException INFO [2019-12-17 12:33:40,327] jepsen worker 57 - jepsen.util 9957 :info :cas [19 [44 17]] :RequestCanceledException INFO [2019-12-17 12:33:40,327] jepsen worker 31 - jepsen.util 6601 :fail :read [10 nil] :RequestTimeoutException INFO [2019-12-17 12:33:40,328] jepsen worker 76 - jepsen.util 10066 :invoke :write [25 12] INFO [2019-12-17 12:33:40,328] jepsen worker 31 - jepsen.util 6601 :invoke :cas [10 [16 34]] INFO [2019-12-17 12:33:40,328] jepsen worker 57 - jepsen.util 10047 :invoke :write [19 43] INFO [2019-12-17 12:33:40,329] jepsen worker 72 - jepsen.util 7272 :invoke :cas [24 [43 20]] INFO [2019-12-17 12:33:40,342] jepsen worker 11 - jepsen.util 7481 :info :write [3 43] :RequestCanceledException INFO [2019-12-17 12:33:40,342] jepsen worker 11 - jepsen.util 7571 :invoke :read [3 nil] INFO [2019-12-17 12:33:40,359] jepsen worker 17 - jepsen.util 10277 :info :cas [5 [6 2]] :RequestCanceledException INFO [2019-12-17 12:33:40,360] jepsen worker 17 - jepsen.util 10367 :invoke :read [5 nil] INFO [2019-12-17 12:33:40,369] jepsen worker 74 - jepsen.util 7544 :ok :read [24 17] INFO [2019-12-17 12:33:40,370] jepsen worker 74 - jepsen.util 7544 :invoke :write [24 13] INFO [2019-12-17 12:33:40,376] jepsen worker 55 - jepsen.util 8335 :ok :read [18 27] INFO [2019-12-17 12:33:40,376] jepsen worker 55 - jepsen.util 8335 :invoke :cas [18 [40 33]] INFO [2019-12-17 12:33:40,381] jepsen worker 88 - jepsen.util 7378 :fail :cas [29 [13 15]] :ValueNotSwapFrom INFO [2019-12-17 12:33:40,381] jepsen worker 88 - jepsen.util 7378 :invoke :write [29 28] INFO [2019-12-17 12:33:40,391] jepsen worker 32 - jepsen.util 7412 :ok :read [10 21] INFO [2019-12-17 12:33:40,391] jepsen worker 32 - jepsen.util 7412 :invoke :read [10 nil] INFO [2019-12-17 12:33:40,394] jepsen worker 63 - jepsen.util 6273 :ok :read [21 36] INFO [2019-12-17 12:33:40,394] jepsen worker 63 - jepsen.util 6273 :invoke :read [21 nil] INFO [2019-12-17 12:33:40,395] jepsen worker 39 - jepsen.util 9849 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:33:40,395] jepsen worker 48 - jepsen.util 9138 :info :write [16 24] :RequestCanceledException INFO [2019-12-17 12:33:40,395] jepsen worker 39 - jepsen.util 9849 :invoke :cas [13 [8 27]] INFO [2019-12-17 12:33:40,396] jepsen worker 48 - jepsen.util 9228 :invoke :read [16 nil] INFO [2019-12-17 12:33:40,456] jepsen worker 47 - jepsen.util 7607 :fail :cas [15 [11 3]] :ValueNotSwapFrom INFO [2019-12-17 12:33:40,456] jepsen worker 47 - jepsen.util 7607 :invoke :read [15 nil] INFO [2019-12-17 12:33:40,465] jepsen worker 11 - jepsen.util 7571 :ok :read [3 23] INFO [2019-12-17 12:33:40,465] jepsen worker 11 - jepsen.util 7571 :invoke :cas [3 [27 48]] INFO [2019-12-17 12:33:40,467] jepsen worker 70 - jepsen.util 7720 :ok :read [23 2] INFO [2019-12-17 12:33:40,467] jepsen worker 70 - jepsen.util 7720 :invoke :write [23 49] INFO [2019-12-17 12:33:40,472] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42788,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:33:40,472] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42788,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:33:40,472] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42792,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:33:40,472] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42792,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:33:40,474] jepsen worker 72 - jepsen.util 7272 :fail :cas [24 [43 20]] :ValueNotSwapFrom INFO [2019-12-17 12:33:40,474] jepsen worker 72 - jepsen.util 7272 :invoke :read [24 nil] INFO [2019-12-17 12:33:40,529] jepsen worker 32 - jepsen.util 7412 :ok :read [10 21] INFO [2019-12-17 12:33:40,530] jepsen worker 32 - jepsen.util 7412 :invoke :cas [10 [40 4]] INFO [2019-12-17 12:33:40,533] jepsen worker 47 - jepsen.util 7607 :ok :read [15 40] INFO [2019-12-17 12:33:40,533] jepsen worker 47 - jepsen.util 7607 :invoke :read [15 nil] INFO [2019-12-17 12:33:40,540] jepsen worker 54 - jepsen.util 8334 :fail :cas [18 [33 14]] :ValueNotSwapFrom INFO [2019-12-17 12:33:40,540] jepsen worker 54 - jepsen.util 8334 :invoke :cas [18 [26 47]] INFO [2019-12-17 12:33:40,543] jepsen worker 16 - jepsen.util 11266 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:33:40,543] jepsen worker 85 - jepsen.util 10885 :info :cas [28 [17 5]] :RequestCanceledException INFO [2019-12-17 12:33:40,543] jepsen worker 16 - jepsen.util 11266 :invoke :write [5 48] INFO [2019-12-17 12:33:40,544] jepsen worker 85 - jepsen.util 10975 :invoke :read [28 nil] INFO [2019-12-17 12:33:40,558] jepsen worker 79 - jepsen.util 7459 :ok :read [26 16] INFO [2019-12-17 12:33:40,559] jepsen worker 79 - jepsen.util 7459 :invoke :write [26 18] INFO [2019-12-17 12:33:40,559] jepsen worker 35 - jepsen.util 9935 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:33:40,559] jepsen worker 40 - jepsen.util 10390 :info :cas [13 [44 3]] :RequestCanceledException INFO [2019-12-17 12:33:40,560] jepsen worker 35 - jepsen.util 9935 :invoke :write [11 2] INFO [2019-12-17 12:33:40,561] jepsen worker 40 - jepsen.util 10480 :invoke :read [13 nil] INFO [2019-12-17 12:33:40,567] jepsen worker 11 - jepsen.util 7571 :fail :cas [3 [27 48]] :ValueNotSwapFrom INFO [2019-12-17 12:33:40,567] jepsen worker 11 - jepsen.util 7571 :invoke :cas [3 [1 49]] INFO [2019-12-17 12:33:40,580] jepsen worker 19 - jepsen.util 7489 :ok :read [6 33] INFO [2019-12-17 12:33:40,581] jepsen worker 19 - jepsen.util 7489 :invoke :read [6 nil] INFO [2019-12-17 12:33:40,589] jepsen worker 72 - jepsen.util 7272 :ok :read [24 17] INFO [2019-12-17 12:33:40,589] jepsen worker 72 - jepsen.util 7272 :invoke :read [24 nil] INFO [2019-12-17 12:33:40,592] jepsen worker 47 - jepsen.util 7607 :ok :read [15 40] INFO [2019-12-17 12:33:40,592] jepsen worker 47 - jepsen.util 7607 :invoke :read [15 nil] INFO [2019-12-17 12:33:40,595] jepsen worker 49 - jepsen.util 9499 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:33:40,595] jepsen worker 7 - jepsen.util 10177 :info :cas [2 [33 48]] :RequestCanceledException INFO [2019-12-17 12:33:40,595] jepsen worker 49 - jepsen.util 9499 :invoke :cas [16 [24 40]] INFO [2019-12-17 12:33:40,596] jepsen worker 7 - jepsen.util 10267 :invoke :read [2 nil] INFO [2019-12-17 12:33:40,613] jepsen worker 44 - jepsen.util 10664 :info :cas [14 [28 32]] :RequestCanceledException INFO [2019-12-17 12:33:40,614] jepsen worker 44 - jepsen.util 10754 :invoke :cas [14 [4 44]] INFO [2019-12-17 12:33:40,614] jepsen worker 88 - jepsen.util 7378 :ok :write [29 28] INFO [2019-12-17 12:33:40,614] jepsen worker 88 - jepsen.util 7378 :invoke :read [29 nil] INFO [2019-12-17 12:33:40,628] jepsen worker 8 - jepsen.util 10088 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:33:40,628] jepsen worker 8 - jepsen.util 10088 :invoke :read [2 nil] INFO [2019-12-17 12:33:40,628] jepsen worker 61 - jepsen.util 10681 :info :write [20 12] :RequestCanceledException INFO [2019-12-17 12:33:40,629] jepsen worker 61 - jepsen.util 10771 :invoke :read [20 nil] INFO [2019-12-17 12:33:40,635] jepsen worker 36 - jepsen.util 7776 :ok :write [12 32] INFO [2019-12-17 12:33:40,636] jepsen worker 36 - jepsen.util 7776 :invoke :read [12 nil] INFO [2019-12-17 12:33:40,642] jepsen worker 69 - jepsen.util 7539 :info :write [23 42] :RequestCanceledException INFO [2019-12-17 12:33:40,643] jepsen worker 69 - jepsen.util 7629 :invoke :cas [23 [40 19]] INFO [2019-12-17 12:33:40,644] jepsen worker 27 - jepsen.util 7407 :ok :write [9 1] INFO [2019-12-17 12:33:40,645] jepsen worker 27 - jepsen.util 7407 :invoke :cas [9 [43 0]] INFO [2019-12-17 12:33:40,647] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42818,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:33:40,648] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42818,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:33:40,648] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42821,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:33:40,648] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42821,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:33:40,648] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42822,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:33:40,648] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42822,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:33:40,683] jepsen worker 32 - jepsen.util 7412 :fail :cas [10 [40 4]] :ValueNotSwapFrom INFO [2019-12-17 12:33:40,683] jepsen worker 32 - jepsen.util 7412 :invoke :cas [10 [36 28]] INFO [2019-12-17 12:33:40,685] jepsen worker 80 - jepsen.util 7460 :ok :write [26 20] INFO [2019-12-17 12:33:40,685] jepsen worker 80 - jepsen.util 7460 :invoke :read [26 nil] INFO [2019-12-17 12:33:40,691] jepsen worker 11 - jepsen.util 7571 :fail :cas [3 [1 49]] :ValueNotSwapFrom INFO [2019-12-17 12:33:40,691] jepsen worker 11 - jepsen.util 7571 :invoke :read [3 nil] INFO [2019-12-17 12:33:40,738] jepsen worker 47 - jepsen.util 7607 :ok :read [15 40] INFO [2019-12-17 12:33:40,738] jepsen worker 47 - jepsen.util 7607 :invoke :read [15 nil] INFO [2019-12-17 12:33:40,759] jepsen worker 27 - jepsen.util 7407 :fail :cas [9 [43 0]] :ValueNotSwapFrom INFO [2019-12-17 12:33:40,759] jepsen worker 27 - jepsen.util 7407 :invoke :read [9 nil] INFO [2019-12-17 12:33:40,770] jepsen worker 14 - jepsen.util 7664 :fail :cas [4 [38 25]] :ValueNotSwapFrom INFO [2019-12-17 12:33:40,770] jepsen worker 14 - jepsen.util 7664 :invoke :write [4 1] INFO [2019-12-17 12:33:40,773] jepsen worker 31 - jepsen.util 6601 :fail :cas [10 [16 34]] :ValueNotSwapFrom INFO [2019-12-17 12:33:40,773] jepsen worker 31 - jepsen.util 6601 :invoke :write [10 5] INFO [2019-12-17 12:33:40,785] jepsen worker 11 - jepsen.util 7571 :ok :read [3 23] INFO [2019-12-17 12:33:40,785] jepsen worker 11 - jepsen.util 7571 :invoke :write [3 14] INFO [2019-12-17 12:33:40,788] jepsen worker 72 - jepsen.util 7272 :ok :read [24 17] INFO [2019-12-17 12:33:40,788] jepsen worker 72 - jepsen.util 7272 :invoke :cas [24 [36 45]] INFO [2019-12-17 12:33:40,802] jepsen worker 63 - jepsen.util 6273 :ok :read [21 36] INFO [2019-12-17 12:33:40,802] jepsen worker 63 - jepsen.util 6273 :invoke :cas [21 [49 22]] INFO [2019-12-17 12:33:40,812] jepsen worker 37 - jepsen.util 6967 :info :write [12 30] :RequestCanceledException INFO [2019-12-17 12:33:40,815] jepsen worker 37 - jepsen.util 7057 :invoke :read [12 nil] INFO [2019-12-17 12:33:40,824] jepsen worker 54 - jepsen.util 8334 :fail :cas [18 [26 47]] :ValueNotSwapFrom INFO [2019-12-17 12:33:40,825] jepsen worker 54 - jepsen.util 8334 :invoke :read [18 nil] INFO [2019-12-17 12:33:40,825] jepsen worker 71 - jepsen.util 8171 :ok :read [23 2] INFO [2019-12-17 12:33:40,825] jepsen worker 71 - jepsen.util 8171 :invoke :read [23 nil] INFO [2019-12-17 12:33:40,828] jepsen worker 46 - jepsen.util 7696 :info :write [15 1] :RequestCanceledException INFO [2019-12-17 12:33:40,833] jepsen worker 46 - jepsen.util 7786 :invoke :read [15 nil] INFO [2019-12-17 12:33:40,843] jepsen worker 32 - jepsen.util 7412 :fail :cas [10 [36 28]] :ValueNotSwapFrom INFO [2019-12-17 12:33:40,843] jepsen worker 32 - jepsen.util 7412 :invoke :read [10 nil] INFO [2019-12-17 12:33:40,844] jepsen worker 19 - jepsen.util 7489 :ok :read [6 33] INFO [2019-12-17 12:33:40,844] jepsen worker 19 - jepsen.util 7489 :invoke :write [6 41] INFO [2019-12-17 12:33:40,844] jepsen worker 88 - jepsen.util 7378 :ok :read [29 28] INFO [2019-12-17 12:33:40,844] jepsen worker 88 - jepsen.util 7378 :invoke :read [29 nil] INFO [2019-12-17 12:33:40,846] jepsen worker 27 - jepsen.util 7407 :ok :read [9 1] INFO [2019-12-17 12:33:40,846] jepsen worker 27 - jepsen.util 7407 :invoke :write [9 14] INFO [2019-12-17 12:33:40,849] jepsen worker 47 - jepsen.util 7607 :ok :read [15 40] INFO [2019-12-17 12:33:40,849] jepsen worker 47 - jepsen.util 7607 :invoke :write [15 24] INFO [2019-12-17 12:33:40,856] jepsen worker 72 - jepsen.util 7272 :fail :cas [24 [36 45]] :ValueNotSwapFrom INFO [2019-12-17 12:33:40,856] jepsen worker 72 - jepsen.util 7272 :invoke :write [24 13] INFO [2019-12-17 12:33:40,871] jepsen worker 32 - jepsen.util 7412 :ok :read [10 21] INFO [2019-12-17 12:33:40,871] jepsen worker 32 - jepsen.util 7412 :invoke :read [10 nil] INFO [2019-12-17 12:33:40,872] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:33:40,873] jepsen nemesis - couchbase.util Rebalance check count: 328 INFO [2019-12-17 12:33:40,873] jepsen nemesis - couchbase.util Rebalance stuck count: 309 INFO [2019-12-17 12:33:40,895] jepsen worker 4 - jepsen.util 6754 :info :write [1 36] :RequestCanceledException INFO [2019-12-17 12:33:40,895] jepsen worker 4 - jepsen.util 6844 :invoke :write [1 37] INFO [2019-12-17 12:33:40,911] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42845,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:33:40,911] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42845,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:33:40,911] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42848,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:33:40,911] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42848,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:33:40,913] jepsen worker 18 - jepsen.util 7488 :info :write [6 35] :RequestCanceledException INFO [2019-12-17 12:33:40,914] jepsen worker 18 - jepsen.util 7578 :invoke :write [6 21] INFO [2019-12-17 12:33:40,928] jepsen worker 13 - jepsen.util 6943 :info :write [4 1] :RequestCanceledException INFO [2019-12-17 12:33:40,928] jepsen worker 87 - jepsen.util 6657 :info :write [29 26] :RequestCanceledException INFO [2019-12-17 12:33:40,929] jepsen worker 87 - jepsen.util 6747 :invoke :write [29 22] INFO [2019-12-17 12:33:40,929] jepsen worker 13 - jepsen.util 7033 :invoke :write [4 27] INFO [2019-12-17 12:33:40,934] jepsen worker 3 - jepsen.util 7473 :ok :read [1 36] INFO [2019-12-17 12:33:40,935] jepsen worker 3 - jepsen.util 7473 :invoke :read [1 nil] INFO [2019-12-17 12:33:40,942] jepsen worker 45 - jepsen.util 7245 :info :cas [15 [12 35]] :RequestCanceledException INFO [2019-12-17 12:33:40,943] jepsen worker 45 - jepsen.util 7335 :invoke :cas [15 [0 40]] INFO [2019-12-17 12:33:40,954] jepsen worker 74 - jepsen.util 7544 :ok :write [24 13] INFO [2019-12-17 12:33:40,954] jepsen worker 74 - jepsen.util 7544 :invoke :read [24 nil] INFO [2019-12-17 12:33:40,974] jepsen worker 52 - jepsen.util 7702 :ok :write [17 30] INFO [2019-12-17 12:33:40,974] jepsen worker 52 - jepsen.util 7702 :invoke :read [17 nil] INFO [2019-12-17 12:33:40,978] jepsen worker 46 - jepsen.util 7786 :ok :read [15 40] INFO [2019-12-17 12:33:40,978] jepsen worker 46 - jepsen.util 7786 :invoke :write [15 5] INFO [2019-12-17 12:33:40,995] jepsen worker 2 - jepsen.util 9722 :info :cas [0 [37 13]] :RequestCanceledException INFO [2019-12-17 12:33:40,996] jepsen worker 74 - jepsen.util 7544 :ok :read [24 13] INFO [2019-12-17 12:33:40,996] jepsen worker 74 - jepsen.util 7544 :invoke :write [24 44] INFO [2019-12-17 12:33:40,997] jepsen worker 2 - jepsen.util 9812 :invoke :write [0 16] INFO [2019-12-17 12:33:41,009] jepsen worker 69 - jepsen.util 7629 :fail :cas [23 [40 19]] :ValueNotSwapFrom INFO [2019-12-17 12:33:41,009] jepsen worker 69 - jepsen.util 7629 :invoke :write [23 35] INFO [2019-12-17 12:33:41,017] jepsen worker 80 - jepsen.util 7460 :ok :read [26 20] INFO [2019-12-17 12:33:41,017] jepsen worker 80 - jepsen.util 7460 :invoke :read [26 nil] INFO [2019-12-17 12:33:41,018] jepsen worker 88 - jepsen.util 7378 :ok :read [29 28] INFO [2019-12-17 12:33:41,018] jepsen worker 88 - jepsen.util 7378 :invoke :cas [29 [7 7]] INFO [2019-12-17 12:33:41,021] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42860,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:33:41,021] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42860,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:33:41,021] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42864,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:33:41,021] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42864,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:33:41,021] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42865,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:33:41,021] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42865,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:33:41,040] jepsen worker 27 - jepsen.util 7407 :ok :write [9 14] INFO [2019-12-17 12:33:41,040] jepsen worker 27 - jepsen.util 7407 :invoke :read [9 nil] INFO [2019-12-17 12:33:41,042] jepsen worker 65 - jepsen.util 6095 :info :cas [21 [7 47]] :RequestCanceledException INFO [2019-12-17 12:33:41,043] jepsen worker 65 - jepsen.util 6185 :invoke :write [21 45] INFO [2019-12-17 12:33:41,059] jepsen worker 27 - jepsen.util 7407 :ok :read [9 14] INFO [2019-12-17 12:33:41,059] jepsen worker 27 - jepsen.util 7407 :invoke :cas [9 [39 2]] INFO [2019-12-17 12:33:41,059] jepsen worker 11 - jepsen.util 7571 :ok :write [3 14] INFO [2019-12-17 12:33:41,060] jepsen worker 11 - jepsen.util 7571 :invoke :read [3 nil] INFO [2019-12-17 12:33:41,060] jepsen worker 31 - jepsen.util 6601 :ok :write [10 5] INFO [2019-12-17 12:33:41,060] jepsen worker 31 - jepsen.util 6601 :invoke :read [10 nil] INFO [2019-12-17 12:33:41,070] jepsen worker 53 - jepsen.util 6983 :ok :write [17 23] INFO [2019-12-17 12:33:41,071] jepsen worker 53 - jepsen.util 6983 :invoke :read [17 nil] INFO [2019-12-17 12:33:41,110] jepsen worker 27 - jepsen.util 7407 :fail :cas [9 [39 2]] :ValueNotSwapFrom INFO [2019-12-17 12:33:41,111] jepsen worker 27 - jepsen.util 7407 :invoke :read [9 nil] INFO [2019-12-17 12:33:41,114] jepsen worker 88 - jepsen.util 7378 :fail :cas [29 [7 7]] :ValueNotSwapFrom INFO [2019-12-17 12:33:41,114] jepsen worker 88 - jepsen.util 7378 :invoke :cas [29 [45 21]] INFO [2019-12-17 12:33:41,126] jepsen worker 20 - jepsen.util 7130 :info :write [6 48] :RequestCanceledException INFO [2019-12-17 12:33:41,127] jepsen worker 20 - jepsen.util 7220 :invoke :read [6 nil] INFO [2019-12-17 12:33:41,129] jepsen worker 31 - jepsen.util 6601 :ok :read [10 5] INFO [2019-12-17 12:33:41,129] jepsen worker 31 - jepsen.util 6601 :invoke :read [10 nil] INFO [2019-12-17 12:33:41,130] jepsen worker 27 - jepsen.util 7407 :ok :read [9 14] INFO [2019-12-17 12:33:41,130] jepsen worker 27 - jepsen.util 7407 :invoke :read [9 nil] INFO [2019-12-17 12:33:41,141] jepsen worker 54 - jepsen.util 8334 :ok :read [18 27] INFO [2019-12-17 12:33:41,141] jepsen worker 54 - jepsen.util 8334 :invoke :cas [18 [3 32]] INFO [2019-12-17 12:33:41,164] jepsen worker 31 - jepsen.util 6601 :ok :read [10 5] INFO [2019-12-17 12:33:41,165] jepsen worker 31 - jepsen.util 6601 :invoke :cas [10 [1 47]] INFO [2019-12-17 12:33:41,165] jepsen worker 27 - jepsen.util 7407 :ok :read [9 14] INFO [2019-12-17 12:33:41,166] jepsen worker 27 - jepsen.util 7407 :invoke :cas [9 [22 30]] INFO [2019-12-17 12:33:41,177] jepsen worker 55 - jepsen.util 8335 :fail :cas [18 [40 33]] :ValueNotSwapFrom INFO [2019-12-17 12:33:41,178] jepsen worker 55 - jepsen.util 8335 :invoke :cas [18 [38 29]] INFO [2019-12-17 12:33:41,186] jepsen worker 27 - jepsen.util 7407 :fail :cas [9 [22 30]] :ValueNotSwapFrom INFO [2019-12-17 12:33:41,186] jepsen worker 27 - jepsen.util 7407 :invoke :write [9 22] INFO [2019-12-17 12:33:41,188] jepsen worker 88 - jepsen.util 7378 :fail :cas [29 [45 21]] :ValueNotSwapFrom INFO [2019-12-17 12:33:41,188] jepsen worker 88 - jepsen.util 7378 :invoke :write [29 45] INFO [2019-12-17 12:33:41,189] jepsen worker 72 - jepsen.util 7272 :ok :write [24 13] INFO [2019-12-17 12:33:41,189] jepsen worker 72 - jepsen.util 7272 :invoke :read [24 nil] INFO [2019-12-17 12:33:41,195] jepsen worker 31 - jepsen.util 6601 :fail :cas [10 [1 47]] :ValueNotSwapFrom INFO [2019-12-17 12:33:41,195] jepsen worker 31 - jepsen.util 6601 :invoke :write [10 11] INFO [2019-12-17 12:33:41,212] jepsen worker 9 - jepsen.util 7839 :info :cas [3 [23 30]] :RequestCanceledException INFO [2019-12-17 12:33:41,212] jepsen worker 73 - jepsen.util 7093 :info :write [24 33] :RequestCanceledException INFO [2019-12-17 12:33:41,213] jepsen worker 9 - jepsen.util 7929 :invoke :read [3 nil] INFO [2019-12-17 12:33:41,214] jepsen worker 73 - jepsen.util 7183 :invoke :write [24 20] INFO [2019-12-17 12:33:41,232] jepsen worker 36 - jepsen.util 7776 :ok :read [12 32] INFO [2019-12-17 12:33:41,232] jepsen worker 36 - jepsen.util 7776 :invoke :write [12 49] INFO [2019-12-17 12:33:41,241] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42889,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:33:41,241] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42889,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:33:41,241] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42890,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:33:41,241] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42890,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:33:41,241] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42891,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:33:41,241] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42891,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:33:41,248] jepsen worker 53 - jepsen.util 6983 :ok :read [17 23] INFO [2019-12-17 12:33:41,249] jepsen worker 53 - jepsen.util 6983 :invoke :read [17 nil] INFO [2019-12-17 12:33:41,252] jepsen worker 65 - jepsen.util 6185 :ok :write [21 45] INFO [2019-12-17 12:33:41,252] jepsen worker 65 - jepsen.util 6185 :invoke :cas [21 [45 29]] INFO [2019-12-17 12:33:41,260] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42898,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:33:41,260] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42898,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:33:41,260] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42901,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:33:41,260] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42901,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:33:41,260] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42902,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:33:41,260] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42902,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:33:41,272] jepsen worker 37 - jepsen.util 7057 :ok :read [12 32] INFO [2019-12-17 12:33:41,272] jepsen worker 37 - jepsen.util 7057 :invoke :read [12 nil] INFO [2019-12-17 12:33:41,275] jepsen worker 47 - jepsen.util 7607 :ok :write [15 24] INFO [2019-12-17 12:33:41,275] jepsen worker 47 - jepsen.util 7607 :invoke :cas [15 [23 36]] INFO [2019-12-17 12:33:41,328] jepsen worker 10 - jepsen.util 7390 :fail :read [3 nil] :RequestTimeoutException INFO [2019-12-17 12:33:41,328] jepsen worker 28 - jepsen.util 5788 :info :cas [9 [20 15]] :RequestCanceledException INFO [2019-12-17 12:33:41,328] jepsen worker 10 - jepsen.util 7390 :invoke :read [3 nil] INFO [2019-12-17 12:33:41,329] jepsen worker 28 - jepsen.util 5878 :invoke :write [9 1] INFO [2019-12-17 12:33:41,329] jepsen worker 47 - jepsen.util 7607 :fail :cas [15 [23 36]] :ValueNotSwapFrom INFO [2019-12-17 12:33:41,330] jepsen worker 47 - jepsen.util 7607 :invoke :read [15 nil] INFO [2019-12-17 12:33:41,342] jepsen worker 56 - jepsen.util 8246 :info :write [18 21] :RequestCanceledException INFO [2019-12-17 12:33:41,343] jepsen worker 56 - jepsen.util 8336 :invoke :read [18 nil] INFO [2019-12-17 12:33:41,348] jepsen worker 55 - jepsen.util 8335 :fail :cas [18 [38 29]] :ValueNotSwapFrom INFO [2019-12-17 12:33:41,348] jepsen worker 55 - jepsen.util 8335 :invoke :cas [18 [29 11]] INFO [2019-12-17 12:33:41,356] jepsen worker 18 - jepsen.util 7578 :ok :write [6 21] INFO [2019-12-17 12:33:41,356] jepsen worker 18 - jepsen.util 7578 :invoke :cas [6 [28 23]] INFO [2019-12-17 12:33:41,359] jepsen worker 5 - jepsen.util 6935 :info :write [1 39] :RequestCanceledException INFO [2019-12-17 12:33:41,359] jepsen worker 30 - jepsen.util 7230 :info :write [10 46] :RequestCanceledException INFO [2019-12-17 12:33:41,359] jepsen worker 64 - jepsen.util 5734 :info :write [21 15] :RequestCanceledException INFO [2019-12-17 12:33:41,361] jepsen worker 64 - jepsen.util 5824 :invoke :read [21 nil] INFO [2019-12-17 12:33:41,361] jepsen worker 30 - jepsen.util 7320 :invoke :cas [10 [6 42]] INFO [2019-12-17 12:33:41,362] jepsen worker 5 - jepsen.util 7025 :invoke :read [1 nil] INFO [2019-12-17 12:33:41,371] jepsen worker 47 - jepsen.util 7607 :ok :read [15 24] INFO [2019-12-17 12:33:41,371] jepsen worker 47 - jepsen.util 7607 :invoke :write [15 33] INFO [2019-12-17 12:33:41,391] jepsen worker 71 - jepsen.util 8171 :ok :read [23 2] INFO [2019-12-17 12:33:41,391] jepsen worker 71 - jepsen.util 8171 :invoke :read [23 nil] INFO [2019-12-17 12:33:41,392] jepsen worker 20 - jepsen.util 7220 :ok :read [6 21] INFO [2019-12-17 12:33:41,392] jepsen worker 20 - jepsen.util 7220 :invoke :write [6 47] INFO [2019-12-17 12:33:41,395] jepsen worker 37 - jepsen.util 7057 :ok :read [12 32] INFO [2019-12-17 12:33:41,395] jepsen worker 37 - jepsen.util 7057 :invoke :cas [12 [5 45]] INFO [2019-12-17 12:33:41,406] jepsen worker 55 - jepsen.util 8335 :fail :cas [18 [29 11]] :ValueNotSwapFrom INFO [2019-12-17 12:33:41,406] jepsen worker 55 - jepsen.util 8335 :invoke :read [18 nil] INFO [2019-12-17 12:33:41,425] jepsen worker 18 - jepsen.util 7578 :fail :cas [6 [28 23]] :ValueNotSwapFrom INFO [2019-12-17 12:33:41,426] jepsen worker 18 - jepsen.util 7578 :invoke :cas [6 [46 32]] INFO [2019-12-17 12:33:41,455] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42926,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:33:41,455] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42926,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:33:41,455] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42929,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:33:41,455] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42929,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:33:41,455] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42930,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:33:41,455] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42930,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:33:41,456] jepsen worker 53 - jepsen.util 6983 :ok :read [17 23] INFO [2019-12-17 12:33:41,457] jepsen worker 53 - jepsen.util 6983 :invoke :cas [17 [1 29]] INFO [2019-12-17 12:33:41,463] jepsen worker 30 - jepsen.util 7320 :fail :cas [10 [6 42]] :ValueNotSwapFrom INFO [2019-12-17 12:33:41,463] jepsen worker 30 - jepsen.util 7320 :invoke :cas [10 [41 39]] INFO [2019-12-17 12:33:41,468] jepsen worker 65 - jepsen.util 6185 :ok :cas [21 [45 29]] INFO [2019-12-17 12:33:41,469] jepsen worker 27 - jepsen.util 7407 :ok :write [9 22] INFO [2019-12-17 12:33:41,469] jepsen worker 65 - jepsen.util 6185 :invoke :write [21 5] INFO [2019-12-17 12:33:41,469] jepsen worker 27 - jepsen.util 7407 :invoke :read [9 nil] INFO [2019-12-17 12:33:41,506] jepsen worker 3 - jepsen.util 7473 :ok :read [1 36] INFO [2019-12-17 12:33:41,506] jepsen worker 3 - jepsen.util 7473 :invoke :read [1 nil] INFO [2019-12-17 12:33:41,508] jepsen worker 56 - jepsen.util 8336 :ok :read [18 27] INFO [2019-12-17 12:33:41,508] jepsen worker 56 - jepsen.util 8336 :invoke :write [18 38] INFO [2019-12-17 12:33:41,529] jepsen worker 80 - jepsen.util 7460 :ok :read [26 20] INFO [2019-12-17 12:33:41,529] jepsen worker 80 - jepsen.util 7460 :invoke :write [26 38] INFO [2019-12-17 12:33:41,533] jepsen worker 64 - jepsen.util 5824 :ok :read [21 29] INFO [2019-12-17 12:33:41,534] jepsen worker 64 - jepsen.util 5824 :invoke :read [21 nil] INFO [2019-12-17 12:33:41,541] jepsen worker 37 - jepsen.util 7057 :fail :cas [12 [5 45]] :ValueNotSwapFrom INFO [2019-12-17 12:33:41,541] jepsen worker 37 - jepsen.util 7057 :invoke :cas [12 [44 31]] INFO [2019-12-17 12:33:41,541] jepsen worker 27 - jepsen.util 7407 :ok :read [9 22] INFO [2019-12-17 12:33:41,541] jepsen worker 27 - jepsen.util 7407 :invoke :read [9 nil] INFO [2019-12-17 12:33:41,576] jepsen worker 3 - jepsen.util 7473 :ok :read [1 36] INFO [2019-12-17 12:33:41,576] jepsen worker 3 - jepsen.util 7473 :invoke :cas [1 [34 38]] INFO [2019-12-17 12:33:41,590] jepsen worker 28 - jepsen.util 5878 :ok :write [9 1] INFO [2019-12-17 12:33:41,590] jepsen worker 28 - jepsen.util 5878 :invoke :read [9 nil] INFO [2019-12-17 12:33:41,591] jepsen worker 27 - jepsen.util 7407 :ok :read [9 22] INFO [2019-12-17 12:33:41,591] jepsen worker 27 - jepsen.util 7407 :invoke :write [9 0] INFO [2019-12-17 12:33:41,600] jepsen worker 65 - jepsen.util 6185 :ok :write [21 5] INFO [2019-12-17 12:33:41,600] jepsen worker 65 - jepsen.util 6185 :invoke :write [21 28] INFO [2019-12-17 12:33:41,610] jepsen worker 52 - jepsen.util 7702 :ok :read [17 23] INFO [2019-12-17 12:33:41,610] jepsen worker 52 - jepsen.util 7702 :invoke :cas [17 [22 3]] INFO [2019-12-17 12:33:41,623] jepsen worker 53 - jepsen.util 6983 :fail :cas [17 [1 29]] :ValueNotSwapFrom INFO [2019-12-17 12:33:41,623] jepsen worker 53 - jepsen.util 6983 :invoke :write [17 47] INFO [2019-12-17 12:33:41,623] jepsen worker 28 - jepsen.util 5878 :ok :read [9 1] INFO [2019-12-17 12:33:41,624] jepsen worker 28 - jepsen.util 5878 :invoke :cas [9 [11 46]] INFO [2019-12-17 12:33:41,647] jepsen worker 28 - jepsen.util 5878 :fail :cas [9 [11 46]] :ValueNotSwapFrom INFO [2019-12-17 12:33:41,647] jepsen worker 28 - jepsen.util 5878 :invoke :cas [9 [7 37]] INFO [2019-12-17 12:33:41,648] jepsen worker 55 - jepsen.util 8335 :ok :read [18 27] INFO [2019-12-17 12:33:41,649] jepsen worker 55 - jepsen.util 8335 :invoke :cas [18 [17 34]] INFO [2019-12-17 12:33:41,684] jepsen worker 28 - jepsen.util 5878 :fail :cas [9 [7 37]] :ValueNotSwapFrom INFO [2019-12-17 12:33:41,685] jepsen worker 28 - jepsen.util 5878 :invoke :cas [9 [14 9]] INFO [2019-12-17 12:33:41,694] jepsen worker 12 - jepsen.util 7932 :info :write [4 34] :RequestCanceledException INFO [2019-12-17 12:33:41,695] jepsen worker 12 - jepsen.util 8022 :invoke :cas [4 [31 39]] INFO [2019-12-17 12:33:41,698] jepsen worker 5 - jepsen.util 7025 :ok :read [1 36] INFO [2019-12-17 12:33:41,698] jepsen worker 5 - jepsen.util 7025 :invoke :read [1 nil] INFO [2019-12-17 12:33:41,699] jepsen worker 65 - jepsen.util 6185 :ok :write [21 28] INFO [2019-12-17 12:33:41,699] jepsen worker 65 - jepsen.util 6185 :invoke :write [21 48] INFO [2019-12-17 12:33:41,718] jepsen worker 28 - jepsen.util 5878 :fail :cas [9 [14 9]] :ValueNotSwapFrom INFO [2019-12-17 12:33:41,718] jepsen worker 28 - jepsen.util 5878 :invoke :cas [9 [12 28]] INFO [2019-12-17 12:33:41,740] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42951,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:33:41,740] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42951,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:33:41,742] jepsen worker 38 - jepsen.util 7598 :info :write [12 18] :RequestCanceledException INFO [2019-12-17 12:33:41,744] jepsen worker 38 - jepsen.util 7688 :invoke :write [12 9] INFO [2019-12-17 12:33:41,746] jepsen worker 37 - jepsen.util 7057 :fail :cas [12 [44 31]] :ValueNotSwapFrom INFO [2019-12-17 12:33:41,746] jepsen worker 37 - jepsen.util 7057 :invoke :cas [12 [19 35]] INFO [2019-12-17 12:33:41,773] jepsen worker 27 - jepsen.util 7407 :ok :write [9 0] INFO [2019-12-17 12:33:41,774] jepsen worker 27 - jepsen.util 7407 :invoke :read [9 nil] INFO [2019-12-17 12:33:41,779] jepsen worker 28 - jepsen.util 5878 :fail :cas [9 [12 28]] :ValueNotSwapFrom INFO [2019-12-17 12:33:41,779] jepsen worker 28 - jepsen.util 5878 :invoke :read [9 nil] INFO [2019-12-17 12:33:41,802] jepsen worker 28 - jepsen.util 5878 :ok :read [9 0] INFO [2019-12-17 12:33:41,803] jepsen worker 28 - jepsen.util 5878 :invoke :write [9 32] INFO [2019-12-17 12:33:41,806] jepsen worker 27 - jepsen.util 7407 :ok :read [9 0] INFO [2019-12-17 12:33:41,806] jepsen worker 27 - jepsen.util 7407 :invoke :read [9 nil] INFO [2019-12-17 12:33:41,814] jepsen worker 65 - jepsen.util 6185 :ok :write [21 48] INFO [2019-12-17 12:33:41,815] jepsen worker 65 - jepsen.util 6185 :invoke :read [21 nil] INFO [2019-12-17 12:33:41,824] jepsen worker 27 - jepsen.util 7407 :ok :read [9 0] INFO [2019-12-17 12:33:41,824] jepsen worker 27 - jepsen.util 7407 :invoke :read [9 nil] INFO [2019-12-17 12:33:41,828] jepsen worker 83 - jepsen.util 10523 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:33:41,828] jepsen worker 83 - jepsen.util 10523 :invoke :read [27 nil] INFO [2019-12-17 12:33:41,828] jepsen worker 34 - jepsen.util 9484 :info :cas [11 [10 6]] :RequestCanceledException INFO [2019-12-17 12:33:41,828] jepsen worker 78 - jepsen.util 7548 :info :write [26 44] :RequestCanceledException INFO [2019-12-17 12:33:41,830] jepsen worker 34 - jepsen.util 9574 :invoke :cas [11 [37 29]] INFO [2019-12-17 12:33:41,830] jepsen worker 78 - jepsen.util 7638 :invoke :cas [26 [30 26]] INFO [2019-12-17 12:33:41,830] jepsen worker 65 - jepsen.util 6185 :ok :read [21 48] INFO [2019-12-17 12:33:41,830] jepsen worker 65 - jepsen.util 6185 :invoke :write [21 41] INFO [2019-12-17 12:33:41,839] jepsen worker 27 - jepsen.util 7407 :ok :read [9 0] INFO [2019-12-17 12:33:41,839] jepsen worker 27 - jepsen.util 7407 :invoke :write [9 5] INFO [2019-12-17 12:33:41,842] jepsen worker 42 - jepsen.util 10752 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:33:41,842] jepsen worker 42 - jepsen.util 10752 :invoke :cas [14 [46 5]] INFO [2019-12-17 12:33:41,874] jepsen worker 43 - jepsen.util 9583 :info :cas [14 [2 8]] :RequestCanceledException INFO [2019-12-17 12:33:41,874] jepsen worker 26 - jepsen.util 10376 :info :cas [8 [20 32]] :RequestCanceledException INFO [2019-12-17 12:33:41,876] jepsen worker 71 - jepsen.util 8171 :ok :read [23 2] INFO [2019-12-17 12:33:41,876] jepsen worker 26 - jepsen.util 10466 :invoke :write [8 44] INFO [2019-12-17 12:33:41,876] jepsen worker 43 - jepsen.util 9673 :invoke :read [14 nil] INFO [2019-12-17 12:33:41,876] jepsen worker 71 - jepsen.util 8171 :invoke :cas [23 [40 16]] INFO [2019-12-17 12:33:41,885] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:33:41,885] jepsen nemesis - couchbase.util Rebalance check count: 329 INFO [2019-12-17 12:33:41,885] jepsen nemesis - couchbase.util Rebalance stuck count: 310 INFO [2019-12-17 12:33:41,893] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42976,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:33:41,893] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42976,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:33:41,894] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42980,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:33:41,894] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":42980,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:33:41,895] jepsen worker 66 - jepsen.util 9966 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:33:41,895] jepsen worker 89 - jepsen.util 7199 :info :write [29 43] :RequestCanceledException INFO [2019-12-17 12:33:41,895] jepsen worker 66 - jepsen.util 9966 :invoke :cas [22 [10 24]] INFO [2019-12-17 12:33:41,896] jepsen worker 89 - jepsen.util 7289 :invoke :cas [29 [3 10]] INFO [2019-12-17 12:33:41,904] jepsen worker 28 - jepsen.util 5878 :ok :write [9 32] INFO [2019-12-17 12:33:41,904] jepsen worker 28 - jepsen.util 5878 :invoke :read [9 nil] INFO [2019-12-17 12:33:41,906] jepsen worker 75 - jepsen.util 9525 :info :write [25 7] :RequestCanceledException INFO [2019-12-17 12:33:41,906] jepsen worker 21 - jepsen.util 9471 :info :cas [7 [47 17]] :RequestCanceledException INFO [2019-12-17 12:33:41,908] jepsen worker 21 - jepsen.util 9561 :invoke :read [7 nil] INFO [2019-12-17 12:33:41,908] jepsen worker 75 - jepsen.util 9615 :invoke :write [25 17] INFO [2019-12-17 12:33:41,909] jepsen worker 55 - jepsen.util 8335 :fail :cas [18 [17 34]] :ValueNotSwapFrom INFO [2019-12-17 12:33:41,909] jepsen worker 55 - jepsen.util 8335 :invoke :write [18 5] INFO [2019-12-17 12:33:41,928] jepsen worker 23 - jepsen.util 10193 :info :write [7 43] :RequestCanceledException INFO [2019-12-17 12:33:41,928] jepsen worker 22 - jepsen.util 9922 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:33:41,929] jepsen worker 22 - jepsen.util 9922 :invoke :cas [7 [15 11]] INFO [2019-12-17 12:33:41,929] jepsen worker 23 - jepsen.util 10283 :invoke :read [7 nil] INFO [2019-12-17 12:33:41,942] jepsen worker 62 - jepsen.util 10142 :info :cas [20 [43 27]] :RequestCanceledException INFO [2019-12-17 12:33:41,943] jepsen worker 62 - jepsen.util 10232 :invoke :read [20 nil] INFO [2019-12-17 12:33:41,950] jepsen worker 65 - jepsen.util 6185 :ok :write [21 41] INFO [2019-12-17 12:33:41,950] jepsen worker 65 - jepsen.util 6185 :invoke :write [21 28] INFO [2019-12-17 12:33:41,958] jepsen worker 28 - jepsen.util 5878 :ok :read [9 32] INFO [2019-12-17 12:33:41,958] jepsen worker 41 - jepsen.util 10571 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:33:41,958] jepsen worker 41 - jepsen.util 10571 :invoke :write [13 26] INFO [2019-12-17 12:33:41,958] jepsen worker 28 - jepsen.util 5878 :invoke :write [9 12] INFO [2019-12-17 12:33:41,983] jepsen worker 89 - jepsen.util 7289 :fail :cas [29 [3 10]] :ValueNotSwapFrom INFO [2019-12-17 12:33:41,983] jepsen worker 89 - jepsen.util 7289 :invoke :write [29 0] INFO [2019-12-17 12:33:41,994] jepsen worker 77 - jepsen.util 10247 :info :cas [25 [8 16]] :RequestCanceledException INFO [2019-12-17 12:33:41,994] jepsen worker 67 - jepsen.util 10057 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:33:41,995] jepsen worker 67 - jepsen.util 10057 :invoke :cas [22 [14 37]] INFO [2019-12-17 12:33:41,995] jepsen worker 77 - jepsen.util 10337 :invoke :write [25 2] INFO [2019-12-17 12:33:42,001] jepsen worker 27 - jepsen.util 7407 :ok :write [9 5] INFO [2019-12-17 12:33:42,001] jepsen worker 27 - jepsen.util 7407 :invoke :read [9 nil] INFO [2019-12-17 12:33:42,003] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":43001,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:33:42,003] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":43001,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:33:42,013] jepsen worker 59 - jepsen.util 9869 :info :cas [19 [35 13]] :RequestCanceledException INFO [2019-12-17 12:33:42,013] jepsen worker 86 - jepsen.util 10616 :info :write [28 46] :RequestCanceledException INFO [2019-12-17 12:33:42,014] jepsen worker 59 - jepsen.util 9959 :invoke :write [19 25] INFO [2019-12-17 12:33:42,015] jepsen worker 86 - jepsen.util 10706 :invoke :write [28 6] INFO [2019-12-17 12:33:42,029] jepsen worker 68 - jepsen.util 9518 :info :write [22 8] :RequestCanceledException INFO [2019-12-17 12:33:42,029] jepsen worker 0 - jepsen.util 10440 :info :cas [0 [13 14]] :RequestCanceledException INFO [2019-12-17 12:33:42,029] jepsen worker 58 - jepsen.util 10768 :info :write [19 13] :RequestCanceledException INFO [2019-12-17 12:33:42,029] jepsen worker 15 - jepsen.util 9015 :info :cas [5 [2 45]] :RequestCanceledException INFO [2019-12-17 12:33:42,030] jepsen worker 0 - jepsen.util 10530 :invoke :write [0 18] INFO [2019-12-17 12:33:42,030] jepsen worker 68 - jepsen.util 9608 :invoke :cas [22 [35 29]] INFO [2019-12-17 12:33:42,030] jepsen worker 58 - jepsen.util 10858 :invoke :cas [19 [24 11]] INFO [2019-12-17 12:33:42,030] jepsen worker 15 - jepsen.util 9105 :invoke :read [5 nil] INFO [2019-12-17 12:33:42,042] jepsen worker 60 - jepsen.util 10950 :info :write [20 43] :RequestCanceledException INFO [2019-12-17 12:33:42,042] jepsen worker 50 - jepsen.util 10760 :info :write [16 10] :RequestCanceledException INFO [2019-12-17 12:33:42,043] jepsen worker 60 - jepsen.util 11040 :invoke :cas [20 [20 24]] INFO [2019-12-17 12:33:42,044] jepsen worker 50 - jepsen.util 10850 :invoke :cas [16 [26 4]] INFO [2019-12-17 12:33:42,053] jepsen worker 37 - jepsen.util 7057 :fail :cas [12 [19 35]] :ValueNotSwapFrom INFO [2019-12-17 12:33:42,054] jepsen worker 37 - jepsen.util 7057 :invoke :write [12 49] INFO [2019-12-17 12:33:42,054] jepsen worker 27 - jepsen.util 7407 :ok :read [9 5] INFO [2019-12-17 12:33:42,054] jepsen worker 27 - jepsen.util 7407 :invoke :cas [9 [33 13]] INFO [2019-12-17 12:33:42,055] jepsen worker 52 - jepsen.util 7702 :fail :cas [17 [22 3]] :ValueNotSwapFrom INFO [2019-12-17 12:33:42,055] jepsen worker 52 - jepsen.util 7702 :invoke :read [17 nil] INFO [2019-12-17 12:33:42,057] jepsen worker 25 - jepsen.util 10555 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:33:42,057] jepsen worker 33 - jepsen.util 9843 :info :cas [11 [27 49]] :RequestCanceledException INFO [2019-12-17 12:33:42,057] jepsen worker 25 - jepsen.util 10555 :invoke :write [8 23] INFO [2019-12-17 12:33:42,058] jepsen worker 33 - jepsen.util 9933 :invoke :read [11 nil] INFO [2019-12-17 12:33:42,071] jepsen worker 65 - jepsen.util 6185 :ok :write [21 28] INFO [2019-12-17 12:33:42,071] jepsen worker 65 - jepsen.util 6185 :invoke :cas [21 [45 43]] INFO [2019-12-17 12:33:42,079] jepsen worker 27 - jepsen.util 7407 :fail :cas [9 [33 13]] :ValueNotSwapFrom INFO [2019-12-17 12:33:42,079] jepsen worker 27 - jepsen.util 7407 :invoke :read [9 nil] INFO [2019-12-17 12:33:42,085] jepsen worker 5 - jepsen.util 7025 :ok :read [1 36] INFO [2019-12-17 12:33:42,085] jepsen worker 5 - jepsen.util 7025 :invoke :cas [1 [35 5]] INFO [2019-12-17 12:33:42,095] jepsen worker 24 - jepsen.util 8754 :info :write [8 6] :RequestCanceledException INFO [2019-12-17 12:33:42,095] jepsen worker 24 - jepsen.util 8844 :invoke :read [8 nil] INFO [2019-12-17 12:33:42,103] jepsen worker 65 - jepsen.util 6185 :fail :cas [21 [45 43]] :ValueNotSwapFrom INFO [2019-12-17 12:33:42,103] jepsen worker 65 - jepsen.util 6185 :invoke :write [21 9] INFO [2019-12-17 12:33:42,110] jepsen worker 27 - jepsen.util 7407 :ok :read [9 5] INFO [2019-12-17 12:33:42,110] jepsen worker 27 - jepsen.util 7407 :invoke :read [9 nil] INFO [2019-12-17 12:33:42,113] jepsen worker 51 - jepsen.util 7161 :info :write [17 11] :RequestCanceledException INFO [2019-12-17 12:33:42,114] jepsen worker 51 - jepsen.util 7251 :invoke :write [17 7] INFO [2019-12-17 12:33:42,127] jepsen worker 82 - jepsen.util 10702 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:33:42,127] jepsen worker 82 - jepsen.util 10702 :invoke :cas [27 [10 2]] INFO [2019-12-17 12:33:42,133] jepsen worker 3 - jepsen.util 7473 :fail :cas [1 [34 38]] :ValueNotSwapFrom INFO [2019-12-17 12:33:42,133] jepsen worker 3 - jepsen.util 7473 :invoke :read [1 nil] INFO [2019-12-17 12:33:42,145] jepsen worker 27 - jepsen.util 7407 :ok :read [9 5] INFO [2019-12-17 12:33:42,145] jepsen worker 27 - jepsen.util 7407 :invoke :read [9 nil] INFO [2019-12-17 12:33:42,148] jepsen worker 28 - jepsen.util 5878 :ok :write [9 12] INFO [2019-12-17 12:33:42,148] jepsen worker 28 - jepsen.util 5878 :invoke :write [9 5] INFO [2019-12-17 12:33:42,176] jepsen worker 12 - jepsen.util 8022 :fail :cas [4 [31 39]] :ValueNotSwapFrom INFO [2019-12-17 12:33:42,176] jepsen worker 12 - jepsen.util 8022 :invoke :write [4 21] INFO [2019-12-17 12:33:42,180] jepsen worker 27 - jepsen.util 7407 :ok :read [9 12] INFO [2019-12-17 12:33:42,180] jepsen worker 27 - jepsen.util 7407 :invoke :read [9 nil] INFO [2019-12-17 12:33:42,200] jepsen worker 27 - jepsen.util 7407 :ok :read [9 12] INFO [2019-12-17 12:33:42,201] jepsen worker 27 - jepsen.util 7407 :invoke :read [9 nil] INFO [2019-12-17 12:33:42,211] jepsen worker 65 - jepsen.util 6185 :ok :write [21 9] INFO [2019-12-17 12:33:42,211] jepsen worker 65 - jepsen.util 6185 :invoke :cas [21 [42 26]] INFO [2019-12-17 12:33:42,213] jepsen worker 1 - jepsen.util 10081 :info :cas [0 [7 43]] :RequestCanceledException INFO [2019-12-17 12:33:42,214] jepsen worker 1 - jepsen.util 10171 :invoke :write [0 44] INFO [2019-12-17 12:33:42,222] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":43032,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:33:42,222] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":43032,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:33:42,222] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":43034,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:33:42,222] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":43034,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:33:42,229] jepsen worker 6 - jepsen.util 10716 :info :cas [2 [3 49]] :RequestCanceledException INFO [2019-12-17 12:33:42,229] jepsen worker 6 - jepsen.util 10806 :invoke :cas [2 [29 8]] INFO [2019-12-17 12:33:42,235] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":43041,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:33:42,235] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":43041,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:33:42,235] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":43045,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:33:42,236] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":43045,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:33:42,239] jepsen worker 27 - jepsen.util 7407 :ok :read [9 12] INFO [2019-12-17 12:33:42,239] jepsen worker 27 - jepsen.util 7407 :invoke :read [9 nil] INFO [2019-12-17 12:33:42,253] jepsen worker 65 - jepsen.util 6185 :fail :cas [21 [42 26]] :ValueNotSwapFrom INFO [2019-12-17 12:33:42,253] jepsen worker 65 - jepsen.util 6185 :invoke :cas [21 [29 47]] INFO [2019-12-17 12:33:42,273] jepsen worker 27 - jepsen.util 7407 :ok :read [9 12] INFO [2019-12-17 12:33:42,273] jepsen worker 27 - jepsen.util 7407 :invoke :read [9 nil] INFO [2019-12-17 12:33:42,278] jepsen worker 20 - jepsen.util 7220 :ok :write [6 47] INFO [2019-12-17 12:33:42,278] jepsen worker 20 - jepsen.util 7220 :invoke :write [6 16] INFO [2019-12-17 12:33:42,278] jepsen worker 31 - jepsen.util 6601 :ok :write [10 11] INFO [2019-12-17 12:33:42,278] jepsen worker 31 - jepsen.util 6601 :invoke :cas [10 [14 18]] INFO [2019-12-17 12:33:42,309] jepsen worker 65 - jepsen.util 6185 :fail :cas [21 [29 47]] :ValueNotSwapFrom INFO [2019-12-17 12:33:42,310] jepsen worker 65 - jepsen.util 6185 :invoke :write [21 28] INFO [2019-12-17 12:33:42,314] jepsen worker 71 - jepsen.util 8171 :fail :cas [23 [40 16]] :ValueNotSwapFrom INFO [2019-12-17 12:33:42,314] jepsen worker 71 - jepsen.util 8171 :invoke :cas [23 [42 18]] INFO [2019-12-17 12:33:42,342] jepsen worker 81 - jepsen.util 9981 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:33:42,342] jepsen worker 84 - jepsen.util 9714 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:33:42,342] jepsen worker 84 - jepsen.util 9714 :invoke :cas [28 [11 47]] INFO [2019-12-17 12:33:42,342] jepsen worker 81 - jepsen.util 9981 :invoke :cas [27 [6 47]] INFO [2019-12-17 12:33:42,353] jepsen worker 27 - jepsen.util 7407 :ok :read [9 12] INFO [2019-12-17 12:33:42,353] jepsen worker 27 - jepsen.util 7407 :invoke :read [9 nil] INFO [2019-12-17 12:33:42,360] jepsen worker 57 - jepsen.util 10047 :info :write [19 43] :RequestCanceledException INFO [2019-12-17 12:33:42,360] jepsen worker 57 - jepsen.util 10137 :invoke :cas [19 [12 45]] INFO [2019-12-17 12:33:42,381] jepsen worker 52 - jepsen.util 7702 :ok :read [17 23] INFO [2019-12-17 12:33:42,381] jepsen worker 52 - jepsen.util 7702 :invoke :cas [17 [15 40]] INFO [2019-12-17 12:33:42,394] jepsen worker 30 - jepsen.util 7320 :fail :cas [10 [41 39]] :ValueNotSwapFrom INFO [2019-12-17 12:33:42,394] jepsen worker 30 - jepsen.util 7320 :invoke :read [10 nil] INFO [2019-12-17 12:33:42,395] jepsen worker 29 - jepsen.util 5789 :info :write [9 0] :RequestCanceledException INFO [2019-12-17 12:33:42,396] jepsen worker 29 - jepsen.util 5879 :invoke :write [9 43] INFO [2019-12-17 12:33:42,407] jepsen worker 28 - jepsen.util 5878 :ok :write [9 5] INFO [2019-12-17 12:33:42,407] jepsen worker 28 - jepsen.util 5878 :invoke :write [9 47] INFO [2019-12-17 12:33:42,409] jepsen worker 5 - jepsen.util 7025 :fail :cas [1 [35 5]] :ValueNotSwapFrom INFO [2019-12-17 12:33:42,409] jepsen worker 5 - jepsen.util 7025 :invoke :write [1 22] INFO [2019-12-17 12:33:42,413] jepsen worker 17 - jepsen.util 10367 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:33:42,413] jepsen worker 17 - jepsen.util 10367 :invoke :cas [5 [33 11]] INFO [2019-12-17 12:33:42,415] jepsen worker 27 - jepsen.util 7407 :ok :read [9 12] INFO [2019-12-17 12:33:42,415] jepsen worker 27 - jepsen.util 7407 :invoke :read [9 nil] INFO [2019-12-17 12:33:42,428] jepsen worker 76 - jepsen.util 10066 :info :write [25 12] :RequestCanceledException INFO [2019-12-17 12:33:42,428] jepsen worker 48 - jepsen.util 9228 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:33:42,428] jepsen worker 48 - jepsen.util 9228 :invoke :write [16 30] INFO [2019-12-17 12:33:42,429] jepsen worker 76 - jepsen.util 10156 :invoke :cas [25 [36 27]] INFO [2019-12-17 12:33:42,436] jepsen worker 27 - jepsen.util 7407 :ok :read [9 5] INFO [2019-12-17 12:33:42,436] jepsen worker 27 - jepsen.util 7407 :invoke :write [9 47] INFO [2019-12-17 12:33:42,440] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":43067,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:33:42,440] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":43067,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:33:42,440] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":43070,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:33:42,440] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":43070,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:33:42,440] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":43071,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:33:42,441] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":43071,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:33:42,462] jepsen worker 65 - jepsen.util 6185 :ok :write [21 28] INFO [2019-12-17 12:33:42,462] jepsen worker 65 - jepsen.util 6185 :invoke :cas [21 [39 1]] INFO [2019-12-17 12:33:42,472] jepsen worker 29 - jepsen.util 5879 :ok :write [9 43] INFO [2019-12-17 12:33:42,472] jepsen worker 29 - jepsen.util 5879 :invoke :read [9 nil] INFO [2019-12-17 12:33:42,495] jepsen worker 39 - jepsen.util 9849 :info :cas [13 [8 27]] :RequestCanceledException INFO [2019-12-17 12:33:42,496] jepsen worker 39 - jepsen.util 9939 :invoke :read [13 nil] INFO [2019-12-17 12:33:42,498] jepsen worker 52 - jepsen.util 7702 :fail :cas [17 [15 40]] :ValueNotSwapFrom INFO [2019-12-17 12:33:42,498] jepsen worker 52 - jepsen.util 7702 :invoke :write [17 39] INFO [2019-12-17 12:33:42,507] jepsen worker 65 - jepsen.util 6185 :fail :cas [21 [39 1]] :ValueNotSwapFrom INFO [2019-12-17 12:33:42,507] jepsen worker 65 - jepsen.util 6185 :invoke :write [21 11] INFO [2019-12-17 12:33:42,529] jepsen worker 70 - jepsen.util 7720 :info :write [23 49] :RequestCanceledException INFO [2019-12-17 12:33:42,530] jepsen worker 70 - jepsen.util 7810 :invoke :write [23 24] INFO [2019-12-17 12:33:42,531] jepsen worker 3 - jepsen.util 7473 :ok :read [1 36] INFO [2019-12-17 12:33:42,532] jepsen worker 3 - jepsen.util 7473 :invoke :read [1 nil] INFO [2019-12-17 12:33:42,549] jepsen worker 29 - jepsen.util 5879 :ok :read [9 43] INFO [2019-12-17 12:33:42,549] jepsen worker 29 - jepsen.util 5879 :invoke :read [9 nil] INFO [2019-12-17 12:33:42,574] jepsen worker 29 - jepsen.util 5879 :ok :read [9 43] INFO [2019-12-17 12:33:42,574] jepsen worker 29 - jepsen.util 5879 :invoke :cas [9 [18 28]] INFO [2019-12-17 12:33:42,608] jepsen worker 29 - jepsen.util 5879 :fail :cas [9 [18 28]] :ValueNotSwapFrom INFO [2019-12-17 12:33:42,608] jepsen worker 29 - jepsen.util 5879 :invoke :write [9 2] INFO [2019-12-17 12:33:42,611] jepsen worker 27 - jepsen.util 7407 :ok :write [9 47] INFO [2019-12-17 12:33:42,611] jepsen worker 27 - jepsen.util 7407 :invoke :write [9 36] INFO [2019-12-17 12:33:42,616] jepsen worker 65 - jepsen.util 6185 :ok :write [21 11] INFO [2019-12-17 12:33:42,617] jepsen worker 65 - jepsen.util 6185 :invoke :cas [21 [46 4]] INFO [2019-12-17 12:33:42,622] jepsen worker 71 - jepsen.util 8171 :fail :cas [23 [42 18]] :ValueNotSwapFrom INFO [2019-12-17 12:33:42,622] jepsen worker 71 - jepsen.util 8171 :invoke :cas [23 [42 7]] INFO [2019-12-17 12:33:42,628] jepsen worker 79 - jepsen.util 7459 :info :write [26 18] :RequestCanceledException INFO [2019-12-17 12:33:42,628] jepsen worker 7 - jepsen.util 10267 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:33:42,628] jepsen worker 7 - jepsen.util 10267 :invoke :write [2 34] INFO [2019-12-17 12:33:42,629] jepsen worker 79 - jepsen.util 7549 :invoke :write [26 4] INFO [2019-12-17 12:33:42,638] jepsen worker 65 - jepsen.util 6185 :fail :cas [21 [46 4]] :ValueNotSwapFrom INFO [2019-12-17 12:33:42,638] jepsen worker 65 - jepsen.util 6185 :invoke :read [21 nil] INFO [2019-12-17 12:33:42,642] jepsen worker 16 - jepsen.util 11266 :info :write [5 48] :RequestCanceledException INFO [2019-12-17 12:33:42,642] jepsen worker 85 - jepsen.util 10975 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:33:42,642] jepsen worker 85 - jepsen.util 10975 :invoke :read [28 nil] INFO [2019-12-17 12:33:42,643] jepsen worker 16 - jepsen.util 11356 :invoke :cas [5 [46 31]] INFO [2019-12-17 12:33:42,658] jepsen worker 35 - jepsen.util 9935 :info :write [11 2] :RequestCanceledException INFO [2019-12-17 12:33:42,658] jepsen worker 40 - jepsen.util 10480 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:33:42,658] jepsen worker 40 - jepsen.util 10480 :invoke :write [13 48] INFO [2019-12-17 12:33:42,659] jepsen worker 35 - jepsen.util 10025 :invoke :read [11 nil] INFO [2019-12-17 12:33:42,668] jepsen worker 73 - jepsen.util 7183 :ok :write [24 20] INFO [2019-12-17 12:33:42,668] jepsen worker 73 - jepsen.util 7183 :invoke :read [24 nil] INFO [2019-12-17 12:33:42,694] jepsen worker 49 - jepsen.util 9499 :info :cas [16 [24 40]] :RequestCanceledException INFO [2019-12-17 12:33:42,694] jepsen worker 44 - jepsen.util 10754 :info :cas [14 [4 44]] :RequestCanceledException INFO [2019-12-17 12:33:42,695] jepsen worker 49 - jepsen.util 9589 :invoke :cas [16 [12 17]] INFO [2019-12-17 12:33:42,695] jepsen worker 44 - jepsen.util 10844 :invoke :cas [14 [2 29]] INFO [2019-12-17 12:33:42,697] jepsen worker 31 - jepsen.util 6601 :fail :cas [10 [14 18]] :ValueNotSwapFrom INFO [2019-12-17 12:33:42,697] jepsen worker 31 - jepsen.util 6601 :invoke :cas [10 [7 7]] INFO [2019-12-17 12:33:42,701] jepsen worker 65 - jepsen.util 6185 :ok :read [21 11] INFO [2019-12-17 12:33:42,701] jepsen worker 65 - jepsen.util 6185 :invoke :write [21 5] INFO [2019-12-17 12:33:42,712] jepsen worker 61 - jepsen.util 10771 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:33:42,712] jepsen worker 61 - jepsen.util 10771 :invoke :write [20 12] INFO [2019-12-17 12:33:42,721] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":43093,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:33:42,721] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":43093,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:33:42,721] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":43095,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:33:42,721] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":43095,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:33:42,721] jepsen worker 29 - jepsen.util 5879 :ok :write [9 2] INFO [2019-12-17 12:33:42,722] jepsen worker 29 - jepsen.util 5879 :invoke :cas [9 [12 6]] INFO [2019-12-17 12:33:42,727] jepsen worker 8 - jepsen.util 10088 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:33:42,728] jepsen worker 8 - jepsen.util 10088 :invoke :read [2 nil] INFO [2019-12-17 12:33:42,749] jepsen worker 89 - jepsen.util 7289 :ok :write [29 0] INFO [2019-12-17 12:33:42,749] jepsen worker 89 - jepsen.util 7289 :invoke :write [29 14] INFO [2019-12-17 12:33:42,756] jepsen worker 73 - jepsen.util 7183 :ok :read [24 20] INFO [2019-12-17 12:33:42,756] jepsen worker 73 - jepsen.util 7183 :invoke :read [24 nil] INFO [2019-12-17 12:33:42,757] jepsen worker 20 - jepsen.util 7220 :ok :write [6 16] INFO [2019-12-17 12:33:42,757] jepsen worker 20 - jepsen.util 7220 :invoke :write [6 32] INFO [2019-12-17 12:33:42,791] jepsen worker 78 - jepsen.util 7638 :fail :cas [26 [30 26]] :ValueNotSwapFrom INFO [2019-12-17 12:33:42,791] jepsen worker 78 - jepsen.util 7638 :invoke :cas [26 [33 12]] INFO [2019-12-17 12:33:42,804] jepsen worker 29 - jepsen.util 5879 :fail :cas [9 [12 6]] :ValueNotSwapFrom INFO [2019-12-17 12:33:42,805] jepsen worker 29 - jepsen.util 5879 :invoke :read [9 nil] INFO [2019-12-17 12:33:42,808] jepsen worker 30 - jepsen.util 7320 :ok :read [10 11] INFO [2019-12-17 12:33:42,808] jepsen worker 30 - jepsen.util 7320 :invoke :write [10 16] INFO [2019-12-17 12:33:42,811] jepsen worker 31 - jepsen.util 6601 :fail :cas [10 [7 7]] :ValueNotSwapFrom INFO [2019-12-17 12:33:42,811] jepsen worker 31 - jepsen.util 6601 :invoke :read [10 nil] INFO [2019-12-17 12:33:42,813] jepsen worker 14 - jepsen.util 7664 :info :write [4 1] :RequestCanceledException INFO [2019-12-17 12:33:42,814] jepsen worker 14 - jepsen.util 7754 :invoke :read [4 nil] INFO [2019-12-17 12:33:42,828] jepsen worker 63 - jepsen.util 6273 :info :cas [21 [49 22]] :RequestCanceledException INFO [2019-12-17 12:33:42,829] jepsen worker 63 - jepsen.util 6363 :invoke :read [21 nil] INFO [2019-12-17 12:33:42,859] jepsen worker 29 - jepsen.util 5879 :ok :read [9 2] INFO [2019-12-17 12:33:42,859] jepsen worker 29 - jepsen.util 5879 :invoke :cas [9 [23 41]] INFO [2019-12-17 12:33:42,875] jepsen worker 65 - jepsen.util 6185 :ok :write [21 5] INFO [2019-12-17 12:33:42,875] jepsen worker 65 - jepsen.util 6185 :invoke :cas [21 [26 18]] INFO [2019-12-17 12:33:42,875] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":43123,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:33:42,876] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":43123,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:33:42,876] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":43124,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:33:42,876] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":43124,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:33:42,877] jepsen worker 73 - jepsen.util 7183 :ok :read [24 20] INFO [2019-12-17 12:33:42,877] jepsen worker 73 - jepsen.util 7183 :invoke :cas [24 [27 42]] INFO [2019-12-17 12:33:42,881] jepsen worker 78 - jepsen.util 7638 :fail :cas [26 [33 12]] :ValueNotSwapFrom INFO [2019-12-17 12:33:42,881] jepsen worker 78 - jepsen.util 7638 :invoke :read [26 nil] INFO [2019-12-17 12:33:42,901] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:33:42,901] jepsen nemesis - couchbase.util Rebalance check count: 330 INFO [2019-12-17 12:33:42,901] jepsen nemesis - couchbase.util Rebalance stuck count: 311 INFO [2019-12-17 12:33:42,903] jepsen worker 65 - jepsen.util 6185 :fail :cas [21 [26 18]] :ValueNotSwapFrom INFO [2019-12-17 12:33:42,903] jepsen worker 65 - jepsen.util 6185 :invoke :read [21 nil] INFO [2019-12-17 12:33:42,928] jepsen worker 32 - jepsen.util 7412 :fail :read [10 nil] :RequestTimeoutException INFO [2019-12-17 12:33:42,928] jepsen worker 4 - jepsen.util 6844 :info :write [1 37] :RequestCanceledException INFO [2019-12-17 12:33:42,928] jepsen worker 32 - jepsen.util 7412 :invoke :write [10 39] INFO [2019-12-17 12:33:42,928] jepsen worker 4 - jepsen.util 6934 :invoke :write [1 33] INFO [2019-12-17 12:33:42,929] jepsen worker 19 - jepsen.util 7489 :info :write [6 41] :RequestCanceledException INFO [2019-12-17 12:33:42,929] jepsen worker 19 - jepsen.util 7579 :invoke :write [6 41] INFO [2019-12-17 12:33:42,969] jepsen worker 31 - jepsen.util 6601 :ok :read [10 11] INFO [2019-12-17 12:33:42,969] jepsen worker 31 - jepsen.util 6601 :invoke :write [10 3] INFO [2019-12-17 12:33:42,976] jepsen worker 29 - jepsen.util 5879 :fail :cas [9 [23 41]] :ValueNotSwapFrom INFO [2019-12-17 12:33:42,976] jepsen worker 29 - jepsen.util 5879 :invoke :cas [9 [30 13]] INFO [2019-12-17 12:33:42,982] jepsen worker 65 - jepsen.util 6185 :ok :read [21 5] INFO [2019-12-17 12:33:42,982] jepsen worker 65 - jepsen.util 6185 :invoke :write [21 28] INFO [2019-12-17 12:33:43,012] jepsen worker 87 - jepsen.util 6747 :info :write [29 22] :RequestCanceledException INFO [2019-12-17 12:33:43,013] jepsen worker 63 - jepsen.util 6363 :ok :read [21 5] INFO [2019-12-17 12:33:43,013] jepsen worker 63 - jepsen.util 6363 :invoke :cas [21 [36 30]] INFO [2019-12-17 12:33:43,013] jepsen worker 87 - jepsen.util 6837 :invoke :read [29 nil] INFO [2019-12-17 12:33:43,028] jepsen worker 13 - jepsen.util 7033 :info :write [4 27] :RequestCanceledException INFO [2019-12-17 12:33:43,029] jepsen worker 13 - jepsen.util 7123 :invoke :write [4 14] INFO [2019-12-17 12:33:43,041] jepsen worker 80 - jepsen.util 7460 :ok :write [26 38] INFO [2019-12-17 12:33:43,041] jepsen worker 80 - jepsen.util 7460 :invoke :cas [26 [13 39]] INFO [2019-12-17 12:33:43,043] jepsen worker 45 - jepsen.util 7335 :info :cas [15 [0 40]] :RequestCanceledException INFO [2019-12-17 12:33:43,044] jepsen worker 45 - jepsen.util 7425 :invoke :read [15 nil] INFO [2019-12-17 12:33:43,046] jepsen worker 78 - jepsen.util 7638 :ok :read [26 20] INFO [2019-12-17 12:33:43,046] jepsen worker 78 - jepsen.util 7638 :invoke :read [26 nil] INFO [2019-12-17 12:33:43,052] jepsen worker 73 - jepsen.util 7183 :fail :cas [24 [27 42]] :ValueNotSwapFrom INFO [2019-12-17 12:33:43,052] jepsen worker 73 - jepsen.util 7183 :invoke :cas [24 [9 8]] INFO [2019-12-17 12:33:43,059] jepsen worker 74 - jepsen.util 7544 :info :write [24 44] :RequestCanceledException INFO [2019-12-17 12:33:43,059] jepsen worker 46 - jepsen.util 7786 :info :write [15 5] :RequestCanceledException INFO [2019-12-17 12:33:43,059] jepsen worker 2 - jepsen.util 9812 :info :write [0 16] :RequestCanceledException INFO [2019-12-17 12:33:43,060] jepsen worker 46 - jepsen.util 7876 :invoke :cas [15 [48 23]] INFO [2019-12-17 12:33:43,060] jepsen worker 2 - jepsen.util 9902 :invoke :read [0 nil] INFO [2019-12-17 12:33:43,063] jepsen worker 74 - jepsen.util 7634 :invoke :read [24 nil] INFO [2019-12-17 12:33:43,085] jepsen worker 45 - jepsen.util 7425 :ok :read [15 24] INFO [2019-12-17 12:33:43,085] jepsen worker 45 - jepsen.util 7425 :invoke :write [15 11] INFO [2019-12-17 12:33:43,095] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":43135,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:33:43,095] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":43135,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:33:43,095] jepsen worker 11 - jepsen.util 7571 :fail :read [3 nil] :RequestTimeoutException INFO [2019-12-17 12:33:43,095] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":43139,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:33:43,095] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":43139,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:33:43,095] jepsen worker 11 - jepsen.util 7571 :invoke :write [3 19] INFO [2019-12-17 12:33:43,095] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":43140,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:33:43,095] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":43140,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:33:43,102] jepsen worker 51 - jepsen.util 7251 :ok :write [17 7] INFO [2019-12-17 12:33:43,102] jepsen worker 51 - jepsen.util 7251 :invoke :read [17 nil] INFO [2019-12-17 12:33:43,111] jepsen worker 4 - jepsen.util 6934 :ok :write [1 33] INFO [2019-12-17 12:33:43,111] jepsen worker 4 - jepsen.util 6934 :invoke :cas [1 [44 14]] INFO [2019-12-17 12:33:43,129] jepsen worker 69 - jepsen.util 7629 :info :write [23 35] :RequestCanceledException INFO [2019-12-17 12:33:43,130] jepsen worker 69 - jepsen.util 7719 :invoke :write [23 35] INFO [2019-12-17 12:33:43,137] jepsen worker 63 - jepsen.util 6363 :fail :cas [21 [36 30]] :ValueNotSwapFrom INFO [2019-12-17 12:33:43,137] jepsen worker 63 - jepsen.util 6363 :invoke :write [21 18] INFO [2019-12-17 12:33:43,145] jepsen worker 73 - jepsen.util 7183 :fail :cas [24 [9 8]] :ValueNotSwapFrom INFO [2019-12-17 12:33:43,146] jepsen worker 73 - jepsen.util 7183 :invoke :read [24 nil] INFO [2019-12-17 12:33:43,162] jepsen worker 46 - jepsen.util 7876 :fail :cas [15 [48 23]] :ValueNotSwapFrom INFO [2019-12-17 12:33:43,162] jepsen worker 46 - jepsen.util 7876 :invoke :write [15 33] INFO [2019-12-17 12:33:43,173] jepsen worker 4 - jepsen.util 6934 :fail :cas [1 [44 14]] :ValueNotSwapFrom INFO [2019-12-17 12:33:43,173] jepsen worker 4 - jepsen.util 6934 :invoke :cas [1 [43 13]] INFO [2019-12-17 12:33:43,197] jepsen worker 32 - jepsen.util 7412 :ok :write [10 39] INFO [2019-12-17 12:33:43,198] jepsen worker 32 - jepsen.util 7412 :invoke :write [10 7] INFO [2019-12-17 12:33:43,204] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":43160,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:33:43,205] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":43160,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:33:43,205] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":43161,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:33:43,205] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":43161,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:33:43,220] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":43169,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:33:43,221] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":43169,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:33:43,221] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":43172,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:33:43,221] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":43172,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:33:43,226] jepsen worker 3 - jepsen.util 7473 :ok :read [1 33] INFO [2019-12-17 12:33:43,226] jepsen worker 3 - jepsen.util 7473 :invoke :read [1 nil] INFO [2019-12-17 12:33:43,228] jepsen worker 72 - jepsen.util 7272 :fail :read [24 nil] :RequestTimeoutException INFO [2019-12-17 12:33:43,228] jepsen worker 54 - jepsen.util 8334 :info :cas [18 [3 32]] :RequestCanceledException INFO [2019-12-17 12:33:43,228] jepsen worker 72 - jepsen.util 7272 :invoke :read [24 nil] INFO [2019-12-17 12:33:43,229] jepsen worker 54 - jepsen.util 8424 :invoke :read [18 nil] INFO [2019-12-17 12:33:43,249] jepsen worker 63 - jepsen.util 6363 :ok :write [21 18] INFO [2019-12-17 12:33:43,249] jepsen worker 63 - jepsen.util 6363 :invoke :write [21 49] INFO [2019-12-17 12:33:43,259] jepsen worker 88 - jepsen.util 7378 :info :write [29 45] :RequestCanceledException INFO [2019-12-17 12:33:43,260] jepsen worker 88 - jepsen.util 7468 :invoke :cas [29 [24 1]] INFO [2019-12-17 12:33:43,261] jepsen worker 53 - jepsen.util 6983 :ok :write [17 47] INFO [2019-12-17 12:33:43,261] jepsen worker 53 - jepsen.util 6983 :invoke :cas [17 [45 42]] INFO [2019-12-17 12:33:43,293] jepsen worker 12 - jepsen.util 8022 :ok :write [4 21] INFO [2019-12-17 12:33:43,296] jepsen worker 12 - jepsen.util 8022 :invoke :write [4 35] INFO [2019-12-17 12:33:43,300] jepsen worker 70 - jepsen.util 7810 :ok :write [23 24] INFO [2019-12-17 12:33:43,300] jepsen worker 70 - jepsen.util 7810 :invoke :cas [23 [44 39]] INFO [2019-12-17 12:33:43,301] jepsen worker 36 - jepsen.util 7776 :ok :write [12 49] INFO [2019-12-17 12:33:43,301] jepsen worker 36 - jepsen.util 7776 :invoke :cas [12 [19 35]] INFO [2019-12-17 12:33:43,312] jepsen worker 9 - jepsen.util 7929 :fail :read [3 nil] :RequestTimeoutException INFO [2019-12-17 12:33:43,313] jepsen worker 9 - jepsen.util 7929 :invoke :read [3 nil] INFO [2019-12-17 12:33:43,372] jepsen worker 45 - jepsen.util 7425 :ok :write [15 11] INFO [2019-12-17 12:33:43,372] jepsen worker 45 - jepsen.util 7425 :invoke :write [15 6] INFO [2019-12-17 12:33:43,382] jepsen worker 20 - jepsen.util 7220 :ok :write [6 32] INFO [2019-12-17 12:33:43,382] jepsen worker 20 - jepsen.util 7220 :invoke :write [6 8] INFO [2019-12-17 12:33:43,392] jepsen worker 71 - jepsen.util 8171 :fail :cas [23 [42 7]] :ValueNotSwapFrom INFO [2019-12-17 12:33:43,392] jepsen worker 71 - jepsen.util 8171 :invoke :read [23 nil] INFO [2019-12-17 12:33:43,412] jepsen worker 9 - jepsen.util 7929 :ok :read [3 14] INFO [2019-12-17 12:33:43,412] jepsen worker 9 - jepsen.util 7929 :invoke :read [3 nil] INFO [2019-12-17 12:33:43,428] jepsen worker 10 - jepsen.util 7390 :fail :read [3 nil] :RequestTimeoutException INFO [2019-12-17 12:33:43,428] jepsen worker 10 - jepsen.util 7390 :invoke :read [3 nil] INFO [2019-12-17 12:33:43,432] jepsen worker 87 - jepsen.util 6837 :ok :read [29 0] INFO [2019-12-17 12:33:43,432] jepsen worker 87 - jepsen.util 6837 :invoke :write [29 34] INFO [2019-12-17 12:33:43,435] jepsen worker 14 - jepsen.util 7754 :ok :read [4 21] INFO [2019-12-17 12:33:43,435] jepsen worker 14 - jepsen.util 7754 :invoke :cas [4 [30 49]] INFO [2019-12-17 12:33:43,442] jepsen worker 47 - jepsen.util 7607 :info :write [15 33] :RequestCanceledException INFO [2019-12-17 12:33:43,442] jepsen worker 53 - jepsen.util 6983 :fail :cas [17 [45 42]] :ValueNotSwapFrom INFO [2019-12-17 12:33:43,442] jepsen worker 53 - jepsen.util 6983 :invoke :read [17 nil] INFO [2019-12-17 12:33:43,443] jepsen worker 47 - jepsen.util 7697 :invoke :read [15 nil] INFO [2019-12-17 12:33:43,446] jepsen worker 78 - jepsen.util 7638 :ok :read [26 38] INFO [2019-12-17 12:33:43,446] jepsen worker 78 - jepsen.util 7638 :invoke :write [26 19] INFO [2019-12-17 12:33:43,456] jepsen worker 47 - jepsen.util 7697 :ok :read [15 11] INFO [2019-12-17 12:33:43,456] jepsen worker 47 - jepsen.util 7697 :invoke :read [15 nil] INFO [2019-12-17 12:33:43,469] jepsen worker 47 - jepsen.util 7697 :ok :read [15 11] INFO [2019-12-17 12:33:43,469] jepsen worker 47 - jepsen.util 7697 :invoke :write [15 21] INFO [2019-12-17 12:33:43,489] jepsen worker 36 - jepsen.util 7776 :fail :cas [12 [19 35]] :ValueNotSwapFrom INFO [2019-12-17 12:33:43,489] jepsen worker 36 - jepsen.util 7776 :invoke :cas [12 [15 41]] INFO [2019-12-17 12:33:43,495] jepsen worker 18 - jepsen.util 7578 :info :cas [6 [46 32]] :RequestCanceledException INFO [2019-12-17 12:33:43,496] jepsen worker 18 - jepsen.util 7668 :invoke :cas [6 [9 20]] INFO [2019-12-17 12:33:43,514] jepsen worker 18 - jepsen.util 7668 :fail :cas [6 [9 20]] :ValueNotSwapFrom INFO [2019-12-17 12:33:43,514] jepsen worker 18 - jepsen.util 7668 :invoke :write [6 9] INFO [2019-12-17 12:33:43,520] jepsen worker 3 - jepsen.util 7473 :ok :read [1 33] INFO [2019-12-17 12:33:43,521] jepsen worker 3 - jepsen.util 7473 :invoke :write [1 38] INFO [2019-12-17 12:33:43,530] jepsen worker 53 - jepsen.util 6983 :ok :read [17 47] INFO [2019-12-17 12:33:43,530] jepsen worker 53 - jepsen.util 6983 :invoke :read [17 nil] INFO [2019-12-17 12:33:43,531] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":43189,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:33:43,532] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":43189,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:33:43,539] jepsen worker 32 - jepsen.util 7412 :ok :write [10 7] INFO [2019-12-17 12:33:43,539] jepsen worker 32 - jepsen.util 7412 :invoke :read [10 nil] INFO [2019-12-17 12:33:43,543] jepsen worker 20 - jepsen.util 7220 :ok :write [6 8] INFO [2019-12-17 12:33:43,543] jepsen worker 20 - jepsen.util 7220 :invoke :read [6 nil] INFO [2019-12-17 12:33:43,544] jepsen worker 45 - jepsen.util 7425 :ok :write [15 6] INFO [2019-12-17 12:33:43,544] jepsen worker 45 - jepsen.util 7425 :invoke :read [15 nil] INFO [2019-12-17 12:33:43,559] jepsen worker 56 - jepsen.util 8336 :info :write [18 38] :RequestCanceledException INFO [2019-12-17 12:33:43,563] jepsen worker 56 - jepsen.util 8426 :invoke :read [18 nil] INFO [2019-12-17 12:33:43,570] jepsen worker 20 - jepsen.util 7220 :ok :read [6 8] INFO [2019-12-17 12:33:43,570] jepsen worker 20 - jepsen.util 7220 :invoke :read [6 nil] INFO [2019-12-17 12:33:43,585] jepsen worker 45 - jepsen.util 7425 :ok :read [15 6] INFO [2019-12-17 12:33:43,585] jepsen worker 45 - jepsen.util 7425 :invoke :write [15 7] INFO [2019-12-17 12:33:43,592] jepsen worker 72 - jepsen.util 7272 :ok :read [24 20] INFO [2019-12-17 12:33:43,592] jepsen worker 72 - jepsen.util 7272 :invoke :cas [24 [32 6]] INFO [2019-12-17 12:33:43,595] jepsen worker 20 - jepsen.util 7220 :ok :read [6 8] INFO [2019-12-17 12:33:43,595] jepsen worker 20 - jepsen.util 7220 :invoke :cas [6 [20 1]] INFO [2019-12-17 12:33:43,606] jepsen worker 80 - jepsen.util 7460 :fail :cas [26 [13 39]] :ValueNotSwapFrom INFO [2019-12-17 12:33:43,606] jepsen worker 80 - jepsen.util 7460 :invoke :write [26 12] INFO [2019-12-17 12:33:43,613] jepsen worker 10 - jepsen.util 7390 :ok :read [3 14] INFO [2019-12-17 12:33:43,613] jepsen worker 10 - jepsen.util 7390 :invoke :write [3 40] INFO [2019-12-17 12:33:43,629] jepsen worker 64 - jepsen.util 5824 :fail :read [21 nil] :RequestTimeoutException INFO [2019-12-17 12:33:43,629] jepsen worker 64 - jepsen.util 5824 :invoke :cas [21 [6 19]] INFO [2019-12-17 12:33:43,638] jepsen worker 32 - jepsen.util 7412 :ok :read [10 7] INFO [2019-12-17 12:33:43,638] jepsen worker 32 - jepsen.util 7412 :invoke :cas [10 [30 14]] INFO [2019-12-17 12:33:43,640] jepsen worker 20 - jepsen.util 7220 :fail :cas [6 [20 1]] :ValueNotSwapFrom INFO [2019-12-17 12:33:43,640] jepsen worker 20 - jepsen.util 7220 :invoke :read [6 nil] INFO [2019-12-17 12:33:43,655] jepsen worker 72 - jepsen.util 7272 :fail :cas [24 [32 6]] :ValueNotSwapFrom INFO [2019-12-17 12:33:43,656] jepsen worker 72 - jepsen.util 7272 :invoke :read [24 nil] INFO [2019-12-17 12:33:43,658] jepsen worker 4 - jepsen.util 6934 :fail :cas [1 [43 13]] :ValueNotSwapFrom INFO [2019-12-17 12:33:43,659] jepsen worker 4 - jepsen.util 6934 :invoke :read [1 nil] INFO [2019-12-17 12:33:43,665] jepsen worker 20 - jepsen.util 7220 :ok :read [6 8] INFO [2019-12-17 12:33:43,665] jepsen worker 20 - jepsen.util 7220 :invoke :write [6 11] INFO [2019-12-17 12:33:43,695] jepsen worker 53 - jepsen.util 6983 :ok :read [17 47] INFO [2019-12-17 12:33:43,695] jepsen worker 53 - jepsen.util 6983 :invoke :cas [17 [3 1]] INFO [2019-12-17 12:33:43,708] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":43222,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:33:43,708] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":43222,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:33:43,708] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":43224,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:33:43,708] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":43224,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:33:43,794] jepsen worker 51 - jepsen.util 7251 :ok :read [17 47] INFO [2019-12-17 12:33:43,794] jepsen worker 51 - jepsen.util 7251 :invoke :cas [17 [16 8]] INFO [2019-12-17 12:33:43,795] jepsen worker 56 - jepsen.util 8426 :ok :read [18 27] INFO [2019-12-17 12:33:43,795] jepsen worker 56 - jepsen.util 8426 :invoke :cas [18 [13 6]] INFO [2019-12-17 12:33:43,815] jepsen worker 88 - jepsen.util 7468 :fail :cas [29 [24 1]] :ValueNotSwapFrom INFO [2019-12-17 12:33:43,815] jepsen worker 88 - jepsen.util 7468 :invoke :read [29 nil] INFO [2019-12-17 12:33:43,842] jepsen worker 38 - jepsen.util 7688 :info :write [12 9] :RequestCanceledException INFO [2019-12-17 12:33:43,842] jepsen worker 38 - jepsen.util 7778 :invoke :read [12 nil] INFO [2019-12-17 12:33:43,847] jepsen worker 70 - jepsen.util 7810 :fail :cas [23 [44 39]] :ValueNotSwapFrom INFO [2019-12-17 12:33:43,847] jepsen worker 70 - jepsen.util 7810 :invoke :write [23 28] INFO [2019-12-17 12:33:43,860] jepsen worker 34 - jepsen.util 9574 :info :cas [11 [37 29]] :RequestCanceledException INFO [2019-12-17 12:33:43,861] jepsen worker 34 - jepsen.util 9664 :invoke :cas [11 [48 43]] INFO [2019-12-17 12:33:43,871] jepsen worker 71 - jepsen.util 8171 :ok :read [23 24] INFO [2019-12-17 12:33:43,871] jepsen worker 71 - jepsen.util 8171 :invoke :cas [23 [49 24]] INFO [2019-12-17 12:33:43,877] jepsen worker 4 - jepsen.util 6934 :ok :read [1 33] INFO [2019-12-17 12:33:43,877] jepsen worker 4 - jepsen.util 6934 :invoke :cas [1 [32 14]] INFO [2019-12-17 12:33:43,895] jepsen worker 26 - jepsen.util 10466 :info :write [8 44] :RequestCanceledException INFO [2019-12-17 12:33:43,896] jepsen worker 26 - jepsen.util 10556 :invoke :write [8 46] INFO [2019-12-17 12:33:43,913] jepsen worker 43 - jepsen.util 9673 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:33:43,913] jepsen worker 43 - jepsen.util 9673 :invoke :write [14 47] INFO [2019-12-17 12:33:43,913] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:33:43,914] jepsen nemesis - couchbase.util Rebalance check count: 331 INFO [2019-12-17 12:33:43,914] jepsen nemesis - couchbase.util Rebalance stuck count: 312 INFO [2019-12-17 12:33:43,928] jepsen worker 83 - jepsen.util 10523 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:33:43,928] jepsen worker 83 - jepsen.util 10523 :invoke :read [27 nil] INFO [2019-12-17 12:33:43,937] jepsen worker 14 - jepsen.util 7754 :fail :cas [4 [30 49]] :ValueNotSwapFrom INFO [2019-12-17 12:33:43,938] jepsen worker 14 - jepsen.util 7754 :invoke :read [4 nil] INFO [2019-12-17 12:33:43,942] jepsen worker 21 - jepsen.util 9561 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:33:43,942] jepsen worker 55 - jepsen.util 8335 :info :write [18 5] :RequestCanceledException INFO [2019-12-17 12:33:43,942] jepsen worker 42 - jepsen.util 10752 :info :cas [14 [46 5]] :RequestCanceledException INFO [2019-12-17 12:33:43,943] jepsen worker 21 - jepsen.util 9561 :invoke :write [7 49] INFO [2019-12-17 12:33:43,943] jepsen worker 55 - jepsen.util 8425 :invoke :read [18 nil] INFO [2019-12-17 12:33:43,943] jepsen worker 42 - jepsen.util 10842 :invoke :write [14 18] INFO [2019-12-17 12:33:43,945] jepsen worker 53 - jepsen.util 6983 :fail :cas [17 [3 1]] :ValueNotSwapFrom INFO [2019-12-17 12:33:43,945] jepsen worker 53 - jepsen.util 6983 :invoke :cas [17 [37 31]] INFO [2019-12-17 12:33:43,951] jepsen worker 36 - jepsen.util 7776 :fail :cas [12 [15 41]] :ValueNotSwapFrom INFO [2019-12-17 12:33:43,951] jepsen worker 36 - jepsen.util 7776 :invoke :write [12 44] INFO [2019-12-17 12:33:43,958] jepsen worker 4 - jepsen.util 6934 :fail :cas [1 [32 14]] :ValueNotSwapFrom INFO [2019-12-17 12:33:43,958] jepsen worker 75 - jepsen.util 9615 :info :write [25 17] :RequestCanceledException INFO [2019-12-17 12:33:43,958] jepsen worker 4 - jepsen.util 6934 :invoke :read [1 nil] INFO [2019-12-17 12:33:43,958] jepsen worker 75 - jepsen.util 9705 :invoke :cas [25 [49 40]] INFO [2019-12-17 12:33:43,970] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":43236,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:33:43,970] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":43236,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:33:43,970] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":43240,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:33:43,970] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":43240,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:33:43,994] jepsen worker 62 - jepsen.util 10232 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:33:43,994] jepsen worker 66 - jepsen.util 9966 :info :cas [22 [10 24]] :RequestCanceledException INFO [2019-12-17 12:33:43,994] jepsen worker 62 - jepsen.util 10232 :invoke :write [20 8] INFO [2019-12-17 12:33:43,995] jepsen worker 66 - jepsen.util 10056 :invoke :write [22 25] INFO [2019-12-17 12:33:44,010] jepsen worker 77 - jepsen.util 10337 :info :write [25 2] :RequestCanceledException INFO [2019-12-17 12:33:44,011] jepsen worker 77 - jepsen.util 10427 :invoke :write [25 30] INFO [2019-12-17 12:33:44,026] jepsen worker 56 - jepsen.util 8426 :fail :cas [18 [13 6]] :ValueNotSwapFrom INFO [2019-12-17 12:33:44,026] jepsen worker 56 - jepsen.util 8426 :invoke :read [18 nil] INFO [2019-12-17 12:33:44,028] jepsen worker 23 - jepsen.util 10283 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:33:44,028] jepsen worker 22 - jepsen.util 9922 :info :cas [7 [15 11]] :RequestCanceledException INFO [2019-12-17 12:33:44,028] jepsen worker 23 - jepsen.util 10283 :invoke :read [7 nil] INFO [2019-12-17 12:33:44,029] jepsen worker 22 - jepsen.util 10012 :invoke :read [7 nil] INFO [2019-12-17 12:33:44,042] jepsen worker 86 - jepsen.util 10706 :info :write [28 6] :RequestCanceledException INFO [2019-12-17 12:33:44,043] jepsen worker 86 - jepsen.util 10796 :invoke :cas [28 [27 29]] INFO [2019-12-17 12:33:44,058] jepsen worker 0 - jepsen.util 10530 :info :write [0 18] :RequestCanceledException INFO [2019-12-17 12:33:44,058] jepsen worker 41 - jepsen.util 10571 :info :write [13 26] :RequestCanceledException INFO [2019-12-17 12:33:44,059] jepsen worker 0 - jepsen.util 10620 :invoke :write [0 19] INFO [2019-12-17 12:33:44,059] jepsen worker 41 - jepsen.util 10661 :invoke :cas [13 [20 20]] INFO [2019-12-17 12:33:44,070] jepsen worker 88 - jepsen.util 7468 :ok :read [29 0] INFO [2019-12-17 12:33:44,070] jepsen worker 88 - jepsen.util 7468 :invoke :cas [29 [11 28]] INFO [2019-12-17 12:33:44,080] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":43254,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:33:44,080] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":43254,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:33:44,080] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":43255,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:33:44,080] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":43255,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:33:44,095] jepsen worker 67 - jepsen.util 10057 :info :cas [22 [14 37]] :RequestCanceledException INFO [2019-12-17 12:33:44,095] jepsen worker 58 - jepsen.util 10858 :info :cas [19 [24 11]] :RequestCanceledException INFO [2019-12-17 12:33:44,096] jepsen worker 67 - jepsen.util 10147 :invoke :write [22 15] INFO [2019-12-17 12:33:44,096] jepsen worker 58 - jepsen.util 10948 :invoke :cas [19 [28 15]] INFO [2019-12-17 12:33:44,112] jepsen worker 15 - jepsen.util 9105 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:33:44,113] jepsen worker 15 - jepsen.util 9105 :invoke :cas [5 [38 48]] INFO [2019-12-17 12:33:44,128] jepsen worker 68 - jepsen.util 9608 :info :cas [22 [35 29]] :RequestCanceledException INFO [2019-12-17 12:33:44,128] jepsen worker 59 - jepsen.util 9959 :info :write [19 25] :RequestCanceledException INFO [2019-12-17 12:33:44,128] jepsen worker 24 - jepsen.util 8844 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:33:44,128] jepsen worker 60 - jepsen.util 11040 :info :cas [20 [20 24]] :RequestCanceledException INFO [2019-12-17 12:33:44,128] jepsen worker 24 - jepsen.util 8844 :invoke :write [8 41] INFO [2019-12-17 12:33:44,129] jepsen worker 68 - jepsen.util 9698 :invoke :cas [22 [47 3]] INFO [2019-12-17 12:33:44,129] jepsen worker 60 - jepsen.util 11130 :invoke :cas [20 [39 27]] INFO [2019-12-17 12:33:44,129] jepsen worker 59 - jepsen.util 10049 :invoke :read [19 nil] INFO [2019-12-17 12:33:44,133] jepsen worker 53 - jepsen.util 6983 :fail :cas [17 [37 31]] :ValueNotSwapFrom INFO [2019-12-17 12:33:44,133] jepsen worker 53 - jepsen.util 6983 :invoke :cas [17 [2 29]] INFO [2019-12-17 12:33:44,141] jepsen worker 38 - jepsen.util 7778 :ok :read [12 49] INFO [2019-12-17 12:33:44,141] jepsen worker 38 - jepsen.util 7778 :invoke :read [12 nil] INFO [2019-12-17 12:33:44,142] jepsen worker 37 - jepsen.util 7057 :info :write [12 49] :RequestCanceledException INFO [2019-12-17 12:33:44,142] jepsen worker 50 - jepsen.util 10850 :info :cas [16 [26 4]] :RequestCanceledException INFO [2019-12-17 12:33:44,142] jepsen worker 50 - jepsen.util 10940 :invoke :write [16 35] INFO [2019-12-17 12:33:44,142] jepsen worker 37 - jepsen.util 7147 :invoke :read [12 nil] INFO [2019-12-17 12:33:44,159] jepsen worker 25 - jepsen.util 10555 :info :write [8 23] :RequestCanceledException INFO [2019-12-17 12:33:44,159] jepsen worker 33 - jepsen.util 9933 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:33:44,160] jepsen worker 33 - jepsen.util 9933 :invoke :cas [11 [22 19]] INFO [2019-12-17 12:33:44,160] jepsen worker 25 - jepsen.util 10645 :invoke :read [8 nil] INFO [2019-12-17 12:33:44,171] jepsen worker 56 - jepsen.util 8426 :ok :read [18 27] INFO [2019-12-17 12:33:44,171] jepsen worker 56 - jepsen.util 8426 :invoke :cas [18 [8 40]] INFO [2019-12-17 12:33:44,187] jepsen worker 54 - jepsen.util 8424 :ok :read [18 27] INFO [2019-12-17 12:33:44,187] jepsen worker 54 - jepsen.util 8424 :invoke :read [18 nil] INFO [2019-12-17 12:33:44,197] jepsen worker 4 - jepsen.util 6934 :ok :read [1 33] INFO [2019-12-17 12:33:44,197] jepsen worker 4 - jepsen.util 6934 :invoke :read [1 nil] INFO [2019-12-17 12:33:44,228] jepsen worker 53 - jepsen.util 6983 :fail :cas [17 [2 29]] :ValueNotSwapFrom INFO [2019-12-17 12:33:44,229] jepsen worker 53 - jepsen.util 6983 :invoke :read [17 nil] INFO [2019-12-17 12:33:44,229] jepsen worker 82 - jepsen.util 10702 :info :cas [27 [10 2]] :RequestCanceledException INFO [2019-12-17 12:33:44,229] jepsen worker 82 - jepsen.util 10792 :invoke :write [27 8] INFO [2019-12-17 12:33:44,265] jepsen worker 56 - jepsen.util 8426 :fail :cas [18 [8 40]] :ValueNotSwapFrom INFO [2019-12-17 12:33:44,265] jepsen worker 56 - jepsen.util 8426 :invoke :write [18 42] INFO [2019-12-17 12:33:44,271] jepsen worker 4 - jepsen.util 6934 :ok :read [1 33] INFO [2019-12-17 12:33:44,271] jepsen worker 4 - jepsen.util 6934 :invoke :read [1 nil] INFO [2019-12-17 12:33:44,289] jepsen worker 71 - jepsen.util 8171 :fail :cas [23 [49 24]] :ValueNotSwapFrom INFO [2019-12-17 12:33:44,289] jepsen worker 71 - jepsen.util 8171 :invoke :cas [23 [16 30]] INFO [2019-12-17 12:33:44,298] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":43285,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:33:44,299] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":43285,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:33:44,299] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":43287,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:33:44,299] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":43287,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:33:44,305] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":43293,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:33:44,305] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":43293,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:33:44,305] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":43296,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:33:44,305] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":43296,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:33:44,306] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":43297,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:33:44,306] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":43297,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:33:44,312] jepsen worker 1 - jepsen.util 10171 :info :write [0 44] :RequestCanceledException INFO [2019-12-17 12:33:44,313] jepsen worker 1 - jepsen.util 10261 :invoke :write [0 9] INFO [2019-12-17 12:33:44,328] jepsen worker 6 - jepsen.util 10806 :info :cas [2 [29 8]] :RequestCanceledException INFO [2019-12-17 12:33:44,329] jepsen worker 6 - jepsen.util 10896 :invoke :read [2 nil] INFO [2019-12-17 12:33:44,349] jepsen worker 54 - jepsen.util 8424 :ok :read [18 27] INFO [2019-12-17 12:33:44,349] jepsen worker 54 - jepsen.util 8424 :invoke :cas [18 [41 4]] INFO [2019-12-17 12:33:44,366] jepsen worker 14 - jepsen.util 7754 :ok :read [4 21] INFO [2019-12-17 12:33:44,366] jepsen worker 14 - jepsen.util 7754 :invoke :write [4 44] INFO [2019-12-17 12:33:44,366] jepsen worker 4 - jepsen.util 6934 :ok :read [1 33] INFO [2019-12-17 12:33:44,366] jepsen worker 4 - jepsen.util 6934 :invoke :read [1 nil] INFO [2019-12-17 12:33:44,372] jepsen worker 51 - jepsen.util 7251 :fail :cas [17 [16 8]] :ValueNotSwapFrom INFO [2019-12-17 12:33:44,372] jepsen worker 51 - jepsen.util 7251 :invoke :write [17 34] INFO [2019-12-17 12:33:44,377] jepsen worker 4 - jepsen.util 6934 :ok :read [1 33] INFO [2019-12-17 12:33:44,378] jepsen worker 4 - jepsen.util 6934 :invoke :write [1 10] INFO [2019-12-17 12:33:44,441] jepsen worker 84 - jepsen.util 9714 :info :cas [28 [11 47]] :RequestCanceledException INFO [2019-12-17 12:33:44,441] jepsen worker 81 - jepsen.util 9981 :info :cas [27 [6 47]] :RequestCanceledException INFO [2019-12-17 12:33:44,441] jepsen worker 57 - jepsen.util 10137 :info :cas [19 [12 45]] :RequestCanceledException INFO [2019-12-17 12:33:44,441] jepsen worker 28 - jepsen.util 5878 :info :write [9 47] :RequestCanceledException INFO [2019-12-17 12:33:44,442] jepsen worker 57 - jepsen.util 10227 :invoke :write [19 4] INFO [2019-12-17 12:33:44,442] jepsen worker 81 - jepsen.util 10071 :invoke :cas [27 [39 4]] INFO [2019-12-17 12:33:44,442] jepsen worker 28 - jepsen.util 5968 :invoke :write [9 17] INFO [2019-12-17 12:33:44,442] jepsen worker 84 - jepsen.util 9804 :invoke :write [28 16] INFO [2019-12-17 12:33:44,449] jepsen worker 55 - jepsen.util 8425 :ok :read [18 27] INFO [2019-12-17 12:33:44,449] jepsen worker 55 - jepsen.util 8425 :invoke :read [18 nil] INFO [2019-12-17 12:33:44,466] jepsen worker 53 - jepsen.util 6983 :ok :read [17 47] INFO [2019-12-17 12:33:44,467] jepsen worker 53 - jepsen.util 6983 :invoke :write [17 36] INFO [2019-12-17 12:33:44,497] jepsen worker 20 - jepsen.util 7220 :ok :write [6 11] INFO [2019-12-17 12:33:44,497] jepsen worker 20 - jepsen.util 7220 :invoke :write [6 46] INFO [2019-12-17 12:33:44,498] jepsen worker 45 - jepsen.util 7425 :ok :write [15 7] INFO [2019-12-17 12:33:44,498] jepsen worker 45 - jepsen.util 7425 :invoke :read [15 nil] INFO [2019-12-17 12:33:44,502] jepsen worker 10 - jepsen.util 7390 :ok :write [3 40] INFO [2019-12-17 12:33:44,502] jepsen worker 10 - jepsen.util 7390 :invoke :cas [3 [4 15]] INFO [2019-12-17 12:33:44,512] jepsen worker 17 - jepsen.util 10367 :info :cas [5 [33 11]] :RequestCanceledException INFO [2019-12-17 12:33:44,512] jepsen worker 5 - jepsen.util 7025 :info :write [1 22] :RequestCanceledException INFO [2019-12-17 12:33:44,512] jepsen worker 52 - jepsen.util 7702 :info :write [17 39] :RequestCanceledException INFO [2019-12-17 12:33:44,514] jepsen worker 38 - jepsen.util 7778 :ok :read [12 49] INFO [2019-12-17 12:33:44,514] jepsen worker 17 - jepsen.util 10457 :invoke :read [5 nil] INFO [2019-12-17 12:33:44,514] jepsen worker 5 - jepsen.util 7115 :invoke :cas [1 [42 27]] INFO [2019-12-17 12:33:44,514] jepsen worker 38 - jepsen.util 7778 :invoke :write [12 3] INFO [2019-12-17 12:33:44,514] jepsen worker 52 - jepsen.util 7792 :invoke :read [17 nil] INFO [2019-12-17 12:33:44,519] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":43311,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:33:44,520] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":43311,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:33:44,520] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":43315,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:33:44,520] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":43315,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:33:44,522] jepsen worker 72 - jepsen.util 7272 :ok :read [24 20] INFO [2019-12-17 12:33:44,522] jepsen worker 72 - jepsen.util 7272 :invoke :write [24 32] INFO [2019-12-17 12:33:44,528] jepsen worker 48 - jepsen.util 9228 :info :write [16 30] :RequestCanceledException INFO [2019-12-17 12:33:44,528] jepsen worker 76 - jepsen.util 10156 :info :cas [25 [36 27]] :RequestCanceledException INFO [2019-12-17 12:33:44,529] jepsen worker 76 - jepsen.util 10246 :invoke :read [25 nil] INFO [2019-12-17 12:33:44,529] jepsen worker 48 - jepsen.util 9318 :invoke :read [16 nil] INFO [2019-12-17 12:33:44,552] jepsen worker 10 - jepsen.util 7390 :fail :cas [3 [4 15]] :ValueNotSwapFrom INFO [2019-12-17 12:33:44,552] jepsen worker 10 - jepsen.util 7390 :invoke :cas [3 [19 39]] INFO [2019-12-17 12:33:44,553] jepsen worker 37 - jepsen.util 7147 :ok :read [12 49] INFO [2019-12-17 12:33:44,553] jepsen worker 37 - jepsen.util 7147 :invoke :write [12 46] INFO [2019-12-17 12:33:44,558] jepsen worker 45 - jepsen.util 7425 :ok :read [15 7] INFO [2019-12-17 12:33:44,559] jepsen worker 45 - jepsen.util 7425 :invoke :cas [15 [40 18]] INFO [2019-12-17 12:33:44,563] jepsen worker 28 - jepsen.util 5968 :ok :write [9 17] INFO [2019-12-17 12:33:44,563] jepsen worker 28 - jepsen.util 5968 :invoke :read [9 nil] INFO [2019-12-17 12:33:44,577] jepsen worker 28 - jepsen.util 5968 :ok :read [9 17] INFO [2019-12-17 12:33:44,577] jepsen worker 28 - jepsen.util 5968 :invoke :cas [9 [12 9]] INFO [2019-12-17 12:33:44,595] jepsen worker 39 - jepsen.util 9939 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:33:44,595] jepsen worker 39 - jepsen.util 9939 :invoke :read [13 nil] INFO [2019-12-17 12:33:44,602] jepsen worker 45 - jepsen.util 7425 :fail :cas [15 [40 18]] :ValueNotSwapFrom INFO [2019-12-17 12:33:44,602] jepsen worker 45 - jepsen.util 7425 :invoke :cas [15 [19 22]] INFO [2019-12-17 12:33:44,606] jepsen worker 28 - jepsen.util 5968 :fail :cas [9 [12 9]] :ValueNotSwapFrom INFO [2019-12-17 12:33:44,606] jepsen worker 28 - jepsen.util 5968 :invoke :write [9 24] INFO [2019-12-17 12:33:44,614] jepsen worker 88 - jepsen.util 7468 :fail :cas [29 [11 28]] :ValueNotSwapFrom INFO [2019-12-17 12:33:44,614] jepsen worker 88 - jepsen.util 7468 :invoke :cas [29 [0 27]] INFO [2019-12-17 12:33:44,628] jepsen worker 45 - jepsen.util 7425 :fail :cas [15 [19 22]] :ValueNotSwapFrom INFO [2019-12-17 12:33:44,628] jepsen worker 45 - jepsen.util 7425 :invoke :cas [15 [24 33]] INFO [2019-12-17 12:33:44,641] jepsen worker 45 - jepsen.util 7425 :fail :cas [15 [24 33]] :ValueNotSwapFrom INFO [2019-12-17 12:33:44,641] jepsen worker 45 - jepsen.util 7425 :invoke :cas [15 [12 4]] INFO [2019-12-17 12:33:44,648] jepsen worker 54 - jepsen.util 8424 :fail :cas [18 [41 4]] :ValueNotSwapFrom INFO [2019-12-17 12:33:44,649] jepsen worker 54 - jepsen.util 8424 :invoke :write [18 22] INFO [2019-12-17 12:33:44,657] jepsen worker 45 - jepsen.util 7425 :fail :cas [15 [12 4]] :ValueNotSwapFrom INFO [2019-12-17 12:33:44,657] jepsen worker 45 - jepsen.util 7425 :invoke :read [15 nil] INFO [2019-12-17 12:33:44,669] jepsen worker 28 - jepsen.util 5968 :ok :write [9 24] INFO [2019-12-17 12:33:44,669] jepsen worker 28 - jepsen.util 5968 :invoke :write [9 29] INFO [2019-12-17 12:33:44,674] jepsen worker 45 - jepsen.util 7425 :ok :read [15 7] INFO [2019-12-17 12:33:44,674] jepsen worker 45 - jepsen.util 7425 :invoke :cas [15 [11 2]] INFO [2019-12-17 12:33:44,686] jepsen worker 5 - jepsen.util 7115 :fail :cas [1 [42 27]] :ValueNotSwapFrom INFO [2019-12-17 12:33:44,686] jepsen worker 5 - jepsen.util 7115 :invoke :write [1 36] INFO [2019-12-17 12:33:44,694] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":43344,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:33:44,694] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":43344,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:33:44,694] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":43347,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:33:44,694] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":43347,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:33:44,694] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":43348,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:33:44,694] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":43348,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:33:44,697] jepsen worker 45 - jepsen.util 7425 :fail :cas [15 [11 2]] :ValueNotSwapFrom INFO [2019-12-17 12:33:44,697] jepsen worker 45 - jepsen.util 7425 :invoke :cas [15 [3 11]] INFO [2019-12-17 12:33:44,705] jepsen worker 45 - jepsen.util 7425 :fail :cas [15 [3 11]] :ValueNotSwapFrom INFO [2019-12-17 12:33:44,706] jepsen worker 45 - jepsen.util 7425 :invoke :cas [15 [32 40]] INFO [2019-12-17 12:33:44,713] jepsen worker 27 - jepsen.util 7407 :info :write [9 36] :RequestCanceledException INFO [2019-12-17 12:33:44,714] jepsen worker 27 - jepsen.util 7497 :invoke :read [9 nil] INFO [2019-12-17 12:33:44,725] jepsen worker 45 - jepsen.util 7425 :fail :cas [15 [32 40]] :ValueNotSwapFrom INFO [2019-12-17 12:33:44,725] jepsen worker 45 - jepsen.util 7425 :invoke :cas [15 [45 45]] INFO [2019-12-17 12:33:44,728] jepsen worker 79 - jepsen.util 7549 :info :write [26 4] :RequestCanceledException INFO [2019-12-17 12:33:44,728] jepsen worker 7 - jepsen.util 10267 :info :write [2 34] :RequestCanceledException INFO [2019-12-17 12:33:44,728] jepsen worker 49 - jepsen.util 9589 :info :cas [16 [12 17]] :RequestCanceledException INFO [2019-12-17 12:33:44,729] jepsen worker 49 - jepsen.util 9679 :invoke :cas [16 [35 27]] INFO [2019-12-17 12:33:44,729] jepsen worker 79 - jepsen.util 7639 :invoke :cas [26 [32 47]] INFO [2019-12-17 12:33:44,729] jepsen worker 7 - jepsen.util 10357 :invoke :cas [2 [24 17]] INFO [2019-12-17 12:33:44,743] jepsen worker 85 - jepsen.util 10975 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:33:44,743] jepsen worker 16 - jepsen.util 11356 :info :cas [5 [46 31]] :RequestCanceledException INFO [2019-12-17 12:33:44,743] jepsen worker 85 - jepsen.util 10975 :invoke :write [28 40] INFO [2019-12-17 12:33:44,744] jepsen worker 16 - jepsen.util 11446 :invoke :write [5 3] INFO [2019-12-17 12:33:44,745] jepsen worker 27 - jepsen.util 7497 :ok :read [9 24] INFO [2019-12-17 12:33:44,745] jepsen worker 27 - jepsen.util 7497 :invoke :cas [9 [16 9]] INFO [2019-12-17 12:33:44,748] jepsen worker 71 - jepsen.util 8171 :fail :cas [23 [16 30]] :ValueNotSwapFrom INFO [2019-12-17 12:33:44,748] jepsen worker 71 - jepsen.util 8171 :invoke :read [23 nil] INFO [2019-12-17 12:33:44,754] jepsen worker 72 - jepsen.util 7272 :ok :write [24 32] INFO [2019-12-17 12:33:44,754] jepsen worker 72 - jepsen.util 7272 :invoke :read [24 nil] INFO [2019-12-17 12:33:44,758] jepsen worker 45 - jepsen.util 7425 :fail :cas [15 [45 45]] :ValueNotSwapFrom INFO [2019-12-17 12:33:44,759] jepsen worker 45 - jepsen.util 7425 :invoke :cas [15 [25 25]] INFO [2019-12-17 12:33:44,759] jepsen worker 40 - jepsen.util 10480 :info :write [13 48] :RequestCanceledException INFO [2019-12-17 12:33:44,759] jepsen worker 35 - jepsen.util 10025 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:33:44,759] jepsen worker 35 - jepsen.util 10025 :invoke :read [11 nil] INFO [2019-12-17 12:33:44,760] jepsen worker 40 - jepsen.util 10570 :invoke :write [13 47] INFO [2019-12-17 12:33:44,761] jepsen worker 28 - jepsen.util 5968 :ok :write [9 29] INFO [2019-12-17 12:33:44,762] jepsen worker 28 - jepsen.util 5968 :invoke :read [9 nil] INFO [2019-12-17 12:33:44,767] jepsen worker 27 - jepsen.util 7497 :fail :cas [9 [16 9]] :ValueNotSwapFrom INFO [2019-12-17 12:33:44,767] jepsen worker 27 - jepsen.util 7497 :invoke :read [9 nil] INFO [2019-12-17 12:33:44,794] jepsen worker 44 - jepsen.util 10844 :info :cas [14 [2 29]] :RequestCanceledException INFO [2019-12-17 12:33:44,794] jepsen worker 44 - jepsen.util 10934 :invoke :read [14 nil] INFO [2019-12-17 12:33:44,813] jepsen worker 61 - jepsen.util 10771 :info :write [20 12] :RequestCanceledException INFO [2019-12-17 12:33:44,814] jepsen worker 61 - jepsen.util 10861 :invoke :cas [20 [30 15]] INFO [2019-12-17 12:33:44,828] jepsen worker 8 - jepsen.util 10088 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:33:44,828] jepsen worker 8 - jepsen.util 10088 :invoke :cas [2 [6 16]] INFO [2019-12-17 12:33:44,829] jepsen worker 89 - jepsen.util 7289 :info :write [29 14] :RequestCanceledException INFO [2019-12-17 12:33:44,830] jepsen worker 89 - jepsen.util 7379 :invoke :read [29 nil] INFO [2019-12-17 12:33:44,842] jepsen worker 72 - jepsen.util 7272 :ok :read [24 32] INFO [2019-12-17 12:33:44,843] jepsen worker 72 - jepsen.util 7272 :invoke :cas [24 [35 17]] INFO [2019-12-17 12:33:44,852] jepsen worker 45 - jepsen.util 7425 :fail :cas [15 [25 25]] :ValueNotSwapFrom INFO [2019-12-17 12:33:44,852] jepsen worker 45 - jepsen.util 7425 :invoke :write [15 34] INFO [2019-12-17 12:33:44,869] jepsen worker 89 - jepsen.util 7379 :ok :read [29 0] INFO [2019-12-17 12:33:44,869] jepsen worker 89 - jepsen.util 7379 :invoke :cas [29 [41 18]] INFO [2019-12-17 12:33:44,885] jepsen worker 28 - jepsen.util 5968 :ok :read [9 29] INFO [2019-12-17 12:33:44,885] jepsen worker 28 - jepsen.util 5968 :invoke :read [9 nil] INFO [2019-12-17 12:33:44,895] jepsen worker 30 - jepsen.util 7320 :info :write [10 16] :RequestCanceledException INFO [2019-12-17 12:33:44,896] jepsen worker 30 - jepsen.util 7410 :invoke :read [10 nil] INFO [2019-12-17 12:33:44,904] jepsen worker 27 - jepsen.util 7497 :ok :read [9 29] INFO [2019-12-17 12:33:44,904] jepsen worker 27 - jepsen.util 7497 :invoke :write [9 9] INFO [2019-12-17 12:33:44,920] jepsen worker 28 - jepsen.util 5968 :ok :read [9 29] INFO [2019-12-17 12:33:44,920] jepsen worker 28 - jepsen.util 5968 :invoke :read [9 nil] INFO [2019-12-17 12:33:44,932] jepsen worker 52 - jepsen.util 7792 :ok :read [17 47] INFO [2019-12-17 12:33:44,932] jepsen worker 52 - jepsen.util 7792 :invoke :write [17 14] INFO [2019-12-17 12:33:44,941] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:33:44,941] jepsen nemesis - couchbase.util Rebalance check count: 332 INFO [2019-12-17 12:33:44,941] jepsen nemesis - couchbase.util Rebalance stuck count: 313 INFO [2019-12-17 12:33:44,947] jepsen worker 30 - jepsen.util 7410 :ok :read [10 7] INFO [2019-12-17 12:33:44,947] jepsen worker 30 - jepsen.util 7410 :invoke :cas [10 [10 14]] INFO [2019-12-17 12:33:44,948] jepsen worker 72 - jepsen.util 7272 :fail :cas [24 [35 17]] :ValueNotSwapFrom INFO [2019-12-17 12:33:44,948] jepsen worker 72 - jepsen.util 7272 :invoke :write [24 15] INFO [2019-12-17 12:33:44,957] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":43378,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:33:44,957] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":43378,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:33:44,957] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":43381,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:33:44,957] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":43381,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:33:44,959] jepsen worker 19 - jepsen.util 7579 :info :write [6 41] :RequestCanceledException INFO [2019-12-17 12:33:44,960] jepsen worker 19 - jepsen.util 7669 :invoke :write [6 0] INFO [2019-12-17 12:33:44,965] jepsen worker 28 - jepsen.util 5968 :ok :read [9 29] INFO [2019-12-17 12:33:44,965] jepsen worker 28 - jepsen.util 5968 :invoke :write [9 6] INFO [2019-12-17 12:33:44,968] jepsen worker 51 - jepsen.util 7251 :ok :write [17 34] INFO [2019-12-17 12:33:44,968] jepsen worker 51 - jepsen.util 7251 :invoke :cas [17 [25 34]] INFO [2019-12-17 12:33:44,984] jepsen worker 30 - jepsen.util 7410 :fail :cas [10 [10 14]] :ValueNotSwapFrom INFO [2019-12-17 12:33:44,985] jepsen worker 30 - jepsen.util 7410 :invoke :read [10 nil] INFO [2019-12-17 12:33:45,000] jepsen worker 78 - jepsen.util 7638 :ok :write [26 19] INFO [2019-12-17 12:33:45,000] jepsen worker 78 - jepsen.util 7638 :invoke :read [26 nil] INFO [2019-12-17 12:33:45,009] jepsen worker 79 - jepsen.util 7639 :fail :cas [26 [32 47]] :ValueNotSwapFrom INFO [2019-12-17 12:33:45,009] jepsen worker 79 - jepsen.util 7639 :invoke :cas [26 [36 10]] INFO [2019-12-17 12:33:45,017] jepsen worker 30 - jepsen.util 7410 :ok :read [10 7] INFO [2019-12-17 12:33:45,017] jepsen worker 30 - jepsen.util 7410 :invoke :write [10 5] INFO [2019-12-17 12:33:45,029] jepsen worker 70 - jepsen.util 7810 :ok :write [23 28] INFO [2019-12-17 12:33:45,029] jepsen worker 65 - jepsen.util 6185 :info :write [21 28] :RequestCanceledException INFO [2019-12-17 12:33:45,029] jepsen worker 31 - jepsen.util 6601 :info :write [10 3] :RequestCanceledException INFO [2019-12-17 12:33:45,029] jepsen worker 70 - jepsen.util 7810 :invoke :write [23 0] INFO [2019-12-17 12:33:45,030] jepsen worker 65 - jepsen.util 6275 :invoke :write [21 35] INFO [2019-12-17 12:33:45,030] jepsen worker 31 - jepsen.util 6691 :invoke :write [10 21] INFO [2019-12-17 12:33:45,035] jepsen worker 79 - jepsen.util 7639 :fail :cas [26 [36 10]] :ValueNotSwapFrom INFO [2019-12-17 12:33:45,035] jepsen worker 79 - jepsen.util 7639 :invoke :read [26 nil] INFO [2019-12-17 12:33:45,059] jepsen worker 29 - jepsen.util 5879 :info :cas [9 [30 13]] :RequestCanceledException INFO [2019-12-17 12:33:45,062] jepsen worker 29 - jepsen.util 5969 :invoke :write [9 9] INFO [2019-12-17 12:33:45,067] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":43396,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:33:45,067] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":43396,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:33:45,067] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":43399,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:33:45,067] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":43399,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:33:45,067] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":43400,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:33:45,067] jepsen worker 28 - jepsen.util 5968 :ok :write [9 6] INFO [2019-12-17 12:33:45,067] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":43400,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:33:45,068] jepsen worker 28 - jepsen.util 5968 :invoke :cas [9 [8 16]] INFO [2019-12-17 12:33:45,095] jepsen worker 13 - jepsen.util 7123 :info :write [4 14] :RequestCanceledException INFO [2019-12-17 12:33:45,103] jepsen worker 79 - jepsen.util 7639 :ok :read [26 19] INFO [2019-12-17 12:33:45,118] jepsen worker 65 - jepsen.util 6275 :ok :write [21 35] INFO [2019-12-17 12:33:45,125] jepsen worker 89 - jepsen.util 7379 :fail :cas [29 [41 18]] :ValueNotSwapFrom INFO [2019-12-17 12:33:45,129] jepsen worker 29 - jepsen.util 5969 :ok :write [9 9] INFO [2019-12-17 12:33:45,141] jepsen worker 72 - jepsen.util 7272 :ok :write [24 15] INFO [2019-12-17 12:33:45,142] jepsen worker 2 - jepsen.util 9902 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:33:45,146] jepsen worker 30 - jepsen.util 7410 :ok :write [10 5] INFO [2019-12-17 12:33:45,156] jepsen worker 79 - jepsen.util 7639 :invoke :read [26 nil] INFO [2019-12-17 12:33:45,156] jepsen worker 65 - jepsen.util 6275 :invoke :write [21 49] INFO [2019-12-17 12:33:45,156] jepsen worker 89 - jepsen.util 7379 :invoke :read [29 nil] INFO [2019-12-17 12:33:45,156] jepsen worker 29 - jepsen.util 5969 :invoke :read [9 nil] INFO [2019-12-17 12:33:45,157] jepsen worker 13 - jepsen.util 7213 :invoke :write [4 18] INFO [2019-12-17 12:33:45,159] jepsen worker 72 - jepsen.util 7272 :invoke :cas [24 [24 23]] INFO [2019-12-17 12:33:45,159] jepsen worker 2 - jepsen.util 9902 :invoke :write [0 43] INFO [2019-12-17 12:33:45,160] jepsen worker 74 - jepsen.util 7634 :fail :read [24 nil] :RequestTimeoutException INFO [2019-12-17 12:33:45,160] jepsen worker 74 - jepsen.util 7634 :invoke :cas [24 [9 22]] INFO [2019-12-17 12:33:45,160] jepsen worker 30 - jepsen.util 7410 :invoke :write [10 0] INFO [2019-12-17 12:33:45,180] jepsen worker 51 - jepsen.util 7251 :fail :cas [17 [25 34]] :ValueNotSwapFrom INFO [2019-12-17 12:33:45,180] jepsen worker 51 - jepsen.util 7251 :invoke :write [17 21] INFO [2019-12-17 12:33:45,183] jepsen worker 71 - jepsen.util 8171 :ok :read [23 28] INFO [2019-12-17 12:33:45,183] jepsen worker 71 - jepsen.util 8171 :invoke :write [23 48] INFO [2019-12-17 12:33:45,195] jepsen worker 11 - jepsen.util 7571 :info :write [3 19] :RequestCanceledException INFO [2019-12-17 12:33:45,196] jepsen worker 11 - jepsen.util 7661 :invoke :read [3 nil] INFO [2019-12-17 12:33:45,204] jepsen worker 74 - jepsen.util 7634 :fail :cas [24 [9 22]] :ValueNotSwapFrom INFO [2019-12-17 12:33:45,204] jepsen worker 74 - jepsen.util 7634 :invoke :cas [24 [20 1]] INFO [2019-12-17 12:33:45,229] jepsen worker 73 - jepsen.util 7183 :fail :read [24 nil] :RequestTimeoutException INFO [2019-12-17 12:33:45,229] jepsen worker 46 - jepsen.util 7876 :info :write [15 33] :RequestCanceledException INFO [2019-12-17 12:33:45,229] jepsen worker 69 - jepsen.util 7719 :info :write [23 35] :RequestCanceledException INFO [2019-12-17 12:33:45,229] jepsen worker 73 - jepsen.util 7183 :invoke :read [24 nil] INFO [2019-12-17 12:33:45,230] jepsen worker 69 - jepsen.util 7809 :invoke :cas [23 [27 4]] INFO [2019-12-17 12:33:45,230] jepsen worker 46 - jepsen.util 7966 :invoke :write [15 47] INFO [2019-12-17 12:33:45,234] jepsen worker 72 - jepsen.util 7272 :fail :cas [24 [24 23]] :ValueNotSwapFrom INFO [2019-12-17 12:33:45,234] jepsen worker 72 - jepsen.util 7272 :invoke :read [24 nil] INFO [2019-12-17 12:33:45,251] jepsen worker 72 - jepsen.util 7272 :ok :read [24 15] INFO [2019-12-17 12:33:45,251] jepsen worker 72 - jepsen.util 7272 :invoke :write [24 3] INFO [2019-12-17 12:33:45,269] jepsen worker 79 - jepsen.util 7639 :ok :read [26 19] INFO [2019-12-17 12:33:45,269] jepsen worker 79 - jepsen.util 7639 :invoke :write [26 39] INFO [2019-12-17 12:33:45,274] jepsen worker 29 - jepsen.util 5969 :ok :read [9 9] INFO [2019-12-17 12:33:45,275] jepsen worker 29 - jepsen.util 5969 :invoke :read [9 nil] INFO [2019-12-17 12:33:45,285] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":43423,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:33:45,286] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":43423,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:33:45,290] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":43431,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:33:45,290] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":43431,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:33:45,290] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":43434,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:33:45,290] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":43434,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:33:45,290] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":43435,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:33:45,290] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":43435,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:33:45,304] jepsen worker 73 - jepsen.util 7183 :ok :read [24 15] INFO [2019-12-17 12:33:45,304] jepsen worker 73 - jepsen.util 7183 :invoke :read [24 nil] INFO [2019-12-17 12:33:45,309] jepsen worker 65 - jepsen.util 6275 :ok :write [21 49] INFO [2019-12-17 12:33:45,309] jepsen worker 65 - jepsen.util 6275 :invoke :read [21 nil] INFO [2019-12-17 12:33:45,318] jepsen worker 78 - jepsen.util 7638 :ok :read [26 19] INFO [2019-12-17 12:33:45,318] jepsen worker 78 - jepsen.util 7638 :invoke :cas [26 [14 12]] INFO [2019-12-17 12:33:45,321] jepsen worker 29 - jepsen.util 5969 :ok :read [9 9] INFO [2019-12-17 12:33:45,321] jepsen worker 29 - jepsen.util 5969 :invoke :cas [9 [44 36]] INFO [2019-12-17 12:33:45,328] jepsen worker 63 - jepsen.util 6363 :info :write [21 49] :RequestCanceledException INFO [2019-12-17 12:33:45,329] jepsen worker 12 - jepsen.util 8022 :info :write [4 35] :RequestCanceledException INFO [2019-12-17 12:33:45,329] jepsen worker 63 - jepsen.util 6453 :invoke :cas [21 [10 29]] INFO [2019-12-17 12:33:45,332] jepsen worker 12 - jepsen.util 8112 :invoke :read [4 nil] INFO [2019-12-17 12:33:45,336] jepsen worker 73 - jepsen.util 7183 :ok :read [24 15] INFO [2019-12-17 12:33:45,336] jepsen worker 73 - jepsen.util 7183 :invoke :read [24 nil] INFO [2019-12-17 12:33:45,357] jepsen worker 69 - jepsen.util 7809 :fail :cas [23 [27 4]] :ValueNotSwapFrom INFO [2019-12-17 12:33:45,357] jepsen worker 69 - jepsen.util 7809 :invoke :cas [23 [36 13]] INFO [2019-12-17 12:33:45,379] jepsen worker 78 - jepsen.util 7638 :fail :cas [26 [14 12]] :ValueNotSwapFrom INFO [2019-12-17 12:33:45,379] jepsen worker 78 - jepsen.util 7638 :invoke :write [26 30] INFO [2019-12-17 12:33:45,380] jepsen worker 29 - jepsen.util 5969 :fail :cas [9 [44 36]] :ValueNotSwapFrom INFO [2019-12-17 12:33:45,380] jepsen worker 29 - jepsen.util 5969 :invoke :read [9 nil] INFO [2019-12-17 12:33:45,413] jepsen worker 73 - jepsen.util 7183 :ok :read [24 15] INFO [2019-12-17 12:33:45,413] jepsen worker 73 - jepsen.util 7183 :invoke :write [24 47] INFO [2019-12-17 12:33:45,417] jepsen worker 89 - jepsen.util 7379 :ok :read [29 0] INFO [2019-12-17 12:33:45,417] jepsen worker 89 - jepsen.util 7379 :invoke :read [29 nil] INFO [2019-12-17 12:33:45,438] jepsen worker 29 - jepsen.util 5969 :ok :read [9 9] INFO [2019-12-17 12:33:45,438] jepsen worker 29 - jepsen.util 5969 :invoke :write [9 0] INFO [2019-12-17 12:33:45,489] jepsen worker 72 - jepsen.util 7272 :ok :write [24 3] INFO [2019-12-17 12:33:45,489] jepsen worker 72 - jepsen.util 7272 :invoke :cas [24 [17 30]] INFO [2019-12-17 12:33:45,501] jepsen worker 72 - jepsen.util 7272 :fail :cas [24 [17 30]] :ValueNotSwapFrom INFO [2019-12-17 12:33:45,501] jepsen worker 72 - jepsen.util 7272 :invoke :cas [24 [19 11]] INFO [2019-12-17 12:33:45,506] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":43456,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:33:45,507] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":43456,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:33:45,507] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":43459,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:33:45,507] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":43459,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:33:45,507] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":43460,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:33:45,507] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":43460,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:33:45,513] jepsen worker 9 - jepsen.util 7929 :fail :read [3 nil] :RequestTimeoutException INFO [2019-12-17 12:33:45,514] jepsen worker 9 - jepsen.util 7929 :invoke :write [3 25] INFO [2019-12-17 12:33:45,514] jepsen worker 89 - jepsen.util 7379 :ok :read [29 0] INFO [2019-12-17 12:33:45,514] jepsen worker 89 - jepsen.util 7379 :invoke :write [29 2] INFO [2019-12-17 12:33:45,517] jepsen worker 72 - jepsen.util 7272 :fail :cas [24 [19 11]] :ValueNotSwapFrom INFO [2019-12-17 12:33:45,518] jepsen worker 72 - jepsen.util 7272 :invoke :write [24 0] INFO [2019-12-17 12:33:45,528] jepsen worker 87 - jepsen.util 6837 :info :write [29 34] :RequestCanceledException INFO [2019-12-17 12:33:45,529] jepsen worker 87 - jepsen.util 6927 :invoke :write [29 26] INFO [2019-12-17 12:33:45,529] jepsen worker 47 - jepsen.util 7697 :info :write [15 21] :RequestCanceledException INFO [2019-12-17 12:33:45,529] jepsen worker 3 - jepsen.util 7473 :info :write [1 38] :RequestCanceledException INFO [2019-12-17 12:33:45,530] jepsen worker 47 - jepsen.util 7787 :invoke :read [15 nil] INFO [2019-12-17 12:33:45,530] jepsen worker 3 - jepsen.util 7563 :invoke :read [1 nil] INFO [2019-12-17 12:33:45,542] jepsen worker 18 - jepsen.util 7668 :info :write [6 9] :RequestCanceledException INFO [2019-12-17 12:33:45,543] jepsen worker 18 - jepsen.util 7758 :invoke :cas [6 [20 21]] INFO [2019-12-17 12:33:45,549] jepsen worker 12 - jepsen.util 8112 :ok :read [4 21] INFO [2019-12-17 12:33:45,550] jepsen worker 12 - jepsen.util 8112 :invoke :read [4 nil] INFO [2019-12-17 12:33:45,628] jepsen worker 80 - jepsen.util 7460 :info :write [26 12] :RequestCanceledException INFO [2019-12-17 12:33:45,629] jepsen worker 80 - jepsen.util 7550 :invoke :cas [26 [16 3]] INFO [2019-12-17 12:33:45,664] jepsen worker 69 - jepsen.util 7809 :fail :cas [23 [36 13]] :ValueNotSwapFrom INFO [2019-12-17 12:33:45,664] jepsen worker 69 - jepsen.util 7809 :invoke :read [23 nil] INFO [2019-12-17 12:33:45,671] jepsen worker 12 - jepsen.util 8112 :ok :read [4 21] INFO [2019-12-17 12:33:45,671] jepsen worker 12 - jepsen.util 8112 :invoke :read [4 nil] INFO [2019-12-17 12:33:45,674] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":43474,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:33:45,674] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":43474,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:33:45,674] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":43477,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:33:45,674] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":43477,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:33:45,714] jepsen worker 72 - jepsen.util 7272 :ok :write [24 0] INFO [2019-12-17 12:33:45,714] jepsen worker 9 - jepsen.util 7929 :ok :write [3 25] INFO [2019-12-17 12:33:45,714] jepsen worker 72 - jepsen.util 7272 :invoke :write [24 23] INFO [2019-12-17 12:33:45,714] jepsen worker 9 - jepsen.util 7929 :invoke :cas [3 [14 9]] INFO [2019-12-17 12:33:45,715] jepsen worker 30 - jepsen.util 7410 :ok :write [10 0] INFO [2019-12-17 12:33:45,715] jepsen worker 30 - jepsen.util 7410 :invoke :read [10 nil] INFO [2019-12-17 12:33:45,719] jepsen worker 46 - jepsen.util 7966 :ok :write [15 47] INFO [2019-12-17 12:33:45,719] jepsen worker 46 - jepsen.util 7966 :invoke :cas [15 [25 11]] INFO [2019-12-17 12:33:45,729] jepsen worker 32 - jepsen.util 7412 :info :cas [10 [30 14]] :RequestCanceledException INFO [2019-12-17 12:33:45,729] jepsen worker 64 - jepsen.util 5824 :info :cas [21 [6 19]] :RequestCanceledException INFO [2019-12-17 12:33:45,730] jepsen worker 32 - jepsen.util 7502 :invoke :write [10 44] INFO [2019-12-17 12:33:45,731] jepsen worker 64 - jepsen.util 5914 :invoke :read [21 nil] INFO [2019-12-17 12:33:45,733] jepsen worker 12 - jepsen.util 8112 :ok :read [4 21] INFO [2019-12-17 12:33:45,734] jepsen worker 12 - jepsen.util 8112 :invoke :write [4 49] INFO [2019-12-17 12:33:45,765] jepsen worker 55 - jepsen.util 8425 :ok :read [18 27] INFO [2019-12-17 12:33:45,765] jepsen worker 55 - jepsen.util 8425 :invoke :write [18 33] INFO [2019-12-17 12:33:45,813] jepsen worker 9 - jepsen.util 7929 :fail :cas [3 [14 9]] :ValueNotSwapFrom INFO [2019-12-17 12:33:45,813] jepsen worker 9 - jepsen.util 7929 :invoke :cas [3 [2 16]] INFO [2019-12-17 12:33:45,820] jepsen worker 18 - jepsen.util 7758 :fail :cas [6 [20 21]] :ValueNotSwapFrom INFO [2019-12-17 12:33:45,820] jepsen worker 18 - jepsen.util 7758 :invoke :read [6 nil] INFO [2019-12-17 12:33:45,846] jepsen worker 64 - jepsen.util 5914 :ok :read [21 49] INFO [2019-12-17 12:33:45,847] jepsen worker 64 - jepsen.util 5914 :invoke :write [21 27] INFO [2019-12-17 12:33:45,849] jepsen worker 47 - jepsen.util 7787 :ok :read [15 47] INFO [2019-12-17 12:33:45,850] jepsen worker 47 - jepsen.util 7787 :invoke :cas [15 [19 38]] INFO [2019-12-17 12:33:45,858] jepsen worker 9 - jepsen.util 7929 :fail :cas [3 [2 16]] :ValueNotSwapFrom INFO [2019-12-17 12:33:45,858] jepsen worker 9 - jepsen.util 7929 :invoke :write [3 1] INFO [2019-12-17 12:33:45,881] jepsen worker 30 - jepsen.util 7410 :ok :read [10 0] INFO [2019-12-17 12:33:45,881] jepsen worker 30 - jepsen.util 7410 :invoke :write [10 7] INFO [2019-12-17 12:33:45,887] jepsen worker 46 - jepsen.util 7966 :fail :cas [15 [25 11]] :ValueNotSwapFrom INFO [2019-12-17 12:33:45,887] jepsen worker 46 - jepsen.util 7966 :invoke :write [15 26] INFO [2019-12-17 12:33:45,895] jepsen worker 34 - jepsen.util 9664 :info :cas [11 [48 43]] :RequestCanceledException INFO [2019-12-17 12:33:45,896] jepsen worker 34 - jepsen.util 9754 :invoke :cas [11 [25 30]] INFO [2019-12-17 12:33:45,907] jepsen worker 3 - jepsen.util 7563 :ok :read [1 33] INFO [2019-12-17 12:33:45,907] jepsen worker 3 - jepsen.util 7563 :invoke :write [1 26] INFO [2019-12-17 12:33:45,913] jepsen worker 26 - jepsen.util 10556 :info :write [8 46] :RequestCanceledException INFO [2019-12-17 12:33:45,913] jepsen worker 26 - jepsen.util 10646 :invoke :write [8 47] INFO [2019-12-17 12:33:45,941] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":43495,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:33:45,941] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":43495,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:33:45,941] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":43499,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:33:45,941] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":43499,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:33:45,949] jepsen worker 69 - jepsen.util 7809 :ok :read [23 28] INFO [2019-12-17 12:33:45,949] jepsen worker 69 - jepsen.util 7809 :invoke :cas [23 [26 18]] INFO [2019-12-17 12:33:45,970] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:33:45,970] jepsen nemesis - couchbase.util Rebalance check count: 333 INFO [2019-12-17 12:33:45,970] jepsen nemesis - couchbase.util Rebalance stuck count: 314 INFO [2019-12-17 12:33:45,976] jepsen worker 47 - jepsen.util 7787 :fail :cas [15 [19 38]] :ValueNotSwapFrom INFO [2019-12-17 12:33:45,976] jepsen worker 47 - jepsen.util 7787 :invoke :write [15 48] INFO [2019-12-17 12:33:45,981] jepsen worker 18 - jepsen.util 7758 :ok :read [6 11] INFO [2019-12-17 12:33:45,982] jepsen worker 18 - jepsen.util 7758 :invoke :read [6 nil] INFO [2019-12-17 12:33:46,013] jepsen worker 43 - jepsen.util 9673 :info :write [14 47] :RequestCanceledException INFO [2019-12-17 12:33:46,014] jepsen worker 43 - jepsen.util 9763 :invoke :read [14 nil] INFO [2019-12-17 12:33:46,028] jepsen worker 83 - jepsen.util 10523 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:33:46,028] jepsen worker 36 - jepsen.util 7776 :info :write [12 44] :RequestCanceledException INFO [2019-12-17 12:33:46,028] jepsen worker 66 - jepsen.util 10056 :info :write [22 25] :RequestCanceledException INFO [2019-12-17 12:33:46,028] jepsen worker 83 - jepsen.util 10523 :invoke :cas [27 [1 49]] INFO [2019-12-17 12:33:46,029] jepsen worker 36 - jepsen.util 7866 :invoke :cas [12 [1 20]] INFO [2019-12-17 12:33:46,029] jepsen worker 66 - jepsen.util 10146 :invoke :cas [22 [7 12]] INFO [2019-12-17 12:33:46,041] jepsen worker 42 - jepsen.util 10842 :info :write [14 18] :RequestCanceledException INFO [2019-12-17 12:33:46,041] jepsen worker 21 - jepsen.util 9561 :info :write [7 49] :RequestCanceledException INFO [2019-12-17 12:33:46,043] jepsen worker 21 - jepsen.util 9651 :invoke :read [7 nil] INFO [2019-12-17 12:33:46,043] jepsen worker 42 - jepsen.util 10932 :invoke :write [14 13] INFO [2019-12-17 12:33:46,048] jepsen worker 64 - jepsen.util 5914 :ok :write [21 27] INFO [2019-12-17 12:33:46,048] jepsen worker 64 - jepsen.util 5914 :invoke :read [21 nil] INFO [2019-12-17 12:33:46,050] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":43511,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:33:46,050] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":43511,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:33:46,059] jepsen worker 75 - jepsen.util 9705 :info :cas [25 [49 40]] :RequestCanceledException INFO [2019-12-17 12:33:46,060] jepsen worker 75 - jepsen.util 9795 :invoke :write [25 0] INFO [2019-12-17 12:33:46,066] jepsen worker 64 - jepsen.util 5914 :ok :read [21 27] INFO [2019-12-17 12:33:46,066] jepsen worker 64 - jepsen.util 5914 :invoke :read [21 nil] INFO [2019-12-17 12:33:46,085] jepsen worker 64 - jepsen.util 5914 :ok :read [21 27] INFO [2019-12-17 12:33:46,085] jepsen worker 64 - jepsen.util 5914 :invoke :write [21 48] INFO [2019-12-17 12:33:46,090] jepsen worker 72 - jepsen.util 7272 :ok :write [24 23] INFO [2019-12-17 12:33:46,090] jepsen worker 72 - jepsen.util 7272 :invoke :write [24 5] INFO [2019-12-17 12:33:46,095] jepsen worker 62 - jepsen.util 10232 :info :write [20 8] :RequestCanceledException INFO [2019-12-17 12:33:46,095] jepsen worker 77 - jepsen.util 10427 :info :write [25 30] :RequestCanceledException INFO [2019-12-17 12:33:46,096] jepsen worker 62 - jepsen.util 10322 :invoke :cas [20 [24 47]] INFO [2019-12-17 12:33:46,096] jepsen worker 77 - jepsen.util 10517 :invoke :cas [25 [35 21]] INFO [2019-12-17 12:33:46,113] jepsen worker 22 - jepsen.util 10012 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:33:46,113] jepsen worker 22 - jepsen.util 10012 :invoke :cas [7 [35 20]] INFO [2019-12-17 12:33:46,128] jepsen worker 67 - jepsen.util 10147 :info :write [22 15] :RequestCanceledException INFO [2019-12-17 12:33:46,128] jepsen worker 23 - jepsen.util 10283 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:33:46,128] jepsen worker 86 - jepsen.util 10796 :info :cas [28 [27 29]] :RequestCanceledException INFO [2019-12-17 12:33:46,128] jepsen worker 23 - jepsen.util 10283 :invoke :read [7 nil] INFO [2019-12-17 12:33:46,129] jepsen worker 67 - jepsen.util 10237 :invoke :read [22 nil] INFO [2019-12-17 12:33:46,129] jepsen worker 86 - jepsen.util 10886 :invoke :write [28 43] INFO [2019-12-17 12:33:46,143] jepsen worker 59 - jepsen.util 10049 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:33:46,143] jepsen worker 0 - jepsen.util 10620 :info :write [0 19] :RequestCanceledException INFO [2019-12-17 12:33:46,143] jepsen worker 59 - jepsen.util 10049 :invoke :read [19 nil] INFO [2019-12-17 12:33:46,143] jepsen worker 0 - jepsen.util 10710 :invoke :cas [0 [18 17]] INFO [2019-12-17 12:33:46,149] jepsen worker 64 - jepsen.util 5914 :ok :write [21 48] INFO [2019-12-17 12:33:46,149] jepsen worker 64 - jepsen.util 5914 :invoke :write [21 9] INFO [2019-12-17 12:33:46,157] jepsen worker 3 - jepsen.util 7563 :ok :write [1 26] INFO [2019-12-17 12:33:46,157] jepsen worker 3 - jepsen.util 7563 :invoke :read [1 nil] INFO [2019-12-17 12:33:46,159] jepsen worker 41 - jepsen.util 10661 :info :cas [13 [20 20]] :RequestCanceledException INFO [2019-12-17 12:33:46,159] jepsen worker 50 - jepsen.util 10940 :info :write [16 35] :RequestCanceledException INFO [2019-12-17 12:33:46,160] jepsen worker 50 - jepsen.util 11030 :invoke :write [16 20] INFO [2019-12-17 12:33:46,160] jepsen worker 41 - jepsen.util 10751 :invoke :read [13 nil] INFO [2019-12-17 12:33:46,193] jepsen worker 25 - jepsen.util 10645 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:33:46,193] jepsen worker 58 - jepsen.util 10948 :info :cas [19 [28 15]] :RequestCanceledException INFO [2019-12-17 12:33:46,193] jepsen worker 25 - jepsen.util 10645 :invoke :write [8 15] INFO [2019-12-17 12:33:46,194] jepsen worker 58 - jepsen.util 11038 :invoke :cas [19 [39 26]] INFO [2019-12-17 12:33:46,203] jepsen worker 64 - jepsen.util 5914 :ok :write [21 9] INFO [2019-12-17 12:33:46,203] jepsen worker 64 - jepsen.util 5914 :invoke :cas [21 [24 35]] INFO [2019-12-17 12:33:46,213] jepsen worker 68 - jepsen.util 9698 :info :cas [22 [47 3]] :RequestCanceledException INFO [2019-12-17 12:33:46,213] jepsen worker 15 - jepsen.util 9105 :info :cas [5 [38 48]] :RequestCanceledException INFO [2019-12-17 12:33:46,214] jepsen worker 68 - jepsen.util 9788 :invoke :read [22 nil] INFO [2019-12-17 12:33:46,214] jepsen worker 15 - jepsen.util 9195 :invoke :cas [5 [22 38]] INFO [2019-12-17 12:33:46,226] jepsen worker 64 - jepsen.util 5914 :fail :cas [21 [24 35]] :ValueNotSwapFrom INFO [2019-12-17 12:33:46,226] jepsen worker 64 - jepsen.util 5914 :invoke :cas [21 [6 1]] INFO [2019-12-17 12:33:46,228] jepsen worker 60 - jepsen.util 11130 :info :cas [20 [39 27]] :RequestCanceledException INFO [2019-12-17 12:33:46,229] jepsen worker 24 - jepsen.util 8844 :info :write [8 41] :RequestCanceledException INFO [2019-12-17 12:33:46,230] jepsen worker 60 - jepsen.util 11220 :invoke :write [20 23] INFO [2019-12-17 12:33:46,230] jepsen worker 24 - jepsen.util 8934 :invoke :cas [8 [26 37]] INFO [2019-12-17 12:33:46,252] jepsen worker 64 - jepsen.util 5914 :fail :cas [21 [6 1]] :ValueNotSwapFrom INFO [2019-12-17 12:33:46,252] jepsen worker 64 - jepsen.util 5914 :invoke :write [21 28] INFO [2019-12-17 12:33:46,260] jepsen worker 33 - jepsen.util 9933 :info :cas [11 [22 19]] :RequestCanceledException INFO [2019-12-17 12:33:46,261] jepsen worker 33 - jepsen.util 10023 :invoke :cas [11 [16 25]] INFO [2019-12-17 12:33:46,272] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":43538,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:33:46,272] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":43538,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:33:46,272] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":43540,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:33:46,272] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":43540,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:33:46,272] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":43541,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:33:46,272] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":43541,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:33:46,273] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":43546,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:33:46,274] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":43546,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:33:46,274] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":43553,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:33:46,274] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":43553,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:33:46,302] jepsen worker 32 - jepsen.util 7502 :ok :write [10 44] INFO [2019-12-17 12:33:46,302] jepsen worker 32 - jepsen.util 7502 :invoke :cas [10 [23 45]] INFO [2019-12-17 12:33:46,302] jepsen worker 46 - jepsen.util 7966 :ok :write [15 26] INFO [2019-12-17 12:33:46,302] jepsen worker 46 - jepsen.util 7966 :invoke :read [15 nil] INFO [2019-12-17 12:33:46,312] jepsen worker 82 - jepsen.util 10792 :info :write [27 8] :RequestCanceledException INFO [2019-12-17 12:33:46,314] jepsen worker 82 - jepsen.util 10882 :invoke :write [27 26] INFO [2019-12-17 12:33:46,321] jepsen worker 9 - jepsen.util 7929 :ok :write [3 1] INFO [2019-12-17 12:33:46,321] jepsen worker 9 - jepsen.util 7929 :invoke :cas [3 [25 17]] INFO [2019-12-17 12:33:46,328] jepsen worker 1 - jepsen.util 10261 :info :write [0 9] :RequestCanceledException INFO [2019-12-17 12:33:46,330] jepsen worker 1 - jepsen.util 10351 :invoke :write [0 49] INFO [2019-12-17 12:33:46,339] jepsen worker 69 - jepsen.util 7809 :fail :cas [23 [26 18]] :ValueNotSwapFrom INFO [2019-12-17 12:33:46,340] jepsen worker 69 - jepsen.util 7809 :invoke :cas [23 [12 46]] INFO [2019-12-17 12:33:46,342] jepsen worker 6 - jepsen.util 10896 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:33:46,342] jepsen worker 6 - jepsen.util 10896 :invoke :read [2 nil] INFO [2019-12-17 12:33:46,345] jepsen worker 46 - jepsen.util 7966 :ok :read [15 26] INFO [2019-12-17 12:33:46,345] jepsen worker 46 - jepsen.util 7966 :invoke :write [15 44] INFO [2019-12-17 12:33:46,350] jepsen worker 32 - jepsen.util 7502 :fail :cas [10 [23 45]] :ValueNotSwapFrom INFO [2019-12-17 12:33:46,351] jepsen worker 32 - jepsen.util 7502 :invoke :write [10 13] INFO [2019-12-17 12:33:46,355] jepsen worker 9 - jepsen.util 7929 :fail :cas [3 [25 17]] :ValueNotSwapFrom INFO [2019-12-17 12:33:46,355] jepsen worker 9 - jepsen.util 7929 :invoke :cas [3 [18 8]] INFO [2019-12-17 12:33:46,359] jepsen worker 56 - jepsen.util 8426 :info :write [18 42] :RequestCanceledException INFO [2019-12-17 12:33:46,360] jepsen worker 56 - jepsen.util 8516 :invoke :read [18 nil] INFO [2019-12-17 12:33:46,372] jepsen worker 72 - jepsen.util 7272 :ok :write [24 5] INFO [2019-12-17 12:33:46,372] jepsen worker 72 - jepsen.util 7272 :invoke :cas [24 [24 6]] INFO [2019-12-17 12:33:46,413] jepsen worker 14 - jepsen.util 7754 :info :write [4 44] :RequestCanceledException INFO [2019-12-17 12:33:46,414] jepsen worker 14 - jepsen.util 7844 :invoke :read [4 nil] INFO [2019-12-17 12:33:46,438] jepsen worker 37 - jepsen.util 7147 :ok :write [12 46] INFO [2019-12-17 12:33:46,438] jepsen worker 37 - jepsen.util 7147 :invoke :read [12 nil] INFO [2019-12-17 12:33:46,441] jepsen worker 4 - jepsen.util 6934 :info :write [1 10] :RequestCanceledException INFO [2019-12-17 12:33:46,442] jepsen worker 4 - jepsen.util 7024 :invoke :read [1 nil] INFO [2019-12-17 12:33:46,456] jepsen worker 9 - jepsen.util 7929 :fail :cas [3 [18 8]] :ValueNotSwapFrom INFO [2019-12-17 12:33:46,458] jepsen worker 64 - jepsen.util 5914 :ok :write [21 28] INFO [2019-12-17 12:33:46,459] jepsen worker 72 - jepsen.util 7272 :fail :cas [24 [24 6]] :ValueNotSwapFrom INFO [2019-12-17 12:33:46,460] jepsen worker 9 - jepsen.util 7929 :invoke :cas [3 [48 12]] INFO [2019-12-17 12:33:46,460] jepsen worker 57 - jepsen.util 10227 :info :write [19 4] :RequestCanceledException INFO [2019-12-17 12:33:46,470] jepsen worker 64 - jepsen.util 5914 :invoke :cas [21 [7 29]] INFO [2019-12-17 12:33:46,470] jepsen worker 72 - jepsen.util 7272 :invoke :read [24 nil] INFO [2019-12-17 12:33:46,471] jepsen worker 57 - jepsen.util 10317 :invoke :read [19 nil] INFO [2019-12-17 12:33:46,490] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":43572,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:33:46,490] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":43572,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:33:46,490] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":43573,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:33:46,490] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":43573,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:33:46,490] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":43574,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:33:46,490] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":43574,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:33:46,507] jepsen worker 72 - jepsen.util 7272 :ok :read [24 5] INFO [2019-12-17 12:33:46,507] jepsen worker 64 - jepsen.util 5914 :fail :cas [21 [7 29]] :ValueNotSwapFrom INFO [2019-12-17 12:33:46,508] jepsen worker 72 - jepsen.util 7272 :invoke :write [24 35] INFO [2019-12-17 12:33:46,508] jepsen worker 64 - jepsen.util 5914 :invoke :read [21 nil] INFO [2019-12-17 12:33:46,508] jepsen worker 9 - jepsen.util 7929 :fail :cas [3 [48 12]] :ValueNotSwapFrom INFO [2019-12-17 12:33:46,508] jepsen worker 9 - jepsen.util 7929 :invoke :read [3 nil] INFO [2019-12-17 12:33:46,513] jepsen worker 84 - jepsen.util 9804 :info :write [28 16] :RequestCanceledException INFO [2019-12-17 12:33:46,514] jepsen worker 84 - jepsen.util 9894 :invoke :write [28 44] INFO [2019-12-17 12:33:46,528] jepsen worker 3 - jepsen.util 7563 :ok :read [1 26] INFO [2019-12-17 12:33:46,528] jepsen worker 17 - jepsen.util 10457 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:33:46,528] jepsen worker 81 - jepsen.util 10071 :info :cas [27 [39 4]] :RequestCanceledException INFO [2019-12-17 12:33:46,528] jepsen worker 38 - jepsen.util 7778 :info :write [12 3] :RequestCanceledException INFO [2019-12-17 12:33:46,528] jepsen worker 3 - jepsen.util 7563 :invoke :write [1 41] INFO [2019-12-17 12:33:46,528] jepsen worker 17 - jepsen.util 10457 :invoke :read [5 nil] INFO [2019-12-17 12:33:46,529] jepsen worker 81 - jepsen.util 10161 :invoke :read [27 nil] INFO [2019-12-17 12:33:46,529] jepsen worker 38 - jepsen.util 7868 :invoke :write [12 32] INFO [2019-12-17 12:33:46,559] jepsen worker 53 - jepsen.util 6983 :info :write [17 36] :RequestCanceledException INFO [2019-12-17 12:33:46,560] jepsen worker 53 - jepsen.util 7073 :invoke :read [17 nil] INFO [2019-12-17 12:33:46,570] jepsen worker 14 - jepsen.util 7844 :ok :read [4 21] INFO [2019-12-17 12:33:46,570] jepsen worker 14 - jepsen.util 7844 :invoke :read [4 nil] INFO [2019-12-17 12:33:46,574] jepsen worker 64 - jepsen.util 5914 :ok :read [21 28] INFO [2019-12-17 12:33:46,574] jepsen worker 64 - jepsen.util 5914 :invoke :write [21 35] INFO [2019-12-17 12:33:46,581] jepsen worker 9 - jepsen.util 7929 :ok :read [3 1] INFO [2019-12-17 12:33:46,581] jepsen worker 9 - jepsen.util 7929 :invoke :cas [3 [0 48]] INFO [2019-12-17 12:33:46,589] jepsen worker 69 - jepsen.util 7809 :fail :cas [23 [12 46]] :ValueNotSwapFrom INFO [2019-12-17 12:33:46,589] jepsen worker 69 - jepsen.util 7809 :invoke :cas [23 [8 5]] INFO [2019-12-17 12:33:46,595] jepsen worker 20 - jepsen.util 7220 :info :write [6 46] :RequestCanceledException INFO [2019-12-17 12:33:46,595] jepsen worker 48 - jepsen.util 9318 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:33:46,595] jepsen worker 48 - jepsen.util 9318 :invoke :write [16 15] INFO [2019-12-17 12:33:46,596] jepsen worker 20 - jepsen.util 7310 :invoke :write [6 27] INFO [2019-12-17 12:33:46,627] jepsen worker 9 - jepsen.util 7929 :fail :cas [3 [0 48]] :ValueNotSwapFrom INFO [2019-12-17 12:33:46,627] jepsen worker 9 - jepsen.util 7929 :invoke :read [3 nil] INFO [2019-12-17 12:33:46,629] jepsen worker 76 - jepsen.util 10246 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:33:46,629] jepsen worker 10 - jepsen.util 7390 :info :cas [3 [19 39]] :RequestCanceledException INFO [2019-12-17 12:33:46,629] jepsen worker 76 - jepsen.util 10246 :invoke :read [25 nil] INFO [2019-12-17 12:33:46,630] jepsen worker 10 - jepsen.util 7480 :invoke :cas [3 [8 1]] INFO [2019-12-17 12:33:46,641] jepsen worker 51 - jepsen.util 7251 :ok :write [17 21] INFO [2019-12-17 12:33:46,641] jepsen worker 51 - jepsen.util 7251 :invoke :cas [17 [4 14]] INFO [2019-12-17 12:33:46,653] jepsen worker 80 - jepsen.util 7550 :fail :cas [26 [16 3]] :ValueNotSwapFrom INFO [2019-12-17 12:33:46,653] jepsen worker 80 - jepsen.util 7550 :invoke :read [26 nil] INFO [2019-12-17 12:33:46,660] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":43604,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:33:46,660] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":43604,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:33:46,660] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":43607,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:33:46,660] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":43607,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:33:46,660] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":43608,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:33:46,660] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":43608,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:33:46,666] jepsen worker 9 - jepsen.util 7929 :ok :read [3 1] INFO [2019-12-17 12:33:46,666] jepsen worker 9 - jepsen.util 7929 :invoke :write [3 22] INFO [2019-12-17 12:33:46,691] jepsen worker 12 - jepsen.util 8112 :ok :write [4 49] INFO [2019-12-17 12:33:46,694] jepsen worker 12 - jepsen.util 8112 :invoke :cas [4 [8 12]] INFO [2019-12-17 12:33:46,695] jepsen worker 54 - jepsen.util 8424 :info :write [18 22] :RequestCanceledException INFO [2019-12-17 12:33:46,695] jepsen worker 39 - jepsen.util 9939 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:33:46,696] jepsen worker 39 - jepsen.util 9939 :invoke :read [13 nil] INFO [2019-12-17 12:33:46,697] jepsen worker 54 - jepsen.util 8514 :invoke :write [18 41] INFO [2019-12-17 12:33:46,698] jepsen worker 51 - jepsen.util 7251 :fail :cas [17 [4 14]] :ValueNotSwapFrom INFO [2019-12-17 12:33:46,698] jepsen worker 51 - jepsen.util 7251 :invoke :write [17 35] INFO [2019-12-17 12:33:46,717] jepsen worker 64 - jepsen.util 5914 :ok :write [21 35] INFO [2019-12-17 12:33:46,718] jepsen worker 64 - jepsen.util 5914 :invoke :write [21 14] INFO [2019-12-17 12:33:46,724] jepsen worker 46 - jepsen.util 7966 :ok :write [15 44] INFO [2019-12-17 12:33:46,724] jepsen worker 32 - jepsen.util 7502 :ok :write [10 13] INFO [2019-12-17 12:33:46,724] jepsen worker 46 - jepsen.util 7966 :invoke :read [15 nil] INFO [2019-12-17 12:33:46,724] jepsen worker 32 - jepsen.util 7502 :invoke :write [10 19] INFO [2019-12-17 12:33:46,742] jepsen worker 5 - jepsen.util 7115 :info :write [1 36] :RequestCanceledException INFO [2019-12-17 12:33:46,743] jepsen worker 5 - jepsen.util 7205 :invoke :write [1 11] INFO [2019-12-17 12:33:46,748] jepsen worker 78 - jepsen.util 7638 :ok :write [26 30] INFO [2019-12-17 12:33:46,748] jepsen worker 78 - jepsen.util 7638 :invoke :cas [26 [41 2]] INFO [2019-12-17 12:33:46,760] jepsen worker 7 - jepsen.util 10357 :info :cas [2 [24 17]] :RequestCanceledException INFO [2019-12-17 12:33:46,761] jepsen worker 7 - jepsen.util 10447 :invoke :read [2 nil] INFO [2019-12-17 12:33:46,772] jepsen worker 10 - jepsen.util 7480 :fail :cas [3 [8 1]] :ValueNotSwapFrom INFO [2019-12-17 12:33:46,773] jepsen worker 10 - jepsen.util 7480 :invoke :cas [3 [49 37]] INFO [2019-12-17 12:33:46,782] jepsen worker 37 - jepsen.util 7147 :ok :read [12 46] INFO [2019-12-17 12:33:46,782] jepsen worker 37 - jepsen.util 7147 :invoke :cas [12 [2 20]] INFO [2019-12-17 12:33:46,785] jepsen worker 46 - jepsen.util 7966 :ok :read [15 44] INFO [2019-12-17 12:33:46,786] jepsen worker 46 - jepsen.util 7966 :invoke :write [15 11] INFO [2019-12-17 12:33:46,795] jepsen worker 40 - jepsen.util 10570 :info :write [13 47] :RequestCanceledException INFO [2019-12-17 12:33:46,795] jepsen worker 40 - jepsen.util 10660 :invoke :write [13 0] INFO [2019-12-17 12:33:46,813] jepsen worker 44 - jepsen.util 10934 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:33:46,813] jepsen worker 44 - jepsen.util 10934 :invoke :read [14 nil] INFO [2019-12-17 12:33:46,824] jepsen worker 89 - jepsen.util 7379 :ok :write [29 2] INFO [2019-12-17 12:33:46,824] jepsen worker 89 - jepsen.util 7379 :invoke :write [29 20] INFO [2019-12-17 12:33:46,826] jepsen worker 10 - jepsen.util 7480 :fail :cas [3 [49 37]] :ValueNotSwapFrom INFO [2019-12-17 12:33:46,827] jepsen worker 10 - jepsen.util 7480 :invoke :write [3 6] INFO [2019-12-17 12:33:46,827] jepsen worker 49 - jepsen.util 9679 :info :cas [16 [35 27]] :RequestCanceledException INFO [2019-12-17 12:33:46,827] jepsen worker 16 - jepsen.util 11446 :info :write [5 3] :RequestCanceledException INFO [2019-12-17 12:33:46,828] jepsen worker 61 - jepsen.util 10861 :info :cas [20 [30 15]] :RequestCanceledException INFO [2019-12-17 12:33:46,828] jepsen worker 49 - jepsen.util 9769 :invoke :write [16 12] INFO [2019-12-17 12:33:46,829] jepsen worker 16 - jepsen.util 11536 :invoke :cas [5 [8 32]] INFO [2019-12-17 12:33:46,829] jepsen worker 61 - jepsen.util 10951 :invoke :read [20 nil] INFO [2019-12-17 12:33:46,843] jepsen worker 85 - jepsen.util 10975 :info :write [28 40] :RequestCanceledException INFO [2019-12-17 12:33:46,844] jepsen worker 85 - jepsen.util 11065 :invoke :cas [28 [49 47]] INFO [2019-12-17 12:33:46,845] jepsen worker 4 - jepsen.util 7024 :ok :read [1 26] INFO [2019-12-17 12:33:46,845] jepsen worker 4 - jepsen.util 7024 :invoke :cas [1 [4 6]] INFO [2019-12-17 12:33:46,859] jepsen worker 35 - jepsen.util 10025 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:33:46,859] jepsen worker 35 - jepsen.util 10025 :invoke :cas [11 [33 1]] INFO [2019-12-17 12:33:46,888] jepsen worker 64 - jepsen.util 5914 :ok :write [21 14] INFO [2019-12-17 12:33:46,889] jepsen worker 64 - jepsen.util 5914 :invoke :cas [21 [49 39]] INFO [2019-12-17 12:33:46,912] jepsen worker 64 - jepsen.util 5914 :fail :cas [21 [49 39]] :ValueNotSwapFrom INFO [2019-12-17 12:33:46,912] jepsen worker 64 - jepsen.util 5914 :invoke :cas [21 [33 4]] INFO [2019-12-17 12:33:46,913] jepsen worker 45 - jepsen.util 7425 :info :write [15 34] :RequestCanceledException INFO [2019-12-17 12:33:46,914] jepsen worker 45 - jepsen.util 7515 :invoke :read [15 nil] INFO [2019-12-17 12:33:46,927] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":43637,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:33:46,927] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":43637,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:33:46,927] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":43639,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:33:46,927] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":43639,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:33:46,929] jepsen worker 8 - jepsen.util 10088 :info :cas [2 [6 16]] :RequestCanceledException INFO [2019-12-17 12:33:46,930] jepsen worker 8 - jepsen.util 10178 :invoke :read [2 nil] INFO [2019-12-17 12:33:46,943] jepsen worker 64 - jepsen.util 5914 :fail :cas [21 [33 4]] :ValueNotSwapFrom INFO [2019-12-17 12:33:46,943] jepsen worker 64 - jepsen.util 5914 :invoke :cas [21 [41 31]] INFO [2019-12-17 12:33:46,951] jepsen worker 45 - jepsen.util 7515 :ok :read [15 44] INFO [2019-12-17 12:33:46,951] jepsen worker 45 - jepsen.util 7515 :invoke :cas [15 [22 14]] INFO [2019-12-17 12:33:46,960] jepsen worker 52 - jepsen.util 7792 :info :write [17 14] :RequestCanceledException INFO [2019-12-17 12:33:46,961] jepsen worker 52 - jepsen.util 7882 :invoke :write [17 7] INFO [2019-12-17 12:33:46,961] jepsen worker 64 - jepsen.util 5914 :fail :cas [21 [41 31]] :ValueNotSwapFrom INFO [2019-12-17 12:33:46,961] jepsen worker 64 - jepsen.util 5914 :invoke :cas [21 [1 33]] INFO [2019-12-17 12:33:46,975] jepsen worker 69 - jepsen.util 7809 :fail :cas [23 [8 5]] :ValueNotSwapFrom INFO [2019-12-17 12:33:46,975] jepsen worker 69 - jepsen.util 7809 :invoke :read [23 nil] INFO [2019-12-17 12:33:46,975] jepsen worker 45 - jepsen.util 7515 :fail :cas [15 [22 14]] :ValueNotSwapFrom INFO [2019-12-17 12:33:46,975] jepsen worker 45 - jepsen.util 7515 :invoke :write [15 24] INFO [2019-12-17 12:33:47,011] jepsen worker 64 - jepsen.util 5914 :fail :cas [21 [1 33]] :ValueNotSwapFrom INFO [2019-12-17 12:33:47,012] jepsen worker 64 - jepsen.util 5914 :invoke :cas [21 [22 19]] INFO [2019-12-17 12:33:47,014] jepsen worker 27 - jepsen.util 7497 :info :write [9 9] :RequestCanceledException INFO [2019-12-17 12:33:47,015] jepsen worker 27 - jepsen.util 7587 :invoke :cas [9 [3 0]] INFO [2019-12-17 12:33:47,017] jepsen worker 4 - jepsen.util 7024 :fail :cas [1 [4 6]] :ValueNotSwapFrom INFO [2019-12-17 12:33:47,017] jepsen worker 4 - jepsen.util 7024 :invoke :read [1 nil] INFO [2019-12-17 12:33:47,020] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:33:47,020] jepsen nemesis - couchbase.util Rebalance check count: 334 INFO [2019-12-17 12:33:47,020] jepsen nemesis - couchbase.util Rebalance stuck count: 315 INFO [2019-12-17 12:33:47,029] jepsen worker 19 - jepsen.util 7669 :info :write [6 0] :RequestCanceledException INFO [2019-12-17 12:33:47,036] jepsen worker 19 - jepsen.util 7759 :invoke :read [6 nil] INFO [2019-12-17 12:33:47,037] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":43650,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:33:47,037] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":43650,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:33:47,037] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":43651,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:33:47,037] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":43651,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:33:47,037] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":43655,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:33:47,037] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":43655,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:33:47,095] jepsen worker 28 - jepsen.util 5968 :info :cas [9 [8 16]] :RequestCanceledException INFO [2019-12-17 12:33:47,096] jepsen worker 28 - jepsen.util 6058 :invoke :cas [9 [5 33]] INFO [2019-12-17 12:33:47,097] jepsen worker 78 - jepsen.util 7638 :fail :cas [26 [41 2]] :ValueNotSwapFrom INFO [2019-12-17 12:33:47,097] jepsen worker 78 - jepsen.util 7638 :invoke :cas [26 [29 26]] INFO [2019-12-17 12:33:47,109] jepsen worker 56 - jepsen.util 8516 :ok :read [18 27] INFO [2019-12-17 12:33:47,109] jepsen worker 56 - jepsen.util 8516 :invoke :write [18 19] INFO [2019-12-17 12:33:47,113] jepsen worker 37 - jepsen.util 7147 :fail :cas [12 [2 20]] :ValueNotSwapFrom INFO [2019-12-17 12:33:47,113] jepsen worker 37 - jepsen.util 7147 :invoke :read [12 nil] INFO [2019-12-17 12:33:47,114] jepsen worker 70 - jepsen.util 7810 :info :write [23 0] :RequestCanceledException INFO [2019-12-17 12:33:47,114] jepsen worker 31 - jepsen.util 6691 :info :write [10 21] :RequestCanceledException INFO [2019-12-17 12:33:47,114] jepsen worker 32 - jepsen.util 7502 :ok :write [10 19] INFO [2019-12-17 12:33:47,114] jepsen worker 32 - jepsen.util 7502 :invoke :read [10 nil] INFO [2019-12-17 12:33:47,114] jepsen worker 9 - jepsen.util 7929 :ok :write [3 22] INFO [2019-12-17 12:33:47,114] jepsen worker 9 - jepsen.util 7929 :invoke :read [3 nil] INFO [2019-12-17 12:33:47,115] jepsen worker 70 - jepsen.util 7900 :invoke :cas [23 [27 10]] INFO [2019-12-17 12:33:47,115] jepsen worker 31 - jepsen.util 6781 :invoke :cas [10 [11 19]] INFO [2019-12-17 12:33:47,115] jepsen worker 72 - jepsen.util 7272 :ok :write [24 35] INFO [2019-12-17 12:33:47,115] jepsen worker 72 - jepsen.util 7272 :invoke :cas [24 [14 0]] INFO [2019-12-17 12:33:47,135] jepsen worker 28 - jepsen.util 6058 :fail :cas [9 [5 33]] :ValueNotSwapFrom INFO [2019-12-17 12:33:47,135] jepsen worker 28 - jepsen.util 6058 :invoke :write [9 47] INFO [2019-12-17 12:33:47,143] jepsen worker 12 - jepsen.util 8112 :fail :cas [4 [8 12]] :ValueNotSwapFrom INFO [2019-12-17 12:33:47,143] jepsen worker 12 - jepsen.util 8112 :invoke :cas [4 [30 29]] INFO [2019-12-17 12:33:47,160] jepsen worker 51 - jepsen.util 7251 :ok :write [17 35] INFO [2019-12-17 12:33:47,161] jepsen worker 51 - jepsen.util 7251 :invoke :read [17 nil] INFO [2019-12-17 12:33:47,162] jepsen worker 19 - jepsen.util 7759 :ok :read [6 11] INFO [2019-12-17 12:33:47,162] jepsen worker 19 - jepsen.util 7759 :invoke :read [6 nil] INFO [2019-12-17 12:33:47,176] jepsen worker 3 - jepsen.util 7563 :ok :write [1 41] INFO [2019-12-17 12:33:47,176] jepsen worker 3 - jepsen.util 7563 :invoke :read [1 nil] INFO [2019-12-17 12:33:47,188] jepsen worker 13 - jepsen.util 7213 :ok :write [4 18] INFO [2019-12-17 12:33:47,188] jepsen worker 13 - jepsen.util 7213 :invoke :cas [4 [34 22]] INFO [2019-12-17 12:33:47,190] jepsen worker 31 - jepsen.util 6781 :fail :cas [10 [11 19]] :ValueNotSwapFrom INFO [2019-12-17 12:33:47,190] jepsen worker 31 - jepsen.util 6781 :invoke :cas [10 [17 10]] INFO [2019-12-17 12:33:47,193] jepsen worker 4 - jepsen.util 7024 :ok :read [1 41] INFO [2019-12-17 12:33:47,193] jepsen worker 4 - jepsen.util 7024 :invoke :read [1 nil] INFO [2019-12-17 12:33:47,207] jepsen worker 32 - jepsen.util 7502 :ok :read [10 19] INFO [2019-12-17 12:33:47,207] jepsen worker 32 - jepsen.util 7502 :invoke :read [10 nil] INFO [2019-12-17 12:33:47,212] jepsen worker 88 - jepsen.util 7468 :info :cas [29 [0 27]] :RequestCanceledException INFO [2019-12-17 12:33:47,212] jepsen worker 71 - jepsen.util 8171 :info :write [23 48] :RequestCanceledException INFO [2019-12-17 12:33:47,214] jepsen worker 71 - jepsen.util 8261 :invoke :read [23 nil] INFO [2019-12-17 12:33:47,216] jepsen worker 88 - jepsen.util 7558 :invoke :read [29 nil] INFO [2019-12-17 12:33:47,223] jepsen worker 9 - jepsen.util 7929 :ok :read [3 22] INFO [2019-12-17 12:33:47,223] jepsen worker 9 - jepsen.util 7929 :invoke :write [3 34] INFO [2019-12-17 12:33:47,233] jepsen worker 46 - jepsen.util 7966 :ok :write [15 11] INFO [2019-12-17 12:33:47,233] jepsen worker 46 - jepsen.util 7966 :invoke :cas [15 [1 10]] INFO [2019-12-17 12:33:47,239] jepsen worker 31 - jepsen.util 6781 :fail :cas [10 [17 10]] :ValueNotSwapFrom INFO [2019-12-17 12:33:47,239] jepsen worker 31 - jepsen.util 6781 :invoke :cas [10 [35 46]] INFO [2019-12-17 12:33:47,242] jepsen worker 2 - jepsen.util 9902 :info :write [0 43] :RequestCanceledException INFO [2019-12-17 12:33:47,243] jepsen worker 2 - jepsen.util 9992 :invoke :cas [0 [45 26]] INFO [2019-12-17 12:33:47,256] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":43677,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:33:47,256] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":43688,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:33:47,256] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":43677,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:33:47,256] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":43688,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:33:47,256] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":43678,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:33:47,257] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":43690,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:33:47,257] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":43678,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:33:47,257] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":43690,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:33:47,257] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":43679,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:33:47,257] jepsen worker 11 - jepsen.util 7661 :fail :read [3 nil] :RequestTimeoutException INFO [2019-12-17 12:33:47,257] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":43679,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:33:47,257] jepsen worker 74 - jepsen.util 7634 :info :cas [24 [20 1]] :RequestCanceledException INFO [2019-12-17 12:33:47,257] jepsen worker 11 - jepsen.util 7661 :invoke :write [3 21] INFO [2019-12-17 12:33:47,258] jepsen worker 74 - jepsen.util 7724 :invoke :cas [24 [7 42]] INFO [2019-12-17 12:33:47,276] jepsen worker 53 - jepsen.util 7073 :ok :read [17 35] INFO [2019-12-17 12:33:47,277] jepsen worker 53 - jepsen.util 7073 :invoke :write [17 14] INFO [2019-12-17 12:33:47,290] jepsen worker 19 - jepsen.util 7759 :ok :read [6 11] INFO [2019-12-17 12:33:47,291] jepsen worker 19 - jepsen.util 7759 :invoke :read [6 nil] INFO [2019-12-17 12:33:47,301] jepsen worker 31 - jepsen.util 6781 :fail :cas [10 [35 46]] :ValueNotSwapFrom INFO [2019-12-17 12:33:47,301] jepsen worker 31 - jepsen.util 6781 :invoke :write [10 8] INFO [2019-12-17 12:33:47,309] jepsen worker 27 - jepsen.util 7587 :fail :cas [9 [3 0]] :ValueNotSwapFrom INFO [2019-12-17 12:33:47,310] jepsen worker 27 - jepsen.util 7587 :invoke :cas [9 [7 40]] INFO [2019-12-17 12:33:47,335] jepsen worker 46 - jepsen.util 7966 :fail :cas [15 [1 10]] :ValueNotSwapFrom INFO [2019-12-17 12:33:47,336] jepsen worker 46 - jepsen.util 7966 :invoke :read [15 nil] INFO [2019-12-17 12:33:47,342] jepsen worker 79 - jepsen.util 7639 :info :write [26 39] :RequestCanceledException INFO [2019-12-17 12:33:47,343] jepsen worker 79 - jepsen.util 7729 :invoke :write [26 30] INFO [2019-12-17 12:33:47,350] jepsen worker 27 - jepsen.util 7587 :fail :cas [9 [7 40]] :ValueNotSwapFrom INFO [2019-12-17 12:33:47,350] jepsen worker 27 - jepsen.util 7587 :invoke :cas [9 [36 15]] INFO [2019-12-17 12:33:47,352] jepsen worker 14 - jepsen.util 7844 :ok :read [4 18] INFO [2019-12-17 12:33:47,352] jepsen worker 14 - jepsen.util 7844 :invoke :write [4 40] INFO [2019-12-17 12:33:47,372] jepsen worker 51 - jepsen.util 7251 :ok :read [17 35] INFO [2019-12-17 12:33:47,372] jepsen worker 51 - jepsen.util 7251 :invoke :write [17 36] INFO [2019-12-17 12:33:47,376] jepsen worker 28 - jepsen.util 6058 :ok :write [9 47] INFO [2019-12-17 12:33:47,376] jepsen worker 28 - jepsen.util 6058 :invoke :write [9 25] INFO [2019-12-17 12:33:47,386] jepsen worker 46 - jepsen.util 7966 :ok :read [15 11] INFO [2019-12-17 12:33:47,386] jepsen worker 46 - jepsen.util 7966 :invoke :read [15 nil] INFO [2019-12-17 12:33:47,395] jepsen worker 65 - jepsen.util 6275 :fail :read [21 nil] :RequestTimeoutException INFO [2019-12-17 12:33:47,396] jepsen worker 65 - jepsen.util 6275 :invoke :write [21 2] INFO [2019-12-17 12:33:47,396] jepsen worker 52 - jepsen.util 7882 :ok :write [17 7] INFO [2019-12-17 12:33:47,397] jepsen worker 52 - jepsen.util 7882 :invoke :cas [17 [2 47]] INFO [2019-12-17 12:33:47,406] jepsen worker 32 - jepsen.util 7502 :ok :read [10 19] INFO [2019-12-17 12:33:47,406] jepsen worker 32 - jepsen.util 7502 :invoke :cas [10 [35 23]] INFO [2019-12-17 12:33:47,412] jepsen worker 63 - jepsen.util 6453 :info :cas [21 [10 29]] :RequestCanceledException INFO [2019-12-17 12:33:47,413] jepsen worker 63 - jepsen.util 6543 :invoke :read [21 nil] INFO [2019-12-17 12:33:47,423] jepsen worker 27 - jepsen.util 7587 :fail :cas [9 [36 15]] :ValueNotSwapFrom INFO [2019-12-17 12:33:47,423] jepsen worker 27 - jepsen.util 7587 :invoke :cas [9 [1 25]] INFO [2019-12-17 12:33:47,428] jepsen worker 73 - jepsen.util 7183 :info :write [24 47] :RequestCanceledException INFO [2019-12-17 12:33:47,428] jepsen worker 74 - jepsen.util 7724 :fail :cas [24 [7 42]] :ValueNotSwapFrom INFO [2019-12-17 12:33:47,428] jepsen worker 74 - jepsen.util 7724 :invoke :cas [24 [5 44]] INFO [2019-12-17 12:33:47,429] jepsen worker 73 - jepsen.util 7273 :invoke :read [24 nil] INFO [2019-12-17 12:33:47,437] jepsen worker 88 - jepsen.util 7558 :ok :read [29 2] INFO [2019-12-17 12:33:47,438] jepsen worker 88 - jepsen.util 7558 :invoke :cas [29 [11 13]] INFO [2019-12-17 12:33:47,454] jepsen worker 63 - jepsen.util 6543 :ok :read [21 14] INFO [2019-12-17 12:33:47,454] jepsen worker 63 - jepsen.util 6543 :invoke :read [21 nil] INFO [2019-12-17 12:33:47,454] jepsen worker 69 - jepsen.util 7809 :ok :read [23 28] INFO [2019-12-17 12:33:47,454] jepsen worker 69 - jepsen.util 7809 :invoke :write [23 46] INFO [2019-12-17 12:33:47,460] jepsen worker 55 - jepsen.util 8425 :ok :write [18 33] INFO [2019-12-17 12:33:47,461] jepsen worker 55 - jepsen.util 8425 :invoke :read [18 nil] INFO [2019-12-17 12:33:47,472] jepsen worker 46 - jepsen.util 7966 :ok :read [15 11] INFO [2019-12-17 12:33:47,472] jepsen worker 46 - jepsen.util 7966 :invoke :write [15 19] INFO [2019-12-17 12:33:47,475] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":43717,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:33:47,475] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":43717,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:33:47,475] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":43720,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:33:47,476] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":43720,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:33:47,479] jepsen worker 28 - jepsen.util 6058 :ok :write [9 25] INFO [2019-12-17 12:33:47,479] jepsen worker 28 - jepsen.util 6058 :invoke :write [9 25] INFO [2019-12-17 12:33:47,486] jepsen worker 63 - jepsen.util 6543 :ok :read [21 14] INFO [2019-12-17 12:33:47,486] jepsen worker 63 - jepsen.util 6543 :invoke :read [21 nil] INFO [2019-12-17 12:33:47,502] jepsen worker 74 - jepsen.util 7724 :fail :cas [24 [5 44]] :ValueNotSwapFrom INFO [2019-12-17 12:33:47,502] jepsen worker 74 - jepsen.util 7724 :invoke :write [24 16] INFO [2019-12-17 12:33:47,521] jepsen worker 63 - jepsen.util 6543 :ok :read [21 14] INFO [2019-12-17 12:33:47,521] jepsen worker 63 - jepsen.util 6543 :invoke :read [21 nil] INFO [2019-12-17 12:33:47,528] jepsen worker 29 - jepsen.util 5969 :info :write [9 0] :RequestCanceledException INFO [2019-12-17 12:33:47,529] jepsen worker 29 - jepsen.util 6059 :invoke :write [9 40] INFO [2019-12-17 12:33:47,540] jepsen worker 65 - jepsen.util 6275 :ok :write [21 2] INFO [2019-12-17 12:33:47,540] jepsen worker 65 - jepsen.util 6275 :invoke :cas [21 [14 36]] INFO [2019-12-17 12:33:47,548] jepsen worker 78 - jepsen.util 7638 :fail :cas [26 [29 26]] :ValueNotSwapFrom INFO [2019-12-17 12:33:47,548] jepsen worker 78 - jepsen.util 7638 :invoke :read [26 nil] INFO [2019-12-17 12:33:47,575] jepsen worker 63 - jepsen.util 6543 :ok :read [21 2] INFO [2019-12-17 12:33:47,575] jepsen worker 63 - jepsen.util 6543 :invoke :read [21 nil] INFO [2019-12-17 12:33:47,594] jepsen worker 28 - jepsen.util 6058 :ok :write [9 25] INFO [2019-12-17 12:33:47,595] jepsen worker 28 - jepsen.util 6058 :invoke :read [9 nil] INFO [2019-12-17 12:33:47,601] jepsen worker 63 - jepsen.util 6543 :ok :read [21 2] INFO [2019-12-17 12:33:47,601] jepsen worker 63 - jepsen.util 6543 :invoke :read [21 nil] INFO [2019-12-17 12:33:47,624] jepsen worker 28 - jepsen.util 6058 :ok :read [9 25] INFO [2019-12-17 12:33:47,624] jepsen worker 28 - jepsen.util 6058 :invoke :read [9 nil] INFO [2019-12-17 12:33:47,624] jepsen worker 37 - jepsen.util 7147 :ok :read [12 46] INFO [2019-12-17 12:33:47,624] jepsen worker 37 - jepsen.util 7147 :invoke :cas [12 [31 8]] INFO [2019-12-17 12:33:47,630] jepsen worker 63 - jepsen.util 6543 :ok :read [21 2] INFO [2019-12-17 12:33:47,630] jepsen worker 63 - jepsen.util 6543 :invoke :write [21 39] INFO [2019-12-17 12:33:47,631] jepsen worker 36 - jepsen.util 7866 :fail :cas [12 [1 20]] :ValueNotSwapFrom INFO [2019-12-17 12:33:47,631] jepsen worker 36 - jepsen.util 7866 :invoke :write [12 47] INFO [2019-12-17 12:33:47,640] jepsen worker 87 - jepsen.util 6927 :info :write [29 26] :RequestCanceledException INFO [2019-12-17 12:33:47,641] jepsen worker 87 - jepsen.util 7017 :invoke :cas [29 [44 44]] INFO [2019-12-17 12:33:47,646] jepsen worker 28 - jepsen.util 6058 :ok :read [9 25] INFO [2019-12-17 12:33:47,646] jepsen worker 28 - jepsen.util 6058 :invoke :write [9 18] INFO [2019-12-17 12:33:47,685] jepsen worker 63 - jepsen.util 6543 :ok :write [21 39] INFO [2019-12-17 12:33:47,685] jepsen worker 63 - jepsen.util 6543 :invoke :read [21 nil] INFO [2019-12-17 12:33:47,699] jepsen worker 3 - jepsen.util 7563 :ok :read [1 41] INFO [2019-12-17 12:33:47,699] jepsen worker 3 - jepsen.util 7563 :invoke :cas [1 [5 2]] INFO [2019-12-17 12:33:47,710] jepsen worker 63 - jepsen.util 6543 :ok :read [21 39] INFO [2019-12-17 12:33:47,710] jepsen worker 63 - jepsen.util 6543 :invoke :write [21 30] INFO [2019-12-17 12:33:47,724] jepsen worker 28 - jepsen.util 6058 :ok :write [9 18] INFO [2019-12-17 12:33:47,724] jepsen worker 28 - jepsen.util 6058 :invoke :read [9 nil] INFO [2019-12-17 12:33:47,745] jepsen worker 4 - jepsen.util 7024 :ok :read [1 41] INFO [2019-12-17 12:33:47,746] jepsen worker 4 - jepsen.util 7024 :invoke :read [1 nil] INFO [2019-12-17 12:33:47,749] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":43744,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:33:47,750] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":43744,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:33:47,750] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":43746,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:33:47,750] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":43746,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:33:47,775] jepsen worker 28 - jepsen.util 6058 :ok :read [9 18] INFO [2019-12-17 12:33:47,775] jepsen worker 28 - jepsen.util 6058 :invoke :write [9 3] INFO [2019-12-17 12:33:47,785] jepsen worker 63 - jepsen.util 6543 :ok :write [21 30] INFO [2019-12-17 12:33:47,785] jepsen worker 63 - jepsen.util 6543 :invoke :write [21 11] INFO [2019-12-17 12:33:47,801] jepsen worker 89 - jepsen.util 7379 :ok :write [29 20] INFO [2019-12-17 12:33:47,801] jepsen worker 89 - jepsen.util 7379 :invoke :read [29 nil] INFO [2019-12-17 12:33:47,804] jepsen worker 71 - jepsen.util 8261 :ok :read [23 48] INFO [2019-12-17 12:33:47,804] jepsen worker 71 - jepsen.util 8261 :invoke :cas [23 [21 23]] INFO [2019-12-17 12:33:47,846] jepsen worker 28 - jepsen.util 6058 :ok :write [9 3] INFO [2019-12-17 12:33:47,846] jepsen worker 28 - jepsen.util 6058 :invoke :read [9 nil] INFO [2019-12-17 12:33:47,849] jepsen worker 52 - jepsen.util 7882 :fail :cas [17 [2 47]] :ValueNotSwapFrom INFO [2019-12-17 12:33:47,849] jepsen worker 52 - jepsen.util 7882 :invoke :cas [17 [36 27]] INFO [2019-12-17 12:33:47,877] jepsen worker 28 - jepsen.util 6058 :ok :read [9 3] INFO [2019-12-17 12:33:47,877] jepsen worker 28 - jepsen.util 6058 :invoke :write [9 40] INFO [2019-12-17 12:33:47,880] jepsen worker 9 - jepsen.util 7929 :ok :write [3 34] INFO [2019-12-17 12:33:47,880] jepsen worker 9 - jepsen.util 7929 :invoke :write [3 13] INFO [2019-12-17 12:33:47,880] jepsen worker 46 - jepsen.util 7966 :ok :write [15 19] INFO [2019-12-17 12:33:47,880] jepsen worker 20 - jepsen.util 7310 :ok :write [6 27] INFO [2019-12-17 12:33:47,881] jepsen worker 46 - jepsen.util 7966 :invoke :read [15 nil] INFO [2019-12-17 12:33:47,881] jepsen worker 20 - jepsen.util 7310 :invoke :cas [6 [16 43]] INFO [2019-12-17 12:33:47,886] jepsen worker 5 - jepsen.util 7205 :ok :write [1 11] INFO [2019-12-17 12:33:47,886] jepsen worker 5 - jepsen.util 7205 :invoke :cas [1 [49 21]] INFO [2019-12-17 12:33:47,914] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":43760,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:33:47,928] jepsen worker 26 - jepsen.util 10646 :info :write [8 47] :RequestCanceledException INFO [2019-12-17 12:33:47,932] jepsen worker 73 - jepsen.util 7273 :ok :read [24 35] INFO [2019-12-17 12:33:47,933] jepsen worker 46 - jepsen.util 7966 :ok :read [15 19] INFO [2019-12-17 12:33:47,942] jepsen worker 51 - jepsen.util 7251 :ok :write [17 36] INFO [2019-12-17 12:33:47,950] jepsen worker 20 - jepsen.util 7310 :fail :cas [6 [16 43]] :ValueNotSwapFrom INFO [2019-12-17 12:33:47,959] jepsen worker 34 - jepsen.util 9754 :info :cas [11 [25 30]] :RequestCanceledException INFO [2019-12-17 12:33:47,959] jepsen worker 30 - jepsen.util 7410 :info :write [10 7] :RequestCanceledException INFO [2019-12-17 12:33:47,959] jepsen worker 63 - jepsen.util 6543 :ok :write [21 11] INFO [2019-12-17 12:33:47,961] jepsen worker 12 - jepsen.util 8112 :fail :cas [4 [30 29]] :ValueNotSwapFrom INFO [2019-12-17 12:33:47,969] jepsen worker 80 - jepsen.util 7550 :ok :read [26 39] INFO [2019-12-17 12:33:47,972] jepsen worker 28 - jepsen.util 6058 :ok :write [9 40] INFO [2019-12-17 12:33:47,991] jepsen worker 38 - jepsen.util 7868 :ok :write [12 32] INFO [2019-12-17 12:33:47,995] jepsen worker 18 - jepsen.util 7758 :fail :read [6 nil] :RequestTimeoutException INFO [2019-12-17 12:33:47,996] jepsen worker 55 - jepsen.util 8425 :ok :read [18 33] INFO [2019-12-17 12:33:48,017] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":43760,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:33:48,017] jepsen worker 73 - jepsen.util 7273 :invoke :read [24 nil] INFO [2019-12-17 12:33:48,018] jepsen worker 46 - jepsen.util 7966 :invoke :read [15 nil] INFO [2019-12-17 12:33:48,018] jepsen worker 26 - jepsen.util 10736 :invoke :read [8 nil] INFO [2019-12-17 12:33:48,018] jepsen worker 51 - jepsen.util 7251 :invoke :cas [17 [20 27]] INFO [2019-12-17 12:33:48,018] jepsen worker 20 - jepsen.util 7310 :invoke :cas [6 [7 36]] INFO [2019-12-17 12:33:48,018] jepsen worker 63 - jepsen.util 6543 :invoke :write [21 8] INFO [2019-12-17 12:33:48,018] jepsen worker 12 - jepsen.util 8112 :invoke :read [4 nil] INFO [2019-12-17 12:33:48,018] jepsen worker 80 - jepsen.util 7550 :invoke :write [26 33] INFO [2019-12-17 12:33:48,018] jepsen worker 28 - jepsen.util 6058 :invoke :write [9 19] INFO [2019-12-17 12:33:48,018] jepsen worker 38 - jepsen.util 7868 :invoke :write [12 42] INFO [2019-12-17 12:33:48,018] jepsen worker 18 - jepsen.util 7758 :invoke :read [6 nil] INFO [2019-12-17 12:33:48,018] jepsen worker 55 - jepsen.util 8425 :invoke :write [18 22] INFO [2019-12-17 12:33:48,018] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":43764,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:33:48,019] jepsen worker 30 - jepsen.util 7500 :invoke :cas [10 [9 30]] INFO [2019-12-17 12:33:48,019] jepsen worker 34 - jepsen.util 9844 :invoke :write [11 32] INFO [2019-12-17 12:33:48,019] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":43764,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:33:48,019] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":43765,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:33:48,019] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":43765,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:33:48,022] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":43772,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:33:48,023] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":43772,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:33:48,023] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":43775,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:33:48,023] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":43775,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:33:48,031] jepsen worker 70 - jepsen.util 7900 :fail :cas [23 [27 10]] :ValueNotSwapFrom INFO [2019-12-17 12:33:48,031] jepsen worker 70 - jepsen.util 7900 :invoke :cas [23 [19 13]] INFO [2019-12-17 12:33:48,038] jepsen worker 69 - jepsen.util 7809 :ok :write [23 46] INFO [2019-12-17 12:33:48,038] jepsen worker 69 - jepsen.util 7809 :invoke :read [23 nil] INFO [2019-12-17 12:33:48,039] jepsen worker 5 - jepsen.util 7205 :fail :cas [1 [49 21]] :ValueNotSwapFrom INFO [2019-12-17 12:33:48,039] jepsen worker 5 - jepsen.util 7205 :invoke :read [1 nil] INFO [2019-12-17 12:33:48,052] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:33:48,053] jepsen nemesis - couchbase.util Rebalance check count: 335 INFO [2019-12-17 12:33:48,053] jepsen nemesis - couchbase.util Rebalance stuck count: 316 INFO [2019-12-17 12:33:48,060] jepsen worker 47 - jepsen.util 7787 :info :write [15 48] :RequestCanceledException INFO [2019-12-17 12:33:48,060] jepsen worker 42 - jepsen.util 10932 :info :write [14 13] :RequestCanceledException INFO [2019-12-17 12:33:48,061] jepsen worker 47 - jepsen.util 7877 :invoke :read [15 nil] INFO [2019-12-17 12:33:48,061] jepsen worker 42 - jepsen.util 11022 :invoke :cas [14 [34 19]] INFO [2019-12-17 12:33:48,083] jepsen worker 13 - jepsen.util 7213 :fail :cas [4 [34 22]] :ValueNotSwapFrom INFO [2019-12-17 12:33:48,083] jepsen worker 13 - jepsen.util 7213 :invoke :read [4 nil] INFO [2019-12-17 12:33:48,095] jepsen worker 43 - jepsen.util 9763 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:33:48,095] jepsen worker 43 - jepsen.util 9763 :invoke :write [14 29] INFO [2019-12-17 12:33:48,096] jepsen worker 37 - jepsen.util 7147 :fail :cas [12 [31 8]] :ValueNotSwapFrom INFO [2019-12-17 12:33:48,096] jepsen worker 37 - jepsen.util 7147 :invoke :read [12 nil] INFO [2019-12-17 12:33:48,107] jepsen worker 78 - jepsen.util 7638 :ok :read [26 39] INFO [2019-12-17 12:33:48,107] jepsen worker 78 - jepsen.util 7638 :invoke :read [26 nil] INFO [2019-12-17 12:33:48,110] jepsen worker 73 - jepsen.util 7273 :ok :read [24 35] INFO [2019-12-17 12:33:48,110] jepsen worker 73 - jepsen.util 7273 :invoke :cas [24 [20 41]] INFO [2019-12-17 12:33:48,113] jepsen worker 77 - jepsen.util 10517 :info :cas [25 [35 21]] :RequestCanceledException INFO [2019-12-17 12:33:48,113] jepsen worker 77 - jepsen.util 10607 :invoke :write [25 49] INFO [2019-12-17 12:33:48,120] jepsen worker 46 - jepsen.util 7966 :ok :read [15 19] INFO [2019-12-17 12:33:48,120] jepsen worker 46 - jepsen.util 7966 :invoke :read [15 nil] INFO [2019-12-17 12:33:48,128] jepsen worker 83 - jepsen.util 10523 :info :cas [27 [1 49]] :RequestCanceledException INFO [2019-12-17 12:33:48,128] jepsen worker 75 - jepsen.util 9795 :info :write [25 0] :RequestCanceledException INFO [2019-12-17 12:33:48,129] jepsen worker 66 - jepsen.util 10146 :info :cas [22 [7 12]] :RequestCanceledException INFO [2019-12-17 12:33:48,129] jepsen worker 83 - jepsen.util 10613 :invoke :write [27 22] INFO [2019-12-17 12:33:48,130] jepsen worker 66 - jepsen.util 10236 :invoke :read [22 nil] INFO [2019-12-17 12:33:48,130] jepsen worker 75 - jepsen.util 9885 :invoke :read [25 nil] INFO [2019-12-17 12:33:48,130] jepsen worker 20 - jepsen.util 7310 :fail :cas [6 [7 36]] :ValueNotSwapFrom INFO [2019-12-17 12:33:48,130] jepsen worker 20 - jepsen.util 7310 :invoke :write [6 47] INFO [2019-12-17 12:33:48,134] jepsen worker 9 - jepsen.util 7929 :ok :write [3 13] INFO [2019-12-17 12:33:48,134] jepsen worker 9 - jepsen.util 7929 :invoke :write [3 23] INFO [2019-12-17 12:33:48,141] jepsen worker 30 - jepsen.util 7500 :fail :cas [10 [9 30]] :ValueNotSwapFrom INFO [2019-12-17 12:33:48,141] jepsen worker 30 - jepsen.util 7500 :invoke :write [10 23] INFO [2019-12-17 12:33:48,143] jepsen worker 21 - jepsen.util 9651 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:33:48,143] jepsen worker 21 - jepsen.util 9651 :invoke :cas [7 [43 5]] INFO [2019-12-17 12:33:48,150] jepsen worker 18 - jepsen.util 7758 :ok :read [6 27] INFO [2019-12-17 12:33:48,150] jepsen worker 18 - jepsen.util 7758 :invoke :cas [6 [42 3]] INFO [2019-12-17 12:33:48,150] jepsen worker 3 - jepsen.util 7563 :fail :cas [1 [5 2]] :ValueNotSwapFrom INFO [2019-12-17 12:33:48,151] jepsen worker 3 - jepsen.util 7563 :invoke :cas [1 [17 19]] INFO [2019-12-17 12:33:48,159] jepsen worker 0 - jepsen.util 10710 :info :cas [0 [18 17]] :RequestCanceledException INFO [2019-12-17 12:33:48,160] jepsen worker 0 - jepsen.util 10800 :invoke :cas [0 [26 34]] INFO [2019-12-17 12:33:48,161] jepsen worker 47 - jepsen.util 7877 :ok :read [15 19] INFO [2019-12-17 12:33:48,161] jepsen worker 47 - jepsen.util 7877 :invoke :read [15 nil] INFO [2019-12-17 12:33:48,173] jepsen worker 88 - jepsen.util 7558 :fail :cas [29 [11 13]] :ValueNotSwapFrom INFO [2019-12-17 12:33:48,173] jepsen worker 88 - jepsen.util 7558 :invoke :write [29 13] INFO [2019-12-17 12:33:48,174] jepsen worker 12 - jepsen.util 8112 :ok :read [4 18] INFO [2019-12-17 12:33:48,174] jepsen worker 12 - jepsen.util 8112 :invoke :read [4 nil] INFO [2019-12-17 12:33:48,176] jepsen worker 46 - jepsen.util 7966 :ok :read [15 19] INFO [2019-12-17 12:33:48,176] jepsen worker 46 - jepsen.util 7966 :invoke :read [15 nil] INFO [2019-12-17 12:33:48,187] jepsen worker 4 - jepsen.util 7024 :ok :read [1 11] INFO [2019-12-17 12:33:48,187] jepsen worker 4 - jepsen.util 7024 :invoke :read [1 nil] INFO [2019-12-17 12:33:48,195] jepsen worker 73 - jepsen.util 7273 :fail :cas [24 [20 41]] :ValueNotSwapFrom INFO [2019-12-17 12:33:48,195] jepsen worker 41 - jepsen.util 10751 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:33:48,195] jepsen worker 62 - jepsen.util 10322 :info :cas [20 [24 47]] :RequestCanceledException INFO [2019-12-17 12:33:48,195] jepsen worker 73 - jepsen.util 7273 :invoke :write [24 40] INFO [2019-12-17 12:33:48,195] jepsen worker 41 - jepsen.util 10751 :invoke :write [13 42] INFO [2019-12-17 12:33:48,196] jepsen worker 62 - jepsen.util 10412 :invoke :write [20 23] INFO [2019-12-17 12:33:48,213] jepsen worker 22 - jepsen.util 10012 :info :cas [7 [35 20]] :RequestCanceledException INFO [2019-12-17 12:33:48,213] jepsen worker 58 - jepsen.util 11038 :info :cas [19 [39 26]] :RequestCanceledException INFO [2019-12-17 12:33:48,214] jepsen worker 22 - jepsen.util 10102 :invoke :write [7 44] INFO [2019-12-17 12:33:48,214] jepsen worker 58 - jepsen.util 11128 :invoke :write [19 29] INFO [2019-12-17 12:33:48,222] jepsen worker 71 - jepsen.util 8261 :fail :cas [23 [21 23]] :ValueNotSwapFrom INFO [2019-12-17 12:33:48,223] jepsen worker 71 - jepsen.util 8261 :invoke :cas [23 [33 18]] INFO [2019-12-17 12:33:48,228] jepsen worker 50 - jepsen.util 11030 :info :write [16 20] :RequestCanceledException INFO [2019-12-17 12:33:48,228] jepsen worker 23 - jepsen.util 10283 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:33:48,228] jepsen worker 67 - jepsen.util 10237 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:33:48,228] jepsen worker 68 - jepsen.util 9788 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:33:48,228] jepsen worker 23 - jepsen.util 10283 :invoke :write [7 23] INFO [2019-12-17 12:33:48,228] jepsen worker 67 - jepsen.util 10237 :invoke :write [22 31] INFO [2019-12-17 12:33:48,228] jepsen worker 68 - jepsen.util 9788 :invoke :write [22 0] INFO [2019-12-17 12:33:48,229] jepsen worker 50 - jepsen.util 11120 :invoke :cas [16 [37 27]] INFO [2019-12-17 12:33:48,242] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":43818,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:33:48,242] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":43828,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:33:48,242] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":43818,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:33:48,242] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":43828,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:33:48,242] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":43819,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:33:48,242] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":43831,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:33:48,242] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":43819,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:33:48,242] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":43831,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:33:48,242] jepsen worker 59 - jepsen.util 10049 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:33:48,242] jepsen worker 86 - jepsen.util 10886 :info :write [28 43] :RequestCanceledException INFO [2019-12-17 12:33:48,242] jepsen worker 15 - jepsen.util 9195 :info :cas [5 [22 38]] :RequestCanceledException INFO [2019-12-17 12:33:48,242] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":43820,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:33:48,242] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":43832,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:33:48,242] jepsen worker 59 - jepsen.util 10049 :invoke :read [19 nil] INFO [2019-12-17 12:33:48,243] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":43820,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:33:48,243] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":43832,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:33:48,243] jepsen worker 15 - jepsen.util 9285 :invoke :read [5 nil] INFO [2019-12-17 12:33:48,243] jepsen worker 86 - jepsen.util 10976 :invoke :read [28 nil] INFO [2019-12-17 12:33:48,248] jepsen worker 54 - jepsen.util 8514 :ok :write [18 41] INFO [2019-12-17 12:33:48,248] jepsen worker 54 - jepsen.util 8514 :invoke :cas [18 [34 13]] INFO [2019-12-17 12:33:48,260] jepsen worker 60 - jepsen.util 11220 :info :write [20 23] :RequestCanceledException INFO [2019-12-17 12:33:48,261] jepsen worker 60 - jepsen.util 11310 :invoke :read [20 nil] INFO [2019-12-17 12:33:48,266] jepsen worker 18 - jepsen.util 7758 :fail :cas [6 [42 3]] :ValueNotSwapFrom INFO [2019-12-17 12:33:48,266] jepsen worker 18 - jepsen.util 7758 :invoke :cas [6 [3 28]] INFO [2019-12-17 12:33:48,269] jepsen worker 51 - jepsen.util 7251 :fail :cas [17 [20 27]] :ValueNotSwapFrom INFO [2019-12-17 12:33:48,270] jepsen worker 51 - jepsen.util 7251 :invoke :read [17 nil] INFO [2019-12-17 12:33:48,291] jepsen worker 46 - jepsen.util 7966 :ok :read [15 19] INFO [2019-12-17 12:33:48,291] jepsen worker 46 - jepsen.util 7966 :invoke :cas [15 [49 29]] INFO [2019-12-17 12:33:48,295] jepsen worker 33 - jepsen.util 10023 :info :cas [11 [16 25]] :RequestCanceledException INFO [2019-12-17 12:33:48,295] jepsen worker 25 - jepsen.util 10645 :info :write [8 15] :RequestCanceledException INFO [2019-12-17 12:33:48,296] jepsen worker 33 - jepsen.util 10113 :invoke :read [11 nil] INFO [2019-12-17 12:33:48,296] jepsen worker 25 - jepsen.util 10735 :invoke :write [8 44] INFO [2019-12-17 12:33:48,309] jepsen worker 18 - jepsen.util 7758 :fail :cas [6 [3 28]] :ValueNotSwapFrom INFO [2019-12-17 12:33:48,310] jepsen worker 18 - jepsen.util 7758 :invoke :write [6 32] INFO [2019-12-17 12:33:48,329] jepsen worker 24 - jepsen.util 8934 :info :cas [8 [26 37]] :RequestCanceledException INFO [2019-12-17 12:33:48,330] jepsen worker 24 - jepsen.util 9024 :invoke :cas [8 [9 40]] INFO [2019-12-17 12:33:48,332] jepsen worker 69 - jepsen.util 7809 :ok :read [23 46] INFO [2019-12-17 12:33:48,332] jepsen worker 69 - jepsen.util 7809 :invoke :write [23 8] INFO [2019-12-17 12:33:48,350] jepsen worker 28 - jepsen.util 6058 :ok :write [9 19] INFO [2019-12-17 12:33:48,350] jepsen worker 28 - jepsen.util 6058 :invoke :cas [9 [23 20]] INFO [2019-12-17 12:33:48,351] jepsen worker 38 - jepsen.util 7868 :ok :write [12 42] INFO [2019-12-17 12:33:48,351] jepsen worker 38 - jepsen.util 7868 :invoke :cas [12 [5 10]] INFO [2019-12-17 12:33:48,359] jepsen worker 46 - jepsen.util 7966 :fail :cas [15 [49 29]] :ValueNotSwapFrom INFO [2019-12-17 12:33:48,360] jepsen worker 46 - jepsen.util 7966 :invoke :read [15 nil] INFO [2019-12-17 12:33:48,376] jepsen worker 87 - jepsen.util 7017 :fail :cas [29 [44 44]] :ValueNotSwapFrom INFO [2019-12-17 12:33:48,376] jepsen worker 87 - jepsen.util 7017 :invoke :read [29 nil] INFO [2019-12-17 12:33:48,379] jepsen worker 37 - jepsen.util 7147 :ok :read [12 32] INFO [2019-12-17 12:33:48,379] jepsen worker 37 - jepsen.util 7147 :invoke :read [12 nil] INFO [2019-12-17 12:33:48,390] jepsen worker 28 - jepsen.util 6058 :fail :cas [9 [23 20]] :ValueNotSwapFrom INFO [2019-12-17 12:33:48,391] jepsen worker 28 - jepsen.util 6058 :invoke :cas [9 [7 3]] INFO [2019-12-17 12:33:48,402] jepsen worker 14 - jepsen.util 7844 :ok :write [4 40] INFO [2019-12-17 12:33:48,403] jepsen worker 14 - jepsen.util 7844 :invoke :cas [4 [44 7]] INFO [2019-12-17 12:33:48,411] jepsen worker 82 - jepsen.util 10882 :info :write [27 26] :RequestCanceledException INFO [2019-12-17 12:33:48,413] jepsen worker 82 - jepsen.util 10972 :invoke :write [27 17] INFO [2019-12-17 12:33:48,417] jepsen worker 63 - jepsen.util 6543 :ok :write [21 8] INFO [2019-12-17 12:33:48,417] jepsen worker 63 - jepsen.util 6543 :invoke :write [21 49] INFO [2019-12-17 12:33:48,423] jepsen worker 28 - jepsen.util 6058 :fail :cas [9 [7 3]] :ValueNotSwapFrom INFO [2019-12-17 12:33:48,423] jepsen worker 28 - jepsen.util 6058 :invoke :read [9 nil] INFO [2019-12-17 12:33:48,428] jepsen worker 1 - jepsen.util 10351 :info :write [0 49] :RequestCanceledException INFO [2019-12-17 12:33:48,428] jepsen worker 1 - jepsen.util 10441 :invoke :cas [0 [35 26]] INFO [2019-12-17 12:33:48,442] jepsen worker 6 - jepsen.util 10896 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:33:48,443] jepsen worker 6 - jepsen.util 10896 :invoke :cas [2 [1 21]] INFO [2019-12-17 12:33:48,445] jepsen worker 78 - jepsen.util 7638 :ok :read [26 39] INFO [2019-12-17 12:33:48,446] jepsen worker 78 - jepsen.util 7638 :invoke :write [26 3] INFO [2019-12-17 12:33:48,461] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":43866,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:33:48,461] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":43866,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:33:48,461] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":43870,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:33:48,461] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":43870,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:33:48,470] jepsen worker 28 - jepsen.util 6058 :ok :read [9 19] INFO [2019-12-17 12:33:48,470] jepsen worker 28 - jepsen.util 6058 :invoke :cas [9 [4 36]] INFO [2019-12-17 12:33:48,487] jepsen worker 3 - jepsen.util 7563 :fail :cas [1 [17 19]] :ValueNotSwapFrom INFO [2019-12-17 12:33:48,487] jepsen worker 3 - jepsen.util 7563 :invoke :write [1 16] INFO [2019-12-17 12:33:48,495] jepsen worker 57 - jepsen.util 10317 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:33:48,495] jepsen worker 57 - jepsen.util 10317 :invoke :read [19 nil] INFO [2019-12-17 12:33:48,521] jepsen worker 28 - jepsen.util 6058 :fail :cas [9 [4 36]] :ValueNotSwapFrom INFO [2019-12-17 12:33:48,521] jepsen worker 28 - jepsen.util 6058 :invoke :write [9 28] INFO [2019-12-17 12:33:48,533] jepsen worker 63 - jepsen.util 6543 :ok :write [21 49] INFO [2019-12-17 12:33:48,534] jepsen worker 63 - jepsen.util 6543 :invoke :cas [21 [16 1]] INFO [2019-12-17 12:33:48,561] jepsen worker 4 - jepsen.util 7024 :ok :read [1 11] INFO [2019-12-17 12:33:48,562] jepsen worker 4 - jepsen.util 7024 :invoke :read [1 nil] INFO [2019-12-17 12:33:48,577] jepsen worker 63 - jepsen.util 6543 :fail :cas [21 [16 1]] :ValueNotSwapFrom INFO [2019-12-17 12:33:48,577] jepsen worker 63 - jepsen.util 6543 :invoke :write [21 7] INFO [2019-12-17 12:33:48,599] jepsen worker 71 - jepsen.util 8261 :fail :cas [23 [33 18]] :ValueNotSwapFrom INFO [2019-12-17 12:33:48,599] jepsen worker 71 - jepsen.util 8261 :invoke :read [23 nil] INFO [2019-12-17 12:33:48,612] jepsen worker 84 - jepsen.util 9894 :info :write [28 44] :RequestCanceledException INFO [2019-12-17 12:33:48,613] jepsen worker 84 - jepsen.util 9984 :invoke :read [28 nil] INFO [2019-12-17 12:33:48,615] jepsen worker 28 - jepsen.util 6058 :ok :write [9 28] INFO [2019-12-17 12:33:48,615] jepsen worker 28 - jepsen.util 6058 :invoke :write [9 1] INFO [2019-12-17 12:33:48,628] jepsen worker 17 - jepsen.util 10457 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:33:48,628] jepsen worker 81 - jepsen.util 10161 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:33:48,628] jepsen worker 81 - jepsen.util 10161 :invoke :write [27 7] INFO [2019-12-17 12:33:48,628] jepsen worker 17 - jepsen.util 10457 :invoke :write [5 4] INFO [2019-12-17 12:33:48,668] jepsen worker 80 - jepsen.util 7550 :ok :write [26 33] INFO [2019-12-17 12:33:48,668] jepsen worker 80 - jepsen.util 7550 :invoke :write [26 15] INFO [2019-12-17 12:33:48,680] jepsen worker 30 - jepsen.util 7500 :ok :write [10 23] INFO [2019-12-17 12:33:48,680] jepsen worker 30 - jepsen.util 7500 :invoke :write [10 49] INFO [2019-12-17 12:33:48,680] jepsen worker 20 - jepsen.util 7310 :ok :write [6 47] INFO [2019-12-17 12:33:48,680] jepsen worker 20 - jepsen.util 7310 :invoke :write [6 39] INFO [2019-12-17 12:33:48,694] jepsen worker 48 - jepsen.util 9318 :info :write [16 15] :RequestCanceledException INFO [2019-12-17 12:33:48,695] jepsen worker 48 - jepsen.util 9408 :invoke :read [16 nil] INFO [2019-12-17 12:33:48,698] jepsen worker 63 - jepsen.util 6543 :ok :write [21 7] INFO [2019-12-17 12:33:48,698] jepsen worker 63 - jepsen.util 6543 :invoke :read [21 nil] INFO [2019-12-17 12:33:48,703] jepsen worker 88 - jepsen.util 7558 :ok :write [29 13] INFO [2019-12-17 12:33:48,703] jepsen worker 88 - jepsen.util 7558 :invoke :read [29 nil] INFO [2019-12-17 12:33:48,703] jepsen worker 54 - jepsen.util 8514 :fail :cas [18 [34 13]] :ValueNotSwapFrom INFO [2019-12-17 12:33:48,703] jepsen worker 54 - jepsen.util 8514 :invoke :read [18 nil] INFO [2019-12-17 12:33:48,723] jepsen worker 9 - jepsen.util 7929 :ok :write [3 23] INFO [2019-12-17 12:33:48,723] jepsen worker 9 - jepsen.util 7929 :invoke :read [3 nil] INFO [2019-12-17 12:33:48,728] jepsen worker 76 - jepsen.util 10246 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:33:48,728] jepsen worker 76 - jepsen.util 10246 :invoke :write [25 26] INFO [2019-12-17 12:33:48,736] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":43885,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:33:48,737] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":43885,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:33:48,737] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":43886,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:33:48,737] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":43886,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:33:48,737] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":43887,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:33:48,737] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":43887,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:33:48,741] jepsen worker 51 - jepsen.util 7251 :ok :read [17 36] INFO [2019-12-17 12:33:48,741] jepsen worker 51 - jepsen.util 7251 :invoke :cas [17 [7 29]] INFO [2019-12-17 12:33:48,743] jepsen worker 63 - jepsen.util 6543 :ok :read [21 7] INFO [2019-12-17 12:33:48,743] jepsen worker 63 - jepsen.util 6543 :invoke :read [21 nil] INFO [2019-12-17 12:33:48,752] jepsen worker 52 - jepsen.util 7882 :ok :cas [17 [36 27]] INFO [2019-12-17 12:33:48,752] jepsen worker 52 - jepsen.util 7882 :invoke :read [17 nil] INFO [2019-12-17 12:33:48,755] jepsen worker 28 - jepsen.util 6058 :ok :write [9 1] INFO [2019-12-17 12:33:48,755] jepsen worker 28 - jepsen.util 6058 :invoke :cas [9 [7 9]] INFO [2019-12-17 12:33:48,795] jepsen worker 39 - jepsen.util 9939 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:33:48,795] jepsen worker 39 - jepsen.util 9939 :invoke :read [13 nil] INFO [2019-12-17 12:33:48,796] jepsen worker 79 - jepsen.util 7729 :ok :write [26 30] INFO [2019-12-17 12:33:48,797] jepsen worker 79 - jepsen.util 7729 :invoke :cas [26 [36 32]] INFO [2019-12-17 12:33:48,800] jepsen worker 28 - jepsen.util 6058 :fail :cas [9 [7 9]] :ValueNotSwapFrom INFO [2019-12-17 12:33:48,800] jepsen worker 28 - jepsen.util 6058 :invoke :write [9 7] INFO [2019-12-17 12:33:48,801] jepsen worker 4 - jepsen.util 7024 :ok :read [1 11] INFO [2019-12-17 12:33:48,802] jepsen worker 4 - jepsen.util 7024 :invoke :cas [1 [25 12]] INFO [2019-12-17 12:33:48,802] jepsen worker 63 - jepsen.util 6543 :ok :read [21 7] INFO [2019-12-17 12:33:48,802] jepsen worker 63 - jepsen.util 6543 :invoke :cas [21 [18 14]] INFO [2019-12-17 12:33:48,824] jepsen worker 63 - jepsen.util 6543 :fail :cas [21 [18 14]] :ValueNotSwapFrom INFO [2019-12-17 12:33:48,824] jepsen worker 63 - jepsen.util 6543 :invoke :cas [21 [45 2]] INFO [2019-12-17 12:33:48,825] jepsen worker 9 - jepsen.util 7929 :ok :read [3 23] INFO [2019-12-17 12:33:48,825] jepsen worker 9 - jepsen.util 7929 :invoke :read [3 nil] INFO [2019-12-17 12:33:48,842] jepsen worker 7 - jepsen.util 10447 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:33:48,842] jepsen worker 7 - jepsen.util 10447 :invoke :read [2 nil] INFO [2019-12-17 12:33:48,853] jepsen worker 9 - jepsen.util 7929 :ok :read [3 23] INFO [2019-12-17 12:33:48,853] jepsen worker 9 - jepsen.util 7929 :invoke :read [3 nil] INFO [2019-12-17 12:33:48,856] jepsen worker 14 - jepsen.util 7844 :fail :cas [4 [44 7]] :ValueNotSwapFrom INFO [2019-12-17 12:33:48,856] jepsen worker 14 - jepsen.util 7844 :invoke :read [4 nil] INFO [2019-12-17 12:33:48,860] jepsen worker 40 - jepsen.util 10660 :info :write [13 0] :RequestCanceledException INFO [2019-12-17 12:33:48,861] jepsen worker 40 - jepsen.util 10750 :invoke :cas [13 [14 30]] INFO [2019-12-17 12:33:48,884] jepsen worker 36 - jepsen.util 7866 :ok :write [12 47] INFO [2019-12-17 12:33:48,885] jepsen worker 36 - jepsen.util 7866 :invoke :cas [12 [23 15]] INFO [2019-12-17 12:33:48,889] jepsen worker 71 - jepsen.util 8261 :ok :read [23 46] INFO [2019-12-17 12:33:48,889] jepsen worker 9 - jepsen.util 7929 :ok :read [3 23] INFO [2019-12-17 12:33:48,889] jepsen worker 9 - jepsen.util 7929 :invoke :read [3 nil] INFO [2019-12-17 12:33:48,889] jepsen worker 71 - jepsen.util 8261 :invoke :cas [23 [15 10]] INFO [2019-12-17 12:33:48,890] jepsen worker 16 - jepsen.util 11536 :info :cas [5 [8 32]] :RequestCanceledException INFO [2019-12-17 12:33:48,890] jepsen worker 10 - jepsen.util 7480 :info :write [3 6] :RequestCanceledException INFO [2019-12-17 12:33:48,891] jepsen worker 10 - jepsen.util 7570 :invoke :read [3 nil] INFO [2019-12-17 12:33:48,891] jepsen worker 16 - jepsen.util 11626 :invoke :read [5 nil] INFO [2019-12-17 12:33:48,895] jepsen worker 89 - jepsen.util 7379 :ok :read [29 13] INFO [2019-12-17 12:33:48,895] jepsen worker 89 - jepsen.util 7379 :invoke :write [29 16] INFO [2019-12-17 12:33:48,897] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":43916,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:33:48,897] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":43916,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:33:48,897] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":43919,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:33:48,897] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":43919,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:33:48,913] jepsen worker 44 - jepsen.util 10934 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:33:48,913] jepsen worker 61 - jepsen.util 10951 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:33:48,913] jepsen worker 63 - jepsen.util 6543 :fail :cas [21 [45 2]] :ValueNotSwapFrom INFO [2019-12-17 12:33:48,914] jepsen worker 61 - jepsen.util 10951 :invoke :write [20 24] INFO [2019-12-17 12:33:48,914] jepsen worker 44 - jepsen.util 10934 :invoke :write [14 17] INFO [2019-12-17 12:33:48,914] jepsen worker 63 - jepsen.util 6543 :invoke :read [21 nil] INFO [2019-12-17 12:33:48,920] jepsen worker 9 - jepsen.util 7929 :ok :read [3 23] INFO [2019-12-17 12:33:48,920] jepsen worker 9 - jepsen.util 7929 :invoke :read [3 nil] INFO [2019-12-17 12:33:48,928] jepsen worker 85 - jepsen.util 11065 :info :cas [28 [49 47]] :RequestCanceledException INFO [2019-12-17 12:33:48,929] jepsen worker 49 - jepsen.util 9769 :info :write [16 12] :RequestCanceledException INFO [2019-12-17 12:33:48,929] jepsen worker 85 - jepsen.util 11155 :invoke :write [28 21] INFO [2019-12-17 12:33:48,930] jepsen worker 49 - jepsen.util 9859 :invoke :write [16 47] INFO [2019-12-17 12:33:48,937] jepsen worker 63 - jepsen.util 6543 :ok :read [21 7] INFO [2019-12-17 12:33:48,938] jepsen worker 63 - jepsen.util 6543 :invoke :write [21 27] INFO [2019-12-17 12:33:48,938] jepsen worker 9 - jepsen.util 7929 :ok :read [3 23] INFO [2019-12-17 12:33:48,938] jepsen worker 9 - jepsen.util 7929 :invoke :read [3 nil] INFO [2019-12-17 12:33:48,945] jepsen worker 28 - jepsen.util 6058 :ok :write [9 7] INFO [2019-12-17 12:33:48,945] jepsen worker 28 - jepsen.util 6058 :invoke :write [9 29] INFO [2019-12-17 12:33:48,949] jepsen worker 70 - jepsen.util 7900 :fail :cas [23 [19 13]] :ValueNotSwapFrom INFO [2019-12-17 12:33:48,950] jepsen worker 70 - jepsen.util 7900 :invoke :cas [23 [14 47]] INFO [2019-12-17 12:33:48,956] jepsen worker 88 - jepsen.util 7558 :ok :read [29 13] INFO [2019-12-17 12:33:48,956] jepsen worker 88 - jepsen.util 7558 :invoke :cas [29 [13 31]] INFO [2019-12-17 12:33:48,959] jepsen worker 35 - jepsen.util 10025 :info :cas [11 [33 1]] :RequestCanceledException INFO [2019-12-17 12:33:48,959] jepsen worker 8 - jepsen.util 10178 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:33:48,959] jepsen worker 8 - jepsen.util 10178 :invoke :cas [2 [40 21]] INFO [2019-12-17 12:33:48,960] jepsen worker 55 - jepsen.util 8425 :ok :write [18 22] INFO [2019-12-17 12:33:48,960] jepsen worker 55 - jepsen.util 8425 :invoke :cas [18 [17 36]] INFO [2019-12-17 12:33:48,960] jepsen worker 35 - jepsen.util 10115 :invoke :cas [11 [33 38]] INFO [2019-12-17 12:33:48,970] jepsen worker 9 - jepsen.util 7929 :ok :read [3 23] INFO [2019-12-17 12:33:48,970] jepsen worker 9 - jepsen.util 7929 :invoke :write [3 31] INFO [2019-12-17 12:33:48,972] jepsen worker 13 - jepsen.util 7213 :ok :read [4 40] INFO [2019-12-17 12:33:48,972] jepsen worker 13 - jepsen.util 7213 :invoke :write [4 46] INFO [2019-12-17 12:33:48,994] jepsen worker 51 - jepsen.util 7251 :fail :cas [17 [7 29]] :ValueNotSwapFrom INFO [2019-12-17 12:33:48,995] jepsen worker 51 - jepsen.util 7251 :invoke :cas [17 [14 25]] INFO [2019-12-17 12:33:48,997] jepsen worker 87 - jepsen.util 7017 :ok :read [29 13] INFO [2019-12-17 12:33:48,997] jepsen worker 87 - jepsen.util 7017 :invoke :cas [29 [27 30]] INFO [2019-12-17 12:33:48,999] jepsen worker 14 - jepsen.util 7844 :ok :read [4 40] INFO [2019-12-17 12:33:49,000] jepsen worker 14 - jepsen.util 7844 :invoke :write [4 18] INFO [2019-12-17 12:33:49,009] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":43945,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:33:49,009] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":43945,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:33:49,009] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":43949,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:33:49,009] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":43949,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:33:49,026] jepsen worker 5 - jepsen.util 7205 :ok :read [1 11] INFO [2019-12-17 12:33:49,026] jepsen worker 5 - jepsen.util 7205 :invoke :read [1 nil] INFO [2019-12-17 12:33:49,042] jepsen worker 45 - jepsen.util 7515 :info :write [15 24] :RequestCanceledException INFO [2019-12-17 12:33:49,042] jepsen worker 64 - jepsen.util 5914 :info :cas [21 [22 19]] :RequestCanceledException INFO [2019-12-17 12:33:49,044] jepsen worker 64 - jepsen.util 6004 :invoke :read [21 nil] INFO [2019-12-17 12:33:49,044] jepsen worker 45 - jepsen.util 7605 :invoke :write [15 32] INFO [2019-12-17 12:33:49,066] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:33:49,066] jepsen nemesis - couchbase.util Rebalance check count: 336 INFO [2019-12-17 12:33:49,066] jepsen nemesis - couchbase.util Rebalance stuck count: 317 INFO [2019-12-17 12:33:49,075] jepsen worker 53 - jepsen.util 7073 :ok :write [17 14] INFO [2019-12-17 12:33:49,075] jepsen worker 53 - jepsen.util 7073 :invoke :cas [17 [44 34]] INFO [2019-12-17 12:33:49,079] jepsen worker 51 - jepsen.util 7251 :fail :cas [17 [14 25]] :ValueNotSwapFrom INFO [2019-12-17 12:33:49,079] jepsen worker 51 - jepsen.util 7251 :invoke :write [17 46] INFO [2019-12-17 12:33:49,083] jepsen worker 69 - jepsen.util 7809 :ok :write [23 8] INFO [2019-12-17 12:33:49,083] jepsen worker 69 - jepsen.util 7809 :invoke :write [23 18] INFO [2019-12-17 12:33:49,083] jepsen worker 12 - jepsen.util 8112 :ok :read [4 40] INFO [2019-12-17 12:33:49,083] jepsen worker 12 - jepsen.util 8112 :invoke :write [4 28] INFO [2019-12-17 12:33:49,088] jepsen worker 80 - jepsen.util 7550 :ok :write [26 15] INFO [2019-12-17 12:33:49,088] jepsen worker 80 - jepsen.util 7550 :invoke :read [26 nil] INFO [2019-12-17 12:33:49,101] jepsen worker 28 - jepsen.util 6058 :ok :write [9 29] INFO [2019-12-17 12:33:49,101] jepsen worker 28 - jepsen.util 6058 :invoke :read [9 nil] INFO [2019-12-17 12:33:49,119] jepsen worker 63 - jepsen.util 6543 :ok :write [21 27] INFO [2019-12-17 12:33:49,119] jepsen worker 63 - jepsen.util 6543 :invoke :cas [21 [18 4]] INFO [2019-12-17 12:33:49,119] jepsen worker 64 - jepsen.util 6004 :ok :read [21 7] INFO [2019-12-17 12:33:49,120] jepsen worker 64 - jepsen.util 6004 :invoke :read [21 nil] INFO [2019-12-17 12:33:49,128] jepsen worker 72 - jepsen.util 7272 :info :cas [24 [14 0]] :RequestCanceledException INFO [2019-12-17 12:33:49,129] jepsen worker 37 - jepsen.util 7147 :ok :read [12 47] INFO [2019-12-17 12:33:49,129] jepsen worker 37 - jepsen.util 7147 :invoke :cas [12 [7 10]] INFO [2019-12-17 12:33:49,130] jepsen worker 72 - jepsen.util 7362 :invoke :write [24 20] INFO [2019-12-17 12:33:49,142] jepsen worker 56 - jepsen.util 8516 :info :write [18 19] :RequestCanceledException INFO [2019-12-17 12:33:49,144] jepsen worker 56 - jepsen.util 8606 :invoke :write [18 36] INFO [2019-12-17 12:33:49,149] jepsen worker 70 - jepsen.util 7900 :fail :cas [23 [14 47]] :ValueNotSwapFrom INFO [2019-12-17 12:33:49,149] jepsen worker 70 - jepsen.util 7900 :invoke :write [23 29] INFO [2019-12-17 12:33:49,153] jepsen worker 54 - jepsen.util 8514 :ok :read [18 22] INFO [2019-12-17 12:33:49,153] jepsen worker 54 - jepsen.util 8514 :invoke :cas [18 [27 11]] INFO [2019-12-17 12:33:49,156] jepsen worker 55 - jepsen.util 8425 :fail :cas [18 [17 36]] :ValueNotSwapFrom INFO [2019-12-17 12:33:49,156] jepsen worker 55 - jepsen.util 8425 :invoke :write [18 3] INFO [2019-12-17 12:33:49,177] jepsen worker 5 - jepsen.util 7205 :ok :read [1 11] INFO [2019-12-17 12:33:49,178] jepsen worker 5 - jepsen.util 7205 :invoke :cas [1 [30 16]] INFO [2019-12-17 12:33:49,187] jepsen worker 63 - jepsen.util 6543 :fail :cas [21 [18 4]] :ValueNotSwapFrom INFO [2019-12-17 12:33:49,187] jepsen worker 63 - jepsen.util 6543 :invoke :cas [21 [20 6]] INFO [2019-12-17 12:33:49,189] jepsen worker 28 - jepsen.util 6058 :ok :read [9 29] INFO [2019-12-17 12:33:49,189] jepsen worker 28 - jepsen.util 6058 :invoke :write [9 38] INFO [2019-12-17 12:33:49,228] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":43981,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:33:49,228] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":43974,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:33:49,228] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":43981,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:33:49,228] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":43974,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:33:49,228] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":43975,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:33:49,228] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":43975,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:33:49,233] jepsen worker 64 - jepsen.util 6004 :ok :read [21 27] INFO [2019-12-17 12:33:49,234] jepsen worker 64 - jepsen.util 6004 :invoke :read [21 nil] INFO [2019-12-17 12:33:49,250] jepsen worker 36 - jepsen.util 7866 :fail :cas [12 [23 15]] :ValueNotSwapFrom INFO [2019-12-17 12:33:49,251] jepsen worker 36 - jepsen.util 7866 :invoke :read [12 nil] INFO [2019-12-17 12:33:49,265] jepsen worker 38 - jepsen.util 7868 :fail :cas [12 [5 10]] :ValueNotSwapFrom INFO [2019-12-17 12:33:49,265] jepsen worker 38 - jepsen.util 7868 :invoke :read [12 nil] INFO [2019-12-17 12:33:49,268] jepsen worker 30 - jepsen.util 7500 :ok :write [10 49] INFO [2019-12-17 12:33:49,268] jepsen worker 20 - jepsen.util 7310 :ok :write [6 39] INFO [2019-12-17 12:33:49,268] jepsen worker 30 - jepsen.util 7500 :invoke :cas [10 [21 33]] INFO [2019-12-17 12:33:49,268] jepsen worker 20 - jepsen.util 7310 :invoke :write [6 21] INFO [2019-12-17 12:33:49,283] jepsen worker 12 - jepsen.util 8112 :ok :write [4 28] INFO [2019-12-17 12:33:49,283] jepsen worker 12 - jepsen.util 8112 :invoke :read [4 nil] INFO [2019-12-17 12:33:49,291] jepsen worker 79 - jepsen.util 7729 :fail :cas [26 [36 32]] :ValueNotSwapFrom INFO [2019-12-17 12:33:49,292] jepsen worker 79 - jepsen.util 7729 :invoke :cas [26 [1 11]] INFO [2019-12-17 12:33:49,310] jepsen worker 2 - jepsen.util 9992 :info :cas [0 [45 26]] :RequestCanceledException INFO [2019-12-17 12:33:49,311] jepsen worker 19 - jepsen.util 7759 :fail :read [6 nil] :RequestTimeoutException INFO [2019-12-17 12:33:49,311] jepsen worker 19 - jepsen.util 7759 :invoke :cas [6 [34 24]] INFO [2019-12-17 12:33:49,313] jepsen worker 10 - jepsen.util 7570 :ok :read [3 23] INFO [2019-12-17 12:33:49,313] jepsen worker 10 - jepsen.util 7570 :invoke :read [3 nil] INFO [2019-12-17 12:33:49,314] jepsen worker 2 - jepsen.util 10082 :invoke :read [0 nil] INFO [2019-12-17 12:33:49,316] jepsen worker 28 - jepsen.util 6058 :ok :write [9 38] INFO [2019-12-17 12:33:49,316] jepsen worker 28 - jepsen.util 6058 :invoke :write [9 9] INFO [2019-12-17 12:33:49,339] jepsen worker 64 - jepsen.util 6004 :ok :read [21 27] INFO [2019-12-17 12:33:49,339] jepsen worker 64 - jepsen.util 6004 :invoke :cas [21 [20 8]] INFO [2019-12-17 12:33:49,346] jepsen worker 87 - jepsen.util 7017 :fail :cas [29 [27 30]] :ValueNotSwapFrom INFO [2019-12-17 12:33:49,346] jepsen worker 87 - jepsen.util 7017 :invoke :read [29 nil] INFO [2019-12-17 12:33:49,357] jepsen worker 9 - jepsen.util 7929 :ok :write [3 31] INFO [2019-12-17 12:33:49,357] jepsen worker 9 - jepsen.util 7929 :invoke :write [3 9] INFO [2019-12-17 12:33:49,359] jepsen worker 31 - jepsen.util 6781 :info :write [10 8] :RequestCanceledException INFO [2019-12-17 12:33:49,359] jepsen worker 11 - jepsen.util 7661 :info :write [3 21] :RequestCanceledException INFO [2019-12-17 12:33:49,360] jepsen worker 31 - jepsen.util 6871 :invoke :read [10 nil] INFO [2019-12-17 12:33:49,360] jepsen worker 11 - jepsen.util 7751 :invoke :write [3 6] INFO [2019-12-17 12:33:49,361] jepsen worker 80 - jepsen.util 7550 :ok :read [26 15] INFO [2019-12-17 12:33:49,361] jepsen worker 80 - jepsen.util 7550 :invoke :read [26 nil] INFO [2019-12-17 12:33:49,376] jepsen worker 19 - jepsen.util 7759 :fail :cas [6 [34 24]] :ValueNotSwapFrom INFO [2019-12-17 12:33:49,376] jepsen worker 19 - jepsen.util 7759 :invoke :cas [6 [0 36]] INFO [2019-12-17 12:33:49,385] jepsen worker 5 - jepsen.util 7205 :fail :cas [1 [30 16]] :ValueNotSwapFrom INFO [2019-12-17 12:33:49,385] jepsen worker 5 - jepsen.util 7205 :invoke :read [1 nil] INFO [2019-12-17 12:33:49,398] jepsen worker 54 - jepsen.util 8514 :fail :cas [18 [27 11]] :ValueNotSwapFrom INFO [2019-12-17 12:33:49,398] jepsen worker 54 - jepsen.util 8514 :invoke :cas [18 [21 37]] INFO [2019-12-17 12:33:49,404] jepsen worker 31 - jepsen.util 6871 :ok :read [10 49] INFO [2019-12-17 12:33:49,404] jepsen worker 31 - jepsen.util 6871 :invoke :write [10 35] INFO [2019-12-17 12:33:49,412] jepsen worker 28 - jepsen.util 6058 :ok :write [9 9] INFO [2019-12-17 12:33:49,413] jepsen worker 28 - jepsen.util 6058 :invoke :read [9 nil] INFO [2019-12-17 12:33:49,422] jepsen worker 12 - jepsen.util 8112 :ok :read [4 28] INFO [2019-12-17 12:33:49,422] jepsen worker 12 - jepsen.util 8112 :invoke :cas [4 [2 42]] INFO [2019-12-17 12:33:49,426] jepsen worker 32 - jepsen.util 7502 :info :cas [10 [35 23]] :RequestCanceledException INFO [2019-12-17 12:33:49,427] jepsen worker 32 - jepsen.util 7592 :invoke :write [10 16] INFO [2019-12-17 12:33:49,447] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44008,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:33:49,448] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44008,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:33:49,448] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44012,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:33:49,448] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44012,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:33:49,451] jepsen worker 64 - jepsen.util 6004 :fail :cas [21 [20 8]] :ValueNotSwapFrom INFO [2019-12-17 12:33:49,451] jepsen worker 64 - jepsen.util 6004 :invoke :cas [21 [2 2]] INFO [2019-12-17 12:33:49,465] jepsen worker 5 - jepsen.util 7205 :ok :read [1 11] INFO [2019-12-17 12:33:49,466] jepsen worker 5 - jepsen.util 7205 :invoke :read [1 nil] INFO [2019-12-17 12:33:49,495] jepsen worker 27 - jepsen.util 7587 :info :cas [9 [1 25]] :RequestCanceledException INFO [2019-12-17 12:33:49,498] jepsen worker 27 - jepsen.util 7677 :invoke :read [9 nil] INFO [2019-12-17 12:33:49,504] jepsen worker 72 - jepsen.util 7362 :ok :write [24 20] INFO [2019-12-17 12:33:49,504] jepsen worker 72 - jepsen.util 7362 :invoke :read [24 nil] INFO [2019-12-17 12:33:49,505] jepsen worker 12 - jepsen.util 8112 :fail :cas [4 [2 42]] :ValueNotSwapFrom INFO [2019-12-17 12:33:49,505] jepsen worker 12 - jepsen.util 8112 :invoke :read [4 nil] INFO [2019-12-17 12:33:49,529] jepsen worker 74 - jepsen.util 7724 :info :write [24 16] :RequestCanceledException INFO [2019-12-17 12:33:49,530] jepsen worker 74 - jepsen.util 7814 :invoke :cas [24 [47 11]] INFO [2019-12-17 12:33:49,539] jepsen worker 72 - jepsen.util 7362 :ok :read [24 20] INFO [2019-12-17 12:33:49,539] jepsen worker 38 - jepsen.util 7868 :ok :read [12 47] INFO [2019-12-17 12:33:49,540] jepsen worker 72 - jepsen.util 7362 :invoke :read [24 nil] INFO [2019-12-17 12:33:49,540] jepsen worker 38 - jepsen.util 7868 :invoke :write [12 45] INFO [2019-12-17 12:33:49,546] jepsen worker 27 - jepsen.util 7677 :ok :read [9 9] INFO [2019-12-17 12:33:49,546] jepsen worker 27 - jepsen.util 7677 :invoke :write [9 26] INFO [2019-12-17 12:33:49,572] jepsen worker 79 - jepsen.util 7729 :fail :cas [26 [1 11]] :ValueNotSwapFrom INFO [2019-12-17 12:33:49,572] jepsen worker 79 - jepsen.util 7729 :invoke :read [26 nil] INFO [2019-12-17 12:33:49,581] jepsen worker 52 - jepsen.util 7882 :ok :read [17 14] INFO [2019-12-17 12:33:49,581] jepsen worker 52 - jepsen.util 7882 :invoke :cas [17 [31 17]] INFO [2019-12-17 12:33:49,595] jepsen worker 29 - jepsen.util 6059 :info :write [9 40] :RequestCanceledException INFO [2019-12-17 12:33:49,595] jepsen worker 65 - jepsen.util 6275 :info :cas [21 [14 36]] :RequestCanceledException INFO [2019-12-17 12:33:49,596] jepsen worker 65 - jepsen.util 6365 :invoke :read [21 nil] INFO [2019-12-17 12:33:49,596] jepsen worker 29 - jepsen.util 6149 :invoke :write [9 5] INFO [2019-12-17 12:33:49,599] jepsen worker 72 - jepsen.util 7362 :ok :read [24 20] INFO [2019-12-17 12:33:49,599] jepsen worker 72 - jepsen.util 7362 :invoke :read [24 nil] INFO [2019-12-17 12:33:49,618] jepsen worker 53 - jepsen.util 7073 :fail :cas [17 [44 34]] :ValueNotSwapFrom INFO [2019-12-17 12:33:49,619] jepsen worker 53 - jepsen.util 7073 :invoke :write [17 44] INFO [2019-12-17 12:33:49,622] jepsen worker 36 - jepsen.util 7866 :ok :read [12 47] INFO [2019-12-17 12:33:49,622] jepsen worker 36 - jepsen.util 7866 :invoke :cas [12 [10 27]] INFO [2019-12-17 12:33:49,631] jepsen worker 20 - jepsen.util 7310 :ok :write [6 21] INFO [2019-12-17 12:33:49,631] jepsen worker 20 - jepsen.util 7310 :invoke :read [6 nil] INFO [2019-12-17 12:33:49,632] jepsen worker 45 - jepsen.util 7605 :ok :write [15 32] INFO [2019-12-17 12:33:49,632] jepsen worker 45 - jepsen.util 7605 :invoke :cas [15 [44 21]] INFO [2019-12-17 12:33:49,644] jepsen worker 4 - jepsen.util 7024 :fail :cas [1 [25 12]] :ValueNotSwapFrom INFO [2019-12-17 12:33:49,645] jepsen worker 4 - jepsen.util 7024 :invoke :read [1 nil] INFO [2019-12-17 12:33:49,665] jepsen worker 65 - jepsen.util 6365 :ok :read [21 27] INFO [2019-12-17 12:33:49,665] jepsen worker 65 - jepsen.util 6365 :invoke :read [21 nil] INFO [2019-12-17 12:33:49,670] jepsen worker 71 - jepsen.util 8261 :fail :cas [23 [15 10]] :ValueNotSwapFrom INFO [2019-12-17 12:33:49,670] jepsen worker 71 - jepsen.util 8261 :invoke :cas [23 [23 40]] INFO [2019-12-17 12:33:49,678] jepsen worker 87 - jepsen.util 7017 :ok :read [29 13] INFO [2019-12-17 12:33:49,678] jepsen worker 87 - jepsen.util 7017 :invoke :read [29 nil] INFO [2019-12-17 12:33:49,679] jepsen worker 27 - jepsen.util 7677 :ok :write [9 26] INFO [2019-12-17 12:33:49,679] jepsen worker 27 - jepsen.util 7677 :invoke :write [9 42] INFO [2019-12-17 12:33:49,693] jepsen worker 80 - jepsen.util 7550 :ok :read [26 15] INFO [2019-12-17 12:33:49,694] jepsen worker 80 - jepsen.util 7550 :invoke :read [26 nil] INFO [2019-12-17 12:33:49,713] jepsen worker 74 - jepsen.util 7814 :fail :cas [24 [47 11]] :ValueNotSwapFrom INFO [2019-12-17 12:33:49,713] jepsen worker 74 - jepsen.util 7814 :invoke :write [24 38] INFO [2019-12-17 12:33:49,714] jepsen worker 10 - jepsen.util 7570 :ok :read [3 31] INFO [2019-12-17 12:33:49,714] jepsen worker 10 - jepsen.util 7570 :invoke :write [3 37] INFO [2019-12-17 12:33:49,720] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44034,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:33:49,721] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44034,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:33:49,721] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44037,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:33:49,721] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44037,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:33:49,721] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44038,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:33:49,722] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44038,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:33:49,759] jepsen worker 20 - jepsen.util 7310 :ok :read [6 21] INFO [2019-12-17 12:33:49,760] jepsen worker 20 - jepsen.util 7310 :invoke :write [6 4] INFO [2019-12-17 12:33:49,763] jepsen worker 5 - jepsen.util 7205 :ok :read [1 11] INFO [2019-12-17 12:33:49,764] jepsen worker 5 - jepsen.util 7205 :invoke :read [1 nil] INFO [2019-12-17 12:33:49,782] jepsen worker 65 - jepsen.util 6365 :ok :read [21 27] INFO [2019-12-17 12:33:49,782] jepsen worker 65 - jepsen.util 6365 :invoke :cas [21 [14 37]] INFO [2019-12-17 12:33:49,785] jepsen worker 54 - jepsen.util 8514 :fail :cas [18 [21 37]] :ValueNotSwapFrom INFO [2019-12-17 12:33:49,785] jepsen worker 54 - jepsen.util 8514 :invoke :read [18 nil] INFO [2019-12-17 12:33:49,787] jepsen worker 72 - jepsen.util 7362 :ok :read [24 20] INFO [2019-12-17 12:33:49,787] jepsen worker 72 - jepsen.util 7362 :invoke :write [24 18] INFO [2019-12-17 12:33:49,825] jepsen worker 36 - jepsen.util 7866 :fail :cas [12 [10 27]] :ValueNotSwapFrom INFO [2019-12-17 12:33:49,825] jepsen worker 36 - jepsen.util 7866 :invoke :read [12 nil] INFO [2019-12-17 12:33:49,845] jepsen worker 65 - jepsen.util 6365 :fail :cas [21 [14 37]] :ValueNotSwapFrom INFO [2019-12-17 12:33:49,845] jepsen worker 65 - jepsen.util 6365 :invoke :write [21 31] INFO [2019-12-17 12:33:49,869] jepsen worker 27 - jepsen.util 7677 :ok :write [9 42] INFO [2019-12-17 12:33:49,870] jepsen worker 27 - jepsen.util 7677 :invoke :read [9 nil] INFO [2019-12-17 12:33:49,882] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44055,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:33:49,882] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44055,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:33:49,882] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44058,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:33:49,882] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44058,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:33:49,882] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44059,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:33:49,883] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44059,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:33:49,921] jepsen worker 65 - jepsen.util 6365 :ok :write [21 31] INFO [2019-12-17 12:33:49,921] jepsen worker 65 - jepsen.util 6365 :invoke :cas [21 [48 29]] INFO [2019-12-17 12:33:49,944] jepsen worker 87 - jepsen.util 7017 :ok :read [29 13] INFO [2019-12-17 12:33:49,944] jepsen worker 87 - jepsen.util 7017 :invoke :cas [29 [46 46]] INFO [2019-12-17 12:33:49,973] jepsen worker 37 - jepsen.util 7147 :fail :cas [12 [7 10]] :ValueNotSwapFrom INFO [2019-12-17 12:33:49,973] jepsen worker 37 - jepsen.util 7147 :invoke :cas [12 [46 8]] INFO [2019-12-17 12:33:49,975] jepsen worker 65 - jepsen.util 6365 :fail :cas [21 [48 29]] :ValueNotSwapFrom INFO [2019-12-17 12:33:49,975] jepsen worker 65 - jepsen.util 6365 :invoke :read [21 nil] INFO [2019-12-17 12:33:50,003] jepsen worker 65 - jepsen.util 6365 :ok :read [21 31] INFO [2019-12-17 12:33:50,003] jepsen worker 65 - jepsen.util 6365 :invoke :cas [21 [3 21]] INFO [2019-12-17 12:33:50,003] jepsen worker 80 - jepsen.util 7550 :ok :read [26 15] INFO [2019-12-17 12:33:50,004] jepsen worker 80 - jepsen.util 7550 :invoke :read [26 nil] INFO [2019-12-17 12:33:50,011] jepsen worker 9 - jepsen.util 7929 :ok :write [3 9] INFO [2019-12-17 12:33:50,011] jepsen worker 9 - jepsen.util 7929 :invoke :write [3 37] INFO [2019-12-17 12:33:50,045] jepsen worker 65 - jepsen.util 6365 :fail :cas [21 [3 21]] :ValueNotSwapFrom INFO [2019-12-17 12:33:50,047] jepsen worker 65 - jepsen.util 6365 :invoke :cas [21 [49 1]] INFO [2019-12-17 12:33:50,051] jepsen worker 29 - jepsen.util 6149 :ok :write [9 5] INFO [2019-12-17 12:33:50,051] jepsen worker 29 - jepsen.util 6149 :invoke :write [9 44] INFO [2019-12-17 12:33:50,059] jepsen worker 34 - jepsen.util 9844 :info :write [11 32] :RequestCanceledException INFO [2019-12-17 12:33:50,061] jepsen worker 34 - jepsen.util 9934 :invoke :write [11 27] INFO [2019-12-17 12:33:50,081] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:33:50,081] jepsen worker 12 - jepsen.util 8112 :ok :read [4 28] INFO [2019-12-17 12:33:50,081] jepsen nemesis - couchbase.util Rebalance check count: 337 INFO [2019-12-17 12:33:50,081] jepsen nemesis - couchbase.util Rebalance stuck count: 318 INFO [2019-12-17 12:33:50,081] jepsen worker 65 - jepsen.util 6365 :fail :cas [21 [49 1]] :ValueNotSwapFrom INFO [2019-12-17 12:33:50,081] jepsen worker 12 - jepsen.util 8112 :invoke :write [4 19] INFO [2019-12-17 12:33:50,082] jepsen worker 65 - jepsen.util 6365 :invoke :read [21 nil] INFO [2019-12-17 12:33:50,096] jepsen worker 42 - jepsen.util 11022 :info :cas [14 [34 19]] :RequestCanceledException INFO [2019-12-17 12:33:50,098] jepsen worker 42 - jepsen.util 11112 :invoke :read [14 nil] INFO [2019-12-17 12:33:50,104] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44067,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:33:50,104] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44067,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:33:50,104] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44070,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:33:50,105] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44070,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:33:50,107] jepsen worker 54 - jepsen.util 8514 :ok :read [18 22] INFO [2019-12-17 12:33:50,107] jepsen worker 54 - jepsen.util 8514 :invoke :write [18 46] INFO [2019-12-17 12:33:50,126] jepsen worker 26 - jepsen.util 10736 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:33:50,127] jepsen worker 26 - jepsen.util 10736 :invoke :cas [8 [18 34]] INFO [2019-12-17 12:33:50,132] jepsen worker 32 - jepsen.util 7592 :ok :write [10 16] INFO [2019-12-17 12:33:50,132] jepsen worker 74 - jepsen.util 7814 :ok :write [24 38] INFO [2019-12-17 12:33:50,132] jepsen worker 32 - jepsen.util 7592 :invoke :write [10 12] INFO [2019-12-17 12:33:50,132] jepsen worker 74 - jepsen.util 7814 :invoke :write [24 44] INFO [2019-12-17 12:33:50,142] jepsen worker 66 - jepsen.util 10236 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:33:50,143] jepsen worker 66 - jepsen.util 10236 :invoke :read [22 nil] INFO [2019-12-17 12:33:50,143] jepsen worker 65 - jepsen.util 6365 :ok :read [21 31] INFO [2019-12-17 12:33:50,143] jepsen worker 65 - jepsen.util 6365 :invoke :read [21 nil] INFO [2019-12-17 12:33:50,160] jepsen worker 75 - jepsen.util 9885 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:33:50,160] jepsen worker 75 - jepsen.util 9885 :invoke :write [25 21] INFO [2019-12-17 12:33:50,180] jepsen worker 65 - jepsen.util 6365 :ok :read [21 31] INFO [2019-12-17 12:33:50,180] jepsen worker 65 - jepsen.util 6365 :invoke :cas [21 [10 21]] INFO [2019-12-17 12:33:50,195] jepsen worker 43 - jepsen.util 9763 :info :write [14 29] :RequestCanceledException INFO [2019-12-17 12:33:50,199] jepsen worker 43 - jepsen.util 9853 :invoke :write [14 37] INFO [2019-12-17 12:33:50,210] jepsen worker 36 - jepsen.util 7866 :ok :read [12 47] INFO [2019-12-17 12:33:50,211] jepsen worker 36 - jepsen.util 7866 :invoke :read [12 nil] INFO [2019-12-17 12:33:50,212] jepsen worker 77 - jepsen.util 10607 :info :write [25 49] :RequestCanceledException INFO [2019-12-17 12:33:50,214] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44088,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:33:50,214] jepsen worker 77 - jepsen.util 10697 :invoke :read [25 nil] INFO [2019-12-17 12:33:50,214] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44088,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:33:50,214] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44090,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:33:50,215] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44090,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:33:50,217] jepsen worker 65 - jepsen.util 6365 :fail :cas [21 [10 21]] :ValueNotSwapFrom INFO [2019-12-17 12:33:50,217] jepsen worker 65 - jepsen.util 6365 :invoke :write [21 23] INFO [2019-12-17 12:33:50,228] jepsen worker 0 - jepsen.util 10800 :info :cas [0 [26 34]] :RequestCanceledException INFO [2019-12-17 12:33:50,228] jepsen worker 47 - jepsen.util 7877 :fail :read [15 nil] :RequestTimeoutException INFO [2019-12-17 12:33:50,228] jepsen worker 73 - jepsen.util 7273 :info :write [24 40] :RequestCanceledException INFO [2019-12-17 12:33:50,228] jepsen worker 83 - jepsen.util 10613 :info :write [27 22] :RequestCanceledException INFO [2019-12-17 12:33:50,228] jepsen worker 47 - jepsen.util 7877 :invoke :cas [15 [23 40]] INFO [2019-12-17 12:33:50,231] jepsen worker 73 - jepsen.util 7363 :invoke :write [24 47] INFO [2019-12-17 12:33:50,231] jepsen worker 0 - jepsen.util 10890 :invoke :read [0 nil] INFO [2019-12-17 12:33:50,232] jepsen worker 83 - jepsen.util 10703 :invoke :cas [27 [15 47]] INFO [2019-12-17 12:33:50,242] jepsen worker 21 - jepsen.util 9651 :info :cas [7 [43 5]] :RequestCanceledException INFO [2019-12-17 12:33:50,242] jepsen worker 50 - jepsen.util 11120 :info :cas [16 [37 27]] :RequestCanceledException INFO [2019-12-17 12:33:50,245] jepsen worker 21 - jepsen.util 9741 :invoke :write [7 10] INFO [2019-12-17 12:33:50,246] jepsen worker 50 - jepsen.util 11210 :invoke :read [16 nil] INFO [2019-12-17 12:33:50,259] jepsen worker 15 - jepsen.util 9285 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:33:50,259] jepsen worker 15 - jepsen.util 9285 :invoke :write [5 25] INFO [2019-12-17 12:33:50,268] jepsen worker 87 - jepsen.util 7017 :fail :cas [29 [46 46]] :ValueNotSwapFrom INFO [2019-12-17 12:33:50,268] jepsen worker 87 - jepsen.util 7017 :invoke :write [29 9] INFO [2019-12-17 12:33:50,278] jepsen worker 65 - jepsen.util 6365 :ok :write [21 23] INFO [2019-12-17 12:33:50,278] jepsen worker 65 - jepsen.util 6365 :invoke :read [21 nil] INFO [2019-12-17 12:33:50,295] jepsen worker 62 - jepsen.util 10412 :info :write [20 23] :RequestCanceledException INFO [2019-12-17 12:33:50,295] jepsen worker 41 - jepsen.util 10751 :info :write [13 42] :RequestCanceledException INFO [2019-12-17 12:33:50,295] jepsen worker 60 - jepsen.util 11310 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:33:50,295] jepsen worker 60 - jepsen.util 11310 :invoke :cas [20 [25 16]] INFO [2019-12-17 12:33:50,296] jepsen worker 80 - jepsen.util 7550 :ok :read [26 15] INFO [2019-12-17 12:33:50,296] jepsen worker 80 - jepsen.util 7550 :invoke :read [26 nil] INFO [2019-12-17 12:33:50,297] jepsen worker 62 - jepsen.util 10502 :invoke :read [20 nil] INFO [2019-12-17 12:33:50,297] jepsen worker 41 - jepsen.util 10841 :invoke :cas [13 [24 0]] INFO [2019-12-17 12:33:50,313] jepsen worker 22 - jepsen.util 10102 :info :write [7 44] :RequestCanceledException INFO [2019-12-17 12:33:50,313] jepsen worker 33 - jepsen.util 10113 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:33:50,313] jepsen worker 33 - jepsen.util 10113 :invoke :read [11 nil] INFO [2019-12-17 12:33:50,314] jepsen worker 22 - jepsen.util 10192 :invoke :write [7 38] INFO [2019-12-17 12:33:50,323] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44097,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:33:50,323] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44097,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:33:50,323] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44101,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:33:50,323] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44101,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:33:50,328] jepsen worker 23 - jepsen.util 10283 :info :write [7 23] :RequestCanceledException INFO [2019-12-17 12:33:50,328] jepsen worker 86 - jepsen.util 10976 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:33:50,328] jepsen worker 86 - jepsen.util 10976 :invoke :read [28 nil] INFO [2019-12-17 12:33:50,329] jepsen worker 58 - jepsen.util 11128 :info :write [19 29] :RequestCanceledException INFO [2019-12-17 12:33:50,329] jepsen worker 68 - jepsen.util 9788 :info :write [22 0] :RequestCanceledException INFO [2019-12-17 12:33:50,329] jepsen worker 67 - jepsen.util 10237 :info :write [22 31] :RequestCanceledException INFO [2019-12-17 12:33:50,329] jepsen worker 25 - jepsen.util 10735 :info :write [8 44] :RequestCanceledException INFO [2019-12-17 12:33:50,332] jepsen worker 67 - jepsen.util 10327 :invoke :read [22 nil] INFO [2019-12-17 12:33:50,332] jepsen worker 25 - jepsen.util 10825 :invoke :cas [8 [16 0]] INFO [2019-12-17 12:33:50,333] jepsen worker 68 - jepsen.util 9878 :invoke :read [22 nil] INFO [2019-12-17 12:33:50,335] jepsen worker 23 - jepsen.util 10373 :invoke :read [7 nil] INFO [2019-12-17 12:33:50,336] jepsen worker 58 - jepsen.util 11218 :invoke :write [19 21] INFO [2019-12-17 12:33:50,342] jepsen worker 59 - jepsen.util 10049 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:33:50,342] jepsen worker 24 - jepsen.util 9024 :info :cas [8 [9 40]] :RequestCanceledException INFO [2019-12-17 12:33:50,342] jepsen worker 59 - jepsen.util 10049 :invoke :read [19 nil] INFO [2019-12-17 12:33:50,346] jepsen worker 24 - jepsen.util 9114 :invoke :cas [8 [7 6]] INFO [2019-12-17 12:33:50,391] jepsen worker 5 - jepsen.util 7205 :ok :read [1 11] INFO [2019-12-17 12:33:50,391] jepsen worker 5 - jepsen.util 7205 :invoke :read [1 nil] INFO [2019-12-17 12:33:50,395] jepsen worker 18 - jepsen.util 7758 :info :write [6 32] :RequestCanceledException INFO [2019-12-17 12:33:50,395] jepsen worker 46 - jepsen.util 7966 :fail :read [15 nil] :RequestTimeoutException INFO [2019-12-17 12:33:50,395] jepsen worker 46 - jepsen.util 7966 :invoke :cas [15 [14 17]] INFO [2019-12-17 12:33:50,397] jepsen worker 18 - jepsen.util 7848 :invoke :cas [6 [38 11]] INFO [2019-12-17 12:33:50,435] jepsen worker 52 - jepsen.util 7882 :fail :cas [17 [31 17]] :ValueNotSwapFrom INFO [2019-12-17 12:33:50,436] jepsen worker 52 - jepsen.util 7882 :invoke :cas [17 [28 5]] INFO [2019-12-17 12:33:50,437] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44131,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:33:50,437] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44131,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:33:50,437] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44134,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:33:50,437] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44134,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:33:50,438] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44135,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:33:50,438] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44135,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:33:50,448] jepsen worker 18 - jepsen.util 7848 :fail :cas [6 [38 11]] :ValueNotSwapFrom INFO [2019-12-17 12:33:50,448] jepsen worker 18 - jepsen.util 7848 :invoke :write [6 48] INFO [2019-12-17 12:33:50,453] jepsen worker 4 - jepsen.util 7024 :ok :read [1 11] INFO [2019-12-17 12:33:50,453] jepsen worker 4 - jepsen.util 7024 :invoke :write [1 13] INFO [2019-12-17 12:33:50,459] jepsen worker 82 - jepsen.util 10972 :info :write [27 17] :RequestCanceledException INFO [2019-12-17 12:33:50,463] jepsen worker 82 - jepsen.util 11062 :invoke :write [27 20] INFO [2019-12-17 12:33:50,529] jepsen worker 78 - jepsen.util 7638 :info :write [26 3] :RequestCanceledException INFO [2019-12-17 12:33:50,529] jepsen worker 1 - jepsen.util 10441 :info :cas [0 [35 26]] :RequestCanceledException INFO [2019-12-17 12:33:50,529] jepsen worker 3 - jepsen.util 7563 :info :write [1 16] :RequestCanceledException INFO [2019-12-17 12:33:50,532] jepsen worker 1 - jepsen.util 10531 :invoke :write [0 46] INFO [2019-12-17 12:33:50,532] jepsen worker 78 - jepsen.util 7728 :invoke :cas [26 [48 49]] INFO [2019-12-17 12:33:50,533] jepsen worker 3 - jepsen.util 7653 :invoke :read [1 nil] INFO [2019-12-17 12:33:50,542] jepsen worker 71 - jepsen.util 8261 :fail :cas [23 [23 40]] :ValueNotSwapFrom INFO [2019-12-17 12:33:50,542] jepsen worker 71 - jepsen.util 8261 :invoke :cas [23 [34 40]] INFO [2019-12-17 12:33:50,542] jepsen worker 6 - jepsen.util 10896 :info :cas [2 [1 21]] :RequestCanceledException INFO [2019-12-17 12:33:50,546] jepsen worker 6 - jepsen.util 10986 :invoke :read [2 nil] INFO [2019-12-17 12:33:50,571] jepsen worker 79 - jepsen.util 7729 :ok :read [26 15] INFO [2019-12-17 12:33:50,571] jepsen worker 79 - jepsen.util 7729 :invoke :cas [26 [49 11]] INFO [2019-12-17 12:33:50,579] jepsen worker 9 - jepsen.util 7929 :ok :write [3 37] INFO [2019-12-17 12:33:50,579] jepsen worker 32 - jepsen.util 7592 :ok :write [10 12] INFO [2019-12-17 12:33:50,579] jepsen worker 20 - jepsen.util 7310 :ok :write [6 4] INFO [2019-12-17 12:33:50,579] jepsen worker 9 - jepsen.util 7929 :invoke :read [3 nil] INFO [2019-12-17 12:33:50,579] jepsen worker 32 - jepsen.util 7592 :invoke :write [10 7] INFO [2019-12-17 12:33:50,579] jepsen worker 20 - jepsen.util 7310 :invoke :cas [6 [17 39]] INFO [2019-12-17 12:33:50,594] jepsen worker 57 - jepsen.util 10317 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:33:50,594] jepsen worker 57 - jepsen.util 10317 :invoke :cas [19 [13 17]] INFO [2019-12-17 12:33:50,614] jepsen worker 69 - jepsen.util 7809 :ok :write [23 18] INFO [2019-12-17 12:33:50,614] jepsen worker 69 - jepsen.util 7809 :invoke :read [23 nil] INFO [2019-12-17 12:33:50,617] jepsen worker 73 - jepsen.util 7363 :ok :write [24 47] INFO [2019-12-17 12:33:50,617] jepsen worker 73 - jepsen.util 7363 :invoke :cas [24 [33 27]] INFO [2019-12-17 12:33:50,654] jepsen worker 9 - jepsen.util 7929 :ok :read [3 37] INFO [2019-12-17 12:33:50,654] jepsen worker 9 - jepsen.util 7929 :invoke :cas [3 [10 47]] INFO [2019-12-17 12:33:50,677] jepsen worker 36 - jepsen.util 7866 :ok :read [12 47] INFO [2019-12-17 12:33:50,677] jepsen worker 36 - jepsen.util 7866 :invoke :cas [12 [48 16]] INFO [2019-12-17 12:33:50,683] jepsen worker 20 - jepsen.util 7310 :fail :cas [6 [17 39]] :ValueNotSwapFrom INFO [2019-12-17 12:33:50,683] jepsen worker 20 - jepsen.util 7310 :invoke :cas [6 [14 5]] INFO [2019-12-17 12:33:50,695] jepsen worker 84 - jepsen.util 9984 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:33:50,695] jepsen worker 84 - jepsen.util 9984 :invoke :write [28 16] INFO [2019-12-17 12:33:50,703] jepsen worker 46 - jepsen.util 7966 :fail :cas [15 [14 17]] :ValueNotSwapFrom INFO [2019-12-17 12:33:50,703] jepsen worker 46 - jepsen.util 7966 :invoke :write [15 44] INFO [2019-12-17 12:33:50,708] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44155,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:33:50,708] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44155,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:33:50,709] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44157,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:33:50,710] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44157,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:33:50,712] jepsen worker 3 - jepsen.util 7653 :ok :read [1 11] INFO [2019-12-17 12:33:50,712] jepsen worker 3 - jepsen.util 7653 :invoke :cas [1 [34 39]] INFO [2019-12-17 12:33:50,713] jepsen worker 48 - jepsen.util 9408 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:33:50,714] jepsen worker 48 - jepsen.util 9408 :invoke :cas [16 [11 38]] INFO [2019-12-17 12:33:50,728] jepsen worker 73 - jepsen.util 7363 :fail :cas [24 [33 27]] :ValueNotSwapFrom INFO [2019-12-17 12:33:50,728] jepsen worker 73 - jepsen.util 7363 :invoke :write [24 23] INFO [2019-12-17 12:33:50,728] jepsen worker 17 - jepsen.util 10457 :info :write [5 4] :RequestCanceledException INFO [2019-12-17 12:33:50,728] jepsen worker 81 - jepsen.util 10161 :info :write [27 7] :RequestCanceledException INFO [2019-12-17 12:33:50,733] jepsen worker 17 - jepsen.util 10547 :invoke :read [5 nil] INFO [2019-12-17 12:33:50,733] jepsen worker 81 - jepsen.util 10251 :invoke :read [27 nil] INFO [2019-12-17 12:33:50,745] jepsen worker 5 - jepsen.util 7205 :ok :read [1 11] INFO [2019-12-17 12:33:50,745] jepsen worker 5 - jepsen.util 7205 :invoke :cas [1 [0 38]] INFO [2019-12-17 12:33:50,753] jepsen worker 9 - jepsen.util 7929 :fail :cas [3 [10 47]] :ValueNotSwapFrom INFO [2019-12-17 12:33:50,753] jepsen worker 9 - jepsen.util 7929 :invoke :cas [3 [20 43]] INFO [2019-12-17 12:33:50,759] jepsen worker 37 - jepsen.util 7147 :fail :cas [12 [46 8]] :ValueNotSwapFrom INFO [2019-12-17 12:33:50,759] jepsen worker 37 - jepsen.util 7147 :invoke :cas [12 [15 2]] INFO [2019-12-17 12:33:50,776] jepsen worker 20 - jepsen.util 7310 :fail :cas [6 [14 5]] :ValueNotSwapFrom INFO [2019-12-17 12:33:50,776] jepsen worker 20 - jepsen.util 7310 :invoke :cas [6 [34 24]] INFO [2019-12-17 12:33:50,811] jepsen worker 9 - jepsen.util 7929 :fail :cas [3 [20 43]] :ValueNotSwapFrom INFO [2019-12-17 12:33:50,812] jepsen worker 9 - jepsen.util 7929 :invoke :read [3 nil] INFO [2019-12-17 12:33:50,820] jepsen worker 20 - jepsen.util 7310 :fail :cas [6 [34 24]] :ValueNotSwapFrom INFO [2019-12-17 12:33:50,820] jepsen worker 20 - jepsen.util 7310 :invoke :write [6 13] INFO [2019-12-17 12:33:50,827] jepsen worker 76 - jepsen.util 10246 :info :write [25 26] :RequestCanceledException INFO [2019-12-17 12:33:50,832] jepsen worker 76 - jepsen.util 10336 :invoke :read [25 nil] INFO [2019-12-17 12:33:50,855] jepsen worker 9 - jepsen.util 7929 :ok :read [3 37] INFO [2019-12-17 12:33:50,855] jepsen worker 9 - jepsen.util 7929 :invoke :write [3 12] INFO [2019-12-17 12:33:50,861] jepsen worker 52 - jepsen.util 7882 :fail :cas [17 [28 5]] :ValueNotSwapFrom INFO [2019-12-17 12:33:50,861] jepsen worker 52 - jepsen.util 7882 :invoke :write [17 46] INFO [2019-12-17 12:33:50,872] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44176,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:33:50,872] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44176,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:33:50,872] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44180,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:33:50,872] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44180,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:33:50,890] jepsen worker 80 - jepsen.util 7550 :ok :read [26 15] INFO [2019-12-17 12:33:50,890] jepsen worker 80 - jepsen.util 7550 :invoke :cas [26 [37 45]] INFO [2019-12-17 12:33:50,895] jepsen worker 39 - jepsen.util 9939 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:33:50,896] jepsen worker 39 - jepsen.util 9939 :invoke :cas [13 [45 43]] INFO [2019-12-17 12:33:50,929] jepsen worker 40 - jepsen.util 10750 :info :cas [13 [14 30]] :RequestCanceledException INFO [2019-12-17 12:33:50,931] jepsen worker 40 - jepsen.util 10840 :invoke :cas [13 [38 23]] INFO [2019-12-17 12:33:50,942] jepsen worker 7 - jepsen.util 10447 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:33:50,942] jepsen worker 89 - jepsen.util 7379 :info :write [29 16] :RequestCanceledException INFO [2019-12-17 12:33:50,942] jepsen worker 7 - jepsen.util 10447 :invoke :write [2 6] INFO [2019-12-17 12:33:50,960] jepsen worker 16 - jepsen.util 11626 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:33:50,960] jepsen worker 16 - jepsen.util 11626 :invoke :write [5 13] INFO [2019-12-17 12:33:50,961] jepsen worker 89 - jepsen.util 7469 :invoke :write [29 45] INFO [2019-12-17 12:33:50,995] jepsen worker 49 - jepsen.util 9859 :info :write [16 47] :RequestCanceledException INFO [2019-12-17 12:33:51,002] jepsen worker 49 - jepsen.util 9949 :invoke :cas [16 [23 7]] INFO [2019-12-17 12:33:51,013] jepsen worker 61 - jepsen.util 10951 :info :write [20 24] :RequestCanceledException INFO [2019-12-17 12:33:51,013] jepsen worker 44 - jepsen.util 10934 :info :write [14 17] :RequestCanceledException INFO [2019-12-17 12:33:51,013] jepsen worker 35 - jepsen.util 10115 :info :cas [11 [33 38]] :RequestCanceledException INFO [2019-12-17 12:33:51,015] jepsen worker 78 - jepsen.util 7728 :fail :cas [26 [48 49]] :ValueNotSwapFrom INFO [2019-12-17 12:33:51,015] jepsen worker 78 - jepsen.util 7728 :invoke :read [26 nil] INFO [2019-12-17 12:33:51,019] jepsen worker 44 - jepsen.util 11024 :invoke :read [14 nil] INFO [2019-12-17 12:33:51,019] jepsen worker 61 - jepsen.util 11041 :invoke :read [20 nil] INFO [2019-12-17 12:33:51,020] jepsen worker 35 - jepsen.util 10205 :invoke :read [11 nil] INFO [2019-12-17 12:33:51,029] jepsen worker 85 - jepsen.util 11155 :info :write [28 21] :RequestCanceledException INFO [2019-12-17 12:33:51,034] jepsen worker 85 - jepsen.util 11245 :invoke :read [28 nil] INFO [2019-12-17 12:33:51,042] jepsen worker 13 - jepsen.util 7213 :info :write [4 46] :RequestCanceledException INFO [2019-12-17 12:33:51,043] jepsen worker 13 - jepsen.util 7303 :invoke :read [4 nil] INFO [2019-12-17 12:33:51,044] jepsen worker 71 - jepsen.util 8261 :fail :cas [23 [34 40]] :ValueNotSwapFrom INFO [2019-12-17 12:33:51,044] jepsen worker 71 - jepsen.util 8261 :invoke :cas [23 [0 40]] INFO [2019-12-17 12:33:51,049] jepsen worker 80 - jepsen.util 7550 :fail :cas [26 [37 45]] :ValueNotSwapFrom INFO [2019-12-17 12:33:51,049] jepsen worker 80 - jepsen.util 7550 :invoke :cas [26 [35 32]] INFO [2019-12-17 12:33:51,051] jepsen worker 69 - jepsen.util 7809 :ok :read [23 18] INFO [2019-12-17 12:33:51,051] jepsen worker 69 - jepsen.util 7809 :invoke :cas [23 [2 33]] INFO [2019-12-17 12:33:51,055] jepsen worker 3 - jepsen.util 7653 :fail :cas [1 [34 39]] :ValueNotSwapFrom INFO [2019-12-17 12:33:51,056] jepsen worker 3 - jepsen.util 7653 :invoke :write [1 31] INFO [2019-12-17 12:33:51,059] jepsen worker 8 - jepsen.util 10178 :info :cas [2 [40 21]] :RequestCanceledException INFO [2019-12-17 12:33:51,059] jepsen worker 14 - jepsen.util 7844 :info :write [4 18] :RequestCanceledException INFO [2019-12-17 12:33:51,062] jepsen worker 14 - jepsen.util 7934 :invoke :write [4 14] INFO [2019-12-17 12:33:51,062] jepsen worker 8 - jepsen.util 10268 :invoke :read [2 nil] INFO [2019-12-17 12:33:51,088] jepsen worker 37 - jepsen.util 7147 :fail :cas [12 [15 2]] :ValueNotSwapFrom INFO [2019-12-17 12:33:51,088] jepsen worker 37 - jepsen.util 7147 :invoke :read [12 nil] INFO [2019-12-17 12:33:51,092] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44187,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:33:51,092] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44187,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:33:51,092] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44192,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:33:51,092] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44192,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:33:51,096] jepsen worker 88 - jepsen.util 7558 :info :cas [29 [13 31]] :RequestCanceledException INFO [2019-12-17 12:33:51,096] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:33:51,096] jepsen nemesis - couchbase.util Rebalance check count: 338 INFO [2019-12-17 12:33:51,096] jepsen nemesis - couchbase.util Rebalance stuck count: 319 INFO [2019-12-17 12:33:51,097] jepsen worker 88 - jepsen.util 7648 :invoke :write [29 1] INFO [2019-12-17 12:33:51,102] jepsen worker 5 - jepsen.util 7205 :fail :cas [1 [0 38]] :ValueNotSwapFrom INFO [2019-12-17 12:33:51,102] jepsen worker 5 - jepsen.util 7205 :invoke :read [1 nil] INFO [2019-12-17 12:33:51,195] jepsen worker 51 - jepsen.util 7251 :info :write [17 46] :RequestCanceledException INFO [2019-12-17 12:33:51,197] jepsen worker 51 - jepsen.util 7341 :invoke :write [17 22] INFO [2019-12-17 12:33:51,202] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44211,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:33:51,202] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44211,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:33:51,202] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44212,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:33:51,202] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44212,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:33:51,228] jepsen worker 56 - jepsen.util 8606 :info :write [18 36] :RequestCanceledException INFO [2019-12-17 12:33:51,229] jepsen worker 56 - jepsen.util 8696 :invoke :write [18 3] INFO [2019-12-17 12:33:51,229] jepsen worker 55 - jepsen.util 8425 :info :write [18 3] :RequestCanceledException INFO [2019-12-17 12:33:51,230] jepsen worker 55 - jepsen.util 8515 :invoke :cas [18 [36 32]] INFO [2019-12-17 12:33:51,242] jepsen worker 70 - jepsen.util 7900 :info :write [23 29] :RequestCanceledException INFO [2019-12-17 12:33:51,246] jepsen worker 70 - jepsen.util 7990 :invoke :write [23 42] INFO [2019-12-17 12:33:51,260] jepsen worker 63 - jepsen.util 6543 :info :cas [21 [20 6]] :RequestCanceledException INFO [2019-12-17 12:33:51,261] jepsen worker 63 - jepsen.util 6633 :invoke :read [21 nil] INFO [2019-12-17 12:33:51,311] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44220,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:33:51,311] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44220,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:33:51,311] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44223,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:33:51,311] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44223,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:33:51,319] jepsen worker 13 - jepsen.util 7303 :ok :read [4 28] INFO [2019-12-17 12:33:51,319] jepsen worker 13 - jepsen.util 7303 :invoke :write [4 34] INFO [2019-12-17 12:33:51,322] jepsen worker 63 - jepsen.util 6633 :ok :read [21 23] INFO [2019-12-17 12:33:51,322] jepsen worker 63 - jepsen.util 6633 :invoke :write [21 29] INFO [2019-12-17 12:33:51,342] jepsen worker 30 - jepsen.util 7500 :info :cas [10 [21 33]] :RequestCanceledException INFO [2019-12-17 12:33:51,346] jepsen worker 30 - jepsen.util 7590 :invoke :write [10 21] INFO [2019-12-17 12:33:51,361] jepsen worker 78 - jepsen.util 7728 :ok :read [26 15] INFO [2019-12-17 12:33:51,362] jepsen worker 78 - jepsen.util 7728 :invoke :cas [26 [27 25]] INFO [2019-12-17 12:33:51,372] jepsen worker 36 - jepsen.util 7866 :fail :cas [12 [48 16]] :ValueNotSwapFrom INFO [2019-12-17 12:33:51,372] jepsen worker 36 - jepsen.util 7866 :invoke :write [12 6] INFO [2019-12-17 12:33:51,395] jepsen worker 2 - jepsen.util 10082 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:33:51,395] jepsen worker 2 - jepsen.util 10082 :invoke :read [0 nil] INFO [2019-12-17 12:33:51,410] jepsen worker 19 - jepsen.util 7759 :info :cas [6 [0 36]] :RequestCanceledException INFO [2019-12-17 12:33:51,410] jepsen worker 31 - jepsen.util 6871 :info :write [10 35] :RequestCanceledException INFO [2019-12-17 12:33:51,412] jepsen worker 31 - jepsen.util 6961 :invoke :read [10 nil] INFO [2019-12-17 12:33:51,413] jepsen worker 19 - jepsen.util 7849 :invoke :cas [6 [44 10]] INFO [2019-12-17 12:33:51,425] jepsen worker 63 - jepsen.util 6633 :ok :write [21 29] INFO [2019-12-17 12:33:51,425] jepsen worker 63 - jepsen.util 6633 :invoke :write [21 48] INFO [2019-12-17 12:33:51,426] jepsen worker 11 - jepsen.util 7751 :info :write [3 6] :RequestCanceledException INFO [2019-12-17 12:33:51,427] jepsen worker 28 - jepsen.util 6058 :fail :read [9 nil] :RequestTimeoutException INFO [2019-12-17 12:33:51,427] jepsen worker 28 - jepsen.util 6058 :invoke :read [9 nil] INFO [2019-12-17 12:33:51,427] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44239,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:33:51,427] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44239,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:33:51,427] jepsen worker 11 - jepsen.util 7841 :invoke :read [3 nil] INFO [2019-12-17 12:33:51,440] jepsen worker 9 - jepsen.util 7929 :ok :write [3 12] INFO [2019-12-17 12:33:51,440] jepsen worker 9 - jepsen.util 7929 :invoke :cas [3 [43 3]] INFO [2019-12-17 12:33:51,446] jepsen worker 46 - jepsen.util 7966 :ok :write [15 44] INFO [2019-12-17 12:33:51,446] jepsen worker 46 - jepsen.util 7966 :invoke :cas [15 [37 23]] INFO [2019-12-17 12:33:51,465] jepsen worker 31 - jepsen.util 6961 :ok :read [10 12] INFO [2019-12-17 12:33:51,465] jepsen worker 31 - jepsen.util 6961 :invoke :write [10 16] INFO [2019-12-17 12:33:51,466] jepsen worker 20 - jepsen.util 7310 :ok :write [6 13] INFO [2019-12-17 12:33:51,466] jepsen worker 20 - jepsen.util 7310 :invoke :read [6 nil] INFO [2019-12-17 12:33:51,469] jepsen worker 32 - jepsen.util 7592 :ok :write [10 7] INFO [2019-12-17 12:33:51,469] jepsen worker 32 - jepsen.util 7592 :invoke :read [10 nil] INFO [2019-12-17 12:33:51,497] jepsen worker 9 - jepsen.util 7929 :fail :cas [3 [43 3]] :ValueNotSwapFrom INFO [2019-12-17 12:33:51,497] jepsen worker 9 - jepsen.util 7929 :invoke :read [3 nil] INFO [2019-12-17 12:33:51,498] jepsen worker 28 - jepsen.util 6058 :ok :read [9 5] INFO [2019-12-17 12:33:51,498] jepsen worker 28 - jepsen.util 6058 :invoke :write [9 35] INFO [2019-12-17 12:33:51,510] jepsen worker 79 - jepsen.util 7729 :fail :cas [26 [49 11]] :ValueNotSwapFrom INFO [2019-12-17 12:33:51,510] jepsen worker 79 - jepsen.util 7729 :invoke :cas [26 [25 40]] INFO [2019-12-17 12:33:51,529] jepsen worker 64 - jepsen.util 6004 :info :cas [21 [2 2]] :RequestCanceledException INFO [2019-12-17 12:33:51,536] jepsen worker 64 - jepsen.util 6094 :invoke :write [21 3] INFO [2019-12-17 12:33:51,544] jepsen worker 19 - jepsen.util 7849 :fail :cas [6 [44 10]] :ValueNotSwapFrom INFO [2019-12-17 12:33:51,544] jepsen worker 19 - jepsen.util 7849 :invoke :cas [6 [9 47]] INFO [2019-12-17 12:33:51,608] jepsen worker 5 - jepsen.util 7205 :ok :read [1 11] INFO [2019-12-17 12:33:51,608] jepsen worker 5 - jepsen.util 7205 :invoke :cas [1 [1 47]] INFO [2019-12-17 12:33:51,635] jepsen worker 20 - jepsen.util 7310 :ok :read [6 13] INFO [2019-12-17 12:33:51,635] jepsen worker 20 - jepsen.util 7310 :invoke :write [6 49] INFO [2019-12-17 12:33:51,639] jepsen worker 9 - jepsen.util 7929 :ok :read [3 12] INFO [2019-12-17 12:33:51,639] jepsen worker 9 - jepsen.util 7929 :invoke :cas [3 [9 34]] INFO [2019-12-17 12:33:51,642] jepsen worker 38 - jepsen.util 7868 :info :write [12 45] :RequestCanceledException INFO [2019-12-17 12:33:51,643] jepsen worker 45 - jepsen.util 7605 :info :cas [15 [44 21]] :RequestCanceledException INFO [2019-12-17 12:33:51,648] jepsen worker 38 - jepsen.util 7958 :invoke :cas [12 [33 13]] INFO [2019-12-17 12:33:51,648] jepsen worker 45 - jepsen.util 7695 :invoke :write [15 38] INFO [2019-12-17 12:33:51,659] jepsen worker 63 - jepsen.util 6633 :ok :write [21 48] INFO [2019-12-17 12:33:51,659] jepsen worker 63 - jepsen.util 6633 :invoke :cas [21 [26 17]] INFO [2019-12-17 12:33:51,660] jepsen worker 53 - jepsen.util 7073 :info :write [17 44] :RequestCanceledException INFO [2019-12-17 12:33:51,663] jepsen worker 71 - jepsen.util 8261 :fail :cas [23 [0 40]] :ValueNotSwapFrom INFO [2019-12-17 12:33:51,663] jepsen worker 71 - jepsen.util 8261 :invoke :cas [23 [47 31]] INFO [2019-12-17 12:33:51,667] jepsen worker 53 - jepsen.util 7163 :invoke :write [17 23] INFO [2019-12-17 12:33:51,696] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44260,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:33:51,696] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44260,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:33:51,708] jepsen worker 55 - jepsen.util 8515 :fail :cas [18 [36 32]] :ValueNotSwapFrom INFO [2019-12-17 12:33:51,708] jepsen worker 55 - jepsen.util 8515 :invoke :write [18 26] INFO [2019-12-17 12:33:51,713] jepsen worker 69 - jepsen.util 7809 :fail :cas [23 [2 33]] :ValueNotSwapFrom INFO [2019-12-17 12:33:51,714] jepsen worker 69 - jepsen.util 7809 :invoke :cas [23 [2 29]] INFO [2019-12-17 12:33:51,723] jepsen worker 73 - jepsen.util 7363 :ok :write [24 23] INFO [2019-12-17 12:33:51,723] jepsen worker 73 - jepsen.util 7363 :invoke :write [24 6] INFO [2019-12-17 12:33:51,737] jepsen worker 63 - jepsen.util 6633 :fail :cas [21 [26 17]] :ValueNotSwapFrom INFO [2019-12-17 12:33:51,737] jepsen worker 63 - jepsen.util 6633 :invoke :write [21 34] INFO [2019-12-17 12:33:51,743] jepsen worker 10 - jepsen.util 7570 :info :write [3 37] :RequestCanceledException INFO [2019-12-17 12:33:51,753] jepsen worker 10 - jepsen.util 7660 :invoke :cas [3 [14 39]] INFO [2019-12-17 12:33:51,761] jepsen worker 9 - jepsen.util 7929 :fail :cas [3 [9 34]] :ValueNotSwapFrom INFO [2019-12-17 12:33:51,762] jepsen worker 9 - jepsen.util 7929 :invoke :write [3 28] INFO [2019-12-17 12:33:51,779] jepsen worker 19 - jepsen.util 7849 :fail :cas [6 [9 47]] :ValueNotSwapFrom INFO [2019-12-17 12:33:51,779] jepsen worker 19 - jepsen.util 7849 :invoke :cas [6 [11 29]] INFO [2019-12-17 12:33:51,797] jepsen worker 30 - jepsen.util 7590 :ok :write [10 21] INFO [2019-12-17 12:33:51,797] jepsen worker 30 - jepsen.util 7590 :invoke :cas [10 [7 43]] INFO [2019-12-17 12:33:51,823] jepsen worker 11 - jepsen.util 7841 :ok :read [3 12] INFO [2019-12-17 12:33:51,823] jepsen worker 11 - jepsen.util 7841 :invoke :cas [3 [43 10]] INFO [2019-12-17 12:33:51,830] jepsen worker 37 - jepsen.util 7147 :ok :read [12 47] INFO [2019-12-17 12:33:51,830] jepsen worker 37 - jepsen.util 7147 :invoke :cas [12 [43 6]] INFO [2019-12-17 12:33:51,853] jepsen worker 46 - jepsen.util 7966 :fail :cas [15 [37 23]] :ValueNotSwapFrom INFO [2019-12-17 12:33:51,853] jepsen worker 46 - jepsen.util 7966 :invoke :read [15 nil] INFO [2019-12-17 12:33:51,858] jepsen worker 72 - jepsen.util 7362 :info :write [24 18] :RequestCanceledException INFO [2019-12-17 12:33:51,862] jepsen worker 72 - jepsen.util 7452 :invoke :cas [24 [26 44]] INFO [2019-12-17 12:33:51,908] jepsen worker 63 - jepsen.util 6633 :ok :write [21 34] INFO [2019-12-17 12:33:51,909] jepsen worker 63 - jepsen.util 6633 :invoke :read [21 nil] INFO [2019-12-17 12:33:51,958] jepsen worker 27 - jepsen.util 7677 :fail :read [9 nil] :RequestTimeoutException INFO [2019-12-17 12:33:51,959] jepsen worker 27 - jepsen.util 7677 :invoke :write [9 10] INFO [2019-12-17 12:33:51,967] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44281,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:33:51,967] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44281,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:33:51,968] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44284,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:33:51,968] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44284,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:33:51,968] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44285,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:33:51,968] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44285,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:33:51,969] jepsen worker 63 - jepsen.util 6633 :ok :read [21 34] INFO [2019-12-17 12:33:51,969] jepsen worker 63 - jepsen.util 6633 :invoke :write [21 15] INFO [2019-12-17 12:33:52,004] jepsen worker 5 - jepsen.util 7205 :fail :cas [1 [1 47]] :ValueNotSwapFrom INFO [2019-12-17 12:33:52,004] jepsen worker 5 - jepsen.util 7205 :invoke :write [1 18] INFO [2019-12-17 12:33:52,015] jepsen worker 78 - jepsen.util 7728 :fail :cas [26 [27 25]] :ValueNotSwapFrom INFO [2019-12-17 12:33:52,015] jepsen worker 78 - jepsen.util 7728 :invoke :write [26 28] INFO [2019-12-17 12:33:52,061] jepsen worker 27 - jepsen.util 7677 :ok :write [9 10] INFO [2019-12-17 12:33:52,061] jepsen worker 27 - jepsen.util 7677 :invoke :read [9 nil] INFO [2019-12-17 12:33:52,078] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44291,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:33:52,078] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44291,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:33:52,093] jepsen worker 20 - jepsen.util 7310 :ok :write [6 49] INFO [2019-12-17 12:33:52,093] jepsen worker 45 - jepsen.util 7695 :ok :write [15 38] INFO [2019-12-17 12:33:52,093] jepsen worker 20 - jepsen.util 7310 :invoke :read [6 nil] INFO [2019-12-17 12:33:52,093] jepsen worker 45 - jepsen.util 7695 :invoke :write [15 38] INFO [2019-12-17 12:33:52,098] jepsen worker 73 - jepsen.util 7363 :ok :write [24 6] INFO [2019-12-17 12:33:52,098] jepsen worker 73 - jepsen.util 7363 :invoke :write [24 10] INFO [2019-12-17 12:33:52,112] jepsen worker 29 - jepsen.util 6149 :info :write [9 44] :RequestCanceledException INFO [2019-12-17 12:33:52,117] jepsen worker 38 - jepsen.util 7958 :fail :cas [12 [33 13]] :ValueNotSwapFrom INFO [2019-12-17 12:33:52,118] jepsen worker 38 - jepsen.util 7958 :invoke :cas [12 [37 4]] INFO [2019-12-17 12:33:52,118] jepsen worker 29 - jepsen.util 6239 :invoke :read [9 nil] INFO [2019-12-17 12:33:52,124] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:33:52,124] jepsen nemesis - couchbase.util Rebalance check count: 339 INFO [2019-12-17 12:33:52,124] jepsen nemesis - couchbase.util Rebalance stuck count: 320 INFO [2019-12-17 12:33:52,127] jepsen worker 34 - jepsen.util 9934 :info :write [11 27] :RequestCanceledException INFO [2019-12-17 12:33:52,127] jepsen worker 12 - jepsen.util 8112 :info :write [4 19] :RequestCanceledException INFO [2019-12-17 12:33:52,129] jepsen worker 12 - jepsen.util 8202 :invoke :read [4 nil] INFO [2019-12-17 12:33:52,131] jepsen worker 34 - jepsen.util 10024 :invoke :cas [11 [4 22]] INFO [2019-12-17 12:33:52,142] jepsen worker 42 - jepsen.util 11112 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:33:52,143] jepsen worker 42 - jepsen.util 11112 :invoke :write [14 19] INFO [2019-12-17 12:33:52,145] jepsen worker 27 - jepsen.util 7677 :ok :read [9 10] INFO [2019-12-17 12:33:52,146] jepsen worker 27 - jepsen.util 7677 :invoke :read [9 nil] INFO [2019-12-17 12:33:52,174] jepsen worker 10 - jepsen.util 7660 :fail :cas [3 [14 39]] :ValueNotSwapFrom INFO [2019-12-17 12:33:52,174] jepsen worker 10 - jepsen.util 7660 :invoke :cas [3 [48 28]] INFO [2019-12-17 12:33:52,204] jepsen worker 63 - jepsen.util 6633 :ok :write [21 15] INFO [2019-12-17 12:33:52,205] jepsen worker 63 - jepsen.util 6633 :invoke :write [21 40] INFO [2019-12-17 12:33:52,207] jepsen worker 11 - jepsen.util 7841 :fail :cas [3 [43 10]] :ValueNotSwapFrom INFO [2019-12-17 12:33:52,207] jepsen worker 11 - jepsen.util 7841 :invoke :read [3 nil] INFO [2019-12-17 12:33:52,213] jepsen worker 54 - jepsen.util 8514 :info :write [18 46] :RequestCanceledException INFO [2019-12-17 12:33:52,216] jepsen worker 54 - jepsen.util 8604 :invoke :write [18 47] INFO [2019-12-17 12:33:52,220] jepsen worker 19 - jepsen.util 7849 :fail :cas [6 [11 29]] :ValueNotSwapFrom INFO [2019-12-17 12:33:52,220] jepsen worker 19 - jepsen.util 7849 :invoke :cas [6 [14 43]] INFO [2019-12-17 12:33:52,227] jepsen worker 71 - jepsen.util 8261 :fail :cas [23 [47 31]] :ValueNotSwapFrom INFO [2019-12-17 12:33:52,227] jepsen worker 71 - jepsen.util 8261 :invoke :cas [23 [37 44]] INFO [2019-12-17 12:33:52,228] jepsen worker 74 - jepsen.util 7814 :info :write [24 44] :RequestCanceledException INFO [2019-12-17 12:33:52,228] jepsen worker 77 - jepsen.util 10697 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:33:52,228] jepsen worker 77 - jepsen.util 10697 :invoke :write [25 22] INFO [2019-12-17 12:33:52,229] jepsen worker 26 - jepsen.util 10736 :info :cas [8 [18 34]] :RequestCanceledException INFO [2019-12-17 12:33:52,233] jepsen worker 74 - jepsen.util 7904 :invoke :cas [24 [6 8]] INFO [2019-12-17 12:33:52,236] jepsen worker 26 - jepsen.util 10826 :invoke :write [8 46] INFO [2019-12-17 12:33:52,243] jepsen worker 66 - jepsen.util 10236 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:33:52,243] jepsen worker 66 - jepsen.util 10236 :invoke :write [22 16] INFO [2019-12-17 12:33:52,260] jepsen worker 75 - jepsen.util 9885 :info :write [25 21] :RequestCanceledException INFO [2019-12-17 12:33:52,262] jepsen worker 43 - jepsen.util 9853 :info :write [14 37] :RequestCanceledException INFO [2019-12-17 12:33:52,262] jepsen worker 50 - jepsen.util 11210 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:33:52,264] jepsen worker 50 - jepsen.util 11210 :invoke :cas [16 [22 45]] INFO [2019-12-17 12:33:52,271] jepsen worker 75 - jepsen.util 9975 :invoke :cas [25 [42 21]] INFO [2019-12-17 12:33:52,272] jepsen worker 43 - jepsen.util 9943 :invoke :cas [14 [17 45]] INFO [2019-12-17 12:33:52,294] jepsen worker 65 - jepsen.util 6365 :fail :read [21 nil] :RequestTimeoutException INFO [2019-12-17 12:33:52,294] jepsen worker 65 - jepsen.util 6365 :invoke :read [21 nil] INFO [2019-12-17 12:33:52,294] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44317,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:33:52,294] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44309,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:33:52,295] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44317,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:33:52,295] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44320,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:33:52,295] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44309,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:33:52,295] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44311,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:33:52,296] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44320,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:33:52,297] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44311,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:33:52,298] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44321,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:33:52,299] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44321,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:33:52,313] jepsen worker 0 - jepsen.util 10890 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:33:52,313] jepsen worker 0 - jepsen.util 10890 :invoke :read [0 nil] INFO [2019-12-17 12:33:52,313] jepsen worker 87 - jepsen.util 7017 :info :write [29 9] :RequestCanceledException INFO [2019-12-17 12:33:52,317] jepsen worker 38 - jepsen.util 7958 :fail :cas [12 [37 4]] :ValueNotSwapFrom INFO [2019-12-17 12:33:52,317] jepsen worker 38 - jepsen.util 7958 :invoke :read [12 nil] INFO [2019-12-17 12:33:52,323] jepsen worker 87 - jepsen.util 7107 :invoke :cas [29 [11 6]] INFO [2019-12-17 12:33:52,328] jepsen worker 47 - jepsen.util 7877 :info :cas [15 [23 40]] :RequestCanceledException INFO [2019-12-17 12:33:52,328] jepsen worker 62 - jepsen.util 10502 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:33:52,328] jepsen worker 62 - jepsen.util 10502 :invoke :cas [20 [27 12]] INFO [2019-12-17 12:33:52,328] jepsen worker 83 - jepsen.util 10703 :info :cas [27 [15 47]] :RequestCanceledException INFO [2019-12-17 12:33:52,338] jepsen worker 47 - jepsen.util 7967 :invoke :write [15 12] INFO [2019-12-17 12:33:52,342] jepsen worker 21 - jepsen.util 9741 :info :write [7 10] :RequestCanceledException INFO [2019-12-17 12:33:52,342] jepsen worker 83 - jepsen.util 10793 :invoke :read [27 nil] INFO [2019-12-17 12:33:52,349] jepsen worker 46 - jepsen.util 7966 :ok :read [15 38] INFO [2019-12-17 12:33:52,350] jepsen worker 46 - jepsen.util 7966 :invoke :cas [15 [43 35]] INFO [2019-12-17 12:33:52,353] jepsen worker 21 - jepsen.util 9831 :invoke :cas [7 [38 22]] INFO [2019-12-17 12:33:52,353] jepsen worker 79 - jepsen.util 7729 :fail :cas [26 [25 40]] :ValueNotSwapFrom INFO [2019-12-17 12:33:52,353] jepsen worker 79 - jepsen.util 7729 :invoke :cas [26 [22 4]] INFO [2019-12-17 12:33:52,356] jepsen worker 15 - jepsen.util 9285 :info :write [5 25] :RequestCanceledException INFO [2019-12-17 12:33:52,357] jepsen worker 68 - jepsen.util 9878 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:33:52,357] jepsen worker 68 - jepsen.util 9878 :invoke :read [22 nil] INFO [2019-12-17 12:33:52,365] jepsen worker 15 - jepsen.util 9375 :invoke :read [5 nil] INFO [2019-12-17 12:33:52,382] jepsen worker 63 - jepsen.util 6633 :ok :write [21 40] INFO [2019-12-17 12:33:52,382] jepsen worker 63 - jepsen.util 6633 :invoke :write [21 20] INFO [2019-12-17 12:33:52,395] jepsen worker 60 - jepsen.util 11310 :info :cas [20 [25 16]] :RequestCanceledException INFO [2019-12-17 12:33:52,395] jepsen worker 41 - jepsen.util 10841 :info :cas [13 [24 0]] :RequestCanceledException INFO [2019-12-17 12:33:52,396] jepsen worker 58 - jepsen.util 11218 :info :write [19 21] :RequestCanceledException INFO [2019-12-17 12:33:52,404] jepsen worker 58 - jepsen.util 11308 :invoke :read [19 nil] INFO [2019-12-17 12:33:52,405] jepsen worker 60 - jepsen.util 11400 :invoke :write [20 22] INFO [2019-12-17 12:33:52,405] jepsen worker 72 - jepsen.util 7452 :fail :cas [24 [26 44]] :ValueNotSwapFrom INFO [2019-12-17 12:33:52,406] jepsen worker 72 - jepsen.util 7452 :invoke :write [24 13] INFO [2019-12-17 12:33:52,407] jepsen worker 27 - jepsen.util 7677 :ok :read [9 10] INFO [2019-12-17 12:33:52,407] jepsen worker 27 - jepsen.util 7677 :invoke :cas [9 [21 9]] INFO [2019-12-17 12:33:52,408] jepsen worker 20 - jepsen.util 7310 :ok :read [6 49] INFO [2019-12-17 12:33:52,408] jepsen worker 12 - jepsen.util 8202 :ok :read [4 28] INFO [2019-12-17 12:33:52,409] jepsen worker 20 - jepsen.util 7310 :invoke :cas [6 [12 20]] INFO [2019-12-17 12:33:52,409] jepsen worker 12 - jepsen.util 8202 :invoke :read [4 nil] INFO [2019-12-17 12:33:52,413] jepsen worker 41 - jepsen.util 10931 :invoke :cas [13 [18 28]] INFO [2019-12-17 12:33:52,413] jepsen worker 33 - jepsen.util 10113 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:33:52,413] jepsen worker 22 - jepsen.util 10192 :info :write [7 38] :RequestCanceledException INFO [2019-12-17 12:33:52,413] jepsen worker 33 - jepsen.util 10113 :invoke :write [11 45] INFO [2019-12-17 12:33:52,413] jepsen worker 24 - jepsen.util 9114 :info :cas [8 [7 6]] :RequestCanceledException INFO [2019-12-17 12:33:52,423] jepsen worker 24 - jepsen.util 9204 :invoke :write [8 11] INFO [2019-12-17 12:33:52,423] jepsen worker 22 - jepsen.util 10282 :invoke :write [7 8] INFO [2019-12-17 12:33:52,429] jepsen worker 23 - jepsen.util 10373 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:33:52,429] jepsen worker 86 - jepsen.util 10976 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:33:52,429] jepsen worker 67 - jepsen.util 10327 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:33:52,429] jepsen worker 23 - jepsen.util 10373 :invoke :read [7 nil] INFO [2019-12-17 12:33:52,429] jepsen worker 86 - jepsen.util 10976 :invoke :write [28 13] INFO [2019-12-17 12:33:52,429] jepsen worker 67 - jepsen.util 10327 :invoke :cas [22 [12 5]] INFO [2019-12-17 12:33:52,442] jepsen worker 10 - jepsen.util 7660 :fail :cas [3 [48 28]] :ValueNotSwapFrom INFO [2019-12-17 12:33:52,442] jepsen worker 10 - jepsen.util 7660 :invoke :write [3 25] INFO [2019-12-17 12:33:52,443] jepsen worker 25 - jepsen.util 10825 :info :cas [8 [16 0]] :RequestCanceledException INFO [2019-12-17 12:33:52,443] jepsen worker 59 - jepsen.util 10049 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:33:52,443] jepsen worker 59 - jepsen.util 10049 :invoke :cas [19 [14 35]] INFO [2019-12-17 12:33:52,451] jepsen worker 25 - jepsen.util 10915 :invoke :cas [8 [37 34]] INFO [2019-12-17 12:33:52,462] jepsen worker 19 - jepsen.util 7849 :fail :cas [6 [14 43]] :ValueNotSwapFrom INFO [2019-12-17 12:33:52,463] jepsen worker 19 - jepsen.util 7849 :invoke :cas [6 [47 6]] INFO [2019-12-17 12:33:52,466] jepsen worker 69 - jepsen.util 7809 :fail :cas [23 [2 29]] :ValueNotSwapFrom INFO [2019-12-17 12:33:52,466] jepsen worker 69 - jepsen.util 7809 :invoke :cas [23 [15 38]] INFO [2019-12-17 12:33:52,515] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44354,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:33:52,515] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44354,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:33:52,515] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44358,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:33:52,515] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44358,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:33:52,522] jepsen worker 20 - jepsen.util 7310 :fail :cas [6 [12 20]] :ValueNotSwapFrom INFO [2019-12-17 12:33:52,522] jepsen worker 20 - jepsen.util 7310 :invoke :read [6 nil] INFO [2019-12-17 12:33:52,528] jepsen worker 4 - jepsen.util 7024 :info :write [1 13] :RequestCanceledException INFO [2019-12-17 12:33:52,530] jepsen worker 18 - jepsen.util 7848 :info :write [6 48] :RequestCanceledException INFO [2019-12-17 12:33:52,532] jepsen worker 4 - jepsen.util 7114 :invoke :cas [1 [36 23]] INFO [2019-12-17 12:33:52,533] jepsen worker 18 - jepsen.util 7938 :invoke :read [6 nil] INFO [2019-12-17 12:33:52,539] jepsen worker 19 - jepsen.util 7849 :fail :cas [6 [47 6]] :ValueNotSwapFrom INFO [2019-12-17 12:33:52,539] jepsen worker 19 - jepsen.util 7849 :invoke :cas [6 [8 2]] INFO [2019-12-17 12:33:52,541] jepsen worker 63 - jepsen.util 6633 :ok :write [21 20] INFO [2019-12-17 12:33:52,541] jepsen worker 63 - jepsen.util 6633 :invoke :read [21 nil] INFO [2019-12-17 12:33:52,542] jepsen worker 82 - jepsen.util 11062 :info :write [27 20] :RequestCanceledException INFO [2019-12-17 12:33:52,545] jepsen worker 82 - jepsen.util 11152 :invoke :cas [27 [6 20]] INFO [2019-12-17 12:33:52,560] jepsen worker 1 - jepsen.util 10531 :info :write [0 46] :RequestCanceledException INFO [2019-12-17 12:33:52,563] jepsen worker 1 - jepsen.util 10621 :invoke :write [0 26] INFO [2019-12-17 12:33:52,578] jepsen worker 37 - jepsen.util 7147 :fail :cas [12 [43 6]] :ValueNotSwapFrom INFO [2019-12-17 12:33:52,579] jepsen worker 37 - jepsen.util 7147 :invoke :write [12 14] INFO [2019-12-17 12:33:52,581] jepsen worker 27 - jepsen.util 7677 :fail :cas [9 [21 9]] :ValueNotSwapFrom INFO [2019-12-17 12:33:52,582] jepsen worker 27 - jepsen.util 7677 :invoke :cas [9 [15 26]] INFO [2019-12-17 12:33:52,613] jepsen worker 6 - jepsen.util 10986 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:33:52,614] jepsen worker 6 - jepsen.util 10986 :invoke :cas [2 [42 17]] INFO [2019-12-17 12:33:52,616] jepsen worker 63 - jepsen.util 6633 :ok :read [21 20] INFO [2019-12-17 12:33:52,616] jepsen worker 63 - jepsen.util 6633 :invoke :read [21 nil] INFO [2019-12-17 12:33:52,627] jepsen worker 27 - jepsen.util 7677 :fail :cas [9 [15 26]] :ValueNotSwapFrom INFO [2019-12-17 12:33:52,628] jepsen worker 27 - jepsen.util 7677 :invoke :write [9 7] INFO [2019-12-17 12:33:52,630] jepsen worker 87 - jepsen.util 7107 :fail :cas [29 [11 6]] :ValueNotSwapFrom INFO [2019-12-17 12:33:52,631] jepsen worker 87 - jepsen.util 7107 :invoke :cas [29 [49 0]] INFO [2019-12-17 12:33:52,659] jepsen worker 63 - jepsen.util 6633 :ok :read [21 20] INFO [2019-12-17 12:33:52,659] jepsen worker 63 - jepsen.util 6633 :invoke :read [21 nil] INFO [2019-12-17 12:33:52,683] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44385,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:33:52,683] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44385,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:33:52,683] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44388,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:33:52,683] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44388,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:33:52,683] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44389,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:33:52,683] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44389,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:33:52,695] jepsen worker 57 - jepsen.util 10317 :info :cas [19 [13 17]] :RequestCanceledException INFO [2019-12-17 12:33:52,697] jepsen worker 57 - jepsen.util 10407 :invoke :cas [19 [21 37]] INFO [2019-12-17 12:33:52,703] jepsen worker 63 - jepsen.util 6633 :ok :read [21 20] INFO [2019-12-17 12:33:52,703] jepsen worker 63 - jepsen.util 6633 :invoke :write [21 40] INFO [2019-12-17 12:33:52,707] jepsen worker 80 - jepsen.util 7550 :fail :cas [26 [35 32]] :ValueNotSwapFrom INFO [2019-12-17 12:33:52,708] jepsen worker 80 - jepsen.util 7550 :invoke :read [26 nil] INFO [2019-12-17 12:33:52,719] jepsen worker 27 - jepsen.util 7677 :ok :write [9 7] INFO [2019-12-17 12:33:52,719] jepsen worker 27 - jepsen.util 7677 :invoke :read [9 nil] INFO [2019-12-17 12:33:52,776] jepsen worker 27 - jepsen.util 7677 :ok :read [9 7] INFO [2019-12-17 12:33:52,776] jepsen worker 27 - jepsen.util 7677 :invoke :write [9 25] INFO [2019-12-17 12:33:52,779] jepsen worker 63 - jepsen.util 6633 :ok :write [21 40] INFO [2019-12-17 12:33:52,779] jepsen worker 63 - jepsen.util 6633 :invoke :write [21 30] INFO [2019-12-17 12:33:52,795] jepsen worker 84 - jepsen.util 9984 :info :write [28 16] :RequestCanceledException INFO [2019-12-17 12:33:52,799] jepsen worker 84 - jepsen.util 10074 :invoke :write [28 39] INFO [2019-12-17 12:33:52,813] jepsen worker 48 - jepsen.util 9408 :info :cas [16 [11 38]] :RequestCanceledException INFO [2019-12-17 12:33:52,818] jepsen worker 48 - jepsen.util 9498 :invoke :cas [16 [7 20]] INFO [2019-12-17 12:33:52,828] jepsen worker 17 - jepsen.util 10547 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:33:52,829] jepsen worker 17 - jepsen.util 10547 :invoke :read [5 nil] INFO [2019-12-17 12:33:52,842] jepsen worker 81 - jepsen.util 10251 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:33:52,842] jepsen worker 81 - jepsen.util 10251 :invoke :read [27 nil] INFO [2019-12-17 12:33:52,860] jepsen worker 76 - jepsen.util 10336 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:33:52,860] jepsen worker 76 - jepsen.util 10336 :invoke :read [25 nil] INFO [2019-12-17 12:33:52,863] jepsen worker 79 - jepsen.util 7729 :fail :cas [26 [22 4]] :ValueNotSwapFrom INFO [2019-12-17 12:33:52,864] jepsen worker 79 - jepsen.util 7729 :invoke :write [26 35] INFO [2019-12-17 12:33:52,878] jepsen worker 38 - jepsen.util 7958 :ok :read [12 47] INFO [2019-12-17 12:33:52,878] jepsen worker 38 - jepsen.util 7958 :invoke :read [12 nil] INFO [2019-12-17 12:33:52,892] jepsen worker 80 - jepsen.util 7550 :ok :read [26 15] INFO [2019-12-17 12:33:52,892] jepsen worker 80 - jepsen.util 7550 :invoke :read [26 nil] INFO [2019-12-17 12:33:52,911] jepsen worker 63 - jepsen.util 6633 :ok :write [21 30] INFO [2019-12-17 12:33:52,911] jepsen worker 63 - jepsen.util 6633 :invoke :read [21 nil] INFO [2019-12-17 12:33:52,927] jepsen worker 27 - jepsen.util 7677 :ok :write [9 25] INFO [2019-12-17 12:33:52,928] jepsen worker 27 - jepsen.util 7677 :invoke :cas [9 [24 20]] INFO [2019-12-17 12:33:52,929] jepsen worker 52 - jepsen.util 7882 :info :write [17 46] :RequestCanceledException INFO [2019-12-17 12:33:52,931] jepsen worker 52 - jepsen.util 7972 :invoke :cas [17 [15 43]] INFO [2019-12-17 12:33:52,941] jepsen worker 9 - jepsen.util 7929 :ok :write [3 28] INFO [2019-12-17 12:33:52,941] jepsen worker 45 - jepsen.util 7695 :ok :write [15 38] INFO [2019-12-17 12:33:52,941] jepsen worker 9 - jepsen.util 7929 :invoke :write [3 2] INFO [2019-12-17 12:33:52,941] jepsen worker 45 - jepsen.util 7695 :invoke :read [15 nil] INFO [2019-12-17 12:33:52,950] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44403,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:33:52,950] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44403,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:33:52,950] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44406,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:33:52,951] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44406,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:33:52,964] jepsen worker 73 - jepsen.util 7363 :ok :write [24 10] INFO [2019-12-17 12:33:52,965] jepsen worker 73 - jepsen.util 7363 :invoke :write [24 18] INFO [2019-12-17 12:33:53,053] jepsen worker 63 - jepsen.util 6633 :ok :read [21 30] INFO [2019-12-17 12:33:53,053] jepsen worker 14 - jepsen.util 7934 :ok :write [4 14] INFO [2019-12-17 12:33:53,053] jepsen worker 63 - jepsen.util 6633 :invoke :read [21 nil] INFO [2019-12-17 12:33:53,053] jepsen worker 40 - jepsen.util 10840 :info :cas [13 [38 23]] :RequestCanceledException INFO [2019-12-17 12:33:53,053] jepsen worker 49 - jepsen.util 9949 :info :cas [16 [23 7]] :RequestCanceledException INFO [2019-12-17 12:33:53,053] jepsen worker 14 - jepsen.util 7934 :invoke :write [4 10] INFO [2019-12-17 12:33:53,053] jepsen worker 7 - jepsen.util 10447 :info :write [2 6] :RequestCanceledException INFO [2019-12-17 12:33:53,053] jepsen worker 61 - jepsen.util 11041 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:33:53,054] jepsen worker 39 - jepsen.util 9939 :info :cas [13 [45 43]] :RequestCanceledException INFO [2019-12-17 12:33:53,054] jepsen worker 61 - jepsen.util 11041 :invoke :cas [20 [18 19]] INFO [2019-12-17 12:33:53,055] jepsen worker 16 - jepsen.util 11626 :info :write [5 13] :RequestCanceledException INFO [2019-12-17 12:33:53,056] jepsen worker 35 - jepsen.util 10205 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:33:53,056] jepsen worker 35 - jepsen.util 10205 :invoke :cas [11 [32 4]] INFO [2019-12-17 12:33:53,063] jepsen worker 49 - jepsen.util 10039 :invoke :read [16 nil] INFO [2019-12-17 12:33:53,065] jepsen worker 16 - jepsen.util 11716 :invoke :cas [5 [38 32]] INFO [2019-12-17 12:33:53,067] jepsen worker 7 - jepsen.util 10537 :invoke :write [2 34] INFO [2019-12-17 12:33:53,068] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44420,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:33:53,068] jepsen worker 39 - jepsen.util 10029 :invoke :write [13 33] INFO [2019-12-17 12:33:53,072] jepsen worker 40 - jepsen.util 10930 :invoke :write [13 15] INFO [2019-12-17 12:33:53,095] jepsen worker 89 - jepsen.util 7469 :info :write [29 45] :RequestCanceledException INFO [2019-12-17 12:33:53,095] jepsen worker 3 - jepsen.util 7653 :info :write [1 31] :RequestCanceledException INFO [2019-12-17 12:33:53,108] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44420,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:33:53,127] jepsen worker 44 - jepsen.util 11024 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:33:53,127] jepsen worker 85 - jepsen.util 11245 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:33:53,127] jepsen worker 8 - jepsen.util 10268 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:33:53,141] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44423,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:33:53,141] jepsen worker 44 - jepsen.util 11024 :invoke :read [14 nil] INFO [2019-12-17 12:33:53,141] jepsen worker 8 - jepsen.util 10268 :invoke :cas [2 [11 6]] INFO [2019-12-17 12:33:53,142] jepsen worker 85 - jepsen.util 11245 :invoke :cas [28 [10 6]] INFO [2019-12-17 12:33:53,143] jepsen worker 88 - jepsen.util 7648 :info :write [29 1] :RequestCanceledException INFO [2019-12-17 12:33:53,147] jepsen worker 89 - jepsen.util 7559 :invoke :write [29 1] INFO [2019-12-17 12:33:53,151] jepsen worker 3 - jepsen.util 7743 :invoke :cas [1 [35 8]] INFO [2019-12-17 12:33:53,155] jepsen worker 29 - jepsen.util 6239 :ok :read [9 25] INFO [2019-12-17 12:33:53,165] jepsen worker 65 - jepsen.util 6365 :ok :read [21 30] INFO [2019-12-17 12:33:53,166] jepsen worker 18 - jepsen.util 7938 :ok :read [6 49] INFO [2019-12-17 12:33:53,168] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44423,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:33:53,169] jepsen worker 12 - jepsen.util 8202 :ok :read [4 28] INFO [2019-12-17 12:33:53,169] jepsen worker 29 - jepsen.util 6239 :invoke :write [9 32] INFO [2019-12-17 12:33:53,169] jepsen worker 65 - jepsen.util 6365 :invoke :read [21 nil] INFO [2019-12-17 12:33:53,169] jepsen worker 18 - jepsen.util 7938 :invoke :write [6 44] INFO [2019-12-17 12:33:53,169] jepsen worker 12 - jepsen.util 8202 :invoke :write [4 7] INFO [2019-12-17 12:33:53,175] jepsen worker 27 - jepsen.util 7677 :fail :cas [9 [24 20]] :ValueNotSwapFrom INFO [2019-12-17 12:33:53,177] jepsen worker 88 - jepsen.util 7738 :invoke :read [29 nil] INFO [2019-12-17 12:33:53,188] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:33:53,213] jepsen worker 4 - jepsen.util 7114 :fail :cas [1 [36 23]] :ValueNotSwapFrom INFO [2019-12-17 12:33:53,215] jepsen worker 27 - jepsen.util 7677 :invoke :cas [9 [30 3]] INFO [2019-12-17 12:33:53,215] jepsen nemesis - couchbase.util Rebalance check count: 340 INFO [2019-12-17 12:33:53,215] jepsen worker 4 - jepsen.util 7114 :invoke :cas [1 [23 32]] INFO [2019-12-17 12:33:53,215] jepsen nemesis - couchbase.util Rebalance stuck count: 321 INFO [2019-12-17 12:33:53,216] jepsen worker 45 - jepsen.util 7695 :ok :read [15 38] INFO [2019-12-17 12:33:53,217] jepsen worker 45 - jepsen.util 7695 :invoke :read [15 nil] INFO [2019-12-17 12:33:53,225] jepsen worker 46 - jepsen.util 7966 :fail :cas [15 [43 35]] :ValueNotSwapFrom INFO [2019-12-17 12:33:53,238] jepsen worker 63 - jepsen.util 6633 :ok :read [21 30] INFO [2019-12-17 12:33:53,259] jepsen worker 51 - jepsen.util 7341 :info :write [17 22] :RequestCanceledException INFO [2019-12-17 12:33:53,260] jepsen worker 46 - jepsen.util 7966 :invoke :read [15 nil] INFO [2019-12-17 12:33:53,260] jepsen worker 63 - jepsen.util 6633 :invoke :cas [21 [42 39]] INFO [2019-12-17 12:33:53,263] jepsen worker 51 - jepsen.util 7431 :invoke :read [17 nil] INFO [2019-12-17 12:33:53,274] jepsen worker 87 - jepsen.util 7107 :fail :cas [29 [49 0]] :ValueNotSwapFrom INFO [2019-12-17 12:33:53,281] jepsen worker 38 - jepsen.util 7958 :ok :read [12 47] INFO [2019-12-17 12:33:53,283] jepsen worker 78 - jepsen.util 7728 :ok :write [26 28] INFO [2019-12-17 12:33:53,286] jepsen worker 72 - jepsen.util 7452 :ok :write [24 13] INFO [2019-12-17 12:33:53,288] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44446,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:33:53,289] jepsen worker 87 - jepsen.util 7107 :invoke :cas [29 [6 39]] INFO [2019-12-17 12:33:53,289] jepsen worker 38 - jepsen.util 7958 :invoke :cas [12 [0 27]] INFO [2019-12-17 12:33:53,289] jepsen worker 78 - jepsen.util 7728 :invoke :cas [26 [45 23]] INFO [2019-12-17 12:33:53,289] jepsen worker 72 - jepsen.util 7452 :invoke :write [24 30] INFO [2019-12-17 12:33:53,289] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44446,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:33:53,290] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44438,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:33:53,293] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44438,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:33:53,294] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44449,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:33:53,294] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44449,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:33:53,301] jepsen worker 13 - jepsen.util 7303 :ok :write [4 34] INFO [2019-12-17 12:33:53,301] jepsen worker 13 - jepsen.util 7303 :invoke :read [4 nil] INFO [2019-12-17 12:33:53,313] jepsen worker 70 - jepsen.util 7990 :info :write [23 42] :RequestCanceledException INFO [2019-12-17 12:33:53,315] jepsen worker 70 - jepsen.util 8080 :invoke :read [23 nil] INFO [2019-12-17 12:33:53,329] jepsen worker 56 - jepsen.util 8696 :info :write [18 3] :RequestCanceledException INFO [2019-12-17 12:33:53,331] jepsen worker 56 - jepsen.util 8786 :invoke :cas [18 [41 14]] INFO [2019-12-17 12:33:53,342] jepsen worker 45 - jepsen.util 7695 :ok :read [15 38] INFO [2019-12-17 12:33:53,343] jepsen worker 45 - jepsen.util 7695 :invoke :write [15 25] INFO [2019-12-17 12:33:53,346] jepsen worker 27 - jepsen.util 7677 :fail :cas [9 [30 3]] :ValueNotSwapFrom INFO [2019-12-17 12:33:53,346] jepsen worker 27 - jepsen.util 7677 :invoke :write [9 40] INFO [2019-12-17 12:33:53,364] jepsen worker 88 - jepsen.util 7738 :ok :read [29 13] INFO [2019-12-17 12:33:53,364] jepsen worker 88 - jepsen.util 7738 :invoke :read [29 nil] INFO [2019-12-17 12:33:53,373] jepsen worker 63 - jepsen.util 6633 :fail :cas [21 [42 39]] :ValueNotSwapFrom INFO [2019-12-17 12:33:53,373] jepsen worker 63 - jepsen.util 6633 :invoke :read [21 nil] INFO [2019-12-17 12:33:53,395] jepsen worker 79 - jepsen.util 7729 :ok :write [26 35] INFO [2019-12-17 12:33:53,395] jepsen worker 79 - jepsen.util 7729 :invoke :write [26 4] INFO [2019-12-17 12:33:53,413] jepsen worker 36 - jepsen.util 7866 :info :write [12 6] :RequestCanceledException INFO [2019-12-17 12:33:53,418] jepsen worker 36 - jepsen.util 7956 :invoke :read [12 nil] INFO [2019-12-17 12:33:53,452] jepsen worker 71 - jepsen.util 8261 :fail :cas [23 [37 44]] :ValueNotSwapFrom INFO [2019-12-17 12:33:53,452] jepsen worker 71 - jepsen.util 8261 :invoke :write [23 38] INFO [2019-12-17 12:33:53,476] jepsen worker 69 - jepsen.util 7809 :fail :cas [23 [15 38]] :ValueNotSwapFrom INFO [2019-12-17 12:33:53,477] jepsen worker 69 - jepsen.util 7809 :invoke :cas [23 [14 0]] INFO [2019-12-17 12:33:53,487] jepsen worker 63 - jepsen.util 6633 :ok :read [21 30] INFO [2019-12-17 12:33:53,488] jepsen worker 63 - jepsen.util 6633 :invoke :read [21 nil] INFO [2019-12-17 12:33:53,488] jepsen worker 5 - jepsen.util 7205 :ok :write [1 18] INFO [2019-12-17 12:33:53,488] jepsen worker 55 - jepsen.util 8515 :ok :write [18 26] INFO [2019-12-17 12:33:53,488] jepsen worker 5 - jepsen.util 7205 :invoke :write [1 48] INFO [2019-12-17 12:33:53,488] jepsen worker 55 - jepsen.util 8515 :invoke :read [18 nil] INFO [2019-12-17 12:33:53,494] jepsen worker 2 - jepsen.util 10082 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:33:53,494] jepsen worker 2 - jepsen.util 10082 :invoke :write [0 35] INFO [2019-12-17 12:33:53,505] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44476,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:33:53,506] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44476,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:33:53,506] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44478,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:33:53,506] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44478,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:33:53,526] jepsen worker 52 - jepsen.util 7972 :fail :cas [17 [15 43]] :ValueNotSwapFrom INFO [2019-12-17 12:33:53,526] jepsen worker 52 - jepsen.util 7972 :invoke :cas [17 [20 39]] INFO [2019-12-17 12:33:53,528] jepsen worker 28 - jepsen.util 6058 :info :write [9 35] :RequestCanceledException INFO [2019-12-17 12:33:53,535] jepsen worker 28 - jepsen.util 6148 :invoke :write [9 26] INFO [2019-12-17 12:33:53,541] jepsen worker 32 - jepsen.util 7592 :fail :read [10 nil] :RequestTimeoutException INFO [2019-12-17 12:33:53,541] jepsen worker 32 - jepsen.util 7592 :invoke :write [10 20] INFO [2019-12-17 12:33:53,556] jepsen worker 31 - jepsen.util 6961 :info :write [10 16] :RequestCanceledException INFO [2019-12-17 12:33:53,567] jepsen worker 31 - jepsen.util 7051 :invoke :read [10 nil] INFO [2019-12-17 12:33:53,606] jepsen worker 64 - jepsen.util 6094 :info :write [21 3] :RequestCanceledException INFO [2019-12-17 12:33:53,610] jepsen worker 64 - jepsen.util 6184 :invoke :cas [21 [9 41]] INFO [2019-12-17 12:33:53,628] jepsen worker 37 - jepsen.util 7147 :ok :write [12 14] INFO [2019-12-17 12:33:53,629] jepsen worker 37 - jepsen.util 7147 :invoke :read [12 nil] INFO [2019-12-17 12:33:53,653] jepsen worker 89 - jepsen.util 7559 :ok :write [29 1] INFO [2019-12-17 12:33:53,653] jepsen worker 89 - jepsen.util 7559 :invoke :read [29 nil] INFO [2019-12-17 12:33:53,656] jepsen worker 27 - jepsen.util 7677 :ok :write [9 40] INFO [2019-12-17 12:33:53,656] jepsen worker 27 - jepsen.util 7677 :invoke :write [9 33] INFO [2019-12-17 12:33:53,684] jepsen worker 18 - jepsen.util 7938 :ok :write [6 44] INFO [2019-12-17 12:33:53,684] jepsen worker 18 - jepsen.util 7938 :invoke :read [6 nil] INFO [2019-12-17 12:33:53,685] jepsen worker 9 - jepsen.util 7929 :ok :write [3 2] INFO [2019-12-17 12:33:53,685] jepsen worker 9 - jepsen.util 7929 :invoke :cas [3 [21 6]] INFO [2019-12-17 12:33:53,699] jepsen worker 73 - jepsen.util 7363 :ok :write [24 18] INFO [2019-12-17 12:33:53,700] jepsen worker 73 - jepsen.util 7363 :invoke :write [24 33] INFO [2019-12-17 12:33:53,714] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44492,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:33:53,716] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44492,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:33:53,719] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44495,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:33:53,724] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44495,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:33:53,726] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44497,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:33:53,728] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44497,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:33:53,751] jepsen worker 28 - jepsen.util 6148 :ok :write [9 26] INFO [2019-12-17 12:33:53,752] jepsen worker 28 - jepsen.util 6148 :invoke :read [9 nil] INFO [2019-12-17 12:33:53,756] jepsen worker 53 - jepsen.util 7163 :info :write [17 23] :RequestCanceledException INFO [2019-12-17 12:33:53,768] jepsen worker 53 - jepsen.util 7253 :invoke :read [17 nil] INFO [2019-12-17 12:33:53,842] jepsen worker 30 - jepsen.util 7590 :info :cas [10 [7 43]] :RequestCanceledException INFO [2019-12-17 12:33:53,851] jepsen worker 30 - jepsen.util 7680 :invoke :write [10 39] INFO [2019-12-17 12:33:53,905] jepsen worker 54 - jepsen.util 8604 :ok :write [18 47] INFO [2019-12-17 12:33:53,909] jepsen worker 54 - jepsen.util 8604 :invoke :cas [18 [27 22]] INFO [2019-12-17 12:33:53,920] jepsen worker 12 - jepsen.util 8202 :ok :write [4 7] INFO [2019-12-17 12:33:53,920] jepsen worker 12 - jepsen.util 8202 :invoke :cas [4 [5 46]] INFO [2019-12-17 12:33:53,935] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44503,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:33:53,936] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44503,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:33:53,936] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44508,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:33:53,937] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44508,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:33:53,984] jepsen worker 45 - jepsen.util 7695 :ok :write [15 25] INFO [2019-12-17 12:33:53,985] jepsen worker 45 - jepsen.util 7695 :invoke :read [15 nil] INFO [2019-12-17 12:33:54,047] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44514,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:33:54,048] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44514,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:33:54,071] jepsen worker 38 - jepsen.util 7958 :fail :cas [12 [0 27]] :ValueNotSwapFrom INFO [2019-12-17 12:33:54,072] jepsen worker 38 - jepsen.util 7958 :invoke :write [12 44] INFO [2019-12-17 12:33:54,098] jepsen worker 51 - jepsen.util 7431 :ok :read [17 14] INFO [2019-12-17 12:33:54,098] jepsen worker 51 - jepsen.util 7431 :invoke :cas [17 [28 21]] INFO [2019-12-17 12:33:54,102] jepsen worker 72 - jepsen.util 7452 :ok :write [24 30] INFO [2019-12-17 12:33:54,103] jepsen worker 72 - jepsen.util 7452 :invoke :cas [24 [18 24]] INFO [2019-12-17 12:33:54,123] jepsen worker 31 - jepsen.util 7051 :ok :read [10 21] INFO [2019-12-17 12:33:54,123] jepsen worker 31 - jepsen.util 7051 :invoke :write [10 32] INFO [2019-12-17 12:33:54,133] jepsen worker 63 - jepsen.util 6633 :ok :read [21 30] INFO [2019-12-17 12:33:54,134] jepsen worker 63 - jepsen.util 6633 :invoke :cas [21 [47 46]] INFO [2019-12-17 12:33:54,143] jepsen worker 34 - jepsen.util 10024 :info :cas [11 [4 22]] :RequestCanceledException INFO [2019-12-17 12:33:54,219] jepsen worker 34 - jepsen.util 10114 :invoke :read [11 nil] INFO [2019-12-17 12:33:54,246] jepsen worker 42 - jepsen.util 11112 :info :write [14 19] :RequestCanceledException INFO [2019-12-17 12:33:54,249] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:33:54,249] jepsen nemesis - couchbase.util Rebalance check count: 341 INFO [2019-12-17 12:33:54,250] jepsen nemesis - couchbase.util Rebalance stuck count: 322 INFO [2019-12-17 12:33:54,254] jepsen worker 13 - jepsen.util 7303 :ok :read [4 34] INFO [2019-12-17 12:33:54,254] jepsen worker 13 - jepsen.util 7303 :invoke :write [4 35] INFO [2019-12-17 12:33:54,263] jepsen worker 42 - jepsen.util 11202 :invoke :cas [14 [8 28]] INFO [2019-12-17 12:33:54,288] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44523,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:33:54,288] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44523,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:33:54,289] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44524,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:33:54,289] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44524,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:33:54,289] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44530,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:33:54,289] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44528,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:33:54,289] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44530,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:33:54,290] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44528,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:33:54,290] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44534,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:33:54,291] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44534,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:33:54,291] jepsen worker 11 - jepsen.util 7841 :fail :read [3 nil] :RequestTimeoutException INFO [2019-12-17 12:33:54,291] jepsen worker 11 - jepsen.util 7841 :invoke :write [3 21] INFO [2019-12-17 12:33:54,291] jepsen worker 26 - jepsen.util 10826 :info :write [8 46] :RequestCanceledException INFO [2019-12-17 12:33:54,314] jepsen worker 75 - jepsen.util 9975 :info :cas [25 [42 21]] :RequestCanceledException INFO [2019-12-17 12:33:54,329] jepsen worker 43 - jepsen.util 9943 :info :cas [14 [17 45]] :RequestCanceledException INFO [2019-12-17 12:33:54,329] jepsen worker 77 - jepsen.util 10697 :info :write [25 22] :RequestCanceledException INFO [2019-12-17 12:33:54,342] jepsen worker 66 - jepsen.util 10236 :info :write [22 16] :RequestCanceledException INFO [2019-12-17 12:33:54,352] jepsen worker 26 - jepsen.util 10916 :invoke :write [8 4] INFO [2019-12-17 12:33:54,359] jepsen worker 50 - jepsen.util 11210 :info :cas [16 [22 45]] :RequestCanceledException INFO [2019-12-17 12:33:54,392] jepsen worker 27 - jepsen.util 7677 :ok :write [9 33] INFO [2019-12-17 12:33:54,399] jepsen worker 21 - jepsen.util 9831 :info :cas [7 [38 22]] :RequestCanceledException INFO [2019-12-17 12:33:54,414] jepsen worker 15 - jepsen.util 9375 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:33:54,414] jepsen worker 0 - jepsen.util 10890 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:33:54,421] jepsen worker 50 - jepsen.util 11300 :invoke :write [16 38] INFO [2019-12-17 12:33:54,422] jepsen worker 55 - jepsen.util 8515 :ok :read [18 26] INFO [2019-12-17 12:33:54,422] jepsen worker 43 - jepsen.util 10033 :invoke :read [14 nil] INFO [2019-12-17 12:33:54,425] jepsen worker 75 - jepsen.util 10065 :invoke :cas [25 [44 28]] INFO [2019-12-17 12:33:54,427] jepsen worker 66 - jepsen.util 10326 :invoke :write [22 36] INFO [2019-12-17 12:33:54,428] jepsen worker 62 - jepsen.util 10502 :info :cas [20 [27 12]] :RequestCanceledException INFO [2019-12-17 12:33:54,429] jepsen worker 58 - jepsen.util 11308 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:33:54,429] jepsen worker 83 - jepsen.util 10793 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:33:54,430] jepsen worker 77 - jepsen.util 10787 :invoke :cas [25 [47 31]] INFO [2019-12-17 12:33:54,439] jepsen worker 52 - jepsen.util 7972 :fail :cas [17 [20 39]] :ValueNotSwapFrom INFO [2019-12-17 12:33:54,451] jepsen worker 27 - jepsen.util 7677 :invoke :write [9 26] INFO [2019-12-17 12:33:54,452] jepsen worker 15 - jepsen.util 9375 :invoke :write [5 42] INFO [2019-12-17 12:33:54,452] jepsen worker 0 - jepsen.util 10890 :invoke :read [0 nil] INFO [2019-12-17 12:33:54,452] jepsen worker 55 - jepsen.util 8515 :invoke :write [18 35] INFO [2019-12-17 12:33:54,454] jepsen worker 58 - jepsen.util 11308 :invoke :cas [19 [20 11]] INFO [2019-12-17 12:33:54,454] jepsen worker 52 - jepsen.util 7972 :invoke :write [17 17] INFO [2019-12-17 12:33:54,454] jepsen worker 83 - jepsen.util 10793 :invoke :cas [27 [39 44]] INFO [2019-12-17 12:33:54,460] jepsen worker 68 - jepsen.util 9878 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:33:54,461] jepsen worker 68 - jepsen.util 9878 :invoke :write [22 7] INFO [2019-12-17 12:33:54,462] jepsen worker 47 - jepsen.util 7967 :info :write [15 12] :RequestCanceledException INFO [2019-12-17 12:33:54,464] jepsen worker 41 - jepsen.util 10931 :info :cas [13 [18 28]] :RequestCanceledException INFO [2019-12-17 12:33:54,496] jepsen worker 22 - jepsen.util 10282 :info :write [7 8] :RequestCanceledException INFO [2019-12-17 12:33:54,519] jepsen worker 25 - jepsen.util 10915 :info :cas [8 [37 34]] :RequestCanceledException INFO [2019-12-17 12:33:54,524] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44542,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:33:54,534] jepsen worker 30 - jepsen.util 7680 :ok :write [10 39] INFO [2019-12-17 12:33:54,536] jepsen worker 23 - jepsen.util 10373 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:33:54,536] jepsen worker 33 - jepsen.util 10113 :info :write [11 45] :RequestCanceledException INFO [2019-12-17 12:33:54,536] jepsen worker 30 - jepsen.util 7680 :invoke :read [10 nil] INFO [2019-12-17 12:33:54,537] jepsen worker 23 - jepsen.util 10373 :invoke :cas [7 [20 16]] INFO [2019-12-17 12:33:54,538] jepsen worker 28 - jepsen.util 6148 :ok :read [9 26] INFO [2019-12-17 12:33:54,539] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44542,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:33:54,543] jepsen worker 28 - jepsen.util 6148 :invoke :write [9 9] INFO [2019-12-17 12:33:54,548] jepsen worker 59 - jepsen.util 10049 :info :cas [19 [14 35]] :RequestCanceledException INFO [2019-12-17 12:33:54,553] jepsen worker 67 - jepsen.util 10327 :info :cas [22 [12 5]] :RequestCanceledException INFO [2019-12-17 12:33:54,553] jepsen worker 24 - jepsen.util 9204 :info :write [8 11] :RequestCanceledException INFO [2019-12-17 12:33:54,554] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44544,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:33:54,563] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44544,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:33:54,570] jepsen worker 60 - jepsen.util 11400 :info :write [20 22] :RequestCanceledException INFO [2019-12-17 12:33:54,573] jepsen worker 86 - jepsen.util 10976 :info :write [28 13] :RequestCanceledException INFO [2019-12-17 12:33:54,574] jepsen worker 82 - jepsen.util 11152 :info :cas [27 [6 20]] :RequestCanceledException INFO [2019-12-17 12:33:54,575] jepsen worker 41 - jepsen.util 11021 :invoke :read [13 nil] INFO [2019-12-17 12:33:54,575] jepsen worker 74 - jepsen.util 7904 :info :cas [24 [6 8]] :RequestCanceledException INFO [2019-12-17 12:33:54,577] jepsen worker 10 - jepsen.util 7660 :info :write [3 25] :RequestCanceledException INFO [2019-12-17 12:33:54,580] jepsen worker 62 - jepsen.util 10592 :invoke :cas [20 [19 10]] INFO [2019-12-17 12:33:54,585] jepsen worker 21 - jepsen.util 9921 :invoke :read [7 nil] INFO [2019-12-17 12:33:54,585] jepsen worker 47 - jepsen.util 8057 :invoke :read [15 nil] INFO [2019-12-17 12:33:54,598] jepsen worker 74 - jepsen.util 7994 :invoke :write [24 5] INFO [2019-12-17 12:33:54,601] jepsen worker 14 - jepsen.util 7934 :ok :write [4 10] INFO [2019-12-17 12:33:54,602] jepsen worker 14 - jepsen.util 7934 :invoke :write [4 2] INFO [2019-12-17 12:33:54,611] jepsen worker 73 - jepsen.util 7363 :ok :write [24 33] INFO [2019-12-17 12:33:54,611] jepsen worker 73 - jepsen.util 7363 :invoke :write [24 19] INFO [2019-12-17 12:33:54,620] jepsen worker 33 - jepsen.util 10203 :invoke :write [11 33] INFO [2019-12-17 12:33:54,631] jepsen worker 20 - jepsen.util 7310 :fail :read [6 nil] :RequestTimeoutException INFO [2019-12-17 12:33:54,631] jepsen worker 20 - jepsen.util 7310 :invoke :write [6 43] INFO [2019-12-17 12:33:54,633] jepsen worker 19 - jepsen.util 7849 :info :cas [6 [8 2]] :RequestCanceledException INFO [2019-12-17 12:33:54,634] jepsen worker 25 - jepsen.util 11005 :invoke :read [8 nil] INFO [2019-12-17 12:33:54,635] jepsen worker 59 - jepsen.util 10139 :invoke :read [19 nil] INFO [2019-12-17 12:33:54,635] jepsen worker 86 - jepsen.util 11066 :invoke :write [28 40] INFO [2019-12-17 12:33:54,635] jepsen worker 67 - jepsen.util 10417 :invoke :cas [22 [36 35]] INFO [2019-12-17 12:33:54,638] jepsen worker 82 - jepsen.util 11242 :invoke :cas [27 [1 29]] INFO [2019-12-17 12:33:54,641] jepsen worker 1 - jepsen.util 10621 :info :write [0 26] :RequestCanceledException INFO [2019-12-17 12:33:54,642] jepsen worker 22 - jepsen.util 10372 :invoke :write [7 41] INFO [2019-12-17 12:33:54,652] jepsen worker 24 - jepsen.util 9294 :invoke :write [8 29] INFO [2019-12-17 12:33:54,660] jepsen worker 60 - jepsen.util 11490 :invoke :cas [20 [18 22]] INFO [2019-12-17 12:33:54,671] jepsen worker 10 - jepsen.util 7750 :invoke :cas [3 [7 10]] INFO [2019-12-17 12:33:54,677] jepsen worker 9 - jepsen.util 7929 :fail :cas [3 [21 6]] :ValueNotSwapFrom INFO [2019-12-17 12:33:54,678] jepsen worker 9 - jepsen.util 7929 :invoke :cas [3 [12 4]] INFO [2019-12-17 12:33:54,697] jepsen worker 1 - jepsen.util 10711 :invoke :cas [0 [7 37]] INFO [2019-12-17 12:33:54,697] jepsen worker 19 - jepsen.util 7939 :invoke :write [6 35] INFO [2019-12-17 12:33:54,712] jepsen worker 6 - jepsen.util 10986 :info :cas [2 [42 17]] :RequestCanceledException INFO [2019-12-17 12:33:54,723] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44564,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:33:54,723] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44564,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:33:54,723] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44568,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:33:54,725] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44568,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:33:54,734] jepsen worker 6 - jepsen.util 11076 :invoke :write [2 26] INFO [2019-12-17 12:33:54,798] jepsen worker 57 - jepsen.util 10407 :info :cas [19 [21 37]] :RequestCanceledException INFO [2019-12-17 12:33:54,845] jepsen worker 48 - jepsen.util 9498 :info :cas [16 [7 20]] :RequestCanceledException INFO [2019-12-17 12:33:54,906] jepsen worker 31 - jepsen.util 7051 :ok :write [10 32] INFO [2019-12-17 12:33:54,910] jepsen worker 31 - jepsen.util 7051 :invoke :read [10 nil] INFO [2019-12-17 12:33:54,918] jepsen worker 84 - jepsen.util 10074 :info :write [28 39] :RequestCanceledException INFO [2019-12-17 12:33:54,925] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44578,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:33:54,933] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44578,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:33:54,937] jepsen worker 11 - jepsen.util 7841 :ok :write [3 21] INFO [2019-12-17 12:33:54,939] jepsen worker 17 - jepsen.util 10547 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:33:54,941] jepsen worker 11 - jepsen.util 7841 :invoke :read [3 nil] INFO [2019-12-17 12:33:54,942] jepsen worker 17 - jepsen.util 10547 :invoke :write [5 0] INFO [2019-12-17 12:33:54,945] jepsen worker 81 - jepsen.util 10251 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:33:54,949] jepsen worker 81 - jepsen.util 10251 :invoke :read [27 nil] INFO [2019-12-17 12:33:54,974] jepsen worker 76 - jepsen.util 10336 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:33:54,979] jepsen worker 76 - jepsen.util 10336 :invoke :read [25 nil] INFO [2019-12-17 12:33:54,981] jepsen worker 48 - jepsen.util 9588 :invoke :read [16 nil] INFO [2019-12-17 12:33:54,982] jepsen worker 57 - jepsen.util 10497 :invoke :write [19 13] INFO [2019-12-17 12:33:55,087] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44592,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:33:55,094] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44592,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:33:55,095] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44597,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:33:55,102] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44597,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:33:55,107] jepsen worker 38 - jepsen.util 7958 :ok :write [12 44] INFO [2019-12-17 12:33:55,114] jepsen worker 38 - jepsen.util 7958 :invoke :write [12 37] INFO [2019-12-17 12:33:55,129] jepsen worker 16 - jepsen.util 11716 :info :cas [5 [38 32]] :RequestCanceledException INFO [2019-12-17 12:33:55,137] jepsen worker 84 - jepsen.util 10164 :invoke :write [28 4] INFO [2019-12-17 12:33:55,159] jepsen worker 61 - jepsen.util 11041 :info :cas [20 [18 19]] :RequestCanceledException INFO [2019-12-17 12:33:55,165] jepsen worker 45 - jepsen.util 7695 :ok :read [15 25] INFO [2019-12-17 12:33:55,166] jepsen worker 45 - jepsen.util 7695 :invoke :write [15 36] INFO [2019-12-17 12:33:55,167] jepsen worker 35 - jepsen.util 10205 :info :cas [11 [32 4]] :RequestCanceledException INFO [2019-12-17 12:33:55,173] jepsen worker 49 - jepsen.util 10039 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:33:55,174] jepsen worker 49 - jepsen.util 10039 :invoke :write [16 32] INFO [2019-12-17 12:33:55,181] jepsen worker 53 - jepsen.util 7253 :ok :read [17 22] INFO [2019-12-17 12:33:55,183] jepsen worker 53 - jepsen.util 7253 :invoke :read [17 nil] INFO [2019-12-17 12:33:55,199] jepsen worker 7 - jepsen.util 10537 :info :write [2 34] :RequestCanceledException INFO [2019-12-17 12:33:55,199] jepsen worker 65 - jepsen.util 6365 :fail :read [21 nil] :RequestTimeoutException INFO [2019-12-17 12:33:55,200] jepsen worker 65 - jepsen.util 6365 :invoke :write [21 36] INFO [2019-12-17 12:33:55,215] jepsen worker 61 - jepsen.util 11131 :invoke :cas [20 [4 46]] INFO [2019-12-17 12:33:55,215] jepsen worker 39 - jepsen.util 10029 :info :write [13 33] :RequestCanceledException INFO [2019-12-17 12:33:55,215] jepsen worker 16 - jepsen.util 11806 :invoke :read [5 nil] INFO [2019-12-17 12:33:55,219] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44605,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:33:55,230] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44605,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:33:55,231] jepsen worker 85 - jepsen.util 11245 :info :cas [28 [10 6]] :RequestCanceledException INFO [2019-12-17 12:33:55,236] jepsen worker 3 - jepsen.util 7743 :fail :cas [1 [35 8]] :ValueNotSwapFrom INFO [2019-12-17 12:33:55,237] jepsen worker 78 - jepsen.util 7728 :fail :cas [26 [45 23]] :ValueNotSwapFrom INFO [2019-12-17 12:33:55,238] jepsen worker 44 - jepsen.util 11024 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:33:55,239] jepsen worker 3 - jepsen.util 7743 :invoke :read [1 nil] INFO [2019-12-17 12:33:55,241] jepsen worker 78 - jepsen.util 7728 :invoke :write [26 6] INFO [2019-12-17 12:33:55,241] jepsen worker 44 - jepsen.util 11024 :invoke :cas [14 [26 24]] INFO [2019-12-17 12:33:55,241] jepsen worker 8 - jepsen.util 10268 :info :cas [2 [11 6]] :RequestCanceledException INFO [2019-12-17 12:33:55,243] jepsen worker 70 - jepsen.util 8080 :ok :read [23 42] INFO [2019-12-17 12:33:55,247] jepsen worker 54 - jepsen.util 8604 :fail :cas [18 [27 22]] :ValueNotSwapFrom INFO [2019-12-17 12:33:55,248] jepsen worker 40 - jepsen.util 10930 :info :write [13 15] :RequestCanceledException INFO [2019-12-17 12:33:55,248] jepsen worker 70 - jepsen.util 8080 :invoke :cas [23 [21 25]] INFO [2019-12-17 12:33:55,249] jepsen worker 54 - jepsen.util 8604 :invoke :write [18 34] INFO [2019-12-17 12:33:55,257] jepsen worker 87 - jepsen.util 7107 :fail :cas [29 [6 39]] :ValueNotSwapFrom INFO [2019-12-17 12:33:55,259] jepsen worker 87 - jepsen.util 7107 :invoke :write [29 15] INFO [2019-12-17 12:33:55,259] jepsen worker 80 - jepsen.util 7550 :ok :read [26 35] INFO [2019-12-17 12:33:55,262] jepsen worker 80 - jepsen.util 7550 :invoke :read [26 nil] INFO [2019-12-17 12:33:55,268] jepsen worker 4 - jepsen.util 7114 :fail :cas [1 [23 32]] :ValueNotSwapFrom INFO [2019-12-17 12:33:55,270] jepsen worker 4 - jepsen.util 7114 :invoke :cas [1 [25 21]] INFO [2019-12-17 12:33:55,271] jepsen worker 35 - jepsen.util 10295 :invoke :read [11 nil] INFO [2019-12-17 12:33:55,280] jepsen worker 64 - jepsen.util 6184 :fail :cas [21 [9 41]] :ValueNotSwapFrom INFO [2019-12-17 12:33:55,281] jepsen worker 56 - jepsen.util 8786 :fail :cas [18 [41 14]] :ValueNotSwapFrom INFO [2019-12-17 12:33:55,281] jepsen worker 64 - jepsen.util 6184 :invoke :write [21 11] INFO [2019-12-17 12:33:55,281] jepsen worker 56 - jepsen.util 8786 :invoke :cas [18 [40 39]] INFO [2019-12-17 12:33:55,292] jepsen worker 46 - jepsen.util 7966 :fail :read [15 nil] :RequestTimeoutException INFO [2019-12-17 12:33:55,299] jepsen worker 46 - jepsen.util 7966 :invoke :read [15 nil] INFO [2019-12-17 12:33:55,301] jepsen worker 18 - jepsen.util 7938 :ok :read [6 44] INFO [2019-12-17 12:33:55,301] jepsen worker 18 - jepsen.util 7938 :invoke :read [6 nil] INFO [2019-12-17 12:33:55,305] jepsen worker 7 - jepsen.util 10627 :invoke :write [2 44] INFO [2019-12-17 12:33:55,309] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:33:55,310] jepsen nemesis - couchbase.util Rebalance check count: 342 INFO [2019-12-17 12:33:55,312] jepsen nemesis - couchbase.util Rebalance stuck count: 323 INFO [2019-12-17 12:33:55,319] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44611,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:33:55,324] jepsen worker 29 - jepsen.util 6239 :info :write [9 32] :RequestCanceledException INFO [2019-12-17 12:33:55,332] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44611,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:33:55,332] jepsen worker 85 - jepsen.util 11335 :invoke :write [28 14] INFO [2019-12-17 12:33:55,336] jepsen worker 39 - jepsen.util 10119 :invoke :cas [13 [19 40]] INFO [2019-12-17 12:33:55,341] jepsen worker 8 - jepsen.util 10358 :invoke :write [2 5] INFO [2019-12-17 12:33:55,378] jepsen worker 14 - jepsen.util 7934 :ok :write [4 2] INFO [2019-12-17 12:33:55,378] jepsen worker 14 - jepsen.util 7934 :invoke :cas [4 [33 5]] INFO [2019-12-17 12:33:55,381] jepsen worker 29 - jepsen.util 6329 :invoke :read [9 nil] INFO [2019-12-17 12:33:55,382] jepsen worker 36 - jepsen.util 7956 :ok :read [12 14] INFO [2019-12-17 12:33:55,383] jepsen worker 36 - jepsen.util 7956 :invoke :cas [12 [8 16]] INFO [2019-12-17 12:33:55,385] jepsen worker 40 - jepsen.util 11020 :invoke :cas [13 [34 36]] INFO [2019-12-17 12:33:55,456] jepsen worker 79 - jepsen.util 7729 :info :write [26 4] :RequestCanceledException INFO [2019-12-17 12:33:55,506] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44641,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:33:55,516] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44641,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:33:55,539] jepsen worker 88 - jepsen.util 7738 :ok :read [29 1] INFO [2019-12-17 12:33:55,540] jepsen worker 88 - jepsen.util 7738 :invoke :cas [29 [2 46]] INFO [2019-12-17 12:33:55,540] jepsen worker 5 - jepsen.util 7205 :info :write [1 48] :RequestCanceledException INFO [2019-12-17 12:33:55,542] jepsen worker 71 - jepsen.util 8261 :info :write [23 38] :RequestCanceledException INFO [2019-12-17 12:33:55,542] jepsen worker 79 - jepsen.util 7819 :invoke :write [26 13] INFO [2019-12-17 12:33:55,547] jepsen worker 51 - jepsen.util 7431 :fail :cas [17 [28 21]] :ValueNotSwapFrom INFO [2019-12-17 12:33:55,548] jepsen worker 51 - jepsen.util 7431 :invoke :cas [17 [22 5]] INFO [2019-12-17 12:33:55,601] jepsen worker 2 - jepsen.util 10082 :info :write [0 35] :RequestCanceledException INFO [2019-12-17 12:33:55,645] jepsen worker 5 - jepsen.util 7295 :invoke :read [1 nil] INFO [2019-12-17 12:33:55,664] jepsen worker 32 - jepsen.util 7592 :info :write [10 20] :RequestCanceledException INFO [2019-12-17 12:33:55,674] jepsen worker 71 - jepsen.util 8351 :invoke :read [23 nil] INFO [2019-12-17 12:33:55,707] jepsen worker 2 - jepsen.util 10172 :invoke :read [0 nil] INFO [2019-12-17 12:33:55,713] jepsen worker 32 - jepsen.util 7682 :invoke :cas [10 [30 30]] INFO [2019-12-17 12:33:55,737] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44652,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:33:55,738] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44652,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:33:55,738] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44655,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:33:55,739] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44655,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:33:55,744] jepsen worker 89 - jepsen.util 7559 :fail :read [29 nil] :RequestTimeoutException INFO [2019-12-17 12:33:55,744] jepsen worker 89 - jepsen.util 7559 :invoke :read [29 nil] INFO [2019-12-17 12:33:55,895] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44673,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:33:55,897] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44673,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:33:55,901] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44677,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:33:55,903] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44677,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:33:55,914] jepsen worker 13 - jepsen.util 7303 :ok :write [4 35] INFO [2019-12-17 12:33:55,915] jepsen worker 13 - jepsen.util 7303 :invoke :cas [4 [32 40]] INFO [2019-12-17 12:33:55,950] jepsen worker 69 - jepsen.util 7809 :fail :cas [23 [14 0]] :ValueNotSwapFrom INFO [2019-12-17 12:33:55,966] jepsen worker 69 - jepsen.util 7809 :invoke :cas [23 [32 34]] INFO [2019-12-17 12:33:55,967] jepsen worker 20 - jepsen.util 7310 :ok :write [6 43] INFO [2019-12-17 12:33:55,967] jepsen worker 20 - jepsen.util 7310 :invoke :cas [6 [6 2]] INFO [2019-12-17 12:33:55,991] jepsen worker 27 - jepsen.util 7677 :ok :write [9 26] INFO [2019-12-17 12:33:55,991] jepsen worker 27 - jepsen.util 7677 :invoke :write [9 5] INFO [2019-12-17 12:33:56,024] jepsen worker 54 - jepsen.util 8604 :ok :write [18 34] INFO [2019-12-17 12:33:56,024] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44682,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:33:56,024] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44682,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:33:56,025] jepsen worker 54 - jepsen.util 8604 :invoke :read [18 nil] INFO [2019-12-17 12:33:56,029] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44686,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:33:56,037] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44686,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:33:56,042] jepsen worker 10 - jepsen.util 7750 :fail :cas [3 [7 10]] :ValueNotSwapFrom INFO [2019-12-17 12:33:56,042] jepsen worker 10 - jepsen.util 7750 :invoke :read [3 nil] INFO [2019-12-17 12:33:56,059] jepsen worker 28 - jepsen.util 6148 :ok :write [9 9] INFO [2019-12-17 12:33:56,062] jepsen worker 28 - jepsen.util 6148 :invoke :write [9 30] INFO [2019-12-17 12:33:56,114] jepsen worker 74 - jepsen.util 7994 :ok :write [24 5] INFO [2019-12-17 12:33:56,118] jepsen worker 74 - jepsen.util 7994 :invoke :cas [24 [8 31]] INFO [2019-12-17 12:33:56,170] jepsen worker 63 - jepsen.util 6633 :fail :cas [21 [47 46]] :ValueNotSwapFrom INFO [2019-12-17 12:33:56,171] jepsen worker 63 - jepsen.util 6633 :invoke :read [21 nil] INFO [2019-12-17 12:33:56,216] jepsen worker 72 - jepsen.util 7452 :info :cas [24 [18 24]] :RequestCanceledException INFO [2019-12-17 12:33:56,242] jepsen worker 31 - jepsen.util 7051 :ok :read [10 32] INFO [2019-12-17 12:33:56,245] jepsen worker 31 - jepsen.util 7051 :invoke :cas [10 [49 40]] INFO [2019-12-17 12:33:56,246] jepsen worker 34 - jepsen.util 10114 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:33:56,246] jepsen worker 37 - jepsen.util 7147 :ok :read [12 14] INFO [2019-12-17 12:33:56,246] jepsen worker 34 - jepsen.util 10114 :invoke :write [11 23] INFO [2019-12-17 12:33:56,247] jepsen worker 37 - jepsen.util 7147 :invoke :read [12 nil] INFO [2019-12-17 12:33:56,279] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44696,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:33:56,281] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44696,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:33:56,282] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44703,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:33:56,289] jepsen worker 72 - jepsen.util 7542 :invoke :read [24 nil] INFO [2019-12-17 12:33:56,297] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44703,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:33:56,298] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44695,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:33:56,299] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44695,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:33:56,299] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44700,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:33:56,300] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44700,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:33:56,364] jepsen worker 42 - jepsen.util 11202 :info :cas [14 [8 28]] :RequestCanceledException INFO [2019-12-17 12:33:56,376] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:33:56,377] jepsen nemesis - couchbase.util Rebalance check count: 343 INFO [2019-12-17 12:33:56,377] jepsen nemesis - couchbase.util Rebalance stuck count: 324 INFO [2019-12-17 12:33:56,396] jepsen worker 11 - jepsen.util 7841 :ok :read [3 21] INFO [2019-12-17 12:33:56,397] jepsen worker 11 - jepsen.util 7841 :invoke :cas [3 [42 29]] INFO [2019-12-17 12:33:56,404] jepsen worker 42 - jepsen.util 11292 :invoke :cas [14 [13 39]] INFO [2019-12-17 12:33:56,422] jepsen worker 47 - jepsen.util 8057 :ok :read [15 25] INFO [2019-12-17 12:33:56,422] jepsen worker 47 - jepsen.util 8057 :invoke :write [15 30] INFO [2019-12-17 12:33:56,458] jepsen worker 70 - jepsen.util 8080 :fail :cas [23 [21 25]] :ValueNotSwapFrom INFO [2019-12-17 12:33:56,460] jepsen worker 70 - jepsen.util 8080 :invoke :cas [23 [35 41]] INFO [2019-12-17 12:33:56,512] jepsen worker 19 - jepsen.util 7939 :ok :write [6 35] INFO [2019-12-17 12:33:56,513] jepsen worker 19 - jepsen.util 7939 :invoke :cas [6 [20 32]] INFO [2019-12-17 12:33:56,514] jepsen worker 0 - jepsen.util 10890 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:33:56,515] jepsen worker 43 - jepsen.util 10033 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:33:56,515] jepsen worker 0 - jepsen.util 10890 :invoke :write [0 38] INFO [2019-12-17 12:33:56,521] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44713,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:33:56,522] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44713,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:33:56,522] jepsen worker 43 - jepsen.util 10033 :invoke :cas [14 [28 21]] INFO [2019-12-17 12:33:56,525] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44715,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:33:56,528] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44715,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:33:56,534] jepsen worker 75 - jepsen.util 10065 :info :cas [25 [44 28]] :RequestCanceledException INFO [2019-12-17 12:33:56,535] jepsen worker 58 - jepsen.util 11308 :info :cas [19 [20 11]] :RequestCanceledException INFO [2019-12-17 12:33:56,542] jepsen worker 83 - jepsen.util 10793 :info :cas [27 [39 44]] :RequestCanceledException INFO [2019-12-17 12:33:56,566] jepsen worker 77 - jepsen.util 10787 :info :cas [25 [47 31]] :RequestCanceledException INFO [2019-12-17 12:33:56,602] jepsen worker 62 - jepsen.util 10592 :info :cas [20 [19 10]] :RequestCanceledException INFO [2019-12-17 12:33:56,613] jepsen worker 58 - jepsen.util 11398 :invoke :cas [19 [35 21]] INFO [2019-12-17 12:33:56,615] jepsen worker 30 - jepsen.util 7680 :ok :read [10 39] INFO [2019-12-17 12:33:56,616] jepsen worker 30 - jepsen.util 7680 :invoke :read [10 nil] INFO [2019-12-17 12:33:56,625] jepsen worker 75 - jepsen.util 10155 :invoke :cas [25 [48 24]] INFO [2019-12-17 12:33:56,625] jepsen worker 9 - jepsen.util 7929 :fail :cas [3 [12 4]] :ValueNotSwapFrom INFO [2019-12-17 12:33:56,626] jepsen worker 21 - jepsen.util 9921 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:33:56,628] jepsen worker 77 - jepsen.util 10877 :invoke :read [25 nil] INFO [2019-12-17 12:33:56,633] jepsen worker 21 - jepsen.util 9921 :invoke :cas [7 [48 20]] INFO [2019-12-17 12:33:56,633] jepsen worker 9 - jepsen.util 7929 :invoke :read [3 nil] INFO [2019-12-17 12:33:56,640] jepsen worker 62 - jepsen.util 10682 :invoke :write [20 44] INFO [2019-12-17 12:33:56,641] jepsen worker 23 - jepsen.util 10373 :info :cas [7 [20 16]] :RequestCanceledException INFO [2019-12-17 12:33:56,642] jepsen worker 83 - jepsen.util 10883 :invoke :write [27 12] INFO [2019-12-17 12:33:56,642] jepsen worker 41 - jepsen.util 11021 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:33:56,649] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44727,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:33:56,649] jepsen worker 41 - jepsen.util 11021 :invoke :write [13 17] INFO [2019-12-17 12:33:56,664] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44727,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:33:56,674] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44730,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:33:56,674] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44730,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:33:56,674] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44731,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:33:56,675] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44731,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:33:56,686] jepsen worker 12 - jepsen.util 8202 :fail :cas [4 [5 46]] :ValueNotSwapFrom INFO [2019-12-17 12:33:56,688] jepsen worker 12 - jepsen.util 8202 :invoke :read [4 nil] INFO [2019-12-17 12:33:56,696] jepsen worker 59 - jepsen.util 10139 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:33:56,698] jepsen worker 59 - jepsen.util 10139 :invoke :cas [19 [17 8]] INFO [2019-12-17 12:33:56,703] jepsen worker 60 - jepsen.util 11490 :info :cas [20 [18 22]] :RequestCanceledException INFO [2019-12-17 12:33:56,729] jepsen worker 73 - jepsen.util 7363 :ok :write [24 19] INFO [2019-12-17 12:33:56,730] jepsen worker 73 - jepsen.util 7363 :invoke :write [24 22] INFO [2019-12-17 12:33:56,731] jepsen worker 25 - jepsen.util 11005 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:33:56,731] jepsen worker 67 - jepsen.util 10417 :info :cas [22 [36 35]] :RequestCanceledException INFO [2019-12-17 12:33:56,732] jepsen worker 1 - jepsen.util 10711 :info :cas [0 [7 37]] :RequestCanceledException INFO [2019-12-17 12:33:56,732] jepsen worker 25 - jepsen.util 11005 :invoke :cas [8 [24 25]] INFO [2019-12-17 12:33:56,744] jepsen worker 23 - jepsen.util 10463 :invoke :read [7 nil] INFO [2019-12-17 12:33:56,746] jepsen worker 82 - jepsen.util 11242 :info :cas [27 [1 29]] :RequestCanceledException INFO [2019-12-17 12:33:56,771] jepsen worker 60 - jepsen.util 11580 :invoke :cas [20 [22 22]] INFO [2019-12-17 12:33:56,780] jepsen worker 1 - jepsen.util 10801 :invoke :read [0 nil] INFO [2019-12-17 12:33:56,796] jepsen worker 67 - jepsen.util 10507 :invoke :write [22 39] INFO [2019-12-17 12:33:56,810] jepsen worker 38 - jepsen.util 7958 :ok :write [12 37] INFO [2019-12-17 12:33:56,810] jepsen worker 38 - jepsen.util 7958 :invoke :read [12 nil] INFO [2019-12-17 12:33:56,826] jepsen worker 82 - jepsen.util 11332 :invoke :cas [27 [9 12]] INFO [2019-12-17 12:33:56,839] jepsen worker 26 - jepsen.util 10916 :info :write [8 4] :RequestCanceledException INFO [2019-12-17 12:33:56,905] jepsen worker 50 - jepsen.util 11300 :info :write [16 38] :RequestCanceledException INFO [2019-12-17 12:33:56,924] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44745,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:33:56,928] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44745,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:33:56,932] jepsen worker 53 - jepsen.util 7253 :ok :read [17 22] INFO [2019-12-17 12:33:56,933] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44747,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:33:56,934] jepsen worker 53 - jepsen.util 7253 :invoke :cas [17 [10 44]] INFO [2019-12-17 12:33:56,939] jepsen worker 3 - jepsen.util 7743 :ok :read [1 18] INFO [2019-12-17 12:33:56,945] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44747,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:33:56,945] jepsen worker 3 - jepsen.util 7743 :invoke :read [1 nil] INFO [2019-12-17 12:33:56,947] jepsen worker 66 - jepsen.util 10326 :info :write [22 36] :RequestCanceledException INFO [2019-12-17 12:33:56,961] jepsen worker 68 - jepsen.util 9878 :info :write [22 7] :RequestCanceledException INFO [2019-12-17 12:33:56,985] jepsen worker 26 - jepsen.util 11006 :invoke :write [8 24] INFO [2019-12-17 12:33:57,017] jepsen worker 15 - jepsen.util 9375 :info :write [5 42] :RequestCanceledException INFO [2019-12-17 12:33:57,017] jepsen worker 50 - jepsen.util 11390 :invoke :read [16 nil] INFO [2019-12-17 12:33:57,038] jepsen worker 48 - jepsen.util 9588 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:33:57,040] jepsen worker 48 - jepsen.util 9588 :invoke :write [16 21] INFO [2019-12-17 12:33:57,049] jepsen worker 81 - jepsen.util 10251 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:33:57,050] jepsen worker 81 - jepsen.util 10251 :invoke :write [27 36] INFO [2019-12-17 12:33:57,059] jepsen worker 76 - jepsen.util 10336 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:33:57,061] jepsen worker 76 - jepsen.util 10336 :invoke :cas [25 [9 40]] INFO [2019-12-17 12:33:57,067] jepsen worker 80 - jepsen.util 7550 :ok :read [26 35] INFO [2019-12-17 12:33:57,068] jepsen worker 80 - jepsen.util 7550 :invoke :read [26 nil] INFO [2019-12-17 12:33:57,075] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44751,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:33:57,079] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44751,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:33:57,086] jepsen worker 68 - jepsen.util 9968 :invoke :cas [22 [6 45]] INFO [2019-12-17 12:33:57,087] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44755,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:33:57,089] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44755,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:33:57,097] jepsen worker 66 - jepsen.util 10416 :invoke :write [22 27] INFO [2019-12-17 12:33:57,118] jepsen worker 86 - jepsen.util 11066 :info :write [28 40] :RequestCanceledException INFO [2019-12-17 12:33:57,127] jepsen worker 15 - jepsen.util 9465 :invoke :write [5 16] INFO [2019-12-17 12:33:57,139] jepsen worker 52 - jepsen.util 7972 :info :write [17 17] :RequestCanceledException INFO [2019-12-17 12:33:57,153] jepsen worker 86 - jepsen.util 11156 :invoke :cas [28 [36 31]] INFO [2019-12-17 12:33:57,154] jepsen worker 52 - jepsen.util 8062 :invoke :read [17 nil] INFO [2019-12-17 12:33:57,205] jepsen worker 55 - jepsen.util 8515 :info :write [18 35] :RequestCanceledException INFO [2019-12-17 12:33:57,207] jepsen worker 78 - jepsen.util 7728 :ok :write [26 6] INFO [2019-12-17 12:33:57,208] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44767,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:33:57,211] jepsen worker 78 - jepsen.util 7728 :invoke :read [26 nil] INFO [2019-12-17 12:33:57,217] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44767,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:33:57,243] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44773,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:33:57,250] jepsen worker 61 - jepsen.util 11131 :info :cas [20 [4 46]] :RequestCanceledException INFO [2019-12-17 12:33:57,254] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44773,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:33:57,255] jepsen worker 16 - jepsen.util 11806 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:33:57,263] jepsen worker 16 - jepsen.util 11806 :invoke :cas [5 [33 23]] INFO [2019-12-17 12:33:57,279] jepsen worker 79 - jepsen.util 7819 :ok :write [26 13] INFO [2019-12-17 12:33:57,279] jepsen worker 55 - jepsen.util 8605 :invoke :write [18 36] INFO [2019-12-17 12:33:57,279] jepsen worker 79 - jepsen.util 7819 :invoke :read [26 nil] INFO [2019-12-17 12:33:57,330] jepsen worker 44 - jepsen.util 11024 :info :cas [14 [26 24]] :RequestCanceledException INFO [2019-12-17 12:33:57,336] jepsen worker 24 - jepsen.util 9294 :info :write [8 29] :RequestCanceledException INFO [2019-12-17 12:33:57,337] jepsen worker 61 - jepsen.util 11221 :invoke :write [20 42] INFO [2019-12-17 12:33:57,365] jepsen worker 33 - jepsen.util 10203 :info :write [11 33] :RequestCanceledException INFO [2019-12-17 12:33:57,365] jepsen worker 35 - jepsen.util 10295 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:33:57,371] jepsen worker 35 - jepsen.util 10295 :invoke :read [11 nil] INFO [2019-12-17 12:33:57,380] jepsen worker 22 - jepsen.util 10372 :info :write [7 41] :RequestCanceledException INFO [2019-12-17 12:33:57,420] jepsen worker 39 - jepsen.util 10119 :info :cas [13 [19 40]] :RequestCanceledException INFO [2019-12-17 12:33:57,445] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:33:57,446] jepsen nemesis - couchbase.util Rebalance check count: 344 INFO [2019-12-17 12:33:57,447] jepsen worker 36 - jepsen.util 7956 :info :cas [12 [8 16]] :RequestCanceledException INFO [2019-12-17 12:33:57,447] jepsen nemesis - couchbase.util Rebalance stuck count: 325 INFO [2019-12-17 12:33:57,463] jepsen worker 40 - jepsen.util 11020 :info :cas [13 [34 36]] :RequestCanceledException INFO [2019-12-17 12:33:57,466] jepsen worker 44 - jepsen.util 11114 :invoke :write [14 12] INFO [2019-12-17 12:33:57,474] jepsen worker 24 - jepsen.util 9384 :invoke :read [8 nil] INFO [2019-12-17 12:33:57,538] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44787,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:33:57,538] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44787,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:33:57,548] jepsen worker 22 - jepsen.util 10462 :invoke :cas [7 [16 15]] INFO [2019-12-17 12:33:57,558] jepsen worker 39 - jepsen.util 10209 :invoke :write [13 42] INFO [2019-12-17 12:33:57,563] jepsen worker 6 - jepsen.util 11076 :info :write [2 26] :RequestCanceledException INFO [2019-12-17 12:33:57,563] jepsen worker 33 - jepsen.util 10293 :invoke :read [11 nil] INFO [2019-12-17 12:33:57,575] jepsen worker 36 - jepsen.util 8046 :invoke :read [12 nil] INFO [2019-12-17 12:33:57,642] jepsen worker 17 - jepsen.util 10547 :info :write [5 0] :RequestCanceledException INFO [2019-12-17 12:33:57,677] jepsen worker 40 - jepsen.util 11110 :invoke :cas [13 [10 30]] INFO [2019-12-17 12:33:57,695] jepsen worker 57 - jepsen.util 10497 :info :write [19 13] :RequestCanceledException INFO [2019-12-17 12:33:57,711] jepsen worker 65 - jepsen.util 6365 :info :write [21 36] :RequestCanceledException INFO [2019-12-17 12:33:57,715] jepsen worker 87 - jepsen.util 7107 :ok :write [29 15] INFO [2019-12-17 12:33:57,717] jepsen worker 87 - jepsen.util 7107 :invoke :read [29 nil] INFO [2019-12-17 12:33:57,731] jepsen worker 2 - jepsen.util 10172 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:33:57,736] jepsen worker 2 - jepsen.util 10172 :invoke :read [0 nil] INFO [2019-12-17 12:33:57,738] jepsen worker 6 - jepsen.util 11166 :invoke :read [2 nil] INFO [2019-12-17 12:33:57,747] jepsen worker 32 - jepsen.util 7682 :info :cas [10 [30 30]] :RequestCanceledException INFO [2019-12-17 12:33:57,759] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44797,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:33:57,761] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44797,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:33:57,764] jepsen worker 49 - jepsen.util 10039 :info :write [16 32] :RequestCanceledException INFO [2019-12-17 12:33:57,787] jepsen worker 17 - jepsen.util 10637 :invoke :write [5 11] INFO [2019-12-17 12:33:57,790] jepsen worker 32 - jepsen.util 7772 :invoke :cas [10 [15 15]] INFO [2019-12-17 12:33:57,799] jepsen worker 85 - jepsen.util 11335 :info :write [28 14] :RequestCanceledException INFO [2019-12-17 12:33:57,800] jepsen worker 64 - jepsen.util 6184 :info :write [21 11] :RequestCanceledException INFO [2019-12-17 12:33:57,818] jepsen worker 84 - jepsen.util 10164 :info :write [28 4] :RequestCanceledException INFO [2019-12-17 12:33:57,820] jepsen worker 65 - jepsen.util 6455 :invoke :cas [21 [24 20]] INFO [2019-12-17 12:33:57,830] jepsen worker 57 - jepsen.util 10587 :invoke :read [19 nil] INFO [2019-12-17 12:33:57,839] jepsen worker 7 - jepsen.util 10627 :info :write [2 44] :RequestCanceledException INFO [2019-12-17 12:33:57,840] jepsen worker 8 - jepsen.util 10358 :info :write [2 5] :RequestCanceledException INFO [2019-12-17 12:33:57,840] jepsen worker 49 - jepsen.util 10129 :invoke :write [16 23] INFO [2019-12-17 12:33:57,849] jepsen worker 45 - jepsen.util 7695 :info :write [15 36] :RequestCanceledException INFO [2019-12-17 12:33:57,899] jepsen worker 20 - jepsen.util 7310 :fail :cas [6 [6 2]] :ValueNotSwapFrom INFO [2019-12-17 12:33:57,900] jepsen worker 20 - jepsen.util 7310 :invoke :read [6 nil] INFO [2019-12-17 12:33:57,902] jepsen worker 64 - jepsen.util 6274 :invoke :read [21 nil] INFO [2019-12-17 12:33:57,902] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44810,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:33:57,903] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44810,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:33:57,910] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44814,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:33:57,915] jepsen worker 85 - jepsen.util 11425 :invoke :read [28 nil] INFO [2019-12-17 12:33:57,922] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44814,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:33:57,925] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44815,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:33:57,932] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44815,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:33:57,953] jepsen worker 84 - jepsen.util 10254 :invoke :read [28 nil] INFO [2019-12-17 12:33:57,966] jepsen worker 7 - jepsen.util 10717 :invoke :read [2 nil] INFO [2019-12-17 12:33:57,975] jepsen worker 8 - jepsen.util 10448 :invoke :write [2 30] INFO [2019-12-17 12:33:57,982] jepsen worker 45 - jepsen.util 7785 :invoke :read [15 nil] INFO [2019-12-17 12:33:57,999] jepsen worker 71 - jepsen.util 8351 :ok :read [23 42] INFO [2019-12-17 12:33:58,000] jepsen worker 71 - jepsen.util 8351 :invoke :read [23 nil] INFO [2019-12-17 12:33:58,020] jepsen worker 29 - jepsen.util 6329 :ok :read [9 9] INFO [2019-12-17 12:33:58,021] jepsen worker 29 - jepsen.util 6329 :invoke :cas [9 [30 41]] INFO [2019-12-17 12:33:58,033] jepsen worker 28 - jepsen.util 6148 :ok :write [9 30] INFO [2019-12-17 12:33:58,034] jepsen worker 28 - jepsen.util 6148 :invoke :write [9 11] INFO [2019-12-17 12:33:58,044] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44826,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:33:58,044] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44826,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:33:58,104] jepsen worker 14 - jepsen.util 7934 :fail :cas [4 [33 5]] :ValueNotSwapFrom INFO [2019-12-17 12:33:58,105] jepsen worker 14 - jepsen.util 7934 :invoke :write [4 45] INFO [2019-12-17 12:33:58,137] jepsen worker 10 - jepsen.util 7750 :ok :read [3 21] INFO [2019-12-17 12:33:58,139] jepsen worker 10 - jepsen.util 7750 :invoke :cas [3 [31 31]] INFO [2019-12-17 12:33:58,145] jepsen worker 63 - jepsen.util 6633 :ok :read [21 30] INFO [2019-12-17 12:33:58,146] jepsen worker 74 - jepsen.util 7994 :info :cas [24 [8 31]] :RequestCanceledException INFO [2019-12-17 12:33:58,146] jepsen worker 63 - jepsen.util 6633 :invoke :read [21 nil] INFO [2019-12-17 12:33:58,152] jepsen worker 54 - jepsen.util 8604 :ok :read [18 34] INFO [2019-12-17 12:33:58,156] jepsen worker 54 - jepsen.util 8604 :invoke :write [18 0] INFO [2019-12-17 12:33:58,156] jepsen worker 31 - jepsen.util 7051 :fail :cas [10 [49 40]] :ValueNotSwapFrom INFO [2019-12-17 12:33:58,168] jepsen worker 31 - jepsen.util 7051 :invoke :read [10 nil] INFO [2019-12-17 12:33:58,219] jepsen worker 18 - jepsen.util 7938 :ok :read [6 43] INFO [2019-12-17 12:33:58,220] jepsen worker 18 - jepsen.util 7938 :invoke :read [6 nil] INFO [2019-12-17 12:33:58,240] jepsen worker 74 - jepsen.util 8084 :invoke :write [24 46] INFO [2019-12-17 12:33:58,296] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44844,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:33:58,296] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44844,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:33:58,303] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44849,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:33:58,308] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44838,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:33:58,310] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44849,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:33:58,310] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44838,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:33:58,311] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44840,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:33:58,313] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44840,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:33:58,314] jepsen worker 73 - jepsen.util 7363 :ok :write [24 22] INFO [2019-12-17 12:33:58,318] jepsen worker 73 - jepsen.util 7363 :invoke :read [24 nil] INFO [2019-12-17 12:33:58,318] jepsen worker 47 - jepsen.util 8057 :ok :write [15 30] INFO [2019-12-17 12:33:58,319] jepsen worker 47 - jepsen.util 8057 :invoke :write [15 38] INFO [2019-12-17 12:33:58,324] jepsen worker 27 - jepsen.util 7677 :ok :write [9 5] INFO [2019-12-17 12:33:58,324] jepsen worker 27 - jepsen.util 7677 :invoke :cas [9 [24 43]] INFO [2019-12-17 12:33:58,335] jepsen worker 37 - jepsen.util 7147 :fail :read [12 nil] :RequestTimeoutException INFO [2019-12-17 12:33:58,335] jepsen worker 37 - jepsen.util 7147 :invoke :read [12 nil] INFO [2019-12-17 12:33:58,439] jepsen worker 42 - jepsen.util 11292 :info :cas [14 [13 39]] :RequestCanceledException INFO [2019-12-17 12:33:58,489] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44856,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:33:58,490] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44856,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:33:58,501] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44860,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:33:58,505] jepsen worker 11 - jepsen.util 7841 :info :cas [3 [42 29]] :RequestCanceledException INFO [2019-12-17 12:33:58,506] jepsen worker 5 - jepsen.util 7295 :ok :read [1 18] INFO [2019-12-17 12:33:58,511] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44860,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:33:58,512] jepsen worker 5 - jepsen.util 7295 :invoke :read [1 nil] INFO [2019-12-17 12:33:58,540] jepsen worker 11 - jepsen.util 7931 :invoke :read [3 nil] INFO [2019-12-17 12:33:58,546] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:33:58,549] jepsen nemesis - couchbase.util Rebalance check count: 345 INFO [2019-12-17 12:33:58,550] jepsen nemesis - couchbase.util Rebalance stuck count: 326 INFO [2019-12-17 12:33:58,578] jepsen worker 42 - jepsen.util 11382 :invoke :read [14 nil] INFO [2019-12-17 12:33:58,615] jepsen worker 43 - jepsen.util 10033 :info :cas [14 [28 21]] :RequestCanceledException INFO [2019-12-17 12:33:58,709] jepsen worker 58 - jepsen.util 11398 :info :cas [19 [35 21]] :RequestCanceledException INFO [2019-12-17 12:33:58,716] jepsen worker 21 - jepsen.util 9921 :info :cas [7 [48 20]] :RequestCanceledException INFO [2019-12-17 12:33:58,717] jepsen worker 75 - jepsen.util 10155 :info :cas [25 [48 24]] :RequestCanceledException INFO [2019-12-17 12:33:58,723] jepsen worker 4 - jepsen.util 7114 :fail :cas [1 [25 21]] :ValueNotSwapFrom INFO [2019-12-17 12:33:58,725] jepsen worker 4 - jepsen.util 7114 :invoke :write [1 22] INFO [2019-12-17 12:33:58,734] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44868,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:33:58,740] jepsen worker 77 - jepsen.util 10877 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:33:58,743] jepsen worker 9 - jepsen.util 7929 :ok :read [3 21] INFO [2019-12-17 12:33:58,743] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44868,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:33:58,745] jepsen worker 9 - jepsen.util 7929 :invoke :cas [3 [27 16]] INFO [2019-12-17 12:33:58,747] jepsen worker 77 - jepsen.util 10877 :invoke :cas [25 [8 8]] INFO [2019-12-17 12:33:58,747] jepsen worker 34 - jepsen.util 10114 :info :write [11 23] :RequestCanceledException INFO [2019-12-17 12:33:58,748] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44873,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:33:58,749] jepsen worker 30 - jepsen.util 7680 :ok :read [10 32] INFO [2019-12-17 12:33:58,750] jepsen worker 30 - jepsen.util 7680 :invoke :write [10 31] INFO [2019-12-17 12:33:58,751] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44873,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:33:58,760] jepsen worker 43 - jepsen.util 10123 :invoke :cas [14 [17 24]] INFO [2019-12-17 12:33:58,788] jepsen worker 56 - jepsen.util 8786 :fail :cas [18 [40 39]] :ValueNotSwapFrom INFO [2019-12-17 12:33:58,788] jepsen worker 56 - jepsen.util 8786 :invoke :write [18 3] INFO [2019-12-17 12:33:58,803] jepsen worker 19 - jepsen.util 7939 :fail :cas [6 [20 32]] :ValueNotSwapFrom INFO [2019-12-17 12:33:58,805] jepsen worker 58 - jepsen.util 11488 :invoke :write [19 3] INFO [2019-12-17 12:33:58,812] jepsen worker 21 - jepsen.util 10011 :invoke :cas [7 [4 44]] INFO [2019-12-17 12:33:58,813] jepsen worker 19 - jepsen.util 7939 :invoke :cas [6 [26 14]] INFO [2019-12-17 12:33:58,813] jepsen worker 59 - jepsen.util 10139 :info :cas [19 [17 8]] :RequestCanceledException INFO [2019-12-17 12:33:58,813] jepsen worker 60 - jepsen.util 11580 :info :cas [20 [22 22]] :RequestCanceledException INFO [2019-12-17 12:33:58,822] jepsen worker 1 - jepsen.util 10801 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:33:58,823] jepsen worker 1 - jepsen.util 10801 :invoke :read [0 nil] INFO [2019-12-17 12:33:58,824] jepsen worker 46 - jepsen.util 7966 :ok :read [15 25] INFO [2019-12-17 12:33:58,830] jepsen worker 23 - jepsen.util 10463 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:33:58,831] jepsen worker 46 - jepsen.util 7966 :invoke :read [15 nil] INFO [2019-12-17 12:33:58,831] jepsen worker 23 - jepsen.util 10463 :invoke :cas [7 [46 40]] INFO [2019-12-17 12:33:58,840] jepsen worker 25 - jepsen.util 11005 :info :cas [8 [24 25]] :RequestCanceledException INFO [2019-12-17 12:33:58,845] jepsen worker 82 - jepsen.util 11332 :info :cas [27 [9 12]] :RequestCanceledException INFO [2019-12-17 12:33:58,847] jepsen worker 12 - jepsen.util 8202 :ok :read [4 35] INFO [2019-12-17 12:33:58,849] jepsen worker 12 - jepsen.util 8202 :invoke :read [4 nil] INFO [2019-12-17 12:33:58,855] jepsen worker 70 - jepsen.util 8080 :fail :cas [23 [35 41]] :ValueNotSwapFrom INFO [2019-12-17 12:33:58,856] jepsen worker 70 - jepsen.util 8080 :invoke :write [23 16] INFO [2019-12-17 12:33:58,884] jepsen worker 75 - jepsen.util 10245 :invoke :read [25 nil] INFO [2019-12-17 12:33:58,886] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44889,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:33:58,887] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44889,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:33:58,938] jepsen worker 0 - jepsen.util 10890 :info :write [0 38] :RequestCanceledException INFO [2019-12-17 12:33:58,949] jepsen worker 34 - jepsen.util 10204 :invoke :cas [11 [29 0]] INFO [2019-12-17 12:33:58,992] jepsen worker 82 - jepsen.util 11422 :invoke :cas [27 [31 27]] INFO [2019-12-17 12:33:58,992] jepsen worker 59 - jepsen.util 10229 :invoke :cas [19 [37 2]] INFO [2019-12-17 12:33:59,066] jepsen worker 60 - jepsen.util 11670 :invoke :read [20 nil] INFO [2019-12-17 12:33:59,081] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44895,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:33:59,083] jepsen worker 0 - jepsen.util 10980 :invoke :write [0 38] INFO [2019-12-17 12:33:59,083] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44895,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:33:59,084] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44899,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:33:59,084] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44899,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:33:59,085] jepsen worker 25 - jepsen.util 11095 :invoke :write [8 11] INFO [2019-12-17 12:33:59,137] jepsen worker 50 - jepsen.util 11390 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:33:59,144] jepsen worker 50 - jepsen.util 11390 :invoke :cas [16 [48 13]] INFO [2019-12-17 12:33:59,168] jepsen worker 76 - jepsen.util 10336 :info :cas [25 [9 40]] :RequestCanceledException INFO [2019-12-17 12:33:59,176] jepsen worker 88 - jepsen.util 7738 :fail :cas [29 [2 46]] :ValueNotSwapFrom INFO [2019-12-17 12:33:59,176] jepsen worker 88 - jepsen.util 7738 :invoke :read [29 nil] INFO [2019-12-17 12:33:59,197] jepsen worker 54 - jepsen.util 8604 :ok :write [18 0] INFO [2019-12-17 12:33:59,198] jepsen worker 54 - jepsen.util 8604 :invoke :cas [18 [9 43]] INFO [2019-12-17 12:33:59,205] jepsen worker 68 - jepsen.util 9968 :info :cas [22 [6 45]] :RequestCanceledException INFO [2019-12-17 12:33:59,232] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44917,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:33:59,232] jepsen worker 3 - jepsen.util 7743 :ok :read [1 48] INFO [2019-12-17 12:33:59,234] jepsen worker 41 - jepsen.util 11021 :info :write [13 17] :RequestCanceledException INFO [2019-12-17 12:33:59,234] jepsen worker 3 - jepsen.util 7743 :invoke :write [1 10] INFO [2019-12-17 12:33:59,236] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44917,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:33:59,242] jepsen worker 76 - jepsen.util 10426 :invoke :read [25 nil] INFO [2019-12-17 12:33:59,242] jepsen worker 62 - jepsen.util 10682 :info :write [20 44] :RequestCanceledException INFO [2019-12-17 12:33:59,242] jepsen worker 86 - jepsen.util 11156 :info :cas [28 [36 31]] :RequestCanceledException INFO [2019-12-17 12:33:59,247] jepsen worker 68 - jepsen.util 10058 :invoke :read [22 nil] INFO [2019-12-17 12:33:59,255] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44906,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:33:59,259] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44906,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:33:59,259] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44911,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:33:59,260] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44911,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:33:59,261] jepsen worker 83 - jepsen.util 10883 :info :write [27 12] :RequestCanceledException INFO [2019-12-17 12:33:59,270] jepsen worker 62 - jepsen.util 10772 :invoke :write [20 34] INFO [2019-12-17 12:33:59,277] jepsen worker 41 - jepsen.util 11111 :invoke :cas [13 [23 7]] INFO [2019-12-17 12:33:59,281] jepsen worker 86 - jepsen.util 11246 :invoke :read [28 nil] INFO [2019-12-17 12:33:59,287] jepsen worker 38 - jepsen.util 7958 :ok :read [12 37] INFO [2019-12-17 12:33:59,287] jepsen worker 38 - jepsen.util 7958 :invoke :write [12 28] INFO [2019-12-17 12:33:59,287] jepsen worker 65 - jepsen.util 6455 :fail :cas [21 [24 20]] :ValueNotSwapFrom INFO [2019-12-17 12:33:59,288] jepsen worker 65 - jepsen.util 6455 :invoke :read [21 nil] INFO [2019-12-17 12:33:59,293] jepsen worker 13 - jepsen.util 7303 :fail :cas [4 [32 40]] :ValueNotSwapFrom INFO [2019-12-17 12:33:59,294] jepsen worker 13 - jepsen.util 7303 :invoke :write [4 20] INFO [2019-12-17 12:33:59,301] jepsen worker 83 - jepsen.util 10973 :invoke :cas [27 [11 48]] INFO [2019-12-17 12:33:59,316] jepsen worker 53 - jepsen.util 7253 :fail :cas [17 [10 44]] :ValueNotSwapFrom INFO [2019-12-17 12:33:59,316] jepsen worker 53 - jepsen.util 7253 :invoke :read [17 nil] INFO [2019-12-17 12:33:59,322] jepsen worker 69 - jepsen.util 7809 :fail :cas [23 [32 34]] :ValueNotSwapFrom INFO [2019-12-17 12:33:59,323] jepsen worker 69 - jepsen.util 7809 :invoke :write [23 21] INFO [2019-12-17 12:33:59,329] jepsen worker 79 - jepsen.util 7819 :fail :read [26 nil] :RequestTimeoutException INFO [2019-12-17 12:33:59,330] jepsen worker 79 - jepsen.util 7819 :invoke :cas [26 [48 38]] INFO [2019-12-17 12:33:59,350] jepsen worker 80 - jepsen.util 7550 :ok :read [26 13] INFO [2019-12-17 12:33:59,352] jepsen worker 16 - jepsen.util 11806 :info :cas [5 [33 23]] :RequestCanceledException INFO [2019-12-17 12:33:59,353] jepsen worker 80 - jepsen.util 7550 :invoke :cas [26 [34 20]] INFO [2019-12-17 12:33:59,357] jepsen worker 52 - jepsen.util 8062 :ok :read [17 22] INFO [2019-12-17 12:33:59,360] jepsen worker 52 - jepsen.util 8062 :invoke :cas [17 [22 1]] INFO [2019-12-17 12:33:59,389] jepsen worker 28 - jepsen.util 6148 :ok :write [9 11] INFO [2019-12-17 12:33:59,389] jepsen worker 28 - jepsen.util 6148 :invoke :write [9 24] INFO [2019-12-17 12:33:59,428] jepsen worker 16 - jepsen.util 11896 :invoke :read [5 nil] INFO [2019-12-17 12:33:59,431] jepsen worker 67 - jepsen.util 10507 :info :write [22 39] :RequestCanceledException INFO [2019-12-17 12:33:59,438] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44931,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:33:59,442] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44931,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:33:59,466] jepsen worker 35 - jepsen.util 10295 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:33:59,473] jepsen worker 35 - jepsen.util 10295 :invoke :cas [11 [39 42]] INFO [2019-12-17 12:33:59,476] jepsen worker 67 - jepsen.util 10597 :invoke :cas [22 [17 18]] INFO [2019-12-17 12:33:59,533] jepsen worker 24 - jepsen.util 9384 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:33:59,534] jepsen worker 24 - jepsen.util 9384 :invoke :write [8 30] INFO [2019-12-17 12:33:59,535] jepsen worker 89 - jepsen.util 7559 :ok :read [29 15] INFO [2019-12-17 12:33:59,537] jepsen worker 89 - jepsen.util 7559 :invoke :write [29 9] INFO [2019-12-17 12:33:59,617] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:33:59,617] jepsen nemesis - couchbase.util Rebalance check count: 346 INFO [2019-12-17 12:33:59,617] jepsen nemesis - couchbase.util Rebalance stuck count: 327 INFO [2019-12-17 12:33:59,633] jepsen worker 33 - jepsen.util 10293 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:33:59,644] jepsen worker 22 - jepsen.util 10462 :info :cas [7 [16 15]] :RequestCanceledException INFO [2019-12-17 12:33:59,645] jepsen worker 33 - jepsen.util 10293 :invoke :cas [11 [5 16]] INFO [2019-12-17 12:34:00,011] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44946,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:34:00,011] jepsen worker 72 - jepsen.util 7542 :ok :read [24 19] INFO [2019-12-17 12:34:00,011] jepsen worker 78 - jepsen.util 7728 :ok :read [26 13] INFO [2019-12-17 12:34:00,018] jepsen worker 27 - jepsen.util 7677 :fail :cas [9 [24 43]] :ValueNotSwapFrom INFO [2019-12-17 12:34:00,018] jepsen worker 45 - jepsen.util 7785 :ok :read [15 25] INFO [2019-12-17 12:34:00,019] jepsen worker 71 - jepsen.util 8351 :ok :read [23 38] INFO [2019-12-17 12:34:00,019] jepsen worker 57 - jepsen.util 10587 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:34:00,020] jepsen worker 78 - jepsen.util 7728 :invoke :read [26 nil] INFO [2019-12-17 12:34:00,023] jepsen worker 36 - jepsen.util 8046 :ok :read [12 37] INFO [2019-12-17 12:34:00,024] jepsen worker 72 - jepsen.util 7542 :invoke :write [24 39] INFO [2019-12-17 12:34:00,025] jepsen worker 81 - jepsen.util 10251 :info :write [27 36] :RequestCanceledException INFO [2019-12-17 12:34:00,025] jepsen worker 27 - jepsen.util 7677 :invoke :read [9 nil] INFO [2019-12-17 12:34:00,028] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44955,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:34:00,031] jepsen worker 40 - jepsen.util 11110 :info :cas [13 [10 30]] :RequestCanceledException INFO [2019-12-17 12:34:00,031] jepsen worker 2 - jepsen.util 10172 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:34:00,031] jepsen worker 85 - jepsen.util 11425 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:34:00,031] jepsen worker 48 - jepsen.util 9588 :info :write [16 21] :RequestCanceledException INFO [2019-12-17 12:34:00,032] jepsen worker 71 - jepsen.util 8351 :invoke :cas [23 [24 18]] INFO [2019-12-17 12:34:00,033] jepsen worker 57 - jepsen.util 10587 :invoke :cas [19 [36 25]] INFO [2019-12-17 12:34:00,033] jepsen worker 45 - jepsen.util 7785 :invoke :write [15 8] INFO [2019-12-17 12:34:00,044] jepsen worker 66 - jepsen.util 10416 :info :write [22 27] :RequestCanceledException INFO [2019-12-17 12:34:00,044] jepsen worker 36 - jepsen.util 8046 :invoke :cas [12 [30 20]] INFO [2019-12-17 12:34:00,044] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44955,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:34:00,046] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44946,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:34:00,046] jepsen worker 85 - jepsen.util 11425 :invoke :read [28 nil] INFO [2019-12-17 12:34:00,052] jepsen worker 2 - jepsen.util 10172 :invoke :write [0 35] INFO [2019-12-17 12:34:00,053] jepsen worker 6 - jepsen.util 11166 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:34:00,057] jepsen worker 32 - jepsen.util 7772 :info :cas [10 [15 15]] :RequestCanceledException INFO [2019-12-17 12:34:00,062] jepsen worker 26 - jepsen.util 11006 :info :write [8 24] :RequestCanceledException INFO [2019-12-17 12:34:00,065] jepsen worker 55 - jepsen.util 8605 :ok :write [18 36] INFO [2019-12-17 12:34:00,066] jepsen worker 15 - jepsen.util 9465 :info :write [5 16] :RequestCanceledException INFO [2019-12-17 12:34:00,068] jepsen worker 84 - jepsen.util 10254 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:34:00,074] jepsen worker 55 - jepsen.util 8605 :invoke :read [18 nil] INFO [2019-12-17 12:34:00,074] jepsen worker 6 - jepsen.util 11166 :invoke :cas [2 [40 45]] INFO [2019-12-17 12:34:00,078] jepsen worker 84 - jepsen.util 10254 :invoke :read [28 nil] INFO [2019-12-17 12:34:00,092] jepsen worker 51 - jepsen.util 7431 :fail :cas [17 [22 5]] :CASMismatchException INFO [2019-12-17 12:34:00,098] jepsen worker 7 - jepsen.util 10717 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:34:00,131] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44965,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:34:00,133] jepsen worker 61 - jepsen.util 11221 :info :write [20 42] :RequestCanceledException INFO [2019-12-17 12:34:00,158] jepsen worker 51 - jepsen.util 7431 :invoke :cas [17 [4 1]] INFO [2019-12-17 12:34:00,161] jepsen worker 7 - jepsen.util 10717 :invoke :write [2 35] INFO [2019-12-17 12:34:00,169] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44965,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:34:00,175] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44968,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:34:00,198] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44968,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:34:00,211] jepsen worker 32 - jepsen.util 7862 :invoke :read [10 nil] INFO [2019-12-17 12:34:00,228] jepsen worker 44 - jepsen.util 11114 :info :write [14 12] :RequestCanceledException INFO [2019-12-17 12:34:00,234] jepsen worker 22 - jepsen.util 10552 :invoke :cas [7 [33 21]] INFO [2019-12-17 12:34:00,273] jepsen worker 39 - jepsen.util 10209 :info :write [13 42] :RequestCanceledException INFO [2019-12-17 12:34:00,279] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44980,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:34:00,283] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44980,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:34:00,285] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44985,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:34:00,292] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44985,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:34:00,303] jepsen worker 14 - jepsen.util 7934 :ok :write [4 45] INFO [2019-12-17 12:34:00,304] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44990,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:34:00,330] jepsen worker 14 - jepsen.util 7934 :invoke :write [4 47] INFO [2019-12-17 12:34:00,331] jepsen worker 40 - jepsen.util 11200 :invoke :cas [13 [36 22]] INFO [2019-12-17 12:34:00,333] jepsen worker 81 - jepsen.util 10341 :invoke :write [27 1] INFO [2019-12-17 12:34:00,341] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44990,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:34:00,350] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44991,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:34:00,351] jepsen worker 48 - jepsen.util 9678 :invoke :read [16 nil] INFO [2019-12-17 12:34:00,357] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44991,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:34:00,361] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44992,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:34:00,362] jepsen worker 15 - jepsen.util 9555 :invoke :cas [5 [17 49]] INFO [2019-12-17 12:34:00,366] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":44992,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:34:00,375] jepsen worker 66 - jepsen.util 10506 :invoke :read [22 nil] INFO [2019-12-17 12:34:00,378] jepsen worker 26 - jepsen.util 11096 :invoke :cas [8 [43 34]] INFO [2019-12-17 12:34:00,381] jepsen worker 61 - jepsen.util 11311 :invoke :cas [20 [29 20]] INFO [2019-12-17 12:34:00,383] jepsen worker 39 - jepsen.util 10299 :invoke :write [13 36] INFO [2019-12-17 12:34:00,437] jepsen worker 44 - jepsen.util 11204 :invoke :read [14 nil] INFO [2019-12-17 12:34:00,441] jepsen worker 37 - jepsen.util 7147 :fail :read [12 nil] :RequestTimeoutException INFO [2019-12-17 12:34:00,442] jepsen worker 37 - jepsen.util 7147 :invoke :write [12 45] INFO [2019-12-17 12:34:00,460] jepsen worker 70 - jepsen.util 8080 :ok :write [23 16] INFO [2019-12-17 12:34:00,462] jepsen worker 70 - jepsen.util 8080 :invoke :write [23 29] INFO [2019-12-17 12:34:00,511] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45000,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:34:00,536] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45000,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:34:00,538] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45003,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:34:00,538] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45003,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:34:00,546] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45005,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:34:00,548] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45005,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:34:00,553] jepsen worker 17 - jepsen.util 10637 :info :write [5 11] :RequestCanceledException INFO [2019-12-17 12:34:00,554] jepsen worker 13 - jepsen.util 7303 :ok :write [4 20] INFO [2019-12-17 12:34:00,555] jepsen worker 13 - jepsen.util 7303 :invoke :read [4 nil] INFO [2019-12-17 12:34:00,644] jepsen worker 74 - jepsen.util 8084 :info :write [24 46] :RequestCanceledException INFO [2019-12-17 12:34:00,656] jepsen worker 47 - jepsen.util 8057 :info :write [15 38] :RequestCanceledException INFO [2019-12-17 12:34:00,663] jepsen worker 42 - jepsen.util 11382 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:34:00,663] jepsen worker 42 - jepsen.util 11382 :invoke :cas [14 [39 1]] INFO [2019-12-17 12:34:00,687] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45010,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:34:00,688] jepsen worker 74 - jepsen.util 8174 :invoke :write [24 30] INFO [2019-12-17 12:34:00,689] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45010,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:34:00,696] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45013,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:34:00,700] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45013,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:34:00,700] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:34:00,701] jepsen worker 17 - jepsen.util 10727 :invoke :read [5 nil] INFO [2019-12-17 12:34:00,708] jepsen nemesis - couchbase.util Rebalance check count: 347 INFO [2019-12-17 12:34:00,709] jepsen nemesis - couchbase.util Rebalance stuck count: 328 INFO [2019-12-17 12:34:00,719] jepsen worker 47 - jepsen.util 8147 :invoke :read [15 nil] INFO [2019-12-17 12:34:00,736] jepsen worker 49 - jepsen.util 10129 :info :write [16 23] :RequestCanceledException INFO [2019-12-17 12:34:00,742] jepsen worker 20 - jepsen.util 7310 :ok :read [6 35] INFO [2019-12-17 12:34:00,743] jepsen worker 20 - jepsen.util 7310 :invoke :cas [6 [39 8]] INFO [2019-12-17 12:34:00,795] jepsen worker 8 - jepsen.util 10448 :info :write [2 30] :RequestCanceledException INFO [2019-12-17 12:34:00,802] jepsen worker 63 - jepsen.util 6633 :ok :read [21 11] INFO [2019-12-17 12:34:00,805] jepsen worker 12 - jepsen.util 8202 :ok :read [4 35] INFO [2019-12-17 12:34:00,806] jepsen worker 12 - jepsen.util 8202 :invoke :read [4 nil] INFO [2019-12-17 12:34:00,807] jepsen worker 64 - jepsen.util 6274 :ok :read [21 11] INFO [2019-12-17 12:34:00,814] jepsen worker 64 - jepsen.util 6274 :invoke :cas [21 [13 25]] INFO [2019-12-17 12:34:00,814] jepsen worker 63 - jepsen.util 6633 :invoke :cas [21 [37 37]] INFO [2019-12-17 12:34:00,831] jepsen worker 49 - jepsen.util 10219 :invoke :read [16 nil] INFO [2019-12-17 12:34:00,831] jepsen worker 43 - jepsen.util 10123 :info :cas [14 [17 24]] :RequestCanceledException INFO [2019-12-17 12:34:00,831] jepsen worker 77 - jepsen.util 10877 :info :cas [25 [8 8]] :RequestCanceledException INFO [2019-12-17 12:34:00,833] jepsen worker 87 - jepsen.util 7107 :ok :read [29 15] INFO [2019-12-17 12:34:00,841] jepsen worker 87 - jepsen.util 7107 :invoke :read [29 nil] INFO [2019-12-17 12:34:00,849] jepsen worker 8 - jepsen.util 10538 :invoke :read [2 nil] INFO [2019-12-17 12:34:00,870] jepsen worker 10 - jepsen.util 7750 :fail :cas [3 [31 31]] :ValueNotSwapFrom INFO [2019-12-17 12:34:00,871] jepsen worker 10 - jepsen.util 7750 :invoke :write [3 14] INFO [2019-12-17 12:34:00,872] jepsen worker 9 - jepsen.util 7929 :fail :cas [3 [27 16]] :ValueNotSwapFrom INFO [2019-12-17 12:34:00,874] jepsen worker 9 - jepsen.util 7929 :invoke :write [3 21] INFO [2019-12-17 12:34:00,876] jepsen worker 11 - jepsen.util 7931 :ok :read [3 21] INFO [2019-12-17 12:34:00,877] jepsen worker 11 - jepsen.util 7931 :invoke :cas [3 [4 37]] INFO [2019-12-17 12:34:00,879] jepsen worker 31 - jepsen.util 7051 :ok :read [10 32] INFO [2019-12-17 12:34:00,880] jepsen worker 18 - jepsen.util 7938 :ok :read [6 35] INFO [2019-12-17 12:34:00,880] jepsen worker 31 - jepsen.util 7051 :invoke :write [10 17] INFO [2019-12-17 12:34:00,880] jepsen worker 18 - jepsen.util 7938 :invoke :cas [6 [4 47]] INFO [2019-12-17 12:34:00,881] jepsen worker 5 - jepsen.util 7295 :ok :read [1 48] INFO [2019-12-17 12:34:00,881] jepsen worker 5 - jepsen.util 7295 :invoke :read [1 nil] INFO [2019-12-17 12:34:00,886] jepsen worker 73 - jepsen.util 7363 :ok :read [24 22] INFO [2019-12-17 12:34:00,888] jepsen worker 73 - jepsen.util 7363 :invoke :write [24 40] INFO [2019-12-17 12:34:00,899] jepsen worker 43 - jepsen.util 10213 :invoke :cas [14 [43 25]] INFO [2019-12-17 12:34:00,919] jepsen worker 21 - jepsen.util 10011 :info :cas [7 [4 44]] :RequestCanceledException INFO [2019-12-17 12:34:00,925] jepsen worker 77 - jepsen.util 10967 :invoke :read [25 nil] INFO [2019-12-17 12:34:00,926] jepsen worker 1 - jepsen.util 10801 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:34:00,927] jepsen worker 1 - jepsen.util 10801 :invoke :read [0 nil] INFO [2019-12-17 12:34:00,936] jepsen worker 23 - jepsen.util 10463 :info :cas [7 [46 40]] :RequestCanceledException INFO [2019-12-17 12:34:00,943] jepsen worker 75 - jepsen.util 10245 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:34:00,943] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45032,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:34:00,944] jepsen worker 75 - jepsen.util 10245 :invoke :read [25 nil] INFO [2019-12-17 12:34:00,946] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45032,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:34:00,959] jepsen worker 46 - jepsen.util 7966 :ok :read [15 30] INFO [2019-12-17 12:34:00,960] jepsen worker 46 - jepsen.util 7966 :invoke :cas [15 [40 29]] INFO [2019-12-17 12:34:01,019] jepsen worker 28 - jepsen.util 6148 :ok :write [9 24] INFO [2019-12-17 12:34:01,021] jepsen worker 28 - jepsen.util 6148 :invoke :read [9 nil] INFO [2019-12-17 12:34:01,031] jepsen worker 59 - jepsen.util 10229 :info :cas [19 [37 2]] :RequestCanceledException INFO [2019-12-17 12:34:01,059] jepsen worker 34 - jepsen.util 10204 :info :cas [11 [29 0]] :RequestCanceledException INFO [2019-12-17 12:34:01,065] jepsen worker 82 - jepsen.util 11422 :info :cas [27 [31 27]] :RequestCanceledException INFO [2019-12-17 12:34:01,091] jepsen worker 3 - jepsen.util 7743 :ok :write [1 10] INFO [2019-12-17 12:34:01,097] jepsen worker 3 - jepsen.util 7743 :invoke :write [1 46] INFO [2019-12-17 12:34:01,106] jepsen worker 60 - jepsen.util 11670 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:34:01,121] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45044,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:34:01,123] jepsen worker 60 - jepsen.util 11670 :invoke :cas [20 [35 18]] INFO [2019-12-17 12:34:01,153] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45044,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:34:01,153] jepsen worker 23 - jepsen.util 10553 :invoke :read [7 nil] INFO [2019-12-17 12:34:01,155] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45049,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:34:01,169] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45049,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:34:01,187] jepsen worker 89 - jepsen.util 7559 :ok :write [29 9] INFO [2019-12-17 12:34:01,187] jepsen worker 21 - jepsen.util 10101 :invoke :cas [7 [37 47]] INFO [2019-12-17 12:34:01,188] jepsen worker 89 - jepsen.util 7559 :invoke :cas [29 [46 34]] INFO [2019-12-17 12:34:01,235] jepsen worker 50 - jepsen.util 11390 :info :cas [16 [48 13]] :RequestCanceledException INFO [2019-12-17 12:34:01,240] jepsen worker 88 - jepsen.util 7738 :fail :read [29 nil] :RequestTimeoutException INFO [2019-12-17 12:34:01,242] jepsen worker 88 - jepsen.util 7738 :invoke :write [29 45] INFO [2019-12-17 12:34:01,252] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45066,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:34:01,258] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45066,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:34:01,260] jepsen worker 59 - jepsen.util 10319 :invoke :cas [19 [16 24]] INFO [2019-12-17 12:34:01,267] jepsen worker 34 - jepsen.util 10294 :invoke :write [11 35] INFO [2019-12-17 12:34:01,275] jepsen worker 68 - jepsen.util 10058 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:34:01,276] jepsen worker 68 - jepsen.util 10058 :invoke :cas [22 [45 0]] INFO [2019-12-17 12:34:01,299] jepsen worker 41 - jepsen.util 11111 :info :cas [13 [23 7]] :RequestCanceledException INFO [2019-12-17 12:34:01,299] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45059,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:34:01,305] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45059,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:34:01,311] jepsen worker 82 - jepsen.util 11512 :invoke :read [27 nil] INFO [2019-12-17 12:34:01,317] jepsen worker 86 - jepsen.util 11246 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:34:01,321] jepsen worker 86 - jepsen.util 11246 :invoke :read [28 nil] INFO [2019-12-17 12:34:01,324] jepsen worker 50 - jepsen.util 11480 :invoke :read [16 nil] INFO [2019-12-17 12:34:01,332] jepsen worker 83 - jepsen.util 10973 :info :cas [27 [11 48]] :RequestCanceledException INFO [2019-12-17 12:34:01,341] jepsen worker 41 - jepsen.util 11201 :invoke :cas [13 [28 38]] INFO [2019-12-17 12:34:01,345] jepsen worker 76 - jepsen.util 10426 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:34:01,347] jepsen worker 76 - jepsen.util 10426 :invoke :cas [25 [24 30]] INFO [2019-12-17 12:34:01,430] jepsen worker 79 - jepsen.util 7819 :info :cas [26 [48 38]] :RequestCanceledException INFO [2019-12-17 12:34:01,438] jepsen worker 83 - jepsen.util 11063 :invoke :read [27 nil] INFO [2019-12-17 12:34:01,444] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45082,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:34:01,444] jepsen worker 16 - jepsen.util 11896 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:34:01,449] jepsen worker 16 - jepsen.util 11896 :invoke :read [5 nil] INFO [2019-12-17 12:34:01,449] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45082,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:34:01,451] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45083,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:34:01,452] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45083,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:34:01,452] jepsen worker 79 - jepsen.util 7909 :invoke :write [26 4] INFO [2019-12-17 12:34:01,454] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45087,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:34:01,459] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45087,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:34:01,475] jepsen worker 19 - jepsen.util 7939 :fail :cas [6 [26 14]] :ValueNotSwapFrom INFO [2019-12-17 12:34:01,475] jepsen worker 19 - jepsen.util 7939 :invoke :cas [6 [43 8]] INFO [2019-12-17 12:34:01,496] jepsen worker 58 - jepsen.util 11488 :info :write [19 3] :RequestCanceledException INFO [2019-12-17 12:34:01,587] jepsen worker 29 - jepsen.util 6329 :fail :cas [9 [30 41]] :CASMismatchException INFO [2019-12-17 12:34:01,588] jepsen worker 29 - jepsen.util 6329 :invoke :cas [9 [34 5]] INFO [2019-12-17 12:34:01,590] jepsen worker 35 - jepsen.util 10295 :info :cas [11 [39 42]] :RequestCanceledException INFO [2019-12-17 12:34:01,590] jepsen worker 67 - jepsen.util 10597 :info :cas [22 [17 18]] :RequestCanceledException INFO [2019-12-17 12:34:01,592] jepsen worker 4 - jepsen.util 7114 :info :write [1 22] :RequestCanceledException INFO [2019-12-17 12:34:01,593] jepsen worker 65 - jepsen.util 6455 :ok :read [21 11] INFO [2019-12-17 12:34:01,594] jepsen worker 54 - jepsen.util 8604 :fail :cas [18 [9 43]] :ValueNotSwapFrom INFO [2019-12-17 12:34:01,594] jepsen worker 52 - jepsen.util 8062 :fail :cas [17 [22 1]] :ValueNotSwapFrom INFO [2019-12-17 12:34:01,595] jepsen worker 54 - jepsen.util 8604 :invoke :read [18 nil] INFO [2019-12-17 12:34:01,595] jepsen worker 65 - jepsen.util 6455 :invoke :read [21 nil] INFO [2019-12-17 12:34:01,595] jepsen worker 52 - jepsen.util 8062 :invoke :read [17 nil] INFO [2019-12-17 12:34:01,596] jepsen worker 30 - jepsen.util 7680 :info :write [10 31] :RequestCanceledException INFO [2019-12-17 12:34:01,597] jepsen worker 56 - jepsen.util 8786 :info :write [18 3] :RequestCanceledException INFO [2019-12-17 12:34:01,648] jepsen worker 58 - jepsen.util 11578 :invoke :write [19 1] INFO [2019-12-17 12:34:01,699] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45102,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:34:01,699] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45102,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:34:01,714] jepsen worker 67 - jepsen.util 10687 :invoke :read [22 nil] INFO [2019-12-17 12:34:01,714] jepsen worker 0 - jepsen.util 10980 :info :write [0 38] :RequestCanceledException INFO [2019-12-17 12:34:01,715] jepsen worker 35 - jepsen.util 10385 :invoke :read [11 nil] INFO [2019-12-17 12:34:01,715] jepsen worker 4 - jepsen.util 7204 :invoke :write [1 24] INFO [2019-12-17 12:34:01,729] jepsen worker 33 - jepsen.util 10293 :info :cas [11 [5 16]] :RequestCanceledException INFO [2019-12-17 12:34:01,729] jepsen worker 25 - jepsen.util 11095 :info :write [8 11] :RequestCanceledException INFO [2019-12-17 12:34:01,730] jepsen worker 30 - jepsen.util 7770 :invoke :read [10 nil] INFO [2019-12-17 12:34:01,735] jepsen worker 62 - jepsen.util 10772 :info :write [20 34] :RequestCanceledException INFO [2019-12-17 12:34:01,764] jepsen worker 31 - jepsen.util 7051 :ok :write [10 17] INFO [2019-12-17 12:34:01,766] jepsen worker 31 - jepsen.util 7051 :invoke :cas [10 [24 19]] INFO [2019-12-17 12:34:01,799] jepsen worker 0 - jepsen.util 11070 :invoke :write [0 34] INFO [2019-12-17 12:34:01,801] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:34:01,801] jepsen worker 56 - jepsen.util 8876 :invoke :cas [18 [33 38]] INFO [2019-12-17 12:34:01,806] jepsen nemesis - couchbase.util Rebalance check count: 348 INFO [2019-12-17 12:34:01,807] jepsen nemesis - couchbase.util Rebalance stuck count: 329 INFO [2019-12-17 12:34:01,808] jepsen worker 62 - jepsen.util 10862 :invoke :write [20 12] INFO [2019-12-17 12:34:01,815] jepsen worker 33 - jepsen.util 10383 :invoke :write [11 18] INFO [2019-12-17 12:34:01,839] jepsen worker 25 - jepsen.util 11185 :invoke :write [8 28] INFO [2019-12-17 12:34:01,922] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45112,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:34:01,930] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45112,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:34:01,939] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45117,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:34:01,941] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45117,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:34:02,030] jepsen worker 69 - jepsen.util 7809 :info :write [23 21] :RequestCanceledException INFO [2019-12-17 12:34:02,030] jepsen worker 38 - jepsen.util 7958 :info :write [12 28] :RequestCanceledException INFO [2019-12-17 12:34:02,035] jepsen worker 47 - jepsen.util 8147 :ok :read [15 30] INFO [2019-12-17 12:34:02,036] jepsen worker 47 - jepsen.util 8147 :invoke :write [15 1] INFO [2019-12-17 12:34:02,041] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45123,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:34:02,043] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45123,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:34:02,045] jepsen worker 36 - jepsen.util 8046 :fail :cas [12 [30 20]] :ValueNotSwapFrom INFO [2019-12-17 12:34:02,046] jepsen worker 53 - jepsen.util 7253 :ok :read [17 17] INFO [2019-12-17 12:34:02,046] jepsen worker 36 - jepsen.util 8046 :invoke :write [12 28] INFO [2019-12-17 12:34:02,047] jepsen worker 53 - jepsen.util 7253 :invoke :write [17 12] INFO [2019-12-17 12:34:02,048] jepsen worker 20 - jepsen.util 7310 :fail :cas [6 [39 8]] :ValueNotSwapFrom INFO [2019-12-17 12:34:02,049] jepsen worker 32 - jepsen.util 7862 :ok :read [10 32] INFO [2019-12-17 12:34:02,049] jepsen worker 80 - jepsen.util 7550 :fail :cas [26 [34 20]] :ValueNotSwapFrom INFO [2019-12-17 12:34:02,049] jepsen worker 80 - jepsen.util 7550 :invoke :write [26 36] INFO [2019-12-17 12:34:02,050] jepsen worker 20 - jepsen.util 7310 :invoke :cas [6 [21 21]] INFO [2019-12-17 12:34:02,050] jepsen worker 32 - jepsen.util 7862 :invoke :cas [10 [42 32]] INFO [2019-12-17 12:34:02,091] jepsen worker 69 - jepsen.util 7899 :invoke :cas [23 [10 41]] INFO [2019-12-17 12:34:02,094] jepsen worker 38 - jepsen.util 8048 :invoke :cas [12 [29 11]] INFO [2019-12-17 12:34:02,097] jepsen worker 57 - jepsen.util 10587 :info :cas [19 [36 25]] :RequestCanceledException INFO [2019-12-17 12:34:02,130] jepsen worker 78 - jepsen.util 7728 :fail :read [26 nil] :RequestTimeoutException INFO [2019-12-17 12:34:02,130] jepsen worker 6 - jepsen.util 11166 :info :cas [2 [40 45]] :RequestCanceledException INFO [2019-12-17 12:34:02,130] jepsen worker 78 - jepsen.util 7728 :invoke :read [26 nil] INFO [2019-12-17 12:34:02,130] jepsen worker 55 - jepsen.util 8605 :fail :read [18 nil] :RequestTimeoutException INFO [2019-12-17 12:34:02,131] jepsen worker 55 - jepsen.util 8605 :invoke :read [18 nil] INFO [2019-12-17 12:34:02,133] jepsen worker 57 - jepsen.util 10677 :invoke :read [19 nil] INFO [2019-12-17 12:34:02,143] jepsen worker 85 - jepsen.util 11425 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:34:02,147] jepsen worker 85 - jepsen.util 11425 :invoke :read [28 nil] INFO [2019-12-17 12:34:02,183] jepsen worker 6 - jepsen.util 11256 :invoke :write [2 16] INFO [2019-12-17 12:34:02,257] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45139,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:34:02,258] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45139,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:34:02,261] jepsen worker 84 - jepsen.util 10254 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:34:02,266] jepsen worker 84 - jepsen.util 10254 :invoke :write [28 25] INFO [2019-12-17 12:34:02,266] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45140,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:34:02,269] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45140,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:34:02,271] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45141,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:34:02,272] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45141,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:34:02,297] jepsen worker 22 - jepsen.util 10552 :info :cas [7 [33 21]] :RequestCanceledException INFO [2019-12-17 12:34:02,300] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45146,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:34:02,302] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45146,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:34:02,305] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45149,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:34:02,308] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45149,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:34:02,313] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45150,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:34:02,315] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45150,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:34:02,335] jepsen worker 73 - jepsen.util 7363 :ok :write [24 40] INFO [2019-12-17 12:34:02,335] jepsen worker 22 - jepsen.util 10642 :invoke :write [7 12] INFO [2019-12-17 12:34:02,335] jepsen worker 73 - jepsen.util 7363 :invoke :cas [24 [13 46]] INFO [2019-12-17 12:34:02,351] jepsen worker 46 - jepsen.util 7966 :fail :cas [15 [40 29]] :ValueNotSwapFrom INFO [2019-12-17 12:34:02,352] jepsen worker 46 - jepsen.util 7966 :invoke :write [15 45] INFO [2019-12-17 12:34:02,387] jepsen worker 71 - jepsen.util 8351 :fail :cas [23 [24 18]] :ValueNotSwapFrom INFO [2019-12-17 12:34:02,388] jepsen worker 71 - jepsen.util 8351 :invoke :read [23 nil] INFO [2019-12-17 12:34:02,396] jepsen worker 51 - jepsen.util 7431 :fail :cas [17 [4 1]] :ValueNotSwapFrom INFO [2019-12-17 12:34:02,396] jepsen worker 51 - jepsen.util 7431 :invoke :cas [17 [28 11]] INFO [2019-12-17 12:34:02,398] jepsen worker 26 - jepsen.util 11096 :info :cas [8 [43 34]] :RequestCanceledException INFO [2019-12-17 12:34:02,399] jepsen worker 87 - jepsen.util 7107 :ok :read [29 9] INFO [2019-12-17 12:34:02,400] jepsen worker 87 - jepsen.util 7107 :invoke :write [29 36] INFO [2019-12-17 12:34:02,404] jepsen worker 89 - jepsen.util 7559 :fail :cas [29 [46 34]] :ValueNotSwapFrom INFO [2019-12-17 12:34:02,406] jepsen worker 89 - jepsen.util 7559 :invoke :write [29 31] INFO [2019-12-17 12:34:02,409] jepsen worker 61 - jepsen.util 11311 :info :cas [20 [29 20]] :RequestCanceledException INFO [2019-12-17 12:34:02,414] jepsen worker 40 - jepsen.util 11200 :info :cas [13 [36 22]] :RequestCanceledException INFO [2019-12-17 12:34:02,429] jepsen worker 66 - jepsen.util 10506 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:34:02,429] jepsen worker 66 - jepsen.util 10506 :invoke :read [22 nil] INFO [2019-12-17 12:34:02,443] jepsen worker 48 - jepsen.util 9678 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:34:02,444] jepsen worker 48 - jepsen.util 9678 :invoke :read [16 nil] INFO [2019-12-17 12:34:02,447] jepsen worker 26 - jepsen.util 11186 :invoke :cas [8 [45 15]] INFO [2019-12-17 12:34:02,450] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45160,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:34:02,450] jepsen worker 27 - jepsen.util 7677 :ok :read [9 24] INFO [2019-12-17 12:34:02,452] jepsen worker 27 - jepsen.util 7677 :invoke :cas [9 [6 37]] INFO [2019-12-17 12:34:02,459] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45160,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:34:02,460] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45165,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:34:02,461] jepsen worker 15 - jepsen.util 9555 :info :cas [5 [17 49]] :RequestCanceledException INFO [2019-12-17 12:34:02,463] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45165,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:34:02,466] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45166,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:34:02,469] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45166,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:34:02,473] jepsen worker 40 - jepsen.util 11290 :invoke :read [13 nil] INFO [2019-12-17 12:34:02,476] jepsen worker 61 - jepsen.util 11401 :invoke :cas [20 [30 47]] INFO [2019-12-17 12:34:02,504] jepsen worker 15 - jepsen.util 9645 :invoke :write [5 3] INFO [2019-12-17 12:34:02,538] jepsen worker 44 - jepsen.util 11204 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:34:02,539] jepsen worker 44 - jepsen.util 11204 :invoke :read [14 nil] INFO [2019-12-17 12:34:02,568] jepsen worker 3 - jepsen.util 7743 :ok :write [1 46] INFO [2019-12-17 12:34:02,569] jepsen worker 3 - jepsen.util 7743 :invoke :read [1 nil] INFO [2019-12-17 12:34:02,632] jepsen worker 13 - jepsen.util 7303 :ok :read [4 20] INFO [2019-12-17 12:34:02,632] jepsen worker 13 - jepsen.util 7303 :invoke :read [4 nil] INFO [2019-12-17 12:34:02,674] jepsen worker 65 - jepsen.util 6455 :ok :read [21 11] INFO [2019-12-17 12:34:02,674] jepsen worker 65 - jepsen.util 6455 :invoke :cas [21 [44 1]] INFO [2019-12-17 12:34:02,678] jepsen worker 5 - jepsen.util 7295 :ok :read [1 48] INFO [2019-12-17 12:34:02,682] jepsen worker 5 - jepsen.util 7295 :invoke :read [1 nil] INFO [2019-12-17 12:34:02,689] jepsen worker 19 - jepsen.util 7939 :fail :cas [6 [43 8]] :ValueNotSwapFrom INFO [2019-12-17 12:34:02,691] jepsen worker 19 - jepsen.util 7939 :invoke :read [6 nil] INFO [2019-12-17 12:34:02,696] jepsen worker 29 - jepsen.util 6329 :fail :cas [9 [34 5]] :ValueNotSwapFrom INFO [2019-12-17 12:34:02,696] jepsen worker 11 - jepsen.util 7931 :fail :cas [3 [4 37]] :ValueNotSwapFrom INFO [2019-12-17 12:34:02,697] jepsen worker 29 - jepsen.util 6329 :invoke :cas [9 [19 11]] INFO [2019-12-17 12:34:02,697] jepsen worker 32 - jepsen.util 7862 :fail :cas [10 [42 32]] :ValueNotSwapFrom INFO [2019-12-17 12:34:02,697] jepsen worker 11 - jepsen.util 7931 :invoke :write [3 27] INFO [2019-12-17 12:34:02,697] jepsen worker 32 - jepsen.util 7862 :invoke :cas [10 [35 24]] INFO [2019-12-17 12:34:02,697] jepsen worker 31 - jepsen.util 7051 :fail :cas [10 [24 19]] :ValueNotSwapFrom INFO [2019-12-17 12:34:02,697] jepsen worker 18 - jepsen.util 7938 :fail :cas [6 [4 47]] :ValueNotSwapFrom INFO [2019-12-17 12:34:02,697] jepsen worker 31 - jepsen.util 7051 :invoke :cas [10 [16 4]] INFO [2019-12-17 12:34:02,698] jepsen worker 18 - jepsen.util 7938 :invoke :read [6 nil] INFO [2019-12-17 12:34:02,700] jepsen worker 20 - jepsen.util 7310 :fail :cas [6 [21 21]] :ValueNotSwapFrom INFO [2019-12-17 12:34:02,701] jepsen worker 20 - jepsen.util 7310 :invoke :write [6 17] INFO [2019-12-17 12:34:02,704] jepsen worker 69 - jepsen.util 7899 :fail :cas [23 [10 41]] :ValueNotSwapFrom INFO [2019-12-17 12:34:02,704] jepsen worker 69 - jepsen.util 7899 :invoke :write [23 33] INFO [2019-12-17 12:34:02,706] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45178,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:34:02,706] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45178,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:34:02,706] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45182,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:34:02,708] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45182,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:34:02,709] jepsen worker 14 - jepsen.util 7934 :ok :write [4 47] INFO [2019-12-17 12:34:02,712] jepsen worker 14 - jepsen.util 7934 :invoke :cas [4 [22 7]] INFO [2019-12-17 12:34:02,720] jepsen worker 24 - jepsen.util 9384 :info :write [8 30] :RequestCanceledException INFO [2019-12-17 12:34:02,723] jepsen worker 30 - jepsen.util 7770 :ok :read [10 17] INFO [2019-12-17 12:34:02,723] jepsen worker 30 - jepsen.util 7770 :invoke :read [10 nil] INFO [2019-12-17 12:34:02,727] jepsen worker 24 - jepsen.util 9474 :invoke :write [8 35] INFO [2019-12-17 12:34:02,758] jepsen worker 42 - jepsen.util 11382 :info :cas [14 [39 1]] :RequestCanceledException INFO [2019-12-17 12:34:02,770] jepsen worker 42 - jepsen.util 11472 :invoke :cas [14 [11 9]] INFO [2019-12-17 12:34:02,783] jepsen worker 28 - jepsen.util 6148 :ok :read [9 24] INFO [2019-12-17 12:34:02,786] jepsen worker 28 - jepsen.util 6148 :invoke :write [9 6] INFO [2019-12-17 12:34:02,793] jepsen worker 73 - jepsen.util 7363 :fail :cas [24 [13 46]] :ValueNotSwapFrom INFO [2019-12-17 12:34:02,793] jepsen worker 73 - jepsen.util 7363 :invoke :write [24 36] INFO [2019-12-17 12:34:02,808] jepsen worker 54 - jepsen.util 8604 :ok :read [18 36] INFO [2019-12-17 12:34:02,808] jepsen worker 54 - jepsen.util 8604 :invoke :read [18 nil] INFO [2019-12-17 12:34:02,820] jepsen worker 12 - jepsen.util 8202 :fail :read [4 nil] :RequestTimeoutException INFO [2019-12-17 12:34:02,820] jepsen worker 17 - jepsen.util 10727 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:34:02,820] jepsen worker 64 - jepsen.util 6274 :info :cas [21 [13 25]] :RequestCanceledException INFO [2019-12-17 12:34:02,820] jepsen worker 12 - jepsen.util 8202 :invoke :write [4 9] INFO [2019-12-17 12:34:02,820] jepsen worker 17 - jepsen.util 10727 :invoke :write [5 35] INFO [2019-12-17 12:34:02,823] jepsen worker 63 - jepsen.util 6633 :info :cas [21 [37 37]] :RequestCanceledException INFO [2019-12-17 12:34:02,824] jepsen worker 64 - jepsen.util 6364 :invoke :write [21 2] INFO [2019-12-17 12:34:02,836] jepsen worker 63 - jepsen.util 6723 :invoke :read [21 nil] INFO [2019-12-17 12:34:02,868] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:34:02,868] jepsen nemesis - couchbase.util Rebalance check count: 349 INFO [2019-12-17 12:34:02,868] jepsen nemesis - couchbase.util Rebalance stuck count: 330 INFO [2019-12-17 12:34:02,874] jepsen worker 52 - jepsen.util 8062 :ok :read [17 17] INFO [2019-12-17 12:34:02,874] jepsen worker 52 - jepsen.util 8062 :invoke :cas [17 [46 39]] INFO [2019-12-17 12:34:02,907] jepsen worker 49 - jepsen.util 10219 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:34:02,907] jepsen worker 45 - jepsen.util 7785 :info :write [15 8] :RequestCanceledException INFO [2019-12-17 12:34:02,907] jepsen worker 49 - jepsen.util 10219 :invoke :read [16 nil] INFO [2019-12-17 12:34:02,910] jepsen worker 65 - jepsen.util 6455 :fail :cas [21 [44 1]] :ValueNotSwapFrom INFO [2019-12-17 12:34:02,910] jepsen worker 65 - jepsen.util 6455 :invoke :read [21 nil] INFO [2019-12-17 12:34:02,917] jepsen worker 19 - jepsen.util 7939 :ok :read [6 35] INFO [2019-12-17 12:34:02,917] jepsen worker 19 - jepsen.util 7939 :invoke :read [6 nil] INFO [2019-12-17 12:34:02,921] jepsen worker 45 - jepsen.util 7875 :invoke :write [15 26] INFO [2019-12-17 12:34:02,929] jepsen worker 7 - jepsen.util 10717 :info :write [2 35] :RequestCanceledException INFO [2019-12-17 12:34:02,929] jepsen worker 2 - jepsen.util 10172 :info :write [0 35] :RequestCanceledException INFO [2019-12-17 12:34:02,935] jepsen worker 8 - jepsen.util 10538 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:34:02,935] jepsen worker 8 - jepsen.util 10538 :invoke :cas [2 [22 26]] INFO [2019-12-17 12:34:02,936] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45210,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:34:02,936] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45210,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:34:02,940] jepsen worker 2 - jepsen.util 10262 :invoke :read [0 nil] INFO [2019-12-17 12:34:02,941] jepsen worker 7 - jepsen.util 10807 :invoke :write [2 42] INFO [2019-12-17 12:34:02,941] jepsen worker 43 - jepsen.util 10213 :info :cas [14 [43 25]] :RequestCanceledException INFO [2019-12-17 12:34:02,951] jepsen worker 18 - jepsen.util 7938 :ok :read [6 35] INFO [2019-12-17 12:34:02,952] jepsen worker 18 - jepsen.util 7938 :invoke :read [6 nil] INFO [2019-12-17 12:34:02,956] jepsen worker 77 - jepsen.util 10967 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:34:02,956] jepsen worker 77 - jepsen.util 10967 :invoke :read [25 nil] INFO [2019-12-17 12:34:02,961] jepsen worker 70 - jepsen.util 8080 :ok :write [23 29] INFO [2019-12-17 12:34:02,962] jepsen worker 70 - jepsen.util 8080 :invoke :write [23 34] INFO [2019-12-17 12:34:02,968] jepsen worker 43 - jepsen.util 10303 :invoke :write [14 8] INFO [2019-12-17 12:34:02,970] jepsen worker 3 - jepsen.util 7743 :ok :read [1 46] INFO [2019-12-17 12:34:02,971] jepsen worker 3 - jepsen.util 7743 :invoke :cas [1 [23 16]] INFO [2019-12-17 12:34:02,974] jepsen worker 29 - jepsen.util 6329 :fail :cas [9 [19 11]] :ValueNotSwapFrom INFO [2019-12-17 12:34:02,974] jepsen worker 29 - jepsen.util 6329 :invoke :write [9 1] INFO [2019-12-17 12:34:02,983] jepsen worker 32 - jepsen.util 7862 :fail :cas [10 [35 24]] :ValueNotSwapFrom INFO [2019-12-17 12:34:02,984] jepsen worker 32 - jepsen.util 7862 :invoke :cas [10 [20 40]] INFO [2019-12-17 12:34:03,005] jepsen worker 1 - jepsen.util 10801 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:34:03,006] jepsen worker 1 - jepsen.util 10801 :invoke :cas [0 [15 31]] INFO [2019-12-17 12:34:03,008] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45227,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:34:03,010] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45227,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:34:03,011] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45229,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:34:03,011] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45229,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:34:03,011] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45231,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:34:03,011] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45231,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:34:03,022] jepsen worker 31 - jepsen.util 7051 :fail :cas [10 [16 4]] :ValueNotSwapFrom INFO [2019-12-17 12:34:03,022] jepsen worker 31 - jepsen.util 7051 :invoke :read [10 nil] INFO [2019-12-17 12:34:03,028] jepsen worker 75 - jepsen.util 10245 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:34:03,029] jepsen worker 75 - jepsen.util 10245 :invoke :write [25 4] INFO [2019-12-17 12:34:03,057] jepsen worker 63 - jepsen.util 6723 :ok :read [21 11] INFO [2019-12-17 12:34:03,057] jepsen worker 63 - jepsen.util 6723 :invoke :write [21 0] INFO [2019-12-17 12:34:03,060] jepsen worker 72 - jepsen.util 7542 :info :write [24 39] :RequestCanceledException INFO [2019-12-17 12:34:03,073] jepsen worker 72 - jepsen.util 7632 :invoke :read [24 nil] INFO [2019-12-17 12:34:03,146] jepsen worker 54 - jepsen.util 8604 :ok :read [18 36] INFO [2019-12-17 12:34:03,147] jepsen worker 54 - jepsen.util 8604 :invoke :read [18 nil] INFO [2019-12-17 12:34:03,149] jepsen worker 79 - jepsen.util 7909 :ok :write [26 4] INFO [2019-12-17 12:34:03,149] jepsen worker 79 - jepsen.util 7909 :invoke :read [26 nil] INFO [2019-12-17 12:34:03,149] jepsen worker 87 - jepsen.util 7107 :ok :write [29 36] INFO [2019-12-17 12:34:03,150] jepsen worker 87 - jepsen.util 7107 :invoke :write [29 0] INFO [2019-12-17 12:34:03,177] jepsen worker 36 - jepsen.util 8046 :ok :write [12 28] INFO [2019-12-17 12:34:03,177] jepsen worker 36 - jepsen.util 8046 :invoke :cas [12 [10 11]] INFO [2019-12-17 12:34:03,193] jepsen worker 60 - jepsen.util 11670 :info :cas [20 [35 18]] :RequestCanceledException INFO [2019-12-17 12:34:03,210] jepsen worker 63 - jepsen.util 6723 :ok :write [21 0] INFO [2019-12-17 12:34:03,210] jepsen worker 63 - jepsen.util 6723 :invoke :read [21 nil] INFO [2019-12-17 12:34:03,215] jepsen worker 60 - jepsen.util 11760 :invoke :cas [20 [42 17]] INFO [2019-12-17 12:34:03,225] jepsen worker 23 - jepsen.util 10553 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:34:03,225] jepsen worker 23 - jepsen.util 10553 :invoke :cas [7 [35 15]] INFO [2019-12-17 12:34:03,226] jepsen worker 39 - jepsen.util 10299 :info :write [13 36] :RequestCanceledException INFO [2019-12-17 12:34:03,231] jepsen worker 37 - jepsen.util 7147 :info :write [12 45] :RequestCanceledException INFO [2019-12-17 12:34:03,244] jepsen worker 39 - jepsen.util 10389 :invoke :write [13 43] INFO [2019-12-17 12:34:03,248] jepsen worker 37 - jepsen.util 7237 :invoke :read [12 nil] INFO [2019-12-17 12:34:03,259] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45243,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:34:03,260] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45243,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:34:03,269] jepsen worker 52 - jepsen.util 8062 :fail :cas [17 [46 39]] :ValueNotSwapFrom INFO [2019-12-17 12:34:03,269] jepsen worker 52 - jepsen.util 8062 :invoke :read [17 nil] INFO [2019-12-17 12:34:03,290] jepsen worker 20 - jepsen.util 7310 :ok :write [6 17] INFO [2019-12-17 12:34:03,290] jepsen worker 20 - jepsen.util 7310 :invoke :write [6 1] INFO [2019-12-17 12:34:03,295] jepsen worker 21 - jepsen.util 10101 :info :cas [7 [37 47]] :RequestCanceledException INFO [2019-12-17 12:34:03,298] jepsen worker 21 - jepsen.util 10191 :invoke :write [7 25] INFO [2019-12-17 12:34:03,300] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45248,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:34:03,301] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45248,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:34:03,301] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45252,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:34:03,301] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45252,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:34:03,301] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45255,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:34:03,301] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45255,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:34:03,308] jepsen worker 59 - jepsen.util 10319 :info :cas [19 [16 24]] :RequestCanceledException INFO [2019-12-17 12:34:03,313] jepsen worker 3 - jepsen.util 7743 :fail :cas [1 [23 16]] :ValueNotSwapFrom INFO [2019-12-17 12:34:03,314] jepsen worker 3 - jepsen.util 7743 :invoke :write [1 47] INFO [2019-12-17 12:34:03,314] jepsen worker 59 - jepsen.util 10409 :invoke :cas [19 [36 0]] INFO [2019-12-17 12:34:03,325] jepsen worker 81 - jepsen.util 10341 :info :write [27 1] :RequestCanceledException INFO [2019-12-17 12:34:03,331] jepsen worker 81 - jepsen.util 10431 :invoke :cas [27 [8 20]] INFO [2019-12-17 12:34:03,340] jepsen worker 38 - jepsen.util 8048 :fail :cas [12 [29 11]] :ValueNotSwapFrom INFO [2019-12-17 12:34:03,341] jepsen worker 38 - jepsen.util 8048 :invoke :write [12 22] INFO [2019-12-17 12:34:03,345] jepsen worker 82 - jepsen.util 11512 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:34:03,345] jepsen worker 82 - jepsen.util 11512 :invoke :read [27 nil] INFO [2019-12-17 12:34:03,346] jepsen worker 51 - jepsen.util 7431 :fail :cas [17 [28 11]] :ValueNotSwapFrom INFO [2019-12-17 12:34:03,346] jepsen worker 51 - jepsen.util 7431 :invoke :cas [17 [26 28]] INFO [2019-12-17 12:34:03,362] jepsen worker 50 - jepsen.util 11480 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:34:03,362] jepsen worker 68 - jepsen.util 10058 :info :cas [22 [45 0]] :RequestCanceledException INFO [2019-12-17 12:34:03,362] jepsen worker 50 - jepsen.util 11480 :invoke :read [16 nil] INFO [2019-12-17 12:34:03,385] jepsen worker 68 - jepsen.util 10148 :invoke :read [22 nil] INFO [2019-12-17 12:34:03,401] jepsen worker 74 - jepsen.util 8174 :ok :write [24 30] INFO [2019-12-17 12:34:03,401] jepsen worker 74 - jepsen.util 8174 :invoke :cas [24 [11 2]] INFO [2019-12-17 12:34:03,405] jepsen worker 11 - jepsen.util 7931 :ok :write [3 27] INFO [2019-12-17 12:34:03,405] jepsen worker 11 - jepsen.util 7931 :invoke :read [3 nil] INFO [2019-12-17 12:34:03,407] jepsen worker 86 - jepsen.util 11246 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:34:03,407] jepsen worker 86 - jepsen.util 11246 :invoke :read [28 nil] INFO [2019-12-17 12:34:03,412] jepsen worker 9 - jepsen.util 7929 :info :write [3 21] :RequestCanceledException INFO [2019-12-17 12:34:03,414] jepsen worker 9 - jepsen.util 8019 :invoke :write [3 43] INFO [2019-12-17 12:34:03,418] jepsen worker 63 - jepsen.util 6723 :ok :read [21 0] INFO [2019-12-17 12:34:03,418] jepsen worker 63 - jepsen.util 6723 :invoke :cas [21 [41 11]] INFO [2019-12-17 12:34:03,426] jepsen worker 41 - jepsen.util 11201 :info :cas [13 [28 38]] :RequestCanceledException INFO [2019-12-17 12:34:03,434] jepsen worker 41 - jepsen.util 11291 :invoke :write [13 27] INFO [2019-12-17 12:34:03,442] jepsen worker 76 - jepsen.util 10426 :info :cas [25 [24 30]] :RequestCanceledException INFO [2019-12-17 12:34:03,444] jepsen worker 56 - jepsen.util 8876 :fail :cas [18 [33 38]] :ValueNotSwapFrom INFO [2019-12-17 12:34:03,444] jepsen worker 56 - jepsen.util 8876 :invoke :cas [18 [1 18]] INFO [2019-12-17 12:34:03,446] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45280,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:34:03,446] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45280,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:34:03,447] jepsen worker 76 - jepsen.util 10516 :invoke :write [25 23] INFO [2019-12-17 12:34:03,454] jepsen worker 64 - jepsen.util 6364 :ok :write [21 2] INFO [2019-12-17 12:34:03,455] jepsen worker 64 - jepsen.util 6364 :invoke :cas [21 [29 27]] INFO [2019-12-17 12:34:03,482] jepsen worker 78 - jepsen.util 7728 :ok :read [26 4] INFO [2019-12-17 12:34:03,484] jepsen worker 37 - jepsen.util 7237 :ok :read [12 28] INFO [2019-12-17 12:34:03,484] jepsen worker 37 - jepsen.util 7237 :invoke :write [12 8] INFO [2019-12-17 12:34:03,485] jepsen worker 19 - jepsen.util 7939 :ok :read [6 17] INFO [2019-12-17 12:34:03,485] jepsen worker 19 - jepsen.util 7939 :invoke :read [6 nil] INFO [2019-12-17 12:34:03,485] jepsen worker 78 - jepsen.util 7728 :invoke :read [26 nil] INFO [2019-12-17 12:34:03,487] jepsen worker 83 - jepsen.util 11063 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:34:03,487] jepsen worker 83 - jepsen.util 11063 :invoke :read [27 nil] INFO [2019-12-17 12:34:03,509] jepsen worker 10 - jepsen.util 7750 :info :write [3 14] :RequestCanceledException INFO [2019-12-17 12:34:03,511] jepsen worker 10 - jepsen.util 7840 :invoke :write [3 41] INFO [2019-12-17 12:34:03,526] jepsen worker 18 - jepsen.util 7938 :ok :read [6 17] INFO [2019-12-17 12:34:03,526] jepsen worker 18 - jepsen.util 7938 :invoke :cas [6 [32 40]] INFO [2019-12-17 12:34:03,529] jepsen worker 88 - jepsen.util 7738 :info :write [29 45] :RequestCanceledException INFO [2019-12-17 12:34:03,531] jepsen worker 88 - jepsen.util 7828 :invoke :read [29 nil] INFO [2019-12-17 12:34:03,544] jepsen worker 16 - jepsen.util 11896 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:34:03,545] jepsen worker 16 - jepsen.util 11896 :invoke :cas [5 [8 22]] INFO [2019-12-17 12:34:03,579] jepsen worker 54 - jepsen.util 8604 :ok :read [18 3] INFO [2019-12-17 12:34:03,579] jepsen worker 54 - jepsen.util 8604 :invoke :cas [18 [7 40]] INFO [2019-12-17 12:34:03,589] jepsen worker 36 - jepsen.util 8046 :fail :cas [12 [10 11]] :ValueNotSwapFrom INFO [2019-12-17 12:34:03,590] jepsen worker 36 - jepsen.util 8046 :invoke :write [12 0] INFO [2019-12-17 12:34:03,608] jepsen worker 63 - jepsen.util 6723 :fail :cas [21 [41 11]] :ValueNotSwapFrom INFO [2019-12-17 12:34:03,609] jepsen worker 63 - jepsen.util 6723 :invoke :read [21 nil] INFO [2019-12-17 12:34:03,616] jepsen worker 51 - jepsen.util 7431 :fail :cas [17 [26 28]] :ValueNotSwapFrom INFO [2019-12-17 12:34:03,616] jepsen worker 51 - jepsen.util 7431 :invoke :read [17 nil] INFO [2019-12-17 12:34:03,625] jepsen worker 28 - jepsen.util 6148 :ok :write [9 6] INFO [2019-12-17 12:34:03,625] jepsen worker 28 - jepsen.util 6148 :invoke :read [9 nil] INFO [2019-12-17 12:34:03,629] jepsen worker 34 - jepsen.util 10294 :info :write [11 35] :RequestCanceledException INFO [2019-12-17 12:34:03,632] jepsen worker 34 - jepsen.util 10384 :invoke :write [11 25] INFO [2019-12-17 12:34:03,657] jepsen worker 52 - jepsen.util 8062 :ok :read [17 17] INFO [2019-12-17 12:34:03,657] jepsen worker 52 - jepsen.util 8062 :invoke :read [17 nil] INFO [2019-12-17 12:34:03,659] jepsen worker 31 - jepsen.util 7051 :ok :read [10 17] INFO [2019-12-17 12:34:03,660] jepsen worker 31 - jepsen.util 7051 :invoke :cas [10 [40 33]] INFO [2019-12-17 12:34:03,693] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45300,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:34:03,694] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45300,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:34:03,697] jepsen worker 63 - jepsen.util 6723 :ok :read [21 2] INFO [2019-12-17 12:34:03,697] jepsen worker 63 - jepsen.util 6723 :invoke :read [21 nil] INFO [2019-12-17 12:34:03,699] jepsen worker 11 - jepsen.util 7931 :ok :read [3 27] INFO [2019-12-17 12:34:03,699] jepsen worker 11 - jepsen.util 7931 :invoke :read [3 nil] INFO [2019-12-17 12:34:03,706] jepsen worker 55 - jepsen.util 8605 :ok :read [18 3] INFO [2019-12-17 12:34:03,707] jepsen worker 55 - jepsen.util 8605 :invoke :read [18 nil] INFO [2019-12-17 12:34:03,712] jepsen worker 72 - jepsen.util 7632 :ok :read [24 30] INFO [2019-12-17 12:34:03,712] jepsen worker 72 - jepsen.util 7632 :invoke :read [24 nil] INFO [2019-12-17 12:34:03,718] jepsen worker 74 - jepsen.util 8174 :fail :cas [24 [11 2]] :ValueNotSwapFrom INFO [2019-12-17 12:34:03,718] jepsen worker 74 - jepsen.util 8174 :invoke :write [24 18] INFO [2019-12-17 12:34:03,743] jepsen worker 67 - jepsen.util 10687 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:34:03,743] jepsen worker 67 - jepsen.util 10687 :invoke :cas [22 [25 49]] INFO [2019-12-17 12:34:03,750] jepsen worker 19 - jepsen.util 7939 :ok :read [6 17] INFO [2019-12-17 12:34:03,750] jepsen worker 19 - jepsen.util 7939 :invoke :read [6 nil] INFO [2019-12-17 12:34:03,757] jepsen worker 35 - jepsen.util 10385 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:34:03,757] jepsen worker 35 - jepsen.util 10385 :invoke :cas [11 [22 40]] INFO [2019-12-17 12:34:03,780] jepsen worker 18 - jepsen.util 7938 :fail :cas [6 [32 40]] :ValueNotSwapFrom INFO [2019-12-17 12:34:03,780] jepsen worker 18 - jepsen.util 7938 :invoke :read [6 nil] INFO [2019-12-17 12:34:03,810] jepsen worker 63 - jepsen.util 6723 :ok :read [21 2] INFO [2019-12-17 12:34:03,810] jepsen worker 63 - jepsen.util 6723 :invoke :write [21 11] INFO [2019-12-17 12:34:03,831] jepsen worker 11 - jepsen.util 7931 :ok :read [3 27] INFO [2019-12-17 12:34:03,832] jepsen worker 11 - jepsen.util 7931 :invoke :read [3 nil] INFO [2019-12-17 12:34:03,835] jepsen worker 19 - jepsen.util 7939 :ok :read [6 17] INFO [2019-12-17 12:34:03,836] jepsen worker 19 - jepsen.util 7939 :invoke :cas [6 [6 39]] INFO [2019-12-17 12:34:03,901] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:34:03,901] jepsen nemesis - couchbase.util Rebalance check count: 350 INFO [2019-12-17 12:34:03,901] jepsen nemesis - couchbase.util Rebalance stuck count: 331 INFO [2019-12-17 12:34:03,907] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45322,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:34:03,907] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45322,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:34:03,907] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45325,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:34:03,907] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45325,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:34:03,927] jepsen worker 11 - jepsen.util 7931 :ok :read [3 27] INFO [2019-12-17 12:34:03,927] jepsen worker 11 - jepsen.util 7931 :invoke :write [3 8] INFO [2019-12-17 12:34:03,930] jepsen worker 18 - jepsen.util 7938 :ok :read [6 17] INFO [2019-12-17 12:34:03,930] jepsen worker 18 - jepsen.util 7938 :invoke :cas [6 [21 4]] INFO [2019-12-17 12:34:03,947] jepsen worker 56 - jepsen.util 8876 :fail :cas [18 [1 18]] :ValueNotSwapFrom INFO [2019-12-17 12:34:03,947] jepsen worker 56 - jepsen.util 8876 :invoke :cas [18 [5 25]] INFO [2019-12-17 12:34:03,951] jepsen worker 78 - jepsen.util 7728 :ok :read [26 4] INFO [2019-12-17 12:34:03,952] jepsen worker 78 - jepsen.util 7728 :invoke :read [26 nil] INFO [2019-12-17 12:34:03,953] jepsen worker 19 - jepsen.util 7939 :fail :cas [6 [6 39]] :ValueNotSwapFrom INFO [2019-12-17 12:34:03,954] jepsen worker 19 - jepsen.util 7939 :invoke :cas [6 [46 20]] INFO [2019-12-17 12:34:03,971] jepsen worker 18 - jepsen.util 7938 :fail :cas [6 [21 4]] :ValueNotSwapFrom INFO [2019-12-17 12:34:03,971] jepsen worker 18 - jepsen.util 7938 :invoke :cas [6 [48 19]] INFO [2019-12-17 12:34:03,980] jepsen worker 71 - jepsen.util 8351 :ok :read [23 29] INFO [2019-12-17 12:34:03,980] jepsen worker 71 - jepsen.util 8351 :invoke :read [23 nil] INFO [2019-12-17 12:34:03,984] jepsen worker 14 - jepsen.util 7934 :fail :cas [4 [22 7]] :ValueNotSwapFrom INFO [2019-12-17 12:34:03,984] jepsen worker 14 - jepsen.util 7934 :invoke :read [4 nil] INFO [2019-12-17 12:34:04,019] jepsen worker 63 - jepsen.util 6723 :ok :write [21 11] INFO [2019-12-17 12:34:04,019] jepsen worker 63 - jepsen.util 6723 :invoke :cas [21 [40 2]] INFO [2019-12-17 12:34:04,062] jepsen worker 13 - jepsen.util 7303 :ok :read [4 47] INFO [2019-12-17 12:34:04,063] jepsen worker 13 - jepsen.util 7303 :invoke :cas [4 [30 25]] INFO [2019-12-17 12:34:04,087] jepsen worker 63 - jepsen.util 6723 :fail :cas [21 [40 2]] :ValueNotSwapFrom INFO [2019-12-17 12:34:04,087] jepsen worker 63 - jepsen.util 6723 :invoke :cas [21 [5 43]] INFO [2019-12-17 12:34:04,101] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45335,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:34:04,104] jepsen worker 20 - jepsen.util 7310 :ok :write [6 1] INFO [2019-12-17 12:34:04,121] jepsen worker 74 - jepsen.util 8174 :ok :write [24 18] INFO [2019-12-17 12:34:04,128] jepsen worker 25 - jepsen.util 11185 :info :write [8 28] :RequestCanceledException INFO [2019-12-17 12:34:04,128] jepsen worker 58 - jepsen.util 11578 :info :write [19 1] :RequestCanceledException INFO [2019-12-17 12:34:04,140] jepsen worker 88 - jepsen.util 7828 :ok :read [29 36] INFO [2019-12-17 12:34:04,143] jepsen worker 62 - jepsen.util 10862 :info :write [20 12] :RequestCanceledException INFO [2019-12-17 12:34:04,144] jepsen worker 79 - jepsen.util 7909 :ok :read [26 4] INFO [2019-12-17 12:34:04,149] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45335,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:34:04,149] jepsen worker 74 - jepsen.util 8174 :invoke :cas [24 [7 8]] INFO [2019-12-17 12:34:04,149] jepsen worker 20 - jepsen.util 7310 :invoke :cas [6 [44 29]] INFO [2019-12-17 12:34:04,149] jepsen worker 88 - jepsen.util 7828 :invoke :read [29 nil] INFO [2019-12-17 12:34:04,149] jepsen worker 79 - jepsen.util 7909 :invoke :write [26 18] INFO [2019-12-17 12:34:04,149] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45338,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:34:04,150] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45338,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:34:04,154] jepsen worker 62 - jepsen.util 10952 :invoke :read [20 nil] INFO [2019-12-17 12:34:04,154] jepsen worker 25 - jepsen.util 11275 :invoke :read [8 nil] INFO [2019-12-17 12:34:04,155] jepsen worker 58 - jepsen.util 11668 :invoke :write [19 34] INFO [2019-12-17 12:34:04,161] jepsen worker 33 - jepsen.util 10383 :info :write [11 18] :RequestCanceledException INFO [2019-12-17 12:34:04,166] jepsen worker 33 - jepsen.util 10473 :invoke :write [11 41] INFO [2019-12-17 12:34:04,168] jepsen worker 18 - jepsen.util 7938 :fail :cas [6 [48 19]] :ValueNotSwapFrom INFO [2019-12-17 12:34:04,169] jepsen worker 18 - jepsen.util 7938 :invoke :write [6 40] INFO [2019-12-17 12:34:04,178] jepsen worker 19 - jepsen.util 7939 :fail :cas [6 [46 20]] :ValueNotSwapFrom INFO [2019-12-17 12:34:04,178] jepsen worker 63 - jepsen.util 6723 :fail :cas [21 [5 43]] :ValueNotSwapFrom INFO [2019-12-17 12:34:04,179] jepsen worker 19 - jepsen.util 7939 :invoke :read [6 nil] INFO [2019-12-17 12:34:04,179] jepsen worker 63 - jepsen.util 6723 :invoke :write [21 46] INFO [2019-12-17 12:34:04,179] jepsen worker 5 - jepsen.util 7295 :ok :read [1 46] INFO [2019-12-17 12:34:04,181] jepsen worker 5 - jepsen.util 7295 :invoke :write [1 15] INFO [2019-12-17 12:34:04,201] jepsen worker 74 - jepsen.util 8174 :fail :cas [24 [7 8]] :ValueNotSwapFrom INFO [2019-12-17 12:34:04,201] jepsen worker 74 - jepsen.util 8174 :invoke :cas [24 [29 47]] INFO [2019-12-17 12:34:04,213] jepsen worker 0 - jepsen.util 11070 :info :write [0 34] :RequestCanceledException INFO [2019-12-17 12:34:04,221] jepsen worker 0 - jepsen.util 11160 :invoke :cas [0 [14 31]] INFO [2019-12-17 12:34:04,229] jepsen worker 4 - jepsen.util 7204 :info :write [1 24] :RequestCanceledException INFO [2019-12-17 12:34:04,229] jepsen worker 57 - jepsen.util 10677 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:34:04,229] jepsen worker 57 - jepsen.util 10677 :invoke :write [19 16] INFO [2019-12-17 12:34:04,232] jepsen worker 4 - jepsen.util 7294 :invoke :write [1 44] INFO [2019-12-17 12:34:04,234] jepsen worker 64 - jepsen.util 6364 :fail :cas [21 [29 27]] :ValueNotSwapFrom INFO [2019-12-17 12:34:04,234] jepsen worker 64 - jepsen.util 6364 :invoke :read [21 nil] INFO [2019-12-17 12:34:04,243] jepsen worker 85 - jepsen.util 11425 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:34:04,243] jepsen worker 85 - jepsen.util 11425 :invoke :write [28 11] INFO [2019-12-17 12:34:04,256] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45354,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:34:04,258] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45354,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:34:04,258] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45356,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:34:04,258] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45356,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:34:04,267] jepsen worker 78 - jepsen.util 7728 :ok :read [26 4] INFO [2019-12-17 12:34:04,268] jepsen worker 78 - jepsen.util 7728 :invoke :cas [26 [8 44]] INFO [2019-12-17 12:34:04,280] jepsen worker 55 - jepsen.util 8605 :ok :read [18 3] INFO [2019-12-17 12:34:04,281] jepsen worker 55 - jepsen.util 8605 :invoke :write [18 9] INFO [2019-12-17 12:34:04,290] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45362,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:34:04,291] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45362,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:34:04,291] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45366,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:34:04,291] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45366,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:34:04,301] jepsen worker 31 - jepsen.util 7051 :fail :cas [10 [40 33]] :ValueNotSwapFrom INFO [2019-12-17 12:34:04,302] jepsen worker 31 - jepsen.util 7051 :invoke :cas [10 [29 47]] INFO [2019-12-17 12:34:04,302] jepsen worker 63 - jepsen.util 6723 :ok :write [21 46] INFO [2019-12-17 12:34:04,303] jepsen worker 63 - jepsen.util 6723 :invoke :cas [21 [9 18]] INFO [2019-12-17 12:34:04,308] jepsen worker 80 - jepsen.util 7550 :info :write [26 36] :RequestCanceledException INFO [2019-12-17 12:34:04,311] jepsen worker 80 - jepsen.util 7640 :invoke :write [26 19] INFO [2019-12-17 12:34:04,314] jepsen worker 74 - jepsen.util 8174 :fail :cas [24 [29 47]] :ValueNotSwapFrom INFO [2019-12-17 12:34:04,315] jepsen worker 74 - jepsen.util 8174 :invoke :read [24 nil] INFO [2019-12-17 12:34:04,343] jepsen worker 20 - jepsen.util 7310 :fail :cas [6 [44 29]] :ValueNotSwapFrom INFO [2019-12-17 12:34:04,344] jepsen worker 20 - jepsen.util 7310 :invoke :read [6 nil] INFO [2019-12-17 12:34:04,352] jepsen worker 71 - jepsen.util 8351 :ok :read [23 29] INFO [2019-12-17 12:34:04,352] jepsen worker 71 - jepsen.util 8351 :invoke :write [23 39] INFO [2019-12-17 12:34:04,364] jepsen worker 63 - jepsen.util 6723 :fail :cas [21 [9 18]] :ValueNotSwapFrom INFO [2019-12-17 12:34:04,364] jepsen worker 63 - jepsen.util 6723 :invoke :read [21 nil] INFO [2019-12-17 12:34:04,367] jepsen worker 72 - jepsen.util 7632 :ok :read [24 18] INFO [2019-12-17 12:34:04,368] jepsen worker 64 - jepsen.util 6364 :ok :read [21 11] INFO [2019-12-17 12:34:04,368] jepsen worker 64 - jepsen.util 6364 :invoke :read [21 nil] INFO [2019-12-17 12:34:04,368] jepsen worker 72 - jepsen.util 7632 :invoke :write [24 17] INFO [2019-12-17 12:34:04,396] jepsen worker 47 - jepsen.util 8147 :info :write [15 1] :RequestCanceledException INFO [2019-12-17 12:34:04,401] jepsen worker 47 - jepsen.util 8237 :invoke :cas [15 [21 43]] INFO [2019-12-17 12:34:04,408] jepsen worker 53 - jepsen.util 7253 :info :write [17 12] :RequestCanceledException INFO [2019-12-17 12:34:04,414] jepsen worker 53 - jepsen.util 7343 :invoke :read [17 nil] INFO [2019-12-17 12:34:04,421] jepsen worker 54 - jepsen.util 8604 :fail :cas [18 [7 40]] :ValueNotSwapFrom INFO [2019-12-17 12:34:04,422] jepsen worker 54 - jepsen.util 8604 :invoke :write [18 8] INFO [2019-12-17 12:34:04,443] jepsen worker 6 - jepsen.util 11256 :info :write [2 16] :RequestCanceledException INFO [2019-12-17 12:34:04,445] jepsen worker 74 - jepsen.util 8174 :ok :read [24 18] INFO [2019-12-17 12:34:04,446] jepsen worker 74 - jepsen.util 8174 :invoke :write [24 4] INFO [2019-12-17 12:34:04,447] jepsen worker 6 - jepsen.util 11346 :invoke :write [2 41] INFO [2019-12-17 12:34:04,448] jepsen worker 31 - jepsen.util 7051 :fail :cas [10 [29 47]] :ValueNotSwapFrom INFO [2019-12-17 12:34:04,449] jepsen worker 31 - jepsen.util 7051 :invoke :cas [10 [16 5]] INFO [2019-12-17 12:34:04,454] jepsen worker 20 - jepsen.util 7310 :ok :read [6 1] INFO [2019-12-17 12:34:04,455] jepsen worker 20 - jepsen.util 7310 :invoke :cas [6 [3 1]] INFO [2019-12-17 12:34:04,460] jepsen worker 84 - jepsen.util 10254 :info :write [28 25] :RequestCanceledException INFO [2019-12-17 12:34:04,462] jepsen worker 84 - jepsen.util 10344 :invoke :cas [28 [33 13]] INFO [2019-12-17 12:34:04,472] jepsen worker 14 - jepsen.util 7934 :ok :read [4 47] INFO [2019-12-17 12:34:04,472] jepsen worker 14 - jepsen.util 7934 :invoke :write [4 21] INFO [2019-12-17 12:34:04,487] jepsen worker 64 - jepsen.util 6364 :ok :read [21 46] INFO [2019-12-17 12:34:04,488] jepsen worker 64 - jepsen.util 6364 :invoke :cas [21 [24 21]] INFO [2019-12-17 12:34:04,495] jepsen worker 40 - jepsen.util 11290 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:34:04,496] jepsen worker 40 - jepsen.util 11290 :invoke :cas [13 [30 12]] INFO [2019-12-17 12:34:04,510] jepsen worker 11 - jepsen.util 7931 :ok :write [3 8] INFO [2019-12-17 12:34:04,510] jepsen worker 11 - jepsen.util 7931 :invoke :write [3 26] INFO [2019-12-17 12:34:04,511] jepsen worker 47 - jepsen.util 8237 :fail :cas [15 [21 43]] :ValueNotSwapFrom INFO [2019-12-17 12:34:04,512] jepsen worker 47 - jepsen.util 8237 :invoke :read [15 nil] INFO [2019-12-17 12:34:04,513] jepsen worker 61 - jepsen.util 11401 :info :cas [20 [30 47]] :RequestCanceledException INFO [2019-12-17 12:34:04,517] jepsen worker 61 - jepsen.util 11491 :invoke :cas [20 [47 46]] INFO [2019-12-17 12:34:04,519] jepsen worker 63 - jepsen.util 6723 :ok :read [21 46] INFO [2019-12-17 12:34:04,519] jepsen worker 63 - jepsen.util 6723 :invoke :write [21 43] INFO [2019-12-17 12:34:04,522] jepsen worker 56 - jepsen.util 8876 :fail :cas [18 [5 25]] :ValueNotSwapFrom INFO [2019-12-17 12:34:04,523] jepsen worker 56 - jepsen.util 8876 :invoke :read [18 nil] INFO [2019-12-17 12:34:04,529] jepsen worker 66 - jepsen.util 10506 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:34:04,529] jepsen worker 26 - jepsen.util 11186 :info :cas [8 [45 15]] :RequestCanceledException INFO [2019-12-17 12:34:04,530] jepsen worker 66 - jepsen.util 10506 :invoke :write [22 3] INFO [2019-12-17 12:34:04,530] jepsen worker 52 - jepsen.util 8062 :ok :read [17 17] INFO [2019-12-17 12:34:04,530] jepsen worker 52 - jepsen.util 8062 :invoke :read [17 nil] INFO [2019-12-17 12:34:04,535] jepsen worker 26 - jepsen.util 11276 :invoke :write [8 3] INFO [2019-12-17 12:34:04,539] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45386,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:34:04,539] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45386,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:34:04,540] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45388,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:34:04,540] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45388,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:34:04,543] jepsen worker 48 - jepsen.util 9678 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:34:04,543] jepsen worker 48 - jepsen.util 9678 :invoke :cas [16 [38 30]] INFO [2019-12-17 12:34:04,560] jepsen worker 27 - jepsen.util 7677 :info :cas [9 [6 37]] :RequestCanceledException INFO [2019-12-17 12:34:04,560] jepsen worker 22 - jepsen.util 10642 :info :write [7 12] :RequestCanceledException INFO [2019-12-17 12:34:04,564] jepsen worker 22 - jepsen.util 10732 :invoke :read [7 nil] INFO [2019-12-17 12:34:04,565] jepsen worker 27 - jepsen.util 7767 :invoke :write [9 38] INFO [2019-12-17 12:34:04,573] jepsen worker 47 - jepsen.util 8237 :ok :read [15 8] INFO [2019-12-17 12:34:04,574] jepsen worker 47 - jepsen.util 8237 :invoke :write [15 9] INFO [2019-12-17 12:34:04,578] jepsen worker 19 - jepsen.util 7939 :ok :read [6 1] INFO [2019-12-17 12:34:04,578] jepsen worker 19 - jepsen.util 7939 :invoke :cas [6 [28 12]] INFO [2019-12-17 12:34:04,636] jepsen worker 51 - jepsen.util 7431 :ok :read [17 12] INFO [2019-12-17 12:34:04,636] jepsen worker 51 - jepsen.util 7431 :invoke :write [17 43] INFO [2019-12-17 12:34:04,642] jepsen worker 44 - jepsen.util 11204 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:34:04,642] jepsen worker 44 - jepsen.util 11204 :invoke :read [14 nil] INFO [2019-12-17 12:34:04,670] jepsen worker 63 - jepsen.util 6723 :ok :write [21 43] INFO [2019-12-17 12:34:04,670] jepsen worker 63 - jepsen.util 6723 :invoke :write [21 46] INFO [2019-12-17 12:34:04,672] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45417,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:34:04,672] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45417,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:34:04,672] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45420,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:34:04,673] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45420,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:34:04,673] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45421,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:34:04,673] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45421,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:34:04,696] jepsen worker 46 - jepsen.util 7966 :info :write [15 45] :RequestCanceledException INFO [2019-12-17 12:34:04,700] jepsen worker 46 - jepsen.util 8056 :invoke :write [15 2] INFO [2019-12-17 12:34:04,716] jepsen worker 13 - jepsen.util 7303 :fail :cas [4 [30 25]] :ValueNotSwapFrom INFO [2019-12-17 12:34:04,717] jepsen worker 13 - jepsen.util 7303 :invoke :write [4 4] INFO [2019-12-17 12:34:04,729] jepsen worker 15 - jepsen.util 9645 :info :write [5 3] :RequestCanceledException INFO [2019-12-17 12:34:04,731] jepsen worker 15 - jepsen.util 9735 :invoke :write [5 4] INFO [2019-12-17 12:34:04,743] jepsen worker 89 - jepsen.util 7559 :info :write [29 31] :RequestCanceledException INFO [2019-12-17 12:34:04,745] jepsen worker 89 - jepsen.util 7649 :invoke :write [29 32] INFO [2019-12-17 12:34:04,777] jepsen worker 63 - jepsen.util 6723 :ok :write [21 46] INFO [2019-12-17 12:34:04,778] jepsen worker 63 - jepsen.util 6723 :invoke :cas [21 [45 10]] INFO [2019-12-17 12:34:04,795] jepsen worker 30 - jepsen.util 7770 :fail :read [10 nil] :RequestTimeoutException INFO [2019-12-17 12:34:04,795] jepsen worker 69 - jepsen.util 7899 :info :write [23 33] :RequestCanceledException INFO [2019-12-17 12:34:04,796] jepsen worker 30 - jepsen.util 7770 :invoke :write [10 27] INFO [2019-12-17 12:34:04,797] jepsen worker 69 - jepsen.util 7989 :invoke :read [23 nil] INFO [2019-12-17 12:34:04,803] jepsen worker 18 - jepsen.util 7938 :ok :write [6 40] INFO [2019-12-17 12:34:04,803] jepsen worker 18 - jepsen.util 7938 :invoke :cas [6 [31 24]] INFO [2019-12-17 12:34:04,803] jepsen worker 72 - jepsen.util 7632 :ok :write [24 17] INFO [2019-12-17 12:34:04,804] jepsen worker 72 - jepsen.util 7632 :invoke :read [24 nil] INFO [2019-12-17 12:34:04,814] jepsen worker 47 - jepsen.util 8237 :ok :write [15 9] INFO [2019-12-17 12:34:04,814] jepsen worker 47 - jepsen.util 8237 :invoke :cas [15 [19 3]] INFO [2019-12-17 12:34:04,843] jepsen worker 24 - jepsen.util 9474 :info :write [8 35] :RequestCanceledException INFO [2019-12-17 12:34:04,853] jepsen worker 24 - jepsen.util 9564 :invoke :write [8 2] INFO [2019-12-17 12:34:04,857] jepsen worker 42 - jepsen.util 11472 :info :cas [14 [11 9]] :RequestCanceledException INFO [2019-12-17 12:34:04,860] jepsen worker 42 - jepsen.util 11562 :invoke :read [14 nil] INFO [2019-12-17 12:34:04,890] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45434,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:34:04,890] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45434,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:34:04,890] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45435,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:34:04,890] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45435,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:34:04,890] jepsen worker 53 - jepsen.util 7343 :ok :read [17 12] INFO [2019-12-17 12:34:04,891] jepsen worker 53 - jepsen.util 7343 :invoke :read [17 nil] INFO [2019-12-17 12:34:04,893] jepsen worker 73 - jepsen.util 7363 :info :write [24 36] :RequestCanceledException INFO [2019-12-17 12:34:04,894] jepsen worker 63 - jepsen.util 6723 :fail :cas [21 [45 10]] :ValueNotSwapFrom INFO [2019-12-17 12:34:04,894] jepsen worker 63 - jepsen.util 6723 :invoke :write [21 14] INFO [2019-12-17 12:34:04,895] jepsen worker 73 - jepsen.util 7453 :invoke :cas [24 [21 9]] INFO [2019-12-17 12:34:04,919] jepsen worker 88 - jepsen.util 7828 :ok :read [29 36] INFO [2019-12-17 12:34:04,919] jepsen worker 88 - jepsen.util 7828 :invoke :cas [29 [15 41]] INFO [2019-12-17 12:34:04,921] jepsen worker 64 - jepsen.util 6364 :fail :cas [21 [24 21]] :ValueNotSwapFrom INFO [2019-12-17 12:34:04,921] jepsen worker 64 - jepsen.util 6364 :invoke :write [21 17] INFO [2019-12-17 12:34:04,929] jepsen worker 12 - jepsen.util 8202 :info :write [4 9] :RequestCanceledException INFO [2019-12-17 12:34:04,929] jepsen worker 17 - jepsen.util 10727 :info :write [5 35] :RequestCanceledException INFO [2019-12-17 12:34:04,929] jepsen worker 65 - jepsen.util 6455 :fail :read [21 nil] :RequestTimeoutException INFO [2019-12-17 12:34:04,929] jepsen worker 65 - jepsen.util 6455 :invoke :write [21 44] INFO [2019-12-17 12:34:04,932] jepsen worker 17 - jepsen.util 10817 :invoke :read [5 nil] INFO [2019-12-17 12:34:04,933] jepsen worker 12 - jepsen.util 8292 :invoke :write [4 35] INFO [2019-12-17 12:34:04,934] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:34:04,934] jepsen nemesis - couchbase.util Rebalance check count: 351 INFO [2019-12-17 12:34:04,934] jepsen nemesis - couchbase.util Rebalance stuck count: 332 INFO [2019-12-17 12:34:04,966] jepsen worker 47 - jepsen.util 8237 :fail :cas [15 [19 3]] :ValueNotSwapFrom INFO [2019-12-17 12:34:04,966] jepsen worker 47 - jepsen.util 8237 :invoke :read [15 nil] INFO [2019-12-17 12:34:04,972] jepsen worker 73 - jepsen.util 7453 :fail :cas [24 [21 9]] :ValueNotSwapFrom INFO [2019-12-17 12:34:04,973] jepsen worker 73 - jepsen.util 7453 :invoke :write [24 46] INFO [2019-12-17 12:34:04,974] jepsen worker 72 - jepsen.util 7632 :ok :read [24 17] INFO [2019-12-17 12:34:04,974] jepsen worker 72 - jepsen.util 7632 :invoke :write [24 42] INFO [2019-12-17 12:34:05,013] jepsen worker 49 - jepsen.util 10219 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:34:05,013] jepsen worker 70 - jepsen.util 8080 :info :write [23 34] :RequestCanceledException INFO [2019-12-17 12:34:05,013] jepsen worker 49 - jepsen.util 10219 :invoke :cas [16 [44 0]] INFO [2019-12-17 12:34:05,014] jepsen worker 45 - jepsen.util 7875 :info :write [15 26] :RequestCanceledException INFO [2019-12-17 12:34:05,017] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45451,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:34:05,017] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45451,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:34:05,017] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45454,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:34:05,017] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45454,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:34:05,017] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45455,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:34:05,017] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45455,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:34:05,029] jepsen worker 8 - jepsen.util 10538 :info :cas [2 [22 26]] :RequestCanceledException INFO [2019-12-17 12:34:05,029] jepsen worker 32 - jepsen.util 7862 :info :cas [10 [20 40]] :RequestCanceledException INFO [2019-12-17 12:34:05,029] jepsen worker 2 - jepsen.util 10262 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:34:05,029] jepsen worker 2 - jepsen.util 10262 :invoke :cas [0 [1 25]] INFO [2019-12-17 12:34:05,031] jepsen worker 11 - jepsen.util 7931 :ok :write [3 26] INFO [2019-12-17 12:34:05,031] jepsen worker 11 - jepsen.util 7931 :invoke :cas [3 [22 4]] INFO [2019-12-17 12:34:05,033] jepsen worker 8 - jepsen.util 10628 :invoke :read [2 nil] INFO [2019-12-17 12:34:05,033] jepsen worker 32 - jepsen.util 7952 :invoke :read [10 nil] INFO [2019-12-17 12:34:05,038] jepsen worker 70 - jepsen.util 8170 :invoke :write [23 17] INFO [2019-12-17 12:34:05,043] jepsen worker 7 - jepsen.util 10807 :info :write [2 42] :RequestCanceledException INFO [2019-12-17 12:34:05,043] jepsen worker 29 - jepsen.util 6329 :info :write [9 1] :RequestCanceledException INFO [2019-12-17 12:34:05,045] jepsen worker 29 - jepsen.util 6419 :invoke :read [9 nil] INFO [2019-12-17 12:34:05,045] jepsen worker 7 - jepsen.util 10897 :invoke :cas [2 [28 0]] INFO [2019-12-17 12:34:05,046] jepsen worker 45 - jepsen.util 7965 :invoke :read [15 nil] INFO [2019-12-17 12:34:05,060] jepsen worker 43 - jepsen.util 10303 :info :write [14 8] :RequestCanceledException INFO [2019-12-17 12:34:05,060] jepsen worker 77 - jepsen.util 10967 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:34:05,061] jepsen worker 77 - jepsen.util 10967 :invoke :write [25 29] INFO [2019-12-17 12:34:05,062] jepsen worker 78 - jepsen.util 7728 :fail :cas [26 [8 44]] :ValueNotSwapFrom INFO [2019-12-17 12:34:05,062] jepsen worker 78 - jepsen.util 7728 :invoke :write [26 36] INFO [2019-12-17 12:34:05,065] jepsen worker 43 - jepsen.util 10393 :invoke :read [14 nil] INFO [2019-12-17 12:34:05,081] jepsen worker 47 - jepsen.util 8237 :ok :read [15 9] INFO [2019-12-17 12:34:05,083] jepsen worker 47 - jepsen.util 8237 :invoke :write [15 29] INFO [2019-12-17 12:34:05,108] jepsen worker 1 - jepsen.util 10801 :info :cas [0 [15 31]] :RequestCanceledException INFO [2019-12-17 12:34:05,111] jepsen worker 1 - jepsen.util 10891 :invoke :cas [0 [5 26]] INFO [2019-12-17 12:34:05,128] jepsen worker 64 - jepsen.util 6364 :ok :write [21 17] INFO [2019-12-17 12:34:05,129] jepsen worker 75 - jepsen.util 10245 :info :write [25 4] :RequestCanceledException INFO [2019-12-17 12:34:05,129] jepsen worker 64 - jepsen.util 6364 :invoke :write [21 46] INFO [2019-12-17 12:34:05,131] jepsen worker 75 - jepsen.util 10335 :invoke :read [25 nil] INFO [2019-12-17 12:34:05,156] jepsen worker 11 - jepsen.util 7931 :fail :cas [3 [22 4]] :ValueNotSwapFrom INFO [2019-12-17 12:34:05,156] jepsen worker 11 - jepsen.util 7931 :invoke :write [3 46] INFO [2019-12-17 12:34:05,160] jepsen worker 52 - jepsen.util 8062 :ok :read [17 12] INFO [2019-12-17 12:34:05,160] jepsen worker 52 - jepsen.util 8062 :invoke :write [17 3] INFO [2019-12-17 12:34:05,180] jepsen worker 32 - jepsen.util 7952 :ok :read [10 17] INFO [2019-12-17 12:34:05,181] jepsen worker 32 - jepsen.util 7952 :invoke :write [10 45] INFO [2019-12-17 12:34:05,204] jepsen worker 53 - jepsen.util 7343 :ok :read [17 12] INFO [2019-12-17 12:34:05,204] jepsen worker 53 - jepsen.util 7343 :invoke :write [17 28] INFO [2019-12-17 12:34:05,234] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45478,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:34:05,234] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45478,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:34:05,235] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45479,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:34:05,235] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45479,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:34:05,242] jepsen worker 30 - jepsen.util 7770 :ok :write [10 27] INFO [2019-12-17 12:34:05,242] jepsen worker 30 - jepsen.util 7770 :invoke :cas [10 [44 0]] INFO [2019-12-17 12:34:05,244] jepsen worker 65 - jepsen.util 6455 :ok :write [21 44] INFO [2019-12-17 12:34:05,244] jepsen worker 65 - jepsen.util 6455 :invoke :write [21 7] INFO [2019-12-17 12:34:05,256] jepsen worker 56 - jepsen.util 8876 :ok :read [18 3] INFO [2019-12-17 12:34:05,256] jepsen worker 56 - jepsen.util 8876 :invoke :read [18 nil] INFO [2019-12-17 12:34:05,267] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45484,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:34:05,267] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45484,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:34:05,295] jepsen worker 60 - jepsen.util 11760 :info :cas [20 [42 17]] :RequestCanceledException INFO [2019-12-17 12:34:05,303] jepsen worker 73 - jepsen.util 7453 :ok :write [24 46] INFO [2019-12-17 12:34:05,313] jepsen worker 39 - jepsen.util 10389 :info :write [13 43] :RequestCanceledException INFO [2019-12-17 12:34:05,324] jepsen worker 63 - jepsen.util 6723 :ok :write [21 14] INFO [2019-12-17 12:34:05,328] jepsen worker 23 - jepsen.util 10553 :info :cas [7 [35 15]] :RequestCanceledException INFO [2019-12-17 12:34:05,342] jepsen worker 87 - jepsen.util 7107 :info :write [29 0] :RequestCanceledException INFO [2019-12-17 12:34:05,342] jepsen worker 21 - jepsen.util 10191 :info :write [7 25] :RequestCanceledException INFO [2019-12-17 12:34:05,360] jepsen worker 59 - jepsen.util 10409 :info :cas [19 [36 0]] :RequestCanceledException INFO [2019-12-17 12:34:05,362] jepsen worker 3 - jepsen.util 7743 :info :write [1 47] :RequestCanceledException INFO [2019-12-17 12:34:05,390] jepsen worker 88 - jepsen.util 7828 :fail :cas [29 [15 41]] :ValueNotSwapFrom INFO [2019-12-17 12:34:05,418] jepsen worker 56 - jepsen.util 8876 :ok :read [18 3] INFO [2019-12-17 12:34:05,430] jepsen worker 81 - jepsen.util 10431 :info :cas [27 [8 20]] :RequestCanceledException INFO [2019-12-17 12:34:05,443] jepsen worker 82 - jepsen.util 11512 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:34:05,445] jepsen worker 47 - jepsen.util 8237 :ok :write [15 29] INFO [2019-12-17 12:34:05,452] jepsen worker 11 - jepsen.util 7931 :ok :write [3 46] INFO [2019-12-17 12:34:05,456] jepsen worker 50 - jepsen.util 11480 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:34:05,487] jepsen worker 69 - jepsen.util 7989 :ok :read [23 29] INFO [2019-12-17 12:34:05,495] jepsen worker 68 - jepsen.util 10148 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:34:05,513] jepsen worker 86 - jepsen.util 11246 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:34:05,513] jepsen worker 38 - jepsen.util 8048 :info :write [12 22] :RequestCanceledException INFO [2019-12-17 12:34:05,513] jepsen worker 9 - jepsen.util 8019 :info :write [3 43] :RequestCanceledException INFO [2019-12-17 12:34:05,520] jepsen worker 73 - jepsen.util 7453 :invoke :cas [24 [33 42]] INFO [2019-12-17 12:34:05,521] jepsen worker 88 - jepsen.util 7828 :invoke :read [29 nil] INFO [2019-12-17 12:34:05,521] jepsen worker 56 - jepsen.util 8876 :invoke :cas [18 [48 43]] INFO [2019-12-17 12:34:05,521] jepsen worker 63 - jepsen.util 6723 :invoke :write [21 39] INFO [2019-12-17 12:34:05,521] jepsen worker 82 - jepsen.util 11512 :invoke :read [27 nil] INFO [2019-12-17 12:34:05,522] jepsen worker 50 - jepsen.util 11480 :invoke :cas [16 [3 44]] INFO [2019-12-17 12:34:05,522] jepsen worker 11 - jepsen.util 7931 :invoke :write [3 42] INFO [2019-12-17 12:34:05,522] jepsen worker 69 - jepsen.util 7989 :invoke :cas [23 [3 3]] INFO [2019-12-17 12:34:05,522] jepsen worker 68 - jepsen.util 10148 :invoke :write [22 30] INFO [2019-12-17 12:34:05,522] jepsen worker 86 - jepsen.util 11246 :invoke :write [28 42] INFO [2019-12-17 12:34:05,523] jepsen worker 47 - jepsen.util 8237 :invoke :cas [15 [0 20]] INFO [2019-12-17 12:34:05,523] jepsen worker 3 - jepsen.util 7833 :invoke :cas [1 [1 22]] INFO [2019-12-17 12:34:05,524] jepsen worker 87 - jepsen.util 7197 :invoke :write [29 40] INFO [2019-12-17 12:34:05,524] jepsen worker 21 - jepsen.util 10281 :invoke :read [7 nil] INFO [2019-12-17 12:34:05,526] jepsen worker 23 - jepsen.util 10643 :invoke :read [7 nil] INFO [2019-12-17 12:34:05,526] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45494,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:34:05,526] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45494,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:34:05,526] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45497,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:34:05,527] jepsen worker 9 - jepsen.util 8109 :invoke :read [3 nil] INFO [2019-12-17 12:34:05,527] jepsen worker 41 - jepsen.util 11291 :info :write [13 27] :RequestCanceledException INFO [2019-12-17 12:34:05,527] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45497,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:34:05,527] jepsen worker 38 - jepsen.util 8138 :invoke :write [12 37] INFO [2019-12-17 12:34:05,528] jepsen worker 37 - jepsen.util 7237 :info :write [12 8] :RequestCanceledException INFO [2019-12-17 12:34:05,528] jepsen worker 59 - jepsen.util 10499 :invoke :read [19 nil] INFO [2019-12-17 12:34:05,528] jepsen worker 60 - jepsen.util 11850 :invoke :write [20 42] INFO [2019-12-17 12:34:05,529] jepsen worker 39 - jepsen.util 10479 :invoke :read [13 nil] INFO [2019-12-17 12:34:05,529] jepsen worker 81 - jepsen.util 10521 :invoke :cas [27 [21 36]] INFO [2019-12-17 12:34:05,531] jepsen worker 37 - jepsen.util 7327 :invoke :cas [12 [23 33]] INFO [2019-12-17 12:34:05,534] jepsen worker 41 - jepsen.util 11381 :invoke :read [13 nil] INFO [2019-12-17 12:34:05,543] jepsen worker 76 - jepsen.util 10516 :info :write [25 23] :RequestCanceledException INFO [2019-12-17 12:34:05,549] jepsen worker 76 - jepsen.util 10606 :invoke :write [25 20] INFO [2019-12-17 12:34:05,557] jepsen worker 10 - jepsen.util 7840 :info :write [3 41] :RequestCanceledException INFO [2019-12-17 12:34:05,561] jepsen worker 10 - jepsen.util 7930 :invoke :write [3 26] INFO [2019-12-17 12:34:05,596] jepsen worker 83 - jepsen.util 11063 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:34:05,597] jepsen worker 83 - jepsen.util 11063 :invoke :read [27 nil] INFO [2019-12-17 12:34:05,606] jepsen worker 73 - jepsen.util 7453 :fail :cas [24 [33 42]] :ValueNotSwapFrom INFO [2019-12-17 12:34:05,606] jepsen worker 73 - jepsen.util 7453 :invoke :read [24 nil] INFO [2019-12-17 12:34:05,641] jepsen worker 16 - jepsen.util 11896 :info :cas [5 [8 22]] :RequestCanceledException INFO [2019-12-17 12:34:05,645] jepsen worker 16 - jepsen.util 11986 :invoke :cas [5 [17 4]] INFO [2019-12-17 12:34:05,647] jepsen worker 73 - jepsen.util 7453 :ok :read [24 46] INFO [2019-12-17 12:34:05,647] jepsen worker 73 - jepsen.util 7453 :invoke :write [24 35] INFO [2019-12-17 12:34:05,652] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45526,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:34:05,652] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45526,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:34:05,653] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45530,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:34:05,653] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45530,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:34:05,656] jepsen worker 28 - jepsen.util 6148 :fail :read [9 nil] :RequestTimeoutException INFO [2019-12-17 12:34:05,656] jepsen worker 28 - jepsen.util 6148 :invoke :cas [9 [30 43]] INFO [2019-12-17 12:34:05,657] jepsen worker 63 - jepsen.util 6723 :ok :write [21 39] INFO [2019-12-17 12:34:05,657] jepsen worker 63 - jepsen.util 6723 :invoke :write [21 8] INFO [2019-12-17 12:34:05,697] jepsen worker 34 - jepsen.util 10384 :info :write [11 25] :RequestCanceledException INFO [2019-12-17 12:34:05,702] jepsen worker 36 - jepsen.util 8046 :info :write [12 0] :RequestCanceledException INFO [2019-12-17 12:34:05,732] jepsen worker 63 - jepsen.util 6723 :ok :write [21 8] INFO [2019-12-17 12:34:05,734] jepsen worker 63 - jepsen.util 6723 :invoke :read [21 nil] INFO [2019-12-17 12:34:05,734] jepsen worker 34 - jepsen.util 10474 :invoke :write [11 5] INFO [2019-12-17 12:34:05,736] jepsen worker 36 - jepsen.util 8136 :invoke :read [12 nil] INFO [2019-12-17 12:34:05,819] jepsen worker 3 - jepsen.util 7833 :fail :cas [1 [1 22]] :ValueNotSwapFrom INFO [2019-12-17 12:34:05,819] jepsen worker 3 - jepsen.util 7833 :invoke :read [1 nil] INFO [2019-12-17 12:34:05,843] jepsen worker 67 - jepsen.util 10687 :info :cas [22 [25 49]] :RequestCanceledException INFO [2019-12-17 12:34:05,845] jepsen worker 88 - jepsen.util 7828 :ok :read [29 36] INFO [2019-12-17 12:34:05,846] jepsen worker 88 - jepsen.util 7828 :invoke :write [29 16] INFO [2019-12-17 12:34:05,847] jepsen worker 67 - jepsen.util 10777 :invoke :write [22 8] INFO [2019-12-17 12:34:05,847] jepsen worker 37 - jepsen.util 7327 :fail :cas [12 [23 33]] :ValueNotSwapFrom INFO [2019-12-17 12:34:05,848] jepsen worker 37 - jepsen.util 7327 :invoke :write [12 20] INFO [2019-12-17 12:34:05,853] jepsen worker 63 - jepsen.util 6723 :ok :read [21 8] INFO [2019-12-17 12:34:05,853] jepsen worker 63 - jepsen.util 6723 :invoke :write [21 46] INFO [2019-12-17 12:34:05,858] jepsen worker 35 - jepsen.util 10385 :info :cas [11 [22 40]] :RequestCanceledException INFO [2019-12-17 12:34:05,861] jepsen worker 56 - jepsen.util 8876 :fail :cas [18 [48 43]] :ValueNotSwapFrom INFO [2019-12-17 12:34:05,866] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45543,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:34:05,888] jepsen worker 56 - jepsen.util 8876 :invoke :read [18 nil] INFO [2019-12-17 12:34:05,888] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45543,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:34:05,888] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45547,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:34:05,888] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45547,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:34:05,898] jepsen worker 35 - jepsen.util 10475 :invoke :cas [11 [1 39]] INFO [2019-12-17 12:34:05,919] jepsen worker 63 - jepsen.util 6723 :ok :write [21 46] INFO [2019-12-17 12:34:05,925] jepsen worker 63 - jepsen.util 6723 :invoke :cas [21 [7 35]] INFO [2019-12-17 12:34:05,948] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:34:05,948] jepsen nemesis - couchbase.util Rebalance check count: 352 INFO [2019-12-17 12:34:05,948] jepsen nemesis - couchbase.util Rebalance stuck count: 333 INFO [2019-12-17 12:34:05,980] jepsen worker 63 - jepsen.util 6723 :fail :cas [21 [7 35]] :ValueNotSwapFrom INFO [2019-12-17 12:34:05,997] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45556,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:34:06,007] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45556,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:34:06,007] jepsen worker 63 - jepsen.util 6723 :invoke :write [21 29] INFO [2019-12-17 12:34:06,026] jepsen worker 36 - jepsen.util 8136 :ok :read [12 28] INFO [2019-12-17 12:34:06,029] jepsen worker 36 - jepsen.util 8136 :invoke :write [12 45] INFO [2019-12-17 12:34:06,086] jepsen worker 3 - jepsen.util 7833 :ok :read [1 46] INFO [2019-12-17 12:34:06,086] jepsen worker 3 - jepsen.util 7833 :invoke :write [1 22] INFO [2019-12-17 12:34:06,088] jepsen worker 56 - jepsen.util 8876 :ok :read [18 3] INFO [2019-12-17 12:34:06,089] jepsen worker 56 - jepsen.util 8876 :invoke :cas [18 [14 31]] INFO [2019-12-17 12:34:06,109] jepsen worker 63 - jepsen.util 6723 :ok :write [21 29] INFO [2019-12-17 12:34:06,109] jepsen worker 63 - jepsen.util 6723 :invoke :write [21 9] INFO [2019-12-17 12:34:06,189] jepsen worker 63 - jepsen.util 6723 :ok :write [21 9] INFO [2019-12-17 12:34:06,189] jepsen worker 63 - jepsen.util 6723 :invoke :cas [21 [39 23]] INFO [2019-12-17 12:34:06,203] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45565,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:34:06,203] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45565,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:34:06,203] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45566,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:34:06,203] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45566,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:34:06,203] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45567,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:34:06,204] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45567,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:34:06,215] jepsen worker 62 - jepsen.util 10952 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:34:06,215] jepsen worker 62 - jepsen.util 10952 :invoke :cas [20 [17 26]] INFO [2019-12-17 12:34:06,216] jepsen worker 79 - jepsen.util 7909 :info :write [26 18] :RequestCanceledException INFO [2019-12-17 12:34:06,217] jepsen worker 69 - jepsen.util 7989 :fail :cas [23 [3 3]] :ValueNotSwapFrom INFO [2019-12-17 12:34:06,217] jepsen worker 69 - jepsen.util 7989 :invoke :write [23 40] INFO [2019-12-17 12:34:06,220] jepsen worker 79 - jepsen.util 7999 :invoke :write [26 45] INFO [2019-12-17 12:34:06,228] jepsen worker 25 - jepsen.util 11275 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:34:06,228] jepsen worker 25 - jepsen.util 11275 :invoke :cas [8 [13 31]] INFO [2019-12-17 12:34:06,242] jepsen worker 58 - jepsen.util 11668 :info :write [19 34] :RequestCanceledException INFO [2019-12-17 12:34:06,250] jepsen worker 58 - jepsen.util 11758 :invoke :read [19 nil] INFO [2019-12-17 12:34:06,254] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45573,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:34:06,258] jepsen worker 63 - jepsen.util 6723 :fail :cas [21 [39 23]] :ValueNotSwapFrom INFO [2019-12-17 12:34:06,261] jepsen worker 33 - jepsen.util 10473 :info :write [11 41] :RequestCanceledException INFO [2019-12-17 12:34:06,280] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45573,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:34:06,280] jepsen worker 63 - jepsen.util 6723 :invoke :cas [21 [46 4]] INFO [2019-12-17 12:34:06,280] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45577,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:34:06,280] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45577,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:34:06,285] jepsen worker 33 - jepsen.util 10563 :invoke :write [11 10] INFO [2019-12-17 12:34:06,296] jepsen worker 5 - jepsen.util 7295 :info :write [1 15] :RequestCanceledException INFO [2019-12-17 12:34:06,296] jepsen worker 4 - jepsen.util 7294 :info :write [1 44] :RequestCanceledException INFO [2019-12-17 12:34:06,298] jepsen worker 4 - jepsen.util 7384 :invoke :write [1 10] INFO [2019-12-17 12:34:06,301] jepsen worker 5 - jepsen.util 7385 :invoke :cas [1 [23 37]] INFO [2019-12-17 12:34:06,320] jepsen worker 63 - jepsen.util 6723 :fail :cas [21 [46 4]] :ValueNotSwapFrom INFO [2019-12-17 12:34:06,325] jepsen worker 63 - jepsen.util 6723 :invoke :write [21 18] INFO [2019-12-17 12:34:06,329] jepsen worker 57 - jepsen.util 10677 :info :write [19 16] :RequestCanceledException INFO [2019-12-17 12:34:06,329] jepsen worker 0 - jepsen.util 11160 :info :cas [0 [14 31]] :RequestCanceledException INFO [2019-12-17 12:34:06,329] jepsen worker 55 - jepsen.util 8605 :info :write [18 9] :RequestCanceledException INFO [2019-12-17 12:34:06,332] jepsen worker 57 - jepsen.util 10767 :invoke :write [19 12] INFO [2019-12-17 12:34:06,332] jepsen worker 0 - jepsen.util 11250 :invoke :write [0 24] INFO [2019-12-17 12:34:06,332] jepsen worker 55 - jepsen.util 8695 :invoke :write [18 23] INFO [2019-12-17 12:34:06,343] jepsen worker 85 - jepsen.util 11425 :info :write [28 11] :RequestCanceledException INFO [2019-12-17 12:34:06,346] jepsen worker 85 - jepsen.util 11515 :invoke :cas [28 [4 41]] INFO [2019-12-17 12:34:06,379] jepsen worker 5 - jepsen.util 7385 :fail :cas [1 [23 37]] :ValueNotSwapFrom INFO [2019-12-17 12:34:06,380] jepsen worker 5 - jepsen.util 7385 :invoke :read [1 nil] INFO [2019-12-17 12:34:06,414] jepsen worker 71 - jepsen.util 8351 :info :write [23 39] :RequestCanceledException INFO [2019-12-17 12:34:06,414] jepsen worker 80 - jepsen.util 7640 :info :write [26 19] :RequestCanceledException INFO [2019-12-17 12:34:06,417] jepsen worker 71 - jepsen.util 8441 :invoke :cas [23 [31 16]] INFO [2019-12-17 12:34:06,418] jepsen worker 11 - jepsen.util 7931 :ok :write [3 42] INFO [2019-12-17 12:34:06,418] jepsen worker 11 - jepsen.util 7931 :invoke :cas [3 [16 17]] INFO [2019-12-17 12:34:06,419] jepsen worker 80 - jepsen.util 7730 :invoke :write [26 7] INFO [2019-12-17 12:34:06,420] jepsen worker 32 - jepsen.util 7952 :ok :write [10 45] INFO [2019-12-17 12:34:06,420] jepsen worker 32 - jepsen.util 7952 :invoke :write [10 31] INFO [2019-12-17 12:34:06,421] jepsen worker 73 - jepsen.util 7453 :ok :write [24 35] INFO [2019-12-17 12:34:06,421] jepsen worker 73 - jepsen.util 7453 :invoke :read [24 nil] INFO [2019-12-17 12:34:06,648] jepsen worker 4 - jepsen.util 7384 :ok :write [1 10] INFO [2019-12-17 12:34:06,648] jepsen worker 14 - jepsen.util 7934 :info :write [4 21] :RequestCanceledException INFO [2019-12-17 12:34:06,648] jepsen worker 26 - jepsen.util 11276 :info :write [8 3] :RequestCanceledException INFO [2019-12-17 12:34:06,648] jepsen worker 40 - jepsen.util 11290 :info :cas [13 [30 12]] :RequestCanceledException INFO [2019-12-17 12:34:06,648] jepsen worker 74 - jepsen.util 8174 :info :write [24 4] :RequestCanceledException INFO [2019-12-17 12:34:06,648] jepsen worker 61 - jepsen.util 11491 :info :cas [20 [47 46]] :RequestCanceledException INFO [2019-12-17 12:34:06,648] jepsen worker 48 - jepsen.util 9678 :info :cas [16 [38 30]] :RequestCanceledException INFO [2019-12-17 12:34:06,649] jepsen worker 54 - jepsen.util 8604 :info :write [18 8] :RequestCanceledException INFO [2019-12-17 12:34:06,649] jepsen worker 6 - jepsen.util 11346 :info :write [2 41] :RequestCanceledException INFO [2019-12-17 12:34:06,649] jepsen worker 27 - jepsen.util 7767 :info :write [9 38] :RequestCanceledException INFO [2019-12-17 12:34:06,649] jepsen worker 22 - jepsen.util 10732 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:34:06,650] jepsen worker 19 - jepsen.util 7939 :info :cas [6 [28 12]] :RequestCanceledException INFO [2019-12-17 12:34:06,650] jepsen worker 44 - jepsen.util 11204 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:34:06,650] jepsen worker 66 - jepsen.util 10506 :info :write [22 3] :RequestCanceledException INFO [2019-12-17 12:34:06,650] jepsen worker 84 - jepsen.util 10344 :info :cas [28 [33 13]] :RequestCanceledException INFO [2019-12-17 12:34:06,650] jepsen worker 20 - jepsen.util 7310 :info :cas [6 [3 1]] :RequestCanceledException INFO [2019-12-17 12:34:06,650] jepsen worker 31 - jepsen.util 7051 :info :cas [10 [16 5]] :RequestCanceledException INFO [2019-12-17 12:34:06,652] jepsen worker 4 - jepsen.util 7384 :invoke :read [1 nil] INFO [2019-12-17 12:34:06,653] jepsen worker 22 - jepsen.util 10732 :invoke :cas [7 [25 27]] INFO [2019-12-17 12:34:06,653] jepsen worker 44 - jepsen.util 11204 :invoke :read [14 nil] INFO [2019-12-17 12:34:06,666] jepsen worker 14 - jepsen.util 8024 :invoke :cas [4 [0 19]] INFO [2019-12-17 12:34:06,666] jepsen worker 40 - jepsen.util 11380 :invoke :read [13 nil] INFO [2019-12-17 12:34:06,667] jepsen worker 54 - jepsen.util 8694 :invoke :write [18 46] INFO [2019-12-17 12:34:06,667] jepsen worker 19 - jepsen.util 8029 :invoke :write [6 21] INFO [2019-12-17 12:34:06,667] jepsen worker 26 - jepsen.util 11366 :invoke :write [8 10] INFO [2019-12-17 12:34:06,667] jepsen worker 6 - jepsen.util 11436 :invoke :read [2 nil] INFO [2019-12-17 12:34:06,668] jepsen worker 48 - jepsen.util 9768 :invoke :write [16 16] INFO [2019-12-17 12:34:06,668] jepsen worker 74 - jepsen.util 8264 :invoke :read [24 nil] INFO [2019-12-17 12:34:06,668] jepsen worker 27 - jepsen.util 7857 :invoke :read [9 nil] INFO [2019-12-17 12:34:06,668] jepsen worker 66 - jepsen.util 10596 :invoke :write [22 37] INFO [2019-12-17 12:34:06,668] jepsen worker 61 - jepsen.util 11581 :invoke :write [20 35] INFO [2019-12-17 12:34:06,669] jepsen worker 84 - jepsen.util 10434 :invoke :write [28 11] INFO [2019-12-17 12:34:06,669] jepsen worker 20 - jepsen.util 7400 :invoke :read [6 nil] INFO [2019-12-17 12:34:06,670] jepsen worker 31 - jepsen.util 7141 :invoke :read [10 nil] INFO [2019-12-17 12:34:06,689] jepsen worker 51 - jepsen.util 7431 :info :write [17 43] :RequestCanceledException INFO [2019-12-17 12:34:06,702] jepsen worker 51 - jepsen.util 7521 :invoke :cas [17 [13 0]] INFO [2019-12-17 12:34:06,754] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45595,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:34:06,754] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45601,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:34:06,755] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45595,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:34:06,755] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45601,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:34:06,755] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45597,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:34:06,755] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45603,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:34:06,755] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45597,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:34:06,755] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45603,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:34:06,758] jepsen worker 46 - jepsen.util 8056 :info :write [15 2] :RequestCanceledException INFO [2019-12-17 12:34:06,760] jepsen worker 28 - jepsen.util 6148 :fail :cas [9 [30 43]] :ValueNotSwapFrom INFO [2019-12-17 12:34:06,760] jepsen worker 28 - jepsen.util 6148 :invoke :read [9 nil] INFO [2019-12-17 12:34:06,760] jepsen worker 47 - jepsen.util 8237 :fail :cas [15 [0 20]] :ValueNotSwapFrom INFO [2019-12-17 12:34:06,760] jepsen worker 11 - jepsen.util 7931 :fail :cas [3 [16 17]] :ValueNotSwapFrom INFO [2019-12-17 12:34:06,760] jepsen worker 47 - jepsen.util 8237 :invoke :read [15 nil] INFO [2019-12-17 12:34:06,761] jepsen worker 11 - jepsen.util 7931 :invoke :write [3 25] INFO [2019-12-17 12:34:06,763] jepsen worker 46 - jepsen.util 8146 :invoke :read [15 nil] INFO [2019-12-17 12:34:06,787] jepsen worker 13 - jepsen.util 7303 :info :write [4 4] :RequestCanceledException INFO [2019-12-17 12:34:06,787] jepsen worker 89 - jepsen.util 7649 :info :write [29 32] :RequestCanceledException INFO [2019-12-17 12:34:06,792] jepsen worker 71 - jepsen.util 8441 :fail :cas [23 [31 16]] :ValueNotSwapFrom INFO [2019-12-17 12:34:06,792] jepsen worker 71 - jepsen.util 8441 :invoke :write [23 19] INFO [2019-12-17 12:34:06,794] jepsen worker 13 - jepsen.util 7393 :invoke :cas [4 [25 37]] INFO [2019-12-17 12:34:06,795] jepsen worker 89 - jepsen.util 7739 :invoke :write [29 41] INFO [2019-12-17 12:34:06,795] jepsen worker 73 - jepsen.util 7453 :ok :read [24 35] INFO [2019-12-17 12:34:06,796] jepsen worker 73 - jepsen.util 7453 :invoke :cas [24 [46 31]] INFO [2019-12-17 12:34:06,821] jepsen worker 74 - jepsen.util 8264 :ok :read [24 35] INFO [2019-12-17 12:34:06,822] jepsen worker 74 - jepsen.util 8264 :invoke :cas [24 [9 45]] INFO [2019-12-17 12:34:06,826] jepsen worker 31 - jepsen.util 7141 :ok :read [10 45] INFO [2019-12-17 12:34:06,826] jepsen worker 31 - jepsen.util 7141 :invoke :write [10 49] INFO [2019-12-17 12:34:06,828] jepsen worker 15 - jepsen.util 9735 :info :write [5 4] :RequestCanceledException INFO [2019-12-17 12:34:06,831] jepsen worker 15 - jepsen.util 9825 :invoke :cas [5 [4 8]] INFO [2019-12-17 12:34:06,832] jepsen worker 46 - jepsen.util 8146 :ok :read [15 29] INFO [2019-12-17 12:34:06,832] jepsen worker 46 - jepsen.util 8146 :invoke :write [15 44] INFO [2019-12-17 12:34:06,842] jepsen worker 18 - jepsen.util 7938 :info :cas [6 [31 24]] :RequestCanceledException INFO [2019-12-17 12:34:06,845] jepsen worker 18 - jepsen.util 8028 :invoke :write [6 2] INFO [2019-12-17 12:34:06,868] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45644,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:34:06,868] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45644,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:34:06,868] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45645,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:34:06,868] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45645,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:34:06,873] jepsen worker 74 - jepsen.util 8264 :fail :cas [24 [9 45]] :ValueNotSwapFrom INFO [2019-12-17 12:34:06,873] jepsen worker 74 - jepsen.util 8264 :invoke :read [24 nil] INFO [2019-12-17 12:34:06,878] jepsen worker 9 - jepsen.util 8109 :ok :read [3 42] INFO [2019-12-17 12:34:06,878] jepsen worker 9 - jepsen.util 8109 :invoke :cas [3 [48 8]] INFO [2019-12-17 12:34:06,909] jepsen worker 20 - jepsen.util 7400 :ok :read [6 40] INFO [2019-12-17 12:34:06,909] jepsen worker 20 - jepsen.util 7400 :invoke :write [6 11] INFO [2019-12-17 12:34:06,929] jepsen worker 24 - jepsen.util 9564 :info :write [8 2] :RequestCanceledException INFO [2019-12-17 12:34:06,933] jepsen worker 63 - jepsen.util 6723 :ok :write [21 18] INFO [2019-12-17 12:34:06,933] jepsen worker 63 - jepsen.util 6723 :invoke :read [21 nil] INFO [2019-12-17 12:34:06,934] jepsen worker 24 - jepsen.util 9654 :invoke :cas [8 [38 1]] INFO [2019-12-17 12:34:06,944] jepsen worker 42 - jepsen.util 11562 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:34:06,944] jepsen worker 42 - jepsen.util 11562 :invoke :cas [14 [14 42]] INFO [2019-12-17 12:34:06,956] jepsen worker 78 - jepsen.util 7728 :ok :write [26 36] INFO [2019-12-17 12:34:06,957] jepsen worker 78 - jepsen.util 7728 :invoke :write [26 21] INFO [2019-12-17 12:34:06,974] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:34:06,980] jepsen nemesis - couchbase.util Rebalance check count: 353 INFO [2019-12-17 12:34:06,980] jepsen nemesis - couchbase.util Rebalance stuck count: 334 INFO [2019-12-17 12:34:06,996] jepsen worker 12 - jepsen.util 8292 :info :write [4 35] :RequestCanceledException INFO [2019-12-17 12:34:07,010] jepsen worker 52 - jepsen.util 8062 :ok :write [17 3] INFO [2019-12-17 12:34:07,010] jepsen worker 52 - jepsen.util 8062 :invoke :read [17 nil] INFO [2019-12-17 12:34:07,025] jepsen worker 12 - jepsen.util 8382 :invoke :write [4 25] INFO [2019-12-17 12:34:07,029] jepsen worker 17 - jepsen.util 10817 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:34:07,030] jepsen worker 17 - jepsen.util 10817 :invoke :read [5 nil] INFO [2019-12-17 12:34:07,060] jepsen worker 29 - jepsen.util 6419 :fail :read [9 nil] :RequestTimeoutException INFO [2019-12-17 12:34:07,061] jepsen worker 29 - jepsen.util 6419 :invoke :read [9 nil] INFO [2019-12-17 12:34:07,090] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45652,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:34:07,090] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45652,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:34:07,091] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45653,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:34:07,091] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45653,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:34:07,092] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45654,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:34:07,092] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45654,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:34:07,092] jepsen worker 28 - jepsen.util 6148 :ok :read [9 6] INFO [2019-12-17 12:34:07,094] jepsen worker 28 - jepsen.util 6148 :invoke :cas [9 [38 35]] INFO [2019-12-17 12:34:07,096] jepsen worker 45 - jepsen.util 7965 :fail :read [15 nil] :RequestTimeoutException INFO [2019-12-17 12:34:07,096] jepsen worker 45 - jepsen.util 7965 :invoke :read [15 nil] INFO [2019-12-17 12:34:07,104] jepsen worker 63 - jepsen.util 6723 :ok :read [21 18] INFO [2019-12-17 12:34:07,105] jepsen worker 63 - jepsen.util 6723 :invoke :write [21 20] INFO [2019-12-17 12:34:07,113] jepsen worker 49 - jepsen.util 10219 :info :cas [16 [44 0]] :RequestCanceledException INFO [2019-12-17 12:34:07,114] jepsen worker 43 - jepsen.util 10393 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:34:07,114] jepsen worker 8 - jepsen.util 10628 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:34:07,115] jepsen worker 43 - jepsen.util 10393 :invoke :read [14 nil] INFO [2019-12-17 12:34:07,116] jepsen worker 8 - jepsen.util 10628 :invoke :cas [2 [34 45]] INFO [2019-12-17 12:34:07,123] jepsen worker 49 - jepsen.util 10309 :invoke :read [16 nil] INFO [2019-12-17 12:34:07,130] jepsen worker 7 - jepsen.util 10897 :info :cas [2 [28 0]] :RequestCanceledException INFO [2019-12-17 12:34:07,130] jepsen worker 72 - jepsen.util 7632 :info :write [24 42] :RequestCanceledException INFO [2019-12-17 12:34:07,130] jepsen worker 2 - jepsen.util 10262 :info :cas [0 [1 25]] :RequestCanceledException INFO [2019-12-17 12:34:07,130] jepsen worker 1 - jepsen.util 10891 :info :cas [0 [5 26]] :RequestCanceledException INFO [2019-12-17 12:34:07,135] jepsen worker 73 - jepsen.util 7453 :fail :cas [24 [46 31]] :ValueNotSwapFrom INFO [2019-12-17 12:34:07,135] jepsen worker 73 - jepsen.util 7453 :invoke :cas [24 [46 0]] INFO [2019-12-17 12:34:07,140] jepsen worker 2 - jepsen.util 10352 :invoke :cas [0 [36 0]] INFO [2019-12-17 12:34:07,141] jepsen worker 72 - jepsen.util 7722 :invoke :cas [24 [9 0]] INFO [2019-12-17 12:34:07,142] jepsen worker 1 - jepsen.util 10981 :invoke :cas [0 [2 44]] INFO [2019-12-17 12:34:07,142] jepsen worker 7 - jepsen.util 10987 :invoke :write [2 32] INFO [2019-12-17 12:34:07,142] jepsen worker 70 - jepsen.util 8170 :info :write [23 17] :RequestCanceledException INFO [2019-12-17 12:34:07,143] jepsen worker 75 - jepsen.util 10335 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:34:07,144] jepsen worker 75 - jepsen.util 10335 :invoke :write [25 25] INFO [2019-12-17 12:34:07,152] jepsen worker 70 - jepsen.util 8260 :invoke :read [23 nil] INFO [2019-12-17 12:34:07,158] jepsen worker 11 - jepsen.util 7931 :ok :write [3 25] INFO [2019-12-17 12:34:07,158] jepsen worker 31 - jepsen.util 7141 :ok :write [10 49] INFO [2019-12-17 12:34:07,158] jepsen worker 31 - jepsen.util 7141 :invoke :cas [10 [41 1]] INFO [2019-12-17 12:34:07,158] jepsen worker 11 - jepsen.util 7931 :invoke :write [3 46] INFO [2019-12-17 12:34:07,161] jepsen worker 77 - jepsen.util 10967 :info :write [25 29] :RequestCanceledException INFO [2019-12-17 12:34:07,177] jepsen worker 77 - jepsen.util 11057 :invoke :write [25 8] INFO [2019-12-17 12:34:07,201] jepsen worker 14 - jepsen.util 8024 :fail :cas [4 [0 19]] :ValueNotSwapFrom INFO [2019-12-17 12:34:07,210] jepsen worker 45 - jepsen.util 7965 :ok :read [15 29] INFO [2019-12-17 12:34:07,222] jepsen worker 31 - jepsen.util 7141 :fail :cas [10 [41 1]] :ValueNotSwapFrom INFO [2019-12-17 12:34:07,229] jepsen worker 64 - jepsen.util 6364 :info :write [21 46] :RequestCanceledException INFO [2019-12-17 12:34:07,234] jepsen worker 56 - jepsen.util 8876 :fail :cas [18 [14 31]] :ValueNotSwapFrom INFO [2019-12-17 12:34:07,243] jepsen worker 53 - jepsen.util 7343 :info :write [17 28] :RequestCanceledException INFO [2019-12-17 12:34:07,256] jepsen worker 52 - jepsen.util 8062 :ok :read [17 3] INFO [2019-12-17 12:34:07,266] jepsen worker 4 - jepsen.util 7384 :ok :read [1 10] INFO [2019-12-17 12:34:07,275] jepsen worker 63 - jepsen.util 6723 :ok :write [21 20] INFO [2019-12-17 12:34:07,275] jepsen worker 14 - jepsen.util 8024 :invoke :write [4 23] INFO [2019-12-17 12:34:07,275] jepsen worker 45 - jepsen.util 7965 :invoke :read [15 nil] INFO [2019-12-17 12:34:07,275] jepsen worker 31 - jepsen.util 7141 :invoke :cas [10 [34 14]] INFO [2019-12-17 12:34:07,275] jepsen worker 56 - jepsen.util 8876 :invoke :write [18 48] INFO [2019-12-17 12:34:07,276] jepsen worker 52 - jepsen.util 8062 :invoke :cas [17 [10 25]] INFO [2019-12-17 12:34:07,276] jepsen worker 4 - jepsen.util 7384 :invoke :read [1 nil] INFO [2019-12-17 12:34:07,276] jepsen worker 63 - jepsen.util 6723 :invoke :cas [21 [30 13]] INFO [2019-12-17 12:34:07,278] jepsen worker 64 - jepsen.util 6454 :invoke :cas [21 [30 12]] INFO [2019-12-17 12:34:07,278] jepsen worker 53 - jepsen.util 7433 :invoke :cas [17 [27 31]] INFO [2019-12-17 12:34:07,296] jepsen worker 30 - jepsen.util 7770 :info :cas [10 [44 0]] :RequestCanceledException INFO [2019-12-17 12:34:07,300] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45678,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:34:07,301] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45678,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:34:07,302] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45679,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:34:07,302] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45679,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:34:07,305] jepsen worker 30 - jepsen.util 7860 :invoke :cas [10 [16 36]] INFO [2019-12-17 12:34:07,312] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45688,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:34:07,312] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45688,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:34:07,312] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45690,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:34:07,313] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45690,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:34:07,327] jepsen worker 65 - jepsen.util 6455 :info :write [21 7] :RequestCanceledException INFO [2019-12-17 12:34:07,333] jepsen worker 65 - jepsen.util 6545 :invoke :cas [21 [31 28]] INFO [2019-12-17 12:34:07,342] jepsen worker 31 - jepsen.util 7141 :fail :cas [10 [34 14]] :ValueNotSwapFrom INFO [2019-12-17 12:34:07,343] jepsen worker 31 - jepsen.util 7141 :invoke :read [10 nil] INFO [2019-12-17 12:34:07,350] jepsen worker 45 - jepsen.util 7965 :ok :read [15 29] INFO [2019-12-17 12:34:07,351] jepsen worker 45 - jepsen.util 7965 :invoke :cas [15 [24 5]] INFO [2019-12-17 12:34:07,366] jepsen worker 30 - jepsen.util 7860 :fail :cas [10 [16 36]] :ValueNotSwapFrom INFO [2019-12-17 12:34:07,366] jepsen worker 30 - jepsen.util 7860 :invoke :read [10 nil] INFO [2019-12-17 12:34:07,368] jepsen worker 63 - jepsen.util 6723 :fail :cas [21 [30 13]] :ValueNotSwapFrom INFO [2019-12-17 12:34:07,368] jepsen worker 63 - jepsen.util 6723 :invoke :read [21 nil] INFO [2019-12-17 12:34:07,433] jepsen worker 31 - jepsen.util 7141 :ok :read [10 49] INFO [2019-12-17 12:34:07,433] jepsen worker 65 - jepsen.util 6545 :fail :cas [21 [31 28]] :ValueNotSwapFrom INFO [2019-12-17 12:34:07,433] jepsen worker 65 - jepsen.util 6545 :invoke :cas [21 [9 12]] INFO [2019-12-17 12:34:07,433] jepsen worker 31 - jepsen.util 7141 :invoke :cas [10 [46 42]] INFO [2019-12-17 12:34:07,444] jepsen worker 13 - jepsen.util 7393 :fail :cas [4 [25 37]] :ValueNotSwapFrom INFO [2019-12-17 12:34:07,444] jepsen worker 13 - jepsen.util 7393 :invoke :cas [4 [27 6]] INFO [2019-12-17 12:34:07,457] jepsen worker 45 - jepsen.util 7965 :fail :cas [15 [24 5]] :ValueNotSwapFrom INFO [2019-12-17 12:34:07,457] jepsen worker 45 - jepsen.util 7965 :invoke :cas [15 [12 16]] INFO [2019-12-17 12:34:07,458] jepsen worker 30 - jepsen.util 7860 :ok :read [10 49] INFO [2019-12-17 12:34:07,458] jepsen worker 30 - jepsen.util 7860 :invoke :write [10 13] INFO [2019-12-17 12:34:07,459] jepsen worker 51 - jepsen.util 7521 :fail :cas [17 [13 0]] :ValueNotSwapFrom INFO [2019-12-17 12:34:07,459] jepsen worker 51 - jepsen.util 7521 :invoke :read [17 nil] INFO [2019-12-17 12:34:07,463] jepsen worker 63 - jepsen.util 6723 :ok :read [21 20] INFO [2019-12-17 12:34:07,463] jepsen worker 63 - jepsen.util 6723 :invoke :read [21 nil] INFO [2019-12-17 12:34:07,488] jepsen worker 31 - jepsen.util 7141 :fail :cas [10 [46 42]] :ValueNotSwapFrom INFO [2019-12-17 12:34:07,488] jepsen worker 31 - jepsen.util 7141 :invoke :write [10 29] INFO [2019-12-17 12:34:07,499] jepsen worker 65 - jepsen.util 6545 :fail :cas [21 [9 12]] :ValueNotSwapFrom INFO [2019-12-17 12:34:07,500] jepsen worker 65 - jepsen.util 6545 :invoke :write [21 48] INFO [2019-12-17 12:34:07,514] jepsen worker 45 - jepsen.util 7965 :fail :cas [15 [12 16]] :ValueNotSwapFrom INFO [2019-12-17 12:34:07,514] jepsen worker 45 - jepsen.util 7965 :invoke :write [15 32] INFO [2019-12-17 12:34:07,521] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45706,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:34:07,521] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45706,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:34:07,521] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45710,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:34:07,522] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45710,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:34:07,536] jepsen worker 63 - jepsen.util 6723 :ok :read [21 20] INFO [2019-12-17 12:34:07,537] jepsen worker 63 - jepsen.util 6723 :invoke :write [21 14] INFO [2019-12-17 12:34:07,542] jepsen worker 82 - jepsen.util 11512 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:34:07,542] jepsen worker 41 - jepsen.util 11381 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:34:07,542] jepsen worker 82 - jepsen.util 11512 :invoke :read [27 nil] INFO [2019-12-17 12:34:07,542] jepsen worker 41 - jepsen.util 11381 :invoke :cas [13 [32 42]] INFO [2019-12-17 12:34:07,560] jepsen worker 50 - jepsen.util 11480 :info :cas [16 [3 44]] :RequestCanceledException INFO [2019-12-17 12:34:07,560] jepsen worker 59 - jepsen.util 10499 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:34:07,561] jepsen worker 59 - jepsen.util 10499 :invoke :read [19 nil] INFO [2019-12-17 12:34:07,563] jepsen worker 50 - jepsen.util 11570 :invoke :read [16 nil] INFO [2019-12-17 12:34:07,565] jepsen worker 46 - jepsen.util 8146 :ok :write [15 44] INFO [2019-12-17 12:34:07,565] jepsen worker 20 - jepsen.util 7400 :ok :write [6 11] INFO [2019-12-17 12:34:07,565] jepsen worker 20 - jepsen.util 7400 :invoke :read [6 nil] INFO [2019-12-17 12:34:07,565] jepsen worker 46 - jepsen.util 8146 :invoke :cas [15 [9 45]] INFO [2019-12-17 12:34:07,596] jepsen worker 39 - jepsen.util 10479 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:34:07,596] jepsen worker 68 - jepsen.util 10148 :info :write [22 30] :RequestCanceledException INFO [2019-12-17 12:34:07,596] jepsen worker 21 - jepsen.util 10281 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:34:07,597] jepsen worker 39 - jepsen.util 10479 :invoke :cas [13 [36 8]] INFO [2019-12-17 12:34:07,597] jepsen worker 21 - jepsen.util 10281 :invoke :read [7 nil] INFO [2019-12-17 12:34:07,600] jepsen worker 68 - jepsen.util 10238 :invoke :write [22 22] INFO [2019-12-17 12:34:07,610] jepsen worker 29 - jepsen.util 6419 :ok :read [9 6] INFO [2019-12-17 12:34:07,611] jepsen worker 29 - jepsen.util 6419 :invoke :write [9 30] INFO [2019-12-17 12:34:07,613] jepsen worker 81 - jepsen.util 10521 :info :cas [27 [21 36]] :RequestCanceledException INFO [2019-12-17 12:34:07,613] jepsen worker 23 - jepsen.util 10643 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:34:07,613] jepsen worker 86 - jepsen.util 11246 :info :write [28 42] :RequestCanceledException INFO [2019-12-17 12:34:07,613] jepsen worker 23 - jepsen.util 10643 :invoke :cas [7 [37 49]] INFO [2019-12-17 12:34:07,618] jepsen worker 86 - jepsen.util 11336 :invoke :cas [28 [49 3]] INFO [2019-12-17 12:34:07,618] jepsen worker 81 - jepsen.util 10611 :invoke :read [27 nil] INFO [2019-12-17 12:34:07,622] jepsen worker 10 - jepsen.util 7930 :info :write [3 26] :RequestCanceledException INFO [2019-12-17 12:34:07,622] jepsen worker 60 - jepsen.util 11850 :info :write [20 42] :RequestCanceledException INFO [2019-12-17 12:34:07,622] jepsen worker 87 - jepsen.util 7197 :info :write [29 40] :RequestCanceledException INFO [2019-12-17 12:34:07,627] jepsen worker 10 - jepsen.util 8020 :invoke :cas [3 [20 4]] INFO [2019-12-17 12:34:07,627] jepsen worker 87 - jepsen.util 7287 :invoke :read [29 nil] INFO [2019-12-17 12:34:07,627] jepsen worker 60 - jepsen.util 11940 :invoke :write [20 27] INFO [2019-12-17 12:34:07,628] jepsen worker 46 - jepsen.util 8146 :fail :cas [15 [9 45]] :ValueNotSwapFrom INFO [2019-12-17 12:34:07,628] jepsen worker 46 - jepsen.util 8146 :invoke :write [15 36] INFO [2019-12-17 12:34:07,638] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45741,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:34:07,638] jepsen worker 38 - jepsen.util 8138 :info :write [12 37] :RequestCanceledException INFO [2019-12-17 12:34:07,639] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45741,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:34:07,639] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45743,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:34:07,639] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45743,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:34:07,643] jepsen worker 38 - jepsen.util 8228 :invoke :cas [12 [17 18]] INFO [2019-12-17 12:34:07,657] jepsen worker 28 - jepsen.util 6148 :fail :cas [9 [38 35]] :ValueNotSwapFrom INFO [2019-12-17 12:34:07,657] jepsen worker 28 - jepsen.util 6148 :invoke :write [9 12] INFO [2019-12-17 12:34:07,657] jepsen worker 76 - jepsen.util 10606 :info :write [25 20] :RequestCanceledException INFO [2019-12-17 12:34:07,660] jepsen worker 76 - jepsen.util 10696 :invoke :read [25 nil] INFO [2019-12-17 12:34:07,675] jepsen worker 65 - jepsen.util 6545 :ok :write [21 48] INFO [2019-12-17 12:34:07,676] jepsen worker 65 - jepsen.util 6545 :invoke :cas [21 [29 0]] INFO [2019-12-17 12:34:07,677] jepsen worker 52 - jepsen.util 8062 :fail :cas [17 [10 25]] :ValueNotSwapFrom INFO [2019-12-17 12:34:07,677] jepsen worker 52 - jepsen.util 8062 :invoke :cas [17 [41 2]] INFO [2019-12-17 12:34:07,692] jepsen worker 73 - jepsen.util 7453 :fail :cas [24 [46 0]] :ValueNotSwapFrom INFO [2019-12-17 12:34:07,692] jepsen worker 73 - jepsen.util 7453 :invoke :read [24 nil] INFO [2019-12-17 12:34:07,695] jepsen worker 83 - jepsen.util 11063 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:34:07,695] jepsen worker 16 - jepsen.util 11986 :info :cas [5 [17 4]] :RequestCanceledException INFO [2019-12-17 12:34:07,695] jepsen worker 83 - jepsen.util 11063 :invoke :cas [27 [12 27]] INFO [2019-12-17 12:34:07,699] jepsen worker 16 - jepsen.util 12076 :invoke :cas [5 [32 6]] INFO [2019-12-17 12:34:07,722] jepsen worker 72 - jepsen.util 7722 :fail :cas [24 [9 0]] :ValueNotSwapFrom INFO [2019-12-17 12:34:07,722] jepsen worker 72 - jepsen.util 7722 :invoke :read [24 nil] INFO [2019-12-17 12:34:07,725] jepsen worker 53 - jepsen.util 7433 :fail :cas [17 [27 31]] :ValueNotSwapFrom INFO [2019-12-17 12:34:07,725] jepsen worker 53 - jepsen.util 7433 :invoke :cas [17 [14 32]] INFO [2019-12-17 12:34:07,734] jepsen worker 20 - jepsen.util 7400 :ok :read [6 11] INFO [2019-12-17 12:34:07,734] jepsen worker 20 - jepsen.util 7400 :invoke :cas [6 [45 27]] INFO [2019-12-17 12:34:07,770] jepsen worker 29 - jepsen.util 6419 :ok :write [9 30] INFO [2019-12-17 12:34:07,770] jepsen worker 29 - jepsen.util 6419 :invoke :write [9 47] INFO [2019-12-17 12:34:07,814] jepsen worker 34 - jepsen.util 10474 :info :write [11 5] :RequestCanceledException INFO [2019-12-17 12:34:07,817] jepsen worker 34 - jepsen.util 10564 :invoke :write [11 4] INFO [2019-12-17 12:34:07,827] jepsen worker 73 - jepsen.util 7453 :ok :read [24 35] INFO [2019-12-17 12:34:07,827] jepsen worker 73 - jepsen.util 7453 :invoke :cas [24 [46 3]] INFO [2019-12-17 12:34:07,839] jepsen worker 63 - jepsen.util 6723 :ok :write [21 14] INFO [2019-12-17 12:34:07,839] jepsen worker 63 - jepsen.util 6723 :invoke :write [21 16] INFO [2019-12-17 12:34:07,887] jepsen worker 72 - jepsen.util 7722 :ok :read [24 35] INFO [2019-12-17 12:34:07,888] jepsen worker 72 - jepsen.util 7722 :invoke :cas [24 [41 9]] INFO [2019-12-17 12:34:07,917] jepsen worker 20 - jepsen.util 7400 :fail :cas [6 [45 27]] :ValueNotSwapFrom INFO [2019-12-17 12:34:07,917] jepsen worker 20 - jepsen.util 7400 :invoke :read [6 nil] INFO [2019-12-17 12:34:07,921] jepsen worker 30 - jepsen.util 7860 :ok :write [10 13] INFO [2019-12-17 12:34:07,921] jepsen worker 37 - jepsen.util 7327 :info :write [12 20] :RequestCanceledException INFO [2019-12-17 12:34:07,921] jepsen worker 88 - jepsen.util 7828 :info :write [29 16] :RequestCanceledException INFO [2019-12-17 12:34:07,921] jepsen worker 30 - jepsen.util 7860 :invoke :read [10 nil] INFO [2019-12-17 12:34:07,925] jepsen worker 88 - jepsen.util 7918 :invoke :cas [29 [10 49]] INFO [2019-12-17 12:34:07,926] jepsen worker 37 - jepsen.util 7417 :invoke :write [12 26] INFO [2019-12-17 12:34:07,934] jepsen worker 11 - jepsen.util 7931 :ok :write [3 46] INFO [2019-12-17 12:34:07,934] jepsen worker 11 - jepsen.util 7931 :invoke :read [3 nil] INFO [2019-12-17 12:34:07,942] jepsen worker 5 - jepsen.util 7385 :ok :read [1 10] INFO [2019-12-17 12:34:07,942] jepsen worker 5 - jepsen.util 7385 :invoke :cas [1 [34 46]] INFO [2019-12-17 12:34:07,943] jepsen worker 67 - jepsen.util 10777 :info :write [22 8] :RequestCanceledException INFO [2019-12-17 12:34:07,947] jepsen worker 67 - jepsen.util 10867 :invoke :write [22 14] INFO [2019-12-17 12:34:07,957] jepsen worker 35 - jepsen.util 10475 :info :cas [11 [1 39]] :RequestCanceledException INFO [2019-12-17 12:34:07,957] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45762,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:34:07,957] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45762,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:34:07,957] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45766,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:34:07,958] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45766,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:34:07,959] jepsen worker 35 - jepsen.util 10565 :invoke :write [11 18] INFO [2019-12-17 12:34:07,977] jepsen worker 29 - jepsen.util 6419 :ok :write [9 47] INFO [2019-12-17 12:34:07,978] jepsen worker 29 - jepsen.util 6419 :invoke :write [9 42] INFO [2019-12-17 12:34:08,009] jepsen worker 64 - jepsen.util 6454 :fail :cas [21 [30 12]] :ValueNotSwapFrom INFO [2019-12-17 12:34:08,009] jepsen worker 64 - jepsen.util 6454 :invoke :write [21 36] INFO [2019-12-17 12:34:08,014] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:34:08,014] jepsen nemesis - couchbase.util Rebalance check count: 354 INFO [2019-12-17 12:34:08,014] jepsen nemesis - couchbase.util Rebalance stuck count: 335 INFO [2019-12-17 12:34:08,040] jepsen worker 11 - jepsen.util 7931 :ok :read [3 46] INFO [2019-12-17 12:34:08,041] jepsen worker 11 - jepsen.util 7931 :invoke :cas [3 [20 19]] INFO [2019-12-17 12:34:08,044] jepsen worker 72 - jepsen.util 7722 :fail :cas [24 [41 9]] :ValueNotSwapFrom INFO [2019-12-17 12:34:08,045] jepsen worker 72 - jepsen.util 7722 :invoke :cas [24 [49 10]] INFO [2019-12-17 12:34:08,067] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45779,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:34:08,067] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45779,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:34:08,067] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45782,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:34:08,067] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45782,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:34:08,077] jepsen worker 20 - jepsen.util 7400 :ok :read [6 11] INFO [2019-12-17 12:34:08,077] jepsen worker 20 - jepsen.util 7400 :invoke :cas [6 [29 45]] INFO [2019-12-17 12:34:08,091] jepsen worker 52 - jepsen.util 8062 :fail :cas [17 [41 2]] :ValueNotSwapFrom INFO [2019-12-17 12:34:08,091] jepsen worker 52 - jepsen.util 8062 :invoke :cas [17 [10 33]] INFO [2019-12-17 12:34:08,097] jepsen worker 51 - jepsen.util 7521 :ok :read [17 3] INFO [2019-12-17 12:34:08,097] jepsen worker 51 - jepsen.util 7521 :invoke :read [17 nil] INFO [2019-12-17 12:34:08,129] jepsen worker 53 - jepsen.util 7433 :fail :cas [17 [14 32]] :ValueNotSwapFrom INFO [2019-12-17 12:34:08,129] jepsen worker 36 - jepsen.util 8136 :info :write [12 45] :RequestCanceledException INFO [2019-12-17 12:34:08,129] jepsen worker 53 - jepsen.util 7433 :invoke :read [17 nil] INFO [2019-12-17 12:34:08,130] jepsen worker 63 - jepsen.util 6723 :ok :write [21 16] INFO [2019-12-17 12:34:08,130] jepsen worker 63 - jepsen.util 6723 :invoke :cas [21 [9 3]] INFO [2019-12-17 12:34:08,131] jepsen worker 36 - jepsen.util 8226 :invoke :cas [12 [16 22]] INFO [2019-12-17 12:34:08,158] jepsen worker 3 - jepsen.util 7833 :info :write [1 22] :RequestCanceledException INFO [2019-12-17 12:34:08,161] jepsen worker 3 - jepsen.util 7923 :invoke :cas [1 [30 19]] INFO [2019-12-17 12:34:08,166] jepsen worker 63 - jepsen.util 6723 :fail :cas [21 [9 3]] :ValueNotSwapFrom INFO [2019-12-17 12:34:08,166] jepsen worker 63 - jepsen.util 6723 :invoke :read [21 nil] INFO [2019-12-17 12:34:08,167] jepsen worker 29 - jepsen.util 6419 :ok :write [9 42] INFO [2019-12-17 12:34:08,167] jepsen worker 29 - jepsen.util 6419 :invoke :write [9 19] INFO [2019-12-17 12:34:08,174] jepsen worker 70 - jepsen.util 8260 :ok :read [23 29] INFO [2019-12-17 12:34:08,174] jepsen worker 70 - jepsen.util 8260 :invoke :read [23 nil] INFO [2019-12-17 12:34:08,204] jepsen worker 63 - jepsen.util 6723 :ok :read [21 16] INFO [2019-12-17 12:34:08,204] jepsen worker 63 - jepsen.util 6723 :invoke :write [21 12] INFO [2019-12-17 12:34:08,212] jepsen worker 87 - jepsen.util 7287 :ok :read [29 36] INFO [2019-12-17 12:34:08,212] jepsen worker 87 - jepsen.util 7287 :invoke :write [29 13] INFO [2019-12-17 12:34:08,242] jepsen worker 88 - jepsen.util 7918 :fail :cas [29 [10 49]] :ValueNotSwapFrom INFO [2019-12-17 12:34:08,242] jepsen worker 88 - jepsen.util 7918 :invoke :read [29 nil] INFO [2019-12-17 12:34:08,275] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45796,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:34:08,275] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45796,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:34:08,275] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45798,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:34:08,275] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45798,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:34:08,277] jepsen worker 52 - jepsen.util 8062 :fail :cas [17 [10 33]] :ValueNotSwapFrom INFO [2019-12-17 12:34:08,277] jepsen worker 52 - jepsen.util 8062 :invoke :cas [17 [40 21]] INFO [2019-12-17 12:34:08,282] jepsen worker 29 - jepsen.util 6419 :ok :write [9 19] INFO [2019-12-17 12:34:08,282] jepsen worker 29 - jepsen.util 6419 :invoke :cas [9 [27 41]] INFO [2019-12-17 12:34:08,286] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45802,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:34:08,286] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45802,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:34:08,286] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45803,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:34:08,286] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45803,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:34:08,286] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45804,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:34:08,286] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45804,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:34:08,296] jepsen worker 58 - jepsen.util 11758 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:34:08,296] jepsen worker 58 - jepsen.util 11758 :invoke :read [19 nil] INFO [2019-12-17 12:34:08,313] jepsen worker 33 - jepsen.util 10563 :info :write [11 10] :RequestCanceledException INFO [2019-12-17 12:34:08,313] jepsen worker 62 - jepsen.util 10952 :info :cas [20 [17 26]] :RequestCanceledException INFO [2019-12-17 12:34:08,313] jepsen worker 69 - jepsen.util 7989 :info :write [23 40] :RequestCanceledException INFO [2019-12-17 12:34:08,317] jepsen worker 33 - jepsen.util 10653 :invoke :write [11 42] INFO [2019-12-17 12:34:08,317] jepsen worker 62 - jepsen.util 11042 :invoke :cas [20 [20 9]] INFO [2019-12-17 12:34:08,319] jepsen worker 69 - jepsen.util 8079 :invoke :read [23 nil] INFO [2019-12-17 12:34:08,320] jepsen worker 53 - jepsen.util 7433 :ok :read [17 3] INFO [2019-12-17 12:34:08,320] jepsen worker 53 - jepsen.util 7433 :invoke :write [17 0] INFO [2019-12-17 12:34:08,326] jepsen worker 79 - jepsen.util 7999 :info :write [26 45] :RequestCanceledException INFO [2019-12-17 12:34:08,326] jepsen worker 25 - jepsen.util 11275 :info :cas [8 [13 31]] :RequestCanceledException INFO [2019-12-17 12:34:08,328] jepsen worker 51 - jepsen.util 7521 :ok :read [17 3] INFO [2019-12-17 12:34:08,328] jepsen worker 51 - jepsen.util 7521 :invoke :read [17 nil] INFO [2019-12-17 12:34:08,331] jepsen worker 25 - jepsen.util 11365 :invoke :write [8 35] INFO [2019-12-17 12:34:08,331] jepsen worker 79 - jepsen.util 8089 :invoke :cas [26 [42 42]] INFO [2019-12-17 12:34:08,337] jepsen worker 29 - jepsen.util 6419 :fail :cas [9 [27 41]] :ValueNotSwapFrom INFO [2019-12-17 12:34:08,337] jepsen worker 29 - jepsen.util 6419 :invoke :write [9 20] INFO [2019-12-17 12:34:08,346] jepsen worker 63 - jepsen.util 6723 :ok :write [21 12] INFO [2019-12-17 12:34:08,347] jepsen worker 63 - jepsen.util 6723 :invoke :read [21 nil] INFO [2019-12-17 12:34:08,356] jepsen worker 5 - jepsen.util 7385 :fail :cas [1 [34 46]] :ValueNotSwapFrom INFO [2019-12-17 12:34:08,356] jepsen worker 5 - jepsen.util 7385 :invoke :read [1 nil] INFO [2019-12-17 12:34:08,360] jepsen worker 57 - jepsen.util 10767 :info :write [19 12] :RequestCanceledException INFO [2019-12-17 12:34:08,361] jepsen worker 57 - jepsen.util 10857 :invoke :write [19 0] INFO [2019-12-17 12:34:08,386] jepsen worker 63 - jepsen.util 6723 :ok :read [21 12] INFO [2019-12-17 12:34:08,386] jepsen worker 63 - jepsen.util 6723 :invoke :write [21 21] INFO [2019-12-17 12:34:08,395] jepsen worker 55 - jepsen.util 8695 :info :write [18 23] :RequestCanceledException INFO [2019-12-17 12:34:08,397] jepsen worker 55 - jepsen.util 8785 :invoke :read [18 nil] INFO [2019-12-17 12:34:08,412] jepsen worker 85 - jepsen.util 11515 :info :cas [28 [4 41]] :RequestCanceledException INFO [2019-12-17 12:34:08,415] jepsen worker 85 - jepsen.util 11605 :invoke :read [28 nil] INFO [2019-12-17 12:34:08,419] jepsen worker 4 - jepsen.util 7384 :ok :read [1 10] INFO [2019-12-17 12:34:08,423] jepsen worker 4 - jepsen.util 7384 :invoke :cas [1 [44 28]] INFO [2019-12-17 12:34:08,429] jepsen worker 0 - jepsen.util 11250 :info :write [0 24] :RequestCanceledException INFO [2019-12-17 12:34:08,431] jepsen worker 0 - jepsen.util 11340 :invoke :cas [0 [16 45]] INFO [2019-12-17 12:34:08,433] jepsen worker 29 - jepsen.util 6419 :ok :write [9 20] INFO [2019-12-17 12:34:08,433] jepsen worker 29 - jepsen.util 6419 :invoke :write [9 48] INFO [2019-12-17 12:34:08,437] jepsen worker 36 - jepsen.util 8226 :fail :cas [12 [16 22]] :ValueNotSwapFrom INFO [2019-12-17 12:34:08,437] jepsen worker 36 - jepsen.util 8226 :invoke :write [12 17] INFO [2019-12-17 12:34:08,471] jepsen worker 63 - jepsen.util 6723 :ok :write [21 21] INFO [2019-12-17 12:34:08,471] jepsen worker 63 - jepsen.util 6723 :invoke :write [21 12] INFO [2019-12-17 12:34:08,494] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45830,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:34:08,494] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45830,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:34:08,495] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45834,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:34:08,495] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45834,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:34:08,496] jepsen worker 79 - jepsen.util 8089 :fail :cas [26 [42 42]] :ValueNotSwapFrom INFO [2019-12-17 12:34:08,496] jepsen worker 79 - jepsen.util 8089 :invoke :write [26 44] INFO [2019-12-17 12:34:08,524] jepsen worker 29 - jepsen.util 6419 :ok :write [9 48] INFO [2019-12-17 12:34:08,524] jepsen worker 29 - jepsen.util 6419 :invoke :write [9 4] INFO [2019-12-17 12:34:08,524] jepsen worker 46 - jepsen.util 8146 :ok :write [15 36] INFO [2019-12-17 12:34:08,524] jepsen worker 31 - jepsen.util 7141 :ok :write [10 29] INFO [2019-12-17 12:34:08,524] jepsen worker 46 - jepsen.util 8146 :invoke :read [15 nil] INFO [2019-12-17 12:34:08,524] jepsen worker 31 - jepsen.util 7141 :invoke :cas [10 [38 21]] INFO [2019-12-17 12:34:08,557] jepsen worker 31 - jepsen.util 7141 :fail :cas [10 [38 21]] :ValueNotSwapFrom INFO [2019-12-17 12:34:08,558] jepsen worker 31 - jepsen.util 7141 :invoke :read [10 nil] INFO [2019-12-17 12:34:08,568] jepsen worker 46 - jepsen.util 8146 :ok :read [15 36] INFO [2019-12-17 12:34:08,568] jepsen worker 46 - jepsen.util 8146 :invoke :write [15 16] INFO [2019-12-17 12:34:08,590] jepsen worker 31 - jepsen.util 7141 :ok :read [10 29] INFO [2019-12-17 12:34:08,591] jepsen worker 31 - jepsen.util 7141 :invoke :write [10 16] INFO [2019-12-17 12:34:08,599] jepsen worker 63 - jepsen.util 6723 :ok :write [21 12] INFO [2019-12-17 12:34:08,599] jepsen worker 63 - jepsen.util 6723 :invoke :write [21 5] INFO [2019-12-17 12:34:08,628] jepsen worker 13 - jepsen.util 7393 :fail :cas [4 [27 6]] :ValueNotSwapFrom INFO [2019-12-17 12:34:08,628] jepsen worker 13 - jepsen.util 7393 :invoke :write [4 4] INFO [2019-12-17 12:34:08,650] jepsen worker 29 - jepsen.util 6419 :ok :write [9 4] INFO [2019-12-17 12:34:08,651] jepsen worker 29 - jepsen.util 6419 :invoke :write [9 28] INFO [2019-12-17 12:34:08,681] jepsen worker 5 - jepsen.util 7385 :ok :read [1 10] INFO [2019-12-17 12:34:08,681] jepsen worker 5 - jepsen.util 7385 :invoke :write [1 22] INFO [2019-12-17 12:34:08,695] jepsen worker 27 - jepsen.util 7857 :fail :read [9 nil] :RequestTimeoutException INFO [2019-12-17 12:34:08,696] jepsen worker 27 - jepsen.util 7857 :invoke :read [9 nil] INFO [2019-12-17 12:34:08,704] jepsen worker 63 - jepsen.util 6723 :ok :write [21 5] INFO [2019-12-17 12:34:08,704] jepsen worker 63 - jepsen.util 6723 :invoke :write [21 47] INFO [2019-12-17 12:34:08,718] jepsen worker 70 - jepsen.util 8260 :ok :read [23 29] INFO [2019-12-17 12:34:08,718] jepsen worker 70 - jepsen.util 8260 :invoke :write [23 37] INFO [2019-12-17 12:34:08,723] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45849,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:34:08,723] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45849,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:34:08,723] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45853,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:34:08,724] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45853,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:34:08,729] jepsen worker 40 - jepsen.util 11380 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:34:08,729] jepsen worker 22 - jepsen.util 10732 :info :cas [7 [25 27]] :RequestCanceledException INFO [2019-12-17 12:34:08,729] jepsen worker 32 - jepsen.util 7952 :info :write [10 31] :RequestCanceledException INFO [2019-12-17 12:34:08,729] jepsen worker 40 - jepsen.util 11380 :invoke :read [13 nil] INFO [2019-12-17 12:34:08,729] jepsen worker 61 - jepsen.util 11581 :info :write [20 35] :RequestCanceledException INFO [2019-12-17 12:34:08,729] jepsen worker 26 - jepsen.util 11366 :info :write [8 10] :RequestCanceledException INFO [2019-12-17 12:34:08,731] jepsen worker 26 - jepsen.util 11456 :invoke :read [8 nil] INFO [2019-12-17 12:34:08,731] jepsen worker 32 - jepsen.util 8042 :invoke :write [10 12] INFO [2019-12-17 12:34:08,732] jepsen worker 61 - jepsen.util 11671 :invoke :write [20 31] INFO [2019-12-17 12:34:08,732] jepsen worker 22 - jepsen.util 10822 :invoke :write [7 28] INFO [2019-12-17 12:34:08,741] jepsen worker 44 - jepsen.util 11204 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:34:08,741] jepsen worker 6 - jepsen.util 11436 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:34:08,741] jepsen worker 80 - jepsen.util 7730 :info :write [26 7] :RequestCanceledException INFO [2019-12-17 12:34:08,741] jepsen worker 44 - jepsen.util 11204 :invoke :read [14 nil] INFO [2019-12-17 12:34:08,741] jepsen worker 6 - jepsen.util 11436 :invoke :read [2 nil] INFO [2019-12-17 12:34:08,741] jepsen worker 84 - jepsen.util 10434 :info :write [28 11] :RequestCanceledException INFO [2019-12-17 12:34:08,746] jepsen worker 80 - jepsen.util 7820 :invoke :write [26 47] INFO [2019-12-17 12:34:08,747] jepsen worker 84 - jepsen.util 10524 :invoke :read [28 nil] INFO [2019-12-17 12:34:08,760] jepsen worker 48 - jepsen.util 9768 :info :write [16 16] :RequestCanceledException INFO [2019-12-17 12:34:08,764] jepsen worker 27 - jepsen.util 7857 :ok :read [9 4] INFO [2019-12-17 12:34:08,764] jepsen worker 27 - jepsen.util 7857 :invoke :write [9 10] INFO [2019-12-17 12:34:08,766] jepsen worker 48 - jepsen.util 9858 :invoke :write [16 16] INFO [2019-12-17 12:34:08,789] jepsen worker 29 - jepsen.util 6419 :ok :write [9 28] INFO [2019-12-17 12:34:08,789] jepsen worker 29 - jepsen.util 6419 :invoke :write [9 41] INFO [2019-12-17 12:34:08,795] jepsen worker 19 - jepsen.util 8029 :info :write [6 21] :RequestCanceledException INFO [2019-12-17 12:34:08,798] jepsen worker 19 - jepsen.util 8119 :invoke :cas [6 [29 33]] INFO [2019-12-17 12:34:08,810] jepsen worker 54 - jepsen.util 8694 :info :write [18 46] :RequestCanceledException INFO [2019-12-17 12:34:08,810] jepsen worker 66 - jepsen.util 10596 :info :write [22 37] :RequestCanceledException INFO [2019-12-17 12:34:08,813] jepsen worker 54 - jepsen.util 8784 :invoke :write [18 49] INFO [2019-12-17 12:34:08,813] jepsen worker 66 - jepsen.util 10686 :invoke :read [22 nil] INFO [2019-12-17 12:34:08,829] jepsen worker 71 - jepsen.util 8441 :info :write [23 19] :RequestCanceledException INFO [2019-12-17 12:34:08,829] jepsen worker 47 - jepsen.util 8237 :fail :read [15 nil] :RequestTimeoutException INFO [2019-12-17 12:34:08,829] jepsen worker 47 - jepsen.util 8237 :invoke :read [15 nil] INFO [2019-12-17 12:34:08,831] jepsen worker 71 - jepsen.util 8531 :invoke :cas [23 [34 47]] INFO [2019-12-17 12:34:08,838] jepsen worker 63 - jepsen.util 6723 :ok :write [21 47] INFO [2019-12-17 12:34:08,839] jepsen worker 63 - jepsen.util 6723 :invoke :cas [21 [5 23]] INFO [2019-12-17 12:34:08,843] jepsen worker 89 - jepsen.util 7739 :info :write [29 41] :RequestCanceledException INFO [2019-12-17 12:34:08,844] jepsen worker 89 - jepsen.util 7829 :invoke :cas [29 [18 6]] INFO [2019-12-17 12:34:08,860] jepsen worker 15 - jepsen.util 9825 :info :cas [5 [4 8]] :RequestCanceledException INFO [2019-12-17 12:34:08,861] jepsen worker 15 - jepsen.util 9915 :invoke :write [5 34] INFO [2019-12-17 12:34:08,885] jepsen worker 38 - jepsen.util 8228 :fail :cas [12 [17 18]] :ValueNotSwapFrom INFO [2019-12-17 12:34:08,885] jepsen worker 38 - jepsen.util 8228 :invoke :read [12 nil] INFO [2019-12-17 12:34:08,898] jepsen worker 88 - jepsen.util 7918 :ok :read [29 36] INFO [2019-12-17 12:34:08,899] jepsen worker 88 - jepsen.util 7918 :invoke :read [29 nil] INFO [2019-12-17 12:34:08,929] jepsen worker 9 - jepsen.util 8109 :info :cas [3 [48 8]] :RequestCanceledException INFO [2019-12-17 12:34:08,929] jepsen worker 18 - jepsen.util 8028 :info :write [6 2] :RequestCanceledException INFO [2019-12-17 12:34:08,929] jepsen worker 74 - jepsen.util 8264 :fail :read [24 nil] :RequestTimeoutException INFO [2019-12-17 12:34:08,929] jepsen worker 74 - jepsen.util 8264 :invoke :cas [24 [47 49]] INFO [2019-12-17 12:34:08,931] jepsen worker 9 - jepsen.util 8199 :invoke :write [3 27] INFO [2019-12-17 12:34:08,932] jepsen worker 18 - jepsen.util 8118 :invoke :write [6 0] INFO [2019-12-17 12:34:08,934] jepsen worker 47 - jepsen.util 8237 :ok :read [15 36] INFO [2019-12-17 12:34:08,935] jepsen worker 47 - jepsen.util 8237 :invoke :cas [15 [18 35]] INFO [2019-12-17 12:34:08,943] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45881,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:34:08,943] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45881,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:34:08,959] jepsen worker 27 - jepsen.util 7857 :ok :write [9 10] INFO [2019-12-17 12:34:08,959] jepsen worker 27 - jepsen.util 7857 :invoke :read [9 nil] INFO [2019-12-17 12:34:08,969] jepsen worker 47 - jepsen.util 8237 :fail :cas [15 [18 35]] :ValueNotSwapFrom INFO [2019-12-17 12:34:08,970] jepsen worker 47 - jepsen.util 8237 :invoke :write [15 20] INFO [2019-12-17 12:34:08,990] jepsen worker 27 - jepsen.util 7857 :ok :read [9 10] INFO [2019-12-17 12:34:08,990] jepsen worker 27 - jepsen.util 7857 :invoke :read [9 nil] INFO [2019-12-17 12:34:08,995] jepsen worker 24 - jepsen.util 9654 :info :cas [8 [38 1]] :RequestCanceledException INFO [2019-12-17 12:34:08,997] jepsen worker 24 - jepsen.util 9744 :invoke :write [8 7] INFO [2019-12-17 12:34:09,005] jepsen worker 51 - jepsen.util 7521 :ok :read [17 3] INFO [2019-12-17 12:34:09,005] jepsen worker 51 - jepsen.util 7521 :invoke :read [17 nil] INFO [2019-12-17 12:34:09,029] jepsen worker 78 - jepsen.util 7728 :info :write [26 21] :RequestCanceledException INFO [2019-12-17 12:34:09,031] jepsen worker 78 - jepsen.util 7818 :invoke :read [26 nil] INFO [2019-12-17 12:34:09,031] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:34:09,032] jepsen nemesis - couchbase.util Rebalance check count: 355 INFO [2019-12-17 12:34:09,032] jepsen nemesis - couchbase.util Rebalance stuck count: 336 INFO [2019-12-17 12:34:09,038] jepsen worker 27 - jepsen.util 7857 :ok :read [9 10] INFO [2019-12-17 12:34:09,039] jepsen worker 27 - jepsen.util 7857 :invoke :read [9 nil] INFO [2019-12-17 12:34:09,043] jepsen worker 42 - jepsen.util 11562 :info :cas [14 [14 42]] :RequestCanceledException INFO [2019-12-17 12:34:09,044] jepsen worker 42 - jepsen.util 11652 :invoke :cas [14 [2 36]] INFO [2019-12-17 12:34:09,050] jepsen worker 52 - jepsen.util 8062 :fail :cas [17 [40 21]] :ValueNotSwapFrom INFO [2019-12-17 12:34:09,051] jepsen worker 52 - jepsen.util 8062 :invoke :read [17 nil] INFO [2019-12-17 12:34:09,051] jepsen worker 4 - jepsen.util 7384 :fail :cas [1 [44 28]] :ValueNotSwapFrom INFO [2019-12-17 12:34:09,051] jepsen worker 4 - jepsen.util 7384 :invoke :cas [1 [27 45]] INFO [2019-12-17 12:34:09,053] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45895,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:34:09,053] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45895,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:34:09,053] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45899,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:34:09,053] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45899,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:34:09,069] jepsen worker 27 - jepsen.util 7857 :ok :read [9 10] INFO [2019-12-17 12:34:09,069] jepsen worker 27 - jepsen.util 7857 :invoke :cas [9 [16 18]] INFO [2019-12-17 12:34:09,075] jepsen worker 55 - jepsen.util 8785 :ok :read [18 3] INFO [2019-12-17 12:34:09,076] jepsen worker 55 - jepsen.util 8785 :invoke :read [18 nil] INFO [2019-12-17 12:34:09,090] jepsen worker 27 - jepsen.util 7857 :fail :cas [9 [16 18]] :ValueNotSwapFrom INFO [2019-12-17 12:34:09,090] jepsen worker 27 - jepsen.util 7857 :invoke :cas [9 [41 23]] INFO [2019-12-17 12:34:09,111] jepsen worker 27 - jepsen.util 7857 :fail :cas [9 [41 23]] :ValueNotSwapFrom INFO [2019-12-17 12:34:09,111] jepsen worker 27 - jepsen.util 7857 :invoke :cas [9 [15 16]] INFO [2019-12-17 12:34:09,113] jepsen worker 12 - jepsen.util 8382 :info :write [4 25] :RequestCanceledException INFO [2019-12-17 12:34:09,115] jepsen worker 12 - jepsen.util 8472 :invoke :cas [4 [30 10]] INFO [2019-12-17 12:34:09,122] jepsen worker 27 - jepsen.util 7857 :fail :cas [9 [15 16]] :ValueNotSwapFrom INFO [2019-12-17 12:34:09,122] jepsen worker 27 - jepsen.util 7857 :invoke :write [9 0] INFO [2019-12-17 12:34:09,128] jepsen worker 17 - jepsen.util 10817 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:34:09,128] jepsen worker 17 - jepsen.util 10817 :invoke :read [5 nil] INFO [2019-12-17 12:34:09,148] jepsen worker 12 - jepsen.util 8472 :fail :cas [4 [30 10]] :ValueNotSwapFrom INFO [2019-12-17 12:34:09,148] jepsen worker 12 - jepsen.util 8472 :invoke :read [4 nil] INFO [2019-12-17 12:34:09,163] jepsen worker 55 - jepsen.util 8785 :ok :read [18 3] INFO [2019-12-17 12:34:09,163] jepsen worker 55 - jepsen.util 8785 :invoke :read [18 nil] INFO [2019-12-17 12:34:09,171] jepsen worker 27 - jepsen.util 7857 :ok :write [9 0] INFO [2019-12-17 12:34:09,172] jepsen worker 27 - jepsen.util 7857 :invoke :read [9 nil] INFO [2019-12-17 12:34:09,174] jepsen worker 69 - jepsen.util 8079 :ok :read [23 29] INFO [2019-12-17 12:34:09,174] jepsen worker 69 - jepsen.util 8079 :invoke :cas [23 [36 6]] INFO [2019-12-17 12:34:09,192] jepsen worker 27 - jepsen.util 7857 :ok :read [9 0] INFO [2019-12-17 12:34:09,192] jepsen worker 27 - jepsen.util 7857 :invoke :write [9 31] INFO [2019-12-17 12:34:09,195] jepsen worker 7 - jepsen.util 10987 :info :write [2 32] :RequestCanceledException INFO [2019-12-17 12:34:09,197] jepsen worker 7 - jepsen.util 11077 :invoke :cas [2 [12 43]] INFO [2019-12-17 12:34:09,213] jepsen worker 43 - jepsen.util 10393 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:34:09,213] jepsen worker 8 - jepsen.util 10628 :info :cas [2 [34 45]] :RequestCanceledException INFO [2019-12-17 12:34:09,213] jepsen worker 43 - jepsen.util 10393 :invoke :read [14 nil] INFO [2019-12-17 12:34:09,214] jepsen worker 8 - jepsen.util 10718 :invoke :cas [2 [35 27]] INFO [2019-12-17 12:34:09,229] jepsen worker 1 - jepsen.util 10981 :info :cas [0 [2 44]] :RequestCanceledException INFO [2019-12-17 12:34:09,229] jepsen worker 49 - jepsen.util 10309 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:34:09,229] jepsen worker 77 - jepsen.util 11057 :info :write [25 8] :RequestCanceledException INFO [2019-12-17 12:34:09,230] jepsen worker 49 - jepsen.util 10309 :invoke :read [16 nil] INFO [2019-12-17 12:34:09,230] jepsen worker 1 - jepsen.util 11071 :invoke :cas [0 [42 26]] INFO [2019-12-17 12:34:09,231] jepsen worker 77 - jepsen.util 11147 :invoke :read [25 nil] INFO [2019-12-17 12:34:09,240] jepsen worker 27 - jepsen.util 7857 :ok :write [9 31] INFO [2019-12-17 12:34:09,240] jepsen worker 27 - jepsen.util 7857 :invoke :cas [9 [43 26]] INFO [2019-12-17 12:34:09,243] jepsen worker 2 - jepsen.util 10352 :info :cas [0 [36 0]] :RequestCanceledException INFO [2019-12-17 12:34:09,243] jepsen worker 75 - jepsen.util 10335 :info :write [25 25] :RequestCanceledException INFO [2019-12-17 12:34:09,244] jepsen worker 75 - jepsen.util 10425 :invoke :read [25 nil] INFO [2019-12-17 12:34:09,244] jepsen worker 2 - jepsen.util 10442 :invoke :write [0 18] INFO [2019-12-17 12:34:09,248] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45920,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:34:09,248] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45920,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:34:09,248] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45921,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:34:09,249] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45921,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:34:09,258] jepsen worker 27 - jepsen.util 7857 :fail :cas [9 [43 26]] :ValueNotSwapFrom INFO [2019-12-17 12:34:09,259] jepsen worker 27 - jepsen.util 7857 :invoke :write [9 6] INFO [2019-12-17 12:34:09,262] jepsen worker 3 - jepsen.util 7923 :fail :cas [1 [30 19]] :ValueNotSwapFrom INFO [2019-12-17 12:34:09,262] jepsen worker 3 - jepsen.util 7923 :invoke :read [1 nil] INFO [2019-12-17 12:34:09,268] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45929,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:34:09,268] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45929,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:34:09,268] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45932,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:34:09,268] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45932,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:34:09,294] jepsen worker 3 - jepsen.util 7923 :ok :read [1 10] INFO [2019-12-17 12:34:09,294] jepsen worker 3 - jepsen.util 7923 :invoke :write [1 27] INFO [2019-12-17 12:34:09,310] jepsen worker 27 - jepsen.util 7857 :ok :write [9 6] INFO [2019-12-17 12:34:09,310] jepsen worker 27 - jepsen.util 7857 :invoke :write [9 35] INFO [2019-12-17 12:34:09,329] jepsen worker 56 - jepsen.util 8876 :info :write [18 48] :RequestCanceledException INFO [2019-12-17 12:34:09,330] jepsen worker 56 - jepsen.util 8966 :invoke :read [18 nil] INFO [2019-12-17 12:34:09,336] jepsen worker 88 - jepsen.util 7918 :ok :read [29 36] INFO [2019-12-17 12:34:09,336] jepsen worker 88 - jepsen.util 7918 :invoke :write [29 3] INFO [2019-12-17 12:34:09,350] jepsen worker 51 - jepsen.util 7521 :ok :read [17 3] INFO [2019-12-17 12:34:09,350] jepsen worker 51 - jepsen.util 7521 :invoke :read [17 nil] INFO [2019-12-17 12:34:09,359] jepsen worker 14 - jepsen.util 8024 :info :write [4 23] :RequestCanceledException INFO [2019-12-17 12:34:09,361] jepsen worker 14 - jepsen.util 8114 :invoke :read [4 nil] INFO [2019-12-17 12:34:09,362] jepsen worker 27 - jepsen.util 7857 :ok :write [9 35] INFO [2019-12-17 12:34:09,362] jepsen worker 27 - jepsen.util 7857 :invoke :cas [9 [37 45]] INFO [2019-12-17 12:34:09,377] jepsen worker 9 - jepsen.util 8199 :ok :write [3 27] INFO [2019-12-17 12:34:09,377] jepsen worker 9 - jepsen.util 8199 :invoke :cas [3 [18 23]] INFO [2019-12-17 12:34:09,377] jepsen worker 46 - jepsen.util 8146 :ok :write [15 16] INFO [2019-12-17 12:34:09,377] jepsen worker 46 - jepsen.util 8146 :invoke :read [15 nil] INFO [2019-12-17 12:34:09,384] jepsen worker 27 - jepsen.util 7857 :fail :cas [9 [37 45]] :ValueNotSwapFrom INFO [2019-12-17 12:34:09,384] jepsen worker 27 - jepsen.util 7857 :invoke :write [9 37] INFO [2019-12-17 12:34:09,390] jepsen worker 31 - jepsen.util 7141 :ok :write [10 16] INFO [2019-12-17 12:34:09,390] jepsen worker 31 - jepsen.util 7141 :invoke :read [10 nil] INFO [2019-12-17 12:34:09,400] jepsen worker 12 - jepsen.util 8472 :ok :read [4 21] INFO [2019-12-17 12:34:09,400] jepsen worker 12 - jepsen.util 8472 :invoke :write [4 9] INFO [2019-12-17 12:34:09,420] jepsen worker 46 - jepsen.util 8146 :ok :read [15 16] INFO [2019-12-17 12:34:09,421] jepsen worker 46 - jepsen.util 8146 :invoke :write [15 22] INFO [2019-12-17 12:34:09,422] jepsen worker 55 - jepsen.util 8785 :ok :read [18 3] INFO [2019-12-17 12:34:09,422] jepsen worker 55 - jepsen.util 8785 :invoke :cas [18 [31 0]] INFO [2019-12-17 12:34:09,431] jepsen worker 9 - jepsen.util 8199 :fail :cas [3 [18 23]] :ValueNotSwapFrom INFO [2019-12-17 12:34:09,432] jepsen worker 9 - jepsen.util 8199 :invoke :read [3 nil] INFO [2019-12-17 12:34:09,442] jepsen worker 74 - jepsen.util 8264 :fail :cas [24 [47 49]] :ValueNotSwapFrom INFO [2019-12-17 12:34:09,442] jepsen worker 74 - jepsen.util 8264 :invoke :write [24 34] INFO [2019-12-17 12:34:09,468] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45958,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:34:09,468] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45958,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:34:09,473] jepsen worker 80 - jepsen.util 7820 :ok :write [26 47] INFO [2019-12-17 12:34:09,474] jepsen worker 80 - jepsen.util 7820 :invoke :read [26 nil] INFO [2019-12-17 12:34:09,495] jepsen worker 9 - jepsen.util 8199 :ok :read [3 27] INFO [2019-12-17 12:34:09,495] jepsen worker 9 - jepsen.util 8199 :invoke :read [3 nil] INFO [2019-12-17 12:34:09,512] jepsen worker 89 - jepsen.util 7829 :fail :cas [29 [18 6]] :ValueNotSwapFrom INFO [2019-12-17 12:34:09,513] jepsen worker 89 - jepsen.util 7829 :invoke :read [29 nil] INFO [2019-12-17 12:34:09,519] jepsen worker 9 - jepsen.util 8199 :ok :read [3 27] INFO [2019-12-17 12:34:09,519] jepsen worker 9 - jepsen.util 8199 :invoke :write [3 48] INFO [2019-12-17 12:34:09,525] jepsen worker 27 - jepsen.util 7857 :ok :write [9 37] INFO [2019-12-17 12:34:09,525] jepsen worker 27 - jepsen.util 7857 :invoke :write [9 11] INFO [2019-12-17 12:34:09,561] jepsen worker 53 - jepsen.util 7433 :ok :write [17 0] INFO [2019-12-17 12:34:09,561] jepsen worker 53 - jepsen.util 7433 :invoke :cas [17 [6 41]] INFO [2019-12-17 12:34:09,566] jepsen worker 3 - jepsen.util 7923 :ok :write [1 27] INFO [2019-12-17 12:34:09,566] jepsen worker 3 - jepsen.util 7923 :invoke :cas [1 [40 34]] INFO [2019-12-17 12:34:09,592] jepsen worker 27 - jepsen.util 7857 :ok :write [9 11] INFO [2019-12-17 12:34:09,593] jepsen worker 27 - jepsen.util 7857 :invoke :write [9 32] INFO [2019-12-17 12:34:09,594] jepsen worker 18 - jepsen.util 8118 :ok :write [6 0] INFO [2019-12-17 12:34:09,594] jepsen worker 18 - jepsen.util 8118 :invoke :read [6 nil] INFO [2019-12-17 12:34:09,596] jepsen worker 45 - jepsen.util 7965 :info :write [15 32] :RequestCanceledException INFO [2019-12-17 12:34:09,598] jepsen worker 45 - jepsen.util 8055 :invoke :cas [15 [23 1]] INFO [2019-12-17 12:34:09,612] jepsen worker 50 - jepsen.util 11570 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:34:09,613] jepsen worker 50 - jepsen.util 11570 :invoke :cas [16 [44 9]] INFO [2019-12-17 12:34:09,619] jepsen worker 38 - jepsen.util 8228 :ok :read [12 28] INFO [2019-12-17 12:34:09,619] jepsen worker 38 - jepsen.util 8228 :invoke :write [12 6] INFO [2019-12-17 12:34:09,626] jepsen worker 31 - jepsen.util 7141 :ok :read [10 16] INFO [2019-12-17 12:34:09,626] jepsen worker 31 - jepsen.util 7141 :invoke :cas [10 [18 7]] INFO [2019-12-17 12:34:09,630] jepsen worker 68 - jepsen.util 10238 :info :write [22 22] :RequestCanceledException INFO [2019-12-17 12:34:09,631] jepsen worker 68 - jepsen.util 10328 :invoke :read [22 nil] INFO [2019-12-17 12:34:09,643] jepsen worker 41 - jepsen.util 11381 :info :cas [13 [32 42]] :RequestCanceledException INFO [2019-12-17 12:34:09,643] jepsen worker 82 - jepsen.util 11512 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:34:09,643] jepsen worker 86 - jepsen.util 11336 :info :cas [28 [49 3]] :RequestCanceledException INFO [2019-12-17 12:34:09,643] jepsen worker 82 - jepsen.util 11512 :invoke :read [27 nil] INFO [2019-12-17 12:34:09,645] jepsen worker 86 - jepsen.util 11426 :invoke :cas [28 [27 8]] INFO [2019-12-17 12:34:09,648] jepsen worker 41 - jepsen.util 11471 :invoke :write [13 43] INFO [2019-12-17 12:34:09,657] jepsen worker 59 - jepsen.util 10499 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:34:09,657] jepsen worker 81 - jepsen.util 10611 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:34:09,657] jepsen worker 59 - jepsen.util 10499 :invoke :read [19 nil] INFO [2019-12-17 12:34:09,657] jepsen worker 81 - jepsen.util 10611 :invoke :read [27 nil] INFO [2019-12-17 12:34:09,695] jepsen worker 65 - jepsen.util 6545 :info :cas [21 [29 0]] :RequestCanceledException INFO [2019-12-17 12:34:09,695] jepsen worker 39 - jepsen.util 10479 :info :cas [13 [36 8]] :RequestCanceledException INFO [2019-12-17 12:34:09,695] jepsen worker 21 - jepsen.util 10281 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:34:09,695] jepsen worker 21 - jepsen.util 10281 :invoke :read [7 nil] INFO [2019-12-17 12:34:09,697] jepsen worker 65 - jepsen.util 6635 :invoke :read [21 nil] INFO [2019-12-17 12:34:09,697] jepsen worker 39 - jepsen.util 10569 :invoke :read [13 nil] INFO [2019-12-17 12:34:09,701] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45980,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:34:09,701] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":45980,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:34:09,704] jepsen worker 27 - jepsen.util 7857 :ok :write [9 32] INFO [2019-12-17 12:34:09,705] jepsen worker 27 - jepsen.util 7857 :invoke :cas [9 [49 44]] INFO [2019-12-17 12:34:09,712] jepsen worker 60 - jepsen.util 11940 :info :write [20 27] :RequestCanceledException INFO [2019-12-17 12:34:09,712] jepsen worker 23 - jepsen.util 10643 :info :cas [7 [37 49]] :RequestCanceledException INFO [2019-12-17 12:34:09,713] jepsen worker 23 - jepsen.util 10733 :invoke :read [7 nil] INFO [2019-12-17 12:34:09,714] jepsen worker 60 - jepsen.util 12030 :invoke :read [20 nil] INFO [2019-12-17 12:34:09,721] jepsen worker 71 - jepsen.util 8531 :fail :cas [23 [34 47]] :ValueNotSwapFrom INFO [2019-12-17 12:34:09,721] jepsen worker 71 - jepsen.util 8531 :invoke :read [23 nil] INFO [2019-12-17 12:34:09,724] jepsen worker 45 - jepsen.util 8055 :fail :cas [15 [23 1]] :ValueNotSwapFrom INFO [2019-12-17 12:34:09,724] jepsen worker 45 - jepsen.util 8055 :invoke :read [15 nil] INFO [2019-12-17 12:34:09,727] jepsen worker 10 - jepsen.util 8020 :info :cas [3 [20 4]] :RequestCanceledException INFO [2019-12-17 12:34:09,729] jepsen worker 10 - jepsen.util 8110 :invoke :write [3 13] INFO [2019-12-17 12:34:09,734] jepsen worker 76 - jepsen.util 10696 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:34:09,734] jepsen worker 76 - jepsen.util 10696 :invoke :cas [25 [2 39]] INFO [2019-12-17 12:34:09,736] jepsen worker 69 - jepsen.util 8079 :fail :cas [23 [36 6]] :ValueNotSwapFrom INFO [2019-12-17 12:34:09,736] jepsen worker 69 - jepsen.util 8079 :invoke :write [23 17] INFO [2019-12-17 12:34:09,739] jepsen worker 27 - jepsen.util 7857 :fail :cas [9 [49 44]] :ValueNotSwapFrom INFO [2019-12-17 12:34:09,739] jepsen worker 27 - jepsen.util 7857 :invoke :write [9 12] INFO [2019-12-17 12:34:09,760] jepsen worker 28 - jepsen.util 6148 :info :write [9 12] :RequestCanceledException INFO [2019-12-17 12:34:09,760] jepsen worker 16 - jepsen.util 12076 :info :cas [5 [32 6]] :RequestCanceledException INFO [2019-12-17 12:34:09,761] jepsen worker 16 - jepsen.util 12166 :invoke :read [5 nil] INFO [2019-12-17 12:34:09,763] jepsen worker 28 - jepsen.util 6238 :invoke :read [9 nil] INFO [2019-12-17 12:34:09,763] jepsen worker 45 - jepsen.util 8055 :ok :read [15 16] INFO [2019-12-17 12:34:09,763] jepsen worker 45 - jepsen.util 8055 :invoke :read [15 nil] INFO [2019-12-17 12:34:09,796] jepsen worker 83 - jepsen.util 11063 :info :cas [27 [12 27]] :RequestCanceledException INFO [2019-12-17 12:34:09,798] jepsen worker 83 - jepsen.util 11153 :invoke :read [27 nil] INFO [2019-12-17 12:34:09,826] jepsen worker 78 - jepsen.util 7818 :ok :read [26 47] INFO [2019-12-17 12:34:09,826] jepsen worker 78 - jepsen.util 7818 :invoke :cas [26 [18 13]] INFO [2019-12-17 12:34:09,841] jepsen worker 74 - jepsen.util 8264 :ok :write [24 34] INFO [2019-12-17 12:34:09,842] jepsen worker 74 - jepsen.util 8264 :invoke :read [24 nil] INFO [2019-12-17 12:34:09,846] jepsen worker 46 - jepsen.util 8146 :ok :write [15 22] INFO [2019-12-17 12:34:09,847] jepsen worker 46 - jepsen.util 8146 :invoke :write [15 43] INFO [2019-12-17 12:34:09,847] jepsen worker 14 - jepsen.util 8114 :ok :read [4 21] INFO [2019-12-17 12:34:09,848] jepsen worker 14 - jepsen.util 8114 :invoke :write [4 18] INFO [2019-12-17 12:34:09,851] jepsen worker 9 - jepsen.util 8199 :ok :write [3 48] INFO [2019-12-17 12:34:09,851] jepsen worker 9 - jepsen.util 8199 :invoke :read [3 nil] INFO [2019-12-17 12:34:09,857] jepsen worker 73 - jepsen.util 7453 :info :cas [24 [46 3]] :RequestCanceledException INFO [2019-12-17 12:34:09,859] jepsen worker 73 - jepsen.util 7543 :invoke :write [24 21] INFO [2019-12-17 12:34:09,884] jepsen worker 27 - jepsen.util 7857 :ok :write [9 12] INFO [2019-12-17 12:34:09,884] jepsen worker 27 - jepsen.util 7857 :invoke :read [9 nil] INFO [2019-12-17 12:34:09,899] jepsen worker 18 - jepsen.util 8118 :ok :read [6 0] INFO [2019-12-17 12:34:09,899] jepsen worker 18 - jepsen.util 8118 :invoke :cas [6 [7 34]] INFO [2019-12-17 12:34:09,901] jepsen worker 65 - jepsen.util 6635 :ok :read [21 47] INFO [2019-12-17 12:34:09,901] jepsen worker 65 - jepsen.util 6635 :invoke :cas [21 [45 17]] INFO [2019-12-17 12:34:09,912] jepsen worker 80 - jepsen.util 7820 :ok :read [26 47] INFO [2019-12-17 12:34:09,912] jepsen worker 80 - jepsen.util 7820 :invoke :read [26 nil] INFO [2019-12-17 12:34:09,919] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46012,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:34:09,920] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46012,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:34:09,920] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46016,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:34:09,920] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46016,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:34:09,929] jepsen worker 34 - jepsen.util 10564 :info :write [11 4] :RequestCanceledException INFO [2019-12-17 12:34:09,933] jepsen worker 34 - jepsen.util 10654 :invoke :write [11 46] INFO [2019-12-17 12:34:09,951] jepsen worker 28 - jepsen.util 6238 :ok :read [9 12] INFO [2019-12-17 12:34:09,952] jepsen worker 3 - jepsen.util 7923 :fail :cas [1 [40 34]] :ValueNotSwapFrom INFO [2019-12-17 12:34:09,952] jepsen worker 28 - jepsen.util 6238 :invoke :write [9 40] INFO [2019-12-17 12:34:09,952] jepsen worker 3 - jepsen.util 7923 :invoke :cas [1 [20 26]] INFO [2019-12-17 12:34:09,952] jepsen worker 31 - jepsen.util 7141 :fail :cas [10 [18 7]] :ValueNotSwapFrom INFO [2019-12-17 12:34:09,952] jepsen worker 31 - jepsen.util 7141 :invoke :write [10 38] INFO [2019-12-17 12:34:09,979] jepsen worker 51 - jepsen.util 7521 :ok :read [17 0] INFO [2019-12-17 12:34:09,980] jepsen worker 51 - jepsen.util 7521 :invoke :cas [17 [47 19]] INFO [2019-12-17 12:34:10,011] jepsen worker 37 - jepsen.util 7417 :info :write [12 26] :RequestCanceledException INFO [2019-12-17 12:34:10,014] jepsen worker 37 - jepsen.util 7507 :invoke :write [12 8] INFO [2019-12-17 12:34:10,028] jepsen worker 30 - jepsen.util 7860 :fail :read [10 nil] :RequestTimeoutException INFO [2019-12-17 12:34:10,029] jepsen worker 30 - jepsen.util 7860 :invoke :cas [10 [15 4]] INFO [2019-12-17 12:34:10,029] jepsen worker 67 - jepsen.util 10867 :info :write [22 14] :RequestCanceledException INFO [2019-12-17 12:34:10,030] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46030,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:34:10,030] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46030,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:34:10,030] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46032,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:34:10,030] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46032,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:34:10,031] jepsen worker 9 - jepsen.util 8199 :ok :read [3 48] INFO [2019-12-17 12:34:10,031] jepsen worker 9 - jepsen.util 8199 :invoke :cas [3 [28 15]] INFO [2019-12-17 12:34:10,033] jepsen worker 67 - jepsen.util 10957 :invoke :cas [22 [19 4]] INFO [2019-12-17 12:34:10,040] jepsen worker 71 - jepsen.util 8531 :ok :read [23 29] INFO [2019-12-17 12:34:10,040] jepsen worker 71 - jepsen.util 8531 :invoke :write [23 41] INFO [2019-12-17 12:34:10,041] jepsen worker 74 - jepsen.util 8264 :ok :read [24 34] INFO [2019-12-17 12:34:10,042] jepsen worker 74 - jepsen.util 8264 :invoke :cas [24 [38 26]] INFO [2019-12-17 12:34:10,043] jepsen worker 35 - jepsen.util 10565 :info :write [11 18] :RequestCanceledException INFO [2019-12-17 12:34:10,043] jepsen worker 64 - jepsen.util 6454 :info :write [21 36] :RequestCanceledException INFO [2019-12-17 12:34:10,045] jepsen worker 64 - jepsen.util 6544 :invoke :read [21 nil] INFO [2019-12-17 12:34:10,048] jepsen worker 35 - jepsen.util 10655 :invoke :read [11 nil] INFO [2019-12-17 12:34:10,050] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:34:10,051] jepsen nemesis - couchbase.util Rebalance check count: 356 INFO [2019-12-17 12:34:10,051] jepsen nemesis - couchbase.util Rebalance stuck count: 337 INFO [2019-12-17 12:34:10,060] jepsen worker 72 - jepsen.util 7722 :info :cas [24 [49 10]] :RequestCanceledException INFO [2019-12-17 12:34:10,064] jepsen worker 72 - jepsen.util 7812 :invoke :write [24 8] INFO [2019-12-17 12:34:10,075] jepsen worker 65 - jepsen.util 6635 :fail :cas [21 [45 17]] :ValueNotSwapFrom INFO [2019-12-17 12:34:10,075] jepsen worker 65 - jepsen.util 6635 :invoke :write [21 40] INFO [2019-12-17 12:34:10,079] jepsen worker 18 - jepsen.util 8118 :fail :cas [6 [7 34]] :ValueNotSwapFrom INFO [2019-12-17 12:34:10,080] jepsen worker 18 - jepsen.util 8118 :invoke :write [6 49] INFO [2019-12-17 12:34:10,106] jepsen worker 55 - jepsen.util 8785 :fail :cas [18 [31 0]] :ValueNotSwapFrom INFO [2019-12-17 12:34:10,106] jepsen worker 55 - jepsen.util 8785 :invoke :cas [18 [37 3]] INFO [2019-12-17 12:34:10,113] jepsen worker 28 - jepsen.util 6238 :ok :write [9 40] INFO [2019-12-17 12:34:10,113] jepsen worker 28 - jepsen.util 6238 :invoke :cas [9 [20 11]] INFO [2019-12-17 12:34:10,143] jepsen worker 11 - jepsen.util 7931 :info :cas [3 [20 19]] :RequestCanceledException INFO [2019-12-17 12:34:10,145] jepsen worker 11 - jepsen.util 8021 :invoke :cas [3 [7 11]] INFO [2019-12-17 12:34:10,160] jepsen worker 20 - jepsen.util 7400 :info :cas [6 [29 45]] :RequestCanceledException INFO [2019-12-17 12:34:10,162] jepsen worker 9 - jepsen.util 8199 :fail :cas [3 [28 15]] :ValueNotSwapFrom INFO [2019-12-17 12:34:10,162] jepsen worker 9 - jepsen.util 8199 :invoke :read [3 nil] INFO [2019-12-17 12:34:10,162] jepsen worker 20 - jepsen.util 7490 :invoke :write [6 44] INFO [2019-12-17 12:34:10,176] jepsen worker 64 - jepsen.util 6544 :ok :read [21 47] INFO [2019-12-17 12:34:10,177] jepsen worker 64 - jepsen.util 6544 :invoke :read [21 nil] INFO [2019-12-17 12:34:10,234] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46050,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:34:10,234] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46050,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:34:10,234] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46051,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:34:10,235] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46051,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:34:10,235] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46052,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:34:10,235] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46052,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:34:10,237] jepsen worker 89 - jepsen.util 7829 :ok :read [29 36] INFO [2019-12-17 12:34:10,237] jepsen worker 89 - jepsen.util 7829 :invoke :write [29 49] INFO [2019-12-17 12:34:10,249] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46057,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:34:10,250] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46057,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:34:10,250] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46061,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:34:10,250] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46061,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:34:10,250] jepsen worker 65 - jepsen.util 6635 :ok :write [21 40] INFO [2019-12-17 12:34:10,251] jepsen worker 65 - jepsen.util 6635 :invoke :cas [21 [49 27]] INFO [2019-12-17 12:34:10,286] jepsen worker 9 - jepsen.util 8199 :ok :read [3 48] INFO [2019-12-17 12:34:10,286] jepsen worker 9 - jepsen.util 8199 :invoke :write [3 28] INFO [2019-12-17 12:34:10,296] jepsen worker 87 - jepsen.util 7287 :info :write [29 13] :RequestCanceledException INFO [2019-12-17 12:34:10,299] jepsen worker 87 - jepsen.util 7377 :invoke :read [29 nil] INFO [2019-12-17 12:34:10,305] jepsen worker 64 - jepsen.util 6544 :ok :read [21 40] INFO [2019-12-17 12:34:10,306] jepsen worker 64 - jepsen.util 6544 :invoke :write [21 10] INFO [2019-12-17 12:34:10,319] jepsen worker 65 - jepsen.util 6635 :fail :cas [21 [49 27]] :ValueNotSwapFrom INFO [2019-12-17 12:34:10,319] jepsen worker 65 - jepsen.util 6635 :invoke :cas [21 [7 9]] INFO [2019-12-17 12:34:10,341] jepsen worker 46 - jepsen.util 8146 :ok :write [15 43] INFO [2019-12-17 12:34:10,341] jepsen worker 46 - jepsen.util 8146 :invoke :write [15 18] INFO [2019-12-17 12:34:10,342] jepsen worker 73 - jepsen.util 7543 :ok :write [24 21] INFO [2019-12-17 12:34:10,343] jepsen worker 73 - jepsen.util 7543 :invoke :cas [24 [36 1]] INFO [2019-12-17 12:34:10,381] jepsen worker 11 - jepsen.util 8021 :fail :cas [3 [7 11]] :ValueNotSwapFrom INFO [2019-12-17 12:34:10,382] jepsen worker 11 - jepsen.util 8021 :invoke :read [3 nil] INFO [2019-12-17 12:34:10,396] jepsen worker 58 - jepsen.util 11758 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:34:10,396] jepsen worker 33 - jepsen.util 10653 :info :write [11 42] :RequestCanceledException INFO [2019-12-17 12:34:10,396] jepsen worker 58 - jepsen.util 11758 :invoke :write [19 7] INFO [2019-12-17 12:34:10,399] jepsen worker 33 - jepsen.util 10743 :invoke :write [11 22] INFO [2019-12-17 12:34:10,411] jepsen worker 62 - jepsen.util 11042 :info :cas [20 [20 9]] :RequestCanceledException INFO [2019-12-17 12:34:10,414] jepsen worker 62 - jepsen.util 11132 :invoke :write [20 25] INFO [2019-12-17 12:34:10,429] jepsen worker 57 - jepsen.util 10857 :info :write [19 0] :RequestCanceledException INFO [2019-12-17 12:34:10,431] jepsen worker 57 - jepsen.util 10947 :invoke :read [19 nil] INFO [2019-12-17 12:34:10,442] jepsen worker 25 - jepsen.util 11365 :info :write [8 35] :RequestCanceledException INFO [2019-12-17 12:34:10,444] jepsen worker 25 - jepsen.util 11455 :invoke :write [8 43] INFO [2019-12-17 12:34:10,453] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46075,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:34:10,454] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46075,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:34:10,454] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46078,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:34:10,454] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46078,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:34:10,454] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46079,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:34:10,454] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46079,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:34:10,460] jepsen worker 36 - jepsen.util 8226 :info :write [12 17] :RequestCanceledException INFO [2019-12-17 12:34:10,462] jepsen worker 36 - jepsen.util 8316 :invoke :write [12 14] INFO [2019-12-17 12:34:10,513] jepsen worker 85 - jepsen.util 11605 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:34:10,513] jepsen worker 85 - jepsen.util 11605 :invoke :write [28 16] INFO [2019-12-17 12:34:10,529] jepsen worker 0 - jepsen.util 11340 :info :cas [0 [16 45]] :RequestCanceledException INFO [2019-12-17 12:34:10,532] jepsen worker 0 - jepsen.util 11430 :invoke :read [0 nil] INFO [2019-12-17 12:34:10,560] jepsen worker 79 - jepsen.util 8089 :info :write [26 44] :RequestCanceledException INFO [2019-12-17 12:34:10,564] jepsen worker 79 - jepsen.util 8179 :invoke :cas [26 [33 0]] INFO [2019-12-17 12:34:10,575] jepsen worker 3 - jepsen.util 7923 :fail :cas [1 [20 26]] :ValueNotSwapFrom INFO [2019-12-17 12:34:10,575] jepsen worker 3 - jepsen.util 7923 :invoke :read [1 nil] INFO [2019-12-17 12:34:10,604] jepsen worker 51 - jepsen.util 7521 :fail :cas [17 [47 19]] :ValueNotSwapFrom INFO [2019-12-17 12:34:10,604] jepsen worker 51 - jepsen.util 7521 :invoke :read [17 nil] INFO [2019-12-17 12:34:10,661] jepsen worker 87 - jepsen.util 7377 :ok :read [29 36] INFO [2019-12-17 12:34:10,661] jepsen worker 87 - jepsen.util 7377 :invoke :read [29 nil] INFO [2019-12-17 12:34:10,676] jepsen worker 55 - jepsen.util 8785 :fail :cas [18 [37 3]] :ValueNotSwapFrom INFO [2019-12-17 12:34:10,676] jepsen worker 55 - jepsen.util 8785 :invoke :write [18 16] INFO [2019-12-17 12:34:10,689] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46089,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:34:10,689] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46089,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:34:10,689] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46093,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:34:10,689] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46093,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:34:10,705] jepsen worker 51 - jepsen.util 7521 :ok :read [17 0] INFO [2019-12-17 12:34:10,705] jepsen worker 51 - jepsen.util 7521 :invoke :cas [17 [49 35]] INFO [2019-12-17 12:34:10,711] jepsen worker 56 - jepsen.util 8966 :ok :read [18 48] INFO [2019-12-17 12:34:10,711] jepsen worker 56 - jepsen.util 8966 :invoke :read [18 nil] INFO [2019-12-17 12:34:10,724] jepsen worker 20 - jepsen.util 7490 :ok :write [6 44] INFO [2019-12-17 12:34:10,724] jepsen worker 20 - jepsen.util 7490 :invoke :cas [6 [14 26]] INFO [2019-12-17 12:34:10,724] jepsen worker 9 - jepsen.util 8199 :ok :write [3 28] INFO [2019-12-17 12:34:10,725] jepsen worker 9 - jepsen.util 8199 :invoke :read [3 nil] INFO [2019-12-17 12:34:10,725] jepsen worker 31 - jepsen.util 7141 :ok :write [10 38] INFO [2019-12-17 12:34:10,725] jepsen worker 31 - jepsen.util 7141 :invoke :cas [10 [20 29]] INFO [2019-12-17 12:34:10,726] jepsen worker 53 - jepsen.util 7433 :fail :cas [17 [6 41]] :ValueNotSwapFrom INFO [2019-12-17 12:34:10,727] jepsen worker 53 - jepsen.util 7433 :invoke :read [17 nil] INFO [2019-12-17 12:34:10,729] jepsen worker 46 - jepsen.util 8146 :ok :write [15 18] INFO [2019-12-17 12:34:10,729] jepsen worker 13 - jepsen.util 7393 :info :write [4 4] :RequestCanceledException INFO [2019-12-17 12:34:10,729] jepsen worker 46 - jepsen.util 8146 :invoke :read [15 nil] INFO [2019-12-17 12:34:10,731] jepsen worker 72 - jepsen.util 7812 :ok :write [24 8] INFO [2019-12-17 12:34:10,731] jepsen worker 72 - jepsen.util 7812 :invoke :read [24 nil] INFO [2019-12-17 12:34:10,731] jepsen worker 13 - jepsen.util 7483 :invoke :read [4 nil] INFO [2019-12-17 12:34:10,742] jepsen worker 26 - jepsen.util 11456 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:34:10,742] jepsen worker 5 - jepsen.util 7385 :info :write [1 22] :RequestCanceledException INFO [2019-12-17 12:34:10,743] jepsen worker 26 - jepsen.util 11456 :invoke :read [8 nil] INFO [2019-12-17 12:34:10,745] jepsen worker 5 - jepsen.util 7475 :invoke :read [1 nil] INFO [2019-12-17 12:34:10,774] jepsen worker 80 - jepsen.util 7820 :ok :read [26 47] INFO [2019-12-17 12:34:10,775] jepsen worker 80 - jepsen.util 7820 :invoke :read [26 nil] INFO [2019-12-17 12:34:10,791] jepsen worker 9 - jepsen.util 8199 :ok :read [3 28] INFO [2019-12-17 12:34:10,791] jepsen worker 9 - jepsen.util 8199 :invoke :write [3 20] INFO [2019-12-17 12:34:10,793] jepsen worker 20 - jepsen.util 7490 :fail :cas [6 [14 26]] :ValueNotSwapFrom INFO [2019-12-17 12:34:10,793] jepsen worker 20 - jepsen.util 7490 :invoke :write [6 4] INFO [2019-12-17 12:34:10,796] jepsen worker 22 - jepsen.util 10822 :info :write [7 28] :RequestCanceledException INFO [2019-12-17 12:34:10,798] jepsen worker 73 - jepsen.util 7543 :fail :cas [24 [36 1]] :ValueNotSwapFrom INFO [2019-12-17 12:34:10,798] jepsen worker 73 - jepsen.util 7543 :invoke :write [24 22] INFO [2019-12-17 12:34:10,802] jepsen worker 22 - jepsen.util 10912 :invoke :read [7 nil] INFO [2019-12-17 12:34:10,814] jepsen worker 70 - jepsen.util 8260 :info :write [23 37] :RequestCanceledException INFO [2019-12-17 12:34:10,817] jepsen worker 72 - jepsen.util 7812 :ok :read [24 8] INFO [2019-12-17 12:34:10,817] jepsen worker 72 - jepsen.util 7812 :invoke :read [24 nil] INFO [2019-12-17 12:34:10,819] jepsen worker 70 - jepsen.util 8350 :invoke :read [23 nil] INFO [2019-12-17 12:34:10,821] jepsen worker 11 - jepsen.util 8021 :ok :read [3 28] INFO [2019-12-17 12:34:10,821] jepsen worker 11 - jepsen.util 8021 :invoke :cas [3 [22 15]] INFO [2019-12-17 12:34:10,826] jepsen worker 40 - jepsen.util 11380 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:34:10,826] jepsen worker 84 - jepsen.util 10524 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:34:10,827] jepsen worker 61 - jepsen.util 11671 :info :write [20 31] :RequestCanceledException INFO [2019-12-17 12:34:10,827] jepsen worker 40 - jepsen.util 11380 :invoke :read [13 nil] INFO [2019-12-17 12:34:10,827] jepsen worker 84 - jepsen.util 10524 :invoke :read [28 nil] INFO [2019-12-17 12:34:10,829] jepsen worker 46 - jepsen.util 8146 :ok :read [15 18] INFO [2019-12-17 12:34:10,829] jepsen worker 46 - jepsen.util 8146 :invoke :read [15 nil] INFO [2019-12-17 12:34:10,831] jepsen worker 61 - jepsen.util 11761 :invoke :read [20 nil] INFO [2019-12-17 12:34:10,843] jepsen worker 6 - jepsen.util 11436 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:34:10,843] jepsen worker 44 - jepsen.util 11204 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:34:10,843] jepsen worker 48 - jepsen.util 9858 :info :write [16 16] :RequestCanceledException INFO [2019-12-17 12:34:10,843] jepsen worker 6 - jepsen.util 11436 :invoke :cas [2 [24 42]] INFO [2019-12-17 12:34:10,843] jepsen worker 44 - jepsen.util 11204 :invoke :read [14 nil] INFO [2019-12-17 12:34:10,846] jepsen worker 48 - jepsen.util 9948 :invoke :cas [16 [45 24]] INFO [2019-12-17 12:34:10,859] jepsen worker 11 - jepsen.util 8021 :fail :cas [3 [22 15]] :ValueNotSwapFrom INFO [2019-12-17 12:34:10,859] jepsen worker 11 - jepsen.util 8021 :invoke :cas [3 [0 38]] INFO [2019-12-17 12:34:10,860] jepsen worker 32 - jepsen.util 8042 :info :write [10 12] :RequestCanceledException INFO [2019-12-17 12:34:10,860] jepsen worker 29 - jepsen.util 6419 :info :write [9 41] :RequestCanceledException INFO [2019-12-17 12:34:10,860] jepsen worker 19 - jepsen.util 8119 :info :cas [6 [29 33]] :RequestCanceledException INFO [2019-12-17 12:34:10,863] jepsen worker 19 - jepsen.util 8209 :invoke :write [6 49] INFO [2019-12-17 12:34:10,863] jepsen worker 29 - jepsen.util 6509 :invoke :write [9 45] INFO [2019-12-17 12:34:10,863] jepsen worker 32 - jepsen.util 8132 :invoke :cas [10 [8 42]] INFO [2019-12-17 12:34:10,896] jepsen worker 66 - jepsen.util 10686 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:34:10,896] jepsen worker 63 - jepsen.util 6723 :info :cas [21 [5 23]] :RequestCanceledException INFO [2019-12-17 12:34:10,896] jepsen worker 66 - jepsen.util 10686 :invoke :read [22 nil] INFO [2019-12-17 12:34:10,899] jepsen worker 63 - jepsen.util 6813 :invoke :read [21 nil] INFO [2019-12-17 12:34:10,899] jepsen worker 46 - jepsen.util 8146 :ok :read [15 18] INFO [2019-12-17 12:34:10,900] jepsen worker 46 - jepsen.util 8146 :invoke :read [15 nil] INFO [2019-12-17 12:34:10,908] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46124,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:34:10,908] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46124,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:34:10,908] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46127,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:34:10,909] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46127,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:34:10,912] jepsen worker 72 - jepsen.util 7812 :ok :read [24 8] INFO [2019-12-17 12:34:10,913] jepsen worker 72 - jepsen.util 7812 :invoke :write [24 21] INFO [2019-12-17 12:34:10,929] jepsen worker 54 - jepsen.util 8784 :info :write [18 49] :RequestCanceledException INFO [2019-12-17 12:34:10,930] jepsen worker 11 - jepsen.util 8021 :fail :cas [3 [0 38]] :ValueNotSwapFrom INFO [2019-12-17 12:34:10,930] jepsen worker 11 - jepsen.util 8021 :invoke :cas [3 [24 45]] INFO [2019-12-17 12:34:10,932] jepsen worker 54 - jepsen.util 8874 :invoke :write [18 11] INFO [2019-12-17 12:34:10,941] jepsen worker 15 - jepsen.util 9915 :info :write [5 34] :RequestCanceledException INFO [2019-12-17 12:34:10,947] jepsen worker 15 - jepsen.util 10005 :invoke :write [5 15] INFO [2019-12-17 12:34:10,950] jepsen worker 32 - jepsen.util 8132 :fail :cas [10 [8 42]] :ValueNotSwapFrom INFO [2019-12-17 12:34:10,950] jepsen worker 32 - jepsen.util 8132 :invoke :read [10 nil] INFO [2019-12-17 12:34:10,970] jepsen worker 46 - jepsen.util 8146 :ok :read [15 18] INFO [2019-12-17 12:34:10,970] jepsen worker 46 - jepsen.util 8146 :invoke :cas [15 [32 42]] INFO [2019-12-17 12:34:10,977] jepsen worker 63 - jepsen.util 6813 :ok :read [21 40] INFO [2019-12-17 12:34:10,978] jepsen worker 63 - jepsen.util 6813 :invoke :cas [21 [8 20]] INFO [2019-12-17 12:34:10,987] jepsen worker 79 - jepsen.util 8179 :fail :cas [26 [33 0]] :ValueNotSwapFrom INFO [2019-12-17 12:34:10,987] jepsen worker 79 - jepsen.util 8179 :invoke :write [26 29] INFO [2019-12-17 12:34:10,993] jepsen worker 11 - jepsen.util 8021 :fail :cas [3 [24 45]] :ValueNotSwapFrom INFO [2019-12-17 12:34:10,993] jepsen worker 11 - jepsen.util 8021 :invoke :cas [3 [26 18]] INFO [2019-12-17 12:34:11,016] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46143,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:34:11,016] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46143,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:34:11,017] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46147,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:34:11,017] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46147,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:34:11,017] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46148,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:34:11,017] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46148,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:34:11,019] jepsen worker 56 - jepsen.util 8966 :ok :read [18 48] INFO [2019-12-17 12:34:11,019] jepsen worker 56 - jepsen.util 8966 :invoke :write [18 46] INFO [2019-12-17 12:34:11,026] jepsen worker 47 - jepsen.util 8237 :info :write [15 20] :RequestCanceledException INFO [2019-12-17 12:34:11,030] jepsen worker 47 - jepsen.util 8327 :invoke :cas [15 [31 12]] INFO [2019-12-17 12:34:11,032] jepsen worker 63 - jepsen.util 6813 :fail :cas [21 [8 20]] :ValueNotSwapFrom INFO [2019-12-17 12:34:11,032] jepsen worker 63 - jepsen.util 6813 :invoke :read [21 nil] INFO [2019-12-17 12:34:11,041] jepsen worker 32 - jepsen.util 8132 :ok :read [10 38] INFO [2019-12-17 12:34:11,041] jepsen worker 32 - jepsen.util 8132 :invoke :read [10 nil] INFO [2019-12-17 12:34:11,052] jepsen worker 46 - jepsen.util 8146 :fail :cas [15 [32 42]] :ValueNotSwapFrom INFO [2019-12-17 12:34:11,052] jepsen worker 46 - jepsen.util 8146 :invoke :read [15 nil] INFO [2019-12-17 12:34:11,059] jepsen worker 11 - jepsen.util 8021 :fail :cas [3 [26 18]] :ValueNotSwapFrom INFO [2019-12-17 12:34:11,059] jepsen worker 11 - jepsen.util 8021 :invoke :cas [3 [9 33]] INFO [2019-12-17 12:34:11,082] jepsen worker 63 - jepsen.util 6813 :ok :read [21 40] INFO [2019-12-17 12:34:11,082] jepsen worker 63 - jepsen.util 6813 :invoke :read [21 nil] INFO [2019-12-17 12:34:11,087] jepsen worker 32 - jepsen.util 8132 :ok :read [10 38] INFO [2019-12-17 12:34:11,087] jepsen worker 32 - jepsen.util 8132 :invoke :cas [10 [10 31]] INFO [2019-12-17 12:34:11,092] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:34:11,092] jepsen nemesis - couchbase.util Rebalance check count: 357 INFO [2019-12-17 12:34:11,092] jepsen nemesis - couchbase.util Rebalance stuck count: 338 INFO [2019-12-17 12:34:11,096] jepsen worker 24 - jepsen.util 9744 :info :write [8 7] :RequestCanceledException INFO [2019-12-17 12:34:11,099] jepsen worker 24 - jepsen.util 9834 :invoke :write [8 46] INFO [2019-12-17 12:34:11,114] jepsen worker 18 - jepsen.util 8118 :ok :write [6 49] INFO [2019-12-17 12:34:11,114] jepsen worker 18 - jepsen.util 8118 :invoke :cas [6 [46 0]] INFO [2019-12-17 12:34:11,129] jepsen worker 42 - jepsen.util 11652 :info :cas [14 [2 36]] :RequestCanceledException INFO [2019-12-17 12:34:11,132] jepsen worker 42 - jepsen.util 11742 :invoke :cas [14 [13 48]] INFO [2019-12-17 12:34:11,156] jepsen worker 63 - jepsen.util 6813 :ok :read [21 40] INFO [2019-12-17 12:34:11,156] jepsen worker 63 - jepsen.util 6813 :invoke :write [21 45] INFO [2019-12-17 12:34:11,159] jepsen worker 46 - jepsen.util 8146 :ok :read [15 18] INFO [2019-12-17 12:34:11,159] jepsen worker 46 - jepsen.util 8146 :invoke :cas [15 [47 9]] INFO [2019-12-17 12:34:11,160] jepsen worker 52 - jepsen.util 8062 :fail :read [17 nil] :RequestTimeoutException INFO [2019-12-17 12:34:11,160] jepsen worker 52 - jepsen.util 8062 :invoke :read [17 nil] INFO [2019-12-17 12:34:11,172] jepsen worker 51 - jepsen.util 7521 :fail :cas [17 [49 35]] :ValueNotSwapFrom INFO [2019-12-17 12:34:11,174] jepsen worker 51 - jepsen.util 7521 :invoke :read [17 nil] INFO [2019-12-17 12:34:11,198] jepsen worker 3 - jepsen.util 7923 :ok :read [1 27] INFO [2019-12-17 12:34:11,198] jepsen worker 3 - jepsen.util 7923 :invoke :read [1 nil] INFO [2019-12-17 12:34:11,218] jepsen worker 13 - jepsen.util 7483 :ok :read [4 21] INFO [2019-12-17 12:34:11,219] jepsen worker 13 - jepsen.util 7483 :invoke :cas [4 [45 7]] INFO [2019-12-17 12:34:11,221] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46166,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:34:11,221] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46166,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:34:11,229] jepsen worker 8 - jepsen.util 10718 :info :cas [2 [35 27]] :RequestCanceledException INFO [2019-12-17 12:34:11,229] jepsen worker 17 - jepsen.util 10817 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:34:11,229] jepsen worker 17 - jepsen.util 10817 :invoke :cas [5 [32 5]] INFO [2019-12-17 12:34:11,232] jepsen worker 11 - jepsen.util 8021 :fail :cas [3 [9 33]] :ValueNotSwapFrom INFO [2019-12-17 12:34:11,232] jepsen worker 11 - jepsen.util 8021 :invoke :cas [3 [28 27]] INFO [2019-12-17 12:34:11,233] jepsen worker 8 - jepsen.util 10808 :invoke :cas [2 [44 17]] INFO [2019-12-17 12:34:11,235] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46173,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:34:11,235] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46173,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:34:11,240] jepsen worker 31 - jepsen.util 7141 :fail :cas [10 [20 29]] :ValueNotSwapFrom INFO [2019-12-17 12:34:11,240] jepsen worker 31 - jepsen.util 7141 :invoke :write [10 47] INFO [2019-12-17 12:34:11,257] jepsen worker 7 - jepsen.util 11077 :info :cas [2 [12 43]] :RequestCanceledException INFO [2019-12-17 12:34:11,262] jepsen worker 7 - jepsen.util 11167 :invoke :cas [2 [24 27]] INFO [2019-12-17 12:34:11,274] jepsen worker 5 - jepsen.util 7475 :ok :read [1 27] INFO [2019-12-17 12:34:11,274] jepsen worker 5 - jepsen.util 7475 :invoke :write [1 2] INFO [2019-12-17 12:34:11,281] jepsen worker 32 - jepsen.util 8132 :fail :cas [10 [10 31]] :ValueNotSwapFrom INFO [2019-12-17 12:34:11,281] jepsen worker 32 - jepsen.util 8132 :invoke :write [10 23] INFO [2019-12-17 12:34:11,292] jepsen worker 63 - jepsen.util 6813 :ok :write [21 45] INFO [2019-12-17 12:34:11,292] jepsen worker 63 - jepsen.util 6813 :invoke :cas [21 [23 46]] INFO [2019-12-17 12:34:11,293] jepsen worker 1 - jepsen.util 11071 :info :cas [0 [42 26]] :RequestCanceledException INFO [2019-12-17 12:34:11,297] jepsen worker 1 - jepsen.util 11161 :invoke :read [0 nil] INFO [2019-12-17 12:34:11,311] jepsen worker 43 - jepsen.util 10393 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:34:11,312] jepsen worker 77 - jepsen.util 11147 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:34:11,312] jepsen worker 77 - jepsen.util 11147 :invoke :cas [25 [17 49]] INFO [2019-12-17 12:34:11,312] jepsen worker 43 - jepsen.util 10393 :invoke :write [14 17] INFO [2019-12-17 12:34:11,324] jepsen worker 46 - jepsen.util 8146 :fail :cas [15 [47 9]] :ValueNotSwapFrom INFO [2019-12-17 12:34:11,324] jepsen worker 46 - jepsen.util 8146 :invoke :write [15 36] INFO [2019-12-17 12:34:11,324] jepsen worker 3 - jepsen.util 7923 :ok :read [1 27] INFO [2019-12-17 12:34:11,324] jepsen worker 3 - jepsen.util 7923 :invoke :read [1 nil] INFO [2019-12-17 12:34:11,326] jepsen worker 49 - jepsen.util 10309 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:34:11,327] jepsen worker 75 - jepsen.util 10425 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:34:11,327] jepsen worker 49 - jepsen.util 10309 :invoke :cas [16 [5 31]] INFO [2019-12-17 12:34:11,327] jepsen worker 75 - jepsen.util 10425 :invoke :read [25 nil] INFO [2019-12-17 12:34:11,341] jepsen worker 2 - jepsen.util 10442 :info :write [0 18] :RequestCanceledException INFO [2019-12-17 12:34:11,345] jepsen worker 2 - jepsen.util 10532 :invoke :write [0 31] INFO [2019-12-17 12:34:11,363] jepsen worker 80 - jepsen.util 7820 :ok :read [26 47] INFO [2019-12-17 12:34:11,363] jepsen worker 80 - jepsen.util 7820 :invoke :write [26 32] INFO [2019-12-17 12:34:11,366] jepsen worker 63 - jepsen.util 6813 :fail :cas [21 [23 46]] :ValueNotSwapFrom INFO [2019-12-17 12:34:11,366] jepsen worker 63 - jepsen.util 6813 :invoke :cas [21 [49 39]] INFO [2019-12-17 12:34:11,396] jepsen worker 88 - jepsen.util 7918 :info :write [29 3] :RequestCanceledException INFO [2019-12-17 12:34:11,400] jepsen worker 88 - jepsen.util 8008 :invoke :write [29 36] INFO [2019-12-17 12:34:11,411] jepsen worker 53 - jepsen.util 7433 :ok :read [17 0] INFO [2019-12-17 12:34:11,411] jepsen worker 63 - jepsen.util 6813 :fail :cas [21 [49 39]] :ValueNotSwapFrom INFO [2019-12-17 12:34:11,411] jepsen worker 53 - jepsen.util 7433 :invoke :read [17 nil] INFO [2019-12-17 12:34:11,411] jepsen worker 63 - jepsen.util 6813 :invoke :write [21 3] INFO [2019-12-17 12:34:11,436] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46202,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:34:11,437] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46202,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:34:11,437] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46205,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:34:11,437] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46205,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:34:11,437] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46206,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:34:11,437] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46206,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:34:11,443] jepsen worker 12 - jepsen.util 8472 :info :write [4 9] :RequestCanceledException INFO [2019-12-17 12:34:11,446] jepsen worker 12 - jepsen.util 8562 :invoke :cas [4 [29 36]] INFO [2019-12-17 12:34:11,448] jepsen worker 78 - jepsen.util 7818 :fail :cas [26 [18 13]] :ValueNotSwapFrom INFO [2019-12-17 12:34:11,448] jepsen worker 78 - jepsen.util 7818 :invoke :read [26 nil] INFO [2019-12-17 12:34:11,489] jepsen worker 63 - jepsen.util 6813 :ok :write [21 3] INFO [2019-12-17 12:34:11,489] jepsen worker 63 - jepsen.util 6813 :invoke :read [21 nil] INFO [2019-12-17 12:34:11,513] jepsen worker 63 - jepsen.util 6813 :ok :read [21 3] INFO [2019-12-17 12:34:11,513] jepsen worker 63 - jepsen.util 6813 :invoke :cas [21 [36 31]] INFO [2019-12-17 12:34:11,541] jepsen worker 63 - jepsen.util 6813 :fail :cas [21 [36 31]] :ValueNotSwapFrom INFO [2019-12-17 12:34:11,542] jepsen worker 63 - jepsen.util 6813 :invoke :cas [21 [39 16]] INFO [2019-12-17 12:34:11,568] jepsen worker 63 - jepsen.util 6813 :fail :cas [21 [39 16]] :ValueNotSwapFrom INFO [2019-12-17 12:34:11,568] jepsen worker 63 - jepsen.util 6813 :invoke :cas [21 [17 0]] INFO [2019-12-17 12:34:11,591] jepsen worker 63 - jepsen.util 6813 :fail :cas [21 [17 0]] :ValueNotSwapFrom INFO [2019-12-17 12:34:11,592] jepsen worker 63 - jepsen.util 6813 :invoke :cas [21 [17 4]] INFO [2019-12-17 12:34:11,616] jepsen worker 63 - jepsen.util 6813 :fail :cas [21 [17 4]] :ValueNotSwapFrom INFO [2019-12-17 12:34:11,617] jepsen worker 63 - jepsen.util 6813 :invoke :write [21 29] INFO [2019-12-17 12:34:11,670] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46218,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:34:11,670] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46218,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:34:11,670] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46220,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:34:11,671] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46220,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:34:11,673] jepsen worker 13 - jepsen.util 7483 :fail :cas [4 [45 7]] :ValueNotSwapFrom INFO [2019-12-17 12:34:11,673] jepsen worker 13 - jepsen.util 7483 :invoke :cas [4 [32 26]] INFO [2019-12-17 12:34:11,680] jepsen worker 63 - jepsen.util 6813 :ok :write [21 29] INFO [2019-12-17 12:34:11,680] jepsen worker 63 - jepsen.util 6813 :invoke :read [21 nil] INFO [2019-12-17 12:34:11,709] jepsen worker 63 - jepsen.util 6813 :ok :read [21 29] INFO [2019-12-17 12:34:11,709] jepsen worker 63 - jepsen.util 6813 :invoke :write [21 11] INFO [2019-12-17 12:34:11,713] jepsen worker 68 - jepsen.util 10328 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:34:11,713] jepsen worker 50 - jepsen.util 11570 :info :cas [16 [44 9]] :RequestCanceledException INFO [2019-12-17 12:34:11,714] jepsen worker 68 - jepsen.util 10328 :invoke :write [22 19] INFO [2019-12-17 12:34:11,717] jepsen worker 50 - jepsen.util 11660 :invoke :write [16 37] INFO [2019-12-17 12:34:11,728] jepsen worker 39 - jepsen.util 10569 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:34:11,729] jepsen worker 39 - jepsen.util 10569 :invoke :read [13 nil] INFO [2019-12-17 12:34:11,729] jepsen worker 38 - jepsen.util 8228 :info :write [12 6] :RequestCanceledException INFO [2019-12-17 12:34:11,729] jepsen worker 86 - jepsen.util 11426 :info :cas [28 [27 8]] :RequestCanceledException INFO [2019-12-17 12:34:11,729] jepsen worker 4 - jepsen.util 7384 :info :cas [1 [27 45]] :RequestCanceledException INFO [2019-12-17 12:34:11,734] jepsen worker 38 - jepsen.util 8318 :invoke :read [12 nil] INFO [2019-12-17 12:34:11,735] jepsen worker 4 - jepsen.util 7474 :invoke :write [1 35] INFO [2019-12-17 12:34:11,735] jepsen worker 86 - jepsen.util 11516 :invoke :cas [28 [48 20]] INFO [2019-12-17 12:34:11,743] jepsen worker 82 - jepsen.util 11512 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:34:11,743] jepsen worker 41 - jepsen.util 11471 :info :write [13 43] :RequestCanceledException INFO [2019-12-17 12:34:11,743] jepsen worker 82 - jepsen.util 11512 :invoke :read [27 nil] INFO [2019-12-17 12:34:11,745] jepsen worker 51 - jepsen.util 7521 :ok :read [17 0] INFO [2019-12-17 12:34:11,745] jepsen worker 51 - jepsen.util 7521 :invoke :read [17 nil] INFO [2019-12-17 12:34:11,746] jepsen worker 41 - jepsen.util 11561 :invoke :write [13 2] INFO [2019-12-17 12:34:11,759] jepsen worker 81 - jepsen.util 10611 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:34:11,759] jepsen worker 59 - jepsen.util 10499 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:34:11,760] jepsen worker 81 - jepsen.util 10611 :invoke :write [27 11] INFO [2019-12-17 12:34:11,760] jepsen worker 59 - jepsen.util 10499 :invoke :cas [19 [13 40]] INFO [2019-12-17 12:34:11,775] jepsen worker 63 - jepsen.util 6813 :ok :write [21 11] INFO [2019-12-17 12:34:11,775] jepsen worker 63 - jepsen.util 6813 :invoke :write [21 18] INFO [2019-12-17 12:34:11,794] jepsen worker 21 - jepsen.util 10281 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:34:11,794] jepsen worker 23 - jepsen.util 10733 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:34:11,794] jepsen worker 45 - jepsen.util 8055 :fail :read [15 nil] :RequestTimeoutException INFO [2019-12-17 12:34:11,794] jepsen worker 23 - jepsen.util 10733 :invoke :read [7 nil] INFO [2019-12-17 12:34:11,794] jepsen worker 21 - jepsen.util 10281 :invoke :read [7 nil] INFO [2019-12-17 12:34:11,794] jepsen worker 45 - jepsen.util 8055 :invoke :write [15 21] INFO [2019-12-17 12:34:11,814] jepsen worker 60 - jepsen.util 12030 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:34:11,814] jepsen worker 60 - jepsen.util 12030 :invoke :read [20 nil] INFO [2019-12-17 12:34:11,829] jepsen worker 83 - jepsen.util 11153 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:34:11,829] jepsen worker 10 - jepsen.util 8110 :info :write [3 13] :RequestCanceledException INFO [2019-12-17 12:34:11,829] jepsen worker 69 - jepsen.util 8079 :info :write [23 17] :RequestCanceledException INFO [2019-12-17 12:34:11,830] jepsen worker 83 - jepsen.util 11153 :invoke :write [27 48] INFO [2019-12-17 12:34:11,830] jepsen worker 3 - jepsen.util 7923 :ok :read [1 27] INFO [2019-12-17 12:34:11,830] jepsen worker 3 - jepsen.util 7923 :invoke :cas [1 [24 44]] INFO [2019-12-17 12:34:11,834] jepsen worker 69 - jepsen.util 8169 :invoke :cas [23 [10 31]] INFO [2019-12-17 12:34:11,834] jepsen worker 10 - jepsen.util 8200 :invoke :cas [3 [39 4]] INFO [2019-12-17 12:34:11,836] jepsen worker 16 - jepsen.util 12166 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:34:11,836] jepsen worker 76 - jepsen.util 10696 :info :cas [25 [2 39]] :RequestCanceledException INFO [2019-12-17 12:34:11,836] jepsen worker 16 - jepsen.util 12166 :invoke :write [5 7] INFO [2019-12-17 12:34:11,839] jepsen worker 76 - jepsen.util 10786 :invoke :cas [25 [38 4]] INFO [2019-12-17 12:34:11,845] jepsen worker 63 - jepsen.util 6813 :ok :write [21 18] INFO [2019-12-17 12:34:11,845] jepsen worker 63 - jepsen.util 6813 :invoke :read [21 nil] INFO [2019-12-17 12:34:11,858] jepsen worker 87 - jepsen.util 7377 :ok :read [29 36] INFO [2019-12-17 12:34:11,858] jepsen worker 87 - jepsen.util 7377 :invoke :write [29 6] INFO [2019-12-17 12:34:11,888] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46247,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:34:11,888] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46247,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:34:11,888] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46251,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:34:11,888] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46251,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:34:11,889] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46252,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:34:11,889] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46252,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:34:11,891] jepsen worker 27 - jepsen.util 7857 :fail :read [9 nil] :RequestTimeoutException INFO [2019-12-17 12:34:11,892] jepsen worker 27 - jepsen.util 7857 :invoke :write [9 21] INFO [2019-12-17 12:34:11,907] jepsen worker 13 - jepsen.util 7483 :fail :cas [4 [32 26]] :ValueNotSwapFrom INFO [2019-12-17 12:34:11,907] jepsen worker 13 - jepsen.util 7483 :invoke :read [4 nil] INFO [2019-12-17 12:34:11,917] jepsen worker 10 - jepsen.util 8200 :fail :cas [3 [39 4]] :ValueNotSwapFrom INFO [2019-12-17 12:34:11,917] jepsen worker 10 - jepsen.util 8200 :invoke :cas [3 [6 31]] INFO [2019-12-17 12:34:11,922] jepsen worker 63 - jepsen.util 6813 :ok :read [21 18] INFO [2019-12-17 12:34:11,923] jepsen worker 63 - jepsen.util 6813 :invoke :read [21 nil] INFO [2019-12-17 12:34:11,929] jepsen worker 14 - jepsen.util 8114 :info :write [4 18] :RequestCanceledException INFO [2019-12-17 12:34:11,931] jepsen worker 14 - jepsen.util 8204 :invoke :write [4 3] INFO [2019-12-17 12:34:11,947] jepsen worker 10 - jepsen.util 8200 :fail :cas [3 [6 31]] :ValueNotSwapFrom INFO [2019-12-17 12:34:11,947] jepsen worker 10 - jepsen.util 8200 :invoke :cas [3 [15 2]] INFO [2019-12-17 12:34:11,956] jepsen worker 63 - jepsen.util 6813 :ok :read [21 18] INFO [2019-12-17 12:34:11,956] jepsen worker 63 - jepsen.util 6813 :invoke :read [21 nil] INFO [2019-12-17 12:34:11,980] jepsen worker 10 - jepsen.util 8200 :fail :cas [3 [15 2]] :ValueNotSwapFrom INFO [2019-12-17 12:34:11,981] jepsen worker 10 - jepsen.util 8200 :invoke :read [3 nil] INFO [2019-12-17 12:34:11,992] jepsen worker 63 - jepsen.util 6813 :ok :read [21 18] INFO [2019-12-17 12:34:11,992] jepsen worker 63 - jepsen.util 6813 :invoke :write [21 12] INFO [2019-12-17 12:34:11,997] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46263,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:34:11,998] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46263,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:34:11,998] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46266,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:34:11,998] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46266,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:34:11,998] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46267,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:34:11,998] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46267,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:34:12,012] jepsen worker 10 - jepsen.util 8200 :ok :read [3 28] INFO [2019-12-17 12:34:12,013] jepsen worker 10 - jepsen.util 8200 :invoke :read [3 nil] INFO [2019-12-17 12:34:12,013] jepsen worker 34 - jepsen.util 10654 :info :write [11 46] :RequestCanceledException INFO [2019-12-17 12:34:12,015] jepsen worker 34 - jepsen.util 10744 :invoke :write [11 28] INFO [2019-12-17 12:34:12,051] jepsen worker 38 - jepsen.util 8318 :ok :read [12 28] INFO [2019-12-17 12:34:12,052] jepsen worker 38 - jepsen.util 8318 :invoke :write [12 43] INFO [2019-12-17 12:34:12,068] jepsen worker 12 - jepsen.util 8562 :fail :cas [4 [29 36]] :ValueNotSwapFrom INFO [2019-12-17 12:34:12,068] jepsen worker 12 - jepsen.util 8562 :invoke :read [4 nil] INFO [2019-12-17 12:34:12,072] jepsen worker 63 - jepsen.util 6813 :ok :write [21 12] INFO [2019-12-17 12:34:12,072] jepsen worker 63 - jepsen.util 6813 :invoke :write [21 13] INFO [2019-12-17 12:34:12,080] jepsen worker 19 - jepsen.util 8209 :ok :write [6 49] INFO [2019-12-17 12:34:12,080] jepsen worker 32 - jepsen.util 8132 :ok :write [10 23] INFO [2019-12-17 12:34:12,080] jepsen worker 19 - jepsen.util 8209 :invoke :read [6 nil] INFO [2019-12-17 12:34:12,080] jepsen worker 32 - jepsen.util 8132 :invoke :read [10 nil] INFO [2019-12-17 12:34:12,091] jepsen worker 78 - jepsen.util 7818 :ok :read [26 47] INFO [2019-12-17 12:34:12,091] jepsen worker 78 - jepsen.util 7818 :invoke :cas [26 [12 20]] INFO [2019-12-17 12:34:12,095] jepsen worker 73 - jepsen.util 7543 :ok :write [24 22] INFO [2019-12-17 12:34:12,095] jepsen worker 73 - jepsen.util 7543 :invoke :write [24 3] INFO [2019-12-17 12:34:12,096] jepsen worker 9 - jepsen.util 8199 :ok :write [3 20] INFO [2019-12-17 12:34:12,096] jepsen worker 9 - jepsen.util 8199 :invoke :cas [3 [9 45]] INFO [2019-12-17 12:34:12,112] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:34:12,112] jepsen nemesis - couchbase.util Rebalance check count: 358 INFO [2019-12-17 12:34:12,112] jepsen nemesis - couchbase.util Rebalance stuck count: 339 INFO [2019-12-17 12:34:12,114] jepsen worker 71 - jepsen.util 8531 :info :write [23 41] :RequestCanceledException INFO [2019-12-17 12:34:12,115] jepsen worker 71 - jepsen.util 8621 :invoke :read [23 nil] INFO [2019-12-17 12:34:12,165] jepsen worker 11 - jepsen.util 8021 :fail :cas [3 [28 27]] :CASMismatchException INFO [2019-12-17 12:34:12,165] jepsen worker 67 - jepsen.util 10957 :info :cas [22 [19 4]] :RequestCanceledException INFO [2019-12-17 12:34:12,165] jepsen worker 28 - jepsen.util 6238 :info :cas [9 [20 11]] :RequestCanceledException INFO [2019-12-17 12:34:12,165] jepsen worker 74 - jepsen.util 8264 :info :cas [24 [38 26]] :RequestCanceledException INFO [2019-12-17 12:34:12,165] jepsen worker 30 - jepsen.util 7860 :info :cas [10 [15 4]] :RequestCanceledException INFO [2019-12-17 12:34:12,165] jepsen worker 37 - jepsen.util 7507 :info :write [12 8] :RequestCanceledException INFO [2019-12-17 12:34:12,165] jepsen worker 11 - jepsen.util 8021 :invoke :cas [3 [3 17]] INFO [2019-12-17 12:34:12,165] jepsen worker 35 - jepsen.util 10655 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:34:12,166] jepsen worker 35 - jepsen.util 10655 :invoke :read [11 nil] INFO [2019-12-17 12:34:12,168] jepsen worker 30 - jepsen.util 7950 :invoke :cas [10 [27 17]] INFO [2019-12-17 12:34:12,168] jepsen worker 37 - jepsen.util 7597 :invoke :write [12 36] INFO [2019-12-17 12:34:12,168] jepsen worker 28 - jepsen.util 6328 :invoke :cas [9 [12 13]] INFO [2019-12-17 12:34:12,169] jepsen worker 74 - jepsen.util 8354 :invoke :cas [24 [14 33]] INFO [2019-12-17 12:34:12,170] jepsen worker 67 - jepsen.util 11047 :invoke :write [22 26] INFO [2019-12-17 12:34:12,198] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46287,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:34:12,198] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46287,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:34:12,199] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46289,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:34:12,199] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46289,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:34:12,212] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46294,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:34:12,212] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46294,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:34:12,213] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46298,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:34:12,213] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46298,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:34:12,230] jepsen worker 52 - jepsen.util 8062 :ok :read [17 0] INFO [2019-12-17 12:34:12,230] jepsen worker 52 - jepsen.util 8062 :invoke :read [17 nil] INFO [2019-12-17 12:34:12,241] jepsen worker 9 - jepsen.util 8199 :fail :cas [3 [9 45]] :ValueNotSwapFrom INFO [2019-12-17 12:34:12,241] jepsen worker 9 - jepsen.util 8199 :invoke :cas [3 [24 16]] INFO [2019-12-17 12:34:12,242] jepsen worker 32 - jepsen.util 8132 :ok :read [10 23] INFO [2019-12-17 12:34:12,242] jepsen worker 32 - jepsen.util 8132 :invoke :cas [10 [31 13]] INFO [2019-12-17 12:34:12,249] jepsen worker 53 - jepsen.util 7433 :ok :read [17 0] INFO [2019-12-17 12:34:12,249] jepsen worker 53 - jepsen.util 7433 :invoke :cas [17 [7 32]] INFO [2019-12-17 12:34:12,252] jepsen worker 28 - jepsen.util 6328 :fail :cas [9 [12 13]] :ValueNotSwapFrom INFO [2019-12-17 12:34:12,253] jepsen worker 28 - jepsen.util 6328 :invoke :cas [9 [9 38]] INFO [2019-12-17 12:34:12,257] jepsen worker 11 - jepsen.util 8021 :fail :cas [3 [3 17]] :ValueNotSwapFrom INFO [2019-12-17 12:34:12,258] jepsen worker 11 - jepsen.util 8021 :invoke :cas [3 [34 7]] INFO [2019-12-17 12:34:12,262] jepsen worker 30 - jepsen.util 7950 :fail :cas [10 [27 17]] :ValueNotSwapFrom INFO [2019-12-17 12:34:12,262] jepsen worker 30 - jepsen.util 7950 :invoke :write [10 24] INFO [2019-12-17 12:34:12,314] jepsen worker 69 - jepsen.util 8169 :fail :cas [23 [10 31]] :ValueNotSwapFrom INFO [2019-12-17 12:34:12,315] jepsen worker 69 - jepsen.util 8169 :invoke :read [23 nil] INFO [2019-12-17 12:34:12,316] jepsen worker 32 - jepsen.util 8132 :fail :cas [10 [31 13]] :ValueNotSwapFrom INFO [2019-12-17 12:34:12,316] jepsen worker 32 - jepsen.util 8132 :invoke :read [10 nil] INFO [2019-12-17 12:34:12,326] jepsen worker 89 - jepsen.util 7829 :info :write [29 49] :RequestCanceledException INFO [2019-12-17 12:34:12,329] jepsen worker 12 - jepsen.util 8562 :ok :read [4 21] INFO [2019-12-17 12:34:12,329] jepsen worker 89 - jepsen.util 7919 :invoke :cas [29 [36 7]] INFO [2019-12-17 12:34:12,329] jepsen worker 12 - jepsen.util 8562 :invoke :write [4 19] INFO [2019-12-17 12:34:12,355] jepsen worker 28 - jepsen.util 6328 :fail :cas [9 [9 38]] :ValueNotSwapFrom INFO [2019-12-17 12:34:12,355] jepsen worker 28 - jepsen.util 6328 :invoke :read [9 nil] INFO [2019-12-17 12:34:12,360] jepsen worker 63 - jepsen.util 6813 :ok :write [21 13] INFO [2019-12-17 12:34:12,360] jepsen worker 63 - jepsen.util 6813 :invoke :cas [21 [8 12]] INFO [2019-12-17 12:34:12,360] jepsen worker 64 - jepsen.util 6544 :info :write [21 10] :RequestCanceledException INFO [2019-12-17 12:34:12,360] jepsen worker 65 - jepsen.util 6635 :info :cas [21 [7 9]] :RequestCanceledException INFO [2019-12-17 12:34:12,363] jepsen worker 74 - jepsen.util 8354 :fail :cas [24 [14 33]] :ValueNotSwapFrom INFO [2019-12-17 12:34:12,364] jepsen worker 11 - jepsen.util 8021 :fail :cas [3 [34 7]] :ValueNotSwapFrom INFO [2019-12-17 12:34:12,364] jepsen worker 74 - jepsen.util 8354 :invoke :cas [24 [43 29]] INFO [2019-12-17 12:34:12,364] jepsen worker 11 - jepsen.util 8021 :invoke :write [3 45] INFO [2019-12-17 12:34:12,364] jepsen worker 64 - jepsen.util 6634 :invoke :write [21 46] INFO [2019-12-17 12:34:12,364] jepsen worker 65 - jepsen.util 6725 :invoke :write [21 20] INFO [2019-12-17 12:34:12,390] jepsen worker 9 - jepsen.util 8199 :fail :cas [3 [24 16]] :ValueNotSwapFrom INFO [2019-12-17 12:34:12,390] jepsen worker 9 - jepsen.util 8199 :invoke :read [3 nil] INFO [2019-12-17 12:34:12,420] jepsen worker 78 - jepsen.util 7818 :fail :cas [26 [12 20]] :ValueNotSwapFrom INFO [2019-12-17 12:34:12,420] jepsen worker 78 - jepsen.util 7818 :invoke :read [26 nil] INFO [2019-12-17 12:34:12,447] jepsen worker 63 - jepsen.util 6813 :fail :cas [21 [8 12]] :ValueNotSwapFrom INFO [2019-12-17 12:34:12,447] jepsen worker 63 - jepsen.util 6813 :invoke :cas [21 [18 7]] INFO [2019-12-17 12:34:12,460] jepsen worker 33 - jepsen.util 10743 :info :write [11 22] :RequestCanceledException INFO [2019-12-17 12:34:12,462] jepsen worker 33 - jepsen.util 10833 :invoke :read [11 nil] INFO [2019-12-17 12:34:12,482] jepsen worker 74 - jepsen.util 8354 :fail :cas [24 [43 29]] :ValueNotSwapFrom INFO [2019-12-17 12:34:12,482] jepsen worker 74 - jepsen.util 8354 :invoke :write [24 41] INFO [2019-12-17 12:34:12,495] jepsen worker 58 - jepsen.util 11758 :info :write [19 7] :RequestCanceledException INFO [2019-12-17 12:34:12,495] jepsen worker 57 - jepsen.util 10947 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:34:12,495] jepsen worker 57 - jepsen.util 10947 :invoke :read [19 nil] INFO [2019-12-17 12:34:12,497] jepsen worker 53 - jepsen.util 7433 :fail :cas [17 [7 32]] :ValueNotSwapFrom INFO [2019-12-17 12:34:12,498] jepsen worker 53 - jepsen.util 7433 :invoke :cas [17 [30 35]] INFO [2019-12-17 12:34:12,499] jepsen worker 58 - jepsen.util 11848 :invoke :read [19 nil] INFO [2019-12-17 12:34:12,512] jepsen worker 25 - jepsen.util 11455 :info :write [8 43] :RequestCanceledException INFO [2019-12-17 12:34:12,516] jepsen worker 25 - jepsen.util 11545 :invoke :write [8 45] INFO [2019-12-17 12:34:12,528] jepsen worker 36 - jepsen.util 8316 :info :write [12 14] :RequestCanceledException INFO [2019-12-17 12:34:12,529] jepsen worker 62 - jepsen.util 11132 :info :write [20 25] :RequestCanceledException INFO [2019-12-17 12:34:12,529] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46319,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:34:12,529] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46319,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:34:12,529] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46322,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:34:12,530] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46322,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:34:12,530] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46323,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:34:12,530] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46323,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:34:12,532] jepsen worker 36 - jepsen.util 8406 :invoke :cas [12 [17 19]] INFO [2019-12-17 12:34:12,534] jepsen worker 62 - jepsen.util 11222 :invoke :write [20 28] INFO [2019-12-17 12:34:12,537] jepsen worker 9 - jepsen.util 8199 :ok :read [3 20] INFO [2019-12-17 12:34:12,537] jepsen worker 9 - jepsen.util 8199 :invoke :write [3 17] INFO [2019-12-17 12:34:12,542] jepsen worker 28 - jepsen.util 6328 :ok :read [9 40] INFO [2019-12-17 12:34:12,542] jepsen worker 28 - jepsen.util 6328 :invoke :cas [9 [25 0]] INFO [2019-12-17 12:34:12,552] jepsen worker 69 - jepsen.util 8169 :ok :read [23 29] INFO [2019-12-17 12:34:12,552] jepsen worker 69 - jepsen.util 8169 :invoke :cas [23 [40 12]] INFO [2019-12-17 12:34:12,589] jepsen worker 51 - jepsen.util 7521 :ok :read [17 0] INFO [2019-12-17 12:34:12,589] jepsen worker 51 - jepsen.util 7521 :invoke :read [17 nil] INFO [2019-12-17 12:34:12,599] jepsen worker 28 - jepsen.util 6328 :fail :cas [9 [25 0]] :ValueNotSwapFrom INFO [2019-12-17 12:34:12,600] jepsen worker 28 - jepsen.util 6328 :invoke :cas [9 [34 43]] INFO [2019-12-17 12:34:12,612] jepsen worker 85 - jepsen.util 11605 :info :write [28 16] :RequestCanceledException INFO [2019-12-17 12:34:12,612] jepsen worker 64 - jepsen.util 6634 :ok :write [21 46] INFO [2019-12-17 12:34:12,612] jepsen worker 64 - jepsen.util 6634 :invoke :cas [21 [40 22]] INFO [2019-12-17 12:34:12,614] jepsen worker 85 - jepsen.util 11695 :invoke :write [28 27] INFO [2019-12-17 12:34:12,637] jepsen worker 28 - jepsen.util 6328 :fail :cas [9 [34 43]] :ValueNotSwapFrom INFO [2019-12-17 12:34:12,637] jepsen worker 28 - jepsen.util 6328 :invoke :cas [9 [20 41]] INFO [2019-12-17 12:34:12,643] jepsen worker 0 - jepsen.util 11430 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:34:12,643] jepsen worker 0 - jepsen.util 11430 :invoke :cas [0 [3 3]] INFO [2019-12-17 12:34:12,654] jepsen worker 78 - jepsen.util 7818 :ok :read [26 47] INFO [2019-12-17 12:34:12,654] jepsen worker 78 - jepsen.util 7818 :invoke :read [26 nil] INFO [2019-12-17 12:34:12,655] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46343,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:34:12,655] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46343,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:34:12,655] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46346,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:34:12,655] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46346,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:34:12,656] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46347,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:34:12,656] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46347,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:34:12,677] jepsen worker 28 - jepsen.util 6328 :fail :cas [9 [20 41]] :ValueNotSwapFrom INFO [2019-12-17 12:34:12,677] jepsen worker 28 - jepsen.util 6328 :invoke :cas [9 [26 49]] INFO [2019-12-17 12:34:12,693] jepsen worker 31 - jepsen.util 7141 :ok :write [10 47] INFO [2019-12-17 12:34:12,693] jepsen worker 31 - jepsen.util 7141 :invoke :write [10 22] INFO [2019-12-17 12:34:12,694] jepsen worker 55 - jepsen.util 8785 :info :write [18 16] :RequestCanceledException INFO [2019-12-17 12:34:12,697] jepsen worker 55 - jepsen.util 8875 :invoke :write [18 7] INFO [2019-12-17 12:34:12,711] jepsen worker 45 - jepsen.util 8055 :ok :write [15 21] INFO [2019-12-17 12:34:12,711] jepsen worker 45 - jepsen.util 8055 :invoke :read [15 nil] INFO [2019-12-17 12:34:12,714] jepsen worker 28 - jepsen.util 6328 :fail :cas [9 [26 49]] :ValueNotSwapFrom INFO [2019-12-17 12:34:12,714] jepsen worker 28 - jepsen.util 6328 :invoke :write [9 23] INFO [2019-12-17 12:34:12,718] jepsen worker 65 - jepsen.util 6725 :ok :write [21 20] INFO [2019-12-17 12:34:12,718] jepsen worker 65 - jepsen.util 6725 :invoke :write [21 39] INFO [2019-12-17 12:34:12,753] jepsen worker 19 - jepsen.util 8209 :ok :read [6 49] INFO [2019-12-17 12:34:12,753] jepsen worker 19 - jepsen.util 8209 :invoke :read [6 nil] INFO [2019-12-17 12:34:12,768] jepsen worker 3 - jepsen.util 7923 :fail :cas [1 [24 44]] :ValueNotSwapFrom INFO [2019-12-17 12:34:12,768] jepsen worker 71 - jepsen.util 8621 :ok :read [23 29] INFO [2019-12-17 12:34:12,768] jepsen worker 3 - jepsen.util 7923 :invoke :write [1 46] INFO [2019-12-17 12:34:12,768] jepsen worker 71 - jepsen.util 8621 :invoke :write [23 28] INFO [2019-12-17 12:34:12,809] jepsen worker 13 - jepsen.util 7483 :ok :read [4 21] INFO [2019-12-17 12:34:12,809] jepsen worker 13 - jepsen.util 7483 :invoke :read [4 nil] INFO [2019-12-17 12:34:12,809] jepsen worker 45 - jepsen.util 8055 :ok :read [15 21] INFO [2019-12-17 12:34:12,809] jepsen worker 45 - jepsen.util 8055 :invoke :cas [15 [12 47]] INFO [2019-12-17 12:34:12,814] jepsen worker 22 - jepsen.util 10912 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:34:12,814] jepsen worker 22 - jepsen.util 10912 :invoke :write [7 24] INFO [2019-12-17 12:34:12,829] jepsen worker 20 - jepsen.util 7490 :info :write [6 4] :RequestCanceledException INFO [2019-12-17 12:34:12,831] jepsen worker 20 - jepsen.util 7580 :invoke :cas [6 [28 21]] INFO [2019-12-17 12:34:12,843] jepsen worker 61 - jepsen.util 11761 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:34:12,843] jepsen worker 26 - jepsen.util 11456 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:34:12,843] jepsen worker 61 - jepsen.util 11761 :invoke :cas [20 [30 21]] INFO [2019-12-17 12:34:12,843] jepsen worker 26 - jepsen.util 11456 :invoke :write [8 7] INFO [2019-12-17 12:34:12,860] jepsen worker 48 - jepsen.util 9948 :info :cas [16 [45 24]] :RequestCanceledException INFO [2019-12-17 12:34:12,862] jepsen worker 51 - jepsen.util 7521 :ok :read [17 0] INFO [2019-12-17 12:34:12,862] jepsen worker 48 - jepsen.util 10038 :invoke :cas [16 [46 29]] INFO [2019-12-17 12:34:12,863] jepsen worker 51 - jepsen.util 7521 :invoke :read [17 nil] INFO [2019-12-17 12:34:12,888] jepsen worker 29 - jepsen.util 6509 :info :write [9 45] :RequestCanceledException INFO [2019-12-17 12:34:12,890] jepsen worker 74 - jepsen.util 8354 :ok :write [24 41] INFO [2019-12-17 12:34:12,890] jepsen worker 74 - jepsen.util 8354 :invoke :read [24 nil] INFO [2019-12-17 12:34:12,890] jepsen worker 29 - jepsen.util 6599 :invoke :read [9 nil] INFO [2019-12-17 12:34:12,909] jepsen worker 28 - jepsen.util 6328 :ok :write [9 23] INFO [2019-12-17 12:34:12,910] jepsen worker 28 - jepsen.util 6328 :invoke :write [9 9] INFO [2019-12-17 12:34:12,916] jepsen worker 45 - jepsen.util 8055 :fail :cas [15 [12 47]] :ValueNotSwapFrom INFO [2019-12-17 12:34:12,916] jepsen worker 45 - jepsen.util 8055 :invoke :read [15 nil] INFO [2019-12-17 12:34:12,925] jepsen worker 65 - jepsen.util 6725 :ok :write [21 39] INFO [2019-12-17 12:34:12,925] jepsen worker 65 - jepsen.util 6725 :invoke :cas [21 [29 7]] INFO [2019-12-17 12:34:12,929] jepsen worker 70 - jepsen.util 8350 :fail :read [23 nil] :RequestTimeoutException INFO [2019-12-17 12:34:12,929] jepsen worker 84 - jepsen.util 10524 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:34:12,929] jepsen worker 40 - jepsen.util 11380 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:34:12,929] jepsen worker 70 - jepsen.util 8350 :invoke :write [23 35] INFO [2019-12-17 12:34:12,929] jepsen worker 84 - jepsen.util 10524 :invoke :cas [28 [8 19]] INFO [2019-12-17 12:34:12,929] jepsen worker 40 - jepsen.util 11380 :invoke :cas [13 [36 13]] INFO [2019-12-17 12:34:12,943] jepsen worker 44 - jepsen.util 11204 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:34:12,943] jepsen worker 6 - jepsen.util 11436 :info :cas [2 [24 42]] :RequestCanceledException INFO [2019-12-17 12:34:12,943] jepsen worker 44 - jepsen.util 11204 :invoke :read [14 nil] INFO [2019-12-17 12:34:12,945] jepsen worker 6 - jepsen.util 11526 :invoke :read [2 nil] INFO [2019-12-17 12:34:12,971] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46365,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:34:12,971] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46365,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:34:12,972] jepsen worker 29 - jepsen.util 6599 :ok :read [9 40] INFO [2019-12-17 12:34:12,972] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46370,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:34:12,972] jepsen worker 29 - jepsen.util 6599 :invoke :cas [9 [3 12]] INFO [2019-12-17 12:34:12,972] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46370,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:34:12,972] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46371,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:34:12,972] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46371,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:34:12,986] jepsen worker 74 - jepsen.util 8354 :ok :read [24 41] INFO [2019-12-17 12:34:12,986] jepsen worker 74 - jepsen.util 8354 :invoke :cas [24 [32 15]] INFO [2019-12-17 12:34:12,995] jepsen worker 66 - jepsen.util 10686 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:34:12,995] jepsen worker 72 - jepsen.util 7812 :info :write [24 21] :RequestCanceledException INFO [2019-12-17 12:34:12,995] jepsen worker 66 - jepsen.util 10686 :invoke :write [22 2] INFO [2019-12-17 12:34:12,998] jepsen worker 72 - jepsen.util 7902 :invoke :read [24 nil] INFO [2019-12-17 12:34:13,044] jepsen worker 54 - jepsen.util 8874 :info :write [18 11] :RequestCanceledException INFO [2019-12-17 12:34:13,047] jepsen worker 54 - jepsen.util 8964 :invoke :cas [18 [28 41]] INFO [2019-12-17 12:34:13,060] jepsen worker 15 - jepsen.util 10005 :info :write [5 15] :RequestCanceledException INFO [2019-12-17 12:34:13,060] jepsen worker 79 - jepsen.util 8179 :info :write [26 29] :RequestCanceledException INFO [2019-12-17 12:34:13,060] jepsen worker 56 - jepsen.util 8966 :info :write [18 46] :RequestCanceledException INFO [2019-12-17 12:34:13,064] jepsen worker 15 - jepsen.util 10095 :invoke :read [5 nil] INFO [2019-12-17 12:34:13,064] jepsen worker 56 - jepsen.util 9056 :invoke :cas [18 [17 18]] INFO [2019-12-17 12:34:13,065] jepsen worker 79 - jepsen.util 8269 :invoke :write [26 16] INFO [2019-12-17 12:34:13,072] jepsen worker 69 - jepsen.util 8169 :fail :cas [23 [40 12]] :ValueNotSwapFrom INFO [2019-12-17 12:34:13,072] jepsen worker 69 - jepsen.util 8169 :invoke :cas [23 [47 20]] INFO [2019-12-17 12:34:13,073] jepsen worker 45 - jepsen.util 8055 :ok :read [15 21] INFO [2019-12-17 12:34:13,073] jepsen worker 45 - jepsen.util 8055 :invoke :write [15 3] INFO [2019-12-17 12:34:13,092] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46385,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:34:13,092] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46385,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:34:13,092] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46389,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:34:13,092] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46389,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:34:13,109] jepsen worker 74 - jepsen.util 8354 :fail :cas [24 [32 15]] :ValueNotSwapFrom INFO [2019-12-17 12:34:13,109] jepsen worker 74 - jepsen.util 8354 :invoke :read [24 nil] INFO [2019-12-17 12:34:13,111] jepsen worker 65 - jepsen.util 6725 :fail :cas [21 [29 7]] :ValueNotSwapFrom INFO [2019-12-17 12:34:13,111] jepsen worker 65 - jepsen.util 6725 :invoke :cas [21 [13 42]] INFO [2019-12-17 12:34:13,126] jepsen worker 47 - jepsen.util 8327 :info :cas [15 [31 12]] :RequestCanceledException INFO [2019-12-17 12:34:13,126] jepsen worker 18 - jepsen.util 8118 :info :cas [6 [46 0]] :RequestCanceledException INFO [2019-12-17 12:34:13,128] jepsen worker 47 - jepsen.util 8417 :invoke :read [15 nil] INFO [2019-12-17 12:34:13,129] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:34:13,129] jepsen nemesis - couchbase.util Rebalance check count: 359 INFO [2019-12-17 12:34:13,129] jepsen nemesis - couchbase.util Rebalance stuck count: 340 INFO [2019-12-17 12:34:13,130] jepsen worker 18 - jepsen.util 8208 :invoke :read [6 nil] INFO [2019-12-17 12:34:13,137] jepsen worker 72 - jepsen.util 7902 :ok :read [24 41] INFO [2019-12-17 12:34:13,137] jepsen worker 72 - jepsen.util 7902 :invoke :read [24 nil] INFO [2019-12-17 12:34:13,137] jepsen worker 29 - jepsen.util 6599 :fail :cas [9 [3 12]] :ValueNotSwapFrom INFO [2019-12-17 12:34:13,137] jepsen worker 29 - jepsen.util 6599 :invoke :cas [9 [47 22]] INFO [2019-12-17 12:34:13,160] jepsen worker 74 - jepsen.util 8354 :ok :read [24 41] INFO [2019-12-17 12:34:13,160] jepsen worker 74 - jepsen.util 8354 :invoke :read [24 nil] INFO [2019-12-17 12:34:13,170] jepsen worker 65 - jepsen.util 6725 :fail :cas [21 [13 42]] :ValueNotSwapFrom INFO [2019-12-17 12:34:13,170] jepsen worker 65 - jepsen.util 6725 :invoke :read [21 nil] INFO [2019-12-17 12:34:13,190] jepsen worker 18 - jepsen.util 8208 :ok :read [6 49] INFO [2019-12-17 12:34:13,190] jepsen worker 18 - jepsen.util 8208 :invoke :read [6 nil] INFO [2019-12-17 12:34:13,196] jepsen worker 24 - jepsen.util 9834 :info :write [8 46] :RequestCanceledException INFO [2019-12-17 12:34:13,199] jepsen worker 24 - jepsen.util 9924 :invoke :write [8 18] INFO [2019-12-17 12:34:13,213] jepsen worker 42 - jepsen.util 11742 :info :cas [14 [13 48]] :RequestCanceledException INFO [2019-12-17 12:34:13,218] jepsen worker 42 - jepsen.util 11832 :invoke :cas [14 [42 30]] INFO [2019-12-17 12:34:13,219] jepsen worker 28 - jepsen.util 6328 :ok :write [9 9] INFO [2019-12-17 12:34:13,219] jepsen worker 28 - jepsen.util 6328 :invoke :read [9 nil] INFO [2019-12-17 12:34:13,229] jepsen worker 36 - jepsen.util 8406 :fail :cas [12 [17 19]] :ValueNotSwapFrom INFO [2019-12-17 12:34:13,230] jepsen worker 36 - jepsen.util 8406 :invoke :read [12 nil] INFO [2019-12-17 12:34:13,246] jepsen worker 74 - jepsen.util 8354 :ok :read [24 41] INFO [2019-12-17 12:34:13,247] jepsen worker 74 - jepsen.util 8354 :invoke :read [24 nil] INFO [2019-12-17 12:34:13,248] jepsen worker 72 - jepsen.util 7902 :ok :read [24 41] INFO [2019-12-17 12:34:13,249] jepsen worker 72 - jepsen.util 7902 :invoke :read [24 nil] INFO [2019-12-17 12:34:13,253] jepsen worker 65 - jepsen.util 6725 :ok :read [21 39] INFO [2019-12-17 12:34:13,254] jepsen worker 65 - jepsen.util 6725 :invoke :read [21 nil] INFO [2019-12-17 12:34:13,273] jepsen worker 28 - jepsen.util 6328 :ok :read [9 9] INFO [2019-12-17 12:34:13,273] jepsen worker 28 - jepsen.util 6328 :invoke :cas [9 [12 12]] INFO [2019-12-17 12:34:13,276] jepsen worker 18 - jepsen.util 8208 :ok :read [6 49] INFO [2019-12-17 12:34:13,276] jepsen worker 18 - jepsen.util 8208 :invoke :cas [6 [8 17]] INFO [2019-12-17 12:34:13,290] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46409,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:34:13,290] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46409,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:34:13,308] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46416,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:34:13,309] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46416,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:34:13,309] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46419,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:34:13,309] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46419,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:34:13,310] jepsen worker 72 - jepsen.util 7902 :ok :read [24 41] INFO [2019-12-17 12:34:13,310] jepsen worker 72 - jepsen.util 7902 :invoke :write [24 45] INFO [2019-12-17 12:34:13,310] jepsen worker 74 - jepsen.util 8354 :ok :read [24 41] INFO [2019-12-17 12:34:13,311] jepsen worker 74 - jepsen.util 8354 :invoke :read [24 nil] INFO [2019-12-17 12:34:13,312] jepsen worker 13 - jepsen.util 7483 :ok :read [4 21] INFO [2019-12-17 12:34:13,313] jepsen worker 13 - jepsen.util 7483 :invoke :write [4 12] INFO [2019-12-17 12:34:13,321] jepsen worker 8 - jepsen.util 10808 :info :cas [2 [44 17]] :RequestCanceledException INFO [2019-12-17 12:34:13,321] jepsen worker 17 - jepsen.util 10817 :info :cas [5 [32 5]] :RequestCanceledException INFO [2019-12-17 12:34:13,325] jepsen worker 17 - jepsen.util 10907 :invoke :write [5 21] INFO [2019-12-17 12:34:13,325] jepsen worker 8 - jepsen.util 10898 :invoke :read [2 nil] INFO [2019-12-17 12:34:13,328] jepsen worker 28 - jepsen.util 6328 :fail :cas [9 [12 12]] :ValueNotSwapFrom INFO [2019-12-17 12:34:13,328] jepsen worker 28 - jepsen.util 6328 :invoke :write [9 0] INFO [2019-12-17 12:34:13,344] jepsen worker 7 - jepsen.util 11167 :info :cas [2 [24 27]] :RequestCanceledException INFO [2019-12-17 12:34:13,345] jepsen worker 65 - jepsen.util 6725 :ok :read [21 39] INFO [2019-12-17 12:34:13,345] jepsen worker 65 - jepsen.util 6725 :invoke :read [21 nil] INFO [2019-12-17 12:34:13,347] jepsen worker 7 - jepsen.util 11257 :invoke :cas [2 [4 26]] INFO [2019-12-17 12:34:13,360] jepsen worker 1 - jepsen.util 11161 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:34:13,361] jepsen worker 1 - jepsen.util 11161 :invoke :cas [0 [30 20]] INFO [2019-12-17 12:34:13,367] jepsen worker 18 - jepsen.util 8208 :fail :cas [6 [8 17]] :ValueNotSwapFrom INFO [2019-12-17 12:34:13,367] jepsen worker 18 - jepsen.util 8208 :invoke :cas [6 [13 39]] INFO [2019-12-17 12:34:13,374] jepsen worker 74 - jepsen.util 8354 :ok :read [24 41] INFO [2019-12-17 12:34:13,374] jepsen worker 74 - jepsen.util 8354 :invoke :read [24 nil] INFO [2019-12-17 12:34:13,396] jepsen worker 5 - jepsen.util 7475 :info :write [1 2] :RequestCanceledException INFO [2019-12-17 12:34:13,398] jepsen worker 5 - jepsen.util 7565 :invoke :write [1 18] INFO [2019-12-17 12:34:13,414] jepsen worker 77 - jepsen.util 11147 :info :cas [25 [17 49]] :RequestCanceledException INFO [2019-12-17 12:34:13,414] jepsen worker 43 - jepsen.util 10393 :info :write [14 17] :RequestCanceledException INFO [2019-12-17 12:34:13,414] jepsen worker 46 - jepsen.util 8146 :info :write [15 36] :RequestCanceledException INFO [2019-12-17 12:34:13,414] jepsen worker 80 - jepsen.util 7820 :info :write [26 32] :RequestCanceledException INFO [2019-12-17 12:34:13,417] jepsen worker 43 - jepsen.util 10483 :invoke :cas [14 [16 16]] INFO [2019-12-17 12:34:13,417] jepsen worker 80 - jepsen.util 7910 :invoke :cas [26 [47 0]] INFO [2019-12-17 12:34:13,418] jepsen worker 46 - jepsen.util 8236 :invoke :cas [15 [27 18]] INFO [2019-12-17 12:34:13,418] jepsen worker 77 - jepsen.util 11237 :invoke :write [25 3] INFO [2019-12-17 12:34:13,426] jepsen worker 2 - jepsen.util 10532 :info :write [0 31] :RequestCanceledException INFO [2019-12-17 12:34:13,426] jepsen worker 49 - jepsen.util 10309 :info :cas [16 [5 31]] :RequestCanceledException INFO [2019-12-17 12:34:13,426] jepsen worker 75 - jepsen.util 10425 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:34:13,427] jepsen worker 75 - jepsen.util 10425 :invoke :write [25 16] INFO [2019-12-17 12:34:13,431] jepsen worker 2 - jepsen.util 10622 :invoke :write [0 19] INFO [2019-12-17 12:34:13,432] jepsen worker 49 - jepsen.util 10399 :invoke :read [16 nil] INFO [2019-12-17 12:34:13,447] jepsen worker 28 - jepsen.util 6328 :ok :write [9 0] INFO [2019-12-17 12:34:13,449] jepsen worker 28 - jepsen.util 6328 :invoke :write [9 35] INFO [2019-12-17 12:34:13,470] jepsen worker 74 - jepsen.util 8354 :ok :read [24 41] INFO [2019-12-17 12:34:13,470] jepsen worker 74 - jepsen.util 8354 :invoke :cas [24 [13 45]] INFO [2019-12-17 12:34:13,491] jepsen worker 56 - jepsen.util 9056 :fail :cas [18 [17 18]] :ValueNotSwapFrom INFO [2019-12-17 12:34:13,491] jepsen worker 56 - jepsen.util 9056 :invoke :read [18 nil] INFO [2019-12-17 12:34:13,497] jepsen worker 88 - jepsen.util 8008 :info :write [29 36] :RequestCanceledException INFO [2019-12-17 12:34:13,502] jepsen worker 51 - jepsen.util 7521 :ok :read [17 0] INFO [2019-12-17 12:34:13,502] jepsen worker 88 - jepsen.util 8098 :invoke :write [29 42] INFO [2019-12-17 12:34:13,502] jepsen worker 51 - jepsen.util 7521 :invoke :cas [17 [33 10]] INFO [2019-12-17 12:34:13,508] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46446,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:34:13,508] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46446,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:34:13,528] jepsen worker 78 - jepsen.util 7818 :ok :read [26 47] INFO [2019-12-17 12:34:13,528] jepsen worker 78 - jepsen.util 7818 :invoke :read [26 nil] INFO [2019-12-17 12:34:13,529] jepsen worker 74 - jepsen.util 8354 :fail :cas [24 [13 45]] :ValueNotSwapFrom INFO [2019-12-17 12:34:13,529] jepsen worker 74 - jepsen.util 8354 :invoke :read [24 nil] INFO [2019-12-17 12:34:13,556] jepsen worker 74 - jepsen.util 8354 :ok :read [24 41] INFO [2019-12-17 12:34:13,557] jepsen worker 74 - jepsen.util 8354 :invoke :read [24 nil] INFO [2019-12-17 12:34:13,596] jepsen worker 28 - jepsen.util 6328 :ok :write [9 35] INFO [2019-12-17 12:34:13,596] jepsen worker 28 - jepsen.util 6328 :invoke :cas [9 [38 29]] INFO [2019-12-17 12:34:13,597] jepsen worker 74 - jepsen.util 8354 :ok :read [24 41] INFO [2019-12-17 12:34:13,597] jepsen worker 74 - jepsen.util 8354 :invoke :write [24 26] INFO [2019-12-17 12:34:13,609] jepsen worker 72 - jepsen.util 7902 :ok :write [24 45] INFO [2019-12-17 12:34:13,609] jepsen worker 45 - jepsen.util 8055 :ok :write [15 3] INFO [2019-12-17 12:34:13,610] jepsen worker 72 - jepsen.util 7902 :invoke :read [24 nil] INFO [2019-12-17 12:34:13,610] jepsen worker 45 - jepsen.util 8055 :invoke :read [15 nil] INFO [2019-12-17 12:34:13,637] jepsen worker 28 - jepsen.util 6328 :fail :cas [9 [38 29]] :ValueNotSwapFrom INFO [2019-12-17 12:34:13,637] jepsen worker 28 - jepsen.util 6328 :invoke :read [9 nil] INFO [2019-12-17 12:34:13,638] jepsen worker 31 - jepsen.util 7141 :ok :write [10 22] INFO [2019-12-17 12:34:13,638] jepsen worker 31 - jepsen.util 7141 :invoke :write [10 36] INFO [2019-12-17 12:34:13,640] jepsen worker 11 - jepsen.util 8021 :ok :write [3 45] INFO [2019-12-17 12:34:13,640] jepsen worker 11 - jepsen.util 8021 :invoke :cas [3 [16 4]] INFO [2019-12-17 12:34:13,668] jepsen worker 56 - jepsen.util 9056 :ok :read [18 48] INFO [2019-12-17 12:34:13,668] jepsen worker 56 - jepsen.util 9056 :invoke :write [18 6] INFO [2019-12-17 12:34:13,692] jepsen worker 52 - jepsen.util 8062 :ok :read [17 0] INFO [2019-12-17 12:34:13,693] jepsen worker 52 - jepsen.util 8062 :invoke :cas [17 [2 23]] INFO [2019-12-17 12:34:13,720] jepsen worker 45 - jepsen.util 8055 :ok :read [15 3] INFO [2019-12-17 12:34:13,721] jepsen worker 45 - jepsen.util 8055 :invoke :read [15 nil] INFO [2019-12-17 12:34:13,724] jepsen worker 53 - jepsen.util 7433 :fail :cas [17 [30 35]] :ValueNotSwapFrom INFO [2019-12-17 12:34:13,724] jepsen worker 53 - jepsen.util 7433 :invoke :cas [17 [24 13]] INFO [2019-12-17 12:34:13,724] jepsen worker 36 - jepsen.util 8406 :ok :read [12 28] INFO [2019-12-17 12:34:13,724] jepsen worker 36 - jepsen.util 8406 :invoke :cas [12 [47 23]] INFO [2019-12-17 12:34:13,726] jepsen worker 72 - jepsen.util 7902 :ok :read [24 45] INFO [2019-12-17 12:34:13,726] jepsen worker 72 - jepsen.util 7902 :invoke :read [24 nil] INFO [2019-12-17 12:34:13,743] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46469,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:34:13,743] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46469,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:34:13,744] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46472,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:34:13,744] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46472,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:34:13,757] jepsen worker 4 - jepsen.util 7474 :info :write [1 35] :RequestCanceledException INFO [2019-12-17 12:34:13,760] jepsen worker 4 - jepsen.util 7564 :invoke :cas [1 [8 4]] INFO [2019-12-17 12:34:13,762] jepsen worker 28 - jepsen.util 6328 :ok :read [9 35] INFO [2019-12-17 12:34:13,762] jepsen worker 28 - jepsen.util 6328 :invoke :write [9 1] INFO [2019-12-17 12:34:13,795] jepsen worker 41 - jepsen.util 11561 :info :write [13 2] :RequestCanceledException INFO [2019-12-17 12:34:13,797] jepsen worker 41 - jepsen.util 11651 :invoke :write [13 22] INFO [2019-12-17 12:34:13,813] jepsen worker 68 - jepsen.util 10328 :info :write [22 19] :RequestCanceledException INFO [2019-12-17 12:34:13,817] jepsen worker 68 - jepsen.util 10418 :invoke :read [22 nil] INFO [2019-12-17 12:34:13,829] jepsen worker 39 - jepsen.util 10569 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:34:13,829] jepsen worker 50 - jepsen.util 11660 :info :write [16 37] :RequestCanceledException INFO [2019-12-17 12:34:13,829] jepsen worker 86 - jepsen.util 11516 :info :cas [28 [48 20]] :RequestCanceledException INFO [2019-12-17 12:34:13,829] jepsen worker 39 - jepsen.util 10569 :invoke :write [13 35] INFO [2019-12-17 12:34:13,832] jepsen worker 86 - jepsen.util 11606 :invoke :write [28 32] INFO [2019-12-17 12:34:13,832] jepsen worker 50 - jepsen.util 11750 :invoke :read [16 nil] INFO [2019-12-17 12:34:13,837] jepsen worker 82 - jepsen.util 11512 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:34:13,837] jepsen worker 82 - jepsen.util 11512 :invoke :write [27 47] INFO [2019-12-17 12:34:13,858] jepsen worker 59 - jepsen.util 10499 :info :cas [19 [13 40]] :RequestCanceledException INFO [2019-12-17 12:34:13,859] jepsen worker 81 - jepsen.util 10611 :info :write [27 11] :RequestCanceledException INFO [2019-12-17 12:34:13,862] jepsen worker 59 - jepsen.util 10589 :invoke :write [19 48] INFO [2019-12-17 12:34:13,863] jepsen worker 81 - jepsen.util 10701 :invoke :write [27 45] INFO [2019-12-17 12:34:13,878] jepsen worker 47 - jepsen.util 8417 :ok :read [15 3] INFO [2019-12-17 12:34:13,878] jepsen worker 47 - jepsen.util 8417 :invoke :write [15 20] INFO [2019-12-17 12:34:13,890] jepsen worker 19 - jepsen.util 8209 :ok :read [6 49] INFO [2019-12-17 12:34:13,891] jepsen worker 19 - jepsen.util 8209 :invoke :cas [6 [32 28]] INFO [2019-12-17 12:34:13,891] jepsen worker 23 - jepsen.util 10733 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:34:13,892] jepsen worker 21 - jepsen.util 10281 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:34:13,892] jepsen worker 23 - jepsen.util 10733 :invoke :cas [7 [6 6]] INFO [2019-12-17 12:34:13,892] jepsen worker 21 - jepsen.util 10281 :invoke :cas [7 [38 22]] INFO [2019-12-17 12:34:13,899] jepsen worker 46 - jepsen.util 8236 :fail :cas [15 [27 18]] :ValueNotSwapFrom INFO [2019-12-17 12:34:13,899] jepsen worker 46 - jepsen.util 8236 :invoke :read [15 nil] INFO [2019-12-17 12:34:13,912] jepsen worker 60 - jepsen.util 12030 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:34:13,912] jepsen worker 60 - jepsen.util 12030 :invoke :read [20 nil] INFO [2019-12-17 12:34:13,929] jepsen worker 83 - jepsen.util 11153 :info :write [27 48] :RequestCanceledException INFO [2019-12-17 12:34:13,933] jepsen worker 83 - jepsen.util 11243 :invoke :cas [27 [42 7]] INFO [2019-12-17 12:34:13,933] jepsen worker 72 - jepsen.util 7902 :ok :read [24 45] INFO [2019-12-17 12:34:13,934] jepsen worker 72 - jepsen.util 7902 :invoke :write [24 4] INFO [2019-12-17 12:34:13,939] jepsen worker 45 - jepsen.util 8055 :ok :read [15 3] INFO [2019-12-17 12:34:13,939] jepsen worker 45 - jepsen.util 8055 :invoke :cas [15 [25 6]] INFO [2019-12-17 12:34:13,941] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46490,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:34:13,941] jepsen worker 76 - jepsen.util 10786 :info :cas [25 [38 4]] :RequestCanceledException INFO [2019-12-17 12:34:13,941] jepsen worker 16 - jepsen.util 12166 :info :write [5 7] :RequestCanceledException INFO [2019-12-17 12:34:13,942] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46490,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:34:13,942] jepsen worker 87 - jepsen.util 7377 :info :write [29 6] :RequestCanceledException INFO [2019-12-17 12:34:13,942] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46495,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:34:13,942] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46495,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:34:13,947] jepsen worker 76 - jepsen.util 10876 :invoke :read [25 nil] INFO [2019-12-17 12:34:13,947] jepsen worker 87 - jepsen.util 7467 :invoke :write [29 35] INFO [2019-12-17 12:34:13,948] jepsen worker 16 - jepsen.util 12256 :invoke :read [5 nil] INFO [2019-12-17 12:34:13,955] jepsen worker 69 - jepsen.util 8169 :fail :cas [23 [47 20]] :ValueNotSwapFrom INFO [2019-12-17 12:34:13,955] jepsen worker 69 - jepsen.util 8169 :invoke :cas [23 [0 18]] INFO [2019-12-17 12:34:13,957] jepsen worker 11 - jepsen.util 8021 :fail :cas [3 [16 4]] :ValueNotSwapFrom INFO [2019-12-17 12:34:13,957] jepsen worker 11 - jepsen.util 8021 :invoke :read [3 nil] INFO [2019-12-17 12:34:13,960] jepsen worker 14 - jepsen.util 8204 :info :write [4 3] :RequestCanceledException INFO [2019-12-17 12:34:13,964] jepsen worker 14 - jepsen.util 8294 :invoke :write [4 3] INFO [2019-12-17 12:34:13,978] jepsen worker 45 - jepsen.util 8055 :fail :cas [15 [25 6]] :ValueNotSwapFrom INFO [2019-12-17 12:34:13,978] jepsen worker 45 - jepsen.util 8055 :invoke :write [15 30] INFO [2019-12-17 12:34:13,996] jepsen worker 27 - jepsen.util 7857 :info :write [9 21] :RequestCanceledException INFO [2019-12-17 12:34:13,997] jepsen worker 27 - jepsen.util 7947 :invoke :read [9 nil] INFO [2019-12-17 12:34:14,019] jepsen worker 28 - jepsen.util 6328 :ok :write [9 1] INFO [2019-12-17 12:34:14,019] jepsen worker 28 - jepsen.util 6328 :invoke :write [9 20] INFO [2019-12-17 12:34:14,029] jepsen worker 10 - jepsen.util 8200 :fail :read [3 nil] :RequestTimeoutException INFO [2019-12-17 12:34:14,030] jepsen worker 10 - jepsen.util 8200 :invoke :cas [3 [46 3]] INFO [2019-12-17 12:34:14,057] jepsen worker 51 - jepsen.util 7521 :fail :cas [17 [33 10]] :ValueNotSwapFrom INFO [2019-12-17 12:34:14,057] jepsen worker 51 - jepsen.util 7521 :invoke :write [17 36] INFO [2019-12-17 12:34:14,078] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46517,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:34:14,078] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46517,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:34:14,078] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46519,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:34:14,079] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46519,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:34:14,083] jepsen worker 10 - jepsen.util 8200 :fail :cas [3 [46 3]] :ValueNotSwapFrom INFO [2019-12-17 12:34:14,083] jepsen worker 10 - jepsen.util 8200 :invoke :cas [3 [48 44]] INFO [2019-12-17 12:34:14,090] jepsen worker 78 - jepsen.util 7818 :ok :read [26 47] INFO [2019-12-17 12:34:14,090] jepsen worker 78 - jepsen.util 7818 :invoke :write [26 22] INFO [2019-12-17 12:34:14,110] jepsen worker 10 - jepsen.util 8200 :fail :cas [3 [48 44]] :ValueNotSwapFrom INFO [2019-12-17 12:34:14,111] jepsen worker 10 - jepsen.util 8200 :invoke :cas [3 [21 16]] INFO [2019-12-17 12:34:14,129] jepsen worker 34 - jepsen.util 10744 :info :write [11 28] :RequestCanceledException INFO [2019-12-17 12:34:14,132] jepsen worker 34 - jepsen.util 10834 :invoke :cas [11 [6 18]] INFO [2019-12-17 12:34:14,138] jepsen worker 28 - jepsen.util 6328 :ok :write [9 20] INFO [2019-12-17 12:34:14,138] jepsen worker 28 - jepsen.util 6328 :invoke :write [9 31] INFO [2019-12-17 12:34:14,142] jepsen worker 38 - jepsen.util 8318 :info :write [12 43] :RequestCanceledException INFO [2019-12-17 12:34:14,145] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:34:14,145] jepsen nemesis - couchbase.util Rebalance check count: 360 INFO [2019-12-17 12:34:14,145] jepsen nemesis - couchbase.util Rebalance stuck count: 341 INFO [2019-12-17 12:34:14,145] jepsen worker 10 - jepsen.util 8200 :fail :cas [3 [21 16]] :ValueNotSwapFrom INFO [2019-12-17 12:34:14,146] jepsen worker 10 - jepsen.util 8200 :invoke :write [3 14] INFO [2019-12-17 12:34:14,147] jepsen worker 38 - jepsen.util 8408 :invoke :read [12 nil] INFO [2019-12-17 12:34:14,181] jepsen worker 69 - jepsen.util 8169 :fail :cas [23 [0 18]] :ValueNotSwapFrom INFO [2019-12-17 12:34:14,181] jepsen worker 69 - jepsen.util 8169 :invoke :cas [23 [20 2]] INFO [2019-12-17 12:34:14,195] jepsen worker 73 - jepsen.util 7543 :info :write [24 3] :RequestCanceledException INFO [2019-12-17 12:34:14,199] jepsen worker 73 - jepsen.util 7633 :invoke :cas [24 [48 24]] INFO [2019-12-17 12:34:14,211] jepsen worker 54 - jepsen.util 8964 :fail :cas [18 [28 41]] :ValueNotSwapFrom INFO [2019-12-17 12:34:14,211] jepsen worker 54 - jepsen.util 8964 :invoke :write [18 31] INFO [2019-12-17 12:34:14,226] jepsen worker 28 - jepsen.util 6328 :ok :write [9 31] INFO [2019-12-17 12:34:14,226] jepsen worker 28 - jepsen.util 6328 :invoke :write [9 18] INFO [2019-12-17 12:34:14,228] jepsen worker 35 - jepsen.util 10655 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:34:14,228] jepsen worker 37 - jepsen.util 7597 :info :write [12 36] :RequestCanceledException INFO [2019-12-17 12:34:14,228] jepsen worker 67 - jepsen.util 11047 :info :write [22 26] :RequestCanceledException INFO [2019-12-17 12:34:14,228] jepsen worker 35 - jepsen.util 10655 :invoke :cas [11 [27 11]] INFO [2019-12-17 12:34:14,230] jepsen worker 37 - jepsen.util 7687 :invoke :write [12 23] INFO [2019-12-17 12:34:14,230] jepsen worker 67 - jepsen.util 11137 :invoke :read [22 nil] INFO [2019-12-17 12:34:14,256] jepsen worker 74 - jepsen.util 8354 :ok :write [24 26] INFO [2019-12-17 12:34:14,256] jepsen worker 74 - jepsen.util 8354 :invoke :cas [24 [49 16]] INFO [2019-12-17 12:34:14,267] jepsen worker 31 - jepsen.util 7141 :ok :write [10 36] INFO [2019-12-17 12:34:14,267] jepsen worker 31 - jepsen.util 7141 :invoke :read [10 nil] INFO [2019-12-17 12:34:14,270] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46532,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:34:14,270] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46532,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:34:14,270] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46533,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:34:14,270] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46533,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:34:14,270] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46534,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:34:14,270] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46534,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:34:14,291] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46540,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:34:14,291] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46540,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:34:14,292] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46544,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:34:14,292] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46544,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:34:14,313] jepsen worker 30 - jepsen.util 7950 :info :write [10 24] :RequestCanceledException INFO [2019-12-17 12:34:14,322] jepsen worker 30 - jepsen.util 8040 :invoke :write [10 5] INFO [2019-12-17 12:34:14,336] jepsen worker 27 - jepsen.util 7947 :ok :read [9 31] INFO [2019-12-17 12:34:14,336] jepsen worker 27 - jepsen.util 7947 :invoke :write [9 10] INFO [2019-12-17 12:34:14,369] jepsen worker 19 - jepsen.util 8209 :fail :cas [6 [32 28]] :ValueNotSwapFrom INFO [2019-12-17 12:34:14,369] jepsen worker 19 - jepsen.util 8209 :invoke :read [6 nil] INFO [2019-12-17 12:34:14,409] jepsen worker 46 - jepsen.util 8236 :ok :read [15 3] INFO [2019-12-17 12:34:14,409] jepsen worker 46 - jepsen.util 8236 :invoke :write [15 29] INFO [2019-12-17 12:34:14,410] jepsen worker 12 - jepsen.util 8562 :info :write [4 19] :RequestCanceledException INFO [2019-12-17 12:34:14,410] jepsen worker 32 - jepsen.util 8132 :fail :read [10 nil] :RequestTimeoutException INFO [2019-12-17 12:34:14,410] jepsen worker 32 - jepsen.util 8132 :invoke :write [10 22] INFO [2019-12-17 12:34:14,415] jepsen worker 12 - jepsen.util 8652 :invoke :write [4 20] INFO [2019-12-17 12:34:14,422] jepsen worker 52 - jepsen.util 8062 :fail :cas [17 [2 23]] :ValueNotSwapFrom INFO [2019-12-17 12:34:14,422] jepsen worker 52 - jepsen.util 8062 :invoke :cas [17 [20 49]] INFO [2019-12-17 12:34:14,447] jepsen worker 31 - jepsen.util 7141 :ok :read [10 36] INFO [2019-12-17 12:34:14,447] jepsen worker 31 - jepsen.util 7141 :invoke :write [10 33] INFO [2019-12-17 12:34:14,453] jepsen worker 19 - jepsen.util 8209 :ok :read [6 49] INFO [2019-12-17 12:34:14,453] jepsen worker 19 - jepsen.util 8209 :invoke :cas [6 [13 30]] INFO [2019-12-17 12:34:14,455] jepsen worker 74 - jepsen.util 8354 :fail :cas [24 [49 16]] :ValueNotSwapFrom INFO [2019-12-17 12:34:14,455] jepsen worker 28 - jepsen.util 6328 :ok :write [9 18] INFO [2019-12-17 12:34:14,455] jepsen worker 74 - jepsen.util 8354 :invoke :read [24 nil] INFO [2019-12-17 12:34:14,455] jepsen worker 28 - jepsen.util 6328 :invoke :cas [9 [26 42]] INFO [2019-12-17 12:34:14,477] jepsen worker 53 - jepsen.util 7433 :fail :cas [17 [24 13]] :ValueNotSwapFrom INFO [2019-12-17 12:34:14,478] jepsen worker 53 - jepsen.util 7433 :invoke :write [17 26] INFO [2019-12-17 12:34:14,486] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46556,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:34:14,486] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46556,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:34:14,487] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46561,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:34:14,487] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46561,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:34:14,501] jepsen worker 28 - jepsen.util 6328 :fail :cas [9 [26 42]] :ValueNotSwapFrom INFO [2019-12-17 12:34:14,501] jepsen worker 28 - jepsen.util 6328 :invoke :write [9 26] INFO [2019-12-17 12:34:14,515] jepsen worker 47 - jepsen.util 8417 :ok :write [15 20] INFO [2019-12-17 12:34:14,515] jepsen worker 47 - jepsen.util 8417 :invoke :write [15 41] INFO [2019-12-17 12:34:14,530] jepsen worker 63 - jepsen.util 6813 :info :cas [21 [18 7]] :RequestCanceledException INFO [2019-12-17 12:34:14,531] jepsen worker 4 - jepsen.util 7564 :fail :cas [1 [8 4]] :ValueNotSwapFrom INFO [2019-12-17 12:34:14,531] jepsen worker 4 - jepsen.util 7564 :invoke :cas [1 [41 11]] INFO [2019-12-17 12:34:14,532] jepsen worker 63 - jepsen.util 6903 :invoke :write [21 15] INFO [2019-12-17 12:34:14,543] jepsen worker 33 - jepsen.util 10833 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:34:14,543] jepsen worker 33 - jepsen.util 10833 :invoke :cas [11 [7 19]] INFO [2019-12-17 12:34:14,556] jepsen worker 58 - jepsen.util 11848 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:34:14,556] jepsen worker 58 - jepsen.util 11848 :invoke :read [19 nil] INFO [2019-12-17 12:34:14,559] jepsen worker 9 - jepsen.util 8199 :info :write [3 17] :RequestCanceledException INFO [2019-12-17 12:34:14,562] jepsen worker 11 - jepsen.util 8021 :ok :read [3 45] INFO [2019-12-17 12:34:14,562] jepsen worker 11 - jepsen.util 8021 :invoke :write [3 30] INFO [2019-12-17 12:34:14,562] jepsen worker 9 - jepsen.util 8289 :invoke :write [3 24] INFO [2019-12-17 12:34:14,595] jepsen worker 57 - jepsen.util 10947 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:34:14,595] jepsen worker 57 - jepsen.util 10947 :invoke :cas [19 [35 19]] INFO [2019-12-17 12:34:14,602] jepsen worker 52 - jepsen.util 8062 :fail :cas [17 [20 49]] :ValueNotSwapFrom INFO [2019-12-17 12:34:14,602] jepsen worker 52 - jepsen.util 8062 :invoke :read [17 nil] INFO [2019-12-17 12:34:14,613] jepsen worker 62 - jepsen.util 11222 :info :write [20 28] :RequestCanceledException INFO [2019-12-17 12:34:14,616] jepsen worker 62 - jepsen.util 11312 :invoke :write [20 6] INFO [2019-12-17 12:34:14,620] jepsen worker 73 - jepsen.util 7633 :fail :cas [24 [48 24]] :ValueNotSwapFrom INFO [2019-12-17 12:34:14,620] jepsen worker 73 - jepsen.util 7633 :invoke :cas [24 [41 34]] INFO [2019-12-17 12:34:14,630] jepsen worker 25 - jepsen.util 11545 :info :write [8 45] :RequestCanceledException INFO [2019-12-17 12:34:14,632] jepsen worker 25 - jepsen.util 11635 :invoke :write [8 32] INFO [2019-12-17 12:34:14,673] jepsen worker 73 - jepsen.util 7633 :fail :cas [24 [41 34]] :ValueNotSwapFrom INFO [2019-12-17 12:34:14,673] jepsen worker 73 - jepsen.util 7633 :invoke :write [24 14] INFO [2019-12-17 12:34:14,676] jepsen worker 10 - jepsen.util 8200 :ok :write [3 14] INFO [2019-12-17 12:34:14,676] jepsen worker 10 - jepsen.util 8200 :invoke :cas [3 [46 27]] INFO [2019-12-17 12:34:14,701] jepsen worker 63 - jepsen.util 6903 :ok :write [21 15] INFO [2019-12-17 12:34:14,701] jepsen worker 63 - jepsen.util 6903 :invoke :read [21 nil] INFO [2019-12-17 12:34:14,708] jepsen worker 10 - jepsen.util 8200 :fail :cas [3 [46 27]] :ValueNotSwapFrom INFO [2019-12-17 12:34:14,708] jepsen worker 10 - jepsen.util 8200 :invoke :write [3 34] INFO [2019-12-17 12:34:14,714] jepsen worker 64 - jepsen.util 6634 :info :cas [21 [40 22]] :RequestCanceledException INFO [2019-12-17 12:34:14,716] jepsen worker 64 - jepsen.util 6724 :invoke :read [21 nil] INFO [2019-12-17 12:34:14,720] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46583,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:34:14,720] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46583,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:34:14,720] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46587,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:34:14,720] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46587,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:34:14,721] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46588,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:34:14,721] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46588,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:34:14,726] jepsen worker 85 - jepsen.util 11695 :info :write [28 27] :RequestCanceledException INFO [2019-12-17 12:34:14,728] jepsen worker 85 - jepsen.util 11785 :invoke :read [28 nil] INFO [2019-12-17 12:34:14,743] jepsen worker 55 - jepsen.util 8875 :info :write [18 7] :RequestCanceledException INFO [2019-12-17 12:34:14,743] jepsen worker 0 - jepsen.util 11430 :info :cas [0 [3 3]] :RequestCanceledException INFO [2019-12-17 12:34:14,746] jepsen worker 0 - jepsen.util 11520 :invoke :read [0 nil] INFO [2019-12-17 12:34:14,746] jepsen worker 55 - jepsen.util 8965 :invoke :read [18 nil] INFO [2019-12-17 12:34:14,747] jepsen worker 63 - jepsen.util 6903 :ok :read [21 15] INFO [2019-12-17 12:34:14,747] jepsen worker 63 - jepsen.util 6903 :invoke :write [21 38] INFO [2019-12-17 12:34:14,787] jepsen worker 28 - jepsen.util 6328 :ok :write [9 26] INFO [2019-12-17 12:34:14,787] jepsen worker 28 - jepsen.util 6328 :invoke :cas [9 [6 40]] INFO [2019-12-17 12:34:14,795] jepsen worker 71 - jepsen.util 8621 :info :write [23 28] :RequestCanceledException INFO [2019-12-17 12:34:14,797] jepsen worker 71 - jepsen.util 8711 :invoke :read [23 nil] INFO [2019-12-17 12:34:14,827] jepsen worker 3 - jepsen.util 7923 :info :write [1 46] :RequestCanceledException INFO [2019-12-17 12:34:14,829] jepsen worker 3 - jepsen.util 8013 :invoke :cas [1 [47 32]] INFO [2019-12-17 12:34:14,844] jepsen worker 36 - jepsen.util 8406 :fail :cas [12 [47 23]] :ValueNotSwapFrom INFO [2019-12-17 12:34:14,844] jepsen worker 36 - jepsen.util 8406 :invoke :cas [12 [15 18]] INFO [2019-12-17 12:34:14,845] jepsen worker 52 - jepsen.util 8062 :ok :read [17 0] INFO [2019-12-17 12:34:14,846] jepsen worker 52 - jepsen.util 8062 :invoke :write [17 2] INFO [2019-12-17 12:34:14,858] jepsen worker 20 - jepsen.util 7580 :info :cas [6 [28 21]] :RequestCanceledException INFO [2019-12-17 12:34:14,862] jepsen worker 20 - jepsen.util 7670 :invoke :write [6 30] INFO [2019-12-17 12:34:14,863] jepsen worker 28 - jepsen.util 6328 :fail :cas [9 [6 40]] :ValueNotSwapFrom INFO [2019-12-17 12:34:14,863] jepsen worker 28 - jepsen.util 6328 :invoke :cas [9 [12 19]] INFO [2019-12-17 12:34:14,870] jepsen worker 69 - jepsen.util 8169 :fail :cas [23 [20 2]] :ValueNotSwapFrom INFO [2019-12-17 12:34:14,870] jepsen worker 69 - jepsen.util 8169 :invoke :write [23 48] INFO [2019-12-17 12:34:14,895] jepsen worker 28 - jepsen.util 6328 :fail :cas [9 [12 19]] :ValueNotSwapFrom INFO [2019-12-17 12:34:14,895] jepsen worker 28 - jepsen.util 6328 :invoke :read [9 nil] INFO [2019-12-17 12:34:14,913] jepsen worker 22 - jepsen.util 10912 :info :write [7 24] :RequestCanceledException INFO [2019-12-17 12:34:14,916] jepsen worker 22 - jepsen.util 11002 :invoke :cas [7 [21 19]] INFO [2019-12-17 12:34:14,920] jepsen worker 28 - jepsen.util 6328 :ok :read [9 26] INFO [2019-12-17 12:34:14,920] jepsen worker 28 - jepsen.util 6328 :invoke :write [9 22] INFO [2019-12-17 12:34:14,923] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46606,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:34:14,924] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46606,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:34:14,924] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46608,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:34:14,924] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46608,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:34:14,929] jepsen worker 48 - jepsen.util 10038 :info :cas [16 [46 29]] :RequestCanceledException INFO [2019-12-17 12:34:14,932] jepsen worker 48 - jepsen.util 10128 :invoke :write [16 8] INFO [2019-12-17 12:34:14,942] jepsen worker 61 - jepsen.util 11761 :info :cas [20 [30 21]] :RequestCanceledException INFO [2019-12-17 12:34:14,942] jepsen worker 26 - jepsen.util 11456 :info :write [8 7] :RequestCanceledException INFO [2019-12-17 12:34:14,944] jepsen worker 26 - jepsen.util 11546 :invoke :read [8 nil] INFO [2019-12-17 12:34:14,944] jepsen worker 61 - jepsen.util 11851 :invoke :cas [20 [30 45]] INFO [2019-12-17 12:34:14,946] jepsen worker 63 - jepsen.util 6903 :ok :write [21 38] INFO [2019-12-17 12:34:14,946] jepsen worker 63 - jepsen.util 6903 :invoke :cas [21 [33 22]] INFO [2019-12-17 12:34:14,964] jepsen worker 30 - jepsen.util 8040 :ok :write [10 5] INFO [2019-12-17 12:34:14,965] jepsen worker 30 - jepsen.util 8040 :invoke :cas [10 [37 48]] INFO [2019-12-17 12:34:14,977] jepsen worker 63 - jepsen.util 6903 :fail :cas [21 [33 22]] :ValueNotSwapFrom INFO [2019-12-17 12:34:14,978] jepsen worker 63 - jepsen.util 6903 :invoke :cas [21 [5 32]] INFO [2019-12-17 12:34:14,990] jepsen worker 28 - jepsen.util 6328 :ok :write [9 22] INFO [2019-12-17 12:34:14,991] jepsen worker 28 - jepsen.util 6328 :invoke :cas [9 [2 7]] INFO [2019-12-17 12:34:15,012] jepsen worker 6 - jepsen.util 11526 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:34:15,012] jepsen worker 6 - jepsen.util 11526 :invoke :write [2 16] INFO [2019-12-17 12:34:15,021] jepsen worker 84 - jepsen.util 10524 :info :cas [28 [8 19]] :RequestCanceledException INFO [2019-12-17 12:34:15,021] jepsen worker 70 - jepsen.util 8350 :info :write [23 35] :RequestCanceledException INFO [2019-12-17 12:34:15,025] jepsen worker 84 - jepsen.util 10614 :invoke :cas [28 [32 32]] INFO [2019-12-17 12:34:15,025] jepsen worker 70 - jepsen.util 8440 :invoke :write [23 31] INFO [2019-12-17 12:34:15,027] jepsen worker 40 - jepsen.util 11380 :info :cas [13 [36 13]] :RequestCanceledException INFO [2019-12-17 12:34:15,032] jepsen worker 40 - jepsen.util 11470 :invoke :read [13 nil] INFO [2019-12-17 12:34:15,043] jepsen worker 44 - jepsen.util 11204 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:34:15,043] jepsen worker 44 - jepsen.util 11204 :invoke :read [14 nil] INFO [2019-12-17 12:34:15,044] jepsen worker 38 - jepsen.util 8408 :ok :read [12 28] INFO [2019-12-17 12:34:15,044] jepsen worker 38 - jepsen.util 8408 :invoke :write [12 30] INFO [2019-12-17 12:34:15,046] jepsen worker 28 - jepsen.util 6328 :fail :cas [9 [2 7]] :ValueNotSwapFrom INFO [2019-12-17 12:34:15,046] jepsen worker 28 - jepsen.util 6328 :invoke :write [9 39] INFO [2019-12-17 12:34:15,055] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46622,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:34:15,056] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46622,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:34:15,056] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46626,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:34:15,056] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46626,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:34:15,056] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46627,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:34:15,056] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46627,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:34:15,063] jepsen worker 27 - jepsen.util 7947 :ok :write [9 10] INFO [2019-12-17 12:34:15,063] jepsen worker 27 - jepsen.util 7947 :invoke :write [9 31] INFO [2019-12-17 12:34:15,082] jepsen worker 63 - jepsen.util 6903 :fail :cas [21 [5 32]] :ValueNotSwapFrom INFO [2019-12-17 12:34:15,083] jepsen worker 63 - jepsen.util 6903 :invoke :read [21 nil] INFO [2019-12-17 12:34:15,096] jepsen worker 66 - jepsen.util 10686 :info :write [22 2] :RequestCanceledException INFO [2019-12-17 12:34:15,096] jepsen worker 79 - jepsen.util 8269 :info :write [26 16] :RequestCanceledException INFO [2019-12-17 12:34:15,096] jepsen worker 89 - jepsen.util 7919 :info :cas [29 [36 7]] :RequestCanceledException INFO [2019-12-17 12:34:15,099] jepsen worker 79 - jepsen.util 8359 :invoke :cas [26 [0 3]] INFO [2019-12-17 12:34:15,099] jepsen worker 89 - jepsen.util 8009 :invoke :write [29 18] INFO [2019-12-17 12:34:15,099] jepsen worker 66 - jepsen.util 10776 :invoke :cas [22 [25 39]] INFO [2019-12-17 12:34:15,110] jepsen worker 36 - jepsen.util 8406 :fail :cas [12 [15 18]] :ValueNotSwapFrom INFO [2019-12-17 12:34:15,110] jepsen worker 36 - jepsen.util 8406 :invoke :cas [12 [41 11]] INFO [2019-12-17 12:34:15,115] jepsen worker 4 - jepsen.util 7564 :fail :cas [1 [41 11]] :ValueNotSwapFrom INFO [2019-12-17 12:34:15,116] jepsen worker 4 - jepsen.util 7564 :invoke :read [1 nil] INFO [2019-12-17 12:34:15,129] jepsen worker 28 - jepsen.util 6328 :ok :write [9 39] INFO [2019-12-17 12:34:15,130] jepsen worker 28 - jepsen.util 6328 :invoke :write [9 47] INFO [2019-12-17 12:34:15,159] jepsen worker 15 - jepsen.util 10095 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:34:15,159] jepsen worker 15 - jepsen.util 10095 :invoke :write [5 16] INFO [2019-12-17 12:34:15,165] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:34:15,165] jepsen nemesis - couchbase.util Rebalance check count: 361 INFO [2019-12-17 12:34:15,166] jepsen nemesis - couchbase.util Rebalance stuck count: 342 INFO [2019-12-17 12:34:15,175] jepsen worker 63 - jepsen.util 6903 :ok :read [21 38] INFO [2019-12-17 12:34:15,175] jepsen worker 63 - jepsen.util 6903 :invoke :cas [21 [15 18]] INFO [2019-12-17 12:34:15,195] jepsen worker 29 - jepsen.util 6599 :info :cas [9 [47 22]] :RequestCanceledException INFO [2019-12-17 12:34:15,207] jepsen worker 63 - jepsen.util 6903 :fail :cas [21 [15 18]] :ValueNotSwapFrom INFO [2019-12-17 12:34:15,207] jepsen worker 63 - jepsen.util 6903 :invoke :read [21 nil] INFO [2019-12-17 12:34:15,208] jepsen worker 29 - jepsen.util 6689 :invoke :cas [9 [2 29]] INFO [2019-12-17 12:34:15,220] jepsen worker 28 - jepsen.util 6328 :ok :write [9 47] INFO [2019-12-17 12:34:15,221] jepsen worker 28 - jepsen.util 6328 :invoke :cas [9 [20 4]] INFO [2019-12-17 12:34:15,237] jepsen worker 63 - jepsen.util 6903 :ok :read [21 38] INFO [2019-12-17 12:34:15,237] jepsen worker 63 - jepsen.util 6903 :invoke :write [21 27] INFO [2019-12-17 12:34:15,242] jepsen worker 24 - jepsen.util 9924 :info :write [8 18] :RequestCanceledException INFO [2019-12-17 12:34:15,246] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46645,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:34:15,247] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46645,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:34:15,247] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46646,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:34:15,247] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46646,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:34:15,247] jepsen worker 24 - jepsen.util 10014 :invoke :read [8 nil] INFO [2019-12-17 12:34:15,253] jepsen worker 42 - jepsen.util 11832 :info :cas [14 [42 30]] :RequestCanceledException INFO [2019-12-17 12:34:15,256] jepsen worker 42 - jepsen.util 11922 :invoke :write [14 48] INFO [2019-12-17 12:34:15,272] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46656,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:34:15,272] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46656,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:34:15,272] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46658,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:34:15,272] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46658,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:34:15,278] jepsen worker 29 - jepsen.util 6689 :fail :cas [9 [2 29]] :ValueNotSwapFrom INFO [2019-12-17 12:34:15,278] jepsen worker 29 - jepsen.util 6689 :invoke :write [9 7] INFO [2019-12-17 12:34:15,278] jepsen worker 71 - jepsen.util 8711 :ok :read [23 29] INFO [2019-12-17 12:34:15,279] jepsen worker 71 - jepsen.util 8711 :invoke :cas [23 [11 25]] INFO [2019-12-17 12:34:15,298] jepsen worker 28 - jepsen.util 6328 :fail :cas [9 [20 4]] :ValueNotSwapFrom INFO [2019-12-17 12:34:15,298] jepsen worker 28 - jepsen.util 6328 :invoke :cas [9 [46 41]] INFO [2019-12-17 12:34:15,326] jepsen worker 63 - jepsen.util 6903 :ok :write [21 27] INFO [2019-12-17 12:34:15,327] jepsen worker 63 - jepsen.util 6903 :invoke :cas [21 [3 37]] INFO [2019-12-17 12:34:15,340] jepsen worker 79 - jepsen.util 8359 :fail :cas [26 [0 3]] :ValueNotSwapFrom INFO [2019-12-17 12:34:15,341] jepsen worker 79 - jepsen.util 8359 :invoke :cas [26 [27 9]] INFO [2019-12-17 12:34:15,355] jepsen worker 63 - jepsen.util 6903 :fail :cas [21 [3 37]] :ValueNotSwapFrom INFO [2019-12-17 12:34:15,355] jepsen worker 63 - jepsen.util 6903 :invoke :cas [21 [43 30]] INFO [2019-12-17 12:34:15,392] jepsen worker 63 - jepsen.util 6903 :fail :cas [21 [43 30]] :ValueNotSwapFrom INFO [2019-12-17 12:34:15,392] jepsen worker 63 - jepsen.util 6903 :invoke :cas [21 [41 46]] INFO [2019-12-17 12:34:15,394] jepsen worker 8 - jepsen.util 10898 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:34:15,394] jepsen worker 8 - jepsen.util 10898 :invoke :write [2 18] INFO [2019-12-17 12:34:15,409] jepsen worker 29 - jepsen.util 6689 :ok :write [9 7] INFO [2019-12-17 12:34:15,409] jepsen worker 29 - jepsen.util 6689 :invoke :cas [9 [48 29]] INFO [2019-12-17 12:34:15,413] jepsen worker 7 - jepsen.util 11257 :info :cas [2 [4 26]] :RequestCanceledException INFO [2019-12-17 12:34:15,415] jepsen worker 7 - jepsen.util 11347 :invoke :read [2 nil] INFO [2019-12-17 12:34:15,427] jepsen worker 65 - jepsen.util 6725 :fail :read [21 nil] :RequestTimeoutException INFO [2019-12-17 12:34:15,427] jepsen worker 13 - jepsen.util 7483 :info :write [4 12] :RequestCanceledException INFO [2019-12-17 12:34:15,427] jepsen worker 18 - jepsen.util 8208 :info :cas [6 [13 39]] :RequestCanceledException INFO [2019-12-17 12:34:15,427] jepsen worker 65 - jepsen.util 6725 :invoke :write [21 24] INFO [2019-12-17 12:34:15,427] jepsen worker 5 - jepsen.util 7565 :info :write [1 18] :RequestCanceledException INFO [2019-12-17 12:34:15,427] jepsen worker 17 - jepsen.util 10907 :info :write [5 21] :RequestCanceledException INFO [2019-12-17 12:34:15,430] jepsen worker 18 - jepsen.util 8298 :invoke :cas [6 [33 10]] INFO [2019-12-17 12:34:15,431] jepsen worker 13 - jepsen.util 7573 :invoke :write [4 43] INFO [2019-12-17 12:34:15,431] jepsen worker 17 - jepsen.util 10997 :invoke :cas [5 [22 39]] INFO [2019-12-17 12:34:15,431] jepsen worker 5 - jepsen.util 7655 :invoke :cas [1 [6 48]] INFO [2019-12-17 12:34:15,434] jepsen worker 63 - jepsen.util 6903 :fail :cas [21 [41 46]] :ValueNotSwapFrom INFO [2019-12-17 12:34:15,435] jepsen worker 63 - jepsen.util 6903 :invoke :write [21 40] INFO [2019-12-17 12:34:15,443] jepsen worker 43 - jepsen.util 10483 :info :cas [14 [16 16]] :RequestCanceledException INFO [2019-12-17 12:34:15,448] jepsen worker 43 - jepsen.util 10573 :invoke :write [14 44] INFO [2019-12-17 12:34:15,451] jepsen worker 36 - jepsen.util 8406 :fail :cas [12 [41 11]] :ValueNotSwapFrom INFO [2019-12-17 12:34:15,452] jepsen worker 36 - jepsen.util 8406 :invoke :write [12 29] INFO [2019-12-17 12:34:15,460] jepsen worker 1 - jepsen.util 11161 :info :cas [0 [30 20]] :RequestCanceledException INFO [2019-12-17 12:34:15,463] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46676,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:34:15,463] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46676,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:34:15,465] jepsen worker 1 - jepsen.util 11251 :invoke :write [0 10] INFO [2019-12-17 12:34:15,471] jepsen worker 29 - jepsen.util 6689 :fail :cas [9 [48 29]] :ValueNotSwapFrom INFO [2019-12-17 12:34:15,471] jepsen worker 29 - jepsen.util 6689 :invoke :write [9 33] INFO [2019-12-17 12:34:15,488] jepsen worker 79 - jepsen.util 8359 :fail :cas [26 [27 9]] :ValueNotSwapFrom INFO [2019-12-17 12:34:15,488] jepsen worker 79 - jepsen.util 8359 :invoke :read [26 nil] INFO [2019-12-17 12:34:15,495] jepsen worker 77 - jepsen.util 11237 :info :write [25 3] :RequestCanceledException INFO [2019-12-17 12:34:15,496] jepsen worker 77 - jepsen.util 11327 :invoke :cas [25 [35 47]] INFO [2019-12-17 12:34:15,513] jepsen worker 2 - jepsen.util 10622 :info :write [0 19] :RequestCanceledException INFO [2019-12-17 12:34:15,514] jepsen worker 2 - jepsen.util 10712 :invoke :write [0 0] INFO [2019-12-17 12:34:15,529] jepsen worker 75 - jepsen.util 10425 :info :write [25 16] :RequestCanceledException INFO [2019-12-17 12:34:15,529] jepsen worker 49 - jepsen.util 10399 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:34:15,529] jepsen worker 49 - jepsen.util 10399 :invoke :write [16 21] INFO [2019-12-17 12:34:15,531] jepsen worker 75 - jepsen.util 10515 :invoke :read [25 nil] INFO [2019-12-17 12:34:15,543] jepsen worker 88 - jepsen.util 8098 :info :write [29 42] :RequestCanceledException INFO [2019-12-17 12:34:15,544] jepsen worker 63 - jepsen.util 6903 :ok :write [21 40] INFO [2019-12-17 12:34:15,544] jepsen worker 63 - jepsen.util 6903 :invoke :read [21 nil] INFO [2019-12-17 12:34:15,545] jepsen worker 88 - jepsen.util 8188 :invoke :cas [29 [42 41]] INFO [2019-12-17 12:34:15,563] jepsen worker 29 - jepsen.util 6689 :ok :write [9 33] INFO [2019-12-17 12:34:15,571] jepsen worker 29 - jepsen.util 6689 :invoke :cas [9 [22 49]] INFO [2019-12-17 12:34:15,571] jepsen worker 63 - jepsen.util 6903 :ok :read [21 40] INFO [2019-12-17 12:34:15,574] jepsen worker 63 - jepsen.util 6903 :invoke :read [21 nil] INFO [2019-12-17 12:34:15,584] jepsen worker 71 - jepsen.util 8711 :fail :cas [23 [11 25]] :ValueNotSwapFrom INFO [2019-12-17 12:34:15,585] jepsen worker 71 - jepsen.util 8711 :invoke :read [23 nil] INFO [2019-12-17 12:34:15,599] jepsen worker 55 - jepsen.util 8965 :ok :read [18 48] INFO [2019-12-17 12:34:15,599] jepsen worker 55 - jepsen.util 8965 :invoke :cas [18 [10 34]] INFO [2019-12-17 12:34:15,634] jepsen worker 63 - jepsen.util 6903 :ok :read [21 40] INFO [2019-12-17 12:34:15,634] jepsen worker 63 - jepsen.util 6903 :invoke :write [21 15] INFO [2019-12-17 12:34:15,643] jepsen worker 29 - jepsen.util 6689 :fail :cas [9 [22 49]] :ValueNotSwapFrom INFO [2019-12-17 12:34:15,644] jepsen worker 29 - jepsen.util 6689 :invoke :cas [9 [35 4]] INFO [2019-12-17 12:34:15,670] jepsen worker 29 - jepsen.util 6689 :fail :cas [9 [35 4]] :ValueNotSwapFrom INFO [2019-12-17 12:34:15,670] jepsen worker 29 - jepsen.util 6689 :invoke :read [9 nil] INFO [2019-12-17 12:34:15,697] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46705,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:34:15,697] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46705,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:34:15,702] jepsen worker 79 - jepsen.util 8359 :ok :read [26 47] INFO [2019-12-17 12:34:15,703] jepsen worker 79 - jepsen.util 8359 :invoke :write [26 26] INFO [2019-12-17 12:34:15,704] jepsen worker 29 - jepsen.util 6689 :ok :read [9 33] INFO [2019-12-17 12:34:15,704] jepsen worker 29 - jepsen.util 6689 :invoke :write [9 6] INFO [2019-12-17 12:34:15,710] jepsen worker 63 - jepsen.util 6903 :ok :write [21 15] INFO [2019-12-17 12:34:15,710] jepsen worker 63 - jepsen.util 6903 :invoke :read [21 nil] INFO [2019-12-17 12:34:15,729] jepsen worker 56 - jepsen.util 9056 :info :write [18 6] :RequestCanceledException INFO [2019-12-17 12:34:15,731] jepsen worker 56 - jepsen.util 9146 :invoke :cas [18 [32 14]] INFO [2019-12-17 12:34:15,754] jepsen worker 63 - jepsen.util 6903 :ok :read [21 15] INFO [2019-12-17 12:34:15,754] jepsen worker 63 - jepsen.util 6903 :invoke :cas [21 [33 49]] INFO [2019-12-17 12:34:15,781] jepsen worker 63 - jepsen.util 6903 :fail :cas [21 [33 49]] :ValueNotSwapFrom INFO [2019-12-17 12:34:15,781] jepsen worker 63 - jepsen.util 6903 :invoke :write [21 14] INFO [2019-12-17 12:34:15,804] jepsen worker 29 - jepsen.util 6689 :ok :write [9 6] INFO [2019-12-17 12:34:15,804] jepsen worker 29 - jepsen.util 6689 :invoke :read [9 nil] INFO [2019-12-17 12:34:15,815] jepsen worker 3 - jepsen.util 8013 :fail :cas [1 [47 32]] :ValueNotSwapFrom INFO [2019-12-17 12:34:15,815] jepsen worker 3 - jepsen.util 8013 :invoke :cas [1 [45 16]] INFO [2019-12-17 12:34:15,821] jepsen worker 41 - jepsen.util 11651 :info :write [13 22] :RequestCanceledException INFO [2019-12-17 12:34:15,824] jepsen worker 41 - jepsen.util 11741 :invoke :read [13 nil] INFO [2019-12-17 12:34:15,842] jepsen worker 29 - jepsen.util 6689 :ok :read [9 6] INFO [2019-12-17 12:34:15,842] jepsen worker 29 - jepsen.util 6689 :invoke :cas [9 [25 27]] INFO [2019-12-17 12:34:15,858] jepsen worker 80 - jepsen.util 7910 :info :cas [26 [47 0]] :RequestCanceledException INFO [2019-12-17 12:34:15,859] jepsen worker 80 - jepsen.util 8000 :invoke :write [26 22] INFO [2019-12-17 12:34:15,890] jepsen worker 29 - jepsen.util 6689 :fail :cas [9 [25 27]] :ValueNotSwapFrom INFO [2019-12-17 12:34:15,890] jepsen worker 29 - jepsen.util 6689 :invoke :read [9 nil] INFO [2019-12-17 12:34:15,890] jepsen worker 63 - jepsen.util 6903 :ok :write [21 14] INFO [2019-12-17 12:34:15,891] jepsen worker 63 - jepsen.util 6903 :invoke :write [21 23] INFO [2019-12-17 12:34:15,894] jepsen worker 68 - jepsen.util 10418 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:34:15,894] jepsen worker 68 - jepsen.util 10418 :invoke :read [22 nil] INFO [2019-12-17 12:34:15,899] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46720,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:34:15,900] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46720,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:34:15,900] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46723,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:34:15,900] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46723,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:34:15,900] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46724,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:34:15,900] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46724,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:34:15,914] jepsen worker 86 - jepsen.util 11606 :info :write [28 32] :RequestCanceledException INFO [2019-12-17 12:34:15,916] jepsen worker 86 - jepsen.util 11696 :invoke :cas [28 [18 42]] INFO [2019-12-17 12:34:15,922] jepsen worker 39 - jepsen.util 10569 :info :write [13 35] :RequestCanceledException INFO [2019-12-17 12:34:15,922] jepsen worker 50 - jepsen.util 11750 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:34:15,922] jepsen worker 50 - jepsen.util 11750 :invoke :cas [16 [49 17]] INFO [2019-12-17 12:34:15,924] jepsen worker 39 - jepsen.util 10659 :invoke :write [13 36] INFO [2019-12-17 12:34:15,942] jepsen worker 59 - jepsen.util 10589 :info :write [19 48] :RequestCanceledException INFO [2019-12-17 12:34:15,942] jepsen worker 82 - jepsen.util 11512 :info :write [27 47] :RequestCanceledException INFO [2019-12-17 12:34:15,945] jepsen worker 59 - jepsen.util 10679 :invoke :cas [19 [4 2]] INFO [2019-12-17 12:34:15,946] jepsen worker 29 - jepsen.util 6689 :ok :read [9 6] INFO [2019-12-17 12:34:15,946] jepsen worker 29 - jepsen.util 6689 :invoke :read [9 nil] INFO [2019-12-17 12:34:15,949] jepsen worker 82 - jepsen.util 11602 :invoke :write [27 1] INFO [2019-12-17 12:34:15,960] jepsen worker 81 - jepsen.util 10701 :info :write [27 45] :RequestCanceledException INFO [2019-12-17 12:34:15,962] jepsen worker 81 - jepsen.util 10791 :invoke :write [27 10] INFO [2019-12-17 12:34:15,976] jepsen worker 29 - jepsen.util 6689 :ok :read [9 6] INFO [2019-12-17 12:34:15,977] jepsen worker 29 - jepsen.util 6689 :invoke :read [9 nil] INFO [2019-12-17 12:34:15,991] jepsen worker 63 - jepsen.util 6903 :ok :write [21 23] INFO [2019-12-17 12:34:15,992] jepsen worker 63 - jepsen.util 6903 :invoke :read [21 nil] INFO [2019-12-17 12:34:15,995] jepsen worker 21 - jepsen.util 10281 :info :cas [7 [38 22]] :RequestCanceledException INFO [2019-12-17 12:34:15,995] jepsen worker 23 - jepsen.util 10733 :info :cas [7 [6 6]] :RequestCanceledException INFO [2019-12-17 12:34:15,995] jepsen worker 76 - jepsen.util 10876 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:34:15,995] jepsen worker 45 - jepsen.util 8055 :info :write [15 30] :RequestCanceledException INFO [2019-12-17 12:34:15,995] jepsen worker 76 - jepsen.util 10876 :invoke :cas [25 [36 45]] INFO [2019-12-17 12:34:15,997] jepsen worker 21 - jepsen.util 10371 :invoke :cas [7 [11 18]] INFO [2019-12-17 12:34:15,997] jepsen worker 23 - jepsen.util 10823 :invoke :write [7 47] INFO [2019-12-17 12:34:15,997] jepsen worker 45 - jepsen.util 8145 :invoke :cas [15 [25 30]] INFO [2019-12-17 12:34:16,008] jepsen worker 29 - jepsen.util 6689 :ok :read [9 6] INFO [2019-12-17 12:34:16,008] jepsen worker 29 - jepsen.util 6689 :invoke :write [9 14] INFO [2019-12-17 12:34:16,013] jepsen worker 60 - jepsen.util 12030 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:34:16,013] jepsen worker 87 - jepsen.util 7467 :info :write [29 35] :RequestCanceledException INFO [2019-12-17 12:34:16,013] jepsen worker 60 - jepsen.util 12030 :invoke :read [20 nil] INFO [2019-12-17 12:34:16,016] jepsen worker 87 - jepsen.util 7557 :invoke :read [29 nil] INFO [2019-12-17 12:34:16,021] jepsen worker 83 - jepsen.util 11243 :info :cas [27 [42 7]] :RequestCanceledException INFO [2019-12-17 12:34:16,021] jepsen worker 16 - jepsen.util 12256 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:34:16,021] jepsen worker 72 - jepsen.util 7902 :info :write [24 4] :RequestCanceledException INFO [2019-12-17 12:34:16,021] jepsen worker 16 - jepsen.util 12256 :invoke :write [5 24] INFO [2019-12-17 12:34:16,023] jepsen worker 72 - jepsen.util 7992 :invoke :cas [24 [8 41]] INFO [2019-12-17 12:34:16,025] jepsen worker 83 - jepsen.util 11333 :invoke :write [27 10] INFO [2019-12-17 12:34:16,032] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46739,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:34:16,032] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46739,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:34:16,032] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46743,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:34:16,032] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46743,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:34:16,043] jepsen worker 14 - jepsen.util 8294 :info :write [4 3] :RequestCanceledException INFO [2019-12-17 12:34:16,043] jepsen worker 45 - jepsen.util 8145 :fail :cas [15 [25 30]] :ValueNotSwapFrom INFO [2019-12-17 12:34:16,044] jepsen worker 45 - jepsen.util 8145 :invoke :write [15 36] INFO [2019-12-17 12:34:16,046] jepsen worker 14 - jepsen.util 8384 :invoke :cas [4 [19 3]] INFO [2019-12-17 12:34:16,049] jepsen worker 63 - jepsen.util 6903 :ok :read [21 23] INFO [2019-12-17 12:34:16,049] jepsen worker 63 - jepsen.util 6903 :invoke :cas [21 [30 36]] INFO [2019-12-17 12:34:16,082] jepsen worker 63 - jepsen.util 6903 :fail :cas [21 [30 36]] :ValueNotSwapFrom INFO [2019-12-17 12:34:16,082] jepsen worker 63 - jepsen.util 6903 :invoke :write [21 44] INFO [2019-12-17 12:34:16,090] jepsen worker 29 - jepsen.util 6689 :ok :write [9 14] INFO [2019-12-17 12:34:16,090] jepsen worker 29 - jepsen.util 6689 :invoke :read [9 nil] INFO [2019-12-17 12:34:16,096] jepsen worker 51 - jepsen.util 7521 :info :write [17 36] :RequestCanceledException INFO [2019-12-17 12:34:16,098] jepsen worker 51 - jepsen.util 7611 :invoke :read [17 nil] INFO [2019-12-17 12:34:16,114] jepsen worker 78 - jepsen.util 7818 :info :write [26 22] :RequestCanceledException INFO [2019-12-17 12:34:16,116] jepsen worker 78 - jepsen.util 7908 :invoke :read [26 nil] INFO [2019-12-17 12:34:16,148] jepsen worker 63 - jepsen.util 6903 :ok :write [21 44] INFO [2019-12-17 12:34:16,148] jepsen worker 63 - jepsen.util 6903 :invoke :read [21 nil] INFO [2019-12-17 12:34:16,187] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:34:16,187] jepsen nemesis - couchbase.util Rebalance check count: 362 INFO [2019-12-17 12:34:16,187] jepsen nemesis - couchbase.util Rebalance stuck count: 343 INFO [2019-12-17 12:34:16,192] jepsen worker 63 - jepsen.util 6903 :ok :read [21 44] INFO [2019-12-17 12:34:16,192] jepsen worker 63 - jepsen.util 6903 :invoke :write [21 4] INFO [2019-12-17 12:34:16,199] jepsen worker 3 - jepsen.util 8013 :fail :cas [1 [45 16]] :ValueNotSwapFrom INFO [2019-12-17 12:34:16,199] jepsen worker 3 - jepsen.util 8013 :invoke :read [1 nil] INFO [2019-12-17 12:34:16,221] jepsen worker 71 - jepsen.util 8711 :ok :read [23 29] INFO [2019-12-17 12:34:16,222] jepsen worker 71 - jepsen.util 8711 :invoke :cas [23 [39 2]] INFO [2019-12-17 12:34:16,229] jepsen worker 34 - jepsen.util 10834 :info :cas [11 [6 18]] :RequestCanceledException INFO [2019-12-17 12:34:16,230] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46762,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:34:16,230] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46762,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:34:16,230] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46764,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:34:16,231] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46764,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:34:16,231] jepsen worker 34 - jepsen.util 10924 :invoke :write [11 25] INFO [2019-12-17 12:34:16,243] jepsen worker 54 - jepsen.util 8964 :info :write [18 31] :RequestCanceledException INFO [2019-12-17 12:34:16,247] jepsen worker 56 - jepsen.util 9146 :fail :cas [18 [32 14]] :ValueNotSwapFrom INFO [2019-12-17 12:34:16,247] jepsen worker 56 - jepsen.util 9146 :invoke :cas [18 [2 46]] INFO [2019-12-17 12:34:16,250] jepsen worker 54 - jepsen.util 9054 :invoke :read [18 nil] INFO [2019-12-17 12:34:16,251] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46771,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:34:16,251] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46771,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:34:16,251] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46774,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:34:16,251] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46774,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:34:16,251] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46775,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:34:16,251] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46775,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:34:16,257] jepsen worker 63 - jepsen.util 6903 :ok :write [21 4] INFO [2019-12-17 12:34:16,258] jepsen worker 63 - jepsen.util 6903 :invoke :cas [21 [2 13]] INFO [2019-12-17 12:34:16,281] jepsen worker 63 - jepsen.util 6903 :fail :cas [21 [2 13]] :ValueNotSwapFrom INFO [2019-12-17 12:34:16,282] jepsen worker 63 - jepsen.util 6903 :invoke :read [21 nil] INFO [2019-12-17 12:34:16,311] jepsen worker 63 - jepsen.util 6903 :ok :read [21 4] INFO [2019-12-17 12:34:16,311] jepsen worker 63 - jepsen.util 6903 :invoke :cas [21 [48 4]] INFO [2019-12-17 12:34:16,320] jepsen worker 56 - jepsen.util 9146 :fail :cas [18 [2 46]] :ValueNotSwapFrom INFO [2019-12-17 12:34:16,320] jepsen worker 56 - jepsen.util 9146 :invoke :cas [18 [18 25]] INFO [2019-12-17 12:34:16,329] jepsen worker 78 - jepsen.util 7908 :ok :read [26 47] INFO [2019-12-17 12:34:16,329] jepsen worker 46 - jepsen.util 8236 :info :write [15 29] :SyncWriteAmbiguous INFO [2019-12-17 12:34:16,329] jepsen worker 35 - jepsen.util 10655 :info :cas [11 [27 11]] :RequestCanceledException INFO [2019-12-17 12:34:16,329] jepsen worker 67 - jepsen.util 11137 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:34:16,329] jepsen worker 78 - jepsen.util 7908 :invoke :cas [26 [10 33]] INFO [2019-12-17 12:34:16,329] jepsen worker 67 - jepsen.util 11137 :invoke :cas [22 [30 37]] INFO [2019-12-17 12:34:16,331] jepsen worker 46 - jepsen.util 8326 :invoke :write [15 19] INFO [2019-12-17 12:34:16,331] jepsen worker 35 - jepsen.util 10745 :invoke :cas [11 [7 38]] INFO [2019-12-17 12:34:16,344] jepsen worker 37 - jepsen.util 7687 :info :write [12 23] :RequestCanceledException INFO [2019-12-17 12:34:16,347] jepsen worker 37 - jepsen.util 7777 :invoke :write [12 10] INFO [2019-12-17 12:34:16,370] jepsen worker 63 - jepsen.util 6903 :fail :cas [21 [48 4]] :ValueNotSwapFrom INFO [2019-12-17 12:34:16,370] jepsen worker 63 - jepsen.util 6903 :invoke :read [21 nil] INFO [2019-12-17 12:34:16,371] jepsen worker 19 - jepsen.util 8209 :fail :cas [6 [13 30]] :ValueNotSwapFrom INFO [2019-12-17 12:34:16,371] jepsen worker 19 - jepsen.util 8209 :invoke :write [6 14] INFO [2019-12-17 12:34:16,373] jepsen worker 72 - jepsen.util 7992 :fail :cas [24 [8 41]] :ValueNotSwapFrom INFO [2019-12-17 12:34:16,373] jepsen worker 72 - jepsen.util 7992 :invoke :write [24 42] INFO [2019-12-17 12:34:16,391] jepsen worker 5 - jepsen.util 7655 :fail :cas [1 [6 48]] :ValueNotSwapFrom INFO [2019-12-17 12:34:16,391] jepsen worker 5 - jepsen.util 7655 :invoke :read [1 nil] INFO [2019-12-17 12:34:16,439] jepsen worker 54 - jepsen.util 9054 :ok :read [18 48] INFO [2019-12-17 12:34:16,439] jepsen worker 54 - jepsen.util 9054 :invoke :write [18 42] INFO [2019-12-17 12:34:16,443] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46794,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:34:16,443] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46794,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:34:16,443] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46798,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:34:16,443] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46798,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:34:16,452] jepsen worker 55 - jepsen.util 8965 :fail :cas [18 [10 34]] :ValueNotSwapFrom INFO [2019-12-17 12:34:16,452] jepsen worker 55 - jepsen.util 8965 :invoke :cas [18 [30 38]] INFO [2019-12-17 12:34:16,513] jepsen worker 32 - jepsen.util 8132 :info :write [10 22] :RequestCanceledException INFO [2019-12-17 12:34:16,518] jepsen worker 32 - jepsen.util 8222 :invoke :cas [10 [3 7]] INFO [2019-12-17 12:34:16,529] jepsen worker 12 - jepsen.util 8652 :info :write [4 20] :RequestCanceledException INFO [2019-12-17 12:34:16,530] jepsen worker 12 - jepsen.util 8742 :invoke :read [4 nil] INFO [2019-12-17 12:34:16,531] jepsen worker 31 - jepsen.util 7141 :info :write [10 33] :RequestCanceledException INFO [2019-12-17 12:34:16,549] jepsen worker 31 - jepsen.util 7231 :invoke :read [10 nil] INFO [2019-12-17 12:34:16,558] jepsen worker 14 - jepsen.util 8384 :fail :cas [4 [19 3]] :ValueNotSwapFrom INFO [2019-12-17 12:34:16,560] jepsen worker 74 - jepsen.util 8354 :fail :read [24 nil] :RequestTimeoutException INFO [2019-12-17 12:34:16,560] jepsen worker 53 - jepsen.util 7433 :info :write [17 26] :RequestCanceledException INFO [2019-12-17 12:34:16,578] jepsen worker 32 - jepsen.util 8222 :fail :cas [10 [3 7]] :ValueNotSwapFrom INFO [2019-12-17 12:34:16,586] jepsen worker 56 - jepsen.util 9146 :fail :cas [18 [18 25]] :ValueNotSwapFrom INFO [2019-12-17 12:34:16,594] jepsen worker 31 - jepsen.util 7231 :ok :read [10 5] INFO [2019-12-17 12:34:16,599] jepsen worker 78 - jepsen.util 7908 :fail :cas [26 [10 33]] :ValueNotSwapFrom INFO [2019-12-17 12:34:16,601] jepsen worker 10 - jepsen.util 8200 :info :write [3 34] :SyncWriteAmbiguous INFO [2019-12-17 12:34:16,605] jepsen worker 73 - jepsen.util 7633 :info :write [24 14] :SyncWriteAmbiguous INFO [2019-12-17 12:34:16,613] jepsen worker 47 - jepsen.util 8417 :info :write [15 41] :RequestCanceledException INFO [2019-12-17 12:34:16,613] jepsen worker 11 - jepsen.util 8021 :info :write [3 30] :RequestCanceledException INFO [2019-12-17 12:34:16,613] jepsen worker 9 - jepsen.util 8289 :info :write [3 24] :RequestCanceledException INFO [2019-12-17 12:34:16,627] jepsen worker 71 - jepsen.util 8711 :fail :cas [23 [39 2]] :ValueNotSwapFrom INFO [2019-12-17 12:34:16,642] jepsen worker 33 - jepsen.util 10833 :info :cas [11 [7 19]] :RequestCanceledException INFO [2019-12-17 12:34:16,645] jepsen worker 18 - jepsen.util 8298 :fail :cas [6 [33 10]] :ValueNotSwapFrom INFO [2019-12-17 12:34:16,645] jepsen worker 51 - jepsen.util 7611 :ok :read [17 0] INFO [2019-12-17 12:34:16,651] jepsen worker 65 - jepsen.util 6725 :ok :write [21 24] INFO [2019-12-17 12:34:16,660] jepsen worker 58 - jepsen.util 11848 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:34:16,672] jepsen worker 88 - jepsen.util 8188 :fail :cas [29 [42 41]] :ValueNotSwapFrom INFO [2019-12-17 12:34:16,679] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46805,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:34:16,693] jepsen worker 3 - jepsen.util 8013 :ok :read [1 27] INFO [2019-12-17 12:34:16,696] jepsen worker 57 - jepsen.util 10947 :info :cas [19 [35 19]] :RequestCanceledException INFO [2019-12-17 12:34:16,717] jepsen worker 14 - jepsen.util 8384 :invoke :cas [4 [32 28]] INFO [2019-12-17 12:34:16,717] jepsen worker 74 - jepsen.util 8354 :invoke :cas [24 [15 49]] INFO [2019-12-17 12:34:16,717] jepsen worker 32 - jepsen.util 8222 :invoke :write [10 27] INFO [2019-12-17 12:34:16,717] jepsen worker 56 - jepsen.util 9146 :invoke :cas [18 [15 13]] INFO [2019-12-17 12:34:16,717] jepsen worker 31 - jepsen.util 7231 :invoke :write [10 5] INFO [2019-12-17 12:34:16,717] jepsen worker 78 - jepsen.util 7908 :invoke :write [26 26] INFO [2019-12-17 12:34:16,717] jepsen worker 71 - jepsen.util 8711 :invoke :write [23 47] INFO [2019-12-17 12:34:16,717] jepsen worker 18 - jepsen.util 8298 :invoke :write [6 30] INFO [2019-12-17 12:34:16,717] jepsen worker 51 - jepsen.util 7611 :invoke :write [17 22] INFO [2019-12-17 12:34:16,718] jepsen worker 65 - jepsen.util 6725 :invoke :write [21 3] INFO [2019-12-17 12:34:16,718] jepsen worker 58 - jepsen.util 11848 :invoke :write [19 29] INFO [2019-12-17 12:34:16,718] jepsen worker 88 - jepsen.util 8188 :invoke :write [29 31] INFO [2019-12-17 12:34:16,718] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46805,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:34:16,718] jepsen worker 3 - jepsen.util 8013 :invoke :cas [1 [35 42]] INFO [2019-12-17 12:34:16,720] jepsen worker 11 - jepsen.util 8111 :invoke :cas [3 [13 47]] INFO [2019-12-17 12:34:16,720] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46809,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:34:16,720] jepsen worker 9 - jepsen.util 8379 :invoke :read [3 nil] INFO [2019-12-17 12:34:16,720] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46809,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:34:16,721] jepsen worker 10 - jepsen.util 8290 :invoke :write [3 45] INFO [2019-12-17 12:34:16,722] jepsen worker 73 - jepsen.util 7723 :invoke :cas [24 [0 48]] INFO [2019-12-17 12:34:16,723] jepsen worker 47 - jepsen.util 8507 :invoke :read [15 nil] INFO [2019-12-17 12:34:16,724] jepsen worker 57 - jepsen.util 11037 :invoke :read [19 nil] INFO [2019-12-17 12:34:16,724] jepsen worker 33 - jepsen.util 10923 :invoke :write [11 22] INFO [2019-12-17 12:34:16,727] jepsen worker 62 - jepsen.util 11312 :info :write [20 6] :RequestCanceledException INFO [2019-12-17 12:34:16,728] jepsen worker 53 - jepsen.util 7523 :invoke :read [17 nil] INFO [2019-12-17 12:34:16,736] jepsen worker 62 - jepsen.util 11402 :invoke :read [20 nil] INFO [2019-12-17 12:34:16,743] jepsen worker 25 - jepsen.util 11635 :info :write [8 32] :RequestCanceledException INFO [2019-12-17 12:34:16,756] jepsen worker 64 - jepsen.util 6724 :fail :read [21 nil] :RequestTimeoutException INFO [2019-12-17 12:34:16,756] jepsen worker 64 - jepsen.util 6724 :invoke :read [21 nil] INFO [2019-12-17 12:34:16,758] jepsen worker 25 - jepsen.util 11725 :invoke :read [8 nil] INFO [2019-12-17 12:34:16,786] jepsen worker 46 - jepsen.util 8326 :ok :write [15 19] INFO [2019-12-17 12:34:16,786] jepsen worker 46 - jepsen.util 8326 :invoke :write [15 17] INFO [2019-12-17 12:34:16,791] jepsen worker 55 - jepsen.util 8965 :fail :cas [18 [30 38]] :ValueNotSwapFrom INFO [2019-12-17 12:34:16,791] jepsen worker 55 - jepsen.util 8965 :invoke :write [18 18] INFO [2019-12-17 12:34:16,795] jepsen worker 0 - jepsen.util 11520 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:34:16,795] jepsen worker 0 - jepsen.util 11520 :invoke :cas [0 [46 31]] INFO [2019-12-17 12:34:16,820] jepsen worker 73 - jepsen.util 7723 :fail :cas [24 [0 48]] :ValueNotSwapFrom INFO [2019-12-17 12:34:16,820] jepsen worker 73 - jepsen.util 7723 :invoke :read [24 nil] INFO [2019-12-17 12:34:16,829] jepsen worker 85 - jepsen.util 11785 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:34:16,829] jepsen worker 85 - jepsen.util 11785 :invoke :cas [28 [10 38]] INFO [2019-12-17 12:34:16,831] jepsen worker 9 - jepsen.util 8379 :ok :read [3 14] INFO [2019-12-17 12:34:16,831] jepsen worker 9 - jepsen.util 8379 :invoke :write [3 34] INFO [2019-12-17 12:34:16,837] jepsen worker 20 - jepsen.util 7670 :ok :write [6 30] INFO [2019-12-17 12:34:16,837] jepsen worker 20 - jepsen.util 7670 :invoke :read [6 nil] INFO [2019-12-17 12:34:16,882] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46844,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:34:16,883] jepsen worker 63 - jepsen.util 6903 :ok :read [21 24] INFO [2019-12-17 12:34:16,885] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46844,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:34:16,886] jepsen worker 63 - jepsen.util 6903 :invoke :read [21 nil] INFO [2019-12-17 12:34:16,911] jepsen worker 69 - jepsen.util 8169 :info :write [23 48] :RequestCanceledException INFO [2019-12-17 12:34:16,914] jepsen worker 87 - jepsen.util 7557 :ok :read [29 36] INFO [2019-12-17 12:34:16,914] jepsen worker 87 - jepsen.util 7557 :invoke :read [29 nil] INFO [2019-12-17 12:34:16,915] jepsen worker 69 - jepsen.util 8259 :invoke :cas [23 [41 15]] INFO [2019-12-17 12:34:16,917] jepsen worker 3 - jepsen.util 8013 :fail :cas [1 [35 42]] :ValueNotSwapFrom INFO [2019-12-17 12:34:16,917] jepsen worker 3 - jepsen.util 8013 :invoke :write [1 13] INFO [2019-12-17 12:34:16,928] jepsen worker 22 - jepsen.util 11002 :info :cas [7 [21 19]] :RequestCanceledException INFO [2019-12-17 12:34:16,936] jepsen worker 22 - jepsen.util 11092 :invoke :cas [7 [7 27]] INFO [2019-12-17 12:34:16,942] jepsen worker 56 - jepsen.util 9146 :fail :cas [18 [15 13]] :ValueNotSwapFrom INFO [2019-12-17 12:34:16,942] jepsen worker 56 - jepsen.util 9146 :invoke :read [18 nil] INFO [2019-12-17 12:34:16,960] jepsen worker 27 - jepsen.util 7947 :ok :write [9 31] INFO [2019-12-17 12:34:16,960] jepsen worker 27 - jepsen.util 7947 :invoke :write [9 24] INFO [2019-12-17 12:34:16,960] jepsen worker 52 - jepsen.util 8062 :info :write [17 2] :RequestCanceledException INFO [2019-12-17 12:34:16,961] jepsen worker 11 - jepsen.util 8111 :fail :cas [3 [13 47]] :ValueNotSwapFrom INFO [2019-12-17 12:34:16,962] jepsen worker 11 - jepsen.util 8111 :invoke :read [3 nil] INFO [2019-12-17 12:34:16,962] jepsen worker 52 - jepsen.util 8152 :invoke :read [17 nil] INFO [2019-12-17 12:34:16,996] jepsen worker 48 - jepsen.util 10128 :info :write [16 8] :RequestCanceledException INFO [2019-12-17 12:34:16,997] jepsen worker 48 - jepsen.util 10218 :invoke :read [16 nil] INFO [2019-12-17 12:34:17,007] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46856,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:34:17,007] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46856,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:34:17,007] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46861,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:34:17,007] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46861,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:34:17,013] jepsen worker 26 - jepsen.util 11546 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:34:17,014] jepsen worker 26 - jepsen.util 11546 :invoke :write [8 30] INFO [2019-12-17 12:34:17,017] jepsen worker 74 - jepsen.util 8354 :fail :cas [24 [15 49]] :ValueNotSwapFrom INFO [2019-12-17 12:34:17,017] jepsen worker 74 - jepsen.util 8354 :invoke :cas [24 [8 29]] INFO [2019-12-17 12:34:17,029] jepsen worker 61 - jepsen.util 11851 :info :cas [20 [30 45]] :RequestCanceledException INFO [2019-12-17 12:34:17,032] jepsen worker 61 - jepsen.util 11941 :invoke :cas [20 [18 35]] INFO [2019-12-17 12:34:17,042] jepsen worker 84 - jepsen.util 10614 :info :cas [28 [32 32]] :RequestCanceledException INFO [2019-12-17 12:34:17,046] jepsen worker 84 - jepsen.util 10704 :invoke :read [28 nil] INFO [2019-12-17 12:34:17,056] jepsen worker 30 - jepsen.util 8040 :info :cas [10 [37 48]] :RequestCanceledException INFO [2019-12-17 12:34:17,056] jepsen worker 70 - jepsen.util 8440 :info :write [23 31] :RequestCanceledException INFO [2019-12-17 12:34:17,059] jepsen worker 30 - jepsen.util 8130 :invoke :cas [10 [42 0]] INFO [2019-12-17 12:34:17,060] jepsen worker 70 - jepsen.util 8530 :invoke :write [23 42] INFO [2019-12-17 12:34:17,063] jepsen worker 47 - jepsen.util 8507 :ok :read [15 19] INFO [2019-12-17 12:34:17,063] jepsen worker 47 - jepsen.util 8507 :invoke :cas [15 [27 48]] INFO [2019-12-17 12:34:17,064] jepsen worker 5 - jepsen.util 7655 :ok :read [1 27] INFO [2019-12-17 12:34:17,065] jepsen worker 5 - jepsen.util 7655 :invoke :write [1 1] INFO [2019-12-17 12:34:17,083] jepsen worker 64 - jepsen.util 6724 :ok :read [21 24] INFO [2019-12-17 12:34:17,083] jepsen worker 64 - jepsen.util 6724 :invoke :write [21 8] INFO [2019-12-17 12:34:17,093] jepsen worker 38 - jepsen.util 8408 :info :write [12 30] :RequestCanceledException INFO [2019-12-17 12:34:17,096] jepsen worker 38 - jepsen.util 8498 :invoke :cas [12 [9 24]] INFO [2019-12-17 12:34:17,103] jepsen worker 20 - jepsen.util 7670 :ok :read [6 30] INFO [2019-12-17 12:34:17,103] jepsen worker 20 - jepsen.util 7670 :invoke :cas [6 [8 7]] INFO [2019-12-17 12:34:17,113] jepsen worker 6 - jepsen.util 11526 :info :write [2 16] :RequestCanceledException INFO [2019-12-17 12:34:17,115] jepsen worker 6 - jepsen.util 11616 :invoke :write [2 32] INFO [2019-12-17 12:34:17,116] jepsen worker 65 - jepsen.util 6725 :ok :write [21 3] INFO [2019-12-17 12:34:17,116] jepsen worker 65 - jepsen.util 6725 :invoke :cas [21 [43 48]] INFO [2019-12-17 12:34:17,127] jepsen worker 66 - jepsen.util 10776 :info :cas [22 [25 39]] :RequestCanceledException INFO [2019-12-17 12:34:17,127] jepsen worker 40 - jepsen.util 11470 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:34:17,127] jepsen worker 40 - jepsen.util 11470 :invoke :read [13 nil] INFO [2019-12-17 12:34:17,129] jepsen worker 66 - jepsen.util 10866 :invoke :write [22 26] INFO [2019-12-17 12:34:17,140] jepsen worker 27 - jepsen.util 7947 :ok :write [9 24] INFO [2019-12-17 12:34:17,140] jepsen worker 27 - jepsen.util 7947 :invoke :read [9 nil] INFO [2019-12-17 12:34:17,143] jepsen worker 44 - jepsen.util 11204 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:34:17,143] jepsen worker 44 - jepsen.util 11204 :invoke :write [14 34] INFO [2019-12-17 12:34:17,149] jepsen worker 46 - jepsen.util 8326 :ok :write [15 17] INFO [2019-12-17 12:34:17,149] jepsen worker 46 - jepsen.util 8326 :invoke :read [15 nil] INFO [2019-12-17 12:34:17,157] jepsen worker 4 - jepsen.util 7564 :fail :read [1 nil] :RequestTimeoutException INFO [2019-12-17 12:34:17,157] jepsen worker 4 - jepsen.util 7564 :invoke :write [1 48] INFO [2019-12-17 12:34:17,193] jepsen worker 56 - jepsen.util 9146 :ok :read [18 48] INFO [2019-12-17 12:34:17,193] jepsen worker 56 - jepsen.util 9146 :invoke :write [18 20] INFO [2019-12-17 12:34:17,206] jepsen worker 30 - jepsen.util 8130 :fail :cas [10 [42 0]] :ValueNotSwapFrom INFO [2019-12-17 12:34:17,206] jepsen worker 30 - jepsen.util 8130 :invoke :write [10 46] INFO [2019-12-17 12:34:17,214] jepsen worker 89 - jepsen.util 8009 :info :write [29 18] :RequestCanceledException INFO [2019-12-17 12:34:17,215] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46883,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:34:17,216] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46883,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:34:17,218] jepsen worker 89 - jepsen.util 8099 :invoke :read [29 nil] INFO [2019-12-17 12:34:17,225] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46890,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:34:17,225] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46890,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:34:17,226] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46893,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:34:17,226] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46893,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:34:17,241] jepsen worker 11 - jepsen.util 8111 :ok :read [3 14] INFO [2019-12-17 12:34:17,241] jepsen worker 11 - jepsen.util 8111 :invoke :write [3 48] INFO [2019-12-17 12:34:17,242] jepsen worker 47 - jepsen.util 8507 :fail :cas [15 [27 48]] :ValueNotSwapFrom INFO [2019-12-17 12:34:17,242] jepsen worker 47 - jepsen.util 8507 :invoke :cas [15 [23 25]] INFO [2019-12-17 12:34:17,260] jepsen worker 24 - jepsen.util 10014 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:34:17,261] jepsen worker 15 - jepsen.util 10095 :info :write [5 16] :RequestCanceledException INFO [2019-12-17 12:34:17,261] jepsen worker 24 - jepsen.util 10014 :invoke :write [8 26] INFO [2019-12-17 12:34:17,261] jepsen worker 64 - jepsen.util 6724 :ok :write [21 8] INFO [2019-12-17 12:34:17,261] jepsen worker 64 - jepsen.util 6724 :invoke :cas [21 [12 0]] INFO [2019-12-17 12:34:17,263] jepsen worker 15 - jepsen.util 10185 :invoke :write [5 43] INFO [2019-12-17 12:34:17,270] jepsen worker 46 - jepsen.util 8326 :ok :read [15 17] INFO [2019-12-17 12:34:17,270] jepsen worker 46 - jepsen.util 8326 :invoke :write [15 26] INFO [2019-12-17 12:34:17,298] jepsen worker 52 - jepsen.util 8152 :ok :read [17 0] INFO [2019-12-17 12:34:17,298] jepsen worker 52 - jepsen.util 8152 :invoke :cas [17 [37 38]] INFO [2019-12-17 12:34:17,304] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:34:17,304] jepsen nemesis - couchbase.util Rebalance check count: 363 INFO [2019-12-17 12:34:17,304] jepsen nemesis - couchbase.util Rebalance stuck count: 344 INFO [2019-12-17 12:34:17,329] jepsen worker 42 - jepsen.util 11922 :info :write [14 48] :RequestCanceledException INFO [2019-12-17 12:34:17,332] jepsen worker 42 - jepsen.util 12012 :invoke :cas [14 [3 47]] INFO [2019-12-17 12:34:17,342] jepsen worker 12 - jepsen.util 8742 :ok :read [4 21] INFO [2019-12-17 12:34:17,343] jepsen worker 12 - jepsen.util 8742 :invoke :write [4 18] INFO [2019-12-17 12:34:17,343] jepsen worker 28 - jepsen.util 6328 :info :cas [9 [46 41]] :RequestCanceledException INFO [2019-12-17 12:34:17,350] jepsen worker 28 - jepsen.util 6418 :invoke :cas [9 [37 6]] INFO [2019-12-17 12:34:17,373] jepsen worker 38 - jepsen.util 8498 :fail :cas [12 [9 24]] :ValueNotSwapFrom INFO [2019-12-17 12:34:17,373] jepsen worker 38 - jepsen.util 8498 :invoke :write [12 22] INFO [2019-12-17 12:34:17,443] jepsen worker 17 - jepsen.util 10997 :info :cas [5 [22 39]] :RequestCanceledException INFO [2019-12-17 12:34:17,444] jepsen worker 17 - jepsen.util 11087 :invoke :read [5 nil] INFO [2019-12-17 12:34:17,495] jepsen worker 52 - jepsen.util 8152 :fail :cas [17 [37 38]] :ValueNotSwapFrom INFO [2019-12-17 12:34:17,495] jepsen worker 52 - jepsen.util 8152 :invoke :read [17 nil] INFO [2019-12-17 12:34:17,496] jepsen worker 8 - jepsen.util 10898 :info :write [2 18] :RequestCanceledException INFO [2019-12-17 12:34:17,496] jepsen worker 7 - jepsen.util 11347 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:34:17,496] jepsen worker 36 - jepsen.util 8406 :info :write [12 29] :RequestCanceledException INFO [2019-12-17 12:34:17,496] jepsen worker 1 - jepsen.util 11251 :info :write [0 10] :RequestCanceledException INFO [2019-12-17 12:34:17,496] jepsen worker 7 - jepsen.util 11347 :invoke :read [2 nil] INFO [2019-12-17 12:34:17,499] jepsen worker 36 - jepsen.util 8496 :invoke :cas [12 [41 9]] INFO [2019-12-17 12:34:17,499] jepsen worker 8 - jepsen.util 10988 :invoke :read [2 nil] INFO [2019-12-17 12:34:17,499] jepsen worker 1 - jepsen.util 11341 :invoke :read [0 nil] INFO [2019-12-17 12:34:17,513] jepsen worker 14 - jepsen.util 8384 :fail :cas [4 [32 28]] :ValueNotSwapFrom INFO [2019-12-17 12:34:17,513] jepsen worker 14 - jepsen.util 8384 :invoke :write [4 43] INFO [2019-12-17 12:34:17,514] jepsen worker 77 - jepsen.util 11327 :info :cas [25 [35 47]] :RequestCanceledException INFO [2019-12-17 12:34:17,515] jepsen worker 77 - jepsen.util 11417 :invoke :cas [25 [11 11]] INFO [2019-12-17 12:34:17,528] jepsen worker 53 - jepsen.util 7523 :ok :read [17 0] INFO [2019-12-17 12:34:17,528] jepsen worker 53 - jepsen.util 7523 :invoke :write [17 38] INFO [2019-12-17 12:34:17,528] jepsen worker 43 - jepsen.util 10573 :info :write [14 44] :RequestCanceledException INFO [2019-12-17 12:34:17,529] jepsen worker 13 - jepsen.util 7573 :info :write [4 43] :RequestCanceledException INFO [2019-12-17 12:34:17,531] jepsen worker 43 - jepsen.util 10663 :invoke :cas [14 [28 8]] INFO [2019-12-17 12:34:17,531] jepsen worker 13 - jepsen.util 7663 :invoke :cas [4 [24 42]] INFO [2019-12-17 12:34:17,535] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46911,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:34:17,535] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46911,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:34:17,535] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46913,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:34:17,535] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46913,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:34:17,542] jepsen worker 75 - jepsen.util 10515 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:34:17,542] jepsen worker 75 - jepsen.util 10515 :invoke :read [25 nil] INFO [2019-12-17 12:34:17,573] jepsen worker 52 - jepsen.util 8152 :ok :read [17 0] INFO [2019-12-17 12:34:17,575] jepsen worker 52 - jepsen.util 8152 :invoke :cas [17 [11 30]] INFO [2019-12-17 12:34:17,627] jepsen worker 2 - jepsen.util 10712 :info :write [0 0] :RequestCanceledException INFO [2019-12-17 12:34:17,627] jepsen worker 49 - jepsen.util 10399 :info :write [16 21] :RequestCanceledException INFO [2019-12-17 12:34:17,628] jepsen worker 2 - jepsen.util 10802 :invoke :read [0 nil] INFO [2019-12-17 12:34:17,628] jepsen worker 49 - jepsen.util 10489 :invoke :write [16 0] INFO [2019-12-17 12:34:17,692] jepsen worker 89 - jepsen.util 8099 :ok :read [29 36] INFO [2019-12-17 12:34:17,693] jepsen worker 89 - jepsen.util 8099 :invoke :write [29 42] INFO [2019-12-17 12:34:17,703] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46931,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:34:17,703] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46931,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:34:17,769] jepsen worker 31 - jepsen.util 7231 :ok :write [10 5] INFO [2019-12-17 12:34:17,770] jepsen worker 18 - jepsen.util 8298 :ok :write [6 30] INFO [2019-12-17 12:34:17,771] jepsen worker 18 - jepsen.util 8298 :invoke :write [6 26] INFO [2019-12-17 12:34:17,772] jepsen worker 46 - jepsen.util 8326 :ok :write [15 26] INFO [2019-12-17 12:34:17,772] jepsen worker 46 - jepsen.util 8326 :invoke :write [15 40] INFO [2019-12-17 12:34:17,774] jepsen worker 31 - jepsen.util 7231 :invoke :write [10 18] INFO [2019-12-17 12:34:17,783] jepsen worker 10 - jepsen.util 8290 :ok :write [3 45] INFO [2019-12-17 12:34:17,783] jepsen worker 10 - jepsen.util 8290 :invoke :cas [3 [33 0]] INFO [2019-12-17 12:34:17,795] jepsen worker 79 - jepsen.util 8359 :info :write [26 26] :RequestCanceledException INFO [2019-12-17 12:34:17,799] jepsen worker 79 - jepsen.util 8449 :invoke :write [26 37] INFO [2019-12-17 12:34:17,804] jepsen worker 87 - jepsen.util 7557 :ok :read [29 36] INFO [2019-12-17 12:34:17,804] jepsen worker 87 - jepsen.util 7557 :invoke :read [29 nil] INFO [2019-12-17 12:34:17,827] jepsen worker 28 - jepsen.util 6418 :fail :cas [9 [37 6]] :ValueNotSwapFrom INFO [2019-12-17 12:34:17,827] jepsen worker 28 - jepsen.util 6418 :invoke :cas [9 [21 20]] INFO [2019-12-17 12:34:17,849] jepsen worker 10 - jepsen.util 8290 :fail :cas [3 [33 0]] :ValueNotSwapFrom INFO [2019-12-17 12:34:17,849] jepsen worker 10 - jepsen.util 8290 :invoke :write [3 40] INFO [2019-12-17 12:34:17,881] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46946,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:34:17,881] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46946,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:34:17,881] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46949,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:34:17,882] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46949,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:34:17,893] jepsen worker 69 - jepsen.util 8259 :fail :cas [23 [41 15]] :ValueNotSwapFrom INFO [2019-12-17 12:34:17,893] jepsen worker 69 - jepsen.util 8259 :invoke :write [23 8] INFO [2019-12-17 12:34:17,894] jepsen worker 41 - jepsen.util 11741 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:34:17,894] jepsen worker 41 - jepsen.util 11741 :invoke :cas [13 [37 10]] INFO [2019-12-17 12:34:17,913] jepsen worker 80 - jepsen.util 8000 :info :write [26 22] :RequestCanceledException INFO [2019-12-17 12:34:17,915] jepsen worker 80 - jepsen.util 8090 :invoke :write [26 11] INFO [2019-12-17 12:34:17,926] jepsen worker 86 - jepsen.util 11696 :info :cas [28 [18 42]] :RequestCanceledException INFO [2019-12-17 12:34:17,929] jepsen worker 86 - jepsen.util 11786 :invoke :cas [28 [31 44]] INFO [2019-12-17 12:34:17,943] jepsen worker 39 - jepsen.util 10659 :info :write [13 36] :RequestCanceledException INFO [2019-12-17 12:34:17,945] jepsen worker 39 - jepsen.util 10749 :invoke :read [13 nil] INFO [2019-12-17 12:34:17,957] jepsen worker 59 - jepsen.util 10679 :info :cas [19 [4 2]] :RequestCanceledException INFO [2019-12-17 12:34:17,959] jepsen worker 59 - jepsen.util 10769 :invoke :read [19 nil] INFO [2019-12-17 12:34:17,995] jepsen worker 81 - jepsen.util 10791 :info :write [27 10] :RequestCanceledException INFO [2019-12-17 12:34:17,995] jepsen worker 68 - jepsen.util 10418 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:34:17,995] jepsen worker 68 - jepsen.util 10418 :invoke :write [22 26] INFO [2019-12-17 12:34:17,997] jepsen worker 81 - jepsen.util 10881 :invoke :write [27 23] INFO [2019-12-17 12:34:18,011] jepsen worker 21 - jepsen.util 10371 :info :cas [7 [11 18]] :RequestCanceledException INFO [2019-12-17 12:34:18,012] jepsen worker 21 - jepsen.util 10461 :invoke :write [7 1] INFO [2019-12-17 12:34:18,029] jepsen worker 82 - jepsen.util 11602 :info :write [27 1] :RequestCanceledException INFO [2019-12-17 12:34:18,029] jepsen worker 50 - jepsen.util 11750 :info :cas [16 [49 17]] :RequestCanceledException INFO [2019-12-17 12:34:18,029] jepsen worker 23 - jepsen.util 10823 :info :write [7 47] :RequestCanceledException INFO [2019-12-17 12:34:18,031] jepsen worker 50 - jepsen.util 11840 :invoke :write [16 13] INFO [2019-12-17 12:34:18,039] jepsen worker 82 - jepsen.util 11692 :invoke :cas [27 [44 17]] INFO [2019-12-17 12:34:18,039] jepsen worker 23 - jepsen.util 10913 :invoke :cas [7 [45 1]] INFO [2019-12-17 12:34:18,042] jepsen worker 37 - jepsen.util 7777 :ok :write [12 10] INFO [2019-12-17 12:34:18,043] jepsen worker 37 - jepsen.util 7777 :invoke :read [12 nil] INFO [2019-12-17 12:34:18,050] jepsen worker 52 - jepsen.util 8152 :fail :cas [17 [11 30]] :ValueNotSwapFrom INFO [2019-12-17 12:34:18,050] jepsen worker 52 - jepsen.util 8152 :invoke :cas [17 [35 10]] INFO [2019-12-17 12:34:18,055] jepsen worker 31 - jepsen.util 7231 :ok :write [10 18] INFO [2019-12-17 12:34:18,055] jepsen worker 31 - jepsen.util 7231 :invoke :write [10 18] INFO [2019-12-17 12:34:18,056] jepsen worker 18 - jepsen.util 8298 :ok :write [6 26] INFO [2019-12-17 12:34:18,056] jepsen worker 18 - jepsen.util 8298 :invoke :cas [6 [8 34]] INFO [2019-12-17 12:34:18,056] jepsen worker 46 - jepsen.util 8326 :ok :write [15 40] INFO [2019-12-17 12:34:18,056] jepsen worker 46 - jepsen.util 8326 :invoke :cas [15 [17 39]] INFO [2019-12-17 12:34:18,090] jepsen worker 36 - jepsen.util 8496 :fail :cas [12 [41 9]] :ValueNotSwapFrom INFO [2019-12-17 12:34:18,090] jepsen worker 36 - jepsen.util 8496 :invoke :write [12 3] INFO [2019-12-17 12:34:18,095] jepsen worker 76 - jepsen.util 10876 :info :cas [25 [36 45]] :RequestCanceledException INFO [2019-12-17 12:34:18,097] jepsen worker 83 - jepsen.util 11333 :info :write [27 10] :RequestCanceledException INFO [2019-12-17 12:34:18,100] jepsen worker 76 - jepsen.util 10966 :invoke :write [25 22] INFO [2019-12-17 12:34:18,100] jepsen worker 83 - jepsen.util 11423 :invoke :cas [27 [3 43]] INFO [2019-12-17 12:34:18,101] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46958,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:34:18,101] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46958,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:34:18,107] jepsen worker 10 - jepsen.util 8290 :ok :write [3 40] INFO [2019-12-17 12:34:18,107] jepsen worker 10 - jepsen.util 8290 :invoke :cas [3 [34 3]] INFO [2019-12-17 12:34:18,114] jepsen worker 60 - jepsen.util 12030 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:34:18,114] jepsen worker 60 - jepsen.util 12030 :invoke :write [20 24] INFO [2019-12-17 12:34:18,126] jepsen worker 16 - jepsen.util 12256 :info :write [5 24] :RequestCanceledException INFO [2019-12-17 12:34:18,135] jepsen worker 16 - jepsen.util 12346 :invoke :read [5 nil] INFO [2019-12-17 12:34:18,137] jepsen worker 18 - jepsen.util 8298 :fail :cas [6 [8 34]] :ValueNotSwapFrom INFO [2019-12-17 12:34:18,137] jepsen worker 18 - jepsen.util 8298 :invoke :read [6 nil] INFO [2019-12-17 12:34:18,137] jepsen worker 46 - jepsen.util 8326 :fail :cas [15 [17 39]] :ValueNotSwapFrom INFO [2019-12-17 12:34:18,137] jepsen worker 46 - jepsen.util 8326 :invoke :cas [15 [14 15]] INFO [2019-12-17 12:34:18,142] jepsen worker 45 - jepsen.util 8145 :info :write [15 36] :RequestCanceledException INFO [2019-12-17 12:34:18,142] jepsen worker 29 - jepsen.util 6689 :fail :read [9 nil] :RequestTimeoutException INFO [2019-12-17 12:34:18,143] jepsen worker 29 - jepsen.util 6689 :invoke :write [9 19] INFO [2019-12-17 12:34:18,147] jepsen worker 45 - jepsen.util 8235 :invoke :read [15 nil] INFO [2019-12-17 12:34:18,159] jepsen worker 28 - jepsen.util 6418 :fail :cas [9 [21 20]] :ValueNotSwapFrom INFO [2019-12-17 12:34:18,159] jepsen worker 28 - jepsen.util 6418 :invoke :read [9 nil] INFO [2019-12-17 12:34:18,167] jepsen worker 10 - jepsen.util 8290 :fail :cas [3 [34 3]] :ValueNotSwapFrom INFO [2019-12-17 12:34:18,167] jepsen worker 10 - jepsen.util 8290 :invoke :cas [3 [23 8]] INFO [2019-12-17 12:34:18,212] jepsen worker 46 - jepsen.util 8326 :fail :cas [15 [14 15]] :ValueNotSwapFrom INFO [2019-12-17 12:34:18,212] jepsen worker 46 - jepsen.util 8326 :invoke :write [15 17] INFO [2019-12-17 12:34:18,212] jepsen worker 10 - jepsen.util 8290 :fail :cas [3 [23 8]] :ValueNotSwapFrom INFO [2019-12-17 12:34:18,212] jepsen worker 10 - jepsen.util 8290 :invoke :write [3 28] INFO [2019-12-17 12:34:18,214] jepsen worker 18 - jepsen.util 8298 :ok :read [6 26] INFO [2019-12-17 12:34:18,214] jepsen worker 18 - jepsen.util 8298 :invoke :cas [6 [44 11]] INFO [2019-12-17 12:34:18,216] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46985,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:34:18,216] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46985,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:34:18,216] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46987,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:34:18,216] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46987,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:34:18,251] jepsen worker 3 - jepsen.util 8013 :ok :write [1 13] INFO [2019-12-17 12:34:18,251] jepsen worker 3 - jepsen.util 8013 :invoke :write [1 23] INFO [2019-12-17 12:34:18,260] jepsen worker 34 - jepsen.util 10924 :info :write [11 25] :RequestCanceledException INFO [2019-12-17 12:34:18,260] jepsen worker 31 - jepsen.util 7231 :ok :write [10 18] INFO [2019-12-17 12:34:18,260] jepsen worker 31 - jepsen.util 7231 :invoke :read [10 nil] INFO [2019-12-17 12:34:18,262] jepsen worker 34 - jepsen.util 11014 :invoke :cas [11 [28 48]] INFO [2019-12-17 12:34:18,273] jepsen worker 18 - jepsen.util 8298 :fail :cas [6 [44 11]] :ValueNotSwapFrom INFO [2019-12-17 12:34:18,273] jepsen worker 18 - jepsen.util 8298 :invoke :cas [6 [30 18]] INFO [2019-12-17 12:34:18,280] jepsen worker 28 - jepsen.util 6418 :ok :read [9 24] INFO [2019-12-17 12:34:18,280] jepsen worker 28 - jepsen.util 6418 :invoke :read [9 nil] INFO [2019-12-17 12:34:18,294] jepsen worker 29 - jepsen.util 6689 :ok :write [9 19] INFO [2019-12-17 12:34:18,294] jepsen worker 29 - jepsen.util 6689 :invoke :read [9 nil] INFO [2019-12-17 12:34:18,299] jepsen worker 45 - jepsen.util 8235 :ok :read [15 40] INFO [2019-12-17 12:34:18,300] jepsen worker 45 - jepsen.util 8235 :invoke :write [15 18] INFO [2019-12-17 12:34:18,319] jepsen worker 78 - jepsen.util 7908 :ok :write [26 26] INFO [2019-12-17 12:34:18,319] jepsen worker 78 - jepsen.util 7908 :invoke :read [26 nil] INFO [2019-12-17 12:34:18,322] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46992,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:34:18,322] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46992,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:34:18,322] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46996,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:34:18,322] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":46996,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:34:18,341] jepsen worker 4 - jepsen.util 7564 :ok :write [1 48] INFO [2019-12-17 12:34:18,341] jepsen worker 4 - jepsen.util 7564 :invoke :write [1 21] INFO [2019-12-17 12:34:18,361] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:34:18,361] jepsen nemesis - couchbase.util Rebalance check count: 364 INFO [2019-12-17 12:34:18,361] jepsen nemesis - couchbase.util Rebalance stuck count: 345 INFO [2019-12-17 12:34:18,398] jepsen worker 54 - jepsen.util 9054 :ok :write [18 42] INFO [2019-12-17 12:34:18,398] jepsen worker 54 - jepsen.util 9054 :invoke :cas [18 [5 23]] INFO [2019-12-17 12:34:18,412] jepsen worker 35 - jepsen.util 10745 :info :cas [11 [7 38]] :RequestCanceledException INFO [2019-12-17 12:34:18,414] jepsen worker 38 - jepsen.util 8498 :ok :write [12 22] INFO [2019-12-17 12:34:18,414] jepsen worker 38 - jepsen.util 8498 :invoke :cas [12 [12 29]] INFO [2019-12-17 12:34:18,415] jepsen worker 35 - jepsen.util 10835 :invoke :cas [11 [33 19]] INFO [2019-12-17 12:34:18,421] jepsen worker 31 - jepsen.util 7231 :ok :read [10 18] INFO [2019-12-17 12:34:18,421] jepsen worker 31 - jepsen.util 7231 :invoke :write [10 5] INFO [2019-12-17 12:34:18,424] jepsen worker 18 - jepsen.util 8298 :fail :cas [6 [30 18]] :ValueNotSwapFrom INFO [2019-12-17 12:34:18,424] jepsen worker 18 - jepsen.util 8298 :invoke :cas [6 [30 5]] INFO [2019-12-17 12:34:18,430] jepsen worker 19 - jepsen.util 8209 :info :write [6 14] :RequestCanceledException INFO [2019-12-17 12:34:18,430] jepsen worker 67 - jepsen.util 11137 :info :cas [22 [30 37]] :RequestCanceledException INFO [2019-12-17 12:34:18,430] jepsen worker 72 - jepsen.util 7992 :info :write [24 42] :RequestCanceledException INFO [2019-12-17 12:34:18,434] jepsen worker 67 - jepsen.util 11227 :invoke :read [22 nil] INFO [2019-12-17 12:34:18,434] jepsen worker 72 - jepsen.util 8082 :invoke :cas [24 [36 33]] INFO [2019-12-17 12:34:18,434] jepsen worker 19 - jepsen.util 8299 :invoke :read [6 nil] INFO [2019-12-17 12:34:18,436] jepsen worker 29 - jepsen.util 6689 :ok :read [9 19] INFO [2019-12-17 12:34:18,436] jepsen worker 29 - jepsen.util 6689 :invoke :cas [9 [32 1]] INFO [2019-12-17 12:34:18,495] jepsen worker 37 - jepsen.util 7777 :ok :read [12 22] INFO [2019-12-17 12:34:18,495] jepsen worker 37 - jepsen.util 7777 :invoke :write [12 18] INFO [2019-12-17 12:34:18,497] jepsen worker 18 - jepsen.util 8298 :fail :cas [6 [30 5]] :ValueNotSwapFrom INFO [2019-12-17 12:34:18,497] jepsen worker 18 - jepsen.util 8298 :invoke :cas [6 [29 19]] INFO [2019-12-17 12:34:18,513] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47014,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:34:18,513] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47014,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:34:18,513] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47017,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:34:18,514] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47017,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:34:18,514] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47018,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:34:18,514] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47018,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:34:18,570] jepsen worker 19 - jepsen.util 8299 :ok :read [6 26] INFO [2019-12-17 12:34:18,571] jepsen worker 19 - jepsen.util 8299 :invoke :cas [6 [6 15]] INFO [2019-12-17 12:34:18,573] jepsen worker 72 - jepsen.util 8082 :fail :cas [24 [36 33]] :ValueNotSwapFrom INFO [2019-12-17 12:34:18,573] jepsen worker 72 - jepsen.util 8082 :invoke :write [24 45] INFO [2019-12-17 12:34:18,589] jepsen worker 18 - jepsen.util 8298 :fail :cas [6 [29 19]] :ValueNotSwapFrom INFO [2019-12-17 12:34:18,590] jepsen worker 18 - jepsen.util 8298 :invoke :read [6 nil] INFO [2019-12-17 12:34:18,595] jepsen worker 46 - jepsen.util 8326 :ok :write [15 17] INFO [2019-12-17 12:34:18,595] jepsen worker 46 - jepsen.util 8326 :invoke :read [15 nil] INFO [2019-12-17 12:34:18,609] jepsen worker 19 - jepsen.util 8299 :fail :cas [6 [6 15]] :ValueNotSwapFrom INFO [2019-12-17 12:34:18,609] jepsen worker 19 - jepsen.util 8299 :invoke :write [6 42] INFO [2019-12-17 12:34:18,666] jepsen worker 29 - jepsen.util 6689 :fail :cas [9 [32 1]] :ValueNotSwapFrom INFO [2019-12-17 12:34:18,667] jepsen worker 29 - jepsen.util 6689 :invoke :read [9 nil] INFO [2019-12-17 12:34:18,680] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47032,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:34:18,681] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47032,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:34:18,681] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47035,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:34:18,681] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47035,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:34:18,681] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47036,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:34:18,681] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47036,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:34:18,691] jepsen worker 28 - jepsen.util 6418 :ok :read [9 19] INFO [2019-12-17 12:34:18,691] jepsen worker 28 - jepsen.util 6418 :invoke :read [9 nil] INFO [2019-12-17 12:34:18,698] jepsen worker 18 - jepsen.util 8298 :ok :read [6 26] INFO [2019-12-17 12:34:18,699] jepsen worker 18 - jepsen.util 8298 :invoke :cas [6 [16 2]] INFO [2019-12-17 12:34:18,708] jepsen worker 46 - jepsen.util 8326 :ok :read [15 17] INFO [2019-12-17 12:34:18,709] jepsen worker 46 - jepsen.util 8326 :invoke :cas [15 [39 8]] INFO [2019-12-17 12:34:18,718] jepsen worker 52 - jepsen.util 8152 :fail :cas [17 [35 10]] :ValueNotSwapFrom INFO [2019-12-17 12:34:18,718] jepsen worker 52 - jepsen.util 8152 :invoke :cas [17 [14 5]] INFO [2019-12-17 12:34:18,764] jepsen worker 28 - jepsen.util 6418 :ok :read [9 19] INFO [2019-12-17 12:34:18,764] jepsen worker 28 - jepsen.util 6418 :invoke :write [9 18] INFO [2019-12-17 12:34:18,786] jepsen worker 46 - jepsen.util 8326 :fail :cas [15 [39 8]] :ValueNotSwapFrom INFO [2019-12-17 12:34:18,786] jepsen worker 46 - jepsen.util 8326 :invoke :write [15 10] INFO [2019-12-17 12:34:18,787] jepsen worker 18 - jepsen.util 8298 :fail :cas [6 [16 2]] :ValueNotSwapFrom INFO [2019-12-17 12:34:18,787] jepsen worker 18 - jepsen.util 8298 :invoke :cas [6 [17 48]] INFO [2019-12-17 12:34:18,794] jepsen worker 33 - jepsen.util 10923 :info :write [11 22] :RequestCanceledException INFO [2019-12-17 12:34:18,799] jepsen worker 33 - jepsen.util 11013 :invoke :write [11 18] INFO [2019-12-17 12:34:18,804] jepsen worker 38 - jepsen.util 8498 :fail :cas [12 [12 29]] :ValueNotSwapFrom INFO [2019-12-17 12:34:18,804] jepsen worker 38 - jepsen.util 8498 :invoke :read [12 nil] INFO [2019-12-17 12:34:18,822] jepsen worker 52 - jepsen.util 8152 :fail :cas [17 [14 5]] :ValueNotSwapFrom INFO [2019-12-17 12:34:18,822] jepsen worker 52 - jepsen.util 8152 :invoke :cas [17 [0 36]] INFO [2019-12-17 12:34:18,829] jepsen worker 62 - jepsen.util 11402 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:34:18,829] jepsen worker 57 - jepsen.util 11037 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:34:18,829] jepsen worker 71 - jepsen.util 8711 :info :write [23 47] :RequestCanceledException INFO [2019-12-17 12:34:18,829] jepsen worker 51 - jepsen.util 7611 :info :write [17 22] :RequestCanceledException INFO [2019-12-17 12:34:18,829] jepsen worker 62 - jepsen.util 11402 :invoke :read [20 nil] INFO [2019-12-17 12:34:18,829] jepsen worker 57 - jepsen.util 11037 :invoke :write [19 44] INFO [2019-12-17 12:34:18,831] jepsen worker 71 - jepsen.util 8801 :invoke :cas [23 [18 2]] INFO [2019-12-17 12:34:18,834] jepsen worker 51 - jepsen.util 7701 :invoke :read [17 nil] INFO [2019-12-17 12:34:18,839] jepsen worker 18 - jepsen.util 8298 :fail :cas [6 [17 48]] :ValueNotSwapFrom INFO [2019-12-17 12:34:18,839] jepsen worker 18 - jepsen.util 8298 :invoke :read [6 nil] INFO [2019-12-17 12:34:18,843] jepsen worker 25 - jepsen.util 11725 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:34:18,843] jepsen worker 32 - jepsen.util 8222 :info :write [10 27] :RequestCanceledException INFO [2019-12-17 12:34:18,843] jepsen worker 73 - jepsen.util 7723 :fail :read [24 nil] :RequestTimeoutException INFO [2019-12-17 12:34:18,843] jepsen worker 25 - jepsen.util 11725 :invoke :read [8 nil] INFO [2019-12-17 12:34:18,843] jepsen worker 73 - jepsen.util 7723 :invoke :cas [24 [20 21]] INFO [2019-12-17 12:34:18,845] jepsen worker 32 - jepsen.util 8312 :invoke :read [10 nil] INFO [2019-12-17 12:34:18,857] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47058,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:34:18,858] jepsen worker 58 - jepsen.util 11848 :info :write [19 29] :RequestCanceledException INFO [2019-12-17 12:34:18,858] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47058,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:34:18,858] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47061,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:34:18,858] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47061,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:34:18,861] jepsen worker 58 - jepsen.util 11938 :invoke :write [19 42] INFO [2019-12-17 12:34:18,876] jepsen worker 13 - jepsen.util 7663 :fail :cas [4 [24 42]] :ValueNotSwapFrom INFO [2019-12-17 12:34:18,876] jepsen worker 13 - jepsen.util 7663 :invoke :read [4 nil] INFO [2019-12-17 12:34:18,876] jepsen worker 18 - jepsen.util 8298 :ok :read [6 26] INFO [2019-12-17 12:34:18,876] jepsen worker 18 - jepsen.util 8298 :invoke :write [6 40] INFO [2019-12-17 12:34:18,884] jepsen worker 28 - jepsen.util 6418 :ok :write [9 18] INFO [2019-12-17 12:34:18,884] jepsen worker 28 - jepsen.util 6418 :invoke :write [9 31] INFO [2019-12-17 12:34:18,889] jepsen worker 37 - jepsen.util 7777 :ok :write [12 18] INFO [2019-12-17 12:34:18,890] jepsen worker 37 - jepsen.util 7777 :invoke :cas [12 [3 23]] INFO [2019-12-17 12:34:18,896] jepsen worker 0 - jepsen.util 11520 :info :cas [0 [46 31]] :RequestCanceledException INFO [2019-12-17 12:34:18,898] jepsen worker 0 - jepsen.util 11610 :invoke :cas [0 [27 33]] INFO [2019-12-17 12:34:18,912] jepsen worker 9 - jepsen.util 8379 :info :write [3 34] :RequestCanceledException INFO [2019-12-17 12:34:18,912] jepsen worker 55 - jepsen.util 8965 :info :write [18 18] :RequestCanceledException INFO [2019-12-17 12:34:18,914] jepsen worker 55 - jepsen.util 9055 :invoke :read [18 nil] INFO [2019-12-17 12:34:18,921] jepsen worker 70 - jepsen.util 8530 :ok :write [23 42] INFO [2019-12-17 12:34:18,921] jepsen worker 70 - jepsen.util 8530 :invoke :write [23 15] INFO [2019-12-17 12:34:18,927] jepsen worker 85 - jepsen.util 11785 :info :cas [28 [10 38]] :RequestCanceledException INFO [2019-12-17 12:34:18,929] jepsen worker 9 - jepsen.util 8469 :invoke :read [3 nil] INFO [2019-12-17 12:34:18,929] jepsen worker 85 - jepsen.util 11875 :invoke :write [28 27] INFO [2019-12-17 12:34:18,947] jepsen worker 28 - jepsen.util 6418 :ok :write [9 31] INFO [2019-12-17 12:34:18,947] jepsen worker 28 - jepsen.util 6418 :invoke :read [9 nil] INFO [2019-12-17 12:34:18,978] jepsen worker 38 - jepsen.util 8498 :ok :read [12 18] INFO [2019-12-17 12:34:18,979] jepsen worker 38 - jepsen.util 8498 :invoke :write [12 30] INFO [2019-12-17 12:34:18,994] jepsen worker 9 - jepsen.util 8469 :ok :read [3 40] INFO [2019-12-17 12:34:18,994] jepsen worker 9 - jepsen.util 8469 :invoke :cas [3 [15 31]] INFO [2019-12-17 12:34:19,000] jepsen worker 63 - jepsen.util 6903 :fail :read [21 nil] :RequestTimeoutException INFO [2019-12-17 12:34:19,000] jepsen worker 63 - jepsen.util 6903 :invoke :write [21 48] INFO [2019-12-17 12:34:19,009] jepsen worker 28 - jepsen.util 6418 :ok :read [9 31] INFO [2019-12-17 12:34:19,010] jepsen worker 28 - jepsen.util 6418 :invoke :cas [9 [22 26]] INFO [2019-12-17 12:34:19,013] jepsen worker 48 - jepsen.util 10218 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:34:19,014] jepsen worker 48 - jepsen.util 10218 :invoke :write [16 20] INFO [2019-12-17 12:34:19,028] jepsen worker 22 - jepsen.util 11092 :info :cas [7 [7 27]] :RequestCanceledException INFO [2019-12-17 12:34:19,031] jepsen worker 22 - jepsen.util 11182 :invoke :cas [7 [9 29]] INFO [2019-12-17 12:34:19,032] jepsen worker 9 - jepsen.util 8469 :fail :cas [3 [15 31]] :ValueNotSwapFrom INFO [2019-12-17 12:34:19,032] jepsen worker 9 - jepsen.util 8469 :invoke :cas [3 [45 1]] INFO [2019-12-17 12:34:19,060] jepsen worker 88 - jepsen.util 8188 :info :write [29 31] :RequestCanceledException INFO [2019-12-17 12:34:19,062] jepsen worker 74 - jepsen.util 8354 :info :cas [24 [8 29]] :RequestCanceledException INFO [2019-12-17 12:34:19,064] jepsen worker 74 - jepsen.util 8444 :invoke :read [24 nil] INFO [2019-12-17 12:34:19,067] jepsen worker 88 - jepsen.util 8278 :invoke :read [29 nil] INFO [2019-12-17 12:34:19,074] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47076,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:34:19,075] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47076,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:34:19,075] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47080,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:34:19,075] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47080,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:34:19,087] jepsen worker 31 - jepsen.util 7231 :ok :write [10 5] INFO [2019-12-17 12:34:19,087] jepsen worker 31 - jepsen.util 7231 :invoke :write [10 44] INFO [2019-12-17 12:34:19,090] jepsen worker 19 - jepsen.util 8299 :ok :write [6 42] INFO [2019-12-17 12:34:19,090] jepsen worker 10 - jepsen.util 8290 :ok :write [3 28] INFO [2019-12-17 12:34:19,091] jepsen worker 19 - jepsen.util 8299 :invoke :write [6 27] INFO [2019-12-17 12:34:19,091] jepsen worker 10 - jepsen.util 8290 :invoke :read [3 nil] INFO [2019-12-17 12:34:19,098] jepsen worker 54 - jepsen.util 9054 :fail :cas [18 [5 23]] :ValueNotSwapFrom INFO [2019-12-17 12:34:19,098] jepsen worker 54 - jepsen.util 9054 :invoke :write [18 45] INFO [2019-12-17 12:34:19,109] jepsen worker 26 - jepsen.util 11546 :info :write [8 30] :RequestCanceledException INFO [2019-12-17 12:34:19,111] jepsen worker 26 - jepsen.util 11636 :invoke :read [8 nil] INFO [2019-12-17 12:34:19,115] jepsen worker 3 - jepsen.util 8013 :ok :write [1 23] INFO [2019-12-17 12:34:19,115] jepsen worker 3 - jepsen.util 8013 :invoke :cas [1 [49 35]] INFO [2019-12-17 12:34:19,125] jepsen worker 32 - jepsen.util 8312 :ok :read [10 5] INFO [2019-12-17 12:34:19,125] jepsen worker 32 - jepsen.util 8312 :invoke :read [10 nil] INFO [2019-12-17 12:34:19,129] jepsen worker 65 - jepsen.util 6725 :info :cas [21 [43 48]] :RequestCanceledException INFO [2019-12-17 12:34:19,129] jepsen worker 61 - jepsen.util 11941 :info :cas [20 [18 35]] :RequestCanceledException INFO [2019-12-17 12:34:19,131] jepsen worker 65 - jepsen.util 6815 :invoke :write [21 16] INFO [2019-12-17 12:34:19,131] jepsen worker 61 - jepsen.util 12031 :invoke :cas [20 [47 17]] INFO [2019-12-17 12:34:19,144] jepsen worker 84 - jepsen.util 10704 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:34:19,144] jepsen worker 84 - jepsen.util 10704 :invoke :write [28 0] INFO [2019-12-17 12:34:19,152] jepsen worker 72 - jepsen.util 8082 :ok :write [24 45] INFO [2019-12-17 12:34:19,156] jepsen worker 72 - jepsen.util 8082 :invoke :write [24 25] INFO [2019-12-17 12:34:19,157] jepsen worker 5 - jepsen.util 7655 :info :write [1 1] :RequestCanceledException INFO [2019-12-17 12:34:19,157] jepsen worker 20 - jepsen.util 7670 :info :cas [6 [8 7]] :RequestCanceledException INFO [2019-12-17 12:34:19,157] jepsen worker 27 - jepsen.util 7947 :fail :read [9 nil] :RequestTimeoutException INFO [2019-12-17 12:34:19,157] jepsen worker 27 - jepsen.util 7947 :invoke :write [9 34] INFO [2019-12-17 12:34:19,161] jepsen worker 20 - jepsen.util 7760 :invoke :write [6 38] INFO [2019-12-17 12:34:19,165] jepsen worker 5 - jepsen.util 7745 :invoke :write [1 6] INFO [2019-12-17 12:34:19,188] jepsen worker 71 - jepsen.util 8801 :fail :cas [23 [18 2]] :ValueNotSwapFrom INFO [2019-12-17 12:34:19,188] jepsen worker 71 - jepsen.util 8801 :invoke :write [23 20] INFO [2019-12-17 12:34:19,190] jepsen worker 63 - jepsen.util 6903 :ok :write [21 48] INFO [2019-12-17 12:34:19,190] jepsen worker 63 - jepsen.util 6903 :invoke :write [21 31] INFO [2019-12-17 12:34:19,197] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47102,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:34:19,198] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47102,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:34:19,198] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47103,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:34:19,198] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47103,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:34:19,199] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47105,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:34:19,199] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47105,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:34:19,212] jepsen worker 6 - jepsen.util 11616 :info :write [2 32] :RequestCanceledException INFO [2019-12-17 12:34:19,220] jepsen worker 6 - jepsen.util 11706 :invoke :cas [2 [29 39]] INFO [2019-12-17 12:34:19,223] jepsen worker 78 - jepsen.util 7908 :ok :read [26 26] INFO [2019-12-17 12:34:19,223] jepsen worker 78 - jepsen.util 7908 :invoke :read [26 nil] INFO [2019-12-17 12:34:19,227] jepsen worker 10 - jepsen.util 8290 :ok :read [3 28] INFO [2019-12-17 12:34:19,227] jepsen worker 10 - jepsen.util 8290 :invoke :write [3 41] INFO [2019-12-17 12:34:19,229] jepsen worker 40 - jepsen.util 11470 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:34:19,229] jepsen worker 56 - jepsen.util 9146 :info :write [18 20] :RequestCanceledException INFO [2019-12-17 12:34:19,230] jepsen worker 66 - jepsen.util 10866 :info :write [22 26] :RequestCanceledException INFO [2019-12-17 12:34:19,230] jepsen worker 40 - jepsen.util 11470 :invoke :cas [13 [27 25]] INFO [2019-12-17 12:34:19,234] jepsen worker 56 - jepsen.util 9236 :invoke :read [18 nil] INFO [2019-12-17 12:34:19,236] jepsen worker 44 - jepsen.util 11204 :info :write [14 34] :RequestCanceledException INFO [2019-12-17 12:34:19,241] jepsen worker 44 - jepsen.util 11294 :invoke :write [14 7] INFO [2019-12-17 12:34:19,244] jepsen worker 66 - jepsen.util 10956 :invoke :write [22 34] INFO [2019-12-17 12:34:19,251] jepsen worker 29 - jepsen.util 6689 :ok :read [9 31] INFO [2019-12-17 12:34:19,251] jepsen worker 29 - jepsen.util 6689 :invoke :read [9 nil] INFO [2019-12-17 12:34:19,252] jepsen worker 32 - jepsen.util 8312 :ok :read [10 5] INFO [2019-12-17 12:34:19,252] jepsen worker 32 - jepsen.util 8312 :invoke :read [10 nil] INFO [2019-12-17 12:34:19,260] jepsen worker 30 - jepsen.util 8130 :info :write [10 46] :RequestCanceledException INFO [2019-12-17 12:34:19,260] jepsen worker 47 - jepsen.util 8507 :info :cas [15 [23 25]] :RequestCanceledException INFO [2019-12-17 12:34:19,260] jepsen worker 73 - jepsen.util 7723 :fail :cas [24 [20 21]] :ValueNotSwapFrom INFO [2019-12-17 12:34:19,260] jepsen worker 73 - jepsen.util 7723 :invoke :write [24 27] INFO [2019-12-17 12:34:19,262] jepsen worker 47 - jepsen.util 8597 :invoke :write [15 21] INFO [2019-12-17 12:34:19,262] jepsen worker 30 - jepsen.util 8220 :invoke :cas [10 [42 34]] INFO [2019-12-17 12:34:19,293] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47113,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:34:19,293] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47113,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:34:19,295] jepsen worker 74 - jepsen.util 8444 :ok :read [24 45] INFO [2019-12-17 12:34:19,295] jepsen worker 74 - jepsen.util 8444 :invoke :read [24 nil] INFO [2019-12-17 12:34:19,295] jepsen worker 11 - jepsen.util 8111 :info :write [3 48] :RequestCanceledException INFO [2019-12-17 12:34:19,297] jepsen worker 11 - jepsen.util 8201 :invoke :cas [3 [24 24]] INFO [2019-12-17 12:34:19,307] jepsen worker 80 - jepsen.util 8090 :ok :write [26 11] INFO [2019-12-17 12:34:19,307] jepsen worker 80 - jepsen.util 8090 :invoke :read [26 nil] INFO [2019-12-17 12:34:19,310] jepsen worker 28 - jepsen.util 6418 :fail :cas [9 [22 26]] :ValueNotSwapFrom INFO [2019-12-17 12:34:19,310] jepsen worker 28 - jepsen.util 6418 :invoke :read [9 nil] INFO [2019-12-17 12:34:19,346] jepsen worker 89 - jepsen.util 8099 :ok :write [29 42] INFO [2019-12-17 12:34:19,346] jepsen worker 89 - jepsen.util 8099 :invoke :write [29 31] INFO [2019-12-17 12:34:19,353] jepsen worker 51 - jepsen.util 7701 :ok :read [17 22] INFO [2019-12-17 12:34:19,354] jepsen worker 51 - jepsen.util 7701 :invoke :read [17 nil] INFO [2019-12-17 12:34:19,358] jepsen worker 64 - jepsen.util 6724 :info :cas [21 [12 0]] :RequestCanceledException INFO [2019-12-17 12:34:19,358] jepsen worker 24 - jepsen.util 10014 :info :write [8 26] :RequestCanceledException INFO [2019-12-17 12:34:19,358] jepsen worker 15 - jepsen.util 10185 :info :write [5 43] :RequestCanceledException INFO [2019-12-17 12:34:19,359] jepsen worker 12 - jepsen.util 8742 :info :write [4 18] :RequestCanceledException INFO [2019-12-17 12:34:19,360] jepsen worker 64 - jepsen.util 6814 :invoke :cas [21 [1 31]] INFO [2019-12-17 12:34:19,360] jepsen worker 12 - jepsen.util 8832 :invoke :cas [4 [31 9]] INFO [2019-12-17 12:34:19,360] jepsen worker 24 - jepsen.util 10104 :invoke :cas [8 [12 26]] INFO [2019-12-17 12:34:19,361] jepsen worker 15 - jepsen.util 10275 :invoke :write [5 10] INFO [2019-12-17 12:34:19,373] jepsen worker 38 - jepsen.util 8498 :ok :write [12 30] INFO [2019-12-17 12:34:19,373] jepsen worker 38 - jepsen.util 8498 :invoke :cas [12 [2 34]] INFO [2019-12-17 12:34:19,382] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:34:19,382] jepsen nemesis - couchbase.util Rebalance check count: 365 INFO [2019-12-17 12:34:19,382] jepsen nemesis - couchbase.util Rebalance stuck count: 346 INFO [2019-12-17 12:34:19,395] jepsen worker 31 - jepsen.util 7231 :ok :write [10 44] INFO [2019-12-17 12:34:19,395] jepsen worker 31 - jepsen.util 7231 :invoke :write [10 32] INFO [2019-12-17 12:34:19,409] jepsen worker 65 - jepsen.util 6815 :ok :write [21 16] INFO [2019-12-17 12:34:19,409] jepsen worker 65 - jepsen.util 6815 :invoke :write [21 39] INFO [2019-12-17 12:34:19,429] jepsen worker 42 - jepsen.util 12012 :info :cas [14 [3 47]] :RequestCanceledException INFO [2019-12-17 12:34:19,434] jepsen worker 42 - jepsen.util 12102 :invoke :write [14 49] INFO [2019-12-17 12:34:19,474] jepsen worker 9 - jepsen.util 8469 :fail :cas [3 [45 1]] :ValueNotSwapFrom INFO [2019-12-17 12:34:19,475] jepsen worker 9 - jepsen.util 8469 :invoke :write [3 2] INFO [2019-12-17 12:34:19,480] jepsen worker 64 - jepsen.util 6814 :fail :cas [21 [1 31]] :ValueNotSwapFrom INFO [2019-12-17 12:34:19,480] jepsen worker 64 - jepsen.util 6814 :invoke :write [21 46] INFO [2019-12-17 12:34:19,495] jepsen worker 3 - jepsen.util 8013 :fail :cas [1 [49 35]] :ValueNotSwapFrom INFO [2019-12-17 12:34:19,496] jepsen worker 3 - jepsen.util 8013 :invoke :write [1 4] INFO [2019-12-17 12:34:19,503] jepsen worker 13 - jepsen.util 7663 :ok :read [4 21] INFO [2019-12-17 12:34:19,504] jepsen worker 13 - jepsen.util 7663 :invoke :write [4 31] INFO [2019-12-17 12:34:19,504] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47144,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:34:19,504] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47144,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:34:19,513] jepsen worker 17 - jepsen.util 11087 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:34:19,513] jepsen worker 17 - jepsen.util 11087 :invoke :cas [5 [23 14]] INFO [2019-12-17 12:34:19,514] jepsen worker 28 - jepsen.util 6418 :ok :read [9 31] INFO [2019-12-17 12:34:19,515] jepsen worker 28 - jepsen.util 6418 :invoke :cas [9 [28 14]] INFO [2019-12-17 12:34:19,527] jepsen worker 77 - jepsen.util 11417 :info :cas [25 [11 11]] :RequestCanceledException INFO [2019-12-17 12:34:19,528] jepsen worker 77 - jepsen.util 11507 :invoke :read [25 nil] INFO [2019-12-17 12:34:19,535] jepsen worker 65 - jepsen.util 6815 :ok :write [21 39] INFO [2019-12-17 12:34:19,536] jepsen worker 65 - jepsen.util 6815 :invoke :cas [21 [47 26]] INFO [2019-12-17 12:34:19,543] jepsen worker 8 - jepsen.util 10988 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:34:19,543] jepsen worker 8 - jepsen.util 10988 :invoke :cas [2 [25 27]] INFO [2019-12-17 12:34:19,558] jepsen worker 1 - jepsen.util 11341 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:34:19,558] jepsen worker 1 - jepsen.util 11341 :invoke :cas [0 [43 38]] INFO [2019-12-17 12:34:19,596] jepsen worker 7 - jepsen.util 11347 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:34:19,596] jepsen worker 43 - jepsen.util 10663 :info :cas [14 [28 8]] :RequestCanceledException INFO [2019-12-17 12:34:19,596] jepsen worker 7 - jepsen.util 11347 :invoke :cas [2 [30 9]] INFO [2019-12-17 12:34:19,604] jepsen worker 37 - jepsen.util 7777 :fail :cas [12 [3 23]] :ValueNotSwapFrom INFO [2019-12-17 12:34:19,605] jepsen worker 37 - jepsen.util 7777 :invoke :cas [12 [28 48]] INFO [2019-12-17 12:34:19,605] jepsen worker 43 - jepsen.util 10753 :invoke :write [14 25] INFO [2019-12-17 12:34:19,606] jepsen worker 51 - jepsen.util 7701 :ok :read [17 22] INFO [2019-12-17 12:34:19,607] jepsen worker 51 - jepsen.util 7701 :invoke :write [17 39] INFO [2019-12-17 12:34:19,614] jepsen worker 14 - jepsen.util 8384 :info :write [4 43] :RequestCanceledException INFO [2019-12-17 12:34:19,614] jepsen worker 53 - jepsen.util 7523 :info :write [17 38] :RequestCanceledException INFO [2019-12-17 12:34:19,618] jepsen worker 14 - jepsen.util 8474 :invoke :read [4 nil] INFO [2019-12-17 12:34:19,622] jepsen worker 28 - jepsen.util 6418 :fail :cas [9 [28 14]] :ValueNotSwapFrom INFO [2019-12-17 12:34:19,622] jepsen worker 28 - jepsen.util 6418 :invoke :read [9 nil] INFO [2019-12-17 12:34:19,622] jepsen worker 65 - jepsen.util 6815 :fail :cas [21 [47 26]] :ValueNotSwapFrom INFO [2019-12-17 12:34:19,622] jepsen worker 65 - jepsen.util 6815 :invoke :write [21 43] INFO [2019-12-17 12:34:19,623] jepsen worker 53 - jepsen.util 7613 :invoke :write [17 36] INFO [2019-12-17 12:34:19,623] jepsen worker 78 - jepsen.util 7908 :ok :read [26 11] INFO [2019-12-17 12:34:19,624] jepsen worker 78 - jepsen.util 7908 :invoke :write [26 21] INFO [2019-12-17 12:34:19,684] jepsen worker 75 - jepsen.util 10515 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:34:19,684] jepsen worker 75 - jepsen.util 10515 :invoke :write [25 4] INFO [2019-12-17 12:34:19,726] jepsen worker 28 - jepsen.util 6418 :ok :read [9 31] INFO [2019-12-17 12:34:19,726] jepsen worker 28 - jepsen.util 6418 :invoke :cas [9 [17 18]] INFO [2019-12-17 12:34:19,729] jepsen worker 2 - jepsen.util 10802 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:34:19,729] jepsen worker 2 - jepsen.util 10802 :invoke :read [0 nil] INFO [2019-12-17 12:34:19,738] jepsen worker 55 - jepsen.util 9055 :ok :read [18 42] INFO [2019-12-17 12:34:19,739] jepsen worker 55 - jepsen.util 9055 :invoke :read [18 nil] INFO [2019-12-17 12:34:19,743] jepsen worker 49 - jepsen.util 10489 :info :write [16 0] :RequestCanceledException INFO [2019-12-17 12:34:19,750] jepsen worker 49 - jepsen.util 10579 :invoke :write [16 19] INFO [2019-12-17 12:34:19,760] jepsen worker 88 - jepsen.util 8278 :ok :read [29 42] INFO [2019-12-17 12:34:19,760] jepsen worker 88 - jepsen.util 8278 :invoke :cas [29 [47 15]] INFO [2019-12-17 12:34:19,786] jepsen worker 72 - jepsen.util 8082 :ok :write [24 25] INFO [2019-12-17 12:34:19,787] jepsen worker 72 - jepsen.util 8082 :invoke :write [24 23] INFO [2019-12-17 12:34:19,787] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47170,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:34:19,787] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47170,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:34:19,787] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47172,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:34:19,787] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47172,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:34:19,796] jepsen worker 65 - jepsen.util 6815 :ok :write [21 43] INFO [2019-12-17 12:34:19,796] jepsen worker 65 - jepsen.util 6815 :invoke :read [21 nil] INFO [2019-12-17 12:34:19,838] jepsen worker 65 - jepsen.util 6815 :ok :read [21 43] INFO [2019-12-17 12:34:19,838] jepsen worker 65 - jepsen.util 6815 :invoke :write [21 9] INFO [2019-12-17 12:34:19,839] jepsen worker 19 - jepsen.util 8299 :ok :write [6 27] INFO [2019-12-17 12:34:19,839] jepsen worker 19 - jepsen.util 8299 :invoke :write [6 1] INFO [2019-12-17 12:34:19,860] jepsen worker 79 - jepsen.util 8449 :info :write [26 37] :RequestCanceledException INFO [2019-12-17 12:34:19,860] jepsen worker 87 - jepsen.util 7557 :fail :read [29 nil] :RequestTimeoutException INFO [2019-12-17 12:34:19,861] jepsen worker 87 - jepsen.util 7557 :invoke :cas [29 [12 1]] INFO [2019-12-17 12:34:19,864] jepsen worker 31 - jepsen.util 7231 :ok :write [10 32] INFO [2019-12-17 12:34:19,864] jepsen worker 31 - jepsen.util 7231 :invoke :cas [10 [34 42]] INFO [2019-12-17 12:34:19,866] jepsen worker 79 - jepsen.util 8539 :invoke :cas [26 [12 36]] INFO [2019-12-17 12:34:19,909] jepsen worker 64 - jepsen.util 6814 :ok :write [21 46] INFO [2019-12-17 12:34:19,909] jepsen worker 64 - jepsen.util 6814 :invoke :cas [21 [5 0]] INFO [2019-12-17 12:34:19,931] jepsen worker 37 - jepsen.util 7777 :fail :cas [12 [28 48]] :ValueNotSwapFrom INFO [2019-12-17 12:34:19,940] jepsen worker 56 - jepsen.util 9236 :ok :read [18 42] INFO [2019-12-17 12:34:19,943] jepsen worker 37 - jepsen.util 7777 :invoke :write [12 41] INFO [2019-12-17 12:34:19,943] jepsen worker 56 - jepsen.util 9236 :invoke :read [18 nil] INFO [2019-12-17 12:34:19,951] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47186,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:34:19,951] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47186,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:34:19,951] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47189,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:34:19,951] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47189,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:34:19,952] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47190,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:34:19,952] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47190,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:34:19,960] jepsen worker 69 - jepsen.util 8259 :info :write [23 8] :RequestCanceledException INFO [2019-12-17 12:34:19,962] jepsen worker 31 - jepsen.util 7231 :fail :cas [10 [34 42]] :ValueNotSwapFrom INFO [2019-12-17 12:34:19,962] jepsen worker 31 - jepsen.util 7231 :invoke :cas [10 [32 3]] INFO [2019-12-17 12:34:19,966] jepsen worker 69 - jepsen.util 8349 :invoke :cas [23 [40 13]] INFO [2019-12-17 12:34:19,967] jepsen worker 64 - jepsen.util 6814 :fail :cas [21 [5 0]] :ValueNotSwapFrom INFO [2019-12-17 12:34:19,967] jepsen worker 64 - jepsen.util 6814 :invoke :write [21 11] INFO [2019-12-17 12:34:19,989] jepsen worker 80 - jepsen.util 8090 :ok :read [26 11] INFO [2019-12-17 12:34:19,989] jepsen worker 80 - jepsen.util 8090 :invoke :cas [26 [26 1]] INFO [2019-12-17 12:34:19,992] jepsen worker 41 - jepsen.util 11741 :info :cas [13 [37 10]] :RequestCanceledException INFO [2019-12-17 12:34:19,992] jepsen worker 86 - jepsen.util 11786 :info :cas [28 [31 44]] :RequestCanceledException INFO [2019-12-17 12:34:19,994] jepsen worker 41 - jepsen.util 11831 :invoke :read [13 nil] INFO [2019-12-17 12:34:19,994] jepsen worker 86 - jepsen.util 11876 :invoke :write [28 44] INFO [2019-12-17 12:34:20,013] jepsen worker 39 - jepsen.util 10749 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:34:20,023] jepsen worker 39 - jepsen.util 10749 :invoke :read [13 nil] INFO [2019-12-17 12:34:20,029] jepsen worker 59 - jepsen.util 10769 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:34:20,030] jepsen worker 59 - jepsen.util 10769 :invoke :cas [19 [1 1]] INFO [2019-12-17 12:34:20,041] jepsen worker 64 - jepsen.util 6814 :ok :write [21 11] INFO [2019-12-17 12:34:20,042] jepsen worker 64 - jepsen.util 6814 :invoke :read [21 nil] INFO [2019-12-17 12:34:20,043] jepsen worker 81 - jepsen.util 10881 :info :write [27 23] :RequestCanceledException INFO [2019-12-17 12:34:20,047] jepsen worker 81 - jepsen.util 10971 :invoke :cas [27 [47 17]] INFO [2019-12-17 12:34:20,050] jepsen worker 72 - jepsen.util 8082 :ok :write [24 23] INFO [2019-12-17 12:34:20,051] jepsen worker 72 - jepsen.util 8082 :invoke :write [24 43] INFO [2019-12-17 12:34:20,177] jepsen worker 50 - jepsen.util 11840 :info :write [16 13] :RequestCanceledException INFO [2019-12-17 12:34:20,177] jepsen worker 68 - jepsen.util 10418 :info :write [22 26] :RequestCanceledException INFO [2019-12-17 12:34:20,177] jepsen worker 36 - jepsen.util 8496 :info :write [12 3] :RequestCanceledException INFO [2019-12-17 12:34:20,177] jepsen worker 83 - jepsen.util 11423 :info :cas [27 [3 43]] :RequestCanceledException INFO [2019-12-17 12:34:20,177] jepsen worker 23 - jepsen.util 10913 :info :cas [7 [45 1]] :RequestCanceledException INFO [2019-12-17 12:34:20,177] jepsen worker 82 - jepsen.util 11692 :info :cas [27 [44 17]] :RequestCanceledException INFO [2019-12-17 12:34:20,177] jepsen worker 19 - jepsen.util 8299 :ok :write [6 1] INFO [2019-12-17 12:34:20,178] jepsen worker 21 - jepsen.util 10461 :info :write [7 1] :RequestCanceledException INFO [2019-12-17 12:34:20,178] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47202,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:34:20,178] jepsen worker 16 - jepsen.util 12346 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:34:20,178] jepsen worker 19 - jepsen.util 8299 :invoke :write [6 19] INFO [2019-12-17 12:34:20,179] jepsen worker 50 - jepsen.util 11930 :invoke :write [16 11] INFO [2019-12-17 12:34:20,179] jepsen worker 36 - jepsen.util 8586 :invoke :cas [12 [19 1]] INFO [2019-12-17 12:34:20,179] jepsen worker 83 - jepsen.util 11513 :invoke :read [27 nil] INFO [2019-12-17 12:34:20,179] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47202,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:34:20,179] jepsen worker 16 - jepsen.util 12346 :invoke :write [5 46] INFO [2019-12-17 12:34:20,179] jepsen worker 68 - jepsen.util 10508 :invoke :read [22 nil] INFO [2019-12-17 12:34:20,179] jepsen worker 82 - jepsen.util 11782 :invoke :read [27 nil] INFO [2019-12-17 12:34:20,179] jepsen worker 23 - jepsen.util 11003 :invoke :cas [7 [45 12]] INFO [2019-12-17 12:34:20,179] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47205,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:34:20,180] jepsen worker 21 - jepsen.util 10551 :invoke :cas [7 [38 17]] INFO [2019-12-17 12:34:20,180] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47205,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:34:20,181] jepsen worker 12 - jepsen.util 8832 :fail :cas [4 [31 9]] :ValueNotSwapFrom INFO [2019-12-17 12:34:20,182] jepsen worker 12 - jepsen.util 8832 :invoke :cas [4 [3 30]] INFO [2019-12-17 12:34:20,194] jepsen worker 38 - jepsen.util 8498 :fail :cas [12 [2 34]] :ValueNotSwapFrom INFO [2019-12-17 12:34:20,194] jepsen worker 38 - jepsen.util 8498 :invoke :write [12 6] INFO [2019-12-17 12:34:20,214] jepsen worker 60 - jepsen.util 12030 :info :write [20 24] :RequestCanceledException INFO [2019-12-17 12:34:20,215] jepsen worker 60 - jepsen.util 12120 :invoke :read [20 nil] INFO [2019-12-17 12:34:20,229] jepsen worker 76 - jepsen.util 10966 :info :write [25 22] :RequestCanceledException INFO [2019-12-17 12:34:20,231] jepsen worker 76 - jepsen.util 11056 :invoke :cas [25 [35 2]] INFO [2019-12-17 12:34:20,238] jepsen worker 14 - jepsen.util 8474 :ok :read [4 21] INFO [2019-12-17 12:34:20,239] jepsen worker 14 - jepsen.util 8474 :invoke :write [4 46] INFO [2019-12-17 12:34:20,271] jepsen worker 55 - jepsen.util 9055 :ok :read [18 42] INFO [2019-12-17 12:34:20,271] jepsen worker 55 - jepsen.util 9055 :invoke :cas [18 [0 32]] INFO [2019-12-17 12:34:20,286] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47224,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:34:20,287] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47233,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:34:20,288] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47224,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:34:20,288] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47233,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:34:20,288] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47226,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:34:20,288] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47236,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:34:20,288] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47226,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:34:20,288] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47236,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:34:20,288] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47237,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:34:20,288] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47237,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:34:20,296] jepsen worker 34 - jepsen.util 11014 :info :cas [11 [28 48]] :RequestCanceledException INFO [2019-12-17 12:34:20,297] jepsen worker 34 - jepsen.util 11104 :invoke :read [11 nil] INFO [2019-12-17 12:34:20,329] jepsen worker 45 - jepsen.util 8235 :info :write [15 18] :RequestCanceledException INFO [2019-12-17 12:34:20,360] jepsen worker 4 - jepsen.util 7564 :info :write [1 21] :RequestCanceledException INFO [2019-12-17 12:34:20,396] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:34:20,404] jepsen nemesis - couchbase.util Rebalance check count: 366 INFO [2019-12-17 12:34:20,404] jepsen nemesis - couchbase.util Rebalance stuck count: 347 INFO [2019-12-17 12:34:20,405] jepsen worker 4 - jepsen.util 7654 :invoke :read [1 nil] INFO [2019-12-17 12:34:20,405] jepsen worker 45 - jepsen.util 8325 :invoke :read [15 nil] INFO [2019-12-17 12:34:20,428] jepsen worker 45 - jepsen.util 8325 :ok :read [15 17] INFO [2019-12-17 12:34:20,428] jepsen worker 45 - jepsen.util 8325 :invoke :write [15 35] INFO [2019-12-17 12:34:20,468] jepsen worker 87 - jepsen.util 7557 :fail :cas [29 [12 1]] :ValueNotSwapFrom INFO [2019-12-17 12:34:20,468] jepsen worker 87 - jepsen.util 7557 :invoke :cas [29 [7 1]] INFO [2019-12-17 12:34:20,506] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47245,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:34:20,506] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47245,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:34:20,506] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47249,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:34:20,506] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47249,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:34:20,513] jepsen worker 35 - jepsen.util 10835 :info :cas [11 [33 19]] :RequestCanceledException INFO [2019-12-17 12:34:20,514] jepsen worker 35 - jepsen.util 10925 :invoke :cas [11 [27 30]] INFO [2019-12-17 12:34:20,529] jepsen worker 67 - jepsen.util 11227 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:34:20,530] jepsen worker 67 - jepsen.util 11227 :invoke :cas [22 [38 22]] INFO [2019-12-17 12:34:20,580] jepsen worker 88 - jepsen.util 8278 :fail :cas [29 [47 15]] :ValueNotSwapFrom INFO [2019-12-17 12:34:20,580] jepsen worker 88 - jepsen.util 8278 :invoke :write [29 49] INFO [2019-12-17 12:34:20,581] jepsen worker 12 - jepsen.util 8832 :fail :cas [4 [3 30]] :ValueNotSwapFrom INFO [2019-12-17 12:34:20,581] jepsen worker 12 - jepsen.util 8832 :invoke :write [4 39] INFO [2019-12-17 12:34:20,592] jepsen worker 69 - jepsen.util 8349 :fail :cas [23 [40 13]] :ValueNotSwapFrom INFO [2019-12-17 12:34:20,592] jepsen worker 69 - jepsen.util 8349 :invoke :cas [23 [46 47]] INFO [2019-12-17 12:34:20,618] jepsen worker 56 - jepsen.util 9236 :ok :read [18 42] INFO [2019-12-17 12:34:20,618] jepsen worker 56 - jepsen.util 9236 :invoke :read [18 nil] INFO [2019-12-17 12:34:20,644] jepsen worker 79 - jepsen.util 8539 :fail :cas [26 [12 36]] :ValueNotSwapFrom INFO [2019-12-17 12:34:20,644] jepsen worker 79 - jepsen.util 8539 :invoke :cas [26 [11 46]] INFO [2019-12-17 12:34:20,657] jepsen worker 87 - jepsen.util 7557 :fail :cas [29 [7 1]] :ValueNotSwapFrom INFO [2019-12-17 12:34:20,657] jepsen worker 87 - jepsen.util 7557 :invoke :read [29 nil] INFO [2019-12-17 12:34:20,682] jepsen worker 80 - jepsen.util 8090 :fail :cas [26 [26 1]] :ValueNotSwapFrom INFO [2019-12-17 12:34:20,683] jepsen worker 80 - jepsen.util 8090 :invoke :write [26 25] INFO [2019-12-17 12:34:20,721] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47260,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:34:20,721] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47260,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:34:20,721] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47263,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:34:20,722] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47263,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:34:20,722] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47264,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:34:20,722] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47264,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:34:20,737] jepsen worker 55 - jepsen.util 9055 :fail :cas [18 [0 32]] :ValueNotSwapFrom INFO [2019-12-17 12:34:20,737] jepsen worker 55 - jepsen.util 9055 :invoke :read [18 nil] INFO [2019-12-17 12:34:20,749] jepsen worker 56 - jepsen.util 9236 :ok :read [18 42] INFO [2019-12-17 12:34:20,750] jepsen worker 56 - jepsen.util 9236 :invoke :read [18 nil] INFO [2019-12-17 12:34:20,806] jepsen worker 4 - jepsen.util 7654 :ok :read [1 23] INFO [2019-12-17 12:34:20,806] jepsen worker 4 - jepsen.util 7654 :invoke :read [1 nil] INFO [2019-12-17 12:34:20,829] jepsen worker 33 - jepsen.util 11013 :info :write [11 18] :RequestCanceledException INFO [2019-12-17 12:34:20,829] jepsen worker 33 - jepsen.util 11103 :invoke :write [11 21] INFO [2019-12-17 12:34:20,841] jepsen worker 36 - jepsen.util 8586 :fail :cas [12 [19 1]] :ValueNotSwapFrom INFO [2019-12-17 12:34:20,841] jepsen worker 36 - jepsen.util 8586 :invoke :cas [12 [19 2]] INFO [2019-12-17 12:34:20,895] jepsen worker 46 - jepsen.util 8326 :info :write [15 10] :RequestCanceledException INFO [2019-12-17 12:34:20,896] jepsen worker 46 - jepsen.util 8416 :invoke :read [15 nil] INFO [2019-12-17 12:34:20,903] jepsen worker 36 - jepsen.util 8586 :fail :cas [12 [19 2]] :ValueNotSwapFrom INFO [2019-12-17 12:34:20,904] jepsen worker 36 - jepsen.util 8586 :invoke :cas [12 [36 45]] INFO [2019-12-17 12:34:20,908] jepsen worker 87 - jepsen.util 7557 :ok :read [29 42] INFO [2019-12-17 12:34:20,908] jepsen worker 87 - jepsen.util 7557 :invoke :write [29 37] INFO [2019-12-17 12:34:20,913] jepsen worker 18 - jepsen.util 8298 :info :write [6 40] :RequestCanceledException INFO [2019-12-17 12:34:20,914] jepsen worker 18 - jepsen.util 8388 :invoke :cas [6 [36 37]] INFO [2019-12-17 12:34:20,929] jepsen worker 62 - jepsen.util 11402 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:34:20,929] jepsen worker 57 - jepsen.util 11037 :info :write [19 44] :RequestCanceledException INFO [2019-12-17 12:34:20,929] jepsen worker 62 - jepsen.util 11402 :invoke :cas [20 [31 19]] INFO [2019-12-17 12:34:20,930] jepsen worker 57 - jepsen.util 11127 :invoke :cas [19 [1 13]] INFO [2019-12-17 12:34:20,941] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47273,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:34:20,941] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47273,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:34:20,941] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47276,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:34:20,941] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47276,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:34:20,943] jepsen worker 25 - jepsen.util 11725 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:34:20,943] jepsen worker 58 - jepsen.util 11938 :info :write [19 42] :RequestCanceledException INFO [2019-12-17 12:34:20,943] jepsen worker 25 - jepsen.util 11725 :invoke :cas [8 [31 15]] INFO [2019-12-17 12:34:20,943] jepsen worker 58 - jepsen.util 12028 :invoke :write [19 17] INFO [2019-12-17 12:34:20,952] jepsen worker 69 - jepsen.util 8349 :fail :cas [23 [46 47]] :ValueNotSwapFrom INFO [2019-12-17 12:34:20,952] jepsen worker 69 - jepsen.util 8349 :invoke :write [23 31] INFO [2019-12-17 12:34:20,960] jepsen worker 0 - jepsen.util 11610 :info :cas [0 [27 33]] :RequestCanceledException INFO [2019-12-17 12:34:20,961] jepsen worker 0 - jepsen.util 11700 :invoke :read [0 nil] INFO [2019-12-17 12:34:21,007] jepsen worker 5 - jepsen.util 7745 :ok :write [1 6] INFO [2019-12-17 12:34:21,007] jepsen worker 5 - jepsen.util 7745 :invoke :read [1 nil] INFO [2019-12-17 12:34:21,014] jepsen worker 85 - jepsen.util 11875 :info :write [28 27] :RequestCanceledException INFO [2019-12-17 12:34:21,015] jepsen worker 85 - jepsen.util 11965 :invoke :read [28 nil] INFO [2019-12-17 12:34:21,029] jepsen worker 70 - jepsen.util 8530 :info :write [23 15] :RequestCanceledException INFO [2019-12-17 12:34:21,030] jepsen worker 70 - jepsen.util 8620 :invoke :read [23 nil] INFO [2019-12-17 12:34:21,056] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47289,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:34:21,056] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47289,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:34:21,056] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47293,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:34:21,056] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47293,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:34:21,087] jepsen worker 38 - jepsen.util 8498 :ok :write [12 6] INFO [2019-12-17 12:34:21,087] jepsen worker 38 - jepsen.util 8498 :invoke :cas [12 [20 47]] INFO [2019-12-17 12:34:21,088] jepsen worker 70 - jepsen.util 8620 :ok :read [23 15] INFO [2019-12-17 12:34:21,088] jepsen worker 70 - jepsen.util 8620 :invoke :cas [23 [31 2]] INFO [2019-12-17 12:34:21,095] jepsen worker 52 - jepsen.util 8152 :info :cas [17 [0 36]] :RequestCanceledException INFO [2019-12-17 12:34:21,096] jepsen worker 52 - jepsen.util 8242 :invoke :read [17 nil] INFO [2019-12-17 12:34:21,114] jepsen worker 48 - jepsen.util 10218 :info :write [16 20] :RequestCanceledException INFO [2019-12-17 12:34:21,115] jepsen worker 48 - jepsen.util 10308 :invoke :write [16 40] INFO [2019-12-17 12:34:21,129] jepsen worker 26 - jepsen.util 11636 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:34:21,129] jepsen worker 26 - jepsen.util 11636 :invoke :cas [8 [40 42]] INFO [2019-12-17 12:34:21,129] jepsen worker 22 - jepsen.util 11182 :info :cas [7 [9 29]] :RequestCanceledException INFO [2019-12-17 12:34:21,130] jepsen worker 22 - jepsen.util 11272 :invoke :read [7 nil] INFO [2019-12-17 12:34:21,195] jepsen worker 4 - jepsen.util 7654 :ok :read [1 6] INFO [2019-12-17 12:34:21,195] jepsen worker 4 - jepsen.util 7654 :invoke :read [1 nil] INFO [2019-12-17 12:34:21,213] jepsen worker 61 - jepsen.util 12031 :info :cas [20 [47 17]] :RequestCanceledException INFO [2019-12-17 12:34:21,214] jepsen worker 61 - jepsen.util 12121 :invoke :write [20 41] INFO [2019-12-17 12:34:21,229] jepsen worker 54 - jepsen.util 9054 :info :write [18 45] :RequestCanceledException INFO [2019-12-17 12:34:21,229] jepsen worker 20 - jepsen.util 7760 :info :write [6 38] :RequestCanceledException INFO [2019-12-17 12:34:21,230] jepsen worker 54 - jepsen.util 9144 :invoke :cas [18 [11 6]] INFO [2019-12-17 12:34:21,230] jepsen worker 20 - jepsen.util 7850 :invoke :write [6 26] INFO [2019-12-17 12:34:21,234] jepsen worker 51 - jepsen.util 7701 :ok :write [17 39] INFO [2019-12-17 12:34:21,235] jepsen worker 51 - jepsen.util 7701 :invoke :cas [17 [19 13]] INFO [2019-12-17 12:34:21,243] jepsen worker 84 - jepsen.util 10704 :info :write [28 0] :RequestCanceledException INFO [2019-12-17 12:34:21,244] jepsen worker 84 - jepsen.util 10794 :invoke :cas [28 [25 48]] INFO [2019-12-17 12:34:21,260] jepsen worker 27 - jepsen.util 7947 :info :write [9 34] :RequestCanceledException INFO [2019-12-17 12:34:21,260] jepsen worker 6 - jepsen.util 11706 :info :cas [2 [29 39]] :RequestCanceledException INFO [2019-12-17 12:34:21,261] jepsen worker 6 - jepsen.util 11796 :invoke :write [2 43] INFO [2019-12-17 12:34:21,261] jepsen worker 27 - jepsen.util 8037 :invoke :read [9 nil] INFO [2019-12-17 12:34:21,270] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47306,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:34:21,270] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47312,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:34:21,271] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47306,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:34:21,271] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47312,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:34:21,271] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47307,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:34:21,271] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47315,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:34:21,271] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47307,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:34:21,271] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47315,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:34:21,286] jepsen worker 4 - jepsen.util 7654 :ok :read [1 6] INFO [2019-12-17 12:34:21,286] jepsen worker 4 - jepsen.util 7654 :invoke :write [1 13] INFO [2019-12-17 12:34:21,295] jepsen worker 44 - jepsen.util 11294 :info :write [14 7] :RequestCanceledException INFO [2019-12-17 12:34:21,295] jepsen worker 71 - jepsen.util 8801 :info :write [23 20] :RequestCanceledException INFO [2019-12-17 12:34:21,295] jepsen worker 63 - jepsen.util 6903 :info :write [21 31] :RequestCanceledException INFO [2019-12-17 12:34:21,296] jepsen worker 44 - jepsen.util 11384 :invoke :read [14 nil] INFO [2019-12-17 12:34:21,296] jepsen worker 63 - jepsen.util 6993 :invoke :read [21 nil] INFO [2019-12-17 12:34:21,296] jepsen worker 71 - jepsen.util 8891 :invoke :read [23 nil] INFO [2019-12-17 12:34:21,313] jepsen worker 66 - jepsen.util 10956 :info :write [22 34] :RequestCanceledException INFO [2019-12-17 12:34:21,314] jepsen worker 66 - jepsen.util 11046 :invoke :cas [22 [8 3]] INFO [2019-12-17 12:34:21,329] jepsen worker 40 - jepsen.util 11470 :info :cas [13 [27 25]] :RequestCanceledException INFO [2019-12-17 12:34:21,330] jepsen worker 10 - jepsen.util 8290 :info :write [3 41] :RequestCanceledException INFO [2019-12-17 12:34:21,330] jepsen worker 32 - jepsen.util 8312 :fail :read [10 nil] :RequestTimeoutException INFO [2019-12-17 12:34:21,330] jepsen worker 47 - jepsen.util 8597 :info :write [15 21] :RequestCanceledException INFO [2019-12-17 12:34:21,330] jepsen worker 32 - jepsen.util 8312 :invoke :write [10 20] INFO [2019-12-17 12:34:21,330] jepsen worker 40 - jepsen.util 11560 :invoke :read [13 nil] INFO [2019-12-17 12:34:21,331] jepsen worker 47 - jepsen.util 8687 :invoke :read [15 nil] INFO [2019-12-17 12:34:21,331] jepsen worker 10 - jepsen.util 8380 :invoke :write [3 46] INFO [2019-12-17 12:34:21,343] jepsen worker 29 - jepsen.util 6689 :fail :read [9 nil] :RequestTimeoutException INFO [2019-12-17 12:34:21,343] jepsen worker 30 - jepsen.util 8220 :info :cas [10 [42 34]] :RequestCanceledException INFO [2019-12-17 12:34:21,343] jepsen worker 73 - jepsen.util 7723 :info :write [24 27] :RequestCanceledException INFO [2019-12-17 12:34:21,343] jepsen worker 74 - jepsen.util 8444 :fail :read [24 nil] :RequestTimeoutException INFO [2019-12-17 12:34:21,343] jepsen worker 29 - jepsen.util 6689 :invoke :read [9 nil] INFO [2019-12-17 12:34:21,344] jepsen worker 74 - jepsen.util 8444 :invoke :read [24 nil] INFO [2019-12-17 12:34:21,344] jepsen worker 30 - jepsen.util 8310 :invoke :read [10 nil] INFO [2019-12-17 12:34:21,344] jepsen worker 73 - jepsen.util 7813 :invoke :write [24 19] INFO [2019-12-17 12:34:21,345] jepsen worker 38 - jepsen.util 8498 :fail :cas [12 [20 47]] :ValueNotSwapFrom INFO [2019-12-17 12:34:21,345] jepsen worker 38 - jepsen.util 8498 :invoke :cas [12 [46 42]] INFO [2019-12-17 12:34:21,361] jepsen worker 11 - jepsen.util 8201 :info :cas [3 [24 24]] :RequestCanceledException INFO [2019-12-17 12:34:21,362] jepsen worker 11 - jepsen.util 8291 :invoke :read [3 nil] INFO [2019-12-17 12:34:21,372] jepsen worker 71 - jepsen.util 8891 :ok :read [23 15] INFO [2019-12-17 12:34:21,372] jepsen worker 71 - jepsen.util 8891 :invoke :write [23 11] INFO [2019-12-17 12:34:21,385] jepsen worker 38 - jepsen.util 8498 :fail :cas [12 [46 42]] :ValueNotSwapFrom INFO [2019-12-17 12:34:21,385] jepsen worker 38 - jepsen.util 8498 :invoke :read [12 nil] INFO [2019-12-17 12:34:21,413] jepsen worker 24 - jepsen.util 10104 :info :cas [8 [12 26]] :RequestCanceledException INFO [2019-12-17 12:34:21,414] jepsen worker 24 - jepsen.util 10194 :invoke :cas [8 [28 29]] INFO [2019-12-17 12:34:21,418] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:34:21,418] jepsen nemesis - couchbase.util Rebalance check count: 367 INFO [2019-12-17 12:34:21,418] jepsen nemesis - couchbase.util Rebalance stuck count: 348 INFO [2019-12-17 12:34:21,421] jepsen worker 55 - jepsen.util 9055 :ok :read [18 42] INFO [2019-12-17 12:34:21,422] jepsen worker 55 - jepsen.util 9055 :invoke :cas [18 [28 15]] INFO [2019-12-17 12:34:21,430] jepsen worker 15 - jepsen.util 10275 :info :write [5 10] :RequestCanceledException INFO [2019-12-17 12:34:21,430] jepsen worker 15 - jepsen.util 10365 :invoke :cas [5 [16 38]] INFO [2019-12-17 12:34:21,443] jepsen worker 89 - jepsen.util 8099 :info :write [29 31] :RequestCanceledException INFO [2019-12-17 12:34:21,444] jepsen worker 89 - jepsen.util 8189 :invoke :read [29 nil] INFO [2019-12-17 12:34:21,449] jepsen worker 38 - jepsen.util 8498 :ok :read [12 6] INFO [2019-12-17 12:34:21,449] jepsen worker 38 - jepsen.util 8498 :invoke :cas [12 [34 11]] INFO [2019-12-17 12:34:21,456] jepsen worker 56 - jepsen.util 9236 :ok :read [18 42] INFO [2019-12-17 12:34:21,456] jepsen worker 56 - jepsen.util 9236 :invoke :cas [18 [0 14]] INFO [2019-12-17 12:34:21,489] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47346,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:34:21,489] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47346,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:34:21,514] jepsen worker 52 - jepsen.util 8242 :ok :read [17 39] INFO [2019-12-17 12:34:21,514] jepsen worker 17 - jepsen.util 11087 :info :cas [5 [23 14]] :RequestCanceledException INFO [2019-12-17 12:34:21,514] jepsen worker 13 - jepsen.util 7663 :info :write [4 31] :RequestCanceledException INFO [2019-12-17 12:34:21,514] jepsen worker 52 - jepsen.util 8242 :invoke :write [17 37] INFO [2019-12-17 12:34:21,515] jepsen worker 13 - jepsen.util 7753 :invoke :write [4 22] INFO [2019-12-17 12:34:21,515] jepsen worker 17 - jepsen.util 11177 :invoke :read [5 nil] INFO [2019-12-17 12:34:21,521] jepsen worker 36 - jepsen.util 8586 :fail :cas [12 [36 45]] :ValueNotSwapFrom INFO [2019-12-17 12:34:21,522] jepsen worker 36 - jepsen.util 8586 :invoke :cas [12 [34 38]] INFO [2019-12-17 12:34:21,542] jepsen worker 51 - jepsen.util 7701 :fail :cas [17 [19 13]] :ValueNotSwapFrom INFO [2019-12-17 12:34:21,542] jepsen worker 42 - jepsen.util 12102 :info :write [14 49] :RequestCanceledException INFO [2019-12-17 12:34:21,542] jepsen worker 3 - jepsen.util 8013 :info :write [1 4] :RequestCanceledException INFO [2019-12-17 12:34:21,543] jepsen worker 51 - jepsen.util 7701 :invoke :read [17 nil] INFO [2019-12-17 12:34:21,544] jepsen worker 42 - jepsen.util 12192 :invoke :write [14 28] INFO [2019-12-17 12:34:21,544] jepsen worker 3 - jepsen.util 8103 :invoke :cas [1 [29 22]] INFO [2019-12-17 12:34:21,552] jepsen worker 55 - jepsen.util 9055 :fail :cas [18 [28 15]] :ValueNotSwapFrom INFO [2019-12-17 12:34:21,553] jepsen worker 55 - jepsen.util 9055 :invoke :cas [18 [39 36]] INFO [2019-12-17 12:34:21,560] jepsen worker 77 - jepsen.util 11507 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:34:21,560] jepsen worker 1 - jepsen.util 11341 :info :cas [0 [43 38]] :RequestCanceledException INFO [2019-12-17 12:34:21,560] jepsen worker 77 - jepsen.util 11507 :invoke :cas [25 [21 38]] INFO [2019-12-17 12:34:21,562] jepsen worker 5 - jepsen.util 7745 :ok :read [1 6] INFO [2019-12-17 12:34:21,563] jepsen worker 5 - jepsen.util 7745 :invoke :cas [1 [7 18]] INFO [2019-12-17 12:34:21,563] jepsen worker 1 - jepsen.util 11431 :invoke :write [0 49] INFO [2019-12-17 12:34:21,596] jepsen worker 9 - jepsen.util 8469 :info :write [3 2] :RequestCanceledException INFO [2019-12-17 12:34:21,596] jepsen worker 9 - jepsen.util 8559 :invoke :write [3 32] INFO [2019-12-17 12:34:21,622] jepsen worker 70 - jepsen.util 8620 :fail :cas [23 [31 2]] :ValueNotSwapFrom INFO [2019-12-17 12:34:21,623] jepsen worker 70 - jepsen.util 8620 :invoke :read [23 nil] INFO [2019-12-17 12:34:21,629] jepsen worker 43 - jepsen.util 10753 :info :write [14 25] :RequestCanceledException INFO [2019-12-17 12:34:21,630] jepsen worker 43 - jepsen.util 10843 :invoke :cas [14 [21 29]] INFO [2019-12-17 12:34:21,639] jepsen worker 80 - jepsen.util 8090 :ok :write [26 25] INFO [2019-12-17 12:34:21,639] jepsen worker 80 - jepsen.util 8090 :invoke :cas [26 [21 49]] INFO [2019-12-17 12:34:21,643] jepsen worker 78 - jepsen.util 7908 :info :write [26 21] :RequestCanceledException INFO [2019-12-17 12:34:21,643] jepsen worker 8 - jepsen.util 10988 :info :cas [2 [25 27]] :RequestCanceledException INFO [2019-12-17 12:34:21,644] jepsen worker 8 - jepsen.util 11078 :invoke :write [2 17] INFO [2019-12-17 12:34:21,644] jepsen worker 78 - jepsen.util 7998 :invoke :write [26 21] INFO [2019-12-17 12:34:21,672] jepsen worker 9 - jepsen.util 8559 :ok :write [3 32] INFO [2019-12-17 12:34:21,673] jepsen worker 9 - jepsen.util 8559 :invoke :write [3 22] INFO [2019-12-17 12:34:21,695] jepsen worker 7 - jepsen.util 11347 :info :cas [2 [30 9]] :RequestCanceledException INFO [2019-12-17 12:34:21,696] jepsen worker 7 - jepsen.util 11437 :invoke :read [2 nil] INFO [2019-12-17 12:34:21,706] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47369,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:34:21,706] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47369,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:34:21,706] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47372,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:34:21,706] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47372,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:34:21,711] jepsen worker 53 - jepsen.util 7613 :info :write [17 36] :RequestCanceledException INFO [2019-12-17 12:34:21,712] jepsen worker 53 - jepsen.util 7703 :invoke :read [17 nil] INFO [2019-12-17 12:34:21,740] jepsen worker 9 - jepsen.util 8559 :ok :write [3 22] INFO [2019-12-17 12:34:21,740] jepsen worker 9 - jepsen.util 8559 :invoke :write [3 48] INFO [2019-12-17 12:34:21,743] jepsen worker 75 - jepsen.util 10515 :info :write [25 4] :RequestCanceledException INFO [2019-12-17 12:34:21,744] jepsen worker 75 - jepsen.util 10605 :invoke :write [25 4] INFO [2019-12-17 12:34:21,788] jepsen worker 54 - jepsen.util 9144 :fail :cas [18 [11 6]] :ValueNotSwapFrom INFO [2019-12-17 12:34:21,789] jepsen worker 54 - jepsen.util 9144 :invoke :read [18 nil] INFO [2019-12-17 12:34:21,789] jepsen worker 51 - jepsen.util 7701 :ok :read [17 39] INFO [2019-12-17 12:34:21,789] jepsen worker 51 - jepsen.util 7701 :invoke :read [17 nil] INFO [2019-12-17 12:34:21,796] jepsen worker 28 - jepsen.util 6418 :info :cas [9 [17 18]] :RequestCanceledException INFO [2019-12-17 12:34:21,797] jepsen worker 28 - jepsen.util 6508 :invoke :write [9 20] INFO [2019-12-17 12:34:21,822] jepsen worker 9 - jepsen.util 8559 :ok :write [3 48] INFO [2019-12-17 12:34:21,822] jepsen worker 9 - jepsen.util 8559 :invoke :write [3 23] INFO [2019-12-17 12:34:21,829] jepsen worker 49 - jepsen.util 10579 :info :write [16 19] :RequestCanceledException INFO [2019-12-17 12:34:21,829] jepsen worker 2 - jepsen.util 10802 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:34:21,830] jepsen worker 2 - jepsen.util 10802 :invoke :cas [0 [22 36]] INFO [2019-12-17 12:34:21,831] jepsen worker 49 - jepsen.util 10669 :invoke :read [16 nil] INFO [2019-12-17 12:34:21,888] jepsen worker 31 - jepsen.util 7231 :info :cas [10 [32 3]] :SyncWriteAmbiguous INFO [2019-12-17 12:34:21,889] jepsen worker 31 - jepsen.util 7321 :invoke :read [10 nil] INFO [2019-12-17 12:34:21,896] jepsen worker 65 - jepsen.util 6815 :info :write [21 9] :RequestCanceledException INFO [2019-12-17 12:34:21,897] jepsen worker 65 - jepsen.util 6905 :invoke :write [21 39] INFO [2019-12-17 12:34:21,906] jepsen worker 18 - jepsen.util 8388 :fail :cas [6 [36 37]] :ValueNotSwapFrom INFO [2019-12-17 12:34:21,906] jepsen worker 18 - jepsen.util 8388 :invoke :cas [6 [15 47]] INFO [2019-12-17 12:34:21,916] jepsen worker 53 - jepsen.util 7703 :ok :read [17 39] INFO [2019-12-17 12:34:21,916] jepsen worker 53 - jepsen.util 7703 :invoke :read [17 nil] INFO [2019-12-17 12:34:21,922] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47388,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:34:21,922] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47388,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:34:21,923] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47392,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:34:21,923] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47392,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:34:21,939] jepsen worker 11 - jepsen.util 8291 :ok :read [3 48] INFO [2019-12-17 12:34:21,939] jepsen worker 11 - jepsen.util 8291 :invoke :cas [3 [27 46]] INFO [2019-12-17 12:34:21,952] jepsen worker 31 - jepsen.util 7321 :ok :read [10 32] INFO [2019-12-17 12:34:21,952] jepsen worker 31 - jepsen.util 7321 :invoke :write [10 23] INFO [2019-12-17 12:34:21,953] jepsen worker 56 - jepsen.util 9236 :fail :cas [18 [0 14]] :ValueNotSwapFrom INFO [2019-12-17 12:34:21,953] jepsen worker 56 - jepsen.util 9236 :invoke :cas [18 [18 12]] INFO [2019-12-17 12:34:21,960] jepsen worker 18 - jepsen.util 8388 :fail :cas [6 [15 47]] :ValueNotSwapFrom INFO [2019-12-17 12:34:21,960] jepsen worker 18 - jepsen.util 8388 :invoke :cas [6 [17 25]] INFO [2019-12-17 12:34:21,988] jepsen worker 55 - jepsen.util 9055 :fail :cas [18 [39 36]] :ValueNotSwapFrom INFO [2019-12-17 12:34:21,988] jepsen worker 55 - jepsen.util 9055 :invoke :read [18 nil] INFO [2019-12-17 12:34:22,001] jepsen worker 87 - jepsen.util 7557 :ok :write [29 37] INFO [2019-12-17 12:34:22,001] jepsen worker 87 - jepsen.util 7557 :invoke :write [29 5] INFO [2019-12-17 12:34:22,006] jepsen worker 65 - jepsen.util 6905 :ok :write [21 39] INFO [2019-12-17 12:34:22,006] jepsen worker 65 - jepsen.util 6905 :invoke :cas [21 [45 7]] INFO [2019-12-17 12:34:22,006] jepsen worker 11 - jepsen.util 8291 :fail :cas [3 [27 46]] :ValueNotSwapFrom INFO [2019-12-17 12:34:22,006] jepsen worker 11 - jepsen.util 8291 :invoke :read [3 nil] INFO [2019-12-17 12:34:22,023] jepsen worker 65 - jepsen.util 6905 :fail :cas [21 [45 7]] :ValueNotSwapFrom INFO [2019-12-17 12:34:22,023] jepsen worker 65 - jepsen.util 6905 :invoke :read [21 nil] INFO [2019-12-17 12:34:22,029] jepsen worker 41 - jepsen.util 11831 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:34:22,029] jepsen worker 41 - jepsen.util 11831 :invoke :write [13 44] INFO [2019-12-17 12:34:22,030] jepsen worker 37 - jepsen.util 7777 :info :write [12 41] :RequestCanceledException INFO [2019-12-17 12:34:22,031] jepsen worker 37 - jepsen.util 7867 :invoke :cas [12 [39 17]] INFO [2019-12-17 12:34:22,039] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47405,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:34:22,039] jepsen worker 72 - jepsen.util 8082 :info :write [24 43] :SyncWriteAmbiguous INFO [2019-12-17 12:34:22,039] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47405,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:34:22,040] jepsen worker 72 - jepsen.util 8172 :invoke :read [24 nil] INFO [2019-12-17 12:34:22,040] jepsen worker 65 - jepsen.util 6905 :ok :read [21 39] INFO [2019-12-17 12:34:22,041] jepsen worker 65 - jepsen.util 6905 :invoke :read [21 nil] INFO [2019-12-17 12:34:22,060] jepsen worker 89 - jepsen.util 8189 :ok :read [29 42] INFO [2019-12-17 12:34:22,060] jepsen worker 89 - jepsen.util 8189 :invoke :read [29 nil] INFO [2019-12-17 12:34:22,069] jepsen worker 19 - jepsen.util 8299 :info :write [6 19] :SyncWriteAmbiguous INFO [2019-12-17 12:34:22,070] jepsen worker 65 - jepsen.util 6905 :ok :read [21 39] INFO [2019-12-17 12:34:22,070] jepsen worker 65 - jepsen.util 6905 :invoke :write [21 32] INFO [2019-12-17 12:34:22,070] jepsen worker 19 - jepsen.util 8389 :invoke :write [6 39] INFO [2019-12-17 12:34:22,083] jepsen worker 80 - jepsen.util 8090 :fail :cas [26 [21 49]] :ValueNotSwapFrom INFO [2019-12-17 12:34:22,083] jepsen worker 80 - jepsen.util 8090 :invoke :cas [26 [8 7]] INFO [2019-12-17 12:34:22,084] jepsen worker 63 - jepsen.util 6993 :ok :read [21 39] INFO [2019-12-17 12:34:22,084] jepsen worker 63 - jepsen.util 6993 :invoke :read [21 nil] INFO [2019-12-17 12:34:22,095] jepsen worker 86 - jepsen.util 11876 :info :write [28 44] :RequestCanceledException INFO [2019-12-17 12:34:22,096] jepsen worker 86 - jepsen.util 11966 :invoke :cas [28 [12 17]] INFO [2019-12-17 12:34:22,114] jepsen worker 39 - jepsen.util 10749 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:34:22,114] jepsen worker 81 - jepsen.util 10971 :info :cas [27 [47 17]] :RequestCanceledException INFO [2019-12-17 12:34:22,114] jepsen worker 39 - jepsen.util 10749 :invoke :cas [13 [30 12]] INFO [2019-12-17 12:34:22,115] jepsen worker 81 - jepsen.util 11061 :invoke :cas [27 [43 40]] INFO [2019-12-17 12:34:22,129] jepsen worker 64 - jepsen.util 6814 :fail :read [21 nil] :RequestTimeoutException INFO [2019-12-17 12:34:22,129] jepsen worker 59 - jepsen.util 10769 :info :cas [19 [1 1]] :RequestCanceledException INFO [2019-12-17 12:34:22,129] jepsen worker 64 - jepsen.util 6814 :invoke :cas [21 [31 23]] INFO [2019-12-17 12:34:22,130] jepsen worker 51 - jepsen.util 7701 :ok :read [17 39] INFO [2019-12-17 12:34:22,130] jepsen worker 59 - jepsen.util 10859 :invoke :write [19 33] INFO [2019-12-17 12:34:22,130] jepsen worker 51 - jepsen.util 7701 :invoke :read [17 nil] INFO [2019-12-17 12:34:22,143] jepsen worker 29 - jepsen.util 6689 :ok :read [9 31] INFO [2019-12-17 12:34:22,143] jepsen worker 29 - jepsen.util 6689 :invoke :read [9 nil] INFO [2019-12-17 12:34:22,147] jepsen worker 55 - jepsen.util 9055 :ok :read [18 42] INFO [2019-12-17 12:34:22,147] jepsen worker 55 - jepsen.util 9055 :invoke :write [18 4] INFO [2019-12-17 12:34:22,150] jepsen worker 46 - jepsen.util 8416 :ok :read [15 17] INFO [2019-12-17 12:34:22,151] jepsen worker 46 - jepsen.util 8416 :invoke :cas [15 [7 20]] INFO [2019-12-17 12:34:22,157] jepsen worker 53 - jepsen.util 7703 :ok :read [17 39] INFO [2019-12-17 12:34:22,158] jepsen worker 53 - jepsen.util 7703 :invoke :cas [17 [12 15]] INFO [2019-12-17 12:34:22,165] jepsen worker 74 - jepsen.util 8444 :ok :read [24 23] INFO [2019-12-17 12:34:22,166] jepsen worker 74 - jepsen.util 8444 :invoke :cas [24 [26 31]] INFO [2019-12-17 12:34:22,176] jepsen worker 30 - jepsen.util 8310 :ok :read [10 32] INFO [2019-12-17 12:34:22,176] jepsen worker 30 - jepsen.util 8310 :invoke :read [10 nil] INFO [2019-12-17 12:34:22,196] jepsen worker 82 - jepsen.util 11782 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:34:22,196] jepsen worker 82 - jepsen.util 11782 :invoke :write [27 2] INFO [2019-12-17 12:34:22,204] jepsen worker 65 - jepsen.util 6905 :ok :write [21 32] INFO [2019-12-17 12:34:22,205] jepsen worker 65 - jepsen.util 6905 :invoke :cas [21 [32 5]] INFO [2019-12-17 12:34:22,213] jepsen worker 23 - jepsen.util 11003 :info :cas [7 [45 12]] :RequestCanceledException INFO [2019-12-17 12:34:22,214] jepsen worker 23 - jepsen.util 11093 :invoke :read [7 nil] INFO [2019-12-17 12:34:22,229] jepsen worker 28 - jepsen.util 6508 :ok :write [9 20] INFO [2019-12-17 12:34:22,229] jepsen worker 68 - jepsen.util 10508 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:34:22,229] jepsen worker 68 - jepsen.util 10508 :invoke :write [22 1] INFO [2019-12-17 12:34:22,230] jepsen worker 50 - jepsen.util 11930 :info :write [16 11] :RequestCanceledException INFO [2019-12-17 12:34:22,230] jepsen worker 21 - jepsen.util 10551 :info :cas [7 [38 17]] :RequestCanceledException INFO [2019-12-17 12:34:22,231] jepsen worker 28 - jepsen.util 6508 :invoke :read [9 nil] INFO [2019-12-17 12:34:22,231] jepsen worker 50 - jepsen.util 12020 :invoke :read [16 nil] INFO [2019-12-17 12:34:22,232] jepsen worker 21 - jepsen.util 10641 :invoke :cas [7 [29 15]] INFO [2019-12-17 12:34:22,243] jepsen worker 60 - jepsen.util 12120 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:34:22,243] jepsen worker 60 - jepsen.util 12120 :invoke :write [20 43] INFO [2019-12-17 12:34:22,247] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47433,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:34:22,247] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47442,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:34:22,247] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47433,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:34:22,247] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47442,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:34:22,247] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47434,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:34:22,247] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47447,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:34:22,247] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47434,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:34:22,247] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47447,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:34:22,247] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47435,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:34:22,247] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47435,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:34:22,256] jepsen worker 36 - jepsen.util 8586 :fail :cas [12 [34 38]] :ValueNotSwapFrom INFO [2019-12-17 12:34:22,256] jepsen worker 36 - jepsen.util 8586 :invoke :write [12 32] INFO [2019-12-17 12:34:22,261] jepsen worker 83 - jepsen.util 11513 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:34:22,261] jepsen worker 16 - jepsen.util 12346 :info :write [5 46] :RequestCanceledException INFO [2019-12-17 12:34:22,261] jepsen worker 76 - jepsen.util 11056 :info :cas [25 [35 2]] :RequestCanceledException INFO [2019-12-17 12:34:22,261] jepsen worker 83 - jepsen.util 11513 :invoke :write [27 6] INFO [2019-12-17 12:34:22,264] jepsen worker 16 - jepsen.util 12436 :invoke :read [5 nil] INFO [2019-12-17 12:34:22,264] jepsen worker 76 - jepsen.util 11146 :invoke :write [25 38] INFO [2019-12-17 12:34:22,265] jepsen worker 5 - jepsen.util 7745 :fail :cas [1 [7 18]] :ValueNotSwapFrom INFO [2019-12-17 12:34:22,265] jepsen worker 5 - jepsen.util 7745 :invoke :write [1 35] INFO [2019-12-17 12:34:22,282] jepsen worker 38 - jepsen.util 8498 :fail :cas [12 [34 11]] :ValueNotSwapFrom INFO [2019-12-17 12:34:22,282] jepsen worker 38 - jepsen.util 8498 :invoke :read [12 nil] INFO [2019-12-17 12:34:22,288] jepsen worker 65 - jepsen.util 6905 :ok :cas [21 [32 5]] INFO [2019-12-17 12:34:22,289] jepsen worker 65 - jepsen.util 6905 :invoke :write [21 24] INFO [2019-12-17 12:34:22,295] jepsen worker 45 - jepsen.util 8325 :info :write [15 35] :SyncWriteAmbiguous INFO [2019-12-17 12:34:22,295] jepsen worker 14 - jepsen.util 8474 :info :write [4 46] :RequestCanceledException INFO [2019-12-17 12:34:22,296] jepsen worker 14 - jepsen.util 8564 :invoke :cas [4 [8 3]] INFO [2019-12-17 12:34:22,296] jepsen worker 45 - jepsen.util 8415 :invoke :read [15 nil] INFO [2019-12-17 12:34:22,318] jepsen worker 27 - jepsen.util 8037 :ok :read [9 20] INFO [2019-12-17 12:34:22,318] jepsen worker 27 - jepsen.util 8037 :invoke :read [9 nil] INFO [2019-12-17 12:34:22,328] jepsen worker 34 - jepsen.util 11104 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:34:22,328] jepsen worker 34 - jepsen.util 11104 :invoke :cas [11 [6 1]] INFO [2019-12-17 12:34:22,332] jepsen worker 47 - jepsen.util 8687 :ok :read [15 17] INFO [2019-12-17 12:34:22,332] jepsen worker 47 - jepsen.util 8687 :invoke :write [15 29] INFO [2019-12-17 12:34:22,346] jepsen worker 45 - jepsen.util 8415 :ok :read [15 17] INFO [2019-12-17 12:34:22,347] jepsen worker 45 - jepsen.util 8415 :invoke :write [15 19] INFO [2019-12-17 12:34:22,365] jepsen worker 27 - jepsen.util 8037 :ok :read [9 20] INFO [2019-12-17 12:34:22,365] jepsen worker 27 - jepsen.util 8037 :invoke :cas [9 [24 24]] INFO [2019-12-17 12:34:22,367] jepsen worker 3 - jepsen.util 8103 :fail :cas [1 [29 22]] :ValueNotSwapFrom INFO [2019-12-17 12:34:22,368] jepsen worker 3 - jepsen.util 8103 :invoke :cas [1 [26 48]] INFO [2019-12-17 12:34:22,373] jepsen worker 65 - jepsen.util 6905 :ok :write [21 24] INFO [2019-12-17 12:34:22,373] jepsen worker 65 - jepsen.util 6905 :invoke :write [21 30] INFO [2019-12-17 12:34:22,385] jepsen worker 89 - jepsen.util 8189 :ok :read [29 37] INFO [2019-12-17 12:34:22,385] jepsen worker 89 - jepsen.util 8189 :invoke :write [29 15] INFO [2019-12-17 12:34:22,428] jepsen worker 65 - jepsen.util 6905 :ok :write [21 30] INFO [2019-12-17 12:34:22,428] jepsen worker 65 - jepsen.util 6905 :invoke :read [21 nil] INFO [2019-12-17 12:34:22,439] jepsen worker 37 - jepsen.util 7867 :fail :cas [12 [39 17]] :ValueNotSwapFrom INFO [2019-12-17 12:34:22,440] jepsen worker 37 - jepsen.util 7867 :invoke :cas [12 [8 2]] INFO [2019-12-17 12:34:22,440] jepsen worker 56 - jepsen.util 9236 :fail :cas [18 [18 12]] :ValueNotSwapFrom INFO [2019-12-17 12:34:22,440] jepsen worker 56 - jepsen.util 9236 :invoke :read [18 nil] INFO [2019-12-17 12:34:22,446] jepsen worker 54 - jepsen.util 9144 :ok :read [18 42] INFO [2019-12-17 12:34:22,446] jepsen worker 54 - jepsen.util 9144 :invoke :write [18 34] INFO [2019-12-17 12:34:22,451] jepsen worker 38 - jepsen.util 8498 :ok :read [12 6] INFO [2019-12-17 12:34:22,451] jepsen worker 38 - jepsen.util 8498 :invoke :read [12 nil] INFO [2019-12-17 12:34:22,456] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:34:22,456] jepsen nemesis - couchbase.util Rebalance check count: 368 INFO [2019-12-17 12:34:22,456] jepsen nemesis - couchbase.util Rebalance stuck count: 349 INFO [2019-12-17 12:34:22,461] jepsen worker 65 - jepsen.util 6905 :ok :read [21 30] INFO [2019-12-17 12:34:22,461] jepsen worker 65 - jepsen.util 6905 :invoke :write [21 2] INFO [2019-12-17 12:34:22,466] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47473,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:34:22,466] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47473,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:34:22,466] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47477,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:34:22,466] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47477,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:34:22,502] jepsen worker 51 - jepsen.util 7701 :ok :read [17 39] INFO [2019-12-17 12:34:22,502] jepsen worker 51 - jepsen.util 7701 :invoke :cas [17 [6 47]] INFO [2019-12-17 12:34:22,505] jepsen worker 65 - jepsen.util 6905 :ok :write [21 2] INFO [2019-12-17 12:34:22,506] jepsen worker 65 - jepsen.util 6905 :invoke :cas [21 [34 43]] INFO [2019-12-17 12:34:22,516] jepsen worker 53 - jepsen.util 7703 :fail :cas [17 [12 15]] :ValueNotSwapFrom INFO [2019-12-17 12:34:22,517] jepsen worker 53 - jepsen.util 7703 :invoke :read [17 nil] INFO [2019-12-17 12:34:22,528] jepsen worker 65 - jepsen.util 6905 :fail :cas [21 [34 43]] :ValueNotSwapFrom INFO [2019-12-17 12:34:22,528] jepsen worker 65 - jepsen.util 6905 :invoke :read [21 nil] INFO [2019-12-17 12:34:22,529] jepsen worker 35 - jepsen.util 10925 :info :cas [11 [27 30]] :RequestCanceledException INFO [2019-12-17 12:34:22,530] jepsen worker 35 - jepsen.util 11015 :invoke :read [11 nil] INFO [2019-12-17 12:34:22,540] jepsen worker 37 - jepsen.util 7867 :fail :cas [12 [8 2]] :ValueNotSwapFrom INFO [2019-12-17 12:34:22,541] jepsen worker 37 - jepsen.util 7867 :invoke :write [12 12] INFO [2019-12-17 12:34:22,547] jepsen worker 14 - jepsen.util 8564 :fail :cas [4 [8 3]] :ValueNotSwapFrom INFO [2019-12-17 12:34:22,548] jepsen worker 14 - jepsen.util 8564 :invoke :read [4 nil] INFO [2019-12-17 12:34:22,550] jepsen worker 65 - jepsen.util 6905 :ok :read [21 2] INFO [2019-12-17 12:34:22,551] jepsen worker 65 - jepsen.util 6905 :invoke :write [21 30] INFO [2019-12-17 12:34:22,563] jepsen worker 51 - jepsen.util 7701 :fail :cas [17 [6 47]] :ValueNotSwapFrom INFO [2019-12-17 12:34:22,563] jepsen worker 51 - jepsen.util 7701 :invoke :read [17 nil] INFO [2019-12-17 12:34:22,596] jepsen worker 12 - jepsen.util 8832 :info :write [4 39] :RequestCanceledException INFO [2019-12-17 12:34:22,597] jepsen worker 12 - jepsen.util 8922 :invoke :write [4 34] INFO [2019-12-17 12:34:22,600] jepsen worker 65 - jepsen.util 6905 :ok :write [21 30] INFO [2019-12-17 12:34:22,600] jepsen worker 65 - jepsen.util 6905 :invoke :write [21 21] INFO [2019-12-17 12:34:22,603] jepsen worker 53 - jepsen.util 7703 :ok :read [17 39] INFO [2019-12-17 12:34:22,604] jepsen worker 53 - jepsen.util 7703 :invoke :read [17 nil] INFO [2019-12-17 12:34:22,610] jepsen worker 55 - jepsen.util 9055 :ok :write [18 4] INFO [2019-12-17 12:34:22,610] jepsen worker 55 - jepsen.util 9055 :invoke :cas [18 [28 35]] INFO [2019-12-17 12:34:22,630] jepsen worker 67 - jepsen.util 11227 :info :cas [22 [38 22]] :RequestCanceledException INFO [2019-12-17 12:34:22,631] jepsen worker 67 - jepsen.util 11317 :invoke :cas [22 [2 5]] INFO [2019-12-17 12:34:22,631] jepsen worker 36 - jepsen.util 8586 :ok :write [12 32] INFO [2019-12-17 12:34:22,632] jepsen worker 36 - jepsen.util 8586 :invoke :cas [12 [34 23]] INFO [2019-12-17 12:34:22,635] jepsen worker 71 - jepsen.util 8891 :ok :write [23 11] INFO [2019-12-17 12:34:22,635] jepsen worker 80 - jepsen.util 8090 :fail :cas [26 [8 7]] :ValueNotSwapFrom INFO [2019-12-17 12:34:22,635] jepsen worker 71 - jepsen.util 8891 :invoke :read [23 nil] INFO [2019-12-17 12:34:22,635] jepsen worker 80 - jepsen.util 8090 :invoke :read [26 nil] INFO [2019-12-17 12:34:22,639] jepsen worker 3 - jepsen.util 8103 :fail :cas [1 [26 48]] :ValueNotSwapFrom INFO [2019-12-17 12:34:22,639] jepsen worker 3 - jepsen.util 8103 :invoke :cas [1 [44 33]] INFO [2019-12-17 12:34:22,660] jepsen worker 88 - jepsen.util 8278 :info :write [29 49] :RequestCanceledException INFO [2019-12-17 12:34:22,662] jepsen worker 88 - jepsen.util 8368 :invoke :write [29 14] INFO [2019-12-17 12:34:22,666] jepsen worker 65 - jepsen.util 6905 :ok :write [21 21] INFO [2019-12-17 12:34:22,666] jepsen worker 65 - jepsen.util 6905 :invoke :read [21 nil] INFO [2019-12-17 12:34:22,685] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47501,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:34:22,685] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47501,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:34:22,685] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47506,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:34:22,685] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47506,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:34:22,686] jepsen worker 65 - jepsen.util 6905 :ok :read [21 21] INFO [2019-12-17 12:34:22,686] jepsen worker 65 - jepsen.util 6905 :invoke :read [21 nil] INFO [2019-12-17 12:34:22,708] jepsen worker 65 - jepsen.util 6905 :ok :read [21 21] INFO [2019-12-17 12:34:22,708] jepsen worker 65 - jepsen.util 6905 :invoke :read [21 nil] INFO [2019-12-17 12:34:22,721] jepsen worker 14 - jepsen.util 8564 :ok :read [4 21] INFO [2019-12-17 12:34:22,721] jepsen worker 14 - jepsen.util 8564 :invoke :read [4 nil] INFO [2019-12-17 12:34:22,726] jepsen worker 65 - jepsen.util 6905 :ok :read [21 21] INFO [2019-12-17 12:34:22,726] jepsen worker 65 - jepsen.util 6905 :invoke :read [21 nil] INFO [2019-12-17 12:34:22,750] jepsen worker 65 - jepsen.util 6905 :ok :read [21 21] INFO [2019-12-17 12:34:22,751] jepsen worker 65 - jepsen.util 6905 :invoke :write [21 13] INFO [2019-12-17 12:34:22,752] jepsen worker 38 - jepsen.util 8498 :ok :read [12 32] INFO [2019-12-17 12:34:22,752] jepsen worker 38 - jepsen.util 8498 :invoke :write [12 27] INFO [2019-12-17 12:34:22,815] jepsen worker 65 - jepsen.util 6905 :ok :write [21 13] INFO [2019-12-17 12:34:22,815] jepsen worker 65 - jepsen.util 6905 :invoke :cas [21 [32 46]] INFO [2019-12-17 12:34:22,829] jepsen worker 65 - jepsen.util 6905 :fail :cas [21 [32 46]] :ValueNotSwapFrom INFO [2019-12-17 12:34:22,830] jepsen worker 65 - jepsen.util 6905 :invoke :read [21 nil] INFO [2019-12-17 12:34:22,838] jepsen worker 53 - jepsen.util 7703 :ok :read [17 39] INFO [2019-12-17 12:34:22,838] jepsen worker 53 - jepsen.util 7703 :invoke :write [17 27] INFO [2019-12-17 12:34:22,847] jepsen worker 51 - jepsen.util 7701 :ok :read [17 39] INFO [2019-12-17 12:34:22,847] jepsen worker 51 - jepsen.util 7701 :invoke :write [17 30] INFO [2019-12-17 12:34:22,856] jepsen worker 65 - jepsen.util 6905 :ok :read [21 13] INFO [2019-12-17 12:34:22,856] jepsen worker 65 - jepsen.util 6905 :invoke :write [21 7] INFO [2019-12-17 12:34:22,904] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47521,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:34:22,904] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47521,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:34:22,904] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47525,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:34:22,905] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47525,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:34:22,908] jepsen worker 65 - jepsen.util 6905 :ok :write [21 7] INFO [2019-12-17 12:34:22,909] jepsen worker 65 - jepsen.util 6905 :invoke :read [21 nil] INFO [2019-12-17 12:34:22,923] jepsen worker 65 - jepsen.util 6905 :ok :read [21 7] INFO [2019-12-17 12:34:22,924] jepsen worker 65 - jepsen.util 6905 :invoke :write [21 20] INFO [2019-12-17 12:34:22,943] jepsen worker 33 - jepsen.util 11103 :info :write [11 21] :RequestCanceledException INFO [2019-12-17 12:34:22,944] jepsen worker 33 - jepsen.util 11193 :invoke :read [11 nil] INFO [2019-12-17 12:34:22,947] jepsen worker 3 - jepsen.util 8103 :fail :cas [1 [44 33]] :ValueNotSwapFrom INFO [2019-12-17 12:34:22,947] jepsen worker 3 - jepsen.util 8103 :invoke :write [1 13] INFO [2019-12-17 12:34:22,948] jepsen worker 70 - jepsen.util 8620 :ok :read [23 11] INFO [2019-12-17 12:34:22,948] jepsen worker 70 - jepsen.util 8620 :invoke :cas [23 [37 47]] INFO [2019-12-17 12:34:22,962] jepsen worker 65 - jepsen.util 6905 :ok :write [21 20] INFO [2019-12-17 12:34:22,962] jepsen worker 65 - jepsen.util 6905 :invoke :write [21 11] INFO [2019-12-17 12:34:22,979] jepsen worker 56 - jepsen.util 9236 :ok :read [18 4] INFO [2019-12-17 12:34:22,979] jepsen worker 56 - jepsen.util 9236 :invoke :write [18 27] INFO [2019-12-17 12:34:22,995] jepsen worker 57 - jepsen.util 11127 :info :cas [19 [1 13]] :RequestCanceledException INFO [2019-12-17 12:34:22,996] jepsen worker 57 - jepsen.util 11217 :invoke :write [19 44] INFO [2019-12-17 12:34:23,006] jepsen worker 14 - jepsen.util 8564 :ok :read [4 21] INFO [2019-12-17 12:34:23,006] jepsen worker 14 - jepsen.util 8564 :invoke :read [4 nil] INFO [2019-12-17 12:34:23,014] jepsen worker 58 - jepsen.util 12028 :info :write [19 17] :RequestCanceledException INFO [2019-12-17 12:34:23,017] jepsen worker 58 - jepsen.util 12118 :invoke :write [19 12] INFO [2019-12-17 12:34:23,018] jepsen worker 55 - jepsen.util 9055 :fail :cas [18 [28 35]] :ValueNotSwapFrom INFO [2019-12-17 12:34:23,018] jepsen worker 55 - jepsen.util 9055 :invoke :write [18 34] INFO [2019-12-17 12:34:23,023] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47535,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:34:23,023] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47535,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:34:23,024] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47538,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:34:23,024] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47538,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:34:23,027] jepsen worker 65 - jepsen.util 6905 :ok :write [21 11] INFO [2019-12-17 12:34:23,027] jepsen worker 65 - jepsen.util 6905 :invoke :write [21 4] INFO [2019-12-17 12:34:23,029] jepsen worker 62 - jepsen.util 11402 :info :cas [20 [31 19]] :RequestCanceledException INFO [2019-12-17 12:34:23,029] jepsen worker 0 - jepsen.util 11700 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:34:23,029] jepsen worker 0 - jepsen.util 11700 :invoke :cas [0 [11 10]] INFO [2019-12-17 12:34:23,030] jepsen worker 62 - jepsen.util 11492 :invoke :read [20 nil] INFO [2019-12-17 12:34:23,044] jepsen worker 85 - jepsen.util 11965 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:34:23,044] jepsen worker 25 - jepsen.util 11725 :info :cas [8 [31 15]] :RequestCanceledException INFO [2019-12-17 12:34:23,044] jepsen worker 85 - jepsen.util 11965 :invoke :cas [28 [48 0]] INFO [2019-12-17 12:34:23,045] jepsen worker 25 - jepsen.util 11815 :invoke :cas [8 [19 14]] INFO [2019-12-17 12:34:23,049] jepsen worker 80 - jepsen.util 8090 :ok :read [26 25] INFO [2019-12-17 12:34:23,049] jepsen worker 80 - jepsen.util 8090 :invoke :cas [26 [3 6]] INFO [2019-12-17 12:34:23,061] jepsen worker 69 - jepsen.util 8349 :info :write [23 31] :RequestCanceledException INFO [2019-12-17 12:34:23,062] jepsen worker 69 - jepsen.util 8439 :invoke :read [23 nil] INFO [2019-12-17 12:34:23,064] jepsen worker 65 - jepsen.util 6905 :ok :write [21 4] INFO [2019-12-17 12:34:23,065] jepsen worker 65 - jepsen.util 6905 :invoke :write [21 37] INFO [2019-12-17 12:34:23,089] jepsen worker 88 - jepsen.util 8368 :ok :write [29 14] INFO [2019-12-17 12:34:23,090] jepsen worker 88 - jepsen.util 8368 :invoke :write [29 39] INFO [2019-12-17 12:34:23,123] jepsen worker 70 - jepsen.util 8620 :fail :cas [23 [37 47]] :ValueNotSwapFrom INFO [2019-12-17 12:34:23,123] jepsen worker 70 - jepsen.util 8620 :invoke :cas [23 [28 27]] INFO [2019-12-17 12:34:23,128] jepsen worker 14 - jepsen.util 8564 :ok :read [4 21] INFO [2019-12-17 12:34:23,128] jepsen worker 14 - jepsen.util 8564 :invoke :read [4 nil] INFO [2019-12-17 12:34:23,172] jepsen worker 65 - jepsen.util 6905 :ok :write [21 37] INFO [2019-12-17 12:34:23,172] jepsen worker 65 - jepsen.util 6905 :invoke :cas [21 [23 20]] INFO [2019-12-17 12:34:23,185] jepsen worker 65 - jepsen.util 6905 :fail :cas [21 [23 20]] :ValueNotSwapFrom INFO [2019-12-17 12:34:23,185] jepsen worker 65 - jepsen.util 6905 :invoke :cas [21 [48 0]] INFO [2019-12-17 12:34:23,196] jepsen worker 48 - jepsen.util 10308 :info :write [16 40] :RequestCanceledException INFO [2019-12-17 12:34:23,197] jepsen worker 48 - jepsen.util 10398 :invoke :write [16 43] INFO [2019-12-17 12:34:23,213] jepsen worker 22 - jepsen.util 11272 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:34:23,213] jepsen worker 22 - jepsen.util 11272 :invoke :cas [7 [6 6]] INFO [2019-12-17 12:34:23,217] jepsen worker 65 - jepsen.util 6905 :fail :cas [21 [48 0]] :ValueNotSwapFrom INFO [2019-12-17 12:34:23,217] jepsen worker 65 - jepsen.util 6905 :invoke :cas [21 [35 38]] INFO [2019-12-17 12:34:23,229] jepsen worker 61 - jepsen.util 12121 :info :write [20 41] :RequestCanceledException INFO [2019-12-17 12:34:23,229] jepsen worker 26 - jepsen.util 11636 :info :cas [8 [40 42]] :RequestCanceledException INFO [2019-12-17 12:34:23,230] jepsen worker 61 - jepsen.util 12211 :invoke :read [20 nil] INFO [2019-12-17 12:34:23,230] jepsen worker 26 - jepsen.util 11726 :invoke :cas [8 [45 13]] INFO [2019-12-17 12:34:23,235] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47567,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:34:23,235] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47567,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:34:23,235] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47569,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:34:23,235] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47569,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:34:23,235] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47556,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:34:23,235] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47556,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:34:23,235] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47557,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:34:23,235] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47557,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:34:23,238] jepsen worker 65 - jepsen.util 6905 :fail :cas [21 [35 38]] :ValueNotSwapFrom INFO [2019-12-17 12:34:23,238] jepsen worker 65 - jepsen.util 6905 :invoke :read [21 nil] INFO [2019-12-17 12:34:23,243] jepsen worker 79 - jepsen.util 8539 :info :cas [26 [11 46]] :RequestCanceledException INFO [2019-12-17 12:34:23,244] jepsen worker 79 - jepsen.util 8629 :invoke :read [26 nil] INFO [2019-12-17 12:34:23,250] jepsen worker 65 - jepsen.util 6905 :ok :read [21 37] INFO [2019-12-17 12:34:23,250] jepsen worker 65 - jepsen.util 6905 :invoke :read [21 nil] INFO [2019-12-17 12:34:23,262] jepsen worker 65 - jepsen.util 6905 :ok :read [21 37] INFO [2019-12-17 12:34:23,262] jepsen worker 71 - jepsen.util 8891 :ok :read [23 11] INFO [2019-12-17 12:34:23,262] jepsen worker 65 - jepsen.util 6905 :invoke :read [21 nil] INFO [2019-12-17 12:34:23,262] jepsen worker 71 - jepsen.util 8891 :invoke :write [23 40] INFO [2019-12-17 12:34:23,263] jepsen worker 20 - jepsen.util 7850 :info :write [6 26] :RequestCanceledException INFO [2019-12-17 12:34:23,278] jepsen worker 20 - jepsen.util 7940 :invoke :write [6 7] INFO [2019-12-17 12:34:23,288] jepsen worker 6 - jepsen.util 11796 :info :write [2 43] :RequestCanceledException INFO [2019-12-17 12:34:23,289] jepsen worker 6 - jepsen.util 11886 :invoke :read [2 nil] INFO [2019-12-17 12:34:23,300] jepsen worker 65 - jepsen.util 6905 :ok :read [21 37] INFO [2019-12-17 12:34:23,300] jepsen worker 65 - jepsen.util 6905 :invoke :cas [21 [7 33]] INFO [2019-12-17 12:34:23,312] jepsen worker 65 - jepsen.util 6905 :fail :cas [21 [7 33]] :ValueNotSwapFrom INFO [2019-12-17 12:34:23,312] jepsen worker 65 - jepsen.util 6905 :invoke :read [21 nil] INFO [2019-12-17 12:34:23,313] jepsen worker 70 - jepsen.util 8620 :fail :cas [23 [28 27]] :ValueNotSwapFrom INFO [2019-12-17 12:34:23,313] jepsen worker 70 - jepsen.util 8620 :invoke :read [23 nil] INFO [2019-12-17 12:34:23,327] jepsen worker 44 - jepsen.util 11384 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:34:23,327] jepsen worker 84 - jepsen.util 10794 :info :cas [28 [25 48]] :RequestCanceledException INFO [2019-12-17 12:34:23,327] jepsen worker 66 - jepsen.util 11046 :info :cas [22 [8 3]] :RequestCanceledException INFO [2019-12-17 12:34:23,327] jepsen worker 44 - jepsen.util 11384 :invoke :cas [14 [45 5]] INFO [2019-12-17 12:34:23,327] jepsen worker 65 - jepsen.util 6905 :ok :read [21 37] INFO [2019-12-17 12:34:23,327] jepsen worker 65 - jepsen.util 6905 :invoke :cas [21 [3 8]] INFO [2019-12-17 12:34:23,328] jepsen worker 84 - jepsen.util 10884 :invoke :write [28 38] INFO [2019-12-17 12:34:23,329] jepsen worker 66 - jepsen.util 11136 :invoke :write [22 15] INFO [2019-12-17 12:34:23,344] jepsen worker 65 - jepsen.util 6905 :fail :cas [21 [3 8]] :ValueNotSwapFrom INFO [2019-12-17 12:34:23,344] jepsen worker 65 - jepsen.util 6905 :invoke :read [21 nil] INFO [2019-12-17 12:34:23,358] jepsen worker 65 - jepsen.util 6905 :ok :read [21 37] INFO [2019-12-17 12:34:23,358] jepsen worker 65 - jepsen.util 6905 :invoke :write [21 6] INFO [2019-12-17 12:34:23,361] jepsen worker 73 - jepsen.util 7813 :info :write [24 19] :RequestCanceledException INFO [2019-12-17 12:34:23,362] jepsen worker 73 - jepsen.util 7903 :invoke :cas [24 [2 14]] INFO [2019-12-17 12:34:23,370] jepsen worker 3 - jepsen.util 8103 :ok :write [1 13] INFO [2019-12-17 12:34:23,371] jepsen worker 3 - jepsen.util 8103 :invoke :read [1 nil] INFO [2019-12-17 12:34:23,396] jepsen worker 40 - jepsen.util 11560 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:34:23,396] jepsen worker 4 - jepsen.util 7654 :info :write [1 13] :RequestCanceledException INFO [2019-12-17 12:34:23,396] jepsen worker 40 - jepsen.util 11560 :invoke :write [13 13] INFO [2019-12-17 12:34:23,399] jepsen worker 4 - jepsen.util 7744 :invoke :cas [1 [8 11]] INFO [2019-12-17 12:34:23,420] jepsen worker 52 - jepsen.util 8242 :ok :write [17 37] INFO [2019-12-17 12:34:23,420] jepsen worker 52 - jepsen.util 8242 :invoke :write [17 35] INFO [2019-12-17 12:34:23,420] jepsen worker 65 - jepsen.util 6905 :ok :write [21 6] INFO [2019-12-17 12:34:23,420] jepsen worker 65 - jepsen.util 6905 :invoke :read [21 nil] INFO [2019-12-17 12:34:23,430] jepsen worker 32 - jepsen.util 8312 :info :write [10 20] :RequestCanceledException INFO [2019-12-17 12:34:23,431] jepsen worker 32 - jepsen.util 8402 :invoke :write [10 47] INFO [2019-12-17 12:34:23,431] jepsen worker 10 - jepsen.util 8380 :info :write [3 46] :RequestCanceledException INFO [2019-12-17 12:34:23,443] jepsen worker 14 - jepsen.util 8564 :ok :read [4 21] INFO [2019-12-17 12:34:23,443] jepsen worker 14 - jepsen.util 8564 :invoke :read [4 nil] INFO [2019-12-17 12:34:23,445] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47595,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:34:23,445] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47595,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:34:23,445] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47596,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:34:23,445] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47596,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:34:23,449] jepsen worker 10 - jepsen.util 8470 :invoke :read [3 nil] INFO [2019-12-17 12:34:23,455] jepsen worker 71 - jepsen.util 8891 :ok :write [23 40] INFO [2019-12-17 12:34:23,455] jepsen worker 71 - jepsen.util 8891 :invoke :cas [23 [22 6]] INFO [2019-12-17 12:34:23,464] jepsen worker 65 - jepsen.util 6905 :ok :read [21 6] INFO [2019-12-17 12:34:23,464] jepsen worker 65 - jepsen.util 6905 :invoke :read [21 nil] INFO [2019-12-17 12:34:23,473] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:34:23,473] jepsen nemesis - couchbase.util Rebalance check count: 369 INFO [2019-12-17 12:34:23,473] jepsen nemesis - couchbase.util Rebalance stuck count: 350 INFO [2019-12-17 12:34:23,477] jepsen worker 65 - jepsen.util 6905 :ok :read [21 6] INFO [2019-12-17 12:34:23,477] jepsen worker 65 - jepsen.util 6905 :invoke :write [21 3] INFO [2019-12-17 12:34:23,496] jepsen worker 24 - jepsen.util 10194 :info :cas [8 [28 29]] :RequestCanceledException INFO [2019-12-17 12:34:23,497] jepsen worker 24 - jepsen.util 10284 :invoke :cas [8 [13 33]] INFO [2019-12-17 12:34:23,504] jepsen worker 70 - jepsen.util 8620 :ok :read [23 40] INFO [2019-12-17 12:34:23,504] jepsen worker 70 - jepsen.util 8620 :invoke :write [23 42] INFO [2019-12-17 12:34:23,514] jepsen worker 15 - jepsen.util 10365 :info :cas [5 [16 38]] :RequestCanceledException INFO [2019-12-17 12:34:23,516] jepsen worker 65 - jepsen.util 6905 :ok :write [21 3] INFO [2019-12-17 12:34:23,516] jepsen worker 65 - jepsen.util 6905 :invoke :read [21 nil] INFO [2019-12-17 12:34:23,517] jepsen worker 15 - jepsen.util 10455 :invoke :read [5 nil] INFO [2019-12-17 12:34:23,543] jepsen worker 13 - jepsen.util 7753 :info :write [4 22] :RequestCanceledException INFO [2019-12-17 12:34:23,544] jepsen worker 13 - jepsen.util 7843 :invoke :cas [4 [3 38]] INFO [2019-12-17 12:34:23,559] jepsen worker 65 - jepsen.util 6905 :ok :read [21 3] INFO [2019-12-17 12:34:23,560] jepsen worker 65 - jepsen.util 6905 :invoke :cas [21 [29 3]] INFO [2019-12-17 12:34:23,560] jepsen worker 17 - jepsen.util 11177 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:34:23,561] jepsen worker 17 - jepsen.util 11177 :invoke :cas [5 [32 30]] INFO [2019-12-17 12:34:23,565] jepsen worker 38 - jepsen.util 8498 :ok :write [12 27] INFO [2019-12-17 12:34:23,565] jepsen worker 38 - jepsen.util 8498 :invoke :read [12 nil] INFO [2019-12-17 12:34:23,574] jepsen worker 71 - jepsen.util 8891 :fail :cas [23 [22 6]] :ValueNotSwapFrom INFO [2019-12-17 12:34:23,574] jepsen worker 71 - jepsen.util 8891 :invoke :cas [23 [7 23]] INFO [2019-12-17 12:34:23,581] jepsen worker 65 - jepsen.util 6905 :fail :cas [21 [29 3]] :ValueNotSwapFrom INFO [2019-12-17 12:34:23,584] jepsen worker 14 - jepsen.util 8564 :ok :read [4 21] INFO [2019-12-17 12:34:23,585] jepsen worker 14 - jepsen.util 8564 :invoke :read [4 nil] INFO [2019-12-17 12:34:23,585] jepsen worker 65 - jepsen.util 6905 :invoke :write [21 44] INFO [2019-12-17 12:34:23,600] jepsen worker 56 - jepsen.util 9236 :ok :write [18 27] INFO [2019-12-17 12:34:23,600] jepsen worker 56 - jepsen.util 9236 :invoke :read [18 nil] INFO [2019-12-17 12:34:23,614] jepsen worker 1 - jepsen.util 11431 :info :write [0 49] :RequestCanceledException INFO [2019-12-17 12:34:23,615] jepsen worker 1 - jepsen.util 11521 :invoke :write [0 38] INFO [2019-12-17 12:34:23,629] jepsen worker 42 - jepsen.util 12192 :info :write [14 28] :RequestCanceledException INFO [2019-12-17 12:34:23,630] jepsen worker 42 - jepsen.util 12282 :invoke :cas [14 [39 0]] INFO [2019-12-17 12:34:23,643] jepsen worker 43 - jepsen.util 10843 :info :cas [14 [21 29]] :RequestCanceledException INFO [2019-12-17 12:34:23,652] jepsen worker 65 - jepsen.util 6905 :ok :write [21 44] INFO [2019-12-17 12:34:23,653] jepsen worker 65 - jepsen.util 6905 :invoke :write [21 19] INFO [2019-12-17 12:34:23,657] jepsen worker 8 - jepsen.util 11078 :info :write [2 17] :RequestCanceledException INFO [2019-12-17 12:34:23,657] jepsen worker 77 - jepsen.util 11507 :info :cas [25 [21 38]] :RequestCanceledException INFO [2019-12-17 12:34:23,658] jepsen worker 43 - jepsen.util 10933 :invoke :write [14 8] INFO [2019-12-17 12:34:23,659] jepsen worker 8 - jepsen.util 11168 :invoke :cas [2 [48 10]] INFO [2019-12-17 12:34:23,660] jepsen worker 77 - jepsen.util 11597 :invoke :read [25 nil] INFO [2019-12-17 12:34:23,668] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47624,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:34:23,668] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47624,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:34:23,669] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47626,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:34:23,669] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47626,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:34:23,681] jepsen worker 3 - jepsen.util 8103 :ok :read [1 13] INFO [2019-12-17 12:34:23,681] jepsen worker 3 - jepsen.util 8103 :invoke :cas [1 [17 43]] INFO [2019-12-17 12:34:23,684] jepsen worker 9 - jepsen.util 8559 :info :write [3 23] :SyncWriteAmbiguous INFO [2019-12-17 12:34:23,685] jepsen worker 9 - jepsen.util 8649 :invoke :read [3 nil] INFO [2019-12-17 12:34:23,710] jepsen worker 72 - jepsen.util 8172 :ok :read [24 23] INFO [2019-12-17 12:34:23,710] jepsen worker 72 - jepsen.util 8172 :invoke :read [24 nil] INFO [2019-12-17 12:34:23,729] jepsen worker 78 - jepsen.util 7998 :info :write [26 21] :RequestCanceledException INFO [2019-12-17 12:34:23,733] jepsen worker 9 - jepsen.util 8649 :ok :read [3 48] INFO [2019-12-17 12:34:23,733] jepsen worker 9 - jepsen.util 8649 :invoke :write [3 43] INFO [2019-12-17 12:34:23,735] jepsen worker 65 - jepsen.util 6905 :ok :write [21 19] INFO [2019-12-17 12:34:23,735] jepsen worker 65 - jepsen.util 6905 :invoke :cas [21 [40 32]] INFO [2019-12-17 12:34:23,744] jepsen worker 73 - jepsen.util 7903 :fail :cas [24 [2 14]] :ValueNotSwapFrom INFO [2019-12-17 12:34:23,744] jepsen worker 73 - jepsen.util 7903 :invoke :write [24 33] INFO [2019-12-17 12:34:23,745] jepsen worker 78 - jepsen.util 8088 :invoke :read [26 nil] INFO [2019-12-17 12:34:23,749] jepsen worker 72 - jepsen.util 8172 :ok :read [24 23] INFO [2019-12-17 12:34:23,749] jepsen worker 72 - jepsen.util 8172 :invoke :write [24 17] INFO [2019-12-17 12:34:23,764] jepsen worker 80 - jepsen.util 8090 :fail :cas [26 [3 6]] :ValueNotSwapFrom INFO [2019-12-17 12:34:23,764] jepsen worker 80 - jepsen.util 8090 :invoke :write [26 49] INFO [2019-12-17 12:34:23,787] jepsen worker 69 - jepsen.util 8439 :ok :read [23 40] INFO [2019-12-17 12:34:23,787] jepsen worker 69 - jepsen.util 8439 :invoke :write [23 19] INFO [2019-12-17 12:34:23,792] jepsen worker 7 - jepsen.util 11437 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:34:23,793] jepsen worker 7 - jepsen.util 11437 :invoke :write [2 42] INFO [2019-12-17 12:34:23,820] jepsen worker 14 - jepsen.util 8564 :ok :read [4 21] INFO [2019-12-17 12:34:23,820] jepsen worker 65 - jepsen.util 6905 :fail :cas [21 [40 32]] :ValueNotSwapFrom INFO [2019-12-17 12:34:23,820] jepsen worker 14 - jepsen.util 8564 :invoke :cas [4 [1 29]] INFO [2019-12-17 12:34:23,820] jepsen worker 65 - jepsen.util 6905 :invoke :cas [21 [30 29]] INFO [2019-12-17 12:34:23,822] jepsen worker 12 - jepsen.util 8922 :ok :write [4 34] INFO [2019-12-17 12:34:23,822] jepsen worker 12 - jepsen.util 8922 :invoke :read [4 nil] INFO [2019-12-17 12:34:23,822] jepsen worker 37 - jepsen.util 7867 :ok :write [12 12] INFO [2019-12-17 12:34:23,823] jepsen worker 37 - jepsen.util 7867 :invoke :read [12 nil] INFO [2019-12-17 12:34:23,828] jepsen worker 38 - jepsen.util 8498 :ok :read [12 27] INFO [2019-12-17 12:34:23,828] jepsen worker 38 - jepsen.util 8498 :invoke :write [12 32] INFO [2019-12-17 12:34:23,829] jepsen worker 75 - jepsen.util 10605 :info :write [25 4] :RequestCanceledException INFO [2019-12-17 12:34:23,830] jepsen worker 75 - jepsen.util 10695 :invoke :cas [25 [40 18]] INFO [2019-12-17 12:34:23,848] jepsen worker 65 - jepsen.util 6905 :fail :cas [21 [30 29]] :ValueNotSwapFrom INFO [2019-12-17 12:34:23,848] jepsen worker 65 - jepsen.util 6905 :invoke :cas [21 [35 49]] INFO [2019-12-17 12:34:23,849] jepsen worker 3 - jepsen.util 8103 :fail :cas [1 [17 43]] :ValueNotSwapFrom INFO [2019-12-17 12:34:23,849] jepsen worker 3 - jepsen.util 8103 :invoke :cas [1 [32 18]] INFO [2019-12-17 12:34:23,854] jepsen worker 70 - jepsen.util 8620 :ok :write [23 42] INFO [2019-12-17 12:34:23,854] jepsen worker 70 - jepsen.util 8620 :invoke :read [23 nil] INFO [2019-12-17 12:34:23,860] jepsen worker 49 - jepsen.util 10669 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:34:23,860] jepsen worker 49 - jepsen.util 10669 :invoke :cas [16 [6 19]] INFO [2019-12-17 12:34:23,880] jepsen worker 31 - jepsen.util 7321 :info :write [10 23] :SyncWriteAmbiguous INFO [2019-12-17 12:34:23,885] jepsen worker 31 - jepsen.util 7411 :invoke :write [10 33] INFO [2019-12-17 12:34:23,886] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47655,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:34:23,888] jepsen worker 14 - jepsen.util 8564 :fail :cas [4 [1 29]] :ValueNotSwapFrom INFO [2019-12-17 12:34:23,888] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47655,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:34:23,888] jepsen worker 14 - jepsen.util 8564 :invoke :read [4 nil] INFO [2019-12-17 12:34:23,888] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47656,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:34:23,889] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47656,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:34:23,889] jepsen worker 65 - jepsen.util 6905 :fail :cas [21 [35 49]] :ValueNotSwapFrom INFO [2019-12-17 12:34:23,889] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47657,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:34:23,889] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47657,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:34:23,889] jepsen worker 65 - jepsen.util 6905 :invoke :write [21 1] INFO [2019-12-17 12:34:23,889] jepsen worker 36 - jepsen.util 8586 :fail :cas [12 [34 23]] :ValueNotSwapFrom INFO [2019-12-17 12:34:23,889] jepsen worker 36 - jepsen.util 8586 :invoke :write [12 1] INFO [2019-12-17 12:34:23,905] jepsen worker 13 - jepsen.util 7843 :fail :cas [4 [3 38]] :ValueNotSwapFrom INFO [2019-12-17 12:34:23,905] jepsen worker 13 - jepsen.util 7843 :invoke :write [4 41] INFO [2019-12-17 12:34:23,916] jepsen worker 11 - jepsen.util 8291 :ok :read [3 48] INFO [2019-12-17 12:34:23,916] jepsen worker 11 - jepsen.util 8291 :invoke :read [3 nil] INFO [2019-12-17 12:34:23,929] jepsen worker 2 - jepsen.util 10802 :info :cas [0 [22 36]] :RequestCanceledException INFO [2019-12-17 12:34:23,934] jepsen worker 2 - jepsen.util 10892 :invoke :cas [0 [0 27]] INFO [2019-12-17 12:34:23,935] jepsen worker 64 - jepsen.util 6814 :fail :cas [21 [31 23]] :ValueNotSwapFrom INFO [2019-12-17 12:34:23,935] jepsen worker 64 - jepsen.util 6814 :invoke :cas [21 [5 15]] INFO [2019-12-17 12:34:23,940] jepsen worker 10 - jepsen.util 8470 :ok :read [3 48] INFO [2019-12-17 12:34:23,940] jepsen worker 10 - jepsen.util 8470 :invoke :read [3 nil] INFO [2019-12-17 12:34:23,956] jepsen worker 65 - jepsen.util 6905 :ok :write [21 1] INFO [2019-12-17 12:34:23,956] jepsen worker 65 - jepsen.util 6905 :invoke :read [21 nil] INFO [2019-12-17 12:34:23,966] jepsen worker 71 - jepsen.util 8891 :fail :cas [23 [7 23]] :ValueNotSwapFrom INFO [2019-12-17 12:34:23,968] jepsen worker 71 - jepsen.util 8891 :invoke :write [23 36] INFO [2019-12-17 12:34:23,981] jepsen worker 70 - jepsen.util 8620 :ok :read [23 42] INFO [2019-12-17 12:34:23,981] jepsen worker 70 - jepsen.util 8620 :invoke :cas [23 [19 29]] INFO [2019-12-17 12:34:24,008] jepsen worker 46 - jepsen.util 8416 :fail :cas [15 [7 20]] :ValueNotSwapFrom INFO [2019-12-17 12:34:24,008] jepsen worker 46 - jepsen.util 8416 :invoke :cas [15 [25 6]] INFO [2019-12-17 12:34:24,010] jepsen worker 53 - jepsen.util 7703 :ok :write [17 27] INFO [2019-12-17 12:34:24,010] jepsen worker 53 - jepsen.util 7703 :invoke :read [17 nil] INFO [2019-12-17 12:34:24,012] jepsen worker 65 - jepsen.util 6905 :ok :read [21 1] INFO [2019-12-17 12:34:24,012] jepsen worker 65 - jepsen.util 6905 :invoke :read [21 nil] INFO [2019-12-17 12:34:24,012] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47673,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:34:24,013] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47673,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:34:24,023] jepsen worker 38 - jepsen.util 8498 :ok :write [12 32] INFO [2019-12-17 12:34:24,023] jepsen worker 38 - jepsen.util 8498 :invoke :cas [12 [1 6]] INFO [2019-12-17 12:34:24,030] jepsen worker 18 - jepsen.util 8388 :info :cas [6 [17 25]] :RequestCanceledException INFO [2019-12-17 12:34:24,031] jepsen worker 18 - jepsen.util 8478 :invoke :write [6 33] INFO [2019-12-17 12:34:24,055] jepsen worker 65 - jepsen.util 6905 :ok :read [21 1] INFO [2019-12-17 12:34:24,055] jepsen worker 65 - jepsen.util 6905 :invoke :read [21 nil] INFO [2019-12-17 12:34:24,059] jepsen worker 87 - jepsen.util 7557 :info :write [29 5] :RequestCanceledException INFO [2019-12-17 12:34:24,060] jepsen worker 87 - jepsen.util 7647 :invoke :read [29 nil] INFO [2019-12-17 12:34:24,079] jepsen worker 65 - jepsen.util 6905 :ok :read [21 1] INFO [2019-12-17 12:34:24,079] jepsen worker 65 - jepsen.util 6905 :invoke :cas [21 [3 48]] INFO [2019-12-17 12:34:24,085] jepsen worker 14 - jepsen.util 8564 :ok :read [4 34] INFO [2019-12-17 12:34:24,086] jepsen worker 14 - jepsen.util 8564 :invoke :read [4 nil] INFO [2019-12-17 12:34:24,102] jepsen worker 65 - jepsen.util 6905 :fail :cas [21 [3 48]] :ValueNotSwapFrom INFO [2019-12-17 12:34:24,102] jepsen worker 65 - jepsen.util 6905 :invoke :write [21 0] INFO [2019-12-17 12:34:24,121] jepsen worker 3 - jepsen.util 8103 :fail :cas [1 [32 18]] :ValueNotSwapFrom INFO [2019-12-17 12:34:24,121] jepsen worker 3 - jepsen.util 8103 :invoke :read [1 nil] INFO [2019-12-17 12:34:24,129] jepsen worker 41 - jepsen.util 11831 :info :write [13 44] :RequestCanceledException INFO [2019-12-17 12:34:24,129] jepsen worker 81 - jepsen.util 11061 :info :cas [27 [43 40]] :RequestCanceledException INFO [2019-12-17 12:34:24,135] jepsen worker 41 - jepsen.util 11921 :invoke :read [13 nil] INFO [2019-12-17 12:34:24,139] jepsen worker 81 - jepsen.util 11151 :invoke :write [27 35] INFO [2019-12-17 12:34:24,141] jepsen worker 38 - jepsen.util 8498 :fail :cas [12 [1 6]] :ValueNotSwapFrom INFO [2019-12-17 12:34:24,141] jepsen worker 38 - jepsen.util 8498 :invoke :read [12 nil] INFO [2019-12-17 12:34:24,143] jepsen worker 19 - jepsen.util 8389 :info :write [6 39] :RequestCanceledException INFO [2019-12-17 12:34:24,143] jepsen worker 63 - jepsen.util 6993 :fail :read [21 nil] :RequestTimeoutException INFO [2019-12-17 12:34:24,143] jepsen worker 63 - jepsen.util 6993 :invoke :write [21 15] INFO [2019-12-17 12:34:24,158] jepsen worker 19 - jepsen.util 8479 :invoke :read [6 nil] INFO [2019-12-17 12:34:24,158] jepsen worker 5 - jepsen.util 7745 :ok :write [1 35] INFO [2019-12-17 12:34:24,158] jepsen worker 5 - jepsen.util 7745 :invoke :read [1 nil] INFO [2019-12-17 12:34:24,162] jepsen worker 86 - jepsen.util 11966 :info :cas [28 [12 17]] :RequestCanceledException INFO [2019-12-17 12:34:24,170] jepsen worker 86 - jepsen.util 12056 :invoke :write [28 14] INFO [2019-12-17 12:34:24,172] jepsen worker 89 - jepsen.util 8189 :ok :write [29 15] INFO [2019-12-17 12:34:24,172] jepsen worker 89 - jepsen.util 8189 :invoke :write [29 44] INFO [2019-12-17 12:34:24,178] jepsen worker 14 - jepsen.util 8564 :ok :read [4 34] INFO [2019-12-17 12:34:24,178] jepsen worker 14 - jepsen.util 8564 :invoke :cas [4 [15 30]] INFO [2019-12-17 12:34:24,189] jepsen worker 65 - jepsen.util 6905 :ok :write [21 0] INFO [2019-12-17 12:34:24,189] jepsen worker 65 - jepsen.util 6905 :invoke :read [21 nil] INFO [2019-12-17 12:34:24,196] jepsen worker 59 - jepsen.util 10859 :info :write [19 33] :RequestCanceledException INFO [2019-12-17 12:34:24,199] jepsen worker 59 - jepsen.util 10949 :invoke :read [19 nil] INFO [2019-12-17 12:34:24,206] jepsen worker 65 - jepsen.util 6905 :ok :read [21 0] INFO [2019-12-17 12:34:24,206] jepsen worker 65 - jepsen.util 6905 :invoke :cas [21 [46 28]] INFO [2019-12-17 12:34:24,214] jepsen worker 39 - jepsen.util 10749 :info :cas [13 [30 12]] :RequestCanceledException INFO [2019-12-17 12:34:24,215] jepsen worker 39 - jepsen.util 10839 :invoke :read [13 nil] INFO [2019-12-17 12:34:24,221] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47697,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:34:24,222] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47697,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:34:24,222] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47699,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:34:24,222] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47699,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:34:24,226] jepsen worker 79 - jepsen.util 8629 :ok :read [26 21] INFO [2019-12-17 12:34:24,226] jepsen worker 79 - jepsen.util 8629 :invoke :read [26 nil] INFO [2019-12-17 12:34:24,227] jepsen worker 47 - jepsen.util 8687 :info :write [15 29] :SyncWriteAmbiguous INFO [2019-12-17 12:34:24,227] jepsen worker 63 - jepsen.util 6993 :ok :write [21 15] INFO [2019-12-17 12:34:24,227] jepsen worker 63 - jepsen.util 6993 :invoke :read [21 nil] INFO [2019-12-17 12:34:24,228] jepsen worker 47 - jepsen.util 8777 :invoke :write [15 47] INFO [2019-12-17 12:34:24,233] jepsen worker 65 - jepsen.util 6905 :fail :cas [21 [46 28]] :ValueNotSwapFrom INFO [2019-12-17 12:34:24,233] jepsen worker 65 - jepsen.util 6905 :invoke :cas [21 [41 24]] INFO [2019-12-17 12:34:24,243] jepsen worker 74 - jepsen.util 8444 :info :cas [24 [26 31]] :RequestCanceledException INFO [2019-12-17 12:34:24,243] jepsen worker 30 - jepsen.util 8310 :fail :read [10 nil] :RequestTimeoutException INFO [2019-12-17 12:34:24,243] jepsen worker 21 - jepsen.util 10641 :info :cas [7 [29 15]] :RequestCanceledException INFO [2019-12-17 12:34:24,243] jepsen worker 29 - jepsen.util 6689 :fail :read [9 nil] :RequestTimeoutException INFO [2019-12-17 12:34:24,243] jepsen worker 30 - jepsen.util 8310 :invoke :write [10 12] INFO [2019-12-17 12:34:24,243] jepsen worker 29 - jepsen.util 6689 :invoke :write [9 14] INFO [2019-12-17 12:34:24,244] jepsen worker 74 - jepsen.util 8534 :invoke :cas [24 [18 7]] INFO [2019-12-17 12:34:24,244] jepsen worker 21 - jepsen.util 10731 :invoke :cas [7 [32 22]] INFO [2019-12-17 12:34:24,245] jepsen worker 28 - jepsen.util 6508 :fail :read [9 nil] :RequestTimeoutException INFO [2019-12-17 12:34:24,245] jepsen worker 28 - jepsen.util 6508 :invoke :write [9 24] INFO [2019-12-17 12:34:24,254] jepsen worker 3 - jepsen.util 8103 :ok :read [1 35] INFO [2019-12-17 12:34:24,254] jepsen worker 3 - jepsen.util 8103 :invoke :cas [1 [19 42]] INFO [2019-12-17 12:34:24,256] jepsen worker 65 - jepsen.util 6905 :fail :cas [21 [41 24]] :ValueNotSwapFrom INFO [2019-12-17 12:34:24,256] jepsen worker 65 - jepsen.util 6905 :invoke :write [21 47] INFO [2019-12-17 12:34:24,273] jepsen worker 4 - jepsen.util 7744 :fail :cas [1 [8 11]] :ValueNotSwapFrom INFO [2019-12-17 12:34:24,273] jepsen worker 4 - jepsen.util 7744 :invoke :cas [1 [29 14]] INFO [2019-12-17 12:34:24,282] jepsen worker 78 - jepsen.util 8088 :ok :read [26 21] INFO [2019-12-17 12:34:24,282] jepsen worker 78 - jepsen.util 8088 :invoke :write [26 34] INFO [2019-12-17 12:34:24,285] jepsen worker 63 - jepsen.util 6993 :ok :read [21 15] INFO [2019-12-17 12:34:24,285] jepsen worker 63 - jepsen.util 6993 :invoke :write [21 4] INFO [2019-12-17 12:34:24,296] jepsen worker 82 - jepsen.util 11782 :info :write [27 2] :RequestCanceledException INFO [2019-12-17 12:34:24,296] jepsen worker 82 - jepsen.util 11872 :invoke :cas [27 [21 3]] INFO [2019-12-17 12:34:24,309] jepsen worker 38 - jepsen.util 8498 :ok :read [12 32] INFO [2019-12-17 12:34:24,309] jepsen worker 38 - jepsen.util 8498 :invoke :read [12 nil] INFO [2019-12-17 12:34:24,313] jepsen worker 23 - jepsen.util 11093 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:34:24,313] jepsen worker 23 - jepsen.util 11093 :invoke :read [7 nil] INFO [2019-12-17 12:34:24,319] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47706,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:34:24,319] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47706,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:34:24,319] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47709,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:34:24,319] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47709,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:34:24,319] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47710,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:34:24,319] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47710,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:34:24,329] jepsen worker 50 - jepsen.util 12020 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:34:24,329] jepsen worker 76 - jepsen.util 11146 :info :write [25 38] :RequestCanceledException INFO [2019-12-17 12:34:24,329] jepsen worker 68 - jepsen.util 10508 :info :write [22 1] :RequestCanceledException INFO [2019-12-17 12:34:24,329] jepsen worker 50 - jepsen.util 12020 :invoke :read [16 nil] INFO [2019-12-17 12:34:24,330] jepsen worker 68 - jepsen.util 10598 :invoke :cas [22 [0 40]] INFO [2019-12-17 12:34:24,330] jepsen worker 76 - jepsen.util 11236 :invoke :write [25 36] INFO [2019-12-17 12:34:24,334] jepsen worker 65 - jepsen.util 6905 :ok :write [21 47] INFO [2019-12-17 12:34:24,334] jepsen worker 65 - jepsen.util 6905 :invoke :cas [21 [6 39]] INFO [2019-12-17 12:34:24,343] jepsen worker 60 - jepsen.util 12120 :info :write [20 43] :RequestCanceledException INFO [2019-12-17 12:34:24,345] jepsen worker 12 - jepsen.util 8922 :ok :read [4 34] INFO [2019-12-17 12:34:24,345] jepsen worker 12 - jepsen.util 8922 :invoke :write [4 48] INFO [2019-12-17 12:34:24,345] jepsen worker 60 - jepsen.util 12210 :invoke :cas [20 [18 1]] INFO [2019-12-17 12:34:24,353] jepsen worker 14 - jepsen.util 8564 :fail :cas [4 [15 30]] :ValueNotSwapFrom INFO [2019-12-17 12:34:24,353] jepsen worker 14 - jepsen.util 8564 :invoke :write [4 6] INFO [2019-12-17 12:34:24,356] jepsen worker 65 - jepsen.util 6905 :fail :cas [21 [6 39]] :ValueNotSwapFrom INFO [2019-12-17 12:34:24,356] jepsen worker 65 - jepsen.util 6905 :invoke :cas [21 [44 49]] INFO [2019-12-17 12:34:24,361] jepsen worker 16 - jepsen.util 12436 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:34:24,361] jepsen worker 16 - jepsen.util 12436 :invoke :read [5 nil] INFO [2019-12-17 12:34:24,363] jepsen worker 83 - jepsen.util 11513 :info :write [27 6] :RequestCanceledException INFO [2019-12-17 12:34:24,367] jepsen worker 83 - jepsen.util 11603 :invoke :cas [27 [29 47]] INFO [2019-12-17 12:34:24,369] jepsen worker 3 - jepsen.util 8103 :fail :cas [1 [19 42]] :ValueNotSwapFrom INFO [2019-12-17 12:34:24,369] jepsen worker 3 - jepsen.util 8103 :invoke :cas [1 [49 36]] INFO [2019-12-17 12:34:24,395] jepsen worker 65 - jepsen.util 6905 :fail :cas [21 [44 49]] :ValueNotSwapFrom INFO [2019-12-17 12:34:24,395] jepsen worker 65 - jepsen.util 6905 :invoke :write [21 2] INFO [2019-12-17 12:34:24,396] jepsen worker 38 - jepsen.util 8498 :ok :read [12 32] INFO [2019-12-17 12:34:24,396] jepsen worker 38 - jepsen.util 8498 :invoke :read [12 nil] INFO [2019-12-17 12:34:24,414] jepsen worker 45 - jepsen.util 8415 :info :write [15 19] :RequestCanceledException INFO [2019-12-17 12:34:24,414] jepsen worker 27 - jepsen.util 8037 :info :cas [9 [24 24]] :RequestCanceledException INFO [2019-12-17 12:34:24,415] jepsen worker 27 - jepsen.util 8127 :invoke :read [9 nil] INFO [2019-12-17 12:34:24,415] jepsen worker 45 - jepsen.util 8505 :invoke :read [15 nil] INFO [2019-12-17 12:34:24,429] jepsen worker 34 - jepsen.util 11104 :info :cas [11 [6 1]] :RequestCanceledException INFO [2019-12-17 12:34:24,430] jepsen worker 34 - jepsen.util 11194 :invoke :read [11 nil] INFO [2019-12-17 12:34:24,435] jepsen worker 65 - jepsen.util 6905 :ok :write [21 2] INFO [2019-12-17 12:34:24,436] jepsen worker 65 - jepsen.util 6905 :invoke :read [21 nil] INFO [2019-12-17 12:34:24,436] jepsen worker 55 - jepsen.util 9055 :ok :write [18 34] INFO [2019-12-17 12:34:24,436] jepsen worker 55 - jepsen.util 9055 :invoke :read [18 nil] INFO [2019-12-17 12:34:24,448] jepsen worker 38 - jepsen.util 8498 :ok :read [12 32] INFO [2019-12-17 12:34:24,448] jepsen worker 38 - jepsen.util 8498 :invoke :read [12 nil] INFO [2019-12-17 12:34:24,458] jepsen worker 45 - jepsen.util 8505 :ok :read [15 17] INFO [2019-12-17 12:34:24,458] jepsen worker 45 - jepsen.util 8505 :invoke :write [15 43] INFO [2019-12-17 12:34:24,470] jepsen worker 71 - jepsen.util 8891 :ok :write [23 36] INFO [2019-12-17 12:34:24,470] jepsen worker 71 - jepsen.util 8891 :invoke :cas [23 [48 13]] INFO [2019-12-17 12:34:24,485] jepsen worker 56 - jepsen.util 9236 :ok :read [18 34] INFO [2019-12-17 12:34:24,485] jepsen worker 56 - jepsen.util 9236 :invoke :cas [18 [24 6]] INFO [2019-12-17 12:34:24,492] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:34:24,492] jepsen nemesis - couchbase.util Rebalance check count: 370 INFO [2019-12-17 12:34:24,492] jepsen nemesis - couchbase.util Rebalance stuck count: 351 INFO [2019-12-17 12:34:24,493] jepsen worker 65 - jepsen.util 6905 :ok :read [21 2] INFO [2019-12-17 12:34:24,493] jepsen worker 65 - jepsen.util 6905 :invoke :cas [21 [12 36]] INFO [2019-12-17 12:34:24,505] jepsen worker 3 - jepsen.util 8103 :fail :cas [1 [49 36]] :ValueNotSwapFrom INFO [2019-12-17 12:34:24,505] jepsen worker 3 - jepsen.util 8103 :invoke :cas [1 [23 48]] INFO [2019-12-17 12:34:24,530] jepsen worker 38 - jepsen.util 8498 :ok :read [12 32] INFO [2019-12-17 12:34:24,531] jepsen worker 38 - jepsen.util 8498 :invoke :cas [12 [43 47]] INFO [2019-12-17 12:34:24,537] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47746,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:34:24,537] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47746,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:34:24,537] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47750,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:34:24,537] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47750,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:34:24,539] jepsen worker 36 - jepsen.util 8586 :ok :write [12 1] INFO [2019-12-17 12:34:24,539] jepsen worker 36 - jepsen.util 8586 :invoke :cas [12 [27 33]] INFO [2019-12-17 12:34:24,542] jepsen worker 54 - jepsen.util 9144 :info :write [18 34] :RequestCanceledException INFO [2019-12-17 12:34:24,543] jepsen worker 54 - jepsen.util 9234 :invoke :read [18 nil] INFO [2019-12-17 12:34:24,553] jepsen worker 51 - jepsen.util 7701 :ok :write [17 30] INFO [2019-12-17 12:34:24,554] jepsen worker 51 - jepsen.util 7701 :invoke :cas [17 [46 6]] INFO [2019-12-17 12:34:24,571] jepsen worker 13 - jepsen.util 7843 :ok :write [4 41] INFO [2019-12-17 12:34:24,571] jepsen worker 13 - jepsen.util 7843 :invoke :write [4 28] INFO [2019-12-17 12:34:24,576] jepsen worker 4 - jepsen.util 7744 :fail :cas [1 [29 14]] :ValueNotSwapFrom INFO [2019-12-17 12:34:24,577] jepsen worker 4 - jepsen.util 7744 :invoke :write [1 11] INFO [2019-12-17 12:34:24,600] jepsen worker 87 - jepsen.util 7647 :ok :read [29 15] INFO [2019-12-17 12:34:24,600] jepsen worker 87 - jepsen.util 7647 :invoke :write [29 19] INFO [2019-12-17 12:34:24,621] jepsen worker 14 - jepsen.util 8564 :ok :write [4 6] INFO [2019-12-17 12:34:24,621] jepsen worker 14 - jepsen.util 8564 :invoke :write [4 40] INFO [2019-12-17 12:34:24,629] jepsen worker 35 - jepsen.util 11015 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:34:24,629] jepsen worker 35 - jepsen.util 11015 :invoke :read [11 nil] INFO [2019-12-17 12:34:24,630] jepsen worker 37 - jepsen.util 7867 :ok :read [12 1] INFO [2019-12-17 12:34:24,630] jepsen worker 37 - jepsen.util 7867 :invoke :write [12 22] INFO [2019-12-17 12:34:24,652] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47770,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:34:24,652] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47770,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:34:24,652] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47773,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:34:24,652] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47773,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:34:24,652] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47774,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:34:24,652] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47774,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:34:24,661] jepsen worker 67 - jepsen.util 11317 :info :cas [22 [2 5]] :RequestCanceledException INFO [2019-12-17 12:34:24,661] jepsen worker 67 - jepsen.util 11407 :invoke :read [22 nil] INFO [2019-12-17 12:34:24,674] jepsen worker 38 - jepsen.util 8498 :fail :cas [12 [43 47]] :ValueNotSwapFrom INFO [2019-12-17 12:34:24,674] jepsen worker 38 - jepsen.util 8498 :invoke :write [12 2] INFO [2019-12-17 12:34:24,711] jepsen worker 53 - jepsen.util 7703 :ok :read [17 30] INFO [2019-12-17 12:34:24,711] jepsen worker 53 - jepsen.util 7703 :invoke :read [17 nil] INFO [2019-12-17 12:34:24,745] jepsen worker 3 - jepsen.util 8103 :fail :cas [1 [23 48]] :ValueNotSwapFrom INFO [2019-12-17 12:34:24,745] jepsen worker 3 - jepsen.util 8103 :invoke :write [1 16] INFO [2019-12-17 12:34:24,752] jepsen worker 51 - jepsen.util 7701 :fail :cas [17 [46 6]] :ValueNotSwapFrom INFO [2019-12-17 12:34:24,752] jepsen worker 51 - jepsen.util 7701 :invoke :cas [17 [34 23]] INFO [2019-12-17 12:34:24,781] jepsen worker 89 - jepsen.util 8189 :ok :write [29 44] INFO [2019-12-17 12:34:24,781] jepsen worker 89 - jepsen.util 8189 :invoke :write [29 21] INFO [2019-12-17 12:34:24,789] jepsen worker 78 - jepsen.util 8088 :ok :write [26 34] INFO [2019-12-17 12:34:24,789] jepsen worker 78 - jepsen.util 8088 :invoke :write [26 36] INFO [2019-12-17 12:34:24,851] jepsen worker 14 - jepsen.util 8564 :ok :write [4 40] INFO [2019-12-17 12:34:24,851] jepsen worker 14 - jepsen.util 8564 :invoke :write [4 36] INFO [2019-12-17 12:34:24,854] jepsen worker 70 - jepsen.util 8620 :fail :cas [23 [19 29]] :ValueNotSwapFrom INFO [2019-12-17 12:34:24,854] jepsen worker 70 - jepsen.util 8620 :invoke :read [23 nil] INFO [2019-12-17 12:34:24,872] jepsen worker 37 - jepsen.util 7867 :ok :write [12 22] INFO [2019-12-17 12:34:24,872] jepsen worker 37 - jepsen.util 7867 :invoke :write [12 40] INFO [2019-12-17 12:34:24,882] jepsen worker 70 - jepsen.util 8620 :ok :read [23 36] INFO [2019-12-17 12:34:24,882] jepsen worker 70 - jepsen.util 8620 :invoke :cas [23 [48 42]] INFO [2019-12-17 12:34:24,916] jepsen worker 53 - jepsen.util 7703 :ok :read [17 30] INFO [2019-12-17 12:34:24,916] jepsen worker 53 - jepsen.util 7703 :invoke :cas [17 [35 27]] INFO [2019-12-17 12:34:24,920] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47787,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:34:24,920] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47787,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:34:24,932] jepsen worker 4 - jepsen.util 7744 :ok :write [1 11] INFO [2019-12-17 12:34:24,932] jepsen worker 4 - jepsen.util 7744 :invoke :write [1 25] INFO [2019-12-17 12:34:24,972] jepsen worker 71 - jepsen.util 8891 :fail :cas [23 [48 13]] :ValueNotSwapFrom INFO [2019-12-17 12:34:24,973] jepsen worker 71 - jepsen.util 8891 :invoke :write [23 17] INFO [2019-12-17 12:34:24,982] jepsen worker 36 - jepsen.util 8586 :fail :cas [12 [27 33]] :ValueNotSwapFrom INFO [2019-12-17 12:34:24,983] jepsen worker 36 - jepsen.util 8586 :invoke :cas [12 [41 26]] INFO [2019-12-17 12:34:24,996] jepsen worker 33 - jepsen.util 11193 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:34:24,996] jepsen worker 33 - jepsen.util 11193 :invoke :cas [11 [43 20]] INFO [2019-12-17 12:34:25,001] jepsen worker 51 - jepsen.util 7701 :fail :cas [17 [34 23]] :ValueNotSwapFrom INFO [2019-12-17 12:34:25,001] jepsen worker 51 - jepsen.util 7701 :invoke :write [17 31] INFO [2019-12-17 12:34:25,013] jepsen worker 57 - jepsen.util 11217 :info :write [19 44] :RequestCanceledException INFO [2019-12-17 12:34:25,014] jepsen worker 57 - jepsen.util 11307 :invoke :read [19 nil] INFO [2019-12-17 12:34:25,038] jepsen worker 70 - jepsen.util 8620 :fail :cas [23 [48 42]] :ValueNotSwapFrom INFO [2019-12-17 12:34:25,038] jepsen worker 70 - jepsen.util 8620 :invoke :read [23 nil] INFO [2019-12-17 12:34:25,043] jepsen worker 62 - jepsen.util 11492 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:34:25,043] jepsen worker 62 - jepsen.util 11492 :invoke :cas [20 [49 13]] INFO [2019-12-17 12:34:25,061] jepsen worker 25 - jepsen.util 11815 :info :cas [8 [19 14]] :RequestCanceledException INFO [2019-12-17 12:34:25,062] jepsen worker 25 - jepsen.util 11905 :invoke :write [8 0] INFO [2019-12-17 12:34:25,064] jepsen worker 5 - jepsen.util 7745 :ok :read [1 11] INFO [2019-12-17 12:34:25,064] jepsen worker 5 - jepsen.util 7745 :invoke :read [1 nil] INFO [2019-12-17 12:34:25,078] jepsen worker 87 - jepsen.util 7647 :ok :write [29 19] INFO [2019-12-17 12:34:25,078] jepsen worker 87 - jepsen.util 7647 :invoke :cas [29 [16 20]] INFO [2019-12-17 12:34:25,099] jepsen worker 79 - jepsen.util 8629 :ok :read [26 34] INFO [2019-12-17 12:34:25,100] jepsen worker 79 - jepsen.util 8629 :invoke :read [26 nil] INFO [2019-12-17 12:34:25,104] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47797,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:34:25,104] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47797,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:34:25,104] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47801,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:34:25,104] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47801,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:34:25,105] jepsen worker 69 - jepsen.util 8439 :ok :write [23 19] INFO [2019-12-17 12:34:25,105] jepsen worker 69 - jepsen.util 8439 :invoke :read [23 nil] INFO [2019-12-17 12:34:25,115] jepsen worker 54 - jepsen.util 9234 :ok :read [18 34] INFO [2019-12-17 12:34:25,115] jepsen worker 54 - jepsen.util 9234 :invoke :cas [18 [47 8]] INFO [2019-12-17 12:34:25,129] jepsen worker 0 - jepsen.util 11700 :info :cas [0 [11 10]] :RequestCanceledException INFO [2019-12-17 12:34:25,129] jepsen worker 58 - jepsen.util 12118 :info :write [19 12] :RequestCanceledException INFO [2019-12-17 12:34:25,129] jepsen worker 88 - jepsen.util 8368 :info :write [29 39] :RequestCanceledException INFO [2019-12-17 12:34:25,130] jepsen worker 0 - jepsen.util 11790 :invoke :read [0 nil] INFO [2019-12-17 12:34:25,130] jepsen worker 88 - jepsen.util 8458 :invoke :write [29 12] INFO [2019-12-17 12:34:25,131] jepsen worker 58 - jepsen.util 12208 :invoke :write [19 29] INFO [2019-12-17 12:34:25,136] jepsen worker 38 - jepsen.util 8498 :ok :write [12 2] INFO [2019-12-17 12:34:25,137] jepsen worker 38 - jepsen.util 8498 :invoke :write [12 6] INFO [2019-12-17 12:34:25,144] jepsen worker 85 - jepsen.util 11965 :info :cas [28 [48 0]] :RequestCanceledException INFO [2019-12-17 12:34:25,145] jepsen worker 85 - jepsen.util 12055 :invoke :cas [28 [27 20]] INFO [2019-12-17 12:34:25,178] jepsen worker 12 - jepsen.util 8922 :ok :write [4 48] INFO [2019-12-17 12:34:25,178] jepsen worker 12 - jepsen.util 8922 :invoke :cas [4 [35 10]] INFO [2019-12-17 12:34:25,189] jepsen worker 71 - jepsen.util 8891 :ok :write [23 17] INFO [2019-12-17 12:34:25,189] jepsen worker 71 - jepsen.util 8891 :invoke :cas [23 [42 10]] INFO [2019-12-17 12:34:25,208] jepsen worker 70 - jepsen.util 8620 :ok :read [23 17] INFO [2019-12-17 12:34:25,208] jepsen worker 70 - jepsen.util 8620 :invoke :cas [23 [34 31]] INFO [2019-12-17 12:34:25,208] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47821,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:34:25,208] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47821,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:34:25,208] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47822,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:34:25,208] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47822,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:34:25,209] jepsen worker 71 - jepsen.util 8891 :fail :cas [23 [42 10]] :ValueNotSwapFrom INFO [2019-12-17 12:34:25,209] jepsen worker 71 - jepsen.util 8891 :invoke :cas [23 [1 11]] INFO [2019-12-17 12:34:25,216] jepsen worker 37 - jepsen.util 7867 :ok :write [12 40] INFO [2019-12-17 12:34:25,216] jepsen worker 37 - jepsen.util 7867 :invoke :cas [12 [44 23]] INFO [2019-12-17 12:34:25,220] jepsen worker 70 - jepsen.util 8620 :fail :cas [23 [34 31]] :ValueNotSwapFrom INFO [2019-12-17 12:34:25,221] jepsen worker 70 - jepsen.util 8620 :invoke :write [23 0] INFO [2019-12-17 12:34:25,254] jepsen worker 71 - jepsen.util 8891 :fail :cas [23 [1 11]] :ValueNotSwapFrom INFO [2019-12-17 12:34:25,254] jepsen worker 71 - jepsen.util 8891 :invoke :read [23 nil] INFO [2019-12-17 12:34:25,278] jepsen worker 87 - jepsen.util 7647 :fail :cas [29 [16 20]] :ValueNotSwapFrom INFO [2019-12-17 12:34:25,278] jepsen worker 87 - jepsen.util 7647 :invoke :write [29 21] INFO [2019-12-17 12:34:25,285] jepsen worker 78 - jepsen.util 8088 :ok :write [26 36] INFO [2019-12-17 12:34:25,286] jepsen worker 78 - jepsen.util 8088 :invoke :write [26 49] INFO [2019-12-17 12:34:25,296] jepsen worker 48 - jepsen.util 10398 :info :write [16 43] :RequestCanceledException INFO [2019-12-17 12:34:25,297] jepsen worker 48 - jepsen.util 10488 :invoke :cas [16 [13 29]] INFO [2019-12-17 12:34:25,297] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47829,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:34:25,297] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47829,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:34:25,297] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47832,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:34:25,298] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47832,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:34:25,313] jepsen worker 61 - jepsen.util 12211 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:34:25,313] jepsen worker 22 - jepsen.util 11272 :info :cas [7 [6 6]] :RequestCanceledException INFO [2019-12-17 12:34:25,313] jepsen worker 61 - jepsen.util 12211 :invoke :read [20 nil] INFO [2019-12-17 12:34:25,314] jepsen worker 22 - jepsen.util 11362 :invoke :read [7 nil] INFO [2019-12-17 12:34:25,329] jepsen worker 6 - jepsen.util 11886 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:34:25,329] jepsen worker 26 - jepsen.util 11726 :info :cas [8 [45 13]] :RequestCanceledException INFO [2019-12-17 12:34:25,329] jepsen worker 6 - jepsen.util 11886 :invoke :write [2 9] INFO [2019-12-17 12:34:25,330] jepsen worker 26 - jepsen.util 11816 :invoke :write [8 3] INFO [2019-12-17 12:34:25,338] jepsen worker 54 - jepsen.util 9234 :fail :cas [18 [47 8]] :ValueNotSwapFrom INFO [2019-12-17 12:34:25,338] jepsen worker 54 - jepsen.util 9234 :invoke :write [18 0] INFO [2019-12-17 12:34:25,353] jepsen worker 20 - jepsen.util 7940 :info :write [6 7] :RequestCanceledException INFO [2019-12-17 12:34:25,355] jepsen worker 20 - jepsen.util 8030 :invoke :write [6 42] INFO [2019-12-17 12:34:25,363] jepsen worker 3 - jepsen.util 8103 :ok :write [1 16] INFO [2019-12-17 12:34:25,363] jepsen worker 3 - jepsen.util 8103 :invoke :cas [1 [6 19]] INFO [2019-12-17 12:34:25,390] jepsen worker 71 - jepsen.util 8891 :ok :read [23 17] INFO [2019-12-17 12:34:25,390] jepsen worker 71 - jepsen.util 8891 :invoke :cas [23 [0 33]] INFO [2019-12-17 12:34:25,396] jepsen worker 84 - jepsen.util 10884 :info :write [28 38] :RequestCanceledException INFO [2019-12-17 12:34:25,396] jepsen worker 84 - jepsen.util 10974 :invoke :read [28 nil] INFO [2019-12-17 12:34:25,399] jepsen worker 88 - jepsen.util 8458 :ok :write [29 12] INFO [2019-12-17 12:34:25,399] jepsen worker 88 - jepsen.util 8458 :invoke :write [29 27] INFO [2019-12-17 12:34:25,410] jepsen worker 69 - jepsen.util 8439 :ok :read [23 17] INFO [2019-12-17 12:34:25,410] jepsen worker 69 - jepsen.util 8439 :invoke :write [23 3] INFO [2019-12-17 12:34:25,414] jepsen worker 66 - jepsen.util 11136 :info :write [22 15] :RequestCanceledException INFO [2019-12-17 12:34:25,415] jepsen worker 66 - jepsen.util 11226 :invoke :cas [22 [30 12]] INFO [2019-12-17 12:34:25,428] jepsen worker 44 - jepsen.util 11384 :info :cas [14 [45 5]] :RequestCanceledException INFO [2019-12-17 12:34:25,428] jepsen worker 52 - jepsen.util 8242 :info :write [17 35] :RequestCanceledException INFO [2019-12-17 12:34:25,429] jepsen worker 44 - jepsen.util 11474 :invoke :cas [14 [3 46]] INFO [2019-12-17 12:34:25,430] jepsen worker 52 - jepsen.util 8332 :invoke :write [17 12] INFO [2019-12-17 12:34:25,436] jepsen worker 9 - jepsen.util 8649 :ok :write [3 43] INFO [2019-12-17 12:34:25,436] jepsen worker 9 - jepsen.util 8649 :invoke :write [3 13] INFO [2019-12-17 12:34:25,436] jepsen worker 31 - jepsen.util 7411 :ok :write [10 33] INFO [2019-12-17 12:34:25,436] jepsen worker 31 - jepsen.util 7411 :invoke :write [10 33] INFO [2019-12-17 12:34:25,486] jepsen worker 64 - jepsen.util 6814 :fail :cas [21 [5 15]] :ValueNotSwapFrom INFO [2019-12-17 12:34:25,486] jepsen worker 64 - jepsen.util 6814 :invoke :read [21 nil] INFO [2019-12-17 12:34:25,496] jepsen worker 40 - jepsen.util 11560 :info :write [13 13] :RequestCanceledException INFO [2019-12-17 12:34:25,497] jepsen worker 40 - jepsen.util 11650 :invoke :read [13 nil] INFO [2019-12-17 12:34:25,508] jepsen worker 47 - jepsen.util 8777 :ok :write [15 47] INFO [2019-12-17 12:34:25,508] jepsen worker 47 - jepsen.util 8777 :invoke :write [15 41] INFO [2019-12-17 12:34:25,515] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47855,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:34:25,515] jepsen worker 71 - jepsen.util 8891 :fail :cas [23 [0 33]] :ValueNotSwapFrom INFO [2019-12-17 12:34:25,515] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47855,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:34:25,515] jepsen worker 71 - jepsen.util 8891 :invoke :read [23 nil] INFO [2019-12-17 12:34:25,515] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47857,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:34:25,515] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47857,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:34:25,515] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:34:25,515] jepsen nemesis - couchbase.util Rebalance check count: 371 INFO [2019-12-17 12:34:25,515] jepsen nemesis - couchbase.util Rebalance stuck count: 352 INFO [2019-12-17 12:34:25,520] jepsen worker 10 - jepsen.util 8470 :ok :read [3 43] INFO [2019-12-17 12:34:25,521] jepsen worker 10 - jepsen.util 8470 :invoke :write [3 16] INFO [2019-12-17 12:34:25,527] jepsen worker 46 - jepsen.util 8416 :fail :cas [15 [25 6]] :ValueNotSwapFrom INFO [2019-12-17 12:34:25,528] jepsen worker 46 - jepsen.util 8416 :invoke :read [15 nil] INFO [2019-12-17 12:34:25,538] jepsen worker 53 - jepsen.util 7703 :fail :cas [17 [35 27]] :ValueNotSwapFrom INFO [2019-12-17 12:34:25,538] jepsen worker 53 - jepsen.util 7703 :invoke :read [17 nil] INFO [2019-12-17 12:34:25,548] jepsen worker 3 - jepsen.util 8103 :fail :cas [1 [6 19]] :ValueNotSwapFrom INFO [2019-12-17 12:34:25,548] jepsen worker 3 - jepsen.util 8103 :invoke :cas [1 [48 48]] INFO [2019-12-17 12:34:25,559] jepsen worker 37 - jepsen.util 7867 :fail :cas [12 [44 23]] :ValueNotSwapFrom INFO [2019-12-17 12:34:25,559] jepsen worker 37 - jepsen.util 7867 :invoke :read [12 nil] INFO [2019-12-17 12:34:25,560] jepsen worker 32 - jepsen.util 8402 :info :write [10 47] :RequestCanceledException INFO [2019-12-17 12:34:25,561] jepsen worker 32 - jepsen.util 8492 :invoke :read [10 nil] INFO [2019-12-17 12:34:25,566] jepsen worker 19 - jepsen.util 8479 :ok :read [6 7] INFO [2019-12-17 12:34:25,566] jepsen worker 19 - jepsen.util 8479 :invoke :read [6 nil] INFO [2019-12-17 12:34:25,573] jepsen worker 74 - jepsen.util 8534 :fail :cas [24 [18 7]] :ValueNotSwapFrom INFO [2019-12-17 12:34:25,573] jepsen worker 74 - jepsen.util 8534 :invoke :write [24 7] INFO [2019-12-17 12:34:25,592] jepsen worker 24 - jepsen.util 10284 :info :cas [8 [13 33]] :RequestCanceledException INFO [2019-12-17 12:34:25,594] jepsen worker 24 - jepsen.util 10374 :invoke :write [8 14] INFO [2019-12-17 12:34:25,597] jepsen worker 27 - jepsen.util 8127 :ok :read [9 20] INFO [2019-12-17 12:34:25,597] jepsen worker 27 - jepsen.util 8127 :invoke :write [9 18] INFO [2019-12-17 12:34:25,600] jepsen worker 46 - jepsen.util 8416 :ok :read [15 47] INFO [2019-12-17 12:34:25,600] jepsen worker 46 - jepsen.util 8416 :invoke :read [15 nil] INFO [2019-12-17 12:34:25,613] jepsen worker 15 - jepsen.util 10455 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:34:25,613] jepsen worker 15 - jepsen.util 10455 :invoke :write [5 14] INFO [2019-12-17 12:34:25,639] jepsen worker 36 - jepsen.util 8586 :fail :cas [12 [41 26]] :ValueNotSwapFrom INFO [2019-12-17 12:34:25,639] jepsen worker 36 - jepsen.util 8586 :invoke :cas [12 [29 31]] INFO [2019-12-17 12:34:25,642] jepsen worker 32 - jepsen.util 8492 :ok :read [10 33] INFO [2019-12-17 12:34:25,643] jepsen worker 32 - jepsen.util 8492 :invoke :read [10 nil] INFO [2019-12-17 12:34:25,643] jepsen worker 1 - jepsen.util 11521 :info :write [0 38] :RequestCanceledException INFO [2019-12-17 12:34:25,644] jepsen worker 1 - jepsen.util 11611 :invoke :write [0 18] INFO [2019-12-17 12:34:25,658] jepsen worker 5 - jepsen.util 7745 :ok :read [1 16] INFO [2019-12-17 12:34:25,659] jepsen worker 5 - jepsen.util 7745 :invoke :cas [1 [9 24]] INFO [2019-12-17 12:34:25,661] jepsen worker 42 - jepsen.util 12282 :info :cas [14 [39 0]] :RequestCanceledException INFO [2019-12-17 12:34:25,661] jepsen worker 17 - jepsen.util 11177 :info :cas [5 [32 30]] :RequestCanceledException INFO [2019-12-17 12:34:25,662] jepsen worker 17 - jepsen.util 11267 :invoke :write [5 45] INFO [2019-12-17 12:34:25,662] jepsen worker 42 - jepsen.util 12372 :invoke :cas [14 [1 42]] INFO [2019-12-17 12:34:25,668] jepsen worker 32 - jepsen.util 8492 :ok :read [10 33] INFO [2019-12-17 12:34:25,668] jepsen worker 32 - jepsen.util 8492 :invoke :cas [10 [15 31]] INFO [2019-12-17 12:34:25,676] jepsen worker 71 - jepsen.util 8891 :ok :read [23 17] INFO [2019-12-17 12:34:25,676] jepsen worker 71 - jepsen.util 8891 :invoke :read [23 nil] INFO [2019-12-17 12:34:25,681] jepsen worker 32 - jepsen.util 8492 :fail :cas [10 [15 31]] :ValueNotSwapFrom INFO [2019-12-17 12:34:25,681] jepsen worker 32 - jepsen.util 8492 :invoke :write [10 41] INFO [2019-12-17 12:34:25,696] jepsen worker 8 - jepsen.util 11168 :info :cas [2 [48 10]] :RequestCanceledException INFO [2019-12-17 12:34:25,697] jepsen worker 8 - jepsen.util 11258 :invoke :cas [2 [49 2]] INFO [2019-12-17 12:34:25,713] jepsen worker 79 - jepsen.util 8629 :ok :read [26 36] INFO [2019-12-17 12:34:25,713] jepsen worker 79 - jepsen.util 8629 :invoke :write [26 29] INFO [2019-12-17 12:34:25,714] jepsen worker 77 - jepsen.util 11597 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:34:25,714] jepsen worker 77 - jepsen.util 11597 :invoke :read [25 nil] INFO [2019-12-17 12:34:25,724] jepsen worker 12 - jepsen.util 8922 :fail :cas [4 [35 10]] :ValueNotSwapFrom INFO [2019-12-17 12:34:25,724] jepsen worker 12 - jepsen.util 8922 :invoke :cas [4 [47 23]] INFO [2019-12-17 12:34:25,729] jepsen worker 43 - jepsen.util 10933 :info :write [14 8] :RequestCanceledException INFO [2019-12-17 12:34:25,731] jepsen worker 43 - jepsen.util 11023 :invoke :cas [14 [27 44]] INFO [2019-12-17 12:34:25,748] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47881,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:34:25,748] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47881,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:34:25,748] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47884,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:34:25,748] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47884,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:34:25,754] jepsen worker 89 - jepsen.util 8189 :ok :write [29 21] INFO [2019-12-17 12:34:25,754] jepsen worker 89 - jepsen.util 8189 :invoke :write [29 34] INFO [2019-12-17 12:34:25,767] jepsen worker 47 - jepsen.util 8777 :ok :write [15 41] INFO [2019-12-17 12:34:25,768] jepsen worker 47 - jepsen.util 8777 :invoke :cas [15 [22 10]] INFO [2019-12-17 12:34:25,800] jepsen worker 47 - jepsen.util 8777 :fail :cas [15 [22 10]] :ValueNotSwapFrom INFO [2019-12-17 12:34:25,800] jepsen worker 47 - jepsen.util 8777 :invoke :write [15 36] INFO [2019-12-17 12:34:25,829] jepsen worker 73 - jepsen.util 7903 :info :write [24 33] :RequestCanceledException INFO [2019-12-17 12:34:25,829] jepsen worker 72 - jepsen.util 8172 :info :write [24 17] :RequestCanceledException INFO [2019-12-17 12:34:25,829] jepsen worker 80 - jepsen.util 8090 :info :write [26 49] :RequestCanceledException INFO [2019-12-17 12:34:25,830] jepsen worker 73 - jepsen.util 7993 :invoke :read [24 nil] INFO [2019-12-17 12:34:25,830] jepsen worker 80 - jepsen.util 8180 :invoke :cas [26 [27 32]] INFO [2019-12-17 12:34:25,830] jepsen worker 72 - jepsen.util 8262 :invoke :write [24 16] INFO [2019-12-17 12:34:25,849] jepsen worker 53 - jepsen.util 7703 :ok :read [17 30] INFO [2019-12-17 12:34:25,849] jepsen worker 53 - jepsen.util 7703 :invoke :cas [17 [27 5]] INFO [2019-12-17 12:34:25,855] jepsen worker 3 - jepsen.util 8103 :fail :cas [1 [48 48]] :ValueNotSwapFrom INFO [2019-12-17 12:34:25,855] jepsen worker 3 - jepsen.util 8103 :invoke :read [1 nil] INFO [2019-12-17 12:34:25,860] jepsen worker 75 - jepsen.util 10695 :info :cas [25 [40 18]] :RequestCanceledException INFO [2019-12-17 12:34:25,861] jepsen worker 75 - jepsen.util 10785 :invoke :cas [25 [48 13]] INFO [2019-12-17 12:34:25,879] jepsen worker 5 - jepsen.util 7745 :fail :cas [1 [9 24]] :ValueNotSwapFrom INFO [2019-12-17 12:34:25,879] jepsen worker 5 - jepsen.util 7745 :invoke :write [1 19] INFO [2019-12-17 12:34:25,890] jepsen worker 37 - jepsen.util 7867 :ok :read [12 40] INFO [2019-12-17 12:34:25,890] jepsen worker 37 - jepsen.util 7867 :invoke :write [12 9] INFO [2019-12-17 12:34:25,896] jepsen worker 7 - jepsen.util 11437 :info :write [2 42] :RequestCanceledException INFO [2019-12-17 12:34:25,897] jepsen worker 7 - jepsen.util 11527 :invoke :write [2 44] INFO [2019-12-17 12:34:25,901] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47908,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:34:25,901] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47908,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:34:25,901] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47911,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:34:25,901] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47911,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:34:25,901] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47912,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:34:25,902] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47912,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:34:25,916] jepsen worker 3 - jepsen.util 8103 :ok :read [1 16] INFO [2019-12-17 12:34:25,916] jepsen worker 3 - jepsen.util 8103 :invoke :write [1 10] INFO [2019-12-17 12:34:25,929] jepsen worker 11 - jepsen.util 8291 :fail :read [3 nil] :RequestTimeoutException INFO [2019-12-17 12:34:25,930] jepsen worker 11 - jepsen.util 8291 :invoke :read [3 nil] INFO [2019-12-17 12:34:25,960] jepsen worker 49 - jepsen.util 10669 :info :cas [16 [6 19]] :RequestCanceledException INFO [2019-12-17 12:34:25,961] jepsen worker 49 - jepsen.util 10759 :invoke :write [16 19] INFO [2019-12-17 12:34:25,985] jepsen worker 36 - jepsen.util 8586 :fail :cas [12 [29 31]] :ValueNotSwapFrom INFO [2019-12-17 12:34:25,985] jepsen worker 36 - jepsen.util 8586 :invoke :read [12 nil] INFO [2019-12-17 12:34:25,998] jepsen worker 2 - jepsen.util 10892 :info :cas [0 [0 27]] :RequestCanceledException INFO [2019-12-17 12:34:25,999] jepsen worker 2 - jepsen.util 10982 :invoke :read [0 nil] INFO [2019-12-17 12:34:26,005] jepsen worker 71 - jepsen.util 8891 :ok :read [23 17] INFO [2019-12-17 12:34:26,005] jepsen worker 71 - jepsen.util 8891 :invoke :cas [23 [36 47]] INFO [2019-12-17 12:34:26,063] jepsen worker 71 - jepsen.util 8891 :fail :cas [23 [36 47]] :ValueNotSwapFrom INFO [2019-12-17 12:34:26,063] jepsen worker 71 - jepsen.util 8891 :invoke :write [23 46] INFO [2019-12-17 12:34:26,072] jepsen worker 12 - jepsen.util 8922 :fail :cas [4 [47 23]] :ValueNotSwapFrom INFO [2019-12-17 12:34:26,072] jepsen worker 12 - jepsen.util 8922 :invoke :read [4 nil] INFO [2019-12-17 12:34:26,084] jepsen worker 55 - jepsen.util 9055 :ok :read [18 34] INFO [2019-12-17 12:34:26,084] jepsen worker 55 - jepsen.util 9055 :invoke :write [18 20] INFO [2019-12-17 12:34:26,085] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47923,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:34:26,085] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47923,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:34:26,085] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47925,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:34:26,085] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47925,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:34:26,088] jepsen worker 56 - jepsen.util 9236 :fail :cas [18 [24 6]] :ValueNotSwapFrom INFO [2019-12-17 12:34:26,089] jepsen worker 56 - jepsen.util 9236 :invoke :write [18 27] INFO [2019-12-17 12:34:26,114] jepsen worker 18 - jepsen.util 8478 :info :write [6 33] :RequestCanceledException INFO [2019-12-17 12:34:26,114] jepsen worker 18 - jepsen.util 8568 :invoke :write [6 30] INFO [2019-12-17 12:34:26,143] jepsen worker 47 - jepsen.util 8777 :ok :write [15 36] INFO [2019-12-17 12:34:26,143] jepsen worker 47 - jepsen.util 8777 :invoke :cas [15 [48 21]] INFO [2019-12-17 12:34:26,144] jepsen worker 31 - jepsen.util 7411 :ok :write [10 33] INFO [2019-12-17 12:34:26,144] jepsen worker 31 - jepsen.util 7411 :invoke :cas [10 [4 32]] INFO [2019-12-17 12:34:26,155] jepsen worker 9 - jepsen.util 8649 :ok :write [3 13] INFO [2019-12-17 12:34:26,155] jepsen worker 9 - jepsen.util 8649 :invoke :write [3 1] INFO [2019-12-17 12:34:26,160] jepsen worker 81 - jepsen.util 11151 :info :write [27 35] :RequestCanceledException INFO [2019-12-17 12:34:26,160] jepsen worker 74 - jepsen.util 8534 :ok :write [24 7] INFO [2019-12-17 12:34:26,161] jepsen worker 74 - jepsen.util 8534 :invoke :cas [24 [41 13]] INFO [2019-12-17 12:34:26,161] jepsen worker 81 - jepsen.util 11241 :invoke :write [27 37] INFO [2019-12-17 12:34:26,186] jepsen worker 80 - jepsen.util 8180 :fail :cas [26 [27 32]] :ValueNotSwapFrom INFO [2019-12-17 12:34:26,186] jepsen worker 80 - jepsen.util 8180 :invoke :write [26 46] INFO [2019-12-17 12:34:26,194] jepsen worker 19 - jepsen.util 8479 :ok :read [6 7] INFO [2019-12-17 12:34:26,194] jepsen worker 19 - jepsen.util 8479 :invoke :read [6 nil] INFO [2019-12-17 12:34:26,196] jepsen worker 86 - jepsen.util 12056 :info :write [28 14] :RequestCanceledException INFO [2019-12-17 12:34:26,198] jepsen worker 86 - jepsen.util 12146 :invoke :write [28 42] INFO [2019-12-17 12:34:26,213] jepsen worker 59 - jepsen.util 10949 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:34:26,213] jepsen worker 59 - jepsen.util 10949 :invoke :write [19 27] INFO [2019-12-17 12:34:26,215] jepsen worker 73 - jepsen.util 7993 :ok :read [24 23] INFO [2019-12-17 12:34:26,215] jepsen worker 46 - jepsen.util 8416 :ok :read [15 36] INFO [2019-12-17 12:34:26,215] jepsen worker 73 - jepsen.util 7993 :invoke :cas [24 [19 11]] INFO [2019-12-17 12:34:26,215] jepsen worker 46 - jepsen.util 8416 :invoke :read [15 nil] INFO [2019-12-17 12:34:26,228] jepsen worker 39 - jepsen.util 10839 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:34:26,228] jepsen worker 39 - jepsen.util 10839 :invoke :read [13 nil] INFO [2019-12-17 12:34:26,243] jepsen worker 41 - jepsen.util 11921 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:34:26,243] jepsen worker 41 - jepsen.util 11921 :invoke :cas [13 [0 34]] INFO [2019-12-17 12:34:26,260] jepsen worker 27 - jepsen.util 8127 :ok :write [9 18] INFO [2019-12-17 12:34:26,260] jepsen worker 27 - jepsen.util 8127 :invoke :read [9 nil] INFO [2019-12-17 12:34:26,267] jepsen worker 53 - jepsen.util 7703 :fail :cas [17 [27 5]] :ValueNotSwapFrom INFO [2019-12-17 12:34:26,267] jepsen worker 53 - jepsen.util 7703 :invoke :cas [17 [13 22]] INFO [2019-12-17 12:34:26,278] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47948,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:34:26,278] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47948,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:34:26,278] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47953,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:34:26,278] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47953,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:34:26,288] jepsen worker 31 - jepsen.util 7411 :fail :cas [10 [4 32]] :ValueNotSwapFrom INFO [2019-12-17 12:34:26,289] jepsen worker 31 - jepsen.util 7411 :invoke :write [10 48] INFO [2019-12-17 12:34:26,303] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47940,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:34:26,303] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47940,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:34:26,304] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47941,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:34:26,304] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47941,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:34:26,304] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47942,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:34:26,304] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47942,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:34:26,316] jepsen worker 19 - jepsen.util 8479 :ok :read [6 7] INFO [2019-12-17 12:34:26,316] jepsen worker 19 - jepsen.util 8479 :invoke :cas [6 [49 47]] INFO [2019-12-17 12:34:26,329] jepsen worker 21 - jepsen.util 10731 :info :cas [7 [32 22]] :RequestCanceledException INFO [2019-12-17 12:34:26,331] jepsen worker 21 - jepsen.util 10821 :invoke :cas [7 [10 18]] INFO [2019-12-17 12:34:26,338] jepsen worker 27 - jepsen.util 8127 :ok :read [9 18] INFO [2019-12-17 12:34:26,338] jepsen worker 27 - jepsen.util 8127 :invoke :cas [9 [9 5]] INFO [2019-12-17 12:34:26,343] jepsen worker 29 - jepsen.util 6689 :info :write [9 14] :RequestCanceledException INFO [2019-12-17 12:34:26,343] jepsen worker 28 - jepsen.util 6508 :info :write [9 24] :RequestCanceledException INFO [2019-12-17 12:34:26,343] jepsen worker 63 - jepsen.util 6993 :info :write [21 4] :RequestCanceledException INFO [2019-12-17 12:34:26,343] jepsen worker 30 - jepsen.util 8310 :info :write [10 12] :RequestCanceledException INFO [2019-12-17 12:34:26,344] jepsen worker 28 - jepsen.util 6598 :invoke :read [9 nil] INFO [2019-12-17 12:34:26,344] jepsen worker 29 - jepsen.util 6779 :invoke :write [9 42] INFO [2019-12-17 12:34:26,345] jepsen worker 63 - jepsen.util 7083 :invoke :read [21 nil] INFO [2019-12-17 12:34:26,346] jepsen worker 30 - jepsen.util 8400 :invoke :cas [10 [4 46]] INFO [2019-12-17 12:34:26,346] jepsen worker 72 - jepsen.util 8262 :ok :write [24 16] INFO [2019-12-17 12:34:26,347] jepsen worker 72 - jepsen.util 8262 :invoke :write [24 14] INFO [2019-12-17 12:34:26,377] jepsen worker 74 - jepsen.util 8534 :fail :cas [24 [41 13]] :ValueNotSwapFrom INFO [2019-12-17 12:34:26,377] jepsen worker 74 - jepsen.util 8534 :invoke :cas [24 [12 3]] INFO [2019-12-17 12:34:26,396] jepsen worker 82 - jepsen.util 11872 :info :cas [27 [21 3]] :RequestCanceledException INFO [2019-12-17 12:34:26,397] jepsen worker 82 - jepsen.util 11962 :invoke :read [27 nil] INFO [2019-12-17 12:34:26,413] jepsen worker 23 - jepsen.util 11093 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:34:26,413] jepsen worker 68 - jepsen.util 10598 :info :cas [22 [0 40]] :RequestCanceledException INFO [2019-12-17 12:34:26,413] jepsen worker 23 - jepsen.util 11093 :invoke :read [7 nil] INFO [2019-12-17 12:34:26,415] jepsen worker 68 - jepsen.util 10688 :invoke :write [22 9] INFO [2019-12-17 12:34:26,419] jepsen worker 28 - jepsen.util 6598 :ok :read [9 18] INFO [2019-12-17 12:34:26,419] jepsen worker 28 - jepsen.util 6598 :invoke :cas [9 [14 32]] INFO [2019-12-17 12:34:26,421] jepsen worker 46 - jepsen.util 8416 :ok :read [15 36] INFO [2019-12-17 12:34:26,422] jepsen worker 46 - jepsen.util 8416 :invoke :read [15 nil] INFO [2019-12-17 12:34:26,429] jepsen worker 76 - jepsen.util 11236 :info :write [25 36] :RequestCanceledException INFO [2019-12-17 12:34:26,429] jepsen worker 50 - jepsen.util 12020 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:34:26,430] jepsen worker 50 - jepsen.util 12020 :invoke :cas [16 [34 21]] INFO [2019-12-17 12:34:26,432] jepsen worker 76 - jepsen.util 11326 :invoke :write [25 0] INFO [2019-12-17 12:34:26,435] jepsen worker 30 - jepsen.util 8400 :fail :cas [10 [4 46]] :ValueNotSwapFrom INFO [2019-12-17 12:34:26,435] jepsen worker 30 - jepsen.util 8400 :invoke :read [10 nil] INFO [2019-12-17 12:34:26,438] jepsen worker 28 - jepsen.util 6598 :fail :cas [9 [14 32]] :ValueNotSwapFrom INFO [2019-12-17 12:34:26,438] jepsen worker 28 - jepsen.util 6598 :invoke :cas [9 [10 42]] INFO [2019-12-17 12:34:26,443] jepsen worker 60 - jepsen.util 12210 :info :cas [20 [18 1]] :RequestCanceledException INFO [2019-12-17 12:34:26,444] jepsen worker 60 - jepsen.util 12300 :invoke :write [20 10] INFO [2019-12-17 12:34:26,453] jepsen worker 36 - jepsen.util 8586 :ok :read [12 40] INFO [2019-12-17 12:34:26,453] jepsen worker 36 - jepsen.util 8586 :invoke :cas [12 [38 26]] INFO [2019-12-17 12:34:26,460] jepsen worker 16 - jepsen.util 12436 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:34:26,460] jepsen worker 83 - jepsen.util 11603 :info :cas [27 [29 47]] :RequestCanceledException INFO [2019-12-17 12:34:26,461] jepsen worker 16 - jepsen.util 12436 :invoke :read [5 nil] INFO [2019-12-17 12:34:26,462] jepsen worker 83 - jepsen.util 11693 :invoke :cas [27 [37 35]] INFO [2019-12-17 12:34:26,470] jepsen worker 28 - jepsen.util 6598 :fail :cas [9 [10 42]] :ValueNotSwapFrom INFO [2019-12-17 12:34:26,470] jepsen worker 28 - jepsen.util 6598 :invoke :read [9 nil] INFO [2019-12-17 12:34:26,484] jepsen worker 28 - jepsen.util 6598 :ok :read [9 18] INFO [2019-12-17 12:34:26,484] jepsen worker 28 - jepsen.util 6598 :invoke :cas [9 [46 35]] INFO [2019-12-17 12:34:26,491] jepsen worker 45 - jepsen.util 8505 :info :write [15 43] :RequestCanceledException INFO [2019-12-17 12:34:26,492] jepsen worker 45 - jepsen.util 8595 :invoke :read [15 nil] INFO [2019-12-17 12:34:26,495] jepsen worker 28 - jepsen.util 6598 :fail :cas [9 [46 35]] :ValueNotSwapFrom INFO [2019-12-17 12:34:26,495] jepsen worker 12 - jepsen.util 8922 :ok :read [4 48] INFO [2019-12-17 12:34:26,495] jepsen worker 28 - jepsen.util 6598 :invoke :read [9 nil] INFO [2019-12-17 12:34:26,495] jepsen worker 12 - jepsen.util 8922 :invoke :read [4 nil] INFO [2019-12-17 12:34:26,498] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47976,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:34:26,498] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47976,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:34:26,498] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47980,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:34:26,498] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47980,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:34:26,498] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47981,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:34:26,498] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":47981,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:34:26,504] jepsen worker 28 - jepsen.util 6598 :ok :read [9 18] INFO [2019-12-17 12:34:26,504] jepsen worker 28 - jepsen.util 6598 :invoke :write [9 16] INFO [2019-12-17 12:34:26,514] jepsen worker 34 - jepsen.util 11194 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:34:26,514] jepsen worker 34 - jepsen.util 11194 :invoke :cas [11 [11 13]] INFO [2019-12-17 12:34:26,533] jepsen worker 9 - jepsen.util 8649 :ok :write [3 1] INFO [2019-12-17 12:34:26,534] jepsen worker 9 - jepsen.util 8649 :invoke :cas [3 [13 1]] INFO [2019-12-17 12:34:26,534] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:34:26,534] jepsen nemesis - couchbase.util Rebalance check count: 372 INFO [2019-12-17 12:34:26,534] jepsen nemesis - couchbase.util Rebalance stuck count: 353 INFO [2019-12-17 12:34:26,537] jepsen worker 31 - jepsen.util 7411 :ok :write [10 48] INFO [2019-12-17 12:34:26,537] jepsen worker 31 - jepsen.util 7411 :invoke :read [10 nil] INFO [2019-12-17 12:34:26,570] jepsen worker 18 - jepsen.util 8568 :ok :write [6 30] INFO [2019-12-17 12:34:26,570] jepsen worker 18 - jepsen.util 8568 :invoke :read [6 nil] INFO [2019-12-17 12:34:26,581] jepsen worker 72 - jepsen.util 8262 :ok :write [24 14] INFO [2019-12-17 12:34:26,581] jepsen worker 72 - jepsen.util 8262 :invoke :cas [24 [5 47]] INFO [2019-12-17 12:34:26,589] jepsen worker 65 - jepsen.util 6905 :info :cas [21 [12 36]] :RequestCanceledException INFO [2019-12-17 12:34:26,590] jepsen worker 65 - jepsen.util 6995 :invoke :cas [21 [38 25]] INFO [2019-12-17 12:34:26,629] jepsen worker 13 - jepsen.util 7843 :info :write [4 28] :RequestCanceledException INFO [2019-12-17 12:34:26,630] jepsen worker 13 - jepsen.util 7933 :invoke :read [4 nil] INFO [2019-12-17 12:34:26,636] jepsen worker 63 - jepsen.util 7083 :ok :read [21 2] INFO [2019-12-17 12:34:26,636] jepsen worker 63 - jepsen.util 7083 :invoke :read [21 nil] INFO [2019-12-17 12:34:26,641] jepsen worker 27 - jepsen.util 8127 :fail :cas [9 [9 5]] :ValueNotSwapFrom INFO [2019-12-17 12:34:26,641] jepsen worker 27 - jepsen.util 8127 :invoke :read [9 nil] INFO [2019-12-17 12:34:26,644] jepsen worker 46 - jepsen.util 8416 :ok :read [15 36] INFO [2019-12-17 12:34:26,645] jepsen worker 46 - jepsen.util 8416 :invoke :read [15 nil] INFO [2019-12-17 12:34:26,650] jepsen worker 80 - jepsen.util 8180 :ok :write [26 46] INFO [2019-12-17 12:34:26,651] jepsen worker 80 - jepsen.util 8180 :invoke :cas [26 [7 16]] INFO [2019-12-17 12:34:26,673] jepsen worker 18 - jepsen.util 8568 :ok :read [6 30] INFO [2019-12-17 12:34:26,673] jepsen worker 18 - jepsen.util 8568 :invoke :cas [6 [1 29]] INFO [2019-12-17 12:34:26,683] jepsen worker 31 - jepsen.util 7411 :ok :read [10 48] INFO [2019-12-17 12:34:26,684] jepsen worker 31 - jepsen.util 7411 :invoke :write [10 47] INFO [2019-12-17 12:34:26,713] jepsen worker 63 - jepsen.util 7083 :ok :read [21 2] INFO [2019-12-17 12:34:26,713] jepsen worker 63 - jepsen.util 7083 :invoke :cas [21 [47 5]] INFO [2019-12-17 12:34:26,730] jepsen worker 35 - jepsen.util 11015 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:34:26,730] jepsen worker 35 - jepsen.util 11015 :invoke :read [11 nil] INFO [2019-12-17 12:34:26,730] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":48005,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:34:26,730] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":48005,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:34:26,730] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":48009,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:34:26,730] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":48009,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:34:26,735] jepsen worker 65 - jepsen.util 6995 :fail :cas [21 [38 25]] :ValueNotSwapFrom INFO [2019-12-17 12:34:26,735] jepsen worker 65 - jepsen.util 6995 :invoke :read [21 nil] INFO [2019-12-17 12:34:26,737] jepsen worker 78 - jepsen.util 8088 :ok :write [26 49] INFO [2019-12-17 12:34:26,738] jepsen worker 78 - jepsen.util 8088 :invoke :write [26 17] INFO [2019-12-17 12:34:26,743] jepsen worker 67 - jepsen.util 11407 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:34:26,744] jepsen worker 67 - jepsen.util 11407 :invoke :write [22 0] INFO [2019-12-17 12:34:26,802] jepsen worker 18 - jepsen.util 8568 :fail :cas [6 [1 29]] :ValueNotSwapFrom INFO [2019-12-17 12:34:26,802] jepsen worker 18 - jepsen.util 8568 :invoke :read [6 nil] INFO [2019-12-17 12:34:26,821] jepsen worker 53 - jepsen.util 7703 :fail :cas [17 [13 22]] :ValueNotSwapFrom INFO [2019-12-17 12:34:26,821] jepsen worker 53 - jepsen.util 7703 :invoke :read [17 nil] INFO [2019-12-17 12:34:26,821] jepsen worker 27 - jepsen.util 8127 :ok :read [9 18] INFO [2019-12-17 12:34:26,821] jepsen worker 27 - jepsen.util 8127 :invoke :write [9 26] INFO [2019-12-17 12:34:26,827] jepsen worker 30 - jepsen.util 8400 :ok :read [10 48] INFO [2019-12-17 12:34:26,827] jepsen worker 30 - jepsen.util 8400 :invoke :read [10 nil] INFO [2019-12-17 12:34:26,834] jepsen worker 45 - jepsen.util 8595 :ok :read [15 36] INFO [2019-12-17 12:34:26,834] jepsen worker 45 - jepsen.util 8595 :invoke :write [15 13] INFO [2019-12-17 12:34:26,835] jepsen worker 36 - jepsen.util 8586 :fail :cas [12 [38 26]] :ValueNotSwapFrom INFO [2019-12-17 12:34:26,835] jepsen worker 36 - jepsen.util 8586 :invoke :read [12 nil] INFO [2019-12-17 12:34:26,852] jepsen worker 72 - jepsen.util 8262 :fail :cas [24 [5 47]] :ValueNotSwapFrom INFO [2019-12-17 12:34:26,852] jepsen worker 72 - jepsen.util 8262 :invoke :write [24 26] INFO [2019-12-17 12:34:26,855] jepsen worker 13 - jepsen.util 7933 :ok :read [4 48] INFO [2019-12-17 12:34:26,856] jepsen worker 13 - jepsen.util 7933 :invoke :cas [4 [9 26]] INFO [2019-12-17 12:34:26,871] jepsen worker 29 - jepsen.util 6779 :ok :write [9 42] INFO [2019-12-17 12:34:26,871] jepsen worker 29 - jepsen.util 6779 :invoke :write [9 12] INFO [2019-12-17 12:34:26,882] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":48027,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:34:26,882] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":48027,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:34:26,882] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":48031,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:34:26,882] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":48031,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:34:26,882] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":48032,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:34:26,882] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":48032,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:34:26,891] jepsen worker 14 - jepsen.util 8564 :info :write [4 36] :RequestCanceledException INFO [2019-12-17 12:34:26,895] jepsen worker 14 - jepsen.util 8654 :invoke :cas [4 [7 34]] INFO [2019-12-17 12:34:26,895] jepsen worker 63 - jepsen.util 7083 :fail :cas [21 [47 5]] :ValueNotSwapFrom INFO [2019-12-17 12:34:26,895] jepsen worker 63 - jepsen.util 7083 :invoke :write [21 8] INFO [2019-12-17 12:34:26,900] jepsen worker 46 - jepsen.util 8416 :ok :read [15 36] INFO [2019-12-17 12:34:26,900] jepsen worker 46 - jepsen.util 8416 :invoke :read [15 nil] INFO [2019-12-17 12:34:26,910] jepsen worker 30 - jepsen.util 8400 :ok :read [10 48] INFO [2019-12-17 12:34:26,910] jepsen worker 30 - jepsen.util 8400 :invoke :write [10 14] INFO [2019-12-17 12:34:26,935] jepsen worker 18 - jepsen.util 8568 :ok :read [6 30] INFO [2019-12-17 12:34:26,935] jepsen worker 18 - jepsen.util 8568 :invoke :read [6 nil] INFO [2019-12-17 12:34:26,973] jepsen worker 12 - jepsen.util 8922 :ok :read [4 48] INFO [2019-12-17 12:34:26,973] jepsen worker 12 - jepsen.util 8922 :invoke :read [4 nil] INFO [2019-12-17 12:34:26,987] jepsen worker 55 - jepsen.util 9055 :ok :write [18 20] INFO [2019-12-17 12:34:26,987] jepsen worker 55 - jepsen.util 9055 :invoke :read [18 nil] INFO [2019-12-17 12:34:26,999] jepsen worker 89 - jepsen.util 8189 :ok :write [29 34] INFO [2019-12-17 12:34:26,999] jepsen worker 89 - jepsen.util 8189 :invoke :write [29 5] INFO [2019-12-17 12:34:27,004] jepsen worker 71 - jepsen.util 8891 :ok :write [23 46] INFO [2019-12-17 12:34:27,004] jepsen worker 71 - jepsen.util 8891 :invoke :read [23 nil] INFO [2019-12-17 12:34:27,011] jepsen worker 28 - jepsen.util 6598 :ok :write [9 16] INFO [2019-12-17 12:34:27,011] jepsen worker 28 - jepsen.util 6598 :invoke :read [9 nil] INFO [2019-12-17 12:34:27,014] jepsen worker 36 - jepsen.util 8586 :ok :read [12 40] INFO [2019-12-17 12:34:27,014] jepsen worker 36 - jepsen.util 8586 :invoke :cas [12 [23 32]] INFO [2019-12-17 12:34:27,025] jepsen worker 63 - jepsen.util 7083 :ok :write [21 8] INFO [2019-12-17 12:34:27,025] jepsen worker 63 - jepsen.util 7083 :invoke :write [21 45] INFO [2019-12-17 12:34:27,043] jepsen worker 4 - jepsen.util 7744 :info :write [1 25] :RequestCanceledException INFO [2019-12-17 12:34:27,044] jepsen worker 4 - jepsen.util 7834 :invoke :read [1 nil] INFO [2019-12-17 12:34:27,050] jepsen worker 28 - jepsen.util 6598 :ok :read [9 16] INFO [2019-12-17 12:34:27,050] jepsen worker 28 - jepsen.util 6598 :invoke :write [9 14] INFO [2019-12-17 12:34:27,053] jepsen worker 31 - jepsen.util 7411 :ok :write [10 47] INFO [2019-12-17 12:34:27,053] jepsen worker 31 - jepsen.util 7411 :invoke :cas [10 [48 39]] INFO [2019-12-17 12:34:27,057] jepsen worker 57 - jepsen.util 11307 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:34:27,058] jepsen worker 57 - jepsen.util 11307 :invoke :cas [19 [48 38]] INFO [2019-12-17 12:34:27,062] jepsen worker 18 - jepsen.util 8568 :ok :read [6 30] INFO [2019-12-17 12:34:27,062] jepsen worker 18 - jepsen.util 8568 :invoke :read [6 nil] INFO [2019-12-17 12:34:27,063] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":48041,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:34:27,063] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":48041,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:34:27,063] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":48045,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:34:27,063] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":48045,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:34:27,080] jepsen worker 13 - jepsen.util 7933 :fail :cas [4 [9 26]] :ValueNotSwapFrom INFO [2019-12-17 12:34:27,080] jepsen worker 13 - jepsen.util 7933 :invoke :cas [4 [14 8]] INFO [2019-12-17 12:34:27,088] jepsen worker 45 - jepsen.util 8595 :ok :write [15 13] INFO [2019-12-17 12:34:27,089] jepsen worker 45 - jepsen.util 8595 :invoke :cas [15 [21 40]] INFO [2019-12-17 12:34:27,096] jepsen worker 33 - jepsen.util 11193 :info :cas [11 [43 20]] :RequestCanceledException INFO [2019-12-17 12:34:27,097] jepsen worker 33 - jepsen.util 11283 :invoke :cas [11 [12 16]] INFO [2019-12-17 12:34:27,111] jepsen worker 46 - jepsen.util 8416 :ok :read [15 13] INFO [2019-12-17 12:34:27,111] jepsen worker 46 - jepsen.util 8416 :invoke :cas [15 [36 40]] INFO [2019-12-17 12:34:27,114] jepsen worker 51 - jepsen.util 7701 :info :write [17 31] :RequestCanceledException INFO [2019-12-17 12:34:27,115] jepsen worker 51 - jepsen.util 7791 :invoke :read [17 nil] INFO [2019-12-17 12:34:27,129] jepsen worker 25 - jepsen.util 11905 :info :write [8 0] :RequestCanceledException INFO [2019-12-17 12:34:27,130] jepsen worker 25 - jepsen.util 11995 :invoke :cas [8 [7 0]] INFO [2019-12-17 12:34:27,139] jepsen worker 27 - jepsen.util 8127 :ok :write [9 26] INFO [2019-12-17 12:34:27,139] jepsen worker 27 - jepsen.util 8127 :invoke :write [9 26] INFO [2019-12-17 12:34:27,143] jepsen worker 62 - jepsen.util 11492 :info :cas [20 [49 13]] :RequestCanceledException INFO [2019-12-17 12:34:27,144] jepsen worker 62 - jepsen.util 11582 :invoke :write [20 29] INFO [2019-12-17 12:34:27,164] jepsen worker 78 - jepsen.util 8088 :ok :write [26 17] INFO [2019-12-17 12:34:27,164] jepsen worker 78 - jepsen.util 8088 :invoke :write [26 47] INFO [2019-12-17 12:34:27,171] jepsen worker 13 - jepsen.util 7933 :fail :cas [4 [14 8]] :ValueNotSwapFrom INFO [2019-12-17 12:34:27,171] jepsen worker 30 - jepsen.util 8400 :ok :write [10 14] INFO [2019-12-17 12:34:27,171] jepsen worker 13 - jepsen.util 7933 :invoke :write [4 27] INFO [2019-12-17 12:34:27,172] jepsen worker 30 - jepsen.util 8400 :invoke :cas [10 [11 22]] INFO [2019-12-17 12:34:27,177] jepsen worker 31 - jepsen.util 7411 :fail :cas [10 [48 39]] :ValueNotSwapFrom INFO [2019-12-17 12:34:27,177] jepsen worker 31 - jepsen.util 7411 :invoke :read [10 nil] INFO [2019-12-17 12:34:27,196] jepsen worker 0 - jepsen.util 11790 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:34:27,196] jepsen worker 38 - jepsen.util 8498 :info :write [12 6] :RequestCanceledException INFO [2019-12-17 12:34:27,196] jepsen worker 0 - jepsen.util 11790 :invoke :cas [0 [21 12]] INFO [2019-12-17 12:34:27,197] jepsen worker 38 - jepsen.util 8588 :invoke :read [12 nil] INFO [2019-12-17 12:34:27,199] jepsen worker 56 - jepsen.util 9236 :ok :write [18 27] INFO [2019-12-17 12:34:27,199] jepsen worker 56 - jepsen.util 9236 :invoke :read [18 nil] INFO [2019-12-17 12:34:27,208] jepsen worker 45 - jepsen.util 8595 :fail :cas [15 [21 40]] :ValueNotSwapFrom INFO [2019-12-17 12:34:27,209] jepsen worker 45 - jepsen.util 8595 :invoke :read [15 nil] INFO [2019-12-17 12:34:27,224] jepsen worker 46 - jepsen.util 8416 :fail :cas [15 [36 40]] :ValueNotSwapFrom INFO [2019-12-17 12:34:27,225] jepsen worker 46 - jepsen.util 8416 :invoke :read [15 nil] INFO [2019-12-17 12:34:27,230] jepsen worker 58 - jepsen.util 12208 :info :write [19 29] :RequestCanceledException INFO [2019-12-17 12:34:27,231] jepsen worker 58 - jepsen.util 12298 :invoke :write [19 7] INFO [2019-12-17 12:34:27,233] jepsen worker 51 - jepsen.util 7791 :ok :read [17 30] INFO [2019-12-17 12:34:27,233] jepsen worker 51 - jepsen.util 7791 :invoke :write [17 29] INFO [2019-12-17 12:34:27,239] jepsen worker 80 - jepsen.util 8180 :fail :cas [26 [7 16]] :ValueNotSwapFrom INFO [2019-12-17 12:34:27,239] jepsen worker 80 - jepsen.util 8180 :invoke :write [26 16] INFO [2019-12-17 12:34:27,243] jepsen worker 85 - jepsen.util 12055 :info :cas [28 [27 20]] :RequestCanceledException INFO [2019-12-17 12:34:27,244] jepsen worker 85 - jepsen.util 12145 :invoke :read [28 nil] INFO [2019-12-17 12:34:27,246] jepsen worker 30 - jepsen.util 8400 :fail :cas [10 [11 22]] :ValueNotSwapFrom INFO [2019-12-17 12:34:27,246] jepsen worker 30 - jepsen.util 8400 :invoke :write [10 10] INFO [2019-12-17 12:34:27,258] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":48082,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:34:27,258] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":48082,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:34:27,258] jepsen worker 70 - jepsen.util 8620 :info :write [23 0] :RequestCanceledException INFO [2019-12-17 12:34:27,258] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":48084,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:34:27,258] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":48084,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:34:27,260] jepsen worker 70 - jepsen.util 8710 :invoke :cas [23 [16 37]] INFO [2019-12-17 12:34:27,280] jepsen worker 71 - jepsen.util 8891 :ok :read [23 46] INFO [2019-12-17 12:34:27,280] jepsen worker 28 - jepsen.util 6598 :ok :write [9 14] INFO [2019-12-17 12:34:27,280] jepsen worker 71 - jepsen.util 8891 :invoke :cas [23 [0 33]] INFO [2019-12-17 12:34:27,280] jepsen worker 28 - jepsen.util 6598 :invoke :cas [9 [6 22]] INFO [2019-12-17 12:34:27,282] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":48070,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:34:27,282] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":48070,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:34:27,282] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":48072,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:34:27,282] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":48072,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:34:27,285] jepsen worker 45 - jepsen.util 8595 :ok :read [15 13] INFO [2019-12-17 12:34:27,286] jepsen worker 45 - jepsen.util 8595 :invoke :write [15 48] INFO [2019-12-17 12:34:27,329] jepsen worker 22 - jepsen.util 11362 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:34:27,329] jepsen worker 87 - jepsen.util 7647 :info :write [29 21] :RequestCanceledException INFO [2019-12-17 12:34:27,329] jepsen worker 22 - jepsen.util 11362 :invoke :write [7 41] INFO [2019-12-17 12:34:27,330] jepsen worker 87 - jepsen.util 7737 :invoke :write [29 27] INFO [2019-12-17 12:34:27,344] jepsen worker 28 - jepsen.util 6598 :fail :cas [9 [6 22]] :ValueNotSwapFrom INFO [2019-12-17 12:34:27,345] jepsen worker 28 - jepsen.util 6598 :invoke :cas [9 [33 46]] INFO [2019-12-17 12:34:27,346] jepsen worker 46 - jepsen.util 8416 :ok :read [15 13] INFO [2019-12-17 12:34:27,347] jepsen worker 46 - jepsen.util 8416 :invoke :write [15 15] INFO [2019-12-17 12:34:27,357] jepsen worker 48 - jepsen.util 10488 :info :cas [16 [13 29]] :RequestCanceledException INFO [2019-12-17 12:34:27,367] jepsen worker 48 - jepsen.util 10578 :invoke :write [16 42] INFO [2019-12-17 12:34:27,377] jepsen worker 30 - jepsen.util 8400 :ok :write [10 10] INFO [2019-12-17 12:34:27,377] jepsen worker 30 - jepsen.util 8400 :invoke :read [10 nil] INFO [2019-12-17 12:34:27,379] jepsen worker 28 - jepsen.util 6598 :fail :cas [9 [33 46]] :ValueNotSwapFrom INFO [2019-12-17 12:34:27,379] jepsen worker 28 - jepsen.util 6598 :invoke :cas [9 [36 34]] INFO [2019-12-17 12:34:27,396] jepsen worker 26 - jepsen.util 11816 :info :write [8 3] :RequestCanceledException INFO [2019-12-17 12:34:27,397] jepsen worker 26 - jepsen.util 11906 :invoke :read [8 nil] INFO [2019-12-17 12:34:27,399] jepsen worker 53 - jepsen.util 7703 :ok :read [17 30] INFO [2019-12-17 12:34:27,399] jepsen worker 53 - jepsen.util 7703 :invoke :cas [17 [20 40]] INFO [2019-12-17 12:34:27,406] jepsen worker 28 - jepsen.util 6598 :fail :cas [9 [36 34]] :ValueNotSwapFrom INFO [2019-12-17 12:34:27,406] jepsen worker 28 - jepsen.util 6598 :invoke :cas [9 [46 7]] INFO [2019-12-17 12:34:27,413] jepsen worker 88 - jepsen.util 8458 :info :write [29 27] :RequestCanceledException INFO [2019-12-17 12:34:27,413] jepsen worker 61 - jepsen.util 12211 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:34:27,413] jepsen worker 20 - jepsen.util 8030 :info :write [6 42] :RequestCanceledException INFO [2019-12-17 12:34:27,413] jepsen worker 61 - jepsen.util 12211 :invoke :read [20 nil] INFO [2019-12-17 12:34:27,414] jepsen worker 20 - jepsen.util 8120 :invoke :read [6 nil] INFO [2019-12-17 12:34:27,414] jepsen worker 88 - jepsen.util 8548 :invoke :write [29 28] INFO [2019-12-17 12:34:27,429] jepsen worker 6 - jepsen.util 11886 :info :write [2 9] :RequestCanceledException INFO [2019-12-17 12:34:27,429] jepsen worker 69 - jepsen.util 8439 :info :write [23 3] :RequestCanceledException INFO [2019-12-17 12:34:27,429] jepsen worker 84 - jepsen.util 10974 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:34:27,429] jepsen worker 54 - jepsen.util 9234 :info :write [18 0] :RequestCanceledException INFO [2019-12-17 12:34:27,430] jepsen worker 84 - jepsen.util 10974 :invoke :read [28 nil] INFO [2019-12-17 12:34:27,431] jepsen worker 54 - jepsen.util 9324 :invoke :cas [18 [46 27]] INFO [2019-12-17 12:34:27,431] jepsen worker 69 - jepsen.util 8529 :invoke :read [23 nil] INFO [2019-12-17 12:34:27,431] jepsen worker 6 - jepsen.util 11976 :invoke :cas [2 [39 25]] INFO [2019-12-17 12:34:27,431] jepsen worker 80 - jepsen.util 8180 :ok :write [26 16] INFO [2019-12-17 12:34:27,431] jepsen worker 80 - jepsen.util 8180 :invoke :read [26 nil] INFO [2019-12-17 12:34:27,433] jepsen worker 28 - jepsen.util 6598 :fail :cas [9 [46 7]] :ValueNotSwapFrom INFO [2019-12-17 12:34:27,433] jepsen worker 28 - jepsen.util 6598 :invoke :cas [9 [42 48]] INFO [2019-12-17 12:34:27,443] jepsen worker 66 - jepsen.util 11226 :info :cas [22 [30 12]] :RequestCanceledException INFO [2019-12-17 12:34:27,445] jepsen worker 66 - jepsen.util 11316 :invoke :read [22 nil] INFO [2019-12-17 12:34:27,448] jepsen worker 14 - jepsen.util 8654 :fail :cas [4 [7 34]] :ValueNotSwapFrom INFO [2019-12-17 12:34:27,448] jepsen worker 14 - jepsen.util 8654 :invoke :read [4 nil] INFO [2019-12-17 12:34:27,457] jepsen worker 44 - jepsen.util 11474 :info :cas [14 [3 46]] :RequestCanceledException INFO [2019-12-17 12:34:27,458] jepsen worker 44 - jepsen.util 11564 :invoke :write [14 6] INFO [2019-12-17 12:34:27,462] jepsen worker 20 - jepsen.util 8120 :ok :read [6 30] INFO [2019-12-17 12:34:27,462] jepsen worker 20 - jepsen.util 8120 :invoke :write [6 32] INFO [2019-12-17 12:34:27,465] jepsen worker 28 - jepsen.util 6598 :fail :cas [9 [42 48]] :ValueNotSwapFrom INFO [2019-12-17 12:34:27,465] jepsen worker 28 - jepsen.util 6598 :invoke :write [9 14] INFO [2019-12-17 12:34:27,468] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":48112,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:34:27,468] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":48112,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:34:27,483] jepsen worker 27 - jepsen.util 8127 :ok :write [9 26] INFO [2019-12-17 12:34:27,483] jepsen worker 27 - jepsen.util 8127 :invoke :read [9 nil] INFO [2019-12-17 12:34:27,489] jepsen worker 12 - jepsen.util 8922 :ok :read [4 48] INFO [2019-12-17 12:34:27,489] jepsen worker 12 - jepsen.util 8922 :invoke :cas [4 [45 3]] INFO [2019-12-17 12:34:27,502] jepsen worker 4 - jepsen.util 7834 :ok :read [1 16] INFO [2019-12-17 12:34:27,502] jepsen worker 4 - jepsen.util 7834 :invoke :cas [1 [38 27]] INFO [2019-12-17 12:34:27,515] jepsen worker 53 - jepsen.util 7703 :fail :cas [17 [20 40]] :ValueNotSwapFrom INFO [2019-12-17 12:34:27,515] jepsen worker 53 - jepsen.util 7703 :invoke :read [17 nil] INFO [2019-12-17 12:34:27,515] jepsen worker 63 - jepsen.util 7083 :ok :write [21 45] INFO [2019-12-17 12:34:27,516] jepsen worker 63 - jepsen.util 7083 :invoke :cas [21 [23 9]] INFO [2019-12-17 12:34:27,519] jepsen worker 55 - jepsen.util 9055 :ok :read [18 27] INFO [2019-12-17 12:34:27,519] jepsen worker 55 - jepsen.util 9055 :invoke :cas [18 [19 0]] INFO [2019-12-17 12:34:27,526] jepsen worker 27 - jepsen.util 8127 :ok :read [9 26] INFO [2019-12-17 12:34:27,526] jepsen worker 27 - jepsen.util 8127 :invoke :write [9 28] INFO [2019-12-17 12:34:27,529] jepsen worker 52 - jepsen.util 8332 :info :write [17 12] :RequestCanceledException INFO [2019-12-17 12:34:27,529] jepsen worker 64 - jepsen.util 6814 :fail :read [21 nil] :RequestTimeoutException INFO [2019-12-17 12:34:27,529] jepsen worker 64 - jepsen.util 6814 :invoke :cas [21 [5 47]] INFO [2019-12-17 12:34:27,532] jepsen worker 52 - jepsen.util 8422 :invoke :cas [17 [34 15]] INFO [2019-12-17 12:34:27,544] jepsen worker 63 - jepsen.util 7083 :fail :cas [21 [23 9]] :ValueNotSwapFrom INFO [2019-12-17 12:34:27,544] jepsen worker 63 - jepsen.util 7083 :invoke :write [21 6] INFO [2019-12-17 12:34:27,552] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:34:27,552] jepsen nemesis - couchbase.util Rebalance check count: 373 INFO [2019-12-17 12:34:27,552] jepsen nemesis - couchbase.util Rebalance stuck count: 354 INFO [2019-12-17 12:34:27,580] jepsen worker 56 - jepsen.util 9236 :ok :read [18 27] INFO [2019-12-17 12:34:27,581] jepsen worker 56 - jepsen.util 9236 :invoke :cas [18 [39 7]] INFO [2019-12-17 12:34:27,585] jepsen worker 64 - jepsen.util 6814 :fail :cas [21 [5 47]] :ValueNotSwapFrom INFO [2019-12-17 12:34:27,585] jepsen worker 64 - jepsen.util 6814 :invoke :write [21 9] INFO [2019-12-17 12:34:27,596] jepsen worker 13 - jepsen.util 7933 :ok :write [4 27] INFO [2019-12-17 12:34:27,596] jepsen worker 40 - jepsen.util 11650 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:34:27,596] jepsen worker 40 - jepsen.util 11650 :invoke :read [13 nil] INFO [2019-12-17 12:34:27,596] jepsen worker 13 - jepsen.util 7933 :invoke :read [4 nil] INFO [2019-12-17 12:34:27,596] jepsen worker 51 - jepsen.util 7791 :ok :write [17 29] INFO [2019-12-17 12:34:27,596] jepsen worker 51 - jepsen.util 7791 :invoke :read [17 nil] INFO [2019-12-17 12:34:27,604] jepsen worker 4 - jepsen.util 7834 :fail :cas [1 [38 27]] :ValueNotSwapFrom INFO [2019-12-17 12:34:27,605] jepsen worker 4 - jepsen.util 7834 :invoke :cas [1 [26 22]] INFO [2019-12-17 12:34:27,613] jepsen worker 28 - jepsen.util 6598 :ok :write [9 14] INFO [2019-12-17 12:34:27,613] jepsen worker 28 - jepsen.util 6598 :invoke :write [9 29] INFO [2019-12-17 12:34:27,629] jepsen worker 10 - jepsen.util 8470 :info :write [3 16] :RequestCanceledException INFO [2019-12-17 12:34:27,630] jepsen worker 10 - jepsen.util 8560 :invoke :read [3 nil] INFO [2019-12-17 12:34:27,630] jepsen worker 24 - jepsen.util 10374 :info :write [8 14] :RequestCanceledException INFO [2019-12-17 12:34:27,631] jepsen worker 46 - jepsen.util 8416 :ok :write [15 15] INFO [2019-12-17 12:34:27,631] jepsen worker 46 - jepsen.util 8416 :invoke :write [15 35] INFO [2019-12-17 12:34:27,631] jepsen worker 24 - jepsen.util 10464 :invoke :cas [8 [49 27]] INFO [2019-12-17 12:34:27,647] jepsen worker 36 - jepsen.util 8586 :fail :cas [12 [23 32]] :ValueNotSwapFrom INFO [2019-12-17 12:34:27,647] jepsen worker 36 - jepsen.util 8586 :invoke :write [12 22] INFO [2019-12-17 12:34:27,658] jepsen worker 70 - jepsen.util 8710 :fail :cas [23 [16 37]] :ValueNotSwapFrom INFO [2019-12-17 12:34:27,658] jepsen worker 70 - jepsen.util 8710 :invoke :cas [23 [35 2]] INFO [2019-12-17 12:34:27,686] jepsen worker 10 - jepsen.util 8560 :ok :read [3 1] INFO [2019-12-17 12:34:27,686] jepsen worker 10 - jepsen.util 8560 :invoke :cas [3 [20 42]] INFO [2019-12-17 12:34:27,693] jepsen worker 71 - jepsen.util 8891 :fail :cas [23 [0 33]] :ValueNotSwapFrom INFO [2019-12-17 12:34:27,693] jepsen worker 71 - jepsen.util 8891 :invoke :cas [23 [3 8]] INFO [2019-12-17 12:34:27,698] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":48148,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:34:27,698] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":48148,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:34:27,701] jepsen worker 20 - jepsen.util 8120 :ok :write [6 32] INFO [2019-12-17 12:34:27,701] jepsen worker 20 - jepsen.util 8120 :invoke :read [6 nil] INFO [2019-12-17 12:34:27,711] jepsen worker 15 - jepsen.util 10455 :info :write [5 14] :RequestCanceledException INFO [2019-12-17 12:34:27,711] jepsen worker 32 - jepsen.util 8492 :info :write [10 41] :RequestCanceledException INFO [2019-12-17 12:34:27,712] jepsen worker 32 - jepsen.util 8582 :invoke :write [10 32] INFO [2019-12-17 12:34:27,713] jepsen worker 15 - jepsen.util 10545 :invoke :write [5 16] INFO [2019-12-17 12:34:27,720] jepsen worker 54 - jepsen.util 9324 :fail :cas [18 [46 27]] :ValueNotSwapFrom INFO [2019-12-17 12:34:27,720] jepsen worker 54 - jepsen.util 9324 :invoke :cas [18 [26 31]] INFO [2019-12-17 12:34:27,729] jepsen worker 42 - jepsen.util 12372 :info :cas [14 [1 42]] :RequestCanceledException INFO [2019-12-17 12:34:27,730] jepsen worker 42 - jepsen.util 12462 :invoke :cas [14 [3 12]] INFO [2019-12-17 12:34:27,735] jepsen worker 38 - jepsen.util 8588 :ok :read [12 40] INFO [2019-12-17 12:34:27,735] jepsen worker 38 - jepsen.util 8588 :invoke :write [12 24] INFO [2019-12-17 12:34:27,743] jepsen worker 79 - jepsen.util 8629 :info :write [26 29] :RequestCanceledException INFO [2019-12-17 12:34:27,743] jepsen worker 1 - jepsen.util 11611 :info :write [0 18] :RequestCanceledException INFO [2019-12-17 12:34:27,744] jepsen worker 79 - jepsen.util 8719 :invoke :write [26 18] INFO [2019-12-17 12:34:27,744] jepsen worker 1 - jepsen.util 11701 :invoke :read [0 nil] INFO [2019-12-17 12:34:27,747] jepsen worker 27 - jepsen.util 8127 :ok :write [9 28] INFO [2019-12-17 12:34:27,747] jepsen worker 27 - jepsen.util 8127 :invoke :read [9 nil] INFO [2019-12-17 12:34:27,752] jepsen worker 69 - jepsen.util 8529 :ok :read [23 46] INFO [2019-12-17 12:34:27,752] jepsen worker 69 - jepsen.util 8529 :invoke :cas [23 [7 17]] INFO [2019-12-17 12:34:27,752] jepsen worker 20 - jepsen.util 8120 :ok :read [6 32] INFO [2019-12-17 12:34:27,752] jepsen worker 20 - jepsen.util 8120 :invoke :write [6 13] INFO [2019-12-17 12:34:27,757] jepsen worker 17 - jepsen.util 11267 :info :write [5 45] :RequestCanceledException INFO [2019-12-17 12:34:27,758] jepsen worker 17 - jepsen.util 11357 :invoke :cas [5 [19 0]] INFO [2019-12-17 12:34:27,795] jepsen worker 52 - jepsen.util 8422 :fail :cas [17 [34 15]] :ValueNotSwapFrom INFO [2019-12-17 12:34:27,795] jepsen worker 52 - jepsen.util 8422 :invoke :write [17 28] INFO [2019-12-17 12:34:27,795] jepsen worker 8 - jepsen.util 11258 :info :cas [2 [49 2]] :RequestCanceledException INFO [2019-12-17 12:34:27,796] jepsen worker 51 - jepsen.util 7791 :ok :read [17 29] INFO [2019-12-17 12:34:27,796] jepsen worker 8 - jepsen.util 11348 :invoke :read [2 nil] INFO [2019-12-17 12:34:27,796] jepsen worker 51 - jepsen.util 7791 :invoke :cas [17 [38 8]] INFO [2019-12-17 12:34:27,797] jepsen worker 63 - jepsen.util 7083 :ok :write [21 6] INFO [2019-12-17 12:34:27,797] jepsen worker 63 - jepsen.util 7083 :invoke :cas [21 [38 32]] INFO [2019-12-17 12:34:27,814] jepsen worker 77 - jepsen.util 11597 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:34:27,814] jepsen worker 43 - jepsen.util 11023 :info :cas [14 [27 44]] :RequestCanceledException INFO [2019-12-17 12:34:27,814] jepsen worker 77 - jepsen.util 11597 :invoke :read [25 nil] INFO [2019-12-17 12:34:27,815] jepsen worker 43 - jepsen.util 11113 :invoke :cas [14 [31 48]] INFO [2019-12-17 12:34:27,823] jepsen worker 63 - jepsen.util 7083 :fail :cas [21 [38 32]] :ValueNotSwapFrom INFO [2019-12-17 12:34:27,823] jepsen worker 63 - jepsen.util 7083 :invoke :read [21 nil] INFO [2019-12-17 12:34:27,828] jepsen worker 14 - jepsen.util 8654 :ok :read [4 27] INFO [2019-12-17 12:34:27,828] jepsen worker 14 - jepsen.util 8654 :invoke :read [4 nil] INFO [2019-12-17 12:34:27,838] jepsen worker 64 - jepsen.util 6814 :ok :write [21 9] INFO [2019-12-17 12:34:27,838] jepsen worker 64 - jepsen.util 6814 :invoke :read [21 nil] INFO [2019-12-17 12:34:27,840] jepsen worker 13 - jepsen.util 7933 :ok :read [4 27] INFO [2019-12-17 12:34:27,840] jepsen worker 13 - jepsen.util 7933 :invoke :cas [4 [33 38]] INFO [2019-12-17 12:34:27,857] jepsen worker 63 - jepsen.util 7083 :ok :read [21 6] INFO [2019-12-17 12:34:27,857] jepsen worker 63 - jepsen.util 7083 :invoke :cas [21 [13 22]] INFO [2019-12-17 12:34:27,860] jepsen worker 27 - jepsen.util 8127 :ok :read [9 28] INFO [2019-12-17 12:34:27,861] jepsen worker 27 - jepsen.util 8127 :invoke :cas [9 [39 39]] INFO [2019-12-17 12:34:27,874] jepsen worker 55 - jepsen.util 9055 :fail :cas [18 [19 0]] :ValueNotSwapFrom INFO [2019-12-17 12:34:27,874] jepsen worker 55 - jepsen.util 9055 :invoke :write [18 12] INFO [2019-12-17 12:34:27,884] jepsen worker 70 - jepsen.util 8710 :fail :cas [23 [35 2]] :ValueNotSwapFrom INFO [2019-12-17 12:34:27,884] jepsen worker 70 - jepsen.util 8710 :invoke :cas [23 [15 5]] INFO [2019-12-17 12:34:27,889] jepsen worker 64 - jepsen.util 6814 :ok :read [21 9] INFO [2019-12-17 12:34:27,889] jepsen worker 64 - jepsen.util 6814 :invoke :write [21 19] INFO [2019-12-17 12:34:27,911] jepsen worker 37 - jepsen.util 7867 :info :write [12 9] :RequestCanceledException INFO [2019-12-17 12:34:27,912] jepsen worker 37 - jepsen.util 7957 :invoke :read [12 nil] INFO [2019-12-17 12:34:27,920] jepsen worker 75 - jepsen.util 10785 :info :cas [25 [48 13]] :RequestCanceledException INFO [2019-12-17 12:34:27,921] jepsen worker 75 - jepsen.util 10875 :invoke :write [25 47] INFO [2019-12-17 12:34:27,925] jepsen worker 63 - jepsen.util 7083 :fail :cas [21 [13 22]] :ValueNotSwapFrom INFO [2019-12-17 12:34:27,925] jepsen worker 63 - jepsen.util 7083 :invoke :read [21 nil] INFO [2019-12-17 12:34:27,933] jepsen worker 27 - jepsen.util 8127 :fail :cas [9 [39 39]] :ValueNotSwapFrom INFO [2019-12-17 12:34:27,933] jepsen worker 27 - jepsen.util 8127 :invoke :cas [9 [48 14]] INFO [2019-12-17 12:34:27,941] jepsen worker 80 - jepsen.util 8180 :ok :read [26 16] INFO [2019-12-17 12:34:27,941] jepsen worker 80 - jepsen.util 8180 :invoke :cas [26 [17 4]] INFO [2019-12-17 12:34:27,943] jepsen worker 5 - jepsen.util 7745 :info :write [1 19] :RequestCanceledException INFO [2019-12-17 12:34:27,944] jepsen worker 5 - jepsen.util 7835 :invoke :cas [1 [16 0]] INFO [2019-12-17 12:34:27,961] jepsen worker 63 - jepsen.util 7083 :ok :read [21 9] INFO [2019-12-17 12:34:27,961] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":48180,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:34:27,961] jepsen worker 63 - jepsen.util 7083 :invoke :cas [21 [20 13]] INFO [2019-12-17 12:34:27,962] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":48180,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:34:27,962] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":48184,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:34:27,962] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":48184,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:34:27,970] jepsen worker 56 - jepsen.util 9236 :fail :cas [18 [39 7]] :ValueNotSwapFrom INFO [2019-12-17 12:34:27,971] jepsen worker 56 - jepsen.util 9236 :invoke :write [18 9] INFO [2019-12-17 12:34:27,980] jepsen worker 63 - jepsen.util 7083 :fail :cas [21 [20 13]] :ValueNotSwapFrom INFO [2019-12-17 12:34:27,981] jepsen worker 63 - jepsen.util 7083 :invoke :write [21 20] INFO [2019-12-17 12:34:27,994] jepsen worker 3 - jepsen.util 8103 :info :write [1 10] :RequestCanceledException INFO [2019-12-17 12:34:27,994] jepsen worker 7 - jepsen.util 11527 :info :write [2 44] :RequestCanceledException INFO [2019-12-17 12:34:27,995] jepsen worker 3 - jepsen.util 8193 :invoke :read [1 nil] INFO [2019-12-17 12:34:27,995] jepsen worker 7 - jepsen.util 11617 :invoke :write [2 23] INFO [2019-12-17 12:34:27,995] jepsen worker 71 - jepsen.util 8891 :fail :cas [23 [3 8]] :ValueNotSwapFrom INFO [2019-12-17 12:34:27,995] jepsen worker 71 - jepsen.util 8891 :invoke :read [23 nil] INFO [2019-12-17 12:34:28,000] jepsen worker 64 - jepsen.util 6814 :ok :write [21 19] INFO [2019-12-17 12:34:28,000] jepsen worker 64 - jepsen.util 6814 :invoke :write [21 29] INFO [2019-12-17 12:34:28,010] jepsen worker 49 - jepsen.util 10759 :info :write [16 19] :RequestCanceledException INFO [2019-12-17 12:34:28,011] jepsen worker 49 - jepsen.util 10849 :invoke :cas [16 [32 41]] INFO [2019-12-17 12:34:28,027] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":48201,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:34:28,027] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":48201,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:34:28,027] jepsen worker 2 - jepsen.util 10982 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:34:28,027] jepsen worker 11 - jepsen.util 8291 :fail :read [3 nil] :RequestTimeoutException INFO [2019-12-17 12:34:28,027] jepsen worker 2 - jepsen.util 10982 :invoke :cas [0 [5 44]] INFO [2019-12-17 12:34:28,027] jepsen worker 11 - jepsen.util 8291 :invoke :cas [3 [26 26]] INFO [2019-12-17 12:34:28,041] jepsen worker 27 - jepsen.util 8127 :fail :cas [9 [48 14]] :ValueNotSwapFrom INFO [2019-12-17 12:34:28,042] jepsen worker 27 - jepsen.util 8127 :invoke :write [9 45] INFO [2019-12-17 12:34:28,076] jepsen worker 54 - jepsen.util 9324 :fail :cas [18 [26 31]] :ValueNotSwapFrom INFO [2019-12-17 12:34:28,076] jepsen worker 54 - jepsen.util 9324 :invoke :write [18 12] INFO [2019-12-17 12:34:28,077] jepsen worker 12 - jepsen.util 8922 :fail :cas [4 [45 3]] :ValueNotSwapFrom INFO [2019-12-17 12:34:28,077] jepsen worker 12 - jepsen.util 8922 :invoke :cas [4 [31 40]] INFO [2019-12-17 12:34:28,084] jepsen worker 64 - jepsen.util 6814 :ok :write [21 29] INFO [2019-12-17 12:34:28,084] jepsen worker 64 - jepsen.util 6814 :invoke :cas [21 [47 49]] INFO [2019-12-17 12:34:28,144] jepsen worker 27 - jepsen.util 8127 :ok :write [9 45] INFO [2019-12-17 12:34:28,145] jepsen worker 27 - jepsen.util 8127 :invoke :read [9 nil] INFO [2019-12-17 12:34:28,160] jepsen worker 27 - jepsen.util 8127 :ok :read [9 45] INFO [2019-12-17 12:34:28,160] jepsen worker 27 - jepsen.util 8127 :invoke :cas [9 [16 1]] INFO [2019-12-17 12:34:28,177] jepsen worker 27 - jepsen.util 8127 :fail :cas [9 [16 1]] :ValueNotSwapFrom INFO [2019-12-17 12:34:28,178] jepsen worker 27 - jepsen.util 8127 :invoke :read [9 nil] INFO [2019-12-17 12:34:28,190] jepsen worker 27 - jepsen.util 8127 :ok :read [9 45] INFO [2019-12-17 12:34:28,190] jepsen worker 27 - jepsen.util 8127 :invoke :read [9 nil] INFO [2019-12-17 12:34:28,205] jepsen worker 51 - jepsen.util 7791 :fail :cas [17 [38 8]] :ValueNotSwapFrom INFO [2019-12-17 12:34:28,205] jepsen worker 51 - jepsen.util 7791 :invoke :cas [17 [6 14]] INFO [2019-12-17 12:34:28,207] jepsen worker 27 - jepsen.util 8127 :ok :read [9 45] INFO [2019-12-17 12:34:28,207] jepsen worker 27 - jepsen.util 8127 :invoke :cas [9 [21 34]] INFO [2019-12-17 12:34:28,222] jepsen worker 27 - jepsen.util 8127 :fail :cas [9 [21 34]] :ValueNotSwapFrom INFO [2019-12-17 12:34:28,222] jepsen worker 27 - jepsen.util 8127 :invoke :write [9 20] INFO [2019-12-17 12:34:28,229] jepsen worker 73 - jepsen.util 7993 :info :cas [24 [19 11]] :RequestCanceledException INFO [2019-12-17 12:34:28,229] jepsen worker 86 - jepsen.util 12146 :info :write [28 42] :RequestCanceledException INFO [2019-12-17 12:34:28,230] jepsen worker 86 - jepsen.util 12236 :invoke :cas [28 [22 34]] INFO [2019-12-17 12:34:28,230] jepsen worker 73 - jepsen.util 8083 :invoke :write [24 36] INFO [2019-12-17 12:34:28,231] jepsen worker 13 - jepsen.util 7933 :fail :cas [4 [33 38]] :ValueNotSwapFrom INFO [2019-12-17 12:34:28,231] jepsen worker 13 - jepsen.util 7933 :invoke :write [4 39] INFO [2019-12-17 12:34:28,237] jepsen worker 4 - jepsen.util 7834 :fail :cas [1 [26 22]] :ValueNotSwapFrom INFO [2019-12-17 12:34:28,237] jepsen worker 4 - jepsen.util 7834 :invoke :cas [1 [16 41]] INFO [2019-12-17 12:34:28,239] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":48227,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:34:28,239] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":48227,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:34:28,239] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":48230,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:34:28,239] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":48230,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:34:28,239] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":48231,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:34:28,239] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":48231,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:34:28,243] jepsen worker 47 - jepsen.util 8777 :info :cas [15 [48 21]] :RequestCanceledException INFO [2019-12-17 12:34:28,244] jepsen worker 47 - jepsen.util 8867 :invoke :cas [15 [2 48]] INFO [2019-12-17 12:34:28,246] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":48218,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:34:28,246] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":48218,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:34:28,246] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":48220,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:34:28,246] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":48220,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:34:28,257] jepsen worker 81 - jepsen.util 11241 :info :write [27 37] :RequestCanceledException INFO [2019-12-17 12:34:28,258] jepsen worker 81 - jepsen.util 11331 :invoke :read [27 nil] INFO [2019-12-17 12:34:28,262] jepsen worker 70 - jepsen.util 8710 :fail :cas [23 [15 5]] :ValueNotSwapFrom INFO [2019-12-17 12:34:28,262] jepsen worker 70 - jepsen.util 8710 :invoke :read [23 nil] INFO [2019-12-17 12:34:28,279] jepsen worker 27 - jepsen.util 8127 :ok :write [9 20] INFO [2019-12-17 12:34:28,280] jepsen worker 27 - jepsen.util 8127 :invoke :write [9 14] INFO [2019-12-17 12:34:28,307] jepsen worker 69 - jepsen.util 8529 :fail :cas [23 [7 17]] :ValueNotSwapFrom INFO [2019-12-17 12:34:28,308] jepsen worker 69 - jepsen.util 8529 :invoke :write [23 26] INFO [2019-12-17 12:34:28,313] jepsen worker 59 - jepsen.util 10949 :info :write [19 27] :RequestCanceledException INFO [2019-12-17 12:34:28,314] jepsen worker 59 - jepsen.util 11039 :invoke :read [19 nil] INFO [2019-12-17 12:34:28,325] jepsen worker 27 - jepsen.util 8127 :ok :write [9 14] INFO [2019-12-17 12:34:28,325] jepsen worker 27 - jepsen.util 8127 :invoke :read [9 nil] INFO [2019-12-17 12:34:28,328] jepsen worker 39 - jepsen.util 10839 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:34:28,328] jepsen worker 19 - jepsen.util 8479 :info :cas [6 [49 47]] :RequestCanceledException INFO [2019-12-17 12:34:28,328] jepsen worker 39 - jepsen.util 10839 :invoke :write [13 48] INFO [2019-12-17 12:34:28,333] jepsen worker 19 - jepsen.util 8569 :invoke :cas [6 [9 30]] INFO [2019-12-17 12:34:28,344] jepsen worker 27 - jepsen.util 8127 :ok :read [9 14] INFO [2019-12-17 12:34:28,344] jepsen worker 41 - jepsen.util 11921 :info :cas [13 [0 34]] :RequestCanceledException INFO [2019-12-17 12:34:28,344] jepsen worker 27 - jepsen.util 8127 :invoke :cas [9 [17 17]] INFO [2019-12-17 12:34:28,345] jepsen worker 41 - jepsen.util 12011 :invoke :cas [13 [43 22]] INFO [2019-12-17 12:34:28,375] jepsen worker 37 - jepsen.util 7957 :ok :read [12 40] INFO [2019-12-17 12:34:28,375] jepsen worker 37 - jepsen.util 7957 :invoke :read [12 nil] INFO [2019-12-17 12:34:28,379] jepsen worker 5 - jepsen.util 7835 :fail :cas [1 [16 0]] :ValueNotSwapFrom INFO [2019-12-17 12:34:28,379] jepsen worker 5 - jepsen.util 7835 :invoke :read [1 nil] INFO [2019-12-17 12:34:28,391] jepsen worker 21 - jepsen.util 10821 :info :cas [7 [10 18]] :RequestCanceledException INFO [2019-12-17 12:34:28,392] jepsen worker 21 - jepsen.util 10911 :invoke :write [7 34] INFO [2019-12-17 12:34:28,395] jepsen worker 71 - jepsen.util 8891 :ok :read [23 46] INFO [2019-12-17 12:34:28,395] jepsen worker 71 - jepsen.util 8891 :invoke :read [23 nil] INFO [2019-12-17 12:34:28,396] jepsen worker 19 - jepsen.util 8569 :fail :cas [6 [9 30]] :ValueNotSwapFrom INFO [2019-12-17 12:34:28,396] jepsen worker 19 - jepsen.util 8569 :invoke :read [6 nil] INFO [2019-12-17 12:34:28,398] jepsen worker 27 - jepsen.util 8127 :fail :cas [9 [17 17]] :ValueNotSwapFrom INFO [2019-12-17 12:34:28,398] jepsen worker 27 - jepsen.util 8127 :invoke :write [9 18] INFO [2019-12-17 12:34:28,424] jepsen worker 52 - jepsen.util 8422 :ok :write [17 28] INFO [2019-12-17 12:34:28,425] jepsen worker 52 - jepsen.util 8422 :invoke :write [17 16] INFO [2019-12-17 12:34:28,429] jepsen worker 82 - jepsen.util 11962 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:34:28,429] jepsen worker 82 - jepsen.util 11962 :invoke :write [27 36] INFO [2019-12-17 12:34:28,441] jepsen worker 19 - jepsen.util 8569 :ok :read [6 32] INFO [2019-12-17 12:34:28,441] jepsen worker 19 - jepsen.util 8569 :invoke :write [6 5] INFO [2019-12-17 12:34:28,447] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":48255,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:34:28,448] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":48255,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:34:28,448] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":48259,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:34:28,448] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":48259,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:34:28,461] jepsen worker 74 - jepsen.util 8534 :info :cas [24 [12 3]] :RequestCanceledException INFO [2019-12-17 12:34:28,462] jepsen worker 74 - jepsen.util 8624 :invoke :read [24 nil] INFO [2019-12-17 12:34:28,479] jepsen worker 27 - jepsen.util 8127 :ok :write [9 18] INFO [2019-12-17 12:34:28,480] jepsen worker 27 - jepsen.util 8127 :invoke :read [9 nil] INFO [2019-12-17 12:34:28,485] jepsen worker 80 - jepsen.util 8180 :fail :cas [26 [17 4]] :ValueNotSwapFrom INFO [2019-12-17 12:34:28,485] jepsen worker 80 - jepsen.util 8180 :invoke :cas [26 [27 18]] INFO [2019-12-17 12:34:28,494] jepsen worker 68 - jepsen.util 10688 :info :write [22 9] :RequestCanceledException INFO [2019-12-17 12:34:28,496] jepsen worker 68 - jepsen.util 10778 :invoke :read [22 nil] INFO [2019-12-17 12:34:28,501] jepsen worker 27 - jepsen.util 8127 :ok :read [9 18] INFO [2019-12-17 12:34:28,501] jepsen worker 27 - jepsen.util 8127 :invoke :cas [9 [12 2]] INFO [2019-12-17 12:34:28,512] jepsen worker 23 - jepsen.util 11093 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:34:28,512] jepsen worker 76 - jepsen.util 11326 :info :write [25 0] :RequestCanceledException INFO [2019-12-17 12:34:28,512] jepsen worker 55 - jepsen.util 9055 :ok :write [18 12] INFO [2019-12-17 12:34:28,512] jepsen worker 23 - jepsen.util 11093 :invoke :read [7 nil] INFO [2019-12-17 12:34:28,512] jepsen worker 55 - jepsen.util 9055 :invoke :cas [18 [24 4]] INFO [2019-12-17 12:34:28,513] jepsen worker 76 - jepsen.util 11416 :invoke :write [25 12] INFO [2019-12-17 12:34:28,530] jepsen worker 60 - jepsen.util 12300 :info :write [20 10] :RequestCanceledException INFO [2019-12-17 12:34:28,530] jepsen worker 50 - jepsen.util 12020 :info :cas [16 [34 21]] :RequestCanceledException INFO [2019-12-17 12:34:28,533] jepsen worker 50 - jepsen.util 12110 :invoke :cas [16 [0 21]] INFO [2019-12-17 12:34:28,535] jepsen worker 60 - jepsen.util 12390 :invoke :write [20 9] INFO [2019-12-17 12:34:28,535] jepsen worker 27 - jepsen.util 8127 :fail :cas [9 [12 2]] :ValueNotSwapFrom INFO [2019-12-17 12:34:28,535] jepsen worker 27 - jepsen.util 8127 :invoke :cas [9 [34 37]] INFO [2019-12-17 12:34:28,535] jepsen worker 83 - jepsen.util 11693 :info :cas [27 [37 35]] :RequestCanceledException INFO [2019-12-17 12:34:28,537] jepsen worker 83 - jepsen.util 11783 :invoke :cas [27 [21 18]] INFO [2019-12-17 12:34:28,557] jepsen worker 46 - jepsen.util 8416 :ok :write [15 35] INFO [2019-12-17 12:34:28,557] jepsen worker 32 - jepsen.util 8582 :ok :write [10 32] INFO [2019-12-17 12:34:28,557] jepsen worker 20 - jepsen.util 8120 :ok :write [6 13] INFO [2019-12-17 12:34:28,557] jepsen worker 46 - jepsen.util 8416 :invoke :read [15 nil] INFO [2019-12-17 12:34:28,557] jepsen worker 32 - jepsen.util 8582 :invoke :cas [10 [40 1]] INFO [2019-12-17 12:34:28,557] jepsen worker 20 - jepsen.util 8120 :invoke :write [6 43] INFO [2019-12-17 12:34:28,559] jepsen worker 16 - jepsen.util 12436 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:34:28,559] jepsen worker 16 - jepsen.util 12436 :invoke :write [5 7] INFO [2019-12-17 12:34:28,560] jepsen worker 27 - jepsen.util 8127 :fail :cas [9 [34 37]] :ValueNotSwapFrom INFO [2019-12-17 12:34:28,560] jepsen worker 27 - jepsen.util 8127 :invoke :write [9 42] INFO [2019-12-17 12:34:28,584] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:34:28,584] jepsen nemesis - couchbase.util Rebalance check count: 374 INFO [2019-12-17 12:34:28,584] jepsen nemesis - couchbase.util Rebalance stuck count: 355 INFO [2019-12-17 12:34:28,589] jepsen worker 53 - jepsen.util 7703 :ok :read [17 28] INFO [2019-12-17 12:34:28,590] jepsen worker 53 - jepsen.util 7703 :invoke :cas [17 [15 49]] INFO [2019-12-17 12:34:28,594] jepsen worker 51 - jepsen.util 7791 :fail :cas [17 [6 14]] :ValueNotSwapFrom INFO [2019-12-17 12:34:28,595] jepsen worker 51 - jepsen.util 7791 :invoke :cas [17 [20 42]] INFO [2019-12-17 12:34:28,599] jepsen worker 73 - jepsen.util 8083 :ok :write [24 36] INFO [2019-12-17 12:34:28,599] jepsen worker 73 - jepsen.util 8083 :invoke :cas [24 [34 49]] INFO [2019-12-17 12:34:28,613] jepsen worker 34 - jepsen.util 11194 :info :cas [11 [11 13]] :RequestCanceledException INFO [2019-12-17 12:34:28,614] jepsen worker 34 - jepsen.util 11284 :invoke :write [11 32] INFO [2019-12-17 12:34:28,630] jepsen worker 9 - jepsen.util 8649 :info :cas [3 [13 1]] :RequestCanceledException INFO [2019-12-17 12:34:28,632] jepsen worker 9 - jepsen.util 8739 :invoke :cas [3 [31 29]] INFO [2019-12-17 12:34:28,634] jepsen worker 47 - jepsen.util 8867 :fail :cas [15 [2 48]] :ValueNotSwapFrom INFO [2019-12-17 12:34:28,634] jepsen worker 47 - jepsen.util 8867 :invoke :cas [15 [33 8]] INFO [2019-12-17 12:34:28,660] jepsen worker 70 - jepsen.util 8710 :ok :read [23 46] INFO [2019-12-17 12:34:28,660] jepsen worker 70 - jepsen.util 8710 :invoke :cas [23 [25 18]] INFO [2019-12-17 12:34:28,667] jepsen worker 73 - jepsen.util 8083 :fail :cas [24 [34 49]] :ValueNotSwapFrom INFO [2019-12-17 12:34:28,667] jepsen worker 73 - jepsen.util 8083 :invoke :read [24 nil] INFO [2019-12-17 12:34:28,679] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":48287,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:34:28,679] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":48287,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:34:28,679] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":48290,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:34:28,679] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":48290,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:34:28,679] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":48291,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:34:28,679] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":48291,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:34:28,696] jepsen worker 3 - jepsen.util 8193 :ok :read [1 10] INFO [2019-12-17 12:34:28,696] jepsen worker 3 - jepsen.util 8193 :invoke :cas [1 [32 9]] INFO [2019-12-17 12:34:28,706] jepsen worker 32 - jepsen.util 8582 :fail :cas [10 [40 1]] :ValueNotSwapFrom INFO [2019-12-17 12:34:28,707] jepsen worker 32 - jepsen.util 8582 :invoke :read [10 nil] INFO [2019-12-17 12:34:28,715] jepsen worker 53 - jepsen.util 7703 :fail :cas [17 [15 49]] :ValueNotSwapFrom INFO [2019-12-17 12:34:28,715] jepsen worker 53 - jepsen.util 7703 :invoke :read [17 nil] INFO [2019-12-17 12:34:28,717] jepsen worker 73 - jepsen.util 8083 :ok :read [24 36] INFO [2019-12-17 12:34:28,717] jepsen worker 73 - jepsen.util 8083 :invoke :write [24 43] INFO [2019-12-17 12:34:28,747] jepsen worker 74 - jepsen.util 8624 :ok :read [24 36] INFO [2019-12-17 12:34:28,747] jepsen worker 74 - jepsen.util 8624 :invoke :read [24 nil] INFO [2019-12-17 12:34:28,762] jepsen worker 5 - jepsen.util 7835 :ok :read [1 10] INFO [2019-12-17 12:34:28,763] jepsen worker 5 - jepsen.util 7835 :invoke :cas [1 [31 15]] INFO [2019-12-17 12:34:28,771] jepsen worker 14 - jepsen.util 8654 :ok :read [4 27] INFO [2019-12-17 12:34:28,771] jepsen worker 14 - jepsen.util 8654 :invoke :read [4 nil] INFO [2019-12-17 12:34:28,783] jepsen worker 71 - jepsen.util 8891 :ok :read [23 46] INFO [2019-12-17 12:34:28,783] jepsen worker 71 - jepsen.util 8891 :invoke :cas [23 [16 27]] INFO [2019-12-17 12:34:28,787] jepsen worker 4 - jepsen.util 7834 :fail :cas [1 [16 41]] :ValueNotSwapFrom INFO [2019-12-17 12:34:28,787] jepsen worker 4 - jepsen.util 7834 :invoke :write [1 5] INFO [2019-12-17 12:34:28,795] jepsen worker 46 - jepsen.util 8416 :ok :read [15 35] INFO [2019-12-17 12:34:28,795] jepsen worker 46 - jepsen.util 8416 :invoke :read [15 nil] INFO [2019-12-17 12:34:28,796] jepsen worker 32 - jepsen.util 8582 :ok :read [10 32] INFO [2019-12-17 12:34:28,796] jepsen worker 32 - jepsen.util 8582 :invoke :cas [10 [18 32]] INFO [2019-12-17 12:34:28,803] jepsen worker 47 - jepsen.util 8867 :fail :cas [15 [33 8]] :ValueNotSwapFrom INFO [2019-12-17 12:34:28,803] jepsen worker 47 - jepsen.util 8867 :invoke :cas [15 [39 1]] INFO [2019-12-17 12:34:28,810] jepsen worker 37 - jepsen.util 7957 :ok :read [12 40] INFO [2019-12-17 12:34:28,810] jepsen worker 37 - jepsen.util 7957 :invoke :write [12 32] INFO [2019-12-17 12:34:28,824] jepsen worker 74 - jepsen.util 8624 :ok :read [24 36] INFO [2019-12-17 12:34:28,824] jepsen worker 74 - jepsen.util 8624 :invoke :cas [24 [17 3]] INFO [2019-12-17 12:34:28,829] jepsen worker 35 - jepsen.util 11015 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:34:28,829] jepsen worker 65 - jepsen.util 6995 :fail :read [21 nil] :RequestTimeoutException INFO [2019-12-17 12:34:28,829] jepsen worker 65 - jepsen.util 6995 :invoke :write [21 44] INFO [2019-12-17 12:34:28,829] jepsen worker 35 - jepsen.util 11015 :invoke :cas [11 [31 31]] INFO [2019-12-17 12:34:28,832] jepsen worker 32 - jepsen.util 8582 :fail :cas [10 [18 32]] :ValueNotSwapFrom INFO [2019-12-17 12:34:28,833] jepsen worker 32 - jepsen.util 8582 :invoke :write [10 43] INFO [2019-12-17 12:34:28,843] jepsen worker 67 - jepsen.util 11407 :info :write [22 0] :RequestCanceledException INFO [2019-12-17 12:34:28,844] jepsen worker 67 - jepsen.util 11497 :invoke :write [22 18] INFO [2019-12-17 12:34:28,849] jepsen worker 46 - jepsen.util 8416 :ok :read [15 35] INFO [2019-12-17 12:34:28,849] jepsen worker 46 - jepsen.util 8416 :invoke :write [15 0] INFO [2019-12-17 12:34:28,886] jepsen worker 47 - jepsen.util 8867 :fail :cas [15 [39 1]] :ValueNotSwapFrom INFO [2019-12-17 12:34:28,886] jepsen worker 47 - jepsen.util 8867 :invoke :write [15 9] INFO [2019-12-17 12:34:28,892] jepsen worker 55 - jepsen.util 9055 :fail :cas [18 [24 4]] :ValueNotSwapFrom INFO [2019-12-17 12:34:28,892] jepsen worker 55 - jepsen.util 9055 :invoke :read [18 nil] INFO [2019-12-17 12:34:28,897] jepsen worker 27 - jepsen.util 8127 :ok :write [9 42] INFO [2019-12-17 12:34:28,897] jepsen worker 27 - jepsen.util 8127 :invoke :write [9 12] INFO [2019-12-17 12:34:28,903] jepsen worker 74 - jepsen.util 8624 :fail :cas [24 [17 3]] :ValueNotSwapFrom INFO [2019-12-17 12:34:28,904] jepsen worker 74 - jepsen.util 8624 :invoke :read [24 nil] INFO [2019-12-17 12:34:28,930] jepsen worker 29 - jepsen.util 6779 :info :write [9 12] :RequestCanceledException INFO [2019-12-17 12:34:28,931] jepsen worker 29 - jepsen.util 6869 :invoke :read [9 nil] INFO [2019-12-17 12:34:28,933] jepsen worker 20 - jepsen.util 8120 :ok :write [6 43] INFO [2019-12-17 12:34:28,933] jepsen worker 20 - jepsen.util 8120 :invoke :cas [6 [18 3]] INFO [2019-12-17 12:34:28,939] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":48314,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:34:28,939] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":48314,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:34:28,939] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":48317,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:34:28,939] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":48317,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:34:28,939] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":48318,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:34:28,939] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":48318,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:34:28,954] jepsen worker 29 - jepsen.util 6869 :ok :read [9 42] INFO [2019-12-17 12:34:28,954] jepsen worker 29 - jepsen.util 6869 :invoke :write [9 40] INFO [2019-12-17 12:34:28,960] jepsen worker 72 - jepsen.util 8262 :info :write [24 26] :RequestCanceledException INFO [2019-12-17 12:34:28,963] jepsen worker 72 - jepsen.util 8352 :invoke :write [24 0] INFO [2019-12-17 12:34:28,964] jepsen worker 20 - jepsen.util 8120 :fail :cas [6 [18 3]] :ValueNotSwapFrom INFO [2019-12-17 12:34:28,964] jepsen worker 20 - jepsen.util 8120 :invoke :cas [6 [38 37]] INFO [2019-12-17 12:34:28,992] jepsen worker 51 - jepsen.util 7791 :fail :cas [17 [20 42]] :ValueNotSwapFrom INFO [2019-12-17 12:34:28,992] jepsen worker 51 - jepsen.util 7791 :invoke :cas [17 [49 29]] INFO [2019-12-17 12:34:28,998] jepsen worker 3 - jepsen.util 8193 :fail :cas [1 [32 9]] :ValueNotSwapFrom INFO [2019-12-17 12:34:28,998] jepsen worker 3 - jepsen.util 8193 :invoke :write [1 28] INFO [2019-12-17 12:34:29,000] jepsen worker 65 - jepsen.util 6995 :ok :write [21 44] INFO [2019-12-17 12:34:29,000] jepsen worker 65 - jepsen.util 6995 :invoke :write [21 28] INFO [2019-12-17 12:34:29,005] jepsen worker 20 - jepsen.util 8120 :fail :cas [6 [38 37]] :ValueNotSwapFrom INFO [2019-12-17 12:34:29,005] jepsen worker 20 - jepsen.util 8120 :invoke :read [6 nil] INFO [2019-12-17 12:34:29,029] jepsen worker 89 - jepsen.util 8189 :info :write [29 5] :RequestCanceledException INFO [2019-12-17 12:34:29,032] jepsen worker 89 - jepsen.util 8279 :invoke :write [29 2] INFO [2019-12-17 12:34:29,041] jepsen worker 70 - jepsen.util 8710 :fail :cas [23 [25 18]] :ValueNotSwapFrom INFO [2019-12-17 12:34:29,041] jepsen worker 70 - jepsen.util 8710 :invoke :write [23 20] INFO [2019-12-17 12:34:29,051] jepsen worker 29 - jepsen.util 6869 :ok :write [9 40] INFO [2019-12-17 12:34:29,051] jepsen worker 29 - jepsen.util 6869 :invoke :read [9 nil] INFO [2019-12-17 12:34:29,061] jepsen worker 5 - jepsen.util 7835 :fail :cas [1 [31 15]] :ValueNotSwapFrom INFO [2019-12-17 12:34:29,061] jepsen worker 5 - jepsen.util 7835 :invoke :read [1 nil] INFO [2019-12-17 12:34:29,066] jepsen worker 29 - jepsen.util 6869 :ok :read [9 40] INFO [2019-12-17 12:34:29,066] jepsen worker 29 - jepsen.util 6869 :invoke :write [9 14] INFO [2019-12-17 12:34:29,093] jepsen worker 65 - jepsen.util 6995 :ok :write [21 28] INFO [2019-12-17 12:34:29,093] jepsen worker 65 - jepsen.util 6995 :invoke :write [21 7] INFO [2019-12-17 12:34:29,097] jepsen worker 71 - jepsen.util 8891 :fail :cas [23 [16 27]] :ValueNotSwapFrom INFO [2019-12-17 12:34:29,097] jepsen worker 71 - jepsen.util 8891 :invoke :cas [23 [48 10]] INFO [2019-12-17 12:34:29,103] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":48330,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:34:29,103] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":48330,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:34:29,104] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":48334,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:34:29,104] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":48334,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:34:29,127] jepsen worker 51 - jepsen.util 7791 :fail :cas [17 [49 29]] :ValueNotSwapFrom INFO [2019-12-17 12:34:29,128] jepsen worker 51 - jepsen.util 7791 :invoke :write [17 1] INFO [2019-12-17 12:34:29,137] jepsen worker 53 - jepsen.util 7703 :ok :read [17 28] INFO [2019-12-17 12:34:29,138] jepsen worker 53 - jepsen.util 7703 :invoke :cas [17 [19 20]] INFO [2019-12-17 12:34:29,139] jepsen worker 29 - jepsen.util 6869 :ok :write [9 14] INFO [2019-12-17 12:34:29,139] jepsen worker 29 - jepsen.util 6869 :invoke :cas [9 [6 40]] INFO [2019-12-17 12:34:29,143] jepsen worker 18 - jepsen.util 8568 :fail :read [6 nil] :RequestTimeoutException INFO [2019-12-17 12:34:29,143] jepsen worker 33 - jepsen.util 11283 :info :cas [11 [12 16]] :RequestCanceledException INFO [2019-12-17 12:34:29,143] jepsen worker 18 - jepsen.util 8568 :invoke :write [6 23] INFO [2019-12-17 12:34:29,144] jepsen worker 33 - jepsen.util 11373 :invoke :write [11 38] INFO [2019-12-17 12:34:29,154] jepsen worker 14 - jepsen.util 8654 :ok :read [4 27] INFO [2019-12-17 12:34:29,154] jepsen worker 14 - jepsen.util 8654 :invoke :read [4 nil] INFO [2019-12-17 12:34:29,160] jepsen worker 57 - jepsen.util 11307 :info :cas [19 [48 38]] :RequestCanceledException INFO [2019-12-17 12:34:29,161] jepsen worker 57 - jepsen.util 11397 :invoke :cas [19 [2 23]] INFO [2019-12-17 12:34:29,165] jepsen worker 29 - jepsen.util 6869 :fail :cas [9 [6 40]] :ValueNotSwapFrom INFO [2019-12-17 12:34:29,165] jepsen worker 29 - jepsen.util 6869 :invoke :cas [9 [3 16]] INFO [2019-12-17 12:34:29,182] jepsen worker 65 - jepsen.util 6995 :ok :write [21 7] INFO [2019-12-17 12:34:29,182] jepsen worker 65 - jepsen.util 6995 :invoke :write [21 48] INFO [2019-12-17 12:34:29,188] jepsen worker 29 - jepsen.util 6869 :fail :cas [9 [3 16]] :ValueNotSwapFrom INFO [2019-12-17 12:34:29,188] jepsen worker 29 - jepsen.util 6869 :invoke :write [9 39] INFO [2019-12-17 12:34:29,196] jepsen worker 62 - jepsen.util 11582 :info :write [20 29] :RequestCanceledException INFO [2019-12-17 12:34:29,196] jepsen worker 53 - jepsen.util 7703 :fail :cas [17 [19 20]] :ValueNotSwapFrom INFO [2019-12-17 12:34:29,197] jepsen worker 53 - jepsen.util 7703 :invoke :read [17 nil] INFO [2019-12-17 12:34:29,197] jepsen worker 62 - jepsen.util 11672 :invoke :cas [20 [32 6]] INFO [2019-12-17 12:34:29,222] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":48355,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:34:29,222] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":48355,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:34:29,222] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":48358,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:34:29,222] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":48358,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:34:29,229] jepsen worker 25 - jepsen.util 11995 :info :cas [8 [7 0]] :RequestCanceledException INFO [2019-12-17 12:34:29,229] jepsen worker 31 - jepsen.util 7411 :fail :read [10 nil] :RequestTimeoutException INFO [2019-12-17 12:34:29,229] jepsen worker 31 - jepsen.util 7411 :invoke :cas [10 [13 22]] INFO [2019-12-17 12:34:29,232] jepsen worker 25 - jepsen.util 12085 :invoke :write [8 29] INFO [2019-12-17 12:34:29,237] jepsen worker 14 - jepsen.util 8654 :ok :read [4 27] INFO [2019-12-17 12:34:29,237] jepsen worker 14 - jepsen.util 8654 :invoke :cas [4 [8 1]] INFO [2019-12-17 12:34:29,238] jepsen worker 78 - jepsen.util 8088 :info :write [26 47] :RequestCanceledException INFO [2019-12-17 12:34:29,239] jepsen worker 78 - jepsen.util 8178 :invoke :cas [26 [30 36]] INFO [2019-12-17 12:34:29,248] jepsen worker 65 - jepsen.util 6995 :ok :write [21 48] INFO [2019-12-17 12:34:29,248] jepsen worker 65 - jepsen.util 6995 :invoke :write [21 11] INFO [2019-12-17 12:34:29,268] jepsen worker 29 - jepsen.util 6869 :ok :write [9 39] INFO [2019-12-17 12:34:29,268] jepsen worker 29 - jepsen.util 6869 :invoke :cas [9 [32 40]] INFO [2019-12-17 12:34:29,275] jepsen worker 31 - jepsen.util 7411 :fail :cas [10 [13 22]] :ValueNotSwapFrom INFO [2019-12-17 12:34:29,275] jepsen worker 31 - jepsen.util 7411 :invoke :write [10 37] INFO [2019-12-17 12:34:29,278] jepsen worker 53 - jepsen.util 7703 :ok :read [17 28] INFO [2019-12-17 12:34:29,278] jepsen worker 53 - jepsen.util 7703 :invoke :cas [17 [45 45]] INFO [2019-12-17 12:34:29,279] jepsen worker 12 - jepsen.util 8922 :fail :cas [4 [31 40]] :ValueNotSwapFrom INFO [2019-12-17 12:34:29,279] jepsen worker 12 - jepsen.util 8922 :invoke :cas [4 [27 3]] INFO [2019-12-17 12:34:29,292] jepsen worker 0 - jepsen.util 11790 :info :cas [0 [21 12]] :RequestCanceledException INFO [2019-12-17 12:34:29,293] jepsen worker 0 - jepsen.util 11880 :invoke :read [0 nil] INFO [2019-12-17 12:34:29,299] jepsen worker 14 - jepsen.util 8654 :fail :cas [4 [8 1]] :ValueNotSwapFrom INFO [2019-12-17 12:34:29,300] jepsen worker 14 - jepsen.util 8654 :invoke :read [4 nil] INFO [2019-12-17 12:34:29,315] jepsen worker 29 - jepsen.util 6869 :fail :cas [9 [32 40]] :ValueNotSwapFrom INFO [2019-12-17 12:34:29,315] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":48365,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:34:29,315] jepsen worker 29 - jepsen.util 6869 :invoke :read [9 nil] INFO [2019-12-17 12:34:29,315] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":48365,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:34:29,316] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":48368,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:34:29,316] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":48368,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:34:29,329] jepsen worker 58 - jepsen.util 12298 :info :write [19 7] :RequestCanceledException INFO [2019-12-17 12:34:29,329] jepsen worker 65 - jepsen.util 6995 :ok :write [21 11] INFO [2019-12-17 12:34:29,329] jepsen worker 65 - jepsen.util 6995 :invoke :read [21 nil] INFO [2019-12-17 12:34:29,330] jepsen worker 58 - jepsen.util 12388 :invoke :read [19 nil] INFO [2019-12-17 12:34:29,337] jepsen worker 29 - jepsen.util 6869 :ok :read [9 39] INFO [2019-12-17 12:34:29,337] jepsen worker 29 - jepsen.util 6869 :invoke :cas [9 [30 10]] INFO [2019-12-17 12:34:29,343] jepsen worker 85 - jepsen.util 12145 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:34:29,343] jepsen worker 85 - jepsen.util 12145 :invoke :cas [28 [11 16]] INFO [2019-12-17 12:34:29,345] jepsen worker 65 - jepsen.util 6995 :ok :read [21 11] INFO [2019-12-17 12:34:29,346] jepsen worker 65 - jepsen.util 6995 :invoke :read [21 nil] INFO [2019-12-17 12:34:29,355] jepsen worker 29 - jepsen.util 6869 :fail :cas [9 [30 10]] :ValueNotSwapFrom INFO [2019-12-17 12:34:29,355] jepsen worker 29 - jepsen.util 6869 :invoke :write [9 28] INFO [2019-12-17 12:34:29,355] jepsen worker 45 - jepsen.util 8595 :info :write [15 48] :RequestCanceledException INFO [2019-12-17 12:34:29,356] jepsen worker 45 - jepsen.util 8685 :invoke :read [15 nil] INFO [2019-12-17 12:34:29,362] jepsen worker 65 - jepsen.util 6995 :ok :read [21 11] INFO [2019-12-17 12:34:29,363] jepsen worker 65 - jepsen.util 6995 :invoke :cas [21 [20 20]] INFO [2019-12-17 12:34:29,371] jepsen worker 45 - jepsen.util 8685 :ok :read [15 35] INFO [2019-12-17 12:34:29,371] jepsen worker 45 - jepsen.util 8685 :invoke :cas [15 [49 5]] INFO [2019-12-17 12:34:29,377] jepsen worker 80 - jepsen.util 8180 :fail :cas [26 [27 18]] :ValueNotSwapFrom INFO [2019-12-17 12:34:29,377] jepsen worker 80 - jepsen.util 8180 :invoke :cas [26 [45 35]] INFO [2019-12-17 12:34:29,385] jepsen worker 5 - jepsen.util 7835 :ok :read [1 10] INFO [2019-12-17 12:34:29,385] jepsen worker 5 - jepsen.util 7835 :invoke :write [1 12] INFO [2019-12-17 12:34:29,394] jepsen worker 65 - jepsen.util 6995 :fail :cas [21 [20 20]] :ValueNotSwapFrom INFO [2019-12-17 12:34:29,394] jepsen worker 65 - jepsen.util 6995 :invoke :read [21 nil] INFO [2019-12-17 12:34:29,396] jepsen worker 48 - jepsen.util 10578 :info :write [16 42] :RequestCanceledException INFO [2019-12-17 12:34:29,397] jepsen worker 48 - jepsen.util 10668 :invoke :cas [16 [41 20]] INFO [2019-12-17 12:34:29,411] jepsen worker 26 - jepsen.util 11906 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:34:29,411] jepsen worker 53 - jepsen.util 7703 :fail :cas [17 [45 45]] :ValueNotSwapFrom INFO [2019-12-17 12:34:29,412] jepsen worker 26 - jepsen.util 11906 :invoke :read [8 nil] INFO [2019-12-17 12:34:29,412] jepsen worker 53 - jepsen.util 7703 :invoke :cas [17 [25 21]] INFO [2019-12-17 12:34:29,413] jepsen worker 71 - jepsen.util 8891 :fail :cas [23 [48 10]] :ValueNotSwapFrom INFO [2019-12-17 12:34:29,413] jepsen worker 71 - jepsen.util 8891 :invoke :read [23 nil] INFO [2019-12-17 12:34:29,422] jepsen worker 45 - jepsen.util 8685 :fail :cas [15 [49 5]] :ValueNotSwapFrom INFO [2019-12-17 12:34:29,422] jepsen worker 45 - jepsen.util 8685 :invoke :write [15 13] INFO [2019-12-17 12:34:29,429] jepsen worker 22 - jepsen.util 11362 :info :write [7 41] :RequestCanceledException INFO [2019-12-17 12:34:29,429] jepsen worker 87 - jepsen.util 7737 :info :write [29 27] :RequestCanceledException INFO [2019-12-17 12:34:29,430] jepsen worker 22 - jepsen.util 11452 :invoke :read [7 nil] INFO [2019-12-17 12:34:29,430] jepsen worker 87 - jepsen.util 7827 :invoke :read [29 nil] INFO [2019-12-17 12:34:29,431] jepsen worker 14 - jepsen.util 8654 :ok :read [4 27] INFO [2019-12-17 12:34:29,431] jepsen worker 14 - jepsen.util 8654 :invoke :read [4 nil] INFO [2019-12-17 12:34:29,436] jepsen worker 65 - jepsen.util 6995 :ok :read [21 11] INFO [2019-12-17 12:34:29,437] jepsen worker 65 - jepsen.util 6995 :invoke :read [21 nil] INFO [2019-12-17 12:34:29,443] jepsen worker 88 - jepsen.util 8548 :info :write [29 28] :RequestCanceledException INFO [2019-12-17 12:34:29,444] jepsen worker 88 - jepsen.util 8638 :invoke :read [29 nil] INFO [2019-12-17 12:34:29,457] jepsen worker 29 - jepsen.util 6869 :ok :write [9 28] INFO [2019-12-17 12:34:29,458] jepsen worker 29 - jepsen.util 6869 :invoke :cas [9 [4 48]] INFO [2019-12-17 12:34:29,460] jepsen worker 30 - jepsen.util 8400 :fail :read [10 nil] :RequestTimeoutException INFO [2019-12-17 12:34:29,460] jepsen worker 30 - jepsen.util 8400 :invoke :read [10 nil] INFO [2019-12-17 12:34:29,476] jepsen worker 29 - jepsen.util 6869 :fail :cas [9 [4 48]] :ValueNotSwapFrom INFO [2019-12-17 12:34:29,476] jepsen worker 29 - jepsen.util 6869 :invoke :read [9 nil] INFO [2019-12-17 12:34:29,487] jepsen worker 71 - jepsen.util 8891 :ok :read [23 46] INFO [2019-12-17 12:34:29,487] jepsen worker 71 - jepsen.util 8891 :invoke :read [23 nil] INFO [2019-12-17 12:34:29,487] jepsen worker 29 - jepsen.util 6869 :ok :read [9 28] INFO [2019-12-17 12:34:29,487] jepsen worker 29 - jepsen.util 6869 :invoke :write [9 6] INFO [2019-12-17 12:34:29,496] jepsen worker 6 - jepsen.util 11976 :info :cas [2 [39 25]] :RequestCanceledException INFO [2019-12-17 12:34:29,497] jepsen worker 6 - jepsen.util 12066 :invoke :read [2 nil] INFO [2019-12-17 12:34:29,514] jepsen worker 61 - jepsen.util 12211 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:34:29,514] jepsen worker 66 - jepsen.util 11316 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:34:29,514] jepsen worker 61 - jepsen.util 12211 :invoke :read [20 nil] INFO [2019-12-17 12:34:29,514] jepsen worker 66 - jepsen.util 11316 :invoke :cas [22 [14 33]] INFO [2019-12-17 12:34:29,524] jepsen worker 29 - jepsen.util 6869 :ok :write [9 6] INFO [2019-12-17 12:34:29,524] jepsen worker 29 - jepsen.util 6869 :invoke :read [9 nil] INFO [2019-12-17 12:34:29,524] jepsen worker 55 - jepsen.util 9055 :ok :read [18 12] INFO [2019-12-17 12:34:29,524] jepsen worker 55 - jepsen.util 9055 :invoke :read [18 nil] INFO [2019-12-17 12:34:29,529] jepsen worker 44 - jepsen.util 11564 :info :write [14 6] :RequestCanceledException INFO [2019-12-17 12:34:29,530] jepsen worker 84 - jepsen.util 10974 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:34:29,530] jepsen worker 84 - jepsen.util 10974 :invoke :cas [28 [33 36]] INFO [2019-12-17 12:34:29,533] jepsen worker 44 - jepsen.util 11654 :invoke :cas [14 [28 13]] INFO [2019-12-17 12:34:29,534] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":48403,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:34:29,534] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":48403,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:34:29,534] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":48405,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:34:29,534] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":48405,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:34:29,543] jepsen worker 29 - jepsen.util 6869 :ok :read [9 6] INFO [2019-12-17 12:34:29,543] jepsen worker 29 - jepsen.util 6869 :invoke :read [9 nil] INFO [2019-12-17 12:34:29,556] jepsen worker 29 - jepsen.util 6869 :ok :read [9 6] INFO [2019-12-17 12:34:29,557] jepsen worker 29 - jepsen.util 6869 :invoke :write [9 47] INFO [2019-12-17 12:34:29,598] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:34:29,598] jepsen nemesis - couchbase.util Rebalance check count: 375 INFO [2019-12-17 12:34:29,598] jepsen nemesis - couchbase.util Rebalance stuck count: 356 INFO [2019-12-17 12:34:29,599] jepsen worker 29 - jepsen.util 6869 :ok :write [9 47] INFO [2019-12-17 12:34:29,600] jepsen worker 29 - jepsen.util 6869 :invoke :write [9 14] INFO [2019-12-17 12:34:29,627] jepsen worker 55 - jepsen.util 9055 :ok :read [18 12] INFO [2019-12-17 12:34:29,627] jepsen worker 55 - jepsen.util 9055 :invoke :read [18 nil] INFO [2019-12-17 12:34:29,650] jepsen worker 29 - jepsen.util 6869 :ok :write [9 14] INFO [2019-12-17 12:34:29,650] jepsen worker 29 - jepsen.util 6869 :invoke :write [9 26] INFO [2019-12-17 12:34:29,650] jepsen worker 78 - jepsen.util 8178 :fail :cas [26 [30 36]] :ValueNotSwapFrom INFO [2019-12-17 12:34:29,650] jepsen worker 78 - jepsen.util 8178 :invoke :cas [26 [34 27]] INFO [2019-12-17 12:34:29,696] jepsen worker 40 - jepsen.util 11650 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:34:29,696] jepsen worker 40 - jepsen.util 11650 :invoke :read [13 nil] INFO [2019-12-17 12:34:29,698] jepsen worker 29 - jepsen.util 6869 :ok :write [9 26] INFO [2019-12-17 12:34:29,699] jepsen worker 29 - jepsen.util 6869 :invoke :write [9 25] INFO [2019-12-17 12:34:29,708] jepsen worker 28 - jepsen.util 6598 :info :write [9 29] :RequestCanceledException INFO [2019-12-17 12:34:29,709] jepsen worker 28 - jepsen.util 6688 :invoke :cas [9 [1 38]] INFO [2019-12-17 12:34:29,727] jepsen worker 55 - jepsen.util 9055 :ok :read [18 12] INFO [2019-12-17 12:34:29,727] jepsen worker 55 - jepsen.util 9055 :invoke :write [18 5] INFO [2019-12-17 12:34:29,729] jepsen worker 24 - jepsen.util 10464 :info :cas [8 [49 27]] :RequestCanceledException INFO [2019-12-17 12:34:29,729] jepsen worker 87 - jepsen.util 7827 :ok :read [29 34] INFO [2019-12-17 12:34:29,729] jepsen worker 87 - jepsen.util 7827 :invoke :read [29 nil] INFO [2019-12-17 12:34:29,731] jepsen worker 24 - jepsen.util 10554 :invoke :cas [8 [35 24]] INFO [2019-12-17 12:34:29,743] jepsen worker 36 - jepsen.util 8586 :info :write [12 22] :RequestCanceledException INFO [2019-12-17 12:34:29,744] jepsen worker 36 - jepsen.util 8676 :invoke :read [12 nil] INFO [2019-12-17 12:34:29,750] jepsen worker 29 - jepsen.util 6869 :ok :write [9 25] INFO [2019-12-17 12:34:29,750] jepsen worker 29 - jepsen.util 6869 :invoke :read [9 nil] INFO [2019-12-17 12:34:29,754] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":48426,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:34:29,754] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":48426,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:34:29,754] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":48429,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:34:29,754] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":48429,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:34:29,761] jepsen worker 10 - jepsen.util 8560 :info :cas [3 [20 42]] :RequestCanceledException INFO [2019-12-17 12:34:29,761] jepsen worker 1 - jepsen.util 11701 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:34:29,761] jepsen worker 1 - jepsen.util 11701 :invoke :read [0 nil] INFO [2019-12-17 12:34:29,764] jepsen worker 10 - jepsen.util 8650 :invoke :read [3 nil] INFO [2019-12-17 12:34:29,770] jepsen worker 29 - jepsen.util 6869 :ok :read [9 25] INFO [2019-12-17 12:34:29,770] jepsen worker 29 - jepsen.util 6869 :invoke :read [9 nil] INFO [2019-12-17 12:34:29,779] jepsen worker 32 - jepsen.util 8582 :ok :write [10 43] INFO [2019-12-17 12:34:29,779] jepsen worker 32 - jepsen.util 8582 :invoke :cas [10 [34 37]] INFO [2019-12-17 12:34:29,780] jepsen worker 73 - jepsen.util 8083 :ok :write [24 43] INFO [2019-12-17 12:34:29,780] jepsen worker 46 - jepsen.util 8416 :ok :write [15 0] INFO [2019-12-17 12:34:29,780] jepsen worker 73 - jepsen.util 8083 :invoke :cas [24 [0 49]] INFO [2019-12-17 12:34:29,780] jepsen worker 46 - jepsen.util 8416 :invoke :write [15 16] INFO [2019-12-17 12:34:29,800] jepsen worker 29 - jepsen.util 6869 :ok :read [9 25] INFO [2019-12-17 12:34:29,800] jepsen worker 29 - jepsen.util 6869 :invoke :cas [9 [17 39]] INFO [2019-12-17 12:34:29,814] jepsen worker 15 - jepsen.util 10545 :info :write [5 16] :RequestCanceledException INFO [2019-12-17 12:34:29,814] jepsen worker 38 - jepsen.util 8588 :info :write [12 24] :RequestCanceledException INFO [2019-12-17 12:34:29,815] jepsen worker 38 - jepsen.util 8678 :invoke :write [12 7] INFO [2019-12-17 12:34:29,817] jepsen worker 15 - jepsen.util 10635 :invoke :write [5 3] INFO [2019-12-17 12:34:29,819] jepsen worker 53 - jepsen.util 7703 :fail :cas [17 [25 21]] :ValueNotSwapFrom INFO [2019-12-17 12:34:29,819] jepsen worker 53 - jepsen.util 7703 :invoke :write [17 10] INFO [2019-12-17 12:34:29,830] jepsen worker 17 - jepsen.util 11357 :info :cas [5 [19 0]] :RequestCanceledException INFO [2019-12-17 12:34:29,830] jepsen worker 42 - jepsen.util 12462 :info :cas [14 [3 12]] :RequestCanceledException INFO [2019-12-17 12:34:29,831] jepsen worker 42 - jepsen.util 12552 :invoke :read [14 nil] INFO [2019-12-17 12:34:29,831] jepsen worker 17 - jepsen.util 11447 :invoke :read [5 nil] INFO [2019-12-17 12:34:29,843] jepsen worker 79 - jepsen.util 8719 :info :write [26 18] :RequestCanceledException INFO [2019-12-17 12:34:29,845] jepsen worker 79 - jepsen.util 8809 :invoke :read [26 nil] INFO [2019-12-17 12:34:29,854] jepsen worker 74 - jepsen.util 8624 :ok :read [24 43] INFO [2019-12-17 12:34:29,854] jepsen worker 74 - jepsen.util 8624 :invoke :read [24 nil] INFO [2019-12-17 12:34:29,855] jepsen worker 14 - jepsen.util 8654 :ok :read [4 27] INFO [2019-12-17 12:34:29,855] jepsen worker 14 - jepsen.util 8654 :invoke :write [4 22] INFO [2019-12-17 12:34:29,862] jepsen worker 28 - jepsen.util 6688 :fail :cas [9 [1 38]] :ValueNotSwapFrom INFO [2019-12-17 12:34:29,862] jepsen worker 28 - jepsen.util 6688 :invoke :write [9 46] INFO [2019-12-17 12:34:29,873] jepsen worker 29 - jepsen.util 6869 :fail :cas [9 [17 39]] :ValueNotSwapFrom INFO [2019-12-17 12:34:29,873] jepsen worker 29 - jepsen.util 6869 :invoke :write [9 12] INFO [2019-12-17 12:34:29,896] jepsen worker 8 - jepsen.util 11348 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:34:29,896] jepsen worker 8 - jepsen.util 11348 :invoke :read [2 nil] INFO [2019-12-17 12:34:29,905] jepsen worker 80 - jepsen.util 8180 :fail :cas [26 [45 35]] :ValueNotSwapFrom INFO [2019-12-17 12:34:29,905] jepsen worker 80 - jepsen.util 8180 :invoke :read [26 nil] INFO [2019-12-17 12:34:29,908] jepsen worker 30 - jepsen.util 8400 :ok :read [10 43] INFO [2019-12-17 12:34:29,908] jepsen worker 30 - jepsen.util 8400 :invoke :cas [10 [27 2]] INFO [2019-12-17 12:34:29,909] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":48457,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:34:29,910] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":48457,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:34:29,910] jepsen worker 77 - jepsen.util 11597 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:34:29,910] jepsen worker 43 - jepsen.util 11113 :info :cas [14 [31 48]] :RequestCanceledException INFO [2019-12-17 12:34:29,910] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":48461,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:34:29,910] jepsen worker 77 - jepsen.util 11597 :invoke :write [25 3] INFO [2019-12-17 12:34:29,910] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":48461,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:34:29,911] jepsen worker 43 - jepsen.util 11203 :invoke :read [14 nil] INFO [2019-12-17 12:34:29,915] jepsen worker 32 - jepsen.util 8582 :fail :cas [10 [34 37]] :ValueNotSwapFrom INFO [2019-12-17 12:34:29,915] jepsen worker 32 - jepsen.util 8582 :invoke :read [10 nil] INFO [2019-12-17 12:34:29,919] jepsen worker 88 - jepsen.util 8638 :ok :read [29 34] INFO [2019-12-17 12:34:29,919] jepsen worker 88 - jepsen.util 8638 :invoke :cas [29 [39 36]] INFO [2019-12-17 12:34:29,943] jepsen worker 75 - jepsen.util 10875 :info :write [25 47] :RequestCanceledException INFO [2019-12-17 12:34:29,944] jepsen worker 28 - jepsen.util 6688 :ok :write [9 46] INFO [2019-12-17 12:34:29,944] jepsen worker 28 - jepsen.util 6688 :invoke :write [9 46] INFO [2019-12-17 12:34:29,944] jepsen worker 75 - jepsen.util 10965 :invoke :read [25 nil] INFO [2019-12-17 12:34:29,973] jepsen worker 32 - jepsen.util 8582 :ok :read [10 43] INFO [2019-12-17 12:34:29,973] jepsen worker 32 - jepsen.util 8582 :invoke :write [10 34] INFO [2019-12-17 12:34:29,990] jepsen worker 78 - jepsen.util 8178 :fail :cas [26 [34 27]] :ValueNotSwapFrom INFO [2019-12-17 12:34:29,990] jepsen worker 78 - jepsen.util 8178 :invoke :read [26 nil] INFO [2019-12-17 12:34:30,008] jepsen worker 28 - jepsen.util 6688 :ok :write [9 46] INFO [2019-12-17 12:34:30,008] jepsen worker 28 - jepsen.util 6688 :invoke :read [9 nil] INFO [2019-12-17 12:34:30,030] jepsen worker 56 - jepsen.util 9236 :info :write [18 9] :RequestCanceledException INFO [2019-12-17 12:34:30,031] jepsen worker 56 - jepsen.util 9326 :invoke :cas [18 [30 41]] INFO [2019-12-17 12:34:30,037] jepsen worker 28 - jepsen.util 6688 :ok :read [9 46] INFO [2019-12-17 12:34:30,038] jepsen worker 28 - jepsen.util 6688 :invoke :write [9 38] INFO [2019-12-17 12:34:30,042] jepsen worker 63 - jepsen.util 7083 :info :write [21 20] :RequestCanceledException INFO [2019-12-17 12:34:30,043] jepsen worker 63 - jepsen.util 7173 :invoke :write [21 20] INFO [2019-12-17 12:34:30,044] jepsen worker 80 - jepsen.util 8180 :ok :read [26 16] INFO [2019-12-17 12:34:30,045] jepsen worker 80 - jepsen.util 8180 :invoke :read [26 nil] INFO [2019-12-17 12:34:30,055] jepsen worker 18 - jepsen.util 8568 :ok :write [6 23] INFO [2019-12-17 12:34:30,056] jepsen worker 18 - jepsen.util 8568 :invoke :write [6 9] INFO [2019-12-17 12:34:30,060] jepsen worker 87 - jepsen.util 7827 :ok :read [29 34] INFO [2019-12-17 12:34:30,060] jepsen worker 87 - jepsen.util 7827 :invoke :cas [29 [0 46]] INFO [2019-12-17 12:34:30,076] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":48479,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:34:30,076] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":48479,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:34:30,076] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":48481,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:34:30,076] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":48481,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:34:30,078] jepsen worker 10 - jepsen.util 8650 :ok :read [3 1] INFO [2019-12-17 12:34:30,079] jepsen worker 10 - jepsen.util 8650 :invoke :write [3 13] INFO [2019-12-17 12:34:30,080] jepsen worker 36 - jepsen.util 8676 :ok :read [12 40] INFO [2019-12-17 12:34:30,080] jepsen worker 36 - jepsen.util 8676 :invoke :read [12 nil] INFO [2019-12-17 12:34:30,085] jepsen worker 46 - jepsen.util 8416 :ok :write [15 16] INFO [2019-12-17 12:34:30,085] jepsen worker 46 - jepsen.util 8416 :invoke :write [15 28] INFO [2019-12-17 12:34:30,095] jepsen worker 7 - jepsen.util 11617 :info :write [2 23] :RequestCanceledException INFO [2019-12-17 12:34:30,096] jepsen worker 7 - jepsen.util 11707 :invoke :write [2 4] INFO [2019-12-17 12:34:30,100] jepsen worker 88 - jepsen.util 8638 :fail :cas [29 [39 36]] :ValueNotSwapFrom INFO [2019-12-17 12:34:30,100] jepsen worker 88 - jepsen.util 8638 :invoke :cas [29 [13 34]] INFO [2019-12-17 12:34:30,114] jepsen worker 49 - jepsen.util 10849 :info :cas [16 [32 41]] :RequestCanceledException INFO [2019-12-17 12:34:30,115] jepsen worker 49 - jepsen.util 10939 :invoke :cas [16 [46 3]] INFO [2019-12-17 12:34:30,129] jepsen worker 2 - jepsen.util 10982 :info :cas [0 [5 44]] :RequestCanceledException INFO [2019-12-17 12:34:30,129] jepsen worker 11 - jepsen.util 8291 :info :cas [3 [26 26]] :RequestCanceledException INFO [2019-12-17 12:34:30,129] jepsen worker 64 - jepsen.util 6814 :info :cas [21 [47 49]] :RequestCanceledException INFO [2019-12-17 12:34:30,130] jepsen worker 2 - jepsen.util 11072 :invoke :write [0 49] INFO [2019-12-17 12:34:30,130] jepsen worker 64 - jepsen.util 6904 :invoke :write [21 20] INFO [2019-12-17 12:34:30,130] jepsen worker 11 - jepsen.util 8381 :invoke :write [3 13] INFO [2019-12-17 12:34:30,132] jepsen worker 73 - jepsen.util 8083 :fail :cas [24 [0 49]] :ValueNotSwapFrom INFO [2019-12-17 12:34:30,132] jepsen worker 73 - jepsen.util 8083 :invoke :read [24 nil] INFO [2019-12-17 12:34:30,139] jepsen worker 74 - jepsen.util 8624 :ok :read [24 43] INFO [2019-12-17 12:34:30,139] jepsen worker 74 - jepsen.util 8624 :invoke :cas [24 [29 36]] INFO [2019-12-17 12:34:30,157] jepsen worker 54 - jepsen.util 9324 :info :write [18 12] :RequestCanceledException INFO [2019-12-17 12:34:30,158] jepsen worker 54 - jepsen.util 9414 :invoke :write [18 36] INFO [2019-12-17 12:34:30,160] jepsen worker 30 - jepsen.util 8400 :fail :cas [10 [27 2]] :ValueNotSwapFrom INFO [2019-12-17 12:34:30,161] jepsen worker 30 - jepsen.util 8400 :invoke :cas [10 [45 43]] INFO [2019-12-17 12:34:30,178] jepsen worker 28 - jepsen.util 6688 :ok :write [9 38] INFO [2019-12-17 12:34:30,178] jepsen worker 28 - jepsen.util 6688 :invoke :read [9 nil] INFO [2019-12-17 12:34:30,194] jepsen worker 88 - jepsen.util 8638 :fail :cas [29 [13 34]] :ValueNotSwapFrom INFO [2019-12-17 12:34:30,194] jepsen worker 88 - jepsen.util 8638 :invoke :read [29 nil] INFO [2019-12-17 12:34:30,196] jepsen worker 71 - jepsen.util 8891 :ok :read [23 46] INFO [2019-12-17 12:34:30,196] jepsen worker 71 - jepsen.util 8891 :invoke :cas [23 [32 6]] INFO [2019-12-17 12:34:30,199] jepsen worker 28 - jepsen.util 6688 :ok :read [9 38] INFO [2019-12-17 12:34:30,199] jepsen worker 28 - jepsen.util 6688 :invoke :cas [9 [0 7]] INFO [2019-12-17 12:34:30,200] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":48505,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:34:30,200] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":48505,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:34:30,200] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":48506,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:34:30,200] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":48506,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:34:30,200] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":48507,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:34:30,200] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":48507,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:34:30,216] jepsen worker 28 - jepsen.util 6688 :fail :cas [9 [0 7]] :ValueNotSwapFrom INFO [2019-12-17 12:34:30,216] jepsen worker 28 - jepsen.util 6688 :invoke :cas [9 [22 39]] INFO [2019-12-17 12:34:30,237] jepsen worker 28 - jepsen.util 6688 :fail :cas [9 [22 39]] :ValueNotSwapFrom INFO [2019-12-17 12:34:30,237] jepsen worker 28 - jepsen.util 6688 :invoke :cas [9 [19 24]] INFO [2019-12-17 12:34:30,250] jepsen worker 28 - jepsen.util 6688 :fail :cas [9 [19 24]] :ValueNotSwapFrom INFO [2019-12-17 12:34:30,250] jepsen worker 28 - jepsen.util 6688 :invoke :write [9 6] INFO [2019-12-17 12:34:30,255] jepsen worker 18 - jepsen.util 8568 :ok :write [6 9] INFO [2019-12-17 12:34:30,255] jepsen worker 18 - jepsen.util 8568 :invoke :cas [6 [1 40]] INFO [2019-12-17 12:34:30,266] jepsen worker 53 - jepsen.util 7703 :ok :write [17 10] INFO [2019-12-17 12:34:30,266] jepsen worker 53 - jepsen.util 7703 :invoke :write [17 22] INFO [2019-12-17 12:34:30,292] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":48513,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:34:30,292] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":48513,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:34:30,292] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":48517,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:34:30,292] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":48517,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:34:30,294] jepsen worker 13 - jepsen.util 7933 :info :write [4 39] :RequestCanceledException INFO [2019-12-17 12:34:30,295] jepsen worker 13 - jepsen.util 8023 :invoke :cas [4 [34 27]] INFO [2019-12-17 12:34:30,302] jepsen worker 18 - jepsen.util 8568 :fail :cas [6 [1 40]] :ValueNotSwapFrom INFO [2019-12-17 12:34:30,303] jepsen worker 18 - jepsen.util 8568 :invoke :read [6 nil] INFO [2019-12-17 12:34:30,329] jepsen worker 86 - jepsen.util 12236 :info :cas [28 [22 34]] :RequestCanceledException INFO [2019-12-17 12:34:30,330] jepsen worker 81 - jepsen.util 11331 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:34:30,330] jepsen worker 69 - jepsen.util 8529 :info :write [23 26] :RequestCanceledException INFO [2019-12-17 12:34:30,330] jepsen worker 81 - jepsen.util 11331 :invoke :cas [27 [3 34]] INFO [2019-12-17 12:34:30,331] jepsen worker 86 - jepsen.util 12326 :invoke :write [28 1] INFO [2019-12-17 12:34:30,332] jepsen worker 69 - jepsen.util 8619 :invoke :cas [23 [27 10]] INFO [2019-12-17 12:34:30,333] jepsen worker 18 - jepsen.util 8568 :ok :read [6 9] INFO [2019-12-17 12:34:30,334] jepsen worker 18 - jepsen.util 8568 :invoke :cas [6 [36 12]] INFO [2019-12-17 12:34:30,349] jepsen worker 54 - jepsen.util 9414 :ok :write [18 36] INFO [2019-12-17 12:34:30,350] jepsen worker 54 - jepsen.util 9414 :invoke :read [18 nil] INFO [2019-12-17 12:34:30,360] jepsen worker 18 - jepsen.util 8568 :fail :cas [6 [36 12]] :ValueNotSwapFrom INFO [2019-12-17 12:34:30,360] jepsen worker 18 - jepsen.util 8568 :invoke :read [6 nil] INFO [2019-12-17 12:34:30,360] jepsen worker 46 - jepsen.util 8416 :ok :write [15 28] INFO [2019-12-17 12:34:30,360] jepsen worker 46 - jepsen.util 8416 :invoke :write [15 29] INFO [2019-12-17 12:34:30,393] jepsen worker 11 - jepsen.util 8381 :ok :write [3 13] INFO [2019-12-17 12:34:30,394] jepsen worker 11 - jepsen.util 8381 :invoke :cas [3 [15 0]] INFO [2019-12-17 12:34:30,395] jepsen worker 13 - jepsen.util 8023 :fail :cas [4 [34 27]] :ValueNotSwapFrom INFO [2019-12-17 12:34:30,396] jepsen worker 13 - jepsen.util 8023 :invoke :cas [4 [14 38]] INFO [2019-12-17 12:34:30,396] jepsen worker 59 - jepsen.util 11039 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:34:30,396] jepsen worker 59 - jepsen.util 11039 :invoke :cas [19 [0 48]] INFO [2019-12-17 12:34:30,402] jepsen worker 18 - jepsen.util 8568 :ok :read [6 9] INFO [2019-12-17 12:34:30,402] jepsen worker 18 - jepsen.util 8568 :invoke :write [6 41] INFO [2019-12-17 12:34:30,412] jepsen worker 74 - jepsen.util 8624 :fail :cas [24 [29 36]] :ValueNotSwapFrom INFO [2019-12-17 12:34:30,412] jepsen worker 74 - jepsen.util 8624 :invoke :read [24 nil] INFO [2019-12-17 12:34:30,429] jepsen worker 39 - jepsen.util 10839 :info :write [13 48] :RequestCanceledException INFO [2019-12-17 12:34:30,429] jepsen worker 41 - jepsen.util 12011 :info :cas [13 [43 22]] :RequestCanceledException INFO [2019-12-17 12:34:30,430] jepsen worker 41 - jepsen.util 12101 :invoke :write [13 42] INFO [2019-12-17 12:34:30,430] jepsen worker 39 - jepsen.util 10929 :invoke :cas [13 [21 47]] INFO [2019-12-17 12:34:30,435] jepsen worker 79 - jepsen.util 8809 :ok :read [26 16] INFO [2019-12-17 12:34:30,435] jepsen worker 79 - jepsen.util 8809 :invoke :read [26 nil] INFO [2019-12-17 12:34:30,443] jepsen worker 21 - jepsen.util 10911 :info :write [7 34] :RequestCanceledException INFO [2019-12-17 12:34:30,444] jepsen worker 21 - jepsen.util 11001 :invoke :read [7 nil] INFO [2019-12-17 12:34:30,450] jepsen worker 28 - jepsen.util 6688 :ok :write [9 6] INFO [2019-12-17 12:34:30,451] jepsen worker 28 - jepsen.util 6688 :invoke :cas [9 [10 3]] INFO [2019-12-17 12:34:30,459] jepsen worker 11 - jepsen.util 8381 :fail :cas [3 [15 0]] :ValueNotSwapFrom INFO [2019-12-17 12:34:30,459] jepsen worker 11 - jepsen.util 8381 :invoke :cas [3 [13 43]] INFO [2019-12-17 12:34:30,463] jepsen worker 30 - jepsen.util 8400 :fail :cas [10 [45 43]] :ValueNotSwapFrom INFO [2019-12-17 12:34:30,463] jepsen worker 30 - jepsen.util 8400 :invoke :read [10 nil] INFO [2019-12-17 12:34:30,469] jepsen worker 78 - jepsen.util 8178 :ok :read [26 16] INFO [2019-12-17 12:34:30,469] jepsen worker 78 - jepsen.util 8178 :invoke :cas [26 [19 5]] INFO [2019-12-17 12:34:30,487] jepsen worker 28 - jepsen.util 6688 :fail :cas [9 [10 3]] :ValueNotSwapFrom INFO [2019-12-17 12:34:30,487] jepsen worker 28 - jepsen.util 6688 :invoke :write [9 6] INFO [2019-12-17 12:34:30,508] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":48538,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:34:30,508] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":48538,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:34:30,508] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":48543,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:34:30,508] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":48543,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:34:30,511] jepsen worker 13 - jepsen.util 8023 :fail :cas [4 [14 38]] :ValueNotSwapFrom INFO [2019-12-17 12:34:30,511] jepsen worker 13 - jepsen.util 8023 :invoke :cas [4 [7 15]] INFO [2019-12-17 12:34:30,513] jepsen worker 19 - jepsen.util 8569 :info :write [6 5] :RequestCanceledException INFO [2019-12-17 12:34:30,514] jepsen worker 19 - jepsen.util 8659 :invoke :cas [6 [13 22]] INFO [2019-12-17 12:34:30,530] jepsen worker 82 - jepsen.util 11962 :info :write [27 36] :RequestCanceledException INFO [2019-12-17 12:34:30,530] jepsen worker 68 - jepsen.util 10778 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:34:30,530] jepsen worker 68 - jepsen.util 10778 :invoke :write [22 21] INFO [2019-12-17 12:34:30,531] jepsen worker 82 - jepsen.util 12052 :invoke :write [27 34] INFO [2019-12-17 12:34:30,540] jepsen worker 19 - jepsen.util 8659 :fail :cas [6 [13 22]] :ValueNotSwapFrom INFO [2019-12-17 12:34:30,540] jepsen worker 19 - jepsen.util 8659 :invoke :cas [6 [47 28]] INFO [2019-12-17 12:34:30,543] jepsen worker 52 - jepsen.util 8422 :info :write [17 16] :RequestCanceledException INFO [2019-12-17 12:34:30,545] jepsen worker 52 - jepsen.util 8512 :invoke :write [17 31] INFO [2019-12-17 12:34:30,570] jepsen worker 19 - jepsen.util 8659 :fail :cas [6 [47 28]] :ValueNotSwapFrom INFO [2019-12-17 12:34:30,571] jepsen worker 19 - jepsen.util 8659 :invoke :write [6 22] INFO [2019-12-17 12:34:30,573] jepsen worker 28 - jepsen.util 6688 :ok :write [9 6] INFO [2019-12-17 12:34:30,573] jepsen worker 28 - jepsen.util 6688 :invoke :cas [9 [18 17]] INFO [2019-12-17 12:34:30,579] jepsen worker 13 - jepsen.util 8023 :fail :cas [4 [7 15]] :ValueNotSwapFrom INFO [2019-12-17 12:34:30,579] jepsen worker 13 - jepsen.util 8023 :invoke :write [4 5] INFO [2019-12-17 12:34:30,587] jepsen worker 87 - jepsen.util 7827 :fail :cas [29 [0 46]] :ValueNotSwapFrom INFO [2019-12-17 12:34:30,587] jepsen worker 87 - jepsen.util 7827 :invoke :write [29 25] INFO [2019-12-17 12:34:30,594] jepsen worker 28 - jepsen.util 6688 :fail :cas [9 [18 17]] :ValueNotSwapFrom INFO [2019-12-17 12:34:30,594] jepsen worker 28 - jepsen.util 6688 :invoke :read [9 nil] INFO [2019-12-17 12:34:30,596] jepsen worker 76 - jepsen.util 11416 :info :write [25 12] :RequestCanceledException INFO [2019-12-17 12:34:30,597] jepsen worker 76 - jepsen.util 11506 :invoke :cas [25 [1 7]] INFO [2019-12-17 12:34:30,611] jepsen worker 23 - jepsen.util 11093 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:34:30,611] jepsen worker 23 - jepsen.util 11093 :invoke :write [7 36] INFO [2019-12-17 12:34:30,611] jepsen worker 50 - jepsen.util 12110 :info :cas [16 [0 21]] :RequestCanceledException INFO [2019-12-17 12:34:30,614] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:34:30,614] jepsen worker 50 - jepsen.util 12200 :invoke :read [16 nil] INFO [2019-12-17 12:34:30,614] jepsen nemesis - couchbase.util Rebalance check count: 376 INFO [2019-12-17 12:34:30,615] jepsen nemesis - couchbase.util Rebalance stuck count: 357 INFO [2019-12-17 12:34:30,622] jepsen worker 18 - jepsen.util 8568 :ok :write [6 41] INFO [2019-12-17 12:34:30,622] jepsen worker 18 - jepsen.util 8568 :invoke :cas [6 [33 5]] INFO [2019-12-17 12:34:30,624] jepsen worker 28 - jepsen.util 6688 :ok :read [9 6] INFO [2019-12-17 12:34:30,624] jepsen worker 28 - jepsen.util 6688 :invoke :cas [9 [2 8]] INFO [2019-12-17 12:34:30,629] jepsen worker 60 - jepsen.util 12390 :info :write [20 9] :RequestCanceledException INFO [2019-12-17 12:34:30,630] jepsen worker 60 - jepsen.util 12480 :invoke :write [20 34] INFO [2019-12-17 12:34:30,644] jepsen worker 83 - jepsen.util 11783 :info :cas [27 [21 18]] :RequestCanceledException INFO [2019-12-17 12:34:30,645] jepsen worker 83 - jepsen.util 11873 :invoke :cas [27 [35 37]] INFO [2019-12-17 12:34:30,647] jepsen worker 18 - jepsen.util 8568 :fail :cas [6 [33 5]] :ValueNotSwapFrom INFO [2019-12-17 12:34:30,647] jepsen worker 18 - jepsen.util 8568 :invoke :read [6 nil] INFO [2019-12-17 12:34:30,655] jepsen worker 28 - jepsen.util 6688 :fail :cas [9 [2 8]] :ValueNotSwapFrom INFO [2019-12-17 12:34:30,655] jepsen worker 28 - jepsen.util 6688 :invoke :read [9 nil] INFO [2019-12-17 12:34:30,658] jepsen worker 16 - jepsen.util 12436 :info :write [5 7] :RequestCanceledException INFO [2019-12-17 12:34:30,658] jepsen worker 34 - jepsen.util 11284 :info :write [11 32] :RequestCanceledException INFO [2019-12-17 12:34:30,659] jepsen worker 16 - jepsen.util 12526 :invoke :read [5 nil] INFO [2019-12-17 12:34:30,659] jepsen worker 34 - jepsen.util 11374 :invoke :write [11 15] INFO [2019-12-17 12:34:30,670] jepsen worker 18 - jepsen.util 8568 :ok :read [6 41] INFO [2019-12-17 12:34:30,670] jepsen worker 18 - jepsen.util 8568 :invoke :cas [6 [18 15]] INFO [2019-12-17 12:34:30,677] jepsen worker 28 - jepsen.util 6688 :ok :read [9 6] INFO [2019-12-17 12:34:30,677] jepsen worker 28 - jepsen.util 6688 :invoke :write [9 32] INFO [2019-12-17 12:34:30,695] jepsen worker 36 - jepsen.util 8676 :ok :read [12 40] INFO [2019-12-17 12:34:30,695] jepsen worker 36 - jepsen.util 8676 :invoke :read [12 nil] INFO [2019-12-17 12:34:30,697] jepsen worker 18 - jepsen.util 8568 :fail :cas [6 [18 15]] :ValueNotSwapFrom INFO [2019-12-17 12:34:30,697] jepsen worker 18 - jepsen.util 8568 :invoke :read [6 nil] INFO [2019-12-17 12:34:30,716] jepsen worker 18 - jepsen.util 8568 :ok :read [6 41] INFO [2019-12-17 12:34:30,716] jepsen worker 18 - jepsen.util 8568 :invoke :read [6 nil] INFO [2019-12-17 12:34:30,729] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":48571,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:34:30,729] jepsen worker 18 - jepsen.util 8568 :ok :read [6 41] INFO [2019-12-17 12:34:30,729] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":48571,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:34:30,729] jepsen worker 18 - jepsen.util 8568 :invoke :read [6 nil] INFO [2019-12-17 12:34:30,729] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":48575,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:34:30,729] jepsen worker 9 - jepsen.util 8739 :info :cas [3 [31 29]] :RequestCanceledException INFO [2019-12-17 12:34:30,729] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":48575,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:34:30,730] jepsen worker 9 - jepsen.util 8829 :invoke :cas [3 [38 21]] INFO [2019-12-17 12:34:30,742] jepsen worker 80 - jepsen.util 8180 :ok :read [26 16] INFO [2019-12-17 12:34:30,742] jepsen worker 80 - jepsen.util 8180 :invoke :cas [26 [26 4]] INFO [2019-12-17 12:34:30,750] jepsen worker 18 - jepsen.util 8568 :ok :read [6 41] INFO [2019-12-17 12:34:30,750] jepsen worker 18 - jepsen.util 8568 :invoke :read [6 nil] INFO [2019-12-17 12:34:30,758] jepsen worker 28 - jepsen.util 6688 :ok :write [9 32] INFO [2019-12-17 12:34:30,760] jepsen worker 28 - jepsen.util 6688 :invoke :read [9 nil] INFO [2019-12-17 12:34:30,768] jepsen worker 18 - jepsen.util 8568 :ok :read [6 41] INFO [2019-12-17 12:34:30,768] jepsen worker 18 - jepsen.util 8568 :invoke :read [6 nil] INFO [2019-12-17 12:34:30,783] jepsen worker 28 - jepsen.util 6688 :ok :read [9 32] INFO [2019-12-17 12:34:30,783] jepsen worker 28 - jepsen.util 6688 :invoke :cas [9 [30 46]] INFO [2019-12-17 12:34:30,788] jepsen worker 18 - jepsen.util 8568 :ok :read [6 41] INFO [2019-12-17 12:34:30,788] jepsen worker 18 - jepsen.util 8568 :invoke :cas [6 [43 18]] INFO [2019-12-17 12:34:30,805] jepsen worker 54 - jepsen.util 9414 :ok :read [18 36] INFO [2019-12-17 12:34:30,805] jepsen worker 54 - jepsen.util 9414 :invoke :read [18 nil] INFO [2019-12-17 12:34:30,816] jepsen worker 18 - jepsen.util 8568 :fail :cas [6 [43 18]] :ValueNotSwapFrom INFO [2019-12-17 12:34:30,817] jepsen worker 18 - jepsen.util 8568 :invoke :write [6 37] INFO [2019-12-17 12:34:30,823] jepsen worker 28 - jepsen.util 6688 :fail :cas [9 [30 46]] :ValueNotSwapFrom INFO [2019-12-17 12:34:30,823] jepsen worker 28 - jepsen.util 6688 :invoke :cas [9 [28 13]] INFO [2019-12-17 12:34:30,830] jepsen worker 37 - jepsen.util 7957 :info :write [12 32] :RequestCanceledException INFO [2019-12-17 12:34:30,830] jepsen worker 4 - jepsen.util 7834 :info :write [1 5] :RequestCanceledException INFO [2019-12-17 12:34:30,831] jepsen worker 4 - jepsen.util 7924 :invoke :write [1 20] INFO [2019-12-17 12:34:30,832] jepsen worker 37 - jepsen.util 8047 :invoke :read [12 nil] INFO [2019-12-17 12:34:30,850] jepsen worker 28 - jepsen.util 6688 :fail :cas [9 [28 13]] :ValueNotSwapFrom INFO [2019-12-17 12:34:30,850] jepsen worker 28 - jepsen.util 6688 :invoke :cas [9 [28 34]] INFO [2019-12-17 12:34:30,852] jepsen worker 36 - jepsen.util 8676 :ok :read [12 40] INFO [2019-12-17 12:34:30,852] jepsen worker 36 - jepsen.util 8676 :invoke :read [12 nil] INFO [2019-12-17 12:34:30,869] jepsen worker 28 - jepsen.util 6688 :fail :cas [9 [28 34]] :ValueNotSwapFrom INFO [2019-12-17 12:34:30,869] jepsen worker 28 - jepsen.util 6688 :invoke :write [9 26] INFO [2019-12-17 12:34:30,875] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":48603,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:34:30,875] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":48603,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:34:30,875] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":48606,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:34:30,875] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":48606,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:34:30,896] jepsen worker 67 - jepsen.util 11497 :info :write [22 18] :RequestCanceledException INFO [2019-12-17 12:34:30,897] jepsen worker 67 - jepsen.util 11587 :invoke :cas [22 [10 23]] INFO [2019-12-17 12:34:30,920] jepsen worker 28 - jepsen.util 6688 :ok :write [9 26] INFO [2019-12-17 12:34:30,921] jepsen worker 28 - jepsen.util 6688 :invoke :write [9 30] INFO [2019-12-17 12:34:30,929] jepsen worker 27 - jepsen.util 8127 :info :write [9 12] :RequestCanceledException INFO [2019-12-17 12:34:30,930] jepsen worker 35 - jepsen.util 11015 :info :cas [11 [31 31]] :RequestCanceledException INFO [2019-12-17 12:34:30,930] jepsen worker 27 - jepsen.util 8217 :invoke :write [9 27] INFO [2019-12-17 12:34:30,934] jepsen worker 35 - jepsen.util 11105 :invoke :cas [11 [43 27]] INFO [2019-12-17 12:34:30,958] jepsen worker 47 - jepsen.util 8867 :info :write [15 9] :RequestCanceledException INFO [2019-12-17 12:34:30,960] jepsen worker 47 - jepsen.util 8957 :invoke :cas [15 [32 48]] INFO [2019-12-17 12:34:30,961] jepsen worker 54 - jepsen.util 9414 :ok :read [18 36] INFO [2019-12-17 12:34:30,961] jepsen worker 54 - jepsen.util 9414 :invoke :write [18 30] INFO [2019-12-17 12:34:30,961] jepsen worker 71 - jepsen.util 8891 :fail :cas [23 [32 6]] :ValueNotSwapFrom INFO [2019-12-17 12:34:30,962] jepsen worker 71 - jepsen.util 8891 :invoke :read [23 nil] INFO [2019-12-17 12:34:30,994] jepsen worker 28 - jepsen.util 6688 :ok :write [9 30] INFO [2019-12-17 12:34:30,995] jepsen worker 28 - jepsen.util 6688 :invoke :write [9 8] INFO [2019-12-17 12:34:31,006] jepsen worker 79 - jepsen.util 8809 :ok :read [26 16] INFO [2019-12-17 12:34:31,006] jepsen worker 79 - jepsen.util 8809 :invoke :read [26 nil] INFO [2019-12-17 12:34:31,029] jepsen worker 3 - jepsen.util 8193 :info :write [1 28] :RequestCanceledException INFO [2019-12-17 12:34:31,029] jepsen worker 20 - jepsen.util 8120 :fail :read [6 nil] :RequestTimeoutException INFO [2019-12-17 12:34:31,029] jepsen worker 72 - jepsen.util 8352 :info :write [24 0] :RequestCanceledException INFO [2019-12-17 12:34:31,029] jepsen worker 20 - jepsen.util 8120 :invoke :write [6 4] INFO [2019-12-17 12:34:31,030] jepsen worker 3 - jepsen.util 8283 :invoke :cas [1 [36 34]] INFO [2019-12-17 12:34:31,031] jepsen worker 56 - jepsen.util 9326 :fail :cas [18 [30 41]] :ValueNotSwapFrom INFO [2019-12-17 12:34:31,031] jepsen worker 56 - jepsen.util 9326 :invoke :write [18 0] INFO [2019-12-17 12:34:31,033] jepsen worker 72 - jepsen.util 8442 :invoke :cas [24 [7 18]] INFO [2019-12-17 12:34:31,057] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":48616,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:34:31,057] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":48616,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:34:31,057] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":48619,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:34:31,057] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":48619,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:34:31,057] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":48620,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:34:31,057] jepsen worker 70 - jepsen.util 8710 :info :write [23 20] :RequestCanceledException INFO [2019-12-17 12:34:31,057] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":48620,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:34:31,057] jepsen worker 28 - jepsen.util 6688 :ok :write [9 8] INFO [2019-12-17 12:34:31,058] jepsen worker 28 - jepsen.util 6688 :invoke :write [9 12] INFO [2019-12-17 12:34:31,058] jepsen worker 70 - jepsen.util 8800 :invoke :write [23 46] INFO [2019-12-17 12:34:31,094] jepsen worker 78 - jepsen.util 8178 :fail :cas [26 [19 5]] :ValueNotSwapFrom INFO [2019-12-17 12:34:31,094] jepsen worker 78 - jepsen.util 8178 :invoke :write [26 15] INFO [2019-12-17 12:34:31,094] jepsen worker 72 - jepsen.util 8442 :fail :cas [24 [7 18]] :ValueNotSwapFrom INFO [2019-12-17 12:34:31,094] jepsen worker 72 - jepsen.util 8442 :invoke :write [24 41] INFO [2019-12-17 12:34:31,120] jepsen worker 28 - jepsen.util 6688 :ok :write [9 12] INFO [2019-12-17 12:34:31,120] jepsen worker 28 - jepsen.util 6688 :invoke :read [9 nil] INFO [2019-12-17 12:34:31,140] jepsen worker 37 - jepsen.util 8047 :ok :read [12 40] INFO [2019-12-17 12:34:31,141] jepsen worker 37 - jepsen.util 8047 :invoke :cas [12 [47 26]] INFO [2019-12-17 12:34:31,143] jepsen worker 89 - jepsen.util 8279 :info :write [29 2] :RequestCanceledException INFO [2019-12-17 12:34:31,144] jepsen worker 89 - jepsen.util 8369 :invoke :write [29 40] INFO [2019-12-17 12:34:31,160] jepsen worker 51 - jepsen.util 7791 :info :write [17 1] :RequestCanceledException INFO [2019-12-17 12:34:31,160] jepsen worker 33 - jepsen.util 11373 :info :write [11 38] :RequestCanceledException INFO [2019-12-17 12:34:31,162] jepsen worker 51 - jepsen.util 7881 :invoke :write [17 47] INFO [2019-12-17 12:34:31,162] jepsen worker 33 - jepsen.util 11463 :invoke :cas [11 [8 48]] INFO [2019-12-17 12:34:31,201] jepsen worker 79 - jepsen.util 8809 :ok :read [26 16] INFO [2019-12-17 12:34:31,201] jepsen worker 79 - jepsen.util 8809 :invoke :cas [26 [6 8]] INFO [2019-12-17 12:34:31,202] jepsen worker 71 - jepsen.util 8891 :ok :read [23 46] INFO [2019-12-17 12:34:31,202] jepsen worker 71 - jepsen.util 8891 :invoke :read [23 nil] INFO [2019-12-17 12:34:31,228] jepsen worker 57 - jepsen.util 11397 :info :cas [19 [2 23]] :RequestCanceledException INFO [2019-12-17 12:34:31,228] jepsen worker 57 - jepsen.util 11487 :invoke :read [19 nil] INFO [2019-12-17 12:34:31,243] jepsen worker 80 - jepsen.util 8180 :fail :cas [26 [26 4]] :ValueNotSwapFrom INFO [2019-12-17 12:34:31,243] jepsen worker 80 - jepsen.util 8180 :invoke :read [26 nil] INFO [2019-12-17 12:34:31,252] jepsen worker 88 - jepsen.util 8638 :ok :read [29 34] INFO [2019-12-17 12:34:31,252] jepsen worker 88 - jepsen.util 8638 :invoke :cas [29 [6 17]] INFO [2019-12-17 12:34:31,276] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":48646,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:34:31,276] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":48646,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:34:31,276] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":48649,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:34:31,276] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":48649,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:34:31,280] jepsen worker 69 - jepsen.util 8619 :fail :cas [23 [27 10]] :ValueNotSwapFrom INFO [2019-12-17 12:34:31,280] jepsen worker 69 - jepsen.util 8619 :invoke :cas [23 [25 4]] INFO [2019-12-17 12:34:31,286] jepsen worker 36 - jepsen.util 8676 :ok :read [12 40] INFO [2019-12-17 12:34:31,286] jepsen worker 36 - jepsen.util 8676 :invoke :read [12 nil] INFO [2019-12-17 12:34:31,293] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":48636,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:34:31,293] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":48636,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:34:31,293] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":48637,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:34:31,293] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":48637,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:34:31,296] jepsen worker 62 - jepsen.util 11672 :info :cas [20 [32 6]] :RequestCanceledException INFO [2019-12-17 12:34:31,296] jepsen worker 62 - jepsen.util 11762 :invoke :write [20 35] INFO [2019-12-17 12:34:31,304] jepsen worker 3 - jepsen.util 8283 :fail :cas [1 [36 34]] :ValueNotSwapFrom INFO [2019-12-17 12:34:31,305] jepsen worker 3 - jepsen.util 8283 :invoke :write [1 38] INFO [2019-12-17 12:34:31,315] jepsen worker 25 - jepsen.util 12085 :info :write [8 29] :RequestCanceledException INFO [2019-12-17 12:34:31,327] jepsen worker 25 - jepsen.util 12175 :invoke :read [8 nil] INFO [2019-12-17 12:34:31,329] jepsen worker 31 - jepsen.util 7411 :info :write [10 37] :RequestCanceledException INFO [2019-12-17 12:34:31,340] jepsen worker 31 - jepsen.util 7501 :invoke :cas [10 [30 12]] INFO [2019-12-17 12:34:31,341] jepsen worker 0 - jepsen.util 11880 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:34:31,341] jepsen worker 0 - jepsen.util 11880 :invoke :write [0 36] INFO [2019-12-17 12:34:31,361] jepsen worker 58 - jepsen.util 12388 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:34:31,361] jepsen worker 58 - jepsen.util 12388 :invoke :write [19 11] INFO [2019-12-17 12:34:31,385] jepsen worker 31 - jepsen.util 7501 :fail :cas [10 [30 12]] :ValueNotSwapFrom INFO [2019-12-17 12:34:31,385] jepsen worker 31 - jepsen.util 7501 :invoke :cas [10 [37 10]] INFO [2019-12-17 12:34:31,413] jepsen worker 79 - jepsen.util 8809 :fail :cas [26 [6 8]] :ValueNotSwapFrom INFO [2019-12-17 12:34:31,413] jepsen worker 79 - jepsen.util 8809 :invoke :write [26 12] INFO [2019-12-17 12:34:31,443] jepsen worker 85 - jepsen.util 12145 :info :cas [28 [11 16]] :RequestCanceledException INFO [2019-12-17 12:34:31,445] jepsen worker 19 - jepsen.util 8659 :ok :write [6 22] INFO [2019-12-17 12:34:31,445] jepsen worker 19 - jepsen.util 8659 :invoke :cas [6 [44 19]] INFO [2019-12-17 12:34:31,446] jepsen worker 72 - jepsen.util 8442 :ok :write [24 41] INFO [2019-12-17 12:34:31,446] jepsen worker 72 - jepsen.util 8442 :invoke :write [24 18] INFO [2019-12-17 12:34:31,446] jepsen worker 11 - jepsen.util 8381 :ok :cas [3 [13 43]] INFO [2019-12-17 12:34:31,447] jepsen worker 11 - jepsen.util 8381 :invoke :read [3 nil] INFO [2019-12-17 12:34:31,447] jepsen worker 31 - jepsen.util 7501 :fail :cas [10 [37 10]] :ValueNotSwapFrom INFO [2019-12-17 12:34:31,447] jepsen worker 85 - jepsen.util 12235 :invoke :cas [28 [46 20]] INFO [2019-12-17 12:34:31,447] jepsen worker 31 - jepsen.util 7501 :invoke :read [10 nil] INFO [2019-12-17 12:34:31,449] jepsen worker 46 - jepsen.util 8416 :ok :write [15 29] INFO [2019-12-17 12:34:31,449] jepsen worker 46 - jepsen.util 8416 :invoke :write [15 35] INFO [2019-12-17 12:34:31,461] jepsen worker 5 - jepsen.util 7835 :info :write [1 12] :RequestCanceledException INFO [2019-12-17 12:34:31,465] jepsen worker 5 - jepsen.util 7925 :invoke :read [1 nil] INFO [2019-12-17 12:34:31,494] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":48666,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:34:31,494] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":48666,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:34:31,496] jepsen worker 48 - jepsen.util 10668 :info :cas [16 [41 20]] :RequestCanceledException INFO [2019-12-17 12:34:31,505] jepsen worker 48 - jepsen.util 10758 :invoke :write [16 9] INFO [2019-12-17 12:34:31,513] jepsen worker 26 - jepsen.util 11906 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:34:31,513] jepsen worker 22 - jepsen.util 11452 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:34:31,513] jepsen worker 22 - jepsen.util 11452 :invoke :cas [7 [5 12]] INFO [2019-12-17 12:34:31,514] jepsen worker 26 - jepsen.util 11906 :invoke :cas [8 [18 45]] INFO [2019-12-17 12:34:31,515] jepsen worker 19 - jepsen.util 8659 :fail :cas [6 [44 19]] :ValueNotSwapFrom INFO [2019-12-17 12:34:31,515] jepsen worker 19 - jepsen.util 8659 :invoke :read [6 nil] INFO [2019-12-17 12:34:31,529] jepsen worker 45 - jepsen.util 8685 :info :write [15 13] :RequestCanceledException INFO [2019-12-17 12:34:31,529] jepsen worker 65 - jepsen.util 6995 :fail :read [21 nil] :RequestTimeoutException INFO [2019-12-17 12:34:31,530] jepsen worker 65 - jepsen.util 6995 :invoke :cas [21 [35 40]] INFO [2019-12-17 12:34:31,532] jepsen worker 11 - jepsen.util 8381 :ok :read [3 43] INFO [2019-12-17 12:34:31,533] jepsen worker 11 - jepsen.util 8381 :invoke :write [3 37] INFO [2019-12-17 12:34:31,533] jepsen worker 45 - jepsen.util 8775 :invoke :cas [15 [15 36]] INFO [2019-12-17 12:34:31,543] jepsen worker 71 - jepsen.util 8891 :ok :read [23 46] INFO [2019-12-17 12:34:31,543] jepsen worker 71 - jepsen.util 8891 :invoke :write [23 40] INFO [2019-12-17 12:34:31,550] jepsen worker 5 - jepsen.util 7925 :ok :read [1 10] INFO [2019-12-17 12:34:31,550] jepsen worker 5 - jepsen.util 7925 :invoke :cas [1 [26 6]] INFO [2019-12-17 12:34:31,557] jepsen worker 6 - jepsen.util 12066 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:34:31,557] jepsen worker 6 - jepsen.util 12066 :invoke :read [2 nil] INFO [2019-12-17 12:34:31,591] jepsen worker 31 - jepsen.util 7501 :ok :read [10 43] INFO [2019-12-17 12:34:31,591] jepsen worker 31 - jepsen.util 7501 :invoke :cas [10 [46 24]] INFO [2019-12-17 12:34:31,598] jepsen worker 9 - jepsen.util 8829 :fail :cas [3 [38 21]] :ValueNotSwapFrom INFO [2019-12-17 12:34:31,598] jepsen worker 9 - jepsen.util 8829 :invoke :write [3 36] INFO [2019-12-17 12:34:31,598] jepsen worker 19 - jepsen.util 8659 :ok :read [6 22] INFO [2019-12-17 12:34:31,599] jepsen worker 19 - jepsen.util 8659 :invoke :cas [6 [47 41]] INFO [2019-12-17 12:34:31,608] jepsen worker 61 - jepsen.util 12211 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:34:31,608] jepsen worker 66 - jepsen.util 11316 :info :cas [22 [14 33]] :RequestCanceledException INFO [2019-12-17 12:34:31,608] jepsen worker 61 - jepsen.util 12211 :invoke :cas [20 [27 49]] INFO [2019-12-17 12:34:31,609] jepsen worker 66 - jepsen.util 11406 :invoke :read [22 nil] INFO [2019-12-17 12:34:31,626] jepsen worker 47 - jepsen.util 8957 :fail :cas [15 [32 48]] :ValueNotSwapFrom INFO [2019-12-17 12:34:31,627] jepsen worker 47 - jepsen.util 8957 :invoke :cas [15 [1 13]] INFO [2019-12-17 12:34:31,627] jepsen worker 44 - jepsen.util 11654 :info :cas [14 [28 13]] :RequestCanceledException INFO [2019-12-17 12:34:31,627] jepsen worker 84 - jepsen.util 10974 :info :cas [28 [33 36]] :RequestCanceledException INFO [2019-12-17 12:34:31,632] jepsen worker 84 - jepsen.util 11064 :invoke :read [28 nil] INFO [2019-12-17 12:34:31,634] jepsen worker 44 - jepsen.util 11744 :invoke :write [14 32] INFO [2019-12-17 12:34:31,646] jepsen worker 65 - jepsen.util 6995 :fail :cas [21 [35 40]] :ValueNotSwapFrom INFO [2019-12-17 12:34:31,646] jepsen worker 65 - jepsen.util 6995 :invoke :write [21 24] INFO [2019-12-17 12:34:31,650] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:34:31,650] jepsen nemesis - couchbase.util Rebalance check count: 377 INFO [2019-12-17 12:34:31,650] jepsen nemesis - couchbase.util Rebalance stuck count: 358 INFO [2019-12-17 12:34:31,656] jepsen worker 80 - jepsen.util 8180 :ok :read [26 16] INFO [2019-12-17 12:34:31,656] jepsen worker 80 - jepsen.util 8180 :invoke :write [26 17] INFO [2019-12-17 12:34:31,671] jepsen worker 19 - jepsen.util 8659 :fail :cas [6 [47 41]] :ValueNotSwapFrom INFO [2019-12-17 12:34:31,671] jepsen worker 19 - jepsen.util 8659 :invoke :read [6 nil] INFO [2019-12-17 12:34:31,678] jepsen worker 31 - jepsen.util 7501 :fail :cas [10 [46 24]] :ValueNotSwapFrom INFO [2019-12-17 12:34:31,678] jepsen worker 31 - jepsen.util 7501 :invoke :write [10 46] INFO [2019-12-17 12:34:31,682] jepsen worker 5 - jepsen.util 7925 :fail :cas [1 [26 6]] :ValueNotSwapFrom INFO [2019-12-17 12:34:31,683] jepsen worker 5 - jepsen.util 7925 :invoke :write [1 27] INFO [2019-12-17 12:34:31,697] jepsen worker 14 - jepsen.util 8654 :ok :write [4 22] INFO [2019-12-17 12:34:31,698] jepsen worker 14 - jepsen.util 8654 :invoke :read [4 nil] INFO [2019-12-17 12:34:31,715] jepsen worker 54 - jepsen.util 9414 :ok :write [18 30] INFO [2019-12-17 12:34:31,715] jepsen worker 54 - jepsen.util 9414 :invoke :cas [18 [5 45]] INFO [2019-12-17 12:34:31,716] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":48694,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:34:31,716] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":48694,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:34:31,716] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":48698,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:34:31,716] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":48698,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:34:31,730] jepsen worker 19 - jepsen.util 8659 :ok :read [6 22] INFO [2019-12-17 12:34:31,730] jepsen worker 19 - jepsen.util 8659 :invoke :write [6 35] INFO [2019-12-17 12:34:31,775] jepsen worker 36 - jepsen.util 8676 :ok :read [12 40] INFO [2019-12-17 12:34:31,775] jepsen worker 36 - jepsen.util 8676 :invoke :write [12 42] INFO [2019-12-17 12:34:31,796] jepsen worker 40 - jepsen.util 11650 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:34:31,796] jepsen worker 40 - jepsen.util 11650 :invoke :read [13 nil] INFO [2019-12-17 12:34:31,803] jepsen worker 37 - jepsen.util 8047 :fail :cas [12 [47 26]] :ValueNotSwapFrom INFO [2019-12-17 12:34:31,803] jepsen worker 37 - jepsen.util 8047 :invoke :cas [12 [37 14]] INFO [2019-12-17 12:34:31,805] jepsen worker 46 - jepsen.util 8416 :ok :write [15 35] INFO [2019-12-17 12:34:31,805] jepsen worker 46 - jepsen.util 8416 :invoke :read [15 nil] INFO [2019-12-17 12:34:31,805] jepsen worker 72 - jepsen.util 8442 :ok :write [24 18] INFO [2019-12-17 12:34:31,805] jepsen worker 72 - jepsen.util 8442 :invoke :write [24 38] INFO [2019-12-17 12:34:31,809] jepsen worker 24 - jepsen.util 10554 :info :cas [8 [35 24]] :RequestCanceledException INFO [2019-12-17 12:34:31,810] jepsen worker 24 - jepsen.util 10644 :invoke :read [8 nil] INFO [2019-12-17 12:34:31,827] jepsen worker 55 - jepsen.util 9055 :info :write [18 5] :RequestCanceledException INFO [2019-12-17 12:34:31,828] jepsen worker 55 - jepsen.util 9145 :invoke :write [18 21] INFO [2019-12-17 12:34:31,840] jepsen worker 47 - jepsen.util 8957 :fail :cas [15 [1 13]] :ValueNotSwapFrom INFO [2019-12-17 12:34:31,840] jepsen worker 47 - jepsen.util 8957 :invoke :write [15 3] INFO [2019-12-17 12:34:31,860] jepsen worker 38 - jepsen.util 8678 :info :write [12 7] :RequestCanceledException INFO [2019-12-17 12:34:31,860] jepsen worker 1 - jepsen.util 11701 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:34:31,860] jepsen worker 1 - jepsen.util 11701 :invoke :cas [0 [44 39]] INFO [2019-12-17 12:34:31,861] jepsen worker 38 - jepsen.util 8768 :invoke :read [12 nil] INFO [2019-12-17 12:34:31,895] jepsen worker 42 - jepsen.util 12552 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:34:31,895] jepsen worker 42 - jepsen.util 12552 :invoke :cas [14 [42 3]] INFO [2019-12-17 12:34:31,914] jepsen worker 29 - jepsen.util 6869 :info :write [9 12] :RequestCanceledException INFO [2019-12-17 12:34:31,914] jepsen worker 17 - jepsen.util 11447 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:34:31,914] jepsen worker 17 - jepsen.util 11447 :invoke :cas [5 [12 46]] INFO [2019-12-17 12:34:31,915] jepsen worker 29 - jepsen.util 6959 :invoke :write [9 5] INFO [2019-12-17 12:34:31,929] jepsen worker 15 - jepsen.util 10635 :info :write [5 3] :RequestCanceledException INFO [2019-12-17 12:34:31,930] jepsen worker 15 - jepsen.util 10725 :invoke :cas [5 [42 21]] INFO [2019-12-17 12:34:31,943] jepsen worker 43 - jepsen.util 11203 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:34:31,943] jepsen worker 12 - jepsen.util 8922 :info :cas [4 [27 3]] :RequestCanceledException INFO [2019-12-17 12:34:31,943] jepsen worker 43 - jepsen.util 11203 :invoke :read [14 nil] INFO [2019-12-17 12:34:31,944] jepsen worker 12 - jepsen.util 9012 :invoke :read [4 nil] INFO [2019-12-17 12:34:31,951] jepsen worker 88 - jepsen.util 8638 :fail :cas [29 [6 17]] :ValueNotSwapFrom INFO [2019-12-17 12:34:31,952] jepsen worker 88 - jepsen.util 8638 :invoke :write [29 37] INFO [2019-12-17 12:34:31,961] jepsen worker 75 - jepsen.util 10965 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:34:31,961] jepsen worker 75 - jepsen.util 10965 :invoke :cas [25 [45 39]] INFO [2019-12-17 12:34:31,964] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":48719,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:34:31,964] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":48719,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:34:31,964] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":48722,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:34:31,964] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":48722,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:34:31,964] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":48723,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:34:31,964] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":48723,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:34:31,989] jepsen worker 54 - jepsen.util 9414 :fail :cas [18 [5 45]] :ValueNotSwapFrom INFO [2019-12-17 12:34:31,989] jepsen worker 54 - jepsen.util 9414 :invoke :read [18 nil] INFO [2019-12-17 12:34:31,991] jepsen worker 69 - jepsen.util 8619 :fail :cas [23 [25 4]] :ValueNotSwapFrom INFO [2019-12-17 12:34:31,991] jepsen worker 69 - jepsen.util 8619 :invoke :read [23 nil] INFO [2019-12-17 12:34:31,996] jepsen worker 32 - jepsen.util 8582 :info :write [10 34] :RequestCanceledException INFO [2019-12-17 12:34:31,996] jepsen worker 8 - jepsen.util 11348 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:34:31,996] jepsen worker 8 - jepsen.util 11348 :invoke :cas [2 [20 23]] INFO [2019-12-17 12:34:31,996] jepsen worker 32 - jepsen.util 8672 :invoke :cas [10 [43 1]] INFO [2019-12-17 12:34:32,007] jepsen worker 11 - jepsen.util 8381 :ok :write [3 37] INFO [2019-12-17 12:34:32,007] jepsen worker 11 - jepsen.util 8381 :invoke :write [3 6] INFO [2019-12-17 12:34:32,008] jepsen worker 31 - jepsen.util 7501 :ok :write [10 46] INFO [2019-12-17 12:34:32,008] jepsen worker 31 - jepsen.util 7501 :invoke :read [10 nil] INFO [2019-12-17 12:34:32,013] jepsen worker 77 - jepsen.util 11597 :info :write [25 3] :RequestCanceledException INFO [2019-12-17 12:34:32,015] jepsen worker 77 - jepsen.util 11687 :invoke :write [25 9] INFO [2019-12-17 12:34:32,033] jepsen worker 31 - jepsen.util 7501 :ok :read [10 46] INFO [2019-12-17 12:34:32,033] jepsen worker 31 - jepsen.util 7501 :invoke :write [10 40] INFO [2019-12-17 12:34:32,043] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":48739,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:34:32,043] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":48739,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:34:32,045] jepsen worker 65 - jepsen.util 6995 :ok :write [21 24] INFO [2019-12-17 12:34:32,046] jepsen worker 65 - jepsen.util 6995 :invoke :read [21 nil] INFO [2019-12-17 12:34:32,058] jepsen worker 37 - jepsen.util 8047 :fail :cas [12 [37 14]] :ValueNotSwapFrom INFO [2019-12-17 12:34:32,058] jepsen worker 37 - jepsen.util 8047 :invoke :read [12 nil] INFO [2019-12-17 12:34:32,074] jepsen worker 38 - jepsen.util 8768 :ok :read [12 40] INFO [2019-12-17 12:34:32,074] jepsen worker 38 - jepsen.util 8768 :invoke :cas [12 [16 21]] INFO [2019-12-17 12:34:32,077] jepsen worker 19 - jepsen.util 8659 :ok :write [6 35] INFO [2019-12-17 12:34:32,077] jepsen worker 19 - jepsen.util 8659 :invoke :read [6 nil] INFO [2019-12-17 12:34:32,080] jepsen worker 72 - jepsen.util 8442 :ok :write [24 38] INFO [2019-12-17 12:34:32,080] jepsen worker 72 - jepsen.util 8442 :invoke :cas [24 [49 21]] INFO [2019-12-17 12:34:32,080] jepsen worker 47 - jepsen.util 8957 :ok :write [15 3] INFO [2019-12-17 12:34:32,080] jepsen worker 47 - jepsen.util 8957 :invoke :read [15 nil] INFO [2019-12-17 12:34:32,087] jepsen worker 54 - jepsen.util 9414 :ok :read [18 30] INFO [2019-12-17 12:34:32,087] jepsen worker 54 - jepsen.util 9414 :invoke :write [18 29] INFO [2019-12-17 12:34:32,096] jepsen worker 63 - jepsen.util 7173 :info :write [21 20] :RequestCanceledException INFO [2019-12-17 12:34:32,097] jepsen worker 63 - jepsen.util 7263 :invoke :write [21 34] INFO [2019-12-17 12:34:32,098] jepsen worker 37 - jepsen.util 8047 :ok :read [12 40] INFO [2019-12-17 12:34:32,099] jepsen worker 37 - jepsen.util 8047 :invoke :write [12 40] INFO [2019-12-17 12:34:32,107] jepsen worker 32 - jepsen.util 8672 :fail :cas [10 [43 1]] :ValueNotSwapFrom INFO [2019-12-17 12:34:32,108] jepsen worker 32 - jepsen.util 8672 :invoke :cas [10 [35 12]] INFO [2019-12-17 12:34:32,113] jepsen worker 7 - jepsen.util 11707 :info :write [2 4] :RequestCanceledException INFO [2019-12-17 12:34:32,125] jepsen worker 38 - jepsen.util 8768 :fail :cas [12 [16 21]] :ValueNotSwapFrom INFO [2019-12-17 12:34:32,129] jepsen worker 49 - jepsen.util 10939 :info :cas [16 [46 3]] :RequestCanceledException INFO [2019-12-17 12:34:32,144] jepsen worker 73 - jepsen.util 8083 :fail :read [24 nil] :RequestTimeoutException INFO [2019-12-17 12:34:32,144] jepsen worker 10 - jepsen.util 8650 :info :write [3 13] :RequestCanceledException INFO [2019-12-17 12:34:32,145] jepsen worker 47 - jepsen.util 8957 :ok :read [15 3] INFO [2019-12-17 12:34:32,151] jepsen worker 72 - jepsen.util 8442 :fail :cas [24 [49 21]] :ValueNotSwapFrom INFO [2019-12-17 12:34:32,160] jepsen worker 64 - jepsen.util 6904 :info :write [21 20] :RequestCanceledException INFO [2019-12-17 12:34:32,161] jepsen worker 38 - jepsen.util 8768 :invoke :cas [12 [41 34]] INFO [2019-12-17 12:34:32,161] jepsen worker 73 - jepsen.util 8083 :invoke :read [24 nil] INFO [2019-12-17 12:34:32,161] jepsen worker 47 - jepsen.util 8957 :invoke :read [15 nil] INFO [2019-12-17 12:34:32,161] jepsen worker 72 - jepsen.util 8442 :invoke :write [24 20] INFO [2019-12-17 12:34:32,161] jepsen worker 49 - jepsen.util 11029 :invoke :read [16 nil] INFO [2019-12-17 12:34:32,162] jepsen worker 7 - jepsen.util 11797 :invoke :read [2 nil] INFO [2019-12-17 12:34:32,162] jepsen worker 64 - jepsen.util 6994 :invoke :cas [21 [28 7]] INFO [2019-12-17 12:34:32,162] jepsen worker 10 - jepsen.util 8740 :invoke :write [3 37] INFO [2019-12-17 12:34:32,181] jepsen worker 32 - jepsen.util 8672 :fail :cas [10 [35 12]] :ValueNotSwapFrom INFO [2019-12-17 12:34:32,181] jepsen worker 32 - jepsen.util 8672 :invoke :cas [10 [14 11]] INFO [2019-12-17 12:34:32,210] jepsen worker 73 - jepsen.util 8083 :ok :read [24 38] INFO [2019-12-17 12:34:32,211] jepsen worker 73 - jepsen.util 8083 :invoke :cas [24 [9 39]] INFO [2019-12-17 12:34:32,222] jepsen worker 47 - jepsen.util 8957 :ok :read [15 3] INFO [2019-12-17 12:34:32,222] jepsen worker 47 - jepsen.util 8957 :invoke :read [15 nil] INFO [2019-12-17 12:34:32,226] jepsen worker 64 - jepsen.util 6994 :fail :cas [21 [28 7]] :ValueNotSwapFrom INFO [2019-12-17 12:34:32,226] jepsen worker 64 - jepsen.util 6994 :invoke :cas [21 [1 22]] INFO [2019-12-17 12:34:32,233] jepsen worker 29 - jepsen.util 6959 :ok :write [9 5] INFO [2019-12-17 12:34:32,233] jepsen worker 29 - jepsen.util 6959 :invoke :write [9 1] INFO [2019-12-17 12:34:32,234] jepsen worker 73 - jepsen.util 8083 :fail :cas [24 [9 39]] :ValueNotSwapFrom INFO [2019-12-17 12:34:32,234] jepsen worker 73 - jepsen.util 8083 :invoke :cas [24 [49 37]] INFO [2019-12-17 12:34:32,241] jepsen worker 11 - jepsen.util 8381 :ok :write [3 6] INFO [2019-12-17 12:34:32,242] jepsen worker 11 - jepsen.util 8381 :invoke :write [3 11] INFO [2019-12-17 12:34:32,243] jepsen worker 2 - jepsen.util 11072 :info :write [0 49] :RequestCanceledException INFO [2019-12-17 12:34:32,244] jepsen worker 2 - jepsen.util 11162 :invoke :write [0 37] INFO [2019-12-17 12:34:32,251] jepsen worker 64 - jepsen.util 6994 :fail :cas [21 [1 22]] :ValueNotSwapFrom INFO [2019-12-17 12:34:32,251] jepsen worker 64 - jepsen.util 6994 :invoke :write [21 26] INFO [2019-12-17 12:34:32,261] jepsen worker 63 - jepsen.util 7263 :ok :write [21 34] INFO [2019-12-17 12:34:32,262] jepsen worker 63 - jepsen.util 7263 :invoke :write [21 3] INFO [2019-12-17 12:34:32,263] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":48773,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:34:32,264] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":48773,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:34:32,264] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":48776,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:34:32,264] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":48776,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:34:32,264] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":48777,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:34:32,264] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":48777,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:34:32,278] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":48765,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:34:32,278] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":48765,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:34:32,278] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":48767,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:34:32,278] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":48767,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:34:32,311] jepsen worker 31 - jepsen.util 7501 :ok :write [10 40] INFO [2019-12-17 12:34:32,311] jepsen worker 31 - jepsen.util 7501 :invoke :read [10 nil] INFO [2019-12-17 12:34:32,314] jepsen worker 53 - jepsen.util 7703 :info :write [17 22] :RequestCanceledException INFO [2019-12-17 12:34:32,315] jepsen worker 53 - jepsen.util 7793 :invoke :read [17 nil] INFO [2019-12-17 12:34:32,320] jepsen worker 73 - jepsen.util 8083 :fail :cas [24 [49 37]] :ValueNotSwapFrom INFO [2019-12-17 12:34:32,321] jepsen worker 73 - jepsen.util 8083 :invoke :write [24 27] INFO [2019-12-17 12:34:32,356] jepsen worker 31 - jepsen.util 7501 :ok :read [10 40] INFO [2019-12-17 12:34:32,356] jepsen worker 31 - jepsen.util 7501 :invoke :write [10 11] INFO [2019-12-17 12:34:32,356] jepsen worker 53 - jepsen.util 7793 :ok :read [17 10] INFO [2019-12-17 12:34:32,356] jepsen worker 53 - jepsen.util 7793 :invoke :read [17 nil] INFO [2019-12-17 12:34:32,392] jepsen worker 29 - jepsen.util 6959 :ok :write [9 1] INFO [2019-12-17 12:34:32,393] jepsen worker 29 - jepsen.util 6959 :invoke :write [9 42] INFO [2019-12-17 12:34:32,399] jepsen worker 14 - jepsen.util 8654 :ok :read [4 22] INFO [2019-12-17 12:34:32,399] jepsen worker 14 - jepsen.util 8654 :invoke :cas [4 [42 46]] INFO [2019-12-17 12:34:32,427] jepsen worker 14 - jepsen.util 8654 :fail :cas [4 [42 46]] :ValueNotSwapFrom INFO [2019-12-17 12:34:32,427] jepsen worker 14 - jepsen.util 8654 :invoke :cas [4 [15 17]] INFO [2019-12-17 12:34:32,429] jepsen worker 81 - jepsen.util 11331 :info :cas [27 [3 34]] :RequestCanceledException INFO [2019-12-17 12:34:32,429] jepsen worker 86 - jepsen.util 12326 :info :write [28 1] :RequestCanceledException INFO [2019-12-17 12:34:32,430] jepsen worker 81 - jepsen.util 11421 :invoke :write [27 17] INFO [2019-12-17 12:34:32,430] jepsen worker 86 - jepsen.util 12416 :invoke :read [28 nil] INFO [2019-12-17 12:34:32,432] jepsen worker 64 - jepsen.util 6994 :ok :write [21 26] INFO [2019-12-17 12:34:32,432] jepsen worker 64 - jepsen.util 6994 :invoke :write [21 49] INFO [2019-12-17 12:34:32,461] jepsen worker 41 - jepsen.util 12101 :info :write [13 42] :RequestCanceledException INFO [2019-12-17 12:34:32,461] jepsen worker 74 - jepsen.util 8624 :fail :read [24 nil] :RequestTimeoutException INFO [2019-12-17 12:34:32,461] jepsen worker 74 - jepsen.util 8624 :invoke :cas [24 [25 19]] INFO [2019-12-17 12:34:32,462] jepsen worker 41 - jepsen.util 12191 :invoke :write [13 6] INFO [2019-12-17 12:34:32,473] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":48798,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:34:32,473] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":48798,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:34:32,473] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":48800,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:34:32,473] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":48800,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:34:32,479] jepsen worker 14 - jepsen.util 8654 :fail :cas [4 [15 17]] :ValueNotSwapFrom INFO [2019-12-17 12:34:32,479] jepsen worker 14 - jepsen.util 8654 :invoke :cas [4 [23 3]] INFO [2019-12-17 12:34:32,496] jepsen worker 74 - jepsen.util 8624 :fail :cas [24 [25 19]] :ValueNotSwapFrom INFO [2019-12-17 12:34:32,496] jepsen worker 74 - jepsen.util 8624 :invoke :cas [24 [12 25]] INFO [2019-12-17 12:34:32,496] jepsen worker 29 - jepsen.util 6959 :ok :write [9 42] INFO [2019-12-17 12:34:32,496] jepsen worker 29 - jepsen.util 6959 :invoke :cas [9 [45 19]] INFO [2019-12-17 12:34:32,496] jepsen worker 21 - jepsen.util 11001 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:34:32,496] jepsen worker 59 - jepsen.util 11039 :info :cas [19 [0 48]] :RequestCanceledException INFO [2019-12-17 12:34:32,496] jepsen worker 21 - jepsen.util 11001 :invoke :write [7 15] INFO [2019-12-17 12:34:32,497] jepsen worker 59 - jepsen.util 11129 :invoke :cas [19 [13 12]] INFO [2019-12-17 12:34:32,529] jepsen worker 39 - jepsen.util 10929 :info :cas [13 [21 47]] :RequestCanceledException INFO [2019-12-17 12:34:32,530] jepsen worker 39 - jepsen.util 11019 :invoke :cas [13 [27 43]] INFO [2019-12-17 12:34:32,538] jepsen worker 74 - jepsen.util 8624 :fail :cas [24 [12 25]] :ValueNotSwapFrom INFO [2019-12-17 12:34:32,538] jepsen worker 74 - jepsen.util 8624 :invoke :write [24 26] INFO [2019-12-17 12:34:32,541] jepsen worker 53 - jepsen.util 7793 :ok :read [17 10] INFO [2019-12-17 12:34:32,541] jepsen worker 53 - jepsen.util 7793 :invoke :read [17 nil] INFO [2019-12-17 12:34:32,552] jepsen worker 29 - jepsen.util 6959 :fail :cas [9 [45 19]] :ValueNotSwapFrom INFO [2019-12-17 12:34:32,553] jepsen worker 29 - jepsen.util 6959 :invoke :write [9 45] INFO [2019-12-17 12:34:32,560] jepsen worker 30 - jepsen.util 8400 :fail :read [10 nil] :RequestTimeoutException INFO [2019-12-17 12:34:32,560] jepsen worker 30 - jepsen.util 8400 :invoke :cas [10 [4 35]] INFO [2019-12-17 12:34:32,593] jepsen worker 14 - jepsen.util 8654 :fail :cas [4 [23 3]] :ValueNotSwapFrom INFO [2019-12-17 12:34:32,593] jepsen worker 14 - jepsen.util 8654 :invoke :cas [4 [12 7]] INFO [2019-12-17 12:34:32,613] jepsen worker 13 - jepsen.util 8023 :info :write [4 5] :RequestCanceledException INFO [2019-12-17 12:34:32,614] jepsen worker 13 - jepsen.util 8113 :invoke :cas [4 [48 6]] INFO [2019-12-17 12:34:32,623] jepsen worker 64 - jepsen.util 6994 :ok :write [21 49] INFO [2019-12-17 12:34:32,624] jepsen worker 64 - jepsen.util 6994 :invoke :read [21 nil] INFO [2019-12-17 12:34:32,629] jepsen worker 82 - jepsen.util 12052 :info :write [27 34] :RequestCanceledException INFO [2019-12-17 12:34:32,629] jepsen worker 87 - jepsen.util 7827 :info :write [29 25] :RequestCanceledException INFO [2019-12-17 12:34:32,630] jepsen worker 68 - jepsen.util 10778 :info :write [22 21] :RequestCanceledException INFO [2019-12-17 12:34:32,630] jepsen worker 50 - jepsen.util 12200 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:34:32,630] jepsen worker 50 - jepsen.util 12200 :invoke :write [16 22] INFO [2019-12-17 12:34:32,630] jepsen worker 87 - jepsen.util 7917 :invoke :write [29 2] INFO [2019-12-17 12:34:32,630] jepsen worker 82 - jepsen.util 12142 :invoke :write [27 0] INFO [2019-12-17 12:34:32,631] jepsen worker 68 - jepsen.util 10868 :invoke :cas [22 [13 43]] INFO [2019-12-17 12:34:32,643] jepsen worker 52 - jepsen.util 8512 :info :write [17 31] :RequestCanceledException INFO [2019-12-17 12:34:32,644] jepsen worker 52 - jepsen.util 8602 :invoke :cas [17 [21 12]] INFO [2019-12-17 12:34:32,650] jepsen worker 64 - jepsen.util 6994 :ok :read [21 49] INFO [2019-12-17 12:34:32,651] jepsen worker 64 - jepsen.util 6994 :invoke :cas [21 [16 13]] INFO [2019-12-17 12:34:32,661] jepsen worker 76 - jepsen.util 11506 :info :cas [25 [1 7]] :RequestCanceledException INFO [2019-12-17 12:34:32,662] jepsen worker 76 - jepsen.util 11596 :invoke :cas [25 [30 11]] INFO [2019-12-17 12:34:32,664] jepsen worker 12 - jepsen.util 9012 :ok :read [4 22] INFO [2019-12-17 12:34:32,664] jepsen worker 12 - jepsen.util 9012 :invoke :cas [4 [16 29]] INFO [2019-12-17 12:34:32,675] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:34:32,675] jepsen nemesis - couchbase.util Rebalance check count: 378 INFO [2019-12-17 12:34:32,675] jepsen nemesis - couchbase.util Rebalance stuck count: 359 INFO [2019-12-17 12:34:32,686] jepsen worker 64 - jepsen.util 6994 :fail :cas [21 [16 13]] :ValueNotSwapFrom INFO [2019-12-17 12:34:32,686] jepsen worker 64 - jepsen.util 6994 :invoke :cas [21 [16 37]] INFO [2019-12-17 12:34:32,691] jepsen worker 72 - jepsen.util 8442 :ok :write [24 20] INFO [2019-12-17 12:34:32,691] jepsen worker 72 - jepsen.util 8442 :invoke :read [24 nil] INFO [2019-12-17 12:34:32,696] jepsen worker 60 - jepsen.util 12480 :info :write [20 34] :RequestCanceledException INFO [2019-12-17 12:34:32,697] jepsen worker 60 - jepsen.util 12570 :invoke :write [20 45] INFO [2019-12-17 12:34:32,708] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":48824,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:34:32,708] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":48824,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:34:32,708] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":48827,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:34:32,708] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":48827,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:34:32,708] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":48828,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:34:32,708] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":48828,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:34:32,714] jepsen worker 23 - jepsen.util 11093 :info :write [7 36] :RequestCanceledException INFO [2019-12-17 12:34:32,714] jepsen worker 83 - jepsen.util 11873 :info :cas [27 [35 37]] :RequestCanceledException INFO [2019-12-17 12:34:32,715] jepsen worker 83 - jepsen.util 11963 :invoke :read [27 nil] INFO [2019-12-17 12:34:32,715] jepsen worker 23 - jepsen.util 11183 :invoke :cas [7 [39 15]] INFO [2019-12-17 12:34:32,727] jepsen worker 16 - jepsen.util 12526 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:34:32,728] jepsen worker 16 - jepsen.util 12526 :invoke :read [5 nil] INFO [2019-12-17 12:34:32,731] jepsen worker 69 - jepsen.util 8619 :ok :read [23 46] INFO [2019-12-17 12:34:32,731] jepsen worker 69 - jepsen.util 8619 :invoke :read [23 nil] INFO [2019-12-17 12:34:32,743] jepsen worker 34 - jepsen.util 11374 :info :write [11 15] :RequestCanceledException INFO [2019-12-17 12:34:32,744] jepsen worker 34 - jepsen.util 11464 :invoke :write [11 37] INFO [2019-12-17 12:34:32,764] jepsen worker 72 - jepsen.util 8442 :ok :read [24 20] INFO [2019-12-17 12:34:32,764] jepsen worker 72 - jepsen.util 8442 :invoke :write [24 22] INFO [2019-12-17 12:34:32,766] jepsen worker 12 - jepsen.util 9012 :fail :cas [4 [16 29]] :ValueNotSwapFrom INFO [2019-12-17 12:34:32,766] jepsen worker 12 - jepsen.util 9012 :invoke :write [4 19] INFO [2019-12-17 12:34:32,785] jepsen worker 69 - jepsen.util 8619 :ok :read [23 46] INFO [2019-12-17 12:34:32,785] jepsen worker 69 - jepsen.util 8619 :invoke :read [23 nil] INFO [2019-12-17 12:34:32,798] jepsen worker 11 - jepsen.util 8381 :ok :write [3 11] INFO [2019-12-17 12:34:32,798] jepsen worker 11 - jepsen.util 8381 :invoke :write [3 9] INFO [2019-12-17 12:34:32,800] jepsen worker 64 - jepsen.util 6994 :fail :cas [21 [16 37]] :ValueNotSwapFrom INFO [2019-12-17 12:34:32,800] jepsen worker 64 - jepsen.util 6994 :invoke :read [21 nil] INFO [2019-12-17 12:34:32,816] jepsen worker 31 - jepsen.util 7501 :ok :write [10 11] INFO [2019-12-17 12:34:32,816] jepsen worker 31 - jepsen.util 7501 :invoke :cas [10 [27 6]] INFO [2019-12-17 12:34:32,837] jepsen worker 64 - jepsen.util 6994 :ok :read [21 49] INFO [2019-12-17 12:34:32,837] jepsen worker 64 - jepsen.util 6994 :invoke :write [21 41] INFO [2019-12-17 12:34:32,837] jepsen worker 29 - jepsen.util 6959 :ok :write [9 45] INFO [2019-12-17 12:34:32,837] jepsen worker 29 - jepsen.util 6959 :invoke :write [9 1] INFO [2019-12-17 12:34:32,843] jepsen worker 18 - jepsen.util 8568 :info :write [6 37] :RequestCanceledException INFO [2019-12-17 12:34:32,845] jepsen worker 18 - jepsen.util 8658 :invoke :cas [6 [29 17]] INFO [2019-12-17 12:34:32,845] jepsen worker 36 - jepsen.util 8676 :ok :write [12 42] INFO [2019-12-17 12:34:32,845] jepsen worker 36 - jepsen.util 8676 :invoke :read [12 nil] INFO [2019-12-17 12:34:32,846] jepsen worker 31 - jepsen.util 7501 :fail :cas [10 [27 6]] :ValueNotSwapFrom INFO [2019-12-17 12:34:32,846] jepsen worker 31 - jepsen.util 7501 :invoke :read [10 nil] INFO [2019-12-17 12:34:32,886] jepsen worker 56 - jepsen.util 9326 :ok :write [18 0] INFO [2019-12-17 12:34:32,886] jepsen worker 38 - jepsen.util 8768 :fail :cas [12 [41 34]] :ValueNotSwapFrom INFO [2019-12-17 12:34:32,886] jepsen worker 56 - jepsen.util 9326 :invoke :cas [18 [34 33]] INFO [2019-12-17 12:34:32,886] jepsen worker 38 - jepsen.util 8768 :invoke :cas [12 [11 23]] INFO [2019-12-17 12:34:32,903] jepsen worker 18 - jepsen.util 8658 :fail :cas [6 [29 17]] :ValueNotSwapFrom INFO [2019-12-17 12:34:32,904] jepsen worker 18 - jepsen.util 8658 :invoke :read [6 nil] INFO [2019-12-17 12:34:32,904] jepsen worker 31 - jepsen.util 7501 :ok :read [10 11] INFO [2019-12-17 12:34:32,904] jepsen worker 31 - jepsen.util 7501 :invoke :cas [10 [39 19]] INFO [2019-12-17 12:34:32,914] jepsen worker 67 - jepsen.util 11587 :info :cas [22 [10 23]] :RequestCanceledException INFO [2019-12-17 12:34:32,916] jepsen worker 67 - jepsen.util 11677 :invoke :cas [22 [43 18]] INFO [2019-12-17 12:34:32,929] jepsen worker 4 - jepsen.util 7924 :info :write [1 20] :RequestCanceledException INFO [2019-12-17 12:34:32,930] jepsen worker 4 - jepsen.util 8014 :invoke :write [1 29] INFO [2019-12-17 12:34:32,945] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":48852,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:34:32,945] jepsen worker 18 - jepsen.util 8658 :ok :read [6 35] INFO [2019-12-17 12:34:32,945] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":48852,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:34:32,945] jepsen worker 18 - jepsen.util 8658 :invoke :write [6 24] INFO [2019-12-17 12:34:32,945] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":48855,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:34:32,945] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":48855,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:34:32,958] jepsen worker 51 - jepsen.util 7881 :ok :write [17 47] INFO [2019-12-17 12:34:32,958] jepsen worker 51 - jepsen.util 7881 :invoke :write [17 8] INFO [2019-12-17 12:34:32,962] jepsen worker 31 - jepsen.util 7501 :fail :cas [10 [39 19]] :ValueNotSwapFrom INFO [2019-12-17 12:34:32,962] jepsen worker 31 - jepsen.util 7501 :invoke :read [10 nil] INFO [2019-12-17 12:34:32,998] jepsen worker 31 - jepsen.util 7501 :ok :read [10 11] INFO [2019-12-17 12:34:32,998] jepsen worker 31 - jepsen.util 7501 :invoke :read [10 nil] INFO [2019-12-17 12:34:33,014] jepsen worker 5 - jepsen.util 7925 :ok :write [1 27] INFO [2019-12-17 12:34:33,014] jepsen worker 5 - jepsen.util 7925 :invoke :read [1 nil] INFO [2019-12-17 12:34:33,028] jepsen worker 31 - jepsen.util 7501 :ok :read [10 11] INFO [2019-12-17 12:34:33,028] jepsen worker 31 - jepsen.util 7501 :invoke :read [10 nil] INFO [2019-12-17 12:34:33,029] jepsen worker 72 - jepsen.util 8442 :ok :write [24 22] INFO [2019-12-17 12:34:33,029] jepsen worker 72 - jepsen.util 8442 :invoke :cas [24 [7 12]] INFO [2019-12-17 12:34:33,029] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":48866,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:34:33,029] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":48866,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:34:33,029] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":48869,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:34:33,029] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":48869,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:34:33,030] jepsen worker 27 - jepsen.util 8217 :info :write [9 27] :RequestCanceledException INFO [2019-12-17 12:34:33,031] jepsen worker 27 - jepsen.util 8307 :invoke :read [9 nil] INFO [2019-12-17 12:34:33,041] jepsen worker 35 - jepsen.util 11105 :info :cas [11 [43 27]] :RequestCanceledException INFO [2019-12-17 12:34:33,042] jepsen worker 35 - jepsen.util 11195 :invoke :cas [11 [33 29]] INFO [2019-12-17 12:34:33,066] jepsen worker 72 - jepsen.util 8442 :fail :cas [24 [7 12]] :ValueNotSwapFrom INFO [2019-12-17 12:34:33,066] jepsen worker 72 - jepsen.util 8442 :invoke :read [24 nil] INFO [2019-12-17 12:34:33,066] jepsen worker 31 - jepsen.util 7501 :ok :read [10 11] INFO [2019-12-17 12:34:33,066] jepsen worker 31 - jepsen.util 7501 :invoke :cas [10 [37 5]] INFO [2019-12-17 12:34:33,076] jepsen worker 3 - jepsen.util 8283 :ok :write [1 38] INFO [2019-12-17 12:34:33,076] jepsen worker 3 - jepsen.util 8283 :invoke :write [1 46] INFO [2019-12-17 12:34:33,082] jepsen worker 64 - jepsen.util 6994 :ok :write [21 41] INFO [2019-12-17 12:34:33,082] jepsen worker 64 - jepsen.util 6994 :invoke :cas [21 [0 39]] INFO [2019-12-17 12:34:33,102] jepsen worker 29 - jepsen.util 6959 :ok :write [9 1] INFO [2019-12-17 12:34:33,102] jepsen worker 29 - jepsen.util 6959 :invoke :write [9 2] INFO [2019-12-17 12:34:33,114] jepsen worker 70 - jepsen.util 8800 :info :write [23 46] :RequestCanceledException INFO [2019-12-17 12:34:33,114] jepsen worker 64 - jepsen.util 6994 :fail :cas [21 [0 39]] :ValueNotSwapFrom INFO [2019-12-17 12:34:33,114] jepsen worker 64 - jepsen.util 6994 :invoke :cas [21 [3 42]] INFO [2019-12-17 12:34:33,115] jepsen worker 70 - jepsen.util 8890 :invoke :read [23 nil] INFO [2019-12-17 12:34:33,125] jepsen worker 27 - jepsen.util 8307 :ok :read [9 1] INFO [2019-12-17 12:34:33,125] jepsen worker 27 - jepsen.util 8307 :invoke :read [9 nil] INFO [2019-12-17 12:34:33,126] jepsen worker 80 - jepsen.util 8180 :ok :write [26 17] INFO [2019-12-17 12:34:33,127] jepsen worker 80 - jepsen.util 8180 :invoke :cas [26 [32 14]] INFO [2019-12-17 12:34:33,127] jepsen worker 78 - jepsen.util 8178 :info :write [26 15] :RequestCanceledException INFO [2019-12-17 12:34:33,127] jepsen worker 20 - jepsen.util 8120 :info :write [6 4] :RequestCanceledException INFO [2019-12-17 12:34:33,128] jepsen worker 78 - jepsen.util 8268 :invoke :read [26 nil] INFO [2019-12-17 12:34:33,129] jepsen worker 52 - jepsen.util 8602 :fail :cas [17 [21 12]] :ValueNotSwapFrom INFO [2019-12-17 12:34:33,129] jepsen worker 52 - jepsen.util 8602 :invoke :cas [17 [9 23]] INFO [2019-12-17 12:34:33,131] jepsen worker 20 - jepsen.util 8210 :invoke :cas [6 [6 44]] INFO [2019-12-17 12:34:33,138] jepsen worker 64 - jepsen.util 6994 :fail :cas [21 [3 42]] :ValueNotSwapFrom INFO [2019-12-17 12:34:33,139] jepsen worker 64 - jepsen.util 6994 :invoke :read [21 nil] INFO [2019-12-17 12:34:33,146] jepsen worker 78 - jepsen.util 8268 :ok :read [26 17] INFO [2019-12-17 12:34:33,146] jepsen worker 78 - jepsen.util 8268 :invoke :cas [26 [42 26]] INFO [2019-12-17 12:34:33,187] jepsen worker 52 - jepsen.util 8602 :fail :cas [17 [9 23]] :ValueNotSwapFrom INFO [2019-12-17 12:34:33,188] jepsen worker 52 - jepsen.util 8602 :invoke :read [17 nil] INFO [2019-12-17 12:34:33,196] jepsen worker 28 - jepsen.util 6688 :fail :read [9 nil] :RequestTimeoutException INFO [2019-12-17 12:34:33,196] jepsen worker 28 - jepsen.util 6688 :invoke :read [9 nil] INFO [2019-12-17 12:34:33,217] jepsen worker 64 - jepsen.util 6994 :ok :read [21 41] INFO [2019-12-17 12:34:33,217] jepsen worker 64 - jepsen.util 6994 :invoke :write [21 38] INFO [2019-12-17 12:34:33,218] jepsen worker 72 - jepsen.util 8442 :ok :read [24 22] INFO [2019-12-17 12:34:33,218] jepsen worker 72 - jepsen.util 8442 :invoke :read [24 nil] INFO [2019-12-17 12:34:33,227] jepsen worker 89 - jepsen.util 8369 :info :write [29 40] :RequestCanceledException INFO [2019-12-17 12:34:33,228] jepsen worker 89 - jepsen.util 8459 :invoke :read [29 nil] INFO [2019-12-17 12:34:33,234] jepsen worker 78 - jepsen.util 8268 :fail :cas [26 [42 26]] :ValueNotSwapFrom INFO [2019-12-17 12:34:33,234] jepsen worker 78 - jepsen.util 8268 :invoke :write [26 19] INFO [2019-12-17 12:34:33,249] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":48901,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:34:33,249] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":48901,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:34:33,249] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":48904,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:34:33,249] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":48904,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:34:33,253] jepsen worker 5 - jepsen.util 7925 :ok :read [1 38] INFO [2019-12-17 12:34:33,254] jepsen worker 5 - jepsen.util 7925 :invoke :read [1 nil] INFO [2019-12-17 12:34:33,255] jepsen worker 79 - jepsen.util 8809 :ok :write [26 12] INFO [2019-12-17 12:34:33,255] jepsen worker 79 - jepsen.util 8809 :invoke :write [26 32] INFO [2019-12-17 12:34:33,256] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":48895,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:34:33,256] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":48895,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:34:33,257] jepsen worker 33 - jepsen.util 11463 :info :cas [11 [8 48]] :RequestCanceledException INFO [2019-12-17 12:34:33,257] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":48896,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:34:33,257] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":48896,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:34:33,259] jepsen worker 33 - jepsen.util 11553 :invoke :write [11 23] INFO [2019-12-17 12:34:33,271] jepsen worker 56 - jepsen.util 9326 :fail :cas [18 [34 33]] :ValueNotSwapFrom INFO [2019-12-17 12:34:33,271] jepsen worker 56 - jepsen.util 9326 :invoke :cas [18 [40 46]] INFO [2019-12-17 12:34:33,272] jepsen worker 31 - jepsen.util 7501 :fail :cas [10 [37 5]] :ValueNotSwapFrom INFO [2019-12-17 12:34:33,272] jepsen worker 31 - jepsen.util 7501 :invoke :read [10 nil] INFO [2019-12-17 12:34:33,309] jepsen worker 28 - jepsen.util 6688 :ok :read [9 1] INFO [2019-12-17 12:34:33,309] jepsen worker 28 - jepsen.util 6688 :invoke :read [9 nil] INFO [2019-12-17 12:34:33,329] jepsen worker 57 - jepsen.util 11487 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:34:33,329] jepsen worker 57 - jepsen.util 11487 :invoke :read [19 nil] INFO [2019-12-17 12:34:33,330] jepsen worker 36 - jepsen.util 8676 :ok :read [12 42] INFO [2019-12-17 12:34:33,330] jepsen worker 36 - jepsen.util 8676 :invoke :cas [12 [25 42]] INFO [2019-12-17 12:34:33,343] jepsen worker 88 - jepsen.util 8638 :ok :write [29 37] INFO [2019-12-17 12:34:33,344] jepsen worker 88 - jepsen.util 8638 :invoke :cas [29 [39 20]] INFO [2019-12-17 12:34:33,348] jepsen worker 31 - jepsen.util 7501 :ok :read [10 11] INFO [2019-12-17 12:34:33,348] jepsen worker 31 - jepsen.util 7501 :invoke :cas [10 [20 18]] INFO [2019-12-17 12:34:33,350] jepsen worker 3 - jepsen.util 8283 :ok :write [1 46] INFO [2019-12-17 12:34:33,350] jepsen worker 3 - jepsen.util 8283 :invoke :read [1 nil] INFO [2019-12-17 12:34:33,363] jepsen worker 72 - jepsen.util 8442 :ok :read [24 22] INFO [2019-12-17 12:34:33,363] jepsen worker 72 - jepsen.util 8442 :invoke :read [24 nil] INFO [2019-12-17 12:34:33,396] jepsen worker 62 - jepsen.util 11762 :info :write [20 35] :RequestCanceledException INFO [2019-12-17 12:34:33,399] jepsen worker 62 - jepsen.util 11852 :invoke :write [20 1] INFO [2019-12-17 12:34:33,399] jepsen worker 56 - jepsen.util 9326 :fail :cas [18 [40 46]] :ValueNotSwapFrom INFO [2019-12-17 12:34:33,400] jepsen worker 56 - jepsen.util 9326 :invoke :write [18 17] INFO [2019-12-17 12:34:33,414] jepsen worker 25 - jepsen.util 12175 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:34:33,414] jepsen worker 25 - jepsen.util 12175 :invoke :write [8 2] INFO [2019-12-17 12:34:33,427] jepsen worker 79 - jepsen.util 8809 :ok :write [26 32] INFO [2019-12-17 12:34:33,427] jepsen worker 79 - jepsen.util 8809 :invoke :cas [26 [48 42]] INFO [2019-12-17 12:34:33,434] jepsen worker 31 - jepsen.util 7501 :fail :cas [10 [20 18]] :ValueNotSwapFrom INFO [2019-12-17 12:34:33,434] jepsen worker 31 - jepsen.util 7501 :invoke :read [10 nil] INFO [2019-12-17 12:34:33,443] jepsen worker 0 - jepsen.util 11880 :info :write [0 36] :RequestCanceledException INFO [2019-12-17 12:34:33,444] jepsen worker 3 - jepsen.util 8283 :ok :read [1 46] INFO [2019-12-17 12:34:33,444] jepsen worker 3 - jepsen.util 8283 :invoke :write [1 14] INFO [2019-12-17 12:34:33,445] jepsen worker 0 - jepsen.util 11970 :invoke :cas [0 [47 3]] INFO [2019-12-17 12:34:33,453] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":48928,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:34:33,453] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":48928,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:34:33,461] jepsen worker 58 - jepsen.util 12388 :info :write [19 11] :RequestCanceledException INFO [2019-12-17 12:34:33,462] jepsen worker 58 - jepsen.util 12478 :invoke :write [19 44] INFO [2019-12-17 12:34:33,462] jepsen worker 28 - jepsen.util 6688 :ok :read [9 1] INFO [2019-12-17 12:34:33,462] jepsen worker 28 - jepsen.util 6688 :invoke :read [9 nil] INFO [2019-12-17 12:34:33,480] jepsen worker 31 - jepsen.util 7501 :ok :read [10 11] INFO [2019-12-17 12:34:33,481] jepsen worker 31 - jepsen.util 7501 :invoke :write [10 25] INFO [2019-12-17 12:34:33,489] jepsen worker 5 - jepsen.util 7925 :ok :read [1 46] INFO [2019-12-17 12:34:33,489] jepsen worker 36 - jepsen.util 8676 :fail :cas [12 [25 42]] :ValueNotSwapFrom INFO [2019-12-17 12:34:33,490] jepsen worker 5 - jepsen.util 7925 :invoke :read [1 nil] INFO [2019-12-17 12:34:33,490] jepsen worker 36 - jepsen.util 8676 :invoke :cas [12 [37 12]] INFO [2019-12-17 12:34:33,492] jepsen worker 29 - jepsen.util 6959 :ok :write [9 2] INFO [2019-12-17 12:34:33,492] jepsen worker 29 - jepsen.util 6959 :invoke :read [9 nil] INFO [2019-12-17 12:34:33,510] jepsen worker 52 - jepsen.util 8602 :ok :read [17 47] INFO [2019-12-17 12:34:33,511] jepsen worker 52 - jepsen.util 8602 :invoke :read [17 nil] INFO [2019-12-17 12:34:33,522] jepsen worker 79 - jepsen.util 8809 :fail :cas [26 [48 42]] :ValueNotSwapFrom INFO [2019-12-17 12:34:33,522] jepsen worker 79 - jepsen.util 8809 :invoke :write [26 40] INFO [2019-12-17 12:34:33,531] jepsen worker 64 - jepsen.util 6994 :ok :write [21 38] INFO [2019-12-17 12:34:33,531] jepsen worker 64 - jepsen.util 6994 :invoke :read [21 nil] INFO [2019-12-17 12:34:33,543] jepsen worker 85 - jepsen.util 12235 :info :cas [28 [46 20]] :RequestCanceledException INFO [2019-12-17 12:34:33,544] jepsen worker 36 - jepsen.util 8676 :fail :cas [12 [37 12]] :ValueNotSwapFrom INFO [2019-12-17 12:34:33,544] jepsen worker 36 - jepsen.util 8676 :invoke :write [12 36] INFO [2019-12-17 12:34:33,545] jepsen worker 85 - jepsen.util 12325 :invoke :write [28 18] INFO [2019-12-17 12:34:33,555] jepsen worker 29 - jepsen.util 6959 :ok :read [9 2] INFO [2019-12-17 12:34:33,555] jepsen worker 29 - jepsen.util 6959 :invoke :read [9 nil] INFO [2019-12-17 12:34:33,578] jepsen worker 72 - jepsen.util 8442 :ok :read [24 22] INFO [2019-12-17 12:34:33,578] jepsen worker 72 - jepsen.util 8442 :invoke :write [24 14] INFO [2019-12-17 12:34:33,589] jepsen worker 12 - jepsen.util 9012 :ok :write [4 19] INFO [2019-12-17 12:34:33,590] jepsen worker 12 - jepsen.util 9012 :invoke :cas [4 [36 6]] INFO [2019-12-17 12:34:33,596] jepsen worker 45 - jepsen.util 8775 :info :cas [15 [15 36]] :RequestCanceledException INFO [2019-12-17 12:34:33,597] jepsen worker 45 - jepsen.util 8865 :invoke :read [15 nil] INFO [2019-12-17 12:34:33,613] jepsen worker 22 - jepsen.util 11452 :info :cas [7 [5 12]] :RequestCanceledException INFO [2019-12-17 12:34:33,613] jepsen worker 26 - jepsen.util 11906 :info :cas [8 [18 45]] :RequestCanceledException INFO [2019-12-17 12:34:33,615] jepsen worker 26 - jepsen.util 11996 :invoke :write [8 4] INFO [2019-12-17 12:34:33,615] jepsen worker 22 - jepsen.util 11542 :invoke :cas [7 [9 22]] INFO [2019-12-17 12:34:33,623] jepsen worker 29 - jepsen.util 6959 :ok :read [9 2] INFO [2019-12-17 12:34:33,623] jepsen worker 29 - jepsen.util 6959 :invoke :write [9 9] INFO [2019-12-17 12:34:33,626] jepsen worker 48 - jepsen.util 10758 :info :write [16 9] :RequestCanceledException INFO [2019-12-17 12:34:33,627] jepsen worker 64 - jepsen.util 6994 :ok :read [21 38] INFO [2019-12-17 12:34:33,627] jepsen worker 64 - jepsen.util 6994 :invoke :cas [21 [45 28]] INFO [2019-12-17 12:34:33,628] jepsen worker 28 - jepsen.util 6688 :ok :read [9 2] INFO [2019-12-17 12:34:33,628] jepsen worker 28 - jepsen.util 6688 :invoke :write [9 48] INFO [2019-12-17 12:34:33,628] jepsen worker 48 - jepsen.util 10848 :invoke :read [16 nil] INFO [2019-12-17 12:34:33,633] jepsen worker 78 - jepsen.util 8268 :ok :write [26 19] INFO [2019-12-17 12:34:33,633] jepsen worker 78 - jepsen.util 8268 :invoke :read [26 nil] INFO [2019-12-17 12:34:33,660] jepsen worker 6 - jepsen.util 12066 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:34:33,660] jepsen worker 71 - jepsen.util 8891 :info :write [23 40] :RequestCanceledException INFO [2019-12-17 12:34:33,660] jepsen worker 6 - jepsen.util 12066 :invoke :write [2 6] INFO [2019-12-17 12:34:33,660] jepsen worker 9 - jepsen.util 8829 :info :write [3 36] :RequestCanceledException INFO [2019-12-17 12:34:33,662] jepsen worker 71 - jepsen.util 8981 :invoke :cas [23 [17 42]] INFO [2019-12-17 12:34:33,662] jepsen worker 9 - jepsen.util 8919 :invoke :cas [3 [41 18]] INFO [2019-12-17 12:34:33,665] jepsen worker 69 - jepsen.util 8619 :ok :read [23 46] INFO [2019-12-17 12:34:33,665] jepsen worker 69 - jepsen.util 8619 :invoke :read [23 nil] INFO [2019-12-17 12:34:33,681] jepsen worker 64 - jepsen.util 6994 :fail :cas [21 [45 28]] :ValueNotSwapFrom INFO [2019-12-17 12:34:33,682] jepsen worker 64 - jepsen.util 6994 :invoke :write [21 6] INFO [2019-12-17 12:34:33,683] jepsen worker 52 - jepsen.util 8602 :ok :read [17 47] INFO [2019-12-17 12:34:33,683] jepsen worker 52 - jepsen.util 8602 :invoke :write [17 41] INFO [2019-12-17 12:34:33,691] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":48958,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:34:33,691] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":48958,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:34:33,691] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":48961,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:34:33,691] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":48961,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:34:33,694] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:34:33,694] jepsen nemesis - couchbase.util Rebalance check count: 379 INFO [2019-12-17 12:34:33,694] jepsen nemesis - couchbase.util Rebalance stuck count: 360 INFO [2019-12-17 12:34:33,714] jepsen worker 61 - jepsen.util 12211 :info :cas [20 [27 49]] :RequestCanceledException INFO [2019-12-17 12:34:33,714] jepsen worker 66 - jepsen.util 11406 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:34:33,714] jepsen worker 66 - jepsen.util 11406 :invoke :cas [22 [47 24]] INFO [2019-12-17 12:34:33,715] jepsen worker 61 - jepsen.util 12301 :invoke :write [20 7] INFO [2019-12-17 12:34:33,730] jepsen worker 84 - jepsen.util 11064 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:34:33,730] jepsen worker 84 - jepsen.util 11064 :invoke :read [28 nil] INFO [2019-12-17 12:34:33,736] jepsen worker 29 - jepsen.util 6959 :ok :write [9 9] INFO [2019-12-17 12:34:33,736] jepsen worker 29 - jepsen.util 6959 :invoke :cas [9 [7 27]] INFO [2019-12-17 12:34:33,741] jepsen worker 44 - jepsen.util 11744 :info :write [14 32] :RequestCanceledException INFO [2019-12-17 12:34:33,743] jepsen worker 44 - jepsen.util 11834 :invoke :cas [14 [38 23]] INFO [2019-12-17 12:34:33,756] jepsen worker 5 - jepsen.util 7925 :ok :read [1 46] INFO [2019-12-17 12:34:33,757] jepsen worker 5 - jepsen.util 7925 :invoke :write [1 49] INFO [2019-12-17 12:34:33,771] jepsen worker 29 - jepsen.util 6959 :fail :cas [9 [7 27]] :ValueNotSwapFrom INFO [2019-12-17 12:34:33,772] jepsen worker 29 - jepsen.util 6959 :invoke :write [9 3] INFO [2019-12-17 12:34:33,773] jepsen worker 78 - jepsen.util 8268 :ok :read [26 19] INFO [2019-12-17 12:34:33,774] jepsen worker 78 - jepsen.util 8268 :invoke :read [26 nil] INFO [2019-12-17 12:34:33,791] jepsen worker 64 - jepsen.util 6994 :ok :write [21 6] INFO [2019-12-17 12:34:33,791] jepsen worker 64 - jepsen.util 6994 :invoke :read [21 nil] INFO [2019-12-17 12:34:33,795] jepsen worker 71 - jepsen.util 8981 :fail :cas [23 [17 42]] :ValueNotSwapFrom INFO [2019-12-17 12:34:33,795] jepsen worker 71 - jepsen.util 8981 :invoke :write [23 41] INFO [2019-12-17 12:34:33,803] jepsen worker 11 - jepsen.util 8381 :ok :write [3 9] INFO [2019-12-17 12:34:33,803] jepsen worker 11 - jepsen.util 8381 :invoke :write [3 8] INFO [2019-12-17 12:34:33,814] jepsen worker 64 - jepsen.util 6994 :ok :read [21 6] INFO [2019-12-17 12:34:33,814] jepsen worker 51 - jepsen.util 7881 :ok :write [17 8] INFO [2019-12-17 12:34:33,814] jepsen worker 64 - jepsen.util 6994 :invoke :write [21 21] INFO [2019-12-17 12:34:33,815] jepsen worker 51 - jepsen.util 7881 :invoke :write [17 34] INFO [2019-12-17 12:34:33,841] jepsen worker 36 - jepsen.util 8676 :ok :write [12 36] INFO [2019-12-17 12:34:33,841] jepsen worker 36 - jepsen.util 8676 :invoke :cas [12 [25 20]] INFO [2019-12-17 12:34:33,848] jepsen worker 54 - jepsen.util 9414 :ok :write [18 29] INFO [2019-12-17 12:34:33,848] jepsen worker 54 - jepsen.util 9414 :invoke :write [18 45] INFO [2019-12-17 12:34:33,858] jepsen worker 70 - jepsen.util 8890 :ok :read [23 46] INFO [2019-12-17 12:34:33,858] jepsen worker 70 - jepsen.util 8890 :invoke :write [23 33] INFO [2019-12-17 12:34:33,861] jepsen worker 24 - jepsen.util 10644 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:34:33,861] jepsen worker 46 - jepsen.util 8416 :fail :read [15 nil] :RequestTimeoutException INFO [2019-12-17 12:34:33,861] jepsen worker 24 - jepsen.util 10644 :invoke :cas [8 [29 47]] INFO [2019-12-17 12:34:33,862] jepsen worker 46 - jepsen.util 8416 :invoke :cas [15 [41 24]] INFO [2019-12-17 12:34:33,892] jepsen worker 20 - jepsen.util 8210 :fail :cas [6 [6 44]] :ValueNotSwapFrom INFO [2019-12-17 12:34:33,892] jepsen worker 20 - jepsen.util 8210 :invoke :write [6 49] INFO [2019-12-17 12:34:33,897] jepsen worker 40 - jepsen.util 11650 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:34:33,897] jepsen worker 40 - jepsen.util 11650 :invoke :cas [13 [35 15]] INFO [2019-12-17 12:34:33,903] jepsen worker 29 - jepsen.util 6959 :ok :write [9 3] INFO [2019-12-17 12:34:33,904] jepsen worker 29 - jepsen.util 6959 :invoke :cas [9 [16 25]] INFO [2019-12-17 12:34:33,912] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":48984,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:34:33,912] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":48984,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:34:33,912] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":48991,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:34:33,912] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":48991,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:34:33,914] jepsen worker 9 - jepsen.util 8919 :fail :cas [3 [41 18]] :ValueNotSwapFrom INFO [2019-12-17 12:34:33,914] jepsen worker 9 - jepsen.util 8919 :invoke :read [3 nil] INFO [2019-12-17 12:34:33,928] jepsen worker 29 - jepsen.util 6959 :fail :cas [9 [16 25]] :ValueNotSwapFrom INFO [2019-12-17 12:34:33,928] jepsen worker 29 - jepsen.util 6959 :invoke :write [9 20] INFO [2019-12-17 12:34:33,929] jepsen worker 55 - jepsen.util 9145 :info :write [18 21] :RequestCanceledException INFO [2019-12-17 12:34:33,933] jepsen worker 55 - jepsen.util 9235 :invoke :cas [18 [4 48]] INFO [2019-12-17 12:34:33,942] jepsen worker 14 - jepsen.util 8654 :fail :cas [4 [12 7]] :ValueNotSwapFrom INFO [2019-12-17 12:34:33,942] jepsen worker 14 - jepsen.util 8654 :invoke :cas [4 [47 28]] INFO [2019-12-17 12:34:33,949] jepsen worker 38 - jepsen.util 8768 :fail :cas [12 [11 23]] :ValueNotSwapFrom INFO [2019-12-17 12:34:33,949] jepsen worker 38 - jepsen.util 8768 :invoke :write [12 30] INFO [2019-12-17 12:34:33,950] jepsen worker 52 - jepsen.util 8602 :ok :write [17 41] INFO [2019-12-17 12:34:33,951] jepsen worker 52 - jepsen.util 8602 :invoke :write [17 35] INFO [2019-12-17 12:34:33,960] jepsen worker 1 - jepsen.util 11701 :info :cas [0 [44 39]] :RequestCanceledException INFO [2019-12-17 12:34:33,961] jepsen worker 1 - jepsen.util 11791 :invoke :read [0 nil] INFO [2019-12-17 12:34:33,974] jepsen worker 80 - jepsen.util 8180 :fail :cas [26 [32 14]] :ValueNotSwapFrom INFO [2019-12-17 12:34:33,974] jepsen worker 80 - jepsen.util 8180 :invoke :cas [26 [19 16]] INFO [2019-12-17 12:34:33,995] jepsen worker 42 - jepsen.util 12552 :info :cas [14 [42 3]] :RequestCanceledException INFO [2019-12-17 12:34:33,997] jepsen worker 42 - jepsen.util 12642 :invoke :cas [14 [43 9]] INFO [2019-12-17 12:34:34,002] jepsen worker 53 - jepsen.util 7793 :ok :read [17 41] INFO [2019-12-17 12:34:34,003] jepsen worker 53 - jepsen.util 7793 :invoke :cas [17 [23 7]] INFO [2019-12-17 12:34:34,011] jepsen worker 64 - jepsen.util 6994 :ok :write [21 21] INFO [2019-12-17 12:34:34,011] jepsen worker 64 - jepsen.util 6994 :invoke :write [21 42] INFO [2019-12-17 12:34:34,013] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":49007,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:34:34,013] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":49007,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:34:34,013] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":49009,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:34:34,013] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":49009,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:34:34,014] jepsen worker 17 - jepsen.util 11447 :info :cas [5 [12 46]] :RequestCanceledException INFO [2019-12-17 12:34:34,015] jepsen worker 17 - jepsen.util 11537 :invoke :cas [5 [30 13]] INFO [2019-12-17 12:34:34,024] jepsen worker 87 - jepsen.util 7917 :ok :write [29 2] INFO [2019-12-17 12:34:34,024] jepsen worker 87 - jepsen.util 7917 :invoke :write [29 40] INFO [2019-12-17 12:34:34,029] jepsen worker 13 - jepsen.util 8113 :fail :cas [4 [48 6]] :ValueNotSwapFrom INFO [2019-12-17 12:34:34,029] jepsen worker 13 - jepsen.util 8113 :invoke :write [4 33] INFO [2019-12-17 12:34:34,030] jepsen worker 15 - jepsen.util 10725 :info :cas [5 [42 21]] :RequestCanceledException INFO [2019-12-17 12:34:34,031] jepsen worker 15 - jepsen.util 10815 :invoke :write [5 0] INFO [2019-12-17 12:34:34,043] jepsen worker 43 - jepsen.util 11203 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:34:34,043] jepsen worker 43 - jepsen.util 11203 :invoke :cas [14 [10 11]] INFO [2019-12-17 12:34:34,060] jepsen worker 75 - jepsen.util 10965 :info :cas [25 [45 39]] :RequestCanceledException INFO [2019-12-17 12:34:34,061] jepsen worker 75 - jepsen.util 11055 :invoke :read [25 nil] INFO [2019-12-17 12:34:34,087] jepsen worker 14 - jepsen.util 8654 :fail :cas [4 [47 28]] :ValueNotSwapFrom INFO [2019-12-17 12:34:34,087] jepsen worker 14 - jepsen.util 8654 :invoke :cas [4 [8 31]] INFO [2019-12-17 12:34:34,091] jepsen worker 69 - jepsen.util 8619 :ok :read [23 46] INFO [2019-12-17 12:34:34,091] jepsen worker 69 - jepsen.util 8619 :invoke :read [23 nil] INFO [2019-12-17 12:34:34,096] jepsen worker 8 - jepsen.util 11348 :info :cas [2 [20 23]] :RequestCanceledException INFO [2019-12-17 12:34:34,097] jepsen worker 8 - jepsen.util 11438 :invoke :read [2 nil] INFO [2019-12-17 12:34:34,097] jepsen worker 29 - jepsen.util 6959 :ok :write [9 20] INFO [2019-12-17 12:34:34,097] jepsen worker 29 - jepsen.util 6959 :invoke :write [9 46] INFO [2019-12-17 12:34:34,113] jepsen worker 19 - jepsen.util 8659 :fail :read [6 nil] :RequestTimeoutException INFO [2019-12-17 12:34:34,113] jepsen worker 19 - jepsen.util 8659 :invoke :read [6 nil] INFO [2019-12-17 12:34:34,130] jepsen worker 77 - jepsen.util 11687 :info :write [25 9] :RequestCanceledException INFO [2019-12-17 12:34:34,130] jepsen worker 65 - jepsen.util 6995 :fail :read [21 nil] :RequestTimeoutException INFO [2019-12-17 12:34:34,130] jepsen worker 65 - jepsen.util 6995 :invoke :read [21 nil] INFO [2019-12-17 12:34:34,131] jepsen worker 77 - jepsen.util 11777 :invoke :cas [25 [11 24]] INFO [2019-12-17 12:34:34,135] jepsen worker 88 - jepsen.util 8638 :fail :cas [29 [39 20]] :ValueNotSwapFrom INFO [2019-12-17 12:34:34,136] jepsen worker 88 - jepsen.util 8638 :invoke :write [29 42] INFO [2019-12-17 12:34:34,155] jepsen worker 18 - jepsen.util 8658 :ok :write [6 24] INFO [2019-12-17 12:34:34,156] jepsen worker 18 - jepsen.util 8658 :invoke :cas [6 [36 34]] INFO [2019-12-17 12:34:34,156] jepsen worker 31 - jepsen.util 7501 :ok :write [10 25] INFO [2019-12-17 12:34:34,156] jepsen worker 31 - jepsen.util 7501 :invoke :write [10 45] INFO [2019-12-17 12:34:34,172] jepsen worker 37 - jepsen.util 8047 :ok :write [12 40] INFO [2019-12-17 12:34:34,172] jepsen worker 37 - jepsen.util 8047 :invoke :write [12 47] INFO [2019-12-17 12:34:34,178] jepsen worker 12 - jepsen.util 9012 :fail :cas [4 [36 6]] :ValueNotSwapFrom INFO [2019-12-17 12:34:34,178] jepsen worker 12 - jepsen.util 9012 :invoke :read [4 nil] INFO [2019-12-17 12:34:34,187] jepsen worker 89 - jepsen.util 8459 :ok :read [29 2] INFO [2019-12-17 12:34:34,187] jepsen worker 89 - jepsen.util 8459 :invoke :write [29 18] INFO [2019-12-17 12:34:34,201] jepsen worker 5 - jepsen.util 7925 :ok :write [1 49] INFO [2019-12-17 12:34:34,201] jepsen worker 5 - jepsen.util 7925 :invoke :read [1 nil] INFO [2019-12-17 12:34:34,214] jepsen worker 49 - jepsen.util 11029 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:34:34,214] jepsen worker 49 - jepsen.util 11029 :invoke :read [16 nil] INFO [2019-12-17 12:34:34,214] jepsen worker 64 - jepsen.util 6994 :ok :write [21 42] INFO [2019-12-17 12:34:34,214] jepsen worker 64 - jepsen.util 6994 :invoke :cas [21 [22 23]] INFO [2019-12-17 12:34:34,225] jepsen worker 46 - jepsen.util 8416 :fail :cas [15 [41 24]] :ValueNotSwapFrom INFO [2019-12-17 12:34:34,225] jepsen worker 46 - jepsen.util 8416 :invoke :read [15 nil] INFO [2019-12-17 12:34:34,225] jepsen worker 78 - jepsen.util 8268 :ok :read [26 40] INFO [2019-12-17 12:34:34,225] jepsen worker 78 - jepsen.util 8268 :invoke :write [26 3] INFO [2019-12-17 12:34:34,230] jepsen worker 7 - jepsen.util 11797 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:34:34,230] jepsen worker 7 - jepsen.util 11797 :invoke :read [2 nil] INFO [2019-12-17 12:34:34,232] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":49042,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:34:34,232] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":49042,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:34:34,232] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":49046,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:34:34,232] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":49046,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:34:34,234] jepsen worker 79 - jepsen.util 8809 :ok :write [26 40] INFO [2019-12-17 12:34:34,234] jepsen worker 79 - jepsen.util 8809 :invoke :cas [26 [5 27]] INFO [2019-12-17 12:34:34,239] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":49032,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:34:34,239] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":49032,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:34:34,239] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":49033,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:34:34,239] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":49033,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:34:34,239] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":49034,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:34:34,239] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":49034,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:34:34,240] jepsen worker 51 - jepsen.util 7881 :ok :write [17 34] INFO [2019-12-17 12:34:34,240] jepsen worker 51 - jepsen.util 7881 :invoke :cas [17 [32 13]] INFO [2019-12-17 12:34:34,243] jepsen worker 53 - jepsen.util 7793 :fail :cas [17 [23 7]] :ValueNotSwapFrom INFO [2019-12-17 12:34:34,244] jepsen worker 53 - jepsen.util 7793 :invoke :write [17 42] INFO [2019-12-17 12:34:34,260] jepsen worker 10 - jepsen.util 8740 :info :write [3 37] :RequestCanceledException INFO [2019-12-17 12:34:34,260] jepsen worker 32 - jepsen.util 8672 :info :cas [10 [14 11]] :RequestCanceledException INFO [2019-12-17 12:34:34,260] jepsen worker 47 - jepsen.util 8957 :fail :read [15 nil] :RequestTimeoutException INFO [2019-12-17 12:34:34,261] jepsen worker 47 - jepsen.util 8957 :invoke :read [15 nil] INFO [2019-12-17 12:34:34,262] jepsen worker 10 - jepsen.util 8830 :invoke :cas [3 [0 3]] INFO [2019-12-17 12:34:34,262] jepsen worker 32 - jepsen.util 8762 :invoke :cas [10 [36 2]] INFO [2019-12-17 12:34:34,262] jepsen worker 14 - jepsen.util 8654 :fail :cas [4 [8 31]] :ValueNotSwapFrom INFO [2019-12-17 12:34:34,262] jepsen worker 14 - jepsen.util 8654 :invoke :cas [4 [47 15]] INFO [2019-12-17 12:34:34,284] jepsen worker 36 - jepsen.util 8676 :fail :cas [12 [25 20]] :ValueNotSwapFrom INFO [2019-12-17 12:34:34,284] jepsen worker 36 - jepsen.util 8676 :invoke :read [12 nil] INFO [2019-12-17 12:34:34,296] jepsen worker 63 - jepsen.util 7263 :info :write [21 3] :RequestCanceledException INFO [2019-12-17 12:34:34,297] jepsen worker 63 - jepsen.util 7353 :invoke :read [21 nil] INFO [2019-12-17 12:34:34,299] jepsen worker 55 - jepsen.util 9235 :fail :cas [18 [4 48]] :ValueNotSwapFrom INFO [2019-12-17 12:34:34,299] jepsen worker 55 - jepsen.util 9235 :invoke :write [18 19] INFO [2019-12-17 12:34:34,316] jepsen worker 14 - jepsen.util 8654 :fail :cas [4 [47 15]] :ValueNotSwapFrom INFO [2019-12-17 12:34:34,316] jepsen worker 14 - jepsen.util 8654 :invoke :write [4 17] INFO [2019-12-17 12:34:34,325] jepsen worker 79 - jepsen.util 8809 :fail :cas [26 [5 27]] :ValueNotSwapFrom INFO [2019-12-17 12:34:34,325] jepsen worker 79 - jepsen.util 8809 :invoke :cas [26 [7 35]] INFO [2019-12-17 12:34:34,326] jepsen worker 64 - jepsen.util 6994 :fail :cas [21 [22 23]] :ValueNotSwapFrom INFO [2019-12-17 12:34:34,326] jepsen worker 64 - jepsen.util 6994 :invoke :read [21 nil] INFO [2019-12-17 12:34:34,329] jepsen worker 18 - jepsen.util 8658 :fail :cas [6 [36 34]] :ValueNotSwapFrom INFO [2019-12-17 12:34:34,330] jepsen worker 18 - jepsen.util 8658 :invoke :write [6 26] INFO [2019-12-17 12:34:34,330] jepsen worker 2 - jepsen.util 11162 :info :write [0 37] :RequestCanceledException INFO [2019-12-17 12:34:34,331] jepsen worker 2 - jepsen.util 11252 :invoke :cas [0 [47 25]] INFO [2019-12-17 12:34:34,338] jepsen worker 80 - jepsen.util 8180 :fail :cas [26 [19 16]] :ValueNotSwapFrom INFO [2019-12-17 12:34:34,338] jepsen worker 80 - jepsen.util 8180 :invoke :read [26 nil] INFO [2019-12-17 12:34:34,343] jepsen worker 73 - jepsen.util 8083 :info :write [24 27] :RequestCanceledException INFO [2019-12-17 12:34:34,344] jepsen worker 73 - jepsen.util 8173 :invoke :write [24 5] INFO [2019-12-17 12:34:34,347] jepsen worker 29 - jepsen.util 6959 :ok :write [9 46] INFO [2019-12-17 12:34:34,347] jepsen worker 29 - jepsen.util 6959 :invoke :write [9 47] INFO [2019-12-17 12:34:34,348] jepsen worker 64 - jepsen.util 6994 :ok :read [21 42] INFO [2019-12-17 12:34:34,348] jepsen worker 64 - jepsen.util 6994 :invoke :read [21 nil] INFO [2019-12-17 12:34:34,375] jepsen worker 36 - jepsen.util 8676 :ok :read [12 40] INFO [2019-12-17 12:34:34,375] jepsen worker 36 - jepsen.util 8676 :invoke :cas [12 [34 1]] INFO [2019-12-17 12:34:34,380] jepsen worker 80 - jepsen.util 8180 :ok :read [26 40] INFO [2019-12-17 12:34:34,380] jepsen worker 80 - jepsen.util 8180 :invoke :cas [26 [35 47]] INFO [2019-12-17 12:34:34,392] jepsen worker 64 - jepsen.util 6994 :ok :read [21 42] INFO [2019-12-17 12:34:34,392] jepsen worker 64 - jepsen.util 6994 :invoke :cas [21 [29 12]] INFO [2019-12-17 12:34:34,408] jepsen worker 51 - jepsen.util 7881 :fail :cas [17 [32 13]] :ValueNotSwapFrom INFO [2019-12-17 12:34:34,408] jepsen worker 51 - jepsen.util 7881 :invoke :cas [17 [43 22]] INFO [2019-12-17 12:34:34,433] jepsen worker 46 - jepsen.util 8416 :ok :read [15 3] INFO [2019-12-17 12:34:34,433] jepsen worker 46 - jepsen.util 8416 :invoke :cas [15 [31 19]] INFO [2019-12-17 12:34:34,440] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":49074,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:34:34,440] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":49074,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:34:34,440] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":49078,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:34:34,441] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":49078,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:34:34,453] jepsen worker 69 - jepsen.util 8619 :ok :read [23 46] INFO [2019-12-17 12:34:34,453] jepsen worker 69 - jepsen.util 8619 :invoke :cas [23 [8 45]] INFO [2019-12-17 12:34:34,455] jepsen worker 47 - jepsen.util 8957 :ok :read [15 3] INFO [2019-12-17 12:34:34,455] jepsen worker 47 - jepsen.util 8957 :invoke :read [15 nil] INFO [2019-12-17 12:34:34,460] jepsen worker 12 - jepsen.util 9012 :ok :read [4 19] INFO [2019-12-17 12:34:34,460] jepsen worker 12 - jepsen.util 9012 :invoke :read [4 nil] INFO [2019-12-17 12:34:34,471] jepsen worker 29 - jepsen.util 6959 :ok :write [9 47] INFO [2019-12-17 12:34:34,471] jepsen worker 29 - jepsen.util 6959 :invoke :cas [9 [11 36]] INFO [2019-12-17 12:34:34,473] jepsen worker 64 - jepsen.util 6994 :fail :cas [21 [29 12]] :ValueNotSwapFrom INFO [2019-12-17 12:34:34,473] jepsen worker 64 - jepsen.util 6994 :invoke :read [21 nil] INFO [2019-12-17 12:34:34,480] jepsen worker 36 - jepsen.util 8676 :fail :cas [12 [34 1]] :ValueNotSwapFrom INFO [2019-12-17 12:34:34,480] jepsen worker 36 - jepsen.util 8676 :invoke :write [12 24] INFO [2019-12-17 12:34:34,488] jepsen worker 10 - jepsen.util 8830 :fail :cas [3 [0 3]] :ValueNotSwapFrom INFO [2019-12-17 12:34:34,488] jepsen worker 10 - jepsen.util 8830 :invoke :cas [3 [10 15]] INFO [2019-12-17 12:34:34,496] jepsen worker 79 - jepsen.util 8809 :fail :cas [26 [7 35]] :ValueNotSwapFrom INFO [2019-12-17 12:34:34,497] jepsen worker 79 - jepsen.util 8809 :invoke :cas [26 [43 26]] INFO [2019-12-17 12:34:34,514] jepsen worker 81 - jepsen.util 11421 :info :write [27 17] :RequestCanceledException INFO [2019-12-17 12:34:34,515] jepsen worker 81 - jepsen.util 11511 :invoke :read [27 nil] INFO [2019-12-17 12:34:34,519] jepsen worker 56 - jepsen.util 9326 :ok :write [18 17] INFO [2019-12-17 12:34:34,519] jepsen worker 56 - jepsen.util 9326 :invoke :cas [18 [0 31]] INFO [2019-12-17 12:34:34,522] jepsen worker 64 - jepsen.util 6994 :ok :read [21 42] INFO [2019-12-17 12:34:34,522] jepsen worker 64 - jepsen.util 6994 :invoke :write [21 42] INFO [2019-12-17 12:34:34,524] jepsen worker 29 - jepsen.util 6959 :fail :cas [9 [11 36]] :ValueNotSwapFrom INFO [2019-12-17 12:34:34,524] jepsen worker 29 - jepsen.util 6959 :invoke :cas [9 [19 7]] INFO [2019-12-17 12:34:34,530] jepsen worker 86 - jepsen.util 12416 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:34:34,530] jepsen worker 86 - jepsen.util 12416 :invoke :cas [28 [35 3]] INFO [2019-12-17 12:34:34,544] jepsen worker 41 - jepsen.util 12191 :info :write [13 6] :RequestCanceledException INFO [2019-12-17 12:34:34,545] jepsen worker 41 - jepsen.util 12281 :invoke :cas [13 [22 24]] INFO [2019-12-17 12:34:34,561] jepsen worker 74 - jepsen.util 8624 :info :write [24 26] :RequestCanceledException INFO [2019-12-17 12:34:34,561] jepsen worker 59 - jepsen.util 11129 :info :cas [19 [13 12]] :RequestCanceledException INFO [2019-12-17 12:34:34,561] jepsen worker 30 - jepsen.util 8400 :info :cas [10 [4 35]] :RequestCanceledException INFO [2019-12-17 12:34:34,562] jepsen worker 59 - jepsen.util 11219 :invoke :read [19 nil] INFO [2019-12-17 12:34:34,562] jepsen worker 30 - jepsen.util 8490 :invoke :cas [10 [49 9]] INFO [2019-12-17 12:34:34,566] jepsen worker 74 - jepsen.util 8714 :invoke :read [24 nil] INFO [2019-12-17 12:34:34,573] jepsen worker 69 - jepsen.util 8619 :fail :cas [23 [8 45]] :ValueNotSwapFrom INFO [2019-12-17 12:34:34,573] jepsen worker 69 - jepsen.util 8619 :invoke :write [23 9] INFO [2019-12-17 12:34:34,576] jepsen worker 29 - jepsen.util 6959 :fail :cas [9 [19 7]] :ValueNotSwapFrom INFO [2019-12-17 12:34:34,576] jepsen worker 29 - jepsen.util 6959 :invoke :read [9 nil] INFO [2019-12-17 12:34:34,589] jepsen worker 56 - jepsen.util 9326 :fail :cas [18 [0 31]] :ValueNotSwapFrom INFO [2019-12-17 12:34:34,589] jepsen worker 56 - jepsen.util 9326 :invoke :read [18 nil] INFO [2019-12-17 12:34:34,597] jepsen worker 21 - jepsen.util 11001 :info :write [7 15] :RequestCanceledException INFO [2019-12-17 12:34:34,597] jepsen worker 21 - jepsen.util 11091 :invoke :write [7 10] INFO [2019-12-17 12:34:34,611] jepsen worker 74 - jepsen.util 8714 :ok :read [24 22] INFO [2019-12-17 12:34:34,613] jepsen worker 74 - jepsen.util 8714 :invoke :cas [24 [4 9]] INFO [2019-12-17 12:34:34,617] jepsen worker 12 - jepsen.util 9012 :ok :read [4 19] INFO [2019-12-17 12:34:34,617] jepsen worker 12 - jepsen.util 9012 :invoke :cas [4 [8 12]] INFO [2019-12-17 12:34:34,630] jepsen worker 39 - jepsen.util 11019 :info :cas [13 [27 43]] :RequestCanceledException INFO [2019-12-17 12:34:34,631] jepsen worker 39 - jepsen.util 11109 :invoke :read [13 nil] INFO [2019-12-17 12:34:34,641] jepsen worker 29 - jepsen.util 6959 :ok :read [9 47] INFO [2019-12-17 12:34:34,641] jepsen worker 29 - jepsen.util 6959 :invoke :cas [9 [15 10]] INFO [2019-12-17 12:34:34,644] jepsen worker 68 - jepsen.util 10868 :info :cas [22 [13 43]] :RequestCanceledException INFO [2019-12-17 12:34:34,647] jepsen worker 68 - jepsen.util 10958 :invoke :write [22 37] INFO [2019-12-17 12:34:34,648] jepsen worker 56 - jepsen.util 9326 :ok :read [18 17] INFO [2019-12-17 12:34:34,648] jepsen worker 56 - jepsen.util 9326 :invoke :read [18 nil] INFO [2019-12-17 12:34:34,664] jepsen worker 80 - jepsen.util 8180 :fail :cas [26 [35 47]] :ValueNotSwapFrom INFO [2019-12-17 12:34:34,664] jepsen worker 80 - jepsen.util 8180 :invoke :write [26 10] INFO [2019-12-17 12:34:34,669] jepsen worker 74 - jepsen.util 8714 :fail :cas [24 [4 9]] :ValueNotSwapFrom INFO [2019-12-17 12:34:34,669] jepsen worker 74 - jepsen.util 8714 :invoke :write [24 16] INFO [2019-12-17 12:34:34,671] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":49106,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:34:34,672] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":49106,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:34:34,672] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":49110,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:34:34,672] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":49110,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:34:34,704] jepsen worker 64 - jepsen.util 6994 :ok :write [21 42] INFO [2019-12-17 12:34:34,705] jepsen worker 64 - jepsen.util 6994 :invoke :cas [21 [15 20]] INFO [2019-12-17 12:34:34,709] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:34:34,709] jepsen nemesis - couchbase.util Rebalance check count: 380 INFO [2019-12-17 12:34:34,709] jepsen nemesis - couchbase.util Rebalance stuck count: 361 INFO [2019-12-17 12:34:34,710] jepsen worker 72 - jepsen.util 8442 :ok :write [24 14] INFO [2019-12-17 12:34:34,710] jepsen worker 72 - jepsen.util 8442 :invoke :write [24 7] INFO [2019-12-17 12:34:34,710] jepsen worker 18 - jepsen.util 8658 :ok :write [6 26] INFO [2019-12-17 12:34:34,711] jepsen worker 18 - jepsen.util 8658 :invoke :read [6 nil] INFO [2019-12-17 12:34:34,715] jepsen worker 29 - jepsen.util 6959 :fail :cas [9 [15 10]] :ValueNotSwapFrom INFO [2019-12-17 12:34:34,715] jepsen worker 29 - jepsen.util 6959 :invoke :read [9 nil] INFO [2019-12-17 12:34:34,729] jepsen worker 76 - jepsen.util 11596 :info :cas [25 [30 11]] :RequestCanceledException INFO [2019-12-17 12:34:34,729] jepsen worker 82 - jepsen.util 12142 :info :write [27 0] :RequestCanceledException INFO [2019-12-17 12:34:34,729] jepsen worker 50 - jepsen.util 12200 :info :write [16 22] :RequestCanceledException INFO [2019-12-17 12:34:34,729] jepsen worker 23 - jepsen.util 11183 :info :cas [7 [39 15]] :RequestCanceledException INFO [2019-12-17 12:34:34,730] jepsen worker 50 - jepsen.util 12290 :invoke :cas [16 [20 46]] INFO [2019-12-17 12:34:34,730] jepsen worker 76 - jepsen.util 11686 :invoke :cas [25 [3 15]] INFO [2019-12-17 12:34:34,730] jepsen worker 82 - jepsen.util 12232 :invoke :cas [27 [25 43]] INFO [2019-12-17 12:34:34,730] jepsen worker 23 - jepsen.util 11273 :invoke :write [7 25] INFO [2019-12-17 12:34:34,736] jepsen worker 5 - jepsen.util 7925 :ok :read [1 49] INFO [2019-12-17 12:34:34,736] jepsen worker 5 - jepsen.util 7925 :invoke :cas [1 [49 47]] INFO [2019-12-17 12:34:34,736] jepsen worker 11 - jepsen.util 8381 :ok :write [3 8] INFO [2019-12-17 12:34:34,736] jepsen worker 11 - jepsen.util 8381 :invoke :write [3 11] INFO [2019-12-17 12:34:34,747] jepsen worker 51 - jepsen.util 7881 :fail :cas [17 [43 22]] :ValueNotSwapFrom INFO [2019-12-17 12:34:34,747] jepsen worker 51 - jepsen.util 7881 :invoke :cas [17 [39 29]] INFO [2019-12-17 12:34:34,766] jepsen worker 47 - jepsen.util 8957 :ok :read [15 3] INFO [2019-12-17 12:34:34,766] jepsen worker 47 - jepsen.util 8957 :invoke :read [15 nil] INFO [2019-12-17 12:34:34,782] jepsen worker 64 - jepsen.util 6994 :fail :cas [21 [15 20]] :ValueNotSwapFrom INFO [2019-12-17 12:34:34,782] jepsen worker 64 - jepsen.util 6994 :invoke :write [21 33] INFO [2019-12-17 12:34:34,795] jepsen worker 60 - jepsen.util 12570 :info :write [20 45] :RequestCanceledException INFO [2019-12-17 12:34:34,796] jepsen worker 60 - jepsen.util 12660 :invoke :read [20 nil] INFO [2019-12-17 12:34:34,814] jepsen worker 83 - jepsen.util 11963 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:34:34,814] jepsen worker 83 - jepsen.util 11963 :invoke :write [27 5] INFO [2019-12-17 12:34:34,824] jepsen worker 63 - jepsen.util 7353 :ok :read [21 42] INFO [2019-12-17 12:34:34,824] jepsen worker 63 - jepsen.util 7353 :invoke :read [21 nil] INFO [2019-12-17 12:34:34,827] jepsen worker 16 - jepsen.util 12526 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:34:34,828] jepsen worker 16 - jepsen.util 12526 :invoke :write [5 28] INFO [2019-12-17 12:34:34,834] jepsen worker 10 - jepsen.util 8830 :fail :cas [3 [10 15]] :ValueNotSwapFrom INFO [2019-12-17 12:34:34,834] jepsen worker 10 - jepsen.util 8830 :invoke :write [3 45] INFO [2019-12-17 12:34:34,844] jepsen worker 34 - jepsen.util 11464 :info :write [11 37] :RequestCanceledException INFO [2019-12-17 12:34:34,845] jepsen worker 34 - jepsen.util 11554 :invoke :write [11 43] INFO [2019-12-17 12:34:34,863] jepsen worker 56 - jepsen.util 9326 :ok :read [18 17] INFO [2019-12-17 12:34:34,863] jepsen worker 56 - jepsen.util 9326 :invoke :cas [18 [1 33]] INFO [2019-12-17 12:34:34,889] jepsen worker 30 - jepsen.util 8490 :fail :cas [10 [49 9]] :ValueNotSwapFrom INFO [2019-12-17 12:34:34,889] jepsen worker 30 - jepsen.util 8490 :invoke :cas [10 [32 16]] INFO [2019-12-17 12:34:34,891] jepsen worker 18 - jepsen.util 8658 :ok :read [6 26] INFO [2019-12-17 12:34:34,891] jepsen worker 18 - jepsen.util 8658 :invoke :write [6 5] INFO [2019-12-17 12:34:34,895] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":49136,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:34:34,896] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":49136,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:34:34,896] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":49139,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:34:34,896] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":49139,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:34:34,896] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":49140,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:34:34,896] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":49140,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:34:34,897] jepsen worker 79 - jepsen.util 8809 :fail :cas [26 [43 26]] :ValueNotSwapFrom INFO [2019-12-17 12:34:34,897] jepsen worker 79 - jepsen.util 8809 :invoke :read [26 nil] INFO [2019-12-17 12:34:34,929] jepsen worker 67 - jepsen.util 11677 :info :cas [22 [43 18]] :RequestCanceledException INFO [2019-12-17 12:34:34,931] jepsen worker 67 - jepsen.util 11767 :invoke :read [22 nil] INFO [2019-12-17 12:34:34,934] jepsen worker 63 - jepsen.util 7353 :ok :read [21 42] INFO [2019-12-17 12:34:34,934] jepsen worker 63 - jepsen.util 7353 :invoke :read [21 nil] INFO [2019-12-17 12:34:34,959] jepsen worker 30 - jepsen.util 8490 :fail :cas [10 [32 16]] :ValueNotSwapFrom INFO [2019-12-17 12:34:34,959] jepsen worker 30 - jepsen.util 8490 :invoke :cas [10 [18 33]] INFO [2019-12-17 12:34:34,963] jepsen worker 12 - jepsen.util 9012 :fail :cas [4 [8 12]] :ValueNotSwapFrom INFO [2019-12-17 12:34:34,963] jepsen worker 12 - jepsen.util 9012 :invoke :read [4 nil] INFO [2019-12-17 12:34:34,965] jepsen worker 64 - jepsen.util 6994 :ok :write [21 33] INFO [2019-12-17 12:34:34,966] jepsen worker 64 - jepsen.util 6994 :invoke :read [21 nil] INFO [2019-12-17 12:34:34,975] jepsen worker 20 - jepsen.util 8210 :ok :write [6 49] INFO [2019-12-17 12:34:34,976] jepsen worker 20 - jepsen.util 8210 :invoke :cas [6 [24 25]] INFO [2019-12-17 12:34:34,984] jepsen worker 63 - jepsen.util 7353 :ok :read [21 42] INFO [2019-12-17 12:34:34,984] jepsen worker 63 - jepsen.util 7353 :invoke :write [21 31] INFO [2019-12-17 12:34:34,984] jepsen worker 64 - jepsen.util 6994 :ok :read [21 33] INFO [2019-12-17 12:34:34,984] jepsen worker 64 - jepsen.util 6994 :invoke :read [21 nil] INFO [2019-12-17 12:34:34,988] jepsen worker 56 - jepsen.util 9326 :fail :cas [18 [1 33]] :ValueNotSwapFrom INFO [2019-12-17 12:34:34,988] jepsen worker 56 - jepsen.util 9326 :invoke :cas [18 [11 23]] INFO [2019-12-17 12:34:34,990] jepsen worker 31 - jepsen.util 7501 :ok :write [10 45] INFO [2019-12-17 12:34:34,990] jepsen worker 31 - jepsen.util 7501 :invoke :cas [10 [2 30]] INFO [2019-12-17 12:34:34,997] jepsen worker 4 - jepsen.util 8014 :info :write [1 29] :RequestCanceledException INFO [2019-12-17 12:34:34,998] jepsen worker 4 - jepsen.util 8104 :invoke :read [1 nil] INFO [2019-12-17 12:34:35,000] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":49155,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:34:35,000] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":49155,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:34:35,000] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":49159,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:34:35,000] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":49159,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:34:35,001] jepsen worker 47 - jepsen.util 8957 :ok :read [15 3] INFO [2019-12-17 12:34:35,001] jepsen worker 47 - jepsen.util 8957 :invoke :cas [15 [9 36]] INFO [2019-12-17 12:34:35,026] jepsen worker 64 - jepsen.util 6994 :ok :read [21 33] INFO [2019-12-17 12:34:35,026] jepsen worker 64 - jepsen.util 6994 :invoke :cas [21 [32 28]] INFO [2019-12-17 12:34:35,027] jepsen worker 32 - jepsen.util 8762 :fail :cas [10 [36 2]] :ValueNotSwapFrom INFO [2019-12-17 12:34:35,027] jepsen worker 32 - jepsen.util 8762 :invoke :cas [10 [12 16]] INFO [2019-12-17 12:34:35,035] jepsen worker 79 - jepsen.util 8809 :ok :read [26 40] INFO [2019-12-17 12:34:35,035] jepsen worker 79 - jepsen.util 8809 :invoke :read [26 nil] INFO [2019-12-17 12:34:35,063] jepsen worker 56 - jepsen.util 9326 :fail :cas [18 [11 23]] :ValueNotSwapFrom INFO [2019-12-17 12:34:35,064] jepsen worker 56 - jepsen.util 9326 :invoke :write [18 14] INFO [2019-12-17 12:34:35,088] jepsen worker 12 - jepsen.util 9012 :ok :read [4 19] INFO [2019-12-17 12:34:35,088] jepsen worker 12 - jepsen.util 9012 :invoke :write [4 42] INFO [2019-12-17 12:34:35,129] jepsen worker 32 - jepsen.util 8762 :fail :cas [10 [12 16]] :ValueNotSwapFrom INFO [2019-12-17 12:34:35,129] jepsen worker 35 - jepsen.util 11195 :info :cas [11 [33 29]] :RequestCanceledException INFO [2019-12-17 12:34:35,129] jepsen worker 32 - jepsen.util 8762 :invoke :read [10 nil] INFO [2019-12-17 12:34:35,131] jepsen worker 35 - jepsen.util 11285 :invoke :cas [11 [15 48]] INFO [2019-12-17 12:34:35,132] jepsen worker 51 - jepsen.util 7881 :fail :cas [17 [39 29]] :ValueNotSwapFrom INFO [2019-12-17 12:34:35,132] jepsen worker 51 - jepsen.util 7881 :invoke :cas [17 [37 43]] INFO [2019-12-17 12:34:35,141] jepsen worker 64 - jepsen.util 6994 :fail :cas [21 [32 28]] :ValueNotSwapFrom INFO [2019-12-17 12:34:35,141] jepsen worker 64 - jepsen.util 6994 :invoke :read [21 nil] INFO [2019-12-17 12:34:35,148] jepsen worker 30 - jepsen.util 8490 :fail :cas [10 [18 33]] :ValueNotSwapFrom INFO [2019-12-17 12:34:35,148] jepsen worker 30 - jepsen.util 8490 :invoke :write [10 1] INFO [2019-12-17 12:34:35,156] jepsen worker 31 - jepsen.util 7501 :fail :cas [10 [2 30]] :ValueNotSwapFrom INFO [2019-12-17 12:34:35,156] jepsen worker 31 - jepsen.util 7501 :invoke :write [10 18] INFO [2019-12-17 12:34:35,159] jepsen worker 32 - jepsen.util 8762 :ok :read [10 45] INFO [2019-12-17 12:34:35,160] jepsen worker 32 - jepsen.util 8762 :invoke :write [10 28] INFO [2019-12-17 12:34:35,168] jepsen worker 64 - jepsen.util 6994 :ok :read [21 33] INFO [2019-12-17 12:34:35,168] jepsen worker 64 - jepsen.util 6994 :invoke :write [21 21] INFO [2019-12-17 12:34:35,183] jepsen worker 63 - jepsen.util 7353 :ok :write [21 31] INFO [2019-12-17 12:34:35,183] jepsen worker 63 - jepsen.util 7353 :invoke :write [21 47] INFO [2019-12-17 12:34:35,213] jepsen worker 27 - jepsen.util 8307 :fail :read [9 nil] :RequestTimeoutException INFO [2019-12-17 12:34:35,213] jepsen worker 27 - jepsen.util 8307 :invoke :write [9 21] INFO [2019-12-17 12:34:35,219] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":49186,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:34:35,219] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":49186,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:34:35,223] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":49178,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:34:35,223] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":49178,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:34:35,256] jepsen worker 64 - jepsen.util 6994 :ok :write [21 21] INFO [2019-12-17 12:34:35,256] jepsen worker 64 - jepsen.util 6994 :invoke :cas [21 [36 9]] INFO [2019-12-17 12:34:35,281] jepsen worker 64 - jepsen.util 6994 :fail :cas [21 [36 9]] :ValueNotSwapFrom INFO [2019-12-17 12:34:35,281] jepsen worker 64 - jepsen.util 6994 :invoke :read [21 nil] INFO [2019-12-17 12:34:35,291] jepsen worker 79 - jepsen.util 8809 :ok :read [26 40] INFO [2019-12-17 12:34:35,292] jepsen worker 79 - jepsen.util 8809 :invoke :write [26 26] INFO [2019-12-17 12:34:35,314] jepsen worker 33 - jepsen.util 11553 :info :write [11 23] :RequestCanceledException INFO [2019-12-17 12:34:35,315] jepsen worker 33 - jepsen.util 11643 :invoke :cas [11 [1 1]] INFO [2019-12-17 12:34:35,315] jepsen worker 64 - jepsen.util 6994 :ok :read [21 21] INFO [2019-12-17 12:34:35,316] jepsen worker 64 - jepsen.util 6994 :invoke :write [21 8] INFO [2019-12-17 12:34:35,316] jepsen worker 63 - jepsen.util 7353 :ok :write [21 47] INFO [2019-12-17 12:34:35,316] jepsen worker 63 - jepsen.util 7353 :invoke :cas [21 [14 13]] INFO [2019-12-17 12:34:35,323] jepsen worker 74 - jepsen.util 8714 :ok :write [24 16] INFO [2019-12-17 12:34:35,323] jepsen worker 74 - jepsen.util 8714 :invoke :cas [24 [9 36]] INFO [2019-12-17 12:34:35,323] jepsen worker 10 - jepsen.util 8830 :ok :write [3 45] INFO [2019-12-17 12:34:35,323] jepsen worker 10 - jepsen.util 8830 :invoke :write [3 7] INFO [2019-12-17 12:34:35,335] jepsen worker 63 - jepsen.util 7353 :fail :cas [21 [14 13]] :ValueNotSwapFrom INFO [2019-12-17 12:34:35,335] jepsen worker 63 - jepsen.util 7353 :invoke :read [21 nil] INFO [2019-12-17 12:34:35,372] jepsen worker 53 - jepsen.util 7793 :ok :write [17 42] INFO [2019-12-17 12:34:35,373] jepsen worker 53 - jepsen.util 7793 :invoke :cas [17 [28 34]] INFO [2019-12-17 12:34:35,381] jepsen worker 47 - jepsen.util 8957 :fail :cas [15 [9 36]] :ValueNotSwapFrom INFO [2019-12-17 12:34:35,382] jepsen worker 47 - jepsen.util 8957 :invoke :write [15 13] INFO [2019-12-17 12:34:35,385] jepsen worker 32 - jepsen.util 8762 :ok :write [10 28] INFO [2019-12-17 12:34:35,385] jepsen worker 32 - jepsen.util 8762 :invoke :read [10 nil] INFO [2019-12-17 12:34:35,389] jepsen worker 63 - jepsen.util 7353 :ok :read [21 47] INFO [2019-12-17 12:34:35,389] jepsen worker 63 - jepsen.util 7353 :invoke :cas [21 [32 31]] INFO [2019-12-17 12:34:35,405] jepsen worker 32 - jepsen.util 8762 :ok :read [10 28] INFO [2019-12-17 12:34:35,405] jepsen worker 32 - jepsen.util 8762 :invoke :write [10 14] INFO [2019-12-17 12:34:35,427] jepsen worker 57 - jepsen.util 11487 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:34:35,427] jepsen worker 62 - jepsen.util 11852 :info :write [20 1] :RequestCanceledException INFO [2019-12-17 12:34:35,427] jepsen worker 57 - jepsen.util 11487 :invoke :write [19 11] INFO [2019-12-17 12:34:35,428] jepsen worker 62 - jepsen.util 11942 :invoke :write [20 38] INFO [2019-12-17 12:34:35,439] jepsen worker 63 - jepsen.util 7353 :fail :cas [21 [32 31]] :ValueNotSwapFrom INFO [2019-12-17 12:34:35,439] jepsen worker 63 - jepsen.util 7353 :invoke :read [21 nil] INFO [2019-12-17 12:34:35,486] jepsen worker 63 - jepsen.util 7353 :ok :read [21 47] INFO [2019-12-17 12:34:35,486] jepsen worker 63 - jepsen.util 7353 :invoke :write [21 44] INFO [2019-12-17 12:34:35,488] jepsen worker 64 - jepsen.util 6994 :ok :write [21 8] INFO [2019-12-17 12:34:35,489] jepsen worker 64 - jepsen.util 6994 :invoke :write [21 33] INFO [2019-12-17 12:34:35,515] jepsen worker 25 - jepsen.util 12175 :info :write [8 2] :RequestCanceledException INFO [2019-12-17 12:34:35,516] jepsen worker 25 - jepsen.util 12265 :invoke :read [8 nil] INFO [2019-12-17 12:34:35,530] jepsen worker 3 - jepsen.util 8283 :info :write [1 14] :RequestCanceledException INFO [2019-12-17 12:34:35,531] jepsen worker 3 - jepsen.util 8373 :invoke :read [1 nil] INFO [2019-12-17 12:34:35,534] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":49207,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:34:35,535] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":49207,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:34:35,543] jepsen worker 0 - jepsen.util 11970 :info :cas [0 [47 3]] :RequestCanceledException INFO [2019-12-17 12:34:35,544] jepsen worker 0 - jepsen.util 12060 :invoke :write [0 48] INFO [2019-12-17 12:34:35,561] jepsen worker 27 - jepsen.util 8307 :ok :write [9 21] INFO [2019-12-17 12:34:35,561] jepsen worker 27 - jepsen.util 8307 :invoke :cas [9 [7 5]] INFO [2019-12-17 12:34:35,561] jepsen worker 58 - jepsen.util 12478 :info :write [19 44] :RequestCanceledException INFO [2019-12-17 12:34:35,562] jepsen worker 58 - jepsen.util 12568 :invoke :read [19 nil] INFO [2019-12-17 12:34:35,578] jepsen worker 69 - jepsen.util 8619 :ok :write [23 9] INFO [2019-12-17 12:34:35,578] jepsen worker 69 - jepsen.util 8619 :invoke :read [23 nil] INFO [2019-12-17 12:34:35,592] jepsen worker 27 - jepsen.util 8307 :fail :cas [9 [7 5]] :ValueNotSwapFrom INFO [2019-12-17 12:34:35,592] jepsen worker 27 - jepsen.util 8307 :invoke :read [9 nil] INFO [2019-12-17 12:34:35,611] jepsen worker 64 - jepsen.util 6994 :ok :write [21 33] INFO [2019-12-17 12:34:35,611] jepsen worker 64 - jepsen.util 6994 :invoke :write [21 42] INFO [2019-12-17 12:34:35,622] jepsen worker 27 - jepsen.util 8307 :ok :read [9 21] INFO [2019-12-17 12:34:35,622] jepsen worker 27 - jepsen.util 8307 :invoke :cas [9 [30 49]] INFO [2019-12-17 12:34:35,630] jepsen worker 85 - jepsen.util 12325 :info :write [28 18] :RequestCanceledException INFO [2019-12-17 12:34:35,630] jepsen worker 22 - jepsen.util 11542 :info :cas [7 [9 22]] :RequestCanceledException INFO [2019-12-17 12:34:35,633] jepsen worker 85 - jepsen.util 12415 :invoke :write [28 1] INFO [2019-12-17 12:34:35,634] jepsen worker 22 - jepsen.util 11632 :invoke :read [7 nil] INFO [2019-12-17 12:34:35,638] jepsen worker 10 - jepsen.util 8830 :ok :write [3 7] INFO [2019-12-17 12:34:35,638] jepsen worker 10 - jepsen.util 8830 :invoke :write [3 45] INFO [2019-12-17 12:34:35,643] jepsen worker 48 - jepsen.util 10848 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:34:35,643] jepsen worker 48 - jepsen.util 10848 :invoke :read [16 nil] INFO [2019-12-17 12:34:35,651] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":49231,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:34:35,651] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":49231,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:34:35,651] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":49233,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:34:35,651] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":49233,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:34:35,653] jepsen worker 32 - jepsen.util 8762 :ok :write [10 14] INFO [2019-12-17 12:34:35,653] jepsen worker 32 - jepsen.util 8762 :invoke :cas [10 [16 30]] INFO [2019-12-17 12:34:35,655] jepsen worker 27 - jepsen.util 8307 :fail :cas [9 [30 49]] :ValueNotSwapFrom INFO [2019-12-17 12:34:35,655] jepsen worker 27 - jepsen.util 8307 :invoke :read [9 nil] INFO [2019-12-17 12:34:35,685] jepsen worker 32 - jepsen.util 8762 :fail :cas [10 [16 30]] :ValueNotSwapFrom INFO [2019-12-17 12:34:35,686] jepsen worker 32 - jepsen.util 8762 :invoke :cas [10 [47 7]] INFO [2019-12-17 12:34:35,692] jepsen worker 28 - jepsen.util 6688 :info :write [9 48] :RequestCanceledException INFO [2019-12-17 12:34:35,692] jepsen worker 45 - jepsen.util 8865 :fail :read [15 nil] :RequestTimeoutException INFO [2019-12-17 12:34:35,692] jepsen worker 45 - jepsen.util 8865 :invoke :write [15 45] INFO [2019-12-17 12:34:35,693] jepsen worker 28 - jepsen.util 6778 :invoke :write [9 21] INFO [2019-12-17 12:34:35,704] jepsen worker 27 - jepsen.util 8307 :ok :read [9 21] INFO [2019-12-17 12:34:35,704] jepsen worker 27 - jepsen.util 8307 :invoke :write [9 45] INFO [2019-12-17 12:34:35,706] jepsen worker 4 - jepsen.util 8104 :ok :read [1 49] INFO [2019-12-17 12:34:35,707] jepsen worker 4 - jepsen.util 8104 :invoke :cas [1 [4 41]] INFO [2019-12-17 12:34:35,712] jepsen worker 26 - jepsen.util 11996 :info :write [8 4] :RequestCanceledException INFO [2019-12-17 12:34:35,713] jepsen worker 26 - jepsen.util 12086 :invoke :write [8 35] INFO [2019-12-17 12:34:35,746] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:34:35,746] jepsen nemesis - couchbase.util Rebalance check count: 381 INFO [2019-12-17 12:34:35,746] jepsen nemesis - couchbase.util Rebalance stuck count: 362 INFO [2019-12-17 12:34:35,754] jepsen worker 64 - jepsen.util 6994 :ok :write [21 42] INFO [2019-12-17 12:34:35,754] jepsen worker 64 - jepsen.util 6994 :invoke :cas [21 [30 15]] INFO [2019-12-17 12:34:35,757] jepsen worker 6 - jepsen.util 12066 :info :write [2 6] :RequestCanceledException INFO [2019-12-17 12:34:35,757] jepsen worker 32 - jepsen.util 8762 :fail :cas [10 [47 7]] :ValueNotSwapFrom INFO [2019-12-17 12:34:35,757] jepsen worker 32 - jepsen.util 8762 :invoke :cas [10 [9 13]] INFO [2019-12-17 12:34:35,758] jepsen worker 6 - jepsen.util 12156 :invoke :read [2 nil] INFO [2019-12-17 12:34:35,791] jepsen worker 61 - jepsen.util 12301 :info :write [20 7] :RequestCanceledException INFO [2019-12-17 12:34:35,792] jepsen worker 61 - jepsen.util 12391 :invoke :read [20 nil] INFO [2019-12-17 12:34:35,801] jepsen worker 64 - jepsen.util 6994 :fail :cas [21 [30 15]] :ValueNotSwapFrom INFO [2019-12-17 12:34:35,801] jepsen worker 64 - jepsen.util 6994 :invoke :cas [21 [9 4]] INFO [2019-12-17 12:34:35,814] jepsen worker 66 - jepsen.util 11406 :info :cas [22 [47 24]] :RequestCanceledException INFO [2019-12-17 12:34:35,814] jepsen worker 44 - jepsen.util 11834 :info :cas [14 [38 23]] :RequestCanceledException INFO [2019-12-17 12:34:35,815] jepsen worker 66 - jepsen.util 11496 :invoke :read [22 nil] INFO [2019-12-17 12:34:35,816] jepsen worker 44 - jepsen.util 11924 :invoke :cas [14 [5 14]] INFO [2019-12-17 12:34:35,819] jepsen worker 32 - jepsen.util 8762 :fail :cas [10 [9 13]] :ValueNotSwapFrom INFO [2019-12-17 12:34:35,820] jepsen worker 32 - jepsen.util 8762 :invoke :cas [10 [28 6]] INFO [2019-12-17 12:34:35,829] jepsen worker 84 - jepsen.util 11064 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:34:35,829] jepsen worker 84 - jepsen.util 11064 :invoke :read [28 nil] INFO [2019-12-17 12:34:35,835] jepsen worker 47 - jepsen.util 8957 :ok :write [15 13] INFO [2019-12-17 12:34:35,835] jepsen worker 47 - jepsen.util 8957 :invoke :write [15 47] INFO [2019-12-17 12:34:35,861] jepsen worker 71 - jepsen.util 8981 :info :write [23 41] :RequestCanceledException INFO [2019-12-17 12:34:35,862] jepsen worker 71 - jepsen.util 9071 :invoke :cas [23 [4 48]] INFO [2019-12-17 12:34:35,868] jepsen worker 53 - jepsen.util 7793 :fail :cas [17 [28 34]] :ValueNotSwapFrom INFO [2019-12-17 12:34:35,868] jepsen worker 53 - jepsen.util 7793 :invoke :cas [17 [35 36]] INFO [2019-12-17 12:34:35,870] jepsen worker 64 - jepsen.util 6994 :fail :cas [21 [9 4]] :ValueNotSwapFrom INFO [2019-12-17 12:34:35,870] jepsen worker 64 - jepsen.util 6994 :invoke :write [21 19] INFO [2019-12-17 12:34:35,872] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":49255,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:34:35,873] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":49255,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:34:35,873] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":49257,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:34:35,873] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":49257,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:34:35,873] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":49258,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:34:35,876] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":49258,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:34:35,883] jepsen worker 32 - jepsen.util 8762 :fail :cas [10 [28 6]] :ValueNotSwapFrom INFO [2019-12-17 12:34:35,883] jepsen worker 32 - jepsen.util 8762 :invoke :read [10 nil] INFO [2019-12-17 12:34:35,896] jepsen worker 54 - jepsen.util 9414 :info :write [18 45] :RequestCanceledException INFO [2019-12-17 12:34:35,901] jepsen worker 54 - jepsen.util 9504 :invoke :read [18 nil] INFO [2019-12-17 12:34:35,929] jepsen worker 9 - jepsen.util 8919 :fail :read [3 nil] :RequestTimeoutException INFO [2019-12-17 12:34:35,929] jepsen worker 9 - jepsen.util 8919 :invoke :read [3 nil] INFO [2019-12-17 12:34:35,934] jepsen worker 32 - jepsen.util 8762 :ok :read [10 14] INFO [2019-12-17 12:34:35,934] jepsen worker 32 - jepsen.util 8762 :invoke :cas [10 [39 35]] INFO [2019-12-17 12:34:35,944] jepsen worker 70 - jepsen.util 8890 :info :write [23 33] :RequestCanceledException INFO [2019-12-17 12:34:35,945] jepsen worker 27 - jepsen.util 8307 :ok :write [9 45] INFO [2019-12-17 12:34:35,945] jepsen worker 27 - jepsen.util 8307 :invoke :read [9 nil] INFO [2019-12-17 12:34:35,948] jepsen worker 70 - jepsen.util 8980 :invoke :write [23 3] INFO [2019-12-17 12:34:35,961] jepsen worker 24 - jepsen.util 10644 :info :cas [8 [29 47]] :RequestCanceledException INFO [2019-12-17 12:34:35,962] jepsen worker 24 - jepsen.util 10734 :invoke :write [8 27] INFO [2019-12-17 12:34:35,969] jepsen worker 51 - jepsen.util 7881 :fail :cas [17 [37 43]] :ValueNotSwapFrom INFO [2019-12-17 12:34:35,969] jepsen worker 51 - jepsen.util 7881 :invoke :cas [17 [28 12]] INFO [2019-12-17 12:34:35,974] jepsen worker 9 - jepsen.util 8919 :ok :read [3 7] INFO [2019-12-17 12:34:35,974] jepsen worker 9 - jepsen.util 8919 :invoke :cas [3 [24 3]] INFO [2019-12-17 12:34:35,988] jepsen worker 27 - jepsen.util 8307 :ok :read [9 45] INFO [2019-12-17 12:34:35,988] jepsen worker 27 - jepsen.util 8307 :invoke :write [9 44] INFO [2019-12-17 12:34:35,993] jepsen worker 32 - jepsen.util 8762 :fail :cas [10 [39 35]] :ValueNotSwapFrom INFO [2019-12-17 12:34:35,993] jepsen worker 32 - jepsen.util 8762 :invoke :write [10 23] INFO [2019-12-17 12:34:35,996] jepsen worker 38 - jepsen.util 8768 :info :write [12 30] :RequestCanceledException INFO [2019-12-17 12:34:35,996] jepsen worker 40 - jepsen.util 11650 :info :cas [13 [35 15]] :RequestCanceledException INFO [2019-12-17 12:34:35,997] jepsen worker 38 - jepsen.util 8858 :invoke :read [12 nil] INFO [2019-12-17 12:34:35,998] jepsen worker 40 - jepsen.util 11740 :invoke :write [13 12] INFO [2019-12-17 12:34:36,005] jepsen worker 64 - jepsen.util 6994 :ok :write [21 19] INFO [2019-12-17 12:34:36,005] jepsen worker 64 - jepsen.util 6994 :invoke :write [21 25] INFO [2019-12-17 12:34:36,007] jepsen worker 69 - jepsen.util 8619 :ok :read [23 9] INFO [2019-12-17 12:34:36,007] jepsen worker 69 - jepsen.util 8619 :invoke :cas [23 [38 32]] INFO [2019-12-17 12:34:36,022] jepsen worker 4 - jepsen.util 8104 :fail :cas [1 [4 41]] :ValueNotSwapFrom INFO [2019-12-17 12:34:36,022] jepsen worker 4 - jepsen.util 8104 :invoke :read [1 nil] INFO [2019-12-17 12:34:36,028] jepsen worker 17 - jepsen.util 11537 :info :cas [5 [30 13]] :RequestCanceledException INFO [2019-12-17 12:34:36,028] jepsen worker 17 - jepsen.util 11627 :invoke :cas [5 [35 37]] INFO [2019-12-17 12:34:36,036] jepsen worker 69 - jepsen.util 8619 :fail :cas [23 [38 32]] :ValueNotSwapFrom INFO [2019-12-17 12:34:36,037] jepsen worker 69 - jepsen.util 8619 :invoke :write [23 14] INFO [2019-12-17 12:34:36,043] jepsen worker 1 - jepsen.util 11791 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:34:36,043] jepsen worker 1 - jepsen.util 11791 :invoke :read [0 nil] INFO [2019-12-17 12:34:36,049] jepsen worker 9 - jepsen.util 8919 :fail :cas [3 [24 3]] :ValueNotSwapFrom INFO [2019-12-17 12:34:36,049] jepsen worker 9 - jepsen.util 8919 :invoke :write [3 23] INFO [2019-12-17 12:34:36,053] jepsen worker 4 - jepsen.util 8104 :ok :read [1 49] INFO [2019-12-17 12:34:36,053] jepsen worker 4 - jepsen.util 8104 :invoke :write [1 14] INFO [2019-12-17 12:34:36,061] jepsen worker 52 - jepsen.util 8602 :info :write [17 35] :RequestCanceledException INFO [2019-12-17 12:34:36,061] jepsen worker 42 - jepsen.util 12642 :info :cas [14 [43 9]] :RequestCanceledException INFO [2019-12-17 12:34:36,062] jepsen worker 42 - jepsen.util 12732 :invoke :read [14 nil] INFO [2019-12-17 12:34:36,062] jepsen worker 52 - jepsen.util 8692 :invoke :cas [17 [41 22]] INFO [2019-12-17 12:34:36,085] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":49272,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:34:36,086] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":49272,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:34:36,096] jepsen worker 15 - jepsen.util 10815 :info :write [5 0] :RequestCanceledException INFO [2019-12-17 12:34:36,096] jepsen worker 13 - jepsen.util 8113 :info :write [4 33] :RequestCanceledException INFO [2019-12-17 12:34:36,097] jepsen worker 15 - jepsen.util 10905 :invoke :read [5 nil] INFO [2019-12-17 12:34:36,100] jepsen worker 13 - jepsen.util 8203 :invoke :cas [4 [36 10]] INFO [2019-12-17 12:34:36,113] jepsen worker 87 - jepsen.util 7917 :info :write [29 40] :RequestCanceledException INFO [2019-12-17 12:34:36,113] jepsen worker 75 - jepsen.util 11055 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:34:36,114] jepsen worker 75 - jepsen.util 11055 :invoke :cas [25 [0 28]] INFO [2019-12-17 12:34:36,117] jepsen worker 87 - jepsen.util 8007 :invoke :write [29 43] INFO [2019-12-17 12:34:36,124] jepsen worker 27 - jepsen.util 8307 :ok :write [9 44] INFO [2019-12-17 12:34:36,125] jepsen worker 27 - jepsen.util 8307 :invoke :read [9 nil] INFO [2019-12-17 12:34:36,129] jepsen worker 8 - jepsen.util 11438 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:34:36,130] jepsen worker 8 - jepsen.util 11438 :invoke :cas [2 [6 7]] INFO [2019-12-17 12:34:36,143] jepsen worker 43 - jepsen.util 11203 :info :cas [14 [10 11]] :RequestCanceledException INFO [2019-12-17 12:34:36,143] jepsen worker 77 - jepsen.util 11777 :info :cas [25 [11 24]] :RequestCanceledException INFO [2019-12-17 12:34:36,144] jepsen worker 43 - jepsen.util 11293 :invoke :cas [14 [32 35]] INFO [2019-12-17 12:34:36,144] jepsen worker 77 - jepsen.util 11867 :invoke :cas [25 [16 46]] INFO [2019-12-17 12:34:36,150] jepsen worker 13 - jepsen.util 8203 :fail :cas [4 [36 10]] :ValueNotSwapFrom INFO [2019-12-17 12:34:36,151] jepsen worker 13 - jepsen.util 8203 :invoke :write [4 25] INFO [2019-12-17 12:34:36,165] jepsen worker 27 - jepsen.util 8307 :ok :read [9 44] INFO [2019-12-17 12:34:36,165] jepsen worker 27 - jepsen.util 8307 :invoke :write [9 13] INFO [2019-12-17 12:34:36,170] jepsen worker 64 - jepsen.util 6994 :ok :write [21 25] INFO [2019-12-17 12:34:36,170] jepsen worker 64 - jepsen.util 6994 :invoke :cas [21 [24 6]] INFO [2019-12-17 12:34:36,196] jepsen worker 37 - jepsen.util 8047 :info :write [12 47] :RequestCanceledException INFO [2019-12-17 12:34:36,196] jepsen worker 89 - jepsen.util 8459 :info :write [29 18] :RequestCanceledException INFO [2019-12-17 12:34:36,197] jepsen worker 37 - jepsen.util 8137 :invoke :read [12 nil] INFO [2019-12-17 12:34:36,197] jepsen worker 89 - jepsen.util 8549 :invoke :write [29 17] INFO [2019-12-17 12:34:36,207] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":49301,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:34:36,207] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":49301,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:34:36,207] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":49303,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:34:36,207] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":49303,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:34:36,213] jepsen worker 19 - jepsen.util 8659 :fail :read [6 nil] :RequestTimeoutException INFO [2019-12-17 12:34:36,213] jepsen worker 19 - jepsen.util 8659 :invoke :read [6 nil] INFO [2019-12-17 12:34:36,224] jepsen worker 64 - jepsen.util 6994 :fail :cas [21 [24 6]] :ValueNotSwapFrom INFO [2019-12-17 12:34:36,224] jepsen worker 64 - jepsen.util 6994 :invoke :cas [21 [12 48]] INFO [2019-12-17 12:34:36,229] jepsen worker 65 - jepsen.util 6995 :fail :read [21 nil] :RequestTimeoutException INFO [2019-12-17 12:34:36,229] jepsen worker 65 - jepsen.util 6995 :invoke :cas [21 [18 20]] INFO [2019-12-17 12:34:36,244] jepsen worker 88 - jepsen.util 8638 :info :write [29 42] :RequestCanceledException INFO [2019-12-17 12:34:36,244] jepsen worker 88 - jepsen.util 8728 :invoke :write [29 12] INFO [2019-12-17 12:34:36,246] jepsen worker 64 - jepsen.util 6994 :fail :cas [21 [12 48]] :ValueNotSwapFrom INFO [2019-12-17 12:34:36,246] jepsen worker 64 - jepsen.util 6994 :invoke :write [21 5] INFO [2019-12-17 12:34:36,256] jepsen worker 27 - jepsen.util 8307 :ok :write [9 13] INFO [2019-12-17 12:34:36,256] jepsen worker 27 - jepsen.util 8307 :invoke :read [9 nil] INFO [2019-12-17 12:34:36,259] jepsen worker 78 - jepsen.util 8268 :info :write [26 3] :RequestCanceledException INFO [2019-12-17 12:34:36,260] jepsen worker 78 - jepsen.util 8358 :invoke :cas [26 [33 27]] INFO [2019-12-17 12:34:36,261] jepsen worker 53 - jepsen.util 7793 :fail :cas [17 [35 36]] :ValueNotSwapFrom INFO [2019-12-17 12:34:36,261] jepsen worker 53 - jepsen.util 7793 :invoke :cas [17 [40 49]] INFO [2019-12-17 12:34:36,291] jepsen worker 19 - jepsen.util 8659 :ok :read [6 49] INFO [2019-12-17 12:34:36,291] jepsen worker 19 - jepsen.util 8659 :invoke :write [6 27] INFO [2019-12-17 12:34:36,304] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":49310,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:34:36,305] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":49310,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:34:36,305] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":49313,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:34:36,305] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":49313,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:34:36,305] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":49314,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:34:36,305] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":49314,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:34:36,306] jepsen worker 27 - jepsen.util 8307 :ok :read [9 13] INFO [2019-12-17 12:34:36,306] jepsen worker 27 - jepsen.util 8307 :invoke :read [9 nil] INFO [2019-12-17 12:34:36,337] jepsen worker 49 - jepsen.util 11029 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:34:36,337] jepsen worker 7 - jepsen.util 11797 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:34:36,337] jepsen worker 55 - jepsen.util 9235 :info :write [18 19] :RequestCanceledException INFO [2019-12-17 12:34:36,337] jepsen worker 49 - jepsen.util 11029 :invoke :read [16 nil] INFO [2019-12-17 12:34:36,337] jepsen worker 7 - jepsen.util 11797 :invoke :write [2 49] INFO [2019-12-17 12:34:36,339] jepsen worker 55 - jepsen.util 9325 :invoke :cas [18 [20 21]] INFO [2019-12-17 12:34:36,346] jepsen worker 3 - jepsen.util 8373 :ok :read [1 49] INFO [2019-12-17 12:34:36,346] jepsen worker 3 - jepsen.util 8373 :invoke :cas [1 [27 12]] INFO [2019-12-17 12:34:36,360] jepsen worker 64 - jepsen.util 6994 :ok :write [21 5] INFO [2019-12-17 12:34:36,361] jepsen worker 64 - jepsen.util 6994 :invoke :cas [21 [25 31]] INFO [2019-12-17 12:34:36,369] jepsen worker 71 - jepsen.util 9071 :fail :cas [23 [4 48]] :ValueNotSwapFrom INFO [2019-12-17 12:34:36,369] jepsen worker 71 - jepsen.util 9071 :invoke :write [23 47] INFO [2019-12-17 12:34:36,382] jepsen worker 64 - jepsen.util 6994 :fail :cas [21 [25 31]] :ValueNotSwapFrom INFO [2019-12-17 12:34:36,382] jepsen worker 64 - jepsen.util 6994 :invoke :cas [21 [29 3]] INFO [2019-12-17 12:34:36,395] jepsen worker 64 - jepsen.util 6994 :fail :cas [21 [29 3]] :ValueNotSwapFrom INFO [2019-12-17 12:34:36,395] jepsen worker 64 - jepsen.util 6994 :invoke :cas [21 [23 22]] INFO [2019-12-17 12:34:36,407] jepsen worker 64 - jepsen.util 6994 :fail :cas [21 [23 22]] :ValueNotSwapFrom INFO [2019-12-17 12:34:36,407] jepsen worker 64 - jepsen.util 6994 :invoke :write [21 44] INFO [2019-12-17 12:34:36,414] jepsen worker 14 - jepsen.util 8654 :info :write [4 17] :RequestCanceledException INFO [2019-12-17 12:34:36,414] jepsen worker 2 - jepsen.util 11252 :info :cas [0 [47 25]] :RequestCanceledException INFO [2019-12-17 12:34:36,415] jepsen worker 2 - jepsen.util 11342 :invoke :read [0 nil] INFO [2019-12-17 12:34:36,415] jepsen worker 14 - jepsen.util 8744 :invoke :cas [4 [21 42]] INFO [2019-12-17 12:34:36,425] jepsen worker 55 - jepsen.util 9325 :fail :cas [18 [20 21]] :ValueNotSwapFrom INFO [2019-12-17 12:34:36,425] jepsen worker 55 - jepsen.util 9325 :invoke :write [18 10] INFO [2019-12-17 12:34:36,429] jepsen worker 73 - jepsen.util 8173 :info :write [24 5] :RequestCanceledException INFO [2019-12-17 12:34:36,430] jepsen worker 73 - jepsen.util 8263 :invoke :write [24 29] INFO [2019-12-17 12:34:36,436] jepsen worker 51 - jepsen.util 7881 :fail :cas [17 [28 12]] :ValueNotSwapFrom INFO [2019-12-17 12:34:36,437] jepsen worker 51 - jepsen.util 7881 :invoke :read [17 nil] INFO [2019-12-17 12:34:36,448] jepsen worker 64 - jepsen.util 6994 :ok :write [21 44] INFO [2019-12-17 12:34:36,448] jepsen worker 64 - jepsen.util 6994 :invoke :read [21 nil] INFO [2019-12-17 12:34:36,460] jepsen worker 46 - jepsen.util 8416 :info :cas [15 [31 19]] :RequestCanceledException INFO [2019-12-17 12:34:36,462] jepsen worker 46 - jepsen.util 8506 :invoke :write [15 27] INFO [2019-12-17 12:34:36,464] jepsen worker 64 - jepsen.util 6994 :ok :read [21 44] INFO [2019-12-17 12:34:36,464] jepsen worker 64 - jepsen.util 6994 :invoke :cas [21 [14 28]] INFO [2019-12-17 12:34:36,487] jepsen worker 64 - jepsen.util 6994 :fail :cas [21 [14 28]] :ValueNotSwapFrom INFO [2019-12-17 12:34:36,487] jepsen worker 64 - jepsen.util 6994 :invoke :write [21 11] INFO [2019-12-17 12:34:36,519] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":49338,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:34:36,519] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":49338,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:34:36,519] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":49342,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:34:36,519] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":49342,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:34:36,530] jepsen worker 36 - jepsen.util 8676 :info :write [12 24] :RequestCanceledException INFO [2019-12-17 12:34:36,531] jepsen worker 36 - jepsen.util 8766 :invoke :cas [12 [30 31]] INFO [2019-12-17 12:34:36,537] jepsen worker 54 - jepsen.util 9504 :ok :read [18 17] INFO [2019-12-17 12:34:36,537] jepsen worker 54 - jepsen.util 9504 :invoke :cas [18 [39 26]] INFO [2019-12-17 12:34:36,542] jepsen worker 64 - jepsen.util 6994 :ok :write [21 11] INFO [2019-12-17 12:34:36,542] jepsen worker 64 - jepsen.util 6994 :invoke :cas [21 [37 24]] INFO [2019-12-17 12:34:36,543] jepsen worker 81 - jepsen.util 11511 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:34:36,543] jepsen worker 81 - jepsen.util 11511 :invoke :read [27 nil] INFO [2019-12-17 12:34:36,558] jepsen worker 64 - jepsen.util 6994 :fail :cas [21 [37 24]] :ValueNotSwapFrom INFO [2019-12-17 12:34:36,558] jepsen worker 64 - jepsen.util 6994 :invoke :read [21 nil] INFO [2019-12-17 12:34:36,561] jepsen worker 41 - jepsen.util 12281 :info :cas [13 [22 24]] :RequestCanceledException INFO [2019-12-17 12:34:36,563] jepsen worker 41 - jepsen.util 12371 :invoke :write [13 8] INFO [2019-12-17 12:34:36,575] jepsen worker 64 - jepsen.util 6994 :ok :read [21 11] INFO [2019-12-17 12:34:36,575] jepsen worker 64 - jepsen.util 6994 :invoke :read [21 nil] INFO [2019-12-17 12:34:36,587] jepsen worker 52 - jepsen.util 8692 :fail :cas [17 [41 22]] :ValueNotSwapFrom INFO [2019-12-17 12:34:36,587] jepsen worker 52 - jepsen.util 8692 :invoke :cas [17 [29 4]] INFO [2019-12-17 12:34:36,590] jepsen worker 36 - jepsen.util 8766 :fail :cas [12 [30 31]] :ValueNotSwapFrom INFO [2019-12-17 12:34:36,590] jepsen worker 36 - jepsen.util 8766 :invoke :write [12 26] INFO [2019-12-17 12:34:36,591] jepsen worker 64 - jepsen.util 6994 :ok :read [21 11] INFO [2019-12-17 12:34:36,591] jepsen worker 64 - jepsen.util 6994 :invoke :write [21 5] INFO [2019-12-17 12:34:36,600] jepsen worker 38 - jepsen.util 8858 :ok :read [12 40] INFO [2019-12-17 12:34:36,600] jepsen worker 38 - jepsen.util 8858 :invoke :read [12 nil] INFO [2019-12-17 12:34:36,612] jepsen worker 51 - jepsen.util 7881 :ok :read [17 42] INFO [2019-12-17 12:34:36,612] jepsen worker 51 - jepsen.util 7881 :invoke :read [17 nil] INFO [2019-12-17 12:34:36,618] jepsen worker 78 - jepsen.util 8358 :fail :cas [26 [33 27]] :ValueNotSwapFrom INFO [2019-12-17 12:34:36,618] jepsen worker 78 - jepsen.util 8358 :invoke :read [26 nil] INFO [2019-12-17 12:34:36,629] jepsen worker 21 - jepsen.util 11091 :info :write [7 10] :RequestCanceledException INFO [2019-12-17 12:34:36,629] jepsen worker 59 - jepsen.util 11219 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:34:36,629] jepsen worker 86 - jepsen.util 12416 :info :cas [28 [35 3]] :RequestCanceledException INFO [2019-12-17 12:34:36,629] jepsen worker 59 - jepsen.util 11219 :invoke :read [19 nil] INFO [2019-12-17 12:34:36,630] jepsen worker 21 - jepsen.util 11181 :invoke :write [7 7] INFO [2019-12-17 12:34:36,630] jepsen worker 86 - jepsen.util 12506 :invoke :read [28 nil] INFO [2019-12-17 12:34:36,633] jepsen worker 64 - jepsen.util 6994 :ok :write [21 5] INFO [2019-12-17 12:34:36,633] jepsen worker 64 - jepsen.util 6994 :invoke :read [21 nil] INFO [2019-12-17 12:34:36,643] jepsen worker 39 - jepsen.util 11109 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:34:36,643] jepsen worker 39 - jepsen.util 11109 :invoke :read [13 nil] INFO [2019-12-17 12:34:36,658] jepsen worker 64 - jepsen.util 6994 :ok :read [21 5] INFO [2019-12-17 12:34:36,658] jepsen worker 64 - jepsen.util 6994 :invoke :write [21 4] INFO [2019-12-17 12:34:36,689] jepsen worker 64 - jepsen.util 6994 :ok :write [21 4] INFO [2019-12-17 12:34:36,689] jepsen worker 64 - jepsen.util 6994 :invoke :read [21 nil] INFO [2019-12-17 12:34:36,700] jepsen worker 51 - jepsen.util 7881 :ok :read [17 42] INFO [2019-12-17 12:34:36,700] jepsen worker 51 - jepsen.util 7881 :invoke :write [17 8] INFO [2019-12-17 12:34:36,709] jepsen worker 64 - jepsen.util 6994 :ok :read [21 4] INFO [2019-12-17 12:34:36,709] jepsen worker 64 - jepsen.util 6994 :invoke :cas [21 [36 44]] INFO [2019-12-17 12:34:36,721] jepsen worker 64 - jepsen.util 6994 :fail :cas [21 [36 44]] :ValueNotSwapFrom INFO [2019-12-17 12:34:36,721] jepsen worker 64 - jepsen.util 6994 :invoke :cas [21 [7 32]] INFO [2019-12-17 12:34:36,722] jepsen worker 37 - jepsen.util 8137 :ok :read [12 40] INFO [2019-12-17 12:34:36,723] jepsen worker 37 - jepsen.util 8137 :invoke :cas [12 [1 42]] INFO [2019-12-17 12:34:36,732] jepsen worker 64 - jepsen.util 6994 :fail :cas [21 [7 32]] :ValueNotSwapFrom INFO [2019-12-17 12:34:36,732] jepsen worker 64 - jepsen.util 6994 :invoke :write [21 19] INFO [2019-12-17 12:34:36,737] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":49368,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:34:36,738] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":49368,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:34:36,738] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":49371,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:34:36,738] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":49371,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:34:36,738] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":49372,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:34:36,738] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":49372,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:34:36,742] jepsen worker 53 - jepsen.util 7793 :fail :cas [17 [40 49]] :ValueNotSwapFrom INFO [2019-12-17 12:34:36,742] jepsen worker 53 - jepsen.util 7793 :invoke :write [17 18] INFO [2019-12-17 12:34:36,744] jepsen worker 80 - jepsen.util 8180 :info :write [26 10] :RequestCanceledException INFO [2019-12-17 12:34:36,745] jepsen worker 80 - jepsen.util 8270 :invoke :read [26 nil] INFO [2019-12-17 12:34:36,759] jepsen worker 78 - jepsen.util 8358 :ok :read [26 40] INFO [2019-12-17 12:34:36,759] jepsen worker 78 - jepsen.util 8358 :invoke :write [26 11] INFO [2019-12-17 12:34:36,760] jepsen worker 68 - jepsen.util 10958 :info :write [22 37] :RequestCanceledException INFO [2019-12-17 12:34:36,761] jepsen worker 68 - jepsen.util 11048 :invoke :read [22 nil] INFO [2019-12-17 12:34:36,761] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:34:36,761] jepsen nemesis - couchbase.util Rebalance check count: 382 INFO [2019-12-17 12:34:36,761] jepsen nemesis - couchbase.util Rebalance stuck count: 363 INFO [2019-12-17 12:34:36,770] jepsen worker 3 - jepsen.util 8373 :fail :cas [1 [27 12]] :ValueNotSwapFrom INFO [2019-12-17 12:34:36,770] jepsen worker 3 - jepsen.util 8373 :invoke :write [1 18] INFO [2019-12-17 12:34:36,775] jepsen worker 64 - jepsen.util 6994 :ok :write [21 19] INFO [2019-12-17 12:34:36,775] jepsen worker 64 - jepsen.util 6994 :invoke :read [21 nil] INFO [2019-12-17 12:34:36,779] jepsen worker 14 - jepsen.util 8744 :fail :cas [4 [21 42]] :ValueNotSwapFrom INFO [2019-12-17 12:34:36,779] jepsen worker 14 - jepsen.util 8744 :invoke :write [4 4] INFO [2019-12-17 12:34:36,791] jepsen worker 64 - jepsen.util 6994 :ok :read [21 19] INFO [2019-12-17 12:34:36,791] jepsen worker 64 - jepsen.util 6994 :invoke :write [21 27] INFO [2019-12-17 12:34:36,795] jepsen worker 76 - jepsen.util 11686 :info :cas [25 [3 15]] :RequestCanceledException INFO [2019-12-17 12:34:36,795] jepsen worker 72 - jepsen.util 8442 :info :write [24 7] :RequestCanceledException INFO [2019-12-17 12:34:36,796] jepsen worker 72 - jepsen.util 8532 :invoke :cas [24 [41 37]] INFO [2019-12-17 12:34:36,797] jepsen worker 76 - jepsen.util 11776 :invoke :write [25 43] INFO [2019-12-17 12:34:36,806] jepsen worker 52 - jepsen.util 8692 :fail :cas [17 [29 4]] :ValueNotSwapFrom INFO [2019-12-17 12:34:36,806] jepsen worker 52 - jepsen.util 8692 :invoke :write [17 18] INFO [2019-12-17 12:34:36,814] jepsen worker 82 - jepsen.util 12232 :info :cas [27 [25 43]] :RequestCanceledException INFO [2019-12-17 12:34:36,814] jepsen worker 29 - jepsen.util 6959 :fail :read [9 nil] :RequestTimeoutException INFO [2019-12-17 12:34:36,814] jepsen worker 29 - jepsen.util 6959 :invoke :write [9 29] INFO [2019-12-17 12:34:36,815] jepsen worker 82 - jepsen.util 12322 :invoke :cas [27 [4 34]] INFO [2019-12-17 12:34:36,828] jepsen worker 64 - jepsen.util 6994 :ok :write [21 27] INFO [2019-12-17 12:34:36,828] jepsen worker 64 - jepsen.util 6994 :invoke :cas [21 [49 41]] INFO [2019-12-17 12:34:36,828] jepsen worker 23 - jepsen.util 11273 :info :write [7 25] :RequestCanceledException INFO [2019-12-17 12:34:36,829] jepsen worker 50 - jepsen.util 12290 :info :cas [16 [20 46]] :RequestCanceledException INFO [2019-12-17 12:34:36,829] jepsen worker 11 - jepsen.util 8381 :info :write [3 11] :RequestCanceledException INFO [2019-12-17 12:34:36,829] jepsen worker 11 - jepsen.util 8471 :invoke :write [3 25] INFO [2019-12-17 12:34:36,830] jepsen worker 23 - jepsen.util 11363 :invoke :read [7 nil] INFO [2019-12-17 12:34:36,830] jepsen worker 50 - jepsen.util 12380 :invoke :read [16 nil] INFO [2019-12-17 12:34:36,844] jepsen worker 60 - jepsen.util 12660 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:34:36,844] jepsen worker 60 - jepsen.util 12660 :invoke :write [20 9] INFO [2019-12-17 12:34:36,861] jepsen worker 34 - jepsen.util 11554 :info :write [11 43] :RequestCanceledException INFO [2019-12-17 12:34:36,862] jepsen worker 34 - jepsen.util 11644 :invoke :write [11 48] INFO [2019-12-17 12:34:36,863] jepsen worker 64 - jepsen.util 6994 :fail :cas [21 [49 41]] :ValueNotSwapFrom INFO [2019-12-17 12:34:36,863] jepsen worker 64 - jepsen.util 6994 :invoke :write [21 32] INFO [2019-12-17 12:34:36,901] jepsen worker 64 - jepsen.util 6994 :ok :write [21 32] INFO [2019-12-17 12:34:36,901] jepsen worker 64 - jepsen.util 6994 :invoke :write [21 15] INFO [2019-12-17 12:34:36,911] jepsen worker 83 - jepsen.util 11963 :info :write [27 5] :RequestCanceledException INFO [2019-12-17 12:34:36,912] jepsen worker 83 - jepsen.util 12053 :invoke :cas [27 [17 30]] INFO [2019-12-17 12:34:36,929] jepsen worker 16 - jepsen.util 12526 :info :write [5 28] :RequestCanceledException INFO [2019-12-17 12:34:36,930] jepsen worker 16 - jepsen.util 12616 :invoke :write [5 23] INFO [2019-12-17 12:34:36,947] jepsen worker 64 - jepsen.util 6994 :ok :write [21 15] INFO [2019-12-17 12:34:36,947] jepsen worker 64 - jepsen.util 6994 :invoke :write [21 41] INFO [2019-12-17 12:34:36,961] jepsen worker 18 - jepsen.util 8658 :info :write [6 5] :RequestCanceledException INFO [2019-12-17 12:34:36,962] jepsen worker 18 - jepsen.util 8748 :invoke :read [6 nil] INFO [2019-12-17 12:34:36,971] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":49399,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:34:36,971] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":49399,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:34:36,971] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":49402,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:34:36,971] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":49402,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:34:36,974] jepsen worker 80 - jepsen.util 8270 :ok :read [26 40] INFO [2019-12-17 12:34:36,974] jepsen worker 80 - jepsen.util 8270 :invoke :read [26 nil] INFO [2019-12-17 12:34:36,987] jepsen worker 54 - jepsen.util 9504 :fail :cas [18 [39 26]] :ValueNotSwapFrom INFO [2019-12-17 12:34:36,987] jepsen worker 54 - jepsen.util 9504 :invoke :cas [18 [34 42]] INFO [2019-12-17 12:34:37,002] jepsen worker 37 - jepsen.util 8137 :fail :cas [12 [1 42]] :ValueNotSwapFrom INFO [2019-12-17 12:34:37,002] jepsen worker 18 - jepsen.util 8748 :ok :read [6 49] INFO [2019-12-17 12:34:37,002] jepsen worker 18 - jepsen.util 8748 :invoke :write [6 35] INFO [2019-12-17 12:34:37,002] jepsen worker 37 - jepsen.util 8137 :invoke :write [12 8] INFO [2019-12-17 12:34:37,006] jepsen worker 64 - jepsen.util 6994 :ok :write [21 41] INFO [2019-12-17 12:34:37,006] jepsen worker 64 - jepsen.util 6994 :invoke :write [21 49] INFO [2019-12-17 12:34:37,029] jepsen worker 67 - jepsen.util 11767 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:34:37,029] jepsen worker 20 - jepsen.util 8210 :info :cas [6 [24 25]] :RequestCanceledException INFO [2019-12-17 12:34:37,030] jepsen worker 67 - jepsen.util 11767 :invoke :cas [22 [12 21]] INFO [2019-12-17 12:34:37,030] jepsen worker 20 - jepsen.util 8300 :invoke :read [6 nil] INFO [2019-12-17 12:34:37,042] jepsen worker 80 - jepsen.util 8270 :ok :read [26 40] INFO [2019-12-17 12:34:37,042] jepsen worker 80 - jepsen.util 8270 :invoke :write [26 2] INFO [2019-12-17 12:34:37,061] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":49414,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:34:37,061] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":49414,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:34:37,061] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":49418,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:34:37,061] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":49418,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:34:37,087] jepsen worker 64 - jepsen.util 6994 :ok :write [21 49] INFO [2019-12-17 12:34:37,087] jepsen worker 64 - jepsen.util 6994 :invoke :read [21 nil] INFO [2019-12-17 12:34:37,096] jepsen worker 38 - jepsen.util 8858 :ok :read [12 40] INFO [2019-12-17 12:34:37,096] jepsen worker 38 - jepsen.util 8858 :invoke :read [12 nil] INFO [2019-12-17 12:34:37,101] jepsen worker 64 - jepsen.util 6994 :ok :read [21 49] INFO [2019-12-17 12:34:37,101] jepsen worker 64 - jepsen.util 6994 :invoke :write [21 42] INFO [2019-12-17 12:34:37,129] jepsen worker 56 - jepsen.util 9326 :info :write [18 14] :RequestCanceledException INFO [2019-12-17 12:34:37,130] jepsen worker 56 - jepsen.util 9416 :invoke :read [18 nil] INFO [2019-12-17 12:34:37,140] jepsen worker 64 - jepsen.util 6994 :ok :write [21 42] INFO [2019-12-17 12:34:37,141] jepsen worker 64 - jepsen.util 6994 :invoke :read [21 nil] INFO [2019-12-17 12:34:37,144] jepsen worker 12 - jepsen.util 9012 :info :write [4 42] :RequestCanceledException INFO [2019-12-17 12:34:37,145] jepsen worker 12 - jepsen.util 9102 :invoke :cas [4 [10 9]] INFO [2019-12-17 12:34:37,160] jepsen worker 5 - jepsen.util 7925 :info :cas [1 [49 47]] :RequestCanceledException INFO [2019-12-17 12:34:37,161] jepsen worker 5 - jepsen.util 8015 :invoke :read [1 nil] INFO [2019-12-17 12:34:37,167] jepsen worker 64 - jepsen.util 6994 :ok :read [21 42] INFO [2019-12-17 12:34:37,167] jepsen worker 64 - jepsen.util 6994 :invoke :cas [21 [1 14]] INFO [2019-12-17 12:34:37,179] jepsen worker 64 - jepsen.util 6994 :fail :cas [21 [1 14]] :ValueNotSwapFrom INFO [2019-12-17 12:34:37,179] jepsen worker 64 - jepsen.util 6994 :invoke :read [21 nil] INFO [2019-12-17 12:34:37,192] jepsen worker 64 - jepsen.util 6994 :ok :read [21 42] INFO [2019-12-17 12:34:37,192] jepsen worker 64 - jepsen.util 6994 :invoke :write [21 16] INFO [2019-12-17 12:34:37,196] jepsen worker 30 - jepsen.util 8490 :info :write [10 1] :RequestCanceledException INFO [2019-12-17 12:34:37,197] jepsen worker 30 - jepsen.util 8580 :invoke :read [10 nil] INFO [2019-12-17 12:34:37,211] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":49437,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:34:37,211] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":49437,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:34:37,211] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":49438,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:34:37,212] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":49438,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:34:37,215] jepsen worker 35 - jepsen.util 11285 :info :cas [11 [15 48]] :RequestCanceledException INFO [2019-12-17 12:34:37,216] jepsen worker 35 - jepsen.util 11375 :invoke :cas [11 [35 43]] INFO [2019-12-17 12:34:37,230] jepsen worker 64 - jepsen.util 6994 :ok :write [21 16] INFO [2019-12-17 12:34:37,230] jepsen worker 31 - jepsen.util 7501 :info :write [10 18] :RequestCanceledException INFO [2019-12-17 12:34:37,230] jepsen worker 64 - jepsen.util 6994 :invoke :write [21 5] INFO [2019-12-17 12:34:37,231] jepsen worker 31 - jepsen.util 7591 :invoke :cas [10 [24 1]] INFO [2019-12-17 12:34:37,280] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":49444,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:34:37,281] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":49444,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:34:37,281] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":49447,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:34:37,281] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":49447,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:34:37,281] jepsen worker 64 - jepsen.util 6994 :ok :write [21 5] INFO [2019-12-17 12:34:37,282] jepsen worker 64 - jepsen.util 6994 :invoke :write [21 37] INFO [2019-12-17 12:34:37,283] jepsen worker 32 - jepsen.util 8762 :ok :write [10 23] INFO [2019-12-17 12:34:37,284] jepsen worker 32 - jepsen.util 8762 :invoke :read [10 nil] INFO [2019-12-17 12:34:37,307] jepsen worker 65 - jepsen.util 6995 :fail :cas [21 [18 20]] :ValueNotSwapFrom INFO [2019-12-17 12:34:37,307] jepsen worker 65 - jepsen.util 6995 :invoke :write [21 11] INFO [2019-12-17 12:34:37,314] jepsen worker 19 - jepsen.util 8659 :ok :write [6 27] INFO [2019-12-17 12:34:37,314] jepsen worker 19 - jepsen.util 8659 :invoke :read [6 nil] INFO [2019-12-17 12:34:37,316] jepsen worker 10 - jepsen.util 8830 :ok :write [3 45] INFO [2019-12-17 12:34:37,316] jepsen worker 10 - jepsen.util 8830 :invoke :write [3 26] INFO [2019-12-17 12:34:37,325] jepsen worker 46 - jepsen.util 8506 :ok :write [15 27] INFO [2019-12-17 12:34:37,325] jepsen worker 46 - jepsen.util 8506 :invoke :write [15 13] INFO [2019-12-17 12:34:37,330] jepsen worker 33 - jepsen.util 11643 :info :cas [11 [1 1]] :RequestCanceledException INFO [2019-12-17 12:34:37,330] jepsen worker 79 - jepsen.util 8809 :info :write [26 26] :RequestCanceledException INFO [2019-12-17 12:34:37,331] jepsen worker 79 - jepsen.util 8899 :invoke :read [26 nil] INFO [2019-12-17 12:34:37,331] jepsen worker 33 - jepsen.util 11733 :invoke :write [11 22] INFO [2019-12-17 12:34:37,345] jepsen worker 30 - jepsen.util 8580 :ok :read [10 23] INFO [2019-12-17 12:34:37,345] jepsen worker 30 - jepsen.util 8580 :invoke :cas [10 [42 5]] INFO [2019-12-17 12:34:37,373] jepsen worker 64 - jepsen.util 6994 :ok :write [21 37] INFO [2019-12-17 12:34:37,373] jepsen worker 64 - jepsen.util 6994 :invoke :read [21 nil] INFO [2019-12-17 12:34:37,375] jepsen worker 32 - jepsen.util 8762 :ok :read [10 23] INFO [2019-12-17 12:34:37,375] jepsen worker 32 - jepsen.util 8762 :invoke :read [10 nil] INFO [2019-12-17 12:34:37,393] jepsen worker 54 - jepsen.util 9504 :fail :cas [18 [34 42]] :ValueNotSwapFrom INFO [2019-12-17 12:34:37,394] jepsen worker 54 - jepsen.util 9504 :invoke :read [18 nil] INFO [2019-12-17 12:34:37,394] jepsen worker 64 - jepsen.util 6994 :ok :read [21 37] INFO [2019-12-17 12:34:37,394] jepsen worker 64 - jepsen.util 6994 :invoke :write [21 13] INFO [2019-12-17 12:34:37,414] jepsen worker 74 - jepsen.util 8714 :info :cas [24 [9 36]] :RequestCanceledException INFO [2019-12-17 12:34:37,415] jepsen worker 74 - jepsen.util 8804 :invoke :read [24 nil] INFO [2019-12-17 12:34:37,466] jepsen worker 64 - jepsen.util 6994 :ok :write [21 13] INFO [2019-12-17 12:34:37,466] jepsen worker 64 - jepsen.util 6994 :invoke :cas [21 [48 0]] INFO [2019-12-17 12:34:37,483] jepsen worker 64 - jepsen.util 6994 :fail :cas [21 [48 0]] :ValueNotSwapFrom INFO [2019-12-17 12:34:37,483] jepsen worker 64 - jepsen.util 6994 :invoke :write [21 22] INFO [2019-12-17 12:34:37,489] jepsen worker 65 - jepsen.util 6995 :ok :write [21 11] INFO [2019-12-17 12:34:37,490] jepsen worker 65 - jepsen.util 6995 :invoke :cas [21 [2 16]] INFO [2019-12-17 12:34:37,497] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":49466,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:34:37,497] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":49466,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:34:37,497] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":49469,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:34:37,497] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":49469,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:34:37,497] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":49470,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:34:37,497] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":49470,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:34:37,516] jepsen worker 38 - jepsen.util 8858 :ok :read [12 40] INFO [2019-12-17 12:34:37,516] jepsen worker 38 - jepsen.util 8858 :invoke :cas [12 [3 49]] INFO [2019-12-17 12:34:37,528] jepsen worker 65 - jepsen.util 6995 :fail :cas [21 [2 16]] :ValueNotSwapFrom INFO [2019-12-17 12:34:37,528] jepsen worker 65 - jepsen.util 6995 :invoke :write [21 48] INFO [2019-12-17 12:34:37,529] jepsen worker 57 - jepsen.util 11487 :info :write [19 11] :RequestCanceledException INFO [2019-12-17 12:34:37,530] jepsen worker 57 - jepsen.util 11577 :invoke :cas [19 [33 24]] INFO [2019-12-17 12:34:37,544] jepsen worker 62 - jepsen.util 11942 :info :write [20 38] :RequestCanceledException INFO [2019-12-17 12:34:37,544] jepsen worker 56 - jepsen.util 9416 :ok :read [18 17] INFO [2019-12-17 12:34:37,544] jepsen worker 56 - jepsen.util 9416 :invoke :read [18 nil] INFO [2019-12-17 12:34:37,545] jepsen worker 62 - jepsen.util 12032 :invoke :read [20 nil] INFO [2019-12-17 12:34:37,559] jepsen worker 79 - jepsen.util 8899 :ok :read [26 40] INFO [2019-12-17 12:34:37,559] jepsen worker 79 - jepsen.util 8899 :invoke :cas [26 [32 7]] INFO [2019-12-17 12:34:37,561] jepsen worker 25 - jepsen.util 12265 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:34:37,562] jepsen worker 25 - jepsen.util 12265 :invoke :write [8 2] INFO [2019-12-17 12:34:37,574] jepsen worker 64 - jepsen.util 6994 :ok :write [21 22] INFO [2019-12-17 12:34:37,574] jepsen worker 64 - jepsen.util 6994 :invoke :write [21 34] INFO [2019-12-17 12:34:37,596] jepsen worker 63 - jepsen.util 7353 :info :write [21 44] :RequestCanceledException INFO [2019-12-17 12:34:37,596] jepsen worker 0 - jepsen.util 12060 :info :write [0 48] :RequestCanceledException INFO [2019-12-17 12:34:37,597] jepsen worker 63 - jepsen.util 7443 :invoke :read [21 nil] INFO [2019-12-17 12:34:37,598] jepsen worker 0 - jepsen.util 12150 :invoke :cas [0 [20 39]] INFO [2019-12-17 12:34:37,600] jepsen worker 65 - jepsen.util 6995 :ok :write [21 48] INFO [2019-12-17 12:34:37,601] jepsen worker 65 - jepsen.util 6995 :invoke :write [21 40] INFO [2019-12-17 12:34:37,614] jepsen worker 58 - jepsen.util 12568 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:34:37,614] jepsen worker 58 - jepsen.util 12568 :invoke :read [19 nil] INFO [2019-12-17 12:34:37,649] jepsen worker 64 - jepsen.util 6994 :ok :write [21 34] INFO [2019-12-17 12:34:37,649] jepsen worker 64 - jepsen.util 6994 :invoke :read [21 nil] INFO [2019-12-17 12:34:37,663] jepsen worker 64 - jepsen.util 6994 :ok :read [21 34] INFO [2019-12-17 12:34:37,663] jepsen worker 64 - jepsen.util 6994 :invoke :cas [21 [44 6]] INFO [2019-12-17 12:34:37,674] jepsen worker 64 - jepsen.util 6994 :fail :cas [21 [44 6]] :ValueNotSwapFrom INFO [2019-12-17 12:34:37,674] jepsen worker 64 - jepsen.util 6994 :invoke :write [21 43] INFO [2019-12-17 12:34:37,707] jepsen worker 64 - jepsen.util 6994 :ok :write [21 43] INFO [2019-12-17 12:34:37,707] jepsen worker 64 - jepsen.util 6994 :invoke :cas [21 [19 8]] INFO [2019-12-17 12:34:37,717] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":49487,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:34:37,717] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":49487,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:34:37,717] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":49490,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:34:37,717] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":49490,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:34:37,722] jepsen worker 64 - jepsen.util 6994 :fail :cas [21 [19 8]] :ValueNotSwapFrom INFO [2019-12-17 12:34:37,723] jepsen worker 64 - jepsen.util 6994 :invoke :read [21 nil] INFO [2019-12-17 12:34:37,729] jepsen worker 26 - jepsen.util 12086 :info :write [8 35] :RequestCanceledException INFO [2019-12-17 12:34:37,729] jepsen worker 85 - jepsen.util 12415 :info :write [28 1] :RequestCanceledException INFO [2019-12-17 12:34:37,730] jepsen worker 26 - jepsen.util 12176 :invoke :cas [8 [46 26]] INFO [2019-12-17 12:34:37,730] jepsen worker 85 - jepsen.util 12505 :invoke :cas [28 [48 18]] INFO [2019-12-17 12:34:37,732] jepsen worker 5 - jepsen.util 8015 :ok :read [1 49] INFO [2019-12-17 12:34:37,733] jepsen worker 5 - jepsen.util 8015 :invoke :write [1 6] INFO [2019-12-17 12:34:37,742] jepsen worker 64 - jepsen.util 6994 :ok :read [21 43] INFO [2019-12-17 12:34:37,743] jepsen worker 64 - jepsen.util 6994 :invoke :read [21 nil] INFO [2019-12-17 12:34:37,743] jepsen worker 22 - jepsen.util 11632 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:34:37,743] jepsen worker 48 - jepsen.util 10848 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:34:37,743] jepsen worker 22 - jepsen.util 11632 :invoke :read [7 nil] INFO [2019-12-17 12:34:37,743] jepsen worker 48 - jepsen.util 10848 :invoke :read [16 nil] INFO [2019-12-17 12:34:37,754] jepsen worker 64 - jepsen.util 6994 :ok :read [21 43] INFO [2019-12-17 12:34:37,754] jepsen worker 64 - jepsen.util 6994 :invoke :write [21 16] INFO [2019-12-17 12:34:37,761] jepsen worker 28 - jepsen.util 6778 :info :write [9 21] :RequestCanceledException INFO [2019-12-17 12:34:37,762] jepsen worker 28 - jepsen.util 6868 :invoke :write [9 6] INFO [2019-12-17 12:34:37,776] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:34:37,777] jepsen nemesis - couchbase.util Rebalance check count: 383 INFO [2019-12-17 12:34:37,777] jepsen nemesis - couchbase.util Rebalance stuck count: 364 INFO [2019-12-17 12:34:37,792] jepsen worker 6 - jepsen.util 12156 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:34:37,792] jepsen worker 45 - jepsen.util 8865 :info :write [15 45] :RequestCanceledException INFO [2019-12-17 12:34:37,793] jepsen worker 6 - jepsen.util 12156 :invoke :cas [2 [42 14]] INFO [2019-12-17 12:34:37,793] jepsen worker 45 - jepsen.util 8955 :invoke :read [15 nil] INFO [2019-12-17 12:34:37,814] jepsen worker 61 - jepsen.util 12391 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:34:37,814] jepsen worker 61 - jepsen.util 12391 :invoke :read [20 nil] INFO [2019-12-17 12:34:37,818] jepsen worker 45 - jepsen.util 8955 :ok :read [15 27] INFO [2019-12-17 12:34:37,818] jepsen worker 45 - jepsen.util 8955 :invoke :read [15 nil] INFO [2019-12-17 12:34:37,830] jepsen worker 66 - jepsen.util 11496 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:34:37,830] jepsen worker 66 - jepsen.util 11496 :invoke :read [22 nil] INFO [2019-12-17 12:34:37,838] jepsen worker 12 - jepsen.util 9102 :fail :cas [4 [10 9]] :ValueNotSwapFrom INFO [2019-12-17 12:34:37,838] jepsen worker 12 - jepsen.util 9102 :invoke :write [4 33] INFO [2019-12-17 12:34:37,842] jepsen worker 64 - jepsen.util 6994 :ok :write [21 16] INFO [2019-12-17 12:34:37,842] jepsen worker 64 - jepsen.util 6994 :invoke :read [21 nil] INFO [2019-12-17 12:34:37,843] jepsen worker 44 - jepsen.util 11924 :info :cas [14 [5 14]] :RequestCanceledException INFO [2019-12-17 12:34:37,844] jepsen worker 44 - jepsen.util 12014 :invoke :cas [14 [45 40]] INFO [2019-12-17 12:34:37,846] jepsen worker 45 - jepsen.util 8955 :ok :read [15 27] INFO [2019-12-17 12:34:37,847] jepsen worker 45 - jepsen.util 8955 :invoke :cas [15 [20 31]] INFO [2019-12-17 12:34:37,861] jepsen worker 47 - jepsen.util 8957 :info :write [15 47] :RequestCanceledException INFO [2019-12-17 12:34:37,862] jepsen worker 47 - jepsen.util 9047 :invoke :read [15 nil] INFO [2019-12-17 12:34:37,863] jepsen worker 64 - jepsen.util 6994 :ok :read [21 16] INFO [2019-12-17 12:34:37,863] jepsen worker 64 - jepsen.util 6994 :invoke :read [21 nil] INFO [2019-12-17 12:34:37,878] jepsen worker 45 - jepsen.util 8955 :fail :cas [15 [20 31]] :ValueNotSwapFrom INFO [2019-12-17 12:34:37,878] jepsen worker 45 - jepsen.util 8955 :invoke :read [15 nil] INFO [2019-12-17 12:34:37,897] jepsen worker 64 - jepsen.util 6994 :ok :read [21 16] INFO [2019-12-17 12:34:37,897] jepsen worker 64 - jepsen.util 6994 :invoke :read [21 nil] INFO [2019-12-17 12:34:37,930] jepsen worker 84 - jepsen.util 11064 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:34:37,930] jepsen worker 84 - jepsen.util 11064 :invoke :write [28 13] INFO [2019-12-17 12:34:37,931] jepsen worker 45 - jepsen.util 8955 :ok :read [15 27] INFO [2019-12-17 12:34:37,931] jepsen worker 28 - jepsen.util 6868 :ok :write [9 6] INFO [2019-12-17 12:34:37,932] jepsen worker 45 - jepsen.util 8955 :invoke :write [15 9] INFO [2019-12-17 12:34:37,932] jepsen worker 28 - jepsen.util 6868 :invoke :read [9 nil] INFO [2019-12-17 12:34:37,940] jepsen worker 64 - jepsen.util 6994 :ok :read [21 16] INFO [2019-12-17 12:34:37,941] jepsen worker 64 - jepsen.util 6994 :invoke :write [21 18] INFO [2019-12-17 12:34:37,956] jepsen worker 28 - jepsen.util 6868 :ok :read [9 6] INFO [2019-12-17 12:34:37,957] jepsen worker 28 - jepsen.util 6868 :invoke :write [9 24] INFO [2019-12-17 12:34:37,957] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":49516,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:34:37,957] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":49516,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:34:37,957] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":49521,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:34:37,957] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":49521,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:34:37,976] jepsen worker 54 - jepsen.util 9504 :ok :read [18 17] INFO [2019-12-17 12:34:37,977] jepsen worker 54 - jepsen.util 9504 :invoke :read [18 nil] INFO [2019-12-17 12:34:37,987] jepsen worker 38 - jepsen.util 8858 :fail :cas [12 [3 49]] :ValueNotSwapFrom INFO [2019-12-17 12:34:37,987] jepsen worker 38 - jepsen.util 8858 :invoke :write [12 27] INFO [2019-12-17 12:34:37,995] jepsen worker 70 - jepsen.util 8980 :info :write [23 3] :RequestCanceledException INFO [2019-12-17 12:34:37,996] jepsen worker 70 - jepsen.util 9070 :invoke :read [23 nil] INFO [2019-12-17 12:34:37,999] jepsen worker 64 - jepsen.util 6994 :ok :write [21 18] INFO [2019-12-17 12:34:38,000] jepsen worker 64 - jepsen.util 6994 :invoke :cas [21 [46 28]] INFO [2019-12-17 12:34:38,013] jepsen worker 24 - jepsen.util 10734 :info :write [8 27] :RequestCanceledException INFO [2019-12-17 12:34:38,014] jepsen worker 24 - jepsen.util 10824 :invoke :write [8 6] INFO [2019-12-17 12:34:38,032] jepsen worker 64 - jepsen.util 6994 :fail :cas [21 [46 28]] :ValueNotSwapFrom INFO [2019-12-17 12:34:38,032] jepsen worker 64 - jepsen.util 6994 :invoke :write [21 28] INFO [2019-12-17 12:34:38,037] jepsen worker 56 - jepsen.util 9416 :ok :read [18 17] INFO [2019-12-17 12:34:38,037] jepsen worker 56 - jepsen.util 9416 :invoke :read [18 nil] INFO [2019-12-17 12:34:38,044] jepsen worker 40 - jepsen.util 11740 :info :write [13 12] :RequestCanceledException INFO [2019-12-17 12:34:38,045] jepsen worker 40 - jepsen.util 11830 :invoke :write [13 3] INFO [2019-12-17 12:34:38,046] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":49533,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:34:38,046] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":49533,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:34:38,046] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":49536,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:34:38,046] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":49536,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:34:38,060] jepsen worker 17 - jepsen.util 11627 :info :cas [5 [35 37]] :RequestCanceledException INFO [2019-12-17 12:34:38,061] jepsen worker 17 - jepsen.util 11717 :invoke :read [5 nil] INFO [2019-12-17 12:34:38,063] jepsen worker 28 - jepsen.util 6868 :ok :write [9 24] INFO [2019-12-17 12:34:38,063] jepsen worker 28 - jepsen.util 6868 :invoke :write [9 22] INFO [2019-12-17 12:34:38,093] jepsen worker 64 - jepsen.util 6994 :ok :write [21 28] INFO [2019-12-17 12:34:38,093] jepsen worker 64 - jepsen.util 6994 :invoke :cas [21 [20 37]] INFO [2019-12-17 12:34:38,096] jepsen worker 4 - jepsen.util 8104 :info :write [1 14] :RequestCanceledException INFO [2019-12-17 12:34:38,097] jepsen worker 4 - jepsen.util 8194 :invoke :read [1 nil] INFO [2019-12-17 12:34:38,107] jepsen worker 79 - jepsen.util 8899 :fail :cas [26 [32 7]] :ValueNotSwapFrom INFO [2019-12-17 12:34:38,107] jepsen worker 79 - jepsen.util 8899 :invoke :cas [26 [8 44]] INFO [2019-12-17 12:34:38,114] jepsen worker 15 - jepsen.util 10905 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:34:38,114] jepsen worker 15 - jepsen.util 10905 :invoke :read [5 nil] INFO [2019-12-17 12:34:38,116] jepsen worker 64 - jepsen.util 6994 :fail :cas [21 [20 37]] :ValueNotSwapFrom INFO [2019-12-17 12:34:38,116] jepsen worker 64 - jepsen.util 6994 :invoke :read [21 nil] INFO [2019-12-17 12:34:38,130] jepsen worker 9 - jepsen.util 8919 :info :write [3 23] :RequestCanceledException INFO [2019-12-17 12:34:38,130] jepsen worker 42 - jepsen.util 12732 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:34:38,130] jepsen worker 42 - jepsen.util 12732 :invoke :read [14 nil] INFO [2019-12-17 12:34:38,131] jepsen worker 9 - jepsen.util 9009 :invoke :write [3 12] INFO [2019-12-17 12:34:38,135] jepsen worker 64 - jepsen.util 6994 :ok :read [21 28] INFO [2019-12-17 12:34:38,135] jepsen worker 64 - jepsen.util 6994 :invoke :cas [21 [43 12]] INFO [2019-12-17 12:34:38,141] jepsen worker 79 - jepsen.util 8899 :fail :cas [26 [8 44]] :ValueNotSwapFrom INFO [2019-12-17 12:34:38,141] jepsen worker 79 - jepsen.util 8899 :invoke :cas [26 [40 17]] INFO [2019-12-17 12:34:38,144] jepsen worker 69 - jepsen.util 8619 :info :write [23 14] :RequestCanceledException INFO [2019-12-17 12:34:38,144] jepsen worker 1 - jepsen.util 11791 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:34:38,144] jepsen worker 87 - jepsen.util 8007 :info :write [29 43] :RequestCanceledException INFO [2019-12-17 12:34:38,144] jepsen worker 1 - jepsen.util 11791 :invoke :write [0 27] INFO [2019-12-17 12:34:38,145] jepsen worker 69 - jepsen.util 8709 :invoke :write [23 22] INFO [2019-12-17 12:34:38,145] jepsen worker 87 - jepsen.util 8097 :invoke :read [29 nil] INFO [2019-12-17 12:34:38,146] jepsen worker 28 - jepsen.util 6868 :ok :write [9 22] INFO [2019-12-17 12:34:38,146] jepsen worker 28 - jepsen.util 6868 :invoke :cas [9 [37 11]] INFO [2019-12-17 12:34:38,161] jepsen worker 43 - jepsen.util 11293 :info :cas [14 [32 35]] :RequestCanceledException INFO [2019-12-17 12:34:38,162] jepsen worker 43 - jepsen.util 11383 :invoke :cas [14 [23 46]] INFO [2019-12-17 12:34:38,171] jepsen worker 28 - jepsen.util 6868 :fail :cas [9 [37 11]] :ValueNotSwapFrom INFO [2019-12-17 12:34:38,171] jepsen worker 28 - jepsen.util 6868 :invoke :read [9 nil] INFO [2019-12-17 12:34:38,178] jepsen worker 64 - jepsen.util 6994 :fail :cas [21 [43 12]] :ValueNotSwapFrom INFO [2019-12-17 12:34:38,179] jepsen worker 64 - jepsen.util 6994 :invoke :cas [21 [16 5]] INFO [2019-12-17 12:34:38,191] jepsen worker 28 - jepsen.util 6868 :ok :read [9 22] INFO [2019-12-17 12:34:38,191] jepsen worker 28 - jepsen.util 6868 :invoke :write [9 14] INFO [2019-12-17 12:34:38,197] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":49562,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:34:38,197] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":49562,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:34:38,197] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":49564,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:34:38,197] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":49564,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:34:38,199] jepsen worker 64 - jepsen.util 6994 :fail :cas [21 [16 5]] :ValueNotSwapFrom INFO [2019-12-17 12:34:38,199] jepsen worker 64 - jepsen.util 6994 :invoke :read [21 nil] INFO [2019-12-17 12:34:38,201] jepsen worker 56 - jepsen.util 9416 :ok :read [18 17] INFO [2019-12-17 12:34:38,201] jepsen worker 56 - jepsen.util 9416 :invoke :write [18 6] INFO [2019-12-17 12:34:38,214] jepsen worker 75 - jepsen.util 11055 :info :cas [25 [0 28]] :RequestCanceledException INFO [2019-12-17 12:34:38,215] jepsen worker 75 - jepsen.util 11145 :invoke :read [25 nil] INFO [2019-12-17 12:34:38,221] jepsen worker 4 - jepsen.util 8194 :ok :read [1 49] INFO [2019-12-17 12:34:38,221] jepsen worker 4 - jepsen.util 8194 :invoke :cas [1 [21 34]] INFO [2019-12-17 12:34:38,227] jepsen worker 8 - jepsen.util 11438 :info :cas [2 [6 7]] :RequestCanceledException INFO [2019-12-17 12:34:38,227] jepsen worker 77 - jepsen.util 11867 :info :cas [25 [16 46]] :RequestCanceledException INFO [2019-12-17 12:34:38,227] jepsen worker 13 - jepsen.util 8203 :info :write [4 25] :RequestCanceledException INFO [2019-12-17 12:34:38,228] jepsen worker 8 - jepsen.util 11528 :invoke :read [2 nil] INFO [2019-12-17 12:34:38,228] jepsen worker 13 - jepsen.util 8293 :invoke :read [4 nil] INFO [2019-12-17 12:34:38,228] jepsen worker 77 - jepsen.util 11957 :invoke :write [25 31] INFO [2019-12-17 12:34:38,232] jepsen worker 64 - jepsen.util 6994 :ok :read [21 28] INFO [2019-12-17 12:34:38,232] jepsen worker 64 - jepsen.util 6994 :invoke :write [21 22] INFO [2019-12-17 12:34:38,266] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":49572,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:34:38,266] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":49572,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:34:38,266] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":49576,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:34:38,266] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":49576,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:34:38,267] jepsen worker 28 - jepsen.util 6868 :ok :write [9 14] INFO [2019-12-17 12:34:38,267] jepsen worker 28 - jepsen.util 6868 :invoke :write [9 37] INFO [2019-12-17 12:34:38,268] jepsen worker 54 - jepsen.util 9504 :ok :read [18 17] INFO [2019-12-17 12:34:38,268] jepsen worker 54 - jepsen.util 9504 :invoke :read [18 nil] INFO [2019-12-17 12:34:38,306] jepsen worker 64 - jepsen.util 6994 :ok :write [21 22] INFO [2019-12-17 12:34:38,306] jepsen worker 64 - jepsen.util 6994 :invoke :read [21 nil] INFO [2019-12-17 12:34:38,310] jepsen worker 46 - jepsen.util 8506 :ok :write [15 13] INFO [2019-12-17 12:34:38,310] jepsen worker 46 - jepsen.util 8506 :invoke :read [15 nil] INFO [2019-12-17 12:34:38,314] jepsen worker 89 - jepsen.util 8549 :info :write [29 17] :RequestCanceledException INFO [2019-12-17 12:34:38,314] jepsen worker 10 - jepsen.util 8830 :ok :write [3 26] INFO [2019-12-17 12:34:38,315] jepsen worker 10 - jepsen.util 8830 :invoke :cas [3 [48 21]] INFO [2019-12-17 12:34:38,315] jepsen worker 89 - jepsen.util 8639 :invoke :cas [29 [41 1]] INFO [2019-12-17 12:34:38,326] jepsen worker 73 - jepsen.util 8263 :ok :write [24 29] INFO [2019-12-17 12:34:38,326] jepsen worker 73 - jepsen.util 8263 :invoke :read [24 nil] INFO [2019-12-17 12:34:38,330] jepsen worker 88 - jepsen.util 8728 :info :write [29 12] :RequestCanceledException INFO [2019-12-17 12:34:38,330] jepsen worker 88 - jepsen.util 8818 :invoke :cas [29 [23 8]] INFO [2019-12-17 12:34:38,337] jepsen worker 54 - jepsen.util 9504 :ok :read [18 17] INFO [2019-12-17 12:34:38,337] jepsen worker 54 - jepsen.util 9504 :invoke :cas [18 [37 38]] INFO [2019-12-17 12:34:38,413] jepsen worker 49 - jepsen.util 11029 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:34:38,413] jepsen worker 27 - jepsen.util 8307 :fail :read [9 nil] :RequestTimeoutException INFO [2019-12-17 12:34:38,414] jepsen worker 49 - jepsen.util 11029 :invoke :cas [16 [40 1]] INFO [2019-12-17 12:34:38,414] jepsen worker 27 - jepsen.util 8307 :invoke :read [9 nil] INFO [2019-12-17 12:34:38,429] jepsen worker 7 - jepsen.util 11797 :info :write [2 49] :RequestCanceledException INFO [2019-12-17 12:34:38,429] jepsen worker 2 - jepsen.util 11342 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:34:38,429] jepsen worker 2 - jepsen.util 11342 :invoke :cas [0 [9 38]] INFO [2019-12-17 12:34:38,430] jepsen worker 7 - jepsen.util 11887 :invoke :write [2 6] INFO [2019-12-17 12:34:38,436] jepsen worker 64 - jepsen.util 6994 :ok :read [21 22] INFO [2019-12-17 12:34:38,436] jepsen worker 64 - jepsen.util 6994 :invoke :read [21 nil] INFO [2019-12-17 12:34:38,460] jepsen worker 55 - jepsen.util 9325 :info :write [18 10] :RequestCanceledException INFO [2019-12-17 12:34:38,460] jepsen worker 71 - jepsen.util 9071 :info :write [23 47] :RequestCanceledException INFO [2019-12-17 12:34:38,461] jepsen worker 55 - jepsen.util 9415 :invoke :read [18 nil] INFO [2019-12-17 12:34:38,461] jepsen worker 71 - jepsen.util 9161 :invoke :read [23 nil] INFO [2019-12-17 12:34:38,469] jepsen worker 47 - jepsen.util 9047 :ok :read [15 13] INFO [2019-12-17 12:34:38,469] jepsen worker 47 - jepsen.util 9047 :invoke :cas [15 [10 20]] INFO [2019-12-17 12:34:38,474] jepsen worker 74 - jepsen.util 8804 :ok :read [24 29] INFO [2019-12-17 12:34:38,474] jepsen worker 74 - jepsen.util 8804 :invoke :read [24 nil] INFO [2019-12-17 12:34:38,477] jepsen worker 64 - jepsen.util 6994 :ok :read [21 22] INFO [2019-12-17 12:34:38,477] jepsen worker 64 - jepsen.util 6994 :invoke :read [21 nil] INFO [2019-12-17 12:34:38,482] jepsen worker 63 - jepsen.util 7443 :ok :read [21 22] INFO [2019-12-17 12:34:38,482] jepsen worker 63 - jepsen.util 7443 :invoke :write [21 20] INFO [2019-12-17 12:34:38,485] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":49595,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:34:38,485] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":49595,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:34:38,485] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":49598,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:34:38,485] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":49598,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:34:38,486] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":49599,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:34:38,486] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":49599,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:34:38,506] jepsen worker 10 - jepsen.util 8830 :fail :cas [3 [48 21]] :ValueNotSwapFrom INFO [2019-12-17 12:34:38,506] jepsen worker 10 - jepsen.util 8830 :invoke :cas [3 [1 3]] INFO [2019-12-17 12:34:38,513] jepsen worker 13 - jepsen.util 8293 :ok :read [4 19] INFO [2019-12-17 12:34:38,513] jepsen worker 13 - jepsen.util 8293 :invoke :cas [4 [7 38]] INFO [2019-12-17 12:34:38,530] jepsen worker 70 - jepsen.util 9070 :ok :read [23 9] INFO [2019-12-17 12:34:38,530] jepsen worker 70 - jepsen.util 9070 :invoke :cas [23 [19 39]] INFO [2019-12-17 12:34:38,546] jepsen worker 73 - jepsen.util 8263 :ok :read [24 29] INFO [2019-12-17 12:34:38,546] jepsen worker 73 - jepsen.util 8263 :invoke :write [24 29] INFO [2019-12-17 12:34:38,546] jepsen worker 64 - jepsen.util 6994 :ok :read [21 22] INFO [2019-12-17 12:34:38,547] jepsen worker 64 - jepsen.util 6994 :invoke :read [21 nil] INFO [2019-12-17 12:34:38,558] jepsen worker 88 - jepsen.util 8818 :fail :cas [29 [23 8]] :ValueNotSwapFrom INFO [2019-12-17 12:34:38,558] jepsen worker 88 - jepsen.util 8818 :invoke :cas [29 [12 39]] INFO [2019-12-17 12:34:38,565] jepsen worker 74 - jepsen.util 8804 :ok :read [24 29] INFO [2019-12-17 12:34:38,565] jepsen worker 74 - jepsen.util 8804 :invoke :cas [24 [27 30]] INFO [2019-12-17 12:34:38,575] jepsen worker 27 - jepsen.util 8307 :ok :read [9 14] INFO [2019-12-17 12:34:38,576] jepsen worker 27 - jepsen.util 8307 :invoke :write [9 11] INFO [2019-12-17 12:34:38,578] jepsen worker 28 - jepsen.util 6868 :ok :write [9 37] INFO [2019-12-17 12:34:38,578] jepsen worker 28 - jepsen.util 6868 :invoke :write [9 21] INFO [2019-12-17 12:34:38,593] jepsen worker 54 - jepsen.util 9504 :fail :cas [18 [37 38]] :ValueNotSwapFrom INFO [2019-12-17 12:34:38,593] jepsen worker 54 - jepsen.util 9504 :invoke :write [18 14] INFO [2019-12-17 12:34:38,622] jepsen worker 10 - jepsen.util 8830 :fail :cas [3 [1 3]] :ValueNotSwapFrom INFO [2019-12-17 12:34:38,623] jepsen worker 10 - jepsen.util 8830 :invoke :cas [3 [21 2]] INFO [2019-12-17 12:34:38,623] jepsen worker 80 - jepsen.util 8270 :ok :write [26 2] INFO [2019-12-17 12:34:38,623] jepsen worker 80 - jepsen.util 8270 :invoke :cas [26 [14 46]] INFO [2019-12-17 12:34:38,638] jepsen worker 74 - jepsen.util 8804 :fail :cas [24 [27 30]] :ValueNotSwapFrom INFO [2019-12-17 12:34:38,638] jepsen worker 74 - jepsen.util 8804 :invoke :write [24 35] INFO [2019-12-17 12:34:38,644] jepsen worker 81 - jepsen.util 11511 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:34:38,644] jepsen worker 36 - jepsen.util 8766 :info :write [12 26] :RequestCanceledException INFO [2019-12-17 12:34:38,644] jepsen worker 81 - jepsen.util 11511 :invoke :write [27 12] INFO [2019-12-17 12:34:38,645] jepsen worker 36 - jepsen.util 8856 :invoke :write [12 46] INFO [2019-12-17 12:34:38,652] jepsen worker 10 - jepsen.util 8830 :fail :cas [3 [21 2]] :ValueNotSwapFrom INFO [2019-12-17 12:34:38,652] jepsen worker 10 - jepsen.util 8830 :invoke :read [3 nil] INFO [2019-12-17 12:34:38,661] jepsen worker 41 - jepsen.util 12371 :info :write [13 8] :RequestCanceledException INFO [2019-12-17 12:34:38,662] jepsen worker 41 - jepsen.util 12461 :invoke :write [13 14] INFO [2019-12-17 12:34:38,680] jepsen worker 4 - jepsen.util 8194 :fail :cas [1 [21 34]] :ValueNotSwapFrom INFO [2019-12-17 12:34:38,681] jepsen worker 4 - jepsen.util 8194 :invoke :read [1 nil] INFO [2019-12-17 12:34:38,682] jepsen worker 64 - jepsen.util 6994 :ok :read [21 22] INFO [2019-12-17 12:34:38,682] jepsen worker 64 - jepsen.util 6994 :invoke :read [21 nil] INFO [2019-12-17 12:34:38,696] jepsen worker 86 - jepsen.util 12506 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:34:38,696] jepsen worker 86 - jepsen.util 12506 :invoke :write [28 49] INFO [2019-12-17 12:34:38,703] jepsen worker 63 - jepsen.util 7443 :ok :write [21 20] INFO [2019-12-17 12:34:38,703] jepsen worker 63 - jepsen.util 7443 :invoke :cas [21 [15 41]] INFO [2019-12-17 12:34:38,704] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":49623,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:34:38,704] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":49623,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:34:38,704] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":49627,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:34:38,704] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":49627,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:34:38,708] jepsen worker 55 - jepsen.util 9415 :ok :read [18 17] INFO [2019-12-17 12:34:38,708] jepsen worker 55 - jepsen.util 9415 :invoke :read [18 nil] INFO [2019-12-17 12:34:38,710] jepsen worker 64 - jepsen.util 6994 :ok :read [21 20] INFO [2019-12-17 12:34:38,710] jepsen worker 64 - jepsen.util 6994 :invoke :read [21 nil] INFO [2019-12-17 12:34:38,720] jepsen worker 53 - jepsen.util 7793 :ok :write [17 18] INFO [2019-12-17 12:34:38,721] jepsen worker 53 - jepsen.util 7793 :invoke :write [17 33] INFO [2019-12-17 12:34:38,730] jepsen worker 59 - jepsen.util 11219 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:34:38,730] jepsen worker 21 - jepsen.util 11181 :info :write [7 7] :RequestCanceledException INFO [2019-12-17 12:34:38,730] jepsen worker 59 - jepsen.util 11219 :invoke :read [19 nil] INFO [2019-12-17 12:34:38,731] jepsen worker 21 - jepsen.util 11271 :invoke :cas [7 [0 38]] INFO [2019-12-17 12:34:38,737] jepsen worker 28 - jepsen.util 6868 :ok :write [9 21] INFO [2019-12-17 12:34:38,737] jepsen worker 28 - jepsen.util 6868 :invoke :cas [9 [35 42]] INFO [2019-12-17 12:34:38,743] jepsen worker 39 - jepsen.util 11109 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:34:38,743] jepsen worker 51 - jepsen.util 7881 :info :write [17 8] :RequestCanceledException INFO [2019-12-17 12:34:38,744] jepsen worker 39 - jepsen.util 11109 :invoke :read [13 nil] INFO [2019-12-17 12:34:38,745] jepsen worker 51 - jepsen.util 7971 :invoke :read [17 nil] INFO [2019-12-17 12:34:38,754] jepsen worker 64 - jepsen.util 6994 :ok :read [21 20] INFO [2019-12-17 12:34:38,754] jepsen worker 64 - jepsen.util 6994 :invoke :cas [21 [22 25]] INFO [2019-12-17 12:34:38,773] jepsen worker 9 - jepsen.util 9009 :ok :write [3 12] INFO [2019-12-17 12:34:38,774] jepsen worker 9 - jepsen.util 9009 :invoke :cas [3 [15 40]] INFO [2019-12-17 12:34:38,779] jepsen worker 64 - jepsen.util 6994 :fail :cas [21 [22 25]] :ValueNotSwapFrom INFO [2019-12-17 12:34:38,779] jepsen worker 28 - jepsen.util 6868 :fail :cas [9 [35 42]] :ValueNotSwapFrom INFO [2019-12-17 12:34:38,779] jepsen worker 64 - jepsen.util 6994 :invoke :read [21 nil] INFO [2019-12-17 12:34:38,779] jepsen worker 28 - jepsen.util 6868 :invoke :cas [9 [45 31]] INFO [2019-12-17 12:34:38,795] jepsen worker 14 - jepsen.util 8744 :info :write [4 4] :RequestCanceledException INFO [2019-12-17 12:34:38,797] jepsen worker 14 - jepsen.util 8834 :invoke :read [4 nil] INFO [2019-12-17 12:34:38,801] jepsen worker 80 - jepsen.util 8270 :fail :cas [26 [14 46]] :ValueNotSwapFrom INFO [2019-12-17 12:34:38,801] jepsen worker 80 - jepsen.util 8270 :invoke :read [26 nil] INFO [2019-12-17 12:34:38,805] jepsen worker 47 - jepsen.util 9047 :fail :cas [15 [10 20]] :ValueNotSwapFrom INFO [2019-12-17 12:34:38,805] jepsen worker 47 - jepsen.util 9047 :invoke :read [15 nil] INFO [2019-12-17 12:34:38,811] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:34:38,811] jepsen nemesis - couchbase.util Rebalance check count: 384 INFO [2019-12-17 12:34:38,811] jepsen nemesis - couchbase.util Rebalance stuck count: 365 INFO [2019-12-17 12:34:38,830] jepsen worker 68 - jepsen.util 11048 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:34:38,830] jepsen worker 3 - jepsen.util 8373 :info :write [1 18] :RequestCanceledException INFO [2019-12-17 12:34:38,830] jepsen worker 82 - jepsen.util 12322 :info :cas [27 [4 34]] :RequestCanceledException INFO [2019-12-17 12:34:38,830] jepsen worker 68 - jepsen.util 11048 :invoke :read [22 nil] INFO [2019-12-17 12:34:38,831] jepsen worker 3 - jepsen.util 8463 :invoke :read [1 nil] INFO [2019-12-17 12:34:38,831] jepsen worker 82 - jepsen.util 12412 :invoke :read [27 nil] INFO [2019-12-17 12:34:38,836] jepsen worker 78 - jepsen.util 8358 :info :write [26 11] :RequestCanceledException INFO [2019-12-17 12:34:38,836] jepsen worker 72 - jepsen.util 8532 :info :cas [24 [41 37]] :RequestCanceledException INFO [2019-12-17 12:34:38,838] jepsen worker 78 - jepsen.util 8448 :invoke :cas [26 [42 35]] INFO [2019-12-17 12:34:38,838] jepsen worker 72 - jepsen.util 8622 :invoke :cas [24 [12 23]] INFO [2019-12-17 12:34:38,841] jepsen worker 64 - jepsen.util 6994 :ok :read [21 20] INFO [2019-12-17 12:34:38,842] jepsen worker 64 - jepsen.util 6994 :invoke :read [21 nil] INFO [2019-12-17 12:34:38,846] jepsen worker 74 - jepsen.util 8804 :ok :write [24 35] INFO [2019-12-17 12:34:38,846] jepsen worker 74 - jepsen.util 8804 :invoke :write [24 34] INFO [2019-12-17 12:34:38,850] jepsen worker 9 - jepsen.util 9009 :fail :cas [3 [15 40]] :ValueNotSwapFrom INFO [2019-12-17 12:34:38,850] jepsen worker 9 - jepsen.util 9009 :invoke :read [3 nil] INFO [2019-12-17 12:34:38,850] jepsen worker 28 - jepsen.util 6868 :fail :cas [9 [45 31]] :ValueNotSwapFrom INFO [2019-12-17 12:34:38,850] jepsen worker 28 - jepsen.util 6868 :invoke :cas [9 [28 1]] INFO [2019-12-17 12:34:38,861] jepsen worker 76 - jepsen.util 11776 :info :write [25 43] :RequestCanceledException INFO [2019-12-17 12:34:38,862] jepsen worker 76 - jepsen.util 11866 :invoke :cas [25 [36 44]] INFO [2019-12-17 12:34:38,864] jepsen worker 12 - jepsen.util 9102 :ok :write [4 33] INFO [2019-12-17 12:34:38,864] jepsen worker 12 - jepsen.util 9102 :invoke :write [4 27] INFO [2019-12-17 12:34:38,870] jepsen worker 4 - jepsen.util 8194 :ok :read [1 49] INFO [2019-12-17 12:34:38,871] jepsen worker 4 - jepsen.util 8194 :invoke :write [1 13] INFO [2019-12-17 12:34:38,892] jepsen worker 52 - jepsen.util 8692 :info :write [17 18] :RequestCanceledException INFO [2019-12-17 12:34:38,892] jepsen worker 11 - jepsen.util 8471 :info :write [3 25] :RequestCanceledException INFO [2019-12-17 12:34:38,893] jepsen worker 52 - jepsen.util 8782 :invoke :write [17 36] INFO [2019-12-17 12:34:38,893] jepsen worker 11 - jepsen.util 8561 :invoke :write [3 33] INFO [2019-12-17 12:34:38,895] jepsen worker 64 - jepsen.util 6994 :ok :read [21 20] INFO [2019-12-17 12:34:38,895] jepsen worker 64 - jepsen.util 6994 :invoke :cas [21 [30 22]] INFO [2019-12-17 12:34:38,896] jepsen worker 72 - jepsen.util 8622 :fail :cas [24 [12 23]] :ValueNotSwapFrom INFO [2019-12-17 12:34:38,897] jepsen worker 72 - jepsen.util 8622 :invoke :read [24 nil] INFO [2019-12-17 12:34:38,900] jepsen worker 47 - jepsen.util 9047 :ok :read [15 13] INFO [2019-12-17 12:34:38,900] jepsen worker 47 - jepsen.util 9047 :invoke :read [15 nil] INFO [2019-12-17 12:34:38,913] jepsen worker 29 - jepsen.util 6959 :info :write [9 29] :RequestCanceledException INFO [2019-12-17 12:34:38,913] jepsen worker 23 - jepsen.util 11363 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:34:38,913] jepsen worker 23 - jepsen.util 11363 :invoke :write [7 37] INFO [2019-12-17 12:34:38,914] jepsen worker 29 - jepsen.util 7049 :invoke :read [9 nil] INFO [2019-12-17 12:34:38,914] jepsen worker 27 - jepsen.util 8307 :ok :write [9 11] INFO [2019-12-17 12:34:38,914] jepsen worker 27 - jepsen.util 8307 :invoke :write [9 20] INFO [2019-12-17 12:34:38,919] jepsen worker 28 - jepsen.util 6868 :fail :cas [9 [28 1]] :ValueNotSwapFrom INFO [2019-12-17 12:34:38,919] jepsen worker 28 - jepsen.util 6868 :invoke :cas [9 [34 45]] INFO [2019-12-17 12:34:38,929] jepsen worker 50 - jepsen.util 12380 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:34:38,929] jepsen worker 50 - jepsen.util 12380 :invoke :cas [16 [47 24]] INFO [2019-12-17 12:34:38,931] jepsen worker 70 - jepsen.util 9070 :fail :cas [23 [19 39]] :ValueNotSwapFrom INFO [2019-12-17 12:34:38,931] jepsen worker 70 - jepsen.util 9070 :invoke :write [23 8] INFO [2019-12-17 12:34:38,932] jepsen worker 71 - jepsen.util 9161 :ok :read [23 9] INFO [2019-12-17 12:34:38,932] jepsen worker 71 - jepsen.util 9161 :invoke :cas [23 [7 28]] INFO [2019-12-17 12:34:38,935] jepsen worker 64 - jepsen.util 6994 :fail :cas [21 [30 22]] :ValueNotSwapFrom INFO [2019-12-17 12:34:38,935] jepsen worker 64 - jepsen.util 6994 :invoke :write [21 8] INFO [2019-12-17 12:34:38,940] jepsen worker 72 - jepsen.util 8622 :ok :read [24 35] INFO [2019-12-17 12:34:38,940] jepsen worker 72 - jepsen.util 8622 :invoke :read [24 nil] INFO [2019-12-17 12:34:38,941] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":49662,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:34:38,941] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":49662,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:34:38,941] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":49665,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:34:38,941] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":49665,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:34:38,941] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":49666,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:34:38,941] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":49666,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:34:38,944] jepsen worker 60 - jepsen.util 12660 :info :write [20 9] :RequestCanceledException INFO [2019-12-17 12:34:38,944] jepsen worker 34 - jepsen.util 11644 :info :write [11 48] :RequestCanceledException INFO [2019-12-17 12:34:38,944] jepsen worker 51 - jepsen.util 7971 :ok :read [17 18] INFO [2019-12-17 12:34:38,944] jepsen worker 51 - jepsen.util 7971 :invoke :write [17 25] INFO [2019-12-17 12:34:38,945] jepsen worker 34 - jepsen.util 11734 :invoke :cas [11 [28 14]] INFO [2019-12-17 12:34:38,945] jepsen worker 60 - jepsen.util 12750 :invoke :read [20 nil] INFO [2019-12-17 12:34:38,960] jepsen worker 13 - jepsen.util 8293 :fail :cas [4 [7 38]] :ValueNotSwapFrom INFO [2019-12-17 12:34:38,960] jepsen worker 13 - jepsen.util 8293 :invoke :write [4 1] INFO [2019-12-17 12:34:38,961] jepsen worker 83 - jepsen.util 12053 :info :cas [27 [17 30]] :RequestCanceledException INFO [2019-12-17 12:34:38,962] jepsen worker 83 - jepsen.util 12143 :invoke :cas [27 [5 23]] INFO [2019-12-17 12:34:38,968] jepsen worker 88 - jepsen.util 8818 :fail :cas [29 [12 39]] :ValueNotSwapFrom INFO [2019-12-17 12:34:38,969] jepsen worker 88 - jepsen.util 8818 :invoke :read [29 nil] INFO [2019-12-17 12:34:38,972] jepsen worker 55 - jepsen.util 9415 :ok :read [18 17] INFO [2019-12-17 12:34:38,972] jepsen worker 55 - jepsen.util 9415 :invoke :read [18 nil] INFO [2019-12-17 12:34:38,982] jepsen worker 14 - jepsen.util 8834 :ok :read [4 33] INFO [2019-12-17 12:34:38,982] jepsen worker 14 - jepsen.util 8834 :invoke :read [4 nil] INFO [2019-12-17 12:34:38,986] jepsen worker 9 - jepsen.util 9009 :ok :read [3 12] INFO [2019-12-17 12:34:38,986] jepsen worker 9 - jepsen.util 9009 :invoke :cas [3 [48 4]] INFO [2019-12-17 12:34:39,006] jepsen worker 29 - jepsen.util 7049 :ok :read [9 11] INFO [2019-12-17 12:34:39,006] jepsen worker 29 - jepsen.util 7049 :invoke :write [9 11] INFO [2019-12-17 12:34:39,012] jepsen worker 80 - jepsen.util 8270 :ok :read [26 2] INFO [2019-12-17 12:34:39,012] jepsen worker 80 - jepsen.util 8270 :invoke :read [26 nil] INFO [2019-12-17 12:34:39,014] jepsen worker 28 - jepsen.util 6868 :fail :cas [9 [34 45]] :ValueNotSwapFrom INFO [2019-12-17 12:34:39,014] jepsen worker 28 - jepsen.util 6868 :invoke :read [9 nil] INFO [2019-12-17 12:34:39,023] jepsen worker 72 - jepsen.util 8622 :ok :read [24 35] INFO [2019-12-17 12:34:39,023] jepsen worker 72 - jepsen.util 8622 :invoke :read [24 nil] INFO [2019-12-17 12:34:39,029] jepsen worker 16 - jepsen.util 12616 :info :write [5 23] :RequestCanceledException INFO [2019-12-17 12:34:39,031] jepsen worker 16 - jepsen.util 12706 :invoke :read [5 nil] INFO [2019-12-17 12:34:39,031] jepsen worker 28 - jepsen.util 6868 :ok :read [9 11] INFO [2019-12-17 12:34:39,031] jepsen worker 28 - jepsen.util 6868 :invoke :write [9 7] INFO [2019-12-17 12:34:39,031] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":49693,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:34:39,031] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":49693,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:34:39,031] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":49696,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:34:39,032] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":49696,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:34:39,032] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":49697,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:34:39,032] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":49697,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:34:39,035] jepsen worker 51 - jepsen.util 7971 :ok :write [17 25] INFO [2019-12-17 12:34:39,035] jepsen worker 51 - jepsen.util 7971 :invoke :cas [17 [22 46]] INFO [2019-12-17 12:34:39,043] jepsen worker 72 - jepsen.util 8622 :ok :read [24 35] INFO [2019-12-17 12:34:39,044] jepsen worker 72 - jepsen.util 8622 :invoke :cas [24 [10 5]] INFO [2019-12-17 12:34:39,096] jepsen worker 18 - jepsen.util 8748 :info :write [6 35] :RequestCanceledException INFO [2019-12-17 12:34:39,096] jepsen worker 37 - jepsen.util 8137 :info :write [12 8] :RequestCanceledException INFO [2019-12-17 12:34:39,097] jepsen worker 18 - jepsen.util 8838 :invoke :write [6 23] INFO [2019-12-17 12:34:39,097] jepsen worker 37 - jepsen.util 8227 :invoke :read [12 nil] INFO [2019-12-17 12:34:39,097] jepsen worker 28 - jepsen.util 6868 :ok :write [9 7] INFO [2019-12-17 12:34:39,097] jepsen worker 28 - jepsen.util 6868 :invoke :write [9 49] INFO [2019-12-17 12:34:39,113] jepsen worker 20 - jepsen.util 8300 :fail :read [6 nil] :RequestTimeoutException INFO [2019-12-17 12:34:39,113] jepsen worker 20 - jepsen.util 8300 :invoke :read [6 nil] INFO [2019-12-17 12:34:39,130] jepsen worker 67 - jepsen.util 11767 :info :cas [22 [12 21]] :RequestCanceledException INFO [2019-12-17 12:34:39,131] jepsen worker 67 - jepsen.util 11857 :invoke :write [22 23] INFO [2019-12-17 12:34:39,152] jepsen worker 72 - jepsen.util 8622 :fail :cas [24 [10 5]] :ValueNotSwapFrom INFO [2019-12-17 12:34:39,152] jepsen worker 72 - jepsen.util 8622 :invoke :cas [24 [33 40]] INFO [2019-12-17 12:34:39,177] jepsen worker 20 - jepsen.util 8300 :ok :read [6 27] INFO [2019-12-17 12:34:39,177] jepsen worker 20 - jepsen.util 8300 :invoke :write [6 26] INFO [2019-12-17 12:34:39,180] jepsen worker 13 - jepsen.util 8293 :ok :write [4 1] INFO [2019-12-17 12:34:39,180] jepsen worker 13 - jepsen.util 8293 :invoke :read [4 nil] INFO [2019-12-17 12:34:39,190] jepsen worker 74 - jepsen.util 8804 :ok :write [24 34] INFO [2019-12-17 12:34:39,191] jepsen worker 74 - jepsen.util 8804 :invoke :cas [24 [48 2]] INFO [2019-12-17 12:34:39,202] jepsen worker 5 - jepsen.util 8015 :ok :write [1 6] INFO [2019-12-17 12:34:39,203] jepsen worker 5 - jepsen.util 8015 :invoke :read [1 nil] INFO [2019-12-17 12:34:39,204] jepsen worker 80 - jepsen.util 8270 :ok :read [26 2] INFO [2019-12-17 12:34:39,204] jepsen worker 80 - jepsen.util 8270 :invoke :cas [26 [43 34]] INFO [2019-12-17 12:34:39,214] jepsen worker 69 - jepsen.util 8709 :ok :write [23 22] INFO [2019-12-17 12:34:39,215] jepsen worker 69 - jepsen.util 8709 :invoke :write [23 1] INFO [2019-12-17 12:34:39,216] jepsen worker 11 - jepsen.util 8561 :ok :write [3 33] INFO [2019-12-17 12:34:39,216] jepsen worker 11 - jepsen.util 8561 :invoke :read [3 nil] INFO [2019-12-17 12:34:39,227] jepsen worker 74 - jepsen.util 8804 :fail :cas [24 [48 2]] :ValueNotSwapFrom INFO [2019-12-17 12:34:39,228] jepsen worker 74 - jepsen.util 8804 :invoke :read [24 nil] INFO [2019-12-17 12:34:39,239] jepsen worker 78 - jepsen.util 8448 :fail :cas [26 [42 35]] :ValueNotSwapFrom INFO [2019-12-17 12:34:39,240] jepsen worker 78 - jepsen.util 8448 :invoke :write [26 20] INFO [2019-12-17 12:34:39,251] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":49724,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:34:39,251] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":49724,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:34:39,251] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":49727,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:34:39,251] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":49727,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:34:39,251] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":49728,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:34:39,251] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":49728,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:34:39,255] jepsen worker 5 - jepsen.util 8015 :ok :read [1 6] INFO [2019-12-17 12:34:39,255] jepsen worker 5 - jepsen.util 8015 :invoke :cas [1 [5 14]] INFO [2019-12-17 12:34:39,264] jepsen worker 74 - jepsen.util 8804 :ok :read [24 34] INFO [2019-12-17 12:34:39,265] jepsen worker 74 - jepsen.util 8804 :invoke :write [24 20] INFO [2019-12-17 12:34:39,267] jepsen worker 64 - jepsen.util 6994 :ok :write [21 8] INFO [2019-12-17 12:34:39,268] jepsen worker 64 - jepsen.util 6994 :invoke :write [21 35] INFO [2019-12-17 12:34:39,286] jepsen worker 88 - jepsen.util 8818 :ok :read [29 2] INFO [2019-12-17 12:34:39,286] jepsen worker 88 - jepsen.util 8818 :invoke :cas [29 [1 16]] INFO [2019-12-17 12:34:39,293] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":49715,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:34:39,293] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":49715,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:34:39,293] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":49716,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:34:39,293] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":49716,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:34:39,293] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":49717,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:34:39,293] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":49717,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:34:39,296] jepsen worker 35 - jepsen.util 11375 :info :cas [11 [35 43]] :RequestCanceledException INFO [2019-12-17 12:34:39,297] jepsen worker 35 - jepsen.util 11465 :invoke :write [11 43] INFO [2019-12-17 12:34:39,302] jepsen worker 56 - jepsen.util 9416 :ok :write [18 6] INFO [2019-12-17 12:34:39,302] jepsen worker 56 - jepsen.util 9416 :invoke :cas [18 [42 29]] INFO [2019-12-17 12:34:39,313] jepsen worker 88 - jepsen.util 8818 :fail :cas [29 [1 16]] :ValueNotSwapFrom INFO [2019-12-17 12:34:39,314] jepsen worker 88 - jepsen.util 8818 :invoke :write [29 47] INFO [2019-12-17 12:34:39,314] jepsen worker 31 - jepsen.util 7591 :info :cas [10 [24 1]] :RequestCanceledException INFO [2019-12-17 12:34:39,315] jepsen worker 31 - jepsen.util 7681 :invoke :write [10 20] INFO [2019-12-17 12:34:39,328] jepsen worker 51 - jepsen.util 7971 :fail :cas [17 [22 46]] :ValueNotSwapFrom INFO [2019-12-17 12:34:39,328] jepsen worker 51 - jepsen.util 7971 :invoke :write [17 0] INFO [2019-12-17 12:34:39,360] jepsen worker 18 - jepsen.util 8838 :ok :write [6 23] INFO [2019-12-17 12:34:39,360] jepsen worker 18 - jepsen.util 8838 :invoke :read [6 nil] INFO [2019-12-17 12:34:39,362] jepsen worker 3 - jepsen.util 8463 :ok :read [1 6] INFO [2019-12-17 12:34:39,362] jepsen worker 3 - jepsen.util 8463 :invoke :read [1 nil] INFO [2019-12-17 12:34:39,383] jepsen worker 28 - jepsen.util 6868 :ok :write [9 49] INFO [2019-12-17 12:34:39,383] jepsen worker 28 - jepsen.util 6868 :invoke :cas [9 [44 21]] INFO [2019-12-17 12:34:39,385] jepsen worker 18 - jepsen.util 8838 :ok :read [6 23] INFO [2019-12-17 12:34:39,385] jepsen worker 18 - jepsen.util 8838 :invoke :cas [6 [10 15]] INFO [2019-12-17 12:34:39,405] jepsen worker 64 - jepsen.util 6994 :ok :write [21 35] INFO [2019-12-17 12:34:39,406] jepsen worker 64 - jepsen.util 6994 :invoke :read [21 nil] INFO [2019-12-17 12:34:39,411] jepsen worker 18 - jepsen.util 8838 :fail :cas [6 [10 15]] :ValueNotSwapFrom INFO [2019-12-17 12:34:39,411] jepsen worker 18 - jepsen.util 8838 :invoke :cas [6 [10 23]] INFO [2019-12-17 12:34:39,414] jepsen worker 19 - jepsen.util 8659 :fail :read [6 nil] :RequestTimeoutException INFO [2019-12-17 12:34:39,414] jepsen worker 19 - jepsen.util 8659 :invoke :write [6 29] INFO [2019-12-17 12:34:39,427] jepsen worker 55 - jepsen.util 9415 :ok :read [18 6] INFO [2019-12-17 12:34:39,427] jepsen worker 55 - jepsen.util 9415 :invoke :read [18 nil] INFO [2019-12-17 12:34:39,429] jepsen worker 30 - jepsen.util 8580 :info :cas [10 [42 5]] :RequestCanceledException INFO [2019-12-17 12:34:39,429] jepsen worker 32 - jepsen.util 8762 :fail :read [10 nil] :RequestTimeoutException INFO [2019-12-17 12:34:39,429] jepsen worker 32 - jepsen.util 8762 :invoke :read [10 nil] INFO [2019-12-17 12:34:39,430] jepsen worker 30 - jepsen.util 8670 :invoke :read [10 nil] INFO [2019-12-17 12:34:39,442] jepsen worker 14 - jepsen.util 8834 :ok :read [4 1] INFO [2019-12-17 12:34:39,442] jepsen worker 33 - jepsen.util 11733 :info :write [11 22] :RequestCanceledException INFO [2019-12-17 12:34:39,442] jepsen worker 14 - jepsen.util 8834 :invoke :cas [4 [7 7]] INFO [2019-12-17 12:34:39,443] jepsen worker 33 - jepsen.util 11823 :invoke :write [11 25] INFO [2019-12-17 12:34:39,451] jepsen worker 71 - jepsen.util 9161 :fail :cas [23 [7 28]] :ValueNotSwapFrom INFO [2019-12-17 12:34:39,451] jepsen worker 71 - jepsen.util 9161 :invoke :cas [23 [10 7]] INFO [2019-12-17 12:34:39,463] jepsen worker 74 - jepsen.util 8804 :ok :write [24 20] INFO [2019-12-17 12:34:39,463] jepsen worker 74 - jepsen.util 8804 :invoke :write [24 40] INFO [2019-12-17 12:34:39,468] jepsen worker 28 - jepsen.util 6868 :fail :cas [9 [44 21]] :ValueNotSwapFrom INFO [2019-12-17 12:34:39,468] jepsen worker 28 - jepsen.util 6868 :invoke :read [9 nil] INFO [2019-12-17 12:34:39,470] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":49753,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:34:39,470] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":49753,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:34:39,470] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":49756,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:34:39,470] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":49756,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:34:39,483] jepsen worker 64 - jepsen.util 6994 :ok :read [21 35] INFO [2019-12-17 12:34:39,483] jepsen worker 64 - jepsen.util 6994 :invoke :read [21 nil] INFO [2019-12-17 12:34:39,491] jepsen worker 18 - jepsen.util 8838 :fail :cas [6 [10 23]] :ValueNotSwapFrom INFO [2019-12-17 12:34:39,492] jepsen worker 18 - jepsen.util 8838 :invoke :read [6 nil] INFO [2019-12-17 12:34:39,493] jepsen worker 5 - jepsen.util 8015 :fail :cas [1 [5 14]] :ValueNotSwapFrom INFO [2019-12-17 12:34:39,493] jepsen worker 5 - jepsen.util 8015 :invoke :cas [1 [2 3]] INFO [2019-12-17 12:34:39,497] jepsen worker 71 - jepsen.util 9161 :fail :cas [23 [10 7]] :ValueNotSwapFrom INFO [2019-12-17 12:34:39,498] jepsen worker 71 - jepsen.util 9161 :invoke :read [23 nil] INFO [2019-12-17 12:34:39,508] jepsen worker 80 - jepsen.util 8270 :fail :cas [26 [43 34]] :ValueNotSwapFrom INFO [2019-12-17 12:34:39,509] jepsen worker 80 - jepsen.util 8270 :invoke :write [26 28] INFO [2019-12-17 12:34:39,528] jepsen worker 5 - jepsen.util 8015 :fail :cas [1 [2 3]] :ValueNotSwapFrom INFO [2019-12-17 12:34:39,528] jepsen worker 5 - jepsen.util 8015 :invoke :write [1 12] INFO [2019-12-17 12:34:39,529] jepsen worker 71 - jepsen.util 9161 :ok :read [23 22] INFO [2019-12-17 12:34:39,529] jepsen worker 71 - jepsen.util 9161 :invoke :cas [23 [43 28]] INFO [2019-12-17 12:34:39,547] jepsen worker 18 - jepsen.util 8838 :ok :read [6 23] INFO [2019-12-17 12:34:39,547] jepsen worker 18 - jepsen.util 8838 :invoke :read [6 nil] INFO [2019-12-17 12:34:39,549] jepsen worker 32 - jepsen.util 8762 :ok :read [10 23] INFO [2019-12-17 12:34:39,549] jepsen worker 32 - jepsen.util 8762 :invoke :read [10 nil] INFO [2019-12-17 12:34:39,564] jepsen worker 64 - jepsen.util 6994 :ok :read [21 35] INFO [2019-12-17 12:34:39,564] jepsen worker 64 - jepsen.util 6994 :invoke :read [21 nil] INFO [2019-12-17 12:34:39,588] jepsen worker 18 - jepsen.util 8838 :ok :read [6 23] INFO [2019-12-17 12:34:39,588] jepsen worker 37 - jepsen.util 8227 :ok :read [12 40] INFO [2019-12-17 12:34:39,588] jepsen worker 18 - jepsen.util 8838 :invoke :read [6 nil] INFO [2019-12-17 12:34:39,588] jepsen worker 37 - jepsen.util 8227 :invoke :write [12 8] INFO [2019-12-17 12:34:39,593] jepsen worker 71 - jepsen.util 9161 :fail :cas [23 [43 28]] :ValueNotSwapFrom INFO [2019-12-17 12:34:39,593] jepsen worker 71 - jepsen.util 9161 :invoke :cas [23 [49 23]] INFO [2019-12-17 12:34:39,595] jepsen worker 62 - jepsen.util 12032 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:34:39,596] jepsen worker 62 - jepsen.util 12032 :invoke :write [20 14] INFO [2019-12-17 12:34:39,606] jepsen worker 28 - jepsen.util 6868 :ok :read [9 49] INFO [2019-12-17 12:34:39,606] jepsen worker 28 - jepsen.util 6868 :invoke :cas [9 [22 36]] INFO [2019-12-17 12:34:39,613] jepsen worker 64 - jepsen.util 6994 :ok :read [21 35] INFO [2019-12-17 12:34:39,613] jepsen worker 64 - jepsen.util 6994 :invoke :write [21 36] INFO [2019-12-17 12:34:39,624] jepsen worker 32 - jepsen.util 8762 :ok :read [10 23] INFO [2019-12-17 12:34:39,624] jepsen worker 32 - jepsen.util 8762 :invoke :read [10 nil] INFO [2019-12-17 12:34:39,629] jepsen worker 57 - jepsen.util 11577 :info :cas [19 [33 24]] :RequestCanceledException INFO [2019-12-17 12:34:39,629] jepsen worker 65 - jepsen.util 6995 :info :write [21 40] :RequestCanceledException INFO [2019-12-17 12:34:39,630] jepsen worker 57 - jepsen.util 11667 :invoke :read [19 nil] INFO [2019-12-17 12:34:39,630] jepsen worker 0 - jepsen.util 12150 :info :cas [0 [20 39]] :RequestCanceledException INFO [2019-12-17 12:34:39,630] jepsen worker 65 - jepsen.util 7085 :invoke :read [21 nil] INFO [2019-12-17 12:34:39,631] jepsen worker 0 - jepsen.util 12240 :invoke :write [0 0] INFO [2019-12-17 12:34:39,636] jepsen worker 13 - jepsen.util 8293 :ok :read [4 1] INFO [2019-12-17 12:34:39,637] jepsen worker 13 - jepsen.util 8293 :invoke :cas [4 [18 35]] INFO [2019-12-17 12:34:39,638] jepsen worker 18 - jepsen.util 8838 :ok :read [6 23] INFO [2019-12-17 12:34:39,638] jepsen worker 18 - jepsen.util 8838 :invoke :write [6 14] INFO [2019-12-17 12:34:39,653] jepsen worker 28 - jepsen.util 6868 :fail :cas [9 [22 36]] :ValueNotSwapFrom INFO [2019-12-17 12:34:39,653] jepsen worker 28 - jepsen.util 6868 :invoke :cas [9 [21 34]] INFO [2019-12-17 12:34:39,661] jepsen worker 56 - jepsen.util 9416 :fail :cas [18 [42 29]] :ValueNotSwapFrom INFO [2019-12-17 12:34:39,661] jepsen worker 25 - jepsen.util 12265 :info :write [8 2] :RequestCanceledException INFO [2019-12-17 12:34:39,661] jepsen worker 56 - jepsen.util 9416 :invoke :cas [18 [21 17]] INFO [2019-12-17 12:34:39,662] jepsen worker 25 - jepsen.util 12355 :invoke :cas [8 [21 39]] INFO [2019-12-17 12:34:39,674] jepsen worker 3 - jepsen.util 8463 :ok :read [1 6] INFO [2019-12-17 12:34:39,674] jepsen worker 3 - jepsen.util 8463 :invoke :read [1 nil] INFO [2019-12-17 12:34:39,689] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":49785,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:34:39,689] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":49785,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:34:39,689] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":49788,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:34:39,689] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":49788,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:34:39,689] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":49789,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:34:39,689] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":49789,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:34:39,706] jepsen worker 32 - jepsen.util 8762 :ok :read [10 23] INFO [2019-12-17 12:34:39,706] jepsen worker 32 - jepsen.util 8762 :invoke :write [10 22] INFO [2019-12-17 12:34:39,714] jepsen worker 58 - jepsen.util 12568 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:34:39,714] jepsen worker 58 - jepsen.util 12568 :invoke :write [19 12] INFO [2019-12-17 12:34:39,734] jepsen worker 56 - jepsen.util 9416 :fail :cas [18 [21 17]] :ValueNotSwapFrom INFO [2019-12-17 12:34:39,734] jepsen worker 56 - jepsen.util 9416 :invoke :read [18 nil] INFO [2019-12-17 12:34:39,741] jepsen worker 3 - jepsen.util 8463 :ok :read [1 6] INFO [2019-12-17 12:34:39,742] jepsen worker 3 - jepsen.util 8463 :invoke :read [1 nil] INFO [2019-12-17 12:34:39,743] jepsen worker 28 - jepsen.util 6868 :fail :cas [9 [21 34]] :ValueNotSwapFrom INFO [2019-12-17 12:34:39,743] jepsen worker 28 - jepsen.util 6868 :invoke :cas [9 [24 45]] INFO [2019-12-17 12:34:39,743] jepsen worker 26 - jepsen.util 12176 :info :cas [8 [46 26]] :RequestCanceledException INFO [2019-12-17 12:34:39,744] jepsen worker 26 - jepsen.util 12266 :invoke :write [8 41] INFO [2019-12-17 12:34:39,760] jepsen worker 85 - jepsen.util 12505 :info :cas [28 [48 18]] :RequestCanceledException INFO [2019-12-17 12:34:39,761] jepsen worker 85 - jepsen.util 12595 :invoke :read [28 nil] INFO [2019-12-17 12:34:39,762] jepsen worker 64 - jepsen.util 6994 :ok :write [21 36] INFO [2019-12-17 12:34:39,762] jepsen worker 64 - jepsen.util 6994 :invoke :write [21 0] INFO [2019-12-17 12:34:39,778] jepsen worker 71 - jepsen.util 9161 :fail :cas [23 [49 23]] :ValueNotSwapFrom INFO [2019-12-17 12:34:39,778] jepsen worker 71 - jepsen.util 9161 :invoke :read [23 nil] INFO [2019-12-17 12:34:39,802] jepsen worker 13 - jepsen.util 8293 :fail :cas [4 [18 35]] :ValueNotSwapFrom INFO [2019-12-17 12:34:39,802] jepsen worker 13 - jepsen.util 8293 :invoke :read [4 nil] INFO [2019-12-17 12:34:39,805] jepsen worker 64 - jepsen.util 6994 :ok :write [21 0] INFO [2019-12-17 12:34:39,805] jepsen worker 64 - jepsen.util 6994 :invoke :cas [21 [4 10]] INFO [2019-12-17 12:34:39,819] jepsen worker 64 - jepsen.util 6994 :fail :cas [21 [4 10]] :ValueNotSwapFrom INFO [2019-12-17 12:34:39,819] jepsen worker 64 - jepsen.util 6994 :invoke :read [21 nil] INFO [2019-12-17 12:34:39,822] jepsen worker 56 - jepsen.util 9416 :ok :read [18 6] INFO [2019-12-17 12:34:39,822] jepsen worker 56 - jepsen.util 9416 :invoke :write [18 13] INFO [2019-12-17 12:34:39,828] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:34:39,828] jepsen nemesis - couchbase.util Rebalance check count: 385 INFO [2019-12-17 12:34:39,828] jepsen nemesis - couchbase.util Rebalance stuck count: 366 INFO [2019-12-17 12:34:39,835] jepsen worker 64 - jepsen.util 6994 :ok :read [21 0] INFO [2019-12-17 12:34:39,835] jepsen worker 64 - jepsen.util 6994 :invoke :write [21 23] INFO [2019-12-17 12:34:39,843] jepsen worker 22 - jepsen.util 11632 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:34:39,843] jepsen worker 48 - jepsen.util 10848 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:34:39,843] jepsen worker 22 - jepsen.util 11632 :invoke :write [7 41] INFO [2019-12-17 12:34:39,843] jepsen worker 48 - jepsen.util 10848 :invoke :read [16 nil] INFO [2019-12-17 12:34:39,847] jepsen worker 3 - jepsen.util 8463 :ok :read [1 6] INFO [2019-12-17 12:34:39,847] jepsen worker 3 - jepsen.util 8463 :invoke :write [1 1] INFO [2019-12-17 12:34:39,879] jepsen worker 64 - jepsen.util 6994 :ok :write [21 23] INFO [2019-12-17 12:34:39,879] jepsen worker 64 - jepsen.util 6994 :invoke :write [21 38] INFO [2019-12-17 12:34:39,892] jepsen worker 6 - jepsen.util 12156 :info :cas [2 [42 14]] :RequestCanceledException INFO [2019-12-17 12:34:39,893] jepsen worker 6 - jepsen.util 12246 :invoke :cas [2 [27 43]] INFO [2019-12-17 12:34:39,914] jepsen worker 61 - jepsen.util 12391 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:34:39,914] jepsen worker 44 - jepsen.util 12014 :info :cas [14 [45 40]] :RequestCanceledException INFO [2019-12-17 12:34:39,914] jepsen worker 61 - jepsen.util 12391 :invoke :write [20 46] INFO [2019-12-17 12:34:39,915] jepsen worker 44 - jepsen.util 12104 :invoke :cas [14 [9 9]] INFO [2019-12-17 12:34:39,924] jepsen worker 64 - jepsen.util 6994 :ok :write [21 38] INFO [2019-12-17 12:34:39,924] jepsen worker 64 - jepsen.util 6994 :invoke :read [21 nil] INFO [2019-12-17 12:34:39,928] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":49813,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:34:39,928] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":49813,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:34:39,928] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":49816,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:34:39,928] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":49816,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:34:39,928] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":49817,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:34:39,928] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":49817,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:34:39,930] jepsen worker 66 - jepsen.util 11496 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:34:39,930] jepsen worker 66 - jepsen.util 11496 :invoke :write [22 15] INFO [2019-12-17 12:34:39,939] jepsen worker 64 - jepsen.util 6994 :ok :read [21 38] INFO [2019-12-17 12:34:39,939] jepsen worker 64 - jepsen.util 6994 :invoke :read [21 nil] INFO [2019-12-17 12:34:39,944] jepsen worker 71 - jepsen.util 9161 :ok :read [23 22] INFO [2019-12-17 12:34:39,945] jepsen worker 71 - jepsen.util 9161 :invoke :write [23 37] INFO [2019-12-17 12:34:39,954] jepsen worker 64 - jepsen.util 6994 :ok :read [21 38] INFO [2019-12-17 12:34:39,954] jepsen worker 64 - jepsen.util 6994 :invoke :read [21 nil] INFO [2019-12-17 12:34:39,966] jepsen worker 64 - jepsen.util 6994 :ok :read [21 38] INFO [2019-12-17 12:34:39,966] jepsen worker 64 - jepsen.util 6994 :invoke :write [21 34] INFO [2019-12-17 12:34:39,967] jepsen worker 13 - jepsen.util 8293 :ok :read [4 1] INFO [2019-12-17 12:34:39,967] jepsen worker 13 - jepsen.util 8293 :invoke :read [4 nil] INFO [2019-12-17 12:34:39,996] jepsen worker 45 - jepsen.util 8955 :info :write [15 9] :RequestCanceledException INFO [2019-12-17 12:34:39,997] jepsen worker 45 - jepsen.util 9045 :invoke :cas [15 [36 3]] INFO [2019-12-17 12:34:40,008] jepsen worker 64 - jepsen.util 6994 :ok :write [21 34] INFO [2019-12-17 12:34:40,008] jepsen worker 64 - jepsen.util 6994 :invoke :cas [21 [0 18]] INFO [2019-12-17 12:34:40,012] jepsen worker 45 - jepsen.util 9045 :fail :cas [15 [36 3]] :ValueNotSwapFrom INFO [2019-12-17 12:34:40,012] jepsen worker 45 - jepsen.util 9045 :invoke :cas [15 [30 2]] INFO [2019-12-17 12:34:40,018] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":49830,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:34:40,019] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":49830,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:34:40,019] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":49831,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:34:40,019] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":49831,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:34:40,027] jepsen worker 38 - jepsen.util 8858 :info :write [12 27] :RequestCanceledException INFO [2019-12-17 12:34:40,027] jepsen worker 84 - jepsen.util 11064 :info :write [28 13] :RequestCanceledException INFO [2019-12-17 12:34:40,028] jepsen worker 84 - jepsen.util 11154 :invoke :write [28 37] INFO [2019-12-17 12:34:40,028] jepsen worker 38 - jepsen.util 8948 :invoke :cas [12 [6 23]] INFO [2019-12-17 12:34:40,032] jepsen worker 64 - jepsen.util 6994 :fail :cas [21 [0 18]] :ValueNotSwapFrom INFO [2019-12-17 12:34:40,032] jepsen worker 64 - jepsen.util 6994 :invoke :read [21 nil] INFO [2019-12-17 12:34:40,042] jepsen worker 45 - jepsen.util 9045 :fail :cas [15 [30 2]] :ValueNotSwapFrom INFO [2019-12-17 12:34:40,042] jepsen worker 45 - jepsen.util 9045 :invoke :write [15 27] INFO [2019-12-17 12:34:40,046] jepsen worker 64 - jepsen.util 6994 :ok :read [21 34] INFO [2019-12-17 12:34:40,046] jepsen worker 64 - jepsen.util 6994 :invoke :write [21 21] INFO [2019-12-17 12:34:40,060] jepsen worker 24 - jepsen.util 10824 :info :write [8 6] :RequestCanceledException INFO [2019-12-17 12:34:40,061] jepsen worker 24 - jepsen.util 10914 :invoke :read [8 nil] INFO [2019-12-17 12:34:40,096] jepsen worker 17 - jepsen.util 11717 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:34:40,096] jepsen worker 17 - jepsen.util 11717 :invoke :cas [5 [46 2]] INFO [2019-12-17 12:34:40,098] jepsen worker 64 - jepsen.util 6994 :ok :write [21 21] INFO [2019-12-17 12:34:40,098] jepsen worker 64 - jepsen.util 6994 :invoke :cas [21 [11 14]] INFO [2019-12-17 12:34:40,113] jepsen worker 64 - jepsen.util 6994 :fail :cas [21 [11 14]] :ValueNotSwapFrom INFO [2019-12-17 12:34:40,113] jepsen worker 64 - jepsen.util 6994 :invoke :cas [21 [46 16]] INFO [2019-12-17 12:34:40,125] jepsen worker 64 - jepsen.util 6994 :fail :cas [21 [46 16]] :ValueNotSwapFrom INFO [2019-12-17 12:34:40,125] jepsen worker 64 - jepsen.util 6994 :invoke :write [21 31] INFO [2019-12-17 12:34:40,129] jepsen worker 40 - jepsen.util 11830 :info :write [13 3] :RequestCanceledException INFO [2019-12-17 12:34:40,129] jepsen worker 40 - jepsen.util 11920 :invoke :cas [13 [24 43]] INFO [2019-12-17 12:34:40,160] jepsen worker 64 - jepsen.util 6994 :ok :write [21 31] INFO [2019-12-17 12:34:40,160] jepsen worker 64 - jepsen.util 6994 :invoke :read [21 nil] INFO [2019-12-17 12:34:40,161] jepsen worker 87 - jepsen.util 8097 :fail :read [29 nil] :RequestTimeoutException INFO [2019-12-17 12:34:40,161] jepsen worker 87 - jepsen.util 8097 :invoke :write [29 12] INFO [2019-12-17 12:34:40,182] jepsen worker 64 - jepsen.util 6994 :ok :read [21 31] INFO [2019-12-17 12:34:40,182] jepsen worker 64 - jepsen.util 6994 :invoke :cas [21 [36 45]] INFO [2019-12-17 12:34:40,191] jepsen worker 55 - jepsen.util 9415 :ok :read [18 6] INFO [2019-12-17 12:34:40,191] jepsen worker 55 - jepsen.util 9415 :invoke :read [18 nil] INFO [2019-12-17 12:34:40,196] jepsen worker 64 - jepsen.util 6994 :fail :cas [21 [36 45]] :ValueNotSwapFrom INFO [2019-12-17 12:34:40,196] jepsen worker 64 - jepsen.util 6994 :invoke :read [21 nil] INFO [2019-12-17 12:34:40,207] jepsen worker 14 - jepsen.util 8834 :fail :cas [4 [7 7]] :ValueNotSwapFrom INFO [2019-12-17 12:34:40,207] jepsen worker 64 - jepsen.util 6994 :ok :read [21 31] INFO [2019-12-17 12:34:40,208] jepsen worker 14 - jepsen.util 8834 :invoke :cas [4 [5 19]] INFO [2019-12-17 12:34:40,208] jepsen worker 64 - jepsen.util 6994 :invoke :write [21 4] INFO [2019-12-17 12:34:40,213] jepsen worker 15 - jepsen.util 10905 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:34:40,214] jepsen worker 15 - jepsen.util 10905 :invoke :read [5 nil] INFO [2019-12-17 12:34:40,229] jepsen worker 42 - jepsen.util 12732 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:34:40,229] jepsen worker 43 - jepsen.util 11383 :info :cas [14 [23 46]] :RequestCanceledException INFO [2019-12-17 12:34:40,230] jepsen worker 42 - jepsen.util 12732 :invoke :write [14 5] INFO [2019-12-17 12:34:40,231] jepsen worker 43 - jepsen.util 11473 :invoke :write [14 6] INFO [2019-12-17 12:34:40,237] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":49864,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:34:40,237] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":49864,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:34:40,237] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":49867,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:34:40,238] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":49867,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:34:40,238] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":49868,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:34:40,238] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":49868,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:34:40,243] jepsen worker 77 - jepsen.util 11957 :info :write [25 31] :RequestCanceledException INFO [2019-12-17 12:34:40,243] jepsen worker 1 - jepsen.util 11791 :info :write [0 27] :RequestCanceledException INFO [2019-12-17 12:34:40,244] jepsen worker 1 - jepsen.util 11881 :invoke :write [0 6] INFO [2019-12-17 12:34:40,244] jepsen worker 77 - jepsen.util 12047 :invoke :cas [25 [9 25]] INFO [2019-12-17 12:34:40,252] jepsen worker 64 - jepsen.util 6994 :ok :write [21 4] INFO [2019-12-17 12:34:40,252] jepsen worker 64 - jepsen.util 6994 :invoke :write [21 34] INFO [2019-12-17 12:34:40,257] jepsen worker 89 - jepsen.util 8639 :fail :cas [29 [41 1]] :ValueNotSwapFrom INFO [2019-12-17 12:34:40,257] jepsen worker 89 - jepsen.util 8639 :invoke :write [29 42] INFO [2019-12-17 12:34:40,273] jepsen worker 31 - jepsen.util 7681 :ok :write [10 20] INFO [2019-12-17 12:34:40,273] jepsen worker 74 - jepsen.util 8804 :ok :write [24 40] INFO [2019-12-17 12:34:40,273] jepsen worker 31 - jepsen.util 7681 :invoke :cas [10 [9 49]] INFO [2019-12-17 12:34:40,273] jepsen worker 74 - jepsen.util 8804 :invoke :write [24 5] INFO [2019-12-17 12:34:40,279] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":49854,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:34:40,279] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":49854,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:34:40,279] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":49856,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:34:40,279] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":49856,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:34:40,284] jepsen worker 45 - jepsen.util 9045 :ok :write [15 27] INFO [2019-12-17 12:34:40,284] jepsen worker 19 - jepsen.util 8659 :ok :write [6 29] INFO [2019-12-17 12:34:40,284] jepsen worker 45 - jepsen.util 9045 :invoke :write [15 10] INFO [2019-12-17 12:34:40,284] jepsen worker 19 - jepsen.util 8659 :invoke :read [6 nil] INFO [2019-12-17 12:34:40,297] jepsen worker 75 - jepsen.util 11145 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:34:40,297] jepsen worker 75 - jepsen.util 11145 :invoke :write [25 42] INFO [2019-12-17 12:34:40,300] jepsen worker 65 - jepsen.util 7085 :ok :read [21 4] INFO [2019-12-17 12:34:40,300] jepsen worker 65 - jepsen.util 7085 :invoke :write [21 9] INFO [2019-12-17 12:34:40,314] jepsen worker 8 - jepsen.util 11528 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:34:40,314] jepsen worker 8 - jepsen.util 11528 :invoke :write [2 9] INFO [2019-12-17 12:34:40,321] jepsen worker 19 - jepsen.util 8659 :ok :read [6 29] INFO [2019-12-17 12:34:40,321] jepsen worker 19 - jepsen.util 8659 :invoke :cas [6 [44 7]] INFO [2019-12-17 12:34:40,329] jepsen worker 46 - jepsen.util 8506 :fail :read [15 nil] :RequestTimeoutException INFO [2019-12-17 12:34:40,329] jepsen worker 46 - jepsen.util 8506 :invoke :write [15 48] INFO [2019-12-17 12:34:40,334] jepsen worker 31 - jepsen.util 7681 :fail :cas [10 [9 49]] :ValueNotSwapFrom INFO [2019-12-17 12:34:40,334] jepsen worker 31 - jepsen.util 7681 :invoke :read [10 nil] INFO [2019-12-17 12:34:40,339] jepsen worker 19 - jepsen.util 8659 :fail :cas [6 [44 7]] :ValueNotSwapFrom INFO [2019-12-17 12:34:40,339] jepsen worker 19 - jepsen.util 8659 :invoke :cas [6 [44 44]] INFO [2019-12-17 12:34:40,357] jepsen worker 64 - jepsen.util 6994 :ok :write [21 34] INFO [2019-12-17 12:34:40,357] jepsen worker 64 - jepsen.util 6994 :invoke :read [21 nil] INFO [2019-12-17 12:34:40,377] jepsen worker 19 - jepsen.util 8659 :fail :cas [6 [44 44]] :ValueNotSwapFrom INFO [2019-12-17 12:34:40,378] jepsen worker 19 - jepsen.util 8659 :invoke :read [6 nil] INFO [2019-12-17 12:34:40,383] jepsen worker 64 - jepsen.util 6994 :ok :read [21 34] INFO [2019-12-17 12:34:40,383] jepsen worker 64 - jepsen.util 6994 :invoke :read [21 nil] INFO [2019-12-17 12:34:40,397] jepsen worker 13 - jepsen.util 8293 :ok :read [4 1] INFO [2019-12-17 12:34:40,397] jepsen worker 13 - jepsen.util 8293 :invoke :write [4 25] INFO [2019-12-17 12:34:40,434] jepsen worker 19 - jepsen.util 8659 :ok :read [6 29] INFO [2019-12-17 12:34:40,435] jepsen worker 19 - jepsen.util 8659 :invoke :write [6 47] INFO [2019-12-17 12:34:40,440] jepsen worker 64 - jepsen.util 6994 :ok :read [21 34] INFO [2019-12-17 12:34:40,440] jepsen worker 64 - jepsen.util 6994 :invoke :cas [21 [35 6]] INFO [2019-12-17 12:34:40,444] jepsen worker 31 - jepsen.util 7681 :ok :read [10 20] INFO [2019-12-17 12:34:40,444] jepsen worker 31 - jepsen.util 7681 :invoke :read [10 nil] INFO [2019-12-17 12:34:40,447] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":49891,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:34:40,447] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":49891,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:34:40,447] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":49894,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:34:40,447] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":49894,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:34:40,447] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":49895,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:34:40,447] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":49895,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:34:40,462] jepsen worker 64 - jepsen.util 6994 :fail :cas [21 [35 6]] :ValueNotSwapFrom INFO [2019-12-17 12:34:40,462] jepsen worker 64 - jepsen.util 6994 :invoke :read [21 nil] INFO [2019-12-17 12:34:40,486] jepsen worker 14 - jepsen.util 8834 :fail :cas [4 [5 19]] :ValueNotSwapFrom INFO [2019-12-17 12:34:40,486] jepsen worker 14 - jepsen.util 8834 :invoke :cas [4 [31 37]] INFO [2019-12-17 12:34:40,492] jepsen worker 7 - jepsen.util 11887 :info :write [2 6] :RequestCanceledException INFO [2019-12-17 12:34:40,493] jepsen worker 7 - jepsen.util 11977 :invoke :write [2 10] INFO [2019-12-17 12:34:40,504] jepsen worker 31 - jepsen.util 7681 :ok :read [10 20] INFO [2019-12-17 12:34:40,504] jepsen worker 31 - jepsen.util 7681 :invoke :cas [10 [49 30]] INFO [2019-12-17 12:34:40,511] jepsen worker 64 - jepsen.util 6994 :ok :read [21 34] INFO [2019-12-17 12:34:40,511] jepsen worker 64 - jepsen.util 6994 :invoke :read [21 nil] INFO [2019-12-17 12:34:40,514] jepsen worker 49 - jepsen.util 11029 :info :cas [16 [40 1]] :RequestCanceledException INFO [2019-12-17 12:34:40,515] jepsen worker 49 - jepsen.util 11119 :invoke :write [16 36] INFO [2019-12-17 12:34:40,530] jepsen worker 2 - jepsen.util 11342 :info :cas [0 [9 38]] :RequestCanceledException INFO [2019-12-17 12:34:40,530] jepsen worker 79 - jepsen.util 8899 :info :cas [26 [40 17]] :RequestCanceledException INFO [2019-12-17 12:34:40,531] jepsen worker 31 - jepsen.util 7681 :fail :cas [10 [49 30]] :ValueNotSwapFrom INFO [2019-12-17 12:34:40,531] jepsen worker 31 - jepsen.util 7681 :invoke :read [10 nil] INFO [2019-12-17 12:34:40,531] jepsen worker 2 - jepsen.util 11432 :invoke :write [0 13] INFO [2019-12-17 12:34:40,531] jepsen worker 79 - jepsen.util 8989 :invoke :read [26 nil] INFO [2019-12-17 12:34:40,545] jepsen worker 46 - jepsen.util 8506 :ok :write [15 48] INFO [2019-12-17 12:34:40,545] jepsen worker 46 - jepsen.util 8506 :invoke :cas [15 [45 1]] INFO [2019-12-17 12:34:40,549] jepsen worker 31 - jepsen.util 7681 :ok :read [10 20] INFO [2019-12-17 12:34:40,550] jepsen worker 31 - jepsen.util 7681 :invoke :read [10 nil] INFO [2019-12-17 12:34:40,574] jepsen worker 46 - jepsen.util 8506 :fail :cas [15 [45 1]] :ValueNotSwapFrom INFO [2019-12-17 12:34:40,574] jepsen worker 46 - jepsen.util 8506 :invoke :write [15 31] INFO [2019-12-17 12:34:40,608] jepsen worker 64 - jepsen.util 6994 :ok :read [21 34] INFO [2019-12-17 12:34:40,608] jepsen worker 64 - jepsen.util 6994 :invoke :write [21 24] INFO [2019-12-17 12:34:40,611] jepsen worker 31 - jepsen.util 7681 :ok :read [10 20] INFO [2019-12-17 12:34:40,611] jepsen worker 31 - jepsen.util 7681 :invoke :read [10 nil] INFO [2019-12-17 12:34:40,614] jepsen worker 73 - jepsen.util 8263 :info :write [24 29] :RequestCanceledException INFO [2019-12-17 12:34:40,616] jepsen worker 73 - jepsen.util 8353 :invoke :read [24 nil] INFO [2019-12-17 12:34:40,630] jepsen worker 54 - jepsen.util 9504 :info :write [18 14] :RequestCanceledException INFO [2019-12-17 12:34:40,631] jepsen worker 54 - jepsen.util 9594 :invoke :cas [18 [40 36]] INFO [2019-12-17 12:34:40,649] jepsen worker 31 - jepsen.util 7681 :ok :read [10 20] INFO [2019-12-17 12:34:40,649] jepsen worker 31 - jepsen.util 7681 :invoke :read [10 nil] INFO [2019-12-17 12:34:40,656] jepsen worker 64 - jepsen.util 6994 :ok :write [21 24] INFO [2019-12-17 12:34:40,656] jepsen worker 64 - jepsen.util 6994 :invoke :read [21 nil] INFO [2019-12-17 12:34:40,666] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":49916,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:34:40,667] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":49916,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:34:40,667] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":49919,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:34:40,667] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":49919,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:34:40,667] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":49920,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:34:40,667] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":49920,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:34:40,679] jepsen worker 74 - jepsen.util 8804 :ok :write [24 5] INFO [2019-12-17 12:34:40,680] jepsen worker 74 - jepsen.util 8804 :invoke :write [24 38] INFO [2019-12-17 12:34:40,681] jepsen worker 64 - jepsen.util 6994 :ok :read [21 24] INFO [2019-12-17 12:34:40,681] jepsen worker 64 - jepsen.util 6994 :invoke :cas [21 [16 40]] INFO [2019-12-17 12:34:40,681] jepsen worker 38 - jepsen.util 8948 :fail :cas [12 [6 23]] :ValueNotSwapFrom INFO [2019-12-17 12:34:40,681] jepsen worker 31 - jepsen.util 7681 :ok :read [10 20] INFO [2019-12-17 12:34:40,681] jepsen worker 38 - jepsen.util 8948 :invoke :read [12 nil] INFO [2019-12-17 12:34:40,681] jepsen worker 31 - jepsen.util 7681 :invoke :write [10 35] INFO [2019-12-17 12:34:40,714] jepsen worker 63 - jepsen.util 7443 :info :cas [21 [15 41]] :RequestCanceledException INFO [2019-12-17 12:34:40,715] jepsen worker 63 - jepsen.util 7533 :invoke :read [21 nil] INFO [2019-12-17 12:34:40,726] jepsen worker 64 - jepsen.util 6994 :fail :cas [21 [16 40]] :ValueNotSwapFrom INFO [2019-12-17 12:34:40,726] jepsen worker 64 - jepsen.util 6994 :invoke :write [21 40] INFO [2019-12-17 12:34:40,742] jepsen worker 55 - jepsen.util 9415 :ok :read [18 6] INFO [2019-12-17 12:34:40,742] jepsen worker 55 - jepsen.util 9415 :invoke :read [18 nil] INFO [2019-12-17 12:34:40,743] jepsen worker 36 - jepsen.util 8856 :info :write [12 46] :RequestCanceledException INFO [2019-12-17 12:34:40,743] jepsen worker 81 - jepsen.util 11511 :info :write [27 12] :RequestCanceledException INFO [2019-12-17 12:34:40,744] jepsen worker 36 - jepsen.util 8946 :invoke :write [12 30] INFO [2019-12-17 12:34:40,744] jepsen worker 81 - jepsen.util 11601 :invoke :cas [27 [18 19]] INFO [2019-12-17 12:34:40,754] jepsen worker 19 - jepsen.util 8659 :ok :write [6 47] INFO [2019-12-17 12:34:40,754] jepsen worker 19 - jepsen.util 8659 :invoke :cas [6 [47 22]] INFO [2019-12-17 12:34:40,760] jepsen worker 10 - jepsen.util 8830 :fail :read [3 nil] :RequestTimeoutException INFO [2019-12-17 12:34:40,760] jepsen worker 41 - jepsen.util 12461 :info :write [13 14] :RequestCanceledException INFO [2019-12-17 12:34:40,760] jepsen worker 10 - jepsen.util 8830 :invoke :read [3 nil] INFO [2019-12-17 12:34:40,762] jepsen worker 41 - jepsen.util 12551 :invoke :write [13 44] INFO [2019-12-17 12:34:40,795] jepsen worker 63 - jepsen.util 7533 :ok :read [21 24] INFO [2019-12-17 12:34:40,795] jepsen worker 63 - jepsen.util 7533 :invoke :write [21 28] INFO [2019-12-17 12:34:40,796] jepsen worker 86 - jepsen.util 12506 :info :write [28 49] :RequestCanceledException INFO [2019-12-17 12:34:40,796] jepsen worker 53 - jepsen.util 7793 :info :write [17 33] :RequestCanceledException INFO [2019-12-17 12:34:40,797] jepsen worker 86 - jepsen.util 12596 :invoke :cas [28 [2 17]] INFO [2019-12-17 12:34:40,798] jepsen worker 53 - jepsen.util 7883 :invoke :cas [17 [16 11]] INFO [2019-12-17 12:34:40,806] jepsen worker 45 - jepsen.util 9045 :ok :write [15 10] INFO [2019-12-17 12:34:40,806] jepsen worker 45 - jepsen.util 9045 :invoke :cas [15 [5 37]] INFO [2019-12-17 12:34:40,807] jepsen worker 64 - jepsen.util 6994 :ok :write [21 40] INFO [2019-12-17 12:34:40,807] jepsen worker 64 - jepsen.util 6994 :invoke :write [21 40] INFO [2019-12-17 12:34:40,827] jepsen worker 55 - jepsen.util 9415 :ok :read [18 6] INFO [2019-12-17 12:34:40,827] jepsen worker 55 - jepsen.util 9415 :invoke :write [18 1] INFO [2019-12-17 12:34:40,830] jepsen worker 59 - jepsen.util 11219 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:34:40,830] jepsen worker 21 - jepsen.util 11271 :info :cas [7 [0 38]] :RequestCanceledException INFO [2019-12-17 12:34:40,830] jepsen worker 59 - jepsen.util 11219 :invoke :cas [19 [7 24]] INFO [2019-12-17 12:34:40,831] jepsen worker 21 - jepsen.util 11361 :invoke :cas [7 [7 12]] INFO [2019-12-17 12:34:40,844] jepsen worker 39 - jepsen.util 11109 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:34:40,844] jepsen worker 82 - jepsen.util 12412 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:34:40,844] jepsen worker 39 - jepsen.util 11109 :invoke :write [13 47] INFO [2019-12-17 12:34:40,844] jepsen worker 82 - jepsen.util 12412 :invoke :read [27 nil] INFO [2019-12-17 12:34:40,846] jepsen worker 73 - jepsen.util 8353 :ok :read [24 5] INFO [2019-12-17 12:34:40,847] jepsen worker 73 - jepsen.util 8353 :invoke :cas [24 [3 18]] INFO [2019-12-17 12:34:40,854] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:34:40,854] jepsen nemesis - couchbase.util Rebalance check count: 386 INFO [2019-12-17 12:34:40,854] jepsen nemesis - couchbase.util Rebalance stuck count: 367 INFO [2019-12-17 12:34:40,880] jepsen worker 45 - jepsen.util 9045 :fail :cas [15 [5 37]] :ValueNotSwapFrom INFO [2019-12-17 12:34:40,880] jepsen worker 45 - jepsen.util 9045 :invoke :read [15 nil] INFO [2019-12-17 12:34:40,887] jepsen worker 73 - jepsen.util 8353 :fail :cas [24 [3 18]] :ValueNotSwapFrom INFO [2019-12-17 12:34:40,887] jepsen worker 73 - jepsen.util 8353 :invoke :write [24 12] INFO [2019-12-17 12:34:40,896] jepsen worker 76 - jepsen.util 11866 :info :cas [25 [36 44]] :RequestCanceledException INFO [2019-12-17 12:34:40,897] jepsen worker 76 - jepsen.util 11956 :invoke :write [25 33] INFO [2019-12-17 12:34:40,902] jepsen worker 45 - jepsen.util 9045 :ok :read [15 10] INFO [2019-12-17 12:34:40,903] jepsen worker 45 - jepsen.util 9045 :invoke :cas [15 [49 17]] INFO [2019-12-17 12:34:40,909] jepsen worker 54 - jepsen.util 9594 :fail :cas [18 [40 36]] :ValueNotSwapFrom INFO [2019-12-17 12:34:40,909] jepsen worker 54 - jepsen.util 9594 :invoke :write [18 21] INFO [2019-12-17 12:34:40,912] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":49948,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:34:40,912] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":49948,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:34:40,912] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":49951,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:34:40,912] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":49951,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:34:40,914] jepsen worker 52 - jepsen.util 8782 :info :write [17 36] :RequestCanceledException INFO [2019-12-17 12:34:40,914] jepsen worker 4 - jepsen.util 8194 :info :write [1 13] :RequestCanceledException INFO [2019-12-17 12:34:40,915] jepsen worker 52 - jepsen.util 8872 :invoke :write [17 10] INFO [2019-12-17 12:34:40,915] jepsen worker 4 - jepsen.util 8284 :invoke :cas [1 [40 47]] INFO [2019-12-17 12:34:40,923] jepsen worker 64 - jepsen.util 6994 :ok :write [21 40] INFO [2019-12-17 12:34:40,923] jepsen worker 64 - jepsen.util 6994 :invoke :cas [21 [4 11]] INFO [2019-12-17 12:34:40,930] jepsen worker 47 - jepsen.util 9047 :fail :read [15 nil] :RequestTimeoutException INFO [2019-12-17 12:34:40,930] jepsen worker 68 - jepsen.util 11048 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:34:40,930] jepsen worker 47 - jepsen.util 9047 :invoke :write [15 0] INFO [2019-12-17 12:34:40,930] jepsen worker 68 - jepsen.util 11048 :invoke :write [22 48] INFO [2019-12-17 12:34:40,940] jepsen worker 64 - jepsen.util 6994 :fail :cas [21 [4 11]] :ValueNotSwapFrom INFO [2019-12-17 12:34:40,940] jepsen worker 64 - jepsen.util 6994 :invoke :write [21 33] INFO [2019-12-17 12:34:40,943] jepsen worker 53 - jepsen.util 7883 :fail :cas [17 [16 11]] :ValueNotSwapFrom INFO [2019-12-17 12:34:40,944] jepsen worker 53 - jepsen.util 7883 :invoke :write [17 19] INFO [2019-12-17 12:34:40,944] jepsen worker 12 - jepsen.util 9102 :info :write [4 27] :RequestCanceledException INFO [2019-12-17 12:34:40,944] jepsen worker 70 - jepsen.util 9070 :info :write [23 8] :RequestCanceledException INFO [2019-12-17 12:34:40,945] jepsen worker 12 - jepsen.util 9192 :invoke :read [4 nil] INFO [2019-12-17 12:34:40,946] jepsen worker 70 - jepsen.util 9160 :invoke :cas [23 [30 41]] INFO [2019-12-17 12:34:40,961] jepsen worker 34 - jepsen.util 11734 :info :cas [11 [28 14]] :RequestCanceledException INFO [2019-12-17 12:34:40,962] jepsen worker 34 - jepsen.util 11824 :invoke :cas [11 [8 32]] INFO [2019-12-17 12:34:40,992] jepsen worker 83 - jepsen.util 12143 :info :cas [27 [5 23]] :RequestCanceledException INFO [2019-12-17 12:34:40,992] jepsen worker 83 - jepsen.util 12233 :invoke :cas [27 [28 21]] INFO [2019-12-17 12:34:41,005] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":49969,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:34:41,005] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":49969,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:34:41,005] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":49971,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:34:41,005] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":49971,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:34:41,011] jepsen worker 64 - jepsen.util 6994 :ok :write [21 33] INFO [2019-12-17 12:34:41,011] jepsen worker 64 - jepsen.util 6994 :invoke :read [21 nil] INFO [2019-12-17 12:34:41,014] jepsen worker 27 - jepsen.util 8307 :info :write [9 20] :RequestCanceledException INFO [2019-12-17 12:34:41,015] jepsen worker 23 - jepsen.util 11363 :info :write [7 37] :RequestCanceledException INFO [2019-12-17 12:34:41,015] jepsen worker 14 - jepsen.util 8834 :fail :cas [4 [31 37]] :ValueNotSwapFrom INFO [2019-12-17 12:34:41,016] jepsen worker 14 - jepsen.util 8834 :invoke :read [4 nil] INFO [2019-12-17 12:34:41,016] jepsen worker 23 - jepsen.util 11453 :invoke :cas [7 [49 5]] INFO [2019-12-17 12:34:41,017] jepsen worker 27 - jepsen.util 8397 :invoke :write [9 29] INFO [2019-12-17 12:34:41,023] jepsen worker 37 - jepsen.util 8227 :ok :write [12 8] INFO [2019-12-17 12:34:41,023] jepsen worker 37 - jepsen.util 8227 :invoke :write [12 20] INFO [2019-12-17 12:34:41,029] jepsen worker 60 - jepsen.util 12750 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:34:41,029] jepsen worker 9 - jepsen.util 9009 :info :cas [3 [48 4]] :RequestCanceledException INFO [2019-12-17 12:34:41,029] jepsen worker 50 - jepsen.util 12380 :info :cas [16 [47 24]] :RequestCanceledException INFO [2019-12-17 12:34:41,029] jepsen worker 60 - jepsen.util 12750 :invoke :cas [20 [10 20]] INFO [2019-12-17 12:34:41,030] jepsen worker 9 - jepsen.util 9099 :invoke :write [3 33] INFO [2019-12-17 12:34:41,030] jepsen worker 50 - jepsen.util 12470 :invoke :read [16 nil] INFO [2019-12-17 12:34:41,041] jepsen worker 64 - jepsen.util 6994 :ok :read [21 33] INFO [2019-12-17 12:34:41,041] jepsen worker 64 - jepsen.util 6994 :invoke :read [21 nil] INFO [2019-12-17 12:34:41,044] jepsen worker 29 - jepsen.util 7049 :info :write [9 11] :RequestCanceledException INFO [2019-12-17 12:34:41,045] jepsen worker 29 - jepsen.util 7139 :invoke :cas [9 [14 15]] INFO [2019-12-17 12:34:41,054] jepsen worker 64 - jepsen.util 6994 :ok :read [21 33] INFO [2019-12-17 12:34:41,054] jepsen worker 64 - jepsen.util 6994 :invoke :write [21 42] INFO [2019-12-17 12:34:41,079] jepsen worker 29 - jepsen.util 7139 :fail :cas [9 [14 15]] :ValueNotSwapFrom INFO [2019-12-17 12:34:41,079] jepsen worker 29 - jepsen.util 7139 :invoke :cas [9 [48 4]] INFO [2019-12-17 12:34:41,089] jepsen worker 38 - jepsen.util 8948 :ok :read [12 8] INFO [2019-12-17 12:34:41,090] jepsen worker 38 - jepsen.util 8948 :invoke :read [12 nil] INFO [2019-12-17 12:34:41,092] jepsen worker 79 - jepsen.util 8989 :ok :read [26 2] INFO [2019-12-17 12:34:41,092] jepsen worker 79 - jepsen.util 8989 :invoke :write [26 28] INFO [2019-12-17 12:34:41,110] jepsen worker 29 - jepsen.util 7139 :fail :cas [9 [48 4]] :ValueNotSwapFrom INFO [2019-12-17 12:34:41,111] jepsen worker 29 - jepsen.util 7139 :invoke :write [9 41] INFO [2019-12-17 12:34:41,114] jepsen worker 16 - jepsen.util 12706 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:34:41,114] jepsen worker 16 - jepsen.util 12706 :invoke :write [5 17] INFO [2019-12-17 12:34:41,115] jepsen worker 74 - jepsen.util 8804 :ok :write [24 38] INFO [2019-12-17 12:34:41,115] jepsen worker 74 - jepsen.util 8804 :invoke :cas [24 [13 6]] INFO [2019-12-17 12:34:41,145] jepsen worker 10 - jepsen.util 8830 :ok :read [3 33] INFO [2019-12-17 12:34:41,146] jepsen worker 10 - jepsen.util 8830 :invoke :cas [3 [42 48]] INFO [2019-12-17 12:34:41,161] jepsen worker 67 - jepsen.util 11857 :info :write [22 23] :RequestCanceledException INFO [2019-12-17 12:34:41,162] jepsen worker 67 - jepsen.util 11947 :invoke :cas [22 [33 25]] INFO [2019-12-17 12:34:41,188] jepsen worker 64 - jepsen.util 6994 :ok :write [21 42] INFO [2019-12-17 12:34:41,188] jepsen worker 64 - jepsen.util 6994 :invoke :read [21 nil] INFO [2019-12-17 12:34:41,201] jepsen worker 27 - jepsen.util 8397 :ok :write [9 29] INFO [2019-12-17 12:34:41,201] jepsen worker 27 - jepsen.util 8397 :invoke :read [9 nil] INFO [2019-12-17 12:34:41,210] jepsen worker 56 - jepsen.util 9416 :ok :write [18 13] INFO [2019-12-17 12:34:41,211] jepsen worker 56 - jepsen.util 9416 :invoke :write [18 46] INFO [2019-12-17 12:34:41,214] jepsen worker 20 - jepsen.util 8300 :info :write [6 26] :RequestCanceledException INFO [2019-12-17 12:34:41,215] jepsen worker 20 - jepsen.util 8390 :invoke :write [6 49] INFO [2019-12-17 12:34:41,224] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":50004,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:34:41,225] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":50004,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:34:41,225] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":50007,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:34:41,225] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":50007,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:34:41,226] jepsen worker 64 - jepsen.util 6994 :ok :read [21 42] INFO [2019-12-17 12:34:41,226] jepsen worker 64 - jepsen.util 6994 :invoke :read [21 nil] INFO [2019-12-17 12:34:41,229] jepsen worker 11 - jepsen.util 8561 :fail :read [3 nil] :RequestTimeoutException INFO [2019-12-17 12:34:41,229] jepsen worker 11 - jepsen.util 8561 :invoke :read [3 nil] INFO [2019-12-17 12:34:41,230] jepsen worker 72 - jepsen.util 8622 :info :cas [24 [33 40]] :RequestCanceledException INFO [2019-12-17 12:34:41,231] jepsen worker 72 - jepsen.util 8712 :invoke :write [24 14] INFO [2019-12-17 12:34:41,243] jepsen worker 69 - jepsen.util 8709 :info :write [23 1] :RequestCanceledException INFO [2019-12-17 12:34:41,248] jepsen worker 69 - jepsen.util 8799 :invoke :cas [23 [5 7]] INFO [2019-12-17 12:34:41,260] jepsen worker 78 - jepsen.util 8448 :info :write [26 20] :RequestCanceledException INFO [2019-12-17 12:34:41,261] jepsen worker 78 - jepsen.util 8538 :invoke :read [26 nil] INFO [2019-12-17 12:34:41,262] jepsen worker 12 - jepsen.util 9192 :ok :read [4 1] INFO [2019-12-17 12:34:41,263] jepsen worker 12 - jepsen.util 9192 :invoke :read [4 nil] INFO [2019-12-17 12:34:41,264] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":49996,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:34:41,264] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":49996,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:34:41,286] jepsen worker 10 - jepsen.util 8830 :fail :cas [3 [42 48]] :ValueNotSwapFrom INFO [2019-12-17 12:34:41,286] jepsen worker 10 - jepsen.util 8830 :invoke :write [3 38] INFO [2019-12-17 12:34:41,297] jepsen worker 38 - jepsen.util 8948 :ok :read [12 8] INFO [2019-12-17 12:34:41,297] jepsen worker 38 - jepsen.util 8948 :invoke :write [12 39] INFO [2019-12-17 12:34:41,330] jepsen worker 70 - jepsen.util 9160 :fail :cas [23 [30 41]] :ValueNotSwapFrom INFO [2019-12-17 12:34:41,330] jepsen worker 88 - jepsen.util 8818 :info :write [29 47] :RequestCanceledException INFO [2019-12-17 12:34:41,330] jepsen worker 35 - jepsen.util 11465 :info :write [11 43] :RequestCanceledException INFO [2019-12-17 12:34:41,330] jepsen worker 70 - jepsen.util 9160 :invoke :cas [23 [19 29]] INFO [2019-12-17 12:34:41,331] jepsen worker 35 - jepsen.util 11555 :invoke :write [11 40] INFO [2019-12-17 12:34:41,331] jepsen worker 88 - jepsen.util 8908 :invoke :write [29 34] INFO [2019-12-17 12:34:41,334] jepsen worker 27 - jepsen.util 8397 :ok :read [9 29] INFO [2019-12-17 12:34:41,335] jepsen worker 27 - jepsen.util 8397 :invoke :write [9 31] INFO [2019-12-17 12:34:41,337] jepsen worker 9 - jepsen.util 9099 :ok :write [3 33] INFO [2019-12-17 12:34:41,337] jepsen worker 9 - jepsen.util 9099 :invoke :write [3 49] INFO [2019-12-17 12:34:41,338] jepsen worker 31 - jepsen.util 7681 :ok :write [10 35] INFO [2019-12-17 12:34:41,338] jepsen worker 31 - jepsen.util 7681 :invoke :write [10 37] INFO [2019-12-17 12:34:41,395] jepsen worker 51 - jepsen.util 7971 :info :write [17 0] :RequestCanceledException INFO [2019-12-17 12:34:41,396] jepsen worker 51 - jepsen.util 8061 :invoke :cas [17 [16 32]] INFO [2019-12-17 12:34:41,409] jepsen worker 47 - jepsen.util 9047 :ok :write [15 0] INFO [2019-12-17 12:34:41,409] jepsen worker 47 - jepsen.util 9047 :invoke :write [15 28] INFO [2019-12-17 12:34:41,420] jepsen worker 14 - jepsen.util 8834 :ok :read [4 1] INFO [2019-12-17 12:34:41,420] jepsen worker 14 - jepsen.util 8834 :invoke :write [4 0] INFO [2019-12-17 12:34:41,434] jepsen worker 11 - jepsen.util 8561 :ok :read [3 33] INFO [2019-12-17 12:34:41,434] jepsen worker 11 - jepsen.util 8561 :invoke :cas [3 [5 41]] INFO [2019-12-17 12:34:41,491] jepsen worker 20 - jepsen.util 8390 :ok :write [6 49] INFO [2019-12-17 12:34:41,491] jepsen worker 20 - jepsen.util 8390 :invoke :write [6 11] INFO [2019-12-17 12:34:41,491] jepsen worker 72 - jepsen.util 8712 :ok :write [24 14] INFO [2019-12-17 12:34:41,491] jepsen worker 72 - jepsen.util 8712 :invoke :write [24 34] INFO [2019-12-17 12:34:41,514] jepsen worker 30 - jepsen.util 8670 :fail :read [10 nil] :RequestTimeoutException INFO [2019-12-17 12:34:41,514] jepsen worker 30 - jepsen.util 8670 :invoke :write [10 29] INFO [2019-12-17 12:34:41,528] jepsen worker 33 - jepsen.util 11823 :info :write [11 25] :RequestCanceledException INFO [2019-12-17 12:34:41,529] jepsen worker 33 - jepsen.util 11913 :invoke :read [11 nil] INFO [2019-12-17 12:34:41,537] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":50028,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:34:41,537] jepsen worker 9 - jepsen.util 9099 :ok :write [3 49] INFO [2019-12-17 12:34:41,537] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":50028,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:34:41,537] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":50031,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:34:41,537] jepsen worker 9 - jepsen.util 9099 :invoke :cas [3 [10 17]] INFO [2019-12-17 12:34:41,537] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":50031,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:34:41,537] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":50032,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:34:41,537] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":50032,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:34:41,539] jepsen worker 31 - jepsen.util 7681 :ok :write [10 37] INFO [2019-12-17 12:34:41,539] jepsen worker 31 - jepsen.util 7681 :invoke :read [10 nil] INFO [2019-12-17 12:34:41,540] jepsen worker 47 - jepsen.util 9047 :ok :write [15 28] INFO [2019-12-17 12:34:41,540] jepsen worker 47 - jepsen.util 9047 :invoke :cas [15 [19 20]] INFO [2019-12-17 12:34:41,549] jepsen worker 4 - jepsen.util 8284 :fail :cas [1 [40 47]] :ValueNotSwapFrom INFO [2019-12-17 12:34:41,549] jepsen worker 4 - jepsen.util 8284 :invoke :read [1 nil] INFO [2019-12-17 12:34:41,561] jepsen worker 5 - jepsen.util 8015 :info :write [1 12] :RequestCanceledException INFO [2019-12-17 12:34:41,562] jepsen worker 5 - jepsen.util 8105 :invoke :read [1 nil] INFO [2019-12-17 12:34:41,573] jepsen worker 69 - jepsen.util 8799 :fail :cas [23 [5 7]] :ValueNotSwapFrom INFO [2019-12-17 12:34:41,573] jepsen worker 69 - jepsen.util 8799 :invoke :write [23 44] INFO [2019-12-17 12:34:41,592] jepsen worker 47 - jepsen.util 9047 :fail :cas [15 [19 20]] :ValueNotSwapFrom INFO [2019-12-17 12:34:41,592] jepsen worker 47 - jepsen.util 9047 :invoke :cas [15 [38 36]] INFO [2019-12-17 12:34:41,606] jepsen worker 12 - jepsen.util 9192 :ok :read [4 1] INFO [2019-12-17 12:34:41,607] jepsen worker 12 - jepsen.util 9192 :invoke :write [4 45] INFO [2019-12-17 12:34:41,614] jepsen worker 9 - jepsen.util 9099 :fail :cas [3 [10 17]] :ValueNotSwapFrom INFO [2019-12-17 12:34:41,614] jepsen worker 31 - jepsen.util 7681 :ok :read [10 37] INFO [2019-12-17 12:34:41,614] jepsen worker 9 - jepsen.util 9099 :invoke :write [3 45] INFO [2019-12-17 12:34:41,614] jepsen worker 31 - jepsen.util 7681 :invoke :read [10 nil] INFO [2019-12-17 12:34:41,614] jepsen worker 80 - jepsen.util 8270 :info :write [26 28] :RequestCanceledException INFO [2019-12-17 12:34:41,615] jepsen worker 80 - jepsen.util 8360 :invoke :write [26 0] INFO [2019-12-17 12:34:41,643] jepsen worker 57 - jepsen.util 11667 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:34:41,643] jepsen worker 57 - jepsen.util 11667 :invoke :read [19 nil] INFO [2019-12-17 12:34:41,647] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":50051,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:34:41,647] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":50051,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:34:41,647] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":50054,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:34:41,647] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":50054,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:34:41,686] jepsen worker 47 - jepsen.util 9047 :fail :cas [15 [38 36]] :ValueNotSwapFrom INFO [2019-12-17 12:34:41,686] jepsen worker 47 - jepsen.util 9047 :invoke :cas [15 [27 11]] INFO [2019-12-17 12:34:41,688] jepsen worker 31 - jepsen.util 7681 :ok :read [10 37] INFO [2019-12-17 12:34:41,688] jepsen worker 31 - jepsen.util 7681 :invoke :write [10 5] INFO [2019-12-17 12:34:41,692] jepsen worker 62 - jepsen.util 12032 :info :write [20 14] :RequestCanceledException INFO [2019-12-17 12:34:41,692] jepsen worker 25 - jepsen.util 12355 :info :cas [8 [21 39]] :RequestCanceledException INFO [2019-12-17 12:34:41,693] jepsen worker 62 - jepsen.util 12122 :invoke :read [20 nil] INFO [2019-12-17 12:34:41,693] jepsen worker 25 - jepsen.util 12445 :invoke :write [8 29] INFO [2019-12-17 12:34:41,701] jepsen worker 47 - jepsen.util 9047 :fail :cas [15 [27 11]] :ValueNotSwapFrom INFO [2019-12-17 12:34:41,701] jepsen worker 47 - jepsen.util 9047 :invoke :read [15 nil] INFO [2019-12-17 12:34:41,714] jepsen worker 5 - jepsen.util 8105 :ok :read [1 13] INFO [2019-12-17 12:34:41,714] jepsen worker 5 - jepsen.util 8105 :invoke :cas [1 [40 41]] INFO [2019-12-17 12:34:41,723] jepsen worker 47 - jepsen.util 9047 :ok :read [15 28] INFO [2019-12-17 12:34:41,723] jepsen worker 47 - jepsen.util 9047 :invoke :read [15 nil] INFO [2019-12-17 12:34:41,730] jepsen worker 32 - jepsen.util 8762 :info :write [10 22] :RequestCanceledException INFO [2019-12-17 12:34:41,730] jepsen worker 0 - jepsen.util 12240 :info :write [0 0] :RequestCanceledException INFO [2019-12-17 12:34:41,730] jepsen worker 18 - jepsen.util 8838 :info :write [6 14] :RequestCanceledException INFO [2019-12-17 12:34:41,731] jepsen worker 0 - jepsen.util 12330 :invoke :read [0 nil] INFO [2019-12-17 12:34:41,731] jepsen worker 32 - jepsen.util 8852 :invoke :write [10 0] INFO [2019-12-17 12:34:41,731] jepsen worker 18 - jepsen.util 8928 :invoke :write [6 40] INFO [2019-12-17 12:34:41,747] jepsen worker 47 - jepsen.util 9047 :ok :read [15 28] INFO [2019-12-17 12:34:41,747] jepsen worker 47 - jepsen.util 9047 :invoke :write [15 27] INFO [2019-12-17 12:34:41,814] jepsen worker 58 - jepsen.util 12568 :info :write [19 12] :RequestCanceledException INFO [2019-12-17 12:34:41,814] jepsen worker 26 - jepsen.util 12266 :info :write [8 41] :RequestCanceledException INFO [2019-12-17 12:34:41,815] jepsen worker 26 - jepsen.util 12356 :invoke :write [8 36] INFO [2019-12-17 12:34:41,815] jepsen worker 58 - jepsen.util 12658 :invoke :write [19 37] INFO [2019-12-17 12:34:41,829] jepsen worker 28 - jepsen.util 6868 :info :cas [9 [24 45]] :RequestCanceledException INFO [2019-12-17 12:34:41,830] jepsen worker 28 - jepsen.util 6958 :invoke :read [9 nil] INFO [2019-12-17 12:34:41,830] jepsen worker 85 - jepsen.util 12595 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:34:41,830] jepsen worker 85 - jepsen.util 12595 :invoke :write [28 42] INFO [2019-12-17 12:34:41,854] jepsen worker 4 - jepsen.util 8284 :ok :read [1 13] INFO [2019-12-17 12:34:41,854] jepsen worker 4 - jepsen.util 8284 :invoke :read [1 nil] INFO [2019-12-17 12:34:41,866] jepsen worker 5 - jepsen.util 8105 :fail :cas [1 [40 41]] :ValueNotSwapFrom INFO [2019-12-17 12:34:41,866] jepsen worker 5 - jepsen.util 8105 :invoke :write [1 47] INFO [2019-12-17 12:34:41,867] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:34:41,867] jepsen nemesis - couchbase.util Rebalance check count: 387 INFO [2019-12-17 12:34:41,867] jepsen nemesis - couchbase.util Rebalance stuck count: 368 INFO [2019-12-17 12:34:41,895] jepsen worker 4 - jepsen.util 8284 :ok :read [1 13] INFO [2019-12-17 12:34:41,896] jepsen worker 4 - jepsen.util 8284 :invoke :write [1 36] INFO [2019-12-17 12:34:41,897] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":50073,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:34:41,897] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":50073,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:34:41,897] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":50076,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:34:41,897] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":50076,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:34:41,897] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":50077,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:34:41,897] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":50077,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:34:41,910] jepsen worker 51 - jepsen.util 8061 :fail :cas [17 [16 32]] :ValueNotSwapFrom INFO [2019-12-17 12:34:41,910] jepsen worker 51 - jepsen.util 8061 :invoke :cas [17 [45 47]] INFO [2019-12-17 12:34:41,930] jepsen worker 3 - jepsen.util 8463 :info :write [1 1] :RequestCanceledException INFO [2019-12-17 12:34:41,930] jepsen worker 3 - jepsen.util 8553 :invoke :read [1 nil] INFO [2019-12-17 12:34:41,942] jepsen worker 6 - jepsen.util 12246 :info :cas [2 [27 43]] :RequestCanceledException INFO [2019-12-17 12:34:41,942] jepsen worker 22 - jepsen.util 11632 :info :write [7 41] :RequestCanceledException INFO [2019-12-17 12:34:41,942] jepsen worker 48 - jepsen.util 10848 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:34:41,942] jepsen worker 48 - jepsen.util 10848 :invoke :write [16 41] INFO [2019-12-17 12:34:41,943] jepsen worker 6 - jepsen.util 12336 :invoke :read [2 nil] INFO [2019-12-17 12:34:41,943] jepsen worker 22 - jepsen.util 11722 :invoke :cas [7 [13 38]] INFO [2019-12-17 12:34:41,960] jepsen worker 44 - jepsen.util 12104 :info :cas [14 [9 9]] :RequestCanceledException INFO [2019-12-17 12:34:41,960] jepsen worker 44 - jepsen.util 12194 :invoke :read [14 nil] INFO [2019-12-17 12:34:41,978] jepsen worker 78 - jepsen.util 8538 :ok :read [26 2] INFO [2019-12-17 12:34:41,978] jepsen worker 78 - jepsen.util 8538 :invoke :write [26 1] INFO [2019-12-17 12:34:42,006] jepsen worker 70 - jepsen.util 9160 :fail :cas [23 [19 29]] :ValueNotSwapFrom INFO [2019-12-17 12:34:42,006] jepsen worker 70 - jepsen.util 9160 :invoke :read [23 nil] INFO [2019-12-17 12:34:42,013] jepsen worker 61 - jepsen.util 12391 :info :write [20 46] :RequestCanceledException INFO [2019-12-17 12:34:42,014] jepsen worker 61 - jepsen.util 12481 :invoke :write [20 33] INFO [2019-12-17 12:34:42,030] jepsen worker 66 - jepsen.util 11496 :info :write [22 15] :RequestCanceledException INFO [2019-12-17 12:34:42,030] jepsen worker 71 - jepsen.util 9161 :info :write [23 37] :RequestCanceledException INFO [2019-12-17 12:34:42,031] jepsen worker 71 - jepsen.util 9251 :invoke :read [23 nil] INFO [2019-12-17 12:34:42,031] jepsen worker 66 - jepsen.util 11586 :invoke :cas [22 [35 3]] INFO [2019-12-17 12:34:42,096] jepsen worker 84 - jepsen.util 11154 :info :write [28 37] :RequestCanceledException INFO [2019-12-17 12:34:42,097] jepsen worker 84 - jepsen.util 11244 :invoke :cas [28 [9 43]] INFO [2019-12-17 12:34:42,099] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":50090,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:34:42,099] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":50090,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:34:42,099] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":50092,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:34:42,099] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":50092,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:34:42,128] jepsen worker 24 - jepsen.util 10914 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:34:42,128] jepsen worker 24 - jepsen.util 10914 :invoke :write [8 1] INFO [2019-12-17 12:34:42,142] jepsen worker 40 - jepsen.util 11920 :info :cas [13 [24 43]] :RequestCanceledException INFO [2019-12-17 12:34:42,143] jepsen worker 40 - jepsen.util 12010 :invoke :read [13 nil] INFO [2019-12-17 12:34:42,195] jepsen worker 17 - jepsen.util 11717 :info :cas [5 [46 2]] :RequestCanceledException INFO [2019-12-17 12:34:42,196] jepsen worker 17 - jepsen.util 11807 :invoke :write [5 14] INFO [2019-12-17 12:34:42,226] jepsen worker 51 - jepsen.util 8061 :fail :cas [17 [45 47]] :ValueNotSwapFrom INFO [2019-12-17 12:34:42,226] jepsen worker 51 - jepsen.util 8061 :invoke :read [17 nil] INFO [2019-12-17 12:34:42,241] jepsen worker 3 - jepsen.util 8553 :ok :read [1 13] INFO [2019-12-17 12:34:42,241] jepsen worker 3 - jepsen.util 8553 :invoke :write [1 12] INFO [2019-12-17 12:34:42,245] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":50100,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:34:42,245] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":50100,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:34:42,245] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":50101,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:34:42,245] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":50101,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:34:42,245] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":50102,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:34:42,245] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":50102,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:34:42,254] jepsen worker 70 - jepsen.util 9160 :ok :read [23 22] INFO [2019-12-17 12:34:42,254] jepsen worker 70 - jepsen.util 9160 :invoke :write [23 31] INFO [2019-12-17 12:34:42,258] jepsen worker 87 - jepsen.util 8097 :info :write [29 12] :RequestCanceledException INFO [2019-12-17 12:34:42,258] jepsen worker 43 - jepsen.util 11473 :info :write [14 6] :RequestCanceledException INFO [2019-12-17 12:34:42,259] jepsen worker 87 - jepsen.util 8187 :invoke :cas [29 [28 11]] INFO [2019-12-17 12:34:42,260] jepsen worker 43 - jepsen.util 11563 :invoke :read [14 nil] INFO [2019-12-17 12:34:42,262] jepsen worker 71 - jepsen.util 9251 :ok :read [23 22] INFO [2019-12-17 12:34:42,262] jepsen worker 71 - jepsen.util 9251 :invoke :write [23 29] INFO [2019-12-17 12:34:42,293] jepsen worker 77 - jepsen.util 12047 :info :cas [25 [9 25]] :RequestCanceledException INFO [2019-12-17 12:34:42,293] jepsen worker 77 - jepsen.util 12137 :invoke :cas [25 [36 42]] INFO [2019-12-17 12:34:42,314] jepsen worker 15 - jepsen.util 10905 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:34:42,314] jepsen worker 15 - jepsen.util 10905 :invoke :read [5 nil] INFO [2019-12-17 12:34:42,319] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":50108,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:34:42,319] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":50108,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:34:42,319] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":50112,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:34:42,319] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":50112,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:34:42,328] jepsen worker 42 - jepsen.util 12732 :info :write [14 5] :RequestCanceledException INFO [2019-12-17 12:34:42,328] jepsen worker 1 - jepsen.util 11881 :info :write [0 6] :RequestCanceledException INFO [2019-12-17 12:34:42,329] jepsen worker 42 - jepsen.util 12822 :invoke :write [14 14] INFO [2019-12-17 12:34:42,329] jepsen worker 1 - jepsen.util 11971 :invoke :write [0 37] INFO [2019-12-17 12:34:42,344] jepsen worker 9 - jepsen.util 9099 :ok :write [3 45] INFO [2019-12-17 12:34:42,344] jepsen worker 9 - jepsen.util 9099 :invoke :read [3 nil] INFO [2019-12-17 12:34:42,344] jepsen worker 30 - jepsen.util 8670 :ok :write [10 29] INFO [2019-12-17 12:34:42,345] jepsen worker 20 - jepsen.util 8390 :ok :write [6 11] INFO [2019-12-17 12:34:42,345] jepsen worker 30 - jepsen.util 8670 :invoke :write [10 37] INFO [2019-12-17 12:34:42,345] jepsen worker 20 - jepsen.util 8390 :invoke :cas [6 [20 21]] INFO [2019-12-17 12:34:42,345] jepsen worker 47 - jepsen.util 9047 :ok :write [15 27] INFO [2019-12-17 12:34:42,345] jepsen worker 72 - jepsen.util 8712 :ok :write [24 34] INFO [2019-12-17 12:34:42,345] jepsen worker 72 - jepsen.util 8712 :invoke :read [24 nil] INFO [2019-12-17 12:34:42,345] jepsen worker 47 - jepsen.util 9047 :invoke :write [15 42] INFO [2019-12-17 12:34:42,361] jepsen worker 65 - jepsen.util 7085 :info :write [21 9] :RequestCanceledException INFO [2019-12-17 12:34:42,361] jepsen worker 89 - jepsen.util 8639 :info :write [29 42] :RequestCanceledException INFO [2019-12-17 12:34:42,362] jepsen worker 65 - jepsen.util 7175 :invoke :write [21 13] INFO [2019-12-17 12:34:42,364] jepsen worker 89 - jepsen.util 8729 :invoke :read [29 nil] INFO [2019-12-17 12:34:42,396] jepsen worker 75 - jepsen.util 11145 :info :write [25 42] :RequestCanceledException INFO [2019-12-17 12:34:42,398] jepsen worker 75 - jepsen.util 11235 :invoke :read [25 nil] INFO [2019-12-17 12:34:42,406] jepsen worker 28 - jepsen.util 6958 :ok :read [9 29] INFO [2019-12-17 12:34:42,406] jepsen worker 28 - jepsen.util 6958 :invoke :write [9 41] INFO [2019-12-17 12:34:42,413] jepsen worker 8 - jepsen.util 11528 :info :write [2 9] :RequestCanceledException INFO [2019-12-17 12:34:42,414] jepsen worker 8 - jepsen.util 11618 :invoke :read [2 nil] INFO [2019-12-17 12:34:42,416] jepsen worker 20 - jepsen.util 8390 :fail :cas [6 [20 21]] :ValueNotSwapFrom INFO [2019-12-17 12:34:42,416] jepsen worker 20 - jepsen.util 8390 :invoke :write [6 40] INFO [2019-12-17 12:34:42,430] jepsen worker 13 - jepsen.util 8293 :info :write [4 25] :RequestCanceledException INFO [2019-12-17 12:34:42,431] jepsen worker 13 - jepsen.util 8383 :invoke :write [4 34] INFO [2019-12-17 12:34:42,471] jepsen worker 65 - jepsen.util 7175 :ok :write [21 13] INFO [2019-12-17 12:34:42,471] jepsen worker 65 - jepsen.util 7175 :invoke :write [21 2] INFO [2019-12-17 12:34:42,474] jepsen worker 32 - jepsen.util 8852 :ok :write [10 0] INFO [2019-12-17 12:34:42,474] jepsen worker 32 - jepsen.util 8852 :invoke :cas [10 [15 4]] INFO [2019-12-17 12:34:42,477] jepsen worker 37 - jepsen.util 8227 :ok :write [12 20] INFO [2019-12-17 12:34:42,477] jepsen worker 37 - jepsen.util 8227 :invoke :read [12 nil] INFO [2019-12-17 12:34:42,491] jepsen worker 4 - jepsen.util 8284 :ok :write [1 36] INFO [2019-12-17 12:34:42,491] jepsen worker 4 - jepsen.util 8284 :invoke :write [1 19] INFO [2019-12-17 12:34:42,495] jepsen worker 87 - jepsen.util 8187 :fail :cas [29 [28 11]] :ValueNotSwapFrom INFO [2019-12-17 12:34:42,495] jepsen worker 87 - jepsen.util 8187 :invoke :write [29 26] INFO [2019-12-17 12:34:42,508] jepsen worker 72 - jepsen.util 8712 :ok :read [24 34] INFO [2019-12-17 12:34:42,508] jepsen worker 72 - jepsen.util 8712 :invoke :write [24 41] INFO [2019-12-17 12:34:42,514] jepsen worker 7 - jepsen.util 11977 :info :write [2 10] :RequestCanceledException INFO [2019-12-17 12:34:42,515] jepsen worker 7 - jepsen.util 12067 :invoke :write [2 35] INFO [2019-12-17 12:34:42,519] jepsen worker 89 - jepsen.util 8729 :ok :read [29 2] INFO [2019-12-17 12:34:42,519] jepsen worker 89 - jepsen.util 8729 :invoke :write [29 20] INFO [2019-12-17 12:34:42,523] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":50135,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:34:42,523] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":50135,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:34:42,523] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":50138,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:34:42,523] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":50138,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:34:42,523] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":50139,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:34:42,523] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":50139,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:34:42,526] jepsen worker 51 - jepsen.util 8061 :ok :read [17 25] INFO [2019-12-17 12:34:42,526] jepsen worker 51 - jepsen.util 8061 :invoke :read [17 nil] INFO [2019-12-17 12:34:42,538] jepsen worker 32 - jepsen.util 8852 :fail :cas [10 [15 4]] :ValueNotSwapFrom INFO [2019-12-17 12:34:42,538] jepsen worker 32 - jepsen.util 8852 :invoke :cas [10 [20 12]] INFO [2019-12-17 12:34:42,557] jepsen worker 65 - jepsen.util 7175 :ok :write [21 2] INFO [2019-12-17 12:34:42,557] jepsen worker 65 - jepsen.util 7175 :invoke :write [21 15] INFO [2019-12-17 12:34:42,557] jepsen worker 30 - jepsen.util 8670 :ok :write [10 37] INFO [2019-12-17 12:34:42,557] jepsen worker 18 - jepsen.util 8928 :ok :write [6 40] INFO [2019-12-17 12:34:42,557] jepsen worker 30 - jepsen.util 8670 :invoke :read [10 nil] INFO [2019-12-17 12:34:42,557] jepsen worker 18 - jepsen.util 8928 :invoke :write [6 48] INFO [2019-12-17 12:34:42,602] jepsen worker 37 - jepsen.util 8227 :ok :read [12 20] INFO [2019-12-17 12:34:42,602] jepsen worker 37 - jepsen.util 8227 :invoke :read [12 nil] INFO [2019-12-17 12:34:42,617] jepsen worker 51 - jepsen.util 8061 :ok :read [17 25] INFO [2019-12-17 12:34:42,617] jepsen worker 51 - jepsen.util 8061 :invoke :cas [17 [17 30]] INFO [2019-12-17 12:34:42,621] jepsen worker 9 - jepsen.util 9099 :ok :read [3 45] INFO [2019-12-17 12:34:42,621] jepsen worker 9 - jepsen.util 9099 :invoke :write [3 4] INFO [2019-12-17 12:34:42,630] jepsen worker 49 - jepsen.util 11119 :info :write [16 36] :RequestCanceledException INFO [2019-12-17 12:34:42,630] jepsen worker 46 - jepsen.util 8506 :info :write [15 31] :RequestCanceledException INFO [2019-12-17 12:34:42,630] jepsen worker 30 - jepsen.util 8670 :ok :read [10 37] INFO [2019-12-17 12:34:42,630] jepsen worker 30 - jepsen.util 8670 :invoke :write [10 28] INFO [2019-12-17 12:34:42,631] jepsen worker 49 - jepsen.util 11209 :invoke :cas [16 [37 7]] INFO [2019-12-17 12:34:42,631] jepsen worker 46 - jepsen.util 8596 :invoke :read [15 nil] INFO [2019-12-17 12:34:42,643] jepsen worker 2 - jepsen.util 11432 :info :write [0 13] :RequestCanceledException INFO [2019-12-17 12:34:42,644] jepsen worker 2 - jepsen.util 11522 :invoke :write [0 48] INFO [2019-12-17 12:34:42,655] jepsen worker 46 - jepsen.util 8596 :ok :read [15 27] INFO [2019-12-17 12:34:42,655] jepsen worker 46 - jepsen.util 8596 :invoke :cas [15 [41 10]] INFO [2019-12-17 12:34:42,660] jepsen worker 37 - jepsen.util 8227 :ok :read [12 20] INFO [2019-12-17 12:34:42,660] jepsen worker 37 - jepsen.util 8227 :invoke :cas [12 [26 7]] INFO [2019-12-17 12:34:42,674] jepsen worker 51 - jepsen.util 8061 :fail :cas [17 [17 30]] :ValueNotSwapFrom INFO [2019-12-17 12:34:42,675] jepsen worker 51 - jepsen.util 8061 :invoke :cas [17 [17 19]] INFO [2019-12-17 12:34:42,711] jepsen worker 65 - jepsen.util 7175 :ok :write [21 15] INFO [2019-12-17 12:34:42,711] jepsen worker 65 - jepsen.util 7175 :invoke :cas [21 [6 48]] INFO [2019-12-17 12:34:42,738] jepsen worker 65 - jepsen.util 7175 :fail :cas [21 [6 48]] :ValueNotSwapFrom INFO [2019-12-17 12:34:42,738] jepsen worker 65 - jepsen.util 7175 :invoke :cas [21 [46 40]] INFO [2019-12-17 12:34:42,742] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":50161,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:34:42,742] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":50161,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:34:42,742] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":50165,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:34:42,742] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":50165,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:34:42,756] jepsen worker 65 - jepsen.util 7175 :fail :cas [21 [46 40]] :ValueNotSwapFrom INFO [2019-12-17 12:34:42,756] jepsen worker 65 - jepsen.util 7175 :invoke :cas [21 [39 31]] INFO [2019-12-17 12:34:42,770] jepsen worker 47 - jepsen.util 9047 :ok :write [15 42] INFO [2019-12-17 12:34:42,770] jepsen worker 72 - jepsen.util 8712 :ok :write [24 41] INFO [2019-12-17 12:34:42,770] jepsen worker 47 - jepsen.util 9047 :invoke :read [15 nil] INFO [2019-12-17 12:34:42,770] jepsen worker 72 - jepsen.util 8712 :invoke :write [24 2] INFO [2019-12-17 12:34:42,779] jepsen worker 65 - jepsen.util 7175 :fail :cas [21 [39 31]] :ValueNotSwapFrom INFO [2019-12-17 12:34:42,779] jepsen worker 65 - jepsen.util 7175 :invoke :cas [21 [7 28]] INFO [2019-12-17 12:34:42,791] jepsen worker 47 - jepsen.util 9047 :ok :read [15 42] INFO [2019-12-17 12:34:42,791] jepsen worker 47 - jepsen.util 9047 :invoke :cas [15 [44 6]] INFO [2019-12-17 12:34:42,815] jepsen worker 63 - jepsen.util 7533 :info :write [21 28] :RequestCanceledException INFO [2019-12-17 12:34:42,816] jepsen worker 63 - jepsen.util 7623 :invoke :cas [21 [35 12]] INFO [2019-12-17 12:34:42,828] jepsen worker 36 - jepsen.util 8946 :info :write [12 30] :RequestCanceledException INFO [2019-12-17 12:34:42,828] jepsen worker 86 - jepsen.util 12596 :info :cas [28 [2 17]] :RequestCanceledException INFO [2019-12-17 12:34:42,830] jepsen worker 86 - jepsen.util 12686 :invoke :read [28 nil] INFO [2019-12-17 12:34:42,830] jepsen worker 36 - jepsen.util 9036 :invoke :read [12 nil] INFO [2019-12-17 12:34:42,841] jepsen worker 81 - jepsen.util 11601 :info :cas [27 [18 19]] :RequestCanceledException INFO [2019-12-17 12:34:42,842] jepsen worker 81 - jepsen.util 11691 :invoke :read [27 nil] INFO [2019-12-17 12:34:42,846] jepsen worker 47 - jepsen.util 9047 :fail :cas [15 [44 6]] :ValueNotSwapFrom INFO [2019-12-17 12:34:42,846] jepsen worker 47 - jepsen.util 9047 :invoke :write [15 20] INFO [2019-12-17 12:34:42,853] jepsen worker 65 - jepsen.util 7175 :fail :cas [21 [7 28]] :ValueNotSwapFrom INFO [2019-12-17 12:34:42,853] jepsen worker 65 - jepsen.util 7175 :invoke :read [21 nil] INFO [2019-12-17 12:34:42,854] jepsen worker 63 - jepsen.util 7623 :fail :cas [21 [35 12]] :ValueNotSwapFrom INFO [2019-12-17 12:34:42,854] jepsen worker 63 - jepsen.util 7623 :invoke :cas [21 [44 42]] INFO [2019-12-17 12:34:42,860] jepsen worker 41 - jepsen.util 12551 :info :write [13 44] :RequestCanceledException INFO [2019-12-17 12:34:42,862] jepsen worker 41 - jepsen.util 12641 :invoke :read [13 nil] INFO [2019-12-17 12:34:42,871] jepsen worker 63 - jepsen.util 7623 :fail :cas [21 [44 42]] :ValueNotSwapFrom INFO [2019-12-17 12:34:42,871] jepsen worker 63 - jepsen.util 7623 :invoke :write [21 10] INFO [2019-12-17 12:34:42,883] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":50187,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:34:42,883] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":50187,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:34:42,889] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:34:42,889] jepsen nemesis - couchbase.util Rebalance check count: 388 INFO [2019-12-17 12:34:42,889] jepsen nemesis - couchbase.util Rebalance stuck count: 369 INFO [2019-12-17 12:34:42,903] jepsen worker 37 - jepsen.util 8227 :fail :cas [12 [26 7]] :ValueNotSwapFrom INFO [2019-12-17 12:34:42,903] jepsen worker 37 - jepsen.util 8227 :invoke :read [12 nil] INFO [2019-12-17 12:34:42,914] jepsen worker 19 - jepsen.util 8659 :info :cas [6 [47 22]] :RequestCanceledException INFO [2019-12-17 12:34:42,914] jepsen worker 55 - jepsen.util 9415 :info :write [18 1] :RequestCanceledException INFO [2019-12-17 12:34:42,914] jepsen worker 21 - jepsen.util 11361 :info :cas [7 [7 12]] :RequestCanceledException INFO [2019-12-17 12:34:42,915] jepsen worker 19 - jepsen.util 8749 :invoke :write [6 1] INFO [2019-12-17 12:34:42,915] jepsen worker 55 - jepsen.util 9505 :invoke :cas [18 [21 36]] INFO [2019-12-17 12:34:42,915] jepsen worker 21 - jepsen.util 11451 :invoke :cas [7 [6 41]] INFO [2019-12-17 12:34:42,929] jepsen worker 73 - jepsen.util 8353 :info :write [24 12] :RequestCanceledException INFO [2019-12-17 12:34:42,929] jepsen worker 59 - jepsen.util 11219 :info :cas [19 [7 24]] :RequestCanceledException INFO [2019-12-17 12:34:42,929] jepsen worker 45 - jepsen.util 9045 :info :cas [15 [49 17]] :RequestCanceledException INFO [2019-12-17 12:34:42,930] jepsen worker 59 - jepsen.util 11309 :invoke :read [19 nil] INFO [2019-12-17 12:34:42,930] jepsen worker 73 - jepsen.util 8443 :invoke :read [24 nil] INFO [2019-12-17 12:34:42,930] jepsen worker 45 - jepsen.util 9135 :invoke :write [15 46] INFO [2019-12-17 12:34:42,931] jepsen worker 76 - jepsen.util 11956 :info :write [25 33] :RequestCanceledException INFO [2019-12-17 12:34:42,935] jepsen worker 76 - jepsen.util 12046 :invoke :cas [25 [13 0]] INFO [2019-12-17 12:34:42,937] jepsen worker 65 - jepsen.util 7175 :ok :read [21 15] INFO [2019-12-17 12:34:42,937] jepsen worker 65 - jepsen.util 7175 :invoke :read [21 nil] INFO [2019-12-17 12:34:42,943] jepsen worker 52 - jepsen.util 8872 :info :write [17 10] :RequestCanceledException INFO [2019-12-17 12:34:42,943] jepsen worker 82 - jepsen.util 12412 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:34:42,943] jepsen worker 39 - jepsen.util 11109 :info :write [13 47] :RequestCanceledException INFO [2019-12-17 12:34:42,943] jepsen worker 82 - jepsen.util 12412 :invoke :write [27 42] INFO [2019-12-17 12:34:42,944] jepsen worker 52 - jepsen.util 8962 :invoke :write [17 16] INFO [2019-12-17 12:34:42,944] jepsen worker 39 - jepsen.util 11199 :invoke :write [13 28] INFO [2019-12-17 12:34:42,991] jepsen worker 73 - jepsen.util 8443 :ok :read [24 41] INFO [2019-12-17 12:34:42,991] jepsen worker 73 - jepsen.util 8443 :invoke :read [24 nil] INFO [2019-12-17 12:34:42,997] jepsen worker 53 - jepsen.util 7883 :info :write [17 19] :RequestCanceledException INFO [2019-12-17 12:34:42,997] jepsen worker 54 - jepsen.util 9594 :info :write [18 21] :RequestCanceledException INFO [2019-12-17 12:34:42,998] jepsen worker 54 - jepsen.util 9684 :invoke :cas [18 [47 26]] INFO [2019-12-17 12:34:42,998] jepsen worker 53 - jepsen.util 7973 :invoke :write [17 25] INFO [2019-12-17 12:34:42,999] jepsen worker 65 - jepsen.util 7175 :ok :read [21 15] INFO [2019-12-17 12:34:42,999] jepsen worker 65 - jepsen.util 7175 :invoke :read [21 nil] INFO [2019-12-17 12:34:43,004] jepsen worker 63 - jepsen.util 7623 :ok :write [21 10] INFO [2019-12-17 12:34:43,004] jepsen worker 63 - jepsen.util 7623 :invoke :cas [21 [13 14]] INFO [2019-12-17 12:34:43,014] jepsen worker 34 - jepsen.util 11824 :info :cas [11 [8 32]] :RequestCanceledException INFO [2019-12-17 12:34:43,015] jepsen worker 34 - jepsen.util 11914 :invoke :cas [11 [11 20]] INFO [2019-12-17 12:34:43,030] jepsen worker 68 - jepsen.util 11048 :info :write [22 48] :RequestCanceledException INFO [2019-12-17 12:34:43,030] jepsen worker 83 - jepsen.util 12233 :info :cas [27 [28 21]] :RequestCanceledException INFO [2019-12-17 12:34:43,031] jepsen worker 83 - jepsen.util 12323 :invoke :read [27 nil] INFO [2019-12-17 12:34:43,031] jepsen worker 68 - jepsen.util 11138 :invoke :write [22 16] INFO [2019-12-17 12:34:43,035] jepsen worker 51 - jepsen.util 8061 :fail :cas [17 [17 19]] :ValueNotSwapFrom INFO [2019-12-17 12:34:43,035] jepsen worker 51 - jepsen.util 8061 :invoke :read [17 nil] INFO [2019-12-17 12:34:43,044] jepsen worker 23 - jepsen.util 11453 :info :cas [7 [49 5]] :RequestCanceledException INFO [2019-12-17 12:34:43,045] jepsen worker 23 - jepsen.util 11543 :invoke :cas [7 [37 39]] INFO [2019-12-17 12:34:43,064] jepsen worker 63 - jepsen.util 7623 :fail :cas [21 [13 14]] :ValueNotSwapFrom INFO [2019-12-17 12:34:43,064] jepsen worker 63 - jepsen.util 7623 :invoke :write [21 3] INFO [2019-12-17 12:34:43,067] jepsen worker 73 - jepsen.util 8443 :ok :read [24 41] INFO [2019-12-17 12:34:43,067] jepsen worker 73 - jepsen.util 8443 :invoke :write [24 28] INFO [2019-12-17 12:34:43,082] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":50208,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:34:43,083] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":50208,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:34:43,083] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":50210,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:34:43,083] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":50210,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:34:43,083] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":50211,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:34:43,083] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":50211,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:34:43,090] jepsen worker 65 - jepsen.util 7175 :ok :read [21 10] INFO [2019-12-17 12:34:43,090] jepsen worker 65 - jepsen.util 7175 :invoke :cas [21 [24 27]] INFO [2019-12-17 12:34:43,097] jepsen worker 50 - jepsen.util 12470 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:34:43,097] jepsen worker 50 - jepsen.util 12470 :invoke :read [16 nil] INFO [2019-12-17 12:34:43,106] jepsen worker 18 - jepsen.util 8928 :ok :write [6 48] INFO [2019-12-17 12:34:43,106] jepsen worker 18 - jepsen.util 8928 :invoke :write [6 16] INFO [2019-12-17 12:34:43,106] jepsen worker 30 - jepsen.util 8670 :ok :write [10 28] INFO [2019-12-17 12:34:43,106] jepsen worker 30 - jepsen.util 8670 :invoke :write [10 14] INFO [2019-12-17 12:34:43,107] jepsen worker 72 - jepsen.util 8712 :ok :write [24 2] INFO [2019-12-17 12:34:43,107] jepsen worker 72 - jepsen.util 8712 :invoke :cas [24 [36 7]] INFO [2019-12-17 12:34:43,126] jepsen worker 37 - jepsen.util 8227 :ok :read [12 20] INFO [2019-12-17 12:34:43,126] jepsen worker 37 - jepsen.util 8227 :invoke :read [12 nil] INFO [2019-12-17 12:34:43,130] jepsen worker 60 - jepsen.util 12750 :info :cas [20 [10 20]] :RequestCanceledException INFO [2019-12-17 12:34:43,131] jepsen worker 60 - jepsen.util 12840 :invoke :read [20 nil] INFO [2019-12-17 12:34:43,134] jepsen worker 47 - jepsen.util 9047 :ok :write [15 20] INFO [2019-12-17 12:34:43,134] jepsen worker 47 - jepsen.util 9047 :invoke :read [15 nil] INFO [2019-12-17 12:34:43,140] jepsen worker 63 - jepsen.util 7623 :ok :write [21 3] INFO [2019-12-17 12:34:43,140] jepsen worker 63 - jepsen.util 7623 :invoke :read [21 nil] INFO [2019-12-17 12:34:43,141] jepsen worker 72 - jepsen.util 8712 :fail :cas [24 [36 7]] :ValueNotSwapFrom INFO [2019-12-17 12:34:43,141] jepsen worker 72 - jepsen.util 8712 :invoke :cas [24 [38 30]] INFO [2019-12-17 12:34:43,152] jepsen worker 55 - jepsen.util 9505 :fail :cas [18 [21 36]] :ValueNotSwapFrom INFO [2019-12-17 12:34:43,152] jepsen worker 55 - jepsen.util 9505 :invoke :write [18 18] INFO [2019-12-17 12:34:43,161] jepsen worker 74 - jepsen.util 8804 :info :cas [24 [13 6]] :RequestCanceledException INFO [2019-12-17 12:34:43,161] jepsen worker 79 - jepsen.util 8989 :info :write [26 28] :RequestCanceledException INFO [2019-12-17 12:34:43,162] jepsen worker 79 - jepsen.util 9079 :invoke :write [26 45] INFO [2019-12-17 12:34:43,162] jepsen worker 74 - jepsen.util 8894 :invoke :read [24 nil] INFO [2019-12-17 12:34:43,163] jepsen worker 63 - jepsen.util 7623 :ok :read [21 3] INFO [2019-12-17 12:34:43,164] jepsen worker 63 - jepsen.util 7623 :invoke :read [21 nil] INFO [2019-12-17 12:34:43,213] jepsen worker 72 - jepsen.util 8712 :fail :cas [24 [38 30]] :ValueNotSwapFrom INFO [2019-12-17 12:34:43,213] jepsen worker 72 - jepsen.util 8712 :invoke :read [24 nil] INFO [2019-12-17 12:34:43,214] jepsen worker 29 - jepsen.util 7139 :info :write [9 41] :RequestCanceledException INFO [2019-12-17 12:34:43,214] jepsen worker 16 - jepsen.util 12706 :info :write [5 17] :RequestCanceledException INFO [2019-12-17 12:34:43,216] jepsen worker 29 - jepsen.util 7229 :invoke :write [9 44] INFO [2019-12-17 12:34:43,216] jepsen worker 16 - jepsen.util 12796 :invoke :cas [5 [43 2]] INFO [2019-12-17 12:34:43,218] jepsen worker 63 - jepsen.util 7623 :ok :read [21 3] INFO [2019-12-17 12:34:43,218] jepsen worker 63 - jepsen.util 7623 :invoke :cas [21 [29 7]] INFO [2019-12-17 12:34:43,229] jepsen worker 67 - jepsen.util 11947 :info :cas [22 [33 25]] :RequestCanceledException INFO [2019-12-17 12:34:43,229] jepsen worker 56 - jepsen.util 9416 :info :write [18 46] :RequestCanceledException INFO [2019-12-17 12:34:43,230] jepsen worker 67 - jepsen.util 12037 :invoke :write [22 24] INFO [2019-12-17 12:34:43,230] jepsen worker 56 - jepsen.util 9506 :invoke :write [18 4] INFO [2019-12-17 12:34:43,233] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":50236,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:34:43,233] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":50236,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:34:43,234] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":50237,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:34:43,234] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":50237,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:34:43,242] jepsen worker 64 - jepsen.util 6994 :fail :read [21 nil] :RequestTimeoutException INFO [2019-12-17 12:34:43,242] jepsen worker 64 - jepsen.util 6994 :invoke :read [21 nil] INFO [2019-12-17 12:34:43,244] jepsen worker 72 - jepsen.util 8712 :ok :read [24 2] INFO [2019-12-17 12:34:43,244] jepsen worker 72 - jepsen.util 8712 :invoke :cas [24 [35 18]] INFO [2019-12-17 12:34:43,257] jepsen worker 63 - jepsen.util 7623 :fail :cas [21 [29 7]] :ValueNotSwapFrom INFO [2019-12-17 12:34:43,258] jepsen worker 63 - jepsen.util 7623 :invoke :read [21 nil] INFO [2019-12-17 12:34:43,285] jepsen worker 64 - jepsen.util 6994 :ok :read [21 3] INFO [2019-12-17 12:34:43,285] jepsen worker 64 - jepsen.util 6994 :invoke :read [21 nil] INFO [2019-12-17 12:34:43,285] jepsen worker 72 - jepsen.util 8712 :fail :cas [24 [35 18]] :ValueNotSwapFrom INFO [2019-12-17 12:34:43,285] jepsen worker 72 - jepsen.util 8712 :invoke :write [24 39] INFO [2019-12-17 12:34:43,293] jepsen worker 63 - jepsen.util 7623 :ok :read [21 3] INFO [2019-12-17 12:34:43,293] jepsen worker 63 - jepsen.util 7623 :invoke :read [21 nil] INFO [2019-12-17 12:34:43,300] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":50243,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:34:43,300] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":50243,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:34:43,300] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":50244,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:34:43,301] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":50244,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:34:43,309] jepsen worker 51 - jepsen.util 8061 :ok :read [17 25] INFO [2019-12-17 12:34:43,309] jepsen worker 51 - jepsen.util 8061 :invoke :write [17 36] INFO [2019-12-17 12:34:43,310] jepsen worker 64 - jepsen.util 6994 :ok :read [21 3] INFO [2019-12-17 12:34:43,311] jepsen worker 64 - jepsen.util 6994 :invoke :cas [21 [31 43]] INFO [2019-12-17 12:34:43,312] jepsen worker 38 - jepsen.util 8948 :info :write [12 39] :RequestCanceledException INFO [2019-12-17 12:34:43,313] jepsen worker 38 - jepsen.util 9038 :invoke :cas [12 [30 32]] INFO [2019-12-17 12:34:43,323] jepsen worker 63 - jepsen.util 7623 :ok :read [21 3] INFO [2019-12-17 12:34:43,323] jepsen worker 63 - jepsen.util 7623 :invoke :read [21 nil] INFO [2019-12-17 12:34:43,359] jepsen worker 45 - jepsen.util 9135 :ok :write [15 46] INFO [2019-12-17 12:34:43,359] jepsen worker 45 - jepsen.util 9135 :invoke :write [15 17] INFO [2019-12-17 12:34:43,361] jepsen worker 64 - jepsen.util 6994 :fail :cas [21 [31 43]] :ValueNotSwapFrom INFO [2019-12-17 12:34:43,361] jepsen worker 64 - jepsen.util 6994 :invoke :write [21 46] INFO [2019-12-17 12:34:43,361] jepsen worker 10 - jepsen.util 8830 :info :write [3 38] :RequestCanceledException INFO [2019-12-17 12:34:43,361] jepsen worker 27 - jepsen.util 8397 :info :write [9 31] :RequestCanceledException INFO [2019-12-17 12:34:43,362] jepsen worker 10 - jepsen.util 8920 :invoke :write [3 38] INFO [2019-12-17 12:34:43,362] jepsen worker 27 - jepsen.util 8487 :invoke :write [9 46] INFO [2019-12-17 12:34:43,389] jepsen worker 36 - jepsen.util 9036 :ok :read [12 20] INFO [2019-12-17 12:34:43,390] jepsen worker 36 - jepsen.util 9036 :invoke :write [12 21] INFO [2019-12-17 12:34:43,398] jepsen worker 63 - jepsen.util 7623 :ok :read [21 3] INFO [2019-12-17 12:34:43,398] jepsen worker 63 - jepsen.util 7623 :invoke :read [21 nil] INFO [2019-12-17 12:34:43,409] jepsen worker 19 - jepsen.util 8749 :ok :write [6 1] INFO [2019-12-17 12:34:43,409] jepsen worker 19 - jepsen.util 8749 :invoke :cas [6 [0 49]] INFO [2019-12-17 12:34:43,414] jepsen worker 35 - jepsen.util 11555 :info :write [11 40] :RequestCanceledException INFO [2019-12-17 12:34:43,415] jepsen worker 35 - jepsen.util 11645 :invoke :write [11 14] INFO [2019-12-17 12:34:43,430] jepsen worker 88 - jepsen.util 8908 :info :write [29 34] :RequestCanceledException INFO [2019-12-17 12:34:43,431] jepsen worker 88 - jepsen.util 8998 :invoke :cas [29 [19 14]] INFO [2019-12-17 12:34:43,434] jepsen worker 72 - jepsen.util 8712 :ok :write [24 39] INFO [2019-12-17 12:34:43,434] jepsen worker 72 - jepsen.util 8712 :invoke :cas [24 [41 34]] INFO [2019-12-17 12:34:43,447] jepsen worker 70 - jepsen.util 9160 :ok :write [23 31] INFO [2019-12-17 12:34:43,447] jepsen worker 70 - jepsen.util 9160 :invoke :read [23 nil] INFO [2019-12-17 12:34:43,449] jepsen worker 30 - jepsen.util 8670 :ok :write [10 14] INFO [2019-12-17 12:34:43,449] jepsen worker 30 - jepsen.util 8670 :invoke :cas [10 [29 15]] INFO [2019-12-17 12:34:43,465] jepsen worker 72 - jepsen.util 8712 :fail :cas [24 [41 34]] :ValueNotSwapFrom INFO [2019-12-17 12:34:43,465] jepsen worker 72 - jepsen.util 8712 :invoke :read [24 nil] INFO [2019-12-17 12:34:43,472] jepsen worker 29 - jepsen.util 7229 :ok :write [9 44] INFO [2019-12-17 12:34:43,473] jepsen worker 29 - jepsen.util 7229 :invoke :write [9 20] INFO [2019-12-17 12:34:43,477] jepsen worker 72 - jepsen.util 8712 :ok :read [24 39] INFO [2019-12-17 12:34:43,477] jepsen worker 72 - jepsen.util 8712 :invoke :read [24 nil] INFO [2019-12-17 12:34:43,481] jepsen worker 37 - jepsen.util 8227 :ok :read [12 20] INFO [2019-12-17 12:34:43,481] jepsen worker 37 - jepsen.util 8227 :invoke :write [12 31] INFO [2019-12-17 12:34:43,493] jepsen worker 64 - jepsen.util 6994 :ok :write [21 46] INFO [2019-12-17 12:34:43,494] jepsen worker 64 - jepsen.util 6994 :invoke :cas [21 [18 38]] INFO [2019-12-17 12:34:43,507] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":50275,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:34:43,508] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":50275,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:34:43,514] jepsen worker 14 - jepsen.util 8834 :info :write [4 0] :RequestCanceledException INFO [2019-12-17 12:34:43,517] jepsen worker 14 - jepsen.util 8924 :invoke :read [4 nil] INFO [2019-12-17 12:34:43,530] jepsen worker 11 - jepsen.util 8561 :info :cas [3 [5 41]] :RequestCanceledException INFO [2019-12-17 12:34:43,531] jepsen worker 11 - jepsen.util 8651 :invoke :read [3 nil] INFO [2019-12-17 12:34:43,541] jepsen worker 72 - jepsen.util 8712 :ok :read [24 39] INFO [2019-12-17 12:34:43,541] jepsen worker 72 - jepsen.util 8712 :invoke :read [24 nil] INFO [2019-12-17 12:34:43,561] jepsen worker 33 - jepsen.util 11913 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:34:43,561] jepsen worker 33 - jepsen.util 11913 :invoke :cas [11 [49 0]] INFO [2019-12-17 12:34:43,566] jepsen worker 64 - jepsen.util 6994 :fail :cas [21 [18 38]] :ValueNotSwapFrom INFO [2019-12-17 12:34:43,566] jepsen worker 64 - jepsen.util 6994 :invoke :cas [21 [14 10]] INFO [2019-12-17 12:34:43,597] jepsen worker 64 - jepsen.util 6994 :fail :cas [21 [14 10]] :ValueNotSwapFrom INFO [2019-12-17 12:34:43,598] jepsen worker 64 - jepsen.util 6994 :invoke :read [21 nil] INFO [2019-12-17 12:34:43,604] jepsen worker 72 - jepsen.util 8712 :ok :read [24 39] INFO [2019-12-17 12:34:43,605] jepsen worker 72 - jepsen.util 8712 :invoke :write [24 9] INFO [2019-12-17 12:34:43,628] jepsen worker 69 - jepsen.util 8799 :info :write [23 44] :RequestCanceledException INFO [2019-12-17 12:34:43,628] jepsen worker 12 - jepsen.util 9192 :info :write [4 45] :RequestCanceledException INFO [2019-12-17 12:34:43,629] jepsen worker 12 - jepsen.util 9282 :invoke :cas [4 [7 31]] INFO [2019-12-17 12:34:43,629] jepsen worker 69 - jepsen.util 8889 :invoke :read [23 nil] INFO [2019-12-17 12:34:43,647] jepsen worker 38 - jepsen.util 9038 :fail :cas [12 [30 32]] :ValueNotSwapFrom INFO [2019-12-17 12:34:43,648] jepsen worker 38 - jepsen.util 9038 :invoke :cas [12 [24 37]] INFO [2019-12-17 12:34:43,655] jepsen worker 11 - jepsen.util 8651 :ok :read [3 45] INFO [2019-12-17 12:34:43,655] jepsen worker 11 - jepsen.util 8651 :invoke :write [3 21] INFO [2019-12-17 12:34:43,665] jepsen worker 70 - jepsen.util 9160 :ok :read [23 31] INFO [2019-12-17 12:34:43,665] jepsen worker 70 - jepsen.util 9160 :invoke :write [23 45] INFO [2019-12-17 12:34:43,671] jepsen worker 64 - jepsen.util 6994 :ok :read [21 46] INFO [2019-12-17 12:34:43,671] jepsen worker 64 - jepsen.util 6994 :invoke :read [21 nil] INFO [2019-12-17 12:34:43,693] jepsen worker 88 - jepsen.util 8998 :fail :cas [29 [19 14]] :ValueNotSwapFrom INFO [2019-12-17 12:34:43,693] jepsen worker 88 - jepsen.util 8998 :invoke :read [29 nil] INFO [2019-12-17 12:34:43,696] jepsen worker 80 - jepsen.util 8360 :info :write [26 0] :RequestCanceledException INFO [2019-12-17 12:34:43,697] jepsen worker 80 - jepsen.util 8450 :invoke :cas [26 [29 3]] INFO [2019-12-17 12:34:43,709] jepsen worker 45 - jepsen.util 9135 :ok :write [15 17] INFO [2019-12-17 12:34:43,709] jepsen worker 45 - jepsen.util 9135 :invoke :cas [15 [5 48]] INFO [2019-12-17 12:34:43,711] jepsen worker 62 - jepsen.util 12122 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:34:43,712] jepsen worker 62 - jepsen.util 12122 :invoke :write [20 23] INFO [2019-12-17 12:34:43,715] jepsen worker 69 - jepsen.util 8889 :ok :read [23 31] INFO [2019-12-17 12:34:43,715] jepsen worker 69 - jepsen.util 8889 :invoke :write [23 10] INFO [2019-12-17 12:34:43,719] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":50298,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:34:43,720] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":50298,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:34:43,720] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":50301,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:34:43,720] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":50301,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:34:43,725] jepsen worker 64 - jepsen.util 6994 :ok :read [21 46] INFO [2019-12-17 12:34:43,725] jepsen worker 64 - jepsen.util 6994 :invoke :read [21 nil] INFO [2019-12-17 12:34:43,730] jepsen worker 25 - jepsen.util 12445 :info :write [8 29] :RequestCanceledException INFO [2019-12-17 12:34:43,731] jepsen worker 25 - jepsen.util 12535 :invoke :write [8 40] INFO [2019-12-17 12:34:43,736] jepsen worker 29 - jepsen.util 7229 :ok :write [9 20] INFO [2019-12-17 12:34:43,736] jepsen worker 29 - jepsen.util 7229 :invoke :cas [9 [15 41]] INFO [2019-12-17 12:34:43,737] jepsen worker 57 - jepsen.util 11667 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:34:43,737] jepsen worker 0 - jepsen.util 12330 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:34:43,737] jepsen worker 57 - jepsen.util 11667 :invoke :write [19 23] INFO [2019-12-17 12:34:43,737] jepsen worker 0 - jepsen.util 12330 :invoke :cas [0 [48 48]] INFO [2019-12-17 12:34:43,743] jepsen worker 45 - jepsen.util 9135 :fail :cas [15 [5 48]] :ValueNotSwapFrom INFO [2019-12-17 12:34:43,743] jepsen worker 45 - jepsen.util 9135 :invoke :cas [15 [33 36]] INFO [2019-12-17 12:34:43,770] jepsen worker 64 - jepsen.util 6994 :ok :read [21 46] INFO [2019-12-17 12:34:43,770] jepsen worker 64 - jepsen.util 6994 :invoke :cas [21 [43 12]] INFO [2019-12-17 12:34:43,788] jepsen worker 45 - jepsen.util 9135 :fail :cas [15 [33 36]] :ValueNotSwapFrom INFO [2019-12-17 12:34:43,788] jepsen worker 45 - jepsen.util 9135 :invoke :read [15 nil] INFO [2019-12-17 12:34:43,791] jepsen worker 29 - jepsen.util 7229 :fail :cas [9 [15 41]] :ValueNotSwapFrom INFO [2019-12-17 12:34:43,791] jepsen worker 29 - jepsen.util 7229 :invoke :read [9 nil] INFO [2019-12-17 12:34:43,797] jepsen worker 31 - jepsen.util 7681 :info :write [10 5] :RequestCanceledException INFO [2019-12-17 12:34:43,797] jepsen worker 31 - jepsen.util 7771 :invoke :cas [10 [48 14]] INFO [2019-12-17 12:34:43,824] jepsen worker 29 - jepsen.util 7229 :ok :read [9 20] INFO [2019-12-17 12:34:43,825] jepsen worker 29 - jepsen.util 7229 :invoke :read [9 nil] INFO [2019-12-17 12:34:43,845] jepsen worker 54 - jepsen.util 9684 :fail :cas [18 [47 26]] :ValueNotSwapFrom INFO [2019-12-17 12:34:43,846] jepsen worker 54 - jepsen.util 9684 :invoke :write [18 49] INFO [2019-12-17 12:34:43,849] jepsen worker 64 - jepsen.util 6994 :fail :cas [21 [43 12]] :ValueNotSwapFrom INFO [2019-12-17 12:34:43,849] jepsen worker 64 - jepsen.util 6994 :invoke :read [21 nil] INFO [2019-12-17 12:34:43,861] jepsen worker 29 - jepsen.util 7229 :ok :read [9 20] INFO [2019-12-17 12:34:43,861] jepsen worker 29 - jepsen.util 7229 :invoke :read [9 nil] INFO [2019-12-17 12:34:43,864] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":50326,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:34:43,864] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":50326,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:34:43,864] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":50330,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:34:43,864] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":50330,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:34:43,871] jepsen worker 45 - jepsen.util 9135 :ok :read [15 17] INFO [2019-12-17 12:34:43,872] jepsen worker 45 - jepsen.util 9135 :invoke :read [15 nil] INFO [2019-12-17 12:34:43,882] jepsen worker 31 - jepsen.util 7771 :fail :cas [10 [48 14]] :ValueNotSwapFrom INFO [2019-12-17 12:34:43,882] jepsen worker 31 - jepsen.util 7771 :invoke :cas [10 [44 9]] INFO [2019-12-17 12:34:43,887] jepsen worker 29 - jepsen.util 7229 :ok :read [9 20] INFO [2019-12-17 12:34:43,887] jepsen worker 29 - jepsen.util 7229 :invoke :cas [9 [4 12]] INFO [2019-12-17 12:34:43,896] jepsen worker 26 - jepsen.util 12356 :info :write [8 36] :RequestCanceledException INFO [2019-12-17 12:34:43,897] jepsen worker 26 - jepsen.util 12446 :invoke :read [8 nil] INFO [2019-12-17 12:34:43,910] jepsen worker 64 - jepsen.util 6994 :ok :read [21 46] INFO [2019-12-17 12:34:43,910] jepsen worker 64 - jepsen.util 6994 :invoke :write [21 20] INFO [2019-12-17 12:34:43,913] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:34:43,913] jepsen nemesis - couchbase.util Rebalance check count: 389 INFO [2019-12-17 12:34:43,913] jepsen nemesis - couchbase.util Rebalance stuck count: 370 INFO [2019-12-17 12:34:43,914] jepsen worker 58 - jepsen.util 12658 :info :write [19 37] :RequestCanceledException INFO [2019-12-17 12:34:43,915] jepsen worker 58 - jepsen.util 12748 :invoke :read [19 nil] INFO [2019-12-17 12:34:43,928] jepsen worker 88 - jepsen.util 8998 :ok :read [29 2] INFO [2019-12-17 12:34:43,928] jepsen worker 88 - jepsen.util 8998 :invoke :cas [29 [11 36]] INFO [2019-12-17 12:34:43,929] jepsen worker 85 - jepsen.util 12595 :info :write [28 42] :RequestCanceledException INFO [2019-12-17 12:34:43,930] jepsen worker 5 - jepsen.util 8105 :info :write [1 47] :RequestCanceledException INFO [2019-12-17 12:34:43,930] jepsen worker 85 - jepsen.util 12685 :invoke :read [28 nil] INFO [2019-12-17 12:34:43,931] jepsen worker 5 - jepsen.util 8195 :invoke :write [1 26] INFO [2019-12-17 12:34:43,948] jepsen worker 45 - jepsen.util 9135 :ok :read [15 17] INFO [2019-12-17 12:34:43,948] jepsen worker 45 - jepsen.util 9135 :invoke :write [15 3] INFO [2019-12-17 12:34:43,958] jepsen worker 29 - jepsen.util 7229 :fail :cas [9 [4 12]] :ValueNotSwapFrom INFO [2019-12-17 12:34:43,958] jepsen worker 29 - jepsen.util 7229 :invoke :write [9 49] INFO [2019-12-17 12:34:43,960] jepsen worker 6 - jepsen.util 12336 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:34:43,960] jepsen worker 6 - jepsen.util 12336 :invoke :read [2 nil] INFO [2019-12-17 12:34:43,974] jepsen worker 88 - jepsen.util 8998 :fail :cas [29 [11 36]] :ValueNotSwapFrom INFO [2019-12-17 12:34:43,974] jepsen worker 88 - jepsen.util 8998 :invoke :cas [29 [19 41]] INFO [2019-12-17 12:34:43,996] jepsen worker 44 - jepsen.util 12194 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:34:43,996] jepsen worker 78 - jepsen.util 8538 :info :write [26 1] :RequestCanceledException INFO [2019-12-17 12:34:43,996] jepsen worker 44 - jepsen.util 12194 :invoke :cas [14 [45 36]] INFO [2019-12-17 12:34:43,997] jepsen worker 78 - jepsen.util 8628 :invoke :write [26 35] INFO [2019-12-17 12:34:44,030] jepsen worker 22 - jepsen.util 11722 :info :cas [7 [13 38]] :RequestCanceledException INFO [2019-12-17 12:34:44,030] jepsen worker 11 - jepsen.util 8651 :ok :write [3 21] INFO [2019-12-17 12:34:44,030] jepsen worker 11 - jepsen.util 8651 :invoke :read [3 nil] INFO [2019-12-17 12:34:44,031] jepsen worker 31 - jepsen.util 7771 :fail :cas [10 [44 9]] :ValueNotSwapFrom INFO [2019-12-17 12:34:44,031] jepsen worker 31 - jepsen.util 7771 :invoke :cas [10 [40 33]] INFO [2019-12-17 12:34:44,031] jepsen worker 22 - jepsen.util 11812 :invoke :read [7 nil] INFO [2019-12-17 12:34:44,043] jepsen worker 66 - jepsen.util 11586 :info :cas [22 [35 3]] :RequestCanceledException INFO [2019-12-17 12:34:44,043] jepsen worker 48 - jepsen.util 10848 :info :write [16 41] :RequestCanceledException INFO [2019-12-17 12:34:44,044] jepsen worker 48 - jepsen.util 10938 :invoke :write [16 19] INFO [2019-12-17 12:34:44,044] jepsen worker 66 - jepsen.util 11676 :invoke :cas [22 [4 41]] INFO [2019-12-17 12:34:44,065] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":50346,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:34:44,065] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":50346,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:34:44,065] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":50349,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:34:44,065] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":50349,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:34:44,077] jepsen worker 72 - jepsen.util 8712 :ok :write [24 9] INFO [2019-12-17 12:34:44,077] jepsen worker 72 - jepsen.util 8712 :invoke :read [24 nil] INFO [2019-12-17 12:34:44,079] jepsen worker 31 - jepsen.util 7771 :fail :cas [10 [40 33]] :ValueNotSwapFrom INFO [2019-12-17 12:34:44,079] jepsen worker 31 - jepsen.util 7771 :invoke :read [10 nil] INFO [2019-12-17 12:34:44,092] jepsen worker 11 - jepsen.util 8651 :ok :read [3 21] INFO [2019-12-17 12:34:44,093] jepsen worker 11 - jepsen.util 8651 :invoke :cas [3 [8 5]] INFO [2019-12-17 12:34:44,098] jepsen worker 72 - jepsen.util 8712 :ok :read [24 9] INFO [2019-12-17 12:34:44,098] jepsen worker 72 - jepsen.util 8712 :invoke :read [24 nil] INFO [2019-12-17 12:34:44,113] jepsen worker 31 - jepsen.util 7771 :ok :read [10 14] INFO [2019-12-17 12:34:44,113] jepsen worker 31 - jepsen.util 7771 :invoke :write [10 14] INFO [2019-12-17 12:34:44,114] jepsen worker 61 - jepsen.util 12481 :info :write [20 33] :RequestCanceledException INFO [2019-12-17 12:34:44,115] jepsen worker 61 - jepsen.util 12571 :invoke :cas [20 [45 4]] INFO [2019-12-17 12:34:44,158] jepsen worker 37 - jepsen.util 8227 :ok :write [12 31] INFO [2019-12-17 12:34:44,159] jepsen worker 37 - jepsen.util 8227 :invoke :cas [12 [40 11]] INFO [2019-12-17 12:34:44,160] jepsen worker 84 - jepsen.util 11244 :info :cas [28 [9 43]] :RequestCanceledException INFO [2019-12-17 12:34:44,162] jepsen worker 84 - jepsen.util 11334 :invoke :read [28 nil] INFO [2019-12-17 12:34:44,162] jepsen worker 12 - jepsen.util 9282 :fail :cas [4 [7 31]] :ValueNotSwapFrom INFO [2019-12-17 12:34:44,163] jepsen worker 12 - jepsen.util 9282 :invoke :cas [4 [23 16]] INFO [2019-12-17 12:34:44,185] jepsen worker 4 - jepsen.util 8284 :ok :write [1 19] INFO [2019-12-17 12:34:44,185] jepsen worker 4 - jepsen.util 8284 :invoke :cas [1 [33 26]] INFO [2019-12-17 12:34:44,193] jepsen worker 72 - jepsen.util 8712 :ok :read [24 9] INFO [2019-12-17 12:34:44,193] jepsen worker 72 - jepsen.util 8712 :invoke :read [24 nil] INFO [2019-12-17 12:34:44,196] jepsen worker 11 - jepsen.util 8651 :fail :cas [3 [8 5]] :ValueNotSwapFrom INFO [2019-12-17 12:34:44,196] jepsen worker 11 - jepsen.util 8651 :invoke :read [3 nil] INFO [2019-12-17 12:34:44,200] jepsen worker 12 - jepsen.util 9282 :fail :cas [4 [23 16]] :ValueNotSwapFrom INFO [2019-12-17 12:34:44,200] jepsen worker 12 - jepsen.util 9282 :invoke :write [4 32] INFO [2019-12-17 12:34:44,210] jepsen worker 88 - jepsen.util 8998 :fail :cas [29 [19 41]] :ValueNotSwapFrom INFO [2019-12-17 12:34:44,211] jepsen worker 88 - jepsen.util 8998 :invoke :read [29 nil] INFO [2019-12-17 12:34:44,215] jepsen worker 64 - jepsen.util 6994 :ok :write [21 20] INFO [2019-12-17 12:34:44,215] jepsen worker 64 - jepsen.util 6994 :invoke :read [21 nil] INFO [2019-12-17 12:34:44,219] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":50369,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:34:44,219] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":50369,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:34:44,219] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":50371,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:34:44,220] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":50371,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:34:44,230] jepsen worker 40 - jepsen.util 12010 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:34:44,230] jepsen worker 24 - jepsen.util 10914 :info :write [8 1] :RequestCanceledException INFO [2019-12-17 12:34:44,230] jepsen worker 40 - jepsen.util 12010 :invoke :read [13 nil] INFO [2019-12-17 12:34:44,231] jepsen worker 24 - jepsen.util 11004 :invoke :read [8 nil] INFO [2019-12-17 12:34:44,248] jepsen worker 72 - jepsen.util 8712 :ok :read [24 9] INFO [2019-12-17 12:34:44,248] jepsen worker 72 - jepsen.util 8712 :invoke :cas [24 [37 40]] INFO [2019-12-17 12:34:44,277] jepsen worker 11 - jepsen.util 8651 :ok :read [3 21] INFO [2019-12-17 12:34:44,277] jepsen worker 11 - jepsen.util 8651 :invoke :write [3 38] INFO [2019-12-17 12:34:44,285] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":50379,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:34:44,285] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":50379,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:34:44,285] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":50382,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:34:44,285] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":50382,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:34:44,285] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":50383,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:34:44,285] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":50383,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:34:44,292] jepsen worker 4 - jepsen.util 8284 :fail :cas [1 [33 26]] :ValueNotSwapFrom INFO [2019-12-17 12:34:44,292] jepsen worker 4 - jepsen.util 8284 :invoke :read [1 nil] INFO [2019-12-17 12:34:44,293] jepsen worker 5 - jepsen.util 8195 :ok :write [1 26] INFO [2019-12-17 12:34:44,293] jepsen worker 5 - jepsen.util 8195 :invoke :write [1 26] INFO [2019-12-17 12:34:44,296] jepsen worker 17 - jepsen.util 11807 :info :write [5 14] :RequestCanceledException INFO [2019-12-17 12:34:44,297] jepsen worker 17 - jepsen.util 11897 :invoke :cas [5 [30 32]] INFO [2019-12-17 12:34:44,313] jepsen worker 64 - jepsen.util 6994 :ok :read [21 20] INFO [2019-12-17 12:34:44,313] jepsen worker 64 - jepsen.util 6994 :invoke :cas [21 [19 18]] INFO [2019-12-17 12:34:44,330] jepsen worker 71 - jepsen.util 9251 :info :write [23 29] :RequestCanceledException INFO [2019-12-17 12:34:44,330] jepsen worker 43 - jepsen.util 11563 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:34:44,330] jepsen worker 43 - jepsen.util 11563 :invoke :write [14 30] INFO [2019-12-17 12:34:44,331] jepsen worker 71 - jepsen.util 9341 :invoke :cas [23 [39 44]] INFO [2019-12-17 12:34:44,332] jepsen worker 72 - jepsen.util 8712 :fail :cas [24 [37 40]] :ValueNotSwapFrom INFO [2019-12-17 12:34:44,332] jepsen worker 72 - jepsen.util 8712 :invoke :cas [24 [16 41]] INFO [2019-12-17 12:34:44,338] jepsen worker 88 - jepsen.util 8998 :ok :read [29 2] INFO [2019-12-17 12:34:44,338] jepsen worker 88 - jepsen.util 8998 :invoke :read [29 nil] INFO [2019-12-17 12:34:44,344] jepsen worker 3 - jepsen.util 8553 :info :write [1 12] :RequestCanceledException INFO [2019-12-17 12:34:44,344] jepsen worker 77 - jepsen.util 12137 :info :cas [25 [36 42]] :RequestCanceledException INFO [2019-12-17 12:34:44,345] jepsen worker 3 - jepsen.util 8643 :invoke :read [1 nil] INFO [2019-12-17 12:34:44,345] jepsen worker 77 - jepsen.util 12227 :invoke :write [25 29] INFO [2019-12-17 12:34:44,355] jepsen worker 4 - jepsen.util 8284 :ok :read [1 26] INFO [2019-12-17 12:34:44,355] jepsen worker 4 - jepsen.util 8284 :invoke :read [1 nil] INFO [2019-12-17 12:34:44,355] jepsen worker 56 - jepsen.util 9506 :ok :write [18 4] INFO [2019-12-17 12:34:44,356] jepsen worker 56 - jepsen.util 9506 :invoke :cas [18 [3 12]] INFO [2019-12-17 12:34:44,360] jepsen worker 42 - jepsen.util 12822 :info :write [14 14] :RequestCanceledException INFO [2019-12-17 12:34:44,361] jepsen worker 42 - jepsen.util 12912 :invoke :cas [14 [8 48]] INFO [2019-12-17 12:34:44,369] jepsen worker 29 - jepsen.util 7229 :ok :write [9 49] INFO [2019-12-17 12:34:44,369] jepsen worker 29 - jepsen.util 7229 :invoke :read [9 nil] INFO [2019-12-17 12:34:44,382] jepsen worker 64 - jepsen.util 6994 :fail :cas [21 [19 18]] :ValueNotSwapFrom INFO [2019-12-17 12:34:44,382] jepsen worker 64 - jepsen.util 6994 :invoke :cas [21 [26 23]] INFO [2019-12-17 12:34:44,414] jepsen worker 15 - jepsen.util 10905 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:34:44,414] jepsen worker 15 - jepsen.util 10905 :invoke :write [5 29] INFO [2019-12-17 12:34:44,429] jepsen worker 1 - jepsen.util 11971 :info :write [0 37] :RequestCanceledException INFO [2019-12-17 12:34:44,429] jepsen worker 75 - jepsen.util 11235 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:34:44,429] jepsen worker 28 - jepsen.util 6958 :info :write [9 41] :RequestCanceledException INFO [2019-12-17 12:34:44,430] jepsen worker 75 - jepsen.util 11235 :invoke :read [25 nil] INFO [2019-12-17 12:34:44,430] jepsen worker 1 - jepsen.util 12061 :invoke :cas [0 [21 31]] INFO [2019-12-17 12:34:44,430] jepsen worker 28 - jepsen.util 7048 :invoke :write [9 5] INFO [2019-12-17 12:34:44,440] jepsen worker 72 - jepsen.util 8712 :fail :cas [24 [16 41]] :ValueNotSwapFrom INFO [2019-12-17 12:34:44,440] jepsen worker 72 - jepsen.util 8712 :invoke :write [24 3] INFO [2019-12-17 12:34:44,444] jepsen worker 20 - jepsen.util 8390 :info :write [6 40] :RequestCanceledException INFO [2019-12-17 12:34:44,444] jepsen worker 8 - jepsen.util 11618 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:34:44,444] jepsen worker 8 - jepsen.util 11618 :invoke :write [2 0] INFO [2019-12-17 12:34:44,445] jepsen worker 20 - jepsen.util 8480 :invoke :cas [6 [32 49]] INFO [2019-12-17 12:34:44,456] jepsen worker 64 - jepsen.util 6994 :fail :cas [21 [26 23]] :ValueNotSwapFrom INFO [2019-12-17 12:34:44,456] jepsen worker 64 - jepsen.util 6994 :invoke :cas [21 [48 45]] INFO [2019-12-17 12:34:44,457] jepsen worker 13 - jepsen.util 8383 :info :write [4 34] :RequestCanceledException INFO [2019-12-17 12:34:44,458] jepsen worker 13 - jepsen.util 8473 :invoke :write [4 18] INFO [2019-12-17 12:34:44,469] jepsen worker 29 - jepsen.util 7229 :ok :read [9 49] INFO [2019-12-17 12:34:44,469] jepsen worker 29 - jepsen.util 7229 :invoke :write [9 39] INFO [2019-12-17 12:34:44,470] jepsen worker 14 - jepsen.util 8924 :ok :read [4 1] INFO [2019-12-17 12:34:44,470] jepsen worker 14 - jepsen.util 8924 :invoke :cas [4 [9 22]] INFO [2019-12-17 12:34:44,493] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":50406,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:34:44,494] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":50406,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:34:44,494] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":50411,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:34:44,494] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":50411,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:34:44,495] jepsen worker 88 - jepsen.util 8998 :ok :read [29 2] INFO [2019-12-17 12:34:44,495] jepsen worker 88 - jepsen.util 8998 :invoke :read [29 nil] INFO [2019-12-17 12:34:44,514] jepsen worker 87 - jepsen.util 8187 :info :write [29 26] :RequestCanceledException INFO [2019-12-17 12:34:44,515] jepsen worker 87 - jepsen.util 8277 :invoke :cas [29 [33 39]] INFO [2019-12-17 12:34:44,515] jepsen worker 4 - jepsen.util 8284 :ok :read [1 26] INFO [2019-12-17 12:34:44,516] jepsen worker 4 - jepsen.util 8284 :invoke :read [1 nil] INFO [2019-12-17 12:34:44,526] jepsen worker 64 - jepsen.util 6994 :fail :cas [21 [48 45]] :ValueNotSwapFrom INFO [2019-12-17 12:34:44,526] jepsen worker 64 - jepsen.util 6994 :invoke :write [21 33] INFO [2019-12-17 12:34:44,530] jepsen worker 7 - jepsen.util 12067 :info :write [2 35] :RequestCanceledException INFO [2019-12-17 12:34:44,531] jepsen worker 7 - jepsen.util 12157 :invoke :cas [2 [10 26]] INFO [2019-12-17 12:34:44,561] jepsen worker 32 - jepsen.util 8852 :info :cas [10 [20 12]] :RequestCanceledException INFO [2019-12-17 12:34:44,562] jepsen worker 32 - jepsen.util 8942 :invoke :write [10 20] INFO [2019-12-17 12:34:44,578] jepsen worker 55 - jepsen.util 9505 :ok :write [18 18] INFO [2019-12-17 12:34:44,578] jepsen worker 55 - jepsen.util 9505 :invoke :cas [18 [1 30]] INFO [2019-12-17 12:34:44,583] jepsen worker 3 - jepsen.util 8643 :ok :read [1 26] INFO [2019-12-17 12:34:44,583] jepsen worker 3 - jepsen.util 8643 :invoke :cas [1 [36 37]] INFO [2019-12-17 12:34:44,600] jepsen worker 64 - jepsen.util 6994 :ok :write [21 33] INFO [2019-12-17 12:34:44,601] jepsen worker 64 - jepsen.util 6994 :invoke :cas [21 [31 10]] INFO [2019-12-17 12:34:44,612] jepsen worker 89 - jepsen.util 8729 :info :write [29 20] :RequestCanceledException INFO [2019-12-17 12:34:44,613] jepsen worker 89 - jepsen.util 8819 :invoke :cas [29 [8 16]] INFO [2019-12-17 12:34:44,624] jepsen worker 64 - jepsen.util 6994 :fail :cas [21 [31 10]] :ValueNotSwapFrom INFO [2019-12-17 12:34:44,624] jepsen worker 64 - jepsen.util 6994 :invoke :read [21 nil] INFO [2019-12-17 12:34:44,625] jepsen worker 80 - jepsen.util 8450 :fail :cas [26 [29 3]] :ValueNotSwapFrom INFO [2019-12-17 12:34:44,625] jepsen worker 80 - jepsen.util 8450 :invoke :write [26 16] INFO [2019-12-17 12:34:44,625] jepsen worker 45 - jepsen.util 9135 :ok :write [15 3] INFO [2019-12-17 12:34:44,626] jepsen worker 45 - jepsen.util 9135 :invoke :read [15 nil] INFO [2019-12-17 12:34:44,626] jepsen worker 31 - jepsen.util 7771 :ok :write [10 14] INFO [2019-12-17 12:34:44,626] jepsen worker 31 - jepsen.util 7771 :invoke :cas [10 [14 45]] INFO [2019-12-17 12:34:44,651] jepsen worker 29 - jepsen.util 7229 :ok :write [9 39] INFO [2019-12-17 12:34:44,651] jepsen worker 29 - jepsen.util 7229 :invoke :read [9 nil] INFO [2019-12-17 12:34:44,693] jepsen worker 64 - jepsen.util 6994 :ok :read [21 33] INFO [2019-12-17 12:34:44,693] jepsen worker 64 - jepsen.util 6994 :invoke :cas [21 [16 26]] INFO [2019-12-17 12:34:44,694] jepsen worker 49 - jepsen.util 11209 :info :cas [16 [37 7]] :RequestCanceledException INFO [2019-12-17 12:34:44,695] jepsen worker 49 - jepsen.util 11299 :invoke :read [16 nil] INFO [2019-12-17 12:34:44,696] jepsen worker 56 - jepsen.util 9506 :fail :cas [18 [3 12]] :ValueNotSwapFrom INFO [2019-12-17 12:34:44,696] jepsen worker 56 - jepsen.util 9506 :invoke :read [18 nil] INFO [2019-12-17 12:34:44,703] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":50435,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:34:44,703] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":50435,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:34:44,703] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":50438,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:34:44,703] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":50438,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:34:44,703] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":50439,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:34:44,703] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":50439,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:34:44,711] jepsen worker 45 - jepsen.util 9135 :ok :read [15 3] INFO [2019-12-17 12:34:44,712] jepsen worker 45 - jepsen.util 9135 :invoke :write [15 36] INFO [2019-12-17 12:34:44,714] jepsen worker 46 - jepsen.util 8596 :info :cas [15 [41 10]] :RequestCanceledException INFO [2019-12-17 12:34:44,715] jepsen worker 46 - jepsen.util 8686 :invoke :read [15 nil] INFO [2019-12-17 12:34:44,729] jepsen worker 2 - jepsen.util 11522 :info :write [0 48] :RequestCanceledException INFO [2019-12-17 12:34:44,729] jepsen worker 9 - jepsen.util 9099 :info :write [3 4] :RequestCanceledException INFO [2019-12-17 12:34:44,729] jepsen worker 29 - jepsen.util 7229 :ok :read [9 39] INFO [2019-12-17 12:34:44,729] jepsen worker 29 - jepsen.util 7229 :invoke :write [9 38] INFO [2019-12-17 12:34:44,730] jepsen worker 2 - jepsen.util 11612 :invoke :cas [0 [38 33]] INFO [2019-12-17 12:34:44,732] jepsen worker 9 - jepsen.util 9189 :invoke :cas [3 [33 41]] INFO [2019-12-17 12:34:44,745] jepsen worker 64 - jepsen.util 6994 :fail :cas [21 [16 26]] :ValueNotSwapFrom INFO [2019-12-17 12:34:44,746] jepsen worker 64 - jepsen.util 6994 :invoke :cas [21 [27 29]] INFO [2019-12-17 12:34:44,762] jepsen worker 55 - jepsen.util 9505 :fail :cas [18 [1 30]] :ValueNotSwapFrom INFO [2019-12-17 12:34:44,762] jepsen worker 55 - jepsen.util 9505 :invoke :cas [18 [15 41]] INFO [2019-12-17 12:34:44,777] jepsen worker 14 - jepsen.util 8924 :fail :cas [4 [9 22]] :ValueNotSwapFrom INFO [2019-12-17 12:34:44,777] jepsen worker 14 - jepsen.util 8924 :invoke :write [4 0] INFO [2019-12-17 12:34:44,780] jepsen worker 9 - jepsen.util 9189 :fail :cas [3 [33 41]] :ValueNotSwapFrom INFO [2019-12-17 12:34:44,780] jepsen worker 9 - jepsen.util 9189 :invoke :cas [3 [49 22]] INFO [2019-12-17 12:34:44,797] jepsen worker 87 - jepsen.util 8277 :fail :cas [29 [33 39]] :ValueNotSwapFrom INFO [2019-12-17 12:34:44,797] jepsen worker 87 - jepsen.util 8277 :invoke :cas [29 [43 12]] INFO [2019-12-17 12:34:44,797] jepsen worker 88 - jepsen.util 8998 :ok :read [29 2] INFO [2019-12-17 12:34:44,798] jepsen worker 88 - jepsen.util 8998 :invoke :cas [29 [26 48]] INFO [2019-12-17 12:34:44,825] jepsen worker 89 - jepsen.util 8819 :fail :cas [29 [8 16]] :ValueNotSwapFrom INFO [2019-12-17 12:34:44,825] jepsen worker 89 - jepsen.util 8819 :invoke :cas [29 [34 11]] INFO [2019-12-17 12:34:44,835] jepsen worker 64 - jepsen.util 6994 :fail :cas [21 [27 29]] :ValueNotSwapFrom INFO [2019-12-17 12:34:44,835] jepsen worker 64 - jepsen.util 6994 :invoke :write [21 24] INFO [2019-12-17 12:34:44,851] jepsen worker 9 - jepsen.util 9189 :fail :cas [3 [49 22]] :ValueNotSwapFrom INFO [2019-12-17 12:34:44,851] jepsen worker 9 - jepsen.util 9189 :invoke :read [3 nil] INFO [2019-12-17 12:34:44,861] jepsen worker 86 - jepsen.util 12686 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:34:44,861] jepsen worker 86 - jepsen.util 12686 :invoke :write [28 19] INFO [2019-12-17 12:34:44,873] jepsen worker 11 - jepsen.util 8651 :ok :write [3 38] INFO [2019-12-17 12:34:44,873] jepsen worker 11 - jepsen.util 8651 :invoke :cas [3 [23 19]] INFO [2019-12-17 12:34:44,887] jepsen worker 28 - jepsen.util 7048 :ok :write [9 5] INFO [2019-12-17 12:34:44,887] jepsen worker 28 - jepsen.util 7048 :invoke :write [9 4] INFO [2019-12-17 12:34:44,888] jepsen worker 20 - jepsen.util 8480 :fail :cas [6 [32 49]] :ValueNotSwapFrom INFO [2019-12-17 12:34:44,888] jepsen worker 20 - jepsen.util 8480 :invoke :cas [6 [7 38]] INFO [2019-12-17 12:34:44,897] jepsen worker 81 - jepsen.util 11691 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:34:44,897] jepsen worker 81 - jepsen.util 11691 :invoke :write [27 9] INFO [2019-12-17 12:34:44,907] jepsen worker 11 - jepsen.util 8651 :fail :cas [3 [23 19]] :ValueNotSwapFrom INFO [2019-12-17 12:34:44,907] jepsen worker 11 - jepsen.util 8651 :invoke :write [3 0] INFO [2019-12-17 12:34:44,915] jepsen worker 41 - jepsen.util 12641 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:34:44,915] jepsen worker 41 - jepsen.util 12641 :invoke :write [13 28] INFO [2019-12-17 12:34:44,932] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:34:44,932] jepsen nemesis - couchbase.util Rebalance check count: 390 INFO [2019-12-17 12:34:44,933] jepsen nemesis - couchbase.util Rebalance stuck count: 371 INFO [2019-12-17 12:34:44,941] jepsen worker 4 - jepsen.util 8284 :ok :read [1 26] INFO [2019-12-17 12:34:44,941] jepsen worker 4 - jepsen.util 8284 :invoke :cas [1 [12 31]] INFO [2019-12-17 12:34:44,942] jepsen worker 20 - jepsen.util 8480 :fail :cas [6 [7 38]] :ValueNotSwapFrom INFO [2019-12-17 12:34:44,942] jepsen worker 20 - jepsen.util 8480 :invoke :cas [6 [27 48]] INFO [2019-12-17 12:34:44,945] jepsen worker 64 - jepsen.util 6994 :ok :write [21 24] INFO [2019-12-17 12:34:44,945] jepsen worker 64 - jepsen.util 6994 :invoke :cas [21 [26 24]] INFO [2019-12-17 12:34:44,955] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":50459,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:34:44,955] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":50459,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:34:44,956] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":50462,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:34:44,956] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":50462,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:34:44,956] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":50463,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:34:44,956] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":50463,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:34:44,960] jepsen worker 21 - jepsen.util 11451 :info :cas [7 [6 41]] :RequestCanceledException INFO [2019-12-17 12:34:44,960] jepsen worker 39 - jepsen.util 11199 :info :write [13 28] :RequestCanceledException INFO [2019-12-17 12:34:44,961] jepsen worker 39 - jepsen.util 11289 :invoke :cas [13 [18 44]] INFO [2019-12-17 12:34:44,961] jepsen worker 21 - jepsen.util 11541 :invoke :write [7 3] INFO [2019-12-17 12:34:44,991] jepsen worker 64 - jepsen.util 6994 :fail :cas [21 [26 24]] :ValueNotSwapFrom INFO [2019-12-17 12:34:44,991] jepsen worker 64 - jepsen.util 6994 :invoke :cas [21 [13 29]] INFO [2019-12-17 12:34:44,991] jepsen worker 37 - jepsen.util 8227 :fail :cas [12 [40 11]] :ValueNotSwapFrom INFO [2019-12-17 12:34:44,991] jepsen worker 37 - jepsen.util 8227 :invoke :read [12 nil] INFO [2019-12-17 12:34:44,992] jepsen worker 53 - jepsen.util 7973 :ok :write [17 25] INFO [2019-12-17 12:34:44,992] jepsen worker 53 - jepsen.util 7973 :invoke :read [17 nil] INFO [2019-12-17 12:34:45,013] jepsen worker 72 - jepsen.util 8712 :ok :write [24 3] INFO [2019-12-17 12:34:45,013] jepsen worker 72 - jepsen.util 8712 :invoke :cas [24 [6 33]] INFO [2019-12-17 12:34:45,024] jepsen worker 28 - jepsen.util 7048 :ok :write [9 4] INFO [2019-12-17 12:34:45,024] jepsen worker 28 - jepsen.util 7048 :invoke :read [9 nil] INFO [2019-12-17 12:34:45,026] jepsen worker 64 - jepsen.util 6994 :fail :cas [21 [13 29]] :ValueNotSwapFrom INFO [2019-12-17 12:34:45,026] jepsen worker 64 - jepsen.util 6994 :invoke :write [21 30] INFO [2019-12-17 12:34:45,029] jepsen worker 76 - jepsen.util 12046 :info :cas [25 [13 0]] :RequestCanceledException INFO [2019-12-17 12:34:45,029] jepsen worker 59 - jepsen.util 11309 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:34:45,030] jepsen worker 59 - jepsen.util 11309 :invoke :read [19 nil] INFO [2019-12-17 12:34:45,031] jepsen worker 76 - jepsen.util 12136 :invoke :write [25 42] INFO [2019-12-17 12:34:45,042] jepsen worker 71 - jepsen.util 9341 :fail :cas [23 [39 44]] :ValueNotSwapFrom INFO [2019-12-17 12:34:45,043] jepsen worker 71 - jepsen.util 9341 :invoke :read [23 nil] INFO [2019-12-17 12:34:45,043] jepsen worker 52 - jepsen.util 8962 :info :write [17 16] :RequestCanceledException INFO [2019-12-17 12:34:45,043] jepsen worker 82 - jepsen.util 12412 :info :write [27 42] :RequestCanceledException INFO [2019-12-17 12:34:45,044] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":50480,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:34:45,045] jepsen worker 82 - jepsen.util 12502 :invoke :cas [27 [48 15]] INFO [2019-12-17 12:34:45,045] jepsen worker 52 - jepsen.util 9052 :invoke :read [17 nil] INFO [2019-12-17 12:34:45,045] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":50480,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:34:45,045] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":50484,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:34:45,045] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":50484,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:34:45,058] jepsen worker 28 - jepsen.util 7048 :ok :read [9 4] INFO [2019-12-17 12:34:45,058] jepsen worker 28 - jepsen.util 7048 :invoke :write [9 5] INFO [2019-12-17 12:34:45,061] jepsen worker 23 - jepsen.util 11543 :info :cas [7 [37 39]] :RequestCanceledException INFO [2019-12-17 12:34:45,063] jepsen worker 23 - jepsen.util 11633 :invoke :read [7 nil] INFO [2019-12-17 12:34:45,068] jepsen worker 69 - jepsen.util 8889 :ok :write [23 10] INFO [2019-12-17 12:34:45,068] jepsen worker 69 - jepsen.util 8889 :invoke :read [23 nil] INFO [2019-12-17 12:34:45,072] jepsen worker 13 - jepsen.util 8473 :ok :write [4 18] INFO [2019-12-17 12:34:45,072] jepsen worker 13 - jepsen.util 8473 :invoke :read [4 nil] INFO [2019-12-17 12:34:45,079] jepsen worker 45 - jepsen.util 9135 :ok :write [15 36] INFO [2019-12-17 12:34:45,079] jepsen worker 45 - jepsen.util 9135 :invoke :cas [15 [47 18]] INFO [2019-12-17 12:34:45,094] jepsen worker 5 - jepsen.util 8195 :ok :write [1 26] INFO [2019-12-17 12:34:45,094] jepsen worker 5 - jepsen.util 8195 :invoke :read [1 nil] INFO [2019-12-17 12:34:45,096] jepsen worker 73 - jepsen.util 8443 :info :write [24 28] :RequestCanceledException INFO [2019-12-17 12:34:45,097] jepsen worker 73 - jepsen.util 8533 :invoke :write [24 20] INFO [2019-12-17 12:34:45,103] jepsen worker 3 - jepsen.util 8643 :fail :cas [1 [36 37]] :ValueNotSwapFrom INFO [2019-12-17 12:34:45,103] jepsen worker 3 - jepsen.util 8643 :invoke :cas [1 [13 27]] INFO [2019-12-17 12:34:45,110] jepsen worker 46 - jepsen.util 8686 :ok :read [15 36] INFO [2019-12-17 12:34:45,110] jepsen worker 46 - jepsen.util 8686 :invoke :cas [15 [28 31]] INFO [2019-12-17 12:34:45,110] jepsen worker 78 - jepsen.util 8628 :ok :write [26 35] INFO [2019-12-17 12:34:45,110] jepsen worker 78 - jepsen.util 8628 :invoke :cas [26 [49 48]] INFO [2019-12-17 12:34:45,115] jepsen worker 34 - jepsen.util 11914 :info :cas [11 [11 20]] :RequestCanceledException INFO [2019-12-17 12:34:45,116] jepsen worker 34 - jepsen.util 12004 :invoke :cas [11 [36 28]] INFO [2019-12-17 12:34:45,117] jepsen worker 71 - jepsen.util 9341 :ok :read [23 10] INFO [2019-12-17 12:34:45,117] jepsen worker 71 - jepsen.util 9341 :invoke :read [23 nil] INFO [2019-12-17 12:34:45,129] jepsen worker 18 - jepsen.util 8928 :info :write [6 16] :RequestCanceledException INFO [2019-12-17 12:34:45,130] jepsen worker 18 - jepsen.util 9018 :invoke :write [6 30] INFO [2019-12-17 12:34:45,130] jepsen worker 83 - jepsen.util 12323 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:34:45,130] jepsen worker 83 - jepsen.util 12323 :invoke :cas [27 [22 22]] INFO [2019-12-17 12:34:45,133] jepsen worker 45 - jepsen.util 9135 :fail :cas [15 [47 18]] :ValueNotSwapFrom INFO [2019-12-17 12:34:45,133] jepsen worker 45 - jepsen.util 9135 :invoke :cas [15 [43 21]] INFO [2019-12-17 12:34:45,144] jepsen worker 68 - jepsen.util 11138 :info :write [22 16] :RequestCanceledException INFO [2019-12-17 12:34:45,145] jepsen worker 68 - jepsen.util 11228 :invoke :write [22 28] INFO [2019-12-17 12:34:45,149] jepsen worker 64 - jepsen.util 6994 :ok :write [21 30] INFO [2019-12-17 12:34:45,149] jepsen worker 64 - jepsen.util 6994 :invoke :write [21 2] INFO [2019-12-17 12:34:45,158] jepsen worker 54 - jepsen.util 9684 :ok :write [18 49] INFO [2019-12-17 12:34:45,158] jepsen worker 54 - jepsen.util 9684 :invoke :write [18 25] INFO [2019-12-17 12:34:45,160] jepsen worker 87 - jepsen.util 8277 :fail :cas [29 [43 12]] :ValueNotSwapFrom INFO [2019-12-17 12:34:45,160] jepsen worker 87 - jepsen.util 8277 :invoke :cas [29 [42 40]] INFO [2019-12-17 12:34:45,163] jepsen worker 53 - jepsen.util 7973 :ok :read [17 25] INFO [2019-12-17 12:34:45,163] jepsen worker 53 - jepsen.util 7973 :invoke :write [17 41] INFO [2019-12-17 12:34:45,178] jepsen worker 46 - jepsen.util 8686 :fail :cas [15 [28 31]] :ValueNotSwapFrom INFO [2019-12-17 12:34:45,178] jepsen worker 46 - jepsen.util 8686 :invoke :read [15 nil] INFO [2019-12-17 12:34:45,190] jepsen worker 55 - jepsen.util 9505 :fail :cas [18 [15 41]] :ValueNotSwapFrom INFO [2019-12-17 12:34:45,190] jepsen worker 55 - jepsen.util 9505 :invoke :write [18 30] INFO [2019-12-17 12:34:45,195] jepsen worker 45 - jepsen.util 9135 :fail :cas [15 [43 21]] :ValueNotSwapFrom INFO [2019-12-17 12:34:45,196] jepsen worker 65 - jepsen.util 7175 :info :cas [21 [24 27]] :RequestCanceledException INFO [2019-12-17 12:34:45,196] jepsen worker 50 - jepsen.util 12470 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:34:45,197] jepsen worker 79 - jepsen.util 9079 :info :write [26 45] :RequestCanceledException INFO [2019-12-17 12:34:45,197] jepsen worker 50 - jepsen.util 12470 :invoke :cas [16 [6 40]] INFO [2019-12-17 12:34:45,197] jepsen worker 45 - jepsen.util 9135 :invoke :read [15 nil] INFO [2019-12-17 12:34:45,197] jepsen worker 65 - jepsen.util 7265 :invoke :read [21 nil] INFO [2019-12-17 12:34:45,198] jepsen worker 79 - jepsen.util 9169 :invoke :cas [26 [30 40]] INFO [2019-12-17 12:34:45,206] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":50518,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:34:45,206] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":50518,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:34:45,206] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":50520,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:34:45,206] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":50520,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:34:45,208] jepsen worker 3 - jepsen.util 8643 :fail :cas [1 [13 27]] :ValueNotSwapFrom INFO [2019-12-17 12:34:45,209] jepsen worker 3 - jepsen.util 8643 :invoke :cas [1 [46 5]] INFO [2019-12-17 12:34:45,215] jepsen worker 74 - jepsen.util 8894 :fail :read [24 nil] :RequestTimeoutException INFO [2019-12-17 12:34:45,215] jepsen worker 74 - jepsen.util 8894 :invoke :write [24 5] INFO [2019-12-17 12:34:45,227] jepsen worker 52 - jepsen.util 9052 :ok :read [17 25] INFO [2019-12-17 12:34:45,228] jepsen worker 52 - jepsen.util 9052 :invoke :read [17 nil] INFO [2019-12-17 12:34:45,229] jepsen worker 60 - jepsen.util 12840 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:34:45,229] jepsen worker 60 - jepsen.util 12840 :invoke :cas [20 [33 7]] INFO [2019-12-17 12:34:45,230] jepsen worker 47 - jepsen.util 9047 :fail :read [15 nil] :RequestTimeoutException INFO [2019-12-17 12:34:45,230] jepsen worker 47 - jepsen.util 9047 :invoke :cas [15 [48 38]] INFO [2019-12-17 12:34:45,239] jepsen worker 11 - jepsen.util 8651 :ok :write [3 0] INFO [2019-12-17 12:34:45,239] jepsen worker 11 - jepsen.util 8651 :invoke :write [3 21] INFO [2019-12-17 12:34:45,241] jepsen worker 65 - jepsen.util 7265 :ok :read [21 30] INFO [2019-12-17 12:34:45,241] jepsen worker 65 - jepsen.util 7265 :invoke :write [21 45] INFO [2019-12-17 12:34:45,244] jepsen worker 16 - jepsen.util 12796 :info :cas [5 [43 2]] :RequestCanceledException INFO [2019-12-17 12:34:45,245] jepsen worker 16 - jepsen.util 12886 :invoke :write [5 14] INFO [2019-12-17 12:34:45,249] jepsen worker 46 - jepsen.util 8686 :ok :read [15 36] INFO [2019-12-17 12:34:45,249] jepsen worker 46 - jepsen.util 8686 :invoke :cas [15 [38 32]] INFO [2019-12-17 12:34:45,257] jepsen worker 89 - jepsen.util 8819 :fail :cas [29 [34 11]] :ValueNotSwapFrom INFO [2019-12-17 12:34:45,257] jepsen worker 89 - jepsen.util 8819 :invoke :write [29 0] INFO [2019-12-17 12:34:45,261] jepsen worker 67 - jepsen.util 12037 :info :write [22 24] :RequestCanceledException INFO [2019-12-17 12:34:45,262] jepsen worker 67 - jepsen.util 12127 :invoke :read [22 nil] INFO [2019-12-17 12:34:45,263] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":50528,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:34:45,264] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":50528,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:34:45,284] jepsen worker 13 - jepsen.util 8473 :ok :read [4 18] INFO [2019-12-17 12:34:45,284] jepsen worker 13 - jepsen.util 8473 :invoke :write [4 14] INFO [2019-12-17 12:34:45,285] jepsen worker 69 - jepsen.util 8889 :ok :read [23 10] INFO [2019-12-17 12:34:45,285] jepsen worker 69 - jepsen.util 8889 :invoke :write [23 43] INFO [2019-12-17 12:34:45,302] jepsen worker 88 - jepsen.util 8998 :fail :cas [29 [26 48]] :ValueNotSwapFrom INFO [2019-12-17 12:34:45,302] jepsen worker 88 - jepsen.util 8998 :invoke :read [29 nil] INFO [2019-12-17 12:34:45,308] jepsen worker 47 - jepsen.util 9047 :fail :cas [15 [48 38]] :ValueNotSwapFrom INFO [2019-12-17 12:34:45,308] jepsen worker 47 - jepsen.util 9047 :invoke :write [15 33] INFO [2019-12-17 12:34:45,309] jepsen worker 32 - jepsen.util 8942 :ok :write [10 20] INFO [2019-12-17 12:34:45,310] jepsen worker 32 - jepsen.util 8942 :invoke :read [10 nil] INFO [2019-12-17 12:34:45,312] jepsen worker 37 - jepsen.util 8227 :ok :read [12 31] INFO [2019-12-17 12:34:45,312] jepsen worker 37 - jepsen.util 8227 :invoke :cas [12 [4 47]] INFO [2019-12-17 12:34:45,330] jepsen worker 4 - jepsen.util 8284 :fail :cas [1 [12 31]] :ValueNotSwapFrom INFO [2019-12-17 12:34:45,330] jepsen worker 4 - jepsen.util 8284 :invoke :write [1 17] INFO [2019-12-17 12:34:45,343] jepsen worker 51 - jepsen.util 8061 :info :write [17 36] :RequestCanceledException INFO [2019-12-17 12:34:45,344] jepsen worker 51 - jepsen.util 8151 :invoke :write [17 27] INFO [2019-12-17 12:34:45,348] jepsen worker 45 - jepsen.util 9135 :ok :read [15 36] INFO [2019-12-17 12:34:45,348] jepsen worker 45 - jepsen.util 9135 :invoke :write [15 23] INFO [2019-12-17 12:34:45,362] jepsen worker 56 - jepsen.util 9506 :ok :read [18 49] INFO [2019-12-17 12:34:45,363] jepsen worker 56 - jepsen.util 9506 :invoke :cas [18 [39 15]] INFO [2019-12-17 12:34:45,378] jepsen worker 20 - jepsen.util 8480 :fail :cas [6 [27 48]] :ValueNotSwapFrom INFO [2019-12-17 12:34:45,378] jepsen worker 20 - jepsen.util 8480 :invoke :write [6 17] INFO [2019-12-17 12:34:45,379] jepsen worker 78 - jepsen.util 8628 :fail :cas [26 [49 48]] :ValueNotSwapFrom INFO [2019-12-17 12:34:45,379] jepsen worker 78 - jepsen.util 8628 :invoke :read [26 nil] INFO [2019-12-17 12:34:45,389] jepsen worker 87 - jepsen.util 8277 :fail :cas [29 [42 40]] :ValueNotSwapFrom INFO [2019-12-17 12:34:45,389] jepsen worker 87 - jepsen.util 8277 :invoke :read [29 nil] INFO [2019-12-17 12:34:45,399] jepsen worker 72 - jepsen.util 8712 :fail :cas [24 [6 33]] :ValueNotSwapFrom INFO [2019-12-17 12:34:45,399] jepsen worker 72 - jepsen.util 8712 :invoke :write [24 31] INFO [2019-12-17 12:34:45,412] jepsen worker 3 - jepsen.util 8643 :fail :cas [1 [46 5]] :ValueNotSwapFrom INFO [2019-12-17 12:34:45,413] jepsen worker 3 - jepsen.util 8643 :invoke :cas [1 [35 10]] INFO [2019-12-17 12:34:45,414] jepsen worker 10 - jepsen.util 8920 :info :write [3 38] :RequestCanceledException INFO [2019-12-17 12:34:45,415] jepsen worker 28 - jepsen.util 7048 :ok :write [9 5] INFO [2019-12-17 12:34:45,415] jepsen worker 28 - jepsen.util 7048 :invoke :read [9 nil] INFO [2019-12-17 12:34:45,415] jepsen worker 10 - jepsen.util 9010 :invoke :write [3 38] INFO [2019-12-17 12:34:45,429] jepsen worker 36 - jepsen.util 9036 :info :write [12 21] :RequestCanceledException INFO [2019-12-17 12:34:45,430] jepsen worker 27 - jepsen.util 8487 :info :write [9 46] :RequestCanceledException INFO [2019-12-17 12:34:45,430] jepsen worker 19 - jepsen.util 8749 :info :cas [6 [0 49]] :RequestCanceledException INFO [2019-12-17 12:34:45,431] jepsen worker 19 - jepsen.util 8839 :invoke :read [6 nil] INFO [2019-12-17 12:34:45,432] jepsen worker 27 - jepsen.util 8577 :invoke :cas [9 [40 35]] INFO [2019-12-17 12:34:45,432] jepsen worker 36 - jepsen.util 9126 :invoke :read [12 nil] INFO [2019-12-17 12:34:45,444] jepsen worker 35 - jepsen.util 11645 :info :write [11 14] :RequestCanceledException INFO [2019-12-17 12:34:45,444] jepsen worker 63 - jepsen.util 7623 :fail :read [21 nil] :RequestTimeoutException INFO [2019-12-17 12:34:45,444] jepsen worker 63 - jepsen.util 7623 :invoke :write [21 12] INFO [2019-12-17 12:34:45,445] jepsen worker 35 - jepsen.util 11735 :invoke :write [11 12] INFO [2019-12-17 12:34:45,451] jepsen worker 52 - jepsen.util 9052 :ok :read [17 25] INFO [2019-12-17 12:34:45,451] jepsen worker 52 - jepsen.util 9052 :invoke :write [17 23] INFO [2019-12-17 12:34:45,466] jepsen worker 88 - jepsen.util 8998 :ok :read [29 2] INFO [2019-12-17 12:34:45,466] jepsen worker 88 - jepsen.util 8998 :invoke :read [29 nil] INFO [2019-12-17 12:34:45,478] jepsen worker 28 - jepsen.util 7048 :ok :read [9 5] INFO [2019-12-17 12:34:45,478] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":50562,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:34:45,478] jepsen worker 28 - jepsen.util 7048 :invoke :read [9 nil] INFO [2019-12-17 12:34:45,478] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":50562,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:34:45,499] jepsen worker 3 - jepsen.util 8643 :fail :cas [1 [35 10]] :ValueNotSwapFrom INFO [2019-12-17 12:34:45,500] jepsen worker 3 - jepsen.util 8643 :invoke :cas [1 [0 13]] INFO [2019-12-17 12:34:45,510] jepsen worker 36 - jepsen.util 9126 :ok :read [12 31] INFO [2019-12-17 12:34:45,510] jepsen worker 36 - jepsen.util 9126 :invoke :read [12 nil] INFO [2019-12-17 12:34:45,514] jepsen worker 30 - jepsen.util 8670 :info :cas [10 [29 15]] :RequestCanceledException INFO [2019-12-17 12:34:45,515] jepsen worker 27 - jepsen.util 8577 :fail :cas [9 [40 35]] :ValueNotSwapFrom INFO [2019-12-17 12:34:45,515] jepsen worker 30 - jepsen.util 8760 :invoke :write [10 33] INFO [2019-12-17 12:34:45,515] jepsen worker 27 - jepsen.util 8577 :invoke :write [9 34] INFO [2019-12-17 12:34:45,517] jepsen worker 65 - jepsen.util 7265 :ok :write [21 45] INFO [2019-12-17 12:34:45,517] jepsen worker 65 - jepsen.util 7265 :invoke :write [21 4] INFO [2019-12-17 12:34:45,528] jepsen worker 28 - jepsen.util 7048 :ok :read [9 5] INFO [2019-12-17 12:34:45,528] jepsen worker 28 - jepsen.util 7048 :invoke :read [9 nil] INFO [2019-12-17 12:34:45,530] jepsen worker 78 - jepsen.util 8628 :ok :read [26 35] INFO [2019-12-17 12:34:45,531] jepsen worker 78 - jepsen.util 8628 :invoke :write [26 3] INFO [2019-12-17 12:34:45,556] jepsen worker 88 - jepsen.util 8998 :ok :read [29 2] INFO [2019-12-17 12:34:45,556] jepsen worker 88 - jepsen.util 8998 :invoke :cas [29 [20 21]] INFO [2019-12-17 12:34:45,558] jepsen worker 87 - jepsen.util 8277 :ok :read [29 2] INFO [2019-12-17 12:34:45,558] jepsen worker 87 - jepsen.util 8277 :invoke :read [29 nil] INFO [2019-12-17 12:34:45,568] jepsen worker 19 - jepsen.util 8839 :ok :read [6 1] INFO [2019-12-17 12:34:45,568] jepsen worker 19 - jepsen.util 8839 :invoke :read [6 nil] INFO [2019-12-17 12:34:45,578] jepsen worker 28 - jepsen.util 7048 :ok :read [9 5] INFO [2019-12-17 12:34:45,579] jepsen worker 28 - jepsen.util 7048 :invoke :cas [9 [0 21]] INFO [2019-12-17 12:34:45,597] jepsen worker 28 - jepsen.util 7048 :fail :cas [9 [0 21]] :ValueNotSwapFrom INFO [2019-12-17 12:34:45,597] jepsen worker 28 - jepsen.util 7048 :invoke :write [9 9] INFO [2019-12-17 12:34:45,597] jepsen worker 63 - jepsen.util 7623 :ok :write [21 12] INFO [2019-12-17 12:34:45,597] jepsen worker 63 - jepsen.util 7623 :invoke :read [21 nil] INFO [2019-12-17 12:34:45,599] jepsen worker 3 - jepsen.util 8643 :fail :cas [1 [0 13]] :ValueNotSwapFrom INFO [2019-12-17 12:34:45,599] jepsen worker 3 - jepsen.util 8643 :invoke :cas [1 [7 49]] INFO [2019-12-17 12:34:45,618] jepsen worker 36 - jepsen.util 9126 :ok :read [12 31] INFO [2019-12-17 12:34:45,618] jepsen worker 36 - jepsen.util 9126 :invoke :write [12 26] INFO [2019-12-17 12:34:45,634] jepsen worker 65 - jepsen.util 7265 :ok :write [21 4] INFO [2019-12-17 12:34:45,634] jepsen worker 65 - jepsen.util 7265 :invoke :write [21 18] INFO [2019-12-17 12:34:45,638] jepsen worker 18 - jepsen.util 9018 :ok :write [6 30] INFO [2019-12-17 12:34:45,638] jepsen worker 18 - jepsen.util 9018 :invoke :write [6 48] INFO [2019-12-17 12:34:45,660] jepsen worker 33 - jepsen.util 11913 :info :cas [11 [49 0]] :RequestCanceledException INFO [2019-12-17 12:34:45,661] jepsen worker 33 - jepsen.util 12003 :invoke :read [11 nil] INFO [2019-12-17 12:34:45,687] jepsen worker 74 - jepsen.util 8894 :ok :write [24 5] INFO [2019-12-17 12:34:45,687] jepsen worker 74 - jepsen.util 8894 :invoke :write [24 1] INFO [2019-12-17 12:34:45,688] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":50591,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:34:45,688] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":50591,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:34:45,688] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":50594,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:34:45,688] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":50594,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:34:45,688] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":50595,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:34:45,688] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":50595,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:34:45,696] jepsen worker 70 - jepsen.util 9160 :info :write [23 45] :RequestCanceledException INFO [2019-12-17 12:34:45,696] jepsen worker 38 - jepsen.util 9038 :info :cas [12 [24 37]] :RequestCanceledException INFO [2019-12-17 12:34:45,697] jepsen worker 70 - jepsen.util 9250 :invoke :read [23 nil] INFO [2019-12-17 12:34:45,698] jepsen worker 38 - jepsen.util 9128 :invoke :cas [12 [35 22]] INFO [2019-12-17 12:34:45,707] jepsen worker 28 - jepsen.util 7048 :ok :write [9 9] INFO [2019-12-17 12:34:45,707] jepsen worker 28 - jepsen.util 7048 :invoke :write [9 21] INFO [2019-12-17 12:34:45,710] jepsen worker 5 - jepsen.util 8195 :ok :read [1 26] INFO [2019-12-17 12:34:45,710] jepsen worker 5 - jepsen.util 8195 :invoke :cas [1 [47 46]] INFO [2019-12-17 12:34:45,722] jepsen worker 79 - jepsen.util 9169 :fail :cas [26 [30 40]] :ValueNotSwapFrom INFO [2019-12-17 12:34:45,722] jepsen worker 79 - jepsen.util 9169 :invoke :write [26 15] INFO [2019-12-17 12:34:45,751] jepsen worker 70 - jepsen.util 9250 :ok :read [23 10] INFO [2019-12-17 12:34:45,751] jepsen worker 70 - jepsen.util 9250 :invoke :write [23 14] INFO [2019-12-17 12:34:45,761] jepsen worker 25 - jepsen.util 12535 :info :write [8 40] :RequestCanceledException INFO [2019-12-17 12:34:45,762] jepsen worker 25 - jepsen.util 12625 :invoke :cas [8 [20 7]] INFO [2019-12-17 12:34:45,766] jepsen worker 65 - jepsen.util 7265 :ok :write [21 18] INFO [2019-12-17 12:34:45,766] jepsen worker 65 - jepsen.util 7265 :invoke :write [21 43] INFO [2019-12-17 12:34:45,774] jepsen worker 27 - jepsen.util 8577 :ok :write [9 34] INFO [2019-12-17 12:34:45,774] jepsen worker 27 - jepsen.util 8577 :invoke :read [9 nil] INFO [2019-12-17 12:34:45,780] jepsen worker 71 - jepsen.util 9341 :ok :read [23 10] INFO [2019-12-17 12:34:45,780] jepsen worker 71 - jepsen.util 9341 :invoke :cas [23 [27 25]] INFO [2019-12-17 12:34:45,802] jepsen worker 27 - jepsen.util 8577 :ok :read [9 34] INFO [2019-12-17 12:34:45,802] jepsen worker 27 - jepsen.util 8577 :invoke :write [9 14] INFO [2019-12-17 12:34:45,802] jepsen worker 37 - jepsen.util 8227 :fail :cas [12 [4 47]] :ValueNotSwapFrom INFO [2019-12-17 12:34:45,802] jepsen worker 37 - jepsen.util 8227 :invoke :read [12 nil] INFO [2019-12-17 12:34:45,814] jepsen worker 62 - jepsen.util 12122 :info :write [20 23] :RequestCanceledException INFO [2019-12-17 12:34:45,815] jepsen worker 62 - jepsen.util 12212 :invoke :read [20 nil] INFO [2019-12-17 12:34:45,817] jepsen worker 88 - jepsen.util 8998 :fail :cas [29 [20 21]] :ValueNotSwapFrom INFO [2019-12-17 12:34:45,817] jepsen worker 88 - jepsen.util 8998 :invoke :read [29 nil] INFO [2019-12-17 12:34:45,842] jepsen worker 57 - jepsen.util 11667 :info :write [19 23] :RequestCanceledException INFO [2019-12-17 12:34:45,842] jepsen worker 0 - jepsen.util 12330 :info :cas [0 [48 48]] :RequestCanceledException INFO [2019-12-17 12:34:45,843] jepsen worker 0 - jepsen.util 12420 :invoke :write [0 12] INFO [2019-12-17 12:34:45,843] jepsen worker 57 - jepsen.util 11757 :invoke :read [19 nil] INFO [2019-12-17 12:34:45,846] jepsen worker 65 - jepsen.util 7265 :ok :write [21 43] INFO [2019-12-17 12:34:45,846] jepsen worker 65 - jepsen.util 7265 :invoke :read [21 nil] INFO [2019-12-17 12:34:45,866] jepsen worker 65 - jepsen.util 7265 :ok :read [21 43] INFO [2019-12-17 12:34:45,866] jepsen worker 65 - jepsen.util 7265 :invoke :write [21 28] INFO [2019-12-17 12:34:45,893] jepsen worker 27 - jepsen.util 8577 :ok :write [9 14] INFO [2019-12-17 12:34:45,893] jepsen worker 27 - jepsen.util 8577 :invoke :cas [9 [23 16]] INFO [2019-12-17 12:34:45,914] jepsen worker 27 - jepsen.util 8577 :fail :cas [9 [23 16]] :ValueNotSwapFrom INFO [2019-12-17 12:34:45,914] jepsen worker 27 - jepsen.util 8577 :invoke :read [9 nil] INFO [2019-12-17 12:34:45,937] jepsen worker 27 - jepsen.util 8577 :ok :read [9 14] INFO [2019-12-17 12:34:45,938] jepsen worker 27 - jepsen.util 8577 :invoke :cas [9 [16 35]] INFO [2019-12-17 12:34:45,941] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":50616,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:34:45,941] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":50616,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:34:45,941] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":50620,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:34:45,941] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":50620,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:34:45,941] jepsen worker 37 - jepsen.util 8227 :ok :read [12 31] INFO [2019-12-17 12:34:45,942] jepsen worker 37 - jepsen.util 8227 :invoke :write [12 35] INFO [2019-12-17 12:34:45,945] jepsen worker 20 - jepsen.util 8480 :ok :write [6 17] INFO [2019-12-17 12:34:45,945] jepsen worker 20 - jepsen.util 8480 :invoke :write [6 0] INFO [2019-12-17 12:34:45,950] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:34:45,950] jepsen nemesis - couchbase.util Rebalance check count: 391 INFO [2019-12-17 12:34:45,950] jepsen nemesis - couchbase.util Rebalance stuck count: 372 INFO [2019-12-17 12:34:45,966] jepsen worker 56 - jepsen.util 9506 :fail :cas [18 [39 15]] :ValueNotSwapFrom INFO [2019-12-17 12:34:45,966] jepsen worker 56 - jepsen.util 9506 :invoke :read [18 nil] INFO [2019-12-17 12:34:45,978] jepsen worker 65 - jepsen.util 7265 :ok :write [21 28] INFO [2019-12-17 12:34:45,978] jepsen worker 65 - jepsen.util 7265 :invoke :cas [21 [8 15]] INFO [2019-12-17 12:34:45,987] jepsen worker 27 - jepsen.util 8577 :fail :cas [9 [16 35]] :ValueNotSwapFrom INFO [2019-12-17 12:34:45,987] jepsen worker 27 - jepsen.util 8577 :invoke :read [9 nil] INFO [2019-12-17 12:34:45,993] jepsen worker 26 - jepsen.util 12446 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:34:45,993] jepsen worker 26 - jepsen.util 12446 :invoke :cas [8 [48 28]] INFO [2019-12-17 12:34:46,013] jepsen worker 58 - jepsen.util 12748 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:34:46,013] jepsen worker 58 - jepsen.util 12748 :invoke :read [19 nil] INFO [2019-12-17 12:34:46,017] jepsen worker 38 - jepsen.util 9128 :fail :cas [12 [35 22]] :ValueNotSwapFrom INFO [2019-12-17 12:34:46,017] jepsen worker 38 - jepsen.util 9128 :invoke :write [12 8] INFO [2019-12-17 12:34:46,019] jepsen worker 27 - jepsen.util 8577 :ok :read [9 14] INFO [2019-12-17 12:34:46,019] jepsen worker 27 - jepsen.util 8577 :invoke :read [9 nil] INFO [2019-12-17 12:34:46,024] jepsen worker 65 - jepsen.util 7265 :fail :cas [21 [8 15]] :ValueNotSwapFrom INFO [2019-12-17 12:34:46,024] jepsen worker 65 - jepsen.util 7265 :invoke :write [21 39] INFO [2019-12-17 12:34:46,025] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":50632,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:34:46,025] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":50632,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:34:46,025] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":50633,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:34:46,025] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":50633,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:34:46,030] jepsen worker 85 - jepsen.util 12685 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:34:46,030] jepsen worker 85 - jepsen.util 12685 :invoke :read [28 nil] INFO [2019-12-17 12:34:46,057] jepsen worker 27 - jepsen.util 8577 :ok :read [9 14] INFO [2019-12-17 12:34:46,057] jepsen worker 27 - jepsen.util 8577 :invoke :write [9 45] INFO [2019-12-17 12:34:46,058] jepsen worker 5 - jepsen.util 8195 :fail :cas [1 [47 46]] :ValueNotSwapFrom INFO [2019-12-17 12:34:46,058] jepsen worker 5 - jepsen.util 8195 :invoke :read [1 nil] INFO [2019-12-17 12:34:46,058] jepsen worker 6 - jepsen.util 12336 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:34:46,058] jepsen worker 6 - jepsen.util 12336 :invoke :write [2 26] INFO [2019-12-17 12:34:46,092] jepsen worker 10 - jepsen.util 9010 :ok :write [3 38] INFO [2019-12-17 12:34:46,092] jepsen worker 10 - jepsen.util 9010 :invoke :read [3 nil] INFO [2019-12-17 12:34:46,096] jepsen worker 44 - jepsen.util 12194 :info :cas [14 [45 36]] :RequestCanceledException INFO [2019-12-17 12:34:46,096] jepsen worker 48 - jepsen.util 10938 :info :write [16 19] :RequestCanceledException INFO [2019-12-17 12:34:46,097] jepsen worker 44 - jepsen.util 12284 :invoke :cas [14 [26 46]] INFO [2019-12-17 12:34:46,097] jepsen worker 48 - jepsen.util 11028 :invoke :write [16 1] INFO [2019-12-17 12:34:46,111] jepsen worker 10 - jepsen.util 9010 :ok :read [3 38] INFO [2019-12-17 12:34:46,111] jepsen worker 10 - jepsen.util 9010 :invoke :read [3 nil] INFO [2019-12-17 12:34:46,114] jepsen worker 66 - jepsen.util 11676 :info :cas [22 [4 41]] :RequestCanceledException INFO [2019-12-17 12:34:46,115] jepsen worker 66 - jepsen.util 11766 :invoke :read [22 nil] INFO [2019-12-17 12:34:46,128] jepsen worker 65 - jepsen.util 7265 :ok :write [21 39] INFO [2019-12-17 12:34:46,128] jepsen worker 65 - jepsen.util 7265 :invoke :write [21 3] INFO [2019-12-17 12:34:46,130] jepsen worker 22 - jepsen.util 11812 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:34:46,130] jepsen worker 22 - jepsen.util 11812 :invoke :cas [7 [17 49]] INFO [2019-12-17 12:34:46,130] jepsen worker 61 - jepsen.util 12571 :info :cas [20 [45 4]] :RequestCanceledException INFO [2019-12-17 12:34:46,132] jepsen worker 61 - jepsen.util 12661 :invoke :write [20 31] INFO [2019-12-17 12:34:46,134] jepsen worker 10 - jepsen.util 9010 :ok :read [3 38] INFO [2019-12-17 12:34:46,134] jepsen worker 10 - jepsen.util 9010 :invoke :read [3 nil] INFO [2019-12-17 12:34:46,138] jepsen worker 27 - jepsen.util 8577 :ok :write [9 45] INFO [2019-12-17 12:34:46,138] jepsen worker 27 - jepsen.util 8577 :invoke :write [9 9] INFO [2019-12-17 12:34:46,174] jepsen worker 87 - jepsen.util 8277 :ok :read [29 2] INFO [2019-12-17 12:34:46,174] jepsen worker 87 - jepsen.util 8277 :invoke :write [29 48] INFO [2019-12-17 12:34:46,175] jepsen worker 88 - jepsen.util 8998 :ok :read [29 2] INFO [2019-12-17 12:34:46,175] jepsen worker 88 - jepsen.util 8998 :invoke :cas [29 [32 15]] INFO [2019-12-17 12:34:46,178] jepsen worker 10 - jepsen.util 9010 :ok :read [3 38] INFO [2019-12-17 12:34:46,178] jepsen worker 10 - jepsen.util 9010 :invoke :read [3 nil] INFO [2019-12-17 12:34:46,192] jepsen worker 79 - jepsen.util 9169 :ok :write [26 15] INFO [2019-12-17 12:34:46,192] jepsen worker 79 - jepsen.util 9169 :invoke :read [26 nil] INFO [2019-12-17 12:34:46,215] jepsen worker 20 - jepsen.util 8480 :ok :write [6 0] INFO [2019-12-17 12:34:46,215] jepsen worker 20 - jepsen.util 8480 :invoke :read [6 nil] INFO [2019-12-17 12:34:46,219] jepsen worker 3 - jepsen.util 8643 :fail :cas [1 [7 49]] :ValueNotSwapFrom INFO [2019-12-17 12:34:46,219] jepsen worker 10 - jepsen.util 9010 :ok :read [3 38] INFO [2019-12-17 12:34:46,219] jepsen worker 3 - jepsen.util 8643 :invoke :read [1 nil] INFO [2019-12-17 12:34:46,219] jepsen worker 10 - jepsen.util 9010 :invoke :cas [3 [31 47]] INFO [2019-12-17 12:34:46,243] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":50666,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:34:46,244] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":50666,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:34:46,244] jepsen worker 84 - jepsen.util 11334 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:34:46,244] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":50668,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:34:46,244] jepsen worker 84 - jepsen.util 11334 :invoke :read [28 nil] INFO [2019-12-17 12:34:46,244] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":50668,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:34:46,261] jepsen worker 24 - jepsen.util 11004 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:34:46,261] jepsen worker 24 - jepsen.util 11004 :invoke :read [8 nil] INFO [2019-12-17 12:34:46,265] jepsen worker 20 - jepsen.util 8480 :ok :read [6 0] INFO [2019-12-17 12:34:46,265] jepsen worker 20 - jepsen.util 8480 :invoke :read [6 nil] INFO [2019-12-17 12:34:46,271] jepsen worker 10 - jepsen.util 9010 :fail :cas [3 [31 47]] :ValueNotSwapFrom INFO [2019-12-17 12:34:46,271] jepsen worker 10 - jepsen.util 9010 :invoke :cas [3 [13 30]] INFO [2019-12-17 12:34:46,295] jepsen worker 20 - jepsen.util 8480 :ok :read [6 0] INFO [2019-12-17 12:34:46,295] jepsen worker 20 - jepsen.util 8480 :invoke :cas [6 [33 49]] INFO [2019-12-17 12:34:46,297] jepsen worker 10 - jepsen.util 9010 :fail :cas [3 [13 30]] :ValueNotSwapFrom INFO [2019-12-17 12:34:46,297] jepsen worker 10 - jepsen.util 9010 :invoke :cas [3 [44 9]] INFO [2019-12-17 12:34:46,300] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":50659,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:34:46,300] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":50659,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:34:46,300] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":50660,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:34:46,300] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":50660,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:34:46,300] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":50661,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:34:46,300] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":50661,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:34:46,314] jepsen worker 12 - jepsen.util 9282 :info :write [4 32] :RequestCanceledException INFO [2019-12-17 12:34:46,321] jepsen worker 12 - jepsen.util 9372 :invoke :cas [4 [29 0]] INFO [2019-12-17 12:34:46,329] jepsen worker 40 - jepsen.util 12010 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:34:46,329] jepsen worker 17 - jepsen.util 11897 :info :cas [5 [30 32]] :RequestCanceledException INFO [2019-12-17 12:34:46,330] jepsen worker 40 - jepsen.util 12010 :invoke :write [13 29] INFO [2019-12-17 12:34:46,331] jepsen worker 17 - jepsen.util 11987 :invoke :read [5 nil] INFO [2019-12-17 12:34:46,351] jepsen worker 20 - jepsen.util 8480 :fail :cas [6 [33 49]] :ValueNotSwapFrom INFO [2019-12-17 12:34:46,351] jepsen worker 20 - jepsen.util 8480 :invoke :cas [6 [8 21]] INFO [2019-12-17 12:34:46,353] jepsen worker 10 - jepsen.util 9010 :fail :cas [3 [44 9]] :ValueNotSwapFrom INFO [2019-12-17 12:34:46,353] jepsen worker 10 - jepsen.util 9010 :invoke :write [3 45] INFO [2019-12-17 12:34:46,357] jepsen worker 27 - jepsen.util 8577 :ok :write [9 9] INFO [2019-12-17 12:34:46,357] jepsen worker 27 - jepsen.util 8577 :invoke :write [9 14] INFO [2019-12-17 12:34:46,374] jepsen worker 65 - jepsen.util 7265 :ok :write [21 3] INFO [2019-12-17 12:34:46,374] jepsen worker 65 - jepsen.util 7265 :invoke :cas [21 [41 15]] INFO [2019-12-17 12:34:46,381] jepsen worker 20 - jepsen.util 8480 :fail :cas [6 [8 21]] :ValueNotSwapFrom INFO [2019-12-17 12:34:46,382] jepsen worker 20 - jepsen.util 8480 :invoke :read [6 nil] INFO [2019-12-17 12:34:46,401] jepsen worker 3 - jepsen.util 8643 :ok :read [1 26] INFO [2019-12-17 12:34:46,401] jepsen worker 3 - jepsen.util 8643 :invoke :cas [1 [37 17]] INFO [2019-12-17 12:34:46,404] jepsen worker 65 - jepsen.util 7265 :fail :cas [21 [41 15]] :ValueNotSwapFrom INFO [2019-12-17 12:34:46,404] jepsen worker 65 - jepsen.util 7265 :invoke :cas [21 [35 3]] INFO [2019-12-17 12:34:46,412] jepsen worker 20 - jepsen.util 8480 :ok :read [6 0] INFO [2019-12-17 12:34:46,412] jepsen worker 20 - jepsen.util 8480 :invoke :cas [6 [15 18]] INFO [2019-12-17 12:34:46,430] jepsen worker 43 - jepsen.util 11563 :info :write [14 30] :RequestCanceledException INFO [2019-12-17 12:34:46,430] jepsen worker 77 - jepsen.util 12227 :info :write [25 29] :RequestCanceledException INFO [2019-12-17 12:34:46,431] jepsen worker 77 - jepsen.util 12317 :invoke :write [25 24] INFO [2019-12-17 12:34:46,431] jepsen worker 43 - jepsen.util 11653 :invoke :write [14 30] INFO [2019-12-17 12:34:46,444] jepsen worker 42 - jepsen.util 12912 :info :cas [14 [8 48]] :RequestCanceledException INFO [2019-12-17 12:34:46,445] jepsen worker 42 - jepsen.util 13002 :invoke :read [14 nil] INFO [2019-12-17 12:34:46,448] jepsen worker 65 - jepsen.util 7265 :fail :cas [21 [35 3]] :ValueNotSwapFrom INFO [2019-12-17 12:34:46,448] jepsen worker 65 - jepsen.util 7265 :invoke :write [21 33] INFO [2019-12-17 12:34:46,451] jepsen worker 54 - jepsen.util 9684 :ok :write [18 25] INFO [2019-12-17 12:34:46,451] jepsen worker 54 - jepsen.util 9684 :invoke :cas [18 [44 23]] INFO [2019-12-17 12:34:46,461] jepsen worker 1 - jepsen.util 12061 :info :cas [0 [21 31]] :RequestCanceledException INFO [2019-12-17 12:34:46,462] jepsen worker 1 - jepsen.util 12151 :invoke :read [0 nil] INFO [2019-12-17 12:34:46,463] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":50692,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:34:46,463] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":50692,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:34:46,463] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":50695,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:34:46,463] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":50695,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:34:46,463] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":50696,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:34:46,463] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":50696,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:34:46,464] jepsen worker 20 - jepsen.util 8480 :fail :cas [6 [15 18]] :ValueNotSwapFrom INFO [2019-12-17 12:34:46,464] jepsen worker 20 - jepsen.util 8480 :invoke :cas [6 [3 48]] INFO [2019-12-17 12:34:46,487] jepsen worker 51 - jepsen.util 8151 :ok :write [17 27] INFO [2019-12-17 12:34:46,488] jepsen worker 51 - jepsen.util 8151 :invoke :read [17 nil] INFO [2019-12-17 12:34:46,496] jepsen worker 20 - jepsen.util 8480 :fail :cas [6 [3 48]] :ValueNotSwapFrom INFO [2019-12-17 12:34:46,503] jepsen worker 56 - jepsen.util 9506 :ok :read [18 25] INFO [2019-12-17 12:34:46,505] jepsen worker 20 - jepsen.util 8480 :invoke :write [6 35] INFO [2019-12-17 12:34:46,505] jepsen worker 56 - jepsen.util 9506 :invoke :read [18 nil] INFO [2019-12-17 12:34:46,509] jepsen worker 27 - jepsen.util 8577 :ok :write [9 14] INFO [2019-12-17 12:34:46,509] jepsen worker 27 - jepsen.util 8577 :invoke :write [9 13] INFO [2019-12-17 12:34:46,512] jepsen worker 15 - jepsen.util 10905 :info :write [5 29] :RequestCanceledException INFO [2019-12-17 12:34:46,515] jepsen worker 10 - jepsen.util 9010 :ok :write [3 45] INFO [2019-12-17 12:34:46,515] jepsen worker 10 - jepsen.util 9010 :invoke :read [3 nil] INFO [2019-12-17 12:34:46,515] jepsen worker 15 - jepsen.util 10995 :invoke :write [5 12] INFO [2019-12-17 12:34:46,530] jepsen worker 75 - jepsen.util 11235 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:34:46,531] jepsen worker 75 - jepsen.util 11235 :invoke :write [25 47] INFO [2019-12-17 12:34:46,543] jepsen worker 8 - jepsen.util 11618 :info :write [2 0] :RequestCanceledException INFO [2019-12-17 12:34:46,543] jepsen worker 7 - jepsen.util 12157 :info :cas [2 [10 26]] :RequestCanceledException INFO [2019-12-17 12:34:46,544] jepsen worker 8 - jepsen.util 11708 :invoke :cas [2 [37 45]] INFO [2019-12-17 12:34:46,544] jepsen worker 7 - jepsen.util 12247 :invoke :read [2 nil] INFO [2019-12-17 12:34:46,553] jepsen worker 37 - jepsen.util 8227 :ok :write [12 35] INFO [2019-12-17 12:34:46,553] jepsen worker 37 - jepsen.util 8227 :invoke :read [12 nil] INFO [2019-12-17 12:34:46,563] jepsen worker 3 - jepsen.util 8643 :fail :cas [1 [37 17]] :ValueNotSwapFrom INFO [2019-12-17 12:34:46,563] jepsen worker 3 - jepsen.util 8643 :invoke :read [1 nil] INFO [2019-12-17 12:34:46,572] jepsen worker 10 - jepsen.util 9010 :ok :read [3 45] INFO [2019-12-17 12:34:46,572] jepsen worker 10 - jepsen.util 9010 :invoke :write [3 25] INFO [2019-12-17 12:34:46,576] jepsen worker 88 - jepsen.util 8998 :fail :cas [29 [32 15]] :ValueNotSwapFrom INFO [2019-12-17 12:34:46,576] jepsen worker 88 - jepsen.util 8998 :invoke :read [29 nil] INFO [2019-12-17 12:34:46,586] jepsen worker 65 - jepsen.util 7265 :ok :write [21 33] INFO [2019-12-17 12:34:46,586] jepsen worker 65 - jepsen.util 7265 :invoke :write [21 1] INFO [2019-12-17 12:34:46,644] jepsen worker 80 - jepsen.util 8450 :info :write [26 16] :RequestCanceledException INFO [2019-12-17 12:34:46,645] jepsen worker 80 - jepsen.util 8540 :invoke :cas [26 [26 25]] INFO [2019-12-17 12:34:46,651] jepsen worker 3 - jepsen.util 8643 :ok :read [1 26] INFO [2019-12-17 12:34:46,651] jepsen worker 3 - jepsen.util 8643 :invoke :cas [1 [3 1]] INFO [2019-12-17 12:34:46,672] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":50719,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:34:46,672] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":50719,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:34:46,672] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":50723,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:34:46,672] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":50723,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:34:46,695] jepsen worker 27 - jepsen.util 8577 :ok :write [9 13] INFO [2019-12-17 12:34:46,696] jepsen worker 27 - jepsen.util 8577 :invoke :write [9 46] INFO [2019-12-17 12:34:46,725] jepsen worker 20 - jepsen.util 8480 :ok :write [6 35] INFO [2019-12-17 12:34:46,725] jepsen worker 20 - jepsen.util 8480 :invoke :write [6 29] INFO [2019-12-17 12:34:46,729] jepsen worker 31 - jepsen.util 7771 :info :cas [10 [14 45]] :RequestCanceledException INFO [2019-12-17 12:34:46,730] jepsen worker 31 - jepsen.util 7861 :invoke :cas [10 [35 28]] INFO [2019-12-17 12:34:46,762] jepsen worker 65 - jepsen.util 7265 :ok :write [21 1] INFO [2019-12-17 12:34:46,763] jepsen worker 65 - jepsen.util 7265 :invoke :cas [21 [10 19]] INFO [2019-12-17 12:34:46,776] jepsen worker 31 - jepsen.util 7861 :fail :cas [10 [35 28]] :ValueNotSwapFrom INFO [2019-12-17 12:34:46,776] jepsen worker 31 - jepsen.util 7861 :invoke :cas [10 [20 21]] INFO [2019-12-17 12:34:46,793] jepsen worker 49 - jepsen.util 11299 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:34:46,793] jepsen worker 49 - jepsen.util 11299 :invoke :cas [16 [23 41]] INFO [2019-12-17 12:34:46,809] jepsen worker 10 - jepsen.util 9010 :ok :write [3 25] INFO [2019-12-17 12:34:46,809] jepsen worker 10 - jepsen.util 9010 :invoke :write [3 13] INFO [2019-12-17 12:34:46,821] jepsen worker 27 - jepsen.util 8577 :ok :write [9 46] INFO [2019-12-17 12:34:46,821] jepsen worker 27 - jepsen.util 8577 :invoke :write [9 21] INFO [2019-12-17 12:34:46,825] jepsen worker 65 - jepsen.util 7265 :fail :cas [21 [10 19]] :ValueNotSwapFrom INFO [2019-12-17 12:34:46,825] jepsen worker 65 - jepsen.util 7265 :invoke :cas [21 [3 49]] INFO [2019-12-17 12:34:46,828] jepsen worker 54 - jepsen.util 9684 :fail :cas [18 [44 23]] :ValueNotSwapFrom INFO [2019-12-17 12:34:46,828] jepsen worker 54 - jepsen.util 9684 :invoke :cas [18 [33 40]] INFO [2019-12-17 12:34:46,829] jepsen worker 2 - jepsen.util 11612 :info :cas [0 [38 33]] :RequestCanceledException INFO [2019-12-17 12:34:46,829] jepsen worker 29 - jepsen.util 7229 :info :write [9 38] :RequestCanceledException INFO [2019-12-17 12:34:46,830] jepsen worker 14 - jepsen.util 8924 :info :write [4 0] :RequestCanceledException INFO [2019-12-17 12:34:46,830] jepsen worker 2 - jepsen.util 11702 :invoke :read [0 nil] INFO [2019-12-17 12:34:46,831] jepsen worker 29 - jepsen.util 7319 :invoke :cas [9 [47 11]] INFO [2019-12-17 12:34:46,831] jepsen worker 14 - jepsen.util 9014 :invoke :write [4 44] INFO [2019-12-17 12:34:46,849] jepsen worker 56 - jepsen.util 9506 :ok :read [18 25] INFO [2019-12-17 12:34:46,850] jepsen worker 56 - jepsen.util 9506 :invoke :write [18 21] INFO [2019-12-17 12:34:46,882] jepsen worker 65 - jepsen.util 7265 :fail :cas [21 [3 49]] :ValueNotSwapFrom INFO [2019-12-17 12:34:46,882] jepsen worker 65 - jepsen.util 7265 :invoke :read [21 nil] INFO [2019-12-17 12:34:46,912] jepsen worker 65 - jepsen.util 7265 :ok :read [21 1] INFO [2019-12-17 12:34:46,912] jepsen worker 65 - jepsen.util 7265 :invoke :cas [21 [48 2]] INFO [2019-12-17 12:34:46,922] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":50740,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:34:46,922] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":50740,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:34:46,922] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":50741,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:34:46,922] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":50741,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:34:46,922] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":50742,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:34:46,922] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":50742,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:34:46,937] jepsen worker 88 - jepsen.util 8998 :ok :read [29 2] INFO [2019-12-17 12:34:46,937] jepsen worker 88 - jepsen.util 8998 :invoke :cas [29 [16 36]] INFO [2019-12-17 12:34:46,942] jepsen worker 71 - jepsen.util 9341 :fail :cas [23 [27 25]] :ValueNotSwapFrom INFO [2019-12-17 12:34:46,942] jepsen worker 71 - jepsen.util 9341 :invoke :read [23 nil] INFO [2019-12-17 12:34:46,943] jepsen worker 9 - jepsen.util 9189 :fail :read [3 nil] :RequestTimeoutException INFO [2019-12-17 12:34:46,943] jepsen worker 9 - jepsen.util 9189 :invoke :cas [3 [49 1]] INFO [2019-12-17 12:34:46,960] jepsen worker 86 - jepsen.util 12686 :info :write [28 19] :RequestCanceledException INFO [2019-12-17 12:34:46,961] jepsen worker 65 - jepsen.util 7265 :fail :cas [21 [48 2]] :ValueNotSwapFrom INFO [2019-12-17 12:34:46,961] jepsen worker 65 - jepsen.util 7265 :invoke :cas [21 [16 0]] INFO [2019-12-17 12:34:46,962] jepsen worker 86 - jepsen.util 12776 :invoke :write [28 44] INFO [2019-12-17 12:34:46,966] jepsen worker 27 - jepsen.util 8577 :ok :write [9 21] INFO [2019-12-17 12:34:46,966] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:34:46,966] jepsen worker 27 - jepsen.util 8577 :invoke :read [9 nil] INFO [2019-12-17 12:34:46,966] jepsen nemesis - couchbase.util Rebalance check count: 392 INFO [2019-12-17 12:34:46,966] jepsen nemesis - couchbase.util Rebalance stuck count: 373 INFO [2019-12-17 12:34:46,996] jepsen worker 81 - jepsen.util 11691 :info :write [27 9] :RequestCanceledException INFO [2019-12-17 12:34:46,998] jepsen worker 81 - jepsen.util 11781 :invoke :read [27 nil] INFO [2019-12-17 12:34:47,007] jepsen worker 9 - jepsen.util 9189 :fail :cas [3 [49 1]] :ValueNotSwapFrom INFO [2019-12-17 12:34:47,007] jepsen worker 9 - jepsen.util 9189 :invoke :write [3 43] INFO [2019-12-17 12:34:47,010] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":50756,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:34:47,010] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":50756,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:34:47,010] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":50759,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:34:47,010] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":50759,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:34:47,010] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":50760,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:34:47,010] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":50760,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:34:47,010] jepsen worker 3 - jepsen.util 8643 :fail :cas [1 [3 1]] :ValueNotSwapFrom INFO [2019-12-17 12:34:47,011] jepsen worker 3 - jepsen.util 8643 :invoke :write [1 12] INFO [2019-12-17 12:34:47,014] jepsen worker 5 - jepsen.util 8195 :ok :read [1 26] INFO [2019-12-17 12:34:47,015] jepsen worker 5 - jepsen.util 8195 :invoke :read [1 nil] INFO [2019-12-17 12:34:47,015] jepsen worker 41 - jepsen.util 12641 :info :write [13 28] :RequestCanceledException INFO [2019-12-17 12:34:47,015] jepsen worker 41 - jepsen.util 12731 :invoke :cas [13 [28 31]] INFO [2019-12-17 12:34:47,024] jepsen worker 27 - jepsen.util 8577 :ok :read [9 21] INFO [2019-12-17 12:34:47,025] jepsen worker 27 - jepsen.util 8577 :invoke :write [9 2] INFO [2019-12-17 12:34:47,029] jepsen worker 51 - jepsen.util 8151 :ok :read [17 27] INFO [2019-12-17 12:34:47,029] jepsen worker 51 - jepsen.util 8151 :invoke :read [17 nil] INFO [2019-12-17 12:34:47,029] jepsen worker 65 - jepsen.util 7265 :fail :cas [21 [16 0]] :ValueNotSwapFrom INFO [2019-12-17 12:34:47,029] jepsen worker 21 - jepsen.util 11541 :info :write [7 3] :RequestCanceledException INFO [2019-12-17 12:34:47,029] jepsen worker 65 - jepsen.util 7265 :invoke :read [21 nil] INFO [2019-12-17 12:34:47,030] jepsen worker 21 - jepsen.util 11631 :invoke :read [7 nil] INFO [2019-12-17 12:34:47,039] jepsen worker 79 - jepsen.util 9169 :ok :read [26 15] INFO [2019-12-17 12:34:47,039] jepsen worker 79 - jepsen.util 9169 :invoke :write [26 39] INFO [2019-12-17 12:34:47,049] jepsen worker 12 - jepsen.util 9372 :fail :cas [4 [29 0]] :ValueNotSwapFrom INFO [2019-12-17 12:34:47,049] jepsen worker 12 - jepsen.util 9372 :invoke :read [4 nil] INFO [2019-12-17 12:34:47,061] jepsen worker 39 - jepsen.util 11289 :info :cas [13 [18 44]] :RequestCanceledException INFO [2019-12-17 12:34:47,062] jepsen worker 39 - jepsen.util 11379 :invoke :write [13 0] INFO [2019-12-17 12:34:47,084] jepsen worker 51 - jepsen.util 8151 :ok :read [17 27] INFO [2019-12-17 12:34:47,084] jepsen worker 51 - jepsen.util 8151 :invoke :cas [17 [32 7]] INFO [2019-12-17 12:34:47,090] jepsen worker 65 - jepsen.util 7265 :ok :read [21 1] INFO [2019-12-17 12:34:47,090] jepsen worker 65 - jepsen.util 7265 :invoke :cas [21 [6 27]] INFO [2019-12-17 12:34:47,096] jepsen worker 76 - jepsen.util 12136 :info :write [25 42] :RequestCanceledException INFO [2019-12-17 12:34:47,097] jepsen worker 76 - jepsen.util 12226 :invoke :cas [25 [2 41]] INFO [2019-12-17 12:34:47,101] jepsen worker 65 - jepsen.util 7265 :fail :cas [21 [6 27]] :ValueNotSwapFrom INFO [2019-12-17 12:34:47,102] jepsen worker 65 - jepsen.util 7265 :invoke :cas [21 [0 3]] INFO [2019-12-17 12:34:47,114] jepsen worker 82 - jepsen.util 12502 :info :cas [27 [48 15]] :RequestCanceledException INFO [2019-12-17 12:34:47,115] jepsen worker 82 - jepsen.util 12592 :invoke :cas [27 [4 31]] INFO [2019-12-17 12:34:47,118] jepsen worker 12 - jepsen.util 9372 :ok :read [4 18] INFO [2019-12-17 12:34:47,118] jepsen worker 12 - jepsen.util 9372 :invoke :read [4 nil] INFO [2019-12-17 12:34:47,123] jepsen worker 37 - jepsen.util 8227 :ok :read [12 35] INFO [2019-12-17 12:34:47,123] jepsen worker 37 - jepsen.util 8227 :invoke :cas [12 [36 39]] INFO [2019-12-17 12:34:47,123] jepsen worker 65 - jepsen.util 7265 :fail :cas [21 [0 3]] :ValueNotSwapFrom INFO [2019-12-17 12:34:47,123] jepsen worker 65 - jepsen.util 7265 :invoke :read [21 nil] INFO [2019-12-17 12:34:47,130] jepsen worker 59 - jepsen.util 11309 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:34:47,130] jepsen worker 34 - jepsen.util 12004 :info :cas [11 [36 28]] :RequestCanceledException INFO [2019-12-17 12:34:47,130] jepsen worker 59 - jepsen.util 11309 :invoke :read [19 nil] INFO [2019-12-17 12:34:47,131] jepsen worker 34 - jepsen.util 12094 :invoke :cas [11 [10 17]] INFO [2019-12-17 12:34:47,134] jepsen worker 14 - jepsen.util 9014 :ok :write [4 44] INFO [2019-12-17 12:34:47,134] jepsen worker 14 - jepsen.util 9014 :invoke :read [4 nil] INFO [2019-12-17 12:34:47,140] jepsen worker 65 - jepsen.util 7265 :ok :read [21 1] INFO [2019-12-17 12:34:47,140] jepsen worker 65 - jepsen.util 7265 :invoke :cas [21 [4 16]] INFO [2019-12-17 12:34:47,144] jepsen worker 23 - jepsen.util 11633 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:34:47,144] jepsen worker 23 - jepsen.util 11633 :invoke :write [7 32] INFO [2019-12-17 12:34:47,154] jepsen worker 65 - jepsen.util 7265 :fail :cas [21 [4 16]] :ValueNotSwapFrom INFO [2019-12-17 12:34:47,154] jepsen worker 65 - jepsen.util 7265 :invoke :read [21 nil] INFO [2019-12-17 12:34:47,161] jepsen worker 73 - jepsen.util 8533 :info :write [24 20] :RequestCanceledException INFO [2019-12-17 12:34:47,162] jepsen worker 73 - jepsen.util 8623 :invoke :write [24 18] INFO [2019-12-17 12:34:47,167] jepsen worker 65 - jepsen.util 7265 :ok :read [21 1] INFO [2019-12-17 12:34:47,167] jepsen worker 65 - jepsen.util 7265 :invoke :cas [21 [25 37]] INFO [2019-12-17 12:34:47,179] jepsen worker 80 - jepsen.util 8540 :fail :cas [26 [26 25]] :ValueNotSwapFrom INFO [2019-12-17 12:34:47,179] jepsen worker 80 - jepsen.util 8540 :invoke :read [26 nil] INFO [2019-12-17 12:34:47,180] jepsen worker 65 - jepsen.util 7265 :fail :cas [21 [25 37]] :ValueNotSwapFrom INFO [2019-12-17 12:34:47,180] jepsen worker 65 - jepsen.util 7265 :invoke :read [21 nil] INFO [2019-12-17 12:34:47,194] jepsen worker 65 - jepsen.util 7265 :ok :read [21 1] INFO [2019-12-17 12:34:47,194] jepsen worker 65 - jepsen.util 7265 :invoke :cas [21 [34 48]] INFO [2019-12-17 12:34:47,196] jepsen worker 51 - jepsen.util 8151 :fail :cas [17 [32 7]] :ValueNotSwapFrom INFO [2019-12-17 12:34:47,196] jepsen worker 51 - jepsen.util 8151 :invoke :read [17 nil] INFO [2019-12-17 12:34:47,197] jepsen worker 53 - jepsen.util 7973 :info :write [17 41] :RequestCanceledException INFO [2019-12-17 12:34:47,198] jepsen worker 53 - jepsen.util 8063 :invoke :cas [17 [17 7]] INFO [2019-12-17 12:34:47,209] jepsen worker 65 - jepsen.util 7265 :fail :cas [21 [34 48]] :ValueNotSwapFrom INFO [2019-12-17 12:34:47,209] jepsen worker 65 - jepsen.util 7265 :invoke :cas [21 [10 37]] INFO [2019-12-17 12:34:47,214] jepsen worker 68 - jepsen.util 11228 :info :write [22 28] :RequestCanceledException INFO [2019-12-17 12:34:47,215] jepsen worker 68 - jepsen.util 11318 :invoke :read [22 nil] INFO [2019-12-17 12:34:47,220] jepsen worker 65 - jepsen.util 7265 :fail :cas [21 [10 37]] :ValueNotSwapFrom INFO [2019-12-17 12:34:47,220] jepsen worker 20 - jepsen.util 8480 :ok :write [6 29] INFO [2019-12-17 12:34:47,220] jepsen worker 65 - jepsen.util 7265 :invoke :cas [21 [46 44]] INFO [2019-12-17 12:34:47,220] jepsen worker 31 - jepsen.util 7861 :ok :cas [10 [20 21]] INFO [2019-12-17 12:34:47,220] jepsen worker 20 - jepsen.util 8480 :invoke :read [6 nil] INFO [2019-12-17 12:34:47,220] jepsen worker 31 - jepsen.util 7861 :invoke :read [10 nil] INFO [2019-12-17 12:34:47,227] jepsen worker 4 - jepsen.util 8284 :ok :write [1 17] INFO [2019-12-17 12:34:47,227] jepsen worker 4 - jepsen.util 8284 :invoke :read [1 nil] INFO [2019-12-17 12:34:47,229] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":50805,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:34:47,230] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":50805,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:34:47,230] jepsen worker 83 - jepsen.util 12323 :info :cas [27 [22 22]] :RequestCanceledException INFO [2019-12-17 12:34:47,230] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":50807,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:34:47,230] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":50807,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:34:47,231] jepsen worker 83 - jepsen.util 12413 :invoke :cas [27 [27 18]] INFO [2019-12-17 12:34:47,244] jepsen worker 55 - jepsen.util 9505 :info :write [18 30] :RequestCanceledException INFO [2019-12-17 12:34:47,244] jepsen worker 64 - jepsen.util 6994 :info :write [21 2] :RequestCanceledException INFO [2019-12-17 12:34:47,245] jepsen worker 55 - jepsen.util 9595 :invoke :write [18 6] INFO [2019-12-17 12:34:47,245] jepsen worker 64 - jepsen.util 7084 :invoke :cas [21 [15 4]] INFO [2019-12-17 12:34:47,257] jepsen worker 71 - jepsen.util 9341 :ok :read [23 10] INFO [2019-12-17 12:34:47,257] jepsen worker 71 - jepsen.util 9341 :invoke :write [23 44] INFO [2019-12-17 12:34:47,262] jepsen worker 16 - jepsen.util 12886 :info :write [5 14] :RequestCanceledException INFO [2019-12-17 12:34:47,262] jepsen worker 10 - jepsen.util 9010 :ok :write [3 13] INFO [2019-12-17 12:34:47,263] jepsen worker 10 - jepsen.util 9010 :invoke :write [3 8] INFO [2019-12-17 12:34:47,263] jepsen worker 16 - jepsen.util 12976 :invoke :write [5 23] INFO [2019-12-17 12:34:47,285] jepsen worker 37 - jepsen.util 8227 :fail :cas [12 [36 39]] :ValueNotSwapFrom INFO [2019-12-17 12:34:47,285] jepsen worker 37 - jepsen.util 8227 :invoke :cas [12 [29 5]] INFO [2019-12-17 12:34:47,286] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":50793,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:34:47,286] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":50793,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:34:47,286] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":50795,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:34:47,286] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":50795,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:34:47,296] jepsen worker 50 - jepsen.util 12470 :info :cas [16 [6 40]] :RequestCanceledException INFO [2019-12-17 12:34:47,296] jepsen worker 11 - jepsen.util 8651 :info :write [3 21] :RequestCanceledException INFO [2019-12-17 12:34:47,297] jepsen worker 50 - jepsen.util 12560 :invoke :cas [16 [27 33]] INFO [2019-12-17 12:34:47,297] jepsen worker 11 - jepsen.util 8741 :invoke :write [3 40] INFO [2019-12-17 12:34:47,314] jepsen worker 13 - jepsen.util 8473 :info :write [4 14] :RequestCanceledException INFO [2019-12-17 12:34:47,314] jepsen worker 46 - jepsen.util 8686 :info :cas [15 [38 32]] :RequestCanceledException INFO [2019-12-17 12:34:47,315] jepsen worker 46 - jepsen.util 8776 :invoke :cas [15 [28 24]] INFO [2019-12-17 12:34:47,315] jepsen worker 13 - jepsen.util 8563 :invoke :read [4 nil] INFO [2019-12-17 12:34:47,329] jepsen worker 60 - jepsen.util 12840 :info :cas [20 [33 7]] :RequestCanceledException INFO [2019-12-17 12:34:47,329] jepsen worker 89 - jepsen.util 8819 :info :write [29 0] :RequestCanceledException INFO [2019-12-17 12:34:47,329] jepsen worker 67 - jepsen.util 12127 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:34:47,330] jepsen worker 67 - jepsen.util 12127 :invoke :read [22 nil] INFO [2019-12-17 12:34:47,330] jepsen worker 69 - jepsen.util 8889 :info :write [23 43] :RequestCanceledException INFO [2019-12-17 12:34:47,330] jepsen worker 47 - jepsen.util 9047 :info :write [15 33] :RequestCanceledException INFO [2019-12-17 12:34:47,331] jepsen worker 60 - jepsen.util 12930 :invoke :write [20 37] INFO [2019-12-17 12:34:47,331] jepsen worker 89 - jepsen.util 8909 :invoke :cas [29 [47 38]] INFO [2019-12-17 12:34:47,331] jepsen worker 69 - jepsen.util 8979 :invoke :cas [23 [17 33]] INFO [2019-12-17 12:34:47,331] jepsen worker 47 - jepsen.util 9137 :invoke :write [15 28] INFO [2019-12-17 12:34:47,349] jepsen worker 64 - jepsen.util 7084 :fail :cas [21 [15 4]] :ValueNotSwapFrom INFO [2019-12-17 12:34:47,349] jepsen worker 64 - jepsen.util 7084 :invoke :read [21 nil] INFO [2019-12-17 12:34:47,359] jepsen worker 13 - jepsen.util 8563 :ok :read [4 44] INFO [2019-12-17 12:34:47,359] jepsen worker 13 - jepsen.util 8563 :invoke :cas [4 [17 15]] INFO [2019-12-17 12:34:47,361] jepsen worker 32 - jepsen.util 8942 :fail :read [10 nil] :RequestTimeoutException INFO [2019-12-17 12:34:47,361] jepsen worker 65 - jepsen.util 7265 :fail :cas [21 [46 44]] :ValueNotSwapFrom INFO [2019-12-17 12:34:47,361] jepsen worker 32 - jepsen.util 8942 :invoke :write [10 39] INFO [2019-12-17 12:34:47,361] jepsen worker 65 - jepsen.util 7265 :invoke :cas [21 [5 44]] INFO [2019-12-17 12:34:47,395] jepsen worker 5 - jepsen.util 8195 :ok :read [1 17] INFO [2019-12-17 12:34:47,395] jepsen worker 5 - jepsen.util 8195 :invoke :write [1 38] INFO [2019-12-17 12:34:47,416] jepsen worker 45 - jepsen.util 9135 :info :write [15 23] :RequestCanceledException INFO [2019-12-17 12:34:47,416] jepsen worker 45 - jepsen.util 9225 :invoke :write [15 2] INFO [2019-12-17 12:34:47,439] jepsen worker 29 - jepsen.util 7319 :fail :cas [9 [47 11]] :ValueNotSwapFrom INFO [2019-12-17 12:34:47,439] jepsen worker 29 - jepsen.util 7319 :invoke :write [9 47] INFO [2019-12-17 12:34:47,449] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":50835,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:34:47,449] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":50835,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:34:47,449] jepsen worker 64 - jepsen.util 7084 :ok :read [21 1] INFO [2019-12-17 12:34:47,449] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":50838,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:34:47,449] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":50838,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:34:47,449] jepsen worker 64 - jepsen.util 7084 :invoke :write [21 39] INFO [2019-12-17 12:34:47,459] jepsen worker 65 - jepsen.util 7265 :fail :cas [21 [5 44]] :ValueNotSwapFrom INFO [2019-12-17 12:34:47,459] jepsen worker 65 - jepsen.util 7265 :invoke :cas [21 [25 13]] INFO [2019-12-17 12:34:47,459] jepsen worker 72 - jepsen.util 8712 :info :write [24 31] :RequestCanceledException INFO [2019-12-17 12:34:47,460] jepsen worker 72 - jepsen.util 8802 :invoke :read [24 nil] INFO [2019-12-17 12:34:47,481] jepsen worker 78 - jepsen.util 8628 :ok :write [26 3] INFO [2019-12-17 12:34:47,481] jepsen worker 78 - jepsen.util 8628 :invoke :cas [26 [23 46]] INFO [2019-12-17 12:34:47,486] jepsen worker 80 - jepsen.util 8540 :ok :read [26 15] INFO [2019-12-17 12:34:47,487] jepsen worker 80 - jepsen.util 8540 :invoke :cas [26 [33 3]] INFO [2019-12-17 12:34:47,513] jepsen worker 20 - jepsen.util 8480 :ok :read [6 29] INFO [2019-12-17 12:34:47,513] jepsen worker 20 - jepsen.util 8480 :invoke :read [6 nil] INFO [2019-12-17 12:34:47,517] jepsen worker 65 - jepsen.util 7265 :fail :cas [21 [25 13]] :ValueNotSwapFrom INFO [2019-12-17 12:34:47,518] jepsen worker 65 - jepsen.util 7265 :invoke :cas [21 [35 38]] INFO [2019-12-17 12:34:47,530] jepsen worker 35 - jepsen.util 11735 :info :write [11 12] :RequestCanceledException INFO [2019-12-17 12:34:47,530] jepsen worker 52 - jepsen.util 9052 :info :write [17 23] :RequestCanceledException INFO [2019-12-17 12:34:47,531] jepsen worker 52 - jepsen.util 9142 :invoke :write [17 42] INFO [2019-12-17 12:34:47,531] jepsen worker 35 - jepsen.util 11825 :invoke :write [11 47] INFO [2019-12-17 12:34:47,544] jepsen worker 89 - jepsen.util 8909 :fail :cas [29 [47 38]] :ValueNotSwapFrom INFO [2019-12-17 12:34:47,544] jepsen worker 89 - jepsen.util 8909 :invoke :read [29 nil] INFO [2019-12-17 12:34:47,544] jepsen worker 72 - jepsen.util 8802 :ok :read [24 5] INFO [2019-12-17 12:34:47,545] jepsen worker 72 - jepsen.util 8802 :invoke :write [24 19] INFO [2019-12-17 12:34:47,561] jepsen worker 88 - jepsen.util 8998 :fail :cas [29 [16 36]] :ValueNotSwapFrom INFO [2019-12-17 12:34:47,561] jepsen worker 88 - jepsen.util 8998 :invoke :cas [29 [22 24]] INFO [2019-12-17 12:34:47,571] jepsen worker 4 - jepsen.util 8284 :ok :read [1 17] INFO [2019-12-17 12:34:47,571] jepsen worker 4 - jepsen.util 8284 :invoke :write [1 13] INFO [2019-12-17 12:34:47,575] jepsen worker 3 - jepsen.util 8643 :ok :write [1 12] INFO [2019-12-17 12:34:47,575] jepsen worker 3 - jepsen.util 8643 :invoke :write [1 44] INFO [2019-12-17 12:34:47,581] jepsen worker 65 - jepsen.util 7265 :fail :cas [21 [35 38]] :ValueNotSwapFrom INFO [2019-12-17 12:34:47,581] jepsen worker 65 - jepsen.util 7265 :invoke :write [21 13] INFO [2019-12-17 12:34:47,594] jepsen worker 38 - jepsen.util 9128 :ok :write [12 8] INFO [2019-12-17 12:34:47,594] jepsen worker 38 - jepsen.util 9128 :invoke :write [12 14] INFO [2019-12-17 12:34:47,594] jepsen worker 14 - jepsen.util 9014 :ok :read [4 14] INFO [2019-12-17 12:34:47,594] jepsen worker 14 - jepsen.util 9014 :invoke :read [4 nil] INFO [2019-12-17 12:34:47,596] jepsen worker 30 - jepsen.util 8760 :info :write [10 33] :RequestCanceledException INFO [2019-12-17 12:34:47,597] jepsen worker 30 - jepsen.util 8850 :invoke :cas [10 [4 6]] INFO [2019-12-17 12:34:47,629] jepsen worker 30 - jepsen.util 8850 :fail :cas [10 [4 6]] :ValueNotSwapFrom INFO [2019-12-17 12:34:47,629] jepsen worker 30 - jepsen.util 8850 :invoke :write [10 22] INFO [2019-12-17 12:34:47,629] jepsen worker 19 - jepsen.util 8839 :fail :read [6 nil] :RequestTimeoutException INFO [2019-12-17 12:34:47,630] jepsen worker 19 - jepsen.util 8839 :invoke :write [6 40] INFO [2019-12-17 12:34:47,633] jepsen worker 37 - jepsen.util 8227 :fail :cas [12 [29 5]] :ValueNotSwapFrom INFO [2019-12-17 12:34:47,633] jepsen worker 37 - jepsen.util 8227 :invoke :write [12 20] INFO [2019-12-17 12:34:47,644] jepsen worker 63 - jepsen.util 7623 :fail :read [21 nil] :RequestTimeoutException INFO [2019-12-17 12:34:47,644] jepsen worker 63 - jepsen.util 7623 :invoke :write [21 25] INFO [2019-12-17 12:34:47,650] jepsen worker 64 - jepsen.util 7084 :ok :write [21 39] INFO [2019-12-17 12:34:47,651] jepsen worker 64 - jepsen.util 7084 :invoke :read [21 nil] INFO [2019-12-17 12:34:47,659] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":50864,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:34:47,659] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":50864,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:34:47,659] jepsen worker 36 - jepsen.util 9126 :info :write [12 26] :RequestCanceledException INFO [2019-12-17 12:34:47,659] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":50867,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:34:47,659] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":50867,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:34:47,660] jepsen worker 36 - jepsen.util 9216 :invoke :read [12 nil] INFO [2019-12-17 12:34:47,669] jepsen worker 89 - jepsen.util 8909 :ok :read [29 2] INFO [2019-12-17 12:34:47,669] jepsen worker 89 - jepsen.util 8909 :invoke :read [29 nil] INFO [2019-12-17 12:34:47,672] jepsen worker 71 - jepsen.util 9341 :ok :write [23 44] INFO [2019-12-17 12:34:47,672] jepsen worker 71 - jepsen.util 9341 :invoke :write [23 40] INFO [2019-12-17 12:34:47,691] jepsen worker 78 - jepsen.util 8628 :fail :cas [26 [23 46]] :ValueNotSwapFrom INFO [2019-12-17 12:34:47,691] jepsen worker 78 - jepsen.util 8628 :invoke :read [26 nil] INFO [2019-12-17 12:34:47,696] jepsen worker 18 - jepsen.util 9018 :info :write [6 48] :RequestCanceledException INFO [2019-12-17 12:34:47,698] jepsen worker 18 - jepsen.util 9108 :invoke :cas [6 [32 21]] INFO [2019-12-17 12:34:47,699] jepsen worker 73 - jepsen.util 8623 :ok :write [24 18] INFO [2019-12-17 12:34:47,699] jepsen worker 73 - jepsen.util 8623 :invoke :read [24 nil] INFO [2019-12-17 12:34:47,702] jepsen worker 10 - jepsen.util 9010 :ok :write [3 8] INFO [2019-12-17 12:34:47,703] jepsen worker 10 - jepsen.util 9010 :invoke :read [3 nil] INFO [2019-12-17 12:34:47,703] jepsen worker 88 - jepsen.util 8998 :fail :cas [29 [22 24]] :ValueNotSwapFrom INFO [2019-12-17 12:34:47,704] jepsen worker 88 - jepsen.util 8998 :invoke :write [29 5] INFO [2019-12-17 12:34:47,712] jepsen worker 33 - jepsen.util 12003 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:34:47,712] jepsen worker 74 - jepsen.util 8894 :info :write [24 1] :RequestCanceledException INFO [2019-12-17 12:34:47,712] jepsen worker 33 - jepsen.util 12003 :invoke :read [11 nil] INFO [2019-12-17 12:34:47,714] jepsen worker 74 - jepsen.util 8984 :invoke :read [24 nil] INFO [2019-12-17 12:34:47,714] jepsen worker 51 - jepsen.util 8151 :ok :read [17 27] INFO [2019-12-17 12:34:47,714] jepsen worker 51 - jepsen.util 8151 :invoke :cas [17 [10 15]] INFO [2019-12-17 12:34:47,730] jepsen worker 28 - jepsen.util 7048 :info :write [9 21] :RequestCanceledException INFO [2019-12-17 12:34:47,731] jepsen worker 28 - jepsen.util 7138 :invoke :cas [9 [10 29]] INFO [2019-12-17 12:34:47,739] jepsen worker 29 - jepsen.util 7319 :ok :write [9 47] INFO [2019-12-17 12:34:47,740] jepsen worker 29 - jepsen.util 7319 :invoke :read [9 nil] INFO [2019-12-17 12:34:47,747] jepsen worker 63 - jepsen.util 7623 :ok :write [21 25] INFO [2019-12-17 12:34:47,747] jepsen worker 63 - jepsen.util 7623 :invoke :write [21 11] INFO [2019-12-17 12:34:47,756] jepsen worker 3 - jepsen.util 8643 :ok :write [1 44] INFO [2019-12-17 12:34:47,757] jepsen worker 3 - jepsen.util 8643 :invoke :write [1 6] INFO [2019-12-17 12:34:47,758] jepsen worker 56 - jepsen.util 9506 :ok :write [18 21] INFO [2019-12-17 12:34:47,759] jepsen worker 56 - jepsen.util 9506 :invoke :cas [18 [37 4]] INFO [2019-12-17 12:34:47,773] jepsen worker 28 - jepsen.util 7138 :fail :cas [9 [10 29]] :ValueNotSwapFrom INFO [2019-12-17 12:34:47,773] jepsen worker 29 - jepsen.util 7319 :ok :read [9 47] INFO [2019-12-17 12:34:47,774] jepsen worker 29 - jepsen.util 7319 :invoke :cas [9 [30 13]] INFO [2019-12-17 12:34:47,774] jepsen worker 28 - jepsen.util 7138 :invoke :write [9 46] INFO [2019-12-17 12:34:47,793] jepsen worker 29 - jepsen.util 7319 :fail :cas [9 [30 13]] :ValueNotSwapFrom INFO [2019-12-17 12:34:47,793] jepsen worker 29 - jepsen.util 7319 :invoke :read [9 nil] INFO [2019-12-17 12:34:47,806] jepsen worker 29 - jepsen.util 7319 :ok :read [9 47] INFO [2019-12-17 12:34:47,806] jepsen worker 29 - jepsen.util 7319 :invoke :read [9 nil] INFO [2019-12-17 12:34:47,824] jepsen worker 29 - jepsen.util 7319 :ok :read [9 47] INFO [2019-12-17 12:34:47,824] jepsen worker 29 - jepsen.util 7319 :invoke :read [9 nil] INFO [2019-12-17 12:34:47,827] jepsen worker 70 - jepsen.util 9250 :info :write [23 14] :RequestCanceledException INFO [2019-12-17 12:34:47,827] jepsen worker 62 - jepsen.util 12212 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:34:47,827] jepsen worker 62 - jepsen.util 12212 :invoke :write [20 36] INFO [2019-12-17 12:34:47,828] jepsen worker 70 - jepsen.util 9340 :invoke :read [23 nil] INFO [2019-12-17 12:34:47,833] jepsen worker 89 - jepsen.util 8909 :ok :read [29 2] INFO [2019-12-17 12:34:47,833] jepsen worker 89 - jepsen.util 8909 :invoke :cas [29 [49 15]] INFO [2019-12-17 12:34:47,835] jepsen worker 63 - jepsen.util 7623 :ok :write [21 11] INFO [2019-12-17 12:34:47,835] jepsen worker 63 - jepsen.util 7623 :invoke :write [21 23] INFO [2019-12-17 12:34:47,837] jepsen worker 14 - jepsen.util 9014 :ok :read [4 14] INFO [2019-12-17 12:34:47,837] jepsen worker 14 - jepsen.util 9014 :invoke :write [4 18] INFO [2019-12-17 12:34:47,857] jepsen worker 29 - jepsen.util 7319 :ok :read [9 47] INFO [2019-12-17 12:34:47,857] jepsen worker 29 - jepsen.util 7319 :invoke :cas [9 [39 42]] INFO [2019-12-17 12:34:47,861] jepsen worker 25 - jepsen.util 12625 :info :cas [8 [20 7]] :RequestCanceledException INFO [2019-12-17 12:34:47,862] jepsen worker 25 - jepsen.util 12715 :invoke :write [8 19] INFO [2019-12-17 12:34:47,872] jepsen worker 29 - jepsen.util 7319 :fail :cas [9 [39 42]] :ValueNotSwapFrom INFO [2019-12-17 12:34:47,873] jepsen worker 29 - jepsen.util 7319 :invoke :write [9 46] INFO [2019-12-17 12:34:47,897] jepsen worker 0 - jepsen.util 12420 :info :write [0 12] :RequestCanceledException INFO [2019-12-17 12:34:47,898] jepsen worker 0 - jepsen.util 12510 :invoke :cas [0 [2 30]] INFO [2019-12-17 12:34:47,907] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":50898,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:34:47,907] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":50898,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:34:47,907] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":50902,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:34:47,907] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":50902,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:34:47,914] jepsen worker 57 - jepsen.util 11757 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:34:47,914] jepsen worker 57 - jepsen.util 11757 :invoke :write [19 11] INFO [2019-12-17 12:34:47,916] jepsen worker 78 - jepsen.util 8628 :ok :read [26 3] INFO [2019-12-17 12:34:47,916] jepsen worker 78 - jepsen.util 8628 :invoke :read [26 nil] INFO [2019-12-17 12:34:47,917] jepsen worker 28 - jepsen.util 7138 :ok :write [9 46] INFO [2019-12-17 12:34:47,917] jepsen worker 28 - jepsen.util 7138 :invoke :read [9 nil] INFO [2019-12-17 12:34:47,921] jepsen worker 19 - jepsen.util 8839 :ok :write [6 40] INFO [2019-12-17 12:34:47,921] jepsen worker 19 - jepsen.util 8839 :invoke :cas [6 [38 39]] INFO [2019-12-17 12:34:47,926] jepsen worker 30 - jepsen.util 8850 :ok :write [10 22] INFO [2019-12-17 12:34:47,926] jepsen worker 30 - jepsen.util 8850 :invoke :cas [10 [5 5]] INFO [2019-12-17 12:34:47,940] jepsen worker 74 - jepsen.util 8984 :ok :read [24 18] INFO [2019-12-17 12:34:47,941] jepsen worker 74 - jepsen.util 8984 :invoke :cas [24 [12 23]] INFO [2019-12-17 12:34:47,942] jepsen worker 28 - jepsen.util 7138 :ok :read [9 46] INFO [2019-12-17 12:34:47,942] jepsen worker 28 - jepsen.util 7138 :invoke :read [9 nil] INFO [2019-12-17 12:34:47,952] jepsen worker 19 - jepsen.util 8839 :fail :cas [6 [38 39]] :ValueNotSwapFrom INFO [2019-12-17 12:34:47,952] jepsen worker 19 - jepsen.util 8839 :invoke :cas [6 [35 18]] INFO [2019-12-17 12:34:47,964] jepsen worker 30 - jepsen.util 8850 :fail :cas [10 [5 5]] :ValueNotSwapFrom INFO [2019-12-17 12:34:47,964] jepsen worker 30 - jepsen.util 8850 :invoke :write [10 21] INFO [2019-12-17 12:34:47,965] jepsen worker 54 - jepsen.util 9684 :fail :cas [18 [33 40]] :ValueNotSwapFrom INFO [2019-12-17 12:34:47,965] jepsen worker 54 - jepsen.util 9684 :invoke :read [18 nil] INFO [2019-12-17 12:34:47,976] jepsen worker 28 - jepsen.util 7138 :ok :read [9 46] INFO [2019-12-17 12:34:47,976] jepsen worker 28 - jepsen.util 7138 :invoke :read [9 nil] INFO [2019-12-17 12:34:47,977] jepsen worker 74 - jepsen.util 8984 :fail :cas [24 [12 23]] :ValueNotSwapFrom INFO [2019-12-17 12:34:47,978] jepsen worker 74 - jepsen.util 8984 :invoke :cas [24 [1 23]] INFO [2019-12-17 12:34:47,984] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:34:47,985] jepsen nemesis - couchbase.util Rebalance check count: 393 INFO [2019-12-17 12:34:47,985] jepsen nemesis - couchbase.util Rebalance stuck count: 374 INFO [2019-12-17 12:34:47,992] jepsen worker 53 - jepsen.util 8063 :fail :cas [17 [17 7]] :ValueNotSwapFrom INFO [2019-12-17 12:34:47,993] jepsen worker 53 - jepsen.util 8063 :invoke :read [17 nil] INFO [2019-12-17 12:34:48,010] jepsen worker 19 - jepsen.util 8839 :fail :cas [6 [35 18]] :ValueNotSwapFrom INFO [2019-12-17 12:34:48,010] jepsen worker 19 - jepsen.util 8839 :invoke :write [6 7] INFO [2019-12-17 12:34:48,019] jepsen worker 28 - jepsen.util 7138 :ok :read [9 46] INFO [2019-12-17 12:34:48,019] jepsen worker 28 - jepsen.util 7138 :invoke :cas [9 [29 43]] INFO [2019-12-17 12:34:48,021] jepsen worker 12 - jepsen.util 9372 :ok :read [4 14] INFO [2019-12-17 12:34:48,021] jepsen worker 12 - jepsen.util 9372 :invoke :write [4 1] INFO [2019-12-17 12:34:48,031] jepsen worker 70 - jepsen.util 9340 :ok :read [23 44] INFO [2019-12-17 12:34:48,031] jepsen worker 70 - jepsen.util 9340 :invoke :write [23 7] INFO [2019-12-17 12:34:48,044] jepsen worker 74 - jepsen.util 8984 :fail :cas [24 [1 23]] :ValueNotSwapFrom INFO [2019-12-17 12:34:48,044] jepsen worker 74 - jepsen.util 8984 :invoke :read [24 nil] INFO [2019-12-17 12:34:48,057] jepsen worker 28 - jepsen.util 7138 :fail :cas [9 [29 43]] :ValueNotSwapFrom INFO [2019-12-17 12:34:48,057] jepsen worker 28 - jepsen.util 7138 :invoke :read [9 nil] INFO [2019-12-17 12:34:48,066] jepsen worker 28 - jepsen.util 7138 :ok :read [9 46] INFO [2019-12-17 12:34:48,066] jepsen worker 28 - jepsen.util 7138 :invoke :cas [9 [10 39]] INFO [2019-12-17 12:34:48,074] jepsen worker 69 - jepsen.util 8979 :fail :cas [23 [17 33]] :ValueNotSwapFrom INFO [2019-12-17 12:34:48,074] jepsen worker 69 - jepsen.util 8979 :invoke :cas [23 [3 21]] INFO [2019-12-17 12:34:48,081] jepsen worker 28 - jepsen.util 7138 :fail :cas [9 [10 39]] :ValueNotSwapFrom INFO [2019-12-17 12:34:48,082] jepsen worker 28 - jepsen.util 7138 :invoke :write [9 24] INFO [2019-12-17 12:34:48,096] jepsen worker 26 - jepsen.util 12446 :info :cas [8 [48 28]] :RequestCanceledException INFO [2019-12-17 12:34:48,097] jepsen worker 26 - jepsen.util 12536 :invoke :write [8 4] INFO [2019-12-17 12:34:48,100] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":50919,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:34:48,100] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":50919,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:34:48,101] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":50920,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:34:48,101] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":50920,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:34:48,102] jepsen worker 56 - jepsen.util 9506 :fail :cas [18 [37 4]] :ValueNotSwapFrom INFO [2019-12-17 12:34:48,102] jepsen worker 56 - jepsen.util 9506 :invoke :read [18 nil] INFO [2019-12-17 12:34:48,111] jepsen worker 54 - jepsen.util 9684 :ok :read [18 21] INFO [2019-12-17 12:34:48,112] jepsen worker 54 - jepsen.util 9684 :invoke :cas [18 [47 32]] INFO [2019-12-17 12:34:48,112] jepsen worker 58 - jepsen.util 12748 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:34:48,112] jepsen worker 58 - jepsen.util 12748 :invoke :write [19 29] INFO [2019-12-17 12:34:48,128] jepsen worker 53 - jepsen.util 8063 :ok :read [17 27] INFO [2019-12-17 12:34:48,129] jepsen worker 53 - jepsen.util 8063 :invoke :write [17 13] INFO [2019-12-17 12:34:48,130] jepsen worker 85 - jepsen.util 12685 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:34:48,130] jepsen worker 44 - jepsen.util 12284 :info :cas [14 [26 46]] :RequestCanceledException INFO [2019-12-17 12:34:48,130] jepsen worker 85 - jepsen.util 12685 :invoke :read [28 nil] INFO [2019-12-17 12:34:48,131] jepsen worker 44 - jepsen.util 12374 :invoke :read [14 nil] INFO [2019-12-17 12:34:48,135] jepsen worker 28 - jepsen.util 7138 :ok :write [9 24] INFO [2019-12-17 12:34:48,135] jepsen worker 28 - jepsen.util 7138 :invoke :read [9 nil] INFO [2019-12-17 12:34:48,143] jepsen worker 48 - jepsen.util 11028 :info :write [16 1] :RequestCanceledException INFO [2019-12-17 12:34:48,144] jepsen worker 48 - jepsen.util 11118 :invoke :read [16 nil] INFO [2019-12-17 12:34:48,148] jepsen worker 28 - jepsen.util 7138 :ok :read [9 24] INFO [2019-12-17 12:34:48,149] jepsen worker 28 - jepsen.util 7138 :invoke :write [9 45] INFO [2019-12-17 12:34:48,155] jepsen worker 36 - jepsen.util 9216 :ok :read [12 8] INFO [2019-12-17 12:34:48,156] jepsen worker 36 - jepsen.util 9216 :invoke :read [12 nil] INFO [2019-12-17 12:34:48,161] jepsen worker 6 - jepsen.util 12336 :info :write [2 26] :RequestCanceledException INFO [2019-12-17 12:34:48,161] jepsen worker 66 - jepsen.util 11766 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:34:48,161] jepsen worker 66 - jepsen.util 11766 :invoke :cas [22 [12 47]] INFO [2019-12-17 12:34:48,162] jepsen worker 6 - jepsen.util 12426 :invoke :cas [2 [0 43]] INFO [2019-12-17 12:34:48,169] jepsen worker 89 - jepsen.util 8909 :fail :cas [29 [49 15]] :ValueNotSwapFrom INFO [2019-12-17 12:34:48,169] jepsen worker 89 - jepsen.util 8909 :invoke :cas [29 [42 15]] INFO [2019-12-17 12:34:48,172] jepsen worker 69 - jepsen.util 8979 :fail :cas [23 [3 21]] :ValueNotSwapFrom INFO [2019-12-17 12:34:48,172] jepsen worker 69 - jepsen.util 8979 :invoke :cas [23 [32 15]] INFO [2019-12-17 12:34:48,188] jepsen worker 28 - jepsen.util 7138 :ok :write [9 45] INFO [2019-12-17 12:34:48,188] jepsen worker 28 - jepsen.util 7138 :invoke :read [9 nil] INFO [2019-12-17 12:34:48,200] jepsen worker 56 - jepsen.util 9506 :ok :read [18 21] INFO [2019-12-17 12:34:48,200] jepsen worker 56 - jepsen.util 9506 :invoke :read [18 nil] INFO [2019-12-17 12:34:48,203] jepsen worker 28 - jepsen.util 7138 :ok :read [9 45] INFO [2019-12-17 12:34:48,203] jepsen worker 28 - jepsen.util 7138 :invoke :write [9 6] INFO [2019-12-17 12:34:48,229] jepsen worker 22 - jepsen.util 11812 :info :cas [7 [17 49]] :RequestCanceledException INFO [2019-12-17 12:34:48,229] jepsen worker 61 - jepsen.util 12661 :info :write [20 31] :RequestCanceledException INFO [2019-12-17 12:34:48,230] jepsen worker 87 - jepsen.util 8277 :info :write [29 48] :RequestCanceledException INFO [2019-12-17 12:34:48,230] jepsen worker 22 - jepsen.util 11902 :invoke :cas [7 [24 3]] INFO [2019-12-17 12:34:48,230] jepsen worker 61 - jepsen.util 12751 :invoke :cas [20 [49 29]] INFO [2019-12-17 12:34:48,231] jepsen worker 87 - jepsen.util 8367 :invoke :cas [29 [12 33]] INFO [2019-12-17 12:34:48,247] jepsen worker 80 - jepsen.util 8540 :fail :cas [26 [33 3]] :ValueNotSwapFrom INFO [2019-12-17 12:34:48,248] jepsen worker 28 - jepsen.util 7138 :ok :write [9 6] INFO [2019-12-17 12:34:48,267] jepsen worker 54 - jepsen.util 9684 :fail :cas [18 [47 32]] :ValueNotSwapFrom INFO [2019-12-17 12:34:48,269] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":50950,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:34:48,319] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":50959,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:34:48,343] jepsen worker 84 - jepsen.util 11334 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:34:48,361] jepsen worker 24 - jepsen.util 11004 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:34:48,369] jepsen worker 78 - jepsen.util 8628 :ok :read [26 3] INFO [2019-12-17 12:34:48,370] jepsen worker 80 - jepsen.util 8540 :invoke :cas [26 [17 36]] INFO [2019-12-17 12:34:48,370] jepsen worker 28 - jepsen.util 7138 :invoke :read [9 nil] INFO [2019-12-17 12:34:48,370] jepsen worker 54 - jepsen.util 9684 :invoke :write [18 21] INFO [2019-12-17 12:34:48,370] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":50959,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:34:48,370] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":50950,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:34:48,370] jepsen worker 84 - jepsen.util 11334 :invoke :write [28 1] INFO [2019-12-17 12:34:48,370] jepsen worker 24 - jepsen.util 11004 :invoke :read [8 nil] INFO [2019-12-17 12:34:48,370] jepsen worker 78 - jepsen.util 8628 :invoke :read [26 nil] INFO [2019-12-17 12:34:48,371] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":50962,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:34:48,371] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":50952,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:34:48,371] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":50962,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:34:48,371] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":50952,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:34:48,371] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":50963,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:34:48,371] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":50963,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:34:48,387] jepsen worker 28 - jepsen.util 7138 :ok :read [9 6] INFO [2019-12-17 12:34:48,387] jepsen worker 28 - jepsen.util 7138 :invoke :write [9 23] INFO [2019-12-17 12:34:48,387] jepsen worker 51 - jepsen.util 8151 :fail :cas [17 [10 15]] :ValueNotSwapFrom INFO [2019-12-17 12:34:48,388] jepsen worker 51 - jepsen.util 8151 :invoke :read [17 nil] INFO [2019-12-17 12:34:48,414] jepsen worker 17 - jepsen.util 11987 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:34:48,414] jepsen worker 17 - jepsen.util 11987 :invoke :cas [5 [48 43]] INFO [2019-12-17 12:34:48,421] jepsen worker 28 - jepsen.util 7138 :ok :write [9 23] INFO [2019-12-17 12:34:48,421] jepsen worker 28 - jepsen.util 7138 :invoke :cas [9 [16 9]] INFO [2019-12-17 12:34:48,429] jepsen worker 40 - jepsen.util 12010 :info :write [13 29] :RequestCanceledException INFO [2019-12-17 12:34:48,431] jepsen worker 40 - jepsen.util 12100 :invoke :cas [13 [7 23]] INFO [2019-12-17 12:34:48,431] jepsen worker 36 - jepsen.util 9216 :ok :read [12 8] INFO [2019-12-17 12:34:48,431] jepsen worker 36 - jepsen.util 9216 :invoke :read [12 nil] INFO [2019-12-17 12:34:48,433] jepsen worker 89 - jepsen.util 8909 :fail :cas [29 [42 15]] :ValueNotSwapFrom INFO [2019-12-17 12:34:48,433] jepsen worker 89 - jepsen.util 8909 :invoke :read [29 nil] INFO [2019-12-17 12:34:48,434] jepsen worker 28 - jepsen.util 7138 :fail :cas [9 [16 9]] :ValueNotSwapFrom INFO [2019-12-17 12:34:48,434] jepsen worker 28 - jepsen.util 7138 :invoke :read [9 nil] INFO [2019-12-17 12:34:48,450] jepsen worker 28 - jepsen.util 7138 :ok :read [9 23] INFO [2019-12-17 12:34:48,450] jepsen worker 28 - jepsen.util 7138 :invoke :cas [9 [7 34]] INFO [2019-12-17 12:34:48,460] jepsen worker 28 - jepsen.util 7138 :fail :cas [9 [7 34]] :ValueNotSwapFrom INFO [2019-12-17 12:34:48,460] jepsen worker 28 - jepsen.util 7138 :invoke :cas [9 [47 39]] INFO [2019-12-17 12:34:48,461] jepsen worker 42 - jepsen.util 13002 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:34:48,462] jepsen worker 42 - jepsen.util 13002 :invoke :cas [14 [49 17]] INFO [2019-12-17 12:34:48,471] jepsen worker 28 - jepsen.util 7138 :fail :cas [9 [47 39]] :ValueNotSwapFrom INFO [2019-12-17 12:34:48,471] jepsen worker 28 - jepsen.util 7138 :invoke :write [9 3] INFO [2019-12-17 12:34:48,486] jepsen worker 87 - jepsen.util 8367 :fail :cas [29 [12 33]] :ValueNotSwapFrom INFO [2019-12-17 12:34:48,486] jepsen worker 87 - jepsen.util 8367 :invoke :cas [29 [47 25]] INFO [2019-12-17 12:34:48,505] jepsen worker 28 - jepsen.util 7138 :ok :write [9 3] INFO [2019-12-17 12:34:48,505] jepsen worker 28 - jepsen.util 7138 :invoke :read [9 nil] INFO [2019-12-17 12:34:48,518] jepsen worker 28 - jepsen.util 7138 :ok :read [9 3] INFO [2019-12-17 12:34:48,518] jepsen worker 28 - jepsen.util 7138 :invoke :read [9 nil] INFO [2019-12-17 12:34:48,527] jepsen worker 28 - jepsen.util 7138 :ok :read [9 3] INFO [2019-12-17 12:34:48,527] jepsen worker 28 - jepsen.util 7138 :invoke :write [9 2] INFO [2019-12-17 12:34:48,530] jepsen worker 77 - jepsen.util 12317 :info :write [25 24] :RequestCanceledException INFO [2019-12-17 12:34:48,530] jepsen worker 1 - jepsen.util 12151 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:34:48,531] jepsen worker 1 - jepsen.util 12151 :invoke :write [0 12] INFO [2019-12-17 12:34:48,531] jepsen worker 77 - jepsen.util 12407 :invoke :write [25 28] INFO [2019-12-17 12:34:48,535] jepsen worker 13 - jepsen.util 8563 :fail :cas [4 [17 15]] :ValueNotSwapFrom INFO [2019-12-17 12:34:48,535] jepsen worker 13 - jepsen.util 8563 :invoke :write [4 13] INFO [2019-12-17 12:34:48,539] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":50978,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:34:48,539] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":50978,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:34:48,539] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":50982,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:34:48,539] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":50982,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:34:48,543] jepsen worker 43 - jepsen.util 11653 :info :write [14 30] :RequestCanceledException INFO [2019-12-17 12:34:48,544] jepsen worker 43 - jepsen.util 11743 :invoke :write [14 43] INFO [2019-12-17 12:34:48,565] jepsen worker 28 - jepsen.util 7138 :ok :write [9 2] INFO [2019-12-17 12:34:48,566] jepsen worker 28 - jepsen.util 7138 :invoke :cas [9 [9 30]] INFO [2019-12-17 12:34:48,576] jepsen worker 28 - jepsen.util 7138 :fail :cas [9 [9 30]] :ValueNotSwapFrom INFO [2019-12-17 12:34:48,576] jepsen worker 28 - jepsen.util 7138 :invoke :read [9 nil] INFO [2019-12-17 12:34:48,584] jepsen worker 19 - jepsen.util 8839 :ok :write [6 7] INFO [2019-12-17 12:34:48,584] jepsen worker 19 - jepsen.util 8839 :invoke :cas [6 [42 48]] INFO [2019-12-17 12:34:48,590] jepsen worker 28 - jepsen.util 7138 :ok :read [9 2] INFO [2019-12-17 12:34:48,590] jepsen worker 28 - jepsen.util 7138 :invoke :read [9 nil] INFO [2019-12-17 12:34:48,597] jepsen worker 15 - jepsen.util 10995 :info :write [5 12] :RequestCanceledException INFO [2019-12-17 12:34:48,598] jepsen worker 15 - jepsen.util 11085 :invoke :read [5 nil] INFO [2019-12-17 12:34:48,602] jepsen worker 30 - jepsen.util 8850 :ok :write [10 21] INFO [2019-12-17 12:34:48,602] jepsen worker 45 - jepsen.util 9225 :ok :write [15 2] INFO [2019-12-17 12:34:48,602] jepsen worker 30 - jepsen.util 8850 :invoke :cas [10 [25 41]] INFO [2019-12-17 12:34:48,602] jepsen worker 45 - jepsen.util 9225 :invoke :read [15 nil] INFO [2019-12-17 12:34:48,614] jepsen worker 8 - jepsen.util 11708 :info :cas [2 [37 45]] :RequestCanceledException INFO [2019-12-17 12:34:48,617] jepsen worker 8 - jepsen.util 11798 :invoke :read [2 nil] INFO [2019-12-17 12:34:48,621] jepsen worker 19 - jepsen.util 8839 :fail :cas [6 [42 48]] :ValueNotSwapFrom INFO [2019-12-17 12:34:48,622] jepsen worker 19 - jepsen.util 8839 :invoke :write [6 10] INFO [2019-12-17 12:34:48,630] jepsen worker 7 - jepsen.util 12247 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:34:48,630] jepsen worker 75 - jepsen.util 11235 :info :write [25 47] :RequestCanceledException INFO [2019-12-17 12:34:48,630] jepsen worker 7 - jepsen.util 12247 :invoke :read [2 nil] INFO [2019-12-17 12:34:48,631] jepsen worker 75 - jepsen.util 11325 :invoke :write [25 21] INFO [2019-12-17 12:34:48,659] jepsen worker 45 - jepsen.util 9225 :ok :read [15 2] INFO [2019-12-17 12:34:48,659] jepsen worker 45 - jepsen.util 9225 :invoke :cas [15 [35 39]] INFO [2019-12-17 12:34:48,661] jepsen worker 28 - jepsen.util 7138 :ok :read [9 2] INFO [2019-12-17 12:34:48,662] jepsen worker 28 - jepsen.util 7138 :invoke :cas [9 [26 12]] INFO [2019-12-17 12:34:48,664] jepsen worker 30 - jepsen.util 8850 :fail :cas [10 [25 41]] :ValueNotSwapFrom INFO [2019-12-17 12:34:48,664] jepsen worker 30 - jepsen.util 8850 :invoke :write [10 35] INFO [2019-12-17 12:34:48,695] jepsen worker 45 - jepsen.util 9225 :fail :cas [15 [35 39]] :ValueNotSwapFrom INFO [2019-12-17 12:34:48,695] jepsen worker 45 - jepsen.util 9225 :invoke :write [15 23] INFO [2019-12-17 12:34:48,716] jepsen worker 28 - jepsen.util 7138 :fail :cas [9 [26 12]] :ValueNotSwapFrom INFO [2019-12-17 12:34:48,716] jepsen worker 28 - jepsen.util 7138 :invoke :write [9 17] INFO [2019-12-17 12:34:48,736] jepsen worker 51 - jepsen.util 8151 :ok :read [17 27] INFO [2019-12-17 12:34:48,737] jepsen worker 51 - jepsen.util 8151 :invoke :cas [17 [34 24]] INFO [2019-12-17 12:34:48,745] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51011,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:34:48,745] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51011,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:34:48,745] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51014,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:34:48,745] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51014,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:34:48,745] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51015,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:34:48,746] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51015,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:34:48,776] jepsen worker 28 - jepsen.util 7138 :ok :write [9 17] INFO [2019-12-17 12:34:48,776] jepsen worker 28 - jepsen.util 7138 :invoke :cas [9 [11 49]] INFO [2019-12-17 12:34:48,795] jepsen worker 28 - jepsen.util 7138 :fail :cas [9 [11 49]] :ValueNotSwapFrom INFO [2019-12-17 12:34:48,795] jepsen worker 28 - jepsen.util 7138 :invoke :cas [9 [18 38]] INFO [2019-12-17 12:34:48,806] jepsen worker 28 - jepsen.util 7138 :fail :cas [9 [18 38]] :ValueNotSwapFrom INFO [2019-12-17 12:34:48,807] jepsen worker 28 - jepsen.util 7138 :invoke :cas [9 [44 20]] INFO [2019-12-17 12:34:48,814] jepsen worker 69 - jepsen.util 8979 :fail :cas [23 [32 15]] :ValueNotSwapFrom INFO [2019-12-17 12:34:48,814] jepsen worker 69 - jepsen.util 8979 :invoke :read [23 nil] INFO [2019-12-17 12:34:48,816] jepsen worker 80 - jepsen.util 8540 :fail :cas [26 [17 36]] :ValueNotSwapFrom INFO [2019-12-17 12:34:48,816] jepsen worker 80 - jepsen.util 8540 :invoke :cas [26 [13 22]] INFO [2019-12-17 12:34:48,820] jepsen worker 28 - jepsen.util 7138 :fail :cas [9 [44 20]] :ValueNotSwapFrom INFO [2019-12-17 12:34:48,820] jepsen worker 28 - jepsen.util 7138 :invoke :read [9 nil] INFO [2019-12-17 12:34:48,832] jepsen worker 28 - jepsen.util 7138 :ok :read [9 17] INFO [2019-12-17 12:34:48,832] jepsen worker 28 - jepsen.util 7138 :invoke :cas [9 [10 10]] INFO [2019-12-17 12:34:48,838] jepsen worker 78 - jepsen.util 8628 :ok :read [26 3] INFO [2019-12-17 12:34:48,838] jepsen worker 78 - jepsen.util 8628 :invoke :read [26 nil] INFO [2019-12-17 12:34:48,847] jepsen worker 19 - jepsen.util 8839 :ok :write [6 10] INFO [2019-12-17 12:34:48,848] jepsen worker 19 - jepsen.util 8839 :invoke :read [6 nil] INFO [2019-12-17 12:34:48,855] jepsen worker 30 - jepsen.util 8850 :ok :write [10 35] INFO [2019-12-17 12:34:48,856] jepsen worker 30 - jepsen.util 8850 :invoke :cas [10 [47 3]] INFO [2019-12-17 12:34:48,857] jepsen worker 28 - jepsen.util 7138 :fail :cas [9 [10 10]] :ValueNotSwapFrom INFO [2019-12-17 12:34:48,857] jepsen worker 28 - jepsen.util 7138 :invoke :write [9 6] INFO [2019-12-17 12:34:48,892] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51032,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:34:48,892] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51032,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:34:48,892] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51037,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:34:48,892] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51037,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:34:48,896] jepsen worker 49 - jepsen.util 11299 :info :cas [16 [23 41]] :RequestCanceledException INFO [2019-12-17 12:34:48,898] jepsen worker 49 - jepsen.util 11389 :invoke :read [16 nil] INFO [2019-12-17 12:34:48,911] jepsen worker 19 - jepsen.util 8839 :ok :read [6 10] INFO [2019-12-17 12:34:48,912] jepsen worker 19 - jepsen.util 8839 :invoke :write [6 44] INFO [2019-12-17 12:34:48,919] jepsen worker 30 - jepsen.util 8850 :fail :cas [10 [47 3]] :ValueNotSwapFrom INFO [2019-12-17 12:34:48,919] jepsen worker 45 - jepsen.util 9225 :ok :write [15 23] INFO [2019-12-17 12:34:48,919] jepsen worker 30 - jepsen.util 8850 :invoke :read [10 nil] INFO [2019-12-17 12:34:48,919] jepsen worker 45 - jepsen.util 9225 :invoke :write [15 27] INFO [2019-12-17 12:34:48,930] jepsen worker 2 - jepsen.util 11702 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:34:48,930] jepsen worker 2 - jepsen.util 11702 :invoke :read [0 nil] INFO [2019-12-17 12:34:48,952] jepsen worker 89 - jepsen.util 8909 :ok :read [29 2] INFO [2019-12-17 12:34:48,953] jepsen worker 89 - jepsen.util 8909 :invoke :cas [29 [32 13]] INFO [2019-12-17 12:34:48,958] jepsen worker 14 - jepsen.util 9014 :ok :write [4 18] INFO [2019-12-17 12:34:48,958] jepsen worker 14 - jepsen.util 9014 :invoke :read [4 nil] INFO [2019-12-17 12:34:48,962] jepsen worker 56 - jepsen.util 9506 :ok :read [18 21] INFO [2019-12-17 12:34:48,962] jepsen worker 56 - jepsen.util 9506 :invoke :read [18 nil] INFO [2019-12-17 12:34:48,974] jepsen worker 30 - jepsen.util 8850 :ok :read [10 35] INFO [2019-12-17 12:34:48,974] jepsen worker 30 - jepsen.util 8850 :invoke :write [10 31] INFO [2019-12-17 12:34:48,995] jepsen worker 28 - jepsen.util 7138 :ok :write [9 6] INFO [2019-12-17 12:34:48,995] jepsen worker 28 - jepsen.util 7138 :invoke :read [9 nil] INFO [2019-12-17 12:34:48,998] jepsen worker 71 - jepsen.util 9341 :ok :write [23 40] INFO [2019-12-17 12:34:48,998] jepsen worker 71 - jepsen.util 9341 :invoke :cas [23 [15 43]] INFO [2019-12-17 12:34:49,000] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:34:49,000] jepsen nemesis - couchbase.util Rebalance check count: 394 INFO [2019-12-17 12:34:49,000] jepsen nemesis - couchbase.util Rebalance stuck count: 375 INFO [2019-12-17 12:34:49,021] jepsen worker 28 - jepsen.util 7138 :ok :read [9 6] INFO [2019-12-17 12:34:49,022] jepsen worker 28 - jepsen.util 7138 :invoke :cas [9 [48 35]] INFO [2019-12-17 12:34:49,026] jepsen worker 36 - jepsen.util 9216 :ok :read [12 8] INFO [2019-12-17 12:34:49,026] jepsen worker 36 - jepsen.util 9216 :invoke :write [12 48] INFO [2019-12-17 12:34:49,030] jepsen worker 86 - jepsen.util 12776 :info :write [28 44] :RequestCanceledException INFO [2019-12-17 12:34:49,031] jepsen worker 86 - jepsen.util 12866 :invoke :read [28 nil] INFO [2019-12-17 12:34:49,035] jepsen worker 28 - jepsen.util 7138 :fail :cas [9 [48 35]] :ValueNotSwapFrom INFO [2019-12-17 12:34:49,035] jepsen worker 28 - jepsen.util 7138 :invoke :write [9 47] INFO [2019-12-17 12:34:49,043] jepsen worker 81 - jepsen.util 11781 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:34:49,043] jepsen worker 9 - jepsen.util 9189 :info :write [3 43] :RequestCanceledException INFO [2019-12-17 12:34:49,044] jepsen worker 27 - jepsen.util 8577 :info :write [9 2] :RequestCanceledException INFO [2019-12-17 12:34:49,044] jepsen worker 81 - jepsen.util 11781 :invoke :cas [27 [22 35]] INFO [2019-12-17 12:34:49,045] jepsen worker 27 - jepsen.util 8667 :invoke :read [9 nil] INFO [2019-12-17 12:34:49,045] jepsen worker 9 - jepsen.util 9279 :invoke :write [3 6] INFO [2019-12-17 12:34:49,061] jepsen worker 87 - jepsen.util 8367 :fail :cas [29 [47 25]] :ValueNotSwapFrom INFO [2019-12-17 12:34:49,061] jepsen worker 87 - jepsen.util 8367 :invoke :read [29 nil] INFO [2019-12-17 12:34:49,061] jepsen worker 41 - jepsen.util 12731 :info :cas [13 [28 31]] :RequestCanceledException INFO [2019-12-17 12:34:49,062] jepsen worker 41 - jepsen.util 12821 :invoke :read [13 nil] INFO [2019-12-17 12:34:49,073] jepsen worker 27 - jepsen.util 8667 :ok :read [9 6] INFO [2019-12-17 12:34:49,073] jepsen worker 27 - jepsen.util 8667 :invoke :read [9 nil] INFO [2019-12-17 12:34:49,087] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51049,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:34:49,087] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51049,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:34:49,087] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51051,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:34:49,087] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51051,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:34:49,088] jepsen worker 27 - jepsen.util 8667 :ok :read [9 6] INFO [2019-12-17 12:34:49,088] jepsen worker 27 - jepsen.util 8667 :invoke :write [9 26] INFO [2019-12-17 12:34:49,096] jepsen worker 39 - jepsen.util 11379 :info :write [13 0] :RequestCanceledException INFO [2019-12-17 12:34:49,097] jepsen worker 39 - jepsen.util 11469 :invoke :cas [13 [46 19]] INFO [2019-12-17 12:34:49,099] jepsen worker 28 - jepsen.util 7138 :ok :write [9 47] INFO [2019-12-17 12:34:49,100] jepsen worker 28 - jepsen.util 7138 :invoke :cas [9 [47 16]] INFO [2019-12-17 12:34:49,108] jepsen worker 80 - jepsen.util 8540 :fail :cas [26 [13 22]] :ValueNotSwapFrom INFO [2019-12-17 12:34:49,109] jepsen worker 80 - jepsen.util 8540 :invoke :cas [26 [28 41]] INFO [2019-12-17 12:34:49,110] jepsen worker 14 - jepsen.util 9014 :ok :read [4 18] INFO [2019-12-17 12:34:49,110] jepsen worker 14 - jepsen.util 9014 :invoke :write [4 40] INFO [2019-12-17 12:34:49,114] jepsen worker 76 - jepsen.util 12226 :info :cas [25 [2 41]] :RequestCanceledException INFO [2019-12-17 12:34:49,116] jepsen worker 76 - jepsen.util 12316 :invoke :write [25 26] INFO [2019-12-17 12:34:49,129] jepsen worker 21 - jepsen.util 11631 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:34:49,129] jepsen worker 82 - jepsen.util 12592 :info :cas [27 [4 31]] :RequestCanceledException INFO [2019-12-17 12:34:49,130] jepsen worker 21 - jepsen.util 11631 :invoke :cas [7 [18 17]] INFO [2019-12-17 12:34:49,131] jepsen worker 82 - jepsen.util 12682 :invoke :cas [27 [41 40]] INFO [2019-12-17 12:34:49,131] jepsen worker 87 - jepsen.util 8367 :ok :read [29 2] INFO [2019-12-17 12:34:49,131] jepsen worker 87 - jepsen.util 8367 :invoke :cas [29 [46 4]] INFO [2019-12-17 12:34:49,143] jepsen worker 79 - jepsen.util 9169 :info :write [26 39] :RequestCanceledException INFO [2019-12-17 12:34:49,143] jepsen worker 34 - jepsen.util 12094 :info :cas [11 [10 17]] :RequestCanceledException INFO [2019-12-17 12:34:49,144] jepsen worker 79 - jepsen.util 9259 :invoke :read [26 nil] INFO [2019-12-17 12:34:49,144] jepsen worker 34 - jepsen.util 12184 :invoke :write [11 3] INFO [2019-12-17 12:34:49,155] jepsen worker 51 - jepsen.util 8151 :fail :cas [17 [34 24]] :ValueNotSwapFrom INFO [2019-12-17 12:34:49,155] jepsen worker 51 - jepsen.util 8151 :invoke :read [17 nil] INFO [2019-12-17 12:34:49,177] jepsen worker 27 - jepsen.util 8667 :ok :write [9 26] INFO [2019-12-17 12:34:49,177] jepsen worker 27 - jepsen.util 8667 :invoke :cas [9 [36 36]] INFO [2019-12-17 12:34:49,193] jepsen worker 27 - jepsen.util 8667 :fail :cas [9 [36 36]] :ValueNotSwapFrom INFO [2019-12-17 12:34:49,193] jepsen worker 27 - jepsen.util 8667 :invoke :read [9 nil] INFO [2019-12-17 12:34:49,198] jepsen worker 71 - jepsen.util 9341 :fail :cas [23 [15 43]] :ValueNotSwapFrom INFO [2019-12-17 12:34:49,198] jepsen worker 71 - jepsen.util 9341 :invoke :read [23 nil] INFO [2019-12-17 12:34:49,208] jepsen worker 27 - jepsen.util 8667 :ok :read [9 26] INFO [2019-12-17 12:34:49,208] jepsen worker 27 - jepsen.util 8667 :invoke :read [9 nil] INFO [2019-12-17 12:34:49,224] jepsen worker 27 - jepsen.util 8667 :ok :read [9 26] INFO [2019-12-17 12:34:49,224] jepsen worker 27 - jepsen.util 8667 :invoke :cas [9 [11 5]] INFO [2019-12-17 12:34:49,230] jepsen worker 59 - jepsen.util 11309 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:34:49,230] jepsen worker 59 - jepsen.util 11309 :invoke :write [19 25] INFO [2019-12-17 12:34:49,239] jepsen worker 27 - jepsen.util 8667 :fail :cas [9 [11 5]] :ValueNotSwapFrom INFO [2019-12-17 12:34:49,239] jepsen worker 27 - jepsen.util 8667 :invoke :write [9 48] INFO [2019-12-17 12:34:49,243] jepsen worker 23 - jepsen.util 11633 :info :write [7 32] :RequestCanceledException INFO [2019-12-17 12:34:49,244] jepsen worker 23 - jepsen.util 11723 :invoke :cas [7 [18 14]] INFO [2019-12-17 12:34:49,246] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51081,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:34:49,246] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51090,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:34:49,246] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51081,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:34:49,246] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51090,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:34:49,246] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51082,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:34:49,246] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51093,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:34:49,246] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51082,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:34:49,246] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51093,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:34:49,246] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51083,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:34:49,246] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51083,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:34:49,254] jepsen worker 13 - jepsen.util 8563 :ok :write [4 13] INFO [2019-12-17 12:34:49,254] jepsen worker 13 - jepsen.util 8563 :invoke :read [4 nil] INFO [2019-12-17 12:34:49,260] jepsen worker 31 - jepsen.util 7861 :fail :read [10 nil] :RequestTimeoutException INFO [2019-12-17 12:34:49,260] jepsen worker 31 - jepsen.util 7861 :invoke :read [10 nil] INFO [2019-12-17 12:34:49,271] jepsen worker 19 - jepsen.util 8839 :ok :write [6 44] INFO [2019-12-17 12:34:49,271] jepsen worker 45 - jepsen.util 9225 :ok :write [15 27] INFO [2019-12-17 12:34:49,271] jepsen worker 19 - jepsen.util 8839 :invoke :write [6 41] INFO [2019-12-17 12:34:49,271] jepsen worker 45 - jepsen.util 9225 :invoke :write [15 37] INFO [2019-12-17 12:34:49,285] jepsen worker 30 - jepsen.util 8850 :ok :write [10 31] INFO [2019-12-17 12:34:49,285] jepsen worker 30 - jepsen.util 8850 :invoke :write [10 35] INFO [2019-12-17 12:34:49,296] jepsen worker 68 - jepsen.util 11318 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:34:49,296] jepsen worker 68 - jepsen.util 11318 :invoke :write [22 48] INFO [2019-12-17 12:34:49,298] jepsen worker 27 - jepsen.util 8667 :ok :write [9 48] INFO [2019-12-17 12:34:49,298] jepsen worker 27 - jepsen.util 8667 :invoke :read [9 nil] INFO [2019-12-17 12:34:49,313] jepsen worker 31 - jepsen.util 7861 :ok :read [10 31] INFO [2019-12-17 12:34:49,313] jepsen worker 31 - jepsen.util 7861 :invoke :cas [10 [11 11]] INFO [2019-12-17 12:34:49,315] jepsen worker 83 - jepsen.util 12413 :info :cas [27 [27 18]] :RequestCanceledException INFO [2019-12-17 12:34:49,316] jepsen worker 83 - jepsen.util 12503 :invoke :cas [27 [38 49]] INFO [2019-12-17 12:34:49,329] jepsen worker 55 - jepsen.util 9595 :info :write [18 6] :RequestCanceledException INFO [2019-12-17 12:34:49,329] jepsen worker 50 - jepsen.util 12560 :info :cas [16 [27 33]] :RequestCanceledException INFO [2019-12-17 12:34:49,330] jepsen worker 50 - jepsen.util 12650 :invoke :read [16 nil] INFO [2019-12-17 12:34:49,330] jepsen worker 55 - jepsen.util 9685 :invoke :cas [18 [24 37]] INFO [2019-12-17 12:34:49,340] jepsen worker 27 - jepsen.util 8667 :ok :read [9 48] INFO [2019-12-17 12:34:49,340] jepsen worker 27 - jepsen.util 8667 :invoke :write [9 5] INFO [2019-12-17 12:34:49,349] jepsen worker 31 - jepsen.util 7861 :fail :cas [10 [11 11]] :ValueNotSwapFrom INFO [2019-12-17 12:34:49,349] jepsen worker 31 - jepsen.util 7861 :invoke :cas [10 [41 5]] INFO [2019-12-17 12:34:49,361] jepsen worker 16 - jepsen.util 12976 :info :write [5 23] :RequestCanceledException INFO [2019-12-17 12:34:49,362] jepsen worker 16 - jepsen.util 13066 :invoke :write [5 27] INFO [2019-12-17 12:34:49,368] jepsen worker 13 - jepsen.util 8563 :ok :read [4 13] INFO [2019-12-17 12:34:49,368] jepsen worker 13 - jepsen.util 8563 :invoke :read [4 nil] INFO [2019-12-17 12:34:49,394] jepsen worker 13 - jepsen.util 8563 :ok :read [4 13] INFO [2019-12-17 12:34:49,395] jepsen worker 13 - jepsen.util 8563 :invoke :write [4 28] INFO [2019-12-17 12:34:49,396] jepsen worker 47 - jepsen.util 9137 :info :write [15 28] :RequestCanceledException INFO [2019-12-17 12:34:49,396] jepsen worker 11 - jepsen.util 8741 :info :write [3 40] :RequestCanceledException INFO [2019-12-17 12:34:49,398] jepsen worker 47 - jepsen.util 9227 :invoke :cas [15 [7 34]] INFO [2019-12-17 12:34:49,398] jepsen worker 11 - jepsen.util 8831 :invoke :write [3 31] INFO [2019-12-17 12:34:49,401] jepsen worker 27 - jepsen.util 8667 :ok :write [9 5] INFO [2019-12-17 12:34:49,402] jepsen worker 27 - jepsen.util 8667 :invoke :read [9 nil] INFO [2019-12-17 12:34:49,404] jepsen worker 69 - jepsen.util 8979 :ok :read [23 40] INFO [2019-12-17 12:34:49,404] jepsen worker 69 - jepsen.util 8979 :invoke :cas [23 [34 13]] INFO [2019-12-17 12:34:49,414] jepsen worker 46 - jepsen.util 8776 :info :cas [15 [28 24]] :RequestCanceledException INFO [2019-12-17 12:34:49,415] jepsen worker 46 - jepsen.util 8866 :invoke :cas [15 [8 28]] INFO [2019-12-17 12:34:49,430] jepsen worker 67 - jepsen.util 12127 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:34:49,430] jepsen worker 67 - jepsen.util 12127 :invoke :read [22 nil] INFO [2019-12-17 12:34:49,442] jepsen worker 87 - jepsen.util 8367 :fail :cas [29 [46 4]] :ValueNotSwapFrom INFO [2019-12-17 12:34:49,442] jepsen worker 87 - jepsen.util 8367 :invoke :read [29 nil] INFO [2019-12-17 12:34:49,443] jepsen worker 56 - jepsen.util 9506 :ok :read [18 21] INFO [2019-12-17 12:34:49,443] jepsen worker 56 - jepsen.util 9506 :invoke :write [18 41] INFO [2019-12-17 12:34:49,444] jepsen worker 5 - jepsen.util 8195 :info :write [1 38] :RequestCanceledException INFO [2019-12-17 12:34:49,444] jepsen worker 60 - jepsen.util 12930 :info :write [20 37] :RequestCanceledException INFO [2019-12-17 12:34:49,445] jepsen worker 60 - jepsen.util 13020 :invoke :write [20 20] INFO [2019-12-17 12:34:49,445] jepsen worker 5 - jepsen.util 8285 :invoke :write [1 22] INFO [2019-12-17 12:34:49,449] jepsen worker 51 - jepsen.util 8151 :ok :read [17 27] INFO [2019-12-17 12:34:49,449] jepsen worker 51 - jepsen.util 8151 :invoke :read [17 nil] INFO [2019-12-17 12:34:49,461] jepsen worker 32 - jepsen.util 8942 :info :write [10 39] :RequestCanceledException INFO [2019-12-17 12:34:49,462] jepsen worker 32 - jepsen.util 9032 :invoke :cas [10 [46 39]] INFO [2019-12-17 12:34:49,463] jepsen worker 71 - jepsen.util 9341 :ok :read [23 40] INFO [2019-12-17 12:34:49,463] jepsen worker 71 - jepsen.util 9341 :invoke :read [23 nil] INFO [2019-12-17 12:34:49,470] jepsen worker 27 - jepsen.util 8667 :ok :read [9 5] INFO [2019-12-17 12:34:49,471] jepsen worker 27 - jepsen.util 8667 :invoke :write [9 6] INFO [2019-12-17 12:34:49,500] jepsen worker 32 - jepsen.util 9032 :fail :cas [10 [46 39]] :ValueNotSwapFrom INFO [2019-12-17 12:34:49,500] jepsen worker 32 - jepsen.util 9032 :invoke :cas [10 [41 35]] INFO [2019-12-17 12:34:49,505] jepsen worker 9 - jepsen.util 9279 :ok :write [3 6] INFO [2019-12-17 12:34:49,505] jepsen worker 9 - jepsen.util 9279 :invoke :write [3 17] INFO [2019-12-17 12:34:49,514] jepsen worker 19 - jepsen.util 8839 :ok :write [6 41] INFO [2019-12-17 12:34:49,514] jepsen worker 19 - jepsen.util 8839 :invoke :read [6 nil] INFO [2019-12-17 12:34:49,524] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51124,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:34:49,524] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51124,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:34:49,524] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51126,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:34:49,524] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51126,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:34:49,525] jepsen worker 45 - jepsen.util 9225 :ok :write [15 37] INFO [2019-12-17 12:34:49,525] jepsen worker 45 - jepsen.util 9225 :invoke :cas [15 [13 38]] INFO [2019-12-17 12:34:49,540] jepsen worker 87 - jepsen.util 8367 :ok :read [29 2] INFO [2019-12-17 12:34:49,540] jepsen worker 87 - jepsen.util 8367 :invoke :cas [29 [0 46]] INFO [2019-12-17 12:34:49,552] jepsen worker 46 - jepsen.util 8866 :fail :cas [15 [8 28]] :ValueNotSwapFrom INFO [2019-12-17 12:34:49,552] jepsen worker 46 - jepsen.util 8866 :invoke :cas [15 [17 34]] INFO [2019-12-17 12:34:49,553] jepsen worker 19 - jepsen.util 8839 :ok :read [6 41] INFO [2019-12-17 12:34:49,553] jepsen worker 19 - jepsen.util 8839 :invoke :cas [6 [12 7]] INFO [2019-12-17 12:34:49,561] jepsen worker 35 - jepsen.util 11825 :info :write [11 47] :RequestCanceledException INFO [2019-12-17 12:34:49,562] jepsen worker 35 - jepsen.util 11915 :invoke :cas [11 [21 29]] INFO [2019-12-17 12:34:49,588] jepsen worker 51 - jepsen.util 8151 :ok :read [17 27] INFO [2019-12-17 12:34:49,589] jepsen worker 51 - jepsen.util 8151 :invoke :read [17 nil] INFO [2019-12-17 12:34:49,597] jepsen worker 20 - jepsen.util 8480 :fail :read [6 nil] :RequestTimeoutException INFO [2019-12-17 12:34:49,598] jepsen worker 20 - jepsen.util 8480 :invoke :read [6 nil] INFO [2019-12-17 12:34:49,598] jepsen worker 45 - jepsen.util 9225 :fail :cas [15 [13 38]] :ValueNotSwapFrom INFO [2019-12-17 12:34:49,598] jepsen worker 71 - jepsen.util 9341 :ok :read [23 40] INFO [2019-12-17 12:34:49,598] jepsen worker 45 - jepsen.util 9225 :invoke :read [15 nil] INFO [2019-12-17 12:34:49,598] jepsen worker 71 - jepsen.util 9341 :invoke :write [23 26] INFO [2019-12-17 12:34:49,613] jepsen worker 51 - jepsen.util 8151 :ok :read [17 27] INFO [2019-12-17 12:34:49,613] jepsen worker 51 - jepsen.util 8151 :invoke :write [17 47] INFO [2019-12-17 12:34:49,621] jepsen worker 19 - jepsen.util 8839 :fail :cas [6 [12 7]] :ValueNotSwapFrom INFO [2019-12-17 12:34:49,621] jepsen worker 19 - jepsen.util 8839 :invoke :write [6 28] INFO [2019-12-17 12:34:49,630] jepsen worker 72 - jepsen.util 8802 :info :write [24 19] :RequestCanceledException INFO [2019-12-17 12:34:49,630] jepsen worker 65 - jepsen.util 7265 :info :write [21 13] :RequestCanceledException INFO [2019-12-17 12:34:49,632] jepsen worker 72 - jepsen.util 8892 :invoke :cas [24 [49 34]] INFO [2019-12-17 12:34:49,632] jepsen worker 65 - jepsen.util 7355 :invoke :cas [21 [28 9]] INFO [2019-12-17 12:34:49,644] jepsen worker 52 - jepsen.util 9142 :info :write [17 42] :RequestCanceledException INFO [2019-12-17 12:34:49,644] jepsen worker 38 - jepsen.util 9128 :info :write [12 14] :RequestCanceledException INFO [2019-12-17 12:34:49,644] jepsen worker 52 - jepsen.util 9232 :invoke :cas [17 [17 27]] INFO [2019-12-17 12:34:49,645] jepsen worker 38 - jepsen.util 9218 :invoke :write [12 18] INFO [2019-12-17 12:34:49,653] jepsen worker 46 - jepsen.util 8866 :fail :cas [15 [17 34]] :ValueNotSwapFrom INFO [2019-12-17 12:34:49,653] jepsen worker 46 - jepsen.util 8866 :invoke :cas [15 [33 26]] INFO [2019-12-17 12:34:49,661] jepsen worker 4 - jepsen.util 8284 :info :write [1 13] :RequestCanceledException INFO [2019-12-17 12:34:49,662] jepsen worker 4 - jepsen.util 8374 :invoke :read [1 nil] INFO [2019-12-17 12:34:49,704] jepsen worker 80 - jepsen.util 8540 :fail :cas [26 [28 41]] :ValueNotSwapFrom INFO [2019-12-17 12:34:49,704] jepsen worker 80 - jepsen.util 8540 :invoke :cas [26 [19 9]] INFO [2019-12-17 12:34:49,710] jepsen worker 10 - jepsen.util 9010 :fail :read [3 nil] :RequestTimeoutException INFO [2019-12-17 12:34:49,710] jepsen worker 18 - jepsen.util 9108 :info :cas [6 [32 21]] :RequestCanceledException INFO [2019-12-17 12:34:49,710] jepsen worker 10 - jepsen.util 9010 :invoke :write [3 20] INFO [2019-12-17 12:34:49,711] jepsen worker 18 - jepsen.util 9198 :invoke :write [6 46] INFO [2019-12-17 12:34:49,717] jepsen worker 45 - jepsen.util 9225 :ok :read [15 37] INFO [2019-12-17 12:34:49,717] jepsen worker 45 - jepsen.util 9225 :invoke :read [15 nil] INFO [2019-12-17 12:34:49,721] jepsen worker 27 - jepsen.util 8667 :ok :write [9 6] INFO [2019-12-17 12:34:49,722] jepsen worker 27 - jepsen.util 8667 :invoke :read [9 nil] INFO [2019-12-17 12:34:49,726] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51153,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:34:49,726] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51153,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:34:49,726] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51156,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:34:49,726] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51156,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:34:49,728] jepsen worker 20 - jepsen.util 8480 :ok :read [6 41] INFO [2019-12-17 12:34:49,728] jepsen worker 20 - jepsen.util 8480 :invoke :read [6 nil] INFO [2019-12-17 12:34:49,743] jepsen worker 64 - jepsen.util 7084 :fail :read [21 nil] :RequestTimeoutException INFO [2019-12-17 12:34:49,743] jepsen worker 37 - jepsen.util 8227 :info :write [12 20] :RequestCanceledException INFO [2019-12-17 12:34:49,743] jepsen worker 64 - jepsen.util 7084 :invoke :write [21 20] INFO [2019-12-17 12:34:49,744] jepsen worker 37 - jepsen.util 8317 :invoke :write [12 23] INFO [2019-12-17 12:34:49,761] jepsen worker 73 - jepsen.util 8623 :fail :read [24 nil] :RequestTimeoutException INFO [2019-12-17 12:34:49,761] jepsen worker 88 - jepsen.util 8998 :info :write [29 5] :RequestCanceledException INFO [2019-12-17 12:34:49,761] jepsen worker 73 - jepsen.util 8623 :invoke :cas [24 [47 28]] INFO [2019-12-17 12:34:49,763] jepsen worker 88 - jepsen.util 9088 :invoke :cas [29 [14 48]] INFO [2019-12-17 12:34:49,774] jepsen worker 55 - jepsen.util 9685 :fail :cas [18 [24 37]] :ValueNotSwapFrom INFO [2019-12-17 12:34:49,774] jepsen worker 55 - jepsen.util 9685 :invoke :write [18 49] INFO [2019-12-17 12:34:49,791] jepsen worker 65 - jepsen.util 7355 :fail :cas [21 [28 9]] :ValueNotSwapFrom INFO [2019-12-17 12:34:49,791] jepsen worker 65 - jepsen.util 7355 :invoke :read [21 nil] INFO [2019-12-17 12:34:49,795] jepsen worker 89 - jepsen.util 8909 :fail :cas [29 [32 13]] :ValueNotSwapFrom INFO [2019-12-17 12:34:49,795] jepsen worker 89 - jepsen.util 8909 :invoke :cas [29 [36 19]] INFO [2019-12-17 12:34:49,810] jepsen worker 72 - jepsen.util 8892 :fail :cas [24 [49 34]] :ValueNotSwapFrom INFO [2019-12-17 12:34:49,810] jepsen worker 72 - jepsen.util 8892 :invoke :read [24 nil] INFO [2019-12-17 12:34:49,814] jepsen worker 27 - jepsen.util 8667 :ok :read [9 6] INFO [2019-12-17 12:34:49,814] jepsen worker 27 - jepsen.util 8667 :invoke :cas [9 [27 44]] INFO [2019-12-17 12:34:49,814] jepsen worker 33 - jepsen.util 12003 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:34:49,814] jepsen worker 3 - jepsen.util 8643 :info :write [1 6] :RequestCanceledException INFO [2019-12-17 12:34:49,814] jepsen worker 33 - jepsen.util 12003 :invoke :write [11 40] INFO [2019-12-17 12:34:49,815] jepsen worker 3 - jepsen.util 8733 :invoke :write [1 8] INFO [2019-12-17 12:34:49,826] jepsen worker 87 - jepsen.util 8367 :fail :cas [29 [0 46]] :ValueNotSwapFrom INFO [2019-12-17 12:34:49,826] jepsen worker 87 - jepsen.util 8367 :invoke :write [29 35] INFO [2019-12-17 12:34:49,843] jepsen worker 80 - jepsen.util 8540 :fail :cas [26 [19 9]] :ValueNotSwapFrom INFO [2019-12-17 12:34:49,843] jepsen worker 80 - jepsen.util 8540 :invoke :cas [26 [0 12]] INFO [2019-12-17 12:34:49,859] jepsen worker 73 - jepsen.util 8623 :fail :cas [24 [47 28]] :ValueNotSwapFrom INFO [2019-12-17 12:34:49,859] jepsen worker 73 - jepsen.util 8623 :invoke :cas [24 [27 36]] INFO [2019-12-17 12:34:49,868] jepsen worker 20 - jepsen.util 8480 :ok :read [6 41] INFO [2019-12-17 12:34:49,868] jepsen worker 20 - jepsen.util 8480 :invoke :write [6 24] INFO [2019-12-17 12:34:49,872] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51182,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:34:49,872] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51182,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:34:49,872] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51186,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:34:49,872] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51186,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:34:49,880] jepsen worker 27 - jepsen.util 8667 :fail :cas [9 [27 44]] :ValueNotSwapFrom INFO [2019-12-17 12:34:49,880] jepsen worker 27 - jepsen.util 8667 :invoke :write [9 34] INFO [2019-12-17 12:34:49,896] jepsen worker 29 - jepsen.util 7319 :info :write [9 46] :RequestCanceledException INFO [2019-12-17 12:34:49,897] jepsen worker 29 - jepsen.util 7409 :invoke :read [9 nil] INFO [2019-12-17 12:34:49,916] jepsen worker 72 - jepsen.util 8892 :ok :read [24 18] INFO [2019-12-17 12:34:49,916] jepsen worker 72 - jepsen.util 8892 :invoke :read [24 nil] INFO [2019-12-17 12:34:49,917] jepsen worker 46 - jepsen.util 8866 :fail :cas [15 [33 26]] :ValueNotSwapFrom INFO [2019-12-17 12:34:49,918] jepsen worker 46 - jepsen.util 8866 :invoke :write [15 48] INFO [2019-12-17 12:34:49,929] jepsen worker 62 - jepsen.util 12212 :info :write [20 36] :RequestCanceledException INFO [2019-12-17 12:34:49,929] jepsen worker 25 - jepsen.util 12715 :info :write [8 19] :RequestCanceledException INFO [2019-12-17 12:34:49,931] jepsen worker 62 - jepsen.util 12302 :invoke :read [20 nil] INFO [2019-12-17 12:34:49,931] jepsen worker 25 - jepsen.util 12805 :invoke :cas [8 [13 16]] INFO [2019-12-17 12:34:49,935] jepsen worker 52 - jepsen.util 9232 :fail :cas [17 [17 27]] :ValueNotSwapFrom INFO [2019-12-17 12:34:49,935] jepsen worker 52 - jepsen.util 9232 :invoke :read [17 nil] INFO [2019-12-17 12:34:49,938] jepsen worker 80 - jepsen.util 8540 :fail :cas [26 [0 12]] :ValueNotSwapFrom INFO [2019-12-17 12:34:49,938] jepsen worker 80 - jepsen.util 8540 :invoke :cas [26 [12 19]] INFO [2019-12-17 12:34:49,940] jepsen worker 63 - jepsen.util 7623 :info :write [21 23] :RequestCanceledException INFO [2019-12-17 12:34:49,941] jepsen worker 63 - jepsen.util 7713 :invoke :write [21 4] INFO [2019-12-17 12:34:49,942] jepsen worker 73 - jepsen.util 8623 :fail :cas [24 [27 36]] :ValueNotSwapFrom INFO [2019-12-17 12:34:49,942] jepsen worker 73 - jepsen.util 8623 :invoke :read [24 nil] INFO [2019-12-17 12:34:49,949] jepsen worker 78 - jepsen.util 8628 :ok :read [26 3] INFO [2019-12-17 12:34:49,949] jepsen worker 78 - jepsen.util 8628 :invoke :read [26 nil] INFO [2019-12-17 12:34:49,965] jepsen worker 64 - jepsen.util 7084 :ok :write [21 20] INFO [2019-12-17 12:34:49,966] jepsen worker 64 - jepsen.util 7084 :invoke :write [21 15] INFO [2019-12-17 12:34:49,983] jepsen worker 29 - jepsen.util 7409 :ok :read [9 6] INFO [2019-12-17 12:34:49,983] jepsen worker 29 - jepsen.util 7409 :invoke :cas [9 [49 22]] INFO [2019-12-17 12:34:49,991] jepsen worker 79 - jepsen.util 9259 :ok :read [26 3] INFO [2019-12-17 12:34:49,991] jepsen worker 79 - jepsen.util 9259 :invoke :cas [26 [15 16]] INFO [2019-12-17 12:34:49,997] jepsen worker 0 - jepsen.util 12510 :info :cas [0 [2 30]] :RequestCanceledException INFO [2019-12-17 12:34:49,998] jepsen worker 0 - jepsen.util 12600 :invoke :write [0 24] INFO [2019-12-17 12:34:50,013] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:34:50,014] jepsen worker 57 - jepsen.util 11757 :info :write [19 11] :RequestCanceledException INFO [2019-12-17 12:34:50,038] jepsen worker 64 - jepsen.util 7084 :ok :write [21 15] INFO [2019-12-17 12:34:50,046] jepsen nemesis - couchbase.util Rebalance check count: 395 INFO [2019-12-17 12:34:50,046] jepsen worker 64 - jepsen.util 7084 :invoke :write [21 12] INFO [2019-12-17 12:34:50,046] jepsen nemesis - couchbase.util Rebalance stuck count: 376 INFO [2019-12-17 12:34:50,047] jepsen worker 57 - jepsen.util 11847 :invoke :read [19 nil] INFO [2019-12-17 12:34:50,056] jepsen worker 88 - jepsen.util 9088 :fail :cas [29 [14 48]] :ValueNotSwapFrom INFO [2019-12-17 12:34:50,057] jepsen worker 88 - jepsen.util 9088 :invoke :read [29 nil] INFO [2019-12-17 12:34:50,061] jepsen worker 70 - jepsen.util 9340 :info :write [23 7] :RequestCanceledException INFO [2019-12-17 12:34:50,062] jepsen worker 70 - jepsen.util 9430 :invoke :write [23 35] INFO [2019-12-17 12:34:50,072] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51205,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:34:50,072] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51205,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:34:50,072] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51207,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:34:50,072] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51207,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:34:50,075] jepsen worker 69 - jepsen.util 8979 :fail :cas [23 [34 13]] :ValueNotSwapFrom INFO [2019-12-17 12:34:50,075] jepsen worker 69 - jepsen.util 8979 :invoke :read [23 nil] INFO [2019-12-17 12:34:50,092] jepsen worker 27 - jepsen.util 8667 :ok :write [9 34] INFO [2019-12-17 12:34:50,092] jepsen worker 27 - jepsen.util 8667 :invoke :cas [9 [40 31]] INFO [2019-12-17 12:34:50,097] jepsen worker 12 - jepsen.util 9372 :info :write [4 1] :RequestCanceledException INFO [2019-12-17 12:34:50,098] jepsen worker 12 - jepsen.util 9462 :invoke :cas [4 [15 37]] INFO [2019-12-17 12:34:50,115] jepsen worker 26 - jepsen.util 12536 :info :write [8 4] :RequestCanceledException INFO [2019-12-17 12:34:50,116] jepsen worker 26 - jepsen.util 12626 :invoke :read [8 nil] INFO [2019-12-17 12:34:50,127] jepsen worker 78 - jepsen.util 8628 :ok :read [26 3] INFO [2019-12-17 12:34:50,127] jepsen worker 78 - jepsen.util 8628 :invoke :write [26 9] INFO [2019-12-17 12:34:50,128] jepsen worker 27 - jepsen.util 8667 :fail :cas [9 [40 31]] :ValueNotSwapFrom INFO [2019-12-17 12:34:50,128] jepsen worker 27 - jepsen.util 8667 :invoke :read [9 nil] INFO [2019-12-17 12:34:50,130] jepsen worker 74 - jepsen.util 8984 :fail :read [24 nil] :RequestTimeoutException INFO [2019-12-17 12:34:50,130] jepsen worker 74 - jepsen.util 8984 :invoke :write [24 21] INFO [2019-12-17 12:34:50,172] jepsen worker 64 - jepsen.util 7084 :ok :write [21 12] INFO [2019-12-17 12:34:50,172] jepsen worker 64 - jepsen.util 7084 :invoke :read [21 nil] INFO [2019-12-17 12:34:50,173] jepsen worker 27 - jepsen.util 8667 :ok :read [9 34] INFO [2019-12-17 12:34:50,174] jepsen worker 27 - jepsen.util 8667 :invoke :read [9 nil] INFO [2019-12-17 12:34:50,183] jepsen worker 87 - jepsen.util 8367 :ok :write [29 35] INFO [2019-12-17 12:34:50,184] jepsen worker 87 - jepsen.util 8367 :invoke :cas [29 [3 46]] INFO [2019-12-17 12:34:50,193] jepsen worker 4 - jepsen.util 8374 :ok :read [1 44] INFO [2019-12-17 12:34:50,193] jepsen worker 4 - jepsen.util 8374 :invoke :write [1 4] INFO [2019-12-17 12:34:50,202] jepsen worker 30 - jepsen.util 8850 :ok :write [10 35] INFO [2019-12-17 12:34:50,202] jepsen worker 30 - jepsen.util 8850 :invoke :write [10 12] INFO [2019-12-17 12:34:50,204] jepsen worker 19 - jepsen.util 8839 :ok :write [6 28] INFO [2019-12-17 12:34:50,204] jepsen worker 19 - jepsen.util 8839 :invoke :cas [6 [3 33]] INFO [2019-12-17 12:34:50,207] jepsen worker 64 - jepsen.util 7084 :ok :read [21 12] INFO [2019-12-17 12:34:50,207] jepsen worker 64 - jepsen.util 7084 :invoke :cas [21 [2 8]] INFO [2019-12-17 12:34:50,214] jepsen worker 58 - jepsen.util 12748 :info :write [19 29] :RequestCanceledException INFO [2019-12-17 12:34:50,215] jepsen worker 58 - jepsen.util 12838 :invoke :write [19 30] INFO [2019-12-17 12:34:50,216] jepsen worker 27 - jepsen.util 8667 :ok :read [9 34] INFO [2019-12-17 12:34:50,216] jepsen worker 27 - jepsen.util 8667 :invoke :cas [9 [34 18]] INFO [2019-12-17 12:34:50,218] jepsen worker 89 - jepsen.util 8909 :fail :cas [29 [36 19]] :ValueNotSwapFrom INFO [2019-12-17 12:34:50,219] jepsen worker 89 - jepsen.util 8909 :invoke :write [29 10] INFO [2019-12-17 12:34:50,230] jepsen worker 85 - jepsen.util 12685 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:34:50,230] jepsen worker 53 - jepsen.util 8063 :info :write [17 13] :RequestCanceledException INFO [2019-12-17 12:34:50,230] jepsen worker 44 - jepsen.util 12374 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:34:50,230] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51235,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:34:50,230] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51223,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:34:50,230] jepsen worker 85 - jepsen.util 12685 :invoke :write [28 44] INFO [2019-12-17 12:34:50,230] jepsen worker 44 - jepsen.util 12374 :invoke :cas [14 [49 1]] INFO [2019-12-17 12:34:50,230] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51235,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:34:50,230] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51223,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:34:50,230] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51239,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:34:50,230] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51225,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:34:50,230] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51239,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:34:50,230] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51225,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:34:50,231] jepsen worker 53 - jepsen.util 8153 :invoke :cas [17 [6 20]] INFO [2019-12-17 12:34:50,244] jepsen worker 48 - jepsen.util 11118 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:34:50,244] jepsen worker 48 - jepsen.util 11118 :invoke :cas [16 [41 38]] INFO [2019-12-17 12:34:50,247] jepsen worker 10 - jepsen.util 9010 :ok :write [3 20] INFO [2019-12-17 12:34:50,247] jepsen worker 10 - jepsen.util 9010 :invoke :read [3 nil] INFO [2019-12-17 12:34:50,251] jepsen worker 79 - jepsen.util 9259 :fail :cas [26 [15 16]] :ValueNotSwapFrom INFO [2019-12-17 12:34:50,252] jepsen worker 79 - jepsen.util 9259 :invoke :write [26 8] INFO [2019-12-17 12:34:50,258] jepsen worker 64 - jepsen.util 7084 :fail :cas [21 [2 8]] :ValueNotSwapFrom INFO [2019-12-17 12:34:50,258] jepsen worker 64 - jepsen.util 7084 :invoke :write [21 41] INFO [2019-12-17 12:34:50,258] jepsen worker 66 - jepsen.util 11766 :info :cas [22 [12 47]] :RequestCanceledException INFO [2019-12-17 12:34:50,258] jepsen worker 6 - jepsen.util 12426 :info :cas [2 [0 43]] :RequestCanceledException INFO [2019-12-17 12:34:50,259] jepsen worker 6 - jepsen.util 12516 :invoke :write [2 8] INFO [2019-12-17 12:34:50,259] jepsen worker 66 - jepsen.util 11856 :invoke :read [22 nil] INFO [2019-12-17 12:34:50,272] jepsen worker 12 - jepsen.util 9462 :fail :cas [4 [15 37]] :ValueNotSwapFrom INFO [2019-12-17 12:34:50,272] jepsen worker 12 - jepsen.util 9462 :invoke :cas [4 [41 14]] INFO [2019-12-17 12:34:50,281] jepsen worker 88 - jepsen.util 9088 :ok :read [29 35] INFO [2019-12-17 12:34:50,281] jepsen worker 88 - jepsen.util 9088 :invoke :cas [29 [32 48]] INFO [2019-12-17 12:34:50,296] jepsen worker 61 - jepsen.util 12751 :info :cas [20 [49 29]] :RequestCanceledException INFO [2019-12-17 12:34:50,297] jepsen worker 61 - jepsen.util 12841 :invoke :cas [20 [22 27]] INFO [2019-12-17 12:34:50,304] jepsen worker 88 - jepsen.util 9088 :fail :cas [29 [32 48]] :ValueNotSwapFrom INFO [2019-12-17 12:34:50,304] jepsen worker 88 - jepsen.util 9088 :invoke :read [29 nil] INFO [2019-12-17 12:34:50,314] jepsen worker 69 - jepsen.util 8979 :ok :read [23 40] INFO [2019-12-17 12:34:50,314] jepsen worker 69 - jepsen.util 8979 :invoke :read [23 nil] INFO [2019-12-17 12:34:50,326] jepsen worker 10 - jepsen.util 9010 :ok :read [3 20] INFO [2019-12-17 12:34:50,326] jepsen worker 10 - jepsen.util 9010 :invoke :write [3 40] INFO [2019-12-17 12:34:50,326] jepsen worker 88 - jepsen.util 9088 :ok :read [29 35] INFO [2019-12-17 12:34:50,327] jepsen worker 88 - jepsen.util 9088 :invoke :cas [29 [12 1]] INFO [2019-12-17 12:34:50,330] jepsen worker 22 - jepsen.util 11902 :info :cas [7 [24 3]] :RequestCanceledException INFO [2019-12-17 12:34:50,331] jepsen worker 22 - jepsen.util 11992 :invoke :read [7 nil] INFO [2019-12-17 12:34:50,343] jepsen worker 12 - jepsen.util 9462 :fail :cas [4 [41 14]] :ValueNotSwapFrom INFO [2019-12-17 12:34:50,343] jepsen worker 12 - jepsen.util 9462 :invoke :read [4 nil] INFO [2019-12-17 12:34:50,364] jepsen worker 87 - jepsen.util 8367 :fail :cas [29 [3 46]] :ValueNotSwapFrom INFO [2019-12-17 12:34:50,364] jepsen worker 87 - jepsen.util 8367 :invoke :read [29 nil] INFO [2019-12-17 12:34:50,414] jepsen worker 74 - jepsen.util 8984 :ok :write [24 21] INFO [2019-12-17 12:34:50,414] jepsen worker 74 - jepsen.util 8984 :invoke :read [24 nil] INFO [2019-12-17 12:34:50,430] jepsen worker 54 - jepsen.util 9684 :info :write [18 21] :RequestCanceledException INFO [2019-12-17 12:34:50,439] jepsen worker 54 - jepsen.util 9774 :invoke :write [18 16] INFO [2019-12-17 12:34:50,443] jepsen worker 74 - jepsen.util 8984 :ok :read [24 21] INFO [2019-12-17 12:34:50,443] jepsen worker 74 - jepsen.util 8984 :invoke :read [24 nil] INFO [2019-12-17 12:34:50,444] jepsen worker 84 - jepsen.util 11334 :info :write [28 1] :RequestCanceledException INFO [2019-12-17 12:34:50,445] jepsen worker 64 - jepsen.util 7084 :ok :write [21 41] INFO [2019-12-17 12:34:50,445] jepsen worker 84 - jepsen.util 11424 :invoke :cas [28 [29 21]] INFO [2019-12-17 12:34:50,446] jepsen worker 64 - jepsen.util 7084 :invoke :cas [21 [39 31]] INFO [2019-12-17 12:34:50,451] jepsen worker 53 - jepsen.util 8153 :fail :cas [17 [6 20]] :ValueNotSwapFrom INFO [2019-12-17 12:34:50,452] jepsen worker 53 - jepsen.util 8153 :invoke :cas [17 [5 1]] INFO [2019-12-17 12:34:50,456] jepsen worker 12 - jepsen.util 9462 :ok :read [4 13] INFO [2019-12-17 12:34:50,456] jepsen worker 12 - jepsen.util 9462 :invoke :read [4 nil] INFO [2019-12-17 12:34:50,460] jepsen worker 24 - jepsen.util 11004 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:34:50,461] jepsen worker 24 - jepsen.util 11004 :invoke :write [8 45] INFO [2019-12-17 12:34:50,470] jepsen worker 27 - jepsen.util 8667 :ok :cas [9 [34 18]] INFO [2019-12-17 12:34:50,470] jepsen worker 27 - jepsen.util 8667 :invoke :read [9 nil] INFO [2019-12-17 12:34:50,481] jepsen worker 64 - jepsen.util 7084 :fail :cas [21 [39 31]] :ValueNotSwapFrom INFO [2019-12-17 12:34:50,481] jepsen worker 64 - jepsen.util 7084 :invoke :write [21 30] INFO [2019-12-17 12:34:50,503] jepsen worker 74 - jepsen.util 8984 :ok :read [24 21] INFO [2019-12-17 12:34:50,503] jepsen worker 74 - jepsen.util 8984 :invoke :write [24 42] INFO [2019-12-17 12:34:50,504] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51264,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:34:50,504] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51264,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:34:50,504] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51267,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:34:50,504] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51267,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:34:50,504] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51268,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:34:50,504] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51268,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:34:50,506] jepsen worker 27 - jepsen.util 8667 :ok :read [9 18] INFO [2019-12-17 12:34:50,507] jepsen worker 27 - jepsen.util 8667 :invoke :write [9 45] INFO [2019-12-17 12:34:50,512] jepsen worker 17 - jepsen.util 11987 :info :cas [5 [48 43]] :RequestCanceledException INFO [2019-12-17 12:34:50,513] jepsen worker 17 - jepsen.util 12077 :invoke :read [5 nil] INFO [2019-12-17 12:34:50,522] jepsen worker 80 - jepsen.util 8540 :fail :cas [26 [12 19]] :ValueNotSwapFrom INFO [2019-12-17 12:34:50,522] jepsen worker 80 - jepsen.util 8540 :invoke :write [26 28] INFO [2019-12-17 12:34:50,527] jepsen worker 14 - jepsen.util 9014 :ok :write [4 40] INFO [2019-12-17 12:34:50,527] jepsen worker 14 - jepsen.util 9014 :invoke :cas [4 [27 37]] INFO [2019-12-17 12:34:50,530] jepsen worker 40 - jepsen.util 12100 :info :cas [13 [7 23]] :RequestCanceledException INFO [2019-12-17 12:34:50,531] jepsen worker 40 - jepsen.util 12190 :invoke :write [13 16] INFO [2019-12-17 12:34:50,543] jepsen worker 87 - jepsen.util 8367 :ok :read [29 35] INFO [2019-12-17 12:34:50,543] jepsen worker 87 - jepsen.util 8367 :invoke :cas [29 [34 36]] INFO [2019-12-17 12:34:50,549] jepsen worker 88 - jepsen.util 9088 :fail :cas [29 [12 1]] :ValueNotSwapFrom INFO [2019-12-17 12:34:50,549] jepsen worker 88 - jepsen.util 9088 :invoke :cas [29 [23 23]] INFO [2019-12-17 12:34:50,561] jepsen worker 43 - jepsen.util 11743 :info :write [14 43] :RequestCanceledException INFO [2019-12-17 12:34:50,561] jepsen worker 42 - jepsen.util 13002 :info :cas [14 [49 17]] :RequestCanceledException INFO [2019-12-17 12:34:50,562] jepsen worker 43 - jepsen.util 11833 :invoke :read [14 nil] INFO [2019-12-17 12:34:50,563] jepsen worker 42 - jepsen.util 13092 :invoke :write [14 30] INFO [2019-12-17 12:34:50,565] jepsen worker 10 - jepsen.util 9010 :ok :write [3 40] INFO [2019-12-17 12:34:50,566] jepsen worker 10 - jepsen.util 9010 :invoke :cas [3 [38 1]] INFO [2019-12-17 12:34:50,600] jepsen worker 10 - jepsen.util 9010 :fail :cas [3 [38 1]] :ValueNotSwapFrom INFO [2019-12-17 12:34:50,600] jepsen worker 10 - jepsen.util 9010 :invoke :write [3 43] INFO [2019-12-17 12:34:50,601] jepsen worker 64 - jepsen.util 7084 :ok :write [21 30] INFO [2019-12-17 12:34:50,601] jepsen worker 64 - jepsen.util 7084 :invoke :cas [21 [11 22]] INFO [2019-12-17 12:34:50,627] jepsen worker 15 - jepsen.util 11085 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:34:50,627] jepsen worker 1 - jepsen.util 12151 :info :write [0 12] :RequestCanceledException INFO [2019-12-17 12:34:50,628] jepsen worker 15 - jepsen.util 11085 :invoke :cas [5 [27 33]] INFO [2019-12-17 12:34:50,632] jepsen worker 1 - jepsen.util 12241 :invoke :read [0 nil] INFO [2019-12-17 12:34:50,633] jepsen worker 27 - jepsen.util 8667 :ok :write [9 45] INFO [2019-12-17 12:34:50,633] jepsen worker 27 - jepsen.util 8667 :invoke :read [9 nil] INFO [2019-12-17 12:34:50,638] jepsen worker 53 - jepsen.util 8153 :fail :cas [17 [5 1]] :ValueNotSwapFrom INFO [2019-12-17 12:34:50,638] jepsen worker 53 - jepsen.util 8153 :invoke :write [17 18] INFO [2019-12-17 12:34:50,639] jepsen worker 64 - jepsen.util 7084 :fail :cas [21 [11 22]] :ValueNotSwapFrom INFO [2019-12-17 12:34:50,640] jepsen worker 64 - jepsen.util 7084 :invoke :cas [21 [21 28]] INFO [2019-12-17 12:34:50,643] jepsen worker 77 - jepsen.util 12407 :info :write [25 28] :RequestCanceledException INFO [2019-12-17 12:34:50,644] jepsen worker 77 - jepsen.util 12497 :invoke :write [25 10] INFO [2019-12-17 12:34:50,666] jepsen worker 14 - jepsen.util 9014 :fail :cas [4 [27 37]] :ValueNotSwapFrom INFO [2019-12-17 12:34:50,667] jepsen worker 27 - jepsen.util 8667 :ok :read [9 45] INFO [2019-12-17 12:34:50,667] jepsen worker 27 - jepsen.util 8667 :invoke :read [9 nil] INFO [2019-12-17 12:34:50,668] jepsen worker 14 - jepsen.util 9014 :invoke :write [4 12] INFO [2019-12-17 12:34:50,675] jepsen worker 64 - jepsen.util 7084 :fail :cas [21 [21 28]] :ValueNotSwapFrom INFO [2019-12-17 12:34:50,675] jepsen worker 64 - jepsen.util 7084 :invoke :cas [21 [12 40]] INFO [2019-12-17 12:34:50,696] jepsen worker 8 - jepsen.util 11798 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:34:50,696] jepsen worker 8 - jepsen.util 11798 :invoke :read [2 nil] INFO [2019-12-17 12:34:50,706] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51296,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:34:50,706] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51296,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:34:50,706] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51300,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:34:50,706] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51300,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:34:50,709] jepsen worker 87 - jepsen.util 8367 :fail :cas [29 [34 36]] :ValueNotSwapFrom INFO [2019-12-17 12:34:50,709] jepsen worker 87 - jepsen.util 8367 :invoke :cas [29 [10 26]] INFO [2019-12-17 12:34:50,714] jepsen worker 75 - jepsen.util 11325 :info :write [25 21] :RequestCanceledException INFO [2019-12-17 12:34:50,715] jepsen worker 75 - jepsen.util 11415 :invoke :read [25 nil] INFO [2019-12-17 12:34:50,723] jepsen worker 27 - jepsen.util 8667 :ok :read [9 45] INFO [2019-12-17 12:34:50,723] jepsen worker 27 - jepsen.util 8667 :invoke :read [9 nil] INFO [2019-12-17 12:34:50,727] jepsen worker 64 - jepsen.util 7084 :fail :cas [21 [12 40]] :ValueNotSwapFrom INFO [2019-12-17 12:34:50,727] jepsen worker 64 - jepsen.util 7084 :invoke :cas [21 [5 6]] INFO [2019-12-17 12:34:50,730] jepsen worker 7 - jepsen.util 12247 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:34:50,730] jepsen worker 7 - jepsen.util 12247 :invoke :read [2 nil] INFO [2019-12-17 12:34:50,732] jepsen worker 88 - jepsen.util 9088 :fail :cas [29 [23 23]] :ValueNotSwapFrom INFO [2019-12-17 12:34:50,732] jepsen worker 88 - jepsen.util 9088 :invoke :cas [29 [9 39]] INFO [2019-12-17 12:34:50,758] jepsen worker 18 - jepsen.util 9198 :ok :write [6 46] INFO [2019-12-17 12:34:50,758] jepsen worker 18 - jepsen.util 9198 :invoke :read [6 nil] INFO [2019-12-17 12:34:50,760] jepsen worker 27 - jepsen.util 8667 :ok :read [9 45] INFO [2019-12-17 12:34:50,760] jepsen worker 27 - jepsen.util 8667 :invoke :cas [9 [32 4]] INFO [2019-12-17 12:34:50,763] jepsen worker 64 - jepsen.util 7084 :fail :cas [21 [5 6]] :ValueNotSwapFrom INFO [2019-12-17 12:34:50,763] jepsen worker 64 - jepsen.util 7084 :invoke :write [21 31] INFO [2019-12-17 12:34:50,788] jepsen worker 18 - jepsen.util 9198 :ok :read [6 46] INFO [2019-12-17 12:34:50,788] jepsen worker 18 - jepsen.util 9198 :invoke :read [6 nil] INFO [2019-12-17 12:34:50,811] jepsen worker 27 - jepsen.util 8667 :fail :cas [9 [32 4]] :ValueNotSwapFrom INFO [2019-12-17 12:34:50,811] jepsen worker 27 - jepsen.util 8667 :invoke :write [9 46] INFO [2019-12-17 12:34:50,818] jepsen worker 52 - jepsen.util 9232 :ok :read [17 27] INFO [2019-12-17 12:34:50,818] jepsen worker 52 - jepsen.util 9232 :invoke :cas [17 [0 42]] INFO [2019-12-17 12:34:50,830] jepsen worker 87 - jepsen.util 8367 :fail :cas [29 [10 26]] :ValueNotSwapFrom INFO [2019-12-17 12:34:50,831] jepsen worker 87 - jepsen.util 8367 :invoke :cas [29 [14 23]] INFO [2019-12-17 12:34:50,852] jepsen worker 74 - jepsen.util 8984 :ok :write [24 42] INFO [2019-12-17 12:34:50,852] jepsen worker 74 - jepsen.util 8984 :invoke :read [24 nil] INFO [2019-12-17 12:34:50,872] jepsen worker 18 - jepsen.util 9198 :ok :read [6 46] INFO [2019-12-17 12:34:50,873] jepsen worker 18 - jepsen.util 9198 :invoke :read [6 nil] INFO [2019-12-17 12:34:50,888] jepsen worker 10 - jepsen.util 9010 :ok :write [3 43] INFO [2019-12-17 12:34:50,888] jepsen worker 10 - jepsen.util 9010 :invoke :write [3 0] INFO [2019-12-17 12:34:50,894] jepsen worker 74 - jepsen.util 8984 :ok :read [24 42] INFO [2019-12-17 12:34:50,894] jepsen worker 74 - jepsen.util 8984 :invoke :cas [24 [20 37]] INFO [2019-12-17 12:34:50,907] jepsen worker 18 - jepsen.util 9198 :ok :read [6 46] INFO [2019-12-17 12:34:50,907] jepsen worker 18 - jepsen.util 9198 :invoke :cas [6 [19 44]] INFO [2019-12-17 12:34:50,930] jepsen worker 49 - jepsen.util 11389 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:34:50,930] jepsen worker 49 - jepsen.util 11389 :invoke :write [16 33] INFO [2019-12-17 12:34:50,949] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51324,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:34:50,949] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51324,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:34:50,984] jepsen worker 74 - jepsen.util 8984 :fail :cas [24 [20 37]] :ValueNotSwapFrom INFO [2019-12-17 12:34:50,984] jepsen worker 74 - jepsen.util 8984 :invoke :cas [24 [11 31]] INFO [2019-12-17 12:34:50,993] jepsen worker 18 - jepsen.util 9198 :fail :cas [6 [19 44]] :ValueNotSwapFrom INFO [2019-12-17 12:34:50,994] jepsen worker 18 - jepsen.util 9198 :invoke :read [6 nil] INFO [2019-12-17 12:34:51,002] jepsen worker 74 - jepsen.util 8984 :fail :cas [24 [11 31]] :ValueNotSwapFrom INFO [2019-12-17 12:34:51,002] jepsen worker 74 - jepsen.util 8984 :invoke :write [24 48] INFO [2019-12-17 12:34:51,005] jepsen worker 27 - jepsen.util 8667 :ok :write [9 46] INFO [2019-12-17 12:34:51,005] jepsen worker 27 - jepsen.util 8667 :invoke :cas [9 [20 48]] INFO [2019-12-17 12:34:51,030] jepsen worker 2 - jepsen.util 11702 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:34:51,030] jepsen worker 2 - jepsen.util 11702 :invoke :cas [0 [11 26]] INFO [2019-12-17 12:34:51,037] jepsen worker 64 - jepsen.util 7084 :ok :write [21 31] INFO [2019-12-17 12:34:51,037] jepsen worker 64 - jepsen.util 7084 :invoke :write [21 21] INFO [2019-12-17 12:34:51,039] jepsen worker 27 - jepsen.util 8667 :fail :cas [9 [20 48]] :ValueNotSwapFrom INFO [2019-12-17 12:34:51,039] jepsen worker 27 - jepsen.util 8667 :invoke :write [9 36] INFO [2019-12-17 12:34:51,044] jepsen worker 86 - jepsen.util 12866 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:34:51,044] jepsen worker 86 - jepsen.util 12866 :invoke :cas [28 [45 37]] INFO [2019-12-17 12:34:51,046] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51333,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:34:51,046] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51333,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:34:51,046] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51337,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:34:51,046] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51337,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:34:51,046] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51338,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:34:51,046] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51338,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:34:51,054] jepsen worker 18 - jepsen.util 9198 :ok :read [6 46] INFO [2019-12-17 12:34:51,055] jepsen worker 18 - jepsen.util 9198 :invoke :write [6 33] INFO [2019-12-17 12:34:51,081] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:34:51,081] jepsen nemesis - couchbase.util Rebalance check count: 396 INFO [2019-12-17 12:34:51,082] jepsen nemesis - couchbase.util Rebalance stuck count: 377 INFO [2019-12-17 12:34:51,095] jepsen worker 12 - jepsen.util 9462 :ok :read [4 40] INFO [2019-12-17 12:34:51,095] jepsen worker 12 - jepsen.util 9462 :invoke :cas [4 [7 21]] INFO [2019-12-17 12:34:51,096] jepsen worker 36 - jepsen.util 9216 :info :write [12 48] :RequestCanceledException INFO [2019-12-17 12:34:51,096] jepsen worker 41 - jepsen.util 12821 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:34:51,096] jepsen worker 41 - jepsen.util 12821 :invoke :read [13 nil] INFO [2019-12-17 12:34:51,097] jepsen worker 36 - jepsen.util 9306 :invoke :cas [12 [5 2]] INFO [2019-12-17 12:34:51,114] jepsen worker 39 - jepsen.util 11469 :info :cas [13 [46 19]] :RequestCanceledException INFO [2019-12-17 12:34:51,115] jepsen worker 39 - jepsen.util 11559 :invoke :write [13 39] INFO [2019-12-17 12:34:51,144] jepsen worker 81 - jepsen.util 11781 :info :cas [27 [22 35]] :RequestCanceledException INFO [2019-12-17 12:34:51,144] jepsen worker 82 - jepsen.util 12682 :info :cas [27 [41 40]] :RequestCanceledException INFO [2019-12-17 12:34:51,145] jepsen worker 81 - jepsen.util 11871 :invoke :read [27 nil] INFO [2019-12-17 12:34:51,145] jepsen worker 82 - jepsen.util 12772 :invoke :cas [27 [47 2]] INFO [2019-12-17 12:34:51,159] jepsen worker 64 - jepsen.util 7084 :ok :write [21 21] INFO [2019-12-17 12:34:51,159] jepsen worker 64 - jepsen.util 7084 :invoke :cas [21 [18 34]] INFO [2019-12-17 12:34:51,192] jepsen worker 27 - jepsen.util 8667 :ok :write [9 36] INFO [2019-12-17 12:34:51,192] jepsen worker 27 - jepsen.util 8667 :invoke :write [9 14] INFO [2019-12-17 12:34:51,212] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51354,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:34:51,212] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51354,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:34:51,212] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51355,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:34:51,213] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51355,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:34:51,220] jepsen worker 64 - jepsen.util 7084 :fail :cas [21 [18 34]] :ValueNotSwapFrom INFO [2019-12-17 12:34:51,220] jepsen worker 64 - jepsen.util 7084 :invoke :read [21 nil] INFO [2019-12-17 12:34:51,230] jepsen worker 21 - jepsen.util 11631 :info :cas [7 [18 17]] :RequestCanceledException INFO [2019-12-17 12:34:51,230] jepsen worker 34 - jepsen.util 12184 :info :write [11 3] :RequestCanceledException INFO [2019-12-17 12:34:51,230] jepsen worker 76 - jepsen.util 12316 :info :write [25 26] :RequestCanceledException INFO [2019-12-17 12:34:51,231] jepsen worker 34 - jepsen.util 12274 :invoke :read [11 nil] INFO [2019-12-17 12:34:51,231] jepsen worker 21 - jepsen.util 11721 :invoke :write [7 16] INFO [2019-12-17 12:34:51,231] jepsen worker 76 - jepsen.util 12406 :invoke :write [25 35] INFO [2019-12-17 12:34:51,244] jepsen worker 28 - jepsen.util 7138 :info :cas [9 [47 16]] :RequestCanceledException INFO [2019-12-17 12:34:51,244] jepsen worker 28 - jepsen.util 7228 :invoke :cas [9 [4 20]] INFO [2019-12-17 12:34:51,248] jepsen worker 64 - jepsen.util 7084 :ok :read [21 21] INFO [2019-12-17 12:34:51,248] jepsen worker 64 - jepsen.util 7084 :invoke :cas [21 [26 25]] INFO [2019-12-17 12:34:51,264] jepsen worker 10 - jepsen.util 9010 :ok :write [3 0] INFO [2019-12-17 12:34:51,264] jepsen worker 10 - jepsen.util 9010 :invoke :write [3 42] INFO [2019-12-17 12:34:51,275] jepsen worker 27 - jepsen.util 8667 :ok :write [9 14] INFO [2019-12-17 12:34:51,275] jepsen worker 27 - jepsen.util 8667 :invoke :read [9 nil] INFO [2019-12-17 12:34:51,283] jepsen worker 36 - jepsen.util 9306 :fail :cas [12 [5 2]] :ValueNotSwapFrom INFO [2019-12-17 12:34:51,283] jepsen worker 36 - jepsen.util 9306 :invoke :write [12 45] INFO [2019-12-17 12:34:51,291] jepsen worker 64 - jepsen.util 7084 :fail :cas [21 [26 25]] :ValueNotSwapFrom INFO [2019-12-17 12:34:51,291] jepsen worker 64 - jepsen.util 7084 :invoke :read [21 nil] INFO [2019-12-17 12:34:51,292] jepsen worker 23 - jepsen.util 11723 :info :cas [7 [18 14]] :RequestCanceledException INFO [2019-12-17 12:34:51,293] jepsen worker 23 - jepsen.util 11813 :invoke :write [7 25] INFO [2019-12-17 12:34:51,298] jepsen worker 28 - jepsen.util 7228 :fail :cas [9 [4 20]] :ValueNotSwapFrom INFO [2019-12-17 12:34:51,298] jepsen worker 28 - jepsen.util 7228 :invoke :cas [9 [30 16]] INFO [2019-12-17 12:34:51,300] jepsen worker 18 - jepsen.util 9198 :ok :write [6 33] INFO [2019-12-17 12:34:51,300] jepsen worker 18 - jepsen.util 9198 :invoke :cas [6 [6 42]] INFO [2019-12-17 12:34:51,304] jepsen worker 74 - jepsen.util 8984 :ok :write [24 48] INFO [2019-12-17 12:34:51,304] jepsen worker 74 - jepsen.util 8984 :invoke :read [24 nil] INFO [2019-12-17 12:34:51,307] jepsen worker 64 - jepsen.util 7084 :ok :read [21 21] INFO [2019-12-17 12:34:51,307] jepsen worker 64 - jepsen.util 7084 :invoke :write [21 9] INFO [2019-12-17 12:34:51,322] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51363,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:34:51,322] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51363,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:34:51,322] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51366,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:34:51,322] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51366,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:34:51,330] jepsen worker 59 - jepsen.util 11309 :info :write [19 25] :RequestCanceledException INFO [2019-12-17 12:34:51,332] jepsen worker 59 - jepsen.util 11399 :invoke :cas [19 [37 6]] INFO [2019-12-17 12:34:51,361] jepsen worker 31 - jepsen.util 7861 :info :cas [10 [41 5]] :RequestCanceledException INFO [2019-12-17 12:34:51,366] jepsen worker 31 - jepsen.util 7951 :invoke :cas [10 [28 39]] INFO [2019-12-17 12:34:51,380] jepsen worker 52 - jepsen.util 9232 :fail :cas [17 [0 42]] :ValueNotSwapFrom INFO [2019-12-17 12:34:51,380] jepsen worker 52 - jepsen.util 9232 :invoke :cas [17 [21 2]] INFO [2019-12-17 12:34:51,392] jepsen worker 27 - jepsen.util 8667 :ok :read [9 14] INFO [2019-12-17 12:34:51,393] jepsen worker 27 - jepsen.util 8667 :invoke :read [9 nil] INFO [2019-12-17 12:34:51,397] jepsen worker 68 - jepsen.util 11318 :info :write [22 48] :RequestCanceledException INFO [2019-12-17 12:34:51,398] jepsen worker 68 - jepsen.util 11408 :invoke :read [22 nil] INFO [2019-12-17 12:34:51,414] jepsen worker 83 - jepsen.util 12503 :info :cas [27 [38 49]] :RequestCanceledException INFO [2019-12-17 12:34:51,415] jepsen worker 83 - jepsen.util 12593 :invoke :write [27 14] INFO [2019-12-17 12:34:51,429] jepsen worker 47 - jepsen.util 9227 :info :cas [15 [7 34]] :RequestCanceledException INFO [2019-12-17 12:34:51,430] jepsen worker 47 - jepsen.util 9317 :invoke :read [15 nil] INFO [2019-12-17 12:34:51,430] jepsen worker 50 - jepsen.util 12650 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:34:51,430] jepsen worker 13 - jepsen.util 8563 :info :write [4 28] :RequestCanceledException INFO [2019-12-17 12:34:51,430] jepsen worker 50 - jepsen.util 12650 :invoke :cas [16 [11 22]] INFO [2019-12-17 12:34:51,431] jepsen worker 13 - jepsen.util 8653 :invoke :write [4 45] INFO [2019-12-17 12:34:51,445] jepsen worker 88 - jepsen.util 9088 :fail :cas [29 [9 39]] :ValueNotSwapFrom INFO [2019-12-17 12:34:51,446] jepsen worker 88 - jepsen.util 9088 :invoke :read [29 nil] INFO [2019-12-17 12:34:51,456] jepsen worker 74 - jepsen.util 8984 :ok :read [24 48] INFO [2019-12-17 12:34:51,457] jepsen worker 74 - jepsen.util 8984 :invoke :cas [24 [15 15]] INFO [2019-12-17 12:34:51,461] jepsen worker 16 - jepsen.util 13066 :info :write [5 27] :RequestCanceledException INFO [2019-12-17 12:34:51,462] jepsen worker 16 - jepsen.util 13156 :invoke :read [5 nil] INFO [2019-12-17 12:34:51,473] jepsen worker 28 - jepsen.util 7228 :fail :cas [9 [30 16]] :ValueNotSwapFrom INFO [2019-12-17 12:34:51,473] jepsen worker 28 - jepsen.util 7228 :invoke :read [9 nil] INFO [2019-12-17 12:34:51,488] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51392,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:34:51,489] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51392,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:34:51,497] jepsen worker 27 - jepsen.util 8667 :ok :read [9 14] INFO [2019-12-17 12:34:51,498] jepsen worker 27 - jepsen.util 8667 :invoke :write [9 36] INFO [2019-12-17 12:34:51,498] jepsen worker 11 - jepsen.util 8831 :info :write [3 31] :RequestCanceledException INFO [2019-12-17 12:34:51,499] jepsen worker 11 - jepsen.util 8921 :invoke :read [3 nil] INFO [2019-12-17 12:34:51,507] jepsen worker 18 - jepsen.util 9198 :fail :cas [6 [6 42]] :ValueNotSwapFrom INFO [2019-12-17 12:34:51,507] jepsen worker 18 - jepsen.util 9198 :invoke :read [6 nil] INFO [2019-12-17 12:34:51,508] jepsen worker 74 - jepsen.util 8984 :fail :cas [24 [15 15]] :ValueNotSwapFrom INFO [2019-12-17 12:34:51,508] jepsen worker 74 - jepsen.util 8984 :invoke :write [24 36] INFO [2019-12-17 12:34:51,530] jepsen worker 67 - jepsen.util 12127 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:34:51,530] jepsen worker 67 - jepsen.util 12127 :invoke :cas [22 [43 21]] INFO [2019-12-17 12:34:51,531] jepsen worker 60 - jepsen.util 13020 :info :write [20 20] :RequestCanceledException INFO [2019-12-17 12:34:51,531] jepsen worker 56 - jepsen.util 9506 :info :write [18 41] :RequestCanceledException INFO [2019-12-17 12:34:51,531] jepsen worker 9 - jepsen.util 9279 :info :write [3 17] :RequestCanceledException INFO [2019-12-17 12:34:51,532] jepsen worker 47 - jepsen.util 9317 :ok :read [15 37] INFO [2019-12-17 12:34:51,532] jepsen worker 9 - jepsen.util 9369 :invoke :write [3 14] INFO [2019-12-17 12:34:51,532] jepsen worker 47 - jepsen.util 9317 :invoke :cas [15 [33 14]] INFO [2019-12-17 12:34:51,535] jepsen worker 28 - jepsen.util 7228 :ok :read [9 14] INFO [2019-12-17 12:34:51,535] jepsen worker 28 - jepsen.util 7228 :invoke :read [9 nil] INFO [2019-12-17 12:34:51,537] jepsen worker 60 - jepsen.util 13110 :invoke :read [20 nil] INFO [2019-12-17 12:34:51,537] jepsen worker 56 - jepsen.util 9596 :invoke :read [18 nil] INFO [2019-12-17 12:34:51,544] jepsen worker 5 - jepsen.util 8285 :info :write [1 22] :RequestCanceledException INFO [2019-12-17 12:34:51,546] jepsen worker 5 - jepsen.util 8375 :invoke :cas [1 [5 15]] INFO [2019-12-17 12:34:51,550] jepsen worker 64 - jepsen.util 7084 :ok :write [21 9] INFO [2019-12-17 12:34:51,551] jepsen worker 64 - jepsen.util 7084 :invoke :read [21 nil] INFO [2019-12-17 12:34:51,554] jepsen worker 69 - jepsen.util 8979 :ok :read [23 40] INFO [2019-12-17 12:34:51,554] jepsen worker 69 - jepsen.util 8979 :invoke :cas [23 [2 7]] INFO [2019-12-17 12:34:51,561] jepsen worker 32 - jepsen.util 9032 :info :cas [10 [41 35]] :RequestCanceledException INFO [2019-12-17 12:34:51,562] jepsen worker 32 - jepsen.util 9122 :invoke :cas [10 [20 39]] INFO [2019-12-17 12:34:51,575] jepsen worker 10 - jepsen.util 9010 :ok :write [3 42] INFO [2019-12-17 12:34:51,575] jepsen worker 10 - jepsen.util 9010 :invoke :cas [3 [27 36]] INFO [2019-12-17 12:34:51,590] jepsen worker 64 - jepsen.util 7084 :ok :read [21 9] INFO [2019-12-17 12:34:51,590] jepsen worker 64 - jepsen.util 7084 :invoke :cas [21 [24 10]] INFO [2019-12-17 12:34:51,623] jepsen worker 52 - jepsen.util 9232 :fail :cas [17 [21 2]] :ValueNotSwapFrom INFO [2019-12-17 12:34:51,623] jepsen worker 52 - jepsen.util 9232 :invoke :write [17 6] INFO [2019-12-17 12:34:51,624] jepsen worker 18 - jepsen.util 9198 :ok :read [6 33] INFO [2019-12-17 12:34:51,624] jepsen worker 18 - jepsen.util 9198 :invoke :write [6 18] INFO [2019-12-17 12:34:51,629] jepsen worker 35 - jepsen.util 11915 :info :cas [11 [21 29]] :RequestCanceledException INFO [2019-12-17 12:34:51,640] jepsen worker 35 - jepsen.util 12005 :invoke :write [11 30] INFO [2019-12-17 12:34:51,650] jepsen worker 11 - jepsen.util 8921 :ok :read [3 42] INFO [2019-12-17 12:34:51,650] jepsen worker 11 - jepsen.util 8921 :invoke :write [3 37] INFO [2019-12-17 12:34:51,665] jepsen worker 27 - jepsen.util 8667 :ok :write [9 36] INFO [2019-12-17 12:34:51,665] jepsen worker 27 - jepsen.util 8667 :invoke :read [9 nil] INFO [2019-12-17 12:34:51,688] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51419,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:34:51,688] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51419,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:34:51,688] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51422,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:34:51,688] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51422,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:34:51,696] jepsen worker 71 - jepsen.util 9341 :info :write [23 26] :RequestCanceledException INFO [2019-12-17 12:34:51,697] jepsen worker 71 - jepsen.util 9431 :invoke :write [23 16] INFO [2019-12-17 12:34:51,697] jepsen worker 87 - jepsen.util 8367 :fail :cas [29 [14 23]] :ValueNotSwapFrom INFO [2019-12-17 12:34:51,697] jepsen worker 87 - jepsen.util 8367 :invoke :cas [29 [30 36]] INFO [2019-12-17 12:34:51,699] jepsen worker 51 - jepsen.util 8151 :info :write [17 47] :RequestCanceledException INFO [2019-12-17 12:34:51,707] jepsen worker 74 - jepsen.util 8984 :ok :write [24 36] INFO [2019-12-17 12:34:51,707] jepsen worker 74 - jepsen.util 8984 :invoke :read [24 nil] INFO [2019-12-17 12:34:51,730] jepsen worker 51 - jepsen.util 8241 :invoke :write [17 42] INFO [2019-12-17 12:34:51,743] jepsen worker 38 - jepsen.util 9218 :info :write [12 18] :RequestCanceledException INFO [2019-12-17 12:34:51,744] jepsen worker 38 - jepsen.util 9308 :invoke :cas [12 [23 37]] INFO [2019-12-17 12:34:51,749] jepsen worker 27 - jepsen.util 8667 :ok :read [9 36] INFO [2019-12-17 12:34:51,749] jepsen worker 27 - jepsen.util 8667 :invoke :read [9 nil] INFO [2019-12-17 12:34:51,799] jepsen worker 37 - jepsen.util 8317 :info :write [12 23] :RequestCanceledException INFO [2019-12-17 12:34:51,806] jepsen worker 12 - jepsen.util 9462 :fail :cas [4 [7 21]] :ValueNotSwapFrom INFO [2019-12-17 12:34:51,806] jepsen worker 12 - jepsen.util 9462 :invoke :write [4 23] INFO [2019-12-17 12:34:51,812] jepsen worker 65 - jepsen.util 7355 :fail :read [21 nil] :RequestTimeoutException INFO [2019-12-17 12:34:51,812] jepsen worker 45 - jepsen.util 9225 :fail :read [15 nil] :RequestTimeoutException INFO [2019-12-17 12:34:51,813] jepsen worker 65 - jepsen.util 7355 :invoke :write [21 37] INFO [2019-12-17 12:34:51,813] jepsen worker 45 - jepsen.util 9225 :invoke :cas [15 [4 14]] INFO [2019-12-17 12:34:51,814] jepsen worker 37 - jepsen.util 8407 :invoke :read [12 nil] INFO [2019-12-17 12:34:51,825] jepsen worker 32 - jepsen.util 9122 :fail :cas [10 [20 39]] :ValueNotSwapFrom INFO [2019-12-17 12:34:51,825] jepsen worker 32 - jepsen.util 9122 :invoke :write [10 4] INFO [2019-12-17 12:34:51,830] jepsen worker 3 - jepsen.util 8733 :info :write [1 8] :RequestCanceledException INFO [2019-12-17 12:34:51,845] jepsen worker 55 - jepsen.util 9685 :info :write [18 49] :RequestCanceledException INFO [2019-12-17 12:34:51,846] jepsen worker 55 - jepsen.util 9775 :invoke :cas [18 [49 28]] INFO [2019-12-17 12:34:51,849] jepsen worker 3 - jepsen.util 8823 :invoke :read [1 nil] INFO [2019-12-17 12:34:51,853] jepsen worker 18 - jepsen.util 9198 :ok :write [6 18] INFO [2019-12-17 12:34:51,854] jepsen worker 18 - jepsen.util 9198 :invoke :read [6 nil] INFO [2019-12-17 12:34:51,873] jepsen worker 10 - jepsen.util 9010 :fail :cas [3 [27 36]] :ValueNotSwapFrom INFO [2019-12-17 12:34:51,874] jepsen worker 10 - jepsen.util 9010 :invoke :read [3 nil] INFO [2019-12-17 12:34:51,875] jepsen worker 64 - jepsen.util 7084 :fail :cas [21 [24 10]] :ValueNotSwapFrom INFO [2019-12-17 12:34:51,875] jepsen worker 64 - jepsen.util 7084 :invoke :read [21 nil] INFO [2019-12-17 12:34:51,885] jepsen worker 27 - jepsen.util 8667 :ok :read [9 36] INFO [2019-12-17 12:34:51,885] jepsen worker 27 - jepsen.util 8667 :invoke :read [9 nil] INFO [2019-12-17 12:34:51,885] jepsen worker 11 - jepsen.util 8921 :ok :write [3 37] INFO [2019-12-17 12:34:51,885] jepsen worker 11 - jepsen.util 8921 :invoke :cas [3 [23 40]] INFO [2019-12-17 12:34:51,893] jepsen worker 88 - jepsen.util 9088 :ok :read [29 35] INFO [2019-12-17 12:34:51,893] jepsen worker 88 - jepsen.util 9088 :invoke :cas [29 [3 35]] INFO [2019-12-17 12:34:51,896] jepsen worker 74 - jepsen.util 8984 :ok :read [24 36] INFO [2019-12-17 12:34:51,896] jepsen worker 74 - jepsen.util 8984 :invoke :read [24 nil] INFO [2019-12-17 12:34:51,897] jepsen worker 20 - jepsen.util 8480 :info :write [6 24] :RequestCanceledException INFO [2019-12-17 12:34:51,898] jepsen worker 20 - jepsen.util 8570 :invoke :cas [6 [16 8]] INFO [2019-12-17 12:34:51,914] jepsen worker 33 - jepsen.util 12003 :info :write [11 40] :RequestCanceledException INFO [2019-12-17 12:34:51,921] jepsen worker 33 - jepsen.util 12093 :invoke :read [11 nil] INFO [2019-12-17 12:34:51,931] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51442,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:34:51,932] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51442,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:34:51,932] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51445,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:34:51,933] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51445,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:34:51,933] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51446,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:34:51,933] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51446,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:34:51,948] jepsen worker 18 - jepsen.util 9198 :ok :read [6 18] INFO [2019-12-17 12:34:51,948] jepsen worker 18 - jepsen.util 9198 :invoke :write [6 13] INFO [2019-12-17 12:34:51,960] jepsen worker 62 - jepsen.util 12302 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:34:51,960] jepsen worker 73 - jepsen.util 8623 :fail :read [24 nil] :RequestTimeoutException INFO [2019-12-17 12:34:51,960] jepsen worker 62 - jepsen.util 12302 :invoke :write [20 11] INFO [2019-12-17 12:34:51,961] jepsen worker 73 - jepsen.util 8623 :invoke :write [24 46] INFO [2019-12-17 12:34:51,961] jepsen worker 45 - jepsen.util 9225 :fail :cas [15 [4 14]] :ValueNotSwapFrom INFO [2019-12-17 12:34:51,961] jepsen worker 45 - jepsen.util 9225 :invoke :write [15 0] INFO [2019-12-17 12:34:51,997] jepsen worker 72 - jepsen.util 8892 :fail :read [24 nil] :RequestTimeoutException INFO [2019-12-17 12:34:51,997] jepsen worker 63 - jepsen.util 7713 :info :write [21 4] :RequestCanceledException INFO [2019-12-17 12:34:51,997] jepsen worker 72 - jepsen.util 8892 :invoke :cas [24 [47 42]] INFO [2019-12-17 12:34:51,998] jepsen worker 63 - jepsen.util 7803 :invoke :read [21 nil] INFO [2019-12-17 12:34:52,003] jepsen worker 38 - jepsen.util 9308 :fail :cas [12 [23 37]] :ValueNotSwapFrom INFO [2019-12-17 12:34:52,003] jepsen worker 38 - jepsen.util 9308 :invoke :cas [12 [40 24]] INFO [2019-12-17 12:34:52,014] jepsen worker 0 - jepsen.util 12600 :info :write [0 24] :RequestCanceledException INFO [2019-12-17 12:34:52,014] jepsen worker 46 - jepsen.util 8866 :info :write [15 48] :RequestCanceledException INFO [2019-12-17 12:34:52,015] jepsen worker 46 - jepsen.util 8956 :invoke :read [15 nil] INFO [2019-12-17 12:34:52,019] jepsen worker 0 - jepsen.util 12690 :invoke :cas [0 [28 15]] INFO [2019-12-17 12:34:52,030] jepsen worker 25 - jepsen.util 12805 :info :cas [8 [13 16]] :RequestCanceledException INFO [2019-12-17 12:34:52,031] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51463,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:34:52,032] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51463,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:34:52,035] jepsen worker 25 - jepsen.util 12895 :invoke :read [8 nil] INFO [2019-12-17 12:34:52,038] jepsen worker 55 - jepsen.util 9775 :fail :cas [18 [49 28]] :ValueNotSwapFrom INFO [2019-12-17 12:34:52,038] jepsen worker 55 - jepsen.util 9775 :invoke :read [18 nil] INFO [2019-12-17 12:34:52,039] jepsen worker 64 - jepsen.util 7084 :ok :read [21 9] INFO [2019-12-17 12:34:52,040] jepsen worker 64 - jepsen.util 7084 :invoke :write [21 17] INFO [2019-12-17 12:34:52,044] jepsen worker 29 - jepsen.util 7409 :info :cas [9 [49 22]] :RequestCanceledException INFO [2019-12-17 12:34:52,046] jepsen worker 29 - jepsen.util 7499 :invoke :read [9 nil] INFO [2019-12-17 12:34:52,047] jepsen worker 5 - jepsen.util 8375 :fail :cas [1 [5 15]] :ValueNotSwapFrom INFO [2019-12-17 12:34:52,047] jepsen worker 5 - jepsen.util 8375 :invoke :read [1 nil] INFO [2019-12-17 12:34:52,068] jepsen worker 27 - jepsen.util 8667 :ok :read [9 36] INFO [2019-12-17 12:34:52,068] jepsen worker 27 - jepsen.util 8667 :invoke :read [9 nil] INFO [2019-12-17 12:34:52,101] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:34:52,101] jepsen nemesis - couchbase.util Rebalance check count: 397 INFO [2019-12-17 12:34:52,101] jepsen nemesis - couchbase.util Rebalance stuck count: 378 INFO [2019-12-17 12:34:52,129] jepsen worker 26 - jepsen.util 12626 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:34:52,129] jepsen worker 26 - jepsen.util 12626 :invoke :write [8 10] INFO [2019-12-17 12:34:52,130] jepsen worker 57 - jepsen.util 11847 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:34:52,130] jepsen worker 57 - jepsen.util 11847 :invoke :read [19 nil] INFO [2019-12-17 12:34:52,138] jepsen worker 20 - jepsen.util 8570 :fail :cas [6 [16 8]] :ValueNotSwapFrom INFO [2019-12-17 12:34:52,138] jepsen worker 20 - jepsen.util 8570 :invoke :write [6 4] INFO [2019-12-17 12:34:52,140] jepsen worker 65 - jepsen.util 7355 :ok :write [21 37] INFO [2019-12-17 12:34:52,140] jepsen worker 65 - jepsen.util 7355 :invoke :cas [21 [36 25]] INFO [2019-12-17 12:34:52,144] jepsen worker 70 - jepsen.util 9430 :info :write [23 35] :RequestCanceledException INFO [2019-12-17 12:34:52,146] jepsen worker 74 - jepsen.util 8984 :ok :read [24 36] INFO [2019-12-17 12:34:52,146] jepsen worker 74 - jepsen.util 8984 :invoke :read [24 nil] INFO [2019-12-17 12:34:52,148] jepsen worker 70 - jepsen.util 9520 :invoke :cas [23 [4 10]] INFO [2019-12-17 12:34:52,160] jepsen worker 78 - jepsen.util 8628 :info :write [26 9] :RequestCanceledException INFO [2019-12-17 12:34:52,162] jepsen worker 78 - jepsen.util 8718 :invoke :cas [26 [43 18]] INFO [2019-12-17 12:34:52,181] jepsen worker 36 - jepsen.util 9306 :ok :write [12 45] INFO [2019-12-17 12:34:52,182] jepsen worker 36 - jepsen.util 9306 :invoke :write [12 44] INFO [2019-12-17 12:34:52,182] jepsen worker 29 - jepsen.util 7499 :ok :read [9 36] INFO [2019-12-17 12:34:52,182] jepsen worker 29 - jepsen.util 7499 :invoke :read [9 nil] INFO [2019-12-17 12:34:52,185] jepsen worker 53 - jepsen.util 8153 :ok :write [17 18] INFO [2019-12-17 12:34:52,185] jepsen worker 53 - jepsen.util 8153 :invoke :read [17 nil] INFO [2019-12-17 12:34:52,198] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51486,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:34:52,199] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51486,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:34:52,199] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51488,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:34:52,199] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51488,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:34:52,202] jepsen worker 69 - jepsen.util 8979 :fail :cas [23 [2 7]] :ValueNotSwapFrom INFO [2019-12-17 12:34:52,203] jepsen worker 69 - jepsen.util 8979 :invoke :cas [23 [42 4]] INFO [2019-12-17 12:34:52,203] jepsen worker 54 - jepsen.util 9774 :ok :write [18 16] INFO [2019-12-17 12:34:52,203] jepsen worker 54 - jepsen.util 9774 :invoke :write [18 28] INFO [2019-12-17 12:34:52,230] jepsen worker 19 - jepsen.util 8839 :info :cas [6 [3 33]] :RequestCanceledException INFO [2019-12-17 12:34:52,230] jepsen worker 30 - jepsen.util 8850 :info :write [10 12] :RequestCanceledException INFO [2019-12-17 12:34:52,231] jepsen worker 19 - jepsen.util 8929 :invoke :write [6 14] INFO [2019-12-17 12:34:52,231] jepsen worker 30 - jepsen.util 8940 :invoke :cas [10 [2 2]] INFO [2019-12-17 12:34:52,250] jepsen worker 65 - jepsen.util 7355 :fail :cas [21 [36 25]] :ValueNotSwapFrom INFO [2019-12-17 12:34:52,250] jepsen worker 65 - jepsen.util 7355 :invoke :write [21 36] INFO [2019-12-17 12:34:52,261] jepsen worker 89 - jepsen.util 8909 :info :write [29 10] :RequestCanceledException INFO [2019-12-17 12:34:52,261] jepsen worker 4 - jepsen.util 8374 :info :write [1 4] :RequestCanceledException INFO [2019-12-17 12:34:52,262] jepsen worker 10 - jepsen.util 9010 :ok :read [3 37] INFO [2019-12-17 12:34:52,262] jepsen worker 10 - jepsen.util 9010 :invoke :write [3 4] INFO [2019-12-17 12:34:52,266] jepsen worker 4 - jepsen.util 8464 :invoke :write [1 42] INFO [2019-12-17 12:34:52,266] jepsen worker 89 - jepsen.util 8999 :invoke :read [29 nil] INFO [2019-12-17 12:34:52,273] jepsen worker 74 - jepsen.util 8984 :ok :read [24 36] INFO [2019-12-17 12:34:52,274] jepsen worker 74 - jepsen.util 8984 :invoke :write [24 8] INFO [2019-12-17 12:34:52,296] jepsen worker 58 - jepsen.util 12838 :info :write [19 30] :RequestCanceledException INFO [2019-12-17 12:34:52,300] jepsen worker 37 - jepsen.util 8407 :ok :read [12 45] INFO [2019-12-17 12:34:52,300] jepsen worker 37 - jepsen.util 8407 :invoke :cas [12 [44 11]] INFO [2019-12-17 12:34:52,303] jepsen worker 58 - jepsen.util 12928 :invoke :read [19 nil] INFO [2019-12-17 12:34:52,308] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51496,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:34:52,308] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51496,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:34:52,308] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51499,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:34:52,310] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51499,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:34:52,310] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51500,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:34:52,310] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51500,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:34:52,331] jepsen worker 85 - jepsen.util 12685 :info :write [28 44] :RequestCanceledException INFO [2019-12-17 12:34:52,331] jepsen worker 6 - jepsen.util 12516 :info :write [2 8] :RequestCanceledException INFO [2019-12-17 12:34:52,332] jepsen worker 44 - jepsen.util 12374 :info :cas [14 [49 1]] :RequestCanceledException INFO [2019-12-17 12:34:52,335] jepsen worker 85 - jepsen.util 12775 :invoke :cas [28 [13 13]] INFO [2019-12-17 12:34:52,335] jepsen worker 44 - jepsen.util 12464 :invoke :read [14 nil] INFO [2019-12-17 12:34:52,336] jepsen worker 6 - jepsen.util 12606 :invoke :read [2 nil] INFO [2019-12-17 12:34:52,344] jepsen worker 48 - jepsen.util 11118 :info :cas [16 [41 38]] :RequestCanceledException INFO [2019-12-17 12:34:52,344] jepsen worker 56 - jepsen.util 9596 :ok :read [18 16] INFO [2019-12-17 12:34:52,344] jepsen worker 56 - jepsen.util 9596 :invoke :cas [18 [29 41]] INFO [2019-12-17 12:34:52,344] jepsen worker 66 - jepsen.util 11856 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:34:52,345] jepsen worker 66 - jepsen.util 11856 :invoke :cas [22 [38 1]] INFO [2019-12-17 12:34:52,347] jepsen worker 48 - jepsen.util 11208 :invoke :read [16 nil] INFO [2019-12-17 12:34:52,355] jepsen worker 55 - jepsen.util 9775 :ok :read [18 16] INFO [2019-12-17 12:34:52,356] jepsen worker 55 - jepsen.util 9775 :invoke :cas [18 [7 39]] INFO [2019-12-17 12:34:52,361] jepsen worker 79 - jepsen.util 9259 :info :write [26 8] :RequestCanceledException INFO [2019-12-17 12:34:52,361] jepsen worker 61 - jepsen.util 12841 :info :cas [20 [22 27]] :RequestCanceledException INFO [2019-12-17 12:34:52,364] jepsen worker 87 - jepsen.util 8367 :fail :cas [29 [30 36]] :ValueNotSwapFrom INFO [2019-12-17 12:34:52,365] jepsen worker 87 - jepsen.util 8367 :invoke :cas [29 [8 30]] INFO [2019-12-17 12:34:52,366] jepsen worker 79 - jepsen.util 9349 :invoke :cas [26 [7 8]] INFO [2019-12-17 12:34:52,367] jepsen worker 65 - jepsen.util 7355 :ok :write [21 36] INFO [2019-12-17 12:34:52,367] jepsen worker 65 - jepsen.util 7355 :invoke :read [21 nil] INFO [2019-12-17 12:34:52,369] jepsen worker 61 - jepsen.util 12931 :invoke :write [20 49] INFO [2019-12-17 12:34:52,385] jepsen worker 69 - jepsen.util 8979 :fail :cas [23 [42 4]] :ValueNotSwapFrom INFO [2019-12-17 12:34:52,385] jepsen worker 69 - jepsen.util 8979 :invoke :read [23 nil] INFO [2019-12-17 12:34:52,413] jepsen worker 52 - jepsen.util 9232 :ok :write [17 6] INFO [2019-12-17 12:34:52,413] jepsen worker 52 - jepsen.util 9232 :invoke :write [17 47] INFO [2019-12-17 12:34:52,428] jepsen worker 80 - jepsen.util 8540 :ok :write [26 28] INFO [2019-12-17 12:34:52,428] jepsen worker 80 - jepsen.util 8540 :invoke :cas [26 [42 22]] INFO [2019-12-17 12:34:52,429] jepsen worker 22 - jepsen.util 11992 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:34:52,429] jepsen worker 22 - jepsen.util 11992 :invoke :read [7 nil] INFO [2019-12-17 12:34:52,457] jepsen worker 73 - jepsen.util 8623 :ok :write [24 46] INFO [2019-12-17 12:34:52,457] jepsen worker 73 - jepsen.util 8623 :invoke :read [24 nil] INFO [2019-12-17 12:34:52,477] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51528,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:34:52,477] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51528,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:34:52,477] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51530,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:34:52,478] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51530,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:34:52,479] jepsen worker 32 - jepsen.util 9122 :ok :write [10 4] INFO [2019-12-17 12:34:52,479] jepsen worker 32 - jepsen.util 9122 :invoke :cas [10 [43 30]] INFO [2019-12-17 12:34:52,490] jepsen worker 54 - jepsen.util 9774 :ok :write [18 28] INFO [2019-12-17 12:34:52,490] jepsen worker 54 - jepsen.util 9774 :invoke :read [18 nil] INFO [2019-12-17 12:34:52,511] jepsen worker 14 - jepsen.util 9014 :ok :write [4 12] INFO [2019-12-17 12:34:52,511] jepsen worker 14 - jepsen.util 9014 :invoke :read [4 nil] INFO [2019-12-17 12:34:52,515] jepsen worker 84 - jepsen.util 11424 :info :cas [28 [29 21]] :RequestCanceledException INFO [2019-12-17 12:34:52,520] jepsen worker 84 - jepsen.util 11514 :invoke :cas [28 [38 25]] INFO [2019-12-17 12:34:52,528] jepsen worker 63 - jepsen.util 7803 :ok :read [21 36] INFO [2019-12-17 12:34:52,528] jepsen worker 63 - jepsen.util 7803 :invoke :read [21 nil] INFO [2019-12-17 12:34:52,530] jepsen worker 17 - jepsen.util 12077 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:34:52,530] jepsen worker 17 - jepsen.util 12077 :invoke :write [5 43] INFO [2019-12-17 12:34:52,561] jepsen worker 24 - jepsen.util 11004 :info :write [8 45] :RequestCanceledException INFO [2019-12-17 12:34:52,572] jepsen worker 24 - jepsen.util 11094 :invoke :write [8 32] INFO [2019-12-17 12:34:52,595] jepsen worker 72 - jepsen.util 8892 :fail :cas [24 [47 42]] :ValueNotSwapFrom INFO [2019-12-17 12:34:52,595] jepsen worker 72 - jepsen.util 8892 :invoke :read [24 nil] INFO [2019-12-17 12:34:52,599] jepsen worker 73 - jepsen.util 8623 :ok :read [24 46] INFO [2019-12-17 12:34:52,599] jepsen worker 73 - jepsen.util 8623 :invoke :read [24 nil] INFO [2019-12-17 12:34:52,601] jepsen worker 30 - jepsen.util 8940 :fail :cas [10 [2 2]] :ValueNotSwapFrom INFO [2019-12-17 12:34:52,601] jepsen worker 30 - jepsen.util 8940 :invoke :write [10 34] INFO [2019-12-17 12:34:52,613] jepsen worker 32 - jepsen.util 9122 :fail :cas [10 [43 30]] :ValueNotSwapFrom INFO [2019-12-17 12:34:52,613] jepsen worker 32 - jepsen.util 9122 :invoke :write [10 36] INFO [2019-12-17 12:34:52,614] jepsen worker 3 - jepsen.util 8823 :ok :read [1 8] INFO [2019-12-17 12:34:52,614] jepsen worker 3 - jepsen.util 8823 :invoke :read [1 nil] INFO [2019-12-17 12:34:52,630] jepsen worker 42 - jepsen.util 13092 :info :write [14 30] :RequestCanceledException INFO [2019-12-17 12:34:52,634] jepsen worker 42 - jepsen.util 13182 :invoke :cas [14 [31 20]] INFO [2019-12-17 12:34:52,636] jepsen worker 63 - jepsen.util 7803 :ok :read [21 36] INFO [2019-12-17 12:34:52,636] jepsen worker 63 - jepsen.util 7803 :invoke :cas [21 [27 36]] INFO [2019-12-17 12:34:52,637] jepsen worker 51 - jepsen.util 8241 :ok :write [17 42] INFO [2019-12-17 12:34:52,638] jepsen worker 51 - jepsen.util 8241 :invoke :write [17 7] INFO [2019-12-17 12:34:52,643] jepsen worker 55 - jepsen.util 9775 :fail :cas [18 [7 39]] :ValueNotSwapFrom INFO [2019-12-17 12:34:52,643] jepsen worker 55 - jepsen.util 9775 :invoke :read [18 nil] INFO [2019-12-17 12:34:52,644] jepsen worker 40 - jepsen.util 12190 :info :write [13 16] :RequestCanceledException INFO [2019-12-17 12:34:52,654] jepsen worker 40 - jepsen.util 12280 :invoke :write [13 33] INFO [2019-12-17 12:34:52,661] jepsen worker 43 - jepsen.util 11833 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:34:52,661] jepsen worker 43 - jepsen.util 11833 :invoke :cas [14 [34 48]] INFO [2019-12-17 12:34:52,681] jepsen worker 18 - jepsen.util 9198 :ok :write [6 13] INFO [2019-12-17 12:34:52,681] jepsen worker 18 - jepsen.util 9198 :invoke :write [6 27] INFO [2019-12-17 12:34:52,683] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51548,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:34:52,683] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51548,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:34:52,683] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51551,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:34:52,683] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51551,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:34:52,683] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51552,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:34:52,684] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51552,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:34:52,693] jepsen worker 73 - jepsen.util 8623 :ok :read [24 46] INFO [2019-12-17 12:34:52,693] jepsen worker 73 - jepsen.util 8623 :invoke :read [24 nil] INFO [2019-12-17 12:34:52,696] jepsen worker 1 - jepsen.util 12241 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:34:52,696] jepsen worker 1 - jepsen.util 12241 :invoke :write [0 24] INFO [2019-12-17 12:34:52,714] jepsen worker 77 - jepsen.util 12497 :info :write [25 10] :RequestCanceledException INFO [2019-12-17 12:34:52,715] jepsen worker 72 - jepsen.util 8892 :ok :read [24 46] INFO [2019-12-17 12:34:52,715] jepsen worker 72 - jepsen.util 8892 :invoke :write [24 30] INFO [2019-12-17 12:34:52,719] jepsen worker 77 - jepsen.util 12587 :invoke :cas [25 [10 10]] INFO [2019-12-17 12:34:52,728] jepsen worker 54 - jepsen.util 9774 :ok :read [18 28] INFO [2019-12-17 12:34:52,728] jepsen worker 54 - jepsen.util 9774 :invoke :read [18 nil] INFO [2019-12-17 12:34:52,729] jepsen worker 15 - jepsen.util 11085 :info :cas [5 [27 33]] :RequestCanceledException INFO [2019-12-17 12:34:52,730] jepsen worker 75 - jepsen.util 11415 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:34:52,731] jepsen worker 75 - jepsen.util 11415 :invoke :cas [25 [29 29]] INFO [2019-12-17 12:34:52,739] jepsen worker 15 - jepsen.util 11175 :invoke :cas [5 [14 15]] INFO [2019-12-17 12:34:52,741] jepsen worker 88 - jepsen.util 9088 :fail :cas [29 [3 35]] :ValueNotSwapFrom INFO [2019-12-17 12:34:52,741] jepsen worker 88 - jepsen.util 9088 :invoke :cas [29 [46 9]] INFO [2019-12-17 12:34:52,769] jepsen worker 63 - jepsen.util 7803 :fail :cas [21 [27 36]] :ValueNotSwapFrom INFO [2019-12-17 12:34:52,769] jepsen worker 63 - jepsen.util 7803 :invoke :read [21 nil] INFO [2019-12-17 12:34:52,771] jepsen worker 13 - jepsen.util 8653 :ok :write [4 45] INFO [2019-12-17 12:34:52,772] jepsen worker 13 - jepsen.util 8653 :invoke :cas [4 [40 31]] INFO [2019-12-17 12:34:52,787] jepsen worker 73 - jepsen.util 8623 :ok :read [24 46] INFO [2019-12-17 12:34:52,787] jepsen worker 73 - jepsen.util 8623 :invoke :read [24 nil] INFO [2019-12-17 12:34:52,797] jepsen worker 8 - jepsen.util 11798 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:34:52,797] jepsen worker 8 - jepsen.util 11798 :invoke :read [2 nil] INFO [2019-12-17 12:34:52,806] jepsen worker 87 - jepsen.util 8367 :fail :cas [29 [8 30]] :ValueNotSwapFrom INFO [2019-12-17 12:34:52,806] jepsen worker 87 - jepsen.util 8367 :invoke :cas [29 [49 22]] INFO [2019-12-17 12:34:52,818] jepsen worker 70 - jepsen.util 9520 :fail :cas [23 [4 10]] :ValueNotSwapFrom INFO [2019-12-17 12:34:52,818] jepsen worker 70 - jepsen.util 9520 :invoke :write [23 13] INFO [2019-12-17 12:34:52,830] jepsen worker 7 - jepsen.util 12247 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:34:52,830] jepsen worker 7 - jepsen.util 12247 :invoke :read [2 nil] INFO [2019-12-17 12:34:52,857] jepsen worker 55 - jepsen.util 9775 :ok :read [18 28] INFO [2019-12-17 12:34:52,857] jepsen worker 55 - jepsen.util 9775 :invoke :read [18 nil] INFO [2019-12-17 12:34:52,868] jepsen worker 63 - jepsen.util 7803 :ok :read [21 36] INFO [2019-12-17 12:34:52,868] jepsen worker 63 - jepsen.util 7803 :invoke :read [21 nil] INFO [2019-12-17 12:34:52,869] jepsen worker 53 - jepsen.util 8153 :ok :read [17 42] INFO [2019-12-17 12:34:52,869] jepsen worker 53 - jepsen.util 8153 :invoke :cas [17 [25 37]] INFO [2019-12-17 12:34:52,898] jepsen worker 73 - jepsen.util 8623 :ok :read [24 46] INFO [2019-12-17 12:34:52,898] jepsen worker 73 - jepsen.util 8623 :invoke :cas [24 [45 47]] INFO [2019-12-17 12:34:52,899] jepsen worker 19 - jepsen.util 8929 :ok :write [6 14] INFO [2019-12-17 12:34:52,899] jepsen worker 19 - jepsen.util 8929 :invoke :cas [6 [7 4]] INFO [2019-12-17 12:34:52,904] jepsen worker 10 - jepsen.util 9010 :ok :write [3 4] INFO [2019-12-17 12:34:52,904] jepsen worker 38 - jepsen.util 9308 :fail :cas [12 [40 24]] :ValueNotSwapFrom INFO [2019-12-17 12:34:52,904] jepsen worker 10 - jepsen.util 9010 :invoke :read [3 nil] INFO [2019-12-17 12:34:52,904] jepsen worker 38 - jepsen.util 9308 :invoke :cas [12 [23 49]] INFO [2019-12-17 12:34:52,908] jepsen worker 54 - jepsen.util 9774 :ok :read [18 28] INFO [2019-12-17 12:34:52,909] jepsen worker 54 - jepsen.util 9774 :invoke :write [18 26] INFO [2019-12-17 12:34:52,915] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51575,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:34:52,915] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51575,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:34:52,937] jepsen worker 78 - jepsen.util 8718 :fail :cas [26 [43 18]] :ValueNotSwapFrom INFO [2019-12-17 12:34:52,937] jepsen worker 78 - jepsen.util 8718 :invoke :read [26 nil] INFO [2019-12-17 12:34:52,941] jepsen worker 71 - jepsen.util 9431 :ok :write [23 16] INFO [2019-12-17 12:34:52,942] jepsen worker 71 - jepsen.util 9431 :invoke :cas [23 [15 12]] INFO [2019-12-17 12:34:52,955] jepsen worker 30 - jepsen.util 8940 :ok :write [10 34] INFO [2019-12-17 12:34:52,955] jepsen worker 30 - jepsen.util 8940 :invoke :write [10 35] INFO [2019-12-17 12:34:52,991] jepsen worker 10 - jepsen.util 9010 :ok :read [3 4] INFO [2019-12-17 12:34:52,991] jepsen worker 10 - jepsen.util 9010 :invoke :read [3 nil] INFO [2019-12-17 12:34:52,996] jepsen worker 4 - jepsen.util 8464 :ok :write [1 42] INFO [2019-12-17 12:34:52,996] jepsen worker 4 - jepsen.util 8464 :invoke :read [1 nil] INFO [2019-12-17 12:34:53,003] jepsen worker 56 - jepsen.util 9596 :fail :cas [18 [29 41]] :ValueNotSwapFrom INFO [2019-12-17 12:34:53,003] jepsen worker 56 - jepsen.util 9596 :invoke :read [18 nil] INFO [2019-12-17 12:34:53,019] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51591,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:34:53,019] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51591,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:34:53,019] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51595,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:34:53,021] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51595,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:34:53,021] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51596,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:34:53,021] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51596,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:34:53,021] jepsen worker 63 - jepsen.util 7803 :ok :read [21 36] INFO [2019-12-17 12:34:53,021] jepsen worker 63 - jepsen.util 7803 :invoke :cas [21 [38 21]] INFO [2019-12-17 12:34:53,030] jepsen worker 49 - jepsen.util 11389 :info :write [16 33] :RequestCanceledException INFO [2019-12-17 12:34:53,039] jepsen worker 49 - jepsen.util 11479 :invoke :cas [16 [6 8]] INFO [2019-12-17 12:34:53,045] jepsen worker 5 - jepsen.util 8375 :ok :read [1 8] INFO [2019-12-17 12:34:53,045] jepsen worker 5 - jepsen.util 8375 :invoke :read [1 nil] INFO [2019-12-17 12:34:53,062] jepsen worker 73 - jepsen.util 8623 :fail :cas [24 [45 47]] :ValueNotSwapFrom INFO [2019-12-17 12:34:53,063] jepsen worker 73 - jepsen.util 8623 :invoke :write [24 4] INFO [2019-12-17 12:34:53,073] jepsen worker 55 - jepsen.util 9775 :ok :read [18 28] INFO [2019-12-17 12:34:53,074] jepsen worker 55 - jepsen.util 9775 :invoke :read [18 nil] INFO [2019-12-17 12:34:53,076] jepsen worker 10 - jepsen.util 9010 :ok :read [3 4] INFO [2019-12-17 12:34:53,076] jepsen worker 10 - jepsen.util 9010 :invoke :write [3 9] INFO [2019-12-17 12:34:53,091] jepsen worker 37 - jepsen.util 8407 :fail :cas [12 [44 11]] :ValueNotSwapFrom INFO [2019-12-17 12:34:53,091] jepsen worker 37 - jepsen.util 8407 :invoke :write [12 36] INFO [2019-12-17 12:34:53,093] jepsen worker 32 - jepsen.util 9122 :ok :write [10 36] INFO [2019-12-17 12:34:53,093] jepsen worker 32 - jepsen.util 9122 :invoke :read [10 nil] INFO [2019-12-17 12:34:53,106] jepsen worker 80 - jepsen.util 8540 :fail :cas [26 [42 22]] :ValueNotSwapFrom INFO [2019-12-17 12:34:53,106] jepsen worker 80 - jepsen.util 8540 :invoke :write [26 38] INFO [2019-12-17 12:34:53,123] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:34:53,123] jepsen nemesis - couchbase.util Rebalance check count: 398 INFO [2019-12-17 12:34:53,123] jepsen nemesis - couchbase.util Rebalance stuck count: 379 INFO [2019-12-17 12:34:53,130] jepsen worker 2 - jepsen.util 11702 :info :cas [0 [11 26]] :RequestCanceledException INFO [2019-12-17 12:34:53,130] jepsen worker 63 - jepsen.util 7803 :fail :cas [21 [38 21]] :ValueNotSwapFrom INFO [2019-12-17 12:34:53,130] jepsen worker 63 - jepsen.util 7803 :invoke :cas [21 [36 9]] INFO [2019-12-17 12:34:53,134] jepsen worker 18 - jepsen.util 9198 :ok :write [6 27] INFO [2019-12-17 12:34:53,134] jepsen worker 18 - jepsen.util 9198 :invoke :write [6 19] INFO [2019-12-17 12:34:53,135] jepsen worker 2 - jepsen.util 11792 :invoke :cas [0 [19 2]] INFO [2019-12-17 12:34:53,143] jepsen worker 86 - jepsen.util 12866 :info :cas [28 [45 37]] :RequestCanceledException INFO [2019-12-17 12:34:53,147] jepsen worker 86 - jepsen.util 12956 :invoke :read [28 nil] INFO [2019-12-17 12:34:53,149] jepsen worker 88 - jepsen.util 9088 :fail :cas [29 [46 9]] :ValueNotSwapFrom INFO [2019-12-17 12:34:53,149] jepsen worker 88 - jepsen.util 9088 :invoke :read [29 nil] INFO [2019-12-17 12:34:53,161] jepsen worker 39 - jepsen.util 11559 :info :write [13 39] :RequestCanceledException INFO [2019-12-17 12:34:53,166] jepsen worker 39 - jepsen.util 11649 :invoke :cas [13 [3 10]] INFO [2019-12-17 12:34:53,180] jepsen worker 69 - jepsen.util 8979 :ok :read [23 16] INFO [2019-12-17 12:34:53,180] jepsen worker 69 - jepsen.util 8979 :invoke :write [23 36] INFO [2019-12-17 12:34:53,196] jepsen worker 41 - jepsen.util 12821 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:34:53,196] jepsen worker 82 - jepsen.util 12772 :info :cas [27 [47 2]] :RequestCanceledException INFO [2019-12-17 12:34:53,196] jepsen worker 41 - jepsen.util 12821 :invoke :cas [13 [27 33]] INFO [2019-12-17 12:34:53,207] jepsen worker 82 - jepsen.util 12862 :invoke :read [27 nil] INFO [2019-12-17 12:34:53,229] jepsen worker 81 - jepsen.util 11871 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:34:53,229] jepsen worker 81 - jepsen.util 11871 :invoke :write [27 23] INFO [2019-12-17 12:34:53,245] jepsen worker 87 - jepsen.util 8367 :fail :cas [29 [49 22]] :ValueNotSwapFrom INFO [2019-12-17 12:34:53,245] jepsen worker 87 - jepsen.util 8367 :invoke :write [29 12] INFO [2019-12-17 12:34:53,251] jepsen worker 13 - jepsen.util 8653 :fail :cas [4 [40 31]] :ValueNotSwapFrom INFO [2019-12-17 12:34:53,252] jepsen worker 13 - jepsen.util 8653 :invoke :read [4 nil] INFO [2019-12-17 12:34:53,283] jepsen worker 30 - jepsen.util 8940 :ok :write [10 35] INFO [2019-12-17 12:34:53,283] jepsen worker 30 - jepsen.util 8940 :invoke :read [10 nil] INFO [2019-12-17 12:34:53,285] jepsen worker 10 - jepsen.util 9010 :ok :write [3 9] INFO [2019-12-17 12:34:53,286] jepsen worker 10 - jepsen.util 9010 :invoke :cas [3 [34 24]] INFO [2019-12-17 12:34:53,295] jepsen worker 5 - jepsen.util 8375 :ok :read [1 42] INFO [2019-12-17 12:34:53,295] jepsen worker 5 - jepsen.util 8375 :invoke :read [1 nil] INFO [2019-12-17 12:34:53,297] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51625,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:34:53,297] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51625,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:34:53,303] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51615,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:34:53,304] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51615,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:34:53,304] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51616,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:34:53,305] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51616,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:34:53,314] jepsen worker 34 - jepsen.util 12274 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:34:53,314] jepsen worker 34 - jepsen.util 12274 :invoke :read [11 nil] INFO [2019-12-17 12:34:53,314] jepsen worker 3 - jepsen.util 8823 :ok :read [1 42] INFO [2019-12-17 12:34:53,314] jepsen worker 3 - jepsen.util 8823 :invoke :write [1 25] INFO [2019-12-17 12:34:53,322] jepsen worker 89 - jepsen.util 8999 :ok :read [29 35] INFO [2019-12-17 12:34:53,323] jepsen worker 89 - jepsen.util 8999 :invoke :cas [29 [2 20]] INFO [2019-12-17 12:34:53,330] jepsen worker 23 - jepsen.util 11813 :info :write [7 25] :RequestCanceledException INFO [2019-12-17 12:34:53,331] jepsen worker 21 - jepsen.util 11721 :info :write [7 16] :RequestCanceledException INFO [2019-12-17 12:34:53,336] jepsen worker 23 - jepsen.util 11903 :invoke :read [7 nil] INFO [2019-12-17 12:34:53,338] jepsen worker 21 - jepsen.util 11811 :invoke :cas [7 [16 8]] INFO [2019-12-17 12:34:53,343] jepsen worker 76 - jepsen.util 12406 :info :write [25 35] :RequestCanceledException INFO [2019-12-17 12:34:53,347] jepsen worker 76 - jepsen.util 12496 :invoke :write [25 3] INFO [2019-12-17 12:34:53,373] jepsen worker 30 - jepsen.util 8940 :ok :read [10 35] INFO [2019-12-17 12:34:53,374] jepsen worker 30 - jepsen.util 8940 :invoke :write [10 9] INFO [2019-12-17 12:34:53,377] jepsen worker 10 - jepsen.util 9010 :fail :cas [3 [34 24]] :ValueNotSwapFrom INFO [2019-12-17 12:34:53,377] jepsen worker 10 - jepsen.util 9010 :invoke :write [3 43] INFO [2019-12-17 12:34:53,378] jepsen worker 55 - jepsen.util 9775 :ok :read [18 28] INFO [2019-12-17 12:34:53,379] jepsen worker 55 - jepsen.util 9775 :invoke :read [18 nil] INFO [2019-12-17 12:34:53,379] jepsen worker 14 - jepsen.util 9014 :ok :read [4 45] INFO [2019-12-17 12:34:53,379] jepsen worker 14 - jepsen.util 9014 :invoke :read [4 nil] INFO [2019-12-17 12:34:53,396] jepsen worker 59 - jepsen.util 11399 :info :cas [19 [37 6]] :RequestCanceledException INFO [2019-12-17 12:34:53,402] jepsen worker 59 - jepsen.util 11489 :invoke :cas [19 [0 27]] INFO [2019-12-17 12:34:53,406] jepsen worker 79 - jepsen.util 9349 :fail :cas [26 [7 8]] :ValueNotSwapFrom INFO [2019-12-17 12:34:53,406] jepsen worker 79 - jepsen.util 9349 :invoke :read [26 nil] INFO [2019-12-17 12:34:53,410] jepsen worker 53 - jepsen.util 8153 :fail :cas [17 [25 37]] :ValueNotSwapFrom INFO [2019-12-17 12:34:53,410] jepsen worker 53 - jepsen.util 8153 :invoke :cas [17 [26 24]] INFO [2019-12-17 12:34:53,413] jepsen worker 31 - jepsen.util 7951 :info :cas [10 [28 39]] :RequestCanceledException INFO [2019-12-17 12:34:53,414] jepsen worker 31 - jepsen.util 8041 :invoke :cas [10 [31 36]] INFO [2019-12-17 12:34:53,430] jepsen worker 68 - jepsen.util 11408 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:34:53,430] jepsen worker 68 - jepsen.util 11408 :invoke :cas [22 [45 13]] INFO [2019-12-17 12:34:53,442] jepsen worker 71 - jepsen.util 9431 :fail :cas [23 [15 12]] :ValueNotSwapFrom INFO [2019-12-17 12:34:53,442] jepsen worker 71 - jepsen.util 9431 :invoke :write [23 30] INFO [2019-12-17 12:34:53,463] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51652,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:34:53,463] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51652,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:34:53,466] jepsen worker 4 - jepsen.util 8464 :ok :read [1 42] INFO [2019-12-17 12:34:53,466] jepsen worker 4 - jepsen.util 8464 :invoke :write [1 36] INFO [2019-12-17 12:34:53,496] jepsen worker 16 - jepsen.util 13156 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:34:53,496] jepsen worker 16 - jepsen.util 13156 :invoke :read [5 nil] INFO [2019-12-17 12:34:53,505] jepsen worker 88 - jepsen.util 9088 :ok :read [29 35] INFO [2019-12-17 12:34:53,505] jepsen worker 88 - jepsen.util 9088 :invoke :write [29 44] INFO [2019-12-17 12:34:53,530] jepsen worker 50 - jepsen.util 12650 :info :cas [16 [11 22]] :RequestCanceledException INFO [2019-12-17 12:34:53,534] jepsen worker 50 - jepsen.util 12740 :invoke :cas [16 [40 24]] INFO [2019-12-17 12:34:53,543] jepsen worker 83 - jepsen.util 12593 :info :write [27 14] :RequestCanceledException INFO [2019-12-17 12:34:53,557] jepsen worker 83 - jepsen.util 12683 :invoke :cas [27 [44 23]] INFO [2019-12-17 12:34:53,560] jepsen worker 47 - jepsen.util 9317 :info :cas [15 [33 14]] :RequestCanceledException INFO [2019-12-17 12:34:53,560] jepsen worker 28 - jepsen.util 7228 :fail :read [9 nil] :RequestTimeoutException INFO [2019-12-17 12:34:53,561] jepsen worker 28 - jepsen.util 7228 :invoke :cas [9 [4 4]] INFO [2019-12-17 12:34:53,567] jepsen worker 47 - jepsen.util 9407 :invoke :cas [15 [34 21]] INFO [2019-12-17 12:34:53,582] jepsen worker 89 - jepsen.util 8999 :fail :cas [29 [2 20]] :ValueNotSwapFrom INFO [2019-12-17 12:34:53,582] jepsen worker 89 - jepsen.util 8999 :invoke :write [29 11] INFO [2019-12-17 12:34:53,604] jepsen worker 38 - jepsen.util 9308 :fail :cas [12 [23 49]] :ValueNotSwapFrom INFO [2019-12-17 12:34:53,604] jepsen worker 38 - jepsen.util 9308 :invoke :write [12 11] INFO [2019-12-17 12:34:53,622] jepsen worker 79 - jepsen.util 9349 :ok :read [26 8] INFO [2019-12-17 12:34:53,622] jepsen worker 79 - jepsen.util 9349 :invoke :read [26 nil] INFO [2019-12-17 12:34:53,629] jepsen worker 67 - jepsen.util 12127 :info :cas [22 [43 21]] :RequestCanceledException INFO [2019-12-17 12:34:53,630] jepsen worker 9 - jepsen.util 9369 :info :write [3 14] :RequestCanceledException INFO [2019-12-17 12:34:53,632] jepsen worker 9 - jepsen.util 9459 :invoke :write [3 0] INFO [2019-12-17 12:34:53,635] jepsen worker 67 - jepsen.util 12217 :invoke :write [22 16] INFO [2019-12-17 12:34:53,644] jepsen worker 60 - jepsen.util 13110 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:34:53,644] jepsen worker 60 - jepsen.util 13110 :invoke :write [20 24] INFO [2019-12-17 12:34:53,675] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51667,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:34:53,675] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51667,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:34:53,675] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51670,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:34:53,676] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51670,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:34:53,685] jepsen worker 78 - jepsen.util 8718 :ok :read [26 8] INFO [2019-12-17 12:34:53,685] jepsen worker 78 - jepsen.util 8718 :invoke :cas [26 [5 22]] INFO [2019-12-17 12:34:53,690] jepsen worker 56 - jepsen.util 9596 :ok :read [18 28] INFO [2019-12-17 12:34:53,690] jepsen worker 56 - jepsen.util 9596 :invoke :cas [18 [46 39]] INFO [2019-12-17 12:34:53,700] jepsen worker 13 - jepsen.util 8653 :ok :read [4 45] INFO [2019-12-17 12:34:53,700] jepsen worker 13 - jepsen.util 8653 :invoke :read [4 nil] INFO [2019-12-17 12:34:53,715] jepsen worker 35 - jepsen.util 12005 :info :write [11 30] :RequestCanceledException INFO [2019-12-17 12:34:53,718] jepsen worker 55 - jepsen.util 9775 :ok :read [18 28] INFO [2019-12-17 12:34:53,719] jepsen worker 55 - jepsen.util 9775 :invoke :write [18 37] INFO [2019-12-17 12:34:53,722] jepsen worker 35 - jepsen.util 12095 :invoke :write [11 46] INFO [2019-12-17 12:34:53,723] jepsen worker 5 - jepsen.util 8375 :ok :read [1 42] INFO [2019-12-17 12:34:53,724] jepsen worker 5 - jepsen.util 8375 :invoke :write [1 34] INFO [2019-12-17 12:34:53,862] jepsen worker 12 - jepsen.util 9462 :info :write [4 23] :RequestCanceledException INFO [2019-12-17 12:34:53,867] jepsen worker 12 - jepsen.util 9552 :invoke :cas [4 [28 41]] INFO [2019-12-17 12:34:53,877] jepsen worker 79 - jepsen.util 9349 :ok :read [26 8] INFO [2019-12-17 12:34:53,877] jepsen worker 79 - jepsen.util 9349 :invoke :write [26 36] INFO [2019-12-17 12:34:53,899] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51682,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:34:53,899] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51682,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:34:53,899] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51686,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:34:53,899] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51686,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:34:53,913] jepsen worker 11 - jepsen.util 8921 :info :cas [3 [23 40]] :RequestCanceledException INFO [2019-12-17 12:34:53,923] jepsen worker 11 - jepsen.util 9011 :invoke :read [3 nil] INFO [2019-12-17 12:34:53,946] jepsen worker 33 - jepsen.util 12093 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:34:53,946] jepsen worker 33 - jepsen.util 12093 :invoke :write [11 12] INFO [2019-12-17 12:34:53,980] jepsen worker 11 - jepsen.util 9011 :ok :read [3 9] INFO [2019-12-17 12:34:53,981] jepsen worker 11 - jepsen.util 9011 :invoke :write [3 26] INFO [2019-12-17 12:34:54,009] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51693,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:34:54,010] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51693,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:34:54,010] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51696,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:34:54,011] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51696,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:34:54,011] jepsen worker 45 - jepsen.util 9225 :info :write [15 0] :RequestCanceledException INFO [2019-12-17 12:34:54,015] jepsen worker 45 - jepsen.util 9315 :invoke :cas [15 [38 15]] INFO [2019-12-17 12:34:54,020] jepsen worker 78 - jepsen.util 8718 :fail :cas [26 [5 22]] :ValueNotSwapFrom INFO [2019-12-17 12:34:54,021] jepsen worker 78 - jepsen.util 8718 :invoke :cas [26 [23 31]] INFO [2019-12-17 12:34:54,028] jepsen worker 46 - jepsen.util 8956 :fail :read [15 nil] :RequestTimeoutException INFO [2019-12-17 12:34:54,028] jepsen worker 46 - jepsen.util 8956 :invoke :read [15 nil] INFO [2019-12-17 12:34:54,060] jepsen worker 62 - jepsen.util 12302 :info :write [20 11] :RequestCanceledException INFO [2019-12-17 12:34:54,064] jepsen worker 62 - jepsen.util 12392 :invoke :read [20 nil] INFO [2019-12-17 12:34:54,070] jepsen worker 56 - jepsen.util 9596 :fail :cas [18 [46 39]] :ValueNotSwapFrom INFO [2019-12-17 12:34:54,070] jepsen worker 56 - jepsen.util 9596 :invoke :read [18 nil] INFO [2019-12-17 12:34:54,086] jepsen worker 78 - jepsen.util 8718 :fail :cas [26 [23 31]] :ValueNotSwapFrom INFO [2019-12-17 12:34:54,086] jepsen worker 78 - jepsen.util 8718 :invoke :write [26 13] INFO [2019-12-17 12:34:54,096] jepsen worker 0 - jepsen.util 12690 :info :cas [0 [28 15]] :RequestCanceledException INFO [2019-12-17 12:34:54,098] jepsen worker 0 - jepsen.util 12780 :invoke :write [0 13] INFO [2019-12-17 12:34:54,111] jepsen worker 25 - jepsen.util 12895 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:34:54,112] jepsen worker 25 - jepsen.util 12895 :invoke :cas [8 [40 12]] INFO [2019-12-17 12:34:54,129] jepsen worker 13 - jepsen.util 8653 :ok :read [4 45] INFO [2019-12-17 12:34:54,130] jepsen worker 13 - jepsen.util 8653 :invoke :write [4 44] INFO [2019-12-17 12:34:54,139] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:34:54,140] jepsen nemesis - couchbase.util Rebalance check count: 399 INFO [2019-12-17 12:34:54,140] jepsen nemesis - couchbase.util Rebalance stuck count: 380 INFO [2019-12-17 12:34:54,143] jepsen worker 64 - jepsen.util 7084 :info :write [21 17] :RequestCanceledException INFO [2019-12-17 12:34:54,148] jepsen worker 64 - jepsen.util 7174 :invoke :cas [21 [7 10]] INFO [2019-12-17 12:34:54,161] jepsen worker 27 - jepsen.util 8667 :fail :read [9 nil] :RequestTimeoutException INFO [2019-12-17 12:34:54,162] jepsen worker 27 - jepsen.util 8667 :invoke :write [9 13] INFO [2019-12-17 12:34:54,197] jepsen worker 56 - jepsen.util 9596 :ok :read [18 28] INFO [2019-12-17 12:34:54,197] jepsen worker 56 - jepsen.util 9596 :invoke :cas [18 [23 16]] INFO [2019-12-17 12:34:54,229] jepsen worker 26 - jepsen.util 12626 :info :write [8 10] :RequestCanceledException INFO [2019-12-17 12:34:54,231] jepsen worker 57 - jepsen.util 11847 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:34:54,231] jepsen worker 57 - jepsen.util 11847 :invoke :write [19 31] INFO [2019-12-17 12:34:54,232] jepsen worker 20 - jepsen.util 8570 :info :write [6 4] :RequestCanceledException INFO [2019-12-17 12:34:54,232] jepsen worker 29 - jepsen.util 7499 :fail :read [9 nil] :RequestTimeoutException INFO [2019-12-17 12:34:54,232] jepsen worker 29 - jepsen.util 7499 :invoke :cas [9 [21 2]] INFO [2019-12-17 12:34:54,237] jepsen worker 20 - jepsen.util 8660 :invoke :read [6 nil] INFO [2019-12-17 12:34:54,237] jepsen worker 26 - jepsen.util 12716 :invoke :cas [8 [16 49]] INFO [2019-12-17 12:34:54,244] jepsen worker 36 - jepsen.util 9306 :info :write [12 44] :RequestCanceledException INFO [2019-12-17 12:34:54,250] jepsen worker 36 - jepsen.util 9396 :invoke :read [12 nil] INFO [2019-12-17 12:34:54,283] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51716,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:34:54,285] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51716,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:34:54,286] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51720,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:34:54,286] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51720,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:34:54,288] jepsen worker 53 - jepsen.util 8153 :fail :cas [17 [26 24]] :ValueNotSwapFrom INFO [2019-12-17 12:34:54,289] jepsen worker 53 - jepsen.util 8153 :invoke :write [17 16] INFO [2019-12-17 12:34:54,296] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51709,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:34:54,296] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51709,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:34:54,297] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51710,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:34:54,297] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51710,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:34:54,297] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51711,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:34:54,297] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51711,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:34:54,314] jepsen worker 20 - jepsen.util 8660 :ok :read [6 27] INFO [2019-12-17 12:34:54,314] jepsen worker 20 - jepsen.util 8660 :invoke :read [6 nil] INFO [2019-12-17 12:34:54,319] jepsen worker 12 - jepsen.util 9552 :fail :cas [4 [28 41]] :ValueNotSwapFrom INFO [2019-12-17 12:34:54,319] jepsen worker 12 - jepsen.util 9552 :invoke :read [4 nil] INFO [2019-12-17 12:34:54,322] jepsen worker 29 - jepsen.util 7499 :fail :cas [9 [21 2]] :ValueNotSwapFrom INFO [2019-12-17 12:34:54,322] jepsen worker 29 - jepsen.util 7499 :invoke :write [9 46] INFO [2019-12-17 12:34:54,330] jepsen worker 74 - jepsen.util 8984 :info :write [24 8] :RequestCanceledException INFO [2019-12-17 12:34:54,333] jepsen worker 74 - jepsen.util 9074 :invoke :write [24 11] INFO [2019-12-17 12:34:54,344] jepsen worker 58 - jepsen.util 12928 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:34:54,344] jepsen worker 58 - jepsen.util 12928 :invoke :read [19 nil] INFO [2019-12-17 12:34:54,361] jepsen worker 85 - jepsen.util 12775 :info :cas [28 [13 13]] :RequestCanceledException INFO [2019-12-17 12:34:54,366] jepsen worker 85 - jepsen.util 12865 :invoke :read [28 nil] INFO [2019-12-17 12:34:54,373] jepsen worker 73 - jepsen.util 8623 :ok :write [24 4] INFO [2019-12-17 12:34:54,373] jepsen worker 73 - jepsen.util 8623 :invoke :read [24 nil] INFO [2019-12-17 12:34:54,373] jepsen worker 30 - jepsen.util 8940 :ok :write [10 9] INFO [2019-12-17 12:34:54,373] jepsen worker 30 - jepsen.util 8940 :invoke :cas [10 [33 4]] INFO [2019-12-17 12:34:54,379] jepsen worker 56 - jepsen.util 9596 :fail :cas [18 [23 16]] :ValueNotSwapFrom INFO [2019-12-17 12:34:54,380] jepsen worker 56 - jepsen.util 9596 :invoke :write [18 45] INFO [2019-12-17 12:34:54,384] jepsen worker 20 - jepsen.util 8660 :ok :read [6 27] INFO [2019-12-17 12:34:54,385] jepsen worker 20 - jepsen.util 8660 :invoke :cas [6 [31 22]] INFO [2019-12-17 12:34:54,390] jepsen worker 14 - jepsen.util 9014 :ok :read [4 45] INFO [2019-12-17 12:34:54,390] jepsen worker 14 - jepsen.util 9014 :invoke :read [4 nil] INFO [2019-12-17 12:34:54,396] jepsen worker 6 - jepsen.util 12606 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:34:54,396] jepsen worker 6 - jepsen.util 12606 :invoke :cas [2 [35 28]] INFO [2019-12-17 12:34:54,415] jepsen worker 48 - jepsen.util 11208 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:34:54,415] jepsen worker 65 - jepsen.util 7355 :fail :read [21 nil] :RequestTimeoutException INFO [2019-12-17 12:34:54,415] jepsen worker 48 - jepsen.util 11208 :invoke :write [16 10] INFO [2019-12-17 12:34:54,415] jepsen worker 65 - jepsen.util 7355 :invoke :write [21 44] INFO [2019-12-17 12:34:54,423] jepsen worker 18 - jepsen.util 9198 :ok :write [6 19] INFO [2019-12-17 12:34:54,423] jepsen worker 18 - jepsen.util 9198 :invoke :write [6 3] INFO [2019-12-17 12:34:54,425] jepsen worker 10 - jepsen.util 9010 :ok :write [3 43] INFO [2019-12-17 12:34:54,425] jepsen worker 10 - jepsen.util 9010 :invoke :cas [3 [26 34]] INFO [2019-12-17 12:34:54,427] jepsen worker 44 - jepsen.util 12464 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:34:54,428] jepsen worker 44 - jepsen.util 12464 :invoke :cas [14 [39 19]] INFO [2019-12-17 12:34:54,443] jepsen worker 66 - jepsen.util 11856 :info :cas [22 [38 1]] :RequestCanceledException INFO [2019-12-17 12:34:54,443] jepsen worker 61 - jepsen.util 12931 :info :write [20 49] :RequestCanceledException INFO [2019-12-17 12:34:54,447] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51743,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:34:54,448] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51743,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:34:54,448] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51747,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:34:54,448] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51747,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:34:54,451] jepsen worker 66 - jepsen.util 11946 :invoke :read [22 nil] INFO [2019-12-17 12:34:54,451] jepsen worker 61 - jepsen.util 13021 :invoke :write [20 36] INFO [2019-12-17 12:34:54,490] jepsen worker 28 - jepsen.util 7228 :fail :cas [9 [4 4]] :ValueNotSwapFrom INFO [2019-12-17 12:34:54,490] jepsen worker 28 - jepsen.util 7228 :invoke :write [9 42] INFO [2019-12-17 12:34:54,515] jepsen worker 30 - jepsen.util 8940 :fail :cas [10 [33 4]] :ValueNotSwapFrom INFO [2019-12-17 12:34:54,515] jepsen worker 30 - jepsen.util 8940 :invoke :read [10 nil] INFO [2019-12-17 12:34:54,529] jepsen worker 22 - jepsen.util 11992 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:34:54,529] jepsen worker 22 - jepsen.util 11992 :invoke :write [7 33] INFO [2019-12-17 12:34:54,530] jepsen worker 52 - jepsen.util 9232 :info :write [17 47] :RequestCanceledException INFO [2019-12-17 12:34:54,535] jepsen worker 52 - jepsen.util 9322 :invoke :cas [17 [46 37]] INFO [2019-12-17 12:34:54,538] jepsen worker 36 - jepsen.util 9396 :ok :read [12 45] INFO [2019-12-17 12:34:54,538] jepsen worker 36 - jepsen.util 9396 :invoke :read [12 nil] INFO [2019-12-17 12:34:54,560] jepsen worker 47 - jepsen.util 9407 :fail :cas [15 [34 21]] :ValueNotSwapFrom INFO [2019-12-17 12:34:54,561] jepsen worker 84 - jepsen.util 11514 :info :cas [28 [38 25]] :RequestCanceledException INFO [2019-12-17 12:34:54,563] jepsen worker 47 - jepsen.util 9407 :invoke :cas [15 [33 27]] INFO [2019-12-17 12:34:54,566] jepsen worker 29 - jepsen.util 7499 :ok :write [9 46] INFO [2019-12-17 12:34:54,567] jepsen worker 29 - jepsen.util 7499 :invoke :read [9 nil] INFO [2019-12-17 12:34:54,571] jepsen worker 84 - jepsen.util 11604 :invoke :write [28 10] INFO [2019-12-17 12:34:54,592] jepsen worker 31 - jepsen.util 8041 :fail :cas [10 [31 36]] :ValueNotSwapFrom INFO [2019-12-17 12:34:54,592] jepsen worker 31 - jepsen.util 8041 :invoke :cas [10 [7 38]] INFO [2019-12-17 12:34:54,605] jepsen worker 45 - jepsen.util 9315 :fail :cas [15 [38 15]] :ValueNotSwapFrom INFO [2019-12-17 12:34:54,605] jepsen worker 45 - jepsen.util 9315 :invoke :read [15 nil] INFO [2019-12-17 12:34:54,630] jepsen worker 24 - jepsen.util 11094 :info :write [8 32] :RequestCanceledException INFO [2019-12-17 12:34:54,630] jepsen worker 17 - jepsen.util 12077 :info :write [5 43] :RequestCanceledException INFO [2019-12-17 12:34:54,632] jepsen worker 17 - jepsen.util 12167 :invoke :read [5 nil] INFO [2019-12-17 12:34:54,634] jepsen worker 24 - jepsen.util 11184 :invoke :write [8 20] INFO [2019-12-17 12:34:54,654] jepsen worker 64 - jepsen.util 7174 :fail :cas [21 [7 10]] :ValueNotSwapFrom INFO [2019-12-17 12:34:54,654] jepsen worker 64 - jepsen.util 7174 :invoke :cas [21 [9 9]] INFO [2019-12-17 12:34:54,658] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51765,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:34:54,658] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51765,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:34:54,658] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51769,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:34:54,659] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51769,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:34:54,696] jepsen worker 42 - jepsen.util 13182 :info :cas [14 [31 20]] :RequestCanceledException INFO [2019-12-17 12:34:54,698] jepsen worker 9 - jepsen.util 9459 :ok :write [3 0] INFO [2019-12-17 12:34:54,698] jepsen worker 9 - jepsen.util 9459 :invoke :cas [3 [14 14]] INFO [2019-12-17 12:34:54,700] jepsen worker 42 - jepsen.util 13272 :invoke :read [14 nil] INFO [2019-12-17 12:34:54,703] jepsen worker 10 - jepsen.util 9010 :fail :cas [3 [26 34]] :ValueNotSwapFrom INFO [2019-12-17 12:34:54,703] jepsen worker 10 - jepsen.util 9010 :invoke :read [3 nil] INFO [2019-12-17 12:34:54,712] jepsen worker 20 - jepsen.util 8660 :fail :cas [6 [31 22]] :ValueNotSwapFrom INFO [2019-12-17 12:34:54,712] jepsen worker 20 - jepsen.util 8660 :invoke :write [6 1] INFO [2019-12-17 12:34:54,715] jepsen worker 40 - jepsen.util 12280 :info :write [13 33] :RequestCanceledException INFO [2019-12-17 12:34:54,718] jepsen worker 40 - jepsen.util 12370 :invoke :read [13 nil] INFO [2019-12-17 12:34:54,722] jepsen worker 30 - jepsen.util 8940 :ok :read [10 9] INFO [2019-12-17 12:34:54,722] jepsen worker 30 - jepsen.util 8940 :invoke :cas [10 [7 38]] INFO [2019-12-17 12:34:54,728] jepsen worker 77 - jepsen.util 12587 :info :cas [25 [10 10]] :RequestCanceledException INFO [2019-12-17 12:34:54,734] jepsen worker 77 - jepsen.util 12677 :invoke :read [25 nil] INFO [2019-12-17 12:34:54,744] jepsen worker 51 - jepsen.util 8241 :info :write [17 7] :RequestCanceledException INFO [2019-12-17 12:34:54,745] jepsen worker 29 - jepsen.util 7499 :ok :read [9 46] INFO [2019-12-17 12:34:54,745] jepsen worker 29 - jepsen.util 7499 :invoke :read [9 nil] INFO [2019-12-17 12:34:54,748] jepsen worker 51 - jepsen.util 8331 :invoke :write [17 9] INFO [2019-12-17 12:34:54,753] jepsen worker 65 - jepsen.util 7355 :ok :write [21 44] INFO [2019-12-17 12:34:54,753] jepsen worker 65 - jepsen.util 7355 :invoke :read [21 nil] INFO [2019-12-17 12:34:54,760] jepsen worker 43 - jepsen.util 11833 :info :cas [14 [34 48]] :RequestCanceledException INFO [2019-12-17 12:34:54,763] jepsen worker 43 - jepsen.util 11923 :invoke :write [14 35] INFO [2019-12-17 12:34:54,778] jepsen worker 36 - jepsen.util 9396 :ok :read [12 45] INFO [2019-12-17 12:34:54,778] jepsen worker 36 - jepsen.util 9396 :invoke :write [12 25] INFO [2019-12-17 12:34:54,796] jepsen worker 72 - jepsen.util 8892 :info :write [24 30] :RequestCanceledException INFO [2019-12-17 12:34:54,796] jepsen worker 1 - jepsen.util 12241 :info :write [0 24] :RequestCanceledException INFO [2019-12-17 12:34:54,800] jepsen worker 1 - jepsen.util 12331 :invoke :read [0 nil] INFO [2019-12-17 12:34:54,802] jepsen worker 72 - jepsen.util 8982 :invoke :write [24 32] INFO [2019-12-17 12:34:54,807] jepsen worker 28 - jepsen.util 7228 :ok :write [9 42] INFO [2019-12-17 12:34:54,807] jepsen worker 28 - jepsen.util 7228 :invoke :write [9 45] INFO [2019-12-17 12:34:54,830] jepsen worker 75 - jepsen.util 11415 :info :cas [25 [29 29]] :RequestCanceledException INFO [2019-12-17 12:34:54,833] jepsen worker 75 - jepsen.util 11505 :invoke :cas [25 [25 7]] INFO [2019-12-17 12:34:54,844] jepsen worker 15 - jepsen.util 11175 :info :cas [5 [14 15]] :RequestCanceledException INFO [2019-12-17 12:34:54,854] jepsen worker 15 - jepsen.util 11265 :invoke :read [5 nil] INFO [2019-12-17 12:34:54,855] jepsen worker 46 - jepsen.util 8956 :ok :read [15 37] INFO [2019-12-17 12:34:54,855] jepsen worker 46 - jepsen.util 8956 :invoke :cas [15 [40 2]] INFO [2019-12-17 12:34:54,881] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51790,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:34:54,881] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51790,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:34:54,881] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51794,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:34:54,882] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51794,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:34:54,882] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51795,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:34:54,882] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51795,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:34:54,896] jepsen worker 8 - jepsen.util 11798 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:34:54,897] jepsen worker 8 - jepsen.util 11798 :invoke :read [2 nil] INFO [2019-12-17 12:34:54,911] jepsen worker 31 - jepsen.util 8041 :fail :cas [10 [7 38]] :ValueNotSwapFrom INFO [2019-12-17 12:34:54,911] jepsen worker 31 - jepsen.util 8041 :invoke :read [10 nil] INFO [2019-12-17 12:34:54,912] jepsen worker 12 - jepsen.util 9552 :ok :read [4 45] INFO [2019-12-17 12:34:54,912] jepsen worker 12 - jepsen.util 9552 :invoke :cas [4 [36 16]] INFO [2019-12-17 12:34:54,930] jepsen worker 19 - jepsen.util 8929 :info :cas [6 [7 4]] :RequestCanceledException INFO [2019-12-17 12:34:54,930] jepsen worker 7 - jepsen.util 12247 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:34:54,930] jepsen worker 7 - jepsen.util 12247 :invoke :cas [2 [18 30]] INFO [2019-12-17 12:34:54,933] jepsen worker 19 - jepsen.util 9019 :invoke :write [6 43] INFO [2019-12-17 12:34:54,944] jepsen worker 70 - jepsen.util 9520 :info :write [23 13] :RequestCanceledException INFO [2019-12-17 12:34:54,949] jepsen worker 30 - jepsen.util 8940 :fail :cas [10 [7 38]] :ValueNotSwapFrom INFO [2019-12-17 12:34:54,950] jepsen worker 30 - jepsen.util 8940 :invoke :read [10 nil] INFO [2019-12-17 12:34:54,950] jepsen worker 70 - jepsen.util 9610 :invoke :write [23 20] INFO [2019-12-17 12:34:54,965] jepsen worker 64 - jepsen.util 7174 :fail :cas [21 [9 9]] :ValueNotSwapFrom INFO [2019-12-17 12:34:54,965] jepsen worker 64 - jepsen.util 7174 :invoke :write [21 32] INFO [2019-12-17 12:34:54,973] jepsen worker 79 - jepsen.util 9349 :ok :write [26 36] INFO [2019-12-17 12:34:54,973] jepsen worker 79 - jepsen.util 9349 :invoke :read [26 nil] INFO [2019-12-17 12:34:54,983] jepsen worker 37 - jepsen.util 8407 :ok :write [12 36] INFO [2019-12-17 12:34:54,983] jepsen worker 37 - jepsen.util 8407 :invoke :write [12 13] INFO [2019-12-17 12:34:54,991] jepsen worker 10 - jepsen.util 9010 :ok :read [3 0] INFO [2019-12-17 12:34:54,991] jepsen worker 10 - jepsen.util 9010 :invoke :write [3 47] INFO [2019-12-17 12:34:54,997] jepsen worker 54 - jepsen.util 9774 :info :write [18 26] :RequestCanceledException INFO [2019-12-17 12:34:55,007] jepsen worker 54 - jepsen.util 9864 :invoke :cas [18 [4 35]] INFO [2019-12-17 12:34:55,059] jepsen worker 13 - jepsen.util 8653 :ok :write [4 44] INFO [2019-12-17 12:34:55,059] jepsen worker 13 - jepsen.util 8653 :invoke :cas [4 [4 40]] INFO [2019-12-17 12:34:55,061] jepsen worker 49 - jepsen.util 11479 :info :cas [16 [6 8]] :RequestCanceledException INFO [2019-12-17 12:34:55,066] jepsen worker 49 - jepsen.util 11569 :invoke :read [16 nil] INFO [2019-12-17 12:34:55,070] jepsen worker 30 - jepsen.util 8940 :ok :read [10 9] INFO [2019-12-17 12:34:55,070] jepsen worker 30 - jepsen.util 8940 :invoke :cas [10 [1 11]] INFO [2019-12-17 12:34:55,105] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51807,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:34:55,105] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51807,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:34:55,105] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51810,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:34:55,105] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51810,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:34:55,105] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51811,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:34:55,105] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51811,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:34:55,131] jepsen worker 74 - jepsen.util 9074 :ok :write [24 11] INFO [2019-12-17 12:34:55,131] jepsen worker 74 - jepsen.util 9074 :invoke :write [24 16] INFO [2019-12-17 12:34:55,137] jepsen worker 20 - jepsen.util 8660 :ok :write [6 1] INFO [2019-12-17 12:34:55,137] jepsen worker 20 - jepsen.util 8660 :invoke :read [6 nil] INFO [2019-12-17 12:34:55,144] jepsen worker 80 - jepsen.util 8540 :info :write [26 38] :RequestCanceledException INFO [2019-12-17 12:34:55,146] jepsen worker 80 - jepsen.util 8630 :invoke :write [26 28] INFO [2019-12-17 12:34:55,157] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:34:55,158] jepsen nemesis - couchbase.util Rebalance check count: 400 INFO [2019-12-17 12:34:55,158] jepsen nemesis - couchbase.util Rebalance stuck count: 381 INFO [2019-12-17 12:34:55,158] jepsen worker 63 - jepsen.util 7803 :info :cas [21 [36 9]] :RequestCanceledException INFO [2019-12-17 12:34:55,163] jepsen worker 63 - jepsen.util 7893 :invoke :read [21 nil] INFO [2019-12-17 12:34:55,197] jepsen worker 32 - jepsen.util 9122 :fail :read [10 nil] :RequestTimeoutException INFO [2019-12-17 12:34:55,197] jepsen worker 86 - jepsen.util 12956 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:34:55,197] jepsen worker 32 - jepsen.util 9122 :invoke :read [10 nil] INFO [2019-12-17 12:34:55,197] jepsen worker 86 - jepsen.util 12956 :invoke :write [28 30] INFO [2019-12-17 12:34:55,198] jepsen worker 30 - jepsen.util 8940 :fail :cas [10 [1 11]] :ValueNotSwapFrom INFO [2019-12-17 12:34:55,198] jepsen worker 30 - jepsen.util 8940 :invoke :cas [10 [37 35]] INFO [2019-12-17 12:34:55,214] jepsen worker 39 - jepsen.util 11649 :info :cas [13 [3 10]] :RequestCanceledException INFO [2019-12-17 12:34:55,215] jepsen worker 28 - jepsen.util 7228 :ok :write [9 45] INFO [2019-12-17 12:34:55,215] jepsen worker 28 - jepsen.util 7228 :invoke :read [9 nil] INFO [2019-12-17 12:34:55,220] jepsen worker 39 - jepsen.util 11739 :invoke :write [13 7] INFO [2019-12-17 12:34:55,230] jepsen worker 82 - jepsen.util 12862 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:34:55,230] jepsen worker 2 - jepsen.util 11792 :info :cas [0 [19 2]] :RequestCanceledException INFO [2019-12-17 12:34:55,230] jepsen worker 82 - jepsen.util 12862 :invoke :cas [27 [18 5]] INFO [2019-12-17 12:34:55,230] jepsen worker 69 - jepsen.util 8979 :info :write [23 36] :RequestCanceledException INFO [2019-12-17 12:34:55,233] jepsen worker 69 - jepsen.util 9069 :invoke :write [23 35] INFO [2019-12-17 12:34:55,240] jepsen worker 2 - jepsen.util 11882 :invoke :read [0 nil] INFO [2019-12-17 12:34:55,240] jepsen worker 65 - jepsen.util 7355 :ok :read [21 44] INFO [2019-12-17 12:34:55,240] jepsen worker 65 - jepsen.util 7355 :invoke :cas [21 [12 41]] INFO [2019-12-17 12:34:55,270] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51834,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:34:55,271] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51834,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:34:55,272] jepsen worker 20 - jepsen.util 8660 :ok :read [6 1] INFO [2019-12-17 12:34:55,272] jepsen worker 20 - jepsen.util 8660 :invoke :cas [6 [14 15]] INFO [2019-12-17 12:34:55,282] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51822,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:34:55,283] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51822,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:34:55,283] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51824,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:34:55,283] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51824,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:34:55,283] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51825,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:34:55,283] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51825,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:34:55,295] jepsen worker 63 - jepsen.util 7893 :ok :read [21 44] INFO [2019-12-17 12:34:55,295] jepsen worker 63 - jepsen.util 7893 :invoke :read [21 nil] INFO [2019-12-17 12:34:55,298] jepsen worker 41 - jepsen.util 12821 :info :cas [13 [27 33]] :RequestCanceledException INFO [2019-12-17 12:34:55,311] jepsen worker 41 - jepsen.util 12911 :invoke :cas [13 [13 15]] INFO [2019-12-17 12:34:55,314] jepsen worker 87 - jepsen.util 8367 :info :write [29 12] :RequestCanceledException INFO [2019-12-17 12:34:55,317] jepsen worker 87 - jepsen.util 8457 :invoke :write [29 0] INFO [2019-12-17 12:34:55,329] jepsen worker 28 - jepsen.util 7228 :ok :read [9 45] INFO [2019-12-17 12:34:55,329] jepsen worker 28 - jepsen.util 7228 :invoke :write [9 15] INFO [2019-12-17 12:34:55,330] jepsen worker 52 - jepsen.util 9322 :fail :cas [17 [46 37]] :ValueNotSwapFrom INFO [2019-12-17 12:34:55,330] jepsen worker 52 - jepsen.util 9322 :invoke :read [17 nil] INFO [2019-12-17 12:34:55,330] jepsen worker 81 - jepsen.util 11871 :info :write [27 23] :RequestCanceledException INFO [2019-12-17 12:34:55,337] jepsen worker 81 - jepsen.util 11961 :invoke :read [27 nil] INFO [2019-12-17 12:34:55,339] jepsen worker 32 - jepsen.util 9122 :ok :read [10 9] INFO [2019-12-17 12:34:55,339] jepsen worker 32 - jepsen.util 9122 :invoke :write [10 34] INFO [2019-12-17 12:34:55,361] jepsen worker 21 - jepsen.util 11811 :info :cas [7 [16 8]] :RequestCanceledException INFO [2019-12-17 12:34:55,370] jepsen worker 21 - jepsen.util 11901 :invoke :cas [7 [17 0]] INFO [2019-12-17 12:34:55,376] jepsen worker 36 - jepsen.util 9396 :ok :write [12 25] INFO [2019-12-17 12:34:55,376] jepsen worker 36 - jepsen.util 9396 :invoke :write [12 21] INFO [2019-12-17 12:34:55,414] jepsen worker 34 - jepsen.util 12274 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:34:55,414] jepsen worker 3 - jepsen.util 8823 :info :write [1 25] :RequestCanceledException INFO [2019-12-17 12:34:55,415] jepsen worker 34 - jepsen.util 12274 :invoke :write [11 10] INFO [2019-12-17 12:34:55,416] jepsen worker 65 - jepsen.util 7355 :fail :cas [21 [12 41]] :ValueNotSwapFrom INFO [2019-12-17 12:34:55,416] jepsen worker 65 - jepsen.util 7355 :invoke :cas [21 [28 8]] INFO [2019-12-17 12:34:55,423] jepsen worker 3 - jepsen.util 8913 :invoke :read [1 nil] INFO [2019-12-17 12:34:55,428] jepsen worker 30 - jepsen.util 8940 :fail :cas [10 [37 35]] :ValueNotSwapFrom INFO [2019-12-17 12:34:55,429] jepsen worker 30 - jepsen.util 8940 :invoke :read [10 nil] INFO [2019-12-17 12:34:55,430] jepsen worker 76 - jepsen.util 12496 :info :write [25 3] :RequestCanceledException INFO [2019-12-17 12:34:55,433] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51856,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:34:55,434] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51856,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:34:55,434] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51859,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:34:55,434] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51859,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:34:55,437] jepsen worker 76 - jepsen.util 12586 :invoke :write [25 24] INFO [2019-12-17 12:34:55,444] jepsen worker 23 - jepsen.util 11903 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:34:55,444] jepsen worker 23 - jepsen.util 11903 :invoke :write [7 48] INFO [2019-12-17 12:34:55,452] jepsen worker 5 - jepsen.util 8375 :ok :write [1 34] INFO [2019-12-17 12:34:55,452] jepsen worker 5 - jepsen.util 8375 :invoke :cas [1 [1 19]] INFO [2019-12-17 12:34:55,452] jepsen worker 71 - jepsen.util 9431 :ok :write [23 30] INFO [2019-12-17 12:34:55,452] jepsen worker 71 - jepsen.util 9431 :invoke :read [23 nil] INFO [2019-12-17 12:34:55,456] jepsen worker 63 - jepsen.util 7893 :ok :read [21 44] INFO [2019-12-17 12:34:55,456] jepsen worker 63 - jepsen.util 7893 :invoke :write [21 3] INFO [2019-12-17 12:34:55,460] jepsen worker 52 - jepsen.util 9322 :ok :read [17 42] INFO [2019-12-17 12:34:55,460] jepsen worker 52 - jepsen.util 9322 :invoke :cas [17 [27 11]] INFO [2019-12-17 12:34:55,461] jepsen worker 51 - jepsen.util 8331 :ok :write [17 9] INFO [2019-12-17 12:34:55,461] jepsen worker 51 - jepsen.util 8331 :invoke :write [17 48] INFO [2019-12-17 12:34:55,466] jepsen worker 20 - jepsen.util 8660 :fail :cas [6 [14 15]] :ValueNotSwapFrom INFO [2019-12-17 12:34:55,466] jepsen worker 20 - jepsen.util 8660 :invoke :cas [6 [28 43]] INFO [2019-12-17 12:34:55,493] jepsen worker 59 - jepsen.util 11489 :info :cas [19 [0 27]] :RequestCanceledException INFO [2019-12-17 12:34:55,496] jepsen worker 59 - jepsen.util 11579 :invoke :read [19 nil] INFO [2019-12-17 12:34:55,510] jepsen worker 55 - jepsen.util 9775 :ok :write [18 37] INFO [2019-12-17 12:34:55,510] jepsen worker 55 - jepsen.util 9775 :invoke :write [18 28] INFO [2019-12-17 12:34:55,514] jepsen worker 4 - jepsen.util 8464 :info :write [1 36] :RequestCanceledException INFO [2019-12-17 12:34:55,518] jepsen worker 4 - jepsen.util 8554 :invoke :cas [1 [21 9]] INFO [2019-12-17 12:34:55,530] jepsen worker 68 - jepsen.util 11408 :info :cas [22 [45 13]] :RequestCanceledException INFO [2019-12-17 12:34:55,533] jepsen worker 65 - jepsen.util 7355 :fail :cas [21 [28 8]] :ValueNotSwapFrom INFO [2019-12-17 12:34:55,533] jepsen worker 65 - jepsen.util 7355 :invoke :write [21 20] INFO [2019-12-17 12:34:55,534] jepsen worker 68 - jepsen.util 11498 :invoke :cas [22 [20 23]] INFO [2019-12-17 12:34:55,564] jepsen worker 20 - jepsen.util 8660 :fail :cas [6 [28 43]] :ValueNotSwapFrom INFO [2019-12-17 12:34:55,583] jepsen worker 74 - jepsen.util 9074 :ok :write [24 16] INFO [2019-12-17 12:34:55,588] jepsen worker 28 - jepsen.util 7228 :ok :write [9 15] INFO [2019-12-17 12:34:55,592] jepsen worker 52 - jepsen.util 9322 :fail :cas [17 [27 11]] :ValueNotSwapFrom INFO [2019-12-17 12:34:55,596] jepsen worker 20 - jepsen.util 8660 :invoke :cas [6 [38 48]] INFO [2019-12-17 12:34:55,596] jepsen worker 16 - jepsen.util 13156 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:34:55,596] jepsen worker 74 - jepsen.util 9074 :invoke :cas [24 [47 8]] INFO [2019-12-17 12:34:55,596] jepsen worker 28 - jepsen.util 7228 :invoke :write [9 47] INFO [2019-12-17 12:34:55,596] jepsen worker 52 - jepsen.util 9322 :invoke :read [17 nil] INFO [2019-12-17 12:34:55,596] jepsen worker 16 - jepsen.util 13156 :invoke :write [5 31] INFO [2019-12-17 12:34:55,596] jepsen worker 79 - jepsen.util 9349 :ok :read [26 38] INFO [2019-12-17 12:34:55,597] jepsen worker 79 - jepsen.util 9349 :invoke :write [26 22] INFO [2019-12-17 12:34:55,610] jepsen worker 88 - jepsen.util 9088 :info :write [29 44] :RequestCanceledException INFO [2019-12-17 12:34:55,631] jepsen worker 50 - jepsen.util 12740 :info :cas [16 [40 24]] :RequestCanceledException INFO [2019-12-17 12:34:55,631] jepsen worker 89 - jepsen.util 8999 :info :write [29 11] :RequestCanceledException INFO [2019-12-17 12:34:55,644] jepsen worker 83 - jepsen.util 12683 :info :cas [27 [44 23]] :RequestCanceledException INFO [2019-12-17 12:34:55,661] jepsen worker 38 - jepsen.util 9308 :info :write [12 11] :RequestCanceledException INFO [2019-12-17 12:34:55,678] jepsen worker 50 - jepsen.util 12830 :invoke :read [16 nil] INFO [2019-12-17 12:34:55,679] jepsen worker 88 - jepsen.util 9178 :invoke :cas [29 [4 44]] INFO [2019-12-17 12:34:55,680] jepsen worker 89 - jepsen.util 9089 :invoke :cas [29 [37 37]] INFO [2019-12-17 12:34:55,687] jepsen worker 83 - jepsen.util 12773 :invoke :write [27 25] INFO [2019-12-17 12:34:55,691] jepsen worker 38 - jepsen.util 9398 :invoke :read [12 nil] INFO [2019-12-17 12:34:55,697] jepsen worker 67 - jepsen.util 12217 :info :write [22 16] :RequestCanceledException INFO [2019-12-17 12:34:55,701] jepsen worker 67 - jepsen.util 12307 :invoke :write [22 46] INFO [2019-12-17 12:34:55,701] jepsen worker 36 - jepsen.util 9396 :ok :write [12 21] INFO [2019-12-17 12:34:55,702] jepsen worker 36 - jepsen.util 9396 :invoke :read [12 nil] INFO [2019-12-17 12:34:55,729] jepsen worker 63 - jepsen.util 7893 :ok :write [21 3] INFO [2019-12-17 12:34:55,729] jepsen worker 63 - jepsen.util 7893 :invoke :read [21 nil] INFO [2019-12-17 12:34:55,743] jepsen worker 60 - jepsen.util 13110 :info :write [20 24] :RequestCanceledException INFO [2019-12-17 12:34:55,753] jepsen worker 60 - jepsen.util 13200 :invoke :cas [20 [6 31]] INFO [2019-12-17 12:34:55,754] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51882,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:34:55,754] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51882,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:34:55,754] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51885,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:34:55,754] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51885,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:34:55,756] jepsen worker 54 - jepsen.util 9864 :fail :cas [18 [4 35]] :ValueNotSwapFrom INFO [2019-12-17 12:34:55,756] jepsen worker 54 - jepsen.util 9864 :invoke :cas [18 [4 8]] INFO [2019-12-17 12:34:55,796] jepsen worker 20 - jepsen.util 8660 :fail :cas [6 [38 48]] :ValueNotSwapFrom INFO [2019-12-17 12:34:55,796] jepsen worker 20 - jepsen.util 8660 :invoke :read [6 nil] INFO [2019-12-17 12:34:55,806] jepsen worker 32 - jepsen.util 9122 :ok :write [10 34] INFO [2019-12-17 12:34:55,807] jepsen worker 32 - jepsen.util 9122 :invoke :cas [10 [12 4]] INFO [2019-12-17 12:34:55,814] jepsen worker 35 - jepsen.util 12095 :info :write [11 46] :RequestCanceledException INFO [2019-12-17 12:34:55,816] jepsen worker 35 - jepsen.util 12185 :invoke :cas [11 [41 28]] INFO [2019-12-17 12:34:55,817] jepsen worker 74 - jepsen.util 9074 :fail :cas [24 [47 8]] :ValueNotSwapFrom INFO [2019-12-17 12:34:55,817] jepsen worker 74 - jepsen.util 9074 :invoke :write [24 49] INFO [2019-12-17 12:34:55,873] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51901,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:34:55,873] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51901,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:34:55,873] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51904,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:34:55,874] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51904,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:34:55,874] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51905,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:34:55,876] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51905,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:34:55,901] jepsen worker 28 - jepsen.util 7228 :ok :write [9 47] INFO [2019-12-17 12:34:55,903] jepsen worker 28 - jepsen.util 7228 :invoke :write [9 48] INFO [2019-12-17 12:34:55,910] jepsen worker 12 - jepsen.util 9552 :fail :cas [4 [36 16]] :ValueNotSwapFrom INFO [2019-12-17 12:34:55,911] jepsen worker 12 - jepsen.util 9552 :invoke :cas [4 [35 34]] INFO [2019-12-17 12:34:55,977] jepsen worker 63 - jepsen.util 7893 :ok :read [21 3] INFO [2019-12-17 12:34:55,977] jepsen worker 63 - jepsen.util 7893 :invoke :read [21 nil] INFO [2019-12-17 12:34:56,041] jepsen worker 70 - jepsen.util 9610 :ok :write [23 20] INFO [2019-12-17 12:34:56,041] jepsen worker 70 - jepsen.util 9610 :invoke :cas [23 [4 36]] INFO [2019-12-17 12:34:56,042] jepsen worker 53 - jepsen.util 8153 :ok :write [17 16] INFO [2019-12-17 12:34:56,043] jepsen worker 53 - jepsen.util 8153 :invoke :cas [17 [38 43]] INFO [2019-12-17 12:34:56,044] jepsen worker 33 - jepsen.util 12093 :info :write [11 12] :RequestCanceledException INFO [2019-12-17 12:34:56,047] jepsen worker 11 - jepsen.util 9011 :info :write [3 26] :RequestCanceledException INFO [2019-12-17 12:34:56,057] jepsen worker 11 - jepsen.util 9101 :invoke :cas [3 [45 1]] INFO [2019-12-17 12:34:56,066] jepsen worker 55 - jepsen.util 9775 :ok :write [18 28] INFO [2019-12-17 12:34:56,067] jepsen worker 55 - jepsen.util 9775 :invoke :read [18 nil] INFO [2019-12-17 12:34:56,088] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51911,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:34:56,088] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51911,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:34:56,102] jepsen worker 33 - jepsen.util 12183 :invoke :write [11 42] INFO [2019-12-17 12:34:56,130] jepsen worker 62 - jepsen.util 12392 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:34:56,131] jepsen worker 62 - jepsen.util 12392 :invoke :cas [20 [9 19]] INFO [2019-12-17 12:34:56,160] jepsen worker 74 - jepsen.util 9074 :ok :write [24 49] INFO [2019-12-17 12:34:56,160] jepsen worker 74 - jepsen.util 9074 :invoke :write [24 47] INFO [2019-12-17 12:34:56,163] jepsen worker 78 - jepsen.util 8718 :info :write [26 13] :RequestCanceledException INFO [2019-12-17 12:34:56,188] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:34:56,190] jepsen nemesis - couchbase.util Rebalance check count: 401 INFO [2019-12-17 12:34:56,190] jepsen nemesis - couchbase.util Rebalance stuck count: 382 INFO [2019-12-17 12:34:56,197] jepsen worker 0 - jepsen.util 12780 :info :write [0 13] :RequestCanceledException INFO [2019-12-17 12:34:56,215] jepsen worker 25 - jepsen.util 12895 :info :cas [8 [40 12]] :RequestCanceledException INFO [2019-12-17 12:34:56,232] jepsen worker 78 - jepsen.util 8808 :invoke :read [26 nil] INFO [2019-12-17 12:34:56,259] jepsen worker 27 - jepsen.util 8667 :info :write [9 13] :RequestCanceledException INFO [2019-12-17 12:34:56,270] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51921,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:34:56,272] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51928,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:34:56,272] jepsen worker 27 - jepsen.util 8757 :invoke :write [9 2] INFO [2019-12-17 12:34:56,274] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51928,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:34:56,276] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51932,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:34:56,277] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51921,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:34:56,279] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51932,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:34:56,279] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51923,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:34:56,283] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51923,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:34:56,292] jepsen worker 25 - jepsen.util 12985 :invoke :cas [8 [18 2]] INFO [2019-12-17 12:34:56,302] jepsen worker 80 - jepsen.util 8630 :ok :write [26 28] INFO [2019-12-17 12:34:56,302] jepsen worker 80 - jepsen.util 8630 :invoke :write [26 45] INFO [2019-12-17 12:34:56,330] jepsen worker 57 - jepsen.util 11847 :info :write [19 31] :RequestCanceledException INFO [2019-12-17 12:34:56,349] jepsen worker 26 - jepsen.util 12716 :info :cas [8 [16 49]] :RequestCanceledException INFO [2019-12-17 12:34:56,370] jepsen worker 0 - jepsen.util 12870 :invoke :cas [0 [8 14]] INFO [2019-12-17 12:34:56,384] jepsen worker 57 - jepsen.util 11937 :invoke :cas [19 [14 13]] INFO [2019-12-17 12:34:56,397] jepsen worker 85 - jepsen.util 12865 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:34:56,400] jepsen worker 85 - jepsen.util 12865 :invoke :cas [28 [17 21]] INFO [2019-12-17 12:34:56,410] jepsen worker 26 - jepsen.util 12806 :invoke :read [8 nil] INFO [2019-12-17 12:34:56,415] jepsen worker 14 - jepsen.util 9014 :fail :read [4 nil] :RequestTimeoutException INFO [2019-12-17 12:34:56,415] jepsen worker 14 - jepsen.util 9014 :invoke :cas [4 [46 11]] INFO [2019-12-17 12:34:56,444] jepsen worker 58 - jepsen.util 12928 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:34:56,445] jepsen worker 58 - jepsen.util 12928 :invoke :read [19 nil] INFO [2019-12-17 12:34:56,462] jepsen worker 73 - jepsen.util 8623 :fail :read [24 nil] :RequestTimeoutException INFO [2019-12-17 12:34:56,463] jepsen worker 73 - jepsen.util 8623 :invoke :read [24 nil] INFO [2019-12-17 12:34:56,463] jepsen worker 56 - jepsen.util 9596 :info :write [18 45] :RequestCanceledException INFO [2019-12-17 12:34:56,477] jepsen worker 28 - jepsen.util 7228 :ok :write [9 48] INFO [2019-12-17 12:34:56,477] jepsen worker 28 - jepsen.util 7228 :invoke :write [9 4] INFO [2019-12-17 12:34:56,478] jepsen worker 56 - jepsen.util 9686 :invoke :cas [18 [18 20]] INFO [2019-12-17 12:34:56,497] jepsen worker 6 - jepsen.util 12606 :info :cas [2 [35 28]] :RequestCanceledException INFO [2019-12-17 12:34:56,516] jepsen worker 48 - jepsen.util 11208 :info :write [16 10] :RequestCanceledException INFO [2019-12-17 12:34:56,517] jepsen worker 66 - jepsen.util 11946 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:34:56,518] jepsen worker 66 - jepsen.util 11946 :invoke :read [22 nil] INFO [2019-12-17 12:34:56,525] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51942,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:34:56,526] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51942,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:34:56,527] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51948,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:34:56,527] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51948,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:34:56,530] jepsen worker 44 - jepsen.util 12464 :info :cas [14 [39 19]] :RequestCanceledException INFO [2019-12-17 12:34:56,531] jepsen worker 61 - jepsen.util 13021 :info :write [20 36] :RequestCanceledException INFO [2019-12-17 12:34:56,533] jepsen worker 18 - jepsen.util 9198 :info :write [6 3] :RequestCanceledException INFO [2019-12-17 12:34:56,546] jepsen worker 18 - jepsen.util 9288 :invoke :cas [6 [4 45]] INFO [2019-12-17 12:34:56,563] jepsen worker 6 - jepsen.util 12696 :invoke :write [2 1] INFO [2019-12-17 12:34:56,565] jepsen worker 48 - jepsen.util 11298 :invoke :write [16 24] INFO [2019-12-17 12:34:56,602] jepsen worker 44 - jepsen.util 12554 :invoke :write [14 8] INFO [2019-12-17 12:34:56,620] jepsen worker 61 - jepsen.util 13111 :invoke :write [20 24] INFO [2019-12-17 12:34:56,624] jepsen worker 22 - jepsen.util 11992 :info :write [7 33] :RequestCanceledException INFO [2019-12-17 12:34:56,654] jepsen worker 87 - jepsen.util 8457 :ok :write [29 0] INFO [2019-12-17 12:34:56,655] jepsen worker 87 - jepsen.util 8457 :invoke :read [29 nil] INFO [2019-12-17 12:34:56,670] jepsen worker 47 - jepsen.util 9407 :info :cas [15 [33 27]] :RequestCanceledException INFO [2019-12-17 12:34:56,670] jepsen worker 45 - jepsen.util 9315 :fail :read [15 nil] :RequestTimeoutException INFO [2019-12-17 12:34:56,670] jepsen worker 84 - jepsen.util 11604 :info :write [28 10] :RequestCanceledException INFO [2019-12-17 12:34:56,671] jepsen worker 45 - jepsen.util 9315 :invoke :read [15 nil] INFO [2019-12-17 12:34:56,701] jepsen worker 24 - jepsen.util 11184 :info :write [8 20] :RequestCanceledException INFO [2019-12-17 12:34:56,717] jepsen worker 42 - jepsen.util 13272 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:34:56,720] jepsen worker 42 - jepsen.util 13272 :invoke :read [14 nil] INFO [2019-12-17 12:34:56,736] jepsen worker 17 - jepsen.util 12167 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:34:56,739] jepsen worker 20 - jepsen.util 8660 :ok :read [6 1] INFO [2019-12-17 12:34:56,739] jepsen worker 17 - jepsen.util 12167 :invoke :read [5 nil] INFO [2019-12-17 12:34:56,740] jepsen worker 40 - jepsen.util 12370 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:34:56,740] jepsen worker 9 - jepsen.util 9459 :info :cas [3 [14 14]] :RequestCanceledException INFO [2019-12-17 12:34:56,740] jepsen worker 20 - jepsen.util 8660 :invoke :write [6 40] INFO [2019-12-17 12:34:56,741] jepsen worker 40 - jepsen.util 12370 :invoke :cas [13 [17 31]] INFO [2019-12-17 12:34:56,744] jepsen worker 22 - jepsen.util 12082 :invoke :cas [7 [13 18]] INFO [2019-12-17 12:34:56,750] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51956,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:34:56,752] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51956,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:34:56,753] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51960,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:34:56,753] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51960,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:34:56,777] jepsen worker 24 - jepsen.util 11274 :invoke :read [8 nil] INFO [2019-12-17 12:34:56,780] jepsen worker 84 - jepsen.util 11694 :invoke :read [28 nil] INFO [2019-12-17 12:34:56,800] jepsen worker 9 - jepsen.util 9549 :invoke :read [3 nil] INFO [2019-12-17 12:34:56,811] jepsen worker 47 - jepsen.util 9497 :invoke :cas [15 [14 23]] INFO [2019-12-17 12:34:56,830] jepsen worker 29 - jepsen.util 7499 :fail :read [9 nil] :RequestTimeoutException INFO [2019-12-17 12:34:56,830] jepsen worker 43 - jepsen.util 11923 :info :write [14 35] :RequestCanceledException INFO [2019-12-17 12:34:56,830] jepsen worker 29 - jepsen.util 7499 :invoke :cas [9 [12 24]] INFO [2019-12-17 12:34:56,845] jepsen worker 77 - jepsen.util 12677 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:34:56,846] jepsen worker 77 - jepsen.util 12677 :invoke :read [25 nil] INFO [2019-12-17 12:34:56,903] jepsen worker 1 - jepsen.util 12331 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:34:56,903] jepsen worker 1 - jepsen.util 12331 :invoke :read [0 nil] INFO [2019-12-17 12:34:56,910] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51976,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:34:56,910] jepsen worker 43 - jepsen.util 12013 :invoke :write [14 0] INFO [2019-12-17 12:34:56,910] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51976,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:34:56,913] jepsen worker 72 - jepsen.util 8982 :info :write [24 32] :RequestCanceledException INFO [2019-12-17 12:34:56,935] jepsen worker 75 - jepsen.util 11505 :info :cas [25 [25 7]] :RequestCanceledException INFO [2019-12-17 12:34:56,935] jepsen worker 46 - jepsen.util 8956 :info :cas [15 [40 2]] :RequestCanceledException INFO [2019-12-17 12:34:56,948] jepsen worker 15 - jepsen.util 11265 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:34:56,948] jepsen worker 15 - jepsen.util 11265 :invoke :cas [5 [9 13]] INFO [2019-12-17 12:34:56,953] jepsen worker 72 - jepsen.util 9072 :invoke :cas [24 [23 10]] INFO [2019-12-17 12:34:56,970] jepsen worker 19 - jepsen.util 9019 :info :write [6 43] :RequestCanceledException INFO [2019-12-17 12:34:56,979] jepsen worker 27 - jepsen.util 8757 :ok :write [9 2] INFO [2019-12-17 12:34:56,980] jepsen worker 27 - jepsen.util 8757 :invoke :cas [9 [35 14]] INFO [2019-12-17 12:34:57,002] jepsen worker 8 - jepsen.util 11798 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:34:57,003] jepsen worker 8 - jepsen.util 11798 :invoke :read [2 nil] INFO [2019-12-17 12:34:57,017] jepsen worker 19 - jepsen.util 9109 :invoke :write [6 16] INFO [2019-12-17 12:34:57,018] jepsen worker 31 - jepsen.util 8041 :fail :read [10 nil] :RequestTimeoutException INFO [2019-12-17 12:34:57,019] jepsen worker 31 - jepsen.util 8041 :invoke :read [10 nil] INFO [2019-12-17 12:34:57,019] jepsen worker 51 - jepsen.util 8331 :ok :write [17 48] INFO [2019-12-17 12:34:57,024] jepsen worker 64 - jepsen.util 7174 :info :write [21 32] :RequestCanceledException INFO [2019-12-17 12:34:57,031] jepsen worker 51 - jepsen.util 8331 :invoke :cas [17 [22 40]] INFO [2019-12-17 12:34:57,032] jepsen worker 7 - jepsen.util 12247 :info :cas [2 [18 30]] :RequestCanceledException INFO [2019-12-17 12:34:57,032] jepsen worker 37 - jepsen.util 8407 :info :write [12 13] :RequestCanceledException INFO [2019-12-17 12:34:57,043] jepsen worker 37 - jepsen.util 8497 :invoke :cas [12 [2 30]] INFO [2019-12-17 12:34:57,051] jepsen worker 32 - jepsen.util 9122 :fail :cas [10 [12 4]] :ValueNotSwapFrom INFO [2019-12-17 12:34:57,054] jepsen worker 32 - jepsen.util 9122 :invoke :write [10 46] INFO [2019-12-17 12:34:57,090] jepsen worker 46 - jepsen.util 9046 :invoke :write [15 46] INFO [2019-12-17 12:34:57,090] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51984,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:34:57,101] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51984,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:34:57,102] jepsen worker 74 - jepsen.util 9074 :ok :write [24 47] INFO [2019-12-17 12:34:57,106] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51989,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:34:57,106] jepsen worker 74 - jepsen.util 9074 :invoke :write [24 8] INFO [2019-12-17 12:34:57,107] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51989,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:34:57,115] jepsen worker 7 - jepsen.util 12337 :invoke :write [2 47] INFO [2019-12-17 12:34:57,131] jepsen worker 10 - jepsen.util 9010 :info :write [3 47] :RequestCanceledException INFO [2019-12-17 12:34:57,132] jepsen worker 49 - jepsen.util 11569 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:34:57,133] jepsen worker 49 - jepsen.util 11569 :invoke :write [16 48] INFO [2019-12-17 12:34:57,162] jepsen worker 75 - jepsen.util 11595 :invoke :read [25 nil] INFO [2019-12-17 12:34:57,211] jepsen worker 64 - jepsen.util 7264 :invoke :read [21 nil] INFO [2019-12-17 12:34:57,241] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:34:57,241] jepsen nemesis - couchbase.util Rebalance check count: 402 INFO [2019-12-17 12:34:57,241] jepsen nemesis - couchbase.util Rebalance stuck count: 383 INFO [2019-12-17 12:34:57,247] jepsen worker 10 - jepsen.util 9100 :invoke :cas [3 [40 28]] INFO [2019-12-17 12:34:57,259] jepsen worker 39 - jepsen.util 11739 :info :write [13 7] :RequestCanceledException INFO [2019-12-17 12:34:57,279] jepsen worker 39 - jepsen.util 11829 :invoke :cas [13 [2 29]] INFO [2019-12-17 12:34:57,293] jepsen worker 52 - jepsen.util 9322 :ok :read [17 9] INFO [2019-12-17 12:34:57,293] jepsen worker 52 - jepsen.util 9322 :invoke :write [17 42] INFO [2019-12-17 12:34:57,294] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51996,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:34:57,295] jepsen worker 86 - jepsen.util 12956 :info :write [28 30] :RequestCanceledException INFO [2019-12-17 12:34:57,295] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51996,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:34:57,295] jepsen worker 2 - jepsen.util 11882 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:34:57,296] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51997,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:34:57,296] jepsen worker 2 - jepsen.util 11882 :invoke :write [0 18] INFO [2019-12-17 12:34:57,297] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":51997,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:34:57,325] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52006,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:34:57,327] jepsen worker 86 - jepsen.util 13046 :invoke :read [28 nil] INFO [2019-12-17 12:34:57,327] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52006,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:34:57,330] jepsen worker 82 - jepsen.util 12862 :info :cas [27 [18 5]] :RequestCanceledException INFO [2019-12-17 12:34:57,331] jepsen worker 69 - jepsen.util 9069 :info :write [23 35] :RequestCanceledException INFO [2019-12-17 12:34:57,382] jepsen worker 63 - jepsen.util 7893 :ok :read [21 3] INFO [2019-12-17 12:34:57,383] jepsen worker 63 - jepsen.util 7893 :invoke :write [21 21] INFO [2019-12-17 12:34:57,408] jepsen worker 69 - jepsen.util 9159 :invoke :read [23 nil] INFO [2019-12-17 12:34:57,408] jepsen worker 82 - jepsen.util 12952 :invoke :read [27 nil] INFO [2019-12-17 12:34:57,422] jepsen worker 41 - jepsen.util 12911 :info :cas [13 [13 15]] :RequestCanceledException INFO [2019-12-17 12:34:57,452] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52026,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:34:57,452] jepsen worker 81 - jepsen.util 11961 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:34:57,453] jepsen worker 11 - jepsen.util 9101 :fail :cas [3 [45 1]] :ValueNotSwapFrom INFO [2019-12-17 12:34:57,453] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52026,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:34:57,453] jepsen worker 81 - jepsen.util 11961 :invoke :read [27 nil] INFO [2019-12-17 12:34:57,454] jepsen worker 11 - jepsen.util 9101 :invoke :write [3 28] INFO [2019-12-17 12:34:57,457] jepsen worker 41 - jepsen.util 13001 :invoke :write [13 12] INFO [2019-12-17 12:34:57,468] jepsen worker 21 - jepsen.util 11901 :info :cas [7 [17 0]] :RequestCanceledException INFO [2019-12-17 12:34:57,503] jepsen worker 30 - jepsen.util 8940 :fail :read [10 nil] :RequestTimeoutException INFO [2019-12-17 12:34:57,503] jepsen worker 30 - jepsen.util 8940 :invoke :cas [10 [44 17]] INFO [2019-12-17 12:34:57,504] jepsen worker 76 - jepsen.util 12586 :info :write [25 24] :RequestCanceledException INFO [2019-12-17 12:34:57,508] jepsen worker 36 - jepsen.util 9396 :ok :read [12 11] INFO [2019-12-17 12:34:57,511] jepsen worker 36 - jepsen.util 9396 :invoke :read [12 nil] INFO [2019-12-17 12:34:57,520] jepsen worker 34 - jepsen.util 12274 :info :write [11 10] :RequestCanceledException INFO [2019-12-17 12:34:57,520] jepsen worker 59 - jepsen.util 11579 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:34:57,521] jepsen worker 89 - jepsen.util 9089 :fail :cas [29 [37 37]] :ValueNotSwapFrom INFO [2019-12-17 12:34:57,522] jepsen worker 89 - jepsen.util 9089 :invoke :cas [29 [25 20]] INFO [2019-12-17 12:34:57,523] jepsen worker 59 - jepsen.util 11579 :invoke :write [19 20] INFO [2019-12-17 12:34:57,544] jepsen worker 23 - jepsen.util 11903 :info :write [7 48] :RequestCanceledException INFO [2019-12-17 12:34:57,601] jepsen worker 21 - jepsen.util 11991 :invoke :write [7 39] INFO [2019-12-17 12:34:57,615] jepsen worker 65 - jepsen.util 7355 :info :write [21 20] :RequestCanceledException INFO [2019-12-17 12:34:57,623] jepsen worker 76 - jepsen.util 12676 :invoke :read [25 nil] INFO [2019-12-17 12:34:57,635] jepsen worker 79 - jepsen.util 9349 :info :write [26 22] :RequestCanceledException INFO [2019-12-17 12:34:57,647] jepsen worker 68 - jepsen.util 11498 :info :cas [22 [20 23]] :RequestCanceledException INFO [2019-12-17 12:34:57,663] jepsen worker 34 - jepsen.util 12364 :invoke :read [11 nil] INFO [2019-12-17 12:34:57,665] jepsen worker 65 - jepsen.util 7445 :invoke :read [21 nil] INFO [2019-12-17 12:34:57,665] jepsen worker 79 - jepsen.util 9439 :invoke :cas [26 [47 15]] INFO [2019-12-17 12:34:57,688] jepsen worker 23 - jepsen.util 11993 :invoke :cas [7 [38 10]] INFO [2019-12-17 12:34:57,697] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52039,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:34:57,701] jepsen worker 16 - jepsen.util 13156 :info :write [5 31] :RequestCanceledException INFO [2019-12-17 12:34:57,708] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52039,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:34:57,709] jepsen worker 3 - jepsen.util 8913 :ok :read [1 25] INFO [2019-12-17 12:34:57,711] jepsen worker 3 - jepsen.util 8913 :invoke :write [1 9] INFO [2019-12-17 12:34:57,717] jepsen worker 68 - jepsen.util 11588 :invoke :write [22 19] INFO [2019-12-17 12:34:57,733] jepsen worker 38 - jepsen.util 9398 :fail :read [12 nil] :RequestTimeoutException INFO [2019-12-17 12:34:57,733] jepsen worker 88 - jepsen.util 9178 :info :cas [29 [4 44]] :RequestCanceledException INFO [2019-12-17 12:34:57,735] jepsen worker 38 - jepsen.util 9398 :invoke :cas [12 [26 22]] INFO [2019-12-17 12:34:57,765] jepsen worker 16 - jepsen.util 13246 :invoke :write [5 26] INFO [2019-12-17 12:34:57,766] jepsen worker 50 - jepsen.util 12830 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:34:57,770] jepsen worker 50 - jepsen.util 12830 :invoke :read [16 nil] INFO [2019-12-17 12:34:57,837] jepsen worker 83 - jepsen.util 12773 :info :write [27 25] :RequestCanceledException INFO [2019-12-17 12:34:57,846] jepsen worker 35 - jepsen.util 12185 :info :cas [11 [41 28]] :RequestCanceledException INFO [2019-12-17 12:34:57,855] jepsen worker 67 - jepsen.util 12307 :info :write [22 46] :RequestCanceledException INFO [2019-12-17 12:34:57,867] jepsen worker 60 - jepsen.util 13200 :info :cas [20 [6 31]] :RequestCanceledException INFO [2019-12-17 12:34:57,886] jepsen worker 12 - jepsen.util 9552 :fail :cas [4 [35 34]] :ValueNotSwapFrom INFO [2019-12-17 12:34:57,894] jepsen worker 12 - jepsen.util 9552 :invoke :cas [4 [23 40]] INFO [2019-12-17 12:34:57,900] jepsen worker 88 - jepsen.util 9268 :invoke :read [29 nil] INFO [2019-12-17 12:34:57,951] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52051,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:34:57,959] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52051,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:34:57,965] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52053,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:34:57,971] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52053,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:34:58,036] jepsen worker 35 - jepsen.util 12275 :invoke :write [11 14] INFO [2019-12-17 12:34:58,038] jepsen worker 60 - jepsen.util 13290 :invoke :read [20 nil] INFO [2019-12-17 12:34:58,044] jepsen worker 83 - jepsen.util 12863 :invoke :read [27 nil] INFO [2019-12-17 12:34:58,055] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52059,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:34:58,058] jepsen worker 28 - jepsen.util 7228 :ok :write [9 4] INFO [2019-12-17 12:34:58,058] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52059,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:34:58,076] jepsen worker 28 - jepsen.util 7228 :invoke :read [9 nil] INFO [2019-12-17 12:34:58,098] jepsen worker 67 - jepsen.util 12397 :invoke :read [22 nil] INFO [2019-12-17 12:34:58,118] jepsen worker 13 - jepsen.util 8653 :fail :cas [4 [4 40]] :ValueNotSwapFrom INFO [2019-12-17 12:34:58,140] jepsen worker 13 - jepsen.util 8653 :invoke :cas [4 [23 12]] INFO [2019-12-17 12:34:58,141] jepsen worker 70 - jepsen.util 9610 :info :cas [23 [4 36]] :RequestCanceledException INFO [2019-12-17 12:34:58,206] jepsen worker 70 - jepsen.util 9700 :invoke :write [23 47] INFO [2019-12-17 12:34:58,232] jepsen worker 53 - jepsen.util 8153 :fail :cas [17 [38 43]] :ValueNotSwapFrom INFO [2019-12-17 12:34:58,237] jepsen worker 62 - jepsen.util 12392 :info :cas [20 [9 19]] :RequestCanceledException INFO [2019-12-17 12:34:58,240] jepsen worker 53 - jepsen.util 8153 :invoke :cas [17 [3 26]] INFO [2019-12-17 12:34:58,259] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52072,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:34:58,263] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52072,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:34:58,270] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52077,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:34:58,273] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52077,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:34:58,273] jepsen worker 54 - jepsen.util 9864 :fail :cas [18 [4 8]] :ValueNotSwapFrom INFO [2019-12-17 12:34:58,281] jepsen worker 54 - jepsen.util 9864 :invoke :write [18 21] INFO [2019-12-17 12:34:58,286] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52081,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:34:58,291] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52081,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:34:58,293] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52084,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:34:58,294] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52084,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:34:58,306] jepsen worker 62 - jepsen.util 12482 :invoke :read [20 nil] INFO [2019-12-17 12:34:58,308] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:34:58,313] jepsen nemesis - couchbase.util Rebalance check count: 403 INFO [2019-12-17 12:34:58,314] jepsen nemesis - couchbase.util Rebalance stuck count: 384 INFO [2019-12-17 12:34:58,323] jepsen worker 78 - jepsen.util 8808 :fail :read [26 nil] :RequestTimeoutException INFO [2019-12-17 12:34:58,323] jepsen worker 78 - jepsen.util 8808 :invoke :write [26 31] INFO [2019-12-17 12:34:58,360] jepsen worker 47 - jepsen.util 9497 :fail :cas [15 [14 23]] :ValueNotSwapFrom INFO [2019-12-17 12:34:58,361] jepsen worker 25 - jepsen.util 12985 :info :cas [8 [18 2]] :RequestCanceledException INFO [2019-12-17 12:34:58,361] jepsen worker 47 - jepsen.util 9497 :invoke :cas [15 [0 29]] INFO [2019-12-17 12:34:58,390] jepsen worker 25 - jepsen.util 13075 :invoke :cas [8 [6 24]] INFO [2019-12-17 12:34:58,397] jepsen worker 52 - jepsen.util 9322 :ok :write [17 42] INFO [2019-12-17 12:34:58,398] jepsen worker 52 - jepsen.util 9322 :invoke :cas [17 [29 6]] INFO [2019-12-17 12:34:58,420] jepsen worker 33 - jepsen.util 12183 :info :write [11 42] :RequestCanceledException INFO [2019-12-17 12:34:58,431] jepsen worker 0 - jepsen.util 12870 :info :cas [0 [8 14]] :RequestCanceledException INFO [2019-12-17 12:34:58,453] jepsen worker 71 - jepsen.util 9431 :ok :read [23 20] INFO [2019-12-17 12:34:58,456] jepsen worker 71 - jepsen.util 9431 :invoke :write [23 35] INFO [2019-12-17 12:34:58,467] jepsen worker 33 - jepsen.util 12273 :invoke :cas [11 [19 6]] INFO [2019-12-17 12:34:58,476] jepsen worker 0 - jepsen.util 12960 :invoke :read [0 nil] INFO [2019-12-17 12:34:58,490] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52095,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:34:58,491] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52095,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:34:58,491] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52096,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:34:58,491] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52096,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:34:58,491] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52099,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:34:58,491] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52099,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:34:58,495] jepsen worker 57 - jepsen.util 11937 :info :cas [19 [14 13]] :RequestCanceledException INFO [2019-12-17 12:34:58,495] jepsen worker 85 - jepsen.util 12865 :info :cas [28 [17 21]] :RequestCanceledException INFO [2019-12-17 12:34:58,511] jepsen worker 26 - jepsen.util 12806 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:34:58,515] jepsen worker 26 - jepsen.util 12806 :invoke :cas [8 [42 22]] INFO [2019-12-17 12:34:58,530] jepsen worker 57 - jepsen.util 12027 :invoke :read [19 nil] INFO [2019-12-17 12:34:58,536] jepsen worker 56 - jepsen.util 9686 :info :cas [18 [18 20]] :RequestCanceledException INFO [2019-12-17 12:34:58,546] jepsen worker 20 - jepsen.util 8660 :ok :write [6 40] INFO [2019-12-17 12:34:58,546] jepsen worker 85 - jepsen.util 12955 :invoke :cas [28 [28 27]] INFO [2019-12-17 12:34:58,547] jepsen worker 58 - jepsen.util 12928 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:34:58,548] jepsen worker 20 - jepsen.util 8660 :invoke :write [6 16] INFO [2019-12-17 12:34:58,548] jepsen worker 58 - jepsen.util 12928 :invoke :cas [19 [27 11]] INFO [2019-12-17 12:34:58,553] jepsen worker 56 - jepsen.util 9776 :invoke :read [18 nil] INFO [2019-12-17 12:34:58,589] jepsen worker 73 - jepsen.util 8623 :ok :read [24 49] INFO [2019-12-17 12:34:58,590] jepsen worker 73 - jepsen.util 8623 :invoke :write [24 10] INFO [2019-12-17 12:34:58,634] jepsen worker 66 - jepsen.util 11946 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:34:58,635] jepsen worker 80 - jepsen.util 8630 :info :write [26 45] :RequestCanceledException INFO [2019-12-17 12:34:58,635] jepsen worker 66 - jepsen.util 11946 :invoke :write [22 13] INFO [2019-12-17 12:34:58,635] jepsen worker 4 - jepsen.util 8554 :fail :cas [1 [21 9]] :ValueNotSwapFrom INFO [2019-12-17 12:34:58,638] jepsen worker 4 - jepsen.util 8554 :invoke :cas [1 [24 42]] INFO [2019-12-17 12:34:58,658] jepsen worker 80 - jepsen.util 8720 :invoke :read [26 nil] INFO [2019-12-17 12:34:58,669] jepsen worker 29 - jepsen.util 7499 :fail :cas [9 [12 24]] :ValueNotSwapFrom INFO [2019-12-17 12:34:58,670] jepsen worker 72 - jepsen.util 9072 :fail :cas [24 [23 10]] :ValueNotSwapFrom INFO [2019-12-17 12:34:58,671] jepsen worker 29 - jepsen.util 7499 :invoke :write [9 49] INFO [2019-12-17 12:34:58,674] jepsen worker 72 - jepsen.util 9072 :invoke :read [24 nil] INFO [2019-12-17 12:34:58,686] jepsen worker 51 - jepsen.util 8331 :fail :cas [17 [22 40]] :ValueNotSwapFrom INFO [2019-12-17 12:34:58,686] jepsen worker 51 - jepsen.util 8331 :invoke :write [17 47] INFO [2019-12-17 12:34:58,689] jepsen worker 55 - jepsen.util 9775 :ok :read [18 28] INFO [2019-12-17 12:34:58,690] jepsen worker 55 - jepsen.util 9775 :invoke :write [18 27] INFO [2019-12-17 12:34:58,731] jepsen worker 87 - jepsen.util 8457 :fail :read [29 nil] :RequestTimeoutException INFO [2019-12-17 12:34:58,732] jepsen worker 87 - jepsen.util 8457 :invoke :cas [29 [43 37]] INFO [2019-12-17 12:34:58,733] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52113,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:34:58,733] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52113,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:34:58,735] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52116,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:34:58,742] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52116,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:34:58,742] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52117,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:34:58,744] jepsen worker 18 - jepsen.util 9288 :fail :cas [6 [4 45]] :ValueNotSwapFrom INFO [2019-12-17 12:34:58,747] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52117,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:34:58,748] jepsen worker 18 - jepsen.util 9288 :invoke :read [6 nil] INFO [2019-12-17 12:34:58,749] jepsen worker 9 - jepsen.util 9549 :ok :read [3 0] INFO [2019-12-17 12:34:58,751] jepsen worker 9 - jepsen.util 9549 :invoke :cas [3 [30 35]] INFO [2019-12-17 12:34:58,792] jepsen worker 45 - jepsen.util 9315 :ok :read [15 37] INFO [2019-12-17 12:34:58,792] jepsen worker 45 - jepsen.util 9315 :invoke :read [15 nil] INFO [2019-12-17 12:34:58,813] jepsen worker 42 - jepsen.util 13272 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:34:58,814] jepsen worker 42 - jepsen.util 13272 :invoke :read [14 nil] INFO [2019-12-17 12:34:58,835] jepsen worker 17 - jepsen.util 12167 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:34:58,836] jepsen worker 17 - jepsen.util 12167 :invoke :read [5 nil] INFO [2019-12-17 12:34:58,837] jepsen worker 22 - jepsen.util 12082 :info :cas [7 [13 18]] :RequestCanceledException INFO [2019-12-17 12:34:58,838] jepsen worker 40 - jepsen.util 12370 :info :cas [13 [17 31]] :RequestCanceledException INFO [2019-12-17 12:34:58,839] jepsen worker 24 - jepsen.util 11274 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:34:58,839] jepsen worker 24 - jepsen.util 11274 :invoke :read [8 nil] INFO [2019-12-17 12:34:58,862] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52132,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:34:58,874] jepsen worker 54 - jepsen.util 9864 :ok :write [18 21] INFO [2019-12-17 12:34:58,878] jepsen worker 84 - jepsen.util 11694 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:34:58,882] jepsen worker 22 - jepsen.util 12172 :invoke :write [7 35] INFO [2019-12-17 12:34:58,891] jepsen worker 40 - jepsen.util 12460 :invoke :write [13 2] INFO [2019-12-17 12:34:58,900] jepsen worker 70 - jepsen.util 9700 :ok :write [23 47] INFO [2019-12-17 12:34:58,900] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52132,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:34:58,900] jepsen worker 54 - jepsen.util 9864 :invoke :write [18 44] INFO [2019-12-17 12:34:58,900] jepsen worker 84 - jepsen.util 11694 :invoke :read [28 nil] INFO [2019-12-17 12:34:58,901] jepsen worker 70 - jepsen.util 9700 :invoke :cas [23 [20 9]] INFO [2019-12-17 12:34:58,943] jepsen worker 77 - jepsen.util 12677 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:34:58,944] jepsen worker 77 - jepsen.util 12677 :invoke :read [25 nil] INFO [2019-12-17 12:34:58,972] jepsen worker 27 - jepsen.util 8757 :fail :cas [9 [35 14]] :ValueNotSwapFrom INFO [2019-12-17 12:34:58,973] jepsen worker 27 - jepsen.util 8757 :invoke :cas [9 [48 8]] INFO [2019-12-17 12:34:59,014] jepsen worker 1 - jepsen.util 12331 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:34:59,018] jepsen worker 1 - jepsen.util 12331 :invoke :read [0 nil] INFO [2019-12-17 12:34:59,050] jepsen worker 15 - jepsen.util 11265 :info :cas [5 [9 13]] :RequestCanceledException INFO [2019-12-17 12:34:59,063] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52141,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:34:59,066] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52141,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:34:59,066] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52144,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:34:59,073] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52144,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:34:59,088] jepsen worker 15 - jepsen.util 11355 :invoke :cas [5 [37 36]] INFO [2019-12-17 12:34:59,100] jepsen worker 8 - jepsen.util 11798 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:34:59,102] jepsen worker 8 - jepsen.util 11798 :invoke :cas [2 [3 3]] INFO [2019-12-17 12:34:59,197] jepsen worker 31 - jepsen.util 8041 :ok :read [10 34] INFO [2019-12-17 12:34:59,199] jepsen worker 31 - jepsen.util 8041 :invoke :cas [10 [42 8]] INFO [2019-12-17 12:34:59,217] jepsen worker 46 - jepsen.util 9046 :ok :write [15 46] INFO [2019-12-17 12:34:59,218] jepsen worker 46 - jepsen.util 9046 :invoke :cas [15 [38 13]] INFO [2019-12-17 12:34:59,232] jepsen worker 69 - jepsen.util 9159 :ok :read [23 20] INFO [2019-12-17 12:34:59,234] jepsen worker 14 - jepsen.util 9014 :fail :cas [4 [46 11]] :ValueNotSwapFrom INFO [2019-12-17 12:34:59,234] jepsen worker 14 - jepsen.util 9014 :invoke :read [4 nil] INFO [2019-12-17 12:34:59,244] jepsen worker 69 - jepsen.util 9159 :invoke :read [23 nil] INFO [2019-12-17 12:34:59,253] jepsen worker 75 - jepsen.util 11595 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:34:59,256] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52161,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:34:59,259] jepsen worker 75 - jepsen.util 11595 :invoke :write [25 7] INFO [2019-12-17 12:34:59,262] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52161,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:34:59,271] jepsen worker 6 - jepsen.util 12696 :info :write [2 1] :RequestCanceledException INFO [2019-12-17 12:34:59,293] jepsen worker 37 - jepsen.util 8497 :fail :cas [12 [2 30]] :ValueNotSwapFrom INFO [2019-12-17 12:34:59,295] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52151,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:34:59,295] jepsen worker 37 - jepsen.util 8497 :invoke :read [12 nil] INFO [2019-12-17 12:34:59,296] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52151,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:34:59,307] jepsen worker 44 - jepsen.util 12554 :info :write [14 8] :RequestCanceledException INFO [2019-12-17 12:34:59,310] jepsen worker 74 - jepsen.util 9074 :ok :write [24 8] INFO [2019-12-17 12:34:59,318] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52152,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:34:59,319] jepsen worker 74 - jepsen.util 9074 :invoke :read [24 nil] INFO [2019-12-17 12:34:59,321] jepsen worker 61 - jepsen.util 13111 :info :write [20 24] :RequestCanceledException INFO [2019-12-17 12:34:59,322] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52152,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:34:59,325] jepsen worker 64 - jepsen.util 7264 :ok :read [21 3] INFO [2019-12-17 12:34:59,334] jepsen worker 64 - jepsen.util 7264 :invoke :cas [21 [30 21]] INFO [2019-12-17 12:34:59,339] jepsen worker 48 - jepsen.util 11298 :info :write [16 24] :RequestCanceledException INFO [2019-12-17 12:34:59,348] jepsen worker 6 - jepsen.util 12786 :invoke :read [2 nil] INFO [2019-12-17 12:34:59,367] jepsen worker 39 - jepsen.util 11829 :info :cas [13 [2 29]] :RequestCanceledException INFO [2019-12-17 12:34:59,375] jepsen worker 5 - jepsen.util 8375 :fail :cas [1 [1 19]] :ValueNotSwapFrom INFO [2019-12-17 12:34:59,376] jepsen worker 5 - jepsen.util 8375 :invoke :cas [1 [23 40]] INFO [2019-12-17 12:34:59,400] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:34:59,403] jepsen worker 7 - jepsen.util 12337 :info :write [2 47] :RequestCanceledException INFO [2019-12-17 12:34:59,409] jepsen nemesis - couchbase.util Rebalance check count: 404 INFO [2019-12-17 12:34:59,409] jepsen nemesis - couchbase.util Rebalance stuck count: 385 INFO [2019-12-17 12:34:59,412] jepsen worker 44 - jepsen.util 12644 :invoke :read [14 nil] INFO [2019-12-17 12:34:59,418] jepsen worker 43 - jepsen.util 12013 :info :write [14 0] :RequestCanceledException INFO [2019-12-17 12:34:59,438] jepsen worker 61 - jepsen.util 13201 :invoke :write [20 36] INFO [2019-12-17 12:34:59,443] jepsen worker 19 - jepsen.util 9109 :ok :write [6 16] INFO [2019-12-17 12:34:59,443] jepsen worker 19 - jepsen.util 9109 :invoke :read [6 nil] INFO [2019-12-17 12:34:59,443] jepsen worker 86 - jepsen.util 13046 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:34:59,444] jepsen worker 86 - jepsen.util 13046 :invoke :write [28 41] INFO [2019-12-17 12:34:59,454] jepsen worker 79 - jepsen.util 9439 :fail :cas [26 [47 15]] :ValueNotSwapFrom INFO [2019-12-17 12:34:59,459] jepsen worker 79 - jepsen.util 9439 :invoke :write [26 39] INFO [2019-12-17 12:34:59,461] jepsen worker 39 - jepsen.util 11919 :invoke :read [13 nil] INFO [2019-12-17 12:34:59,477] jepsen worker 7 - jepsen.util 12427 :invoke :read [2 nil] INFO [2019-12-17 12:34:59,478] jepsen worker 48 - jepsen.util 11388 :invoke :read [16 nil] INFO [2019-12-17 12:34:59,480] jepsen worker 43 - jepsen.util 12103 :invoke :write [14 41] INFO [2019-12-17 12:34:59,482] jepsen worker 30 - jepsen.util 8940 :fail :cas [10 [44 17]] :ValueNotSwapFrom INFO [2019-12-17 12:34:59,485] jepsen worker 30 - jepsen.util 8940 :invoke :read [10 nil] INFO [2019-12-17 12:34:59,518] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52180,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:34:59,519] jepsen worker 82 - jepsen.util 12952 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:34:59,519] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52180,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:34:59,519] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52182,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:34:59,519] jepsen worker 82 - jepsen.util 12952 :invoke :cas [27 [35 0]] INFO [2019-12-17 12:34:59,520] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52182,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:34:59,526] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52183,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:34:59,527] jepsen worker 63 - jepsen.util 7893 :ok :write [21 21] INFO [2019-12-17 12:34:59,533] jepsen worker 63 - jepsen.util 7893 :invoke :cas [21 [46 31]] INFO [2019-12-17 12:34:59,534] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52183,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:34:59,544] jepsen worker 81 - jepsen.util 11961 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:34:59,547] jepsen worker 81 - jepsen.util 11961 :invoke :write [27 25] INFO [2019-12-17 12:34:59,577] jepsen worker 89 - jepsen.util 9089 :fail :cas [29 [25 20]] :ValueNotSwapFrom INFO [2019-12-17 12:34:59,578] jepsen worker 89 - jepsen.util 9089 :invoke :cas [29 [29 42]] INFO [2019-12-17 12:34:59,618] jepsen worker 49 - jepsen.util 11569 :info :write [16 48] :RequestCanceledException INFO [2019-12-17 12:34:59,671] jepsen worker 76 - jepsen.util 12676 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:34:59,672] jepsen worker 76 - jepsen.util 12676 :invoke :read [25 nil] INFO [2019-12-17 12:34:59,711] jepsen worker 49 - jepsen.util 11659 :invoke :read [16 nil] INFO [2019-12-17 12:34:59,727] jepsen worker 36 - jepsen.util 9396 :ok :read [12 11] INFO [2019-12-17 12:34:59,735] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52196,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:34:59,738] jepsen worker 36 - jepsen.util 9396 :invoke :cas [12 [29 16]] INFO [2019-12-17 12:34:59,738] jepsen worker 23 - jepsen.util 11993 :info :cas [7 [38 10]] :RequestCanceledException INFO [2019-12-17 12:34:59,739] jepsen worker 34 - jepsen.util 12364 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:34:59,739] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52196,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:34:59,740] jepsen worker 34 - jepsen.util 12364 :invoke :read [11 nil] INFO [2019-12-17 12:34:59,747] jepsen worker 3 - jepsen.util 8913 :ok :write [1 9] INFO [2019-12-17 12:34:59,748] jepsen worker 3 - jepsen.util 8913 :invoke :write [1 43] INFO [2019-12-17 12:34:59,810] jepsen worker 23 - jepsen.util 12083 :invoke :read [7 nil] INFO [2019-12-17 12:34:59,830] jepsen worker 53 - jepsen.util 8153 :fail :cas [17 [3 26]] :ValueNotSwapFrom INFO [2019-12-17 12:34:59,831] jepsen worker 53 - jepsen.util 8153 :invoke :write [17 25] INFO [2019-12-17 12:34:59,831] jepsen worker 38 - jepsen.util 9398 :info :cas [12 [26 22]] :RequestCanceledException INFO [2019-12-17 12:34:59,847] jepsen worker 38 - jepsen.util 9488 :invoke :cas [12 [13 3]] INFO [2019-12-17 12:34:59,864] jepsen worker 50 - jepsen.util 12830 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:34:59,871] jepsen worker 28 - jepsen.util 7228 :ok :read [9 4] INFO [2019-12-17 12:34:59,873] jepsen worker 52 - jepsen.util 9322 :fail :cas [17 [29 6]] :ValueNotSwapFrom INFO [2019-12-17 12:34:59,902] jepsen worker 2 - jepsen.util 11882 :info :write [0 18] :RequestCanceledException INFO [2019-12-17 12:34:59,903] jepsen worker 65 - jepsen.util 7445 :ok :read [21 3] INFO [2019-12-17 12:34:59,926] jepsen worker 50 - jepsen.util 12830 :invoke :cas [16 [26 44]] INFO [2019-12-17 12:34:59,926] jepsen worker 28 - jepsen.util 7228 :invoke :write [9 38] INFO [2019-12-17 12:34:59,927] jepsen worker 65 - jepsen.util 7445 :invoke :read [21 nil] INFO [2019-12-17 12:34:59,928] jepsen worker 52 - jepsen.util 9322 :invoke :cas [17 [43 20]] INFO [2019-12-17 12:34:59,929] jepsen worker 12 - jepsen.util 9552 :info :cas [4 [23 40]] :RequestCanceledException INFO [2019-12-17 12:34:59,936] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52208,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:34:59,937] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52208,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:34:59,944] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52210,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:34:59,946] jepsen worker 2 - jepsen.util 11972 :invoke :read [0 nil] INFO [2019-12-17 12:34:59,946] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52210,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:34:59,966] jepsen worker 12 - jepsen.util 9642 :invoke :write [4 11] INFO [2019-12-17 12:34:59,992] jepsen worker 51 - jepsen.util 8331 :ok :write [17 47] INFO [2019-12-17 12:34:59,994] jepsen worker 51 - jepsen.util 8331 :invoke :cas [17 [47 30]] INFO [2019-12-17 12:34:59,996] jepsen worker 32 - jepsen.util 9122 :info :write [10 46] :RequestCanceledException INFO [2019-12-17 12:35:00,008] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52220,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:35:00,009] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52220,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:35:00,009] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52223,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:35:00,010] jepsen worker 32 - jepsen.util 9212 :invoke :cas [10 [17 21]] INFO [2019-12-17 12:35:00,020] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52223,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:35:00,022] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52224,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:35:00,025] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52224,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:35:00,069] jepsen worker 73 - jepsen.util 8623 :ok :write [24 10] INFO [2019-12-17 12:35:00,070] jepsen worker 73 - jepsen.util 8623 :invoke :write [24 34] INFO [2019-12-17 12:35:00,120] jepsen worker 60 - jepsen.util 13290 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:35:00,121] jepsen worker 60 - jepsen.util 13290 :invoke :write [20 23] INFO [2019-12-17 12:35:00,136] jepsen worker 83 - jepsen.util 12863 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:35:00,136] jepsen worker 83 - jepsen.util 12863 :invoke :read [27 nil] INFO [2019-12-17 12:35:00,150] jepsen worker 67 - jepsen.util 12397 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:35:00,151] jepsen worker 67 - jepsen.util 12397 :invoke :cas [22 [19 28]] INFO [2019-12-17 12:35:00,158] jepsen worker 11 - jepsen.util 9101 :ok :write [3 28] INFO [2019-12-17 12:35:00,159] jepsen worker 11 - jepsen.util 9101 :invoke :write [3 8] INFO [2019-12-17 12:35:00,220] jepsen worker 59 - jepsen.util 11579 :info :write [19 20] :RequestCanceledException INFO [2019-12-17 12:35:00,235] jepsen worker 41 - jepsen.util 13001 :info :write [13 12] :RequestCanceledException INFO [2019-12-17 12:35:00,270] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52230,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:35:00,272] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52230,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:35:00,274] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52236,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:35:00,274] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52236,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:35:00,287] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52238,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:35:00,287] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52238,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:35:00,290] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52242,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:35:00,292] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52242,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:35:00,297] jepsen worker 59 - jepsen.util 11669 :invoke :write [19 35] INFO [2019-12-17 12:35:00,325] jepsen worker 41 - jepsen.util 13091 :invoke :write [13 17] INFO [2019-12-17 12:35:00,333] jepsen worker 62 - jepsen.util 12482 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:35:00,334] jepsen worker 62 - jepsen.util 12482 :invoke :write [20 37] INFO [2019-12-17 12:35:00,367] jepsen worker 63 - jepsen.util 7893 :fail :cas [21 [46 31]] :ValueNotSwapFrom INFO [2019-12-17 12:35:00,369] jepsen worker 63 - jepsen.util 7893 :invoke :cas [21 [43 46]] INFO [2019-12-17 12:35:00,398] jepsen worker 47 - jepsen.util 9497 :info :cas [15 [0 29]] :RequestCanceledException INFO [2019-12-17 12:35:00,413] jepsen worker 20 - jepsen.util 8660 :ok :write [6 16] INFO [2019-12-17 12:35:00,414] jepsen worker 20 - jepsen.util 8660 :invoke :read [6 nil] INFO [2019-12-17 12:35:00,421] jepsen worker 47 - jepsen.util 9587 :invoke :write [15 2] INFO [2019-12-17 12:35:00,447] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52249,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:35:00,448] jepsen worker 68 - jepsen.util 11588 :info :write [22 19] :RequestCanceledException INFO [2019-12-17 12:35:00,450] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52249,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:35:00,451] jepsen worker 21 - jepsen.util 11991 :info :write [7 39] :RequestCanceledException INFO [2019-12-17 12:35:00,455] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52253,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:35:00,455] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52253,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:35:00,457] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:35:00,457] jepsen nemesis - couchbase.util Rebalance check count: 405 INFO [2019-12-17 12:35:00,457] jepsen nemesis - couchbase.util Rebalance stuck count: 386 INFO [2019-12-17 12:35:00,463] jepsen worker 16 - jepsen.util 13246 :info :write [5 26] :RequestCanceledException INFO [2019-12-17 12:35:00,470] jepsen worker 29 - jepsen.util 7499 :ok :write [9 49] INFO [2019-12-17 12:35:00,482] jepsen worker 29 - jepsen.util 7499 :invoke :write [9 48] INFO [2019-12-17 12:35:00,499] jepsen worker 25 - jepsen.util 13075 :info :cas [8 [6 24]] :RequestCanceledException INFO [2019-12-17 12:35:00,508] jepsen worker 35 - jepsen.util 12275 :info :write [11 14] :RequestCanceledException INFO [2019-12-17 12:35:00,517] jepsen worker 33 - jepsen.util 12273 :info :cas [11 [19 6]] :RequestCanceledException INFO [2019-12-17 12:35:00,543] jepsen worker 78 - jepsen.util 8808 :info :write [26 31] :RequestCanceledException INFO [2019-12-17 12:35:00,545] jepsen worker 0 - jepsen.util 12960 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:35:00,984] jepsen worker 14 - jepsen.util 9014 :ok :read [4 44] INFO [2019-12-17 12:35:00,984] jepsen worker 14 - jepsen.util 9014 :invoke :read [4 nil] INFO [2019-12-17 12:35:00,984] jepsen worker 57 - jepsen.util 12027 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:35:00,986] jepsen worker 80 - jepsen.util 8720 :ok :read [26 45] INFO [2019-12-17 12:35:00,986] jepsen worker 72 - jepsen.util 9072 :ok :read [24 47] INFO [2019-12-17 12:35:00,987] jepsen worker 45 - jepsen.util 9315 :ok :read [15 37] INFO [2019-12-17 12:35:00,987] jepsen worker 69 - jepsen.util 9159 :ok :read [23 47] INFO [2019-12-17 12:35:00,987] jepsen worker 10 - jepsen.util 9100 :fail :cas [3 [40 28]] :ValueNotSwapFrom INFO [2019-12-17 12:35:00,987] jepsen worker 72 - jepsen.util 9072 :invoke :cas [24 [31 16]] INFO [2019-12-17 12:35:00,987] jepsen worker 88 - jepsen.util 9268 :ok :read [29 0] INFO [2019-12-17 12:35:00,987] jepsen worker 0 - jepsen.util 12960 :invoke :cas [0 [9 14]] INFO [2019-12-17 12:35:00,987] jepsen worker 74 - jepsen.util 9074 :ok :read [24 8] INFO [2019-12-17 12:35:00,988] jepsen worker 57 - jepsen.util 12027 :invoke :cas [19 [25 5]] INFO [2019-12-17 12:35:00,988] jepsen worker 70 - jepsen.util 9700 :fail :cas [23 [20 9]] :ValueNotSwapFrom INFO [2019-12-17 12:35:00,988] jepsen worker 80 - jepsen.util 8720 :invoke :cas [26 [43 44]] INFO [2019-12-17 12:35:00,990] jepsen worker 58 - jepsen.util 12928 :info :cas [19 [27 11]] :RequestCanceledException INFO [2019-12-17 12:35:00,990] jepsen worker 85 - jepsen.util 12955 :info :cas [28 [28 27]] :RequestCanceledException INFO [2019-12-17 12:35:00,990] jepsen worker 45 - jepsen.util 9315 :invoke :cas [15 [44 38]] INFO [2019-12-17 12:35:00,990] jepsen worker 87 - jepsen.util 8457 :info :cas [29 [43 37]] :RequestCanceledException INFO [2019-12-17 12:35:00,990] jepsen worker 18 - jepsen.util 9288 :fail :read [6 nil] :RequestTimeoutException INFO [2019-12-17 12:35:00,990] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52259,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:35:00,990] jepsen worker 10 - jepsen.util 9100 :invoke :read [3 nil] INFO [2019-12-17 12:35:00,990] jepsen worker 66 - jepsen.util 11946 :info :write [22 13] :RequestCanceledException INFO [2019-12-17 12:35:00,995] jepsen worker 69 - jepsen.util 9159 :invoke :read [23 nil] INFO [2019-12-17 12:35:00,995] jepsen worker 4 - jepsen.util 8554 :info :cas [1 [24 42]] :RequestCanceledException INFO [2019-12-17 12:35:00,995] jepsen worker 17 - jepsen.util 12167 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:35:00,995] jepsen worker 42 - jepsen.util 13272 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:35:00,996] jepsen worker 70 - jepsen.util 9700 :invoke :cas [23 [18 30]] INFO [2019-12-17 12:35:00,996] jepsen worker 68 - jepsen.util 11678 :invoke :read [22 nil] INFO [2019-12-17 12:35:00,996] jepsen worker 88 - jepsen.util 9268 :invoke :cas [29 [48 14]] INFO [2019-12-17 12:35:00,997] jepsen worker 74 - jepsen.util 9074 :invoke :read [24 nil] INFO [2019-12-17 12:35:00,998] jepsen worker 18 - jepsen.util 9288 :invoke :read [6 nil] INFO [2019-12-17 12:35:01,008] jepsen worker 26 - jepsen.util 12806 :info :cas [8 [42 22]] :RequestCanceledException INFO [2019-12-17 12:35:01,008] jepsen worker 84 - jepsen.util 11694 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:35:01,011] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52259,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:35:01,011] jepsen worker 9 - jepsen.util 9549 :info :cas [3 [30 35]] :RequestCanceledException INFO [2019-12-17 12:35:01,012] jepsen worker 24 - jepsen.util 11274 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:35:01,012] jepsen worker 77 - jepsen.util 12677 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:35:01,013] jepsen worker 42 - jepsen.util 13272 :invoke :read [14 nil] INFO [2019-12-17 12:35:01,020] jepsen worker 27 - jepsen.util 8757 :info :cas [9 [48 8]] :RequestCanceledException INFO [2019-12-17 12:35:01,021] jepsen worker 17 - jepsen.util 12167 :invoke :read [5 nil] INFO [2019-12-17 12:35:01,029] jepsen worker 84 - jepsen.util 11694 :invoke :cas [28 [24 2]] INFO [2019-12-17 12:35:01,031] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52263,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:35:01,032] jepsen worker 25 - jepsen.util 13165 :invoke :cas [8 [23 36]] INFO [2019-12-17 12:35:01,032] jepsen worker 24 - jepsen.util 11274 :invoke :write [8 45] INFO [2019-12-17 12:35:01,036] jepsen worker 71 - jepsen.util 9431 :info :write [23 35] :RequestCanceledException INFO [2019-12-17 12:35:01,038] jepsen worker 77 - jepsen.util 12677 :invoke :cas [25 [17 35]] INFO [2019-12-17 12:35:01,044] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52263,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:35:01,077] jepsen worker 78 - jepsen.util 8898 :invoke :read [26 nil] INFO [2019-12-17 12:35:01,089] jepsen worker 55 - jepsen.util 9775 :ok :write [18 27] INFO [2019-12-17 12:35:01,089] jepsen worker 35 - jepsen.util 12365 :invoke :read [11 nil] INFO [2019-12-17 12:35:01,097] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52261,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:35:01,097] jepsen worker 55 - jepsen.util 9775 :invoke :cas [18 [13 10]] INFO [2019-12-17 12:35:01,108] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52261,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:35:01,109] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52275,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:35:01,110] jepsen worker 1 - jepsen.util 12331 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:35:01,110] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52275,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:35:01,111] jepsen worker 1 - jepsen.util 12331 :invoke :write [0 27] INFO [2019-12-17 12:35:01,111] jepsen worker 27 - jepsen.util 8847 :invoke :read [9 nil] INFO [2019-12-17 12:35:01,115] jepsen worker 4 - jepsen.util 8644 :invoke :read [1 nil] INFO [2019-12-17 12:35:01,115] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52280,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:35:01,115] jepsen worker 58 - jepsen.util 13018 :invoke :cas [19 [21 45]] INFO [2019-12-17 12:35:01,116] jepsen worker 33 - jepsen.util 12363 :invoke :read [11 nil] INFO [2019-12-17 12:35:01,121] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52280,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:35:01,122] jepsen worker 9 - jepsen.util 9639 :invoke :cas [3 [47 23]] INFO [2019-12-17 12:35:01,158] jepsen worker 54 - jepsen.util 9864 :ok :write [18 44] INFO [2019-12-17 12:35:01,160] jepsen worker 54 - jepsen.util 9864 :invoke :cas [18 [7 45]] INFO [2019-12-17 12:35:01,161] jepsen worker 21 - jepsen.util 12081 :invoke :write [7 18] INFO [2019-12-17 12:35:01,164] jepsen worker 71 - jepsen.util 9521 :invoke :read [23 nil] INFO [2019-12-17 12:35:01,165] jepsen worker 15 - jepsen.util 11355 :info :cas [5 [37 36]] :RequestCanceledException INFO [2019-12-17 12:35:01,181] jepsen worker 26 - jepsen.util 12896 :invoke :read [8 nil] INFO [2019-12-17 12:35:01,191] jepsen worker 85 - jepsen.util 13045 :invoke :write [28 39] INFO [2019-12-17 12:35:01,196] jepsen worker 16 - jepsen.util 13336 :invoke :write [5 23] INFO [2019-12-17 12:35:01,197] jepsen worker 66 - jepsen.util 12036 :invoke :read [22 nil] INFO [2019-12-17 12:35:01,213] jepsen worker 8 - jepsen.util 11798 :info :cas [2 [3 3]] :RequestCanceledException INFO [2019-12-17 12:35:01,226] jepsen worker 87 - jepsen.util 8547 :invoke :write [29 43] INFO [2019-12-17 12:35:01,236] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52305,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:35:01,236] jepsen worker 46 - jepsen.util 9046 :info :cas [15 [38 13]] :RequestCanceledException INFO [2019-12-17 12:35:01,240] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52312,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:35:01,240] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52305,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:35:01,242] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52312,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:35:01,247] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52315,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:35:01,251] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52315,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:35:01,293] jepsen worker 46 - jepsen.util 9136 :invoke :cas [15 [18 2]] INFO [2019-12-17 12:35:01,311] jepsen worker 15 - jepsen.util 11445 :invoke :cas [5 [41 12]] INFO [2019-12-17 12:35:01,315] jepsen worker 8 - jepsen.util 11888 :invoke :write [2 34] INFO [2019-12-17 12:35:01,318] jepsen worker 22 - jepsen.util 12172 :info :write [7 35] :RequestCanceledException INFO [2019-12-17 12:35:01,368] jepsen worker 6 - jepsen.util 12786 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:35:01,369] jepsen worker 6 - jepsen.util 12786 :invoke :cas [2 [35 41]] INFO [2019-12-17 12:35:01,410] jepsen worker 22 - jepsen.util 12262 :invoke :read [7 nil] INFO [2019-12-17 12:35:01,432] jepsen worker 5 - jepsen.util 8375 :info :cas [1 [23 40]] :RequestCanceledException INFO [2019-12-17 12:35:01,432] jepsen worker 40 - jepsen.util 12460 :info :write [13 2] :RequestCanceledException INFO [2019-12-17 12:35:01,434] jepsen worker 44 - jepsen.util 12644 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:35:01,437] jepsen worker 44 - jepsen.util 12644 :invoke :read [14 nil] INFO [2019-12-17 12:35:01,457] jepsen worker 12 - jepsen.util 9642 :ok :write [4 11] INFO [2019-12-17 12:35:01,457] jepsen worker 12 - jepsen.util 9642 :invoke :read [4 nil] INFO [2019-12-17 12:35:01,479] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52326,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:35:01,481] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52326,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:35:01,481] jepsen worker 40 - jepsen.util 12550 :invoke :cas [13 [20 11]] INFO [2019-12-17 12:35:01,492] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52327,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:35:01,498] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52327,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:35:01,512] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:35:01,517] jepsen worker 5 - jepsen.util 8465 :invoke :cas [1 [3 10]] INFO [2019-12-17 12:35:01,520] jepsen nemesis - couchbase.util Rebalance check count: 406 INFO [2019-12-17 12:35:01,525] jepsen nemesis - couchbase.util Rebalance stuck count: 387 INFO [2019-12-17 12:35:01,525] jepsen worker 39 - jepsen.util 11919 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:35:01,527] jepsen worker 39 - jepsen.util 11919 :invoke :read [13 nil] INFO [2019-12-17 12:35:01,535] jepsen worker 7 - jepsen.util 12427 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:35:01,536] jepsen worker 7 - jepsen.util 12427 :invoke :read [2 nil] INFO [2019-12-17 12:35:01,547] jepsen worker 19 - jepsen.util 9109 :fail :read [6 nil] :RequestTimeoutException INFO [2019-12-17 12:35:01,548] jepsen worker 19 - jepsen.util 9109 :invoke :cas [6 [33 45]] INFO [2019-12-17 12:35:01,562] jepsen worker 48 - jepsen.util 11388 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:35:01,565] jepsen worker 48 - jepsen.util 11388 :invoke :cas [16 [34 8]] INFO [2019-12-17 12:35:01,600] jepsen worker 30 - jepsen.util 8940 :fail :read [10 nil] :RequestTimeoutException INFO [2019-12-17 12:35:01,600] jepsen worker 30 - jepsen.util 8940 :invoke :cas [10 [21 18]] INFO [2019-12-17 12:35:01,620] jepsen worker 82 - jepsen.util 12952 :info :cas [27 [35 0]] :RequestCanceledException INFO [2019-12-17 12:35:01,704] jepsen worker 31 - jepsen.util 8041 :fail :cas [10 [42 8]] :ValueNotSwapFrom INFO [2019-12-17 12:35:01,705] jepsen worker 31 - jepsen.util 8041 :invoke :write [10 30] INFO [2019-12-17 12:35:01,711] jepsen worker 82 - jepsen.util 13042 :invoke :read [27 nil] INFO [2019-12-17 12:35:01,723] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52346,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:35:01,724] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52346,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:35:01,735] jepsen worker 49 - jepsen.util 11659 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:35:01,736] jepsen worker 49 - jepsen.util 11659 :invoke :write [16 34] INFO [2019-12-17 12:35:01,763] jepsen worker 76 - jepsen.util 12676 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:35:01,769] jepsen worker 76 - jepsen.util 12676 :invoke :cas [25 [20 48]] INFO [2019-12-17 12:35:01,813] jepsen worker 79 - jepsen.util 9439 :ok :write [26 39] INFO [2019-12-17 12:35:01,814] jepsen worker 79 - jepsen.util 9439 :invoke :read [26 nil] INFO [2019-12-17 12:35:01,832] jepsen worker 34 - jepsen.util 12364 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:35:01,843] jepsen worker 34 - jepsen.util 12364 :invoke :cas [11 [41 9]] INFO [2019-12-17 12:35:01,861] jepsen worker 47 - jepsen.util 9587 :ok :write [15 2] INFO [2019-12-17 12:35:01,865] jepsen worker 47 - jepsen.util 9587 :invoke :write [15 1] INFO [2019-12-17 12:35:01,875] jepsen worker 11 - jepsen.util 9101 :ok :write [3 8] INFO [2019-12-17 12:35:01,879] jepsen worker 11 - jepsen.util 9101 :invoke :write [3 47] INFO [2019-12-17 12:35:01,905] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52358,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:35:01,906] jepsen worker 23 - jepsen.util 12083 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:35:01,906] jepsen worker 61 - jepsen.util 13201 :info :write [20 36] :RequestCanceledException INFO [2019-12-17 12:35:01,912] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52358,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:35:01,914] jepsen worker 23 - jepsen.util 12083 :invoke :write [7 17] INFO [2019-12-17 12:35:01,917] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52361,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:35:01,917] jepsen worker 75 - jepsen.util 11595 :info :write [25 7] :RequestCanceledException INFO [2019-12-17 12:35:01,923] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52361,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:35:01,924] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52362,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:35:01,925] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52362,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:35:01,947] jepsen worker 81 - jepsen.util 11961 :info :write [27 25] :RequestCanceledException INFO [2019-12-17 12:35:01,972] jepsen worker 50 - jepsen.util 12830 :info :cas [16 [26 44]] :RequestCanceledException INFO [2019-12-17 12:35:02,033] jepsen worker 3 - jepsen.util 8913 :ok :write [1 43] INFO [2019-12-17 12:35:02,034] jepsen worker 3 - jepsen.util 8913 :invoke :read [1 nil] INFO [2019-12-17 12:35:02,034] jepsen worker 65 - jepsen.util 7445 :fail :read [21 nil] :RequestTimeoutException INFO [2019-12-17 12:35:02,035] jepsen worker 2 - jepsen.util 11972 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:35:02,035] jepsen worker 86 - jepsen.util 13046 :info :write [28 41] :RequestCanceledException INFO [2019-12-17 12:35:02,040] jepsen worker 2 - jepsen.util 11972 :invoke :cas [0 [42 41]] INFO [2019-12-17 12:35:02,041] jepsen worker 65 - jepsen.util 7445 :invoke :cas [21 [1 7]] INFO [2019-12-17 12:35:02,112] jepsen worker 75 - jepsen.util 11685 :invoke :write [25 25] INFO [2019-12-17 12:35:02,144] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52369,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:35:02,145] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52369,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:35:02,160] jepsen worker 61 - jepsen.util 13291 :invoke :read [20 nil] INFO [2019-12-17 12:35:02,172] jepsen worker 43 - jepsen.util 12103 :info :write [14 41] :RequestCanceledException INFO [2019-12-17 12:35:02,196] jepsen worker 81 - jepsen.util 12051 :invoke :cas [27 [16 46]] INFO [2019-12-17 12:35:02,212] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52376,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:35:02,214] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52376,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:35:02,252] jepsen worker 83 - jepsen.util 12863 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:35:02,256] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52382,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:35:02,260] jepsen worker 83 - jepsen.util 12863 :invoke :write [27 43] INFO [2019-12-17 12:35:02,270] jepsen worker 67 - jepsen.util 12397 :info :cas [22 [19 28]] :RequestCanceledException INFO [2019-12-17 12:35:02,272] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52381,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:35:02,279] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52382,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:35:02,279] jepsen worker 50 - jepsen.util 12920 :invoke :write [16 14] INFO [2019-12-17 12:35:02,280] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52381,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:35:02,288] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52387,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:35:02,304] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52387,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:35:02,376] jepsen worker 86 - jepsen.util 13136 :invoke :read [28 nil] INFO [2019-12-17 12:35:02,404] jepsen worker 43 - jepsen.util 12193 :invoke :write [14 21] INFO [2019-12-17 12:35:02,424] jepsen worker 56 - jepsen.util 9776 :ok :read [18 21] INFO [2019-12-17 12:35:02,427] jepsen worker 53 - jepsen.util 8153 :info :write [17 25] :RequestCanceledException INFO [2019-12-17 12:35:02,427] jepsen worker 56 - jepsen.util 9776 :invoke :write [18 42] INFO [2019-12-17 12:35:02,455] jepsen worker 67 - jepsen.util 12487 :invoke :read [22 nil] INFO [2019-12-17 12:35:02,465] jepsen worker 28 - jepsen.util 7228 :info :write [9 38] :RequestCanceledException INFO [2019-12-17 12:35:02,504] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52389,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:35:02,513] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52389,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:35:02,516] jepsen worker 53 - jepsen.util 8243 :invoke :cas [17 [44 21]] INFO [2019-12-17 12:35:02,523] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52393,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:35:02,523] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52393,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:35:02,533] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52394,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:35:02,549] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52394,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:35:02,552] jepsen worker 28 - jepsen.util 7318 :invoke :cas [9 [6 17]] INFO [2019-12-17 12:35:02,621] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:35:02,630] jepsen nemesis - couchbase.util Rebalance check count: 407 INFO [2019-12-17 12:35:02,640] jepsen worker 37 - jepsen.util 8497 :ok :read [12 13] INFO [2019-12-17 12:35:02,640] jepsen nemesis - couchbase.util Rebalance stuck count: 388 INFO [2019-12-17 12:35:02,641] jepsen worker 37 - jepsen.util 8497 :invoke :cas [12 [15 4]] INFO [2019-12-17 12:35:02,662] jepsen worker 13 - jepsen.util 8653 :fail :cas [4 [23 12]] :ValueNotSwapFrom INFO [2019-12-17 12:35:02,663] jepsen worker 13 - jepsen.util 8653 :invoke :read [4 nil] INFO [2019-12-17 12:35:02,672] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52399,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:35:02,677] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52399,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:35:02,677] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52402,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:35:02,677] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52402,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:35:02,688] jepsen worker 73 - jepsen.util 8623 :ok :write [24 34] INFO [2019-12-17 12:35:02,689] jepsen worker 73 - jepsen.util 8623 :invoke :read [24 nil] INFO [2019-12-17 12:35:02,740] jepsen worker 74 - jepsen.util 9074 :ok :read [24 10] INFO [2019-12-17 12:35:02,740] jepsen worker 63 - jepsen.util 7893 :fail :cas [21 [43 46]] :ValueNotSwapFrom INFO [2019-12-17 12:35:02,740] jepsen worker 74 - jepsen.util 9074 :invoke :write [24 36] INFO [2019-12-17 12:35:02,747] jepsen worker 63 - jepsen.util 7893 :invoke :cas [21 [33 35]] INFO [2019-12-17 12:35:02,833] jepsen worker 64 - jepsen.util 7264 :fail :cas [21 [30 21]] :ValueNotSwapFrom INFO [2019-12-17 12:35:02,836] jepsen worker 64 - jepsen.util 7264 :invoke :read [21 nil] INFO [2019-12-17 12:35:02,840] jepsen worker 52 - jepsen.util 9322 :fail :cas [17 [43 20]] :ValueNotSwapFrom INFO [2019-12-17 12:35:02,844] jepsen worker 52 - jepsen.util 9322 :invoke :write [17 11] INFO [2019-12-17 12:35:02,864] jepsen worker 29 - jepsen.util 7499 :ok :write [9 48] INFO [2019-12-17 12:35:02,865] jepsen worker 29 - jepsen.util 7499 :invoke :cas [9 [41 35]] INFO [2019-12-17 12:35:02,880] jepsen worker 20 - jepsen.util 8660 :ok :read [6 16] INFO [2019-12-17 12:35:02,881] jepsen worker 20 - jepsen.util 8660 :invoke :write [6 25] INFO [2019-12-17 12:35:02,905] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52417,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:35:02,907] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52417,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:35:03,019] jepsen worker 60 - jepsen.util 13290 :info :write [20 23] :RequestCanceledException INFO [2019-12-17 12:35:03,020] jepsen worker 14 - jepsen.util 9014 :fail :read [4 nil] :RequestTimeoutException INFO [2019-12-17 12:35:03,021] jepsen worker 14 - jepsen.util 9014 :invoke :read [4 nil] INFO [2019-12-17 12:35:03,035] jepsen worker 0 - jepsen.util 12960 :info :cas [0 [9 14]] :RequestCanceledException INFO [2019-12-17 12:35:03,048] jepsen worker 68 - jepsen.util 11678 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:35:03,056] jepsen worker 57 - jepsen.util 12027 :info :cas [19 [25 5]] :RequestCanceledException INFO [2019-12-17 12:35:03,056] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52420,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:35:03,062] jepsen worker 68 - jepsen.util 11678 :invoke :cas [22 [47 38]] INFO [2019-12-17 12:35:03,063] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52420,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:35:03,064] jepsen worker 45 - jepsen.util 9315 :info :cas [15 [44 38]] :RequestCanceledException INFO [2019-12-17 12:35:03,065] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52425,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:35:03,073] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52425,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:35:03,074] jepsen worker 84 - jepsen.util 11694 :info :cas [28 [24 2]] :RequestCanceledException INFO [2019-12-17 12:35:03,100] jepsen worker 69 - jepsen.util 9159 :fail :read [23 nil] :RequestTimeoutException INFO [2019-12-17 12:35:03,101] jepsen worker 69 - jepsen.util 9159 :invoke :cas [23 [9 19]] INFO [2019-12-17 12:35:03,118] jepsen worker 45 - jepsen.util 9405 :invoke :cas [15 [25 31]] INFO [2019-12-17 12:35:03,118] jepsen worker 42 - jepsen.util 13272 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:35:03,120] jepsen worker 42 - jepsen.util 13272 :invoke :write [14 22] INFO [2019-12-17 12:35:03,131] jepsen worker 17 - jepsen.util 12167 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:35:03,138] jepsen worker 17 - jepsen.util 12167 :invoke :write [5 33] INFO [2019-12-17 12:35:03,140] jepsen worker 35 - jepsen.util 12365 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:35:03,146] jepsen worker 35 - jepsen.util 12365 :invoke :read [11 nil] INFO [2019-12-17 12:35:03,147] jepsen worker 60 - jepsen.util 13380 :invoke :read [20 nil] INFO [2019-12-17 12:35:03,152] jepsen worker 25 - jepsen.util 13165 :info :cas [8 [23 36]] :RequestCanceledException INFO [2019-12-17 12:35:03,153] jepsen worker 77 - jepsen.util 12677 :info :cas [25 [17 35]] :RequestCanceledException INFO [2019-12-17 12:35:03,155] jepsen worker 57 - jepsen.util 12117 :invoke :read [19 nil] INFO [2019-12-17 12:35:03,156] jepsen worker 33 - jepsen.util 12363 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:35:03,159] jepsen worker 33 - jepsen.util 12363 :invoke :cas [11 [23 28]] INFO [2019-12-17 12:35:03,162] jepsen worker 9 - jepsen.util 9639 :info :cas [3 [47 23]] :RequestCanceledException INFO [2019-12-17 12:35:03,166] jepsen worker 0 - jepsen.util 13050 :invoke :read [0 nil] INFO [2019-12-17 12:35:03,191] jepsen worker 84 - jepsen.util 11784 :invoke :cas [28 [25 44]] INFO [2019-12-17 12:35:03,196] jepsen worker 54 - jepsen.util 9864 :info :cas [18 [7 45]] :RequestCanceledException INFO [2019-12-17 12:35:03,197] jepsen worker 71 - jepsen.util 9521 :fail :read [23 nil] :RequestTimeoutException INFO [2019-12-17 12:35:03,197] jepsen worker 71 - jepsen.util 9521 :invoke :cas [23 [16 5]] INFO [2019-12-17 12:35:03,205] jepsen worker 9 - jepsen.util 9729 :invoke :cas [3 [11 37]] INFO [2019-12-17 12:35:03,211] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52451,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:35:03,212] jepsen worker 4 - jepsen.util 8644 :fail :read [1 nil] :RequestTimeoutException INFO [2019-12-17 12:35:03,217] jepsen worker 25 - jepsen.util 13255 :invoke :write [8 24] INFO [2019-12-17 12:35:03,217] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52451,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:35:03,222] jepsen worker 4 - jepsen.util 8644 :invoke :read [1 nil] INFO [2019-12-17 12:35:03,222] jepsen worker 26 - jepsen.util 12896 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:35:03,229] jepsen worker 26 - jepsen.util 12896 :invoke :cas [8 [35 22]] INFO [2019-12-17 12:35:03,231] jepsen worker 77 - jepsen.util 12767 :invoke :write [25 11] INFO [2019-12-17 12:35:03,233] jepsen worker 36 - jepsen.util 9396 :fail :cas [12 [29 16]] :ValueNotSwapFrom INFO [2019-12-17 12:35:03,239] jepsen worker 36 - jepsen.util 9396 :invoke :write [12 42] INFO [2019-12-17 12:35:03,249] jepsen worker 58 - jepsen.util 13018 :info :cas [19 [21 45]] :RequestCanceledException INFO [2019-12-17 12:35:03,258] jepsen worker 54 - jepsen.util 9954 :invoke :write [18 10] INFO [2019-12-17 12:35:03,271] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52441,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:35:03,271] jepsen worker 66 - jepsen.util 12036 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:35:03,271] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52441,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:35:03,272] jepsen worker 66 - jepsen.util 12036 :invoke :write [22 2] INFO [2019-12-17 12:35:03,272] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52443,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:35:03,272] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52443,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:35:03,290] jepsen worker 58 - jepsen.util 13108 :invoke :write [19 26] INFO [2019-12-17 12:35:03,417] jepsen worker 15 - jepsen.util 11445 :info :cas [5 [41 12]] :RequestCanceledException INFO [2019-12-17 12:35:03,425] jepsen worker 10 - jepsen.util 9100 :ok :read [3 28] INFO [2019-12-17 12:35:03,446] jepsen worker 22 - jepsen.util 12262 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:35:03,463] jepsen worker 6 - jepsen.util 12786 :info :cas [2 [35 41]] :RequestCanceledException INFO [2019-12-17 12:35:03,499] jepsen worker 41 - jepsen.util 13091 :info :write [13 17] :RequestCanceledException INFO [2019-12-17 12:35:03,510] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52463,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:35:03,538] jepsen worker 62 - jepsen.util 12482 :info :write [20 37] :RequestCanceledException INFO [2019-12-17 12:35:03,541] jepsen worker 44 - jepsen.util 12644 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:35:03,544] jepsen worker 59 - jepsen.util 11669 :info :write [19 35] :RequestCanceledException INFO [2019-12-17 12:35:03,564] jepsen worker 40 - jepsen.util 12550 :info :cas [13 [20 11]] :RequestCanceledException INFO [2019-12-17 12:35:03,642] jepsen worker 21 - jepsen.util 12081 :info :write [7 18] :RequestCanceledException INFO [2019-12-17 12:35:03,645] jepsen worker 7 - jepsen.util 12427 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:35:03,692] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52471,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:35:03,692] jepsen worker 85 - jepsen.util 13045 :info :write [28 39] :RequestCanceledException INFO [2019-12-17 12:35:03,696] jepsen worker 32 - jepsen.util 9212 :fail :cas [10 [17 21]] :ValueNotSwapFrom INFO [2019-12-17 12:35:03,733] jepsen worker 48 - jepsen.util 11388 :info :cas [16 [34 8]] :RequestCanceledException INFO [2019-12-17 12:35:03,749] jepsen worker 39 - jepsen.util 11919 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:35:03,773] jepsen worker 38 - jepsen.util 9488 :info :cas [12 [13 3]] :RequestCanceledException INFO [2019-12-17 12:35:03,791] jepsen worker 24 - jepsen.util 11274 :info :write [8 45] :RequestCanceledException INFO [2019-12-17 12:35:03,791] jepsen worker 30 - jepsen.util 8940 :info :cas [10 [21 18]] :RequestCanceledException INFO [2019-12-17 12:35:03,825] jepsen worker 10 - jepsen.util 9100 :invoke :write [3 16] INFO [2019-12-17 12:35:03,831] jepsen worker 22 - jepsen.util 12262 :invoke :read [7 nil] INFO [2019-12-17 12:35:03,832] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52463,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:35:03,835] jepsen worker 44 - jepsen.util 12644 :invoke :cas [14 [20 18]] INFO [2019-12-17 12:35:03,836] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52471,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:35:03,848] jepsen worker 1 - jepsen.util 12331 :info :write [0 27] :RequestCanceledException INFO [2019-12-17 12:35:03,848] jepsen worker 82 - jepsen.util 13042 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:35:03,852] jepsen worker 39 - jepsen.util 11919 :invoke :read [13 nil] INFO [2019-12-17 12:35:03,857] jepsen worker 7 - jepsen.util 12427 :invoke :write [2 33] INFO [2019-12-17 12:35:03,892] jepsen worker 76 - jepsen.util 12676 :info :cas [25 [20 48]] :RequestCanceledException INFO [2019-12-17 12:35:03,896] jepsen worker 32 - jepsen.util 9212 :invoke :read [10 nil] INFO [2019-12-17 12:35:03,897] jepsen worker 82 - jepsen.util 13042 :invoke :write [27 12] INFO [2019-12-17 12:35:03,898] jepsen worker 18 - jepsen.util 9288 :ok :read [6 16] INFO [2019-12-17 12:35:03,899] jepsen worker 16 - jepsen.util 13336 :info :write [5 23] :RequestCanceledException INFO [2019-12-17 12:35:03,899] jepsen worker 18 - jepsen.util 9288 :invoke :write [6 30] INFO [2019-12-17 12:35:03,918] jepsen worker 79 - jepsen.util 9439 :fail :read [26 nil] :RequestTimeoutException INFO [2019-12-17 12:35:03,921] jepsen worker 79 - jepsen.util 9439 :invoke :read [26 nil] INFO [2019-12-17 12:35:03,942] jepsen worker 34 - jepsen.util 12364 :info :cas [11 [41 9]] :RequestCanceledException INFO [2019-12-17 12:35:03,953] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52475,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:35:03,955] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52475,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:35:03,957] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52481,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:35:03,958] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:35:03,964] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52481,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:35:03,973] jepsen nemesis - couchbase.util Rebalance check count: 408 INFO [2019-12-17 12:35:03,986] jepsen nemesis - couchbase.util Rebalance stuck count: 389 INFO [2019-12-17 12:35:04,005] jepsen worker 48 - jepsen.util 11478 :invoke :read [16 nil] INFO [2019-12-17 12:35:04,006] jepsen worker 15 - jepsen.util 11535 :invoke :write [5 38] INFO [2019-12-17 12:35:04,009] jepsen worker 34 - jepsen.util 12454 :invoke :write [11 29] INFO [2019-12-17 12:35:04,019] jepsen worker 41 - jepsen.util 13181 :invoke :write [13 31] INFO [2019-12-17 12:35:04,027] jepsen worker 6 - jepsen.util 12876 :invoke :read [2 nil] INFO [2019-12-17 12:35:04,030] jepsen worker 87 - jepsen.util 8547 :info :write [29 43] :RequestCanceledException INFO [2019-12-17 12:35:04,030] jepsen worker 1 - jepsen.util 12421 :invoke :write [0 48] INFO [2019-12-17 12:35:04,030] jepsen worker 62 - jepsen.util 12572 :invoke :cas [20 [44 4]] INFO [2019-12-17 12:35:04,036] jepsen worker 30 - jepsen.util 9030 :invoke :read [10 nil] INFO [2019-12-17 12:35:04,037] jepsen worker 38 - jepsen.util 9578 :invoke :read [12 nil] INFO [2019-12-17 12:35:04,037] jepsen worker 16 - jepsen.util 13426 :invoke :cas [5 [46 23]] INFO [2019-12-17 12:35:04,039] jepsen worker 87 - jepsen.util 8637 :invoke :read [29 nil] INFO [2019-12-17 12:35:04,043] jepsen worker 76 - jepsen.util 12766 :invoke :read [25 nil] INFO [2019-12-17 12:35:04,044] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52486,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:35:04,046] jepsen worker 24 - jepsen.util 11364 :invoke :cas [8 [44 2]] INFO [2019-12-17 12:35:04,051] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52486,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:35:04,052] jepsen worker 85 - jepsen.util 13135 :invoke :cas [28 [6 6]] INFO [2019-12-17 12:35:04,057] jepsen worker 40 - jepsen.util 12640 :invoke :cas [13 [23 8]] INFO [2019-12-17 12:35:04,072] jepsen worker 21 - jepsen.util 12171 :invoke :read [7 nil] INFO [2019-12-17 12:35:04,080] jepsen worker 59 - jepsen.util 11759 :invoke :cas [19 [17 9]] INFO [2019-12-17 12:35:04,094] jepsen worker 89 - jepsen.util 9089 :fail :cas [29 [29 42]] :ValueNotSwapFrom INFO [2019-12-17 12:35:04,095] jepsen worker 89 - jepsen.util 9089 :invoke :write [29 16] INFO [2019-12-17 12:35:04,099] jepsen worker 65 - jepsen.util 7445 :info :cas [21 [1 7]] :RequestCanceledException INFO [2019-12-17 12:35:04,123] jepsen worker 65 - jepsen.util 7535 :invoke :cas [21 [32 36]] INFO [2019-12-17 12:35:04,133] jepsen worker 3 - jepsen.util 8913 :fail :read [1 nil] :RequestTimeoutException INFO [2019-12-17 12:35:04,134] jepsen worker 3 - jepsen.util 8913 :invoke :cas [1 [23 27]] INFO [2019-12-17 12:35:04,134] jepsen worker 8 - jepsen.util 11888 :info :write [2 34] :RequestCanceledException INFO [2019-12-17 12:35:04,134] jepsen worker 2 - jepsen.util 11972 :info :cas [0 [42 41]] :RequestCanceledException INFO [2019-12-17 12:35:04,155] jepsen worker 2 - jepsen.util 12062 :invoke :cas [0 [43 17]] INFO [2019-12-17 12:35:04,174] jepsen worker 8 - jepsen.util 11978 :invoke :cas [2 [32 31]] INFO [2019-12-17 12:35:04,191] jepsen worker 52 - jepsen.util 9322 :ok :write [17 11] INFO [2019-12-17 12:35:04,191] jepsen worker 52 - jepsen.util 9322 :invoke :cas [17 [2 28]] INFO [2019-12-17 12:35:04,229] jepsen worker 72 - jepsen.util 9072 :fail :cas [24 [31 16]] :ValueNotSwapFrom INFO [2019-12-17 12:35:04,254] jepsen worker 81 - jepsen.util 12051 :info :cas [27 [16 46]] :RequestCanceledException INFO [2019-12-17 12:35:04,271] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52521,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:35:04,272] jepsen worker 72 - jepsen.util 9072 :invoke :read [24 nil] INFO [2019-12-17 12:35:04,315] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52515,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:35:04,341] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52521,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:35:04,354] jepsen worker 61 - jepsen.util 13291 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:35:04,368] jepsen worker 81 - jepsen.util 12141 :invoke :cas [27 [24 39]] INFO [2019-12-17 12:35:04,376] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52526,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:35:04,377] jepsen worker 61 - jepsen.util 13291 :invoke :write [20 17] INFO [2019-12-17 12:35:04,383] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52515,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:35:04,387] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52526,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:35:04,396] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52517,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:35:04,413] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52517,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:35:04,434] jepsen worker 86 - jepsen.util 13136 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:35:04,434] jepsen worker 86 - jepsen.util 13136 :invoke :write [28 32] INFO [2019-12-17 12:35:04,474] jepsen worker 70 - jepsen.util 9700 :fail :cas [23 [18 30]] :ValueNotSwapFrom INFO [2019-12-17 12:35:04,484] jepsen worker 70 - jepsen.util 9700 :invoke :write [23 35] INFO [2019-12-17 12:35:04,495] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52530,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:35:04,497] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52530,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:35:04,497] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52534,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:35:04,498] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52534,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:35:04,517] jepsen worker 67 - jepsen.util 12487 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:35:04,519] jepsen worker 67 - jepsen.util 12487 :invoke :read [22 nil] INFO [2019-12-17 12:35:04,522] jepsen worker 12 - jepsen.util 9642 :ok :read [4 11] INFO [2019-12-17 12:35:04,524] jepsen worker 12 - jepsen.util 9642 :invoke :write [4 25] INFO [2019-12-17 12:35:04,559] jepsen worker 55 - jepsen.util 9775 :fail :cas [18 [13 10]] :ValueNotSwapFrom INFO [2019-12-17 12:35:04,560] jepsen worker 55 - jepsen.util 9775 :invoke :read [18 nil] INFO [2019-12-17 12:35:04,594] jepsen worker 74 - jepsen.util 9074 :ok :write [24 36] INFO [2019-12-17 12:35:04,609] jepsen worker 74 - jepsen.util 9074 :invoke :read [24 nil] INFO [2019-12-17 12:35:04,621] jepsen worker 31 - jepsen.util 8041 :ok :write [10 30] INFO [2019-12-17 12:35:04,622] jepsen worker 78 - jepsen.util 8898 :ok :read [26 45] INFO [2019-12-17 12:35:04,623] jepsen worker 78 - jepsen.util 8898 :invoke :write [26 7] INFO [2019-12-17 12:35:04,623] jepsen worker 31 - jepsen.util 8041 :invoke :cas [10 [18 49]] INFO [2019-12-17 12:35:04,695] jepsen worker 47 - jepsen.util 9587 :ok :write [15 1] INFO [2019-12-17 12:35:04,695] jepsen worker 47 - jepsen.util 9587 :invoke :write [15 37] INFO [2019-12-17 12:35:04,724] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52542,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:35:04,741] jepsen worker 20 - jepsen.util 8660 :ok :write [6 25] INFO [2019-12-17 12:35:04,755] jepsen worker 49 - jepsen.util 11659 :info :write [16 34] :RequestCanceledException INFO [2019-12-17 12:35:04,757] jepsen worker 13 - jepsen.util 8653 :fail :read [4 nil] :RequestTimeoutException INFO [2019-12-17 12:35:04,761] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52542,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:35:04,762] jepsen worker 20 - jepsen.util 8660 :invoke :read [6 nil] INFO [2019-12-17 12:35:04,762] jepsen worker 13 - jepsen.util 8653 :invoke :cas [4 [40 17]] INFO [2019-12-17 12:35:04,764] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52545,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:35:04,771] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52545,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:35:04,810] jepsen worker 49 - jepsen.util 11749 :invoke :write [16 11] INFO [2019-12-17 12:35:04,821] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestTimeoutException: CarrierBucketConfigRequest {"retried":0,"reason":"TIMEOUT","requestId":52380,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"timings":null,"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:35:04,857] jepsen worker 19 - jepsen.util 9109 :fail :cas [6 [33 45]] :ValueNotSwapFrom INFO [2019-12-17 12:35:04,858] jepsen worker 19 - jepsen.util 9109 :invoke :cas [6 [3 35]] INFO [2019-12-17 12:35:04,884] jepsen worker 80 - jepsen.util 8720 :fail :cas [26 [43 44]] :ValueNotSwapFrom INFO [2019-12-17 12:35:04,884] jepsen worker 5 - jepsen.util 8465 :fail :cas [1 [3 10]] :ValueNotSwapFrom INFO [2019-12-17 12:35:04,885] jepsen worker 80 - jepsen.util 8720 :invoke :read [26 nil] INFO [2019-12-17 12:35:04,885] jepsen worker 5 - jepsen.util 8465 :invoke :cas [1 [30 11]] INFO [2019-12-17 12:35:04,886] jepsen worker 37 - jepsen.util 8497 :fail :cas [12 [15 4]] :ValueNotSwapFrom INFO [2019-12-17 12:35:04,887] jepsen worker 37 - jepsen.util 8497 :invoke :write [12 9] INFO [2019-12-17 12:35:04,891] jepsen worker 88 - jepsen.util 9268 :fail :cas [29 [48 14]] :ValueNotSwapFrom INFO [2019-12-17 12:35:04,892] jepsen worker 46 - jepsen.util 9136 :fail :cas [15 [18 2]] :ValueNotSwapFrom INFO [2019-12-17 12:35:04,892] jepsen worker 88 - jepsen.util 9268 :invoke :cas [29 [29 43]] INFO [2019-12-17 12:35:04,892] jepsen worker 46 - jepsen.util 9136 :invoke :read [15 nil] INFO [2019-12-17 12:35:04,893] jepsen worker 64 - jepsen.util 7264 :ok :read [21 21] INFO [2019-12-17 12:35:04,894] jepsen worker 64 - jepsen.util 7264 :invoke :read [21 nil] INFO [2019-12-17 12:35:04,968] jepsen worker 56 - jepsen.util 9776 :ok :write [18 42] INFO [2019-12-17 12:35:04,968] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52564,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:35:04,969] jepsen worker 56 - jepsen.util 9776 :invoke :write [18 20] INFO [2019-12-17 12:35:04,969] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52564,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:35:04,973] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52567,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:35:04,989] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52567,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:35:05,933] jepsen worker 27 - jepsen.util 8847 :ok :read [9 49] INFO [2019-12-17 12:35:05,934] jepsen worker 27 - jepsen.util 8847 :invoke :read [9 nil] INFO [2019-12-17 12:35:05,936] jepsen worker 28 - jepsen.util 7318 :fail :cas [9 [6 17]] :ValueNotSwapFrom INFO [2019-12-17 12:35:05,936] jepsen worker 51 - jepsen.util 8331 :fail :cas [17 [47 30]] :CASMismatchException INFO [2019-12-17 12:35:05,939] jepsen worker 30 - jepsen.util 9030 :ok :read [10 46] INFO [2019-12-17 12:35:05,939] jepsen worker 44 - jepsen.util 12644 :info :cas [14 [20 18]] :RequestCanceledException INFO [2019-12-17 12:35:05,939] jepsen worker 0 - jepsen.util 13050 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:35:05,940] jepsen worker 29 - jepsen.util 7499 :fail :cas [9 [41 35]] :ValueNotSwapFrom INFO [2019-12-17 12:35:05,940] jepsen worker 51 - jepsen.util 8331 :invoke :write [17 23] INFO [2019-12-17 12:35:05,940] jepsen worker 28 - jepsen.util 7318 :invoke :write [9 20] INFO [2019-12-17 12:35:05,955] jepsen worker 29 - jepsen.util 7499 :invoke :read [9 nil] INFO [2019-12-17 12:35:05,961] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52579,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:35:05,961] jepsen worker 35 - jepsen.util 12365 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:35:05,964] jepsen worker 52 - jepsen.util 9322 :fail :cas [17 [2 28]] :ValueNotSwapFrom INFO [2019-12-17 12:35:05,964] jepsen worker 30 - jepsen.util 9030 :invoke :write [10 43] INFO [2019-12-17 12:35:05,964] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52598,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:35:05,964] jepsen worker 23 - jepsen.util 12083 :info :write [7 17] :RequestCanceledException INFO [2019-12-17 12:35:05,967] jepsen worker 50 - jepsen.util 12920 :info :write [16 14] :RequestCanceledException INFO [2019-12-17 12:35:05,967] jepsen worker 33 - jepsen.util 12363 :info :cas [11 [23 28]] :RequestCanceledException INFO [2019-12-17 12:35:05,959] jepsen worker 60 - jepsen.util 13380 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:35:05,968] jepsen worker 22 - jepsen.util 12262 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:35:05,969] jepsen worker 68 - jepsen.util 11678 :info :cas [22 [47 38]] :RequestCanceledException INFO [2019-12-17 12:35:05,970] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52583,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:35:05,970] jepsen worker 26 - jepsen.util 12896 :info :cas [8 [35 22]] :RequestCanceledException INFO [2019-12-17 12:35:05,970] jepsen worker 39 - jepsen.util 11919 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:35:05,970] jepsen worker 83 - jepsen.util 12863 :info :write [27 43] :RequestCanceledException INFO [2019-12-17 12:35:05,970] jepsen worker 11 - jepsen.util 9101 :info :write [3 47] :RequestCanceledException INFO [2019-12-17 12:35:05,971] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52596,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:35:05,975] jepsen worker 57 - jepsen.util 12117 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:35:05,976] jepsen worker 32 - jepsen.util 9212 :fail :read [10 nil] :RequestTimeoutException INFO [2019-12-17 12:35:05,976] jepsen worker 0 - jepsen.util 13050 :invoke :cas [0 [2 37]] INFO [2019-12-17 12:35:05,978] jepsen worker 84 - jepsen.util 11784 :info :cas [28 [25 44]] :RequestCanceledException INFO [2019-12-17 12:35:05,978] jepsen worker 52 - jepsen.util 9322 :invoke :read [17 nil] INFO [2019-12-17 12:35:05,978] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52592,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:35:05,979] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52598,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:35:05,981] jepsen worker 71 - jepsen.util 9521 :info :cas [23 [16 5]] :RequestCanceledException INFO [2019-12-17 12:35:05,981] jepsen worker 43 - jepsen.util 12193 :info :write [14 21] :RequestCanceledException INFO [2019-12-17 12:35:05,982] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52579,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:35:05,984] jepsen worker 22 - jepsen.util 12262 :invoke :write [7 42] INFO [2019-12-17 12:35:05,985] jepsen worker 75 - jepsen.util 11685 :info :write [25 25] :RequestCanceledException INFO [2019-12-17 12:35:05,985] jepsen worker 60 - jepsen.util 13380 :invoke :read [20 nil] INFO [2019-12-17 12:35:05,993] jepsen worker 35 - jepsen.util 12365 :invoke :cas [11 [20 2]] INFO [2019-12-17 12:35:06,004] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52583,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:35:06,037] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52596,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:35:06,048] jepsen worker 39 - jepsen.util 11919 :invoke :write [13 21] INFO [2019-12-17 12:35:06,051] jepsen worker 48 - jepsen.util 11478 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:35:06,053] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52595,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:35:06,053] jepsen worker 32 - jepsen.util 9212 :invoke :cas [10 [32 18]] INFO [2019-12-17 12:35:06,055] jepsen worker 57 - jepsen.util 12117 :invoke :cas [19 [9 7]] INFO [2019-12-17 12:35:06,059] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:35:06,062] jepsen worker 38 - jepsen.util 9578 :fail :read [12 nil] :RequestTimeoutException INFO [2019-12-17 12:35:06,070] jepsen worker 12 - jepsen.util 9642 :ok :write [4 25] INFO [2019-12-17 12:35:06,081] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52592,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:35:06,086] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52580,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:35:06,101] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52595,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:35:06,101] jepsen worker 48 - jepsen.util 11478 :invoke :write [16 24] INFO [2019-12-17 12:35:06,103] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52594,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:35:06,104] jepsen nemesis - couchbase.util Rebalance check count: 409 INFO [2019-12-17 12:35:06,109] jepsen worker 12 - jepsen.util 9642 :invoke :read [4 nil] INFO [2019-12-17 12:35:06,120] jepsen worker 38 - jepsen.util 9578 :invoke :cas [12 [19 40]] INFO [2019-12-17 12:35:06,125] jepsen worker 6 - jepsen.util 12876 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:35:06,126] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52580,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:35:06,127] jepsen worker 87 - jepsen.util 8637 :fail :read [29 nil] :RequestTimeoutException INFO [2019-12-17 12:35:06,128] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52605,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:35:06,129] jepsen worker 42 - jepsen.util 13272 :info :write [14 22] :RequestCanceledException INFO [2019-12-17 12:35:06,132] jepsen worker 76 - jepsen.util 12766 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:35:06,132] jepsen worker 59 - jepsen.util 11759 :info :cas [19 [17 9]] :RequestCanceledException INFO [2019-12-17 12:35:06,134] jepsen worker 62 - jepsen.util 12572 :info :cas [20 [44 4]] :RequestCanceledException INFO [2019-12-17 12:35:06,136] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52594,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:35:06,136] jepsen nemesis - couchbase.util Rebalance stuck count: 390 INFO [2019-12-17 12:35:06,138] jepsen worker 16 - jepsen.util 13426 :info :cas [5 [46 23]] :RequestCanceledException INFO [2019-12-17 12:35:06,139] jepsen worker 24 - jepsen.util 11364 :info :cas [8 [44 2]] :RequestCanceledException INFO [2019-12-17 12:35:06,140] jepsen worker 17 - jepsen.util 12167 :info :write [5 33] :RequestCanceledException INFO [2019-12-17 12:35:06,141] jepsen worker 6 - jepsen.util 12876 :invoke :cas [2 [32 9]] INFO [2019-12-17 12:35:06,143] jepsen worker 21 - jepsen.util 12171 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:35:06,144] jepsen worker 87 - jepsen.util 8637 :invoke :cas [29 [8 41]] INFO [2019-12-17 12:35:06,145] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52581,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:35:06,146] jepsen worker 68 - jepsen.util 11768 :invoke :cas [22 [25 19]] INFO [2019-12-17 12:35:06,146] jepsen worker 23 - jepsen.util 12173 :invoke :cas [7 [5 36]] INFO [2019-12-17 12:35:06,148] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52605,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:35:06,148] jepsen worker 77 - jepsen.util 12767 :info :write [25 11] :RequestCanceledException INFO [2019-12-17 12:35:06,151] jepsen worker 76 - jepsen.util 12766 :invoke :read [25 nil] INFO [2019-12-17 12:35:06,151] jepsen worker 85 - jepsen.util 13135 :info :cas [28 [6 6]] :RequestCanceledException INFO [2019-12-17 12:35:06,154] jepsen worker 71 - jepsen.util 9611 :invoke :write [23 4] INFO [2019-12-17 12:35:06,154] jepsen worker 50 - jepsen.util 13010 :invoke :write [16 36] INFO [2019-12-17 12:35:06,155] jepsen worker 44 - jepsen.util 12734 :invoke :cas [14 [44 33]] INFO [2019-12-17 12:35:06,158] jepsen worker 84 - jepsen.util 11874 :invoke :write [28 4] INFO [2019-12-17 12:35:06,158] jepsen worker 11 - jepsen.util 9191 :invoke :cas [3 [19 46]] INFO [2019-12-17 12:35:06,162] jepsen worker 21 - jepsen.util 12171 :invoke :write [7 19] INFO [2019-12-17 12:35:06,163] jepsen worker 40 - jepsen.util 12640 :info :cas [13 [23 8]] :RequestCanceledException INFO [2019-12-17 12:35:06,165] jepsen worker 66 - jepsen.util 12036 :info :write [22 2] :RequestCanceledException INFO [2019-12-17 12:35:06,165] jepsen worker 25 - jepsen.util 13255 :info :write [8 24] :RequestCanceledException INFO [2019-12-17 12:35:06,166] jepsen worker 42 - jepsen.util 13362 :invoke :read [14 nil] INFO [2019-12-17 12:35:06,167] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52606,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:35:06,167] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52581,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:35:06,174] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52606,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:35:06,174] jepsen worker 33 - jepsen.util 12453 :invoke :write [11 14] INFO [2019-12-17 12:35:06,179] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52616,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:35:06,180] jepsen worker 83 - jepsen.util 12953 :invoke :read [27 nil] INFO [2019-12-17 12:35:06,182] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52616,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:35:06,201] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52617,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:35:06,213] jepsen worker 54 - jepsen.util 9954 :info :write [18 10] :RequestCanceledException INFO [2019-12-17 12:35:06,231] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52617,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:35:06,234] jepsen worker 43 - jepsen.util 12283 :invoke :read [14 nil] INFO [2019-12-17 12:35:06,248] jepsen worker 65 - jepsen.util 7535 :info :cas [21 [32 36]] :RequestCanceledException INFO [2019-12-17 12:35:06,253] jepsen worker 47 - jepsen.util 9587 :ok :write [15 37] INFO [2019-12-17 12:35:06,256] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52637,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:35:06,269] jepsen worker 2 - jepsen.util 12062 :info :cas [0 [43 17]] :RequestCanceledException INFO [2019-12-17 12:35:06,272] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52637,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:35:06,278] jepsen worker 59 - jepsen.util 11849 :invoke :write [19 24] INFO [2019-12-17 12:35:06,283] jepsen worker 47 - jepsen.util 9587 :invoke :read [15 nil] INFO [2019-12-17 12:35:06,287] jepsen worker 16 - jepsen.util 13516 :invoke :write [5 18] INFO [2019-12-17 12:35:06,289] jepsen worker 26 - jepsen.util 12986 :invoke :cas [8 [47 43]] INFO [2019-12-17 12:35:06,301] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52645,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:35:06,332] jepsen worker 85 - jepsen.util 13225 :invoke :cas [28 [3 16]] INFO [2019-12-17 12:35:06,332] jepsen worker 8 - jepsen.util 11978 :info :cas [2 [32 31]] :RequestCanceledException INFO [2019-12-17 12:35:06,332] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52641,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:35:06,357] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52645,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:35:06,365] jepsen worker 24 - jepsen.util 11454 :invoke :read [8 nil] INFO [2019-12-17 12:35:06,375] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52641,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:35:06,375] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52649,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:35:06,381] jepsen worker 62 - jepsen.util 12662 :invoke :cas [20 [37 3]] INFO [2019-12-17 12:35:06,433] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52656,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:35:06,511] jepsen worker 17 - jepsen.util 12257 :invoke :write [5 27] INFO [2019-12-17 12:35:06,512] jepsen worker 81 - jepsen.util 12141 :info :cas [27 [24 39]] :RequestCanceledException INFO [2019-12-17 12:35:06,534] jepsen worker 58 - jepsen.util 13108 :info :write [19 26] :RequestCanceledException INFO [2019-12-17 12:35:06,540] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52649,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:35:06,562] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52656,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:35:06,569] jepsen worker 36 - jepsen.util 9396 :info :write [12 42] :RequestCanceledException INFO [2019-12-17 12:35:06,587] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52659,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:35:06,589] jepsen worker 77 - jepsen.util 12857 :invoke :write [25 7] INFO [2019-12-17 12:35:06,610] jepsen worker 54 - jepsen.util 10044 :invoke :read [18 nil] INFO [2019-12-17 12:35:06,612] jepsen worker 55 - jepsen.util 9775 :fail :read [18 nil] :RequestTimeoutException INFO [2019-12-17 12:35:06,626] jepsen worker 67 - jepsen.util 12487 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:35:06,631] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52659,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:35:06,631] jepsen worker 55 - jepsen.util 9775 :invoke :cas [18 [45 46]] INFO [2019-12-17 12:35:06,633] jepsen worker 67 - jepsen.util 12487 :invoke :write [22 16] INFO [2019-12-17 12:35:06,640] jepsen worker 75 - jepsen.util 11775 :invoke :cas [25 [8 0]] INFO [2019-12-17 12:35:06,656] jepsen worker 40 - jepsen.util 12730 :invoke :cas [13 [16 36]] INFO [2019-12-17 12:35:06,661] jepsen worker 2 - jepsen.util 12152 :invoke :write [0 5] INFO [2019-12-17 12:35:06,661] jepsen worker 66 - jepsen.util 12126 :invoke :cas [22 [36 5]] INFO [2019-12-17 12:35:06,663] jepsen worker 25 - jepsen.util 13345 :invoke :cas [8 [33 28]] INFO [2019-12-17 12:35:06,682] jepsen worker 65 - jepsen.util 7625 :invoke :write [21 34] INFO [2019-12-17 12:35:06,750] jepsen worker 58 - jepsen.util 13198 :invoke :read [19 nil] INFO [2019-12-17 12:35:06,753] jepsen worker 18 - jepsen.util 9288 :ok :write [6 30] INFO [2019-12-17 12:35:06,753] jepsen worker 1 - jepsen.util 12421 :info :write [0 48] :RequestCanceledException INFO [2019-12-17 12:35:06,758] jepsen worker 31 - jepsen.util 8041 :info :cas [10 [18 49]] :RequestCanceledException INFO [2019-12-17 12:35:06,762] jepsen worker 18 - jepsen.util 9288 :invoke :write [6 26] INFO [2019-12-17 12:35:06,766] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52663,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:35:06,769] jepsen worker 8 - jepsen.util 12068 :invoke :cas [2 [9 34]] INFO [2019-12-17 12:35:06,777] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52663,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:35:06,783] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52669,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:35:06,808] jepsen worker 82 - jepsen.util 13042 :info :write [27 12] :RequestCanceledException INFO [2019-12-17 12:35:06,808] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52669,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:35:06,811] jepsen worker 9 - jepsen.util 9729 :fail :cas [3 [11 37]] :ValueNotSwapFrom INFO [2019-12-17 12:35:06,814] jepsen worker 9 - jepsen.util 9729 :invoke :read [3 nil] INFO [2019-12-17 12:35:06,819] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52671,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:35:06,823] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52671,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:35:06,841] jepsen worker 20 - jepsen.util 8660 :fail :read [6 nil] :RequestTimeoutException INFO [2019-12-17 12:35:06,842] jepsen worker 7 - jepsen.util 12427 :info :write [2 33] :RequestCanceledException INFO [2019-12-17 12:35:06,842] jepsen worker 20 - jepsen.util 8660 :invoke :read [6 nil] INFO [2019-12-17 12:35:06,843] jepsen worker 61 - jepsen.util 13291 :info :write [20 17] :RequestCanceledException INFO [2019-12-17 12:35:06,852] jepsen worker 10 - jepsen.util 9100 :ok :write [3 16] INFO [2019-12-17 12:35:06,853] jepsen worker 15 - jepsen.util 11535 :info :write [5 38] :RequestCanceledException INFO [2019-12-17 12:35:06,853] jepsen worker 10 - jepsen.util 9100 :invoke :cas [3 [44 14]] INFO [2019-12-17 12:35:06,859] jepsen worker 36 - jepsen.util 9486 :invoke :cas [12 [40 39]] INFO [2019-12-17 12:35:06,883] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52681,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:35:06,888] jepsen worker 78 - jepsen.util 8898 :info :write [26 7] :RequestCanceledException INFO [2019-12-17 12:35:06,897] jepsen worker 34 - jepsen.util 12454 :info :write [11 29] :RequestCanceledException INFO [2019-12-17 12:35:06,906] jepsen worker 89 - jepsen.util 9089 :info :write [29 16] :RequestCanceledException INFO [2019-12-17 12:35:06,909] jepsen worker 81 - jepsen.util 12231 :invoke :read [27 nil] INFO [2019-12-17 12:35:06,913] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52681,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:35:06,928] jepsen worker 73 - jepsen.util 8623 :ok :read [24 34] INFO [2019-12-17 12:35:06,931] jepsen worker 80 - jepsen.util 8720 :fail :read [26 nil] :RequestTimeoutException INFO [2019-12-17 12:35:06,934] jepsen worker 86 - jepsen.util 13136 :info :write [28 32] :RequestCanceledException INFO [2019-12-17 12:35:06,940] jepsen worker 73 - jepsen.util 8623 :invoke :cas [24 [5 7]] INFO [2019-12-17 12:35:06,941] jepsen worker 80 - jepsen.util 8720 :invoke :cas [26 [30 10]] INFO [2019-12-17 12:35:06,941] jepsen worker 64 - jepsen.util 7264 :fail :read [21 nil] :RequestTimeoutException INFO [2019-12-17 12:35:06,941] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52685,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:35:06,950] jepsen worker 41 - jepsen.util 13181 :info :write [13 31] :RequestCanceledException INFO [2019-12-17 12:35:06,951] jepsen worker 64 - jepsen.util 7264 :invoke :write [21 35] INFO [2019-12-17 12:35:06,951] jepsen worker 5 - jepsen.util 8465 :info :cas [1 [30 11]] :RequestCanceledException INFO [2019-12-17 12:35:06,953] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52685,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:35:06,954] jepsen worker 19 - jepsen.util 9109 :info :cas [6 [3 35]] :RequestCanceledException INFO [2019-12-17 12:35:06,963] jepsen worker 70 - jepsen.util 9700 :info :write [23 35] :RequestCanceledException INFO [2019-12-17 12:35:06,963] jepsen worker 88 - jepsen.util 9268 :info :cas [29 [29 43]] :RequestCanceledException INFO [2019-12-17 12:35:06,981] jepsen worker 63 - jepsen.util 7893 :fail :cas [21 [33 35]] :ValueNotSwapFrom INFO [2019-12-17 12:35:06,982] jepsen worker 63 - jepsen.util 7893 :invoke :read [21 nil] INFO [2019-12-17 12:35:06,985] jepsen worker 1 - jepsen.util 12511 :invoke :cas [0 [47 24]] INFO [2019-12-17 12:35:06,999] jepsen worker 46 - jepsen.util 9136 :fail :read [15 nil] :RequestTimeoutException INFO [2019-12-17 12:35:07,000] jepsen worker 53 - jepsen.util 8243 :fail :cas [17 [44 21]] :ValueNotSwapFrom INFO [2019-12-17 12:35:07,010] jepsen worker 53 - jepsen.util 8243 :invoke :write [17 19] INFO [2019-12-17 12:35:07,012] jepsen worker 46 - jepsen.util 9136 :invoke :read [15 nil] INFO [2019-12-17 12:35:07,019] jepsen worker 19 - jepsen.util 9199 :invoke :cas [6 [27 11]] INFO [2019-12-17 12:35:07,024] jepsen worker 5 - jepsen.util 8555 :invoke :write [1 49] INFO [2019-12-17 12:35:07,025] jepsen worker 70 - jepsen.util 9790 :invoke :read [23 nil] INFO [2019-12-17 12:35:07,037] jepsen worker 82 - jepsen.util 13132 :invoke :cas [27 [20 9]] INFO [2019-12-17 12:35:07,040] jepsen worker 61 - jepsen.util 13381 :invoke :read [20 nil] INFO [2019-12-17 12:35:07,048] jepsen worker 31 - jepsen.util 8131 :invoke :read [10 nil] INFO [2019-12-17 12:35:07,052] jepsen worker 15 - jepsen.util 11625 :invoke :write [5 34] INFO [2019-12-17 12:35:07,059] jepsen worker 34 - jepsen.util 12544 :invoke :cas [11 [3 36]] INFO [2019-12-17 12:35:07,063] jepsen worker 86 - jepsen.util 13226 :invoke :write [28 46] INFO [2019-12-17 12:35:07,070] jepsen worker 78 - jepsen.util 8988 :invoke :write [26 11] INFO [2019-12-17 12:35:07,074] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52695,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:35:07,085] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52695,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:35:07,093] jepsen worker 7 - jepsen.util 12517 :invoke :read [2 nil] INFO [2019-12-17 12:35:07,093] jepsen worker 89 - jepsen.util 9179 :invoke :cas [29 [18 22]] INFO [2019-12-17 12:35:07,094] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52698,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:35:07,094] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52698,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:35:07,100] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52700,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:35:07,102] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52700,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:35:07,141] jepsen worker 41 - jepsen.util 13271 :invoke :read [13 nil] INFO [2019-12-17 12:35:07,141] jepsen worker 88 - jepsen.util 9358 :invoke :write [29 2] INFO [2019-12-17 12:35:07,198] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:35:07,198] jepsen nemesis - couchbase.util Rebalance check count: 410 INFO [2019-12-17 12:35:07,198] jepsen nemesis - couchbase.util Rebalance stuck count: 391 INFO [2019-12-17 12:35:07,250] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52716,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:35:07,253] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52716,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:35:07,296] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52724,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:35:07,296] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52724,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:35:07,441] jepsen worker 71 - jepsen.util 9611 :ok :write [23 4] INFO [2019-12-17 12:35:07,446] jepsen worker 71 - jepsen.util 9611 :invoke :write [23 28] INFO [2019-12-17 12:35:07,477] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52740,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:35:07,484] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52740,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:35:07,673] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52751,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:35:07,674] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52751,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:35:07,837] jepsen worker 69 - jepsen.util 9159 :fail :cas [23 [9 19]] :ValueNotSwapFrom INFO [2019-12-17 12:35:07,846] jepsen worker 69 - jepsen.util 9159 :invoke :cas [23 [29 19]] INFO [2019-12-17 12:35:07,892] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52757,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:35:07,900] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52757,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:35:07,908] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52758,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:35:07,918] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52758,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:35:07,925] jepsen worker 74 - jepsen.util 9074 :ok :read [24 36] INFO [2019-12-17 12:35:07,931] jepsen worker 74 - jepsen.util 9074 :invoke :write [24 45] INFO [2019-12-17 12:35:07,933] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52762,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:35:07,952] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52762,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:35:08,037] jepsen worker 49 - jepsen.util 11749 :info :write [16 11] :RequestCanceledException INFO [2019-12-17 12:35:08,125] jepsen worker 14 - jepsen.util 9014 :ok :read [4 11] INFO [2019-12-17 12:35:08,127] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52770,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:35:08,132] jepsen worker 14 - jepsen.util 9014 :invoke :write [4 3] INFO [2019-12-17 12:35:08,138] jepsen worker 0 - jepsen.util 13050 :info :cas [0 [2 37]] :RequestCanceledException INFO [2019-12-17 12:35:08,149] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52770,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:35:08,154] jepsen worker 35 - jepsen.util 12365 :info :cas [11 [20 2]] :RequestCanceledException INFO [2019-12-17 12:35:08,155] jepsen worker 60 - jepsen.util 13380 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:35:08,167] jepsen worker 60 - jepsen.util 13380 :invoke :cas [20 [13 34]] INFO [2019-12-17 12:35:08,169] jepsen worker 38 - jepsen.util 9578 :info :cas [12 [19 40]] :RequestCanceledException INFO [2019-12-17 12:35:08,196] jepsen worker 57 - jepsen.util 12117 :info :cas [19 [9 7]] :RequestCanceledException INFO [2019-12-17 12:35:08,205] jepsen worker 6 - jepsen.util 12876 :info :cas [2 [32 9]] :RequestCanceledException INFO [2019-12-17 12:35:08,209] jepsen worker 38 - jepsen.util 9668 :invoke :read [12 nil] INFO [2019-12-17 12:35:08,210] jepsen worker 87 - jepsen.util 8637 :info :cas [29 [8 41]] :RequestCanceledException INFO [2019-12-17 12:35:08,220] jepsen worker 23 - jepsen.util 12173 :info :cas [7 [5 36]] :RequestCanceledException INFO [2019-12-17 12:35:08,221] jepsen worker 68 - jepsen.util 11768 :info :cas [22 [25 19]] :RequestCanceledException INFO [2019-12-17 12:35:08,222] jepsen worker 76 - jepsen.util 12766 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:35:08,222] jepsen worker 76 - jepsen.util 12766 :invoke :read [25 nil] INFO [2019-12-17 12:35:08,223] jepsen worker 42 - jepsen.util 13362 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:35:08,223] jepsen worker 42 - jepsen.util 13362 :invoke :read [14 nil] INFO [2019-12-17 12:35:08,240] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52782,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:35:08,241] jepsen worker 87 - jepsen.util 8727 :invoke :read [29 nil] INFO [2019-12-17 12:35:08,245] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52782,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:35:08,246] jepsen worker 83 - jepsen.util 12953 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:35:08,252] jepsen worker 83 - jepsen.util 12953 :invoke :write [27 9] INFO [2019-12-17 12:35:08,252] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52786,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:35:08,267] jepsen worker 44 - jepsen.util 12734 :info :cas [14 [44 33]] :RequestCanceledException INFO [2019-12-17 12:35:08,267] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52777,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:35:08,268] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52786,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:35:08,269] jepsen worker 0 - jepsen.util 13140 :invoke :write [0 44] INFO [2019-12-17 12:35:08,274] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52777,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:35:08,276] jepsen worker 43 - jepsen.util 12283 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:35:08,283] jepsen worker 43 - jepsen.util 12283 :invoke :read [14 nil] INFO [2019-12-17 12:35:08,286] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52779,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:35:08,307] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52779,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:35:08,308] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:35:08,308] jepsen worker 6 - jepsen.util 12966 :invoke :cas [2 [47 5]] INFO [2019-12-17 12:35:08,318] jepsen nemesis - couchbase.util Rebalance check count: 411 INFO [2019-12-17 12:35:08,321] jepsen worker 49 - jepsen.util 11839 :invoke :write [16 4] INFO [2019-12-17 12:35:08,321] jepsen worker 57 - jepsen.util 12207 :invoke :cas [19 [31 40]] INFO [2019-12-17 12:35:08,323] jepsen nemesis - couchbase.util Rebalance stuck count: 392 INFO [2019-12-17 12:35:08,334] jepsen worker 35 - jepsen.util 12455 :invoke :cas [11 [37 34]] INFO [2019-12-17 12:35:08,348] jepsen worker 56 - jepsen.util 9776 :ok :write [18 20] INFO [2019-12-17 12:35:08,350] jepsen worker 56 - jepsen.util 9776 :invoke :read [18 nil] INFO [2019-12-17 12:35:08,351] jepsen worker 68 - jepsen.util 11858 :invoke :write [22 10] INFO [2019-12-17 12:35:08,364] jepsen worker 23 - jepsen.util 12263 :invoke :write [7 10] INFO [2019-12-17 12:35:08,397] jepsen worker 26 - jepsen.util 12986 :info :cas [8 [47 43]] :RequestCanceledException INFO [2019-12-17 12:35:08,412] jepsen worker 44 - jepsen.util 12824 :invoke :cas [14 [41 0]] INFO [2019-12-17 12:35:08,417] jepsen worker 85 - jepsen.util 13225 :info :cas [28 [3 16]] :RequestCanceledException INFO [2019-12-17 12:35:08,442] jepsen worker 26 - jepsen.util 13076 :invoke :write [8 7] INFO [2019-12-17 12:35:08,444] jepsen worker 24 - jepsen.util 11454 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:35:08,445] jepsen worker 24 - jepsen.util 11454 :invoke :cas [8 [6 39]] INFO [2019-12-17 12:35:08,446] jepsen worker 85 - jepsen.util 13315 :invoke :write [28 5] INFO [2019-12-17 12:35:08,462] jepsen worker 62 - jepsen.util 12662 :info :cas [20 [37 3]] :RequestCanceledException INFO [2019-12-17 12:35:08,480] jepsen worker 62 - jepsen.util 12752 :invoke :write [20 5] INFO [2019-12-17 12:35:08,525] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52797,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:35:08,526] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52797,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:35:08,527] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52801,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:35:08,527] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52801,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:35:08,541] jepsen worker 4 - jepsen.util 8644 :ok :read [1 43] INFO [2019-12-17 12:35:08,544] jepsen worker 4 - jepsen.util 8644 :invoke :read [1 nil] INFO [2019-12-17 12:35:08,595] jepsen worker 52 - jepsen.util 9322 :ok :read [17 11] INFO [2019-12-17 12:35:08,596] jepsen worker 52 - jepsen.util 9322 :invoke :write [17 17] INFO [2019-12-17 12:35:08,651] jepsen worker 45 - jepsen.util 9405 :fail :cas [15 [25 31]] :ValueNotSwapFrom INFO [2019-12-17 12:35:08,656] jepsen worker 45 - jepsen.util 9405 :invoke :cas [15 [22 29]] INFO [2019-12-17 12:35:08,681] jepsen worker 64 - jepsen.util 7264 :ok :write [21 35] INFO [2019-12-17 12:35:08,683] jepsen worker 64 - jepsen.util 7264 :invoke :cas [21 [4 10]] INFO [2019-12-17 12:35:08,696] jepsen worker 51 - jepsen.util 8331 :ok :write [17 23] INFO [2019-12-17 12:35:08,697] jepsen worker 51 - jepsen.util 8331 :invoke :cas [17 [25 5]] INFO [2019-12-17 12:35:08,699] jepsen worker 55 - jepsen.util 9775 :info :cas [18 [45 46]] :RequestCanceledException INFO [2019-12-17 12:35:08,700] jepsen worker 25 - jepsen.util 13345 :info :cas [8 [33 28]] :RequestCanceledException INFO [2019-12-17 12:35:08,714] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52810,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:35:08,719] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52810,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:35:08,722] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52814,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:35:08,728] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52814,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:35:08,731] jepsen worker 66 - jepsen.util 12126 :info :cas [22 [36 5]] :RequestCanceledException INFO [2019-12-17 12:35:08,731] jepsen worker 75 - jepsen.util 11775 :info :cas [25 [8 0]] :RequestCanceledException INFO [2019-12-17 12:35:08,746] jepsen worker 40 - jepsen.util 12730 :info :cas [13 [16 36]] :RequestCanceledException INFO [2019-12-17 12:35:08,747] jepsen worker 55 - jepsen.util 9865 :invoke :cas [18 [22 46]] INFO [2019-12-17 12:35:08,752] jepsen worker 3 - jepsen.util 8913 :fail :cas [1 [23 27]] :ValueNotSwapFrom INFO [2019-12-17 12:35:08,753] jepsen worker 3 - jepsen.util 8913 :invoke :read [1 nil] INFO [2019-12-17 12:35:08,761] jepsen worker 75 - jepsen.util 11865 :invoke :write [25 18] INFO [2019-12-17 12:35:08,774] jepsen worker 12 - jepsen.util 9642 :ok :read [4 25] INFO [2019-12-17 12:35:08,776] jepsen worker 12 - jepsen.util 9642 :invoke :write [4 10] INFO [2019-12-17 12:35:08,785] jepsen worker 72 - jepsen.util 9072 :ok :read [24 36] INFO [2019-12-17 12:35:08,787] jepsen worker 25 - jepsen.util 13435 :invoke :cas [8 [35 17]] INFO [2019-12-17 12:35:08,789] jepsen worker 72 - jepsen.util 9072 :invoke :read [24 nil] INFO [2019-12-17 12:35:08,807] jepsen worker 66 - jepsen.util 12216 :invoke :cas [22 [20 27]] INFO [2019-12-17 12:35:08,819] jepsen worker 40 - jepsen.util 12820 :invoke :cas [13 [16 48]] INFO [2019-12-17 12:35:08,825] jepsen worker 65 - jepsen.util 7625 :ok :write [21 34] INFO [2019-12-17 12:35:08,825] jepsen worker 65 - jepsen.util 7625 :invoke :cas [21 [14 6]] INFO [2019-12-17 12:35:08,827] jepsen worker 27 - jepsen.util 8847 :ok :read [9 48] INFO [2019-12-17 12:35:08,829] jepsen worker 27 - jepsen.util 8847 :invoke :read [9 nil] INFO [2019-12-17 12:35:08,832] jepsen worker 58 - jepsen.util 13198 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:35:08,833] jepsen worker 58 - jepsen.util 13198 :invoke :cas [19 [33 23]] INFO [2019-12-17 12:35:08,854] jepsen worker 8 - jepsen.util 12068 :info :cas [2 [9 34]] :RequestCanceledException INFO [2019-12-17 12:35:08,856] jepsen worker 9 - jepsen.util 9729 :fail :read [3 nil] :RequestTimeoutException INFO [2019-12-17 12:35:08,857] jepsen worker 9 - jepsen.util 9729 :invoke :read [3 nil] INFO [2019-12-17 12:35:08,879] jepsen worker 79 - jepsen.util 9439 :ok :read [26 31] INFO [2019-12-17 12:35:08,880] jepsen worker 79 - jepsen.util 9439 :invoke :write [26 23] INFO [2019-12-17 12:35:08,887] jepsen worker 13 - jepsen.util 8653 :fail :cas [4 [40 17]] :ValueNotSwapFrom INFO [2019-12-17 12:35:08,895] jepsen worker 13 - jepsen.util 8653 :invoke :write [4 10] INFO [2019-12-17 12:35:08,910] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52836,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:35:08,912] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52836,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:35:08,927] jepsen worker 5 - jepsen.util 8555 :ok :write [1 49] INFO [2019-12-17 12:35:08,930] jepsen worker 5 - jepsen.util 8555 :invoke :read [1 nil] INFO [2019-12-17 12:35:08,932] jepsen worker 8 - jepsen.util 12158 :invoke :write [2 17] INFO [2019-12-17 12:35:08,945] jepsen worker 81 - jepsen.util 12231 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:35:08,946] jepsen worker 54 - jepsen.util 10044 :ok :read [18 42] INFO [2019-12-17 12:35:08,949] jepsen worker 81 - jepsen.util 12231 :invoke :write [27 11] INFO [2019-12-17 12:35:08,957] jepsen worker 10 - jepsen.util 9100 :info :cas [3 [44 14]] :RequestCanceledException INFO [2019-12-17 12:35:08,957] jepsen worker 54 - jepsen.util 10044 :invoke :write [18 46] INFO [2019-12-17 12:35:08,995] jepsen worker 74 - jepsen.util 9074 :ok :write [24 45] INFO [2019-12-17 12:35:08,999] jepsen worker 74 - jepsen.util 9074 :invoke :cas [24 [30 29]] INFO [2019-12-17 12:35:08,999] jepsen worker 80 - jepsen.util 8720 :info :cas [26 [30 10]] :RequestCanceledException INFO [2019-12-17 12:35:09,045] jepsen worker 28 - jepsen.util 7318 :info :write [9 20] :RequestCanceledException INFO [2019-12-17 12:35:09,047] jepsen worker 63 - jepsen.util 7893 :fail :read [21 nil] :RequestTimeoutException INFO [2019-12-17 12:35:09,048] jepsen worker 63 - jepsen.util 7893 :invoke :write [21 17] INFO [2019-12-17 12:35:09,048] jepsen worker 80 - jepsen.util 8810 :invoke :write [26 41] INFO [2019-12-17 12:35:09,061] jepsen worker 10 - jepsen.util 9190 :invoke :cas [3 [11 41]] INFO [2019-12-17 12:35:09,063] jepsen worker 82 - jepsen.util 13132 :info :cas [27 [20 9]] :RequestCanceledException INFO [2019-12-17 12:35:09,071] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52841,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:35:09,077] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52841,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:35:09,077] jepsen worker 28 - jepsen.util 7408 :invoke :read [9 nil] INFO [2019-12-17 12:35:09,081] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52846,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:35:09,081] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52846,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:35:09,083] jepsen worker 47 - jepsen.util 9587 :ok :read [15 37] INFO [2019-12-17 12:35:09,091] jepsen worker 47 - jepsen.util 9587 :invoke :cas [15 [40 25]] INFO [2019-12-17 12:35:09,095] jepsen worker 82 - jepsen.util 13222 :invoke :read [27 nil] INFO [2019-12-17 12:35:09,100] jepsen worker 1 - jepsen.util 12511 :info :cas [0 [47 24]] :RequestCanceledException INFO [2019-12-17 12:35:09,133] jepsen worker 61 - jepsen.util 13381 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:35:09,134] jepsen worker 7 - jepsen.util 12517 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:35:09,134] jepsen worker 61 - jepsen.util 13381 :invoke :cas [20 [8 1]] INFO [2019-12-17 12:35:09,134] jepsen worker 7 - jepsen.util 12517 :invoke :cas [2 [7 49]] INFO [2019-12-17 12:35:09,150] jepsen worker 1 - jepsen.util 12601 :invoke :cas [0 [16 2]] INFO [2019-12-17 12:35:09,151] jepsen worker 34 - jepsen.util 12544 :info :cas [11 [3 36]] :RequestCanceledException INFO [2019-12-17 12:35:09,170] jepsen worker 37 - jepsen.util 8497 :info :write [12 9] :RequestCanceledException INFO [2019-12-17 12:35:09,220] jepsen worker 41 - jepsen.util 13271 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:35:09,221] jepsen worker 41 - jepsen.util 13271 :invoke :write [13 11] INFO [2019-12-17 12:35:09,233] jepsen worker 70 - jepsen.util 9790 :ok :read [23 35] INFO [2019-12-17 12:35:09,234] jepsen worker 48 - jepsen.util 11478 :info :write [16 24] :RequestCanceledException INFO [2019-12-17 12:35:09,234] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52855,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:35:09,237] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52855,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:35:09,248] jepsen worker 70 - jepsen.util 9790 :invoke :read [23 nil] INFO [2019-12-17 12:35:09,249] jepsen worker 22 - jepsen.util 12262 :info :write [7 42] :RequestCanceledException INFO [2019-12-17 12:35:09,249] jepsen worker 30 - jepsen.util 9030 :info :write [10 43] :RequestCanceledException INFO [2019-12-17 12:35:09,251] jepsen worker 36 - jepsen.util 9486 :fail :cas [12 [40 39]] :ValueNotSwapFrom INFO [2019-12-17 12:35:09,252] jepsen worker 36 - jepsen.util 9486 :invoke :write [12 25] INFO [2019-12-17 12:35:09,285] jepsen worker 34 - jepsen.util 12634 :invoke :write [11 46] INFO [2019-12-17 12:35:09,318] jepsen worker 30 - jepsen.util 9120 :invoke :write [10 45] INFO [2019-12-17 12:35:09,325] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52861,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:35:09,326] jepsen worker 22 - jepsen.util 12352 :invoke :write [7 27] INFO [2019-12-17 12:35:09,327] jepsen worker 39 - jepsen.util 11919 :info :write [13 21] :RequestCanceledException INFO [2019-12-17 12:35:09,334] jepsen worker 46 - jepsen.util 9136 :ok :read [15 37] INFO [2019-12-17 12:35:09,334] jepsen worker 33 - jepsen.util 12453 :info :write [11 14] :RequestCanceledException INFO [2019-12-17 12:35:09,335] jepsen worker 21 - jepsen.util 12171 :info :write [7 19] :RequestCanceledException INFO [2019-12-17 12:35:09,336] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52861,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:35:09,336] jepsen worker 84 - jepsen.util 11874 :info :write [28 4] :RequestCanceledException INFO [2019-12-17 12:35:09,336] jepsen worker 48 - jepsen.util 11568 :invoke :read [16 nil] INFO [2019-12-17 12:35:09,336] jepsen worker 37 - jepsen.util 8587 :invoke :cas [12 [36 34]] INFO [2019-12-17 12:35:09,338] jepsen worker 46 - jepsen.util 9136 :invoke :write [15 17] INFO [2019-12-17 12:35:09,433] jepsen worker 17 - jepsen.util 12257 :info :write [5 27] :RequestCanceledException INFO [2019-12-17 12:35:09,439] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:35:09,439] jepsen worker 59 - jepsen.util 11849 :info :write [19 24] :RequestCanceledException INFO [2019-12-17 12:35:09,440] jepsen nemesis - couchbase.util Rebalance check count: 412 INFO [2019-12-17 12:35:09,444] jepsen nemesis - couchbase.util Rebalance stuck count: 393 INFO [2019-12-17 12:35:09,449] jepsen worker 50 - jepsen.util 13010 :info :write [16 36] :RequestCanceledException INFO [2019-12-17 12:35:09,464] jepsen worker 21 - jepsen.util 12261 :invoke :write [7 27] INFO [2019-12-17 12:35:09,466] jepsen worker 2 - jepsen.util 12152 :info :write [0 5] :RequestCanceledException INFO [2019-12-17 12:35:09,469] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52871,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:35:09,473] jepsen worker 84 - jepsen.util 11964 :invoke :write [28 25] INFO [2019-12-17 12:35:09,476] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52871,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:35:09,482] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52876,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:35:09,486] jepsen worker 33 - jepsen.util 12543 :invoke :cas [11 [41 28]] INFO [2019-12-17 12:35:09,488] jepsen worker 19 - jepsen.util 9199 :fail :cas [6 [27 11]] :ValueNotSwapFrom INFO [2019-12-17 12:35:09,494] jepsen worker 19 - jepsen.util 9199 :invoke :read [6 nil] INFO [2019-12-17 12:35:09,497] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52876,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:35:09,503] jepsen worker 39 - jepsen.util 12009 :invoke :cas [13 [45 49]] INFO [2019-12-17 12:35:09,518] jepsen worker 67 - jepsen.util 12487 :info :write [22 16] :RequestCanceledException INFO [2019-12-17 12:35:09,533] jepsen worker 88 - jepsen.util 9358 :info :write [29 2] :RequestCanceledException INFO [2019-12-17 12:35:09,536] jepsen worker 16 - jepsen.util 13516 :info :write [5 18] :RequestCanceledException INFO [2019-12-17 12:35:09,550] jepsen worker 50 - jepsen.util 13100 :invoke :write [16 30] INFO [2019-12-17 12:35:09,555] jepsen worker 88 - jepsen.util 9448 :invoke :read [29 nil] INFO [2019-12-17 12:35:09,556] jepsen worker 59 - jepsen.util 11939 :invoke :read [19 nil] INFO [2019-12-17 12:35:09,559] jepsen worker 17 - jepsen.util 12347 :invoke :write [5 34] INFO [2019-12-17 12:35:09,575] jepsen worker 11 - jepsen.util 9191 :fail :cas [3 [19 46]] :ValueNotSwapFrom INFO [2019-12-17 12:35:09,576] jepsen worker 2 - jepsen.util 12242 :invoke :read [0 nil] INFO [2019-12-17 12:35:09,576] jepsen worker 11 - jepsen.util 9191 :invoke :cas [3 [24 33]] INFO [2019-12-17 12:35:09,590] jepsen worker 67 - jepsen.util 12577 :invoke :write [22 14] INFO [2019-12-17 12:35:09,602] jepsen worker 16 - jepsen.util 13606 :invoke :cas [5 [14 27]] INFO [2019-12-17 12:35:09,603] jepsen worker 32 - jepsen.util 9212 :fail :cas [10 [32 18]] :ValueNotSwapFrom INFO [2019-12-17 12:35:09,604] jepsen worker 32 - jepsen.util 9212 :invoke :cas [10 [47 17]] INFO [2019-12-17 12:35:09,626] jepsen worker 29 - jepsen.util 7499 :ok :read [9 48] INFO [2019-12-17 12:35:09,629] jepsen worker 29 - jepsen.util 7499 :invoke :read [9 nil] INFO [2019-12-17 12:35:09,637] jepsen worker 53 - jepsen.util 8243 :info :write [17 19] :RequestCanceledException INFO [2019-12-17 12:35:09,639] jepsen worker 31 - jepsen.util 8131 :ok :read [10 30] INFO [2019-12-17 12:35:09,639] jepsen worker 31 - jepsen.util 8131 :invoke :read [10 nil] INFO [2019-12-17 12:35:09,662] jepsen worker 73 - jepsen.util 8623 :fail :cas [24 [5 7]] :ValueNotSwapFrom INFO [2019-12-17 12:35:09,665] jepsen worker 53 - jepsen.util 8333 :invoke :cas [17 [15 0]] INFO [2019-12-17 12:35:09,667] jepsen worker 73 - jepsen.util 8623 :invoke :cas [24 [5 16]] INFO [2019-12-17 12:35:09,698] jepsen worker 77 - jepsen.util 12857 :info :write [25 7] :RequestCanceledException INFO [2019-12-17 12:35:09,742] jepsen worker 77 - jepsen.util 12947 :invoke :write [25 36] INFO [2019-12-17 12:35:09,755] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52893,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:35:09,756] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52893,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:35:09,756] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52896,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:35:09,756] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52896,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:35:09,831] jepsen worker 86 - jepsen.util 13226 :info :write [28 46] :RequestCanceledException INFO [2019-12-17 12:35:09,858] jepsen worker 78 - jepsen.util 8988 :info :write [26 11] :RequestCanceledException INFO [2019-12-17 12:35:09,861] jepsen worker 86 - jepsen.util 13316 :invoke :cas [28 [42 11]] INFO [2019-12-17 12:35:09,862] jepsen worker 78 - jepsen.util 9078 :invoke :write [26 23] INFO [2019-12-17 12:35:09,904] jepsen worker 20 - jepsen.util 8660 :ok :read [6 30] INFO [2019-12-17 12:35:09,905] jepsen worker 20 - jepsen.util 8660 :invoke :write [6 31] INFO [2019-12-17 12:35:09,916] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52906,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:35:09,917] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52906,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:35:09,917] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52910,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:35:09,919] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52910,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:35:10,084] jepsen worker 89 - jepsen.util 9179 :fail :cas [29 [18 22]] :ValueNotSwapFrom INFO [2019-12-17 12:35:10,085] jepsen worker 89 - jepsen.util 9179 :invoke :cas [29 [13 15]] INFO [2019-12-17 12:35:10,085] jepsen worker 65 - jepsen.util 7625 :fail :cas [21 [14 6]] :ValueNotSwapFrom INFO [2019-12-17 12:35:10,085] jepsen worker 65 - jepsen.util 7625 :invoke :cas [21 [24 47]] INFO [2019-12-17 12:35:10,088] jepsen worker 64 - jepsen.util 7264 :fail :cas [21 [4 10]] :ValueNotSwapFrom INFO [2019-12-17 12:35:10,088] jepsen worker 64 - jepsen.util 7264 :invoke :write [21 30] INFO [2019-12-17 12:35:10,089] jepsen worker 74 - jepsen.util 9074 :fail :cas [24 [30 29]] :ValueNotSwapFrom INFO [2019-12-17 12:35:10,090] jepsen worker 4 - jepsen.util 8644 :ok :read [1 43] INFO [2019-12-17 12:35:10,090] jepsen worker 45 - jepsen.util 9405 :fail :cas [15 [22 29]] :ValueNotSwapFrom INFO [2019-12-17 12:35:10,090] jepsen worker 69 - jepsen.util 9159 :fail :cas [23 [29 19]] :ValueNotSwapFrom INFO [2019-12-17 12:35:10,090] jepsen worker 38 - jepsen.util 9668 :ok :read [12 9] INFO [2019-12-17 12:35:10,091] jepsen worker 56 - jepsen.util 9776 :ok :read [18 20] INFO [2019-12-17 12:35:10,091] jepsen worker 74 - jepsen.util 9074 :invoke :read [24 nil] INFO [2019-12-17 12:35:10,091] jepsen worker 45 - jepsen.util 9405 :invoke :cas [15 [0 4]] INFO [2019-12-17 12:35:10,092] jepsen worker 69 - jepsen.util 9159 :invoke :cas [23 [26 20]] INFO [2019-12-17 12:35:10,092] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52927,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:35:10,092] jepsen worker 15 - jepsen.util 11625 :info :write [5 34] :RequestCanceledException INFO [2019-12-17 12:35:10,092] jepsen worker 14 - jepsen.util 9014 :ok :write [4 3] INFO [2019-12-17 12:35:10,093] jepsen worker 4 - jepsen.util 8644 :invoke :read [1 nil] INFO [2019-12-17 12:35:10,093] jepsen worker 18 - jepsen.util 9288 :info :write [6 26] :RequestCanceledException INFO [2019-12-17 12:35:10,098] jepsen worker 5 - jepsen.util 8555 :ok :read [1 49] INFO [2019-12-17 12:35:10,098] jepsen worker 38 - jepsen.util 9668 :invoke :write [12 40] INFO [2019-12-17 12:35:10,099] jepsen worker 56 - jepsen.util 9776 :invoke :read [18 nil] INFO [2019-12-17 12:35:10,100] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52927,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:35:10,100] jepsen worker 14 - jepsen.util 9014 :invoke :cas [4 [42 15]] INFO [2019-12-17 12:35:10,102] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52930,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:35:10,102] jepsen worker 5 - jepsen.util 8555 :invoke :cas [1 [26 47]] INFO [2019-12-17 12:35:10,103] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52930,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:35:10,106] jepsen worker 27 - jepsen.util 8847 :ok :read [9 48] INFO [2019-12-17 12:35:10,107] jepsen worker 27 - jepsen.util 8847 :invoke :read [9 nil] INFO [2019-12-17 12:35:10,107] jepsen worker 18 - jepsen.util 9378 :invoke :write [6 6] INFO [2019-12-17 12:35:10,108] jepsen worker 3 - jepsen.util 8913 :ok :read [1 43] INFO [2019-12-17 12:35:10,108] jepsen worker 3 - jepsen.util 8913 :invoke :cas [1 [23 37]] INFO [2019-12-17 12:35:10,114] jepsen worker 15 - jepsen.util 11715 :invoke :read [5 nil] INFO [2019-12-17 12:35:10,119] jepsen worker 28 - jepsen.util 7408 :ok :read [9 48] INFO [2019-12-17 12:35:10,119] jepsen worker 28 - jepsen.util 7408 :invoke :write [9 15] INFO [2019-12-17 12:35:10,122] jepsen worker 10 - jepsen.util 9190 :fail :cas [3 [11 41]] :ValueNotSwapFrom INFO [2019-12-17 12:35:10,123] jepsen worker 10 - jepsen.util 9190 :invoke :read [3 nil] INFO [2019-12-17 12:35:10,199] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52946,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:35:10,199] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52946,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:35:10,199] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52947,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:35:10,199] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52947,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:35:10,199] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52948,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:35:10,200] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52948,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:35:10,204] jepsen worker 47 - jepsen.util 9587 :fail :cas [15 [40 25]] :ValueNotSwapFrom INFO [2019-12-17 12:35:10,205] jepsen worker 47 - jepsen.util 9587 :invoke :cas [15 [27 7]] INFO [2019-12-17 12:35:10,206] jepsen worker 29 - jepsen.util 7499 :ok :read [9 48] INFO [2019-12-17 12:35:10,207] jepsen worker 29 - jepsen.util 7499 :invoke :cas [9 [13 38]] INFO [2019-12-17 12:35:10,227] jepsen worker 60 - jepsen.util 13380 :info :cas [20 [13 34]] :RequestCanceledException INFO [2019-12-17 12:35:10,227] jepsen worker 72 - jepsen.util 9072 :ok :read [24 45] INFO [2019-12-17 12:35:10,227] jepsen worker 72 - jepsen.util 9072 :invoke :cas [24 [8 29]] INFO [2019-12-17 12:35:10,231] jepsen worker 71 - jepsen.util 9611 :info :write [23 28] :RequestCanceledException INFO [2019-12-17 12:35:10,240] jepsen worker 60 - jepsen.util 13470 :invoke :write [20 19] INFO [2019-12-17 12:35:10,264] jepsen worker 87 - jepsen.util 8727 :fail :read [29 nil] :RequestTimeoutException INFO [2019-12-17 12:35:10,265] jepsen worker 71 - jepsen.util 9701 :invoke :cas [23 [8 38]] INFO [2019-12-17 12:35:10,267] jepsen worker 87 - jepsen.util 8727 :invoke :write [29 9] INFO [2019-12-17 12:35:10,300] jepsen worker 37 - jepsen.util 8587 :fail :cas [12 [36 34]] :ValueNotSwapFrom INFO [2019-12-17 12:35:10,301] jepsen worker 37 - jepsen.util 8587 :invoke :write [12 10] INFO [2019-12-17 12:35:10,301] jepsen worker 32 - jepsen.util 9212 :fail :cas [10 [47 17]] :ValueNotSwapFrom INFO [2019-12-17 12:35:10,301] jepsen worker 32 - jepsen.util 9212 :invoke :cas [10 [48 5]] INFO [2019-12-17 12:35:10,308] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52955,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:35:10,308] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52955,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:35:10,308] jepsen worker 76 - jepsen.util 12766 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:35:10,308] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52959,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:35:10,308] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52959,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:35:10,308] jepsen worker 76 - jepsen.util 12766 :invoke :write [25 17] INFO [2019-12-17 12:35:10,311] jepsen worker 42 - jepsen.util 13362 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:35:10,315] jepsen worker 42 - jepsen.util 13362 :invoke :write [14 15] INFO [2019-12-17 12:35:10,364] jepsen worker 43 - jepsen.util 12283 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:35:10,364] jepsen worker 43 - jepsen.util 12283 :invoke :write [14 5] INFO [2019-12-17 12:35:10,390] jepsen worker 6 - jepsen.util 12966 :info :cas [2 [47 5]] :RequestCanceledException INFO [2019-12-17 12:35:10,399] jepsen worker 11 - jepsen.util 9191 :fail :cas [3 [24 33]] :ValueNotSwapFrom INFO [2019-12-17 12:35:10,399] jepsen worker 11 - jepsen.util 9191 :invoke :write [3 25] INFO [2019-12-17 12:35:10,410] jepsen worker 6 - jepsen.util 13056 :invoke :read [2 nil] INFO [2019-12-17 12:35:10,430] jepsen worker 57 - jepsen.util 12207 :info :cas [19 [31 40]] :RequestCanceledException INFO [2019-12-17 12:35:10,433] jepsen worker 88 - jepsen.util 9448 :ok :read [29 16] INFO [2019-12-17 12:35:10,433] jepsen worker 88 - jepsen.util 9448 :invoke :write [29 31] INFO [2019-12-17 12:35:10,434] jepsen worker 57 - jepsen.util 12297 :invoke :read [19 nil] INFO [2019-12-17 12:35:10,436] jepsen worker 19 - jepsen.util 9199 :ok :read [6 30] INFO [2019-12-17 12:35:10,436] jepsen worker 19 - jepsen.util 9199 :invoke :read [6 nil] INFO [2019-12-17 12:35:10,441] jepsen worker 73 - jepsen.util 8623 :fail :cas [24 [5 16]] :ValueNotSwapFrom INFO [2019-12-17 12:35:10,443] jepsen worker 73 - jepsen.util 8623 :invoke :read [24 nil] INFO [2019-12-17 12:35:10,444] jepsen worker 35 - jepsen.util 12455 :info :cas [11 [37 34]] :RequestCanceledException INFO [2019-12-17 12:35:10,458] jepsen worker 35 - jepsen.util 12545 :invoke :write [11 4] INFO [2019-12-17 12:35:10,497] jepsen worker 44 - jepsen.util 12824 :info :cas [14 [41 0]] :RequestCanceledException INFO [2019-12-17 12:35:10,500] jepsen worker 51 - jepsen.util 8331 :fail :cas [17 [25 5]] :ValueNotSwapFrom INFO [2019-12-17 12:35:10,500] jepsen worker 51 - jepsen.util 8331 :invoke :read [17 nil] INFO [2019-12-17 12:35:10,502] jepsen worker 44 - jepsen.util 12914 :invoke :read [14 nil] INFO [2019-12-17 12:35:10,512] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:35:10,512] jepsen nemesis - couchbase.util Rebalance check count: 413 INFO [2019-12-17 12:35:10,512] jepsen nemesis - couchbase.util Rebalance stuck count: 394 INFO [2019-12-17 12:35:10,544] jepsen worker 24 - jepsen.util 11454 :info :cas [8 [6 39]] :RequestCanceledException INFO [2019-12-17 12:35:10,546] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52973,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:35:10,548] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52973,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:35:10,548] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52976,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:35:10,548] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52976,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:35:10,549] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52977,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:35:10,549] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52977,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:35:10,550] jepsen worker 24 - jepsen.util 11544 :invoke :cas [8 [32 43]] INFO [2019-12-17 12:35:10,615] jepsen worker 26 - jepsen.util 13076 :info :write [8 7] :RequestCanceledException INFO [2019-12-17 12:35:10,622] jepsen worker 53 - jepsen.util 8333 :fail :cas [17 [15 0]] :ValueNotSwapFrom INFO [2019-12-17 12:35:10,631] jepsen worker 85 - jepsen.util 13315 :info :write [28 5] :RequestCanceledException INFO [2019-12-17 12:35:10,632] jepsen worker 23 - jepsen.util 12263 :info :write [7 10] :RequestCanceledException INFO [2019-12-17 12:35:10,645] jepsen worker 62 - jepsen.util 12752 :info :write [20 5] :RequestCanceledException INFO [2019-12-17 12:35:10,651] jepsen worker 53 - jepsen.util 8333 :invoke :write [17 25] INFO [2019-12-17 12:35:10,657] jepsen worker 31 - jepsen.util 8131 :ok :read [10 30] INFO [2019-12-17 12:35:10,658] jepsen worker 31 - jepsen.util 8131 :invoke :read [10 nil] INFO [2019-12-17 12:35:10,671] jepsen worker 26 - jepsen.util 13166 :invoke :cas [8 [44 30]] INFO [2019-12-17 12:35:10,671] jepsen worker 62 - jepsen.util 12842 :invoke :read [20 nil] INFO [2019-12-17 12:35:10,672] jepsen worker 23 - jepsen.util 12353 :invoke :cas [7 [18 40]] INFO [2019-12-17 12:35:10,672] jepsen worker 85 - jepsen.util 13405 :invoke :write [28 28] INFO [2019-12-17 12:35:10,716] jepsen worker 49 - jepsen.util 11839 :info :write [16 4] :RequestCanceledException INFO [2019-12-17 12:35:10,734] jepsen worker 49 - jepsen.util 11929 :invoke :write [16 25] INFO [2019-12-17 12:35:10,738] jepsen worker 74 - jepsen.util 9074 :ok :read [24 45] INFO [2019-12-17 12:35:10,738] jepsen worker 74 - jepsen.util 9074 :invoke :read [24 nil] INFO [2019-12-17 12:35:10,744] jepsen worker 55 - jepsen.util 9865 :fail :cas [18 [22 46]] :ValueNotSwapFrom INFO [2019-12-17 12:35:10,744] jepsen worker 55 - jepsen.util 9865 :invoke :read [18 nil] INFO [2019-12-17 12:35:10,744] jepsen worker 83 - jepsen.util 12953 :info :write [27 9] :RequestCanceledException INFO [2019-12-17 12:35:10,748] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52988,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:35:10,749] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52988,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:35:10,749] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52991,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:35:10,751] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52991,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:35:10,751] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52992,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:35:10,752] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":52992,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:35:10,756] jepsen worker 83 - jepsen.util 13043 :invoke :write [27 8] INFO [2019-12-17 12:35:10,782] jepsen worker 65 - jepsen.util 7625 :fail :cas [21 [24 47]] :ValueNotSwapFrom INFO [2019-12-17 12:35:10,783] jepsen worker 65 - jepsen.util 7625 :invoke :write [21 40] INFO [2019-12-17 12:35:10,784] jepsen worker 89 - jepsen.util 9179 :fail :cas [29 [13 15]] :ValueNotSwapFrom INFO [2019-12-17 12:35:10,784] jepsen worker 89 - jepsen.util 9179 :invoke :write [29 33] INFO [2019-12-17 12:35:10,791] jepsen worker 10 - jepsen.util 9190 :ok :read [3 16] INFO [2019-12-17 12:35:10,792] jepsen worker 10 - jepsen.util 9190 :invoke :write [3 28] INFO [2019-12-17 12:35:10,792] jepsen worker 19 - jepsen.util 9199 :ok :read [6 30] INFO [2019-12-17 12:35:10,792] jepsen worker 19 - jepsen.util 9199 :invoke :read [6 nil] INFO [2019-12-17 12:35:10,796] jepsen worker 47 - jepsen.util 9587 :fail :cas [15 [27 7]] :ValueNotSwapFrom INFO [2019-12-17 12:35:10,799] jepsen worker 47 - jepsen.util 9587 :invoke :write [15 4] INFO [2019-12-17 12:35:10,827] jepsen worker 69 - jepsen.util 9159 :fail :cas [23 [26 20]] :ValueNotSwapFrom INFO [2019-12-17 12:35:10,827] jepsen worker 69 - jepsen.util 9159 :invoke :read [23 nil] INFO [2019-12-17 12:35:10,861] jepsen worker 68 - jepsen.util 11858 :info :write [22 10] :RequestCanceledException INFO [2019-12-17 12:35:10,863] jepsen worker 68 - jepsen.util 11948 :invoke :read [22 nil] INFO [2019-12-17 12:35:10,866] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53013,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:35:10,866] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53013,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:35:10,866] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53015,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:35:10,866] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53015,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:35:10,892] jepsen worker 25 - jepsen.util 13435 :info :cas [8 [35 17]] :RequestCanceledException INFO [2019-12-17 12:35:10,901] jepsen worker 25 - jepsen.util 13525 :invoke :write [8 7] INFO [2019-12-17 12:35:10,914] jepsen worker 66 - jepsen.util 12216 :info :cas [22 [20 27]] :RequestCanceledException INFO [2019-12-17 12:35:10,918] jepsen worker 66 - jepsen.util 12306 :invoke :cas [22 [0 49]] INFO [2019-12-17 12:35:10,925] jepsen worker 58 - jepsen.util 13198 :info :cas [19 [33 23]] :RequestCanceledException INFO [2019-12-17 12:35:10,925] jepsen worker 40 - jepsen.util 12820 :info :cas [13 [16 48]] :RequestCanceledException INFO [2019-12-17 12:35:10,932] jepsen worker 58 - jepsen.util 13288 :invoke :read [19 nil] INFO [2019-12-17 12:35:10,934] jepsen worker 40 - jepsen.util 12910 :invoke :read [13 nil] INFO [2019-12-17 12:35:10,945] jepsen worker 9 - jepsen.util 9729 :fail :read [3 nil] :RequestTimeoutException INFO [2019-12-17 12:35:10,945] jepsen worker 9 - jepsen.util 9729 :invoke :write [3 30] INFO [2019-12-17 12:35:11,038] jepsen worker 0 - jepsen.util 13140 :info :write [0 44] :RequestCanceledException INFO [2019-12-17 12:35:11,041] jepsen worker 0 - jepsen.util 13230 :invoke :write [0 12] INFO [2019-12-17 12:35:11,082] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53021,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:35:11,082] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53021,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:35:11,083] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53025,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:35:11,083] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53025,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:35:11,083] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53026,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:35:11,083] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53026,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:35:11,138] jepsen worker 13 - jepsen.util 8653 :ok :write [4 10] INFO [2019-12-17 12:35:11,138] jepsen worker 82 - jepsen.util 13222 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:35:11,139] jepsen worker 13 - jepsen.util 8653 :invoke :cas [4 [23 15]] INFO [2019-12-17 12:35:11,139] jepsen worker 82 - jepsen.util 13222 :invoke :cas [27 [26 25]] INFO [2019-12-17 12:35:11,139] jepsen worker 52 - jepsen.util 9322 :ok :write [17 17] INFO [2019-12-17 12:35:11,139] jepsen worker 52 - jepsen.util 9322 :invoke :read [17 nil] INFO [2019-12-17 12:35:11,155] jepsen worker 74 - jepsen.util 9074 :ok :read [24 45] INFO [2019-12-17 12:35:11,156] jepsen worker 74 - jepsen.util 9074 :invoke :cas [24 [41 8]] INFO [2019-12-17 12:35:11,173] jepsen worker 31 - jepsen.util 8131 :ok :read [10 30] INFO [2019-12-17 12:35:11,174] jepsen worker 31 - jepsen.util 8131 :invoke :read [10 nil] INFO [2019-12-17 12:35:11,197] jepsen worker 19 - jepsen.util 9199 :ok :read [6 30] INFO [2019-12-17 12:35:11,197] jepsen worker 19 - jepsen.util 9199 :invoke :cas [6 [1 4]] INFO [2019-12-17 12:35:11,205] jepsen worker 79 - jepsen.util 9439 :ok :write [26 23] INFO [2019-12-17 12:35:11,205] jepsen worker 79 - jepsen.util 9439 :invoke :read [26 nil] INFO [2019-12-17 12:35:11,229] jepsen worker 61 - jepsen.util 13381 :info :cas [20 [8 1]] :RequestCanceledException INFO [2019-12-17 12:35:11,229] jepsen worker 7 - jepsen.util 12517 :info :cas [2 [7 49]] :RequestCanceledException INFO [2019-12-17 12:35:11,231] jepsen worker 1 - jepsen.util 12601 :info :cas [0 [16 2]] :RequestCanceledException INFO [2019-12-17 12:35:11,232] jepsen worker 61 - jepsen.util 13471 :invoke :read [20 nil] INFO [2019-12-17 12:35:11,232] jepsen worker 7 - jepsen.util 12607 :invoke :read [2 nil] INFO [2019-12-17 12:35:11,232] jepsen worker 1 - jepsen.util 12691 :invoke :write [0 2] INFO [2019-12-17 12:35:11,272] jepsen worker 56 - jepsen.util 9776 :ok :read [18 20] INFO [2019-12-17 12:35:11,273] jepsen worker 56 - jepsen.util 9776 :invoke :read [18 nil] INFO [2019-12-17 12:35:11,280] jepsen worker 37 - jepsen.util 8587 :ok :write [12 10] INFO [2019-12-17 12:35:11,280] jepsen worker 37 - jepsen.util 8587 :invoke :cas [12 [41 1]] INFO [2019-12-17 12:35:11,282] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53035,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:35:11,283] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53044,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:35:11,283] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53035,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:35:11,283] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53044,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:35:11,283] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53046,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:35:11,283] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53046,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:35:11,328] jepsen worker 75 - jepsen.util 11865 :info :write [25 18] :RequestCanceledException INFO [2019-12-17 12:35:11,328] jepsen worker 70 - jepsen.util 9790 :fail :read [23 nil] :RequestTimeoutException INFO [2019-12-17 12:35:11,328] jepsen worker 70 - jepsen.util 9790 :invoke :write [23 2] INFO [2019-12-17 12:35:11,340] jepsen worker 3 - jepsen.util 8913 :fail :cas [1 [23 37]] :ValueNotSwapFrom INFO [2019-12-17 12:35:11,340] jepsen worker 3 - jepsen.util 8913 :invoke :read [1 nil] INFO [2019-12-17 12:35:11,342] jepsen worker 75 - jepsen.util 11955 :invoke :write [25 24] INFO [2019-12-17 12:35:11,353] jepsen worker 12 - jepsen.util 9642 :ok :write [4 10] INFO [2019-12-17 12:35:11,353] jepsen worker 12 - jepsen.util 9642 :invoke :read [4 nil] INFO [2019-12-17 12:35:11,374] jepsen worker 65 - jepsen.util 7625 :ok :write [21 40] INFO [2019-12-17 12:35:11,375] jepsen worker 65 - jepsen.util 7625 :invoke :write [21 8] INFO [2019-12-17 12:35:11,400] jepsen worker 56 - jepsen.util 9776 :ok :read [18 20] INFO [2019-12-17 12:35:11,401] jepsen worker 56 - jepsen.util 9776 :invoke :read [18 nil] INFO [2019-12-17 12:35:11,407] jepsen worker 48 - jepsen.util 11568 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:35:11,407] jepsen worker 48 - jepsen.util 11568 :invoke :write [16 7] INFO [2019-12-17 12:35:11,439] jepsen worker 13 - jepsen.util 8653 :fail :cas [4 [23 15]] :ValueNotSwapFrom INFO [2019-12-17 12:35:11,439] jepsen worker 13 - jepsen.util 8653 :invoke :cas [4 [24 29]] INFO [2019-12-17 12:35:11,451] jepsen worker 30 - jepsen.util 9120 :ok :write [10 45] INFO [2019-12-17 12:35:11,451] jepsen worker 30 - jepsen.util 9120 :invoke :read [10 nil] INFO [2019-12-17 12:35:11,451] jepsen worker 20 - jepsen.util 8660 :ok :write [6 31] INFO [2019-12-17 12:35:11,451] jepsen worker 20 - jepsen.util 8660 :invoke :write [6 3] INFO [2019-12-17 12:35:11,476] jepsen worker 54 - jepsen.util 10044 :ok :write [18 46] INFO [2019-12-17 12:35:11,476] jepsen worker 54 - jepsen.util 10044 :invoke :write [18 27] INFO [2019-12-17 12:35:11,486] jepsen worker 46 - jepsen.util 9136 :ok :write [15 17] INFO [2019-12-17 12:35:11,488] jepsen worker 46 - jepsen.util 9136 :invoke :read [15 nil] INFO [2019-12-17 12:35:11,509] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53059,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:35:11,509] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53059,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:35:11,511] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53062,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:35:11,512] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53062,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:35:11,522] jepsen worker 5 - jepsen.util 8555 :fail :cas [1 [26 47]] :ValueNotSwapFrom INFO [2019-12-17 12:35:11,522] jepsen worker 5 - jepsen.util 8555 :invoke :read [1 nil] INFO [2019-12-17 12:35:11,528] jepsen worker 81 - jepsen.util 12231 :info :write [27 11] :RequestCanceledException INFO [2019-12-17 12:35:11,528] jepsen worker 39 - jepsen.util 12009 :info :cas [13 [45 49]] :RequestCanceledException INFO [2019-12-17 12:35:11,529] jepsen worker 78 - jepsen.util 9078 :ok :write [26 23] INFO [2019-12-17 12:35:11,530] jepsen worker 78 - jepsen.util 9078 :invoke :cas [26 [5 22]] INFO [2019-12-17 12:35:11,536] jepsen worker 81 - jepsen.util 12321 :invoke :write [27 5] INFO [2019-12-17 12:35:11,538] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:35:11,538] jepsen nemesis - couchbase.util Rebalance check count: 414 INFO [2019-12-17 12:35:11,538] jepsen nemesis - couchbase.util Rebalance stuck count: 395 INFO [2019-12-17 12:35:11,543] jepsen worker 63 - jepsen.util 7893 :info :write [21 17] :RequestCanceledException INFO [2019-12-17 12:35:11,550] jepsen worker 64 - jepsen.util 7264 :ok :write [21 30] INFO [2019-12-17 12:35:11,550] jepsen worker 64 - jepsen.util 7264 :invoke :cas [21 [2 17]] INFO [2019-12-17 12:35:11,556] jepsen worker 39 - jepsen.util 12099 :invoke :cas [13 [46 21]] INFO [2019-12-17 12:35:11,557] jepsen worker 33 - jepsen.util 12543 :info :cas [11 [41 28]] :RequestCanceledException INFO [2019-12-17 12:35:11,560] jepsen worker 63 - jepsen.util 7983 :invoke :read [21 nil] INFO [2019-12-17 12:35:11,560] jepsen worker 33 - jepsen.util 12633 :invoke :read [11 nil] INFO [2019-12-17 12:35:11,565] jepsen worker 56 - jepsen.util 9776 :ok :read [18 46] INFO [2019-12-17 12:35:11,567] jepsen worker 56 - jepsen.util 9776 :invoke :read [18 nil] INFO [2019-12-17 12:35:11,567] jepsen worker 10 - jepsen.util 9190 :ok :write [3 28] INFO [2019-12-17 12:35:11,567] jepsen worker 10 - jepsen.util 9190 :invoke :read [3 nil] INFO [2019-12-17 12:35:11,581] jepsen worker 13 - jepsen.util 8653 :fail :cas [4 [24 29]] :ValueNotSwapFrom INFO [2019-12-17 12:35:11,581] jepsen worker 13 - jepsen.util 8653 :invoke :read [4 nil] INFO [2019-12-17 12:35:11,593] jepsen worker 3 - jepsen.util 8913 :ok :read [1 49] INFO [2019-12-17 12:35:11,593] jepsen worker 3 - jepsen.util 8913 :invoke :write [1 39] INFO [2019-12-17 12:35:11,605] jepsen worker 31 - jepsen.util 8131 :ok :read [10 30] INFO [2019-12-17 12:35:11,605] jepsen worker 31 - jepsen.util 8131 :invoke :cas [10 [43 45]] INFO [2019-12-17 12:35:11,628] jepsen worker 59 - jepsen.util 11939 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:35:11,645] jepsen worker 19 - jepsen.util 9199 :fail :cas [6 [1 4]] :ValueNotSwapFrom INFO [2019-12-17 12:35:11,661] jepsen worker 2 - jepsen.util 12242 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:35:11,668] jepsen worker 19 - jepsen.util 9199 :invoke :read [6 nil] INFO [2019-12-17 12:35:11,668] jepsen worker 2 - jepsen.util 12242 :invoke :cas [0 [45 3]] INFO [2019-12-17 12:35:11,670] jepsen worker 59 - jepsen.util 11939 :invoke :read [19 nil] INFO [2019-12-17 12:35:11,696] jepsen worker 16 - jepsen.util 13606 :info :cas [5 [14 27]] :RequestCanceledException INFO [2019-12-17 12:35:11,714] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53086,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:35:11,720] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53086,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:35:11,721] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53087,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:35:11,722] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53087,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:35:11,727] jepsen worker 16 - jepsen.util 13696 :invoke :write [5 3] INFO [2019-12-17 12:35:11,795] jepsen worker 38 - jepsen.util 9668 :ok :write [12 40] INFO [2019-12-17 12:35:11,796] jepsen worker 38 - jepsen.util 9668 :invoke :write [12 24] INFO [2019-12-17 12:35:11,815] jepsen worker 74 - jepsen.util 9074 :fail :cas [24 [41 8]] :ValueNotSwapFrom INFO [2019-12-17 12:35:11,816] jepsen worker 74 - jepsen.util 9074 :invoke :cas [24 [35 3]] INFO [2019-12-17 12:35:11,828] jepsen worker 4 - jepsen.util 8644 :ok :read [1 49] INFO [2019-12-17 12:35:11,829] jepsen worker 4 - jepsen.util 8644 :invoke :write [1 31] INFO [2019-12-17 12:35:11,832] jepsen worker 30 - jepsen.util 9120 :ok :read [10 45] INFO [2019-12-17 12:35:11,834] jepsen worker 30 - jepsen.util 9120 :invoke :write [10 25] INFO [2019-12-17 12:35:11,835] jepsen worker 64 - jepsen.util 7264 :fail :cas [21 [2 17]] :ValueNotSwapFrom INFO [2019-12-17 12:35:11,835] jepsen worker 64 - jepsen.util 7264 :invoke :write [21 35] INFO [2019-12-17 12:35:11,842] jepsen worker 51 - jepsen.util 8331 :ok :read [17 17] INFO [2019-12-17 12:35:11,842] jepsen worker 51 - jepsen.util 8331 :invoke :read [17 nil] INFO [2019-12-17 12:35:11,843] jepsen worker 8 - jepsen.util 12158 :info :write [2 17] :RequestCanceledException INFO [2019-12-17 12:35:11,847] jepsen worker 14 - jepsen.util 9014 :fail :cas [4 [42 15]] :ValueNotSwapFrom INFO [2019-12-17 12:35:11,847] jepsen worker 14 - jepsen.util 9014 :invoke :read [4 nil] INFO [2019-12-17 12:35:11,854] jepsen worker 8 - jepsen.util 12248 :invoke :read [2 nil] INFO [2019-12-17 12:35:11,858] jepsen worker 80 - jepsen.util 8810 :info :write [26 41] :RequestCanceledException INFO [2019-12-17 12:35:11,864] jepsen worker 80 - jepsen.util 8900 :invoke :cas [26 [19 14]] INFO [2019-12-17 12:35:11,868] jepsen worker 10 - jepsen.util 9190 :ok :read [3 28] INFO [2019-12-17 12:35:11,869] jepsen worker 10 - jepsen.util 9190 :invoke :cas [3 [44 8]] INFO [2019-12-17 12:35:11,888] jepsen worker 28 - jepsen.util 7408 :ok :write [9 15] INFO [2019-12-17 12:35:11,889] jepsen worker 28 - jepsen.util 7408 :invoke :write [9 30] INFO [2019-12-17 12:35:11,912] jepsen worker 65 - jepsen.util 7625 :ok :write [21 8] INFO [2019-12-17 12:35:11,912] jepsen worker 65 - jepsen.util 7625 :invoke :cas [21 [7 5]] INFO [2019-12-17 12:35:11,916] jepsen worker 41 - jepsen.util 13271 :info :write [13 11] :RequestCanceledException INFO [2019-12-17 12:35:11,930] jepsen worker 77 - jepsen.util 12947 :info :write [25 36] :RequestCanceledException INFO [2019-12-17 12:35:11,933] jepsen worker 22 - jepsen.util 12352 :info :write [7 27] :RequestCanceledException INFO [2019-12-17 12:35:11,944] jepsen worker 84 - jepsen.util 11964 :info :write [28 25] :RequestCanceledException INFO [2019-12-17 12:35:11,944] jepsen worker 34 - jepsen.util 12634 :info :write [11 46] :RequestCanceledException INFO [2019-12-17 12:35:11,944] jepsen worker 22 - jepsen.util 12442 :invoke :read [7 nil] INFO [2019-12-17 12:35:11,946] jepsen worker 86 - jepsen.util 13316 :info :cas [28 [42 11]] :RequestCanceledException INFO [2019-12-17 12:35:11,948] jepsen worker 77 - jepsen.util 13037 :invoke :cas [25 [8 33]] INFO [2019-12-17 12:35:11,954] jepsen worker 34 - jepsen.util 12724 :invoke :read [11 nil] INFO [2019-12-17 12:35:11,954] jepsen worker 84 - jepsen.util 12054 :invoke :cas [28 [46 2]] INFO [2019-12-17 12:35:11,957] jepsen worker 86 - jepsen.util 13406 :invoke :write [28 31] INFO [2019-12-17 12:35:11,958] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53098,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:35:11,958] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53098,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:35:11,959] jepsen worker 36 - jepsen.util 9486 :info :write [12 25] :RequestCanceledException INFO [2019-12-17 12:35:11,959] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53101,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:35:11,959] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53101,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:35:11,959] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53102,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:35:11,960] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53102,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:35:11,966] jepsen worker 41 - jepsen.util 13361 :invoke :read [13 nil] INFO [2019-12-17 12:35:11,972] jepsen worker 36 - jepsen.util 9576 :invoke :write [12 36] INFO [2019-12-17 12:35:11,996] jepsen worker 50 - jepsen.util 13100 :info :write [16 30] :RequestCanceledException INFO [2019-12-17 12:35:12,029] jepsen worker 21 - jepsen.util 12261 :info :write [7 27] :RequestCanceledException INFO [2019-12-17 12:35:12,031] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53120,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:35:12,031] jepsen worker 67 - jepsen.util 12577 :info :write [22 14] :RequestCanceledException INFO [2019-12-17 12:35:12,032] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53120,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:35:12,032] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53121,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:35:12,033] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53121,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:35:12,036] jepsen worker 50 - jepsen.util 13190 :invoke :cas [16 [39 2]] INFO [2019-12-17 12:35:12,038] jepsen worker 63 - jepsen.util 7983 :ok :read [21 30] INFO [2019-12-17 12:35:12,038] jepsen worker 63 - jepsen.util 7983 :invoke :cas [21 [17 10]] INFO [2019-12-17 12:35:12,041] jepsen worker 21 - jepsen.util 12351 :invoke :cas [7 [1 25]] INFO [2019-12-17 12:35:12,042] jepsen worker 67 - jepsen.util 12667 :invoke :cas [22 [42 29]] INFO [2019-12-17 12:35:12,073] jepsen worker 19 - jepsen.util 9199 :ok :read [6 31] INFO [2019-12-17 12:35:12,073] jepsen worker 19 - jepsen.util 9199 :invoke :write [6 42] INFO [2019-12-17 12:35:12,074] jepsen worker 74 - jepsen.util 9074 :fail :cas [24 [35 3]] :ValueNotSwapFrom INFO [2019-12-17 12:35:12,074] jepsen worker 74 - jepsen.util 9074 :invoke :cas [24 [19 0]] INFO [2019-12-17 12:35:12,083] jepsen worker 55 - jepsen.util 9865 :ok :read [18 46] INFO [2019-12-17 12:35:12,083] jepsen worker 55 - jepsen.util 9865 :invoke :cas [18 [9 31]] INFO [2019-12-17 12:35:12,088] jepsen worker 65 - jepsen.util 7625 :fail :cas [21 [7 5]] :ValueNotSwapFrom INFO [2019-12-17 12:35:12,088] jepsen worker 65 - jepsen.util 7625 :invoke :write [21 3] INFO [2019-12-17 12:35:12,088] jepsen worker 10 - jepsen.util 9190 :fail :cas [3 [44 8]] :ValueNotSwapFrom INFO [2019-12-17 12:35:12,089] jepsen worker 10 - jepsen.util 9190 :invoke :write [3 0] INFO [2019-12-17 12:35:12,144] jepsen worker 17 - jepsen.util 12347 :info :write [5 34] :RequestCanceledException INFO [2019-12-17 12:35:12,146] jepsen worker 17 - jepsen.util 12437 :invoke :write [5 22] INFO [2019-12-17 12:35:12,197] jepsen worker 88 - jepsen.util 9448 :ok :write [29 31] INFO [2019-12-17 12:35:12,197] jepsen worker 88 - jepsen.util 9448 :invoke :cas [29 [19 0]] INFO [2019-12-17 12:35:12,197] jepsen worker 15 - jepsen.util 11715 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:35:12,198] jepsen worker 15 - jepsen.util 11715 :invoke :read [5 nil] INFO [2019-12-17 12:35:12,198] jepsen worker 46 - jepsen.util 9136 :ok :read [15 17] INFO [2019-12-17 12:35:12,198] jepsen worker 46 - jepsen.util 9136 :invoke :write [15 48] INFO [2019-12-17 12:35:12,198] jepsen worker 27 - jepsen.util 8847 :fail :read [9 nil] :RequestTimeoutException INFO [2019-12-17 12:35:12,198] jepsen worker 27 - jepsen.util 8847 :invoke :cas [9 [31 22]] INFO [2019-12-17 12:35:12,198] jepsen worker 45 - jepsen.util 9405 :info :cas [15 [0 4]] :RequestCanceledException INFO [2019-12-17 12:35:12,206] jepsen worker 45 - jepsen.util 9495 :invoke :write [15 5] INFO [2019-12-17 12:35:12,222] jepsen worker 18 - jepsen.util 9378 :info :write [6 6] :RequestCanceledException INFO [2019-12-17 12:35:12,225] jepsen worker 29 - jepsen.util 7499 :info :cas [9 [13 38]] :RequestCanceledException INFO [2019-12-17 12:35:12,230] jepsen worker 18 - jepsen.util 9468 :invoke :cas [6 [22 20]] INFO [2019-12-17 12:35:12,247] jepsen worker 29 - jepsen.util 7589 :invoke :write [9 38] INFO [2019-12-17 12:35:12,248] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53132,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:35:12,248] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53132,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:35:12,248] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53134,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:35:12,248] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53134,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:35:12,250] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53142,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:35:12,251] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53142,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:35:12,251] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53145,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:35:12,251] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53145,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:35:12,251] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53146,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:35:12,251] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53146,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:35:12,302] jepsen worker 37 - jepsen.util 8587 :fail :cas [12 [41 1]] :ValueNotSwapFrom INFO [2019-12-17 12:35:12,302] jepsen worker 37 - jepsen.util 8587 :invoke :cas [12 [24 6]] INFO [2019-12-17 12:35:12,304] jepsen worker 28 - jepsen.util 7408 :ok :write [9 30] INFO [2019-12-17 12:35:12,304] jepsen worker 28 - jepsen.util 7408 :invoke :write [9 37] INFO [2019-12-17 12:35:12,307] jepsen worker 74 - jepsen.util 9074 :fail :cas [24 [19 0]] :ValueNotSwapFrom INFO [2019-12-17 12:35:12,307] jepsen worker 74 - jepsen.util 9074 :invoke :read [24 nil] INFO [2019-12-17 12:35:12,330] jepsen worker 69 - jepsen.util 9159 :ok :read [23 4] INFO [2019-12-17 12:35:12,330] jepsen worker 71 - jepsen.util 9701 :info :cas [23 [8 38]] :RequestCanceledException INFO [2019-12-17 12:35:12,330] jepsen worker 72 - jepsen.util 9072 :info :cas [24 [8 29]] :RequestCanceledException INFO [2019-12-17 12:35:12,330] jepsen worker 69 - jepsen.util 9159 :invoke :cas [23 [45 25]] INFO [2019-12-17 12:35:12,350] jepsen worker 13 - jepsen.util 8653 :ok :read [4 10] INFO [2019-12-17 12:35:12,350] jepsen worker 13 - jepsen.util 8653 :invoke :read [4 nil] INFO [2019-12-17 12:35:12,357] jepsen worker 72 - jepsen.util 9162 :invoke :cas [24 [46 37]] INFO [2019-12-17 12:35:12,360] jepsen worker 56 - jepsen.util 9776 :ok :read [18 46] INFO [2019-12-17 12:35:12,361] jepsen worker 32 - jepsen.util 9212 :info :cas [10 [48 5]] :RequestCanceledException INFO [2019-12-17 12:35:12,361] jepsen worker 87 - jepsen.util 8727 :info :write [29 9] :RequestCanceledException INFO [2019-12-17 12:35:12,363] jepsen worker 56 - jepsen.util 9776 :invoke :cas [18 [12 16]] INFO [2019-12-17 12:35:12,366] jepsen worker 87 - jepsen.util 8817 :invoke :write [29 36] INFO [2019-12-17 12:35:12,369] jepsen worker 32 - jepsen.util 9302 :invoke :write [10 9] INFO [2019-12-17 12:35:12,373] jepsen worker 71 - jepsen.util 9791 :invoke :cas [23 [45 6]] INFO [2019-12-17 12:35:12,386] jepsen worker 20 - jepsen.util 8660 :ok :write [6 3] INFO [2019-12-17 12:35:12,387] jepsen worker 20 - jepsen.util 8660 :invoke :cas [6 [24 45]] INFO [2019-12-17 12:35:12,408] jepsen worker 60 - jepsen.util 13470 :info :write [20 19] :RequestCanceledException INFO [2019-12-17 12:35:12,409] jepsen worker 42 - jepsen.util 13362 :info :write [14 15] :RequestCanceledException INFO [2019-12-17 12:35:12,410] jepsen worker 76 - jepsen.util 12766 :info :write [25 17] :RequestCanceledException INFO [2019-12-17 12:35:12,416] jepsen worker 60 - jepsen.util 13560 :invoke :cas [20 [5 45]] INFO [2019-12-17 12:35:12,422] jepsen worker 42 - jepsen.util 13452 :invoke :write [14 2] INFO [2019-12-17 12:35:12,423] jepsen worker 76 - jepsen.util 12856 :invoke :write [25 38] INFO [2019-12-17 12:35:12,432] jepsen worker 78 - jepsen.util 9078 :fail :cas [26 [5 22]] :ValueNotSwapFrom INFO [2019-12-17 12:35:12,432] jepsen worker 78 - jepsen.util 9078 :invoke :cas [26 [12 2]] INFO [2019-12-17 12:35:12,443] jepsen worker 80 - jepsen.util 8900 :fail :cas [26 [19 14]] :ValueNotSwapFrom INFO [2019-12-17 12:35:12,443] jepsen worker 80 - jepsen.util 8900 :invoke :cas [26 [37 4]] INFO [2019-12-17 12:35:12,444] jepsen worker 6 - jepsen.util 13056 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:35:12,444] jepsen worker 6 - jepsen.util 13056 :invoke :cas [2 [6 30]] INFO [2019-12-17 12:35:12,445] jepsen worker 27 - jepsen.util 8847 :fail :cas [9 [31 22]] :ValueNotSwapFrom INFO [2019-12-17 12:35:12,445] jepsen worker 27 - jepsen.util 8847 :invoke :read [9 nil] INFO [2019-12-17 12:35:12,459] jepsen worker 43 - jepsen.util 12283 :info :write [14 5] :RequestCanceledException INFO [2019-12-17 12:35:12,459] jepsen worker 57 - jepsen.util 12297 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:35:12,459] jepsen worker 73 - jepsen.util 8623 :fail :read [24 nil] :RequestTimeoutException INFO [2019-12-17 12:35:12,459] jepsen worker 57 - jepsen.util 12297 :invoke :read [19 nil] INFO [2019-12-17 12:35:12,460] jepsen worker 73 - jepsen.util 8623 :invoke :read [24 nil] INFO [2019-12-17 12:35:12,466] jepsen worker 43 - jepsen.util 12373 :invoke :write [14 3] INFO [2019-12-17 12:35:12,473] jepsen worker 30 - jepsen.util 9120 :ok :write [10 25] INFO [2019-12-17 12:35:12,473] jepsen worker 30 - jepsen.util 9120 :invoke :cas [10 [24 19]] INFO [2019-12-17 12:35:12,497] jepsen worker 11 - jepsen.util 9191 :info :write [3 25] :RequestCanceledException INFO [2019-12-17 12:35:12,500] jepsen worker 11 - jepsen.util 9281 :invoke :write [3 48] INFO [2019-12-17 12:35:12,514] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53165,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:35:12,514] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53165,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:35:12,515] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53168,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:35:12,515] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53168,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:35:12,516] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53169,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:35:12,516] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53169,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:35:12,524] jepsen worker 72 - jepsen.util 9162 :fail :cas [24 [46 37]] :ValueNotSwapFrom INFO [2019-12-17 12:35:12,525] jepsen worker 72 - jepsen.util 9162 :invoke :cas [24 [49 6]] INFO [2019-12-17 12:35:12,546] jepsen worker 64 - jepsen.util 7264 :ok :write [21 35] INFO [2019-12-17 12:35:12,547] jepsen worker 64 - jepsen.util 7264 :invoke :read [21 nil] INFO [2019-12-17 12:35:12,573] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:35:12,574] jepsen nemesis - couchbase.util Rebalance check count: 415 INFO [2019-12-17 12:35:12,574] jepsen worker 37 - jepsen.util 8587 :fail :cas [12 [24 6]] :ValueNotSwapFrom INFO [2019-12-17 12:35:12,574] jepsen nemesis - couchbase.util Rebalance stuck count: 396 INFO [2019-12-17 12:35:12,574] jepsen worker 37 - jepsen.util 8587 :invoke :cas [12 [28 40]] INFO [2019-12-17 12:35:12,576] jepsen worker 3 - jepsen.util 8913 :ok :write [1 39] INFO [2019-12-17 12:35:12,576] jepsen worker 3 - jepsen.util 8913 :invoke :cas [1 [27 14]] INFO [2019-12-17 12:35:12,597] jepsen worker 44 - jepsen.util 12914 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:35:12,597] jepsen worker 44 - jepsen.util 12914 :invoke :read [14 nil] INFO [2019-12-17 12:35:12,609] jepsen worker 24 - jepsen.util 11544 :info :cas [8 [32 43]] :RequestCanceledException INFO [2019-12-17 12:35:12,626] jepsen worker 10 - jepsen.util 9190 :ok :write [3 0] INFO [2019-12-17 12:35:12,626] jepsen worker 10 - jepsen.util 9190 :invoke :write [3 4] INFO [2019-12-17 12:35:12,630] jepsen worker 35 - jepsen.util 12545 :info :write [11 4] :RequestCanceledException INFO [2019-12-17 12:35:12,630] jepsen worker 24 - jepsen.util 11634 :invoke :read [8 nil] INFO [2019-12-17 12:35:12,638] jepsen worker 29 - jepsen.util 7589 :ok :write [9 38] INFO [2019-12-17 12:35:12,640] jepsen worker 29 - jepsen.util 7589 :invoke :cas [9 [48 33]] INFO [2019-12-17 12:35:12,680] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53188,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:35:12,680] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53188,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:35:12,680] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53192,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:35:12,681] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53192,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:35:12,681] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53193,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:35:12,684] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53193,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:35:12,688] jepsen worker 46 - jepsen.util 9136 :ok :write [15 48] INFO [2019-12-17 12:35:12,690] jepsen worker 46 - jepsen.util 9136 :invoke :write [15 40] INFO [2019-12-17 12:35:12,711] jepsen worker 35 - jepsen.util 12635 :invoke :cas [11 [24 32]] INFO [2019-12-17 12:35:12,741] jepsen worker 26 - jepsen.util 13166 :info :cas [8 [44 30]] :RequestCanceledException INFO [2019-12-17 12:35:12,754] jepsen worker 62 - jepsen.util 12842 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:35:12,758] jepsen worker 62 - jepsen.util 12842 :invoke :read [20 nil] INFO [2019-12-17 12:35:12,773] jepsen worker 23 - jepsen.util 12353 :info :cas [7 [18 40]] :RequestCanceledException INFO [2019-12-17 12:35:12,784] jepsen worker 26 - jepsen.util 13256 :invoke :write [8 4] INFO [2019-12-17 12:35:12,801] jepsen worker 49 - jepsen.util 11929 :info :write [16 25] :RequestCanceledException INFO [2019-12-17 12:35:12,820] jepsen worker 53 - jepsen.util 8333 :info :write [17 25] :RequestCanceledException INFO [2019-12-17 12:35:12,822] jepsen worker 83 - jepsen.util 13043 :info :write [27 8] :RequestCanceledException INFO [2019-12-17 12:35:12,833] jepsen worker 85 - jepsen.util 13405 :info :write [28 28] :RequestCanceledException INFO [2019-12-17 12:35:12,849] jepsen worker 45 - jepsen.util 9495 :ok :write [15 5] INFO [2019-12-17 12:35:12,851] jepsen worker 45 - jepsen.util 9495 :invoke :write [15 0] INFO [2019-12-17 12:35:12,860] jepsen worker 23 - jepsen.util 12443 :invoke :cas [7 [28 35]] INFO [2019-12-17 12:35:12,869] jepsen worker 53 - jepsen.util 8423 :invoke :write [17 16] INFO [2019-12-17 12:35:12,883] jepsen worker 49 - jepsen.util 12019 :invoke :write [16 4] INFO [2019-12-17 12:35:12,906] jepsen worker 83 - jepsen.util 13133 :invoke :cas [27 [27 8]] INFO [2019-12-17 12:35:12,906] jepsen worker 89 - jepsen.util 9179 :info :write [29 33] :RequestCanceledException INFO [2019-12-17 12:35:12,916] jepsen worker 47 - jepsen.util 9587 :info :write [15 4] :RequestCanceledException INFO [2019-12-17 12:35:12,947] jepsen worker 68 - jepsen.util 11948 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:35:12,960] jepsen worker 68 - jepsen.util 11948 :invoke :write [22 34] INFO [2019-12-17 12:35:12,960] jepsen worker 25 - jepsen.util 13525 :info :write [8 7] :RequestCanceledException INFO [2019-12-17 12:35:12,961] jepsen worker 85 - jepsen.util 13495 :invoke :cas [28 [21 45]] INFO [2019-12-17 12:35:12,970] jepsen worker 66 - jepsen.util 12306 :info :cas [22 [0 49]] :RequestCanceledException INFO [2019-12-17 12:35:12,972] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53201,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:35:12,973] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53201,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:35:13,022] jepsen worker 40 - jepsen.util 12910 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:35:13,033] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53205,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:35:13,038] jepsen worker 58 - jepsen.util 13288 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:35:13,039] jepsen worker 40 - jepsen.util 12910 :invoke :cas [13 [33 23]] INFO [2019-12-17 12:35:13,039] jepsen worker 58 - jepsen.util 13288 :invoke :write [19 30] INFO [2019-12-17 12:35:13,047] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53205,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:35:13,049] jepsen worker 47 - jepsen.util 9677 :invoke :write [15 17] INFO [2019-12-17 12:35:13,064] jepsen worker 32 - jepsen.util 9302 :ok :write [10 9] INFO [2019-12-17 12:35:13,066] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53210,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:35:13,067] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53210,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:35:13,071] jepsen worker 9 - jepsen.util 9729 :info :write [3 30] :RequestCanceledException INFO [2019-12-17 12:35:13,072] jepsen worker 32 - jepsen.util 9302 :invoke :write [10 33] INFO [2019-12-17 12:35:13,103] jepsen worker 89 - jepsen.util 9269 :invoke :read [29 nil] INFO [2019-12-17 12:35:13,104] jepsen worker 73 - jepsen.util 8623 :ok :read [24 45] INFO [2019-12-17 12:35:13,106] jepsen worker 73 - jepsen.util 8623 :invoke :write [24 37] INFO [2019-12-17 12:35:13,112] jepsen worker 66 - jepsen.util 12396 :invoke :read [22 nil] INFO [2019-12-17 12:35:13,133] jepsen worker 9 - jepsen.util 9819 :invoke :cas [3 [26 39]] INFO [2019-12-17 12:35:13,140] jepsen worker 25 - jepsen.util 13615 :invoke :write [8 11] INFO [2019-12-17 12:35:13,222] jepsen worker 82 - jepsen.util 13222 :info :cas [27 [26 25]] :RequestCanceledException INFO [2019-12-17 12:35:13,224] jepsen worker 0 - jepsen.util 13230 :info :write [0 12] :RequestCanceledException INFO [2019-12-17 12:35:13,238] jepsen worker 7 - jepsen.util 12607 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:35:13,244] jepsen worker 7 - jepsen.util 12607 :invoke :write [2 4] INFO [2019-12-17 12:35:13,251] jepsen worker 0 - jepsen.util 13320 :invoke :read [0 nil] INFO [2019-12-17 12:35:13,252] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53218,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:35:13,254] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53218,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:35:13,255] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53222,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:35:13,256] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53222,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:35:13,262] jepsen worker 1 - jepsen.util 12691 :info :write [0 2] :RequestCanceledException INFO [2019-12-17 12:35:13,268] jepsen worker 82 - jepsen.util 13312 :invoke :write [27 34] INFO [2019-12-17 12:35:13,269] jepsen worker 72 - jepsen.util 9162 :fail :cas [24 [49 6]] :ValueNotSwapFrom INFO [2019-12-17 12:35:13,270] jepsen worker 72 - jepsen.util 9162 :invoke :cas [24 [43 47]] INFO [2019-12-17 12:35:13,298] jepsen worker 1 - jepsen.util 12781 :invoke :cas [0 [37 30]] INFO [2019-12-17 12:35:13,306] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53224,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:35:13,307] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53224,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:35:13,309] jepsen worker 30 - jepsen.util 9120 :fail :cas [10 [24 19]] :ValueNotSwapFrom INFO [2019-12-17 12:35:13,309] jepsen worker 30 - jepsen.util 9120 :invoke :write [10 8] INFO [2019-12-17 12:35:13,321] jepsen worker 61 - jepsen.util 13471 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:35:13,321] jepsen worker 61 - jepsen.util 13471 :invoke :cas [20 [13 4]] INFO [2019-12-17 12:35:13,380] jepsen worker 52 - jepsen.util 9322 :ok :read [17 19] INFO [2019-12-17 12:35:13,381] jepsen worker 52 - jepsen.util 9322 :invoke :cas [17 [49 8]] INFO [2019-12-17 12:35:13,383] jepsen worker 79 - jepsen.util 9439 :ok :read [26 41] INFO [2019-12-17 12:35:13,383] jepsen worker 79 - jepsen.util 9439 :invoke :cas [26 [8 26]] INFO [2019-12-17 12:35:13,418] jepsen worker 13 - jepsen.util 8653 :ok :read [4 10] INFO [2019-12-17 12:35:13,418] jepsen worker 12 - jepsen.util 9642 :ok :read [4 10] INFO [2019-12-17 12:35:13,418] jepsen worker 13 - jepsen.util 8653 :invoke :write [4 13] INFO [2019-12-17 12:35:13,418] jepsen worker 12 - jepsen.util 9642 :invoke :cas [4 [11 19]] INFO [2019-12-17 12:35:13,419] jepsen worker 29 - jepsen.util 7589 :fail :cas [9 [48 33]] :ValueNotSwapFrom INFO [2019-12-17 12:35:13,419] jepsen worker 29 - jepsen.util 7589 :invoke :read [9 nil] INFO [2019-12-17 12:35:13,430] jepsen worker 75 - jepsen.util 11955 :info :write [25 24] :RequestCanceledException INFO [2019-12-17 12:35:13,430] jepsen worker 70 - jepsen.util 9790 :info :write [23 2] :RequestCanceledException INFO [2019-12-17 12:35:13,436] jepsen worker 69 - jepsen.util 9159 :fail :cas [23 [45 25]] :ValueNotSwapFrom INFO [2019-12-17 12:35:13,437] jepsen worker 75 - jepsen.util 12045 :invoke :cas [25 [47 7]] INFO [2019-12-17 12:35:13,437] jepsen worker 69 - jepsen.util 9159 :invoke :cas [23 [26 3]] INFO [2019-12-17 12:35:13,441] jepsen worker 70 - jepsen.util 9880 :invoke :read [23 nil] INFO [2019-12-17 12:35:13,448] jepsen worker 74 - jepsen.util 9074 :ok :read [24 45] INFO [2019-12-17 12:35:13,449] jepsen worker 74 - jepsen.util 9074 :invoke :read [24 nil] INFO [2019-12-17 12:35:13,470] jepsen worker 71 - jepsen.util 9791 :fail :cas [23 [45 6]] :ValueNotSwapFrom INFO [2019-12-17 12:35:13,471] jepsen worker 71 - jepsen.util 9791 :invoke :read [23 nil] INFO [2019-12-17 12:35:13,481] jepsen worker 5 - jepsen.util 8555 :ok :read [1 39] INFO [2019-12-17 12:35:13,482] jepsen worker 5 - jepsen.util 8555 :invoke :read [1 nil] INFO [2019-12-17 12:35:13,497] jepsen worker 9 - jepsen.util 9819 :fail :cas [3 [26 39]] :ValueNotSwapFrom INFO [2019-12-17 12:35:13,500] jepsen worker 9 - jepsen.util 9819 :invoke :read [3 nil] INFO [2019-12-17 12:35:13,509] jepsen worker 48 - jepsen.util 11568 :info :write [16 7] :RequestCanceledException INFO [2019-12-17 12:35:13,534] jepsen worker 56 - jepsen.util 9776 :fail :cas [18 [12 16]] :ValueNotSwapFrom INFO [2019-12-17 12:35:13,534] jepsen worker 56 - jepsen.util 9776 :invoke :cas [18 [18 49]] INFO [2019-12-17 12:35:13,540] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53250,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:35:13,540] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53250,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:35:13,540] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53254,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:35:13,540] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53254,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:35:13,580] jepsen worker 72 - jepsen.util 9162 :fail :cas [24 [43 47]] :ValueNotSwapFrom INFO [2019-12-17 12:35:13,580] jepsen worker 72 - jepsen.util 9162 :invoke :read [24 nil] INFO [2019-12-17 12:35:13,580] jepsen worker 48 - jepsen.util 11658 :invoke :cas [16 [4 47]] INFO [2019-12-17 12:35:13,592] jepsen worker 81 - jepsen.util 12321 :info :write [27 5] :RequestCanceledException INFO [2019-12-17 12:35:13,595] jepsen worker 81 - jepsen.util 12411 :invoke :read [27 nil] INFO [2019-12-17 12:35:13,603] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:35:13,603] jepsen nemesis - couchbase.util Rebalance check count: 416 INFO [2019-12-17 12:35:13,603] jepsen nemesis - couchbase.util Rebalance stuck count: 397 INFO [2019-12-17 12:35:13,606] jepsen worker 51 - jepsen.util 8331 :ok :read [17 19] INFO [2019-12-17 12:35:13,606] jepsen worker 51 - jepsen.util 8331 :invoke :cas [17 [28 24]] INFO [2019-12-17 12:35:13,614] jepsen worker 54 - jepsen.util 10044 :info :write [18 27] :RequestCanceledException INFO [2019-12-17 12:35:13,615] jepsen worker 39 - jepsen.util 12099 :info :cas [13 [46 21]] :RequestCanceledException INFO [2019-12-17 12:35:13,617] jepsen worker 54 - jepsen.util 10134 :invoke :write [18 4] INFO [2019-12-17 12:35:13,618] jepsen worker 39 - jepsen.util 12189 :invoke :cas [13 [42 35]] INFO [2019-12-17 12:35:13,629] jepsen worker 72 - jepsen.util 9162 :ok :read [24 45] INFO [2019-12-17 12:35:13,629] jepsen worker 72 - jepsen.util 9162 :invoke :write [24 2] INFO [2019-12-17 12:35:13,645] jepsen worker 33 - jepsen.util 12633 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:35:13,645] jepsen worker 33 - jepsen.util 12633 :invoke :cas [11 [13 10]] INFO [2019-12-17 12:35:13,671] jepsen worker 55 - jepsen.util 9865 :fail :cas [18 [9 31]] :ValueNotSwapFrom INFO [2019-12-17 12:35:13,672] jepsen worker 55 - jepsen.util 9865 :invoke :write [18 44] INFO [2019-12-17 12:35:14,200] jepsen worker 73 - jepsen.util 8623 :ok :write [24 37] INFO [2019-12-17 12:35:14,200] jepsen worker 89 - jepsen.util 9269 :ok :read [29 31] INFO [2019-12-17 12:35:14,200] jepsen worker 73 - jepsen.util 8623 :invoke :cas [24 [33 31]] INFO [2019-12-17 12:35:14,200] jepsen worker 89 - jepsen.util 9269 :invoke :cas [29 [29 9]] INFO [2019-12-17 12:35:14,200] jepsen worker 80 - jepsen.util 8900 :fail :cas [26 [37 4]] :ValueNotSwapFrom INFO [2019-12-17 12:35:14,200] jepsen worker 2 - jepsen.util 12242 :info :cas [0 [45 3]] :RequestCanceledException INFO [2019-12-17 12:35:14,200] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53283,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:35:14,200] jepsen worker 78 - jepsen.util 9078 :fail :cas [26 [12 2]] :ValueNotSwapFrom INFO [2019-12-17 12:35:14,200] jepsen worker 8 - jepsen.util 12248 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:35:14,200] jepsen worker 34 - jepsen.util 12724 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:35:14,201] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53283,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:35:14,201] jepsen worker 78 - jepsen.util 9078 :invoke :write [26 35] INFO [2019-12-17 12:35:14,201] jepsen worker 38 - jepsen.util 9668 :info :write [12 24] :RequestCanceledException INFO [2019-12-17 12:35:14,201] jepsen worker 16 - jepsen.util 13696 :info :write [5 3] :RequestCanceledException INFO [2019-12-17 12:35:14,201] jepsen worker 47 - jepsen.util 9677 :ok :write [15 17] INFO [2019-12-17 12:35:14,201] jepsen worker 80 - jepsen.util 8900 :invoke :cas [26 [0 22]] INFO [2019-12-17 12:35:14,201] jepsen worker 31 - jepsen.util 8131 :info :cas [10 [43 45]] :RequestCanceledException INFO [2019-12-17 12:35:14,201] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53272,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:35:14,201] jepsen worker 59 - jepsen.util 11939 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:35:14,201] jepsen worker 84 - jepsen.util 12054 :info :cas [28 [46 2]] :RequestCanceledException INFO [2019-12-17 12:35:14,201] jepsen worker 86 - jepsen.util 13406 :info :write [28 31] :RequestCanceledException INFO [2019-12-17 12:35:14,201] jepsen worker 65 - jepsen.util 7625 :info :write [21 3] :RequestCanceledException INFO [2019-12-17 12:35:14,201] jepsen worker 21 - jepsen.util 12351 :info :cas [7 [1 25]] :RequestCanceledException INFO [2019-12-17 12:35:14,201] jepsen worker 34 - jepsen.util 12724 :invoke :read [11 nil] INFO [2019-12-17 12:35:14,201] jepsen worker 8 - jepsen.util 12248 :invoke :read [2 nil] INFO [2019-12-17 12:35:14,201] jepsen worker 14 - jepsen.util 9014 :fail :read [4 nil] :RequestTimeoutException INFO [2019-12-17 12:35:14,202] jepsen worker 77 - jepsen.util 13037 :info :cas [25 [8 33]] :RequestCanceledException INFO [2019-12-17 12:35:14,202] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53288,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:35:14,202] jepsen worker 22 - jepsen.util 12442 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:35:14,202] jepsen worker 50 - jepsen.util 13190 :info :cas [16 [39 2]] :RequestCanceledException INFO [2019-12-17 12:35:14,202] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53291,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:35:14,202] jepsen worker 4 - jepsen.util 8644 :info :write [1 31] :RequestCanceledException INFO [2019-12-17 12:35:14,202] jepsen worker 67 - jepsen.util 12667 :info :cas [22 [42 29]] :RequestCanceledException INFO [2019-12-17 12:35:14,203] jepsen worker 41 - jepsen.util 13361 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:35:14,203] jepsen worker 36 - jepsen.util 9576 :info :write [12 36] :RequestCanceledException INFO [2019-12-17 12:35:14,203] jepsen worker 19 - jepsen.util 9199 :info :write [6 42] :RequestCanceledException INFO [2019-12-17 12:35:14,203] jepsen worker 63 - jepsen.util 7983 :info :cas [21 [17 10]] :RequestCanceledException INFO [2019-12-17 12:35:14,203] jepsen worker 10 - jepsen.util 9190 :ok :write [3 4] INFO [2019-12-17 12:35:14,204] jepsen worker 15 - jepsen.util 11715 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:35:14,204] jepsen worker 47 - jepsen.util 9677 :invoke :cas [15 [36 36]] INFO [2019-12-17 12:35:14,204] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53272,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:35:14,204] jepsen worker 59 - jepsen.util 11939 :invoke :read [19 nil] INFO [2019-12-17 12:35:14,205] jepsen worker 14 - jepsen.util 9014 :invoke :write [4 42] INFO [2019-12-17 12:35:14,206] jepsen worker 22 - jepsen.util 12442 :invoke :write [7 33] INFO [2019-12-17 12:35:14,206] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53288,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:35:14,207] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53291,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:35:14,207] jepsen worker 88 - jepsen.util 9448 :info :cas [29 [19 0]] :RequestCanceledException INFO [2019-12-17 12:35:14,208] jepsen worker 10 - jepsen.util 9190 :invoke :read [3 nil] INFO [2019-12-17 12:35:14,208] jepsen worker 41 - jepsen.util 13361 :invoke :write [13 46] INFO [2019-12-17 12:35:14,208] jepsen worker 15 - jepsen.util 11715 :invoke :cas [5 [18 39]] INFO [2019-12-17 12:35:14,209] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53275,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:35:14,209] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53294,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:35:14,212] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53275,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:35:14,213] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53294,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:35:14,213] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53295,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:35:14,214] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53295,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:35:14,215] jepsen worker 2 - jepsen.util 12332 :invoke :write [0 25] INFO [2019-12-17 12:35:14,219] jepsen worker 38 - jepsen.util 9758 :invoke :cas [12 [25 13]] INFO [2019-12-17 12:35:14,220] jepsen worker 16 - jepsen.util 13786 :invoke :cas [5 [27 22]] INFO [2019-12-17 12:35:14,220] jepsen worker 86 - jepsen.util 13496 :invoke :write [28 23] INFO [2019-12-17 12:35:14,221] jepsen worker 84 - jepsen.util 12144 :invoke :write [28 23] INFO [2019-12-17 12:35:14,221] jepsen worker 31 - jepsen.util 8221 :invoke :cas [10 [19 46]] INFO [2019-12-17 12:35:14,222] jepsen worker 21 - jepsen.util 12441 :invoke :cas [7 [37 9]] INFO [2019-12-17 12:35:14,222] jepsen worker 65 - jepsen.util 7715 :invoke :read [21 nil] INFO [2019-12-17 12:35:14,223] jepsen worker 77 - jepsen.util 13127 :invoke :write [25 44] INFO [2019-12-17 12:35:14,224] jepsen worker 4 - jepsen.util 8734 :invoke :read [1 nil] INFO [2019-12-17 12:35:14,224] jepsen worker 19 - jepsen.util 9289 :invoke :write [6 3] INFO [2019-12-17 12:35:14,224] jepsen worker 50 - jepsen.util 13280 :invoke :read [16 nil] INFO [2019-12-17 12:35:14,224] jepsen worker 67 - jepsen.util 12757 :invoke :cas [22 [49 46]] INFO [2019-12-17 12:35:14,224] jepsen worker 63 - jepsen.util 8073 :invoke :cas [21 [20 39]] INFO [2019-12-17 12:35:14,225] jepsen worker 36 - jepsen.util 9666 :invoke :cas [12 [13 31]] INFO [2019-12-17 12:35:14,226] jepsen worker 88 - jepsen.util 9538 :invoke :cas [29 [29 36]] INFO [2019-12-17 12:35:14,295] jepsen worker 37 - jepsen.util 8587 :fail :cas [12 [28 40]] :ValueNotSwapFrom INFO [2019-12-17 12:35:14,309] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53282,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:35:14,309] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53309,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:35:14,310] jepsen worker 69 - jepsen.util 9159 :fail :cas [23 [26 3]] :ValueNotSwapFrom INFO [2019-12-17 12:35:14,312] jepsen worker 52 - jepsen.util 9322 :fail :cas [17 [49 8]] :ValueNotSwapFrom INFO [2019-12-17 12:35:14,314] jepsen worker 70 - jepsen.util 9880 :ok :read [23 4] INFO [2019-12-17 12:35:14,314] jepsen worker 18 - jepsen.util 9468 :info :cas [6 [22 20]] :RequestCanceledException INFO [2019-12-17 12:35:14,331] jepsen worker 17 - jepsen.util 12437 :info :write [5 22] :RequestCanceledException INFO [2019-12-17 12:35:14,336] jepsen worker 47 - jepsen.util 9677 :fail :cas [15 [36 36]] :ValueNotSwapFrom INFO [2019-12-17 12:35:14,338] jepsen worker 73 - jepsen.util 8623 :fail :cas [24 [33 31]] :ValueNotSwapFrom INFO [2019-12-17 12:35:14,346] jepsen worker 12 - jepsen.util 9642 :fail :cas [4 [11 19]] :ValueNotSwapFrom INFO [2019-12-17 12:35:14,376] jepsen worker 71 - jepsen.util 9791 :ok :read [23 4] INFO [2019-12-17 12:35:14,383] jepsen worker 5 - jepsen.util 8555 :ok :read [1 39] INFO [2019-12-17 12:35:14,391] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53282,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:35:14,391] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53309,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:35:14,391] jepsen worker 69 - jepsen.util 9159 :invoke :write [23 45] INFO [2019-12-17 12:35:14,391] jepsen worker 37 - jepsen.util 8587 :invoke :write [12 45] INFO [2019-12-17 12:35:14,391] jepsen worker 52 - jepsen.util 9322 :invoke :read [17 nil] INFO [2019-12-17 12:35:14,391] jepsen worker 47 - jepsen.util 9677 :invoke :write [15 15] INFO [2019-12-17 12:35:14,391] jepsen worker 70 - jepsen.util 9880 :invoke :cas [23 [28 24]] INFO [2019-12-17 12:35:14,391] jepsen worker 12 - jepsen.util 9642 :invoke :read [4 nil] INFO [2019-12-17 12:35:14,391] jepsen worker 73 - jepsen.util 8623 :invoke :write [24 21] INFO [2019-12-17 12:35:14,392] jepsen worker 5 - jepsen.util 8555 :invoke :cas [1 [6 39]] INFO [2019-12-17 12:35:14,392] jepsen worker 71 - jepsen.util 9791 :invoke :write [23 16] INFO [2019-12-17 12:35:14,392] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53311,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:35:14,392] jepsen worker 28 - jepsen.util 7408 :info :write [9 37] :RequestCanceledException INFO [2019-12-17 12:35:14,393] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53311,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:35:14,394] jepsen worker 17 - jepsen.util 12527 :invoke :read [5 nil] INFO [2019-12-17 12:35:14,394] jepsen worker 18 - jepsen.util 9558 :invoke :read [6 nil] INFO [2019-12-17 12:35:14,396] jepsen worker 28 - jepsen.util 7498 :invoke :cas [9 [10 38]] INFO [2019-12-17 12:35:14,430] jepsen worker 20 - jepsen.util 8660 :info :cas [6 [24 45]] :RequestCanceledException INFO [2019-12-17 12:35:14,433] jepsen worker 20 - jepsen.util 8750 :invoke :read [6 nil] INFO [2019-12-17 12:35:14,450] jepsen worker 28 - jepsen.util 7498 :fail :cas [9 [10 38]] :ValueNotSwapFrom INFO [2019-12-17 12:35:14,451] jepsen worker 28 - jepsen.util 7498 :invoke :read [9 nil] INFO [2019-12-17 12:35:14,462] jepsen worker 87 - jepsen.util 8817 :info :write [29 36] :RequestCanceledException INFO [2019-12-17 12:35:14,464] jepsen worker 87 - jepsen.util 8907 :invoke :read [29 nil] INFO [2019-12-17 12:35:14,497] jepsen worker 27 - jepsen.util 8847 :fail :read [9 nil] :RequestTimeoutException INFO [2019-12-17 12:35:14,497] jepsen worker 27 - jepsen.util 8847 :invoke :cas [9 [14 24]] INFO [2019-12-17 12:35:14,506] jepsen worker 28 - jepsen.util 7498 :ok :read [9 38] INFO [2019-12-17 12:35:14,506] jepsen worker 28 - jepsen.util 7498 :invoke :cas [9 [20 44]] INFO [2019-12-17 12:35:14,508] jepsen worker 60 - jepsen.util 13560 :info :cas [20 [5 45]] :RequestCanceledException INFO [2019-12-17 12:35:14,510] jepsen worker 80 - jepsen.util 8900 :fail :cas [26 [0 22]] :ValueNotSwapFrom INFO [2019-12-17 12:35:14,510] jepsen worker 80 - jepsen.util 8900 :invoke :read [26 nil] INFO [2019-12-17 12:35:14,511] jepsen worker 60 - jepsen.util 13650 :invoke :cas [20 [20 4]] INFO [2019-12-17 12:35:14,520] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53346,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:35:14,520] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53346,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:35:14,520] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53349,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:35:14,521] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53349,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:35:14,521] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53350,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:35:14,521] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53350,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:35:14,528] jepsen worker 32 - jepsen.util 9302 :ok :write [10 33] INFO [2019-12-17 12:35:14,528] jepsen worker 32 - jepsen.util 9302 :invoke :cas [10 [43 19]] INFO [2019-12-17 12:35:14,528] jepsen worker 42 - jepsen.util 13452 :info :write [14 2] :RequestCanceledException INFO [2019-12-17 12:35:14,528] jepsen worker 11 - jepsen.util 9281 :info :write [3 48] :RequestCanceledException INFO [2019-12-17 12:35:14,531] jepsen worker 42 - jepsen.util 13542 :invoke :write [14 28] INFO [2019-12-17 12:35:14,531] jepsen worker 11 - jepsen.util 9371 :invoke :cas [3 [8 26]] INFO [2019-12-17 12:35:14,543] jepsen worker 6 - jepsen.util 13056 :info :cas [2 [6 30]] :RequestCanceledException INFO [2019-12-17 12:35:14,543] jepsen worker 76 - jepsen.util 12856 :info :write [25 38] :RequestCanceledException INFO [2019-12-17 12:35:14,547] jepsen worker 6 - jepsen.util 13146 :invoke :write [2 44] INFO [2019-12-17 12:35:14,547] jepsen worker 76 - jepsen.util 12946 :invoke :read [25 nil] INFO [2019-12-17 12:35:14,561] jepsen worker 57 - jepsen.util 12297 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:35:14,561] jepsen worker 43 - jepsen.util 12373 :info :write [14 3] :RequestCanceledException INFO [2019-12-17 12:35:14,562] jepsen worker 57 - jepsen.util 12297 :invoke :write [19 1] INFO [2019-12-17 12:35:14,568] jepsen worker 43 - jepsen.util 12463 :invoke :read [14 nil] INFO [2019-12-17 12:35:14,583] jepsen worker 27 - jepsen.util 8847 :fail :cas [9 [14 24]] :ValueNotSwapFrom INFO [2019-12-17 12:35:14,583] jepsen worker 27 - jepsen.util 8847 :invoke :read [9 nil] INFO [2019-12-17 12:35:14,611] jepsen worker 10 - jepsen.util 9190 :ok :read [3 4] INFO [2019-12-17 12:35:14,611] jepsen worker 10 - jepsen.util 9190 :invoke :read [3 nil] INFO [2019-12-17 12:35:14,619] jepsen worker 28 - jepsen.util 7498 :fail :cas [9 [20 44]] :ValueNotSwapFrom INFO [2019-12-17 12:35:14,619] jepsen worker 52 - jepsen.util 9322 :ok :read [17 19] INFO [2019-12-17 12:35:14,619] jepsen worker 28 - jepsen.util 7498 :invoke :read [9 nil] INFO [2019-12-17 12:35:14,619] jepsen worker 52 - jepsen.util 9322 :invoke :read [17 nil] INFO [2019-12-17 12:35:14,630] jepsen worker 3 - jepsen.util 8913 :info :cas [1 [27 14]] :RequestCanceledException INFO [2019-12-17 12:35:14,630] jepsen worker 65 - jepsen.util 7715 :ok :read [21 35] INFO [2019-12-17 12:35:14,631] jepsen worker 65 - jepsen.util 7715 :invoke :read [21 nil] INFO [2019-12-17 12:35:14,631] jepsen worker 64 - jepsen.util 7264 :fail :read [21 nil] :RequestTimeoutException INFO [2019-12-17 12:35:14,631] jepsen worker 64 - jepsen.util 7264 :invoke :write [21 9] INFO [2019-12-17 12:35:14,635] jepsen worker 3 - jepsen.util 9003 :invoke :cas [1 [21 19]] INFO [2019-12-17 12:35:14,647] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:35:14,648] jepsen nemesis - couchbase.util Rebalance check count: 417 INFO [2019-12-17 12:35:14,648] jepsen nemesis - couchbase.util Rebalance stuck count: 398 INFO [2019-12-17 12:35:14,649] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53372,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:35:14,649] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53372,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:35:14,649] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53376,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:35:14,649] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53376,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:35:14,651] jepsen worker 63 - jepsen.util 8073 :fail :cas [21 [20 39]] :ValueNotSwapFrom INFO [2019-12-17 12:35:14,651] jepsen worker 63 - jepsen.util 8073 :invoke :read [21 nil] INFO [2019-12-17 12:35:14,670] jepsen worker 20 - jepsen.util 8750 :ok :read [6 3] INFO [2019-12-17 12:35:14,670] jepsen worker 20 - jepsen.util 8750 :invoke :read [6 nil] INFO [2019-12-17 12:35:14,682] jepsen worker 36 - jepsen.util 9666 :fail :cas [12 [13 31]] :ValueNotSwapFrom INFO [2019-12-17 12:35:14,682] jepsen worker 36 - jepsen.util 9666 :invoke :write [12 4] INFO [2019-12-17 12:35:14,690] jepsen worker 51 - jepsen.util 8331 :fail :cas [17 [28 24]] :ValueNotSwapFrom INFO [2019-12-17 12:35:14,691] jepsen worker 51 - jepsen.util 8331 :invoke :write [17 29] INFO [2019-12-17 12:35:14,698] jepsen worker 44 - jepsen.util 12914 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:35:14,698] jepsen worker 24 - jepsen.util 11634 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:35:14,698] jepsen worker 44 - jepsen.util 12914 :invoke :write [14 41] INFO [2019-12-17 12:35:14,698] jepsen worker 24 - jepsen.util 11634 :invoke :read [8 nil] INFO [2019-12-17 12:35:14,729] jepsen worker 32 - jepsen.util 9302 :fail :cas [10 [43 19]] :ValueNotSwapFrom INFO [2019-12-17 12:35:14,729] jepsen worker 32 - jepsen.util 9302 :invoke :read [10 nil] INFO [2019-12-17 12:35:14,731] jepsen worker 27 - jepsen.util 8847 :ok :read [9 38] INFO [2019-12-17 12:35:14,731] jepsen worker 27 - jepsen.util 8847 :invoke :read [9 nil] INFO [2019-12-17 12:35:14,736] jepsen worker 10 - jepsen.util 9190 :ok :read [3 4] INFO [2019-12-17 12:35:14,736] jepsen worker 10 - jepsen.util 9190 :invoke :cas [3 [31 30]] INFO [2019-12-17 12:35:14,739] jepsen worker 28 - jepsen.util 7498 :ok :read [9 38] INFO [2019-12-17 12:35:14,739] jepsen worker 28 - jepsen.util 7498 :invoke :cas [9 [45 21]] INFO [2019-12-17 12:35:14,742] jepsen worker 65 - jepsen.util 7715 :ok :read [21 35] INFO [2019-12-17 12:35:14,742] jepsen worker 65 - jepsen.util 7715 :invoke :cas [21 [37 33]] INFO [2019-12-17 12:35:14,771] jepsen worker 63 - jepsen.util 8073 :ok :read [21 35] INFO [2019-12-17 12:35:14,781] jepsen worker 63 - jepsen.util 8073 :invoke :read [21 nil] INFO [2019-12-17 12:35:14,808] jepsen worker 27 - jepsen.util 8847 :ok :read [9 38] INFO [2019-12-17 12:35:14,809] jepsen worker 27 - jepsen.util 8847 :invoke :cas [9 [48 2]] INFO [2019-12-17 12:35:14,809] jepsen worker 35 - jepsen.util 12635 :info :cas [11 [24 32]] :RequestCanceledException INFO [2019-12-17 12:35:14,811] jepsen worker 20 - jepsen.util 8750 :ok :read [6 3] INFO [2019-12-17 12:35:14,812] jepsen worker 20 - jepsen.util 8750 :invoke :cas [6 [29 43]] INFO [2019-12-17 12:35:14,813] jepsen worker 35 - jepsen.util 12725 :invoke :read [11 nil] INFO [2019-12-17 12:35:14,816] jepsen worker 19 - jepsen.util 9289 :ok :write [6 3] INFO [2019-12-17 12:35:14,817] jepsen worker 19 - jepsen.util 9289 :invoke :read [6 nil] INFO [2019-12-17 12:35:14,827] jepsen worker 28 - jepsen.util 7498 :fail :cas [9 [45 21]] :ValueNotSwapFrom INFO [2019-12-17 12:35:14,827] jepsen worker 28 - jepsen.util 7498 :invoke :read [9 nil] INFO [2019-12-17 12:35:14,839] jepsen worker 62 - jepsen.util 12842 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:35:14,840] jepsen worker 62 - jepsen.util 12842 :invoke :cas [20 [39 27]] INFO [2019-12-17 12:35:14,849] jepsen worker 32 - jepsen.util 9302 :ok :read [10 33] INFO [2019-12-17 12:35:14,850] jepsen worker 32 - jepsen.util 9302 :invoke :write [10 33] INFO [2019-12-17 12:35:14,854] jepsen worker 10 - jepsen.util 9190 :fail :cas [3 [31 30]] :ValueNotSwapFrom INFO [2019-12-17 12:35:14,854] jepsen worker 10 - jepsen.util 9190 :invoke :cas [3 [14 32]] INFO [2019-12-17 12:35:14,867] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53401,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:35:14,867] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53401,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:35:14,867] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53402,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:35:14,867] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53402,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:35:14,870] jepsen worker 65 - jepsen.util 7715 :fail :cas [21 [37 33]] :ValueNotSwapFrom INFO [2019-12-17 12:35:14,870] jepsen worker 65 - jepsen.util 7715 :invoke :cas [21 [9 13]] INFO [2019-12-17 12:35:14,898] jepsen worker 72 - jepsen.util 9162 :ok :write [24 2] INFO [2019-12-17 12:35:14,899] jepsen worker 72 - jepsen.util 9162 :invoke :cas [24 [31 49]] INFO [2019-12-17 12:35:14,944] jepsen worker 23 - jepsen.util 12443 :info :cas [7 [28 35]] :RequestCanceledException INFO [2019-12-17 12:35:14,946] jepsen worker 28 - jepsen.util 7498 :ok :read [9 38] INFO [2019-12-17 12:35:14,947] jepsen worker 27 - jepsen.util 8847 :fail :cas [9 [48 2]] :ValueNotSwapFrom INFO [2019-12-17 12:35:14,948] jepsen worker 27 - jepsen.util 8847 :invoke :cas [9 [22 35]] INFO [2019-12-17 12:35:14,948] jepsen worker 28 - jepsen.util 7498 :invoke :cas [9 [43 43]] INFO [2019-12-17 12:35:14,950] jepsen worker 23 - jepsen.util 12533 :invoke :write [7 11] INFO [2019-12-17 12:35:14,976] jepsen worker 12 - jepsen.util 9642 :ok :read [4 10] INFO [2019-12-17 12:35:14,976] jepsen worker 12 - jepsen.util 9642 :invoke :write [4 13] INFO [2019-12-17 12:35:14,988] jepsen worker 63 - jepsen.util 8073 :ok :read [21 35] INFO [2019-12-17 12:35:14,989] jepsen worker 63 - jepsen.util 8073 :invoke :cas [21 [38 8]] INFO [2019-12-17 12:35:14,992] jepsen worker 5 - jepsen.util 8555 :fail :cas [1 [6 39]] :ValueNotSwapFrom INFO [2019-12-17 12:35:14,993] jepsen worker 5 - jepsen.util 8555 :invoke :cas [1 [18 6]] INFO [2019-12-17 12:35:14,997] jepsen worker 83 - jepsen.util 13133 :info :cas [27 [27 8]] :RequestCanceledException INFO [2019-12-17 12:35:14,998] jepsen worker 20 - jepsen.util 8750 :fail :cas [6 [29 43]] :ValueNotSwapFrom INFO [2019-12-17 12:35:14,998] jepsen worker 20 - jepsen.util 8750 :invoke :write [6 45] INFO [2019-12-17 12:35:15,002] jepsen worker 83 - jepsen.util 13223 :invoke :read [27 nil] INFO [2019-12-17 12:35:15,012] jepsen worker 85 - jepsen.util 13495 :info :cas [28 [21 45]] :RequestCanceledException INFO [2019-12-17 12:35:15,018] jepsen worker 85 - jepsen.util 13585 :invoke :read [28 nil] INFO [2019-12-17 12:35:15,019] jepsen worker 89 - jepsen.util 9269 :fail :cas [29 [29 9]] :ValueNotSwapFrom INFO [2019-12-17 12:35:15,019] jepsen worker 89 - jepsen.util 9269 :invoke :read [29 nil] INFO [2019-12-17 12:35:15,044] jepsen worker 56 - jepsen.util 9776 :fail :cas [18 [18 49]] :ValueNotSwapFrom INFO [2019-12-17 12:35:15,044] jepsen worker 56 - jepsen.util 9776 :invoke :write [18 33] INFO [2019-12-17 12:35:15,045] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53412,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:35:15,045] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53412,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:35:15,046] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53416,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:35:15,046] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53416,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:35:15,047] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53417,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:35:15,047] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53417,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:35:15,054] jepsen worker 10 - jepsen.util 9190 :fail :cas [3 [14 32]] :ValueNotSwapFrom INFO [2019-12-17 12:35:15,054] jepsen worker 10 - jepsen.util 9190 :invoke :cas [3 [45 21]] INFO [2019-12-17 12:35:15,058] jepsen worker 47 - jepsen.util 9677 :ok :write [15 15] INFO [2019-12-17 12:35:15,058] jepsen worker 47 - jepsen.util 9677 :invoke :cas [15 [30 22]] INFO [2019-12-17 12:35:15,071] jepsen worker 28 - jepsen.util 7498 :fail :cas [9 [43 43]] :ValueNotSwapFrom INFO [2019-12-17 12:35:15,072] jepsen worker 28 - jepsen.util 7498 :invoke :cas [9 [25 16]] INFO [2019-12-17 12:35:15,096] jepsen worker 40 - jepsen.util 12910 :info :cas [13 [33 23]] :RequestCanceledException INFO [2019-12-17 12:35:15,098] jepsen worker 65 - jepsen.util 7715 :fail :cas [21 [9 13]] :ValueNotSwapFrom INFO [2019-12-17 12:35:15,098] jepsen worker 65 - jepsen.util 7715 :invoke :write [21 41] INFO [2019-12-17 12:35:15,102] jepsen worker 40 - jepsen.util 13000 :invoke :read [13 nil] INFO [2019-12-17 12:35:15,120] jepsen worker 27 - jepsen.util 8847 :fail :cas [9 [22 35]] :ValueNotSwapFrom INFO [2019-12-17 12:35:15,121] jepsen worker 27 - jepsen.util 8847 :invoke :read [9 nil] INFO [2019-12-17 12:35:15,125] jepsen worker 38 - jepsen.util 9758 :fail :cas [12 [25 13]] :ValueNotSwapFrom INFO [2019-12-17 12:35:15,125] jepsen worker 18 - jepsen.util 9558 :ok :read [6 3] INFO [2019-12-17 12:35:15,126] jepsen worker 38 - jepsen.util 9758 :invoke :cas [12 [46 4]] INFO [2019-12-17 12:35:15,126] jepsen worker 18 - jepsen.util 9558 :invoke :write [6 41] INFO [2019-12-17 12:35:15,137] jepsen worker 11 - jepsen.util 9371 :fail :cas [3 [8 26]] :ValueNotSwapFrom INFO [2019-12-17 12:35:15,137] jepsen worker 11 - jepsen.util 9371 :invoke :write [3 3] INFO [2019-12-17 12:35:15,160] jepsen worker 64 - jepsen.util 7264 :ok :write [21 9] INFO [2019-12-17 12:35:15,160] jepsen worker 64 - jepsen.util 7264 :invoke :write [21 34] INFO [2019-12-17 12:35:15,161] jepsen worker 66 - jepsen.util 12396 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:35:15,162] jepsen worker 66 - jepsen.util 12396 :invoke :cas [22 [30 49]] INFO [2019-12-17 12:35:15,192] jepsen worker 27 - jepsen.util 8847 :ok :read [9 38] INFO [2019-12-17 12:35:15,192] jepsen worker 28 - jepsen.util 7498 :fail :cas [9 [25 16]] :ValueNotSwapFrom INFO [2019-12-17 12:35:15,192] jepsen worker 27 - jepsen.util 8847 :invoke :read [9 nil] INFO [2019-12-17 12:35:15,192] jepsen worker 28 - jepsen.util 7498 :invoke :write [9 38] INFO [2019-12-17 12:35:15,197] jepsen worker 46 - jepsen.util 9136 :info :write [15 40] :RequestCanceledException INFO [2019-12-17 12:35:15,201] jepsen worker 46 - jepsen.util 9226 :invoke :write [15 30] INFO [2019-12-17 12:35:15,225] jepsen worker 70 - jepsen.util 9880 :fail :cas [23 [28 24]] :ValueNotSwapFrom INFO [2019-12-17 12:35:15,225] jepsen worker 70 - jepsen.util 9880 :invoke :read [23 nil] INFO [2019-12-17 12:35:15,232] jepsen worker 47 - jepsen.util 9677 :fail :cas [15 [30 22]] :ValueNotSwapFrom INFO [2019-12-17 12:35:15,232] jepsen worker 47 - jepsen.util 9677 :invoke :read [15 nil] INFO [2019-12-17 12:35:15,256] jepsen worker 72 - jepsen.util 9162 :fail :cas [24 [31 49]] :ValueNotSwapFrom INFO [2019-12-17 12:35:15,256] jepsen worker 72 - jepsen.util 9162 :invoke :read [24 nil] INFO [2019-12-17 12:35:15,262] jepsen worker 53 - jepsen.util 8423 :info :write [17 16] :RequestCanceledException INFO [2019-12-17 12:35:15,263] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53445,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:35:15,264] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53445,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:35:15,264] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53447,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:35:15,264] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53447,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:35:15,266] jepsen worker 53 - jepsen.util 8513 :invoke :read [17 nil] INFO [2019-12-17 12:35:15,266] jepsen worker 64 - jepsen.util 7264 :ok :write [21 34] INFO [2019-12-17 12:35:15,272] jepsen worker 64 - jepsen.util 7264 :invoke :cas [21 [1 0]] INFO [2019-12-17 12:35:15,297] jepsen worker 45 - jepsen.util 9495 :info :write [15 0] :RequestCanceledException INFO [2019-12-17 12:35:15,306] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53435,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:35:15,307] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53435,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:35:15,307] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53436,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:35:15,307] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53436,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:35:15,315] jepsen worker 0 - jepsen.util 13320 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:35:15,315] jepsen worker 0 - jepsen.util 13320 :invoke :cas [0 [27 14]] INFO [2019-12-17 12:35:15,326] jepsen worker 47 - jepsen.util 9677 :ok :read [15 15] INFO [2019-12-17 12:35:15,326] jepsen worker 47 - jepsen.util 9677 :invoke :read [15 nil] INFO [2019-12-17 12:35:15,330] jepsen worker 49 - jepsen.util 12019 :info :write [16 4] :RequestCanceledException INFO [2019-12-17 12:35:15,330] jepsen worker 26 - jepsen.util 13256 :info :write [8 4] :RequestCanceledException INFO [2019-12-17 12:35:15,331] jepsen worker 80 - jepsen.util 8900 :ok :read [26 41] INFO [2019-12-17 12:35:15,331] jepsen worker 45 - jepsen.util 9585 :invoke :read [15 nil] INFO [2019-12-17 12:35:15,331] jepsen worker 80 - jepsen.util 8900 :invoke :read [26 nil] INFO [2019-12-17 12:35:15,335] jepsen worker 26 - jepsen.util 13346 :invoke :read [8 nil] INFO [2019-12-17 12:35:15,336] jepsen worker 49 - jepsen.util 12109 :invoke :write [16 31] INFO [2019-12-17 12:35:15,344] jepsen worker 1 - jepsen.util 12781 :info :cas [0 [37 30]] :RequestCanceledException INFO [2019-12-17 12:35:15,351] jepsen worker 1 - jepsen.util 12871 :invoke :cas [0 [24 45]] INFO [2019-12-17 12:35:15,370] jepsen worker 87 - jepsen.util 8907 :ok :read [29 36] INFO [2019-12-17 12:35:15,372] jepsen worker 87 - jepsen.util 8907 :invoke :cas [29 [13 42]] INFO [2019-12-17 12:35:15,376] jepsen worker 28 - jepsen.util 7498 :ok :write [9 38] INFO [2019-12-17 12:35:15,376] jepsen worker 28 - jepsen.util 7498 :invoke :cas [9 [46 31]] INFO [2019-12-17 12:35:15,382] jepsen worker 64 - jepsen.util 7264 :fail :cas [21 [1 0]] :ValueNotSwapFrom INFO [2019-12-17 12:35:15,382] jepsen worker 64 - jepsen.util 7264 :invoke :read [21 nil] INFO [2019-12-17 12:35:15,383] jepsen worker 27 - jepsen.util 8847 :ok :read [9 38] INFO [2019-12-17 12:35:15,383] jepsen worker 27 - jepsen.util 8847 :invoke :read [9 nil] INFO [2019-12-17 12:35:15,397] jepsen worker 25 - jepsen.util 13615 :info :write [8 11] :RequestCanceledException INFO [2019-12-17 12:35:15,399] jepsen worker 25 - jepsen.util 13705 :invoke :write [8 36] INFO [2019-12-17 12:35:15,413] jepsen worker 79 - jepsen.util 9439 :info :cas [26 [8 26]] :RequestCanceledException INFO [2019-12-17 12:35:15,415] jepsen worker 79 - jepsen.util 9529 :invoke :write [26 17] INFO [2019-12-17 12:35:15,426] jepsen worker 47 - jepsen.util 9677 :ok :read [15 15] INFO [2019-12-17 12:35:15,426] jepsen worker 47 - jepsen.util 9677 :invoke :cas [15 [32 47]] INFO [2019-12-17 12:35:15,430] jepsen worker 58 - jepsen.util 13288 :info :write [19 30] :RequestCanceledException INFO [2019-12-17 12:35:15,430] jepsen worker 61 - jepsen.util 13471 :info :cas [20 [13 4]] :RequestCanceledException INFO [2019-12-17 12:35:15,430] jepsen worker 68 - jepsen.util 11948 :info :write [22 34] :RequestCanceledException INFO [2019-12-17 12:35:15,430] jepsen worker 82 - jepsen.util 13312 :info :write [27 34] :RequestCanceledException INFO [2019-12-17 12:35:15,430] jepsen worker 29 - jepsen.util 7589 :fail :read [9 nil] :RequestTimeoutException INFO [2019-12-17 12:35:15,430] jepsen worker 29 - jepsen.util 7589 :invoke :write [9 30] INFO [2019-12-17 12:35:15,436] jepsen worker 58 - jepsen.util 13378 :invoke :read [19 nil] INFO [2019-12-17 12:35:15,436] jepsen worker 82 - jepsen.util 13402 :invoke :read [27 nil] INFO [2019-12-17 12:35:15,436] jepsen worker 61 - jepsen.util 13561 :invoke :cas [20 [28 29]] INFO [2019-12-17 12:35:15,436] jepsen worker 68 - jepsen.util 12038 :invoke :write [22 39] INFO [2019-12-17 12:35:15,445] jepsen worker 7 - jepsen.util 12607 :info :write [2 4] :RequestCanceledException INFO [2019-12-17 12:35:15,450] jepsen worker 7 - jepsen.util 12697 :invoke :write [2 41] INFO [2019-12-17 12:35:15,462] jepsen worker 30 - jepsen.util 9120 :info :write [10 8] :RequestCanceledException INFO [2019-12-17 12:35:15,462] jepsen worker 75 - jepsen.util 12045 :info :cas [25 [47 7]] :RequestCanceledException INFO [2019-12-17 12:35:15,468] jepsen worker 75 - jepsen.util 12135 :invoke :read [25 nil] INFO [2019-12-17 12:35:15,468] jepsen worker 30 - jepsen.util 9210 :invoke :cas [10 [12 16]] INFO [2019-12-17 12:35:15,480] jepsen worker 64 - jepsen.util 7264 :ok :read [21 34] INFO [2019-12-17 12:35:15,480] jepsen worker 64 - jepsen.util 7264 :invoke :read [21 nil] INFO [2019-12-17 12:35:15,486] jepsen worker 28 - jepsen.util 7498 :fail :cas [9 [46 31]] :ValueNotSwapFrom INFO [2019-12-17 12:35:15,486] jepsen worker 28 - jepsen.util 7498 :invoke :cas [9 [15 33]] INFO [2019-12-17 12:35:15,491] jepsen worker 47 - jepsen.util 9677 :fail :cas [15 [32 47]] :ValueNotSwapFrom INFO [2019-12-17 12:35:15,491] jepsen worker 47 - jepsen.util 9677 :invoke :cas [15 [8 13]] INFO [2019-12-17 12:35:15,500] jepsen worker 27 - jepsen.util 8847 :ok :read [9 38] INFO [2019-12-17 12:35:15,500] jepsen worker 27 - jepsen.util 8847 :invoke :read [9 nil] INFO [2019-12-17 12:35:15,505] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53469,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:35:15,505] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53469,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:35:15,505] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53472,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:35:15,505] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53472,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:35:15,524] jepsen worker 52 - jepsen.util 9322 :ok :read [17 19] INFO [2019-12-17 12:35:15,524] jepsen worker 52 - jepsen.util 9322 :invoke :write [17 5] INFO [2019-12-17 12:35:15,530] jepsen worker 13 - jepsen.util 8653 :info :write [4 13] :RequestCanceledException INFO [2019-12-17 12:35:15,530] jepsen worker 74 - jepsen.util 9074 :fail :read [24 nil] :RequestTimeoutException INFO [2019-12-17 12:35:15,531] jepsen worker 9 - jepsen.util 9819 :fail :read [3 nil] :RequestTimeoutException INFO [2019-12-17 12:35:15,531] jepsen worker 74 - jepsen.util 9074 :invoke :cas [24 [48 2]] INFO [2019-12-17 12:35:15,531] jepsen worker 9 - jepsen.util 9819 :invoke :write [3 2] INFO [2019-12-17 12:35:15,537] jepsen worker 13 - jepsen.util 8743 :invoke :write [4 29] INFO [2019-12-17 12:35:15,549] jepsen worker 3 - jepsen.util 9003 :fail :cas [1 [21 19]] :ValueNotSwapFrom INFO [2019-12-17 12:35:15,551] jepsen worker 3 - jepsen.util 9003 :invoke :cas [1 [30 47]] INFO [2019-12-17 12:35:15,552] jepsen worker 29 - jepsen.util 7589 :ok :write [9 30] INFO [2019-12-17 12:35:15,552] jepsen worker 29 - jepsen.util 7589 :invoke :cas [9 [23 19]] INFO [2019-12-17 12:35:15,586] jepsen worker 5 - jepsen.util 8555 :fail :cas [1 [18 6]] :ValueNotSwapFrom INFO [2019-12-17 12:35:15,586] jepsen worker 5 - jepsen.util 8555 :invoke :write [1 22] INFO [2019-12-17 12:35:15,588] jepsen worker 28 - jepsen.util 7498 :fail :cas [9 [15 33]] :ValueNotSwapFrom INFO [2019-12-17 12:35:15,589] jepsen worker 28 - jepsen.util 7498 :invoke :cas [9 [20 15]] INFO [2019-12-17 12:35:15,594] jepsen worker 48 - jepsen.util 11658 :info :cas [16 [4 47]] :RequestCanceledException INFO [2019-12-17 12:35:15,598] jepsen worker 48 - jepsen.util 11748 :invoke :read [16 nil] INFO [2019-12-17 12:35:15,614] jepsen worker 81 - jepsen.util 12411 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:35:15,614] jepsen worker 81 - jepsen.util 12411 :invoke :write [27 31] INFO [2019-12-17 12:35:15,619] jepsen worker 27 - jepsen.util 8847 :ok :read [9 38] INFO [2019-12-17 12:35:15,620] jepsen worker 27 - jepsen.util 8847 :invoke :write [9 3] INFO [2019-12-17 12:35:15,621] jepsen worker 18 - jepsen.util 9558 :ok :write [6 41] INFO [2019-12-17 12:35:15,621] jepsen worker 18 - jepsen.util 9558 :invoke :cas [6 [9 14]] INFO [2019-12-17 12:35:15,622] jepsen worker 32 - jepsen.util 9302 :ok :write [10 33] INFO [2019-12-17 12:35:15,623] jepsen worker 32 - jepsen.util 9302 :invoke :read [10 nil] INFO [2019-12-17 12:35:15,644] jepsen worker 46 - jepsen.util 9226 :ok :write [15 30] INFO [2019-12-17 12:35:15,644] jepsen worker 46 - jepsen.util 9226 :invoke :write [15 2] INFO [2019-12-17 12:35:15,644] jepsen worker 39 - jepsen.util 12189 :info :cas [13 [42 35]] :RequestCanceledException INFO [2019-12-17 12:35:15,648] jepsen worker 39 - jepsen.util 12279 :invoke :read [13 nil] INFO [2019-12-17 12:35:15,650] jepsen worker 29 - jepsen.util 7589 :fail :cas [9 [23 19]] :ValueNotSwapFrom INFO [2019-12-17 12:35:15,650] jepsen worker 29 - jepsen.util 7589 :invoke :write [9 44] INFO [2019-12-17 12:35:15,660] jepsen worker 28 - jepsen.util 7498 :fail :cas [9 [20 15]] :ValueNotSwapFrom INFO [2019-12-17 12:35:15,661] jepsen worker 28 - jepsen.util 7498 :invoke :cas [9 [45 20]] INFO [2019-12-17 12:35:15,664] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:35:15,664] jepsen nemesis - couchbase.util Rebalance check count: 418 INFO [2019-12-17 12:35:15,664] jepsen nemesis - couchbase.util Rebalance stuck count: 399 INFO [2019-12-17 12:35:15,687] jepsen worker 4 - jepsen.util 8734 :ok :read [1 39] INFO [2019-12-17 12:35:15,688] jepsen worker 4 - jepsen.util 8734 :invoke :read [1 nil] INFO [2019-12-17 12:35:15,688] jepsen worker 88 - jepsen.util 9538 :fail :cas [29 [29 36]] :ValueNotSwapFrom INFO [2019-12-17 12:35:15,688] jepsen worker 88 - jepsen.util 9538 :invoke :write [29 30] INFO [2019-12-17 12:35:15,708] jepsen worker 47 - jepsen.util 9677 :fail :cas [15 [8 13]] :ValueNotSwapFrom INFO [2019-12-17 12:35:15,708] jepsen worker 47 - jepsen.util 9677 :invoke :read [15 nil] INFO [2019-12-17 12:35:15,728] jepsen worker 54 - jepsen.util 10134 :info :write [18 4] :RequestCanceledException INFO [2019-12-17 12:35:15,737] jepsen worker 54 - jepsen.util 10224 :invoke :write [18 38] INFO [2019-12-17 12:35:15,740] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53499,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:35:15,741] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53499,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:35:15,741] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53500,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:35:15,741] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53500,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:35:15,741] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53501,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:35:15,742] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53501,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:35:15,744] jepsen worker 33 - jepsen.util 12633 :info :cas [11 [13 10]] :RequestCanceledException INFO [2019-12-17 12:35:15,746] jepsen worker 89 - jepsen.util 9269 :ok :read [29 36] INFO [2019-12-17 12:35:15,746] jepsen worker 89 - jepsen.util 9269 :invoke :cas [29 [42 45]] INFO [2019-12-17 12:35:15,752] jepsen worker 33 - jepsen.util 12723 :invoke :read [11 nil] INFO [2019-12-17 12:35:15,755] jepsen worker 38 - jepsen.util 9758 :fail :cas [12 [46 4]] :ValueNotSwapFrom INFO [2019-12-17 12:35:15,755] jepsen worker 27 - jepsen.util 8847 :ok :write [9 3] INFO [2019-12-17 12:35:15,755] jepsen worker 38 - jepsen.util 9758 :invoke :write [12 27] INFO [2019-12-17 12:35:15,755] jepsen worker 27 - jepsen.util 8847 :invoke :cas [9 [49 9]] INFO [2019-12-17 12:35:15,775] jepsen worker 4 - jepsen.util 8734 :ok :read [1 39] INFO [2019-12-17 12:35:15,776] jepsen worker 4 - jepsen.util 8734 :invoke :write [1 19] INFO [2019-12-17 12:35:15,822] jepsen worker 70 - jepsen.util 9880 :ok :read [23 2] INFO [2019-12-17 12:35:15,822] jepsen worker 70 - jepsen.util 9880 :invoke :write [23 44] INFO [2019-12-17 12:35:15,827] jepsen worker 27 - jepsen.util 8847 :fail :cas [9 [49 9]] :ValueNotSwapFrom INFO [2019-12-17 12:35:15,828] jepsen worker 27 - jepsen.util 8847 :invoke :write [9 6] INFO [2019-12-17 12:35:15,831] jepsen worker 29 - jepsen.util 7589 :ok :write [9 44] INFO [2019-12-17 12:35:15,831] jepsen worker 29 - jepsen.util 7589 :invoke :read [9 nil] INFO [2019-12-17 12:35:15,844] jepsen worker 28 - jepsen.util 7498 :fail :cas [9 [45 20]] :ValueNotSwapFrom INFO [2019-12-17 12:35:15,845] jepsen worker 28 - jepsen.util 7498 :invoke :cas [9 [21 32]] INFO [2019-12-17 12:35:15,873] jepsen worker 29 - jepsen.util 7589 :ok :read [9 44] INFO [2019-12-17 12:35:15,874] jepsen worker 29 - jepsen.util 7589 :invoke :write [9 20] INFO [2019-12-17 12:35:15,944] jepsen worker 27 - jepsen.util 8847 :ok :write [9 6] INFO [2019-12-17 12:35:15,944] jepsen worker 27 - jepsen.util 8847 :invoke :cas [9 [24 10]] INFO [2019-12-17 12:35:15,958] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53523,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:35:15,959] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53523,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:35:15,959] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53526,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:35:15,959] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53526,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:35:15,959] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53527,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:35:15,959] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53527,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:35:15,980] jepsen worker 29 - jepsen.util 7589 :ok :write [9 20] INFO [2019-12-17 12:35:15,980] jepsen worker 29 - jepsen.util 7589 :invoke :read [9 nil] INFO [2019-12-17 12:35:15,982] jepsen worker 27 - jepsen.util 8847 :fail :cas [9 [24 10]] :ValueNotSwapFrom INFO [2019-12-17 12:35:15,982] jepsen worker 27 - jepsen.util 8847 :invoke :cas [9 [16 20]] INFO [2019-12-17 12:35:16,001] jepsen worker 11 - jepsen.util 9371 :ok :write [3 3] INFO [2019-12-17 12:35:16,001] jepsen worker 11 - jepsen.util 9371 :invoke :write [3 20] INFO [2019-12-17 12:35:16,002] jepsen worker 20 - jepsen.util 8750 :ok :write [6 45] INFO [2019-12-17 12:35:16,002] jepsen worker 20 - jepsen.util 8750 :invoke :read [6 nil] INFO [2019-12-17 12:35:16,003] jepsen worker 28 - jepsen.util 7498 :fail :cas [9 [21 32]] :ValueNotSwapFrom INFO [2019-12-17 12:35:16,003] jepsen worker 28 - jepsen.util 7498 :invoke :cas [9 [45 2]] INFO [2019-12-17 12:35:16,027] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53534,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:35:16,027] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53534,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:35:16,027] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53536,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:35:16,027] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53536,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:35:16,073] jepsen worker 53 - jepsen.util 8513 :ok :read [17 19] INFO [2019-12-17 12:35:16,073] jepsen worker 53 - jepsen.util 8513 :invoke :write [17 35] INFO [2019-12-17 12:35:16,076] jepsen worker 28 - jepsen.util 7498 :fail :cas [9 [45 2]] :ValueNotSwapFrom INFO [2019-12-17 12:35:16,076] jepsen worker 28 - jepsen.util 7498 :invoke :read [9 nil] INFO [2019-12-17 12:35:16,087] jepsen worker 46 - jepsen.util 9226 :ok :write [15 2] INFO [2019-12-17 12:35:16,087] jepsen worker 46 - jepsen.util 9226 :invoke :read [15 nil] INFO [2019-12-17 12:35:16,304] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53546,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:35:16,304] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53544,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:35:16,304] jepsen worker 9 - jepsen.util 9819 :ok :write [3 2] INFO [2019-12-17 12:35:16,304] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53546,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:35:16,304] jepsen worker 34 - jepsen.util 12724 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:35:16,304] jepsen worker 9 - jepsen.util 9819 :invoke :cas [3 [6 47]] INFO [2019-12-17 12:35:16,304] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53544,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:35:16,304] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53553,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:35:16,304] jepsen worker 2 - jepsen.util 12332 :info :write [0 25] :RequestCanceledException INFO [2019-12-17 12:35:16,304] jepsen worker 34 - jepsen.util 12724 :invoke :read [11 nil] INFO [2019-12-17 12:35:16,304] jepsen worker 50 - jepsen.util 13280 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:35:16,304] jepsen worker 21 - jepsen.util 12441 :info :cas [7 [37 9]] :RequestCanceledException INFO [2019-12-17 12:35:16,304] jepsen worker 67 - jepsen.util 12757 :info :cas [22 [49 46]] :RequestCanceledException INFO [2019-12-17 12:35:16,304] jepsen worker 8 - jepsen.util 12248 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:35:16,304] jepsen worker 86 - jepsen.util 13496 :info :write [28 23] :RequestCanceledException INFO [2019-12-17 12:35:16,304] jepsen worker 31 - jepsen.util 8221 :info :cas [10 [19 46]] :RequestCanceledException INFO [2019-12-17 12:35:16,304] jepsen worker 59 - jepsen.util 11939 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:35:16,304] jepsen worker 15 - jepsen.util 11715 :info :cas [5 [18 39]] :RequestCanceledException INFO [2019-12-17 12:35:16,304] jepsen worker 55 - jepsen.util 9865 :info :write [18 44] :RequestCanceledException INFO [2019-12-17 12:35:16,305] jepsen worker 22 - jepsen.util 12442 :info :write [7 33] :RequestCanceledException INFO [2019-12-17 12:35:16,305] jepsen worker 84 - jepsen.util 12144 :info :write [28 23] :RequestCanceledException INFO [2019-12-17 12:35:16,305] jepsen worker 14 - jepsen.util 9014 :info :write [4 42] :RequestCanceledException INFO [2019-12-17 12:35:16,305] jepsen worker 16 - jepsen.util 13786 :info :cas [5 [27 22]] :RequestCanceledException INFO [2019-12-17 12:35:16,305] jepsen worker 41 - jepsen.util 13361 :info :write [13 46] :RequestCanceledException INFO [2019-12-17 12:35:16,305] jepsen worker 77 - jepsen.util 13127 :info :write [25 44] :RequestCanceledException INFO [2019-12-17 12:35:16,305] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53551,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:35:16,305] jepsen worker 74 - jepsen.util 9074 :fail :cas [24 [48 2]] :ValueNotSwapFrom INFO [2019-12-17 12:35:16,305] jepsen worker 3 - jepsen.util 9003 :fail :cas [1 [30 47]] :ValueNotSwapFrom INFO [2019-12-17 12:35:16,305] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53553,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:35:16,306] jepsen worker 50 - jepsen.util 13280 :invoke :write [16 40] INFO [2019-12-17 12:35:16,306] jepsen worker 8 - jepsen.util 12248 :invoke :write [2 2] INFO [2019-12-17 12:35:16,307] jepsen worker 59 - jepsen.util 11939 :invoke :cas [19 [38 27]] INFO [2019-12-17 12:35:16,307] jepsen worker 22 - jepsen.util 12532 :invoke :cas [7 [36 23]] INFO [2019-12-17 12:35:16,308] jepsen worker 15 - jepsen.util 11805 :invoke :read [5 nil] INFO [2019-12-17 12:35:16,308] jepsen worker 67 - jepsen.util 12847 :invoke :read [22 nil] INFO [2019-12-17 12:35:16,308] jepsen worker 29 - jepsen.util 7589 :ok :read [9 20] INFO [2019-12-17 12:35:16,308] jepsen worker 2 - jepsen.util 12422 :invoke :cas [0 [12 38]] INFO [2019-12-17 12:35:16,308] jepsen worker 84 - jepsen.util 12234 :invoke :cas [28 [42 16]] INFO [2019-12-17 12:35:16,308] jepsen worker 14 - jepsen.util 9104 :invoke :read [4 nil] INFO [2019-12-17 12:35:16,308] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53551,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:35:16,308] jepsen worker 86 - jepsen.util 13586 :invoke :write [28 43] INFO [2019-12-17 12:35:16,308] jepsen worker 21 - jepsen.util 12531 :invoke :read [7 nil] INFO [2019-12-17 12:35:16,308] jepsen worker 74 - jepsen.util 9074 :invoke :read [24 nil] INFO [2019-12-17 12:35:16,308] jepsen worker 55 - jepsen.util 9955 :invoke :read [18 nil] INFO [2019-12-17 12:35:16,308] jepsen worker 16 - jepsen.util 13876 :invoke :read [5 nil] INFO [2019-12-17 12:35:16,308] jepsen worker 3 - jepsen.util 9003 :invoke :cas [1 [44 44]] INFO [2019-12-17 12:35:16,308] jepsen worker 31 - jepsen.util 8311 :invoke :read [10 nil] INFO [2019-12-17 12:35:16,309] jepsen worker 29 - jepsen.util 7589 :invoke :cas [9 [7 47]] INFO [2019-12-17 12:35:16,309] jepsen worker 41 - jepsen.util 13451 :invoke :write [13 36] INFO [2019-12-17 12:35:16,309] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53554,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:35:16,310] jepsen worker 77 - jepsen.util 13217 :invoke :write [25 9] INFO [2019-12-17 12:35:16,310] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53554,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:35:16,314] jepsen worker 27 - jepsen.util 8847 :fail :cas [9 [16 20]] :ValueNotSwapFrom INFO [2019-12-17 12:35:16,315] jepsen worker 27 - jepsen.util 8847 :invoke :cas [9 [48 37]] INFO [2019-12-17 12:35:16,320] jepsen worker 28 - jepsen.util 7498 :ok :read [9 20] INFO [2019-12-17 12:35:16,320] jepsen worker 28 - jepsen.util 7498 :invoke :write [9 44] INFO [2019-12-17 12:35:16,320] jepsen worker 46 - jepsen.util 9226 :ok :read [15 2] INFO [2019-12-17 12:35:16,321] jepsen worker 46 - jepsen.util 9226 :invoke :write [15 27] INFO [2019-12-17 12:35:16,321] jepsen worker 20 - jepsen.util 8750 :ok :read [6 45] INFO [2019-12-17 12:35:16,321] jepsen worker 20 - jepsen.util 8750 :invoke :cas [6 [31 31]] INFO [2019-12-17 12:35:16,361] jepsen worker 78 - jepsen.util 9078 :info :write [26 35] :RequestCanceledException INFO [2019-12-17 12:35:16,362] jepsen worker 78 - jepsen.util 9168 :invoke :write [26 8] INFO [2019-12-17 12:35:16,374] jepsen worker 29 - jepsen.util 7589 :fail :cas [9 [7 47]] :ValueNotSwapFrom INFO [2019-12-17 12:35:16,374] jepsen worker 29 - jepsen.util 7589 :invoke :read [9 nil] INFO [2019-12-17 12:35:16,428] jepsen worker 17 - jepsen.util 12527 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:35:16,428] jepsen worker 17 - jepsen.util 12527 :invoke :read [5 nil] INFO [2019-12-17 12:35:16,431] jepsen worker 31 - jepsen.util 8311 :ok :read [10 33] INFO [2019-12-17 12:35:16,431] jepsen worker 31 - jepsen.util 8311 :invoke :read [10 nil] INFO [2019-12-17 12:35:16,435] jepsen worker 9 - jepsen.util 9819 :fail :cas [3 [6 47]] :ValueNotSwapFrom INFO [2019-12-17 12:35:16,435] jepsen worker 9 - jepsen.util 9819 :invoke :read [3 nil] INFO [2019-12-17 12:35:16,448] jepsen worker 74 - jepsen.util 9074 :ok :read [24 2] INFO [2019-12-17 12:35:16,448] jepsen worker 74 - jepsen.util 9074 :invoke :cas [24 [44 31]] INFO [2019-12-17 12:35:16,454] jepsen worker 5 - jepsen.util 8555 :ok :write [1 22] INFO [2019-12-17 12:35:16,459] jepsen worker 73 - jepsen.util 8623 :info :write [24 21] :RequestCanceledException INFO [2019-12-17 12:35:16,482] jepsen worker 27 - jepsen.util 8847 :fail :cas [9 [48 37]] :ValueNotSwapFrom INFO [2019-12-17 12:35:16,490] jepsen worker 29 - jepsen.util 7589 :ok :read [9 20] INFO [2019-12-17 12:35:16,490] jepsen worker 20 - jepsen.util 8750 :fail :cas [6 [31 31]] :ValueNotSwapFrom INFO [2019-12-17 12:35:16,497] jepsen worker 69 - jepsen.util 9159 :info :write [23 45] :RequestCanceledException INFO [2019-12-17 12:35:16,497] jepsen worker 71 - jepsen.util 9791 :info :write [23 16] :RequestCanceledException INFO [2019-12-17 12:35:16,497] jepsen worker 37 - jepsen.util 8587 :info :write [12 45] :RequestCanceledException INFO [2019-12-17 12:35:16,509] jepsen worker 79 - jepsen.util 9529 :ok :write [26 17] INFO [2019-12-17 12:35:16,512] jepsen worker 9 - jepsen.util 9819 :ok :read [3 2] INFO [2019-12-17 12:35:16,522] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53584,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:35:16,522] jepsen worker 89 - jepsen.util 9269 :fail :cas [29 [42 45]] :ValueNotSwapFrom INFO [2019-12-17 12:35:16,528] jepsen worker 5 - jepsen.util 8555 :invoke :write [1 46] INFO [2019-12-17 12:35:16,529] jepsen worker 27 - jepsen.util 8847 :invoke :cas [9 [8 3]] INFO [2019-12-17 12:35:16,529] jepsen worker 29 - jepsen.util 7589 :invoke :read [9 nil] INFO [2019-12-17 12:35:16,529] jepsen worker 20 - jepsen.util 8750 :invoke :cas [6 [26 22]] INFO [2019-12-17 12:35:16,530] jepsen worker 79 - jepsen.util 9529 :invoke :write [26 18] INFO [2019-12-17 12:35:16,530] jepsen worker 9 - jepsen.util 9819 :invoke :read [3 nil] INFO [2019-12-17 12:35:16,530] jepsen worker 73 - jepsen.util 8713 :invoke :write [24 34] INFO [2019-12-17 12:35:16,530] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53584,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:35:16,530] jepsen worker 89 - jepsen.util 9269 :invoke :read [29 nil] INFO [2019-12-17 12:35:16,530] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53588,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:35:16,530] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53588,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:35:16,531] jepsen worker 37 - jepsen.util 8677 :invoke :write [12 28] INFO [2019-12-17 12:35:16,531] jepsen worker 71 - jepsen.util 9881 :invoke :read [23 nil] INFO [2019-12-17 12:35:16,531] jepsen worker 69 - jepsen.util 9249 :invoke :read [23 nil] INFO [2019-12-17 12:35:16,536] jepsen worker 74 - jepsen.util 9074 :fail :cas [24 [44 31]] :ValueNotSwapFrom INFO [2019-12-17 12:35:16,536] jepsen worker 74 - jepsen.util 9074 :invoke :read [24 nil] INFO [2019-12-17 12:35:16,548] jepsen worker 52 - jepsen.util 9322 :ok :write [17 5] INFO [2019-12-17 12:35:16,548] jepsen worker 52 - jepsen.util 9322 :invoke :read [17 nil] INFO [2019-12-17 12:35:16,568] jepsen worker 29 - jepsen.util 7589 :ok :read [9 20] INFO [2019-12-17 12:35:16,568] jepsen worker 29 - jepsen.util 7589 :invoke :write [9 12] INFO [2019-12-17 12:35:16,569] jepsen worker 27 - jepsen.util 8847 :fail :cas [9 [8 3]] :ValueNotSwapFrom INFO [2019-12-17 12:35:16,569] jepsen worker 27 - jepsen.util 8847 :invoke :cas [9 [10 37]] INFO [2019-12-17 12:35:16,573] jepsen worker 89 - jepsen.util 9269 :ok :read [29 36] INFO [2019-12-17 12:35:16,574] jepsen worker 89 - jepsen.util 9269 :invoke :read [29 nil] INFO [2019-12-17 12:35:16,580] jepsen worker 28 - jepsen.util 7498 :ok :write [9 44] INFO [2019-12-17 12:35:16,580] jepsen worker 28 - jepsen.util 7498 :invoke :write [9 29] INFO [2019-12-17 12:35:16,596] jepsen worker 43 - jepsen.util 12463 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:35:16,596] jepsen worker 43 - jepsen.util 12463 :invoke :write [14 13] INFO [2019-12-17 12:35:16,604] jepsen worker 27 - jepsen.util 8847 :fail :cas [9 [10 37]] :ValueNotSwapFrom INFO [2019-12-17 12:35:16,605] jepsen worker 27 - jepsen.util 8847 :invoke :cas [9 [2 37]] INFO [2019-12-17 12:35:16,609] jepsen worker 60 - jepsen.util 13650 :info :cas [20 [20 4]] :RequestCanceledException INFO [2019-12-17 12:35:16,611] jepsen worker 60 - jepsen.util 13740 :invoke :read [20 nil] INFO [2019-12-17 12:35:16,630] jepsen worker 76 - jepsen.util 12946 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:35:16,630] jepsen worker 42 - jepsen.util 13542 :info :write [14 28] :RequestCanceledException INFO [2019-12-17 12:35:16,630] jepsen worker 76 - jepsen.util 12946 :invoke :read [25 nil] INFO [2019-12-17 12:35:16,631] jepsen worker 42 - jepsen.util 13632 :invoke :write [14 30] INFO [2019-12-17 12:35:16,637] jepsen worker 20 - jepsen.util 8750 :fail :cas [6 [26 22]] :ValueNotSwapFrom INFO [2019-12-17 12:35:16,637] jepsen worker 20 - jepsen.util 8750 :invoke :cas [6 [46 11]] INFO [2019-12-17 12:35:16,638] jepsen worker 56 - jepsen.util 9776 :ok :write [18 33] INFO [2019-12-17 12:35:16,638] jepsen worker 56 - jepsen.util 9776 :invoke :cas [18 [41 39]] INFO [2019-12-17 12:35:16,657] jepsen worker 9 - jepsen.util 9819 :ok :read [3 2] INFO [2019-12-17 12:35:16,657] jepsen worker 9 - jepsen.util 9819 :invoke :cas [3 [8 31]] INFO [2019-12-17 12:35:16,662] jepsen worker 6 - jepsen.util 13146 :info :write [2 44] :RequestCanceledException INFO [2019-12-17 12:35:16,662] jepsen worker 57 - jepsen.util 12297 :info :write [19 1] :RequestCanceledException INFO [2019-12-17 12:35:16,663] jepsen worker 6 - jepsen.util 13236 :invoke :cas [2 [13 46]] INFO [2019-12-17 12:35:16,663] jepsen worker 57 - jepsen.util 12387 :invoke :cas [19 [2 24]] INFO [2019-12-17 12:35:16,670] jepsen worker 53 - jepsen.util 8513 :ok :write [17 35] INFO [2019-12-17 12:35:16,670] jepsen worker 53 - jepsen.util 8513 :invoke :read [17 nil] INFO [2019-12-17 12:35:16,685] jepsen worker 27 - jepsen.util 8847 :fail :cas [9 [2 37]] :ValueNotSwapFrom INFO [2019-12-17 12:35:16,685] jepsen worker 27 - jepsen.util 8847 :invoke :write [9 27] INFO [2019-12-17 12:35:16,695] jepsen worker 69 - jepsen.util 9249 :ok :read [23 2] INFO [2019-12-17 12:35:16,695] jepsen worker 69 - jepsen.util 9249 :invoke :read [23 nil] INFO [2019-12-17 12:35:16,697] jepsen worker 5 - jepsen.util 8555 :ok :write [1 46] INFO [2019-12-17 12:35:16,697] jepsen worker 5 - jepsen.util 8555 :invoke :write [1 37] INFO [2019-12-17 12:35:16,699] jepsen worker 28 - jepsen.util 7498 :ok :write [9 29] INFO [2019-12-17 12:35:16,699] jepsen worker 28 - jepsen.util 7498 :invoke :cas [9 [29 9]] INFO [2019-12-17 12:35:16,711] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:35:16,711] jepsen nemesis - couchbase.util Rebalance check count: 419 INFO [2019-12-17 12:35:16,711] jepsen nemesis - couchbase.util Rebalance stuck count: 400 INFO [2019-12-17 12:35:16,739] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53618,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:35:16,739] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53618,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:35:16,739] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53621,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:35:16,740] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53621,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:35:16,740] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53622,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:35:16,740] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53622,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:35:16,760] jepsen worker 51 - jepsen.util 8331 :info :write [17 29] :RequestCanceledException INFO [2019-12-17 12:35:16,762] jepsen worker 51 - jepsen.util 8421 :invoke :cas [17 [23 6]] INFO [2019-12-17 12:35:16,768] jepsen worker 29 - jepsen.util 7589 :ok :write [9 12] INFO [2019-12-17 12:35:16,768] jepsen worker 38 - jepsen.util 9758 :ok :write [12 27] INFO [2019-12-17 12:35:16,768] jepsen worker 29 - jepsen.util 7589 :invoke :cas [9 [26 25]] INFO [2019-12-17 12:35:16,768] jepsen worker 38 - jepsen.util 9758 :invoke :write [12 21] INFO [2019-12-17 12:35:16,795] jepsen worker 24 - jepsen.util 11634 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:35:16,795] jepsen worker 24 - jepsen.util 11634 :invoke :read [8 nil] INFO [2019-12-17 12:35:16,796] jepsen worker 44 - jepsen.util 12914 :info :write [14 41] :RequestCanceledException INFO [2019-12-17 12:35:16,796] jepsen worker 36 - jepsen.util 9666 :info :write [12 4] :RequestCanceledException INFO [2019-12-17 12:35:16,798] jepsen worker 36 - jepsen.util 9756 :invoke :read [12 nil] INFO [2019-12-17 12:35:16,799] jepsen worker 44 - jepsen.util 13004 :invoke :write [14 10] INFO [2019-12-17 12:35:16,800] jepsen worker 29 - jepsen.util 7589 :fail :cas [9 [26 25]] :ValueNotSwapFrom INFO [2019-12-17 12:35:16,800] jepsen worker 29 - jepsen.util 7589 :invoke :read [9 nil] INFO [2019-12-17 12:35:16,803] jepsen worker 79 - jepsen.util 9529 :ok :write [26 18] INFO [2019-12-17 12:35:16,803] jepsen worker 79 - jepsen.util 9529 :invoke :cas [26 [41 36]] INFO [2019-12-17 12:35:16,829] jepsen worker 35 - jepsen.util 12725 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:35:16,829] jepsen worker 19 - jepsen.util 9289 :fail :read [6 nil] :RequestTimeoutException INFO [2019-12-17 12:35:16,830] jepsen worker 35 - jepsen.util 12725 :invoke :read [11 nil] INFO [2019-12-17 12:35:16,830] jepsen worker 19 - jepsen.util 9289 :invoke :write [6 12] INFO [2019-12-17 12:35:16,834] jepsen worker 29 - jepsen.util 7589 :ok :read [9 12] INFO [2019-12-17 12:35:16,834] jepsen worker 29 - jepsen.util 7589 :invoke :write [9 10] INFO [2019-12-17 12:35:16,835] jepsen worker 69 - jepsen.util 9249 :ok :read [23 45] INFO [2019-12-17 12:35:16,835] jepsen worker 69 - jepsen.util 9249 :invoke :cas [23 [29 24]] INFO [2019-12-17 12:35:16,860] jepsen worker 27 - jepsen.util 8847 :ok :write [9 27] INFO [2019-12-17 12:35:16,860] jepsen worker 28 - jepsen.util 7498 :fail :cas [9 [29 9]] :CASMismatchException INFO [2019-12-17 12:35:16,860] jepsen worker 27 - jepsen.util 8847 :invoke :write [9 12] INFO [2019-12-17 12:35:16,860] jepsen worker 28 - jepsen.util 7498 :invoke :read [9 nil] INFO [2019-12-17 12:35:16,868] jepsen worker 53 - jepsen.util 8513 :ok :read [17 35] INFO [2019-12-17 12:35:16,868] jepsen worker 53 - jepsen.util 8513 :invoke :read [17 nil] INFO [2019-12-17 12:35:16,892] jepsen worker 36 - jepsen.util 9756 :ok :read [12 27] INFO [2019-12-17 12:35:16,892] jepsen worker 36 - jepsen.util 9756 :invoke :read [12 nil] INFO [2019-12-17 12:35:16,901] jepsen worker 28 - jepsen.util 7498 :ok :read [9 27] INFO [2019-12-17 12:35:16,901] jepsen worker 28 - jepsen.util 7498 :invoke :write [9 45] INFO [2019-12-17 12:35:16,911] jepsen worker 79 - jepsen.util 9529 :fail :cas [26 [41 36]] :ValueNotSwapFrom INFO [2019-12-17 12:35:16,911] jepsen worker 79 - jepsen.util 9529 :invoke :cas [26 [35 14]] INFO [2019-12-17 12:35:16,924] jepsen worker 14 - jepsen.util 9104 :ok :read [4 10] INFO [2019-12-17 12:35:16,924] jepsen worker 14 - jepsen.util 9104 :invoke :write [4 42] INFO [2019-12-17 12:35:16,925] jepsen worker 4 - jepsen.util 8734 :ok :write [1 19] INFO [2019-12-17 12:35:16,925] jepsen worker 4 - jepsen.util 8734 :invoke :cas [1 [9 19]] INFO [2019-12-17 12:35:16,928] jepsen worker 12 - jepsen.util 9642 :ok :write [4 13] INFO [2019-12-17 12:35:16,928] jepsen worker 12 - jepsen.util 9642 :invoke :read [4 nil] INFO [2019-12-17 12:35:16,930] jepsen worker 70 - jepsen.util 9880 :ok :write [23 44] INFO [2019-12-17 12:35:16,930] jepsen worker 70 - jepsen.util 9880 :invoke :read [23 nil] INFO [2019-12-17 12:35:16,945] jepsen worker 62 - jepsen.util 12842 :info :cas [20 [39 27]] :RequestCanceledException INFO [2019-12-17 12:35:16,946] jepsen worker 62 - jepsen.util 12932 :invoke :cas [20 [8 35]] INFO [2019-12-17 12:35:16,968] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53646,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:35:16,969] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53646,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:35:16,969] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53649,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:35:16,969] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53649,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:35:16,998] jepsen worker 28 - jepsen.util 7498 :ok :write [9 45] INFO [2019-12-17 12:35:16,998] jepsen worker 28 - jepsen.util 7498 :invoke :write [9 40] INFO [2019-12-17 12:35:17,021] jepsen worker 51 - jepsen.util 8421 :fail :cas [17 [23 6]] :ValueNotSwapFrom INFO [2019-12-17 12:35:17,021] jepsen worker 69 - jepsen.util 9249 :fail :cas [23 [29 24]] :ValueNotSwapFrom INFO [2019-12-17 12:35:17,021] jepsen worker 51 - jepsen.util 8421 :invoke :write [17 24] INFO [2019-12-17 12:35:17,021] jepsen worker 63 - jepsen.util 8073 :info :cas [21 [38 8]] :RequestCanceledException INFO [2019-12-17 12:35:17,021] jepsen worker 69 - jepsen.util 9249 :invoke :read [23 nil] INFO [2019-12-17 12:35:17,021] jepsen worker 85 - jepsen.util 13585 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:35:17,021] jepsen worker 52 - jepsen.util 9322 :ok :read [17 35] INFO [2019-12-17 12:35:17,022] jepsen worker 85 - jepsen.util 13585 :invoke :cas [28 [21 18]] INFO [2019-12-17 12:35:17,022] jepsen worker 52 - jepsen.util 9322 :invoke :read [17 nil] INFO [2019-12-17 12:35:17,022] jepsen worker 63 - jepsen.util 8163 :invoke :write [21 2] INFO [2019-12-17 12:35:17,044] jepsen worker 23 - jepsen.util 12533 :info :write [7 11] :RequestCanceledException INFO [2019-12-17 12:35:17,046] jepsen worker 23 - jepsen.util 12623 :invoke :read [7 nil] INFO [2019-12-17 12:35:17,054] jepsen worker 27 - jepsen.util 8847 :ok :write [9 12] INFO [2019-12-17 12:35:17,054] jepsen worker 27 - jepsen.util 8847 :invoke :write [9 15] INFO [2019-12-17 12:35:17,061] jepsen worker 83 - jepsen.util 13223 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:35:17,061] jepsen worker 83 - jepsen.util 13223 :invoke :write [27 38] INFO [2019-12-17 12:35:17,076] jepsen worker 28 - jepsen.util 7498 :ok :write [9 40] INFO [2019-12-17 12:35:17,076] jepsen worker 28 - jepsen.util 7498 :invoke :cas [9 [42 1]] INFO [2019-12-17 12:35:17,080] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53662,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:35:17,080] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53662,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:35:17,081] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53666,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:35:17,082] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53666,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:35:17,089] jepsen worker 80 - jepsen.util 8900 :ok :read [26 18] INFO [2019-12-17 12:35:17,089] jepsen worker 80 - jepsen.util 8900 :invoke :write [26 7] INFO [2019-12-17 12:35:17,090] jepsen worker 53 - jepsen.util 8513 :ok :read [17 35] INFO [2019-12-17 12:35:17,091] jepsen worker 53 - jepsen.util 8513 :invoke :write [17 30] INFO [2019-12-17 12:35:17,106] jepsen worker 73 - jepsen.util 8713 :ok :write [24 34] INFO [2019-12-17 12:35:17,106] jepsen worker 73 - jepsen.util 8713 :invoke :read [24 nil] INFO [2019-12-17 12:35:17,107] jepsen worker 46 - jepsen.util 9226 :ok :write [15 27] INFO [2019-12-17 12:35:17,107] jepsen worker 46 - jepsen.util 9226 :invoke :cas [15 [21 39]] INFO [2019-12-17 12:35:17,111] jepsen worker 28 - jepsen.util 7498 :fail :cas [9 [42 1]] :ValueNotSwapFrom INFO [2019-12-17 12:35:17,111] jepsen worker 28 - jepsen.util 7498 :invoke :write [9 38] INFO [2019-12-17 12:35:17,125] jepsen worker 55 - jepsen.util 9955 :ok :read [18 44] INFO [2019-12-17 12:35:17,125] jepsen worker 55 - jepsen.util 9955 :invoke :read [18 nil] INFO [2019-12-17 12:35:17,131] jepsen worker 65 - jepsen.util 7715 :info :write [21 41] :RequestCanceledException INFO [2019-12-17 12:35:17,131] jepsen worker 10 - jepsen.util 9190 :info :cas [3 [45 21]] :RequestCanceledException INFO [2019-12-17 12:35:17,132] jepsen worker 10 - jepsen.util 9280 :invoke :read [3 nil] INFO [2019-12-17 12:35:17,132] jepsen worker 65 - jepsen.util 7805 :invoke :write [21 17] INFO [2019-12-17 12:35:17,147] jepsen worker 74 - jepsen.util 9074 :ok :read [24 34] INFO [2019-12-17 12:35:17,147] jepsen worker 74 - jepsen.util 9074 :invoke :write [24 22] INFO [2019-12-17 12:35:17,165] jepsen worker 36 - jepsen.util 9756 :ok :read [12 4] INFO [2019-12-17 12:35:17,166] jepsen worker 36 - jepsen.util 9756 :invoke :write [12 21] INFO [2019-12-17 12:35:17,183] jepsen worker 89 - jepsen.util 9269 :ok :read [29 36] INFO [2019-12-17 12:35:17,183] jepsen worker 89 - jepsen.util 9269 :invoke :cas [29 [33 13]] INFO [2019-12-17 12:35:17,187] jepsen worker 46 - jepsen.util 9226 :fail :cas [15 [21 39]] :ValueNotSwapFrom INFO [2019-12-17 12:35:17,187] jepsen worker 46 - jepsen.util 9226 :invoke :cas [15 [23 16]] INFO [2019-12-17 12:35:17,194] jepsen worker 40 - jepsen.util 13000 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:35:17,194] jepsen worker 40 - jepsen.util 13000 :invoke :write [13 44] INFO [2019-12-17 12:35:17,197] jepsen worker 87 - jepsen.util 8907 :fail :cas [29 [13 42]] :ValueNotSwapFrom INFO [2019-12-17 12:35:17,197] jepsen worker 87 - jepsen.util 8907 :invoke :cas [29 [30 30]] INFO [2019-12-17 12:35:17,200] jepsen worker 20 - jepsen.util 8750 :fail :cas [6 [46 11]] :ValueNotSwapFrom INFO [2019-12-17 12:35:17,200] jepsen worker 20 - jepsen.util 8750 :invoke :cas [6 [32 5]] INFO [2019-12-17 12:35:17,221] jepsen worker 27 - jepsen.util 8847 :ok :write [9 15] INFO [2019-12-17 12:35:17,221] jepsen worker 27 - jepsen.util 8847 :invoke :read [9 nil] INFO [2019-12-17 12:35:17,224] jepsen worker 10 - jepsen.util 9280 :ok :read [3 2] INFO [2019-12-17 12:35:17,224] jepsen worker 10 - jepsen.util 9280 :invoke :write [3 45] INFO [2019-12-17 12:35:17,229] jepsen worker 9 - jepsen.util 9819 :fail :cas [3 [8 31]] :ValueNotSwapFrom INFO [2019-12-17 12:35:17,229] jepsen worker 9 - jepsen.util 9819 :invoke :cas [3 [23 8]] INFO [2019-12-17 12:35:17,232] jepsen worker 56 - jepsen.util 9776 :fail :cas [18 [41 39]] :ValueNotSwapFrom INFO [2019-12-17 12:35:17,232] jepsen worker 56 - jepsen.util 9776 :invoke :read [18 nil] INFO [2019-12-17 12:35:17,243] jepsen worker 27 - jepsen.util 8847 :ok :read [9 15] INFO [2019-12-17 12:35:17,243] jepsen worker 27 - jepsen.util 8847 :invoke :write [9 39] INFO [2019-12-17 12:35:17,248] jepsen worker 79 - jepsen.util 9529 :fail :cas [26 [35 14]] :ValueNotSwapFrom INFO [2019-12-17 12:35:17,248] jepsen worker 79 - jepsen.util 9529 :invoke :read [26 nil] INFO [2019-12-17 12:35:17,262] jepsen worker 66 - jepsen.util 12396 :info :cas [22 [30 49]] :RequestCanceledException INFO [2019-12-17 12:35:17,263] jepsen worker 66 - jepsen.util 12486 :invoke :read [22 nil] INFO [2019-12-17 12:35:17,263] jepsen worker 78 - jepsen.util 9168 :ok :write [26 8] INFO [2019-12-17 12:35:17,264] jepsen worker 78 - jepsen.util 9168 :invoke :read [26 nil] INFO [2019-12-17 12:35:17,276] jepsen worker 69 - jepsen.util 9249 :ok :read [23 44] INFO [2019-12-17 12:35:17,276] jepsen worker 69 - jepsen.util 9249 :invoke :read [23 nil] INFO [2019-12-17 12:35:17,298] jepsen worker 46 - jepsen.util 9226 :fail :cas [15 [23 16]] :ValueNotSwapFrom INFO [2019-12-17 12:35:17,299] jepsen worker 46 - jepsen.util 9226 :invoke :cas [15 [40 31]] INFO [2019-12-17 12:35:17,299] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53689,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:35:17,299] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53699,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:35:17,299] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53689,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:35:17,299] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53699,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:35:17,299] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53690,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:35:17,299] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53702,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:35:17,299] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53690,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:35:17,300] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53702,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:35:17,319] jepsen worker 52 - jepsen.util 9322 :ok :read [17 35] INFO [2019-12-17 12:35:17,319] jepsen worker 52 - jepsen.util 9322 :invoke :cas [17 [24 30]] INFO [2019-12-17 12:35:17,322] jepsen worker 63 - jepsen.util 8163 :ok :write [21 2] INFO [2019-12-17 12:35:17,323] jepsen worker 63 - jepsen.util 8163 :invoke :read [21 nil] INFO [2019-12-17 12:35:17,327] jepsen worker 70 - jepsen.util 9880 :ok :read [23 44] INFO [2019-12-17 12:35:17,328] jepsen worker 70 - jepsen.util 9880 :invoke :read [23 nil] INFO [2019-12-17 12:35:17,336] jepsen worker 46 - jepsen.util 9226 :fail :cas [15 [40 31]] :ValueNotSwapFrom INFO [2019-12-17 12:35:17,336] jepsen worker 46 - jepsen.util 9226 :invoke :cas [15 [46 34]] INFO [2019-12-17 12:35:17,347] jepsen worker 72 - jepsen.util 9162 :fail :read [24 nil] :RequestTimeoutException INFO [2019-12-17 12:35:17,347] jepsen worker 45 - jepsen.util 9585 :fail :read [15 nil] :RequestTimeoutException INFO [2019-12-17 12:35:17,347] jepsen worker 45 - jepsen.util 9585 :invoke :write [15 1] INFO [2019-12-17 12:35:17,347] jepsen worker 72 - jepsen.util 9162 :invoke :cas [24 [49 23]] INFO [2019-12-17 12:35:17,354] jepsen worker 27 - jepsen.util 8847 :ok :write [9 39] INFO [2019-12-17 12:35:17,354] jepsen worker 27 - jepsen.util 8847 :invoke :write [9 26] INFO [2019-12-17 12:35:17,356] jepsen worker 12 - jepsen.util 9642 :ok :read [4 13] INFO [2019-12-17 12:35:17,357] jepsen worker 12 - jepsen.util 9642 :invoke :cas [4 [10 20]] INFO [2019-12-17 12:35:17,358] jepsen worker 46 - jepsen.util 9226 :fail :cas [15 [46 34]] :ValueNotSwapFrom INFO [2019-12-17 12:35:17,358] jepsen worker 46 - jepsen.util 9226 :invoke :write [15 30] INFO [2019-12-17 12:35:17,361] jepsen worker 26 - jepsen.util 13346 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:35:17,362] jepsen worker 26 - jepsen.util 13346 :invoke :read [8 nil] INFO [2019-12-17 12:35:17,374] jepsen worker 11 - jepsen.util 9371 :ok :write [3 20] INFO [2019-12-17 12:35:17,374] jepsen worker 11 - jepsen.util 9371 :invoke :cas [3 [11 37]] INFO [2019-12-17 12:35:17,397] jepsen worker 1 - jepsen.util 12871 :info :cas [0 [24 45]] :RequestCanceledException INFO [2019-12-17 12:35:17,406] jepsen worker 69 - jepsen.util 9249 :ok :read [23 44] INFO [2019-12-17 12:35:17,415] jepsen worker 0 - jepsen.util 13320 :info :cas [0 [27 14]] :RequestCanceledException INFO [2019-12-17 12:35:17,418] jepsen worker 79 - jepsen.util 9529 :ok :read [26 8] INFO [2019-12-17 12:35:17,423] jepsen worker 72 - jepsen.util 9162 :fail :cas [24 [49 23]] :ValueNotSwapFrom INFO [2019-12-17 12:35:17,428] jepsen worker 19 - jepsen.util 9289 :ok :write [6 12] INFO [2019-12-17 12:35:17,431] jepsen worker 49 - jepsen.util 12109 :info :write [16 31] :RequestCanceledException INFO [2019-12-17 12:35:17,435] jepsen worker 55 - jepsen.util 9955 :ok :read [18 44] INFO [2019-12-17 12:35:17,447] jepsen worker 27 - jepsen.util 8847 :ok :write [9 26] INFO [2019-12-17 12:35:17,461] jepsen worker 82 - jepsen.util 13402 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:35:17,474] jepsen worker 65 - jepsen.util 7805 :ok :write [21 17] INFO [2019-12-17 12:35:17,476] jepsen worker 69 - jepsen.util 9249 :invoke :write [23 6] INFO [2019-12-17 12:35:17,476] jepsen worker 79 - jepsen.util 9529 :invoke :read [26 nil] INFO [2019-12-17 12:35:17,476] jepsen worker 72 - jepsen.util 9162 :invoke :cas [24 [35 49]] INFO [2019-12-17 12:35:17,476] jepsen worker 19 - jepsen.util 9289 :invoke :write [6 11] INFO [2019-12-17 12:35:17,477] jepsen worker 55 - jepsen.util 9955 :invoke :write [18 9] INFO [2019-12-17 12:35:17,477] jepsen worker 27 - jepsen.util 8847 :invoke :write [9 15] INFO [2019-12-17 12:35:17,477] jepsen worker 82 - jepsen.util 13402 :invoke :write [27 34] INFO [2019-12-17 12:35:17,477] jepsen worker 65 - jepsen.util 7805 :invoke :write [21 17] INFO [2019-12-17 12:35:17,477] jepsen worker 1 - jepsen.util 12961 :invoke :cas [0 [12 3]] INFO [2019-12-17 12:35:17,477] jepsen worker 0 - jepsen.util 13410 :invoke :cas [0 [0 17]] INFO [2019-12-17 12:35:17,478] jepsen worker 49 - jepsen.util 12199 :invoke :cas [16 [22 21]] INFO [2019-12-17 12:35:17,479] jepsen worker 89 - jepsen.util 9269 :fail :cas [29 [33 13]] :ValueNotSwapFrom INFO [2019-12-17 12:35:17,480] jepsen worker 89 - jepsen.util 9269 :invoke :cas [29 [45 26]] INFO [2019-12-17 12:35:17,496] jepsen worker 68 - jepsen.util 12038 :info :write [22 39] :RequestCanceledException INFO [2019-12-17 12:35:17,499] jepsen worker 68 - jepsen.util 12128 :invoke :write [22 46] INFO [2019-12-17 12:35:17,515] jepsen worker 7 - jepsen.util 12697 :info :write [2 41] :RequestCanceledException INFO [2019-12-17 12:35:17,515] jepsen worker 25 - jepsen.util 13705 :info :write [8 36] :RequestCanceledException INFO [2019-12-17 12:35:17,515] jepsen worker 46 - jepsen.util 9226 :ok :write [15 30] INFO [2019-12-17 12:35:17,516] jepsen worker 46 - jepsen.util 9226 :invoke :read [15 nil] INFO [2019-12-17 12:35:17,516] jepsen worker 7 - jepsen.util 12787 :invoke :cas [2 [23 31]] INFO [2019-12-17 12:35:17,516] jepsen worker 25 - jepsen.util 13795 :invoke :read [8 nil] INFO [2019-12-17 12:35:17,518] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53729,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:35:17,518] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53729,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:35:17,519] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53730,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:35:17,519] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53730,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:35:17,525] jepsen worker 52 - jepsen.util 9322 :fail :cas [17 [24 30]] :ValueNotSwapFrom INFO [2019-12-17 12:35:17,525] jepsen worker 52 - jepsen.util 9322 :invoke :read [17 nil] INFO [2019-12-17 12:35:17,530] jepsen worker 75 - jepsen.util 12135 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:35:17,531] jepsen worker 75 - jepsen.util 12135 :invoke :read [25 nil] INFO [2019-12-17 12:35:17,533] jepsen worker 64 - jepsen.util 7264 :fail :read [21 nil] :RequestTimeoutException INFO [2019-12-17 12:35:17,533] jepsen worker 61 - jepsen.util 13561 :info :cas [20 [28 29]] :RequestCanceledException INFO [2019-12-17 12:35:17,534] jepsen worker 64 - jepsen.util 7264 :invoke :write [21 23] INFO [2019-12-17 12:35:17,535] jepsen worker 61 - jepsen.util 13651 :invoke :cas [20 [9 2]] INFO [2019-12-17 12:35:17,540] jepsen worker 46 - jepsen.util 9226 :ok :read [15 30] INFO [2019-12-17 12:35:17,541] jepsen worker 72 - jepsen.util 9162 :fail :cas [24 [35 49]] :ValueNotSwapFrom INFO [2019-12-17 12:35:17,541] jepsen worker 72 - jepsen.util 9162 :invoke :write [24 10] INFO [2019-12-17 12:35:17,543] jepsen worker 46 - jepsen.util 9226 :invoke :cas [15 [34 32]] INFO [2019-12-17 12:35:17,544] jepsen worker 58 - jepsen.util 13378 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:35:17,544] jepsen worker 30 - jepsen.util 9210 :info :cas [10 [12 16]] :RequestCanceledException INFO [2019-12-17 12:35:17,545] jepsen worker 58 - jepsen.util 13378 :invoke :read [19 nil] INFO [2019-12-17 12:35:17,546] jepsen worker 30 - jepsen.util 9300 :invoke :write [10 35] INFO [2019-12-17 12:35:17,556] jepsen worker 56 - jepsen.util 9776 :ok :read [18 44] INFO [2019-12-17 12:35:17,556] jepsen worker 56 - jepsen.util 9776 :invoke :cas [18 [28 25]] INFO [2019-12-17 12:35:17,573] jepsen worker 46 - jepsen.util 9226 :fail :cas [15 [34 32]] :ValueNotSwapFrom INFO [2019-12-17 12:35:17,573] jepsen worker 46 - jepsen.util 9226 :invoke :cas [15 [20 6]] INFO [2019-12-17 12:35:17,589] jepsen worker 27 - jepsen.util 8847 :ok :write [9 15] INFO [2019-12-17 12:35:17,590] jepsen worker 27 - jepsen.util 8847 :invoke :cas [9 [47 26]] INFO [2019-12-17 12:35:17,597] jepsen worker 46 - jepsen.util 9226 :fail :cas [15 [20 6]] :ValueNotSwapFrom INFO [2019-12-17 12:35:17,597] jepsen worker 46 - jepsen.util 9226 :invoke :write [15 27] INFO [2019-12-17 12:35:17,612] jepsen worker 27 - jepsen.util 8847 :fail :cas [9 [47 26]] :ValueNotSwapFrom INFO [2019-12-17 12:35:17,612] jepsen worker 27 - jepsen.util 8847 :invoke :cas [9 [47 46]] INFO [2019-12-17 12:35:17,621] jepsen worker 74 - jepsen.util 9074 :ok :write [24 22] INFO [2019-12-17 12:35:17,622] jepsen worker 74 - jepsen.util 9074 :invoke :read [24 nil] INFO [2019-12-17 12:35:17,626] jepsen worker 65 - jepsen.util 7805 :ok :write [21 17] INFO [2019-12-17 12:35:17,627] jepsen worker 65 - jepsen.util 7805 :invoke :cas [21 [24 16]] INFO [2019-12-17 12:35:17,628] jepsen worker 48 - jepsen.util 11748 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:35:17,628] jepsen worker 48 - jepsen.util 11748 :invoke :read [16 nil] INFO [2019-12-17 12:35:17,631] jepsen worker 27 - jepsen.util 8847 :fail :cas [9 [47 46]] :ValueNotSwapFrom INFO [2019-12-17 12:35:17,631] jepsen worker 27 - jepsen.util 8847 :invoke :read [9 nil] INFO [2019-12-17 12:35:17,636] jepsen worker 12 - jepsen.util 9642 :fail :cas [4 [10 20]] :ValueNotSwapFrom INFO [2019-12-17 12:35:17,636] jepsen worker 12 - jepsen.util 9642 :invoke :cas [4 [30 38]] INFO [2019-12-17 12:35:17,644] jepsen worker 18 - jepsen.util 9558 :info :cas [6 [9 14]] :RequestCanceledException INFO [2019-12-17 12:35:17,645] jepsen worker 18 - jepsen.util 9648 :invoke :cas [6 [2 11]] INFO [2019-12-17 12:35:17,651] jepsen worker 70 - jepsen.util 9880 :ok :read [23 44] INFO [2019-12-17 12:35:17,651] jepsen worker 70 - jepsen.util 9880 :invoke :write [23 43] INFO [2019-12-17 12:35:17,658] jepsen worker 13 - jepsen.util 8743 :info :write [4 29] :RequestCanceledException INFO [2019-12-17 12:35:17,659] jepsen worker 13 - jepsen.util 8833 :invoke :cas [4 [23 7]] INFO [2019-12-17 12:35:17,675] jepsen worker 27 - jepsen.util 8847 :ok :read [9 15] INFO [2019-12-17 12:35:17,675] jepsen worker 27 - jepsen.util 8847 :invoke :read [9 nil] INFO [2019-12-17 12:35:17,697] jepsen worker 39 - jepsen.util 12279 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:35:17,697] jepsen worker 39 - jepsen.util 12279 :invoke :read [13 nil] INFO [2019-12-17 12:35:17,702] jepsen worker 79 - jepsen.util 9529 :ok :read [26 8] INFO [2019-12-17 12:35:17,702] jepsen worker 79 - jepsen.util 9529 :invoke :read [26 nil] INFO [2019-12-17 12:35:17,704] jepsen worker 18 - jepsen.util 9648 :fail :cas [6 [2 11]] :ValueNotSwapFrom INFO [2019-12-17 12:35:17,705] jepsen worker 18 - jepsen.util 9648 :invoke :cas [6 [28 45]] INFO [2019-12-17 12:35:17,710] jepsen worker 27 - jepsen.util 8847 :ok :read [9 15] INFO [2019-12-17 12:35:17,710] jepsen worker 27 - jepsen.util 8847 :invoke :cas [9 [22 8]] INFO [2019-12-17 12:35:17,715] jepsen worker 81 - jepsen.util 12411 :info :write [27 31] :RequestCanceledException INFO [2019-12-17 12:35:17,715] jepsen worker 88 - jepsen.util 9538 :info :write [29 30] :RequestCanceledException INFO [2019-12-17 12:35:17,716] jepsen worker 81 - jepsen.util 12501 :invoke :read [27 nil] INFO [2019-12-17 12:35:17,716] jepsen worker 88 - jepsen.util 9628 :invoke :write [29 27] INFO [2019-12-17 12:35:17,726] jepsen worker 3 - jepsen.util 9003 :fail :cas [1 [44 44]] :ValueNotSwapFrom INFO [2019-12-17 12:35:17,726] jepsen worker 3 - jepsen.util 9003 :invoke :write [1 39] INFO [2019-12-17 12:35:17,730] jepsen worker 47 - jepsen.util 9677 :fail :read [15 nil] :RequestTimeoutException INFO [2019-12-17 12:35:17,730] jepsen worker 32 - jepsen.util 9302 :fail :read [10 nil] :RequestTimeoutException INFO [2019-12-17 12:35:17,731] jepsen worker 32 - jepsen.util 9302 :invoke :write [10 37] INFO [2019-12-17 12:35:17,731] jepsen worker 47 - jepsen.util 9677 :invoke :cas [15 [2 17]] INFO [2019-12-17 12:35:17,733] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53767,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:35:17,734] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53767,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:35:17,734] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53770,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:35:17,734] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53770,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:35:17,738] jepsen worker 18 - jepsen.util 9648 :fail :cas [6 [28 45]] :ValueNotSwapFrom INFO [2019-12-17 12:35:17,738] jepsen worker 18 - jepsen.util 9648 :invoke :read [6 nil] INFO [2019-12-17 12:35:17,752] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:35:17,752] jepsen nemesis - couchbase.util Rebalance check count: 420 INFO [2019-12-17 12:35:17,752] jepsen nemesis - couchbase.util Rebalance stuck count: 401 INFO [2019-12-17 12:35:17,758] jepsen worker 71 - jepsen.util 9881 :ok :read [23 44] INFO [2019-12-17 12:35:17,758] jepsen worker 71 - jepsen.util 9881 :invoke :read [23 nil] INFO [2019-12-17 12:35:17,767] jepsen worker 27 - jepsen.util 8847 :fail :cas [9 [22 8]] :ValueNotSwapFrom INFO [2019-12-17 12:35:17,767] jepsen worker 27 - jepsen.util 8847 :invoke :write [9 35] INFO [2019-12-17 12:35:17,778] jepsen worker 18 - jepsen.util 9648 :ok :read [6 12] INFO [2019-12-17 12:35:17,778] jepsen worker 18 - jepsen.util 9648 :invoke :cas [6 [24 22]] INFO [2019-12-17 12:35:17,783] jepsen worker 87 - jepsen.util 8907 :fail :cas [29 [30 30]] :ValueNotSwapFrom INFO [2019-12-17 12:35:17,783] jepsen worker 87 - jepsen.util 8907 :invoke :read [29 nil] INFO [2019-12-17 12:35:17,812] jepsen worker 54 - jepsen.util 10224 :info :write [18 38] :RequestCanceledException INFO [2019-12-17 12:35:17,817] jepsen worker 54 - jepsen.util 10314 :invoke :cas [18 [24 47]] INFO [2019-12-17 12:35:17,821] jepsen worker 4 - jepsen.util 8734 :fail :cas [1 [9 19]] :ValueNotSwapFrom INFO [2019-12-17 12:35:17,821] jepsen worker 4 - jepsen.util 8734 :invoke :write [1 17] INFO [2019-12-17 12:35:17,828] jepsen worker 33 - jepsen.util 12723 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:35:17,829] jepsen worker 33 - jepsen.util 12723 :invoke :read [11 nil] INFO [2019-12-17 12:35:17,838] jepsen worker 27 - jepsen.util 8847 :ok :write [9 35] INFO [2019-12-17 12:35:17,838] jepsen worker 27 - jepsen.util 8847 :invoke :write [9 26] INFO [2019-12-17 12:35:17,898] jepsen worker 27 - jepsen.util 8847 :ok :write [9 26] INFO [2019-12-17 12:35:17,898] jepsen worker 27 - jepsen.util 8847 :invoke :cas [9 [25 1]] INFO [2019-12-17 12:35:17,931] jepsen worker 27 - jepsen.util 8847 :fail :cas [9 [25 1]] :ValueNotSwapFrom INFO [2019-12-17 12:35:17,932] jepsen worker 27 - jepsen.util 8847 :invoke :cas [9 [36 31]] INFO [2019-12-17 12:35:17,941] jepsen worker 78 - jepsen.util 9168 :ok :read [26 8] INFO [2019-12-17 12:35:17,941] jepsen worker 78 - jepsen.util 9168 :invoke :cas [26 [22 6]] INFO [2019-12-17 12:35:17,952] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53794,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:35:17,953] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53794,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:35:17,953] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53798,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:35:17,953] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53798,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:35:18,037] jepsen worker 27 - jepsen.util 8847 :fail :cas [9 [36 31]] :ValueNotSwapFrom INFO [2019-12-17 12:35:18,037] jepsen worker 27 - jepsen.util 8847 :invoke :read [9 nil] INFO [2019-12-17 12:35:18,040] jepsen worker 54 - jepsen.util 10314 :fail :cas [18 [24 47]] :ValueNotSwapFrom INFO [2019-12-17 12:35:18,040] jepsen worker 54 - jepsen.util 10314 :invoke :read [18 nil] INFO [2019-12-17 12:35:18,041] jepsen worker 89 - jepsen.util 9269 :fail :cas [29 [45 26]] :ValueNotSwapFrom INFO [2019-12-17 12:35:18,041] jepsen worker 89 - jepsen.util 9269 :invoke :cas [29 [36 38]] INFO [2019-12-17 12:35:18,052] jepsen worker 87 - jepsen.util 8907 :ok :read [29 36] INFO [2019-12-17 12:35:18,052] jepsen worker 87 - jepsen.util 8907 :invoke :write [29 41] INFO [2019-12-17 12:35:18,056] jepsen worker 27 - jepsen.util 8847 :ok :read [9 26] INFO [2019-12-17 12:35:18,056] jepsen worker 27 - jepsen.util 8847 :invoke :write [9 12] INFO [2019-12-17 12:35:18,062] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53805,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:35:18,063] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53805,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:35:18,064] jepsen worker 52 - jepsen.util 9322 :ok :read [17 35] INFO [2019-12-17 12:35:18,064] jepsen worker 52 - jepsen.util 9322 :invoke :cas [17 [35 43]] INFO [2019-12-17 12:35:18,100] jepsen worker 27 - jepsen.util 8847 :ok :write [9 12] INFO [2019-12-17 12:35:18,100] jepsen worker 27 - jepsen.util 8847 :invoke :write [9 21] INFO [2019-12-17 12:35:18,170] jepsen worker 56 - jepsen.util 9776 :fail :cas [18 [28 25]] :ValueNotSwapFrom INFO [2019-12-17 12:35:18,183] jepsen worker 78 - jepsen.util 9168 :fail :cas [26 [22 6]] :ValueNotSwapFrom INFO [2019-12-17 12:35:18,197] jepsen worker 27 - jepsen.util 8847 :ok :write [9 21] INFO [2019-12-17 12:35:18,209] jepsen worker 79 - jepsen.util 9529 :ok :read [26 8] INFO [2019-12-17 12:35:18,228] jepsen worker 12 - jepsen.util 9642 :fail :cas [4 [30 38]] :ValueNotSwapFrom INFO [2019-12-17 12:35:18,234] jepsen worker 78 - jepsen.util 9168 :invoke :read [26 nil] INFO [2019-12-17 12:35:18,234] jepsen worker 56 - jepsen.util 9776 :invoke :read [18 nil] INFO [2019-12-17 12:35:18,234] jepsen worker 27 - jepsen.util 8847 :invoke :write [9 45] INFO [2019-12-17 12:35:18,234] jepsen worker 79 - jepsen.util 9529 :invoke :cas [26 [35 26]] INFO [2019-12-17 12:35:18,234] jepsen worker 12 - jepsen.util 9642 :invoke :read [4 nil] INFO [2019-12-17 12:35:18,278] jepsen worker 27 - jepsen.util 8847 :ok :write [9 45] INFO [2019-12-17 12:35:18,278] jepsen worker 27 - jepsen.util 8847 :invoke :read [9 nil] INFO [2019-12-17 12:35:18,282] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53815,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:35:18,282] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53822,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:35:18,282] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53815,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:35:18,282] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53822,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:35:18,282] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53816,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:35:18,282] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53826,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:35:18,282] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53816,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:35:18,282] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53826,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:35:18,282] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53817,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:35:18,282] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53817,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:35:18,305] jepsen worker 54 - jepsen.util 10314 :ok :read [18 44] INFO [2019-12-17 12:35:18,306] jepsen worker 54 - jepsen.util 10314 :invoke :cas [18 [44 1]] INFO [2019-12-17 12:35:18,309] jepsen worker 27 - jepsen.util 8847 :ok :read [9 45] INFO [2019-12-17 12:35:18,309] jepsen worker 27 - jepsen.util 8847 :invoke :read [9 nil] INFO [2019-12-17 12:35:18,322] jepsen worker 78 - jepsen.util 9168 :ok :read [26 8] INFO [2019-12-17 12:35:18,322] jepsen worker 78 - jepsen.util 9168 :invoke :write [26 1] INFO [2019-12-17 12:35:18,329] jepsen worker 27 - jepsen.util 8847 :ok :read [9 45] INFO [2019-12-17 12:35:18,329] jepsen worker 27 - jepsen.util 8847 :invoke :write [9 39] INFO [2019-12-17 12:35:18,330] jepsen worker 67 - jepsen.util 12847 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:35:18,330] jepsen worker 8 - jepsen.util 12248 :info :write [2 2] :RequestCanceledException INFO [2019-12-17 12:35:18,330] jepsen worker 67 - jepsen.util 12847 :invoke :read [22 nil] INFO [2019-12-17 12:35:18,330] jepsen worker 59 - jepsen.util 11939 :info :cas [19 [38 27]] :RequestCanceledException INFO [2019-12-17 12:35:18,330] jepsen worker 86 - jepsen.util 13586 :info :write [28 43] :RequestCanceledException INFO [2019-12-17 12:35:18,331] jepsen worker 41 - jepsen.util 13451 :info :write [13 36] :RequestCanceledException INFO [2019-12-17 12:35:18,331] jepsen worker 8 - jepsen.util 12338 :invoke :write [2 6] INFO [2019-12-17 12:35:18,331] jepsen worker 41 - jepsen.util 13541 :invoke :cas [13 [44 17]] INFO [2019-12-17 12:35:18,331] jepsen worker 59 - jepsen.util 12029 :invoke :cas [19 [8 41]] INFO [2019-12-17 12:35:18,332] jepsen worker 86 - jepsen.util 13676 :invoke :read [28 nil] INFO [2019-12-17 12:35:18,344] jepsen worker 34 - jepsen.util 12724 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:35:18,344] jepsen worker 22 - jepsen.util 12532 :info :cas [7 [36 23]] :RequestCanceledException INFO [2019-12-17 12:35:18,344] jepsen worker 21 - jepsen.util 12531 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:35:18,344] jepsen worker 50 - jepsen.util 13280 :info :write [16 40] :RequestCanceledException INFO [2019-12-17 12:35:18,344] jepsen worker 77 - jepsen.util 13217 :info :write [25 9] :RequestCanceledException INFO [2019-12-17 12:35:18,344] jepsen worker 34 - jepsen.util 12724 :invoke :cas [11 [9 9]] INFO [2019-12-17 12:35:18,344] jepsen worker 21 - jepsen.util 12531 :invoke :read [7 nil] INFO [2019-12-17 12:35:18,345] jepsen worker 50 - jepsen.util 13370 :invoke :read [16 nil] INFO [2019-12-17 12:35:18,345] jepsen worker 22 - jepsen.util 12622 :invoke :cas [7 [33 44]] INFO [2019-12-17 12:35:18,345] jepsen worker 77 - jepsen.util 13307 :invoke :read [25 nil] INFO [2019-12-17 12:35:18,362] jepsen worker 15 - jepsen.util 11805 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:35:18,362] jepsen worker 15 - jepsen.util 11805 :invoke :cas [5 [13 20]] INFO [2019-12-17 12:35:18,369] jepsen worker 27 - jepsen.util 8847 :ok :write [9 39] INFO [2019-12-17 12:35:18,370] jepsen worker 27 - jepsen.util 8847 :invoke :read [9 nil] INFO [2019-12-17 12:35:18,374] jepsen worker 12 - jepsen.util 9642 :ok :read [4 13] INFO [2019-12-17 12:35:18,374] jepsen worker 12 - jepsen.util 9642 :invoke :cas [4 [17 9]] INFO [2019-12-17 12:35:18,398] jepsen worker 2 - jepsen.util 12422 :info :cas [0 [12 38]] :RequestCanceledException INFO [2019-12-17 12:35:18,398] jepsen worker 16 - jepsen.util 13876 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:35:18,398] jepsen worker 16 - jepsen.util 13876 :invoke :cas [5 [0 27]] INFO [2019-12-17 12:35:18,399] jepsen worker 2 - jepsen.util 12512 :invoke :cas [0 [43 26]] INFO [2019-12-17 12:35:18,400] jepsen worker 27 - jepsen.util 8847 :ok :read [9 39] INFO [2019-12-17 12:35:18,400] jepsen worker 27 - jepsen.util 8847 :invoke :cas [9 [17 19]] INFO [2019-12-17 12:35:18,412] jepsen worker 27 - jepsen.util 8847 :fail :cas [9 [17 19]] :ValueNotSwapFrom INFO [2019-12-17 12:35:18,412] jepsen worker 27 - jepsen.util 8847 :invoke :read [9 nil] INFO [2019-12-17 12:35:18,414] jepsen worker 84 - jepsen.util 12234 :info :cas [28 [42 16]] :RequestCanceledException INFO [2019-12-17 12:35:18,415] jepsen worker 84 - jepsen.util 12324 :invoke :cas [28 [49 5]] INFO [2019-12-17 12:35:18,424] jepsen worker 27 - jepsen.util 8847 :ok :read [9 39] INFO [2019-12-17 12:35:18,424] jepsen worker 27 - jepsen.util 8847 :invoke :write [9 38] INFO [2019-12-17 12:35:18,465] jepsen worker 27 - jepsen.util 8847 :ok :write [9 38] INFO [2019-12-17 12:35:18,465] jepsen worker 27 - jepsen.util 8847 :invoke :read [9 nil] INFO [2019-12-17 12:35:18,478] jepsen worker 27 - jepsen.util 8847 :ok :read [9 38] INFO [2019-12-17 12:35:18,479] jepsen worker 27 - jepsen.util 8847 :invoke :cas [9 [42 24]] INFO [2019-12-17 12:35:18,483] jepsen worker 13 - jepsen.util 8833 :fail :cas [4 [23 7]] :ValueNotSwapFrom INFO [2019-12-17 12:35:18,483] jepsen worker 13 - jepsen.util 8833 :invoke :cas [4 [26 43]] INFO [2019-12-17 12:35:18,491] jepsen worker 27 - jepsen.util 8847 :fail :cas [9 [42 24]] :ValueNotSwapFrom INFO [2019-12-17 12:35:18,491] jepsen worker 27 - jepsen.util 8847 :invoke :cas [9 [41 46]] INFO [2019-12-17 12:35:18,501] jepsen worker 27 - jepsen.util 8847 :fail :cas [9 [41 46]] :ValueNotSwapFrom INFO [2019-12-17 12:35:18,501] jepsen worker 27 - jepsen.util 8847 :invoke :write [9 8] INFO [2019-12-17 12:35:18,504] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53859,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:35:18,504] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53859,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:35:18,504] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53862,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:35:18,504] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53862,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:35:18,504] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53863,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:35:18,504] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53863,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:35:18,516] jepsen worker 31 - jepsen.util 8311 :fail :read [10 nil] :RequestTimeoutException INFO [2019-12-17 12:35:18,516] jepsen worker 31 - jepsen.util 8311 :invoke :cas [10 [6 26]] INFO [2019-12-17 12:35:18,530] jepsen worker 17 - jepsen.util 12527 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:35:18,530] jepsen worker 17 - jepsen.util 12527 :invoke :read [5 nil] INFO [2019-12-17 12:35:18,549] jepsen worker 27 - jepsen.util 8847 :ok :write [9 8] INFO [2019-12-17 12:35:18,549] jepsen worker 27 - jepsen.util 8847 :invoke :read [9 nil] INFO [2019-12-17 12:35:18,560] jepsen worker 27 - jepsen.util 8847 :ok :read [9 8] INFO [2019-12-17 12:35:18,560] jepsen worker 27 - jepsen.util 8847 :invoke :read [9 nil] INFO [2019-12-17 12:35:18,573] jepsen worker 27 - jepsen.util 8847 :ok :read [9 8] INFO [2019-12-17 12:35:18,573] jepsen worker 27 - jepsen.util 8847 :invoke :cas [9 [33 37]] INFO [2019-12-17 12:35:18,588] jepsen worker 12 - jepsen.util 9642 :fail :cas [4 [17 9]] :ValueNotSwapFrom INFO [2019-12-17 12:35:18,588] jepsen worker 12 - jepsen.util 9642 :invoke :read [4 nil] INFO [2019-12-17 12:35:18,588] jepsen worker 27 - jepsen.util 8847 :fail :cas [9 [33 37]] :ValueNotSwapFrom INFO [2019-12-17 12:35:18,589] jepsen worker 27 - jepsen.util 8847 :invoke :write [9 19] INFO [2019-12-17 12:35:18,596] jepsen worker 37 - jepsen.util 8677 :info :write [12 28] :RequestCanceledException INFO [2019-12-17 12:35:18,597] jepsen worker 37 - jepsen.util 8767 :invoke :write [12 5] INFO [2019-12-17 12:35:18,620] jepsen worker 12 - jepsen.util 9642 :ok :read [4 13] INFO [2019-12-17 12:35:18,620] jepsen worker 12 - jepsen.util 9642 :invoke :read [4 nil] INFO [2019-12-17 12:35:18,624] jepsen worker 27 - jepsen.util 8847 :ok :write [9 19] INFO [2019-12-17 12:35:18,624] jepsen worker 27 - jepsen.util 8847 :invoke :read [9 nil] INFO [2019-12-17 12:35:18,638] jepsen worker 27 - jepsen.util 8847 :ok :read [9 19] INFO [2019-12-17 12:35:18,639] jepsen worker 27 - jepsen.util 8847 :invoke :write [9 32] INFO [2019-12-17 12:35:18,644] jepsen worker 60 - jepsen.util 13740 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:35:18,644] jepsen worker 60 - jepsen.util 13740 :invoke :write [20 49] INFO [2019-12-17 12:35:18,662] jepsen worker 42 - jepsen.util 13632 :info :write [14 30] :RequestCanceledException INFO [2019-12-17 12:35:18,663] jepsen worker 42 - jepsen.util 13722 :invoke :cas [14 [28 29]] INFO [2019-12-17 12:35:18,665] jepsen worker 19 - jepsen.util 9289 :ok :write [6 11] INFO [2019-12-17 12:35:18,665] jepsen worker 19 - jepsen.util 9289 :invoke :cas [6 [3 11]] INFO [2019-12-17 12:35:18,665] jepsen worker 72 - jepsen.util 9162 :ok :write [24 10] INFO [2019-12-17 12:35:18,665] jepsen worker 72 - jepsen.util 9162 :invoke :read [24 nil] INFO [2019-12-17 12:35:18,665] jepsen worker 46 - jepsen.util 9226 :ok :write [15 27] INFO [2019-12-17 12:35:18,665] jepsen worker 46 - jepsen.util 9226 :invoke :read [15 nil] INFO [2019-12-17 12:35:18,665] jepsen worker 30 - jepsen.util 9300 :ok :write [10 35] INFO [2019-12-17 12:35:18,665] jepsen worker 30 - jepsen.util 9300 :invoke :write [10 17] INFO [2019-12-17 12:35:18,687] jepsen worker 19 - jepsen.util 9289 :fail :cas [6 [3 11]] :ValueNotSwapFrom INFO [2019-12-17 12:35:18,688] jepsen worker 12 - jepsen.util 9642 :ok :read [4 13] INFO [2019-12-17 12:35:18,688] jepsen worker 19 - jepsen.util 9289 :invoke :write [6 30] INFO [2019-12-17 12:35:18,688] jepsen worker 12 - jepsen.util 9642 :invoke :read [4 nil] INFO [2019-12-17 12:35:18,697] jepsen worker 43 - jepsen.util 12463 :info :write [14 13] :RequestCanceledException INFO [2019-12-17 12:35:18,697] jepsen worker 57 - jepsen.util 12387 :info :cas [19 [2 24]] :RequestCanceledException INFO [2019-12-17 12:35:18,698] jepsen worker 57 - jepsen.util 12477 :invoke :read [19 nil] INFO [2019-12-17 12:35:18,698] jepsen worker 43 - jepsen.util 12553 :invoke :write [14 31] INFO [2019-12-17 12:35:18,705] jepsen worker 71 - jepsen.util 9881 :ok :read [23 44] INFO [2019-12-17 12:35:18,705] jepsen worker 71 - jepsen.util 9881 :invoke :read [23 nil] INFO [2019-12-17 12:35:18,707] jepsen worker 27 - jepsen.util 8847 :ok :write [9 32] INFO [2019-12-17 12:35:18,707] jepsen worker 27 - jepsen.util 8847 :invoke :read [9 nil] INFO [2019-12-17 12:35:18,721] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53882,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:35:18,721] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53882,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:35:18,721] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53886,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:35:18,721] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53886,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:35:18,731] jepsen worker 6 - jepsen.util 13236 :info :cas [2 [13 46]] :RequestCanceledException INFO [2019-12-17 12:35:18,731] jepsen worker 76 - jepsen.util 12946 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:35:18,731] jepsen worker 76 - jepsen.util 12946 :invoke :write [25 32] INFO [2019-12-17 12:35:18,732] jepsen worker 6 - jepsen.util 13326 :invoke :read [2 nil] INFO [2019-12-17 12:35:18,743] jepsen worker 5 - jepsen.util 8555 :info :write [1 37] :RequestCanceledException INFO [2019-12-17 12:35:18,744] jepsen worker 5 - jepsen.util 8645 :invoke :write [1 31] INFO [2019-12-17 12:35:18,798] jepsen worker 31 - jepsen.util 8311 :fail :cas [10 [6 26]] :ValueNotSwapFrom INFO [2019-12-17 12:35:18,798] jepsen worker 31 - jepsen.util 8311 :invoke :write [10 12] INFO [2019-12-17 12:35:18,805] jepsen worker 72 - jepsen.util 9162 :ok :read [24 10] INFO [2019-12-17 12:35:18,805] jepsen worker 72 - jepsen.util 9162 :invoke :read [24 nil] INFO [2019-12-17 12:35:18,806] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:35:18,806] jepsen nemesis - couchbase.util Rebalance check count: 421 INFO [2019-12-17 12:35:18,806] jepsen nemesis - couchbase.util Rebalance stuck count: 402 INFO [2019-12-17 12:35:18,828] jepsen worker 56 - jepsen.util 9776 :ok :read [18 44] INFO [2019-12-17 12:35:18,828] jepsen worker 56 - jepsen.util 9776 :invoke :read [18 nil] INFO [2019-12-17 12:35:18,829] jepsen worker 38 - jepsen.util 9758 :info :write [12 21] :RequestCanceledException INFO [2019-12-17 12:35:18,829] jepsen worker 38 - jepsen.util 9848 :invoke :read [12 nil] INFO [2019-12-17 12:35:18,845] jepsen worker 44 - jepsen.util 13004 :info :write [14 10] :RequestCanceledException INFO [2019-12-17 12:35:18,846] jepsen worker 44 - jepsen.util 13094 :invoke :cas [14 [11 14]] INFO [2019-12-17 12:35:18,865] jepsen worker 27 - jepsen.util 8847 :ok :read [9 32] INFO [2019-12-17 12:35:18,865] jepsen worker 27 - jepsen.util 8847 :invoke :read [9 nil] INFO [2019-12-17 12:35:18,885] jepsen worker 72 - jepsen.util 9162 :ok :read [24 10] INFO [2019-12-17 12:35:18,885] jepsen worker 72 - jepsen.util 9162 :invoke :write [24 25] INFO [2019-12-17 12:35:18,885] jepsen worker 46 - jepsen.util 9226 :ok :read [15 27] INFO [2019-12-17 12:35:18,885] jepsen worker 46 - jepsen.util 9226 :invoke :write [15 48] INFO [2019-12-17 12:35:18,898] jepsen worker 24 - jepsen.util 11634 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:35:18,898] jepsen worker 24 - jepsen.util 11634 :invoke :write [8 13] INFO [2019-12-17 12:35:18,913] jepsen worker 32 - jepsen.util 9302 :ok :write [10 37] INFO [2019-12-17 12:35:18,913] jepsen worker 32 - jepsen.util 9302 :invoke :cas [10 [23 16]] INFO [2019-12-17 12:35:18,920] jepsen worker 79 - jepsen.util 9529 :fail :cas [26 [35 26]] :ValueNotSwapFrom INFO [2019-12-17 12:35:18,920] jepsen worker 27 - jepsen.util 8847 :ok :read [9 32] INFO [2019-12-17 12:35:18,920] jepsen worker 79 - jepsen.util 9529 :invoke :read [26 nil] INFO [2019-12-17 12:35:18,921] jepsen worker 27 - jepsen.util 8847 :invoke :cas [9 [0 12]] INFO [2019-12-17 12:35:18,930] jepsen worker 35 - jepsen.util 12725 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:35:18,930] jepsen worker 29 - jepsen.util 7589 :info :write [9 10] :RequestCanceledException INFO [2019-12-17 12:35:18,930] jepsen worker 35 - jepsen.util 12725 :invoke :write [11 19] INFO [2019-12-17 12:35:18,931] jepsen worker 56 - jepsen.util 9776 :ok :read [18 44] INFO [2019-12-17 12:35:18,931] jepsen worker 56 - jepsen.util 9776 :invoke :cas [18 [5 16]] INFO [2019-12-17 12:35:18,931] jepsen worker 29 - jepsen.util 7679 :invoke :cas [9 [49 25]] INFO [2019-12-17 12:35:18,937] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53909,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:35:18,937] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53909,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:35:18,938] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53912,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:35:18,938] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53912,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:35:18,955] jepsen worker 29 - jepsen.util 7679 :fail :cas [9 [49 25]] :ValueNotSwapFrom INFO [2019-12-17 12:35:18,955] jepsen worker 29 - jepsen.util 7679 :invoke :read [9 nil] INFO [2019-12-17 12:35:18,961] jepsen worker 27 - jepsen.util 8847 :fail :cas [9 [0 12]] :ValueNotSwapFrom INFO [2019-12-17 12:35:18,961] jepsen worker 27 - jepsen.util 8847 :invoke :cas [9 [44 44]] INFO [2019-12-17 12:35:18,961] jepsen worker 62 - jepsen.util 12932 :info :cas [20 [8 35]] :RequestCanceledException INFO [2019-12-17 12:35:18,961] jepsen worker 14 - jepsen.util 9104 :info :write [4 42] :RequestCanceledException INFO [2019-12-17 12:35:18,962] jepsen worker 14 - jepsen.util 9194 :invoke :read [4 nil] INFO [2019-12-17 12:35:18,962] jepsen worker 62 - jepsen.util 13022 :invoke :read [20 nil] INFO [2019-12-17 12:35:18,973] jepsen worker 29 - jepsen.util 7679 :ok :read [9 32] INFO [2019-12-17 12:35:18,973] jepsen worker 29 - jepsen.util 7679 :invoke :read [9 nil] INFO [2019-12-17 12:35:19,009] jepsen worker 29 - jepsen.util 7679 :ok :read [9 32] INFO [2019-12-17 12:35:19,009] jepsen worker 29 - jepsen.util 7679 :invoke :read [9 nil] INFO [2019-12-17 12:35:19,010] jepsen worker 27 - jepsen.util 8847 :fail :cas [9 [44 44]] :ValueNotSwapFrom INFO [2019-12-17 12:35:19,011] jepsen worker 27 - jepsen.util 8847 :invoke :read [9 nil] INFO [2019-12-17 12:35:19,047] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53929,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:35:19,047] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53929,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:35:19,047] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53932,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:35:19,047] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53932,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:35:19,047] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53933,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:35:19,047] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53933,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:35:19,059] jepsen worker 29 - jepsen.util 7679 :ok :read [9 32] INFO [2019-12-17 12:35:19,059] jepsen worker 29 - jepsen.util 7679 :invoke :write [9 47] INFO [2019-12-17 12:35:19,060] jepsen worker 56 - jepsen.util 9776 :fail :cas [18 [5 16]] :ValueNotSwapFrom INFO [2019-12-17 12:35:19,060] jepsen worker 56 - jepsen.util 9776 :invoke :read [18 nil] INFO [2019-12-17 12:35:19,061] jepsen worker 72 - jepsen.util 9162 :ok :write [24 25] INFO [2019-12-17 12:35:19,061] jepsen worker 19 - jepsen.util 9289 :ok :write [6 30] INFO [2019-12-17 12:35:19,061] jepsen worker 72 - jepsen.util 9162 :invoke :read [24 nil] INFO [2019-12-17 12:35:19,061] jepsen worker 19 - jepsen.util 9289 :invoke :read [6 nil] INFO [2019-12-17 12:35:19,062] jepsen worker 27 - jepsen.util 8847 :ok :read [9 32] INFO [2019-12-17 12:35:19,062] jepsen worker 27 - jepsen.util 8847 :invoke :write [9 38] INFO [2019-12-17 12:35:19,089] jepsen worker 14 - jepsen.util 9194 :ok :read [4 13] INFO [2019-12-17 12:35:19,089] jepsen worker 14 - jepsen.util 9194 :invoke :write [4 29] INFO [2019-12-17 12:35:19,097] jepsen worker 23 - jepsen.util 12623 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:35:19,097] jepsen worker 23 - jepsen.util 12623 :invoke :cas [7 [27 47]] INFO [2019-12-17 12:35:19,115] jepsen worker 51 - jepsen.util 8421 :info :write [17 24] :RequestCanceledException INFO [2019-12-17 12:35:19,116] jepsen worker 51 - jepsen.util 8511 :invoke :write [17 23] INFO [2019-12-17 12:35:19,130] jepsen worker 85 - jepsen.util 13585 :info :cas [28 [21 18]] :RequestCanceledException INFO [2019-12-17 12:35:19,130] jepsen worker 53 - jepsen.util 8513 :info :write [17 30] :RequestCanceledException INFO [2019-12-17 12:35:19,130] jepsen worker 73 - jepsen.util 8713 :fail :read [24 nil] :RequestTimeoutException INFO [2019-12-17 12:35:19,131] jepsen worker 73 - jepsen.util 8713 :invoke :write [24 20] INFO [2019-12-17 12:35:19,131] jepsen worker 53 - jepsen.util 8603 :invoke :write [17 13] INFO [2019-12-17 12:35:19,131] jepsen worker 85 - jepsen.util 13675 :invoke :read [28 nil] INFO [2019-12-17 12:35:19,142] jepsen worker 30 - jepsen.util 9300 :ok :write [10 17] INFO [2019-12-17 12:35:19,142] jepsen worker 30 - jepsen.util 9300 :invoke :cas [10 [3 26]] INFO [2019-12-17 12:35:19,152] jepsen worker 72 - jepsen.util 9162 :ok :read [24 25] INFO [2019-12-17 12:35:19,152] jepsen worker 72 - jepsen.util 9162 :invoke :cas [24 [3 46]] INFO [2019-12-17 12:35:19,157] jepsen worker 19 - jepsen.util 9289 :ok :read [6 30] INFO [2019-12-17 12:35:19,157] jepsen worker 19 - jepsen.util 9289 :invoke :write [6 22] INFO [2019-12-17 12:35:19,159] jepsen worker 56 - jepsen.util 9776 :ok :read [18 44] INFO [2019-12-17 12:35:19,159] jepsen worker 56 - jepsen.util 9776 :invoke :cas [18 [1 23]] INFO [2019-12-17 12:35:19,161] jepsen worker 28 - jepsen.util 7498 :info :write [9 38] :RequestCanceledException INFO [2019-12-17 12:35:19,161] jepsen worker 83 - jepsen.util 13223 :info :write [27 38] :RequestCanceledException INFO [2019-12-17 12:35:19,162] jepsen worker 28 - jepsen.util 7588 :invoke :read [9 nil] INFO [2019-12-17 12:35:19,162] jepsen worker 83 - jepsen.util 13313 :invoke :cas [27 [7 29]] INFO [2019-12-17 12:35:19,175] jepsen worker 72 - jepsen.util 9162 :fail :cas [24 [3 46]] :ValueNotSwapFrom INFO [2019-12-17 12:35:19,175] jepsen worker 72 - jepsen.util 9162 :invoke :read [24 nil] INFO [2019-12-17 12:35:19,197] jepsen worker 80 - jepsen.util 8900 :info :write [26 7] :RequestCanceledException INFO [2019-12-17 12:35:19,198] jepsen worker 80 - jepsen.util 8990 :invoke :cas [26 [8 36]] INFO [2019-12-17 12:35:19,211] jepsen worker 56 - jepsen.util 9776 :fail :cas [18 [1 23]] :ValueNotSwapFrom INFO [2019-12-17 12:35:19,211] jepsen worker 56 - jepsen.util 9776 :invoke :write [18 30] INFO [2019-12-17 12:35:19,213] jepsen worker 72 - jepsen.util 9162 :ok :read [24 25] INFO [2019-12-17 12:35:19,213] jepsen worker 72 - jepsen.util 9162 :invoke :read [24 nil] INFO [2019-12-17 12:35:19,227] jepsen worker 72 - jepsen.util 9162 :ok :read [24 25] INFO [2019-12-17 12:35:19,227] jepsen worker 72 - jepsen.util 9162 :invoke :cas [24 [46 2]] INFO [2019-12-17 12:35:19,230] jepsen worker 36 - jepsen.util 9756 :info :write [12 21] :RequestCanceledException INFO [2019-12-17 12:35:19,230] jepsen worker 20 - jepsen.util 8750 :info :cas [6 [32 5]] :RequestCanceledException INFO [2019-12-17 12:35:19,231] jepsen worker 20 - jepsen.util 8840 :invoke :cas [6 [12 47]] INFO [2019-12-17 12:35:19,231] jepsen worker 36 - jepsen.util 9846 :invoke :write [12 35] INFO [2019-12-17 12:35:19,261] jepsen worker 72 - jepsen.util 9162 :fail :cas [24 [46 2]] :ValueNotSwapFrom INFO [2019-12-17 12:35:19,261] jepsen worker 72 - jepsen.util 9162 :invoke :cas [24 [39 14]] INFO [2019-12-17 12:35:19,267] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53965,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:35:19,267] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53965,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:35:19,267] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53968,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:35:19,267] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53955,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:35:19,267] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53968,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:35:19,267] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53955,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:35:19,271] jepsen worker 20 - jepsen.util 8840 :fail :cas [6 [12 47]] :ValueNotSwapFrom INFO [2019-12-17 12:35:19,271] jepsen worker 20 - jepsen.util 8840 :invoke :cas [6 [4 46]] INFO [2019-12-17 12:35:19,273] jepsen worker 79 - jepsen.util 9529 :ok :read [26 8] INFO [2019-12-17 12:35:19,273] jepsen worker 79 - jepsen.util 9529 :invoke :cas [26 [17 17]] INFO [2019-12-17 12:35:19,277] jepsen worker 29 - jepsen.util 7679 :ok :write [9 47] INFO [2019-12-17 12:35:19,277] jepsen worker 29 - jepsen.util 7679 :invoke :read [9 nil] INFO [2019-12-17 12:35:19,279] jepsen worker 72 - jepsen.util 9162 :fail :cas [24 [39 14]] :ValueNotSwapFrom INFO [2019-12-17 12:35:19,279] jepsen worker 72 - jepsen.util 9162 :invoke :read [24 nil] INFO [2019-12-17 12:35:19,284] jepsen worker 20 - jepsen.util 8840 :fail :cas [6 [4 46]] :ValueNotSwapFrom INFO [2019-12-17 12:35:19,284] jepsen worker 20 - jepsen.util 8840 :invoke :cas [6 [45 9]] INFO [2019-12-17 12:35:19,295] jepsen worker 29 - jepsen.util 7679 :ok :read [9 47] INFO [2019-12-17 12:35:19,295] jepsen worker 29 - jepsen.util 7679 :invoke :cas [9 [8 9]] INFO [2019-12-17 12:35:19,297] jepsen worker 40 - jepsen.util 13000 :info :write [13 44] :RequestCanceledException INFO [2019-12-17 12:35:19,298] jepsen worker 40 - jepsen.util 13090 :invoke :read [13 nil] INFO [2019-12-17 12:35:19,299] jepsen worker 38 - jepsen.util 9848 :ok :read [12 4] INFO [2019-12-17 12:35:19,299] jepsen worker 38 - jepsen.util 9848 :invoke :read [12 nil] INFO [2019-12-17 12:35:19,315] jepsen worker 10 - jepsen.util 9280 :info :write [3 45] :RequestCanceledException INFO [2019-12-17 12:35:19,316] jepsen worker 10 - jepsen.util 9370 :invoke :write [3 18] INFO [2019-12-17 12:35:19,331] jepsen worker 9 - jepsen.util 9819 :info :cas [3 [23 8]] :RequestCanceledException INFO [2019-12-17 12:35:19,332] jepsen worker 9 - jepsen.util 9909 :invoke :read [3 nil] INFO [2019-12-17 12:35:19,337] jepsen worker 29 - jepsen.util 7679 :fail :cas [9 [8 9]] :ValueNotSwapFrom INFO [2019-12-17 12:35:19,337] jepsen worker 29 - jepsen.util 7679 :invoke :read [9 nil] INFO [2019-12-17 12:35:19,344] jepsen worker 66 - jepsen.util 12486 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:35:19,344] jepsen worker 66 - jepsen.util 12486 :invoke :cas [22 [46 39]] INFO [2019-12-17 12:35:19,354] jepsen worker 72 - jepsen.util 9162 :ok :read [24 25] INFO [2019-12-17 12:35:19,354] jepsen worker 72 - jepsen.util 9162 :invoke :cas [24 [20 9]] INFO [2019-12-17 12:35:19,355] jepsen worker 27 - jepsen.util 8847 :ok :write [9 38] INFO [2019-12-17 12:35:19,355] jepsen worker 27 - jepsen.util 8847 :invoke :write [9 13] INFO [2019-12-17 12:35:19,411] jepsen worker 20 - jepsen.util 8840 :fail :cas [6 [45 9]] :ValueNotSwapFrom INFO [2019-12-17 12:35:19,411] jepsen worker 20 - jepsen.util 8840 :invoke :read [6 nil] INFO [2019-12-17 12:35:19,412] jepsen worker 72 - jepsen.util 9162 :fail :cas [24 [20 9]] :ValueNotSwapFrom INFO [2019-12-17 12:35:19,412] jepsen worker 72 - jepsen.util 9162 :invoke :write [24 35] INFO [2019-12-17 12:35:19,430] jepsen worker 63 - jepsen.util 8163 :fail :read [21 nil] :RequestTimeoutException INFO [2019-12-17 12:35:19,430] jepsen worker 20 - jepsen.util 8840 :ok :read [6 30] INFO [2019-12-17 12:35:19,431] jepsen worker 63 - jepsen.util 8163 :invoke :cas [21 [15 48]] INFO [2019-12-17 12:35:19,431] jepsen worker 20 - jepsen.util 8840 :invoke :read [6 nil] INFO [2019-12-17 12:35:19,445] jepsen worker 45 - jepsen.util 9585 :info :write [15 1] :RequestCanceledException INFO [2019-12-17 12:35:19,445] jepsen worker 11 - jepsen.util 9371 :info :cas [3 [11 37]] :RequestCanceledException INFO [2019-12-17 12:35:19,446] jepsen worker 45 - jepsen.util 9675 :invoke :cas [15 [12 46]] INFO [2019-12-17 12:35:19,447] jepsen worker 11 - jepsen.util 9461 :invoke :write [3 27] INFO [2019-12-17 12:35:19,461] jepsen worker 26 - jepsen.util 13346 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:35:19,462] jepsen worker 26 - jepsen.util 13346 :invoke :read [8 nil] INFO [2019-12-17 12:35:19,485] jepsen worker 19 - jepsen.util 9289 :ok :write [6 22] INFO [2019-12-17 12:35:19,485] jepsen worker 19 - jepsen.util 9289 :invoke :cas [6 [30 9]] INFO [2019-12-17 12:35:19,486] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53992,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:35:19,486] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53992,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:35:19,486] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53995,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:35:19,486] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":53995,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:35:19,489] jepsen worker 73 - jepsen.util 8713 :ok :write [24 20] INFO [2019-12-17 12:35:19,489] jepsen worker 73 - jepsen.util 8713 :invoke :write [24 47] INFO [2019-12-17 12:35:19,497] jepsen worker 20 - jepsen.util 8840 :ok :read [6 30] INFO [2019-12-17 12:35:19,497] jepsen worker 49 - jepsen.util 12199 :info :cas [16 [22 21]] :RequestCanceledException INFO [2019-12-17 12:35:19,497] jepsen worker 20 - jepsen.util 8840 :invoke :read [6 nil] INFO [2019-12-17 12:35:19,498] jepsen worker 49 - jepsen.util 12289 :invoke :write [16 41] INFO [2019-12-17 12:35:19,502] jepsen worker 10 - jepsen.util 9370 :ok :write [3 18] INFO [2019-12-17 12:35:19,502] jepsen worker 10 - jepsen.util 9370 :invoke :cas [3 [22 32]] INFO [2019-12-17 12:35:19,514] jepsen worker 68 - jepsen.util 12128 :info :write [22 46] :RequestCanceledException INFO [2019-12-17 12:35:19,515] jepsen worker 68 - jepsen.util 12218 :invoke :write [22 0] INFO [2019-12-17 12:35:19,529] jepsen worker 55 - jepsen.util 9955 :info :write [18 9] :RequestCanceledException INFO [2019-12-17 12:35:19,529] jepsen worker 0 - jepsen.util 13410 :info :cas [0 [0 17]] :RequestCanceledException INFO [2019-12-17 12:35:19,531] jepsen worker 7 - jepsen.util 12787 :info :cas [2 [23 31]] :RequestCanceledException INFO [2019-12-17 12:35:19,531] jepsen worker 69 - jepsen.util 9249 :info :write [23 6] :RequestCanceledException INFO [2019-12-17 12:35:19,531] jepsen worker 0 - jepsen.util 13500 :invoke :read [0 nil] INFO [2019-12-17 12:35:19,531] jepsen worker 55 - jepsen.util 10045 :invoke :write [18 27] INFO [2019-12-17 12:35:19,532] jepsen worker 69 - jepsen.util 9339 :invoke :read [23 nil] INFO [2019-12-17 12:35:19,532] jepsen worker 7 - jepsen.util 12877 :invoke :read [2 nil] INFO [2019-12-17 12:35:19,544] jepsen worker 25 - jepsen.util 13795 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:35:19,544] jepsen worker 25 - jepsen.util 13795 :invoke :write [8 11] INFO [2019-12-17 12:35:19,555] jepsen worker 9 - jepsen.util 9909 :ok :read [3 20] INFO [2019-12-17 12:35:19,555] jepsen worker 9 - jepsen.util 9909 :invoke :read [3 nil] INFO [2019-12-17 12:35:19,561] jepsen worker 82 - jepsen.util 13402 :info :write [27 34] :RequestCanceledException INFO [2019-12-17 12:35:19,561] jepsen worker 1 - jepsen.util 12961 :info :cas [0 [12 3]] :RequestCanceledException INFO [2019-12-17 12:35:19,561] jepsen worker 61 - jepsen.util 13651 :info :cas [20 [9 2]] :RequestCanceledException INFO [2019-12-17 12:35:19,563] jepsen worker 61 - jepsen.util 13741 :invoke :write [20 38] INFO [2019-12-17 12:35:19,563] jepsen worker 82 - jepsen.util 13492 :invoke :cas [27 [1 24]] INFO [2019-12-17 12:35:19,563] jepsen worker 1 - jepsen.util 13051 :invoke :read [0 nil] INFO [2019-12-17 12:35:19,572] jepsen worker 71 - jepsen.util 9881 :ok :read [23 44] INFO [2019-12-17 12:35:19,572] jepsen worker 71 - jepsen.util 9881 :invoke :read [23 nil] INFO [2019-12-17 12:35:19,593] jepsen worker 19 - jepsen.util 9289 :fail :cas [6 [30 9]] :ValueNotSwapFrom INFO [2019-12-17 12:35:19,593] jepsen worker 19 - jepsen.util 9289 :invoke :write [6 9] INFO [2019-12-17 12:35:19,598] jepsen worker 13 - jepsen.util 8833 :fail :cas [4 [26 43]] :ValueNotSwapFrom INFO [2019-12-17 12:35:19,598] jepsen worker 13 - jepsen.util 8833 :invoke :write [4 43] INFO [2019-12-17 12:35:19,630] jepsen worker 10 - jepsen.util 9370 :fail :cas [3 [22 32]] :ValueNotSwapFrom INFO [2019-12-17 12:35:19,630] jepsen worker 10 - jepsen.util 9370 :invoke :read [3 nil] INFO [2019-12-17 12:35:19,630] jepsen worker 64 - jepsen.util 7264 :info :write [21 23] :RequestCanceledException INFO [2019-12-17 12:35:19,630] jepsen worker 75 - jepsen.util 12135 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:35:19,630] jepsen worker 75 - jepsen.util 12135 :invoke :write [25 47] INFO [2019-12-17 12:35:19,631] jepsen worker 64 - jepsen.util 7354 :invoke :read [21 nil] INFO [2019-12-17 12:35:19,645] jepsen worker 58 - jepsen.util 13378 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:35:19,645] jepsen worker 63 - jepsen.util 8163 :fail :cas [21 [15 48]] :ValueNotSwapFrom INFO [2019-12-17 12:35:19,645] jepsen worker 58 - jepsen.util 13378 :invoke :read [19 nil] INFO [2019-12-17 12:35:19,645] jepsen worker 63 - jepsen.util 8163 :invoke :cas [21 [12 23]] INFO [2019-12-17 12:35:19,675] jepsen worker 27 - jepsen.util 8847 :ok :write [9 13] INFO [2019-12-17 12:35:19,675] jepsen worker 27 - jepsen.util 8847 :invoke :cas [9 [41 4]] INFO [2019-12-17 12:35:19,678] jepsen worker 10 - jepsen.util 9370 :ok :read [3 18] INFO [2019-12-17 12:35:19,678] jepsen worker 10 - jepsen.util 9370 :invoke :read [3 nil] INFO [2019-12-17 12:35:19,699] jepsen worker 11 - jepsen.util 9461 :ok :write [3 27] INFO [2019-12-17 12:35:19,700] jepsen worker 11 - jepsen.util 9461 :invoke :write [3 7] INFO [2019-12-17 12:35:19,705] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54028,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:35:19,706] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54028,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:35:19,706] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54029,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:35:19,706] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54029,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:35:19,707] jepsen worker 27 - jepsen.util 8847 :fail :cas [9 [41 4]] :ValueNotSwapFrom INFO [2019-12-17 12:35:19,707] jepsen worker 27 - jepsen.util 8847 :invoke :read [9 nil] INFO [2019-12-17 12:35:19,712] jepsen worker 78 - jepsen.util 9168 :ok :write [26 1] INFO [2019-12-17 12:35:19,712] jepsen worker 78 - jepsen.util 9168 :invoke :write [26 39] INFO [2019-12-17 12:35:19,731] jepsen worker 65 - jepsen.util 7805 :info :cas [21 [24 16]] :RequestCanceledException INFO [2019-12-17 12:35:19,731] jepsen worker 81 - jepsen.util 12501 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:35:19,731] jepsen worker 48 - jepsen.util 11748 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:35:19,731] jepsen worker 70 - jepsen.util 9880 :info :write [23 43] :RequestCanceledException INFO [2019-12-17 12:35:19,731] jepsen worker 81 - jepsen.util 12501 :invoke :write [27 5] INFO [2019-12-17 12:35:19,731] jepsen worker 48 - jepsen.util 11748 :invoke :cas [16 [38 15]] INFO [2019-12-17 12:35:19,731] jepsen worker 74 - jepsen.util 9074 :fail :read [24 nil] :RequestTimeoutException INFO [2019-12-17 12:35:19,732] jepsen worker 74 - jepsen.util 9074 :invoke :write [24 28] INFO [2019-12-17 12:35:19,732] jepsen worker 65 - jepsen.util 7895 :invoke :write [21 46] INFO [2019-12-17 12:35:19,732] jepsen worker 70 - jepsen.util 9970 :invoke :write [23 16] INFO [2019-12-17 12:35:19,742] jepsen worker 88 - jepsen.util 9628 :info :write [29 27] :RequestCanceledException INFO [2019-12-17 12:35:19,744] jepsen worker 88 - jepsen.util 9718 :invoke :read [29 nil] INFO [2019-12-17 12:35:19,758] jepsen worker 10 - jepsen.util 9370 :ok :read [3 18] INFO [2019-12-17 12:35:19,758] jepsen worker 10 - jepsen.util 9370 :invoke :read [3 nil] INFO [2019-12-17 12:35:19,790] jepsen worker 20 - jepsen.util 8840 :ok :read [6 22] INFO [2019-12-17 12:35:19,790] jepsen worker 20 - jepsen.util 8840 :invoke :cas [6 [43 6]] INFO [2019-12-17 12:35:19,797] jepsen worker 5 - jepsen.util 8645 :ok :write [1 31] INFO [2019-12-17 12:35:19,797] jepsen worker 5 - jepsen.util 8645 :invoke :cas [1 [49 13]] INFO [2019-12-17 12:35:19,798] jepsen worker 39 - jepsen.util 12279 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:35:19,798] jepsen worker 18 - jepsen.util 9648 :info :cas [6 [24 22]] :RequestCanceledException INFO [2019-12-17 12:35:19,798] jepsen worker 39 - jepsen.util 12279 :invoke :cas [13 [43 43]] INFO [2019-12-17 12:35:19,802] jepsen worker 18 - jepsen.util 9738 :invoke :write [6 5] INFO [2019-12-17 12:35:19,815] jepsen worker 3 - jepsen.util 9003 :info :write [1 39] :RequestCanceledException INFO [2019-12-17 12:35:19,818] jepsen worker 3 - jepsen.util 9093 :invoke :read [1 nil] INFO [2019-12-17 12:35:19,829] jepsen worker 47 - jepsen.util 9677 :info :cas [15 [2 17]] :RequestCanceledException INFO [2019-12-17 12:35:19,830] jepsen worker 47 - jepsen.util 9767 :invoke :cas [15 [18 44]] INFO [2019-12-17 12:35:19,836] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:35:19,836] jepsen nemesis - couchbase.util Rebalance check count: 422 INFO [2019-12-17 12:35:19,836] jepsen nemesis - couchbase.util Rebalance stuck count: 403 INFO [2019-12-17 12:35:19,850] jepsen worker 27 - jepsen.util 8847 :ok :read [9 13] INFO [2019-12-17 12:35:19,850] jepsen worker 27 - jepsen.util 8847 :invoke :cas [9 [9 25]] INFO [2019-12-17 12:35:19,861] jepsen worker 69 - jepsen.util 9339 :ok :read [23 44] INFO [2019-12-17 12:35:19,861] jepsen worker 69 - jepsen.util 9339 :invoke :cas [23 [43 36]] INFO [2019-12-17 12:35:19,869] jepsen worker 10 - jepsen.util 9370 :ok :read [3 27] INFO [2019-12-17 12:35:19,869] jepsen worker 10 - jepsen.util 9370 :invoke :cas [3 [39 5]] INFO [2019-12-17 12:35:19,915] jepsen worker 4 - jepsen.util 8734 :info :write [1 17] :RequestCanceledException INFO [2019-12-17 12:35:19,917] jepsen worker 4 - jepsen.util 8824 :invoke :write [1 44] INFO [2019-12-17 12:35:19,918] jepsen worker 10 - jepsen.util 9370 :fail :cas [3 [39 5]] :ValueNotSwapFrom INFO [2019-12-17 12:35:19,918] jepsen worker 10 - jepsen.util 9370 :invoke :write [3 10] INFO [2019-12-17 12:35:19,924] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54049,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:35:19,925] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54049,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:35:19,925] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54053,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:35:19,925] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54053,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:35:19,925] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54054,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:35:19,925] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54054,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:35:19,930] jepsen worker 33 - jepsen.util 12723 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:35:19,931] jepsen worker 33 - jepsen.util 12723 :invoke :write [11 9] INFO [2019-12-17 12:35:19,954] jepsen worker 80 - jepsen.util 8990 :fail :cas [26 [8 36]] :CASMismatchException INFO [2019-12-17 12:35:19,954] jepsen worker 80 - jepsen.util 8990 :invoke :read [26 nil] INFO [2019-12-17 12:35:19,954] jepsen worker 37 - jepsen.util 8767 :ok :write [12 5] INFO [2019-12-17 12:35:19,954] jepsen worker 65 - jepsen.util 7895 :ok :write [21 46] INFO [2019-12-17 12:35:19,954] jepsen worker 37 - jepsen.util 8767 :invoke :cas [12 [21 8]] INFO [2019-12-17 12:35:19,954] jepsen worker 65 - jepsen.util 7895 :invoke :cas [21 [19 27]] INFO [2019-12-17 12:35:19,957] jepsen worker 52 - jepsen.util 9322 :ok :cas [17 [35 43]] INFO [2019-12-17 12:35:19,957] jepsen worker 52 - jepsen.util 9322 :invoke :write [17 35] INFO [2019-12-17 12:35:19,991] jepsen worker 65 - jepsen.util 7895 :fail :cas [21 [19 27]] :ValueNotSwapFrom INFO [2019-12-17 12:35:19,991] jepsen worker 65 - jepsen.util 7895 :invoke :cas [21 [30 3]] INFO [2019-12-17 12:35:20,008] jepsen worker 65 - jepsen.util 7895 :fail :cas [21 [30 3]] :ValueNotSwapFrom INFO [2019-12-17 12:35:20,008] jepsen worker 65 - jepsen.util 7895 :invoke :cas [21 [47 46]] INFO [2019-12-17 12:35:20,010] jepsen worker 51 - jepsen.util 8511 :ok :write [17 23] INFO [2019-12-17 12:35:20,010] jepsen worker 51 - jepsen.util 8511 :invoke :read [17 nil] INFO [2019-12-17 12:35:20,022] jepsen worker 65 - jepsen.util 7895 :fail :cas [21 [47 46]] :ValueNotSwapFrom INFO [2019-12-17 12:35:20,022] jepsen worker 65 - jepsen.util 7895 :invoke :read [21 nil] INFO [2019-12-17 12:35:20,026] jepsen worker 89 - jepsen.util 9269 :ok :cas [29 [36 38]] INFO [2019-12-17 12:35:20,026] jepsen worker 89 - jepsen.util 9269 :invoke :read [29 nil] INFO [2019-12-17 12:35:20,034] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54066,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:35:20,035] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54066,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:35:20,035] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54069,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:35:20,035] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54069,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:35:20,046] jepsen worker 71 - jepsen.util 9881 :ok :read [23 44] INFO [2019-12-17 12:35:20,046] jepsen worker 71 - jepsen.util 9881 :invoke :write [23 23] INFO [2019-12-17 12:35:20,052] jepsen worker 65 - jepsen.util 7895 :ok :read [21 46] INFO [2019-12-17 12:35:20,053] jepsen worker 65 - jepsen.util 7895 :invoke :cas [21 [32 39]] INFO [2019-12-17 12:35:20,064] jepsen worker 69 - jepsen.util 9339 :fail :cas [23 [43 36]] :ValueNotSwapFrom INFO [2019-12-17 12:35:20,064] jepsen worker 69 - jepsen.util 9339 :invoke :write [23 20] INFO [2019-12-17 12:35:20,082] jepsen worker 73 - jepsen.util 8713 :ok :write [24 47] INFO [2019-12-17 12:35:20,083] jepsen worker 73 - jepsen.util 8713 :invoke :write [24 32] INFO [2019-12-17 12:35:20,097] jepsen worker 87 - jepsen.util 8907 :info :write [29 41] :RequestCanceledException INFO [2019-12-17 12:35:20,098] jepsen worker 87 - jepsen.util 8997 :invoke :cas [29 [48 9]] INFO [2019-12-17 12:35:20,099] jepsen worker 65 - jepsen.util 7895 :fail :cas [21 [32 39]] :ValueNotSwapFrom INFO [2019-12-17 12:35:20,099] jepsen worker 65 - jepsen.util 7895 :invoke :read [21 nil] INFO [2019-12-17 12:35:20,100] jepsen worker 19 - jepsen.util 9289 :ok :write [6 9] INFO [2019-12-17 12:35:20,101] jepsen worker 19 - jepsen.util 9289 :invoke :cas [6 [43 4]] INFO [2019-12-17 12:35:20,133] jepsen worker 64 - jepsen.util 7354 :ok :read [21 46] INFO [2019-12-17 12:35:20,133] jepsen worker 64 - jepsen.util 7354 :invoke :write [21 17] INFO [2019-12-17 12:35:20,139] jepsen worker 11 - jepsen.util 9461 :ok :write [3 7] INFO [2019-12-17 12:35:20,139] jepsen worker 11 - jepsen.util 9461 :invoke :read [3 nil] INFO [2019-12-17 12:35:20,173] jepsen worker 78 - jepsen.util 9168 :ok :write [26 39] INFO [2019-12-17 12:35:20,173] jepsen worker 78 - jepsen.util 9168 :invoke :write [26 38] INFO [2019-12-17 12:35:20,184] jepsen worker 56 - jepsen.util 9776 :ok :write [18 30] INFO [2019-12-17 12:35:20,185] jepsen worker 56 - jepsen.util 9776 :invoke :write [18 27] INFO [2019-12-17 12:35:20,206] jepsen worker 13 - jepsen.util 8833 :ok :write [4 43] INFO [2019-12-17 12:35:20,206] jepsen worker 13 - jepsen.util 8833 :invoke :cas [4 [24 4]] INFO [2019-12-17 12:35:20,212] jepsen worker 63 - jepsen.util 8163 :fail :cas [21 [12 23]] :ValueNotSwapFrom INFO [2019-12-17 12:35:20,212] jepsen worker 63 - jepsen.util 8163 :invoke :cas [21 [31 17]] INFO [2019-12-17 12:35:20,231] jepsen worker 4 - jepsen.util 8824 :ok :write [1 44] INFO [2019-12-17 12:35:20,232] jepsen worker 4 - jepsen.util 8824 :invoke :write [1 17] INFO [2019-12-17 12:35:20,250] jepsen worker 65 - jepsen.util 7895 :ok :read [21 46] INFO [2019-12-17 12:35:20,250] jepsen worker 65 - jepsen.util 7895 :invoke :read [21 nil] INFO [2019-12-17 12:35:20,254] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54094,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:35:20,254] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54086,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:35:20,254] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54094,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:35:20,255] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54086,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:35:20,255] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54096,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:35:20,255] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54087,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:35:20,255] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54096,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:35:20,255] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54087,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:35:20,255] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54088,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:35:20,255] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54088,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:35:20,258] jepsen worker 19 - jepsen.util 9289 :fail :cas [6 [43 4]] :ValueNotSwapFrom INFO [2019-12-17 12:35:20,258] jepsen worker 19 - jepsen.util 9289 :invoke :read [6 nil] INFO [2019-12-17 12:35:20,271] jepsen worker 11 - jepsen.util 9461 :ok :read [3 7] INFO [2019-12-17 12:35:20,271] jepsen worker 11 - jepsen.util 9461 :invoke :read [3 nil] INFO [2019-12-17 12:35:20,275] jepsen worker 3 - jepsen.util 9093 :ok :read [1 31] INFO [2019-12-17 12:35:20,275] jepsen worker 3 - jepsen.util 9093 :invoke :write [1 35] INFO [2019-12-17 12:35:20,289] jepsen worker 52 - jepsen.util 9322 :ok :write [17 35] INFO [2019-12-17 12:35:20,289] jepsen worker 52 - jepsen.util 9322 :invoke :write [17 2] INFO [2019-12-17 12:35:20,307] jepsen worker 19 - jepsen.util 9289 :ok :read [6 9] INFO [2019-12-17 12:35:20,308] jepsen worker 19 - jepsen.util 9289 :invoke :write [6 47] INFO [2019-12-17 12:35:20,310] jepsen worker 65 - jepsen.util 7895 :ok :read [21 46] INFO [2019-12-17 12:35:20,310] jepsen worker 65 - jepsen.util 7895 :invoke :read [21 nil] INFO [2019-12-17 12:35:20,319] jepsen worker 11 - jepsen.util 9461 :ok :read [3 7] INFO [2019-12-17 12:35:20,319] jepsen worker 11 - jepsen.util 9461 :invoke :write [3 17] INFO [2019-12-17 12:35:20,325] jepsen worker 14 - jepsen.util 9194 :ok :write [4 29] INFO [2019-12-17 12:35:20,326] jepsen worker 14 - jepsen.util 9194 :invoke :write [4 0] INFO [2019-12-17 12:35:20,326] jepsen worker 74 - jepsen.util 9074 :ok :write [24 28] INFO [2019-12-17 12:35:20,326] jepsen worker 74 - jepsen.util 9074 :invoke :read [24 nil] INFO [2019-12-17 12:35:20,353] jepsen worker 89 - jepsen.util 9269 :ok :read [29 38] INFO [2019-12-17 12:35:20,353] jepsen worker 89 - jepsen.util 9269 :invoke :write [29 16] INFO [2019-12-17 12:35:20,361] jepsen worker 65 - jepsen.util 7895 :ok :read [21 46] INFO [2019-12-17 12:35:20,361] jepsen worker 65 - jepsen.util 7895 :invoke :read [21 nil] INFO [2019-12-17 12:35:20,362] jepsen worker 22 - jepsen.util 12622 :info :cas [7 [33 44]] :RequestCanceledException INFO [2019-12-17 12:35:20,363] jepsen worker 22 - jepsen.util 12712 :invoke :cas [7 [1 48]] INFO [2019-12-17 12:35:20,384] jepsen worker 51 - jepsen.util 8511 :ok :read [17 35] INFO [2019-12-17 12:35:20,384] jepsen worker 51 - jepsen.util 8511 :invoke :write [17 47] INFO [2019-12-17 12:35:20,397] jepsen worker 41 - jepsen.util 13541 :info :cas [13 [44 17]] :RequestCanceledException INFO [2019-12-17 12:35:20,397] jepsen worker 74 - jepsen.util 9074 :ok :read [24 28] INFO [2019-12-17 12:35:20,397] jepsen worker 74 - jepsen.util 9074 :invoke :cas [24 [38 3]] INFO [2019-12-17 12:35:20,398] jepsen worker 41 - jepsen.util 13631 :invoke :cas [13 [19 39]] INFO [2019-12-17 12:35:20,412] jepsen worker 54 - jepsen.util 10314 :fail :cas [18 [44 1]] :CASMismatchException INFO [2019-12-17 12:35:20,412] jepsen worker 54 - jepsen.util 10314 :invoke :cas [18 [32 31]] INFO [2019-12-17 12:35:20,413] jepsen worker 59 - jepsen.util 12029 :info :cas [19 [8 41]] :RequestCanceledException INFO [2019-12-17 12:35:20,415] jepsen worker 59 - jepsen.util 12119 :invoke :cas [19 [14 5]] INFO [2019-12-17 12:35:20,428] jepsen worker 52 - jepsen.util 9322 :ok :write [17 2] INFO [2019-12-17 12:35:20,428] jepsen worker 52 - jepsen.util 9322 :invoke :cas [17 [36 24]] INFO [2019-12-17 12:35:20,430] jepsen worker 86 - jepsen.util 13676 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:35:20,430] jepsen worker 67 - jepsen.util 12847 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:35:20,430] jepsen worker 8 - jepsen.util 12338 :info :write [2 6] :RequestCanceledException INFO [2019-12-17 12:35:20,430] jepsen worker 77 - jepsen.util 13307 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:35:20,430] jepsen worker 86 - jepsen.util 13676 :invoke :read [28 nil] INFO [2019-12-17 12:35:20,430] jepsen worker 67 - jepsen.util 12847 :invoke :read [22 nil] INFO [2019-12-17 12:35:20,431] jepsen worker 77 - jepsen.util 13307 :invoke :cas [25 [44 20]] INFO [2019-12-17 12:35:20,435] jepsen worker 8 - jepsen.util 12428 :invoke :cas [2 [4 26]] INFO [2019-12-17 12:35:20,444] jepsen worker 34 - jepsen.util 12724 :info :cas [11 [9 9]] :RequestCanceledException INFO [2019-12-17 12:35:20,444] jepsen worker 50 - jepsen.util 13370 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:35:20,444] jepsen worker 21 - jepsen.util 12531 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:35:20,445] jepsen worker 21 - jepsen.util 12531 :invoke :cas [7 [12 49]] INFO [2019-12-17 12:35:20,445] jepsen worker 50 - jepsen.util 13370 :invoke :write [16 48] INFO [2019-12-17 12:35:20,449] jepsen worker 34 - jepsen.util 12814 :invoke :write [11 21] INFO [2019-12-17 12:35:20,461] jepsen worker 15 - jepsen.util 11805 :info :cas [5 [13 20]] :RequestCanceledException INFO [2019-12-17 12:35:20,466] jepsen worker 15 - jepsen.util 11895 :invoke :cas [5 [4 36]] INFO [2019-12-17 12:35:20,474] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54124,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:35:20,474] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54124,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:35:20,474] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54128,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:35:20,475] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54128,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:35:20,480] jepsen worker 65 - jepsen.util 7895 :ok :read [21 46] INFO [2019-12-17 12:35:20,480] jepsen worker 65 - jepsen.util 7895 :invoke :read [21 nil] INFO [2019-12-17 12:35:20,498] jepsen worker 2 - jepsen.util 12512 :info :cas [0 [43 26]] :RequestCanceledException INFO [2019-12-17 12:35:20,498] jepsen worker 16 - jepsen.util 13876 :info :cas [5 [0 27]] :RequestCanceledException INFO [2019-12-17 12:35:20,499] jepsen worker 52 - jepsen.util 9322 :fail :cas [17 [36 24]] :ValueNotSwapFrom INFO [2019-12-17 12:35:20,499] jepsen worker 52 - jepsen.util 9322 :invoke :cas [17 [27 0]] INFO [2019-12-17 12:35:20,503] jepsen worker 2 - jepsen.util 12602 :invoke :cas [0 [42 47]] INFO [2019-12-17 12:35:20,503] jepsen worker 16 - jepsen.util 13966 :invoke :cas [5 [17 13]] INFO [2019-12-17 12:35:20,507] jepsen worker 74 - jepsen.util 9074 :fail :cas [24 [38 3]] :ValueNotSwapFrom INFO [2019-12-17 12:35:20,508] jepsen worker 74 - jepsen.util 9074 :invoke :write [24 9] INFO [2019-12-17 12:35:20,514] jepsen worker 84 - jepsen.util 12324 :info :cas [28 [49 5]] :RequestCanceledException INFO [2019-12-17 12:35:20,521] jepsen worker 69 - jepsen.util 9339 :ok :write [23 20] INFO [2019-12-17 12:35:20,521] jepsen worker 19 - jepsen.util 9289 :ok :write [6 47] INFO [2019-12-17 12:35:20,521] jepsen worker 19 - jepsen.util 9289 :invoke :read [6 nil] INFO [2019-12-17 12:35:20,522] jepsen worker 84 - jepsen.util 12414 :invoke :cas [28 [16 36]] INFO [2019-12-17 12:35:20,522] jepsen worker 69 - jepsen.util 9339 :invoke :cas [23 [8 4]] INFO [2019-12-17 12:35:20,539] jepsen worker 65 - jepsen.util 7895 :ok :read [21 46] INFO [2019-12-17 12:35:20,540] jepsen worker 65 - jepsen.util 7895 :invoke :cas [21 [12 48]] INFO [2019-12-17 12:35:20,549] jepsen worker 5 - jepsen.util 8645 :fail :cas [1 [49 13]] :ValueNotSwapFrom INFO [2019-12-17 12:35:20,550] jepsen worker 5 - jepsen.util 8645 :invoke :write [1 40] INFO [2019-12-17 12:35:20,557] jepsen worker 55 - jepsen.util 10045 :ok :write [18 27] INFO [2019-12-17 12:35:20,557] jepsen worker 55 - jepsen.util 10045 :invoke :write [18 13] INFO [2019-12-17 12:35:20,562] jepsen worker 11 - jepsen.util 9461 :ok :write [3 17] INFO [2019-12-17 12:35:20,562] jepsen worker 11 - jepsen.util 9461 :invoke :cas [3 [41 2]] INFO [2019-12-17 12:35:20,574] jepsen worker 19 - jepsen.util 9289 :ok :read [6 47] INFO [2019-12-17 12:35:20,574] jepsen worker 19 - jepsen.util 9289 :invoke :read [6 nil] INFO [2019-12-17 12:35:20,612] jepsen worker 36 - jepsen.util 9846 :ok :write [12 35] INFO [2019-12-17 12:35:20,612] jepsen worker 36 - jepsen.util 9846 :invoke :write [12 34] INFO [2019-12-17 12:35:20,627] jepsen worker 65 - jepsen.util 7895 :fail :cas [21 [12 48]] :ValueNotSwapFrom INFO [2019-12-17 12:35:20,627] jepsen worker 65 - jepsen.util 7895 :invoke :read [21 nil] INFO [2019-12-17 12:35:20,631] jepsen worker 17 - jepsen.util 12527 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:35:20,631] jepsen worker 17 - jepsen.util 12527 :invoke :read [5 nil] INFO [2019-12-17 12:35:20,633] jepsen worker 19 - jepsen.util 9289 :ok :read [6 47] INFO [2019-12-17 12:35:20,633] jepsen worker 19 - jepsen.util 9289 :invoke :cas [6 [11 7]] INFO [2019-12-17 12:35:20,652] jepsen worker 89 - jepsen.util 9269 :ok :write [29 16] INFO [2019-12-17 12:35:20,652] jepsen worker 89 - jepsen.util 9269 :invoke :cas [29 [32 30]] INFO [2019-12-17 12:35:20,687] jepsen worker 4 - jepsen.util 8824 :ok :write [1 17] INFO [2019-12-17 12:35:20,687] jepsen worker 4 - jepsen.util 8824 :invoke :write [1 40] INFO [2019-12-17 12:35:20,695] jepsen worker 11 - jepsen.util 9461 :fail :cas [3 [41 2]] :ValueNotSwapFrom INFO [2019-12-17 12:35:20,695] jepsen worker 11 - jepsen.util 9461 :invoke :read [3 nil] INFO [2019-12-17 12:35:20,696] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54151,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:35:20,696] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54151,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:35:20,696] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54156,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:35:20,696] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54156,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:35:20,701] jepsen worker 79 - jepsen.util 9529 :fail :cas [26 [17 17]] :ValueNotSwapFrom INFO [2019-12-17 12:35:20,701] jepsen worker 79 - jepsen.util 9529 :invoke :read [26 nil] INFO [2019-12-17 12:35:20,734] jepsen worker 43 - jepsen.util 12553 :info :write [14 31] :RequestCanceledException INFO [2019-12-17 12:35:20,746] jepsen worker 60 - jepsen.util 13740 :info :write [20 49] :RequestCanceledException INFO [2019-12-17 12:35:20,746] jepsen worker 42 - jepsen.util 13722 :info :cas [14 [28 29]] :RequestCanceledException INFO [2019-12-17 12:35:20,746] jepsen worker 12 - jepsen.util 9642 :fail :read [4 nil] :RequestTimeoutException INFO [2019-12-17 12:35:20,764] jepsen worker 57 - jepsen.util 12477 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:35:20,772] jepsen worker 12 - jepsen.util 9642 :invoke :write [4 25] INFO [2019-12-17 12:35:20,773] jepsen worker 57 - jepsen.util 12477 :invoke :read [19 nil] INFO [2019-12-17 12:35:20,778] jepsen worker 71 - jepsen.util 9881 :ok :write [23 23] INFO [2019-12-17 12:35:20,778] jepsen worker 71 - jepsen.util 9881 :invoke :write [23 30] INFO [2019-12-17 12:35:20,780] jepsen worker 42 - jepsen.util 13812 :invoke :cas [14 [1 43]] INFO [2019-12-17 12:35:20,781] jepsen worker 43 - jepsen.util 12643 :invoke :cas [14 [9 37]] INFO [2019-12-17 12:35:20,783] jepsen worker 60 - jepsen.util 13830 :invoke :write [20 24] INFO [2019-12-17 12:35:20,797] jepsen worker 6 - jepsen.util 13326 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:35:20,797] jepsen worker 6 - jepsen.util 13326 :invoke :read [2 nil] INFO [2019-12-17 12:35:20,810] jepsen worker 19 - jepsen.util 9289 :fail :cas [6 [11 7]] :ValueNotSwapFrom INFO [2019-12-17 12:35:20,810] jepsen worker 19 - jepsen.util 9289 :invoke :write [6 30] INFO [2019-12-17 12:35:20,811] jepsen worker 65 - jepsen.util 7895 :ok :read [21 46] INFO [2019-12-17 12:35:20,811] jepsen worker 65 - jepsen.util 7895 :invoke :write [21 9] INFO [2019-12-17 12:35:20,815] jepsen worker 31 - jepsen.util 8311 :info :write [10 12] :RequestCanceledException INFO [2019-12-17 12:35:20,820] jepsen worker 31 - jepsen.util 8401 :invoke :read [10 nil] INFO [2019-12-17 12:35:20,827] jepsen worker 11 - jepsen.util 9461 :ok :read [3 17] INFO [2019-12-17 12:35:20,827] jepsen worker 11 - jepsen.util 9461 :invoke :write [3 46] INFO [2019-12-17 12:35:20,829] jepsen worker 76 - jepsen.util 12946 :info :write [25 32] :RequestCanceledException INFO [2019-12-17 12:35:20,841] jepsen worker 76 - jepsen.util 13036 :invoke :read [25 nil] INFO [2019-12-17 12:35:20,882] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:35:20,883] jepsen nemesis - couchbase.util Rebalance check count: 423 INFO [2019-12-17 12:35:20,883] jepsen nemesis - couchbase.util Rebalance stuck count: 404 INFO [2019-12-17 12:35:20,884] jepsen worker 38 - jepsen.util 9848 :ok :read [12 35] INFO [2019-12-17 12:35:20,885] jepsen worker 38 - jepsen.util 9848 :invoke :cas [12 [36 45]] INFO [2019-12-17 12:35:20,906] jepsen worker 52 - jepsen.util 9322 :fail :cas [17 [27 0]] :ValueNotSwapFrom INFO [2019-12-17 12:35:20,907] jepsen worker 52 - jepsen.util 9322 :invoke :cas [17 [24 10]] INFO [2019-12-17 12:35:20,915] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54177,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:35:20,916] jepsen worker 46 - jepsen.util 9226 :info :write [15 48] :RequestCanceledException INFO [2019-12-17 12:35:20,917] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54177,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:35:20,927] jepsen worker 46 - jepsen.util 9316 :invoke :write [15 8] INFO [2019-12-17 12:35:20,932] jepsen worker 32 - jepsen.util 9302 :info :cas [10 [23 16]] :RequestCanceledException INFO [2019-12-17 12:35:20,941] jepsen worker 87 - jepsen.util 8997 :fail :cas [29 [48 9]] :ValueNotSwapFrom INFO [2019-12-17 12:35:20,941] jepsen worker 87 - jepsen.util 8997 :invoke :read [29 nil] INFO [2019-12-17 12:35:20,941] jepsen worker 32 - jepsen.util 9392 :invoke :write [10 31] INFO [2019-12-17 12:35:20,945] jepsen worker 44 - jepsen.util 13094 :info :cas [14 [11 14]] :RequestCanceledException INFO [2019-12-17 12:35:20,947] jepsen worker 80 - jepsen.util 8990 :ok :read [26 39] INFO [2019-12-17 12:35:20,947] jepsen worker 80 - jepsen.util 8990 :invoke :cas [26 [0 27]] INFO [2019-12-17 12:35:20,952] jepsen worker 44 - jepsen.util 13184 :invoke :read [14 nil] INFO [2019-12-17 12:35:20,963] jepsen worker 31 - jepsen.util 8401 :ok :read [10 17] INFO [2019-12-17 12:35:20,963] jepsen worker 31 - jepsen.util 8401 :invoke :read [10 nil] INFO [2019-12-17 12:35:20,987] jepsen worker 89 - jepsen.util 9269 :fail :cas [29 [32 30]] :ValueNotSwapFrom INFO [2019-12-17 12:35:20,987] jepsen worker 89 - jepsen.util 9269 :invoke :read [29 nil] INFO [2019-12-17 12:35:20,989] jepsen worker 54 - jepsen.util 10314 :fail :cas [18 [32 31]] :ValueNotSwapFrom INFO [2019-12-17 12:35:20,989] jepsen worker 54 - jepsen.util 10314 :invoke :read [18 nil] INFO [2019-12-17 12:35:20,997] jepsen worker 62 - jepsen.util 13022 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:35:20,997] jepsen worker 24 - jepsen.util 11634 :info :write [8 13] :RequestCanceledException INFO [2019-12-17 12:35:20,997] jepsen worker 62 - jepsen.util 13022 :invoke :write [20 31] INFO [2019-12-17 12:35:21,001] jepsen worker 24 - jepsen.util 11724 :invoke :write [8 23] INFO [2019-12-17 12:35:21,003] jepsen worker 65 - jepsen.util 7895 :ok :write [21 9] INFO [2019-12-17 12:35:21,003] jepsen worker 65 - jepsen.util 7895 :invoke :cas [21 [45 17]] INFO [2019-12-17 12:35:21,024] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54190,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:35:21,024] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54190,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:35:21,024] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54194,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:35:21,024] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54194,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:35:21,031] jepsen worker 35 - jepsen.util 12725 :info :write [11 19] :RequestCanceledException INFO [2019-12-17 12:35:21,040] jepsen worker 35 - jepsen.util 12815 :invoke :read [11 nil] INFO [2019-12-17 12:35:21,070] jepsen worker 65 - jepsen.util 7895 :fail :cas [21 [45 17]] :ValueNotSwapFrom INFO [2019-12-17 12:35:21,070] jepsen worker 65 - jepsen.util 7895 :invoke :read [21 nil] INFO [2019-12-17 12:35:21,107] jepsen worker 79 - jepsen.util 9529 :ok :read [26 39] INFO [2019-12-17 12:35:21,107] jepsen worker 79 - jepsen.util 9529 :invoke :write [26 1] INFO [2019-12-17 12:35:21,119] jepsen worker 69 - jepsen.util 9339 :fail :cas [23 [8 4]] :ValueNotSwapFrom INFO [2019-12-17 12:35:21,119] jepsen worker 69 - jepsen.util 9339 :invoke :cas [23 [18 40]] INFO [2019-12-17 12:35:21,122] jepsen worker 65 - jepsen.util 7895 :ok :read [21 9] INFO [2019-12-17 12:35:21,123] jepsen worker 65 - jepsen.util 7895 :invoke :write [21 13] INFO [2019-12-17 12:35:21,144] jepsen worker 85 - jepsen.util 13675 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:35:21,145] jepsen worker 85 - jepsen.util 13675 :invoke :cas [28 [34 37]] INFO [2019-12-17 12:35:21,168] jepsen worker 88 - jepsen.util 9718 :ok :read [29 16] INFO [2019-12-17 12:35:21,168] jepsen worker 88 - jepsen.util 9718 :invoke :read [29 nil] INFO [2019-12-17 12:35:21,181] jepsen worker 74 - jepsen.util 9074 :ok :write [24 9] INFO [2019-12-17 12:35:21,181] jepsen worker 74 - jepsen.util 9074 :invoke :read [24 nil] INFO [2019-12-17 12:35:21,190] jepsen worker 65 - jepsen.util 7895 :ok :write [21 13] INFO [2019-12-17 12:35:21,190] jepsen worker 65 - jepsen.util 7895 :invoke :cas [21 [17 19]] INFO [2019-12-17 12:35:21,190] jepsen worker 19 - jepsen.util 9289 :ok :write [6 30] INFO [2019-12-17 12:35:21,190] jepsen worker 19 - jepsen.util 9289 :invoke :read [6 nil] INFO [2019-12-17 12:35:21,198] jepsen worker 23 - jepsen.util 12623 :info :cas [7 [27 47]] :RequestCanceledException INFO [2019-12-17 12:35:21,200] jepsen worker 11 - jepsen.util 9461 :ok :write [3 46] INFO [2019-12-17 12:35:21,201] jepsen worker 11 - jepsen.util 9461 :invoke :write [3 16] INFO [2019-12-17 12:35:21,205] jepsen worker 23 - jepsen.util 12713 :invoke :write [7 26] INFO [2019-12-17 12:35:21,220] jepsen worker 89 - jepsen.util 9269 :ok :read [29 16] INFO [2019-12-17 12:35:21,220] jepsen worker 89 - jepsen.util 9269 :invoke :cas [29 [0 46]] INFO [2019-12-17 12:35:21,231] jepsen worker 30 - jepsen.util 9300 :info :cas [10 [3 26]] :RequestCanceledException INFO [2019-12-17 12:35:21,231] jepsen worker 53 - jepsen.util 8603 :info :write [17 13] :RequestCanceledException INFO [2019-12-17 12:35:21,231] jepsen worker 83 - jepsen.util 13313 :info :cas [27 [7 29]] :RequestCanceledException INFO [2019-12-17 12:35:21,233] jepsen worker 46 - jepsen.util 9316 :ok :write [15 8] INFO [2019-12-17 12:35:21,234] jepsen worker 46 - jepsen.util 9316 :invoke :read [15 nil] INFO [2019-12-17 12:35:21,234] jepsen worker 52 - jepsen.util 9322 :fail :cas [17 [24 10]] :ValueNotSwapFrom INFO [2019-12-17 12:35:21,235] jepsen worker 52 - jepsen.util 9322 :invoke :write [17 11] INFO [2019-12-17 12:35:21,237] jepsen worker 30 - jepsen.util 9390 :invoke :cas [10 [28 43]] INFO [2019-12-17 12:35:21,238] jepsen worker 53 - jepsen.util 8693 :invoke :write [17 21] INFO [2019-12-17 12:35:21,242] jepsen worker 83 - jepsen.util 13403 :invoke :cas [27 [36 38]] INFO [2019-12-17 12:35:21,245] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54209,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:35:21,246] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54209,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:35:21,247] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54215,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:35:21,247] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54210,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:35:21,247] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54215,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:35:21,247] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54210,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:35:21,248] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54218,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:35:21,248] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54218,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:35:21,261] jepsen worker 28 - jepsen.util 7588 :fail :read [9 nil] :RequestTimeoutException INFO [2019-12-17 12:35:21,262] jepsen worker 28 - jepsen.util 7588 :invoke :write [9 8] INFO [2019-12-17 12:35:21,297] jepsen worker 37 - jepsen.util 8767 :fail :cas [12 [21 8]] :ValueNotSwapFrom INFO [2019-12-17 12:35:21,297] jepsen worker 37 - jepsen.util 8767 :invoke :read [12 nil] INFO [2019-12-17 12:35:21,305] jepsen worker 31 - jepsen.util 8401 :ok :read [10 17] INFO [2019-12-17 12:35:21,305] jepsen worker 31 - jepsen.util 8401 :invoke :write [10 33] INFO [2019-12-17 12:35:21,310] jepsen worker 19 - jepsen.util 9289 :ok :read [6 30] INFO [2019-12-17 12:35:21,310] jepsen worker 19 - jepsen.util 9289 :invoke :cas [6 [7 1]] INFO [2019-12-17 12:35:21,319] jepsen worker 65 - jepsen.util 7895 :fail :cas [21 [17 19]] :ValueNotSwapFrom INFO [2019-12-17 12:35:21,319] jepsen worker 65 - jepsen.util 7895 :invoke :cas [21 [3 20]] INFO [2019-12-17 12:35:21,344] jepsen worker 40 - jepsen.util 13090 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:35:21,344] jepsen worker 40 - jepsen.util 13090 :invoke :write [13 36] INFO [2019-12-17 12:35:21,350] jepsen worker 46 - jepsen.util 9316 :ok :read [15 8] INFO [2019-12-17 12:35:21,351] jepsen worker 46 - jepsen.util 9316 :invoke :cas [15 [43 37]] INFO [2019-12-17 12:35:21,363] jepsen worker 29 - jepsen.util 7679 :fail :read [9 nil] :RequestTimeoutException INFO [2019-12-17 12:35:21,363] jepsen worker 29 - jepsen.util 7679 :invoke :write [9 21] INFO [2019-12-17 12:35:21,373] jepsen worker 30 - jepsen.util 9390 :fail :cas [10 [28 43]] :ValueNotSwapFrom INFO [2019-12-17 12:35:21,373] jepsen worker 30 - jepsen.util 9390 :invoke :write [10 14] INFO [2019-12-17 12:35:21,388] jepsen worker 19 - jepsen.util 9289 :fail :cas [6 [7 1]] :ValueNotSwapFrom INFO [2019-12-17 12:35:21,388] jepsen worker 19 - jepsen.util 9289 :invoke :cas [6 [38 35]] INFO [2019-12-17 12:35:21,418] jepsen worker 65 - jepsen.util 7895 :fail :cas [21 [3 20]] :ValueNotSwapFrom INFO [2019-12-17 12:35:21,418] jepsen worker 65 - jepsen.util 7895 :invoke :read [21 nil] INFO [2019-12-17 12:35:21,444] jepsen worker 66 - jepsen.util 12486 :info :cas [22 [46 39]] :RequestCanceledException INFO [2019-12-17 12:35:21,444] jepsen worker 72 - jepsen.util 9162 :info :write [24 35] :RequestCanceledException INFO [2019-12-17 12:35:21,457] jepsen worker 89 - jepsen.util 9269 :fail :cas [29 [0 46]] :ValueNotSwapFrom INFO [2019-12-17 12:35:21,467] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54240,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:35:21,475] jepsen worker 89 - jepsen.util 9269 :invoke :read [29 nil] INFO [2019-12-17 12:35:21,478] jepsen worker 72 - jepsen.util 9252 :invoke :cas [24 [9 21]] INFO [2019-12-17 12:35:21,478] jepsen worker 66 - jepsen.util 12576 :invoke :read [22 nil] INFO [2019-12-17 12:35:21,478] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54240,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:35:21,478] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54243,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:35:21,478] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54243,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:35:21,488] jepsen worker 28 - jepsen.util 7588 :ok :write [9 8] INFO [2019-12-17 12:35:21,488] jepsen worker 28 - jepsen.util 7588 :invoke :read [9 nil] INFO [2019-12-17 12:35:21,496] jepsen worker 45 - jepsen.util 9675 :info :cas [15 [12 46]] :RequestCanceledException INFO [2019-12-17 12:35:21,501] jepsen worker 45 - jepsen.util 9765 :invoke :write [15 34] INFO [2019-12-17 12:35:21,504] jepsen worker 65 - jepsen.util 7895 :ok :read [21 13] INFO [2019-12-17 12:35:21,504] jepsen worker 65 - jepsen.util 7895 :invoke :cas [21 [39 7]] INFO [2019-12-17 12:35:21,510] jepsen worker 80 - jepsen.util 8990 :fail :cas [26 [0 27]] :ValueNotSwapFrom INFO [2019-12-17 12:35:21,512] jepsen worker 80 - jepsen.util 8990 :invoke :cas [26 [38 46]] INFO [2019-12-17 12:35:21,529] jepsen worker 49 - jepsen.util 12289 :info :write [16 41] :RequestCanceledException INFO [2019-12-17 12:35:21,532] jepsen worker 49 - jepsen.util 12379 :invoke :cas [16 [0 25]] INFO [2019-12-17 12:35:21,544] jepsen worker 68 - jepsen.util 12218 :info :write [22 0] :RequestCanceledException INFO [2019-12-17 12:35:21,547] jepsen worker 68 - jepsen.util 12308 :invoke :write [22 41] INFO [2019-12-17 12:35:21,550] jepsen worker 38 - jepsen.util 9848 :fail :cas [12 [36 45]] :ValueNotSwapFrom INFO [2019-12-17 12:35:21,551] jepsen worker 38 - jepsen.util 9848 :invoke :write [12 19] INFO [2019-12-17 12:35:21,552] jepsen worker 69 - jepsen.util 9339 :fail :cas [23 [18 40]] :ValueNotSwapFrom INFO [2019-12-17 12:35:21,553] jepsen worker 69 - jepsen.util 9339 :invoke :write [23 7] INFO [2019-12-17 12:35:21,562] jepsen worker 26 - jepsen.util 13346 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:35:21,562] jepsen worker 0 - jepsen.util 13500 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:35:21,563] jepsen worker 0 - jepsen.util 13500 :invoke :cas [0 [20 44]] INFO [2019-12-17 12:35:21,563] jepsen worker 26 - jepsen.util 13346 :invoke :cas [8 [14 14]] INFO [2019-12-17 12:35:21,566] jepsen worker 29 - jepsen.util 7679 :ok :write [9 21] INFO [2019-12-17 12:35:21,566] jepsen worker 29 - jepsen.util 7679 :invoke :cas [9 [39 16]] INFO [2019-12-17 12:35:21,589] jepsen worker 65 - jepsen.util 7895 :fail :cas [21 [39 7]] :ValueNotSwapFrom INFO [2019-12-17 12:35:21,590] jepsen worker 65 - jepsen.util 7895 :invoke :read [21 nil] INFO [2019-12-17 12:35:21,615] jepsen worker 7 - jepsen.util 12877 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:35:21,615] jepsen worker 7 - jepsen.util 12877 :invoke :read [2 nil] INFO [2019-12-17 12:35:21,629] jepsen worker 13 - jepsen.util 8833 :fail :cas [4 [24 4]] :ValueNotSwapFrom INFO [2019-12-17 12:35:21,629] jepsen worker 13 - jepsen.util 8833 :invoke :write [4 1] INFO [2019-12-17 12:35:21,630] jepsen worker 9 - jepsen.util 9909 :fail :read [3 nil] :RequestTimeoutException INFO [2019-12-17 12:35:21,630] jepsen worker 61 - jepsen.util 13741 :info :write [20 38] :RequestCanceledException INFO [2019-12-17 12:35:21,630] jepsen worker 9 - jepsen.util 9909 :invoke :cas [3 [27 34]] INFO [2019-12-17 12:35:21,637] jepsen worker 61 - jepsen.util 13831 :invoke :write [20 32] INFO [2019-12-17 12:35:21,638] jepsen worker 65 - jepsen.util 7895 :ok :read [21 13] INFO [2019-12-17 12:35:21,638] jepsen worker 65 - jepsen.util 7895 :invoke :read [21 nil] INFO [2019-12-17 12:35:21,644] jepsen worker 25 - jepsen.util 13795 :info :write [8 11] :RequestCanceledException INFO [2019-12-17 12:35:21,644] jepsen worker 82 - jepsen.util 13492 :info :cas [27 [1 24]] :RequestCanceledException INFO [2019-12-17 12:35:21,649] jepsen worker 25 - jepsen.util 13885 :invoke :cas [8 [48 6]] INFO [2019-12-17 12:35:21,649] jepsen worker 82 - jepsen.util 13582 :invoke :write [27 39] INFO [2019-12-17 12:35:21,661] jepsen worker 1 - jepsen.util 13051 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:35:21,662] jepsen worker 1 - jepsen.util 13051 :invoke :read [0 nil] INFO [2019-12-17 12:35:21,686] jepsen worker 65 - jepsen.util 7895 :ok :read [21 13] INFO [2019-12-17 12:35:21,686] jepsen worker 65 - jepsen.util 7895 :invoke :read [21 nil] INFO [2019-12-17 12:35:21,687] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54266,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:35:21,688] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54266,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:35:21,688] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54267,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:35:21,688] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54267,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:35:21,732] jepsen worker 75 - jepsen.util 12135 :info :write [25 47] :RequestCanceledException INFO [2019-12-17 12:35:21,739] jepsen worker 75 - jepsen.util 12225 :invoke :write [25 43] INFO [2019-12-17 12:35:21,744] jepsen worker 58 - jepsen.util 13378 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:35:21,744] jepsen worker 58 - jepsen.util 13378 :invoke :write [19 13] INFO [2019-12-17 12:35:21,791] jepsen worker 88 - jepsen.util 9718 :ok :read [29 16] INFO [2019-12-17 12:35:21,791] jepsen worker 88 - jepsen.util 9718 :invoke :read [29 nil] INFO [2019-12-17 12:35:21,809] jepsen worker 65 - jepsen.util 7895 :ok :read [21 13] INFO [2019-12-17 12:35:21,809] jepsen worker 65 - jepsen.util 7895 :invoke :read [21 nil] INFO [2019-12-17 12:35:21,815] jepsen worker 70 - jepsen.util 9970 :info :write [23 16] :RequestCanceledException INFO [2019-12-17 12:35:21,816] jepsen worker 20 - jepsen.util 8840 :info :cas [6 [43 6]] :RequestCanceledException INFO [2019-12-17 12:35:21,837] jepsen worker 48 - jepsen.util 11748 :info :cas [16 [38 15]] :RequestCanceledException INFO [2019-12-17 12:35:21,845] jepsen worker 81 - jepsen.util 12501 :info :write [27 5] :RequestCanceledException INFO [2019-12-17 12:35:21,853] jepsen worker 18 - jepsen.util 9738 :info :write [6 5] :RequestCanceledException INFO [2019-12-17 12:35:21,904] jepsen worker 39 - jepsen.util 12279 :info :cas [13 [43 43]] :RequestCanceledException INFO [2019-12-17 12:35:21,905] jepsen worker 27 - jepsen.util 8847 :info :cas [9 [9 25]] :RequestCanceledException INFO [2019-12-17 12:35:21,919] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54280,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:35:21,931] jepsen worker 47 - jepsen.util 9767 :info :cas [15 [18 44]] :RequestCanceledException INFO [2019-12-17 12:35:21,950] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54280,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:35:21,955] jepsen worker 70 - jepsen.util 10060 :invoke :cas [23 [21 45]] INFO [2019-12-17 12:35:21,973] jepsen worker 10 - jepsen.util 9370 :info :write [3 10] :RequestCanceledException INFO [2019-12-17 12:35:22,008] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:35:22,012] jepsen nemesis - couchbase.util Rebalance check count: 424 INFO [2019-12-17 12:35:22,016] jepsen worker 45 - jepsen.util 9765 :ok :write [15 34] INFO [2019-12-17 12:35:22,016] jepsen worker 11 - jepsen.util 9461 :ok :write [3 16] INFO [2019-12-17 12:35:22,016] jepsen worker 11 - jepsen.util 9461 :invoke :write [3 17] INFO [2019-12-17 12:35:22,017] jepsen worker 47 - jepsen.util 9857 :invoke :cas [15 [1 2]] INFO [2019-12-17 12:35:22,018] jepsen nemesis - couchbase.util Rebalance stuck count: 405 INFO [2019-12-17 12:35:22,018] jepsen worker 45 - jepsen.util 9765 :invoke :read [15 nil] INFO [2019-12-17 12:35:22,019] jepsen worker 32 - jepsen.util 9392 :ok :write [10 31] INFO [2019-12-17 12:35:22,027] jepsen worker 32 - jepsen.util 9392 :invoke :read [10 nil] INFO [2019-12-17 12:35:22,030] jepsen worker 20 - jepsen.util 8930 :invoke :write [6 16] INFO [2019-12-17 12:35:22,034] jepsen worker 33 - jepsen.util 12723 :info :write [11 9] :RequestCanceledException INFO [2019-12-17 12:35:22,038] jepsen worker 81 - jepsen.util 12591 :invoke :cas [27 [7 37]] INFO [2019-12-17 12:35:22,046] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54286,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:35:22,048] jepsen worker 18 - jepsen.util 9828 :invoke :read [6 nil] INFO [2019-12-17 12:35:22,049] jepsen worker 27 - jepsen.util 8937 :invoke :write [9 38] INFO [2019-12-17 12:35:22,062] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54286,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:35:22,086] jepsen worker 48 - jepsen.util 11838 :invoke :cas [16 [45 38]] INFO [2019-12-17 12:35:22,105] jepsen worker 10 - jepsen.util 9460 :invoke :read [3 nil] INFO [2019-12-17 12:35:22,111] jepsen worker 39 - jepsen.util 12369 :invoke :read [13 nil] INFO [2019-12-17 12:35:22,138] jepsen worker 33 - jepsen.util 12813 :invoke :write [11 42] INFO [2019-12-17 12:35:22,140] jepsen worker 73 - jepsen.util 8713 :info :write [24 32] :RequestCanceledException INFO [2019-12-17 12:35:22,207] jepsen worker 73 - jepsen.util 8803 :invoke :read [24 nil] INFO [2019-12-17 12:35:22,234] jepsen worker 64 - jepsen.util 7354 :info :write [21 17] :RequestCanceledException INFO [2019-12-17 12:35:22,234] jepsen worker 56 - jepsen.util 9776 :info :write [18 27] :RequestCanceledException INFO [2019-12-17 12:35:22,235] jepsen worker 63 - jepsen.util 8163 :info :cas [21 [31 17]] :RequestCanceledException INFO [2019-12-17 12:35:22,260] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54300,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:35:22,263] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54300,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:35:22,265] jepsen worker 78 - jepsen.util 9168 :info :write [26 38] :RequestCanceledException INFO [2019-12-17 12:35:22,265] jepsen worker 63 - jepsen.util 8253 :invoke :read [21 nil] INFO [2019-12-17 12:35:22,273] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54302,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:35:22,280] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54302,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:35:22,280] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54309,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:35:22,284] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54309,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:35:22,307] jepsen worker 64 - jepsen.util 7444 :invoke :cas [21 [44 11]] INFO [2019-12-17 12:35:22,311] jepsen worker 56 - jepsen.util 9866 :invoke :write [18 40] INFO [2019-12-17 12:35:22,321] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54305,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:35:22,324] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54305,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:35:22,343] jepsen worker 78 - jepsen.util 9258 :invoke :cas [26 [27 7]] INFO [2019-12-17 12:35:22,363] jepsen worker 3 - jepsen.util 9093 :info :write [1 35] :RequestCanceledException INFO [2019-12-17 12:35:22,372] jepsen worker 3 - jepsen.util 9183 :invoke :cas [1 [14 31]] INFO [2019-12-17 12:35:22,416] jepsen worker 51 - jepsen.util 8511 :info :write [17 47] :RequestCanceledException INFO [2019-12-17 12:35:22,429] jepsen worker 14 - jepsen.util 9194 :info :write [4 0] :RequestCanceledException INFO [2019-12-17 12:35:22,430] jepsen worker 22 - jepsen.util 12712 :info :cas [7 [1 48]] :RequestCanceledException INFO [2019-12-17 12:35:22,446] jepsen worker 41 - jepsen.util 13631 :info :cas [13 [19 39]] :RequestCanceledException INFO [2019-12-17 12:35:22,447] jepsen worker 51 - jepsen.util 8601 :invoke :read [17 nil] INFO [2019-12-17 12:35:22,449] jepsen worker 14 - jepsen.util 9284 :invoke :read [4 nil] INFO [2019-12-17 12:35:22,453] jepsen worker 22 - jepsen.util 12802 :invoke :cas [7 [9 22]] INFO [2019-12-17 12:35:22,458] jepsen worker 41 - jepsen.util 13721 :invoke :cas [13 [3 7]] INFO [2019-12-17 12:35:22,462] jepsen worker 59 - jepsen.util 12119 :info :cas [19 [14 5]] :RequestCanceledException INFO [2019-12-17 12:35:22,474] jepsen worker 59 - jepsen.util 12209 :invoke :read [19 nil] INFO [2019-12-17 12:35:22,494] jepsen worker 34 - jepsen.util 12814 :info :write [11 21] :RequestCanceledException INFO [2019-12-17 12:35:22,505] jepsen worker 34 - jepsen.util 12904 :invoke :write [11 41] INFO [2019-12-17 12:35:22,506] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54319,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:35:22,507] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54319,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:35:22,507] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54323,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:35:22,508] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54323,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:35:22,508] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54324,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:35:22,508] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54324,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:35:22,513] jepsen worker 15 - jepsen.util 11895 :info :cas [5 [4 36]] :RequestCanceledException INFO [2019-12-17 12:35:22,521] jepsen worker 15 - jepsen.util 11985 :invoke :cas [5 [32 34]] INFO [2019-12-17 12:35:22,525] jepsen worker 86 - jepsen.util 13676 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:35:22,525] jepsen worker 2 - jepsen.util 12602 :info :cas [0 [42 47]] :RequestCanceledException INFO [2019-12-17 12:35:22,525] jepsen worker 67 - jepsen.util 12847 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:35:22,525] jepsen worker 77 - jepsen.util 13307 :info :cas [25 [44 20]] :RequestCanceledException INFO [2019-12-17 12:35:22,525] jepsen worker 8 - jepsen.util 12428 :info :cas [2 [4 26]] :RequestCanceledException INFO [2019-12-17 12:35:22,525] jepsen worker 86 - jepsen.util 13676 :invoke :read [28 nil] INFO [2019-12-17 12:35:22,525] jepsen worker 67 - jepsen.util 12847 :invoke :read [22 nil] INFO [2019-12-17 12:35:22,534] jepsen worker 2 - jepsen.util 12692 :invoke :cas [0 [26 12]] INFO [2019-12-17 12:35:22,534] jepsen worker 77 - jepsen.util 13397 :invoke :cas [25 [1 46]] INFO [2019-12-17 12:35:22,535] jepsen worker 8 - jepsen.util 12518 :invoke :cas [2 [17 6]] INFO [2019-12-17 12:35:22,536] jepsen worker 18 - jepsen.util 9828 :ok :read [6 30] INFO [2019-12-17 12:35:22,536] jepsen worker 18 - jepsen.util 9828 :invoke :cas [6 [41 42]] INFO [2019-12-17 12:35:22,537] jepsen worker 37 - jepsen.util 8767 :ok :read [12 35] INFO [2019-12-17 12:35:22,537] jepsen worker 37 - jepsen.util 8767 :invoke :write [12 21] INFO [2019-12-17 12:35:22,541] jepsen worker 80 - jepsen.util 8990 :fail :cas [26 [38 46]] :ValueNotSwapFrom INFO [2019-12-17 12:35:22,542] jepsen worker 80 - jepsen.util 8990 :invoke :write [26 28] INFO [2019-12-17 12:35:22,542] jepsen worker 65 - jepsen.util 7895 :ok :read [21 13] INFO [2019-12-17 12:35:22,542] jepsen worker 65 - jepsen.util 7895 :invoke :read [21 nil] INFO [2019-12-17 12:35:22,543] jepsen worker 21 - jepsen.util 12531 :info :cas [7 [12 49]] :RequestCanceledException INFO [2019-12-17 12:35:22,543] jepsen worker 50 - jepsen.util 13370 :info :write [16 48] :RequestCanceledException INFO [2019-12-17 12:35:22,543] jepsen worker 16 - jepsen.util 13966 :info :cas [5 [17 13]] :RequestCanceledException INFO [2019-12-17 12:35:22,551] jepsen worker 16 - jepsen.util 14056 :invoke :read [5 nil] INFO [2019-12-17 12:35:22,552] jepsen worker 21 - jepsen.util 12621 :invoke :read [7 nil] INFO [2019-12-17 12:35:22,553] jepsen worker 50 - jepsen.util 13460 :invoke :write [16 43] INFO [2019-12-17 12:35:22,554] jepsen worker 47 - jepsen.util 9857 :fail :cas [15 [1 2]] :ValueNotSwapFrom INFO [2019-12-17 12:35:22,555] jepsen worker 47 - jepsen.util 9857 :invoke :write [15 49] INFO [2019-12-17 12:35:22,562] jepsen worker 84 - jepsen.util 12414 :info :cas [28 [16 36]] :RequestCanceledException INFO [2019-12-17 12:35:22,566] jepsen worker 54 - jepsen.util 10314 :ok :read [18 27] INFO [2019-12-17 12:35:22,567] jepsen worker 54 - jepsen.util 10314 :invoke :cas [18 [34 0]] INFO [2019-12-17 12:35:22,570] jepsen worker 84 - jepsen.util 12504 :invoke :read [28 nil] INFO [2019-12-17 12:35:22,570] jepsen worker 10 - jepsen.util 9460 :ok :read [3 16] INFO [2019-12-17 12:35:22,571] jepsen worker 10 - jepsen.util 9460 :invoke :cas [3 [35 34]] INFO [2019-12-17 12:35:22,574] jepsen worker 45 - jepsen.util 9765 :ok :read [15 34] INFO [2019-12-17 12:35:22,574] jepsen worker 45 - jepsen.util 9765 :invoke :read [15 nil] INFO [2019-12-17 12:35:22,576] jepsen worker 9 - jepsen.util 9909 :fail :cas [3 [27 34]] :ValueNotSwapFrom INFO [2019-12-17 12:35:22,576] jepsen worker 9 - jepsen.util 9909 :invoke :cas [3 [4 36]] INFO [2019-12-17 12:35:22,577] jepsen worker 32 - jepsen.util 9392 :ok :read [10 31] INFO [2019-12-17 12:35:22,577] jepsen worker 32 - jepsen.util 9392 :invoke :read [10 nil] INFO [2019-12-17 12:35:22,578] jepsen worker 89 - jepsen.util 9269 :ok :read [29 16] INFO [2019-12-17 12:35:22,578] jepsen worker 89 - jepsen.util 9269 :invoke :cas [29 [25 10]] INFO [2019-12-17 12:35:22,591] jepsen worker 63 - jepsen.util 8253 :ok :read [21 13] INFO [2019-12-17 12:35:22,592] jepsen worker 63 - jepsen.util 8253 :invoke :cas [21 [21 48]] INFO [2019-12-17 12:35:22,592] jepsen worker 73 - jepsen.util 8803 :ok :read [24 9] INFO [2019-12-17 12:35:22,592] jepsen worker 73 - jepsen.util 8803 :invoke :cas [24 [34 44]] INFO [2019-12-17 12:35:22,593] jepsen worker 64 - jepsen.util 7444 :fail :cas [21 [44 11]] :ValueNotSwapFrom INFO [2019-12-17 12:35:22,594] jepsen worker 64 - jepsen.util 7444 :invoke :write [21 26] INFO [2019-12-17 12:35:22,607] jepsen worker 5 - jepsen.util 8645 :info :write [1 40] :RequestCanceledException INFO [2019-12-17 12:35:22,611] jepsen worker 5 - jepsen.util 8735 :invoke :read [1 nil] INFO [2019-12-17 12:35:22,614] jepsen worker 87 - jepsen.util 8997 :ok :read [29 16] INFO [2019-12-17 12:35:22,615] jepsen worker 87 - jepsen.util 8997 :invoke :read [29 nil] INFO [2019-12-17 12:35:22,628] jepsen worker 55 - jepsen.util 10045 :info :write [18 13] :RequestCanceledException INFO [2019-12-17 12:35:22,632] jepsen worker 55 - jepsen.util 10135 :invoke :cas [18 [27 2]] INFO [2019-12-17 12:35:22,634] jepsen worker 88 - jepsen.util 9718 :ok :read [29 16] INFO [2019-12-17 12:35:22,634] jepsen worker 88 - jepsen.util 9718 :invoke :read [29 nil] INFO [2019-12-17 12:35:22,647] jepsen worker 10 - jepsen.util 9460 :fail :cas [3 [35 34]] :ValueNotSwapFrom INFO [2019-12-17 12:35:22,649] jepsen worker 10 - jepsen.util 9460 :invoke :read [3 nil] INFO [2019-12-17 12:35:22,754] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54362,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:35:22,754] jepsen worker 36 - jepsen.util 9846 :info :write [12 34] :RequestCanceledException INFO [2019-12-17 12:35:22,754] jepsen worker 17 - jepsen.util 12527 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:35:22,758] jepsen worker 9 - jepsen.util 9909 :fail :cas [3 [4 36]] :ValueNotSwapFrom INFO [2019-12-17 12:35:22,761] jepsen worker 70 - jepsen.util 10060 :fail :cas [23 [21 45]] :ValueNotSwapFrom INFO [2019-12-17 12:35:22,762] jepsen worker 18 - jepsen.util 9828 :fail :cas [6 [41 42]] :ValueNotSwapFrom INFO [2019-12-17 12:35:22,773] jepsen worker 73 - jepsen.util 8803 :fail :cas [24 [34 44]] :ValueNotSwapFrom INFO [2019-12-17 12:35:22,776] jepsen worker 32 - jepsen.util 9392 :ok :read [10 31] INFO [2019-12-17 12:35:22,784] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54362,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:35:22,784] jepsen worker 17 - jepsen.util 12527 :invoke :read [5 nil] INFO [2019-12-17 12:35:22,784] jepsen worker 70 - jepsen.util 10060 :invoke :write [23 34] INFO [2019-12-17 12:35:22,784] jepsen worker 9 - jepsen.util 9909 :invoke :read [3 nil] INFO [2019-12-17 12:35:22,784] jepsen worker 18 - jepsen.util 9828 :invoke :read [6 nil] INFO [2019-12-17 12:35:22,784] jepsen worker 73 - jepsen.util 8803 :invoke :write [24 49] INFO [2019-12-17 12:35:22,784] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54365,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:35:22,784] jepsen worker 32 - jepsen.util 9392 :invoke :write [10 48] INFO [2019-12-17 12:35:22,784] jepsen worker 36 - jepsen.util 9936 :invoke :read [12 nil] INFO [2019-12-17 12:35:22,784] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54365,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:35:22,785] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54366,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:35:22,785] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54366,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:35:22,797] jepsen worker 43 - jepsen.util 12643 :info :cas [14 [9 37]] :RequestCanceledException INFO [2019-12-17 12:35:22,797] jepsen worker 4 - jepsen.util 8824 :info :write [1 40] :RequestCanceledException INFO [2019-12-17 12:35:22,798] jepsen worker 4 - jepsen.util 8914 :invoke :write [1 15] INFO [2019-12-17 12:35:22,799] jepsen worker 43 - jepsen.util 12733 :invoke :read [14 nil] INFO [2019-12-17 12:35:22,810] jepsen worker 18 - jepsen.util 9828 :ok :read [6 30] INFO [2019-12-17 12:35:22,810] jepsen worker 18 - jepsen.util 9828 :invoke :cas [6 [43 17]] INFO [2019-12-17 12:35:22,830] jepsen worker 42 - jepsen.util 13812 :info :cas [14 [1 43]] :RequestCanceledException INFO [2019-12-17 12:35:22,831] jepsen worker 42 - jepsen.util 13902 :invoke :cas [14 [23 4]] INFO [2019-12-17 12:35:22,844] jepsen worker 12 - jepsen.util 9642 :info :write [4 25] :RequestCanceledException INFO [2019-12-17 12:35:22,845] jepsen worker 12 - jepsen.util 9732 :invoke :cas [4 [23 12]] INFO [2019-12-17 12:35:22,862] jepsen worker 57 - jepsen.util 12477 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:35:22,862] jepsen worker 57 - jepsen.util 12477 :invoke :cas [19 [19 35]] INFO [2019-12-17 12:35:22,897] jepsen worker 6 - jepsen.util 13326 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:35:22,897] jepsen worker 6 - jepsen.util 13326 :invoke :write [2 49] INFO [2019-12-17 12:35:22,915] jepsen worker 71 - jepsen.util 9881 :info :write [23 30] :RequestCanceledException INFO [2019-12-17 12:35:22,915] jepsen worker 60 - jepsen.util 13830 :info :write [20 24] :RequestCanceledException INFO [2019-12-17 12:35:22,916] jepsen worker 60 - jepsen.util 13920 :invoke :cas [20 [3 45]] INFO [2019-12-17 12:35:22,916] jepsen worker 71 - jepsen.util 9971 :invoke :read [23 nil] INFO [2019-12-17 12:35:22,924] jepsen worker 64 - jepsen.util 7444 :ok :write [21 26] INFO [2019-12-17 12:35:22,924] jepsen worker 64 - jepsen.util 7444 :invoke :write [21 39] INFO [2019-12-17 12:35:22,945] jepsen worker 76 - jepsen.util 13036 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:35:22,945] jepsen worker 76 - jepsen.util 13036 :invoke :write [25 6] INFO [2019-12-17 12:35:22,955] jepsen worker 27 - jepsen.util 8937 :ok :write [9 38] INFO [2019-12-17 12:35:22,955] jepsen worker 27 - jepsen.util 8937 :invoke :cas [9 [12 13]] INFO [2019-12-17 12:35:22,973] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54384,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:35:22,973] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54384,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:35:22,973] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54386,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:35:22,973] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54386,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:35:22,977] jepsen worker 27 - jepsen.util 8937 :fail :cas [9 [12 13]] :ValueNotSwapFrom INFO [2019-12-17 12:35:22,977] jepsen worker 27 - jepsen.util 8937 :invoke :read [9 nil] INFO [2019-12-17 12:35:22,986] jepsen worker 69 - jepsen.util 9339 :ok :write [23 7] INFO [2019-12-17 12:35:22,986] jepsen worker 69 - jepsen.util 9339 :invoke :cas [23 [5 21]] INFO [2019-12-17 12:35:22,996] jepsen worker 44 - jepsen.util 13184 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:35:22,996] jepsen worker 44 - jepsen.util 13184 :invoke :write [14 38] INFO [2019-12-17 12:35:23,006] jepsen worker 27 - jepsen.util 8937 :ok :read [9 38] INFO [2019-12-17 12:35:23,006] jepsen worker 27 - jepsen.util 8937 :invoke :cas [9 [25 0]] INFO [2019-12-17 12:35:23,019] jepsen worker 52 - jepsen.util 9322 :ok :write [17 11] INFO [2019-12-17 12:35:23,020] jepsen worker 52 - jepsen.util 9322 :invoke :read [17 nil] INFO [2019-12-17 12:35:23,035] jepsen worker 64 - jepsen.util 7444 :ok :write [21 39] INFO [2019-12-17 12:35:23,036] jepsen worker 64 - jepsen.util 7444 :invoke :cas [21 [42 17]] INFO [2019-12-17 12:35:23,043] jepsen worker 47 - jepsen.util 9857 :ok :write [15 49] INFO [2019-12-17 12:35:23,044] jepsen worker 47 - jepsen.util 9857 :invoke :cas [15 [47 28]] INFO [2019-12-17 12:35:23,049] jepsen worker 27 - jepsen.util 8937 :fail :cas [9 [25 0]] :ValueNotSwapFrom INFO [2019-12-17 12:35:23,050] jepsen worker 27 - jepsen.util 8937 :invoke :read [9 nil] INFO [2019-12-17 12:35:23,053] jepsen worker 14 - jepsen.util 9284 :ok :read [4 25] INFO [2019-12-17 12:35:23,053] jepsen worker 14 - jepsen.util 9284 :invoke :write [4 34] INFO [2019-12-17 12:35:23,063] jepsen worker 64 - jepsen.util 7444 :fail :cas [21 [42 17]] :ValueNotSwapFrom INFO [2019-12-17 12:35:23,063] jepsen worker 64 - jepsen.util 7444 :invoke :write [21 12] INFO [2019-12-17 12:35:23,068] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:35:23,068] jepsen nemesis - couchbase.util Rebalance check count: 425 INFO [2019-12-17 12:35:23,068] jepsen nemesis - couchbase.util Rebalance stuck count: 406 INFO [2019-12-17 12:35:23,078] jepsen worker 9 - jepsen.util 9909 :ok :read [3 16] INFO [2019-12-17 12:35:23,078] jepsen worker 9 - jepsen.util 9909 :invoke :cas [3 [41 20]] INFO [2019-12-17 12:35:23,083] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54396,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:35:23,083] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54396,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:35:23,083] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54399,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:35:23,083] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54399,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:35:23,083] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54400,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:35:23,083] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54400,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:35:23,086] jepsen worker 89 - jepsen.util 9269 :fail :cas [29 [25 10]] :ValueNotSwapFrom INFO [2019-12-17 12:35:23,087] jepsen worker 89 - jepsen.util 9269 :invoke :write [29 16] INFO [2019-12-17 12:35:23,094] jepsen worker 62 - jepsen.util 13022 :info :write [20 31] :RequestCanceledException INFO [2019-12-17 12:35:23,095] jepsen worker 62 - jepsen.util 13112 :invoke :cas [20 [18 24]] INFO [2019-12-17 12:35:23,097] jepsen worker 27 - jepsen.util 8937 :ok :read [9 38] INFO [2019-12-17 12:35:23,097] jepsen worker 27 - jepsen.util 8937 :invoke :cas [9 [44 20]] INFO [2019-12-17 12:35:23,110] jepsen worker 27 - jepsen.util 8937 :fail :cas [9 [44 20]] :ValueNotSwapFrom INFO [2019-12-17 12:35:23,110] jepsen worker 27 - jepsen.util 8937 :invoke :cas [9 [12 26]] INFO [2019-12-17 12:35:23,116] jepsen worker 24 - jepsen.util 11724 :info :write [8 23] :RequestCanceledException INFO [2019-12-17 12:35:23,116] jepsen worker 52 - jepsen.util 9322 :ok :read [17 11] INFO [2019-12-17 12:35:23,117] jepsen worker 52 - jepsen.util 9322 :invoke :read [17 nil] INFO [2019-12-17 12:35:23,117] jepsen worker 24 - jepsen.util 11814 :invoke :read [8 nil] INFO [2019-12-17 12:35:23,127] jepsen worker 27 - jepsen.util 8937 :fail :cas [9 [12 26]] :ValueNotSwapFrom INFO [2019-12-17 12:35:23,127] jepsen worker 27 - jepsen.util 8937 :invoke :cas [9 [17 35]] INFO [2019-12-17 12:35:23,131] jepsen worker 35 - jepsen.util 12815 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:35:23,131] jepsen worker 35 - jepsen.util 12815 :invoke :write [11 31] INFO [2019-12-17 12:35:23,139] jepsen worker 64 - jepsen.util 7444 :ok :write [21 12] INFO [2019-12-17 12:35:23,139] jepsen worker 64 - jepsen.util 7444 :invoke :cas [21 [1 27]] INFO [2019-12-17 12:35:23,140] jepsen worker 27 - jepsen.util 8937 :fail :cas [9 [17 35]] :ValueNotSwapFrom INFO [2019-12-17 12:35:23,140] jepsen worker 27 - jepsen.util 8937 :invoke :cas [9 [25 47]] INFO [2019-12-17 12:35:23,159] jepsen worker 3 - jepsen.util 9183 :fail :cas [1 [14 31]] :ValueNotSwapFrom INFO [2019-12-17 12:35:23,159] jepsen worker 3 - jepsen.util 9183 :invoke :read [1 nil] INFO [2019-12-17 12:35:23,178] jepsen worker 64 - jepsen.util 7444 :fail :cas [21 [1 27]] :ValueNotSwapFrom INFO [2019-12-17 12:35:23,178] jepsen worker 64 - jepsen.util 7444 :invoke :read [21 nil] INFO [2019-12-17 12:35:23,187] jepsen worker 27 - jepsen.util 8937 :fail :cas [9 [25 47]] :ValueNotSwapFrom INFO [2019-12-17 12:35:23,187] jepsen worker 27 - jepsen.util 8937 :invoke :cas [9 [26 30]] INFO [2019-12-17 12:35:23,188] jepsen worker 87 - jepsen.util 8997 :ok :read [29 16] INFO [2019-12-17 12:35:23,189] jepsen worker 87 - jepsen.util 8997 :invoke :read [29 nil] INFO [2019-12-17 12:35:23,203] jepsen worker 78 - jepsen.util 9258 :fail :cas [26 [27 7]] :ValueNotSwapFrom INFO [2019-12-17 12:35:23,204] jepsen worker 78 - jepsen.util 9258 :invoke :write [26 41] INFO [2019-12-17 12:35:23,213] jepsen worker 27 - jepsen.util 8937 :fail :cas [9 [26 30]] :ValueNotSwapFrom INFO [2019-12-17 12:35:23,214] jepsen worker 27 - jepsen.util 8937 :invoke :read [9 nil] INFO [2019-12-17 12:35:23,216] jepsen worker 64 - jepsen.util 7444 :ok :read [21 12] INFO [2019-12-17 12:35:23,217] jepsen worker 64 - jepsen.util 7444 :invoke :cas [21 [37 28]] INFO [2019-12-17 12:35:23,230] jepsen worker 74 - jepsen.util 9074 :fail :read [24 nil] :RequestTimeoutException INFO [2019-12-17 12:35:23,230] jepsen worker 79 - jepsen.util 9529 :info :write [26 1] :RequestCanceledException INFO [2019-12-17 12:35:23,231] jepsen worker 74 - jepsen.util 9074 :invoke :cas [24 [17 29]] INFO [2019-12-17 12:35:23,231] jepsen worker 79 - jepsen.util 9619 :invoke :write [26 22] INFO [2019-12-17 12:35:23,242] jepsen worker 23 - jepsen.util 12713 :info :write [7 26] :RequestCanceledException INFO [2019-12-17 12:35:23,242] jepsen worker 85 - jepsen.util 13675 :info :cas [28 [34 37]] :RequestCanceledException INFO [2019-12-17 12:35:23,243] jepsen worker 85 - jepsen.util 13765 :invoke :read [28 nil] INFO [2019-12-17 12:35:23,243] jepsen worker 23 - jepsen.util 12803 :invoke :write [7 40] INFO [2019-12-17 12:35:23,251] jepsen worker 27 - jepsen.util 8937 :ok :read [9 38] INFO [2019-12-17 12:35:23,251] jepsen worker 27 - jepsen.util 8937 :invoke :read [9 nil] INFO [2019-12-17 12:35:23,268] jepsen worker 27 - jepsen.util 8937 :ok :read [9 38] INFO [2019-12-17 12:35:23,268] jepsen worker 27 - jepsen.util 8937 :invoke :cas [9 [11 25]] INFO [2019-12-17 12:35:23,269] jepsen worker 64 - jepsen.util 7444 :fail :cas [21 [37 28]] :ValueNotSwapFrom INFO [2019-12-17 12:35:23,269] jepsen worker 64 - jepsen.util 7444 :invoke :cas [21 [6 37]] INFO [2019-12-17 12:35:23,279] jepsen worker 36 - jepsen.util 9936 :ok :read [12 35] INFO [2019-12-17 12:35:23,279] jepsen worker 36 - jepsen.util 9936 :invoke :cas [12 [41 10]] INFO [2019-12-17 12:35:23,290] jepsen worker 64 - jepsen.util 7444 :fail :cas [21 [6 37]] :ValueNotSwapFrom INFO [2019-12-17 12:35:23,290] jepsen worker 64 - jepsen.util 7444 :invoke :read [21 nil] INFO [2019-12-17 12:35:23,293] jepsen worker 73 - jepsen.util 8803 :ok :write [24 49] INFO [2019-12-17 12:35:23,293] jepsen worker 20 - jepsen.util 8930 :ok :write [6 16] INFO [2019-12-17 12:35:23,293] jepsen worker 73 - jepsen.util 8803 :invoke :write [24 29] INFO [2019-12-17 12:35:23,293] jepsen worker 20 - jepsen.util 8930 :invoke :cas [6 [28 33]] INFO [2019-12-17 12:35:23,298] jepsen worker 83 - jepsen.util 13403 :info :cas [27 [36 38]] :RequestCanceledException INFO [2019-12-17 12:35:23,299] jepsen worker 83 - jepsen.util 13493 :invoke :read [27 nil] INFO [2019-12-17 12:35:23,301] jepsen worker 71 - jepsen.util 9971 :ok :read [23 7] INFO [2019-12-17 12:35:23,301] jepsen worker 71 - jepsen.util 9971 :invoke :read [23 nil] INFO [2019-12-17 12:35:23,302] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54425,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:35:23,302] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54435,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:35:23,302] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54425,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:35:23,302] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54435,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:35:23,309] jepsen worker 13 - jepsen.util 8833 :ok :write [4 1] INFO [2019-12-17 12:35:23,309] jepsen worker 13 - jepsen.util 8833 :invoke :read [4 nil] INFO [2019-12-17 12:35:23,319] jepsen worker 27 - jepsen.util 8937 :fail :cas [9 [11 25]] :ValueNotSwapFrom INFO [2019-12-17 12:35:23,319] jepsen worker 27 - jepsen.util 8937 :invoke :read [9 nil] INFO [2019-12-17 12:35:23,330] jepsen worker 53 - jepsen.util 8693 :info :write [17 21] :RequestCanceledException INFO [2019-12-17 12:35:23,331] jepsen worker 53 - jepsen.util 8783 :invoke :cas [17 [43 2]] INFO [2019-12-17 12:35:23,361] jepsen worker 46 - jepsen.util 9316 :info :cas [15 [43 37]] :RequestCanceledException INFO [2019-12-17 12:35:23,362] jepsen worker 46 - jepsen.util 9406 :invoke :cas [15 [17 0]] INFO [2019-12-17 12:35:23,370] jepsen worker 3 - jepsen.util 9183 :ok :read [1 35] INFO [2019-12-17 12:35:23,371] jepsen worker 3 - jepsen.util 9183 :invoke :read [1 nil] INFO [2019-12-17 12:35:23,392] jepsen worker 64 - jepsen.util 7444 :ok :read [21 12] INFO [2019-12-17 12:35:23,392] jepsen worker 64 - jepsen.util 7444 :invoke :write [21 17] INFO [2019-12-17 12:35:23,398] jepsen worker 27 - jepsen.util 8937 :ok :read [9 38] INFO [2019-12-17 12:35:23,398] jepsen worker 27 - jepsen.util 8937 :invoke :write [9 12] INFO [2019-12-17 12:35:23,403] jepsen worker 20 - jepsen.util 8930 :fail :cas [6 [28 33]] :ValueNotSwapFrom INFO [2019-12-17 12:35:23,403] jepsen worker 20 - jepsen.util 8930 :invoke :cas [6 [16 45]] INFO [2019-12-17 12:35:23,420] jepsen worker 54 - jepsen.util 10314 :fail :cas [18 [34 0]] :ValueNotSwapFrom INFO [2019-12-17 12:35:23,420] jepsen worker 54 - jepsen.util 10314 :invoke :write [18 2] INFO [2019-12-17 12:35:23,430] jepsen worker 87 - jepsen.util 8997 :ok :read [29 16] INFO [2019-12-17 12:35:23,430] jepsen worker 87 - jepsen.util 8997 :invoke :cas [29 [29 44]] INFO [2019-12-17 12:35:23,431] jepsen worker 31 - jepsen.util 8401 :info :write [10 33] :RequestCanceledException INFO [2019-12-17 12:35:23,431] jepsen worker 19 - jepsen.util 9289 :info :cas [6 [38 35]] :RequestCanceledException INFO [2019-12-17 12:35:23,432] jepsen worker 31 - jepsen.util 8491 :invoke :read [10 nil] INFO [2019-12-17 12:35:23,432] jepsen worker 19 - jepsen.util 9379 :invoke :cas [6 [45 34]] INFO [2019-12-17 12:35:23,444] jepsen worker 40 - jepsen.util 13090 :info :write [13 36] :RequestCanceledException INFO [2019-12-17 12:35:23,445] jepsen worker 40 - jepsen.util 13180 :invoke :read [13 nil] INFO [2019-12-17 12:35:23,461] jepsen worker 30 - jepsen.util 9390 :info :write [10 14] :RequestCanceledException INFO [2019-12-17 12:35:23,463] jepsen worker 13 - jepsen.util 8833 :ok :read [4 1] INFO [2019-12-17 12:35:23,463] jepsen worker 13 - jepsen.util 8833 :invoke :write [4 24] INFO [2019-12-17 12:35:23,463] jepsen worker 30 - jepsen.util 9480 :invoke :read [10 nil] INFO [2019-12-17 12:35:23,489] jepsen worker 5 - jepsen.util 8735 :ok :read [1 35] INFO [2019-12-17 12:35:23,489] jepsen worker 5 - jepsen.util 8735 :invoke :write [1 13] INFO [2019-12-17 12:35:23,505] jepsen worker 30 - jepsen.util 9480 :ok :read [10 31] INFO [2019-12-17 12:35:23,505] jepsen worker 30 - jepsen.util 9480 :invoke :write [10 7] INFO [2019-12-17 12:35:23,521] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54463,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:35:23,522] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54463,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:35:23,522] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54464,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:35:23,522] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54464,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:35:23,526] jepsen worker 69 - jepsen.util 9339 :fail :cas [23 [5 21]] :ValueNotSwapFrom INFO [2019-12-17 12:35:23,526] jepsen worker 69 - jepsen.util 9339 :invoke :read [23 nil] INFO [2019-12-17 12:35:23,530] jepsen worker 12 - jepsen.util 9732 :fail :cas [4 [23 12]] :ValueNotSwapFrom INFO [2019-12-17 12:35:23,530] jepsen worker 12 - jepsen.util 9732 :invoke :read [4 nil] INFO [2019-12-17 12:35:23,531] jepsen worker 66 - jepsen.util 12576 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:35:23,531] jepsen worker 66 - jepsen.util 12576 :invoke :cas [22 [29 17]] INFO [2019-12-17 12:35:23,562] jepsen worker 49 - jepsen.util 12379 :info :cas [16 [0 25]] :RequestCanceledException INFO [2019-12-17 12:35:23,562] jepsen worker 28 - jepsen.util 7588 :fail :read [9 nil] :RequestTimeoutException INFO [2019-12-17 12:35:23,563] jepsen worker 28 - jepsen.util 7588 :invoke :write [9 9] INFO [2019-12-17 12:35:23,563] jepsen worker 49 - jepsen.util 12469 :invoke :cas [16 [2 12]] INFO [2019-12-17 12:35:23,587] jepsen worker 36 - jepsen.util 9936 :fail :cas [12 [41 10]] :ValueNotSwapFrom INFO [2019-12-17 12:35:23,587] jepsen worker 36 - jepsen.util 9936 :invoke :write [12 12] INFO [2019-12-17 12:35:23,612] jepsen worker 27 - jepsen.util 8937 :ok :write [9 12] INFO [2019-12-17 12:35:23,612] jepsen worker 27 - jepsen.util 8937 :invoke :cas [9 [43 42]] INFO [2019-12-17 12:35:23,628] jepsen worker 52 - jepsen.util 9322 :ok :read [17 21] INFO [2019-12-17 12:35:23,628] jepsen worker 52 - jepsen.util 9322 :invoke :read [17 nil] INFO [2019-12-17 12:35:23,630] jepsen worker 68 - jepsen.util 12308 :info :write [22 41] :RequestCanceledException INFO [2019-12-17 12:35:23,630] jepsen worker 38 - jepsen.util 9848 :info :write [12 19] :RequestCanceledException INFO [2019-12-17 12:35:23,631] jepsen worker 68 - jepsen.util 12398 :invoke :read [22 nil] INFO [2019-12-17 12:35:23,632] jepsen worker 27 - jepsen.util 8937 :fail :cas [9 [43 42]] :ValueNotSwapFrom INFO [2019-12-17 12:35:23,632] jepsen worker 27 - jepsen.util 8937 :invoke :read [9 nil] INFO [2019-12-17 12:35:23,632] jepsen worker 38 - jepsen.util 9938 :invoke :read [12 nil] INFO [2019-12-17 12:35:23,634] jepsen worker 53 - jepsen.util 8783 :fail :cas [17 [43 2]] :ValueNotSwapFrom INFO [2019-12-17 12:35:23,635] jepsen worker 53 - jepsen.util 8783 :invoke :write [17 11] INFO [2019-12-17 12:35:23,653] jepsen worker 64 - jepsen.util 7444 :ok :write [21 17] INFO [2019-12-17 12:35:23,653] jepsen worker 64 - jepsen.util 7444 :invoke :cas [21 [47 34]] INFO [2019-12-17 12:35:23,656] jepsen worker 27 - jepsen.util 8937 :ok :read [9 12] INFO [2019-12-17 12:35:23,656] jepsen worker 27 - jepsen.util 8937 :invoke :cas [9 [7 48]] INFO [2019-12-17 12:35:23,658] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54493,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:35:23,658] jepsen worker 0 - jepsen.util 13500 :info :cas [0 [20 44]] :RequestCanceledException INFO [2019-12-17 12:35:23,658] jepsen worker 26 - jepsen.util 13346 :info :cas [8 [14 14]] :RequestCanceledException INFO [2019-12-17 12:35:23,658] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54493,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:35:23,658] jepsen worker 29 - jepsen.util 7679 :info :cas [9 [39 16]] :RequestCanceledException INFO [2019-12-17 12:35:23,658] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54494,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:35:23,658] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54494,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:35:23,659] jepsen worker 0 - jepsen.util 13590 :invoke :read [0 nil] INFO [2019-12-17 12:35:23,659] jepsen worker 29 - jepsen.util 7769 :invoke :read [9 nil] INFO [2019-12-17 12:35:23,659] jepsen worker 26 - jepsen.util 13436 :invoke :read [8 nil] INFO [2019-12-17 12:35:23,670] jepsen worker 12 - jepsen.util 9732 :ok :read [4 1] INFO [2019-12-17 12:35:23,670] jepsen worker 12 - jepsen.util 9732 :invoke :read [4 nil] INFO [2019-12-17 12:35:23,679] jepsen worker 64 - jepsen.util 7444 :fail :cas [21 [47 34]] :ValueNotSwapFrom INFO [2019-12-17 12:35:23,679] jepsen worker 64 - jepsen.util 7444 :invoke :cas [21 [3 47]] INFO [2019-12-17 12:35:23,697] jepsen worker 61 - jepsen.util 13831 :info :write [20 32] :RequestCanceledException INFO [2019-12-17 12:35:23,698] jepsen worker 61 - jepsen.util 13921 :invoke :cas [20 [29 23]] INFO [2019-12-17 12:35:23,699] jepsen worker 27 - jepsen.util 8937 :fail :cas [9 [7 48]] :ValueNotSwapFrom INFO [2019-12-17 12:35:23,699] jepsen worker 27 - jepsen.util 8937 :invoke :write [9 6] INFO [2019-12-17 12:35:23,700] jepsen worker 64 - jepsen.util 7444 :fail :cas [21 [3 47]] :ValueNotSwapFrom INFO [2019-12-17 12:35:23,700] jepsen worker 64 - jepsen.util 7444 :invoke :cas [21 [15 49]] INFO [2019-12-17 12:35:23,715] jepsen worker 25 - jepsen.util 13885 :info :cas [8 [48 6]] :RequestCanceledException INFO [2019-12-17 12:35:23,715] jepsen worker 7 - jepsen.util 12877 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:35:23,716] jepsen worker 7 - jepsen.util 12877 :invoke :write [2 4] INFO [2019-12-17 12:35:23,717] jepsen worker 25 - jepsen.util 13975 :invoke :write [8 27] INFO [2019-12-17 12:35:23,719] jepsen worker 3 - jepsen.util 9183 :ok :read [1 35] INFO [2019-12-17 12:35:23,719] jepsen worker 3 - jepsen.util 9183 :invoke :read [1 nil] INFO [2019-12-17 12:35:23,726] jepsen worker 73 - jepsen.util 8803 :ok :write [24 29] INFO [2019-12-17 12:35:23,726] jepsen worker 73 - jepsen.util 8803 :invoke :write [24 15] INFO [2019-12-17 12:35:23,731] jepsen worker 82 - jepsen.util 13582 :info :write [27 39] :RequestCanceledException INFO [2019-12-17 12:35:23,733] jepsen worker 82 - jepsen.util 13672 :invoke :cas [27 [13 33]] INFO [2019-12-17 12:35:23,742] jepsen worker 32 - jepsen.util 9392 :ok :write [10 48] INFO [2019-12-17 12:35:23,742] jepsen worker 32 - jepsen.util 9392 :invoke :read [10 nil] INFO [2019-12-17 12:35:23,744] jepsen worker 87 - jepsen.util 8997 :fail :cas [29 [29 44]] :ValueNotSwapFrom INFO [2019-12-17 12:35:23,744] jepsen worker 87 - jepsen.util 8997 :invoke :read [29 nil] INFO [2019-12-17 12:35:23,749] jepsen worker 11 - jepsen.util 9461 :ok :write [3 17] INFO [2019-12-17 12:35:23,749] jepsen worker 11 - jepsen.util 9461 :invoke :write [3 6] INFO [2019-12-17 12:35:23,758] jepsen worker 71 - jepsen.util 9971 :ok :read [23 7] INFO [2019-12-17 12:35:23,759] jepsen worker 71 - jepsen.util 9971 :invoke :write [23 1] INFO [2019-12-17 12:35:23,762] jepsen worker 1 - jepsen.util 13051 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:35:23,762] jepsen worker 1 - jepsen.util 13051 :invoke :cas [0 [10 0]] INFO [2019-12-17 12:35:23,771] jepsen worker 69 - jepsen.util 9339 :ok :read [23 7] INFO [2019-12-17 12:35:23,771] jepsen worker 69 - jepsen.util 9339 :invoke :read [23 nil] INFO [2019-12-17 12:35:23,775] jepsen worker 64 - jepsen.util 7444 :fail :cas [21 [15 49]] :ValueNotSwapFrom INFO [2019-12-17 12:35:23,775] jepsen worker 64 - jepsen.util 7444 :invoke :read [21 nil] INFO [2019-12-17 12:35:23,840] jepsen worker 74 - jepsen.util 9074 :fail :cas [24 [17 29]] :ValueNotSwapFrom INFO [2019-12-17 12:35:23,840] jepsen worker 74 - jepsen.util 9074 :invoke :cas [24 [26 19]] INFO [2019-12-17 12:35:23,842] jepsen worker 58 - jepsen.util 13378 :info :write [19 13] :RequestCanceledException INFO [2019-12-17 12:35:23,842] jepsen worker 75 - jepsen.util 12225 :info :write [25 43] :RequestCanceledException INFO [2019-12-17 12:35:23,843] jepsen worker 58 - jepsen.util 13468 :invoke :write [19 33] INFO [2019-12-17 12:35:23,843] jepsen worker 75 - jepsen.util 12315 :invoke :write [25 31] INFO [2019-12-17 12:35:23,851] jepsen worker 20 - jepsen.util 8930 :ok :cas [6 [16 45]] INFO [2019-12-17 12:35:23,851] jepsen worker 20 - jepsen.util 8930 :invoke :write [6 31] INFO [2019-12-17 12:35:23,875] jepsen worker 3 - jepsen.util 9183 :ok :read [1 35] INFO [2019-12-17 12:35:23,875] jepsen worker 3 - jepsen.util 9183 :invoke :write [1 42] INFO [2019-12-17 12:35:23,887] jepsen worker 29 - jepsen.util 7769 :ok :read [9 12] INFO [2019-12-17 12:35:23,887] jepsen worker 29 - jepsen.util 7769 :invoke :cas [9 [14 34]] INFO [2019-12-17 12:35:23,907] jepsen worker 64 - jepsen.util 7444 :ok :read [21 17] INFO [2019-12-17 12:35:23,908] jepsen worker 64 - jepsen.util 7444 :invoke :read [21 nil] INFO [2019-12-17 12:35:23,942] jepsen worker 32 - jepsen.util 9392 :ok :read [10 48] INFO [2019-12-17 12:35:23,942] jepsen worker 32 - jepsen.util 9392 :invoke :cas [10 [1 20]] INFO [2019-12-17 12:35:23,942] jepsen worker 87 - jepsen.util 8997 :ok :read [29 16] INFO [2019-12-17 12:35:23,942] jepsen worker 87 - jepsen.util 8997 :invoke :read [29 nil] INFO [2019-12-17 12:35:23,961] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54524,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:35:23,961] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54524,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:35:23,961] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54525,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:35:23,961] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54525,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:35:23,961] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54526,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:35:23,961] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54526,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:35:23,968] jepsen worker 88 - jepsen.util 9718 :ok :read [29 16] INFO [2019-12-17 12:35:23,968] jepsen worker 88 - jepsen.util 9718 :invoke :write [29 40] INFO [2019-12-17 12:35:23,984] jepsen worker 74 - jepsen.util 9074 :fail :cas [24 [26 19]] :ValueNotSwapFrom INFO [2019-12-17 12:35:23,984] jepsen worker 74 - jepsen.util 9074 :invoke :write [24 44] INFO [2019-12-17 12:35:23,990] jepsen worker 64 - jepsen.util 7444 :ok :read [21 17] INFO [2019-12-17 12:35:23,990] jepsen worker 64 - jepsen.util 7444 :invoke :read [21 nil] INFO [2019-12-17 12:35:24,070] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54536,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:35:24,070] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54536,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:35:24,070] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54539,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:35:24,071] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54539,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:35:24,071] jepsen worker 14 - jepsen.util 9284 :ok :write [4 34] INFO [2019-12-17 12:35:24,071] jepsen worker 14 - jepsen.util 9284 :invoke :cas [4 [8 5]] INFO [2019-12-17 12:35:24,076] jepsen worker 29 - jepsen.util 7769 :fail :cas [9 [14 34]] :ValueNotSwapFrom INFO [2019-12-17 12:35:24,077] jepsen worker 29 - jepsen.util 7769 :invoke :write [9 26] INFO [2019-12-17 12:35:24,080] jepsen worker 64 - jepsen.util 7444 :ok :read [21 17] INFO [2019-12-17 12:35:24,080] jepsen worker 64 - jepsen.util 7444 :invoke :write [21 19] INFO [2019-12-17 12:35:24,100] jepsen worker 53 - jepsen.util 8783 :ok :write [17 11] INFO [2019-12-17 12:35:24,100] jepsen worker 53 - jepsen.util 8783 :invoke :cas [17 [1 18]] INFO [2019-12-17 12:35:24,115] jepsen worker 81 - jepsen.util 12591 :info :cas [27 [7 37]] :RequestCanceledException INFO [2019-12-17 12:35:24,116] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:35:24,116] jepsen nemesis - couchbase.util Rebalance check count: 426 INFO [2019-12-17 12:35:24,116] jepsen nemesis - couchbase.util Rebalance stuck count: 407 INFO [2019-12-17 12:35:24,116] jepsen worker 81 - jepsen.util 12681 :invoke :cas [27 [9 2]] INFO [2019-12-17 12:35:24,138] jepsen worker 52 - jepsen.util 9322 :ok :read [17 11] INFO [2019-12-17 12:35:24,138] jepsen worker 52 - jepsen.util 9322 :invoke :cas [17 [15 5]] INFO [2019-12-17 12:35:24,162] jepsen worker 48 - jepsen.util 11838 :info :cas [16 [45 38]] :RequestCanceledException INFO [2019-12-17 12:35:24,163] jepsen worker 48 - jepsen.util 11928 :invoke :write [16 23] INFO [2019-12-17 12:35:24,173] jepsen worker 53 - jepsen.util 8783 :fail :cas [17 [1 18]] :ValueNotSwapFrom INFO [2019-12-17 12:35:24,173] jepsen worker 53 - jepsen.util 8783 :invoke :read [17 nil] INFO [2019-12-17 12:35:24,184] jepsen worker 38 - jepsen.util 9938 :ok :read [12 35] INFO [2019-12-17 12:35:24,184] jepsen worker 38 - jepsen.util 9938 :invoke :read [12 nil] INFO [2019-12-17 12:35:24,197] jepsen worker 39 - jepsen.util 12369 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:35:24,197] jepsen worker 39 - jepsen.util 12369 :invoke :cas [13 [6 19]] INFO [2019-12-17 12:35:24,209] jepsen worker 27 - jepsen.util 8937 :ok :write [9 6] INFO [2019-12-17 12:35:24,210] jepsen worker 27 - jepsen.util 8937 :invoke :write [9 3] INFO [2019-12-17 12:35:24,214] jepsen worker 32 - jepsen.util 9392 :fail :cas [10 [1 20]] :ValueNotSwapFrom INFO [2019-12-17 12:35:24,215] jepsen worker 32 - jepsen.util 9392 :invoke :cas [10 [3 47]] INFO [2019-12-17 12:35:24,218] jepsen worker 73 - jepsen.util 8803 :ok :write [24 15] INFO [2019-12-17 12:35:24,218] jepsen worker 64 - jepsen.util 7444 :ok :write [21 19] INFO [2019-12-17 12:35:24,218] jepsen worker 73 - jepsen.util 8803 :invoke :write [24 29] INFO [2019-12-17 12:35:24,218] jepsen worker 64 - jepsen.util 7444 :invoke :cas [21 [33 44]] INFO [2019-12-17 12:35:24,229] jepsen worker 12 - jepsen.util 9732 :ok :read [4 34] INFO [2019-12-17 12:35:24,229] jepsen worker 12 - jepsen.util 9732 :invoke :read [4 nil] INFO [2019-12-17 12:35:24,241] jepsen worker 64 - jepsen.util 7444 :fail :cas [21 [33 44]] :ValueNotSwapFrom INFO [2019-12-17 12:35:24,241] jepsen worker 64 - jepsen.util 7444 :invoke :cas [21 [3 18]] INFO [2019-12-17 12:35:24,246] jepsen worker 55 - jepsen.util 10135 :ok :cas [18 [27 2]] INFO [2019-12-17 12:35:24,246] jepsen worker 55 - jepsen.util 10135 :invoke :write [18 44] INFO [2019-12-17 12:35:24,266] jepsen worker 64 - jepsen.util 7444 :fail :cas [21 [3 18]] :ValueNotSwapFrom INFO [2019-12-17 12:35:24,266] jepsen worker 64 - jepsen.util 7444 :invoke :write [21 24] INFO [2019-12-17 12:35:24,268] jepsen worker 88 - jepsen.util 9718 :ok :write [29 40] INFO [2019-12-17 12:35:24,268] jepsen worker 20 - jepsen.util 8930 :ok :write [6 31] INFO [2019-12-17 12:35:24,268] jepsen worker 88 - jepsen.util 9718 :invoke :cas [29 [5 24]] INFO [2019-12-17 12:35:24,268] jepsen worker 20 - jepsen.util 8930 :invoke :cas [6 [8 45]] INFO [2019-12-17 12:35:24,286] jepsen worker 20 - jepsen.util 8930 :fail :cas [6 [8 45]] :ValueNotSwapFrom INFO [2019-12-17 12:35:24,286] jepsen worker 20 - jepsen.util 8930 :invoke :write [6 40] INFO [2019-12-17 12:35:24,289] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54551,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:35:24,289] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54560,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:35:24,290] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54551,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:35:24,290] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54560,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:35:24,290] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54552,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:35:24,290] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54563,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:35:24,290] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54552,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:35:24,290] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54563,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:35:24,290] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54553,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:35:24,290] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54564,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:35:24,290] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54553,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:35:24,290] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54564,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:35:24,313] jepsen worker 87 - jepsen.util 8997 :ok :read [29 40] INFO [2019-12-17 12:35:24,314] jepsen worker 87 - jepsen.util 8997 :invoke :read [29 nil] INFO [2019-12-17 12:35:24,329] jepsen worker 27 - jepsen.util 8937 :ok :write [9 3] INFO [2019-12-17 12:35:24,329] jepsen worker 27 - jepsen.util 8937 :invoke :read [9 nil] INFO [2019-12-17 12:35:24,341] jepsen worker 88 - jepsen.util 9718 :fail :cas [29 [5 24]] :ValueNotSwapFrom INFO [2019-12-17 12:35:24,341] jepsen worker 88 - jepsen.util 9718 :invoke :write [29 32] INFO [2019-12-17 12:35:24,344] jepsen worker 27 - jepsen.util 8937 :ok :read [9 3] INFO [2019-12-17 12:35:24,344] jepsen worker 27 - jepsen.util 8937 :invoke :write [9 23] INFO [2019-12-17 12:35:24,348] jepsen worker 11 - jepsen.util 9461 :ok :write [3 6] INFO [2019-12-17 12:35:24,348] jepsen worker 11 - jepsen.util 9461 :invoke :cas [3 [8 32]] INFO [2019-12-17 12:35:24,394] jepsen worker 53 - jepsen.util 8783 :ok :read [17 11] INFO [2019-12-17 12:35:24,394] jepsen worker 53 - jepsen.util 8783 :invoke :cas [17 [32 11]] INFO [2019-12-17 12:35:24,410] jepsen worker 51 - jepsen.util 8601 :ok :read [17 11] INFO [2019-12-17 12:35:24,411] jepsen worker 51 - jepsen.util 8601 :invoke :read [17 nil] INFO [2019-12-17 12:35:24,411] jepsen worker 64 - jepsen.util 7444 :ok :write [21 24] INFO [2019-12-17 12:35:24,411] jepsen worker 64 - jepsen.util 7444 :invoke :read [21 nil] INFO [2019-12-17 12:35:24,411] jepsen worker 11 - jepsen.util 9461 :fail :cas [3 [8 32]] :ValueNotSwapFrom INFO [2019-12-17 12:35:24,411] jepsen worker 11 - jepsen.util 9461 :invoke :cas [3 [36 35]] INFO [2019-12-17 12:35:24,416] jepsen worker 33 - jepsen.util 12813 :info :write [11 42] :RequestCanceledException INFO [2019-12-17 12:35:24,417] jepsen worker 33 - jepsen.util 12903 :invoke :read [11 nil] INFO [2019-12-17 12:35:24,427] jepsen worker 64 - jepsen.util 7444 :ok :read [21 24] INFO [2019-12-17 12:35:24,427] jepsen worker 64 - jepsen.util 7444 :invoke :read [21 nil] INFO [2019-12-17 12:35:24,459] jepsen worker 74 - jepsen.util 9074 :ok :write [24 44] INFO [2019-12-17 12:35:24,459] jepsen worker 74 - jepsen.util 9074 :invoke :cas [24 [11 46]] INFO [2019-12-17 12:35:24,462] jepsen worker 64 - jepsen.util 7444 :ok :read [21 24] INFO [2019-12-17 12:35:24,462] jepsen worker 64 - jepsen.util 7444 :invoke :read [21 nil] INFO [2019-12-17 12:35:24,462] jepsen worker 22 - jepsen.util 12802 :info :cas [7 [9 22]] :RequestCanceledException INFO [2019-12-17 12:35:24,463] jepsen worker 22 - jepsen.util 12892 :invoke :read [7 nil] INFO [2019-12-17 12:35:24,476] jepsen worker 64 - jepsen.util 7444 :ok :read [21 24] INFO [2019-12-17 12:35:24,476] jepsen worker 64 - jepsen.util 7444 :invoke :cas [21 [37 46]] INFO [2019-12-17 12:35:24,494] jepsen worker 59 - jepsen.util 12209 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:35:24,494] jepsen worker 59 - jepsen.util 12209 :invoke :read [19 nil] INFO [2019-12-17 12:35:24,495] jepsen worker 27 - jepsen.util 8937 :ok :write [9 23] INFO [2019-12-17 12:35:24,495] jepsen worker 27 - jepsen.util 8937 :invoke :write [9 21] INFO [2019-12-17 12:35:24,499] jepsen worker 64 - jepsen.util 7444 :fail :cas [21 [37 46]] :ValueNotSwapFrom INFO [2019-12-17 12:35:24,500] jepsen worker 64 - jepsen.util 7444 :invoke :write [21 45] INFO [2019-12-17 12:35:24,502] jepsen worker 11 - jepsen.util 9461 :fail :cas [3 [36 35]] :ValueNotSwapFrom INFO [2019-12-17 12:35:24,503] jepsen worker 11 - jepsen.util 9461 :invoke :write [3 2] INFO [2019-12-17 12:35:24,504] jepsen worker 74 - jepsen.util 9074 :fail :cas [24 [11 46]] :ValueNotSwapFrom INFO [2019-12-17 12:35:24,504] jepsen worker 74 - jepsen.util 9074 :invoke :read [24 nil] INFO [2019-12-17 12:35:24,507] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54586,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:35:24,508] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54586,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:35:24,508] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54590,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:35:24,508] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54590,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:35:24,516] jepsen worker 87 - jepsen.util 8997 :ok :read [29 40] INFO [2019-12-17 12:35:24,516] jepsen worker 87 - jepsen.util 8997 :invoke :write [29 27] INFO [2019-12-17 12:35:24,524] jepsen worker 12 - jepsen.util 9732 :ok :read [4 34] INFO [2019-12-17 12:35:24,524] jepsen worker 12 - jepsen.util 9732 :invoke :write [4 19] INFO [2019-12-17 12:35:24,530] jepsen worker 41 - jepsen.util 13721 :info :cas [13 [3 7]] :RequestCanceledException INFO [2019-12-17 12:35:24,530] jepsen worker 56 - jepsen.util 9866 :info :write [18 40] :RequestCanceledException INFO [2019-12-17 12:35:24,531] jepsen worker 56 - jepsen.util 9956 :invoke :read [18 nil] INFO [2019-12-17 12:35:24,531] jepsen worker 41 - jepsen.util 13811 :invoke :write [13 44] INFO [2019-12-17 12:35:24,547] jepsen worker 52 - jepsen.util 9322 :fail :cas [17 [15 5]] :ValueNotSwapFrom INFO [2019-12-17 12:35:24,547] jepsen worker 52 - jepsen.util 9322 :invoke :write [17 24] INFO [2019-12-17 12:35:24,562] jepsen worker 77 - jepsen.util 13397 :info :cas [25 [1 46]] :RequestCanceledException INFO [2019-12-17 12:35:24,563] jepsen worker 77 - jepsen.util 13487 :invoke :write [25 49] INFO [2019-12-17 12:35:24,568] jepsen worker 14 - jepsen.util 9284 :fail :cas [4 [8 5]] :ValueNotSwapFrom INFO [2019-12-17 12:35:24,568] jepsen worker 14 - jepsen.util 9284 :invoke :cas [4 [42 8]] INFO [2019-12-17 12:35:24,573] jepsen worker 53 - jepsen.util 8783 :fail :cas [17 [32 11]] :ValueNotSwapFrom INFO [2019-12-17 12:35:24,573] jepsen worker 53 - jepsen.util 8783 :invoke :cas [17 [15 28]] INFO [2019-12-17 12:35:24,592] jepsen worker 16 - jepsen.util 14056 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:35:24,592] jepsen worker 65 - jepsen.util 7895 :fail :read [21 nil] :RequestTimeoutException INFO [2019-12-17 12:35:24,593] jepsen worker 16 - jepsen.util 14056 :invoke :cas [5 [39 38]] INFO [2019-12-17 12:35:24,593] jepsen worker 65 - jepsen.util 7895 :invoke :read [21 nil] INFO [2019-12-17 12:35:24,596] jepsen worker 74 - jepsen.util 9074 :ok :read [24 44] INFO [2019-12-17 12:35:24,596] jepsen worker 74 - jepsen.util 9074 :invoke :cas [24 [35 12]] INFO [2019-12-17 12:35:24,615] jepsen worker 34 - jepsen.util 12904 :info :write [11 41] :RequestCanceledException INFO [2019-12-17 12:35:24,615] jepsen worker 21 - jepsen.util 12621 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:35:24,615] jepsen worker 72 - jepsen.util 9252 :info :cas [24 [9 21]] :RequestCanceledException INFO [2019-12-17 12:35:24,616] jepsen worker 21 - jepsen.util 12621 :invoke :read [7 nil] INFO [2019-12-17 12:35:24,616] jepsen worker 34 - jepsen.util 12994 :invoke :read [11 nil] INFO [2019-12-17 12:35:24,616] jepsen worker 72 - jepsen.util 9342 :invoke :write [24 9] INFO [2019-12-17 12:35:24,630] jepsen worker 8 - jepsen.util 12518 :info :cas [2 [17 6]] :RequestCanceledException INFO [2019-12-17 12:35:24,630] jepsen worker 67 - jepsen.util 12847 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:35:24,630] jepsen worker 67 - jepsen.util 12847 :invoke :read [22 nil] INFO [2019-12-17 12:35:24,631] jepsen worker 15 - jepsen.util 11985 :info :cas [5 [32 34]] :RequestCanceledException INFO [2019-12-17 12:35:24,631] jepsen worker 86 - jepsen.util 13676 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:35:24,631] jepsen worker 37 - jepsen.util 8767 :info :write [12 21] :RequestCanceledException INFO [2019-12-17 12:35:24,631] jepsen worker 50 - jepsen.util 13460 :info :write [16 43] :RequestCanceledException INFO [2019-12-17 12:35:24,631] jepsen worker 86 - jepsen.util 13676 :invoke :write [28 47] INFO [2019-12-17 12:35:24,631] jepsen worker 8 - jepsen.util 12608 :invoke :read [2 nil] INFO [2019-12-17 12:35:24,632] jepsen worker 37 - jepsen.util 8857 :invoke :cas [12 [46 42]] INFO [2019-12-17 12:35:24,632] jepsen worker 15 - jepsen.util 12075 :invoke :write [5 48] INFO [2019-12-17 12:35:24,632] jepsen worker 50 - jepsen.util 13550 :invoke :cas [16 [37 41]] INFO [2019-12-17 12:35:24,634] jepsen worker 69 - jepsen.util 9339 :ok :read [23 7] INFO [2019-12-17 12:35:24,634] jepsen worker 69 - jepsen.util 9339 :invoke :write [23 32] INFO [2019-12-17 12:35:24,645] jepsen worker 2 - jepsen.util 12692 :info :cas [0 [26 12]] :RequestCanceledException INFO [2019-12-17 12:35:24,645] jepsen worker 45 - jepsen.util 9765 :fail :read [15 nil] :RequestTimeoutException INFO [2019-12-17 12:35:24,645] jepsen worker 84 - jepsen.util 12504 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:35:24,645] jepsen worker 63 - jepsen.util 8253 :info :cas [21 [21 48]] :RequestCanceledException INFO [2019-12-17 12:35:24,645] jepsen worker 45 - jepsen.util 9765 :invoke :write [15 7] INFO [2019-12-17 12:35:24,645] jepsen worker 84 - jepsen.util 12504 :invoke :cas [28 [38 29]] INFO [2019-12-17 12:35:24,646] jepsen worker 63 - jepsen.util 8343 :invoke :cas [21 [30 3]] INFO [2019-12-17 12:35:24,646] jepsen worker 2 - jepsen.util 12782 :invoke :cas [0 [36 0]] INFO [2019-12-17 12:35:24,647] jepsen worker 64 - jepsen.util 7444 :ok :write [21 45] INFO [2019-12-17 12:35:24,647] jepsen worker 64 - jepsen.util 7444 :invoke :write [21 2] INFO [2019-12-17 12:35:24,676] jepsen worker 74 - jepsen.util 9074 :fail :cas [24 [35 12]] :ValueNotSwapFrom INFO [2019-12-17 12:35:24,676] jepsen worker 74 - jepsen.util 9074 :invoke :write [24 45] INFO [2019-12-17 12:35:24,691] jepsen worker 27 - jepsen.util 8937 :ok :write [9 21] INFO [2019-12-17 12:35:24,691] jepsen worker 27 - jepsen.util 8937 :invoke :cas [9 [4 45]] INFO [2019-12-17 12:35:24,693] jepsen worker 38 - jepsen.util 9938 :ok :read [12 35] INFO [2019-12-17 12:35:24,693] jepsen worker 38 - jepsen.util 9938 :invoke :cas [12 [30 5]] INFO [2019-12-17 12:35:24,697] jepsen worker 80 - jepsen.util 8990 :info :write [26 28] :RequestCanceledException INFO [2019-12-17 12:35:24,698] jepsen worker 80 - jepsen.util 9080 :invoke :write [26 27] INFO [2019-12-17 12:35:24,727] jepsen worker 37 - jepsen.util 8857 :fail :cas [12 [46 42]] :ValueNotSwapFrom INFO [2019-12-17 12:35:24,727] jepsen worker 37 - jepsen.util 8857 :invoke :cas [12 [42 13]] INFO [2019-12-17 12:35:24,731] jepsen worker 10 - jepsen.util 9460 :fail :read [3 nil] :RequestTimeoutException INFO [2019-12-17 12:35:24,731] jepsen worker 10 - jepsen.util 9460 :invoke :write [3 4] INFO [2019-12-17 12:35:24,737] jepsen worker 64 - jepsen.util 7444 :ok :write [21 2] INFO [2019-12-17 12:35:24,737] jepsen worker 64 - jepsen.util 7444 :invoke :cas [21 [40 25]] INFO [2019-12-17 12:35:24,748] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54623,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:35:24,748] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54623,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:35:24,748] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54627,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:35:24,748] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54627,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:35:24,748] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54628,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:35:24,748] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54628,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:35:24,793] jepsen worker 20 - jepsen.util 8930 :ok :write [6 40] INFO [2019-12-17 12:35:24,793] jepsen worker 72 - jepsen.util 9342 :ok :write [24 9] INFO [2019-12-17 12:35:24,793] jepsen worker 20 - jepsen.util 8930 :invoke :cas [6 [17 26]] INFO [2019-12-17 12:35:24,794] jepsen worker 72 - jepsen.util 9342 :invoke :read [24 nil] INFO [2019-12-17 12:35:24,794] jepsen worker 64 - jepsen.util 7444 :fail :cas [21 [40 25]] :ValueNotSwapFrom INFO [2019-12-17 12:35:24,794] jepsen worker 64 - jepsen.util 7444 :invoke :read [21 nil] INFO [2019-12-17 12:35:24,807] jepsen worker 65 - jepsen.util 7895 :ok :read [21 2] INFO [2019-12-17 12:35:24,807] jepsen worker 65 - jepsen.util 7895 :invoke :write [21 9] INFO [2019-12-17 12:35:24,815] jepsen worker 4 - jepsen.util 8914 :info :write [1 15] :RequestCanceledException INFO [2019-12-17 12:35:24,816] jepsen worker 4 - jepsen.util 9004 :invoke :cas [1 [31 35]] INFO [2019-12-17 12:35:24,825] jepsen worker 51 - jepsen.util 8601 :ok :read [17 11] INFO [2019-12-17 12:35:24,825] jepsen worker 51 - jepsen.util 8601 :invoke :write [17 17] INFO [2019-12-17 12:35:24,830] jepsen worker 18 - jepsen.util 9828 :info :cas [6 [43 17]] :RequestCanceledException INFO [2019-12-17 12:35:24,830] jepsen worker 43 - jepsen.util 12733 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:35:24,830] jepsen worker 17 - jepsen.util 12527 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:35:24,830] jepsen worker 43 - jepsen.util 12733 :invoke :cas [14 [12 31]] INFO [2019-12-17 12:35:24,830] jepsen worker 17 - jepsen.util 12527 :invoke :cas [5 [7 8]] INFO [2019-12-17 12:35:24,832] jepsen worker 18 - jepsen.util 9918 :invoke :write [6 43] INFO [2019-12-17 12:35:24,845] jepsen worker 42 - jepsen.util 13902 :info :cas [14 [23 4]] :RequestCanceledException INFO [2019-12-17 12:35:24,846] jepsen worker 42 - jepsen.util 13992 :invoke :cas [14 [47 23]] INFO [2019-12-17 12:35:24,861] jepsen worker 70 - jepsen.util 10060 :info :write [23 34] :RequestCanceledException INFO [2019-12-17 12:35:24,862] jepsen worker 70 - jepsen.util 10150 :invoke :cas [23 [31 18]] INFO [2019-12-17 12:35:24,863] jepsen worker 53 - jepsen.util 8783 :fail :cas [17 [15 28]] :ValueNotSwapFrom INFO [2019-12-17 12:35:24,863] jepsen worker 53 - jepsen.util 8783 :invoke :write [17 23] INFO [2019-12-17 12:35:24,896] jepsen worker 72 - jepsen.util 9342 :ok :read [24 9] INFO [2019-12-17 12:35:24,897] jepsen worker 72 - jepsen.util 9342 :invoke :read [24 nil] INFO [2019-12-17 12:35:24,905] jepsen worker 64 - jepsen.util 7444 :ok :read [21 2] INFO [2019-12-17 12:35:24,905] jepsen worker 64 - jepsen.util 7444 :invoke :cas [21 [19 27]] INFO [2019-12-17 12:35:24,908] jepsen worker 20 - jepsen.util 8930 :fail :cas [6 [17 26]] :ValueNotSwapFrom INFO [2019-12-17 12:35:24,908] jepsen worker 20 - jepsen.util 8930 :invoke :cas [6 [27 26]] INFO [2019-12-17 12:35:24,930] jepsen worker 60 - jepsen.util 13920 :info :cas [20 [3 45]] :RequestCanceledException INFO [2019-12-17 12:35:24,931] jepsen worker 60 - jepsen.util 14010 :invoke :write [20 8] INFO [2019-12-17 12:35:24,938] jepsen worker 11 - jepsen.util 9461 :ok :write [3 2] INFO [2019-12-17 12:35:24,938] jepsen worker 11 - jepsen.util 9461 :invoke :write [3 7] INFO [2019-12-17 12:35:24,942] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54651,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:35:24,942] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54651,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:35:24,942] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54654,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:35:24,942] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54654,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:35:24,950] jepsen worker 64 - jepsen.util 7444 :fail :cas [21 [19 27]] :ValueNotSwapFrom INFO [2019-12-17 12:35:24,950] jepsen worker 64 - jepsen.util 7444 :invoke :write [21 9] INFO [2019-12-17 12:35:24,951] jepsen worker 72 - jepsen.util 9342 :ok :read [24 9] INFO [2019-12-17 12:35:24,951] jepsen worker 72 - jepsen.util 9342 :invoke :write [24 3] INFO [2019-12-17 12:35:24,957] jepsen worker 70 - jepsen.util 10150 :fail :cas [23 [31 18]] :ValueNotSwapFrom INFO [2019-12-17 12:35:24,957] jepsen worker 70 - jepsen.util 10150 :invoke :read [23 nil] INFO [2019-12-17 12:35:24,959] jepsen worker 57 - jepsen.util 12477 :info :cas [19 [19 35]] :RequestCanceledException INFO [2019-12-17 12:35:24,960] jepsen worker 57 - jepsen.util 12567 :invoke :read [19 nil] INFO [2019-12-17 12:35:24,971] jepsen worker 65 - jepsen.util 7895 :ok :write [21 9] INFO [2019-12-17 12:35:24,971] jepsen worker 65 - jepsen.util 7895 :invoke :write [21 12] INFO [2019-12-17 12:35:24,997] jepsen worker 6 - jepsen.util 13326 :info :write [2 49] :RequestCanceledException INFO [2019-12-17 12:35:24,998] jepsen worker 6 - jepsen.util 13416 :invoke :write [2 45] INFO [2019-12-17 12:35:25,011] jepsen worker 70 - jepsen.util 10150 :ok :read [23 7] INFO [2019-12-17 12:35:25,011] jepsen worker 70 - jepsen.util 10150 :invoke :write [23 20] INFO [2019-12-17 12:35:25,032] jepsen worker 20 - jepsen.util 8930 :fail :cas [6 [27 26]] :ValueNotSwapFrom INFO [2019-12-17 12:35:25,032] jepsen worker 20 - jepsen.util 8930 :invoke :read [6 nil] INFO [2019-12-17 12:35:25,045] jepsen worker 76 - jepsen.util 13036 :info :write [25 6] :RequestCanceledException INFO [2019-12-17 12:35:25,045] jepsen worker 76 - jepsen.util 13126 :invoke :cas [25 [12 41]] INFO [2019-12-17 12:35:25,050] jepsen worker 64 - jepsen.util 7444 :ok :write [21 9] INFO [2019-12-17 12:35:25,050] jepsen worker 64 - jepsen.util 7444 :invoke :cas [21 [47 21]] INFO [2019-12-17 12:35:25,051] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54668,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:35:25,051] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54668,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:35:25,051] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54671,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:35:25,052] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54671,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:35:25,052] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54672,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:35:25,052] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54672,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:35:25,091] jepsen worker 56 - jepsen.util 9956 :ok :read [18 2] INFO [2019-12-17 12:35:25,091] jepsen worker 56 - jepsen.util 9956 :invoke :write [18 21] INFO [2019-12-17 12:35:25,097] jepsen worker 44 - jepsen.util 13184 :info :write [14 38] :RequestCanceledException INFO [2019-12-17 12:35:25,100] jepsen worker 44 - jepsen.util 13274 :invoke :cas [14 [32 3]] INFO [2019-12-17 12:35:25,106] jepsen worker 20 - jepsen.util 8930 :ok :read [6 40] INFO [2019-12-17 12:35:25,106] jepsen worker 20 - jepsen.util 8930 :invoke :read [6 nil] INFO [2019-12-17 12:35:25,108] jepsen worker 64 - jepsen.util 7444 :fail :cas [21 [47 21]] :ValueNotSwapFrom INFO [2019-12-17 12:35:25,108] jepsen worker 64 - jepsen.util 7444 :invoke :read [21 nil] INFO [2019-12-17 12:35:25,114] jepsen worker 62 - jepsen.util 13112 :info :cas [20 [18 24]] :RequestCanceledException INFO [2019-12-17 12:35:25,115] jepsen worker 62 - jepsen.util 13202 :invoke :cas [20 [39 3]] INFO [2019-12-17 12:35:25,131] jepsen worker 47 - jepsen.util 9857 :info :cas [15 [47 28]] :RequestCanceledException INFO [2019-12-17 12:35:25,131] jepsen worker 24 - jepsen.util 11814 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:35:25,131] jepsen worker 9 - jepsen.util 9909 :info :cas [3 [41 20]] :RequestCanceledException INFO [2019-12-17 12:35:25,131] jepsen worker 24 - jepsen.util 11814 :invoke :write [8 38] INFO [2019-12-17 12:35:25,132] jepsen worker 9 - jepsen.util 9999 :invoke :cas [3 [41 13]] INFO [2019-12-17 12:35:25,132] jepsen worker 47 - jepsen.util 9947 :invoke :read [15 nil] INFO [2019-12-17 12:35:25,142] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:35:25,143] jepsen nemesis - couchbase.util Rebalance check count: 427 INFO [2019-12-17 12:35:25,143] jepsen nemesis - couchbase.util Rebalance stuck count: 408 INFO [2019-12-17 12:35:25,143] jepsen worker 64 - jepsen.util 7444 :ok :read [21 9] INFO [2019-12-17 12:35:25,144] jepsen worker 64 - jepsen.util 7444 :invoke :cas [21 [29 48]] INFO [2019-12-17 12:35:25,145] jepsen worker 89 - jepsen.util 9269 :info :write [29 16] :RequestCanceledException INFO [2019-12-17 12:35:25,146] jepsen worker 20 - jepsen.util 8930 :ok :read [6 40] INFO [2019-12-17 12:35:25,146] jepsen worker 20 - jepsen.util 8930 :invoke :write [6 47] INFO [2019-12-17 12:35:25,146] jepsen worker 89 - jepsen.util 9359 :invoke :write [29 16] INFO [2019-12-17 12:35:25,192] jepsen worker 64 - jepsen.util 7444 :fail :cas [21 [29 48]] :ValueNotSwapFrom INFO [2019-12-17 12:35:25,192] jepsen worker 64 - jepsen.util 7444 :invoke :write [21 42] INFO [2019-12-17 12:35:25,231] jepsen worker 35 - jepsen.util 12815 :info :write [11 31] :RequestCanceledException INFO [2019-12-17 12:35:25,232] jepsen worker 35 - jepsen.util 12905 :invoke :read [11 nil] INFO [2019-12-17 12:35:25,244] jepsen worker 79 - jepsen.util 9619 :info :write [26 22] :RequestCanceledException INFO [2019-12-17 12:35:25,244] jepsen worker 79 - jepsen.util 9709 :invoke :read [26 nil] INFO [2019-12-17 12:35:25,258] jepsen worker 64 - jepsen.util 7444 :ok :write [21 42] INFO [2019-12-17 12:35:25,258] jepsen worker 64 - jepsen.util 7444 :invoke :cas [21 [13 14]] INFO [2019-12-17 12:35:25,262] jepsen worker 85 - jepsen.util 13765 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:35:25,262] jepsen worker 85 - jepsen.util 13765 :invoke :read [28 nil] INFO [2019-12-17 12:35:25,270] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54691,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:35:25,270] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54691,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:35:25,271] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54692,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:35:25,271] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54692,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:35:25,271] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54697,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:35:25,271] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54697,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:35:25,271] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54700,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:35:25,271] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54700,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:35:25,279] jepsen worker 64 - jepsen.util 7444 :fail :cas [21 [13 14]] :ValueNotSwapFrom INFO [2019-12-17 12:35:25,279] jepsen worker 64 - jepsen.util 7444 :invoke :write [21 20] INFO [2019-12-17 12:35:25,297] jepsen worker 78 - jepsen.util 9258 :info :write [26 41] :RequestCanceledException INFO [2019-12-17 12:35:25,298] jepsen worker 78 - jepsen.util 9348 :invoke :read [26 nil] INFO [2019-12-17 12:35:25,316] jepsen worker 64 - jepsen.util 7444 :ok :write [21 20] INFO [2019-12-17 12:35:25,316] jepsen worker 64 - jepsen.util 7444 :invoke :cas [21 [4 17]] INFO [2019-12-17 12:35:25,330] jepsen worker 23 - jepsen.util 12803 :info :write [7 40] :RequestCanceledException INFO [2019-12-17 12:35:25,331] jepsen worker 23 - jepsen.util 12893 :invoke :write [7 2] INFO [2019-12-17 12:35:25,331] jepsen worker 64 - jepsen.util 7444 :fail :cas [21 [4 17]] :ValueNotSwapFrom INFO [2019-12-17 12:35:25,331] jepsen worker 64 - jepsen.util 7444 :invoke :cas [21 [41 22]] INFO [2019-12-17 12:35:25,352] jepsen worker 64 - jepsen.util 7444 :fail :cas [21 [41 22]] :ValueNotSwapFrom INFO [2019-12-17 12:35:25,352] jepsen worker 64 - jepsen.util 7444 :invoke :write [21 39] INFO [2019-12-17 12:35:25,391] jepsen worker 64 - jepsen.util 7444 :ok :write [21 39] INFO [2019-12-17 12:35:25,391] jepsen worker 64 - jepsen.util 7444 :invoke :read [21 nil] INFO [2019-12-17 12:35:25,394] jepsen worker 83 - jepsen.util 13493 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:35:25,394] jepsen worker 83 - jepsen.util 13493 :invoke :read [27 nil] INFO [2019-12-17 12:35:25,409] jepsen worker 64 - jepsen.util 7444 :ok :read [21 39] INFO [2019-12-17 12:35:25,410] jepsen worker 64 - jepsen.util 7444 :invoke :write [21 28] INFO [2019-12-17 12:35:25,430] jepsen worker 46 - jepsen.util 9406 :info :cas [15 [17 0]] :RequestCanceledException INFO [2019-12-17 12:35:25,431] jepsen worker 46 - jepsen.util 9496 :invoke :cas [15 [41 3]] INFO [2019-12-17 12:35:25,444] jepsen worker 31 - jepsen.util 8491 :fail :read [10 nil] :RequestTimeoutException INFO [2019-12-17 12:35:25,444] jepsen worker 31 - jepsen.util 8491 :invoke :write [10 47] INFO [2019-12-17 12:35:25,458] jepsen worker 54 - jepsen.util 10314 :info :write [18 2] :RequestCanceledException INFO [2019-12-17 12:35:25,458] jepsen worker 19 - jepsen.util 9379 :info :cas [6 [45 34]] :RequestCanceledException INFO [2019-12-17 12:35:25,459] jepsen worker 19 - jepsen.util 9469 :invoke :write [6 41] INFO [2019-12-17 12:35:25,459] jepsen worker 54 - jepsen.util 10404 :invoke :cas [18 [5 15]] INFO [2019-12-17 12:35:25,460] jepsen worker 64 - jepsen.util 7444 :ok :write [21 28] INFO [2019-12-17 12:35:25,461] jepsen worker 64 - jepsen.util 7444 :invoke :read [21 nil] INFO [2019-12-17 12:35:25,488] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54718,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:35:25,489] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54718,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:35:25,489] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54721,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:35:25,489] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54721,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:35:25,500] jepsen worker 64 - jepsen.util 7444 :ok :read [21 28] INFO [2019-12-17 12:35:25,500] jepsen worker 64 - jepsen.util 7444 :invoke :cas [21 [35 34]] INFO [2019-12-17 12:35:25,510] jepsen worker 38 - jepsen.util 9938 :fail :cas [12 [30 5]] :ValueNotSwapFrom INFO [2019-12-17 12:35:25,510] jepsen worker 38 - jepsen.util 9938 :invoke :write [12 41] INFO [2019-12-17 12:35:25,516] jepsen worker 13 - jepsen.util 8833 :info :write [4 24] :RequestCanceledException INFO [2019-12-17 12:35:25,517] jepsen worker 13 - jepsen.util 8923 :invoke :read [4 nil] INFO [2019-12-17 12:35:25,519] jepsen worker 78 - jepsen.util 9348 :ok :read [26 39] INFO [2019-12-17 12:35:25,519] jepsen worker 78 - jepsen.util 9348 :invoke :read [26 nil] INFO [2019-12-17 12:35:25,529] jepsen worker 64 - jepsen.util 7444 :fail :cas [21 [35 34]] :ValueNotSwapFrom INFO [2019-12-17 12:35:25,529] jepsen worker 64 - jepsen.util 7444 :invoke :cas [21 [18 29]] INFO [2019-12-17 12:35:25,531] jepsen worker 40 - jepsen.util 13180 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:35:25,531] jepsen worker 40 - jepsen.util 13180 :invoke :write [13 43] INFO [2019-12-17 12:35:25,536] jepsen worker 37 - jepsen.util 8857 :fail :cas [12 [42 13]] :ValueNotSwapFrom INFO [2019-12-17 12:35:25,536] jepsen worker 37 - jepsen.util 8857 :invoke :read [12 nil] INFO [2019-12-17 12:35:25,549] jepsen worker 64 - jepsen.util 7444 :fail :cas [21 [18 29]] :ValueNotSwapFrom INFO [2019-12-17 12:35:25,550] jepsen worker 64 - jepsen.util 7444 :invoke :read [21 nil] INFO [2019-12-17 12:35:25,562] jepsen worker 5 - jepsen.util 8735 :info :write [1 13] :RequestCanceledException INFO [2019-12-17 12:35:25,563] jepsen worker 5 - jepsen.util 8825 :invoke :write [1 47] INFO [2019-12-17 12:35:25,564] jepsen worker 64 - jepsen.util 7444 :ok :read [21 28] INFO [2019-12-17 12:35:25,564] jepsen worker 64 - jepsen.util 7444 :invoke :cas [21 [4 14]] INFO [2019-12-17 12:35:25,575] jepsen worker 64 - jepsen.util 7444 :fail :cas [21 [4 14]] :ValueNotSwapFrom INFO [2019-12-17 12:35:25,575] jepsen worker 64 - jepsen.util 7444 :invoke :write [21 39] INFO [2019-12-17 12:35:25,593] jepsen worker 79 - jepsen.util 9709 :ok :read [26 39] INFO [2019-12-17 12:35:25,593] jepsen worker 79 - jepsen.util 9709 :invoke :cas [26 [19 25]] INFO [2019-12-17 12:35:25,597] jepsen worker 30 - jepsen.util 9480 :info :write [10 7] :RequestCanceledException INFO [2019-12-17 12:35:25,597] jepsen worker 36 - jepsen.util 9936 :info :write [12 12] :RequestCanceledException INFO [2019-12-17 12:35:25,598] jepsen worker 30 - jepsen.util 9570 :invoke :cas [10 [25 20]] INFO [2019-12-17 12:35:25,598] jepsen worker 36 - jepsen.util 10026 :invoke :write [12 5] INFO [2019-12-17 12:35:25,616] jepsen worker 49 - jepsen.util 12469 :info :cas [16 [2 12]] :RequestCanceledException INFO [2019-12-17 12:35:25,617] jepsen worker 49 - jepsen.util 12559 :invoke :cas [16 [36 19]] INFO [2019-12-17 12:35:25,620] jepsen worker 79 - jepsen.util 9709 :fail :cas [26 [19 25]] :ValueNotSwapFrom INFO [2019-12-17 12:35:25,620] jepsen worker 79 - jepsen.util 9709 :invoke :cas [26 [48 47]] INFO [2019-12-17 12:35:25,624] jepsen worker 4 - jepsen.util 9004 :fail :cas [1 [31 35]] :ValueNotSwapFrom INFO [2019-12-17 12:35:25,624] jepsen worker 4 - jepsen.util 9004 :invoke :read [1 nil] INFO [2019-12-17 12:35:25,630] jepsen worker 66 - jepsen.util 12576 :info :cas [22 [29 17]] :RequestCanceledException INFO [2019-12-17 12:35:25,631] jepsen worker 66 - jepsen.util 12666 :invoke :cas [22 [27 26]] INFO [2019-12-17 12:35:25,632] jepsen worker 64 - jepsen.util 7444 :ok :write [21 39] INFO [2019-12-17 12:35:25,632] jepsen worker 64 - jepsen.util 7444 :invoke :write [21 19] INFO [2019-12-17 12:35:25,661] jepsen worker 28 - jepsen.util 7588 :info :write [9 9] :RequestCanceledException INFO [2019-12-17 12:35:25,663] jepsen worker 28 - jepsen.util 7678 :invoke :read [9 nil] INFO [2019-12-17 12:35:25,680] jepsen worker 28 - jepsen.util 7678 :ok :read [9 21] INFO [2019-12-17 12:35:25,680] jepsen worker 28 - jepsen.util 7678 :invoke :write [9 19] INFO [2019-12-17 12:35:25,681] jepsen worker 64 - jepsen.util 7444 :ok :write [21 19] INFO [2019-12-17 12:35:25,681] jepsen worker 64 - jepsen.util 7444 :invoke :write [21 19] INFO [2019-12-17 12:35:25,697] jepsen worker 26 - jepsen.util 13436 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:35:25,697] jepsen worker 26 - jepsen.util 13436 :invoke :cas [8 [4 12]] INFO [2019-12-17 12:35:25,715] jepsen worker 61 - jepsen.util 13921 :info :cas [20 [29 23]] :RequestCanceledException INFO [2019-12-17 12:35:25,716] jepsen worker 61 - jepsen.util 14011 :invoke :read [20 nil] INFO [2019-12-17 12:35:25,718] jepsen worker 13 - jepsen.util 8923 :ok :read [4 34] INFO [2019-12-17 12:35:25,719] jepsen worker 13 - jepsen.util 8923 :invoke :cas [4 [21 25]] INFO [2019-12-17 12:35:25,731] jepsen worker 68 - jepsen.util 12398 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:35:25,731] jepsen worker 68 - jepsen.util 12398 :invoke :write [22 38] INFO [2019-12-17 12:35:25,733] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54746,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:35:25,733] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54746,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:35:25,733] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54749,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:35:25,733] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54749,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:35:25,733] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54750,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:35:25,733] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54750,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:35:25,738] jepsen worker 37 - jepsen.util 8857 :ok :read [12 35] INFO [2019-12-17 12:35:25,738] jepsen worker 37 - jepsen.util 8857 :invoke :write [12 39] INFO [2019-12-17 12:35:25,739] jepsen worker 45 - jepsen.util 9765 :ok :write [15 7] INFO [2019-12-17 12:35:25,739] jepsen worker 45 - jepsen.util 9765 :invoke :cas [15 [30 26]] INFO [2019-12-17 12:35:25,739] jepsen worker 18 - jepsen.util 9918 :ok :write [6 43] INFO [2019-12-17 12:35:25,740] jepsen worker 18 - jepsen.util 9918 :invoke :write [6 38] INFO [2019-12-17 12:35:25,745] jepsen worker 82 - jepsen.util 13672 :info :cas [27 [13 33]] :RequestCanceledException INFO [2019-12-17 12:35:25,746] jepsen worker 82 - jepsen.util 13762 :invoke :read [27 nil] INFO [2019-12-17 12:35:25,760] jepsen worker 47 - jepsen.util 9947 :ok :read [15 7] INFO [2019-12-17 12:35:25,760] jepsen worker 47 - jepsen.util 9947 :invoke :write [15 37] INFO [2019-12-17 12:35:25,762] jepsen worker 0 - jepsen.util 13590 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:35:25,762] jepsen worker 0 - jepsen.util 13590 :invoke :write [0 19] INFO [2019-12-17 12:35:25,784] jepsen worker 78 - jepsen.util 9348 :ok :read [26 39] INFO [2019-12-17 12:35:25,785] jepsen worker 78 - jepsen.util 9348 :invoke :write [26 3] INFO [2019-12-17 12:35:25,790] jepsen worker 71 - jepsen.util 9971 :info :write [23 1] :RequestCanceledException INFO [2019-12-17 12:35:25,791] jepsen worker 71 - jepsen.util 10061 :invoke :read [23 nil] INFO [2019-12-17 12:35:25,800] jepsen worker 54 - jepsen.util 10404 :fail :cas [18 [5 15]] :ValueNotSwapFrom INFO [2019-12-17 12:35:25,800] jepsen worker 54 - jepsen.util 10404 :invoke :cas [18 [19 13]] INFO [2019-12-17 12:35:25,804] jepsen worker 46 - jepsen.util 9496 :fail :cas [15 [41 3]] :ValueNotSwapFrom INFO [2019-12-17 12:35:25,804] jepsen worker 46 - jepsen.util 9496 :invoke :read [15 nil] INFO [2019-12-17 12:35:25,816] jepsen worker 7 - jepsen.util 12877 :info :write [2 4] :RequestCanceledException INFO [2019-12-17 12:35:25,817] jepsen worker 7 - jepsen.util 12967 :invoke :cas [2 [28 14]] INFO [2019-12-17 12:35:25,825] jepsen worker 64 - jepsen.util 7444 :ok :write [21 19] INFO [2019-12-17 12:35:25,826] jepsen worker 64 - jepsen.util 7444 :invoke :write [21 31] INFO [2019-12-17 12:35:25,830] jepsen worker 25 - jepsen.util 13975 :info :write [8 27] :RequestCanceledException INFO [2019-12-17 12:35:25,834] jepsen worker 25 - jepsen.util 14065 :invoke :cas [8 [35 19]] INFO [2019-12-17 12:35:25,860] jepsen worker 1 - jepsen.util 13051 :info :cas [0 [10 0]] :RequestCanceledException INFO [2019-12-17 12:35:25,861] jepsen worker 58 - jepsen.util 13468 :info :write [19 33] :RequestCanceledException INFO [2019-12-17 12:35:25,864] jepsen worker 1 - jepsen.util 13141 :invoke :cas [0 [34 45]] INFO [2019-12-17 12:35:25,864] jepsen worker 58 - jepsen.util 13558 :invoke :read [19 nil] INFO [2019-12-17 12:35:25,889] jepsen worker 28 - jepsen.util 7678 :ok :write [9 19] INFO [2019-12-17 12:35:25,889] jepsen worker 28 - jepsen.util 7678 :invoke :write [9 0] INFO [2019-12-17 12:35:25,916] jepsen worker 3 - jepsen.util 9183 :info :write [1 42] :RequestCanceledException INFO [2019-12-17 12:35:25,917] jepsen worker 3 - jepsen.util 9273 :invoke :write [1 11] INFO [2019-12-17 12:35:25,924] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54772,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:35:25,924] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54772,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:35:25,924] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54775,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:35:25,924] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54775,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:35:25,924] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54776,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:35:25,924] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54776,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:35:25,930] jepsen worker 75 - jepsen.util 12315 :info :write [25 31] :RequestCanceledException INFO [2019-12-17 12:35:25,931] jepsen worker 75 - jepsen.util 12405 :invoke :cas [25 [21 32]] INFO [2019-12-17 12:35:25,943] jepsen worker 64 - jepsen.util 7444 :ok :write [21 31] INFO [2019-12-17 12:35:25,943] jepsen worker 64 - jepsen.util 7444 :invoke :read [21 nil] INFO [2019-12-17 12:35:25,983] jepsen worker 64 - jepsen.util 7444 :ok :read [21 31] INFO [2019-12-17 12:35:25,984] jepsen worker 64 - jepsen.util 7444 :invoke :read [21 nil] INFO [2019-12-17 12:35:26,013] jepsen worker 28 - jepsen.util 7678 :ok :write [9 0] INFO [2019-12-17 12:35:26,014] jepsen worker 28 - jepsen.util 7678 :invoke :cas [9 [49 5]] INFO [2019-12-17 12:35:26,017] jepsen worker 64 - jepsen.util 7444 :ok :read [21 31] INFO [2019-12-17 12:35:26,017] jepsen worker 64 - jepsen.util 7444 :invoke :read [21 nil] INFO [2019-12-17 12:35:26,025] jepsen worker 4 - jepsen.util 9004 :ok :read [1 35] INFO [2019-12-17 12:35:26,026] jepsen worker 4 - jepsen.util 9004 :invoke :cas [1 [29 43]] INFO [2019-12-17 12:35:26,033] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54788,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:35:26,034] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54788,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:35:26,043] jepsen worker 64 - jepsen.util 7444 :ok :read [21 31] INFO [2019-12-17 12:35:26,044] jepsen worker 64 - jepsen.util 7444 :invoke :cas [21 [45 31]] INFO [2019-12-17 12:35:26,046] jepsen worker 28 - jepsen.util 7678 :fail :cas [9 [49 5]] :ValueNotSwapFrom INFO [2019-12-17 12:35:26,046] jepsen worker 28 - jepsen.util 7678 :invoke :write [9 31] INFO [2019-12-17 12:35:26,060] jepsen worker 64 - jepsen.util 7444 :fail :cas [21 [45 31]] :ValueNotSwapFrom INFO [2019-12-17 12:35:26,060] jepsen worker 64 - jepsen.util 7444 :invoke :write [21 20] INFO [2019-12-17 12:35:26,089] jepsen worker 79 - jepsen.util 9709 :fail :cas [26 [48 47]] :ValueNotSwapFrom INFO [2019-12-17 12:35:26,089] jepsen worker 79 - jepsen.util 9709 :invoke :cas [26 [8 1]] INFO [2019-12-17 12:35:26,107] jepsen worker 71 - jepsen.util 10061 :ok :read [23 7] INFO [2019-12-17 12:35:26,108] jepsen worker 71 - jepsen.util 10061 :invoke :write [23 10] INFO [2019-12-17 12:35:26,118] jepsen worker 13 - jepsen.util 8923 :fail :cas [4 [21 25]] :ValueNotSwapFrom INFO [2019-12-17 12:35:26,118] jepsen worker 13 - jepsen.util 8923 :invoke :read [4 nil] INFO [2019-12-17 12:35:26,130] jepsen worker 29 - jepsen.util 7769 :info :write [9 26] :RequestCanceledException INFO [2019-12-17 12:35:26,131] jepsen worker 29 - jepsen.util 7859 :invoke :read [9 nil] INFO [2019-12-17 12:35:26,134] jepsen worker 28 - jepsen.util 7678 :ok :write [9 31] INFO [2019-12-17 12:35:26,134] jepsen worker 28 - jepsen.util 7678 :invoke :write [9 23] INFO [2019-12-17 12:35:26,135] jepsen worker 18 - jepsen.util 9918 :ok :write [6 38] INFO [2019-12-17 12:35:26,135] jepsen worker 18 - jepsen.util 9918 :invoke :read [6 nil] INFO [2019-12-17 12:35:26,135] jepsen worker 11 - jepsen.util 9461 :ok :write [3 7] INFO [2019-12-17 12:35:26,135] jepsen worker 11 - jepsen.util 9461 :invoke :cas [3 [7 8]] INFO [2019-12-17 12:35:26,135] jepsen worker 31 - jepsen.util 8491 :ok :write [10 47] INFO [2019-12-17 12:35:26,135] jepsen worker 31 - jepsen.util 8491 :invoke :read [10 nil] INFO [2019-12-17 12:35:26,142] jepsen worker 72 - jepsen.util 9342 :ok :write [24 3] INFO [2019-12-17 12:35:26,142] jepsen worker 72 - jepsen.util 9342 :invoke :read [24 nil] INFO [2019-12-17 12:35:26,161] jepsen worker 13 - jepsen.util 8923 :ok :read [4 34] INFO [2019-12-17 12:35:26,161] jepsen worker 13 - jepsen.util 8923 :invoke :cas [4 [3 0]] INFO [2019-12-17 12:35:26,171] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:35:26,171] jepsen nemesis - couchbase.util Rebalance check count: 428 INFO [2019-12-17 12:35:26,171] jepsen nemesis - couchbase.util Rebalance stuck count: 409 INFO [2019-12-17 12:35:26,179] jepsen worker 45 - jepsen.util 9765 :fail :cas [15 [30 26]] :ValueNotSwapFrom INFO [2019-12-17 12:35:26,179] jepsen worker 45 - jepsen.util 9765 :invoke :cas [15 [5 4]] INFO [2019-12-17 12:35:26,193] jepsen worker 64 - jepsen.util 7444 :ok :write [21 20] INFO [2019-12-17 12:35:26,193] jepsen worker 64 - jepsen.util 7444 :invoke :cas [21 [3 3]] INFO [2019-12-17 12:35:26,197] jepsen worker 81 - jepsen.util 12681 :info :cas [27 [9 2]] :RequestCanceledException INFO [2019-12-17 12:35:26,198] jepsen worker 81 - jepsen.util 12771 :invoke :cas [27 [23 35]] INFO [2019-12-17 12:35:26,202] jepsen worker 9 - jepsen.util 9999 :fail :cas [3 [41 13]] :ValueNotSwapFrom INFO [2019-12-17 12:35:26,202] jepsen worker 9 - jepsen.util 9999 :invoke :read [3 nil] INFO [2019-12-17 12:35:26,215] jepsen worker 48 - jepsen.util 11928 :info :write [16 23] :RequestCanceledException INFO [2019-12-17 12:35:26,216] jepsen worker 48 - jepsen.util 12018 :invoke :cas [16 [11 22]] INFO [2019-12-17 12:35:26,231] jepsen worker 32 - jepsen.util 9392 :info :cas [10 [3 47]] :RequestCanceledException INFO [2019-12-17 12:35:26,232] jepsen worker 32 - jepsen.util 9482 :invoke :cas [10 [30 41]] INFO [2019-12-17 12:35:26,247] jepsen worker 29 - jepsen.util 7859 :ok :read [9 31] INFO [2019-12-17 12:35:26,247] jepsen worker 29 - jepsen.util 7859 :invoke :read [9 nil] INFO [2019-12-17 12:35:26,253] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54808,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:35:26,253] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54818,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:35:26,253] jepsen worker 13 - jepsen.util 8923 :fail :cas [4 [3 0]] :ValueNotSwapFrom INFO [2019-12-17 12:35:26,253] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54808,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:35:26,253] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54818,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:35:26,253] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54809,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:35:26,253] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54822,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:35:26,253] jepsen worker 13 - jepsen.util 8923 :invoke :cas [4 [19 18]] INFO [2019-12-17 12:35:26,253] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54809,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:35:26,253] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54822,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:35:26,254] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54811,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:35:26,254] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54811,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:35:26,280] jepsen worker 30 - jepsen.util 9570 :fail :cas [10 [25 20]] :ValueNotSwapFrom INFO [2019-12-17 12:35:26,280] jepsen worker 30 - jepsen.util 9570 :invoke :read [10 nil] INFO [2019-12-17 12:35:26,288] jepsen worker 18 - jepsen.util 9918 :ok :read [6 38] INFO [2019-12-17 12:35:26,297] jepsen worker 18 - jepsen.util 9918 :invoke :read [6 nil] INFO [2019-12-17 12:35:26,297] jepsen worker 39 - jepsen.util 12369 :info :cas [13 [6 19]] :RequestCanceledException INFO [2019-12-17 12:35:26,302] jepsen worker 39 - jepsen.util 12459 :invoke :cas [13 [22 43]] INFO [2019-12-17 12:35:26,331] jepsen worker 73 - jepsen.util 8803 :info :write [24 29] :RequestCanceledException INFO [2019-12-17 12:35:26,331] jepsen worker 55 - jepsen.util 10135 :info :write [18 44] :RequestCanceledException INFO [2019-12-17 12:35:26,333] jepsen worker 73 - jepsen.util 8893 :invoke :write [24 9] INFO [2019-12-17 12:35:26,333] jepsen worker 55 - jepsen.util 10225 :invoke :cas [18 [25 11]] INFO [2019-12-17 12:35:26,368] jepsen worker 28 - jepsen.util 7678 :ok :write [9 23] INFO [2019-12-17 12:35:26,368] jepsen worker 28 - jepsen.util 7678 :invoke :write [9 29] INFO [2019-12-17 12:35:26,391] jepsen worker 64 - jepsen.util 7444 :fail :cas [21 [3 3]] :ValueNotSwapFrom INFO [2019-12-17 12:35:26,391] jepsen worker 64 - jepsen.util 7444 :invoke :read [21 nil] INFO [2019-12-17 12:35:26,432] jepsen worker 33 - jepsen.util 12903 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:35:26,432] jepsen worker 33 - jepsen.util 12903 :invoke :cas [11 [31 28]] INFO [2019-12-17 12:35:26,443] jepsen worker 88 - jepsen.util 9718 :info :write [29 32] :RequestCanceledException INFO [2019-12-17 12:35:26,471] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54834,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:35:26,471] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54834,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:35:26,471] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54838,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:35:26,475] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54838,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:35:26,475] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54839,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:35:26,477] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54839,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:35:26,508] jepsen worker 47 - jepsen.util 9947 :ok :write [15 37] INFO [2019-12-17 12:35:26,508] jepsen worker 47 - jepsen.util 9947 :invoke :read [15 nil] INFO [2019-12-17 12:35:26,512] jepsen worker 53 - jepsen.util 8783 :ok :write [17 23] INFO [2019-12-17 12:35:26,512] jepsen worker 53 - jepsen.util 8783 :invoke :write [17 14] INFO [2019-12-17 12:35:26,536] jepsen worker 30 - jepsen.util 9570 :ok :read [10 47] INFO [2019-12-17 12:35:26,536] jepsen worker 30 - jepsen.util 9570 :invoke :read [10 nil] INFO [2019-12-17 12:35:26,545] jepsen worker 22 - jepsen.util 12892 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:35:26,545] jepsen worker 22 - jepsen.util 12892 :invoke :read [7 nil] INFO [2019-12-17 12:35:26,553] jepsen worker 88 - jepsen.util 9808 :invoke :read [29 nil] INFO [2019-12-17 12:35:26,563] jepsen worker 12 - jepsen.util 9732 :info :write [4 19] :RequestCanceledException INFO [2019-12-17 12:35:26,564] jepsen worker 72 - jepsen.util 9342 :ok :read [24 3] INFO [2019-12-17 12:35:26,565] jepsen worker 72 - jepsen.util 9342 :invoke :cas [24 [11 28]] INFO [2019-12-17 12:35:26,566] jepsen worker 12 - jepsen.util 9822 :invoke :cas [4 [31 19]] INFO [2019-12-17 12:35:26,572] jepsen worker 79 - jepsen.util 9709 :fail :cas [26 [8 1]] :ValueNotSwapFrom INFO [2019-12-17 12:35:26,572] jepsen worker 79 - jepsen.util 9709 :invoke :write [26 42] INFO [2019-12-17 12:35:26,597] jepsen worker 77 - jepsen.util 13487 :info :write [25 49] :RequestCanceledException INFO [2019-12-17 12:35:26,597] jepsen worker 59 - jepsen.util 12209 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:35:26,605] jepsen worker 59 - jepsen.util 12209 :invoke :cas [19 [12 40]] INFO [2019-12-17 12:35:26,607] jepsen worker 77 - jepsen.util 13577 :invoke :read [25 nil] INFO [2019-12-17 12:35:26,616] jepsen worker 87 - jepsen.util 8997 :info :write [29 27] :RequestCanceledException INFO [2019-12-17 12:35:26,618] jepsen worker 87 - jepsen.util 9087 :invoke :cas [29 [48 44]] INFO [2019-12-17 12:35:26,630] jepsen worker 41 - jepsen.util 13811 :info :write [13 44] :RequestCanceledException INFO [2019-12-17 12:35:26,634] jepsen worker 41 - jepsen.util 13901 :invoke :write [13 43] INFO [2019-12-17 12:35:26,645] jepsen worker 52 - jepsen.util 9322 :info :write [17 24] :RequestCanceledException INFO [2019-12-17 12:35:26,645] jepsen worker 14 - jepsen.util 9284 :info :cas [4 [42 8]] :RequestCanceledException INFO [2019-12-17 12:35:26,645] jepsen worker 8 - jepsen.util 12608 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:35:26,645] jepsen worker 8 - jepsen.util 12608 :invoke :cas [2 [12 8]] INFO [2019-12-17 12:35:26,646] jepsen worker 52 - jepsen.util 9412 :invoke :read [17 nil] INFO [2019-12-17 12:35:26,646] jepsen worker 14 - jepsen.util 9374 :invoke :write [4 11] INFO [2019-12-17 12:35:26,653] jepsen worker 80 - jepsen.util 9080 :ok :write [26 27] INFO [2019-12-17 12:35:26,653] jepsen worker 80 - jepsen.util 9080 :invoke :write [26 6] INFO [2019-12-17 12:35:26,669] jepsen worker 29 - jepsen.util 7859 :ok :read [9 31] INFO [2019-12-17 12:35:26,670] jepsen worker 29 - jepsen.util 7859 :invoke :read [9 nil] INFO [2019-12-17 12:35:26,671] jepsen worker 31 - jepsen.util 8491 :ok :read [10 47] INFO [2019-12-17 12:35:26,671] jepsen worker 31 - jepsen.util 8491 :invoke :write [10 29] INFO [2019-12-17 12:35:26,697] jepsen worker 16 - jepsen.util 14056 :info :cas [5 [39 38]] :RequestCanceledException INFO [2019-12-17 12:35:26,697] jepsen worker 50 - jepsen.util 13550 :info :cas [16 [37 41]] :RequestCanceledException INFO [2019-12-17 12:35:26,697] jepsen worker 69 - jepsen.util 9339 :info :write [23 32] :RequestCanceledException INFO [2019-12-17 12:35:26,699] jepsen worker 16 - jepsen.util 14146 :invoke :read [5 nil] INFO [2019-12-17 12:35:26,700] jepsen worker 50 - jepsen.util 13640 :invoke :cas [16 [2 46]] INFO [2019-12-17 12:35:26,715] jepsen worker 21 - jepsen.util 12621 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:35:26,715] jepsen worker 34 - jepsen.util 12994 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:35:26,715] jepsen worker 63 - jepsen.util 8343 :info :cas [21 [30 3]] :RequestCanceledException INFO [2019-12-17 12:35:26,715] jepsen worker 21 - jepsen.util 12621 :invoke :write [7 34] INFO [2019-12-17 12:35:26,715] jepsen worker 34 - jepsen.util 12994 :invoke :cas [11 [42 45]] INFO [2019-12-17 12:35:26,717] jepsen worker 63 - jepsen.util 8433 :invoke :read [21 nil] INFO [2019-12-17 12:35:26,719] jepsen worker 30 - jepsen.util 9570 :ok :read [10 47] INFO [2019-12-17 12:35:26,719] jepsen worker 30 - jepsen.util 9570 :invoke :read [10 nil] INFO [2019-12-17 12:35:26,723] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54855,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:35:26,723] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54855,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:35:26,723] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54858,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:35:26,723] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54858,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:35:26,724] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54859,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:35:26,731] jepsen worker 86 - jepsen.util 13676 :info :write [28 47] :RequestCanceledException INFO [2019-12-17 12:35:26,731] jepsen worker 67 - jepsen.util 12847 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:35:26,731] jepsen worker 2 - jepsen.util 12782 :info :cas [0 [36 0]] :RequestCanceledException INFO [2019-12-17 12:35:26,731] jepsen worker 15 - jepsen.util 12075 :info :write [5 48] :RequestCanceledException INFO [2019-12-17 12:35:26,731] jepsen worker 74 - jepsen.util 9074 :info :write [24 45] :RequestCanceledException INFO [2019-12-17 12:35:26,731] jepsen worker 29 - jepsen.util 7859 :ok :read [9 23] INFO [2019-12-17 12:35:26,741] jepsen worker 63 - jepsen.util 8433 :ok :read [21 20] INFO [2019-12-17 12:35:26,744] jepsen worker 67 - jepsen.util 12847 :invoke :read [22 nil] INFO [2019-12-17 12:35:26,744] jepsen worker 29 - jepsen.util 7859 :invoke :write [9 2] INFO [2019-12-17 12:35:26,744] jepsen worker 63 - jepsen.util 8433 :invoke :write [21 44] INFO [2019-12-17 12:35:26,744] jepsen worker 27 - jepsen.util 8937 :info :cas [9 [4 45]] :RequestCanceledException INFO [2019-12-17 12:35:26,744] jepsen worker 84 - jepsen.util 12504 :info :cas [28 [38 29]] :RequestCanceledException INFO [2019-12-17 12:35:26,746] jepsen worker 15 - jepsen.util 12165 :invoke :write [5 2] INFO [2019-12-17 12:35:26,746] jepsen worker 2 - jepsen.util 12872 :invoke :cas [0 [38 5]] INFO [2019-12-17 12:35:26,746] jepsen worker 74 - jepsen.util 9164 :invoke :cas [24 [20 39]] INFO [2019-12-17 12:35:26,746] jepsen worker 84 - jepsen.util 12594 :invoke :cas [28 [25 27]] INFO [2019-12-17 12:35:26,748] jepsen worker 27 - jepsen.util 9027 :invoke :cas [9 [2 0]] INFO [2019-12-17 12:35:26,748] jepsen worker 3 - jepsen.util 9273 :ok :write [1 11] INFO [2019-12-17 12:35:26,748] jepsen worker 3 - jepsen.util 9273 :invoke :read [1 nil] INFO [2019-12-17 12:35:26,749] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54859,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:35:26,756] jepsen worker 72 - jepsen.util 9342 :fail :cas [24 [11 28]] :ValueNotSwapFrom INFO [2019-12-17 12:35:26,761] jepsen worker 54 - jepsen.util 10404 :fail :cas [18 [19 13]] :ValueNotSwapFrom INFO [2019-12-17 12:35:26,767] jepsen worker 72 - jepsen.util 9342 :invoke :read [24 nil] INFO [2019-12-17 12:35:26,767] jepsen worker 54 - jepsen.util 10404 :invoke :write [18 25] INFO [2019-12-17 12:35:26,776] jepsen worker 86 - jepsen.util 13766 :invoke :write [28 40] INFO [2019-12-17 12:35:26,819] jepsen worker 30 - jepsen.util 9570 :ok :read [10 47] INFO [2019-12-17 12:35:26,819] jepsen worker 30 - jepsen.util 9570 :invoke :cas [10 [20 40]] INFO [2019-12-17 12:35:26,822] jepsen worker 13 - jepsen.util 8923 :fail :cas [4 [19 18]] :ValueNotSwapFrom INFO [2019-12-17 12:35:26,822] jepsen worker 13 - jepsen.util 8923 :invoke :read [4 nil] INFO [2019-12-17 12:35:26,871] jepsen worker 10 - jepsen.util 9460 :info :write [3 4] :RequestCanceledException INFO [2019-12-17 12:35:26,896] jepsen worker 42 - jepsen.util 13992 :info :cas [14 [47 23]] :RequestCanceledException INFO [2019-12-17 12:35:26,904] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54889,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:35:26,906] jepsen worker 42 - jepsen.util 14082 :invoke :read [14 nil] INFO [2019-12-17 12:35:26,919] jepsen worker 28 - jepsen.util 7678 :ok :write [9 29] INFO [2019-12-17 12:35:26,920] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54889,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:35:26,920] jepsen worker 28 - jepsen.util 7678 :invoke :cas [9 [15 16]] INFO [2019-12-17 12:35:26,920] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54890,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:35:26,921] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54890,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:35:26,924] jepsen worker 64 - jepsen.util 7444 :ok :read [21 20] INFO [2019-12-17 12:35:26,925] jepsen worker 64 - jepsen.util 7444 :invoke :read [21 nil] INFO [2019-12-17 12:35:26,931] jepsen worker 51 - jepsen.util 8601 :info :write [17 17] :RequestCanceledException INFO [2019-12-17 12:35:26,933] jepsen worker 51 - jepsen.util 8691 :invoke :write [17 30] INFO [2019-12-17 12:35:26,934] jepsen worker 17 - jepsen.util 12527 :info :cas [5 [7 8]] :RequestCanceledException INFO [2019-12-17 12:35:26,935] jepsen worker 43 - jepsen.util 12733 :info :cas [14 [12 31]] :RequestCanceledException INFO [2019-12-17 12:35:26,937] jepsen worker 69 - jepsen.util 9429 :invoke :read [23 nil] INFO [2019-12-17 12:35:26,937] jepsen worker 17 - jepsen.util 12617 :invoke :write [5 32] INFO [2019-12-17 12:35:26,957] jepsen worker 43 - jepsen.util 12823 :invoke :write [14 12] INFO [2019-12-17 12:35:26,958] jepsen worker 10 - jepsen.util 9550 :invoke :read [3 nil] INFO [2019-12-17 12:35:26,976] jepsen worker 9 - jepsen.util 9999 :ok :read [3 7] INFO [2019-12-17 12:35:26,977] jepsen worker 9 - jepsen.util 9999 :invoke :write [3 36] INFO [2019-12-17 12:35:26,983] jepsen worker 45 - jepsen.util 9765 :fail :cas [15 [5 4]] :ValueNotSwapFrom INFO [2019-12-17 12:35:26,983] jepsen worker 45 - jepsen.util 9765 :invoke :cas [15 [40 36]] INFO [2019-12-17 12:35:27,008] jepsen worker 65 - jepsen.util 7895 :info :write [21 12] :RequestCanceledException INFO [2019-12-17 12:35:27,012] jepsen worker 65 - jepsen.util 7985 :invoke :write [21 48] INFO [2019-12-17 12:35:27,014] jepsen worker 32 - jepsen.util 9482 :fail :cas [10 [30 41]] :ValueNotSwapFrom INFO [2019-12-17 12:35:27,014] jepsen worker 32 - jepsen.util 9482 :invoke :write [10 25] INFO [2019-12-17 12:35:27,014] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54900,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:35:27,014] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54900,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:35:27,014] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54903,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:35:27,016] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54903,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:35:27,016] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54904,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:35:27,016] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54904,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:35:27,031] jepsen worker 60 - jepsen.util 14010 :info :write [20 8] :RequestCanceledException INFO [2019-12-17 12:35:27,033] jepsen worker 60 - jepsen.util 14100 :invoke :read [20 nil] INFO [2019-12-17 12:35:27,045] jepsen worker 57 - jepsen.util 12567 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:35:27,045] jepsen worker 57 - jepsen.util 12567 :invoke :read [19 nil] INFO [2019-12-17 12:35:27,052] jepsen worker 6 - jepsen.util 13416 :info :write [2 45] :RequestCanceledException INFO [2019-12-17 12:35:27,071] jepsen worker 63 - jepsen.util 8433 :ok :write [21 44] INFO [2019-12-17 12:35:27,072] jepsen worker 63 - jepsen.util 8433 :invoke :write [21 7] INFO [2019-12-17 12:35:27,093] jepsen worker 56 - jepsen.util 9956 :ok :write [18 21] INFO [2019-12-17 12:35:27,094] jepsen worker 56 - jepsen.util 9956 :invoke :cas [18 [38 27]] INFO [2019-12-17 12:35:27,096] jepsen worker 6 - jepsen.util 13506 :invoke :write [2 23] INFO [2019-12-17 12:35:27,115] jepsen worker 70 - jepsen.util 10150 :info :write [23 20] :RequestCanceledException INFO [2019-12-17 12:35:27,131] jepsen worker 76 - jepsen.util 13126 :info :cas [25 [12 41]] :RequestCanceledException INFO [2019-12-17 12:35:27,135] jepsen worker 18 - jepsen.util 9918 :ok :read [6 38] INFO [2019-12-17 12:35:27,135] jepsen worker 18 - jepsen.util 9918 :invoke :write [6 30] INFO [2019-12-17 12:35:27,138] jepsen worker 70 - jepsen.util 10240 :invoke :cas [23 [3 15]] INFO [2019-12-17 12:35:27,147] jepsen worker 78 - jepsen.util 9348 :ok :write [26 3] INFO [2019-12-17 12:35:27,147] jepsen worker 78 - jepsen.util 9348 :invoke :read [26 nil] INFO [2019-12-17 12:35:27,159] jepsen worker 76 - jepsen.util 13216 :invoke :read [25 nil] INFO [2019-12-17 12:35:27,164] jepsen worker 89 - jepsen.util 9359 :info :write [29 16] :RequestCanceledException INFO [2019-12-17 12:35:27,181] jepsen worker 89 - jepsen.util 9449 :invoke :cas [29 [5 31]] INFO [2019-12-17 12:35:27,199] jepsen worker 44 - jepsen.util 13274 :info :cas [14 [32 3]] :RequestCanceledException INFO [2019-12-17 12:35:27,199] jepsen worker 20 - jepsen.util 8930 :info :write [6 47] :RequestCanceledException INFO [2019-12-17 12:35:27,201] jepsen worker 20 - jepsen.util 9020 :invoke :write [6 21] INFO [2019-12-17 12:35:27,215] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:35:27,215] jepsen worker 62 - jepsen.util 13202 :info :cas [20 [39 3]] :RequestCanceledException INFO [2019-12-17 12:35:27,215] jepsen nemesis - couchbase.util Rebalance check count: 429 INFO [2019-12-17 12:35:27,216] jepsen nemesis - couchbase.util Rebalance stuck count: 410 INFO [2019-12-17 12:35:27,228] jepsen worker 44 - jepsen.util 13364 :invoke :read [14 nil] INFO [2019-12-17 12:35:27,230] jepsen worker 24 - jepsen.util 11814 :info :write [8 38] :RequestCanceledException INFO [2019-12-17 12:35:27,236] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54921,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:35:27,238] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54921,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:35:27,239] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54913,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:35:27,239] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54924,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:35:27,240] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54913,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:35:27,240] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54924,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:35:27,241] jepsen worker 62 - jepsen.util 13292 :invoke :write [20 5] INFO [2019-12-17 12:35:27,241] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54914,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:35:27,242] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54914,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:35:27,258] jepsen worker 24 - jepsen.util 11904 :invoke :write [8 34] INFO [2019-12-17 12:35:27,261] jepsen worker 27 - jepsen.util 9027 :fail :cas [9 [2 0]] :ValueNotSwapFrom INFO [2019-12-17 12:35:27,261] jepsen worker 27 - jepsen.util 9027 :invoke :cas [9 [44 2]] INFO [2019-12-17 12:35:27,331] jepsen worker 35 - jepsen.util 12905 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:35:27,331] jepsen worker 35 - jepsen.util 12905 :invoke :cas [11 [38 29]] INFO [2019-12-17 12:35:27,350] jepsen worker 29 - jepsen.util 7859 :ok :write [9 2] INFO [2019-12-17 12:35:27,350] jepsen worker 29 - jepsen.util 7859 :invoke :cas [9 [23 16]] INFO [2019-12-17 12:35:27,360] jepsen worker 85 - jepsen.util 13765 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:35:27,360] jepsen worker 85 - jepsen.util 13765 :invoke :write [28 22] INFO [2019-12-17 12:35:27,386] jepsen worker 38 - jepsen.util 9938 :ok :write [12 41] INFO [2019-12-17 12:35:27,386] jepsen worker 38 - jepsen.util 9938 :invoke :read [12 nil] INFO [2019-12-17 12:35:27,390] jepsen worker 11 - jepsen.util 9461 :ok :cas [3 [7 8]] INFO [2019-12-17 12:35:27,390] jepsen worker 11 - jepsen.util 9461 :invoke :read [3 nil] INFO [2019-12-17 12:35:27,405] jepsen worker 5 - jepsen.util 8825 :ok :write [1 47] INFO [2019-12-17 12:35:27,411] jepsen worker 5 - jepsen.util 8825 :invoke :cas [1 [20 35]] INFO [2019-12-17 12:35:27,429] jepsen worker 23 - jepsen.util 12893 :info :write [7 2] :RequestCanceledException INFO [2019-12-17 12:35:27,434] jepsen worker 10 - jepsen.util 9550 :ok :read [3 7] INFO [2019-12-17 12:35:27,434] jepsen worker 10 - jepsen.util 9550 :invoke :cas [3 [19 8]] INFO [2019-12-17 12:35:27,452] jepsen worker 23 - jepsen.util 12983 :invoke :cas [7 [41 17]] INFO [2019-12-17 12:35:27,455] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54945,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:35:27,457] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54945,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:35:27,460] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54946,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:35:27,463] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54946,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:35:27,484] jepsen worker 73 - jepsen.util 8893 :ok :write [24 9] INFO [2019-12-17 12:35:27,484] jepsen worker 73 - jepsen.util 8893 :invoke :read [24 nil] INFO [2019-12-17 12:35:27,495] jepsen worker 83 - jepsen.util 13493 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:35:27,495] jepsen worker 83 - jepsen.util 13493 :invoke :read [27 nil] INFO [2019-12-17 12:35:27,513] jepsen worker 28 - jepsen.util 7678 :fail :cas [9 [15 16]] :ValueNotSwapFrom INFO [2019-12-17 12:35:27,514] jepsen worker 28 - jepsen.util 7678 :invoke :read [9 nil] INFO [2019-12-17 12:35:27,558] jepsen worker 19 - jepsen.util 9469 :info :write [6 41] :RequestCanceledException INFO [2019-12-17 12:35:27,561] jepsen worker 19 - jepsen.util 9559 :invoke :write [6 4] INFO [2019-12-17 12:35:27,606] jepsen worker 4 - jepsen.util 9004 :fail :cas [1 [29 43]] :ValueNotSwapFrom INFO [2019-12-17 12:35:27,606] jepsen worker 4 - jepsen.util 9004 :invoke :read [1 nil] INFO [2019-12-17 12:35:27,610] jepsen worker 87 - jepsen.util 9087 :fail :cas [29 [48 44]] :ValueNotSwapFrom INFO [2019-12-17 12:35:27,610] jepsen worker 87 - jepsen.util 9087 :invoke :read [29 nil] INFO [2019-12-17 12:35:27,617] jepsen worker 12 - jepsen.util 9822 :fail :cas [4 [31 19]] :ValueNotSwapFrom INFO [2019-12-17 12:35:27,617] jepsen worker 12 - jepsen.util 9822 :invoke :write [4 6] INFO [2019-12-17 12:35:27,625] jepsen worker 64 - jepsen.util 7444 :ok :read [21 44] INFO [2019-12-17 12:35:27,625] jepsen worker 64 - jepsen.util 7444 :invoke :read [21 nil] INFO [2019-12-17 12:35:27,628] jepsen worker 29 - jepsen.util 7859 :fail :cas [9 [23 16]] :ValueNotSwapFrom INFO [2019-12-17 12:35:27,629] jepsen worker 29 - jepsen.util 7859 :invoke :cas [9 [0 22]] INFO [2019-12-17 12:35:27,629] jepsen worker 3 - jepsen.util 9273 :ok :read [1 11] INFO [2019-12-17 12:35:27,629] jepsen worker 3 - jepsen.util 9273 :invoke :write [1 48] INFO [2019-12-17 12:35:27,632] jepsen worker 40 - jepsen.util 13180 :info :write [13 43] :RequestCanceledException INFO [2019-12-17 12:35:27,634] jepsen worker 40 - jepsen.util 13270 :invoke :cas [13 [38 45]] INFO [2019-12-17 12:35:27,638] jepsen worker 27 - jepsen.util 9027 :fail :cas [9 [44 2]] :ValueNotSwapFrom INFO [2019-12-17 12:35:27,638] jepsen worker 27 - jepsen.util 9027 :invoke :cas [9 [21 25]] INFO [2019-12-17 12:35:27,639] jepsen worker 45 - jepsen.util 9765 :fail :cas [15 [40 36]] :ValueNotSwapFrom INFO [2019-12-17 12:35:27,639] jepsen worker 45 - jepsen.util 9765 :invoke :read [15 nil] INFO [2019-12-17 12:35:27,643] jepsen worker 36 - jepsen.util 10026 :info :write [12 5] :RequestCanceledException INFO [2019-12-17 12:35:27,647] jepsen worker 36 - jepsen.util 10116 :invoke :write [12 13] INFO [2019-12-17 12:35:27,658] jepsen worker 49 - jepsen.util 12559 :info :cas [16 [36 19]] :RequestCanceledException INFO [2019-12-17 12:35:27,660] jepsen worker 55 - jepsen.util 10225 :fail :cas [18 [25 11]] :ValueNotSwapFrom INFO [2019-12-17 12:35:27,660] jepsen worker 55 - jepsen.util 10225 :invoke :read [18 nil] INFO [2019-12-17 12:35:27,660] jepsen worker 28 - jepsen.util 7678 :ok :read [9 2] INFO [2019-12-17 12:35:27,660] jepsen worker 28 - jepsen.util 7678 :invoke :cas [9 [47 41]] INFO [2019-12-17 12:35:27,661] jepsen worker 49 - jepsen.util 12649 :invoke :cas [16 [1 47]] INFO [2019-12-17 12:35:27,676] jepsen worker 79 - jepsen.util 9709 :ok :write [26 42] INFO [2019-12-17 12:35:27,676] jepsen worker 79 - jepsen.util 9709 :invoke :cas [26 [23 1]] INFO [2019-12-17 12:35:27,681] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54963,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:35:27,681] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54963,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:35:27,682] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54966,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:35:27,682] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54966,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:35:27,682] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54967,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:35:27,682] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54967,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:35:27,728] jepsen worker 66 - jepsen.util 12666 :info :cas [22 [27 26]] :RequestCanceledException INFO [2019-12-17 12:35:27,731] jepsen worker 66 - jepsen.util 12756 :invoke :write [22 22] INFO [2019-12-17 12:35:27,732] jepsen worker 64 - jepsen.util 7444 :ok :read [21 44] INFO [2019-12-17 12:35:27,732] jepsen worker 64 - jepsen.util 7444 :invoke :read [21 nil] INFO [2019-12-17 12:35:27,746] jepsen worker 13 - jepsen.util 8923 :ok :read [4 34] INFO [2019-12-17 12:35:27,746] jepsen worker 13 - jepsen.util 8923 :invoke :cas [4 [28 26]] INFO [2019-12-17 12:35:27,749] jepsen worker 54 - jepsen.util 10404 :ok :write [18 25] INFO [2019-12-17 12:35:27,749] jepsen worker 54 - jepsen.util 10404 :invoke :read [18 nil] INFO [2019-12-17 12:35:27,761] jepsen worker 37 - jepsen.util 8857 :info :write [12 39] :RequestCanceledException INFO [2019-12-17 12:35:27,765] jepsen worker 37 - jepsen.util 8947 :invoke :read [12 nil] INFO [2019-12-17 12:35:27,777] jepsen worker 27 - jepsen.util 9027 :fail :cas [9 [21 25]] :ValueNotSwapFrom INFO [2019-12-17 12:35:27,777] jepsen worker 27 - jepsen.util 9027 :invoke :read [9 nil] INFO [2019-12-17 12:35:27,778] jepsen worker 70 - jepsen.util 10240 :fail :cas [23 [3 15]] :ValueNotSwapFrom INFO [2019-12-17 12:35:27,778] jepsen worker 70 - jepsen.util 10240 :invoke :read [23 nil] INFO [2019-12-17 12:35:27,781] jepsen worker 78 - jepsen.util 9348 :ok :read [26 42] INFO [2019-12-17 12:35:27,781] jepsen worker 78 - jepsen.util 9348 :invoke :write [26 10] INFO [2019-12-17 12:35:27,797] jepsen worker 26 - jepsen.util 13436 :info :cas [8 [4 12]] :RequestCanceledException INFO [2019-12-17 12:35:27,805] jepsen worker 29 - jepsen.util 7859 :fail :cas [9 [0 22]] :ValueNotSwapFrom INFO [2019-12-17 12:35:27,805] jepsen worker 29 - jepsen.util 7859 :invoke :cas [9 [47 40]] INFO [2019-12-17 12:35:27,806] jepsen worker 26 - jepsen.util 13526 :invoke :read [8 nil] INFO [2019-12-17 12:35:27,816] jepsen worker 61 - jepsen.util 14011 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:35:27,816] jepsen worker 61 - jepsen.util 14011 :invoke :read [20 nil] INFO [2019-12-17 12:35:27,818] jepsen worker 5 - jepsen.util 8825 :fail :cas [1 [20 35]] :ValueNotSwapFrom INFO [2019-12-17 12:35:27,818] jepsen worker 5 - jepsen.util 8825 :invoke :cas [1 [8 45]] INFO [2019-12-17 12:35:27,829] jepsen worker 68 - jepsen.util 12398 :info :write [22 38] :RequestCanceledException INFO [2019-12-17 12:35:27,829] jepsen worker 82 - jepsen.util 13762 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:35:27,830] jepsen worker 82 - jepsen.util 13762 :invoke :read [27 nil] INFO [2019-12-17 12:35:27,834] jepsen worker 68 - jepsen.util 12488 :invoke :write [22 34] INFO [2019-12-17 12:35:27,844] jepsen worker 46 - jepsen.util 9496 :fail :read [15 nil] :RequestTimeoutException INFO [2019-12-17 12:35:27,844] jepsen worker 46 - jepsen.util 9496 :invoke :write [15 39] INFO [2019-12-17 12:35:27,852] jepsen worker 65 - jepsen.util 7985 :ok :write [21 48] INFO [2019-12-17 12:35:27,852] jepsen worker 65 - jepsen.util 7985 :invoke :write [21 5] INFO [2019-12-17 12:35:27,858] jepsen worker 0 - jepsen.util 13590 :info :write [0 19] :RequestCanceledException INFO [2019-12-17 12:35:27,858] jepsen worker 7 - jepsen.util 12967 :info :cas [2 [28 14]] :RequestCanceledException INFO [2019-12-17 12:35:27,859] jepsen worker 28 - jepsen.util 7678 :fail :cas [9 [47 41]] :ValueNotSwapFrom INFO [2019-12-17 12:35:27,859] jepsen worker 28 - jepsen.util 7678 :invoke :read [9 nil] INFO [2019-12-17 12:35:27,862] jepsen worker 0 - jepsen.util 13680 :invoke :write [0 9] INFO [2019-12-17 12:35:27,862] jepsen worker 7 - jepsen.util 13057 :invoke :read [2 nil] INFO [2019-12-17 12:35:27,880] jepsen worker 64 - jepsen.util 7444 :ok :read [21 44] INFO [2019-12-17 12:35:27,880] jepsen worker 64 - jepsen.util 7444 :invoke :read [21 nil] INFO [2019-12-17 12:35:27,894] jepsen worker 1 - jepsen.util 13141 :info :cas [0 [34 45]] :RequestCanceledException INFO [2019-12-17 12:35:27,897] jepsen worker 1 - jepsen.util 13231 :invoke :read [0 nil] INFO [2019-12-17 12:35:27,913] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54990,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:35:27,913] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54990,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:35:27,913] jepsen worker 58 - jepsen.util 13558 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:35:27,913] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54993,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:35:27,913] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54993,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:35:27,913] jepsen worker 58 - jepsen.util 13558 :invoke :write [19 26] INFO [2019-12-17 12:35:27,914] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54994,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:35:27,914] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":54994,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:35:27,916] jepsen worker 27 - jepsen.util 9027 :ok :read [9 2] INFO [2019-12-17 12:35:27,916] jepsen worker 27 - jepsen.util 9027 :invoke :cas [9 [47 13]] INFO [2019-12-17 12:35:27,916] jepsen worker 52 - jepsen.util 9412 :ok :read [17 17] INFO [2019-12-17 12:35:27,916] jepsen worker 52 - jepsen.util 9412 :invoke :write [17 44] INFO [2019-12-17 12:35:27,927] jepsen worker 25 - jepsen.util 14065 :info :cas [8 [35 19]] :RequestCanceledException INFO [2019-12-17 12:35:27,931] jepsen worker 25 - jepsen.util 14155 :invoke :read [8 nil] INFO [2019-12-17 12:35:27,944] jepsen worker 37 - jepsen.util 8947 :ok :read [12 41] INFO [2019-12-17 12:35:27,944] jepsen worker 75 - jepsen.util 12405 :info :cas [25 [21 32]] :RequestCanceledException INFO [2019-12-17 12:35:27,944] jepsen worker 37 - jepsen.util 8947 :invoke :write [12 11] INFO [2019-12-17 12:35:27,947] jepsen worker 75 - jepsen.util 12495 :invoke :read [25 nil] INFO [2019-12-17 12:35:27,949] jepsen worker 29 - jepsen.util 7859 :fail :cas [9 [47 40]] :ValueNotSwapFrom INFO [2019-12-17 12:35:27,949] jepsen worker 29 - jepsen.util 7859 :invoke :write [9 17] INFO [2019-12-17 12:35:27,974] jepsen worker 64 - jepsen.util 7444 :ok :read [21 48] INFO [2019-12-17 12:35:27,974] jepsen worker 64 - jepsen.util 7444 :invoke :read [21 nil] INFO [2019-12-17 12:35:28,000] jepsen worker 28 - jepsen.util 7678 :ok :read [9 2] INFO [2019-12-17 12:35:28,000] jepsen worker 28 - jepsen.util 7678 :invoke :cas [9 [48 34]] INFO [2019-12-17 12:35:28,005] jepsen worker 32 - jepsen.util 9482 :ok :write [10 25] INFO [2019-12-17 12:35:28,005] jepsen worker 32 - jepsen.util 9482 :invoke :write [10 19] INFO [2019-12-17 12:35:28,010] jepsen worker 27 - jepsen.util 9027 :fail :cas [9 [47 13]] :ValueNotSwapFrom INFO [2019-12-17 12:35:28,010] jepsen worker 27 - jepsen.util 9027 :invoke :cas [9 [25 2]] INFO [2019-12-17 12:35:28,067] jepsen worker 56 - jepsen.util 9956 :fail :cas [18 [38 27]] :ValueNotSwapFrom INFO [2019-12-17 12:35:28,067] jepsen worker 56 - jepsen.util 9956 :invoke :write [18 1] INFO [2019-12-17 12:35:28,082] jepsen worker 28 - jepsen.util 7678 :fail :cas [9 [48 34]] :ValueNotSwapFrom INFO [2019-12-17 12:35:28,082] jepsen worker 28 - jepsen.util 7678 :invoke :write [9 45] INFO [2019-12-17 12:35:28,085] jepsen worker 29 - jepsen.util 7859 :ok :write [9 17] INFO [2019-12-17 12:35:28,085] jepsen worker 29 - jepsen.util 7859 :invoke :cas [9 [12 11]] INFO [2019-12-17 12:35:28,089] jepsen worker 27 - jepsen.util 9027 :fail :cas [9 [25 2]] :ValueNotSwapFrom INFO [2019-12-17 12:35:28,089] jepsen worker 27 - jepsen.util 9027 :invoke :cas [9 [26 33]] INFO [2019-12-17 12:35:28,103] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55011,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:35:28,104] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55011,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:35:28,129] jepsen worker 29 - jepsen.util 7859 :fail :cas [9 [12 11]] :ValueNotSwapFrom INFO [2019-12-17 12:35:28,129] jepsen worker 29 - jepsen.util 7859 :invoke :write [9 7] INFO [2019-12-17 12:35:28,140] jepsen worker 71 - jepsen.util 10061 :info :write [23 10] :RequestCanceledException INFO [2019-12-17 12:35:28,144] jepsen worker 71 - jepsen.util 10151 :invoke :cas [23 [26 46]] INFO [2019-12-17 12:35:28,170] jepsen worker 88 - jepsen.util 9808 :ok :read [29 40] INFO [2019-12-17 12:35:28,170] jepsen worker 88 - jepsen.util 9808 :invoke :read [29 nil] INFO [2019-12-17 12:35:28,172] jepsen worker 27 - jepsen.util 9027 :fail :cas [9 [26 33]] :ValueNotSwapFrom INFO [2019-12-17 12:35:28,172] jepsen worker 27 - jepsen.util 9027 :invoke :cas [9 [29 26]] INFO [2019-12-17 12:35:28,189] jepsen worker 9 - jepsen.util 9999 :ok :write [3 36] INFO [2019-12-17 12:35:28,189] jepsen worker 9 - jepsen.util 9999 :invoke :write [3 8] INFO [2019-12-17 12:35:28,217] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55027,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:35:28,217] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55034,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:35:28,217] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55027,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:35:28,217] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55034,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:35:28,217] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55029,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:35:28,217] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55037,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:35:28,217] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55029,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:35:28,217] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55037,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:35:28,217] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55038,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:35:28,217] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55038,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:35:28,219] jepsen worker 28 - jepsen.util 7678 :ok :write [9 45] INFO [2019-12-17 12:35:28,219] jepsen worker 28 - jepsen.util 7678 :invoke :cas [9 [20 7]] INFO [2019-12-17 12:35:28,228] jepsen worker 81 - jepsen.util 12771 :info :cas [27 [23 35]] :RequestCanceledException INFO [2019-12-17 12:35:28,231] jepsen worker 81 - jepsen.util 12861 :invoke :cas [27 [24 41]] INFO [2019-12-17 12:35:28,233] jepsen worker 69 - jepsen.util 9429 :ok :read [23 32] INFO [2019-12-17 12:35:28,234] jepsen worker 69 - jepsen.util 9429 :invoke :read [23 nil] INFO [2019-12-17 12:35:28,245] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:35:28,245] jepsen nemesis - couchbase.util Rebalance check count: 430 INFO [2019-12-17 12:35:28,245] jepsen nemesis - couchbase.util Rebalance stuck count: 411 INFO [2019-12-17 12:35:28,269] jepsen worker 27 - jepsen.util 9027 :fail :cas [9 [29 26]] :ValueNotSwapFrom INFO [2019-12-17 12:35:28,269] jepsen worker 27 - jepsen.util 9027 :invoke :read [9 nil] INFO [2019-12-17 12:35:28,273] jepsen worker 54 - jepsen.util 10404 :ok :read [18 25] INFO [2019-12-17 12:35:28,274] jepsen worker 54 - jepsen.util 10404 :invoke :read [18 nil] INFO [2019-12-17 12:35:28,278] jepsen worker 29 - jepsen.util 7859 :ok :write [9 7] INFO [2019-12-17 12:35:28,278] jepsen worker 29 - jepsen.util 7859 :invoke :write [9 39] INFO [2019-12-17 12:35:28,284] jepsen worker 28 - jepsen.util 7678 :fail :cas [9 [20 7]] :ValueNotSwapFrom INFO [2019-12-17 12:35:28,284] jepsen worker 28 - jepsen.util 7678 :invoke :read [9 nil] INFO [2019-12-17 12:35:28,298] jepsen worker 48 - jepsen.util 12018 :info :cas [16 [11 22]] :RequestCanceledException INFO [2019-12-17 12:35:28,301] jepsen worker 48 - jepsen.util 12108 :invoke :read [16 nil] INFO [2019-12-17 12:35:28,301] jepsen worker 4 - jepsen.util 9004 :ok :read [1 47] INFO [2019-12-17 12:35:28,302] jepsen worker 4 - jepsen.util 9004 :invoke :read [1 nil] INFO [2019-12-17 12:35:28,330] jepsen worker 39 - jepsen.util 12459 :info :cas [13 [22 43]] :RequestCanceledException INFO [2019-12-17 12:35:28,333] jepsen worker 39 - jepsen.util 12549 :invoke :cas [13 [39 29]] INFO [2019-12-17 12:35:28,346] jepsen worker 27 - jepsen.util 9027 :ok :read [9 7] INFO [2019-12-17 12:35:28,346] jepsen worker 27 - jepsen.util 9027 :invoke :write [9 22] INFO [2019-12-17 12:35:28,378] jepsen worker 28 - jepsen.util 7678 :ok :read [9 7] INFO [2019-12-17 12:35:28,378] jepsen worker 28 - jepsen.util 7678 :invoke :write [9 15] INFO [2019-12-17 12:35:28,380] jepsen worker 55 - jepsen.util 10225 :ok :read [18 25] INFO [2019-12-17 12:35:28,380] jepsen worker 55 - jepsen.util 10225 :invoke :write [18 10] INFO [2019-12-17 12:35:28,386] jepsen worker 63 - jepsen.util 8433 :ok :write [21 7] INFO [2019-12-17 12:35:28,386] jepsen worker 63 - jepsen.util 8433 :invoke :cas [21 [8 13]] INFO [2019-12-17 12:35:28,406] jepsen worker 38 - jepsen.util 9938 :ok :read [12 41] INFO [2019-12-17 12:35:28,406] jepsen worker 38 - jepsen.util 9938 :invoke :read [12 nil] INFO [2019-12-17 12:35:28,433] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55055,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:35:28,433] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55055,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:35:28,433] jepsen worker 5 - jepsen.util 8825 :fail :cas [1 [8 45]] :ValueNotSwapFrom INFO [2019-12-17 12:35:28,433] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55059,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:35:28,434] jepsen worker 5 - jepsen.util 8825 :invoke :write [1 48] INFO [2019-12-17 12:35:28,434] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55059,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:35:28,438] jepsen worker 63 - jepsen.util 8433 :fail :cas [21 [8 13]] :ValueNotSwapFrom INFO [2019-12-17 12:35:28,438] jepsen worker 63 - jepsen.util 8433 :invoke :cas [21 [42 26]] INFO [2019-12-17 12:35:28,458] jepsen worker 29 - jepsen.util 7859 :ok :write [9 39] INFO [2019-12-17 12:35:28,458] jepsen worker 29 - jepsen.util 7859 :invoke :write [9 47] INFO [2019-12-17 12:35:28,489] jepsen worker 63 - jepsen.util 8433 :fail :cas [21 [42 26]] :ValueNotSwapFrom INFO [2019-12-17 12:35:28,489] jepsen worker 63 - jepsen.util 8433 :invoke :cas [21 [12 30]] INFO [2019-12-17 12:35:28,523] jepsen worker 33 - jepsen.util 12903 :info :cas [11 [31 28]] :RequestCanceledException INFO [2019-12-17 12:35:28,526] jepsen worker 33 - jepsen.util 12993 :invoke :write [11 15] INFO [2019-12-17 12:35:28,545] jepsen worker 47 - jepsen.util 9947 :fail :read [15 nil] :RequestTimeoutException INFO [2019-12-17 12:35:28,545] jepsen worker 47 - jepsen.util 9947 :invoke :write [15 16] INFO [2019-12-17 12:35:28,572] jepsen worker 71 - jepsen.util 10151 :fail :cas [23 [26 46]] :ValueNotSwapFrom INFO [2019-12-17 12:35:28,572] jepsen worker 71 - jepsen.util 10151 :invoke :read [23 nil] INFO [2019-12-17 12:35:28,574] jepsen worker 63 - jepsen.util 8433 :fail :cas [21 [12 30]] :ValueNotSwapFrom INFO [2019-12-17 12:35:28,574] jepsen worker 63 - jepsen.util 8433 :invoke :read [21 nil] INFO [2019-12-17 12:35:28,591] jepsen worker 89 - jepsen.util 9449 :fail :cas [29 [5 31]] :ValueNotSwapFrom INFO [2019-12-17 12:35:28,591] jepsen worker 89 - jepsen.util 9449 :invoke :cas [29 [35 22]] INFO [2019-12-17 12:35:28,613] jepsen worker 13 - jepsen.util 8923 :fail :cas [4 [28 26]] :ValueNotSwapFrom INFO [2019-12-17 12:35:28,613] jepsen worker 13 - jepsen.util 8923 :invoke :read [4 nil] INFO [2019-12-17 12:35:28,616] jepsen worker 53 - jepsen.util 8783 :info :write [17 14] :RequestCanceledException INFO [2019-12-17 12:35:28,618] jepsen worker 53 - jepsen.util 8873 :invoke :cas [17 [23 41]] INFO [2019-12-17 12:35:28,628] jepsen worker 63 - jepsen.util 8433 :ok :read [21 7] INFO [2019-12-17 12:35:28,628] jepsen worker 63 - jepsen.util 8433 :invoke :cas [21 [27 17]] INFO [2019-12-17 12:35:28,643] jepsen worker 22 - jepsen.util 12892 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:35:28,643] jepsen worker 22 - jepsen.util 12892 :invoke :cas [7 [45 2]] INFO [2019-12-17 12:35:28,649] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55073,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:35:28,649] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55073,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:35:28,649] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55076,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:35:28,649] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55076,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:35:28,649] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55077,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:35:28,649] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55077,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:35:28,653] jepsen worker 29 - jepsen.util 7859 :ok :write [9 47] INFO [2019-12-17 12:35:28,653] jepsen worker 29 - jepsen.util 7859 :invoke :read [9 nil] INFO [2019-12-17 12:35:28,660] jepsen worker 87 - jepsen.util 9087 :ok :read [29 40] INFO [2019-12-17 12:35:28,660] jepsen worker 87 - jepsen.util 9087 :invoke :write [29 7] INFO [2019-12-17 12:35:28,686] jepsen worker 63 - jepsen.util 8433 :fail :cas [21 [27 17]] :ValueNotSwapFrom INFO [2019-12-17 12:35:28,686] jepsen worker 63 - jepsen.util 8433 :invoke :read [21 nil] INFO [2019-12-17 12:35:28,697] jepsen worker 59 - jepsen.util 12209 :info :cas [19 [12 40]] :RequestCanceledException INFO [2019-12-17 12:35:28,699] jepsen worker 38 - jepsen.util 9938 :ok :read [12 41] INFO [2019-12-17 12:35:28,699] jepsen worker 38 - jepsen.util 9938 :invoke :cas [12 [16 18]] INFO [2019-12-17 12:35:28,700] jepsen worker 59 - jepsen.util 12299 :invoke :write [19 15] INFO [2019-12-17 12:35:28,701] jepsen worker 29 - jepsen.util 7859 :ok :read [9 47] INFO [2019-12-17 12:35:28,701] jepsen worker 29 - jepsen.util 7859 :invoke :cas [9 [8 1]] INFO [2019-12-17 12:35:28,715] jepsen worker 77 - jepsen.util 13577 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:35:28,715] jepsen worker 50 - jepsen.util 13640 :info :cas [16 [2 46]] :RequestCanceledException INFO [2019-12-17 12:35:28,715] jepsen worker 77 - jepsen.util 13577 :invoke :cas [25 [14 41]] INFO [2019-12-17 12:35:28,717] jepsen worker 50 - jepsen.util 13730 :invoke :write [16 21] INFO [2019-12-17 12:35:28,731] jepsen worker 14 - jepsen.util 9374 :info :write [4 11] :RequestCanceledException INFO [2019-12-17 12:35:28,734] jepsen worker 14 - jepsen.util 9464 :invoke :write [4 48] INFO [2019-12-17 12:35:28,745] jepsen worker 8 - jepsen.util 12608 :info :cas [2 [12 8]] :RequestCanceledException INFO [2019-12-17 12:35:28,745] jepsen worker 41 - jepsen.util 13901 :info :write [13 43] :RequestCanceledException INFO [2019-12-17 12:35:28,745] jepsen worker 31 - jepsen.util 8491 :info :write [10 29] :RequestCanceledException INFO [2019-12-17 12:35:28,746] jepsen worker 31 - jepsen.util 8581 :invoke :read [10 nil] INFO [2019-12-17 12:35:28,749] jepsen worker 63 - jepsen.util 8433 :ok :read [21 7] INFO [2019-12-17 12:35:28,749] jepsen worker 63 - jepsen.util 8433 :invoke :write [21 30] INFO [2019-12-17 12:35:28,750] jepsen worker 8 - jepsen.util 12698 :invoke :write [2 46] INFO [2019-12-17 12:35:28,750] jepsen worker 41 - jepsen.util 13991 :invoke :cas [13 [16 48]] INFO [2019-12-17 12:35:28,758] jepsen worker 2 - jepsen.util 12872 :info :cas [0 [38 5]] :RequestCanceledException INFO [2019-12-17 12:35:28,765] jepsen worker 2 - jepsen.util 12962 :invoke :cas [0 [18 29]] INFO [2019-12-17 12:35:28,778] jepsen worker 88 - jepsen.util 9808 :ok :read [29 40] INFO [2019-12-17 12:35:28,778] jepsen worker 88 - jepsen.util 9808 :invoke :read [29 nil] INFO [2019-12-17 12:35:28,791] jepsen worker 29 - jepsen.util 7859 :fail :cas [9 [8 1]] :ValueNotSwapFrom INFO [2019-12-17 12:35:28,792] jepsen worker 29 - jepsen.util 7859 :invoke :cas [9 [46 47]] INFO [2019-12-17 12:35:28,797] jepsen worker 16 - jepsen.util 14146 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:35:28,797] jepsen worker 16 - jepsen.util 14146 :invoke :cas [5 [29 36]] INFO [2019-12-17 12:35:28,812] jepsen worker 4 - jepsen.util 9004 :ok :read [1 47] INFO [2019-12-17 12:35:28,812] jepsen worker 4 - jepsen.util 9004 :invoke :cas [1 [16 1]] INFO [2019-12-17 12:35:28,818] jepsen worker 31 - jepsen.util 8581 :ok :read [10 25] INFO [2019-12-17 12:35:28,819] jepsen worker 31 - jepsen.util 8581 :invoke :write [10 44] INFO [2019-12-17 12:35:28,819] jepsen worker 34 - jepsen.util 12994 :info :cas [11 [42 45]] :RequestCanceledException INFO [2019-12-17 12:35:28,819] jepsen worker 86 - jepsen.util 13766 :info :write [28 40] :RequestCanceledException INFO [2019-12-17 12:35:28,819] jepsen worker 21 - jepsen.util 12621 :info :write [7 34] :RequestCanceledException INFO [2019-12-17 12:35:28,824] jepsen worker 34 - jepsen.util 13084 :invoke :read [11 nil] INFO [2019-12-17 12:35:28,825] jepsen worker 86 - jepsen.util 13856 :invoke :cas [28 [17 35]] INFO [2019-12-17 12:35:28,825] jepsen worker 21 - jepsen.util 12711 :invoke :read [7 nil] INFO [2019-12-17 12:35:28,831] jepsen worker 67 - jepsen.util 12847 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:35:28,831] jepsen worker 84 - jepsen.util 12594 :info :cas [28 [25 27]] :RequestCanceledException INFO [2019-12-17 12:35:28,831] jepsen worker 72 - jepsen.util 9342 :fail :read [24 nil] :RequestTimeoutException INFO [2019-12-17 12:35:28,831] jepsen worker 74 - jepsen.util 9164 :info :cas [24 [20 39]] :RequestCanceledException INFO [2019-12-17 12:35:28,831] jepsen worker 30 - jepsen.util 9570 :info :cas [10 [20 40]] :RequestCanceledException INFO [2019-12-17 12:35:28,831] jepsen worker 67 - jepsen.util 12847 :invoke :cas [22 [18 30]] INFO [2019-12-17 12:35:28,831] jepsen worker 72 - jepsen.util 9342 :invoke :write [24 46] INFO [2019-12-17 12:35:28,832] jepsen worker 74 - jepsen.util 9254 :invoke :write [24 34] INFO [2019-12-17 12:35:28,835] jepsen worker 30 - jepsen.util 9660 :invoke :read [10 nil] INFO [2019-12-17 12:35:28,838] jepsen worker 69 - jepsen.util 9429 :ok :read [23 32] INFO [2019-12-17 12:35:28,839] jepsen worker 69 - jepsen.util 9429 :invoke :read [23 nil] INFO [2019-12-17 12:35:28,842] jepsen worker 84 - jepsen.util 12684 :invoke :read [28 nil] INFO [2019-12-17 12:35:28,843] jepsen worker 15 - jepsen.util 12165 :info :write [5 2] :RequestCanceledException INFO [2019-12-17 12:35:28,843] jepsen worker 80 - jepsen.util 9080 :info :write [26 6] :RequestCanceledException INFO [2019-12-17 12:35:28,844] jepsen worker 80 - jepsen.util 9170 :invoke :read [26 nil] INFO [2019-12-17 12:35:28,848] jepsen worker 15 - jepsen.util 12255 :invoke :read [5 nil] INFO [2019-12-17 12:35:28,874] jepsen worker 38 - jepsen.util 9938 :fail :cas [12 [16 18]] :ValueNotSwapFrom INFO [2019-12-17 12:35:28,875] jepsen worker 38 - jepsen.util 9938 :invoke :write [12 30] INFO [2019-12-17 12:35:28,879] jepsen worker 54 - jepsen.util 10404 :ok :read [18 25] INFO [2019-12-17 12:35:28,879] jepsen worker 54 - jepsen.util 10404 :invoke :cas [18 [17 7]] INFO [2019-12-17 12:35:28,886] jepsen worker 29 - jepsen.util 7859 :fail :cas [9 [46 47]] :ValueNotSwapFrom INFO [2019-12-17 12:35:28,887] jepsen worker 29 - jepsen.util 7859 :invoke :cas [9 [35 0]] INFO [2019-12-17 12:35:28,888] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55110,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:35:28,889] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55110,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:35:28,889] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55113,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:35:28,889] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55113,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:35:28,945] jepsen worker 42 - jepsen.util 14082 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:35:28,945] jepsen worker 42 - jepsen.util 14082 :invoke :write [14 8] INFO [2019-12-17 12:35:28,960] jepsen worker 70 - jepsen.util 10240 :ok :read [23 32] INFO [2019-12-17 12:35:28,960] jepsen worker 70 - jepsen.util 10240 :invoke :read [23 nil] INFO [2019-12-17 12:35:28,960] jepsen worker 51 - jepsen.util 8691 :info :write [17 30] :RequestCanceledException INFO [2019-12-17 12:35:28,962] jepsen worker 63 - jepsen.util 8433 :ok :write [21 30] INFO [2019-12-17 12:35:28,962] jepsen worker 63 - jepsen.util 8433 :invoke :write [21 31] INFO [2019-12-17 12:35:28,963] jepsen worker 51 - jepsen.util 8781 :invoke :read [17 nil] INFO [2019-12-17 12:35:28,965] jepsen worker 29 - jepsen.util 7859 :fail :cas [9 [35 0]] :ValueNotSwapFrom INFO [2019-12-17 12:35:28,965] jepsen worker 29 - jepsen.util 7859 :invoke :read [9 nil] INFO [2019-12-17 12:35:28,985] jepsen worker 79 - jepsen.util 9709 :fail :cas [26 [23 1]] :ValueNotSwapFrom INFO [2019-12-17 12:35:28,985] jepsen worker 79 - jepsen.util 9709 :invoke :write [26 6] INFO [2019-12-17 12:35:29,014] jepsen worker 43 - jepsen.util 12823 :info :write [14 12] :RequestCanceledException INFO [2019-12-17 12:35:29,018] jepsen worker 43 - jepsen.util 12913 :invoke :write [14 14] INFO [2019-12-17 12:35:29,022] jepsen worker 71 - jepsen.util 10151 :ok :read [23 32] INFO [2019-12-17 12:35:29,022] jepsen worker 71 - jepsen.util 10151 :invoke :write [23 36] INFO [2019-12-17 12:35:29,046] jepsen worker 29 - jepsen.util 7859 :ok :read [9 47] INFO [2019-12-17 12:35:29,047] jepsen worker 29 - jepsen.util 7859 :invoke :read [9 nil] INFO [2019-12-17 12:35:29,062] jepsen worker 60 - jepsen.util 14100 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:35:29,062] jepsen worker 60 - jepsen.util 14100 :invoke :write [20 32] INFO [2019-12-17 12:35:29,069] jepsen worker 29 - jepsen.util 7859 :ok :read [9 47] INFO [2019-12-17 12:35:29,070] jepsen worker 29 - jepsen.util 7859 :invoke :cas [9 [9 40]] INFO [2019-12-17 12:35:29,083] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55125,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:35:29,083] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55125,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:35:29,083] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55129,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:35:29,083] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55129,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:35:29,089] jepsen worker 29 - jepsen.util 7859 :fail :cas [9 [9 40]] :ValueNotSwapFrom INFO [2019-12-17 12:35:29,089] jepsen worker 29 - jepsen.util 7859 :invoke :read [9 nil] INFO [2019-12-17 12:35:29,097] jepsen worker 17 - jepsen.util 12617 :info :write [5 32] :RequestCanceledException INFO [2019-12-17 12:35:29,099] jepsen worker 17 - jepsen.util 12707 :invoke :read [5 nil] INFO [2019-12-17 12:35:29,111] jepsen worker 29 - jepsen.util 7859 :ok :read [9 47] INFO [2019-12-17 12:35:29,111] jepsen worker 29 - jepsen.util 7859 :invoke :cas [9 [9 28]] INFO [2019-12-17 12:35:29,139] jepsen worker 29 - jepsen.util 7859 :fail :cas [9 [9 28]] :ValueNotSwapFrom INFO [2019-12-17 12:35:29,139] jepsen worker 29 - jepsen.util 7859 :invoke :write [9 41] INFO [2019-12-17 12:35:29,143] jepsen worker 89 - jepsen.util 9449 :fail :cas [29 [35 22]] :ValueNotSwapFrom INFO [2019-12-17 12:35:29,143] jepsen worker 89 - jepsen.util 9449 :invoke :read [29 nil] INFO [2019-12-17 12:35:29,145] jepsen worker 57 - jepsen.util 12567 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:35:29,145] jepsen worker 57 - jepsen.util 12567 :invoke :cas [19 [20 5]] INFO [2019-12-17 12:35:29,182] jepsen worker 53 - jepsen.util 8873 :fail :cas [17 [23 41]] :ValueNotSwapFrom INFO [2019-12-17 12:35:29,182] jepsen worker 53 - jepsen.util 8873 :invoke :write [17 25] INFO [2019-12-17 12:35:29,191] jepsen worker 29 - jepsen.util 7859 :ok :write [9 41] INFO [2019-12-17 12:35:29,192] jepsen worker 29 - jepsen.util 7859 :invoke :write [9 39] INFO [2019-12-17 12:35:29,215] jepsen worker 76 - jepsen.util 13216 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:35:29,215] jepsen worker 76 - jepsen.util 13216 :invoke :cas [25 [1 2]] INFO [2019-12-17 12:35:29,257] jepsen worker 29 - jepsen.util 7859 :ok :write [9 39] INFO [2019-12-17 12:35:29,258] jepsen worker 29 - jepsen.util 7859 :invoke :cas [9 [46 19]] INFO [2019-12-17 12:35:29,259] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:35:29,260] jepsen nemesis - couchbase.util Rebalance check count: 431 INFO [2019-12-17 12:35:29,260] jepsen nemesis - couchbase.util Rebalance stuck count: 412 INFO [2019-12-17 12:35:29,262] jepsen worker 44 - jepsen.util 13364 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:35:29,262] jepsen worker 44 - jepsen.util 13364 :invoke :read [14 nil] INFO [2019-12-17 12:35:29,294] jepsen worker 13 - jepsen.util 8923 :ok :read [4 34] INFO [2019-12-17 12:35:29,294] jepsen worker 13 - jepsen.util 8923 :invoke :write [4 14] INFO [2019-12-17 12:35:29,295] jepsen worker 29 - jepsen.util 7859 :fail :cas [9 [46 19]] :ValueNotSwapFrom INFO [2019-12-17 12:35:29,295] jepsen worker 29 - jepsen.util 7859 :invoke :write [9 31] INFO [2019-12-17 12:35:29,301] jepsen worker 88 - jepsen.util 9808 :ok :read [29 40] INFO [2019-12-17 12:35:29,301] jepsen worker 88 - jepsen.util 9808 :invoke :cas [29 [12 27]] INFO [2019-12-17 12:35:29,302] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55143,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:35:29,302] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55152,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:35:29,303] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55143,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:35:29,303] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55152,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:35:29,330] jepsen worker 6 - jepsen.util 13506 :info :write [2 23] :RequestCanceledException INFO [2019-12-17 12:35:29,331] jepsen worker 18 - jepsen.util 9918 :info :write [6 30] :RequestCanceledException INFO [2019-12-17 12:35:29,333] jepsen worker 18 - jepsen.util 10008 :invoke :read [6 nil] INFO [2019-12-17 12:35:29,334] jepsen worker 6 - jepsen.util 13596 :invoke :read [2 nil] INFO [2019-12-17 12:35:29,334] jepsen worker 70 - jepsen.util 10240 :ok :read [23 32] INFO [2019-12-17 12:35:29,335] jepsen worker 70 - jepsen.util 10240 :invoke :write [23 26] INFO [2019-12-17 12:35:29,346] jepsen worker 69 - jepsen.util 9429 :ok :read [23 32] INFO [2019-12-17 12:35:29,346] jepsen worker 69 - jepsen.util 9429 :invoke :cas [23 [45 41]] INFO [2019-12-17 12:35:29,352] jepsen worker 54 - jepsen.util 10404 :fail :cas [18 [17 7]] :ValueNotSwapFrom INFO [2019-12-17 12:35:29,353] jepsen worker 54 - jepsen.util 10404 :invoke :cas [18 [12 3]] INFO [2019-12-17 12:35:29,359] jepsen worker 29 - jepsen.util 7859 :ok :write [9 31] INFO [2019-12-17 12:35:29,359] jepsen worker 29 - jepsen.util 7859 :invoke :cas [9 [40 43]] INFO [2019-12-17 12:35:29,389] jepsen worker 51 - jepsen.util 8781 :ok :read [17 17] INFO [2019-12-17 12:35:29,389] jepsen worker 51 - jepsen.util 8781 :invoke :write [17 19] INFO [2019-12-17 12:35:29,397] jepsen worker 29 - jepsen.util 7859 :fail :cas [9 [40 43]] :ValueNotSwapFrom INFO [2019-12-17 12:35:29,397] jepsen worker 29 - jepsen.util 7859 :invoke :read [9 nil] INFO [2019-12-17 12:35:29,398] jepsen worker 24 - jepsen.util 11904 :info :write [8 34] :RequestCanceledException INFO [2019-12-17 12:35:29,399] jepsen worker 24 - jepsen.util 11994 :invoke :write [8 3] INFO [2019-12-17 12:35:29,415] jepsen worker 11 - jepsen.util 9461 :fail :read [3 nil] :RequestTimeoutException INFO [2019-12-17 12:35:29,416] jepsen worker 11 - jepsen.util 9461 :invoke :cas [3 [24 36]] INFO [2019-12-17 12:35:29,418] jepsen worker 29 - jepsen.util 7859 :ok :read [9 31] INFO [2019-12-17 12:35:29,418] jepsen worker 29 - jepsen.util 7859 :invoke :cas [9 [5 22]] INFO [2019-12-17 12:35:29,430] jepsen worker 35 - jepsen.util 12905 :info :cas [11 [38 29]] :RequestCanceledException INFO [2019-12-17 12:35:29,430] jepsen worker 62 - jepsen.util 13292 :info :write [20 5] :RequestCanceledException INFO [2019-12-17 12:35:29,433] jepsen worker 35 - jepsen.util 12995 :invoke :write [11 44] INFO [2019-12-17 12:35:29,433] jepsen worker 62 - jepsen.util 13382 :invoke :cas [20 [47 6]] INFO [2019-12-17 12:35:29,445] jepsen worker 20 - jepsen.util 9020 :info :write [6 21] :RequestCanceledException INFO [2019-12-17 12:35:29,448] jepsen worker 20 - jepsen.util 9110 :invoke :write [6 24] INFO [2019-12-17 12:35:29,451] jepsen worker 29 - jepsen.util 7859 :fail :cas [9 [5 22]] :ValueNotSwapFrom INFO [2019-12-17 12:35:29,452] jepsen worker 29 - jepsen.util 7859 :invoke :write [9 35] INFO [2019-12-17 12:35:29,509] jepsen worker 88 - jepsen.util 9808 :fail :cas [29 [12 27]] :ValueNotSwapFrom INFO [2019-12-17 12:35:29,509] jepsen worker 88 - jepsen.util 9808 :invoke :read [29 nil] INFO [2019-12-17 12:35:29,514] jepsen worker 29 - jepsen.util 7859 :ok :write [9 35] INFO [2019-12-17 12:35:29,514] jepsen worker 29 - jepsen.util 7859 :invoke :write [9 3] INFO [2019-12-17 12:35:29,516] jepsen worker 23 - jepsen.util 12983 :info :cas [7 [41 17]] :RequestCanceledException INFO [2019-12-17 12:35:29,520] jepsen worker 23 - jepsen.util 13073 :invoke :cas [7 [11 28]] INFO [2019-12-17 12:35:29,522] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55174,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:35:29,522] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55174,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:35:29,522] jepsen worker 69 - jepsen.util 9429 :fail :cas [23 [45 41]] :ValueNotSwapFrom INFO [2019-12-17 12:35:29,523] jepsen worker 69 - jepsen.util 9429 :invoke :cas [23 [13 40]] INFO [2019-12-17 12:35:29,530] jepsen worker 10 - jepsen.util 9550 :info :cas [3 [19 8]] :RequestCanceledException INFO [2019-12-17 12:35:29,533] jepsen worker 10 - jepsen.util 9640 :invoke :write [3 39] INFO [2019-12-17 12:35:29,544] jepsen worker 73 - jepsen.util 8893 :fail :read [24 nil] :RequestTimeoutException INFO [2019-12-17 12:35:29,545] jepsen worker 73 - jepsen.util 8893 :invoke :write [24 46] INFO [2019-12-17 12:35:29,549] jepsen worker 54 - jepsen.util 10404 :fail :cas [18 [12 3]] :ValueNotSwapFrom INFO [2019-12-17 12:35:29,549] jepsen worker 54 - jepsen.util 10404 :invoke :read [18 nil] INFO [2019-12-17 12:35:29,555] jepsen worker 4 - jepsen.util 9004 :fail :cas [1 [16 1]] :ValueNotSwapFrom INFO [2019-12-17 12:35:29,556] jepsen worker 4 - jepsen.util 9004 :invoke :read [1 nil] INFO [2019-12-17 12:35:29,561] jepsen worker 85 - jepsen.util 13765 :info :write [28 22] :RequestCanceledException INFO [2019-12-17 12:35:29,567] jepsen worker 85 - jepsen.util 13855 :invoke :cas [28 [49 29]] INFO [2019-12-17 12:35:29,587] jepsen worker 80 - jepsen.util 9170 :ok :read [26 42] INFO [2019-12-17 12:35:29,587] jepsen worker 80 - jepsen.util 9170 :invoke :write [26 36] INFO [2019-12-17 12:35:29,588] jepsen worker 88 - jepsen.util 9808 :ok :read [29 40] INFO [2019-12-17 12:35:29,588] jepsen worker 88 - jepsen.util 9808 :invoke :read [29 nil] INFO [2019-12-17 12:35:29,593] jepsen worker 69 - jepsen.util 9429 :fail :cas [23 [13 40]] :ValueNotSwapFrom INFO [2019-12-17 12:35:29,593] jepsen worker 69 - jepsen.util 9429 :invoke :cas [23 [6 41]] INFO [2019-12-17 12:35:29,593] jepsen worker 29 - jepsen.util 7859 :ok :write [9 3] INFO [2019-12-17 12:35:29,593] jepsen worker 29 - jepsen.util 7859 :invoke :read [9 nil] INFO [2019-12-17 12:35:29,597] jepsen worker 83 - jepsen.util 13493 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:35:29,597] jepsen worker 83 - jepsen.util 13493 :invoke :write [27 11] INFO [2019-12-17 12:35:29,619] jepsen worker 29 - jepsen.util 7859 :ok :read [9 3] INFO [2019-12-17 12:35:29,620] jepsen worker 29 - jepsen.util 7859 :invoke :write [9 24] INFO [2019-12-17 12:35:29,631] jepsen worker 19 - jepsen.util 9559 :info :write [6 4] :RequestCanceledException INFO [2019-12-17 12:35:29,633] jepsen worker 19 - jepsen.util 9649 :invoke :write [6 34] INFO [2019-12-17 12:35:29,662] jepsen worker 36 - jepsen.util 10116 :info :write [12 13] :RequestCanceledException INFO [2019-12-17 12:35:29,665] jepsen worker 36 - jepsen.util 10206 :invoke :cas [12 [13 34]] INFO [2019-12-17 12:35:29,666] jepsen worker 4 - jepsen.util 9004 :ok :read [1 47] INFO [2019-12-17 12:35:29,667] jepsen worker 4 - jepsen.util 9004 :invoke :cas [1 [6 47]] INFO [2019-12-17 12:35:29,683] jepsen worker 29 - jepsen.util 7859 :ok :write [9 24] INFO [2019-12-17 12:35:29,684] jepsen worker 29 - jepsen.util 7859 :invoke :write [9 31] INFO [2019-12-17 12:35:29,697] jepsen worker 12 - jepsen.util 9822 :info :write [4 6] :RequestCanceledException INFO [2019-12-17 12:35:29,700] jepsen worker 12 - jepsen.util 9912 :invoke :cas [4 [21 23]] INFO [2019-12-17 12:35:29,730] jepsen worker 49 - jepsen.util 12649 :info :cas [16 [1 47]] :RequestCanceledException INFO [2019-12-17 12:35:29,731] jepsen worker 3 - jepsen.util 9273 :info :write [1 48] :RequestCanceledException INFO [2019-12-17 12:35:29,733] jepsen worker 29 - jepsen.util 7859 :ok :write [9 31] INFO [2019-12-17 12:35:29,733] jepsen worker 49 - jepsen.util 12739 :invoke :write [16 45] INFO [2019-12-17 12:35:29,733] jepsen worker 29 - jepsen.util 7859 :invoke :cas [9 [26 38]] INFO [2019-12-17 12:35:29,733] jepsen worker 3 - jepsen.util 9363 :invoke :cas [1 [12 6]] INFO [2019-12-17 12:35:29,740] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55199,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:35:29,741] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55199,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:35:29,741] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55202,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:35:29,741] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55202,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:35:29,741] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55203,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:35:29,741] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55203,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:35:29,745] jepsen worker 45 - jepsen.util 9765 :fail :read [15 nil] :RequestTimeoutException INFO [2019-12-17 12:35:29,745] jepsen worker 40 - jepsen.util 13270 :info :cas [13 [38 45]] :RequestCanceledException INFO [2019-12-17 12:35:29,745] jepsen worker 45 - jepsen.util 9765 :invoke :cas [15 [34 23]] INFO [2019-12-17 12:35:29,748] jepsen worker 40 - jepsen.util 13360 :invoke :read [13 nil] INFO [2019-12-17 12:35:29,765] jepsen worker 29 - jepsen.util 7859 :fail :cas [9 [26 38]] :ValueNotSwapFrom INFO [2019-12-17 12:35:29,765] jepsen worker 29 - jepsen.util 7859 :invoke :read [9 nil] INFO [2019-12-17 12:35:29,784] jepsen worker 29 - jepsen.util 7859 :ok :read [9 31] INFO [2019-12-17 12:35:29,784] jepsen worker 29 - jepsen.util 7859 :invoke :write [9 20] INFO [2019-12-17 12:35:29,796] jepsen worker 66 - jepsen.util 12756 :info :write [22 22] :RequestCanceledException INFO [2019-12-17 12:35:29,798] jepsen worker 66 - jepsen.util 12846 :invoke :write [22 11] INFO [2019-12-17 12:35:29,810] jepsen worker 46 - jepsen.util 9496 :info :write [15 39] :SyncWriteAmbiguous INFO [2019-12-17 12:35:29,811] jepsen worker 46 - jepsen.util 9586 :invoke :cas [15 [45 39]] INFO [2019-12-17 12:35:29,815] jepsen worker 78 - jepsen.util 9348 :info :write [26 10] :RequestCanceledException INFO [2019-12-17 12:35:29,819] jepsen worker 78 - jepsen.util 9438 :invoke :write [26 25] INFO [2019-12-17 12:35:29,822] jepsen worker 89 - jepsen.util 9449 :ok :read [29 40] INFO [2019-12-17 12:35:29,822] jepsen worker 89 - jepsen.util 9449 :invoke :read [29 nil] INFO [2019-12-17 12:35:29,830] jepsen worker 26 - jepsen.util 13526 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:35:29,830] jepsen worker 26 - jepsen.util 13526 :invoke :cas [8 [8 45]] INFO [2019-12-17 12:35:29,837] jepsen worker 54 - jepsen.util 10404 :ok :read [18 25] INFO [2019-12-17 12:35:29,837] jepsen worker 54 - jepsen.util 10404 :invoke :cas [18 [12 26]] INFO [2019-12-17 12:35:29,848] jepsen worker 29 - jepsen.util 7859 :ok :write [9 20] INFO [2019-12-17 12:35:29,848] jepsen worker 29 - jepsen.util 7859 :invoke :write [9 13] INFO [2019-12-17 12:35:29,852] jepsen worker 88 - jepsen.util 9808 :ok :read [29 40] INFO [2019-12-17 12:35:29,852] jepsen worker 88 - jepsen.util 9808 :invoke :write [29 31] INFO [2019-12-17 12:35:29,855] jepsen worker 65 - jepsen.util 7985 :ok :write [21 5] INFO [2019-12-17 12:35:29,856] jepsen worker 65 - jepsen.util 7985 :invoke :write [21 6] INFO [2019-12-17 12:35:29,873] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55223,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:35:29,873] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55223,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:35:29,873] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55227,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:35:29,873] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55227,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:35:29,887] jepsen worker 69 - jepsen.util 9429 :fail :cas [23 [6 41]] :ValueNotSwapFrom INFO [2019-12-17 12:35:29,887] jepsen worker 32 - jepsen.util 9482 :info :write [10 19] :SyncWriteAmbiguous INFO [2019-12-17 12:35:29,887] jepsen worker 69 - jepsen.util 9429 :invoke :write [23 4] INFO [2019-12-17 12:35:29,890] jepsen worker 32 - jepsen.util 9572 :invoke :write [10 26] INFO [2019-12-17 12:35:29,914] jepsen worker 61 - jepsen.util 14011 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:35:29,914] jepsen worker 61 - jepsen.util 14011 :invoke :write [20 40] INFO [2019-12-17 12:35:29,931] jepsen worker 82 - jepsen.util 13762 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:35:29,931] jepsen worker 7 - jepsen.util 13057 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:35:29,931] jepsen worker 82 - jepsen.util 13762 :invoke :read [27 nil] INFO [2019-12-17 12:35:29,931] jepsen worker 7 - jepsen.util 13057 :invoke :cas [2 [4 5]] INFO [2019-12-17 12:35:29,944] jepsen worker 68 - jepsen.util 12488 :info :write [22 34] :RequestCanceledException INFO [2019-12-17 12:35:29,948] jepsen worker 54 - jepsen.util 10404 :fail :cas [18 [12 26]] :ValueNotSwapFrom INFO [2019-12-17 12:35:29,948] jepsen worker 54 - jepsen.util 10404 :invoke :write [18 37] INFO [2019-12-17 12:35:29,948] jepsen worker 68 - jepsen.util 12578 :invoke :write [22 0] INFO [2019-12-17 12:35:29,962] jepsen worker 52 - jepsen.util 9412 :info :write [17 44] :RequestCanceledException INFO [2019-12-17 12:35:29,962] jepsen worker 0 - jepsen.util 13680 :info :write [0 9] :RequestCanceledException INFO [2019-12-17 12:35:29,963] jepsen worker 46 - jepsen.util 9586 :fail :cas [15 [45 39]] :ValueNotSwapFrom INFO [2019-12-17 12:35:29,963] jepsen worker 46 - jepsen.util 9586 :invoke :read [15 nil] INFO [2019-12-17 12:35:29,964] jepsen worker 52 - jepsen.util 9502 :invoke :write [17 16] INFO [2019-12-17 12:35:29,964] jepsen worker 11 - jepsen.util 9461 :fail :cas [3 [24 36]] :ValueNotSwapFrom INFO [2019-12-17 12:35:29,964] jepsen worker 11 - jepsen.util 9461 :invoke :read [3 nil] INFO [2019-12-17 12:35:29,965] jepsen worker 0 - jepsen.util 13770 :invoke :write [0 4] INFO [2019-12-17 12:35:29,992] jepsen worker 29 - jepsen.util 7859 :ok :write [9 13] INFO [2019-12-17 12:35:29,992] jepsen worker 29 - jepsen.util 7859 :invoke :cas [9 [21 41]] INFO [2019-12-17 12:35:29,996] jepsen worker 1 - jepsen.util 13231 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:35:29,996] jepsen worker 1 - jepsen.util 13231 :invoke :read [0 nil] INFO [2019-12-17 12:35:29,997] jepsen worker 46 - jepsen.util 9586 :ok :read [15 37] INFO [2019-12-17 12:35:29,997] jepsen worker 46 - jepsen.util 9586 :invoke :write [15 21] INFO [2019-12-17 12:35:30,015] jepsen worker 25 - jepsen.util 14155 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:35:30,015] jepsen worker 58 - jepsen.util 13558 :info :write [19 26] :RequestCanceledException INFO [2019-12-17 12:35:30,015] jepsen worker 37 - jepsen.util 8947 :info :write [12 11] :RequestCanceledException INFO [2019-12-17 12:35:30,016] jepsen worker 25 - jepsen.util 14155 :invoke :cas [8 [1 27]] INFO [2019-12-17 12:35:30,017] jepsen worker 11 - jepsen.util 9461 :ok :read [3 36] INFO [2019-12-17 12:35:30,017] jepsen worker 11 - jepsen.util 9461 :invoke :read [3 nil] INFO [2019-12-17 12:35:30,019] jepsen worker 37 - jepsen.util 9037 :invoke :read [12 nil] INFO [2019-12-17 12:35:30,019] jepsen worker 58 - jepsen.util 13648 :invoke :write [19 7] INFO [2019-12-17 12:35:30,031] jepsen worker 75 - jepsen.util 12495 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:35:30,031] jepsen worker 75 - jepsen.util 12495 :invoke :write [25 36] INFO [2019-12-17 12:35:30,036] jepsen worker 29 - jepsen.util 7859 :fail :cas [9 [21 41]] :ValueNotSwapFrom INFO [2019-12-17 12:35:30,036] jepsen worker 29 - jepsen.util 7859 :invoke :cas [9 [27 17]] INFO [2019-12-17 12:35:30,062] jepsen worker 64 - jepsen.util 7444 :fail :read [21 nil] :RequestTimeoutException INFO [2019-12-17 12:35:30,062] jepsen worker 64 - jepsen.util 7444 :invoke :read [21 nil] INFO [2019-12-17 12:35:30,066] jepsen worker 36 - jepsen.util 10206 :fail :cas [12 [13 34]] :ValueNotSwapFrom INFO [2019-12-17 12:35:30,066] jepsen worker 36 - jepsen.util 10206 :invoke :read [12 nil] INFO [2019-12-17 12:35:30,067] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55245,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:35:30,068] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55245,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:35:30,068] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55248,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:35:30,068] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55248,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:35:30,079] jepsen worker 29 - jepsen.util 7859 :fail :cas [9 [27 17]] :ValueNotSwapFrom INFO [2019-12-17 12:35:30,079] jepsen worker 29 - jepsen.util 7859 :invoke :read [9 nil] INFO [2019-12-17 12:35:30,091] jepsen worker 64 - jepsen.util 7444 :ok :read [21 5] INFO [2019-12-17 12:35:30,091] jepsen worker 64 - jepsen.util 7444 :invoke :read [21 nil] INFO [2019-12-17 12:35:30,102] jepsen worker 11 - jepsen.util 9461 :ok :read [3 36] INFO [2019-12-17 12:35:30,103] jepsen worker 11 - jepsen.util 9461 :invoke :cas [3 [8 38]] INFO [2019-12-17 12:35:30,111] jepsen worker 29 - jepsen.util 7859 :ok :read [9 13] INFO [2019-12-17 12:35:30,111] jepsen worker 29 - jepsen.util 7859 :invoke :write [9 12] INFO [2019-12-17 12:35:30,136] jepsen worker 64 - jepsen.util 7444 :ok :read [21 5] INFO [2019-12-17 12:35:30,136] jepsen worker 64 - jepsen.util 7444 :invoke :read [21 nil] INFO [2019-12-17 12:35:30,139] jepsen worker 11 - jepsen.util 9461 :fail :cas [3 [8 38]] :ValueNotSwapFrom INFO [2019-12-17 12:35:30,139] jepsen worker 11 - jepsen.util 9461 :invoke :write [3 11] INFO [2019-12-17 12:35:30,161] jepsen worker 56 - jepsen.util 9956 :info :write [18 1] :RequestCanceledException INFO [2019-12-17 12:35:30,163] jepsen worker 56 - jepsen.util 10046 :invoke :cas [18 [11 38]] INFO [2019-12-17 12:35:30,176] jepsen worker 37 - jepsen.util 9037 :ok :read [12 41] INFO [2019-12-17 12:35:30,176] jepsen worker 37 - jepsen.util 9037 :invoke :write [12 20] INFO [2019-12-17 12:35:30,177] jepsen worker 29 - jepsen.util 7859 :ok :write [9 12] INFO [2019-12-17 12:35:30,177] jepsen worker 29 - jepsen.util 7859 :invoke :read [9 nil] INFO [2019-12-17 12:35:30,208] jepsen worker 64 - jepsen.util 7444 :ok :read [21 5] INFO [2019-12-17 12:35:30,209] jepsen worker 64 - jepsen.util 7444 :invoke :write [21 5] INFO [2019-12-17 12:35:30,215] jepsen worker 36 - jepsen.util 10206 :ok :read [12 41] INFO [2019-12-17 12:35:30,215] jepsen worker 36 - jepsen.util 10206 :invoke :read [12 nil] INFO [2019-12-17 12:35:30,225] jepsen worker 29 - jepsen.util 7859 :ok :read [9 12] INFO [2019-12-17 12:35:30,225] jepsen worker 29 - jepsen.util 7859 :invoke :write [9 32] INFO [2019-12-17 12:35:30,230] jepsen worker 9 - jepsen.util 9999 :info :write [3 8] :RequestCanceledException INFO [2019-12-17 12:35:30,232] jepsen worker 9 - jepsen.util 10089 :invoke :cas [3 [37 5]] INFO [2019-12-17 12:35:30,261] jepsen worker 64 - jepsen.util 7444 :ok :write [21 5] INFO [2019-12-17 12:35:30,262] jepsen worker 64 - jepsen.util 7444 :invoke :write [21 44] INFO [2019-12-17 12:35:30,262] jepsen worker 81 - jepsen.util 12861 :info :cas [27 [24 41]] :RequestCanceledException INFO [2019-12-17 12:35:30,265] jepsen worker 81 - jepsen.util 12951 :invoke :read [27 nil] INFO [2019-12-17 12:35:30,277] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:35:30,277] jepsen nemesis - couchbase.util Rebalance check count: 432 INFO [2019-12-17 12:35:30,277] jepsen nemesis - couchbase.util Rebalance stuck count: 413 INFO [2019-12-17 12:35:30,286] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55267,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:35:30,286] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55274,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:35:30,286] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55267,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:35:30,286] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55274,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:35:30,286] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55268,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:35:30,287] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55276,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:35:30,287] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55268,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:35:30,287] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55276,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:35:30,287] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55269,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:35:30,287] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55269,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:35:30,290] jepsen worker 89 - jepsen.util 9449 :ok :read [29 40] INFO [2019-12-17 12:35:30,290] jepsen worker 89 - jepsen.util 9449 :invoke :write [29 48] INFO [2019-12-17 12:35:30,321] jepsen worker 64 - jepsen.util 7444 :ok :write [21 44] INFO [2019-12-17 12:35:30,322] jepsen worker 64 - jepsen.util 7444 :invoke :cas [21 [16 10]] INFO [2019-12-17 12:35:30,330] jepsen worker 48 - jepsen.util 12108 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:35:30,330] jepsen worker 48 - jepsen.util 12108 :invoke :read [16 nil] INFO [2019-12-17 12:35:30,358] jepsen worker 64 - jepsen.util 7444 :fail :cas [21 [16 10]] :ValueNotSwapFrom INFO [2019-12-17 12:35:30,358] jepsen worker 64 - jepsen.util 7444 :invoke :write [21 24] INFO [2019-12-17 12:35:30,363] jepsen worker 4 - jepsen.util 9004 :fail :cas [1 [6 47]] :ValueNotSwapFrom INFO [2019-12-17 12:35:30,364] jepsen worker 4 - jepsen.util 9004 :invoke :write [1 12] INFO [2019-12-17 12:35:30,381] jepsen worker 12 - jepsen.util 9912 :fail :cas [4 [21 23]] :ValueNotSwapFrom INFO [2019-12-17 12:35:30,381] jepsen worker 12 - jepsen.util 9912 :invoke :cas [4 [27 35]] INFO [2019-12-17 12:35:30,393] jepsen worker 39 - jepsen.util 12549 :info :cas [13 [39 29]] :RequestCanceledException INFO [2019-12-17 12:35:30,393] jepsen worker 27 - jepsen.util 9027 :info :write [9 22] :RequestCanceledException INFO [2019-12-17 12:35:30,395] jepsen worker 39 - jepsen.util 12639 :invoke :read [13 nil] INFO [2019-12-17 12:35:30,395] jepsen worker 27 - jepsen.util 9117 :invoke :write [9 30] INFO [2019-12-17 12:35:30,413] jepsen worker 64 - jepsen.util 7444 :ok :write [21 24] INFO [2019-12-17 12:35:30,413] jepsen worker 64 - jepsen.util 7444 :invoke :write [21 12] INFO [2019-12-17 12:35:30,420] jepsen worker 36 - jepsen.util 10206 :ok :read [12 41] INFO [2019-12-17 12:35:30,420] jepsen worker 36 - jepsen.util 10206 :invoke :cas [12 [36 16]] INFO [2019-12-17 12:35:30,461] jepsen worker 64 - jepsen.util 7444 :ok :write [21 12] INFO [2019-12-17 12:35:30,461] jepsen worker 64 - jepsen.util 7444 :invoke :cas [21 [39 31]] INFO [2019-12-17 12:35:30,462] jepsen worker 55 - jepsen.util 10225 :info :write [18 10] :RequestCanceledException INFO [2019-12-17 12:35:30,464] jepsen worker 55 - jepsen.util 10315 :invoke :read [18 nil] INFO [2019-12-17 12:35:30,497] jepsen worker 64 - jepsen.util 7444 :fail :cas [21 [39 31]] :ValueNotSwapFrom INFO [2019-12-17 12:35:30,497] jepsen worker 64 - jepsen.util 7444 :invoke :read [21 nil] INFO [2019-12-17 12:35:30,498] jepsen worker 28 - jepsen.util 7678 :info :write [9 15] :RequestCanceledException INFO [2019-12-17 12:35:30,500] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55296,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:35:30,500] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55296,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:35:30,500] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55300,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:35:30,500] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55300,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:35:30,501] jepsen worker 28 - jepsen.util 7768 :invoke :cas [9 [18 10]] INFO [2019-12-17 12:35:30,515] jepsen worker 5 - jepsen.util 8825 :info :write [1 48] :RequestCanceledException INFO [2019-12-17 12:35:30,517] jepsen worker 5 - jepsen.util 8915 :invoke :write [1 26] INFO [2019-12-17 12:35:30,524] jepsen worker 64 - jepsen.util 7444 :ok :read [21 12] INFO [2019-12-17 12:35:30,524] jepsen worker 64 - jepsen.util 7444 :invoke :cas [21 [25 9]] INFO [2019-12-17 12:35:30,567] jepsen worker 36 - jepsen.util 10206 :fail :cas [12 [36 16]] :ValueNotSwapFrom INFO [2019-12-17 12:35:30,567] jepsen worker 36 - jepsen.util 10206 :invoke :read [12 nil] INFO [2019-12-17 12:35:30,568] jepsen worker 64 - jepsen.util 7444 :fail :cas [21 [25 9]] :ValueNotSwapFrom INFO [2019-12-17 12:35:30,568] jepsen worker 64 - jepsen.util 7444 :invoke :write [21 26] INFO [2019-12-17 12:35:30,615] jepsen worker 33 - jepsen.util 12993 :info :write [11 15] :RequestCanceledException INFO [2019-12-17 12:35:30,617] jepsen worker 33 - jepsen.util 13083 :invoke :read [11 nil] INFO [2019-12-17 12:35:30,646] jepsen worker 47 - jepsen.util 9947 :info :write [15 16] :RequestCanceledException INFO [2019-12-17 12:35:30,647] jepsen worker 36 - jepsen.util 10206 :ok :read [12 41] INFO [2019-12-17 12:35:30,648] jepsen worker 36 - jepsen.util 10206 :invoke :cas [12 [2 34]] INFO [2019-12-17 12:35:30,648] jepsen worker 47 - jepsen.util 10037 :invoke :write [15 14] INFO [2019-12-17 12:35:30,654] jepsen worker 64 - jepsen.util 7444 :ok :write [21 26] INFO [2019-12-17 12:35:30,654] jepsen worker 64 - jepsen.util 7444 :invoke :read [21 nil] INFO [2019-12-17 12:35:30,691] jepsen worker 64 - jepsen.util 7444 :ok :read [21 26] INFO [2019-12-17 12:35:30,692] jepsen worker 64 - jepsen.util 7444 :invoke :cas [21 [19 22]] INFO [2019-12-17 12:35:30,696] jepsen worker 3 - jepsen.util 9363 :fail :cas [1 [12 6]] :ValueNotSwapFrom INFO [2019-12-17 12:35:30,696] jepsen worker 3 - jepsen.util 9363 :invoke :cas [1 [39 24]] INFO [2019-12-17 12:35:30,719] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55311,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:35:30,719] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55311,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:35:30,719] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55315,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:35:30,720] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55315,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:35:30,726] jepsen worker 64 - jepsen.util 7444 :fail :cas [21 [19 22]] :ValueNotSwapFrom INFO [2019-12-17 12:35:30,726] jepsen worker 64 - jepsen.util 7444 :invoke :read [21 nil] INFO [2019-12-17 12:35:30,731] jepsen worker 87 - jepsen.util 9087 :info :write [29 7] :RequestCanceledException INFO [2019-12-17 12:35:30,733] jepsen worker 87 - jepsen.util 9177 :invoke :read [29 nil] INFO [2019-12-17 12:35:30,745] jepsen worker 22 - jepsen.util 12892 :info :cas [7 [45 2]] :RequestCanceledException INFO [2019-12-17 12:35:30,745] jepsen worker 59 - jepsen.util 12299 :info :write [19 15] :RequestCanceledException INFO [2019-12-17 12:35:30,749] jepsen worker 22 - jepsen.util 12982 :invoke :cas [7 [31 4]] INFO [2019-12-17 12:35:30,749] jepsen worker 59 - jepsen.util 12389 :invoke :cas [19 [19 23]] INFO [2019-12-17 12:35:30,753] jepsen worker 64 - jepsen.util 7444 :ok :read [21 26] INFO [2019-12-17 12:35:30,753] jepsen worker 64 - jepsen.util 7444 :invoke :read [21 nil] INFO [2019-12-17 12:35:30,759] jepsen worker 50 - jepsen.util 13730 :info :write [16 21] :RequestCanceledException INFO [2019-12-17 12:35:30,762] jepsen worker 50 - jepsen.util 13820 :invoke :write [16 42] INFO [2019-12-17 12:35:30,785] jepsen worker 56 - jepsen.util 10046 :fail :cas [18 [11 38]] :ValueNotSwapFrom INFO [2019-12-17 12:35:30,785] jepsen worker 56 - jepsen.util 10046 :invoke :cas [18 [31 37]] INFO [2019-12-17 12:35:30,788] jepsen worker 64 - jepsen.util 7444 :ok :read [21 26] INFO [2019-12-17 12:35:30,788] jepsen worker 64 - jepsen.util 7444 :invoke :write [21 28] INFO [2019-12-17 12:35:30,815] jepsen worker 77 - jepsen.util 13577 :info :cas [25 [14 41]] :RequestCanceledException INFO [2019-12-17 12:35:30,815] jepsen worker 8 - jepsen.util 12698 :info :write [2 46] :RequestCanceledException INFO [2019-12-17 12:35:30,817] jepsen worker 77 - jepsen.util 13667 :invoke :cas [25 [33 5]] INFO [2019-12-17 12:35:30,817] jepsen worker 8 - jepsen.util 12788 :invoke :read [2 nil] INFO [2019-12-17 12:35:30,829] jepsen worker 14 - jepsen.util 9464 :info :write [4 48] :RequestCanceledException INFO [2019-12-17 12:35:30,829] jepsen worker 41 - jepsen.util 13991 :info :cas [13 [16 48]] :RequestCanceledException INFO [2019-12-17 12:35:30,831] jepsen worker 14 - jepsen.util 9554 :invoke :cas [4 [10 22]] INFO [2019-12-17 12:35:30,832] jepsen worker 41 - jepsen.util 14081 :invoke :cas [13 [6 48]] INFO [2019-12-17 12:35:30,836] jepsen worker 72 - jepsen.util 9342 :info :write [24 46] :SyncWriteAmbiguous INFO [2019-12-17 12:35:30,838] jepsen worker 72 - jepsen.util 9432 :invoke :read [24 nil] INFO [2019-12-17 12:35:30,845] jepsen worker 2 - jepsen.util 12962 :info :cas [0 [18 29]] :RequestCanceledException INFO [2019-12-17 12:35:30,847] jepsen worker 2 - jepsen.util 13052 :invoke :write [0 9] INFO [2019-12-17 12:35:30,854] jepsen worker 64 - jepsen.util 7444 :ok :write [21 28] INFO [2019-12-17 12:35:30,854] jepsen worker 64 - jepsen.util 7444 :invoke :read [21 nil] INFO [2019-12-17 12:35:30,862] jepsen worker 34 - jepsen.util 13084 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:35:30,862] jepsen worker 34 - jepsen.util 13084 :invoke :read [11 nil] INFO [2019-12-17 12:35:30,888] jepsen worker 64 - jepsen.util 7444 :ok :read [21 28] INFO [2019-12-17 12:35:30,888] jepsen worker 64 - jepsen.util 7444 :invoke :write [21 41] INFO [2019-12-17 12:35:30,898] jepsen worker 16 - jepsen.util 14146 :info :cas [5 [29 36]] :RequestCanceledException INFO [2019-12-17 12:35:30,898] jepsen worker 31 - jepsen.util 8581 :info :write [10 44] :RequestCanceledException INFO [2019-12-17 12:35:30,898] jepsen worker 21 - jepsen.util 12711 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:35:30,898] jepsen worker 21 - jepsen.util 12711 :invoke :read [7 nil] INFO [2019-12-17 12:35:30,901] jepsen worker 31 - jepsen.util 8671 :invoke :cas [10 [48 10]] INFO [2019-12-17 12:35:30,902] jepsen worker 16 - jepsen.util 14236 :invoke :write [5 18] INFO [2019-12-17 12:35:30,915] jepsen worker 74 - jepsen.util 9254 :info :write [24 34] :RequestCanceledException INFO [2019-12-17 12:35:30,918] jepsen worker 74 - jepsen.util 9344 :invoke :cas [24 [15 4]] INFO [2019-12-17 12:35:30,922] jepsen worker 36 - jepsen.util 10206 :fail :cas [12 [2 34]] :ValueNotSwapFrom INFO [2019-12-17 12:35:30,922] jepsen worker 36 - jepsen.util 10206 :invoke :write [12 41] INFO [2019-12-17 12:35:30,928] jepsen worker 67 - jepsen.util 12847 :info :cas [22 [18 30]] :RequestCanceledException INFO [2019-12-17 12:35:30,928] jepsen worker 30 - jepsen.util 9660 :fail :read [10 nil] :RequestTimeoutException INFO [2019-12-17 12:35:30,928] jepsen worker 86 - jepsen.util 13856 :info :cas [28 [17 35]] :RequestCanceledException INFO [2019-12-17 12:35:30,928] jepsen worker 15 - jepsen.util 12255 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:35:30,928] jepsen worker 30 - jepsen.util 9660 :invoke :read [10 nil] INFO [2019-12-17 12:35:30,928] jepsen worker 15 - jepsen.util 12255 :invoke :cas [5 [4 19]] INFO [2019-12-17 12:35:30,931] jepsen worker 86 - jepsen.util 13946 :invoke :read [28 nil] INFO [2019-12-17 12:35:30,931] jepsen worker 67 - jepsen.util 12937 :invoke :cas [22 [30 48]] INFO [2019-12-17 12:35:30,944] jepsen worker 84 - jepsen.util 12684 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:35:30,944] jepsen worker 38 - jepsen.util 9938 :info :write [12 30] :RequestCanceledException INFO [2019-12-17 12:35:30,945] jepsen worker 84 - jepsen.util 12684 :invoke :cas [28 [37 32]] INFO [2019-12-17 12:35:30,945] jepsen worker 31 - jepsen.util 8671 :fail :cas [10 [48 10]] :ValueNotSwapFrom INFO [2019-12-17 12:35:30,945] jepsen worker 31 - jepsen.util 8671 :invoke :read [10 nil] INFO [2019-12-17 12:35:30,947] jepsen worker 38 - jepsen.util 10028 :invoke :write [12 1] INFO [2019-12-17 12:35:30,968] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55339,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:35:30,969] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55339,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:35:30,969] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55342,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:35:30,969] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55342,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:35:30,972] jepsen worker 31 - jepsen.util 8671 :ok :read [10 25] INFO [2019-12-17 12:35:30,973] jepsen worker 31 - jepsen.util 8671 :invoke :write [10 26] INFO [2019-12-17 12:35:30,989] jepsen worker 64 - jepsen.util 7444 :ok :write [21 41] INFO [2019-12-17 12:35:30,989] jepsen worker 64 - jepsen.util 7444 :invoke :read [21 nil] INFO [2019-12-17 12:35:30,998] jepsen worker 79 - jepsen.util 9709 :info :write [26 6] :RequestCanceledException INFO [2019-12-17 12:35:31,000] jepsen worker 79 - jepsen.util 9799 :invoke :read [26 nil] INFO [2019-12-17 12:35:31,021] jepsen worker 64 - jepsen.util 7444 :ok :read [21 41] INFO [2019-12-17 12:35:31,021] jepsen worker 64 - jepsen.util 7444 :invoke :read [21 nil] INFO [2019-12-17 12:35:31,022] jepsen worker 30 - jepsen.util 9660 :ok :read [10 25] INFO [2019-12-17 12:35:31,022] jepsen worker 30 - jepsen.util 9660 :invoke :cas [10 [48 47]] INFO [2019-12-17 12:35:31,031] jepsen worker 63 - jepsen.util 8433 :info :write [21 31] :RequestCanceledException INFO [2019-12-17 12:35:31,033] jepsen worker 63 - jepsen.util 8523 :invoke :read [21 nil] INFO [2019-12-17 12:35:31,045] jepsen worker 42 - jepsen.util 14082 :info :write [14 8] :RequestCanceledException INFO [2019-12-17 12:35:31,045] jepsen worker 71 - jepsen.util 10151 :info :write [23 36] :RequestCanceledException INFO [2019-12-17 12:35:31,046] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55358,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:35:31,046] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55358,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:35:31,047] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55362,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:35:31,047] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55362,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:35:31,047] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55363,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:35:31,047] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55363,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:35:31,049] jepsen worker 71 - jepsen.util 10241 :invoke :read [23 nil] INFO [2019-12-17 12:35:31,049] jepsen worker 42 - jepsen.util 14172 :invoke :write [14 17] INFO [2019-12-17 12:35:31,072] jepsen worker 30 - jepsen.util 9660 :fail :cas [10 [48 47]] :ValueNotSwapFrom INFO [2019-12-17 12:35:31,072] jepsen worker 30 - jepsen.util 9660 :invoke :write [10 20] INFO [2019-12-17 12:35:31,077] jepsen worker 64 - jepsen.util 7444 :ok :read [21 41] INFO [2019-12-17 12:35:31,077] jepsen worker 64 - jepsen.util 7444 :invoke :read [21 nil] INFO [2019-12-17 12:35:31,107] jepsen worker 64 - jepsen.util 7444 :ok :read [21 41] INFO [2019-12-17 12:35:31,107] jepsen worker 64 - jepsen.util 7444 :invoke :cas [21 [15 19]] INFO [2019-12-17 12:35:31,114] jepsen worker 43 - jepsen.util 12913 :info :write [14 14] :RequestCanceledException INFO [2019-12-17 12:35:31,116] jepsen worker 43 - jepsen.util 13003 :invoke :cas [14 [6 25]] INFO [2019-12-17 12:35:31,131] jepsen worker 17 - jepsen.util 12707 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:35:31,132] jepsen worker 17 - jepsen.util 12707 :invoke :write [5 35] INFO [2019-12-17 12:35:31,155] jepsen worker 87 - jepsen.util 9177 :ok :read [29 40] INFO [2019-12-17 12:35:31,155] jepsen worker 87 - jepsen.util 9177 :invoke :read [29 nil] INFO [2019-12-17 12:35:31,158] jepsen worker 56 - jepsen.util 10046 :fail :cas [18 [31 37]] :ValueNotSwapFrom INFO [2019-12-17 12:35:31,158] jepsen worker 56 - jepsen.util 10046 :invoke :write [18 0] INFO [2019-12-17 12:35:31,162] jepsen worker 60 - jepsen.util 14100 :info :write [20 32] :RequestCanceledException INFO [2019-12-17 12:35:31,164] jepsen worker 64 - jepsen.util 7444 :fail :cas [21 [15 19]] :ValueNotSwapFrom INFO [2019-12-17 12:35:31,164] jepsen worker 64 - jepsen.util 7444 :invoke :write [21 27] INFO [2019-12-17 12:35:31,165] jepsen worker 60 - jepsen.util 14190 :invoke :write [20 44] INFO [2019-12-17 12:35:31,186] jepsen worker 79 - jepsen.util 9799 :ok :read [26 42] INFO [2019-12-17 12:35:31,186] jepsen worker 79 - jepsen.util 9799 :invoke :read [26 nil] INFO [2019-12-17 12:35:31,231] jepsen worker 53 - jepsen.util 8873 :info :write [17 25] :RequestCanceledException INFO [2019-12-17 12:35:31,238] jepsen worker 64 - jepsen.util 7444 :ok :write [21 27] INFO [2019-12-17 12:35:31,244] jepsen worker 64 - jepsen.util 7444 :invoke :read [21 nil] INFO [2019-12-17 12:35:31,245] jepsen worker 57 - jepsen.util 12567 :info :cas [19 [20 5]] :RequestCanceledException INFO [2019-12-17 12:35:31,247] jepsen worker 53 - jepsen.util 8963 :invoke :write [17 34] INFO [2019-12-17 12:35:31,247] jepsen worker 57 - jepsen.util 12657 :invoke :write [19 42] INFO [2019-12-17 12:35:31,249] jepsen worker 14 - jepsen.util 9554 :fail :cas [4 [10 22]] :ValueNotSwapFrom INFO [2019-12-17 12:35:31,250] jepsen worker 14 - jepsen.util 9554 :invoke :cas [4 [21 43]] INFO [2019-12-17 12:35:31,265] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55389,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:35:31,266] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55389,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:35:31,266] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55391,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:35:31,266] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55391,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:35:31,268] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55380,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:35:31,268] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55380,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:35:31,268] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55381,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:35:31,268] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55381,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:35:31,272] jepsen worker 64 - jepsen.util 7444 :ok :read [21 27] INFO [2019-12-17 12:35:31,274] jepsen worker 64 - jepsen.util 7444 :invoke :cas [21 [13 0]] INFO [2019-12-17 12:35:31,292] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:35:31,292] jepsen nemesis - couchbase.util Rebalance check count: 433 INFO [2019-12-17 12:35:31,292] jepsen nemesis - couchbase.util Rebalance stuck count: 414 INFO [2019-12-17 12:35:31,308] jepsen worker 76 - jepsen.util 13216 :info :cas [25 [1 2]] :RequestCanceledException INFO [2019-12-17 12:35:31,309] jepsen worker 64 - jepsen.util 7444 :fail :cas [21 [13 0]] :ValueNotSwapFrom INFO [2019-12-17 12:35:31,309] jepsen worker 64 - jepsen.util 7444 :invoke :read [21 nil] INFO [2019-12-17 12:35:31,311] jepsen worker 76 - jepsen.util 13306 :invoke :cas [25 [42 3]] INFO [2019-12-17 12:35:31,325] jepsen worker 3 - jepsen.util 9363 :fail :cas [1 [39 24]] :ValueNotSwapFrom INFO [2019-12-17 12:35:31,325] jepsen worker 3 - jepsen.util 9363 :invoke :write [1 28] INFO [2019-12-17 12:35:31,344] jepsen worker 18 - jepsen.util 10008 :fail :read [6 nil] :RequestTimeoutException INFO [2019-12-17 12:35:31,345] jepsen worker 18 - jepsen.util 10008 :invoke :read [6 nil] INFO [2019-12-17 12:35:31,346] jepsen worker 64 - jepsen.util 7444 :ok :read [21 27] INFO [2019-12-17 12:35:31,346] jepsen worker 64 - jepsen.util 7444 :invoke :write [21 12] INFO [2019-12-17 12:35:31,360] jepsen worker 44 - jepsen.util 13364 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:35:31,360] jepsen worker 6 - jepsen.util 13596 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:35:31,360] jepsen worker 44 - jepsen.util 13364 :invoke :write [14 7] INFO [2019-12-17 12:35:31,360] jepsen worker 6 - jepsen.util 13596 :invoke :cas [2 [21 40]] INFO [2019-12-17 12:35:31,397] jepsen worker 13 - jepsen.util 8923 :info :write [4 14] :RequestCanceledException INFO [2019-12-17 12:35:31,397] jepsen worker 70 - jepsen.util 10240 :info :write [23 26] :RequestCanceledException INFO [2019-12-17 12:35:31,398] jepsen worker 13 - jepsen.util 9013 :invoke :read [4 nil] INFO [2019-12-17 12:35:31,398] jepsen worker 70 - jepsen.util 10330 :invoke :read [23 nil] INFO [2019-12-17 12:35:31,424] jepsen worker 12 - jepsen.util 9912 :fail :cas [4 [27 35]] :ValueNotSwapFrom INFO [2019-12-17 12:35:31,424] jepsen worker 12 - jepsen.util 9912 :invoke :write [4 11] INFO [2019-12-17 12:35:31,424] jepsen worker 71 - jepsen.util 10241 :ok :read [23 32] INFO [2019-12-17 12:35:31,424] jepsen worker 71 - jepsen.util 10241 :invoke :write [23 46] INFO [2019-12-17 12:35:31,430] jepsen worker 64 - jepsen.util 7444 :ok :write [21 12] INFO [2019-12-17 12:35:31,430] jepsen worker 64 - jepsen.util 7444 :invoke :write [21 6] INFO [2019-12-17 12:35:31,431] jepsen worker 24 - jepsen.util 11994 :info :write [8 3] :RequestCanceledException INFO [2019-12-17 12:35:31,434] jepsen worker 24 - jepsen.util 12084 :invoke :read [8 nil] INFO [2019-12-17 12:35:31,464] jepsen worker 55 - jepsen.util 10315 :ok :read [18 25] INFO [2019-12-17 12:35:31,464] jepsen worker 55 - jepsen.util 10315 :invoke :cas [18 [15 29]] INFO [2019-12-17 12:35:31,483] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55407,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:35:31,483] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55407,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:35:31,483] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55410,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:35:31,483] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55410,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:35:31,485] jepsen worker 14 - jepsen.util 9554 :fail :cas [4 [21 43]] :ValueNotSwapFrom INFO [2019-12-17 12:35:31,486] jepsen worker 14 - jepsen.util 9554 :invoke :read [4 nil] INFO [2019-12-17 12:35:31,497] jepsen worker 51 - jepsen.util 8781 :info :write [17 19] :RequestCanceledException INFO [2019-12-17 12:35:31,497] jepsen worker 35 - jepsen.util 12995 :info :write [11 44] :RequestCanceledException INFO [2019-12-17 12:35:31,498] jepsen worker 51 - jepsen.util 8871 :invoke :write [17 46] INFO [2019-12-17 12:35:31,499] jepsen worker 35 - jepsen.util 13085 :invoke :read [11 nil] INFO [2019-12-17 12:35:31,499] jepsen worker 87 - jepsen.util 9177 :ok :read [29 40] INFO [2019-12-17 12:35:31,499] jepsen worker 87 - jepsen.util 9177 :invoke :read [29 nil] INFO [2019-12-17 12:35:31,522] jepsen worker 19 - jepsen.util 9649 :info :write [6 34] :SyncWriteAmbiguous INFO [2019-12-17 12:35:31,522] jepsen worker 62 - jepsen.util 13382 :info :cas [20 [47 6]] :RequestCanceledException INFO [2019-12-17 12:35:31,522] jepsen worker 64 - jepsen.util 7444 :ok :write [21 6] INFO [2019-12-17 12:35:31,522] jepsen worker 64 - jepsen.util 7444 :invoke :cas [21 [45 20]] INFO [2019-12-17 12:35:31,524] jepsen worker 19 - jepsen.util 9739 :invoke :cas [6 [45 43]] INFO [2019-12-17 12:35:31,524] jepsen worker 62 - jepsen.util 13472 :invoke :write [20 47] INFO [2019-12-17 12:35:31,531] jepsen worker 20 - jepsen.util 9110 :info :write [6 24] :RequestCanceledException INFO [2019-12-17 12:35:31,533] jepsen worker 20 - jepsen.util 9200 :invoke :cas [6 [19 5]] INFO [2019-12-17 12:35:31,544] jepsen worker 23 - jepsen.util 13073 :info :cas [7 [11 28]] :RequestCanceledException INFO [2019-12-17 12:35:31,546] jepsen worker 23 - jepsen.util 13163 :invoke :write [7 44] INFO [2019-12-17 12:35:31,557] jepsen worker 63 - jepsen.util 8523 :ok :read [21 6] INFO [2019-12-17 12:35:31,557] jepsen worker 63 - jepsen.util 8523 :invoke :write [21 26] INFO [2019-12-17 12:35:31,562] jepsen worker 10 - jepsen.util 9640 :info :write [3 39] :RequestCanceledException INFO [2019-12-17 12:35:31,565] jepsen worker 10 - jepsen.util 9730 :invoke :cas [3 [36 0]] INFO [2019-12-17 12:35:31,579] jepsen worker 64 - jepsen.util 7444 :fail :cas [21 [45 20]] :ValueNotSwapFrom INFO [2019-12-17 12:35:31,580] jepsen worker 64 - jepsen.util 7444 :invoke :read [21 nil] INFO [2019-12-17 12:35:31,580] jepsen worker 79 - jepsen.util 9799 :ok :read [26 42] INFO [2019-12-17 12:35:31,580] jepsen worker 79 - jepsen.util 9799 :invoke :cas [26 [15 38]] INFO [2019-12-17 12:35:31,613] jepsen worker 64 - jepsen.util 7444 :ok :read [21 6] INFO [2019-12-17 12:35:31,613] jepsen worker 64 - jepsen.util 7444 :invoke :write [21 18] INFO [2019-12-17 12:35:31,630] jepsen worker 85 - jepsen.util 13855 :info :cas [28 [49 29]] :RequestCanceledException INFO [2019-12-17 12:35:31,637] jepsen worker 85 - jepsen.util 13945 :invoke :write [28 29] INFO [2019-12-17 12:35:31,645] jepsen worker 73 - jepsen.util 8893 :info :write [24 46] :RequestCanceledException INFO [2019-12-17 12:35:31,649] jepsen worker 73 - jepsen.util 8983 :invoke :read [24 nil] INFO [2019-12-17 12:35:31,662] jepsen worker 80 - jepsen.util 9170 :info :write [26 36] :RequestCanceledException INFO [2019-12-17 12:35:31,664] jepsen worker 80 - jepsen.util 9260 :invoke :cas [26 [32 48]] INFO [2019-12-17 12:35:31,664] jepsen worker 63 - jepsen.util 8523 :ok :write [21 26] INFO [2019-12-17 12:35:31,664] jepsen worker 63 - jepsen.util 8523 :invoke :read [21 nil] INFO [2019-12-17 12:35:31,676] jepsen worker 79 - jepsen.util 9799 :fail :cas [26 [15 38]] :ValueNotSwapFrom INFO [2019-12-17 12:35:31,677] jepsen worker 79 - jepsen.util 9799 :invoke :write [26 39] INFO [2019-12-17 12:35:31,694] jepsen worker 83 - jepsen.util 13493 :info :write [27 11] :RequestCanceledException INFO [2019-12-17 12:35:31,701] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55433,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:35:31,702] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55433,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:35:31,702] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55436,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:35:31,702] jepsen worker 63 - jepsen.util 8523 :ok :read [21 26] INFO [2019-12-17 12:35:31,702] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55436,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:35:31,702] jepsen worker 63 - jepsen.util 8523 :invoke :cas [21 [40 4]] INFO [2019-12-17 12:35:31,702] jepsen worker 83 - jepsen.util 13583 :invoke :read [27 nil] INFO [2019-12-17 12:35:31,720] jepsen worker 64 - jepsen.util 7444 :ok :write [21 18] INFO [2019-12-17 12:35:31,720] jepsen worker 64 - jepsen.util 7444 :invoke :write [21 24] INFO [2019-12-17 12:35:31,746] jepsen worker 63 - jepsen.util 8523 :fail :cas [21 [40 4]] :ValueNotSwapFrom INFO [2019-12-17 12:35:31,746] jepsen worker 63 - jepsen.util 8523 :invoke :write [21 11] INFO [2019-12-17 12:35:31,806] jepsen worker 63 - jepsen.util 8523 :ok :write [21 11] INFO [2019-12-17 12:35:31,806] jepsen worker 63 - jepsen.util 8523 :invoke :write [21 47] INFO [2019-12-17 12:35:31,831] jepsen worker 40 - jepsen.util 13360 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:35:31,831] jepsen worker 40 - jepsen.util 13360 :invoke :write [13 45] INFO [2019-12-17 12:35:31,845] jepsen worker 49 - jepsen.util 12739 :info :write [16 45] :RequestCanceledException INFO [2019-12-17 12:35:31,845] jepsen worker 45 - jepsen.util 9765 :info :cas [15 [34 23]] :RequestCanceledException INFO [2019-12-17 12:35:31,847] jepsen worker 45 - jepsen.util 9855 :invoke :read [15 nil] INFO [2019-12-17 12:35:31,847] jepsen worker 49 - jepsen.util 12829 :invoke :write [16 38] INFO [2019-12-17 12:35:31,849] jepsen worker 64 - jepsen.util 7444 :ok :write [21 24] INFO [2019-12-17 12:35:31,849] jepsen worker 64 - jepsen.util 7444 :invoke :read [21 nil] INFO [2019-12-17 12:35:31,859] jepsen worker 13 - jepsen.util 9013 :ok :read [4 34] INFO [2019-12-17 12:35:31,859] jepsen worker 13 - jepsen.util 9013 :invoke :write [4 41] INFO [2019-12-17 12:35:31,897] jepsen worker 66 - jepsen.util 12846 :info :write [22 11] :RequestCanceledException INFO [2019-12-17 12:35:31,901] jepsen worker 66 - jepsen.util 12936 :invoke :read [22 nil] INFO [2019-12-17 12:35:31,908] jepsen worker 45 - jepsen.util 9855 :ok :read [15 37] INFO [2019-12-17 12:35:31,908] jepsen worker 45 - jepsen.util 9855 :invoke :read [15 nil] INFO [2019-12-17 12:35:31,927] jepsen worker 63 - jepsen.util 8523 :ok :write [21 47] INFO [2019-12-17 12:35:31,928] jepsen worker 63 - jepsen.util 8523 :invoke :cas [21 [16 17]] INFO [2019-12-17 12:35:31,930] jepsen worker 69 - jepsen.util 9429 :info :write [23 4] :RequestCanceledException INFO [2019-12-17 12:35:31,930] jepsen worker 88 - jepsen.util 9808 :info :write [29 31] :RequestCanceledException INFO [2019-12-17 12:35:31,931] jepsen worker 78 - jepsen.util 9438 :info :write [26 25] :RequestCanceledException INFO [2019-12-17 12:35:31,931] jepsen worker 26 - jepsen.util 13526 :info :cas [8 [8 45]] :RequestCanceledException INFO [2019-12-17 12:35:31,937] jepsen worker 78 - jepsen.util 9528 :invoke :cas [26 [37 17]] INFO [2019-12-17 12:35:31,937] jepsen worker 26 - jepsen.util 13616 :invoke :write [8 31] INFO [2019-12-17 12:35:31,938] jepsen worker 88 - jepsen.util 9898 :invoke :read [29 nil] INFO [2019-12-17 12:35:31,940] jepsen worker 69 - jepsen.util 9519 :invoke :write [23 22] INFO [2019-12-17 12:35:31,943] jepsen worker 14 - jepsen.util 9554 :ok :read [4 34] INFO [2019-12-17 12:35:31,943] jepsen worker 14 - jepsen.util 9554 :invoke :read [4 nil] INFO [2019-12-17 12:35:31,944] jepsen worker 32 - jepsen.util 9572 :info :write [10 26] :RequestCanceledException INFO [2019-12-17 12:35:31,944] jepsen worker 65 - jepsen.util 7985 :info :write [21 6] :RequestCanceledException INFO [2019-12-17 12:35:31,949] jepsen worker 65 - jepsen.util 8075 :invoke :read [21 nil] INFO [2019-12-17 12:35:31,950] jepsen worker 32 - jepsen.util 9662 :invoke :write [10 22] INFO [2019-12-17 12:35:31,956] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55451,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:35:31,957] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55451,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:35:31,957] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55454,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:35:31,957] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55454,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:35:31,957] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55455,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:35:31,958] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55455,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:35:31,964] jepsen worker 64 - jepsen.util 7444 :ok :read [21 24] INFO [2019-12-17 12:35:31,965] jepsen worker 64 - jepsen.util 7444 :invoke :read [21 nil] INFO [2019-12-17 12:35:31,992] jepsen worker 0 - jepsen.util 13770 :info :write [0 4] :RequestCanceledException INFO [2019-12-17 12:35:31,996] jepsen worker 55 - jepsen.util 10315 :fail :cas [18 [15 29]] :ValueNotSwapFrom INFO [2019-12-17 12:35:31,996] jepsen worker 55 - jepsen.util 10315 :invoke :cas [18 [44 26]] INFO [2019-12-17 12:35:31,997] jepsen worker 0 - jepsen.util 13860 :invoke :write [0 46] INFO [2019-12-17 12:35:32,002] jepsen worker 63 - jepsen.util 8523 :fail :cas [21 [16 17]] :ValueNotSwapFrom INFO [2019-12-17 12:35:32,002] jepsen worker 63 - jepsen.util 8523 :invoke :cas [21 [46 19]] INFO [2019-12-17 12:35:32,002] jepsen worker 45 - jepsen.util 9855 :ok :read [15 37] INFO [2019-12-17 12:35:32,003] jepsen worker 45 - jepsen.util 9855 :invoke :write [15 30] INFO [2019-12-17 12:35:32,007] jepsen worker 11 - jepsen.util 9461 :info :write [3 11] :SyncWriteAmbiguous INFO [2019-12-17 12:35:32,010] jepsen worker 11 - jepsen.util 9551 :invoke :read [3 nil] INFO [2019-12-17 12:35:32,016] jepsen worker 61 - jepsen.util 14011 :info :write [20 40] :RequestCanceledException INFO [2019-12-17 12:35:32,019] jepsen worker 61 - jepsen.util 14101 :invoke :read [20 nil] INFO [2019-12-17 12:35:32,022] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55472,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:35:32,022] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55472,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:35:32,022] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55475,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:35:32,022] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55475,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:35:32,030] jepsen worker 64 - jepsen.util 7444 :ok :read [21 47] INFO [2019-12-17 12:35:32,030] jepsen worker 64 - jepsen.util 7444 :invoke :write [21 45] INFO [2019-12-17 12:35:32,030] jepsen worker 54 - jepsen.util 10404 :info :write [18 37] :RequestCanceledException INFO [2019-12-17 12:35:32,030] jepsen worker 7 - jepsen.util 13057 :info :cas [2 [4 5]] :RequestCanceledException INFO [2019-12-17 12:35:32,030] jepsen worker 52 - jepsen.util 9502 :info :write [17 16] :RequestCanceledException INFO [2019-12-17 12:35:32,030] jepsen worker 82 - jepsen.util 13762 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:35:32,030] jepsen worker 82 - jepsen.util 13762 :invoke :write [27 41] INFO [2019-12-17 12:35:32,034] jepsen worker 52 - jepsen.util 9592 :invoke :write [17 24] INFO [2019-12-17 12:35:32,035] jepsen worker 7 - jepsen.util 13147 :invoke :cas [2 [38 42]] INFO [2019-12-17 12:35:32,035] jepsen worker 54 - jepsen.util 10494 :invoke :write [18 36] INFO [2019-12-17 12:35:32,058] jepsen worker 68 - jepsen.util 12578 :info :write [22 0] :RequestCanceledException INFO [2019-12-17 12:35:32,062] jepsen worker 68 - jepsen.util 12668 :invoke :read [22 nil] INFO [2019-12-17 12:35:32,067] jepsen worker 27 - jepsen.util 9117 :ok :write [9 30] INFO [2019-12-17 12:35:32,067] jepsen worker 27 - jepsen.util 9117 :invoke :write [9 24] INFO [2019-12-17 12:35:32,074] jepsen worker 11 - jepsen.util 9551 :ok :read [3 36] INFO [2019-12-17 12:35:32,074] jepsen worker 11 - jepsen.util 9551 :invoke :read [3 nil] INFO [2019-12-17 12:35:32,093] jepsen worker 74 - jepsen.util 9344 :fail :cas [24 [15 4]] :ValueNotSwapFrom INFO [2019-12-17 12:35:32,093] jepsen worker 74 - jepsen.util 9344 :invoke :write [24 11] INFO [2019-12-17 12:35:32,097] jepsen worker 1 - jepsen.util 13231 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:35:32,097] jepsen worker 1 - jepsen.util 13231 :invoke :cas [0 [29 30]] INFO [2019-12-17 12:35:32,115] jepsen worker 25 - jepsen.util 14155 :info :cas [8 [1 27]] :RequestCanceledException INFO [2019-12-17 12:35:32,115] jepsen worker 46 - jepsen.util 9586 :info :write [15 21] :RequestCanceledException INFO [2019-12-17 12:35:32,120] jepsen worker 25 - jepsen.util 14245 :invoke :write [8 30] INFO [2019-12-17 12:35:32,120] jepsen worker 46 - jepsen.util 9676 :invoke :write [15 5] INFO [2019-12-17 12:35:32,131] jepsen worker 58 - jepsen.util 13648 :info :write [19 7] :RequestCanceledException INFO [2019-12-17 12:35:32,131] jepsen worker 75 - jepsen.util 12495 :info :write [25 36] :RequestCanceledException INFO [2019-12-17 12:35:32,136] jepsen worker 75 - jepsen.util 12585 :invoke :cas [25 [31 30]] INFO [2019-12-17 12:35:32,136] jepsen worker 58 - jepsen.util 13738 :invoke :read [19 nil] INFO [2019-12-17 12:35:32,144] jepsen worker 87 - jepsen.util 9177 :ok :read [29 40] INFO [2019-12-17 12:35:32,145] jepsen worker 87 - jepsen.util 9177 :invoke :write [29 24] INFO [2019-12-17 12:35:32,150] jepsen worker 19 - jepsen.util 9739 :fail :cas [6 [45 43]] :ValueNotSwapFrom INFO [2019-12-17 12:35:32,151] jepsen worker 19 - jepsen.util 9739 :invoke :cas [6 [37 10]] INFO [2019-12-17 12:35:32,155] jepsen worker 80 - jepsen.util 9260 :fail :cas [26 [32 48]] :ValueNotSwapFrom INFO [2019-12-17 12:35:32,156] jepsen worker 80 - jepsen.util 9260 :invoke :read [26 nil] INFO [2019-12-17 12:35:32,165] jepsen worker 73 - jepsen.util 8983 :ok :read [24 9] INFO [2019-12-17 12:35:32,165] jepsen worker 73 - jepsen.util 8983 :invoke :read [24 nil] INFO [2019-12-17 12:35:32,176] jepsen worker 11 - jepsen.util 9551 :ok :read [3 36] INFO [2019-12-17 12:35:32,176] jepsen worker 11 - jepsen.util 9551 :invoke :cas [3 [5 37]] INFO [2019-12-17 12:35:32,182] jepsen worker 27 - jepsen.util 9117 :ok :write [9 24] INFO [2019-12-17 12:35:32,182] jepsen worker 27 - jepsen.util 9117 :invoke :cas [9 [7 33]] INFO [2019-12-17 12:35:32,216] jepsen worker 37 - jepsen.util 9037 :info :write [12 20] :RequestCanceledException INFO [2019-12-17 12:35:32,216] jepsen worker 11 - jepsen.util 9551 :fail :cas [3 [5 37]] :ValueNotSwapFrom INFO [2019-12-17 12:35:32,217] jepsen worker 11 - jepsen.util 9551 :invoke :read [3 nil] INFO [2019-12-17 12:35:32,220] jepsen worker 37 - jepsen.util 9127 :invoke :cas [12 [7 25]] INFO [2019-12-17 12:35:32,237] jepsen worker 19 - jepsen.util 9739 :fail :cas [6 [37 10]] :ValueNotSwapFrom INFO [2019-12-17 12:35:32,238] jepsen worker 19 - jepsen.util 9739 :invoke :read [6 nil] INFO [2019-12-17 12:35:32,242] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55501,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:35:32,242] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55509,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:35:32,242] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55501,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:35:32,242] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55509,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:35:32,242] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55503,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:35:32,242] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55513,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:35:32,242] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55503,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:35:32,242] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55513,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:35:32,260] jepsen worker 73 - jepsen.util 8983 :ok :read [24 9] INFO [2019-12-17 12:35:32,261] jepsen worker 73 - jepsen.util 8983 :invoke :read [24 nil] INFO [2019-12-17 12:35:32,262] jepsen worker 29 - jepsen.util 7859 :info :write [9 32] :RequestCanceledException INFO [2019-12-17 12:35:32,262] jepsen worker 9 - jepsen.util 10089 :info :cas [3 [37 5]] :RequestCanceledException INFO [2019-12-17 12:35:32,264] jepsen worker 9 - jepsen.util 10179 :invoke :cas [3 [16 12]] INFO [2019-12-17 12:35:32,266] jepsen worker 29 - jepsen.util 7949 :invoke :cas [9 [28 27]] INFO [2019-12-17 12:35:32,280] jepsen worker 27 - jepsen.util 9117 :fail :cas [9 [7 33]] :ValueNotSwapFrom INFO [2019-12-17 12:35:32,280] jepsen worker 27 - jepsen.util 9117 :invoke :read [9 nil] INFO [2019-12-17 12:35:32,285] jepsen worker 11 - jepsen.util 9551 :ok :read [3 36] INFO [2019-12-17 12:35:32,286] jepsen worker 11 - jepsen.util 9551 :invoke :cas [3 [27 7]] INFO [2019-12-17 12:35:32,297] jepsen worker 19 - jepsen.util 9739 :ok :read [6 38] INFO [2019-12-17 12:35:32,297] jepsen worker 19 - jepsen.util 9739 :invoke :write [6 1] INFO [2019-12-17 12:35:32,315] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:35:32,316] jepsen nemesis - couchbase.util Rebalance check count: 434 INFO [2019-12-17 12:35:32,316] jepsen nemesis - couchbase.util Rebalance stuck count: 415 INFO [2019-12-17 12:35:32,330] jepsen worker 89 - jepsen.util 9449 :info :write [29 48] :RequestCanceledException INFO [2019-12-17 12:35:32,330] jepsen worker 81 - jepsen.util 12951 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:35:32,331] jepsen worker 81 - jepsen.util 12951 :invoke :read [27 nil] INFO [2019-12-17 12:35:32,332] jepsen worker 89 - jepsen.util 9539 :invoke :write [29 48] INFO [2019-12-17 12:35:32,339] jepsen worker 73 - jepsen.util 8983 :ok :read [24 9] INFO [2019-12-17 12:35:32,340] jepsen worker 73 - jepsen.util 8983 :invoke :write [24 35] INFO [2019-12-17 12:35:32,364] jepsen worker 9 - jepsen.util 10179 :fail :cas [3 [16 12]] :ValueNotSwapFrom INFO [2019-12-17 12:35:32,364] jepsen worker 9 - jepsen.util 10179 :invoke :write [3 8] INFO [2019-12-17 12:35:32,371] jepsen worker 29 - jepsen.util 7949 :fail :cas [9 [28 27]] :ValueNotSwapFrom INFO [2019-12-17 12:35:32,371] jepsen worker 29 - jepsen.util 7949 :invoke :write [9 30] INFO [2019-12-17 12:35:32,384] jepsen worker 27 - jepsen.util 9117 :ok :read [9 24] INFO [2019-12-17 12:35:32,384] jepsen worker 27 - jepsen.util 9117 :invoke :cas [9 [30 38]] INFO [2019-12-17 12:35:32,403] jepsen worker 11 - jepsen.util 9551 :fail :cas [3 [27 7]] :ValueNotSwapFrom INFO [2019-12-17 12:35:32,403] jepsen worker 11 - jepsen.util 9551 :invoke :cas [3 [18 43]] INFO [2019-12-17 12:35:32,431] jepsen worker 48 - jepsen.util 12108 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:35:32,431] jepsen worker 48 - jepsen.util 12108 :invoke :cas [16 [10 21]] INFO [2019-12-17 12:35:32,454] jepsen worker 29 - jepsen.util 7949 :ok :write [9 30] INFO [2019-12-17 12:35:32,454] jepsen worker 29 - jepsen.util 7949 :invoke :cas [9 [25 48]] INFO [2019-12-17 12:35:32,461] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55531,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:35:32,461] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55531,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:35:32,462] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55535,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:35:32,462] jepsen worker 4 - jepsen.util 9004 :info :write [1 12] :RequestCanceledException INFO [2019-12-17 12:35:32,462] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55535,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:35:32,463] jepsen worker 88 - jepsen.util 9898 :ok :read [29 40] INFO [2019-12-17 12:35:32,463] jepsen worker 88 - jepsen.util 9898 :invoke :read [29 nil] INFO [2019-12-17 12:35:32,464] jepsen worker 78 - jepsen.util 9528 :fail :cas [26 [37 17]] :ValueNotSwapFrom INFO [2019-12-17 12:35:32,464] jepsen worker 78 - jepsen.util 9528 :invoke :write [26 26] INFO [2019-12-17 12:35:32,465] jepsen worker 4 - jepsen.util 9094 :invoke :cas [1 [31 2]] INFO [2019-12-17 12:35:32,498] jepsen worker 39 - jepsen.util 12639 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:35:32,498] jepsen worker 39 - jepsen.util 12639 :invoke :cas [13 [19 1]] INFO [2019-12-17 12:35:32,509] jepsen worker 29 - jepsen.util 7949 :fail :cas [9 [25 48]] :ValueNotSwapFrom INFO [2019-12-17 12:35:32,510] jepsen worker 29 - jepsen.util 7949 :invoke :cas [9 [29 20]] INFO [2019-12-17 12:35:32,514] jepsen worker 55 - jepsen.util 10315 :fail :cas [18 [44 26]] :ValueNotSwapFrom INFO [2019-12-17 12:35:32,515] jepsen worker 55 - jepsen.util 10315 :invoke :cas [18 [15 37]] INFO [2019-12-17 12:35:32,517] jepsen worker 70 - jepsen.util 10330 :ok :read [23 32] INFO [2019-12-17 12:35:32,518] jepsen worker 70 - jepsen.util 10330 :invoke :write [23 10] INFO [2019-12-17 12:35:32,521] jepsen worker 14 - jepsen.util 9554 :ok :read [4 34] INFO [2019-12-17 12:35:32,522] jepsen worker 14 - jepsen.util 9554 :invoke :read [4 nil] INFO [2019-12-17 12:35:32,534] jepsen worker 29 - jepsen.util 7949 :fail :cas [9 [29 20]] :ValueNotSwapFrom INFO [2019-12-17 12:35:32,534] jepsen worker 29 - jepsen.util 7949 :invoke :read [9 nil] INFO [2019-12-17 12:35:32,545] jepsen worker 28 - jepsen.util 7768 :info :cas [9 [18 10]] :RequestCanceledException INFO [2019-12-17 12:35:32,547] jepsen worker 28 - jepsen.util 7858 :invoke :read [9 nil] INFO [2019-12-17 12:35:32,562] jepsen worker 5 - jepsen.util 8915 :info :write [1 26] :RequestCanceledException INFO [2019-12-17 12:35:32,563] jepsen worker 29 - jepsen.util 7949 :ok :read [9 30] INFO [2019-12-17 12:35:32,563] jepsen worker 29 - jepsen.util 7949 :invoke :read [9 nil] INFO [2019-12-17 12:35:32,564] jepsen worker 5 - jepsen.util 9005 :invoke :write [1 9] INFO [2019-12-17 12:35:32,587] jepsen worker 29 - jepsen.util 7949 :ok :read [9 30] INFO [2019-12-17 12:35:32,587] jepsen worker 29 - jepsen.util 7949 :invoke :read [9 nil] INFO [2019-12-17 12:35:32,608] jepsen worker 29 - jepsen.util 7949 :ok :read [9 30] INFO [2019-12-17 12:35:32,608] jepsen worker 29 - jepsen.util 7949 :invoke :write [9 23] INFO [2019-12-17 12:35:32,630] jepsen worker 33 - jepsen.util 13083 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:35:32,631] jepsen worker 33 - jepsen.util 13083 :invoke :write [11 22] INFO [2019-12-17 12:35:32,637] jepsen worker 80 - jepsen.util 9260 :ok :read [26 42] INFO [2019-12-17 12:35:32,637] jepsen worker 80 - jepsen.util 9260 :invoke :write [26 5] INFO [2019-12-17 12:35:32,647] jepsen worker 37 - jepsen.util 9127 :fail :cas [12 [7 25]] :ValueNotSwapFrom INFO [2019-12-17 12:35:32,647] jepsen worker 37 - jepsen.util 9127 :invoke :read [12 nil] INFO [2019-12-17 12:35:32,663] jepsen worker 29 - jepsen.util 7949 :ok :write [9 23] INFO [2019-12-17 12:35:32,663] jepsen worker 29 - jepsen.util 7949 :invoke :read [9 nil] INFO [2019-12-17 12:35:32,680] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55553,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:35:32,680] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55553,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:35:32,680] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55556,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:35:32,680] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55556,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:35:32,680] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55558,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:35:32,680] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55558,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:35:32,685] jepsen worker 19 - jepsen.util 9739 :ok :write [6 1] INFO [2019-12-17 12:35:32,685] jepsen worker 19 - jepsen.util 9739 :invoke :write [6 49] INFO [2019-12-17 12:35:32,688] jepsen worker 31 - jepsen.util 8671 :ok :write [10 26] INFO [2019-12-17 12:35:32,688] jepsen worker 45 - jepsen.util 9855 :ok :write [15 30] INFO [2019-12-17 12:35:32,688] jepsen worker 31 - jepsen.util 8671 :invoke :write [10 49] INFO [2019-12-17 12:35:32,688] jepsen worker 45 - jepsen.util 9855 :invoke :cas [15 [26 16]] INFO [2019-12-17 12:35:32,698] jepsen worker 47 - jepsen.util 10037 :info :write [15 14] :RequestCanceledException INFO [2019-12-17 12:35:32,700] jepsen worker 47 - jepsen.util 10127 :invoke :cas [15 [3 3]] INFO [2019-12-17 12:35:32,703] jepsen worker 29 - jepsen.util 7949 :ok :read [9 23] INFO [2019-12-17 12:35:32,703] jepsen worker 29 - jepsen.util 7949 :invoke :read [9 nil] INFO [2019-12-17 12:35:32,749] jepsen worker 12 - jepsen.util 9912 :ok :write [4 11] INFO [2019-12-17 12:35:32,749] jepsen worker 12 - jepsen.util 9912 :invoke :write [4 15] INFO [2019-12-17 12:35:32,831] jepsen worker 77 - jepsen.util 13667 :info :cas [25 [33 5]] :RequestCanceledException INFO [2019-12-17 12:35:32,831] jepsen worker 22 - jepsen.util 12982 :info :cas [7 [31 4]] :RequestCanceledException INFO [2019-12-17 12:35:32,836] jepsen worker 77 - jepsen.util 13757 :invoke :write [25 48] INFO [2019-12-17 12:35:32,837] jepsen worker 22 - jepsen.util 13072 :invoke :read [7 nil] INFO [2019-12-17 12:35:32,845] jepsen worker 59 - jepsen.util 12389 :info :cas [19 [19 23]] :RequestCanceledException INFO [2019-12-17 12:35:32,847] jepsen worker 55 - jepsen.util 10315 :fail :cas [18 [15 37]] :ValueNotSwapFrom INFO [2019-12-17 12:35:32,848] jepsen worker 55 - jepsen.util 10315 :invoke :read [18 nil] INFO [2019-12-17 12:35:32,850] jepsen worker 29 - jepsen.util 7949 :ok :read [9 23] INFO [2019-12-17 12:35:32,850] jepsen worker 59 - jepsen.util 12479 :invoke :cas [19 [39 13]] INFO [2019-12-17 12:35:32,851] jepsen worker 29 - jepsen.util 7949 :invoke :cas [9 [35 35]] INFO [2019-12-17 12:35:32,855] jepsen worker 45 - jepsen.util 9855 :fail :cas [15 [26 16]] :ValueNotSwapFrom INFO [2019-12-17 12:35:32,855] jepsen worker 45 - jepsen.util 9855 :invoke :write [15 13] INFO [2019-12-17 12:35:32,861] jepsen worker 50 - jepsen.util 13820 :info :write [16 42] :RequestCanceledException INFO [2019-12-17 12:35:32,865] jepsen worker 50 - jepsen.util 13910 :invoke :write [16 35] INFO [2019-12-17 12:35:32,871] jepsen worker 65 - jepsen.util 8075 :ok :read [21 47] INFO [2019-12-17 12:35:32,871] jepsen worker 65 - jepsen.util 8075 :invoke :cas [21 [45 39]] INFO [2019-12-17 12:35:32,897] jepsen worker 8 - jepsen.util 12788 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:35:32,897] jepsen worker 8 - jepsen.util 12788 :invoke :write [2 1] INFO [2019-12-17 12:35:32,915] jepsen worker 29 - jepsen.util 7949 :fail :cas [9 [35 35]] :ValueNotSwapFrom INFO [2019-12-17 12:35:32,916] jepsen worker 29 - jepsen.util 7949 :invoke :write [9 22] INFO [2019-12-17 12:35:32,923] jepsen worker 88 - jepsen.util 9898 :ok :read [29 40] INFO [2019-12-17 12:35:32,923] jepsen worker 88 - jepsen.util 9898 :invoke :write [29 25] INFO [2019-12-17 12:35:32,931] jepsen worker 41 - jepsen.util 14081 :info :cas [13 [6 48]] :RequestCanceledException INFO [2019-12-17 12:35:32,936] jepsen worker 41 - jepsen.util 14171 :invoke :write [13 26] INFO [2019-12-17 12:35:32,943] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55574,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:35:32,943] jepsen worker 72 - jepsen.util 9432 :fail :read [24 nil] :RequestTimeoutException INFO [2019-12-17 12:35:32,943] jepsen worker 67 - jepsen.util 12937 :info :cas [22 [30 48]] :RequestCanceledException INFO [2019-12-17 12:35:32,943] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55574,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:35:32,943] jepsen worker 72 - jepsen.util 9432 :invoke :read [24 nil] INFO [2019-12-17 12:35:32,944] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55577,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:35:32,944] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55577,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:35:32,950] jepsen worker 67 - jepsen.util 13027 :invoke :write [22 8] INFO [2019-12-17 12:35:32,959] jepsen worker 2 - jepsen.util 13052 :info :write [0 9] :RequestCanceledException INFO [2019-12-17 12:35:32,959] jepsen worker 34 - jepsen.util 13084 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:35:32,959] jepsen worker 34 - jepsen.util 13084 :invoke :write [11 41] INFO [2019-12-17 12:35:32,965] jepsen worker 2 - jepsen.util 13142 :invoke :read [0 nil] INFO [2019-12-17 12:35:32,970] jepsen worker 13 - jepsen.util 9013 :ok :write [4 41] INFO [2019-12-17 12:35:32,970] jepsen worker 13 - jepsen.util 9013 :invoke :read [4 nil] INFO [2019-12-17 12:35:32,972] jepsen worker 28 - jepsen.util 7858 :ok :read [9 23] INFO [2019-12-17 12:35:32,972] jepsen worker 28 - jepsen.util 7858 :invoke :write [9 24] INFO [2019-12-17 12:35:32,997] jepsen worker 21 - jepsen.util 12711 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:35:32,997] jepsen worker 16 - jepsen.util 14236 :info :write [5 18] :RequestCanceledException INFO [2019-12-17 12:35:32,997] jepsen worker 21 - jepsen.util 12711 :invoke :read [7 nil] INFO [2019-12-17 12:35:32,999] jepsen worker 16 - jepsen.util 14326 :invoke :write [5 35] INFO [2019-12-17 12:35:33,010] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55593,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:35:33,010] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55593,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:35:33,010] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55598,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:35:33,010] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55598,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:35:33,015] jepsen worker 36 - jepsen.util 10206 :info :write [12 41] :RequestCanceledException INFO [2019-12-17 12:35:33,019] jepsen worker 36 - jepsen.util 10296 :invoke :write [12 48] INFO [2019-12-17 12:35:33,029] jepsen worker 52 - jepsen.util 9592 :ok :write [17 24] INFO [2019-12-17 12:35:33,029] jepsen worker 52 - jepsen.util 9592 :invoke :read [17 nil] INFO [2019-12-17 12:35:33,031] jepsen worker 86 - jepsen.util 13946 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:35:33,031] jepsen worker 15 - jepsen.util 12255 :info :cas [5 [4 19]] :RequestCanceledException INFO [2019-12-17 12:35:33,031] jepsen worker 86 - jepsen.util 13946 :invoke :cas [28 [18 30]] INFO [2019-12-17 12:35:33,032] jepsen worker 18 - jepsen.util 10008 :ok :read [6 1] INFO [2019-12-17 12:35:33,032] jepsen worker 18 - jepsen.util 10008 :invoke :cas [6 [28 44]] INFO [2019-12-17 12:35:33,034] jepsen worker 15 - jepsen.util 12345 :invoke :write [5 26] INFO [2019-12-17 12:35:33,040] jepsen worker 20 - jepsen.util 9200 :fail :cas [6 [19 5]] :ValueNotSwapFrom INFO [2019-12-17 12:35:33,040] jepsen worker 20 - jepsen.util 9200 :invoke :cas [6 [38 34]] INFO [2019-12-17 12:35:33,045] jepsen worker 84 - jepsen.util 12684 :info :cas [28 [37 32]] :RequestCanceledException INFO [2019-12-17 12:35:33,047] jepsen worker 84 - jepsen.util 12774 :invoke :write [28 46] INFO [2019-12-17 12:35:33,054] jepsen worker 29 - jepsen.util 7949 :ok :write [9 22] INFO [2019-12-17 12:35:33,054] jepsen worker 29 - jepsen.util 7949 :invoke :write [9 40] INFO [2019-12-17 12:35:33,062] jepsen worker 38 - jepsen.util 10028 :info :write [12 1] :RequestCanceledException INFO [2019-12-17 12:35:33,064] jepsen worker 38 - jepsen.util 10118 :invoke :write [12 23] INFO [2019-12-17 12:35:33,086] jepsen worker 72 - jepsen.util 9432 :ok :read [24 46] INFO [2019-12-17 12:35:33,087] jepsen worker 72 - jepsen.util 9432 :invoke :write [24 38] INFO [2019-12-17 12:35:33,121] jepsen worker 18 - jepsen.util 10008 :fail :cas [6 [28 44]] :ValueNotSwapFrom INFO [2019-12-17 12:35:33,121] jepsen worker 18 - jepsen.util 10008 :invoke :read [6 nil] INFO [2019-12-17 12:35:33,131] jepsen worker 42 - jepsen.util 14172 :info :write [14 17] :RequestCanceledException INFO [2019-12-17 12:35:33,131] jepsen worker 30 - jepsen.util 9660 :info :write [10 20] :RequestCanceledException INFO [2019-12-17 12:35:33,133] jepsen worker 42 - jepsen.util 14262 :invoke :read [14 nil] INFO [2019-12-17 12:35:33,134] jepsen worker 30 - jepsen.util 9750 :invoke :cas [10 [34 21]] INFO [2019-12-17 12:35:33,140] jepsen worker 45 - jepsen.util 9855 :ok :write [15 13] INFO [2019-12-17 12:35:33,140] jepsen worker 45 - jepsen.util 9855 :invoke :read [15 nil] INFO [2019-12-17 12:35:33,146] jepsen worker 43 - jepsen.util 13003 :info :cas [14 [6 25]] :RequestCanceledException INFO [2019-12-17 12:35:33,147] jepsen worker 43 - jepsen.util 13093 :invoke :read [14 nil] INFO [2019-12-17 12:35:33,148] jepsen worker 73 - jepsen.util 8983 :ok :write [24 35] INFO [2019-12-17 12:35:33,148] jepsen worker 73 - jepsen.util 8983 :invoke :read [24 nil] INFO [2019-12-17 12:35:33,148] jepsen worker 31 - jepsen.util 8671 :ok :write [10 49] INFO [2019-12-17 12:35:33,148] jepsen worker 31 - jepsen.util 8671 :invoke :cas [10 [10 48]] INFO [2019-12-17 12:35:33,149] jepsen worker 20 - jepsen.util 9200 :fail :cas [6 [38 34]] :ValueNotSwapFrom INFO [2019-12-17 12:35:33,149] jepsen worker 20 - jepsen.util 9200 :invoke :cas [6 [30 17]] INFO [2019-12-17 12:35:33,178] jepsen worker 45 - jepsen.util 9855 :ok :read [15 13] INFO [2019-12-17 12:35:33,178] jepsen worker 45 - jepsen.util 9855 :invoke :cas [15 [17 41]] INFO [2019-12-17 12:35:33,227] jepsen worker 14 - jepsen.util 9554 :ok :read [4 41] INFO [2019-12-17 12:35:33,228] jepsen worker 14 - jepsen.util 9554 :invoke :cas [4 [7 34]] INFO [2019-12-17 12:35:33,229] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55627,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:35:33,229] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55627,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:35:33,229] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55620,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:35:33,229] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55620,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:35:33,230] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55621,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:35:33,230] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55621,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:35:33,231] jepsen worker 17 - jepsen.util 12707 :info :write [5 35] :RequestCanceledException INFO [2019-12-17 12:35:33,236] jepsen worker 17 - jepsen.util 12797 :invoke :write [5 14] INFO [2019-12-17 12:35:33,239] jepsen worker 56 - jepsen.util 10046 :info :write [18 0] :RequestCanceledException INFO [2019-12-17 12:35:33,243] jepsen worker 56 - jepsen.util 10136 :invoke :write [18 13] INFO [2019-12-17 12:35:33,249] jepsen worker 29 - jepsen.util 7949 :ok :write [9 40] INFO [2019-12-17 12:35:33,249] jepsen worker 29 - jepsen.util 7949 :invoke :write [9 13] INFO [2019-12-17 12:35:33,262] jepsen worker 60 - jepsen.util 14190 :info :write [20 44] :RequestCanceledException INFO [2019-12-17 12:35:33,268] jepsen worker 60 - jepsen.util 14280 :invoke :cas [20 [44 24]] INFO [2019-12-17 12:35:33,278] jepsen worker 27 - jepsen.util 9117 :fail :cas [9 [30 38]] :ValueNotSwapFrom INFO [2019-12-17 12:35:33,278] jepsen worker 27 - jepsen.util 9117 :invoke :read [9 nil] INFO [2019-12-17 12:35:33,279] jepsen worker 45 - jepsen.util 9855 :fail :cas [15 [17 41]] :ValueNotSwapFrom INFO [2019-12-17 12:35:33,279] jepsen worker 45 - jepsen.util 9855 :invoke :read [15 nil] INFO [2019-12-17 12:35:33,287] jepsen worker 73 - jepsen.util 8983 :ok :read [24 35] INFO [2019-12-17 12:35:33,287] jepsen worker 73 - jepsen.util 8983 :invoke :write [24 4] INFO [2019-12-17 12:35:33,297] jepsen worker 57 - jepsen.util 12657 :info :write [19 42] :RequestCanceledException INFO [2019-12-17 12:35:33,300] jepsen worker 57 - jepsen.util 12747 :invoke :cas [19 [41 0]] INFO [2019-12-17 12:35:33,330] jepsen worker 27 - jepsen.util 9117 :ok :read [9 40] INFO [2019-12-17 12:35:33,330] jepsen worker 27 - jepsen.util 9117 :invoke :write [9 20] INFO [2019-12-17 12:35:33,331] jepsen worker 53 - jepsen.util 8963 :info :write [17 34] :RequestCanceledException INFO [2019-12-17 12:35:33,334] jepsen worker 53 - jepsen.util 9053 :invoke :cas [17 [19 49]] INFO [2019-12-17 12:35:33,344] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:35:33,345] jepsen nemesis - couchbase.util Rebalance check count: 435 INFO [2019-12-17 12:35:33,345] jepsen nemesis - couchbase.util Rebalance stuck count: 416 INFO [2019-12-17 12:35:33,352] jepsen worker 55 - jepsen.util 10315 :ok :read [18 25] INFO [2019-12-17 12:35:33,352] jepsen worker 55 - jepsen.util 10315 :invoke :read [18 nil] INFO [2019-12-17 12:35:33,361] jepsen worker 3 - jepsen.util 9363 :info :write [1 28] :RequestCanceledException INFO [2019-12-17 12:35:33,364] jepsen worker 3 - jepsen.util 9453 :invoke :write [1 42] INFO [2019-12-17 12:35:33,378] jepsen worker 45 - jepsen.util 9855 :ok :read [15 13] INFO [2019-12-17 12:35:33,378] jepsen worker 45 - jepsen.util 9855 :invoke :read [15 nil] INFO [2019-12-17 12:35:33,384] jepsen worker 13 - jepsen.util 9013 :ok :read [4 41] INFO [2019-12-17 12:35:33,384] jepsen worker 13 - jepsen.util 9013 :invoke :read [4 nil] INFO [2019-12-17 12:35:33,409] jepsen worker 76 - jepsen.util 13306 :info :cas [25 [42 3]] :RequestCanceledException INFO [2019-12-17 12:35:33,411] jepsen worker 76 - jepsen.util 13396 :invoke :write [25 21] INFO [2019-12-17 12:35:33,426] jepsen worker 29 - jepsen.util 7949 :ok :write [9 13] INFO [2019-12-17 12:35:33,426] jepsen worker 29 - jepsen.util 7949 :invoke :cas [9 [20 49]] INFO [2019-12-17 12:35:33,429] jepsen worker 45 - jepsen.util 9855 :ok :read [15 13] INFO [2019-12-17 12:35:33,429] jepsen worker 45 - jepsen.util 9855 :invoke :read [15 nil] INFO [2019-12-17 12:35:33,448] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55653,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:35:33,449] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55653,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:35:33,449] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55654,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:35:33,449] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55654,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:35:33,462] jepsen worker 6 - jepsen.util 13596 :info :cas [2 [21 40]] :RequestCanceledException INFO [2019-12-17 12:35:33,462] jepsen worker 44 - jepsen.util 13364 :info :write [14 7] :RequestCanceledException INFO [2019-12-17 12:35:33,462] jepsen worker 24 - jepsen.util 12084 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:35:33,462] jepsen worker 24 - jepsen.util 12084 :invoke :read [8 nil] INFO [2019-12-17 12:35:33,464] jepsen worker 10 - jepsen.util 9730 :ok :cas [3 [36 0]] INFO [2019-12-17 12:35:33,464] jepsen worker 10 - jepsen.util 9730 :invoke :cas [3 [18 24]] INFO [2019-12-17 12:35:33,465] jepsen worker 6 - jepsen.util 13686 :invoke :read [2 nil] INFO [2019-12-17 12:35:33,465] jepsen worker 44 - jepsen.util 13454 :invoke :cas [14 [25 25]] INFO [2019-12-17 12:35:33,488] jepsen worker 29 - jepsen.util 7949 :fail :cas [9 [20 49]] :ValueNotSwapFrom INFO [2019-12-17 12:35:33,488] jepsen worker 29 - jepsen.util 7949 :invoke :read [9 nil] INFO [2019-12-17 12:35:33,502] jepsen worker 45 - jepsen.util 9855 :ok :read [15 13] INFO [2019-12-17 12:35:33,503] jepsen worker 45 - jepsen.util 9855 :invoke :write [15 1] INFO [2019-12-17 12:35:33,515] jepsen worker 71 - jepsen.util 10241 :info :write [23 46] :RequestCanceledException INFO [2019-12-17 12:35:33,516] jepsen worker 71 - jepsen.util 10331 :invoke :read [23 nil] INFO [2019-12-17 12:35:33,577] jepsen worker 72 - jepsen.util 9432 :ok :write [24 38] INFO [2019-12-17 12:35:33,577] jepsen worker 72 - jepsen.util 9432 :invoke :write [24 8] INFO [2019-12-17 12:35:33,595] jepsen worker 53 - jepsen.util 9053 :fail :cas [17 [19 49]] :ValueNotSwapFrom INFO [2019-12-17 12:35:33,595] jepsen worker 53 - jepsen.util 9053 :invoke :read [17 nil] INFO [2019-12-17 12:35:33,597] jepsen worker 35 - jepsen.util 13085 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:35:33,597] jepsen worker 35 - jepsen.util 13085 :invoke :write [11 45] INFO [2019-12-17 12:35:33,597] jepsen worker 31 - jepsen.util 8671 :fail :cas [10 [10 48]] :ValueNotSwapFrom INFO [2019-12-17 12:35:33,597] jepsen worker 31 - jepsen.util 8671 :invoke :read [10 nil] INFO [2019-12-17 12:35:33,607] jepsen worker 29 - jepsen.util 7949 :ok :read [9 13] INFO [2019-12-17 12:35:33,607] jepsen worker 29 - jepsen.util 7949 :invoke :cas [9 [40 42]] INFO [2019-12-17 12:35:33,631] jepsen worker 51 - jepsen.util 8871 :info :write [17 46] :RequestCanceledException INFO [2019-12-17 12:35:33,631] jepsen worker 62 - jepsen.util 13472 :info :write [20 47] :RequestCanceledException INFO [2019-12-17 12:35:33,637] jepsen worker 62 - jepsen.util 13562 :invoke :read [20 nil] INFO [2019-12-17 12:35:33,638] jepsen worker 51 - jepsen.util 8961 :invoke :cas [17 [8 16]] INFO [2019-12-17 12:35:33,650] jepsen worker 71 - jepsen.util 10331 :ok :read [23 32] INFO [2019-12-17 12:35:33,650] jepsen worker 71 - jepsen.util 10331 :invoke :write [23 10] INFO [2019-12-17 12:35:33,653] jepsen worker 10 - jepsen.util 9730 :fail :cas [3 [18 24]] :ValueNotSwapFrom INFO [2019-12-17 12:35:33,654] jepsen worker 10 - jepsen.util 9730 :invoke :cas [3 [42 0]] INFO [2019-12-17 12:35:33,661] jepsen worker 23 - jepsen.util 13163 :info :write [7 44] :RequestCanceledException INFO [2019-12-17 12:35:33,663] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55670,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:35:33,664] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55670,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:35:33,664] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55673,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:35:33,665] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55673,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:35:33,669] jepsen worker 23 - jepsen.util 13253 :invoke :read [7 nil] INFO [2019-12-17 12:35:33,681] jepsen worker 30 - jepsen.util 9750 :fail :cas [10 [34 21]] :ValueNotSwapFrom INFO [2019-12-17 12:35:33,681] jepsen worker 30 - jepsen.util 9750 :invoke :cas [10 [9 24]] INFO [2019-12-17 12:35:33,692] jepsen worker 55 - jepsen.util 10315 :ok :read [18 25] INFO [2019-12-17 12:35:33,692] jepsen worker 55 - jepsen.util 10315 :invoke :read [18 nil] INFO [2019-12-17 12:35:33,694] jepsen worker 20 - jepsen.util 9200 :fail :cas [6 [30 17]] :ValueNotSwapFrom INFO [2019-12-17 12:35:33,694] jepsen worker 20 - jepsen.util 9200 :invoke :read [6 nil] INFO [2019-12-17 12:35:33,695] jepsen worker 31 - jepsen.util 8671 :ok :read [10 49] INFO [2019-12-17 12:35:33,695] jepsen worker 31 - jepsen.util 8671 :invoke :write [10 24] INFO [2019-12-17 12:35:33,698] jepsen worker 85 - jepsen.util 13945 :info :write [28 29] :RequestCanceledException INFO [2019-12-17 12:35:33,700] jepsen worker 85 - jepsen.util 14035 :invoke :read [28 nil] INFO [2019-12-17 12:35:33,728] jepsen worker 29 - jepsen.util 7949 :fail :cas [9 [40 42]] :ValueNotSwapFrom INFO [2019-12-17 12:35:33,728] jepsen worker 29 - jepsen.util 7949 :invoke :read [9 nil] INFO [2019-12-17 12:35:33,730] jepsen worker 79 - jepsen.util 9799 :info :write [26 39] :RequestCanceledException INFO [2019-12-17 12:35:33,732] jepsen worker 79 - jepsen.util 9889 :invoke :write [26 10] INFO [2019-12-17 12:35:33,745] jepsen worker 83 - jepsen.util 13583 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:35:33,745] jepsen worker 83 - jepsen.util 13583 :invoke :write [27 43] INFO [2019-12-17 12:35:33,794] jepsen worker 29 - jepsen.util 7949 :ok :read [9 13] INFO [2019-12-17 12:35:33,794] jepsen worker 29 - jepsen.util 7949 :invoke :read [9 nil] INFO [2019-12-17 12:35:33,796] jepsen worker 10 - jepsen.util 9730 :fail :cas [3 [42 0]] :ValueNotSwapFrom INFO [2019-12-17 12:35:33,797] jepsen worker 10 - jepsen.util 9730 :invoke :cas [3 [27 41]] INFO [2019-12-17 12:35:33,834] jepsen worker 29 - jepsen.util 7949 :ok :read [9 13] INFO [2019-12-17 12:35:33,835] jepsen worker 29 - jepsen.util 7949 :invoke :read [9 nil] INFO [2019-12-17 12:35:33,877] jepsen worker 13 - jepsen.util 9013 :ok :read [4 41] INFO [2019-12-17 12:35:33,877] jepsen worker 13 - jepsen.util 9013 :invoke :write [4 32] INFO [2019-12-17 12:35:33,886] jepsen worker 29 - jepsen.util 7949 :ok :read [9 13] INFO [2019-12-17 12:35:33,886] jepsen worker 29 - jepsen.util 7949 :invoke :read [9 nil] INFO [2019-12-17 12:35:33,927] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55690,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:35:33,927] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55690,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:35:33,928] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55694,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:35:33,928] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55694,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:35:33,929] jepsen worker 29 - jepsen.util 7949 :ok :read [9 13] INFO [2019-12-17 12:35:33,929] jepsen worker 29 - jepsen.util 7949 :invoke :write [9 6] INFO [2019-12-17 12:35:33,931] jepsen worker 40 - jepsen.util 13360 :info :write [13 45] :RequestCanceledException INFO [2019-12-17 12:35:33,932] jepsen worker 49 - jepsen.util 12829 :info :write [16 38] :RequestCanceledException INFO [2019-12-17 12:35:33,935] jepsen worker 49 - jepsen.util 12919 :invoke :write [16 8] INFO [2019-12-17 12:35:33,935] jepsen worker 40 - jepsen.util 13450 :invoke :cas [13 [17 15]] INFO [2019-12-17 12:35:33,998] jepsen worker 66 - jepsen.util 12936 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:35:33,998] jepsen worker 32 - jepsen.util 9662 :info :write [10 22] :RequestCanceledException INFO [2019-12-17 12:35:33,998] jepsen worker 66 - jepsen.util 12936 :invoke :read [22 nil] INFO [2019-12-17 12:35:33,999] jepsen worker 29 - jepsen.util 7949 :ok :write [9 6] INFO [2019-12-17 12:35:34,000] jepsen worker 32 - jepsen.util 9752 :invoke :write [10 24] INFO [2019-12-17 12:35:34,002] jepsen worker 29 - jepsen.util 7949 :invoke :read [9 nil] INFO [2019-12-17 12:35:34,027] jepsen worker 72 - jepsen.util 9432 :ok :write [24 8] INFO [2019-12-17 12:35:34,027] jepsen worker 72 - jepsen.util 9432 :invoke :write [24 23] INFO [2019-12-17 12:35:34,029] jepsen worker 26 - jepsen.util 13616 :info :write [8 31] :RequestCanceledException INFO [2019-12-17 12:35:34,029] jepsen worker 45 - jepsen.util 9855 :ok :write [15 1] INFO [2019-12-17 12:35:34,030] jepsen worker 45 - jepsen.util 9855 :invoke :cas [15 [9 9]] INFO [2019-12-17 12:35:34,033] jepsen worker 26 - jepsen.util 13706 :invoke :cas [8 [27 28]] INFO [2019-12-17 12:35:34,045] jepsen worker 61 - jepsen.util 14101 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:35:34,045] jepsen worker 61 - jepsen.util 14101 :invoke :read [20 nil] INFO [2019-12-17 12:35:34,054] jepsen worker 29 - jepsen.util 7949 :ok :read [9 6] INFO [2019-12-17 12:35:34,060] jepsen worker 29 - jepsen.util 7949 :invoke :cas [9 [27 41]] INFO [2019-12-17 12:35:34,061] jepsen worker 69 - jepsen.util 9519 :info :write [23 22] :RequestCanceledException INFO [2019-12-17 12:35:34,073] jepsen worker 69 - jepsen.util 9609 :invoke :write [23 4] INFO [2019-12-17 12:35:34,077] jepsen worker 14 - jepsen.util 9554 :fail :cas [4 [7 34]] :ValueNotSwapFrom INFO [2019-12-17 12:35:34,077] jepsen worker 14 - jepsen.util 9554 :invoke :cas [4 [1 46]] INFO [2019-12-17 12:35:34,097] jepsen worker 54 - jepsen.util 10494 :info :write [18 36] :RequestCanceledException INFO [2019-12-17 12:35:34,097] jepsen worker 63 - jepsen.util 8523 :info :cas [21 [46 19]] :RequestCanceledException INFO [2019-12-17 12:35:34,100] jepsen worker 54 - jepsen.util 10584 :invoke :write [18 18] INFO [2019-12-17 12:35:34,100] jepsen worker 63 - jepsen.util 8613 :invoke :write [21 2] INFO [2019-12-17 12:35:34,100] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55704,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:35:34,100] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55704,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:35:34,100] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55706,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:35:34,100] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55706,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:35:34,108] jepsen worker 29 - jepsen.util 7949 :fail :cas [9 [27 41]] :ValueNotSwapFrom INFO [2019-12-17 12:35:34,108] jepsen worker 29 - jepsen.util 7949 :invoke :write [9 43] INFO [2019-12-17 12:35:34,112] jepsen worker 0 - jepsen.util 13860 :info :write [0 46] :RequestCanceledException INFO [2019-12-17 12:35:34,112] jepsen worker 68 - jepsen.util 12668 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:35:34,112] jepsen worker 74 - jepsen.util 9344 :info :write [24 11] :RequestCanceledException INFO [2019-12-17 12:35:34,112] jepsen worker 68 - jepsen.util 12668 :invoke :cas [22 [13 33]] INFO [2019-12-17 12:35:34,116] jepsen worker 0 - jepsen.util 13950 :invoke :cas [0 [13 12]] INFO [2019-12-17 12:35:34,116] jepsen worker 74 - jepsen.util 9434 :invoke :cas [24 [28 29]] INFO [2019-12-17 12:35:34,131] jepsen worker 82 - jepsen.util 13762 :info :write [27 41] :RequestCanceledException INFO [2019-12-17 12:35:34,131] jepsen worker 7 - jepsen.util 13147 :info :cas [2 [38 42]] :RequestCanceledException INFO [2019-12-17 12:35:34,134] jepsen worker 7 - jepsen.util 13237 :invoke :read [2 nil] INFO [2019-12-17 12:35:34,134] jepsen worker 82 - jepsen.util 13852 :invoke :read [27 nil] INFO [2019-12-17 12:35:34,144] jepsen worker 46 - jepsen.util 9676 :info :write [15 5] :RequestCanceledException INFO [2019-12-17 12:35:34,147] jepsen worker 46 - jepsen.util 9766 :invoke :write [15 35] INFO [2019-12-17 12:35:34,148] jepsen worker 74 - jepsen.util 9434 :fail :cas [24 [28 29]] :ValueNotSwapFrom INFO [2019-12-17 12:35:34,148] jepsen worker 74 - jepsen.util 9434 :invoke :read [24 nil] INFO [2019-12-17 12:35:34,152] jepsen worker 55 - jepsen.util 10315 :ok :read [18 25] INFO [2019-12-17 12:35:34,153] jepsen worker 55 - jepsen.util 10315 :invoke :cas [18 [7 44]] INFO [2019-12-17 12:35:34,159] jepsen worker 75 - jepsen.util 12585 :info :cas [25 [31 30]] :RequestCanceledException INFO [2019-12-17 12:35:34,159] jepsen worker 64 - jepsen.util 7444 :info :write [21 45] :RequestCanceledException INFO [2019-12-17 12:35:34,162] jepsen worker 75 - jepsen.util 12675 :invoke :cas [25 [13 30]] INFO [2019-12-17 12:35:34,163] jepsen worker 64 - jepsen.util 7534 :invoke :read [21 nil] INFO [2019-12-17 12:35:34,182] jepsen worker 53 - jepsen.util 9053 :ok :read [17 24] INFO [2019-12-17 12:35:34,183] jepsen worker 53 - jepsen.util 9053 :invoke :write [17 25] INFO [2019-12-17 12:35:34,198] jepsen worker 1 - jepsen.util 13231 :info :cas [0 [29 30]] :RequestCanceledException INFO [2019-12-17 12:35:34,200] jepsen worker 1 - jepsen.util 13321 :invoke :read [0 nil] INFO [2019-12-17 12:35:34,205] jepsen worker 63 - jepsen.util 8613 :ok :write [21 2] INFO [2019-12-17 12:35:34,206] jepsen worker 63 - jepsen.util 8613 :invoke :cas [21 [25 48]] INFO [2019-12-17 12:35:34,208] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55730,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:35:34,208] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55730,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:35:34,208] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55732,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:35:34,209] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55732,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:35:34,212] jepsen worker 87 - jepsen.util 9177 :info :write [29 24] :RequestCanceledException INFO [2019-12-17 12:35:34,216] jepsen worker 87 - jepsen.util 9267 :invoke :read [29 nil] INFO [2019-12-17 12:35:34,221] jepsen worker 74 - jepsen.util 9434 :ok :read [24 8] INFO [2019-12-17 12:35:34,221] jepsen worker 74 - jepsen.util 9434 :invoke :cas [24 [16 39]] INFO [2019-12-17 12:35:34,231] jepsen worker 25 - jepsen.util 14245 :info :write [8 30] :RequestCanceledException INFO [2019-12-17 12:35:34,232] jepsen worker 58 - jepsen.util 13738 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:35:34,232] jepsen worker 58 - jepsen.util 13738 :invoke :cas [19 [36 48]] INFO [2019-12-17 12:35:34,233] jepsen worker 25 - jepsen.util 14335 :invoke :read [8 nil] INFO [2019-12-17 12:35:34,277] jepsen worker 74 - jepsen.util 9434 :fail :cas [24 [16 39]] :ValueNotSwapFrom INFO [2019-12-17 12:35:34,277] jepsen worker 74 - jepsen.util 9434 :invoke :write [24 43] INFO [2019-12-17 12:35:34,277] jepsen worker 29 - jepsen.util 7949 :ok :write [9 43] INFO [2019-12-17 12:35:34,278] jepsen worker 29 - jepsen.util 7949 :invoke :read [9 nil] INFO [2019-12-17 12:35:34,302] jepsen worker 29 - jepsen.util 7949 :ok :read [9 43] INFO [2019-12-17 12:35:34,303] jepsen worker 29 - jepsen.util 7949 :invoke :write [9 21] INFO [2019-12-17 12:35:34,311] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55740,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:35:34,311] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55740,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:35:34,312] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55744,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:35:34,312] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55744,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:35:34,329] jepsen worker 87 - jepsen.util 9267 :ok :read [29 40] INFO [2019-12-17 12:35:34,329] jepsen worker 87 - jepsen.util 9267 :invoke :write [29 8] INFO [2019-12-17 12:35:34,360] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:35:34,360] jepsen nemesis - couchbase.util Rebalance check count: 436 INFO [2019-12-17 12:35:34,360] jepsen nemesis - couchbase.util Rebalance stuck count: 417 INFO [2019-12-17 12:35:34,368] jepsen worker 37 - jepsen.util 9127 :ok :read [12 41] INFO [2019-12-17 12:35:34,368] jepsen worker 37 - jepsen.util 9127 :invoke :read [12 nil] INFO [2019-12-17 12:35:34,375] jepsen worker 29 - jepsen.util 7949 :ok :write [9 21] INFO [2019-12-17 12:35:34,376] jepsen worker 29 - jepsen.util 7949 :invoke :write [9 7] INFO [2019-12-17 12:35:34,413] jepsen worker 4 - jepsen.util 9094 :fail :cas [1 [31 2]] :ValueNotSwapFrom INFO [2019-12-17 12:35:34,413] jepsen worker 4 - jepsen.util 9094 :invoke :write [1 4] INFO [2019-12-17 12:35:34,414] jepsen worker 9 - jepsen.util 10179 :info :write [3 8] :RequestCanceledException INFO [2019-12-17 12:35:34,418] jepsen worker 9 - jepsen.util 10269 :invoke :cas [3 [1 45]] INFO [2019-12-17 12:35:34,431] jepsen worker 81 - jepsen.util 12951 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:35:34,431] jepsen worker 81 - jepsen.util 12951 :invoke :cas [27 [38 27]] INFO [2019-12-17 12:35:34,431] jepsen worker 11 - jepsen.util 9551 :info :cas [3 [18 43]] :RequestCanceledException INFO [2019-12-17 12:35:34,434] jepsen worker 11 - jepsen.util 9641 :invoke :read [3 nil] INFO [2019-12-17 12:35:34,438] jepsen worker 29 - jepsen.util 7949 :ok :write [9 7] INFO [2019-12-17 12:35:34,438] jepsen worker 29 - jepsen.util 7949 :invoke :read [9 nil] INFO [2019-12-17 12:35:34,444] jepsen worker 89 - jepsen.util 9539 :info :write [29 48] :RequestCanceledException INFO [2019-12-17 12:35:34,446] jepsen worker 89 - jepsen.util 9629 :invoke :write [29 45] INFO [2019-12-17 12:35:34,470] jepsen worker 11 - jepsen.util 9641 :ok :read [3 0] INFO [2019-12-17 12:35:34,470] jepsen worker 11 - jepsen.util 9641 :invoke :read [3 nil] INFO [2019-12-17 12:35:34,487] jepsen worker 29 - jepsen.util 7949 :ok :read [9 7] INFO [2019-12-17 12:35:34,488] jepsen worker 29 - jepsen.util 7949 :invoke :write [9 13] INFO [2019-12-17 12:35:34,496] jepsen worker 52 - jepsen.util 9592 :ok :read [17 24] INFO [2019-12-17 12:35:34,496] jepsen worker 52 - jepsen.util 9592 :invoke :write [17 13] INFO [2019-12-17 12:35:34,511] jepsen worker 11 - jepsen.util 9641 :ok :read [3 0] INFO [2019-12-17 12:35:34,511] jepsen worker 11 - jepsen.util 9641 :invoke :read [3 nil] INFO [2019-12-17 12:35:34,515] jepsen worker 78 - jepsen.util 9528 :info :write [26 26] :RequestCanceledException INFO [2019-12-17 12:35:34,517] jepsen worker 78 - jepsen.util 9618 :invoke :write [26 35] INFO [2019-12-17 12:35:34,529] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55758,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:35:34,529] jepsen worker 48 - jepsen.util 12108 :info :cas [16 [10 21]] :RequestCanceledException INFO [2019-12-17 12:35:34,529] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55758,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:35:34,529] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55761,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:35:34,530] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55761,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:35:34,530] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55762,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:35:34,530] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55762,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:35:34,532] jepsen worker 48 - jepsen.util 12198 :invoke :write [16 0] INFO [2019-12-17 12:35:34,544] jepsen worker 70 - jepsen.util 10330 :info :write [23 10] :RequestCanceledException INFO [2019-12-17 12:35:34,546] jepsen worker 29 - jepsen.util 7949 :ok :write [9 13] INFO [2019-12-17 12:35:34,546] jepsen worker 29 - jepsen.util 7949 :invoke :write [9 0] INFO [2019-12-17 12:35:34,546] jepsen worker 70 - jepsen.util 10420 :invoke :cas [23 [19 49]] INFO [2019-12-17 12:35:34,596] jepsen worker 39 - jepsen.util 12639 :info :cas [13 [19 1]] :RequestCanceledException INFO [2019-12-17 12:35:34,596] jepsen worker 5 - jepsen.util 9005 :info :write [1 9] :RequestCanceledException INFO [2019-12-17 12:35:34,598] jepsen worker 29 - jepsen.util 7949 :ok :write [9 0] INFO [2019-12-17 12:35:34,598] jepsen worker 39 - jepsen.util 12729 :invoke :write [13 20] INFO [2019-12-17 12:35:34,598] jepsen worker 29 - jepsen.util 7949 :invoke :write [9 23] INFO [2019-12-17 12:35:34,598] jepsen worker 5 - jepsen.util 9095 :invoke :read [1 nil] INFO [2019-12-17 12:35:34,615] jepsen worker 55 - jepsen.util 10315 :fail :cas [18 [7 44]] :ValueNotSwapFrom INFO [2019-12-17 12:35:34,615] jepsen worker 55 - jepsen.util 10315 :invoke :read [18 nil] INFO [2019-12-17 12:35:34,645] jepsen worker 14 - jepsen.util 9554 :fail :cas [4 [1 46]] :ValueNotSwapFrom INFO [2019-12-17 12:35:34,645] jepsen worker 14 - jepsen.util 9554 :invoke :read [4 nil] INFO [2019-12-17 12:35:34,650] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55780,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:35:34,650] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55780,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:35:34,651] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55784,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:35:34,651] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55784,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:35:34,653] jepsen worker 29 - jepsen.util 7949 :ok :write [9 23] INFO [2019-12-17 12:35:34,653] jepsen worker 29 - jepsen.util 7949 :invoke :write [9 20] INFO [2019-12-17 12:35:34,674] jepsen worker 37 - jepsen.util 9127 :ok :read [12 41] INFO [2019-12-17 12:35:34,675] jepsen worker 37 - jepsen.util 9127 :invoke :write [12 10] INFO [2019-12-17 12:35:34,711] jepsen worker 19 - jepsen.util 9739 :info :write [6 49] :RequestCanceledException INFO [2019-12-17 12:35:34,712] jepsen worker 47 - jepsen.util 10127 :info :cas [15 [3 3]] :RequestCanceledException INFO [2019-12-17 12:35:34,714] jepsen worker 19 - jepsen.util 9829 :invoke :cas [6 [27 6]] INFO [2019-12-17 12:35:34,716] jepsen worker 47 - jepsen.util 10217 :invoke :write [15 13] INFO [2019-12-17 12:35:34,722] jepsen worker 29 - jepsen.util 7949 :ok :write [9 20] INFO [2019-12-17 12:35:34,722] jepsen worker 29 - jepsen.util 7949 :invoke :write [9 43] INFO [2019-12-17 12:35:34,727] jepsen worker 51 - jepsen.util 8961 :fail :cas [17 [8 16]] :ValueNotSwapFrom INFO [2019-12-17 12:35:34,727] jepsen worker 51 - jepsen.util 8961 :invoke :write [17 36] INFO [2019-12-17 12:35:34,731] jepsen worker 80 - jepsen.util 9260 :info :write [26 5] :RequestCanceledException INFO [2019-12-17 12:35:34,731] jepsen worker 33 - jepsen.util 13083 :info :write [11 22] :RequestCanceledException INFO [2019-12-17 12:35:34,737] jepsen worker 80 - jepsen.util 9350 :invoke :cas [26 [33 11]] INFO [2019-12-17 12:35:34,737] jepsen worker 33 - jepsen.util 13173 :invoke :write [11 44] INFO [2019-12-17 12:35:34,793] jepsen worker 29 - jepsen.util 7949 :ok :write [9 43] INFO [2019-12-17 12:35:34,794] jepsen worker 29 - jepsen.util 7949 :invoke :cas [9 [48 28]] INFO [2019-12-17 12:35:34,831] jepsen worker 12 - jepsen.util 9912 :info :write [4 15] :RequestCanceledException INFO [2019-12-17 12:35:34,834] jepsen worker 29 - jepsen.util 7949 :fail :cas [9 [48 28]] :ValueNotSwapFrom INFO [2019-12-17 12:35:34,834] jepsen worker 29 - jepsen.util 7949 :invoke :read [9 nil] INFO [2019-12-17 12:35:34,836] jepsen worker 12 - jepsen.util 10002 :invoke :cas [4 [35 15]] INFO [2019-12-17 12:35:34,843] jepsen worker 80 - jepsen.util 9350 :fail :cas [26 [33 11]] :ValueNotSwapFrom INFO [2019-12-17 12:35:34,844] jepsen worker 80 - jepsen.util 9350 :invoke :cas [26 [11 47]] INFO [2019-12-17 12:35:34,846] jepsen worker 5 - jepsen.util 9095 :ok :read [1 47] INFO [2019-12-17 12:35:34,846] jepsen worker 5 - jepsen.util 9095 :invoke :cas [1 [29 11]] INFO [2019-12-17 12:35:34,859] jepsen worker 59 - jepsen.util 12479 :info :cas [19 [39 13]] :RequestCanceledException INFO [2019-12-17 12:35:34,864] jepsen worker 59 - jepsen.util 12569 :invoke :cas [19 [39 44]] INFO [2019-12-17 12:35:34,873] jepsen worker 29 - jepsen.util 7949 :ok :read [9 43] INFO [2019-12-17 12:35:34,873] jepsen worker 29 - jepsen.util 7949 :invoke :cas [9 [26 12]] INFO [2019-12-17 12:35:34,881] jepsen worker 14 - jepsen.util 9554 :ok :read [4 41] INFO [2019-12-17 12:35:34,881] jepsen worker 14 - jepsen.util 9554 :invoke :read [4 nil] INFO [2019-12-17 12:35:34,896] jepsen worker 29 - jepsen.util 7949 :fail :cas [9 [26 12]] :ValueNotSwapFrom INFO [2019-12-17 12:35:34,896] jepsen worker 29 - jepsen.util 7949 :invoke :read [9 nil] INFO [2019-12-17 12:35:34,903] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55803,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:35:34,903] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55803,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:35:34,903] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55804,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:35:34,903] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55804,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:35:34,922] jepsen worker 29 - jepsen.util 7949 :ok :read [9 43] INFO [2019-12-17 12:35:34,923] jepsen worker 29 - jepsen.util 7949 :invoke :write [9 48] INFO [2019-12-17 12:35:34,930] jepsen worker 65 - jepsen.util 8075 :info :cas [21 [45 39]] :RequestCanceledException INFO [2019-12-17 12:35:34,931] jepsen worker 77 - jepsen.util 13757 :info :write [25 48] :RequestCanceledException INFO [2019-12-17 12:35:34,931] jepsen worker 50 - jepsen.util 13910 :info :write [16 35] :RequestCanceledException INFO [2019-12-17 12:35:34,935] jepsen worker 65 - jepsen.util 8165 :invoke :read [21 nil] INFO [2019-12-17 12:35:34,935] jepsen worker 77 - jepsen.util 13847 :invoke :cas [25 [17 21]] INFO [2019-12-17 12:35:34,935] jepsen worker 50 - jepsen.util 14000 :invoke :cas [16 [19 17]] INFO [2019-12-17 12:35:34,945] jepsen worker 22 - jepsen.util 13072 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:35:34,945] jepsen worker 88 - jepsen.util 9898 :info :write [29 25] :RequestCanceledException INFO [2019-12-17 12:35:34,945] jepsen worker 22 - jepsen.util 13072 :invoke :read [7 nil] INFO [2019-12-17 12:35:34,957] jepsen worker 88 - jepsen.util 9988 :invoke :read [29 nil] INFO [2019-12-17 12:35:34,999] jepsen worker 41 - jepsen.util 14171 :info :write [13 26] :RequestCanceledException INFO [2019-12-17 12:35:34,999] jepsen worker 8 - jepsen.util 12788 :info :write [2 1] :RequestCanceledException INFO [2019-12-17 12:35:35,014] jepsen worker 67 - jepsen.util 13027 :info :write [22 8] :RequestCanceledException INFO [2019-12-17 12:35:35,019] jepsen worker 41 - jepsen.util 14261 :invoke :write [13 47] INFO [2019-12-17 12:35:35,019] jepsen worker 8 - jepsen.util 12878 :invoke :read [2 nil] INFO [2019-12-17 12:35:35,026] jepsen worker 29 - jepsen.util 7949 :ok :write [9 48] INFO [2019-12-17 12:35:35,026] jepsen worker 29 - jepsen.util 7949 :invoke :read [9 nil] INFO [2019-12-17 12:35:35,031] jepsen worker 2 - jepsen.util 13142 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:35:35,031] jepsen worker 28 - jepsen.util 7858 :info :write [9 24] :RequestCanceledException INFO [2019-12-17 12:35:35,032] jepsen worker 2 - jepsen.util 13142 :invoke :write [0 33] INFO [2019-12-17 12:35:35,032] jepsen worker 67 - jepsen.util 13117 :invoke :cas [22 [12 20]] INFO [2019-12-17 12:35:35,042] jepsen worker 28 - jepsen.util 7948 :invoke :read [9 nil] INFO [2019-12-17 12:35:35,046] jepsen worker 16 - jepsen.util 14326 :info :write [5 35] :RequestCanceledException INFO [2019-12-17 12:35:35,052] jepsen worker 16 - jepsen.util 14416 :invoke :write [5 33] INFO [2019-12-17 12:35:35,054] jepsen worker 55 - jepsen.util 10315 :ok :read [18 25] INFO [2019-12-17 12:35:35,054] jepsen worker 55 - jepsen.util 10315 :invoke :cas [18 [18 9]] INFO [2019-12-17 12:35:35,062] jepsen worker 34 - jepsen.util 13084 :info :write [11 41] :RequestCanceledException INFO [2019-12-17 12:35:35,063] jepsen worker 36 - jepsen.util 10296 :info :write [12 48] :RequestCanceledException INFO [2019-12-17 12:35:35,069] jepsen worker 34 - jepsen.util 13174 :invoke :write [11 32] INFO [2019-12-17 12:35:35,071] jepsen worker 36 - jepsen.util 10386 :invoke :cas [12 [13 30]] INFO [2019-12-17 12:35:35,075] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55816,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:35:35,075] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55816,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:35:35,076] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55818,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:35:35,076] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55818,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:35:35,077] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55819,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:35:35,077] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55819,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:35:35,088] jepsen worker 14 - jepsen.util 9554 :ok :read [4 41] INFO [2019-12-17 12:35:35,088] jepsen worker 14 - jepsen.util 9554 :invoke :cas [4 [12 11]] INFO [2019-12-17 12:35:35,098] jepsen worker 84 - jepsen.util 12774 :info :write [28 46] :RequestCanceledException INFO [2019-12-17 12:35:35,098] jepsen worker 21 - jepsen.util 12711 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:35:35,098] jepsen worker 21 - jepsen.util 12711 :invoke :cas [7 [33 14]] INFO [2019-12-17 12:35:35,099] jepsen worker 80 - jepsen.util 9350 :fail :cas [26 [11 47]] :ValueNotSwapFrom INFO [2019-12-17 12:35:35,100] jepsen worker 80 - jepsen.util 9350 :invoke :cas [26 [43 20]] INFO [2019-12-17 12:35:35,102] jepsen worker 84 - jepsen.util 12864 :invoke :write [28 12] INFO [2019-12-17 12:35:35,107] jepsen worker 28 - jepsen.util 7948 :ok :read [9 48] INFO [2019-12-17 12:35:35,108] jepsen worker 28 - jepsen.util 7948 :invoke :write [9 36] INFO [2019-12-17 12:35:35,108] jepsen worker 29 - jepsen.util 7949 :ok :read [9 48] INFO [2019-12-17 12:35:35,108] jepsen worker 29 - jepsen.util 7949 :invoke :read [9 nil] INFO [2019-12-17 12:35:35,116] jepsen worker 38 - jepsen.util 10118 :info :write [12 23] :RequestCanceledException INFO [2019-12-17 12:35:35,129] jepsen worker 38 - jepsen.util 10208 :invoke :cas [12 [42 9]] INFO [2019-12-17 12:35:35,131] jepsen worker 15 - jepsen.util 12345 :info :write [5 26] :RequestCanceledException INFO [2019-12-17 12:35:35,132] jepsen worker 86 - jepsen.util 13946 :info :cas [28 [18 30]] :RequestCanceledException INFO [2019-12-17 12:35:35,134] jepsen worker 15 - jepsen.util 12435 :invoke :cas [5 [19 38]] INFO [2019-12-17 12:35:35,135] jepsen worker 86 - jepsen.util 14036 :invoke :read [28 nil] INFO [2019-12-17 12:35:35,143] jepsen worker 29 - jepsen.util 7949 :ok :read [9 48] INFO [2019-12-17 12:35:35,143] jepsen worker 29 - jepsen.util 7949 :invoke :write [9 30] INFO [2019-12-17 12:35:35,145] jepsen worker 18 - jepsen.util 10008 :fail :read [6 nil] :RequestTimeoutException INFO [2019-12-17 12:35:35,145] jepsen worker 18 - jepsen.util 10008 :invoke :read [6 nil] INFO [2019-12-17 12:35:35,163] jepsen worker 43 - jepsen.util 13093 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:35:35,163] jepsen worker 43 - jepsen.util 13093 :invoke :cas [14 [49 29]] INFO [2019-12-17 12:35:35,197] jepsen worker 5 - jepsen.util 9095 :fail :cas [1 [29 11]] :ValueNotSwapFrom INFO [2019-12-17 12:35:35,197] jepsen worker 5 - jepsen.util 9095 :invoke :read [1 nil] INFO [2019-12-17 12:35:35,202] jepsen worker 28 - jepsen.util 7948 :ok :write [9 36] INFO [2019-12-17 12:35:35,203] jepsen worker 28 - jepsen.util 7948 :invoke :cas [9 [17 32]] INFO [2019-12-17 12:35:35,233] jepsen worker 28 - jepsen.util 7948 :fail :cas [9 [17 32]] :ValueNotSwapFrom INFO [2019-12-17 12:35:35,234] jepsen worker 28 - jepsen.util 7948 :invoke :read [9 nil] INFO [2019-12-17 12:35:35,250] jepsen worker 42 - jepsen.util 14262 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:35:35,250] jepsen worker 42 - jepsen.util 14262 :invoke :write [14 15] INFO [2019-12-17 12:35:35,255] jepsen worker 29 - jepsen.util 7949 :ok :write [9 30] INFO [2019-12-17 12:35:35,255] jepsen worker 29 - jepsen.util 7949 :invoke :write [9 38] INFO [2019-12-17 12:35:35,275] jepsen worker 55 - jepsen.util 10315 :fail :cas [18 [18 9]] :ValueNotSwapFrom INFO [2019-12-17 12:35:35,275] jepsen worker 55 - jepsen.util 10315 :invoke :cas [18 [28 38]] INFO [2019-12-17 12:35:35,282] jepsen worker 28 - jepsen.util 7948 :ok :read [9 36] INFO [2019-12-17 12:35:35,282] jepsen worker 28 - jepsen.util 7948 :invoke :read [9 nil] INFO [2019-12-17 12:35:35,292] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55853,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:35:35,292] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55853,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:35:35,292] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55855,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:35:35,292] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55855,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:35:35,293] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55843,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:35:35,293] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55843,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:35:35,293] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55844,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:35:35,293] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55844,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:35:35,300] jepsen worker 56 - jepsen.util 10136 :info :write [18 13] :RequestCanceledException INFO [2019-12-17 12:35:35,303] jepsen worker 56 - jepsen.util 10226 :invoke :read [18 nil] INFO [2019-12-17 12:35:35,315] jepsen worker 60 - jepsen.util 14280 :info :cas [20 [44 24]] :RequestCanceledException INFO [2019-12-17 12:35:35,315] jepsen worker 73 - jepsen.util 8983 :info :write [24 4] :RequestCanceledException INFO [2019-12-17 12:35:35,319] jepsen worker 73 - jepsen.util 9073 :invoke :write [24 26] INFO [2019-12-17 12:35:35,319] jepsen worker 60 - jepsen.util 14370 :invoke :read [20 nil] INFO [2019-12-17 12:35:35,329] jepsen worker 38 - jepsen.util 10208 :fail :cas [12 [42 9]] :ValueNotSwapFrom INFO [2019-12-17 12:35:35,329] jepsen worker 38 - jepsen.util 10208 :invoke :write [12 48] INFO [2019-12-17 12:35:35,331] jepsen worker 17 - jepsen.util 12797 :info :write [5 14] :RequestCanceledException INFO [2019-12-17 12:35:35,332] jepsen worker 57 - jepsen.util 12747 :info :cas [19 [41 0]] :RequestCanceledException INFO [2019-12-17 12:35:35,333] jepsen worker 29 - jepsen.util 7949 :ok :write [9 38] INFO [2019-12-17 12:35:35,333] jepsen worker 29 - jepsen.util 7949 :invoke :cas [9 [23 49]] INFO [2019-12-17 12:35:35,336] jepsen worker 57 - jepsen.util 12837 :invoke :cas [19 [11 40]] INFO [2019-12-17 12:35:35,336] jepsen worker 17 - jepsen.util 12887 :invoke :read [5 nil] INFO [2019-12-17 12:35:35,340] jepsen worker 28 - jepsen.util 7948 :ok :read [9 30] INFO [2019-12-17 12:35:35,340] jepsen worker 28 - jepsen.util 7948 :invoke :write [9 16] INFO [2019-12-17 12:35:35,365] jepsen worker 29 - jepsen.util 7949 :fail :cas [9 [23 49]] :ValueNotSwapFrom INFO [2019-12-17 12:35:35,365] jepsen worker 29 - jepsen.util 7949 :invoke :cas [9 [45 42]] INFO [2019-12-17 12:35:35,373] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:35:35,373] jepsen nemesis - couchbase.util Rebalance check count: 437 INFO [2019-12-17 12:35:35,373] jepsen nemesis - couchbase.util Rebalance stuck count: 418 INFO [2019-12-17 12:35:35,384] jepsen worker 72 - jepsen.util 9432 :ok :write [24 23] INFO [2019-12-17 12:35:35,384] jepsen worker 72 - jepsen.util 9432 :invoke :read [24 nil] INFO [2019-12-17 12:35:35,385] jepsen worker 31 - jepsen.util 8671 :ok :write [10 24] INFO [2019-12-17 12:35:35,385] jepsen worker 31 - jepsen.util 8671 :invoke :write [10 2] INFO [2019-12-17 12:35:35,388] jepsen worker 46 - jepsen.util 9766 :ok :write [15 35] INFO [2019-12-17 12:35:35,388] jepsen worker 46 - jepsen.util 9766 :invoke :read [15 nil] INFO [2019-12-17 12:35:35,411] jepsen worker 5 - jepsen.util 9095 :ok :read [1 47] INFO [2019-12-17 12:35:35,411] jepsen worker 5 - jepsen.util 9095 :invoke :read [1 nil] INFO [2019-12-17 12:35:35,415] jepsen worker 27 - jepsen.util 9117 :info :write [9 20] :RequestCanceledException INFO [2019-12-17 12:35:35,420] jepsen worker 27 - jepsen.util 9207 :invoke :cas [9 [44 13]] INFO [2019-12-17 12:35:35,431] jepsen worker 76 - jepsen.util 13396 :info :write [25 21] :RequestCanceledException INFO [2019-12-17 12:35:35,436] jepsen worker 76 - jepsen.util 13486 :invoke :read [25 nil] INFO [2019-12-17 12:35:35,448] jepsen worker 29 - jepsen.util 7949 :fail :cas [9 [45 42]] :ValueNotSwapFrom INFO [2019-12-17 12:35:35,449] jepsen worker 29 - jepsen.util 7949 :invoke :read [9 nil] INFO [2019-12-17 12:35:35,456] jepsen worker 12 - jepsen.util 10002 :fail :cas [4 [35 15]] :ValueNotSwapFrom INFO [2019-12-17 12:35:35,456] jepsen worker 12 - jepsen.util 10002 :invoke :read [4 nil] INFO [2019-12-17 12:35:35,463] jepsen worker 3 - jepsen.util 9453 :info :write [1 42] :RequestCanceledException INFO [2019-12-17 12:35:35,467] jepsen worker 3 - jepsen.util 9543 :invoke :read [1 nil] INFO [2019-12-17 12:35:35,498] jepsen worker 6 - jepsen.util 13686 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:35:35,498] jepsen worker 6 - jepsen.util 13686 :invoke :cas [2 [16 34]] INFO [2019-12-17 12:35:35,506] jepsen worker 28 - jepsen.util 7948 :ok :write [9 16] INFO [2019-12-17 12:35:35,506] jepsen worker 28 - jepsen.util 7948 :invoke :write [9 26] INFO [2019-12-17 12:35:35,511] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55877,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:35:35,511] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55877,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:35:35,515] jepsen worker 44 - jepsen.util 13454 :info :cas [14 [25 25]] :RequestCanceledException INFO [2019-12-17 12:35:35,517] jepsen worker 64 - jepsen.util 7534 :ok :read [21 2] INFO [2019-12-17 12:35:35,517] jepsen worker 64 - jepsen.util 7534 :invoke :write [21 7] INFO [2019-12-17 12:35:35,521] jepsen worker 44 - jepsen.util 13544 :invoke :read [14 nil] INFO [2019-12-17 12:35:35,527] jepsen worker 10 - jepsen.util 9730 :fail :cas [3 [27 41]] :ValueNotSwapFrom INFO [2019-12-17 12:35:35,527] jepsen worker 10 - jepsen.util 9730 :invoke :cas [3 [26 42]] INFO [2019-12-17 12:35:35,549] jepsen worker 55 - jepsen.util 10315 :fail :cas [18 [28 38]] :ValueNotSwapFrom INFO [2019-12-17 12:35:35,549] jepsen worker 55 - jepsen.util 10315 :invoke :read [18 nil] INFO [2019-12-17 12:35:35,561] jepsen worker 24 - jepsen.util 12084 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:35:35,561] jepsen worker 24 - jepsen.util 12084 :invoke :read [8 nil] INFO [2019-12-17 12:35:35,574] jepsen worker 9 - jepsen.util 10269 :fail :cas [3 [1 45]] :ValueNotSwapFrom INFO [2019-12-17 12:35:35,574] jepsen worker 9 - jepsen.util 10269 :invoke :write [3 6] INFO [2019-12-17 12:35:35,592] jepsen worker 29 - jepsen.util 7949 :ok :read [9 38] INFO [2019-12-17 12:35:35,592] jepsen worker 29 - jepsen.util 7949 :invoke :read [9 nil] INFO [2019-12-17 12:35:35,595] jepsen worker 19 - jepsen.util 9829 :fail :cas [6 [27 6]] :ValueNotSwapFrom INFO [2019-12-17 12:35:35,595] jepsen worker 19 - jepsen.util 9829 :invoke :cas [6 [32 12]] INFO [2019-12-17 12:35:35,611] jepsen worker 5 - jepsen.util 9095 :ok :read [1 47] INFO [2019-12-17 12:35:35,611] jepsen worker 5 - jepsen.util 9095 :invoke :cas [1 [45 17]] INFO [2019-12-17 12:35:35,615] jepsen worker 65 - jepsen.util 8165 :ok :read [21 2] INFO [2019-12-17 12:35:35,615] jepsen worker 65 - jepsen.util 8165 :invoke :write [21 11] INFO [2019-12-17 12:35:35,627] jepsen worker 18 - jepsen.util 10008 :ok :read [6 1] INFO [2019-12-17 12:35:35,628] jepsen worker 18 - jepsen.util 10008 :invoke :cas [6 [25 17]] INFO [2019-12-17 12:35:35,647] jepsen worker 72 - jepsen.util 9432 :ok :read [24 23] INFO [2019-12-17 12:35:35,647] jepsen worker 72 - jepsen.util 9432 :invoke :write [24 15] INFO [2019-12-17 12:35:35,652] jepsen worker 29 - jepsen.util 7949 :ok :read [9 16] INFO [2019-12-17 12:35:35,652] jepsen worker 29 - jepsen.util 7949 :invoke :cas [9 [13 10]] INFO [2019-12-17 12:35:35,675] jepsen worker 36 - jepsen.util 10386 :fail :cas [12 [13 30]] :ValueNotSwapFrom INFO [2019-12-17 12:35:35,676] jepsen worker 36 - jepsen.util 10386 :invoke :cas [12 [30 19]] INFO [2019-12-17 12:35:35,699] jepsen worker 35 - jepsen.util 13085 :info :write [11 45] :RequestCanceledException INFO [2019-12-17 12:35:35,704] jepsen worker 35 - jepsen.util 13175 :invoke :cas [11 [44 28]] INFO [2019-12-17 12:35:35,722] jepsen worker 14 - jepsen.util 9554 :fail :cas [4 [12 11]] :ValueNotSwapFrom INFO [2019-12-17 12:35:35,722] jepsen worker 14 - jepsen.util 9554 :invoke :read [4 nil] INFO [2019-12-17 12:35:35,724] jepsen worker 80 - jepsen.util 9350 :fail :cas [26 [43 20]] :ValueNotSwapFrom INFO [2019-12-17 12:35:35,724] jepsen worker 80 - jepsen.util 9350 :invoke :cas [26 [45 41]] INFO [2019-12-17 12:35:35,726] jepsen worker 28 - jepsen.util 7948 :ok :write [9 26] INFO [2019-12-17 12:35:35,726] jepsen worker 28 - jepsen.util 7948 :invoke :write [9 38] INFO [2019-12-17 12:35:35,727] jepsen worker 29 - jepsen.util 7949 :fail :cas [9 [13 10]] :ValueNotSwapFrom INFO [2019-12-17 12:35:35,727] jepsen worker 29 - jepsen.util 7949 :invoke :write [9 21] INFO [2019-12-17 12:35:35,731] jepsen worker 71 - jepsen.util 10331 :info :write [23 10] :RequestCanceledException INFO [2019-12-17 12:35:35,732] jepsen worker 23 - jepsen.util 13253 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:35:35,733] jepsen worker 23 - jepsen.util 13253 :invoke :cas [7 [32 46]] INFO [2019-12-17 12:35:35,734] jepsen worker 71 - jepsen.util 10421 :invoke :read [23 nil] INFO [2019-12-17 12:35:35,737] jepsen worker 64 - jepsen.util 7534 :ok :write [21 7] INFO [2019-12-17 12:35:35,737] jepsen worker 64 - jepsen.util 7534 :invoke :cas [21 [44 39]] INFO [2019-12-17 12:35:35,739] jepsen worker 12 - jepsen.util 10002 :ok :read [4 41] INFO [2019-12-17 12:35:35,740] jepsen worker 12 - jepsen.util 10002 :invoke :read [4 nil] INFO [2019-12-17 12:35:35,745] jepsen worker 20 - jepsen.util 9200 :fail :read [6 nil] :RequestTimeoutException INFO [2019-12-17 12:35:35,745] jepsen worker 20 - jepsen.util 9200 :invoke :cas [6 [8 28]] INFO [2019-12-17 12:35:35,746] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55901,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:35:35,746] jepsen worker 30 - jepsen.util 9750 :info :cas [10 [9 24]] :RequestCanceledException INFO [2019-12-17 12:35:35,746] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55901,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:35:35,746] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55903,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:35:35,746] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55903,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:35:35,746] jepsen worker 62 - jepsen.util 13562 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:35:35,747] jepsen worker 62 - jepsen.util 13562 :invoke :cas [20 [23 0]] INFO [2019-12-17 12:35:35,750] jepsen worker 30 - jepsen.util 9840 :invoke :read [10 nil] INFO [2019-12-17 12:35:35,767] jepsen worker 70 - jepsen.util 10420 :fail :cas [23 [19 49]] :ValueNotSwapFrom INFO [2019-12-17 12:35:35,768] jepsen worker 70 - jepsen.util 10420 :invoke :write [23 7] INFO [2019-12-17 12:35:35,797] jepsen worker 85 - jepsen.util 14035 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:35:35,798] jepsen worker 85 - jepsen.util 14035 :invoke :write [28 33] INFO [2019-12-17 12:35:35,816] jepsen worker 79 - jepsen.util 9889 :info :write [26 10] :RequestCanceledException INFO [2019-12-17 12:35:35,834] jepsen worker 79 - jepsen.util 9979 :invoke :read [26 nil] INFO [2019-12-17 12:35:35,845] jepsen worker 83 - jepsen.util 13583 :info :write [27 43] :RequestCanceledException INFO [2019-12-17 12:35:35,847] jepsen worker 83 - jepsen.util 13673 :invoke :read [27 nil] INFO [2019-12-17 12:35:35,851] jepsen worker 28 - jepsen.util 7948 :ok :write [9 38] INFO [2019-12-17 12:35:35,851] jepsen worker 28 - jepsen.util 7948 :invoke :cas [9 [12 2]] INFO [2019-12-17 12:35:35,864] jepsen worker 3 - jepsen.util 9543 :ok :read [1 47] INFO [2019-12-17 12:35:35,864] jepsen worker 3 - jepsen.util 9543 :invoke :read [1 nil] INFO [2019-12-17 12:35:35,876] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55926,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:35:35,877] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55926,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:35:35,878] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55928,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:35:35,879] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55928,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:35:35,879] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55929,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:35:35,880] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55929,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:35:35,900] jepsen worker 80 - jepsen.util 9350 :fail :cas [26 [45 41]] :ValueNotSwapFrom INFO [2019-12-17 12:35:35,901] jepsen worker 80 - jepsen.util 9350 :invoke :cas [26 [30 15]] INFO [2019-12-17 12:35:35,932] jepsen worker 13 - jepsen.util 9013 :info :write [4 32] :RequestCanceledException INFO [2019-12-17 12:35:35,945] jepsen worker 28 - jepsen.util 7948 :fail :cas [9 [12 2]] :ValueNotSwapFrom INFO [2019-12-17 12:35:35,945] jepsen worker 28 - jepsen.util 7948 :invoke :cas [9 [20 15]] INFO [2019-12-17 12:35:35,949] jepsen worker 13 - jepsen.util 9103 :invoke :cas [4 [23 3]] INFO [2019-12-17 12:35:35,982] jepsen worker 88 - jepsen.util 9988 :ok :read [29 40] INFO [2019-12-17 12:35:35,982] jepsen worker 88 - jepsen.util 9988 :invoke :cas [29 [35 38]] INFO [2019-12-17 12:35:35,990] jepsen worker 28 - jepsen.util 7948 :fail :cas [9 [20 15]] :ValueNotSwapFrom INFO [2019-12-17 12:35:35,994] jepsen worker 28 - jepsen.util 7948 :invoke :read [9 nil] INFO [2019-12-17 12:35:36,021] jepsen worker 28 - jepsen.util 7948 :ok :read [9 38] INFO [2019-12-17 12:35:36,021] jepsen worker 28 - jepsen.util 7948 :invoke :cas [9 [20 2]] INFO [2019-12-17 12:35:36,027] jepsen worker 36 - jepsen.util 10386 :fail :cas [12 [30 19]] :ValueNotSwapFrom INFO [2019-12-17 12:35:36,027] jepsen worker 36 - jepsen.util 10386 :invoke :cas [12 [1 10]] INFO [2019-12-17 12:35:36,030] jepsen worker 49 - jepsen.util 12919 :info :write [16 8] :RequestCanceledException INFO [2019-12-17 12:35:36,032] jepsen worker 49 - jepsen.util 13009 :invoke :write [16 44] INFO [2019-12-17 12:35:36,045] jepsen worker 40 - jepsen.util 13450 :info :cas [13 [17 15]] :RequestCanceledException INFO [2019-12-17 12:35:36,048] jepsen worker 40 - jepsen.util 13540 :invoke :cas [13 [44 15]] INFO [2019-12-17 12:35:36,049] jepsen worker 28 - jepsen.util 7948 :fail :cas [9 [20 2]] :ValueNotSwapFrom INFO [2019-12-17 12:35:36,049] jepsen worker 28 - jepsen.util 7948 :invoke :write [9 12] INFO [2019-12-17 12:35:36,061] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55938,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:35:36,061] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55938,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:35:36,062] jepsen worker 45 - jepsen.util 9855 :info :cas [15 [9 9]] :RequestCanceledException INFO [2019-12-17 12:35:36,062] jepsen worker 32 - jepsen.util 9752 :info :write [10 24] :RequestCanceledException INFO [2019-12-17 12:35:36,064] jepsen worker 45 - jepsen.util 9945 :invoke :write [15 43] INFO [2019-12-17 12:35:36,064] jepsen worker 32 - jepsen.util 9842 :invoke :cas [10 [14 45]] INFO [2019-12-17 12:35:36,082] jepsen worker 56 - jepsen.util 10226 :ok :read [18 25] INFO [2019-12-17 12:35:36,082] jepsen worker 56 - jepsen.util 10226 :invoke :read [18 nil] INFO [2019-12-17 12:35:36,094] jepsen worker 32 - jepsen.util 9842 :fail :cas [10 [14 45]] :ValueNotSwapFrom INFO [2019-12-17 12:35:36,095] jepsen worker 32 - jepsen.util 9842 :invoke :write [10 17] INFO [2019-12-17 12:35:36,097] jepsen worker 66 - jepsen.util 12936 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:35:36,098] jepsen worker 66 - jepsen.util 12936 :invoke :write [22 12] INFO [2019-12-17 12:35:36,100] jepsen worker 69 - jepsen.util 9609 :info :write [23 4] :RequestCanceledException INFO [2019-12-17 12:35:36,105] jepsen worker 69 - jepsen.util 9699 :invoke :cas [23 [3 19]] INFO [2019-12-17 12:35:36,131] jepsen worker 26 - jepsen.util 13706 :info :cas [8 [27 28]] :RequestCanceledException INFO [2019-12-17 12:35:36,137] jepsen worker 26 - jepsen.util 13796 :invoke :read [8 nil] INFO [2019-12-17 12:35:36,145] jepsen worker 61 - jepsen.util 14101 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:35:36,145] jepsen worker 61 - jepsen.util 14101 :invoke :write [20 13] INFO [2019-12-17 12:35:36,146] jepsen worker 0 - jepsen.util 13950 :info :cas [0 [13 12]] :RequestCanceledException INFO [2019-12-17 12:35:36,146] jepsen worker 28 - jepsen.util 7948 :ok :write [9 12] INFO [2019-12-17 12:35:36,146] jepsen worker 28 - jepsen.util 7948 :invoke :read [9 nil] INFO [2019-12-17 12:35:36,148] jepsen worker 0 - jepsen.util 14040 :invoke :write [0 4] INFO [2019-12-17 12:35:36,179] jepsen worker 28 - jepsen.util 7948 :ok :read [9 12] INFO [2019-12-17 12:35:36,180] jepsen worker 28 - jepsen.util 7948 :invoke :write [9 0] INFO [2019-12-17 12:35:36,187] jepsen worker 14 - jepsen.util 9554 :ok :read [4 41] INFO [2019-12-17 12:35:36,187] jepsen worker 14 - jepsen.util 9554 :invoke :cas [4 [3 45]] INFO [2019-12-17 12:35:36,198] jepsen worker 82 - jepsen.util 13852 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:35:36,198] jepsen worker 82 - jepsen.util 13852 :invoke :read [27 nil] INFO [2019-12-17 12:35:36,205] jepsen worker 88 - jepsen.util 9988 :fail :cas [29 [35 38]] :ValueNotSwapFrom INFO [2019-12-17 12:35:36,205] jepsen worker 88 - jepsen.util 9988 :invoke :cas [29 [35 7]] INFO [2019-12-17 12:35:36,215] jepsen worker 68 - jepsen.util 12668 :info :cas [22 [13 33]] :RequestCanceledException INFO [2019-12-17 12:35:36,217] jepsen worker 54 - jepsen.util 10584 :info :write [18 18] :RequestCanceledException INFO [2019-12-17 12:35:36,227] jepsen worker 68 - jepsen.util 12758 :invoke :read [22 nil] INFO [2019-12-17 12:35:36,228] jepsen worker 54 - jepsen.util 10674 :invoke :write [18 0] INFO [2019-12-17 12:35:36,231] jepsen worker 75 - jepsen.util 12675 :info :cas [25 [13 30]] :RequestCanceledException INFO [2019-12-17 12:35:36,231] jepsen worker 63 - jepsen.util 8613 :info :cas [21 [25 48]] :RequestCanceledException INFO [2019-12-17 12:35:36,231] jepsen worker 7 - jepsen.util 13237 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:35:36,231] jepsen worker 7 - jepsen.util 13237 :invoke :cas [2 [0 14]] INFO [2019-12-17 12:35:36,233] jepsen worker 63 - jepsen.util 8703 :invoke :read [21 nil] INFO [2019-12-17 12:35:36,234] jepsen worker 75 - jepsen.util 12765 :invoke :write [25 31] INFO [2019-12-17 12:35:36,243] jepsen worker 53 - jepsen.util 9053 :info :write [17 25] :RequestCanceledException INFO [2019-12-17 12:35:36,248] jepsen worker 53 - jepsen.util 9143 :invoke :write [17 2] INFO [2019-12-17 12:35:36,255] jepsen worker 1 - jepsen.util 13321 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:35:36,255] jepsen worker 1 - jepsen.util 13321 :invoke :cas [0 [7 40]] INFO [2019-12-17 12:35:36,276] jepsen worker 80 - jepsen.util 9350 :fail :cas [26 [30 15]] :ValueNotSwapFrom INFO [2019-12-17 12:35:36,277] jepsen worker 80 - jepsen.util 9350 :invoke :cas [26 [32 43]] INFO [2019-12-17 12:35:36,279] jepsen worker 12 - jepsen.util 10002 :ok :read [4 41] INFO [2019-12-17 12:35:36,279] jepsen worker 12 - jepsen.util 10002 :invoke :read [4 nil] INFO [2019-12-17 12:35:36,279] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55970,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:35:36,279] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55970,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:35:36,280] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55960,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:35:36,280] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55972,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:35:36,280] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55960,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:35:36,280] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55961,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:35:36,280] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55972,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:35:36,281] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55961,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:35:36,281] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55962,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:35:36,281] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55962,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:35:36,288] jepsen worker 28 - jepsen.util 7948 :ok :write [9 0] INFO [2019-12-17 12:35:36,288] jepsen worker 28 - jepsen.util 7948 :invoke :write [9 20] INFO [2019-12-17 12:35:36,295] jepsen worker 63 - jepsen.util 8703 :ok :read [21 7] INFO [2019-12-17 12:35:36,295] jepsen worker 63 - jepsen.util 8703 :invoke :read [21 nil] INFO [2019-12-17 12:35:36,298] jepsen worker 25 - jepsen.util 14335 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:35:36,298] jepsen worker 25 - jepsen.util 14335 :invoke :read [8 nil] INFO [2019-12-17 12:35:36,306] jepsen worker 3 - jepsen.util 9543 :ok :read [1 47] INFO [2019-12-17 12:35:36,306] jepsen worker 3 - jepsen.util 9543 :invoke :read [1 nil] INFO [2019-12-17 12:35:36,328] jepsen worker 63 - jepsen.util 8703 :ok :read [21 7] INFO [2019-12-17 12:35:36,328] jepsen worker 63 - jepsen.util 8703 :invoke :write [21 28] INFO [2019-12-17 12:35:36,330] jepsen worker 58 - jepsen.util 13738 :info :cas [19 [36 48]] :RequestCanceledException INFO [2019-12-17 12:35:36,333] jepsen worker 58 - jepsen.util 13828 :invoke :read [19 nil] INFO [2019-12-17 12:35:36,346] jepsen worker 36 - jepsen.util 10386 :fail :cas [12 [1 10]] :ValueNotSwapFrom INFO [2019-12-17 12:35:36,347] jepsen worker 36 - jepsen.util 10386 :invoke :cas [12 [10 37]] INFO [2019-12-17 12:35:36,362] jepsen worker 74 - jepsen.util 9434 :info :write [24 43] :RequestCanceledException INFO [2019-12-17 12:35:36,364] jepsen worker 74 - jepsen.util 9524 :invoke :cas [24 [26 34]] INFO [2019-12-17 12:35:36,389] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:35:36,389] jepsen nemesis - couchbase.util Rebalance check count: 438 INFO [2019-12-17 12:35:36,390] jepsen nemesis - couchbase.util Rebalance stuck count: 419 INFO [2019-12-17 12:35:36,426] jepsen worker 79 - jepsen.util 9979 :ok :read [26 42] INFO [2019-12-17 12:35:36,426] jepsen worker 79 - jepsen.util 9979 :invoke :cas [26 [3 14]] INFO [2019-12-17 12:35:36,432] jepsen worker 87 - jepsen.util 9267 :info :write [29 8] :RequestCanceledException INFO [2019-12-17 12:35:36,434] jepsen worker 28 - jepsen.util 7948 :ok :write [9 20] INFO [2019-12-17 12:35:36,434] jepsen worker 28 - jepsen.util 7948 :invoke :write [9 40] INFO [2019-12-17 12:35:36,436] jepsen worker 87 - jepsen.util 9357 :invoke :cas [29 [31 38]] INFO [2019-12-17 12:35:36,463] jepsen worker 4 - jepsen.util 9094 :info :write [1 4] :RequestCanceledException INFO [2019-12-17 12:35:36,468] jepsen worker 4 - jepsen.util 9184 :invoke :cas [1 [40 45]] INFO [2019-12-17 12:35:36,471] jepsen worker 69 - jepsen.util 9699 :fail :cas [23 [3 19]] :ValueNotSwapFrom INFO [2019-12-17 12:35:36,471] jepsen worker 69 - jepsen.util 9699 :invoke :cas [23 [7 0]] INFO [2019-12-17 12:35:36,481] jepsen worker 74 - jepsen.util 9524 :fail :cas [24 [26 34]] :ValueNotSwapFrom INFO [2019-12-17 12:35:36,481] jepsen worker 74 - jepsen.util 9524 :invoke :cas [24 [17 26]] INFO [2019-12-17 12:35:36,489] jepsen worker 63 - jepsen.util 8703 :ok :write [21 28] INFO [2019-12-17 12:35:36,490] jepsen worker 63 - jepsen.util 8703 :invoke :write [21 33] INFO [2019-12-17 12:35:36,499] jepsen worker 14 - jepsen.util 9554 :fail :cas [4 [3 45]] :ValueNotSwapFrom INFO [2019-12-17 12:35:36,499] jepsen worker 14 - jepsen.util 9554 :invoke :cas [4 [9 5]] INFO [2019-12-17 12:35:36,500] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55992,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:35:36,501] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55992,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:35:36,501] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55996,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:35:36,501] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":55996,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:35:36,531] jepsen worker 81 - jepsen.util 12951 :info :cas [27 [38 27]] :RequestCanceledException INFO [2019-12-17 12:35:36,532] jepsen worker 11 - jepsen.util 9641 :fail :read [3 nil] :RequestTimeoutException INFO [2019-12-17 12:35:36,532] jepsen worker 11 - jepsen.util 9641 :invoke :cas [3 [23 32]] INFO [2019-12-17 12:35:36,533] jepsen worker 81 - jepsen.util 13041 :invoke :read [27 nil] INFO [2019-12-17 12:35:36,545] jepsen worker 89 - jepsen.util 9629 :info :write [29 45] :RequestCanceledException INFO [2019-12-17 12:35:36,545] jepsen worker 13 - jepsen.util 9103 :fail :cas [4 [23 3]] :ValueNotSwapFrom INFO [2019-12-17 12:35:36,546] jepsen worker 13 - jepsen.util 9103 :invoke :cas [4 [23 30]] INFO [2019-12-17 12:35:36,547] jepsen worker 89 - jepsen.util 9719 :invoke :write [29 3] INFO [2019-12-17 12:35:36,559] jepsen worker 52 - jepsen.util 9592 :info :write [17 13] :RequestCanceledException INFO [2019-12-17 12:35:36,560] jepsen worker 78 - jepsen.util 9618 :info :write [26 35] :RequestCanceledException INFO [2019-12-17 12:35:36,562] jepsen worker 52 - jepsen.util 9682 :invoke :cas [17 [44 19]] INFO [2019-12-17 12:35:36,563] jepsen worker 78 - jepsen.util 9708 :invoke :write [26 9] INFO [2019-12-17 12:35:36,590] jepsen worker 28 - jepsen.util 7948 :ok :write [9 40] INFO [2019-12-17 12:35:36,590] jepsen worker 28 - jepsen.util 7948 :invoke :cas [9 [41 17]] INFO [2019-12-17 12:35:36,593] jepsen worker 11 - jepsen.util 9641 :fail :cas [3 [23 32]] :ValueNotSwapFrom INFO [2019-12-17 12:35:36,593] jepsen worker 11 - jepsen.util 9641 :invoke :cas [3 [7 46]] INFO [2019-12-17 12:35:36,599] jepsen worker 12 - jepsen.util 10002 :ok :read [4 41] INFO [2019-12-17 12:35:36,599] jepsen worker 12 - jepsen.util 10002 :invoke :read [4 nil] INFO [2019-12-17 12:35:36,620] jepsen worker 63 - jepsen.util 8703 :ok :write [21 33] INFO [2019-12-17 12:35:36,621] jepsen worker 63 - jepsen.util 8703 :invoke :read [21 nil] INFO [2019-12-17 12:35:36,631] jepsen worker 48 - jepsen.util 12198 :info :write [16 0] :RequestCanceledException INFO [2019-12-17 12:35:36,639] jepsen worker 48 - jepsen.util 12288 :invoke :read [16 nil] INFO [2019-12-17 12:35:36,661] jepsen worker 28 - jepsen.util 7948 :fail :cas [9 [41 17]] :ValueNotSwapFrom INFO [2019-12-17 12:35:36,661] jepsen worker 28 - jepsen.util 7948 :invoke :cas [9 [9 42]] INFO [2019-12-17 12:35:36,670] jepsen worker 80 - jepsen.util 9350 :fail :cas [26 [32 43]] :ValueNotSwapFrom INFO [2019-12-17 12:35:36,670] jepsen worker 80 - jepsen.util 9350 :invoke :write [26 32] INFO [2019-12-17 12:35:36,672] jepsen worker 74 - jepsen.util 9524 :fail :cas [24 [17 26]] :ValueNotSwapFrom INFO [2019-12-17 12:35:36,672] jepsen worker 74 - jepsen.util 9524 :invoke :cas [24 [5 17]] INFO [2019-12-17 12:35:36,679] jepsen worker 3 - jepsen.util 9543 :ok :read [1 47] INFO [2019-12-17 12:35:36,680] jepsen worker 3 - jepsen.util 9543 :invoke :cas [1 [39 17]] INFO [2019-12-17 12:35:36,681] jepsen worker 11 - jepsen.util 9641 :fail :cas [3 [7 46]] :ValueNotSwapFrom INFO [2019-12-17 12:35:36,682] jepsen worker 11 - jepsen.util 9641 :invoke :cas [3 [45 38]] INFO [2019-12-17 12:35:36,694] jepsen worker 37 - jepsen.util 9127 :info :write [12 10] :RequestCanceledException INFO [2019-12-17 12:35:36,697] jepsen worker 37 - jepsen.util 9217 :invoke :read [12 nil] INFO [2019-12-17 12:35:36,715] jepsen worker 39 - jepsen.util 12729 :info :write [13 20] :RequestCanceledException INFO [2019-12-17 12:35:36,716] jepsen worker 63 - jepsen.util 8703 :ok :read [21 33] INFO [2019-12-17 12:35:36,716] jepsen worker 63 - jepsen.util 8703 :invoke :read [21 nil] INFO [2019-12-17 12:35:36,719] jepsen worker 39 - jepsen.util 12819 :invoke :write [13 14] INFO [2019-12-17 12:35:36,730] jepsen worker 28 - jepsen.util 7948 :fail :cas [9 [9 42]] :ValueNotSwapFrom INFO [2019-12-17 12:35:36,730] jepsen worker 28 - jepsen.util 7948 :invoke :cas [9 [36 26]] INFO [2019-12-17 12:35:36,733] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":56016,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:35:36,733] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":56016,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:35:36,733] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":56019,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:35:36,734] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":56019,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:35:36,734] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":56020,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:35:36,734] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":56020,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:35:36,746] jepsen worker 11 - jepsen.util 9641 :fail :cas [3 [45 38]] :ValueNotSwapFrom INFO [2019-12-17 12:35:36,746] jepsen worker 11 - jepsen.util 9641 :invoke :cas [3 [43 15]] INFO [2019-12-17 12:35:36,754] jepsen worker 74 - jepsen.util 9524 :fail :cas [24 [5 17]] :ValueNotSwapFrom INFO [2019-12-17 12:35:36,754] jepsen worker 74 - jepsen.util 9524 :invoke :write [24 48] INFO [2019-12-17 12:35:36,762] jepsen worker 47 - jepsen.util 10217 :info :write [15 13] :RequestCanceledException INFO [2019-12-17 12:35:36,762] jepsen worker 51 - jepsen.util 8961 :info :write [17 36] :RequestCanceledException INFO [2019-12-17 12:35:36,902] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":56032,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:35:36,903] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":56032,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:35:36,903] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":56035,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:35:36,903] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":56035,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:35:36,903] jepsen worker 33 - jepsen.util 13173 :info :write [11 44] :RequestCanceledException INFO [2019-12-17 12:35:36,905] jepsen worker 33 - jepsen.util 13263 :invoke :cas [11 [39 20]] INFO [2019-12-17 12:35:36,908] jepsen worker 36 - jepsen.util 10386 :fail :cas [12 [10 37]] :ValueNotSwapFrom INFO [2019-12-17 12:35:36,908] jepsen worker 47 - jepsen.util 10307 :invoke :cas [15 [13 8]] INFO [2019-12-17 12:35:36,908] jepsen worker 63 - jepsen.util 8703 :ok :read [21 33] INFO [2019-12-17 12:35:36,908] jepsen worker 51 - jepsen.util 9051 :invoke :cas [17 [22 21]] INFO [2019-12-17 12:35:36,908] jepsen worker 36 - jepsen.util 10386 :invoke :read [12 nil] INFO [2019-12-17 12:35:36,908] jepsen worker 87 - jepsen.util 9357 :fail :cas [29 [31 38]] :ValueNotSwapFrom INFO [2019-12-17 12:35:36,908] jepsen worker 63 - jepsen.util 8703 :invoke :write [21 24] INFO [2019-12-17 12:35:36,908] jepsen worker 87 - jepsen.util 9357 :invoke :write [29 19] INFO [2019-12-17 12:35:36,911] jepsen worker 28 - jepsen.util 7948 :fail :cas [9 [36 26]] :ValueNotSwapFrom INFO [2019-12-17 12:35:36,912] jepsen worker 28 - jepsen.util 7948 :invoke :cas [9 [49 40]] INFO [2019-12-17 12:35:36,913] jepsen worker 55 - jepsen.util 10315 :ok :read [18 25] INFO [2019-12-17 12:35:36,914] jepsen worker 55 - jepsen.util 10315 :invoke :write [18 38] INFO [2019-12-17 12:35:36,917] jepsen worker 11 - jepsen.util 9641 :fail :cas [3 [43 15]] :ValueNotSwapFrom INFO [2019-12-17 12:35:36,918] jepsen worker 11 - jepsen.util 9641 :invoke :write [3 42] INFO [2019-12-17 12:35:36,918] jepsen worker 69 - jepsen.util 9699 :fail :cas [23 [7 0]] :ValueNotSwapFrom INFO [2019-12-17 12:35:36,918] jepsen worker 69 - jepsen.util 9699 :invoke :write [23 13] INFO [2019-12-17 12:35:36,923] jepsen worker 59 - jepsen.util 12569 :info :cas [19 [39 44]] :RequestCanceledException INFO [2019-12-17 12:35:36,924] jepsen worker 59 - jepsen.util 12659 :invoke :read [19 nil] INFO [2019-12-17 12:35:36,928] jepsen worker 5 - jepsen.util 9095 :fail :cas [1 [45 17]] :ValueNotSwapFrom INFO [2019-12-17 12:35:36,928] jepsen worker 5 - jepsen.util 9095 :invoke :cas [1 [46 8]] INFO [2019-12-17 12:35:36,947] jepsen worker 47 - jepsen.util 10307 :fail :cas [15 [13 8]] :ValueNotSwapFrom INFO [2019-12-17 12:35:36,947] jepsen worker 47 - jepsen.util 10307 :invoke :cas [15 [48 3]] INFO [2019-12-17 12:35:36,962] jepsen worker 77 - jepsen.util 13847 :info :cas [25 [17 21]] :RequestCanceledException INFO [2019-12-17 12:35:36,963] jepsen worker 77 - jepsen.util 13937 :invoke :cas [25 [12 47]] INFO [2019-12-17 12:35:36,969] jepsen worker 47 - jepsen.util 10307 :fail :cas [15 [48 3]] :ValueNotSwapFrom INFO [2019-12-17 12:35:36,969] jepsen worker 47 - jepsen.util 10307 :invoke :write [15 2] INFO [2019-12-17 12:35:36,970] jepsen worker 14 - jepsen.util 9554 :fail :cas [4 [9 5]] :ValueNotSwapFrom INFO [2019-12-17 12:35:36,970] jepsen worker 14 - jepsen.util 9554 :invoke :write [4 48] INFO [2019-12-17 12:35:36,990] jepsen worker 13 - jepsen.util 9103 :fail :cas [4 [23 30]] :ValueNotSwapFrom INFO [2019-12-17 12:35:36,990] jepsen worker 13 - jepsen.util 9103 :invoke :cas [4 [47 43]] INFO [2019-12-17 12:35:36,998] jepsen worker 12 - jepsen.util 10002 :ok :read [4 41] INFO [2019-12-17 12:35:36,998] jepsen worker 12 - jepsen.util 10002 :invoke :write [4 25] INFO [2019-12-17 12:35:37,003] jepsen worker 71 - jepsen.util 10421 :ok :read [23 32] INFO [2019-12-17 12:35:37,003] jepsen worker 71 - jepsen.util 10421 :invoke :cas [23 [26 21]] INFO [2019-12-17 12:35:37,012] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":56056,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:35:37,012] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":56056,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:35:37,012] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":56059,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:35:37,012] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":56059,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:35:37,012] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":56060,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:35:37,012] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":56060,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:35:37,031] jepsen worker 50 - jepsen.util 14000 :info :cas [16 [19 17]] :RequestCanceledException INFO [2019-12-17 12:35:37,031] jepsen worker 8 - jepsen.util 12878 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:35:37,032] jepsen worker 8 - jepsen.util 12878 :invoke :write [2 30] INFO [2019-12-17 12:35:37,032] jepsen worker 50 - jepsen.util 14090 :invoke :read [16 nil] INFO [2019-12-17 12:35:37,045] jepsen worker 22 - jepsen.util 13072 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:35:37,045] jepsen worker 67 - jepsen.util 13117 :info :cas [22 [12 20]] :RequestCanceledException INFO [2019-12-17 12:35:37,045] jepsen worker 22 - jepsen.util 13072 :invoke :read [7 nil] INFO [2019-12-17 12:35:37,046] jepsen worker 67 - jepsen.util 13207 :invoke :read [22 nil] INFO [2019-12-17 12:35:37,055] jepsen worker 63 - jepsen.util 8703 :ok :write [21 24] INFO [2019-12-17 12:35:37,055] jepsen worker 63 - jepsen.util 8703 :invoke :cas [21 [9 26]] INFO [2019-12-17 12:35:37,076] jepsen worker 3 - jepsen.util 9543 :fail :cas [1 [39 17]] :ValueNotSwapFrom INFO [2019-12-17 12:35:37,077] jepsen worker 3 - jepsen.util 9543 :invoke :cas [1 [17 19]] INFO [2019-12-17 12:35:37,079] jepsen worker 63 - jepsen.util 8703 :fail :cas [21 [9 26]] :ValueNotSwapFrom INFO [2019-12-17 12:35:37,080] jepsen worker 63 - jepsen.util 8703 :invoke :write [21 19] INFO [2019-12-17 12:35:37,115] jepsen worker 41 - jepsen.util 14261 :info :write [13 47] :RequestCanceledException INFO [2019-12-17 12:35:37,117] jepsen worker 41 - jepsen.util 14351 :invoke :cas [13 [34 44]] INFO [2019-12-17 12:35:37,119] jepsen worker 36 - jepsen.util 10386 :ok :read [12 41] INFO [2019-12-17 12:35:37,119] jepsen worker 36 - jepsen.util 10386 :invoke :read [12 nil] INFO [2019-12-17 12:35:37,129] jepsen worker 2 - jepsen.util 13142 :info :write [0 33] :RequestCanceledException INFO [2019-12-17 12:35:37,129] jepsen worker 84 - jepsen.util 12864 :info :write [28 12] :RequestCanceledException INFO [2019-12-17 12:35:37,129] jepsen worker 16 - jepsen.util 14416 :info :write [5 33] :RequestCanceledException INFO [2019-12-17 12:35:37,130] jepsen worker 2 - jepsen.util 13232 :invoke :read [0 nil] INFO [2019-12-17 12:35:37,130] jepsen worker 16 - jepsen.util 14506 :invoke :read [5 nil] INFO [2019-12-17 12:35:37,130] jepsen worker 84 - jepsen.util 12954 :invoke :cas [28 [15 23]] INFO [2019-12-17 12:35:37,145] jepsen worker 63 - jepsen.util 8703 :ok :write [21 19] INFO [2019-12-17 12:35:37,146] jepsen worker 63 - jepsen.util 8703 :invoke :read [21 nil] INFO [2019-12-17 12:35:37,162] jepsen worker 15 - jepsen.util 12435 :info :cas [5 [19 38]] :RequestCanceledException INFO [2019-12-17 12:35:37,163] jepsen worker 15 - jepsen.util 12525 :invoke :cas [5 [38 39]] INFO [2019-12-17 12:35:37,165] jepsen worker 63 - jepsen.util 8703 :ok :read [21 19] INFO [2019-12-17 12:35:37,165] jepsen worker 63 - jepsen.util 8703 :invoke :write [21 44] INFO [2019-12-17 12:35:37,169] jepsen worker 56 - jepsen.util 10226 :ok :read [18 25] INFO [2019-12-17 12:35:37,169] jepsen worker 56 - jepsen.util 10226 :invoke :write [18 38] INFO [2019-12-17 12:35:37,196] jepsen worker 21 - jepsen.util 12711 :info :cas [7 [33 14]] :RequestCanceledException INFO [2019-12-17 12:35:37,196] jepsen worker 34 - jepsen.util 13174 :info :write [11 32] :RequestCanceledException INFO [2019-12-17 12:35:37,197] jepsen worker 21 - jepsen.util 12801 :invoke :write [7 11] INFO [2019-12-17 12:35:37,197] jepsen worker 34 - jepsen.util 13264 :invoke :cas [11 [31 44]] INFO [2019-12-17 12:35:37,210] jepsen worker 63 - jepsen.util 8703 :ok :write [21 44] INFO [2019-12-17 12:35:37,210] jepsen worker 63 - jepsen.util 8703 :invoke :read [21 nil] INFO [2019-12-17 12:35:37,222] jepsen worker 63 - jepsen.util 8703 :ok :read [21 44] INFO [2019-12-17 12:35:37,222] jepsen worker 63 - jepsen.util 8703 :invoke :read [21 nil] INFO [2019-12-17 12:35:37,231] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":56089,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:35:37,231] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":56089,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:35:37,231] jepsen worker 86 - jepsen.util 14036 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:35:37,231] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":56092,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:35:37,231] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":56092,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:35:37,231] jepsen worker 86 - jepsen.util 14036 :invoke :write [28 26] INFO [2019-12-17 12:35:37,231] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":56081,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:35:37,231] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":56081,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:35:37,232] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":56082,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:35:37,232] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":56082,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:35:37,236] jepsen worker 63 - jepsen.util 8703 :ok :read [21 44] INFO [2019-12-17 12:35:37,236] jepsen worker 63 - jepsen.util 8703 :invoke :read [21 nil] INFO [2019-12-17 12:35:37,247] jepsen worker 63 - jepsen.util 8703 :ok :read [21 44] INFO [2019-12-17 12:35:37,247] jepsen worker 63 - jepsen.util 8703 :invoke :cas [21 [41 34]] INFO [2019-12-17 12:35:37,259] jepsen worker 63 - jepsen.util 8703 :fail :cas [21 [41 34]] :ValueNotSwapFrom INFO [2019-12-17 12:35:37,259] jepsen worker 63 - jepsen.util 8703 :invoke :cas [21 [3 32]] INFO [2019-12-17 12:35:37,262] jepsen worker 43 - jepsen.util 13093 :info :cas [14 [49 29]] :RequestCanceledException INFO [2019-12-17 12:35:37,264] jepsen worker 43 - jepsen.util 13183 :invoke :cas [14 [9 20]] INFO [2019-12-17 12:35:37,277] jepsen worker 63 - jepsen.util 8703 :fail :cas [21 [3 32]] :ValueNotSwapFrom INFO [2019-12-17 12:35:37,277] jepsen worker 63 - jepsen.util 8703 :invoke :write [21 38] INFO [2019-12-17 12:35:37,301] jepsen worker 31 - jepsen.util 8671 :info :write [10 2] :SyncWriteAmbiguous INFO [2019-12-17 12:35:37,302] jepsen worker 31 - jepsen.util 8761 :invoke :read [10 nil] INFO [2019-12-17 12:35:37,321] jepsen worker 88 - jepsen.util 9988 :fail :cas [29 [35 7]] :ValueNotSwapFrom INFO [2019-12-17 12:35:37,321] jepsen worker 88 - jepsen.util 9988 :invoke :read [29 nil] INFO [2019-12-17 12:35:37,321] jepsen worker 10 - jepsen.util 9730 :fail :cas [3 [26 42]] :ValueNotSwapFrom INFO [2019-12-17 12:35:37,321] jepsen worker 10 - jepsen.util 9730 :invoke :write [3 2] INFO [2019-12-17 12:35:37,331] jepsen worker 42 - jepsen.util 14262 :info :write [14 15] :RequestCanceledException INFO [2019-12-17 12:35:37,331] jepsen worker 60 - jepsen.util 14370 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:35:37,331] jepsen worker 60 - jepsen.util 14370 :invoke :read [20 nil] INFO [2019-12-17 12:35:37,332] jepsen worker 42 - jepsen.util 14352 :invoke :read [14 nil] INFO [2019-12-17 12:35:37,334] jepsen worker 30 - jepsen.util 9840 :ok :read [10 24] INFO [2019-12-17 12:35:37,334] jepsen worker 30 - jepsen.util 9840 :invoke :write [10 2] INFO [2019-12-17 12:35:37,359] jepsen worker 73 - jepsen.util 9073 :info :write [24 26] :SyncWriteAmbiguous INFO [2019-12-17 12:35:37,360] jepsen worker 73 - jepsen.util 9163 :invoke :write [24 7] INFO [2019-12-17 12:35:37,360] jepsen worker 17 - jepsen.util 12887 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:35:37,360] jepsen worker 17 - jepsen.util 12887 :invoke :write [5 14] INFO [2019-12-17 12:35:37,362] jepsen worker 52 - jepsen.util 9682 :fail :cas [17 [44 19]] :ValueNotSwapFrom INFO [2019-12-17 12:35:37,363] jepsen worker 52 - jepsen.util 9682 :invoke :write [17 17] INFO [2019-12-17 12:35:37,368] jepsen worker 5 - jepsen.util 9095 :fail :cas [1 [46 8]] :ValueNotSwapFrom INFO [2019-12-17 12:35:37,368] jepsen worker 5 - jepsen.util 9095 :invoke :read [1 nil] INFO [2019-12-17 12:35:37,373] jepsen worker 63 - jepsen.util 8703 :ok :write [21 38] INFO [2019-12-17 12:35:37,373] jepsen worker 63 - jepsen.util 8703 :invoke :read [21 nil] INFO [2019-12-17 12:35:37,413] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:35:37,413] jepsen nemesis - couchbase.util Rebalance check count: 439 INFO [2019-12-17 12:35:37,413] jepsen nemesis - couchbase.util Rebalance stuck count: 420 INFO [2019-12-17 12:35:37,414] jepsen worker 31 - jepsen.util 8761 :ok :read [10 24] INFO [2019-12-17 12:35:37,414] jepsen worker 31 - jepsen.util 8761 :invoke :cas [10 [23 47]] INFO [2019-12-17 12:35:37,414] jepsen worker 46 - jepsen.util 9766 :fail :read [15 nil] :RequestTimeoutException INFO [2019-12-17 12:35:37,414] jepsen worker 46 - jepsen.util 9766 :invoke :read [15 nil] INFO [2019-12-17 12:35:37,419] jepsen worker 63 - jepsen.util 8703 :ok :read [21 38] INFO [2019-12-17 12:35:37,419] jepsen worker 63 - jepsen.util 8703 :invoke :read [21 nil] INFO [2019-12-17 12:35:37,423] jepsen worker 13 - jepsen.util 9103 :fail :cas [4 [47 43]] :ValueNotSwapFrom INFO [2019-12-17 12:35:37,423] jepsen worker 13 - jepsen.util 9103 :invoke :read [4 nil] INFO [2019-12-17 12:35:37,424] jepsen worker 3 - jepsen.util 9543 :fail :cas [1 [17 19]] :ValueNotSwapFrom INFO [2019-12-17 12:35:37,424] jepsen worker 3 - jepsen.util 9543 :invoke :write [1 2] INFO [2019-12-17 12:35:37,430] jepsen worker 27 - jepsen.util 9207 :info :cas [9 [44 13]] :RequestCanceledException INFO [2019-12-17 12:35:37,432] jepsen worker 27 - jepsen.util 9297 :invoke :write [9 33] INFO [2019-12-17 12:35:37,439] jepsen worker 38 - jepsen.util 10208 :info :write [12 48] :RequestCanceledException INFO [2019-12-17 12:35:37,439] jepsen worker 57 - jepsen.util 12837 :info :cas [19 [11 40]] :RequestCanceledException INFO [2019-12-17 12:35:37,440] jepsen worker 38 - jepsen.util 10298 :invoke :write [12 23] INFO [2019-12-17 12:35:37,441] jepsen worker 57 - jepsen.util 12927 :invoke :read [19 nil] INFO [2019-12-17 12:35:37,444] jepsen worker 63 - jepsen.util 8703 :ok :read [21 38] INFO [2019-12-17 12:35:37,444] jepsen worker 63 - jepsen.util 8703 :invoke :write [21 46] INFO [2019-12-17 12:35:37,445] jepsen worker 79 - jepsen.util 9979 :fail :cas [26 [3 14]] :ValueNotSwapFrom INFO [2019-12-17 12:35:37,445] jepsen worker 79 - jepsen.util 9979 :invoke :write [26 38] INFO [2019-12-17 12:35:37,447] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":56120,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:35:37,447] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":56120,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:35:37,447] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":56122,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:35:37,447] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":56122,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:35:37,451] jepsen worker 36 - jepsen.util 10386 :ok :read [12 41] INFO [2019-12-17 12:35:37,451] jepsen worker 36 - jepsen.util 10386 :invoke :read [12 nil] INFO [2019-12-17 12:35:37,474] jepsen worker 4 - jepsen.util 9184 :fail :cas [1 [40 45]] :ValueNotSwapFrom INFO [2019-12-17 12:35:37,474] jepsen worker 4 - jepsen.util 9184 :invoke :cas [1 [10 35]] INFO [2019-12-17 12:35:37,496] jepsen worker 31 - jepsen.util 8761 :fail :cas [10 [23 47]] :ValueNotSwapFrom INFO [2019-12-17 12:35:37,497] jepsen worker 76 - jepsen.util 13486 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:35:37,508] jepsen worker 63 - jepsen.util 8703 :ok :write [21 46] INFO [2019-12-17 12:35:37,512] jepsen worker 37 - jepsen.util 9217 :ok :read [12 41] INFO [2019-12-17 12:35:37,598] jepsen worker 6 - jepsen.util 13686 :info :cas [2 [16 34]] :RequestCanceledException INFO [2019-12-17 12:35:37,612] jepsen worker 51 - jepsen.util 9051 :fail :cas [17 [22 21]] :ValueNotSwapFrom INFO [2019-12-17 12:35:37,616] jepsen worker 9 - jepsen.util 10269 :info :write [3 6] :RequestCanceledException INFO [2019-12-17 12:35:37,631] jepsen worker 44 - jepsen.util 13544 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:35:37,644] jepsen worker 18 - jepsen.util 10008 :info :cas [6 [25 17]] :RequestCanceledException INFO [2019-12-17 12:35:37,644] jepsen worker 65 - jepsen.util 8165 :info :write [21 11] :RequestCanceledException INFO [2019-12-17 12:35:37,644] jepsen worker 19 - jepsen.util 9829 :info :cas [6 [32 12]] :RequestCanceledException INFO [2019-12-17 12:35:37,652] jepsen worker 31 - jepsen.util 8761 :invoke :cas [10 [35 14]] INFO [2019-12-17 12:35:37,652] jepsen worker 76 - jepsen.util 13486 :invoke :cas [25 [5 11]] INFO [2019-12-17 12:35:37,652] jepsen worker 63 - jepsen.util 8703 :invoke :read [21 nil] INFO [2019-12-17 12:35:37,652] jepsen worker 37 - jepsen.util 9217 :invoke :cas [12 [10 42]] INFO [2019-12-17 12:35:37,652] jepsen worker 51 - jepsen.util 9051 :invoke :read [17 nil] INFO [2019-12-17 12:35:37,653] jepsen worker 44 - jepsen.util 13544 :invoke :cas [14 [6 20]] INFO [2019-12-17 12:35:37,654] jepsen worker 9 - jepsen.util 10359 :invoke :write [3 11] INFO [2019-12-17 12:35:37,654] jepsen worker 18 - jepsen.util 10098 :invoke :read [6 nil] INFO [2019-12-17 12:35:37,654] jepsen worker 6 - jepsen.util 13776 :invoke :cas [2 [21 11]] INFO [2019-12-17 12:35:37,654] jepsen worker 65 - jepsen.util 8255 :invoke :write [21 38] INFO [2019-12-17 12:35:37,654] jepsen worker 19 - jepsen.util 9919 :invoke :cas [6 [24 18]] INFO [2019-12-17 12:35:37,662] jepsen worker 24 - jepsen.util 12084 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:35:37,663] jepsen worker 24 - jepsen.util 12084 :invoke :cas [8 [1 21]] INFO [2019-12-17 12:35:37,663] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":56131,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:35:37,664] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":56131,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:35:37,664] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":56134,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:35:37,664] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":56134,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:35:37,681] jepsen worker 31 - jepsen.util 8761 :fail :cas [10 [35 14]] :ValueNotSwapFrom INFO [2019-12-17 12:35:37,681] jepsen worker 31 - jepsen.util 8761 :invoke :read [10 nil] INFO [2019-12-17 12:35:37,692] jepsen worker 71 - jepsen.util 10421 :fail :cas [23 [26 21]] :ValueNotSwapFrom INFO [2019-12-17 12:35:37,692] jepsen worker 71 - jepsen.util 10421 :invoke :cas [23 [47 17]] INFO [2019-12-17 12:35:37,698] jepsen worker 63 - jepsen.util 8703 :ok :read [21 46] INFO [2019-12-17 12:35:37,698] jepsen worker 63 - jepsen.util 8703 :invoke :cas [21 [31 5]] INFO [2019-12-17 12:35:37,721] jepsen worker 18 - jepsen.util 10098 :ok :read [6 1] INFO [2019-12-17 12:35:37,721] jepsen worker 18 - jepsen.util 10098 :invoke :write [6 21] INFO [2019-12-17 12:35:37,728] jepsen worker 63 - jepsen.util 8703 :fail :cas [21 [31 5]] :ValueNotSwapFrom INFO [2019-12-17 12:35:37,728] jepsen worker 63 - jepsen.util 8703 :invoke :write [21 38] INFO [2019-12-17 12:35:37,730] jepsen worker 51 - jepsen.util 9051 :ok :read [17 24] INFO [2019-12-17 12:35:37,730] jepsen worker 51 - jepsen.util 9051 :invoke :cas [17 [45 26]] INFO [2019-12-17 12:35:37,745] jepsen worker 35 - jepsen.util 13175 :info :cas [11 [44 28]] :RequestCanceledException INFO [2019-12-17 12:35:37,745] jepsen worker 72 - jepsen.util 9432 :info :write [24 15] :RequestCanceledException INFO [2019-12-17 12:35:37,746] jepsen worker 72 - jepsen.util 9522 :invoke :write [24 12] INFO [2019-12-17 12:35:37,746] jepsen worker 35 - jepsen.util 13265 :invoke :read [11 nil] INFO [2019-12-17 12:35:37,753] jepsen worker 13 - jepsen.util 9103 :ok :read [4 41] INFO [2019-12-17 12:35:37,753] jepsen worker 13 - jepsen.util 9103 :invoke :write [4 11] INFO [2019-12-17 12:35:37,798] jepsen worker 70 - jepsen.util 10420 :info :write [23 7] :RequestCanceledException INFO [2019-12-17 12:35:37,799] jepsen worker 70 - jepsen.util 10510 :invoke :write [23 2] INFO [2019-12-17 12:35:37,827] jepsen worker 29 - jepsen.util 7949 :info :write [9 21] :RequestCanceledException INFO [2019-12-17 12:35:37,828] jepsen worker 23 - jepsen.util 13253 :info :cas [7 [32 46]] :RequestCanceledException INFO [2019-12-17 12:35:37,828] jepsen worker 29 - jepsen.util 8039 :invoke :write [9 32] INFO [2019-12-17 12:35:37,829] jepsen worker 23 - jepsen.util 13343 :invoke :read [7 nil] INFO [2019-12-17 12:35:37,836] jepsen worker 64 - jepsen.util 7534 :info :cas [21 [44 39]] :RequestCanceledException INFO [2019-12-17 12:35:37,836] jepsen worker 20 - jepsen.util 9200 :info :cas [6 [8 28]] :RequestCanceledException INFO [2019-12-17 12:35:37,836] jepsen worker 62 - jepsen.util 13562 :info :cas [20 [23 0]] :RequestCanceledException INFO [2019-12-17 12:35:37,837] jepsen worker 62 - jepsen.util 13652 :invoke :cas [20 [19 6]] INFO [2019-12-17 12:35:37,837] jepsen worker 20 - jepsen.util 9290 :invoke :read [6 nil] INFO [2019-12-17 12:35:37,840] jepsen worker 64 - jepsen.util 7624 :invoke :cas [21 [33 29]] INFO [2019-12-17 12:35:37,865] jepsen worker 64 - jepsen.util 7624 :fail :cas [21 [33 29]] :ValueNotSwapFrom INFO [2019-12-17 12:35:37,865] jepsen worker 64 - jepsen.util 7624 :invoke :read [21 nil] INFO [2019-12-17 12:35:37,868] jepsen worker 20 - jepsen.util 9290 :ok :read [6 1] INFO [2019-12-17 12:35:37,868] jepsen worker 20 - jepsen.util 9290 :invoke :read [6 nil] INFO [2019-12-17 12:35:37,878] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":56163,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:35:37,878] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":56163,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:35:37,878] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":56169,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:35:37,879] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":56169,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:35:37,894] jepsen worker 85 - jepsen.util 14035 :info :write [28 33] :RequestCanceledException INFO [2019-12-17 12:35:37,896] jepsen worker 85 - jepsen.util 14125 :invoke :read [28 nil] INFO [2019-12-17 12:35:37,896] jepsen worker 20 - jepsen.util 9290 :ok :read [6 1] INFO [2019-12-17 12:35:37,896] jepsen worker 20 - jepsen.util 9290 :invoke :write [6 40] INFO [2019-12-17 12:35:37,905] jepsen worker 64 - jepsen.util 7624 :ok :read [21 46] INFO [2019-12-17 12:35:37,905] jepsen worker 64 - jepsen.util 7624 :invoke :read [21 nil] INFO [2019-12-17 12:35:37,915] jepsen worker 83 - jepsen.util 13673 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:35:37,916] jepsen worker 83 - jepsen.util 13673 :invoke :read [27 nil] INFO [2019-12-17 12:35:37,921] jepsen worker 64 - jepsen.util 7624 :ok :read [21 46] INFO [2019-12-17 12:35:37,921] jepsen worker 64 - jepsen.util 7624 :invoke :cas [21 [46 42]] INFO [2019-12-17 12:35:37,975] jepsen worker 88 - jepsen.util 9988 :ok :read [29 40] INFO [2019-12-17 12:35:37,975] jepsen worker 88 - jepsen.util 9988 :invoke :write [29 1] INFO [2019-12-17 12:35:37,996] jepsen worker 64 - jepsen.util 7624 :ok :cas [21 [46 42]] INFO [2019-12-17 12:35:37,996] jepsen worker 64 - jepsen.util 7624 :invoke :read [21 nil] INFO [2019-12-17 12:35:38,014] jepsen worker 64 - jepsen.util 7624 :ok :read [21 42] INFO [2019-12-17 12:35:38,014] jepsen worker 64 - jepsen.util 7624 :invoke :cas [21 [41 37]] INFO [2019-12-17 12:35:38,043] jepsen worker 64 - jepsen.util 7624 :fail :cas [21 [41 37]] :ValueNotSwapFrom INFO [2019-12-17 12:35:38,043] jepsen worker 64 - jepsen.util 7624 :invoke :read [21 nil] INFO [2019-12-17 12:35:38,061] jepsen worker 64 - jepsen.util 7624 :ok :read [21 42] INFO [2019-12-17 12:35:38,061] jepsen worker 64 - jepsen.util 7624 :invoke :write [21 10] INFO [2019-12-17 12:35:38,062] jepsen worker 40 - jepsen.util 13540 :info :cas [13 [44 15]] :RequestCanceledException INFO [2019-12-17 12:35:38,063] jepsen worker 5 - jepsen.util 9095 :ok :read [1 47] INFO [2019-12-17 12:35:38,063] jepsen worker 5 - jepsen.util 9095 :invoke :read [1 nil] INFO [2019-12-17 12:35:38,063] jepsen worker 40 - jepsen.util 13630 :invoke :cas [13 [42 31]] INFO [2019-12-17 12:35:38,087] jepsen worker 71 - jepsen.util 10421 :fail :cas [23 [47 17]] :ValueNotSwapFrom INFO [2019-12-17 12:35:38,087] jepsen worker 71 - jepsen.util 10421 :invoke :read [23 nil] INFO [2019-12-17 12:35:38,097] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":56180,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:35:38,098] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":56180,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:35:38,098] jepsen worker 51 - jepsen.util 9051 :fail :cas [17 [45 26]] :ValueNotSwapFrom INFO [2019-12-17 12:35:38,098] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":56181,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:35:38,098] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":56181,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:35:38,098] jepsen worker 51 - jepsen.util 9051 :invoke :cas [17 [44 26]] INFO [2019-12-17 12:35:38,112] jepsen worker 4 - jepsen.util 9184 :fail :cas [1 [10 35]] :ValueNotSwapFrom INFO [2019-12-17 12:35:38,112] jepsen worker 4 - jepsen.util 9184 :invoke :cas [1 [45 13]] INFO [2019-12-17 12:35:38,122] jepsen worker 64 - jepsen.util 7624 :ok :write [21 10] INFO [2019-12-17 12:35:38,122] jepsen worker 64 - jepsen.util 7624 :invoke :cas [21 [21 40]] INFO [2019-12-17 12:35:38,131] jepsen worker 45 - jepsen.util 9945 :info :write [15 43] :RequestCanceledException INFO [2019-12-17 12:35:38,132] jepsen worker 45 - jepsen.util 10035 :invoke :read [15 nil] INFO [2019-12-17 12:35:38,145] jepsen worker 49 - jepsen.util 13009 :info :write [16 44] :RequestCanceledException INFO [2019-12-17 12:35:38,146] jepsen worker 49 - jepsen.util 13099 :invoke :write [16 0] INFO [2019-12-17 12:35:38,149] jepsen worker 64 - jepsen.util 7624 :fail :cas [21 [21 40]] :ValueNotSwapFrom INFO [2019-12-17 12:35:38,150] jepsen worker 64 - jepsen.util 7624 :invoke :read [21 nil] INFO [2019-12-17 12:35:38,167] jepsen worker 64 - jepsen.util 7624 :ok :read [21 10] INFO [2019-12-17 12:35:38,167] jepsen worker 64 - jepsen.util 7624 :invoke :cas [21 [34 21]] INFO [2019-12-17 12:35:38,187] jepsen worker 64 - jepsen.util 7624 :fail :cas [21 [34 21]] :ValueNotSwapFrom INFO [2019-12-17 12:35:38,187] jepsen worker 64 - jepsen.util 7624 :invoke :cas [21 [44 18]] INFO [2019-12-17 12:35:38,197] jepsen worker 66 - jepsen.util 12936 :info :write [22 12] :RequestCanceledException INFO [2019-12-17 12:35:38,197] jepsen worker 32 - jepsen.util 9842 :info :write [10 17] :RequestCanceledException INFO [2019-12-17 12:35:38,198] jepsen worker 66 - jepsen.util 13026 :invoke :read [22 nil] INFO [2019-12-17 12:35:38,198] jepsen worker 32 - jepsen.util 9932 :invoke :cas [10 [25 4]] INFO [2019-12-17 12:35:38,202] jepsen worker 64 - jepsen.util 7624 :fail :cas [21 [44 18]] :ValueNotSwapFrom INFO [2019-12-17 12:35:38,202] jepsen worker 64 - jepsen.util 7624 :invoke :write [21 36] INFO [2019-12-17 12:35:38,207] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":56201,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:35:38,207] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":56201,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:35:38,207] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":56202,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:35:38,207] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":56202,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:35:38,207] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":56203,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:35:38,207] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":56203,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:35:38,226] jepsen worker 37 - jepsen.util 9217 :fail :cas [12 [10 42]] :ValueNotSwapFrom INFO [2019-12-17 12:35:38,226] jepsen worker 37 - jepsen.util 9217 :invoke :write [12 15] INFO [2019-12-17 12:35:38,232] jepsen worker 26 - jepsen.util 13796 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:35:38,233] jepsen worker 26 - jepsen.util 13796 :invoke :write [8 39] INFO [2019-12-17 12:35:38,236] jepsen worker 32 - jepsen.util 9932 :fail :cas [10 [25 4]] :ValueNotSwapFrom INFO [2019-12-17 12:35:38,236] jepsen worker 32 - jepsen.util 9932 :invoke :read [10 nil] INFO [2019-12-17 12:35:38,239] jepsen worker 71 - jepsen.util 10421 :ok :read [23 32] INFO [2019-12-17 12:35:38,240] jepsen worker 71 - jepsen.util 10421 :invoke :write [23 11] INFO [2019-12-17 12:35:38,245] jepsen worker 61 - jepsen.util 14101 :info :write [20 13] :RequestCanceledException INFO [2019-12-17 12:35:38,245] jepsen worker 0 - jepsen.util 14040 :info :write [0 4] :RequestCanceledException INFO [2019-12-17 12:35:38,246] jepsen worker 0 - jepsen.util 14130 :invoke :write [0 4] INFO [2019-12-17 12:35:38,246] jepsen worker 61 - jepsen.util 14191 :invoke :read [20 nil] INFO [2019-12-17 12:35:38,259] jepsen worker 68 - jepsen.util 12758 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:35:38,259] jepsen worker 68 - jepsen.util 12758 :invoke :read [22 nil] INFO [2019-12-17 12:35:38,281] jepsen worker 64 - jepsen.util 7624 :ok :write [21 36] INFO [2019-12-17 12:35:38,281] jepsen worker 64 - jepsen.util 7624 :invoke :read [21 nil] INFO [2019-12-17 12:35:38,297] jepsen worker 64 - jepsen.util 7624 :ok :read [21 36] INFO [2019-12-17 12:35:38,297] jepsen worker 64 - jepsen.util 7624 :invoke :write [21 36] INFO [2019-12-17 12:35:38,297] jepsen worker 82 - jepsen.util 13852 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:35:38,297] jepsen worker 82 - jepsen.util 13852 :invoke :read [27 nil] INFO [2019-12-17 12:35:38,315] jepsen worker 75 - jepsen.util 12765 :info :write [25 31] :RequestCanceledException INFO [2019-12-17 12:35:38,316] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":56209,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:35:38,316] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":56209,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:35:38,316] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":56213,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:35:38,317] jepsen worker 75 - jepsen.util 12855 :invoke :cas [25 [38 34]] INFO [2019-12-17 12:35:38,317] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":56213,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:35:38,330] jepsen worker 54 - jepsen.util 10674 :info :write [18 0] :RequestCanceledException INFO [2019-12-17 12:35:38,330] jepsen worker 53 - jepsen.util 9143 :info :write [17 2] :RequestCanceledException INFO [2019-12-17 12:35:38,330] jepsen worker 7 - jepsen.util 13237 :info :cas [2 [0 14]] :RequestCanceledException INFO [2019-12-17 12:35:38,331] jepsen worker 54 - jepsen.util 10764 :invoke :read [18 nil] INFO [2019-12-17 12:35:38,331] jepsen worker 7 - jepsen.util 13327 :invoke :cas [2 [18 5]] INFO [2019-12-17 12:35:38,333] jepsen worker 53 - jepsen.util 9233 :invoke :write [17 4] INFO [2019-12-17 12:35:38,345] jepsen worker 58 - jepsen.util 13828 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:35:38,345] jepsen worker 58 - jepsen.util 13828 :invoke :cas [19 [14 36]] INFO [2019-12-17 12:35:38,352] jepsen worker 36 - jepsen.util 10386 :ok :read [12 41] INFO [2019-12-17 12:35:38,353] jepsen worker 36 - jepsen.util 10386 :invoke :cas [12 [37 39]] INFO [2019-12-17 12:35:38,362] jepsen worker 1 - jepsen.util 13321 :info :cas [0 [7 40]] :RequestCanceledException INFO [2019-12-17 12:35:38,363] jepsen worker 1 - jepsen.util 13411 :invoke :cas [0 [43 16]] INFO [2019-12-17 12:35:38,363] jepsen worker 64 - jepsen.util 7624 :ok :write [21 36] INFO [2019-12-17 12:35:38,364] jepsen worker 64 - jepsen.util 7624 :invoke :write [21 6] INFO [2019-12-17 12:35:38,365] jepsen worker 51 - jepsen.util 9051 :fail :cas [17 [44 26]] :ValueNotSwapFrom INFO [2019-12-17 12:35:38,366] jepsen worker 51 - jepsen.util 9051 :invoke :cas [17 [18 18]] INFO [2019-12-17 12:35:38,369] jepsen worker 18 - jepsen.util 10098 :ok :write [6 21] INFO [2019-12-17 12:35:38,369] jepsen worker 18 - jepsen.util 10098 :invoke :cas [6 [42 29]] INFO [2019-12-17 12:35:38,388] jepsen worker 11 - jepsen.util 9641 :ok :write [3 42] INFO [2019-12-17 12:35:38,388] jepsen worker 11 - jepsen.util 9641 :invoke :read [3 nil] INFO [2019-12-17 12:35:38,394] jepsen worker 47 - jepsen.util 10307 :ok :write [15 2] INFO [2019-12-17 12:35:38,394] jepsen worker 73 - jepsen.util 9163 :ok :write [24 7] INFO [2019-12-17 12:35:38,395] jepsen worker 47 - jepsen.util 10307 :invoke :cas [15 [13 39]] INFO [2019-12-17 12:35:38,395] jepsen worker 73 - jepsen.util 9163 :invoke :cas [24 [40 37]] INFO [2019-12-17 12:35:38,397] jepsen worker 25 - jepsen.util 14335 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:35:38,397] jepsen worker 25 - jepsen.util 14335 :invoke :read [8 nil] INFO [2019-12-17 12:35:38,424] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:35:38,424] jepsen nemesis - couchbase.util Rebalance check count: 440 INFO [2019-12-17 12:35:38,425] jepsen nemesis - couchbase.util Rebalance stuck count: 421 INFO [2019-12-17 12:35:38,434] jepsen worker 51 - jepsen.util 9051 :fail :cas [17 [18 18]] :ValueNotSwapFrom INFO [2019-12-17 12:35:38,434] jepsen worker 51 - jepsen.util 9051 :invoke :write [17 28] INFO [2019-12-17 12:35:38,445] jepsen worker 18 - jepsen.util 10098 :fail :cas [6 [42 29]] :ValueNotSwapFrom INFO [2019-12-17 12:35:38,446] jepsen worker 18 - jepsen.util 10098 :invoke :read [6 nil] INFO [2019-12-17 12:35:38,459] jepsen worker 19 - jepsen.util 9919 :fail :cas [6 [24 18]] :ValueNotSwapFrom INFO [2019-12-17 12:35:38,459] jepsen worker 19 - jepsen.util 9919 :invoke :cas [6 [12 14]] INFO [2019-12-17 12:35:38,469] jepsen worker 18 - jepsen.util 10098 :ok :read [6 21] INFO [2019-12-17 12:35:38,469] jepsen worker 18 - jepsen.util 10098 :invoke :cas [6 [16 1]] INFO [2019-12-17 12:35:38,476] jepsen worker 36 - jepsen.util 10386 :fail :cas [12 [37 39]] :ValueNotSwapFrom INFO [2019-12-17 12:35:38,476] jepsen worker 36 - jepsen.util 10386 :invoke :write [12 3] INFO [2019-12-17 12:35:38,513] jepsen worker 27 - jepsen.util 9297 :ok :write [9 33] INFO [2019-12-17 12:35:38,513] jepsen worker 27 - jepsen.util 9297 :invoke :write [9 31] INFO [2019-12-17 12:35:38,516] jepsen worker 46 - jepsen.util 9766 :ok :read [15 2] INFO [2019-12-17 12:35:38,516] jepsen worker 46 - jepsen.util 9766 :invoke :cas [15 [17 29]] INFO [2019-12-17 12:35:38,516] jepsen worker 47 - jepsen.util 10307 :fail :cas [15 [13 39]] :ValueNotSwapFrom INFO [2019-12-17 12:35:38,516] jepsen worker 47 - jepsen.util 10307 :invoke :cas [15 [28 45]] INFO [2019-12-17 12:35:38,521] jepsen worker 18 - jepsen.util 10098 :fail :cas [6 [16 1]] :ValueNotSwapFrom INFO [2019-12-17 12:35:38,522] jepsen worker 18 - jepsen.util 10098 :invoke :read [6 nil] INFO [2019-12-17 12:35:38,535] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":56239,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:35:38,535] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":56239,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:35:38,535] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":56242,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:35:38,535] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":56242,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:35:38,535] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":56243,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:35:38,535] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":56243,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:35:38,548] jepsen worker 45 - jepsen.util 10035 :ok :read [15 2] INFO [2019-12-17 12:35:38,548] jepsen worker 45 - jepsen.util 10035 :invoke :cas [15 [3 0]] INFO [2019-12-17 12:35:38,610] jepsen worker 18 - jepsen.util 10098 :ok :read [6 21] INFO [2019-12-17 12:35:38,610] jepsen worker 18 - jepsen.util 10098 :invoke :write [6 29] INFO [2019-12-17 12:35:38,614] jepsen worker 11 - jepsen.util 9641 :ok :read [3 42] INFO [2019-12-17 12:35:38,614] jepsen worker 11 - jepsen.util 9641 :invoke :write [3 4] INFO [2019-12-17 12:35:38,615] jepsen worker 81 - jepsen.util 13041 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:35:38,615] jepsen worker 81 - jepsen.util 13041 :invoke :read [27 nil] INFO [2019-12-17 12:35:38,627] jepsen worker 29 - jepsen.util 8039 :ok :write [9 32] INFO [2019-12-17 12:35:38,627] jepsen worker 29 - jepsen.util 8039 :invoke :write [9 25] INFO [2019-12-17 12:35:38,632] jepsen worker 89 - jepsen.util 9719 :info :write [29 3] :RequestCanceledException INFO [2019-12-17 12:35:38,633] jepsen worker 89 - jepsen.util 9809 :invoke :write [29 7] INFO [2019-12-17 12:35:38,662] jepsen worker 78 - jepsen.util 9708 :info :write [26 9] :RequestCanceledException INFO [2019-12-17 12:35:38,663] jepsen worker 78 - jepsen.util 9798 :invoke :read [26 nil] INFO [2019-12-17 12:35:38,667] jepsen worker 19 - jepsen.util 9919 :fail :cas [6 [12 14]] :ValueNotSwapFrom INFO [2019-12-17 12:35:38,667] jepsen worker 19 - jepsen.util 9919 :invoke :read [6 nil] INFO [2019-12-17 12:35:38,680] jepsen worker 73 - jepsen.util 9163 :fail :cas [24 [40 37]] :ValueNotSwapFrom INFO [2019-12-17 12:35:38,680] jepsen worker 73 - jepsen.util 9163 :invoke :write [24 40] INFO [2019-12-17 12:35:38,703] jepsen worker 47 - jepsen.util 10307 :fail :cas [15 [28 45]] :ValueNotSwapFrom INFO [2019-12-17 12:35:38,703] jepsen worker 47 - jepsen.util 10307 :invoke :read [15 nil] INFO [2019-12-17 12:35:38,716] jepsen worker 46 - jepsen.util 9766 :fail :cas [15 [17 29]] :ValueNotSwapFrom INFO [2019-12-17 12:35:38,716] jepsen worker 46 - jepsen.util 9766 :invoke :read [15 nil] INFO [2019-12-17 12:35:38,731] jepsen worker 48 - jepsen.util 12288 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:35:38,731] jepsen worker 80 - jepsen.util 9350 :info :write [26 32] :RequestCanceledException INFO [2019-12-17 12:35:38,731] jepsen worker 48 - jepsen.util 12288 :invoke :read [16 nil] INFO [2019-12-17 12:35:38,732] jepsen worker 80 - jepsen.util 9440 :invoke :write [26 34] INFO [2019-12-17 12:35:38,743] jepsen worker 5 - jepsen.util 9095 :ok :read [1 47] INFO [2019-12-17 12:35:38,743] jepsen worker 5 - jepsen.util 9095 :invoke :write [1 39] INFO [2019-12-17 12:35:38,754] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":56259,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:35:38,754] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":56259,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:35:38,754] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":56264,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:35:38,754] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":56264,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:35:38,762] jepsen worker 19 - jepsen.util 9919 :ok :read [6 21] INFO [2019-12-17 12:35:38,762] jepsen worker 19 - jepsen.util 9919 :invoke :cas [6 [30 29]] INFO [2019-12-17 12:35:38,776] jepsen worker 47 - jepsen.util 10307 :ok :read [15 2] INFO [2019-12-17 12:35:38,776] jepsen worker 47 - jepsen.util 10307 :invoke :cas [15 [48 30]] INFO [2019-12-17 12:35:38,776] jepsen worker 64 - jepsen.util 7624 :ok :write [21 6] INFO [2019-12-17 12:35:38,776] jepsen worker 64 - jepsen.util 7624 :invoke :read [21 nil] INFO [2019-12-17 12:35:38,801] jepsen worker 19 - jepsen.util 9919 :fail :cas [6 [30 29]] :ValueNotSwapFrom INFO [2019-12-17 12:35:38,802] jepsen worker 19 - jepsen.util 9919 :invoke :read [6 nil] INFO [2019-12-17 12:35:38,815] jepsen worker 39 - jepsen.util 12819 :info :write [13 14] :RequestCanceledException INFO [2019-12-17 12:35:38,816] jepsen worker 39 - jepsen.util 12909 :invoke :write [13 49] INFO [2019-12-17 12:35:38,820] jepsen worker 27 - jepsen.util 9297 :ok :write [9 31] INFO [2019-12-17 12:35:38,820] jepsen worker 27 - jepsen.util 9297 :invoke :write [9 38] INFO [2019-12-17 12:35:38,824] jepsen worker 64 - jepsen.util 7624 :ok :read [21 6] INFO [2019-12-17 12:35:38,824] jepsen worker 64 - jepsen.util 7624 :invoke :read [21 nil] INFO [2019-12-17 12:35:38,847] jepsen worker 54 - jepsen.util 10764 :ok :read [18 25] INFO [2019-12-17 12:35:38,847] jepsen worker 54 - jepsen.util 10764 :invoke :write [18 32] INFO [2019-12-17 12:35:38,855] jepsen worker 47 - jepsen.util 10307 :fail :cas [15 [48 30]] :ValueNotSwapFrom INFO [2019-12-17 12:35:38,855] jepsen worker 47 - jepsen.util 10307 :invoke :cas [15 [17 38]] INFO [2019-12-17 12:35:38,894] jepsen worker 64 - jepsen.util 7624 :ok :read [21 6] INFO [2019-12-17 12:35:38,894] jepsen worker 64 - jepsen.util 7624 :invoke :write [21 42] INFO [2019-12-17 12:35:38,907] jepsen worker 19 - jepsen.util 9919 :ok :read [6 21] INFO [2019-12-17 12:35:38,907] jepsen worker 19 - jepsen.util 9919 :invoke :write [6 40] INFO [2019-12-17 12:35:38,931] jepsen worker 87 - jepsen.util 9357 :info :write [29 19] :RequestCanceledException INFO [2019-12-17 12:35:38,931] jepsen worker 33 - jepsen.util 13263 :info :cas [11 [39 20]] :RequestCanceledException INFO [2019-12-17 12:35:38,931] jepsen worker 55 - jepsen.util 10315 :info :write [18 38] :RequestCanceledException INFO [2019-12-17 12:35:38,932] jepsen worker 33 - jepsen.util 13353 :invoke :read [11 nil] INFO [2019-12-17 12:35:38,932] jepsen worker 55 - jepsen.util 10405 :invoke :write [18 34] INFO [2019-12-17 12:35:38,932] jepsen worker 87 - jepsen.util 9447 :invoke :write [29 41] INFO [2019-12-17 12:35:38,951] jepsen worker 47 - jepsen.util 10307 :fail :cas [15 [17 38]] :ValueNotSwapFrom INFO [2019-12-17 12:35:38,952] jepsen worker 47 - jepsen.util 10307 :invoke :cas [15 [29 48]] INFO [2019-12-17 12:35:38,962] jepsen worker 28 - jepsen.util 7948 :info :cas [9 [49 40]] :RequestCanceledException INFO [2019-12-17 12:35:38,962] jepsen worker 74 - jepsen.util 9524 :info :write [24 48] :RequestCanceledException INFO [2019-12-17 12:35:38,963] jepsen worker 28 - jepsen.util 8038 :invoke :write [9 3] INFO [2019-12-17 12:35:38,963] jepsen worker 74 - jepsen.util 9614 :invoke :cas [24 [13 39]] INFO [2019-12-17 12:35:38,973] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":56286,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:35:38,973] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":56286,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:35:38,996] jepsen worker 47 - jepsen.util 10307 :fail :cas [15 [29 48]] :ValueNotSwapFrom INFO [2019-12-17 12:35:38,996] jepsen worker 47 - jepsen.util 10307 :invoke :write [15 18] INFO [2019-12-17 12:35:38,997] jepsen worker 77 - jepsen.util 13937 :info :cas [25 [12 47]] :RequestCanceledException INFO [2019-12-17 12:35:38,999] jepsen worker 77 - jepsen.util 14027 :invoke :cas [25 [7 28]] INFO [2019-12-17 12:35:39,005] jepsen worker 74 - jepsen.util 9614 :fail :cas [24 [13 39]] :ValueNotSwapFrom INFO [2019-12-17 12:35:39,005] jepsen worker 74 - jepsen.util 9614 :invoke :write [24 47] INFO [2019-12-17 12:35:39,015] jepsen worker 14 - jepsen.util 9554 :info :write [4 48] :RequestCanceledException INFO [2019-12-17 12:35:39,015] jepsen worker 69 - jepsen.util 9699 :info :write [23 13] :RequestCanceledException INFO [2019-12-17 12:35:39,015] jepsen worker 12 - jepsen.util 10002 :info :write [4 25] :RequestCanceledException INFO [2019-12-17 12:35:39,016] jepsen worker 14 - jepsen.util 9644 :invoke :read [4 nil] INFO [2019-12-17 12:35:39,016] jepsen worker 12 - jepsen.util 10092 :invoke :cas [4 [34 13]] INFO [2019-12-17 12:35:39,017] jepsen worker 69 - jepsen.util 9789 :invoke :read [23 nil] INFO [2019-12-17 12:35:39,030] jepsen worker 59 - jepsen.util 12659 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:35:39,030] jepsen worker 59 - jepsen.util 12659 :invoke :read [19 nil] INFO [2019-12-17 12:35:39,061] jepsen worker 64 - jepsen.util 7624 :ok :write [21 42] INFO [2019-12-17 12:35:39,061] jepsen worker 64 - jepsen.util 7624 :invoke :write [21 28] INFO [2019-12-17 12:35:39,082] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":56296,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:35:39,082] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":56296,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:35:39,082] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":56297,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:35:39,082] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":56297,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:35:39,082] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":56298,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:35:39,082] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":56298,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:35:39,115] jepsen worker 50 - jepsen.util 14090 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:35:39,116] jepsen worker 50 - jepsen.util 14090 :invoke :cas [16 [32 34]] INFO [2019-12-17 12:35:39,116] jepsen worker 4 - jepsen.util 9184 :fail :cas [1 [45 13]] :ValueNotSwapFrom INFO [2019-12-17 12:35:39,116] jepsen worker 4 - jepsen.util 9184 :invoke :cas [1 [7 43]] INFO [2019-12-17 12:35:39,131] jepsen worker 8 - jepsen.util 12878 :info :write [2 30] :RequestCanceledException INFO [2019-12-17 12:35:39,131] jepsen worker 67 - jepsen.util 13207 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:35:39,131] jepsen worker 67 - jepsen.util 13207 :invoke :write [22 9] INFO [2019-12-17 12:35:39,132] jepsen worker 8 - jepsen.util 12968 :invoke :read [2 nil] INFO [2019-12-17 12:35:39,133] jepsen worker 64 - jepsen.util 7624 :ok :write [21 28] INFO [2019-12-17 12:35:39,133] jepsen worker 64 - jepsen.util 7624 :invoke :write [21 20] INFO [2019-12-17 12:35:39,145] jepsen worker 22 - jepsen.util 13072 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:35:39,145] jepsen worker 41 - jepsen.util 14351 :info :cas [13 [34 44]] :RequestCanceledException INFO [2019-12-17 12:35:39,145] jepsen worker 22 - jepsen.util 13072 :invoke :read [7 nil] INFO [2019-12-17 12:35:39,146] jepsen worker 41 - jepsen.util 14441 :invoke :read [13 nil] INFO [2019-12-17 12:35:39,162] jepsen worker 2 - jepsen.util 13232 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:35:39,162] jepsen worker 2 - jepsen.util 13232 :invoke :read [0 nil] INFO [2019-12-17 12:35:39,189] jepsen worker 78 - jepsen.util 9798 :ok :read [26 42] INFO [2019-12-17 12:35:39,189] jepsen worker 78 - jepsen.util 9798 :invoke :cas [26 [44 32]] INFO [2019-12-17 12:35:39,193] jepsen worker 64 - jepsen.util 7624 :ok :write [21 20] INFO [2019-12-17 12:35:39,194] jepsen worker 64 - jepsen.util 7624 :invoke :cas [21 [37 48]] INFO [2019-12-17 12:35:39,198] jepsen worker 84 - jepsen.util 12954 :info :cas [28 [15 23]] :RequestCanceledException INFO [2019-12-17 12:35:39,198] jepsen worker 56 - jepsen.util 10226 :info :write [18 38] :RequestCanceledException INFO [2019-12-17 12:35:39,199] jepsen worker 56 - jepsen.util 10316 :invoke :write [18 46] INFO [2019-12-17 12:35:39,200] jepsen worker 84 - jepsen.util 13044 :invoke :read [28 nil] INFO [2019-12-17 12:35:39,207] jepsen worker 64 - jepsen.util 7624 :fail :cas [21 [37 48]] :ValueNotSwapFrom INFO [2019-12-17 12:35:39,208] jepsen worker 64 - jepsen.util 7624 :invoke :write [21 27] INFO [2019-12-17 12:35:39,216] jepsen worker 15 - jepsen.util 12525 :info :cas [5 [38 39]] :RequestCanceledException INFO [2019-12-17 12:35:39,217] jepsen worker 15 - jepsen.util 12615 :invoke :read [5 nil] INFO [2019-12-17 12:35:39,230] jepsen worker 16 - jepsen.util 14506 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:35:39,230] jepsen worker 21 - jepsen.util 12801 :info :write [7 11] :RequestCanceledException INFO [2019-12-17 12:35:39,230] jepsen worker 16 - jepsen.util 14506 :invoke :write [5 0] INFO [2019-12-17 12:35:39,231] jepsen worker 21 - jepsen.util 12891 :invoke :cas [7 [1 0]] INFO [2019-12-17 12:35:39,245] jepsen worker 34 - jepsen.util 13264 :info :cas [11 [31 44]] :RequestCanceledException INFO [2019-12-17 12:35:39,246] jepsen worker 34 - jepsen.util 13354 :invoke :write [11 30] INFO [2019-12-17 12:35:39,258] jepsen worker 64 - jepsen.util 7624 :ok :write [21 27] INFO [2019-12-17 12:35:39,259] jepsen worker 64 - jepsen.util 7624 :invoke :write [21 36] INFO [2019-12-17 12:35:39,301] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":56321,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:35:39,301] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":56332,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:35:39,301] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":56321,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:35:39,301] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":56332,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:35:39,301] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":56335,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:35:39,301] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":56335,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:35:39,318] jepsen worker 64 - jepsen.util 7624 :ok :write [21 36] INFO [2019-12-17 12:35:39,318] jepsen worker 64 - jepsen.util 7624 :invoke :read [21 nil] INFO [2019-12-17 12:35:39,331] jepsen worker 86 - jepsen.util 14036 :info :write [28 26] :RequestCanceledException INFO [2019-12-17 12:35:39,332] jepsen worker 86 - jepsen.util 14126 :invoke :cas [28 [44 23]] INFO [2019-12-17 12:35:39,348] jepsen worker 64 - jepsen.util 7624 :ok :read [21 36] INFO [2019-12-17 12:35:39,348] jepsen worker 64 - jepsen.util 7624 :invoke :read [21 nil] INFO [2019-12-17 12:35:39,348] jepsen worker 14 - jepsen.util 9644 :ok :read [4 41] INFO [2019-12-17 12:35:39,349] jepsen worker 14 - jepsen.util 9644 :invoke :write [4 22] INFO [2019-12-17 12:35:39,358] jepsen worker 43 - jepsen.util 13183 :info :cas [14 [9 20]] :RequestCanceledException INFO [2019-12-17 12:35:39,359] jepsen worker 43 - jepsen.util 13273 :invoke :read [14 nil] INFO [2019-12-17 12:35:39,364] jepsen worker 64 - jepsen.util 7624 :ok :read [21 36] INFO [2019-12-17 12:35:39,364] jepsen worker 64 - jepsen.util 7624 :invoke :read [21 nil] INFO [2019-12-17 12:35:39,381] jepsen worker 64 - jepsen.util 7624 :ok :read [21 36] INFO [2019-12-17 12:35:39,381] jepsen worker 64 - jepsen.util 7624 :invoke :read [21 nil] INFO [2019-12-17 12:35:39,390] jepsen worker 78 - jepsen.util 9798 :fail :cas [26 [44 32]] :ValueNotSwapFrom INFO [2019-12-17 12:35:39,390] jepsen worker 78 - jepsen.util 9798 :invoke :read [26 nil] INFO [2019-12-17 12:35:39,395] jepsen worker 64 - jepsen.util 7624 :ok :read [21 36] INFO [2019-12-17 12:35:39,395] jepsen worker 64 - jepsen.util 7624 :invoke :read [21 nil] INFO [2019-12-17 12:35:39,396] jepsen worker 42 - jepsen.util 14352 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:35:39,396] jepsen worker 42 - jepsen.util 14352 :invoke :read [14 nil] INFO [2019-12-17 12:35:39,406] jepsen worker 64 - jepsen.util 7624 :ok :read [21 36] INFO [2019-12-17 12:35:39,407] jepsen worker 64 - jepsen.util 7624 :invoke :read [21 nil] INFO [2019-12-17 12:35:39,419] jepsen worker 64 - jepsen.util 7624 :ok :read [21 36] INFO [2019-12-17 12:35:39,419] jepsen worker 64 - jepsen.util 7624 :invoke :cas [21 [48 15]] INFO [2019-12-17 12:35:39,431] jepsen worker 10 - jepsen.util 9730 :info :write [3 2] :RequestCanceledException INFO [2019-12-17 12:35:39,431] jepsen worker 30 - jepsen.util 9840 :info :write [10 2] :RequestCanceledException INFO [2019-12-17 12:35:39,431] jepsen worker 60 - jepsen.util 14370 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:35:39,431] jepsen worker 64 - jepsen.util 7624 :fail :cas [21 [48 15]] :ValueNotSwapFrom INFO [2019-12-17 12:35:39,432] jepsen worker 60 - jepsen.util 14370 :invoke :read [20 nil] INFO [2019-12-17 12:35:39,432] jepsen worker 64 - jepsen.util 7624 :invoke :cas [21 [12 48]] INFO [2019-12-17 12:35:39,432] jepsen worker 10 - jepsen.util 9820 :invoke :write [3 18] INFO [2019-12-17 12:35:39,434] jepsen worker 30 - jepsen.util 9930 :invoke :write [10 3] INFO [2019-12-17 12:35:39,442] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:35:39,443] jepsen nemesis - couchbase.util Rebalance check count: 441 INFO [2019-12-17 12:35:39,443] jepsen nemesis - couchbase.util Rebalance stuck count: 422 INFO [2019-12-17 12:35:39,443] jepsen worker 52 - jepsen.util 9682 :info :write [17 17] :RequestCanceledException INFO [2019-12-17 12:35:39,445] jepsen worker 52 - jepsen.util 9772 :invoke :cas [17 [24 15]] INFO [2019-12-17 12:35:39,448] jepsen worker 64 - jepsen.util 7624 :fail :cas [21 [12 48]] :ValueNotSwapFrom INFO [2019-12-17 12:35:39,448] jepsen worker 64 - jepsen.util 7624 :invoke :cas [21 [17 3]] INFO [2019-12-17 12:35:39,462] jepsen worker 17 - jepsen.util 12887 :info :write [5 14] :RequestCanceledException INFO [2019-12-17 12:35:39,462] jepsen worker 57 - jepsen.util 12927 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:35:39,462] jepsen worker 57 - jepsen.util 12927 :invoke :read [19 nil] INFO [2019-12-17 12:35:39,463] jepsen worker 17 - jepsen.util 12977 :invoke :write [5 30] INFO [2019-12-17 12:35:39,468] jepsen worker 64 - jepsen.util 7624 :fail :cas [21 [17 3]] :ValueNotSwapFrom INFO [2019-12-17 12:35:39,469] jepsen worker 64 - jepsen.util 7624 :invoke :read [21 nil] INFO [2019-12-17 12:35:39,482] jepsen worker 64 - jepsen.util 7624 :ok :read [21 36] INFO [2019-12-17 12:35:39,482] jepsen worker 64 - jepsen.util 7624 :invoke :write [21 34] INFO [2019-12-17 12:35:39,497] jepsen worker 79 - jepsen.util 9979 :info :write [26 38] :RequestCanceledException INFO [2019-12-17 12:35:39,499] jepsen worker 79 - jepsen.util 10069 :invoke :read [26 nil] INFO [2019-12-17 12:35:39,516] jepsen worker 3 - jepsen.util 9543 :info :write [1 2] :RequestCanceledException INFO [2019-12-17 12:35:39,517] jepsen worker 3 - jepsen.util 9633 :invoke :write [1 12] INFO [2019-12-17 12:35:39,521] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":56354,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:35:39,521] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":56354,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:35:39,521] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":56357,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:35:39,521] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":56357,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:35:39,525] jepsen worker 64 - jepsen.util 7624 :ok :write [21 34] INFO [2019-12-17 12:35:39,525] jepsen worker 64 - jepsen.util 7624 :invoke :write [21 13] INFO [2019-12-17 12:35:39,545] jepsen worker 38 - jepsen.util 10298 :info :write [12 23] :RequestCanceledException INFO [2019-12-17 12:35:39,546] jepsen worker 38 - jepsen.util 10388 :invoke :cas [12 [10 14]] INFO [2019-12-17 12:35:39,575] jepsen worker 64 - jepsen.util 7624 :ok :write [21 13] INFO [2019-12-17 12:35:39,575] jepsen worker 64 - jepsen.util 7624 :invoke :read [21 nil] INFO [2019-12-17 12:35:39,592] jepsen worker 64 - jepsen.util 7624 :ok :read [21 13] INFO [2019-12-17 12:35:39,592] jepsen worker 64 - jepsen.util 7624 :invoke :read [21 nil] INFO [2019-12-17 12:35:39,606] jepsen worker 64 - jepsen.util 7624 :ok :read [21 13] INFO [2019-12-17 12:35:39,606] jepsen worker 64 - jepsen.util 7624 :invoke :cas [21 [25 18]] INFO [2019-12-17 12:35:39,608] jepsen worker 78 - jepsen.util 9798 :ok :read [26 42] INFO [2019-12-17 12:35:39,608] jepsen worker 78 - jepsen.util 9798 :invoke :cas [26 [47 30]] INFO [2019-12-17 12:35:39,619] jepsen worker 64 - jepsen.util 7624 :fail :cas [21 [25 18]] :ValueNotSwapFrom INFO [2019-12-17 12:35:39,619] jepsen worker 64 - jepsen.util 7624 :invoke :read [21 nil] INFO [2019-12-17 12:35:39,638] jepsen worker 64 - jepsen.util 7624 :ok :read [21 13] INFO [2019-12-17 12:35:39,639] jepsen worker 64 - jepsen.util 7624 :invoke :write [21 28] INFO [2019-12-17 12:35:39,645] jepsen worker 69 - jepsen.util 9789 :ok :read [23 32] INFO [2019-12-17 12:35:39,645] jepsen worker 69 - jepsen.util 9789 :invoke :write [23 29] INFO [2019-12-17 12:35:39,673] jepsen worker 78 - jepsen.util 9798 :fail :cas [26 [47 30]] :ValueNotSwapFrom INFO [2019-12-17 12:35:39,673] jepsen worker 78 - jepsen.util 9798 :invoke :cas [26 [40 16]] INFO [2019-12-17 12:35:39,690] jepsen worker 64 - jepsen.util 7624 :ok :write [21 28] INFO [2019-12-17 12:35:39,690] jepsen worker 64 - jepsen.util 7624 :invoke :read [21 nil] INFO [2019-12-17 12:35:39,693] jepsen worker 76 - jepsen.util 13486 :info :cas [25 [5 11]] :RequestCanceledException INFO [2019-12-17 12:35:39,694] jepsen worker 76 - jepsen.util 13576 :invoke :read [25 nil] INFO [2019-12-17 12:35:39,705] jepsen worker 64 - jepsen.util 7624 :ok :read [21 28] INFO [2019-12-17 12:35:39,705] jepsen worker 64 - jepsen.util 7624 :invoke :cas [21 [31 10]] INFO [2019-12-17 12:35:39,713] jepsen worker 6 - jepsen.util 13776 :info :cas [2 [21 11]] :RequestCanceledException INFO [2019-12-17 12:35:39,714] jepsen worker 6 - jepsen.util 13866 :invoke :cas [2 [36 29]] INFO [2019-12-17 12:35:39,720] jepsen worker 64 - jepsen.util 7624 :fail :cas [21 [31 10]] :ValueNotSwapFrom INFO [2019-12-17 12:35:39,720] jepsen worker 64 - jepsen.util 7624 :invoke :read [21 nil] INFO [2019-12-17 12:35:39,729] jepsen worker 64 - jepsen.util 7624 :ok :read [21 28] INFO [2019-12-17 12:35:39,729] jepsen worker 64 - jepsen.util 7624 :invoke :write [21 42] INFO [2019-12-17 12:35:39,731] jepsen worker 31 - jepsen.util 8761 :fail :read [10 nil] :RequestTimeoutException INFO [2019-12-17 12:35:39,731] jepsen worker 9 - jepsen.util 10359 :info :write [3 11] :RequestCanceledException INFO [2019-12-17 12:35:39,731] jepsen worker 44 - jepsen.util 13544 :info :cas [14 [6 20]] :RequestCanceledException INFO [2019-12-17 12:35:39,731] jepsen worker 65 - jepsen.util 8255 :info :write [21 38] :RequestCanceledException INFO [2019-12-17 12:35:39,731] jepsen worker 31 - jepsen.util 8761 :invoke :write [10 25] INFO [2019-12-17 12:35:39,732] jepsen worker 44 - jepsen.util 13634 :invoke :read [14 nil] INFO [2019-12-17 12:35:39,732] jepsen worker 65 - jepsen.util 8345 :invoke :write [21 32] INFO [2019-12-17 12:35:39,732] jepsen worker 9 - jepsen.util 10449 :invoke :cas [3 [14 23]] INFO [2019-12-17 12:35:39,735] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":56381,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:35:39,736] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":56381,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:35:39,736] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":56383,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:35:39,736] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":56383,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:35:39,754] jepsen worker 9 - jepsen.util 10449 :fail :cas [3 [14 23]] :ValueNotSwapFrom INFO [2019-12-17 12:35:39,754] jepsen worker 9 - jepsen.util 10449 :invoke :write [3 28] INFO [2019-12-17 12:35:39,758] jepsen worker 24 - jepsen.util 12084 :info :cas [8 [1 21]] :RequestCanceledException INFO [2019-12-17 12:35:39,758] jepsen worker 72 - jepsen.util 9522 :info :write [24 12] :RequestCanceledException INFO [2019-12-17 12:35:39,759] jepsen worker 72 - jepsen.util 9612 :invoke :write [24 32] INFO [2019-12-17 12:35:39,760] jepsen worker 24 - jepsen.util 12174 :invoke :write [8 34] INFO [2019-12-17 12:35:39,797] jepsen worker 63 - jepsen.util 8703 :info :write [21 38] :RequestCanceledException INFO [2019-12-17 12:35:39,798] jepsen worker 63 - jepsen.util 8793 :invoke :write [21 8] INFO [2019-12-17 12:35:39,819] jepsen worker 64 - jepsen.util 7624 :ok :write [21 42] INFO [2019-12-17 12:35:39,819] jepsen worker 64 - jepsen.util 7624 :invoke :read [21 nil] INFO [2019-12-17 12:35:39,831] jepsen worker 35 - jepsen.util 13265 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:35:39,831] jepsen worker 35 - jepsen.util 13265 :invoke :read [11 nil] INFO [2019-12-17 12:35:39,832] jepsen worker 13 - jepsen.util 9103 :info :write [4 11] :RequestCanceledException INFO [2019-12-17 12:35:39,833] jepsen worker 13 - jepsen.util 9193 :invoke :read [4 nil] INFO [2019-12-17 12:35:39,889] jepsen worker 78 - jepsen.util 9798 :fail :cas [26 [40 16]] :ValueNotSwapFrom INFO [2019-12-17 12:35:39,889] jepsen worker 78 - jepsen.util 9798 :invoke :cas [26 [20 29]] INFO [2019-12-17 12:35:39,898] jepsen worker 70 - jepsen.util 10510 :info :write [23 2] :RequestCanceledException INFO [2019-12-17 12:35:39,899] jepsen worker 70 - jepsen.util 10600 :invoke :read [23 nil] INFO [2019-12-17 12:35:39,931] jepsen worker 23 - jepsen.util 13343 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:35:39,931] jepsen worker 23 - jepsen.util 13343 :invoke :write [7 6] INFO [2019-12-17 12:35:39,946] jepsen worker 62 - jepsen.util 13652 :info :cas [20 [19 6]] :RequestCanceledException INFO [2019-12-17 12:35:39,947] jepsen worker 62 - jepsen.util 13742 :invoke :cas [20 [28 48]] INFO [2019-12-17 12:35:39,958] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":56405,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:35:39,958] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":56405,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:35:39,958] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":56408,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:35:39,958] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":56408,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:35:39,958] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":56409,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:35:39,958] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":56409,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:35:39,962] jepsen worker 20 - jepsen.util 9290 :info :write [6 40] :RequestCanceledException INFO [2019-12-17 12:35:39,963] jepsen worker 20 - jepsen.util 9380 :invoke :write [6 8] INFO [2019-12-17 12:35:39,974] jepsen worker 13 - jepsen.util 9193 :ok :read [4 41] INFO [2019-12-17 12:35:39,974] jepsen worker 13 - jepsen.util 9193 :invoke :cas [4 [45 5]] INFO [2019-12-17 12:35:39,980] jepsen worker 38 - jepsen.util 10388 :fail :cas [12 [10 14]] :ValueNotSwapFrom INFO [2019-12-17 12:35:39,980] jepsen worker 38 - jepsen.util 10388 :invoke :read [12 nil] INFO [2019-12-17 12:35:39,994] jepsen worker 85 - jepsen.util 14125 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:35:39,994] jepsen worker 88 - jepsen.util 9988 :info :write [29 1] :RequestCanceledException INFO [2019-12-17 12:35:39,994] jepsen worker 85 - jepsen.util 14125 :invoke :cas [28 [0 28]] INFO [2019-12-17 12:35:39,995] jepsen worker 88 - jepsen.util 10078 :invoke :write [29 38] INFO [2019-12-17 12:35:40,012] jepsen worker 38 - jepsen.util 10388 :ok :read [12 41] INFO [2019-12-17 12:35:40,012] jepsen worker 38 - jepsen.util 10388 :invoke :write [12 29] INFO [2019-12-17 12:35:40,015] jepsen worker 83 - jepsen.util 13673 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:35:40,016] jepsen worker 83 - jepsen.util 13673 :invoke :write [27 40] INFO [2019-12-17 12:35:40,037] jepsen worker 78 - jepsen.util 9798 :fail :cas [26 [20 29]] :ValueNotSwapFrom INFO [2019-12-17 12:35:40,037] jepsen worker 78 - jepsen.util 9798 :invoke :read [26 nil] INFO [2019-12-17 12:35:40,065] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":56418,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:35:40,065] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":56418,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:35:40,066] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":56419,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:35:40,066] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":56419,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:35:40,071] jepsen worker 12 - jepsen.util 10092 :fail :cas [4 [34 13]] :ValueNotSwapFrom INFO [2019-12-17 12:35:40,071] jepsen worker 12 - jepsen.util 10092 :invoke :cas [4 [42 20]] INFO [2019-12-17 12:35:40,092] jepsen worker 13 - jepsen.util 9193 :fail :cas [4 [45 5]] :ValueNotSwapFrom INFO [2019-12-17 12:35:40,092] jepsen worker 13 - jepsen.util 9193 :invoke :cas [4 [31 5]] INFO [2019-12-17 12:35:40,113] jepsen worker 11 - jepsen.util 9641 :ok :write [3 4] INFO [2019-12-17 12:35:40,113] jepsen worker 11 - jepsen.util 9641 :invoke :cas [3 [43 8]] INFO [2019-12-17 12:35:40,114] jepsen worker 73 - jepsen.util 9163 :ok :write [24 40] INFO [2019-12-17 12:35:40,114] jepsen worker 73 - jepsen.util 9163 :invoke :write [24 17] INFO [2019-12-17 12:35:40,116] jepsen worker 79 - jepsen.util 10069 :ok :read [26 42] INFO [2019-12-17 12:35:40,116] jepsen worker 79 - jepsen.util 10069 :invoke :cas [26 [22 21]] INFO [2019-12-17 12:35:40,116] jepsen worker 40 - jepsen.util 13630 :info :cas [13 [42 31]] :RequestCanceledException INFO [2019-12-17 12:35:40,117] jepsen worker 40 - jepsen.util 13720 :invoke :read [13 nil] INFO [2019-12-17 12:35:40,128] jepsen worker 47 - jepsen.util 10307 :ok :write [15 18] INFO [2019-12-17 12:35:40,128] jepsen worker 47 - jepsen.util 10307 :invoke :read [15 nil] INFO [2019-12-17 12:35:40,132] jepsen worker 18 - jepsen.util 10098 :ok :write [6 29] INFO [2019-12-17 12:35:40,132] jepsen worker 18 - jepsen.util 10098 :invoke :read [6 nil] INFO [2019-12-17 12:35:40,161] jepsen worker 63 - jepsen.util 8793 :ok :write [21 8] INFO [2019-12-17 12:35:40,161] jepsen worker 63 - jepsen.util 8793 :invoke :read [21 nil] INFO [2019-12-17 12:35:40,186] jepsen worker 28 - jepsen.util 8038 :ok :write [9 3] INFO [2019-12-17 12:35:40,186] jepsen worker 28 - jepsen.util 8038 :invoke :read [9 nil] INFO [2019-12-17 12:35:40,211] jepsen worker 11 - jepsen.util 9641 :fail :cas [3 [43 8]] :ValueNotSwapFrom INFO [2019-12-17 12:35:40,211] jepsen worker 11 - jepsen.util 9641 :invoke :cas [3 [13 46]] INFO [2019-12-17 12:35:40,236] jepsen worker 28 - jepsen.util 8038 :ok :read [9 3] INFO [2019-12-17 12:35:40,236] jepsen worker 28 - jepsen.util 8038 :invoke :cas [9 [38 19]] INFO [2019-12-17 12:35:40,245] jepsen worker 49 - jepsen.util 13099 :info :write [16 0] :RequestCanceledException INFO [2019-12-17 12:35:40,246] jepsen worker 49 - jepsen.util 13189 :invoke :cas [16 [38 27]] INFO [2019-12-17 12:35:40,256] jepsen worker 11 - jepsen.util 9641 :fail :cas [3 [13 46]] :ValueNotSwapFrom INFO [2019-12-17 12:35:40,256] jepsen worker 11 - jepsen.util 9641 :invoke :write [3 23] INFO [2019-12-17 12:35:40,257] jepsen worker 37 - jepsen.util 9217 :ok :write [12 15] INFO [2019-12-17 12:35:40,258] jepsen worker 37 - jepsen.util 9217 :invoke :write [12 16] INFO [2019-12-17 12:35:40,262] jepsen worker 66 - jepsen.util 13026 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:35:40,262] jepsen worker 71 - jepsen.util 10421 :info :write [23 11] :RequestCanceledException INFO [2019-12-17 12:35:40,262] jepsen worker 66 - jepsen.util 13026 :invoke :write [22 13] INFO [2019-12-17 12:35:40,263] jepsen worker 71 - jepsen.util 10511 :invoke :write [23 22] INFO [2019-12-17 12:35:40,266] jepsen worker 4 - jepsen.util 9184 :fail :cas [1 [7 43]] :ValueNotSwapFrom INFO [2019-12-17 12:35:40,266] jepsen worker 4 - jepsen.util 9184 :invoke :read [1 nil] INFO [2019-12-17 12:35:40,274] jepsen worker 28 - jepsen.util 8038 :fail :cas [9 [38 19]] :ValueNotSwapFrom INFO [2019-12-17 12:35:40,274] jepsen worker 28 - jepsen.util 8038 :invoke :cas [9 [42 1]] INFO [2019-12-17 12:35:40,284] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":56446,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:35:40,284] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":56446,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:35:40,284] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":56449,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:35:40,284] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":56449,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:35:40,284] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":56450,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:35:40,284] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":56450,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:35:40,287] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":56438,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:35:40,287] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":56438,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:35:40,287] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":56440,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:35:40,287] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":56440,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:35:40,292] jepsen worker 12 - jepsen.util 10092 :fail :cas [4 [42 20]] :ValueNotSwapFrom INFO [2019-12-17 12:35:40,292] jepsen worker 12 - jepsen.util 10092 :invoke :read [4 nil] INFO [2019-12-17 12:35:40,298] jepsen worker 0 - jepsen.util 14130 :info :write [0 4] :RequestCanceledException INFO [2019-12-17 12:35:40,299] jepsen worker 0 - jepsen.util 14220 :invoke :write [0 30] INFO [2019-12-17 12:35:40,310] jepsen worker 61 - jepsen.util 14191 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:35:40,310] jepsen worker 61 - jepsen.util 14191 :invoke :write [20 19] INFO [2019-12-17 12:35:40,331] jepsen worker 26 - jepsen.util 13796 :info :write [8 39] :RequestCanceledException INFO [2019-12-17 12:35:40,331] jepsen worker 75 - jepsen.util 12855 :info :cas [25 [38 34]] :RequestCanceledException INFO [2019-12-17 12:35:40,331] jepsen worker 32 - jepsen.util 9932 :fail :read [10 nil] :RequestTimeoutException INFO [2019-12-17 12:35:40,331] jepsen worker 32 - jepsen.util 9932 :invoke :read [10 nil] INFO [2019-12-17 12:35:40,332] jepsen worker 75 - jepsen.util 12945 :invoke :read [25 nil] INFO [2019-12-17 12:35:40,332] jepsen worker 26 - jepsen.util 13886 :invoke :write [8 11] INFO [2019-12-17 12:35:40,348] jepsen worker 74 - jepsen.util 9614 :ok :write [24 47] INFO [2019-12-17 12:35:40,348] jepsen worker 74 - jepsen.util 9614 :invoke :write [24 13] INFO [2019-12-17 12:35:40,354] jepsen worker 32 - jepsen.util 9932 :ok :read [10 24] INFO [2019-12-17 12:35:40,354] jepsen worker 32 - jepsen.util 9932 :invoke :read [10 nil] INFO [2019-12-17 12:35:40,360] jepsen worker 68 - jepsen.util 12758 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:35:40,360] jepsen worker 7 - jepsen.util 13327 :info :cas [2 [18 5]] :RequestCanceledException INFO [2019-12-17 12:35:40,360] jepsen worker 68 - jepsen.util 12758 :invoke :read [22 nil] INFO [2019-12-17 12:35:40,363] jepsen worker 11 - jepsen.util 9641 :ok :write [3 23] INFO [2019-12-17 12:35:40,363] jepsen worker 30 - jepsen.util 9930 :ok :write [10 3] INFO [2019-12-17 12:35:40,363] jepsen worker 7 - jepsen.util 13417 :invoke :write [2 30] INFO [2019-12-17 12:35:40,363] jepsen worker 11 - jepsen.util 9641 :invoke :read [3 nil] INFO [2019-12-17 12:35:40,363] jepsen worker 30 - jepsen.util 9930 :invoke :cas [10 [8 24]] INFO [2019-12-17 12:35:40,383] jepsen worker 70 - jepsen.util 10600 :ok :read [23 32] INFO [2019-12-17 12:35:40,383] jepsen worker 70 - jepsen.util 10600 :invoke :read [23 nil] INFO [2019-12-17 12:35:40,394] jepsen worker 82 - jepsen.util 13852 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:35:40,394] jepsen worker 1 - jepsen.util 13411 :info :cas [0 [43 16]] :RequestCanceledException INFO [2019-12-17 12:35:40,394] jepsen worker 82 - jepsen.util 13852 :invoke :read [27 nil] INFO [2019-12-17 12:35:40,395] jepsen worker 1 - jepsen.util 13501 :invoke :read [0 nil] INFO [2019-12-17 12:35:40,403] jepsen worker 11 - jepsen.util 9641 :ok :read [3 23] INFO [2019-12-17 12:35:40,403] jepsen worker 11 - jepsen.util 9641 :invoke :read [3 nil] INFO [2019-12-17 12:35:40,407] jepsen worker 18 - jepsen.util 10098 :ok :read [6 29] INFO [2019-12-17 12:35:40,407] jepsen worker 18 - jepsen.util 10098 :invoke :read [6 nil] INFO [2019-12-17 12:35:40,430] jepsen worker 53 - jepsen.util 9233 :info :write [17 4] :RequestCanceledException INFO [2019-12-17 12:35:40,432] jepsen worker 32 - jepsen.util 9932 :ok :read [10 3] INFO [2019-12-17 12:35:40,432] jepsen worker 32 - jepsen.util 9932 :invoke :read [10 nil] INFO [2019-12-17 12:35:40,435] jepsen worker 53 - jepsen.util 9323 :invoke :cas [17 [49 16]] INFO [2019-12-17 12:35:40,445] jepsen worker 58 - jepsen.util 13828 :info :cas [19 [14 36]] :RequestCanceledException INFO [2019-12-17 12:35:40,446] jepsen worker 58 - jepsen.util 13918 :invoke :read [19 nil] INFO [2019-12-17 12:35:40,457] jepsen worker 51 - jepsen.util 9051 :info :write [17 28] :RequestCanceledException INFO [2019-12-17 12:35:40,457] jepsen worker 78 - jepsen.util 9798 :ok :read [26 42] INFO [2019-12-17 12:35:40,458] jepsen worker 78 - jepsen.util 9798 :invoke :read [26 nil] INFO [2019-12-17 12:35:40,458] jepsen worker 51 - jepsen.util 9141 :invoke :cas [17 [48 28]] INFO [2019-12-17 12:35:40,466] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:35:40,466] jepsen nemesis - couchbase.util Rebalance check count: 442 INFO [2019-12-17 12:35:40,466] jepsen nemesis - couchbase.util Rebalance stuck count: 423 INFO [2019-12-17 12:35:40,470] jepsen worker 11 - jepsen.util 9641 :ok :read [3 23] INFO [2019-12-17 12:35:40,470] jepsen worker 11 - jepsen.util 9641 :invoke :cas [3 [15 29]] INFO [2019-12-17 12:35:40,486] jepsen worker 32 - jepsen.util 9932 :ok :read [10 3] INFO [2019-12-17 12:35:40,486] jepsen worker 32 - jepsen.util 9932 :invoke :cas [10 [35 42]] INFO [2019-12-17 12:35:40,486] jepsen worker 30 - jepsen.util 9930 :fail :cas [10 [8 24]] :ValueNotSwapFrom INFO [2019-12-17 12:35:40,486] jepsen worker 30 - jepsen.util 9930 :invoke :read [10 nil] INFO [2019-12-17 12:35:40,498] jepsen worker 25 - jepsen.util 14335 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:35:40,498] jepsen worker 25 - jepsen.util 14335 :invoke :read [8 nil] INFO [2019-12-17 12:35:40,503] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":56477,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:35:40,503] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":56477,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:35:40,503] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":56481,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:35:40,503] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":56481,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:35:40,516] jepsen worker 36 - jepsen.util 10386 :info :write [12 3] :RequestCanceledException INFO [2019-12-17 12:35:40,516] jepsen worker 36 - jepsen.util 10476 :invoke :write [12 45] INFO [2019-12-17 12:35:40,520] jepsen worker 11 - jepsen.util 9641 :fail :cas [3 [15 29]] :ValueNotSwapFrom INFO [2019-12-17 12:35:40,520] jepsen worker 11 - jepsen.util 9641 :invoke :cas [3 [35 38]] INFO [2019-12-17 12:35:40,532] jepsen worker 18 - jepsen.util 10098 :ok :read [6 29] INFO [2019-12-17 12:35:40,532] jepsen worker 18 - jepsen.util 10098 :invoke :write [6 28] INFO [2019-12-17 12:35:40,543] jepsen worker 32 - jepsen.util 9932 :fail :cas [10 [35 42]] :ValueNotSwapFrom INFO [2019-12-17 12:35:40,543] jepsen worker 32 - jepsen.util 9932 :invoke :cas [10 [23 33]] INFO [2019-12-17 12:35:40,551] jepsen worker 13 - jepsen.util 9193 :fail :cas [4 [31 5]] :ValueNotSwapFrom INFO [2019-12-17 12:35:40,551] jepsen worker 13 - jepsen.util 9193 :invoke :write [4 38] INFO [2019-12-17 12:35:40,584] jepsen worker 65 - jepsen.util 8345 :ok :write [21 32] INFO [2019-12-17 12:35:40,584] jepsen worker 65 - jepsen.util 8345 :invoke :write [21 10] INFO [2019-12-17 12:35:40,585] jepsen worker 78 - jepsen.util 9798 :ok :read [26 42] INFO [2019-12-17 12:35:40,585] jepsen worker 78 - jepsen.util 9798 :invoke :read [26 nil] INFO [2019-12-17 12:35:40,596] jepsen worker 79 - jepsen.util 10069 :fail :cas [26 [22 21]] :ValueNotSwapFrom INFO [2019-12-17 12:35:40,596] jepsen worker 79 - jepsen.util 10069 :invoke :read [26 nil] INFO [2019-12-17 12:35:40,631] jepsen worker 45 - jepsen.util 10035 :info :cas [15 [3 0]] :RequestCanceledException INFO [2019-12-17 12:35:40,632] jepsen worker 45 - jepsen.util 10125 :invoke :read [15 nil] INFO [2019-12-17 12:35:40,653] jepsen worker 11 - jepsen.util 9641 :fail :cas [3 [35 38]] :ValueNotSwapFrom INFO [2019-12-17 12:35:40,653] jepsen worker 11 - jepsen.util 9641 :invoke :write [3 48] INFO [2019-12-17 12:35:40,656] jepsen worker 12 - jepsen.util 10092 :ok :read [4 41] INFO [2019-12-17 12:35:40,656] jepsen worker 12 - jepsen.util 10092 :invoke :write [4 48] INFO [2019-12-17 12:35:40,679] jepsen worker 30 - jepsen.util 9930 :ok :read [10 3] INFO [2019-12-17 12:35:40,679] jepsen worker 30 - jepsen.util 9930 :invoke :cas [10 [39 26]] INFO [2019-12-17 12:35:40,687] jepsen worker 32 - jepsen.util 9932 :fail :cas [10 [23 33]] :ValueNotSwapFrom INFO [2019-12-17 12:35:40,687] jepsen worker 32 - jepsen.util 9932 :invoke :write [10 26] INFO [2019-12-17 12:35:40,695] jepsen worker 47 - jepsen.util 10307 :ok :read [15 18] INFO [2019-12-17 12:35:40,695] jepsen worker 47 - jepsen.util 10307 :invoke :cas [15 [8 18]] INFO [2019-12-17 12:35:40,712] jepsen worker 3 - jepsen.util 9633 :ok :write [1 12] INFO [2019-12-17 12:35:40,712] jepsen worker 3 - jepsen.util 9633 :invoke :write [1 27] INFO [2019-12-17 12:35:40,716] jepsen worker 81 - jepsen.util 13041 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:35:40,716] jepsen worker 29 - jepsen.util 8039 :info :write [9 25] :RequestCanceledException INFO [2019-12-17 12:35:40,716] jepsen worker 89 - jepsen.util 9809 :info :write [29 7] :RequestCanceledException INFO [2019-12-17 12:35:40,716] jepsen worker 81 - jepsen.util 13041 :invoke :write [27 10] INFO [2019-12-17 12:35:40,717] jepsen worker 89 - jepsen.util 9899 :invoke :read [29 nil] INFO [2019-12-17 12:35:40,717] jepsen worker 29 - jepsen.util 8129 :invoke :cas [9 [43 12]] INFO [2019-12-17 12:35:40,722] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":56501,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:35:40,722] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":56501,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:35:40,722] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":56504,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:35:40,722] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":56504,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:35:40,722] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":56505,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:35:40,722] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":56505,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:35:40,745] jepsen worker 4 - jepsen.util 9184 :ok :read [1 12] INFO [2019-12-17 12:35:40,745] jepsen worker 4 - jepsen.util 9184 :invoke :read [1 nil] INFO [2019-12-17 12:35:40,745] jepsen worker 80 - jepsen.util 9440 :info :write [26 34] :RequestCanceledException INFO [2019-12-17 12:35:40,746] jepsen worker 80 - jepsen.util 9530 :invoke :cas [26 [35 2]] INFO [2019-12-17 12:35:40,757] jepsen worker 45 - jepsen.util 10125 :ok :read [15 18] INFO [2019-12-17 12:35:40,758] jepsen worker 45 - jepsen.util 10125 :invoke :read [15 nil] INFO [2019-12-17 12:35:40,785] jepsen worker 45 - jepsen.util 10125 :ok :read [15 18] INFO [2019-12-17 12:35:40,786] jepsen worker 45 - jepsen.util 10125 :invoke :write [15 49] INFO [2019-12-17 12:35:40,816] jepsen worker 46 - jepsen.util 9766 :fail :read [15 nil] :RequestTimeoutException INFO [2019-12-17 12:35:40,816] jepsen worker 46 - jepsen.util 9766 :invoke :cas [15 [39 18]] INFO [2019-12-17 12:35:40,820] jepsen worker 14 - jepsen.util 9644 :ok :write [4 22] INFO [2019-12-17 12:35:40,820] jepsen worker 14 - jepsen.util 9644 :invoke :write [4 1] INFO [2019-12-17 12:35:40,827] jepsen worker 53 - jepsen.util 9323 :fail :cas [17 [49 16]] :ValueNotSwapFrom INFO [2019-12-17 12:35:40,827] jepsen worker 53 - jepsen.util 9323 :invoke :cas [17 [23 19]] INFO [2019-12-17 12:35:40,830] jepsen worker 5 - jepsen.util 9095 :info :write [1 39] :RequestCanceledException INFO [2019-12-17 12:35:40,831] jepsen worker 48 - jepsen.util 12288 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:35:40,831] jepsen worker 48 - jepsen.util 12288 :invoke :read [16 nil] INFO [2019-12-17 12:35:40,831] jepsen worker 5 - jepsen.util 9185 :invoke :write [1 48] INFO [2019-12-17 12:35:40,863] jepsen worker 39 - jepsen.util 12909 :info :write [13 49] :RequestCanceledException INFO [2019-12-17 12:35:40,863] jepsen worker 39 - jepsen.util 12999 :invoke :read [13 nil] INFO [2019-12-17 12:35:40,877] jepsen worker 88 - jepsen.util 10078 :ok :write [29 38] INFO [2019-12-17 12:35:40,877] jepsen worker 88 - jepsen.util 10078 :invoke :cas [29 [3 45]] INFO [2019-12-17 12:35:40,929] jepsen worker 27 - jepsen.util 9297 :info :write [9 38] :RequestCanceledException INFO [2019-12-17 12:35:40,930] jepsen worker 27 - jepsen.util 9387 :invoke :cas [9 [49 31]] INFO [2019-12-17 12:35:40,938] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":56525,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:35:40,938] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":56525,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:35:40,938] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":56529,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:35:40,938] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":56529,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:35:40,945] jepsen worker 54 - jepsen.util 10764 :info :write [18 32] :RequestCanceledException INFO [2019-12-17 12:35:40,945] jepsen worker 19 - jepsen.util 9919 :info :write [6 40] :RequestCanceledException INFO [2019-12-17 12:35:40,946] jepsen worker 19 - jepsen.util 10009 :invoke :read [6 nil] INFO [2019-12-17 12:35:40,949] jepsen worker 54 - jepsen.util 10854 :invoke :read [18 nil] INFO [2019-12-17 12:35:40,976] jepsen worker 71 - jepsen.util 10511 :ok :write [23 22] INFO [2019-12-17 12:35:40,976] jepsen worker 71 - jepsen.util 10511 :invoke :read [23 nil] INFO [2019-12-17 12:35:40,988] jepsen worker 38 - jepsen.util 10388 :ok :write [12 29] INFO [2019-12-17 12:35:40,988] jepsen worker 38 - jepsen.util 10388 :invoke :write [12 44] INFO [2019-12-17 12:35:40,998] jepsen worker 55 - jepsen.util 10405 :info :write [18 34] :RequestCanceledException INFO [2019-12-17 12:35:40,999] jepsen worker 55 - jepsen.util 10495 :invoke :write [18 47] INFO [2019-12-17 12:35:41,015] jepsen worker 87 - jepsen.util 9447 :info :write [29 41] :RequestCanceledException INFO [2019-12-17 12:35:41,016] jepsen worker 87 - jepsen.util 9537 :invoke :read [29 nil] INFO [2019-12-17 12:35:41,026] jepsen worker 70 - jepsen.util 10600 :ok :read [23 22] INFO [2019-12-17 12:35:41,026] jepsen worker 70 - jepsen.util 10600 :invoke :read [23 nil] INFO [2019-12-17 12:35:41,030] jepsen worker 33 - jepsen.util 13353 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:35:41,031] jepsen worker 33 - jepsen.util 13353 :invoke :cas [11 [2 18]] INFO [2019-12-17 12:35:41,035] jepsen worker 56 - jepsen.util 10316 :ok :write [18 46] INFO [2019-12-17 12:35:41,035] jepsen worker 56 - jepsen.util 10316 :invoke :cas [18 [22 15]] INFO [2019-12-17 12:35:41,050] jepsen worker 51 - jepsen.util 9141 :fail :cas [17 [48 28]] :ValueNotSwapFrom INFO [2019-12-17 12:35:41,050] jepsen worker 51 - jepsen.util 9141 :invoke :read [17 nil] INFO [2019-12-17 12:35:41,052] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":56537,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:35:41,052] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":56537,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:35:41,052] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":56539,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:35:41,052] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":56539,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:35:41,061] jepsen worker 77 - jepsen.util 14027 :info :cas [25 [7 28]] :RequestCanceledException INFO [2019-12-17 12:35:41,062] jepsen worker 77 - jepsen.util 14117 :invoke :cas [25 [39 31]] INFO [2019-12-17 12:35:41,091] jepsen worker 79 - jepsen.util 10069 :ok :read [26 42] INFO [2019-12-17 12:35:41,092] jepsen worker 79 - jepsen.util 10069 :invoke :read [26 nil] INFO [2019-12-17 12:35:41,095] jepsen worker 80 - jepsen.util 9530 :fail :cas [26 [35 2]] :ValueNotSwapFrom INFO [2019-12-17 12:35:41,095] jepsen worker 80 - jepsen.util 9530 :invoke :read [26 nil] INFO [2019-12-17 12:35:41,131] jepsen worker 59 - jepsen.util 12659 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:35:41,131] jepsen worker 59 - jepsen.util 12659 :invoke :cas [19 [44 35]] INFO [2019-12-17 12:35:41,136] jepsen worker 18 - jepsen.util 10098 :ok :write [6 28] INFO [2019-12-17 12:35:41,136] jepsen worker 32 - jepsen.util 9932 :ok :write [10 26] INFO [2019-12-17 12:35:41,136] jepsen worker 11 - jepsen.util 9641 :ok :write [3 48] INFO [2019-12-17 12:35:41,136] jepsen worker 74 - jepsen.util 9614 :ok :write [24 13] INFO [2019-12-17 12:35:41,136] jepsen worker 18 - jepsen.util 10098 :invoke :write [6 10] INFO [2019-12-17 12:35:41,137] jepsen worker 32 - jepsen.util 9932 :invoke :read [10 nil] INFO [2019-12-17 12:35:41,137] jepsen worker 11 - jepsen.util 9641 :invoke :read [3 nil] INFO [2019-12-17 12:35:41,137] jepsen worker 74 - jepsen.util 9614 :invoke :write [24 38] INFO [2019-12-17 12:35:41,161] jepsen worker 30 - jepsen.util 9930 :fail :cas [10 [39 26]] :ValueNotSwapFrom INFO [2019-12-17 12:35:41,161] jepsen worker 30 - jepsen.util 9930 :invoke :cas [10 [30 47]] INFO [2019-12-17 12:35:41,192] jepsen worker 52 - jepsen.util 9772 :ok :cas [17 [24 15]] INFO [2019-12-17 12:35:41,192] jepsen worker 52 - jepsen.util 9772 :invoke :write [17 20] INFO [2019-12-17 12:35:41,192] jepsen worker 45 - jepsen.util 10125 :ok :write [15 49] INFO [2019-12-17 12:35:41,192] jepsen worker 45 - jepsen.util 10125 :invoke :cas [15 [3 13]] INFO [2019-12-17 12:35:41,207] jepsen worker 53 - jepsen.util 9323 :fail :cas [17 [23 19]] :ValueNotSwapFrom INFO [2019-12-17 12:35:41,207] jepsen worker 53 - jepsen.util 9323 :invoke :write [17 4] INFO [2019-12-17 12:35:41,216] jepsen worker 50 - jepsen.util 14090 :info :cas [16 [32 34]] :RequestCanceledException INFO [2019-12-17 12:35:41,217] jepsen worker 50 - jepsen.util 14180 :invoke :read [16 nil] INFO [2019-12-17 12:35:41,225] jepsen worker 32 - jepsen.util 9932 :ok :read [10 26] INFO [2019-12-17 12:35:41,225] jepsen worker 32 - jepsen.util 9932 :invoke :cas [10 [28 19]] INFO [2019-12-17 12:35:41,229] jepsen worker 12 - jepsen.util 10092 :ok :write [4 48] INFO [2019-12-17 12:35:41,230] jepsen worker 12 - jepsen.util 10092 :invoke :read [4 nil] INFO [2019-12-17 12:35:41,231] jepsen worker 8 - jepsen.util 12968 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:35:41,231] jepsen worker 84 - jepsen.util 13044 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:35:41,231] jepsen worker 67 - jepsen.util 13207 :info :write [22 9] :RequestCanceledException INFO [2019-12-17 12:35:41,231] jepsen worker 8 - jepsen.util 12968 :invoke :write [2 19] INFO [2019-12-17 12:35:41,231] jepsen worker 84 - jepsen.util 13044 :invoke :read [28 nil] INFO [2019-12-17 12:35:41,233] jepsen worker 67 - jepsen.util 13297 :invoke :cas [22 [17 15]] INFO [2019-12-17 12:35:41,233] jepsen worker 19 - jepsen.util 10009 :ok :read [6 28] INFO [2019-12-17 12:35:41,233] jepsen worker 19 - jepsen.util 10009 :invoke :write [6 19] INFO [2019-12-17 12:35:41,245] jepsen worker 22 - jepsen.util 13072 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:35:41,245] jepsen worker 41 - jepsen.util 14441 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:35:41,245] jepsen worker 22 - jepsen.util 13072 :invoke :read [7 nil] INFO [2019-12-17 12:35:41,245] jepsen worker 41 - jepsen.util 14441 :invoke :read [13 nil] INFO [2019-12-17 12:35:41,262] jepsen worker 2 - jepsen.util 13232 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:35:41,262] jepsen worker 2 - jepsen.util 13232 :invoke :read [0 nil] INFO [2019-12-17 12:35:41,263] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":56561,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:35:41,263] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":56570,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:35:41,263] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":56561,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:35:41,263] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":56570,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:35:41,263] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":56562,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:35:41,263] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":56562,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:35:41,298] jepsen worker 15 - jepsen.util 12615 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:35:41,298] jepsen worker 15 - jepsen.util 12615 :invoke :read [5 nil] INFO [2019-12-17 12:35:41,300] jepsen worker 56 - jepsen.util 10316 :fail :cas [18 [22 15]] :ValueNotSwapFrom INFO [2019-12-17 12:35:41,300] jepsen worker 56 - jepsen.util 10316 :invoke :write [18 47] INFO [2019-12-17 12:35:41,315] jepsen worker 21 - jepsen.util 12891 :info :cas [7 [1 0]] :RequestCanceledException INFO [2019-12-17 12:35:41,317] jepsen worker 21 - jepsen.util 12981 :invoke :write [7 41] INFO [2019-12-17 12:35:41,322] jepsen worker 32 - jepsen.util 9932 :fail :cas [10 [28 19]] :ValueNotSwapFrom INFO [2019-12-17 12:35:41,322] jepsen worker 32 - jepsen.util 9932 :invoke :write [10 11] INFO [2019-12-17 12:35:41,325] jepsen worker 11 - jepsen.util 9641 :ok :read [3 48] INFO [2019-12-17 12:35:41,326] jepsen worker 11 - jepsen.util 9641 :invoke :write [3 16] INFO [2019-12-17 12:35:41,328] jepsen worker 34 - jepsen.util 13354 :info :write [11 30] :RequestCanceledException INFO [2019-12-17 12:35:41,328] jepsen worker 16 - jepsen.util 14506 :info :write [5 0] :RequestCanceledException INFO [2019-12-17 12:35:41,330] jepsen worker 34 - jepsen.util 13444 :invoke :read [11 nil] INFO [2019-12-17 12:35:41,330] jepsen worker 16 - jepsen.util 14596 :invoke :cas [5 [2 26]] INFO [2019-12-17 12:35:41,336] jepsen worker 46 - jepsen.util 9766 :fail :cas [15 [39 18]] :ValueNotSwapFrom INFO [2019-12-17 12:35:41,336] jepsen worker 46 - jepsen.util 9766 :invoke :cas [15 [35 48]] INFO [2019-12-17 12:35:41,345] jepsen worker 86 - jepsen.util 14126 :info :cas [28 [44 23]] :RequestCanceledException INFO [2019-12-17 12:35:41,346] jepsen worker 86 - jepsen.util 14216 :invoke :write [28 12] INFO [2019-12-17 12:35:41,351] jepsen worker 27 - jepsen.util 9387 :fail :cas [9 [49 31]] :ValueNotSwapFrom INFO [2019-12-17 12:35:41,351] jepsen worker 27 - jepsen.util 9387 :invoke :read [9 nil] INFO [2019-12-17 12:35:41,372] jepsen worker 45 - jepsen.util 10125 :fail :cas [15 [3 13]] :ValueNotSwapFrom INFO [2019-12-17 12:35:41,372] jepsen worker 45 - jepsen.util 10125 :invoke :read [15 nil] INFO [2019-12-17 12:35:41,394] jepsen worker 65 - jepsen.util 8345 :ok :write [21 10] INFO [2019-12-17 12:35:41,394] jepsen worker 65 - jepsen.util 8345 :invoke :write [21 3] INFO [2019-12-17 12:35:41,396] jepsen worker 46 - jepsen.util 9766 :fail :cas [15 [35 48]] :ValueNotSwapFrom INFO [2019-12-17 12:35:41,396] jepsen worker 46 - jepsen.util 9766 :invoke :read [15 nil] INFO [2019-12-17 12:35:41,429] jepsen worker 27 - jepsen.util 9387 :ok :read [9 3] INFO [2019-12-17 12:35:41,430] jepsen worker 27 - jepsen.util 9387 :invoke :cas [9 [28 33]] INFO [2019-12-17 12:35:41,454] jepsen worker 80 - jepsen.util 9530 :ok :read [26 42] INFO [2019-12-17 12:35:41,454] jepsen worker 80 - jepsen.util 9530 :invoke :write [26 9] INFO [2019-12-17 12:35:41,459] jepsen worker 45 - jepsen.util 10125 :ok :read [15 49] INFO [2019-12-17 12:35:41,460] jepsen worker 45 - jepsen.util 10125 :invoke :cas [15 [2 40]] INFO [2019-12-17 12:35:41,462] jepsen worker 43 - jepsen.util 13273 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:35:41,463] jepsen worker 43 - jepsen.util 13273 :invoke :cas [14 [34 4]] INFO [2019-12-17 12:35:41,484] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:35:41,484] jepsen nemesis - couchbase.util Rebalance check count: 443 INFO [2019-12-17 12:35:41,484] jepsen nemesis - couchbase.util Rebalance stuck count: 424 INFO [2019-12-17 12:35:41,490] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":56599,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:35:41,492] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":56599,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:35:41,492] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":56601,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:35:41,492] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":56601,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:35:41,497] jepsen worker 4 - jepsen.util 9184 :ok :read [1 39] INFO [2019-12-17 12:35:41,498] jepsen worker 4 - jepsen.util 9184 :invoke :read [1 nil] INFO [2019-12-17 12:35:41,498] jepsen worker 42 - jepsen.util 14352 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:35:41,498] jepsen worker 42 - jepsen.util 14352 :invoke :write [14 24] INFO [2019-12-17 12:35:41,507] jepsen worker 46 - jepsen.util 9766 :ok :read [15 49] INFO [2019-12-17 12:35:41,507] jepsen worker 46 - jepsen.util 9766 :invoke :read [15 nil] INFO [2019-12-17 12:35:41,531] jepsen worker 10 - jepsen.util 9820 :info :write [3 18] :RequestCanceledException INFO [2019-12-17 12:35:41,531] jepsen worker 60 - jepsen.util 14370 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:35:41,531] jepsen worker 17 - jepsen.util 12977 :info :write [5 30] :RequestCanceledException INFO [2019-12-17 12:35:41,532] jepsen worker 60 - jepsen.util 14370 :invoke :read [20 nil] INFO [2019-12-17 12:35:41,534] jepsen worker 10 - jepsen.util 9910 :invoke :read [3 nil] INFO [2019-12-17 12:35:41,537] jepsen worker 88 - jepsen.util 10078 :fail :cas [29 [3 45]] :ValueNotSwapFrom INFO [2019-12-17 12:35:41,537] jepsen worker 88 - jepsen.util 10078 :invoke :read [29 nil] INFO [2019-12-17 12:35:41,538] jepsen worker 18 - jepsen.util 10098 :ok :write [6 10] INFO [2019-12-17 12:35:41,538] jepsen worker 18 - jepsen.util 10098 :invoke :write [6 32] INFO [2019-12-17 12:35:41,538] jepsen worker 17 - jepsen.util 13067 :invoke :cas [5 [27 29]] INFO [2019-12-17 12:35:41,544] jepsen worker 45 - jepsen.util 10125 :fail :cas [15 [2 40]] :ValueNotSwapFrom INFO [2019-12-17 12:35:41,544] jepsen worker 45 - jepsen.util 10125 :invoke :read [15 nil] INFO [2019-12-17 12:35:41,546] jepsen worker 27 - jepsen.util 9387 :fail :cas [9 [28 33]] :ValueNotSwapFrom INFO [2019-12-17 12:35:41,546] jepsen worker 27 - jepsen.util 9387 :invoke :read [9 nil] INFO [2019-12-17 12:35:41,547] jepsen worker 46 - jepsen.util 9766 :ok :read [15 49] INFO [2019-12-17 12:35:41,547] jepsen worker 46 - jepsen.util 9766 :invoke :cas [15 [46 27]] INFO [2019-12-17 12:35:41,558] jepsen worker 10 - jepsen.util 9910 :ok :read [3 48] INFO [2019-12-17 12:35:41,558] jepsen worker 10 - jepsen.util 9910 :invoke :write [3 13] INFO [2019-12-17 12:35:41,562] jepsen worker 57 - jepsen.util 12927 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:35:41,563] jepsen worker 57 - jepsen.util 12927 :invoke :cas [19 [17 34]] INFO [2019-12-17 12:35:41,577] jepsen worker 45 - jepsen.util 10125 :ok :read [15 49] INFO [2019-12-17 12:35:41,577] jepsen worker 45 - jepsen.util 10125 :invoke :write [15 29] INFO [2019-12-17 12:35:41,619] jepsen worker 27 - jepsen.util 9387 :ok :read [9 3] INFO [2019-12-17 12:35:41,619] jepsen worker 27 - jepsen.util 9387 :invoke :read [9 nil] INFO [2019-12-17 12:35:41,633] jepsen worker 46 - jepsen.util 9766 :fail :cas [15 [46 27]] :ValueNotSwapFrom INFO [2019-12-17 12:35:41,633] jepsen worker 46 - jepsen.util 9766 :invoke :cas [15 [14 25]] INFO [2019-12-17 12:35:41,698] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":56622,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:35:41,698] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":56622,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:35:41,698] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":56623,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:35:41,698] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":56623,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:35:41,715] jepsen worker 69 - jepsen.util 9789 :info :write [23 29] :RequestCanceledException INFO [2019-12-17 12:35:41,717] jepsen worker 69 - jepsen.util 9879 :invoke :cas [23 [0 15]] INFO [2019-12-17 12:35:41,721] jepsen worker 78 - jepsen.util 9798 :ok :read [26 42] INFO [2019-12-17 12:35:41,721] jepsen worker 78 - jepsen.util 9798 :invoke :write [26 7] INFO [2019-12-17 12:35:41,759] jepsen worker 32 - jepsen.util 9932 :ok :write [10 11] INFO [2019-12-17 12:35:41,761] jepsen worker 74 - jepsen.util 9614 :ok :write [24 38] INFO [2019-12-17 12:35:41,761] jepsen worker 18 - jepsen.util 10098 :ok :write [6 32] INFO [2019-12-17 12:35:41,776] jepsen worker 32 - jepsen.util 9932 :invoke :write [10 35] INFO [2019-12-17 12:35:41,776] jepsen worker 74 - jepsen.util 9614 :invoke :read [24 nil] INFO [2019-12-17 12:35:41,776] jepsen worker 18 - jepsen.util 10098 :invoke :write [6 25] INFO [2019-12-17 12:35:41,798] jepsen worker 76 - jepsen.util 13576 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:35:41,798] jepsen worker 24 - jepsen.util 12174 :info :write [8 34] :RequestCanceledException INFO [2019-12-17 12:35:41,798] jepsen worker 76 - jepsen.util 13576 :invoke :write [25 29] INFO [2019-12-17 12:35:41,800] jepsen worker 24 - jepsen.util 12264 :invoke :read [8 nil] INFO [2019-12-17 12:35:41,810] jepsen worker 74 - jepsen.util 9614 :ok :read [24 38] INFO [2019-12-17 12:35:41,811] jepsen worker 74 - jepsen.util 9614 :invoke :write [24 9] INFO [2019-12-17 12:35:41,815] jepsen worker 6 - jepsen.util 13866 :info :cas [2 [36 29]] :RequestCanceledException INFO [2019-12-17 12:35:41,816] jepsen worker 6 - jepsen.util 13956 :invoke :cas [2 [5 7]] INFO [2019-12-17 12:35:41,830] jepsen worker 72 - jepsen.util 9612 :info :write [24 32] :RequestCanceledException INFO [2019-12-17 12:35:41,830] jepsen worker 31 - jepsen.util 8761 :info :write [10 25] :RequestCanceledException INFO [2019-12-17 12:35:41,830] jepsen worker 64 - jepsen.util 7624 :fail :read [21 nil] :RequestTimeoutException INFO [2019-12-17 12:35:41,831] jepsen worker 64 - jepsen.util 7624 :invoke :read [21 nil] INFO [2019-12-17 12:35:41,831] jepsen worker 44 - jepsen.util 13634 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:35:41,831] jepsen worker 31 - jepsen.util 8851 :invoke :read [10 nil] INFO [2019-12-17 12:35:41,831] jepsen worker 44 - jepsen.util 13634 :invoke :cas [14 [26 8]] INFO [2019-12-17 12:35:41,833] jepsen worker 72 - jepsen.util 9702 :invoke :cas [24 [25 0]] INFO [2019-12-17 12:35:41,858] jepsen worker 9 - jepsen.util 10449 :info :write [3 28] :RequestCanceledException INFO [2019-12-17 12:35:41,859] jepsen worker 9 - jepsen.util 10539 :invoke :cas [3 [18 0]] INFO [2019-12-17 12:35:41,874] jepsen worker 89 - jepsen.util 9899 :ok :read [29 38] INFO [2019-12-17 12:35:41,874] jepsen worker 89 - jepsen.util 9899 :invoke :cas [29 [29 16]] INFO [2019-12-17 12:35:41,879] jepsen worker 9 - jepsen.util 10539 :fail :cas [3 [18 0]] :ValueNotSwapFrom INFO [2019-12-17 12:35:41,879] jepsen worker 9 - jepsen.util 10539 :invoke :cas [3 [33 28]] INFO [2019-12-17 12:35:41,899] jepsen worker 9 - jepsen.util 10539 :fail :cas [3 [33 28]] :ValueNotSwapFrom INFO [2019-12-17 12:35:41,899] jepsen worker 9 - jepsen.util 10539 :invoke :cas [3 [11 36]] INFO [2019-12-17 12:35:41,910] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":56642,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:35:41,910] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":56642,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:35:41,910] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":56646,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:35:41,910] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":56646,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:35:41,916] jepsen worker 54 - jepsen.util 10854 :ok :read [18 46] INFO [2019-12-17 12:35:41,916] jepsen worker 54 - jepsen.util 10854 :invoke :read [18 nil] INFO [2019-12-17 12:35:41,919] jepsen worker 9 - jepsen.util 10539 :fail :cas [3 [11 36]] :ValueNotSwapFrom INFO [2019-12-17 12:35:41,919] jepsen worker 9 - jepsen.util 10539 :invoke :read [3 nil] INFO [2019-12-17 12:35:41,931] jepsen worker 35 - jepsen.util 13265 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:35:41,931] jepsen worker 9 - jepsen.util 10539 :ok :read [3 48] INFO [2019-12-17 12:35:41,931] jepsen worker 35 - jepsen.util 13265 :invoke :read [11 nil] INFO [2019-12-17 12:35:41,931] jepsen worker 9 - jepsen.util 10539 :invoke :write [3 44] INFO [2019-12-17 12:35:41,944] jepsen worker 87 - jepsen.util 9537 :ok :read [29 38] INFO [2019-12-17 12:35:41,944] jepsen worker 87 - jepsen.util 9537 :invoke :read [29 nil] INFO [2019-12-17 12:35:41,959] jepsen worker 51 - jepsen.util 9141 :ok :read [17 15] INFO [2019-12-17 12:35:41,959] jepsen worker 51 - jepsen.util 9141 :invoke :write [17 18] INFO [2019-12-17 12:35:41,962] jepsen worker 62 - jepsen.util 13742 :info :cas [20 [28 48]] :RequestCanceledException INFO [2019-12-17 12:35:41,963] jepsen worker 62 - jepsen.util 13832 :invoke :cas [20 [46 20]] INFO [2019-12-17 12:35:41,986] jepsen worker 10 - jepsen.util 9910 :ok :write [3 13] INFO [2019-12-17 12:35:41,986] jepsen worker 10 - jepsen.util 9910 :invoke :read [3 nil] INFO [2019-12-17 12:35:42,022] jepsen worker 46 - jepsen.util 9766 :fail :cas [15 [14 25]] :ValueNotSwapFrom INFO [2019-12-17 12:35:42,022] jepsen worker 46 - jepsen.util 9766 :invoke :cas [15 [6 46]] INFO [2019-12-17 12:35:42,031] jepsen worker 20 - jepsen.util 9380 :info :write [6 8] :RequestCanceledException INFO [2019-12-17 12:35:42,032] jepsen worker 23 - jepsen.util 13343 :info :write [7 6] :RequestCanceledException INFO [2019-12-17 12:35:42,032] jepsen worker 27 - jepsen.util 9387 :ok :read [9 3] INFO [2019-12-17 12:35:42,032] jepsen worker 27 - jepsen.util 9387 :invoke :cas [9 [22 3]] INFO [2019-12-17 12:35:42,032] jepsen worker 20 - jepsen.util 9470 :invoke :read [6 nil] INFO [2019-12-17 12:35:42,032] jepsen worker 23 - jepsen.util 13433 :invoke :cas [7 [2 6]] INFO [2019-12-17 12:35:42,035] jepsen worker 72 - jepsen.util 9702 :fail :cas [24 [25 0]] :ValueNotSwapFrom INFO [2019-12-17 12:35:42,035] jepsen worker 72 - jepsen.util 9702 :invoke :cas [24 [29 10]] INFO [2019-12-17 12:35:42,039] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":56662,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:35:42,039] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":56662,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:35:42,053] jepsen worker 10 - jepsen.util 9910 :ok :read [3 13] INFO [2019-12-17 12:35:42,053] jepsen worker 10 - jepsen.util 9910 :invoke :read [3 nil] INFO [2019-12-17 12:35:42,057] jepsen worker 20 - jepsen.util 9470 :ok :read [6 32] INFO [2019-12-17 12:35:42,057] jepsen worker 20 - jepsen.util 9470 :invoke :cas [6 [9 41]] INFO [2019-12-17 12:35:42,063] jepsen worker 46 - jepsen.util 9766 :fail :cas [15 [6 46]] :ValueNotSwapFrom INFO [2019-12-17 12:35:42,063] jepsen worker 46 - jepsen.util 9766 :invoke :read [15 nil] INFO [2019-12-17 12:35:42,078] jepsen worker 27 - jepsen.util 9387 :fail :cas [9 [22 3]] :ValueNotSwapFrom INFO [2019-12-17 12:35:42,078] jepsen worker 27 - jepsen.util 9387 :invoke :cas [9 [28 31]] INFO [2019-12-17 12:35:42,083] jepsen worker 20 - jepsen.util 9470 :fail :cas [6 [9 41]] :ValueNotSwapFrom INFO [2019-12-17 12:35:42,083] jepsen worker 20 - jepsen.util 9470 :invoke :read [6 nil] INFO [2019-12-17 12:35:42,088] jepsen worker 72 - jepsen.util 9702 :fail :cas [24 [29 10]] :ValueNotSwapFrom INFO [2019-12-17 12:35:42,088] jepsen worker 72 - jepsen.util 9702 :invoke :read [24 nil] INFO [2019-12-17 12:35:42,095] jepsen worker 71 - jepsen.util 10511 :ok :read [23 22] INFO [2019-12-17 12:35:42,095] jepsen worker 71 - jepsen.util 10511 :invoke :cas [23 [34 9]] INFO [2019-12-17 12:35:42,097] jepsen worker 85 - jepsen.util 14125 :info :cas [28 [0 28]] :RequestCanceledException INFO [2019-12-17 12:35:42,098] jepsen worker 85 - jepsen.util 14215 :invoke :read [28 nil] INFO [2019-12-17 12:35:42,108] jepsen worker 83 - jepsen.util 13673 :info :write [27 40] :RequestCanceledException INFO [2019-12-17 12:35:42,109] jepsen worker 83 - jepsen.util 13763 :invoke :write [27 18] INFO [2019-12-17 12:35:42,112] jepsen worker 10 - jepsen.util 9910 :ok :read [3 13] INFO [2019-12-17 12:35:42,112] jepsen worker 10 - jepsen.util 9910 :invoke :read [3 nil] INFO [2019-12-17 12:35:42,119] jepsen worker 46 - jepsen.util 9766 :ok :read [15 49] INFO [2019-12-17 12:35:42,120] jepsen worker 46 - jepsen.util 9766 :invoke :cas [15 [15 15]] INFO [2019-12-17 12:35:42,121] jepsen worker 20 - jepsen.util 9470 :ok :read [6 32] INFO [2019-12-17 12:35:42,121] jepsen worker 20 - jepsen.util 9470 :invoke :read [6 nil] INFO [2019-12-17 12:35:42,123] jepsen worker 70 - jepsen.util 10600 :ok :read [23 22] INFO [2019-12-17 12:35:42,123] jepsen worker 70 - jepsen.util 10600 :invoke :cas [23 [5 36]] INFO [2019-12-17 12:35:42,139] jepsen worker 27 - jepsen.util 9387 :fail :cas [9 [28 31]] :ValueNotSwapFrom INFO [2019-12-17 12:35:42,139] jepsen worker 27 - jepsen.util 9387 :invoke :write [9 49] INFO [2019-12-17 12:35:42,140] jepsen worker 20 - jepsen.util 9470 :ok :read [6 32] INFO [2019-12-17 12:35:42,141] jepsen worker 20 - jepsen.util 9470 :invoke :write [6 8] INFO [2019-12-17 12:35:42,145] jepsen worker 72 - jepsen.util 9702 :ok :read [24 38] INFO [2019-12-17 12:35:42,145] jepsen worker 72 - jepsen.util 9702 :invoke :cas [24 [18 22]] INFO [2019-12-17 12:35:42,150] jepsen worker 10 - jepsen.util 9910 :ok :read [3 13] INFO [2019-12-17 12:35:42,150] jepsen worker 10 - jepsen.util 9910 :invoke :read [3 nil] INFO [2019-12-17 12:35:42,157] jepsen worker 79 - jepsen.util 10069 :ok :read [26 42] INFO [2019-12-17 12:35:42,157] jepsen worker 79 - jepsen.util 10069 :invoke :write [26 22] INFO [2019-12-17 12:35:42,166] jepsen worker 46 - jepsen.util 9766 :fail :cas [15 [15 15]] :ValueNotSwapFrom INFO [2019-12-17 12:35:42,166] jepsen worker 46 - jepsen.util 9766 :invoke :write [15 49] INFO [2019-12-17 12:35:42,199] jepsen worker 12 - jepsen.util 10092 :ok :read [4 48] INFO [2019-12-17 12:35:42,199] jepsen worker 12 - jepsen.util 10092 :invoke :cas [4 [30 12]] INFO [2019-12-17 12:35:42,207] jepsen worker 69 - jepsen.util 9879 :fail :cas [23 [0 15]] :ValueNotSwapFrom INFO [2019-12-17 12:35:42,208] jepsen worker 69 - jepsen.util 9879 :invoke :cas [23 [35 25]] INFO [2019-12-17 12:35:42,211] jepsen worker 27 - jepsen.util 9387 :ok :write [9 49] INFO [2019-12-17 12:35:42,211] jepsen worker 27 - jepsen.util 9387 :invoke :cas [9 [38 37]] INFO [2019-12-17 12:35:42,216] jepsen worker 63 - jepsen.util 8793 :fail :read [21 nil] :RequestTimeoutException INFO [2019-12-17 12:35:42,216] jepsen worker 40 - jepsen.util 13720 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:35:42,216] jepsen worker 73 - jepsen.util 9163 :info :write [24 17] :RequestCanceledException INFO [2019-12-17 12:35:42,216] jepsen worker 63 - jepsen.util 8793 :invoke :write [21 18] INFO [2019-12-17 12:35:42,216] jepsen worker 40 - jepsen.util 13720 :invoke :write [13 45] INFO [2019-12-17 12:35:42,217] jepsen worker 73 - jepsen.util 9253 :invoke :read [24 nil] INFO [2019-12-17 12:35:42,237] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":56699,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:35:42,237] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":56689,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:35:42,238] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":56699,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:35:42,238] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":56689,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:35:42,238] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":56703,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:35:42,238] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":56690,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:35:42,238] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":56703,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:35:42,238] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":56690,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:35:42,238] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":56691,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:35:42,238] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":56691,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:35:42,255] jepsen worker 4 - jepsen.util 9184 :ok :read [1 39] INFO [2019-12-17 12:35:42,255] jepsen worker 4 - jepsen.util 9184 :invoke :cas [1 [4 39]] INFO [2019-12-17 12:35:42,257] jepsen worker 72 - jepsen.util 9702 :fail :cas [24 [18 22]] :ValueNotSwapFrom INFO [2019-12-17 12:35:42,258] jepsen worker 72 - jepsen.util 9702 :invoke :read [24 nil] INFO [2019-12-17 12:35:42,289] jepsen worker 73 - jepsen.util 9253 :ok :read [24 38] INFO [2019-12-17 12:35:42,289] jepsen worker 73 - jepsen.util 9253 :invoke :cas [24 [37 12]] INFO [2019-12-17 12:35:42,289] jepsen worker 10 - jepsen.util 9910 :ok :read [3 13] INFO [2019-12-17 12:35:42,289] jepsen worker 10 - jepsen.util 9910 :invoke :cas [3 [19 29]] INFO [2019-12-17 12:35:42,297] jepsen worker 37 - jepsen.util 9217 :info :write [12 16] :RequestCanceledException INFO [2019-12-17 12:35:42,300] jepsen worker 37 - jepsen.util 9307 :invoke :cas [12 [48 16]] INFO [2019-12-17 12:35:42,302] jepsen worker 73 - jepsen.util 9253 :fail :cas [24 [37 12]] :ValueNotSwapFrom INFO [2019-12-17 12:35:42,302] jepsen worker 73 - jepsen.util 9253 :invoke :cas [24 [45 3]] INFO [2019-12-17 12:35:42,318] jepsen worker 73 - jepsen.util 9253 :fail :cas [24 [45 3]] :ValueNotSwapFrom INFO [2019-12-17 12:35:42,319] jepsen worker 73 - jepsen.util 9253 :invoke :read [24 nil] INFO [2019-12-17 12:35:42,328] jepsen worker 88 - jepsen.util 10078 :ok :read [29 38] INFO [2019-12-17 12:35:42,329] jepsen worker 88 - jepsen.util 10078 :invoke :cas [29 [40 29]] INFO [2019-12-17 12:35:42,331] jepsen worker 49 - jepsen.util 13189 :info :cas [16 [38 27]] :RequestCanceledException INFO [2019-12-17 12:35:42,332] jepsen worker 49 - jepsen.util 13279 :invoke :cas [16 [36 11]] INFO [2019-12-17 12:35:42,333] jepsen worker 73 - jepsen.util 9253 :ok :read [24 38] INFO [2019-12-17 12:35:42,333] jepsen worker 73 - jepsen.util 9253 :invoke :write [24 41] INFO [2019-12-17 12:35:42,334] jepsen worker 28 - jepsen.util 8038 :info :cas [9 [42 1]] :RequestCanceledException INFO [2019-12-17 12:35:42,348] jepsen worker 71 - jepsen.util 10511 :fail :cas [23 [34 9]] :ValueNotSwapFrom INFO [2019-12-17 12:35:42,348] jepsen worker 71 - jepsen.util 10511 :invoke :read [23 nil] INFO [2019-12-17 12:35:42,348] jepsen worker 28 - jepsen.util 8128 :invoke :write [9 32] INFO [2019-12-17 12:35:42,362] jepsen worker 0 - jepsen.util 14220 :info :write [0 30] :RequestCanceledException INFO [2019-12-17 12:35:42,362] jepsen worker 66 - jepsen.util 13026 :info :write [22 13] :RequestCanceledException INFO [2019-12-17 12:35:42,363] jepsen worker 0 - jepsen.util 14310 :invoke :read [0 nil] INFO [2019-12-17 12:35:42,363] jepsen worker 66 - jepsen.util 13116 :invoke :write [22 19] INFO [2019-12-17 12:35:42,364] jepsen worker 70 - jepsen.util 10600 :fail :cas [23 [5 36]] :ValueNotSwapFrom INFO [2019-12-17 12:35:42,364] jepsen worker 70 - jepsen.util 10600 :invoke :write [23 0] INFO [2019-12-17 12:35:42,397] jepsen worker 26 - jepsen.util 13886 :info :write [8 11] :RequestCanceledException INFO [2019-12-17 12:35:42,405] jepsen worker 26 - jepsen.util 13976 :invoke :write [8 30] INFO [2019-12-17 12:35:42,414] jepsen worker 7 - jepsen.util 13417 :info :write [2 30] :RequestCanceledException INFO [2019-12-17 12:35:42,414] jepsen worker 61 - jepsen.util 14191 :info :write [20 19] :RequestCanceledException INFO [2019-12-17 12:35:42,415] jepsen worker 7 - jepsen.util 13507 :invoke :read [2 nil] INFO [2019-12-17 12:35:42,415] jepsen worker 61 - jepsen.util 14281 :invoke :read [20 nil] INFO [2019-12-17 12:35:42,430] jepsen worker 75 - jepsen.util 12945 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:35:42,430] jepsen worker 75 - jepsen.util 12945 :invoke :cas [25 [19 41]] INFO [2019-12-17 12:35:42,431] jepsen worker 1 - jepsen.util 13501 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:35:42,431] jepsen worker 1 - jepsen.util 13501 :invoke :write [0 17] INFO [2019-12-17 12:35:42,460] jepsen worker 54 - jepsen.util 10854 :ok :read [18 46] INFO [2019-12-17 12:35:42,460] jepsen worker 54 - jepsen.util 10854 :invoke :cas [18 [47 39]] INFO [2019-12-17 12:35:42,462] jepsen worker 68 - jepsen.util 12758 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:35:42,462] jepsen worker 58 - jepsen.util 13918 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:35:42,462] jepsen worker 68 - jepsen.util 12758 :invoke :write [22 5] INFO [2019-12-17 12:35:42,462] jepsen worker 58 - jepsen.util 13918 :invoke :cas [19 [46 40]] INFO [2019-12-17 12:35:42,469] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":56727,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:35:42,469] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":56727,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:35:42,469] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":56730,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:35:42,469] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":56730,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:35:42,470] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":56731,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:35:42,470] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":56731,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:35:42,480] jepsen worker 87 - jepsen.util 9537 :ok :read [29 38] INFO [2019-12-17 12:35:42,480] jepsen worker 87 - jepsen.util 9537 :invoke :cas [29 [42 2]] INFO [2019-12-17 12:35:42,486] jepsen worker 69 - jepsen.util 9879 :fail :cas [23 [35 25]] :ValueNotSwapFrom INFO [2019-12-17 12:35:42,486] jepsen worker 74 - jepsen.util 9614 :ok :write [24 9] INFO [2019-12-17 12:35:42,486] jepsen worker 69 - jepsen.util 9879 :invoke :cas [23 [4 27]] INFO [2019-12-17 12:35:42,486] jepsen worker 74 - jepsen.util 9614 :invoke :read [24 nil] INFO [2019-12-17 12:35:42,486] jepsen worker 32 - jepsen.util 9932 :ok :write [10 35] INFO [2019-12-17 12:35:42,486] jepsen worker 32 - jepsen.util 9932 :invoke :read [10 nil] INFO [2019-12-17 12:35:42,491] jepsen worker 18 - jepsen.util 10098 :ok :write [6 25] INFO [2019-12-17 12:35:42,491] jepsen worker 18 - jepsen.util 10098 :invoke :read [6 nil] INFO [2019-12-17 12:35:42,498] jepsen worker 82 - jepsen.util 13852 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:35:42,498] jepsen worker 82 - jepsen.util 13852 :invoke :read [27 nil] INFO [2019-12-17 12:35:42,504] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:35:42,504] jepsen nemesis - couchbase.util Rebalance check count: 444 INFO [2019-12-17 12:35:42,504] jepsen nemesis - couchbase.util Rebalance stuck count: 425 INFO [2019-12-17 12:35:42,508] jepsen worker 46 - jepsen.util 9766 :ok :write [15 49] INFO [2019-12-17 12:35:42,509] jepsen worker 46 - jepsen.util 9766 :invoke :read [15 nil] INFO [2019-12-17 12:35:42,518] jepsen worker 31 - jepsen.util 8851 :ok :read [10 35] INFO [2019-12-17 12:35:42,518] jepsen worker 31 - jepsen.util 8851 :invoke :cas [10 [10 4]] INFO [2019-12-17 12:35:42,532] jepsen worker 18 - jepsen.util 10098 :ok :read [6 25] INFO [2019-12-17 12:35:42,532] jepsen worker 18 - jepsen.util 10098 :invoke :read [6 nil] INFO [2019-12-17 12:35:42,533] jepsen worker 12 - jepsen.util 10092 :fail :cas [4 [30 12]] :ValueNotSwapFrom INFO [2019-12-17 12:35:42,534] jepsen worker 12 - jepsen.util 10092 :invoke :cas [4 [29 21]] INFO [2019-12-17 12:35:42,564] jepsen worker 31 - jepsen.util 8851 :fail :cas [10 [10 4]] :ValueNotSwapFrom INFO [2019-12-17 12:35:42,565] jepsen worker 31 - jepsen.util 8851 :invoke :read [10 nil] INFO [2019-12-17 12:35:42,574] jepsen worker 37 - jepsen.util 9307 :fail :cas [12 [48 16]] :ValueNotSwapFrom INFO [2019-12-17 12:35:42,574] jepsen worker 37 - jepsen.util 9307 :invoke :cas [12 [37 26]] INFO [2019-12-17 12:35:42,585] jepsen worker 74 - jepsen.util 9614 :ok :read [24 9] INFO [2019-12-17 12:35:42,585] jepsen worker 74 - jepsen.util 9614 :invoke :write [24 42] INFO [2019-12-17 12:35:42,591] jepsen worker 18 - jepsen.util 10098 :ok :read [6 25] INFO [2019-12-17 12:35:42,592] jepsen worker 18 - jepsen.util 10098 :invoke :read [6 nil] INFO [2019-12-17 12:35:42,598] jepsen worker 25 - jepsen.util 14335 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:35:42,598] jepsen worker 36 - jepsen.util 10476 :info :write [12 45] :RequestCanceledException INFO [2019-12-17 12:35:42,598] jepsen worker 25 - jepsen.util 14335 :invoke :read [8 nil] INFO [2019-12-17 12:35:42,599] jepsen worker 36 - jepsen.util 10566 :invoke :read [12 nil] INFO [2019-12-17 12:35:42,600] jepsen worker 87 - jepsen.util 9537 :fail :cas [29 [42 2]] :ValueNotSwapFrom INFO [2019-12-17 12:35:42,600] jepsen worker 87 - jepsen.util 9537 :invoke :read [29 nil] INFO [2019-12-17 12:35:42,604] jepsen worker 69 - jepsen.util 9879 :fail :cas [23 [4 27]] :ValueNotSwapFrom INFO [2019-12-17 12:35:42,604] jepsen worker 69 - jepsen.util 9879 :invoke :write [23 29] INFO [2019-12-17 12:35:42,609] jepsen worker 64 - jepsen.util 7624 :ok :read [21 10] INFO [2019-12-17 12:35:42,609] jepsen worker 64 - jepsen.util 7624 :invoke :cas [21 [6 42]] INFO [2019-12-17 12:35:42,614] jepsen worker 31 - jepsen.util 8851 :ok :read [10 35] INFO [2019-12-17 12:35:42,614] jepsen worker 31 - jepsen.util 8851 :invoke :cas [10 [21 9]] INFO [2019-12-17 12:35:42,622] jepsen worker 18 - jepsen.util 10098 :ok :read [6 25] INFO [2019-12-17 12:35:42,622] jepsen worker 18 - jepsen.util 10098 :invoke :read [6 nil] INFO [2019-12-17 12:35:42,631] jepsen worker 13 - jepsen.util 9193 :info :write [4 38] :RequestCanceledException INFO [2019-12-17 12:35:42,633] jepsen worker 13 - jepsen.util 9283 :invoke :read [4 nil] INFO [2019-12-17 12:35:42,634] jepsen worker 4 - jepsen.util 9184 :fail :cas [1 [4 39]] :ValueNotSwapFrom INFO [2019-12-17 12:35:42,634] jepsen worker 4 - jepsen.util 9184 :invoke :read [1 nil] INFO [2019-12-17 12:35:42,639] jepsen worker 31 - jepsen.util 8851 :fail :cas [10 [21 9]] :ValueNotSwapFrom INFO [2019-12-17 12:35:42,639] jepsen worker 31 - jepsen.util 8851 :invoke :read [10 nil] INFO [2019-12-17 12:35:42,668] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":56761,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:35:42,668] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":56761,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:35:42,668] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":56765,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:35:42,669] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":56765,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:35:42,672] jepsen worker 18 - jepsen.util 10098 :ok :read [6 25] INFO [2019-12-17 12:35:42,672] jepsen worker 18 - jepsen.util 10098 :invoke :read [6 nil] INFO [2019-12-17 12:35:42,675] jepsen worker 63 - jepsen.util 8793 :ok :write [21 18] INFO [2019-12-17 12:35:42,676] jepsen worker 63 - jepsen.util 8793 :invoke :write [21 22] INFO [2019-12-17 12:35:42,705] jepsen worker 31 - jepsen.util 8851 :ok :read [10 35] INFO [2019-12-17 12:35:42,705] jepsen worker 31 - jepsen.util 8851 :invoke :cas [10 [26 38]] INFO [2019-12-17 12:35:42,714] jepsen worker 28 - jepsen.util 8128 :ok :write [9 32] INFO [2019-12-17 12:35:42,714] jepsen worker 28 - jepsen.util 8128 :invoke :write [9 31] INFO [2019-12-17 12:35:42,717] jepsen worker 71 - jepsen.util 10511 :ok :read [23 22] INFO [2019-12-17 12:35:42,717] jepsen worker 71 - jepsen.util 10511 :invoke :write [23 20] INFO [2019-12-17 12:35:42,732] jepsen worker 36 - jepsen.util 10566 :ok :read [12 29] INFO [2019-12-17 12:35:42,732] jepsen worker 36 - jepsen.util 10566 :invoke :read [12 nil] INFO [2019-12-17 12:35:42,744] jepsen worker 47 - jepsen.util 10307 :info :cas [15 [8 18]] :RequestCanceledException INFO [2019-12-17 12:35:42,744] jepsen worker 29 - jepsen.util 8129 :info :cas [9 [43 12]] :RequestCanceledException INFO [2019-12-17 12:35:42,745] jepsen worker 29 - jepsen.util 8219 :invoke :read [9 nil] INFO [2019-12-17 12:35:42,745] jepsen worker 47 - jepsen.util 10397 :invoke :read [15 nil] INFO [2019-12-17 12:35:42,756] jepsen worker 88 - jepsen.util 10078 :fail :cas [29 [40 29]] :ValueNotSwapFrom INFO [2019-12-17 12:35:42,756] jepsen worker 88 - jepsen.util 10078 :invoke :write [29 27] INFO [2019-12-17 12:35:42,759] jepsen worker 3 - jepsen.util 9633 :info :write [1 27] :RequestCanceledException INFO [2019-12-17 12:35:42,760] jepsen worker 3 - jepsen.util 9723 :invoke :write [1 6] INFO [2019-12-17 12:35:42,767] jepsen worker 89 - jepsen.util 9899 :fail :cas [29 [29 16]] :ValueNotSwapFrom INFO [2019-12-17 12:35:42,768] jepsen worker 89 - jepsen.util 9899 :invoke :write [29 20] INFO [2019-12-17 12:35:42,787] jepsen worker 12 - jepsen.util 10092 :fail :cas [4 [29 21]] :ValueNotSwapFrom INFO [2019-12-17 12:35:42,787] jepsen worker 12 - jepsen.util 10092 :invoke :write [4 38] INFO [2019-12-17 12:35:42,809] jepsen worker 13 - jepsen.util 9283 :ok :read [4 48] INFO [2019-12-17 12:35:42,809] jepsen worker 13 - jepsen.util 9283 :invoke :write [4 44] INFO [2019-12-17 12:35:42,816] jepsen worker 81 - jepsen.util 13041 :info :write [27 10] :RequestCanceledException INFO [2019-12-17 12:35:42,817] jepsen worker 81 - jepsen.util 13131 :invoke :cas [27 [5 35]] INFO [2019-12-17 12:35:42,845] jepsen worker 4 - jepsen.util 9184 :ok :read [1 39] INFO [2019-12-17 12:35:42,845] jepsen worker 64 - jepsen.util 7624 :fail :cas [21 [6 42]] :ValueNotSwapFrom INFO [2019-12-17 12:35:42,845] jepsen worker 4 - jepsen.util 9184 :invoke :cas [1 [40 48]] INFO [2019-12-17 12:35:42,846] jepsen worker 64 - jepsen.util 7624 :invoke :cas [21 [40 11]] INFO [2019-12-17 12:35:42,847] jepsen worker 72 - jepsen.util 9702 :ok :read [24 9] INFO [2019-12-17 12:35:42,848] jepsen worker 72 - jepsen.util 9702 :invoke :cas [24 [48 11]] INFO [2019-12-17 12:35:42,866] jepsen worker 47 - jepsen.util 10397 :ok :read [15 49] INFO [2019-12-17 12:35:42,866] jepsen worker 47 - jepsen.util 10397 :invoke :read [15 nil] INFO [2019-12-17 12:35:42,867] jepsen worker 18 - jepsen.util 10098 :ok :read [6 25] INFO [2019-12-17 12:35:42,867] jepsen worker 18 - jepsen.util 10098 :invoke :cas [6 [2 3]] INFO [2019-12-17 12:35:42,876] jepsen worker 87 - jepsen.util 9537 :ok :read [29 38] INFO [2019-12-17 12:35:42,876] jepsen worker 87 - jepsen.util 9537 :invoke :write [29 48] INFO [2019-12-17 12:35:42,887] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":56786,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:35:42,887] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":56786,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:35:42,887] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":56788,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:35:42,887] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":56788,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:35:42,898] jepsen worker 39 - jepsen.util 12999 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:35:42,898] jepsen worker 39 - jepsen.util 12999 :invoke :write [13 40] INFO [2019-12-17 12:35:42,911] jepsen worker 14 - jepsen.util 9644 :ok :write [4 1] INFO [2019-12-17 12:35:42,911] jepsen worker 14 - jepsen.util 9644 :invoke :read [4 nil] INFO [2019-12-17 12:35:42,920] jepsen worker 88 - jepsen.util 10078 :ok :write [29 27] INFO [2019-12-17 12:35:42,921] jepsen worker 88 - jepsen.util 10078 :invoke :write [29 5] INFO [2019-12-17 12:35:42,930] jepsen worker 31 - jepsen.util 8851 :fail :cas [10 [26 38]] :ValueNotSwapFrom INFO [2019-12-17 12:35:42,930] jepsen worker 31 - jepsen.util 8851 :invoke :cas [10 [5 30]] INFO [2019-12-17 12:35:42,930] jepsen worker 5 - jepsen.util 9185 :info :write [1 48] :RequestCanceledException INFO [2019-12-17 12:35:42,930] jepsen worker 48 - jepsen.util 12288 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:35:42,931] jepsen worker 48 - jepsen.util 12288 :invoke :read [16 nil] INFO [2019-12-17 12:35:42,931] jepsen worker 5 - jepsen.util 9275 :invoke :cas [1 [29 2]] INFO [2019-12-17 12:35:42,981] jepsen worker 10 - jepsen.util 9910 :fail :cas [3 [19 29]] :ValueNotSwapFrom INFO [2019-12-17 12:35:42,981] jepsen worker 10 - jepsen.util 9910 :invoke :read [3 nil] INFO [2019-12-17 12:35:43,004] jepsen worker 13 - jepsen.util 9283 :ok :write [4 44] INFO [2019-12-17 12:35:43,004] jepsen worker 13 - jepsen.util 9283 :invoke :read [4 nil] INFO [2019-12-17 12:35:43,009] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":56800,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:35:43,009] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":56800,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:35:43,009] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":56803,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:35:43,009] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":56803,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:35:43,009] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":56804,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:35:43,009] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":56804,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:35:43,019] jepsen worker 47 - jepsen.util 10397 :ok :read [15 49] INFO [2019-12-17 12:35:43,019] jepsen worker 47 - jepsen.util 10397 :invoke :write [15 17] INFO [2019-12-17 12:35:43,022] jepsen worker 46 - jepsen.util 9766 :ok :read [15 49] INFO [2019-12-17 12:35:43,022] jepsen worker 46 - jepsen.util 9766 :invoke :cas [15 [6 26]] INFO [2019-12-17 12:35:43,027] jepsen worker 18 - jepsen.util 10098 :fail :cas [6 [2 3]] :ValueNotSwapFrom INFO [2019-12-17 12:35:43,027] jepsen worker 18 - jepsen.util 10098 :invoke :write [6 22] INFO [2019-12-17 12:35:43,031] jepsen worker 38 - jepsen.util 10388 :info :write [12 44] :RequestCanceledException INFO [2019-12-17 12:35:43,032] jepsen worker 38 - jepsen.util 10478 :invoke :write [12 27] INFO [2019-12-17 12:35:43,042] jepsen worker 52 - jepsen.util 9772 :ok :write [17 20] INFO [2019-12-17 12:35:43,043] jepsen worker 52 - jepsen.util 9772 :invoke :cas [17 [24 20]] INFO [2019-12-17 12:35:43,044] jepsen worker 31 - jepsen.util 8851 :fail :cas [10 [5 30]] :ValueNotSwapFrom INFO [2019-12-17 12:35:43,044] jepsen worker 31 - jepsen.util 8851 :invoke :write [10 18] INFO [2019-12-17 12:35:43,062] jepsen worker 55 - jepsen.util 10495 :info :write [18 47] :RequestCanceledException INFO [2019-12-17 12:35:43,063] jepsen worker 55 - jepsen.util 10585 :invoke :read [18 nil] INFO [2019-12-17 12:35:43,073] jepsen worker 74 - jepsen.util 9614 :ok :write [24 42] INFO [2019-12-17 12:35:43,073] jepsen worker 74 - jepsen.util 9614 :invoke :write [24 12] INFO [2019-12-17 12:35:43,087] jepsen worker 70 - jepsen.util 10600 :ok :write [23 0] INFO [2019-12-17 12:35:43,087] jepsen worker 70 - jepsen.util 10600 :invoke :write [23 45] INFO [2019-12-17 12:35:43,097] jepsen worker 77 - jepsen.util 14117 :info :cas [25 [39 31]] :RequestCanceledException INFO [2019-12-17 12:35:43,098] jepsen worker 77 - jepsen.util 14207 :invoke :cas [25 [30 24]] INFO [2019-12-17 12:35:43,121] jepsen worker 28 - jepsen.util 8128 :ok :write [9 31] INFO [2019-12-17 12:35:43,121] jepsen worker 28 - jepsen.util 8128 :invoke :write [9 31] INFO [2019-12-17 12:35:43,122] jepsen worker 52 - jepsen.util 9772 :fail :cas [17 [24 20]] :ValueNotSwapFrom INFO [2019-12-17 12:35:43,122] jepsen worker 52 - jepsen.util 9772 :invoke :read [17 nil] INFO [2019-12-17 12:35:43,131] jepsen worker 33 - jepsen.util 13353 :info :cas [11 [2 18]] :RequestCanceledException INFO [2019-12-17 12:35:43,132] jepsen worker 33 - jepsen.util 13443 :invoke :cas [11 [7 32]] INFO [2019-12-17 12:35:43,141] jepsen worker 64 - jepsen.util 7624 :fail :cas [21 [40 11]] :ValueNotSwapFrom INFO [2019-12-17 12:35:43,141] jepsen worker 64 - jepsen.util 7624 :invoke :cas [21 [18 32]] INFO [2019-12-17 12:35:43,155] jepsen worker 55 - jepsen.util 10585 :ok :read [18 46] INFO [2019-12-17 12:35:43,155] jepsen worker 55 - jepsen.util 10585 :invoke :read [18 nil] INFO [2019-12-17 12:35:43,162] jepsen worker 54 - jepsen.util 10854 :fail :cas [18 [47 39]] :ValueNotSwapFrom INFO [2019-12-17 12:35:43,162] jepsen worker 54 - jepsen.util 10854 :invoke :read [18 nil] INFO [2019-12-17 12:35:43,168] jepsen worker 63 - jepsen.util 8793 :ok :write [21 22] INFO [2019-12-17 12:35:43,168] jepsen worker 63 - jepsen.util 8793 :invoke :read [21 nil] INFO [2019-12-17 12:35:43,182] jepsen worker 52 - jepsen.util 9772 :ok :read [17 20] INFO [2019-12-17 12:35:43,182] jepsen worker 52 - jepsen.util 9772 :invoke :write [17 48] INFO [2019-12-17 12:35:43,184] jepsen worker 46 - jepsen.util 9766 :fail :cas [15 [6 26]] :ValueNotSwapFrom INFO [2019-12-17 12:35:43,184] jepsen worker 46 - jepsen.util 9766 :invoke :read [15 nil] INFO [2019-12-17 12:35:43,194] jepsen worker 63 - jepsen.util 8793 :ok :read [21 22] INFO [2019-12-17 12:35:43,194] jepsen worker 63 - jepsen.util 8793 :invoke :cas [21 [43 24]] INFO [2019-12-17 12:35:43,198] jepsen worker 30 - jepsen.util 9930 :info :cas [10 [30 47]] :RequestCanceledException INFO [2019-12-17 12:35:43,199] jepsen worker 30 - jepsen.util 10020 :invoke :read [10 nil] INFO [2019-12-17 12:35:43,211] jepsen worker 55 - jepsen.util 10585 :ok :read [18 46] INFO [2019-12-17 12:35:43,211] jepsen worker 55 - jepsen.util 10585 :invoke :read [18 nil] INFO [2019-12-17 12:35:43,215] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":56826,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:35:43,215] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":56826,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:35:43,215] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":56827,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:35:43,215] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":56827,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:35:43,223] jepsen worker 12 - jepsen.util 10092 :ok :write [4 38] INFO [2019-12-17 12:35:43,223] jepsen worker 12 - jepsen.util 10092 :invoke :read [4 nil] INFO [2019-12-17 12:35:43,225] jepsen worker 78 - jepsen.util 9798 :ok :write [26 7] INFO [2019-12-17 12:35:43,225] jepsen worker 78 - jepsen.util 9798 :invoke :write [26 26] INFO [2019-12-17 12:35:43,231] jepsen worker 59 - jepsen.util 12659 :info :cas [19 [44 35]] :RequestCanceledException INFO [2019-12-17 12:35:43,232] jepsen worker 59 - jepsen.util 12749 :invoke :cas [19 [2 29]] INFO [2019-12-17 12:35:43,237] jepsen worker 46 - jepsen.util 9766 :ok :read [15 49] INFO [2019-12-17 12:35:43,237] jepsen worker 46 - jepsen.util 9766 :invoke :read [15 nil] INFO [2019-12-17 12:35:43,244] jepsen worker 30 - jepsen.util 10020 :ok :read [10 35] INFO [2019-12-17 12:35:43,244] jepsen worker 30 - jepsen.util 10020 :invoke :write [10 29] INFO [2019-12-17 12:35:43,245] jepsen worker 53 - jepsen.util 9323 :info :write [17 4] :RequestCanceledException INFO [2019-12-17 12:35:43,246] jepsen worker 53 - jepsen.util 9413 :invoke :cas [17 [3 32]] INFO [2019-12-17 12:35:43,264] jepsen worker 28 - jepsen.util 8128 :ok :write [9 31] INFO [2019-12-17 12:35:43,265] jepsen worker 28 - jepsen.util 8128 :invoke :cas [9 [44 48]] INFO [2019-12-17 12:35:43,271] jepsen worker 63 - jepsen.util 8793 :fail :cas [21 [43 24]] :ValueNotSwapFrom INFO [2019-12-17 12:35:43,271] jepsen worker 63 - jepsen.util 8793 :invoke :read [21 nil] INFO [2019-12-17 12:35:43,280] jepsen worker 64 - jepsen.util 7624 :fail :cas [21 [18 32]] :ValueNotSwapFrom INFO [2019-12-17 12:35:43,280] jepsen worker 64 - jepsen.util 7624 :invoke :cas [21 [37 46]] INFO [2019-12-17 12:35:43,286] jepsen worker 4 - jepsen.util 9184 :fail :cas [1 [40 48]] :ValueNotSwapFrom INFO [2019-12-17 12:35:43,286] jepsen worker 4 - jepsen.util 9184 :invoke :write [1 6] INFO [2019-12-17 12:35:43,296] jepsen worker 46 - jepsen.util 9766 :ok :read [15 49] INFO [2019-12-17 12:35:43,296] jepsen worker 46 - jepsen.util 9766 :invoke :read [15 nil] INFO [2019-12-17 12:35:43,315] jepsen worker 50 - jepsen.util 14180 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:35:43,315] jepsen worker 50 - jepsen.util 14180 :invoke :read [16 nil] INFO [2019-12-17 12:35:43,324] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":56837,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:35:43,325] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":56837,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:35:43,325] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":56839,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:35:43,325] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":56839,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:35:43,330] jepsen worker 8 - jepsen.util 12968 :info :write [2 19] :RequestCanceledException INFO [2019-12-17 12:35:43,330] jepsen worker 67 - jepsen.util 13297 :info :cas [22 [17 15]] :RequestCanceledException INFO [2019-12-17 12:35:43,330] jepsen worker 19 - jepsen.util 10009 :info :write [6 19] :RequestCanceledException INFO [2019-12-17 12:35:43,331] jepsen worker 8 - jepsen.util 13058 :invoke :write [2 16] INFO [2019-12-17 12:35:43,331] jepsen worker 84 - jepsen.util 13044 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:35:43,331] jepsen worker 19 - jepsen.util 10099 :invoke :read [6 nil] INFO [2019-12-17 12:35:43,331] jepsen worker 84 - jepsen.util 13044 :invoke :cas [28 [2 12]] INFO [2019-12-17 12:35:43,331] jepsen worker 67 - jepsen.util 13387 :invoke :cas [22 [35 23]] INFO [2019-12-17 12:35:43,341] jepsen worker 28 - jepsen.util 8128 :fail :cas [9 [44 48]] :ValueNotSwapFrom INFO [2019-12-17 12:35:43,341] jepsen worker 28 - jepsen.util 8128 :invoke :cas [9 [14 41]] INFO [2019-12-17 12:35:43,345] jepsen worker 21 - jepsen.util 12981 :info :write [7 41] :RequestCanceledException INFO [2019-12-17 12:35:43,345] jepsen worker 22 - jepsen.util 13072 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:35:43,345] jepsen worker 41 - jepsen.util 14441 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:35:43,345] jepsen worker 22 - jepsen.util 13072 :invoke :read [7 nil] INFO [2019-12-17 12:35:43,345] jepsen worker 41 - jepsen.util 14441 :invoke :cas [13 [17 8]] INFO [2019-12-17 12:35:43,346] jepsen worker 21 - jepsen.util 13071 :invoke :read [7 nil] INFO [2019-12-17 12:35:43,356] jepsen worker 63 - jepsen.util 8793 :ok :read [21 22] INFO [2019-12-17 12:35:43,357] jepsen worker 63 - jepsen.util 8793 :invoke :cas [21 [17 43]] INFO [2019-12-17 12:35:43,362] jepsen worker 34 - jepsen.util 13444 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:35:43,362] jepsen worker 2 - jepsen.util 13232 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:35:43,362] jepsen worker 56 - jepsen.util 10316 :info :write [18 47] :RequestCanceledException INFO [2019-12-17 12:35:43,362] jepsen worker 34 - jepsen.util 13444 :invoke :cas [11 [46 31]] INFO [2019-12-17 12:35:43,363] jepsen worker 2 - jepsen.util 13232 :invoke :read [0 nil] INFO [2019-12-17 12:35:43,364] jepsen worker 56 - jepsen.util 10406 :invoke :cas [18 [5 10]] INFO [2019-12-17 12:35:43,374] jepsen worker 36 - jepsen.util 10566 :ok :read [12 44] INFO [2019-12-17 12:35:43,374] jepsen worker 36 - jepsen.util 10566 :invoke :write [12 27] INFO [2019-12-17 12:35:43,393] jepsen worker 64 - jepsen.util 7624 :fail :cas [21 [37 46]] :ValueNotSwapFrom INFO [2019-12-17 12:35:43,393] jepsen worker 64 - jepsen.util 7624 :invoke :cas [21 [27 22]] INFO [2019-12-17 12:35:43,395] jepsen worker 15 - jepsen.util 12615 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:35:43,395] jepsen worker 86 - jepsen.util 14216 :info :write [28 12] :RequestCanceledException INFO [2019-12-17 12:35:43,395] jepsen worker 15 - jepsen.util 12615 :invoke :read [5 nil] INFO [2019-12-17 12:35:43,396] jepsen worker 86 - jepsen.util 14306 :invoke :cas [28 [37 31]] INFO [2019-12-17 12:35:43,401] jepsen worker 46 - jepsen.util 9766 :ok :read [15 49] INFO [2019-12-17 12:35:43,401] jepsen worker 46 - jepsen.util 9766 :invoke :write [15 1] INFO [2019-12-17 12:35:43,430] jepsen worker 16 - jepsen.util 14596 :info :cas [5 [2 26]] :RequestCanceledException INFO [2019-12-17 12:35:43,430] jepsen worker 11 - jepsen.util 9641 :info :write [3 16] :RequestCanceledException INFO [2019-12-17 12:35:43,431] jepsen worker 16 - jepsen.util 14686 :invoke :write [5 38] INFO [2019-12-17 12:35:43,431] jepsen worker 11 - jepsen.util 9731 :invoke :cas [3 [4 44]] INFO [2019-12-17 12:35:43,441] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":56872,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:35:43,441] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":56872,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:35:43,443] jepsen worker 13 - jepsen.util 9283 :ok :read [4 38] INFO [2019-12-17 12:35:43,443] jepsen worker 13 - jepsen.util 9283 :invoke :read [4 nil] INFO [2019-12-17 12:35:43,445] jepsen worker 65 - jepsen.util 8345 :info :write [21 3] :RequestCanceledException INFO [2019-12-17 12:35:43,446] jepsen worker 65 - jepsen.util 8435 :invoke :cas [21 [17 47]] INFO [2019-12-17 12:35:43,453] jepsen worker 63 - jepsen.util 8793 :fail :cas [21 [17 43]] :ValueNotSwapFrom INFO [2019-12-17 12:35:43,453] jepsen worker 63 - jepsen.util 8793 :invoke :write [21 17] INFO [2019-12-17 12:35:43,455] jepsen worker 28 - jepsen.util 8128 :fail :cas [9 [14 41]] :ValueNotSwapFrom INFO [2019-12-17 12:35:43,455] jepsen worker 28 - jepsen.util 8128 :invoke :cas [9 [20 2]] INFO [2019-12-17 12:35:43,461] jepsen worker 47 - jepsen.util 10397 :ok :write [15 17] INFO [2019-12-17 12:35:43,461] jepsen worker 18 - jepsen.util 10098 :ok :write [6 22] INFO [2019-12-17 12:35:43,461] jepsen worker 47 - jepsen.util 10397 :invoke :write [15 1] INFO [2019-12-17 12:35:43,461] jepsen worker 18 - jepsen.util 10098 :invoke :cas [6 [8 41]] INFO [2019-12-17 12:35:43,484] jepsen worker 37 - jepsen.util 9307 :fail :cas [12 [37 26]] :ValueNotSwapFrom INFO [2019-12-17 12:35:43,484] jepsen worker 37 - jepsen.util 9307 :invoke :write [12 13] INFO [2019-12-17 12:35:43,531] jepsen worker 14 - jepsen.util 9644 :ok :read [4 38] INFO [2019-12-17 12:35:43,531] jepsen worker 14 - jepsen.util 9644 :invoke :cas [4 [10 46]] INFO [2019-12-17 12:35:43,554] jepsen worker 64 - jepsen.util 7624 :fail :cas [21 [27 22]] :ValueNotSwapFrom INFO [2019-12-17 12:35:43,554] jepsen worker 64 - jepsen.util 7624 :invoke :write [21 21] INFO [2019-12-17 12:35:43,560] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:35:43,560] jepsen nemesis - couchbase.util Rebalance check count: 445 INFO [2019-12-17 12:35:43,561] jepsen nemesis - couchbase.util Rebalance stuck count: 426 INFO [2019-12-17 12:35:43,562] jepsen worker 43 - jepsen.util 13273 :info :cas [14 [34 4]] :RequestCanceledException INFO [2019-12-17 12:35:43,562] jepsen worker 80 - jepsen.util 9530 :info :write [26 9] :RequestCanceledException INFO [2019-12-17 12:35:43,564] jepsen worker 80 - jepsen.util 9620 :invoke :read [26 nil] INFO [2019-12-17 12:35:43,564] jepsen worker 43 - jepsen.util 13363 :invoke :cas [14 [22 22]] INFO [2019-12-17 12:35:43,568] jepsen worker 19 - jepsen.util 10099 :ok :read [6 22] INFO [2019-12-17 12:35:43,568] jepsen worker 19 - jepsen.util 10099 :invoke :read [6 nil] INFO [2019-12-17 12:35:43,597] jepsen worker 42 - jepsen.util 14352 :info :write [14 24] :RequestCanceledException INFO [2019-12-17 12:35:43,598] jepsen worker 42 - jepsen.util 14442 :invoke :read [14 nil] INFO [2019-12-17 12:35:43,606] jepsen worker 45 - jepsen.util 10125 :info :write [15 29] :RequestCanceledException INFO [2019-12-17 12:35:43,607] jepsen worker 45 - jepsen.util 10215 :invoke :read [15 nil] INFO [2019-12-17 12:35:43,609] jepsen worker 18 - jepsen.util 10098 :fail :cas [6 [8 41]] :ValueNotSwapFrom INFO [2019-12-17 12:35:43,609] jepsen worker 18 - jepsen.util 10098 :invoke :read [6 nil] INFO [2019-12-17 12:35:43,631] jepsen worker 60 - jepsen.util 14370 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:35:43,631] jepsen worker 17 - jepsen.util 13067 :info :cas [5 [27 29]] :RequestCanceledException INFO [2019-12-17 12:35:43,631] jepsen worker 60 - jepsen.util 14370 :invoke :read [20 nil] INFO [2019-12-17 12:35:43,632] jepsen worker 17 - jepsen.util 13157 :invoke :cas [5 [27 4]] INFO [2019-12-17 12:35:43,648] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":56897,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:35:43,648] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":56897,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:35:43,648] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":56899,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:35:43,648] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":56899,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:35:43,651] jepsen worker 11 - jepsen.util 9731 :fail :cas [3 [4 44]] :ValueNotSwapFrom INFO [2019-12-17 12:35:43,652] jepsen worker 11 - jepsen.util 9731 :invoke :cas [3 [25 38]] INFO [2019-12-17 12:35:43,653] jepsen worker 28 - jepsen.util 8128 :fail :cas [9 [20 2]] :ValueNotSwapFrom INFO [2019-12-17 12:35:43,653] jepsen worker 28 - jepsen.util 8128 :invoke :write [9 36] INFO [2019-12-17 12:35:43,655] jepsen worker 55 - jepsen.util 10585 :ok :read [18 46] INFO [2019-12-17 12:35:43,655] jepsen worker 55 - jepsen.util 10585 :invoke :write [18 5] INFO [2019-12-17 12:35:43,662] jepsen worker 57 - jepsen.util 12927 :info :cas [19 [17 34]] :RequestCanceledException INFO [2019-12-17 12:35:43,663] jepsen worker 57 - jepsen.util 13017 :invoke :write [19 38] INFO [2019-12-17 12:35:43,670] jepsen worker 80 - jepsen.util 9620 :ok :read [26 7] INFO [2019-12-17 12:35:43,670] jepsen worker 80 - jepsen.util 9620 :invoke :cas [26 [39 23]] INFO [2019-12-17 12:35:43,674] jepsen worker 5 - jepsen.util 9275 :fail :cas [1 [29 2]] :ValueNotSwapFrom INFO [2019-12-17 12:35:43,674] jepsen worker 5 - jepsen.util 9275 :invoke :cas [1 [24 34]] INFO [2019-12-17 12:35:43,716] jepsen worker 80 - jepsen.util 9620 :fail :cas [26 [39 23]] :ValueNotSwapFrom INFO [2019-12-17 12:35:43,716] jepsen worker 80 - jepsen.util 9620 :invoke :write [26 38] INFO [2019-12-17 12:35:43,726] jepsen worker 18 - jepsen.util 10098 :ok :read [6 22] INFO [2019-12-17 12:35:43,726] jepsen worker 18 - jepsen.util 10098 :invoke :cas [6 [9 28]] INFO [2019-12-17 12:35:43,734] jepsen worker 64 - jepsen.util 7624 :ok :write [21 21] INFO [2019-12-17 12:35:43,734] jepsen worker 64 - jepsen.util 7624 :invoke :read [21 nil] INFO [2019-12-17 12:35:43,741] jepsen worker 56 - jepsen.util 10406 :fail :cas [18 [5 10]] :ValueNotSwapFrom INFO [2019-12-17 12:35:43,741] jepsen worker 56 - jepsen.util 10406 :invoke :cas [18 [35 37]] INFO [2019-12-17 12:35:43,746] jepsen worker 19 - jepsen.util 10099 :ok :read [6 22] INFO [2019-12-17 12:35:43,746] jepsen worker 19 - jepsen.util 10099 :invoke :cas [6 [9 33]] INFO [2019-12-17 12:35:43,765] jepsen worker 11 - jepsen.util 9731 :fail :cas [3 [25 38]] :ValueNotSwapFrom INFO [2019-12-17 12:35:43,765] jepsen worker 11 - jepsen.util 9731 :invoke :write [3 44] INFO [2019-12-17 12:35:43,780] jepsen worker 74 - jepsen.util 9614 :ok :write [24 12] INFO [2019-12-17 12:35:43,780] jepsen worker 74 - jepsen.util 9614 :invoke :cas [24 [28 37]] INFO [2019-12-17 12:35:43,782] jepsen worker 18 - jepsen.util 10098 :fail :cas [6 [9 28]] :ValueNotSwapFrom INFO [2019-12-17 12:35:43,782] jepsen worker 18 - jepsen.util 10098 :invoke :read [6 nil] INFO [2019-12-17 12:35:43,795] jepsen worker 19 - jepsen.util 10099 :fail :cas [6 [9 33]] :ValueNotSwapFrom INFO [2019-12-17 12:35:43,795] jepsen worker 19 - jepsen.util 10099 :invoke :cas [6 [11 19]] INFO [2019-12-17 12:35:43,795] jepsen worker 30 - jepsen.util 10020 :ok :write [10 29] INFO [2019-12-17 12:35:43,795] jepsen worker 30 - jepsen.util 10020 :invoke :cas [10 [40 48]] INFO [2019-12-17 12:35:43,804] jepsen worker 65 - jepsen.util 8435 :fail :cas [21 [17 47]] :ValueNotSwapFrom INFO [2019-12-17 12:35:43,804] jepsen worker 65 - jepsen.util 8435 :invoke :cas [21 [26 22]] INFO [2019-12-17 12:35:43,808] jepsen worker 5 - jepsen.util 9275 :fail :cas [1 [24 34]] :ValueNotSwapFrom INFO [2019-12-17 12:35:43,809] jepsen worker 5 - jepsen.util 9275 :invoke :cas [1 [22 45]] INFO [2019-12-17 12:35:43,831] jepsen worker 6 - jepsen.util 13956 :info :cas [2 [5 7]] :RequestCanceledException INFO [2019-12-17 12:35:43,832] jepsen worker 6 - jepsen.util 14046 :invoke :read [2 nil] INFO [2019-12-17 12:35:43,834] jepsen worker 74 - jepsen.util 9614 :fail :cas [24 [28 37]] :ValueNotSwapFrom INFO [2019-12-17 12:35:43,834] jepsen worker 74 - jepsen.util 9614 :invoke :write [24 22] INFO [2019-12-17 12:35:43,862] jepsen worker 24 - jepsen.util 12264 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:35:43,862] jepsen worker 24 - jepsen.util 12264 :invoke :cas [8 [42 26]] INFO [2019-12-17 12:35:43,865] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":56922,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:35:43,866] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":56922,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:35:43,866] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":56926,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:35:43,866] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":56926,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:35:43,875] jepsen worker 28 - jepsen.util 8128 :ok :write [9 36] INFO [2019-12-17 12:35:43,876] jepsen worker 28 - jepsen.util 8128 :invoke :read [9 nil] INFO [2019-12-17 12:35:43,893] jepsen worker 76 - jepsen.util 13576 :info :write [25 29] :RequestCanceledException INFO [2019-12-17 12:35:43,895] jepsen worker 76 - jepsen.util 13666 :invoke :write [25 4] INFO [2019-12-17 12:35:43,902] jepsen worker 30 - jepsen.util 10020 :fail :cas [10 [40 48]] :ValueNotSwapFrom INFO [2019-12-17 12:35:43,902] jepsen worker 30 - jepsen.util 10020 :invoke :cas [10 [42 35]] INFO [2019-12-17 12:35:43,904] jepsen worker 65 - jepsen.util 8435 :fail :cas [21 [26 22]] :ValueNotSwapFrom INFO [2019-12-17 12:35:43,904] jepsen worker 65 - jepsen.util 8435 :invoke :read [21 nil] INFO [2019-12-17 12:35:43,912] jepsen worker 18 - jepsen.util 10098 :ok :read [6 22] INFO [2019-12-17 12:35:43,913] jepsen worker 18 - jepsen.util 10098 :invoke :write [6 48] INFO [2019-12-17 12:35:43,930] jepsen worker 44 - jepsen.util 13634 :info :cas [14 [26 8]] :RequestCanceledException INFO [2019-12-17 12:35:43,931] jepsen worker 44 - jepsen.util 13724 :invoke :cas [14 [24 7]] INFO [2019-12-17 12:35:43,943] jepsen worker 12 - jepsen.util 10092 :ok :read [4 38] INFO [2019-12-17 12:35:43,943] jepsen worker 12 - jepsen.util 10092 :invoke :write [4 24] INFO [2019-12-17 12:35:43,956] jepsen worker 30 - jepsen.util 10020 :fail :cas [10 [42 35]] :ValueNotSwapFrom INFO [2019-12-17 12:35:43,956] jepsen worker 30 - jepsen.util 10020 :invoke :write [10 19] INFO [2019-12-17 12:35:43,957] jepsen worker 28 - jepsen.util 8128 :ok :read [9 36] INFO [2019-12-17 12:35:43,957] jepsen worker 28 - jepsen.util 8128 :invoke :cas [9 [30 3]] INFO [2019-12-17 12:35:43,962] jepsen worker 80 - jepsen.util 9620 :ok :write [26 38] INFO [2019-12-17 12:35:43,963] jepsen worker 80 - jepsen.util 9620 :invoke :cas [26 [8 17]] INFO [2019-12-17 12:35:43,981] jepsen worker 87 - jepsen.util 9537 :ok :write [29 48] INFO [2019-12-17 12:35:43,982] jepsen worker 87 - jepsen.util 9537 :invoke :write [29 43] INFO [2019-12-17 12:35:43,988] jepsen worker 53 - jepsen.util 9413 :fail :cas [17 [3 32]] :ValueNotSwapFrom INFO [2019-12-17 12:35:43,988] jepsen worker 53 - jepsen.util 9413 :invoke :read [17 nil] INFO [2019-12-17 12:35:44,023] jepsen worker 3 - jepsen.util 9723 :ok :write [1 6] INFO [2019-12-17 12:35:44,023] jepsen worker 3 - jepsen.util 9723 :invoke :cas [1 [1 22]] INFO [2019-12-17 12:35:44,028] jepsen worker 35 - jepsen.util 13265 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:35:44,028] jepsen worker 9 - jepsen.util 10539 :info :write [3 44] :RequestCanceledException INFO [2019-12-17 12:35:44,029] jepsen worker 51 - jepsen.util 9141 :info :write [17 18] :RequestCanceledException INFO [2019-12-17 12:35:44,029] jepsen worker 35 - jepsen.util 13265 :invoke :cas [11 [37 43]] INFO [2019-12-17 12:35:44,030] jepsen worker 9 - jepsen.util 10629 :invoke :cas [3 [47 25]] INFO [2019-12-17 12:35:44,030] jepsen worker 51 - jepsen.util 9231 :invoke :read [17 nil] INFO [2019-12-17 12:35:44,039] jepsen worker 28 - jepsen.util 8128 :fail :cas [9 [30 3]] :ValueNotSwapFrom INFO [2019-12-17 12:35:44,039] jepsen worker 28 - jepsen.util 8128 :invoke :read [9 nil] INFO [2019-12-17 12:35:44,044] jepsen worker 62 - jepsen.util 13832 :info :cas [20 [46 20]] :RequestCanceledException INFO [2019-12-17 12:35:44,045] jepsen worker 62 - jepsen.util 13922 :invoke :read [20 nil] INFO [2019-12-17 12:35:44,058] jepsen worker 28 - jepsen.util 8128 :ok :read [9 36] INFO [2019-12-17 12:35:44,059] jepsen worker 28 - jepsen.util 8128 :invoke :read [9 nil] INFO [2019-12-17 12:35:44,060] jepsen worker 14 - jepsen.util 9644 :fail :cas [4 [10 46]] :ValueNotSwapFrom INFO [2019-12-17 12:35:44,060] jepsen worker 14 - jepsen.util 9644 :invoke :cas [4 [9 28]] INFO [2019-12-17 12:35:44,084] jepsen worker 28 - jepsen.util 8128 :ok :read [9 36] INFO [2019-12-17 12:35:44,084] jepsen worker 28 - jepsen.util 8128 :invoke :cas [9 [29 15]] INFO [2019-12-17 12:35:44,100] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":56939,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:35:44,101] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":56939,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:35:44,101] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":56940,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:35:44,101] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":56940,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:35:44,105] jepsen worker 28 - jepsen.util 8128 :fail :cas [9 [29 15]] :ValueNotSwapFrom INFO [2019-12-17 12:35:44,105] jepsen worker 28 - jepsen.util 8128 :invoke :cas [9 [19 30]] INFO [2019-12-17 12:35:44,125] jepsen worker 28 - jepsen.util 8128 :fail :cas [9 [19 30]] :ValueNotSwapFrom INFO [2019-12-17 12:35:44,125] jepsen worker 28 - jepsen.util 8128 :invoke :read [9 nil] INFO [2019-12-17 12:35:44,131] jepsen worker 23 - jepsen.util 13433 :info :cas [7 [2 6]] :RequestCanceledException INFO [2019-12-17 12:35:44,132] jepsen worker 23 - jepsen.util 13523 :invoke :cas [7 [8 10]] INFO [2019-12-17 12:35:44,150] jepsen worker 28 - jepsen.util 8128 :ok :read [9 36] INFO [2019-12-17 12:35:44,150] jepsen worker 28 - jepsen.util 8128 :invoke :cas [9 [32 9]] INFO [2019-12-17 12:35:44,161] jepsen worker 20 - jepsen.util 9470 :info :write [6 8] :RequestCanceledException INFO [2019-12-17 12:35:44,162] jepsen worker 20 - jepsen.util 9560 :invoke :cas [6 [8 41]] INFO [2019-12-17 12:35:44,189] jepsen worker 28 - jepsen.util 8128 :fail :cas [9 [32 9]] :ValueNotSwapFrom INFO [2019-12-17 12:35:44,190] jepsen worker 28 - jepsen.util 8128 :invoke :write [9 29] INFO [2019-12-17 12:35:44,195] jepsen worker 85 - jepsen.util 14215 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:35:44,195] jepsen worker 85 - jepsen.util 14215 :invoke :cas [28 [46 22]] INFO [2019-12-17 12:35:44,200] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":56960,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:35:44,200] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":56960,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:35:44,200] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":56962,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:35:44,200] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":56962,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:35:44,201] jepsen worker 20 - jepsen.util 9560 :fail :cas [6 [8 41]] :ValueNotSwapFrom INFO [2019-12-17 12:35:44,201] jepsen worker 20 - jepsen.util 9560 :invoke :write [6 23] INFO [2019-12-17 12:35:44,206] jepsen worker 83 - jepsen.util 13763 :info :write [27 18] :RequestCanceledException INFO [2019-12-17 12:35:44,207] jepsen worker 83 - jepsen.util 13853 :invoke :write [27 30] INFO [2019-12-17 12:35:44,207] jepsen worker 56 - jepsen.util 10406 :fail :cas [18 [35 37]] :ValueNotSwapFrom INFO [2019-12-17 12:35:44,208] jepsen worker 56 - jepsen.util 10406 :invoke :cas [18 [19 4]] INFO [2019-12-17 12:35:44,219] jepsen worker 80 - jepsen.util 9620 :fail :cas [26 [8 17]] :ValueNotSwapFrom INFO [2019-12-17 12:35:44,219] jepsen worker 80 - jepsen.util 9620 :invoke :read [26 nil] INFO [2019-12-17 12:35:44,235] jepsen worker 54 - jepsen.util 10854 :ok :read [18 46] INFO [2019-12-17 12:35:44,235] jepsen worker 54 - jepsen.util 10854 :invoke :cas [18 [15 39]] INFO [2019-12-17 12:35:44,239] jepsen worker 79 - jepsen.util 10069 :info :write [26 22] :RequestCanceledException INFO [2019-12-17 12:35:44,240] jepsen worker 79 - jepsen.util 10159 :invoke :write [26 32] INFO [2019-12-17 12:35:44,251] jepsen worker 28 - jepsen.util 8128 :ok :write [9 29] INFO [2019-12-17 12:35:44,252] jepsen worker 28 - jepsen.util 8128 :invoke :read [9 nil] INFO [2019-12-17 12:35:44,260] jepsen worker 80 - jepsen.util 9620 :ok :read [26 38] INFO [2019-12-17 12:35:44,260] jepsen worker 80 - jepsen.util 9620 :invoke :write [26 36] INFO [2019-12-17 12:35:44,302] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":56967,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:35:44,302] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":56967,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:35:44,302] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":56972,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:35:44,302] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":56972,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:35:44,302] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":56973,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:35:44,302] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":56973,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:35:44,314] jepsen worker 27 - jepsen.util 9387 :info :cas [9 [38 37]] :RequestCanceledException INFO [2019-12-17 12:35:44,314] jepsen worker 40 - jepsen.util 13720 :info :write [13 45] :RequestCanceledException INFO [2019-12-17 12:35:44,315] jepsen worker 40 - jepsen.util 13810 :invoke :write [13 34] INFO [2019-12-17 12:35:44,315] jepsen worker 27 - jepsen.util 9477 :invoke :read [9 nil] INFO [2019-12-17 12:35:44,330] jepsen worker 74 - jepsen.util 9614 :ok :write [24 22] INFO [2019-12-17 12:35:44,330] jepsen worker 74 - jepsen.util 9614 :invoke :cas [24 [21 42]] INFO [2019-12-17 12:35:44,330] jepsen worker 46 - jepsen.util 9766 :ok :write [15 1] INFO [2019-12-17 12:35:44,330] jepsen worker 46 - jepsen.util 9766 :invoke :write [15 42] INFO [2019-12-17 12:35:44,332] jepsen worker 18 - jepsen.util 10098 :ok :write [6 48] INFO [2019-12-17 12:35:44,332] jepsen worker 30 - jepsen.util 10020 :ok :write [10 19] INFO [2019-12-17 12:35:44,332] jepsen worker 18 - jepsen.util 10098 :invoke :read [6 nil] INFO [2019-12-17 12:35:44,332] jepsen worker 30 - jepsen.util 10020 :invoke :cas [10 [3 7]] INFO [2019-12-17 12:35:44,350] jepsen worker 53 - jepsen.util 9413 :ok :read [17 20] INFO [2019-12-17 12:35:44,350] jepsen worker 53 - jepsen.util 9413 :invoke :read [17 nil] INFO [2019-12-17 12:35:44,360] jepsen worker 49 - jepsen.util 13279 :info :cas [16 [36 11]] :RequestCanceledException INFO [2019-12-17 12:35:44,361] jepsen worker 49 - jepsen.util 13369 :invoke :write [16 14] INFO [2019-12-17 12:35:44,383] jepsen worker 27 - jepsen.util 9477 :ok :read [9 29] INFO [2019-12-17 12:35:44,384] jepsen worker 27 - jepsen.util 9477 :invoke :cas [9 [47 49]] INFO [2019-12-17 12:35:44,393] jepsen worker 0 - jepsen.util 14310 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:35:44,393] jepsen worker 0 - jepsen.util 14310 :invoke :cas [0 [39 49]] INFO [2019-12-17 12:35:44,407] jepsen worker 11 - jepsen.util 9731 :ok :write [3 44] INFO [2019-12-17 12:35:44,407] jepsen worker 11 - jepsen.util 9731 :invoke :read [3 nil] INFO [2019-12-17 12:35:44,407] jepsen worker 66 - jepsen.util 13116 :info :write [22 19] :RequestCanceledException INFO [2019-12-17 12:35:44,408] jepsen worker 66 - jepsen.util 13206 :invoke :read [22 nil] INFO [2019-12-17 12:35:44,421] jepsen worker 36 - jepsen.util 10566 :ok :write [12 27] INFO [2019-12-17 12:35:44,421] jepsen worker 36 - jepsen.util 10566 :invoke :cas [12 [7 35]] INFO [2019-12-17 12:35:44,424] jepsen worker 45 - jepsen.util 10215 :ok :read [15 1] INFO [2019-12-17 12:35:44,424] jepsen worker 45 - jepsen.util 10215 :invoke :read [15 nil] INFO [2019-12-17 12:35:44,432] jepsen worker 73 - jepsen.util 9253 :info :write [24 41] :RequestCanceledException INFO [2019-12-17 12:35:44,432] jepsen worker 26 - jepsen.util 13976 :info :write [8 30] :RequestCanceledException INFO [2019-12-17 12:35:44,433] jepsen worker 73 - jepsen.util 9343 :invoke :read [24 nil] INFO [2019-12-17 12:35:44,433] jepsen worker 26 - jepsen.util 14066 :invoke :cas [8 [39 36]] INFO [2019-12-17 12:35:44,438] jepsen worker 5 - jepsen.util 9275 :fail :cas [1 [22 45]] :ValueNotSwapFrom INFO [2019-12-17 12:35:44,438] jepsen worker 5 - jepsen.util 9275 :invoke :write [1 35] INFO [2019-12-17 12:35:44,444] jepsen worker 7 - jepsen.util 13507 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:35:44,444] jepsen worker 7 - jepsen.util 13507 :invoke :write [2 27] INFO [2019-12-17 12:35:44,448] jepsen worker 30 - jepsen.util 10020 :fail :cas [10 [3 7]] :ValueNotSwapFrom INFO [2019-12-17 12:35:44,448] jepsen worker 30 - jepsen.util 10020 :invoke :cas [10 [31 4]] INFO [2019-12-17 12:35:44,453] jepsen worker 80 - jepsen.util 9620 :ok :write [26 36] INFO [2019-12-17 12:35:44,453] jepsen worker 80 - jepsen.util 9620 :invoke :write [26 45] INFO [2019-12-17 12:35:44,462] jepsen worker 61 - jepsen.util 14281 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:35:44,462] jepsen worker 61 - jepsen.util 14281 :invoke :cas [20 [30 2]] INFO [2019-12-17 12:35:44,477] jepsen worker 36 - jepsen.util 10566 :fail :cas [12 [7 35]] :ValueNotSwapFrom INFO [2019-12-17 12:35:44,477] jepsen worker 36 - jepsen.util 10566 :invoke :write [12 23] INFO [2019-12-17 12:35:44,485] jepsen worker 51 - jepsen.util 9231 :ok :read [17 20] INFO [2019-12-17 12:35:44,485] jepsen worker 51 - jepsen.util 9231 :invoke :read [17 nil] INFO [2019-12-17 12:35:44,490] jepsen worker 18 - jepsen.util 10098 :ok :read [6 48] INFO [2019-12-17 12:35:44,490] jepsen worker 18 - jepsen.util 10098 :invoke :read [6 nil] INFO [2019-12-17 12:35:44,490] jepsen worker 74 - jepsen.util 9614 :fail :cas [24 [21 42]] :ValueNotSwapFrom INFO [2019-12-17 12:35:44,490] jepsen worker 74 - jepsen.util 9614 :invoke :write [24 2] INFO [2019-12-17 12:35:44,512] jepsen worker 14 - jepsen.util 9644 :fail :cas [4 [9 28]] :ValueNotSwapFrom INFO [2019-12-17 12:35:44,512] jepsen worker 14 - jepsen.util 9644 :invoke :write [4 14] INFO [2019-12-17 12:35:44,518] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":56998,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:35:44,519] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":56998,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:35:44,519] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":57002,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:35:44,519] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":57002,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:35:44,520] jepsen worker 11 - jepsen.util 9731 :ok :read [3 44] INFO [2019-12-17 12:35:44,520] jepsen worker 11 - jepsen.util 9731 :invoke :read [3 nil] INFO [2019-12-17 12:35:44,531] jepsen worker 75 - jepsen.util 12945 :info :cas [25 [19 41]] :RequestCanceledException INFO [2019-12-17 12:35:44,531] jepsen worker 32 - jepsen.util 9932 :fail :read [10 nil] :RequestTimeoutException INFO [2019-12-17 12:35:44,531] jepsen worker 1 - jepsen.util 13501 :info :write [0 17] :RequestCanceledException INFO [2019-12-17 12:35:44,531] jepsen worker 32 - jepsen.util 9932 :invoke :read [10 nil] INFO [2019-12-17 12:35:44,532] jepsen worker 75 - jepsen.util 13035 :invoke :read [25 nil] INFO [2019-12-17 12:35:44,532] jepsen worker 1 - jepsen.util 13591 :invoke :cas [0 [43 18]] INFO [2019-12-17 12:35:44,546] jepsen worker 56 - jepsen.util 10406 :fail :cas [18 [19 4]] :ValueNotSwapFrom INFO [2019-12-17 12:35:44,546] jepsen worker 56 - jepsen.util 10406 :invoke :cas [18 [11 5]] INFO [2019-12-17 12:35:44,547] jepsen worker 30 - jepsen.util 10020 :fail :cas [10 [31 4]] :ValueNotSwapFrom INFO [2019-12-17 12:35:44,547] jepsen worker 30 - jepsen.util 10020 :invoke :cas [10 [14 5]] INFO [2019-12-17 12:35:44,562] jepsen worker 58 - jepsen.util 13918 :info :cas [19 [46 40]] :RequestCanceledException INFO [2019-12-17 12:35:44,562] jepsen worker 68 - jepsen.util 12758 :info :write [22 5] :RequestCanceledException INFO [2019-12-17 12:35:44,563] jepsen worker 58 - jepsen.util 14008 :invoke :read [19 nil] INFO [2019-12-17 12:35:44,563] jepsen worker 68 - jepsen.util 12848 :invoke :write [22 37] INFO [2019-12-17 12:35:44,586] jepsen worker 9 - jepsen.util 10629 :fail :cas [3 [47 25]] :ValueNotSwapFrom INFO [2019-12-17 12:35:44,586] jepsen worker 9 - jepsen.util 10629 :invoke :read [3 nil] INFO [2019-12-17 12:35:44,589] jepsen worker 5 - jepsen.util 9275 :ok :write [1 35] INFO [2019-12-17 12:35:44,589] jepsen worker 5 - jepsen.util 9275 :invoke :read [1 nil] INFO [2019-12-17 12:35:44,598] jepsen worker 82 - jepsen.util 13852 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:35:44,598] jepsen worker 82 - jepsen.util 13852 :invoke :cas [27 [18 17]] INFO [2019-12-17 12:35:44,632] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:35:44,632] jepsen nemesis - couchbase.util Rebalance check count: 446 INFO [2019-12-17 12:35:44,632] jepsen nemesis - couchbase.util Rebalance stuck count: 427 INFO [2019-12-17 12:35:44,637] jepsen worker 88 - jepsen.util 10078 :ok :write [29 5] INFO [2019-12-17 12:35:44,638] jepsen worker 88 - jepsen.util 10078 :invoke :cas [29 [25 9]] INFO [2019-12-17 12:35:44,641] jepsen worker 69 - jepsen.util 9879 :info :write [23 29] :RequestCanceledException INFO [2019-12-17 12:35:44,644] jepsen worker 27 - jepsen.util 9477 :fail :cas [9 [47 49]] :ValueNotSwapFrom INFO [2019-12-17 12:35:44,645] jepsen worker 27 - jepsen.util 9477 :invoke :write [9 44] INFO [2019-12-17 12:35:44,652] jepsen worker 69 - jepsen.util 9969 :invoke :write [23 4] INFO [2019-12-17 12:35:44,657] jepsen worker 73 - jepsen.util 9343 :ok :read [24 22] INFO [2019-12-17 12:35:44,658] jepsen worker 73 - jepsen.util 9343 :invoke :cas [24 [43 23]] INFO [2019-12-17 12:35:44,660] jepsen worker 11 - jepsen.util 9731 :ok :read [3 44] INFO [2019-12-17 12:35:44,661] jepsen worker 11 - jepsen.util 9731 :invoke :read [3 nil] INFO [2019-12-17 12:35:44,662] jepsen worker 80 - jepsen.util 9620 :ok :write [26 45] INFO [2019-12-17 12:35:44,662] jepsen worker 80 - jepsen.util 9620 :invoke :cas [26 [34 44]] INFO [2019-12-17 12:35:44,697] jepsen worker 25 - jepsen.util 14335 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:35:44,698] jepsen worker 25 - jepsen.util 14335 :invoke :write [8 8] INFO [2019-12-17 12:35:44,708] jepsen worker 80 - jepsen.util 9620 :fail :cas [26 [34 44]] :ValueNotSwapFrom INFO [2019-12-17 12:35:44,709] jepsen worker 80 - jepsen.util 9620 :invoke :cas [26 [16 18]] INFO [2019-12-17 12:35:44,716] jepsen worker 54 - jepsen.util 10854 :fail :cas [18 [15 39]] :ValueNotSwapFrom INFO [2019-12-17 12:35:44,717] jepsen worker 54 - jepsen.util 10854 :invoke :read [18 nil] INFO [2019-12-17 12:35:44,734] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":57027,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:35:44,734] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":57027,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:35:44,734] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":57030,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:35:44,734] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":57030,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:35:44,745] jepsen worker 71 - jepsen.util 10511 :info :write [23 20] :RequestCanceledException INFO [2019-12-17 12:35:44,746] jepsen worker 71 - jepsen.util 10601 :invoke :cas [23 [9 3]] INFO [2019-12-17 12:35:44,749] jepsen worker 30 - jepsen.util 10020 :fail :cas [10 [14 5]] :ValueNotSwapFrom INFO [2019-12-17 12:35:44,749] jepsen worker 30 - jepsen.util 10020 :invoke :read [10 nil] INFO [2019-12-17 12:35:44,752] jepsen worker 3 - jepsen.util 9723 :fail :cas [1 [1 22]] :ValueNotSwapFrom INFO [2019-12-17 12:35:44,753] jepsen worker 3 - jepsen.util 9723 :invoke :read [1 nil] INFO [2019-12-17 12:35:44,753] jepsen worker 32 - jepsen.util 9932 :ok :read [10 19] INFO [2019-12-17 12:35:44,753] jepsen worker 32 - jepsen.util 9932 :invoke :write [10 47] INFO [2019-12-17 12:35:44,764] jepsen worker 80 - jepsen.util 9620 :fail :cas [26 [16 18]] :ValueNotSwapFrom INFO [2019-12-17 12:35:44,764] jepsen worker 80 - jepsen.util 9620 :invoke :read [26 nil] INFO [2019-12-17 12:35:44,799] jepsen worker 80 - jepsen.util 9620 :ok :read [26 45] INFO [2019-12-17 12:35:44,799] jepsen worker 80 - jepsen.util 9620 :invoke :read [26 nil] INFO [2019-12-17 12:35:44,814] jepsen worker 29 - jepsen.util 8219 :fail :read [9 nil] :RequestTimeoutException INFO [2019-12-17 12:35:44,814] jepsen worker 29 - jepsen.util 8219 :invoke :cas [9 [44 46]] INFO [2019-12-17 12:35:44,815] jepsen worker 18 - jepsen.util 10098 :ok :read [6 48] INFO [2019-12-17 12:35:44,815] jepsen worker 18 - jepsen.util 10098 :invoke :read [6 nil] INFO [2019-12-17 12:35:44,819] jepsen worker 11 - jepsen.util 9731 :ok :read [3 44] INFO [2019-12-17 12:35:44,819] jepsen worker 11 - jepsen.util 9731 :invoke :cas [3 [30 35]] INFO [2019-12-17 12:35:44,819] jepsen worker 80 - jepsen.util 9620 :ok :read [26 45] INFO [2019-12-17 12:35:44,819] jepsen worker 80 - jepsen.util 9620 :invoke :write [26 12] INFO [2019-12-17 12:35:44,825] jepsen worker 4 - jepsen.util 9184 :ok :write [1 6] INFO [2019-12-17 12:35:44,825] jepsen worker 4 - jepsen.util 9184 :invoke :read [1 nil] INFO [2019-12-17 12:35:44,825] jepsen worker 30 - jepsen.util 10020 :ok :read [10 19] INFO [2019-12-17 12:35:44,826] jepsen worker 30 - jepsen.util 10020 :invoke :read [10 nil] INFO [2019-12-17 12:35:44,832] jepsen worker 89 - jepsen.util 9899 :info :write [29 20] :RequestCanceledException INFO [2019-12-17 12:35:44,834] jepsen worker 89 - jepsen.util 9989 :invoke :read [29 nil] INFO [2019-12-17 12:35:44,835] jepsen worker 73 - jepsen.util 9343 :fail :cas [24 [43 23]] :ValueNotSwapFrom INFO [2019-12-17 12:35:44,835] jepsen worker 73 - jepsen.util 9343 :invoke :read [24 nil] INFO [2019-12-17 12:35:44,857] jepsen worker 11 - jepsen.util 9731 :fail :cas [3 [30 35]] :ValueNotSwapFrom INFO [2019-12-17 12:35:44,857] jepsen worker 11 - jepsen.util 9731 :invoke :write [3 10] INFO [2019-12-17 12:35:44,859] jepsen worker 81 - jepsen.util 13131 :info :cas [27 [5 35]] :RequestCanceledException INFO [2019-12-17 12:35:44,860] jepsen worker 81 - jepsen.util 13221 :invoke :write [27 28] INFO [2019-12-17 12:35:44,873] jepsen worker 30 - jepsen.util 10020 :ok :read [10 19] INFO [2019-12-17 12:35:44,873] jepsen worker 30 - jepsen.util 10020 :invoke :read [10 nil] INFO [2019-12-17 12:35:44,873] jepsen worker 18 - jepsen.util 10098 :ok :read [6 48] INFO [2019-12-17 12:35:44,873] jepsen worker 18 - jepsen.util 10098 :invoke :read [6 nil] INFO [2019-12-17 12:35:44,886] jepsen worker 73 - jepsen.util 9343 :ok :read [24 22] INFO [2019-12-17 12:35:44,886] jepsen worker 73 - jepsen.util 9343 :invoke :write [24 9] INFO [2019-12-17 12:35:44,899] jepsen worker 13 - jepsen.util 9283 :ok :read [4 38] INFO [2019-12-17 12:35:44,899] jepsen worker 13 - jepsen.util 9283 :invoke :write [4 13] INFO [2019-12-17 12:35:44,913] jepsen worker 18 - jepsen.util 10098 :ok :read [6 48] INFO [2019-12-17 12:35:44,913] jepsen worker 18 - jepsen.util 10098 :invoke :cas [6 [25 4]] INFO [2019-12-17 12:35:44,916] jepsen worker 72 - jepsen.util 9702 :info :cas [24 [48 11]] :RequestCanceledException INFO [2019-12-17 12:35:44,923] jepsen worker 30 - jepsen.util 10020 :ok :read [10 19] INFO [2019-12-17 12:35:44,923] jepsen worker 30 - jepsen.util 10020 :invoke :read [10 nil] INFO [2019-12-17 12:35:44,924] jepsen worker 72 - jepsen.util 9792 :invoke :cas [24 [8 26]] INFO [2019-12-17 12:35:44,953] jepsen worker 36 - jepsen.util 10566 :ok :write [12 23] INFO [2019-12-17 12:35:44,953] jepsen worker 36 - jepsen.util 10566 :invoke :read [12 nil] INFO [2019-12-17 12:35:44,953] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":57054,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:35:44,953] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":57054,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:35:44,953] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":57057,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:35:44,953] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":57057,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:35:44,985] jepsen worker 53 - jepsen.util 9413 :ok :read [17 20] INFO [2019-12-17 12:35:44,985] jepsen worker 53 - jepsen.util 9413 :invoke :write [17 49] INFO [2019-12-17 12:35:44,990] jepsen worker 30 - jepsen.util 10020 :ok :read [10 19] INFO [2019-12-17 12:35:44,991] jepsen worker 30 - jepsen.util 10020 :invoke :read [10 nil] INFO [2019-12-17 12:35:44,993] jepsen worker 56 - jepsen.util 10406 :fail :cas [18 [11 5]] :ValueNotSwapFrom INFO [2019-12-17 12:35:44,993] jepsen worker 56 - jepsen.util 10406 :invoke :read [18 nil] INFO [2019-12-17 12:35:44,997] jepsen worker 39 - jepsen.util 12999 :info :write [13 40] :RequestCanceledException INFO [2019-12-17 12:35:44,998] jepsen worker 39 - jepsen.util 13089 :invoke :write [13 10] INFO [2019-12-17 12:35:45,015] jepsen worker 10 - jepsen.util 9910 :fail :read [3 nil] :RequestTimeoutException INFO [2019-12-17 12:35:45,016] jepsen worker 10 - jepsen.util 9910 :invoke :write [3 19] INFO [2019-12-17 12:35:45,022] jepsen worker 72 - jepsen.util 9792 :fail :cas [24 [8 26]] :ValueNotSwapFrom INFO [2019-12-17 12:35:45,023] jepsen worker 72 - jepsen.util 9792 :invoke :write [24 12] INFO [2019-12-17 12:35:45,031] jepsen worker 48 - jepsen.util 12288 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:35:45,031] jepsen worker 48 - jepsen.util 12288 :invoke :cas [16 [36 16]] INFO [2019-12-17 12:35:45,040] jepsen worker 30 - jepsen.util 10020 :ok :read [10 19] INFO [2019-12-17 12:35:45,040] jepsen worker 30 - jepsen.util 10020 :invoke :read [10 nil] INFO [2019-12-17 12:35:45,040] jepsen worker 5 - jepsen.util 9275 :ok :read [1 6] INFO [2019-12-17 12:35:45,040] jepsen worker 5 - jepsen.util 9275 :invoke :cas [1 [46 44]] INFO [2019-12-17 12:35:45,067] jepsen worker 80 - jepsen.util 9620 :ok :write [26 12] INFO [2019-12-17 12:35:45,068] jepsen worker 80 - jepsen.util 9620 :invoke :cas [26 [16 25]] INFO [2019-12-17 12:35:45,072] jepsen worker 30 - jepsen.util 10020 :ok :read [10 19] INFO [2019-12-17 12:35:45,072] jepsen worker 30 - jepsen.util 10020 :invoke :read [10 nil] INFO [2019-12-17 12:35:45,077] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":57073,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:35:45,078] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":57073,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:35:45,078] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":57077,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:35:45,078] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":57077,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:35:45,091] jepsen worker 30 - jepsen.util 10020 :ok :read [10 19] INFO [2019-12-17 12:35:45,091] jepsen worker 30 - jepsen.util 10020 :invoke :read [10 nil] INFO [2019-12-17 12:35:45,097] jepsen worker 31 - jepsen.util 8851 :info :write [10 18] :RequestCanceledException INFO [2019-12-17 12:35:45,097] jepsen worker 38 - jepsen.util 10478 :info :write [12 27] :RequestCanceledException INFO [2019-12-17 12:35:45,098] jepsen worker 38 - jepsen.util 10568 :invoke :write [12 36] INFO [2019-12-17 12:35:45,102] jepsen worker 31 - jepsen.util 8941 :invoke :read [10 nil] INFO [2019-12-17 12:35:45,104] jepsen worker 88 - jepsen.util 10078 :fail :cas [29 [25 9]] :ValueNotSwapFrom INFO [2019-12-17 12:35:45,104] jepsen worker 88 - jepsen.util 10078 :invoke :write [29 28] INFO [2019-12-17 12:35:45,109] jepsen worker 51 - jepsen.util 9231 :ok :read [17 20] INFO [2019-12-17 12:35:45,109] jepsen worker 51 - jepsen.util 9231 :invoke :write [17 19] INFO [2019-12-17 12:35:45,122] jepsen worker 30 - jepsen.util 10020 :ok :read [10 19] INFO [2019-12-17 12:35:45,123] jepsen worker 30 - jepsen.util 10020 :invoke :read [10 nil] INFO [2019-12-17 12:35:45,130] jepsen worker 77 - jepsen.util 14207 :info :cas [25 [30 24]] :RequestCanceledException INFO [2019-12-17 12:35:45,131] jepsen worker 77 - jepsen.util 14297 :invoke :read [25 nil] INFO [2019-12-17 12:35:45,136] jepsen worker 30 - jepsen.util 10020 :ok :read [10 19] INFO [2019-12-17 12:35:45,136] jepsen worker 30 - jepsen.util 10020 :invoke :write [10 24] INFO [2019-12-17 12:35:45,145] jepsen worker 70 - jepsen.util 10600 :info :write [23 45] :RequestCanceledException INFO [2019-12-17 12:35:45,145] jepsen worker 33 - jepsen.util 13443 :info :cas [11 [7 32]] :RequestCanceledException INFO [2019-12-17 12:35:45,146] jepsen worker 70 - jepsen.util 10690 :invoke :cas [23 [38 43]] INFO [2019-12-17 12:35:45,146] jepsen worker 33 - jepsen.util 13533 :invoke :cas [11 [6 46]] INFO [2019-12-17 12:35:45,151] jepsen worker 80 - jepsen.util 9620 :fail :cas [26 [16 25]] :ValueNotSwapFrom INFO [2019-12-17 12:35:45,152] jepsen worker 80 - jepsen.util 9620 :invoke :read [26 nil] INFO [2019-12-17 12:35:45,153] jepsen worker 36 - jepsen.util 10566 :ok :read [12 23] INFO [2019-12-17 12:35:45,154] jepsen worker 36 - jepsen.util 10566 :invoke :read [12 nil] INFO [2019-12-17 12:35:45,190] jepsen worker 56 - jepsen.util 10406 :ok :read [18 46] INFO [2019-12-17 12:35:45,190] jepsen worker 56 - jepsen.util 10406 :invoke :write [18 43] INFO [2019-12-17 12:35:45,211] jepsen worker 55 - jepsen.util 10585 :ok :write [18 5] INFO [2019-12-17 12:35:45,211] jepsen worker 55 - jepsen.util 10585 :invoke :write [18 37] INFO [2019-12-17 12:35:45,215] jepsen worker 52 - jepsen.util 9772 :info :write [17 48] :RequestCanceledException INFO [2019-12-17 12:35:45,216] jepsen worker 52 - jepsen.util 9862 :invoke :read [17 nil] INFO [2019-12-17 12:35:45,267] jepsen worker 5 - jepsen.util 9275 :fail :cas [1 [46 44]] :ValueNotSwapFrom INFO [2019-12-17 12:35:45,267] jepsen worker 5 - jepsen.util 9275 :invoke :read [1 nil] INFO [2019-12-17 12:35:45,275] jepsen worker 80 - jepsen.util 9620 :ok :read [26 12] INFO [2019-12-17 12:35:45,275] jepsen worker 80 - jepsen.util 9620 :invoke :read [26 nil] INFO [2019-12-17 12:35:45,278] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":57108,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:35:45,278] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":57108,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:35:45,278] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":57111,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:35:45,278] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":57111,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:35:45,278] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":57112,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:35:45,278] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":57112,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:35:45,282] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":57100,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:35:45,282] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":57100,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:35:45,283] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":57101,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:35:45,283] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":57101,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:35:45,294] jepsen worker 32 - jepsen.util 9932 :ok :write [10 47] INFO [2019-12-17 12:35:45,294] jepsen worker 46 - jepsen.util 9766 :ok :write [15 42] INFO [2019-12-17 12:35:45,294] jepsen worker 32 - jepsen.util 9932 :invoke :write [10 49] INFO [2019-12-17 12:35:45,294] jepsen worker 46 - jepsen.util 9766 :invoke :read [15 nil] INFO [2019-12-17 12:35:45,295] jepsen worker 74 - jepsen.util 9614 :ok :write [24 2] INFO [2019-12-17 12:35:45,295] jepsen worker 74 - jepsen.util 9614 :invoke :read [24 nil] INFO [2019-12-17 12:35:45,320] jepsen worker 54 - jepsen.util 10854 :ok :read [18 5] INFO [2019-12-17 12:35:45,320] jepsen worker 54 - jepsen.util 10854 :invoke :cas [18 [34 40]] INFO [2019-12-17 12:35:45,331] jepsen worker 59 - jepsen.util 12749 :info :cas [19 [2 29]] :RequestCanceledException INFO [2019-12-17 12:35:45,331] jepsen worker 78 - jepsen.util 9798 :info :write [26 26] :RequestCanceledException INFO [2019-12-17 12:35:45,331] jepsen worker 59 - jepsen.util 12839 :invoke :read [19 nil] INFO [2019-12-17 12:35:45,332] jepsen worker 78 - jepsen.util 9888 :invoke :cas [26 [16 5]] INFO [2019-12-17 12:35:45,338] jepsen worker 11 - jepsen.util 9731 :ok :write [3 10] INFO [2019-12-17 12:35:45,338] jepsen worker 11 - jepsen.util 9731 :invoke :cas [3 [24 38]] INFO [2019-12-17 12:35:45,341] jepsen worker 45 - jepsen.util 10215 :ok :read [15 42] INFO [2019-12-17 12:35:45,341] jepsen worker 45 - jepsen.util 10215 :invoke :read [15 nil] INFO [2019-12-17 12:35:45,343] jepsen worker 67 - jepsen.util 13387 :info :cas [22 [35 23]] :RequestCanceledException INFO [2019-12-17 12:35:45,344] jepsen worker 67 - jepsen.util 13477 :invoke :read [22 nil] INFO [2019-12-17 12:35:45,353] jepsen worker 14 - jepsen.util 9644 :ok :write [4 14] INFO [2019-12-17 12:35:45,353] jepsen worker 14 - jepsen.util 9644 :invoke :cas [4 [40 27]] INFO [2019-12-17 12:35:45,361] jepsen worker 21 - jepsen.util 13071 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:35:45,362] jepsen worker 21 - jepsen.util 13071 :invoke :write [7 12] INFO [2019-12-17 12:35:45,384] jepsen worker 11 - jepsen.util 9731 :fail :cas [3 [24 38]] :ValueNotSwapFrom INFO [2019-12-17 12:35:45,384] jepsen worker 11 - jepsen.util 9731 :invoke :write [3 0] INFO [2019-12-17 12:35:45,415] jepsen worker 50 - jepsen.util 14180 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:35:45,415] jepsen worker 50 - jepsen.util 14180 :invoke :read [16 nil] INFO [2019-12-17 12:35:45,416] jepsen worker 8 - jepsen.util 13058 :info :write [2 16] :RequestCanceledException INFO [2019-12-17 12:35:45,419] jepsen worker 8 - jepsen.util 13148 :invoke :write [2 7] INFO [2019-12-17 12:35:45,421] jepsen worker 9 - jepsen.util 10629 :ok :read [3 44] INFO [2019-12-17 12:35:45,421] jepsen worker 9 - jepsen.util 10629 :invoke :write [3 16] INFO [2019-12-17 12:35:45,431] jepsen worker 84 - jepsen.util 13044 :info :cas [28 [2 12]] :RequestCanceledException INFO [2019-12-17 12:35:45,432] jepsen worker 84 - jepsen.util 13134 :invoke :read [28 nil] INFO [2019-12-17 12:35:45,437] jepsen worker 29 - jepsen.util 8219 :fail :cas [9 [44 46]] :ValueNotSwapFrom INFO [2019-12-17 12:35:45,437] jepsen worker 29 - jepsen.util 8219 :invoke :write [9 31] INFO [2019-12-17 12:35:45,442] jepsen worker 27 - jepsen.util 9477 :ok :write [9 44] INFO [2019-12-17 12:35:45,442] jepsen worker 27 - jepsen.util 9477 :invoke :write [9 22] INFO [2019-12-17 12:35:45,445] jepsen worker 41 - jepsen.util 14441 :info :cas [13 [17 8]] :RequestCanceledException INFO [2019-12-17 12:35:45,445] jepsen worker 22 - jepsen.util 13072 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:35:45,446] jepsen worker 22 - jepsen.util 13072 :invoke :cas [7 [42 16]] INFO [2019-12-17 12:35:45,446] jepsen worker 41 - jepsen.util 14531 :invoke :write [13 1] INFO [2019-12-17 12:35:45,448] jepsen worker 69 - jepsen.util 9969 :ok :write [23 4] INFO [2019-12-17 12:35:45,448] jepsen worker 69 - jepsen.util 9969 :invoke :cas [23 [28 24]] INFO [2019-12-17 12:35:45,451] jepsen worker 80 - jepsen.util 9620 :ok :read [26 12] INFO [2019-12-17 12:35:45,452] jepsen worker 80 - jepsen.util 9620 :invoke :cas [26 [13 9]] INFO [2019-12-17 12:35:45,463] jepsen worker 34 - jepsen.util 13444 :info :cas [11 [46 31]] :RequestCanceledException INFO [2019-12-17 12:35:45,463] jepsen worker 2 - jepsen.util 13232 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:35:45,463] jepsen worker 2 - jepsen.util 13232 :invoke :write [0 46] INFO [2019-12-17 12:35:45,464] jepsen worker 34 - jepsen.util 13534 :invoke :cas [11 [35 10]] INFO [2019-12-17 12:35:45,497] jepsen worker 15 - jepsen.util 12615 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:35:45,497] jepsen worker 86 - jepsen.util 14306 :info :cas [28 [37 31]] :RequestCanceledException INFO [2019-12-17 12:35:45,498] jepsen worker 15 - jepsen.util 12615 :invoke :read [5 nil] INFO [2019-12-17 12:35:45,498] jepsen worker 86 - jepsen.util 14396 :invoke :write [28 2] INFO [2019-12-17 12:35:45,502] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":57131,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:35:45,502] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":57131,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:35:45,502] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":57135,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:35:45,502] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":57135,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:35:45,504] jepsen worker 87 - jepsen.util 9537 :ok :write [29 43] INFO [2019-12-17 12:35:45,504] jepsen worker 87 - jepsen.util 9537 :invoke :read [29 nil] INFO [2019-12-17 12:35:45,512] jepsen worker 27 - jepsen.util 9477 :ok :write [9 22] INFO [2019-12-17 12:35:45,512] jepsen worker 27 - jepsen.util 9477 :invoke :read [9 nil] INFO [2019-12-17 12:35:45,515] jepsen worker 63 - jepsen.util 8793 :info :write [21 17] :RequestCanceledException INFO [2019-12-17 12:35:45,515] jepsen worker 16 - jepsen.util 14686 :info :write [5 38] :RequestCanceledException INFO [2019-12-17 12:35:45,517] jepsen worker 16 - jepsen.util 14776 :invoke :cas [5 [35 42]] INFO [2019-12-17 12:35:45,520] jepsen worker 63 - jepsen.util 8883 :invoke :cas [21 [41 3]] INFO [2019-12-17 12:35:45,530] jepsen worker 27 - jepsen.util 9477 :ok :read [9 22] INFO [2019-12-17 12:35:45,530] jepsen worker 27 - jepsen.util 9477 :invoke :write [9 24] INFO [2019-12-17 12:35:45,531] jepsen worker 47 - jepsen.util 10397 :info :write [15 1] :RequestCanceledException INFO [2019-12-17 12:35:45,532] jepsen worker 47 - jepsen.util 10487 :invoke :write [15 18] INFO [2019-12-17 12:35:45,544] jepsen worker 37 - jepsen.util 9307 :info :write [12 13] :RequestCanceledException INFO [2019-12-17 12:35:45,545] jepsen worker 37 - jepsen.util 9397 :invoke :write [12 23] INFO [2019-12-17 12:35:45,547] jepsen worker 63 - jepsen.util 8883 :fail :cas [21 [41 3]] :ValueNotSwapFrom INFO [2019-12-17 12:35:45,548] jepsen worker 63 - jepsen.util 8883 :invoke :write [21 32] INFO [2019-12-17 12:35:45,611] jepsen worker 36 - jepsen.util 10566 :ok :read [12 23] INFO [2019-12-17 12:35:45,611] jepsen worker 36 - jepsen.util 10566 :invoke :read [12 nil] INFO [2019-12-17 12:35:45,618] jepsen worker 89 - jepsen.util 9989 :ok :read [29 43] INFO [2019-12-17 12:35:45,618] jepsen worker 89 - jepsen.util 9989 :invoke :cas [29 [15 33]] INFO [2019-12-17 12:35:45,630] jepsen worker 32 - jepsen.util 9932 :ok :write [10 49] INFO [2019-12-17 12:35:45,630] jepsen worker 32 - jepsen.util 9932 :invoke :write [10 38] INFO [2019-12-17 12:35:45,632] jepsen worker 43 - jepsen.util 13363 :info :cas [14 [22 22]] :RequestCanceledException INFO [2019-12-17 12:35:45,633] jepsen worker 43 - jepsen.util 13453 :invoke :cas [14 [34 35]] INFO [2019-12-17 12:35:45,652] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:35:45,652] jepsen nemesis - couchbase.util Rebalance check count: 447 INFO [2019-12-17 12:35:45,652] jepsen nemesis - couchbase.util Rebalance stuck count: 428 INFO [2019-12-17 12:35:45,656] jepsen worker 10 - jepsen.util 9910 :ok :write [3 19] INFO [2019-12-17 12:35:45,656] jepsen worker 10 - jepsen.util 9910 :invoke :cas [3 [38 49]] INFO [2019-12-17 12:35:45,673] jepsen worker 52 - jepsen.util 9862 :ok :read [17 48] INFO [2019-12-17 12:35:45,673] jepsen worker 52 - jepsen.util 9862 :invoke :cas [17 [41 39]] INFO [2019-12-17 12:35:45,685] jepsen worker 27 - jepsen.util 9477 :ok :write [9 24] INFO [2019-12-17 12:35:45,685] jepsen worker 27 - jepsen.util 9477 :invoke :read [9 nil] INFO [2019-12-17 12:35:45,690] jepsen worker 5 - jepsen.util 9275 :ok :read [1 6] INFO [2019-12-17 12:35:45,690] jepsen worker 5 - jepsen.util 9275 :invoke :write [1 6] INFO [2019-12-17 12:35:45,693] jepsen worker 42 - jepsen.util 14442 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:35:45,693] jepsen worker 42 - jepsen.util 14442 :invoke :read [14 nil] INFO [2019-12-17 12:35:45,696] jepsen worker 80 - jepsen.util 9620 :fail :cas [26 [13 9]] :ValueNotSwapFrom INFO [2019-12-17 12:35:45,696] jepsen worker 80 - jepsen.util 9620 :invoke :cas [26 [35 46]] INFO [2019-12-17 12:35:45,700] jepsen worker 46 - jepsen.util 9766 :ok :read [15 42] INFO [2019-12-17 12:35:45,700] jepsen worker 46 - jepsen.util 9766 :invoke :write [15 4] INFO [2019-12-17 12:35:45,713] jepsen worker 14 - jepsen.util 9644 :fail :cas [4 [40 27]] :ValueNotSwapFrom INFO [2019-12-17 12:35:45,713] jepsen worker 14 - jepsen.util 9644 :invoke :cas [4 [41 47]] INFO [2019-12-17 12:35:45,715] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":57162,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:35:45,715] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":57162,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:35:45,723] jepsen worker 80 - jepsen.util 9620 :fail :cas [26 [35 46]] :ValueNotSwapFrom INFO [2019-12-17 12:35:45,723] jepsen worker 80 - jepsen.util 9620 :invoke :cas [26 [34 7]] INFO [2019-12-17 12:35:45,724] jepsen worker 87 - jepsen.util 9537 :ok :read [29 43] INFO [2019-12-17 12:35:45,724] jepsen worker 87 - jepsen.util 9537 :invoke :cas [29 [33 42]] INFO [2019-12-17 12:35:45,731] jepsen worker 60 - jepsen.util 14370 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:35:45,731] jepsen worker 17 - jepsen.util 13157 :info :cas [5 [27 4]] :RequestCanceledException INFO [2019-12-17 12:35:45,731] jepsen worker 60 - jepsen.util 14370 :invoke :write [20 2] INFO [2019-12-17 12:35:45,732] jepsen worker 17 - jepsen.util 13247 :invoke :write [5 45] INFO [2019-12-17 12:35:45,745] jepsen worker 57 - jepsen.util 13017 :info :write [19 38] :RequestCanceledException INFO [2019-12-17 12:35:45,746] jepsen worker 57 - jepsen.util 13107 :invoke :write [19 37] INFO [2019-12-17 12:35:45,754] jepsen worker 47 - jepsen.util 10487 :ok :write [15 18] INFO [2019-12-17 12:35:45,754] jepsen worker 47 - jepsen.util 10487 :invoke :cas [15 [42 11]] INFO [2019-12-17 12:35:45,759] jepsen worker 45 - jepsen.util 10215 :ok :read [15 42] INFO [2019-12-17 12:35:45,759] jepsen worker 45 - jepsen.util 10215 :invoke :read [15 nil] INFO [2019-12-17 12:35:45,762] jepsen worker 27 - jepsen.util 9477 :ok :read [9 24] INFO [2019-12-17 12:35:45,762] jepsen worker 27 - jepsen.util 9477 :invoke :read [9 nil] INFO [2019-12-17 12:35:45,771] jepsen worker 53 - jepsen.util 9413 :ok :write [17 49] INFO [2019-12-17 12:35:45,772] jepsen worker 53 - jepsen.util 9413 :invoke :cas [17 [24 48]] INFO [2019-12-17 12:35:45,781] jepsen worker 47 - jepsen.util 10487 :fail :cas [15 [42 11]] :ValueNotSwapFrom INFO [2019-12-17 12:35:45,781] jepsen worker 47 - jepsen.util 10487 :invoke :write [15 22] INFO [2019-12-17 12:35:45,789] jepsen worker 79 - jepsen.util 10159 :ok :write [26 32] INFO [2019-12-17 12:35:45,789] jepsen worker 79 - jepsen.util 10159 :invoke :cas [26 [35 33]] INFO [2019-12-17 12:35:45,790] jepsen worker 32 - jepsen.util 9932 :ok :write [10 38] INFO [2019-12-17 12:35:45,790] jepsen worker 32 - jepsen.util 9932 :invoke :read [10 nil] INFO [2019-12-17 12:35:45,817] jepsen worker 27 - jepsen.util 9477 :ok :read [9 24] INFO [2019-12-17 12:35:45,819] jepsen worker 27 - jepsen.util 9477 :invoke :cas [9 [41 33]] INFO [2019-12-17 12:35:45,830] jepsen worker 27 - jepsen.util 9477 :fail :cas [9 [41 33]] :ValueNotSwapFrom INFO [2019-12-17 12:35:45,831] jepsen worker 27 - jepsen.util 9477 :invoke :cas [9 [20 41]] INFO [2019-12-17 12:35:45,831] jepsen worker 19 - jepsen.util 10099 :info :cas [6 [11 19]] :RequestCanceledException INFO [2019-12-17 12:35:45,831] jepsen worker 64 - jepsen.util 7624 :fail :read [21 nil] :RequestTimeoutException INFO [2019-12-17 12:35:45,831] jepsen worker 64 - jepsen.util 7624 :invoke :cas [21 [38 17]] INFO [2019-12-17 12:35:45,832] jepsen worker 19 - jepsen.util 10189 :invoke :read [6 nil] INFO [2019-12-17 12:35:45,837] jepsen worker 32 - jepsen.util 9932 :ok :read [10 38] INFO [2019-12-17 12:35:45,837] jepsen worker 32 - jepsen.util 9932 :invoke :read [10 nil] INFO [2019-12-17 12:35:45,863] jepsen worker 6 - jepsen.util 14046 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:35:45,863] jepsen worker 6 - jepsen.util 14046 :invoke :cas [2 [38 20]] INFO [2019-12-17 12:35:45,866] jepsen worker 80 - jepsen.util 9620 :fail :cas [26 [34 7]] :ValueNotSwapFrom INFO [2019-12-17 12:35:45,867] jepsen worker 80 - jepsen.util 9620 :invoke :read [26 nil] INFO [2019-12-17 12:35:45,883] jepsen worker 27 - jepsen.util 9477 :fail :cas [9 [20 41]] :ValueNotSwapFrom INFO [2019-12-17 12:35:45,883] jepsen worker 27 - jepsen.util 9477 :invoke :read [9 nil] INFO [2019-12-17 12:35:45,897] jepsen worker 19 - jepsen.util 10189 :ok :read [6 48] INFO [2019-12-17 12:35:45,897] jepsen worker 19 - jepsen.util 10189 :invoke :read [6 nil] INFO [2019-12-17 12:35:45,905] jepsen worker 64 - jepsen.util 7624 :fail :cas [21 [38 17]] :ValueNotSwapFrom INFO [2019-12-17 12:35:45,906] jepsen worker 64 - jepsen.util 7624 :invoke :cas [21 [30 44]] INFO [2019-12-17 12:35:45,922] jepsen worker 3 - jepsen.util 9723 :ok :read [1 6] INFO [2019-12-17 12:35:45,922] jepsen worker 36 - jepsen.util 10566 :ok :read [12 13] INFO [2019-12-17 12:35:45,922] jepsen worker 3 - jepsen.util 9723 :invoke :read [1 nil] INFO [2019-12-17 12:35:45,922] jepsen worker 36 - jepsen.util 10566 :invoke :write [12 18] INFO [2019-12-17 12:35:45,929] jepsen worker 76 - jepsen.util 13666 :info :write [25 4] :RequestCanceledException INFO [2019-12-17 12:35:45,930] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":57192,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:35:45,930] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":57192,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:35:45,930] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":57196,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:35:45,930] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":57196,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:35:45,931] jepsen worker 32 - jepsen.util 9932 :ok :read [10 38] INFO [2019-12-17 12:35:45,931] jepsen worker 32 - jepsen.util 9932 :invoke :cas [10 [8 40]] INFO [2019-12-17 12:35:45,933] jepsen worker 76 - jepsen.util 13756 :invoke :read [25 nil] INFO [2019-12-17 12:35:45,933] jepsen worker 27 - jepsen.util 9477 :ok :read [9 24] INFO [2019-12-17 12:35:45,933] jepsen worker 27 - jepsen.util 9477 :invoke :write [9 13] INFO [2019-12-17 12:35:45,935] jepsen worker 4 - jepsen.util 9184 :ok :read [1 6] INFO [2019-12-17 12:35:45,936] jepsen worker 4 - jepsen.util 9184 :invoke :cas [1 [47 10]] INFO [2019-12-17 12:35:45,945] jepsen worker 65 - jepsen.util 8435 :fail :read [21 nil] :RequestTimeoutException INFO [2019-12-17 12:35:45,945] jepsen worker 65 - jepsen.util 8435 :invoke :write [21 28] INFO [2019-12-17 12:35:45,963] jepsen worker 24 - jepsen.util 12264 :info :cas [8 [42 26]] :RequestCanceledException INFO [2019-12-17 12:35:45,964] jepsen worker 24 - jepsen.util 12354 :invoke :read [8 nil] INFO [2019-12-17 12:35:45,973] jepsen worker 64 - jepsen.util 7624 :fail :cas [21 [30 44]] :ValueNotSwapFrom INFO [2019-12-17 12:35:45,973] jepsen worker 64 - jepsen.util 7624 :invoke :cas [21 [40 21]] INFO [2019-12-17 12:35:45,997] jepsen worker 12 - jepsen.util 10092 :info :write [4 24] :RequestCanceledException INFO [2019-12-17 12:35:45,997] jepsen worker 44 - jepsen.util 13724 :info :cas [14 [24 7]] :RequestCanceledException INFO [2019-12-17 12:35:45,999] jepsen worker 44 - jepsen.util 13814 :invoke :read [14 nil] INFO [2019-12-17 12:35:46,000] jepsen worker 12 - jepsen.util 10182 :invoke :read [4 nil] INFO [2019-12-17 12:35:46,000] jepsen worker 19 - jepsen.util 10189 :ok :read [6 48] INFO [2019-12-17 12:35:46,000] jepsen worker 19 - jepsen.util 10189 :invoke :write [6 1] INFO [2019-12-17 12:35:46,000] jepsen worker 70 - jepsen.util 10690 :fail :cas [23 [38 43]] :ValueNotSwapFrom INFO [2019-12-17 12:35:46,001] jepsen worker 70 - jepsen.util 10690 :invoke :cas [23 [38 26]] INFO [2019-12-17 12:35:46,012] jepsen worker 32 - jepsen.util 9932 :fail :cas [10 [8 40]] :ValueNotSwapFrom INFO [2019-12-17 12:35:46,012] jepsen worker 32 - jepsen.util 9932 :invoke :read [10 nil] INFO [2019-12-17 12:35:46,029] jepsen worker 52 - jepsen.util 9862 :fail :cas [17 [41 39]] :ValueNotSwapFrom INFO [2019-12-17 12:35:46,029] jepsen worker 52 - jepsen.util 9862 :invoke :cas [17 [0 34]] INFO [2019-12-17 12:35:46,044] jepsen worker 64 - jepsen.util 7624 :fail :cas [21 [40 21]] :ValueNotSwapFrom INFO [2019-12-17 12:35:46,044] jepsen worker 64 - jepsen.util 7624 :invoke :write [21 13] INFO [2019-12-17 12:35:46,047] jepsen worker 71 - jepsen.util 10601 :fail :cas [23 [9 3]] :ValueNotSwapFrom INFO [2019-12-17 12:35:46,047] jepsen worker 71 - jepsen.util 10601 :invoke :cas [23 [20 13]] INFO [2019-12-17 12:35:46,064] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":57212,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:35:46,064] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":57212,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:35:46,064] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":57215,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:35:46,064] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":57215,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:35:46,072] jepsen worker 32 - jepsen.util 9932 :ok :read [10 38] INFO [2019-12-17 12:35:46,072] jepsen worker 32 - jepsen.util 9932 :invoke :read [10 nil] INFO [2019-12-17 12:35:46,079] jepsen worker 27 - jepsen.util 9477 :ok :write [9 13] INFO [2019-12-17 12:35:46,079] jepsen worker 27 - jepsen.util 9477 :invoke :read [9 nil] INFO [2019-12-17 12:35:46,083] jepsen worker 54 - jepsen.util 10854 :fail :cas [18 [34 40]] :ValueNotSwapFrom INFO [2019-12-17 12:35:46,083] jepsen worker 54 - jepsen.util 10854 :invoke :read [18 nil] INFO [2019-12-17 12:35:46,085] jepsen worker 89 - jepsen.util 9989 :fail :cas [29 [15 33]] :ValueNotSwapFrom INFO [2019-12-17 12:35:46,085] jepsen worker 89 - jepsen.util 9989 :invoke :read [29 nil] INFO [2019-12-17 12:35:46,131] jepsen worker 35 - jepsen.util 13265 :info :cas [11 [37 43]] :RequestCanceledException INFO [2019-12-17 12:35:46,132] jepsen worker 35 - jepsen.util 13355 :invoke :cas [11 [10 25]] INFO [2019-12-17 12:35:46,140] jepsen worker 78 - jepsen.util 9888 :fail :cas [26 [16 5]] :ValueNotSwapFrom INFO [2019-12-17 12:35:46,140] jepsen worker 78 - jepsen.util 9888 :invoke :write [26 7] INFO [2019-12-17 12:35:46,145] jepsen worker 62 - jepsen.util 13922 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:35:46,153] jepsen worker 3 - jepsen.util 9723 :ok :read [1 6] INFO [2019-12-17 12:35:46,162] jepsen worker 23 - jepsen.util 13523 :info :cas [7 [8 10]] :RequestCanceledException INFO [2019-12-17 12:35:46,166] jepsen worker 64 - jepsen.util 7624 :ok :write [21 13] INFO [2019-12-17 12:35:46,174] jepsen worker 52 - jepsen.util 9862 :fail :cas [17 [0 34]] :ValueNotSwapFrom INFO [2019-12-17 12:35:46,180] jepsen worker 70 - jepsen.util 10690 :fail :cas [23 [38 26]] :ValueNotSwapFrom INFO [2019-12-17 12:35:46,182] jepsen worker 79 - jepsen.util 10159 :fail :cas [26 [35 33]] :ValueNotSwapFrom INFO [2019-12-17 12:35:46,186] jepsen worker 9 - jepsen.util 10629 :ok :write [3 16] INFO [2019-12-17 12:35:46,202] jepsen worker 12 - jepsen.util 10182 :ok :read [4 14] INFO [2019-12-17 12:35:46,206] jepsen worker 10 - jepsen.util 9910 :fail :cas [3 [38 49]] :ValueNotSwapFrom INFO [2019-12-17 12:35:46,209] jepsen worker 80 - jepsen.util 9620 :ok :read [26 32] INFO [2019-12-17 12:35:46,231] jepsen worker 62 - jepsen.util 13922 :invoke :cas [20 [11 20]] INFO [2019-12-17 12:35:46,231] jepsen worker 3 - jepsen.util 9723 :invoke :read [1 nil] INFO [2019-12-17 12:35:46,231] jepsen worker 64 - jepsen.util 7624 :invoke :read [21 nil] INFO [2019-12-17 12:35:46,231] jepsen worker 52 - jepsen.util 9862 :invoke :cas [17 [29 12]] INFO [2019-12-17 12:35:46,231] jepsen worker 20 - jepsen.util 9560 :info :write [6 23] :RequestCanceledException INFO [2019-12-17 12:35:46,231] jepsen worker 70 - jepsen.util 10690 :invoke :read [23 nil] INFO [2019-12-17 12:35:46,231] jepsen worker 79 - jepsen.util 10159 :invoke :cas [26 [28 36]] INFO [2019-12-17 12:35:46,231] jepsen worker 9 - jepsen.util 10629 :invoke :cas [3 [19 5]] INFO [2019-12-17 12:35:46,231] jepsen worker 12 - jepsen.util 10182 :invoke :write [4 6] INFO [2019-12-17 12:35:46,231] jepsen worker 10 - jepsen.util 9910 :invoke :read [3 nil] INFO [2019-12-17 12:35:46,231] jepsen worker 80 - jepsen.util 9620 :invoke :cas [26 [40 39]] INFO [2019-12-17 12:35:46,234] jepsen worker 23 - jepsen.util 13613 :invoke :write [7 11] INFO [2019-12-17 12:35:46,234] jepsen worker 20 - jepsen.util 9650 :invoke :write [6 27] INFO [2019-12-17 12:35:46,237] jepsen worker 45 - jepsen.util 10215 :ok :read [15 18] INFO [2019-12-17 12:35:46,238] jepsen worker 45 - jepsen.util 10215 :invoke :read [15 nil] INFO [2019-12-17 12:35:46,258] jepsen worker 64 - jepsen.util 7624 :ok :read [21 13] INFO [2019-12-17 12:35:46,259] jepsen worker 64 - jepsen.util 7624 :invoke :read [21 nil] INFO [2019-12-17 12:35:46,266] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":57230,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:35:46,267] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":57230,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:35:46,267] jepsen worker 9 - jepsen.util 10629 :fail :cas [3 [19 5]] :ValueNotSwapFrom INFO [2019-12-17 12:35:46,267] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":57231,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:35:46,267] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":57231,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:35:46,267] jepsen worker 9 - jepsen.util 10629 :invoke :cas [3 [46 43]] INFO [2019-12-17 12:35:46,267] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":57232,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:35:46,267] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":57232,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:35:46,268] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":57237,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:35:46,269] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":57237,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:35:46,269] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":57241,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:35:46,269] jepsen worker 47 - jepsen.util 10487 :ok :write [15 22] INFO [2019-12-17 12:35:46,269] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":57241,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:35:46,269] jepsen worker 47 - jepsen.util 10487 :invoke :read [15 nil] INFO [2019-12-17 12:35:46,273] jepsen worker 10 - jepsen.util 9910 :ok :read [3 16] INFO [2019-12-17 12:35:46,273] jepsen worker 80 - jepsen.util 9620 :fail :cas [26 [40 39]] :ValueNotSwapFrom INFO [2019-12-17 12:35:46,273] jepsen worker 10 - jepsen.util 9910 :invoke :write [3 25] INFO [2019-12-17 12:35:46,273] jepsen worker 80 - jepsen.util 9620 :invoke :read [26 nil] INFO [2019-12-17 12:35:46,292] jepsen worker 69 - jepsen.util 9969 :fail :cas [23 [28 24]] :ValueNotSwapFrom INFO [2019-12-17 12:35:46,292] jepsen worker 69 - jepsen.util 9969 :invoke :read [23 nil] INFO [2019-12-17 12:35:46,296] jepsen worker 54 - jepsen.util 10854 :ok :read [18 5] INFO [2019-12-17 12:35:46,296] jepsen worker 54 - jepsen.util 10854 :invoke :write [18 37] INFO [2019-12-17 12:35:46,298] jepsen worker 83 - jepsen.util 13853 :info :write [27 30] :RequestCanceledException INFO [2019-12-17 12:35:46,298] jepsen worker 85 - jepsen.util 14215 :info :cas [28 [46 22]] :RequestCanceledException INFO [2019-12-17 12:35:46,299] jepsen worker 85 - jepsen.util 14305 :invoke :write [28 18] INFO [2019-12-17 12:35:46,299] jepsen worker 83 - jepsen.util 13943 :invoke :read [27 nil] INFO [2019-12-17 12:35:46,301] jepsen worker 64 - jepsen.util 7624 :ok :read [21 13] INFO [2019-12-17 12:35:46,302] jepsen worker 64 - jepsen.util 7624 :invoke :read [21 nil] INFO [2019-12-17 12:35:46,307] jepsen worker 45 - jepsen.util 10215 :ok :read [15 22] INFO [2019-12-17 12:35:46,308] jepsen worker 45 - jepsen.util 10215 :invoke :read [15 nil] INFO [2019-12-17 12:35:46,318] jepsen worker 9 - jepsen.util 10629 :fail :cas [3 [46 43]] :ValueNotSwapFrom INFO [2019-12-17 12:35:46,318] jepsen worker 9 - jepsen.util 10629 :invoke :read [3 nil] INFO [2019-12-17 12:35:46,321] jepsen worker 53 - jepsen.util 9413 :fail :cas [17 [24 48]] :ValueNotSwapFrom INFO [2019-12-17 12:35:46,321] jepsen worker 53 - jepsen.util 9413 :invoke :write [17 0] INFO [2019-12-17 12:35:46,331] jepsen worker 28 - jepsen.util 8128 :fail :read [9 nil] :RequestTimeoutException INFO [2019-12-17 12:35:46,331] jepsen worker 28 - jepsen.util 8128 :invoke :cas [9 [0 49]] INFO [2019-12-17 12:35:46,331] jepsen worker 40 - jepsen.util 13810 :info :write [13 34] :RequestCanceledException INFO [2019-12-17 12:35:46,337] jepsen worker 40 - jepsen.util 13900 :invoke :write [13 40] INFO [2019-12-17 12:35:46,340] jepsen worker 4 - jepsen.util 9184 :fail :cas [1 [47 10]] :ValueNotSwapFrom INFO [2019-12-17 12:35:46,340] jepsen worker 4 - jepsen.util 9184 :invoke :cas [1 [24 28]] INFO [2019-12-17 12:35:46,345] jepsen worker 64 - jepsen.util 7624 :ok :read [21 13] INFO [2019-12-17 12:35:46,346] jepsen worker 64 - jepsen.util 7624 :invoke :read [21 nil] INFO [2019-12-17 12:35:46,354] jepsen worker 28 - jepsen.util 8128 :fail :cas [9 [0 49]] :ValueNotSwapFrom INFO [2019-12-17 12:35:46,354] jepsen worker 28 - jepsen.util 8128 :invoke :cas [9 [8 22]] INFO [2019-12-17 12:35:46,360] jepsen worker 87 - jepsen.util 9537 :fail :cas [29 [33 42]] :ValueNotSwapFrom INFO [2019-12-17 12:35:46,361] jepsen worker 87 - jepsen.util 9537 :invoke :cas [29 [24 25]] INFO [2019-12-17 12:35:46,366] jepsen worker 64 - jepsen.util 7624 :ok :read [21 13] INFO [2019-12-17 12:35:46,366] jepsen worker 64 - jepsen.util 7624 :invoke :write [21 25] INFO [2019-12-17 12:35:46,370] jepsen worker 19 - jepsen.util 10189 :ok :write [6 1] INFO [2019-12-17 12:35:46,371] jepsen worker 19 - jepsen.util 10189 :invoke :write [6 45] INFO [2019-12-17 12:35:46,374] jepsen worker 28 - jepsen.util 8128 :fail :cas [9 [8 22]] :ValueNotSwapFrom INFO [2019-12-17 12:35:46,375] jepsen worker 28 - jepsen.util 8128 :invoke :write [9 48] INFO [2019-12-17 12:35:46,396] jepsen worker 10 - jepsen.util 9910 :ok :write [3 25] INFO [2019-12-17 12:35:46,396] jepsen worker 10 - jepsen.util 9910 :invoke :read [3 nil] INFO [2019-12-17 12:35:46,412] jepsen worker 14 - jepsen.util 9644 :fail :cas [4 [41 47]] :ValueNotSwapFrom INFO [2019-12-17 12:35:46,412] jepsen worker 14 - jepsen.util 9644 :invoke :write [4 33] INFO [2019-12-17 12:35:46,416] jepsen worker 3 - jepsen.util 9723 :ok :read [1 6] INFO [2019-12-17 12:35:46,417] jepsen worker 3 - jepsen.util 9723 :invoke :read [1 nil] INFO [2019-12-17 12:35:46,431] jepsen worker 66 - jepsen.util 13206 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:35:46,431] jepsen worker 66 - jepsen.util 13206 :invoke :cas [22 [17 43]] INFO [2019-12-17 12:35:46,435] jepsen worker 70 - jepsen.util 10690 :ok :read [23 4] INFO [2019-12-17 12:35:46,435] jepsen worker 70 - jepsen.util 10690 :invoke :write [23 31] INFO [2019-12-17 12:35:46,437] jepsen worker 71 - jepsen.util 10601 :fail :cas [23 [20 13]] :ValueNotSwapFrom INFO [2019-12-17 12:35:46,437] jepsen worker 71 - jepsen.util 10601 :invoke :write [23 16] INFO [2019-12-17 12:35:46,438] jepsen worker 45 - jepsen.util 10215 :ok :read [15 22] INFO [2019-12-17 12:35:46,438] jepsen worker 45 - jepsen.util 10215 :invoke :cas [15 [28 23]] INFO [2019-12-17 12:35:46,460] jepsen worker 9 - jepsen.util 10629 :ok :read [3 25] INFO [2019-12-17 12:35:46,462] jepsen worker 49 - jepsen.util 13369 :info :write [16 14] :RequestCanceledException INFO [2019-12-17 12:35:46,467] jepsen worker 10 - jepsen.util 9910 :ok :read [3 25] INFO [2019-12-17 12:35:46,467] jepsen worker 9 - jepsen.util 10629 :invoke :write [3 18] INFO [2019-12-17 12:35:46,467] jepsen worker 10 - jepsen.util 9910 :invoke :cas [3 [26 10]] INFO [2019-12-17 12:35:46,468] jepsen worker 49 - jepsen.util 13459 :invoke :write [16 40] INFO [2019-12-17 12:35:46,475] jepsen worker 45 - jepsen.util 10215 :fail :cas [15 [28 23]] :ValueNotSwapFrom INFO [2019-12-17 12:35:46,478] jepsen worker 45 - jepsen.util 10215 :invoke :write [15 43] INFO [2019-12-17 12:35:46,480] jepsen worker 10 - jepsen.util 9910 :fail :cas [3 [26 10]] :ValueNotSwapFrom INFO [2019-12-17 12:35:46,480] jepsen worker 10 - jepsen.util 9910 :invoke :read [3 nil] INFO [2019-12-17 12:35:46,486] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":57281,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:35:46,486] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":57281,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:35:46,486] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":57286,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:35:46,486] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":57286,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:35:46,497] jepsen worker 0 - jepsen.util 14310 :info :cas [0 [39 49]] :RequestCanceledException INFO [2019-12-17 12:35:46,498] jepsen worker 0 - jepsen.util 14400 :invoke :cas [0 [26 7]] INFO [2019-12-17 12:35:46,513] jepsen worker 79 - jepsen.util 10159 :fail :cas [26 [28 36]] :ValueNotSwapFrom INFO [2019-12-17 12:35:46,514] jepsen worker 79 - jepsen.util 10159 :invoke :read [26 nil] INFO [2019-12-17 12:35:46,515] jepsen worker 26 - jepsen.util 14066 :info :cas [8 [39 36]] :RequestCanceledException INFO [2019-12-17 12:35:46,515] jepsen worker 89 - jepsen.util 9989 :ok :read [29 43] INFO [2019-12-17 12:35:46,516] jepsen worker 89 - jepsen.util 9989 :invoke :cas [29 [10 13]] INFO [2019-12-17 12:35:46,516] jepsen worker 26 - jepsen.util 14156 :invoke :cas [8 [13 19]] INFO [2019-12-17 12:35:46,517] jepsen worker 19 - jepsen.util 10189 :ok :write [6 45] INFO [2019-12-17 12:35:46,517] jepsen worker 19 - jepsen.util 10189 :invoke :cas [6 [24 18]] INFO [2019-12-17 12:35:46,517] jepsen worker 64 - jepsen.util 7624 :ok :write [21 25] INFO [2019-12-17 12:35:46,517] jepsen worker 64 - jepsen.util 7624 :invoke :read [21 nil] INFO [2019-12-17 12:35:46,536] jepsen worker 7 - jepsen.util 13507 :info :write [2 27] :RequestCanceledException INFO [2019-12-17 12:35:46,536] jepsen worker 1 - jepsen.util 13591 :info :cas [0 [43 18]] :RequestCanceledException INFO [2019-12-17 12:35:46,537] jepsen worker 1 - jepsen.util 13681 :invoke :read [0 nil] INFO [2019-12-17 12:35:46,537] jepsen worker 7 - jepsen.util 13597 :invoke :cas [2 [6 36]] INFO [2019-12-17 12:35:46,545] jepsen worker 80 - jepsen.util 9620 :ok :read [26 32] INFO [2019-12-17 12:35:46,545] jepsen worker 80 - jepsen.util 9620 :invoke :cas [26 [49 8]] INFO [2019-12-17 12:35:46,548] jepsen worker 64 - jepsen.util 7624 :ok :read [21 25] INFO [2019-12-17 12:35:46,548] jepsen worker 64 - jepsen.util 7624 :invoke :cas [21 [39 13]] INFO [2019-12-17 12:35:46,549] jepsen worker 19 - jepsen.util 10189 :fail :cas [6 [24 18]] :ValueNotSwapFrom INFO [2019-12-17 12:35:46,549] jepsen worker 19 - jepsen.util 10189 :invoke :cas [6 [34 38]] INFO [2019-12-17 12:35:46,558] jepsen worker 61 - jepsen.util 14281 :info :cas [20 [30 2]] :RequestCanceledException INFO [2019-12-17 12:35:46,559] jepsen worker 61 - jepsen.util 14371 :invoke :cas [20 [43 29]] INFO [2019-12-17 12:35:46,574] jepsen worker 64 - jepsen.util 7624 :fail :cas [21 [39 13]] :ValueNotSwapFrom INFO [2019-12-17 12:35:46,575] jepsen worker 64 - jepsen.util 7624 :invoke :read [21 nil] INFO [2019-12-17 12:35:46,575] jepsen worker 19 - jepsen.util 10189 :fail :cas [6 [34 38]] :ValueNotSwapFrom INFO [2019-12-17 12:35:46,576] jepsen worker 19 - jepsen.util 10189 :invoke :write [6 10] INFO [2019-12-17 12:35:46,588] jepsen worker 64 - jepsen.util 7624 :ok :read [21 25] INFO [2019-12-17 12:35:46,588] jepsen worker 64 - jepsen.util 7624 :invoke :cas [21 [10 18]] INFO [2019-12-17 12:35:46,610] jepsen worker 89 - jepsen.util 9989 :fail :cas [29 [10 13]] :ValueNotSwapFrom INFO [2019-12-17 12:35:46,610] jepsen worker 89 - jepsen.util 9989 :invoke :cas [29 [1 44]] INFO [2019-12-17 12:35:46,628] jepsen worker 75 - jepsen.util 13035 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:35:46,628] jepsen worker 68 - jepsen.util 12848 :info :write [22 37] :RequestCanceledException INFO [2019-12-17 12:35:46,629] jepsen worker 75 - jepsen.util 13035 :invoke :read [25 nil] INFO [2019-12-17 12:35:46,630] jepsen worker 68 - jepsen.util 12938 :invoke :write [22 37] INFO [2019-12-17 12:35:46,636] jepsen worker 64 - jepsen.util 7624 :fail :cas [21 [10 18]] :ValueNotSwapFrom INFO [2019-12-17 12:35:46,636] jepsen worker 64 - jepsen.util 7624 :invoke :read [21 nil] INFO [2019-12-17 12:35:46,643] jepsen worker 80 - jepsen.util 9620 :fail :cas [26 [49 8]] :ValueNotSwapFrom INFO [2019-12-17 12:35:46,643] jepsen worker 80 - jepsen.util 9620 :invoke :read [26 nil] INFO [2019-12-17 12:35:46,653] jepsen worker 64 - jepsen.util 7624 :ok :read [21 25] INFO [2019-12-17 12:35:46,653] jepsen worker 64 - jepsen.util 7624 :invoke :read [21 nil] INFO [2019-12-17 12:35:46,662] jepsen worker 58 - jepsen.util 14008 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:35:46,662] jepsen worker 58 - jepsen.util 14008 :invoke :cas [19 [28 34]] INFO [2019-12-17 12:35:46,664] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:35:46,666] jepsen nemesis - couchbase.util Rebalance check count: 448 INFO [2019-12-17 12:35:46,666] jepsen nemesis - couchbase.util Rebalance stuck count: 429 INFO [2019-12-17 12:35:46,668] jepsen worker 64 - jepsen.util 7624 :ok :read [21 25] INFO [2019-12-17 12:35:46,668] jepsen worker 64 - jepsen.util 7624 :invoke :read [21 nil] INFO [2019-12-17 12:35:46,670] jepsen worker 9 - jepsen.util 10629 :ok :write [3 18] INFO [2019-12-17 12:35:46,670] jepsen worker 9 - jepsen.util 10629 :invoke :cas [3 [49 6]] INFO [2019-12-17 12:35:46,681] jepsen worker 19 - jepsen.util 10189 :ok :write [6 10] INFO [2019-12-17 12:35:46,681] jepsen worker 19 - jepsen.util 10189 :invoke :write [6 26] INFO [2019-12-17 12:35:46,697] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":57316,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:35:46,697] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":57316,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:35:46,697] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":57320,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:35:46,697] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":57320,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:35:46,698] jepsen worker 82 - jepsen.util 13852 :info :cas [27 [18 17]] :RequestCanceledException INFO [2019-12-17 12:35:46,701] jepsen worker 82 - jepsen.util 13942 :invoke :read [27 nil] INFO [2019-12-17 12:35:46,708] jepsen worker 80 - jepsen.util 9620 :ok :read [26 32] INFO [2019-12-17 12:35:46,709] jepsen worker 80 - jepsen.util 9620 :invoke :cas [26 [35 3]] INFO [2019-12-17 12:35:46,730] jepsen worker 79 - jepsen.util 10159 :ok :read [26 32] INFO [2019-12-17 12:35:46,730] jepsen worker 79 - jepsen.util 10159 :invoke :read [26 nil] INFO [2019-12-17 12:35:46,743] jepsen worker 80 - jepsen.util 9620 :fail :cas [26 [35 3]] :ValueNotSwapFrom INFO [2019-12-17 12:35:46,743] jepsen worker 80 - jepsen.util 9620 :invoke :cas [26 [1 32]] INFO [2019-12-17 12:35:46,750] jepsen worker 64 - jepsen.util 7624 :ok :read [21 25] INFO [2019-12-17 12:35:46,751] jepsen worker 64 - jepsen.util 7624 :invoke :write [21 7] INFO [2019-12-17 12:35:46,751] jepsen worker 9 - jepsen.util 10629 :fail :cas [3 [49 6]] :ValueNotSwapFrom INFO [2019-12-17 12:35:46,751] jepsen worker 9 - jepsen.util 10629 :invoke :read [3 nil] INFO [2019-12-17 12:35:46,757] jepsen worker 3 - jepsen.util 9723 :ok :read [1 6] INFO [2019-12-17 12:35:46,758] jepsen worker 3 - jepsen.util 9723 :invoke :write [1 18] INFO [2019-12-17 12:35:46,770] jepsen worker 89 - jepsen.util 9989 :fail :cas [29 [1 44]] :ValueNotSwapFrom INFO [2019-12-17 12:35:46,771] jepsen worker 89 - jepsen.util 9989 :invoke :read [29 nil] INFO [2019-12-17 12:35:46,774] jepsen worker 80 - jepsen.util 9620 :fail :cas [26 [1 32]] :ValueNotSwapFrom INFO [2019-12-17 12:35:46,774] jepsen worker 80 - jepsen.util 9620 :invoke :write [26 3] INFO [2019-12-17 12:35:46,796] jepsen worker 25 - jepsen.util 14335 :info :write [8 8] :RequestCanceledException INFO [2019-12-17 12:35:46,798] jepsen worker 25 - jepsen.util 14425 :invoke :cas [8 [1 26]] INFO [2019-12-17 12:35:46,811] jepsen worker 9 - jepsen.util 10629 :ok :read [3 18] INFO [2019-12-17 12:35:46,811] jepsen worker 89 - jepsen.util 9989 :ok :read [29 43] INFO [2019-12-17 12:35:46,811] jepsen worker 9 - jepsen.util 10629 :invoke :cas [3 [39 35]] INFO [2019-12-17 12:35:46,811] jepsen worker 89 - jepsen.util 9989 :invoke :cas [29 [15 3]] INFO [2019-12-17 12:35:46,826] jepsen worker 9 - jepsen.util 10629 :fail :cas [3 [39 35]] :ValueNotSwapFrom INFO [2019-12-17 12:35:46,826] jepsen worker 9 - jepsen.util 10629 :invoke :write [3 39] INFO [2019-12-17 12:35:46,829] jepsen worker 64 - jepsen.util 7624 :ok :write [21 7] INFO [2019-12-17 12:35:46,829] jepsen worker 64 - jepsen.util 7624 :invoke :cas [21 [24 12]] INFO [2019-12-17 12:35:46,848] jepsen worker 64 - jepsen.util 7624 :fail :cas [21 [24 12]] :ValueNotSwapFrom INFO [2019-12-17 12:35:46,848] jepsen worker 64 - jepsen.util 7624 :invoke :read [21 nil] INFO [2019-12-17 12:35:46,869] jepsen worker 89 - jepsen.util 9989 :fail :cas [29 [15 3]] :ValueNotSwapFrom INFO [2019-12-17 12:35:46,869] jepsen worker 89 - jepsen.util 9989 :invoke :read [29 nil] INFO [2019-12-17 12:35:46,891] jepsen worker 64 - jepsen.util 7624 :ok :read [21 7] INFO [2019-12-17 12:35:46,891] jepsen worker 64 - jepsen.util 7624 :invoke :cas [21 [0 21]] INFO [2019-12-17 12:35:46,895] jepsen worker 19 - jepsen.util 10189 :ok :write [6 26] INFO [2019-12-17 12:35:46,896] jepsen worker 19 - jepsen.util 10189 :invoke :cas [6 [28 48]] INFO [2019-12-17 12:35:46,912] jepsen worker 52 - jepsen.util 9862 :fail :cas [17 [29 12]] :ValueNotSwapFrom INFO [2019-12-17 12:35:46,912] jepsen worker 52 - jepsen.util 9862 :invoke :cas [17 [42 17]] INFO [2019-12-17 12:35:46,913] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":57343,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:35:46,914] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":57343,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:35:46,914] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":57346,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:35:46,914] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":57346,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:35:46,924] jepsen worker 64 - jepsen.util 7624 :fail :cas [21 [0 21]] :ValueNotSwapFrom INFO [2019-12-17 12:35:46,924] jepsen worker 64 - jepsen.util 7624 :invoke :cas [21 [6 44]] INFO [2019-12-17 12:35:46,944] jepsen worker 81 - jepsen.util 13221 :info :write [27 28] :RequestCanceledException INFO [2019-12-17 12:35:46,946] jepsen worker 81 - jepsen.util 13311 :invoke :read [27 nil] INFO [2019-12-17 12:35:46,947] jepsen worker 19 - jepsen.util 10189 :fail :cas [6 [28 48]] :ValueNotSwapFrom INFO [2019-12-17 12:35:46,947] jepsen worker 19 - jepsen.util 10189 :invoke :cas [6 [7 33]] INFO [2019-12-17 12:35:46,951] jepsen worker 89 - jepsen.util 9989 :ok :read [29 43] INFO [2019-12-17 12:35:46,951] jepsen worker 89 - jepsen.util 9989 :invoke :write [29 10] INFO [2019-12-17 12:35:46,957] jepsen worker 64 - jepsen.util 7624 :fail :cas [21 [6 44]] :ValueNotSwapFrom INFO [2019-12-17 12:35:46,957] jepsen worker 64 - jepsen.util 7624 :invoke :write [21 27] INFO [2019-12-17 12:35:46,960] jepsen worker 19 - jepsen.util 10189 :fail :cas [6 [7 33]] :ValueNotSwapFrom INFO [2019-12-17 12:35:46,960] jepsen worker 19 - jepsen.util 10189 :invoke :read [6 nil] INFO [2019-12-17 12:35:46,973] jepsen worker 87 - jepsen.util 9537 :fail :cas [29 [24 25]] :ValueNotSwapFrom INFO [2019-12-17 12:35:46,973] jepsen worker 87 - jepsen.util 9537 :invoke :cas [29 [38 8]] INFO [2019-12-17 12:35:46,981] jepsen worker 19 - jepsen.util 10189 :ok :read [6 26] INFO [2019-12-17 12:35:46,981] jepsen worker 19 - jepsen.util 10189 :invoke :read [6 nil] INFO [2019-12-17 12:35:46,997] jepsen worker 73 - jepsen.util 9343 :info :write [24 9] :RequestCanceledException INFO [2019-12-17 12:35:46,997] jepsen worker 13 - jepsen.util 9283 :info :write [4 13] :RequestCanceledException INFO [2019-12-17 12:35:46,997] jepsen worker 18 - jepsen.util 10098 :info :cas [6 [25 4]] :RequestCanceledException INFO [2019-12-17 12:35:46,999] jepsen worker 73 - jepsen.util 9433 :invoke :write [24 20] INFO [2019-12-17 12:35:46,999] jepsen worker 13 - jepsen.util 9373 :invoke :cas [4 [31 46]] INFO [2019-12-17 12:35:46,999] jepsen worker 18 - jepsen.util 10188 :invoke :read [6 nil] INFO [2019-12-17 12:35:47,007] jepsen worker 69 - jepsen.util 9969 :ok :read [23 4] INFO [2019-12-17 12:35:47,007] jepsen worker 69 - jepsen.util 9969 :invoke :write [23 11] INFO [2019-12-17 12:35:47,031] jepsen worker 39 - jepsen.util 13089 :info :write [13 10] :RequestCanceledException INFO [2019-12-17 12:35:47,044] jepsen worker 39 - jepsen.util 13179 :invoke :read [13 nil] INFO [2019-12-17 12:35:47,051] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":57362,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:35:47,051] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":57362,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:35:47,051] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":57365,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:35:47,052] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":57365,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:35:47,052] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":57366,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:35:47,052] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":57366,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:35:47,062] jepsen worker 72 - jepsen.util 9792 :info :write [24 12] :RequestCanceledException INFO [2019-12-17 12:35:47,071] jepsen worker 72 - jepsen.util 9882 :invoke :write [24 9] INFO [2019-12-17 12:35:47,076] jepsen worker 19 - jepsen.util 10189 :ok :read [6 26] INFO [2019-12-17 12:35:47,077] jepsen worker 19 - jepsen.util 10189 :invoke :cas [6 [15 32]] INFO [2019-12-17 12:35:47,077] jepsen worker 18 - jepsen.util 10188 :ok :read [6 26] INFO [2019-12-17 12:35:47,077] jepsen worker 18 - jepsen.util 10188 :invoke :write [6 38] INFO [2019-12-17 12:35:47,091] jepsen worker 64 - jepsen.util 7624 :ok :write [21 27] INFO [2019-12-17 12:35:47,091] jepsen worker 64 - jepsen.util 7624 :invoke :cas [21 [20 21]] INFO [2019-12-17 12:35:47,102] jepsen worker 19 - jepsen.util 10189 :fail :cas [6 [15 32]] :ValueNotSwapFrom INFO [2019-12-17 12:35:47,102] jepsen worker 19 - jepsen.util 10189 :invoke :cas [6 [43 21]] INFO [2019-12-17 12:35:47,115] jepsen worker 31 - jepsen.util 8941 :fail :read [10 nil] :RequestTimeoutException INFO [2019-12-17 12:35:47,116] jepsen worker 31 - jepsen.util 8941 :invoke :read [10 nil] INFO [2019-12-17 12:35:47,131] jepsen worker 48 - jepsen.util 12288 :info :cas [16 [36 16]] :RequestCanceledException INFO [2019-12-17 12:35:47,131] jepsen worker 51 - jepsen.util 9231 :info :write [17 19] :RequestCanceledException INFO [2019-12-17 12:35:47,132] jepsen worker 51 - jepsen.util 9321 :invoke :write [17 24] INFO [2019-12-17 12:35:47,133] jepsen worker 48 - jepsen.util 12378 :invoke :write [16 5] INFO [2019-12-17 12:35:47,147] jepsen worker 64 - jepsen.util 7624 :fail :cas [21 [20 21]] :ValueNotSwapFrom INFO [2019-12-17 12:35:47,147] jepsen worker 64 - jepsen.util 7624 :invoke :read [21 nil] INFO [2019-12-17 12:35:47,151] jepsen worker 19 - jepsen.util 10189 :fail :cas [6 [43 21]] :ValueNotSwapFrom INFO [2019-12-17 12:35:47,151] jepsen worker 19 - jepsen.util 10189 :invoke :write [6 36] INFO [2019-12-17 12:35:47,162] jepsen worker 33 - jepsen.util 13533 :info :cas [11 [6 46]] :RequestCanceledException INFO [2019-12-17 12:35:47,162] jepsen worker 30 - jepsen.util 10020 :info :write [10 24] :RequestCanceledException INFO [2019-12-17 12:35:47,165] jepsen worker 33 - jepsen.util 13623 :invoke :write [11 47] INFO [2019-12-17 12:35:47,165] jepsen worker 30 - jepsen.util 10110 :invoke :read [10 nil] INFO [2019-12-17 12:35:47,170] jepsen worker 64 - jepsen.util 7624 :ok :read [21 27] INFO [2019-12-17 12:35:47,170] jepsen worker 64 - jepsen.util 7624 :invoke :cas [21 [49 41]] INFO [2019-12-17 12:35:47,189] jepsen worker 64 - jepsen.util 7624 :fail :cas [21 [49 41]] :ValueNotSwapFrom INFO [2019-12-17 12:35:47,189] jepsen worker 64 - jepsen.util 7624 :invoke :write [21 41] INFO [2019-12-17 12:35:47,197] jepsen worker 38 - jepsen.util 10568 :info :write [12 36] :RequestCanceledException INFO [2019-12-17 12:35:47,197] jepsen worker 88 - jepsen.util 10078 :info :write [29 28] :RequestCanceledException INFO [2019-12-17 12:35:47,198] jepsen worker 88 - jepsen.util 10168 :invoke :read [29 nil] INFO [2019-12-17 12:35:47,198] jepsen worker 38 - jepsen.util 10658 :invoke :write [12 25] INFO [2019-12-17 12:35:47,230] jepsen worker 56 - jepsen.util 10406 :info :write [18 43] :RequestCanceledException INFO [2019-12-17 12:35:47,231] jepsen worker 77 - jepsen.util 14297 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:35:47,231] jepsen worker 77 - jepsen.util 14297 :invoke :read [25 nil] INFO [2019-12-17 12:35:47,231] jepsen worker 56 - jepsen.util 10496 :invoke :write [18 38] INFO [2019-12-17 12:35:47,236] jepsen worker 79 - jepsen.util 10159 :ok :read [26 32] INFO [2019-12-17 12:35:47,236] jepsen worker 79 - jepsen.util 10159 :invoke :cas [26 [10 23]] INFO [2019-12-17 12:35:47,251] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":57384,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:35:47,251] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":57395,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:35:47,251] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":57384,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:35:47,251] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":57395,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:35:47,251] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":57385,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:35:47,252] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":57397,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:35:47,252] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":57385,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:35:47,252] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":57397,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:35:47,258] jepsen worker 64 - jepsen.util 7624 :ok :write [21 41] INFO [2019-12-17 12:35:47,258] jepsen worker 64 - jepsen.util 7624 :invoke :read [21 nil] INFO [2019-12-17 12:35:47,272] jepsen worker 64 - jepsen.util 7624 :ok :read [21 41] INFO [2019-12-17 12:35:47,272] jepsen worker 64 - jepsen.util 7624 :invoke :write [21 19] INFO [2019-12-17 12:35:47,310] jepsen worker 64 - jepsen.util 7624 :ok :write [21 19] INFO [2019-12-17 12:35:47,310] jepsen worker 64 - jepsen.util 7624 :invoke :read [21 nil] INFO [2019-12-17 12:35:47,315] jepsen worker 55 - jepsen.util 10585 :info :write [18 37] :RequestCanceledException INFO [2019-12-17 12:35:47,316] jepsen worker 55 - jepsen.util 10675 :invoke :read [18 nil] INFO [2019-12-17 12:35:47,330] jepsen worker 64 - jepsen.util 7624 :ok :read [21 19] INFO [2019-12-17 12:35:47,330] jepsen worker 64 - jepsen.util 7624 :invoke :read [21 nil] INFO [2019-12-17 12:35:47,342] jepsen worker 64 - jepsen.util 7624 :ok :read [21 19] INFO [2019-12-17 12:35:47,342] jepsen worker 64 - jepsen.util 7624 :invoke :cas [21 [1 30]] INFO [2019-12-17 12:35:47,345] jepsen worker 74 - jepsen.util 9614 :fail :read [24 nil] :RequestTimeoutException INFO [2019-12-17 12:35:47,345] jepsen worker 74 - jepsen.util 9614 :invoke :read [24 nil] INFO [2019-12-17 12:35:47,363] jepsen worker 64 - jepsen.util 7624 :fail :cas [21 [1 30]] :ValueNotSwapFrom INFO [2019-12-17 12:35:47,363] jepsen worker 64 - jepsen.util 7624 :invoke :read [21 nil] INFO [2019-12-17 12:35:47,370] jepsen worker 4 - jepsen.util 9184 :fail :cas [1 [24 28]] :ValueNotSwapFrom INFO [2019-12-17 12:35:47,370] jepsen worker 4 - jepsen.util 9184 :invoke :read [1 nil] INFO [2019-12-17 12:35:47,377] jepsen worker 64 - jepsen.util 7624 :ok :read [21 19] INFO [2019-12-17 12:35:47,377] jepsen worker 64 - jepsen.util 7624 :invoke :cas [21 [31 32]] INFO [2019-12-17 12:35:47,388] jepsen worker 52 - jepsen.util 9862 :fail :cas [17 [42 17]] :ValueNotSwapFrom INFO [2019-12-17 12:35:47,388] jepsen worker 52 - jepsen.util 9862 :invoke :write [17 45] INFO [2019-12-17 12:35:47,388] jepsen worker 64 - jepsen.util 7624 :fail :cas [21 [31 32]] :ValueNotSwapFrom INFO [2019-12-17 12:35:47,388] jepsen worker 64 - jepsen.util 7624 :invoke :write [21 15] INFO [2019-12-17 12:35:47,397] jepsen worker 59 - jepsen.util 12839 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:35:47,397] jepsen worker 59 - jepsen.util 12839 :invoke :write [19 7] INFO [2019-12-17 12:35:47,428] jepsen worker 55 - jepsen.util 10675 :ok :read [18 5] INFO [2019-12-17 12:35:47,428] jepsen worker 55 - jepsen.util 10675 :invoke :cas [18 [22 41]] INFO [2019-12-17 12:35:47,430] jepsen worker 67 - jepsen.util 13477 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:35:47,430] jepsen worker 11 - jepsen.util 9731 :info :write [3 0] :RequestCanceledException INFO [2019-12-17 12:35:47,431] jepsen worker 67 - jepsen.util 13477 :invoke :write [22 40] INFO [2019-12-17 12:35:47,432] jepsen worker 11 - jepsen.util 9821 :invoke :cas [3 [46 13]] INFO [2019-12-17 12:35:47,435] jepsen worker 64 - jepsen.util 7624 :ok :write [21 15] INFO [2019-12-17 12:35:47,435] jepsen worker 64 - jepsen.util 7624 :invoke :write [21 27] INFO [2019-12-17 12:35:47,445] jepsen worker 8 - jepsen.util 13148 :info :write [2 7] :RequestCanceledException INFO [2019-12-17 12:35:47,446] jepsen worker 8 - jepsen.util 13238 :invoke :write [2 19] INFO [2019-12-17 12:35:47,458] jepsen worker 87 - jepsen.util 9537 :fail :cas [29 [38 8]] :ValueNotSwapFrom INFO [2019-12-17 12:35:47,458] jepsen worker 87 - jepsen.util 9537 :invoke :cas [29 [21 17]] INFO [2019-12-17 12:35:47,463] jepsen worker 21 - jepsen.util 13071 :info :write [7 12] :RequestCanceledException INFO [2019-12-17 12:35:47,463] jepsen worker 84 - jepsen.util 13134 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:35:47,463] jepsen worker 84 - jepsen.util 13134 :invoke :write [28 17] INFO [2019-12-17 12:35:47,464] jepsen worker 21 - jepsen.util 13161 :invoke :write [7 20] INFO [2019-12-17 12:35:47,470] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":57417,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:35:47,470] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":57417,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:35:47,473] jepsen worker 4 - jepsen.util 9184 :ok :read [1 6] INFO [2019-12-17 12:35:47,473] jepsen worker 4 - jepsen.util 9184 :invoke :write [1 25] INFO [2019-12-17 12:35:47,483] jepsen worker 64 - jepsen.util 7624 :ok :write [21 27] INFO [2019-12-17 12:35:47,483] jepsen worker 64 - jepsen.util 7624 :invoke :write [21 12] INFO [2019-12-17 12:35:47,498] jepsen worker 34 - jepsen.util 13534 :info :cas [11 [35 10]] :RequestCanceledException INFO [2019-12-17 12:35:47,498] jepsen worker 34 - jepsen.util 13624 :invoke :write [11 28] INFO [2019-12-17 12:35:47,515] jepsen worker 86 - jepsen.util 14396 :info :write [28 2] :RequestCanceledException INFO [2019-12-17 12:35:47,515] jepsen worker 50 - jepsen.util 14180 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:35:47,515] jepsen worker 29 - jepsen.util 8219 :info :write [9 31] :RequestCanceledException INFO [2019-12-17 12:35:47,515] jepsen worker 50 - jepsen.util 14180 :invoke :cas [16 [31 40]] INFO [2019-12-17 12:35:47,516] jepsen worker 29 - jepsen.util 8309 :invoke :cas [9 [35 11]] INFO [2019-12-17 12:35:47,517] jepsen worker 86 - jepsen.util 14486 :invoke :write [28 46] INFO [2019-12-17 12:35:47,530] jepsen worker 64 - jepsen.util 7624 :ok :write [21 12] INFO [2019-12-17 12:35:47,530] jepsen worker 64 - jepsen.util 7624 :invoke :write [21 44] INFO [2019-12-17 12:35:47,531] jepsen worker 16 - jepsen.util 14776 :info :cas [5 [35 42]] :RequestCanceledException INFO [2019-12-17 12:35:47,531] jepsen worker 41 - jepsen.util 14531 :info :write [13 1] :RequestCanceledException INFO [2019-12-17 12:35:47,532] jepsen worker 16 - jepsen.util 14866 :invoke :write [5 0] INFO [2019-12-17 12:35:47,533] jepsen worker 41 - jepsen.util 14621 :invoke :cas [13 [37 22]] INFO [2019-12-17 12:35:47,538] jepsen worker 22 - jepsen.util 13072 :info :cas [7 [42 16]] :RequestCanceledException INFO [2019-12-17 12:35:47,540] jepsen worker 22 - jepsen.util 13162 :invoke :write [7 5] INFO [2019-12-17 12:35:47,550] jepsen worker 29 - jepsen.util 8309 :fail :cas [9 [35 11]] :ValueNotSwapFrom INFO [2019-12-17 12:35:47,550] jepsen worker 29 - jepsen.util 8309 :invoke :read [9 nil] INFO [2019-12-17 12:35:47,551] jepsen worker 87 - jepsen.util 9537 :fail :cas [29 [21 17]] :ValueNotSwapFrom INFO [2019-12-17 12:35:47,552] jepsen worker 87 - jepsen.util 9537 :invoke :write [29 5] INFO [2019-12-17 12:35:47,563] jepsen worker 2 - jepsen.util 13232 :info :write [0 46] :RequestCanceledException INFO [2019-12-17 12:35:47,564] jepsen worker 2 - jepsen.util 13322 :invoke :cas [0 [22 14]] INFO [2019-12-17 12:35:47,565] jepsen worker 13 - jepsen.util 9373 :fail :cas [4 [31 46]] :ValueNotSwapFrom INFO [2019-12-17 12:35:47,565] jepsen worker 13 - jepsen.util 9373 :invoke :read [4 nil] INFO [2019-12-17 12:35:47,580] jepsen worker 88 - jepsen.util 10168 :ok :read [29 43] INFO [2019-12-17 12:35:47,580] jepsen worker 88 - jepsen.util 10168 :invoke :cas [29 [5 8]] INFO [2019-12-17 12:35:47,590] jepsen worker 64 - jepsen.util 7624 :ok :write [21 44] INFO [2019-12-17 12:35:47,591] jepsen worker 64 - jepsen.util 7624 :invoke :read [21 nil] INFO [2019-12-17 12:35:47,597] jepsen worker 15 - jepsen.util 12615 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:35:47,598] jepsen worker 15 - jepsen.util 12615 :invoke :read [5 nil] INFO [2019-12-17 12:35:47,604] jepsen worker 29 - jepsen.util 8309 :ok :read [9 13] INFO [2019-12-17 12:35:47,604] jepsen worker 29 - jepsen.util 8309 :invoke :cas [9 [7 5]] INFO [2019-12-17 12:35:47,631] jepsen worker 29 - jepsen.util 8309 :fail :cas [9 [7 5]] :ValueNotSwapFrom INFO [2019-12-17 12:35:47,631] jepsen worker 29 - jepsen.util 8309 :invoke :write [9 14] INFO [2019-12-17 12:35:47,631] jepsen worker 37 - jepsen.util 9397 :info :write [12 23] :RequestCanceledException INFO [2019-12-17 12:35:47,632] jepsen worker 37 - jepsen.util 9487 :invoke :write [12 43] INFO [2019-12-17 12:35:47,632] jepsen worker 64 - jepsen.util 7624 :ok :read [21 44] INFO [2019-12-17 12:35:47,632] jepsen worker 64 - jepsen.util 7624 :invoke :write [21 14] INFO [2019-12-17 12:35:47,638] jepsen worker 63 - jepsen.util 8883 :info :write [21 32] :RequestCanceledException INFO [2019-12-17 12:35:47,639] jepsen worker 63 - jepsen.util 8973 :invoke :write [21 9] INFO [2019-12-17 12:35:47,674] jepsen worker 29 - jepsen.util 8309 :ok :write [9 14] INFO [2019-12-17 12:35:47,674] jepsen worker 29 - jepsen.util 8309 :invoke :read [9 nil] INFO [2019-12-17 12:35:47,681] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":57453,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:35:47,681] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":57453,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:35:47,681] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":57455,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:35:47,681] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":57455,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:35:47,684] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:35:47,684] jepsen nemesis - couchbase.util Rebalance check count: 449 INFO [2019-12-17 12:35:47,684] jepsen nemesis - couchbase.util Rebalance stuck count: 430 INFO [2019-12-17 12:35:47,688] jepsen worker 73 - jepsen.util 9433 :ok :write [24 20] INFO [2019-12-17 12:35:47,689] jepsen worker 73 - jepsen.util 9433 :invoke :cas [24 [39 38]] INFO [2019-12-17 12:35:47,689] jepsen worker 29 - jepsen.util 8309 :ok :read [9 14] INFO [2019-12-17 12:35:47,689] jepsen worker 9 - jepsen.util 10629 :ok :write [3 39] INFO [2019-12-17 12:35:47,689] jepsen worker 29 - jepsen.util 8309 :invoke :write [9 3] INFO [2019-12-17 12:35:47,691] jepsen worker 9 - jepsen.util 10629 :invoke :read [3 nil] INFO [2019-12-17 12:35:47,697] jepsen worker 19 - jepsen.util 10189 :ok :write [6 36] INFO [2019-12-17 12:35:47,697] jepsen worker 19 - jepsen.util 10189 :invoke :read [6 nil] INFO [2019-12-17 12:35:47,698] jepsen worker 43 - jepsen.util 13453 :info :cas [14 [34 35]] :RequestCanceledException INFO [2019-12-17 12:35:47,699] jepsen worker 43 - jepsen.util 13543 :invoke :read [14 nil] INFO [2019-12-17 12:35:47,716] jepsen worker 46 - jepsen.util 9766 :info :write [15 4] :RequestCanceledException INFO [2019-12-17 12:35:47,717] jepsen worker 46 - jepsen.util 9856 :invoke :read [15 nil] INFO [2019-12-17 12:35:47,722] jepsen worker 31 - jepsen.util 8941 :ok :read [10 38] INFO [2019-12-17 12:35:47,722] jepsen worker 31 - jepsen.util 8941 :invoke :write [10 8] INFO [2019-12-17 12:35:47,731] jepsen worker 5 - jepsen.util 9275 :info :write [1 6] :RequestCanceledException INFO [2019-12-17 12:35:47,732] jepsen worker 5 - jepsen.util 9365 :invoke :cas [1 [27 35]] INFO [2019-12-17 12:35:47,748] jepsen worker 46 - jepsen.util 9856 :ok :read [15 22] INFO [2019-12-17 12:35:47,748] jepsen worker 46 - jepsen.util 9856 :invoke :cas [15 [14 45]] INFO [2019-12-17 12:35:47,753] jepsen worker 11 - jepsen.util 9821 :fail :cas [3 [46 13]] :ValueNotSwapFrom INFO [2019-12-17 12:35:47,753] jepsen worker 11 - jepsen.util 9821 :invoke :cas [3 [5 16]] INFO [2019-12-17 12:35:47,755] jepsen worker 74 - jepsen.util 9614 :ok :read [24 20] INFO [2019-12-17 12:35:47,756] jepsen worker 74 - jepsen.util 9614 :invoke :cas [24 [29 25]] INFO [2019-12-17 12:35:47,782] jepsen worker 46 - jepsen.util 9856 :fail :cas [15 [14 45]] :ValueNotSwapFrom INFO [2019-12-17 12:35:47,782] jepsen worker 9 - jepsen.util 10629 :ok :read [3 39] INFO [2019-12-17 12:35:47,782] jepsen worker 46 - jepsen.util 9856 :invoke :write [15 36] INFO [2019-12-17 12:35:47,783] jepsen worker 9 - jepsen.util 10629 :invoke :write [3 0] INFO [2019-12-17 12:35:47,790] jepsen worker 19 - jepsen.util 10189 :ok :read [6 36] INFO [2019-12-17 12:35:47,790] jepsen worker 19 - jepsen.util 10189 :invoke :write [6 36] INFO [2019-12-17 12:35:47,797] jepsen worker 42 - jepsen.util 14442 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:35:47,798] jepsen worker 42 - jepsen.util 14442 :invoke :write [14 16] INFO [2019-12-17 12:35:47,801] jepsen worker 74 - jepsen.util 9614 :fail :cas [24 [29 25]] :ValueNotSwapFrom INFO [2019-12-17 12:35:47,801] jepsen worker 74 - jepsen.util 9614 :invoke :write [24 15] INFO [2019-12-17 12:35:47,817] jepsen worker 17 - jepsen.util 13247 :info :write [5 45] :RequestCanceledException INFO [2019-12-17 12:35:47,829] jepsen worker 17 - jepsen.util 13337 :invoke :cas [5 [13 22]] INFO [2019-12-17 12:35:47,831] jepsen worker 57 - jepsen.util 13107 :info :write [19 37] :RequestCanceledException INFO [2019-12-17 12:35:47,831] jepsen worker 60 - jepsen.util 14370 :info :write [20 2] :RequestCanceledException INFO [2019-12-17 12:35:47,833] jepsen worker 60 - jepsen.util 14460 :invoke :cas [20 [30 6]] INFO [2019-12-17 12:35:47,833] jepsen worker 57 - jepsen.util 13197 :invoke :read [19 nil] INFO [2019-12-17 12:35:47,864] jepsen worker 5 - jepsen.util 9365 :fail :cas [1 [27 35]] :ValueNotSwapFrom INFO [2019-12-17 12:35:47,864] jepsen worker 5 - jepsen.util 9365 :invoke :write [1 19] INFO [2019-12-17 12:35:47,871] jepsen worker 63 - jepsen.util 8973 :ok :write [21 9] INFO [2019-12-17 12:35:47,871] jepsen worker 63 - jepsen.util 8973 :invoke :read [21 nil] INFO [2019-12-17 12:35:47,894] jepsen worker 63 - jepsen.util 8973 :ok :read [21 9] INFO [2019-12-17 12:35:47,894] jepsen worker 63 - jepsen.util 8973 :invoke :write [21 42] INFO [2019-12-17 12:35:47,900] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":57477,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:35:47,900] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":57477,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:35:47,900] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":57480,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:35:47,900] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":57480,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:35:47,900] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":57481,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:35:47,900] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":57481,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:35:47,902] jepsen worker 29 - jepsen.util 8309 :ok :write [9 3] INFO [2019-12-17 12:35:47,902] jepsen worker 29 - jepsen.util 8309 :invoke :write [9 45] INFO [2019-12-17 12:35:47,957] jepsen worker 78 - jepsen.util 9888 :ok :write [26 7] INFO [2019-12-17 12:35:47,957] jepsen worker 78 - jepsen.util 9888 :invoke :read [26 nil] INFO [2019-12-17 12:35:47,959] jepsen worker 31 - jepsen.util 8941 :ok :write [10 8] INFO [2019-12-17 12:35:47,959] jepsen worker 31 - jepsen.util 8941 :invoke :cas [10 [6 41]] INFO [2019-12-17 12:35:47,960] jepsen worker 6 - jepsen.util 14046 :info :cas [2 [38 20]] :RequestCanceledException INFO [2019-12-17 12:35:47,960] jepsen worker 76 - jepsen.util 13756 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:35:47,960] jepsen worker 76 - jepsen.util 13756 :invoke :write [25 14] INFO [2019-12-17 12:35:47,961] jepsen worker 6 - jepsen.util 14136 :invoke :read [2 nil] INFO [2019-12-17 12:35:47,973] jepsen worker 69 - jepsen.util 9969 :ok :write [23 11] INFO [2019-12-17 12:35:47,973] jepsen worker 69 - jepsen.util 9969 :invoke :write [23 37] INFO [2019-12-17 12:35:47,981] jepsen worker 11 - jepsen.util 9821 :fail :cas [3 [5 16]] :ValueNotSwapFrom INFO [2019-12-17 12:35:47,981] jepsen worker 11 - jepsen.util 9821 :invoke :read [3 nil] INFO [2019-12-17 12:35:47,982] jepsen worker 79 - jepsen.util 10159 :fail :cas [26 [10 23]] :ValueNotSwapFrom INFO [2019-12-17 12:35:47,982] jepsen worker 79 - jepsen.util 10159 :invoke :write [26 26] INFO [2019-12-17 12:35:47,997] jepsen worker 46 - jepsen.util 9856 :ok :write [15 36] INFO [2019-12-17 12:35:47,997] jepsen worker 46 - jepsen.util 9856 :invoke :read [15 nil] INFO [2019-12-17 12:35:48,006] jepsen worker 63 - jepsen.util 8973 :ok :write [21 42] INFO [2019-12-17 12:35:48,006] jepsen worker 63 - jepsen.util 8973 :invoke :cas [21 [5 9]] INFO [2019-12-17 12:35:48,016] jepsen worker 29 - jepsen.util 8309 :ok :write [9 45] INFO [2019-12-17 12:35:48,016] jepsen worker 29 - jepsen.util 8309 :invoke :cas [9 [19 26]] INFO [2019-12-17 12:35:48,022] jepsen worker 63 - jepsen.util 8973 :fail :cas [21 [5 9]] :ValueNotSwapFrom INFO [2019-12-17 12:35:48,022] jepsen worker 63 - jepsen.util 8973 :invoke :read [21 nil] INFO [2019-12-17 12:35:48,023] jepsen worker 46 - jepsen.util 9856 :ok :read [15 36] INFO [2019-12-17 12:35:48,023] jepsen worker 46 - jepsen.util 9856 :invoke :write [15 34] INFO [2019-12-17 12:35:48,030] jepsen worker 36 - jepsen.util 10566 :info :write [12 18] :RequestCanceledException INFO [2019-12-17 12:35:48,030] jepsen worker 24 - jepsen.util 12354 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:35:48,031] jepsen worker 24 - jepsen.util 12354 :invoke :read [8 nil] INFO [2019-12-17 12:35:48,031] jepsen worker 36 - jepsen.util 10656 :invoke :write [12 4] INFO [2019-12-17 12:35:48,034] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":57495,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:35:48,034] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":57495,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:35:48,034] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":57498,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:35:48,034] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":57498,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:35:48,041] jepsen worker 29 - jepsen.util 8309 :fail :cas [9 [19 26]] :ValueNotSwapFrom INFO [2019-12-17 12:35:48,041] jepsen worker 29 - jepsen.util 8309 :invoke :read [9 nil] INFO [2019-12-17 12:35:48,045] jepsen worker 65 - jepsen.util 8435 :info :write [21 28] :RequestCanceledException INFO [2019-12-17 12:35:48,046] jepsen worker 65 - jepsen.util 8525 :invoke :write [21 21] INFO [2019-12-17 12:35:48,054] jepsen worker 63 - jepsen.util 8973 :ok :read [21 42] INFO [2019-12-17 12:35:48,054] jepsen worker 63 - jepsen.util 8973 :invoke :write [21 38] INFO [2019-12-17 12:35:48,058] jepsen worker 55 - jepsen.util 10675 :fail :cas [18 [22 41]] :ValueNotSwapFrom INFO [2019-12-17 12:35:48,058] jepsen worker 55 - jepsen.util 10675 :invoke :write [18 17] INFO [2019-12-17 12:35:48,066] jepsen worker 29 - jepsen.util 8309 :ok :read [9 45] INFO [2019-12-17 12:35:48,066] jepsen worker 29 - jepsen.util 8309 :invoke :write [9 44] INFO [2019-12-17 12:35:48,098] jepsen worker 44 - jepsen.util 13814 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:35:48,098] jepsen worker 44 - jepsen.util 13814 :invoke :cas [14 [33 10]] INFO [2019-12-17 12:35:48,114] jepsen worker 29 - jepsen.util 8309 :ok :write [9 44] INFO [2019-12-17 12:35:48,114] jepsen worker 29 - jepsen.util 8309 :invoke :write [9 49] INFO [2019-12-17 12:35:48,131] jepsen worker 32 - jepsen.util 9932 :fail :read [10 nil] :RequestTimeoutException INFO [2019-12-17 12:35:48,131] jepsen worker 32 - jepsen.util 9932 :invoke :write [10 44] INFO [2019-12-17 12:35:48,145] jepsen worker 27 - jepsen.util 9477 :fail :read [9 nil] :RequestTimeoutException INFO [2019-12-17 12:35:48,146] jepsen worker 27 - jepsen.util 9477 :invoke :write [9 1] INFO [2019-12-17 12:35:48,174] jepsen worker 13 - jepsen.util 9373 :ok :read [4 14] INFO [2019-12-17 12:35:48,174] jepsen worker 13 - jepsen.util 9373 :invoke :cas [4 [42 40]] INFO [2019-12-17 12:35:48,183] jepsen worker 63 - jepsen.util 8973 :ok :write [21 38] INFO [2019-12-17 12:35:48,184] jepsen worker 63 - jepsen.util 8973 :invoke :read [21 nil] INFO [2019-12-17 12:35:48,198] jepsen worker 29 - jepsen.util 8309 :ok :write [9 49] INFO [2019-12-17 12:35:48,199] jepsen worker 29 - jepsen.util 8309 :invoke :write [9 38] INFO [2019-12-17 12:35:48,211] jepsen worker 63 - jepsen.util 8973 :ok :read [21 38] INFO [2019-12-17 12:35:48,212] jepsen worker 63 - jepsen.util 8973 :invoke :read [21 nil] INFO [2019-12-17 12:35:48,226] jepsen worker 63 - jepsen.util 8973 :ok :read [21 38] INFO [2019-12-17 12:35:48,226] jepsen worker 63 - jepsen.util 8973 :invoke :read [21 nil] INFO [2019-12-17 12:35:48,231] jepsen worker 35 - jepsen.util 13355 :info :cas [11 [10 25]] :RequestCanceledException INFO [2019-12-17 12:35:48,232] jepsen worker 35 - jepsen.util 13445 :invoke :cas [11 [44 22]] INFO [2019-12-17 12:35:48,236] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":57517,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:35:48,236] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":57517,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:35:48,236] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":57518,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:35:48,236] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":57518,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:35:48,236] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":57526,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:35:48,236] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":57519,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:35:48,237] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":57526,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:35:48,237] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":57519,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:35:48,237] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":57529,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:35:48,237] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":57529,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:35:48,237] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":57530,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:35:48,237] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":57530,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:35:48,244] jepsen worker 62 - jepsen.util 13922 :info :cas [20 [11 20]] :RequestCanceledException INFO [2019-12-17 12:35:48,245] jepsen worker 62 - jepsen.util 14012 :invoke :read [20 nil] INFO [2019-12-17 12:35:48,250] jepsen worker 63 - jepsen.util 8973 :ok :read [21 38] INFO [2019-12-17 12:35:48,250] jepsen worker 63 - jepsen.util 8973 :invoke :read [21 nil] INFO [2019-12-17 12:35:48,259] jepsen worker 13 - jepsen.util 9373 :fail :cas [4 [42 40]] :ValueNotSwapFrom INFO [2019-12-17 12:35:48,259] jepsen worker 13 - jepsen.util 9373 :invoke :read [4 nil] INFO [2019-12-17 12:35:48,260] jepsen worker 63 - jepsen.util 8973 :ok :read [21 38] INFO [2019-12-17 12:35:48,261] jepsen worker 63 - jepsen.util 8973 :invoke :read [21 nil] INFO [2019-12-17 12:35:48,262] jepsen worker 20 - jepsen.util 9650 :info :write [6 27] :RequestCanceledException INFO [2019-12-17 12:35:48,263] jepsen worker 20 - jepsen.util 9740 :invoke :cas [6 [28 44]] INFO [2019-12-17 12:35:48,277] jepsen worker 63 - jepsen.util 8973 :ok :read [21 38] INFO [2019-12-17 12:35:48,277] jepsen worker 63 - jepsen.util 8973 :invoke :read [21 nil] INFO [2019-12-17 12:35:48,283] jepsen worker 88 - jepsen.util 10168 :fail :cas [29 [5 8]] :ValueNotSwapFrom INFO [2019-12-17 12:35:48,283] jepsen worker 88 - jepsen.util 10168 :invoke :read [29 nil] INFO [2019-12-17 12:35:48,285] jepsen worker 20 - jepsen.util 9740 :fail :cas [6 [28 44]] :ValueNotSwapFrom INFO [2019-12-17 12:35:48,285] jepsen worker 20 - jepsen.util 9740 :invoke :read [6 nil] INFO [2019-12-17 12:35:48,295] jepsen worker 63 - jepsen.util 8973 :ok :read [21 38] INFO [2019-12-17 12:35:48,295] jepsen worker 63 - jepsen.util 8973 :invoke :read [21 nil] INFO [2019-12-17 12:35:48,301] jepsen worker 20 - jepsen.util 9740 :ok :read [6 36] INFO [2019-12-17 12:35:48,301] jepsen worker 20 - jepsen.util 9740 :invoke :cas [6 [41 10]] INFO [2019-12-17 12:35:48,310] jepsen worker 63 - jepsen.util 8973 :ok :read [21 38] INFO [2019-12-17 12:35:48,310] jepsen worker 63 - jepsen.util 8973 :invoke :read [21 nil] INFO [2019-12-17 12:35:48,312] jepsen worker 19 - jepsen.util 10189 :ok :write [6 36] INFO [2019-12-17 12:35:48,312] jepsen worker 19 - jepsen.util 10189 :invoke :read [6 nil] INFO [2019-12-17 12:35:48,316] jepsen worker 12 - jepsen.util 10182 :info :write [4 6] :RequestCanceledException INFO [2019-12-17 12:35:48,317] jepsen worker 12 - jepsen.util 10272 :invoke :write [4 13] INFO [2019-12-17 12:35:48,331] jepsen worker 85 - jepsen.util 14305 :info :write [28 18] :RequestCanceledException INFO [2019-12-17 12:35:48,339] jepsen worker 20 - jepsen.util 9740 :fail :cas [6 [41 10]] :ValueNotSwapFrom INFO [2019-12-17 12:35:48,344] jepsen worker 54 - jepsen.util 10854 :info :write [18 37] :RequestCanceledException INFO [2019-12-17 12:35:48,344] jepsen worker 23 - jepsen.util 13613 :info :write [7 11] :RequestCanceledException INFO [2019-12-17 12:35:48,356] jepsen worker 63 - jepsen.util 8973 :ok :read [21 38] INFO [2019-12-17 12:35:48,362] jepsen worker 47 - jepsen.util 10487 :fail :read [15 nil] :RequestTimeoutException INFO [2019-12-17 12:35:48,366] jepsen worker 65 - jepsen.util 8525 :ok :write [21 21] INFO [2019-12-17 12:35:48,397] jepsen worker 83 - jepsen.util 13943 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:35:48,397] jepsen worker 53 - jepsen.util 9413 :info :write [17 0] :RequestCanceledException INFO [2019-12-17 12:35:48,411] jepsen worker 88 - jepsen.util 10168 :ok :read [29 43] INFO [2019-12-17 12:35:48,414] jepsen worker 40 - jepsen.util 13900 :info :write [13 40] :RequestCanceledException INFO [2019-12-17 12:35:48,426] jepsen worker 9 - jepsen.util 10629 :ok :write [3 0] INFO [2019-12-17 12:35:48,426] jepsen worker 46 - jepsen.util 9856 :ok :write [15 34] INFO [2019-12-17 12:35:48,427] jepsen worker 20 - jepsen.util 9740 :invoke :write [6 14] INFO [2019-12-17 12:35:48,427] jepsen worker 63 - jepsen.util 8973 :invoke :write [21 10] INFO [2019-12-17 12:35:48,427] jepsen worker 47 - jepsen.util 10487 :invoke :read [15 nil] INFO [2019-12-17 12:35:48,427] jepsen worker 65 - jepsen.util 8525 :invoke :read [21 nil] INFO [2019-12-17 12:35:48,427] jepsen worker 83 - jepsen.util 13943 :invoke :write [27 15] INFO [2019-12-17 12:35:48,427] jepsen worker 88 - jepsen.util 10168 :invoke :read [29 nil] INFO [2019-12-17 12:35:48,427] jepsen worker 9 - jepsen.util 10629 :invoke :read [3 nil] INFO [2019-12-17 12:35:48,428] jepsen worker 46 - jepsen.util 9856 :invoke :cas [15 [30 22]] INFO [2019-12-17 12:35:48,428] jepsen worker 54 - jepsen.util 10944 :invoke :write [18 20] INFO [2019-12-17 12:35:48,428] jepsen worker 23 - jepsen.util 13703 :invoke :read [7 nil] INFO [2019-12-17 12:35:48,428] jepsen worker 53 - jepsen.util 9503 :invoke :read [17 nil] INFO [2019-12-17 12:35:48,428] jepsen worker 40 - jepsen.util 13990 :invoke :write [13 10] INFO [2019-12-17 12:35:48,428] jepsen worker 28 - jepsen.util 8128 :info :write [9 48] :RequestCanceledException INFO [2019-12-17 12:35:48,428] jepsen worker 14 - jepsen.util 9644 :info :write [4 33] :RequestCanceledException INFO [2019-12-17 12:35:48,429] jepsen worker 85 - jepsen.util 14395 :invoke :cas [28 [23 14]] INFO [2019-12-17 12:35:48,430] jepsen worker 28 - jepsen.util 8218 :invoke :write [9 10] INFO [2019-12-17 12:35:48,430] jepsen worker 14 - jepsen.util 9734 :invoke :cas [4 [13 16]] INFO [2019-12-17 12:35:48,445] jepsen worker 70 - jepsen.util 10690 :info :write [23 31] :RequestCanceledException INFO [2019-12-17 12:35:48,447] jepsen worker 70 - jepsen.util 10780 :invoke :read [23 nil] INFO [2019-12-17 12:35:48,450] jepsen worker 78 - jepsen.util 9888 :ok :read [26 7] INFO [2019-12-17 12:35:48,451] jepsen worker 78 - jepsen.util 9888 :invoke :write [26 10] INFO [2019-12-17 12:35:48,455] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":57553,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:35:48,455] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":57553,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:35:48,455] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":57558,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:35:48,455] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":57558,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:35:48,516] jepsen worker 71 - jepsen.util 10601 :info :write [23 16] :RequestCanceledException INFO [2019-12-17 12:35:48,516] jepsen worker 10 - jepsen.util 9910 :fail :read [3 nil] :RequestTimeoutException INFO [2019-12-17 12:35:48,516] jepsen worker 45 - jepsen.util 10215 :info :write [15 43] :RequestCanceledException INFO [2019-12-17 12:35:48,516] jepsen worker 10 - jepsen.util 9910 :invoke :cas [3 [13 49]] INFO [2019-12-17 12:35:48,516] jepsen worker 71 - jepsen.util 10691 :invoke :write [23 23] INFO [2019-12-17 12:35:48,516] jepsen worker 45 - jepsen.util 10305 :invoke :cas [15 [40 16]] INFO [2019-12-17 12:35:48,531] jepsen worker 49 - jepsen.util 13459 :info :write [16 40] :RequestCanceledException INFO [2019-12-17 12:35:48,531] jepsen worker 66 - jepsen.util 13206 :info :cas [22 [17 43]] :RequestCanceledException INFO [2019-12-17 12:35:48,532] jepsen worker 66 - jepsen.util 13296 :invoke :read [22 nil] INFO [2019-12-17 12:35:48,532] jepsen worker 49 - jepsen.util 13549 :invoke :read [16 nil] INFO [2019-12-17 12:35:48,545] jepsen worker 0 - jepsen.util 14400 :info :cas [0 [26 7]] :RequestCanceledException INFO [2019-12-17 12:35:48,546] jepsen worker 0 - jepsen.util 14490 :invoke :read [0 nil] INFO [2019-12-17 12:35:48,562] jepsen worker 26 - jepsen.util 14156 :info :cas [8 [13 19]] :RequestCanceledException INFO [2019-12-17 12:35:48,563] jepsen worker 26 - jepsen.util 14246 :invoke :write [8 42] INFO [2019-12-17 12:35:48,571] jepsen worker 47 - jepsen.util 10487 :ok :read [15 34] INFO [2019-12-17 12:35:48,571] jepsen worker 47 - jepsen.util 10487 :invoke :read [15 nil] INFO [2019-12-17 12:35:48,585] jepsen worker 14 - jepsen.util 9734 :fail :cas [4 [13 16]] :ValueNotSwapFrom INFO [2019-12-17 12:35:48,585] jepsen worker 14 - jepsen.util 9734 :invoke :write [4 16] INFO [2019-12-17 12:35:48,598] jepsen worker 7 - jepsen.util 13597 :info :cas [2 [6 36]] :RequestCanceledException INFO [2019-12-17 12:35:48,599] jepsen worker 7 - jepsen.util 13687 :invoke :read [2 nil] INFO [2019-12-17 12:35:48,615] jepsen worker 61 - jepsen.util 14371 :info :cas [20 [43 29]] :RequestCanceledException INFO [2019-12-17 12:35:48,620] jepsen worker 61 - jepsen.util 14461 :invoke :read [20 nil] INFO [2019-12-17 12:35:48,631] jepsen worker 1 - jepsen.util 13681 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:35:48,631] jepsen worker 1 - jepsen.util 13681 :invoke :write [0 43] INFO [2019-12-17 12:35:48,638] jepsen worker 46 - jepsen.util 9856 :fail :cas [15 [30 22]] :ValueNotSwapFrom INFO [2019-12-17 12:35:48,638] jepsen worker 46 - jepsen.util 9856 :invoke :write [15 4] INFO [2019-12-17 12:35:48,650] jepsen worker 9 - jepsen.util 10629 :ok :read [3 0] INFO [2019-12-17 12:35:48,650] jepsen worker 9 - jepsen.util 10629 :invoke :cas [3 [37 27]] INFO [2019-12-17 12:35:48,660] jepsen worker 65 - jepsen.util 8525 :ok :read [21 21] INFO [2019-12-17 12:35:48,660] jepsen worker 65 - jepsen.util 8525 :invoke :read [21 nil] INFO [2019-12-17 12:35:48,668] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":57583,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:35:48,668] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":57583,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:35:48,668] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":57587,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:35:48,668] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":57587,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:35:48,680] jepsen worker 65 - jepsen.util 8525 :ok :read [21 21] INFO [2019-12-17 12:35:48,680] jepsen worker 65 - jepsen.util 8525 :invoke :cas [21 [32 7]] INFO [2019-12-17 12:35:48,714] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:35:48,714] jepsen nemesis - couchbase.util Rebalance check count: 450 INFO [2019-12-17 12:35:48,714] jepsen nemesis - couchbase.util Rebalance stuck count: 431 INFO [2019-12-17 12:35:48,720] jepsen worker 28 - jepsen.util 8218 :ok :write [9 10] INFO [2019-12-17 12:35:48,720] jepsen worker 28 - jepsen.util 8218 :invoke :write [9 44] INFO [2019-12-17 12:35:48,726] jepsen worker 47 - jepsen.util 10487 :ok :read [15 34] INFO [2019-12-17 12:35:48,726] jepsen worker 47 - jepsen.util 10487 :invoke :read [15 nil] INFO [2019-12-17 12:35:48,731] jepsen worker 75 - jepsen.util 13035 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:35:48,731] jepsen worker 68 - jepsen.util 12938 :info :write [22 37] :RequestCanceledException INFO [2019-12-17 12:35:48,731] jepsen worker 75 - jepsen.util 13035 :invoke :write [25 41] INFO [2019-12-17 12:35:48,737] jepsen worker 68 - jepsen.util 13028 :invoke :cas [22 [35 6]] INFO [2019-12-17 12:35:48,743] jepsen worker 82 - jepsen.util 13942 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:35:48,743] jepsen worker 82 - jepsen.util 13942 :invoke :cas [27 [4 41]] INFO [2019-12-17 12:35:48,749] jepsen worker 13 - jepsen.util 9373 :ok :read [4 14] INFO [2019-12-17 12:35:48,749] jepsen worker 13 - jepsen.util 9373 :invoke :cas [4 [5 46]] INFO [2019-12-17 12:35:48,760] jepsen worker 88 - jepsen.util 10168 :ok :read [29 43] INFO [2019-12-17 12:35:48,760] jepsen worker 88 - jepsen.util 10168 :invoke :cas [29 [8 25]] INFO [2019-12-17 12:35:48,762] jepsen worker 58 - jepsen.util 14008 :info :cas [19 [28 34]] :RequestCanceledException INFO [2019-12-17 12:35:48,766] jepsen worker 58 - jepsen.util 14098 :invoke :cas [19 [13 17]] INFO [2019-12-17 12:35:48,767] jepsen worker 65 - jepsen.util 8525 :fail :cas [21 [32 7]] :ValueNotSwapFrom INFO [2019-12-17 12:35:48,768] jepsen worker 65 - jepsen.util 8525 :invoke :cas [21 [45 3]] INFO [2019-12-17 12:35:48,772] jepsen worker 70 - jepsen.util 10780 :ok :read [23 11] INFO [2019-12-17 12:35:48,772] jepsen worker 70 - jepsen.util 10780 :invoke :write [23 31] INFO [2019-12-17 12:35:48,777] jepsen worker 9 - jepsen.util 10629 :fail :cas [3 [37 27]] :ValueNotSwapFrom INFO [2019-12-17 12:35:48,777] jepsen worker 9 - jepsen.util 10629 :invoke :read [3 nil] INFO [2019-12-17 12:35:48,785] jepsen worker 47 - jepsen.util 10487 :ok :read [15 34] INFO [2019-12-17 12:35:48,785] jepsen worker 47 - jepsen.util 10487 :invoke :read [15 nil] INFO [2019-12-17 12:35:48,829] jepsen worker 47 - jepsen.util 10487 :ok :read [15 34] INFO [2019-12-17 12:35:48,829] jepsen worker 47 - jepsen.util 10487 :invoke :write [15 21] INFO [2019-12-17 12:35:48,845] jepsen worker 3 - jepsen.util 9723 :info :write [1 18] :RequestCanceledException INFO [2019-12-17 12:35:48,847] jepsen worker 3 - jepsen.util 9813 :invoke :read [1 nil] INFO [2019-12-17 12:35:48,852] jepsen worker 65 - jepsen.util 8525 :fail :cas [21 [45 3]] :ValueNotSwapFrom INFO [2019-12-17 12:35:48,852] jepsen worker 65 - jepsen.util 8525 :invoke :cas [21 [30 25]] INFO [2019-12-17 12:35:48,853] jepsen worker 32 - jepsen.util 9932 :ok :write [10 44] INFO [2019-12-17 12:35:48,853] jepsen worker 32 - jepsen.util 9932 :invoke :read [10 nil] INFO [2019-12-17 12:35:48,863] jepsen worker 80 - jepsen.util 9620 :info :write [26 3] :RequestCanceledException INFO [2019-12-17 12:35:48,863] jepsen worker 25 - jepsen.util 14425 :info :cas [8 [1 26]] :RequestCanceledException INFO [2019-12-17 12:35:48,864] jepsen worker 80 - jepsen.util 9710 :invoke :cas [26 [30 24]] INFO [2019-12-17 12:35:48,865] jepsen worker 25 - jepsen.util 14515 :invoke :write [8 47] INFO [2019-12-17 12:35:48,877] jepsen worker 28 - jepsen.util 8218 :ok :write [9 44] INFO [2019-12-17 12:35:48,877] jepsen worker 28 - jepsen.util 8218 :invoke :write [9 12] INFO [2019-12-17 12:35:48,886] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":57609,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:35:48,886] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":57609,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:35:48,886] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":57612,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:35:48,886] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":57612,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:35:48,897] jepsen worker 65 - jepsen.util 8525 :fail :cas [21 [30 25]] :ValueNotSwapFrom INFO [2019-12-17 12:35:48,897] jepsen worker 65 - jepsen.util 8525 :invoke :write [21 14] INFO [2019-12-17 12:35:48,924] jepsen worker 45 - jepsen.util 10305 :fail :cas [15 [40 16]] :ValueNotSwapFrom INFO [2019-12-17 12:35:48,924] jepsen worker 45 - jepsen.util 10305 :invoke :write [15 15] INFO [2019-12-17 12:35:48,939] jepsen worker 9 - jepsen.util 10629 :ok :read [3 0] INFO [2019-12-17 12:35:48,939] jepsen worker 9 - jepsen.util 10629 :invoke :cas [3 [47 16]] INFO [2019-12-17 12:35:48,949] jepsen worker 32 - jepsen.util 9932 :ok :read [10 44] INFO [2019-12-17 12:35:48,950] jepsen worker 32 - jepsen.util 9932 :invoke :cas [10 [39 8]] INFO [2019-12-17 12:35:48,975] jepsen worker 20 - jepsen.util 9740 :ok :write [6 14] INFO [2019-12-17 12:35:48,975] jepsen worker 20 - jepsen.util 9740 :invoke :cas [6 [11 5]] INFO [2019-12-17 12:35:49,004] jepsen worker 46 - jepsen.util 9856 :ok :write [15 4] INFO [2019-12-17 12:35:49,004] jepsen worker 28 - jepsen.util 8218 :ok :write [9 12] INFO [2019-12-17 12:35:49,004] jepsen worker 46 - jepsen.util 9856 :invoke :write [15 26] INFO [2019-12-17 12:35:49,004] jepsen worker 28 - jepsen.util 8218 :invoke :read [9 nil] INFO [2019-12-17 12:35:49,009] jepsen worker 65 - jepsen.util 8525 :ok :write [21 14] INFO [2019-12-17 12:35:49,009] jepsen worker 65 - jepsen.util 8525 :invoke :read [21 nil] INFO [2019-12-17 12:35:49,017] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":57622,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:35:49,017] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":57622,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:35:49,017] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":57626,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:35:49,017] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":57626,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:35:49,031] jepsen worker 81 - jepsen.util 13311 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:35:49,031] jepsen worker 89 - jepsen.util 9989 :info :write [29 10] :RequestCanceledException INFO [2019-12-17 12:35:49,031] jepsen worker 81 - jepsen.util 13311 :invoke :write [27 4] INFO [2019-12-17 12:35:49,033] jepsen worker 89 - jepsen.util 10079 :invoke :write [29 34] INFO [2019-12-17 12:35:49,067] jepsen worker 9 - jepsen.util 10629 :fail :cas [3 [47 16]] :ValueNotSwapFrom INFO [2019-12-17 12:35:49,067] jepsen worker 9 - jepsen.util 10629 :invoke :cas [3 [30 5]] INFO [2019-12-17 12:35:49,114] jepsen worker 65 - jepsen.util 8525 :ok :read [21 14] INFO [2019-12-17 12:35:49,115] jepsen worker 65 - jepsen.util 8525 :invoke :write [21 26] INFO [2019-12-17 12:35:49,131] jepsen worker 18 - jepsen.util 10188 :info :write [6 38] :RequestCanceledException INFO [2019-12-17 12:35:49,135] jepsen worker 18 - jepsen.util 10278 :invoke :read [6 nil] INFO [2019-12-17 12:35:49,136] jepsen worker 28 - jepsen.util 8218 :ok :read [9 12] INFO [2019-12-17 12:35:49,136] jepsen worker 28 - jepsen.util 8218 :invoke :cas [9 [33 12]] INFO [2019-12-17 12:35:49,145] jepsen worker 39 - jepsen.util 13179 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:35:49,145] jepsen worker 39 - jepsen.util 13179 :invoke :read [13 nil] INFO [2019-12-17 12:35:49,154] jepsen worker 9 - jepsen.util 10629 :fail :cas [3 [30 5]] :ValueNotSwapFrom INFO [2019-12-17 12:35:49,154] jepsen worker 9 - jepsen.util 10629 :invoke :read [3 nil] INFO [2019-12-17 12:35:49,159] jepsen worker 88 - jepsen.util 10168 :fail :cas [29 [8 25]] :ValueNotSwapFrom INFO [2019-12-17 12:35:49,159] jepsen worker 88 - jepsen.util 10168 :invoke :write [29 20] INFO [2019-12-17 12:35:49,163] jepsen worker 72 - jepsen.util 9882 :info :write [24 9] :RequestCanceledException INFO [2019-12-17 12:35:49,169] jepsen worker 72 - jepsen.util 9972 :invoke :read [24 nil] INFO [2019-12-17 12:35:49,198] jepsen worker 48 - jepsen.util 12378 :info :write [16 5] :RequestCanceledException INFO [2019-12-17 12:35:49,199] jepsen worker 48 - jepsen.util 12468 :invoke :read [16 nil] INFO [2019-12-17 12:35:49,216] jepsen worker 53 - jepsen.util 9503 :ok :read [17 49] INFO [2019-12-17 12:35:49,216] jepsen worker 53 - jepsen.util 9503 :invoke :cas [17 [48 33]] INFO [2019-12-17 12:35:49,216] jepsen worker 33 - jepsen.util 13623 :info :write [11 47] :RequestCanceledException INFO [2019-12-17 12:35:49,219] jepsen worker 33 - jepsen.util 13713 :invoke :write [11 22] INFO [2019-12-17 12:35:49,221] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":57650,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:35:49,221] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":57650,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:35:49,228] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":57641,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:35:49,228] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":57641,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:35:49,231] jepsen worker 30 - jepsen.util 10110 :fail :read [10 nil] :RequestTimeoutException INFO [2019-12-17 12:35:49,231] jepsen worker 51 - jepsen.util 9321 :info :write [17 24] :RequestCanceledException INFO [2019-12-17 12:35:49,231] jepsen worker 30 - jepsen.util 10110 :invoke :read [10 nil] INFO [2019-12-17 12:35:49,233] jepsen worker 51 - jepsen.util 9411 :invoke :read [17 nil] INFO [2019-12-17 12:35:49,256] jepsen worker 18 - jepsen.util 10278 :ok :read [6 14] INFO [2019-12-17 12:35:49,256] jepsen worker 18 - jepsen.util 10278 :invoke :cas [6 [5 18]] INFO [2019-12-17 12:35:49,262] jepsen worker 38 - jepsen.util 10658 :info :write [12 25] :RequestCanceledException INFO [2019-12-17 12:35:49,267] jepsen worker 38 - jepsen.util 10748 :invoke :write [12 48] INFO [2019-12-17 12:35:49,279] jepsen worker 28 - jepsen.util 8218 :fail :cas [9 [33 12]] :ValueNotSwapFrom INFO [2019-12-17 12:35:49,279] jepsen worker 45 - jepsen.util 10305 :ok :write [15 15] INFO [2019-12-17 12:35:49,279] jepsen worker 28 - jepsen.util 8218 :invoke :write [9 41] INFO [2019-12-17 12:35:49,279] jepsen worker 45 - jepsen.util 10305 :invoke :cas [15 [42 21]] INFO [2019-12-17 12:35:49,301] jepsen worker 13 - jepsen.util 9373 :fail :cas [4 [5 46]] :ValueNotSwapFrom INFO [2019-12-17 12:35:49,302] jepsen worker 13 - jepsen.util 9373 :invoke :write [4 26] INFO [2019-12-17 12:35:49,312] jepsen worker 30 - jepsen.util 10110 :ok :read [10 44] INFO [2019-12-17 12:35:49,312] jepsen worker 30 - jepsen.util 10110 :invoke :read [10 nil] INFO [2019-12-17 12:35:49,329] jepsen worker 18 - jepsen.util 10278 :fail :cas [6 [5 18]] :ValueNotSwapFrom INFO [2019-12-17 12:35:49,330] jepsen worker 18 - jepsen.util 10278 :invoke :read [6 nil] INFO [2019-12-17 12:35:49,331] jepsen worker 77 - jepsen.util 14297 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:35:49,331] jepsen worker 56 - jepsen.util 10496 :info :write [18 38] :RequestCanceledException INFO [2019-12-17 12:35:49,331] jepsen worker 77 - jepsen.util 14297 :invoke :write [25 0] INFO [2019-12-17 12:35:49,332] jepsen worker 56 - jepsen.util 10586 :invoke :cas [18 [9 19]] INFO [2019-12-17 12:35:49,356] jepsen worker 30 - jepsen.util 10110 :ok :read [10 44] INFO [2019-12-17 12:35:49,356] jepsen worker 30 - jepsen.util 10110 :invoke :write [10 32] INFO [2019-12-17 12:35:49,357] jepsen worker 9 - jepsen.util 10629 :ok :read [3 0] INFO [2019-12-17 12:35:49,357] jepsen worker 9 - jepsen.util 10629 :invoke :cas [3 [16 33]] INFO [2019-12-17 12:35:49,370] jepsen worker 65 - jepsen.util 8525 :ok :write [21 26] INFO [2019-12-17 12:35:49,370] jepsen worker 65 - jepsen.util 8525 :invoke :cas [21 [47 48]] INFO [2019-12-17 12:35:49,372] jepsen worker 45 - jepsen.util 10305 :fail :cas [15 [42 21]] :ValueNotSwapFrom INFO [2019-12-17 12:35:49,372] jepsen worker 45 - jepsen.util 10305 :invoke :cas [15 [6 19]] INFO [2019-12-17 12:35:49,406] jepsen worker 18 - jepsen.util 10278 :ok :read [6 14] INFO [2019-12-17 12:35:49,406] jepsen worker 18 - jepsen.util 10278 :invoke :cas [6 [44 14]] INFO [2019-12-17 12:35:49,431] jepsen worker 52 - jepsen.util 9862 :info :write [17 45] :RequestCanceledException INFO [2019-12-17 12:35:49,432] jepsen worker 52 - jepsen.util 9952 :invoke :cas [17 [42 37]] INFO [2019-12-17 12:35:49,432] jepsen worker 72 - jepsen.util 9972 :ok :read [24 20] INFO [2019-12-17 12:35:49,432] jepsen worker 72 - jepsen.util 9972 :invoke :write [24 32] INFO [2019-12-17 12:35:49,441] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":57673,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:35:49,442] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":57673,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:35:49,442] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":57676,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:35:49,442] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":57676,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:35:49,479] jepsen worker 28 - jepsen.util 8218 :ok :write [9 41] INFO [2019-12-17 12:35:49,479] jepsen worker 28 - jepsen.util 8218 :invoke :write [9 29] INFO [2019-12-17 12:35:49,497] jepsen worker 53 - jepsen.util 9503 :fail :cas [17 [48 33]] :ValueNotSwapFrom INFO [2019-12-17 12:35:49,497] jepsen worker 53 - jepsen.util 9503 :invoke :write [17 30] INFO [2019-12-17 12:35:49,498] jepsen worker 4 - jepsen.util 9184 :info :write [1 25] :RequestCanceledException INFO [2019-12-17 12:35:49,498] jepsen worker 59 - jepsen.util 12839 :info :write [19 7] :RequestCanceledException INFO [2019-12-17 12:35:49,504] jepsen worker 4 - jepsen.util 9274 :invoke :write [1 48] INFO [2019-12-17 12:35:49,506] jepsen worker 59 - jepsen.util 12929 :invoke :read [19 nil] INFO [2019-12-17 12:35:49,515] jepsen worker 18 - jepsen.util 10278 :fail :cas [6 [44 14]] :ValueNotSwapFrom INFO [2019-12-17 12:35:49,516] jepsen worker 51 - jepsen.util 9411 :ok :read [17 49] INFO [2019-12-17 12:35:49,516] jepsen worker 18 - jepsen.util 10278 :invoke :write [6 36] INFO [2019-12-17 12:35:49,516] jepsen worker 51 - jepsen.util 9411 :invoke :read [17 nil] INFO [2019-12-17 12:35:49,518] jepsen worker 45 - jepsen.util 10305 :fail :cas [15 [6 19]] :ValueNotSwapFrom INFO [2019-12-17 12:35:49,518] jepsen worker 45 - jepsen.util 10305 :invoke :write [15 40] INFO [2019-12-17 12:35:49,518] jepsen worker 65 - jepsen.util 8525 :fail :cas [21 [47 48]] :ValueNotSwapFrom INFO [2019-12-17 12:35:49,519] jepsen worker 65 - jepsen.util 8525 :invoke :read [21 nil] INFO [2019-12-17 12:35:49,530] jepsen worker 67 - jepsen.util 13477 :info :write [22 40] :RequestCanceledException INFO [2019-12-17 12:35:49,530] jepsen worker 8 - jepsen.util 13238 :info :write [2 19] :RequestCanceledException INFO [2019-12-17 12:35:49,533] jepsen worker 67 - jepsen.util 13567 :invoke :write [22 5] INFO [2019-12-17 12:35:49,533] jepsen worker 8 - jepsen.util 13328 :invoke :read [2 nil] INFO [2019-12-17 12:35:49,545] jepsen worker 21 - jepsen.util 13161 :info :write [7 20] :RequestCanceledException INFO [2019-12-17 12:35:49,550] jepsen worker 21 - jepsen.util 13251 :invoke :read [7 nil] INFO [2019-12-17 12:35:49,558] jepsen worker 3 - jepsen.util 9813 :ok :read [1 6] INFO [2019-12-17 12:35:49,558] jepsen worker 3 - jepsen.util 9813 :invoke :read [1 nil] INFO [2019-12-17 12:35:49,562] jepsen worker 34 - jepsen.util 13624 :info :write [11 28] :RequestCanceledException INFO [2019-12-17 12:35:49,562] jepsen worker 84 - jepsen.util 13134 :info :write [28 17] :RequestCanceledException INFO [2019-12-17 12:35:49,563] jepsen worker 65 - jepsen.util 8525 :ok :read [21 26] INFO [2019-12-17 12:35:49,563] jepsen worker 65 - jepsen.util 8525 :invoke :cas [21 [32 47]] INFO [2019-12-17 12:35:49,564] jepsen worker 34 - jepsen.util 13714 :invoke :write [11 11] INFO [2019-12-17 12:35:49,564] jepsen worker 84 - jepsen.util 13224 :invoke :cas [28 [20 32]] INFO [2019-12-17 12:35:49,598] jepsen worker 86 - jepsen.util 14486 :info :write [28 46] :RequestCanceledException INFO [2019-12-17 12:35:49,600] jepsen worker 65 - jepsen.util 8525 :fail :cas [21 [32 47]] :ValueNotSwapFrom INFO [2019-12-17 12:35:49,600] jepsen worker 86 - jepsen.util 14576 :invoke :write [28 34] INFO [2019-12-17 12:35:49,600] jepsen worker 65 - jepsen.util 8525 :invoke :write [21 0] INFO [2019-12-17 12:35:49,616] jepsen worker 50 - jepsen.util 14180 :info :cas [16 [31 40]] :RequestCanceledException INFO [2019-12-17 12:35:49,616] jepsen worker 16 - jepsen.util 14866 :info :write [5 0] :RequestCanceledException INFO [2019-12-17 12:35:49,618] jepsen worker 50 - jepsen.util 14270 :invoke :cas [16 [12 44]] INFO [2019-12-17 12:35:49,619] jepsen worker 16 - jepsen.util 14956 :invoke :write [5 3] INFO [2019-12-17 12:35:49,631] jepsen worker 41 - jepsen.util 14621 :info :cas [13 [37 22]] :RequestCanceledException INFO [2019-12-17 12:35:49,631] jepsen worker 87 - jepsen.util 9537 :info :write [29 5] :RequestCanceledException INFO [2019-12-17 12:35:49,632] jepsen worker 87 - jepsen.util 9627 :invoke :read [29 nil] INFO [2019-12-17 12:35:49,635] jepsen worker 41 - jepsen.util 14711 :invoke :write [13 8] INFO [2019-12-17 12:35:49,638] jepsen worker 52 - jepsen.util 9952 :fail :cas [17 [42 37]] :ValueNotSwapFrom INFO [2019-12-17 12:35:49,638] jepsen worker 52 - jepsen.util 9952 :invoke :write [17 18] INFO [2019-12-17 12:35:49,645] jepsen worker 22 - jepsen.util 13162 :info :write [7 5] :RequestCanceledException INFO [2019-12-17 12:35:49,650] jepsen worker 22 - jepsen.util 13252 :invoke :write [7 20] INFO [2019-12-17 12:35:49,653] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":57701,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:35:49,653] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":57701,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:35:49,662] jepsen worker 2 - jepsen.util 13322 :info :cas [0 [22 14]] :RequestCanceledException INFO [2019-12-17 12:35:49,665] jepsen worker 2 - jepsen.util 13412 :invoke :cas [0 [29 39]] INFO [2019-12-17 12:35:49,669] jepsen worker 70 - jepsen.util 10780 :ok :write [23 31] INFO [2019-12-17 12:35:49,669] jepsen worker 70 - jepsen.util 10780 :invoke :read [23 nil] INFO [2019-12-17 12:35:49,674] jepsen worker 65 - jepsen.util 8525 :ok :write [21 0] INFO [2019-12-17 12:35:49,675] jepsen worker 65 - jepsen.util 8525 :invoke :cas [21 [38 3]] INFO [2019-12-17 12:35:49,698] jepsen worker 15 - jepsen.util 12615 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:35:49,698] jepsen worker 15 - jepsen.util 12615 :invoke :read [5 nil] INFO [2019-12-17 12:35:49,705] jepsen worker 65 - jepsen.util 8525 :fail :cas [21 [38 3]] :ValueNotSwapFrom INFO [2019-12-17 12:35:49,706] jepsen worker 65 - jepsen.util 8525 :invoke :read [21 nil] INFO [2019-12-17 12:35:49,714] jepsen worker 43 - jepsen.util 13543 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:35:49,714] jepsen worker 43 - jepsen.util 13543 :invoke :cas [14 [47 47]] INFO [2019-12-17 12:35:49,731] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:35:49,731] jepsen nemesis - couchbase.util Rebalance check count: 451 INFO [2019-12-17 12:35:49,731] jepsen worker 37 - jepsen.util 9487 :info :write [12 43] :RequestCanceledException INFO [2019-12-17 12:35:49,731] jepsen nemesis - couchbase.util Rebalance stuck count: 432 INFO [2019-12-17 12:35:49,731] jepsen worker 64 - jepsen.util 7624 :info :write [21 14] :RequestCanceledException INFO [2019-12-17 12:35:49,733] jepsen worker 37 - jepsen.util 9577 :invoke :read [12 nil] INFO [2019-12-17 12:35:49,736] jepsen worker 64 - jepsen.util 7714 :invoke :cas [21 [45 5]] INFO [2019-12-17 12:35:49,748] jepsen worker 65 - jepsen.util 8525 :ok :read [21 0] INFO [2019-12-17 12:35:49,748] jepsen worker 65 - jepsen.util 8525 :invoke :write [21 25] INFO [2019-12-17 12:35:49,750] jepsen worker 18 - jepsen.util 10278 :ok :write [6 36] INFO [2019-12-17 12:35:49,750] jepsen worker 30 - jepsen.util 10110 :ok :write [10 32] INFO [2019-12-17 12:35:49,751] jepsen worker 30 - jepsen.util 10110 :invoke :write [10 17] INFO [2019-12-17 12:35:49,751] jepsen worker 18 - jepsen.util 10278 :invoke :read [6 nil] INFO [2019-12-17 12:35:49,780] jepsen worker 80 - jepsen.util 9710 :fail :cas [26 [30 24]] :ValueNotSwapFrom INFO [2019-12-17 12:35:49,780] jepsen worker 80 - jepsen.util 9710 :invoke :cas [26 [46 28]] INFO [2019-12-17 12:35:49,795] jepsen worker 18 - jepsen.util 10278 :ok :read [6 36] INFO [2019-12-17 12:35:49,796] jepsen worker 18 - jepsen.util 10278 :invoke :cas [6 [15 33]] INFO [2019-12-17 12:35:49,796] jepsen worker 51 - jepsen.util 9411 :ok :read [17 49] INFO [2019-12-17 12:35:49,796] jepsen worker 51 - jepsen.util 9411 :invoke :read [17 nil] INFO [2019-12-17 12:35:49,798] jepsen worker 73 - jepsen.util 9433 :info :cas [24 [39 38]] :RequestCanceledException INFO [2019-12-17 12:35:49,799] jepsen worker 73 - jepsen.util 9523 :invoke :cas [24 [23 23]] INFO [2019-12-17 12:35:49,845] jepsen worker 74 - jepsen.util 9614 :info :write [24 15] :RequestCanceledException INFO [2019-12-17 12:35:49,847] jepsen worker 74 - jepsen.util 9704 :invoke :write [24 6] INFO [2019-12-17 12:35:49,859] jepsen worker 17 - jepsen.util 13337 :info :cas [5 [13 22]] :RequestCanceledException INFO [2019-12-17 12:35:49,864] jepsen worker 17 - jepsen.util 13427 :invoke :write [5 42] INFO [2019-12-17 12:35:49,872] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":57724,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:35:49,872] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":57724,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:35:49,872] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":57729,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:35:49,873] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":57729,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:35:49,887] jepsen worker 18 - jepsen.util 10278 :fail :cas [6 [15 33]] :ValueNotSwapFrom INFO [2019-12-17 12:35:49,888] jepsen worker 18 - jepsen.util 10278 :invoke :read [6 nil] INFO [2019-12-17 12:35:49,894] jepsen worker 57 - jepsen.util 13197 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:35:49,894] jepsen worker 42 - jepsen.util 14442 :info :write [14 16] :RequestCanceledException INFO [2019-12-17 12:35:49,894] jepsen worker 57 - jepsen.util 13197 :invoke :read [19 nil] INFO [2019-12-17 12:35:49,897] jepsen worker 42 - jepsen.util 14532 :invoke :write [14 44] INFO [2019-12-17 12:35:49,922] jepsen worker 65 - jepsen.util 8525 :ok :write [21 25] INFO [2019-12-17 12:35:49,923] jepsen worker 65 - jepsen.util 8525 :invoke :write [21 4] INFO [2019-12-17 12:35:49,924] jepsen worker 73 - jepsen.util 9523 :fail :cas [24 [23 23]] :ValueNotSwapFrom INFO [2019-12-17 12:35:49,924] jepsen worker 73 - jepsen.util 9523 :invoke :cas [24 [37 15]] INFO [2019-12-17 12:35:49,930] jepsen worker 60 - jepsen.util 14460 :info :cas [20 [30 6]] :RequestCanceledException INFO [2019-12-17 12:35:49,935] jepsen worker 60 - jepsen.util 14550 :invoke :read [20 nil] INFO [2019-12-17 12:35:49,938] jepsen worker 18 - jepsen.util 10278 :ok :read [6 36] INFO [2019-12-17 12:35:49,938] jepsen worker 18 - jepsen.util 10278 :invoke :write [6 27] INFO [2019-12-17 12:35:49,945] jepsen worker 5 - jepsen.util 9365 :info :write [1 19] :RequestCanceledException INFO [2019-12-17 12:35:49,949] jepsen worker 5 - jepsen.util 9455 :invoke :write [1 9] INFO [2019-12-17 12:35:49,975] jepsen worker 73 - jepsen.util 9523 :fail :cas [24 [37 15]] :ValueNotSwapFrom INFO [2019-12-17 12:35:49,975] jepsen worker 73 - jepsen.util 9523 :invoke :write [24 31] INFO [2019-12-17 12:35:49,993] jepsen worker 69 - jepsen.util 9969 :info :write [23 37] :RequestCanceledException INFO [2019-12-17 12:35:49,994] jepsen worker 69 - jepsen.util 10059 :invoke :cas [23 [36 29]] INFO [2019-12-17 12:35:50,004] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":57742,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:35:50,004] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":57742,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:35:50,004] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":57745,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:35:50,004] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":57745,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:35:50,006] jepsen worker 37 - jepsen.util 9577 :ok :read [12 13] INFO [2019-12-17 12:35:50,006] jepsen worker 37 - jepsen.util 9577 :invoke :write [12 27] INFO [2019-12-17 12:35:50,008] jepsen worker 65 - jepsen.util 8525 :ok :write [21 4] INFO [2019-12-17 12:35:50,008] jepsen worker 65 - jepsen.util 8525 :invoke :cas [21 [10 27]] INFO [2019-12-17 12:35:50,016] jepsen worker 31 - jepsen.util 8941 :info :cas [10 [6 41]] :RequestCanceledException INFO [2019-12-17 12:35:50,016] jepsen worker 6 - jepsen.util 14136 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:35:50,016] jepsen worker 11 - jepsen.util 9821 :fail :read [3 nil] :RequestTimeoutException INFO [2019-12-17 12:35:50,016] jepsen worker 6 - jepsen.util 14136 :invoke :read [2 nil] INFO [2019-12-17 12:35:50,016] jepsen worker 11 - jepsen.util 9821 :invoke :read [3 nil] INFO [2019-12-17 12:35:50,016] jepsen worker 79 - jepsen.util 10159 :info :write [26 26] :RequestCanceledException INFO [2019-12-17 12:35:50,019] jepsen worker 70 - jepsen.util 10780 :ok :read [23 31] INFO [2019-12-17 12:35:50,019] jepsen worker 70 - jepsen.util 10780 :invoke :cas [23 [36 23]] INFO [2019-12-17 12:35:50,019] jepsen worker 79 - jepsen.util 10249 :invoke :write [26 40] INFO [2019-12-17 12:35:50,019] jepsen worker 31 - jepsen.util 9031 :invoke :cas [10 [13 46]] INFO [2019-12-17 12:35:50,040] jepsen worker 65 - jepsen.util 8525 :fail :cas [21 [10 27]] :ValueNotSwapFrom INFO [2019-12-17 12:35:50,040] jepsen worker 65 - jepsen.util 8525 :invoke :read [21 nil] INFO [2019-12-17 12:35:50,062] jepsen worker 76 - jepsen.util 13756 :info :write [25 14] :RequestCanceledException INFO [2019-12-17 12:35:50,065] jepsen worker 76 - jepsen.util 13846 :invoke :read [25 nil] INFO [2019-12-17 12:35:50,077] jepsen worker 80 - jepsen.util 9710 :fail :cas [26 [46 28]] :ValueNotSwapFrom INFO [2019-12-17 12:35:50,077] jepsen worker 80 - jepsen.util 9710 :invoke :write [26 9] INFO [2019-12-17 12:35:50,097] jepsen worker 65 - jepsen.util 8525 :ok :read [21 4] INFO [2019-12-17 12:35:50,098] jepsen worker 65 - jepsen.util 8525 :invoke :cas [21 [19 32]] INFO [2019-12-17 12:35:50,098] jepsen worker 55 - jepsen.util 10675 :info :write [18 17] :RequestCanceledException INFO [2019-12-17 12:35:50,101] jepsen worker 55 - jepsen.util 10765 :invoke :write [18 7] INFO [2019-12-17 12:35:50,103] jepsen worker 56 - jepsen.util 10586 :fail :cas [18 [9 19]] :ValueNotSwapFrom INFO [2019-12-17 12:35:50,103] jepsen worker 56 - jepsen.util 10586 :invoke :read [18 nil] INFO [2019-12-17 12:35:50,118] jepsen worker 51 - jepsen.util 9411 :ok :read [17 49] INFO [2019-12-17 12:35:50,118] jepsen worker 51 - jepsen.util 9411 :invoke :read [17 nil] INFO [2019-12-17 12:35:50,128] jepsen worker 65 - jepsen.util 8525 :fail :cas [21 [19 32]] :ValueNotSwapFrom INFO [2019-12-17 12:35:50,128] jepsen worker 65 - jepsen.util 8525 :invoke :write [21 14] INFO [2019-12-17 12:35:50,131] jepsen worker 24 - jepsen.util 12354 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:35:50,131] jepsen worker 36 - jepsen.util 10656 :info :write [12 4] :RequestCanceledException INFO [2019-12-17 12:35:50,131] jepsen worker 24 - jepsen.util 12354 :invoke :cas [8 [8 5]] INFO [2019-12-17 12:35:50,132] jepsen worker 36 - jepsen.util 10746 :invoke :read [12 nil] INFO [2019-12-17 12:35:50,155] jepsen worker 70 - jepsen.util 10780 :fail :cas [23 [36 23]] :ValueNotSwapFrom INFO [2019-12-17 12:35:50,155] jepsen worker 70 - jepsen.util 10780 :invoke :read [23 nil] INFO [2019-12-17 12:35:50,170] jepsen worker 87 - jepsen.util 9627 :ok :read [29 43] INFO [2019-12-17 12:35:50,170] jepsen worker 87 - jepsen.util 9627 :invoke :cas [29 [26 19]] INFO [2019-12-17 12:35:50,178] jepsen worker 65 - jepsen.util 8525 :ok :write [21 14] INFO [2019-12-17 12:35:50,178] jepsen worker 65 - jepsen.util 8525 :invoke :cas [21 [7 39]] INFO [2019-12-17 12:35:50,195] jepsen worker 44 - jepsen.util 13814 :info :cas [14 [33 10]] :RequestCanceledException INFO [2019-12-17 12:35:50,199] jepsen worker 44 - jepsen.util 13904 :invoke :write [14 6] INFO [2019-12-17 12:35:50,208] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":57767,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:35:50,208] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":57767,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:35:50,208] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":57768,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:35:50,208] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":57768,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:35:50,208] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":57769,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:35:50,209] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":57769,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:35:50,211] jepsen worker 65 - jepsen.util 8525 :fail :cas [21 [7 39]] :ValueNotSwapFrom INFO [2019-12-17 12:35:50,211] jepsen worker 65 - jepsen.util 8525 :invoke :write [21 37] INFO [2019-12-17 12:35:50,231] jepsen worker 29 - jepsen.util 8309 :info :write [9 38] :RequestCanceledException INFO [2019-12-17 12:35:50,232] jepsen worker 29 - jepsen.util 8399 :invoke :cas [9 [18 46]] INFO [2019-12-17 12:35:50,245] jepsen worker 27 - jepsen.util 9477 :info :write [9 1] :RequestCanceledException INFO [2019-12-17 12:35:50,249] jepsen worker 27 - jepsen.util 9567 :invoke :write [9 20] INFO [2019-12-17 12:35:50,255] jepsen worker 69 - jepsen.util 10059 :fail :cas [23 [36 29]] :ValueNotSwapFrom INFO [2019-12-17 12:35:50,255] jepsen worker 69 - jepsen.util 10059 :invoke :cas [23 [8 10]] INFO [2019-12-17 12:35:50,262] jepsen worker 3 - jepsen.util 9813 :ok :read [1 25] INFO [2019-12-17 12:35:50,262] jepsen worker 35 - jepsen.util 13445 :info :cas [11 [44 22]] :RequestCanceledException INFO [2019-12-17 12:35:50,262] jepsen worker 3 - jepsen.util 9813 :invoke :cas [1 [43 30]] INFO [2019-12-17 12:35:50,264] jepsen worker 35 - jepsen.util 13535 :invoke :write [11 42] INFO [2019-12-17 12:35:50,275] jepsen worker 65 - jepsen.util 8525 :ok :write [21 37] INFO [2019-12-17 12:35:50,275] jepsen worker 65 - jepsen.util 8525 :invoke :read [21 nil] INFO [2019-12-17 12:35:50,295] jepsen worker 56 - jepsen.util 10586 :ok :read [18 5] INFO [2019-12-17 12:35:50,295] jepsen worker 56 - jepsen.util 10586 :invoke :cas [18 [17 38]] INFO [2019-12-17 12:35:50,306] jepsen worker 65 - jepsen.util 8525 :ok :read [21 37] INFO [2019-12-17 12:35:50,306] jepsen worker 65 - jepsen.util 8525 :invoke :cas [21 [1 32]] INFO [2019-12-17 12:35:50,317] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":57775,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:35:50,317] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":57775,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:35:50,317] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":57779,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:35:50,317] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":57779,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:35:50,317] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":57780,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:35:50,317] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":57780,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:35:50,325] jepsen worker 65 - jepsen.util 8525 :fail :cas [21 [1 32]] :ValueNotSwapFrom INFO [2019-12-17 12:35:50,325] jepsen worker 65 - jepsen.util 8525 :invoke :write [21 40] INFO [2019-12-17 12:35:50,331] jepsen worker 62 - jepsen.util 14012 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:35:50,331] jepsen worker 62 - jepsen.util 14012 :invoke :cas [20 [25 6]] INFO [2019-12-17 12:35:50,344] jepsen worker 19 - jepsen.util 10189 :fail :read [6 nil] :RequestTimeoutException INFO [2019-12-17 12:35:50,344] jepsen worker 19 - jepsen.util 10189 :invoke :read [6 nil] INFO [2019-12-17 12:35:50,357] jepsen worker 87 - jepsen.util 9627 :fail :cas [29 [26 19]] :ValueNotSwapFrom INFO [2019-12-17 12:35:50,358] jepsen worker 87 - jepsen.util 9627 :invoke :write [29 34] INFO [2019-12-17 12:35:50,376] jepsen worker 65 - jepsen.util 8525 :ok :write [21 40] INFO [2019-12-17 12:35:50,376] jepsen worker 65 - jepsen.util 8525 :invoke :cas [21 [30 2]] INFO [2019-12-17 12:35:50,396] jepsen worker 69 - jepsen.util 10059 :fail :cas [23 [8 10]] :ValueNotSwapFrom INFO [2019-12-17 12:35:50,396] jepsen worker 69 - jepsen.util 10059 :invoke :write [23 38] INFO [2019-12-17 12:35:50,410] jepsen worker 19 - jepsen.util 10189 :ok :read [6 36] INFO [2019-12-17 12:35:50,410] jepsen worker 19 - jepsen.util 10189 :invoke :read [6 nil] INFO [2019-12-17 12:35:50,415] jepsen worker 12 - jepsen.util 10272 :info :write [4 13] :RequestCanceledException INFO [2019-12-17 12:35:50,417] jepsen worker 12 - jepsen.util 10362 :invoke :read [4 nil] INFO [2019-12-17 12:35:50,430] jepsen worker 65 - jepsen.util 8525 :fail :cas [21 [30 2]] :ValueNotSwapFrom INFO [2019-12-17 12:35:50,430] jepsen worker 65 - jepsen.util 8525 :invoke :cas [21 [39 44]] INFO [2019-12-17 12:35:50,439] jepsen worker 19 - jepsen.util 10189 :ok :read [6 36] INFO [2019-12-17 12:35:50,439] jepsen worker 19 - jepsen.util 10189 :invoke :cas [6 [16 7]] INFO [2019-12-17 12:35:50,446] jepsen worker 23 - jepsen.util 13703 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:35:50,446] jepsen worker 23 - jepsen.util 13703 :invoke :cas [7 [34 14]] INFO [2019-12-17 12:35:50,462] jepsen worker 65 - jepsen.util 8525 :fail :cas [21 [39 44]] :ValueNotSwapFrom INFO [2019-12-17 12:35:50,462] jepsen worker 65 - jepsen.util 8525 :invoke :write [21 14] INFO [2019-12-17 12:35:50,466] jepsen worker 19 - jepsen.util 10189 :fail :cas [6 [16 7]] :ValueNotSwapFrom INFO [2019-12-17 12:35:50,466] jepsen worker 19 - jepsen.util 10189 :invoke :cas [6 [4 0]] INFO [2019-12-17 12:35:50,490] jepsen worker 19 - jepsen.util 10189 :fail :cas [6 [4 0]] :ValueNotSwapFrom INFO [2019-12-17 12:35:50,490] jepsen worker 19 - jepsen.util 10189 :invoke :read [6 nil] INFO [2019-12-17 12:35:50,497] jepsen worker 63 - jepsen.util 8973 :info :write [21 10] :RequestCanceledException INFO [2019-12-17 12:35:50,497] jepsen worker 83 - jepsen.util 13943 :info :write [27 15] :RequestCanceledException INFO [2019-12-17 12:35:50,497] jepsen worker 85 - jepsen.util 14395 :info :cas [28 [23 14]] :RequestCanceledException INFO [2019-12-17 12:35:50,498] jepsen worker 63 - jepsen.util 9063 :invoke :cas [21 [18 29]] INFO [2019-12-17 12:35:50,500] jepsen worker 3 - jepsen.util 9813 :fail :cas [1 [43 30]] :ValueNotSwapFrom INFO [2019-12-17 12:35:50,500] jepsen worker 83 - jepsen.util 14033 :invoke :read [27 nil] INFO [2019-12-17 12:35:50,500] jepsen worker 3 - jepsen.util 9813 :invoke :write [1 13] INFO [2019-12-17 12:35:50,500] jepsen worker 85 - jepsen.util 14485 :invoke :cas [28 [37 23]] INFO [2019-12-17 12:35:50,515] jepsen worker 78 - jepsen.util 9888 :info :write [26 10] :RequestCanceledException INFO [2019-12-17 12:35:50,517] jepsen worker 78 - jepsen.util 9978 :invoke :read [26 nil] INFO [2019-12-17 12:35:50,521] jepsen worker 65 - jepsen.util 8525 :ok :write [21 14] INFO [2019-12-17 12:35:50,521] jepsen worker 65 - jepsen.util 8525 :invoke :cas [21 [20 33]] INFO [2019-12-17 12:35:50,532] jepsen worker 40 - jepsen.util 13990 :info :write [13 10] :RequestCanceledException INFO [2019-12-17 12:35:50,532] jepsen worker 54 - jepsen.util 10944 :info :write [18 20] :RequestCanceledException INFO [2019-12-17 12:35:50,533] jepsen worker 54 - jepsen.util 11034 :invoke :read [18 nil] INFO [2019-12-17 12:35:50,535] jepsen worker 40 - jepsen.util 14080 :invoke :cas [13 [14 16]] INFO [2019-12-17 12:35:50,536] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":57799,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:35:50,536] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":57799,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:35:50,536] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":57803,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:35:50,536] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":57803,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:35:50,560] jepsen worker 19 - jepsen.util 10189 :ok :read [6 36] INFO [2019-12-17 12:35:50,560] jepsen worker 19 - jepsen.util 10189 :invoke :cas [6 [49 41]] INFO [2019-12-17 12:35:50,563] jepsen worker 71 - jepsen.util 10691 :info :write [23 23] :RequestCanceledException INFO [2019-12-17 12:35:50,563] jepsen worker 71 - jepsen.util 10781 :invoke :read [23 nil] INFO [2019-12-17 12:35:50,566] jepsen worker 56 - jepsen.util 10586 :fail :cas [18 [17 38]] :ValueNotSwapFrom INFO [2019-12-17 12:35:50,566] jepsen worker 56 - jepsen.util 10586 :invoke :write [18 22] INFO [2019-12-17 12:35:50,584] jepsen worker 65 - jepsen.util 8525 :fail :cas [21 [20 33]] :ValueNotSwapFrom INFO [2019-12-17 12:35:50,584] jepsen worker 65 - jepsen.util 8525 :invoke :cas [21 [15 39]] INFO [2019-12-17 12:35:50,595] jepsen worker 19 - jepsen.util 10189 :fail :cas [6 [49 41]] :ValueNotSwapFrom INFO [2019-12-17 12:35:50,595] jepsen worker 19 - jepsen.util 10189 :invoke :cas [6 [25 36]] INFO [2019-12-17 12:35:50,598] jepsen worker 66 - jepsen.util 13296 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:35:50,598] jepsen worker 66 - jepsen.util 13296 :invoke :cas [22 [11 9]] INFO [2019-12-17 12:35:50,616] jepsen worker 49 - jepsen.util 13549 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:35:50,616] jepsen worker 10 - jepsen.util 9910 :info :cas [3 [13 49]] :RequestCanceledException INFO [2019-12-17 12:35:50,616] jepsen worker 49 - jepsen.util 13549 :invoke :write [16 49] INFO [2019-12-17 12:35:50,620] jepsen worker 10 - jepsen.util 10000 :invoke :write [3 40] INFO [2019-12-17 12:35:50,624] jepsen worker 65 - jepsen.util 8525 :fail :cas [21 [15 39]] :ValueNotSwapFrom INFO [2019-12-17 12:35:50,624] jepsen worker 65 - jepsen.util 8525 :invoke :read [21 nil] INFO [2019-12-17 12:35:50,625] jepsen worker 70 - jepsen.util 10780 :ok :read [23 31] INFO [2019-12-17 12:35:50,625] jepsen worker 70 - jepsen.util 10780 :invoke :write [23 39] INFO [2019-12-17 12:35:50,631] jepsen worker 0 - jepsen.util 14490 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:35:50,631] jepsen worker 14 - jepsen.util 9734 :info :write [4 16] :RequestCanceledException INFO [2019-12-17 12:35:50,631] jepsen worker 61 - jepsen.util 14461 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:35:50,631] jepsen worker 0 - jepsen.util 14490 :invoke :read [0 nil] INFO [2019-12-17 12:35:50,631] jepsen worker 61 - jepsen.util 14461 :invoke :read [20 nil] INFO [2019-12-17 12:35:50,633] jepsen worker 45 - jepsen.util 10305 :ok :write [15 40] INFO [2019-12-17 12:35:50,633] jepsen worker 45 - jepsen.util 10305 :invoke :read [15 nil] INFO [2019-12-17 12:35:50,634] jepsen worker 30 - jepsen.util 10110 :ok :write [10 17] INFO [2019-12-17 12:35:50,634] jepsen worker 30 - jepsen.util 10110 :invoke :write [10 16] INFO [2019-12-17 12:35:50,637] jepsen worker 14 - jepsen.util 9824 :invoke :read [4 nil] INFO [2019-12-17 12:35:50,645] jepsen worker 26 - jepsen.util 14246 :info :write [8 42] :RequestCanceledException INFO [2019-12-17 12:35:50,652] jepsen worker 26 - jepsen.util 14336 :invoke :write [8 8] INFO [2019-12-17 12:35:50,662] jepsen worker 7 - jepsen.util 13687 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:35:50,663] jepsen worker 7 - jepsen.util 13687 :invoke :write [2 23] INFO [2019-12-17 12:35:50,670] jepsen worker 19 - jepsen.util 10189 :fail :cas [6 [25 36]] :ValueNotSwapFrom INFO [2019-12-17 12:35:50,670] jepsen worker 19 - jepsen.util 10189 :invoke :read [6 nil] INFO [2019-12-17 12:35:50,682] jepsen worker 18 - jepsen.util 10278 :ok :write [6 27] INFO [2019-12-17 12:35:50,683] jepsen worker 18 - jepsen.util 10278 :invoke :read [6 nil] INFO [2019-12-17 12:35:50,691] jepsen worker 72 - jepsen.util 9972 :ok :write [24 32] INFO [2019-12-17 12:35:50,691] jepsen worker 72 - jepsen.util 9972 :invoke :cas [24 [33 26]] INFO [2019-12-17 12:35:50,714] jepsen worker 27 - jepsen.util 9567 :ok :write [9 20] INFO [2019-12-17 12:35:50,714] jepsen worker 27 - jepsen.util 9567 :invoke :cas [9 [11 13]] INFO [2019-12-17 12:35:50,727] jepsen worker 64 - jepsen.util 7714 :fail :cas [21 [45 5]] :ValueNotSwapFrom INFO [2019-12-17 12:35:50,727] jepsen worker 64 - jepsen.util 7714 :invoke :write [21 28] INFO [2019-12-17 12:35:50,731] jepsen worker 1 - jepsen.util 13681 :info :write [0 43] :RequestCanceledException INFO [2019-12-17 12:35:50,735] jepsen worker 1 - jepsen.util 13771 :invoke :cas [0 [35 31]] INFO [2019-12-17 12:35:50,740] jepsen worker 31 - jepsen.util 9031 :fail :cas [10 [13 46]] :ValueNotSwapFrom INFO [2019-12-17 12:35:50,740] jepsen worker 65 - jepsen.util 8525 :ok :read [21 14] INFO [2019-12-17 12:35:50,740] jepsen worker 31 - jepsen.util 9031 :invoke :write [10 6] INFO [2019-12-17 12:35:50,741] jepsen worker 65 - jepsen.util 8525 :invoke :cas [21 [47 36]] INFO [2019-12-17 12:35:50,748] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:35:50,749] jepsen nemesis - couchbase.util Rebalance check count: 452 INFO [2019-12-17 12:35:50,749] jepsen nemesis - couchbase.util Rebalance stuck count: 433 INFO [2019-12-17 12:35:50,749] jepsen worker 63 - jepsen.util 9063 :fail :cas [21 [18 29]] :ValueNotSwapFrom INFO [2019-12-17 12:35:50,749] jepsen worker 63 - jepsen.util 9063 :invoke :read [21 nil] INFO [2019-12-17 12:35:50,753] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":57832,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:35:50,753] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":57832,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:35:50,753] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":57835,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:35:50,753] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":57835,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:35:50,760] jepsen worker 45 - jepsen.util 10305 :ok :read [15 40] INFO [2019-12-17 12:35:50,761] jepsen worker 45 - jepsen.util 10305 :invoke :write [15 0] INFO [2019-12-17 12:35:50,764] jepsen worker 19 - jepsen.util 10189 :ok :read [6 36] INFO [2019-12-17 12:35:50,765] jepsen worker 19 - jepsen.util 10189 :invoke :write [6 33] INFO [2019-12-17 12:35:50,798] jepsen worker 68 - jepsen.util 13028 :info :cas [22 [35 6]] :RequestCanceledException INFO [2019-12-17 12:35:50,801] jepsen worker 68 - jepsen.util 13118 :invoke :cas [22 [27 33]] INFO [2019-12-17 12:35:50,806] jepsen worker 51 - jepsen.util 9411 :ok :read [17 49] INFO [2019-12-17 12:35:50,807] jepsen worker 51 - jepsen.util 9411 :invoke :write [17 29] INFO [2019-12-17 12:35:50,812] jepsen worker 65 - jepsen.util 8525 :fail :cas [21 [47 36]] :ValueNotSwapFrom INFO [2019-12-17 12:35:50,813] jepsen worker 65 - jepsen.util 8525 :invoke :cas [21 [40 7]] INFO [2019-12-17 12:35:50,816] jepsen worker 58 - jepsen.util 14098 :info :cas [19 [13 17]] :RequestCanceledException INFO [2019-12-17 12:35:50,818] jepsen worker 58 - jepsen.util 14188 :invoke :cas [19 [26 4]] INFO [2019-12-17 12:35:50,831] jepsen worker 75 - jepsen.util 13035 :info :write [25 41] :RequestCanceledException INFO [2019-12-17 12:35:50,836] jepsen worker 75 - jepsen.util 13125 :invoke :write [25 40] INFO [2019-12-17 12:35:50,840] jepsen worker 72 - jepsen.util 9972 :fail :cas [24 [33 26]] :ValueNotSwapFrom INFO [2019-12-17 12:35:50,840] jepsen worker 72 - jepsen.util 9972 :invoke :cas [24 [24 49]] INFO [2019-12-17 12:35:50,843] jepsen worker 73 - jepsen.util 9523 :ok :write [24 31] INFO [2019-12-17 12:35:50,843] jepsen worker 73 - jepsen.util 9523 :invoke :write [24 20] INFO [2019-12-17 12:35:50,845] jepsen worker 82 - jepsen.util 13942 :info :cas [27 [4 41]] :RequestCanceledException INFO [2019-12-17 12:35:50,850] jepsen worker 82 - jepsen.util 14032 :invoke :write [27 10] INFO [2019-12-17 12:35:50,857] jepsen worker 63 - jepsen.util 9063 :ok :read [21 14] INFO [2019-12-17 12:35:50,857] jepsen worker 63 - jepsen.util 9063 :invoke :write [21 34] INFO [2019-12-17 12:35:50,860] jepsen worker 47 - jepsen.util 10487 :info :write [15 21] :RequestCanceledException INFO [2019-12-17 12:35:50,864] jepsen worker 47 - jepsen.util 10577 :invoke :read [15 nil] INFO [2019-12-17 12:35:50,895] jepsen worker 36 - jepsen.util 10746 :ok :read [12 13] INFO [2019-12-17 12:35:50,895] jepsen worker 36 - jepsen.util 10746 :invoke :read [12 nil] INFO [2019-12-17 12:35:50,906] jepsen worker 72 - jepsen.util 9972 :fail :cas [24 [24 49]] :ValueNotSwapFrom INFO [2019-12-17 12:35:50,907] jepsen worker 72 - jepsen.util 9972 :invoke :cas [24 [45 23]] INFO [2019-12-17 12:35:50,946] jepsen worker 64 - jepsen.util 7714 :ok :write [21 28] INFO [2019-12-17 12:35:50,946] jepsen worker 64 - jepsen.util 7714 :invoke :write [21 49] INFO [2019-12-17 12:35:50,962] jepsen worker 25 - jepsen.util 14515 :info :write [8 47] :RequestCanceledException INFO [2019-12-17 12:35:50,963] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":57859,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:35:50,964] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":57859,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:35:50,964] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":57860,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:35:50,964] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":57860,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:35:50,966] jepsen worker 25 - jepsen.util 14605 :invoke :cas [8 [15 35]] INFO [2019-12-17 12:35:50,973] jepsen worker 72 - jepsen.util 9972 :fail :cas [24 [45 23]] :ValueNotSwapFrom INFO [2019-12-17 12:35:50,973] jepsen worker 3 - jepsen.util 9813 :ok :write [1 13] INFO [2019-12-17 12:35:50,973] jepsen worker 72 - jepsen.util 9972 :invoke :cas [24 [22 47]] INFO [2019-12-17 12:35:50,973] jepsen worker 3 - jepsen.util 9813 :invoke :read [1 nil] INFO [2019-12-17 12:35:50,996] jepsen worker 20 - jepsen.util 9740 :info :cas [6 [11 5]] :RequestCanceledException INFO [2019-12-17 12:35:50,998] jepsen worker 20 - jepsen.util 9830 :invoke :read [6 nil] INFO [2019-12-17 12:35:51,015] jepsen worker 30 - jepsen.util 10110 :ok :write [10 16] INFO [2019-12-17 12:35:51,015] jepsen worker 30 - jepsen.util 10110 :invoke :cas [10 [49 3]] INFO [2019-12-17 12:35:51,030] jepsen worker 72 - jepsen.util 9972 :fail :cas [24 [22 47]] :ValueNotSwapFrom INFO [2019-12-17 12:35:51,030] jepsen worker 72 - jepsen.util 9972 :invoke :write [24 31] INFO [2019-12-17 12:35:51,031] jepsen worker 32 - jepsen.util 9932 :info :cas [10 [39 8]] :RequestCanceledException INFO [2019-12-17 12:35:51,031] jepsen worker 46 - jepsen.util 9856 :info :write [15 26] :RequestCanceledException INFO [2019-12-17 12:35:51,033] jepsen worker 46 - jepsen.util 9946 :invoke :read [15 nil] INFO [2019-12-17 12:35:51,033] jepsen worker 32 - jepsen.util 10022 :invoke :cas [10 [35 49]] INFO [2019-12-17 12:35:51,052] jepsen worker 52 - jepsen.util 9952 :ok :write [17 18] INFO [2019-12-17 12:35:51,052] jepsen worker 52 - jepsen.util 9952 :invoke :cas [17 [21 15]] INFO [2019-12-17 12:35:51,087] jepsen worker 20 - jepsen.util 9830 :ok :read [6 27] INFO [2019-12-17 12:35:51,087] jepsen worker 20 - jepsen.util 9830 :invoke :write [6 34] INFO [2019-12-17 12:35:51,099] jepsen worker 78 - jepsen.util 9978 :ok :read [26 7] INFO [2019-12-17 12:35:51,099] jepsen worker 78 - jepsen.util 9978 :invoke :write [26 31] INFO [2019-12-17 12:35:51,101] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":57874,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:35:51,101] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":57874,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:35:51,101] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":57877,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:35:51,101] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":57877,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:35:51,101] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":57878,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:35:51,101] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":57878,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:35:51,101] jepsen worker 10 - jepsen.util 10000 :ok :write [3 40] INFO [2019-12-17 12:35:51,102] jepsen worker 10 - jepsen.util 10000 :invoke :write [3 40] INFO [2019-12-17 12:35:51,129] jepsen worker 12 - jepsen.util 10362 :ok :read [4 14] INFO [2019-12-17 12:35:51,129] jepsen worker 12 - jepsen.util 10362 :invoke :write [4 15] INFO [2019-12-17 12:35:51,131] jepsen worker 89 - jepsen.util 10079 :info :write [29 34] :RequestCanceledException INFO [2019-12-17 12:35:51,131] jepsen worker 81 - jepsen.util 13311 :info :write [27 4] :RequestCanceledException INFO [2019-12-17 12:35:51,134] jepsen worker 89 - jepsen.util 10169 :invoke :cas [29 [33 45]] INFO [2019-12-17 12:35:51,137] jepsen worker 81 - jepsen.util 13401 :invoke :read [27 nil] INFO [2019-12-17 12:35:51,142] jepsen worker 63 - jepsen.util 9063 :ok :write [21 34] INFO [2019-12-17 12:35:51,142] jepsen worker 63 - jepsen.util 9063 :invoke :cas [21 [0 29]] INFO [2019-12-17 12:35:51,148] jepsen worker 71 - jepsen.util 10781 :ok :read [23 31] INFO [2019-12-17 12:35:51,149] jepsen worker 71 - jepsen.util 10781 :invoke :cas [23 [2 33]] INFO [2019-12-17 12:35:51,160] jepsen worker 56 - jepsen.util 10586 :ok :write [18 22] INFO [2019-12-17 12:35:51,161] jepsen worker 56 - jepsen.util 10586 :invoke :cas [18 [42 20]] INFO [2019-12-17 12:35:51,174] jepsen worker 46 - jepsen.util 9946 :ok :read [15 40] INFO [2019-12-17 12:35:51,174] jepsen worker 46 - jepsen.util 9946 :invoke :write [15 49] INFO [2019-12-17 12:35:51,194] jepsen worker 63 - jepsen.util 9063 :fail :cas [21 [0 29]] :ValueNotSwapFrom INFO [2019-12-17 12:35:51,195] jepsen worker 63 - jepsen.util 9063 :invoke :write [21 37] INFO [2019-12-17 12:35:51,197] jepsen worker 19 - jepsen.util 10189 :ok :write [6 33] INFO [2019-12-17 12:35:51,197] jepsen worker 19 - jepsen.util 10189 :invoke :cas [6 [18 34]] INFO [2019-12-17 12:35:51,230] jepsen worker 48 - jepsen.util 12468 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:35:51,231] jepsen worker 48 - jepsen.util 12468 :invoke :read [16 nil] INFO [2019-12-17 12:35:51,243] jepsen worker 39 - jepsen.util 13179 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:35:51,243] jepsen worker 33 - jepsen.util 13713 :info :write [11 22] :RequestCanceledException INFO [2019-12-17 12:35:51,243] jepsen worker 88 - jepsen.util 10168 :info :write [29 20] :RequestCanceledException INFO [2019-12-17 12:35:51,243] jepsen worker 39 - jepsen.util 13179 :invoke :write [13 2] INFO [2019-12-17 12:35:51,246] jepsen worker 88 - jepsen.util 10258 :invoke :cas [29 [45 9]] INFO [2019-12-17 12:35:51,248] jepsen worker 33 - jepsen.util 13803 :invoke :read [11 nil] INFO [2019-12-17 12:35:51,267] jepsen worker 89 - jepsen.util 10169 :fail :cas [29 [33 45]] :ValueNotSwapFrom INFO [2019-12-17 12:35:51,267] jepsen worker 89 - jepsen.util 10169 :invoke :read [29 nil] INFO [2019-12-17 12:35:51,272] jepsen worker 19 - jepsen.util 10189 :fail :cas [6 [18 34]] :ValueNotSwapFrom INFO [2019-12-17 12:35:51,272] jepsen worker 19 - jepsen.util 10189 :invoke :write [6 25] INFO [2019-12-17 12:35:51,287] jepsen worker 54 - jepsen.util 11034 :ok :read [18 22] INFO [2019-12-17 12:35:51,287] jepsen worker 54 - jepsen.util 11034 :invoke :write [18 40] INFO [2019-12-17 12:35:51,305] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":57896,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:35:51,305] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":57896,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:35:51,305] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":57907,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:35:51,305] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":57897,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:35:51,306] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":57907,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:35:51,306] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":57897,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:35:51,306] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":57898,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:35:51,306] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":57898,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:35:51,331] jepsen worker 38 - jepsen.util 10748 :info :write [12 48] :RequestCanceledException INFO [2019-12-17 12:35:51,333] jepsen worker 38 - jepsen.util 10838 :invoke :cas [12 [41 0]] INFO [2019-12-17 12:35:51,397] jepsen worker 36 - jepsen.util 10746 :ok :read [12 13] INFO [2019-12-17 12:35:51,397] jepsen worker 36 - jepsen.util 10746 :invoke :write [12 36] INFO [2019-12-17 12:35:51,414] jepsen worker 9 - jepsen.util 10629 :info :cas [3 [16 33]] :RequestCanceledException INFO [2019-12-17 12:35:51,414] jepsen worker 13 - jepsen.util 9373 :info :write [4 26] :RequestCanceledException INFO [2019-12-17 12:35:51,416] jepsen worker 9 - jepsen.util 10719 :invoke :write [3 37] INFO [2019-12-17 12:35:51,419] jepsen worker 13 - jepsen.util 9463 :invoke :cas [4 [49 31]] INFO [2019-12-17 12:35:51,432] jepsen worker 77 - jepsen.util 14297 :info :write [25 0] :RequestCanceledException INFO [2019-12-17 12:35:51,436] jepsen worker 77 - jepsen.util 14387 :invoke :write [25 4] INFO [2019-12-17 12:35:51,440] jepsen worker 56 - jepsen.util 10586 :fail :cas [18 [42 20]] :ValueNotSwapFrom INFO [2019-12-17 12:35:51,440] jepsen worker 56 - jepsen.util 10586 :invoke :read [18 nil] INFO [2019-12-17 12:35:51,515] jepsen worker 28 - jepsen.util 8218 :info :write [9 29] :RequestCanceledException INFO [2019-12-17 12:35:51,521] jepsen worker 28 - jepsen.util 8308 :invoke :cas [9 [18 40]] INFO [2019-12-17 12:35:51,523] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":57921,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:35:51,523] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":57921,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:35:51,524] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":57924,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:35:51,524] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":57924,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:35:51,545] jepsen worker 59 - jepsen.util 12929 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:35:51,545] jepsen worker 59 - jepsen.util 12929 :invoke :read [19 nil] INFO [2019-12-17 12:35:51,563] jepsen worker 53 - jepsen.util 9503 :info :write [17 30] :RequestCanceledException INFO [2019-12-17 12:35:51,564] jepsen worker 53 - jepsen.util 9593 :invoke :cas [17 [36 37]] INFO [2019-12-17 12:35:51,586] jepsen worker 14 - jepsen.util 9824 :ok :read [4 14] INFO [2019-12-17 12:35:51,587] jepsen worker 14 - jepsen.util 9824 :invoke :read [4 nil] INFO [2019-12-17 12:35:51,593] jepsen worker 88 - jepsen.util 10258 :fail :cas [29 [45 9]] :ValueNotSwapFrom INFO [2019-12-17 12:35:51,593] jepsen worker 88 - jepsen.util 10258 :invoke :write [29 44] INFO [2019-12-17 12:35:51,598] jepsen worker 21 - jepsen.util 13251 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:35:51,598] jepsen worker 21 - jepsen.util 13251 :invoke :read [7 nil] INFO [2019-12-17 12:35:51,608] jepsen worker 52 - jepsen.util 9952 :fail :cas [17 [21 15]] :ValueNotSwapFrom INFO [2019-12-17 12:35:51,608] jepsen worker 52 - jepsen.util 9952 :invoke :cas [17 [41 17]] INFO [2019-12-17 12:35:51,615] jepsen worker 34 - jepsen.util 13714 :info :write [11 11] :RequestCanceledException INFO [2019-12-17 12:35:51,618] jepsen worker 34 - jepsen.util 13804 :invoke :cas [11 [44 18]] INFO [2019-12-17 12:35:51,631] jepsen worker 4 - jepsen.util 9274 :info :write [1 48] :RequestCanceledException INFO [2019-12-17 12:35:51,631] jepsen worker 84 - jepsen.util 13224 :info :cas [28 [20 32]] :RequestCanceledException INFO [2019-12-17 12:35:51,631] jepsen worker 8 - jepsen.util 13328 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:35:51,631] jepsen worker 8 - jepsen.util 13328 :invoke :cas [2 [10 28]] INFO [2019-12-17 12:35:51,633] jepsen worker 4 - jepsen.util 9364 :invoke :cas [1 [19 2]] INFO [2019-12-17 12:35:51,635] jepsen worker 84 - jepsen.util 13314 :invoke :write [28 44] INFO [2019-12-17 12:35:51,635] jepsen worker 89 - jepsen.util 10169 :ok :read [29 43] INFO [2019-12-17 12:35:51,635] jepsen worker 89 - jepsen.util 10169 :invoke :cas [29 [49 33]] INFO [2019-12-17 12:35:51,645] jepsen worker 86 - jepsen.util 14576 :info :write [28 34] :RequestCanceledException INFO [2019-12-17 12:35:51,649] jepsen worker 86 - jepsen.util 14666 :invoke :cas [28 [30 19]] INFO [2019-12-17 12:35:51,652] jepsen worker 13 - jepsen.util 9463 :fail :cas [4 [49 31]] :ValueNotSwapFrom INFO [2019-12-17 12:35:51,653] jepsen worker 13 - jepsen.util 9463 :invoke :write [4 42] INFO [2019-12-17 12:35:51,662] jepsen worker 50 - jepsen.util 14270 :info :cas [16 [12 44]] :RequestCanceledException INFO [2019-12-17 12:35:51,662] jepsen worker 67 - jepsen.util 13567 :info :write [22 5] :RequestCanceledException INFO [2019-12-17 12:35:51,668] jepsen worker 50 - jepsen.util 14360 :invoke :write [16 20] INFO [2019-12-17 12:35:51,668] jepsen worker 67 - jepsen.util 13657 :invoke :cas [22 [12 34]] INFO [2019-12-17 12:35:51,680] jepsen worker 3 - jepsen.util 9813 :ok :read [1 13] INFO [2019-12-17 12:35:51,681] jepsen worker 3 - jepsen.util 9813 :invoke :read [1 nil] INFO [2019-12-17 12:35:51,704] jepsen worker 14 - jepsen.util 9824 :ok :read [4 14] INFO [2019-12-17 12:35:51,704] jepsen worker 14 - jepsen.util 9824 :invoke :read [4 nil] INFO [2019-12-17 12:35:51,716] jepsen worker 41 - jepsen.util 14711 :info :write [13 8] :RequestCanceledException INFO [2019-12-17 12:35:51,721] jepsen worker 41 - jepsen.util 14801 :invoke :cas [13 [39 34]] INFO [2019-12-17 12:35:51,723] jepsen worker 52 - jepsen.util 9952 :fail :cas [17 [41 17]] :ValueNotSwapFrom INFO [2019-12-17 12:35:51,723] jepsen worker 52 - jepsen.util 9952 :invoke :cas [17 [30 0]] INFO [2019-12-17 12:35:51,731] jepsen worker 22 - jepsen.util 13252 :info :write [7 20] :RequestCanceledException INFO [2019-12-17 12:35:51,731] jepsen worker 16 - jepsen.util 14956 :info :write [5 3] :RequestCanceledException INFO [2019-12-17 12:35:51,737] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":57944,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:35:51,737] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":57944,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:35:51,737] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":57945,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:35:51,737] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":57945,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:35:51,739] jepsen worker 22 - jepsen.util 13342 :invoke :write [7 25] INFO [2019-12-17 12:35:51,740] jepsen worker 16 - jepsen.util 15046 :invoke :cas [5 [30 17]] INFO [2019-12-17 12:35:51,745] jepsen worker 2 - jepsen.util 13412 :info :cas [0 [29 39]] :RequestCanceledException INFO [2019-12-17 12:35:51,747] jepsen worker 2 - jepsen.util 13502 :invoke :read [0 nil] INFO [2019-12-17 12:35:51,764] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:35:51,765] jepsen nemesis - couchbase.util Rebalance check count: 453 INFO [2019-12-17 12:35:51,765] jepsen nemesis - couchbase.util Rebalance stuck count: 434 INFO [2019-12-17 12:35:51,795] jepsen worker 15 - jepsen.util 12615 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:35:51,795] jepsen worker 15 - jepsen.util 12615 :invoke :write [5 18] INFO [2019-12-17 12:35:51,795] jepsen worker 3 - jepsen.util 9813 :ok :read [1 13] INFO [2019-12-17 12:35:51,795] jepsen worker 3 - jepsen.util 9813 :invoke :read [1 nil] INFO [2019-12-17 12:35:51,816] jepsen worker 43 - jepsen.util 13543 :info :cas [14 [47 47]] :RequestCanceledException INFO [2019-12-17 12:35:51,818] jepsen worker 43 - jepsen.util 13633 :invoke :write [14 4] INFO [2019-12-17 12:35:51,857] jepsen worker 3 - jepsen.util 9813 :ok :read [1 13] INFO [2019-12-17 12:35:51,857] jepsen worker 3 - jepsen.util 9813 :invoke :write [1 4] INFO [2019-12-17 12:35:51,915] jepsen worker 74 - jepsen.util 9704 :info :write [24 6] :RequestCanceledException INFO [2019-12-17 12:35:51,918] jepsen worker 74 - jepsen.util 9794 :invoke :write [24 42] INFO [2019-12-17 12:35:51,921] jepsen worker 71 - jepsen.util 10781 :fail :cas [23 [2 33]] :ValueNotSwapFrom INFO [2019-12-17 12:35:51,921] jepsen worker 71 - jepsen.util 10781 :invoke :cas [23 [1 40]] INFO [2019-12-17 12:35:51,932] jepsen worker 17 - jepsen.util 13427 :info :write [5 42] :RequestCanceledException INFO [2019-12-17 12:35:51,937] jepsen worker 17 - jepsen.util 13517 :invoke :write [5 34] INFO [2019-12-17 12:35:51,945] jepsen worker 42 - jepsen.util 14532 :info :write [14 44] :RequestCanceledException INFO [2019-12-17 12:35:51,947] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":57963,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:35:51,947] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":57963,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:35:51,947] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":57964,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:35:51,947] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":57964,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:35:51,947] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":57965,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:35:51,947] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":57965,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:35:51,957] jepsen worker 42 - jepsen.util 14622 :invoke :write [14 2] INFO [2019-12-17 12:35:51,962] jepsen worker 60 - jepsen.util 14550 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:35:51,962] jepsen worker 60 - jepsen.util 14550 :invoke :write [20 13] INFO [2019-12-17 12:35:51,973] jepsen worker 89 - jepsen.util 10169 :fail :cas [29 [49 33]] :ValueNotSwapFrom INFO [2019-12-17 12:35:51,973] jepsen worker 89 - jepsen.util 10169 :invoke :cas [29 [22 49]] INFO [2019-12-17 12:35:51,998] jepsen worker 57 - jepsen.util 13197 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:35:51,998] jepsen worker 57 - jepsen.util 13197 :invoke :write [19 39] INFO [2019-12-17 12:35:52,003] jepsen worker 53 - jepsen.util 9593 :fail :cas [17 [36 37]] :ValueNotSwapFrom INFO [2019-12-17 12:35:52,003] jepsen worker 53 - jepsen.util 9593 :invoke :cas [17 [23 25]] INFO [2019-12-17 12:35:52,010] jepsen worker 56 - jepsen.util 10586 :ok :read [18 22] INFO [2019-12-17 12:35:52,010] jepsen worker 56 - jepsen.util 10586 :invoke :read [18 nil] INFO [2019-12-17 12:35:52,027] jepsen worker 38 - jepsen.util 10838 :fail :cas [12 [41 0]] :ValueNotSwapFrom INFO [2019-12-17 12:35:52,028] jepsen worker 38 - jepsen.util 10838 :invoke :cas [12 [49 38]] INFO [2019-12-17 12:35:52,031] jepsen worker 5 - jepsen.util 9455 :info :write [1 9] :RequestCanceledException INFO [2019-12-17 12:35:52,033] jepsen worker 5 - jepsen.util 9545 :invoke :write [1 24] INFO [2019-12-17 12:35:52,063] jepsen worker 37 - jepsen.util 9577 :info :write [12 27] :RequestCanceledException INFO [2019-12-17 12:35:52,065] jepsen worker 37 - jepsen.util 9667 :invoke :read [12 nil] INFO [2019-12-17 12:35:52,085] jepsen worker 4 - jepsen.util 9364 :fail :cas [1 [19 2]] :ValueNotSwapFrom INFO [2019-12-17 12:35:52,085] jepsen worker 4 - jepsen.util 9364 :invoke :cas [1 [24 44]] INFO [2019-12-17 12:35:52,089] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":57973,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:35:52,090] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":57973,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:35:52,090] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":57974,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:35:52,090] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":57974,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:35:52,090] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":57975,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:35:52,090] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":57975,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:35:52,115] jepsen worker 11 - jepsen.util 9821 :fail :read [3 nil] :RequestTimeoutException INFO [2019-12-17 12:35:52,115] jepsen worker 6 - jepsen.util 14136 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:35:52,115] jepsen worker 79 - jepsen.util 10249 :info :write [26 40] :RequestCanceledException INFO [2019-12-17 12:35:52,116] jepsen worker 6 - jepsen.util 14136 :invoke :read [2 nil] INFO [2019-12-17 12:35:52,116] jepsen worker 11 - jepsen.util 9821 :invoke :write [3 1] INFO [2019-12-17 12:35:52,118] jepsen worker 79 - jepsen.util 10339 :invoke :write [26 32] INFO [2019-12-17 12:35:52,145] jepsen worker 76 - jepsen.util 13846 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:35:52,145] jepsen worker 80 - jepsen.util 9710 :info :write [26 9] :RequestCanceledException INFO [2019-12-17 12:35:52,145] jepsen worker 76 - jepsen.util 13846 :invoke :cas [25 [28 35]] INFO [2019-12-17 12:35:52,148] jepsen worker 80 - jepsen.util 9800 :invoke :read [26 nil] INFO [2019-12-17 12:35:52,155] jepsen worker 4 - jepsen.util 9364 :fail :cas [1 [24 44]] :ValueNotSwapFrom INFO [2019-12-17 12:35:52,155] jepsen worker 4 - jepsen.util 9364 :invoke :cas [1 [23 44]] INFO [2019-12-17 12:35:52,162] jepsen worker 55 - jepsen.util 10765 :info :write [18 7] :RequestCanceledException INFO [2019-12-17 12:35:52,164] jepsen worker 55 - jepsen.util 10855 :invoke :cas [18 [1 46]] INFO [2019-12-17 12:35:52,173] jepsen worker 71 - jepsen.util 10781 :fail :cas [23 [1 40]] :ValueNotSwapFrom INFO [2019-12-17 12:35:52,173] jepsen worker 71 - jepsen.util 10781 :invoke :read [23 nil] INFO [2019-12-17 12:35:52,227] jepsen worker 14 - jepsen.util 9824 :ok :read [4 14] INFO [2019-12-17 12:35:52,227] jepsen worker 14 - jepsen.util 9824 :invoke :cas [4 [21 31]] INFO [2019-12-17 12:35:52,232] jepsen worker 24 - jepsen.util 12354 :info :cas [8 [8 5]] :RequestCanceledException INFO [2019-12-17 12:35:52,234] jepsen worker 24 - jepsen.util 12444 :invoke :read [8 nil] INFO [2019-12-17 12:35:52,270] jepsen worker 52 - jepsen.util 9952 :fail :cas [17 [30 0]] :ValueNotSwapFrom INFO [2019-12-17 12:35:52,270] jepsen worker 52 - jepsen.util 9952 :invoke :write [17 27] INFO [2019-12-17 12:35:52,290] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58001,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:35:52,291] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58001,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:35:52,291] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58004,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:35:52,291] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58004,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:35:52,291] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58005,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:35:52,291] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58005,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:35:52,294] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":57994,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:35:52,294] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":57994,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:35:52,294] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":57996,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:35:52,294] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":57996,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:35:52,298] jepsen worker 44 - jepsen.util 13904 :info :write [14 6] :RequestCanceledException INFO [2019-12-17 12:35:52,300] jepsen worker 44 - jepsen.util 13994 :invoke :write [14 7] INFO [2019-12-17 12:35:52,315] jepsen worker 56 - jepsen.util 10586 :ok :read [18 22] INFO [2019-12-17 12:35:52,315] jepsen worker 56 - jepsen.util 10586 :invoke :write [18 39] INFO [2019-12-17 12:35:52,316] jepsen worker 29 - jepsen.util 8399 :info :cas [9 [18 46]] :RequestCanceledException INFO [2019-12-17 12:35:52,317] jepsen worker 29 - jepsen.util 8489 :invoke :cas [9 [27 25]] INFO [2019-12-17 12:35:52,344] jepsen worker 35 - jepsen.util 13535 :info :write [11 42] :RequestCanceledException INFO [2019-12-17 12:35:52,347] jepsen worker 35 - jepsen.util 13625 :invoke :cas [11 [35 10]] INFO [2019-12-17 12:35:52,350] jepsen worker 38 - jepsen.util 10838 :fail :cas [12 [49 38]] :ValueNotSwapFrom INFO [2019-12-17 12:35:52,351] jepsen worker 38 - jepsen.util 10838 :invoke :read [12 nil] INFO [2019-12-17 12:35:52,398] jepsen worker 87 - jepsen.util 9627 :info :write [29 34] :RequestCanceledException INFO [2019-12-17 12:35:52,400] jepsen worker 87 - jepsen.util 9717 :invoke :read [29 nil] INFO [2019-12-17 12:35:52,402] jepsen worker 37 - jepsen.util 9667 :ok :read [12 13] INFO [2019-12-17 12:35:52,402] jepsen worker 37 - jepsen.util 9667 :invoke :read [12 nil] INFO [2019-12-17 12:35:52,402] jepsen worker 53 - jepsen.util 9593 :fail :cas [17 [23 25]] :ValueNotSwapFrom INFO [2019-12-17 12:35:52,402] jepsen worker 53 - jepsen.util 9593 :invoke :read [17 nil] INFO [2019-12-17 12:35:52,431] jepsen worker 62 - jepsen.util 14012 :info :cas [20 [25 6]] :RequestCanceledException INFO [2019-12-17 12:35:52,435] jepsen worker 62 - jepsen.util 14102 :invoke :read [20 nil] INFO [2019-12-17 12:35:52,450] jepsen worker 55 - jepsen.util 10855 :fail :cas [18 [1 46]] :ValueNotSwapFrom INFO [2019-12-17 12:35:52,450] jepsen worker 55 - jepsen.util 10855 :invoke :write [18 41] INFO [2019-12-17 12:35:52,459] jepsen worker 71 - jepsen.util 10781 :ok :read [23 31] INFO [2019-12-17 12:35:52,460] jepsen worker 71 - jepsen.util 10781 :invoke :cas [23 [31 47]] INFO [2019-12-17 12:35:52,479] jepsen worker 38 - jepsen.util 10838 :ok :read [12 13] INFO [2019-12-17 12:35:52,479] jepsen worker 38 - jepsen.util 10838 :invoke :read [12 nil] INFO [2019-12-17 12:35:52,480] jepsen worker 80 - jepsen.util 9800 :ok :read [26 7] INFO [2019-12-17 12:35:52,480] jepsen worker 80 - jepsen.util 9800 :invoke :read [26 nil] INFO [2019-12-17 12:35:52,498] jepsen worker 69 - jepsen.util 10059 :info :write [23 38] :RequestCanceledException INFO [2019-12-17 12:35:52,502] jepsen worker 69 - jepsen.util 10149 :invoke :write [23 48] INFO [2019-12-17 12:35:52,516] jepsen worker 85 - jepsen.util 14485 :info :cas [28 [37 23]] :RequestCanceledException INFO [2019-12-17 12:35:52,518] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58018,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:35:52,518] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58018,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:35:52,518] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58022,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:35:52,518] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58022,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:35:52,520] jepsen worker 85 - jepsen.util 14575 :invoke :cas [28 [9 47]] INFO [2019-12-17 12:35:52,526] jepsen worker 19 - jepsen.util 10189 :ok :write [6 25] INFO [2019-12-17 12:35:52,527] jepsen worker 19 - jepsen.util 10189 :invoke :cas [6 [24 42]] INFO [2019-12-17 12:35:52,536] jepsen worker 31 - jepsen.util 9031 :ok :write [10 6] INFO [2019-12-17 12:35:52,537] jepsen worker 31 - jepsen.util 9031 :invoke :read [10 nil] INFO [2019-12-17 12:35:52,538] jepsen worker 73 - jepsen.util 9523 :ok :write [24 20] INFO [2019-12-17 12:35:52,539] jepsen worker 73 - jepsen.util 9523 :invoke :read [24 nil] INFO [2019-12-17 12:35:52,541] jepsen worker 23 - jepsen.util 13703 :info :cas [7 [34 14]] :RequestCanceledException INFO [2019-12-17 12:35:52,544] jepsen worker 23 - jepsen.util 13793 :invoke :write [7 4] INFO [2019-12-17 12:35:52,544] jepsen worker 37 - jepsen.util 9667 :ok :read [12 13] INFO [2019-12-17 12:35:52,544] jepsen worker 37 - jepsen.util 9667 :invoke :read [12 nil] INFO [2019-12-17 12:35:52,548] jepsen worker 11 - jepsen.util 9821 :ok :write [3 1] INFO [2019-12-17 12:35:52,548] jepsen worker 46 - jepsen.util 9946 :ok :write [15 49] INFO [2019-12-17 12:35:52,548] jepsen worker 11 - jepsen.util 9821 :invoke :cas [3 [18 40]] INFO [2019-12-17 12:35:52,548] jepsen worker 46 - jepsen.util 9946 :invoke :cas [15 [1 22]] INFO [2019-12-17 12:35:52,560] jepsen worker 40 - jepsen.util 14080 :info :cas [13 [14 16]] :RequestCanceledException INFO [2019-12-17 12:35:52,565] jepsen worker 40 - jepsen.util 14170 :invoke :write [13 10] INFO [2019-12-17 12:35:52,583] jepsen worker 4 - jepsen.util 9364 :fail :cas [1 [23 44]] :ValueNotSwapFrom INFO [2019-12-17 12:35:52,583] jepsen worker 4 - jepsen.util 9364 :invoke :cas [1 [15 18]] INFO [2019-12-17 12:35:52,588] jepsen worker 28 - jepsen.util 8308 :fail :cas [9 [18 40]] :ValueNotSwapFrom INFO [2019-12-17 12:35:52,589] jepsen worker 28 - jepsen.util 8308 :invoke :cas [9 [0 37]] INFO [2019-12-17 12:35:52,596] jepsen worker 83 - jepsen.util 14033 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:35:52,596] jepsen worker 83 - jepsen.util 14033 :invoke :read [27 nil] INFO [2019-12-17 12:35:52,610] jepsen worker 19 - jepsen.util 10189 :fail :cas [6 [24 42]] :ValueNotSwapFrom INFO [2019-12-17 12:35:52,610] jepsen worker 19 - jepsen.util 10189 :invoke :cas [6 [34 33]] INFO [2019-12-17 12:35:52,619] jepsen worker 46 - jepsen.util 9946 :fail :cas [15 [1 22]] :ValueNotSwapFrom INFO [2019-12-17 12:35:52,619] jepsen worker 46 - jepsen.util 9946 :invoke :cas [15 [2 14]] INFO [2019-12-17 12:35:52,627] jepsen worker 30 - jepsen.util 10110 :fail :cas [10 [49 3]] :ValueNotSwapFrom INFO [2019-12-17 12:35:52,627] jepsen worker 30 - jepsen.util 10110 :invoke :read [10 nil] INFO [2019-12-17 12:35:52,656] jepsen worker 11 - jepsen.util 9821 :fail :cas [3 [18 40]] :ValueNotSwapFrom INFO [2019-12-17 12:35:52,656] jepsen worker 11 - jepsen.util 9821 :invoke :read [3 nil] INFO [2019-12-17 12:35:52,689] jepsen worker 87 - jepsen.util 9717 :ok :read [29 43] INFO [2019-12-17 12:35:52,689] jepsen worker 87 - jepsen.util 9717 :invoke :read [29 nil] INFO [2019-12-17 12:35:52,694] jepsen worker 66 - jepsen.util 13296 :info :cas [22 [11 9]] :RequestCanceledException INFO [2019-12-17 12:35:52,694] jepsen worker 18 - jepsen.util 10278 :fail :read [6 nil] :RequestTimeoutException INFO [2019-12-17 12:35:52,694] jepsen worker 18 - jepsen.util 10278 :invoke :write [6 41] INFO [2019-12-17 12:35:52,699] jepsen worker 66 - jepsen.util 13386 :invoke :cas [22 [4 31]] INFO [2019-12-17 12:35:52,702] jepsen worker 29 - jepsen.util 8489 :fail :cas [9 [27 25]] :ValueNotSwapFrom INFO [2019-12-17 12:35:52,702] jepsen worker 29 - jepsen.util 8489 :invoke :cas [9 [28 24]] INFO [2019-12-17 12:35:52,706] jepsen worker 28 - jepsen.util 8308 :fail :cas [9 [0 37]] :ValueNotSwapFrom INFO [2019-12-17 12:35:52,707] jepsen worker 28 - jepsen.util 8308 :invoke :write [9 7] INFO [2019-12-17 12:35:52,713] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58045,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:35:52,713] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58045,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:35:52,714] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58048,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:35:52,714] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58048,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:35:52,714] jepsen worker 49 - jepsen.util 13549 :info :write [16 49] :RequestCanceledException INFO [2019-12-17 12:35:52,714] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58049,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:35:52,714] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58049,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:35:52,720] jepsen worker 49 - jepsen.util 13639 :invoke :cas [16 [7 7]] INFO [2019-12-17 12:35:52,725] jepsen worker 53 - jepsen.util 9593 :ok :read [17 18] INFO [2019-12-17 12:35:52,725] jepsen worker 53 - jepsen.util 9593 :invoke :cas [17 [27 36]] INFO [2019-12-17 12:35:52,730] jepsen worker 61 - jepsen.util 14461 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:35:52,731] jepsen worker 0 - jepsen.util 14490 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:35:52,731] jepsen worker 61 - jepsen.util 14461 :invoke :read [20 nil] INFO [2019-12-17 12:35:52,731] jepsen worker 0 - jepsen.util 14490 :invoke :cas [0 [30 35]] INFO [2019-12-17 12:35:52,731] jepsen worker 26 - jepsen.util 14336 :info :write [8 8] :RequestCanceledException INFO [2019-12-17 12:35:52,731] jepsen worker 27 - jepsen.util 9567 :info :cas [9 [11 13]] :RequestCanceledException INFO [2019-12-17 12:35:52,734] jepsen worker 27 - jepsen.util 9657 :invoke :cas [9 [33 43]] INFO [2019-12-17 12:35:52,736] jepsen worker 26 - jepsen.util 14426 :invoke :cas [8 [16 43]] INFO [2019-12-17 12:35:52,738] jepsen worker 31 - jepsen.util 9031 :ok :read [10 6] INFO [2019-12-17 12:35:52,738] jepsen worker 31 - jepsen.util 9031 :invoke :cas [10 [25 17]] INFO [2019-12-17 12:35:52,742] jepsen worker 46 - jepsen.util 9946 :fail :cas [15 [2 14]] :ValueNotSwapFrom INFO [2019-12-17 12:35:52,742] jepsen worker 46 - jepsen.util 9946 :invoke :read [15 nil] INFO [2019-12-17 12:35:52,746] jepsen worker 70 - jepsen.util 10780 :info :write [23 39] :RequestCanceledException INFO [2019-12-17 12:35:52,748] jepsen worker 70 - jepsen.util 10870 :invoke :write [23 27] INFO [2019-12-17 12:35:52,748] jepsen worker 30 - jepsen.util 10110 :ok :read [10 6] INFO [2019-12-17 12:35:52,748] jepsen worker 30 - jepsen.util 10110 :invoke :read [10 nil] INFO [2019-12-17 12:35:52,762] jepsen worker 7 - jepsen.util 13687 :info :write [2 23] :RequestCanceledException INFO [2019-12-17 12:35:52,778] jepsen worker 7 - jepsen.util 13777 :invoke :cas [2 [44 29]] INFO [2019-12-17 12:35:52,799] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:35:52,800] jepsen nemesis - couchbase.util Rebalance check count: 454 INFO [2019-12-17 12:35:52,800] jepsen nemesis - couchbase.util Rebalance stuck count: 435 INFO [2019-12-17 12:35:52,806] jepsen worker 29 - jepsen.util 8489 :fail :cas [9 [28 24]] :ValueNotSwapFrom INFO [2019-12-17 12:35:52,806] jepsen worker 29 - jepsen.util 8489 :invoke :read [9 nil] INFO [2019-12-17 12:35:52,814] jepsen worker 31 - jepsen.util 9031 :fail :cas [10 [25 17]] :ValueNotSwapFrom INFO [2019-12-17 12:35:52,815] jepsen worker 31 - jepsen.util 9031 :invoke :read [10 nil] INFO [2019-12-17 12:35:52,818] jepsen worker 89 - jepsen.util 10169 :fail :cas [29 [22 49]] :ValueNotSwapFrom INFO [2019-12-17 12:35:52,819] jepsen worker 89 - jepsen.util 10169 :invoke :cas [29 [27 46]] INFO [2019-12-17 12:35:52,820] jepsen worker 19 - jepsen.util 10189 :fail :cas [6 [34 33]] :ValueNotSwapFrom INFO [2019-12-17 12:35:52,821] jepsen worker 19 - jepsen.util 10189 :invoke :write [6 13] INFO [2019-12-17 12:35:52,831] jepsen worker 45 - jepsen.util 10305 :info :write [15 0] :RequestCanceledException INFO [2019-12-17 12:35:52,831] jepsen worker 58 - jepsen.util 14188 :info :cas [19 [26 4]] :RequestCanceledException INFO [2019-12-17 12:35:52,833] jepsen worker 45 - jepsen.util 10395 :invoke :read [15 nil] INFO [2019-12-17 12:35:52,834] jepsen worker 58 - jepsen.util 14278 :invoke :cas [19 [22 5]] INFO [2019-12-17 12:35:52,844] jepsen worker 1 - jepsen.util 13771 :info :cas [0 [35 31]] :RequestCanceledException INFO [2019-12-17 12:35:52,844] jepsen worker 65 - jepsen.util 8525 :info :cas [21 [40 7]] :RequestCanceledException INFO [2019-12-17 12:35:52,848] jepsen worker 1 - jepsen.util 13861 :invoke :read [0 nil] INFO [2019-12-17 12:35:52,852] jepsen worker 65 - jepsen.util 8615 :invoke :cas [21 [47 16]] INFO [2019-12-17 12:35:52,859] jepsen worker 68 - jepsen.util 13118 :info :cas [22 [27 33]] :RequestCanceledException INFO [2019-12-17 12:35:52,859] jepsen worker 51 - jepsen.util 9411 :info :write [17 29] :RequestCanceledException INFO [2019-12-17 12:35:52,862] jepsen worker 68 - jepsen.util 13208 :invoke :read [22 nil] INFO [2019-12-17 12:35:52,863] jepsen worker 51 - jepsen.util 9501 :invoke :cas [17 [38 41]] INFO [2019-12-17 12:35:52,870] jepsen worker 11 - jepsen.util 9821 :ok :read [3 1] INFO [2019-12-17 12:35:52,870] jepsen worker 11 - jepsen.util 9821 :invoke :read [3 nil] INFO [2019-12-17 12:35:52,878] jepsen worker 30 - jepsen.util 10110 :ok :read [10 6] INFO [2019-12-17 12:35:52,878] jepsen worker 30 - jepsen.util 10110 :invoke :read [10 nil] INFO [2019-12-17 12:35:52,898] jepsen worker 29 - jepsen.util 8489 :ok :read [9 20] INFO [2019-12-17 12:35:52,898] jepsen worker 75 - jepsen.util 13125 :info :write [25 40] :RequestCanceledException INFO [2019-12-17 12:35:52,898] jepsen worker 29 - jepsen.util 8489 :invoke :cas [9 [35 30]] INFO [2019-12-17 12:35:52,902] jepsen worker 75 - jepsen.util 13215 :invoke :write [25 38] INFO [2019-12-17 12:35:52,916] jepsen worker 82 - jepsen.util 14032 :info :write [27 10] :RequestCanceledException INFO [2019-12-17 12:35:52,922] jepsen worker 82 - jepsen.util 14122 :invoke :cas [27 [30 20]] INFO [2019-12-17 12:35:52,931] jepsen worker 47 - jepsen.util 10577 :fail :read [15 nil] :RequestTimeoutException INFO [2019-12-17 12:35:52,932] jepsen worker 47 - jepsen.util 10577 :invoke :cas [15 [35 44]] INFO [2019-12-17 12:35:52,932] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58076,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:35:52,932] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58076,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:35:52,933] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58079,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:35:52,933] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58079,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:35:52,960] jepsen worker 31 - jepsen.util 9031 :ok :read [10 6] INFO [2019-12-17 12:35:52,961] jepsen worker 31 - jepsen.util 9031 :invoke :read [10 nil] INFO [2019-12-17 12:35:52,963] jepsen worker 45 - jepsen.util 10395 :ok :read [15 49] INFO [2019-12-17 12:35:52,963] jepsen worker 45 - jepsen.util 10395 :invoke :cas [15 [33 13]] INFO [2019-12-17 12:35:52,974] jepsen worker 30 - jepsen.util 10110 :ok :read [10 6] INFO [2019-12-17 12:35:52,975] jepsen worker 30 - jepsen.util 10110 :invoke :read [10 nil] INFO [2019-12-17 12:35:52,979] jepsen worker 29 - jepsen.util 8489 :fail :cas [9 [35 30]] :ValueNotSwapFrom INFO [2019-12-17 12:35:52,979] jepsen worker 29 - jepsen.util 8489 :invoke :write [9 13] INFO [2019-12-17 12:35:53,012] jepsen worker 47 - jepsen.util 10577 :fail :cas [15 [35 44]] :ValueNotSwapFrom INFO [2019-12-17 12:35:53,012] jepsen worker 47 - jepsen.util 10577 :invoke :write [15 46] INFO [2019-12-17 12:35:53,029] jepsen worker 37 - jepsen.util 9667 :ok :read [12 13] INFO [2019-12-17 12:35:53,029] jepsen worker 37 - jepsen.util 9667 :invoke :cas [12 [43 13]] INFO [2019-12-17 12:35:53,030] jepsen worker 46 - jepsen.util 9946 :ok :read [15 49] INFO [2019-12-17 12:35:53,031] jepsen worker 46 - jepsen.util 9946 :invoke :cas [15 [6 20]] INFO [2019-12-17 12:35:53,031] jepsen worker 64 - jepsen.util 7714 :info :write [21 49] :RequestCanceledException INFO [2019-12-17 12:35:53,034] jepsen worker 64 - jepsen.util 7804 :invoke :write [21 44] INFO [2019-12-17 12:35:53,045] jepsen worker 25 - jepsen.util 14605 :info :cas [8 [15 35]] :RequestCanceledException INFO [2019-12-17 12:35:53,052] jepsen worker 31 - jepsen.util 9031 :ok :read [10 6] INFO [2019-12-17 12:35:53,052] jepsen worker 31 - jepsen.util 9031 :invoke :read [10 nil] INFO [2019-12-17 12:35:53,057] jepsen worker 25 - jepsen.util 14695 :invoke :read [8 nil] INFO [2019-12-17 12:35:53,066] jepsen worker 45 - jepsen.util 10395 :fail :cas [15 [33 13]] :ValueNotSwapFrom INFO [2019-12-17 12:35:53,066] jepsen worker 45 - jepsen.util 10395 :invoke :write [15 1] INFO [2019-12-17 12:35:53,068] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58095,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:35:53,068] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58095,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:35:53,068] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58098,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:35:53,068] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58098,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:35:53,068] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58099,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:35:53,068] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58099,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:35:53,087] jepsen worker 10 - jepsen.util 10000 :ok :write [3 40] INFO [2019-12-17 12:35:53,087] jepsen worker 10 - jepsen.util 10000 :invoke :cas [3 [49 23]] INFO [2019-12-17 12:35:53,088] jepsen worker 28 - jepsen.util 8308 :ok :write [9 7] INFO [2019-12-17 12:35:53,088] jepsen worker 28 - jepsen.util 8308 :invoke :write [9 3] INFO [2019-12-17 12:35:53,096] jepsen worker 32 - jepsen.util 10022 :info :cas [10 [35 49]] :RequestCanceledException INFO [2019-12-17 12:35:53,103] jepsen worker 32 - jepsen.util 10112 :invoke :cas [10 [15 10]] INFO [2019-12-17 12:35:53,112] jepsen worker 30 - jepsen.util 10110 :ok :read [10 6] INFO [2019-12-17 12:35:53,112] jepsen worker 30 - jepsen.util 10110 :invoke :write [10 0] INFO [2019-12-17 12:35:53,114] jepsen worker 72 - jepsen.util 9972 :info :write [24 31] :RequestCanceledException INFO [2019-12-17 12:35:53,117] jepsen worker 72 - jepsen.util 10062 :invoke :write [24 7] INFO [2019-12-17 12:35:53,132] jepsen worker 78 - jepsen.util 9978 :info :write [26 31] :RequestCanceledException INFO [2019-12-17 12:35:53,136] jepsen worker 78 - jepsen.util 10068 :invoke :read [26 nil] INFO [2019-12-17 12:35:53,144] jepsen worker 14 - jepsen.util 9824 :fail :cas [4 [21 31]] :ValueNotSwapFrom INFO [2019-12-17 12:35:53,144] jepsen worker 14 - jepsen.util 9824 :invoke :write [4 3] INFO [2019-12-17 12:35:53,162] jepsen worker 20 - jepsen.util 9830 :info :write [6 34] :RequestCanceledException INFO [2019-12-17 12:35:53,163] jepsen worker 12 - jepsen.util 10362 :info :write [4 15] :RequestCanceledException INFO [2019-12-17 12:35:53,165] jepsen worker 65 - jepsen.util 8615 :fail :cas [21 [47 16]] :ValueNotSwapFrom INFO [2019-12-17 12:35:53,165] jepsen worker 65 - jepsen.util 8615 :invoke :cas [21 [26 25]] INFO [2019-12-17 12:35:53,166] jepsen worker 20 - jepsen.util 9920 :invoke :write [6 6] INFO [2019-12-17 12:35:53,166] jepsen worker 12 - jepsen.util 10452 :invoke :write [4 9] INFO [2019-12-17 12:35:53,177] jepsen worker 38 - jepsen.util 10838 :ok :read [12 13] INFO [2019-12-17 12:35:53,177] jepsen worker 38 - jepsen.util 10838 :invoke :read [12 nil] INFO [2019-12-17 12:35:53,198] jepsen worker 29 - jepsen.util 8489 :ok :write [9 13] INFO [2019-12-17 12:35:53,198] jepsen worker 29 - jepsen.util 8489 :invoke :read [9 nil] INFO [2019-12-17 12:35:53,222] jepsen worker 73 - jepsen.util 9523 :ok :read [24 20] INFO [2019-12-17 12:35:53,222] jepsen worker 73 - jepsen.util 9523 :invoke :write [24 30] INFO [2019-12-17 12:35:53,231] jepsen worker 63 - jepsen.util 9063 :info :write [21 37] :RequestCanceledException INFO [2019-12-17 12:35:53,231] jepsen worker 81 - jepsen.util 13401 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:35:53,231] jepsen worker 81 - jepsen.util 13401 :invoke :cas [27 [32 43]] INFO [2019-12-17 12:35:53,233] jepsen worker 63 - jepsen.util 9153 :invoke :write [21 31] INFO [2019-12-17 12:35:53,246] jepsen worker 11 - jepsen.util 9821 :ok :read [3 40] INFO [2019-12-17 12:35:53,246] jepsen worker 11 - jepsen.util 9821 :invoke :write [3 47] INFO [2019-12-17 12:35:53,259] jepsen worker 33 - jepsen.util 13803 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:35:53,260] jepsen worker 33 - jepsen.util 13803 :invoke :read [11 nil] INFO [2019-12-17 12:35:53,265] jepsen worker 46 - jepsen.util 9946 :fail :cas [15 [6 20]] :ValueNotSwapFrom INFO [2019-12-17 12:35:53,266] jepsen worker 46 - jepsen.util 9946 :invoke :write [15 47] INFO [2019-12-17 12:35:53,270] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58126,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:35:53,270] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58126,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:35:53,270] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58129,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:35:53,270] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58129,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:35:53,282] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58119,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:35:53,282] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58119,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:35:53,282] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58120,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:35:53,282] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58120,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:35:53,282] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58121,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:35:53,282] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58121,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:35:53,314] jepsen worker 65 - jepsen.util 8615 :fail :cas [21 [26 25]] :ValueNotSwapFrom INFO [2019-12-17 12:35:53,314] jepsen worker 65 - jepsen.util 8615 :invoke :read [21 nil] INFO [2019-12-17 12:35:53,316] jepsen worker 4 - jepsen.util 9364 :fail :cas [1 [15 18]] :ValueNotSwapFrom INFO [2019-12-17 12:35:53,316] jepsen worker 4 - jepsen.util 9364 :invoke :read [1 nil] INFO [2019-12-17 12:35:53,332] jepsen worker 48 - jepsen.util 12468 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:35:53,332] jepsen worker 48 - jepsen.util 12468 :invoke :cas [16 [43 2]] INFO [2019-12-17 12:35:53,346] jepsen worker 39 - jepsen.util 13179 :info :write [13 2] :RequestCanceledException INFO [2019-12-17 12:35:53,346] jepsen worker 54 - jepsen.util 11034 :info :write [18 40] :RequestCanceledException INFO [2019-12-17 12:35:53,348] jepsen worker 54 - jepsen.util 11124 :invoke :read [18 nil] INFO [2019-12-17 12:35:53,355] jepsen worker 39 - jepsen.util 13269 :invoke :cas [13 [38 17]] INFO [2019-12-17 12:35:53,375] jepsen worker 80 - jepsen.util 9800 :ok :read [26 7] INFO [2019-12-17 12:35:53,375] jepsen worker 37 - jepsen.util 9667 :fail :cas [12 [43 13]] :ValueNotSwapFrom INFO [2019-12-17 12:35:53,376] jepsen worker 80 - jepsen.util 9800 :invoke :cas [26 [7 6]] INFO [2019-12-17 12:35:53,376] jepsen worker 37 - jepsen.util 9667 :invoke :read [12 nil] INFO [2019-12-17 12:35:53,380] jepsen worker 63 - jepsen.util 9153 :ok :write [21 31] INFO [2019-12-17 12:35:53,380] jepsen worker 63 - jepsen.util 9153 :invoke :read [21 nil] INFO [2019-12-17 12:35:53,431] jepsen worker 36 - jepsen.util 10746 :info :write [12 36] :RequestCanceledException INFO [2019-12-17 12:35:53,437] jepsen worker 36 - jepsen.util 10836 :invoke :write [12 19] INFO [2019-12-17 12:35:53,439] jepsen worker 87 - jepsen.util 9717 :ok :read [29 43] INFO [2019-12-17 12:35:53,440] jepsen worker 87 - jepsen.util 9717 :invoke :cas [29 [1 38]] INFO [2019-12-17 12:35:53,440] jepsen worker 78 - jepsen.util 10068 :ok :read [26 7] INFO [2019-12-17 12:35:53,440] jepsen worker 89 - jepsen.util 10169 :fail :cas [29 [27 46]] :ValueNotSwapFrom INFO [2019-12-17 12:35:53,441] jepsen worker 78 - jepsen.util 10068 :invoke :cas [26 [36 5]] INFO [2019-12-17 12:35:53,441] jepsen worker 89 - jepsen.util 10169 :invoke :write [29 42] INFO [2019-12-17 12:35:53,497] jepsen worker 9 - jepsen.util 10719 :info :write [3 37] :RequestCanceledException INFO [2019-12-17 12:35:53,499] jepsen worker 9 - jepsen.util 10809 :invoke :write [3 41] INFO [2019-12-17 12:35:53,500] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58148,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:35:53,500] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58148,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:35:53,500] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58150,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:35:53,500] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58150,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:35:53,506] jepsen worker 53 - jepsen.util 9593 :fail :cas [17 [27 36]] :ValueNotSwapFrom INFO [2019-12-17 12:35:53,506] jepsen worker 53 - jepsen.util 9593 :invoke :read [17 nil] INFO [2019-12-17 12:35:53,531] jepsen worker 77 - jepsen.util 14387 :info :write [25 4] :RequestCanceledException INFO [2019-12-17 12:35:53,534] jepsen worker 77 - jepsen.util 14477 :invoke :cas [25 [45 43]] INFO [2019-12-17 12:35:53,609] jepsen worker 19 - jepsen.util 10189 :ok :write [6 13] INFO [2019-12-17 12:35:53,609] jepsen worker 19 - jepsen.util 10189 :invoke :write [6 4] INFO [2019-12-17 12:35:53,610] jepsen worker 30 - jepsen.util 10110 :ok :write [10 0] INFO [2019-12-17 12:35:53,610] jepsen worker 45 - jepsen.util 10395 :ok :write [15 1] INFO [2019-12-17 12:35:53,611] jepsen worker 45 - jepsen.util 10395 :invoke :read [15 nil] INFO [2019-12-17 12:35:53,611] jepsen worker 30 - jepsen.util 10110 :invoke :cas [10 [5 44]] INFO [2019-12-17 12:35:53,613] jepsen worker 38 - jepsen.util 10838 :ok :read [12 13] INFO [2019-12-17 12:35:53,613] jepsen worker 38 - jepsen.util 10838 :invoke :read [12 nil] INFO [2019-12-17 12:35:53,617] jepsen worker 51 - jepsen.util 9501 :fail :cas [17 [38 41]] :ValueNotSwapFrom INFO [2019-12-17 12:35:53,618] jepsen worker 51 - jepsen.util 9501 :invoke :read [17 nil] INFO [2019-12-17 12:35:53,632] jepsen worker 87 - jepsen.util 9717 :fail :cas [29 [1 38]] :ValueNotSwapFrom INFO [2019-12-17 12:35:53,632] jepsen worker 34 - jepsen.util 13804 :info :cas [11 [44 18]] :RequestCanceledException INFO [2019-12-17 12:35:53,632] jepsen worker 87 - jepsen.util 9717 :invoke :read [29 nil] INFO [2019-12-17 12:35:53,633] jepsen worker 34 - jepsen.util 13894 :invoke :write [11 31] INFO [2019-12-17 12:35:53,645] jepsen worker 59 - jepsen.util 12929 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:35:53,646] jepsen worker 88 - jepsen.util 10258 :info :write [29 44] :RequestCanceledException INFO [2019-12-17 12:35:53,646] jepsen worker 59 - jepsen.util 12929 :invoke :write [19 34] INFO [2019-12-17 12:35:53,652] jepsen worker 88 - jepsen.util 10348 :invoke :write [29 27] INFO [2019-12-17 12:35:53,666] jepsen worker 78 - jepsen.util 10068 :fail :cas [26 [36 5]] :ValueNotSwapFrom INFO [2019-12-17 12:35:53,667] jepsen worker 78 - jepsen.util 10068 :invoke :read [26 nil] INFO [2019-12-17 12:35:53,678] jepsen worker 11 - jepsen.util 9821 :ok :write [3 47] INFO [2019-12-17 12:35:53,679] jepsen worker 11 - jepsen.util 9821 :invoke :write [3 9] INFO [2019-12-17 12:35:53,693] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58165,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:35:53,694] jepsen worker 21 - jepsen.util 13251 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:35:53,694] jepsen worker 21 - jepsen.util 13251 :invoke :cas [7 [27 13]] INFO [2019-12-17 12:35:53,694] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58165,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:35:53,694] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58167,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:35:53,695] jepsen worker 45 - jepsen.util 10395 :ok :read [15 1] INFO [2019-12-17 12:35:53,695] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58167,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:35:53,695] jepsen worker 45 - jepsen.util 10395 :invoke :cas [15 [28 18]] INFO [2019-12-17 12:35:53,696] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58170,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:35:53,697] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58170,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:35:53,703] jepsen worker 28 - jepsen.util 8308 :ok :write [9 3] INFO [2019-12-17 12:35:53,703] jepsen worker 28 - jepsen.util 8308 :invoke :read [9 nil] INFO [2019-12-17 12:35:53,707] jepsen worker 30 - jepsen.util 10110 :fail :cas [10 [5 44]] :ValueNotSwapFrom INFO [2019-12-17 12:35:53,708] jepsen worker 30 - jepsen.util 10110 :invoke :read [10 nil] INFO [2019-12-17 12:35:53,716] jepsen worker 84 - jepsen.util 13314 :info :write [28 44] :RequestCanceledException INFO [2019-12-17 12:35:53,716] jepsen worker 13 - jepsen.util 9463 :info :write [4 42] :RequestCanceledException INFO [2019-12-17 12:35:53,721] jepsen worker 84 - jepsen.util 13404 :invoke :write [28 23] INFO [2019-12-17 12:35:53,722] jepsen worker 13 - jepsen.util 9553 :invoke :write [4 25] INFO [2019-12-17 12:35:53,731] jepsen worker 86 - jepsen.util 14666 :info :cas [28 [30 19]] :RequestCanceledException INFO [2019-12-17 12:35:53,731] jepsen worker 8 - jepsen.util 13328 :info :cas [2 [10 28]] :RequestCanceledException INFO [2019-12-17 12:35:53,734] jepsen worker 4 - jepsen.util 9364 :ok :read [1 13] INFO [2019-12-17 12:35:53,735] jepsen worker 4 - jepsen.util 9364 :invoke :write [1 44] INFO [2019-12-17 12:35:53,736] jepsen worker 8 - jepsen.util 13418 :invoke :read [2 nil] INFO [2019-12-17 12:35:53,737] jepsen worker 86 - jepsen.util 14756 :invoke :write [28 3] INFO [2019-12-17 12:35:53,746] jepsen worker 50 - jepsen.util 14360 :info :write [16 20] :RequestCanceledException INFO [2019-12-17 12:35:53,752] jepsen worker 50 - jepsen.util 14450 :invoke :write [16 12] INFO [2019-12-17 12:35:53,762] jepsen worker 67 - jepsen.util 13657 :info :cas [22 [12 34]] :RequestCanceledException INFO [2019-12-17 12:35:53,765] jepsen worker 38 - jepsen.util 10838 :ok :read [12 13] INFO [2019-12-17 12:35:53,765] jepsen worker 38 - jepsen.util 10838 :invoke :read [12 nil] INFO [2019-12-17 12:35:53,767] jepsen worker 67 - jepsen.util 13747 :invoke :write [22 3] INFO [2019-12-17 12:35:53,798] jepsen worker 22 - jepsen.util 13342 :info :write [7 25] :RequestCanceledException INFO [2019-12-17 12:35:53,802] jepsen worker 22 - jepsen.util 13432 :invoke :write [7 34] INFO [2019-12-17 12:35:53,810] jepsen worker 16 - jepsen.util 15046 :info :cas [5 [30 17]] :RequestCanceledException INFO [2019-12-17 12:35:53,815] jepsen worker 16 - jepsen.util 15136 :invoke :read [5 nil] INFO [2019-12-17 12:35:53,818] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:35:53,819] jepsen nemesis - couchbase.util Rebalance check count: 455 INFO [2019-12-17 12:35:53,819] jepsen nemesis - couchbase.util Rebalance stuck count: 436 INFO [2019-12-17 12:35:53,821] jepsen worker 45 - jepsen.util 10395 :fail :cas [15 [28 18]] :ValueNotSwapFrom INFO [2019-12-17 12:35:53,822] jepsen worker 45 - jepsen.util 10395 :invoke :write [15 17] INFO [2019-12-17 12:35:53,827] jepsen worker 30 - jepsen.util 10110 :ok :read [10 0] INFO [2019-12-17 12:35:53,827] jepsen worker 30 - jepsen.util 10110 :invoke :cas [10 [15 32]] INFO [2019-12-17 12:35:53,831] jepsen worker 2 - jepsen.util 13502 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:35:53,831] jepsen worker 41 - jepsen.util 14801 :info :cas [13 [39 34]] :RequestCanceledException INFO [2019-12-17 12:35:53,831] jepsen worker 2 - jepsen.util 13502 :invoke :write [0 49] INFO [2019-12-17 12:35:53,835] jepsen worker 41 - jepsen.util 14891 :invoke :read [13 nil] INFO [2019-12-17 12:35:53,846] jepsen worker 43 - jepsen.util 13633 :info :write [14 4] :RequestCanceledException INFO [2019-12-17 12:35:53,850] jepsen worker 43 - jepsen.util 13723 :invoke :read [14 nil] INFO [2019-12-17 12:35:53,888] jepsen worker 55 - jepsen.util 10855 :ok :write [18 41] INFO [2019-12-17 12:35:53,889] jepsen worker 55 - jepsen.util 10855 :invoke :write [18 35] INFO [2019-12-17 12:35:53,898] jepsen worker 15 - jepsen.util 12615 :info :write [5 18] :RequestCanceledException INFO [2019-12-17 12:35:53,899] jepsen worker 30 - jepsen.util 10110 :fail :cas [10 [15 32]] :ValueNotSwapFrom INFO [2019-12-17 12:35:53,899] jepsen worker 30 - jepsen.util 10110 :invoke :cas [10 [16 18]] INFO [2019-12-17 12:35:53,903] jepsen worker 15 - jepsen.util 12705 :invoke :write [5 18] INFO [2019-12-17 12:35:53,911] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58198,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:35:53,912] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58198,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:35:53,912] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58200,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:35:53,912] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58200,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:35:53,932] jepsen worker 3 - jepsen.util 9813 :info :write [1 4] :RequestCanceledException INFO [2019-12-17 12:35:53,935] jepsen worker 3 - jepsen.util 9903 :invoke :read [1 nil] INFO [2019-12-17 12:35:53,962] jepsen worker 74 - jepsen.util 9794 :info :write [24 42] :RequestCanceledException INFO [2019-12-17 12:35:53,964] jepsen worker 74 - jepsen.util 9884 :invoke :read [24 nil] INFO [2019-12-17 12:35:53,973] jepsen worker 54 - jepsen.util 11124 :ok :read [18 41] INFO [2019-12-17 12:35:53,973] jepsen worker 54 - jepsen.util 11124 :invoke :write [18 39] INFO [2019-12-17 12:35:53,977] jepsen worker 30 - jepsen.util 10110 :fail :cas [10 [16 18]] :ValueNotSwapFrom INFO [2019-12-17 12:35:53,977] jepsen worker 30 - jepsen.util 10110 :invoke :read [10 nil] INFO [2019-12-17 12:35:53,990] jepsen worker 19 - jepsen.util 10189 :ok :write [6 4] INFO [2019-12-17 12:35:53,990] jepsen worker 11 - jepsen.util 9821 :ok :write [3 9] INFO [2019-12-17 12:35:53,990] jepsen worker 19 - jepsen.util 10189 :invoke :read [6 nil] INFO [2019-12-17 12:35:53,990] jepsen worker 11 - jepsen.util 9821 :invoke :write [3 21] INFO [2019-12-17 12:35:53,995] jepsen worker 42 - jepsen.util 14622 :info :write [14 2] :RequestCanceledException INFO [2019-12-17 12:35:53,998] jepsen worker 42 - jepsen.util 14712 :invoke :write [14 46] INFO [2019-12-17 12:35:54,032] jepsen worker 17 - jepsen.util 13517 :info :write [5 34] :RequestCanceledException INFO [2019-12-17 12:35:54,043] jepsen worker 17 - jepsen.util 13607 :invoke :cas [5 [1 47]] INFO [2019-12-17 12:35:54,053] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58213,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:35:54,053] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58213,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:35:54,053] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58214,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:35:54,053] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58214,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:35:54,061] jepsen worker 30 - jepsen.util 10110 :ok :read [10 0] INFO [2019-12-17 12:35:54,061] jepsen worker 30 - jepsen.util 10110 :invoke :read [10 nil] INFO [2019-12-17 12:35:54,063] jepsen worker 60 - jepsen.util 14550 :info :write [20 13] :RequestCanceledException INFO [2019-12-17 12:35:54,067] jepsen worker 60 - jepsen.util 14640 :invoke :read [20 nil] INFO [2019-12-17 12:35:54,090] jepsen worker 19 - jepsen.util 10189 :ok :read [6 4] INFO [2019-12-17 12:35:54,091] jepsen worker 19 - jepsen.util 10189 :invoke :write [6 8] INFO [2019-12-17 12:35:54,094] jepsen worker 74 - jepsen.util 9884 :ok :read [24 20] INFO [2019-12-17 12:35:54,094] jepsen worker 74 - jepsen.util 9884 :invoke :read [24 nil] INFO [2019-12-17 12:35:54,097] jepsen worker 57 - jepsen.util 13197 :info :write [19 39] :RequestCanceledException INFO [2019-12-17 12:35:54,102] jepsen worker 57 - jepsen.util 13287 :invoke :cas [19 [47 19]] INFO [2019-12-17 12:35:54,115] jepsen worker 5 - jepsen.util 9545 :info :write [1 24] :RequestCanceledException INFO [2019-12-17 12:35:54,116] jepsen worker 53 - jepsen.util 9593 :ok :read [17 18] INFO [2019-12-17 12:35:54,117] jepsen worker 53 - jepsen.util 9593 :invoke :read [17 nil] INFO [2019-12-17 12:35:54,118] jepsen worker 5 - jepsen.util 9635 :invoke :write [1 37] INFO [2019-12-17 12:35:54,128] jepsen worker 30 - jepsen.util 10110 :ok :read [10 0] INFO [2019-12-17 12:35:54,129] jepsen worker 30 - jepsen.util 10110 :invoke :cas [10 [43 41]] INFO [2019-12-17 12:35:54,145] jepsen worker 79 - jepsen.util 10339 :info :write [26 32] :RequestCanceledException INFO [2019-12-17 12:35:54,147] jepsen worker 79 - jepsen.util 10429 :invoke :write [26 15] INFO [2019-12-17 12:35:54,174] jepsen worker 74 - jepsen.util 9884 :ok :read [24 20] INFO [2019-12-17 12:35:54,174] jepsen worker 74 - jepsen.util 9884 :invoke :write [24 45] INFO [2019-12-17 12:35:54,182] jepsen worker 45 - jepsen.util 10395 :ok :write [15 17] INFO [2019-12-17 12:35:54,182] jepsen worker 45 - jepsen.util 10395 :invoke :read [15 nil] INFO [2019-12-17 12:35:54,216] jepsen worker 6 - jepsen.util 14136 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:35:54,216] jepsen worker 6 - jepsen.util 14136 :invoke :write [2 13] INFO [2019-12-17 12:35:54,219] jepsen worker 37 - jepsen.util 9667 :ok :read [12 13] INFO [2019-12-17 12:35:54,219] jepsen worker 37 - jepsen.util 9667 :invoke :write [12 12] INFO [2019-12-17 12:35:54,232] jepsen worker 30 - jepsen.util 10110 :fail :cas [10 [43 41]] :ValueNotSwapFrom INFO [2019-12-17 12:35:54,232] jepsen worker 30 - jepsen.util 10110 :invoke :cas [10 [46 34]] INFO [2019-12-17 12:35:54,245] jepsen worker 76 - jepsen.util 13846 :info :cas [25 [28 35]] :RequestCanceledException INFO [2019-12-17 12:35:54,251] jepsen worker 76 - jepsen.util 13936 :invoke :write [25 47] INFO [2019-12-17 12:35:54,261] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58235,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:35:54,261] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58235,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:35:54,261] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58240,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:35:54,261] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58240,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:35:54,266] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58229,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:35:54,266] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58229,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:35:54,266] jepsen worker 45 - jepsen.util 10395 :ok :read [15 17] INFO [2019-12-17 12:35:54,266] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58230,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:35:54,267] jepsen worker 45 - jepsen.util 10395 :invoke :write [15 47] INFO [2019-12-17 12:35:54,267] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58230,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:35:54,267] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58231,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:35:54,267] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58231,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:35:54,293] jepsen worker 24 - jepsen.util 12444 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:35:54,293] jepsen worker 24 - jepsen.util 12444 :invoke :read [8 nil] INFO [2019-12-17 12:35:54,297] jepsen worker 30 - jepsen.util 10110 :fail :cas [10 [46 34]] :ValueNotSwapFrom INFO [2019-12-17 12:35:54,298] jepsen worker 30 - jepsen.util 10110 :invoke :write [10 19] INFO [2019-12-17 12:35:54,316] jepsen worker 52 - jepsen.util 9952 :info :write [17 27] :RequestCanceledException INFO [2019-12-17 12:35:54,318] jepsen worker 52 - jepsen.util 10042 :invoke :read [17 nil] INFO [2019-12-17 12:35:54,328] jepsen worker 51 - jepsen.util 9501 :ok :read [17 18] INFO [2019-12-17 12:35:54,328] jepsen worker 51 - jepsen.util 9501 :invoke :write [17 28] INFO [2019-12-17 12:35:54,365] jepsen worker 3 - jepsen.util 9903 :ok :read [1 13] INFO [2019-12-17 12:35:54,365] jepsen worker 3 - jepsen.util 9903 :invoke :cas [1 [40 31]] INFO [2019-12-17 12:35:54,397] jepsen worker 56 - jepsen.util 10586 :info :write [18 39] :RequestCanceledException INFO [2019-12-17 12:35:54,404] jepsen worker 56 - jepsen.util 10676 :invoke :read [18 nil] INFO [2019-12-17 12:35:54,415] jepsen worker 44 - jepsen.util 13994 :info :write [14 7] :RequestCanceledException INFO [2019-12-17 12:35:54,418] jepsen worker 44 - jepsen.util 14084 :invoke :read [14 nil] INFO [2019-12-17 12:35:54,418] jepsen worker 38 - jepsen.util 10838 :ok :read [12 13] INFO [2019-12-17 12:35:54,418] jepsen worker 38 - jepsen.util 10838 :invoke :read [12 nil] INFO [2019-12-17 12:35:54,446] jepsen worker 35 - jepsen.util 13625 :info :cas [11 [35 10]] :RequestCanceledException INFO [2019-12-17 12:35:54,450] jepsen worker 35 - jepsen.util 13715 :invoke :cas [11 [35 3]] INFO [2019-12-17 12:35:54,464] jepsen worker 87 - jepsen.util 9717 :ok :read [29 43] INFO [2019-12-17 12:35:54,464] jepsen worker 87 - jepsen.util 9717 :invoke :cas [29 [12 47]] INFO [2019-12-17 12:35:54,485] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58255,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:35:54,485] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58255,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:35:54,485] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58258,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:35:54,486] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58258,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:35:54,486] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58259,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:35:54,486] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58259,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:35:54,506] jepsen worker 11 - jepsen.util 9821 :ok :write [3 21] INFO [2019-12-17 12:35:54,506] jepsen worker 11 - jepsen.util 9821 :invoke :write [3 47] INFO [2019-12-17 12:35:54,531] jepsen worker 62 - jepsen.util 14102 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:35:54,532] jepsen worker 62 - jepsen.util 14102 :invoke :write [20 46] INFO [2019-12-17 12:35:54,555] jepsen worker 78 - jepsen.util 10068 :ok :read [26 7] INFO [2019-12-17 12:35:54,555] jepsen worker 78 - jepsen.util 10068 :invoke :read [26 nil] INFO [2019-12-17 12:35:54,564] jepsen worker 38 - jepsen.util 10838 :ok :read [12 13] INFO [2019-12-17 12:35:54,565] jepsen worker 38 - jepsen.util 10838 :invoke :write [12 30] INFO [2019-12-17 12:35:54,598] jepsen worker 69 - jepsen.util 10149 :info :write [23 48] :RequestCanceledException INFO [2019-12-17 12:35:54,599] jepsen worker 69 - jepsen.util 10239 :invoke :cas [23 [31 38]] INFO [2019-12-17 12:35:54,616] jepsen worker 85 - jepsen.util 14575 :info :cas [28 [9 47]] :RequestCanceledException INFO [2019-12-17 12:35:54,619] jepsen worker 85 - jepsen.util 14665 :invoke :read [28 nil] INFO [2019-12-17 12:35:54,631] jepsen worker 23 - jepsen.util 13793 :info :write [7 4] :RequestCanceledException INFO [2019-12-17 12:35:54,635] jepsen worker 23 - jepsen.util 13883 :invoke :read [7 nil] INFO [2019-12-17 12:35:54,645] jepsen worker 40 - jepsen.util 14170 :info :write [13 10] :RequestCanceledException INFO [2019-12-17 12:35:54,649] jepsen worker 40 - jepsen.util 14260 :invoke :read [13 nil] INFO [2019-12-17 12:35:54,684] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58272,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:35:54,685] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58272,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:35:54,685] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58274,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:35:54,685] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58274,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:35:54,690] jepsen worker 52 - jepsen.util 10042 :ok :read [17 18] INFO [2019-12-17 12:35:54,690] jepsen worker 52 - jepsen.util 10042 :invoke :cas [17 [29 10]] INFO [2019-12-17 12:35:54,698] jepsen worker 83 - jepsen.util 14033 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:35:54,698] jepsen worker 83 - jepsen.util 14033 :invoke :cas [27 [20 9]] INFO [2019-12-17 12:35:54,731] jepsen worker 49 - jepsen.util 13639 :info :cas [16 [7 7]] :RequestCanceledException INFO [2019-12-17 12:35:54,733] jepsen worker 49 - jepsen.util 13729 :invoke :cas [16 [46 41]] INFO [2019-12-17 12:35:54,762] jepsen worker 66 - jepsen.util 13386 :info :cas [22 [4 31]] :RequestCanceledException INFO [2019-12-17 12:35:54,768] jepsen worker 66 - jepsen.util 13476 :invoke :write [22 17] INFO [2019-12-17 12:35:54,799] jepsen worker 18 - jepsen.util 10278 :info :write [6 41] :RequestCanceledException INFO [2019-12-17 12:35:54,799] jepsen worker 27 - jepsen.util 9657 :info :cas [9 [33 43]] :RequestCanceledException INFO [2019-12-17 12:35:54,801] jepsen worker 18 - jepsen.util 10368 :invoke :cas [6 [35 16]] INFO [2019-12-17 12:35:54,801] jepsen worker 27 - jepsen.util 9747 :invoke :read [9 nil] INFO [2019-12-17 12:35:54,816] jepsen worker 26 - jepsen.util 14426 :info :cas [8 [16 43]] :RequestCanceledException INFO [2019-12-17 12:35:54,818] jepsen worker 26 - jepsen.util 14516 :invoke :read [8 nil] INFO [2019-12-17 12:35:54,823] jepsen worker 19 - jepsen.util 10189 :ok :write [6 8] INFO [2019-12-17 12:35:54,823] jepsen worker 19 - jepsen.util 10189 :invoke :read [6 nil] INFO [2019-12-17 12:35:54,829] jepsen worker 30 - jepsen.util 10110 :ok :write [10 19] INFO [2019-12-17 12:35:54,830] jepsen worker 30 - jepsen.util 10110 :invoke :read [10 nil] INFO [2019-12-17 12:35:54,831] jepsen worker 0 - jepsen.util 14490 :info :cas [0 [30 35]] :RequestCanceledException INFO [2019-12-17 12:35:54,831] jepsen worker 61 - jepsen.util 14461 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:35:54,831] jepsen worker 70 - jepsen.util 10870 :info :write [23 27] :RequestCanceledException INFO [2019-12-17 12:35:54,831] jepsen worker 61 - jepsen.util 14461 :invoke :write [20 20] INFO [2019-12-17 12:35:54,833] jepsen worker 70 - jepsen.util 10960 :invoke :read [23 nil] INFO [2019-12-17 12:35:54,834] jepsen worker 45 - jepsen.util 10395 :ok :write [15 47] INFO [2019-12-17 12:35:54,834] jepsen worker 45 - jepsen.util 10395 :invoke :write [15 6] INFO [2019-12-17 12:35:54,839] jepsen worker 0 - jepsen.util 14580 :invoke :read [0 nil] INFO [2019-12-17 12:35:54,842] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:35:54,842] jepsen nemesis - couchbase.util Rebalance check count: 456 INFO [2019-12-17 12:35:54,843] jepsen nemesis - couchbase.util Rebalance stuck count: 437 INFO [2019-12-17 12:35:54,843] jepsen worker 7 - jepsen.util 13777 :info :cas [2 [44 29]] :RequestCanceledException INFO [2019-12-17 12:35:54,851] jepsen worker 7 - jepsen.util 13867 :invoke :write [2 21] INFO [2019-12-17 12:35:54,879] jepsen worker 11 - jepsen.util 9821 :ok :write [3 47] INFO [2019-12-17 12:35:54,880] jepsen worker 11 - jepsen.util 9821 :invoke :write [3 24] INFO [2019-12-17 12:35:54,896] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58293,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:35:54,896] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58293,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:35:54,896] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58294,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:35:54,897] jepsen worker 1 - jepsen.util 13861 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:35:54,897] jepsen worker 1 - jepsen.util 13861 :invoke :read [0 nil] INFO [2019-12-17 12:35:54,897] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58294,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:35:55,077] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58299,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:35:55,077] jepsen worker 58 - jepsen.util 14278 :info :cas [19 [22 5]] :RequestCanceledException INFO [2019-12-17 12:35:55,077] jepsen worker 68 - jepsen.util 13208 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:35:55,077] jepsen worker 82 - jepsen.util 14122 :info :cas [27 [30 20]] :RequestCanceledException INFO [2019-12-17 12:35:55,077] jepsen worker 75 - jepsen.util 13215 :info :write [25 38] :RequestCanceledException INFO [2019-12-17 12:35:55,077] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58299,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:35:55,077] jepsen worker 71 - jepsen.util 10781 :info :cas [23 [31 47]] :RequestCanceledException INFO [2019-12-17 12:35:55,077] jepsen worker 47 - jepsen.util 10577 :info :write [15 46] :RequestCanceledException INFO [2019-12-17 12:35:55,077] jepsen worker 68 - jepsen.util 13208 :invoke :cas [22 [4 46]] INFO [2019-12-17 12:35:55,077] jepsen worker 25 - jepsen.util 14695 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:35:55,077] jepsen worker 74 - jepsen.util 9884 :ok :write [24 45] INFO [2019-12-17 12:35:55,078] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58303,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:35:55,078] jepsen worker 58 - jepsen.util 14368 :invoke :read [19 nil] INFO [2019-12-17 12:35:55,078] jepsen worker 82 - jepsen.util 14212 :invoke :read [27 nil] INFO [2019-12-17 12:35:55,079] jepsen worker 71 - jepsen.util 10871 :invoke :write [23 33] INFO [2019-12-17 12:35:55,079] jepsen worker 47 - jepsen.util 10667 :invoke :read [15 nil] INFO [2019-12-17 12:35:55,079] jepsen worker 75 - jepsen.util 13305 :invoke :write [25 15] INFO [2019-12-17 12:35:55,079] jepsen worker 56 - jepsen.util 10676 :ok :read [18 41] INFO [2019-12-17 12:35:55,079] jepsen worker 25 - jepsen.util 14695 :invoke :cas [8 [7 24]] INFO [2019-12-17 12:35:55,079] jepsen worker 74 - jepsen.util 9884 :invoke :write [24 3] INFO [2019-12-17 12:35:55,080] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58303,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:35:55,082] jepsen worker 56 - jepsen.util 10676 :invoke :read [18 nil] INFO [2019-12-17 12:35:55,086] jepsen worker 30 - jepsen.util 10110 :ok :read [10 19] INFO [2019-12-17 12:35:55,086] jepsen worker 30 - jepsen.util 10110 :invoke :write [10 42] INFO [2019-12-17 12:35:55,086] jepsen worker 70 - jepsen.util 10960 :ok :read [23 31] INFO [2019-12-17 12:35:55,087] jepsen worker 70 - jepsen.util 10960 :invoke :write [23 34] INFO [2019-12-17 12:35:55,089] jepsen worker 10 - jepsen.util 10000 :info :cas [3 [49 23]] :RequestCanceledException INFO [2019-12-17 12:35:55,089] jepsen worker 64 - jepsen.util 7804 :info :write [21 44] :RequestCanceledException INFO [2019-12-17 12:35:55,090] jepsen worker 27 - jepsen.util 9747 :ok :read [9 3] INFO [2019-12-17 12:35:55,090] jepsen worker 64 - jepsen.util 7894 :invoke :write [21 23] INFO [2019-12-17 12:35:55,090] jepsen worker 27 - jepsen.util 9747 :invoke :write [9 37] INFO [2019-12-17 12:35:55,090] jepsen worker 10 - jepsen.util 10090 :invoke :cas [3 [38 14]] INFO [2019-12-17 12:35:55,095] jepsen worker 18 - jepsen.util 10368 :fail :cas [6 [35 16]] :ValueNotSwapFrom INFO [2019-12-17 12:35:55,095] jepsen worker 18 - jepsen.util 10368 :invoke :write [6 9] INFO [2019-12-17 12:35:55,097] jepsen worker 3 - jepsen.util 9903 :fail :cas [1 [40 31]] :ValueNotSwapFrom INFO [2019-12-17 12:35:55,098] jepsen worker 3 - jepsen.util 9903 :invoke :read [1 nil] INFO [2019-12-17 12:35:55,118] jepsen worker 19 - jepsen.util 10189 :ok :read [6 8] INFO [2019-12-17 12:35:55,118] jepsen worker 19 - jepsen.util 10189 :invoke :read [6 nil] INFO [2019-12-17 12:35:55,132] jepsen worker 31 - jepsen.util 9031 :fail :read [10 nil] :RequestTimeoutException INFO [2019-12-17 12:35:55,132] jepsen worker 32 - jepsen.util 10112 :info :cas [10 [15 10]] :RequestCanceledException INFO [2019-12-17 12:35:55,132] jepsen worker 31 - jepsen.util 9031 :invoke :read [10 nil] INFO [2019-12-17 12:35:55,133] jepsen worker 32 - jepsen.util 10202 :invoke :read [10 nil] INFO [2019-12-17 12:35:55,138] jepsen worker 47 - jepsen.util 10667 :ok :read [15 47] INFO [2019-12-17 12:35:55,138] jepsen worker 47 - jepsen.util 10667 :invoke :cas [15 [37 10]] INFO [2019-12-17 12:35:55,145] jepsen worker 72 - jepsen.util 10062 :info :write [24 7] :RequestCanceledException INFO [2019-12-17 12:35:55,146] jepsen worker 72 - jepsen.util 10152 :invoke :read [24 nil] INFO [2019-12-17 12:35:55,180] jepsen worker 47 - jepsen.util 10667 :fail :cas [15 [37 10]] :ValueNotSwapFrom INFO [2019-12-17 12:35:55,180] jepsen worker 47 - jepsen.util 10667 :invoke :cas [15 [22 43]] INFO [2019-12-17 12:35:55,216] jepsen worker 14 - jepsen.util 9824 :info :write [4 3] :RequestCanceledException INFO [2019-12-17 12:35:55,217] jepsen worker 14 - jepsen.util 9914 :invoke :cas [4 [10 30]] INFO [2019-12-17 12:35:55,230] jepsen worker 20 - jepsen.util 9920 :info :write [6 6] :RequestCanceledException INFO [2019-12-17 12:35:55,230] jepsen worker 29 - jepsen.util 8489 :fail :read [9 nil] :RequestTimeoutException INFO [2019-12-17 12:35:55,231] jepsen worker 29 - jepsen.util 8489 :invoke :read [9 nil] INFO [2019-12-17 12:35:55,231] jepsen worker 47 - jepsen.util 10667 :fail :cas [15 [22 43]] :ValueNotSwapFrom INFO [2019-12-17 12:35:55,231] jepsen worker 47 - jepsen.util 10667 :invoke :cas [15 [4 1]] INFO [2019-12-17 12:35:55,231] jepsen worker 20 - jepsen.util 10010 :invoke :cas [6 [44 3]] INFO [2019-12-17 12:35:55,234] jepsen worker 72 - jepsen.util 10152 :ok :read [24 45] INFO [2019-12-17 12:35:55,234] jepsen worker 72 - jepsen.util 10152 :invoke :read [24 nil] INFO [2019-12-17 12:35:55,245] jepsen worker 5 - jepsen.util 9635 :ok :write [1 37] INFO [2019-12-17 12:35:55,246] jepsen worker 5 - jepsen.util 9635 :invoke :write [1 34] INFO [2019-12-17 12:35:55,246] jepsen worker 13 - jepsen.util 9553 :ok :write [4 25] INFO [2019-12-17 12:35:55,247] jepsen worker 13 - jepsen.util 9553 :invoke :write [4 47] INFO [2019-12-17 12:35:55,267] jepsen worker 72 - jepsen.util 10152 :ok :read [24 45] INFO [2019-12-17 12:35:55,267] jepsen worker 72 - jepsen.util 10152 :invoke :write [24 40] INFO [2019-12-17 12:35:55,268] jepsen worker 47 - jepsen.util 10667 :fail :cas [15 [4 1]] :ValueNotSwapFrom INFO [2019-12-17 12:35:55,268] jepsen worker 47 - jepsen.util 10667 :invoke :write [15 12] INFO [2019-12-17 12:35:55,287] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58327,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:35:55,287] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58334,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:35:55,287] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58327,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:35:55,287] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58334,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:35:55,287] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58328,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:35:55,287] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58338,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:35:55,287] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58328,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:35:55,287] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58338,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:35:55,297] jepsen worker 12 - jepsen.util 10452 :info :write [4 9] :RequestCanceledException INFO [2019-12-17 12:35:55,297] jepsen worker 73 - jepsen.util 9523 :info :write [24 30] :RequestCanceledException INFO [2019-12-17 12:35:55,298] jepsen worker 12 - jepsen.util 10542 :invoke :write [4 31] INFO [2019-12-17 12:35:55,299] jepsen worker 73 - jepsen.util 9613 :invoke :write [24 23] INFO [2019-12-17 12:35:55,331] jepsen worker 81 - jepsen.util 13401 :info :cas [27 [32 43]] :RequestCanceledException INFO [2019-12-17 12:35:55,331] jepsen worker 46 - jepsen.util 9946 :info :write [15 47] :RequestCanceledException INFO [2019-12-17 12:35:55,333] jepsen worker 46 - jepsen.util 10036 :invoke :read [15 nil] INFO [2019-12-17 12:35:55,333] jepsen worker 81 - jepsen.util 13491 :invoke :read [27 nil] INFO [2019-12-17 12:35:55,355] jepsen worker 78 - jepsen.util 10068 :ok :read [26 7] INFO [2019-12-17 12:35:55,355] jepsen worker 78 - jepsen.util 10068 :invoke :write [26 34] INFO [2019-12-17 12:35:55,359] jepsen worker 33 - jepsen.util 13803 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:35:55,360] jepsen worker 33 - jepsen.util 13803 :invoke :write [11 49] INFO [2019-12-17 12:35:55,389] jepsen worker 87 - jepsen.util 9717 :fail :cas [29 [12 47]] :ValueNotSwapFrom INFO [2019-12-17 12:35:55,389] jepsen worker 87 - jepsen.util 9717 :invoke :read [29 nil] INFO [2019-12-17 12:35:55,415] jepsen worker 74 - jepsen.util 9884 :ok :write [24 3] INFO [2019-12-17 12:35:55,415] jepsen worker 45 - jepsen.util 10395 :ok :write [15 6] INFO [2019-12-17 12:35:55,415] jepsen worker 74 - jepsen.util 9884 :invoke :cas [24 [17 37]] INFO [2019-12-17 12:35:55,416] jepsen worker 45 - jepsen.util 10395 :invoke :write [15 33] INFO [2019-12-17 12:35:55,416] jepsen worker 65 - jepsen.util 8615 :fail :read [21 nil] :RequestTimeoutException INFO [2019-12-17 12:35:55,416] jepsen worker 63 - jepsen.util 9153 :fail :read [21 nil] :RequestTimeoutException INFO [2019-12-17 12:35:55,416] jepsen worker 65 - jepsen.util 8615 :invoke :cas [21 [0 33]] INFO [2019-12-17 12:35:55,416] jepsen worker 63 - jepsen.util 9153 :invoke :cas [21 [34 0]] INFO [2019-12-17 12:35:55,431] jepsen worker 48 - jepsen.util 12468 :info :cas [16 [43 2]] :RequestCanceledException INFO [2019-12-17 12:35:55,432] jepsen worker 48 - jepsen.util 12558 :invoke :write [16 7] INFO [2019-12-17 12:35:55,445] jepsen worker 39 - jepsen.util 13269 :info :cas [13 [38 17]] :RequestCanceledException INFO [2019-12-17 12:35:55,446] jepsen worker 39 - jepsen.util 13359 :invoke :write [13 25] INFO [2019-12-17 12:35:55,477] jepsen worker 56 - jepsen.util 10676 :ok :read [18 41] INFO [2019-12-17 12:35:55,477] jepsen worker 32 - jepsen.util 10202 :ok :read [10 19] INFO [2019-12-17 12:35:55,477] jepsen worker 56 - jepsen.util 10676 :invoke :write [18 34] INFO [2019-12-17 12:35:55,477] jepsen worker 32 - jepsen.util 10202 :invoke :cas [10 [30 47]] INFO [2019-12-17 12:35:55,497] jepsen worker 52 - jepsen.util 10042 :fail :cas [17 [29 10]] :ValueNotSwapFrom INFO [2019-12-17 12:35:55,497] jepsen worker 52 - jepsen.util 10042 :invoke :write [17 34] INFO [2019-12-17 12:35:55,502] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58360,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:35:55,503] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58360,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:35:55,503] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58363,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:35:55,503] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58363,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:35:55,515] jepsen worker 89 - jepsen.util 10169 :info :write [29 42] :RequestCanceledException INFO [2019-12-17 12:35:55,516] jepsen worker 89 - jepsen.util 10259 :invoke :read [29 nil] INFO [2019-12-17 12:35:55,520] jepsen worker 46 - jepsen.util 10036 :ok :read [15 6] INFO [2019-12-17 12:35:55,521] jepsen worker 46 - jepsen.util 10036 :invoke :write [15 44] INFO [2019-12-17 12:35:55,523] jepsen worker 10 - jepsen.util 10090 :fail :cas [3 [38 14]] :ValueNotSwapFrom INFO [2019-12-17 12:35:55,523] jepsen worker 10 - jepsen.util 10090 :invoke :cas [3 [36 5]] INFO [2019-12-17 12:35:55,531] jepsen worker 13 - jepsen.util 9553 :ok :write [4 47] INFO [2019-12-17 12:35:55,531] jepsen worker 13 - jepsen.util 9553 :invoke :write [4 17] INFO [2019-12-17 12:35:55,532] jepsen worker 9 - jepsen.util 10809 :info :write [3 41] :RequestCanceledException INFO [2019-12-17 12:35:55,534] jepsen worker 9 - jepsen.util 10899 :invoke :write [3 13] INFO [2019-12-17 12:35:55,549] jepsen worker 69 - jepsen.util 10239 :ok :cas [23 [31 38]] INFO [2019-12-17 12:35:55,550] jepsen worker 69 - jepsen.util 10239 :invoke :write [23 22] INFO [2019-12-17 12:35:55,553] jepsen worker 87 - jepsen.util 9717 :ok :read [29 43] INFO [2019-12-17 12:35:55,553] jepsen worker 87 - jepsen.util 9717 :invoke :cas [29 [3 12]] INFO [2019-12-17 12:35:55,562] jepsen worker 36 - jepsen.util 10836 :info :write [12 19] :RequestCanceledException INFO [2019-12-17 12:35:55,563] jepsen worker 36 - jepsen.util 10926 :invoke :cas [12 [49 28]] INFO [2019-12-17 12:35:55,567] jepsen worker 30 - jepsen.util 10110 :ok :write [10 42] INFO [2019-12-17 12:35:55,568] jepsen worker 30 - jepsen.util 10110 :invoke :write [10 12] INFO [2019-12-17 12:35:55,589] jepsen worker 29 - jepsen.util 8489 :ok :read [9 3] INFO [2019-12-17 12:35:55,589] jepsen worker 29 - jepsen.util 8489 :invoke :write [9 0] INFO [2019-12-17 12:35:55,598] jepsen worker 77 - jepsen.util 14477 :info :cas [25 [45 43]] :RequestCanceledException INFO [2019-12-17 12:35:55,599] jepsen worker 77 - jepsen.util 14567 :invoke :read [25 nil] INFO [2019-12-17 12:35:55,614] jepsen worker 38 - jepsen.util 10838 :ok :write [12 30] INFO [2019-12-17 12:35:55,614] jepsen worker 38 - jepsen.util 10838 :invoke :write [12 30] INFO [2019-12-17 12:35:55,617] jepsen worker 87 - jepsen.util 9717 :fail :cas [29 [3 12]] :ValueNotSwapFrom INFO [2019-12-17 12:35:55,617] jepsen worker 87 - jepsen.util 9717 :invoke :write [29 45] INFO [2019-12-17 12:35:55,640] jepsen worker 54 - jepsen.util 11124 :ok :write [18 39] INFO [2019-12-17 12:35:55,640] jepsen worker 54 - jepsen.util 11124 :invoke :read [18 nil] INFO [2019-12-17 12:35:55,644] jepsen worker 79 - jepsen.util 10429 :ok :write [26 15] INFO [2019-12-17 12:35:55,644] jepsen worker 79 - jepsen.util 10429 :invoke :cas [26 [27 6]] INFO [2019-12-17 12:35:55,662] jepsen worker 80 - jepsen.util 9800 :info :cas [26 [7 6]] :RequestCanceledException INFO [2019-12-17 12:35:55,663] jepsen worker 80 - jepsen.util 9890 :invoke :write [26 14] INFO [2019-12-17 12:35:55,698] jepsen worker 21 - jepsen.util 13251 :info :cas [7 [27 13]] :RequestCanceledException INFO [2019-12-17 12:35:55,699] jepsen worker 21 - jepsen.util 13341 :invoke :write [7 42] INFO [2019-12-17 12:35:55,715] jepsen worker 34 - jepsen.util 13894 :info :write [11 31] :RequestCanceledException INFO [2019-12-17 12:35:55,716] jepsen worker 34 - jepsen.util 13984 :invoke :write [11 2] INFO [2019-12-17 12:35:55,718] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58386,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:35:55,718] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58386,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:35:55,718] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58389,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:35:55,718] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58389,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:35:55,718] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58390,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:35:55,718] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58390,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:35:55,731] jepsen worker 88 - jepsen.util 10348 :info :write [29 27] :RequestCanceledException INFO [2019-12-17 12:35:55,733] jepsen worker 88 - jepsen.util 10438 :invoke :read [29 nil] INFO [2019-12-17 12:35:55,738] jepsen worker 11 - jepsen.util 9821 :ok :write [3 24] INFO [2019-12-17 12:35:55,738] jepsen worker 11 - jepsen.util 9821 :invoke :write [3 6] INFO [2019-12-17 12:35:55,745] jepsen worker 59 - jepsen.util 12929 :info :write [19 34] :RequestCanceledException INFO [2019-12-17 12:35:55,745] jepsen worker 28 - jepsen.util 8308 :fail :read [9 nil] :RequestTimeoutException INFO [2019-12-17 12:35:55,745] jepsen worker 28 - jepsen.util 8308 :invoke :read [9 nil] INFO [2019-12-17 12:35:55,746] jepsen worker 59 - jepsen.util 13019 :invoke :cas [19 [45 18]] INFO [2019-12-17 12:35:55,748] jepsen worker 45 - jepsen.util 10395 :ok :write [15 33] INFO [2019-12-17 12:35:55,748] jepsen worker 45 - jepsen.util 10395 :invoke :read [15 nil] INFO [2019-12-17 12:35:55,751] jepsen worker 13 - jepsen.util 9553 :ok :write [4 17] INFO [2019-12-17 12:35:55,751] jepsen worker 13 - jepsen.util 9553 :invoke :write [4 34] INFO [2019-12-17 12:35:55,767] jepsen worker 65 - jepsen.util 8615 :fail :cas [21 [0 33]] :ValueNotSwapFrom INFO [2019-12-17 12:35:55,767] jepsen worker 65 - jepsen.util 8615 :invoke :read [21 nil] INFO [2019-12-17 12:35:55,779] jepsen worker 88 - jepsen.util 10438 :ok :read [29 43] INFO [2019-12-17 12:35:55,779] jepsen worker 88 - jepsen.util 10438 :invoke :read [29 nil] INFO [2019-12-17 12:35:55,789] jepsen worker 45 - jepsen.util 10395 :ok :read [15 33] INFO [2019-12-17 12:35:55,789] jepsen worker 45 - jepsen.util 10395 :invoke :read [15 nil] INFO [2019-12-17 12:35:55,798] jepsen worker 86 - jepsen.util 14756 :info :write [28 3] :RequestCanceledException INFO [2019-12-17 12:35:55,798] jepsen worker 4 - jepsen.util 9364 :info :write [1 44] :RequestCanceledException INFO [2019-12-17 12:35:55,799] jepsen worker 4 - jepsen.util 9454 :invoke :cas [1 [40 29]] INFO [2019-12-17 12:35:55,799] jepsen worker 86 - jepsen.util 14846 :invoke :cas [28 [41 42]] INFO [2019-12-17 12:35:55,815] jepsen worker 50 - jepsen.util 14450 :info :write [16 12] :RequestCanceledException INFO [2019-12-17 12:35:55,815] jepsen worker 45 - jepsen.util 10395 :ok :read [15 33] INFO [2019-12-17 12:35:55,815] jepsen worker 45 - jepsen.util 10395 :invoke :write [15 20] INFO [2019-12-17 12:35:55,816] jepsen worker 50 - jepsen.util 14540 :invoke :cas [16 [28 32]] INFO [2019-12-17 12:35:55,820] jepsen worker 79 - jepsen.util 10429 :fail :cas [26 [27 6]] :ValueNotSwapFrom INFO [2019-12-17 12:35:55,820] jepsen worker 79 - jepsen.util 10429 :invoke :read [26 nil] INFO [2019-12-17 12:35:55,823] jepsen worker 63 - jepsen.util 9153 :fail :cas [21 [34 0]] :ValueNotSwapFrom INFO [2019-12-17 12:35:55,823] jepsen worker 63 - jepsen.util 9153 :invoke :cas [21 [10 28]] INFO [2019-12-17 12:35:55,831] jepsen worker 8 - jepsen.util 13418 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:35:55,831] jepsen worker 8 - jepsen.util 13418 :invoke :cas [2 [16 47]] INFO [2019-12-17 12:35:55,831] jepsen worker 67 - jepsen.util 13747 :info :write [22 3] :RequestCanceledException INFO [2019-12-17 12:35:55,832] jepsen worker 67 - jepsen.util 13837 :invoke :cas [22 [21 33]] INFO [2019-12-17 12:35:55,846] jepsen worker 22 - jepsen.util 13432 :info :write [7 34] :RequestCanceledException INFO [2019-12-17 12:35:55,846] jepsen worker 84 - jepsen.util 13404 :info :write [28 23] :RequestCanceledException INFO [2019-12-17 12:35:55,847] jepsen worker 22 - jepsen.util 13522 :invoke :cas [7 [47 46]] INFO [2019-12-17 12:35:55,847] jepsen worker 84 - jepsen.util 13494 :invoke :read [28 nil] INFO [2019-12-17 12:35:55,857] jepsen worker 14 - jepsen.util 9914 :fail :cas [4 [10 30]] :ValueNotSwapFrom INFO [2019-12-17 12:35:55,857] jepsen worker 14 - jepsen.util 9914 :invoke :cas [4 [40 10]] INFO [2019-12-17 12:35:55,859] jepsen worker 36 - jepsen.util 10926 :fail :cas [12 [49 28]] :ValueNotSwapFrom INFO [2019-12-17 12:35:55,859] jepsen worker 36 - jepsen.util 10926 :invoke :read [12 nil] INFO [2019-12-17 12:35:55,861] jepsen worker 16 - jepsen.util 15136 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:35:55,861] jepsen worker 16 - jepsen.util 15136 :invoke :cas [5 [22 0]] INFO [2019-12-17 12:35:55,861] jepsen worker 52 - jepsen.util 10042 :ok :write [17 34] INFO [2019-12-17 12:35:55,862] jepsen worker 52 - jepsen.util 10042 :invoke :write [17 33] INFO [2019-12-17 12:35:55,865] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:35:55,866] jepsen nemesis - couchbase.util Rebalance check count: 457 INFO [2019-12-17 12:35:55,866] jepsen nemesis - couchbase.util Rebalance stuck count: 438 INFO [2019-12-17 12:35:55,897] jepsen worker 12 - jepsen.util 10542 :ok :write [4 31] INFO [2019-12-17 12:35:55,897] jepsen worker 12 - jepsen.util 10542 :invoke :read [4 nil] INFO [2019-12-17 12:35:55,897] jepsen worker 43 - jepsen.util 13723 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:35:55,897] jepsen worker 43 - jepsen.util 13723 :invoke :write [14 5] INFO [2019-12-17 12:35:55,931] jepsen worker 55 - jepsen.util 10855 :info :write [18 35] :RequestCanceledException INFO [2019-12-17 12:35:55,931] jepsen worker 41 - jepsen.util 14891 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:35:55,931] jepsen worker 41 - jepsen.util 14891 :invoke :write [13 15] INFO [2019-12-17 12:35:55,931] jepsen worker 2 - jepsen.util 13502 :info :write [0 49] :RequestCanceledException INFO [2019-12-17 12:35:55,932] jepsen worker 55 - jepsen.util 10945 :invoke :read [18 nil] INFO [2019-12-17 12:35:55,932] jepsen worker 2 - jepsen.util 13592 :invoke :read [0 nil] INFO [2019-12-17 12:35:55,933] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58417,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:35:55,933] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58417,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:35:55,933] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58420,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:35:55,933] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58420,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:35:55,933] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58421,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:35:55,934] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58421,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:35:55,938] jepsen worker 3 - jepsen.util 9903 :ok :read [1 37] INFO [2019-12-17 12:35:55,939] jepsen worker 3 - jepsen.util 9903 :invoke :cas [1 [21 42]] INFO [2019-12-17 12:35:55,965] jepsen worker 73 - jepsen.util 9613 :ok :write [24 23] INFO [2019-12-17 12:35:55,965] jepsen worker 73 - jepsen.util 9613 :invoke :write [24 31] INFO [2019-12-17 12:35:55,989] jepsen worker 78 - jepsen.util 10068 :ok :write [26 34] INFO [2019-12-17 12:35:55,989] jepsen worker 78 - jepsen.util 10068 :invoke :write [26 6] INFO [2019-12-17 12:35:56,008] jepsen worker 10 - jepsen.util 10090 :fail :cas [3 [36 5]] :ValueNotSwapFrom INFO [2019-12-17 12:35:56,008] jepsen worker 10 - jepsen.util 10090 :invoke :cas [3 [44 27]] INFO [2019-12-17 12:35:56,016] jepsen worker 15 - jepsen.util 12705 :info :write [5 18] :RequestCanceledException INFO [2019-12-17 12:35:56,017] jepsen worker 15 - jepsen.util 12795 :invoke :cas [5 [19 16]] INFO [2019-12-17 12:35:56,032] jepsen worker 30 - jepsen.util 10110 :ok :write [10 12] INFO [2019-12-17 12:35:56,032] jepsen worker 30 - jepsen.util 10110 :invoke :read [10 nil] INFO [2019-12-17 12:35:56,046] jepsen worker 11 - jepsen.util 9821 :ok :write [3 6] INFO [2019-12-17 12:35:56,046] jepsen worker 11 - jepsen.util 9821 :invoke :read [3 nil] INFO [2019-12-17 12:35:56,056] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58431,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:35:56,056] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58431,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:35:56,056] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58435,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:35:56,056] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58435,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:35:56,057] jepsen worker 45 - jepsen.util 10395 :ok :write [15 20] INFO [2019-12-17 12:35:56,058] jepsen worker 45 - jepsen.util 10395 :invoke :cas [15 [32 16]] INFO [2019-12-17 12:35:56,059] jepsen worker 42 - jepsen.util 14712 :info :write [14 46] :RequestCanceledException INFO [2019-12-17 12:35:56,061] jepsen worker 42 - jepsen.util 14802 :invoke :cas [14 [43 19]] INFO [2019-12-17 12:35:56,062] jepsen worker 11 - jepsen.util 9821 :ok :read [3 6] INFO [2019-12-17 12:35:56,062] jepsen worker 11 - jepsen.util 9821 :invoke :write [3 26] INFO [2019-12-17 12:35:56,082] jepsen worker 45 - jepsen.util 10395 :fail :cas [15 [32 16]] :ValueNotSwapFrom INFO [2019-12-17 12:35:56,082] jepsen worker 45 - jepsen.util 10395 :invoke :cas [15 [35 4]] INFO [2019-12-17 12:35:56,094] jepsen worker 71 - jepsen.util 10871 :ok :write [23 33] INFO [2019-12-17 12:35:56,094] jepsen worker 71 - jepsen.util 10871 :invoke :write [23 45] INFO [2019-12-17 12:35:56,098] jepsen worker 60 - jepsen.util 14640 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:35:56,098] jepsen worker 60 - jepsen.util 14640 :invoke :write [20 6] INFO [2019-12-17 12:35:56,110] jepsen worker 45 - jepsen.util 10395 :fail :cas [15 [35 4]] :ValueNotSwapFrom INFO [2019-12-17 12:35:56,111] jepsen worker 45 - jepsen.util 10395 :invoke :cas [15 [38 14]] INFO [2019-12-17 12:35:56,116] jepsen worker 57 - jepsen.util 13287 :info :cas [19 [47 19]] :RequestCanceledException INFO [2019-12-17 12:35:56,117] jepsen worker 57 - jepsen.util 13377 :invoke :cas [19 [22 27]] INFO [2019-12-17 12:35:56,118] jepsen worker 56 - jepsen.util 10676 :ok :write [18 34] INFO [2019-12-17 12:35:56,118] jepsen worker 56 - jepsen.util 10676 :invoke :read [18 nil] INFO [2019-12-17 12:35:56,130] jepsen worker 17 - jepsen.util 13607 :info :cas [5 [1 47]] :RequestCanceledException INFO [2019-12-17 12:35:56,132] jepsen worker 29 - jepsen.util 8489 :ok :write [9 0] INFO [2019-12-17 12:35:56,132] jepsen worker 29 - jepsen.util 8489 :invoke :write [9 30] INFO [2019-12-17 12:35:56,137] jepsen worker 17 - jepsen.util 13697 :invoke :write [5 43] INFO [2019-12-17 12:35:56,141] jepsen worker 45 - jepsen.util 10395 :fail :cas [15 [38 14]] :ValueNotSwapFrom INFO [2019-12-17 12:35:56,142] jepsen worker 45 - jepsen.util 10395 :invoke :write [15 30] INFO [2019-12-17 12:35:56,148] jepsen worker 30 - jepsen.util 10110 :ok :read [10 12] INFO [2019-12-17 12:35:56,148] jepsen worker 30 - jepsen.util 10110 :invoke :write [10 36] INFO [2019-12-17 12:35:56,162] jepsen worker 53 - jepsen.util 9593 :fail :read [17 nil] :RequestTimeoutException INFO [2019-12-17 12:35:56,162] jepsen worker 53 - jepsen.util 9593 :invoke :write [17 15] INFO [2019-12-17 12:35:56,184] jepsen worker 89 - jepsen.util 10259 :ok :read [29 43] INFO [2019-12-17 12:35:56,184] jepsen worker 89 - jepsen.util 10259 :invoke :write [29 29] INFO [2019-12-17 12:35:56,194] jepsen worker 36 - jepsen.util 10926 :ok :read [12 30] INFO [2019-12-17 12:35:56,194] jepsen worker 36 - jepsen.util 10926 :invoke :write [12 33] INFO [2019-12-17 12:35:56,211] jepsen worker 29 - jepsen.util 8489 :ok :write [9 30] INFO [2019-12-17 12:35:56,211] jepsen worker 29 - jepsen.util 8489 :invoke :cas [9 [31 8]] INFO [2019-12-17 12:35:56,219] jepsen worker 54 - jepsen.util 11124 :ok :read [18 34] INFO [2019-12-17 12:35:56,219] jepsen worker 54 - jepsen.util 11124 :invoke :read [18 nil] INFO [2019-12-17 12:35:56,228] jepsen worker 88 - jepsen.util 10438 :ok :read [29 43] INFO [2019-12-17 12:35:56,228] jepsen worker 88 - jepsen.util 10438 :invoke :write [29 1] INFO [2019-12-17 12:35:56,262] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58457,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:35:56,262] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58464,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:35:56,262] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58457,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:35:56,262] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58459,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:35:56,262] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58464,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:35:56,262] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58459,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:35:56,262] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58469,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:35:56,262] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58469,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:35:56,298] jepsen worker 24 - jepsen.util 12444 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:35:56,298] jepsen worker 24 - jepsen.util 12444 :invoke :write [8 32] INFO [2019-12-17 12:35:56,315] jepsen worker 6 - jepsen.util 14136 :info :write [2 13] :RequestCanceledException INFO [2019-12-17 12:35:56,317] jepsen worker 6 - jepsen.util 14226 :invoke :write [2 49] INFO [2019-12-17 12:35:56,332] jepsen worker 37 - jepsen.util 9667 :info :write [12 12] :RequestCanceledException INFO [2019-12-17 12:35:56,333] jepsen worker 37 - jepsen.util 9757 :invoke :cas [12 [23 34]] INFO [2019-12-17 12:35:56,347] jepsen worker 12 - jepsen.util 10542 :ok :read [4 31] INFO [2019-12-17 12:35:56,347] jepsen worker 12 - jepsen.util 10542 :invoke :write [4 12] INFO [2019-12-17 12:35:56,349] jepsen worker 38 - jepsen.util 10838 :ok :write [12 30] INFO [2019-12-17 12:35:56,350] jepsen worker 38 - jepsen.util 10838 :invoke :write [12 25] INFO [2019-12-17 12:35:56,354] jepsen worker 79 - jepsen.util 10429 :ok :read [26 34] INFO [2019-12-17 12:35:56,354] jepsen worker 79 - jepsen.util 10429 :invoke :write [26 19] INFO [2019-12-17 12:35:56,363] jepsen worker 76 - jepsen.util 13936 :info :write [25 47] :RequestCanceledException INFO [2019-12-17 12:35:56,364] jepsen worker 76 - jepsen.util 14026 :invoke :cas [25 [39 25]] INFO [2019-12-17 12:35:56,387] jepsen worker 73 - jepsen.util 9613 :ok :write [24 31] INFO [2019-12-17 12:35:56,387] jepsen worker 11 - jepsen.util 9821 :ok :write [3 26] INFO [2019-12-17 12:35:56,387] jepsen worker 73 - jepsen.util 9613 :invoke :write [24 27] INFO [2019-12-17 12:35:56,387] jepsen worker 11 - jepsen.util 9821 :invoke :write [3 21] INFO [2019-12-17 12:35:56,387] jepsen worker 30 - jepsen.util 10110 :ok :write [10 36] INFO [2019-12-17 12:35:56,388] jepsen worker 30 - jepsen.util 10110 :invoke :read [10 nil] INFO [2019-12-17 12:35:56,394] jepsen worker 3 - jepsen.util 9903 :fail :cas [1 [21 42]] :ValueNotSwapFrom INFO [2019-12-17 12:35:56,394] jepsen worker 3 - jepsen.util 9903 :invoke :read [1 nil] INFO [2019-12-17 12:35:56,413] jepsen worker 45 - jepsen.util 10395 :ok :write [15 30] INFO [2019-12-17 12:35:56,413] jepsen worker 45 - jepsen.util 10395 :invoke :write [15 47] INFO [2019-12-17 12:35:56,424] jepsen worker 30 - jepsen.util 10110 :ok :read [10 36] INFO [2019-12-17 12:35:56,424] jepsen worker 30 - jepsen.util 10110 :invoke :write [10 29] INFO [2019-12-17 12:35:56,432] jepsen worker 4 - jepsen.util 9454 :fail :cas [1 [40 29]] :ValueNotSwapFrom INFO [2019-12-17 12:35:56,432] jepsen worker 4 - jepsen.util 9454 :invoke :cas [1 [10 22]] INFO [2019-12-17 12:35:56,436] jepsen worker 51 - jepsen.util 9501 :info :write [17 28] :RequestCanceledException INFO [2019-12-17 12:35:56,437] jepsen worker 51 - jepsen.util 9591 :invoke :cas [17 [44 48]] INFO [2019-12-17 12:35:56,481] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58485,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:35:56,481] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58485,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:35:56,481] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58490,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:35:56,481] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58490,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:35:56,502] jepsen worker 55 - jepsen.util 10945 :ok :read [18 34] INFO [2019-12-17 12:35:56,502] jepsen worker 55 - jepsen.util 10945 :invoke :read [18 nil] INFO [2019-12-17 12:35:56,515] jepsen worker 54 - jepsen.util 11124 :ok :read [18 34] INFO [2019-12-17 12:35:56,515] jepsen worker 44 - jepsen.util 14084 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:35:56,516] jepsen worker 54 - jepsen.util 11124 :invoke :read [18 nil] INFO [2019-12-17 12:35:56,516] jepsen worker 44 - jepsen.util 14084 :invoke :cas [14 [27 9]] INFO [2019-12-17 12:35:56,531] jepsen worker 35 - jepsen.util 13715 :info :cas [11 [35 3]] :RequestCanceledException INFO [2019-12-17 12:35:56,532] jepsen worker 35 - jepsen.util 13805 :invoke :cas [11 [14 9]] INFO [2019-12-17 12:35:56,606] jepsen worker 54 - jepsen.util 11124 :ok :read [18 34] INFO [2019-12-17 12:35:56,606] jepsen worker 54 - jepsen.util 11124 :invoke :write [18 39] INFO [2019-12-17 12:35:56,612] jepsen worker 55 - jepsen.util 10945 :ok :read [18 34] INFO [2019-12-17 12:35:56,612] jepsen worker 55 - jepsen.util 10945 :invoke :cas [18 [18 39]] INFO [2019-12-17 12:35:56,631] jepsen worker 62 - jepsen.util 14102 :info :write [20 46] :RequestCanceledException INFO [2019-12-17 12:35:56,632] jepsen worker 62 - jepsen.util 14192 :invoke :cas [20 [37 30]] INFO [2019-12-17 12:35:56,642] jepsen worker 3 - jepsen.util 9903 :ok :read [1 37] INFO [2019-12-17 12:35:56,643] jepsen worker 3 - jepsen.util 9903 :invoke :write [1 47] INFO [2019-12-17 12:35:56,662] jepsen worker 85 - jepsen.util 14665 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:35:56,662] jepsen worker 85 - jepsen.util 14665 :invoke :cas [28 [8 15]] INFO [2019-12-17 12:35:56,667] jepsen worker 4 - jepsen.util 9454 :fail :cas [1 [10 22]] :ValueNotSwapFrom INFO [2019-12-17 12:35:56,667] jepsen worker 4 - jepsen.util 9454 :invoke :read [1 nil] INFO [2019-12-17 12:35:56,698] jepsen worker 40 - jepsen.util 14260 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:35:56,699] jepsen worker 40 - jepsen.util 14260 :invoke :cas [13 [8 34]] INFO [2019-12-17 12:35:56,700] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58502,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:35:56,700] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58502,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:35:56,700] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58505,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:35:56,700] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58505,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:35:56,700] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58506,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:35:56,700] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58506,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:35:56,713] jepsen worker 4 - jepsen.util 9454 :ok :read [1 37] INFO [2019-12-17 12:35:56,713] jepsen worker 4 - jepsen.util 9454 :invoke :read [1 nil] INFO [2019-12-17 12:35:56,728] jepsen worker 4 - jepsen.util 9454 :ok :read [1 37] INFO [2019-12-17 12:35:56,729] jepsen worker 4 - jepsen.util 9454 :invoke :read [1 nil] INFO [2019-12-17 12:35:56,729] jepsen worker 23 - jepsen.util 13883 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:35:56,729] jepsen worker 23 - jepsen.util 13883 :invoke :read [7 nil] INFO [2019-12-17 12:35:56,754] jepsen worker 4 - jepsen.util 9454 :ok :read [1 37] INFO [2019-12-17 12:35:56,754] jepsen worker 4 - jepsen.util 9454 :invoke :cas [1 [2 2]] INFO [2019-12-17 12:35:56,765] jepsen worker 51 - jepsen.util 9591 :fail :cas [17 [44 48]] :ValueNotSwapFrom INFO [2019-12-17 12:35:56,765] jepsen worker 51 - jepsen.util 9591 :invoke :read [17 nil] INFO [2019-12-17 12:35:56,794] jepsen worker 83 - jepsen.util 14033 :info :cas [27 [20 9]] :RequestCanceledException INFO [2019-12-17 12:35:56,796] jepsen worker 83 - jepsen.util 14123 :invoke :read [27 nil] INFO [2019-12-17 12:35:56,816] jepsen worker 49 - jepsen.util 13729 :info :cas [16 [46 41]] :RequestCanceledException INFO [2019-12-17 12:35:56,817] jepsen worker 49 - jepsen.util 13819 :invoke :cas [16 [34 7]] INFO [2019-12-17 12:35:56,831] jepsen worker 66 - jepsen.util 13476 :info :write [22 17] :RequestCanceledException INFO [2019-12-17 12:35:56,831] jepsen worker 26 - jepsen.util 14516 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:35:56,831] jepsen worker 26 - jepsen.util 14516 :invoke :read [8 nil] INFO [2019-12-17 12:35:56,832] jepsen worker 66 - jepsen.util 13566 :invoke :cas [22 [25 18]] INFO [2019-12-17 12:35:56,879] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:35:56,879] jepsen nemesis - couchbase.util Rebalance check count: 458 INFO [2019-12-17 12:35:56,879] jepsen nemesis - couchbase.util Rebalance stuck count: 439 INFO [2019-12-17 12:35:56,916] jepsen worker 0 - jepsen.util 14580 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:35:56,916] jepsen worker 0 - jepsen.util 14580 :invoke :cas [0 [30 41]] INFO [2019-12-17 12:35:56,920] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58519,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:35:56,920] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58519,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:35:56,920] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58523,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:35:56,920] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58523,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:35:56,931] jepsen worker 61 - jepsen.util 14461 :info :write [20 20] :RequestCanceledException INFO [2019-12-17 12:35:56,931] jepsen worker 7 - jepsen.util 13867 :info :write [2 21] :RequestCanceledException INFO [2019-12-17 12:35:56,932] jepsen worker 61 - jepsen.util 14551 :invoke :read [20 nil] INFO [2019-12-17 12:35:56,932] jepsen worker 7 - jepsen.util 13957 :invoke :cas [2 [36 6]] INFO [2019-12-17 12:35:56,965] jepsen worker 56 - jepsen.util 10676 :ok :read [18 34] INFO [2019-12-17 12:35:56,965] jepsen worker 56 - jepsen.util 10676 :invoke :cas [18 [20 0]] INFO [2019-12-17 12:35:56,987] jepsen worker 37 - jepsen.util 9757 :fail :cas [12 [23 34]] :ValueNotSwapFrom INFO [2019-12-17 12:35:56,987] jepsen worker 37 - jepsen.util 9757 :invoke :read [12 nil] INFO [2019-12-17 12:35:56,998] jepsen worker 1 - jepsen.util 13861 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:35:56,998] jepsen worker 1 - jepsen.util 13861 :invoke :cas [0 [4 4]] INFO [2019-12-17 12:35:57,024] jepsen worker 55 - jepsen.util 10945 :fail :cas [18 [18 39]] :ValueNotSwapFrom INFO [2019-12-17 12:35:57,024] jepsen worker 55 - jepsen.util 10945 :invoke :read [18 nil] INFO [2019-12-17 12:35:57,035] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58531,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:35:57,035] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58531,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:35:57,035] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58533,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:35:57,035] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58533,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:35:57,087] jepsen worker 4 - jepsen.util 9454 :fail :cas [1 [2 2]] :ValueNotSwapFrom INFO [2019-12-17 12:35:57,087] jepsen worker 4 - jepsen.util 9454 :invoke :read [1 nil] INFO [2019-12-17 12:35:57,115] jepsen worker 75 - jepsen.util 13305 :info :write [25 15] :RequestCanceledException INFO [2019-12-17 12:35:57,115] jepsen worker 25 - jepsen.util 14695 :info :cas [8 [7 24]] :RequestCanceledException INFO [2019-12-17 12:35:57,116] jepsen worker 75 - jepsen.util 13395 :invoke :write [25 13] INFO [2019-12-17 12:35:57,116] jepsen worker 25 - jepsen.util 14785 :invoke :write [8 39] INFO [2019-12-17 12:35:57,131] jepsen worker 68 - jepsen.util 13208 :info :cas [22 [4 46]] :RequestCanceledException INFO [2019-12-17 12:35:57,131] jepsen worker 64 - jepsen.util 7894 :info :write [21 23] :RequestCanceledException INFO [2019-12-17 12:35:57,132] jepsen worker 64 - jepsen.util 7984 :invoke :read [21 nil] INFO [2019-12-17 12:35:57,132] jepsen worker 68 - jepsen.util 13298 :invoke :cas [22 [23 4]] INFO [2019-12-17 12:35:57,144] jepsen worker 64 - jepsen.util 7984 :ok :read [21 31] INFO [2019-12-17 12:35:57,145] jepsen worker 64 - jepsen.util 7984 :invoke :cas [21 [21 23]] INFO [2019-12-17 12:35:57,146] jepsen worker 58 - jepsen.util 14368 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:35:57,146] jepsen worker 18 - jepsen.util 10368 :info :write [6 9] :RequestCanceledException INFO [2019-12-17 12:35:57,146] jepsen worker 19 - jepsen.util 10189 :fail :read [6 nil] :RequestTimeoutException INFO [2019-12-17 12:35:57,146] jepsen worker 58 - jepsen.util 14368 :invoke :write [19 28] INFO [2019-12-17 12:35:57,146] jepsen worker 19 - jepsen.util 10189 :invoke :read [6 nil] INFO [2019-12-17 12:35:57,147] jepsen worker 18 - jepsen.util 10458 :invoke :read [6 nil] INFO [2019-12-17 12:35:57,154] jepsen worker 11 - jepsen.util 9821 :ok :write [3 21] INFO [2019-12-17 12:35:57,154] jepsen worker 11 - jepsen.util 9821 :invoke :write [3 5] INFO [2019-12-17 12:35:57,154] jepsen worker 45 - jepsen.util 10395 :ok :write [15 47] INFO [2019-12-17 12:35:57,154] jepsen worker 45 - jepsen.util 10395 :invoke :read [15 nil] INFO [2019-12-17 12:35:57,155] jepsen worker 30 - jepsen.util 10110 :ok :write [10 29] INFO [2019-12-17 12:35:57,156] jepsen worker 30 - jepsen.util 10110 :invoke :write [10 37] INFO [2019-12-17 12:35:57,159] jepsen worker 73 - jepsen.util 9613 :ok :write [24 27] INFO [2019-12-17 12:35:57,160] jepsen worker 73 - jepsen.util 9613 :invoke :write [24 31] INFO [2019-12-17 12:35:57,160] jepsen worker 82 - jepsen.util 14212 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:35:57,160] jepsen worker 82 - jepsen.util 14212 :invoke :read [27 nil] INFO [2019-12-17 12:35:57,160] jepsen worker 27 - jepsen.util 9747 :info :write [9 37] :RequestCanceledException INFO [2019-12-17 12:35:57,161] jepsen worker 64 - jepsen.util 7984 :fail :cas [21 [21 23]] :ValueNotSwapFrom INFO [2019-12-17 12:35:57,161] jepsen worker 64 - jepsen.util 7984 :invoke :read [21 nil] INFO [2019-12-17 12:35:57,162] jepsen worker 27 - jepsen.util 9837 :invoke :write [9 12] INFO [2019-12-17 12:35:57,198] jepsen worker 70 - jepsen.util 10960 :info :write [23 34] :RequestCanceledException INFO [2019-12-17 12:35:57,199] jepsen worker 70 - jepsen.util 11050 :invoke :write [23 10] INFO [2019-12-17 12:35:57,213] jepsen worker 4 - jepsen.util 9454 :ok :read [1 37] INFO [2019-12-17 12:35:57,213] jepsen worker 4 - jepsen.util 9454 :invoke :read [1 nil] INFO [2019-12-17 12:35:57,219] jepsen worker 51 - jepsen.util 9591 :ok :read [17 34] INFO [2019-12-17 12:35:57,219] jepsen worker 51 - jepsen.util 9591 :invoke :cas [17 [12 24]] INFO [2019-12-17 12:35:57,232] jepsen worker 31 - jepsen.util 9031 :fail :read [10 nil] :RequestTimeoutException INFO [2019-12-17 12:35:57,232] jepsen worker 31 - jepsen.util 9031 :invoke :cas [10 [24 22]] INFO [2019-12-17 12:35:57,249] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58555,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:35:57,249] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58565,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:35:57,249] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58555,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:35:57,250] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58565,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:35:57,250] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58556,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:35:57,250] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58566,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:35:57,250] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58556,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:35:57,250] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58566,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:35:57,279] jepsen worker 51 - jepsen.util 9591 :fail :cas [17 [12 24]] :ValueNotSwapFrom INFO [2019-12-17 12:35:57,279] jepsen worker 51 - jepsen.util 9591 :invoke :write [17 30] INFO [2019-12-17 12:35:57,285] jepsen worker 55 - jepsen.util 10945 :ok :read [18 34] INFO [2019-12-17 12:35:57,285] jepsen worker 55 - jepsen.util 10945 :invoke :cas [18 [14 8]] INFO [2019-12-17 12:35:57,297] jepsen worker 47 - jepsen.util 10667 :info :write [15 12] :RequestCanceledException INFO [2019-12-17 12:35:57,298] jepsen worker 47 - jepsen.util 10757 :invoke :read [15 nil] INFO [2019-12-17 12:35:57,313] jepsen worker 18 - jepsen.util 10458 :ok :read [6 8] INFO [2019-12-17 12:35:57,313] jepsen worker 18 - jepsen.util 10458 :invoke :cas [6 [4 42]] INFO [2019-12-17 12:35:57,315] jepsen worker 20 - jepsen.util 10010 :info :cas [6 [44 3]] :RequestCanceledException INFO [2019-12-17 12:35:57,316] jepsen worker 20 - jepsen.util 10100 :invoke :read [6 nil] INFO [2019-12-17 12:35:57,322] jepsen worker 45 - jepsen.util 10395 :ok :read [15 47] INFO [2019-12-17 12:35:57,323] jepsen worker 45 - jepsen.util 10395 :invoke :write [15 49] INFO [2019-12-17 12:35:57,331] jepsen worker 72 - jepsen.util 10152 :info :write [24 40] :RequestCanceledException INFO [2019-12-17 12:35:57,331] jepsen worker 5 - jepsen.util 9635 :info :write [1 34] :RequestCanceledException INFO [2019-12-17 12:35:57,332] jepsen worker 72 - jepsen.util 10242 :invoke :cas [24 [10 11]] INFO [2019-12-17 12:35:57,332] jepsen worker 5 - jepsen.util 9725 :invoke :write [1 31] INFO [2019-12-17 12:35:57,335] jepsen worker 64 - jepsen.util 7984 :ok :read [21 31] INFO [2019-12-17 12:35:57,335] jepsen worker 64 - jepsen.util 7984 :invoke :write [21 27] INFO [2019-12-17 12:35:57,338] jepsen worker 19 - jepsen.util 10189 :ok :read [6 8] INFO [2019-12-17 12:35:57,338] jepsen worker 19 - jepsen.util 10189 :invoke :cas [6 [38 30]] INFO [2019-12-17 12:35:57,401] jepsen worker 80 - jepsen.util 9890 :ok :write [26 14] INFO [2019-12-17 12:35:57,401] jepsen worker 80 - jepsen.util 9890 :invoke :write [26 48] INFO [2019-12-17 12:35:57,431] jepsen worker 81 - jepsen.util 13491 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:35:57,431] jepsen worker 81 - jepsen.util 13491 :invoke :write [27 10] INFO [2019-12-17 12:35:57,433] jepsen worker 56 - jepsen.util 10676 :fail :cas [18 [20 0]] :ValueNotSwapFrom INFO [2019-12-17 12:35:57,434] jepsen worker 56 - jepsen.util 10676 :invoke :cas [18 [20 16]] INFO [2019-12-17 12:35:57,444] jepsen worker 47 - jepsen.util 10757 :ok :read [15 47] INFO [2019-12-17 12:35:57,444] jepsen worker 47 - jepsen.util 10757 :invoke :read [15 nil] INFO [2019-12-17 12:35:57,445] jepsen worker 20 - jepsen.util 10100 :ok :read [6 8] INFO [2019-12-17 12:35:57,445] jepsen worker 20 - jepsen.util 10100 :invoke :read [6 nil] INFO [2019-12-17 12:35:57,445] jepsen worker 74 - jepsen.util 9884 :info :cas [24 [17 37]] :RequestCanceledException INFO [2019-12-17 12:35:57,446] jepsen worker 74 - jepsen.util 9974 :invoke :cas [24 [7 33]] INFO [2019-12-17 12:35:57,462] jepsen worker 33 - jepsen.util 13803 :info :write [11 49] :RequestCanceledException INFO [2019-12-17 12:35:57,463] jepsen worker 33 - jepsen.util 13893 :invoke :write [11 37] INFO [2019-12-17 12:35:57,466] jepsen worker 47 - jepsen.util 10757 :ok :read [15 47] INFO [2019-12-17 12:35:57,466] jepsen worker 47 - jepsen.util 10757 :invoke :write [15 35] INFO [2019-12-17 12:35:57,468] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58581,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:35:57,468] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58581,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:35:57,468] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58585,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:35:57,468] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58585,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:35:57,497] jepsen worker 48 - jepsen.util 12558 :info :write [16 7] :RequestCanceledException INFO [2019-12-17 12:35:57,498] jepsen worker 48 - jepsen.util 12648 :invoke :cas [16 [48 1]] INFO [2019-12-17 12:35:57,505] jepsen worker 37 - jepsen.util 9757 :ok :read [12 30] INFO [2019-12-17 12:35:57,506] jepsen worker 37 - jepsen.util 9757 :invoke :cas [12 [18 38]] INFO [2019-12-17 12:35:57,516] jepsen worker 39 - jepsen.util 13359 :info :write [13 25] :RequestCanceledException INFO [2019-12-17 12:35:57,517] jepsen worker 39 - jepsen.util 13449 :invoke :write [13 28] INFO [2019-12-17 12:35:57,542] jepsen worker 64 - jepsen.util 7984 :ok :write [21 27] INFO [2019-12-17 12:35:57,542] jepsen worker 64 - jepsen.util 7984 :invoke :write [21 20] INFO [2019-12-17 12:35:57,563] jepsen worker 46 - jepsen.util 10036 :info :write [15 44] :RequestCanceledException INFO [2019-12-17 12:35:57,563] jepsen worker 32 - jepsen.util 10202 :info :cas [10 [30 47]] :RequestCanceledException INFO [2019-12-17 12:35:57,564] jepsen worker 46 - jepsen.util 10126 :invoke :cas [15 [17 1]] INFO [2019-12-17 12:35:57,564] jepsen worker 32 - jepsen.util 10292 :invoke :read [10 nil] INFO [2019-12-17 12:35:57,609] jepsen worker 64 - jepsen.util 7984 :ok :write [21 20] INFO [2019-12-17 12:35:57,609] jepsen worker 64 - jepsen.util 7984 :invoke :read [21 nil] INFO [2019-12-17 12:35:57,627] jepsen worker 64 - jepsen.util 7984 :ok :read [21 20] INFO [2019-12-17 12:35:57,627] jepsen worker 64 - jepsen.util 7984 :invoke :read [21 nil] INFO [2019-12-17 12:35:57,632] jepsen worker 77 - jepsen.util 14567 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:35:57,632] jepsen worker 77 - jepsen.util 14567 :invoke :cas [25 [18 43]] INFO [2019-12-17 12:35:57,643] jepsen worker 64 - jepsen.util 7984 :ok :read [21 20] INFO [2019-12-17 12:35:57,643] jepsen worker 64 - jepsen.util 7984 :invoke :write [21 16] INFO [2019-12-17 12:35:57,645] jepsen worker 69 - jepsen.util 10239 :info :write [23 22] :RequestCanceledException INFO [2019-12-17 12:35:57,645] jepsen worker 9 - jepsen.util 10899 :info :write [3 13] :RequestCanceledException INFO [2019-12-17 12:35:57,646] jepsen worker 9 - jepsen.util 10989 :invoke :write [3 34] INFO [2019-12-17 12:35:57,646] jepsen worker 69 - jepsen.util 10329 :invoke :cas [23 [0 0]] INFO [2019-12-17 12:35:57,658] jepsen worker 56 - jepsen.util 10676 :fail :cas [18 [20 16]] :ValueNotSwapFrom INFO [2019-12-17 12:35:57,658] jepsen worker 56 - jepsen.util 10676 :invoke :read [18 nil] INFO [2019-12-17 12:35:57,659] jepsen worker 54 - jepsen.util 11124 :ok :write [18 39] INFO [2019-12-17 12:35:57,659] jepsen worker 54 - jepsen.util 11124 :invoke :write [18 37] INFO [2019-12-17 12:35:57,661] jepsen worker 11 - jepsen.util 9821 :ok :write [3 5] INFO [2019-12-17 12:35:57,661] jepsen worker 11 - jepsen.util 9821 :invoke :read [3 nil] INFO [2019-12-17 12:35:57,662] jepsen worker 87 - jepsen.util 9717 :info :write [29 45] :RequestCanceledException INFO [2019-12-17 12:35:57,662] jepsen worker 45 - jepsen.util 10395 :ok :write [15 49] INFO [2019-12-17 12:35:57,662] jepsen worker 45 - jepsen.util 10395 :invoke :write [15 38] INFO [2019-12-17 12:35:57,665] jepsen worker 4 - jepsen.util 9454 :ok :read [1 37] INFO [2019-12-17 12:35:57,666] jepsen worker 4 - jepsen.util 9454 :invoke :read [1 nil] INFO [2019-12-17 12:35:57,671] jepsen worker 87 - jepsen.util 9807 :invoke :cas [29 [48 32]] INFO [2019-12-17 12:35:57,685] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58608,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:35:57,685] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58608,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:35:57,685] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58610,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:35:57,685] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58610,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:35:57,689] jepsen worker 55 - jepsen.util 10945 :fail :cas [18 [14 8]] :ValueNotSwapFrom INFO [2019-12-17 12:35:57,689] jepsen worker 55 - jepsen.util 10945 :invoke :read [18 nil] INFO [2019-12-17 12:35:57,702] jepsen worker 46 - jepsen.util 10126 :fail :cas [15 [17 1]] :ValueNotSwapFrom INFO [2019-12-17 12:35:57,702] jepsen worker 46 - jepsen.util 10126 :invoke :cas [15 [27 1]] INFO [2019-12-17 12:35:57,716] jepsen worker 21 - jepsen.util 13341 :info :write [7 42] :RequestCanceledException INFO [2019-12-17 12:35:57,718] jepsen worker 21 - jepsen.util 13431 :invoke :cas [7 [11 34]] INFO [2019-12-17 12:35:57,732] jepsen worker 37 - jepsen.util 9757 :fail :cas [12 [18 38]] :ValueNotSwapFrom INFO [2019-12-17 12:35:57,732] jepsen worker 37 - jepsen.util 9757 :invoke :write [12 23] INFO [2019-12-17 12:35:57,736] jepsen worker 55 - jepsen.util 10945 :ok :read [18 39] INFO [2019-12-17 12:35:57,736] jepsen worker 55 - jepsen.util 10945 :invoke :cas [18 [11 32]] INFO [2019-12-17 12:35:57,748] jepsen worker 11 - jepsen.util 9821 :ok :read [3 5] INFO [2019-12-17 12:35:57,748] jepsen worker 11 - jepsen.util 9821 :invoke :cas [3 [1 46]] INFO [2019-12-17 12:35:57,759] jepsen worker 59 - jepsen.util 13019 :info :cas [19 [45 18]] :RequestCanceledException INFO [2019-12-17 12:35:57,759] jepsen worker 13 - jepsen.util 9553 :info :write [4 34] :RequestCanceledException INFO [2019-12-17 12:35:57,760] jepsen worker 59 - jepsen.util 13109 :invoke :write [19 24] INFO [2019-12-17 12:35:57,760] jepsen worker 13 - jepsen.util 9643 :invoke :cas [4 [11 32]] INFO [2019-12-17 12:35:57,774] jepsen worker 46 - jepsen.util 10126 :fail :cas [15 [27 1]] :ValueNotSwapFrom INFO [2019-12-17 12:35:57,774] jepsen worker 46 - jepsen.util 10126 :invoke :read [15 nil] INFO [2019-12-17 12:35:57,791] jepsen worker 11 - jepsen.util 9821 :fail :cas [3 [1 46]] :ValueNotSwapFrom INFO [2019-12-17 12:35:57,792] jepsen worker 11 - jepsen.util 9821 :invoke :cas [3 [7 36]] INFO [2019-12-17 12:35:57,801] jepsen worker 69 - jepsen.util 10329 :fail :cas [23 [0 0]] :ValueNotSwapFrom INFO [2019-12-17 12:35:57,802] jepsen worker 69 - jepsen.util 10329 :invoke :cas [23 [13 39]] INFO [2019-12-17 12:35:57,809] jepsen worker 4 - jepsen.util 9454 :ok :read [1 37] INFO [2019-12-17 12:35:57,810] jepsen worker 4 - jepsen.util 9454 :invoke :cas [1 [44 18]] INFO [2019-12-17 12:35:57,814] jepsen worker 46 - jepsen.util 10126 :ok :read [15 49] INFO [2019-12-17 12:35:57,814] jepsen worker 46 - jepsen.util 10126 :invoke :cas [15 [16 27]] INFO [2019-12-17 12:35:57,814] jepsen worker 65 - jepsen.util 8615 :fail :read [21 nil] :RequestTimeoutException INFO [2019-12-17 12:35:57,814] jepsen worker 65 - jepsen.util 8615 :invoke :cas [21 [23 43]] INFO [2019-12-17 12:35:57,827] jepsen worker 87 - jepsen.util 9807 :fail :cas [29 [48 32]] :ValueNotSwapFrom INFO [2019-12-17 12:35:57,827] jepsen worker 87 - jepsen.util 9807 :invoke :read [29 nil] INFO [2019-12-17 12:35:57,831] jepsen worker 34 - jepsen.util 13984 :info :write [11 2] :RequestCanceledException INFO [2019-12-17 12:35:57,832] jepsen worker 34 - jepsen.util 14074 :invoke :cas [11 [38 7]] INFO [2019-12-17 12:35:57,838] jepsen worker 11 - jepsen.util 9821 :fail :cas [3 [7 36]] :ValueNotSwapFrom INFO [2019-12-17 12:35:57,838] jepsen worker 11 - jepsen.util 9821 :invoke :read [3 nil] INFO [2019-12-17 12:35:57,839] jepsen worker 56 - jepsen.util 10676 :ok :read [18 39] INFO [2019-12-17 12:35:57,839] jepsen worker 56 - jepsen.util 10676 :invoke :read [18 nil] INFO [2019-12-17 12:35:57,846] jepsen worker 28 - jepsen.util 8308 :fail :read [9 nil] :RequestTimeoutException INFO [2019-12-17 12:35:57,846] jepsen worker 28 - jepsen.util 8308 :invoke :write [9 29] INFO [2019-12-17 12:35:57,856] jepsen worker 46 - jepsen.util 10126 :fail :cas [15 [16 27]] :ValueNotSwapFrom INFO [2019-12-17 12:35:57,856] jepsen worker 46 - jepsen.util 10126 :invoke :cas [15 [27 40]] INFO [2019-12-17 12:35:57,862] jepsen worker 84 - jepsen.util 13494 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:35:57,862] jepsen worker 84 - jepsen.util 13494 :invoke :write [28 17] INFO [2019-12-17 12:35:57,896] jepsen worker 65 - jepsen.util 8615 :fail :cas [21 [23 43]] :ValueNotSwapFrom INFO [2019-12-17 12:35:57,896] jepsen worker 65 - jepsen.util 8615 :invoke :cas [21 [9 27]] INFO [2019-12-17 12:35:57,897] jepsen worker 14 - jepsen.util 9914 :info :cas [4 [40 10]] :RequestCanceledException INFO [2019-12-17 12:35:57,897] jepsen worker 86 - jepsen.util 14846 :info :cas [28 [41 42]] :RequestCanceledException INFO [2019-12-17 12:35:57,899] jepsen worker 14 - jepsen.util 10004 :invoke :read [4 nil] INFO [2019-12-17 12:35:57,899] jepsen worker 86 - jepsen.util 14936 :invoke :write [28 14] INFO [2019-12-17 12:35:57,899] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:35:57,899] jepsen nemesis - couchbase.util Rebalance check count: 459 INFO [2019-12-17 12:35:57,899] jepsen nemesis - couchbase.util Rebalance stuck count: 440 INFO [2019-12-17 12:35:57,906] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58639,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:35:57,906] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58639,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:35:57,906] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58642,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:35:57,906] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58642,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:35:57,906] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58643,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:35:57,906] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58643,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:35:57,909] jepsen worker 11 - jepsen.util 9821 :ok :read [3 5] INFO [2019-12-17 12:35:57,910] jepsen worker 11 - jepsen.util 9821 :invoke :read [3 nil] INFO [2019-12-17 12:35:57,912] jepsen worker 4 - jepsen.util 9454 :fail :cas [1 [44 18]] :ValueNotSwapFrom INFO [2019-12-17 12:35:57,913] jepsen worker 4 - jepsen.util 9454 :invoke :cas [1 [25 20]] INFO [2019-12-17 12:35:57,915] jepsen worker 50 - jepsen.util 14540 :info :cas [16 [28 32]] :RequestCanceledException INFO [2019-12-17 12:35:57,915] jepsen worker 63 - jepsen.util 9153 :info :cas [21 [10 28]] :RequestCanceledException INFO [2019-12-17 12:35:57,917] jepsen worker 50 - jepsen.util 14630 :invoke :read [16 nil] INFO [2019-12-17 12:35:57,917] jepsen worker 63 - jepsen.util 9243 :invoke :write [21 41] INFO [2019-12-17 12:35:57,923] jepsen worker 64 - jepsen.util 7984 :ok :write [21 16] INFO [2019-12-17 12:35:57,923] jepsen worker 64 - jepsen.util 7984 :invoke :cas [21 [43 47]] INFO [2019-12-17 12:35:57,927] jepsen worker 46 - jepsen.util 10126 :fail :cas [15 [27 40]] :ValueNotSwapFrom INFO [2019-12-17 12:35:57,927] jepsen worker 46 - jepsen.util 10126 :invoke :cas [15 [30 35]] INFO [2019-12-17 12:35:57,931] jepsen worker 67 - jepsen.util 13837 :info :cas [22 [21 33]] :RequestCanceledException INFO [2019-12-17 12:35:57,931] jepsen worker 8 - jepsen.util 13418 :info :cas [2 [16 47]] :RequestCanceledException INFO [2019-12-17 12:35:57,931] jepsen worker 52 - jepsen.util 10042 :info :write [17 33] :RequestCanceledException INFO [2019-12-17 12:35:57,932] jepsen worker 52 - jepsen.util 10132 :invoke :read [17 nil] INFO [2019-12-17 12:35:57,932] jepsen worker 67 - jepsen.util 13927 :invoke :write [22 6] INFO [2019-12-17 12:35:57,933] jepsen worker 8 - jepsen.util 13508 :invoke :write [2 19] INFO [2019-12-17 12:35:57,946] jepsen worker 22 - jepsen.util 13522 :info :cas [7 [47 46]] :RequestCanceledException INFO [2019-12-17 12:35:57,946] jepsen worker 87 - jepsen.util 9807 :ok :read [29 43] INFO [2019-12-17 12:35:57,946] jepsen worker 87 - jepsen.util 9807 :invoke :write [29 40] INFO [2019-12-17 12:35:57,947] jepsen worker 22 - jepsen.util 13612 :invoke :cas [7 [14 47]] INFO [2019-12-17 12:35:57,948] jepsen worker 64 - jepsen.util 7984 :fail :cas [21 [43 47]] :ValueNotSwapFrom INFO [2019-12-17 12:35:57,949] jepsen worker 64 - jepsen.util 7984 :invoke :cas [21 [10 25]] INFO [2019-12-17 12:35:57,963] jepsen worker 16 - jepsen.util 15136 :info :cas [5 [22 0]] :RequestCanceledException INFO [2019-12-17 12:35:57,964] jepsen worker 16 - jepsen.util 15226 :invoke :cas [5 [19 44]] INFO [2019-12-17 12:35:57,992] jepsen worker 65 - jepsen.util 8615 :fail :cas [21 [9 27]] :ValueNotSwapFrom INFO [2019-12-17 12:35:57,992] jepsen worker 65 - jepsen.util 8615 :invoke :write [21 12] INFO [2019-12-17 12:35:57,998] jepsen worker 2 - jepsen.util 13592 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:35:57,998] jepsen worker 43 - jepsen.util 13723 :info :write [14 5] :RequestCanceledException INFO [2019-12-17 12:35:57,998] jepsen worker 2 - jepsen.util 13592 :invoke :read [0 nil] INFO [2019-12-17 12:35:57,999] jepsen worker 43 - jepsen.util 13813 :invoke :cas [14 [5 19]] INFO [2019-12-17 12:35:58,013] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58666,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:35:58,013] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58666,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:35:58,013] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58668,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:35:58,013] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58668,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:35:58,031] jepsen worker 41 - jepsen.util 14891 :info :write [13 15] :RequestCanceledException INFO [2019-12-17 12:35:58,032] jepsen worker 41 - jepsen.util 14981 :invoke :read [13 nil] INFO [2019-12-17 12:35:58,045] jepsen worker 10 - jepsen.util 10090 :info :cas [3 [44 27]] :RequestCanceledException INFO [2019-12-17 12:35:58,046] jepsen worker 10 - jepsen.util 10180 :invoke :cas [3 [25 26]] INFO [2019-12-17 12:35:58,054] jepsen worker 11 - jepsen.util 9821 :ok :read [3 5] INFO [2019-12-17 12:35:58,054] jepsen worker 11 - jepsen.util 9821 :invoke :cas [3 [47 10]] INFO [2019-12-17 12:35:58,056] jepsen worker 64 - jepsen.util 7984 :fail :cas [21 [10 25]] :ValueNotSwapFrom INFO [2019-12-17 12:35:58,056] jepsen worker 64 - jepsen.util 7984 :invoke :cas [21 [30 27]] INFO [2019-12-17 12:35:58,063] jepsen worker 78 - jepsen.util 10068 :info :write [26 6] :RequestCanceledException INFO [2019-12-17 12:35:58,070] jepsen worker 78 - jepsen.util 10158 :invoke :cas [26 [44 6]] INFO [2019-12-17 12:35:58,072] jepsen worker 64 - jepsen.util 7984 :fail :cas [21 [30 27]] :ValueNotSwapFrom INFO [2019-12-17 12:35:58,072] jepsen worker 64 - jepsen.util 7984 :invoke :write [21 13] INFO [2019-12-17 12:35:58,075] jepsen worker 55 - jepsen.util 10945 :fail :cas [18 [11 32]] :ValueNotSwapFrom INFO [2019-12-17 12:35:58,075] jepsen worker 55 - jepsen.util 10945 :invoke :cas [18 [44 16]] INFO [2019-12-17 12:35:58,084] jepsen worker 73 - jepsen.util 9613 :ok :write [24 31] INFO [2019-12-17 12:35:58,084] jepsen worker 73 - jepsen.util 9613 :invoke :cas [24 [22 2]] INFO [2019-12-17 12:35:58,088] jepsen worker 30 - jepsen.util 10110 :ok :write [10 37] INFO [2019-12-17 12:35:58,088] jepsen worker 30 - jepsen.util 10110 :invoke :write [10 45] INFO [2019-12-17 12:35:58,107] jepsen worker 4 - jepsen.util 9454 :fail :cas [1 [25 20]] :ValueNotSwapFrom INFO [2019-12-17 12:35:58,107] jepsen worker 4 - jepsen.util 9454 :invoke :write [1 3] INFO [2019-12-17 12:35:58,116] jepsen worker 15 - jepsen.util 12795 :info :cas [5 [19 16]] :RequestCanceledException INFO [2019-12-17 12:35:58,118] jepsen worker 15 - jepsen.util 12885 :invoke :write [5 43] INFO [2019-12-17 12:35:58,123] jepsen worker 32 - jepsen.util 10292 :ok :read [10 37] INFO [2019-12-17 12:35:58,123] jepsen worker 32 - jepsen.util 10292 :invoke :write [10 11] INFO [2019-12-17 12:35:58,131] jepsen worker 71 - jepsen.util 10871 :info :write [23 45] :RequestCanceledException INFO [2019-12-17 12:35:58,131] jepsen worker 42 - jepsen.util 14802 :info :cas [14 [43 19]] :RequestCanceledException INFO [2019-12-17 12:35:58,132] jepsen worker 71 - jepsen.util 10961 :invoke :write [23 41] INFO [2019-12-17 12:35:58,132] jepsen worker 42 - jepsen.util 14892 :invoke :read [14 nil] INFO [2019-12-17 12:35:58,141] jepsen worker 63 - jepsen.util 9243 :ok :write [21 41] INFO [2019-12-17 12:35:58,141] jepsen worker 63 - jepsen.util 9243 :invoke :read [21 nil] INFO [2019-12-17 12:35:58,145] jepsen worker 57 - jepsen.util 13377 :info :cas [19 [22 27]] :RequestCanceledException INFO [2019-12-17 12:35:58,147] jepsen worker 57 - jepsen.util 13467 :invoke :cas [19 [43 35]] INFO [2019-12-17 12:35:58,158] jepsen worker 45 - jepsen.util 10395 :ok :write [15 38] INFO [2019-12-17 12:35:58,158] jepsen worker 45 - jepsen.util 10395 :invoke :write [15 24] INFO [2019-12-17 12:35:58,161] jepsen worker 11 - jepsen.util 9821 :fail :cas [3 [47 10]] :ValueNotSwapFrom INFO [2019-12-17 12:35:58,161] jepsen worker 11 - jepsen.util 9821 :invoke :read [3 nil] INFO [2019-12-17 12:35:58,163] jepsen worker 17 - jepsen.util 13697 :info :write [5 43] :RequestCanceledException INFO [2019-12-17 12:35:58,164] jepsen worker 17 - jepsen.util 13787 :invoke :read [5 nil] INFO [2019-12-17 12:35:58,167] jepsen worker 63 - jepsen.util 9243 :ok :read [21 41] INFO [2019-12-17 12:35:58,167] jepsen worker 63 - jepsen.util 9243 :invoke :write [21 23] INFO [2019-12-17 12:35:58,197] jepsen worker 11 - jepsen.util 9821 :ok :read [3 5] INFO [2019-12-17 12:35:58,197] jepsen worker 11 - jepsen.util 9821 :invoke :read [3 nil] INFO [2019-12-17 12:35:58,198] jepsen worker 60 - jepsen.util 14640 :info :write [20 6] :RequestCanceledException INFO [2019-12-17 12:35:58,198] jepsen worker 60 - jepsen.util 14730 :invoke :cas [20 [49 18]] INFO [2019-12-17 12:35:58,210] jepsen worker 27 - jepsen.util 9837 :ok :write [9 12] INFO [2019-12-17 12:35:58,210] jepsen worker 27 - jepsen.util 9837 :invoke :read [9 nil] INFO [2019-12-17 12:35:58,217] jepsen worker 56 - jepsen.util 10676 :ok :read [18 39] INFO [2019-12-17 12:35:58,217] jepsen worker 56 - jepsen.util 10676 :invoke :read [18 nil] INFO [2019-12-17 12:35:58,231] jepsen worker 29 - jepsen.util 8489 :info :cas [9 [31 8]] :RequestCanceledException INFO [2019-12-17 12:35:58,231] jepsen worker 89 - jepsen.util 10259 :info :write [29 29] :RequestCanceledException INFO [2019-12-17 12:35:58,232] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58700,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:35:58,232] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58692,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:35:58,232] jepsen worker 29 - jepsen.util 8579 :invoke :write [9 13] INFO [2019-12-17 12:35:58,233] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58700,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:35:58,233] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58692,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:35:58,233] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58703,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:35:58,233] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58693,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:35:58,233] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58703,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:35:58,233] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58704,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:35:58,233] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58693,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:35:58,233] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58704,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:35:58,233] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58694,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:35:58,233] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58694,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:35:58,236] jepsen worker 89 - jepsen.util 10349 :invoke :write [29 27] INFO [2019-12-17 12:35:58,243] jepsen worker 64 - jepsen.util 7984 :ok :write [21 13] INFO [2019-12-17 12:35:58,243] jepsen worker 64 - jepsen.util 7984 :invoke :cas [21 [34 3]] INFO [2019-12-17 12:35:58,245] jepsen worker 10 - jepsen.util 10180 :fail :cas [3 [25 26]] :ValueNotSwapFrom INFO [2019-12-17 12:35:58,245] jepsen worker 10 - jepsen.util 10180 :invoke :read [3 nil] INFO [2019-12-17 12:35:58,245] jepsen worker 88 - jepsen.util 10438 :info :write [29 1] :RequestCanceledException INFO [2019-12-17 12:35:58,246] jepsen worker 88 - jepsen.util 10528 :invoke :cas [29 [37 15]] INFO [2019-12-17 12:35:58,259] jepsen worker 36 - jepsen.util 10926 :info :write [12 33] :RequestCanceledException INFO [2019-12-17 12:35:58,259] jepsen worker 53 - jepsen.util 9593 :info :write [17 15] :RequestCanceledException INFO [2019-12-17 12:35:58,260] jepsen worker 36 - jepsen.util 11016 :invoke :cas [12 [11 1]] INFO [2019-12-17 12:35:58,260] jepsen worker 53 - jepsen.util 9683 :invoke :read [17 nil] INFO [2019-12-17 12:35:58,290] jepsen worker 88 - jepsen.util 10528 :fail :cas [29 [37 15]] :ValueNotSwapFrom INFO [2019-12-17 12:35:58,290] jepsen worker 88 - jepsen.util 10528 :invoke :read [29 nil] INFO [2019-12-17 12:35:58,304] jepsen worker 64 - jepsen.util 7984 :fail :cas [21 [34 3]] :ValueNotSwapFrom INFO [2019-12-17 12:35:58,304] jepsen worker 64 - jepsen.util 7984 :invoke :cas [21 [3 38]] INFO [2019-12-17 12:35:58,304] jepsen worker 73 - jepsen.util 9613 :fail :cas [24 [22 2]] :ValueNotSwapFrom INFO [2019-12-17 12:35:58,304] jepsen worker 73 - jepsen.util 9613 :invoke :read [24 nil] INFO [2019-12-17 12:35:58,331] jepsen worker 6 - jepsen.util 14226 :info :write [2 49] :RequestCanceledException INFO [2019-12-17 12:35:58,332] jepsen worker 6 - jepsen.util 14316 :invoke :read [2 nil] INFO [2019-12-17 12:35:58,385] jepsen worker 64 - jepsen.util 7984 :fail :cas [21 [3 38]] :ValueNotSwapFrom INFO [2019-12-17 12:35:58,386] jepsen worker 64 - jepsen.util 7984 :invoke :cas [21 [15 34]] INFO [2019-12-17 12:35:58,397] jepsen worker 24 - jepsen.util 12444 :info :write [8 32] :RequestCanceledException INFO [2019-12-17 12:35:58,404] jepsen worker 24 - jepsen.util 12534 :invoke :cas [8 [26 33]] INFO [2019-12-17 12:35:58,405] jepsen worker 64 - jepsen.util 7984 :fail :cas [21 [15 34]] :ValueNotSwapFrom INFO [2019-12-17 12:35:58,405] jepsen worker 64 - jepsen.util 7984 :invoke :read [21 nil] INFO [2019-12-17 12:35:58,413] jepsen worker 76 - jepsen.util 14026 :info :cas [25 [39 25]] :RequestCanceledException INFO [2019-12-17 12:35:58,415] jepsen worker 76 - jepsen.util 14116 :invoke :cas [25 [17 14]] INFO [2019-12-17 12:35:58,426] jepsen worker 64 - jepsen.util 7984 :ok :read [21 13] INFO [2019-12-17 12:35:58,426] jepsen worker 64 - jepsen.util 7984 :invoke :write [21 44] INFO [2019-12-17 12:35:58,428] jepsen worker 87 - jepsen.util 9807 :ok :write [29 40] INFO [2019-12-17 12:35:58,428] jepsen worker 87 - jepsen.util 9807 :invoke :cas [29 [13 43]] INFO [2019-12-17 12:35:58,429] jepsen worker 12 - jepsen.util 10542 :info :write [4 12] :RequestCanceledException INFO [2019-12-17 12:35:58,429] jepsen worker 38 - jepsen.util 10838 :info :write [12 25] :RequestCanceledException INFO [2019-12-17 12:35:58,430] jepsen worker 12 - jepsen.util 10632 :invoke :write [4 1] INFO [2019-12-17 12:35:58,430] jepsen worker 38 - jepsen.util 10928 :invoke :write [12 0] INFO [2019-12-17 12:35:58,445] jepsen worker 79 - jepsen.util 10429 :info :write [26 19] :RequestCanceledException INFO [2019-12-17 12:35:58,446] jepsen worker 79 - jepsen.util 10519 :invoke :cas [26 [17 12]] INFO [2019-12-17 12:35:58,447] jepsen worker 63 - jepsen.util 9243 :ok :write [21 23] INFO [2019-12-17 12:35:58,447] jepsen worker 63 - jepsen.util 9243 :invoke :cas [21 [39 10]] INFO [2019-12-17 12:35:58,452] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58723,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:35:58,452] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58723,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:35:58,452] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58727,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:35:58,452] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58727,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:35:58,459] jepsen worker 88 - jepsen.util 10528 :ok :read [29 40] INFO [2019-12-17 12:35:58,459] jepsen worker 88 - jepsen.util 10528 :invoke :write [29 22] INFO [2019-12-17 12:35:58,461] jepsen worker 27 - jepsen.util 9837 :ok :read [9 12] INFO [2019-12-17 12:35:58,462] jepsen worker 27 - jepsen.util 9837 :invoke :cas [9 [7 2]] INFO [2019-12-17 12:35:58,504] jepsen worker 53 - jepsen.util 9683 :ok :read [17 34] INFO [2019-12-17 12:35:58,504] jepsen worker 53 - jepsen.util 9683 :invoke :write [17 14] INFO [2019-12-17 12:35:58,511] jepsen worker 10 - jepsen.util 10180 :ok :read [3 5] INFO [2019-12-17 12:35:58,511] jepsen worker 10 - jepsen.util 10180 :invoke :write [3 4] INFO [2019-12-17 12:35:58,513] jepsen worker 63 - jepsen.util 9243 :fail :cas [21 [39 10]] :ValueNotSwapFrom INFO [2019-12-17 12:35:58,514] jepsen worker 63 - jepsen.util 9243 :invoke :write [21 6] INFO [2019-12-17 12:35:58,522] jepsen worker 29 - jepsen.util 8579 :ok :write [9 13] INFO [2019-12-17 12:35:58,522] jepsen worker 29 - jepsen.util 8579 :invoke :read [9 nil] INFO [2019-12-17 12:35:58,524] jepsen worker 73 - jepsen.util 9613 :ok :read [24 31] INFO [2019-12-17 12:35:58,524] jepsen worker 69 - jepsen.util 10329 :fail :cas [23 [13 39]] :ValueNotSwapFrom INFO [2019-12-17 12:35:58,524] jepsen worker 73 - jepsen.util 9613 :invoke :cas [24 [4 33]] INFO [2019-12-17 12:35:58,524] jepsen worker 69 - jepsen.util 10329 :invoke :read [23 nil] INFO [2019-12-17 12:35:58,546] jepsen worker 35 - jepsen.util 13805 :info :cas [11 [14 9]] :RequestCanceledException INFO [2019-12-17 12:35:58,547] jepsen worker 35 - jepsen.util 13895 :invoke :cas [11 [47 3]] INFO [2019-12-17 12:35:58,565] jepsen worker 27 - jepsen.util 9837 :fail :cas [9 [7 2]] :ValueNotSwapFrom INFO [2019-12-17 12:35:58,565] jepsen worker 27 - jepsen.util 9837 :invoke :read [9 nil] INFO [2019-12-17 12:35:58,608] jepsen worker 44 - jepsen.util 14084 :info :cas [14 [27 9]] :RequestCanceledException INFO [2019-12-17 12:35:58,610] jepsen worker 13 - jepsen.util 9643 :fail :cas [4 [11 32]] :ValueNotSwapFrom INFO [2019-12-17 12:35:58,610] jepsen worker 44 - jepsen.util 14174 :invoke :write [14 20] INFO [2019-12-17 12:35:58,610] jepsen worker 13 - jepsen.util 9643 :invoke :read [4 nil] INFO [2019-12-17 12:35:58,619] jepsen worker 29 - jepsen.util 8579 :ok :read [9 13] INFO [2019-12-17 12:35:58,621] jepsen worker 29 - jepsen.util 8579 :invoke :read [9 nil] INFO [2019-12-17 12:35:58,624] jepsen worker 14 - jepsen.util 10004 :ok :read [4 31] INFO [2019-12-17 12:35:58,624] jepsen worker 14 - jepsen.util 10004 :invoke :write [4 7] INFO [2019-12-17 12:35:58,628] jepsen worker 73 - jepsen.util 9613 :fail :cas [24 [4 33]] :ValueNotSwapFrom INFO [2019-12-17 12:35:58,628] jepsen worker 73 - jepsen.util 9613 :invoke :write [24 26] INFO [2019-12-17 12:35:58,628] jepsen worker 64 - jepsen.util 7984 :ok :write [21 44] INFO [2019-12-17 12:35:58,629] jepsen worker 64 - jepsen.util 7984 :invoke :cas [21 [4 6]] INFO [2019-12-17 12:35:58,639] jepsen worker 79 - jepsen.util 10519 :fail :cas [26 [17 12]] :ValueNotSwapFrom INFO [2019-12-17 12:35:58,639] jepsen worker 79 - jepsen.util 10519 :invoke :cas [26 [10 8]] INFO [2019-12-17 12:35:58,661] jepsen worker 62 - jepsen.util 14192 :info :cas [20 [37 30]] :RequestCanceledException INFO [2019-12-17 12:35:58,662] jepsen worker 62 - jepsen.util 14282 :invoke :write [20 12] INFO [2019-12-17 12:35:58,665] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58751,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:35:58,665] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58751,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:35:58,665] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58754,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:35:58,665] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58754,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:35:58,665] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58755,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:35:58,665] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58755,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:35:58,674] jepsen worker 30 - jepsen.util 10110 :ok :write [10 45] INFO [2019-12-17 12:35:58,674] jepsen worker 30 - jepsen.util 10110 :invoke :read [10 nil] INFO [2019-12-17 12:35:58,674] jepsen worker 64 - jepsen.util 7984 :fail :cas [21 [4 6]] :ValueNotSwapFrom INFO [2019-12-17 12:35:58,674] jepsen worker 64 - jepsen.util 7984 :invoke :read [21 nil] INFO [2019-12-17 12:35:58,699] jepsen worker 78 - jepsen.util 10158 :fail :cas [26 [44 6]] :ValueNotSwapFrom INFO [2019-12-17 12:35:58,700] jepsen worker 78 - jepsen.util 10158 :invoke :read [26 nil] INFO [2019-12-17 12:35:58,707] jepsen worker 29 - jepsen.util 8579 :ok :read [9 13] INFO [2019-12-17 12:35:58,708] jepsen worker 29 - jepsen.util 8579 :invoke :read [9 nil] INFO [2019-12-17 12:35:58,718] jepsen worker 30 - jepsen.util 10110 :ok :read [10 45] INFO [2019-12-17 12:35:58,718] jepsen worker 30 - jepsen.util 10110 :invoke :cas [10 [47 47]] INFO [2019-12-17 12:35:58,731] jepsen worker 3 - jepsen.util 9903 :info :write [1 47] :RequestCanceledException INFO [2019-12-17 12:35:58,732] jepsen worker 3 - jepsen.util 9993 :invoke :write [1 16] INFO [2019-12-17 12:35:58,740] jepsen worker 64 - jepsen.util 7984 :ok :read [21 44] INFO [2019-12-17 12:35:58,740] jepsen worker 64 - jepsen.util 7984 :invoke :read [21 nil] INFO [2019-12-17 12:35:58,756] jepsen worker 85 - jepsen.util 14665 :info :cas [28 [8 15]] :RequestCanceledException INFO [2019-12-17 12:35:58,757] jepsen worker 85 - jepsen.util 14755 :invoke :write [28 20] INFO [2019-12-17 12:35:58,760] jepsen worker 29 - jepsen.util 8579 :ok :read [9 13] INFO [2019-12-17 12:35:58,760] jepsen worker 29 - jepsen.util 8579 :invoke :cas [9 [41 22]] INFO [2019-12-17 12:35:58,769] jepsen worker 30 - jepsen.util 10110 :fail :cas [10 [47 47]] :ValueNotSwapFrom INFO [2019-12-17 12:35:58,769] jepsen worker 30 - jepsen.util 10110 :invoke :cas [10 [26 31]] INFO [2019-12-17 12:35:58,776] jepsen worker 64 - jepsen.util 7984 :ok :read [21 44] INFO [2019-12-17 12:35:58,776] jepsen worker 64 - jepsen.util 7984 :invoke :read [21 nil] INFO [2019-12-17 12:35:58,797] jepsen worker 40 - jepsen.util 14260 :info :cas [13 [8 34]] :RequestCanceledException INFO [2019-12-17 12:35:58,798] jepsen worker 40 - jepsen.util 14350 :invoke :cas [13 [34 39]] INFO [2019-12-17 12:35:58,799] jepsen worker 29 - jepsen.util 8579 :fail :cas [9 [41 22]] :ValueNotSwapFrom INFO [2019-12-17 12:35:58,799] jepsen worker 29 - jepsen.util 8579 :invoke :write [9 44] INFO [2019-12-17 12:35:58,814] jepsen worker 30 - jepsen.util 10110 :fail :cas [10 [26 31]] :ValueNotSwapFrom INFO [2019-12-17 12:35:58,814] jepsen worker 30 - jepsen.util 10110 :invoke :write [10 12] INFO [2019-12-17 12:35:58,831] jepsen worker 23 - jepsen.util 13883 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:35:58,831] jepsen worker 83 - jepsen.util 14123 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:35:58,831] jepsen worker 83 - jepsen.util 14123 :invoke :read [27 nil] INFO [2019-12-17 12:35:58,831] jepsen worker 23 - jepsen.util 13883 :invoke :write [7 40] INFO [2019-12-17 12:35:58,832] jepsen worker 64 - jepsen.util 7984 :ok :read [21 44] INFO [2019-12-17 12:35:58,832] jepsen worker 64 - jepsen.util 7984 :invoke :write [21 33] INFO [2019-12-17 12:35:58,881] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58778,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:35:58,881] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58778,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:35:58,882] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58779,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:35:58,882] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58779,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:35:58,887] jepsen worker 63 - jepsen.util 9243 :ok :write [21 6] INFO [2019-12-17 12:35:58,887] jepsen worker 63 - jepsen.util 9243 :invoke :cas [21 [1 20]] INFO [2019-12-17 12:35:58,889] jepsen worker 49 - jepsen.util 13819 :info :cas [16 [34 7]] :RequestCanceledException INFO [2019-12-17 12:35:58,889] jepsen worker 55 - jepsen.util 10945 :fail :cas [18 [44 16]] :ValueNotSwapFrom INFO [2019-12-17 12:35:58,890] jepsen worker 55 - jepsen.util 10945 :invoke :cas [18 [29 35]] INFO [2019-12-17 12:35:58,890] jepsen worker 49 - jepsen.util 13909 :invoke :cas [16 [20 46]] INFO [2019-12-17 12:35:58,898] jepsen worker 79 - jepsen.util 10519 :fail :cas [26 [10 8]] :ValueNotSwapFrom INFO [2019-12-17 12:35:58,898] jepsen worker 79 - jepsen.util 10519 :invoke :write [26 17] INFO [2019-12-17 12:35:58,913] jepsen worker 66 - jepsen.util 13566 :info :cas [22 [25 18]] :RequestCanceledException INFO [2019-12-17 12:35:58,915] jepsen worker 66 - jepsen.util 13656 :invoke :write [22 31] INFO [2019-12-17 12:35:58,916] jepsen worker 87 - jepsen.util 9807 :fail :cas [29 [13 43]] :ValueNotSwapFrom INFO [2019-12-17 12:35:58,916] jepsen worker 87 - jepsen.util 9807 :invoke :cas [29 [17 26]] INFO [2019-12-17 12:35:58,928] jepsen worker 63 - jepsen.util 9243 :fail :cas [21 [1 20]] :ValueNotSwapFrom INFO [2019-12-17 12:35:58,928] jepsen worker 63 - jepsen.util 9243 :invoke :write [21 21] INFO [2019-12-17 12:35:58,928] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:35:58,929] jepsen nemesis - couchbase.util Rebalance check count: 460 INFO [2019-12-17 12:35:58,929] jepsen nemesis - couchbase.util Rebalance stuck count: 441 INFO [2019-12-17 12:35:58,932] jepsen worker 26 - jepsen.util 14516 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:35:58,932] jepsen worker 26 - jepsen.util 14516 :invoke :cas [8 [37 29]] INFO [2019-12-17 12:35:58,946] jepsen worker 7 - jepsen.util 13957 :info :cas [2 [36 6]] :RequestCanceledException INFO [2019-12-17 12:35:58,947] jepsen worker 7 - jepsen.util 14047 :invoke :cas [2 [3 16]] INFO [2019-12-17 12:35:58,953] jepsen worker 45 - jepsen.util 10395 :ok :write [15 24] INFO [2019-12-17 12:35:58,953] jepsen worker 45 - jepsen.util 10395 :invoke :read [15 nil] INFO [2019-12-17 12:35:59,017] jepsen worker 52 - jepsen.util 10132 :ok :read [17 34] INFO [2019-12-17 12:35:59,017] jepsen worker 52 - jepsen.util 10132 :invoke :write [17 11] INFO [2019-12-17 12:35:59,017] jepsen worker 0 - jepsen.util 14580 :info :cas [0 [30 41]] :RequestCanceledException INFO [2019-12-17 12:35:59,019] jepsen worker 0 - jepsen.util 14670 :invoke :cas [0 [28 18]] INFO [2019-12-17 12:35:59,026] jepsen worker 45 - jepsen.util 10395 :ok :read [15 24] INFO [2019-12-17 12:35:59,026] jepsen worker 45 - jepsen.util 10395 :invoke :write [15 30] INFO [2019-12-17 12:35:59,032] jepsen worker 61 - jepsen.util 14551 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:35:59,032] jepsen worker 61 - jepsen.util 14551 :invoke :write [20 20] INFO [2019-12-17 12:35:59,053] jepsen worker 10 - jepsen.util 10180 :ok :write [3 4] INFO [2019-12-17 12:35:59,053] jepsen worker 10 - jepsen.util 10180 :invoke :cas [3 [20 19]] INFO [2019-12-17 12:35:59,064] jepsen worker 64 - jepsen.util 7984 :ok :write [21 33] INFO [2019-12-17 12:35:59,064] jepsen worker 64 - jepsen.util 7984 :invoke :write [21 20] INFO [2019-12-17 12:35:59,080] jepsen worker 36 - jepsen.util 11016 :fail :cas [12 [11 1]] :ValueNotSwapFrom INFO [2019-12-17 12:35:59,080] jepsen worker 36 - jepsen.util 11016 :invoke :cas [12 [35 35]] INFO [2019-12-17 12:35:59,083] jepsen worker 29 - jepsen.util 8579 :ok :write [9 44] INFO [2019-12-17 12:35:59,083] jepsen worker 29 - jepsen.util 8579 :invoke :cas [9 [42 5]] INFO [2019-12-17 12:35:59,097] jepsen worker 1 - jepsen.util 13861 :info :cas [0 [4 4]] :RequestCanceledException INFO [2019-12-17 12:35:59,101] jepsen worker 1 - jepsen.util 13951 :invoke :read [0 nil] INFO [2019-12-17 12:35:59,101] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58790,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:35:59,101] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58790,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:35:59,102] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58791,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:35:59,102] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58791,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:35:59,102] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58793,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:35:59,103] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58793,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:35:59,107] jepsen worker 29 - jepsen.util 8579 :fail :cas [9 [42 5]] :ValueNotSwapFrom INFO [2019-12-17 12:35:59,108] jepsen worker 29 - jepsen.util 8579 :invoke :cas [9 [8 34]] INFO [2019-12-17 12:35:59,112] jepsen worker 55 - jepsen.util 10945 :fail :cas [18 [29 35]] :ValueNotSwapFrom INFO [2019-12-17 12:35:59,112] jepsen worker 55 - jepsen.util 10945 :invoke :read [18 nil] INFO [2019-12-17 12:35:59,132] jepsen worker 25 - jepsen.util 14785 :info :write [8 39] :RequestCanceledException INFO [2019-12-17 12:35:59,133] jepsen worker 25 - jepsen.util 14875 :invoke :write [8 45] INFO [2019-12-17 12:35:59,133] jepsen worker 29 - jepsen.util 8579 :fail :cas [9 [8 34]] :ValueNotSwapFrom INFO [2019-12-17 12:35:59,133] jepsen worker 29 - jepsen.util 8579 :invoke :cas [9 [2 10]] INFO [2019-12-17 12:35:59,148] jepsen worker 87 - jepsen.util 9807 :fail :cas [29 [17 26]] :ValueNotSwapFrom INFO [2019-12-17 12:35:59,148] jepsen worker 87 - jepsen.util 9807 :invoke :read [29 nil] INFO [2019-12-17 12:35:59,158] jepsen worker 29 - jepsen.util 8579 :fail :cas [9 [2 10]] :ValueNotSwapFrom INFO [2019-12-17 12:35:59,159] jepsen worker 29 - jepsen.util 8579 :invoke :cas [9 [7 25]] INFO [2019-12-17 12:35:59,159] jepsen worker 75 - jepsen.util 13395 :info :write [25 13] :RequestCanceledException INFO [2019-12-17 12:35:59,160] jepsen worker 75 - jepsen.util 13485 :invoke :read [25 nil] INFO [2019-12-17 12:35:59,165] jepsen worker 64 - jepsen.util 7984 :ok :write [21 20] INFO [2019-12-17 12:35:59,165] jepsen worker 64 - jepsen.util 7984 :invoke :read [21 nil] INFO [2019-12-17 12:35:59,175] jepsen worker 29 - jepsen.util 8579 :fail :cas [9 [7 25]] :ValueNotSwapFrom INFO [2019-12-17 12:35:59,175] jepsen worker 29 - jepsen.util 8579 :invoke :read [9 nil] INFO [2019-12-17 12:35:59,183] jepsen worker 64 - jepsen.util 7984 :ok :read [21 20] INFO [2019-12-17 12:35:59,184] jepsen worker 64 - jepsen.util 7984 :invoke :write [21 18] INFO [2019-12-17 12:35:59,203] jepsen worker 29 - jepsen.util 8579 :ok :read [9 44] INFO [2019-12-17 12:35:59,203] jepsen worker 29 - jepsen.util 8579 :invoke :write [9 0] INFO [2019-12-17 12:35:59,211] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58816,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:35:59,211] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58816,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:35:59,216] jepsen worker 68 - jepsen.util 13298 :info :cas [22 [23 4]] :RequestCanceledException INFO [2019-12-17 12:35:59,217] jepsen worker 68 - jepsen.util 13388 :invoke :write [22 11] INFO [2019-12-17 12:35:59,246] jepsen worker 58 - jepsen.util 14368 :info :write [19 28] :RequestCanceledException INFO [2019-12-17 12:35:59,247] jepsen worker 58 - jepsen.util 14458 :invoke :read [19 nil] INFO [2019-12-17 12:35:59,250] jepsen worker 30 - jepsen.util 10110 :ok :write [10 12] INFO [2019-12-17 12:35:59,250] jepsen worker 30 - jepsen.util 10110 :invoke :write [10 7] INFO [2019-12-17 12:35:59,252] jepsen worker 73 - jepsen.util 9613 :ok :write [24 26] INFO [2019-12-17 12:35:59,252] jepsen worker 73 - jepsen.util 9613 :invoke :cas [24 [28 20]] INFO [2019-12-17 12:35:59,259] jepsen worker 82 - jepsen.util 14212 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:35:59,259] jepsen worker 70 - jepsen.util 11050 :info :write [23 10] :RequestCanceledException INFO [2019-12-17 12:35:59,259] jepsen worker 82 - jepsen.util 14212 :invoke :read [27 nil] INFO [2019-12-17 12:35:59,261] jepsen worker 70 - jepsen.util 11140 :invoke :read [23 nil] INFO [2019-12-17 12:35:59,273] jepsen worker 87 - jepsen.util 9807 :ok :read [29 40] INFO [2019-12-17 12:35:59,273] jepsen worker 87 - jepsen.util 9807 :invoke :cas [29 [9 23]] INFO [2019-12-17 12:35:59,280] jepsen worker 10 - jepsen.util 10180 :fail :cas [3 [20 19]] :ValueNotSwapFrom INFO [2019-12-17 12:35:59,280] jepsen worker 10 - jepsen.util 10180 :invoke :write [3 34] INFO [2019-12-17 12:35:59,288] jepsen worker 73 - jepsen.util 9613 :fail :cas [24 [28 20]] :ValueNotSwapFrom INFO [2019-12-17 12:35:59,289] jepsen worker 73 - jepsen.util 9613 :invoke :write [24 11] INFO [2019-12-17 12:35:59,317] jepsen worker 55 - jepsen.util 10945 :ok :read [18 39] INFO [2019-12-17 12:35:59,317] jepsen worker 64 - jepsen.util 7984 :ok :write [21 18] INFO [2019-12-17 12:35:59,317] jepsen worker 64 - jepsen.util 7984 :invoke :read [21 nil] INFO [2019-12-17 12:35:59,317] jepsen worker 55 - jepsen.util 10945 :invoke :read [18 nil] INFO [2019-12-17 12:35:59,320] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58825,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:35:59,320] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58825,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:35:59,329] jepsen worker 31 - jepsen.util 9031 :info :cas [10 [24 22]] :RequestCanceledException INFO [2019-12-17 12:35:59,329] jepsen worker 51 - jepsen.util 9591 :info :write [17 30] :RequestCanceledException INFO [2019-12-17 12:35:59,329] jepsen worker 18 - jepsen.util 10458 :info :cas [6 [4 42]] :RequestCanceledException INFO [2019-12-17 12:35:59,330] jepsen worker 31 - jepsen.util 9121 :invoke :write [10 24] INFO [2019-12-17 12:35:59,331] jepsen worker 18 - jepsen.util 10548 :invoke :write [6 41] INFO [2019-12-17 12:35:59,331] jepsen worker 51 - jepsen.util 9681 :invoke :read [17 nil] INFO [2019-12-17 12:35:59,343] jepsen worker 64 - jepsen.util 7984 :ok :read [21 18] INFO [2019-12-17 12:35:59,343] jepsen worker 64 - jepsen.util 7984 :invoke :read [21 nil] INFO [2019-12-17 12:35:59,353] jepsen worker 69 - jepsen.util 10329 :ok :read [23 33] INFO [2019-12-17 12:35:59,353] jepsen worker 69 - jepsen.util 10329 :invoke :write [23 47] INFO [2019-12-17 12:35:59,367] jepsen worker 29 - jepsen.util 8579 :ok :write [9 0] INFO [2019-12-17 12:35:59,367] jepsen worker 29 - jepsen.util 8579 :invoke :write [9 10] INFO [2019-12-17 12:35:59,370] jepsen worker 64 - jepsen.util 7984 :ok :read [21 18] INFO [2019-12-17 12:35:59,370] jepsen worker 64 - jepsen.util 7984 :invoke :read [21 nil] INFO [2019-12-17 12:35:59,392] jepsen worker 56 - jepsen.util 10676 :ok :read [18 39] INFO [2019-12-17 12:35:59,393] jepsen worker 56 - jepsen.util 10676 :invoke :read [18 nil] INFO [2019-12-17 12:35:59,403] jepsen worker 64 - jepsen.util 7984 :ok :read [21 18] INFO [2019-12-17 12:35:59,403] jepsen worker 64 - jepsen.util 7984 :invoke :write [21 11] INFO [2019-12-17 12:35:59,409] jepsen worker 78 - jepsen.util 10158 :ok :read [26 14] INFO [2019-12-17 12:35:59,410] jepsen worker 78 - jepsen.util 10158 :invoke :read [26 nil] INFO [2019-12-17 12:35:59,413] jepsen worker 72 - jepsen.util 10242 :info :cas [24 [10 11]] :RequestCanceledException INFO [2019-12-17 12:35:59,414] jepsen worker 72 - jepsen.util 10332 :invoke :write [24 19] INFO [2019-12-17 12:35:59,432] jepsen worker 5 - jepsen.util 9725 :info :write [1 31] :RequestCanceledException INFO [2019-12-17 12:35:59,433] jepsen worker 5 - jepsen.util 9815 :invoke :write [1 10] INFO [2019-12-17 12:35:59,445] jepsen worker 19 - jepsen.util 10189 :info :cas [6 [38 30]] :RequestCanceledException INFO [2019-12-17 12:35:59,448] jepsen worker 19 - jepsen.util 10279 :invoke :cas [6 [5 27]] INFO [2019-12-17 12:35:59,473] jepsen worker 19 - jepsen.util 10279 :fail :cas [6 [5 27]] :ValueNotSwapFrom INFO [2019-12-17 12:35:59,473] jepsen worker 19 - jepsen.util 10279 :invoke :cas [6 [9 44]] INFO [2019-12-17 12:35:59,498] jepsen worker 20 - jepsen.util 10100 :fail :read [6 nil] :RequestTimeoutException INFO [2019-12-17 12:35:59,498] jepsen worker 80 - jepsen.util 9890 :info :write [26 48] :RequestCanceledException INFO [2019-12-17 12:35:59,498] jepsen worker 20 - jepsen.util 10100 :invoke :read [6 nil] INFO [2019-12-17 12:35:59,499] jepsen worker 80 - jepsen.util 9980 :invoke :write [26 15] INFO [2019-12-17 12:35:59,503] jepsen worker 19 - jepsen.util 10279 :fail :cas [6 [9 44]] :ValueNotSwapFrom INFO [2019-12-17 12:35:59,503] jepsen worker 19 - jepsen.util 10279 :invoke :cas [6 [3 14]] INFO [2019-12-17 12:35:59,513] jepsen worker 29 - jepsen.util 8579 :ok :write [9 10] INFO [2019-12-17 12:35:59,513] jepsen worker 29 - jepsen.util 8579 :invoke :cas [9 [15 49]] INFO [2019-12-17 12:35:59,515] jepsen worker 64 - jepsen.util 7984 :ok :write [21 11] INFO [2019-12-17 12:35:59,515] jepsen worker 64 - jepsen.util 7984 :invoke :read [21 nil] INFO [2019-12-17 12:35:59,528] jepsen worker 78 - jepsen.util 10158 :ok :read [26 14] INFO [2019-12-17 12:35:59,528] jepsen worker 78 - jepsen.util 10158 :invoke :cas [26 [39 44]] INFO [2019-12-17 12:35:59,530] jepsen worker 19 - jepsen.util 10279 :fail :cas [6 [3 14]] :ValueNotSwapFrom INFO [2019-12-17 12:35:59,531] jepsen worker 19 - jepsen.util 10279 :invoke :write [6 36] INFO [2019-12-17 12:35:59,531] jepsen worker 47 - jepsen.util 10757 :info :write [15 35] :RequestCanceledException INFO [2019-12-17 12:35:59,531] jepsen worker 81 - jepsen.util 13491 :info :write [27 10] :RequestCanceledException INFO [2019-12-17 12:35:59,531] jepsen worker 48 - jepsen.util 12648 :info :cas [16 [48 1]] :RequestCanceledException INFO [2019-12-17 12:35:59,531] jepsen worker 13 - jepsen.util 9643 :ok :read [4 31] INFO [2019-12-17 12:35:59,532] jepsen worker 13 - jepsen.util 9643 :invoke :cas [4 [18 12]] INFO [2019-12-17 12:35:59,532] jepsen worker 47 - jepsen.util 10847 :invoke :read [15 nil] INFO [2019-12-17 12:35:59,533] jepsen worker 48 - jepsen.util 12738 :invoke :cas [16 [42 43]] INFO [2019-12-17 12:35:59,533] jepsen worker 81 - jepsen.util 13581 :invoke :cas [27 [32 12]] INFO [2019-12-17 12:35:59,534] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58851,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:35:59,534] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58851,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:35:59,534] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58854,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:35:59,534] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58854,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:35:59,534] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58855,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:35:59,534] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58855,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:35:59,545] jepsen worker 74 - jepsen.util 9974 :info :cas [24 [7 33]] :RequestCanceledException INFO [2019-12-17 12:35:59,546] jepsen worker 74 - jepsen.util 10064 :invoke :cas [24 [27 13]] INFO [2019-12-17 12:35:59,563] jepsen worker 33 - jepsen.util 13893 :info :write [11 37] :RequestCanceledException INFO [2019-12-17 12:35:59,564] jepsen worker 33 - jepsen.util 13983 :invoke :cas [11 [8 37]] INFO [2019-12-17 12:35:59,566] jepsen worker 20 - jepsen.util 10100 :ok :read [6 8] INFO [2019-12-17 12:35:59,566] jepsen worker 20 - jepsen.util 10100 :invoke :write [6 3] INFO [2019-12-17 12:35:59,582] jepsen worker 36 - jepsen.util 11016 :fail :cas [12 [35 35]] :ValueNotSwapFrom INFO [2019-12-17 12:35:59,582] jepsen worker 36 - jepsen.util 11016 :invoke :read [12 nil] INFO [2019-12-17 12:35:59,589] jepsen worker 29 - jepsen.util 8579 :fail :cas [9 [15 49]] :ValueNotSwapFrom INFO [2019-12-17 12:35:59,589] jepsen worker 29 - jepsen.util 8579 :invoke :cas [9 [41 23]] INFO [2019-12-17 12:35:59,594] jepsen worker 55 - jepsen.util 10945 :ok :read [18 39] INFO [2019-12-17 12:35:59,594] jepsen worker 55 - jepsen.util 10945 :invoke :write [18 20] INFO [2019-12-17 12:35:59,598] jepsen worker 39 - jepsen.util 13449 :info :write [13 28] :RequestCanceledException INFO [2019-12-17 12:35:59,599] jepsen worker 39 - jepsen.util 13539 :invoke :cas [13 [5 47]] INFO [2019-12-17 12:35:59,610] jepsen worker 64 - jepsen.util 7984 :ok :read [21 11] INFO [2019-12-17 12:35:59,612] jepsen worker 64 - jepsen.util 7984 :invoke :read [21 nil] INFO [2019-12-17 12:35:59,624] jepsen worker 30 - jepsen.util 10110 :ok :write [10 7] INFO [2019-12-17 12:35:59,624] jepsen worker 30 - jepsen.util 10110 :invoke :cas [10 [21 47]] INFO [2019-12-17 12:35:59,628] jepsen worker 45 - jepsen.util 10395 :ok :write [15 30] INFO [2019-12-17 12:35:59,628] jepsen worker 45 - jepsen.util 10395 :invoke :read [15 nil] INFO [2019-12-17 12:35:59,636] jepsen worker 70 - jepsen.util 11140 :ok :read [23 33] INFO [2019-12-17 12:35:59,636] jepsen worker 70 - jepsen.util 11140 :invoke :read [23 nil] INFO [2019-12-17 12:35:59,640] jepsen worker 10 - jepsen.util 10180 :ok :write [3 34] INFO [2019-12-17 12:35:59,640] jepsen worker 10 - jepsen.util 10180 :invoke :write [3 48] INFO [2019-12-17 12:35:59,650] jepsen worker 74 - jepsen.util 10064 :fail :cas [24 [27 13]] :ValueNotSwapFrom INFO [2019-12-17 12:35:59,650] jepsen worker 29 - jepsen.util 8579 :fail :cas [9 [41 23]] :ValueNotSwapFrom INFO [2019-12-17 12:35:59,650] jepsen worker 74 - jepsen.util 10064 :invoke :write [24 16] INFO [2019-12-17 12:35:59,650] jepsen worker 29 - jepsen.util 8579 :invoke :write [9 11] INFO [2019-12-17 12:35:59,681] jepsen worker 87 - jepsen.util 9807 :fail :cas [29 [9 23]] :ValueNotSwapFrom INFO [2019-12-17 12:35:59,681] jepsen worker 87 - jepsen.util 9807 :invoke :read [29 nil] INFO [2019-12-17 12:35:59,726] jepsen worker 45 - jepsen.util 10395 :ok :read [15 30] INFO [2019-12-17 12:35:59,726] jepsen worker 45 - jepsen.util 10395 :invoke :cas [15 [20 17]] INFO [2019-12-17 12:35:59,728] jepsen worker 80 - jepsen.util 9980 :ok :write [26 15] INFO [2019-12-17 12:35:59,728] jepsen worker 80 - jepsen.util 9980 :invoke :write [26 4] INFO [2019-12-17 12:35:59,731] jepsen worker 54 - jepsen.util 11124 :info :write [18 37] :RequestCanceledException INFO [2019-12-17 12:35:59,731] jepsen worker 77 - jepsen.util 14567 :info :cas [25 [18 43]] :RequestCanceledException INFO [2019-12-17 12:35:59,732] jepsen worker 54 - jepsen.util 11214 :invoke :read [18 nil] INFO [2019-12-17 12:35:59,732] jepsen worker 3 - jepsen.util 9993 :ok :write [1 16] INFO [2019-12-17 12:35:59,732] jepsen worker 3 - jepsen.util 9993 :invoke :write [1 6] INFO [2019-12-17 12:35:59,733] jepsen worker 77 - jepsen.util 14657 :invoke :write [25 19] INFO [2019-12-17 12:35:59,745] jepsen worker 9 - jepsen.util 10989 :info :write [3 34] :RequestCanceledException INFO [2019-12-17 12:35:59,745] jepsen worker 9 - jepsen.util 11079 :invoke :write [3 14] INFO [2019-12-17 12:35:59,752] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58883,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:35:59,752] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58883,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:35:59,752] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58886,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:35:59,752] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58886,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:35:59,752] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58887,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:35:59,752] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58887,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:35:59,773] jepsen worker 70 - jepsen.util 11140 :ok :read [23 33] INFO [2019-12-17 12:35:59,773] jepsen worker 70 - jepsen.util 11140 :invoke :read [23 nil] INFO [2019-12-17 12:35:59,781] jepsen worker 45 - jepsen.util 10395 :fail :cas [15 [20 17]] :ValueNotSwapFrom INFO [2019-12-17 12:35:59,781] jepsen worker 45 - jepsen.util 10395 :invoke :read [15 nil] INFO [2019-12-17 12:35:59,786] jepsen worker 5 - jepsen.util 9815 :ok :write [1 10] INFO [2019-12-17 12:35:59,786] jepsen worker 5 - jepsen.util 9815 :invoke :write [1 49] INFO [2019-12-17 12:35:59,798] jepsen worker 37 - jepsen.util 9757 :info :write [12 23] :RequestCanceledException INFO [2019-12-17 12:35:59,798] jepsen worker 21 - jepsen.util 13431 :info :cas [7 [11 34]] :RequestCanceledException INFO [2019-12-17 12:35:59,799] jepsen worker 21 - jepsen.util 13521 :invoke :write [7 38] INFO [2019-12-17 12:35:59,801] jepsen worker 37 - jepsen.util 9847 :invoke :read [12 nil] INFO [2019-12-17 12:35:59,816] jepsen worker 59 - jepsen.util 13109 :info :write [19 24] :RequestCanceledException INFO [2019-12-17 12:35:59,817] jepsen worker 59 - jepsen.util 13199 :invoke :read [19 nil] INFO [2019-12-17 12:35:59,819] jepsen worker 45 - jepsen.util 10395 :ok :read [15 30] INFO [2019-12-17 12:35:59,819] jepsen worker 45 - jepsen.util 10395 :invoke :write [15 38] INFO [2019-12-17 12:35:59,822] jepsen worker 29 - jepsen.util 8579 :ok :write [9 11] INFO [2019-12-17 12:35:59,822] jepsen worker 29 - jepsen.util 8579 :invoke :cas [9 [37 22]] INFO [2019-12-17 12:35:59,845] jepsen worker 34 - jepsen.util 14074 :info :cas [11 [38 7]] :RequestCanceledException INFO [2019-12-17 12:35:59,846] jepsen worker 34 - jepsen.util 14164 :invoke :read [11 nil] INFO [2019-12-17 12:35:59,908] jepsen worker 31 - jepsen.util 9121 :ok :write [10 24] INFO [2019-12-17 12:35:59,908] jepsen worker 31 - jepsen.util 9121 :invoke :read [10 nil] INFO [2019-12-17 12:35:59,908] jepsen worker 19 - jepsen.util 10279 :ok :write [6 36] INFO [2019-12-17 12:35:59,909] jepsen worker 19 - jepsen.util 10279 :invoke :cas [6 [39 21]] INFO [2019-12-17 12:35:59,931] jepsen worker 86 - jepsen.util 14936 :info :write [28 14] :RequestCanceledException INFO [2019-12-17 12:35:59,932] jepsen worker 86 - jepsen.util 15026 :invoke :write [28 37] INFO [2019-12-17 12:35:59,933] jepsen worker 19 - jepsen.util 10279 :fail :cas [6 [39 21]] :ValueNotSwapFrom INFO [2019-12-17 12:35:59,933] jepsen worker 19 - jepsen.util 10279 :invoke :cas [6 [36 46]] INFO [2019-12-17 12:35:59,945] jepsen worker 28 - jepsen.util 8308 :info :write [9 29] :RequestCanceledException INFO [2019-12-17 12:35:59,946] jepsen worker 28 - jepsen.util 8398 :invoke :write [9 9] INFO [2019-12-17 12:35:59,949] jepsen worker 51 - jepsen.util 9681 :ok :read [17 34] INFO [2019-12-17 12:35:59,949] jepsen worker 51 - jepsen.util 9681 :invoke :cas [17 [41 48]] INFO [2019-12-17 12:35:59,962] jepsen worker 84 - jepsen.util 13494 :info :write [28 17] :RequestCanceledException INFO [2019-12-17 12:35:59,962] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:35:59,962] jepsen worker 8 - jepsen.util 13508 :info :write [2 19] :RequestCanceledException INFO [2019-12-17 12:35:59,962] jepsen nemesis - couchbase.util Rebalance check count: 461 INFO [2019-12-17 12:35:59,962] jepsen nemesis - couchbase.util Rebalance stuck count: 442 INFO [2019-12-17 12:35:59,963] jepsen worker 84 - jepsen.util 13584 :invoke :write [28 6] INFO [2019-12-17 12:35:59,963] jepsen worker 8 - jepsen.util 13598 :invoke :read [2 nil] INFO [2019-12-17 12:35:59,971] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58907,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:35:59,971] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58907,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:35:59,971] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58910,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:35:59,971] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58910,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:35:59,971] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58911,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:35:59,971] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58911,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:35:59,980] jepsen worker 30 - jepsen.util 10110 :fail :cas [10 [21 47]] :ValueNotSwapFrom INFO [2019-12-17 12:35:59,980] jepsen worker 30 - jepsen.util 10110 :invoke :cas [10 [7 7]] INFO [2019-12-17 12:35:59,980] jepsen worker 36 - jepsen.util 11016 :ok :read [12 30] INFO [2019-12-17 12:35:59,980] jepsen worker 36 - jepsen.util 11016 :invoke :write [12 24] INFO [2019-12-17 12:35:59,998] jepsen worker 50 - jepsen.util 14630 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:35:59,998] jepsen worker 16 - jepsen.util 15226 :info :cas [5 [19 44]] :RequestCanceledException INFO [2019-12-17 12:35:59,998] jepsen worker 50 - jepsen.util 14630 :invoke :cas [16 [5 1]] INFO [2019-12-17 12:35:59,999] jepsen worker 16 - jepsen.util 15316 :invoke :write [5 21] INFO [2019-12-17 12:36:00,008] jepsen worker 31 - jepsen.util 9121 :ok :read [10 24] INFO [2019-12-17 12:36:00,008] jepsen worker 31 - jepsen.util 9121 :invoke :cas [10 [3 45]] INFO [2019-12-17 12:36:00,016] jepsen worker 46 - jepsen.util 10126 :info :cas [15 [30 35]] :RequestCanceledException INFO [2019-12-17 12:36:00,016] jepsen worker 65 - jepsen.util 8615 :info :write [21 12] :RequestCanceledException INFO [2019-12-17 12:36:00,016] jepsen worker 43 - jepsen.util 13813 :info :cas [14 [5 19]] :RequestCanceledException INFO [2019-12-17 12:36:00,017] jepsen worker 46 - jepsen.util 10216 :invoke :cas [15 [18 11]] INFO [2019-12-17 12:36:00,018] jepsen worker 43 - jepsen.util 13903 :invoke :write [14 22] INFO [2019-12-17 12:36:00,020] jepsen worker 65 - jepsen.util 8705 :invoke :cas [21 [35 30]] INFO [2019-12-17 12:36:00,028] jepsen worker 56 - jepsen.util 10676 :ok :read [18 39] INFO [2019-12-17 12:36:00,028] jepsen worker 56 - jepsen.util 10676 :invoke :read [18 nil] INFO [2019-12-17 12:36:00,031] jepsen worker 22 - jepsen.util 13612 :info :cas [7 [14 47]] :RequestCanceledException INFO [2019-12-17 12:36:00,032] jepsen worker 22 - jepsen.util 13702 :invoke :cas [7 [44 43]] INFO [2019-12-17 12:36:00,042] jepsen worker 30 - jepsen.util 10110 :fail :cas [10 [7 7]] :ValueNotSwapFrom INFO [2019-12-17 12:36:00,043] jepsen worker 30 - jepsen.util 10110 :invoke :read [10 nil] INFO [2019-12-17 12:36:00,046] jepsen worker 67 - jepsen.util 13927 :info :write [22 6] :RequestCanceledException INFO [2019-12-17 12:36:00,048] jepsen worker 67 - jepsen.util 14017 :invoke :write [22 33] INFO [2019-12-17 12:36:00,076] jepsen worker 31 - jepsen.util 9121 :fail :cas [10 [3 45]] :ValueNotSwapFrom INFO [2019-12-17 12:36:00,085] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58925,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:36:00,086] jepsen worker 30 - jepsen.util 10110 :ok :read [10 24] INFO [2019-12-17 12:36:00,095] jepsen worker 46 - jepsen.util 10216 :fail :cas [15 [18 11]] :ValueNotSwapFrom INFO [2019-12-17 12:36:00,097] jepsen worker 2 - jepsen.util 13592 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:36:00,099] jepsen worker 13 - jepsen.util 9643 :fail :cas [4 [18 12]] :ValueNotSwapFrom INFO [2019-12-17 12:36:00,114] jepsen worker 45 - jepsen.util 10395 :ok :write [15 38] INFO [2019-12-17 12:36:00,117] jepsen worker 74 - jepsen.util 10064 :ok :write [24 16] INFO [2019-12-17 12:36:00,131] jepsen worker 4 - jepsen.util 9454 :info :write [1 3] :RequestCanceledException INFO [2019-12-17 12:36:00,131] jepsen worker 41 - jepsen.util 14981 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:36:00,151] jepsen worker 87 - jepsen.util 9807 :ok :read [29 40] INFO [2019-12-17 12:36:00,175] jepsen worker 70 - jepsen.util 11140 :ok :read [23 33] INFO [2019-12-17 12:36:00,198] jepsen worker 71 - jepsen.util 10961 :info :write [23 41] :RequestCanceledException INFO [2019-12-17 12:36:00,210] jepsen worker 19 - jepsen.util 10279 :ok :cas [6 [36 46]] INFO [2019-12-17 12:36:00,210] jepsen worker 9 - jepsen.util 11079 :ok :write [3 14] INFO [2019-12-17 12:36:00,213] jepsen worker 42 - jepsen.util 14892 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:36:00,213] jepsen worker 11 - jepsen.util 9821 :fail :read [3 nil] :RequestTimeoutException INFO [2019-12-17 12:36:00,231] jepsen worker 15 - jepsen.util 12885 :info :write [5 43] :RequestCanceledException INFO [2019-12-17 12:36:00,231] jepsen worker 32 - jepsen.util 10292 :info :write [10 11] :RequestCanceledException INFO [2019-12-17 12:36:00,231] jepsen worker 57 - jepsen.util 13467 :info :cas [19 [43 35]] :RequestCanceledException INFO [2019-12-17 12:36:00,232] jepsen worker 37 - jepsen.util 9847 :ok :read [12 30] INFO [2019-12-17 12:36:00,245] jepsen worker 28 - jepsen.util 8398 :ok :write [9 9] INFO [2019-12-17 12:36:00,245] jepsen worker 17 - jepsen.util 13787 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:36:00,247] jepsen worker 31 - jepsen.util 9121 :invoke :write [10 32] INFO [2019-12-17 12:36:00,248] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58925,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:36:00,248] jepsen worker 30 - jepsen.util 10110 :invoke :read [10 nil] INFO [2019-12-17 12:36:00,248] jepsen worker 46 - jepsen.util 10216 :invoke :read [15 nil] INFO [2019-12-17 12:36:00,248] jepsen worker 2 - jepsen.util 13592 :invoke :read [0 nil] INFO [2019-12-17 12:36:00,248] jepsen worker 13 - jepsen.util 9643 :invoke :cas [4 [34 39]] INFO [2019-12-17 12:36:00,248] jepsen worker 45 - jepsen.util 10395 :invoke :write [15 32] INFO [2019-12-17 12:36:00,248] jepsen worker 74 - jepsen.util 10064 :invoke :write [24 33] INFO [2019-12-17 12:36:00,248] jepsen worker 41 - jepsen.util 14981 :invoke :cas [13 [47 4]] INFO [2019-12-17 12:36:00,248] jepsen worker 87 - jepsen.util 9807 :invoke :read [29 nil] INFO [2019-12-17 12:36:00,248] jepsen worker 70 - jepsen.util 11140 :invoke :read [23 nil] INFO [2019-12-17 12:36:00,248] jepsen worker 19 - jepsen.util 10279 :invoke :read [6 nil] INFO [2019-12-17 12:36:00,248] jepsen worker 9 - jepsen.util 11079 :invoke :cas [3 [37 48]] INFO [2019-12-17 12:36:00,248] jepsen worker 42 - jepsen.util 14892 :invoke :cas [14 [24 38]] INFO [2019-12-17 12:36:00,248] jepsen worker 11 - jepsen.util 9821 :invoke :cas [3 [5 6]] INFO [2019-12-17 12:36:00,249] jepsen worker 37 - jepsen.util 9847 :invoke :cas [12 [17 23]] INFO [2019-12-17 12:36:00,249] jepsen worker 28 - jepsen.util 8398 :invoke :read [9 nil] INFO [2019-12-17 12:36:00,249] jepsen worker 17 - jepsen.util 13787 :invoke :read [5 nil] INFO [2019-12-17 12:36:00,249] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58928,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:36:00,249] jepsen worker 4 - jepsen.util 9544 :invoke :write [1 11] INFO [2019-12-17 12:36:00,249] jepsen worker 71 - jepsen.util 11051 :invoke :cas [23 [40 38]] INFO [2019-12-17 12:36:00,250] jepsen worker 15 - jepsen.util 12975 :invoke :cas [5 [25 17]] INFO [2019-12-17 12:36:00,250] jepsen worker 57 - jepsen.util 13557 :invoke :cas [19 [25 6]] INFO [2019-12-17 12:36:00,250] jepsen worker 32 - jepsen.util 10382 :invoke :cas [10 [48 18]] INFO [2019-12-17 12:36:00,250] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58928,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:36:00,263] jepsen worker 60 - jepsen.util 14730 :info :cas [20 [49 18]] :RequestCanceledException INFO [2019-12-17 12:36:00,264] jepsen worker 60 - jepsen.util 14820 :invoke :write [20 37] INFO [2019-12-17 12:36:00,288] jepsen worker 36 - jepsen.util 11016 :ok :write [12 24] INFO [2019-12-17 12:36:00,288] jepsen worker 36 - jepsen.util 11016 :invoke :cas [12 [35 46]] INFO [2019-12-17 12:36:00,291] jepsen worker 78 - jepsen.util 10158 :fail :cas [26 [39 44]] :ValueNotSwapFrom INFO [2019-12-17 12:36:00,291] jepsen worker 78 - jepsen.util 10158 :invoke :read [26 nil] INFO [2019-12-17 12:36:00,298] jepsen worker 89 - jepsen.util 10349 :info :write [29 27] :RequestCanceledException INFO [2019-12-17 12:36:00,299] jepsen worker 89 - jepsen.util 10439 :invoke :write [29 35] INFO [2019-12-17 12:36:00,300] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58939,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:36:00,300] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58946,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:36:00,300] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58939,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:36:00,300] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58946,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:36:00,300] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58940,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:36:00,300] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58949,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:36:00,300] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58940,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:36:00,300] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58949,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:36:00,300] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58941,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:36:00,300] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58950,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:36:00,300] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58941,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:36:00,300] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58950,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:36:00,325] jepsen worker 54 - jepsen.util 11214 :ok :read [18 39] INFO [2019-12-17 12:36:00,325] jepsen worker 54 - jepsen.util 11214 :invoke :read [18 nil] INFO [2019-12-17 12:36:00,338] jepsen worker 30 - jepsen.util 10110 :ok :read [10 24] INFO [2019-12-17 12:36:00,338] jepsen worker 30 - jepsen.util 10110 :invoke :read [10 nil] INFO [2019-12-17 12:36:00,361] jepsen worker 5 - jepsen.util 9815 :ok :write [1 49] INFO [2019-12-17 12:36:00,361] jepsen worker 5 - jepsen.util 9815 :invoke :cas [1 [12 19]] INFO [2019-12-17 12:36:00,363] jepsen worker 6 - jepsen.util 14316 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:36:00,363] jepsen worker 6 - jepsen.util 14316 :invoke :write [2 10] INFO [2019-12-17 12:36:00,381] jepsen worker 46 - jepsen.util 10216 :ok :read [15 38] INFO [2019-12-17 12:36:00,381] jepsen worker 46 - jepsen.util 10216 :invoke :cas [15 [42 37]] INFO [2019-12-17 12:36:00,383] jepsen worker 9 - jepsen.util 11079 :fail :cas [3 [37 48]] :ValueNotSwapFrom INFO [2019-12-17 12:36:00,383] jepsen worker 9 - jepsen.util 11079 :invoke :read [3 nil] INFO [2019-12-17 12:36:00,384] jepsen worker 19 - jepsen.util 10279 :ok :read [6 46] INFO [2019-12-17 12:36:00,384] jepsen worker 19 - jepsen.util 10279 :invoke :cas [6 [47 20]] INFO [2019-12-17 12:36:00,394] jepsen worker 56 - jepsen.util 10676 :ok :read [18 39] INFO [2019-12-17 12:36:00,394] jepsen worker 56 - jepsen.util 10676 :invoke :write [18 32] INFO [2019-12-17 12:36:00,398] jepsen worker 32 - jepsen.util 10382 :fail :cas [10 [48 18]] :ValueNotSwapFrom INFO [2019-12-17 12:36:00,398] jepsen worker 32 - jepsen.util 10382 :invoke :cas [10 [44 40]] INFO [2019-12-17 12:36:00,403] jepsen worker 52 - jepsen.util 10132 :ok :write [17 11] INFO [2019-12-17 12:36:00,403] jepsen worker 52 - jepsen.util 10132 :invoke :read [17 nil] INFO [2019-12-17 12:36:00,411] jepsen worker 5 - jepsen.util 9815 :fail :cas [1 [12 19]] :ValueNotSwapFrom INFO [2019-12-17 12:36:00,411] jepsen worker 5 - jepsen.util 9815 :invoke :read [1 nil] INFO [2019-12-17 12:36:00,421] jepsen worker 46 - jepsen.util 10216 :fail :cas [15 [42 37]] :ValueNotSwapFrom INFO [2019-12-17 12:36:00,421] jepsen worker 46 - jepsen.util 10216 :invoke :read [15 nil] INFO [2019-12-17 12:36:00,425] jepsen worker 28 - jepsen.util 8398 :ok :read [9 9] INFO [2019-12-17 12:36:00,426] jepsen worker 28 - jepsen.util 8398 :invoke :cas [9 [42 14]] INFO [2019-12-17 12:36:00,431] jepsen worker 24 - jepsen.util 12534 :info :cas [8 [26 33]] :RequestCanceledException INFO [2019-12-17 12:36:00,432] jepsen worker 24 - jepsen.util 12624 :invoke :read [8 nil] INFO [2019-12-17 12:36:00,443] jepsen worker 52 - jepsen.util 10132 :ok :read [17 11] INFO [2019-12-17 12:36:00,443] jepsen worker 52 - jepsen.util 10132 :invoke :write [17 19] INFO [2019-12-17 12:36:00,491] jepsen worker 53 - jepsen.util 9683 :ok :write [17 14] INFO [2019-12-17 12:36:00,491] jepsen worker 53 - jepsen.util 9683 :invoke :cas [17 [49 26]] INFO [2019-12-17 12:36:00,497] jepsen worker 76 - jepsen.util 14116 :info :cas [25 [17 14]] :RequestCanceledException INFO [2019-12-17 12:36:00,498] jepsen worker 76 - jepsen.util 14206 :invoke :write [25 22] INFO [2019-12-17 12:36:00,499] jepsen worker 11 - jepsen.util 9821 :fail :cas [3 [5 6]] :ValueNotSwapFrom INFO [2019-12-17 12:36:00,499] jepsen worker 11 - jepsen.util 9821 :invoke :write [3 34] INFO [2019-12-17 12:36:00,516] jepsen worker 88 - jepsen.util 10528 :info :write [29 22] :RequestCanceledException INFO [2019-12-17 12:36:00,516] jepsen worker 12 - jepsen.util 10632 :info :write [4 1] :RequestCanceledException INFO [2019-12-17 12:36:00,517] jepsen worker 12 - jepsen.util 10722 :invoke :cas [4 [46 32]] INFO [2019-12-17 12:36:00,518] jepsen worker 88 - jepsen.util 10618 :invoke :cas [29 [7 40]] INFO [2019-12-17 12:36:00,519] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58992,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:36:00,520] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58992,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:36:00,520] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58996,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:36:00,520] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":58996,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:36:00,527] jepsen worker 5 - jepsen.util 9815 :ok :read [1 49] INFO [2019-12-17 12:36:00,528] jepsen worker 5 - jepsen.util 9815 :invoke :read [1 nil] INFO [2019-12-17 12:36:00,531] jepsen worker 46 - jepsen.util 10216 :ok :read [15 38] INFO [2019-12-17 12:36:00,531] jepsen worker 46 - jepsen.util 10216 :invoke :write [15 45] INFO [2019-12-17 12:36:00,531] jepsen worker 38 - jepsen.util 10928 :info :write [12 0] :RequestCanceledException INFO [2019-12-17 12:36:00,532] jepsen worker 38 - jepsen.util 11018 :invoke :write [12 39] INFO [2019-12-17 12:36:00,537] jepsen worker 9 - jepsen.util 11079 :ok :read [3 14] INFO [2019-12-17 12:36:00,537] jepsen worker 9 - jepsen.util 11079 :invoke :write [3 11] INFO [2019-12-17 12:36:00,563] jepsen worker 19 - jepsen.util 10279 :fail :cas [6 [47 20]] :ValueNotSwapFrom INFO [2019-12-17 12:36:00,563] jepsen worker 19 - jepsen.util 10279 :invoke :write [6 36] INFO [2019-12-17 12:36:00,563] jepsen worker 35 - jepsen.util 13895 :info :cas [11 [47 3]] :RequestCanceledException INFO [2019-12-17 12:36:00,564] jepsen worker 35 - jepsen.util 13985 :invoke :cas [11 [14 47]] INFO [2019-12-17 12:36:00,573] jepsen worker 28 - jepsen.util 8398 :fail :cas [9 [42 14]] :ValueNotSwapFrom INFO [2019-12-17 12:36:00,573] jepsen worker 28 - jepsen.util 8398 :invoke :read [9 nil] INFO [2019-12-17 12:36:00,587] jepsen worker 31 - jepsen.util 9121 :ok :write [10 32] INFO [2019-12-17 12:36:00,587] jepsen worker 31 - jepsen.util 9121 :invoke :cas [10 [26 6]] INFO [2019-12-17 12:36:00,592] jepsen worker 5 - jepsen.util 9815 :ok :read [1 49] INFO [2019-12-17 12:36:00,592] jepsen worker 5 - jepsen.util 9815 :invoke :cas [1 [46 28]] INFO [2019-12-17 12:36:00,596] jepsen worker 13 - jepsen.util 9643 :fail :cas [4 [34 39]] :ValueNotSwapFrom INFO [2019-12-17 12:36:00,596] jepsen worker 13 - jepsen.util 9643 :invoke :read [4 nil] INFO [2019-12-17 12:36:00,603] jepsen worker 30 - jepsen.util 10110 :ok :read [10 32] INFO [2019-12-17 12:36:00,604] jepsen worker 30 - jepsen.util 10110 :invoke :write [10 24] INFO [2019-12-17 12:36:00,606] jepsen worker 71 - jepsen.util 11051 :fail :cas [23 [40 38]] :ValueNotSwapFrom INFO [2019-12-17 12:36:00,606] jepsen worker 71 - jepsen.util 11051 :invoke :write [23 11] INFO [2019-12-17 12:36:00,619] jepsen worker 31 - jepsen.util 9121 :fail :cas [10 [26 6]] :ValueNotSwapFrom INFO [2019-12-17 12:36:00,619] jepsen worker 31 - jepsen.util 9121 :invoke :read [10 nil] INFO [2019-12-17 12:36:00,620] jepsen worker 70 - jepsen.util 11140 :ok :read [23 33] INFO [2019-12-17 12:36:00,620] jepsen worker 70 - jepsen.util 11140 :invoke :read [23 nil] INFO [2019-12-17 12:36:00,630] jepsen worker 44 - jepsen.util 14174 :info :write [14 20] :RequestCanceledException INFO [2019-12-17 12:36:00,632] jepsen worker 44 - jepsen.util 14264 :invoke :cas [14 [7 22]] INFO [2019-12-17 12:36:00,633] jepsen worker 37 - jepsen.util 9847 :fail :cas [12 [17 23]] :ValueNotSwapFrom INFO [2019-12-17 12:36:00,633] jepsen worker 37 - jepsen.util 9847 :invoke :read [12 nil] INFO [2019-12-17 12:36:00,645] jepsen worker 27 - jepsen.util 9837 :fail :read [9 nil] :RequestTimeoutException INFO [2019-12-17 12:36:00,645] jepsen worker 27 - jepsen.util 9837 :invoke :write [9 26] INFO [2019-12-17 12:36:00,651] jepsen worker 45 - jepsen.util 10395 :ok :write [15 32] INFO [2019-12-17 12:36:00,651] jepsen worker 45 - jepsen.util 10395 :invoke :cas [15 [45 30]] INFO [2019-12-17 12:36:00,663] jepsen worker 14 - jepsen.util 10004 :info :write [4 7] :RequestCanceledException INFO [2019-12-17 12:36:00,664] jepsen worker 14 - jepsen.util 10094 :invoke :write [4 1] INFO [2019-12-17 12:36:00,685] jepsen worker 54 - jepsen.util 11214 :ok :read [18 39] INFO [2019-12-17 12:36:00,685] jepsen worker 54 - jepsen.util 11214 :invoke :read [18 nil] INFO [2019-12-17 12:36:00,697] jepsen worker 62 - jepsen.util 14282 :info :write [20 12] :RequestCanceledException INFO [2019-12-17 12:36:00,698] jepsen worker 53 - jepsen.util 9683 :fail :cas [17 [49 26]] :ValueNotSwapFrom INFO [2019-12-17 12:36:00,698] jepsen worker 53 - jepsen.util 9683 :invoke :read [17 nil] INFO [2019-12-17 12:36:00,698] jepsen worker 45 - jepsen.util 10395 :fail :cas [15 [45 30]] :ValueNotSwapFrom INFO [2019-12-17 12:36:00,698] jepsen worker 45 - jepsen.util 10395 :invoke :cas [15 [20 43]] INFO [2019-12-17 12:36:00,699] jepsen worker 62 - jepsen.util 14372 :invoke :cas [20 [46 13]] INFO [2019-12-17 12:36:00,717] jepsen worker 36 - jepsen.util 11016 :fail :cas [12 [35 46]] :ValueNotSwapFrom INFO [2019-12-17 12:36:00,717] jepsen worker 36 - jepsen.util 11016 :invoke :write [12 36] INFO [2019-12-17 12:36:00,718] jepsen worker 45 - jepsen.util 10395 :fail :cas [15 [20 43]] :ValueNotSwapFrom INFO [2019-12-17 12:36:00,718] jepsen worker 45 - jepsen.util 10395 :invoke :cas [15 [27 29]] INFO [2019-12-17 12:36:00,720] jepsen worker 9 - jepsen.util 11079 :ok :write [3 11] INFO [2019-12-17 12:36:00,721] jepsen worker 9 - jepsen.util 11079 :invoke :cas [3 [42 48]] INFO [2019-12-17 12:36:00,738] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":59020,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:36:00,739] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":59020,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:36:00,739] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":59023,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:36:00,739] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":59023,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:36:00,739] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":59024,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:36:00,739] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":59024,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:36:00,754] jepsen worker 45 - jepsen.util 10395 :fail :cas [15 [27 29]] :ValueNotSwapFrom INFO [2019-12-17 12:36:00,755] jepsen worker 45 - jepsen.util 10395 :invoke :write [15 24] INFO [2019-12-17 12:36:00,757] jepsen worker 9 - jepsen.util 11079 :fail :cas [3 [42 48]] :ValueNotSwapFrom INFO [2019-12-17 12:36:00,758] jepsen worker 9 - jepsen.util 11079 :invoke :read [3 nil] INFO [2019-12-17 12:36:00,782] jepsen worker 9 - jepsen.util 11079 :ok :read [3 11] INFO [2019-12-17 12:36:00,783] jepsen worker 9 - jepsen.util 11079 :invoke :read [3 nil] INFO [2019-12-17 12:36:00,803] jepsen worker 27 - jepsen.util 9837 :ok :write [9 26] INFO [2019-12-17 12:36:00,804] jepsen worker 27 - jepsen.util 9837 :invoke :read [9 nil] INFO [2019-12-17 12:36:00,811] jepsen worker 9 - jepsen.util 11079 :ok :read [3 11] INFO [2019-12-17 12:36:00,811] jepsen worker 9 - jepsen.util 11079 :invoke :read [3 nil] INFO [2019-12-17 12:36:00,820] jepsen worker 27 - jepsen.util 9837 :ok :read [9 26] INFO [2019-12-17 12:36:00,821] jepsen worker 27 - jepsen.util 9837 :invoke :read [9 nil] INFO [2019-12-17 12:36:00,829] jepsen worker 9 - jepsen.util 11079 :ok :read [3 11] INFO [2019-12-17 12:36:00,829] jepsen worker 9 - jepsen.util 11079 :invoke :read [3 nil] INFO [2019-12-17 12:36:00,832] jepsen worker 85 - jepsen.util 14755 :info :write [28 20] :RequestCanceledException INFO [2019-12-17 12:36:00,833] jepsen worker 85 - jepsen.util 14845 :invoke :read [28 nil] INFO [2019-12-17 12:36:00,845] jepsen worker 40 - jepsen.util 14350 :info :cas [13 [34 39]] :RequestCanceledException INFO [2019-12-17 12:36:00,846] jepsen worker 40 - jepsen.util 14440 :invoke :write [13 40] INFO [2019-12-17 12:36:00,854] jepsen worker 54 - jepsen.util 11214 :ok :read [18 39] INFO [2019-12-17 12:36:00,854] jepsen worker 54 - jepsen.util 11214 :invoke :write [18 23] INFO [2019-12-17 12:36:00,861] jepsen worker 5 - jepsen.util 9815 :fail :cas [1 [46 28]] :ValueNotSwapFrom INFO [2019-12-17 12:36:00,861] jepsen worker 5 - jepsen.util 9815 :invoke :write [1 4] INFO [2019-12-17 12:36:00,867] jepsen worker 69 - jepsen.util 10329 :ok :write [23 47] INFO [2019-12-17 12:36:00,868] jepsen worker 69 - jepsen.util 10329 :invoke :cas [23 [43 13]] INFO [2019-12-17 12:36:00,875] jepsen worker 27 - jepsen.util 9837 :ok :read [9 26] INFO [2019-12-17 12:36:00,875] jepsen worker 27 - jepsen.util 9837 :invoke :read [9 nil] INFO [2019-12-17 12:36:00,878] jepsen worker 9 - jepsen.util 11079 :ok :read [3 11] INFO [2019-12-17 12:36:00,878] jepsen worker 9 - jepsen.util 11079 :invoke :read [3 nil] INFO [2019-12-17 12:36:00,900] jepsen worker 27 - jepsen.util 9837 :ok :read [9 26] INFO [2019-12-17 12:36:00,900] jepsen worker 27 - jepsen.util 9837 :invoke :write [9 30] INFO [2019-12-17 12:36:00,916] jepsen worker 9 - jepsen.util 11079 :ok :read [3 11] INFO [2019-12-17 12:36:00,916] jepsen worker 9 - jepsen.util 11079 :invoke :cas [3 [16 32]] INFO [2019-12-17 12:36:00,931] jepsen worker 83 - jepsen.util 14123 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:36:00,931] jepsen worker 66 - jepsen.util 13656 :info :write [22 31] :RequestCanceledException INFO [2019-12-17 12:36:00,931] jepsen worker 23 - jepsen.util 13883 :info :write [7 40] :RequestCanceledException INFO [2019-12-17 12:36:00,931] jepsen worker 83 - jepsen.util 14123 :invoke :read [27 nil] INFO [2019-12-17 12:36:00,932] jepsen worker 66 - jepsen.util 13746 :invoke :read [22 nil] INFO [2019-12-17 12:36:00,932] jepsen worker 23 - jepsen.util 13973 :invoke :cas [7 [12 49]] INFO [2019-12-17 12:36:00,937] jepsen worker 9 - jepsen.util 11079 :fail :cas [3 [16 32]] :ValueNotSwapFrom INFO [2019-12-17 12:36:00,937] jepsen worker 9 - jepsen.util 11079 :invoke :cas [3 [35 37]] INFO [2019-12-17 12:36:00,944] jepsen worker 79 - jepsen.util 10519 :info :write [26 17] :RequestCanceledException INFO [2019-12-17 12:36:00,945] jepsen worker 70 - jepsen.util 11140 :ok :read [23 47] INFO [2019-12-17 12:36:00,945] jepsen worker 70 - jepsen.util 11140 :invoke :cas [23 [18 8]] INFO [2019-12-17 12:36:00,950] jepsen worker 79 - jepsen.util 10609 :invoke :write [26 40] INFO [2019-12-17 12:36:00,958] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":59045,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:36:00,958] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":59045,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:36:00,958] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":59049,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:36:00,958] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":59049,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:36:00,958] jepsen worker 53 - jepsen.util 9683 :ok :read [17 14] INFO [2019-12-17 12:36:00,959] jepsen worker 53 - jepsen.util 9683 :invoke :read [17 nil] INFO [2019-12-17 12:36:00,962] jepsen worker 49 - jepsen.util 13909 :info :cas [16 [20 46]] :RequestCanceledException INFO [2019-12-17 12:36:00,963] jepsen worker 51 - jepsen.util 9681 :fail :cas [17 [41 48]] :ValueNotSwapFrom INFO [2019-12-17 12:36:00,963] jepsen worker 51 - jepsen.util 9681 :invoke :write [17 34] INFO [2019-12-17 12:36:00,963] jepsen worker 49 - jepsen.util 13999 :invoke :read [16 nil] INFO [2019-12-17 12:36:00,969] jepsen worker 9 - jepsen.util 11079 :fail :cas [3 [35 37]] :ValueNotSwapFrom INFO [2019-12-17 12:36:00,971] jepsen worker 9 - jepsen.util 11079 :invoke :read [3 nil] INFO [2019-12-17 12:36:00,972] jepsen worker 37 - jepsen.util 9847 :ok :read [12 24] INFO [2019-12-17 12:36:00,972] jepsen worker 37 - jepsen.util 9847 :invoke :read [12 nil] INFO [2019-12-17 12:36:00,978] jepsen worker 88 - jepsen.util 10618 :fail :cas [29 [7 40]] :ValueNotSwapFrom INFO [2019-12-17 12:36:00,978] jepsen worker 88 - jepsen.util 10618 :invoke :write [29 0] INFO [2019-12-17 12:36:00,982] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:36:00,982] jepsen nemesis - couchbase.util Rebalance check count: 462 INFO [2019-12-17 12:36:00,982] jepsen nemesis - couchbase.util Rebalance stuck count: 443 INFO [2019-12-17 12:36:00,987] jepsen worker 27 - jepsen.util 9837 :ok :write [9 30] INFO [2019-12-17 12:36:00,988] jepsen worker 27 - jepsen.util 9837 :invoke :read [9 nil] INFO [2019-12-17 12:36:00,993] jepsen worker 7 - jepsen.util 14047 :info :cas [2 [3 16]] :RequestCanceledException INFO [2019-12-17 12:36:00,993] jepsen worker 9 - jepsen.util 11079 :ok :read [3 11] INFO [2019-12-17 12:36:00,993] jepsen worker 9 - jepsen.util 11079 :invoke :read [3 nil] INFO [2019-12-17 12:36:00,994] jepsen worker 7 - jepsen.util 14137 :invoke :read [2 nil] INFO [2019-12-17 12:36:00,995] jepsen worker 55 - jepsen.util 10945 :ok :write [18 20] INFO [2019-12-17 12:36:00,995] jepsen worker 55 - jepsen.util 10945 :invoke :write [18 20] INFO [2019-12-17 12:36:01,016] jepsen worker 63 - jepsen.util 9243 :info :write [21 21] :RequestCanceledException INFO [2019-12-17 12:36:01,017] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":59073,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:36:01,017] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":59073,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:36:01,017] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":59077,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:36:01,017] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":59077,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:36:01,017] jepsen worker 63 - jepsen.util 9333 :invoke :cas [21 [41 49]] INFO [2019-12-17 12:36:01,025] jepsen worker 9 - jepsen.util 11079 :ok :read [3 11] INFO [2019-12-17 12:36:01,025] jepsen worker 27 - jepsen.util 9837 :ok :read [9 30] INFO [2019-12-17 12:36:01,025] jepsen worker 9 - jepsen.util 11079 :invoke :cas [3 [38 8]] INFO [2019-12-17 12:36:01,025] jepsen worker 27 - jepsen.util 9837 :invoke :write [9 31] INFO [2019-12-17 12:36:01,026] jepsen worker 30 - jepsen.util 10110 :ok :write [10 24] INFO [2019-12-17 12:36:01,026] jepsen worker 30 - jepsen.util 10110 :invoke :write [10 0] INFO [2019-12-17 12:36:01,030] jepsen worker 26 - jepsen.util 14516 :info :cas [8 [37 29]] :RequestCanceledException INFO [2019-12-17 12:36:01,031] jepsen worker 26 - jepsen.util 14606 :invoke :read [8 nil] INFO [2019-12-17 12:36:01,043] jepsen worker 78 - jepsen.util 10158 :ok :read [26 15] INFO [2019-12-17 12:36:01,043] jepsen worker 78 - jepsen.util 10158 :invoke :write [26 31] INFO [2019-12-17 12:36:01,052] jepsen worker 63 - jepsen.util 9333 :fail :cas [21 [41 49]] :ValueNotSwapFrom INFO [2019-12-17 12:36:01,052] jepsen worker 63 - jepsen.util 9333 :invoke :cas [21 [39 46]] INFO [2019-12-17 12:36:01,065] jepsen worker 87 - jepsen.util 9807 :ok :read [29 27] INFO [2019-12-17 12:36:01,065] jepsen worker 87 - jepsen.util 9807 :invoke :write [29 41] INFO [2019-12-17 12:36:01,075] jepsen worker 9 - jepsen.util 11079 :fail :cas [3 [38 8]] :ValueNotSwapFrom INFO [2019-12-17 12:36:01,076] jepsen worker 9 - jepsen.util 11079 :invoke :write [3 48] INFO [2019-12-17 12:36:01,114] jepsen worker 38 - jepsen.util 11018 :ok :write [12 39] INFO [2019-12-17 12:36:01,114] jepsen worker 38 - jepsen.util 11018 :invoke :write [12 13] INFO [2019-12-17 12:36:01,116] jepsen worker 0 - jepsen.util 14670 :info :cas [0 [28 18]] :RequestCanceledException INFO [2019-12-17 12:36:01,117] jepsen worker 0 - jepsen.util 14760 :invoke :cas [0 [36 34]] INFO [2019-12-17 12:36:01,131] jepsen worker 61 - jepsen.util 14551 :info :write [20 20] :RequestCanceledException INFO [2019-12-17 12:36:01,131] jepsen worker 1 - jepsen.util 13951 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:36:01,132] jepsen worker 1 - jepsen.util 13951 :invoke :cas [0 [34 15]] INFO [2019-12-17 12:36:01,132] jepsen worker 61 - jepsen.util 14641 :invoke :cas [20 [37 41]] INFO [2019-12-17 12:36:01,150] jepsen worker 63 - jepsen.util 9333 :fail :cas [21 [39 46]] :ValueNotSwapFrom INFO [2019-12-17 12:36:01,150] jepsen worker 63 - jepsen.util 9333 :invoke :write [21 20] INFO [2019-12-17 12:36:01,167] jepsen worker 27 - jepsen.util 9837 :ok :write [9 31] INFO [2019-12-17 12:36:01,167] jepsen worker 27 - jepsen.util 9837 :invoke :write [9 10] INFO [2019-12-17 12:36:01,170] jepsen worker 74 - jepsen.util 10064 :ok :write [24 33] INFO [2019-12-17 12:36:01,170] jepsen worker 74 - jepsen.util 10064 :invoke :read [24 nil] INFO [2019-12-17 12:36:01,185] jepsen worker 37 - jepsen.util 9847 :ok :read [12 39] INFO [2019-12-17 12:36:01,185] jepsen worker 37 - jepsen.util 9847 :invoke :cas [12 [31 24]] INFO [2019-12-17 12:36:01,215] jepsen worker 74 - jepsen.util 10064 :ok :read [24 33] INFO [2019-12-17 12:36:01,215] jepsen worker 74 - jepsen.util 10064 :invoke :write [24 3] INFO [2019-12-17 12:36:01,246] jepsen worker 25 - jepsen.util 14875 :info :write [8 45] :RequestCanceledException INFO [2019-12-17 12:36:01,251] jepsen worker 25 - jepsen.util 14965 :invoke :cas [8 [39 24]] INFO [2019-12-17 12:36:01,262] jepsen worker 12 - jepsen.util 10722 :fail :cas [4 [46 32]] :ValueNotSwapFrom INFO [2019-12-17 12:36:01,263] jepsen worker 12 - jepsen.util 10722 :invoke :write [4 6] INFO [2019-12-17 12:36:01,263] jepsen worker 75 - jepsen.util 13485 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:36:01,263] jepsen worker 75 - jepsen.util 13485 :invoke :read [25 nil] INFO [2019-12-17 12:36:01,281] jepsen worker 9 - jepsen.util 11079 :ok :write [3 48] INFO [2019-12-17 12:36:01,281] jepsen worker 45 - jepsen.util 10395 :ok :write [15 24] INFO [2019-12-17 12:36:01,281] jepsen worker 9 - jepsen.util 11079 :invoke :write [3 29] INFO [2019-12-17 12:36:01,281] jepsen worker 45 - jepsen.util 10395 :invoke :write [15 18] INFO [2019-12-17 12:36:01,287] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":59097,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:36:01,287] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":59105,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:36:01,287] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":59097,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:36:01,287] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":59105,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:36:01,287] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":59098,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:36:01,287] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":59107,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:36:01,287] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":59098,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:36:01,287] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":59107,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:36:01,294] jepsen worker 58 - jepsen.util 14458 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:36:01,294] jepsen worker 58 - jepsen.util 14458 :invoke :read [19 nil] INFO [2019-12-17 12:36:01,313] jepsen worker 27 - jepsen.util 9837 :ok :write [9 10] INFO [2019-12-17 12:36:01,313] jepsen worker 27 - jepsen.util 9837 :invoke :write [9 18] INFO [2019-12-17 12:36:01,317] jepsen worker 70 - jepsen.util 11140 :fail :cas [23 [18 8]] :ValueNotSwapFrom INFO [2019-12-17 12:36:01,317] jepsen worker 70 - jepsen.util 11140 :invoke :read [23 nil] INFO [2019-12-17 12:36:01,331] jepsen worker 68 - jepsen.util 13388 :info :write [22 11] :RequestCanceledException INFO [2019-12-17 12:36:01,332] jepsen worker 68 - jepsen.util 13478 :invoke :write [22 12] INFO [2019-12-17 12:36:01,338] jepsen worker 63 - jepsen.util 9333 :ok :write [21 20] INFO [2019-12-17 12:36:01,338] jepsen worker 63 - jepsen.util 9333 :invoke :read [21 nil] INFO [2019-12-17 12:36:01,340] jepsen worker 30 - jepsen.util 10110 :ok :write [10 0] INFO [2019-12-17 12:36:01,340] jepsen worker 30 - jepsen.util 10110 :invoke :read [10 nil] INFO [2019-12-17 12:36:01,341] jepsen worker 37 - jepsen.util 9847 :fail :cas [12 [31 24]] :ValueNotSwapFrom INFO [2019-12-17 12:36:01,341] jepsen worker 37 - jepsen.util 9847 :invoke :write [12 4] INFO [2019-12-17 12:36:01,345] jepsen worker 73 - jepsen.util 9613 :info :write [24 11] :RequestCanceledException INFO [2019-12-17 12:36:01,350] jepsen worker 73 - jepsen.util 9703 :invoke :read [24 nil] INFO [2019-12-17 12:36:01,363] jepsen worker 82 - jepsen.util 14212 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:36:01,363] jepsen worker 82 - jepsen.util 14212 :invoke :read [27 nil] INFO [2019-12-17 12:36:01,365] jepsen worker 13 - jepsen.util 9643 :ok :read [4 7] INFO [2019-12-17 12:36:01,365] jepsen worker 13 - jepsen.util 9643 :invoke :cas [4 [15 41]] INFO [2019-12-17 12:36:01,389] jepsen worker 69 - jepsen.util 10329 :fail :cas [23 [43 13]] :ValueNotSwapFrom INFO [2019-12-17 12:36:01,389] jepsen worker 69 - jepsen.util 10329 :invoke :read [23 nil] INFO [2019-12-17 12:36:01,394] jepsen worker 30 - jepsen.util 10110 :ok :read [10 0] INFO [2019-12-17 12:36:01,394] jepsen worker 30 - jepsen.util 10110 :invoke :cas [10 [17 32]] INFO [2019-12-17 12:36:01,409] jepsen worker 30 - jepsen.util 10110 :fail :cas [10 [17 32]] :ValueNotSwapFrom INFO [2019-12-17 12:36:01,410] jepsen worker 30 - jepsen.util 10110 :invoke :write [10 28] INFO [2019-12-17 12:36:01,417] jepsen worker 63 - jepsen.util 9333 :ok :read [21 20] INFO [2019-12-17 12:36:01,417] jepsen worker 63 - jepsen.util 9333 :invoke :cas [21 [28 40]] INFO [2019-12-17 12:36:01,431] jepsen worker 72 - jepsen.util 10332 :info :write [24 19] :RequestCanceledException INFO [2019-12-17 12:36:01,432] jepsen worker 72 - jepsen.util 10422 :invoke :cas [24 [10 17]] INFO [2019-12-17 12:36:01,443] jepsen worker 18 - jepsen.util 10548 :info :write [6 41] :RequestCanceledException INFO [2019-12-17 12:36:01,444] jepsen worker 18 - jepsen.util 10638 :invoke :cas [6 [36 10]] INFO [2019-12-17 12:36:01,456] jepsen worker 73 - jepsen.util 9703 :ok :read [24 33] INFO [2019-12-17 12:36:01,456] jepsen worker 73 - jepsen.util 9703 :invoke :cas [24 [16 25]] INFO [2019-12-17 12:36:01,478] jepsen worker 63 - jepsen.util 9333 :fail :cas [21 [28 40]] :ValueNotSwapFrom INFO [2019-12-17 12:36:01,478] jepsen worker 63 - jepsen.util 9333 :invoke :write [21 39] INFO [2019-12-17 12:36:01,494] jepsen worker 73 - jepsen.util 9703 :fail :cas [24 [16 25]] :ValueNotSwapFrom INFO [2019-12-17 12:36:01,494] jepsen worker 73 - jepsen.util 9703 :invoke :read [24 nil] INFO [2019-12-17 12:36:01,506] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":59129,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:36:01,506] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":59129,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:36:01,506] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":59130,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:36:01,506] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":59130,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:36:01,506] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":59131,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:36:01,506] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":59131,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:36:01,510] jepsen worker 27 - jepsen.util 9837 :ok :write [9 18] INFO [2019-12-17 12:36:01,511] jepsen worker 27 - jepsen.util 9837 :invoke :cas [9 [42 18]] INFO [2019-12-17 12:36:01,527] jepsen worker 53 - jepsen.util 9683 :ok :read [17 14] INFO [2019-12-17 12:36:01,527] jepsen worker 53 - jepsen.util 9683 :invoke :cas [17 [48 20]] INFO [2019-12-17 12:36:01,532] jepsen worker 18 - jepsen.util 10638 :fail :cas [6 [36 10]] :ValueNotSwapFrom INFO [2019-12-17 12:36:01,532] jepsen worker 18 - jepsen.util 10638 :invoke :cas [6 [34 25]] INFO [2019-12-17 12:36:01,545] jepsen worker 47 - jepsen.util 10847 :fail :read [15 nil] :RequestTimeoutException INFO [2019-12-17 12:36:01,545] jepsen worker 47 - jepsen.util 10847 :invoke :cas [15 [45 24]] INFO [2019-12-17 12:36:01,558] jepsen worker 27 - jepsen.util 9837 :fail :cas [9 [42 18]] :ValueNotSwapFrom INFO [2019-12-17 12:36:01,558] jepsen worker 27 - jepsen.util 9837 :invoke :write [9 27] INFO [2019-12-17 12:36:01,563] jepsen worker 48 - jepsen.util 12738 :info :cas [16 [42 43]] :RequestCanceledException INFO [2019-12-17 12:36:01,564] jepsen worker 48 - jepsen.util 12828 :invoke :write [16 2] INFO [2019-12-17 12:36:01,569] jepsen worker 18 - jepsen.util 10638 :fail :cas [6 [34 25]] :ValueNotSwapFrom INFO [2019-12-17 12:36:01,569] jepsen worker 18 - jepsen.util 10638 :invoke :cas [6 [43 21]] INFO [2019-12-17 12:36:01,595] jepsen worker 20 - jepsen.util 10100 :info :write [6 3] :RequestCanceledException INFO [2019-12-17 12:36:01,595] jepsen worker 20 - jepsen.util 10190 :invoke :cas [6 [15 45]] INFO [2019-12-17 12:36:01,596] jepsen worker 53 - jepsen.util 9683 :fail :cas [17 [48 20]] :ValueNotSwapFrom INFO [2019-12-17 12:36:01,596] jepsen worker 53 - jepsen.util 9683 :invoke :write [17 3] INFO [2019-12-17 12:36:01,598] jepsen worker 47 - jepsen.util 10847 :fail :cas [15 [45 24]] :ValueNotSwapFrom INFO [2019-12-17 12:36:01,598] jepsen worker 47 - jepsen.util 10847 :invoke :cas [15 [10 39]] INFO [2019-12-17 12:36:01,606] jepsen worker 74 - jepsen.util 10064 :ok :write [24 3] INFO [2019-12-17 12:36:01,606] jepsen worker 74 - jepsen.util 10064 :invoke :read [24 nil] INFO [2019-12-17 12:36:01,616] jepsen worker 33 - jepsen.util 13983 :info :cas [11 [8 37]] :RequestCanceledException INFO [2019-12-17 12:36:01,622] jepsen worker 74 - jepsen.util 10064 :ok :read [24 3] INFO [2019-12-17 12:36:01,623] jepsen worker 74 - jepsen.util 10064 :invoke :write [24 17] INFO [2019-12-17 12:36:01,623] jepsen worker 33 - jepsen.util 14073 :invoke :read [11 nil] INFO [2019-12-17 12:36:01,626] jepsen worker 13 - jepsen.util 9643 :fail :cas [4 [15 41]] :ValueNotSwapFrom INFO [2019-12-17 12:36:01,626] jepsen worker 13 - jepsen.util 9643 :invoke :write [4 13] INFO [2019-12-17 12:36:01,628] jepsen worker 63 - jepsen.util 9333 :ok :write [21 39] INFO [2019-12-17 12:36:01,628] jepsen worker 63 - jepsen.util 9333 :invoke :cas [21 [13 28]] INFO [2019-12-17 12:36:01,631] jepsen worker 81 - jepsen.util 13581 :info :cas [27 [32 12]] :RequestCanceledException INFO [2019-12-17 12:36:01,632] jepsen worker 39 - jepsen.util 13539 :info :cas [13 [5 47]] :RequestCanceledException INFO [2019-12-17 12:36:01,632] jepsen worker 81 - jepsen.util 13671 :invoke :write [27 25] INFO [2019-12-17 12:36:01,633] jepsen worker 39 - jepsen.util 13629 :invoke :write [13 16] INFO [2019-12-17 12:36:01,660] jepsen worker 63 - jepsen.util 9333 :fail :cas [21 [13 28]] :ValueNotSwapFrom INFO [2019-12-17 12:36:01,660] jepsen worker 63 - jepsen.util 9333 :invoke :write [21 13] INFO [2019-12-17 12:36:01,662] jepsen worker 70 - jepsen.util 11140 :ok :read [23 47] INFO [2019-12-17 12:36:01,662] jepsen worker 70 - jepsen.util 11140 :invoke :read [23 nil] INFO [2019-12-17 12:36:01,662] jepsen worker 27 - jepsen.util 9837 :ok :write [9 27] INFO [2019-12-17 12:36:01,663] jepsen worker 27 - jepsen.util 9837 :invoke :cas [9 [25 9]] INFO [2019-12-17 12:36:01,684] jepsen worker 45 - jepsen.util 10395 :ok :write [15 18] INFO [2019-12-17 12:36:01,684] jepsen worker 45 - jepsen.util 10395 :invoke :cas [15 [33 24]] INFO [2019-12-17 12:36:01,698] jepsen worker 64 - jepsen.util 7984 :fail :read [21 nil] :RequestTimeoutException INFO [2019-12-17 12:36:01,698] jepsen worker 64 - jepsen.util 7984 :invoke :cas [21 [12 41]] INFO [2019-12-17 12:36:01,700] jepsen worker 9 - jepsen.util 11079 :ok :write [3 29] INFO [2019-12-17 12:36:01,700] jepsen worker 9 - jepsen.util 11079 :invoke :read [3 nil] INFO [2019-12-17 12:36:01,707] jepsen worker 27 - jepsen.util 9837 :fail :cas [9 [25 9]] :ValueNotSwapFrom INFO [2019-12-17 12:36:01,707] jepsen worker 27 - jepsen.util 9837 :invoke :write [9 12] INFO [2019-12-17 12:36:01,725] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":59157,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:36:01,725] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":59157,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:36:01,725] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":59160,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:36:01,725] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":59160,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:36:01,727] jepsen worker 45 - jepsen.util 10395 :fail :cas [15 [33 24]] :ValueNotSwapFrom INFO [2019-12-17 12:36:01,727] jepsen worker 45 - jepsen.util 10395 :invoke :read [15 nil] INFO [2019-12-17 12:36:01,737] jepsen worker 20 - jepsen.util 10190 :fail :cas [6 [15 45]] :ValueNotSwapFrom INFO [2019-12-17 12:36:01,738] jepsen worker 20 - jepsen.util 10190 :invoke :write [6 42] INFO [2019-12-17 12:36:01,745] jepsen worker 10 - jepsen.util 10180 :info :write [3 48] :RequestCanceledException INFO [2019-12-17 12:36:01,746] jepsen worker 10 - jepsen.util 10270 :invoke :cas [3 [49 19]] INFO [2019-12-17 12:36:01,754] jepsen worker 45 - jepsen.util 10395 :ok :read [15 18] INFO [2019-12-17 12:36:01,754] jepsen worker 45 - jepsen.util 10395 :invoke :write [15 20] INFO [2019-12-17 12:36:01,763] jepsen worker 77 - jepsen.util 14657 :info :write [25 19] :RequestCanceledException INFO [2019-12-17 12:36:01,764] jepsen worker 77 - jepsen.util 14747 :invoke :read [25 nil] INFO [2019-12-17 12:36:01,790] jepsen worker 9 - jepsen.util 11079 :ok :read [3 29] INFO [2019-12-17 12:36:01,790] jepsen worker 9 - jepsen.util 11079 :invoke :read [3 nil] INFO [2019-12-17 12:36:01,804] jepsen worker 63 - jepsen.util 9333 :ok :write [21 13] INFO [2019-12-17 12:36:01,804] jepsen worker 63 - jepsen.util 9333 :invoke :read [21 nil] INFO [2019-12-17 12:36:01,816] jepsen worker 3 - jepsen.util 9993 :info :write [1 6] :RequestCanceledException INFO [2019-12-17 12:36:01,817] jepsen worker 3 - jepsen.util 10083 :invoke :write [1 33] INFO [2019-12-17 12:36:01,822] jepsen worker 63 - jepsen.util 9333 :ok :read [21 13] INFO [2019-12-17 12:36:01,823] jepsen worker 63 - jepsen.util 9333 :invoke :read [21 nil] INFO [2019-12-17 12:36:01,831] jepsen worker 80 - jepsen.util 9980 :info :write [26 4] :RequestCanceledException INFO [2019-12-17 12:36:01,831] jepsen worker 59 - jepsen.util 13199 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:36:01,832] jepsen worker 59 - jepsen.util 13199 :invoke :read [19 nil] INFO [2019-12-17 12:36:01,832] jepsen worker 80 - jepsen.util 10070 :invoke :cas [26 [27 31]] INFO [2019-12-17 12:36:01,851] jepsen worker 63 - jepsen.util 9333 :ok :read [21 13] INFO [2019-12-17 12:36:01,851] jepsen worker 63 - jepsen.util 9333 :invoke :read [21 nil] INFO [2019-12-17 12:36:01,872] jepsen worker 63 - jepsen.util 9333 :ok :read [21 13] INFO [2019-12-17 12:36:01,872] jepsen worker 63 - jepsen.util 9333 :invoke :write [21 18] INFO [2019-12-17 12:36:01,895] jepsen worker 70 - jepsen.util 11140 :ok :read [23 47] INFO [2019-12-17 12:36:01,895] jepsen worker 70 - jepsen.util 11140 :invoke :cas [23 [18 7]] INFO [2019-12-17 12:36:01,898] jepsen worker 21 - jepsen.util 13521 :info :write [7 38] :RequestCanceledException INFO [2019-12-17 12:36:01,899] jepsen worker 21 - jepsen.util 13611 :invoke :cas [7 [28 15]] INFO [2019-12-17 12:36:01,912] jepsen worker 27 - jepsen.util 9837 :ok :write [9 12] INFO [2019-12-17 12:36:01,913] jepsen worker 27 - jepsen.util 9837 :invoke :read [9 nil] INFO [2019-12-17 12:36:01,922] jepsen worker 70 - jepsen.util 11140 :fail :cas [23 [18 7]] :ValueNotSwapFrom INFO [2019-12-17 12:36:01,922] jepsen worker 70 - jepsen.util 11140 :invoke :read [23 nil] INFO [2019-12-17 12:36:01,931] jepsen worker 29 - jepsen.util 8579 :info :cas [9 [37 22]] :RequestCanceledException INFO [2019-12-17 12:36:01,931] jepsen worker 27 - jepsen.util 9837 :ok :read [9 12] INFO [2019-12-17 12:36:01,931] jepsen worker 27 - jepsen.util 9837 :invoke :read [9 nil] INFO [2019-12-17 12:36:01,932] jepsen worker 29 - jepsen.util 8669 :invoke :write [9 16] INFO [2019-12-17 12:36:01,945] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":59182,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:36:01,945] jepsen worker 27 - jepsen.util 9837 :ok :read [9 12] INFO [2019-12-17 12:36:01,945] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":59182,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:36:01,945] jepsen worker 34 - jepsen.util 14164 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:36:01,945] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":59186,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:36:01,945] jepsen worker 27 - jepsen.util 9837 :invoke :cas [9 [36 38]] INFO [2019-12-17 12:36:01,945] jepsen worker 34 - jepsen.util 14164 :invoke :cas [11 [27 41]] INFO [2019-12-17 12:36:01,945] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":59186,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:36:01,954] jepsen worker 63 - jepsen.util 9333 :ok :write [21 18] INFO [2019-12-17 12:36:01,955] jepsen worker 63 - jepsen.util 9333 :invoke :cas [21 [12 7]] INFO [2019-12-17 12:36:01,957] jepsen worker 27 - jepsen.util 9837 :fail :cas [9 [36 38]] :ValueNotSwapFrom INFO [2019-12-17 12:36:01,957] jepsen worker 27 - jepsen.util 9837 :invoke :write [9 5] INFO [2019-12-17 12:36:01,957] jepsen worker 86 - jepsen.util 15026 :info :write [28 37] :RequestCanceledException INFO [2019-12-17 12:36:01,958] jepsen worker 86 - jepsen.util 15116 :invoke :cas [28 [48 14]] INFO [2019-12-17 12:36:01,972] jepsen worker 63 - jepsen.util 9333 :fail :cas [21 [12 7]] :ValueNotSwapFrom INFO [2019-12-17 12:36:01,972] jepsen worker 63 - jepsen.util 9333 :invoke :cas [21 [27 32]] INFO [2019-12-17 12:36:01,985] jepsen worker 63 - jepsen.util 9333 :fail :cas [21 [27 32]] :ValueNotSwapFrom INFO [2019-12-17 12:36:01,986] jepsen worker 63 - jepsen.util 9333 :invoke :cas [21 [25 40]] INFO [2019-12-17 12:36:01,994] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:36:01,994] jepsen nemesis - couchbase.util Rebalance check count: 463 INFO [2019-12-17 12:36:01,994] jepsen nemesis - couchbase.util Rebalance stuck count: 444 INFO [2019-12-17 12:36:01,997] jepsen worker 84 - jepsen.util 13584 :info :write [28 6] :RequestCanceledException INFO [2019-12-17 12:36:01,998] jepsen worker 84 - jepsen.util 13674 :invoke :cas [28 [8 6]] INFO [2019-12-17 12:36:02,005] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":59203,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:36:02,006] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":59203,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:36:02,006] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":59207,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:36:02,006] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":59207,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:36:02,014] jepsen worker 63 - jepsen.util 9333 :fail :cas [21 [25 40]] :ValueNotSwapFrom INFO [2019-12-17 12:36:02,014] jepsen worker 63 - jepsen.util 9333 :invoke :read [21 nil] INFO [2019-12-17 12:36:02,016] jepsen worker 8 - jepsen.util 13598 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:36:02,016] jepsen worker 8 - jepsen.util 13598 :invoke :cas [2 [25 16]] INFO [2019-12-17 12:36:02,017] jepsen worker 27 - jepsen.util 9837 :ok :write [9 5] INFO [2019-12-17 12:36:02,017] jepsen worker 27 - jepsen.util 9837 :invoke :write [9 4] INFO [2019-12-17 12:36:02,030] jepsen worker 63 - jepsen.util 9333 :ok :read [21 18] INFO [2019-12-17 12:36:02,030] jepsen worker 63 - jepsen.util 9333 :invoke :read [21 nil] INFO [2019-12-17 12:36:02,031] jepsen worker 65 - jepsen.util 8705 :info :cas [21 [35 30]] :RequestCanceledException INFO [2019-12-17 12:36:02,032] jepsen worker 65 - jepsen.util 8795 :invoke :cas [21 [29 12]] INFO [2019-12-17 12:36:02,032] jepsen worker 16 - jepsen.util 15316 :info :write [5 21] :RequestCanceledException INFO [2019-12-17 12:36:02,033] jepsen worker 16 - jepsen.util 15406 :invoke :write [5 9] INFO [2019-12-17 12:36:02,050] jepsen worker 63 - jepsen.util 9333 :ok :read [21 18] INFO [2019-12-17 12:36:02,050] jepsen worker 63 - jepsen.util 9333 :invoke :cas [21 [3 9]] INFO [2019-12-17 12:36:02,063] jepsen worker 43 - jepsen.util 13903 :info :write [14 22] :RequestCanceledException INFO [2019-12-17 12:36:02,064] jepsen worker 43 - jepsen.util 13993 :invoke :write [14 43] INFO [2019-12-17 12:36:02,066] jepsen worker 27 - jepsen.util 9837 :ok :write [9 4] INFO [2019-12-17 12:36:02,066] jepsen worker 27 - jepsen.util 9837 :invoke :write [9 22] INFO [2019-12-17 12:36:02,070] jepsen worker 63 - jepsen.util 9333 :fail :cas [21 [3 9]] :ValueNotSwapFrom INFO [2019-12-17 12:36:02,070] jepsen worker 63 - jepsen.util 9333 :invoke :read [21 nil] INFO [2019-12-17 12:36:02,098] jepsen worker 22 - jepsen.util 13702 :info :cas [7 [44 43]] :RequestCanceledException INFO [2019-12-17 12:36:02,098] jepsen worker 50 - jepsen.util 14630 :info :cas [16 [5 1]] :RequestCanceledException INFO [2019-12-17 12:36:02,099] jepsen worker 22 - jepsen.util 13792 :invoke :write [7 23] INFO [2019-12-17 12:36:02,099] jepsen worker 50 - jepsen.util 14720 :invoke :read [16 nil] INFO [2019-12-17 12:36:02,107] jepsen worker 63 - jepsen.util 9333 :ok :read [21 18] INFO [2019-12-17 12:36:02,107] jepsen worker 63 - jepsen.util 9333 :invoke :write [21 39] INFO [2019-12-17 12:36:02,112] jepsen worker 69 - jepsen.util 10329 :ok :read [23 47] INFO [2019-12-17 12:36:02,113] jepsen worker 69 - jepsen.util 10329 :invoke :write [23 46] INFO [2019-12-17 12:36:02,117] jepsen worker 67 - jepsen.util 14017 :info :write [22 33] :RequestCanceledException INFO [2019-12-17 12:36:02,118] jepsen worker 67 - jepsen.util 14107 :invoke :write [22 32] INFO [2019-12-17 12:36:02,155] jepsen worker 27 - jepsen.util 9837 :ok :write [9 22] INFO [2019-12-17 12:36:02,155] jepsen worker 27 - jepsen.util 9837 :invoke :write [9 7] INFO [2019-12-17 12:36:02,169] jepsen worker 63 - jepsen.util 9333 :ok :write [21 39] INFO [2019-12-17 12:36:02,169] jepsen worker 63 - jepsen.util 9333 :invoke :write [21 3] INFO [2019-12-17 12:36:02,229] jepsen worker 27 - jepsen.util 9837 :ok :write [9 7] INFO [2019-12-17 12:36:02,229] jepsen worker 27 - jepsen.util 9837 :invoke :read [9 nil] INFO [2019-12-17 12:36:02,241] jepsen worker 27 - jepsen.util 9837 :ok :read [9 7] INFO [2019-12-17 12:36:02,241] jepsen worker 27 - jepsen.util 9837 :invoke :cas [9 [38 34]] INFO [2019-12-17 12:36:02,242] jepsen worker 63 - jepsen.util 9333 :ok :write [21 3] INFO [2019-12-17 12:36:02,242] jepsen worker 63 - jepsen.util 9333 :invoke :write [21 39] INFO [2019-12-17 12:36:02,263] jepsen worker 15 - jepsen.util 12975 :info :cas [5 [25 17]] :RequestCanceledException INFO [2019-12-17 12:36:02,264] jepsen worker 15 - jepsen.util 13065 :invoke :cas [5 [2 15]] INFO [2019-12-17 12:36:02,266] jepsen worker 27 - jepsen.util 9837 :fail :cas [9 [38 34]] :ValueNotSwapFrom INFO [2019-12-17 12:36:02,266] jepsen worker 27 - jepsen.util 9837 :invoke :cas [9 [21 18]] INFO [2019-12-17 12:36:02,274] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":59225,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:36:02,274] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":59232,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:36:02,274] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":59225,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:36:02,274] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":59232,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:36:02,274] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":59226,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:36:02,274] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":59236,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:36:02,274] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":59226,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:36:02,274] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":59236,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:36:02,274] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":59227,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:36:02,274] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":59227,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:36:02,294] jepsen worker 70 - jepsen.util 11140 :ok :read [23 47] INFO [2019-12-17 12:36:02,294] jepsen worker 70 - jepsen.util 11140 :invoke :read [23 nil] INFO [2019-12-17 12:36:02,297] jepsen worker 27 - jepsen.util 9837 :fail :cas [9 [21 18]] :ValueNotSwapFrom INFO [2019-12-17 12:36:02,297] jepsen worker 27 - jepsen.util 9837 :invoke :write [9 35] INFO [2019-12-17 12:36:02,297] jepsen worker 2 - jepsen.util 13592 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:36:02,297] jepsen worker 2 - jepsen.util 13592 :invoke :cas [0 [32 33]] INFO [2019-12-17 12:36:02,316] jepsen worker 60 - jepsen.util 14820 :info :write [20 37] :RequestCanceledException INFO [2019-12-17 12:36:02,316] jepsen worker 42 - jepsen.util 14892 :info :cas [14 [24 38]] :RequestCanceledException INFO [2019-12-17 12:36:02,317] jepsen worker 42 - jepsen.util 14982 :invoke :read [14 nil] INFO [2019-12-17 12:36:02,317] jepsen worker 60 - jepsen.util 14910 :invoke :cas [20 [19 3]] INFO [2019-12-17 12:36:02,323] jepsen worker 63 - jepsen.util 9333 :ok :write [21 39] INFO [2019-12-17 12:36:02,323] jepsen worker 63 - jepsen.util 9333 :invoke :write [21 26] INFO [2019-12-17 12:36:02,331] jepsen worker 57 - jepsen.util 13557 :info :cas [19 [25 6]] :RequestCanceledException INFO [2019-12-17 12:36:02,331] jepsen worker 41 - jepsen.util 14981 :info :cas [13 [47 4]] :RequestCanceledException INFO [2019-12-17 12:36:02,331] jepsen worker 4 - jepsen.util 9544 :info :write [1 11] :RequestCanceledException INFO [2019-12-17 12:36:02,331] jepsen worker 89 - jepsen.util 10439 :info :write [29 35] :RequestCanceledException INFO [2019-12-17 12:36:02,332] jepsen worker 41 - jepsen.util 15071 :invoke :read [13 nil] INFO [2019-12-17 12:36:02,332] jepsen worker 57 - jepsen.util 13647 :invoke :cas [19 [40 19]] INFO [2019-12-17 12:36:02,332] jepsen worker 4 - jepsen.util 9634 :invoke :read [1 nil] INFO [2019-12-17 12:36:02,334] jepsen worker 89 - jepsen.util 10529 :invoke :write [29 4] INFO [2019-12-17 12:36:02,345] jepsen worker 17 - jepsen.util 13787 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:36:02,346] jepsen worker 17 - jepsen.util 13787 :invoke :cas [5 [12 17]] INFO [2019-12-17 12:36:02,359] jepsen worker 27 - jepsen.util 9837 :ok :write [9 35] INFO [2019-12-17 12:36:02,359] jepsen worker 27 - jepsen.util 9837 :invoke :write [9 37] INFO [2019-12-17 12:36:02,385] jepsen worker 4 - jepsen.util 9634 :ok :read [1 49] INFO [2019-12-17 12:36:02,385] jepsen worker 4 - jepsen.util 9634 :invoke :cas [1 [48 26]] INFO [2019-12-17 12:36:02,387] jepsen worker 63 - jepsen.util 9333 :ok :write [21 26] INFO [2019-12-17 12:36:02,388] jepsen worker 63 - jepsen.util 9333 :invoke :read [21 nil] INFO [2019-12-17 12:36:02,403] jepsen worker 63 - jepsen.util 9333 :ok :read [21 26] INFO [2019-12-17 12:36:02,403] jepsen worker 63 - jepsen.util 9333 :invoke :cas [21 [14 43]] INFO [2019-12-17 12:36:02,418] jepsen worker 63 - jepsen.util 9333 :fail :cas [21 [14 43]] :ValueNotSwapFrom INFO [2019-12-17 12:36:02,418] jepsen worker 63 - jepsen.util 9333 :invoke :cas [21 [37 23]] INFO [2019-12-17 12:36:02,422] jepsen worker 27 - jepsen.util 9837 :ok :write [9 37] INFO [2019-12-17 12:36:02,422] jepsen worker 27 - jepsen.util 9837 :invoke :write [9 6] INFO [2019-12-17 12:36:02,431] jepsen worker 80 - jepsen.util 10070 :fail :cas [26 [27 31]] :ValueNotSwapFrom INFO [2019-12-17 12:36:02,432] jepsen worker 80 - jepsen.util 10070 :invoke :cas [26 [11 5]] INFO [2019-12-17 12:36:02,439] jepsen worker 63 - jepsen.util 9333 :fail :cas [21 [37 23]] :ValueNotSwapFrom INFO [2019-12-17 12:36:02,439] jepsen worker 63 - jepsen.util 9333 :invoke :read [21 nil] INFO [2019-12-17 12:36:02,446] jepsen worker 24 - jepsen.util 12624 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:36:02,446] jepsen worker 24 - jepsen.util 12624 :invoke :read [8 nil] INFO [2019-12-17 12:36:02,452] jepsen worker 63 - jepsen.util 9333 :ok :read [21 26] INFO [2019-12-17 12:36:02,452] jepsen worker 63 - jepsen.util 9333 :invoke :write [21 7] INFO [2019-12-17 12:36:02,463] jepsen worker 6 - jepsen.util 14316 :info :write [2 10] :RequestCanceledException INFO [2019-12-17 12:36:02,464] jepsen worker 6 - jepsen.util 14406 :invoke :write [2 34] INFO [2019-12-17 12:36:02,478] jepsen worker 80 - jepsen.util 10070 :fail :cas [26 [11 5]] :ValueNotSwapFrom INFO [2019-12-17 12:36:02,478] jepsen worker 80 - jepsen.util 10070 :invoke :cas [26 [2 46]] INFO [2019-12-17 12:36:02,488] jepsen worker 27 - jepsen.util 9837 :ok :write [9 6] INFO [2019-12-17 12:36:02,488] jepsen worker 27 - jepsen.util 9837 :invoke :cas [9 [42 19]] INFO [2019-12-17 12:36:02,493] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":59259,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:36:02,493] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":59259,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:36:02,493] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":59264,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:36:02,493] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":59264,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:36:02,498] jepsen worker 56 - jepsen.util 10676 :info :write [18 32] :RequestCanceledException INFO [2019-12-17 12:36:02,499] jepsen worker 32 - jepsen.util 10382 :info :cas [10 [44 40]] :RequestCanceledException INFO [2019-12-17 12:36:02,499] jepsen worker 56 - jepsen.util 10766 :invoke :write [18 1] INFO [2019-12-17 12:36:02,506] jepsen worker 32 - jepsen.util 10472 :invoke :cas [10 [9 19]] INFO [2019-12-17 12:36:02,508] jepsen worker 27 - jepsen.util 9837 :fail :cas [9 [42 19]] :ValueNotSwapFrom INFO [2019-12-17 12:36:02,508] jepsen worker 27 - jepsen.util 9837 :invoke :write [9 46] INFO [2019-12-17 12:36:02,514] jepsen worker 63 - jepsen.util 9333 :ok :write [21 7] INFO [2019-12-17 12:36:02,514] jepsen worker 63 - jepsen.util 9333 :invoke :read [21 nil] INFO [2019-12-17 12:36:02,516] jepsen worker 11 - jepsen.util 9821 :info :write [3 34] :RequestCanceledException INFO [2019-12-17 12:36:02,518] jepsen worker 11 - jepsen.util 9911 :invoke :read [3 nil] INFO [2019-12-17 12:36:02,528] jepsen worker 80 - jepsen.util 10070 :fail :cas [26 [2 46]] :ValueNotSwapFrom INFO [2019-12-17 12:36:02,528] jepsen worker 80 - jepsen.util 10070 :invoke :read [26 nil] INFO [2019-12-17 12:36:02,531] jepsen worker 52 - jepsen.util 10132 :info :write [17 19] :RequestCanceledException INFO [2019-12-17 12:36:02,533] jepsen worker 52 - jepsen.util 10222 :invoke :write [17 2] INFO [2019-12-17 12:36:02,542] jepsen worker 63 - jepsen.util 9333 :ok :read [21 7] INFO [2019-12-17 12:36:02,543] jepsen worker 63 - jepsen.util 9333 :invoke :write [21 49] INFO [2019-12-17 12:36:02,545] jepsen worker 46 - jepsen.util 10216 :info :write [15 45] :RequestCanceledException INFO [2019-12-17 12:36:02,546] jepsen worker 46 - jepsen.util 10306 :invoke :write [15 36] INFO [2019-12-17 12:36:02,562] jepsen worker 76 - jepsen.util 14206 :info :write [25 22] :RequestCanceledException INFO [2019-12-17 12:36:02,564] jepsen worker 76 - jepsen.util 14296 :invoke :write [25 47] INFO [2019-12-17 12:36:02,608] jepsen worker 27 - jepsen.util 9837 :ok :write [9 46] INFO [2019-12-17 12:36:02,608] jepsen worker 27 - jepsen.util 9837 :invoke :cas [9 [23 47]] INFO [2019-12-17 12:36:02,614] jepsen worker 19 - jepsen.util 10279 :info :write [6 36] :RequestCanceledException INFO [2019-12-17 12:36:02,619] jepsen worker 19 - jepsen.util 10369 :invoke :cas [6 [26 45]] INFO [2019-12-17 12:36:02,626] jepsen worker 63 - jepsen.util 9333 :ok :write [21 49] INFO [2019-12-17 12:36:02,626] jepsen worker 63 - jepsen.util 9333 :invoke :read [21 nil] INFO [2019-12-17 12:36:02,629] jepsen worker 27 - jepsen.util 9837 :fail :cas [9 [23 47]] :ValueNotSwapFrom INFO [2019-12-17 12:36:02,629] jepsen worker 27 - jepsen.util 9837 :invoke :read [9 nil] INFO [2019-12-17 12:36:02,631] jepsen worker 28 - jepsen.util 8398 :fail :read [9 nil] :RequestTimeoutException INFO [2019-12-17 12:36:02,631] jepsen worker 35 - jepsen.util 13985 :info :cas [11 [14 47]] :RequestCanceledException INFO [2019-12-17 12:36:02,631] jepsen worker 31 - jepsen.util 9121 :fail :read [10 nil] :RequestTimeoutException INFO [2019-12-17 12:36:02,631] jepsen worker 28 - jepsen.util 8398 :invoke :read [9 nil] INFO [2019-12-17 12:36:02,631] jepsen worker 31 - jepsen.util 9121 :invoke :read [10 nil] INFO [2019-12-17 12:36:02,633] jepsen worker 35 - jepsen.util 14075 :invoke :read [11 nil] INFO [2019-12-17 12:36:02,646] jepsen worker 44 - jepsen.util 14264 :info :cas [14 [7 22]] :RequestCanceledException INFO [2019-12-17 12:36:02,646] jepsen worker 71 - jepsen.util 11051 :info :write [23 11] :RequestCanceledException INFO [2019-12-17 12:36:02,647] jepsen worker 44 - jepsen.util 14354 :invoke :cas [14 [39 6]] INFO [2019-12-17 12:36:02,648] jepsen worker 71 - jepsen.util 11141 :invoke :write [23 12] INFO [2019-12-17 12:36:02,652] jepsen worker 19 - jepsen.util 10369 :fail :cas [6 [26 45]] :ValueNotSwapFrom INFO [2019-12-17 12:36:02,652] jepsen worker 19 - jepsen.util 10369 :invoke :cas [6 [48 28]] INFO [2019-12-17 12:36:02,667] jepsen worker 27 - jepsen.util 9837 :ok :read [9 46] INFO [2019-12-17 12:36:02,668] jepsen worker 27 - jepsen.util 9837 :invoke :read [9 nil] INFO [2019-12-17 12:36:02,673] jepsen worker 63 - jepsen.util 9333 :ok :read [21 49] INFO [2019-12-17 12:36:02,673] jepsen worker 63 - jepsen.util 9333 :invoke :read [21 nil] INFO [2019-12-17 12:36:02,683] jepsen worker 19 - jepsen.util 10369 :fail :cas [6 [48 28]] :ValueNotSwapFrom INFO [2019-12-17 12:36:02,683] jepsen worker 19 - jepsen.util 10369 :invoke :read [6 nil] INFO [2019-12-17 12:36:02,690] jepsen worker 27 - jepsen.util 9837 :ok :read [9 46] INFO [2019-12-17 12:36:02,690] jepsen worker 27 - jepsen.util 9837 :invoke :cas [9 [5 4]] INFO [2019-12-17 12:36:02,693] jepsen worker 63 - jepsen.util 9333 :ok :read [21 49] INFO [2019-12-17 12:36:02,693] jepsen worker 63 - jepsen.util 9333 :invoke :cas [21 [28 46]] INFO [2019-12-17 12:36:02,702] jepsen worker 19 - jepsen.util 10369 :ok :read [6 46] INFO [2019-12-17 12:36:02,702] jepsen worker 19 - jepsen.util 10369 :invoke :write [6 35] INFO [2019-12-17 12:36:02,711] jepsen worker 27 - jepsen.util 9837 :fail :cas [9 [5 4]] :ValueNotSwapFrom INFO [2019-12-17 12:36:02,711] jepsen worker 27 - jepsen.util 9837 :invoke :read [9 nil] INFO [2019-12-17 12:36:02,712] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":59290,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:36:02,712] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":59290,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:36:02,712] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":59294,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:36:02,712] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":59294,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:36:02,715] jepsen worker 63 - jepsen.util 9333 :fail :cas [21 [28 46]] :ValueNotSwapFrom INFO [2019-12-17 12:36:02,715] jepsen worker 63 - jepsen.util 9333 :invoke :cas [21 [11 0]] INFO [2019-12-17 12:36:02,720] jepsen worker 80 - jepsen.util 10070 :ok :read [26 15] INFO [2019-12-17 12:36:02,720] jepsen worker 80 - jepsen.util 10070 :invoke :cas [26 [43 0]] INFO [2019-12-17 12:36:02,731] jepsen worker 62 - jepsen.util 14372 :info :cas [20 [46 13]] :RequestCanceledException INFO [2019-12-17 12:36:02,733] jepsen worker 62 - jepsen.util 14462 :invoke :write [20 22] INFO [2019-12-17 12:36:02,739] jepsen worker 27 - jepsen.util 9837 :ok :read [9 46] INFO [2019-12-17 12:36:02,739] jepsen worker 27 - jepsen.util 9837 :invoke :write [9 29] INFO [2019-12-17 12:36:02,739] jepsen worker 63 - jepsen.util 9333 :fail :cas [21 [11 0]] :ValueNotSwapFrom INFO [2019-12-17 12:36:02,739] jepsen worker 63 - jepsen.util 9333 :invoke :read [21 nil] INFO [2019-12-17 12:36:02,754] jepsen worker 63 - jepsen.util 9333 :ok :read [21 49] INFO [2019-12-17 12:36:02,754] jepsen worker 63 - jepsen.util 9333 :invoke :write [21 30] INFO [2019-12-17 12:36:02,762] jepsen worker 14 - jepsen.util 10094 :info :write [4 1] :RequestCanceledException INFO [2019-12-17 12:36:02,763] jepsen worker 14 - jepsen.util 10184 :invoke :cas [4 [42 9]] INFO [2019-12-17 12:36:02,765] jepsen worker 30 - jepsen.util 10110 :ok :write [10 28] INFO [2019-12-17 12:36:02,765] jepsen worker 30 - jepsen.util 10110 :invoke :cas [10 [2 10]] INFO [2019-12-17 12:36:02,765] jepsen worker 74 - jepsen.util 10064 :ok :write [24 17] INFO [2019-12-17 12:36:02,765] jepsen worker 45 - jepsen.util 10395 :ok :write [15 20] INFO [2019-12-17 12:36:02,765] jepsen worker 74 - jepsen.util 10064 :invoke :cas [24 [41 46]] INFO [2019-12-17 12:36:02,765] jepsen worker 45 - jepsen.util 10395 :invoke :write [15 15] INFO [2019-12-17 12:36:02,771] jepsen worker 20 - jepsen.util 10190 :ok :write [6 42] INFO [2019-12-17 12:36:02,771] jepsen worker 20 - jepsen.util 10190 :invoke :write [6 32] INFO [2019-12-17 12:36:02,802] jepsen worker 70 - jepsen.util 11140 :ok :read [23 47] INFO [2019-12-17 12:36:02,803] jepsen worker 70 - jepsen.util 11140 :invoke :read [23 nil] INFO [2019-12-17 12:36:02,817] jepsen worker 27 - jepsen.util 9837 :ok :write [9 29] INFO [2019-12-17 12:36:02,818] jepsen worker 27 - jepsen.util 9837 :invoke :read [9 nil] INFO [2019-12-17 12:36:02,829] jepsen worker 36 - jepsen.util 11016 :info :write [12 36] :RequestCanceledException INFO [2019-12-17 12:36:02,831] jepsen worker 36 - jepsen.util 11106 :invoke :read [12 nil] INFO [2019-12-17 12:36:02,832] jepsen worker 9 - jepsen.util 11079 :ok :read [3 29] INFO [2019-12-17 12:36:02,833] jepsen worker 9 - jepsen.util 11079 :invoke :cas [3 [26 20]] INFO [2019-12-17 12:36:02,871] jepsen worker 32 - jepsen.util 10472 :fail :cas [10 [9 19]] :ValueNotSwapFrom INFO [2019-12-17 12:36:02,871] jepsen worker 32 - jepsen.util 10472 :invoke :write [10 32] INFO [2019-12-17 12:36:02,877] jepsen worker 64 - jepsen.util 7984 :fail :cas [21 [12 41]] :ValueNotSwapFrom INFO [2019-12-17 12:36:02,877] jepsen worker 64 - jepsen.util 7984 :invoke :write [21 8] INFO [2019-12-17 12:36:02,895] jepsen worker 5 - jepsen.util 9815 :info :write [1 4] :RequestCanceledException INFO [2019-12-17 12:36:02,895] jepsen worker 85 - jepsen.util 14845 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:36:02,895] jepsen worker 85 - jepsen.util 14845 :invoke :write [28 36] INFO [2019-12-17 12:36:02,896] jepsen worker 5 - jepsen.util 9905 :invoke :cas [1 [39 48]] INFO [2019-12-17 12:36:02,917] jepsen worker 40 - jepsen.util 14440 :info :write [13 40] :RequestCanceledException INFO [2019-12-17 12:36:02,918] jepsen worker 40 - jepsen.util 14530 :invoke :write [13 16] INFO [2019-12-17 12:36:02,930] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":59321,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:36:02,930] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":59321,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:36:02,930] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":59324,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:36:02,930] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":59324,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:36:02,930] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":59325,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:36:02,930] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":59325,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:36:02,935] jepsen worker 5 - jepsen.util 9905 :fail :cas [1 [39 48]] :ValueNotSwapFrom INFO [2019-12-17 12:36:02,936] jepsen worker 5 - jepsen.util 9905 :invoke :cas [1 [48 10]] INFO [2019-12-17 12:36:02,938] jepsen worker 23 - jepsen.util 13973 :info :cas [7 [12 49]] :RequestCanceledException INFO [2019-12-17 12:36:02,938] jepsen worker 54 - jepsen.util 11214 :info :write [18 23] :RequestCanceledException INFO [2019-12-17 12:36:02,939] jepsen worker 27 - jepsen.util 9837 :ok :read [9 29] INFO [2019-12-17 12:36:02,939] jepsen worker 27 - jepsen.util 9837 :invoke :write [9 20] INFO [2019-12-17 12:36:02,940] jepsen worker 23 - jepsen.util 14063 :invoke :write [7 0] INFO [2019-12-17 12:36:02,940] jepsen worker 54 - jepsen.util 11304 :invoke :write [18 5] INFO [2019-12-17 12:36:02,940] jepsen worker 4 - jepsen.util 9634 :fail :cas [1 [48 26]] :ValueNotSwapFrom INFO [2019-12-17 12:36:02,941] jepsen worker 4 - jepsen.util 9634 :invoke :read [1 nil] INFO [2019-12-17 12:36:02,949] jepsen worker 65 - jepsen.util 8795 :fail :cas [21 [29 12]] :ValueNotSwapFrom INFO [2019-12-17 12:36:02,949] jepsen worker 65 - jepsen.util 8795 :invoke :cas [21 [6 9]] INFO [2019-12-17 12:36:02,968] jepsen worker 74 - jepsen.util 10064 :fail :cas [24 [41 46]] :ValueNotSwapFrom INFO [2019-12-17 12:36:02,968] jepsen worker 74 - jepsen.util 10064 :invoke :cas [24 [45 37]] INFO [2019-12-17 12:36:02,968] jepsen worker 31 - jepsen.util 9121 :ok :read [10 28] INFO [2019-12-17 12:36:02,969] jepsen worker 31 - jepsen.util 9121 :invoke :cas [10 [10 32]] INFO [2019-12-17 12:36:02,979] jepsen worker 30 - jepsen.util 10110 :fail :cas [10 [2 10]] :ValueNotSwapFrom INFO [2019-12-17 12:36:02,979] jepsen worker 30 - jepsen.util 10110 :invoke :write [10 7] INFO [2019-12-17 12:36:02,998] jepsen worker 88 - jepsen.util 10618 :info :write [29 0] :RequestCanceledException INFO [2019-12-17 12:36:02,999] jepsen worker 88 - jepsen.util 10708 :invoke :write [29 35] INFO [2019-12-17 12:36:03,010] jepsen worker 80 - jepsen.util 10070 :fail :cas [26 [43 0]] :ValueNotSwapFrom INFO [2019-12-17 12:36:03,010] jepsen worker 80 - jepsen.util 10070 :invoke :read [26 nil] INFO [2019-12-17 12:36:03,012] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:36:03,012] jepsen nemesis - couchbase.util Rebalance check count: 464 INFO [2019-12-17 12:36:03,012] jepsen nemesis - couchbase.util Rebalance stuck count: 445 INFO [2019-12-17 12:36:03,014] jepsen worker 63 - jepsen.util 9333 :ok :write [21 30] INFO [2019-12-17 12:36:03,014] jepsen worker 63 - jepsen.util 9333 :invoke :cas [21 [23 2]] INFO [2019-12-17 12:36:03,031] jepsen worker 83 - jepsen.util 14123 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:36:03,031] jepsen worker 49 - jepsen.util 13999 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:36:03,031] jepsen worker 66 - jepsen.util 13746 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:36:03,031] jepsen worker 55 - jepsen.util 10945 :info :write [18 20] :RequestCanceledException INFO [2019-12-17 12:36:03,032] jepsen worker 49 - jepsen.util 13999 :invoke :read [16 nil] INFO [2019-12-17 12:36:03,032] jepsen worker 83 - jepsen.util 14123 :invoke :cas [27 [20 8]] INFO [2019-12-17 12:36:03,032] jepsen worker 66 - jepsen.util 13746 :invoke :read [22 nil] INFO [2019-12-17 12:36:03,032] jepsen worker 55 - jepsen.util 11035 :invoke :read [18 nil] INFO [2019-12-17 12:36:03,059] jepsen worker 3 - jepsen.util 10083 :ok :write [1 33] INFO [2019-12-17 12:36:03,059] jepsen worker 63 - jepsen.util 9333 :fail :cas [21 [23 2]] :ValueNotSwapFrom INFO [2019-12-17 12:36:03,059] jepsen worker 3 - jepsen.util 10083 :invoke :cas [1 [30 28]] INFO [2019-12-17 12:36:03,059] jepsen worker 63 - jepsen.util 9333 :invoke :cas [21 [32 45]] INFO [2019-12-17 12:36:03,060] jepsen worker 79 - jepsen.util 10609 :info :write [26 40] :RequestCanceledException INFO [2019-12-17 12:36:03,060] jepsen worker 78 - jepsen.util 10158 :info :write [26 31] :RequestCanceledException INFO [2019-12-17 12:36:03,060] jepsen worker 51 - jepsen.util 9681 :info :write [17 34] :RequestCanceledException INFO [2019-12-17 12:36:03,061] jepsen worker 79 - jepsen.util 10699 :invoke :cas [26 [12 33]] INFO [2019-12-17 12:36:03,061] jepsen worker 51 - jepsen.util 9771 :invoke :write [17 9] INFO [2019-12-17 12:36:03,061] jepsen worker 78 - jepsen.util 10248 :invoke :cas [26 [47 27]] INFO [2019-12-17 12:36:03,079] jepsen worker 63 - jepsen.util 9333 :fail :cas [21 [32 45]] :ValueNotSwapFrom INFO [2019-12-17 12:36:03,079] jepsen worker 63 - jepsen.util 9333 :invoke :cas [21 [28 2]] INFO [2019-12-17 12:36:03,095] jepsen worker 7 - jepsen.util 14137 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:36:03,095] jepsen worker 7 - jepsen.util 14137 :invoke :cas [2 [43 48]] INFO [2019-12-17 12:36:03,100] jepsen worker 63 - jepsen.util 9333 :fail :cas [21 [28 2]] :ValueNotSwapFrom INFO [2019-12-17 12:36:03,100] jepsen worker 63 - jepsen.util 9333 :invoke :cas [21 [29 2]] INFO [2019-12-17 12:36:03,102] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":59341,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:36:03,102] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":59341,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:36:03,102] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":59345,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:36:03,102] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":59345,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:36:03,129] jepsen worker 27 - jepsen.util 9837 :ok :write [9 20] INFO [2019-12-17 12:36:03,129] jepsen worker 27 - jepsen.util 9837 :invoke :write [9 19] INFO [2019-12-17 12:36:03,131] jepsen worker 87 - jepsen.util 9807 :info :write [29 41] :RequestCanceledException INFO [2019-12-17 12:36:03,132] jepsen worker 26 - jepsen.util 14606 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:36:03,132] jepsen worker 26 - jepsen.util 14606 :invoke :write [8 43] INFO [2019-12-17 12:36:03,133] jepsen worker 87 - jepsen.util 9897 :invoke :write [29 2] INFO [2019-12-17 12:36:03,135] jepsen worker 63 - jepsen.util 9333 :fail :cas [21 [29 2]] :ValueNotSwapFrom INFO [2019-12-17 12:36:03,135] jepsen worker 63 - jepsen.util 9333 :invoke :write [21 16] INFO [2019-12-17 12:36:03,138] jepsen worker 64 - jepsen.util 7984 :ok :write [21 8] INFO [2019-12-17 12:36:03,138] jepsen worker 0 - jepsen.util 14760 :info :cas [0 [36 34]] :RequestCanceledException INFO [2019-12-17 12:36:03,138] jepsen worker 64 - jepsen.util 7984 :invoke :cas [21 [26 0]] INFO [2019-12-17 12:36:03,141] jepsen worker 0 - jepsen.util 14850 :invoke :read [0 nil] INFO [2019-12-17 12:36:03,143] jepsen worker 55 - jepsen.util 11035 :ok :read [18 20] INFO [2019-12-17 12:36:03,143] jepsen worker 55 - jepsen.util 11035 :invoke :read [18 nil] INFO [2019-12-17 12:36:03,153] jepsen worker 14 - jepsen.util 10184 :fail :cas [4 [42 9]] :ValueNotSwapFrom INFO [2019-12-17 12:36:03,153] jepsen worker 14 - jepsen.util 10184 :invoke :cas [4 [2 34]] INFO [2019-12-17 12:36:03,163] jepsen worker 61 - jepsen.util 14641 :info :cas [20 [37 41]] :RequestCanceledException INFO [2019-12-17 12:36:03,164] jepsen worker 61 - jepsen.util 14731 :invoke :write [20 27] INFO [2019-12-17 12:36:03,165] jepsen worker 5 - jepsen.util 9905 :fail :cas [1 [48 10]] :ValueNotSwapFrom INFO [2019-12-17 12:36:03,166] jepsen worker 5 - jepsen.util 9905 :invoke :write [1 16] INFO [2019-12-17 12:36:03,179] jepsen worker 64 - jepsen.util 7984 :fail :cas [21 [26 0]] :ValueNotSwapFrom INFO [2019-12-17 12:36:03,179] jepsen worker 64 - jepsen.util 7984 :invoke :cas [21 [20 16]] INFO [2019-12-17 12:36:03,193] jepsen worker 64 - jepsen.util 7984 :fail :cas [21 [20 16]] :ValueNotSwapFrom INFO [2019-12-17 12:36:03,193] jepsen worker 64 - jepsen.util 7984 :invoke :cas [21 [36 5]] INFO [2019-12-17 12:36:03,210] jepsen worker 64 - jepsen.util 7984 :fail :cas [21 [36 5]] :ValueNotSwapFrom INFO [2019-12-17 12:36:03,210] jepsen worker 64 - jepsen.util 7984 :invoke :cas [21 [41 30]] INFO [2019-12-17 12:36:03,217] jepsen worker 38 - jepsen.util 11018 :info :write [12 13] :RequestCanceledException INFO [2019-12-17 12:36:03,217] jepsen worker 38 - jepsen.util 11108 :invoke :write [12 13] INFO [2019-12-17 12:36:03,226] jepsen worker 64 - jepsen.util 7984 :fail :cas [21 [41 30]] :ValueNotSwapFrom INFO [2019-12-17 12:36:03,226] jepsen worker 64 - jepsen.util 7984 :invoke :cas [21 [19 39]] INFO [2019-12-17 12:36:03,231] jepsen worker 1 - jepsen.util 13951 :info :cas [0 [34 15]] :RequestCanceledException INFO [2019-12-17 12:36:03,232] jepsen worker 1 - jepsen.util 14041 :invoke :write [0 2] INFO [2019-12-17 12:36:03,241] jepsen worker 79 - jepsen.util 10699 :fail :cas [26 [12 33]] :ValueNotSwapFrom INFO [2019-12-17 12:36:03,241] jepsen worker 79 - jepsen.util 10699 :invoke :cas [26 [8 48]] INFO [2019-12-17 12:36:03,243] jepsen worker 64 - jepsen.util 7984 :fail :cas [21 [19 39]] :ValueNotSwapFrom INFO [2019-12-17 12:36:03,243] jepsen worker 64 - jepsen.util 7984 :invoke :read [21 nil] INFO [2019-12-17 12:36:03,246] jepsen worker 27 - jepsen.util 9837 :ok :write [9 19] INFO [2019-12-17 12:36:03,246] jepsen worker 27 - jepsen.util 9837 :invoke :write [9 45] INFO [2019-12-17 12:36:03,257] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":59381,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:36:03,257] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":59381,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:36:03,257] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":59384,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:36:03,257] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":59384,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:36:03,257] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":59385,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:36:03,257] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":59385,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:36:03,259] jepsen worker 64 - jepsen.util 7984 :ok :read [21 8] INFO [2019-12-17 12:36:03,259] jepsen worker 64 - jepsen.util 7984 :invoke :cas [21 [18 46]] INFO [2019-12-17 12:36:03,259] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":59373,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:36:03,259] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":59373,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:36:03,259] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":59374,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:36:03,259] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":59374,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:36:03,267] jepsen worker 70 - jepsen.util 11140 :ok :read [23 47] INFO [2019-12-17 12:36:03,267] jepsen worker 70 - jepsen.util 11140 :invoke :cas [23 [8 42]] INFO [2019-12-17 12:36:03,296] jepsen worker 36 - jepsen.util 11106 :ok :read [12 39] INFO [2019-12-17 12:36:03,296] jepsen worker 36 - jepsen.util 11106 :invoke :read [12 nil] INFO [2019-12-17 12:36:03,298] jepsen worker 64 - jepsen.util 7984 :fail :cas [21 [18 46]] :ValueNotSwapFrom INFO [2019-12-17 12:36:03,299] jepsen worker 64 - jepsen.util 7984 :invoke :cas [21 [19 47]] INFO [2019-12-17 12:36:03,324] jepsen worker 64 - jepsen.util 7984 :fail :cas [21 [19 47]] :ValueNotSwapFrom INFO [2019-12-17 12:36:03,324] jepsen worker 64 - jepsen.util 7984 :invoke :read [21 nil] INFO [2019-12-17 12:36:03,331] jepsen worker 12 - jepsen.util 10722 :info :write [4 6] :RequestCanceledException INFO [2019-12-17 12:36:03,331] jepsen worker 25 - jepsen.util 14965 :info :cas [8 [39 24]] :RequestCanceledException INFO [2019-12-17 12:36:03,332] jepsen worker 12 - jepsen.util 10812 :invoke :write [4 49] INFO [2019-12-17 12:36:03,336] jepsen worker 63 - jepsen.util 9333 :ok :write [21 16] INFO [2019-12-17 12:36:03,336] jepsen worker 63 - jepsen.util 9333 :invoke :write [21 14] INFO [2019-12-17 12:36:03,337] jepsen worker 25 - jepsen.util 15055 :invoke :write [8 23] INFO [2019-12-17 12:36:03,350] jepsen worker 64 - jepsen.util 7984 :ok :read [21 16] INFO [2019-12-17 12:36:03,350] jepsen worker 64 - jepsen.util 7984 :invoke :cas [21 [31 20]] INFO [2019-12-17 12:36:03,353] jepsen worker 4 - jepsen.util 9634 :ok :read [1 33] INFO [2019-12-17 12:36:03,354] jepsen worker 4 - jepsen.util 9634 :invoke :read [1 nil] INFO [2019-12-17 12:36:03,362] jepsen worker 75 - jepsen.util 13485 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:36:03,362] jepsen worker 75 - jepsen.util 13485 :invoke :cas [25 [27 11]] INFO [2019-12-17 12:36:03,367] jepsen worker 32 - jepsen.util 10472 :ok :write [10 32] INFO [2019-12-17 12:36:03,367] jepsen worker 32 - jepsen.util 10472 :invoke :read [10 nil] INFO [2019-12-17 12:36:03,369] jepsen worker 46 - jepsen.util 10306 :ok :write [15 36] INFO [2019-12-17 12:36:03,370] jepsen worker 46 - jepsen.util 10306 :invoke :write [15 31] INFO [2019-12-17 12:36:03,394] jepsen worker 68 - jepsen.util 13478 :info :write [22 12] :RequestCanceledException INFO [2019-12-17 12:36:03,394] jepsen worker 58 - jepsen.util 14458 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:36:03,394] jepsen worker 58 - jepsen.util 14458 :invoke :read [19 nil] INFO [2019-12-17 12:36:03,394] jepsen worker 68 - jepsen.util 13568 :invoke :read [22 nil] INFO [2019-12-17 12:36:03,398] jepsen worker 80 - jepsen.util 10070 :ok :read [26 15] INFO [2019-12-17 12:36:03,398] jepsen worker 80 - jepsen.util 10070 :invoke :write [26 27] INFO [2019-12-17 12:36:03,415] jepsen worker 64 - jepsen.util 7984 :fail :cas [21 [31 20]] :ValueNotSwapFrom INFO [2019-12-17 12:36:03,415] jepsen worker 64 - jepsen.util 7984 :invoke :write [21 8] INFO [2019-12-17 12:36:03,416] jepsen worker 37 - jepsen.util 9847 :info :write [12 4] :RequestCanceledException INFO [2019-12-17 12:36:03,417] jepsen worker 37 - jepsen.util 9937 :invoke :cas [12 [19 33]] INFO [2019-12-17 12:36:03,421] jepsen worker 31 - jepsen.util 9121 :fail :cas [10 [10 32]] :ValueNotSwapFrom INFO [2019-12-17 12:36:03,421] jepsen worker 31 - jepsen.util 9121 :invoke :read [10 nil] INFO [2019-12-17 12:36:03,434] jepsen worker 27 - jepsen.util 9837 :ok :write [9 45] INFO [2019-12-17 12:36:03,434] jepsen worker 27 - jepsen.util 9837 :invoke :read [9 nil] INFO [2019-12-17 12:36:03,450] jepsen worker 14 - jepsen.util 10184 :fail :cas [4 [2 34]] :ValueNotSwapFrom INFO [2019-12-17 12:36:03,450] jepsen worker 14 - jepsen.util 10184 :invoke :write [4 18] INFO [2019-12-17 12:36:03,452] jepsen worker 70 - jepsen.util 11140 :fail :cas [23 [8 42]] :ValueNotSwapFrom INFO [2019-12-17 12:36:03,452] jepsen worker 70 - jepsen.util 11140 :invoke :cas [23 [41 15]] INFO [2019-12-17 12:36:03,452] jepsen worker 27 - jepsen.util 9837 :ok :read [9 45] INFO [2019-12-17 12:36:03,452] jepsen worker 27 - jepsen.util 9837 :invoke :read [9 nil] INFO [2019-12-17 12:36:03,463] jepsen worker 82 - jepsen.util 14212 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:36:03,463] jepsen worker 82 - jepsen.util 14212 :invoke :cas [27 [30 37]] INFO [2019-12-17 12:36:03,472] jepsen worker 27 - jepsen.util 9837 :ok :read [9 45] INFO [2019-12-17 12:36:03,473] jepsen worker 27 - jepsen.util 9837 :invoke :read [9 nil] INFO [2019-12-17 12:36:03,474] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":59411,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:36:03,474] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":59411,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:36:03,474] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":59412,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:36:03,474] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":59412,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:36:03,489] jepsen worker 36 - jepsen.util 11106 :ok :read [12 39] INFO [2019-12-17 12:36:03,489] jepsen worker 36 - jepsen.util 11106 :invoke :cas [12 [31 40]] INFO [2019-12-17 12:36:03,490] jepsen worker 63 - jepsen.util 9333 :ok :write [21 14] INFO [2019-12-17 12:36:03,490] jepsen worker 63 - jepsen.util 9333 :invoke :read [21 nil] INFO [2019-12-17 12:36:03,497] jepsen worker 27 - jepsen.util 9837 :ok :read [9 45] INFO [2019-12-17 12:36:03,497] jepsen worker 27 - jepsen.util 9837 :invoke :read [9 nil] INFO [2019-12-17 12:36:03,504] jepsen worker 63 - jepsen.util 9333 :ok :read [21 14] INFO [2019-12-17 12:36:03,504] jepsen worker 63 - jepsen.util 9333 :invoke :read [21 nil] INFO [2019-12-17 12:36:03,515] jepsen worker 27 - jepsen.util 9837 :ok :read [9 45] INFO [2019-12-17 12:36:03,516] jepsen worker 27 - jepsen.util 9837 :invoke :cas [9 [4 33]] INFO [2019-12-17 12:36:03,516] jepsen worker 73 - jepsen.util 9703 :fail :read [24 nil] :RequestTimeoutException INFO [2019-12-17 12:36:03,516] jepsen worker 73 - jepsen.util 9703 :invoke :cas [24 [4 11]] INFO [2019-12-17 12:36:03,527] jepsen worker 4 - jepsen.util 9634 :ok :read [1 33] INFO [2019-12-17 12:36:03,527] jepsen worker 4 - jepsen.util 9634 :invoke :cas [1 [28 36]] INFO [2019-12-17 12:36:03,531] jepsen worker 72 - jepsen.util 10422 :info :cas [24 [10 17]] :RequestCanceledException INFO [2019-12-17 12:36:03,532] jepsen worker 72 - jepsen.util 10512 :invoke :read [24 nil] INFO [2019-12-17 12:36:03,540] jepsen worker 63 - jepsen.util 9333 :ok :read [21 14] INFO [2019-12-17 12:36:03,540] jepsen worker 63 - jepsen.util 9333 :invoke :read [21 nil] INFO [2019-12-17 12:36:03,551] jepsen worker 27 - jepsen.util 9837 :fail :cas [9 [4 33]] :ValueNotSwapFrom INFO [2019-12-17 12:36:03,551] jepsen worker 27 - jepsen.util 9837 :invoke :read [9 nil] INFO [2019-12-17 12:36:03,556] jepsen worker 73 - jepsen.util 9703 :fail :cas [24 [4 11]] :ValueNotSwapFrom INFO [2019-12-17 12:36:03,556] jepsen worker 73 - jepsen.util 9703 :invoke :write [24 42] INFO [2019-12-17 12:36:03,560] jepsen worker 55 - jepsen.util 11035 :ok :read [18 20] INFO [2019-12-17 12:36:03,560] jepsen worker 55 - jepsen.util 11035 :invoke :cas [18 [35 44]] INFO [2019-12-17 12:36:03,575] jepsen worker 78 - jepsen.util 10248 :fail :cas [26 [47 27]] :ValueNotSwapFrom INFO [2019-12-17 12:36:03,575] jepsen worker 78 - jepsen.util 10248 :invoke :read [26 nil] INFO [2019-12-17 12:36:03,633] jepsen worker 63 - jepsen.util 9333 :ok :read [21 14] INFO [2019-12-17 12:36:03,633] jepsen worker 63 - jepsen.util 9333 :invoke :cas [21 [32 20]] INFO [2019-12-17 12:36:03,645] jepsen worker 47 - jepsen.util 10847 :info :cas [15 [10 39]] :RequestCanceledException INFO [2019-12-17 12:36:03,645] jepsen worker 18 - jepsen.util 10638 :info :cas [6 [43 21]] :RequestCanceledException INFO [2019-12-17 12:36:03,645] jepsen worker 53 - jepsen.util 9683 :info :write [17 3] :RequestCanceledException INFO [2019-12-17 12:36:03,646] jepsen worker 18 - jepsen.util 10728 :invoke :write [6 14] INFO [2019-12-17 12:36:03,646] jepsen worker 53 - jepsen.util 9773 :invoke :cas [17 [34 19]] INFO [2019-12-17 12:36:03,647] jepsen worker 20 - jepsen.util 10190 :ok :write [6 32] INFO [2019-12-17 12:36:03,648] jepsen worker 20 - jepsen.util 10190 :invoke :read [6 nil] INFO [2019-12-17 12:36:03,649] jepsen worker 47 - jepsen.util 10937 :invoke :write [15 27] INFO [2019-12-17 12:36:03,662] jepsen worker 48 - jepsen.util 12828 :info :write [16 2] :RequestCanceledException INFO [2019-12-17 12:36:03,666] jepsen worker 48 - jepsen.util 12918 :invoke :cas [16 [43 32]] INFO [2019-12-17 12:36:03,670] jepsen worker 27 - jepsen.util 9837 :ok :read [9 45] INFO [2019-12-17 12:36:03,670] jepsen worker 27 - jepsen.util 9837 :invoke :cas [9 [25 20]] INFO [2019-12-17 12:36:03,671] jepsen worker 30 - jepsen.util 10110 :ok :write [10 7] INFO [2019-12-17 12:36:03,671] jepsen worker 30 - jepsen.util 10110 :invoke :cas [10 [27 44]] INFO [2019-12-17 12:36:03,678] jepsen worker 52 - jepsen.util 10222 :ok :write [17 2] INFO [2019-12-17 12:36:03,678] jepsen worker 52 - jepsen.util 10222 :invoke :read [17 nil] INFO [2019-12-17 12:36:03,680] jepsen worker 9 - jepsen.util 11079 :fail :cas [3 [26 20]] :ValueNotSwapFrom INFO [2019-12-17 12:36:03,680] jepsen worker 9 - jepsen.util 11079 :invoke :read [3 nil] INFO [2019-12-17 12:36:03,693] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":59441,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:36:03,693] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":59441,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:36:03,693] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":59443,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:36:03,693] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":59443,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:36:03,698] jepsen worker 33 - jepsen.util 14073 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:36:03,698] jepsen worker 33 - jepsen.util 14073 :invoke :cas [11 [48 46]] INFO [2019-12-17 12:36:03,714] jepsen worker 81 - jepsen.util 13671 :info :write [27 25] :RequestCanceledException INFO [2019-12-17 12:36:03,715] jepsen worker 81 - jepsen.util 13761 :invoke :read [27 nil] INFO [2019-12-17 12:36:03,725] jepsen worker 32 - jepsen.util 10472 :ok :read [10 7] INFO [2019-12-17 12:36:03,725] jepsen worker 32 - jepsen.util 10472 :invoke :write [10 24] INFO [2019-12-17 12:36:03,728] jepsen worker 38 - jepsen.util 11108 :ok :write [12 13] INFO [2019-12-17 12:36:03,728] jepsen worker 38 - jepsen.util 11108 :invoke :read [12 nil] INFO [2019-12-17 12:36:03,731] jepsen worker 39 - jepsen.util 13629 :info :write [13 16] :RequestCanceledException INFO [2019-12-17 12:36:03,731] jepsen worker 13 - jepsen.util 9643 :info :write [4 13] :RequestCanceledException INFO [2019-12-17 12:36:03,734] jepsen worker 39 - jepsen.util 13719 :invoke :read [13 nil] INFO [2019-12-17 12:36:03,734] jepsen worker 13 - jepsen.util 9733 :invoke :write [4 19] INFO [2019-12-17 12:36:03,770] jepsen worker 31 - jepsen.util 9121 :ok :read [10 7] INFO [2019-12-17 12:36:03,770] jepsen worker 31 - jepsen.util 9121 :invoke :read [10 nil] INFO [2019-12-17 12:36:03,805] jepsen worker 65 - jepsen.util 8795 :fail :cas [21 [6 9]] :ValueNotSwapFrom INFO [2019-12-17 12:36:03,805] jepsen worker 65 - jepsen.util 8795 :invoke :cas [21 [34 36]] INFO [2019-12-17 12:36:03,831] jepsen worker 72 - jepsen.util 10512 :ok :read [24 17] INFO [2019-12-17 12:36:03,831] jepsen worker 72 - jepsen.util 10512 :invoke :cas [24 [14 20]] INFO [2019-12-17 12:36:03,846] jepsen worker 10 - jepsen.util 10270 :info :cas [3 [49 19]] :RequestCanceledException INFO [2019-12-17 12:36:03,847] jepsen worker 10 - jepsen.util 10360 :invoke :read [3 nil] INFO [2019-12-17 12:36:03,863] jepsen worker 77 - jepsen.util 14747 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:36:03,863] jepsen worker 77 - jepsen.util 14747 :invoke :write [25 28] INFO [2019-12-17 12:36:03,866] jepsen worker 56 - jepsen.util 10766 :ok :write [18 1] INFO [2019-12-17 12:36:03,866] jepsen worker 56 - jepsen.util 10766 :invoke :write [18 25] INFO [2019-12-17 12:36:03,871] jepsen worker 30 - jepsen.util 10110 :fail :cas [10 [27 44]] :ValueNotSwapFrom INFO [2019-12-17 12:36:03,871] jepsen worker 30 - jepsen.util 10110 :invoke :cas [10 [29 20]] INFO [2019-12-17 12:36:03,896] jepsen worker 10 - jepsen.util 10360 :ok :read [3 29] INFO [2019-12-17 12:36:03,896] jepsen worker 10 - jepsen.util 10360 :invoke :cas [3 [4 23]] INFO [2019-12-17 12:36:03,905] jepsen worker 13 - jepsen.util 9733 :ok :write [4 19] INFO [2019-12-17 12:36:03,905] jepsen worker 13 - jepsen.util 9733 :invoke :write [4 10] INFO [2019-12-17 12:36:03,913] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":59463,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:36:03,913] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":59463,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:36:03,913] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":59467,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:36:03,913] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":59467,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:36:03,916] jepsen worker 21 - jepsen.util 13611 :info :cas [7 [28 15]] :RequestCanceledException INFO [2019-12-17 12:36:03,916] jepsen worker 21 - jepsen.util 13701 :invoke :cas [7 [5 14]] INFO [2019-12-17 12:36:03,917] jepsen worker 10 - jepsen.util 10360 :fail :cas [3 [4 23]] :ValueNotSwapFrom INFO [2019-12-17 12:36:03,917] jepsen worker 10 - jepsen.util 10360 :invoke :cas [3 [4 35]] INFO [2019-12-17 12:36:03,923] jepsen worker 36 - jepsen.util 11106 :fail :cas [12 [31 40]] :ValueNotSwapFrom INFO [2019-12-17 12:36:03,923] jepsen worker 36 - jepsen.util 11106 :invoke :write [12 27] INFO [2019-12-17 12:36:03,932] jepsen worker 59 - jepsen.util 13199 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:36:03,932] jepsen worker 59 - jepsen.util 13199 :invoke :cas [19 [24 24]] INFO [2019-12-17 12:36:03,947] jepsen worker 10 - jepsen.util 10360 :fail :cas [3 [4 35]] :ValueNotSwapFrom INFO [2019-12-17 12:36:03,947] jepsen worker 10 - jepsen.util 10360 :invoke :read [3 nil] INFO [2019-12-17 12:36:03,992] jepsen worker 10 - jepsen.util 10360 :ok :read [3 29] INFO [2019-12-17 12:36:03,992] jepsen worker 10 - jepsen.util 10360 :invoke :read [3 nil] INFO [2019-12-17 12:36:03,999] jepsen worker 79 - jepsen.util 10699 :fail :cas [26 [8 48]] :ValueNotSwapFrom INFO [2019-12-17 12:36:03,999] jepsen worker 79 - jepsen.util 10699 :invoke :write [26 47] INFO [2019-12-17 12:36:04,025] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:36:04,025] jepsen nemesis - couchbase.util Rebalance check count: 465 INFO [2019-12-17 12:36:04,025] jepsen nemesis - couchbase.util Rebalance stuck count: 446 INFO [2019-12-17 12:36:04,030] jepsen worker 54 - jepsen.util 11304 :ok :write [18 5] INFO [2019-12-17 12:36:04,030] jepsen worker 54 - jepsen.util 11304 :invoke :cas [18 [39 16]] INFO [2019-12-17 12:36:04,031] jepsen worker 29 - jepsen.util 8669 :info :write [9 16] :RequestCanceledException INFO [2019-12-17 12:36:04,032] jepsen worker 29 - jepsen.util 8759 :invoke :cas [9 [20 21]] INFO [2019-12-17 12:36:04,045] jepsen worker 37 - jepsen.util 9937 :fail :cas [12 [19 33]] :ValueNotSwapFrom INFO [2019-12-17 12:36:04,045] jepsen worker 37 - jepsen.util 9937 :invoke :write [12 24] INFO [2019-12-17 12:36:04,045] jepsen worker 86 - jepsen.util 15116 :info :cas [28 [48 14]] :RequestCanceledException INFO [2019-12-17 12:36:04,045] jepsen worker 34 - jepsen.util 14164 :info :cas [11 [27 41]] :RequestCanceledException INFO [2019-12-17 12:36:04,046] jepsen worker 34 - jepsen.util 14254 :invoke :write [11 5] INFO [2019-12-17 12:36:04,046] jepsen worker 86 - jepsen.util 15206 :invoke :write [28 4] INFO [2019-12-17 12:36:04,047] jepsen worker 10 - jepsen.util 10360 :ok :read [3 29] INFO [2019-12-17 12:36:04,047] jepsen worker 10 - jepsen.util 10360 :invoke :read [3 nil] INFO [2019-12-17 12:36:04,047] jepsen worker 70 - jepsen.util 11140 :fail :cas [23 [41 15]] :ValueNotSwapFrom INFO [2019-12-17 12:36:04,047] jepsen worker 70 - jepsen.util 11140 :invoke :read [23 nil] INFO [2019-12-17 12:36:04,049] jepsen worker 29 - jepsen.util 8759 :fail :cas [9 [20 21]] :ValueNotSwapFrom INFO [2019-12-17 12:36:04,049] jepsen worker 29 - jepsen.util 8759 :invoke :read [9 nil] INFO [2019-12-17 12:36:04,062] jepsen worker 84 - jepsen.util 13674 :info :cas [28 [8 6]] :RequestCanceledException INFO [2019-12-17 12:36:04,063] jepsen worker 84 - jepsen.util 13764 :invoke :write [28 28] INFO [2019-12-17 12:36:04,072] jepsen worker 3 - jepsen.util 10083 :fail :cas [1 [30 28]] :ValueNotSwapFrom INFO [2019-12-17 12:36:04,072] jepsen worker 3 - jepsen.util 10083 :invoke :read [1 nil] INFO [2019-12-17 12:36:04,076] jepsen worker 55 - jepsen.util 11035 :fail :cas [18 [35 44]] :ValueNotSwapFrom INFO [2019-12-17 12:36:04,076] jepsen worker 55 - jepsen.util 11035 :invoke :cas [18 [39 23]] INFO [2019-12-17 12:36:04,083] jepsen worker 29 - jepsen.util 8759 :ok :read [9 45] INFO [2019-12-17 12:36:04,083] jepsen worker 10 - jepsen.util 10360 :ok :read [3 29] INFO [2019-12-17 12:36:04,083] jepsen worker 29 - jepsen.util 8759 :invoke :cas [9 [3 31]] INFO [2019-12-17 12:36:04,083] jepsen worker 10 - jepsen.util 10360 :invoke :cas [3 [20 8]] INFO [2019-12-17 12:36:04,089] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":59480,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:36:04,089] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":59480,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:36:04,089] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":59483,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:36:04,089] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":59483,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:36:04,098] jepsen worker 16 - jepsen.util 15406 :info :write [5 9] :RequestCanceledException INFO [2019-12-17 12:36:04,098] jepsen worker 16 - jepsen.util 15496 :invoke :cas [5 [6 19]] INFO [2019-12-17 12:36:04,101] jepsen worker 4 - jepsen.util 9634 :fail :cas [1 [28 36]] :ValueNotSwapFrom INFO [2019-12-17 12:36:04,101] jepsen worker 4 - jepsen.util 9634 :invoke :cas [1 [14 22]] INFO [2019-12-17 12:36:04,108] jepsen worker 29 - jepsen.util 8759 :fail :cas [9 [3 31]] :ValueNotSwapFrom INFO [2019-12-17 12:36:04,108] jepsen worker 29 - jepsen.util 8759 :invoke :write [9 11] INFO [2019-12-17 12:36:04,114] jepsen worker 10 - jepsen.util 10360 :fail :cas [3 [20 8]] :ValueNotSwapFrom INFO [2019-12-17 12:36:04,114] jepsen worker 10 - jepsen.util 10360 :invoke :read [3 nil] INFO [2019-12-17 12:36:04,114] jepsen worker 8 - jepsen.util 13598 :info :cas [2 [25 16]] :RequestCanceledException INFO [2019-12-17 12:36:04,114] jepsen worker 43 - jepsen.util 13993 :info :write [14 43] :RequestCanceledException INFO [2019-12-17 12:36:04,115] jepsen worker 8 - jepsen.util 13688 :invoke :read [2 nil] INFO [2019-12-17 12:36:04,115] jepsen worker 43 - jepsen.util 14083 :invoke :read [14 nil] INFO [2019-12-17 12:36:04,123] jepsen worker 78 - jepsen.util 10248 :ok :read [26 15] INFO [2019-12-17 12:36:04,123] jepsen worker 78 - jepsen.util 10248 :invoke :cas [26 [37 39]] INFO [2019-12-17 12:36:04,132] jepsen worker 22 - jepsen.util 13792 :info :write [7 23] :RequestCanceledException INFO [2019-12-17 12:36:04,133] jepsen worker 22 - jepsen.util 13882 :invoke :read [7 nil] INFO [2019-12-17 12:36:04,137] jepsen worker 53 - jepsen.util 9773 :fail :cas [17 [34 19]] :ValueNotSwapFrom INFO [2019-12-17 12:36:04,137] jepsen worker 53 - jepsen.util 9773 :invoke :cas [17 [41 41]] INFO [2019-12-17 12:36:04,138] jepsen worker 10 - jepsen.util 10360 :ok :read [3 29] INFO [2019-12-17 12:36:04,138] jepsen worker 10 - jepsen.util 10360 :invoke :write [3 33] INFO [2019-12-17 12:36:04,146] jepsen worker 50 - jepsen.util 14720 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:36:04,146] jepsen worker 50 - jepsen.util 14720 :invoke :read [16 nil] INFO [2019-12-17 12:36:04,152] jepsen worker 52 - jepsen.util 10222 :ok :read [17 2] INFO [2019-12-17 12:36:04,152] jepsen worker 52 - jepsen.util 10222 :invoke :write [17 9] INFO [2019-12-17 12:36:04,162] jepsen worker 69 - jepsen.util 10329 :info :write [23 46] :RequestCanceledException INFO [2019-12-17 12:36:04,162] jepsen worker 67 - jepsen.util 14107 :info :write [22 32] :RequestCanceledException INFO [2019-12-17 12:36:04,163] jepsen worker 67 - jepsen.util 14197 :invoke :read [22 nil] INFO [2019-12-17 12:36:04,163] jepsen worker 69 - jepsen.util 10419 :invoke :read [23 nil] INFO [2019-12-17 12:36:04,170] jepsen worker 29 - jepsen.util 8759 :ok :write [9 11] INFO [2019-12-17 12:36:04,170] jepsen worker 29 - jepsen.util 8759 :invoke :read [9 nil] INFO [2019-12-17 12:36:04,177] jepsen worker 3 - jepsen.util 10083 :ok :read [1 33] INFO [2019-12-17 12:36:04,177] jepsen worker 3 - jepsen.util 10083 :invoke :write [1 34] INFO [2019-12-17 12:36:04,187] jepsen worker 29 - jepsen.util 8759 :ok :read [9 11] INFO [2019-12-17 12:36:04,187] jepsen worker 29 - jepsen.util 8759 :invoke :write [9 45] INFO [2019-12-17 12:36:04,198] jepsen worker 46 - jepsen.util 10306 :ok :write [15 31] INFO [2019-12-17 12:36:04,198] jepsen worker 46 - jepsen.util 10306 :invoke :write [15 36] INFO [2019-12-17 12:36:04,202] jepsen worker 32 - jepsen.util 10472 :ok :write [10 24] INFO [2019-12-17 12:36:04,202] jepsen worker 32 - jepsen.util 10472 :invoke :cas [10 [13 47]] INFO [2019-12-17 12:36:04,203] jepsen worker 73 - jepsen.util 9703 :ok :write [24 42] INFO [2019-12-17 12:36:04,203] jepsen worker 73 - jepsen.util 9703 :invoke :write [24 10] INFO [2019-12-17 12:36:04,218] jepsen worker 54 - jepsen.util 11304 :fail :cas [18 [39 16]] :ValueNotSwapFrom INFO [2019-12-17 12:36:04,218] jepsen worker 54 - jepsen.util 11304 :invoke :cas [18 [7 29]] INFO [2019-12-17 12:36:04,241] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":59515,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:36:04,241] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":59526,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:36:04,242] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":59515,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:36:04,242] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":59526,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:36:04,242] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":59516,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:36:04,242] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":59529,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:36:04,242] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":59516,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:36:04,242] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":59529,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:36:04,242] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":59517,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:36:04,242] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":59530,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:36:04,242] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":59517,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:36:04,242] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":59530,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:36:04,266] jepsen worker 38 - jepsen.util 11108 :ok :read [12 13] INFO [2019-12-17 12:36:04,266] jepsen worker 38 - jepsen.util 11108 :invoke :cas [12 [2 15]] INFO [2019-12-17 12:36:04,285] jepsen worker 72 - jepsen.util 10512 :fail :cas [24 [14 20]] :ValueNotSwapFrom INFO [2019-12-17 12:36:04,285] jepsen worker 72 - jepsen.util 10512 :invoke :read [24 nil] INFO [2019-12-17 12:36:04,287] jepsen worker 20 - jepsen.util 10190 :ok :read [6 32] INFO [2019-12-17 12:36:04,287] jepsen worker 20 - jepsen.util 10190 :invoke :read [6 nil] INFO [2019-12-17 12:36:04,304] jepsen worker 38 - jepsen.util 11108 :fail :cas [12 [2 15]] :ValueNotSwapFrom INFO [2019-12-17 12:36:04,304] jepsen worker 38 - jepsen.util 11108 :invoke :cas [12 [15 27]] INFO [2019-12-17 12:36:04,325] jepsen worker 30 - jepsen.util 10110 :fail :cas [10 [29 20]] :ValueNotSwapFrom INFO [2019-12-17 12:36:04,325] jepsen worker 30 - jepsen.util 10110 :invoke :write [10 44] INFO [2019-12-17 12:36:04,332] jepsen worker 15 - jepsen.util 13065 :info :cas [5 [2 15]] :RequestCanceledException INFO [2019-12-17 12:36:04,333] jepsen worker 15 - jepsen.util 13155 :invoke :read [5 nil] INFO [2019-12-17 12:36:04,344] jepsen worker 57 - jepsen.util 13647 :info :cas [19 [40 19]] :RequestCanceledException INFO [2019-12-17 12:36:04,370] jepsen worker 54 - jepsen.util 11304 :fail :cas [18 [7 29]] :ValueNotSwapFrom INFO [2019-12-17 12:36:04,370] jepsen worker 54 - jepsen.util 11304 :invoke :read [18 nil] INFO [2019-12-17 12:36:04,374] jepsen worker 57 - jepsen.util 13737 :invoke :cas [19 [24 13]] INFO [2019-12-17 12:36:04,388] jepsen worker 32 - jepsen.util 10472 :fail :cas [10 [13 47]] :ValueNotSwapFrom INFO [2019-12-17 12:36:04,388] jepsen worker 32 - jepsen.util 10472 :invoke :write [10 37] INFO [2019-12-17 12:36:04,398] jepsen worker 42 - jepsen.util 14982 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:36:04,398] jepsen worker 2 - jepsen.util 13592 :info :cas [0 [32 33]] :RequestCanceledException INFO [2019-12-17 12:36:04,398] jepsen worker 42 - jepsen.util 14982 :invoke :read [14 nil] INFO [2019-12-17 12:36:04,407] jepsen worker 2 - jepsen.util 13682 :invoke :write [0 19] INFO [2019-12-17 12:36:04,416] jepsen worker 60 - jepsen.util 14910 :info :cas [20 [19 3]] :RequestCanceledException INFO [2019-12-17 12:36:04,417] jepsen worker 60 - jepsen.util 15000 :invoke :write [20 12] INFO [2019-12-17 12:36:04,420] jepsen worker 38 - jepsen.util 11108 :fail :cas [12 [15 27]] :ValueNotSwapFrom INFO [2019-12-17 12:36:04,420] jepsen worker 38 - jepsen.util 11108 :invoke :cas [12 [0 11]] INFO [2019-12-17 12:36:04,430] jepsen worker 72 - jepsen.util 10512 :ok :read [24 42] INFO [2019-12-17 12:36:04,430] jepsen worker 72 - jepsen.util 10512 :invoke :read [24 nil] INFO [2019-12-17 12:36:04,431] jepsen worker 41 - jepsen.util 15071 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:36:04,431] jepsen worker 41 - jepsen.util 15071 :invoke :read [13 nil] INFO [2019-12-17 12:36:04,431] jepsen worker 89 - jepsen.util 10529 :info :write [29 4] :RequestCanceledException INFO [2019-12-17 12:36:04,432] jepsen worker 89 - jepsen.util 10619 :invoke :read [29 nil] INFO [2019-12-17 12:36:04,445] jepsen worker 17 - jepsen.util 13787 :info :cas [5 [12 17]] :RequestCanceledException INFO [2019-12-17 12:36:04,446] jepsen worker 17 - jepsen.util 13877 :invoke :cas [5 [24 36]] INFO [2019-12-17 12:36:04,461] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":59547,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:36:04,461] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":59547,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:36:04,461] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":59549,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:36:04,461] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":59549,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:36:04,461] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":59550,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:36:04,461] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":59550,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:36:04,465] jepsen worker 29 - jepsen.util 8759 :ok :write [9 45] INFO [2019-12-17 12:36:04,466] jepsen worker 29 - jepsen.util 8759 :invoke :write [9 2] INFO [2019-12-17 12:36:04,492] jepsen worker 69 - jepsen.util 10419 :ok :read [23 47] INFO [2019-12-17 12:36:04,492] jepsen worker 69 - jepsen.util 10419 :invoke :write [23 44] INFO [2019-12-17 12:36:04,497] jepsen worker 12 - jepsen.util 10812 :ok :write [4 49] INFO [2019-12-17 12:36:04,497] jepsen worker 12 - jepsen.util 10812 :invoke :cas [4 [7 30]] INFO [2019-12-17 12:36:04,497] jepsen worker 88 - jepsen.util 10708 :ok :write [29 35] INFO [2019-12-17 12:36:04,497] jepsen worker 88 - jepsen.util 10708 :invoke :cas [29 [36 38]] INFO [2019-12-17 12:36:04,498] jepsen worker 6 - jepsen.util 14406 :info :write [2 34] :RequestCanceledException INFO [2019-12-17 12:36:04,499] jepsen worker 6 - jepsen.util 14496 :invoke :write [2 47] INFO [2019-12-17 12:36:04,526] jepsen worker 29 - jepsen.util 8759 :ok :write [9 2] INFO [2019-12-17 12:36:04,526] jepsen worker 29 - jepsen.util 8759 :invoke :cas [9 [35 27]] INFO [2019-12-17 12:36:04,546] jepsen worker 11 - jepsen.util 9911 :fail :read [3 nil] :RequestTimeoutException INFO [2019-12-17 12:36:04,546] jepsen worker 24 - jepsen.util 12624 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:36:04,546] jepsen worker 11 - jepsen.util 9911 :invoke :read [3 nil] INFO [2019-12-17 12:36:04,546] jepsen worker 24 - jepsen.util 12624 :invoke :write [8 45] INFO [2019-12-17 12:36:04,547] jepsen worker 29 - jepsen.util 8759 :fail :cas [9 [35 27]] :ValueNotSwapFrom INFO [2019-12-17 12:36:04,547] jepsen worker 29 - jepsen.util 8759 :invoke :write [9 40] INFO [2019-12-17 12:36:04,583] jepsen worker 29 - jepsen.util 8759 :ok :write [9 40] INFO [2019-12-17 12:36:04,583] jepsen worker 29 - jepsen.util 8759 :invoke :read [9 nil] INFO [2019-12-17 12:36:04,598] jepsen worker 29 - jepsen.util 8759 :ok :read [9 40] INFO [2019-12-17 12:36:04,598] jepsen worker 76 - jepsen.util 14296 :info :write [25 47] :RequestCanceledException INFO [2019-12-17 12:36:04,599] jepsen worker 29 - jepsen.util 8759 :invoke :cas [9 [45 7]] INFO [2019-12-17 12:36:04,599] jepsen worker 76 - jepsen.util 14386 :invoke :cas [25 [25 0]] INFO [2019-12-17 12:36:04,610] jepsen worker 54 - jepsen.util 11304 :ok :read [18 5] INFO [2019-12-17 12:36:04,610] jepsen worker 54 - jepsen.util 11304 :invoke :write [18 20] INFO [2019-12-17 12:36:04,624] jepsen worker 38 - jepsen.util 11108 :fail :cas [12 [0 11]] :ValueNotSwapFrom INFO [2019-12-17 12:36:04,624] jepsen worker 38 - jepsen.util 11108 :invoke :cas [12 [18 18]] INFO [2019-12-17 12:36:04,630] jepsen worker 29 - jepsen.util 8759 :fail :cas [9 [45 7]] :ValueNotSwapFrom INFO [2019-12-17 12:36:04,630] jepsen worker 29 - jepsen.util 8759 :invoke :cas [9 [34 18]] INFO [2019-12-17 12:36:04,651] jepsen worker 18 - jepsen.util 10728 :ok :write [6 14] INFO [2019-12-17 12:36:04,651] jepsen worker 18 - jepsen.util 10728 :invoke :cas [6 [0 47]] INFO [2019-12-17 12:36:04,653] jepsen worker 73 - jepsen.util 9703 :ok :write [24 10] INFO [2019-12-17 12:36:04,653] jepsen worker 30 - jepsen.util 10110 :ok :write [10 44] INFO [2019-12-17 12:36:04,653] jepsen worker 30 - jepsen.util 10110 :invoke :write [10 23] INFO [2019-12-17 12:36:04,653] jepsen worker 73 - jepsen.util 9703 :invoke :write [24 7] INFO [2019-12-17 12:36:04,663] jepsen worker 71 - jepsen.util 11141 :info :write [23 12] :RequestCanceledException INFO [2019-12-17 12:36:04,663] jepsen worker 29 - jepsen.util 8759 :fail :cas [9 [34 18]] :ValueNotSwapFrom INFO [2019-12-17 12:36:04,663] jepsen worker 29 - jepsen.util 8759 :invoke :read [9 nil] INFO [2019-12-17 12:36:04,664] jepsen worker 71 - jepsen.util 11231 :invoke :cas [23 [30 13]] INFO [2019-12-17 12:36:04,667] jepsen worker 10 - jepsen.util 10360 :ok :write [3 33] INFO [2019-12-17 12:36:04,667] jepsen worker 10 - jepsen.util 10360 :invoke :write [3 4] INFO [2019-12-17 12:36:04,677] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":59570,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:36:04,677] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":59570,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:36:04,677] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":59571,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:36:04,678] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":59571,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:36:04,678] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":59572,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:36:04,678] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":59572,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:36:04,685] jepsen worker 89 - jepsen.util 10619 :ok :read [29 35] INFO [2019-12-17 12:36:04,686] jepsen worker 89 - jepsen.util 10619 :invoke :read [29 nil] INFO [2019-12-17 12:36:04,716] jepsen worker 19 - jepsen.util 10369 :info :write [6 35] :RequestCanceledException INFO [2019-12-17 12:36:04,717] jepsen worker 19 - jepsen.util 10459 :invoke :read [6 nil] INFO [2019-12-17 12:36:04,731] jepsen worker 35 - jepsen.util 14075 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:36:04,731] jepsen worker 28 - jepsen.util 8398 :fail :read [9 nil] :RequestTimeoutException INFO [2019-12-17 12:36:04,731] jepsen worker 35 - jepsen.util 14075 :invoke :read [11 nil] INFO [2019-12-17 12:36:04,731] jepsen worker 28 - jepsen.util 8398 :invoke :write [9 7] INFO [2019-12-17 12:36:04,732] jepsen worker 38 - jepsen.util 11108 :fail :cas [12 [18 18]] :ValueNotSwapFrom INFO [2019-12-17 12:36:04,732] jepsen worker 38 - jepsen.util 11108 :invoke :write [12 39] INFO [2019-12-17 12:36:04,744] jepsen worker 3 - jepsen.util 10083 :ok :write [1 34] INFO [2019-12-17 12:36:04,744] jepsen worker 3 - jepsen.util 10083 :invoke :read [1 nil] INFO [2019-12-17 12:36:04,744] jepsen worker 44 - jepsen.util 14354 :info :cas [14 [39 6]] :RequestCanceledException INFO [2019-12-17 12:36:04,745] jepsen worker 44 - jepsen.util 14444 :invoke :read [14 nil] INFO [2019-12-17 12:36:04,756] jepsen worker 70 - jepsen.util 11140 :ok :read [23 47] INFO [2019-12-17 12:36:04,756] jepsen worker 70 - jepsen.util 11140 :invoke :read [23 nil] INFO [2019-12-17 12:36:04,770] jepsen worker 11 - jepsen.util 9911 :ok :read [3 29] INFO [2019-12-17 12:36:04,770] jepsen worker 11 - jepsen.util 9911 :invoke :write [3 36] INFO [2019-12-17 12:36:04,774] jepsen worker 71 - jepsen.util 11231 :fail :cas [23 [30 13]] :ValueNotSwapFrom INFO [2019-12-17 12:36:04,774] jepsen worker 71 - jepsen.util 11231 :invoke :cas [23 [8 43]] INFO [2019-12-17 12:36:04,801] jepsen worker 29 - jepsen.util 8759 :ok :read [9 40] INFO [2019-12-17 12:36:04,801] jepsen worker 29 - jepsen.util 8759 :invoke :write [9 36] INFO [2019-12-17 12:36:04,804] jepsen worker 18 - jepsen.util 10728 :fail :cas [6 [0 47]] :ValueNotSwapFrom INFO [2019-12-17 12:36:04,804] jepsen worker 18 - jepsen.util 10728 :invoke :cas [6 [28 49]] INFO [2019-12-17 12:36:04,832] jepsen worker 62 - jepsen.util 14462 :info :write [20 22] :RequestCanceledException INFO [2019-12-17 12:36:04,833] jepsen worker 62 - jepsen.util 14552 :invoke :read [20 nil] INFO [2019-12-17 12:36:04,857] jepsen worker 89 - jepsen.util 10619 :ok :read [29 35] INFO [2019-12-17 12:36:04,857] jepsen worker 89 - jepsen.util 10619 :invoke :cas [29 [40 1]] INFO [2019-12-17 12:36:04,862] jepsen worker 45 - jepsen.util 10395 :info :write [15 15] :RequestCanceledException INFO [2019-12-17 12:36:04,863] jepsen worker 45 - jepsen.util 10485 :invoke :write [15 22] INFO [2019-12-17 12:36:04,891] jepsen worker 3 - jepsen.util 10083 :ok :read [1 34] INFO [2019-12-17 12:36:04,891] jepsen worker 3 - jepsen.util 10083 :invoke :read [1 nil] INFO [2019-12-17 12:36:04,902] jepsen worker 29 - jepsen.util 8759 :ok :write [9 36] INFO [2019-12-17 12:36:04,902] jepsen worker 29 - jepsen.util 8759 :invoke :write [9 27] INFO [2019-12-17 12:36:04,916] jepsen worker 88 - jepsen.util 10708 :fail :cas [29 [36 38]] :ValueNotSwapFrom INFO [2019-12-17 12:36:04,916] jepsen worker 88 - jepsen.util 10708 :invoke :write [29 41] INFO [2019-12-17 12:36:04,920] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":59599,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:36:04,920] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":59599,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:36:04,935] jepsen worker 47 - jepsen.util 10937 :ok :write [15 27] INFO [2019-12-17 12:36:04,935] jepsen worker 47 - jepsen.util 10937 :invoke :write [15 38] INFO [2019-12-17 12:36:04,945] jepsen worker 40 - jepsen.util 14530 :info :write [13 16] :RequestCanceledException INFO [2019-12-17 12:36:04,946] jepsen worker 40 - jepsen.util 14620 :invoke :read [13 nil] INFO [2019-12-17 12:36:04,963] jepsen worker 23 - jepsen.util 14063 :info :write [7 0] :RequestCanceledException INFO [2019-12-17 12:36:04,964] jepsen worker 23 - jepsen.util 14153 :invoke :write [7 39] INFO [2019-12-17 12:36:04,993] jepsen worker 29 - jepsen.util 8759 :ok :write [9 27] INFO [2019-12-17 12:36:04,993] jepsen worker 29 - jepsen.util 8759 :invoke :cas [9 [13 11]] INFO [2019-12-17 12:36:04,998] jepsen worker 85 - jepsen.util 14845 :info :write [28 36] :RequestCanceledException INFO [2019-12-17 12:36:04,998] jepsen worker 74 - jepsen.util 10064 :info :cas [24 [45 37]] :RequestCanceledException INFO [2019-12-17 12:36:04,999] jepsen worker 85 - jepsen.util 14935 :invoke :cas [28 [36 26]] INFO [2019-12-17 12:36:04,999] jepsen worker 74 - jepsen.util 10154 :invoke :write [24 39] INFO [2019-12-17 12:36:05,011] jepsen worker 29 - jepsen.util 8759 :fail :cas [9 [13 11]] :ValueNotSwapFrom INFO [2019-12-17 12:36:05,011] jepsen worker 29 - jepsen.util 8759 :invoke :write [9 4] INFO [2019-12-17 12:36:05,029] jepsen worker 4 - jepsen.util 9634 :fail :cas [1 [14 22]] :ValueNotSwapFrom INFO [2019-12-17 12:36:05,029] jepsen worker 4 - jepsen.util 9634 :invoke :write [1 15] INFO [2019-12-17 12:36:05,030] jepsen worker 3 - jepsen.util 10083 :ok :read [1 34] INFO [2019-12-17 12:36:05,030] jepsen worker 3 - jepsen.util 10083 :invoke :cas [1 [27 42]] INFO [2019-12-17 12:36:05,048] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:36:05,049] jepsen nemesis - couchbase.util Rebalance check count: 466 INFO [2019-12-17 12:36:05,049] jepsen nemesis - couchbase.util Rebalance stuck count: 447 INFO [2019-12-17 12:36:05,052] jepsen worker 71 - jepsen.util 11231 :fail :cas [23 [8 43]] :ValueNotSwapFrom INFO [2019-12-17 12:36:05,053] jepsen worker 71 - jepsen.util 11231 :invoke :cas [23 [9 48]] INFO [2019-12-17 12:36:05,056] jepsen worker 55 - jepsen.util 11035 :fail :cas [18 [39 23]] :ValueNotSwapFrom INFO [2019-12-17 12:36:05,056] jepsen worker 55 - jepsen.util 11035 :invoke :cas [18 [1 19]] INFO [2019-12-17 12:36:05,073] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":59611,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:36:05,074] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":59611,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:36:05,074] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":59615,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:36:05,074] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":59615,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:36:05,079] jepsen worker 78 - jepsen.util 10248 :fail :cas [26 [37 39]] :ValueNotSwapFrom INFO [2019-12-17 12:36:05,079] jepsen worker 78 - jepsen.util 10248 :invoke :cas [26 [37 1]] INFO [2019-12-17 12:36:05,090] jepsen worker 29 - jepsen.util 8759 :ok :write [9 4] INFO [2019-12-17 12:36:05,090] jepsen worker 29 - jepsen.util 8759 :invoke :cas [9 [23 5]] INFO [2019-12-17 12:36:05,105] jepsen worker 53 - jepsen.util 9773 :fail :cas [17 [41 41]] :ValueNotSwapFrom INFO [2019-12-17 12:36:05,105] jepsen worker 53 - jepsen.util 9773 :invoke :write [17 36] INFO [2019-12-17 12:36:05,107] jepsen worker 32 - jepsen.util 10472 :ok :write [10 37] INFO [2019-12-17 12:36:05,107] jepsen worker 32 - jepsen.util 10472 :invoke :write [10 5] INFO [2019-12-17 12:36:05,107] jepsen worker 10 - jepsen.util 10360 :ok :write [3 4] INFO [2019-12-17 12:36:05,107] jepsen worker 10 - jepsen.util 10360 :invoke :cas [3 [2 1]] INFO [2019-12-17 12:36:05,123] jepsen worker 29 - jepsen.util 8759 :fail :cas [9 [23 5]] :ValueNotSwapFrom INFO [2019-12-17 12:36:05,123] jepsen worker 29 - jepsen.util 8759 :invoke :write [9 38] INFO [2019-12-17 12:36:05,132] jepsen worker 66 - jepsen.util 13746 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:36:05,132] jepsen worker 83 - jepsen.util 14123 :info :cas [27 [20 8]] :RequestCanceledException INFO [2019-12-17 12:36:05,132] jepsen worker 49 - jepsen.util 13999 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:36:05,132] jepsen worker 66 - jepsen.util 13746 :invoke :write [22 13] INFO [2019-12-17 12:36:05,132] jepsen worker 49 - jepsen.util 13999 :invoke :read [16 nil] INFO [2019-12-17 12:36:05,133] jepsen worker 83 - jepsen.util 14213 :invoke :write [27 48] INFO [2019-12-17 12:36:05,146] jepsen worker 51 - jepsen.util 9771 :info :write [17 9] :RequestCanceledException INFO [2019-12-17 12:36:05,146] jepsen worker 51 - jepsen.util 9861 :invoke :cas [17 [40 21]] INFO [2019-12-17 12:36:05,152] jepsen worker 3 - jepsen.util 10083 :fail :cas [1 [27 42]] :ValueNotSwapFrom INFO [2019-12-17 12:36:05,152] jepsen worker 3 - jepsen.util 10083 :invoke :write [1 47] INFO [2019-12-17 12:36:05,174] jepsen worker 70 - jepsen.util 11140 :ok :read [23 12] INFO [2019-12-17 12:36:05,174] jepsen worker 70 - jepsen.util 11140 :invoke :read [23 nil] INFO [2019-12-17 12:36:05,194] jepsen worker 12 - jepsen.util 10812 :fail :cas [4 [7 30]] :ValueNotSwapFrom INFO [2019-12-17 12:36:05,195] jepsen worker 29 - jepsen.util 8759 :ok :write [9 38] INFO [2019-12-17 12:36:05,195] jepsen worker 12 - jepsen.util 10812 :invoke :read [4 nil] INFO [2019-12-17 12:36:05,195] jepsen worker 29 - jepsen.util 8759 :invoke :read [9 nil] INFO [2019-12-17 12:36:05,197] jepsen worker 0 - jepsen.util 14850 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:36:05,197] jepsen worker 7 - jepsen.util 14137 :info :cas [2 [43 48]] :RequestCanceledException INFO [2019-12-17 12:36:05,197] jepsen worker 0 - jepsen.util 14850 :invoke :read [0 nil] INFO [2019-12-17 12:36:05,199] jepsen worker 7 - jepsen.util 14227 :invoke :cas [2 [2 6]] INFO [2019-12-17 12:36:05,212] jepsen worker 29 - jepsen.util 8759 :ok :read [9 38] INFO [2019-12-17 12:36:05,212] jepsen worker 29 - jepsen.util 8759 :invoke :cas [9 [0 7]] INFO [2019-12-17 12:36:05,214] jepsen worker 61 - jepsen.util 14731 :info :write [20 27] :RequestCanceledException INFO [2019-12-17 12:36:05,218] jepsen worker 61 - jepsen.util 14821 :invoke :cas [20 [26 26]] INFO [2019-12-17 12:36:05,223] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":59647,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:36:05,223] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":59647,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:36:05,223] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":59650,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:36:05,223] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":59650,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:36:05,231] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":59638,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:36:05,231] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":59638,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:36:05,231] jepsen worker 26 - jepsen.util 14606 :info :write [8 43] :RequestCanceledException INFO [2019-12-17 12:36:05,232] jepsen worker 87 - jepsen.util 9897 :info :write [29 2] :RequestCanceledException INFO [2019-12-17 12:36:05,232] jepsen worker 5 - jepsen.util 9905 :info :write [1 16] :RequestCanceledException INFO [2019-12-17 12:36:05,233] jepsen worker 87 - jepsen.util 9987 :invoke :read [29 nil] INFO [2019-12-17 12:36:05,233] jepsen worker 26 - jepsen.util 14696 :invoke :write [8 40] INFO [2019-12-17 12:36:05,235] jepsen worker 5 - jepsen.util 9995 :invoke :write [1 28] INFO [2019-12-17 12:36:05,236] jepsen worker 29 - jepsen.util 8759 :fail :cas [9 [0 7]] :ValueNotSwapFrom INFO [2019-12-17 12:36:05,236] jepsen worker 29 - jepsen.util 8759 :invoke :read [9 nil] INFO [2019-12-17 12:36:05,237] jepsen worker 73 - jepsen.util 9703 :ok :write [24 7] INFO [2019-12-17 12:36:05,237] jepsen worker 73 - jepsen.util 9703 :invoke :read [24 nil] INFO [2019-12-17 12:36:05,242] jepsen worker 30 - jepsen.util 10110 :ok :write [10 23] INFO [2019-12-17 12:36:05,242] jepsen worker 30 - jepsen.util 10110 :invoke :cas [10 [5 46]] INFO [2019-12-17 12:36:05,246] jepsen worker 4 - jepsen.util 9634 :ok :write [1 15] INFO [2019-12-17 12:36:05,246] jepsen worker 4 - jepsen.util 9634 :invoke :read [1 nil] INFO [2019-12-17 12:36:05,259] jepsen worker 45 - jepsen.util 10485 :ok :write [15 22] INFO [2019-12-17 12:36:05,259] jepsen worker 45 - jepsen.util 10485 :invoke :write [15 25] INFO [2019-12-17 12:36:05,291] jepsen worker 29 - jepsen.util 8759 :ok :read [9 38] INFO [2019-12-17 12:36:05,291] jepsen worker 29 - jepsen.util 8759 :invoke :write [9 3] INFO [2019-12-17 12:36:05,291] jepsen worker 73 - jepsen.util 9703 :ok :read [24 7] INFO [2019-12-17 12:36:05,292] jepsen worker 73 - jepsen.util 9703 :invoke :read [24 nil] INFO [2019-12-17 12:36:05,298] jepsen worker 12 - jepsen.util 10812 :ok :read [4 49] INFO [2019-12-17 12:36:05,298] jepsen worker 12 - jepsen.util 10812 :invoke :write [4 28] INFO [2019-12-17 12:36:05,308] jepsen worker 73 - jepsen.util 9703 :ok :read [24 7] INFO [2019-12-17 12:36:05,309] jepsen worker 73 - jepsen.util 9703 :invoke :cas [24 [46 16]] INFO [2019-12-17 12:36:05,311] jepsen worker 4 - jepsen.util 9634 :ok :read [1 15] INFO [2019-12-17 12:36:05,311] jepsen worker 4 - jepsen.util 9634 :invoke :cas [1 [37 42]] INFO [2019-12-17 12:36:05,325] jepsen worker 73 - jepsen.util 9703 :fail :cas [24 [46 16]] :ValueNotSwapFrom INFO [2019-12-17 12:36:05,326] jepsen worker 73 - jepsen.util 9703 :invoke :write [24 15] INFO [2019-12-17 12:36:05,336] jepsen worker 30 - jepsen.util 10110 :fail :cas [10 [5 46]] :ValueNotSwapFrom INFO [2019-12-17 12:36:05,336] jepsen worker 30 - jepsen.util 10110 :invoke :read [10 nil] INFO [2019-12-17 12:36:05,345] jepsen worker 1 - jepsen.util 14041 :info :write [0 2] :RequestCanceledException INFO [2019-12-17 12:36:05,346] jepsen worker 1 - jepsen.util 14131 :invoke :cas [0 [48 10]] INFO [2019-12-17 12:36:05,368] jepsen worker 78 - jepsen.util 10248 :fail :cas [26 [37 1]] :ValueNotSwapFrom INFO [2019-12-17 12:36:05,368] jepsen worker 78 - jepsen.util 10248 :invoke :write [26 13] INFO [2019-12-17 12:36:05,375] jepsen worker 30 - jepsen.util 10110 :ok :read [10 23] INFO [2019-12-17 12:36:05,375] jepsen worker 30 - jepsen.util 10110 :invoke :write [10 8] INFO [2019-12-17 12:36:05,381] jepsen worker 29 - jepsen.util 8759 :ok :write [9 3] INFO [2019-12-17 12:36:05,381] jepsen worker 29 - jepsen.util 8759 :invoke :read [9 nil] INFO [2019-12-17 12:36:05,409] jepsen worker 51 - jepsen.util 9861 :fail :cas [17 [40 21]] :ValueNotSwapFrom INFO [2019-12-17 12:36:05,410] jepsen worker 51 - jepsen.util 9861 :invoke :write [17 30] INFO [2019-12-17 12:36:05,421] jepsen worker 29 - jepsen.util 8759 :ok :read [9 3] INFO [2019-12-17 12:36:05,421] jepsen worker 29 - jepsen.util 8759 :invoke :write [9 49] INFO [2019-12-17 12:36:05,423] jepsen worker 89 - jepsen.util 10619 :fail :cas [29 [40 1]] :ValueNotSwapFrom INFO [2019-12-17 12:36:05,423] jepsen worker 89 - jepsen.util 10619 :invoke :write [29 13] INFO [2019-12-17 12:36:05,432] jepsen worker 25 - jepsen.util 15055 :info :write [8 23] :RequestCanceledException INFO [2019-12-17 12:36:05,433] jepsen worker 25 - jepsen.util 15145 :invoke :write [8 21] INFO [2019-12-17 12:36:05,443] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":59678,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:36:05,443] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":59678,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:36:05,444] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":59680,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:36:05,444] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":59680,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:36:05,459] jepsen worker 75 - jepsen.util 13485 :info :cas [25 [27 11]] :RequestCanceledException INFO [2019-12-17 12:36:05,460] jepsen worker 75 - jepsen.util 13575 :invoke :read [25 nil] INFO [2019-12-17 12:36:05,473] jepsen worker 29 - jepsen.util 8759 :ok :write [9 49] INFO [2019-12-17 12:36:05,473] jepsen worker 29 - jepsen.util 8759 :invoke :cas [9 [15 5]] INFO [2019-12-17 12:36:05,495] jepsen worker 29 - jepsen.util 8759 :fail :cas [9 [15 5]] :ValueNotSwapFrom INFO [2019-12-17 12:36:05,495] jepsen worker 29 - jepsen.util 8759 :invoke :read [9 nil] INFO [2019-12-17 12:36:05,499] jepsen worker 64 - jepsen.util 7984 :info :write [21 8] :RequestCanceledException INFO [2019-12-17 12:36:05,499] jepsen worker 58 - jepsen.util 14458 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:36:05,499] jepsen worker 80 - jepsen.util 10070 :info :write [26 27] :RequestCanceledException INFO [2019-12-17 12:36:05,499] jepsen worker 14 - jepsen.util 10184 :info :write [4 18] :RequestCanceledException INFO [2019-12-17 12:36:05,499] jepsen worker 68 - jepsen.util 13568 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:36:05,499] jepsen worker 58 - jepsen.util 14458 :invoke :read [19 nil] INFO [2019-12-17 12:36:05,499] jepsen worker 68 - jepsen.util 13568 :invoke :write [22 4] INFO [2019-12-17 12:36:05,500] jepsen worker 64 - jepsen.util 8074 :invoke :read [21 nil] INFO [2019-12-17 12:36:05,500] jepsen worker 80 - jepsen.util 10160 :invoke :cas [26 [47 14]] INFO [2019-12-17 12:36:05,500] jepsen worker 14 - jepsen.util 10274 :invoke :write [4 26] INFO [2019-12-17 12:36:05,508] jepsen worker 29 - jepsen.util 8759 :ok :read [9 49] INFO [2019-12-17 12:36:05,508] jepsen worker 29 - jepsen.util 8759 :invoke :cas [9 [3 35]] INFO [2019-12-17 12:36:05,510] jepsen worker 73 - jepsen.util 9703 :ok :write [24 15] INFO [2019-12-17 12:36:05,510] jepsen worker 73 - jepsen.util 9703 :invoke :cas [24 [34 32]] INFO [2019-12-17 12:36:05,511] jepsen worker 45 - jepsen.util 10485 :ok :write [15 25] INFO [2019-12-17 12:36:05,511] jepsen worker 45 - jepsen.util 10485 :invoke :read [15 nil] INFO [2019-12-17 12:36:05,513] jepsen worker 55 - jepsen.util 11035 :fail :cas [18 [1 19]] :ValueNotSwapFrom INFO [2019-12-17 12:36:05,513] jepsen worker 55 - jepsen.util 11035 :invoke :read [18 nil] INFO [2019-12-17 12:36:05,515] jepsen worker 30 - jepsen.util 10110 :ok :write [10 8] INFO [2019-12-17 12:36:05,515] jepsen worker 30 - jepsen.util 10110 :invoke :cas [10 [37 12]] INFO [2019-12-17 12:36:05,562] jepsen worker 29 - jepsen.util 8759 :fail :cas [9 [3 35]] :ValueNotSwapFrom INFO [2019-12-17 12:36:05,562] jepsen worker 29 - jepsen.util 8759 :invoke :cas [9 [0 33]] INFO [2019-12-17 12:36:05,563] jepsen worker 82 - jepsen.util 14212 :info :cas [27 [30 37]] :RequestCanceledException INFO [2019-12-17 12:36:05,563] jepsen worker 30 - jepsen.util 10110 :fail :cas [10 [37 12]] :ValueNotSwapFrom INFO [2019-12-17 12:36:05,564] jepsen worker 30 - jepsen.util 10110 :invoke :write [10 44] INFO [2019-12-17 12:36:05,564] jepsen worker 82 - jepsen.util 14302 :invoke :cas [27 [16 47]] INFO [2019-12-17 12:36:05,567] jepsen worker 64 - jepsen.util 8074 :ok :read [21 14] INFO [2019-12-17 12:36:05,567] jepsen worker 64 - jepsen.util 8074 :invoke :read [21 nil] INFO [2019-12-17 12:36:05,642] jepsen worker 45 - jepsen.util 10485 :ok :read [15 25] INFO [2019-12-17 12:36:05,642] jepsen worker 45 - jepsen.util 10485 :invoke :read [15 nil] INFO [2019-12-17 12:36:05,648] jepsen worker 73 - jepsen.util 9703 :fail :cas [24 [34 32]] :ValueNotSwapFrom INFO [2019-12-17 12:36:05,648] jepsen worker 73 - jepsen.util 9703 :invoke :cas [24 [6 22]] INFO [2019-12-17 12:36:05,657] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":59701,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:36:05,657] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":59701,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:36:05,657] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":59704,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:36:05,658] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":59704,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:36:05,665] jepsen worker 29 - jepsen.util 8759 :fail :cas [9 [0 33]] :ValueNotSwapFrom INFO [2019-12-17 12:36:05,665] jepsen worker 29 - jepsen.util 8759 :invoke :write [9 12] INFO [2019-12-17 12:36:05,672] jepsen worker 64 - jepsen.util 8074 :ok :read [21 14] INFO [2019-12-17 12:36:05,673] jepsen worker 64 - jepsen.util 8074 :invoke :write [21 27] INFO [2019-12-17 12:36:05,675] jepsen worker 73 - jepsen.util 9703 :fail :cas [24 [6 22]] :ValueNotSwapFrom INFO [2019-12-17 12:36:05,675] jepsen worker 73 - jepsen.util 9703 :invoke :cas [24 [8 38]] INFO [2019-12-17 12:36:05,687] jepsen worker 45 - jepsen.util 10485 :ok :read [15 25] INFO [2019-12-17 12:36:05,687] jepsen worker 45 - jepsen.util 10485 :invoke :cas [15 [33 20]] INFO [2019-12-17 12:36:05,687] jepsen worker 71 - jepsen.util 11231 :fail :cas [23 [9 48]] :ValueNotSwapFrom INFO [2019-12-17 12:36:05,687] jepsen worker 71 - jepsen.util 11231 :invoke :read [23 nil] INFO [2019-12-17 12:36:05,697] jepsen worker 48 - jepsen.util 12918 :info :cas [16 [43 32]] :RequestCanceledException INFO [2019-12-17 12:36:05,698] jepsen worker 48 - jepsen.util 13008 :invoke :read [16 nil] INFO [2019-12-17 12:36:05,699] jepsen worker 70 - jepsen.util 11140 :ok :read [23 12] INFO [2019-12-17 12:36:05,699] jepsen worker 70 - jepsen.util 11140 :invoke :write [23 26] INFO [2019-12-17 12:36:05,716] jepsen worker 63 - jepsen.util 9333 :info :cas [21 [32 20]] :RequestCanceledException INFO [2019-12-17 12:36:05,717] jepsen worker 63 - jepsen.util 9423 :invoke :read [21 nil] INFO [2019-12-17 12:36:05,719] jepsen worker 87 - jepsen.util 9987 :ok :read [29 35] INFO [2019-12-17 12:36:05,719] jepsen worker 87 - jepsen.util 9987 :invoke :write [29 9] INFO [2019-12-17 12:36:05,731] jepsen worker 9 - jepsen.util 11079 :fail :read [3 nil] :RequestTimeoutException INFO [2019-12-17 12:36:05,731] jepsen worker 9 - jepsen.util 11079 :invoke :cas [3 [36 49]] INFO [2019-12-17 12:36:05,742] jepsen worker 73 - jepsen.util 9703 :fail :cas [24 [8 38]] :ValueNotSwapFrom INFO [2019-12-17 12:36:05,742] jepsen worker 73 - jepsen.util 9703 :invoke :read [24 nil] INFO [2019-12-17 12:36:05,744] jepsen worker 27 - jepsen.util 9837 :info :cas [9 [25 20]] :RequestCanceledException INFO [2019-12-17 12:36:05,746] jepsen worker 27 - jepsen.util 9927 :invoke :write [9 34] INFO [2019-12-17 12:36:05,755] jepsen worker 9 - jepsen.util 11079 :fail :cas [3 [36 49]] :ValueNotSwapFrom INFO [2019-12-17 12:36:05,755] jepsen worker 9 - jepsen.util 11079 :invoke :read [3 nil] INFO [2019-12-17 12:36:05,761] jepsen worker 45 - jepsen.util 10485 :fail :cas [15 [33 20]] :ValueNotSwapFrom INFO [2019-12-17 12:36:05,761] jepsen worker 45 - jepsen.util 10485 :invoke :cas [15 [28 9]] INFO [2019-12-17 12:36:05,783] jepsen worker 45 - jepsen.util 10485 :fail :cas [15 [28 9]] :ValueNotSwapFrom INFO [2019-12-17 12:36:05,783] jepsen worker 45 - jepsen.util 10485 :invoke :cas [15 [18 6]] INFO [2019-12-17 12:36:05,793] jepsen worker 4 - jepsen.util 9634 :fail :cas [1 [37 42]] :ValueNotSwapFrom INFO [2019-12-17 12:36:05,793] jepsen worker 4 - jepsen.util 9634 :invoke :write [1 13] INFO [2019-12-17 12:36:05,798] jepsen worker 33 - jepsen.util 14073 :info :cas [11 [48 46]] :RequestCanceledException INFO [2019-12-17 12:36:05,799] jepsen worker 33 - jepsen.util 14163 :invoke :read [11 nil] INFO [2019-12-17 12:36:05,809] jepsen worker 71 - jepsen.util 11231 :ok :read [23 12] INFO [2019-12-17 12:36:05,809] jepsen worker 71 - jepsen.util 11231 :invoke :write [23 14] INFO [2019-12-17 12:36:05,811] jepsen worker 9 - jepsen.util 11079 :ok :read [3 4] INFO [2019-12-17 12:36:05,811] jepsen worker 9 - jepsen.util 11079 :invoke :cas [3 [17 18]] INFO [2019-12-17 12:36:05,816] jepsen worker 81 - jepsen.util 13761 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:36:05,816] jepsen worker 81 - jepsen.util 13761 :invoke :cas [27 [14 14]] INFO [2019-12-17 12:36:05,828] jepsen worker 45 - jepsen.util 10485 :fail :cas [15 [18 6]] :ValueNotSwapFrom INFO [2019-12-17 12:36:05,829] jepsen worker 65 - jepsen.util 8795 :info :cas [21 [34 36]] :RequestCanceledException INFO [2019-12-17 12:36:05,829] jepsen worker 45 - jepsen.util 10485 :invoke :cas [15 [27 18]] INFO [2019-12-17 12:36:05,830] jepsen worker 65 - jepsen.util 8885 :invoke :cas [21 [7 12]] INFO [2019-12-17 12:36:05,832] jepsen worker 39 - jepsen.util 13719 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:36:05,832] jepsen worker 31 - jepsen.util 9121 :fail :read [10 nil] :RequestTimeoutException INFO [2019-12-17 12:36:05,832] jepsen worker 39 - jepsen.util 13719 :invoke :read [13 nil] INFO [2019-12-17 12:36:05,832] jepsen worker 31 - jepsen.util 9121 :invoke :cas [10 [25 14]] INFO [2019-12-17 12:36:05,862] jepsen worker 9 - jepsen.util 11079 :fail :cas [3 [17 18]] :ValueNotSwapFrom INFO [2019-12-17 12:36:05,862] jepsen worker 9 - jepsen.util 11079 :invoke :cas [3 [29 39]] INFO [2019-12-17 12:36:05,868] jepsen worker 37 - jepsen.util 9937 :ok :write [12 24] INFO [2019-12-17 12:36:05,868] jepsen worker 37 - jepsen.util 9937 :invoke :read [12 nil] INFO [2019-12-17 12:36:05,892] jepsen worker 65 - jepsen.util 8885 :fail :cas [21 [7 12]] :ValueNotSwapFrom INFO [2019-12-17 12:36:05,892] jepsen worker 65 - jepsen.util 8885 :invoke :write [21 31] INFO [2019-12-17 12:36:05,895] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":59732,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:36:05,895] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":59732,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:36:05,895] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":59736,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:36:05,895] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":59736,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:36:05,897] jepsen worker 45 - jepsen.util 10485 :fail :cas [15 [27 18]] :ValueNotSwapFrom INFO [2019-12-17 12:36:05,897] jepsen worker 45 - jepsen.util 10485 :invoke :write [15 25] INFO [2019-12-17 12:36:05,897] jepsen worker 9 - jepsen.util 11079 :fail :cas [3 [29 39]] :ValueNotSwapFrom INFO [2019-12-17 12:36:05,897] jepsen worker 9 - jepsen.util 11079 :invoke :cas [3 [16 11]] INFO [2019-12-17 12:36:05,915] jepsen worker 56 - jepsen.util 10766 :info :write [18 25] :RequestCanceledException INFO [2019-12-17 12:36:05,916] jepsen worker 56 - jepsen.util 10856 :invoke :read [18 nil] INFO [2019-12-17 12:36:05,923] jepsen worker 9 - jepsen.util 11079 :fail :cas [3 [16 11]] :ValueNotSwapFrom INFO [2019-12-17 12:36:05,923] jepsen worker 9 - jepsen.util 11079 :invoke :cas [3 [19 24]] INFO [2019-12-17 12:36:05,932] jepsen worker 21 - jepsen.util 13701 :info :cas [7 [5 14]] :RequestCanceledException INFO [2019-12-17 12:36:05,933] jepsen worker 21 - jepsen.util 13791 :invoke :read [7 nil] INFO [2019-12-17 12:36:05,943] jepsen worker 13 - jepsen.util 9733 :info :write [4 10] :RequestCanceledException INFO [2019-12-17 12:36:05,944] jepsen worker 29 - jepsen.util 8759 :ok :write [9 12] INFO [2019-12-17 12:36:05,944] jepsen worker 29 - jepsen.util 8759 :invoke :read [9 nil] INFO [2019-12-17 12:36:05,944] jepsen worker 13 - jepsen.util 9823 :invoke :read [4 nil] INFO [2019-12-17 12:36:05,950] jepsen worker 30 - jepsen.util 10110 :ok :write [10 44] INFO [2019-12-17 12:36:05,950] jepsen worker 30 - jepsen.util 10110 :invoke :read [10 nil] INFO [2019-12-17 12:36:05,958] jepsen worker 63 - jepsen.util 9423 :ok :read [21 14] INFO [2019-12-17 12:36:05,958] jepsen worker 63 - jepsen.util 9423 :invoke :write [21 15] INFO [2019-12-17 12:36:05,961] jepsen worker 64 - jepsen.util 8074 :ok :write [21 27] INFO [2019-12-17 12:36:05,961] jepsen worker 64 - jepsen.util 8074 :invoke :read [21 nil] INFO [2019-12-17 12:36:05,962] jepsen worker 77 - jepsen.util 14747 :info :write [25 28] :RequestCanceledException INFO [2019-12-17 12:36:05,964] jepsen worker 77 - jepsen.util 14837 :invoke :read [25 nil] INFO [2019-12-17 12:36:05,970] jepsen worker 9 - jepsen.util 11079 :fail :cas [3 [19 24]] :ValueNotSwapFrom INFO [2019-12-17 12:36:05,970] jepsen worker 9 - jepsen.util 11079 :invoke :write [3 25] INFO [2019-12-17 12:36:05,982] jepsen worker 13 - jepsen.util 9823 :ok :read [4 49] INFO [2019-12-17 12:36:05,982] jepsen worker 13 - jepsen.util 9823 :invoke :write [4 37] INFO [2019-12-17 12:36:05,991] jepsen worker 64 - jepsen.util 8074 :ok :read [21 27] INFO [2019-12-17 12:36:05,991] jepsen worker 64 - jepsen.util 8074 :invoke :write [21 1] INFO [2019-12-17 12:36:06,010] jepsen worker 31 - jepsen.util 9121 :fail :cas [10 [25 14]] :ValueNotSwapFrom INFO [2019-12-17 12:36:06,010] jepsen worker 31 - jepsen.util 9121 :invoke :write [10 31] INFO [2019-12-17 12:36:06,014] jepsen worker 36 - jepsen.util 11106 :info :write [12 27] :RequestCanceledException INFO [2019-12-17 12:36:06,017] jepsen worker 36 - jepsen.util 11196 :invoke :cas [12 [31 44]] INFO [2019-12-17 12:36:06,027] jepsen worker 55 - jepsen.util 11035 :ok :read [18 5] INFO [2019-12-17 12:36:06,027] jepsen worker 55 - jepsen.util 11035 :invoke :cas [18 [8 41]] INFO [2019-12-17 12:36:06,031] jepsen worker 59 - jepsen.util 13199 :info :cas [19 [24 24]] :RequestCanceledException INFO [2019-12-17 12:36:06,031] jepsen worker 79 - jepsen.util 10699 :info :write [26 47] :RequestCanceledException INFO [2019-12-17 12:36:06,033] jepsen worker 79 - jepsen.util 10789 :invoke :cas [26 [16 37]] INFO [2019-12-17 12:36:06,033] jepsen worker 59 - jepsen.util 13289 :invoke :cas [19 [7 9]] INFO [2019-12-17 12:36:06,079] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":59755,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:36:06,079] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":59755,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:36:06,079] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":59758,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:36:06,079] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":59758,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:36:06,079] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":59759,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:36:06,079] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":59759,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:36:06,088] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:36:06,088] jepsen nemesis - couchbase.util Rebalance check count: 467 INFO [2019-12-17 12:36:06,088] jepsen nemesis - couchbase.util Rebalance stuck count: 448 INFO [2019-12-17 12:36:06,088] jepsen worker 29 - jepsen.util 8759 :ok :read [9 12] INFO [2019-12-17 12:36:06,088] jepsen worker 29 - jepsen.util 8759 :invoke :cas [9 [16 16]] INFO [2019-12-17 12:36:06,090] jepsen worker 30 - jepsen.util 10110 :ok :read [10 44] INFO [2019-12-17 12:36:06,091] jepsen worker 30 - jepsen.util 10110 :invoke :read [10 nil] INFO [2019-12-17 12:36:06,115] jepsen worker 34 - jepsen.util 14254 :info :write [11 5] :RequestCanceledException INFO [2019-12-17 12:36:06,116] jepsen worker 34 - jepsen.util 14344 :invoke :cas [11 [37 43]] INFO [2019-12-17 12:36:06,131] jepsen worker 86 - jepsen.util 15206 :info :write [28 4] :RequestCanceledException INFO [2019-12-17 12:36:06,131] jepsen worker 8 - jepsen.util 13688 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:36:06,131] jepsen worker 8 - jepsen.util 13688 :invoke :write [2 25] INFO [2019-12-17 12:36:06,132] jepsen worker 86 - jepsen.util 15296 :invoke :cas [28 [48 49]] INFO [2019-12-17 12:36:06,146] jepsen worker 84 - jepsen.util 13764 :info :write [28 28] :RequestCanceledException INFO [2019-12-17 12:36:06,146] jepsen worker 54 - jepsen.util 11304 :ok :write [18 20] INFO [2019-12-17 12:36:06,146] jepsen worker 54 - jepsen.util 11304 :invoke :read [18 nil] INFO [2019-12-17 12:36:06,147] jepsen worker 84 - jepsen.util 13854 :invoke :write [28 5] INFO [2019-12-17 12:36:06,154] jepsen worker 64 - jepsen.util 8074 :ok :write [21 1] INFO [2019-12-17 12:36:06,155] jepsen worker 64 - jepsen.util 8074 :invoke :cas [21 [34 39]] INFO [2019-12-17 12:36:06,162] jepsen worker 16 - jepsen.util 15496 :info :cas [5 [6 19]] :RequestCanceledException INFO [2019-12-17 12:36:06,162] jepsen worker 52 - jepsen.util 10222 :info :write [17 9] :RequestCanceledException INFO [2019-12-17 12:36:06,163] jepsen worker 16 - jepsen.util 15586 :invoke :read [5 nil] INFO [2019-12-17 12:36:06,163] jepsen worker 52 - jepsen.util 10312 :invoke :write [17 17] INFO [2019-12-17 12:36:06,183] jepsen worker 64 - jepsen.util 8074 :fail :cas [21 [34 39]] :ValueNotSwapFrom INFO [2019-12-17 12:36:06,183] jepsen worker 64 - jepsen.util 8074 :invoke :cas [21 [31 37]] INFO [2019-12-17 12:36:06,197] jepsen worker 43 - jepsen.util 14083 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:36:06,198] jepsen worker 43 - jepsen.util 14083 :invoke :write [14 40] INFO [2019-12-17 12:36:06,202] jepsen worker 63 - jepsen.util 9423 :ok :write [21 15] INFO [2019-12-17 12:36:06,202] jepsen worker 63 - jepsen.util 9423 :invoke :read [21 nil] INFO [2019-12-17 12:36:06,206] jepsen worker 64 - jepsen.util 8074 :fail :cas [21 [31 37]] :ValueNotSwapFrom INFO [2019-12-17 12:36:06,206] jepsen worker 64 - jepsen.util 8074 :invoke :read [21 nil] INFO [2019-12-17 12:36:06,213] jepsen worker 80 - jepsen.util 10160 :fail :cas [26 [47 14]] :ValueNotSwapFrom INFO [2019-12-17 12:36:06,213] jepsen worker 80 - jepsen.util 10160 :invoke :write [26 11] INFO [2019-12-17 12:36:06,216] jepsen worker 22 - jepsen.util 13882 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:36:06,216] jepsen worker 22 - jepsen.util 13882 :invoke :cas [7 [1 39]] INFO [2019-12-17 12:36:06,217] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":59776,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:36:06,217] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":59776,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:36:06,217] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":59778,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:36:06,217] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":59778,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:36:06,226] jepsen worker 3 - jepsen.util 10083 :ok :write [1 47] INFO [2019-12-17 12:36:06,226] jepsen worker 3 - jepsen.util 10083 :invoke :cas [1 [34 5]] INFO [2019-12-17 12:36:06,231] jepsen worker 46 - jepsen.util 10306 :info :write [15 36] :RequestCanceledException INFO [2019-12-17 12:36:06,231] jepsen worker 67 - jepsen.util 14197 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:36:06,232] jepsen worker 67 - jepsen.util 14197 :invoke :write [22 34] INFO [2019-12-17 12:36:06,233] jepsen worker 46 - jepsen.util 10396 :invoke :write [15 39] INFO [2019-12-17 12:36:06,245] jepsen worker 50 - jepsen.util 14720 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:36:06,246] jepsen worker 50 - jepsen.util 14720 :invoke :cas [16 [2 27]] INFO [2019-12-17 12:36:06,246] jepsen worker 29 - jepsen.util 8759 :fail :cas [9 [16 16]] :ValueNotSwapFrom INFO [2019-12-17 12:36:06,246] jepsen worker 29 - jepsen.util 8759 :invoke :write [9 20] INFO [2019-12-17 12:36:06,248] jepsen worker 13 - jepsen.util 9823 :ok :write [4 37] INFO [2019-12-17 12:36:06,249] jepsen worker 13 - jepsen.util 9823 :invoke :write [4 5] INFO [2019-12-17 12:36:06,269] jepsen worker 45 - jepsen.util 10485 :ok :write [15 25] INFO [2019-12-17 12:36:06,269] jepsen worker 45 - jepsen.util 10485 :invoke :write [15 44] INFO [2019-12-17 12:36:06,271] jepsen worker 3 - jepsen.util 10083 :fail :cas [1 [34 5]] :ValueNotSwapFrom INFO [2019-12-17 12:36:06,271] jepsen worker 3 - jepsen.util 10083 :invoke :read [1 nil] INFO [2019-12-17 12:36:06,273] jepsen worker 64 - jepsen.util 8074 :ok :read [21 15] INFO [2019-12-17 12:36:06,274] jepsen worker 64 - jepsen.util 8074 :invoke :write [21 14] INFO [2019-12-17 12:36:06,274] jepsen worker 63 - jepsen.util 9423 :ok :read [21 15] INFO [2019-12-17 12:36:06,274] jepsen worker 63 - jepsen.util 9423 :invoke :cas [21 [24 42]] INFO [2019-12-17 12:36:06,309] jepsen worker 54 - jepsen.util 11304 :ok :read [18 20] INFO [2019-12-17 12:36:06,309] jepsen worker 54 - jepsen.util 11304 :invoke :cas [18 [30 5]] INFO [2019-12-17 12:36:06,311] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":59788,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:36:06,311] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":59788,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:36:06,311] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":59792,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:36:06,311] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":59792,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:36:06,331] jepsen worker 20 - jepsen.util 10190 :fail :read [6 nil] :RequestTimeoutException INFO [2019-12-17 12:36:06,332] jepsen worker 20 - jepsen.util 10190 :invoke :cas [6 [49 34]] INFO [2019-12-17 12:36:06,363] jepsen worker 15 - jepsen.util 13155 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:36:06,363] jepsen worker 15 - jepsen.util 13155 :invoke :read [5 nil] INFO [2019-12-17 12:36:06,369] jepsen worker 9 - jepsen.util 11079 :ok :write [3 25] INFO [2019-12-17 12:36:06,369] jepsen worker 9 - jepsen.util 11079 :invoke :read [3 nil] INFO [2019-12-17 12:36:06,379] jepsen worker 79 - jepsen.util 10789 :fail :cas [26 [16 37]] :ValueNotSwapFrom INFO [2019-12-17 12:36:06,379] jepsen worker 79 - jepsen.util 10789 :invoke :read [26 nil] INFO [2019-12-17 12:36:06,383] jepsen worker 3 - jepsen.util 10083 :ok :read [1 47] INFO [2019-12-17 12:36:06,383] jepsen worker 3 - jepsen.util 10083 :invoke :cas [1 [43 17]] INFO [2019-12-17 12:36:06,393] jepsen worker 20 - jepsen.util 10190 :fail :cas [6 [49 34]] :ValueNotSwapFrom INFO [2019-12-17 12:36:06,393] jepsen worker 20 - jepsen.util 10190 :invoke :read [6 nil] INFO [2019-12-17 12:36:06,396] jepsen worker 64 - jepsen.util 8074 :ok :write [21 14] INFO [2019-12-17 12:36:06,396] jepsen worker 64 - jepsen.util 8074 :invoke :read [21 nil] INFO [2019-12-17 12:36:06,400] jepsen worker 29 - jepsen.util 8759 :ok :write [9 20] INFO [2019-12-17 12:36:06,400] jepsen worker 29 - jepsen.util 8759 :invoke :write [9 8] INFO [2019-12-17 12:36:06,402] jepsen worker 9 - jepsen.util 11079 :ok :read [3 25] INFO [2019-12-17 12:36:06,403] jepsen worker 9 - jepsen.util 11079 :invoke :cas [3 [24 37]] INFO [2019-12-17 12:36:06,425] jepsen worker 36 - jepsen.util 11196 :fail :cas [12 [31 44]] :ValueNotSwapFrom INFO [2019-12-17 12:36:06,425] jepsen worker 36 - jepsen.util 11196 :invoke :read [12 nil] INFO [2019-12-17 12:36:06,433] jepsen worker 57 - jepsen.util 13737 :info :cas [19 [24 13]] :RequestCanceledException INFO [2019-12-17 12:36:06,433] jepsen worker 37 - jepsen.util 9937 :ok :read [12 24] INFO [2019-12-17 12:36:06,433] jepsen worker 37 - jepsen.util 9937 :invoke :write [12 10] INFO [2019-12-17 12:36:06,434] jepsen worker 57 - jepsen.util 13827 :invoke :cas [19 [36 3]] INFO [2019-12-17 12:36:06,444] jepsen worker 31 - jepsen.util 9121 :ok :write [10 31] INFO [2019-12-17 12:36:06,444] jepsen worker 31 - jepsen.util 9121 :invoke :read [10 nil] INFO [2019-12-17 12:36:06,469] jepsen worker 64 - jepsen.util 8074 :ok :read [21 14] INFO [2019-12-17 12:36:06,469] jepsen worker 64 - jepsen.util 8074 :invoke :write [21 42] INFO [2019-12-17 12:36:06,470] jepsen worker 20 - jepsen.util 10190 :ok :read [6 14] INFO [2019-12-17 12:36:06,470] jepsen worker 20 - jepsen.util 10190 :invoke :write [6 43] INFO [2019-12-17 12:36:06,498] jepsen worker 42 - jepsen.util 14982 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:36:06,498] jepsen worker 2 - jepsen.util 13682 :info :write [0 19] :RequestCanceledException INFO [2019-12-17 12:36:06,498] jepsen worker 42 - jepsen.util 14982 :invoke :read [14 nil] INFO [2019-12-17 12:36:06,499] jepsen worker 2 - jepsen.util 13772 :invoke :read [0 nil] INFO [2019-12-17 12:36:06,500] jepsen worker 54 - jepsen.util 11304 :fail :cas [18 [30 5]] :ValueNotSwapFrom INFO [2019-12-17 12:36:06,501] jepsen worker 54 - jepsen.util 11304 :invoke :cas [18 [44 31]] INFO [2019-12-17 12:36:06,507] jepsen worker 31 - jepsen.util 9121 :ok :read [10 31] INFO [2019-12-17 12:36:06,507] jepsen worker 31 - jepsen.util 9121 :invoke :cas [10 [17 39]] INFO [2019-12-17 12:36:06,516] jepsen worker 60 - jepsen.util 15000 :info :write [20 12] :RequestCanceledException INFO [2019-12-17 12:36:06,517] jepsen worker 60 - jepsen.util 15090 :invoke :read [20 nil] INFO [2019-12-17 12:36:06,521] jepsen worker 9 - jepsen.util 11079 :fail :cas [3 [24 37]] :ValueNotSwapFrom INFO [2019-12-17 12:36:06,521] jepsen worker 9 - jepsen.util 11079 :invoke :write [3 29] INFO [2019-12-17 12:36:06,531] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":59815,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:36:06,531] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":59815,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:36:06,531] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":59818,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:36:06,531] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":59818,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:36:06,531] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":59819,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:36:06,532] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":59819,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:36:06,532] jepsen worker 41 - jepsen.util 15071 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:36:06,532] jepsen worker 72 - jepsen.util 10512 :fail :read [24 nil] :RequestTimeoutException INFO [2019-12-17 12:36:06,533] jepsen worker 72 - jepsen.util 10512 :invoke :read [24 nil] INFO [2019-12-17 12:36:06,533] jepsen worker 41 - jepsen.util 15071 :invoke :cas [13 [9 17]] INFO [2019-12-17 12:36:06,534] jepsen worker 55 - jepsen.util 11035 :fail :cas [18 [8 41]] :ValueNotSwapFrom INFO [2019-12-17 12:36:06,534] jepsen worker 55 - jepsen.util 11035 :invoke :read [18 nil] INFO [2019-12-17 12:36:06,535] jepsen worker 46 - jepsen.util 10396 :ok :write [15 39] INFO [2019-12-17 12:36:06,535] jepsen worker 46 - jepsen.util 10396 :invoke :write [15 44] INFO [2019-12-17 12:36:06,546] jepsen worker 17 - jepsen.util 13877 :info :cas [5 [24 36]] :RequestCanceledException INFO [2019-12-17 12:36:06,546] jepsen worker 69 - jepsen.util 10419 :info :write [23 44] :RequestCanceledException INFO [2019-12-17 12:36:06,547] jepsen worker 69 - jepsen.util 10509 :invoke :read [23 nil] INFO [2019-12-17 12:36:06,547] jepsen worker 17 - jepsen.util 13967 :invoke :write [5 43] INFO [2019-12-17 12:36:06,552] jepsen worker 31 - jepsen.util 9121 :fail :cas [10 [17 39]] :ValueNotSwapFrom INFO [2019-12-17 12:36:06,552] jepsen worker 31 - jepsen.util 9121 :invoke :write [10 17] INFO [2019-12-17 12:36:06,563] jepsen worker 6 - jepsen.util 14496 :info :write [2 47] :RequestCanceledException INFO [2019-12-17 12:36:06,564] jepsen worker 6 - jepsen.util 14586 :invoke :read [2 nil] INFO [2019-12-17 12:36:06,582] jepsen worker 3 - jepsen.util 10083 :fail :cas [1 [43 17]] :ValueNotSwapFrom INFO [2019-12-17 12:36:06,582] jepsen worker 3 - jepsen.util 10083 :invoke :read [1 nil] INFO [2019-12-17 12:36:06,594] jepsen worker 54 - jepsen.util 11304 :fail :cas [18 [44 31]] :ValueNotSwapFrom INFO [2019-12-17 12:36:06,595] jepsen worker 54 - jepsen.util 11304 :invoke :write [18 17] INFO [2019-12-17 12:36:06,611] jepsen worker 72 - jepsen.util 10512 :ok :read [24 15] INFO [2019-12-17 12:36:06,611] jepsen worker 72 - jepsen.util 10512 :invoke :cas [24 [49 32]] INFO [2019-12-17 12:36:06,622] jepsen worker 29 - jepsen.util 8759 :ok :write [9 8] INFO [2019-12-17 12:36:06,622] jepsen worker 29 - jepsen.util 8759 :invoke :cas [9 [0 38]] INFO [2019-12-17 12:36:06,631] jepsen worker 76 - jepsen.util 14386 :info :cas [25 [25 0]] :RequestCanceledException INFO [2019-12-17 12:36:06,632] jepsen worker 76 - jepsen.util 14476 :invoke :cas [25 [46 0]] INFO [2019-12-17 12:36:06,639] jepsen worker 3 - jepsen.util 10083 :ok :read [1 47] INFO [2019-12-17 12:36:06,640] jepsen worker 3 - jepsen.util 10083 :invoke :write [1 31] INFO [2019-12-17 12:36:06,643] jepsen worker 24 - jepsen.util 12624 :info :write [8 45] :RequestCanceledException INFO [2019-12-17 12:36:06,644] jepsen worker 24 - jepsen.util 12714 :invoke :write [8 2] INFO [2019-12-17 12:36:06,650] jepsen worker 29 - jepsen.util 8759 :fail :cas [9 [0 38]] :ValueNotSwapFrom INFO [2019-12-17 12:36:06,651] jepsen worker 29 - jepsen.util 8759 :invoke :read [9 nil] INFO [2019-12-17 12:36:06,676] jepsen worker 56 - jepsen.util 10856 :ok :read [18 20] INFO [2019-12-17 12:36:06,676] jepsen worker 56 - jepsen.util 10856 :invoke :read [18 nil] INFO [2019-12-17 12:36:06,701] jepsen worker 64 - jepsen.util 8074 :ok :write [21 42] INFO [2019-12-17 12:36:06,702] jepsen worker 64 - jepsen.util 8074 :invoke :read [21 nil] INFO [2019-12-17 12:36:06,703] jepsen worker 29 - jepsen.util 8759 :ok :read [9 8] INFO [2019-12-17 12:36:06,703] jepsen worker 29 - jepsen.util 8759 :invoke :cas [9 [32 4]] INFO [2019-12-17 12:36:06,716] jepsen worker 64 - jepsen.util 8074 :ok :read [21 42] INFO [2019-12-17 12:36:06,716] jepsen worker 64 - jepsen.util 8074 :invoke :write [21 47] INFO [2019-12-17 12:36:06,749] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":59845,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:36:06,749] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":59845,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:36:06,749] jepsen worker 29 - jepsen.util 8759 :fail :cas [9 [32 4]] :ValueNotSwapFrom INFO [2019-12-17 12:36:06,750] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":59849,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:36:06,750] jepsen worker 29 - jepsen.util 8759 :invoke :write [9 37] INFO [2019-12-17 12:36:06,750] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":59849,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:36:06,760] jepsen worker 5 - jepsen.util 9995 :ok :write [1 28] INFO [2019-12-17 12:36:06,760] jepsen worker 5 - jepsen.util 9995 :invoke :cas [1 [6 37]] INFO [2019-12-17 12:36:06,761] jepsen worker 70 - jepsen.util 11140 :ok :write [23 26] INFO [2019-12-17 12:36:06,761] jepsen worker 70 - jepsen.util 11140 :invoke :write [23 5] INFO [2019-12-17 12:36:06,776] jepsen worker 31 - jepsen.util 9121 :ok :write [10 17] INFO [2019-12-17 12:36:06,776] jepsen worker 31 - jepsen.util 9121 :invoke :read [10 nil] INFO [2019-12-17 12:36:06,802] jepsen worker 64 - jepsen.util 8074 :ok :write [21 47] INFO [2019-12-17 12:36:06,802] jepsen worker 64 - jepsen.util 8074 :invoke :read [21 nil] INFO [2019-12-17 12:36:06,803] jepsen worker 38 - jepsen.util 11108 :ok :write [12 39] INFO [2019-12-17 12:36:06,804] jepsen worker 38 - jepsen.util 11108 :invoke :write [12 13] INFO [2019-12-17 12:36:06,816] jepsen worker 19 - jepsen.util 10459 :fail :read [6 nil] :RequestTimeoutException INFO [2019-12-17 12:36:06,816] jepsen worker 19 - jepsen.util 10459 :invoke :cas [6 [13 41]] INFO [2019-12-17 12:36:06,826] jepsen worker 72 - jepsen.util 10512 :fail :cas [24 [49 32]] :ValueNotSwapFrom INFO [2019-12-17 12:36:06,826] jepsen worker 72 - jepsen.util 10512 :invoke :read [24 nil] INFO [2019-12-17 12:36:06,831] jepsen worker 44 - jepsen.util 14444 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:36:06,831] jepsen worker 35 - jepsen.util 14075 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:36:06,831] jepsen worker 28 - jepsen.util 8398 :info :write [9 7] :RequestCanceledException INFO [2019-12-17 12:36:06,831] jepsen worker 44 - jepsen.util 14444 :invoke :read [14 nil] INFO [2019-12-17 12:36:06,831] jepsen worker 35 - jepsen.util 14075 :invoke :write [11 3] INFO [2019-12-17 12:36:06,832] jepsen worker 64 - jepsen.util 8074 :ok :read [21 47] INFO [2019-12-17 12:36:06,832] jepsen worker 64 - jepsen.util 8074 :invoke :cas [21 [29 35]] INFO [2019-12-17 12:36:06,838] jepsen worker 28 - jepsen.util 8488 :invoke :cas [9 [8 39]] INFO [2019-12-17 12:36:06,845] jepsen worker 62 - jepsen.util 14552 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:36:06,845] jepsen worker 18 - jepsen.util 10728 :info :cas [6 [28 49]] :RequestCanceledException INFO [2019-12-17 12:36:06,845] jepsen worker 11 - jepsen.util 9911 :info :write [3 36] :RequestCanceledException INFO [2019-12-17 12:36:06,845] jepsen worker 62 - jepsen.util 14552 :invoke :read [20 nil] INFO [2019-12-17 12:36:06,846] jepsen worker 18 - jepsen.util 10818 :invoke :read [6 nil] INFO [2019-12-17 12:36:06,846] jepsen worker 11 - jepsen.util 10001 :invoke :write [3 40] INFO [2019-12-17 12:36:06,874] jepsen worker 36 - jepsen.util 11196 :ok :read [12 39] INFO [2019-12-17 12:36:06,874] jepsen worker 36 - jepsen.util 11196 :invoke :cas [12 [17 25]] INFO [2019-12-17 12:36:06,882] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":59875,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:36:06,882] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":59875,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:36:06,882] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":59877,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:36:06,882] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":59877,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:36:06,910] jepsen worker 87 - jepsen.util 9987 :ok :write [29 9] INFO [2019-12-17 12:36:06,910] jepsen worker 87 - jepsen.util 9987 :invoke :cas [29 [31 29]] INFO [2019-12-17 12:36:06,921] jepsen worker 19 - jepsen.util 10459 :fail :cas [6 [13 41]] :ValueNotSwapFrom INFO [2019-12-17 12:36:06,921] jepsen worker 19 - jepsen.util 10459 :invoke :read [6 nil] INFO [2019-12-17 12:36:06,935] jepsen worker 12 - jepsen.util 10812 :ok :write [4 28] INFO [2019-12-17 12:36:06,936] jepsen worker 12 - jepsen.util 10812 :invoke :cas [4 [1 9]] INFO [2019-12-17 12:36:06,949] jepsen worker 31 - jepsen.util 9121 :ok :read [10 17] INFO [2019-12-17 12:36:06,950] jepsen worker 31 - jepsen.util 9121 :invoke :write [10 15] INFO [2019-12-17 12:36:06,950] jepsen worker 52 - jepsen.util 10312 :ok :write [17 17] INFO [2019-12-17 12:36:06,950] jepsen worker 52 - jepsen.util 10312 :invoke :write [17 34] INFO [2019-12-17 12:36:06,964] jepsen worker 64 - jepsen.util 8074 :fail :cas [21 [29 35]] :ValueNotSwapFrom INFO [2019-12-17 12:36:06,964] jepsen worker 64 - jepsen.util 8074 :invoke :write [21 31] INFO [2019-12-17 12:36:06,971] jepsen worker 80 - jepsen.util 10160 :ok :write [26 11] INFO [2019-12-17 12:36:06,971] jepsen worker 80 - jepsen.util 10160 :invoke :cas [26 [15 29]] INFO [2019-12-17 12:36:06,998] jepsen worker 23 - jepsen.util 14153 :info :write [7 39] :RequestCanceledException INFO [2019-12-17 12:36:06,998] jepsen worker 88 - jepsen.util 10708 :info :write [29 41] :RequestCanceledException INFO [2019-12-17 12:36:06,999] jepsen worker 23 - jepsen.util 14243 :invoke :read [7 nil] INFO [2019-12-17 12:36:06,999] jepsen worker 88 - jepsen.util 10798 :invoke :cas [29 [1 40]] INFO [2019-12-17 12:36:07,004] jepsen worker 69 - jepsen.util 10509 :ok :read [23 26] INFO [2019-12-17 12:36:07,004] jepsen worker 69 - jepsen.util 10509 :invoke :read [23 nil] INFO [2019-12-17 12:36:07,015] jepsen worker 79 - jepsen.util 10789 :ok :read [26 11] INFO [2019-12-17 12:36:07,015] jepsen worker 79 - jepsen.util 10789 :invoke :read [26 nil] INFO [2019-12-17 12:36:07,016] jepsen worker 85 - jepsen.util 14935 :info :cas [28 [36 26]] :RequestCanceledException INFO [2019-12-17 12:36:07,017] jepsen worker 85 - jepsen.util 15025 :invoke :write [28 18] INFO [2019-12-17 12:36:07,022] jepsen worker 5 - jepsen.util 9995 :fail :cas [1 [6 37]] :ValueNotSwapFrom INFO [2019-12-17 12:36:07,022] jepsen worker 5 - jepsen.util 9995 :invoke :cas [1 [28 19]] INFO [2019-12-17 12:36:07,031] jepsen worker 47 - jepsen.util 10937 :info :write [15 38] :RequestCanceledException INFO [2019-12-17 12:36:07,031] jepsen worker 40 - jepsen.util 14620 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:36:07,031] jepsen worker 74 - jepsen.util 10154 :info :write [24 39] :RequestCanceledException INFO [2019-12-17 12:36:07,032] jepsen worker 40 - jepsen.util 14620 :invoke :read [13 nil] INFO [2019-12-17 12:36:07,033] jepsen worker 74 - jepsen.util 10244 :invoke :cas [24 [28 41]] INFO [2019-12-17 12:36:07,033] jepsen worker 47 - jepsen.util 11027 :invoke :write [15 29] INFO [2019-12-17 12:36:07,060] jepsen worker 36 - jepsen.util 11196 :fail :cas [12 [17 25]] :ValueNotSwapFrom INFO [2019-12-17 12:36:07,060] jepsen worker 36 - jepsen.util 11196 :invoke :read [12 nil] INFO [2019-12-17 12:36:07,064] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":59889,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:36:07,064] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":59889,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:36:07,064] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":59892,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:36:07,064] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":59892,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:36:07,065] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":59893,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:36:07,065] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":59893,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:36:07,070] jepsen worker 29 - jepsen.util 8759 :ok :write [9 37] INFO [2019-12-17 12:36:07,070] jepsen worker 29 - jepsen.util 8759 :invoke :read [9 nil] INFO [2019-12-17 12:36:07,098] jepsen worker 72 - jepsen.util 10512 :ok :read [24 15] INFO [2019-12-17 12:36:07,098] jepsen worker 72 - jepsen.util 10512 :invoke :write [24 35] INFO [2019-12-17 12:36:07,099] jepsen worker 14 - jepsen.util 10274 :ok :write [4 26] INFO [2019-12-17 12:36:07,099] jepsen worker 14 - jepsen.util 10274 :invoke :cas [4 [31 15]] INFO [2019-12-17 12:36:07,113] jepsen worker 4 - jepsen.util 9634 :ok :write [1 13] INFO [2019-12-17 12:36:07,113] jepsen worker 4 - jepsen.util 9634 :invoke :cas [1 [17 19]] INFO [2019-12-17 12:36:07,115] jepsen worker 88 - jepsen.util 10798 :fail :cas [29 [1 40]] :ValueNotSwapFrom INFO [2019-12-17 12:36:07,116] jepsen worker 88 - jepsen.util 10798 :invoke :write [29 43] INFO [2019-12-17 12:36:07,132] jepsen worker 32 - jepsen.util 10472 :info :write [10 5] :RequestCanceledException INFO [2019-12-17 12:36:07,133] jepsen worker 32 - jepsen.util 10562 :invoke :write [10 17] INFO [2019-12-17 12:36:07,134] jepsen worker 37 - jepsen.util 9937 :ok :write [12 10] INFO [2019-12-17 12:36:07,134] jepsen worker 37 - jepsen.util 9937 :invoke :write [12 33] INFO [2019-12-17 12:36:07,140] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:36:07,140] jepsen nemesis - couchbase.util Rebalance check count: 468 INFO [2019-12-17 12:36:07,140] jepsen nemesis - couchbase.util Rebalance stuck count: 449 INFO [2019-12-17 12:36:07,162] jepsen worker 10 - jepsen.util 10360 :info :cas [3 [2 1]] :RequestCanceledException INFO [2019-12-17 12:36:07,162] jepsen worker 53 - jepsen.util 9773 :info :write [17 36] :RequestCanceledException INFO [2019-12-17 12:36:07,164] jepsen worker 79 - jepsen.util 10789 :ok :read [26 11] INFO [2019-12-17 12:36:07,165] jepsen worker 79 - jepsen.util 10789 :invoke :write [26 43] INFO [2019-12-17 12:36:07,166] jepsen worker 10 - jepsen.util 10450 :invoke :cas [3 [26 10]] INFO [2019-12-17 12:36:07,169] jepsen worker 19 - jepsen.util 10459 :ok :read [6 14] INFO [2019-12-17 12:36:07,169] jepsen worker 19 - jepsen.util 10459 :invoke :read [6 nil] INFO [2019-12-17 12:36:07,170] jepsen worker 53 - jepsen.util 9863 :invoke :read [17 nil] INFO [2019-12-17 12:36:07,188] jepsen worker 29 - jepsen.util 8759 :ok :read [9 37] INFO [2019-12-17 12:36:07,188] jepsen worker 29 - jepsen.util 8759 :invoke :read [9 nil] INFO [2019-12-17 12:36:07,192] jepsen worker 18 - jepsen.util 10818 :ok :read [6 14] INFO [2019-12-17 12:36:07,192] jepsen worker 18 - jepsen.util 10818 :invoke :cas [6 [11 19]] INFO [2019-12-17 12:36:07,192] jepsen worker 80 - jepsen.util 10160 :fail :cas [26 [15 29]] :ValueNotSwapFrom INFO [2019-12-17 12:36:07,192] jepsen worker 80 - jepsen.util 10160 :invoke :write [26 17] INFO [2019-12-17 12:36:07,203] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":59917,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:36:07,203] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":59917,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:36:07,203] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":59918,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:36:07,203] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":59918,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:36:07,229] jepsen worker 29 - jepsen.util 8759 :ok :read [9 37] INFO [2019-12-17 12:36:07,229] jepsen worker 29 - jepsen.util 8759 :invoke :read [9 nil] INFO [2019-12-17 12:36:07,229] jepsen worker 56 - jepsen.util 10856 :ok :read [18 20] INFO [2019-12-17 12:36:07,229] jepsen worker 56 - jepsen.util 10856 :invoke :read [18 nil] INFO [2019-12-17 12:36:07,230] jepsen worker 64 - jepsen.util 8074 :ok :write [21 31] INFO [2019-12-17 12:36:07,230] jepsen worker 64 - jepsen.util 8074 :invoke :write [21 8] INFO [2019-12-17 12:36:07,232] jepsen worker 49 - jepsen.util 13999 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:36:07,232] jepsen worker 66 - jepsen.util 13746 :info :write [22 13] :RequestCanceledException INFO [2019-12-17 12:36:07,232] jepsen worker 7 - jepsen.util 14227 :info :cas [2 [2 6]] :RequestCanceledException INFO [2019-12-17 12:36:07,232] jepsen worker 49 - jepsen.util 13999 :invoke :read [16 nil] INFO [2019-12-17 12:36:07,233] jepsen worker 66 - jepsen.util 13836 :invoke :write [22 27] INFO [2019-12-17 12:36:07,233] jepsen worker 7 - jepsen.util 14317 :invoke :read [2 nil] INFO [2019-12-17 12:36:07,239] jepsen worker 69 - jepsen.util 10509 :ok :read [23 26] INFO [2019-12-17 12:36:07,239] jepsen worker 69 - jepsen.util 10509 :invoke :read [23 nil] INFO [2019-12-17 12:36:07,245] jepsen worker 83 - jepsen.util 14213 :info :write [27 48] :RequestCanceledException INFO [2019-12-17 12:36:07,249] jepsen worker 83 - jepsen.util 14303 :invoke :write [27 7] INFO [2019-12-17 12:36:07,252] jepsen worker 5 - jepsen.util 9995 :fail :cas [1 [28 19]] :ValueNotSwapFrom INFO [2019-12-17 12:36:07,252] jepsen worker 5 - jepsen.util 9995 :invoke :write [1 15] INFO [2019-12-17 12:36:07,254] jepsen worker 20 - jepsen.util 10190 :ok :write [6 43] INFO [2019-12-17 12:36:07,254] jepsen worker 20 - jepsen.util 10190 :invoke :cas [6 [43 21]] INFO [2019-12-17 12:36:07,280] jepsen worker 12 - jepsen.util 10812 :fail :cas [4 [1 9]] :ValueNotSwapFrom INFO [2019-12-17 12:36:07,280] jepsen worker 12 - jepsen.util 10812 :invoke :cas [4 [26 36]] INFO [2019-12-17 12:36:07,284] jepsen worker 29 - jepsen.util 8759 :ok :read [9 37] INFO [2019-12-17 12:36:07,284] jepsen worker 29 - jepsen.util 8759 :invoke :write [9 13] INFO [2019-12-17 12:36:07,285] jepsen worker 19 - jepsen.util 10459 :ok :read [6 14] INFO [2019-12-17 12:36:07,285] jepsen worker 19 - jepsen.util 10459 :invoke :write [6 16] INFO [2019-12-17 12:36:07,294] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":59925,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:36:07,294] jepsen worker 0 - jepsen.util 14850 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:36:07,294] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":59925,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:36:07,294] jepsen worker 61 - jepsen.util 14821 :info :cas [20 [26 26]] :RequestCanceledException INFO [2019-12-17 12:36:07,294] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":59928,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:36:07,294] jepsen worker 0 - jepsen.util 14850 :invoke :write [0 49] INFO [2019-12-17 12:36:07,295] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":59928,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:36:07,295] jepsen worker 61 - jepsen.util 14911 :invoke :cas [20 [5 25]] INFO [2019-12-17 12:36:07,298] jepsen worker 4 - jepsen.util 9634 :fail :cas [1 [17 19]] :ValueNotSwapFrom INFO [2019-12-17 12:36:07,298] jepsen worker 4 - jepsen.util 9634 :invoke :write [1 35] INFO [2019-12-17 12:36:07,315] jepsen worker 56 - jepsen.util 10856 :ok :read [18 20] INFO [2019-12-17 12:36:07,315] jepsen worker 56 - jepsen.util 10856 :invoke :write [18 2] INFO [2019-12-17 12:36:07,326] jepsen worker 38 - jepsen.util 11108 :ok :write [12 13] INFO [2019-12-17 12:36:07,326] jepsen worker 38 - jepsen.util 11108 :invoke :cas [12 [49 8]] INFO [2019-12-17 12:36:07,331] jepsen worker 26 - jepsen.util 14696 :info :write [8 40] :RequestCanceledException INFO [2019-12-17 12:36:07,332] jepsen worker 26 - jepsen.util 14786 :invoke :cas [8 [21 22]] INFO [2019-12-17 12:36:07,363] jepsen worker 1 - jepsen.util 14131 :info :cas [0 [48 10]] :RequestCanceledException INFO [2019-12-17 12:36:07,364] jepsen worker 1 - jepsen.util 14221 :invoke :write [0 15] INFO [2019-12-17 12:36:07,364] jepsen worker 14 - jepsen.util 10274 :fail :cas [4 [31 15]] :ValueNotSwapFrom INFO [2019-12-17 12:36:07,365] jepsen worker 14 - jepsen.util 10274 :invoke :cas [4 [30 19]] INFO [2019-12-17 12:36:07,366] jepsen worker 55 - jepsen.util 11035 :ok :read [18 20] INFO [2019-12-17 12:36:07,367] jepsen worker 55 - jepsen.util 11035 :invoke :cas [18 [40 39]] INFO [2019-12-17 12:36:07,373] jepsen worker 71 - jepsen.util 11231 :ok :write [23 14] INFO [2019-12-17 12:36:07,373] jepsen worker 71 - jepsen.util 11231 :invoke :read [23 nil] INFO [2019-12-17 12:36:07,381] jepsen worker 64 - jepsen.util 8074 :ok :write [21 8] INFO [2019-12-17 12:36:07,381] jepsen worker 64 - jepsen.util 8074 :invoke :cas [21 [7 6]] INFO [2019-12-17 12:36:07,407] jepsen worker 64 - jepsen.util 8074 :fail :cas [21 [7 6]] :ValueNotSwapFrom INFO [2019-12-17 12:36:07,407] jepsen worker 64 - jepsen.util 8074 :invoke :read [21 nil] INFO [2019-12-17 12:36:07,419] jepsen worker 38 - jepsen.util 11108 :fail :cas [12 [49 8]] :ValueNotSwapFrom INFO [2019-12-17 12:36:07,419] jepsen worker 38 - jepsen.util 11108 :invoke :read [12 nil] INFO [2019-12-17 12:36:07,438] jepsen worker 29 - jepsen.util 8759 :ok :write [9 13] INFO [2019-12-17 12:36:07,439] jepsen worker 29 - jepsen.util 8759 :invoke :cas [9 [33 26]] INFO [2019-12-17 12:36:07,442] jepsen worker 64 - jepsen.util 8074 :ok :read [21 8] INFO [2019-12-17 12:36:07,442] jepsen worker 64 - jepsen.util 8074 :invoke :cas [21 [22 37]] INFO [2019-12-17 12:36:07,443] jepsen worker 71 - jepsen.util 11231 :ok :read [23 14] INFO [2019-12-17 12:36:07,444] jepsen worker 71 - jepsen.util 11231 :invoke :write [23 14] INFO [2019-12-17 12:36:07,462] jepsen worker 78 - jepsen.util 10248 :info :write [26 13] :RequestCanceledException INFO [2019-12-17 12:36:07,462] jepsen worker 51 - jepsen.util 9861 :info :write [17 30] :RequestCanceledException INFO [2019-12-17 12:36:07,463] jepsen worker 78 - jepsen.util 10338 :invoke :read [26 nil] INFO [2019-12-17 12:36:07,463] jepsen worker 51 - jepsen.util 9951 :invoke :cas [17 [5 2]] INFO [2019-12-17 12:36:07,476] jepsen worker 64 - jepsen.util 8074 :fail :cas [21 [22 37]] :ValueNotSwapFrom INFO [2019-12-17 12:36:07,476] jepsen worker 64 - jepsen.util 8074 :invoke :read [21 nil] INFO [2019-12-17 12:36:07,483] jepsen worker 29 - jepsen.util 8759 :fail :cas [9 [33 26]] :ValueNotSwapFrom INFO [2019-12-17 12:36:07,483] jepsen worker 29 - jepsen.util 8759 :invoke :read [9 nil] INFO [2019-12-17 12:36:07,498] jepsen worker 64 - jepsen.util 8074 :ok :read [21 8] INFO [2019-12-17 12:36:07,498] jepsen worker 75 - jepsen.util 13575 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:36:07,498] jepsen worker 75 - jepsen.util 13575 :invoke :cas [25 [39 3]] INFO [2019-12-17 12:36:07,499] jepsen worker 64 - jepsen.util 8074 :invoke :write [21 47] INFO [2019-12-17 12:36:07,499] jepsen worker 36 - jepsen.util 11196 :ok :read [12 13] INFO [2019-12-17 12:36:07,499] jepsen worker 36 - jepsen.util 11196 :invoke :read [12 nil] INFO [2019-12-17 12:36:07,510] jepsen worker 29 - jepsen.util 8759 :ok :read [9 13] INFO [2019-12-17 12:36:07,511] jepsen worker 29 - jepsen.util 8759 :invoke :cas [9 [4 12]] INFO [2019-12-17 12:36:07,514] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":59958,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:36:07,514] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":59958,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:36:07,526] jepsen worker 29 - jepsen.util 8759 :fail :cas [9 [4 12]] :ValueNotSwapFrom INFO [2019-12-17 12:36:07,526] jepsen worker 29 - jepsen.util 8759 :invoke :read [9 nil] INFO [2019-12-17 12:36:07,531] jepsen worker 25 - jepsen.util 15145 :info :write [8 21] :RequestCanceledException INFO [2019-12-17 12:36:07,532] jepsen worker 89 - jepsen.util 10619 :info :write [29 13] :RequestCanceledException INFO [2019-12-17 12:36:07,532] jepsen worker 25 - jepsen.util 15235 :invoke :write [8 35] INFO [2019-12-17 12:36:07,533] jepsen worker 89 - jepsen.util 10709 :invoke :write [29 7] INFO [2019-12-17 12:36:07,556] jepsen worker 29 - jepsen.util 8759 :ok :read [9 13] INFO [2019-12-17 12:36:07,556] jepsen worker 29 - jepsen.util 8759 :invoke :cas [9 [49 12]] INFO [2019-12-17 12:36:07,573] jepsen worker 29 - jepsen.util 8759 :fail :cas [9 [49 12]] :ValueNotSwapFrom INFO [2019-12-17 12:36:07,574] jepsen worker 29 - jepsen.util 8759 :invoke :write [9 25] INFO [2019-12-17 12:36:07,574] jepsen worker 64 - jepsen.util 8074 :ok :write [21 47] INFO [2019-12-17 12:36:07,574] jepsen worker 64 - jepsen.util 8074 :invoke :read [21 nil] INFO [2019-12-17 12:36:07,598] jepsen worker 58 - jepsen.util 14458 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:36:07,598] jepsen worker 68 - jepsen.util 13568 :info :write [22 4] :RequestCanceledException INFO [2019-12-17 12:36:07,599] jepsen worker 58 - jepsen.util 14458 :invoke :cas [19 [29 11]] INFO [2019-12-17 12:36:07,599] jepsen worker 68 - jepsen.util 13658 :invoke :cas [22 [34 22]] INFO [2019-12-17 12:36:07,600] jepsen worker 69 - jepsen.util 10509 :ok :read [23 14] INFO [2019-12-17 12:36:07,601] jepsen worker 69 - jepsen.util 10509 :invoke :write [23 16] INFO [2019-12-17 12:36:07,615] jepsen worker 64 - jepsen.util 8074 :ok :read [21 47] INFO [2019-12-17 12:36:07,616] jepsen worker 64 - jepsen.util 8074 :invoke :read [21 nil] INFO [2019-12-17 12:36:07,620] jepsen worker 38 - jepsen.util 11108 :ok :read [12 13] INFO [2019-12-17 12:36:07,621] jepsen worker 38 - jepsen.util 11108 :invoke :write [12 14] INFO [2019-12-17 12:36:07,630] jepsen worker 64 - jepsen.util 8074 :ok :read [21 47] INFO [2019-12-17 12:36:07,630] jepsen worker 64 - jepsen.util 8074 :invoke :read [21 nil] INFO [2019-12-17 12:36:07,652] jepsen worker 36 - jepsen.util 11196 :ok :read [12 13] INFO [2019-12-17 12:36:07,652] jepsen worker 36 - jepsen.util 11196 :invoke :write [12 14] INFO [2019-12-17 12:36:07,656] jepsen worker 64 - jepsen.util 8074 :ok :read [21 47] INFO [2019-12-17 12:36:07,657] jepsen worker 46 - jepsen.util 10396 :ok :write [15 44] INFO [2019-12-17 12:36:07,657] jepsen worker 31 - jepsen.util 9121 :ok :write [10 15] INFO [2019-12-17 12:36:07,657] jepsen worker 64 - jepsen.util 8074 :invoke :cas [21 [25 46]] INFO [2019-12-17 12:36:07,657] jepsen worker 46 - jepsen.util 10396 :invoke :cas [15 [30 41]] INFO [2019-12-17 12:36:07,657] jepsen worker 31 - jepsen.util 9121 :invoke :read [10 nil] INFO [2019-12-17 12:36:07,657] jepsen worker 11 - jepsen.util 10001 :ok :write [3 40] INFO [2019-12-17 12:36:07,657] jepsen worker 11 - jepsen.util 10001 :invoke :read [3 nil] INFO [2019-12-17 12:36:07,663] jepsen worker 82 - jepsen.util 14302 :info :cas [27 [16 47]] :RequestCanceledException INFO [2019-12-17 12:36:07,664] jepsen worker 82 - jepsen.util 14392 :invoke :cas [27 [32 22]] INFO [2019-12-17 12:36:07,664] jepsen worker 29 - jepsen.util 8759 :ok :write [9 25] INFO [2019-12-17 12:36:07,664] jepsen worker 29 - jepsen.util 8759 :invoke :read [9 nil] INFO [2019-12-17 12:36:07,692] jepsen worker 18 - jepsen.util 10818 :fail :cas [6 [11 19]] :ValueNotSwapFrom INFO [2019-12-17 12:36:07,692] jepsen worker 18 - jepsen.util 10818 :invoke :cas [6 [12 45]] INFO [2019-12-17 12:36:07,719] jepsen worker 10 - jepsen.util 10450 :fail :cas [3 [26 10]] :ValueNotSwapFrom INFO [2019-12-17 12:36:07,719] jepsen worker 10 - jepsen.util 10450 :invoke :read [3 nil] INFO [2019-12-17 12:36:07,728] jepsen worker 78 - jepsen.util 10338 :ok :read [26 11] INFO [2019-12-17 12:36:07,728] jepsen worker 78 - jepsen.util 10338 :invoke :cas [26 [49 32]] INFO [2019-12-17 12:36:07,731] jepsen worker 48 - jepsen.util 13008 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:36:07,732] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":59988,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:36:07,732] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":59988,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:36:07,732] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":59991,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:36:07,732] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":59991,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:36:07,732] jepsen worker 48 - jepsen.util 13008 :invoke :cas [16 [10 44]] INFO [2019-12-17 12:36:07,751] jepsen worker 78 - jepsen.util 10338 :fail :cas [26 [49 32]] :ValueNotSwapFrom INFO [2019-12-17 12:36:07,751] jepsen worker 78 - jepsen.util 10338 :invoke :cas [26 [43 27]] INFO [2019-12-17 12:36:07,753] jepsen worker 53 - jepsen.util 9863 :ok :read [17 17] INFO [2019-12-17 12:36:07,753] jepsen worker 53 - jepsen.util 9863 :invoke :cas [17 [11 17]] INFO [2019-12-17 12:36:07,753] jepsen worker 29 - jepsen.util 8759 :ok :read [9 25] INFO [2019-12-17 12:36:07,754] jepsen worker 29 - jepsen.util 8759 :invoke :read [9 nil] INFO [2019-12-17 12:36:07,761] jepsen worker 31 - jepsen.util 9121 :ok :read [10 15] INFO [2019-12-17 12:36:07,761] jepsen worker 31 - jepsen.util 9121 :invoke :cas [10 [3 11]] INFO [2019-12-17 12:36:07,766] jepsen worker 64 - jepsen.util 8074 :fail :cas [21 [25 46]] :ValueNotSwapFrom INFO [2019-12-17 12:36:07,766] jepsen worker 46 - jepsen.util 10396 :fail :cas [15 [30 41]] :ValueNotSwapFrom INFO [2019-12-17 12:36:07,766] jepsen worker 64 - jepsen.util 8074 :invoke :read [21 nil] INFO [2019-12-17 12:36:07,766] jepsen worker 46 - jepsen.util 10396 :invoke :cas [15 [44 31]] INFO [2019-12-17 12:36:07,773] jepsen worker 87 - jepsen.util 9987 :fail :cas [29 [31 29]] :ValueNotSwapFrom INFO [2019-12-17 12:36:07,773] jepsen worker 87 - jepsen.util 9987 :invoke :write [29 6] INFO [2019-12-17 12:36:07,798] jepsen worker 78 - jepsen.util 10338 :fail :cas [26 [43 27]] :ValueNotSwapFrom INFO [2019-12-17 12:36:07,798] jepsen worker 78 - jepsen.util 10338 :invoke :write [26 5] INFO [2019-12-17 12:36:07,816] jepsen worker 73 - jepsen.util 9703 :fail :read [24 nil] :RequestTimeoutException INFO [2019-12-17 12:36:07,816] jepsen worker 27 - jepsen.util 9927 :info :write [9 34] :RequestCanceledException INFO [2019-12-17 12:36:07,816] jepsen worker 73 - jepsen.util 9703 :invoke :cas [24 [33 34]] INFO [2019-12-17 12:36:07,817] jepsen worker 27 - jepsen.util 10017 :invoke :write [9 44] INFO [2019-12-17 12:36:07,830] jepsen worker 18 - jepsen.util 10818 :fail :cas [6 [12 45]] :ValueNotSwapFrom INFO [2019-12-17 12:36:07,830] jepsen worker 18 - jepsen.util 10818 :invoke :write [6 20] INFO [2019-12-17 12:36:07,830] jepsen worker 33 - jepsen.util 14163 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:36:07,831] jepsen worker 33 - jepsen.util 14163 :invoke :cas [11 [45 39]] INFO [2019-12-17 12:36:07,848] jepsen worker 29 - jepsen.util 8759 :ok :read [9 25] INFO [2019-12-17 12:36:07,848] jepsen worker 29 - jepsen.util 8759 :invoke :cas [9 [36 32]] INFO [2019-12-17 12:36:07,855] jepsen worker 10 - jepsen.util 10450 :ok :read [3 40] INFO [2019-12-17 12:36:07,857] jepsen worker 10 - jepsen.util 10450 :invoke :read [3 nil] INFO [2019-12-17 12:36:07,867] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":60018,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:36:07,867] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":60018,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:36:07,867] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":60021,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:36:07,867] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":60021,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:36:07,867] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":60022,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:36:07,867] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":60022,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:36:07,873] jepsen worker 64 - jepsen.util 8074 :ok :read [21 47] INFO [2019-12-17 12:36:07,873] jepsen worker 64 - jepsen.util 8074 :invoke :write [21 17] INFO [2019-12-17 12:36:07,878] jepsen worker 55 - jepsen.util 11035 :fail :cas [18 [40 39]] :ValueNotSwapFrom INFO [2019-12-17 12:36:07,878] jepsen worker 55 - jepsen.util 11035 :invoke :read [18 nil] INFO [2019-12-17 12:36:07,895] jepsen worker 73 - jepsen.util 9703 :fail :cas [24 [33 34]] :ValueNotSwapFrom INFO [2019-12-17 12:36:07,896] jepsen worker 73 - jepsen.util 9703 :invoke :cas [24 [17 13]] INFO [2019-12-17 12:36:07,903] jepsen worker 31 - jepsen.util 9121 :fail :cas [10 [3 11]] :ValueNotSwapFrom INFO [2019-12-17 12:36:07,904] jepsen worker 31 - jepsen.util 9121 :invoke :cas [10 [8 19]] INFO [2019-12-17 12:36:07,916] jepsen worker 81 - jepsen.util 13761 :info :cas [27 [14 14]] :RequestCanceledException INFO [2019-12-17 12:36:07,917] jepsen worker 81 - jepsen.util 13851 :invoke :cas [27 [43 2]] INFO [2019-12-17 12:36:07,930] jepsen worker 51 - jepsen.util 9951 :fail :cas [17 [5 2]] :ValueNotSwapFrom INFO [2019-12-17 12:36:07,930] jepsen worker 51 - jepsen.util 9951 :invoke :write [17 22] INFO [2019-12-17 12:36:07,930] jepsen worker 10 - jepsen.util 10450 :ok :read [3 40] INFO [2019-12-17 12:36:07,930] jepsen worker 10 - jepsen.util 10450 :invoke :read [3 nil] INFO [2019-12-17 12:36:07,932] jepsen worker 39 - jepsen.util 13719 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:36:07,932] jepsen worker 39 - jepsen.util 13719 :invoke :write [13 44] INFO [2019-12-17 12:36:07,944] jepsen worker 65 - jepsen.util 8885 :info :write [21 31] :RequestCanceledException INFO [2019-12-17 12:36:07,945] jepsen worker 20 - jepsen.util 10190 :ok :cas [6 [43 21]] INFO [2019-12-17 12:36:07,945] jepsen worker 20 - jepsen.util 10190 :invoke :read [6 nil] INFO [2019-12-17 12:36:07,946] jepsen worker 65 - jepsen.util 8975 :invoke :write [21 11] INFO [2019-12-17 12:36:07,986] jepsen worker 20 - jepsen.util 10190 :ok :read [6 21] INFO [2019-12-17 12:36:07,986] jepsen worker 20 - jepsen.util 10190 :invoke :cas [6 [20 23]] INFO [2019-12-17 12:36:08,007] jepsen worker 20 - jepsen.util 10190 :fail :cas [6 [20 23]] :ValueNotSwapFrom INFO [2019-12-17 12:36:08,008] jepsen worker 20 - jepsen.util 10190 :invoke :cas [6 [48 1]] INFO [2019-12-17 12:36:08,008] jepsen worker 64 - jepsen.util 8074 :ok :write [21 17] INFO [2019-12-17 12:36:08,008] jepsen worker 64 - jepsen.util 8074 :invoke :cas [21 [22 46]] INFO [2019-12-17 12:36:08,015] jepsen worker 77 - jepsen.util 14837 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:36:08,016] jepsen worker 77 - jepsen.util 14837 :invoke :cas [25 [35 46]] INFO [2019-12-17 12:36:08,020] jepsen worker 27 - jepsen.util 10017 :ok :write [9 44] INFO [2019-12-17 12:36:08,020] jepsen worker 27 - jepsen.util 10017 :invoke :write [9 21] INFO [2019-12-17 12:36:08,031] jepsen worker 21 - jepsen.util 13791 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:36:08,031] jepsen worker 21 - jepsen.util 13791 :invoke :read [7 nil] INFO [2019-12-17 12:36:08,035] jepsen worker 32 - jepsen.util 10562 :ok :write [10 17] INFO [2019-12-17 12:36:08,035] jepsen worker 32 - jepsen.util 10562 :invoke :write [10 41] INFO [2019-12-17 12:36:08,050] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":60038,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:36:08,050] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":60038,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:36:08,060] jepsen worker 20 - jepsen.util 10190 :fail :cas [6 [48 1]] :ValueNotSwapFrom INFO [2019-12-17 12:36:08,060] jepsen worker 20 - jepsen.util 10190 :invoke :read [6 nil] INFO [2019-12-17 12:36:08,062] jepsen worker 59 - jepsen.util 13289 :info :cas [19 [7 9]] :RequestCanceledException INFO [2019-12-17 12:36:08,063] jepsen worker 59 - jepsen.util 13379 :invoke :cas [19 [42 33]] INFO [2019-12-17 12:36:08,070] jepsen worker 64 - jepsen.util 8074 :fail :cas [21 [22 46]] :ValueNotSwapFrom INFO [2019-12-17 12:36:08,070] jepsen worker 64 - jepsen.util 8074 :invoke :cas [21 [22 33]] INFO [2019-12-17 12:36:08,094] jepsen worker 11 - jepsen.util 10001 :ok :read [3 40] INFO [2019-12-17 12:36:08,094] jepsen worker 11 - jepsen.util 10001 :invoke :read [3 nil] INFO [2019-12-17 12:36:08,125] jepsen worker 46 - jepsen.util 10396 :ok :cas [15 [44 31]] INFO [2019-12-17 12:36:08,125] jepsen worker 46 - jepsen.util 10396 :invoke :write [15 3] INFO [2019-12-17 12:36:08,131] jepsen worker 18 - jepsen.util 10818 :ok :write [6 20] INFO [2019-12-17 12:36:08,131] jepsen worker 18 - jepsen.util 10818 :invoke :cas [6 [21 24]] INFO [2019-12-17 12:36:08,132] jepsen worker 34 - jepsen.util 14344 :info :cas [11 [37 43]] :RequestCanceledException INFO [2019-12-17 12:36:08,132] jepsen worker 30 - jepsen.util 10110 :fail :read [10 nil] :RequestTimeoutException INFO [2019-12-17 12:36:08,133] jepsen worker 30 - jepsen.util 10110 :invoke :read [10 nil] INFO [2019-12-17 12:36:08,133] jepsen worker 34 - jepsen.util 14434 :invoke :cas [11 [39 43]] INFO [2019-12-17 12:36:08,152] jepsen worker 27 - jepsen.util 10017 :ok :write [9 21] INFO [2019-12-17 12:36:08,152] jepsen worker 27 - jepsen.util 10017 :invoke :write [9 0] INFO [2019-12-17 12:36:08,161] jepsen worker 18 - jepsen.util 10818 :fail :cas [6 [21 24]] :ValueNotSwapFrom INFO [2019-12-17 12:36:08,161] jepsen worker 18 - jepsen.util 10818 :invoke :cas [6 [31 43]] INFO [2019-12-17 12:36:08,162] jepsen worker 64 - jepsen.util 8074 :fail :cas [21 [22 33]] :ValueNotSwapFrom INFO [2019-12-17 12:36:08,163] jepsen worker 64 - jepsen.util 8074 :invoke :read [21 nil] INFO [2019-12-17 12:36:08,163] jepsen worker 20 - jepsen.util 10190 :ok :read [6 21] INFO [2019-12-17 12:36:08,164] jepsen worker 20 - jepsen.util 10190 :invoke :cas [6 [22 31]] INFO [2019-12-17 12:36:08,172] jepsen worker 30 - jepsen.util 10110 :ok :read [10 17] INFO [2019-12-17 12:36:08,172] jepsen worker 30 - jepsen.util 10110 :invoke :read [10 nil] INFO [2019-12-17 12:36:08,181] jepsen worker 14 - jepsen.util 10274 :fail :cas [4 [30 19]] :ValueNotSwapFrom INFO [2019-12-17 12:36:08,182] jepsen worker 14 - jepsen.util 10274 :invoke :read [4 nil] INFO [2019-12-17 12:36:08,185] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:36:08,185] jepsen nemesis - couchbase.util Rebalance check count: 469 INFO [2019-12-17 12:36:08,185] jepsen nemesis - couchbase.util Rebalance stuck count: 450 INFO [2019-12-17 12:36:08,198] jepsen worker 86 - jepsen.util 15296 :info :cas [28 [48 49]] :RequestCanceledException INFO [2019-12-17 12:36:08,198] jepsen worker 86 - jepsen.util 15386 :invoke :read [28 nil] INFO [2019-12-17 12:36:08,213] jepsen worker 84 - jepsen.util 13854 :info :write [28 5] :RequestCanceledException INFO [2019-12-17 12:36:08,214] jepsen worker 84 - jepsen.util 13944 :invoke :cas [28 [19 6]] INFO [2019-12-17 12:36:08,220] jepsen worker 4 - jepsen.util 9634 :ok :write [1 35] INFO [2019-12-17 12:36:08,220] jepsen worker 4 - jepsen.util 9634 :invoke :read [1 nil] INFO [2019-12-17 12:36:08,231] jepsen worker 16 - jepsen.util 15586 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:36:08,231] jepsen worker 8 - jepsen.util 13688 :info :write [2 25] :RequestCanceledException INFO [2019-12-17 12:36:08,232] jepsen worker 16 - jepsen.util 15586 :invoke :cas [5 [16 37]] INFO [2019-12-17 12:36:08,232] jepsen worker 8 - jepsen.util 13778 :invoke :write [2 8] INFO [2019-12-17 12:36:08,253] jepsen worker 79 - jepsen.util 10789 :ok :write [26 43] INFO [2019-12-17 12:36:08,253] jepsen worker 79 - jepsen.util 10789 :invoke :cas [26 [20 16]] INFO [2019-12-17 12:36:08,281] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":60069,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:36:08,282] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":60069,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:36:08,282] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":60072,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:36:08,282] jepsen worker 53 - jepsen.util 9863 :fail :cas [17 [11 17]] :ValueNotSwapFrom INFO [2019-12-17 12:36:08,282] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":60072,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:36:08,282] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":60073,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:36:08,282] jepsen worker 53 - jepsen.util 9863 :invoke :cas [17 [14 28]] INFO [2019-12-17 12:36:08,282] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":60073,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:36:08,287] jepsen worker 64 - jepsen.util 8074 :ok :read [21 17] INFO [2019-12-17 12:36:08,287] jepsen worker 64 - jepsen.util 8074 :invoke :write [21 47] INFO [2019-12-17 12:36:08,290] jepsen worker 20 - jepsen.util 10190 :fail :cas [6 [22 31]] :ValueNotSwapFrom INFO [2019-12-17 12:36:08,290] jepsen worker 20 - jepsen.util 10190 :invoke :read [6 nil] INFO [2019-12-17 12:36:08,298] jepsen worker 43 - jepsen.util 14083 :info :write [14 40] :RequestCanceledException INFO [2019-12-17 12:36:08,298] jepsen worker 13 - jepsen.util 9823 :info :write [4 5] :RequestCanceledException INFO [2019-12-17 12:36:08,298] jepsen worker 63 - jepsen.util 9423 :info :cas [21 [24 42]] :RequestCanceledException INFO [2019-12-17 12:36:08,299] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":60062,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:36:08,299] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":60062,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:36:08,299] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":60064,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:36:08,299] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":60064,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:36:08,299] jepsen worker 43 - jepsen.util 14173 :invoke :read [14 nil] INFO [2019-12-17 12:36:08,299] jepsen worker 63 - jepsen.util 9513 :invoke :read [21 nil] INFO [2019-12-17 12:36:08,304] jepsen worker 13 - jepsen.util 9913 :invoke :cas [4 [41 3]] INFO [2019-12-17 12:36:08,312] jepsen worker 18 - jepsen.util 10818 :fail :cas [6 [31 43]] :ValueNotSwapFrom INFO [2019-12-17 12:36:08,313] jepsen worker 18 - jepsen.util 10818 :invoke :read [6 nil] INFO [2019-12-17 12:36:08,313] jepsen worker 22 - jepsen.util 13882 :info :cas [7 [1 39]] :RequestCanceledException INFO [2019-12-17 12:36:08,317] jepsen worker 22 - jepsen.util 13972 :invoke :cas [7 [35 25]] INFO [2019-12-17 12:36:08,331] jepsen worker 67 - jepsen.util 14197 :info :write [22 34] :RequestCanceledException INFO [2019-12-17 12:36:08,335] jepsen worker 67 - jepsen.util 14287 :invoke :write [22 37] INFO [2019-12-17 12:36:08,345] jepsen worker 20 - jepsen.util 10190 :ok :read [6 20] INFO [2019-12-17 12:36:08,345] jepsen worker 20 - jepsen.util 10190 :invoke :read [6 nil] INFO [2019-12-17 12:36:08,345] jepsen worker 50 - jepsen.util 14720 :info :cas [16 [2 27]] :RequestCanceledException INFO [2019-12-17 12:36:08,346] jepsen worker 50 - jepsen.util 14810 :invoke :read [16 nil] INFO [2019-12-17 12:36:08,362] jepsen worker 45 - jepsen.util 10485 :info :write [15 44] :RequestCanceledException INFO [2019-12-17 12:36:08,364] jepsen worker 45 - jepsen.util 10575 :invoke :read [15 nil] INFO [2019-12-17 12:36:08,382] jepsen worker 30 - jepsen.util 10110 :ok :read [10 17] INFO [2019-12-17 12:36:08,382] jepsen worker 30 - jepsen.util 10110 :invoke :read [10 nil] INFO [2019-12-17 12:36:08,395] jepsen worker 14 - jepsen.util 10274 :ok :read [4 26] INFO [2019-12-17 12:36:08,396] jepsen worker 14 - jepsen.util 10274 :invoke :cas [4 [25 48]] INFO [2019-12-17 12:36:08,406] jepsen worker 18 - jepsen.util 10818 :ok :read [6 20] INFO [2019-12-17 12:36:08,406] jepsen worker 20 - jepsen.util 10190 :ok :read [6 20] INFO [2019-12-17 12:36:08,406] jepsen worker 18 - jepsen.util 10818 :invoke :cas [6 [36 39]] INFO [2019-12-17 12:36:08,406] jepsen worker 20 - jepsen.util 10190 :invoke :read [6 nil] INFO [2019-12-17 12:36:08,436] jepsen worker 20 - jepsen.util 10190 :ok :read [6 20] INFO [2019-12-17 12:36:08,436] jepsen worker 20 - jepsen.util 10190 :invoke :write [6 44] INFO [2019-12-17 12:36:08,440] jepsen worker 45 - jepsen.util 10575 :ok :read [15 31] INFO [2019-12-17 12:36:08,441] jepsen worker 45 - jepsen.util 10575 :invoke :write [15 40] INFO [2019-12-17 12:36:08,451] jepsen worker 79 - jepsen.util 10789 :fail :cas [26 [20 16]] :ValueNotSwapFrom INFO [2019-12-17 12:36:08,451] jepsen worker 79 - jepsen.util 10789 :invoke :write [26 7] INFO [2019-12-17 12:36:08,461] jepsen worker 13 - jepsen.util 9913 :fail :cas [4 [41 3]] :ValueNotSwapFrom INFO [2019-12-17 12:36:08,461] jepsen worker 13 - jepsen.util 9913 :invoke :write [4 49] INFO [2019-12-17 12:36:08,463] jepsen worker 15 - jepsen.util 13155 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:36:08,463] jepsen worker 15 - jepsen.util 13155 :invoke :write [5 15] INFO [2019-12-17 12:36:08,485] jepsen worker 30 - jepsen.util 10110 :ok :read [10 17] INFO [2019-12-17 12:36:08,485] jepsen worker 30 - jepsen.util 10110 :invoke :read [10 nil] INFO [2019-12-17 12:36:08,500] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":60096,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:36:08,501] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":60096,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:36:08,501] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":60100,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:36:08,501] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":60100,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:36:08,503] jepsen worker 27 - jepsen.util 10017 :ok :write [9 0] INFO [2019-12-17 12:36:08,503] jepsen worker 27 - jepsen.util 10017 :invoke :read [9 nil] INFO [2019-12-17 12:36:08,505] jepsen worker 32 - jepsen.util 10562 :ok :write [10 41] INFO [2019-12-17 12:36:08,505] jepsen worker 64 - jepsen.util 8074 :ok :write [21 47] INFO [2019-12-17 12:36:08,506] jepsen worker 32 - jepsen.util 10562 :invoke :read [10 nil] INFO [2019-12-17 12:36:08,506] jepsen worker 64 - jepsen.util 8074 :invoke :cas [21 [22 33]] INFO [2019-12-17 12:36:08,520] jepsen worker 18 - jepsen.util 10818 :fail :cas [6 [36 39]] :ValueNotSwapFrom INFO [2019-12-17 12:36:08,520] jepsen worker 18 - jepsen.util 10818 :invoke :read [6 nil] INFO [2019-12-17 12:36:08,528] jepsen worker 14 - jepsen.util 10274 :fail :cas [4 [25 48]] :ValueNotSwapFrom INFO [2019-12-17 12:36:08,528] jepsen worker 14 - jepsen.util 10274 :invoke :cas [4 [37 14]] INFO [2019-12-17 12:36:08,531] jepsen worker 57 - jepsen.util 13827 :info :cas [19 [36 3]] :RequestCanceledException INFO [2019-12-17 12:36:08,533] jepsen worker 57 - jepsen.util 13917 :invoke :read [19 nil] INFO [2019-12-17 12:36:08,545] jepsen worker 11 - jepsen.util 10001 :ok :read [3 40] INFO [2019-12-17 12:36:08,545] jepsen worker 11 - jepsen.util 10001 :invoke :write [3 4] INFO [2019-12-17 12:36:08,582] jepsen worker 64 - jepsen.util 8074 :fail :cas [21 [22 33]] :ValueNotSwapFrom INFO [2019-12-17 12:36:08,582] jepsen worker 64 - jepsen.util 8074 :invoke :cas [21 [49 8]] INFO [2019-12-17 12:36:08,598] jepsen worker 42 - jepsen.util 14982 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:36:08,598] jepsen worker 2 - jepsen.util 13772 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:36:08,598] jepsen worker 42 - jepsen.util 14982 :invoke :write [14 3] INFO [2019-12-17 12:36:08,598] jepsen worker 2 - jepsen.util 13772 :invoke :read [0 nil] INFO [2019-12-17 12:36:08,616] jepsen worker 60 - jepsen.util 15090 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:36:08,616] jepsen worker 60 - jepsen.util 15090 :invoke :cas [20 [32 34]] INFO [2019-12-17 12:36:08,622] jepsen worker 63 - jepsen.util 9513 :ok :read [21 47] INFO [2019-12-17 12:36:08,622] jepsen worker 63 - jepsen.util 9513 :invoke :read [21 nil] INFO [2019-12-17 12:36:08,630] jepsen worker 41 - jepsen.util 15071 :info :cas [13 [9 17]] :RequestCanceledException INFO [2019-12-17 12:36:08,630] jepsen worker 9 - jepsen.util 11079 :info :write [3 29] :RequestCanceledException INFO [2019-12-17 12:36:08,631] jepsen worker 54 - jepsen.util 11304 :info :write [18 17] :RequestCanceledException INFO [2019-12-17 12:36:08,631] jepsen worker 9 - jepsen.util 11169 :invoke :read [3 nil] INFO [2019-12-17 12:36:08,632] jepsen worker 41 - jepsen.util 15161 :invoke :read [13 nil] INFO [2019-12-17 12:36:08,632] jepsen worker 87 - jepsen.util 9987 :ok :write [29 6] INFO [2019-12-17 12:36:08,632] jepsen worker 87 - jepsen.util 9987 :invoke :write [29 43] INFO [2019-12-17 12:36:08,633] jepsen worker 54 - jepsen.util 11394 :invoke :write [18 38] INFO [2019-12-17 12:36:08,645] jepsen worker 17 - jepsen.util 13967 :info :write [5 43] :RequestCanceledException INFO [2019-12-17 12:36:08,647] jepsen worker 17 - jepsen.util 14057 :invoke :write [5 22] INFO [2019-12-17 12:36:08,657] jepsen worker 27 - jepsen.util 10017 :ok :read [9 0] INFO [2019-12-17 12:36:08,658] jepsen worker 27 - jepsen.util 10017 :invoke :cas [9 [21 6]] INFO [2019-12-17 12:36:08,662] jepsen worker 6 - jepsen.util 14586 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:36:08,662] jepsen worker 6 - jepsen.util 14586 :invoke :read [2 nil] INFO [2019-12-17 12:36:08,674] jepsen worker 55 - jepsen.util 11035 :ok :read [18 20] INFO [2019-12-17 12:36:08,674] jepsen worker 55 - jepsen.util 11035 :invoke :write [18 7] INFO [2019-12-17 12:36:08,686] jepsen worker 64 - jepsen.util 8074 :fail :cas [21 [49 8]] :ValueNotSwapFrom INFO [2019-12-17 12:36:08,686] jepsen worker 64 - jepsen.util 8074 :invoke :read [21 nil] INFO [2019-12-17 12:36:08,697] jepsen worker 24 - jepsen.util 12714 :info :write [8 2] :RequestCanceledException INFO [2019-12-17 12:36:08,698] jepsen worker 24 - jepsen.util 12804 :invoke :cas [8 [40 30]] INFO [2019-12-17 12:36:08,702] jepsen worker 52 - jepsen.util 10312 :ok :write [17 34] INFO [2019-12-17 12:36:08,702] jepsen worker 52 - jepsen.util 10312 :invoke :cas [17 [5 45]] INFO [2019-12-17 12:36:08,718] jepsen worker 4 - jepsen.util 9634 :ok :read [1 35] INFO [2019-12-17 12:36:08,718] jepsen worker 4 - jepsen.util 9634 :invoke :cas [1 [16 19]] INFO [2019-12-17 12:36:08,719] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":60124,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:36:08,719] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":60124,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:36:08,719] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":60127,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:36:08,719] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":60127,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:36:08,719] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":60128,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:36:08,719] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":60128,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:36:08,731] jepsen worker 3 - jepsen.util 10083 :info :write [1 31] :RequestCanceledException INFO [2019-12-17 12:36:08,731] jepsen worker 76 - jepsen.util 14476 :info :cas [25 [46 0]] :RequestCanceledException INFO [2019-12-17 12:36:08,732] jepsen worker 76 - jepsen.util 14566 :invoke :read [25 nil] INFO [2019-12-17 12:36:08,733] jepsen worker 3 - jepsen.util 10173 :invoke :write [1 33] INFO [2019-12-17 12:36:08,751] jepsen worker 32 - jepsen.util 10562 :ok :read [10 41] INFO [2019-12-17 12:36:08,751] jepsen worker 32 - jepsen.util 10562 :invoke :read [10 nil] INFO [2019-12-17 12:36:08,775] jepsen worker 64 - jepsen.util 8074 :ok :read [21 47] INFO [2019-12-17 12:36:08,775] jepsen worker 64 - jepsen.util 8074 :invoke :write [21 29] INFO [2019-12-17 12:36:08,775] jepsen worker 27 - jepsen.util 10017 :fail :cas [9 [21 6]] :ValueNotSwapFrom INFO [2019-12-17 12:36:08,775] jepsen worker 27 - jepsen.util 10017 :invoke :read [9 nil] INFO [2019-12-17 12:36:08,798] jepsen worker 51 - jepsen.util 9951 :ok :write [17 22] INFO [2019-12-17 12:36:08,798] jepsen worker 51 - jepsen.util 9951 :invoke :cas [17 [48 7]] INFO [2019-12-17 12:36:08,809] jepsen worker 27 - jepsen.util 10017 :ok :read [9 0] INFO [2019-12-17 12:36:08,810] jepsen worker 27 - jepsen.util 10017 :invoke :write [9 6] INFO [2019-12-17 12:36:08,817] jepsen worker 70 - jepsen.util 11140 :info :write [23 5] :RequestCanceledException INFO [2019-12-17 12:36:08,818] jepsen worker 70 - jepsen.util 11230 :invoke :cas [23 [17 8]] INFO [2019-12-17 12:36:08,829] jepsen worker 51 - jepsen.util 9951 :fail :cas [17 [48 7]] :ValueNotSwapFrom INFO [2019-12-17 12:36:08,829] jepsen worker 51 - jepsen.util 9951 :invoke :write [17 25] INFO [2019-12-17 12:36:08,882] jepsen worker 64 - jepsen.util 8074 :ok :write [21 29] INFO [2019-12-17 12:36:08,882] jepsen worker 64 - jepsen.util 8074 :invoke :write [21 11] INFO [2019-12-17 12:36:08,890] jepsen worker 27 - jepsen.util 10017 :ok :write [9 6] INFO [2019-12-17 12:36:08,890] jepsen worker 27 - jepsen.util 10017 :invoke :read [9 nil] INFO [2019-12-17 12:36:08,917] jepsen worker 27 - jepsen.util 10017 :ok :read [9 6] INFO [2019-12-17 12:36:08,917] jepsen worker 27 - jepsen.util 10017 :invoke :read [9 nil] INFO [2019-12-17 12:36:08,931] jepsen worker 44 - jepsen.util 14444 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:36:08,931] jepsen worker 35 - jepsen.util 14075 :info :write [11 3] :RequestCanceledException INFO [2019-12-17 12:36:08,931] jepsen worker 44 - jepsen.util 14444 :invoke :cas [14 [24 46]] INFO [2019-12-17 12:36:08,932] jepsen worker 35 - jepsen.util 14165 :invoke :cas [11 [23 23]] INFO [2019-12-17 12:36:08,940] jepsen worker 5 - jepsen.util 9995 :ok :write [1 15] INFO [2019-12-17 12:36:08,940] jepsen worker 5 - jepsen.util 9995 :invoke :read [1 nil] INFO [2019-12-17 12:36:08,946] jepsen worker 62 - jepsen.util 14552 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:36:08,946] jepsen worker 62 - jepsen.util 14552 :invoke :read [20 nil] INFO [2019-12-17 12:36:08,950] jepsen worker 27 - jepsen.util 10017 :ok :read [9 6] INFO [2019-12-17 12:36:08,950] jepsen worker 27 - jepsen.util 10017 :invoke :read [9 nil] INFO [2019-12-17 12:36:08,958] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":60147,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:36:08,959] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":60147,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:36:08,959] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":60150,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:36:08,959] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":60150,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:36:08,974] jepsen worker 12 - jepsen.util 10812 :ok :cas [4 [26 36]] INFO [2019-12-17 12:36:08,974] jepsen worker 12 - jepsen.util 10812 :invoke :read [4 nil] INFO [2019-12-17 12:36:08,987] jepsen worker 27 - jepsen.util 10017 :ok :read [9 6] INFO [2019-12-17 12:36:08,987] jepsen worker 27 - jepsen.util 10017 :invoke :write [9 16] INFO [2019-12-17 12:36:08,994] jepsen worker 64 - jepsen.util 8074 :ok :write [21 11] INFO [2019-12-17 12:36:08,994] jepsen worker 64 - jepsen.util 8074 :invoke :cas [21 [26 8]] INFO [2019-12-17 12:36:09,016] jepsen worker 87 - jepsen.util 9987 :ok :write [29 43] INFO [2019-12-17 12:36:09,017] jepsen worker 87 - jepsen.util 9987 :invoke :cas [29 [26 32]] INFO [2019-12-17 12:36:09,031] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":60164,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:36:09,032] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":60164,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:36:09,032] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":60167,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:36:09,032] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":60167,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:36:09,033] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":60168,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:36:09,033] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":60168,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:36:09,041] jepsen worker 64 - jepsen.util 8074 :fail :cas [21 [26 8]] :ValueNotSwapFrom INFO [2019-12-17 12:36:09,041] jepsen worker 64 - jepsen.util 8074 :invoke :cas [21 [1 11]] INFO [2019-12-17 12:36:09,062] jepsen worker 64 - jepsen.util 8074 :fail :cas [21 [1 11]] :ValueNotSwapFrom INFO [2019-12-17 12:36:09,062] jepsen worker 64 - jepsen.util 8074 :invoke :read [21 nil] INFO [2019-12-17 12:36:09,063] jepsen worker 23 - jepsen.util 14243 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:36:09,063] jepsen worker 23 - jepsen.util 14243 :invoke :read [7 nil] INFO [2019-12-17 12:36:09,085] jepsen worker 64 - jepsen.util 8074 :ok :read [21 11] INFO [2019-12-17 12:36:09,086] jepsen worker 64 - jepsen.util 8074 :invoke :cas [21 [11 34]] INFO [2019-12-17 12:36:09,086] jepsen worker 27 - jepsen.util 10017 :ok :write [9 16] INFO [2019-12-17 12:36:09,087] jepsen worker 27 - jepsen.util 10017 :invoke :read [9 nil] INFO [2019-12-17 12:36:09,087] jepsen worker 5 - jepsen.util 9995 :ok :read [1 15] INFO [2019-12-17 12:36:09,087] jepsen worker 5 - jepsen.util 9995 :invoke :write [1 45] INFO [2019-12-17 12:36:09,098] jepsen worker 85 - jepsen.util 15025 :info :write [28 18] :RequestCanceledException INFO [2019-12-17 12:36:09,099] jepsen worker 85 - jepsen.util 15115 :invoke :read [28 nil] INFO [2019-12-17 12:36:09,116] jepsen worker 28 - jepsen.util 8488 :info :cas [9 [8 39]] :RequestCanceledException INFO [2019-12-17 12:36:09,116] jepsen worker 74 - jepsen.util 10244 :info :cas [24 [28 41]] :RequestCanceledException INFO [2019-12-17 12:36:09,117] jepsen worker 28 - jepsen.util 8578 :invoke :cas [9 [43 10]] INFO [2019-12-17 12:36:09,117] jepsen worker 74 - jepsen.util 10334 :invoke :cas [24 [19 22]] INFO [2019-12-17 12:36:09,131] jepsen worker 40 - jepsen.util 14620 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:36:09,131] jepsen worker 47 - jepsen.util 11027 :info :write [15 29] :RequestCanceledException INFO [2019-12-17 12:36:09,131] jepsen worker 72 - jepsen.util 10512 :info :write [24 35] :RequestCanceledException INFO [2019-12-17 12:36:09,131] jepsen worker 40 - jepsen.util 14620 :invoke :read [13 nil] INFO [2019-12-17 12:36:09,132] jepsen worker 47 - jepsen.util 11117 :invoke :read [15 nil] INFO [2019-12-17 12:36:09,132] jepsen worker 72 - jepsen.util 10602 :invoke :read [24 nil] INFO [2019-12-17 12:36:09,152] jepsen worker 27 - jepsen.util 10017 :ok :read [9 16] INFO [2019-12-17 12:36:09,152] jepsen worker 27 - jepsen.util 10017 :invoke :read [9 nil] INFO [2019-12-17 12:36:09,167] jepsen worker 80 - jepsen.util 10160 :ok :write [26 17] INFO [2019-12-17 12:36:09,167] jepsen worker 80 - jepsen.util 10160 :invoke :cas [26 [38 9]] INFO [2019-12-17 12:36:09,176] jepsen worker 28 - jepsen.util 8578 :fail :cas [9 [43 10]] :ValueNotSwapFrom INFO [2019-12-17 12:36:09,176] jepsen worker 28 - jepsen.util 8578 :invoke :read [9 nil] INFO [2019-12-17 12:36:09,177] jepsen worker 46 - jepsen.util 10396 :ok :write [15 3] INFO [2019-12-17 12:36:09,177] jepsen worker 46 - jepsen.util 10396 :invoke :write [15 33] INFO [2019-12-17 12:36:09,194] jepsen worker 20 - jepsen.util 10190 :ok :write [6 44] INFO [2019-12-17 12:36:09,194] jepsen worker 20 - jepsen.util 10190 :invoke :write [6 39] INFO [2019-12-17 12:36:09,200] jepsen worker 4 - jepsen.util 9634 :fail :cas [1 [16 19]] :ValueNotSwapFrom INFO [2019-12-17 12:36:09,201] jepsen worker 4 - jepsen.util 9634 :invoke :read [1 nil] INFO [2019-12-17 12:36:09,207] jepsen worker 27 - jepsen.util 10017 :ok :read [9 16] INFO [2019-12-17 12:36:09,207] jepsen worker 27 - jepsen.util 10017 :invoke :cas [9 [31 20]] INFO [2019-12-17 12:36:09,213] jepsen worker 87 - jepsen.util 9987 :fail :cas [29 [26 32]] :ValueNotSwapFrom INFO [2019-12-17 12:36:09,213] jepsen worker 87 - jepsen.util 9987 :invoke :write [29 48] INFO [2019-12-17 12:36:09,216] jepsen worker 37 - jepsen.util 9937 :info :write [12 33] :RequestCanceledException INFO [2019-12-17 12:36:09,218] jepsen worker 79 - jepsen.util 10789 :ok :write [26 7] INFO [2019-12-17 12:36:09,218] jepsen worker 37 - jepsen.util 10027 :invoke :write [12 4] INFO [2019-12-17 12:36:09,218] jepsen worker 79 - jepsen.util 10789 :invoke :write [26 22] INFO [2019-12-17 12:36:09,232] jepsen worker 11 - jepsen.util 10001 :ok :write [3 4] INFO [2019-12-17 12:36:09,232] jepsen worker 88 - jepsen.util 10798 :info :write [29 43] :RequestCanceledException INFO [2019-12-17 12:36:09,232] jepsen worker 11 - jepsen.util 10001 :invoke :cas [3 [5 44]] INFO [2019-12-17 12:36:09,233] jepsen worker 88 - jepsen.util 10888 :invoke :read [29 nil] INFO [2019-12-17 12:36:09,241] jepsen worker 14 - jepsen.util 10274 :fail :cas [4 [37 14]] :ValueNotSwapFrom INFO [2019-12-17 12:36:09,241] jepsen worker 14 - jepsen.util 10274 :invoke :read [4 nil] INFO [2019-12-17 12:36:09,249] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:36:09,249] jepsen nemesis - couchbase.util Rebalance check count: 470 INFO [2019-12-17 12:36:09,249] jepsen nemesis - couchbase.util Rebalance stuck count: 451 INFO [2019-12-17 12:36:09,251] jepsen worker 12 - jepsen.util 10812 :ok :read [4 36] INFO [2019-12-17 12:36:09,251] jepsen worker 12 - jepsen.util 10812 :invoke :write [4 34] INFO [2019-12-17 12:36:09,262] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":60199,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:36:09,263] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":60199,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:36:09,263] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":60202,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:36:09,263] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":60202,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:36:09,282] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":60188,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:36:09,282] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":60188,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:36:09,294] jepsen worker 9 - jepsen.util 11169 :ok :read [3 40] INFO [2019-12-17 12:36:09,294] jepsen worker 9 - jepsen.util 11169 :invoke :read [3 nil] INFO [2019-12-17 12:36:09,298] jepsen worker 66 - jepsen.util 13836 :info :write [22 27] :RequestCanceledException INFO [2019-12-17 12:36:09,299] jepsen worker 66 - jepsen.util 13926 :invoke :write [22 32] INFO [2019-12-17 12:36:09,315] jepsen worker 52 - jepsen.util 10312 :fail :cas [17 [5 45]] :ValueNotSwapFrom INFO [2019-12-17 12:36:09,316] jepsen worker 52 - jepsen.util 10312 :invoke :cas [17 [34 0]] INFO [2019-12-17 12:36:09,316] jepsen worker 7 - jepsen.util 14317 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:36:09,316] jepsen worker 19 - jepsen.util 10459 :info :write [6 16] :RequestCanceledException INFO [2019-12-17 12:36:09,316] jepsen worker 7 - jepsen.util 14317 :invoke :cas [2 [46 44]] INFO [2019-12-17 12:36:09,319] jepsen worker 28 - jepsen.util 8578 :ok :read [9 16] INFO [2019-12-17 12:36:09,319] jepsen worker 28 - jepsen.util 8578 :invoke :write [9 20] INFO [2019-12-17 12:36:09,319] jepsen worker 19 - jepsen.util 10549 :invoke :write [6 36] INFO [2019-12-17 12:36:09,331] jepsen worker 49 - jepsen.util 13999 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:36:09,332] jepsen worker 83 - jepsen.util 14303 :info :write [27 7] :RequestCanceledException INFO [2019-12-17 12:36:09,332] jepsen worker 49 - jepsen.util 13999 :invoke :write [16 20] INFO [2019-12-17 12:36:09,332] jepsen worker 83 - jepsen.util 14393 :invoke :write [27 44] INFO [2019-12-17 12:36:09,345] jepsen worker 4 - jepsen.util 9634 :ok :read [1 15] INFO [2019-12-17 12:36:09,345] jepsen worker 61 - jepsen.util 14911 :info :cas [20 [5 25]] :RequestCanceledException INFO [2019-12-17 12:36:09,345] jepsen worker 4 - jepsen.util 9634 :invoke :cas [1 [35 10]] INFO [2019-12-17 12:36:09,346] jepsen worker 61 - jepsen.util 15001 :invoke :cas [20 [25 8]] INFO [2019-12-17 12:36:09,362] jepsen worker 56 - jepsen.util 10856 :info :write [18 2] :RequestCanceledException INFO [2019-12-17 12:36:09,362] jepsen worker 26 - jepsen.util 14786 :info :cas [8 [21 22]] :RequestCanceledException INFO [2019-12-17 12:36:09,363] jepsen worker 56 - jepsen.util 10946 :invoke :cas [18 [12 23]] INFO [2019-12-17 12:36:09,363] jepsen worker 26 - jepsen.util 14876 :invoke :read [8 nil] INFO [2019-12-17 12:36:09,397] jepsen worker 14 - jepsen.util 10274 :ok :read [4 36] INFO [2019-12-17 12:36:09,397] jepsen worker 14 - jepsen.util 10274 :invoke :write [4 7] INFO [2019-12-17 12:36:09,398] jepsen worker 0 - jepsen.util 14850 :info :write [0 49] :RequestCanceledException INFO [2019-12-17 12:36:09,399] jepsen worker 0 - jepsen.util 14940 :invoke :write [0 3] INFO [2019-12-17 12:36:09,430] jepsen worker 88 - jepsen.util 10888 :ok :read [29 43] INFO [2019-12-17 12:36:09,430] jepsen worker 1 - jepsen.util 14221 :info :write [0 15] :RequestCanceledException INFO [2019-12-17 12:36:09,430] jepsen worker 88 - jepsen.util 10888 :invoke :write [29 15] INFO [2019-12-17 12:36:09,432] jepsen worker 1 - jepsen.util 14311 :invoke :read [0 nil] INFO [2019-12-17 12:36:09,435] jepsen worker 5 - jepsen.util 9995 :ok :write [1 45] INFO [2019-12-17 12:36:09,435] jepsen worker 5 - jepsen.util 9995 :invoke :read [1 nil] INFO [2019-12-17 12:36:09,437] jepsen worker 72 - jepsen.util 10602 :ok :read [24 15] INFO [2019-12-17 12:36:09,437] jepsen worker 72 - jepsen.util 10602 :invoke :write [24 22] INFO [2019-12-17 12:36:09,439] jepsen worker 47 - jepsen.util 11117 :ok :read [15 3] INFO [2019-12-17 12:36:09,439] jepsen worker 47 - jepsen.util 11117 :invoke :write [15 38] INFO [2019-12-17 12:36:09,458] jepsen worker 32 - jepsen.util 10562 :ok :read [10 41] INFO [2019-12-17 12:36:09,458] jepsen worker 32 - jepsen.util 10562 :invoke :cas [10 [26 18]] INFO [2019-12-17 12:36:09,460] jepsen worker 27 - jepsen.util 10017 :fail :cas [9 [31 20]] :ValueNotSwapFrom INFO [2019-12-17 12:36:09,460] jepsen worker 27 - jepsen.util 10017 :invoke :read [9 nil] INFO [2019-12-17 12:36:09,484] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":60224,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:36:09,484] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":60224,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:36:09,484] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":60227,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:36:09,484] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":60227,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:36:09,498] jepsen worker 71 - jepsen.util 11231 :info :write [23 14] :RequestCanceledException INFO [2019-12-17 12:36:09,499] jepsen worker 71 - jepsen.util 11321 :invoke :read [23 nil] INFO [2019-12-17 12:36:09,504] jepsen worker 27 - jepsen.util 10017 :ok :read [9 16] INFO [2019-12-17 12:36:09,504] jepsen worker 27 - jepsen.util 10017 :invoke :cas [9 [21 24]] INFO [2019-12-17 12:36:09,523] jepsen worker 27 - jepsen.util 10017 :fail :cas [9 [21 24]] :ValueNotSwapFrom INFO [2019-12-17 12:36:09,523] jepsen worker 27 - jepsen.util 10017 :invoke :write [9 39] INFO [2019-12-17 12:36:09,542] jepsen worker 79 - jepsen.util 10789 :ok :write [26 22] INFO [2019-12-17 12:36:09,542] jepsen worker 79 - jepsen.util 10789 :invoke :write [26 22] INFO [2019-12-17 12:36:09,545] jepsen worker 80 - jepsen.util 10160 :fail :cas [26 [38 9]] :ValueNotSwapFrom INFO [2019-12-17 12:36:09,545] jepsen worker 80 - jepsen.util 10160 :invoke :write [26 23] INFO [2019-12-17 12:36:09,545] jepsen worker 89 - jepsen.util 10709 :info :write [29 7] :RequestCanceledException INFO [2019-12-17 12:36:09,547] jepsen worker 89 - jepsen.util 10799 :invoke :cas [29 [38 46]] INFO [2019-12-17 12:36:09,547] jepsen worker 74 - jepsen.util 10334 :fail :cas [24 [19 22]] :ValueNotSwapFrom INFO [2019-12-17 12:36:09,547] jepsen worker 74 - jepsen.util 10334 :invoke :cas [24 [46 3]] INFO [2019-12-17 12:36:09,566] jepsen worker 53 - jepsen.util 9863 :fail :cas [17 [14 28]] :ValueNotSwapFrom INFO [2019-12-17 12:36:09,566] jepsen worker 53 - jepsen.util 9863 :invoke :write [17 39] INFO [2019-12-17 12:36:09,568] jepsen worker 11 - jepsen.util 10001 :fail :cas [3 [5 44]] :ValueNotSwapFrom INFO [2019-12-17 12:36:09,568] jepsen worker 11 - jepsen.util 10001 :invoke :cas [3 [26 21]] INFO [2019-12-17 12:36:09,573] jepsen worker 70 - jepsen.util 11230 :fail :cas [23 [17 8]] :ValueNotSwapFrom INFO [2019-12-17 12:36:09,573] jepsen worker 70 - jepsen.util 11230 :invoke :write [23 9] INFO [2019-12-17 12:36:09,588] jepsen worker 28 - jepsen.util 8578 :ok :write [9 20] INFO [2019-12-17 12:36:09,588] jepsen worker 28 - jepsen.util 8578 :invoke :read [9 nil] INFO [2019-12-17 12:36:09,597] jepsen worker 64 - jepsen.util 8074 :ok :cas [21 [11 34]] INFO [2019-12-17 12:36:09,597] jepsen worker 64 - jepsen.util 8074 :invoke :write [21 29] INFO [2019-12-17 12:36:09,598] jepsen worker 75 - jepsen.util 13575 :info :cas [25 [39 3]] :RequestCanceledException INFO [2019-12-17 12:36:09,599] jepsen worker 75 - jepsen.util 13665 :invoke :cas [25 [31 25]] INFO [2019-12-17 12:36:09,632] jepsen worker 25 - jepsen.util 15235 :info :write [8 35] :RequestCanceledException INFO [2019-12-17 12:36:09,632] jepsen worker 69 - jepsen.util 10509 :info :write [23 16] :RequestCanceledException INFO [2019-12-17 12:36:09,633] jepsen worker 69 - jepsen.util 10599 :invoke :read [23 nil] INFO [2019-12-17 12:36:09,633] jepsen worker 25 - jepsen.util 15325 :invoke :cas [8 [34 26]] INFO [2019-12-17 12:36:09,634] jepsen worker 28 - jepsen.util 8578 :ok :read [9 20] INFO [2019-12-17 12:36:09,634] jepsen worker 28 - jepsen.util 8578 :invoke :read [9 nil] INFO [2019-12-17 12:36:09,661] jepsen worker 68 - jepsen.util 13658 :info :cas [22 [34 22]] :RequestCanceledException INFO [2019-12-17 12:36:09,662] jepsen worker 68 - jepsen.util 13748 :invoke :read [22 nil] INFO [2019-12-17 12:36:09,670] jepsen worker 28 - jepsen.util 8578 :ok :read [9 20] INFO [2019-12-17 12:36:09,670] jepsen worker 28 - jepsen.util 8578 :invoke :write [9 19] INFO [2019-12-17 12:36:09,671] jepsen worker 87 - jepsen.util 9987 :ok :write [29 48] INFO [2019-12-17 12:36:09,671] jepsen worker 87 - jepsen.util 9987 :invoke :cas [29 [9 7]] INFO [2019-12-17 12:36:09,698] jepsen worker 58 - jepsen.util 14458 :info :cas [19 [29 11]] :RequestCanceledException INFO [2019-12-17 12:36:09,699] jepsen worker 58 - jepsen.util 14548 :invoke :cas [19 [48 10]] INFO [2019-12-17 12:36:09,703] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":60254,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:36:09,703] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":60254,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:36:09,707] jepsen worker 64 - jepsen.util 8074 :ok :write [21 29] INFO [2019-12-17 12:36:09,707] jepsen worker 64 - jepsen.util 8074 :invoke :cas [21 [22 16]] INFO [2019-12-17 12:36:09,707] jepsen worker 89 - jepsen.util 10799 :fail :cas [29 [38 46]] :ValueNotSwapFrom INFO [2019-12-17 12:36:09,707] jepsen worker 89 - jepsen.util 10799 :invoke :read [29 nil] INFO [2019-12-17 12:36:09,726] jepsen worker 27 - jepsen.util 10017 :ok :write [9 39] INFO [2019-12-17 12:36:09,726] jepsen worker 27 - jepsen.util 10017 :invoke :cas [9 [49 19]] INFO [2019-12-17 12:36:09,732] jepsen worker 38 - jepsen.util 11108 :info :write [12 14] :RequestCanceledException INFO [2019-12-17 12:36:09,732] jepsen worker 36 - jepsen.util 11196 :info :write [12 14] :RequestCanceledException INFO [2019-12-17 12:36:09,732] jepsen worker 82 - jepsen.util 14392 :info :cas [27 [32 22]] :RequestCanceledException INFO [2019-12-17 12:36:09,733] jepsen worker 36 - jepsen.util 11286 :invoke :cas [12 [35 0]] INFO [2019-12-17 12:36:09,733] jepsen worker 38 - jepsen.util 11198 :invoke :cas [12 [30 21]] INFO [2019-12-17 12:36:09,733] jepsen worker 82 - jepsen.util 14482 :invoke :read [27 nil] INFO [2019-12-17 12:36:09,744] jepsen worker 64 - jepsen.util 8074 :fail :cas [21 [22 16]] :ValueNotSwapFrom INFO [2019-12-17 12:36:09,744] jepsen worker 64 - jepsen.util 8074 :invoke :read [21 nil] INFO [2019-12-17 12:36:09,752] jepsen worker 14 - jepsen.util 10274 :ok :write [4 7] INFO [2019-12-17 12:36:09,752] jepsen worker 14 - jepsen.util 10274 :invoke :read [4 nil] INFO [2019-12-17 12:36:09,761] jepsen worker 4 - jepsen.util 9634 :fail :cas [1 [35 10]] :ValueNotSwapFrom INFO [2019-12-17 12:36:09,761] jepsen worker 4 - jepsen.util 9634 :invoke :cas [1 [1 12]] INFO [2019-12-17 12:36:09,778] jepsen worker 64 - jepsen.util 8074 :ok :read [21 29] INFO [2019-12-17 12:36:09,779] jepsen worker 64 - jepsen.util 8074 :invoke :write [21 40] INFO [2019-12-17 12:36:09,790] jepsen worker 27 - jepsen.util 10017 :fail :cas [9 [49 19]] :ValueNotSwapFrom INFO [2019-12-17 12:36:09,790] jepsen worker 27 - jepsen.util 10017 :invoke :write [9 9] INFO [2019-12-17 12:36:09,818] jepsen worker 46 - jepsen.util 10396 :ok :write [15 33] INFO [2019-12-17 12:36:09,818] jepsen worker 46 - jepsen.util 10396 :invoke :read [15 nil] INFO [2019-12-17 12:36:09,832] jepsen worker 48 - jepsen.util 13008 :info :cas [16 [10 44]] :RequestCanceledException INFO [2019-12-17 12:36:09,835] jepsen worker 48 - jepsen.util 13098 :invoke :write [16 34] INFO [2019-12-17 12:36:09,840] jepsen worker 19 - jepsen.util 10549 :ok :write [6 36] INFO [2019-12-17 12:36:09,840] jepsen worker 19 - jepsen.util 10549 :invoke :write [6 22] INFO [2019-12-17 12:36:09,840] jepsen worker 46 - jepsen.util 10396 :ok :read [15 33] INFO [2019-12-17 12:36:09,841] jepsen worker 46 - jepsen.util 10396 :invoke :read [15 nil] INFO [2019-12-17 12:36:09,860] jepsen worker 5 - jepsen.util 9995 :ok :read [1 45] INFO [2019-12-17 12:36:09,860] jepsen worker 5 - jepsen.util 9995 :invoke :write [1 5] INFO [2019-12-17 12:36:09,887] jepsen worker 46 - jepsen.util 10396 :ok :read [15 33] INFO [2019-12-17 12:36:09,887] jepsen worker 46 - jepsen.util 10396 :invoke :cas [15 [34 25]] INFO [2019-12-17 12:36:09,889] jepsen worker 54 - jepsen.util 11394 :ok :write [18 38] INFO [2019-12-17 12:36:09,889] jepsen worker 54 - jepsen.util 11394 :invoke :read [18 nil] INFO [2019-12-17 12:36:09,894] jepsen worker 29 - jepsen.util 8759 :info :cas [9 [36 32]] :RequestCanceledException INFO [2019-12-17 12:36:09,894] jepsen worker 78 - jepsen.util 10338 :info :write [26 5] :RequestCanceledException INFO [2019-12-17 12:36:09,896] jepsen worker 29 - jepsen.util 8849 :invoke :cas [9 [35 12]] INFO [2019-12-17 12:36:09,897] jepsen worker 52 - jepsen.util 10312 :fail :cas [17 [34 0]] :ValueNotSwapFrom INFO [2019-12-17 12:36:09,897] jepsen worker 52 - jepsen.util 10312 :invoke :write [17 48] INFO [2019-12-17 12:36:09,897] jepsen worker 78 - jepsen.util 10428 :invoke :read [26 nil] INFO [2019-12-17 12:36:09,904] jepsen worker 69 - jepsen.util 10599 :ok :read [23 14] INFO [2019-12-17 12:36:09,905] jepsen worker 69 - jepsen.util 10599 :invoke :cas [23 [41 45]] INFO [2019-12-17 12:36:09,911] jepsen worker 87 - jepsen.util 9987 :fail :cas [29 [9 7]] :ValueNotSwapFrom INFO [2019-12-17 12:36:09,912] jepsen worker 87 - jepsen.util 9987 :invoke :write [29 27] INFO [2019-12-17 12:36:09,916] jepsen worker 73 - jepsen.util 9703 :info :cas [24 [17 13]] :RequestCanceledException INFO [2019-12-17 12:36:09,917] jepsen worker 73 - jepsen.util 9793 :invoke :cas [24 [44 13]] INFO [2019-12-17 12:36:09,924] jepsen worker 69 - jepsen.util 10599 :fail :cas [23 [41 45]] :ValueNotSwapFrom INFO [2019-12-17 12:36:09,924] jepsen worker 69 - jepsen.util 10599 :invoke :cas [23 [32 34]] INFO [2019-12-17 12:36:09,932] jepsen worker 33 - jepsen.util 14163 :info :cas [11 [45 39]] :RequestCanceledException INFO [2019-12-17 12:36:09,932] jepsen worker 31 - jepsen.util 9121 :info :cas [10 [8 19]] :RequestCanceledException INFO [2019-12-17 12:36:09,933] jepsen worker 31 - jepsen.util 9211 :invoke :read [10 nil] INFO [2019-12-17 12:36:09,933] jepsen worker 33 - jepsen.util 14253 :invoke :cas [11 [0 12]] INFO [2019-12-17 12:36:09,938] jepsen worker 46 - jepsen.util 10396 :fail :cas [15 [34 25]] :ValueNotSwapFrom INFO [2019-12-17 12:36:09,939] jepsen worker 46 - jepsen.util 10396 :invoke :read [15 nil] INFO [2019-12-17 12:36:09,942] jepsen worker 32 - jepsen.util 10562 :fail :cas [10 [26 18]] :ValueNotSwapFrom INFO [2019-12-17 12:36:09,942] jepsen worker 32 - jepsen.util 10562 :invoke :read [10 nil] INFO [2019-12-17 12:36:09,943] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":60278,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:36:09,944] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":60278,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:36:09,944] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":60280,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:36:09,944] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":60280,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:36:09,953] jepsen worker 64 - jepsen.util 8074 :ok :write [21 40] INFO [2019-12-17 12:36:09,953] jepsen worker 64 - jepsen.util 8074 :invoke :read [21 nil] INFO [2019-12-17 12:36:09,960] jepsen worker 46 - jepsen.util 10396 :ok :read [15 33] INFO [2019-12-17 12:36:09,960] jepsen worker 46 - jepsen.util 10396 :invoke :read [15 nil] INFO [2019-12-17 12:36:09,962] jepsen worker 10 - jepsen.util 10450 :fail :read [3 nil] :RequestTimeoutException INFO [2019-12-17 12:36:09,963] jepsen worker 10 - jepsen.util 10450 :invoke :write [3 11] INFO [2019-12-17 12:36:09,974] jepsen worker 71 - jepsen.util 11321 :ok :read [23 14] INFO [2019-12-17 12:36:09,974] jepsen worker 71 - jepsen.util 11321 :invoke :cas [23 [43 40]] INFO [2019-12-17 12:36:09,992] jepsen worker 74 - jepsen.util 10334 :fail :cas [24 [46 3]] :ValueNotSwapFrom INFO [2019-12-17 12:36:09,993] jepsen worker 74 - jepsen.util 10334 :invoke :cas [24 [8 21]] INFO [2019-12-17 12:36:10,016] jepsen worker 81 - jepsen.util 13851 :info :cas [27 [43 2]] :RequestCanceledException INFO [2019-12-17 12:36:10,017] jepsen worker 81 - jepsen.util 13941 :invoke :cas [27 [0 43]] INFO [2019-12-17 12:36:10,019] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":60304,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:36:10,019] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":60304,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:36:10,019] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":60307,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:36:10,019] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":60307,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:36:10,031] jepsen worker 39 - jepsen.util 13719 :info :write [13 44] :RequestCanceledException INFO [2019-12-17 12:36:10,031] jepsen worker 65 - jepsen.util 8975 :info :write [21 11] :RequestCanceledException INFO [2019-12-17 12:36:10,032] jepsen worker 39 - jepsen.util 13809 :invoke :write [13 9] INFO [2019-12-17 12:36:10,036] jepsen worker 65 - jepsen.util 9065 :invoke :read [21 nil] INFO [2019-12-17 12:36:10,042] jepsen worker 78 - jepsen.util 10428 :ok :read [26 22] INFO [2019-12-17 12:36:10,042] jepsen worker 78 - jepsen.util 10428 :invoke :write [26 17] INFO [2019-12-17 12:36:10,051] jepsen worker 46 - jepsen.util 10396 :ok :read [15 33] INFO [2019-12-17 12:36:10,051] jepsen worker 46 - jepsen.util 10396 :invoke :read [15 nil] INFO [2019-12-17 12:36:10,062] jepsen worker 14 - jepsen.util 10274 :ok :read [4 7] INFO [2019-12-17 12:36:10,062] jepsen worker 14 - jepsen.util 10274 :invoke :cas [4 [22 24]] INFO [2019-12-17 12:36:10,071] jepsen worker 73 - jepsen.util 9793 :fail :cas [24 [44 13]] :ValueNotSwapFrom INFO [2019-12-17 12:36:10,072] jepsen worker 73 - jepsen.util 9793 :invoke :read [24 nil] INFO [2019-12-17 12:36:10,080] jepsen worker 11 - jepsen.util 10001 :fail :cas [3 [26 21]] :ValueNotSwapFrom INFO [2019-12-17 12:36:10,080] jepsen worker 11 - jepsen.util 10001 :invoke :write [3 4] INFO [2019-12-17 12:36:10,082] jepsen worker 64 - jepsen.util 8074 :ok :read [21 40] INFO [2019-12-17 12:36:10,082] jepsen worker 64 - jepsen.util 8074 :invoke :cas [21 [9 15]] INFO [2019-12-17 12:36:10,112] jepsen worker 4 - jepsen.util 9634 :fail :cas [1 [1 12]] :ValueNotSwapFrom INFO [2019-12-17 12:36:10,113] jepsen worker 4 - jepsen.util 9634 :invoke :read [1 nil] INFO [2019-12-17 12:36:10,116] jepsen worker 77 - jepsen.util 14837 :info :cas [25 [35 46]] :RequestCanceledException INFO [2019-12-17 12:36:10,116] jepsen worker 77 - jepsen.util 14927 :invoke :read [25 nil] INFO [2019-12-17 12:36:10,121] jepsen worker 27 - jepsen.util 10017 :ok :write [9 9] INFO [2019-12-17 12:36:10,122] jepsen worker 27 - jepsen.util 10017 :invoke :write [9 11] INFO [2019-12-17 12:36:10,132] jepsen worker 21 - jepsen.util 13791 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:36:10,132] jepsen worker 21 - jepsen.util 13791 :invoke :write [7 0] INFO [2019-12-17 12:36:10,145] jepsen worker 59 - jepsen.util 13379 :info :cas [19 [42 33]] :RequestCanceledException INFO [2019-12-17 12:36:10,147] jepsen worker 73 - jepsen.util 9793 :ok :read [24 15] INFO [2019-12-17 12:36:10,147] jepsen worker 73 - jepsen.util 9793 :invoke :cas [24 [21 11]] INFO [2019-12-17 12:36:10,149] jepsen worker 59 - jepsen.util 13469 :invoke :read [19 nil] INFO [2019-12-17 12:36:10,162] jepsen worker 34 - jepsen.util 14434 :info :cas [11 [39 43]] :RequestCanceledException INFO [2019-12-17 12:36:10,163] jepsen worker 34 - jepsen.util 14524 :invoke :cas [11 [34 45]] INFO [2019-12-17 12:36:10,165] jepsen worker 64 - jepsen.util 8074 :fail :cas [21 [9 15]] :ValueNotSwapFrom INFO [2019-12-17 12:36:10,166] jepsen worker 64 - jepsen.util 8074 :invoke :read [21 nil] INFO [2019-12-17 12:36:10,213] jepsen worker 4 - jepsen.util 9634 :ok :read [1 45] INFO [2019-12-17 12:36:10,223] jepsen worker 73 - jepsen.util 9793 :fail :cas [24 [21 11]] :ValueNotSwapFrom INFO [2019-12-17 12:36:10,227] jepsen worker 54 - jepsen.util 11394 :ok :read [18 38] INFO [2019-12-17 12:36:10,232] jepsen worker 86 - jepsen.util 15386 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:36:10,237] jepsen worker 73 - jepsen.util 9793 :invoke :read [24 nil] INFO [2019-12-17 12:36:10,237] jepsen worker 54 - jepsen.util 11394 :invoke :read [18 nil] INFO [2019-12-17 12:36:10,237] jepsen worker 4 - jepsen.util 9634 :invoke :cas [1 [33 29]] INFO [2019-12-17 12:36:10,237] jepsen worker 86 - jepsen.util 15386 :invoke :write [28 27] INFO [2019-12-17 12:36:10,242] jepsen worker 71 - jepsen.util 11321 :fail :cas [23 [43 40]] :ValueNotSwapFrom INFO [2019-12-17 12:36:10,242] jepsen worker 71 - jepsen.util 11321 :invoke :cas [23 [48 41]] INFO [2019-12-17 12:36:10,248] jepsen worker 64 - jepsen.util 8074 :ok :read [21 40] INFO [2019-12-17 12:36:10,248] jepsen worker 64 - jepsen.util 8074 :invoke :cas [21 [25 21]] INFO [2019-12-17 12:36:10,249] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":60331,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:36:10,249] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":60331,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:36:10,249] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":60336,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:36:10,249] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":60336,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:36:10,267] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":60325,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:36:10,267] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":60325,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:36:10,267] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":60326,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:36:10,267] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":60326,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:36:10,267] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":60327,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:36:10,268] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":60327,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:36:10,270] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:36:10,270] jepsen nemesis - couchbase.util Rebalance check count: 471 INFO [2019-12-17 12:36:10,271] jepsen nemesis - couchbase.util Rebalance stuck count: 452 INFO [2019-12-17 12:36:10,272] jepsen worker 73 - jepsen.util 9793 :ok :read [24 15] INFO [2019-12-17 12:36:10,272] jepsen worker 73 - jepsen.util 9793 :invoke :cas [24 [9 42]] INFO [2019-12-17 12:36:10,278] jepsen worker 64 - jepsen.util 8074 :fail :cas [21 [25 21]] :ValueNotSwapFrom INFO [2019-12-17 12:36:10,278] jepsen worker 64 - jepsen.util 8074 :invoke :cas [21 [46 25]] INFO [2019-12-17 12:36:10,294] jepsen worker 73 - jepsen.util 9793 :fail :cas [24 [9 42]] :ValueNotSwapFrom INFO [2019-12-17 12:36:10,294] jepsen worker 56 - jepsen.util 10946 :fail :cas [18 [12 23]] :ValueNotSwapFrom INFO [2019-12-17 12:36:10,294] jepsen worker 73 - jepsen.util 9793 :invoke :read [24 nil] INFO [2019-12-17 12:36:10,294] jepsen worker 56 - jepsen.util 10946 :invoke :read [18 nil] INFO [2019-12-17 12:36:10,299] jepsen worker 84 - jepsen.util 13944 :info :cas [28 [19 6]] :RequestCanceledException INFO [2019-12-17 12:36:10,300] jepsen worker 84 - jepsen.util 14034 :invoke :read [28 nil] INFO [2019-12-17 12:36:10,310] jepsen worker 64 - jepsen.util 8074 :fail :cas [21 [46 25]] :ValueNotSwapFrom INFO [2019-12-17 12:36:10,310] jepsen worker 64 - jepsen.util 8074 :invoke :cas [21 [26 2]] INFO [2019-12-17 12:36:10,316] jepsen worker 8 - jepsen.util 13778 :info :write [2 8] :RequestCanceledException INFO [2019-12-17 12:36:10,317] jepsen worker 8 - jepsen.util 13868 :invoke :cas [2 [42 27]] INFO [2019-12-17 12:36:10,322] jepsen worker 4 - jepsen.util 9634 :fail :cas [1 [33 29]] :ValueNotSwapFrom INFO [2019-12-17 12:36:10,322] jepsen worker 4 - jepsen.util 9634 :invoke :cas [1 [15 7]] INFO [2019-12-17 12:36:10,331] jepsen worker 16 - jepsen.util 15586 :info :cas [5 [16 37]] :RequestCanceledException INFO [2019-12-17 12:36:10,331] jepsen worker 43 - jepsen.util 14173 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:36:10,331] jepsen worker 22 - jepsen.util 13972 :info :cas [7 [35 25]] :RequestCanceledException INFO [2019-12-17 12:36:10,332] jepsen worker 43 - jepsen.util 14173 :invoke :cas [14 [29 36]] INFO [2019-12-17 12:36:10,332] jepsen worker 16 - jepsen.util 15676 :invoke :read [5 nil] INFO [2019-12-17 12:36:10,333] jepsen worker 22 - jepsen.util 14062 :invoke :cas [7 [42 18]] INFO [2019-12-17 12:36:10,343] jepsen worker 71 - jepsen.util 11321 :fail :cas [23 [48 41]] :ValueNotSwapFrom INFO [2019-12-17 12:36:10,343] jepsen worker 71 - jepsen.util 11321 :invoke :read [23 nil] INFO [2019-12-17 12:36:10,348] jepsen worker 73 - jepsen.util 9793 :ok :read [24 15] INFO [2019-12-17 12:36:10,348] jepsen worker 73 - jepsen.util 9793 :invoke :read [24 nil] INFO [2019-12-17 12:36:10,351] jepsen worker 52 - jepsen.util 10312 :ok :write [17 48] INFO [2019-12-17 12:36:10,352] jepsen worker 52 - jepsen.util 10312 :invoke :write [17 18] INFO [2019-12-17 12:36:10,362] jepsen worker 64 - jepsen.util 8074 :fail :cas [21 [26 2]] :ValueNotSwapFrom INFO [2019-12-17 12:36:10,362] jepsen worker 64 - jepsen.util 8074 :invoke :cas [21 [28 32]] INFO [2019-12-17 12:36:10,380] jepsen worker 73 - jepsen.util 9793 :ok :read [24 15] INFO [2019-12-17 12:36:10,380] jepsen worker 73 - jepsen.util 9793 :invoke :write [24 21] INFO [2019-12-17 12:36:10,399] jepsen worker 67 - jepsen.util 14287 :info :write [22 37] :RequestCanceledException INFO [2019-12-17 12:36:10,402] jepsen worker 67 - jepsen.util 14377 :invoke :cas [22 [27 7]] INFO [2019-12-17 12:36:10,403] jepsen worker 64 - jepsen.util 8074 :fail :cas [21 [28 32]] :ValueNotSwapFrom INFO [2019-12-17 12:36:10,404] jepsen worker 64 - jepsen.util 8074 :invoke :write [21 26] INFO [2019-12-17 12:36:10,407] jepsen worker 27 - jepsen.util 10017 :ok :write [9 11] INFO [2019-12-17 12:36:10,408] jepsen worker 27 - jepsen.util 10017 :invoke :write [9 2] INFO [2019-12-17 12:36:10,416] jepsen worker 50 - jepsen.util 14810 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:36:10,416] jepsen worker 50 - jepsen.util 14810 :invoke :read [16 nil] INFO [2019-12-17 12:36:10,426] jepsen worker 14 - jepsen.util 10274 :fail :cas [4 [22 24]] :ValueNotSwapFrom INFO [2019-12-17 12:36:10,426] jepsen worker 14 - jepsen.util 10274 :invoke :read [4 nil] INFO [2019-12-17 12:36:10,452] jepsen worker 4 - jepsen.util 9634 :fail :cas [1 [15 7]] :ValueNotSwapFrom INFO [2019-12-17 12:36:10,452] jepsen worker 4 - jepsen.util 9634 :invoke :read [1 nil] INFO [2019-12-17 12:36:10,457] jepsen worker 19 - jepsen.util 10549 :ok :write [6 22] INFO [2019-12-17 12:36:10,457] jepsen worker 19 - jepsen.util 10549 :invoke :cas [6 [20 16]] INFO [2019-12-17 12:36:10,467] jepsen worker 10 - jepsen.util 10450 :ok :write [3 11] INFO [2019-12-17 12:36:10,467] jepsen worker 10 - jepsen.util 10450 :invoke :cas [3 [43 30]] INFO [2019-12-17 12:36:10,471] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":60364,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:36:10,471] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":60364,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:36:10,471] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":60366,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:36:10,471] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":60366,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:36:10,471] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":60369,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:36:10,471] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":60369,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:36:10,472] jepsen worker 74 - jepsen.util 10334 :fail :cas [24 [8 21]] :ValueNotSwapFrom INFO [2019-12-17 12:36:10,473] jepsen worker 74 - jepsen.util 10334 :invoke :write [24 1] INFO [2019-12-17 12:36:10,484] jepsen worker 55 - jepsen.util 11035 :ok :write [18 7] INFO [2019-12-17 12:36:10,484] jepsen worker 55 - jepsen.util 11035 :invoke :cas [18 [25 41]] INFO [2019-12-17 12:36:10,499] jepsen worker 69 - jepsen.util 10599 :fail :cas [23 [32 34]] :ValueNotSwapFrom INFO [2019-12-17 12:36:10,499] jepsen worker 69 - jepsen.util 10599 :invoke :write [23 42] INFO [2019-12-17 12:36:10,504] jepsen worker 10 - jepsen.util 10450 :fail :cas [3 [43 30]] :ValueNotSwapFrom INFO [2019-12-17 12:36:10,504] jepsen worker 10 - jepsen.util 10450 :invoke :cas [3 [6 12]] INFO [2019-12-17 12:36:10,505] jepsen worker 64 - jepsen.util 8074 :ok :write [21 26] INFO [2019-12-17 12:36:10,505] jepsen worker 64 - jepsen.util 8074 :invoke :read [21 nil] INFO [2019-12-17 12:36:10,524] jepsen worker 27 - jepsen.util 10017 :ok :write [9 2] INFO [2019-12-17 12:36:10,525] jepsen worker 27 - jepsen.util 10017 :invoke :read [9 nil] INFO [2019-12-17 12:36:10,531] jepsen worker 64 - jepsen.util 8074 :ok :read [21 26] INFO [2019-12-17 12:36:10,531] jepsen worker 64 - jepsen.util 8074 :invoke :cas [21 [36 30]] INFO [2019-12-17 12:36:10,532] jepsen worker 18 - jepsen.util 10818 :fail :read [6 nil] :RequestTimeoutException INFO [2019-12-17 12:36:10,532] jepsen worker 45 - jepsen.util 10575 :info :write [15 40] :RequestCanceledException INFO [2019-12-17 12:36:10,532] jepsen worker 30 - jepsen.util 10110 :fail :read [10 nil] :RequestTimeoutException INFO [2019-12-17 12:36:10,532] jepsen worker 18 - jepsen.util 10818 :invoke :cas [6 [19 42]] INFO [2019-12-17 12:36:10,532] jepsen worker 30 - jepsen.util 10110 :invoke :read [10 nil] INFO [2019-12-17 12:36:10,533] jepsen worker 45 - jepsen.util 10665 :invoke :read [15 nil] INFO [2019-12-17 12:36:10,533] jepsen worker 10 - jepsen.util 10450 :fail :cas [3 [6 12]] :ValueNotSwapFrom INFO [2019-12-17 12:36:10,533] jepsen worker 10 - jepsen.util 10450 :invoke :cas [3 [31 0]] INFO [2019-12-17 12:36:10,545] jepsen worker 57 - jepsen.util 13917 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:36:10,545] jepsen worker 57 - jepsen.util 13917 :invoke :cas [19 [16 36]] INFO [2019-12-17 12:36:10,561] jepsen worker 38 - jepsen.util 11198 :fail :cas [12 [30 21]] :ValueNotSwapFrom INFO [2019-12-17 12:36:10,561] jepsen worker 38 - jepsen.util 11198 :invoke :cas [12 [8 3]] INFO [2019-12-17 12:36:10,562] jepsen worker 15 - jepsen.util 13155 :info :write [5 15] :RequestCanceledException INFO [2019-12-17 12:36:10,563] jepsen worker 13 - jepsen.util 9913 :info :write [4 49] :RequestCanceledException INFO [2019-12-17 12:36:10,564] jepsen worker 15 - jepsen.util 13245 :invoke :read [5 nil] INFO [2019-12-17 12:36:10,565] jepsen worker 13 - jepsen.util 10003 :invoke :cas [4 [13 7]] INFO [2019-12-17 12:36:10,565] jepsen worker 27 - jepsen.util 10017 :ok :read [9 2] INFO [2019-12-17 12:36:10,565] jepsen worker 27 - jepsen.util 10017 :invoke :read [9 nil] INFO [2019-12-17 12:36:10,574] jepsen worker 71 - jepsen.util 11321 :ok :read [23 14] INFO [2019-12-17 12:36:10,574] jepsen worker 71 - jepsen.util 11321 :invoke :cas [23 [47 47]] INFO [2019-12-17 12:36:10,575] jepsen worker 55 - jepsen.util 11035 :fail :cas [18 [25 41]] :ValueNotSwapFrom INFO [2019-12-17 12:36:10,576] jepsen worker 55 - jepsen.util 11035 :invoke :write [18 26] INFO [2019-12-17 12:36:10,601] jepsen worker 10 - jepsen.util 10450 :fail :cas [3 [31 0]] :ValueNotSwapFrom INFO [2019-12-17 12:36:10,601] jepsen worker 10 - jepsen.util 10450 :invoke :write [3 0] INFO [2019-12-17 12:36:10,612] jepsen worker 64 - jepsen.util 8074 :fail :cas [21 [36 30]] :ValueNotSwapFrom INFO [2019-12-17 12:36:10,612] jepsen worker 64 - jepsen.util 8074 :invoke :write [21 15] INFO [2019-12-17 12:36:10,620] jepsen worker 18 - jepsen.util 10818 :fail :cas [6 [19 42]] :ValueNotSwapFrom INFO [2019-12-17 12:36:10,621] jepsen worker 18 - jepsen.util 10818 :invoke :write [6 35] INFO [2019-12-17 12:36:10,646] jepsen worker 63 - jepsen.util 9513 :fail :read [21 nil] :RequestTimeoutException INFO [2019-12-17 12:36:10,646] jepsen worker 63 - jepsen.util 9513 :invoke :read [21 nil] INFO [2019-12-17 12:36:10,667] jepsen worker 30 - jepsen.util 10110 :ok :read [10 41] INFO [2019-12-17 12:36:10,668] jepsen worker 30 - jepsen.util 10110 :invoke :read [10 nil] INFO [2019-12-17 12:36:10,669] jepsen worker 73 - jepsen.util 9793 :ok :write [24 21] INFO [2019-12-17 12:36:10,670] jepsen worker 73 - jepsen.util 9793 :invoke :cas [24 [34 41]] INFO [2019-12-17 12:36:10,671] jepsen worker 27 - jepsen.util 10017 :ok :read [9 2] INFO [2019-12-17 12:36:10,671] jepsen worker 27 - jepsen.util 10017 :invoke :read [9 nil] INFO [2019-12-17 12:36:10,675] jepsen worker 45 - jepsen.util 10665 :ok :read [15 33] INFO [2019-12-17 12:36:10,675] jepsen worker 45 - jepsen.util 10665 :invoke :write [15 29] INFO [2019-12-17 12:36:10,690] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":60396,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:36:10,691] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":60396,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:36:10,691] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":60399,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:36:10,691] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":60399,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:36:10,691] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":60400,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:36:10,691] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":60400,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:36:10,695] jepsen worker 27 - jepsen.util 10017 :ok :read [9 2] INFO [2019-12-17 12:36:10,695] jepsen worker 27 - jepsen.util 10017 :invoke :write [9 11] INFO [2019-12-17 12:36:10,698] jepsen worker 2 - jepsen.util 13772 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:36:10,698] jepsen worker 42 - jepsen.util 14982 :info :write [14 3] :RequestCanceledException INFO [2019-12-17 12:36:10,698] jepsen worker 2 - jepsen.util 13772 :invoke :write [0 44] INFO [2019-12-17 12:36:10,701] jepsen worker 42 - jepsen.util 15072 :invoke :read [14 nil] INFO [2019-12-17 12:36:10,706] jepsen worker 4 - jepsen.util 9634 :ok :read [1 45] INFO [2019-12-17 12:36:10,707] jepsen worker 4 - jepsen.util 9634 :invoke :write [1 14] INFO [2019-12-17 12:36:10,708] jepsen worker 11 - jepsen.util 10001 :ok :write [3 4] INFO [2019-12-17 12:36:10,708] jepsen worker 11 - jepsen.util 10001 :invoke :cas [3 [45 22]] INFO [2019-12-17 12:36:10,717] jepsen worker 60 - jepsen.util 15090 :info :cas [20 [32 34]] :RequestCanceledException INFO [2019-12-17 12:36:10,717] jepsen worker 17 - jepsen.util 14057 :info :write [5 22] :RequestCanceledException INFO [2019-12-17 12:36:10,717] jepsen worker 17 - jepsen.util 14147 :invoke :write [5 13] INFO [2019-12-17 12:36:10,718] jepsen worker 60 - jepsen.util 15180 :invoke :read [20 nil] INFO [2019-12-17 12:36:10,730] jepsen worker 73 - jepsen.util 9793 :fail :cas [24 [34 41]] :ValueNotSwapFrom INFO [2019-12-17 12:36:10,730] jepsen worker 73 - jepsen.util 9793 :invoke :cas [24 [0 43]] INFO [2019-12-17 12:36:10,732] jepsen worker 41 - jepsen.util 15161 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:36:10,732] jepsen worker 24 - jepsen.util 12804 :info :cas [8 [40 30]] :RequestCanceledException INFO [2019-12-17 12:36:10,732] jepsen worker 41 - jepsen.util 15161 :invoke :read [13 nil] INFO [2019-12-17 12:36:10,733] jepsen worker 24 - jepsen.util 12894 :invoke :cas [8 [13 46]] INFO [2019-12-17 12:36:10,745] jepsen worker 76 - jepsen.util 14566 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:36:10,745] jepsen worker 76 - jepsen.util 14566 :invoke :write [25 34] INFO [2019-12-17 12:36:10,749] jepsen worker 14 - jepsen.util 10274 :ok :read [4 7] INFO [2019-12-17 12:36:10,749] jepsen worker 14 - jepsen.util 10274 :invoke :write [4 8] INFO [2019-12-17 12:36:10,763] jepsen worker 6 - jepsen.util 14586 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:36:10,763] jepsen worker 3 - jepsen.util 10173 :info :write [1 33] :RequestCanceledException INFO [2019-12-17 12:36:10,763] jepsen worker 6 - jepsen.util 14586 :invoke :read [2 nil] INFO [2019-12-17 12:36:10,764] jepsen worker 3 - jepsen.util 10263 :invoke :write [1 22] INFO [2019-12-17 12:36:10,791] jepsen worker 19 - jepsen.util 10549 :fail :cas [6 [20 16]] :ValueNotSwapFrom INFO [2019-12-17 12:36:10,791] jepsen worker 19 - jepsen.util 10549 :invoke :write [6 5] INFO [2019-12-17 12:36:10,802] jepsen worker 71 - jepsen.util 11321 :fail :cas [23 [47 47]] :ValueNotSwapFrom INFO [2019-12-17 12:36:10,802] jepsen worker 71 - jepsen.util 11321 :invoke :write [23 40] INFO [2019-12-17 12:36:10,813] jepsen worker 73 - jepsen.util 9793 :fail :cas [24 [0 43]] :ValueNotSwapFrom INFO [2019-12-17 12:36:10,813] jepsen worker 73 - jepsen.util 9793 :invoke :write [24 40] INFO [2019-12-17 12:36:10,816] jepsen worker 64 - jepsen.util 8074 :ok :write [21 15] INFO [2019-12-17 12:36:10,817] jepsen worker 64 - jepsen.util 8074 :invoke :read [21 nil] INFO [2019-12-17 12:36:10,853] jepsen worker 10 - jepsen.util 10450 :ok :write [3 0] INFO [2019-12-17 12:36:10,853] jepsen worker 10 - jepsen.util 10450 :invoke :cas [3 [35 10]] INFO [2019-12-17 12:36:10,886] jepsen worker 37 - jepsen.util 10027 :ok :write [12 4] INFO [2019-12-17 12:36:10,886] jepsen worker 37 - jepsen.util 10027 :invoke :write [12 15] INFO [2019-12-17 12:36:10,893] jepsen worker 10 - jepsen.util 10450 :fail :cas [3 [35 10]] :ValueNotSwapFrom INFO [2019-12-17 12:36:10,893] jepsen worker 10 - jepsen.util 10450 :invoke :write [3 36] INFO [2019-12-17 12:36:10,916] jepsen worker 51 - jepsen.util 9951 :info :write [17 25] :RequestCanceledException INFO [2019-12-17 12:36:10,917] jepsen worker 51 - jepsen.util 10041 :invoke :write [17 0] INFO [2019-12-17 12:36:10,923] jepsen worker 27 - jepsen.util 10017 :ok :write [9 11] INFO [2019-12-17 12:36:10,923] jepsen worker 27 - jepsen.util 10017 :invoke :cas [9 [35 45]] INFO [2019-12-17 12:36:10,931] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":60428,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:36:10,931] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":60428,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:36:10,931] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":60429,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:36:10,931] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":60429,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:36:10,964] jepsen worker 27 - jepsen.util 10017 :fail :cas [9 [35 45]] :ValueNotSwapFrom INFO [2019-12-17 12:36:10,964] jepsen worker 27 - jepsen.util 10017 :invoke :read [9 nil] INFO [2019-12-17 12:36:10,998] jepsen worker 27 - jepsen.util 10017 :ok :read [9 11] INFO [2019-12-17 12:36:10,998] jepsen worker 35 - jepsen.util 14165 :info :cas [11 [23 23]] :RequestCanceledException INFO [2019-12-17 12:36:10,998] jepsen worker 27 - jepsen.util 10017 :invoke :cas [9 [43 8]] INFO [2019-12-17 12:36:10,999] jepsen worker 35 - jepsen.util 14255 :invoke :cas [11 [11 24]] INFO [2019-12-17 12:36:11,005] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":60438,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:36:11,005] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":60438,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:36:11,005] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":60441,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:36:11,005] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":60441,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:36:11,005] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":60442,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:36:11,006] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":60442,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:36:11,015] jepsen worker 74 - jepsen.util 10334 :ok :write [24 1] INFO [2019-12-17 12:36:11,016] jepsen worker 74 - jepsen.util 10334 :invoke :cas [24 [1 23]] INFO [2019-12-17 12:36:11,020] jepsen worker 27 - jepsen.util 10017 :fail :cas [9 [43 8]] :ValueNotSwapFrom INFO [2019-12-17 12:36:11,020] jepsen worker 27 - jepsen.util 10017 :invoke :write [9 26] INFO [2019-12-17 12:36:11,031] jepsen worker 44 - jepsen.util 14444 :info :cas [14 [24 46]] :RequestCanceledException INFO [2019-12-17 12:36:11,032] jepsen worker 44 - jepsen.util 14534 :invoke :read [14 nil] INFO [2019-12-17 12:36:11,045] jepsen worker 62 - jepsen.util 14552 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:36:11,045] jepsen worker 62 - jepsen.util 14552 :invoke :cas [20 [15 25]] INFO [2019-12-17 12:36:11,112] jepsen worker 18 - jepsen.util 10818 :ok :write [6 35] INFO [2019-12-17 12:36:11,116] jepsen worker 85 - jepsen.util 15115 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:36:11,134] jepsen worker 27 - jepsen.util 10017 :ok :write [9 26] INFO [2019-12-17 12:36:11,140] jepsen worker 56 - jepsen.util 10946 :ok :read [18 7] INFO [2019-12-17 12:36:11,162] jepsen worker 23 - jepsen.util 14243 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:36:11,175] jepsen worker 85 - jepsen.util 15115 :invoke :cas [28 [1 16]] INFO [2019-12-17 12:36:11,175] jepsen worker 27 - jepsen.util 10017 :invoke :cas [9 [40 36]] INFO [2019-12-17 12:36:11,175] jepsen worker 18 - jepsen.util 10818 :invoke :write [6 8] INFO [2019-12-17 12:36:11,175] jepsen worker 56 - jepsen.util 10946 :invoke :write [18 49] INFO [2019-12-17 12:36:11,175] jepsen worker 23 - jepsen.util 14243 :invoke :write [7 27] INFO [2019-12-17 12:36:11,184] jepsen worker 54 - jepsen.util 11394 :ok :read [18 7] INFO [2019-12-17 12:36:11,184] jepsen worker 54 - jepsen.util 11394 :invoke :read [18 nil] INFO [2019-12-17 12:36:11,206] jepsen worker 27 - jepsen.util 10017 :fail :cas [9 [40 36]] :ValueNotSwapFrom INFO [2019-12-17 12:36:11,206] jepsen worker 27 - jepsen.util 10017 :invoke :read [9 nil] INFO [2019-12-17 12:36:11,219] jepsen worker 38 - jepsen.util 11198 :fail :cas [12 [8 3]] :ValueNotSwapFrom INFO [2019-12-17 12:36:11,219] jepsen worker 38 - jepsen.util 11198 :invoke :read [12 nil] INFO [2019-12-17 12:36:11,228] jepsen worker 10 - jepsen.util 10450 :ok :write [3 36] INFO [2019-12-17 12:36:11,228] jepsen worker 10 - jepsen.util 10450 :invoke :cas [3 [29 11]] INFO [2019-12-17 12:36:11,231] jepsen worker 40 - jepsen.util 14620 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:36:11,231] jepsen worker 20 - jepsen.util 10190 :info :write [6 39] :RequestCanceledException INFO [2019-12-17 12:36:11,232] jepsen worker 40 - jepsen.util 14620 :invoke :write [13 24] INFO [2019-12-17 12:36:11,233] jepsen worker 20 - jepsen.util 10280 :invoke :write [6 44] INFO [2019-12-17 12:36:11,233] jepsen worker 27 - jepsen.util 10017 :ok :read [9 26] INFO [2019-12-17 12:36:11,233] jepsen worker 27 - jepsen.util 10017 :invoke :write [9 37] INFO [2019-12-17 12:36:11,236] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":60464,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:36:11,236] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":60464,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:36:11,236] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":60466,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:36:11,236] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":60466,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:36:11,251] jepsen worker 10 - jepsen.util 10450 :fail :cas [3 [29 11]] :ValueNotSwapFrom INFO [2019-12-17 12:36:11,251] jepsen worker 10 - jepsen.util 10450 :invoke :cas [3 [32 16]] INFO [2019-12-17 12:36:11,253] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":60454,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:36:11,253] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":60454,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:36:11,253] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":60455,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:36:11,253] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":60455,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:36:11,272] jepsen worker 13 - jepsen.util 10003 :fail :cas [4 [13 7]] :ValueNotSwapFrom INFO [2019-12-17 12:36:11,272] jepsen worker 13 - jepsen.util 10003 :invoke :read [4 nil] INFO [2019-12-17 12:36:11,285] jepsen worker 10 - jepsen.util 10450 :fail :cas [3 [32 16]] :ValueNotSwapFrom INFO [2019-12-17 12:36:11,285] jepsen worker 10 - jepsen.util 10450 :invoke :cas [3 [28 15]] INFO [2019-12-17 12:36:11,285] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:36:11,285] jepsen nemesis - couchbase.util Rebalance check count: 472 INFO [2019-12-17 12:36:11,285] jepsen nemesis - couchbase.util Rebalance stuck count: 453 INFO [2019-12-17 12:36:11,330] jepsen worker 10 - jepsen.util 10450 :fail :cas [3 [28 15]] :ValueNotSwapFrom INFO [2019-12-17 12:36:11,330] jepsen worker 10 - jepsen.util 10450 :invoke :write [3 38] INFO [2019-12-17 12:36:11,332] jepsen worker 66 - jepsen.util 13926 :info :write [22 32] :RequestCanceledException INFO [2019-12-17 12:36:11,333] jepsen worker 66 - jepsen.util 14016 :invoke :cas [22 [27 5]] INFO [2019-12-17 12:36:11,340] jepsen worker 74 - jepsen.util 10334 :ok :cas [24 [1 23]] INFO [2019-12-17 12:36:11,340] jepsen worker 45 - jepsen.util 10665 :ok :write [15 29] INFO [2019-12-17 12:36:11,340] jepsen worker 74 - jepsen.util 10334 :invoke :cas [24 [14 3]] INFO [2019-12-17 12:36:11,340] jepsen worker 45 - jepsen.util 10665 :invoke :cas [15 [26 29]] INFO [2019-12-17 12:36:11,363] jepsen worker 12 - jepsen.util 10812 :info :write [4 34] :RequestCanceledException INFO [2019-12-17 12:36:11,363] jepsen worker 83 - jepsen.util 14393 :info :write [27 44] :RequestCanceledException INFO [2019-12-17 12:36:11,363] jepsen worker 9 - jepsen.util 11169 :fail :read [3 nil] :RequestTimeoutException INFO [2019-12-17 12:36:11,363] jepsen worker 9 - jepsen.util 11169 :invoke :write [3 40] INFO [2019-12-17 12:36:11,364] jepsen worker 12 - jepsen.util 10902 :invoke :cas [4 [12 18]] INFO [2019-12-17 12:36:11,367] jepsen worker 83 - jepsen.util 14483 :invoke :write [27 33] INFO [2019-12-17 12:36:11,388] jepsen worker 74 - jepsen.util 10334 :fail :cas [24 [14 3]] :ValueNotSwapFrom INFO [2019-12-17 12:36:11,389] jepsen worker 74 - jepsen.util 10334 :invoke :cas [24 [20 11]] INFO [2019-12-17 12:36:11,412] jepsen worker 54 - jepsen.util 11394 :ok :read [18 7] INFO [2019-12-17 12:36:11,412] jepsen worker 54 - jepsen.util 11394 :invoke :write [18 39] INFO [2019-12-17 12:36:11,417] jepsen worker 7 - jepsen.util 14317 :info :cas [2 [46 44]] :RequestCanceledException INFO [2019-12-17 12:36:11,417] jepsen worker 26 - jepsen.util 14876 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:36:11,417] jepsen worker 26 - jepsen.util 14876 :invoke :read [8 nil] INFO [2019-12-17 12:36:11,418] jepsen worker 7 - jepsen.util 14407 :invoke :read [2 nil] INFO [2019-12-17 12:36:11,419] jepsen worker 45 - jepsen.util 10665 :fail :cas [15 [26 29]] :ValueNotSwapFrom INFO [2019-12-17 12:36:11,419] jepsen worker 45 - jepsen.util 10665 :invoke :cas [15 [13 26]] INFO [2019-12-17 12:36:11,422] jepsen worker 74 - jepsen.util 10334 :fail :cas [24 [20 11]] :ValueNotSwapFrom INFO [2019-12-17 12:36:11,422] jepsen worker 74 - jepsen.util 10334 :invoke :read [24 nil] INFO [2019-12-17 12:36:11,431] jepsen worker 0 - jepsen.util 14940 :info :write [0 3] :RequestCanceledException INFO [2019-12-17 12:36:11,431] jepsen worker 61 - jepsen.util 15001 :info :cas [20 [25 8]] :RequestCanceledException INFO [2019-12-17 12:36:11,431] jepsen worker 49 - jepsen.util 13999 :info :write [16 20] :RequestCanceledException INFO [2019-12-17 12:36:11,432] jepsen worker 0 - jepsen.util 15030 :invoke :read [0 nil] INFO [2019-12-17 12:36:11,432] jepsen worker 49 - jepsen.util 14089 :invoke :cas [16 [42 17]] INFO [2019-12-17 12:36:11,432] jepsen worker 61 - jepsen.util 15091 :invoke :cas [20 [24 12]] INFO [2019-12-17 12:36:11,433] jepsen worker 27 - jepsen.util 10017 :ok :write [9 37] INFO [2019-12-17 12:36:11,433] jepsen worker 27 - jepsen.util 10017 :invoke :write [9 38] INFO [2019-12-17 12:36:11,476] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":60492,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:36:11,476] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":60492,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:36:11,476] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":60495,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:36:11,476] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":60495,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:36:11,476] jepsen worker 47 - jepsen.util 11117 :info :write [15 38] :RequestCanceledException INFO [2019-12-17 12:36:11,476] jepsen worker 1 - jepsen.util 14311 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:36:11,476] jepsen worker 1 - jepsen.util 14311 :invoke :read [0 nil] INFO [2019-12-17 12:36:11,477] jepsen worker 38 - jepsen.util 11198 :ok :read [12 4] INFO [2019-12-17 12:36:11,477] jepsen worker 38 - jepsen.util 11198 :invoke :write [12 15] INFO [2019-12-17 12:36:11,478] jepsen worker 47 - jepsen.util 11207 :invoke :read [15 nil] INFO [2019-12-17 12:36:11,478] jepsen worker 45 - jepsen.util 10665 :fail :cas [15 [13 26]] :ValueNotSwapFrom INFO [2019-12-17 12:36:11,478] jepsen worker 45 - jepsen.util 10665 :invoke :write [15 47] INFO [2019-12-17 12:36:11,484] jepsen worker 74 - jepsen.util 10334 :ok :read [24 23] INFO [2019-12-17 12:36:11,484] jepsen worker 74 - jepsen.util 10334 :invoke :write [24 49] INFO [2019-12-17 12:36:11,516] jepsen worker 88 - jepsen.util 10888 :info :write [29 15] :RequestCanceledException INFO [2019-12-17 12:36:11,521] jepsen worker 88 - jepsen.util 10978 :invoke :write [29 8] INFO [2019-12-17 12:36:11,531] jepsen worker 72 - jepsen.util 10602 :info :write [24 22] :RequestCanceledException INFO [2019-12-17 12:36:11,532] jepsen worker 72 - jepsen.util 10692 :invoke :cas [24 [40 37]] INFO [2019-12-17 12:36:11,554] jepsen worker 27 - jepsen.util 10017 :ok :write [9 38] INFO [2019-12-17 12:36:11,554] jepsen worker 27 - jepsen.util 10017 :invoke :cas [9 [10 47]] INFO [2019-12-17 12:36:11,575] jepsen worker 27 - jepsen.util 10017 :fail :cas [9 [10 47]] :ValueNotSwapFrom INFO [2019-12-17 12:36:11,575] jepsen worker 27 - jepsen.util 10017 :invoke :read [9 nil] INFO [2019-12-17 12:36:11,592] jepsen worker 27 - jepsen.util 10017 :ok :read [9 38] INFO [2019-12-17 12:36:11,593] jepsen worker 27 - jepsen.util 10017 :invoke :write [9 11] INFO [2019-12-17 12:36:11,632] jepsen worker 70 - jepsen.util 11230 :info :write [23 9] :RequestCanceledException INFO [2019-12-17 12:36:11,632] jepsen worker 53 - jepsen.util 9863 :info :write [17 39] :RequestCanceledException INFO [2019-12-17 12:36:11,632] jepsen worker 80 - jepsen.util 10160 :info :write [26 23] :RequestCanceledException INFO [2019-12-17 12:36:11,633] jepsen worker 53 - jepsen.util 9953 :invoke :read [17 nil] INFO [2019-12-17 12:36:11,633] jepsen worker 70 - jepsen.util 11320 :invoke :read [23 nil] INFO [2019-12-17 12:36:11,634] jepsen worker 80 - jepsen.util 10250 :invoke :write [26 29] INFO [2019-12-17 12:36:11,642] jepsen worker 27 - jepsen.util 10017 :ok :write [9 11] INFO [2019-12-17 12:36:11,642] jepsen worker 27 - jepsen.util 10017 :invoke :write [9 26] INFO [2019-12-17 12:36:11,645] jepsen worker 79 - jepsen.util 10789 :info :write [26 22] :RequestCanceledException INFO [2019-12-17 12:36:11,647] jepsen worker 79 - jepsen.util 10879 :invoke :write [26 22] INFO [2019-12-17 12:36:11,690] jepsen worker 27 - jepsen.util 10017 :ok :write [9 26] INFO [2019-12-17 12:36:11,690] jepsen worker 27 - jepsen.util 10017 :invoke :cas [9 [41 3]] INFO [2019-12-17 12:36:11,695] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":60513,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:36:11,695] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":60513,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:36:11,698] jepsen worker 75 - jepsen.util 13665 :info :cas [25 [31 25]] :RequestCanceledException INFO [2019-12-17 12:36:11,699] jepsen worker 75 - jepsen.util 13755 :invoke :write [25 17] INFO [2019-12-17 12:36:11,715] jepsen worker 27 - jepsen.util 10017 :fail :cas [9 [41 3]] :ValueNotSwapFrom INFO [2019-12-17 12:36:11,715] jepsen worker 27 - jepsen.util 10017 :invoke :write [9 40] INFO [2019-12-17 12:36:11,724] jepsen worker 89 - jepsen.util 10799 :fail :read [29 nil] :RequestTimeoutException INFO [2019-12-17 12:36:11,724] jepsen worker 25 - jepsen.util 15325 :info :cas [8 [34 26]] :RequestCanceledException INFO [2019-12-17 12:36:11,724] jepsen worker 28 - jepsen.util 8578 :info :write [9 19] :RequestCanceledException INFO [2019-12-17 12:36:11,724] jepsen worker 89 - jepsen.util 10799 :invoke :cas [29 [21 49]] INFO [2019-12-17 12:36:11,725] jepsen worker 28 - jepsen.util 8668 :invoke :cas [9 [2 45]] INFO [2019-12-17 12:36:11,725] jepsen worker 25 - jepsen.util 15415 :invoke :cas [8 [32 40]] INFO [2019-12-17 12:36:11,746] jepsen worker 68 - jepsen.util 13748 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:36:11,746] jepsen worker 68 - jepsen.util 13748 :invoke :read [22 nil] INFO [2019-12-17 12:36:11,757] jepsen worker 12 - jepsen.util 10902 :fail :cas [4 [12 18]] :ValueNotSwapFrom INFO [2019-12-17 12:36:11,757] jepsen worker 12 - jepsen.util 10902 :invoke :write [4 0] INFO [2019-12-17 12:36:11,758] jepsen worker 27 - jepsen.util 10017 :ok :write [9 40] INFO [2019-12-17 12:36:11,758] jepsen worker 27 - jepsen.util 10017 :invoke :write [9 25] INFO [2019-12-17 12:36:11,763] jepsen worker 58 - jepsen.util 14548 :info :cas [19 [48 10]] :RequestCanceledException INFO [2019-12-17 12:36:11,765] jepsen worker 58 - jepsen.util 14638 :invoke :read [19 nil] INFO [2019-12-17 12:36:11,804] jepsen worker 27 - jepsen.util 10017 :ok :write [9 25] INFO [2019-12-17 12:36:11,804] jepsen worker 27 - jepsen.util 10017 :invoke :cas [9 [37 6]] INFO [2019-12-17 12:36:11,806] jepsen worker 53 - jepsen.util 9953 :ok :read [17 25] INFO [2019-12-17 12:36:11,806] jepsen worker 53 - jepsen.util 9953 :invoke :cas [17 [29 15]] INFO [2019-12-17 12:36:11,816] jepsen worker 82 - jepsen.util 14482 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:36:11,816] jepsen worker 82 - jepsen.util 14482 :invoke :read [27 nil] INFO [2019-12-17 12:36:11,822] jepsen worker 27 - jepsen.util 10017 :fail :cas [9 [37 6]] :ValueNotSwapFrom INFO [2019-12-17 12:36:11,823] jepsen worker 27 - jepsen.util 10017 :invoke :cas [9 [12 48]] INFO [2019-12-17 12:36:11,825] jepsen worker 36 - jepsen.util 11286 :info :cas [12 [35 0]] :RequestCanceledException INFO [2019-12-17 12:36:11,826] jepsen worker 36 - jepsen.util 11376 :invoke :read [12 nil] INFO [2019-12-17 12:36:11,836] jepsen worker 13 - jepsen.util 10003 :ok :read [4 7] INFO [2019-12-17 12:36:11,836] jepsen worker 13 - jepsen.util 10003 :invoke :write [4 27] INFO [2019-12-17 12:36:11,842] jepsen worker 27 - jepsen.util 10017 :fail :cas [9 [12 48]] :ValueNotSwapFrom INFO [2019-12-17 12:36:11,843] jepsen worker 27 - jepsen.util 10017 :invoke :read [9 nil] INFO [2019-12-17 12:36:11,862] jepsen worker 27 - jepsen.util 10017 :ok :read [9 25] INFO [2019-12-17 12:36:11,862] jepsen worker 27 - jepsen.util 10017 :invoke :write [9 28] INFO [2019-12-17 12:36:11,887] jepsen worker 89 - jepsen.util 10799 :fail :cas [29 [21 49]] :ValueNotSwapFrom INFO [2019-12-17 12:36:11,887] jepsen worker 89 - jepsen.util 10799 :invoke :read [29 nil] INFO [2019-12-17 12:36:11,896] jepsen worker 53 - jepsen.util 9953 :fail :cas [17 [29 15]] :ValueNotSwapFrom INFO [2019-12-17 12:36:11,896] jepsen worker 53 - jepsen.util 9953 :invoke :write [17 20] INFO [2019-12-17 12:36:11,911] jepsen worker 89 - jepsen.util 10799 :ok :read [29 48] INFO [2019-12-17 12:36:11,911] jepsen worker 89 - jepsen.util 10799 :invoke :read [29 nil] INFO [2019-12-17 12:36:11,914] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":60537,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:36:11,915] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":60537,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:36:11,915] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":60540,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:36:11,915] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":60540,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:36:11,915] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":60541,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:36:11,915] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":60541,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:36:11,915] jepsen worker 27 - jepsen.util 10017 :ok :write [9 28] INFO [2019-12-17 12:36:11,915] jepsen worker 27 - jepsen.util 10017 :invoke :write [9 41] INFO [2019-12-17 12:36:11,932] jepsen worker 48 - jepsen.util 13098 :info :write [16 34] :RequestCanceledException INFO [2019-12-17 12:36:11,933] jepsen worker 48 - jepsen.util 13188 :invoke :write [16 40] INFO [2019-12-17 12:36:11,946] jepsen worker 29 - jepsen.util 8849 :info :cas [9 [35 12]] :RequestCanceledException INFO [2019-12-17 12:36:11,946] jepsen worker 5 - jepsen.util 9995 :info :write [1 5] :RequestCanceledException INFO [2019-12-17 12:36:11,947] jepsen worker 5 - jepsen.util 10085 :invoke :write [1 14] INFO [2019-12-17 12:36:11,947] jepsen worker 29 - jepsen.util 8939 :invoke :write [9 36] INFO [2019-12-17 12:36:11,972] jepsen worker 27 - jepsen.util 10017 :ok :write [9 41] INFO [2019-12-17 12:36:11,973] jepsen worker 27 - jepsen.util 10017 :invoke :cas [9 [11 11]] INFO [2019-12-17 12:36:11,986] jepsen worker 89 - jepsen.util 10799 :ok :read [29 48] INFO [2019-12-17 12:36:11,986] jepsen worker 89 - jepsen.util 10799 :invoke :read [29 nil] INFO [2019-12-17 12:36:11,989] jepsen worker 27 - jepsen.util 10017 :fail :cas [9 [11 11]] :ValueNotSwapFrom INFO [2019-12-17 12:36:11,989] jepsen worker 27 - jepsen.util 10017 :invoke :read [9 nil] INFO [2019-12-17 12:36:11,998] jepsen worker 31 - jepsen.util 9211 :fail :read [10 nil] :RequestTimeoutException INFO [2019-12-17 12:36:11,998] jepsen worker 31 - jepsen.util 9211 :invoke :read [10 nil] INFO [2019-12-17 12:36:12,006] jepsen worker 27 - jepsen.util 10017 :ok :read [9 41] INFO [2019-12-17 12:36:12,006] jepsen worker 27 - jepsen.util 10017 :invoke :read [9 nil] INFO [2019-12-17 12:36:12,015] jepsen worker 87 - jepsen.util 9987 :info :write [29 27] :RequestCanceledException INFO [2019-12-17 12:36:12,015] jepsen worker 33 - jepsen.util 14253 :info :cas [11 [0 12]] :RequestCanceledException INFO [2019-12-17 12:36:12,016] jepsen worker 31 - jepsen.util 9211 :ok :read [10 41] INFO [2019-12-17 12:36:12,016] jepsen worker 31 - jepsen.util 9211 :invoke :write [10 34] INFO [2019-12-17 12:36:12,016] jepsen worker 87 - jepsen.util 10077 :invoke :read [29 nil] INFO [2019-12-17 12:36:12,016] jepsen worker 33 - jepsen.util 14343 :invoke :write [11 21] INFO [2019-12-17 12:36:12,024] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":60554,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:36:12,024] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":60554,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:36:12,024] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":60555,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:36:12,024] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":60555,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:36:12,031] jepsen worker 27 - jepsen.util 10017 :ok :read [9 41] INFO [2019-12-17 12:36:12,031] jepsen worker 27 - jepsen.util 10017 :invoke :write [9 13] INFO [2019-12-17 12:36:12,032] jepsen worker 81 - jepsen.util 13941 :info :cas [27 [0 43]] :RequestCanceledException INFO [2019-12-17 12:36:12,033] jepsen worker 81 - jepsen.util 14031 :invoke :read [27 nil] INFO [2019-12-17 12:36:12,041] jepsen worker 89 - jepsen.util 10799 :ok :read [29 48] INFO [2019-12-17 12:36:12,041] jepsen worker 89 - jepsen.util 10799 :invoke :read [29 nil] INFO [2019-12-17 12:36:12,045] jepsen worker 32 - jepsen.util 10562 :fail :read [10 nil] :RequestTimeoutException INFO [2019-12-17 12:36:12,045] jepsen worker 32 - jepsen.util 10562 :invoke :read [10 nil] INFO [2019-12-17 12:36:12,054] jepsen worker 87 - jepsen.util 10077 :ok :read [29 48] INFO [2019-12-17 12:36:12,054] jepsen worker 87 - jepsen.util 10077 :invoke :read [29 nil] INFO [2019-12-17 12:36:12,063] jepsen worker 65 - jepsen.util 9065 :fail :read [21 nil] :RequestTimeoutException INFO [2019-12-17 12:36:12,063] jepsen worker 46 - jepsen.util 10396 :fail :read [15 nil] :RequestTimeoutException INFO [2019-12-17 12:36:12,063] jepsen worker 78 - jepsen.util 10428 :info :write [26 17] :RequestCanceledException INFO [2019-12-17 12:36:12,063] jepsen worker 65 - jepsen.util 9065 :invoke :write [21 2] INFO [2019-12-17 12:36:12,063] jepsen worker 46 - jepsen.util 10396 :invoke :read [15 nil] INFO [2019-12-17 12:36:12,064] jepsen worker 78 - jepsen.util 10518 :invoke :read [26 nil] INFO [2019-12-17 12:36:12,131] jepsen worker 77 - jepsen.util 14927 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:36:12,132] jepsen worker 77 - jepsen.util 14927 :invoke :write [25 11] INFO [2019-12-17 12:36:12,146] jepsen worker 39 - jepsen.util 13809 :info :write [13 9] :RequestCanceledException INFO [2019-12-17 12:36:12,147] jepsen worker 39 - jepsen.util 13899 :invoke :read [13 nil] INFO [2019-12-17 12:36:12,181] jepsen worker 89 - jepsen.util 10799 :ok :read [29 48] INFO [2019-12-17 12:36:12,181] jepsen worker 89 - jepsen.util 10799 :invoke :write [29 8] INFO [2019-12-17 12:36:12,198] jepsen worker 59 - jepsen.util 13469 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:36:12,198] jepsen worker 59 - jepsen.util 13469 :invoke :cas [19 [43 22]] INFO [2019-12-17 12:36:12,209] jepsen worker 87 - jepsen.util 10077 :ok :read [29 48] INFO [2019-12-17 12:36:12,210] jepsen worker 87 - jepsen.util 10077 :invoke :cas [29 [13 19]] INFO [2019-12-17 12:36:12,215] jepsen worker 31 - jepsen.util 9211 :ok :write [10 34] INFO [2019-12-17 12:36:12,215] jepsen worker 31 - jepsen.util 9211 :invoke :read [10 nil] INFO [2019-12-17 12:36:12,216] jepsen worker 45 - jepsen.util 10665 :ok :write [15 47] INFO [2019-12-17 12:36:12,216] jepsen worker 74 - jepsen.util 10334 :ok :write [24 49] INFO [2019-12-17 12:36:12,216] jepsen worker 45 - jepsen.util 10665 :invoke :cas [15 [33 30]] INFO [2019-12-17 12:36:12,216] jepsen worker 74 - jepsen.util 10334 :invoke :cas [24 [12 27]] INFO [2019-12-17 12:36:12,216] jepsen worker 34 - jepsen.util 14524 :info :cas [11 [34 45]] :RequestCanceledException INFO [2019-12-17 12:36:12,217] jepsen worker 34 - jepsen.util 14614 :invoke :cas [11 [1 1]] INFO [2019-12-17 12:36:12,229] jepsen worker 21 - jepsen.util 13791 :info :write [7 0] :RequestCanceledException INFO [2019-12-17 12:36:12,230] jepsen worker 21 - jepsen.util 13881 :invoke :read [7 nil] INFO [2019-12-17 12:36:12,243] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":60576,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:36:12,243] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":60588,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:36:12,243] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":60576,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:36:12,243] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":60588,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:36:12,243] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":60578,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:36:12,243] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":60591,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:36:12,244] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":60578,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:36:12,244] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":60591,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:36:12,244] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":60592,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:36:12,244] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":60592,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:36:12,247] jepsen worker 18 - jepsen.util 10818 :ok :write [6 8] INFO [2019-12-17 12:36:12,247] jepsen worker 18 - jepsen.util 10818 :invoke :write [6 1] INFO [2019-12-17 12:36:12,248] jepsen worker 32 - jepsen.util 10562 :ok :read [10 34] INFO [2019-12-17 12:36:12,248] jepsen worker 32 - jepsen.util 10562 :invoke :read [10 nil] INFO [2019-12-17 12:36:12,251] jepsen worker 10 - jepsen.util 10450 :ok :write [3 38] INFO [2019-12-17 12:36:12,251] jepsen worker 10 - jepsen.util 10450 :invoke :read [3 nil] INFO [2019-12-17 12:36:12,255] jepsen worker 45 - jepsen.util 10665 :fail :cas [15 [33 30]] :ValueNotSwapFrom INFO [2019-12-17 12:36:12,255] jepsen worker 45 - jepsen.util 10665 :invoke :cas [15 [19 21]] INFO [2019-12-17 12:36:12,309] jepsen worker 74 - jepsen.util 10334 :fail :cas [24 [12 27]] :ValueNotSwapFrom INFO [2019-12-17 12:36:12,309] jepsen worker 74 - jepsen.util 10334 :invoke :write [24 1] INFO [2019-12-17 12:36:12,324] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:36:12,324] jepsen nemesis - couchbase.util Rebalance check count: 473 INFO [2019-12-17 12:36:12,324] jepsen nemesis - couchbase.util Rebalance stuck count: 454 INFO [2019-12-17 12:36:12,331] jepsen worker 86 - jepsen.util 15386 :info :write [28 27] :RequestCanceledException INFO [2019-12-17 12:36:12,332] jepsen worker 84 - jepsen.util 14034 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:36:12,332] jepsen worker 84 - jepsen.util 14034 :invoke :read [28 nil] INFO [2019-12-17 12:36:12,333] jepsen worker 86 - jepsen.util 15476 :invoke :read [28 nil] INFO [2019-12-17 12:36:12,345] jepsen worker 8 - jepsen.util 13868 :info :cas [2 [42 27]] :RequestCanceledException INFO [2019-12-17 12:36:12,346] jepsen worker 8 - jepsen.util 13958 :invoke :write [2 17] INFO [2019-12-17 12:36:12,360] jepsen worker 47 - jepsen.util 11207 :ok :read [15 47] INFO [2019-12-17 12:36:12,360] jepsen worker 47 - jepsen.util 11207 :invoke :read [15 nil] INFO [2019-12-17 12:36:12,362] jepsen worker 16 - jepsen.util 15676 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:36:12,363] jepsen worker 16 - jepsen.util 15676 :invoke :cas [5 [16 22]] INFO [2019-12-17 12:36:12,386] jepsen worker 72 - jepsen.util 10692 :fail :cas [24 [40 37]] :ValueNotSwapFrom INFO [2019-12-17 12:36:12,386] jepsen worker 72 - jepsen.util 10692 :invoke :cas [24 [6 7]] INFO [2019-12-17 12:36:12,390] jepsen worker 45 - jepsen.util 10665 :fail :cas [15 [19 21]] :ValueNotSwapFrom INFO [2019-12-17 12:36:12,390] jepsen worker 45 - jepsen.util 10665 :invoke :cas [15 [3 5]] INFO [2019-12-17 12:36:12,395] jepsen worker 32 - jepsen.util 10562 :ok :read [10 34] INFO [2019-12-17 12:36:12,395] jepsen worker 32 - jepsen.util 10562 :invoke :cas [10 [14 9]] INFO [2019-12-17 12:36:12,410] jepsen worker 45 - jepsen.util 10665 :fail :cas [15 [3 5]] :ValueNotSwapFrom INFO [2019-12-17 12:36:12,411] jepsen worker 45 - jepsen.util 10665 :invoke :write [15 13] INFO [2019-12-17 12:36:12,420] jepsen worker 27 - jepsen.util 10017 :ok :write [9 13] INFO [2019-12-17 12:36:12,420] jepsen worker 27 - jepsen.util 10017 :invoke :write [9 29] INFO [2019-12-17 12:36:12,431] jepsen worker 43 - jepsen.util 14173 :info :cas [14 [29 36]] :RequestCanceledException INFO [2019-12-17 12:36:12,431] jepsen worker 22 - jepsen.util 14062 :info :cas [7 [42 18]] :RequestCanceledException INFO [2019-12-17 12:36:12,431] jepsen worker 28 - jepsen.util 8668 :fail :cas [9 [2 45]] :ValueNotSwapFrom INFO [2019-12-17 12:36:12,431] jepsen worker 28 - jepsen.util 8668 :invoke :cas [9 [37 0]] INFO [2019-12-17 12:36:12,432] jepsen worker 22 - jepsen.util 14152 :invoke :read [7 nil] INFO [2019-12-17 12:36:12,432] jepsen worker 43 - jepsen.util 14263 :invoke :cas [14 [11 23]] INFO [2019-12-17 12:36:12,446] jepsen worker 52 - jepsen.util 10312 :info :write [17 18] :RequestCanceledException INFO [2019-12-17 12:36:12,447] jepsen worker 52 - jepsen.util 10402 :invoke :write [17 5] INFO [2019-12-17 12:36:12,458] jepsen worker 78 - jepsen.util 10518 :ok :read [26 22] INFO [2019-12-17 12:36:12,458] jepsen worker 78 - jepsen.util 10518 :invoke :write [26 6] INFO [2019-12-17 12:36:12,463] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":60610,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:36:12,463] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":60610,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:36:12,463] jepsen worker 31 - jepsen.util 9211 :ok :read [10 34] INFO [2019-12-17 12:36:12,463] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":60614,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:36:12,463] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":60614,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:36:12,463] jepsen worker 31 - jepsen.util 9211 :invoke :cas [10 [36 43]] INFO [2019-12-17 12:36:12,486] jepsen worker 29 - jepsen.util 8939 :ok :write [9 36] INFO [2019-12-17 12:36:12,486] jepsen worker 29 - jepsen.util 8939 :invoke :read [9 nil] INFO [2019-12-17 12:36:12,494] jepsen worker 67 - jepsen.util 14377 :info :cas [22 [27 7]] :RequestCanceledException INFO [2019-12-17 12:36:12,495] jepsen worker 31 - jepsen.util 9211 :fail :cas [10 [36 43]] :ValueNotSwapFrom INFO [2019-12-17 12:36:12,495] jepsen worker 31 - jepsen.util 9211 :invoke :cas [10 [8 33]] INFO [2019-12-17 12:36:12,495] jepsen worker 67 - jepsen.util 14467 :invoke :cas [22 [37 5]] INFO [2019-12-17 12:36:12,505] jepsen worker 72 - jepsen.util 10692 :fail :cas [24 [6 7]] :ValueNotSwapFrom INFO [2019-12-17 12:36:12,505] jepsen worker 72 - jepsen.util 10692 :invoke :write [24 21] INFO [2019-12-17 12:36:12,516] jepsen worker 69 - jepsen.util 10599 :info :write [23 42] :RequestCanceledException INFO [2019-12-17 12:36:12,516] jepsen worker 50 - jepsen.util 14810 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:36:12,516] jepsen worker 50 - jepsen.util 14810 :invoke :read [16 nil] INFO [2019-12-17 12:36:12,517] jepsen worker 69 - jepsen.util 10689 :invoke :cas [23 [26 13]] INFO [2019-12-17 12:36:12,527] jepsen worker 31 - jepsen.util 9211 :fail :cas [10 [8 33]] :ValueNotSwapFrom INFO [2019-12-17 12:36:12,527] jepsen worker 31 - jepsen.util 9211 :invoke :read [10 nil] INFO [2019-12-17 12:36:12,530] jepsen worker 28 - jepsen.util 8668 :fail :cas [9 [37 0]] :ValueNotSwapFrom INFO [2019-12-17 12:36:12,530] jepsen worker 28 - jepsen.util 8668 :invoke :cas [9 [28 46]] INFO [2019-12-17 12:36:12,562] jepsen worker 29 - jepsen.util 8939 :ok :read [9 36] INFO [2019-12-17 12:36:12,562] jepsen worker 29 - jepsen.util 8939 :invoke :cas [9 [45 47]] INFO [2019-12-17 12:36:12,562] jepsen worker 31 - jepsen.util 9211 :ok :read [10 34] INFO [2019-12-17 12:36:12,562] jepsen worker 31 - jepsen.util 9211 :invoke :cas [10 [13 22]] INFO [2019-12-17 12:36:12,598] jepsen worker 29 - jepsen.util 8939 :fail :cas [9 [45 47]] :ValueNotSwapFrom INFO [2019-12-17 12:36:12,598] jepsen worker 29 - jepsen.util 8939 :invoke :read [9 nil] INFO [2019-12-17 12:36:12,599] jepsen worker 31 - jepsen.util 9211 :fail :cas [10 [13 22]] :ValueNotSwapFrom INFO [2019-12-17 12:36:12,599] jepsen worker 31 - jepsen.util 9211 :invoke :write [10 22] INFO [2019-12-17 12:36:12,601] jepsen worker 87 - jepsen.util 10077 :fail :cas [29 [13 19]] :ValueNotSwapFrom INFO [2019-12-17 12:36:12,601] jepsen worker 87 - jepsen.util 10077 :invoke :cas [29 [1 38]] INFO [2019-12-17 12:36:12,610] jepsen worker 18 - jepsen.util 10818 :ok :write [6 1] INFO [2019-12-17 12:36:12,610] jepsen worker 18 - jepsen.util 10818 :invoke :read [6 nil] INFO [2019-12-17 12:36:12,614] jepsen worker 29 - jepsen.util 8939 :ok :read [9 36] INFO [2019-12-17 12:36:12,614] jepsen worker 29 - jepsen.util 8939 :invoke :read [9 nil] INFO [2019-12-17 12:36:12,616] jepsen worker 55 - jepsen.util 11035 :info :write [18 26] :RequestCanceledException INFO [2019-12-17 12:36:12,617] jepsen worker 55 - jepsen.util 11125 :invoke :write [18 27] INFO [2019-12-17 12:36:12,631] jepsen worker 15 - jepsen.util 13245 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:36:12,632] jepsen worker 15 - jepsen.util 13245 :invoke :read [5 nil] INFO [2019-12-17 12:36:12,646] jepsen worker 57 - jepsen.util 13917 :info :cas [19 [16 36]] :RequestCanceledException INFO [2019-12-17 12:36:12,647] jepsen worker 57 - jepsen.util 14007 :invoke :cas [19 [8 4]] INFO [2019-12-17 12:36:12,650] jepsen worker 47 - jepsen.util 11207 :ok :read [15 47] INFO [2019-12-17 12:36:12,651] jepsen worker 47 - jepsen.util 11207 :invoke :read [15 nil] INFO [2019-12-17 12:36:12,653] jepsen worker 89 - jepsen.util 10799 :ok :write [29 8] INFO [2019-12-17 12:36:12,653] jepsen worker 89 - jepsen.util 10799 :invoke :read [29 nil] INFO [2019-12-17 12:36:12,675] jepsen worker 29 - jepsen.util 8939 :ok :read [9 36] INFO [2019-12-17 12:36:12,675] jepsen worker 29 - jepsen.util 8939 :invoke :cas [9 [2 48]] INFO [2019-12-17 12:36:12,680] jepsen worker 18 - jepsen.util 10818 :ok :read [6 1] INFO [2019-12-17 12:36:12,680] jepsen worker 18 - jepsen.util 10818 :invoke :read [6 nil] INFO [2019-12-17 12:36:12,682] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":60640,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:36:12,682] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":60640,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:36:12,682] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":60643,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:36:12,682] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":60643,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:36:12,682] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":60644,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:36:12,682] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":60644,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:36:12,686] jepsen worker 47 - jepsen.util 11207 :ok :read [15 47] INFO [2019-12-17 12:36:12,686] jepsen worker 47 - jepsen.util 11207 :invoke :read [15 nil] INFO [2019-12-17 12:36:12,689] jepsen worker 87 - jepsen.util 10077 :fail :cas [29 [1 38]] :ValueNotSwapFrom INFO [2019-12-17 12:36:12,690] jepsen worker 87 - jepsen.util 10077 :invoke :cas [29 [34 18]] INFO [2019-12-17 12:36:12,732] jepsen worker 30 - jepsen.util 10110 :fail :read [10 nil] :RequestTimeoutException INFO [2019-12-17 12:36:12,732] jepsen worker 30 - jepsen.util 10110 :invoke :cas [10 [49 15]] INFO [2019-12-17 12:36:12,744] jepsen worker 11 - jepsen.util 10001 :info :cas [3 [45 22]] :RequestCanceledException INFO [2019-12-17 12:36:12,744] jepsen worker 63 - jepsen.util 9513 :fail :read [21 nil] :RequestTimeoutException INFO [2019-12-17 12:36:12,745] jepsen worker 63 - jepsen.util 9513 :invoke :read [21 nil] INFO [2019-12-17 12:36:12,745] jepsen worker 11 - jepsen.util 10091 :invoke :read [3 nil] INFO [2019-12-17 12:36:12,753] jepsen worker 18 - jepsen.util 10818 :ok :read [6 1] INFO [2019-12-17 12:36:12,754] jepsen worker 18 - jepsen.util 10818 :invoke :write [6 43] INFO [2019-12-17 12:36:12,756] jepsen worker 29 - jepsen.util 8939 :fail :cas [9 [2 48]] :ValueNotSwapFrom INFO [2019-12-17 12:36:12,756] jepsen worker 29 - jepsen.util 8939 :invoke :write [9 22] INFO [2019-12-17 12:36:12,763] jepsen worker 42 - jepsen.util 15072 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:36:12,763] jepsen worker 4 - jepsen.util 9634 :info :write [1 14] :RequestCanceledException INFO [2019-12-17 12:36:12,763] jepsen worker 42 - jepsen.util 15072 :invoke :read [14 nil] INFO [2019-12-17 12:36:12,764] jepsen worker 4 - jepsen.util 9724 :invoke :read [1 nil] INFO [2019-12-17 12:36:12,775] jepsen worker 36 - jepsen.util 11376 :ok :read [12 4] INFO [2019-12-17 12:36:12,775] jepsen worker 36 - jepsen.util 11376 :invoke :read [12 nil] INFO [2019-12-17 12:36:12,787] jepsen worker 47 - jepsen.util 11207 :ok :read [15 47] INFO [2019-12-17 12:36:12,787] jepsen worker 47 - jepsen.util 11207 :invoke :read [15 nil] INFO [2019-12-17 12:36:12,798] jepsen worker 2 - jepsen.util 13772 :info :write [0 44] :RequestCanceledException INFO [2019-12-17 12:36:12,798] jepsen worker 60 - jepsen.util 15180 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:36:12,799] jepsen worker 60 - jepsen.util 15180 :invoke :write [20 10] INFO [2019-12-17 12:36:12,799] jepsen worker 2 - jepsen.util 13862 :invoke :write [0 4] INFO [2019-12-17 12:36:12,816] jepsen worker 24 - jepsen.util 12894 :info :cas [8 [13 46]] :RequestCanceledException INFO [2019-12-17 12:36:12,817] jepsen worker 24 - jepsen.util 12984 :invoke :read [8 nil] INFO [2019-12-17 12:36:12,818] jepsen worker 30 - jepsen.util 10110 :fail :cas [10 [49 15]] :ValueNotSwapFrom INFO [2019-12-17 12:36:12,818] jepsen worker 30 - jepsen.util 10110 :invoke :write [10 30] INFO [2019-12-17 12:36:12,831] jepsen worker 3 - jepsen.util 10263 :info :write [1 22] :RequestCanceledException INFO [2019-12-17 12:36:12,831] jepsen worker 41 - jepsen.util 15161 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:36:12,831] jepsen worker 17 - jepsen.util 14147 :info :write [5 13] :RequestCanceledException INFO [2019-12-17 12:36:12,832] jepsen worker 73 - jepsen.util 9793 :info :write [24 40] :RequestCanceledException INFO [2019-12-17 12:36:12,832] jepsen worker 41 - jepsen.util 15161 :invoke :write [13 32] INFO [2019-12-17 12:36:12,833] jepsen worker 3 - jepsen.util 10353 :invoke :cas [1 [21 30]] INFO [2019-12-17 12:36:12,833] jepsen worker 73 - jepsen.util 9883 :invoke :cas [24 [30 48]] INFO [2019-12-17 12:36:12,834] jepsen worker 74 - jepsen.util 10334 :ok :write [24 1] INFO [2019-12-17 12:36:12,834] jepsen worker 74 - jepsen.util 10334 :invoke :read [24 nil] INFO [2019-12-17 12:36:12,835] jepsen worker 17 - jepsen.util 14237 :invoke :cas [5 [44 42]] INFO [2019-12-17 12:36:12,843] jepsen worker 19 - jepsen.util 10549 :info :write [6 5] :RequestCanceledException INFO [2019-12-17 12:36:12,843] jepsen worker 76 - jepsen.util 14566 :info :write [25 34] :RequestCanceledException INFO [2019-12-17 12:36:12,843] jepsen worker 14 - jepsen.util 10274 :info :write [4 8] :RequestCanceledException INFO [2019-12-17 12:36:12,844] jepsen worker 14 - jepsen.util 10364 :invoke :write [4 8] INFO [2019-12-17 12:36:12,844] jepsen worker 19 - jepsen.util 10639 :invoke :cas [6 [38 26]] INFO [2019-12-17 12:36:12,844] jepsen worker 76 - jepsen.util 14656 :invoke :write [25 31] INFO [2019-12-17 12:36:12,861] jepsen worker 6 - jepsen.util 14586 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:36:12,861] jepsen worker 71 - jepsen.util 11321 :info :write [23 40] :RequestCanceledException INFO [2019-12-17 12:36:12,861] jepsen worker 6 - jepsen.util 14586 :invoke :write [2 12] INFO [2019-12-17 12:36:12,862] jepsen worker 71 - jepsen.util 11411 :invoke :cas [23 [23 8]] INFO [2019-12-17 12:36:12,868] jepsen worker 36 - jepsen.util 11376 :ok :read [12 4] INFO [2019-12-17 12:36:12,868] jepsen worker 36 - jepsen.util 11376 :invoke :write [12 31] INFO [2019-12-17 12:36:12,892] jepsen worker 32 - jepsen.util 10562 :fail :cas [10 [14 9]] :ValueNotSwapFrom INFO [2019-12-17 12:36:12,892] jepsen worker 32 - jepsen.util 10562 :invoke :read [10 nil] INFO [2019-12-17 12:36:12,898] jepsen worker 37 - jepsen.util 10027 :info :write [12 15] :RequestCanceledException INFO [2019-12-17 12:36:12,899] jepsen worker 37 - jepsen.util 10117 :invoke :read [12 nil] INFO [2019-12-17 12:36:12,901] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":60674,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:36:12,901] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":60674,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:36:12,901] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":60678,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:36:12,901] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":60678,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:36:12,903] jepsen worker 19 - jepsen.util 10639 :fail :cas [6 [38 26]] :ValueNotSwapFrom INFO [2019-12-17 12:36:12,903] jepsen worker 19 - jepsen.util 10639 :invoke :read [6 nil] INFO [2019-12-17 12:36:12,913] jepsen worker 64 - jepsen.util 8074 :fail :read [21 nil] :RequestTimeoutException INFO [2019-12-17 12:36:12,913] jepsen worker 64 - jepsen.util 8074 :invoke :cas [21 [40 21]] INFO [2019-12-17 12:36:12,917] jepsen worker 63 - jepsen.util 9513 :ok :read [21 15] INFO [2019-12-17 12:36:12,917] jepsen worker 63 - jepsen.util 9513 :invoke :cas [21 [14 5]] INFO [2019-12-17 12:36:12,946] jepsen worker 51 - jepsen.util 10041 :info :write [17 0] :RequestCanceledException INFO [2019-12-17 12:36:12,947] jepsen worker 51 - jepsen.util 10131 :invoke :read [17 nil] INFO [2019-12-17 12:36:12,973] jepsen worker 11 - jepsen.util 10091 :ok :read [3 38] INFO [2019-12-17 12:36:12,973] jepsen worker 11 - jepsen.util 10091 :invoke :cas [3 [30 19]] INFO [2019-12-17 12:36:12,988] jepsen worker 3 - jepsen.util 10353 :fail :cas [1 [21 30]] :ValueNotSwapFrom INFO [2019-12-17 12:36:12,989] jepsen worker 3 - jepsen.util 10353 :invoke :read [1 nil] INFO [2019-12-17 12:36:13,001] jepsen worker 19 - jepsen.util 10639 :ok :read [6 1] INFO [2019-12-17 12:36:13,002] jepsen worker 19 - jepsen.util 10639 :invoke :write [6 42] INFO [2019-12-17 12:36:13,007] jepsen worker 74 - jepsen.util 10334 :ok :read [24 1] INFO [2019-12-17 12:36:13,007] jepsen worker 74 - jepsen.util 10334 :invoke :cas [24 [2 43]] INFO [2019-12-17 12:36:13,008] jepsen worker 29 - jepsen.util 8939 :ok :write [9 22] INFO [2019-12-17 12:36:13,008] jepsen worker 29 - jepsen.util 8939 :invoke :read [9 nil] INFO [2019-12-17 12:36:13,011] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":60690,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:36:13,011] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":60690,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:36:13,011] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":60694,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:36:13,011] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":60694,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:36:13,051] jepsen worker 71 - jepsen.util 11411 :fail :cas [23 [23 8]] :ValueNotSwapFrom INFO [2019-12-17 12:36:13,051] jepsen worker 71 - jepsen.util 11411 :invoke :read [23 nil] INFO [2019-12-17 12:36:13,062] jepsen worker 35 - jepsen.util 14255 :info :cas [11 [11 24]] :RequestCanceledException INFO [2019-12-17 12:36:13,063] jepsen worker 35 - jepsen.util 14345 :invoke :cas [11 [44 3]] INFO [2019-12-17 12:36:13,083] jepsen worker 29 - jepsen.util 8939 :ok :read [9 22] INFO [2019-12-17 12:36:13,083] jepsen worker 29 - jepsen.util 8939 :invoke :cas [9 [8 49]] INFO [2019-12-17 12:36:13,084] jepsen worker 32 - jepsen.util 10562 :ok :read [10 34] INFO [2019-12-17 12:36:13,085] jepsen worker 32 - jepsen.util 10562 :invoke :cas [10 [27 26]] INFO [2019-12-17 12:36:13,110] jepsen worker 89 - jepsen.util 10799 :ok :read [29 8] INFO [2019-12-17 12:36:13,110] jepsen worker 89 - jepsen.util 10799 :invoke :read [29 nil] INFO [2019-12-17 12:36:13,115] jepsen worker 37 - jepsen.util 10117 :ok :read [12 4] INFO [2019-12-17 12:36:13,115] jepsen worker 37 - jepsen.util 10117 :invoke :write [12 15] INFO [2019-12-17 12:36:13,130] jepsen worker 44 - jepsen.util 14534 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:36:13,130] jepsen worker 44 - jepsen.util 14534 :invoke :write [14 6] INFO [2019-12-17 12:36:13,145] jepsen worker 62 - jepsen.util 14552 :info :cas [20 [15 25]] :RequestCanceledException INFO [2019-12-17 12:36:13,146] jepsen worker 62 - jepsen.util 14642 :invoke :cas [20 [27 5]] INFO [2019-12-17 12:36:13,146] jepsen worker 29 - jepsen.util 8939 :fail :cas [9 [8 49]] :ValueNotSwapFrom INFO [2019-12-17 12:36:13,146] jepsen worker 29 - jepsen.util 8939 :invoke :read [9 nil] INFO [2019-12-17 12:36:13,160] jepsen worker 29 - jepsen.util 8939 :ok :read [9 22] INFO [2019-12-17 12:36:13,160] jepsen worker 29 - jepsen.util 8939 :invoke :cas [9 [24 6]] INFO [2019-12-17 12:36:13,163] jepsen worker 63 - jepsen.util 9513 :fail :cas [21 [14 5]] :ValueNotSwapFrom INFO [2019-12-17 12:36:13,164] jepsen worker 63 - jepsen.util 9513 :invoke :read [21 nil] INFO [2019-12-17 12:36:13,172] jepsen worker 11 - jepsen.util 10091 :fail :cas [3 [30 19]] :ValueNotSwapFrom INFO [2019-12-17 12:36:13,172] jepsen worker 11 - jepsen.util 10091 :invoke :cas [3 [39 14]] INFO [2019-12-17 12:36:13,187] jepsen worker 29 - jepsen.util 8939 :fail :cas [9 [24 6]] :ValueNotSwapFrom INFO [2019-12-17 12:36:13,187] jepsen worker 29 - jepsen.util 8939 :invoke :write [9 32] INFO [2019-12-17 12:36:13,187] jepsen worker 74 - jepsen.util 10334 :fail :cas [24 [2 43]] :ValueNotSwapFrom INFO [2019-12-17 12:36:13,187] jepsen worker 74 - jepsen.util 10334 :invoke :cas [24 [27 3]] INFO [2019-12-17 12:36:13,195] jepsen worker 32 - jepsen.util 10562 :fail :cas [10 [27 26]] :ValueNotSwapFrom INFO [2019-12-17 12:36:13,195] jepsen worker 32 - jepsen.util 10562 :invoke :write [10 28] INFO [2019-12-17 12:36:13,198] jepsen worker 56 - jepsen.util 10946 :info :write [18 49] :RequestCanceledException INFO [2019-12-17 12:36:13,199] jepsen worker 56 - jepsen.util 11036 :invoke :write [18 12] INFO [2019-12-17 12:36:13,203] jepsen worker 87 - jepsen.util 10077 :fail :cas [29 [34 18]] :ValueNotSwapFrom INFO [2019-12-17 12:36:13,203] jepsen worker 87 - jepsen.util 10077 :invoke :write [29 4] INFO [2019-12-17 12:36:13,214] jepsen worker 85 - jepsen.util 15115 :info :cas [28 [1 16]] :RequestCanceledException INFO [2019-12-17 12:36:13,214] jepsen worker 51 - jepsen.util 10131 :ok :read [17 25] INFO [2019-12-17 12:36:13,214] jepsen worker 51 - jepsen.util 10131 :invoke :read [17 nil] INFO [2019-12-17 12:36:13,215] jepsen worker 85 - jepsen.util 15205 :invoke :read [28 nil] INFO [2019-12-17 12:36:13,221] jepsen worker 63 - jepsen.util 9513 :ok :read [21 15] INFO [2019-12-17 12:36:13,221] jepsen worker 63 - jepsen.util 9513 :invoke :read [21 nil] INFO [2019-12-17 12:36:13,229] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":60722,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:36:13,229] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":60714,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:36:13,229] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":60722,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:36:13,229] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":60714,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:36:13,229] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":60725,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:36:13,229] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":60715,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:36:13,229] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":60725,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:36:13,229] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":60715,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:36:13,244] jepsen worker 11 - jepsen.util 10091 :fail :cas [3 [39 14]] :ValueNotSwapFrom INFO [2019-12-17 12:36:13,245] jepsen worker 11 - jepsen.util 10091 :invoke :write [3 41] INFO [2019-12-17 12:36:13,256] jepsen worker 74 - jepsen.util 10334 :fail :cas [24 [27 3]] :ValueNotSwapFrom INFO [2019-12-17 12:36:13,256] jepsen worker 74 - jepsen.util 10334 :invoke :cas [24 [12 33]] INFO [2019-12-17 12:36:13,257] jepsen worker 4 - jepsen.util 9724 :ok :read [1 45] INFO [2019-12-17 12:36:13,257] jepsen worker 4 - jepsen.util 9724 :invoke :read [1 nil] INFO [2019-12-17 12:36:13,262] jepsen worker 23 - jepsen.util 14243 :info :write [7 27] :RequestCanceledException INFO [2019-12-17 12:36:13,264] jepsen worker 23 - jepsen.util 14333 :invoke :write [7 24] INFO [2019-12-17 12:36:13,297] jepsen worker 29 - jepsen.util 8939 :ok :write [9 32] INFO [2019-12-17 12:36:13,297] jepsen worker 29 - jepsen.util 8939 :invoke :write [9 0] INFO [2019-12-17 12:36:13,298] jepsen worker 20 - jepsen.util 10280 :info :write [6 44] :RequestCanceledException INFO [2019-12-17 12:36:13,307] jepsen worker 20 - jepsen.util 10370 :invoke :read [6 nil] INFO [2019-12-17 12:36:13,332] jepsen worker 40 - jepsen.util 14620 :info :write [13 24] :RequestCanceledException INFO [2019-12-17 12:36:13,333] jepsen worker 40 - jepsen.util 14710 :invoke :write [13 8] INFO [2019-12-17 12:36:13,340] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:36:13,340] jepsen nemesis - couchbase.util Rebalance check count: 474 INFO [2019-12-17 12:36:13,340] jepsen nemesis - couchbase.util Rebalance stuck count: 455 INFO [2019-12-17 12:36:13,345] jepsen worker 89 - jepsen.util 10799 :ok :read [29 8] INFO [2019-12-17 12:36:13,346] jepsen worker 89 - jepsen.util 10799 :invoke :write [29 25] INFO [2019-12-17 12:36:13,350] jepsen worker 29 - jepsen.util 8939 :ok :write [9 0] INFO [2019-12-17 12:36:13,350] jepsen worker 29 - jepsen.util 8939 :invoke :write [9 45] INFO [2019-12-17 12:36:13,353] jepsen worker 51 - jepsen.util 10131 :ok :read [17 25] INFO [2019-12-17 12:36:13,353] jepsen worker 51 - jepsen.util 10131 :invoke :read [17 nil] INFO [2019-12-17 12:36:13,393] jepsen worker 69 - jepsen.util 10689 :fail :cas [23 [26 13]] :ValueNotSwapFrom INFO [2019-12-17 12:36:13,393] jepsen worker 69 - jepsen.util 10689 :invoke :read [23 nil] INFO [2019-12-17 12:36:13,413] jepsen worker 45 - jepsen.util 10665 :ok :write [15 13] INFO [2019-12-17 12:36:13,414] jepsen worker 45 - jepsen.util 10665 :invoke :read [15 nil] INFO [2019-12-17 12:36:13,414] jepsen worker 31 - jepsen.util 9211 :ok :write [10 22] INFO [2019-12-17 12:36:13,414] jepsen worker 18 - jepsen.util 10818 :ok :write [6 43] INFO [2019-12-17 12:36:13,414] jepsen worker 31 - jepsen.util 9211 :invoke :read [10 nil] INFO [2019-12-17 12:36:13,414] jepsen worker 18 - jepsen.util 10818 :invoke :write [6 9] INFO [2019-12-17 12:36:13,416] jepsen worker 66 - jepsen.util 14016 :info :cas [22 [27 5]] :RequestCanceledException INFO [2019-12-17 12:36:13,417] jepsen worker 66 - jepsen.util 14106 :invoke :read [22 nil] INFO [2019-12-17 12:36:13,425] jepsen worker 29 - jepsen.util 8939 :ok :write [9 45] INFO [2019-12-17 12:36:13,425] jepsen worker 29 - jepsen.util 8939 :invoke :read [9 nil] INFO [2019-12-17 12:36:13,429] jepsen worker 64 - jepsen.util 8074 :fail :cas [21 [40 21]] :ValueNotSwapFrom INFO [2019-12-17 12:36:13,430] jepsen worker 64 - jepsen.util 8074 :invoke :cas [21 [49 2]] INFO [2019-12-17 12:36:13,445] jepsen worker 83 - jepsen.util 14483 :info :write [27 33] :RequestCanceledException INFO [2019-12-17 12:36:13,446] jepsen worker 83 - jepsen.util 14573 :invoke :read [27 nil] INFO [2019-12-17 12:36:13,448] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":60748,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:36:13,448] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":60748,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:36:13,448] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":60751,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:36:13,448] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":60751,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:36:13,449] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":60752,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:36:13,449] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":60752,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:36:13,456] jepsen worker 38 - jepsen.util 11198 :ok :write [12 15] INFO [2019-12-17 12:36:13,456] jepsen worker 38 - jepsen.util 11198 :invoke :write [12 30] INFO [2019-12-17 12:36:13,462] jepsen worker 88 - jepsen.util 10978 :ok :write [29 8] INFO [2019-12-17 12:36:13,462] jepsen worker 88 - jepsen.util 10978 :invoke :cas [29 [47 4]] INFO [2019-12-17 12:36:13,463] jepsen worker 9 - jepsen.util 11169 :info :write [3 40] :RequestCanceledException INFO [2019-12-17 12:36:13,463] jepsen worker 7 - jepsen.util 14407 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:36:13,463] jepsen worker 7 - jepsen.util 14407 :invoke :write [2 30] INFO [2019-12-17 12:36:13,464] jepsen worker 9 - jepsen.util 11259 :invoke :cas [3 [18 46]] INFO [2019-12-17 12:36:13,498] jepsen worker 49 - jepsen.util 14089 :info :cas [16 [42 17]] :RequestCanceledException INFO [2019-12-17 12:36:13,498] jepsen worker 54 - jepsen.util 11394 :info :write [18 39] :RequestCanceledException INFO [2019-12-17 12:36:13,498] jepsen worker 54 - jepsen.util 11484 :invoke :read [18 nil] INFO [2019-12-17 12:36:13,498] jepsen worker 49 - jepsen.util 14179 :invoke :cas [16 [38 26]] INFO [2019-12-17 12:36:13,516] jepsen worker 26 - jepsen.util 14876 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:36:13,516] jepsen worker 61 - jepsen.util 15091 :info :cas [20 [24 12]] :RequestCanceledException INFO [2019-12-17 12:36:13,517] jepsen worker 26 - jepsen.util 14876 :invoke :write [8 37] INFO [2019-12-17 12:36:13,517] jepsen worker 61 - jepsen.util 15181 :invoke :read [20 nil] INFO [2019-12-17 12:36:13,528] jepsen worker 69 - jepsen.util 10689 :ok :read [23 14] INFO [2019-12-17 12:36:13,528] jepsen worker 69 - jepsen.util 10689 :invoke :cas [23 [20 43]] INFO [2019-12-17 12:36:13,531] jepsen worker 0 - jepsen.util 15030 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:36:13,531] jepsen worker 0 - jepsen.util 15030 :invoke :write [0 24] INFO [2019-12-17 12:36:13,538] jepsen worker 89 - jepsen.util 10799 :ok :write [29 25] INFO [2019-12-17 12:36:13,538] jepsen worker 89 - jepsen.util 10799 :invoke :read [29 nil] INFO [2019-12-17 12:36:13,546] jepsen worker 45 - jepsen.util 10665 :ok :read [15 13] INFO [2019-12-17 12:36:13,546] jepsen worker 45 - jepsen.util 10665 :invoke :cas [15 [4 22]] INFO [2019-12-17 12:36:13,546] jepsen worker 1 - jepsen.util 14311 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:36:13,546] jepsen worker 1 - jepsen.util 14311 :invoke :write [0 38] INFO [2019-12-17 12:36:13,550] jepsen worker 29 - jepsen.util 8939 :ok :read [9 45] INFO [2019-12-17 12:36:13,550] jepsen worker 29 - jepsen.util 8939 :invoke :read [9 nil] INFO [2019-12-17 12:36:13,575] jepsen worker 31 - jepsen.util 9211 :ok :read [10 22] INFO [2019-12-17 12:36:13,575] jepsen worker 31 - jepsen.util 9211 :invoke :cas [10 [1 23]] INFO [2019-12-17 12:36:13,578] jepsen worker 64 - jepsen.util 8074 :fail :cas [21 [49 2]] :ValueNotSwapFrom INFO [2019-12-17 12:36:13,578] jepsen worker 64 - jepsen.util 8074 :invoke :cas [21 [19 48]] INFO [2019-12-17 12:36:13,588] jepsen worker 5 - jepsen.util 10085 :ok :write [1 14] INFO [2019-12-17 12:36:13,589] jepsen worker 5 - jepsen.util 10085 :invoke :cas [1 [28 48]] INFO [2019-12-17 12:36:13,612] jepsen worker 69 - jepsen.util 10689 :fail :cas [23 [20 43]] :ValueNotSwapFrom INFO [2019-12-17 12:36:13,612] jepsen worker 69 - jepsen.util 10689 :invoke :cas [23 [32 45]] INFO [2019-12-17 12:36:13,621] jepsen worker 29 - jepsen.util 8939 :ok :read [9 45] INFO [2019-12-17 12:36:13,621] jepsen worker 29 - jepsen.util 8939 :invoke :cas [9 [24 38]] INFO [2019-12-17 12:36:13,633] jepsen worker 9 - jepsen.util 11259 :fail :cas [3 [18 46]] :ValueNotSwapFrom INFO [2019-12-17 12:36:13,633] jepsen worker 9 - jepsen.util 11259 :invoke :write [3 48] INFO [2019-12-17 12:36:13,664] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":60777,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:36:13,664] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":60777,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:36:13,668] jepsen worker 31 - jepsen.util 9211 :fail :cas [10 [1 23]] :ValueNotSwapFrom INFO [2019-12-17 12:36:13,668] jepsen worker 31 - jepsen.util 9211 :invoke :write [10 46] INFO [2019-12-17 12:36:13,693] jepsen worker 52 - jepsen.util 10402 :ok :write [17 5] INFO [2019-12-17 12:36:13,693] jepsen worker 52 - jepsen.util 10402 :invoke :write [17 4] INFO [2019-12-17 12:36:13,698] jepsen worker 70 - jepsen.util 11320 :fail :read [23 nil] :RequestTimeoutException INFO [2019-12-17 12:36:13,698] jepsen worker 70 - jepsen.util 11320 :invoke :write [23 0] INFO [2019-12-17 12:36:13,704] jepsen worker 51 - jepsen.util 10131 :ok :read [17 25] INFO [2019-12-17 12:36:13,704] jepsen worker 51 - jepsen.util 10131 :invoke :cas [17 [27 48]] INFO [2019-12-17 12:36:13,713] jepsen worker 45 - jepsen.util 10665 :fail :cas [15 [4 22]] :ValueNotSwapFrom INFO [2019-12-17 12:36:13,713] jepsen worker 45 - jepsen.util 10665 :invoke :read [15 nil] INFO [2019-12-17 12:36:13,715] jepsen worker 29 - jepsen.util 8939 :fail :cas [9 [24 38]] :ValueNotSwapFrom INFO [2019-12-17 12:36:13,715] jepsen worker 29 - jepsen.util 8939 :invoke :cas [9 [7 15]] INFO [2019-12-17 12:36:13,716] jepsen worker 80 - jepsen.util 10250 :info :write [26 29] :RequestCanceledException INFO [2019-12-17 12:36:13,717] jepsen worker 80 - jepsen.util 10340 :invoke :cas [26 [36 47]] INFO [2019-12-17 12:36:13,732] jepsen worker 79 - jepsen.util 10879 :info :write [26 22] :RequestCanceledException INFO [2019-12-17 12:36:13,733] jepsen worker 79 - jepsen.util 10969 :invoke :read [26 nil] INFO [2019-12-17 12:36:13,738] jepsen worker 64 - jepsen.util 8074 :fail :cas [21 [19 48]] :ValueNotSwapFrom INFO [2019-12-17 12:36:13,738] jepsen worker 64 - jepsen.util 8074 :invoke :cas [21 [22 34]] INFO [2019-12-17 12:36:13,745] jepsen worker 75 - jepsen.util 13755 :info :write [25 17] :RequestCanceledException INFO [2019-12-17 12:36:13,746] jepsen worker 75 - jepsen.util 13845 :invoke :write [25 11] INFO [2019-12-17 12:36:13,754] jepsen worker 88 - jepsen.util 10978 :fail :cas [29 [47 4]] :ValueNotSwapFrom INFO [2019-12-17 12:36:13,754] jepsen worker 88 - jepsen.util 10978 :invoke :cas [29 [8 37]] INFO [2019-12-17 12:36:13,763] jepsen worker 11 - jepsen.util 10091 :ok :write [3 41] INFO [2019-12-17 12:36:13,763] jepsen worker 11 - jepsen.util 10091 :invoke :read [3 nil] INFO [2019-12-17 12:36:13,764] jepsen worker 29 - jepsen.util 8939 :fail :cas [9 [7 15]] :ValueNotSwapFrom INFO [2019-12-17 12:36:13,764] jepsen worker 29 - jepsen.util 8939 :invoke :cas [9 [27 9]] INFO [2019-12-17 12:36:13,782] jepsen worker 5 - jepsen.util 10085 :fail :cas [1 [28 48]] :ValueNotSwapFrom INFO [2019-12-17 12:36:13,782] jepsen worker 5 - jepsen.util 10085 :invoke :cas [1 [12 13]] INFO [2019-12-17 12:36:13,788] jepsen worker 4 - jepsen.util 9724 :ok :read [1 14] INFO [2019-12-17 12:36:13,788] jepsen worker 4 - jepsen.util 9724 :invoke :read [1 nil] INFO [2019-12-17 12:36:13,797] jepsen worker 38 - jepsen.util 11198 :ok :write [12 30] INFO [2019-12-17 12:36:13,798] jepsen worker 38 - jepsen.util 11198 :invoke :read [12 nil] INFO [2019-12-17 12:36:13,798] jepsen worker 58 - jepsen.util 14638 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:36:13,798] jepsen worker 58 - jepsen.util 14638 :invoke :cas [19 [21 8]] INFO [2019-12-17 12:36:13,800] jepsen worker 51 - jepsen.util 10131 :fail :cas [17 [27 48]] :ValueNotSwapFrom INFO [2019-12-17 12:36:13,800] jepsen worker 51 - jepsen.util 10131 :invoke :cas [17 [16 22]] INFO [2019-12-17 12:36:13,813] jepsen worker 29 - jepsen.util 8939 :fail :cas [9 [27 9]] :ValueNotSwapFrom INFO [2019-12-17 12:36:13,813] jepsen worker 29 - jepsen.util 8939 :invoke :read [9 nil] INFO [2019-12-17 12:36:13,824] jepsen worker 53 - jepsen.util 9953 :ok :write [17 20] INFO [2019-12-17 12:36:13,824] jepsen worker 53 - jepsen.util 9953 :invoke :read [17 nil] INFO [2019-12-17 12:36:13,829] jepsen worker 20 - jepsen.util 10370 :ok :read [6 43] INFO [2019-12-17 12:36:13,829] jepsen worker 20 - jepsen.util 10370 :invoke :write [6 0] INFO [2019-12-17 12:36:13,831] jepsen worker 12 - jepsen.util 10902 :info :write [4 0] :RequestCanceledException INFO [2019-12-17 12:36:13,831] jepsen worker 25 - jepsen.util 15415 :info :cas [8 [32 40]] :RequestCanceledException INFO [2019-12-17 12:36:13,832] jepsen worker 25 - jepsen.util 15505 :invoke :read [8 nil] INFO [2019-12-17 12:36:13,832] jepsen worker 12 - jepsen.util 10992 :invoke :write [4 19] INFO [2019-12-17 12:36:13,846] jepsen worker 68 - jepsen.util 13748 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:36:13,846] jepsen worker 68 - jepsen.util 13748 :invoke :read [22 nil] INFO [2019-12-17 12:36:13,848] jepsen worker 29 - jepsen.util 8939 :ok :read [9 45] INFO [2019-12-17 12:36:13,849] jepsen worker 29 - jepsen.util 8939 :invoke :cas [9 [37 18]] INFO [2019-12-17 12:36:13,859] jepsen worker 11 - jepsen.util 10091 :ok :read [3 41] INFO [2019-12-17 12:36:13,859] jepsen worker 11 - jepsen.util 10091 :invoke :read [3 nil] INFO [2019-12-17 12:36:13,879] jepsen worker 30 - jepsen.util 10110 :ok :write [10 30] INFO [2019-12-17 12:36:13,880] jepsen worker 30 - jepsen.util 10110 :invoke :cas [10 [4 1]] INFO [2019-12-17 12:36:13,880] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":60807,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:36:13,880] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":60807,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:36:13,880] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":60811,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:36:13,880] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":60811,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:36:13,881] jepsen worker 13 - jepsen.util 10003 :ok :write [4 27] INFO [2019-12-17 12:36:13,882] jepsen worker 13 - jepsen.util 10003 :invoke :read [4 nil] INFO [2019-12-17 12:36:13,887] jepsen worker 80 - jepsen.util 10340 :fail :cas [26 [36 47]] :ValueNotSwapFrom INFO [2019-12-17 12:36:13,887] jepsen worker 80 - jepsen.util 10340 :invoke :cas [26 [23 35]] INFO [2019-12-17 12:36:13,912] jepsen worker 29 - jepsen.util 8939 :fail :cas [9 [37 18]] :ValueNotSwapFrom INFO [2019-12-17 12:36:13,912] jepsen worker 29 - jepsen.util 8939 :invoke :read [9 nil] INFO [2019-12-17 12:36:13,915] jepsen worker 82 - jepsen.util 14482 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:36:13,916] jepsen worker 82 - jepsen.util 14482 :invoke :cas [27 [35 22]] INFO [2019-12-17 12:36:13,929] jepsen worker 29 - jepsen.util 8939 :ok :read [9 45] INFO [2019-12-17 12:36:13,929] jepsen worker 29 - jepsen.util 8939 :invoke :read [9 nil] INFO [2019-12-17 12:36:13,963] jepsen worker 89 - jepsen.util 10799 :ok :read [29 25] INFO [2019-12-17 12:36:13,963] jepsen worker 89 - jepsen.util 10799 :invoke :read [29 nil] INFO [2019-12-17 12:36:13,964] jepsen worker 29 - jepsen.util 8939 :ok :read [9 45] INFO [2019-12-17 12:36:13,964] jepsen worker 29 - jepsen.util 8939 :invoke :read [9 nil] INFO [2019-12-17 12:36:13,989] jepsen worker 29 - jepsen.util 8939 :ok :read [9 45] INFO [2019-12-17 12:36:13,989] jepsen worker 29 - jepsen.util 8939 :invoke :write [9 30] INFO [2019-12-17 12:36:13,999] jepsen worker 38 - jepsen.util 11198 :ok :read [12 30] INFO [2019-12-17 12:36:14,000] jepsen worker 38 - jepsen.util 11198 :invoke :cas [12 [19 0]] INFO [2019-12-17 12:36:14,014] jepsen worker 5 - jepsen.util 10085 :fail :cas [1 [12 13]] :ValueNotSwapFrom INFO [2019-12-17 12:36:14,014] jepsen worker 5 - jepsen.util 10085 :invoke :read [1 nil] INFO [2019-12-17 12:36:14,032] jepsen worker 48 - jepsen.util 13188 :info :write [16 40] :RequestCanceledException INFO [2019-12-17 12:36:14,033] jepsen worker 48 - jepsen.util 13278 :invoke :cas [16 [25 19]] INFO [2019-12-17 12:36:14,041] jepsen worker 29 - jepsen.util 8939 :ok :write [9 30] INFO [2019-12-17 12:36:14,098] jepsen worker 33 - jepsen.util 14343 :info :write [11 21] :RequestCanceledException INFO [2019-12-17 12:36:14,099] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":60824,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:36:14,108] jepsen worker 29 - jepsen.util 8939 :invoke :cas [9 [42 6]] INFO [2019-12-17 12:36:14,108] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":60824,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:36:14,108] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":60826,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:36:14,108] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":60826,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:36:14,109] jepsen worker 33 - jepsen.util 14433 :invoke :write [11 3] INFO [2019-12-17 12:36:14,111] jepsen worker 9 - jepsen.util 11259 :ok :write [3 48] INFO [2019-12-17 12:36:14,111] jepsen worker 9 - jepsen.util 11259 :invoke :write [3 28] INFO [2019-12-17 12:36:14,116] jepsen worker 81 - jepsen.util 14031 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:36:14,116] jepsen worker 81 - jepsen.util 14031 :invoke :read [27 nil] INFO [2019-12-17 12:36:14,129] jepsen worker 69 - jepsen.util 10689 :fail :cas [23 [32 45]] :ValueNotSwapFrom INFO [2019-12-17 12:36:14,129] jepsen worker 69 - jepsen.util 10689 :invoke :cas [23 [15 7]] INFO [2019-12-17 12:36:14,139] jepsen worker 51 - jepsen.util 10131 :fail :cas [17 [16 22]] :ValueNotSwapFrom INFO [2019-12-17 12:36:14,140] jepsen worker 51 - jepsen.util 10131 :invoke :cas [17 [11 18]] INFO [2019-12-17 12:36:14,143] jepsen worker 29 - jepsen.util 8939 :fail :cas [9 [42 6]] :ValueNotSwapFrom INFO [2019-12-17 12:36:14,143] jepsen worker 29 - jepsen.util 8939 :invoke :write [9 15] INFO [2019-12-17 12:36:14,155] jepsen worker 80 - jepsen.util 10340 :fail :cas [26 [23 35]] :ValueNotSwapFrom INFO [2019-12-17 12:36:14,156] jepsen worker 80 - jepsen.util 10340 :invoke :read [26 nil] INFO [2019-12-17 12:36:14,159] jepsen worker 18 - jepsen.util 10818 :ok :write [6 9] INFO [2019-12-17 12:36:14,159] jepsen worker 18 - jepsen.util 10818 :invoke :read [6 nil] INFO [2019-12-17 12:36:14,162] jepsen worker 46 - jepsen.util 10396 :fail :read [15 nil] :RequestTimeoutException INFO [2019-12-17 12:36:14,162] jepsen worker 65 - jepsen.util 9065 :info :write [21 2] :RequestCanceledException INFO [2019-12-17 12:36:14,162] jepsen worker 46 - jepsen.util 10396 :invoke :read [15 nil] INFO [2019-12-17 12:36:14,163] jepsen worker 65 - jepsen.util 9155 :invoke :cas [21 [28 6]] INFO [2019-12-17 12:36:14,166] jepsen worker 55 - jepsen.util 11125 :ok :write [18 27] INFO [2019-12-17 12:36:14,167] jepsen worker 55 - jepsen.util 11125 :invoke :read [18 nil] INFO [2019-12-17 12:36:14,167] jepsen worker 14 - jepsen.util 10364 :ok :write [4 8] INFO [2019-12-17 12:36:14,167] jepsen worker 14 - jepsen.util 10364 :invoke :write [4 45] INFO [2019-12-17 12:36:14,193] jepsen worker 53 - jepsen.util 9953 :ok :read [17 20] INFO [2019-12-17 12:36:14,193] jepsen worker 53 - jepsen.util 9953 :invoke :write [17 25] INFO [2019-12-17 12:36:14,205] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":60844,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:36:14,205] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":60844,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:36:14,205] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":60845,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:36:14,205] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":60845,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:36:14,205] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":60846,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:36:14,205] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":60846,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:36:14,208] jepsen worker 30 - jepsen.util 10110 :fail :cas [10 [4 1]] :ValueNotSwapFrom INFO [2019-12-17 12:36:14,209] jepsen worker 30 - jepsen.util 10110 :invoke :read [10 nil] INFO [2019-12-17 12:36:14,214] jepsen worker 3 - jepsen.util 10353 :ok :read [1 14] INFO [2019-12-17 12:36:14,214] jepsen worker 3 - jepsen.util 10353 :invoke :write [1 40] INFO [2019-12-17 12:36:14,222] jepsen worker 18 - jepsen.util 10818 :ok :read [6 9] INFO [2019-12-17 12:36:14,223] jepsen worker 18 - jepsen.util 10818 :invoke :cas [6 [41 2]] INFO [2019-12-17 12:36:14,226] jepsen worker 88 - jepsen.util 10978 :fail :cas [29 [8 37]] :ValueNotSwapFrom INFO [2019-12-17 12:36:14,226] jepsen worker 88 - jepsen.util 10978 :invoke :read [29 nil] INFO [2019-12-17 12:36:14,231] jepsen worker 77 - jepsen.util 14927 :info :write [25 11] :RequestCanceledException INFO [2019-12-17 12:36:14,232] jepsen worker 77 - jepsen.util 15017 :invoke :write [25 12] INFO [2019-12-17 12:36:14,240] jepsen worker 30 - jepsen.util 10110 :ok :read [10 30] INFO [2019-12-17 12:36:14,241] jepsen worker 30 - jepsen.util 10110 :invoke :read [10 nil] INFO [2019-12-17 12:36:14,245] jepsen worker 39 - jepsen.util 13899 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:36:14,245] jepsen worker 39 - jepsen.util 13899 :invoke :cas [13 [22 20]] INFO [2019-12-17 12:36:14,246] jepsen worker 13 - jepsen.util 10003 :ok :read [4 8] INFO [2019-12-17 12:36:14,247] jepsen worker 13 - jepsen.util 10003 :invoke :read [4 nil] INFO [2019-12-17 12:36:14,255] jepsen worker 29 - jepsen.util 8939 :ok :write [9 15] INFO [2019-12-17 12:36:14,255] jepsen worker 29 - jepsen.util 8939 :invoke :cas [9 [49 36]] INFO [2019-12-17 12:36:14,261] jepsen worker 10 - jepsen.util 10450 :fail :read [3 nil] :RequestTimeoutException INFO [2019-12-17 12:36:14,261] jepsen worker 34 - jepsen.util 14614 :info :cas [11 [1 1]] :RequestCanceledException INFO [2019-12-17 12:36:14,262] jepsen worker 10 - jepsen.util 10450 :invoke :write [3 12] INFO [2019-12-17 12:36:14,263] jepsen worker 34 - jepsen.util 14704 :invoke :read [11 nil] INFO [2019-12-17 12:36:14,268] jepsen worker 18 - jepsen.util 10818 :fail :cas [6 [41 2]] :ValueNotSwapFrom INFO [2019-12-17 12:36:14,269] jepsen worker 18 - jepsen.util 10818 :invoke :cas [6 [38 8]] INFO [2019-12-17 12:36:14,269] jepsen worker 79 - jepsen.util 10969 :ok :read [26 22] INFO [2019-12-17 12:36:14,269] jepsen worker 79 - jepsen.util 10969 :invoke :read [26 nil] INFO [2019-12-17 12:36:14,277] jepsen worker 30 - jepsen.util 10110 :ok :read [10 30] INFO [2019-12-17 12:36:14,277] jepsen worker 30 - jepsen.util 10110 :invoke :write [10 13] INFO [2019-12-17 12:36:14,298] jepsen worker 59 - jepsen.util 13469 :info :cas [19 [43 22]] :RequestCanceledException INFO [2019-12-17 12:36:14,299] jepsen worker 59 - jepsen.util 13559 :invoke :read [19 nil] INFO [2019-12-17 12:36:14,304] jepsen worker 29 - jepsen.util 8939 :fail :cas [9 [49 36]] :ValueNotSwapFrom INFO [2019-12-17 12:36:14,304] jepsen worker 29 - jepsen.util 8939 :invoke :read [9 nil] INFO [2019-12-17 12:36:14,313] jepsen worker 18 - jepsen.util 10818 :fail :cas [6 [38 8]] :ValueNotSwapFrom INFO [2019-12-17 12:36:14,314] jepsen worker 18 - jepsen.util 10818 :invoke :read [6 nil] INFO [2019-12-17 12:36:14,315] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":60852,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:36:14,315] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":60852,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:36:14,315] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":60856,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:36:14,315] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":60856,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:36:14,320] jepsen worker 89 - jepsen.util 10799 :ok :read [29 25] INFO [2019-12-17 12:36:14,320] jepsen worker 89 - jepsen.util 10799 :invoke :cas [29 [21 40]] INFO [2019-12-17 12:36:14,332] jepsen worker 21 - jepsen.util 13881 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:36:14,332] jepsen worker 21 - jepsen.util 13881 :invoke :write [7 26] INFO [2019-12-17 12:36:14,346] jepsen worker 31 - jepsen.util 9211 :ok :write [10 46] INFO [2019-12-17 12:36:14,347] jepsen worker 31 - jepsen.util 9211 :invoke :cas [10 [22 4]] INFO [2019-12-17 12:36:14,358] jepsen worker 5 - jepsen.util 10085 :ok :read [1 14] INFO [2019-12-17 12:36:14,358] jepsen worker 5 - jepsen.util 10085 :invoke :cas [1 [21 3]] INFO [2019-12-17 12:36:14,360] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:36:14,360] jepsen nemesis - couchbase.util Rebalance check count: 475 INFO [2019-12-17 12:36:14,360] jepsen nemesis - couchbase.util Rebalance stuck count: 456 INFO [2019-12-17 12:36:14,362] jepsen worker 56 - jepsen.util 11036 :ok :write [18 12] INFO [2019-12-17 12:36:14,362] jepsen worker 56 - jepsen.util 11036 :invoke :cas [18 [34 23]] INFO [2019-12-17 12:36:14,367] jepsen worker 65 - jepsen.util 9155 :fail :cas [21 [28 6]] :ValueNotSwapFrom INFO [2019-12-17 12:36:14,367] jepsen worker 65 - jepsen.util 9155 :invoke :read [21 nil] INFO [2019-12-17 12:36:14,382] jepsen worker 71 - jepsen.util 11411 :ok :read [23 14] INFO [2019-12-17 12:36:14,382] jepsen worker 71 - jepsen.util 11411 :invoke :write [23 31] INFO [2019-12-17 12:36:14,397] jepsen worker 31 - jepsen.util 9211 :fail :cas [10 [22 4]] :ValueNotSwapFrom INFO [2019-12-17 12:36:14,398] jepsen worker 31 - jepsen.util 9211 :invoke :read [10 nil] INFO [2019-12-17 12:36:14,398] jepsen worker 38 - jepsen.util 11198 :fail :cas [12 [19 0]] :ValueNotSwapFrom INFO [2019-12-17 12:36:14,398] jepsen worker 38 - jepsen.util 11198 :invoke :read [12 nil] INFO [2019-12-17 12:36:14,398] jepsen worker 86 - jepsen.util 15476 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:36:14,398] jepsen worker 86 - jepsen.util 15476 :invoke :read [28 nil] INFO [2019-12-17 12:36:14,408] jepsen worker 4 - jepsen.util 9724 :ok :read [1 14] INFO [2019-12-17 12:36:14,408] jepsen worker 9 - jepsen.util 11259 :ok :write [3 28] INFO [2019-12-17 12:36:14,408] jepsen worker 4 - jepsen.util 9724 :invoke :cas [1 [22 21]] INFO [2019-12-17 12:36:14,408] jepsen worker 9 - jepsen.util 11259 :invoke :cas [3 [30 46]] INFO [2019-12-17 12:36:14,415] jepsen worker 8 - jepsen.util 13958 :info :write [2 17] :RequestCanceledException INFO [2019-12-17 12:36:14,416] jepsen worker 8 - jepsen.util 14048 :invoke :cas [2 [0 14]] INFO [2019-12-17 12:36:14,430] jepsen worker 84 - jepsen.util 14034 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:36:14,430] jepsen worker 84 - jepsen.util 14034 :invoke :read [28 nil] INFO [2019-12-17 12:36:14,434] jepsen worker 87 - jepsen.util 10077 :ok :write [29 4] INFO [2019-12-17 12:36:14,434] jepsen worker 87 - jepsen.util 10077 :invoke :read [29 nil] INFO [2019-12-17 12:36:14,437] jepsen worker 69 - jepsen.util 10689 :fail :cas [23 [15 7]] :ValueNotSwapFrom INFO [2019-12-17 12:36:14,437] jepsen worker 69 - jepsen.util 10689 :invoke :cas [23 [44 25]] INFO [2019-12-17 12:36:14,442] jepsen worker 65 - jepsen.util 9155 :ok :read [21 15] INFO [2019-12-17 12:36:14,442] jepsen worker 65 - jepsen.util 9155 :invoke :write [21 20] INFO [2019-12-17 12:36:14,445] jepsen worker 43 - jepsen.util 14263 :info :cas [14 [11 23]] :RequestCanceledException INFO [2019-12-17 12:36:14,446] jepsen worker 43 - jepsen.util 14353 :invoke :cas [14 [32 41]] INFO [2019-12-17 12:36:14,449] jepsen worker 9 - jepsen.util 11259 :fail :cas [3 [30 46]] :ValueNotSwapFrom INFO [2019-12-17 12:36:14,449] jepsen worker 9 - jepsen.util 11259 :invoke :read [3 nil] INFO [2019-12-17 12:36:14,457] jepsen worker 31 - jepsen.util 9211 :ok :read [10 46] INFO [2019-12-17 12:36:14,457] jepsen worker 31 - jepsen.util 9211 :invoke :read [10 nil] INFO [2019-12-17 12:36:14,461] jepsen worker 16 - jepsen.util 15676 :info :cas [5 [16 22]] :RequestCanceledException INFO [2019-12-17 12:36:14,462] jepsen worker 55 - jepsen.util 11125 :ok :read [18 12] INFO [2019-12-17 12:36:14,462] jepsen worker 55 - jepsen.util 11125 :invoke :cas [18 [42 0]] INFO [2019-12-17 12:36:14,463] jepsen worker 51 - jepsen.util 10131 :fail :cas [17 [11 18]] :ValueNotSwapFrom INFO [2019-12-17 12:36:14,464] jepsen worker 51 - jepsen.util 10131 :invoke :write [17 14] INFO [2019-12-17 12:36:14,464] jepsen worker 16 - jepsen.util 15766 :invoke :cas [5 [12 3]] INFO [2019-12-17 12:36:14,470] jepsen worker 9 - jepsen.util 11259 :ok :read [3 28] INFO [2019-12-17 12:36:14,471] jepsen worker 9 - jepsen.util 11259 :invoke :read [3 nil] INFO [2019-12-17 12:36:14,477] jepsen worker 80 - jepsen.util 10340 :ok :read [26 22] INFO [2019-12-17 12:36:14,478] jepsen worker 80 - jepsen.util 10340 :invoke :cas [26 [23 40]] INFO [2019-12-17 12:36:14,489] jepsen worker 9 - jepsen.util 11259 :ok :read [3 28] INFO [2019-12-17 12:36:14,489] jepsen worker 9 - jepsen.util 11259 :invoke :write [3 29] INFO [2019-12-17 12:36:14,499] jepsen worker 27 - jepsen.util 10017 :info :write [9 29] :RequestCanceledException INFO [2019-12-17 12:36:14,500] jepsen worker 27 - jepsen.util 10107 :invoke :read [9 nil] INFO [2019-12-17 12:36:14,519] jepsen worker 56 - jepsen.util 11036 :fail :cas [18 [34 23]] :ValueNotSwapFrom INFO [2019-12-17 12:36:14,519] jepsen worker 56 - jepsen.util 11036 :invoke :read [18 nil] INFO [2019-12-17 12:36:14,522] jepsen worker 65 - jepsen.util 9155 :ok :write [21 20] INFO [2019-12-17 12:36:14,522] jepsen worker 65 - jepsen.util 9155 :invoke :cas [21 [20 38]] INFO [2019-12-17 12:36:14,523] jepsen worker 54 - jepsen.util 11484 :ok :read [18 12] INFO [2019-12-17 12:36:14,523] jepsen worker 54 - jepsen.util 11484 :invoke :cas [18 [12 20]] INFO [2019-12-17 12:36:14,531] jepsen worker 22 - jepsen.util 14152 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:36:14,531] jepsen worker 67 - jepsen.util 14467 :info :cas [22 [37 5]] :RequestCanceledException INFO [2019-12-17 12:36:14,531] jepsen worker 78 - jepsen.util 10518 :info :write [26 6] :RequestCanceledException INFO [2019-12-17 12:36:14,532] jepsen worker 22 - jepsen.util 14152 :invoke :cas [7 [42 25]] INFO [2019-12-17 12:36:14,532] jepsen worker 78 - jepsen.util 10608 :invoke :read [26 nil] INFO [2019-12-17 12:36:14,534] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":60887,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:36:14,534] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":60887,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:36:14,534] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":60891,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:36:14,534] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":60891,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:36:14,541] jepsen worker 67 - jepsen.util 14557 :invoke :write [22 19] INFO [2019-12-17 12:36:14,563] jepsen worker 72 - jepsen.util 10692 :info :write [24 21] :RequestCanceledException INFO [2019-12-17 12:36:14,563] jepsen worker 28 - jepsen.util 8668 :info :cas [9 [28 46]] :RequestCanceledException INFO [2019-12-17 12:36:14,564] jepsen worker 72 - jepsen.util 10782 :invoke :cas [24 [40 35]] INFO [2019-12-17 12:36:14,564] jepsen worker 28 - jepsen.util 8758 :invoke :write [9 10] INFO [2019-12-17 12:36:14,579] jepsen worker 12 - jepsen.util 10992 :ok :write [4 19] INFO [2019-12-17 12:36:14,579] jepsen worker 12 - jepsen.util 10992 :invoke :read [4 nil] INFO [2019-12-17 12:36:14,586] jepsen worker 53 - jepsen.util 9953 :ok :write [17 25] INFO [2019-12-17 12:36:14,586] jepsen worker 53 - jepsen.util 9953 :invoke :cas [17 [32 23]] INFO [2019-12-17 12:36:14,594] jepsen worker 72 - jepsen.util 10782 :fail :cas [24 [40 35]] :ValueNotSwapFrom INFO [2019-12-17 12:36:14,594] jepsen worker 72 - jepsen.util 10782 :invoke :write [24 38] INFO [2019-12-17 12:36:14,616] jepsen worker 50 - jepsen.util 14810 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:36:14,617] jepsen worker 50 - jepsen.util 14810 :invoke :write [16 7] INFO [2019-12-17 12:36:14,633] jepsen worker 65 - jepsen.util 9155 :ok :cas [21 [20 38]] INFO [2019-12-17 12:36:14,636] jepsen worker 87 - jepsen.util 10077 :ok :read [29 4] INFO [2019-12-17 12:36:14,636] jepsen worker 65 - jepsen.util 9155 :invoke :cas [21 [21 22]] INFO [2019-12-17 12:36:14,636] jepsen worker 87 - jepsen.util 10077 :invoke :cas [29 [17 40]] INFO [2019-12-17 12:36:14,638] jepsen worker 79 - jepsen.util 10969 :ok :read [26 22] INFO [2019-12-17 12:36:14,639] jepsen worker 79 - jepsen.util 10969 :invoke :write [26 47] INFO [2019-12-17 12:36:14,642] jepsen worker 88 - jepsen.util 10978 :ok :read [29 4] INFO [2019-12-17 12:36:14,642] jepsen worker 88 - jepsen.util 10978 :invoke :cas [29 [21 33]] INFO [2019-12-17 12:36:14,656] jepsen worker 28 - jepsen.util 8758 :ok :write [9 10] INFO [2019-12-17 12:36:14,657] jepsen worker 28 - jepsen.util 8758 :invoke :cas [9 [46 26]] INFO [2019-12-17 12:36:14,678] jepsen worker 30 - jepsen.util 10110 :ok :write [10 13] INFO [2019-12-17 12:36:14,678] jepsen worker 30 - jepsen.util 10110 :invoke :cas [10 [46 45]] INFO [2019-12-17 12:36:14,683] jepsen worker 38 - jepsen.util 11198 :ok :read [12 30] INFO [2019-12-17 12:36:14,683] jepsen worker 38 - jepsen.util 11198 :invoke :cas [12 [36 45]] INFO [2019-12-17 12:36:14,683] jepsen worker 28 - jepsen.util 8758 :fail :cas [9 [46 26]] :ValueNotSwapFrom INFO [2019-12-17 12:36:14,683] jepsen worker 28 - jepsen.util 8758 :invoke :read [9 nil] INFO [2019-12-17 12:36:14,684] jepsen worker 65 - jepsen.util 9155 :fail :cas [21 [21 22]] :ValueNotSwapFrom INFO [2019-12-17 12:36:14,684] jepsen worker 65 - jepsen.util 9155 :invoke :write [21 11] INFO [2019-12-17 12:36:14,696] jepsen worker 13 - jepsen.util 10003 :ok :read [4 19] INFO [2019-12-17 12:36:14,696] jepsen worker 13 - jepsen.util 10003 :invoke :read [4 nil] INFO [2019-12-17 12:36:14,700] jepsen worker 30 - jepsen.util 10110 :fail :cas [10 [46 45]] :ValueNotSwapFrom INFO [2019-12-17 12:36:14,700] jepsen worker 30 - jepsen.util 10110 :invoke :read [10 nil] INFO [2019-12-17 12:36:14,717] jepsen worker 69 - jepsen.util 10689 :fail :cas [23 [44 25]] :ValueNotSwapFrom INFO [2019-12-17 12:36:14,717] jepsen worker 69 - jepsen.util 10689 :invoke :read [23 nil] INFO [2019-12-17 12:36:14,724] jepsen worker 28 - jepsen.util 8758 :ok :read [9 10] INFO [2019-12-17 12:36:14,724] jepsen worker 28 - jepsen.util 8758 :invoke :read [9 nil] INFO [2019-12-17 12:36:14,729] jepsen worker 5 - jepsen.util 10085 :fail :cas [1 [21 3]] :ValueNotSwapFrom INFO [2019-12-17 12:36:14,730] jepsen worker 5 - jepsen.util 10085 :invoke :cas [1 [21 30]] INFO [2019-12-17 12:36:14,732] jepsen worker 15 - jepsen.util 13245 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:36:14,732] jepsen worker 57 - jepsen.util 14007 :info :cas [19 [8 4]] :RequestCanceledException INFO [2019-12-17 12:36:14,732] jepsen worker 15 - jepsen.util 13245 :invoke :read [5 nil] INFO [2019-12-17 12:36:14,733] jepsen worker 57 - jepsen.util 14097 :invoke :cas [19 [41 6]] INFO [2019-12-17 12:36:14,734] jepsen worker 80 - jepsen.util 10340 :fail :cas [26 [23 40]] :ValueNotSwapFrom INFO [2019-12-17 12:36:14,734] jepsen worker 80 - jepsen.util 10340 :invoke :read [26 nil] INFO [2019-12-17 12:36:14,748] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":60924,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:36:14,749] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":60924,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:36:14,749] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":60928,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:36:14,749] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":60928,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:36:14,769] jepsen worker 89 - jepsen.util 10799 :fail :cas [29 [21 40]] :ValueNotSwapFrom INFO [2019-12-17 12:36:14,769] jepsen worker 89 - jepsen.util 10799 :invoke :write [29 42] INFO [2019-12-17 12:36:14,773] jepsen worker 30 - jepsen.util 10110 :ok :read [10 13] INFO [2019-12-17 12:36:14,773] jepsen worker 30 - jepsen.util 10110 :invoke :write [10 12] INFO [2019-12-17 12:36:14,799] jepsen worker 78 - jepsen.util 10608 :ok :read [26 22] INFO [2019-12-17 12:36:14,799] jepsen worker 78 - jepsen.util 10608 :invoke :cas [26 [27 8]] INFO [2019-12-17 12:36:14,801] jepsen worker 28 - jepsen.util 8758 :ok :read [9 10] INFO [2019-12-17 12:36:14,801] jepsen worker 28 - jepsen.util 8758 :invoke :read [9 nil] INFO [2019-12-17 12:36:14,819] jepsen worker 28 - jepsen.util 8758 :ok :read [9 10] INFO [2019-12-17 12:36:14,819] jepsen worker 28 - jepsen.util 8758 :invoke :write [9 27] INFO [2019-12-17 12:36:14,822] jepsen worker 65 - jepsen.util 9155 :ok :write [21 11] INFO [2019-12-17 12:36:14,822] jepsen worker 65 - jepsen.util 9155 :invoke :cas [21 [6 3]] INFO [2019-12-17 12:36:14,831] jepsen worker 2 - jepsen.util 13862 :info :write [0 4] :RequestCanceledException INFO [2019-12-17 12:36:14,831] jepsen worker 47 - jepsen.util 11207 :fail :read [15 nil] :RequestTimeoutException INFO [2019-12-17 12:36:14,831] jepsen worker 47 - jepsen.util 11207 :invoke :read [15 nil] INFO [2019-12-17 12:36:14,832] jepsen worker 2 - jepsen.util 13952 :invoke :cas [0 [26 17]] INFO [2019-12-17 12:36:14,842] jepsen worker 53 - jepsen.util 9953 :fail :cas [17 [32 23]] :ValueNotSwapFrom INFO [2019-12-17 12:36:14,843] jepsen worker 53 - jepsen.util 9953 :invoke :cas [17 [42 45]] INFO [2019-12-17 12:36:14,844] jepsen worker 88 - jepsen.util 10978 :fail :cas [29 [21 33]] :ValueNotSwapFrom INFO [2019-12-17 12:36:14,844] jepsen worker 88 - jepsen.util 10978 :invoke :read [29 nil] INFO [2019-12-17 12:36:14,849] jepsen worker 17 - jepsen.util 14237 :info :cas [5 [44 42]] :RequestCanceledException INFO [2019-12-17 12:36:14,851] jepsen worker 9 - jepsen.util 11259 :ok :write [3 29] INFO [2019-12-17 12:36:14,851] jepsen worker 9 - jepsen.util 11259 :invoke :write [3 34] INFO [2019-12-17 12:36:14,855] jepsen worker 65 - jepsen.util 9155 :fail :cas [21 [6 3]] :ValueNotSwapFrom INFO [2019-12-17 12:36:14,855] jepsen worker 65 - jepsen.util 9155 :invoke :cas [21 [40 29]] INFO [2019-12-17 12:36:14,860] jepsen worker 42 - jepsen.util 15072 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:36:14,860] jepsen worker 42 - jepsen.util 15072 :invoke :read [14 nil] INFO [2019-12-17 12:36:14,862] jepsen worker 17 - jepsen.util 14327 :invoke :write [5 47] INFO [2019-12-17 12:36:14,866] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":60958,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:36:14,866] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":60958,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:36:14,866] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":60961,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:36:14,866] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":60961,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:36:14,879] jepsen worker 65 - jepsen.util 9155 :fail :cas [21 [40 29]] :ValueNotSwapFrom INFO [2019-12-17 12:36:14,879] jepsen worker 65 - jepsen.util 9155 :invoke :read [21 nil] INFO [2019-12-17 12:36:14,894] jepsen worker 76 - jepsen.util 14656 :info :write [25 31] :RequestCanceledException INFO [2019-12-17 12:36:14,894] jepsen worker 24 - jepsen.util 12984 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:36:14,894] jepsen worker 60 - jepsen.util 15180 :info :write [20 10] :RequestCanceledException INFO [2019-12-17 12:36:14,895] jepsen worker 24 - jepsen.util 12984 :invoke :read [8 nil] INFO [2019-12-17 12:36:14,896] jepsen worker 76 - jepsen.util 14746 :invoke :cas [25 [27 1]] INFO [2019-12-17 12:36:14,896] jepsen worker 60 - jepsen.util 15270 :invoke :cas [20 [5 15]] INFO [2019-12-17 12:36:14,897] jepsen worker 78 - jepsen.util 10608 :fail :cas [26 [27 8]] :ValueNotSwapFrom INFO [2019-12-17 12:36:14,898] jepsen worker 78 - jepsen.util 10608 :invoke :write [26 47] INFO [2019-12-17 12:36:14,913] jepsen worker 65 - jepsen.util 9155 :ok :read [21 11] INFO [2019-12-17 12:36:14,913] jepsen worker 65 - jepsen.util 9155 :invoke :cas [21 [3 15]] INFO [2019-12-17 12:36:14,916] jepsen worker 36 - jepsen.util 11376 :info :write [12 31] :RequestCanceledException INFO [2019-12-17 12:36:14,917] jepsen worker 36 - jepsen.util 11466 :invoke :write [12 23] INFO [2019-12-17 12:36:14,919] jepsen worker 28 - jepsen.util 8758 :ok :write [9 27] INFO [2019-12-17 12:36:14,920] jepsen worker 28 - jepsen.util 8758 :invoke :cas [9 [46 39]] INFO [2019-12-17 12:36:14,931] jepsen worker 41 - jepsen.util 15161 :info :write [13 32] :RequestCanceledException INFO [2019-12-17 12:36:14,931] jepsen worker 73 - jepsen.util 9883 :info :cas [24 [30 48]] :RequestCanceledException INFO [2019-12-17 12:36:14,932] jepsen worker 41 - jepsen.util 15251 :invoke :write [13 40] INFO [2019-12-17 12:36:14,932] jepsen worker 73 - jepsen.util 9973 :invoke :read [24 nil] INFO [2019-12-17 12:36:14,941] jepsen worker 87 - jepsen.util 10077 :fail :cas [29 [17 40]] :ValueNotSwapFrom INFO [2019-12-17 12:36:14,941] jepsen worker 87 - jepsen.util 10077 :invoke :write [29 8] INFO [2019-12-17 12:36:14,956] jepsen worker 6 - jepsen.util 14586 :info :write [2 12] :RequestCanceledException INFO [2019-12-17 12:36:14,956] jepsen worker 65 - jepsen.util 9155 :fail :cas [21 [3 15]] :ValueNotSwapFrom INFO [2019-12-17 12:36:14,956] jepsen worker 65 - jepsen.util 9155 :invoke :cas [21 [16 42]] INFO [2019-12-17 12:36:14,957] jepsen worker 12 - jepsen.util 10992 :ok :read [4 19] INFO [2019-12-17 12:36:14,957] jepsen worker 12 - jepsen.util 10992 :invoke :read [4 nil] INFO [2019-12-17 12:36:14,957] jepsen worker 6 - jepsen.util 14676 :invoke :read [2 nil] INFO [2019-12-17 12:36:14,960] jepsen worker 28 - jepsen.util 8758 :fail :cas [9 [46 39]] :ValueNotSwapFrom INFO [2019-12-17 12:36:14,961] jepsen worker 28 - jepsen.util 8758 :invoke :write [9 12] INFO [2019-12-17 12:36:14,983] jepsen worker 13 - jepsen.util 10003 :ok :read [4 19] INFO [2019-12-17 12:36:14,983] jepsen worker 13 - jepsen.util 10003 :invoke :cas [4 [19 13]] INFO [2019-12-17 12:36:14,983] jepsen worker 65 - jepsen.util 9155 :fail :cas [21 [16 42]] :ValueNotSwapFrom INFO [2019-12-17 12:36:14,983] jepsen worker 65 - jepsen.util 9155 :invoke :cas [21 [8 11]] INFO [2019-12-17 12:36:14,984] jepsen worker 52 - jepsen.util 10402 :ok :write [17 4] INFO [2019-12-17 12:36:14,984] jepsen worker 52 - jepsen.util 10402 :invoke :write [17 3] INFO [2019-12-17 12:36:14,990] jepsen worker 70 - jepsen.util 11320 :ok :write [23 0] INFO [2019-12-17 12:36:14,990] jepsen worker 70 - jepsen.util 11320 :invoke :read [23 nil] INFO [2019-12-17 12:36:15,032] jepsen worker 65 - jepsen.util 9155 :fail :cas [21 [8 11]] :ValueNotSwapFrom INFO [2019-12-17 12:36:15,032] jepsen worker 19 - jepsen.util 10639 :info :write [6 42] :RequestCanceledException INFO [2019-12-17 12:36:15,033] jepsen worker 65 - jepsen.util 9155 :invoke :write [21 46] INFO [2019-12-17 12:36:15,034] jepsen worker 19 - jepsen.util 10729 :invoke :cas [6 [5 14]] INFO [2019-12-17 12:36:15,035] jepsen worker 80 - jepsen.util 10340 :ok :read [26 22] INFO [2019-12-17 12:36:15,035] jepsen worker 80 - jepsen.util 10340 :invoke :write [26 12] INFO [2019-12-17 12:36:15,037] jepsen worker 72 - jepsen.util 10782 :ok :write [24 38] INFO [2019-12-17 12:36:15,037] jepsen worker 72 - jepsen.util 10782 :invoke :cas [24 [48 38]] INFO [2019-12-17 12:36:15,037] jepsen worker 4 - jepsen.util 9724 :fail :cas [1 [22 21]] :ValueNotSwapFrom INFO [2019-12-17 12:36:15,037] jepsen worker 4 - jepsen.util 9724 :invoke :read [1 nil] INFO [2019-12-17 12:36:15,053] jepsen worker 69 - jepsen.util 10689 :ok :read [23 0] INFO [2019-12-17 12:36:15,053] jepsen worker 69 - jepsen.util 10689 :invoke :read [23 nil] INFO [2019-12-17 12:36:15,070] jepsen worker 56 - jepsen.util 11036 :ok :read [18 12] INFO [2019-12-17 12:36:15,071] jepsen worker 56 - jepsen.util 11036 :invoke :read [18 nil] INFO [2019-12-17 12:36:15,085] jepsen worker 19 - jepsen.util 10729 :fail :cas [6 [5 14]] :ValueNotSwapFrom INFO [2019-12-17 12:36:15,086] jepsen worker 19 - jepsen.util 10729 :invoke :read [6 nil] INFO [2019-12-17 12:36:15,092] jepsen worker 5 - jepsen.util 10085 :fail :cas [1 [21 30]] :ValueNotSwapFrom INFO [2019-12-17 12:36:15,092] jepsen worker 5 - jepsen.util 10085 :invoke :write [1 14] INFO [2019-12-17 12:36:15,095] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":60983,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:36:15,095] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":60983,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:36:15,095] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":60986,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:36:15,096] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":60986,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:36:15,096] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":60987,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:36:15,096] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":60987,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:36:15,101] jepsen worker 72 - jepsen.util 10782 :fail :cas [24 [48 38]] :ValueNotSwapFrom INFO [2019-12-17 12:36:15,101] jepsen worker 72 - jepsen.util 10782 :invoke :write [24 16] INFO [2019-12-17 12:36:15,127] jepsen worker 88 - jepsen.util 10978 :ok :read [29 4] INFO [2019-12-17 12:36:15,127] jepsen worker 88 - jepsen.util 10978 :invoke :read [29 nil] INFO [2019-12-17 12:36:15,132] jepsen worker 37 - jepsen.util 10117 :info :write [12 15] :RequestCanceledException INFO [2019-12-17 12:36:15,134] jepsen worker 37 - jepsen.util 10207 :invoke :cas [12 [16 19]] INFO [2019-12-17 12:36:15,139] jepsen worker 38 - jepsen.util 11198 :fail :cas [12 [36 45]] :ValueNotSwapFrom INFO [2019-12-17 12:36:15,139] jepsen worker 38 - jepsen.util 11198 :invoke :write [12 12] INFO [2019-12-17 12:36:15,154] jepsen worker 19 - jepsen.util 10729 :ok :read [6 9] INFO [2019-12-17 12:36:15,154] jepsen worker 19 - jepsen.util 10729 :invoke :read [6 nil] INFO [2019-12-17 12:36:15,159] jepsen worker 35 - jepsen.util 14345 :info :cas [11 [44 3]] :RequestCanceledException INFO [2019-12-17 12:36:15,160] jepsen worker 69 - jepsen.util 10689 :ok :read [23 0] INFO [2019-12-17 12:36:15,160] jepsen worker 69 - jepsen.util 10689 :invoke :cas [23 [3 14]] INFO [2019-12-17 12:36:15,161] jepsen worker 35 - jepsen.util 14435 :invoke :write [11 12] INFO [2019-12-17 12:36:15,189] jepsen worker 19 - jepsen.util 10729 :ok :read [6 9] INFO [2019-12-17 12:36:15,190] jepsen worker 19 - jepsen.util 10729 :invoke :read [6 nil] INFO [2019-12-17 12:36:15,194] jepsen worker 28 - jepsen.util 8758 :ok :write [9 12] INFO [2019-12-17 12:36:15,194] jepsen worker 28 - jepsen.util 8758 :invoke :write [9 33] INFO [2019-12-17 12:36:15,207] jepsen worker 65 - jepsen.util 9155 :ok :write [21 46] INFO [2019-12-17 12:36:15,207] jepsen worker 65 - jepsen.util 9155 :invoke :read [21 nil] INFO [2019-12-17 12:36:15,212] jepsen worker 19 - jepsen.util 10729 :ok :read [6 9] INFO [2019-12-17 12:36:15,212] jepsen worker 19 - jepsen.util 10729 :invoke :read [6 nil] INFO [2019-12-17 12:36:15,214] jepsen worker 37 - jepsen.util 10207 :fail :cas [12 [16 19]] :ValueNotSwapFrom INFO [2019-12-17 12:36:15,214] jepsen worker 37 - jepsen.util 10207 :invoke :write [12 32] INFO [2019-12-17 12:36:15,224] jepsen worker 56 - jepsen.util 11036 :ok :read [18 12] INFO [2019-12-17 12:36:15,224] jepsen worker 56 - jepsen.util 11036 :invoke :write [18 29] INFO [2019-12-17 12:36:15,225] jepsen worker 65 - jepsen.util 9155 :ok :read [21 46] INFO [2019-12-17 12:36:15,225] jepsen worker 65 - jepsen.util 9155 :invoke :write [21 12] INFO [2019-12-17 12:36:15,232] jepsen worker 44 - jepsen.util 14534 :info :write [14 6] :RequestCanceledException INFO [2019-12-17 12:36:15,232] jepsen worker 62 - jepsen.util 14642 :info :cas [20 [27 5]] :RequestCanceledException INFO [2019-12-17 12:36:15,233] jepsen worker 44 - jepsen.util 14624 :invoke :write [14 6] INFO [2019-12-17 12:36:15,236] jepsen worker 62 - jepsen.util 14732 :invoke :cas [20 [40 35]] INFO [2019-12-17 12:36:15,245] jepsen worker 32 - jepsen.util 10562 :info :write [10 28] :RequestCanceledException INFO [2019-12-17 12:36:15,245] jepsen worker 63 - jepsen.util 9513 :fail :read [21 nil] :RequestTimeoutException INFO [2019-12-17 12:36:15,246] jepsen worker 63 - jepsen.util 9513 :invoke :cas [21 [9 15]] INFO [2019-12-17 12:36:15,247] jepsen worker 32 - jepsen.util 10652 :invoke :read [10 nil] INFO [2019-12-17 12:36:15,250] jepsen worker 70 - jepsen.util 11320 :ok :read [23 0] INFO [2019-12-17 12:36:15,250] jepsen worker 70 - jepsen.util 11320 :invoke :write [23 5] INFO [2019-12-17 12:36:15,277] jepsen worker 87 - jepsen.util 10077 :ok :write [29 8] INFO [2019-12-17 12:36:15,277] jepsen worker 87 - jepsen.util 10077 :invoke :cas [29 [10 0]] INFO [2019-12-17 12:36:15,283] jepsen worker 19 - jepsen.util 10729 :ok :read [6 9] INFO [2019-12-17 12:36:15,284] jepsen worker 19 - jepsen.util 10729 :invoke :write [6 2] INFO [2019-12-17 12:36:15,297] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":61006,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:36:15,297] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":61016,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:36:15,297] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":61006,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:36:15,297] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":61016,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:36:15,297] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":61007,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:36:15,297] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":61019,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:36:15,297] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":61007,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:36:15,297] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":61019,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:36:15,309] jepsen worker 63 - jepsen.util 9513 :fail :cas [21 [9 15]] :ValueNotSwapFrom INFO [2019-12-17 12:36:15,309] jepsen worker 63 - jepsen.util 9513 :invoke :cas [21 [3 22]] INFO [2019-12-17 12:36:15,316] jepsen worker 85 - jepsen.util 15205 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:36:15,316] jepsen worker 85 - jepsen.util 15205 :invoke :write [28 17] INFO [2019-12-17 12:36:15,322] jepsen worker 88 - jepsen.util 10978 :ok :read [29 8] INFO [2019-12-17 12:36:15,322] jepsen worker 88 - jepsen.util 10978 :invoke :read [29 nil] INFO [2019-12-17 12:36:15,332] jepsen worker 23 - jepsen.util 14333 :info :write [7 24] :RequestCanceledException INFO [2019-12-17 12:36:15,333] jepsen worker 23 - jepsen.util 14423 :invoke :cas [7 [35 18]] INFO [2019-12-17 12:36:15,344] jepsen worker 65 - jepsen.util 9155 :ok :write [21 12] INFO [2019-12-17 12:36:15,344] jepsen worker 65 - jepsen.util 9155 :invoke :write [21 7] INFO [2019-12-17 12:36:15,345] jepsen worker 74 - jepsen.util 10334 :info :cas [24 [12 33]] :RequestCanceledException INFO [2019-12-17 12:36:15,347] jepsen worker 74 - jepsen.util 10424 :invoke :cas [24 [14 23]] INFO [2019-12-17 12:36:15,351] jepsen worker 12 - jepsen.util 10992 :ok :read [4 19] INFO [2019-12-17 12:36:15,351] jepsen worker 12 - jepsen.util 10992 :invoke :write [4 36] INFO [2019-12-17 12:36:15,355] jepsen worker 28 - jepsen.util 8758 :ok :write [9 33] INFO [2019-12-17 12:36:15,355] jepsen worker 28 - jepsen.util 8758 :invoke :cas [9 [26 22]] INFO [2019-12-17 12:36:15,363] jepsen worker 40 - jepsen.util 14710 :info :write [13 8] :RequestCanceledException INFO [2019-12-17 12:36:15,364] jepsen worker 40 - jepsen.util 14800 :invoke :write [13 49] INFO [2019-12-17 12:36:15,400] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:36:15,400] jepsen nemesis - couchbase.util Rebalance check count: 476 INFO [2019-12-17 12:36:15,400] jepsen nemesis - couchbase.util Rebalance stuck count: 457 INFO [2019-12-17 12:36:15,406] jepsen worker 28 - jepsen.util 8758 :fail :cas [9 [26 22]] :ValueNotSwapFrom INFO [2019-12-17 12:36:15,406] jepsen worker 28 - jepsen.util 8758 :invoke :write [9 18] INFO [2019-12-17 12:36:15,432] jepsen worker 69 - jepsen.util 10689 :fail :cas [23 [3 14]] :ValueNotSwapFrom INFO [2019-12-17 12:36:15,432] jepsen worker 69 - jepsen.util 10689 :invoke :write [23 16] INFO [2019-12-17 12:36:15,432] jepsen worker 66 - jepsen.util 14106 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:36:15,432] jepsen worker 66 - jepsen.util 14106 :invoke :cas [22 [19 24]] INFO [2019-12-17 12:36:15,467] jepsen worker 72 - jepsen.util 10782 :ok :write [24 16] INFO [2019-12-17 12:36:15,467] jepsen worker 9 - jepsen.util 11259 :ok :write [3 34] INFO [2019-12-17 12:36:15,467] jepsen worker 72 - jepsen.util 10782 :invoke :write [24 35] INFO [2019-12-17 12:36:15,467] jepsen worker 9 - jepsen.util 11259 :invoke :cas [3 [27 22]] INFO [2019-12-17 12:36:15,467] jepsen worker 30 - jepsen.util 10110 :ok :write [10 12] INFO [2019-12-17 12:36:15,467] jepsen worker 30 - jepsen.util 10110 :invoke :write [10 36] INFO [2019-12-17 12:36:15,481] jepsen worker 54 - jepsen.util 11484 :ok :cas [18 [12 20]] INFO [2019-12-17 12:36:15,481] jepsen worker 54 - jepsen.util 11484 :invoke :read [18 nil] INFO [2019-12-17 12:36:15,482] jepsen worker 52 - jepsen.util 10402 :ok :write [17 3] INFO [2019-12-17 12:36:15,482] jepsen worker 52 - jepsen.util 10402 :invoke :write [17 19] INFO [2019-12-17 12:36:15,493] jepsen worker 5 - jepsen.util 10085 :ok :write [1 14] INFO [2019-12-17 12:36:15,493] jepsen worker 5 - jepsen.util 10085 :invoke :read [1 nil] INFO [2019-12-17 12:36:15,498] jepsen worker 83 - jepsen.util 14573 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:36:15,498] jepsen worker 83 - jepsen.util 14573 :invoke :cas [27 [34 27]] INFO [2019-12-17 12:36:15,503] jepsen worker 53 - jepsen.util 9953 :fail :cas [17 [42 45]] :ValueNotSwapFrom INFO [2019-12-17 12:36:15,503] jepsen worker 53 - jepsen.util 9953 :invoke :write [17 36] INFO [2019-12-17 12:36:15,516] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":61043,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:36:15,516] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":61043,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:36:15,516] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":61046,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:36:15,517] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":61046,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:36:15,533] jepsen worker 55 - jepsen.util 11125 :fail :cas [18 [42 0]] :ValueNotSwapFrom INFO [2019-12-17 12:36:15,533] jepsen worker 55 - jepsen.util 11125 :invoke :cas [18 [26 40]] INFO [2019-12-17 12:36:15,542] jepsen worker 19 - jepsen.util 10729 :ok :write [6 2] INFO [2019-12-17 12:36:15,543] jepsen worker 19 - jepsen.util 10729 :invoke :read [6 nil] INFO [2019-12-17 12:36:15,546] jepsen worker 49 - jepsen.util 14179 :info :cas [16 [38 26]] :RequestCanceledException INFO [2019-12-17 12:36:15,548] jepsen worker 49 - jepsen.util 14269 :invoke :read [16 nil] INFO [2019-12-17 12:36:15,563] jepsen worker 7 - jepsen.util 14407 :info :write [2 30] :RequestCanceledException INFO [2019-12-17 12:36:15,563] jepsen worker 61 - jepsen.util 15181 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:36:15,564] jepsen worker 61 - jepsen.util 15181 :invoke :write [20 40] INFO [2019-12-17 12:36:15,564] jepsen worker 7 - jepsen.util 14497 :invoke :read [2 nil] INFO [2019-12-17 12:36:15,568] jepsen worker 74 - jepsen.util 10424 :fail :cas [24 [14 23]] :ValueNotSwapFrom INFO [2019-12-17 12:36:15,568] jepsen worker 74 - jepsen.util 10424 :invoke :cas [24 [1 36]] INFO [2019-12-17 12:36:15,578] jepsen worker 73 - jepsen.util 9973 :ok :read [24 16] INFO [2019-12-17 12:36:15,578] jepsen worker 73 - jepsen.util 9973 :invoke :read [24 nil] INFO [2019-12-17 12:36:15,591] jepsen worker 80 - jepsen.util 10340 :ok :write [26 12] INFO [2019-12-17 12:36:15,591] jepsen worker 80 - jepsen.util 10340 :invoke :write [26 43] INFO [2019-12-17 12:36:15,593] jepsen worker 47 - jepsen.util 11207 :ok :read [15 13] INFO [2019-12-17 12:36:15,593] jepsen worker 47 - jepsen.util 11207 :invoke :read [15 nil] INFO [2019-12-17 12:36:15,597] jepsen worker 4 - jepsen.util 9724 :ok :read [1 14] INFO [2019-12-17 12:36:15,597] jepsen worker 4 - jepsen.util 9724 :invoke :read [1 nil] INFO [2019-12-17 12:36:15,617] jepsen worker 26 - jepsen.util 14876 :info :write [8 37] :RequestCanceledException INFO [2019-12-17 12:36:15,620] jepsen worker 26 - jepsen.util 14966 :invoke :write [8 29] INFO [2019-12-17 12:36:15,624] jepsen worker 54 - jepsen.util 11484 :ok :read [18 20] INFO [2019-12-17 12:36:15,624] jepsen worker 54 - jepsen.util 11484 :invoke :write [18 24] INFO [2019-12-17 12:36:15,628] jepsen worker 4 - jepsen.util 9724 :ok :read [1 14] INFO [2019-12-17 12:36:15,628] jepsen worker 4 - jepsen.util 9724 :invoke :cas [1 [2 10]] INFO [2019-12-17 12:36:15,632] jepsen worker 0 - jepsen.util 15030 :info :write [0 24] :RequestCanceledException INFO [2019-12-17 12:36:15,633] jepsen worker 0 - jepsen.util 15120 :invoke :cas [0 [15 30]] INFO [2019-12-17 12:36:15,646] jepsen worker 1 - jepsen.util 14311 :info :write [0 38] :RequestCanceledException INFO [2019-12-17 12:36:15,648] jepsen worker 1 - jepsen.util 14401 :invoke :write [0 4] INFO [2019-12-17 12:36:15,653] jepsen worker 4 - jepsen.util 9724 :fail :cas [1 [2 10]] :ValueNotSwapFrom INFO [2019-12-17 12:36:15,654] jepsen worker 4 - jepsen.util 9724 :invoke :read [1 nil] INFO [2019-12-17 12:36:15,663] jepsen worker 13 - jepsen.util 10003 :ok :cas [4 [19 13]] INFO [2019-12-17 12:36:15,663] jepsen worker 13 - jepsen.util 10003 :invoke :read [4 nil] INFO [2019-12-17 12:36:15,664] jepsen worker 3 - jepsen.util 10353 :ok :write [1 40] INFO [2019-12-17 12:36:15,664] jepsen worker 3 - jepsen.util 10353 :invoke :write [1 41] INFO [2019-12-17 12:36:15,703] jepsen worker 32 - jepsen.util 10652 :ok :read [10 12] INFO [2019-12-17 12:36:15,703] jepsen worker 32 - jepsen.util 10652 :invoke :write [10 24] INFO [2019-12-17 12:36:15,710] jepsen worker 4 - jepsen.util 9724 :ok :read [1 40] INFO [2019-12-17 12:36:15,710] jepsen worker 4 - jepsen.util 9724 :invoke :cas [1 [44 13]] INFO [2019-12-17 12:36:15,712] jepsen worker 19 - jepsen.util 10729 :ok :read [6 2] INFO [2019-12-17 12:36:15,713] jepsen worker 19 - jepsen.util 10729 :invoke :cas [6 [39 22]] INFO [2019-12-17 12:36:15,719] jepsen worker 87 - jepsen.util 10077 :fail :cas [29 [10 0]] :ValueNotSwapFrom INFO [2019-12-17 12:36:15,720] jepsen worker 87 - jepsen.util 10077 :invoke :read [29 nil] INFO [2019-12-17 12:36:15,720] jepsen worker 9 - jepsen.util 11259 :fail :cas [3 [27 22]] :ValueNotSwapFrom INFO [2019-12-17 12:36:15,720] jepsen worker 9 - jepsen.util 11259 :invoke :write [3 16] INFO [2019-12-17 12:36:15,734] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":61072,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:36:15,734] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":61072,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:36:15,734] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":61073,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:36:15,734] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":61073,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:36:15,781] jepsen worker 73 - jepsen.util 9973 :ok :read [24 16] INFO [2019-12-17 12:36:15,781] jepsen worker 73 - jepsen.util 9973 :invoke :read [24 nil] INFO [2019-12-17 12:36:15,810] jepsen worker 36 - jepsen.util 11466 :ok :write [12 23] INFO [2019-12-17 12:36:15,811] jepsen worker 36 - jepsen.util 11466 :invoke :write [12 27] INFO [2019-12-17 12:36:15,816] jepsen worker 64 - jepsen.util 8074 :info :cas [21 [22 34]] :RequestCanceledException INFO [2019-12-17 12:36:15,816] jepsen worker 45 - jepsen.util 10665 :fail :read [15 nil] :RequestTimeoutException INFO [2019-12-17 12:36:15,816] jepsen worker 75 - jepsen.util 13845 :info :write [25 11] :RequestCanceledException INFO [2019-12-17 12:36:15,816] jepsen worker 45 - jepsen.util 10665 :invoke :write [15 27] INFO [2019-12-17 12:36:15,817] jepsen worker 64 - jepsen.util 8164 :invoke :cas [21 [23 14]] INFO [2019-12-17 12:36:15,818] jepsen worker 75 - jepsen.util 13935 :invoke :cas [25 [4 39]] INFO [2019-12-17 12:36:15,825] jepsen worker 19 - jepsen.util 10729 :fail :cas [6 [39 22]] :ValueNotSwapFrom INFO [2019-12-17 12:36:15,825] jepsen worker 19 - jepsen.util 10729 :invoke :write [6 47] INFO [2019-12-17 12:36:15,833] jepsen worker 28 - jepsen.util 8758 :ok :write [9 18] INFO [2019-12-17 12:36:15,833] jepsen worker 28 - jepsen.util 8758 :invoke :write [9 9] INFO [2019-12-17 12:36:15,846] jepsen worker 20 - jepsen.util 10370 :info :write [6 0] :RequestCanceledException INFO [2019-12-17 12:36:15,846] jepsen worker 20 - jepsen.util 10460 :invoke :cas [6 [5 37]] INFO [2019-12-17 12:36:15,870] jepsen worker 65 - jepsen.util 9155 :ok :write [21 7] INFO [2019-12-17 12:36:15,871] jepsen worker 65 - jepsen.util 9155 :invoke :read [21 nil] INFO [2019-12-17 12:36:15,895] jepsen worker 51 - jepsen.util 10131 :ok :write [17 14] INFO [2019-12-17 12:36:15,895] jepsen worker 51 - jepsen.util 10131 :invoke :read [17 nil] INFO [2019-12-17 12:36:15,898] jepsen worker 58 - jepsen.util 14638 :info :cas [19 [21 8]] :RequestCanceledException INFO [2019-12-17 12:36:15,899] jepsen worker 58 - jepsen.util 14728 :invoke :write [19 39] INFO [2019-12-17 12:36:15,908] jepsen worker 37 - jepsen.util 10207 :ok :write [12 32] INFO [2019-12-17 12:36:15,908] jepsen worker 37 - jepsen.util 10207 :invoke :cas [12 [18 23]] INFO [2019-12-17 12:36:15,921] jepsen worker 55 - jepsen.util 11125 :fail :cas [18 [26 40]] :ValueNotSwapFrom INFO [2019-12-17 12:36:15,921] jepsen worker 55 - jepsen.util 11125 :invoke :read [18 nil] INFO [2019-12-17 12:36:15,932] jepsen worker 25 - jepsen.util 15505 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:36:15,932] jepsen worker 25 - jepsen.util 15505 :invoke :read [8 nil] INFO [2019-12-17 12:36:15,944] jepsen worker 88 - jepsen.util 10978 :ok :read [29 8] INFO [2019-12-17 12:36:15,944] jepsen worker 88 - jepsen.util 10978 :invoke :cas [29 [47 45]] INFO [2019-12-17 12:36:15,946] jepsen worker 68 - jepsen.util 13748 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:36:15,946] jepsen worker 11 - jepsen.util 10091 :fail :read [3 nil] :RequestTimeoutException INFO [2019-12-17 12:36:15,946] jepsen worker 68 - jepsen.util 13748 :invoke :write [22 11] INFO [2019-12-17 12:36:15,946] jepsen worker 11 - jepsen.util 10091 :invoke :cas [3 [6 31]] INFO [2019-12-17 12:36:15,953] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":61095,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:36:15,953] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":61095,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:36:15,953] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":61098,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:36:15,953] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":61098,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:36:15,953] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":61099,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:36:15,953] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":61099,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:36:15,978] jepsen worker 5 - jepsen.util 10085 :ok :read [1 40] INFO [2019-12-17 12:36:15,978] jepsen worker 5 - jepsen.util 10085 :invoke :write [1 10] INFO [2019-12-17 12:36:15,981] jepsen worker 64 - jepsen.util 8164 :fail :cas [21 [23 14]] :ValueNotSwapFrom INFO [2019-12-17 12:36:15,981] jepsen worker 64 - jepsen.util 8164 :invoke :write [21 33] INFO [2019-12-17 12:36:15,983] jepsen worker 65 - jepsen.util 9155 :ok :read [21 7] INFO [2019-12-17 12:36:15,984] jepsen worker 65 - jepsen.util 9155 :invoke :read [21 nil] INFO [2019-12-17 12:36:16,014] jepsen worker 82 - jepsen.util 14482 :info :cas [27 [35 22]] :RequestCanceledException INFO [2019-12-17 12:36:16,016] jepsen worker 82 - jepsen.util 14572 :invoke :read [27 nil] INFO [2019-12-17 12:36:16,022] jepsen worker 88 - jepsen.util 10978 :fail :cas [29 [47 45]] :ValueNotSwapFrom INFO [2019-12-17 12:36:16,022] jepsen worker 88 - jepsen.util 10978 :invoke :read [29 nil] INFO [2019-12-17 12:36:16,031] jepsen worker 13 - jepsen.util 10003 :ok :read [4 13] INFO [2019-12-17 12:36:16,031] jepsen worker 13 - jepsen.util 10003 :invoke :cas [4 [45 36]] INFO [2019-12-17 12:36:16,037] jepsen worker 65 - jepsen.util 9155 :ok :read [21 7] INFO [2019-12-17 12:36:16,038] jepsen worker 65 - jepsen.util 9155 :invoke :write [21 14] INFO [2019-12-17 12:36:16,039] jepsen worker 51 - jepsen.util 10131 :ok :read [17 14] INFO [2019-12-17 12:36:16,039] jepsen worker 51 - jepsen.util 10131 :invoke :read [17 nil] INFO [2019-12-17 12:36:16,053] jepsen worker 70 - jepsen.util 11320 :ok :write [23 5] INFO [2019-12-17 12:36:16,053] jepsen worker 70 - jepsen.util 11320 :invoke :cas [23 [44 19]] INFO [2019-12-17 12:36:16,063] jepsen worker 48 - jepsen.util 13278 :info :cas [16 [25 19]] :RequestCanceledException INFO [2019-12-17 12:36:16,064] jepsen worker 48 - jepsen.util 13368 :invoke :write [16 31] INFO [2019-12-17 12:36:16,080] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":61113,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:36:16,080] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":61113,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:36:16,080] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":61116,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:36:16,080] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":61116,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:36:16,095] jepsen worker 54 - jepsen.util 11484 :ok :write [18 24] INFO [2019-12-17 12:36:16,095] jepsen worker 54 - jepsen.util 11484 :invoke :cas [18 [7 18]] INFO [2019-12-17 12:36:16,116] jepsen worker 88 - jepsen.util 10978 :ok :read [29 8] INFO [2019-12-17 12:36:16,116] jepsen worker 88 - jepsen.util 10978 :invoke :cas [29 [14 45]] INFO [2019-12-17 12:36:16,123] jepsen worker 13 - jepsen.util 10003 :fail :cas [4 [45 36]] :ValueNotSwapFrom INFO [2019-12-17 12:36:16,123] jepsen worker 13 - jepsen.util 10003 :invoke :write [4 25] INFO [2019-12-17 12:36:16,125] jepsen worker 65 - jepsen.util 9155 :ok :write [21 14] INFO [2019-12-17 12:36:16,125] jepsen worker 65 - jepsen.util 9155 :invoke :read [21 nil] INFO [2019-12-17 12:36:16,132] jepsen worker 33 - jepsen.util 14433 :info :write [11 3] :RequestCanceledException INFO [2019-12-17 12:36:16,133] jepsen worker 33 - jepsen.util 14523 :invoke :read [11 nil] INFO [2019-12-17 12:36:16,138] jepsen worker 51 - jepsen.util 10131 :ok :read [17 14] INFO [2019-12-17 12:36:16,138] jepsen worker 51 - jepsen.util 10131 :invoke :write [17 5] INFO [2019-12-17 12:36:16,142] jepsen worker 30 - jepsen.util 10110 :ok :write [10 36] INFO [2019-12-17 12:36:16,142] jepsen worker 30 - jepsen.util 10110 :invoke :cas [10 [42 32]] INFO [2019-12-17 12:36:16,148] jepsen worker 9 - jepsen.util 11259 :ok :write [3 16] INFO [2019-12-17 12:36:16,148] jepsen worker 9 - jepsen.util 11259 :invoke :cas [3 [14 28]] INFO [2019-12-17 12:36:16,159] jepsen worker 65 - jepsen.util 9155 :ok :read [21 14] INFO [2019-12-17 12:36:16,159] jepsen worker 65 - jepsen.util 9155 :invoke :cas [21 [41 29]] INFO [2019-12-17 12:36:16,166] jepsen worker 47 - jepsen.util 11207 :ok :read [15 13] INFO [2019-12-17 12:36:16,167] jepsen worker 47 - jepsen.util 11207 :invoke :cas [15 [48 41]] INFO [2019-12-17 12:36:16,174] jepsen worker 87 - jepsen.util 10077 :ok :read [29 8] INFO [2019-12-17 12:36:16,174] jepsen worker 87 - jepsen.util 10077 :invoke :write [29 47] INFO [2019-12-17 12:36:16,214] jepsen worker 65 - jepsen.util 9155 :fail :cas [21 [41 29]] :ValueNotSwapFrom INFO [2019-12-17 12:36:16,214] jepsen worker 65 - jepsen.util 9155 :invoke :write [21 7] INFO [2019-12-17 12:36:16,216] jepsen worker 81 - jepsen.util 14031 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:36:16,216] jepsen worker 81 - jepsen.util 14031 :invoke :read [27 nil] INFO [2019-12-17 12:36:16,225] jepsen worker 28 - jepsen.util 8758 :ok :write [9 9] INFO [2019-12-17 12:36:16,226] jepsen worker 28 - jepsen.util 8758 :invoke :cas [9 [43 48]] INFO [2019-12-17 12:36:16,231] jepsen worker 54 - jepsen.util 11484 :fail :cas [18 [7 18]] :ValueNotSwapFrom INFO [2019-12-17 12:36:16,231] jepsen worker 54 - jepsen.util 11484 :invoke :write [18 43] INFO [2019-12-17 12:36:16,242] jepsen worker 28 - jepsen.util 8758 :fail :cas [9 [43 48]] :ValueNotSwapFrom INFO [2019-12-17 12:36:16,242] jepsen worker 28 - jepsen.util 8758 :invoke :cas [9 [27 38]] INFO [2019-12-17 12:36:16,263] jepsen worker 46 - jepsen.util 10396 :fail :read [15 nil] :RequestTimeoutException INFO [2019-12-17 12:36:16,263] jepsen worker 14 - jepsen.util 10364 :info :write [4 45] :RequestCanceledException INFO [2019-12-17 12:36:16,263] jepsen worker 46 - jepsen.util 10396 :invoke :read [15 nil] INFO [2019-12-17 12:36:16,264] jepsen worker 28 - jepsen.util 8758 :fail :cas [9 [27 38]] :ValueNotSwapFrom INFO [2019-12-17 12:36:16,264] jepsen worker 28 - jepsen.util 8758 :invoke :write [9 8] INFO [2019-12-17 12:36:16,264] jepsen worker 14 - jepsen.util 10454 :invoke :cas [4 [10 34]] INFO [2019-12-17 12:36:16,266] jepsen worker 72 - jepsen.util 10782 :ok :write [24 35] INFO [2019-12-17 12:36:16,267] jepsen worker 72 - jepsen.util 10782 :invoke :read [24 nil] INFO [2019-12-17 12:36:16,282] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":61136,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:36:16,282] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":61144,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:36:16,282] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":61136,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:36:16,282] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":61144,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:36:16,282] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":61138,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:36:16,282] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":61147,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:36:16,283] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":61138,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:36:16,283] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":61147,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:36:16,283] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":61148,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:36:16,283] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":61148,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:36:16,296] jepsen worker 4 - jepsen.util 9724 :fail :cas [1 [44 13]] :ValueNotSwapFrom INFO [2019-12-17 12:36:16,297] jepsen worker 4 - jepsen.util 9724 :invoke :read [1 nil] INFO [2019-12-17 12:36:16,312] jepsen worker 46 - jepsen.util 10396 :ok :read [15 13] INFO [2019-12-17 12:36:16,312] jepsen worker 46 - jepsen.util 10396 :invoke :cas [15 [3 25]] INFO [2019-12-17 12:36:16,318] jepsen worker 77 - jepsen.util 15017 :info :write [25 12] :RequestCanceledException INFO [2019-12-17 12:36:16,319] jepsen worker 77 - jepsen.util 15107 :invoke :read [25 nil] INFO [2019-12-17 12:36:16,325] jepsen worker 37 - jepsen.util 10207 :fail :cas [12 [18 23]] :ValueNotSwapFrom INFO [2019-12-17 12:36:16,325] jepsen worker 37 - jepsen.util 10207 :invoke :cas [12 [1 0]] INFO [2019-12-17 12:36:16,328] jepsen worker 72 - jepsen.util 10782 :ok :read [24 35] INFO [2019-12-17 12:36:16,328] jepsen worker 72 - jepsen.util 10782 :invoke :cas [24 [3 45]] INFO [2019-12-17 12:36:16,333] jepsen worker 34 - jepsen.util 14704 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:36:16,333] jepsen worker 18 - jepsen.util 10818 :fail :read [6 nil] :RequestTimeoutException INFO [2019-12-17 12:36:16,333] jepsen worker 34 - jepsen.util 14704 :invoke :cas [11 [23 42]] INFO [2019-12-17 12:36:16,333] jepsen worker 18 - jepsen.util 10818 :invoke :write [6 39] INFO [2019-12-17 12:36:16,342] jepsen worker 46 - jepsen.util 10396 :fail :cas [15 [3 25]] :ValueNotSwapFrom INFO [2019-12-17 12:36:16,342] jepsen worker 46 - jepsen.util 10396 :invoke :read [15 nil] INFO [2019-12-17 12:36:16,345] jepsen worker 59 - jepsen.util 13559 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:36:16,345] jepsen worker 39 - jepsen.util 13899 :info :cas [13 [22 20]] :RequestCanceledException INFO [2019-12-17 12:36:16,346] jepsen worker 59 - jepsen.util 13559 :invoke :read [19 nil] INFO [2019-12-17 12:36:16,347] jepsen worker 39 - jepsen.util 13989 :invoke :write [13 1] INFO [2019-12-17 12:36:16,354] jepsen worker 72 - jepsen.util 10782 :fail :cas [24 [3 45]] :ValueNotSwapFrom INFO [2019-12-17 12:36:16,354] jepsen worker 72 - jepsen.util 10782 :invoke :read [24 nil] INFO [2019-12-17 12:36:16,363] jepsen worker 10 - jepsen.util 10450 :info :write [3 12] :RequestCanceledException INFO [2019-12-17 12:36:16,363] jepsen worker 29 - jepsen.util 8939 :fail :read [9 nil] :RequestTimeoutException INFO [2019-12-17 12:36:16,363] jepsen worker 29 - jepsen.util 8939 :invoke :write [9 17] INFO [2019-12-17 12:36:16,367] jepsen worker 10 - jepsen.util 10540 :invoke :read [3 nil] INFO [2019-12-17 12:36:16,375] jepsen worker 46 - jepsen.util 10396 :ok :read [15 13] INFO [2019-12-17 12:36:16,376] jepsen worker 46 - jepsen.util 10396 :invoke :read [15 nil] INFO [2019-12-17 12:36:16,376] jepsen worker 55 - jepsen.util 11125 :ok :read [18 24] INFO [2019-12-17 12:36:16,376] jepsen worker 55 - jepsen.util 11125 :invoke :write [18 47] INFO [2019-12-17 12:36:16,385] jepsen worker 65 - jepsen.util 9155 :ok :write [21 7] INFO [2019-12-17 12:36:16,386] jepsen worker 65 - jepsen.util 9155 :invoke :write [21 16] INFO [2019-12-17 12:36:16,391] jepsen worker 72 - jepsen.util 10782 :ok :read [24 35] INFO [2019-12-17 12:36:16,391] jepsen worker 45 - jepsen.util 10665 :ok :write [15 27] INFO [2019-12-17 12:36:16,391] jepsen worker 72 - jepsen.util 10782 :invoke :read [24 nil] INFO [2019-12-17 12:36:16,391] jepsen worker 45 - jepsen.util 10665 :invoke :cas [15 [11 35]] INFO [2019-12-17 12:36:16,391] jepsen worker 19 - jepsen.util 10729 :ok :write [6 47] INFO [2019-12-17 12:36:16,391] jepsen worker 28 - jepsen.util 8758 :ok :write [9 8] INFO [2019-12-17 12:36:16,392] jepsen worker 19 - jepsen.util 10729 :invoke :read [6 nil] INFO [2019-12-17 12:36:16,392] jepsen worker 28 - jepsen.util 8758 :invoke :write [9 18] INFO [2019-12-17 12:36:16,416] jepsen worker 71 - jepsen.util 11411 :info :write [23 31] :RequestCanceledException INFO [2019-12-17 12:36:16,418] jepsen worker 71 - jepsen.util 11501 :invoke :write [23 10] INFO [2019-12-17 12:36:16,432] jepsen worker 21 - jepsen.util 13881 :info :write [7 26] :RequestCanceledException INFO [2019-12-17 12:36:16,433] jepsen worker 21 - jepsen.util 13971 :invoke :read [7 nil] INFO [2019-12-17 12:36:16,439] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:36:16,440] jepsen nemesis - couchbase.util Rebalance check count: 477 INFO [2019-12-17 12:36:16,440] jepsen nemesis - couchbase.util Rebalance stuck count: 458 INFO [2019-12-17 12:36:16,445] jepsen worker 4 - jepsen.util 9724 :ok :read [1 40] INFO [2019-12-17 12:36:16,445] jepsen worker 4 - jepsen.util 9724 :invoke :cas [1 [34 18]] INFO [2019-12-17 12:36:16,457] jepsen worker 32 - jepsen.util 10652 :ok :write [10 24] INFO [2019-12-17 12:36:16,463] jepsen worker 32 - jepsen.util 10652 :invoke :write [10 5] INFO [2019-12-17 12:36:16,463] jepsen worker 8 - jepsen.util 14048 :info :cas [2 [0 14]] :RequestCanceledException INFO [2019-12-17 12:36:16,465] jepsen worker 8 - jepsen.util 14138 :invoke :write [2 16] INFO [2019-12-17 12:36:16,471] jepsen worker 10 - jepsen.util 10540 :ok :read [3 16] INFO [2019-12-17 12:36:16,471] jepsen worker 10 - jepsen.util 10540 :invoke :read [3 nil] INFO [2019-12-17 12:36:16,475] jepsen worker 37 - jepsen.util 10207 :fail :cas [12 [1 0]] :ValueNotSwapFrom INFO [2019-12-17 12:36:16,475] jepsen worker 37 - jepsen.util 10207 :invoke :read [12 nil] INFO [2019-12-17 12:36:16,483] jepsen worker 45 - jepsen.util 10665 :fail :cas [15 [11 35]] :ValueNotSwapFrom INFO [2019-12-17 12:36:16,483] jepsen worker 45 - jepsen.util 10665 :invoke :cas [15 [1 24]] INFO [2019-12-17 12:36:16,498] jepsen worker 31 - jepsen.util 9211 :fail :read [10 nil] :RequestTimeoutException INFO [2019-12-17 12:36:16,498] jepsen worker 86 - jepsen.util 15476 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:36:16,498] jepsen worker 16 - jepsen.util 15766 :info :cas [5 [12 3]] :RequestCanceledException INFO [2019-12-17 12:36:16,499] jepsen worker 31 - jepsen.util 9211 :invoke :write [10 6] INFO [2019-12-17 12:36:16,499] jepsen worker 86 - jepsen.util 15476 :invoke :read [28 nil] INFO [2019-12-17 12:36:16,499] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":61180,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:36:16,499] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":61180,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:36:16,499] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":61183,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:36:16,499] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":61183,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:36:16,499] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":61184,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:36:16,499] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":61184,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:36:16,500] jepsen worker 16 - jepsen.util 15856 :invoke :cas [5 [37 17]] INFO [2019-12-17 12:36:16,515] jepsen worker 19 - jepsen.util 10729 :ok :read [6 47] INFO [2019-12-17 12:36:16,515] jepsen worker 19 - jepsen.util 10729 :invoke :cas [6 [22 9]] INFO [2019-12-17 12:36:16,516] jepsen worker 27 - jepsen.util 10107 :fail :read [9 nil] :RequestTimeoutException INFO [2019-12-17 12:36:16,516] jepsen worker 27 - jepsen.util 10107 :invoke :write [9 37] INFO [2019-12-17 12:36:16,531] jepsen worker 43 - jepsen.util 14353 :info :cas [14 [32 41]] :RequestCanceledException INFO [2019-12-17 12:36:16,531] jepsen worker 84 - jepsen.util 14034 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:36:16,531] jepsen worker 84 - jepsen.util 14034 :invoke :write [28 41] INFO [2019-12-17 12:36:16,532] jepsen worker 43 - jepsen.util 14443 :invoke :cas [14 [40 49]] INFO [2019-12-17 12:36:16,565] jepsen worker 45 - jepsen.util 10665 :fail :cas [15 [1 24]] :ValueNotSwapFrom INFO [2019-12-17 12:36:16,565] jepsen worker 45 - jepsen.util 10665 :invoke :cas [15 [12 49]] INFO [2019-12-17 12:36:16,581] jepsen worker 88 - jepsen.util 10978 :fail :cas [29 [14 45]] :ValueNotSwapFrom INFO [2019-12-17 12:36:16,581] jepsen worker 88 - jepsen.util 10978 :invoke :write [29 8] INFO [2019-12-17 12:36:16,588] jepsen worker 10 - jepsen.util 10540 :ok :read [3 16] INFO [2019-12-17 12:36:16,588] jepsen worker 10 - jepsen.util 10540 :invoke :cas [3 [48 29]] INFO [2019-12-17 12:36:16,622] jepsen worker 19 - jepsen.util 10729 :fail :cas [6 [22 9]] :ValueNotSwapFrom INFO [2019-12-17 12:36:16,622] jepsen worker 19 - jepsen.util 10729 :invoke :read [6 nil] INFO [2019-12-17 12:36:16,632] jepsen worker 22 - jepsen.util 14152 :info :cas [7 [42 25]] :RequestCanceledException INFO [2019-12-17 12:36:16,633] jepsen worker 22 - jepsen.util 14242 :invoke :read [7 nil] INFO [2019-12-17 12:36:16,646] jepsen worker 67 - jepsen.util 14557 :info :write [22 19] :RequestCanceledException INFO [2019-12-17 12:36:16,647] jepsen worker 67 - jepsen.util 14647 :invoke :cas [22 [39 42]] INFO [2019-12-17 12:36:16,658] jepsen worker 18 - jepsen.util 10818 :ok :write [6 39] INFO [2019-12-17 12:36:16,658] jepsen worker 18 - jepsen.util 10818 :invoke :read [6 nil] INFO [2019-12-17 12:36:16,664] jepsen worker 10 - jepsen.util 10540 :fail :cas [3 [48 29]] :ValueNotSwapFrom INFO [2019-12-17 12:36:16,665] jepsen worker 10 - jepsen.util 10540 :invoke :cas [3 [8 22]] INFO [2019-12-17 12:36:16,684] jepsen worker 52 - jepsen.util 10402 :ok :write [17 19] INFO [2019-12-17 12:36:16,685] jepsen worker 52 - jepsen.util 10402 :invoke :cas [17 [23 4]] INFO [2019-12-17 12:36:16,695] jepsen worker 18 - jepsen.util 10818 :ok :read [6 39] INFO [2019-12-17 12:36:16,695] jepsen worker 18 - jepsen.util 10818 :invoke :cas [6 [32 48]] INFO [2019-12-17 12:36:16,698] jepsen worker 79 - jepsen.util 10969 :info :write [26 47] :RequestCanceledException INFO [2019-12-17 12:36:16,701] jepsen worker 19 - jepsen.util 10729 :ok :read [6 39] INFO [2019-12-17 12:36:16,701] jepsen worker 19 - jepsen.util 10729 :invoke :cas [6 [15 48]] INFO [2019-12-17 12:36:16,708] jepsen worker 3 - jepsen.util 10353 :ok :write [1 41] INFO [2019-12-17 12:36:16,708] jepsen worker 3 - jepsen.util 10353 :invoke :cas [1 [6 32]] INFO [2019-12-17 12:36:16,710] jepsen worker 79 - jepsen.util 11059 :invoke :read [26 nil] INFO [2019-12-17 12:36:16,717] jepsen worker 50 - jepsen.util 14810 :info :write [16 7] :RequestCanceledException INFO [2019-12-17 12:36:16,718] jepsen worker 50 - jepsen.util 14900 :invoke :cas [16 [32 14]] INFO [2019-12-17 12:36:16,723] jepsen worker 18 - jepsen.util 10818 :fail :cas [6 [32 48]] :ValueNotSwapFrom INFO [2019-12-17 12:36:16,724] jepsen worker 18 - jepsen.util 10818 :invoke :cas [6 [9 42]] INFO [2019-12-17 12:36:16,724] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":61203,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:36:16,724] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":61203,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:36:16,724] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":61208,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:36:16,724] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":61208,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:36:16,724] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":61209,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:36:16,724] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":61209,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:36:16,747] jepsen worker 70 - jepsen.util 11320 :fail :cas [23 [44 19]] :ValueNotSwapFrom INFO [2019-12-17 12:36:16,747] jepsen worker 70 - jepsen.util 11320 :invoke :read [23 nil] INFO [2019-12-17 12:36:16,754] jepsen worker 18 - jepsen.util 10818 :fail :cas [6 [9 42]] :ValueNotSwapFrom INFO [2019-12-17 12:36:16,754] jepsen worker 18 - jepsen.util 10818 :invoke :read [6 nil] INFO [2019-12-17 12:36:16,764] jepsen worker 65 - jepsen.util 9155 :ok :write [21 16] INFO [2019-12-17 12:36:16,764] jepsen worker 65 - jepsen.util 9155 :invoke :write [21 27] INFO [2019-12-17 12:36:16,771] jepsen worker 18 - jepsen.util 10818 :ok :read [6 39] INFO [2019-12-17 12:36:16,772] jepsen worker 18 - jepsen.util 10818 :invoke :read [6 nil] INFO [2019-12-17 12:36:16,778] jepsen worker 10 - jepsen.util 10540 :fail :cas [3 [8 22]] :ValueNotSwapFrom INFO [2019-12-17 12:36:16,778] jepsen worker 10 - jepsen.util 10540 :invoke :write [3 49] INFO [2019-12-17 12:36:16,784] jepsen worker 27 - jepsen.util 10107 :ok :write [9 37] INFO [2019-12-17 12:36:16,784] jepsen worker 27 - jepsen.util 10107 :invoke :cas [9 [6 36]] INFO [2019-12-17 12:36:16,796] jepsen worker 14 - jepsen.util 10454 :fail :cas [4 [10 34]] :ValueNotSwapFrom INFO [2019-12-17 12:36:16,796] jepsen worker 14 - jepsen.util 10454 :invoke :read [4 nil] INFO [2019-12-17 12:36:16,797] jepsen worker 57 - jepsen.util 14097 :info :cas [19 [41 6]] :RequestCanceledException INFO [2019-12-17 12:36:16,799] jepsen worker 57 - jepsen.util 14187 :invoke :read [19 nil] INFO [2019-12-17 12:36:16,832] jepsen worker 15 - jepsen.util 13245 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:36:16,832] jepsen worker 89 - jepsen.util 10799 :info :write [29 42] :RequestCanceledException INFO [2019-12-17 12:36:16,832] jepsen worker 15 - jepsen.util 13245 :invoke :cas [5 [15 15]] INFO [2019-12-17 12:36:16,833] jepsen worker 89 - jepsen.util 10889 :invoke :cas [29 [42 41]] INFO [2019-12-17 12:36:16,842] jepsen worker 3 - jepsen.util 10353 :fail :cas [1 [6 32]] :ValueNotSwapFrom INFO [2019-12-17 12:36:16,843] jepsen worker 3 - jepsen.util 10353 :invoke :write [1 49] INFO [2019-12-17 12:36:16,867] jepsen worker 19 - jepsen.util 10729 :fail :cas [6 [15 48]] :ValueNotSwapFrom INFO [2019-12-17 12:36:16,867] jepsen worker 19 - jepsen.util 10729 :invoke :read [6 nil] INFO [2019-12-17 12:36:16,870] jepsen worker 18 - jepsen.util 10818 :ok :read [6 39] INFO [2019-12-17 12:36:16,870] jepsen worker 18 - jepsen.util 10818 :invoke :read [6 nil] INFO [2019-12-17 12:36:16,899] jepsen worker 27 - jepsen.util 10107 :fail :cas [9 [6 36]] :ValueNotSwapFrom INFO [2019-12-17 12:36:16,899] jepsen worker 27 - jepsen.util 10107 :invoke :read [9 nil] INFO [2019-12-17 12:36:16,913] jepsen worker 2 - jepsen.util 13952 :info :cas [0 [26 17]] :RequestCanceledException INFO [2019-12-17 12:36:16,915] jepsen worker 2 - jepsen.util 14042 :invoke :write [0 15] INFO [2019-12-17 12:36:16,932] jepsen worker 17 - jepsen.util 14327 :info :write [5 47] :RequestCanceledException INFO [2019-12-17 12:36:16,932] jepsen worker 78 - jepsen.util 10608 :info :write [26 47] :RequestCanceledException INFO [2019-12-17 12:36:16,933] jepsen worker 78 - jepsen.util 10698 :invoke :cas [26 [16 22]] INFO [2019-12-17 12:36:16,933] jepsen worker 17 - jepsen.util 14417 :invoke :read [5 nil] INFO [2019-12-17 12:36:16,934] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":61231,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:36:16,934] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":61231,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:36:16,934] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":61234,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:36:16,934] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":61234,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:36:16,943] jepsen worker 76 - jepsen.util 14746 :info :cas [25 [27 1]] :RequestCanceledException INFO [2019-12-17 12:36:16,947] jepsen worker 18 - jepsen.util 10818 :ok :read [6 39] INFO [2019-12-17 12:36:16,950] jepsen worker 18 - jepsen.util 10818 :invoke :cas [6 [26 10]] INFO [2019-12-17 12:36:16,951] jepsen worker 76 - jepsen.util 14836 :invoke :read [25 nil] INFO [2019-12-17 12:36:16,963] jepsen worker 60 - jepsen.util 15270 :info :cas [20 [5 15]] :RequestCanceledException INFO [2019-12-17 12:36:16,963] jepsen worker 42 - jepsen.util 15072 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:36:16,963] jepsen worker 42 - jepsen.util 15072 :invoke :read [14 nil] INFO [2019-12-17 12:36:16,964] jepsen worker 60 - jepsen.util 15360 :invoke :write [20 27] INFO [2019-12-17 12:36:16,994] jepsen worker 14 - jepsen.util 10454 :ok :read [4 13] INFO [2019-12-17 12:36:16,994] jepsen worker 14 - jepsen.util 10454 :invoke :read [4 nil] INFO [2019-12-17 12:36:16,997] jepsen worker 41 - jepsen.util 15251 :info :write [13 40] :RequestCanceledException INFO [2019-12-17 12:36:16,997] jepsen worker 24 - jepsen.util 12984 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:36:16,997] jepsen worker 24 - jepsen.util 12984 :invoke :cas [8 [6 7]] INFO [2019-12-17 12:36:16,998] jepsen worker 41 - jepsen.util 15341 :invoke :read [13 nil] INFO [2019-12-17 12:36:16,999] jepsen worker 27 - jepsen.util 10107 :ok :read [9 37] INFO [2019-12-17 12:36:17,000] jepsen worker 27 - jepsen.util 10107 :invoke :write [9 9] INFO [2019-12-17 12:36:17,008] jepsen worker 18 - jepsen.util 10818 :fail :cas [6 [26 10]] :ValueNotSwapFrom INFO [2019-12-17 12:36:17,010] jepsen worker 18 - jepsen.util 10818 :invoke :cas [6 [46 46]] INFO [2019-12-17 12:36:17,018] jepsen worker 79 - jepsen.util 11059 :ok :read [26 12] INFO [2019-12-17 12:36:17,018] jepsen worker 79 - jepsen.util 11059 :invoke :cas [26 [37 40]] INFO [2019-12-17 12:36:17,032] jepsen worker 6 - jepsen.util 14676 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:36:17,032] jepsen worker 6 - jepsen.util 14676 :invoke :read [2 nil] INFO [2019-12-17 12:36:17,043] jepsen worker 65 - jepsen.util 9155 :ok :write [21 27] INFO [2019-12-17 12:36:17,045] jepsen worker 18 - jepsen.util 10818 :fail :cas [6 [46 46]] :ValueNotSwapFrom INFO [2019-12-17 12:36:17,048] jepsen worker 65 - jepsen.util 9155 :invoke :cas [21 [15 31]] INFO [2019-12-17 12:36:17,048] jepsen worker 18 - jepsen.util 10818 :invoke :read [6 nil] INFO [2019-12-17 12:36:17,052] jepsen worker 10 - jepsen.util 10540 :ok :write [3 49] INFO [2019-12-17 12:36:17,052] jepsen worker 10 - jepsen.util 10540 :invoke :write [3 3] INFO [2019-12-17 12:36:17,066] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":61250,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:36:17,066] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":61250,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:36:17,066] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":61254,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:36:17,066] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":61254,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:36:17,080] jepsen worker 79 - jepsen.util 11059 :fail :cas [26 [37 40]] :ValueNotSwapFrom INFO [2019-12-17 12:36:17,081] jepsen worker 79 - jepsen.util 11059 :invoke :cas [26 [38 44]] INFO [2019-12-17 12:36:17,128] jepsen worker 4 - jepsen.util 9724 :fail :cas [1 [34 18]] :ValueNotSwapFrom INFO [2019-12-17 12:36:17,138] jepsen worker 65 - jepsen.util 9155 :fail :cas [21 [15 31]] :ValueNotSwapFrom INFO [2019-12-17 12:36:17,142] jepsen worker 18 - jepsen.util 10818 :ok :read [6 39] INFO [2019-12-17 12:36:17,152] jepsen worker 4 - jepsen.util 9724 :invoke :write [1 6] INFO [2019-12-17 12:36:17,152] jepsen worker 65 - jepsen.util 9155 :invoke :write [21 49] INFO [2019-12-17 12:36:17,152] jepsen worker 18 - jepsen.util 10818 :invoke :read [6 nil] INFO [2019-12-17 12:36:17,162] jepsen worker 27 - jepsen.util 10107 :ok :write [9 9] INFO [2019-12-17 12:36:17,178] jepsen worker 18 - jepsen.util 10818 :ok :read [6 39] INFO [2019-12-17 12:36:17,184] jepsen worker 37 - jepsen.util 10207 :ok :read [12 32] INFO [2019-12-17 12:36:17,196] jepsen worker 27 - jepsen.util 10107 :invoke :read [9 nil] INFO [2019-12-17 12:36:17,196] jepsen worker 18 - jepsen.util 10818 :invoke :cas [6 [1 13]] INFO [2019-12-17 12:36:17,196] jepsen worker 37 - jepsen.util 10207 :invoke :cas [12 [35 15]] INFO [2019-12-17 12:36:17,198] jepsen worker 38 - jepsen.util 11198 :info :write [12 12] :RequestCanceledException INFO [2019-12-17 12:36:17,207] jepsen worker 38 - jepsen.util 11288 :invoke :cas [12 [30 2]] INFO [2019-12-17 12:36:17,227] jepsen worker 52 - jepsen.util 10402 :fail :cas [17 [23 4]] :ValueNotSwapFrom INFO [2019-12-17 12:36:17,227] jepsen worker 27 - jepsen.util 10107 :ok :read [9 9] INFO [2019-12-17 12:36:17,227] jepsen worker 52 - jepsen.util 10402 :invoke :write [17 28] INFO [2019-12-17 12:36:17,227] jepsen worker 27 - jepsen.util 10107 :invoke :cas [9 [22 4]] INFO [2019-12-17 12:36:17,232] jepsen worker 35 - jepsen.util 14435 :info :write [11 12] :RequestCanceledException INFO [2019-12-17 12:36:17,234] jepsen worker 35 - jepsen.util 14525 :invoke :read [11 nil] INFO [2019-12-17 12:36:17,236] jepsen worker 79 - jepsen.util 11059 :fail :cas [26 [38 44]] :ValueNotSwapFrom INFO [2019-12-17 12:36:17,236] jepsen worker 79 - jepsen.util 11059 :invoke :write [26 31] INFO [2019-12-17 12:36:17,250] jepsen worker 27 - jepsen.util 10107 :fail :cas [9 [22 4]] :ValueNotSwapFrom INFO [2019-12-17 12:36:17,250] jepsen worker 27 - jepsen.util 10107 :invoke :cas [9 [2 26]] INFO [2019-12-17 12:36:17,253] jepsen worker 78 - jepsen.util 10698 :fail :cas [26 [16 22]] :ValueNotSwapFrom INFO [2019-12-17 12:36:17,253] jepsen worker 78 - jepsen.util 10698 :invoke :write [26 12] INFO [2019-12-17 12:36:17,260] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":61270,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:36:17,260] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":61277,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:36:17,262] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":61270,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:36:17,262] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":61277,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:36:17,262] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":61271,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:36:17,262] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":61280,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:36:17,262] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":61271,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:36:17,262] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":61280,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:36:17,266] jepsen worker 65 - jepsen.util 9155 :ok :write [21 49] INFO [2019-12-17 12:36:17,266] jepsen worker 65 - jepsen.util 9155 :invoke :read [21 nil] INFO [2019-12-17 12:36:17,269] jepsen worker 18 - jepsen.util 10818 :fail :cas [6 [1 13]] :ValueNotSwapFrom INFO [2019-12-17 12:36:17,269] jepsen worker 18 - jepsen.util 10818 :invoke :cas [6 [16 11]] INFO [2019-12-17 12:36:17,278] jepsen worker 56 - jepsen.util 11036 :ok :write [18 29] INFO [2019-12-17 12:36:17,278] jepsen worker 56 - jepsen.util 11036 :invoke :cas [18 [42 31]] INFO [2019-12-17 12:36:17,284] jepsen worker 27 - jepsen.util 10107 :fail :cas [9 [2 26]] :ValueNotSwapFrom INFO [2019-12-17 12:36:17,284] jepsen worker 27 - jepsen.util 10107 :invoke :read [9 nil] INFO [2019-12-17 12:36:17,299] jepsen worker 44 - jepsen.util 14624 :info :write [14 6] :RequestCanceledException INFO [2019-12-17 12:36:17,302] jepsen worker 44 - jepsen.util 14714 :invoke :write [14 17] INFO [2019-12-17 12:36:17,305] jepsen worker 70 - jepsen.util 11320 :ok :read [23 5] INFO [2019-12-17 12:36:17,305] jepsen worker 70 - jepsen.util 11320 :invoke :write [23 19] INFO [2019-12-17 12:36:17,316] jepsen worker 62 - jepsen.util 14732 :info :cas [20 [40 35]] :RequestCanceledException INFO [2019-12-17 12:36:17,317] jepsen worker 62 - jepsen.util 14822 :invoke :cas [20 [13 39]] INFO [2019-12-17 12:36:17,318] jepsen worker 12 - jepsen.util 10992 :ok :write [4 36] INFO [2019-12-17 12:36:17,318] jepsen worker 12 - jepsen.util 10992 :invoke :read [4 nil] INFO [2019-12-17 12:36:17,342] jepsen worker 18 - jepsen.util 10818 :fail :cas [6 [16 11]] :ValueNotSwapFrom INFO [2019-12-17 12:36:17,342] jepsen worker 18 - jepsen.util 10818 :invoke :write [6 21] INFO [2019-12-17 12:36:17,345] jepsen worker 23 - jepsen.util 14423 :info :cas [7 [35 18]] :RequestCanceledException INFO [2019-12-17 12:36:17,345] jepsen worker 63 - jepsen.util 9513 :info :cas [21 [3 22]] :RequestCanceledException INFO [2019-12-17 12:36:17,346] jepsen worker 23 - jepsen.util 14513 :invoke :read [7 nil] INFO [2019-12-17 12:36:17,346] jepsen worker 63 - jepsen.util 9603 :invoke :write [21 39] INFO [2019-12-17 12:36:17,349] jepsen worker 53 - jepsen.util 9953 :ok :write [17 36] INFO [2019-12-17 12:36:17,349] jepsen worker 53 - jepsen.util 9953 :invoke :read [17 nil] INFO [2019-12-17 12:36:17,354] jepsen worker 65 - jepsen.util 9155 :ok :read [21 49] INFO [2019-12-17 12:36:17,354] jepsen worker 65 - jepsen.util 9155 :invoke :write [21 31] INFO [2019-12-17 12:36:17,366] jepsen worker 27 - jepsen.util 10107 :ok :read [9 9] INFO [2019-12-17 12:36:17,367] jepsen worker 27 - jepsen.util 10107 :invoke :cas [9 [28 28]] INFO [2019-12-17 12:36:17,388] jepsen worker 89 - jepsen.util 10889 :fail :cas [29 [42 41]] :ValueNotSwapFrom INFO [2019-12-17 12:36:17,394] jepsen worker 89 - jepsen.util 10889 :invoke :cas [29 [15 36]] INFO [2019-12-17 12:36:17,409] jepsen worker 27 - jepsen.util 10107 :fail :cas [9 [28 28]] :ValueNotSwapFrom INFO [2019-12-17 12:36:17,410] jepsen worker 27 - jepsen.util 10107 :invoke :write [9 38] INFO [2019-12-17 12:36:17,417] jepsen worker 85 - jepsen.util 15205 :info :write [28 17] :RequestCanceledException INFO [2019-12-17 12:36:17,418] jepsen worker 85 - jepsen.util 15295 :invoke :cas [28 [46 29]] INFO [2019-12-17 12:36:17,431] jepsen worker 40 - jepsen.util 14800 :info :write [13 49] :RequestCanceledException INFO [2019-12-17 12:36:17,434] jepsen worker 40 - jepsen.util 14890 :invoke :read [13 nil] INFO [2019-12-17 12:36:17,442] jepsen worker 10 - jepsen.util 10540 :ok :write [3 3] INFO [2019-12-17 12:36:17,442] jepsen worker 10 - jepsen.util 10540 :invoke :read [3 nil] INFO [2019-12-17 12:36:17,459] jepsen worker 51 - jepsen.util 10131 :ok :write [17 5] INFO [2019-12-17 12:36:17,460] jepsen worker 51 - jepsen.util 10131 :invoke :write [17 42] INFO [2019-12-17 12:36:17,470] jepsen worker 69 - jepsen.util 10689 :ok :write [23 16] INFO [2019-12-17 12:36:17,471] jepsen worker 69 - jepsen.util 10689 :invoke :cas [23 [25 16]] INFO [2019-12-17 12:36:17,471] jepsen worker 89 - jepsen.util 10889 :fail :cas [29 [15 36]] :ValueNotSwapFrom INFO [2019-12-17 12:36:17,471] jepsen worker 89 - jepsen.util 10889 :invoke :write [29 15] INFO [2019-12-17 12:36:17,473] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:36:17,473] jepsen nemesis - couchbase.util Rebalance check count: 478 INFO [2019-12-17 12:36:17,473] jepsen nemesis - couchbase.util Rebalance stuck count: 459 INFO [2019-12-17 12:36:17,479] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":61307,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:36:17,479] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":61307,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:36:17,492] jepsen worker 10 - jepsen.util 10540 :ok :read [3 3] INFO [2019-12-17 12:36:17,492] jepsen worker 10 - jepsen.util 10540 :invoke :read [3 nil] INFO [2019-12-17 12:36:17,494] jepsen worker 27 - jepsen.util 10107 :ok :write [9 38] INFO [2019-12-17 12:36:17,494] jepsen worker 27 - jepsen.util 10107 :invoke :read [9 nil] INFO [2019-12-17 12:36:17,507] jepsen worker 56 - jepsen.util 11036 :fail :cas [18 [42 31]] :ValueNotSwapFrom INFO [2019-12-17 12:36:17,508] jepsen worker 56 - jepsen.util 11036 :invoke :write [18 25] INFO [2019-12-17 12:36:17,516] jepsen worker 55 - jepsen.util 11125 :ok :write [18 47] INFO [2019-12-17 12:36:17,516] jepsen worker 55 - jepsen.util 11125 :invoke :cas [18 [16 11]] INFO [2019-12-17 12:36:17,532] jepsen worker 10 - jepsen.util 10540 :ok :read [3 3] INFO [2019-12-17 12:36:17,532] jepsen worker 66 - jepsen.util 14106 :info :cas [22 [19 24]] :RequestCanceledException INFO [2019-12-17 12:36:17,533] jepsen worker 10 - jepsen.util 10540 :invoke :read [3 nil] INFO [2019-12-17 12:36:17,534] jepsen worker 66 - jepsen.util 14196 :invoke :cas [22 [40 20]] INFO [2019-12-17 12:36:17,538] jepsen worker 27 - jepsen.util 10107 :ok :read [9 38] INFO [2019-12-17 12:36:17,538] jepsen worker 27 - jepsen.util 10107 :invoke :write [9 10] INFO [2019-12-17 12:36:17,545] jepsen worker 14 - jepsen.util 10454 :ok :read [4 36] INFO [2019-12-17 12:36:17,546] jepsen worker 14 - jepsen.util 10454 :invoke :write [4 29] INFO [2019-12-17 12:36:17,548] jepsen worker 10 - jepsen.util 10540 :ok :read [3 3] INFO [2019-12-17 12:36:17,548] jepsen worker 10 - jepsen.util 10540 :invoke :write [3 26] INFO [2019-12-17 12:36:17,581] jepsen worker 65 - jepsen.util 9155 :ok :write [21 31] INFO [2019-12-17 12:36:17,581] jepsen worker 65 - jepsen.util 9155 :invoke :read [21 nil] INFO [2019-12-17 12:36:17,599] jepsen worker 49 - jepsen.util 14269 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:36:17,599] jepsen worker 83 - jepsen.util 14573 :info :cas [27 [34 27]] :RequestCanceledException INFO [2019-12-17 12:36:17,599] jepsen worker 49 - jepsen.util 14269 :invoke :cas [16 [29 29]] INFO [2019-12-17 12:36:17,600] jepsen worker 83 - jepsen.util 14663 :invoke :read [27 nil] INFO [2019-12-17 12:36:17,617] jepsen worker 7 - jepsen.util 14497 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:36:17,617] jepsen worker 7 - jepsen.util 14497 :invoke :read [2 nil] INFO [2019-12-17 12:36:17,618] jepsen worker 18 - jepsen.util 10818 :ok :write [6 21] INFO [2019-12-17 12:36:17,619] jepsen worker 18 - jepsen.util 10818 :invoke :cas [6 [48 44]] INFO [2019-12-17 12:36:17,624] jepsen worker 65 - jepsen.util 9155 :ok :read [21 31] INFO [2019-12-17 12:36:17,627] jepsen worker 65 - jepsen.util 9155 :invoke :read [21 nil] INFO [2019-12-17 12:36:17,630] jepsen worker 69 - jepsen.util 10689 :fail :cas [23 [25 16]] :ValueNotSwapFrom INFO [2019-12-17 12:36:17,631] jepsen worker 69 - jepsen.util 10689 :invoke :write [23 27] INFO [2019-12-17 12:36:17,631] jepsen worker 80 - jepsen.util 10340 :info :write [26 43] :RequestCanceledException INFO [2019-12-17 12:36:17,632] jepsen worker 88 - jepsen.util 10978 :ok :write [29 8] INFO [2019-12-17 12:36:17,632] jepsen worker 88 - jepsen.util 10978 :invoke :cas [29 [1 1]] INFO [2019-12-17 12:36:17,632] jepsen worker 80 - jepsen.util 10430 :invoke :write [26 36] INFO [2019-12-17 12:36:17,641] jepsen worker 78 - jepsen.util 10698 :ok :write [26 12] INFO [2019-12-17 12:36:17,641] jepsen worker 78 - jepsen.util 10698 :invoke :write [26 18] INFO [2019-12-17 12:36:17,645] jepsen worker 18 - jepsen.util 10818 :fail :cas [6 [48 44]] :ValueNotSwapFrom INFO [2019-12-17 12:36:17,646] jepsen worker 18 - jepsen.util 10818 :invoke :cas [6 [36 41]] INFO [2019-12-17 12:36:17,646] jepsen worker 0 - jepsen.util 15120 :info :cas [0 [15 30]] :RequestCanceledException INFO [2019-12-17 12:36:17,648] jepsen worker 0 - jepsen.util 15210 :invoke :cas [0 [43 44]] INFO [2019-12-17 12:36:17,655] jepsen worker 65 - jepsen.util 9155 :ok :read [21 31] INFO [2019-12-17 12:36:17,655] jepsen worker 65 - jepsen.util 9155 :invoke :read [21 nil] INFO [2019-12-17 12:36:17,663] jepsen worker 61 - jepsen.util 15181 :info :write [20 40] :RequestCanceledException INFO [2019-12-17 12:36:17,663] jepsen worker 74 - jepsen.util 10424 :info :cas [24 [1 36]] :RequestCanceledException INFO [2019-12-17 12:36:17,664] jepsen worker 74 - jepsen.util 10514 :invoke :write [24 1] INFO [2019-12-17 12:36:17,664] jepsen worker 61 - jepsen.util 15271 :invoke :write [20 22] INFO [2019-12-17 12:36:17,675] jepsen worker 89 - jepsen.util 10889 :ok :write [29 15] INFO [2019-12-17 12:36:17,675] jepsen worker 89 - jepsen.util 10889 :invoke :cas [29 [37 35]] INFO [2019-12-17 12:36:17,689] jepsen worker 18 - jepsen.util 10818 :fail :cas [6 [36 41]] :ValueNotSwapFrom INFO [2019-12-17 12:36:17,689] jepsen worker 18 - jepsen.util 10818 :invoke :write [6 42] INFO [2019-12-17 12:36:17,705] jepsen worker 36 - jepsen.util 11466 :ok :write [12 27] INFO [2019-12-17 12:36:17,705] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":61337,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:36:17,706] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":61337,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:36:17,706] jepsen worker 36 - jepsen.util 11466 :invoke :cas [12 [37 22]] INFO [2019-12-17 12:36:17,707] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":61340,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:36:17,707] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":61340,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:36:17,710] jepsen worker 27 - jepsen.util 10107 :ok :write [9 10] INFO [2019-12-17 12:36:17,710] jepsen worker 27 - jepsen.util 10107 :invoke :cas [9 [33 3]] INFO [2019-12-17 12:36:17,713] jepsen worker 65 - jepsen.util 9155 :ok :read [21 31] INFO [2019-12-17 12:36:17,713] jepsen worker 65 - jepsen.util 9155 :invoke :cas [21 [29 15]] INFO [2019-12-17 12:36:17,726] jepsen worker 3 - jepsen.util 10353 :ok :write [1 49] INFO [2019-12-17 12:36:17,726] jepsen worker 3 - jepsen.util 10353 :invoke :read [1 nil] INFO [2019-12-17 12:36:17,732] jepsen worker 26 - jepsen.util 14966 :info :write [8 29] :RequestCanceledException INFO [2019-12-17 12:36:17,733] jepsen worker 26 - jepsen.util 15056 :invoke :cas [8 [8 33]] INFO [2019-12-17 12:36:17,747] jepsen worker 27 - jepsen.util 10107 :fail :cas [9 [33 3]] :ValueNotSwapFrom INFO [2019-12-17 12:36:17,747] jepsen worker 27 - jepsen.util 10107 :invoke :read [9 nil] INFO [2019-12-17 12:36:17,759] jepsen worker 4 - jepsen.util 9724 :ok :write [1 6] INFO [2019-12-17 12:36:17,759] jepsen worker 4 - jepsen.util 9724 :invoke :cas [1 [41 36]] INFO [2019-12-17 12:36:17,762] jepsen worker 1 - jepsen.util 14401 :info :write [0 4] :RequestCanceledException INFO [2019-12-17 12:36:17,765] jepsen worker 1 - jepsen.util 14491 :invoke :cas [0 [46 48]] INFO [2019-12-17 12:36:17,775] jepsen worker 65 - jepsen.util 9155 :fail :cas [21 [29 15]] :ValueNotSwapFrom INFO [2019-12-17 12:36:17,775] jepsen worker 65 - jepsen.util 9155 :invoke :cas [21 [27 19]] INFO [2019-12-17 12:36:17,796] jepsen worker 37 - jepsen.util 10207 :fail :cas [12 [35 15]] :ValueNotSwapFrom INFO [2019-12-17 12:36:17,797] jepsen worker 37 - jepsen.util 10207 :invoke :read [12 nil] INFO [2019-12-17 12:36:17,806] jepsen worker 55 - jepsen.util 11125 :fail :cas [18 [16 11]] :ValueNotSwapFrom INFO [2019-12-17 12:36:17,806] jepsen worker 55 - jepsen.util 11125 :invoke :cas [18 [45 39]] INFO [2019-12-17 12:36:17,811] jepsen worker 27 - jepsen.util 10107 :ok :read [9 10] INFO [2019-12-17 12:36:17,811] jepsen worker 27 - jepsen.util 10107 :invoke :read [9 nil] INFO [2019-12-17 12:36:17,816] jepsen worker 73 - jepsen.util 9973 :fail :read [24 nil] :RequestTimeoutException INFO [2019-12-17 12:36:17,816] jepsen worker 73 - jepsen.util 9973 :invoke :write [24 5] INFO [2019-12-17 12:36:17,841] jepsen worker 52 - jepsen.util 10402 :ok :write [17 28] INFO [2019-12-17 12:36:17,841] jepsen worker 52 - jepsen.util 10402 :invoke :cas [17 [33 44]] INFO [2019-12-17 12:36:17,896] jepsen worker 75 - jepsen.util 13935 :info :cas [25 [4 39]] :RequestCanceledException INFO [2019-12-17 12:36:17,897] jepsen worker 75 - jepsen.util 14025 :invoke :read [25 nil] INFO [2019-12-17 12:36:17,899] jepsen worker 27 - jepsen.util 10107 :ok :read [9 10] INFO [2019-12-17 12:36:17,900] jepsen worker 27 - jepsen.util 10107 :invoke :read [9 nil] INFO [2019-12-17 12:36:17,901] jepsen worker 65 - jepsen.util 9155 :fail :cas [21 [27 19]] :ValueNotSwapFrom INFO [2019-12-17 12:36:17,901] jepsen worker 65 - jepsen.util 9155 :invoke :read [21 nil] INFO [2019-12-17 12:36:17,916] jepsen worker 20 - jepsen.util 10460 :info :cas [6 [5 37]] :RequestCanceledException INFO [2019-12-17 12:36:17,917] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":61363,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:36:17,920] jepsen worker 38 - jepsen.util 11288 :fail :cas [12 [30 2]] :ValueNotSwapFrom INFO [2019-12-17 12:36:17,920] jepsen worker 65 - jepsen.util 9155 :ok :read [21 31] INFO [2019-12-17 12:36:17,923] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":61363,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:36:17,923] jepsen worker 38 - jepsen.util 11288 :invoke :read [12 nil] INFO [2019-12-17 12:36:17,923] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":61367,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:36:17,923] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":61367,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:36:17,923] jepsen worker 65 - jepsen.util 9155 :invoke :cas [21 [39 39]] INFO [2019-12-17 12:36:17,928] jepsen worker 20 - jepsen.util 10550 :invoke :cas [6 [20 8]] INFO [2019-12-17 12:36:17,932] jepsen worker 58 - jepsen.util 14728 :info :write [19 39] :RequestCanceledException INFO [2019-12-17 12:36:17,933] jepsen worker 58 - jepsen.util 14818 :invoke :cas [19 [38 36]] INFO [2019-12-17 12:36:17,937] jepsen worker 4 - jepsen.util 9724 :fail :cas [1 [41 36]] :ValueNotSwapFrom INFO [2019-12-17 12:36:17,938] jepsen worker 4 - jepsen.util 9724 :invoke :write [1 14] INFO [2019-12-17 12:36:17,947] jepsen worker 65 - jepsen.util 9155 :fail :cas [21 [39 39]] :ValueNotSwapFrom INFO [2019-12-17 12:36:17,947] jepsen worker 65 - jepsen.util 9155 :invoke :read [21 nil] INFO [2019-12-17 12:36:17,965] jepsen worker 65 - jepsen.util 9155 :ok :read [21 31] INFO [2019-12-17 12:36:17,965] jepsen worker 65 - jepsen.util 9155 :invoke :write [21 25] INFO [2019-12-17 12:36:18,006] jepsen worker 37 - jepsen.util 10207 :ok :read [12 27] INFO [2019-12-17 12:36:18,006] jepsen worker 37 - jepsen.util 10207 :invoke :write [12 1] INFO [2019-12-17 12:36:18,019] jepsen worker 10 - jepsen.util 10540 :ok :write [3 26] INFO [2019-12-17 12:36:18,019] jepsen worker 10 - jepsen.util 10540 :invoke :cas [3 [17 2]] INFO [2019-12-17 12:36:18,020] jepsen worker 52 - jepsen.util 10402 :fail :cas [17 [33 44]] :ValueNotSwapFrom INFO [2019-12-17 12:36:18,025] jepsen worker 52 - jepsen.util 10402 :invoke :write [17 6] INFO [2019-12-17 12:36:18,031] jepsen worker 64 - jepsen.util 8164 :info :write [21 33] :RequestCanceledException INFO [2019-12-17 12:36:18,031] jepsen worker 25 - jepsen.util 15505 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:36:18,031] jepsen worker 25 - jepsen.util 15505 :invoke :write [8 35] INFO [2019-12-17 12:36:18,031] jepsen worker 12 - jepsen.util 10992 :ok :read [4 36] INFO [2019-12-17 12:36:18,032] jepsen worker 12 - jepsen.util 10992 :invoke :read [4 nil] INFO [2019-12-17 12:36:18,032] jepsen worker 64 - jepsen.util 8254 :invoke :cas [21 [10 38]] INFO [2019-12-17 12:36:18,032] jepsen worker 65 - jepsen.util 9155 :ok :write [21 25] INFO [2019-12-17 12:36:18,032] jepsen worker 65 - jepsen.util 9155 :invoke :write [21 1] INFO [2019-12-17 12:36:18,045] jepsen worker 5 - jepsen.util 10085 :info :write [1 10] :RequestCanceledException INFO [2019-12-17 12:36:18,045] jepsen worker 11 - jepsen.util 10091 :info :cas [3 [6 31]] :RequestCanceledException INFO [2019-12-17 12:36:18,045] jepsen worker 68 - jepsen.util 13748 :info :write [22 11] :RequestCanceledException INFO [2019-12-17 12:36:18,045] jepsen worker 82 - jepsen.util 14572 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:36:18,048] jepsen worker 82 - jepsen.util 14572 :invoke :cas [27 [27 41]] INFO [2019-12-17 12:36:18,049] jepsen worker 5 - jepsen.util 10175 :invoke :write [1 45] INFO [2019-12-17 12:36:18,049] jepsen worker 68 - jepsen.util 13838 :invoke :cas [22 [9 33]] INFO [2019-12-17 12:36:18,050] jepsen worker 11 - jepsen.util 10181 :invoke :write [3 46] INFO [2019-12-17 12:36:18,053] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":61382,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:36:18,053] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":61382,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:36:18,053] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":61383,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:36:18,053] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":61383,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:36:18,058] jepsen worker 70 - jepsen.util 11320 :ok :write [23 19] INFO [2019-12-17 12:36:18,058] jepsen worker 70 - jepsen.util 11320 :invoke :read [23 nil] INFO [2019-12-17 12:36:18,083] jepsen worker 64 - jepsen.util 8254 :fail :cas [21 [10 38]] :ValueNotSwapFrom INFO [2019-12-17 12:36:18,084] jepsen worker 64 - jepsen.util 8254 :invoke :read [21 nil] INFO [2019-12-17 12:36:18,105] jepsen worker 54 - jepsen.util 11484 :ok :write [18 43] INFO [2019-12-17 12:36:18,106] jepsen worker 54 - jepsen.util 11484 :invoke :write [18 37] INFO [2019-12-17 12:36:18,107] jepsen worker 64 - jepsen.util 8254 :ok :read [21 25] INFO [2019-12-17 12:36:18,107] jepsen worker 64 - jepsen.util 8254 :invoke :cas [21 [24 44]] INFO [2019-12-17 12:36:18,141] jepsen worker 64 - jepsen.util 8254 :fail :cas [21 [24 44]] :ValueNotSwapFrom INFO [2019-12-17 12:36:18,141] jepsen worker 64 - jepsen.util 8254 :invoke :cas [21 [23 21]] INFO [2019-12-17 12:36:18,146] jepsen worker 13 - jepsen.util 10003 :info :write [4 25] :RequestCanceledException INFO [2019-12-17 12:36:18,150] jepsen worker 55 - jepsen.util 11125 :fail :cas [18 [45 39]] :ValueNotSwapFrom INFO [2019-12-17 12:36:18,150] jepsen worker 55 - jepsen.util 11125 :invoke :write [18 30] INFO [2019-12-17 12:36:18,156] jepsen worker 13 - jepsen.util 10093 :invoke :write [4 28] INFO [2019-12-17 12:36:18,163] jepsen worker 73 - jepsen.util 9973 :ok :write [24 5] INFO [2019-12-17 12:36:18,163] jepsen worker 73 - jepsen.util 9973 :invoke :cas [24 [2 12]] INFO [2019-12-17 12:36:18,166] jepsen worker 48 - jepsen.util 13368 :info :write [16 31] :RequestCanceledException INFO [2019-12-17 12:36:18,167] jepsen worker 48 - jepsen.util 13458 :invoke :write [16 3] INFO [2019-12-17 12:36:18,169] jepsen worker 89 - jepsen.util 10889 :fail :cas [29 [37 35]] :ValueNotSwapFrom INFO [2019-12-17 12:36:18,170] jepsen worker 89 - jepsen.util 10889 :invoke :cas [29 [4 4]] INFO [2019-12-17 12:36:18,180] jepsen worker 3 - jepsen.util 10353 :ok :read [1 6] INFO [2019-12-17 12:36:18,180] jepsen worker 3 - jepsen.util 10353 :invoke :read [1 nil] INFO [2019-12-17 12:36:18,186] jepsen worker 18 - jepsen.util 10818 :ok :write [6 42] INFO [2019-12-17 12:36:18,186] jepsen worker 18 - jepsen.util 10818 :invoke :cas [6 [43 33]] INFO [2019-12-17 12:36:18,188] jepsen worker 88 - jepsen.util 10978 :fail :cas [29 [1 1]] :ValueNotSwapFrom INFO [2019-12-17 12:36:18,188] jepsen worker 88 - jepsen.util 10978 :invoke :cas [29 [40 12]] INFO [2019-12-17 12:36:18,192] jepsen worker 53 - jepsen.util 9953 :ok :read [17 28] INFO [2019-12-17 12:36:18,192] jepsen worker 53 - jepsen.util 9953 :invoke :write [17 23] INFO [2019-12-17 12:36:18,202] jepsen worker 36 - jepsen.util 11466 :fail :cas [12 [37 22]] :ValueNotSwapFrom INFO [2019-12-17 12:36:18,202] jepsen worker 36 - jepsen.util 11466 :invoke :cas [12 [38 27]] INFO [2019-12-17 12:36:18,216] jepsen worker 9 - jepsen.util 11259 :info :cas [3 [14 28]] :RequestCanceledException INFO [2019-12-17 12:36:18,217] jepsen worker 9 - jepsen.util 11349 :invoke :cas [3 [44 11]] INFO [2019-12-17 12:36:18,228] jepsen worker 89 - jepsen.util 10889 :fail :cas [29 [4 4]] :ValueNotSwapFrom INFO [2019-12-17 12:36:18,229] jepsen worker 89 - jepsen.util 10889 :invoke :write [29 20] INFO [2019-12-17 12:36:18,232] jepsen worker 87 - jepsen.util 10077 :info :write [29 47] :RequestCanceledException INFO [2019-12-17 12:36:18,232] jepsen worker 30 - jepsen.util 10110 :info :cas [10 [42 32]] :RequestCanceledException INFO [2019-12-17 12:36:18,232] jepsen worker 33 - jepsen.util 14523 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:36:18,232] jepsen worker 47 - jepsen.util 11207 :info :cas [15 [48 41]] :RequestCanceledException INFO [2019-12-17 12:36:18,233] jepsen worker 33 - jepsen.util 14523 :invoke :read [11 nil] INFO [2019-12-17 12:36:18,233] jepsen worker 47 - jepsen.util 11297 :invoke :cas [15 [10 15]] INFO [2019-12-17 12:36:18,233] jepsen worker 87 - jepsen.util 10167 :invoke :write [29 22] INFO [2019-12-17 12:36:18,234] jepsen worker 30 - jepsen.util 10200 :invoke :cas [10 [12 32]] INFO [2019-12-17 12:36:18,244] jepsen worker 64 - jepsen.util 8254 :fail :cas [21 [23 21]] :ValueNotSwapFrom INFO [2019-12-17 12:36:18,244] jepsen worker 64 - jepsen.util 8254 :invoke :cas [21 [15 44]] INFO [2019-12-17 12:36:18,245] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":61409,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:36:18,245] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":61417,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:36:18,245] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":61409,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:36:18,245] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":61417,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:36:18,245] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":61411,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:36:18,245] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":61418,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:36:18,245] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":61411,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:36:18,246] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":61418,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:36:18,246] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":61419,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:36:18,246] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":61419,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:36:18,263] jepsen worker 12 - jepsen.util 10992 :ok :read [4 36] INFO [2019-12-17 12:36:18,263] jepsen worker 12 - jepsen.util 10992 :invoke :read [4 nil] INFO [2019-12-17 12:36:18,276] jepsen worker 20 - jepsen.util 10550 :fail :cas [6 [20 8]] :ValueNotSwapFrom INFO [2019-12-17 12:36:18,276] jepsen worker 20 - jepsen.util 10550 :invoke :write [6 44] INFO [2019-12-17 12:36:18,293] jepsen worker 27 - jepsen.util 10107 :ok :read [9 10] INFO [2019-12-17 12:36:18,293] jepsen worker 27 - jepsen.util 10107 :invoke :read [9 nil] INFO [2019-12-17 12:36:18,310] jepsen worker 9 - jepsen.util 11349 :fail :cas [3 [44 11]] :ValueNotSwapFrom INFO [2019-12-17 12:36:18,312] jepsen worker 9 - jepsen.util 11349 :invoke :write [3 9] INFO [2019-12-17 12:36:18,317] jepsen worker 81 - jepsen.util 14031 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:36:18,317] jepsen worker 81 - jepsen.util 14031 :invoke :cas [27 [30 2]] INFO [2019-12-17 12:36:18,321] jepsen worker 18 - jepsen.util 10818 :fail :cas [6 [43 33]] :ValueNotSwapFrom INFO [2019-12-17 12:36:18,322] jepsen worker 18 - jepsen.util 10818 :invoke :read [6 nil] INFO [2019-12-17 12:36:18,336] jepsen worker 65 - jepsen.util 9155 :ok :write [21 1] INFO [2019-12-17 12:36:18,336] jepsen worker 65 - jepsen.util 9155 :invoke :write [21 4] INFO [2019-12-17 12:36:18,339] jepsen worker 47 - jepsen.util 11297 :fail :cas [15 [10 15]] :ValueNotSwapFrom INFO [2019-12-17 12:36:18,339] jepsen worker 47 - jepsen.util 11297 :invoke :read [15 nil] INFO [2019-12-17 12:36:18,344] jepsen worker 64 - jepsen.util 8254 :fail :cas [21 [15 44]] :ValueNotSwapFrom INFO [2019-12-17 12:36:18,344] jepsen worker 64 - jepsen.util 8254 :invoke :read [21 nil] INFO [2019-12-17 12:36:18,358] jepsen worker 47 - jepsen.util 11297 :ok :read [15 27] INFO [2019-12-17 12:36:18,359] jepsen worker 47 - jepsen.util 11297 :invoke :write [15 10] INFO [2019-12-17 12:36:18,401] jepsen worker 36 - jepsen.util 11466 :fail :cas [12 [38 27]] :ValueNotSwapFrom INFO [2019-12-17 12:36:18,401] jepsen worker 36 - jepsen.util 11466 :invoke :write [12 18] INFO [2019-12-17 12:36:18,416] jepsen worker 77 - jepsen.util 15107 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:36:18,417] jepsen worker 77 - jepsen.util 15107 :invoke :cas [25 [33 17]] INFO [2019-12-17 12:36:18,421] jepsen worker 70 - jepsen.util 11320 :ok :read [23 19] INFO [2019-12-17 12:36:18,421] jepsen worker 70 - jepsen.util 11320 :invoke :cas [23 [28 3]] INFO [2019-12-17 12:36:18,423] jepsen worker 38 - jepsen.util 11288 :ok :read [12 27] INFO [2019-12-17 12:36:18,423] jepsen worker 38 - jepsen.util 11288 :invoke :read [12 nil] INFO [2019-12-17 12:36:18,427] jepsen worker 65 - jepsen.util 9155 :ok :write [21 4] INFO [2019-12-17 12:36:18,428] jepsen worker 65 - jepsen.util 9155 :invoke :read [21 nil] INFO [2019-12-17 12:36:18,432] jepsen worker 39 - jepsen.util 13989 :info :write [13 1] :RequestCanceledException INFO [2019-12-17 12:36:18,432] jepsen worker 34 - jepsen.util 14704 :info :cas [11 [23 42]] :RequestCanceledException INFO [2019-12-17 12:36:18,432] jepsen worker 28 - jepsen.util 8758 :info :write [9 18] :RequestCanceledException INFO [2019-12-17 12:36:18,433] jepsen worker 34 - jepsen.util 14794 :invoke :read [11 nil] INFO [2019-12-17 12:36:18,433] jepsen worker 28 - jepsen.util 8848 :invoke :read [9 nil] INFO [2019-12-17 12:36:18,433] jepsen worker 39 - jepsen.util 14079 :invoke :write [13 27] INFO [2019-12-17 12:36:18,444] jepsen worker 88 - jepsen.util 10978 :fail :cas [29 [40 12]] :ValueNotSwapFrom INFO [2019-12-17 12:36:18,444] jepsen worker 88 - jepsen.util 10978 :invoke :write [29 18] INFO [2019-12-17 12:36:18,445] jepsen worker 65 - jepsen.util 9155 :ok :read [21 4] INFO [2019-12-17 12:36:18,446] jepsen worker 65 - jepsen.util 9155 :invoke :write [21 8] INFO [2019-12-17 12:36:18,446] jepsen worker 59 - jepsen.util 13559 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:36:18,446] jepsen worker 59 - jepsen.util 13559 :invoke :write [19 4] INFO [2019-12-17 12:36:18,448] jepsen worker 28 - jepsen.util 8848 :ok :read [9 10] INFO [2019-12-17 12:36:18,448] jepsen worker 28 - jepsen.util 8848 :invoke :write [9 31] INFO [2019-12-17 12:36:18,463] jepsen worker 29 - jepsen.util 8939 :info :write [9 17] :RequestCanceledException INFO [2019-12-17 12:36:18,463] jepsen worker 46 - jepsen.util 10396 :fail :read [15 nil] :RequestTimeoutException INFO [2019-12-17 12:36:18,463] jepsen worker 72 - jepsen.util 10782 :fail :read [24 nil] :RequestTimeoutException INFO [2019-12-17 12:36:18,463] jepsen worker 71 - jepsen.util 11501 :info :write [23 10] :RequestCanceledException INFO [2019-12-17 12:36:18,463] jepsen worker 46 - jepsen.util 10396 :invoke :read [15 nil] INFO [2019-12-17 12:36:18,464] jepsen worker 72 - jepsen.util 10782 :invoke :cas [24 [20 9]] INFO [2019-12-17 12:36:18,464] jepsen worker 29 - jepsen.util 9029 :invoke :read [9 nil] INFO [2019-12-17 12:36:18,464] jepsen worker 71 - jepsen.util 11591 :invoke :cas [23 [23 38]] INFO [2019-12-17 12:36:18,465] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":61447,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:36:18,465] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":61447,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:36:18,465] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":61450,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:36:18,465] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":61450,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:36:18,465] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":61451,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:36:18,465] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":61451,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:36:18,480] jepsen worker 46 - jepsen.util 10396 :ok :read [15 27] INFO [2019-12-17 12:36:18,480] jepsen worker 46 - jepsen.util 10396 :invoke :write [15 46] INFO [2019-12-17 12:36:18,491] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:36:18,491] jepsen nemesis - couchbase.util Rebalance check count: 479 INFO [2019-12-17 12:36:18,491] jepsen nemesis - couchbase.util Rebalance stuck count: 460 INFO [2019-12-17 12:36:18,498] jepsen worker 8 - jepsen.util 14138 :info :write [2 16] :RequestCanceledException INFO [2019-12-17 12:36:18,499] jepsen worker 8 - jepsen.util 14228 :invoke :cas [2 [0 14]] INFO [2019-12-17 12:36:18,510] jepsen worker 11 - jepsen.util 10181 :ok :write [3 46] INFO [2019-12-17 12:36:18,510] jepsen worker 11 - jepsen.util 10181 :invoke :cas [3 [11 37]] INFO [2019-12-17 12:36:18,512] jepsen worker 72 - jepsen.util 10782 :fail :cas [24 [20 9]] :ValueNotSwapFrom INFO [2019-12-17 12:36:18,512] jepsen worker 72 - jepsen.util 10782 :invoke :read [24 nil] INFO [2019-12-17 12:36:18,516] jepsen worker 16 - jepsen.util 15856 :info :cas [5 [37 17]] :RequestCanceledException INFO [2019-12-17 12:36:18,518] jepsen worker 16 - jepsen.util 15946 :invoke :read [5 nil] INFO [2019-12-17 12:36:18,531] jepsen worker 27 - jepsen.util 10107 :ok :read [9 10] INFO [2019-12-17 12:36:18,531] jepsen worker 27 - jepsen.util 10107 :invoke :read [9 nil] INFO [2019-12-17 12:36:18,531] jepsen worker 21 - jepsen.util 13971 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:36:18,531] jepsen worker 32 - jepsen.util 10652 :info :write [10 5] :RequestCanceledException INFO [2019-12-17 12:36:18,531] jepsen worker 21 - jepsen.util 13971 :invoke :write [7 3] INFO [2019-12-17 12:36:18,533] jepsen worker 32 - jepsen.util 10742 :invoke :read [10 nil] INFO [2019-12-17 12:36:18,550] jepsen worker 11 - jepsen.util 10181 :fail :cas [3 [11 37]] :ValueNotSwapFrom INFO [2019-12-17 12:36:18,550] jepsen worker 11 - jepsen.util 10181 :invoke :cas [3 [48 36]] INFO [2019-12-17 12:36:18,559] jepsen worker 28 - jepsen.util 8848 :ok :write [9 31] INFO [2019-12-17 12:36:18,559] jepsen worker 28 - jepsen.util 8848 :invoke :write [9 8] INFO [2019-12-17 12:36:18,559] jepsen worker 27 - jepsen.util 10107 :ok :read [9 10] INFO [2019-12-17 12:36:18,559] jepsen worker 27 - jepsen.util 10107 :invoke :cas [9 [35 17]] INFO [2019-12-17 12:36:18,572] jepsen worker 3 - jepsen.util 10353 :ok :read [1 6] INFO [2019-12-17 12:36:18,572] jepsen worker 3 - jepsen.util 10353 :invoke :read [1 nil] INFO [2019-12-17 12:36:18,590] jepsen worker 11 - jepsen.util 10181 :fail :cas [3 [48 36]] :ValueNotSwapFrom INFO [2019-12-17 12:36:18,590] jepsen worker 32 - jepsen.util 10742 :ok :read [10 24] INFO [2019-12-17 12:36:18,590] jepsen worker 11 - jepsen.util 10181 :invoke :cas [3 [10 37]] INFO [2019-12-17 12:36:18,590] jepsen worker 32 - jepsen.util 10742 :invoke :write [10 38] INFO [2019-12-17 12:36:18,598] jepsen worker 86 - jepsen.util 15476 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:36:18,598] jepsen worker 31 - jepsen.util 9211 :info :write [10 6] :RequestCanceledException INFO [2019-12-17 12:36:18,599] jepsen worker 86 - jepsen.util 15476 :invoke :read [28 nil] INFO [2019-12-17 12:36:18,599] jepsen worker 31 - jepsen.util 9301 :invoke :cas [10 [34 49]] INFO [2019-12-17 12:36:18,612] jepsen worker 65 - jepsen.util 9155 :ok :write [21 8] INFO [2019-12-17 12:36:18,612] jepsen worker 65 - jepsen.util 9155 :invoke :read [21 nil] INFO [2019-12-17 12:36:18,615] jepsen worker 45 - jepsen.util 10665 :info :cas [15 [12 49]] :RequestCanceledException INFO [2019-12-17 12:36:18,617] jepsen worker 45 - jepsen.util 10755 :invoke :write [15 38] INFO [2019-12-17 12:36:18,618] jepsen worker 31 - jepsen.util 9301 :fail :cas [10 [34 49]] :ValueNotSwapFrom INFO [2019-12-17 12:36:18,618] jepsen worker 31 - jepsen.util 9301 :invoke :read [10 nil] INFO [2019-12-17 12:36:18,632] jepsen worker 43 - jepsen.util 14443 :info :cas [14 [40 49]] :RequestCanceledException INFO [2019-12-17 12:36:18,632] jepsen worker 84 - jepsen.util 14034 :info :write [28 41] :RequestCanceledException INFO [2019-12-17 12:36:18,633] jepsen worker 84 - jepsen.util 14124 :invoke :write [28 7] INFO [2019-12-17 12:36:18,633] jepsen worker 43 - jepsen.util 14533 :invoke :read [14 nil] INFO [2019-12-17 12:36:18,646] jepsen worker 22 - jepsen.util 14242 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:36:18,646] jepsen worker 22 - jepsen.util 14242 :invoke :write [7 26] INFO [2019-12-17 12:36:18,652] jepsen worker 12 - jepsen.util 10992 :ok :read [4 36] INFO [2019-12-17 12:36:18,652] jepsen worker 12 - jepsen.util 10992 :invoke :cas [4 [23 19]] INFO [2019-12-17 12:36:18,655] jepsen worker 27 - jepsen.util 10107 :fail :cas [9 [35 17]] :ValueNotSwapFrom INFO [2019-12-17 12:36:18,655] jepsen worker 27 - jepsen.util 10107 :invoke :cas [9 [35 0]] INFO [2019-12-17 12:36:18,659] jepsen worker 65 - jepsen.util 9155 :ok :read [21 8] INFO [2019-12-17 12:36:18,659] jepsen worker 65 - jepsen.util 9155 :invoke :read [21 nil] INFO [2019-12-17 12:36:18,663] jepsen worker 67 - jepsen.util 14647 :info :cas [22 [39 42]] :RequestCanceledException INFO [2019-12-17 12:36:18,664] jepsen worker 67 - jepsen.util 14737 :invoke :write [22 23] INFO [2019-12-17 12:36:18,670] jepsen worker 38 - jepsen.util 11288 :ok :read [12 27] INFO [2019-12-17 12:36:18,671] jepsen worker 38 - jepsen.util 11288 :invoke :write [12 47] INFO [2019-12-17 12:36:18,673] jepsen worker 31 - jepsen.util 9301 :ok :read [10 24] INFO [2019-12-17 12:36:18,673] jepsen worker 31 - jepsen.util 9301 :invoke :write [10 7] INFO [2019-12-17 12:36:18,687] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":61483,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:36:18,688] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":61483,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:36:18,688] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":61487,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:36:18,688] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":61487,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:36:18,692] jepsen worker 11 - jepsen.util 10181 :fail :cas [3 [10 37]] :ValueNotSwapFrom INFO [2019-12-17 12:36:18,692] jepsen worker 11 - jepsen.util 10181 :invoke :cas [3 [18 46]] INFO [2019-12-17 12:36:18,713] jepsen worker 65 - jepsen.util 9155 :ok :read [21 8] INFO [2019-12-17 12:36:18,713] jepsen worker 65 - jepsen.util 9155 :invoke :cas [21 [43 49]] INFO [2019-12-17 12:36:18,720] jepsen worker 3 - jepsen.util 10353 :ok :read [1 6] INFO [2019-12-17 12:36:18,721] jepsen worker 3 - jepsen.util 10353 :invoke :cas [1 [33 3]] INFO [2019-12-17 12:36:18,724] jepsen worker 27 - jepsen.util 10107 :fail :cas [9 [35 0]] :ValueNotSwapFrom INFO [2019-12-17 12:36:18,724] jepsen worker 27 - jepsen.util 10107 :invoke :cas [9 [25 45]] INFO [2019-12-17 12:36:18,739] jepsen worker 65 - jepsen.util 9155 :fail :cas [21 [43 49]] :ValueNotSwapFrom INFO [2019-12-17 12:36:18,739] jepsen worker 65 - jepsen.util 9155 :invoke :read [21 nil] INFO [2019-12-17 12:36:18,741] jepsen worker 11 - jepsen.util 10181 :fail :cas [3 [18 46]] :ValueNotSwapFrom INFO [2019-12-17 12:36:18,741] jepsen worker 11 - jepsen.util 10181 :invoke :read [3 nil] INFO [2019-12-17 12:36:18,744] jepsen worker 28 - jepsen.util 8848 :ok :write [9 8] INFO [2019-12-17 12:36:18,744] jepsen worker 28 - jepsen.util 8848 :invoke :cas [9 [11 36]] INFO [2019-12-17 12:36:18,748] jepsen worker 27 - jepsen.util 10107 :fail :cas [9 [25 45]] :ValueNotSwapFrom INFO [2019-12-17 12:36:18,748] jepsen worker 27 - jepsen.util 10107 :invoke :write [9 41] INFO [2019-12-17 12:36:18,771] jepsen worker 11 - jepsen.util 10181 :ok :read [3 46] INFO [2019-12-17 12:36:18,772] jepsen worker 11 - jepsen.util 10181 :invoke :cas [3 [41 3]] INFO [2019-12-17 12:36:18,793] jepsen worker 28 - jepsen.util 8848 :fail :cas [9 [11 36]] :ValueNotSwapFrom INFO [2019-12-17 12:36:18,794] jepsen worker 28 - jepsen.util 8848 :invoke :read [9 nil] INFO [2019-12-17 12:36:18,794] jepsen worker 65 - jepsen.util 9155 :ok :read [21 8] INFO [2019-12-17 12:36:18,795] jepsen worker 65 - jepsen.util 9155 :invoke :write [21 11] INFO [2019-12-17 12:36:18,799] jepsen worker 50 - jepsen.util 14900 :info :cas [16 [32 14]] :RequestCanceledException INFO [2019-12-17 12:36:18,800] jepsen worker 50 - jepsen.util 14990 :invoke :write [16 36] INFO [2019-12-17 12:36:18,816] jepsen worker 57 - jepsen.util 14187 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:36:18,816] jepsen worker 57 - jepsen.util 14187 :invoke :cas [19 [10 28]] INFO [2019-12-17 12:36:18,821] jepsen worker 28 - jepsen.util 8848 :ok :read [9 8] INFO [2019-12-17 12:36:18,821] jepsen worker 28 - jepsen.util 8848 :invoke :cas [9 [21 43]] INFO [2019-12-17 12:36:18,839] jepsen worker 28 - jepsen.util 8848 :fail :cas [9 [21 43]] :ValueNotSwapFrom INFO [2019-12-17 12:36:18,839] jepsen worker 28 - jepsen.util 8848 :invoke :read [9 nil] INFO [2019-12-17 12:36:18,839] jepsen worker 27 - jepsen.util 10107 :ok :write [9 41] INFO [2019-12-17 12:36:18,839] jepsen worker 27 - jepsen.util 10107 :invoke :read [9 nil] INFO [2019-12-17 12:36:18,853] jepsen worker 47 - jepsen.util 11297 :ok :write [15 10] INFO [2019-12-17 12:36:18,853] jepsen worker 47 - jepsen.util 11297 :invoke :write [15 8] INFO [2019-12-17 12:36:18,859] jepsen worker 20 - jepsen.util 10550 :ok :write [6 44] INFO [2019-12-17 12:36:18,859] jepsen worker 20 - jepsen.util 10550 :invoke :read [6 nil] INFO [2019-12-17 12:36:18,868] jepsen worker 28 - jepsen.util 8848 :ok :read [9 41] INFO [2019-12-17 12:36:18,868] jepsen worker 28 - jepsen.util 8848 :invoke :read [9 nil] INFO [2019-12-17 12:36:18,875] jepsen worker 27 - jepsen.util 10107 :ok :read [9 41] INFO [2019-12-17 12:36:18,875] jepsen worker 27 - jepsen.util 10107 :invoke :cas [9 [49 43]] INFO [2019-12-17 12:36:18,892] jepsen worker 31 - jepsen.util 9301 :ok :write [10 7] INFO [2019-12-17 12:36:18,893] jepsen worker 31 - jepsen.util 9301 :invoke :cas [10 [48 10]] INFO [2019-12-17 12:36:18,906] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":61516,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:36:18,906] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":61516,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:36:18,907] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":61518,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:36:18,907] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":61518,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:36:18,917] jepsen worker 20 - jepsen.util 10550 :ok :read [6 44] INFO [2019-12-17 12:36:18,918] jepsen worker 20 - jepsen.util 10550 :invoke :cas [6 [43 27]] INFO [2019-12-17 12:36:18,919] jepsen worker 36 - jepsen.util 11466 :ok :write [12 18] INFO [2019-12-17 12:36:18,920] jepsen worker 36 - jepsen.util 11466 :invoke :cas [12 [47 10]] INFO [2019-12-17 12:36:18,922] jepsen worker 29 - jepsen.util 9029 :ok :read [9 41] INFO [2019-12-17 12:36:18,922] jepsen worker 29 - jepsen.util 9029 :invoke :write [9 46] INFO [2019-12-17 12:36:18,931] jepsen worker 15 - jepsen.util 13245 :info :cas [5 [15 15]] :RequestCanceledException INFO [2019-12-17 12:36:18,933] jepsen worker 15 - jepsen.util 13335 :invoke :write [5 1] INFO [2019-12-17 12:36:18,935] jepsen worker 28 - jepsen.util 8848 :ok :read [9 41] INFO [2019-12-17 12:36:18,936] jepsen worker 28 - jepsen.util 8848 :invoke :cas [9 [40 10]] INFO [2019-12-17 12:36:18,943] jepsen worker 31 - jepsen.util 9301 :fail :cas [10 [48 10]] :ValueNotSwapFrom INFO [2019-12-17 12:36:18,943] jepsen worker 27 - jepsen.util 10107 :fail :cas [9 [49 43]] :ValueNotSwapFrom INFO [2019-12-17 12:36:18,943] jepsen worker 31 - jepsen.util 9301 :invoke :cas [10 [42 28]] INFO [2019-12-17 12:36:18,943] jepsen worker 27 - jepsen.util 10107 :invoke :write [9 38] INFO [2019-12-17 12:36:18,943] jepsen worker 19 - jepsen.util 10729 :fail :read [6 nil] :RequestTimeoutException INFO [2019-12-17 12:36:18,943] jepsen worker 2 - jepsen.util 14042 :info :write [0 15] :RequestCanceledException INFO [2019-12-17 12:36:18,943] jepsen worker 19 - jepsen.util 10729 :invoke :cas [6 [6 5]] INFO [2019-12-17 12:36:18,944] jepsen worker 2 - jepsen.util 14132 :invoke :cas [0 [39 49]] INFO [2019-12-17 12:36:18,977] jepsen worker 28 - jepsen.util 8848 :fail :cas [9 [40 10]] :ValueNotSwapFrom INFO [2019-12-17 12:36:18,977] jepsen worker 28 - jepsen.util 8848 :invoke :read [9 nil] INFO [2019-12-17 12:36:18,984] jepsen worker 20 - jepsen.util 10550 :fail :cas [6 [43 27]] :ValueNotSwapFrom INFO [2019-12-17 12:36:18,984] jepsen worker 20 - jepsen.util 10550 :invoke :read [6 nil] INFO [2019-12-17 12:36:18,996] jepsen worker 51 - jepsen.util 10131 :ok :write [17 42] INFO [2019-12-17 12:36:18,996] jepsen worker 51 - jepsen.util 10131 :invoke :cas [17 [10 3]] INFO [2019-12-17 12:36:18,998] jepsen worker 76 - jepsen.util 14836 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:36:18,998] jepsen worker 76 - jepsen.util 14836 :invoke :read [25 nil] INFO [2019-12-17 12:36:19,002] jepsen worker 36 - jepsen.util 11466 :fail :cas [12 [47 10]] :ValueNotSwapFrom INFO [2019-12-17 12:36:19,002] jepsen worker 36 - jepsen.util 11466 :invoke :cas [12 [46 22]] INFO [2019-12-17 12:36:19,012] jepsen worker 65 - jepsen.util 9155 :ok :write [21 11] INFO [2019-12-17 12:36:19,012] jepsen worker 65 - jepsen.util 9155 :invoke :write [21 42] INFO [2019-12-17 12:36:19,014] jepsen worker 60 - jepsen.util 15360 :info :write [20 27] :RequestCanceledException INFO [2019-12-17 12:36:19,014] jepsen worker 80 - jepsen.util 10430 :ok :write [26 36] INFO [2019-12-17 12:36:19,014] jepsen worker 80 - jepsen.util 10430 :invoke :write [26 16] INFO [2019-12-17 12:36:19,022] jepsen worker 60 - jepsen.util 15450 :invoke :read [20 nil] INFO [2019-12-17 12:36:19,025] jepsen worker 31 - jepsen.util 9301 :fail :cas [10 [42 28]] :ValueNotSwapFrom INFO [2019-12-17 12:36:19,025] jepsen worker 31 - jepsen.util 9301 :invoke :cas [10 [32 35]] INFO [2019-12-17 12:36:19,027] jepsen worker 19 - jepsen.util 10729 :fail :cas [6 [6 5]] :ValueNotSwapFrom INFO [2019-12-17 12:36:19,027] jepsen worker 19 - jepsen.util 10729 :invoke :read [6 nil] INFO [2019-12-17 12:36:19,032] jepsen worker 41 - jepsen.util 15341 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:36:19,032] jepsen worker 17 - jepsen.util 14417 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:36:19,032] jepsen worker 17 - jepsen.util 14417 :invoke :read [5 nil] INFO [2019-12-17 12:36:19,032] jepsen worker 41 - jepsen.util 15341 :invoke :cas [13 [35 48]] INFO [2019-12-17 12:36:19,034] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":61536,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:36:19,034] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":61536,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:36:19,034] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":61539,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:36:19,034] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":61539,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:36:19,034] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":61540,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:36:19,034] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":61540,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:36:19,037] jepsen worker 20 - jepsen.util 10550 :ok :read [6 44] INFO [2019-12-17 12:36:19,037] jepsen worker 20 - jepsen.util 10550 :invoke :write [6 14] INFO [2019-12-17 12:36:19,056] jepsen worker 31 - jepsen.util 9301 :fail :cas [10 [32 35]] :ValueNotSwapFrom INFO [2019-12-17 12:36:19,056] jepsen worker 31 - jepsen.util 9301 :invoke :cas [10 [34 47]] INFO [2019-12-17 12:36:19,059] jepsen worker 42 - jepsen.util 15072 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:36:19,059] jepsen worker 42 - jepsen.util 15072 :invoke :cas [14 [21 29]] INFO [2019-12-17 12:36:19,073] jepsen worker 28 - jepsen.util 8848 :ok :read [9 41] INFO [2019-12-17 12:36:19,073] jepsen worker 28 - jepsen.util 8848 :invoke :cas [9 [29 28]] INFO [2019-12-17 12:36:19,094] jepsen worker 19 - jepsen.util 10729 :ok :read [6 44] INFO [2019-12-17 12:36:19,094] jepsen worker 19 - jepsen.util 10729 :invoke :read [6 nil] INFO [2019-12-17 12:36:19,099] jepsen worker 24 - jepsen.util 12984 :info :cas [8 [6 7]] :RequestCanceledException INFO [2019-12-17 12:36:19,100] jepsen worker 24 - jepsen.util 13074 :invoke :cas [8 [46 1]] INFO [2019-12-17 12:36:19,100] jepsen worker 3 - jepsen.util 10353 :fail :cas [1 [33 3]] :ValueNotSwapFrom INFO [2019-12-17 12:36:19,101] jepsen worker 3 - jepsen.util 10353 :invoke :cas [1 [11 39]] INFO [2019-12-17 12:36:19,129] jepsen worker 28 - jepsen.util 8848 :fail :cas [9 [29 28]] :ValueNotSwapFrom INFO [2019-12-17 12:36:19,129] jepsen worker 28 - jepsen.util 8848 :invoke :cas [9 [22 14]] INFO [2019-12-17 12:36:19,132] jepsen worker 6 - jepsen.util 14676 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:36:19,132] jepsen worker 6 - jepsen.util 14676 :invoke :write [2 39] INFO [2019-12-17 12:36:19,143] jepsen worker 19 - jepsen.util 10729 :ok :read [6 44] INFO [2019-12-17 12:36:19,143] jepsen worker 19 - jepsen.util 10729 :invoke :write [6 36] INFO [2019-12-17 12:36:19,153] jepsen worker 28 - jepsen.util 8848 :fail :cas [9 [22 14]] :ValueNotSwapFrom INFO [2019-12-17 12:36:19,153] jepsen worker 28 - jepsen.util 8848 :invoke :read [9 nil] INFO [2019-12-17 12:36:19,154] jepsen worker 31 - jepsen.util 9301 :fail :cas [10 [34 47]] :ValueNotSwapFrom INFO [2019-12-17 12:36:19,155] jepsen worker 31 - jepsen.util 9301 :invoke :read [10 nil] INFO [2019-12-17 12:36:19,182] jepsen worker 31 - jepsen.util 9301 :ok :read [10 7] INFO [2019-12-17 12:36:19,197] jepsen worker 54 - jepsen.util 11484 :ok :write [18 37] INFO [2019-12-17 12:36:19,212] jepsen worker 87 - jepsen.util 10167 :ok :write [29 22] INFO [2019-12-17 12:36:19,213] jepsen worker 65 - jepsen.util 9155 :ok :write [21 42] INFO [2019-12-17 12:36:19,219] jepsen worker 27 - jepsen.util 10107 :ok :write [9 38] INFO [2019-12-17 12:36:19,232] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":61561,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:36:19,232] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":61569,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:36:19,234] jepsen worker 70 - jepsen.util 11320 :fail :cas [23 [28 3]] :ValueNotSwapFrom INFO [2019-12-17 12:36:19,252] jepsen worker 71 - jepsen.util 11591 :fail :cas [23 [23 38]] :ValueNotSwapFrom INFO [2019-12-17 12:36:19,263] jepsen worker 35 - jepsen.util 14525 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:36:19,265] jepsen worker 45 - jepsen.util 10755 :ok :write [15 38] INFO [2019-12-17 12:36:19,287] jepsen worker 31 - jepsen.util 9301 :invoke :read [10 nil] INFO [2019-12-17 12:36:19,287] jepsen worker 87 - jepsen.util 10167 :invoke :read [29 nil] INFO [2019-12-17 12:36:19,287] jepsen worker 54 - jepsen.util 11484 :invoke :cas [18 [23 25]] INFO [2019-12-17 12:36:19,287] jepsen worker 27 - jepsen.util 10107 :invoke :read [9 nil] INFO [2019-12-17 12:36:19,287] jepsen worker 65 - jepsen.util 9155 :invoke :write [21 38] INFO [2019-12-17 12:36:19,287] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":61561,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:36:19,287] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":61569,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:36:19,287] jepsen worker 70 - jepsen.util 11320 :invoke :read [23 nil] INFO [2019-12-17 12:36:19,287] jepsen worker 71 - jepsen.util 11591 :invoke :cas [23 [7 48]] INFO [2019-12-17 12:36:19,287] jepsen worker 35 - jepsen.util 14525 :invoke :read [11 nil] INFO [2019-12-17 12:36:19,288] jepsen worker 45 - jepsen.util 10755 :invoke :write [15 17] INFO [2019-12-17 12:36:19,288] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":61562,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:36:19,288] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":61572,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:36:19,288] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":61562,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:36:19,288] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":61572,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:36:19,288] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":61563,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:36:19,288] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":61563,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:36:19,297] jepsen worker 79 - jepsen.util 11059 :ok :write [26 31] INFO [2019-12-17 12:36:19,297] jepsen worker 79 - jepsen.util 11059 :invoke :cas [26 [41 43]] INFO [2019-12-17 12:36:19,302] jepsen worker 12 - jepsen.util 10992 :fail :cas [4 [23 19]] :ValueNotSwapFrom INFO [2019-12-17 12:36:19,302] jepsen worker 12 - jepsen.util 10992 :invoke :cas [4 [41 36]] INFO [2019-12-17 12:36:19,310] jepsen worker 14 - jepsen.util 10454 :ok :write [4 29] INFO [2019-12-17 12:36:19,310] jepsen worker 14 - jepsen.util 10454 :invoke :write [4 14] INFO [2019-12-17 12:36:19,331] jepsen worker 44 - jepsen.util 14714 :info :write [14 17] :RequestCanceledException INFO [2019-12-17 12:36:19,332] jepsen worker 44 - jepsen.util 14804 :invoke :write [14 48] INFO [2019-12-17 12:36:19,339] jepsen worker 87 - jepsen.util 10167 :ok :read [29 22] INFO [2019-12-17 12:36:19,339] jepsen worker 87 - jepsen.util 10167 :invoke :write [29 31] INFO [2019-12-17 12:36:19,360] jepsen worker 31 - jepsen.util 9301 :ok :read [10 7] INFO [2019-12-17 12:36:19,360] jepsen worker 20 - jepsen.util 10550 :ok :write [6 14] INFO [2019-12-17 12:36:19,360] jepsen worker 31 - jepsen.util 9301 :invoke :cas [10 [10 21]] INFO [2019-12-17 12:36:19,360] jepsen worker 20 - jepsen.util 10550 :invoke :write [6 34] INFO [2019-12-17 12:36:19,384] jepsen worker 27 - jepsen.util 10107 :ok :read [9 38] INFO [2019-12-17 12:36:19,385] jepsen worker 27 - jepsen.util 10107 :invoke :write [9 41] INFO [2019-12-17 12:36:19,393] jepsen worker 80 - jepsen.util 10430 :ok :write [26 16] INFO [2019-12-17 12:36:19,393] jepsen worker 70 - jepsen.util 11320 :ok :read [23 10] INFO [2019-12-17 12:36:19,393] jepsen worker 80 - jepsen.util 10430 :invoke :write [26 28] INFO [2019-12-17 12:36:19,393] jepsen worker 70 - jepsen.util 11320 :invoke :read [23 nil] INFO [2019-12-17 12:36:19,399] jepsen worker 62 - jepsen.util 14822 :info :cas [20 [13 39]] :RequestCanceledException INFO [2019-12-17 12:36:19,400] jepsen worker 62 - jepsen.util 14912 :invoke :read [20 nil] INFO [2019-12-17 12:36:19,415] jepsen worker 31 - jepsen.util 9301 :fail :cas [10 [10 21]] :ValueNotSwapFrom INFO [2019-12-17 12:36:19,415] jepsen worker 31 - jepsen.util 9301 :invoke :write [10 7] INFO [2019-12-17 12:36:19,416] jepsen worker 23 - jepsen.util 14513 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:36:19,417] jepsen worker 23 - jepsen.util 14513 :invoke :read [7 nil] INFO [2019-12-17 12:36:19,417] jepsen worker 3 - jepsen.util 10353 :fail :cas [1 [11 39]] :ValueNotSwapFrom INFO [2019-12-17 12:36:19,417] jepsen worker 3 - jepsen.util 10353 :invoke :read [1 nil] INFO [2019-12-17 12:36:19,432] jepsen worker 63 - jepsen.util 9603 :info :write [21 39] :RequestCanceledException INFO [2019-12-17 12:36:19,433] jepsen worker 63 - jepsen.util 9693 :invoke :read [21 nil] INFO [2019-12-17 12:36:19,446] jepsen worker 85 - jepsen.util 15295 :info :cas [28 [46 29]] :RequestCanceledException INFO [2019-12-17 12:36:19,448] jepsen worker 85 - jepsen.util 15385 :invoke :write [28 47] INFO [2019-12-17 12:36:19,451] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":61598,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:36:19,451] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":61598,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:36:19,451] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":61601,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:36:19,452] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":61601,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:36:19,453] jepsen worker 71 - jepsen.util 11591 :fail :cas [23 [7 48]] :ValueNotSwapFrom INFO [2019-12-17 12:36:19,453] jepsen worker 71 - jepsen.util 11591 :invoke :cas [23 [20 3]] INFO [2019-12-17 12:36:19,463] jepsen worker 40 - jepsen.util 14890 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:36:19,463] jepsen worker 40 - jepsen.util 14890 :invoke :read [13 nil] INFO [2019-12-17 12:36:19,473] jepsen worker 29 - jepsen.util 9029 :ok :write [9 46] INFO [2019-12-17 12:36:19,473] jepsen worker 29 - jepsen.util 9029 :invoke :cas [9 [8 30]] INFO [2019-12-17 12:36:19,474] jepsen worker 38 - jepsen.util 11288 :ok :write [12 47] INFO [2019-12-17 12:36:19,474] jepsen worker 38 - jepsen.util 11288 :invoke :cas [12 [20 5]] INFO [2019-12-17 12:36:19,495] jepsen worker 29 - jepsen.util 9029 :fail :cas [9 [8 30]] :ValueNotSwapFrom INFO [2019-12-17 12:36:19,495] jepsen worker 29 - jepsen.util 9029 :invoke :cas [9 [10 1]] INFO [2019-12-17 12:36:19,512] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:36:19,512] jepsen nemesis - couchbase.util Rebalance check count: 480 INFO [2019-12-17 12:36:19,512] jepsen nemesis - couchbase.util Rebalance stuck count: 461 INFO [2019-12-17 12:36:19,523] jepsen worker 29 - jepsen.util 9029 :fail :cas [9 [10 1]] :ValueNotSwapFrom INFO [2019-12-17 12:36:19,523] jepsen worker 29 - jepsen.util 9029 :invoke :cas [9 [18 19]] INFO [2019-12-17 12:36:19,532] jepsen worker 53 - jepsen.util 9953 :ok :write [17 23] INFO [2019-12-17 12:36:19,532] jepsen worker 53 - jepsen.util 9953 :invoke :read [17 nil] INFO [2019-12-17 12:36:19,539] jepsen worker 29 - jepsen.util 9029 :fail :cas [9 [18 19]] :ValueNotSwapFrom INFO [2019-12-17 12:36:19,539] jepsen worker 29 - jepsen.util 9029 :invoke :read [9 nil] INFO [2019-12-17 12:36:19,557] jepsen worker 65 - jepsen.util 9155 :ok :write [21 38] INFO [2019-12-17 12:36:19,557] jepsen worker 65 - jepsen.util 9155 :invoke :read [21 nil] INFO [2019-12-17 12:36:19,562] jepsen worker 29 - jepsen.util 9029 :ok :read [9 46] INFO [2019-12-17 12:36:19,563] jepsen worker 29 - jepsen.util 9029 :invoke :write [9 25] INFO [2019-12-17 12:36:19,572] jepsen worker 27 - jepsen.util 10107 :ok :write [9 41] INFO [2019-12-17 12:36:19,572] jepsen worker 27 - jepsen.util 10107 :invoke :cas [9 [48 19]] INFO [2019-12-17 12:36:19,579] jepsen worker 65 - jepsen.util 9155 :ok :read [21 38] INFO [2019-12-17 12:36:19,579] jepsen worker 65 - jepsen.util 9155 :invoke :cas [21 [49 15]] INFO [2019-12-17 12:36:19,592] jepsen worker 88 - jepsen.util 10978 :ok :write [29 18] INFO [2019-12-17 12:36:19,592] jepsen worker 88 - jepsen.util 10978 :invoke :write [29 4] INFO [2019-12-17 12:36:19,592] jepsen worker 20 - jepsen.util 10550 :ok :write [6 34] INFO [2019-12-17 12:36:19,593] jepsen worker 20 - jepsen.util 10550 :invoke :cas [6 [3 22]] INFO [2019-12-17 12:36:19,596] jepsen worker 56 - jepsen.util 11036 :info :write [18 25] :RequestCanceledException INFO [2019-12-17 12:36:19,597] jepsen worker 56 - jepsen.util 11126 :invoke :write [18 10] INFO [2019-12-17 12:36:19,600] jepsen worker 65 - jepsen.util 9155 :fail :cas [21 [49 15]] :ValueNotSwapFrom INFO [2019-12-17 12:36:19,600] jepsen worker 65 - jepsen.util 9155 :invoke :read [21 nil] INFO [2019-12-17 12:36:19,600] jepsen worker 27 - jepsen.util 10107 :fail :cas [9 [48 19]] :ValueNotSwapFrom INFO [2019-12-17 12:36:19,600] jepsen worker 27 - jepsen.util 10107 :invoke :write [9 47] INFO [2019-12-17 12:36:19,632] jepsen worker 66 - jepsen.util 14196 :info :cas [22 [40 20]] :RequestCanceledException INFO [2019-12-17 12:36:19,633] jepsen worker 66 - jepsen.util 14286 :invoke :write [22 41] INFO [2019-12-17 12:36:19,660] jepsen worker 20 - jepsen.util 10550 :fail :cas [6 [3 22]] :ValueNotSwapFrom INFO [2019-12-17 12:36:19,661] jepsen worker 54 - jepsen.util 11484 :fail :cas [18 [23 25]] :ValueNotSwapFrom INFO [2019-12-17 12:36:19,661] jepsen worker 20 - jepsen.util 10550 :invoke :read [6 nil] INFO [2019-12-17 12:36:19,661] jepsen worker 54 - jepsen.util 11484 :invoke :cas [18 [26 21]] INFO [2019-12-17 12:36:19,663] jepsen worker 3 - jepsen.util 10353 :ok :read [1 6] INFO [2019-12-17 12:36:19,663] jepsen worker 78 - jepsen.util 10698 :info :write [26 18] :RequestCanceledException INFO [2019-12-17 12:36:19,663] jepsen worker 3 - jepsen.util 10353 :invoke :cas [1 [11 3]] INFO [2019-12-17 12:36:19,664] jepsen worker 78 - jepsen.util 10788 :invoke :write [26 27] INFO [2019-12-17 12:36:19,671] jepsen worker 36 - jepsen.util 11466 :fail :cas [12 [46 22]] :ValueNotSwapFrom INFO [2019-12-17 12:36:19,671] jepsen worker 36 - jepsen.util 11466 :invoke :cas [12 [31 46]] INFO [2019-12-17 12:36:19,671] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":61625,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:36:19,672] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":61625,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:36:19,672] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":61627,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:36:19,672] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":61627,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:36:19,690] jepsen worker 79 - jepsen.util 11059 :fail :cas [26 [41 43]] :ValueNotSwapFrom INFO [2019-12-17 12:36:19,690] jepsen worker 79 - jepsen.util 11059 :invoke :cas [26 [32 36]] INFO [2019-12-17 12:36:19,699] jepsen worker 49 - jepsen.util 14269 :info :cas [16 [29 29]] :RequestCanceledException INFO [2019-12-17 12:36:19,699] jepsen worker 69 - jepsen.util 10689 :info :write [23 27] :RequestCanceledException INFO [2019-12-17 12:36:19,699] jepsen worker 83 - jepsen.util 14663 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:36:19,699] jepsen worker 83 - jepsen.util 14663 :invoke :read [27 nil] INFO [2019-12-17 12:36:19,700] jepsen worker 49 - jepsen.util 14359 :invoke :read [16 nil] INFO [2019-12-17 12:36:19,700] jepsen worker 69 - jepsen.util 10779 :invoke :read [23 nil] INFO [2019-12-17 12:36:19,700] jepsen worker 31 - jepsen.util 9301 :ok :write [10 7] INFO [2019-12-17 12:36:19,700] jepsen worker 31 - jepsen.util 9301 :invoke :read [10 nil] INFO [2019-12-17 12:36:19,716] jepsen worker 7 - jepsen.util 14497 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:36:19,716] jepsen worker 7 - jepsen.util 14497 :invoke :cas [2 [37 47]] INFO [2019-12-17 12:36:19,722] jepsen worker 53 - jepsen.util 9953 :ok :read [17 23] INFO [2019-12-17 12:36:19,722] jepsen worker 53 - jepsen.util 9953 :invoke :cas [17 [18 17]] INFO [2019-12-17 12:36:19,723] jepsen worker 54 - jepsen.util 11484 :fail :cas [18 [26 21]] :ValueNotSwapFrom INFO [2019-12-17 12:36:19,723] jepsen worker 54 - jepsen.util 11484 :invoke :read [18 nil] INFO [2019-12-17 12:36:19,724] jepsen worker 65 - jepsen.util 9155 :ok :read [21 38] INFO [2019-12-17 12:36:19,725] jepsen worker 65 - jepsen.util 9155 :invoke :cas [21 [44 22]] INFO [2019-12-17 12:36:19,732] jepsen worker 0 - jepsen.util 15210 :info :cas [0 [43 44]] :RequestCanceledException INFO [2019-12-17 12:36:19,735] jepsen worker 0 - jepsen.util 15300 :invoke :cas [0 [21 12]] INFO [2019-12-17 12:36:19,745] jepsen worker 74 - jepsen.util 10514 :info :write [24 1] :RequestCanceledException INFO [2019-12-17 12:36:19,746] jepsen worker 74 - jepsen.util 10604 :invoke :read [24 nil] INFO [2019-12-17 12:36:19,763] jepsen worker 61 - jepsen.util 15271 :info :write [20 22] :RequestCanceledException INFO [2019-12-17 12:36:19,764] jepsen worker 61 - jepsen.util 15361 :invoke :cas [20 [21 48]] INFO [2019-12-17 12:36:19,776] jepsen worker 54 - jepsen.util 11484 :ok :read [18 37] INFO [2019-12-17 12:36:19,777] jepsen worker 54 - jepsen.util 11484 :invoke :write [18 32] INFO [2019-12-17 12:36:19,786] jepsen worker 36 - jepsen.util 11466 :fail :cas [12 [31 46]] :ValueNotSwapFrom INFO [2019-12-17 12:36:19,786] jepsen worker 36 - jepsen.util 11466 :invoke :cas [12 [17 33]] INFO [2019-12-17 12:36:19,797] jepsen worker 20 - jepsen.util 10550 :ok :read [6 34] INFO [2019-12-17 12:36:19,797] jepsen worker 20 - jepsen.util 10550 :invoke :read [6 nil] INFO [2019-12-17 12:36:19,799] jepsen worker 1 - jepsen.util 14491 :info :cas [0 [46 48]] :RequestCanceledException INFO [2019-12-17 12:36:19,799] jepsen worker 31 - jepsen.util 9301 :ok :read [10 7] INFO [2019-12-17 12:36:19,799] jepsen worker 31 - jepsen.util 9301 :invoke :write [10 16] INFO [2019-12-17 12:36:19,800] jepsen worker 1 - jepsen.util 14581 :invoke :read [0 nil] INFO [2019-12-17 12:36:19,811] jepsen worker 65 - jepsen.util 9155 :fail :cas [21 [44 22]] :ValueNotSwapFrom INFO [2019-12-17 12:36:19,811] jepsen worker 65 - jepsen.util 9155 :invoke :read [21 nil] INFO [2019-12-17 12:36:19,832] jepsen worker 26 - jepsen.util 15056 :info :cas [8 [8 33]] :RequestCanceledException INFO [2019-12-17 12:36:19,833] jepsen worker 26 - jepsen.util 15146 :invoke :read [8 nil] INFO [2019-12-17 12:36:19,862] jepsen worker 79 - jepsen.util 11059 :fail :cas [26 [32 36]] :ValueNotSwapFrom INFO [2019-12-17 12:36:19,862] jepsen worker 79 - jepsen.util 11059 :invoke :read [26 nil] INFO [2019-12-17 12:36:19,870] jepsen worker 20 - jepsen.util 10550 :ok :read [6 34] INFO [2019-12-17 12:36:19,871] jepsen worker 20 - jepsen.util 10550 :invoke :write [6 8] INFO [2019-12-17 12:36:19,879] jepsen worker 13 - jepsen.util 10093 :ok :write [4 28] INFO [2019-12-17 12:36:19,879] jepsen worker 13 - jepsen.util 10093 :invoke :cas [4 [0 26]] INFO [2019-12-17 12:36:19,883] jepsen worker 74 - jepsen.util 10604 :ok :read [24 5] INFO [2019-12-17 12:36:19,884] jepsen worker 74 - jepsen.util 10604 :invoke :cas [24 [34 17]] INFO [2019-12-17 12:36:19,888] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":61654,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:36:19,888] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":61654,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:36:19,888] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":61657,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:36:19,888] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":61657,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:36:19,888] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":61658,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:36:19,888] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":61658,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:36:19,892] jepsen worker 53 - jepsen.util 9953 :fail :cas [17 [18 17]] :ValueNotSwapFrom INFO [2019-12-17 12:36:19,892] jepsen worker 53 - jepsen.util 9953 :invoke :write [17 36] INFO [2019-12-17 12:36:19,917] jepsen worker 75 - jepsen.util 14025 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:36:19,917] jepsen worker 75 - jepsen.util 14025 :invoke :write [25 11] INFO [2019-12-17 12:36:19,923] jepsen worker 65 - jepsen.util 9155 :ok :read [21 38] INFO [2019-12-17 12:36:19,923] jepsen worker 65 - jepsen.util 9155 :invoke :write [21 49] INFO [2019-12-17 12:36:19,935] jepsen worker 51 - jepsen.util 10131 :fail :cas [17 [10 3]] :ValueNotSwapFrom INFO [2019-12-17 12:36:19,935] jepsen worker 51 - jepsen.util 10131 :invoke :read [17 nil] INFO [2019-12-17 12:36:19,939] jepsen worker 27 - jepsen.util 10107 :ok :write [9 47] INFO [2019-12-17 12:36:19,940] jepsen worker 27 - jepsen.util 10107 :invoke :write [9 39] INFO [2019-12-17 12:36:19,946] jepsen worker 58 - jepsen.util 14818 :info :cas [19 [38 36]] :RequestCanceledException INFO [2019-12-17 12:36:19,947] jepsen worker 58 - jepsen.util 14908 :invoke :write [19 38] INFO [2019-12-17 12:36:19,954] jepsen worker 12 - jepsen.util 10992 :fail :cas [4 [41 36]] :ValueNotSwapFrom INFO [2019-12-17 12:36:19,954] jepsen worker 12 - jepsen.util 10992 :invoke :cas [4 [38 36]] INFO [2019-12-17 12:36:19,955] jepsen worker 74 - jepsen.util 10604 :fail :cas [24 [34 17]] :ValueNotSwapFrom INFO [2019-12-17 12:36:19,955] jepsen worker 74 - jepsen.util 10604 :invoke :read [24 nil] INFO [2019-12-17 12:36:19,965] jepsen worker 4 - jepsen.util 9724 :info :write [1 14] :RequestCanceledException INFO [2019-12-17 12:36:19,975] jepsen worker 4 - jepsen.util 9814 :invoke :read [1 nil] INFO [2019-12-17 12:36:19,980] jepsen worker 89 - jepsen.util 10889 :ok :write [29 20] INFO [2019-12-17 12:36:19,981] jepsen worker 89 - jepsen.util 10889 :invoke :cas [29 [23 5]] INFO [2019-12-17 12:36:20,005] jepsen worker 80 - jepsen.util 10430 :ok :write [26 28] INFO [2019-12-17 12:36:20,005] jepsen worker 80 - jepsen.util 10430 :invoke :cas [26 [42 5]] INFO [2019-12-17 12:36:20,007] jepsen worker 38 - jepsen.util 11288 :fail :cas [12 [20 5]] :ValueNotSwapFrom INFO [2019-12-17 12:36:20,007] jepsen worker 38 - jepsen.util 11288 :invoke :cas [12 [41 40]] INFO [2019-12-17 12:36:20,020] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":61672,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:36:20,020] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":61672,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:36:20,020] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":61673,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:36:20,020] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":61673,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:36:20,039] jepsen worker 3 - jepsen.util 10353 :fail :cas [1 [11 3]] :ValueNotSwapFrom INFO [2019-12-17 12:36:20,039] jepsen worker 3 - jepsen.util 10353 :invoke :cas [1 [38 34]] INFO [2019-12-17 12:36:20,044] jepsen worker 10 - jepsen.util 10540 :info :cas [3 [17 2]] :RequestCanceledException INFO [2019-12-17 12:36:20,045] jepsen worker 10 - jepsen.util 10630 :invoke :cas [3 [9 10]] INFO [2019-12-17 12:36:20,062] jepsen worker 52 - jepsen.util 10402 :info :write [17 6] :RequestCanceledException INFO [2019-12-17 12:36:20,062] jepsen worker 37 - jepsen.util 10207 :info :write [12 1] :RequestCanceledException INFO [2019-12-17 12:36:20,063] jepsen worker 37 - jepsen.util 10297 :invoke :read [12 nil] INFO [2019-12-17 12:36:20,063] jepsen worker 52 - jepsen.util 10492 :invoke :read [17 nil] INFO [2019-12-17 12:36:20,075] jepsen worker 65 - jepsen.util 9155 :ok :write [21 49] INFO [2019-12-17 12:36:20,076] jepsen worker 65 - jepsen.util 9155 :invoke :read [21 nil] INFO [2019-12-17 12:36:20,093] jepsen worker 27 - jepsen.util 10107 :ok :write [9 39] INFO [2019-12-17 12:36:20,093] jepsen worker 27 - jepsen.util 10107 :invoke :write [9 19] INFO [2019-12-17 12:36:20,098] jepsen worker 10 - jepsen.util 10630 :fail :cas [3 [9 10]] :ValueNotSwapFrom INFO [2019-12-17 12:36:20,098] jepsen worker 10 - jepsen.util 10630 :invoke :write [3 48] INFO [2019-12-17 12:36:20,099] jepsen worker 68 - jepsen.util 13838 :info :cas [22 [9 33]] :RequestCanceledException INFO [2019-12-17 12:36:20,099] jepsen worker 68 - jepsen.util 13928 :invoke :read [22 nil] INFO [2019-12-17 12:36:20,110] jepsen worker 51 - jepsen.util 10131 :ok :read [17 23] INFO [2019-12-17 12:36:20,110] jepsen worker 51 - jepsen.util 10131 :invoke :write [17 33] INFO [2019-12-17 12:36:20,132] jepsen worker 5 - jepsen.util 10175 :info :write [1 45] :RequestCanceledException INFO [2019-12-17 12:36:20,132] jepsen worker 25 - jepsen.util 15505 :info :write [8 35] :RequestCanceledException INFO [2019-12-17 12:36:20,133] jepsen worker 5 - jepsen.util 10265 :invoke :read [1 nil] INFO [2019-12-17 12:36:20,133] jepsen worker 25 - jepsen.util 15595 :invoke :read [8 nil] INFO [2019-12-17 12:36:20,141] jepsen worker 65 - jepsen.util 9155 :ok :read [21 49] INFO [2019-12-17 12:36:20,141] jepsen worker 65 - jepsen.util 9155 :invoke :write [21 1] INFO [2019-12-17 12:36:20,146] jepsen worker 82 - jepsen.util 14572 :info :cas [27 [27 41]] :RequestCanceledException INFO [2019-12-17 12:36:20,146] jepsen worker 82 - jepsen.util 14662 :invoke :write [27 31] INFO [2019-12-17 12:36:20,158] jepsen worker 78 - jepsen.util 10788 :ok :write [26 27] INFO [2019-12-17 12:36:20,158] jepsen worker 78 - jepsen.util 10788 :invoke :write [26 44] INFO [2019-12-17 12:36:20,195] jepsen worker 88 - jepsen.util 10978 :ok :write [29 4] INFO [2019-12-17 12:36:20,195] jepsen worker 88 - jepsen.util 10978 :invoke :read [29 nil] INFO [2019-12-17 12:36:20,215] jepsen worker 55 - jepsen.util 11125 :info :write [18 30] :RequestCanceledException INFO [2019-12-17 12:36:20,215] jepsen worker 73 - jepsen.util 9973 :info :cas [24 [2 12]] :RequestCanceledException INFO [2019-12-17 12:36:20,216] jepsen worker 73 - jepsen.util 10063 :invoke :cas [24 [38 30]] INFO [2019-12-17 12:36:20,216] jepsen worker 55 - jepsen.util 11215 :invoke :cas [18 [26 6]] INFO [2019-12-17 12:36:20,231] jepsen worker 27 - jepsen.util 10107 :ok :write [9 19] INFO [2019-12-17 12:36:20,231] jepsen worker 27 - jepsen.util 10107 :invoke :cas [9 [18 43]] INFO [2019-12-17 12:36:20,245] jepsen worker 48 - jepsen.util 13458 :info :write [16 3] :RequestCanceledException INFO [2019-12-17 12:36:20,246] jepsen worker 48 - jepsen.util 13548 :invoke :cas [16 [17 49]] INFO [2019-12-17 12:36:20,258] jepsen worker 73 - jepsen.util 10063 :fail :cas [24 [38 30]] :ValueNotSwapFrom INFO [2019-12-17 12:36:20,258] jepsen worker 73 - jepsen.util 10063 :invoke :write [24 0] INFO [2019-12-17 12:36:20,273] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":61695,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:36:20,273] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":61704,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:36:20,273] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":61695,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:36:20,273] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":61704,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:36:20,273] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":61697,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:36:20,273] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":61708,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:36:20,273] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":61697,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:36:20,273] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":61708,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:36:20,274] jepsen worker 27 - jepsen.util 10107 :fail :cas [9 [18 43]] :ValueNotSwapFrom INFO [2019-12-17 12:36:20,274] jepsen worker 27 - jepsen.util 10107 :invoke :read [9 nil] INFO [2019-12-17 12:36:20,303] jepsen worker 13 - jepsen.util 10093 :fail :cas [4 [0 26]] :ValueNotSwapFrom INFO [2019-12-17 12:36:20,303] jepsen worker 13 - jepsen.util 10093 :invoke :write [4 21] INFO [2019-12-17 12:36:20,315] jepsen worker 30 - jepsen.util 10200 :info :cas [10 [12 32]] :RequestCanceledException INFO [2019-12-17 12:36:20,316] jepsen worker 30 - jepsen.util 10290 :invoke :cas [10 [16 21]] INFO [2019-12-17 12:36:20,331] jepsen worker 33 - jepsen.util 14523 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:36:20,331] jepsen worker 33 - jepsen.util 14523 :invoke :read [11 nil] INFO [2019-12-17 12:36:20,362] jepsen worker 64 - jepsen.util 8254 :fail :read [21 nil] :RequestTimeoutException INFO [2019-12-17 12:36:20,362] jepsen worker 9 - jepsen.util 11349 :info :write [3 9] :RequestCanceledException INFO [2019-12-17 12:36:20,362] jepsen worker 64 - jepsen.util 8254 :invoke :read [21 nil] INFO [2019-12-17 12:36:20,363] jepsen worker 9 - jepsen.util 11439 :invoke :read [3 nil] INFO [2019-12-17 12:36:20,375] jepsen worker 9 - jepsen.util 11439 :ok :read [3 46] INFO [2019-12-17 12:36:20,375] jepsen worker 9 - jepsen.util 11439 :invoke :read [3 nil] INFO [2019-12-17 12:36:20,384] jepsen worker 9 - jepsen.util 11439 :ok :read [3 46] INFO [2019-12-17 12:36:20,385] jepsen worker 9 - jepsen.util 11439 :invoke :read [3 nil] INFO [2019-12-17 12:36:20,403] jepsen worker 69 - jepsen.util 10779 :ok :read [23 10] INFO [2019-12-17 12:36:20,403] jepsen worker 69 - jepsen.util 10779 :invoke :read [23 nil] INFO [2019-12-17 12:36:20,404] jepsen worker 9 - jepsen.util 11439 :ok :read [3 46] INFO [2019-12-17 12:36:20,405] jepsen worker 9 - jepsen.util 11439 :invoke :cas [3 [49 5]] INFO [2019-12-17 12:36:20,410] jepsen worker 36 - jepsen.util 11466 :fail :cas [12 [17 33]] :ValueNotSwapFrom INFO [2019-12-17 12:36:20,410] jepsen worker 36 - jepsen.util 11466 :invoke :write [12 45] INFO [2019-12-17 12:36:20,416] jepsen worker 9 - jepsen.util 11439 :fail :cas [3 [49 5]] :ValueNotSwapFrom INFO [2019-12-17 12:36:20,416] jepsen worker 9 - jepsen.util 11439 :invoke :write [3 42] INFO [2019-12-17 12:36:20,417] jepsen worker 81 - jepsen.util 14031 :info :cas [27 [30 2]] :RequestCanceledException INFO [2019-12-17 12:36:20,417] jepsen worker 81 - jepsen.util 14121 :invoke :read [27 nil] INFO [2019-12-17 12:36:20,431] jepsen worker 18 - jepsen.util 10818 :fail :read [6 nil] :RequestTimeoutException INFO [2019-12-17 12:36:20,431] jepsen worker 18 - jepsen.util 10818 :invoke :write [6 12] INFO [2019-12-17 12:36:20,433] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":61726,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:36:20,433] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":61726,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:36:20,433] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":61730,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:36:20,434] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":61730,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:36:20,452] jepsen worker 80 - jepsen.util 10430 :fail :cas [26 [42 5]] :ValueNotSwapFrom INFO [2019-12-17 12:36:20,453] jepsen worker 80 - jepsen.util 10430 :invoke :cas [26 [32 16]] INFO [2019-12-17 12:36:20,459] jepsen worker 31 - jepsen.util 9301 :ok :write [10 16] INFO [2019-12-17 12:36:20,459] jepsen worker 31 - jepsen.util 9301 :invoke :read [10 nil] INFO [2019-12-17 12:36:20,463] jepsen worker 39 - jepsen.util 14079 :info :write [13 27] :RequestCanceledException INFO [2019-12-17 12:36:20,464] jepsen worker 39 - jepsen.util 14169 :invoke :cas [13 [9 49]] INFO [2019-12-17 12:36:20,484] jepsen worker 64 - jepsen.util 8254 :ok :read [21 49] INFO [2019-12-17 12:36:20,484] jepsen worker 64 - jepsen.util 8254 :invoke :write [21 9] INFO [2019-12-17 12:36:20,494] jepsen worker 31 - jepsen.util 9301 :ok :read [10 16] INFO [2019-12-17 12:36:20,494] jepsen worker 31 - jepsen.util 9301 :invoke :cas [10 [32 16]] INFO [2019-12-17 12:36:20,503] jepsen worker 55 - jepsen.util 11215 :fail :cas [18 [26 6]] :ValueNotSwapFrom INFO [2019-12-17 12:36:20,503] jepsen worker 55 - jepsen.util 11215 :invoke :write [18 35] INFO [2019-12-17 12:36:20,509] jepsen worker 70 - jepsen.util 11320 :ok :read [23 27] INFO [2019-12-17 12:36:20,509] jepsen worker 70 - jepsen.util 11320 :invoke :read [23 nil] INFO [2019-12-17 12:36:20,510] jepsen worker 5 - jepsen.util 10265 :ok :read [1 14] INFO [2019-12-17 12:36:20,510] jepsen worker 5 - jepsen.util 10265 :invoke :read [1 nil] INFO [2019-12-17 12:36:20,513] jepsen worker 79 - jepsen.util 11059 :ok :read [26 27] INFO [2019-12-17 12:36:20,513] jepsen worker 79 - jepsen.util 11059 :invoke :read [26 nil] INFO [2019-12-17 12:36:20,516] jepsen worker 8 - jepsen.util 14228 :info :cas [2 [0 14]] :RequestCanceledException INFO [2019-12-17 12:36:20,516] jepsen worker 77 - jepsen.util 15107 :info :cas [25 [33 17]] :RequestCanceledException INFO [2019-12-17 12:36:20,517] jepsen worker 8 - jepsen.util 14318 :invoke :cas [2 [18 39]] INFO [2019-12-17 12:36:20,517] jepsen worker 77 - jepsen.util 15197 :invoke :write [25 22] INFO [2019-12-17 12:36:20,525] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:36:20,526] jepsen nemesis - couchbase.util Rebalance check count: 481 INFO [2019-12-17 12:36:20,526] jepsen nemesis - couchbase.util Rebalance stuck count: 462 INFO [2019-12-17 12:36:20,532] jepsen worker 16 - jepsen.util 15946 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:36:20,532] jepsen worker 34 - jepsen.util 14794 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:36:20,532] jepsen worker 16 - jepsen.util 15946 :invoke :read [5 nil] INFO [2019-12-17 12:36:20,532] jepsen worker 34 - jepsen.util 14794 :invoke :cas [11 [27 25]] INFO [2019-12-17 12:36:20,542] jepsen worker 51 - jepsen.util 10131 :ok :write [17 33] INFO [2019-12-17 12:36:20,542] jepsen worker 51 - jepsen.util 10131 :invoke :cas [17 [11 9]] INFO [2019-12-17 12:36:20,545] jepsen worker 59 - jepsen.util 13559 :info :write [19 4] :RequestCanceledException INFO [2019-12-17 12:36:20,546] jepsen worker 59 - jepsen.util 13649 :invoke :read [19 nil] INFO [2019-12-17 12:36:20,563] jepsen worker 72 - jepsen.util 10782 :fail :read [24 nil] :RequestTimeoutException INFO [2019-12-17 12:36:20,563] jepsen worker 46 - jepsen.util 10396 :info :write [15 46] :RequestCanceledException INFO [2019-12-17 12:36:20,563] jepsen worker 72 - jepsen.util 10782 :invoke :cas [24 [39 27]] INFO [2019-12-17 12:36:20,564] jepsen worker 46 - jepsen.util 10486 :invoke :read [15 nil] INFO [2019-12-17 12:36:20,573] jepsen worker 3 - jepsen.util 10353 :fail :cas [1 [38 34]] :ValueNotSwapFrom INFO [2019-12-17 12:36:20,575] jepsen worker 3 - jepsen.util 10353 :invoke :cas [1 [29 28]] INFO [2019-12-17 12:36:20,575] jepsen worker 46 - jepsen.util 10486 :ok :read [15 38] INFO [2019-12-17 12:36:20,576] jepsen worker 46 - jepsen.util 10486 :invoke :write [15 22] INFO [2019-12-17 12:36:20,584] jepsen worker 69 - jepsen.util 10779 :ok :read [23 27] INFO [2019-12-17 12:36:20,584] jepsen worker 69 - jepsen.util 10779 :invoke :read [23 nil] INFO [2019-12-17 12:36:20,595] jepsen worker 89 - jepsen.util 10889 :fail :cas [29 [23 5]] :ValueNotSwapFrom INFO [2019-12-17 12:36:20,595] jepsen worker 89 - jepsen.util 10889 :invoke :read [29 nil] INFO [2019-12-17 12:36:20,603] jepsen worker 12 - jepsen.util 10992 :fail :cas [4 [38 36]] :ValueNotSwapFrom INFO [2019-12-17 12:36:20,603] jepsen worker 12 - jepsen.util 10992 :invoke :read [4 nil] INFO [2019-12-17 12:36:20,618] jepsen worker 37 - jepsen.util 10297 :ok :read [12 1] INFO [2019-12-17 12:36:20,618] jepsen worker 37 - jepsen.util 10297 :invoke :cas [12 [25 2]] INFO [2019-12-17 12:36:20,630] jepsen worker 5 - jepsen.util 10265 :ok :read [1 14] INFO [2019-12-17 12:36:20,630] jepsen worker 5 - jepsen.util 10265 :invoke :read [1 nil] INFO [2019-12-17 12:36:20,631] jepsen worker 21 - jepsen.util 13971 :info :write [7 3] :RequestCanceledException INFO [2019-12-17 12:36:20,632] jepsen worker 21 - jepsen.util 14061 :invoke :read [7 nil] INFO [2019-12-17 12:36:20,646] jepsen worker 32 - jepsen.util 10742 :info :write [10 38] :RequestCanceledException INFO [2019-12-17 12:36:20,648] jepsen worker 32 - jepsen.util 10832 :invoke :cas [10 [6 16]] INFO [2019-12-17 12:36:20,648] jepsen worker 52 - jepsen.util 10492 :ok :read [17 33] INFO [2019-12-17 12:36:20,648] jepsen worker 52 - jepsen.util 10492 :invoke :cas [17 [12 16]] INFO [2019-12-17 12:36:20,652] jepsen worker 38 - jepsen.util 11288 :fail :cas [12 [41 40]] :ValueNotSwapFrom INFO [2019-12-17 12:36:20,652] jepsen worker 38 - jepsen.util 11288 :invoke :cas [12 [9 47]] INFO [2019-12-17 12:36:20,655] jepsen worker 80 - jepsen.util 10430 :fail :cas [26 [32 16]] :ValueNotSwapFrom INFO [2019-12-17 12:36:20,655] jepsen worker 80 - jepsen.util 10430 :invoke :write [26 14] INFO [2019-12-17 12:36:20,659] jepsen worker 69 - jepsen.util 10779 :ok :read [23 27] INFO [2019-12-17 12:36:20,659] jepsen worker 69 - jepsen.util 10779 :invoke :read [23 nil] INFO [2019-12-17 12:36:20,661] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":61759,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:36:20,661] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":61759,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:36:20,662] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":61763,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:36:20,662] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":61763,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:36:20,691] jepsen worker 3 - jepsen.util 10353 :fail :cas [1 [29 28]] :ValueNotSwapFrom INFO [2019-12-17 12:36:20,692] jepsen worker 3 - jepsen.util 10353 :invoke :write [1 39] INFO [2019-12-17 12:36:20,698] jepsen worker 86 - jepsen.util 15476 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:36:20,699] jepsen worker 84 - jepsen.util 14124 :info :write [28 7] :RequestCanceledException INFO [2019-12-17 12:36:20,699] jepsen worker 86 - jepsen.util 15476 :invoke :cas [28 [47 35]] INFO [2019-12-17 12:36:20,699] jepsen worker 84 - jepsen.util 14214 :invoke :read [28 nil] INFO [2019-12-17 12:36:20,716] jepsen worker 43 - jepsen.util 14533 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:36:20,716] jepsen worker 43 - jepsen.util 14533 :invoke :cas [14 [29 25]] INFO [2019-12-17 12:36:20,723] jepsen worker 5 - jepsen.util 10265 :ok :read [1 14] INFO [2019-12-17 12:36:20,723] jepsen worker 5 - jepsen.util 10265 :invoke :cas [1 [33 29]] INFO [2019-12-17 12:36:20,731] jepsen worker 67 - jepsen.util 14737 :info :write [22 23] :RequestCanceledException INFO [2019-12-17 12:36:20,733] jepsen worker 67 - jepsen.util 14827 :invoke :read [22 nil] INFO [2019-12-17 12:36:20,738] jepsen worker 88 - jepsen.util 10978 :ok :read [29 4] INFO [2019-12-17 12:36:20,738] jepsen worker 88 - jepsen.util 10978 :invoke :write [29 25] INFO [2019-12-17 12:36:20,746] jepsen worker 22 - jepsen.util 14242 :info :write [7 26] :RequestCanceledException INFO [2019-12-17 12:36:20,747] jepsen worker 22 - jepsen.util 14332 :invoke :cas [7 [36 45]] INFO [2019-12-17 12:36:20,757] jepsen worker 37 - jepsen.util 10297 :fail :cas [12 [25 2]] :ValueNotSwapFrom INFO [2019-12-17 12:36:20,757] jepsen worker 37 - jepsen.util 10297 :invoke :cas [12 [3 41]] INFO [2019-12-17 12:36:20,767] jepsen worker 69 - jepsen.util 10779 :ok :read [23 27] INFO [2019-12-17 12:36:20,767] jepsen worker 69 - jepsen.util 10779 :invoke :cas [23 [49 30]] INFO [2019-12-17 12:36:20,794] jepsen worker 5 - jepsen.util 10265 :fail :cas [1 [33 29]] :ValueNotSwapFrom INFO [2019-12-17 12:36:20,794] jepsen worker 5 - jepsen.util 10265 :invoke :read [1 nil] INFO [2019-12-17 12:36:20,810] jepsen worker 51 - jepsen.util 10131 :fail :cas [17 [11 9]] :ValueNotSwapFrom INFO [2019-12-17 12:36:20,811] jepsen worker 51 - jepsen.util 10131 :invoke :cas [17 [22 43]] INFO [2019-12-17 12:36:20,816] jepsen worker 11 - jepsen.util 10181 :info :cas [3 [41 3]] :RequestCanceledException INFO [2019-12-17 12:36:20,817] jepsen worker 11 - jepsen.util 10271 :invoke :read [3 nil] INFO [2019-12-17 12:36:20,835] jepsen worker 10 - jepsen.util 10630 :ok :write [3 48] INFO [2019-12-17 12:36:20,835] jepsen worker 10 - jepsen.util 10630 :invoke :write [3 6] INFO [2019-12-17 12:36:20,835] jepsen worker 20 - jepsen.util 10550 :ok :write [6 8] INFO [2019-12-17 12:36:20,836] jepsen worker 20 - jepsen.util 10550 :invoke :read [6 nil] INFO [2019-12-17 12:36:20,844] jepsen worker 50 - jepsen.util 14990 :info :write [16 36] :RequestCanceledException INFO [2019-12-17 12:36:20,846] jepsen worker 50 - jepsen.util 15080 :invoke :write [16 6] INFO [2019-12-17 12:36:20,851] jepsen worker 69 - jepsen.util 10779 :fail :cas [23 [49 30]] :ValueNotSwapFrom INFO [2019-12-17 12:36:20,851] jepsen worker 69 - jepsen.util 10779 :invoke :cas [23 [33 2]] INFO [2019-12-17 12:36:20,857] jepsen worker 12 - jepsen.util 10992 :ok :read [4 28] INFO [2019-12-17 12:36:20,857] jepsen worker 12 - jepsen.util 10992 :invoke :write [4 27] INFO [2019-12-17 12:36:20,870] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":61790,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:36:20,871] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":61790,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:36:20,893] jepsen worker 5 - jepsen.util 10265 :ok :read [1 14] INFO [2019-12-17 12:36:20,893] jepsen worker 5 - jepsen.util 10265 :invoke :cas [1 [30 22]] INFO [2019-12-17 12:36:20,917] jepsen worker 57 - jepsen.util 14187 :info :cas [19 [10 28]] :RequestCanceledException INFO [2019-12-17 12:36:20,917] jepsen worker 57 - jepsen.util 14277 :invoke :write [19 22] INFO [2019-12-17 12:36:20,930] jepsen worker 47 - jepsen.util 11297 :info :write [15 8] :RequestCanceledException INFO [2019-12-17 12:36:20,931] jepsen worker 47 - jepsen.util 11387 :invoke :cas [15 [12 37]] INFO [2019-12-17 12:36:20,932] jepsen worker 38 - jepsen.util 11288 :fail :cas [12 [9 47]] :ValueNotSwapFrom INFO [2019-12-17 12:36:20,932] jepsen worker 38 - jepsen.util 11288 :invoke :cas [12 [22 39]] INFO [2019-12-17 12:36:20,943] jepsen worker 53 - jepsen.util 9953 :ok :write [17 36] INFO [2019-12-17 12:36:20,943] jepsen worker 53 - jepsen.util 9953 :invoke :read [17 nil] INFO [2019-12-17 12:36:20,963] jepsen worker 15 - jepsen.util 13335 :info :write [5 1] :RequestCanceledException INFO [2019-12-17 12:36:20,963] jepsen worker 15 - jepsen.util 13425 :invoke :cas [5 [28 46]] INFO [2019-12-17 12:36:20,976] jepsen worker 69 - jepsen.util 10779 :fail :cas [23 [33 2]] :ValueNotSwapFrom INFO [2019-12-17 12:36:20,976] jepsen worker 69 - jepsen.util 10779 :invoke :cas [23 [1 4]] INFO [2019-12-17 12:36:20,976] jepsen worker 4 - jepsen.util 9814 :ok :read [1 14] INFO [2019-12-17 12:36:20,976] jepsen worker 4 - jepsen.util 9814 :invoke :cas [1 [1 30]] INFO [2019-12-17 12:36:20,991] jepsen worker 80 - jepsen.util 10430 :ok :write [26 14] INFO [2019-12-17 12:36:20,991] jepsen worker 80 - jepsen.util 10430 :invoke :write [26 3] INFO [2019-12-17 12:36:20,993] jepsen worker 88 - jepsen.util 10978 :ok :write [29 25] INFO [2019-12-17 12:36:20,993] jepsen worker 88 - jepsen.util 10978 :invoke :cas [29 [19 4]] INFO [2019-12-17 12:36:21,002] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":61804,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:36:21,002] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":61804,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:36:21,002] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":61808,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:36:21,002] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":61808,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:36:21,009] jepsen worker 14 - jepsen.util 10454 :ok :write [4 14] INFO [2019-12-17 12:36:21,009] jepsen worker 14 - jepsen.util 10454 :invoke :read [4 nil] INFO [2019-12-17 12:36:21,032] jepsen worker 2 - jepsen.util 14132 :info :cas [0 [39 49]] :RequestCanceledException INFO [2019-12-17 12:36:21,032] jepsen worker 2 - jepsen.util 14222 :invoke :write [0 17] INFO [2019-12-17 12:36:21,058] jepsen worker 37 - jepsen.util 10297 :fail :cas [12 [3 41]] :ValueNotSwapFrom INFO [2019-12-17 12:36:21,058] jepsen worker 37 - jepsen.util 10297 :invoke :read [12 nil] INFO [2019-12-17 12:36:21,065] jepsen worker 5 - jepsen.util 10265 :fail :cas [1 [30 22]] :ValueNotSwapFrom INFO [2019-12-17 12:36:21,065] jepsen worker 5 - jepsen.util 10265 :invoke :write [1 27] INFO [2019-12-17 12:36:21,069] jepsen worker 51 - jepsen.util 10131 :fail :cas [17 [22 43]] :ValueNotSwapFrom INFO [2019-12-17 12:36:21,069] jepsen worker 51 - jepsen.util 10131 :invoke :cas [17 [41 15]] INFO [2019-12-17 12:36:21,096] jepsen worker 46 - jepsen.util 10486 :ok :write [15 22] INFO [2019-12-17 12:36:21,096] jepsen worker 73 - jepsen.util 10063 :ok :write [24 0] INFO [2019-12-17 12:36:21,096] jepsen worker 46 - jepsen.util 10486 :invoke :cas [15 [6 22]] INFO [2019-12-17 12:36:21,096] jepsen worker 73 - jepsen.util 10063 :invoke :read [24 nil] INFO [2019-12-17 12:36:21,097] jepsen worker 60 - jepsen.util 15450 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:36:21,097] jepsen worker 76 - jepsen.util 14836 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:36:21,098] jepsen worker 60 - jepsen.util 15450 :invoke :read [20 nil] INFO [2019-12-17 12:36:21,098] jepsen worker 76 - jepsen.util 14836 :invoke :write [25 1] INFO [2019-12-17 12:36:21,117] jepsen worker 24 - jepsen.util 13074 :info :cas [8 [46 1]] :RequestCanceledException INFO [2019-12-17 12:36:21,119] jepsen worker 24 - jepsen.util 13164 :invoke :cas [8 [37 21]] INFO [2019-12-17 12:36:21,120] jepsen worker 69 - jepsen.util 10779 :fail :cas [23 [1 4]] :ValueNotSwapFrom INFO [2019-12-17 12:36:21,120] jepsen worker 69 - jepsen.util 10779 :invoke :read [23 nil] INFO [2019-12-17 12:36:21,131] jepsen worker 17 - jepsen.util 14417 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:36:21,132] jepsen worker 17 - jepsen.util 14417 :invoke :read [5 nil] INFO [2019-12-17 12:36:21,132] jepsen worker 41 - jepsen.util 15341 :info :cas [13 [35 48]] :RequestCanceledException INFO [2019-12-17 12:36:21,133] jepsen worker 41 - jepsen.util 15431 :invoke :write [13 30] INFO [2019-12-17 12:36:21,145] jepsen worker 11 - jepsen.util 10271 :ok :read [3 48] INFO [2019-12-17 12:36:21,145] jepsen worker 11 - jepsen.util 10271 :invoke :cas [3 [48 9]] INFO [2019-12-17 12:36:21,162] jepsen worker 42 - jepsen.util 15072 :info :cas [14 [21 29]] :RequestCanceledException INFO [2019-12-17 12:36:21,164] jepsen worker 42 - jepsen.util 15162 :invoke :cas [14 [11 31]] INFO [2019-12-17 12:36:21,169] jepsen worker 32 - jepsen.util 10832 :fail :cas [10 [6 16]] :ValueNotSwapFrom INFO [2019-12-17 12:36:21,169] jepsen worker 32 - jepsen.util 10832 :invoke :write [10 13] INFO [2019-12-17 12:36:21,197] jepsen worker 69 - jepsen.util 10779 :ok :read [23 27] INFO [2019-12-17 12:36:21,197] jepsen worker 69 - jepsen.util 10779 :invoke :cas [23 [4 24]] INFO [2019-12-17 12:36:21,212] jepsen worker 69 - jepsen.util 10779 :fail :cas [23 [4 24]] :ValueNotSwapFrom INFO [2019-12-17 12:36:21,212] jepsen worker 69 - jepsen.util 10779 :invoke :cas [23 [46 13]] INFO [2019-12-17 12:36:21,212] jepsen worker 46 - jepsen.util 10486 :fail :cas [15 [6 22]] :ValueNotSwapFrom INFO [2019-12-17 12:36:21,212] jepsen worker 46 - jepsen.util 10486 :invoke :read [15 nil] INFO [2019-12-17 12:36:21,232] jepsen worker 6 - jepsen.util 14676 :info :write [2 39] :RequestCanceledException INFO [2019-12-17 12:36:21,233] jepsen worker 6 - jepsen.util 14766 :invoke :cas [2 [2 24]] INFO [2019-12-17 12:36:21,234] jepsen worker 69 - jepsen.util 10779 :fail :cas [23 [46 13]] :ValueNotSwapFrom INFO [2019-12-17 12:36:21,234] jepsen worker 69 - jepsen.util 10779 :invoke :read [23 nil] INFO [2019-12-17 12:36:21,242] jepsen worker 12 - jepsen.util 10992 :ok :write [4 27] INFO [2019-12-17 12:36:21,242] jepsen worker 12 - jepsen.util 10992 :invoke :read [4 nil] INFO [2019-12-17 12:36:21,246] jepsen worker 19 - jepsen.util 10729 :info :write [6 36] :RequestCanceledException INFO [2019-12-17 12:36:21,246] jepsen worker 28 - jepsen.util 8848 :fail :read [9 nil] :RequestTimeoutException INFO [2019-12-17 12:36:21,246] jepsen worker 28 - jepsen.util 8848 :invoke :read [9 nil] INFO [2019-12-17 12:36:21,246] jepsen worker 36 - jepsen.util 11466 :ok :write [12 45] INFO [2019-12-17 12:36:21,246] jepsen worker 36 - jepsen.util 11466 :invoke :cas [12 [20 19]] INFO [2019-12-17 12:36:21,246] jepsen worker 19 - jepsen.util 10819 :invoke :cas [6 [38 1]] INFO [2019-12-17 12:36:21,258] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":61827,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:36:21,258] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":61827,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:36:21,258] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":61835,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:36:21,258] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":61835,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:36:21,259] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":61838,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:36:21,259] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":61838,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:36:21,259] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":61839,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:36:21,259] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":61839,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:36:21,263] jepsen worker 87 - jepsen.util 10167 :ok :write [29 31] INFO [2019-12-17 12:36:21,263] jepsen worker 87 - jepsen.util 10167 :invoke :write [29 39] INFO [2019-12-17 12:36:21,265] jepsen worker 69 - jepsen.util 10779 :ok :read [23 27] INFO [2019-12-17 12:36:21,265] jepsen worker 69 - jepsen.util 10779 :invoke :write [23 47] INFO [2019-12-17 12:36:21,280] jepsen worker 73 - jepsen.util 10063 :ok :read [24 0] INFO [2019-12-17 12:36:21,280] jepsen worker 73 - jepsen.util 10063 :invoke :read [24 nil] INFO [2019-12-17 12:36:21,291] jepsen worker 28 - jepsen.util 8848 :ok :read [9 19] INFO [2019-12-17 12:36:21,292] jepsen worker 28 - jepsen.util 8848 :invoke :read [9 nil] INFO [2019-12-17 12:36:21,292] jepsen worker 3 - jepsen.util 10353 :ok :write [1 39] INFO [2019-12-17 12:36:21,292] jepsen worker 3 - jepsen.util 10353 :invoke :write [1 32] INFO [2019-12-17 12:36:21,297] jepsen worker 19 - jepsen.util 10819 :fail :cas [6 [38 1]] :ValueNotSwapFrom INFO [2019-12-17 12:36:21,297] jepsen worker 19 - jepsen.util 10819 :invoke :cas [6 [8 34]] INFO [2019-12-17 12:36:21,321] jepsen worker 46 - jepsen.util 10486 :ok :read [15 22] INFO [2019-12-17 12:36:21,321] jepsen worker 46 - jepsen.util 10486 :invoke :cas [15 [7 17]] INFO [2019-12-17 12:36:21,326] jepsen worker 28 - jepsen.util 8848 :ok :read [9 19] INFO [2019-12-17 12:36:21,327] jepsen worker 28 - jepsen.util 8848 :invoke :read [9 nil] INFO [2019-12-17 12:36:21,332] jepsen worker 70 - jepsen.util 11320 :ok :read [23 27] INFO [2019-12-17 12:36:21,332] jepsen worker 45 - jepsen.util 10755 :info :write [15 17] :RequestCanceledException INFO [2019-12-17 12:36:21,332] jepsen worker 70 - jepsen.util 11320 :invoke :cas [23 [15 19]] INFO [2019-12-17 12:36:21,333] jepsen worker 45 - jepsen.util 10845 :invoke :read [15 nil] INFO [2019-12-17 12:36:21,337] jepsen worker 73 - jepsen.util 10063 :ok :read [24 0] INFO [2019-12-17 12:36:21,337] jepsen worker 73 - jepsen.util 10063 :invoke :cas [24 [14 19]] INFO [2019-12-17 12:36:21,362] jepsen worker 73 - jepsen.util 10063 :fail :cas [24 [14 19]] :ValueNotSwapFrom INFO [2019-12-17 12:36:21,363] jepsen worker 73 - jepsen.util 10063 :invoke :read [24 nil] INFO [2019-12-17 12:36:21,363] jepsen worker 35 - jepsen.util 14525 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:36:21,363] jepsen worker 35 - jepsen.util 14525 :invoke :write [11 40] INFO [2019-12-17 12:36:21,364] jepsen worker 38 - jepsen.util 11288 :fail :cas [12 [22 39]] :ValueNotSwapFrom INFO [2019-12-17 12:36:21,364] jepsen worker 38 - jepsen.util 11288 :invoke :read [12 nil] INFO [2019-12-17 12:36:21,366] jepsen worker 28 - jepsen.util 8848 :ok :read [9 19] INFO [2019-12-17 12:36:21,367] jepsen worker 28 - jepsen.util 8848 :invoke :write [9 42] INFO [2019-12-17 12:36:21,377] jepsen worker 79 - jepsen.util 11059 :ok :read [26 14] INFO [2019-12-17 12:36:21,377] jepsen worker 79 - jepsen.util 11059 :invoke :cas [26 [40 4]] INFO [2019-12-17 12:36:21,389] jepsen worker 73 - jepsen.util 10063 :ok :read [24 0] INFO [2019-12-17 12:36:21,389] jepsen worker 73 - jepsen.util 10063 :invoke :cas [24 [31 2]] INFO [2019-12-17 12:36:21,415] jepsen worker 73 - jepsen.util 10063 :fail :cas [24 [31 2]] :ValueNotSwapFrom INFO [2019-12-17 12:36:21,415] jepsen worker 73 - jepsen.util 10063 :invoke :cas [24 [8 7]] INFO [2019-12-17 12:36:21,426] jepsen worker 88 - jepsen.util 10978 :fail :cas [29 [19 4]] :ValueNotSwapFrom INFO [2019-12-17 12:36:21,426] jepsen worker 88 - jepsen.util 10978 :invoke :cas [29 [33 42]] INFO [2019-12-17 12:36:21,431] jepsen worker 28 - jepsen.util 8848 :ok :write [9 42] INFO [2019-12-17 12:36:21,431] jepsen worker 28 - jepsen.util 8848 :invoke :write [9 7] INFO [2019-12-17 12:36:21,432] jepsen worker 44 - jepsen.util 14804 :info :write [14 48] :RequestCanceledException INFO [2019-12-17 12:36:21,433] jepsen worker 73 - jepsen.util 10063 :fail :cas [24 [8 7]] :ValueNotSwapFrom INFO [2019-12-17 12:36:21,433] jepsen worker 73 - jepsen.util 10063 :invoke :write [24 2] INFO [2019-12-17 12:36:21,434] jepsen worker 44 - jepsen.util 14894 :invoke :cas [14 [3 46]] INFO [2019-12-17 12:36:21,437] jepsen worker 54 - jepsen.util 11484 :ok :write [18 32] INFO [2019-12-17 12:36:21,438] jepsen worker 54 - jepsen.util 11484 :invoke :read [18 nil] INFO [2019-12-17 12:36:21,444] jepsen worker 62 - jepsen.util 14912 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:36:21,444] jepsen worker 62 - jepsen.util 14912 :invoke :write [20 44] INFO [2019-12-17 12:36:21,463] jepsen worker 63 - jepsen.util 9693 :fail :read [21 nil] :RequestTimeoutException INFO [2019-12-17 12:36:21,463] jepsen worker 63 - jepsen.util 9693 :invoke :cas [21 [1 40]] INFO [2019-12-17 12:36:21,484] jepsen worker 51 - jepsen.util 10131 :fail :cas [17 [41 15]] :ValueNotSwapFrom INFO [2019-12-17 12:36:21,485] jepsen worker 51 - jepsen.util 10131 :invoke :write [17 36] INFO [2019-12-17 12:36:21,489] jepsen worker 28 - jepsen.util 8848 :ok :write [9 7] INFO [2019-12-17 12:36:21,489] jepsen worker 28 - jepsen.util 8848 :invoke :read [9 nil] INFO [2019-12-17 12:36:21,497] jepsen worker 18 - jepsen.util 10818 :ok :write [6 12] INFO [2019-12-17 12:36:21,497] jepsen worker 30 - jepsen.util 10290 :ok :cas [10 [16 21]] INFO [2019-12-17 12:36:21,497] jepsen worker 18 - jepsen.util 10818 :invoke :write [6 23] INFO [2019-12-17 12:36:21,497] jepsen worker 30 - jepsen.util 10290 :invoke :write [10 33] INFO [2019-12-17 12:36:21,498] jepsen worker 71 - jepsen.util 11591 :info :cas [23 [20 3]] :RequestCanceledException INFO [2019-12-17 12:36:21,499] jepsen worker 71 - jepsen.util 11681 :invoke :cas [23 [24 35]] INFO [2019-12-17 12:36:21,513] jepsen worker 28 - jepsen.util 8848 :ok :read [9 7] INFO [2019-12-17 12:36:21,513] jepsen worker 28 - jepsen.util 8848 :invoke :write [9 12] INFO [2019-12-17 12:36:21,516] jepsen worker 23 - jepsen.util 14513 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:36:21,516] jepsen worker 23 - jepsen.util 14513 :invoke :read [7 nil] INFO [2019-12-17 12:36:21,529] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":61871,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:36:21,529] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":61871,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:36:21,529] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":61872,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:36:21,530] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":61872,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:36:21,532] jepsen worker 85 - jepsen.util 15385 :info :write [28 47] :RequestCanceledException INFO [2019-12-17 12:36:21,533] jepsen worker 85 - jepsen.util 15475 :invoke :write [28 15] INFO [2019-12-17 12:36:21,553] jepsen worker 12 - jepsen.util 10992 :ok :read [4 27] INFO [2019-12-17 12:36:21,554] jepsen worker 12 - jepsen.util 10992 :invoke :read [4 nil] INFO [2019-12-17 12:36:21,564] jepsen worker 40 - jepsen.util 14890 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:36:21,564] jepsen worker 40 - jepsen.util 14890 :invoke :read [13 nil] INFO [2019-12-17 12:36:21,564] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:36:21,564] jepsen nemesis - couchbase.util Rebalance check count: 482 INFO [2019-12-17 12:36:21,565] jepsen nemesis - couchbase.util Rebalance stuck count: 463 INFO [2019-12-17 12:36:21,565] jepsen worker 20 - jepsen.util 10550 :ok :read [6 12] INFO [2019-12-17 12:36:21,565] jepsen worker 20 - jepsen.util 10550 :invoke :write [6 45] INFO [2019-12-17 12:36:21,565] jepsen worker 80 - jepsen.util 10430 :ok :write [26 3] INFO [2019-12-17 12:36:21,565] jepsen worker 80 - jepsen.util 10430 :invoke :cas [26 [45 37]] INFO [2019-12-17 12:36:21,596] jepsen worker 47 - jepsen.util 11387 :fail :cas [15 [12 37]] :ValueNotSwapFrom INFO [2019-12-17 12:36:21,596] jepsen worker 47 - jepsen.util 11387 :invoke :write [15 22] INFO [2019-12-17 12:36:21,602] jepsen worker 5 - jepsen.util 10265 :ok :write [1 27] INFO [2019-12-17 12:36:21,602] jepsen worker 5 - jepsen.util 10265 :invoke :write [1 24] INFO [2019-12-17 12:36:21,610] jepsen worker 64 - jepsen.util 8254 :ok :write [21 9] INFO [2019-12-17 12:36:21,611] jepsen worker 64 - jepsen.util 8254 :invoke :write [21 1] INFO [2019-12-17 12:36:21,611] jepsen worker 4 - jepsen.util 9814 :fail :cas [1 [1 30]] :ValueNotSwapFrom INFO [2019-12-17 12:36:21,611] jepsen worker 4 - jepsen.util 9814 :invoke :write [1 5] INFO [2019-12-17 12:36:21,626] jepsen worker 14 - jepsen.util 10454 :ok :read [4 27] INFO [2019-12-17 12:36:21,626] jepsen worker 14 - jepsen.util 10454 :invoke :read [4 nil] INFO [2019-12-17 12:36:21,632] jepsen worker 29 - jepsen.util 9029 :info :write [9 25] :RequestCanceledException INFO [2019-12-17 12:36:21,633] jepsen worker 29 - jepsen.util 9119 :invoke :read [9 nil] INFO [2019-12-17 12:36:21,638] jepsen worker 45 - jepsen.util 10845 :ok :read [15 22] INFO [2019-12-17 12:36:21,638] jepsen worker 45 - jepsen.util 10845 :invoke :read [15 nil] INFO [2019-12-17 12:36:21,649] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":61899,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:36:21,650] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":61899,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:36:21,650] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":61902,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:36:21,650] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":61902,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:36:21,664] jepsen worker 63 - jepsen.util 9693 :fail :cas [21 [1 40]] :ValueNotSwapFrom INFO [2019-12-17 12:36:21,664] jepsen worker 63 - jepsen.util 9693 :invoke :cas [21 [21 4]] INFO [2019-12-17 12:36:21,666] jepsen worker 28 - jepsen.util 8848 :ok :write [9 12] INFO [2019-12-17 12:36:21,666] jepsen worker 28 - jepsen.util 8848 :invoke :write [9 10] INFO [2019-12-17 12:36:21,668] jepsen worker 53 - jepsen.util 9953 :ok :read [17 36] INFO [2019-12-17 12:36:21,668] jepsen worker 53 - jepsen.util 9953 :invoke :cas [17 [23 41]] INFO [2019-12-17 12:36:21,698] jepsen worker 56 - jepsen.util 11126 :info :write [18 10] :RequestCanceledException INFO [2019-12-17 12:36:21,699] jepsen worker 56 - jepsen.util 11216 :invoke :write [18 0] INFO [2019-12-17 12:36:21,717] jepsen worker 66 - jepsen.util 14286 :info :write [22 41] :RequestCanceledException INFO [2019-12-17 12:36:21,730] jepsen worker 66 - jepsen.util 14376 :invoke :cas [22 [27 14]] INFO [2019-12-17 12:36:21,736] jepsen worker 28 - jepsen.util 8848 :ok :write [9 10] INFO [2019-12-17 12:36:21,737] jepsen worker 28 - jepsen.util 8848 :invoke :cas [9 [40 25]] INFO [2019-12-17 12:36:21,744] jepsen worker 38 - jepsen.util 11288 :ok :read [12 45] INFO [2019-12-17 12:36:21,744] jepsen worker 38 - jepsen.util 11288 :invoke :write [12 25] INFO [2019-12-17 12:36:21,746] jepsen worker 49 - jepsen.util 14359 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:36:21,746] jepsen worker 49 - jepsen.util 14359 :invoke :write [16 48] INFO [2019-12-17 12:36:21,756] jepsen worker 28 - jepsen.util 8848 :fail :cas [9 [40 25]] :ValueNotSwapFrom INFO [2019-12-17 12:36:21,756] jepsen worker 28 - jepsen.util 8848 :invoke :write [9 22] INFO [2019-12-17 12:36:21,786] jepsen worker 37 - jepsen.util 10297 :ok :read [12 45] INFO [2019-12-17 12:36:21,786] jepsen worker 37 - jepsen.util 10297 :invoke :write [12 19] INFO [2019-12-17 12:36:21,789] jepsen worker 36 - jepsen.util 11466 :fail :cas [12 [20 19]] :ValueNotSwapFrom INFO [2019-12-17 12:36:21,789] jepsen worker 36 - jepsen.util 11466 :invoke :write [12 2] INFO [2019-12-17 12:36:21,798] jepsen worker 83 - jepsen.util 14663 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:36:21,798] jepsen worker 83 - jepsen.util 14663 :invoke :cas [27 [26 2]] INFO [2019-12-17 12:36:21,807] jepsen worker 28 - jepsen.util 8848 :ok :write [9 22] INFO [2019-12-17 12:36:21,807] jepsen worker 28 - jepsen.util 8848 :invoke :read [9 nil] INFO [2019-12-17 12:36:21,816] jepsen worker 61 - jepsen.util 15361 :info :cas [20 [21 48]] :RequestCanceledException INFO [2019-12-17 12:36:21,816] jepsen worker 7 - jepsen.util 14497 :info :cas [2 [37 47]] :RequestCanceledException INFO [2019-12-17 12:36:21,818] jepsen worker 7 - jepsen.util 14587 :invoke :read [2 nil] INFO [2019-12-17 12:36:21,818] jepsen worker 61 - jepsen.util 15451 :invoke :read [20 nil] INFO [2019-12-17 12:36:21,819] jepsen worker 28 - jepsen.util 8848 :ok :read [9 22] INFO [2019-12-17 12:36:21,819] jepsen worker 28 - jepsen.util 8848 :invoke :write [9 33] INFO [2019-12-17 12:36:21,832] jepsen worker 0 - jepsen.util 15300 :info :cas [0 [21 12]] :RequestCanceledException INFO [2019-12-17 12:36:21,832] jepsen worker 1 - jepsen.util 14581 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:36:21,832] jepsen worker 1 - jepsen.util 14581 :invoke :write [0 12] INFO [2019-12-17 12:36:21,833] jepsen worker 0 - jepsen.util 15390 :invoke :cas [0 [49 15]] INFO [2019-12-17 12:36:21,846] jepsen worker 26 - jepsen.util 15146 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:36:21,846] jepsen worker 26 - jepsen.util 15146 :invoke :cas [8 [29 44]] INFO [2019-12-17 12:36:21,861] jepsen worker 88 - jepsen.util 10978 :fail :cas [29 [33 42]] :ValueNotSwapFrom INFO [2019-12-17 12:36:21,861] jepsen worker 88 - jepsen.util 10978 :invoke :read [29 nil] INFO [2019-12-17 12:36:21,880] jepsen worker 28 - jepsen.util 8848 :ok :write [9 33] INFO [2019-12-17 12:36:21,880] jepsen worker 28 - jepsen.util 8848 :invoke :cas [9 [31 4]] INFO [2019-12-17 12:36:21,893] jepsen worker 79 - jepsen.util 11059 :fail :cas [26 [40 4]] :ValueNotSwapFrom INFO [2019-12-17 12:36:21,893] jepsen worker 79 - jepsen.util 11059 :invoke :read [26 nil] INFO [2019-12-17 12:36:21,898] jepsen worker 28 - jepsen.util 8848 :fail :cas [9 [31 4]] :ValueNotSwapFrom INFO [2019-12-17 12:36:21,898] jepsen worker 28 - jepsen.util 8848 :invoke :read [9 nil] INFO [2019-12-17 12:36:21,924] jepsen worker 28 - jepsen.util 8848 :ok :read [9 33] INFO [2019-12-17 12:36:21,925] jepsen worker 28 - jepsen.util 8848 :invoke :read [9 nil] INFO [2019-12-17 12:36:21,945] jepsen worker 10 - jepsen.util 10630 :ok :write [3 6] INFO [2019-12-17 12:36:21,945] jepsen worker 10 - jepsen.util 10630 :invoke :cas [3 [33 6]] INFO [2019-12-17 12:36:21,948] jepsen worker 28 - jepsen.util 8848 :ok :read [9 33] INFO [2019-12-17 12:36:21,948] jepsen worker 28 - jepsen.util 8848 :invoke :read [9 nil] INFO [2019-12-17 12:36:21,959] jepsen worker 55 - jepsen.util 11215 :ok :write [18 35] INFO [2019-12-17 12:36:21,959] jepsen worker 55 - jepsen.util 11215 :invoke :write [18 9] INFO [2019-12-17 12:36:21,964] jepsen worker 58 - jepsen.util 14908 :info :write [19 38] :RequestCanceledException INFO [2019-12-17 12:36:21,965] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":61925,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:36:21,965] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":61925,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:36:21,965] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":61929,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:36:21,965] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":61929,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:36:21,965] jepsen worker 58 - jepsen.util 14998 :invoke :write [19 5] INFO [2019-12-17 12:36:21,987] jepsen worker 10 - jepsen.util 10630 :fail :cas [3 [33 6]] :ValueNotSwapFrom INFO [2019-12-17 12:36:21,987] jepsen worker 10 - jepsen.util 10630 :invoke :cas [3 [0 27]] INFO [2019-12-17 12:36:21,994] jepsen worker 74 - jepsen.util 10604 :fail :read [24 nil] :RequestTimeoutException INFO [2019-12-17 12:36:21,994] jepsen worker 74 - jepsen.util 10604 :invoke :read [24 nil] INFO [2019-12-17 12:36:21,996] jepsen worker 28 - jepsen.util 8848 :ok :read [9 33] INFO [2019-12-17 12:36:21,996] jepsen worker 28 - jepsen.util 8848 :invoke :cas [9 [3 34]] INFO [2019-12-17 12:36:22,016] jepsen worker 75 - jepsen.util 14025 :info :write [25 11] :RequestCanceledException INFO [2019-12-17 12:36:22,018] jepsen worker 75 - jepsen.util 14115 :invoke :write [25 47] INFO [2019-12-17 12:36:22,021] jepsen worker 28 - jepsen.util 8848 :fail :cas [9 [3 34]] :ValueNotSwapFrom INFO [2019-12-17 12:36:22,021] jepsen worker 28 - jepsen.util 8848 :invoke :read [9 nil] INFO [2019-12-17 12:36:22,026] jepsen worker 89 - jepsen.util 10889 :ok :read [29 31] INFO [2019-12-17 12:36:22,027] jepsen worker 89 - jepsen.util 10889 :invoke :write [29 34] INFO [2019-12-17 12:36:22,027] jepsen worker 10 - jepsen.util 10630 :fail :cas [3 [0 27]] :ValueNotSwapFrom INFO [2019-12-17 12:36:22,029] jepsen worker 74 - jepsen.util 10604 :ok :read [24 0] INFO [2019-12-17 12:36:22,029] jepsen worker 10 - jepsen.util 10630 :invoke :write [3 9] INFO [2019-12-17 12:36:22,029] jepsen worker 74 - jepsen.util 10604 :invoke :read [24 nil] INFO [2019-12-17 12:36:22,043] jepsen worker 28 - jepsen.util 8848 :ok :read [9 33] INFO [2019-12-17 12:36:22,043] jepsen worker 28 - jepsen.util 8848 :invoke :read [9 nil] INFO [2019-12-17 12:36:22,053] jepsen worker 74 - jepsen.util 10604 :ok :read [24 0] INFO [2019-12-17 12:36:22,054] jepsen worker 74 - jepsen.util 10604 :invoke :cas [24 [10 27]] INFO [2019-12-17 12:36:22,065] jepsen worker 28 - jepsen.util 8848 :ok :read [9 33] INFO [2019-12-17 12:36:22,065] jepsen worker 28 - jepsen.util 8848 :invoke :cas [9 [21 40]] INFO [2019-12-17 12:36:22,083] jepsen worker 52 - jepsen.util 10492 :fail :cas [17 [12 16]] :ValueNotSwapFrom INFO [2019-12-17 12:36:22,083] jepsen worker 52 - jepsen.util 10492 :invoke :read [17 nil] INFO [2019-12-17 12:36:22,083] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":61942,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:36:22,083] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":61942,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:36:22,083] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":61943,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:36:22,083] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":61943,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:36:22,085] jepsen worker 28 - jepsen.util 8848 :fail :cas [9 [21 40]] :ValueNotSwapFrom INFO [2019-12-17 12:36:22,085] jepsen worker 28 - jepsen.util 8848 :invoke :write [9 28] INFO [2019-12-17 12:36:22,090] jepsen worker 4 - jepsen.util 9814 :ok :write [1 5] INFO [2019-12-17 12:36:22,090] jepsen worker 4 - jepsen.util 9814 :invoke :write [1 0] INFO [2019-12-17 12:36:22,115] jepsen worker 53 - jepsen.util 9953 :fail :cas [17 [23 41]] :ValueNotSwapFrom INFO [2019-12-17 12:36:22,115] jepsen worker 53 - jepsen.util 9953 :invoke :write [17 27] INFO [2019-12-17 12:36:22,146] jepsen worker 68 - jepsen.util 13928 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:36:22,146] jepsen worker 68 - jepsen.util 13928 :invoke :read [22 nil] INFO [2019-12-17 12:36:22,168] jepsen worker 69 - jepsen.util 10779 :ok :write [23 47] INFO [2019-12-17 12:36:22,168] jepsen worker 69 - jepsen.util 10779 :invoke :write [23 27] INFO [2019-12-17 12:36:22,178] jepsen worker 28 - jepsen.util 8848 :ok :write [9 28] INFO [2019-12-17 12:36:22,178] jepsen worker 28 - jepsen.util 8848 :invoke :write [9 31] INFO [2019-12-17 12:36:22,193] jepsen worker 73 - jepsen.util 10063 :ok :write [24 2] INFO [2019-12-17 12:36:22,193] jepsen worker 73 - jepsen.util 10063 :invoke :cas [24 [29 47]] INFO [2019-12-17 12:36:22,193] jepsen worker 30 - jepsen.util 10290 :ok :write [10 33] INFO [2019-12-17 12:36:22,193] jepsen worker 30 - jepsen.util 10290 :invoke :write [10 42] INFO [2019-12-17 12:36:22,194] jepsen worker 65 - jepsen.util 9155 :info :write [21 1] :RequestCanceledException INFO [2019-12-17 12:36:22,194] jepsen worker 82 - jepsen.util 14662 :info :write [27 31] :RequestCanceledException INFO [2019-12-17 12:36:22,197] jepsen worker 82 - jepsen.util 14752 :invoke :read [27 nil] INFO [2019-12-17 12:36:22,199] jepsen worker 65 - jepsen.util 9245 :invoke :read [21 nil] INFO [2019-12-17 12:36:22,204] jepsen worker 78 - jepsen.util 10788 :ok :write [26 44] INFO [2019-12-17 12:36:22,204] jepsen worker 78 - jepsen.util 10788 :invoke :write [26 36] INFO [2019-12-17 12:36:22,232] jepsen worker 25 - jepsen.util 15595 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:36:22,232] jepsen worker 25 - jepsen.util 15595 :invoke :cas [8 [14 0]] INFO [2019-12-17 12:36:22,243] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":61964,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:36:22,243] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":61974,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:36:22,243] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":61964,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:36:22,243] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":61974,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:36:22,243] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":61965,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:36:22,243] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":61978,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:36:22,244] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":61965,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:36:22,244] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":61978,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:36:22,244] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":61967,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:36:22,244] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":61967,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:36:22,246] jepsen worker 52 - jepsen.util 10492 :ok :read [17 36] INFO [2019-12-17 12:36:22,246] jepsen worker 52 - jepsen.util 10492 :invoke :write [17 4] INFO [2019-12-17 12:36:22,280] jepsen worker 3 - jepsen.util 10353 :ok :write [1 32] INFO [2019-12-17 12:36:22,281] jepsen worker 3 - jepsen.util 10353 :invoke :write [1 2] INFO [2019-12-17 12:36:22,287] jepsen worker 87 - jepsen.util 10167 :ok :write [29 39] INFO [2019-12-17 12:36:22,287] jepsen worker 45 - jepsen.util 10845 :ok :read [15 22] INFO [2019-12-17 12:36:22,287] jepsen worker 87 - jepsen.util 10167 :invoke :cas [29 [13 33]] INFO [2019-12-17 12:36:22,287] jepsen worker 45 - jepsen.util 10845 :invoke :read [15 nil] INFO [2019-12-17 12:36:22,308] jepsen worker 70 - jepsen.util 11320 :fail :cas [23 [15 19]] :ValueNotSwapFrom INFO [2019-12-17 12:36:22,308] jepsen worker 70 - jepsen.util 11320 :invoke :cas [23 [31 7]] INFO [2019-12-17 12:36:22,314] jepsen worker 73 - jepsen.util 10063 :fail :cas [24 [29 47]] :ValueNotSwapFrom INFO [2019-12-17 12:36:22,314] jepsen worker 73 - jepsen.util 10063 :invoke :write [24 39] INFO [2019-12-17 12:36:22,315] jepsen worker 5 - jepsen.util 10265 :ok :write [1 24] INFO [2019-12-17 12:36:22,315] jepsen worker 5 - jepsen.util 10265 :invoke :read [1 nil] INFO [2019-12-17 12:36:22,316] jepsen worker 27 - jepsen.util 10107 :fail :read [9 nil] :RequestTimeoutException INFO [2019-12-17 12:36:22,316] jepsen worker 27 - jepsen.util 10107 :invoke :write [9 33] INFO [2019-12-17 12:36:22,321] jepsen worker 71 - jepsen.util 11681 :fail :cas [23 [24 35]] :ValueNotSwapFrom INFO [2019-12-17 12:36:22,322] jepsen worker 71 - jepsen.util 11681 :invoke :cas [23 [14 0]] INFO [2019-12-17 12:36:22,332] jepsen worker 13 - jepsen.util 10093 :info :write [4 21] :RequestCanceledException INFO [2019-12-17 12:36:22,333] jepsen worker 13 - jepsen.util 10183 :invoke :read [4 nil] INFO [2019-12-17 12:36:22,346] jepsen worker 48 - jepsen.util 13548 :info :cas [16 [17 49]] :RequestCanceledException INFO [2019-12-17 12:36:22,347] jepsen worker 48 - jepsen.util 13638 :invoke :write [16 5] INFO [2019-12-17 12:36:22,352] jepsen worker 14 - jepsen.util 10454 :ok :read [4 27] INFO [2019-12-17 12:36:22,352] jepsen worker 14 - jepsen.util 10454 :invoke :write [4 32] INFO [2019-12-17 12:36:22,354] jepsen worker 63 - jepsen.util 9693 :fail :cas [21 [21 4]] :ValueNotSwapFrom INFO [2019-12-17 12:36:22,354] jepsen worker 63 - jepsen.util 9693 :invoke :cas [21 [1 22]] INFO [2019-12-17 12:36:22,379] jepsen worker 65 - jepsen.util 9245 :ok :read [21 9] INFO [2019-12-17 12:36:22,379] jepsen worker 65 - jepsen.util 9245 :invoke :write [21 35] INFO [2019-12-17 12:36:22,405] jepsen worker 28 - jepsen.util 8848 :ok :write [9 31] INFO [2019-12-17 12:36:22,405] jepsen worker 28 - jepsen.util 8848 :invoke :cas [9 [10 32]] INFO [2019-12-17 12:36:22,428] jepsen worker 28 - jepsen.util 8848 :fail :cas [9 [10 32]] :ValueNotSwapFrom INFO [2019-12-17 12:36:22,428] jepsen worker 28 - jepsen.util 8848 :invoke :read [9 nil] INFO [2019-12-17 12:36:22,432] jepsen worker 33 - jepsen.util 14523 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:36:22,432] jepsen worker 9 - jepsen.util 11439 :info :write [3 42] :RequestCanceledException INFO [2019-12-17 12:36:22,432] jepsen worker 33 - jepsen.util 14523 :invoke :write [11 41] INFO [2019-12-17 12:36:22,435] jepsen worker 9 - jepsen.util 11529 :invoke :read [3 nil] INFO [2019-12-17 12:36:22,450] jepsen worker 51 - jepsen.util 10131 :ok :write [17 36] INFO [2019-12-17 12:36:22,450] jepsen worker 51 - jepsen.util 10131 :invoke :read [17 nil] INFO [2019-12-17 12:36:22,450] jepsen worker 36 - jepsen.util 11466 :ok :write [12 2] INFO [2019-12-17 12:36:22,450] jepsen worker 36 - jepsen.util 11466 :invoke :write [12 17] INFO [2019-12-17 12:36:22,461] jepsen worker 28 - jepsen.util 8848 :ok :read [9 31] INFO [2019-12-17 12:36:22,461] jepsen worker 28 - jepsen.util 8848 :invoke :cas [9 [7 14]] INFO [2019-12-17 12:36:22,469] jepsen worker 18 - jepsen.util 10818 :ok :write [6 23] INFO [2019-12-17 12:36:22,469] jepsen worker 18 - jepsen.util 10818 :invoke :read [6 nil] INFO [2019-12-17 12:36:22,485] jepsen worker 79 - jepsen.util 11059 :ok :read [26 44] INFO [2019-12-17 12:36:22,485] jepsen worker 79 - jepsen.util 11059 :invoke :cas [26 [22 19]] INFO [2019-12-17 12:36:22,485] jepsen worker 28 - jepsen.util 8848 :fail :cas [9 [7 14]] :ValueNotSwapFrom INFO [2019-12-17 12:36:22,485] jepsen worker 28 - jepsen.util 8848 :invoke :write [9 16] INFO [2019-12-17 12:36:22,487] jepsen worker 87 - jepsen.util 10167 :fail :cas [29 [13 33]] :ValueNotSwapFrom INFO [2019-12-17 12:36:22,487] jepsen worker 87 - jepsen.util 10167 :invoke :write [29 19] INFO [2019-12-17 12:36:22,498] jepsen worker 81 - jepsen.util 14121 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:36:22,498] jepsen worker 81 - jepsen.util 14121 :invoke :cas [27 [17 23]] INFO [2019-12-17 12:36:22,514] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":61997,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:36:22,514] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":61997,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:36:22,514] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":62000,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:36:22,514] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":62000,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:36:22,514] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":62001,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:36:22,514] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":62001,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:36:22,516] jepsen worker 39 - jepsen.util 14169 :info :cas [13 [9 49]] :RequestCanceledException INFO [2019-12-17 12:36:22,517] jepsen worker 39 - jepsen.util 14259 :invoke :cas [13 [39 40]] INFO [2019-12-17 12:36:22,532] jepsen worker 70 - jepsen.util 11320 :fail :cas [23 [31 7]] :ValueNotSwapFrom INFO [2019-12-17 12:36:22,532] jepsen worker 70 - jepsen.util 11320 :invoke :read [23 nil] INFO [2019-12-17 12:36:22,532] jepsen worker 8 - jepsen.util 14318 :info :cas [2 [18 39]] :RequestCanceledException INFO [2019-12-17 12:36:22,533] jepsen worker 8 - jepsen.util 14408 :invoke :write [2 9] INFO [2019-12-17 12:36:22,536] jepsen worker 12 - jepsen.util 10992 :ok :read [4 21] INFO [2019-12-17 12:36:22,537] jepsen worker 12 - jepsen.util 10992 :invoke :cas [4 [31 48]] INFO [2019-12-17 12:36:22,545] jepsen worker 77 - jepsen.util 15197 :info :write [25 22] :RequestCanceledException INFO [2019-12-17 12:36:22,547] jepsen worker 77 - jepsen.util 15287 :invoke :read [25 nil] INFO [2019-12-17 12:36:22,560] jepsen worker 10 - jepsen.util 10630 :ok :write [3 9] INFO [2019-12-17 12:36:22,560] jepsen worker 10 - jepsen.util 10630 :invoke :write [3 46] INFO [2019-12-17 12:36:22,562] jepsen worker 73 - jepsen.util 10063 :ok :write [24 39] INFO [2019-12-17 12:36:22,562] jepsen worker 28 - jepsen.util 8848 :ok :write [9 16] INFO [2019-12-17 12:36:22,562] jepsen worker 73 - jepsen.util 10063 :invoke :write [24 14] INFO [2019-12-17 12:36:22,562] jepsen worker 28 - jepsen.util 8848 :invoke :cas [9 [10 20]] INFO [2019-12-17 12:36:22,563] jepsen worker 59 - jepsen.util 13649 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:36:22,563] jepsen worker 31 - jepsen.util 9301 :info :cas [10 [32 16]] :RequestCanceledException INFO [2019-12-17 12:36:22,563] jepsen worker 59 - jepsen.util 13649 :invoke :read [19 nil] INFO [2019-12-17 12:36:22,575] jepsen worker 69 - jepsen.util 10779 :ok :write [23 27] INFO [2019-12-17 12:36:22,575] jepsen worker 69 - jepsen.util 10779 :invoke :write [23 3] INFO [2019-12-17 12:36:22,575] jepsen worker 31 - jepsen.util 9391 :invoke :read [10 nil] INFO [2019-12-17 12:36:22,592] jepsen worker 65 - jepsen.util 9245 :ok :write [21 35] INFO [2019-12-17 12:36:22,593] jepsen worker 65 - jepsen.util 9245 :invoke :cas [21 [32 30]] INFO [2019-12-17 12:36:22,597] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:36:22,597] jepsen nemesis - couchbase.util Rebalance check count: 483 INFO [2019-12-17 12:36:22,597] jepsen nemesis - couchbase.util Rebalance stuck count: 464 INFO [2019-12-17 12:36:22,607] jepsen worker 71 - jepsen.util 11681 :fail :cas [23 [14 0]] :ValueNotSwapFrom INFO [2019-12-17 12:36:22,607] jepsen worker 71 - jepsen.util 11681 :invoke :cas [23 [40 25]] INFO [2019-12-17 12:36:22,629] jepsen worker 54 - jepsen.util 11484 :ok :read [18 35] INFO [2019-12-17 12:36:22,629] jepsen worker 54 - jepsen.util 11484 :invoke :cas [18 [13 38]] INFO [2019-12-17 12:36:22,632] jepsen worker 16 - jepsen.util 15946 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:36:22,632] jepsen worker 34 - jepsen.util 14794 :info :cas [11 [27 25]] :RequestCanceledException INFO [2019-12-17 12:36:22,632] jepsen worker 16 - jepsen.util 15946 :invoke :cas [5 [11 35]] INFO [2019-12-17 12:36:22,633] jepsen worker 34 - jepsen.util 14884 :invoke :write [11 10] INFO [2019-12-17 12:36:22,648] jepsen worker 28 - jepsen.util 8848 :fail :cas [9 [10 20]] :ValueNotSwapFrom INFO [2019-12-17 12:36:22,648] jepsen worker 28 - jepsen.util 8848 :invoke :write [9 16] INFO [2019-12-17 12:36:22,663] jepsen worker 72 - jepsen.util 10782 :info :cas [24 [39 27]] :RequestCanceledException INFO [2019-12-17 12:36:22,664] jepsen worker 65 - jepsen.util 9245 :fail :cas [21 [32 30]] :ValueNotSwapFrom INFO [2019-12-17 12:36:22,665] jepsen worker 65 - jepsen.util 9245 :invoke :read [21 nil] INFO [2019-12-17 12:36:22,667] jepsen worker 72 - jepsen.util 10872 :invoke :write [24 26] INFO [2019-12-17 12:36:22,698] jepsen worker 21 - jepsen.util 14061 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:36:22,698] jepsen worker 21 - jepsen.util 14061 :invoke :read [7 nil] INFO [2019-12-17 12:36:22,701] jepsen worker 5 - jepsen.util 10265 :ok :read [1 24] INFO [2019-12-17 12:36:22,701] jepsen worker 5 - jepsen.util 10265 :invoke :read [1 nil] INFO [2019-12-17 12:36:22,708] jepsen worker 65 - jepsen.util 9245 :ok :read [21 35] INFO [2019-12-17 12:36:22,708] jepsen worker 65 - jepsen.util 9245 :invoke :cas [21 [47 43]] INFO [2019-12-17 12:36:22,716] jepsen worker 13 - jepsen.util 10183 :ok :read [4 21] INFO [2019-12-17 12:36:22,716] jepsen worker 13 - jepsen.util 10183 :invoke :write [4 29] INFO [2019-12-17 12:36:22,719] jepsen worker 37 - jepsen.util 10297 :ok :write [12 19] INFO [2019-12-17 12:36:22,719] jepsen worker 37 - jepsen.util 10297 :invoke :cas [12 [33 48]] INFO [2019-12-17 12:36:22,732] jepsen worker 84 - jepsen.util 14214 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:36:22,732] jepsen worker 84 - jepsen.util 14214 :invoke :cas [28 [9 36]] INFO [2019-12-17 12:36:22,734] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":62029,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:36:22,735] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":62029,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:36:22,735] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":62033,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:36:22,735] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":62033,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:36:22,746] jepsen worker 67 - jepsen.util 14827 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:36:22,746] jepsen worker 67 - jepsen.util 14827 :invoke :read [22 nil] INFO [2019-12-17 12:36:22,763] jepsen worker 22 - jepsen.util 14332 :info :cas [7 [36 45]] :RequestCanceledException INFO [2019-12-17 12:36:22,763] jepsen worker 22 - jepsen.util 14422 :invoke :write [7 6] INFO [2019-12-17 12:36:22,770] jepsen worker 65 - jepsen.util 9245 :fail :cas [21 [47 43]] :ValueNotSwapFrom INFO [2019-12-17 12:36:22,770] jepsen worker 65 - jepsen.util 9245 :invoke :write [21 32] INFO [2019-12-17 12:36:22,779] jepsen worker 38 - jepsen.util 11288 :ok :write [12 25] INFO [2019-12-17 12:36:22,779] jepsen worker 38 - jepsen.util 11288 :invoke :write [12 49] INFO [2019-12-17 12:36:22,787] jepsen worker 30 - jepsen.util 10290 :ok :write [10 42] INFO [2019-12-17 12:36:22,787] jepsen worker 30 - jepsen.util 10290 :invoke :cas [10 [20 43]] INFO [2019-12-17 12:36:22,792] jepsen worker 20 - jepsen.util 10550 :ok :write [6 45] INFO [2019-12-17 12:36:22,792] jepsen worker 20 - jepsen.util 10550 :invoke :read [6 nil] INFO [2019-12-17 12:36:22,799] jepsen worker 86 - jepsen.util 15476 :info :cas [28 [47 35]] :RequestCanceledException INFO [2019-12-17 12:36:22,800] jepsen worker 86 - jepsen.util 15566 :invoke :read [28 nil] INFO [2019-12-17 12:36:22,809] jepsen worker 5 - jepsen.util 10265 :ok :read [1 24] INFO [2019-12-17 12:36:22,809] jepsen worker 5 - jepsen.util 10265 :invoke :read [1 nil] INFO [2019-12-17 12:36:22,816] jepsen worker 43 - jepsen.util 14533 :info :cas [14 [29 25]] :RequestCanceledException INFO [2019-12-17 12:36:22,820] jepsen worker 43 - jepsen.util 14623 :invoke :write [14 6] INFO [2019-12-17 12:36:22,828] jepsen worker 10 - jepsen.util 10630 :ok :write [3 46] INFO [2019-12-17 12:36:22,828] jepsen worker 10 - jepsen.util 10630 :invoke :read [3 nil] INFO [2019-12-17 12:36:22,851] jepsen worker 88 - jepsen.util 10978 :ok :read [29 39] INFO [2019-12-17 12:36:22,851] jepsen worker 88 - jepsen.util 10978 :invoke :cas [29 [46 0]] INFO [2019-12-17 12:36:22,854] jepsen worker 70 - jepsen.util 11320 :ok :read [23 27] INFO [2019-12-17 12:36:22,854] jepsen worker 70 - jepsen.util 11320 :invoke :read [23 nil] INFO [2019-12-17 12:36:22,879] jepsen worker 5 - jepsen.util 10265 :ok :read [1 24] INFO [2019-12-17 12:36:22,880] jepsen worker 5 - jepsen.util 10265 :invoke :cas [1 [10 21]] INFO [2019-12-17 12:36:22,881] jepsen worker 55 - jepsen.util 11215 :ok :write [18 9] INFO [2019-12-17 12:36:22,881] jepsen worker 55 - jepsen.util 11215 :invoke :write [18 42] INFO [2019-12-17 12:36:22,896] jepsen worker 63 - jepsen.util 9693 :fail :cas [21 [1 22]] :ValueNotSwapFrom INFO [2019-12-17 12:36:22,896] jepsen worker 63 - jepsen.util 9693 :invoke :write [21 23] INFO [2019-12-17 12:36:22,896] jepsen worker 50 - jepsen.util 15080 :info :write [16 6] :RequestCanceledException INFO [2019-12-17 12:36:22,897] jepsen worker 50 - jepsen.util 15170 :invoke :read [16 nil] INFO [2019-12-17 12:36:22,903] jepsen worker 80 - jepsen.util 10430 :fail :cas [26 [45 37]] :ValueNotSwapFrom INFO [2019-12-17 12:36:22,903] jepsen worker 80 - jepsen.util 10430 :invoke :write [26 20] INFO [2019-12-17 12:36:22,905] jepsen worker 12 - jepsen.util 10992 :fail :cas [4 [31 48]] :ValueNotSwapFrom INFO [2019-12-17 12:36:22,905] jepsen worker 12 - jepsen.util 10992 :invoke :cas [4 [39 37]] INFO [2019-12-17 12:36:22,906] jepsen worker 28 - jepsen.util 8848 :ok :write [9 16] INFO [2019-12-17 12:36:22,906] jepsen worker 28 - jepsen.util 8848 :invoke :cas [9 [42 22]] INFO [2019-12-17 12:36:22,925] jepsen worker 71 - jepsen.util 11681 :fail :cas [23 [40 25]] :ValueNotSwapFrom INFO [2019-12-17 12:36:22,925] jepsen worker 71 - jepsen.util 11681 :invoke :cas [23 [4 48]] INFO [2019-12-17 12:36:22,936] jepsen worker 28 - jepsen.util 8848 :fail :cas [9 [42 22]] :ValueNotSwapFrom INFO [2019-12-17 12:36:22,936] jepsen worker 28 - jepsen.util 8848 :invoke :read [9 nil] INFO [2019-12-17 12:36:22,948] jepsen worker 9 - jepsen.util 11529 :ok :read [3 9] INFO [2019-12-17 12:36:22,949] jepsen worker 9 - jepsen.util 11529 :invoke :read [3 nil] INFO [2019-12-17 12:36:22,949] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":62057,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:36:22,949] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":62057,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:36:22,949] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":62060,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:36:22,949] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":62060,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:36:22,967] jepsen worker 30 - jepsen.util 10290 :fail :cas [10 [20 43]] :ValueNotSwapFrom INFO [2019-12-17 12:36:22,967] jepsen worker 30 - jepsen.util 10290 :invoke :read [10 nil] INFO [2019-12-17 12:36:22,979] jepsen worker 10 - jepsen.util 10630 :ok :read [3 46] INFO [2019-12-17 12:36:22,979] jepsen worker 10 - jepsen.util 10630 :invoke :read [3 nil] INFO [2019-12-17 12:36:22,983] jepsen worker 18 - jepsen.util 10818 :ok :read [6 45] INFO [2019-12-17 12:36:22,983] jepsen worker 18 - jepsen.util 10818 :invoke :write [6 45] INFO [2019-12-17 12:36:22,984] jepsen worker 65 - jepsen.util 9245 :ok :write [21 32] INFO [2019-12-17 12:36:22,984] jepsen worker 65 - jepsen.util 9245 :invoke :write [21 37] INFO [2019-12-17 12:36:22,999] jepsen worker 51 - jepsen.util 10131 :ok :read [17 36] INFO [2019-12-17 12:36:22,999] jepsen worker 51 - jepsen.util 10131 :invoke :read [17 nil] INFO [2019-12-17 12:36:23,008] jepsen worker 28 - jepsen.util 8848 :ok :read [9 16] INFO [2019-12-17 12:36:23,008] jepsen worker 28 - jepsen.util 8848 :invoke :write [9 35] INFO [2019-12-17 12:36:23,016] jepsen worker 57 - jepsen.util 14277 :info :write [19 22] :RequestCanceledException INFO [2019-12-17 12:36:23,017] jepsen worker 57 - jepsen.util 14367 :invoke :write [19 34] INFO [2019-12-17 12:36:23,038] jepsen worker 9 - jepsen.util 11529 :ok :read [3 46] INFO [2019-12-17 12:36:23,038] jepsen worker 9 - jepsen.util 11529 :invoke :cas [3 [0 5]] INFO [2019-12-17 12:36:23,046] jepsen worker 15 - jepsen.util 13425 :info :cas [5 [28 46]] :RequestCanceledException INFO [2019-12-17 12:36:23,047] jepsen worker 15 - jepsen.util 13515 :invoke :cas [5 [17 18]] INFO [2019-12-17 12:36:23,055] jepsen worker 10 - jepsen.util 10630 :ok :read [3 46] INFO [2019-12-17 12:36:23,055] jepsen worker 10 - jepsen.util 10630 :invoke :cas [3 [6 12]] INFO [2019-12-17 12:36:23,056] jepsen worker 37 - jepsen.util 10297 :fail :cas [12 [33 48]] :ValueNotSwapFrom INFO [2019-12-17 12:36:23,057] jepsen worker 37 - jepsen.util 10297 :invoke :read [12 nil] INFO [2019-12-17 12:36:23,059] jepsen worker 63 - jepsen.util 9693 :ok :write [21 23] INFO [2019-12-17 12:36:23,059] jepsen worker 63 - jepsen.util 9693 :invoke :read [21 nil] INFO [2019-12-17 12:36:23,063] jepsen worker 2 - jepsen.util 14222 :info :write [0 17] :RequestCanceledException INFO [2019-12-17 12:36:23,064] jepsen worker 2 - jepsen.util 14312 :invoke :cas [0 [20 13]] INFO [2019-12-17 12:36:23,068] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":62077,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:36:23,068] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":62077,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:36:23,068] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":62081,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:36:23,068] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":62081,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:36:23,083] jepsen worker 10 - jepsen.util 10630 :fail :cas [3 [6 12]] :ValueNotSwapFrom INFO [2019-12-17 12:36:23,084] jepsen worker 10 - jepsen.util 10630 :invoke :cas [3 [25 5]] INFO [2019-12-17 12:36:23,096] jepsen worker 63 - jepsen.util 9693 :ok :read [21 23] INFO [2019-12-17 12:36:23,096] jepsen worker 63 - jepsen.util 9693 :invoke :read [21 nil] INFO [2019-12-17 12:36:23,107] jepsen worker 10 - jepsen.util 10630 :fail :cas [3 [25 5]] :ValueNotSwapFrom INFO [2019-12-17 12:36:23,107] jepsen worker 10 - jepsen.util 10630 :invoke :read [3 nil] INFO [2019-12-17 12:36:23,114] jepsen worker 63 - jepsen.util 9693 :ok :read [21 23] INFO [2019-12-17 12:36:23,115] jepsen worker 63 - jepsen.util 9693 :invoke :read [21 nil] INFO [2019-12-17 12:36:23,129] jepsen worker 65 - jepsen.util 9245 :ok :write [21 37] INFO [2019-12-17 12:36:23,129] jepsen worker 65 - jepsen.util 9245 :invoke :cas [21 [33 46]] INFO [2019-12-17 12:36:23,131] jepsen worker 24 - jepsen.util 13164 :info :cas [8 [37 21]] :RequestCanceledException INFO [2019-12-17 12:36:23,132] jepsen worker 24 - jepsen.util 13254 :invoke :cas [8 [18 16]] INFO [2019-12-17 12:36:23,137] jepsen worker 87 - jepsen.util 10167 :ok :write [29 19] INFO [2019-12-17 12:36:23,137] jepsen worker 87 - jepsen.util 10167 :invoke :write [29 30] INFO [2019-12-17 12:36:23,145] jepsen worker 63 - jepsen.util 9693 :ok :read [21 23] INFO [2019-12-17 12:36:23,145] jepsen worker 63 - jepsen.util 9693 :invoke :read [21 nil] INFO [2019-12-17 12:36:23,146] jepsen worker 54 - jepsen.util 11484 :fail :cas [18 [13 38]] :ValueNotSwapFrom INFO [2019-12-17 12:36:23,147] jepsen worker 54 - jepsen.util 11484 :invoke :cas [18 [10 8]] INFO [2019-12-17 12:36:23,153] jepsen worker 73 - jepsen.util 10063 :ok :write [24 14] INFO [2019-12-17 12:36:23,153] jepsen worker 28 - jepsen.util 8848 :ok :write [9 35] INFO [2019-12-17 12:36:23,153] jepsen worker 28 - jepsen.util 8848 :invoke :read [9 nil] INFO [2019-12-17 12:36:23,153] jepsen worker 73 - jepsen.util 10063 :invoke :cas [24 [2 31]] INFO [2019-12-17 12:36:23,156] jepsen worker 65 - jepsen.util 9245 :fail :cas [21 [33 46]] :ValueNotSwapFrom INFO [2019-12-17 12:36:23,156] jepsen worker 65 - jepsen.util 9245 :invoke :write [21 6] INFO [2019-12-17 12:36:23,166] jepsen worker 10 - jepsen.util 10630 :ok :read [3 46] INFO [2019-12-17 12:36:23,166] jepsen worker 10 - jepsen.util 10630 :invoke :write [3 39] INFO [2019-12-17 12:36:23,183] jepsen worker 79 - jepsen.util 11059 :fail :cas [26 [22 19]] :ValueNotSwapFrom INFO [2019-12-17 12:36:23,183] jepsen worker 79 - jepsen.util 11059 :invoke :cas [26 [30 22]] INFO [2019-12-17 12:36:23,199] jepsen worker 41 - jepsen.util 15431 :info :write [13 30] :RequestCanceledException INFO [2019-12-17 12:36:23,199] jepsen worker 60 - jepsen.util 15450 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:36:23,199] jepsen worker 76 - jepsen.util 14836 :info :write [25 1] :RequestCanceledException INFO [2019-12-17 12:36:23,199] jepsen worker 60 - jepsen.util 15450 :invoke :cas [20 [14 37]] INFO [2019-12-17 12:36:23,200] jepsen worker 41 - jepsen.util 15521 :invoke :read [13 nil] INFO [2019-12-17 12:36:23,200] jepsen worker 76 - jepsen.util 14926 :invoke :write [25 31] INFO [2019-12-17 12:36:23,201] jepsen worker 28 - jepsen.util 8848 :ok :read [9 35] INFO [2019-12-17 12:36:23,201] jepsen worker 28 - jepsen.util 8848 :invoke :read [9 nil] INFO [2019-12-17 12:36:23,216] jepsen worker 32 - jepsen.util 10832 :info :write [10 13] :RequestCanceledException INFO [2019-12-17 12:36:23,216] jepsen worker 42 - jepsen.util 15162 :info :cas [14 [11 31]] :RequestCanceledException INFO [2019-12-17 12:36:23,218] jepsen worker 70 - jepsen.util 11320 :ok :read [23 27] INFO [2019-12-17 12:36:23,218] jepsen worker 42 - jepsen.util 15252 :invoke :cas [14 [26 32]] INFO [2019-12-17 12:36:23,219] jepsen worker 70 - jepsen.util 11320 :invoke :read [23 nil] INFO [2019-12-17 12:36:23,220] jepsen worker 32 - jepsen.util 10922 :invoke :write [10 5] INFO [2019-12-17 12:36:23,227] jepsen worker 73 - jepsen.util 10063 :fail :cas [24 [2 31]] :ValueNotSwapFrom INFO [2019-12-17 12:36:23,227] jepsen worker 73 - jepsen.util 10063 :invoke :write [24 20] INFO [2019-12-17 12:36:23,230] jepsen worker 18 - jepsen.util 10818 :ok :write [6 45] INFO [2019-12-17 12:36:23,230] jepsen worker 18 - jepsen.util 10818 :invoke :write [6 37] INFO [2019-12-17 12:36:23,230] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":62117,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:36:23,230] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":62105,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:36:23,230] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":62117,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:36:23,230] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":62105,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:36:23,230] jepsen worker 17 - jepsen.util 14417 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:36:23,231] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":62119,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:36:23,231] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":62107,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:36:23,231] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":62119,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:36:23,231] jepsen worker 17 - jepsen.util 14417 :invoke :cas [5 [21 15]] INFO [2019-12-17 12:36:23,231] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":62107,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:36:23,231] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":62108,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:36:23,231] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":62108,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:36:23,234] jepsen worker 63 - jepsen.util 9693 :ok :read [21 37] INFO [2019-12-17 12:36:23,234] jepsen worker 63 - jepsen.util 9693 :invoke :read [21 nil] INFO [2019-12-17 12:36:23,237] jepsen worker 71 - jepsen.util 11681 :fail :cas [23 [4 48]] :ValueNotSwapFrom INFO [2019-12-17 12:36:23,237] jepsen worker 71 - jepsen.util 11681 :invoke :write [23 36] INFO [2019-12-17 12:36:23,297] jepsen worker 5 - jepsen.util 10265 :fail :cas [1 [10 21]] :ValueNotSwapFrom INFO [2019-12-17 12:36:23,297] jepsen worker 5 - jepsen.util 10265 :invoke :write [1 46] INFO [2019-12-17 12:36:23,325] jepsen worker 28 - jepsen.util 8848 :ok :read [9 35] INFO [2019-12-17 12:36:23,325] jepsen worker 63 - jepsen.util 9693 :ok :read [21 37] INFO [2019-12-17 12:36:23,325] jepsen worker 28 - jepsen.util 8848 :invoke :write [9 47] INFO [2019-12-17 12:36:23,325] jepsen worker 63 - jepsen.util 9693 :invoke :write [21 13] INFO [2019-12-17 12:36:23,325] jepsen worker 88 - jepsen.util 10978 :fail :cas [29 [46 0]] :ValueNotSwapFrom INFO [2019-12-17 12:36:23,327] jepsen worker 88 - jepsen.util 10978 :invoke :cas [29 [39 43]] INFO [2019-12-17 12:36:23,332] jepsen worker 6 - jepsen.util 14766 :info :cas [2 [2 24]] :RequestCanceledException INFO [2019-12-17 12:36:23,332] jepsen worker 11 - jepsen.util 10271 :info :cas [3 [48 9]] :RequestCanceledException INFO [2019-12-17 12:36:23,333] jepsen worker 6 - jepsen.util 14856 :invoke :cas [2 [8 25]] INFO [2019-12-17 12:36:23,333] jepsen worker 11 - jepsen.util 10361 :invoke :cas [3 [11 17]] INFO [2019-12-17 12:36:23,368] jepsen worker 12 - jepsen.util 10992 :fail :cas [4 [39 37]] :ValueNotSwapFrom INFO [2019-12-17 12:36:23,368] jepsen worker 12 - jepsen.util 10992 :invoke :read [4 nil] INFO [2019-12-17 12:36:23,374] jepsen worker 51 - jepsen.util 10131 :ok :read [17 36] INFO [2019-12-17 12:36:23,374] jepsen worker 51 - jepsen.util 10131 :invoke :cas [17 [25 16]] INFO [2019-12-17 12:36:23,403] jepsen worker 65 - jepsen.util 9245 :ok :write [21 6] INFO [2019-12-17 12:36:23,403] jepsen worker 65 - jepsen.util 9245 :invoke :read [21 nil] INFO [2019-12-17 12:36:23,411] jepsen worker 88 - jepsen.util 10978 :fail :cas [29 [39 43]] :ValueNotSwapFrom INFO [2019-12-17 12:36:23,412] jepsen worker 88 - jepsen.util 10978 :invoke :write [29 26] INFO [2019-12-17 12:36:23,413] jepsen worker 12 - jepsen.util 10992 :ok :read [4 21] INFO [2019-12-17 12:36:23,414] jepsen worker 12 - jepsen.util 10992 :invoke :write [4 14] INFO [2019-12-17 12:36:23,432] jepsen worker 46 - jepsen.util 10486 :info :cas [15 [7 17]] :RequestCanceledException INFO [2019-12-17 12:36:23,433] jepsen worker 46 - jepsen.util 10576 :invoke :read [15 nil] INFO [2019-12-17 12:36:23,444] jepsen worker 51 - jepsen.util 10131 :fail :cas [17 [25 16]] :ValueNotSwapFrom INFO [2019-12-17 12:36:23,445] jepsen worker 51 - jepsen.util 10131 :invoke :cas [17 [22 27]] INFO [2019-12-17 12:36:23,445] jepsen worker 65 - jepsen.util 9245 :ok :read [21 6] INFO [2019-12-17 12:36:23,445] jepsen worker 65 - jepsen.util 9245 :invoke :cas [21 [11 22]] INFO [2019-12-17 12:36:23,446] jepsen worker 19 - jepsen.util 10819 :info :cas [6 [8 34]] :RequestCanceledException INFO [2019-12-17 12:36:23,447] jepsen worker 19 - jepsen.util 10909 :invoke :cas [6 [39 8]] INFO [2019-12-17 12:36:23,462] jepsen worker 35 - jepsen.util 14525 :info :write [11 40] :RequestCanceledException INFO [2019-12-17 12:36:23,463] jepsen worker 35 - jepsen.util 14615 :invoke :cas [11 [40 10]] INFO [2019-12-17 12:36:23,469] jepsen worker 46 - jepsen.util 10576 :ok :read [15 22] INFO [2019-12-17 12:36:23,469] jepsen worker 46 - jepsen.util 10576 :invoke :write [15 9] INFO [2019-12-17 12:36:23,481] jepsen worker 51 - jepsen.util 10131 :fail :cas [17 [22 27]] :ValueNotSwapFrom INFO [2019-12-17 12:36:23,481] jepsen worker 51 - jepsen.util 10131 :invoke :read [17 nil] INFO [2019-12-17 12:36:23,494] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":62143,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:36:23,494] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":62143,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:36:23,499] jepsen worker 65 - jepsen.util 9245 :fail :cas [21 [11 22]] :ValueNotSwapFrom INFO [2019-12-17 12:36:23,499] jepsen worker 65 - jepsen.util 9245 :invoke :read [21 nil] INFO [2019-12-17 12:36:23,522] jepsen worker 70 - jepsen.util 11320 :ok :read [23 27] INFO [2019-12-17 12:36:23,523] jepsen worker 70 - jepsen.util 11320 :invoke :cas [23 [26 28]] INFO [2019-12-17 12:36:23,530] jepsen worker 37 - jepsen.util 10297 :ok :read [12 25] INFO [2019-12-17 12:36:23,530] jepsen worker 44 - jepsen.util 14894 :info :cas [14 [3 46]] :RequestCanceledException INFO [2019-12-17 12:36:23,530] jepsen worker 37 - jepsen.util 10297 :invoke :write [12 26] INFO [2019-12-17 12:36:23,531] jepsen worker 44 - jepsen.util 14984 :invoke :cas [14 [24 18]] INFO [2019-12-17 12:36:23,545] jepsen worker 62 - jepsen.util 14912 :info :write [20 44] :RequestCanceledException INFO [2019-12-17 12:36:23,546] jepsen worker 62 - jepsen.util 15002 :invoke :read [20 nil] INFO [2019-12-17 12:36:23,555] jepsen worker 73 - jepsen.util 10063 :ok :write [24 20] INFO [2019-12-17 12:36:23,555] jepsen worker 73 - jepsen.util 10063 :invoke :cas [24 [1 45]] INFO [2019-12-17 12:36:23,562] jepsen worker 63 - jepsen.util 9693 :ok :write [21 13] INFO [2019-12-17 12:36:23,562] jepsen worker 63 - jepsen.util 9693 :invoke :write [21 10] INFO [2019-12-17 12:36:23,616] jepsen worker 85 - jepsen.util 15475 :info :write [28 15] :RequestCanceledException INFO [2019-12-17 12:36:23,616] jepsen worker 23 - jepsen.util 14513 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:36:23,616] jepsen worker 23 - jepsen.util 14513 :invoke :read [7 nil] INFO [2019-12-17 12:36:23,617] jepsen worker 85 - jepsen.util 15565 :invoke :read [28 nil] INFO [2019-12-17 12:36:23,626] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:36:23,626] jepsen nemesis - couchbase.util Rebalance check count: 484 INFO [2019-12-17 12:36:23,626] jepsen nemesis - couchbase.util Rebalance stuck count: 465 INFO [2019-12-17 12:36:23,632] jepsen worker 47 - jepsen.util 11387 :info :write [15 22] :RequestCanceledException INFO [2019-12-17 12:36:23,634] jepsen worker 47 - jepsen.util 11477 :invoke :read [15 nil] INFO [2019-12-17 12:36:23,652] jepsen worker 73 - jepsen.util 10063 :fail :cas [24 [1 45]] :ValueNotSwapFrom INFO [2019-12-17 12:36:23,652] jepsen worker 73 - jepsen.util 10063 :invoke :write [24 11] INFO [2019-12-17 12:36:23,663] jepsen worker 40 - jepsen.util 14890 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:36:23,663] jepsen worker 64 - jepsen.util 8254 :info :write [21 1] :RequestCanceledException INFO [2019-12-17 12:36:23,663] jepsen worker 40 - jepsen.util 14890 :invoke :cas [13 [3 18]] INFO [2019-12-17 12:36:23,668] jepsen worker 64 - jepsen.util 8344 :invoke :read [21 nil] INFO [2019-12-17 12:36:23,707] jepsen worker 32 - jepsen.util 10922 :ok :write [10 5] INFO [2019-12-17 12:36:23,707] jepsen worker 32 - jepsen.util 10922 :invoke :cas [10 [43 11]] INFO [2019-12-17 12:36:23,709] jepsen worker 10 - jepsen.util 10630 :ok :write [3 39] INFO [2019-12-17 12:36:23,709] jepsen worker 10 - jepsen.util 10630 :invoke :cas [3 [2 29]] INFO [2019-12-17 12:36:23,719] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":62163,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:36:23,719] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":62163,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:36:23,719] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":62166,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:36:23,719] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":62166,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:36:23,719] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":62167,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:36:23,719] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":62167,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:36:23,732] jepsen worker 29 - jepsen.util 9119 :fail :read [9 nil] :RequestTimeoutException INFO [2019-12-17 12:36:23,732] jepsen worker 63 - jepsen.util 9693 :ok :write [21 10] INFO [2019-12-17 12:36:23,733] jepsen worker 29 - jepsen.util 9119 :invoke :write [9 45] INFO [2019-12-17 12:36:23,733] jepsen worker 63 - jepsen.util 9693 :invoke :cas [21 [23 30]] INFO [2019-12-17 12:36:23,746] jepsen worker 56 - jepsen.util 11216 :info :write [18 0] :RequestCanceledException INFO [2019-12-17 12:36:23,747] jepsen worker 56 - jepsen.util 11306 :invoke :cas [18 [45 31]] INFO [2019-12-17 12:36:23,763] jepsen worker 66 - jepsen.util 14376 :info :cas [22 [27 14]] :RequestCanceledException INFO [2019-12-17 12:36:23,766] jepsen worker 66 - jepsen.util 14466 :invoke :read [22 nil] INFO [2019-12-17 12:36:23,782] jepsen worker 28 - jepsen.util 8848 :ok :write [9 47] INFO [2019-12-17 12:36:23,782] jepsen worker 28 - jepsen.util 8848 :invoke :write [9 3] INFO [2019-12-17 12:36:23,784] jepsen worker 79 - jepsen.util 11059 :fail :cas [26 [30 22]] :ValueNotSwapFrom INFO [2019-12-17 12:36:23,784] jepsen worker 79 - jepsen.util 11059 :invoke :cas [26 [13 44]] INFO [2019-12-17 12:36:23,803] jepsen worker 87 - jepsen.util 10167 :ok :write [29 30] INFO [2019-12-17 12:36:23,803] jepsen worker 87 - jepsen.util 10167 :invoke :read [29 nil] INFO [2019-12-17 12:36:23,808] jepsen worker 32 - jepsen.util 10922 :fail :cas [10 [43 11]] :ValueNotSwapFrom INFO [2019-12-17 12:36:23,808] jepsen worker 32 - jepsen.util 10922 :invoke :write [10 38] INFO [2019-12-17 12:36:23,829] jepsen worker 64 - jepsen.util 8344 :ok :read [21 13] INFO [2019-12-17 12:36:23,829] jepsen worker 64 - jepsen.util 8344 :invoke :read [21 nil] INFO [2019-12-17 12:36:23,832] jepsen worker 7 - jepsen.util 14587 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:36:23,832] jepsen worker 7 - jepsen.util 14587 :invoke :cas [2 [19 4]] INFO [2019-12-17 12:36:23,845] jepsen worker 49 - jepsen.util 14359 :info :write [16 48] :RequestCanceledException INFO [2019-12-17 12:36:23,846] jepsen worker 49 - jepsen.util 14449 :invoke :read [16 nil] INFO [2019-12-17 12:36:23,847] jepsen worker 63 - jepsen.util 9693 :fail :cas [21 [23 30]] :ValueNotSwapFrom INFO [2019-12-17 12:36:23,847] jepsen worker 63 - jepsen.util 9693 :invoke :cas [21 [43 3]] INFO [2019-12-17 12:36:23,868] jepsen worker 54 - jepsen.util 11484 :fail :cas [18 [10 8]] :ValueNotSwapFrom INFO [2019-12-17 12:36:23,868] jepsen worker 54 - jepsen.util 11484 :invoke :write [18 9] INFO [2019-12-17 12:36:23,898] jepsen worker 83 - jepsen.util 14663 :info :cas [27 [26 2]] :RequestCanceledException INFO [2019-12-17 12:36:23,898] jepsen worker 61 - jepsen.util 15451 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:36:23,898] jepsen worker 61 - jepsen.util 15451 :invoke :write [20 46] INFO [2019-12-17 12:36:23,899] jepsen worker 83 - jepsen.util 14753 :invoke :cas [27 [27 25]] INFO [2019-12-17 12:36:23,913] jepsen worker 38 - jepsen.util 11288 :ok :write [12 49] INFO [2019-12-17 12:36:23,913] jepsen worker 38 - jepsen.util 11288 :invoke :read [12 nil] INFO [2019-12-17 12:36:23,914] jepsen worker 0 - jepsen.util 15390 :info :cas [0 [49 15]] :RequestCanceledException INFO [2019-12-17 12:36:23,915] jepsen worker 0 - jepsen.util 15480 :invoke :write [0 9] INFO [2019-12-17 12:36:23,916] jepsen worker 56 - jepsen.util 11306 :fail :cas [18 [45 31]] :ValueNotSwapFrom INFO [2019-12-17 12:36:23,916] jepsen worker 56 - jepsen.util 11306 :invoke :read [18 nil] INFO [2019-12-17 12:36:23,930] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":62186,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:36:23,930] jepsen worker 1 - jepsen.util 14581 :info :write [0 12] :RequestCanceledException INFO [2019-12-17 12:36:23,930] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":62186,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:36:23,930] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":62189,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:36:23,930] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":62189,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:36:23,931] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":62190,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:36:23,931] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":62190,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:36:23,932] jepsen worker 1 - jepsen.util 14671 :invoke :write [0 25] INFO [2019-12-17 12:36:23,933] jepsen worker 10 - jepsen.util 10630 :fail :cas [3 [2 29]] :ValueNotSwapFrom INFO [2019-12-17 12:36:23,934] jepsen worker 10 - jepsen.util 10630 :invoke :cas [3 [11 32]] INFO [2019-12-17 12:36:23,938] jepsen worker 46 - jepsen.util 10576 :ok :write [15 9] INFO [2019-12-17 12:36:23,938] jepsen worker 46 - jepsen.util 10576 :invoke :write [15 8] INFO [2019-12-17 12:36:23,944] jepsen worker 26 - jepsen.util 15146 :info :cas [8 [29 44]] :RequestCanceledException INFO [2019-12-17 12:36:23,948] jepsen worker 26 - jepsen.util 15236 :invoke :cas [8 [8 48]] INFO [2019-12-17 12:36:23,953] jepsen worker 63 - jepsen.util 9693 :fail :cas [21 [43 3]] :ValueNotSwapFrom INFO [2019-12-17 12:36:23,953] jepsen worker 63 - jepsen.util 9693 :invoke :read [21 nil] INFO [2019-12-17 12:36:23,963] jepsen worker 64 - jepsen.util 8344 :ok :read [21 10] INFO [2019-12-17 12:36:23,963] jepsen worker 64 - jepsen.util 8344 :invoke :cas [21 [27 11]] INFO [2019-12-17 12:36:23,969] jepsen worker 73 - jepsen.util 10063 :ok :write [24 11] INFO [2019-12-17 12:36:23,969] jepsen worker 73 - jepsen.util 10063 :invoke :write [24 37] INFO [2019-12-17 12:36:23,982] jepsen worker 29 - jepsen.util 9119 :ok :write [9 45] INFO [2019-12-17 12:36:23,982] jepsen worker 29 - jepsen.util 9119 :invoke :write [9 0] INFO [2019-12-17 12:36:24,004] jepsen worker 53 - jepsen.util 9953 :ok :write [17 27] INFO [2019-12-17 12:36:24,004] jepsen worker 53 - jepsen.util 9953 :invoke :cas [17 [48 49]] INFO [2019-12-17 12:36:24,028] jepsen worker 47 - jepsen.util 11477 :ok :read [15 9] INFO [2019-12-17 12:36:24,028] jepsen worker 47 - jepsen.util 11477 :invoke :cas [15 [7 35]] INFO [2019-12-17 12:36:24,031] jepsen worker 58 - jepsen.util 14998 :info :write [19 5] :RequestCanceledException INFO [2019-12-17 12:36:24,032] jepsen worker 58 - jepsen.util 15088 :invoke :cas [19 [7 12]] INFO [2019-12-17 12:36:24,042] jepsen worker 37 - jepsen.util 10297 :ok :write [12 26] INFO [2019-12-17 12:36:24,042] jepsen worker 37 - jepsen.util 10297 :invoke :write [12 5] INFO [2019-12-17 12:36:24,046] jepsen worker 5 - jepsen.util 10265 :ok :write [1 46] INFO [2019-12-17 12:36:24,046] jepsen worker 5 - jepsen.util 10265 :invoke :read [1 nil] INFO [2019-12-17 12:36:24,046] jepsen worker 75 - jepsen.util 14115 :info :write [25 47] :RequestCanceledException INFO [2019-12-17 12:36:24,047] jepsen worker 75 - jepsen.util 14205 :invoke :cas [25 [2 10]] INFO [2019-12-17 12:36:24,050] jepsen worker 10 - jepsen.util 10630 :fail :cas [3 [11 32]] :ValueNotSwapFrom INFO [2019-12-17 12:36:24,050] jepsen worker 10 - jepsen.util 10630 :invoke :write [3 6] INFO [2019-12-17 12:36:24,053] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":62205,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:36:24,053] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":62205,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:36:24,053] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":62206,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:36:24,053] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":62206,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:36:24,098] jepsen worker 74 - jepsen.util 10604 :info :cas [24 [10 27]] :RequestCanceledException INFO [2019-12-17 12:36:24,099] jepsen worker 74 - jepsen.util 10694 :invoke :read [24 nil] INFO [2019-12-17 12:36:24,115] jepsen worker 63 - jepsen.util 9693 :ok :read [21 10] INFO [2019-12-17 12:36:24,115] jepsen worker 63 - jepsen.util 9693 :invoke :read [21 nil] INFO [2019-12-17 12:36:24,132] jepsen worker 89 - jepsen.util 10889 :info :write [29 34] :RequestCanceledException INFO [2019-12-17 12:36:24,132] jepsen worker 4 - jepsen.util 9814 :info :write [1 0] :RequestCanceledException INFO [2019-12-17 12:36:24,133] jepsen worker 4 - jepsen.util 9904 :invoke :cas [1 [5 13]] INFO [2019-12-17 12:36:24,133] jepsen worker 89 - jepsen.util 10979 :invoke :cas [29 [34 24]] INFO [2019-12-17 12:36:24,143] jepsen worker 70 - jepsen.util 11320 :fail :cas [23 [26 28]] :ValueNotSwapFrom INFO [2019-12-17 12:36:24,143] jepsen worker 70 - jepsen.util 11320 :invoke :read [23 nil] INFO [2019-12-17 12:36:24,172] jepsen worker 88 - jepsen.util 10978 :ok :write [29 26] INFO [2019-12-17 12:36:24,172] jepsen worker 88 - jepsen.util 10978 :invoke :read [29 nil] INFO [2019-12-17 12:36:24,186] jepsen worker 64 - jepsen.util 8344 :fail :cas [21 [27 11]] :ValueNotSwapFrom INFO [2019-12-17 12:36:24,187] jepsen worker 64 - jepsen.util 8344 :invoke :write [21 20] INFO [2019-12-17 12:36:24,199] jepsen worker 51 - jepsen.util 10131 :ok :read [17 27] INFO [2019-12-17 12:36:24,199] jepsen worker 51 - jepsen.util 10131 :invoke :cas [17 [34 49]] INFO [2019-12-17 12:36:24,203] jepsen worker 3 - jepsen.util 10353 :ok :write [1 2] INFO [2019-12-17 12:36:24,204] jepsen worker 3 - jepsen.util 10353 :invoke :write [1 2] INFO [2019-12-17 12:36:24,210] jepsen worker 38 - jepsen.util 11288 :ok :read [12 26] INFO [2019-12-17 12:36:24,210] jepsen worker 38 - jepsen.util 11288 :invoke :write [12 37] INFO [2019-12-17 12:36:24,217] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":62226,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:36:24,217] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":62226,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:36:24,217] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":62228,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:36:24,217] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":62228,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:36:24,232] jepsen worker 78 - jepsen.util 10788 :info :write [26 36] :RequestCanceledException INFO [2019-12-17 12:36:24,235] jepsen worker 78 - jepsen.util 10878 :invoke :write [26 17] INFO [2019-12-17 12:36:24,241] jepsen worker 74 - jepsen.util 10694 :ok :read [24 11] INFO [2019-12-17 12:36:24,241] jepsen worker 74 - jepsen.util 10694 :invoke :write [24 3] INFO [2019-12-17 12:36:24,245] jepsen worker 68 - jepsen.util 13928 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:36:24,246] jepsen worker 68 - jepsen.util 13928 :invoke :read [22 nil] INFO [2019-12-17 12:36:24,259] jepsen worker 82 - jepsen.util 14752 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:36:24,259] jepsen worker 82 - jepsen.util 14752 :invoke :cas [27 [9 9]] INFO [2019-12-17 12:36:24,261] jepsen worker 5 - jepsen.util 10265 :ok :read [1 2] INFO [2019-12-17 12:36:24,262] jepsen worker 5 - jepsen.util 10265 :invoke :cas [1 [1 37]] INFO [2019-12-17 12:36:24,264] jepsen worker 79 - jepsen.util 11059 :fail :cas [26 [13 44]] :ValueNotSwapFrom INFO [2019-12-17 12:36:24,264] jepsen worker 79 - jepsen.util 11059 :invoke :write [26 30] INFO [2019-12-17 12:36:24,267] jepsen worker 51 - jepsen.util 10131 :fail :cas [17 [34 49]] :ValueNotSwapFrom INFO [2019-12-17 12:36:24,267] jepsen worker 51 - jepsen.util 10131 :invoke :cas [17 [45 25]] INFO [2019-12-17 12:36:24,279] jepsen worker 63 - jepsen.util 9693 :ok :read [21 10] INFO [2019-12-17 12:36:24,280] jepsen worker 63 - jepsen.util 9693 :invoke :cas [21 [3 14]] INFO [2019-12-17 12:36:24,301] jepsen worker 87 - jepsen.util 10167 :ok :read [29 26] INFO [2019-12-17 12:36:24,301] jepsen worker 87 - jepsen.util 10167 :invoke :read [29 nil] INFO [2019-12-17 12:36:24,317] jepsen worker 29 - jepsen.util 9119 :ok :write [9 0] INFO [2019-12-17 12:36:24,318] jepsen worker 29 - jepsen.util 9119 :invoke :read [9 nil] INFO [2019-12-17 12:36:24,327] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":62235,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:36:24,327] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":62235,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:36:24,327] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":62238,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:36:24,327] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":62238,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:36:24,327] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":62239,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:36:24,328] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":62239,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:36:24,332] jepsen worker 25 - jepsen.util 15595 :info :cas [8 [14 0]] :RequestCanceledException INFO [2019-12-17 12:36:24,332] jepsen worker 52 - jepsen.util 10492 :info :write [17 4] :RequestCanceledException INFO [2019-12-17 12:36:24,333] jepsen worker 52 - jepsen.util 10582 :invoke :read [17 nil] INFO [2019-12-17 12:36:24,333] jepsen worker 25 - jepsen.util 15685 :invoke :read [8 nil] INFO [2019-12-17 12:36:24,352] jepsen worker 29 - jepsen.util 9119 :ok :read [9 0] INFO [2019-12-17 12:36:24,353] jepsen worker 29 - jepsen.util 9119 :invoke :cas [9 [23 23]] INFO [2019-12-17 12:36:24,363] jepsen worker 45 - jepsen.util 10845 :fail :read [15 nil] :RequestTimeoutException INFO [2019-12-17 12:36:24,363] jepsen worker 89 - jepsen.util 10979 :fail :cas [29 [34 24]] :ValueNotSwapFrom INFO [2019-12-17 12:36:24,363] jepsen worker 45 - jepsen.util 10845 :invoke :cas [15 [41 18]] INFO [2019-12-17 12:36:24,363] jepsen worker 89 - jepsen.util 10979 :invoke :write [29 34] INFO [2019-12-17 12:36:24,389] jepsen worker 70 - jepsen.util 11320 :ok :read [23 27] INFO [2019-12-17 12:36:24,389] jepsen worker 70 - jepsen.util 11320 :invoke :cas [23 [0 17]] INFO [2019-12-17 12:36:24,396] jepsen worker 29 - jepsen.util 9119 :fail :cas [9 [23 23]] :ValueNotSwapFrom INFO [2019-12-17 12:36:24,396] jepsen worker 29 - jepsen.util 9119 :invoke :write [9 28] INFO [2019-12-17 12:36:24,398] jepsen worker 14 - jepsen.util 10454 :info :write [4 32] :RequestCanceledException INFO [2019-12-17 12:36:24,399] jepsen worker 14 - jepsen.util 10544 :invoke :read [4 nil] INFO [2019-12-17 12:36:24,405] jepsen worker 53 - jepsen.util 9953 :fail :cas [17 [48 49]] :ValueNotSwapFrom INFO [2019-12-17 12:36:24,405] jepsen worker 53 - jepsen.util 9953 :invoke :write [17 12] INFO [2019-12-17 12:36:24,416] jepsen worker 27 - jepsen.util 10107 :info :write [9 33] :RequestCanceledException INFO [2019-12-17 12:36:24,416] jepsen worker 48 - jepsen.util 13638 :info :write [16 5] :RequestCanceledException INFO [2019-12-17 12:36:24,417] jepsen worker 27 - jepsen.util 10197 :invoke :write [9 21] INFO [2019-12-17 12:36:24,417] jepsen worker 48 - jepsen.util 13728 :invoke :read [16 nil] INFO [2019-12-17 12:36:24,425] jepsen worker 45 - jepsen.util 10845 :fail :cas [15 [41 18]] :ValueNotSwapFrom INFO [2019-12-17 12:36:24,425] jepsen worker 45 - jepsen.util 10845 :invoke :write [15 5] INFO [2019-12-17 12:36:24,437] jepsen worker 51 - jepsen.util 10131 :fail :cas [17 [45 25]] :ValueNotSwapFrom INFO [2019-12-17 12:36:24,437] jepsen worker 51 - jepsen.util 10131 :invoke :cas [17 [37 36]] INFO [2019-12-17 12:36:24,438] jepsen worker 4 - jepsen.util 9904 :fail :cas [1 [5 13]] :ValueNotSwapFrom INFO [2019-12-17 12:36:24,438] jepsen worker 4 - jepsen.util 9904 :invoke :read [1 nil] INFO [2019-12-17 12:36:24,470] jepsen worker 70 - jepsen.util 11320 :fail :cas [23 [0 17]] :ValueNotSwapFrom INFO [2019-12-17 12:36:24,470] jepsen worker 70 - jepsen.util 11320 :invoke :write [23 21] INFO [2019-12-17 12:36:24,475] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":62263,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:36:24,476] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":62263,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:36:24,476] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":62266,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:36:24,476] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":62266,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:36:24,476] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":62267,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:36:24,476] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":62267,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:36:24,487] jepsen worker 14 - jepsen.util 10544 :ok :read [4 21] INFO [2019-12-17 12:36:24,488] jepsen worker 14 - jepsen.util 10544 :invoke :read [4 nil] INFO [2019-12-17 12:36:24,489] jepsen worker 29 - jepsen.util 9119 :ok :write [9 28] INFO [2019-12-17 12:36:24,489] jepsen worker 29 - jepsen.util 9119 :invoke :cas [9 [15 36]] INFO [2019-12-17 12:36:24,501] jepsen worker 87 - jepsen.util 10167 :ok :read [29 26] INFO [2019-12-17 12:36:24,501] jepsen worker 87 - jepsen.util 10167 :invoke :read [29 nil] INFO [2019-12-17 12:36:24,511] jepsen worker 29 - jepsen.util 9119 :fail :cas [9 [15 36]] :ValueNotSwapFrom INFO [2019-12-17 12:36:24,512] jepsen worker 29 - jepsen.util 9119 :invoke :read [9 nil] INFO [2019-12-17 12:36:24,530] jepsen worker 29 - jepsen.util 9119 :ok :read [9 28] INFO [2019-12-17 12:36:24,530] jepsen worker 29 - jepsen.util 9119 :invoke :read [9 nil] INFO [2019-12-17 12:36:24,532] jepsen worker 36 - jepsen.util 11466 :info :write [12 17] :RequestCanceledException INFO [2019-12-17 12:36:24,532] jepsen worker 33 - jepsen.util 14523 :info :write [11 41] :RequestCanceledException INFO [2019-12-17 12:36:24,533] jepsen worker 33 - jepsen.util 14613 :invoke :write [11 16] INFO [2019-12-17 12:36:24,533] jepsen worker 36 - jepsen.util 11556 :invoke :cas [12 [16 10]] INFO [2019-12-17 12:36:24,541] jepsen worker 5 - jepsen.util 10265 :fail :cas [1 [1 37]] :ValueNotSwapFrom INFO [2019-12-17 12:36:24,542] jepsen worker 5 - jepsen.util 10265 :invoke :write [1 33] INFO [2019-12-17 12:36:24,545] jepsen worker 73 - jepsen.util 10063 :ok :write [24 37] INFO [2019-12-17 12:36:24,545] jepsen worker 46 - jepsen.util 10576 :ok :write [15 8] INFO [2019-12-17 12:36:24,545] jepsen worker 73 - jepsen.util 10063 :invoke :write [24 16] INFO [2019-12-17 12:36:24,545] jepsen worker 46 - jepsen.util 10576 :invoke :write [15 32] INFO [2019-12-17 12:36:24,545] jepsen worker 29 - jepsen.util 9119 :ok :read [9 28] INFO [2019-12-17 12:36:24,545] jepsen worker 39 - jepsen.util 14259 :info :cas [13 [39 40]] :RequestCanceledException INFO [2019-12-17 12:36:24,546] jepsen worker 29 - jepsen.util 9119 :invoke :write [9 20] INFO [2019-12-17 12:36:24,546] jepsen worker 39 - jepsen.util 14349 :invoke :read [13 nil] INFO [2019-12-17 12:36:24,563] jepsen worker 8 - jepsen.util 14408 :info :write [2 9] :RequestCanceledException INFO [2019-12-17 12:36:24,564] jepsen worker 8 - jepsen.util 14498 :invoke :read [2 nil] INFO [2019-12-17 12:36:24,580] jepsen worker 10 - jepsen.util 10630 :ok :write [3 6] INFO [2019-12-17 12:36:24,580] jepsen worker 10 - jepsen.util 10630 :invoke :write [3 1] INFO [2019-12-17 12:36:24,582] jepsen worker 27 - jepsen.util 10197 :ok :write [9 21] INFO [2019-12-17 12:36:24,582] jepsen worker 27 - jepsen.util 10197 :invoke :read [9 nil] INFO [2019-12-17 12:36:24,596] jepsen worker 14 - jepsen.util 10544 :ok :read [4 21] INFO [2019-12-17 12:36:24,596] jepsen worker 14 - jepsen.util 10544 :invoke :read [4 nil] INFO [2019-12-17 12:36:24,598] jepsen worker 81 - jepsen.util 14121 :info :cas [27 [17 23]] :RequestCanceledException INFO [2019-12-17 12:36:24,598] jepsen worker 31 - jepsen.util 9391 :fail :read [10 nil] :RequestTimeoutException INFO [2019-12-17 12:36:24,598] jepsen worker 31 - jepsen.util 9391 :invoke :read [10 nil] INFO [2019-12-17 12:36:24,599] jepsen worker 81 - jepsen.util 14211 :invoke :cas [27 [38 34]] INFO [2019-12-17 12:36:24,612] jepsen worker 88 - jepsen.util 10978 :ok :read [29 26] INFO [2019-12-17 12:36:24,612] jepsen worker 88 - jepsen.util 10978 :invoke :read [29 nil] INFO [2019-12-17 12:36:24,624] jepsen worker 77 - jepsen.util 15287 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:36:24,624] jepsen worker 77 - jepsen.util 15287 :invoke :cas [25 [42 24]] INFO [2019-12-17 12:36:24,639] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:36:24,639] jepsen nemesis - couchbase.util Rebalance check count: 485 INFO [2019-12-17 12:36:24,639] jepsen nemesis - couchbase.util Rebalance stuck count: 466 INFO [2019-12-17 12:36:24,661] jepsen worker 69 - jepsen.util 10779 :info :write [23 3] :RequestCanceledException INFO [2019-12-17 12:36:24,661] jepsen worker 59 - jepsen.util 13649 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:36:24,661] jepsen worker 59 - jepsen.util 13649 :invoke :cas [19 [4 2]] INFO [2019-12-17 12:36:24,662] jepsen worker 69 - jepsen.util 10869 :invoke :read [23 nil] INFO [2019-12-17 12:36:24,690] jepsen worker 4 - jepsen.util 9904 :ok :read [1 2] INFO [2019-12-17 12:36:24,690] jepsen worker 4 - jepsen.util 9904 :invoke :cas [1 [23 6]] INFO [2019-12-17 12:36:24,698] jepsen worker 56 - jepsen.util 11306 :ok :read [18 9] INFO [2019-12-17 12:36:24,698] jepsen worker 56 - jepsen.util 11306 :invoke :cas [18 [42 0]] INFO [2019-12-17 12:36:24,698] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":62291,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:36:24,698] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":62291,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:36:24,699] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":62293,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:36:24,699] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":62293,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:36:24,715] jepsen worker 34 - jepsen.util 14884 :info :write [11 10] :RequestCanceledException INFO [2019-12-17 12:36:24,716] jepsen worker 34 - jepsen.util 14974 :invoke :write [11 32] INFO [2019-12-17 12:36:24,730] jepsen worker 12 - jepsen.util 10992 :ok :write [4 14] INFO [2019-12-17 12:36:24,730] jepsen worker 12 - jepsen.util 10992 :invoke :read [4 nil] INFO [2019-12-17 12:36:24,732] jepsen worker 16 - jepsen.util 15946 :info :cas [5 [11 35]] :RequestCanceledException INFO [2019-12-17 12:36:24,732] jepsen worker 72 - jepsen.util 10872 :info :write [24 26] :RequestCanceledException INFO [2019-12-17 12:36:24,733] jepsen worker 16 - jepsen.util 16036 :invoke :cas [5 [6 1]] INFO [2019-12-17 12:36:24,733] jepsen worker 72 - jepsen.util 10962 :invoke :read [24 nil] INFO [2019-12-17 12:36:24,750] jepsen worker 72 - jepsen.util 10962 :ok :read [24 37] INFO [2019-12-17 12:36:24,750] jepsen worker 72 - jepsen.util 10962 :invoke :write [24 32] INFO [2019-12-17 12:36:24,751] jepsen worker 73 - jepsen.util 10063 :ok :write [24 16] INFO [2019-12-17 12:36:24,751] jepsen worker 46 - jepsen.util 10576 :ok :write [15 32] INFO [2019-12-17 12:36:24,751] jepsen worker 73 - jepsen.util 10063 :invoke :read [24 nil] INFO [2019-12-17 12:36:24,751] jepsen worker 46 - jepsen.util 10576 :invoke :write [15 37] INFO [2019-12-17 12:36:24,760] jepsen worker 70 - jepsen.util 11320 :ok :write [23 21] INFO [2019-12-17 12:36:24,760] jepsen worker 70 - jepsen.util 11320 :invoke :write [23 29] INFO [2019-12-17 12:36:24,771] jepsen worker 10 - jepsen.util 10630 :ok :write [3 1] INFO [2019-12-17 12:36:24,771] jepsen worker 10 - jepsen.util 10630 :invoke :write [3 19] INFO [2019-12-17 12:36:24,798] jepsen worker 13 - jepsen.util 10183 :info :write [4 29] :RequestCanceledException INFO [2019-12-17 12:36:24,798] jepsen worker 21 - jepsen.util 14061 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:36:24,798] jepsen worker 21 - jepsen.util 14061 :invoke :write [7 26] INFO [2019-12-17 12:36:24,799] jepsen worker 13 - jepsen.util 10273 :invoke :cas [4 [43 26]] INFO [2019-12-17 12:36:24,811] jepsen worker 69 - jepsen.util 10869 :ok :read [23 21] INFO [2019-12-17 12:36:24,812] jepsen worker 69 - jepsen.util 10869 :invoke :cas [23 [40 36]] INFO [2019-12-17 12:36:24,822] jepsen worker 51 - jepsen.util 10131 :fail :cas [17 [37 36]] :ValueNotSwapFrom INFO [2019-12-17 12:36:24,822] jepsen worker 51 - jepsen.util 10131 :invoke :cas [17 [48 6]] INFO [2019-12-17 12:36:24,832] jepsen worker 84 - jepsen.util 14214 :info :cas [28 [9 36]] :RequestCanceledException INFO [2019-12-17 12:36:24,832] jepsen worker 20 - jepsen.util 10550 :fail :read [6 nil] :RequestTimeoutException INFO [2019-12-17 12:36:24,832] jepsen worker 20 - jepsen.util 10550 :invoke :cas [6 [25 18]] INFO [2019-12-17 12:36:24,834] jepsen worker 84 - jepsen.util 14304 :invoke :cas [28 [23 6]] INFO [2019-12-17 12:36:24,844] jepsen worker 73 - jepsen.util 10063 :ok :read [24 16] INFO [2019-12-17 12:36:24,844] jepsen worker 73 - jepsen.util 10063 :invoke :read [24 nil] INFO [2019-12-17 12:36:24,846] jepsen worker 67 - jepsen.util 14827 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:36:24,846] jepsen worker 22 - jepsen.util 14422 :info :write [7 6] :RequestCanceledException INFO [2019-12-17 12:36:24,846] jepsen worker 67 - jepsen.util 14827 :invoke :read [22 nil] INFO [2019-12-17 12:36:24,847] jepsen worker 22 - jepsen.util 14512 :invoke :cas [7 [18 29]] INFO [2019-12-17 12:36:24,858] jepsen worker 14 - jepsen.util 10544 :ok :read [4 14] INFO [2019-12-17 12:36:24,858] jepsen worker 14 - jepsen.util 10544 :invoke :cas [4 [28 13]] INFO [2019-12-17 12:36:24,863] jepsen worker 86 - jepsen.util 15566 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:36:24,863] jepsen worker 86 - jepsen.util 15566 :invoke :read [28 nil] INFO [2019-12-17 12:36:24,876] jepsen worker 73 - jepsen.util 10063 :ok :read [24 16] INFO [2019-12-17 12:36:24,876] jepsen worker 73 - jepsen.util 10063 :invoke :cas [24 [49 39]] INFO [2019-12-17 12:36:24,885] jepsen worker 20 - jepsen.util 10550 :fail :cas [6 [25 18]] :ValueNotSwapFrom INFO [2019-12-17 12:36:24,885] jepsen worker 20 - jepsen.util 10550 :invoke :write [6 2] INFO [2019-12-17 12:36:24,894] jepsen worker 88 - jepsen.util 10978 :ok :read [29 26] INFO [2019-12-17 12:36:24,895] jepsen worker 88 - jepsen.util 10978 :invoke :cas [29 [20 7]] INFO [2019-12-17 12:36:24,915] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":62322,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:36:24,915] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":62322,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:36:24,916] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":62325,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:36:24,916] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":62325,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:36:24,918] jepsen worker 12 - jepsen.util 10992 :ok :read [4 14] INFO [2019-12-17 12:36:24,918] jepsen worker 12 - jepsen.util 10992 :invoke :write [4 33] INFO [2019-12-17 12:36:24,926] jepsen worker 73 - jepsen.util 10063 :fail :cas [24 [49 39]] :ValueNotSwapFrom INFO [2019-12-17 12:36:24,926] jepsen worker 73 - jepsen.util 10063 :invoke :cas [24 [48 17]] INFO [2019-12-17 12:36:24,932] jepsen worker 55 - jepsen.util 11215 :info :write [18 42] :RequestCanceledException INFO [2019-12-17 12:36:24,932] jepsen worker 43 - jepsen.util 14623 :info :write [14 6] :RequestCanceledException INFO [2019-12-17 12:36:24,933] jepsen worker 55 - jepsen.util 11305 :invoke :cas [18 [5 0]] INFO [2019-12-17 12:36:24,933] jepsen worker 43 - jepsen.util 14713 :invoke :read [14 nil] INFO [2019-12-17 12:36:24,942] jepsen worker 13 - jepsen.util 10273 :fail :cas [4 [43 26]] :ValueNotSwapFrom INFO [2019-12-17 12:36:24,942] jepsen worker 13 - jepsen.util 10273 :invoke :read [4 nil] INFO [2019-12-17 12:36:24,942] jepsen worker 73 - jepsen.util 10063 :fail :cas [24 [48 17]] :ValueNotSwapFrom INFO [2019-12-17 12:36:24,942] jepsen worker 73 - jepsen.util 10063 :invoke :cas [24 [39 30]] INFO [2019-12-17 12:36:24,946] jepsen worker 87 - jepsen.util 10167 :ok :read [29 26] INFO [2019-12-17 12:36:24,946] jepsen worker 87 - jepsen.util 10167 :invoke :cas [29 [7 7]] INFO [2019-12-17 12:36:24,962] jepsen worker 14 - jepsen.util 10544 :fail :cas [4 [28 13]] :ValueNotSwapFrom INFO [2019-12-17 12:36:24,963] jepsen worker 14 - jepsen.util 10544 :invoke :read [4 nil] INFO [2019-12-17 12:36:24,969] jepsen worker 73 - jepsen.util 10063 :fail :cas [24 [39 30]] :ValueNotSwapFrom INFO [2019-12-17 12:36:24,970] jepsen worker 73 - jepsen.util 10063 :invoke :write [24 47] INFO [2019-12-17 12:36:24,985] jepsen worker 56 - jepsen.util 11306 :fail :cas [18 [42 0]] :ValueNotSwapFrom INFO [2019-12-17 12:36:24,985] jepsen worker 56 - jepsen.util 11306 :invoke :cas [18 [43 2]] INFO [2019-12-17 12:36:24,998] jepsen worker 50 - jepsen.util 15170 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:36:24,998] jepsen worker 50 - jepsen.util 15170 :invoke :cas [16 [28 35]] INFO [2019-12-17 12:36:25,016] jepsen worker 80 - jepsen.util 10430 :info :write [26 20] :RequestCanceledException INFO [2019-12-17 12:36:25,017] jepsen worker 80 - jepsen.util 10520 :invoke :read [26 nil] INFO [2019-12-17 12:36:25,027] jepsen worker 52 - jepsen.util 10582 :ok :read [17 27] INFO [2019-12-17 12:36:25,027] jepsen worker 52 - jepsen.util 10582 :invoke :write [17 3] INFO [2019-12-17 12:36:25,038] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":62342,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:36:25,038] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":62342,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:36:25,038] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":62346,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:36:25,038] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":62346,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:36:25,058] jepsen worker 36 - jepsen.util 11556 :fail :cas [12 [16 10]] :ValueNotSwapFrom INFO [2019-12-17 12:36:25,058] jepsen worker 36 - jepsen.util 11556 :invoke :read [12 nil] INFO [2019-12-17 12:36:25,063] jepsen worker 30 - jepsen.util 10290 :fail :read [10 nil] :RequestTimeoutException INFO [2019-12-17 12:36:25,063] jepsen worker 30 - jepsen.util 10290 :invoke :read [10 nil] INFO [2019-12-17 12:36:25,076] jepsen worker 30 - jepsen.util 10290 :ok :read [10 5] INFO [2019-12-17 12:36:25,076] jepsen worker 30 - jepsen.util 10290 :invoke :write [10 33] INFO [2019-12-17 12:36:25,116] jepsen worker 37 - jepsen.util 10297 :ok :write [12 5] INFO [2019-12-17 12:36:25,116] jepsen worker 37 - jepsen.util 10297 :invoke :cas [12 [43 11]] INFO [2019-12-17 12:36:25,117] jepsen worker 57 - jepsen.util 14367 :info :write [19 34] :RequestCanceledException INFO [2019-12-17 12:36:25,118] jepsen worker 57 - jepsen.util 14457 :invoke :write [19 5] INFO [2019-12-17 12:36:25,120] jepsen worker 4 - jepsen.util 9904 :fail :cas [1 [23 6]] :ValueNotSwapFrom INFO [2019-12-17 12:36:25,120] jepsen worker 4 - jepsen.util 9904 :invoke :cas [1 [22 43]] INFO [2019-12-17 12:36:25,131] jepsen worker 9 - jepsen.util 11529 :info :cas [3 [0 5]] :RequestCanceledException INFO [2019-12-17 12:36:25,131] jepsen worker 15 - jepsen.util 13515 :info :cas [5 [17 18]] :RequestCanceledException INFO [2019-12-17 12:36:25,133] jepsen worker 15 - jepsen.util 13605 :invoke :cas [5 [4 18]] INFO [2019-12-17 12:36:25,134] jepsen worker 51 - jepsen.util 10131 :fail :cas [17 [48 6]] :ValueNotSwapFrom INFO [2019-12-17 12:36:25,134] jepsen worker 51 - jepsen.util 10131 :invoke :read [17 nil] INFO [2019-12-17 12:36:25,137] jepsen worker 9 - jepsen.util 11619 :invoke :cas [3 [46 42]] INFO [2019-12-17 12:36:25,145] jepsen worker 2 - jepsen.util 14312 :info :cas [0 [20 13]] :RequestCanceledException INFO [2019-12-17 12:36:25,146] jepsen worker 2 - jepsen.util 14402 :invoke :read [0 nil] INFO [2019-12-17 12:36:25,151] jepsen worker 89 - jepsen.util 10979 :ok :write [29 34] INFO [2019-12-17 12:36:25,151] jepsen worker 89 - jepsen.util 10979 :invoke :read [29 nil] INFO [2019-12-17 12:36:25,163] jepsen worker 24 - jepsen.util 13254 :info :cas [8 [18 16]] :RequestCanceledException INFO [2019-12-17 12:36:25,164] jepsen worker 24 - jepsen.util 13344 :invoke :read [8 nil] INFO [2019-12-17 12:36:25,203] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":62362,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:36:25,203] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":62362,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:36:25,203] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":62363,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:36:25,203] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":62363,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:36:25,227] jepsen worker 52 - jepsen.util 10582 :ok :write [17 3] INFO [2019-12-17 12:36:25,227] jepsen worker 52 - jepsen.util 10582 :invoke :read [17 nil] INFO [2019-12-17 12:36:25,232] jepsen worker 41 - jepsen.util 15521 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:36:25,232] jepsen worker 41 - jepsen.util 15521 :invoke :cas [13 [44 23]] INFO [2019-12-17 12:36:25,245] jepsen worker 38 - jepsen.util 11288 :ok :write [12 37] INFO [2019-12-17 12:36:25,245] jepsen worker 38 - jepsen.util 11288 :invoke :cas [12 [27 21]] INFO [2019-12-17 12:36:25,290] jepsen worker 14 - jepsen.util 10544 :ok :read [4 14] INFO [2019-12-17 12:36:25,290] jepsen worker 14 - jepsen.util 10544 :invoke :write [4 7] INFO [2019-12-17 12:36:25,298] jepsen worker 60 - jepsen.util 15450 :info :cas [20 [14 37]] :RequestCanceledException INFO [2019-12-17 12:36:25,298] jepsen worker 76 - jepsen.util 14926 :info :write [25 31] :RequestCanceledException INFO [2019-12-17 12:36:25,298] jepsen worker 71 - jepsen.util 11681 :info :write [23 36] :RequestCanceledException INFO [2019-12-17 12:36:25,299] jepsen worker 71 - jepsen.util 11771 :invoke :write [23 38] INFO [2019-12-17 12:36:25,299] jepsen worker 60 - jepsen.util 15540 :invoke :cas [20 [43 5]] INFO [2019-12-17 12:36:25,299] jepsen worker 76 - jepsen.util 15016 :invoke :cas [25 [23 4]] INFO [2019-12-17 12:36:25,306] jepsen worker 13 - jepsen.util 10273 :ok :read [4 14] INFO [2019-12-17 12:36:25,306] jepsen worker 13 - jepsen.util 10273 :invoke :read [4 nil] INFO [2019-12-17 12:36:25,313] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":62368,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:36:25,313] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":62368,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:36:25,313] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":62371,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:36:25,313] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":62371,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:36:25,316] jepsen worker 42 - jepsen.util 15252 :info :cas [14 [26 32]] :RequestCanceledException INFO [2019-12-17 12:36:25,317] jepsen worker 42 - jepsen.util 15342 :invoke :cas [14 [41 26]] INFO [2019-12-17 12:36:25,332] jepsen worker 17 - jepsen.util 14417 :info :cas [5 [21 15]] :RequestCanceledException INFO [2019-12-17 12:36:25,332] jepsen worker 18 - jepsen.util 10818 :info :write [6 37] :RequestCanceledException INFO [2019-12-17 12:36:25,332] jepsen worker 17 - jepsen.util 14507 :invoke :cas [5 [3 44]] INFO [2019-12-17 12:36:25,333] jepsen worker 18 - jepsen.util 10908 :invoke :read [6 nil] INFO [2019-12-17 12:36:25,342] jepsen worker 53 - jepsen.util 9953 :ok :write [17 12] INFO [2019-12-17 12:36:25,342] jepsen worker 53 - jepsen.util 9953 :invoke :read [17 nil] INFO [2019-12-17 12:36:25,345] jepsen worker 6 - jepsen.util 14856 :info :cas [2 [8 25]] :RequestCanceledException INFO [2019-12-17 12:36:25,346] jepsen worker 6 - jepsen.util 14946 :invoke :write [2 29] INFO [2019-12-17 12:36:25,359] jepsen worker 11 - jepsen.util 10361 :info :cas [3 [11 17]] :RequestCanceledException INFO [2019-12-17 12:36:25,360] jepsen worker 11 - jepsen.util 10451 :invoke :read [3 nil] INFO [2019-12-17 12:36:25,362] jepsen worker 51 - jepsen.util 10131 :ok :read [17 3] INFO [2019-12-17 12:36:25,362] jepsen worker 51 - jepsen.util 10131 :invoke :cas [17 [40 5]] INFO [2019-12-17 12:36:25,376] jepsen worker 88 - jepsen.util 10978 :fail :cas [29 [20 7]] :ValueNotSwapFrom INFO [2019-12-17 12:36:25,376] jepsen worker 88 - jepsen.util 10978 :invoke :read [29 nil] INFO [2019-12-17 12:36:25,392] jepsen worker 10 - jepsen.util 10630 :ok :write [3 19] INFO [2019-12-17 12:36:25,392] jepsen worker 20 - jepsen.util 10550 :ok :write [6 2] INFO [2019-12-17 12:36:25,392] jepsen worker 10 - jepsen.util 10630 :invoke :write [3 39] INFO [2019-12-17 12:36:25,392] jepsen worker 20 - jepsen.util 10550 :invoke :write [6 23] INFO [2019-12-17 12:36:25,397] jepsen worker 46 - jepsen.util 10576 :ok :write [15 37] INFO [2019-12-17 12:36:25,398] jepsen worker 46 - jepsen.util 10576 :invoke :read [15 nil] INFO [2019-12-17 12:36:25,400] jepsen worker 72 - jepsen.util 10962 :ok :write [24 32] INFO [2019-12-17 12:36:25,401] jepsen worker 72 - jepsen.util 10962 :invoke :write [24 49] INFO [2019-12-17 12:36:25,407] jepsen worker 78 - jepsen.util 10878 :ok :write [26 17] INFO [2019-12-17 12:36:25,407] jepsen worker 78 - jepsen.util 10878 :invoke :read [26 nil] INFO [2019-12-17 12:36:25,411] jepsen worker 5 - jepsen.util 10265 :ok :write [1 33] INFO [2019-12-17 12:36:25,411] jepsen worker 5 - jepsen.util 10265 :invoke :read [1 nil] INFO [2019-12-17 12:36:25,436] jepsen worker 30 - jepsen.util 10290 :ok :write [10 33] INFO [2019-12-17 12:36:25,436] jepsen worker 30 - jepsen.util 10290 :invoke :write [10 16] INFO [2019-12-17 12:36:25,460] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":62396,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:36:25,460] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":62396,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:36:25,461] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":62399,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:36:25,461] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":62399,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:36:25,464] jepsen worker 80 - jepsen.util 10520 :ok :read [26 17] INFO [2019-12-17 12:36:25,464] jepsen worker 80 - jepsen.util 10520 :invoke :read [26 nil] INFO [2019-12-17 12:36:25,467] jepsen worker 55 - jepsen.util 11305 :fail :cas [18 [5 0]] :ValueNotSwapFrom INFO [2019-12-17 12:36:25,467] jepsen worker 55 - jepsen.util 11305 :invoke :read [18 nil] INFO [2019-12-17 12:36:25,486] jepsen worker 69 - jepsen.util 10869 :fail :cas [23 [40 36]] :ValueNotSwapFrom INFO [2019-12-17 12:36:25,486] jepsen worker 69 - jepsen.util 10869 :invoke :write [23 8] INFO [2019-12-17 12:36:25,486] jepsen worker 79 - jepsen.util 11059 :ok :write [26 30] INFO [2019-12-17 12:36:25,486] jepsen worker 79 - jepsen.util 11059 :invoke :read [26 nil] INFO [2019-12-17 12:36:25,487] jepsen worker 36 - jepsen.util 11556 :ok :read [12 37] INFO [2019-12-17 12:36:25,487] jepsen worker 36 - jepsen.util 11556 :invoke :read [12 nil] INFO [2019-12-17 12:36:25,497] jepsen worker 19 - jepsen.util 10909 :info :cas [6 [39 8]] :RequestCanceledException INFO [2019-12-17 12:36:25,498] jepsen worker 19 - jepsen.util 10999 :invoke :write [6 38] INFO [2019-12-17 12:36:25,513] jepsen worker 52 - jepsen.util 10582 :ok :read [17 12] INFO [2019-12-17 12:36:25,513] jepsen worker 52 - jepsen.util 10582 :invoke :write [17 5] INFO [2019-12-17 12:36:25,514] jepsen worker 35 - jepsen.util 14615 :info :cas [11 [40 10]] :RequestCanceledException INFO [2019-12-17 12:36:25,515] jepsen worker 35 - jepsen.util 14705 :invoke :read [11 nil] INFO [2019-12-17 12:36:25,517] jepsen worker 51 - jepsen.util 10131 :fail :cas [17 [40 5]] :ValueNotSwapFrom INFO [2019-12-17 12:36:25,517] jepsen worker 51 - jepsen.util 10131 :invoke :read [17 nil] INFO [2019-12-17 12:36:25,528] jepsen worker 18 - jepsen.util 10908 :ok :read [6 2] INFO [2019-12-17 12:36:25,528] jepsen worker 18 - jepsen.util 10908 :invoke :cas [6 [14 8]] INFO [2019-12-17 12:36:25,531] jepsen worker 56 - jepsen.util 11306 :fail :cas [18 [43 2]] :ValueNotSwapFrom INFO [2019-12-17 12:36:25,531] jepsen worker 56 - jepsen.util 11306 :invoke :write [18 43] INFO [2019-12-17 12:36:25,532] jepsen worker 65 - jepsen.util 9245 :fail :read [21 nil] :RequestTimeoutException INFO [2019-12-17 12:36:25,532] jepsen worker 65 - jepsen.util 9245 :invoke :read [21 nil] INFO [2019-12-17 12:36:25,558] jepsen worker 4 - jepsen.util 9904 :fail :cas [1 [22 43]] :ValueNotSwapFrom INFO [2019-12-17 12:36:25,559] jepsen worker 4 - jepsen.util 9904 :invoke :cas [1 [20 37]] INFO [2019-12-17 12:36:25,596] jepsen worker 87 - jepsen.util 10167 :fail :cas [29 [7 7]] :ValueNotSwapFrom INFO [2019-12-17 12:36:25,596] jepsen worker 87 - jepsen.util 10167 :invoke :cas [29 [25 37]] INFO [2019-12-17 12:36:25,623] jepsen worker 11 - jepsen.util 10451 :ok :read [3 19] INFO [2019-12-17 12:36:25,623] jepsen worker 11 - jepsen.util 10451 :invoke :write [3 7] INFO [2019-12-17 12:36:25,625] jepsen worker 9 - jepsen.util 11619 :fail :cas [3 [46 42]] :ValueNotSwapFrom INFO [2019-12-17 12:36:25,626] jepsen worker 9 - jepsen.util 11619 :invoke :read [3 nil] INFO [2019-12-17 12:36:25,629] jepsen worker 44 - jepsen.util 14984 :info :cas [14 [24 18]] :RequestCanceledException INFO [2019-12-17 12:36:25,633] jepsen worker 53 - jepsen.util 9953 :ok :read [17 12] INFO [2019-12-17 12:36:25,633] jepsen worker 53 - jepsen.util 9953 :invoke :cas [17 [49 0]] INFO [2019-12-17 12:36:25,634] jepsen worker 44 - jepsen.util 15074 :invoke :write [14 26] INFO [2019-12-17 12:36:25,635] jepsen worker 36 - jepsen.util 11556 :ok :read [12 37] INFO [2019-12-17 12:36:25,635] jepsen worker 36 - jepsen.util 11556 :invoke :cas [12 [12 49]] INFO [2019-12-17 12:36:25,636] jepsen worker 55 - jepsen.util 11305 :ok :read [18 9] INFO [2019-12-17 12:36:25,636] jepsen worker 55 - jepsen.util 11305 :invoke :write [18 3] INFO [2019-12-17 12:36:25,637] jepsen worker 88 - jepsen.util 10978 :ok :read [29 34] INFO [2019-12-17 12:36:25,637] jepsen worker 88 - jepsen.util 10978 :invoke :read [29 nil] INFO [2019-12-17 12:36:25,638] jepsen worker 62 - jepsen.util 15002 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:36:25,638] jepsen worker 62 - jepsen.util 15002 :invoke :cas [20 [13 0]] INFO [2019-12-17 12:36:25,639] jepsen worker 46 - jepsen.util 10576 :ok :read [15 37] INFO [2019-12-17 12:36:25,639] jepsen worker 46 - jepsen.util 10576 :invoke :read [15 nil] INFO [2019-12-17 12:36:25,655] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:36:25,655] jepsen nemesis - couchbase.util Rebalance check count: 486 INFO [2019-12-17 12:36:25,655] jepsen nemesis - couchbase.util Rebalance stuck count: 467 INFO [2019-12-17 12:36:25,659] jepsen worker 85 - jepsen.util 15565 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:36:25,659] jepsen worker 85 - jepsen.util 15565 :invoke :read [28 nil] INFO [2019-12-17 12:36:25,673] jepsen worker 89 - jepsen.util 10979 :ok :read [29 34] INFO [2019-12-17 12:36:25,674] jepsen worker 89 - jepsen.util 10979 :invoke :cas [29 [4 34]] INFO [2019-12-17 12:36:25,675] jepsen worker 78 - jepsen.util 10878 :ok :read [26 30] INFO [2019-12-17 12:36:25,675] jepsen worker 78 - jepsen.util 10878 :invoke :cas [26 [30 46]] INFO [2019-12-17 12:36:25,676] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":62429,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:36:25,676] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":62429,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:36:25,676] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":62430,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:36:25,676] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":62430,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:36:25,677] jepsen worker 79 - jepsen.util 11059 :ok :read [26 30] INFO [2019-12-17 12:36:25,677] jepsen worker 79 - jepsen.util 11059 :invoke :read [26 nil] INFO [2019-12-17 12:36:25,679] jepsen worker 38 - jepsen.util 11288 :fail :cas [12 [27 21]] :ValueNotSwapFrom INFO [2019-12-17 12:36:25,679] jepsen worker 38 - jepsen.util 11288 :invoke :cas [12 [35 32]] INFO [2019-12-17 12:36:25,715] jepsen worker 23 - jepsen.util 14513 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:36:25,715] jepsen worker 23 - jepsen.util 14513 :invoke :write [7 15] INFO [2019-12-17 12:36:25,715] jepsen worker 4 - jepsen.util 9904 :fail :cas [1 [20 37]] :ValueNotSwapFrom INFO [2019-12-17 12:36:25,715] jepsen worker 4 - jepsen.util 9904 :invoke :cas [1 [28 43]] INFO [2019-12-17 12:36:25,717] jepsen worker 37 - jepsen.util 10297 :fail :cas [12 [43 11]] :ValueNotSwapFrom INFO [2019-12-17 12:36:25,717] jepsen worker 37 - jepsen.util 10297 :invoke :write [12 25] INFO [2019-12-17 12:36:25,757] jepsen worker 53 - jepsen.util 9953 :fail :cas [17 [49 0]] :ValueNotSwapFrom INFO [2019-12-17 12:36:25,758] jepsen worker 52 - jepsen.util 10582 :ok :write [17 5] INFO [2019-12-17 12:36:25,758] jepsen worker 53 - jepsen.util 9953 :invoke :cas [17 [16 40]] INFO [2019-12-17 12:36:25,758] jepsen worker 52 - jepsen.util 10582 :invoke :read [17 nil] INFO [2019-12-17 12:36:25,763] jepsen worker 40 - jepsen.util 14890 :info :cas [13 [3 18]] :RequestCanceledException INFO [2019-12-17 12:36:25,764] jepsen worker 40 - jepsen.util 14980 :invoke :write [13 19] INFO [2019-12-17 12:36:25,789] jepsen worker 79 - jepsen.util 11059 :ok :read [26 30] INFO [2019-12-17 12:36:25,789] jepsen worker 79 - jepsen.util 11059 :invoke :read [26 nil] INFO [2019-12-17 12:36:25,830] jepsen worker 32 - jepsen.util 10922 :info :write [10 38] :RequestCanceledException INFO [2019-12-17 12:36:25,830] jepsen worker 66 - jepsen.util 14466 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:36:25,830] jepsen worker 66 - jepsen.util 14466 :invoke :cas [22 [48 8]] INFO [2019-12-17 12:36:25,830] jepsen worker 32 - jepsen.util 11012 :invoke :read [10 nil] INFO [2019-12-17 12:36:25,846] jepsen worker 10 - jepsen.util 10630 :ok :write [3 39] INFO [2019-12-17 12:36:25,846] jepsen worker 28 - jepsen.util 8848 :info :write [9 3] :RequestCanceledException INFO [2019-12-17 12:36:25,846] jepsen worker 4 - jepsen.util 9904 :fail :cas [1 [28 43]] :ValueNotSwapFrom INFO [2019-12-17 12:36:25,846] jepsen worker 10 - jepsen.util 10630 :invoke :cas [3 [12 15]] INFO [2019-12-17 12:36:25,846] jepsen worker 4 - jepsen.util 9904 :invoke :read [1 nil] INFO [2019-12-17 12:36:25,846] jepsen worker 28 - jepsen.util 8938 :invoke :cas [9 [12 17]] INFO [2019-12-17 12:36:25,870] jepsen worker 53 - jepsen.util 9953 :fail :cas [17 [16 40]] :ValueNotSwapFrom INFO [2019-12-17 12:36:25,870] jepsen worker 53 - jepsen.util 9953 :invoke :read [17 nil] INFO [2019-12-17 12:36:25,872] jepsen worker 14 - jepsen.util 10544 :ok :write [4 7] INFO [2019-12-17 12:36:25,872] jepsen worker 14 - jepsen.util 10544 :invoke :write [4 32] INFO [2019-12-17 12:36:25,873] jepsen worker 10 - jepsen.util 10630 :fail :cas [3 [12 15]] :ValueNotSwapFrom INFO [2019-12-17 12:36:25,873] jepsen worker 10 - jepsen.util 10630 :invoke :read [3 nil] INFO [2019-12-17 12:36:25,879] jepsen worker 3 - jepsen.util 10353 :ok :write [1 2] INFO [2019-12-17 12:36:25,880] jepsen worker 3 - jepsen.util 10353 :invoke :cas [1 [1 43]] INFO [2019-12-17 12:36:25,893] jepsen worker 71 - jepsen.util 11771 :ok :write [23 38] INFO [2019-12-17 12:36:25,893] jepsen worker 71 - jepsen.util 11771 :invoke :cas [23 [8 28]] INFO [2019-12-17 12:36:25,894] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":62449,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:36:25,894] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":62449,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:36:25,894] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":62453,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:36:25,894] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":62453,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:36:25,899] jepsen worker 10 - jepsen.util 10630 :ok :read [3 39] INFO [2019-12-17 12:36:25,899] jepsen worker 10 - jepsen.util 10630 :invoke :read [3 nil] INFO [2019-12-17 12:36:25,911] jepsen worker 10 - jepsen.util 10630 :ok :read [3 39] INFO [2019-12-17 12:36:25,911] jepsen worker 10 - jepsen.util 10630 :invoke :read [3 nil] INFO [2019-12-17 12:36:25,912] jepsen worker 13 - jepsen.util 10273 :ok :read [4 7] INFO [2019-12-17 12:36:25,912] jepsen worker 13 - jepsen.util 10273 :invoke :cas [4 [3 30]] INFO [2019-12-17 12:36:25,926] jepsen worker 10 - jepsen.util 10630 :ok :read [3 39] INFO [2019-12-17 12:36:25,926] jepsen worker 10 - jepsen.util 10630 :invoke :read [3 nil] INFO [2019-12-17 12:36:25,932] jepsen worker 0 - jepsen.util 15480 :info :write [0 9] :RequestCanceledException INFO [2019-12-17 12:36:25,932] jepsen worker 7 - jepsen.util 14587 :info :cas [2 [19 4]] :RequestCanceledException INFO [2019-12-17 12:36:25,932] jepsen worker 54 - jepsen.util 11484 :info :write [18 9] :RequestCanceledException INFO [2019-12-17 12:36:25,933] jepsen worker 0 - jepsen.util 15570 :invoke :cas [0 [40 5]] INFO [2019-12-17 12:36:25,933] jepsen worker 7 - jepsen.util 14677 :invoke :cas [2 [34 40]] INFO [2019-12-17 12:36:25,933] jepsen worker 54 - jepsen.util 11574 :invoke :write [18 11] INFO [2019-12-17 12:36:25,941] jepsen worker 5 - jepsen.util 10265 :ok :read [1 2] INFO [2019-12-17 12:36:25,942] jepsen worker 5 - jepsen.util 10265 :invoke :cas [1 [47 10]] INFO [2019-12-17 12:36:25,946] jepsen worker 49 - jepsen.util 14449 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:36:25,947] jepsen worker 49 - jepsen.util 14449 :invoke :read [16 nil] INFO [2019-12-17 12:36:25,953] jepsen worker 79 - jepsen.util 11059 :ok :read [26 30] INFO [2019-12-17 12:36:25,954] jepsen worker 10 - jepsen.util 10630 :ok :read [3 39] INFO [2019-12-17 12:36:25,954] jepsen worker 79 - jepsen.util 11059 :invoke :cas [26 [26 34]] INFO [2019-12-17 12:36:25,954] jepsen worker 10 - jepsen.util 10630 :invoke :cas [3 [0 25]] INFO [2019-12-17 12:36:25,963] jepsen worker 83 - jepsen.util 14753 :info :cas [27 [27 25]] :RequestCanceledException INFO [2019-12-17 12:36:25,965] jepsen worker 83 - jepsen.util 14843 :invoke :cas [27 [46 0]] INFO [2019-12-17 12:36:25,967] jepsen worker 10 - jepsen.util 10630 :fail :cas [3 [0 25]] :ValueNotSwapFrom INFO [2019-12-17 12:36:25,967] jepsen worker 10 - jepsen.util 10630 :invoke :read [3 nil] INFO [2019-12-17 12:36:25,971] jepsen worker 69 - jepsen.util 10869 :ok :write [23 8] INFO [2019-12-17 12:36:25,971] jepsen worker 69 - jepsen.util 10869 :invoke :write [23 31] INFO [2019-12-17 12:36:25,989] jepsen worker 19 - jepsen.util 10999 :ok :write [6 38] INFO [2019-12-17 12:36:25,989] jepsen worker 72 - jepsen.util 10962 :ok :write [24 49] INFO [2019-12-17 12:36:25,989] jepsen worker 19 - jepsen.util 10999 :invoke :cas [6 [4 42]] INFO [2019-12-17 12:36:25,989] jepsen worker 72 - jepsen.util 10962 :invoke :cas [24 [48 20]] INFO [2019-12-17 12:36:25,991] jepsen worker 10 - jepsen.util 10630 :ok :read [3 39] INFO [2019-12-17 12:36:25,991] jepsen worker 10 - jepsen.util 10630 :invoke :read [3 nil] INFO [2019-12-17 12:36:25,993] jepsen worker 88 - jepsen.util 10978 :ok :read [29 34] INFO [2019-12-17 12:36:25,993] jepsen worker 88 - jepsen.util 10978 :invoke :write [29 19] INFO [2019-12-17 12:36:25,993] jepsen worker 61 - jepsen.util 15451 :info :write [20 46] :RequestCanceledException INFO [2019-12-17 12:36:25,993] jepsen worker 1 - jepsen.util 14671 :info :write [0 25] :RequestCanceledException INFO [2019-12-17 12:36:25,994] jepsen worker 61 - jepsen.util 15541 :invoke :write [20 4] INFO [2019-12-17 12:36:25,994] jepsen worker 1 - jepsen.util 14761 :invoke :write [0 3] INFO [2019-12-17 12:36:26,016] jepsen worker 26 - jepsen.util 15236 :info :cas [8 [8 48]] :RequestCanceledException INFO [2019-12-17 12:36:26,017] jepsen worker 26 - jepsen.util 15326 :invoke :write [8 36] INFO [2019-12-17 12:36:26,018] jepsen worker 36 - jepsen.util 11556 :fail :cas [12 [12 49]] :ValueNotSwapFrom INFO [2019-12-17 12:36:26,018] jepsen worker 36 - jepsen.util 11556 :invoke :cas [12 [31 23]] INFO [2019-12-17 12:36:26,024] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":62481,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:36:26,024] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":62481,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:36:26,026] jepsen worker 80 - jepsen.util 10520 :ok :read [26 30] INFO [2019-12-17 12:36:26,026] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":62485,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:36:26,026] jepsen worker 80 - jepsen.util 10520 :invoke :read [26 nil] INFO [2019-12-17 12:36:26,027] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":62485,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:36:26,038] jepsen worker 87 - jepsen.util 10167 :fail :cas [29 [25 37]] :ValueNotSwapFrom INFO [2019-12-17 12:36:26,038] jepsen worker 87 - jepsen.util 10167 :invoke :read [29 nil] INFO [2019-12-17 12:36:26,045] jepsen worker 32 - jepsen.util 11012 :ok :read [10 33] INFO [2019-12-17 12:36:26,045] jepsen worker 32 - jepsen.util 11012 :invoke :write [10 11] INFO [2019-12-17 12:36:26,057] jepsen worker 46 - jepsen.util 10576 :ok :read [15 37] INFO [2019-12-17 12:36:26,057] jepsen worker 46 - jepsen.util 10576 :invoke :write [15 14] INFO [2019-12-17 12:36:26,075] jepsen worker 51 - jepsen.util 10131 :ok :read [17 5] INFO [2019-12-17 12:36:26,075] jepsen worker 51 - jepsen.util 10131 :invoke :read [17 nil] INFO [2019-12-17 12:36:26,088] jepsen worker 5 - jepsen.util 10265 :fail :cas [1 [47 10]] :ValueNotSwapFrom INFO [2019-12-17 12:36:26,088] jepsen worker 5 - jepsen.util 10265 :invoke :write [1 49] INFO [2019-12-17 12:36:26,088] jepsen worker 30 - jepsen.util 10290 :ok :write [10 16] INFO [2019-12-17 12:36:26,088] jepsen worker 30 - jepsen.util 10290 :invoke :read [10 nil] INFO [2019-12-17 12:36:26,097] jepsen worker 10 - jepsen.util 10630 :ok :read [3 39] INFO [2019-12-17 12:36:26,097] jepsen worker 10 - jepsen.util 10630 :invoke :cas [3 [18 22]] INFO [2019-12-17 12:36:26,108] jepsen worker 52 - jepsen.util 10582 :ok :read [17 5] INFO [2019-12-17 12:36:26,108] jepsen worker 52 - jepsen.util 10582 :invoke :read [17 nil] INFO [2019-12-17 12:36:26,124] jepsen worker 19 - jepsen.util 10999 :fail :cas [6 [4 42]] :ValueNotSwapFrom INFO [2019-12-17 12:36:26,124] jepsen worker 19 - jepsen.util 10999 :invoke :write [6 15] INFO [2019-12-17 12:36:26,125] jepsen worker 38 - jepsen.util 11288 :fail :cas [12 [35 32]] :ValueNotSwapFrom INFO [2019-12-17 12:36:26,125] jepsen worker 38 - jepsen.util 11288 :invoke :read [12 nil] INFO [2019-12-17 12:36:26,127] jepsen worker 28 - jepsen.util 8938 :fail :cas [9 [12 17]] :ValueNotSwapFrom INFO [2019-12-17 12:36:26,127] jepsen worker 28 - jepsen.util 8938 :invoke :cas [9 [36 4]] INFO [2019-12-17 12:36:26,131] jepsen worker 89 - jepsen.util 10979 :fail :cas [29 [4 34]] :ValueNotSwapFrom INFO [2019-12-17 12:36:26,131] jepsen worker 89 - jepsen.util 10979 :invoke :write [29 6] INFO [2019-12-17 12:36:26,131] jepsen worker 47 - jepsen.util 11477 :info :cas [15 [7 35]] :RequestCanceledException INFO [2019-12-17 12:36:26,132] jepsen worker 47 - jepsen.util 11567 :invoke :cas [15 [8 42]] INFO [2019-12-17 12:36:26,133] jepsen worker 58 - jepsen.util 15088 :info :cas [19 [7 12]] :RequestCanceledException INFO [2019-12-17 12:36:26,133] jepsen worker 58 - jepsen.util 15178 :invoke :cas [19 [21 12]] INFO [2019-12-17 12:36:26,142] jepsen worker 10 - jepsen.util 10630 :fail :cas [3 [18 22]] :ValueNotSwapFrom INFO [2019-12-17 12:36:26,143] jepsen worker 10 - jepsen.util 10630 :invoke :cas [3 [9 22]] INFO [2019-12-17 12:36:26,145] jepsen worker 72 - jepsen.util 10962 :fail :cas [24 [48 20]] :ValueNotSwapFrom INFO [2019-12-17 12:36:26,145] jepsen worker 72 - jepsen.util 10962 :invoke :write [24 35] INFO [2019-12-17 12:36:26,145] jepsen worker 75 - jepsen.util 14205 :info :cas [25 [2 10]] :RequestCanceledException INFO [2019-12-17 12:36:26,146] jepsen worker 75 - jepsen.util 14295 :invoke :write [25 44] INFO [2019-12-17 12:36:26,164] jepsen worker 30 - jepsen.util 10290 :ok :read [10 16] INFO [2019-12-17 12:36:26,164] jepsen worker 30 - jepsen.util 10290 :invoke :write [10 16] INFO [2019-12-17 12:36:26,168] jepsen worker 47 - jepsen.util 11567 :fail :cas [15 [8 42]] :ValueNotSwapFrom INFO [2019-12-17 12:36:26,168] jepsen worker 47 - jepsen.util 11567 :invoke :cas [15 [5 45]] INFO [2019-12-17 12:36:26,184] jepsen worker 4 - jepsen.util 9904 :ok :read [1 2] INFO [2019-12-17 12:36:26,184] jepsen worker 4 - jepsen.util 9904 :invoke :write [1 4] INFO [2019-12-17 12:36:26,196] jepsen worker 13 - jepsen.util 10273 :fail :cas [4 [3 30]] :ValueNotSwapFrom INFO [2019-12-17 12:36:26,196] jepsen worker 13 - jepsen.util 10273 :invoke :read [4 nil] INFO [2019-12-17 12:36:26,208] jepsen worker 79 - jepsen.util 11059 :fail :cas [26 [26 34]] :ValueNotSwapFrom INFO [2019-12-17 12:36:26,208] jepsen worker 79 - jepsen.util 11059 :invoke :write [26 30] INFO [2019-12-17 12:36:26,208] jepsen worker 36 - jepsen.util 11556 :fail :cas [12 [31 23]] :ValueNotSwapFrom INFO [2019-12-17 12:36:26,208] jepsen worker 36 - jepsen.util 11556 :invoke :read [12 nil] INFO [2019-12-17 12:36:26,210] jepsen worker 53 - jepsen.util 9953 :ok :read [17 5] INFO [2019-12-17 12:36:26,210] jepsen worker 53 - jepsen.util 9953 :invoke :cas [17 [9 45]] INFO [2019-12-17 12:36:26,219] jepsen worker 28 - jepsen.util 8938 :fail :cas [9 [36 4]] :ValueNotSwapFrom INFO [2019-12-17 12:36:26,219] jepsen worker 28 - jepsen.util 8938 :invoke :write [9 28] INFO [2019-12-17 12:36:26,225] jepsen worker 10 - jepsen.util 10630 :fail :cas [3 [9 22]] :ValueNotSwapFrom INFO [2019-12-17 12:36:26,225] jepsen worker 10 - jepsen.util 10630 :invoke :read [3 nil] INFO [2019-12-17 12:36:26,231] jepsen worker 52 - jepsen.util 10582 :ok :read [17 5] INFO [2019-12-17 12:36:26,231] jepsen worker 52 - jepsen.util 10582 :invoke :write [17 10] INFO [2019-12-17 12:36:26,239] jepsen worker 47 - jepsen.util 11567 :fail :cas [15 [5 45]] :ValueNotSwapFrom INFO [2019-12-17 12:36:26,239] jepsen worker 47 - jepsen.util 11567 :invoke :cas [15 [24 31]] INFO [2019-12-17 12:36:26,250] jepsen worker 36 - jepsen.util 11556 :ok :read [12 37] INFO [2019-12-17 12:36:26,250] jepsen worker 36 - jepsen.util 11556 :invoke :read [12 nil] INFO [2019-12-17 12:36:26,250] jepsen worker 10 - jepsen.util 10630 :ok :read [3 39] INFO [2019-12-17 12:36:26,250] jepsen worker 10 - jepsen.util 10630 :invoke :read [3 nil] INFO [2019-12-17 12:36:26,263] jepsen worker 87 - jepsen.util 10167 :ok :read [29 34] INFO [2019-12-17 12:36:26,263] jepsen worker 74 - jepsen.util 10694 :info :write [24 3] :RequestCanceledException INFO [2019-12-17 12:36:26,263] jepsen worker 87 - jepsen.util 10167 :invoke :read [29 nil] INFO [2019-12-17 12:36:26,264] jepsen worker 74 - jepsen.util 10784 :invoke :read [24 nil] INFO [2019-12-17 12:36:26,270] jepsen worker 47 - jepsen.util 11567 :fail :cas [15 [24 31]] :ValueNotSwapFrom INFO [2019-12-17 12:36:26,270] jepsen worker 47 - jepsen.util 11567 :invoke :read [15 nil] INFO [2019-12-17 12:36:26,278] jepsen worker 80 - jepsen.util 10520 :ok :read [26 30] INFO [2019-12-17 12:36:26,278] jepsen worker 80 - jepsen.util 10520 :invoke :cas [26 [32 32]] INFO [2019-12-17 12:36:26,287] jepsen worker 10 - jepsen.util 10630 :ok :read [3 39] INFO [2019-12-17 12:36:26,287] jepsen worker 10 - jepsen.util 10630 :invoke :cas [3 [26 24]] INFO [2019-12-17 12:36:26,295] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":62520,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:36:26,295] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":62520,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:36:26,295] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":62524,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:36:26,295] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":62510,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:36:26,295] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":62524,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:36:26,295] jepsen worker 64 - jepsen.util 8344 :info :write [21 20] :RequestCanceledException INFO [2019-12-17 12:36:26,296] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":62510,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:36:26,296] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":62511,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:36:26,296] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":62511,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:36:26,296] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":62512,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:36:26,296] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":62512,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:36:26,297] jepsen worker 64 - jepsen.util 8434 :invoke :read [21 nil] INFO [2019-12-17 12:36:26,297] jepsen worker 47 - jepsen.util 11567 :ok :read [15 37] INFO [2019-12-17 12:36:26,298] jepsen worker 47 - jepsen.util 11567 :invoke :cas [15 [7 36]] INFO [2019-12-17 12:36:26,304] jepsen worker 10 - jepsen.util 10630 :fail :cas [3 [26 24]] :ValueNotSwapFrom INFO [2019-12-17 12:36:26,304] jepsen worker 10 - jepsen.util 10630 :invoke :cas [3 [40 41]] INFO [2019-12-17 12:36:26,309] jepsen worker 46 - jepsen.util 10576 :ok :write [15 14] INFO [2019-12-17 12:36:26,309] jepsen worker 46 - jepsen.util 10576 :invoke :cas [15 [37 33]] INFO [2019-12-17 12:36:26,310] jepsen worker 47 - jepsen.util 11567 :fail :cas [15 [7 36]] :ValueNotSwapFrom INFO [2019-12-17 12:36:26,310] jepsen worker 47 - jepsen.util 11567 :invoke :cas [15 [37 41]] INFO [2019-12-17 12:36:26,330] jepsen worker 10 - jepsen.util 10630 :fail :cas [3 [40 41]] :ValueNotSwapFrom INFO [2019-12-17 12:36:26,330] jepsen worker 10 - jepsen.util 10630 :invoke :cas [3 [14 11]] INFO [2019-12-17 12:36:26,333] jepsen worker 46 - jepsen.util 10576 :fail :cas [15 [37 33]] :ValueNotSwapFrom INFO [2019-12-17 12:36:26,333] jepsen worker 46 - jepsen.util 10576 :invoke :cas [15 [40 14]] INFO [2019-12-17 12:36:26,346] jepsen worker 68 - jepsen.util 13928 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:36:26,346] jepsen worker 25 - jepsen.util 15685 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:36:26,346] jepsen worker 68 - jepsen.util 13928 :invoke :read [22 nil] INFO [2019-12-17 12:36:26,346] jepsen worker 25 - jepsen.util 15685 :invoke :cas [8 [34 15]] INFO [2019-12-17 12:36:26,347] jepsen worker 47 - jepsen.util 11567 :fail :cas [15 [37 41]] :ValueNotSwapFrom INFO [2019-12-17 12:36:26,347] jepsen worker 47 - jepsen.util 11567 :invoke :read [15 nil] INFO [2019-12-17 12:36:26,354] jepsen worker 10 - jepsen.util 10630 :fail :cas [3 [14 11]] :ValueNotSwapFrom INFO [2019-12-17 12:36:26,354] jepsen worker 10 - jepsen.util 10630 :invoke :write [3 33] INFO [2019-12-17 12:36:26,363] jepsen worker 63 - jepsen.util 9693 :info :cas [21 [3 14]] :RequestCanceledException INFO [2019-12-17 12:36:26,363] jepsen worker 82 - jepsen.util 14752 :info :cas [27 [9 9]] :RequestCanceledException INFO [2019-12-17 12:36:26,364] jepsen worker 63 - jepsen.util 9783 :invoke :read [21 nil] INFO [2019-12-17 12:36:26,365] jepsen worker 82 - jepsen.util 14842 :invoke :read [27 nil] INFO [2019-12-17 12:36:26,401] jepsen worker 38 - jepsen.util 11288 :ok :read [12 37] INFO [2019-12-17 12:36:26,402] jepsen worker 38 - jepsen.util 11288 :invoke :cas [12 [14 28]] INFO [2019-12-17 12:36:26,404] jepsen worker 36 - jepsen.util 11556 :ok :read [12 37] INFO [2019-12-17 12:36:26,404] jepsen worker 36 - jepsen.util 11556 :invoke :cas [12 [8 19]] INFO [2019-12-17 12:36:26,408] jepsen worker 53 - jepsen.util 9953 :fail :cas [17 [9 45]] :ValueNotSwapFrom INFO [2019-12-17 12:36:26,408] jepsen worker 53 - jepsen.util 9953 :invoke :cas [17 [44 32]] INFO [2019-12-17 12:36:26,424] jepsen worker 30 - jepsen.util 10290 :ok :write [10 16] INFO [2019-12-17 12:36:26,424] jepsen worker 30 - jepsen.util 10290 :invoke :read [10 nil] INFO [2019-12-17 12:36:26,425] jepsen worker 47 - jepsen.util 11567 :ok :read [15 14] INFO [2019-12-17 12:36:26,425] jepsen worker 47 - jepsen.util 11567 :invoke :read [15 nil] INFO [2019-12-17 12:36:26,429] jepsen worker 46 - jepsen.util 10576 :fail :cas [15 [40 14]] :ValueNotSwapFrom INFO [2019-12-17 12:36:26,429] jepsen worker 46 - jepsen.util 10576 :invoke :write [15 15] INFO [2019-12-17 12:36:26,438] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":62556,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:36:26,438] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":62556,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:36:26,438] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":62560,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:36:26,438] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":62560,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:36:26,442] jepsen worker 36 - jepsen.util 11556 :fail :cas [12 [8 19]] :ValueNotSwapFrom INFO [2019-12-17 12:36:26,443] jepsen worker 36 - jepsen.util 11556 :invoke :read [12 nil] INFO [2019-12-17 12:36:26,445] jepsen worker 64 - jepsen.util 8434 :ok :read [21 10] INFO [2019-12-17 12:36:26,445] jepsen worker 64 - jepsen.util 8434 :invoke :cas [21 [6 23]] INFO [2019-12-17 12:36:26,463] jepsen worker 45 - jepsen.util 10845 :info :write [15 5] :RequestCanceledException INFO [2019-12-17 12:36:26,465] jepsen worker 45 - jepsen.util 10935 :invoke :cas [15 [5 40]] INFO [2019-12-17 12:36:26,477] jepsen worker 51 - jepsen.util 10131 :ok :read [17 5] INFO [2019-12-17 12:36:26,477] jepsen worker 51 - jepsen.util 10131 :invoke :write [17 3] INFO [2019-12-17 12:36:26,487] jepsen worker 3 - jepsen.util 10353 :fail :cas [1 [1 43]] :ValueNotSwapFrom INFO [2019-12-17 12:36:26,488] jepsen worker 3 - jepsen.util 10353 :invoke :write [1 11] INFO [2019-12-17 12:36:26,495] jepsen worker 63 - jepsen.util 9783 :ok :read [21 10] INFO [2019-12-17 12:36:26,495] jepsen worker 63 - jepsen.util 9783 :invoke :write [21 17] INFO [2019-12-17 12:36:26,498] jepsen worker 48 - jepsen.util 13728 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:36:26,498] jepsen worker 48 - jepsen.util 13728 :invoke :cas [16 [45 33]] INFO [2019-12-17 12:36:26,514] jepsen worker 80 - jepsen.util 10520 :fail :cas [26 [32 32]] :ValueNotSwapFrom INFO [2019-12-17 12:36:26,514] jepsen worker 80 - jepsen.util 10520 :invoke :read [26 nil] INFO [2019-12-17 12:36:26,549] jepsen worker 30 - jepsen.util 10290 :ok :read [10 16] INFO [2019-12-17 12:36:26,549] jepsen worker 30 - jepsen.util 10290 :invoke :write [10 40] INFO [2019-12-17 12:36:26,555] jepsen worker 45 - jepsen.util 10935 :fail :cas [15 [5 40]] :ValueNotSwapFrom INFO [2019-12-17 12:36:26,555] jepsen worker 45 - jepsen.util 10935 :invoke :write [15 31] INFO [2019-12-17 12:36:26,570] jepsen worker 64 - jepsen.util 8434 :fail :cas [21 [6 23]] :ValueNotSwapFrom INFO [2019-12-17 12:36:26,570] jepsen worker 64 - jepsen.util 8434 :invoke :read [21 nil] INFO [2019-12-17 12:36:26,581] jepsen worker 13 - jepsen.util 10273 :ok :read [4 7] INFO [2019-12-17 12:36:26,581] jepsen worker 13 - jepsen.util 10273 :invoke :write [4 14] INFO [2019-12-17 12:36:26,586] jepsen worker 36 - jepsen.util 11556 :ok :read [12 37] INFO [2019-12-17 12:36:26,586] jepsen worker 36 - jepsen.util 11556 :invoke :read [12 nil] INFO [2019-12-17 12:36:26,614] jepsen worker 33 - jepsen.util 14613 :info :write [11 16] :RequestCanceledException INFO [2019-12-17 12:36:26,615] jepsen worker 33 - jepsen.util 14703 :invoke :read [11 nil] INFO [2019-12-17 12:36:26,632] jepsen worker 27 - jepsen.util 10197 :fail :read [9 nil] :RequestTimeoutException INFO [2019-12-17 12:36:26,632] jepsen worker 81 - jepsen.util 14211 :info :cas [27 [38 34]] :RequestCanceledException INFO [2019-12-17 12:36:26,632] jepsen worker 27 - jepsen.util 10197 :invoke :cas [9 [40 10]] INFO [2019-12-17 12:36:26,632] jepsen worker 29 - jepsen.util 9119 :info :write [9 20] :RequestCanceledException INFO [2019-12-17 12:36:26,633] jepsen worker 81 - jepsen.util 14301 :invoke :write [27 12] INFO [2019-12-17 12:36:26,637] jepsen worker 29 - jepsen.util 9209 :invoke :read [9 nil] INFO [2019-12-17 12:36:26,640] jepsen worker 36 - jepsen.util 11556 :ok :read [12 37] INFO [2019-12-17 12:36:26,640] jepsen worker 36 - jepsen.util 11556 :invoke :write [12 9] INFO [2019-12-17 12:36:26,645] jepsen worker 39 - jepsen.util 14349 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:36:26,645] jepsen worker 39 - jepsen.util 14349 :invoke :write [13 36] INFO [2019-12-17 12:36:26,657] jepsen worker 46 - jepsen.util 10576 :ok :write [15 15] INFO [2019-12-17 12:36:26,657] jepsen worker 46 - jepsen.util 10576 :invoke :read [15 nil] INFO [2019-12-17 12:36:26,662] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":62581,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:36:26,663] jepsen worker 8 - jepsen.util 14498 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:36:26,663] jepsen worker 8 - jepsen.util 14498 :invoke :read [2 nil] INFO [2019-12-17 12:36:26,663] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":62581,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:36:26,664] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":62585,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:36:26,664] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":62585,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:36:26,665] jepsen worker 10 - jepsen.util 10630 :ok :write [3 33] INFO [2019-12-17 12:36:26,665] jepsen worker 10 - jepsen.util 10630 :invoke :cas [3 [3 45]] INFO [2019-12-17 12:36:26,676] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:36:26,676] jepsen nemesis - couchbase.util Rebalance check count: 487 INFO [2019-12-17 12:36:26,676] jepsen nemesis - couchbase.util Rebalance stuck count: 468 INFO [2019-12-17 12:36:26,681] jepsen worker 27 - jepsen.util 10197 :fail :cas [9 [40 10]] :ValueNotSwapFrom INFO [2019-12-17 12:36:26,681] jepsen worker 27 - jepsen.util 10197 :invoke :write [9 41] INFO [2019-12-17 12:36:26,685] jepsen worker 30 - jepsen.util 10290 :ok :write [10 40] INFO [2019-12-17 12:36:26,685] jepsen worker 30 - jepsen.util 10290 :invoke :write [10 49] INFO [2019-12-17 12:36:26,698] jepsen worker 31 - jepsen.util 9391 :fail :read [10 nil] :RequestTimeoutException INFO [2019-12-17 12:36:26,698] jepsen worker 31 - jepsen.util 9391 :invoke :read [10 nil] INFO [2019-12-17 12:36:26,731] jepsen worker 77 - jepsen.util 15287 :info :cas [25 [42 24]] :RequestCanceledException INFO [2019-12-17 12:36:26,732] jepsen worker 77 - jepsen.util 15377 :invoke :read [25 nil] INFO [2019-12-17 12:36:26,734] jepsen worker 10 - jepsen.util 10630 :fail :cas [3 [3 45]] :ValueNotSwapFrom INFO [2019-12-17 12:36:26,734] jepsen worker 10 - jepsen.util 10630 :invoke :write [3 36] INFO [2019-12-17 12:36:26,753] jepsen worker 53 - jepsen.util 9953 :fail :cas [17 [44 32]] :ValueNotSwapFrom INFO [2019-12-17 12:36:26,753] jepsen worker 53 - jepsen.util 9953 :invoke :cas [17 [25 10]] INFO [2019-12-17 12:36:26,763] jepsen worker 59 - jepsen.util 13649 :info :cas [19 [4 2]] :RequestCanceledException INFO [2019-12-17 12:36:26,763] jepsen worker 59 - jepsen.util 13739 :invoke :write [19 26] INFO [2019-12-17 12:36:26,777] jepsen worker 46 - jepsen.util 10576 :ok :read [15 15] INFO [2019-12-17 12:36:26,778] jepsen worker 46 - jepsen.util 10576 :invoke :write [15 39] INFO [2019-12-17 12:36:26,790] jepsen worker 29 - jepsen.util 9209 :ok :read [9 21] INFO [2019-12-17 12:36:26,791] jepsen worker 29 - jepsen.util 9209 :invoke :read [9 nil] INFO [2019-12-17 12:36:26,793] jepsen worker 31 - jepsen.util 9391 :ok :read [10 40] INFO [2019-12-17 12:36:26,793] jepsen worker 31 - jepsen.util 9391 :invoke :read [10 nil] INFO [2019-12-17 12:36:26,799] jepsen worker 70 - jepsen.util 11320 :info :write [23 29] :RequestCanceledException INFO [2019-12-17 12:36:26,799] jepsen worker 70 - jepsen.util 11410 :invoke :read [23 nil] INFO [2019-12-17 12:36:26,816] jepsen worker 34 - jepsen.util 14974 :info :write [11 32] :RequestCanceledException INFO [2019-12-17 12:36:26,818] jepsen worker 34 - jepsen.util 15064 :invoke :write [11 1] INFO [2019-12-17 12:36:26,833] jepsen worker 16 - jepsen.util 16036 :info :cas [5 [6 1]] :RequestCanceledException INFO [2019-12-17 12:36:26,838] jepsen worker 16 - jepsen.util 16126 :invoke :read [5 nil] INFO [2019-12-17 12:36:26,849] jepsen worker 79 - jepsen.util 11059 :ok :write [26 30] INFO [2019-12-17 12:36:26,849] jepsen worker 79 - jepsen.util 11059 :invoke :write [26 33] INFO [2019-12-17 12:36:26,867] jepsen worker 27 - jepsen.util 10197 :ok :write [9 41] INFO [2019-12-17 12:36:26,868] jepsen worker 27 - jepsen.util 10197 :invoke :cas [9 [25 9]] INFO [2019-12-17 12:36:26,876] jepsen worker 30 - jepsen.util 10290 :ok :write [10 49] INFO [2019-12-17 12:36:26,876] jepsen worker 30 - jepsen.util 10290 :invoke :read [10 nil] INFO [2019-12-17 12:36:26,880] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":62607,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:36:26,880] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":62607,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:36:26,880] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":62609,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:36:26,880] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":62609,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:36:26,891] jepsen worker 10 - jepsen.util 10630 :ok :write [3 36] INFO [2019-12-17 12:36:26,891] jepsen worker 10 - jepsen.util 10630 :invoke :cas [3 [2 9]] INFO [2019-12-17 12:36:26,893] jepsen worker 27 - jepsen.util 10197 :fail :cas [9 [25 9]] :ValueNotSwapFrom INFO [2019-12-17 12:36:26,893] jepsen worker 89 - jepsen.util 10979 :ok :write [29 6] INFO [2019-12-17 12:36:26,893] jepsen worker 27 - jepsen.util 10197 :invoke :write [9 16] INFO [2019-12-17 12:36:26,893] jepsen worker 89 - jepsen.util 10979 :invoke :read [29 nil] INFO [2019-12-17 12:36:26,895] jepsen worker 38 - jepsen.util 11288 :fail :cas [12 [14 28]] :ValueNotSwapFrom INFO [2019-12-17 12:36:26,895] jepsen worker 38 - jepsen.util 11288 :invoke :read [12 nil] INFO [2019-12-17 12:36:26,896] jepsen worker 21 - jepsen.util 14061 :info :write [7 26] :RequestCanceledException INFO [2019-12-17 12:36:26,896] jepsen worker 22 - jepsen.util 14512 :info :cas [7 [18 29]] :RequestCanceledException INFO [2019-12-17 12:36:26,897] jepsen worker 22 - jepsen.util 14602 :invoke :cas [7 [37 12]] INFO [2019-12-17 12:36:26,898] jepsen worker 29 - jepsen.util 9209 :ok :read [9 41] INFO [2019-12-17 12:36:26,898] jepsen worker 21 - jepsen.util 14151 :invoke :read [7 nil] INFO [2019-12-17 12:36:26,898] jepsen worker 29 - jepsen.util 9209 :invoke :read [9 nil] INFO [2019-12-17 12:36:26,932] jepsen worker 84 - jepsen.util 14304 :info :cas [28 [23 6]] :RequestCanceledException INFO [2019-12-17 12:36:26,933] jepsen worker 84 - jepsen.util 14394 :invoke :cas [28 [23 18]] INFO [2019-12-17 12:36:26,946] jepsen worker 12 - jepsen.util 10992 :info :write [4 33] :RequestCanceledException INFO [2019-12-17 12:36:26,946] jepsen worker 67 - jepsen.util 14827 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:36:26,946] jepsen worker 67 - jepsen.util 14827 :invoke :cas [22 [35 41]] INFO [2019-12-17 12:36:26,947] jepsen worker 12 - jepsen.util 11082 :invoke :write [4 34] INFO [2019-12-17 12:36:26,948] jepsen worker 30 - jepsen.util 10290 :ok :read [10 49] INFO [2019-12-17 12:36:26,948] jepsen worker 30 - jepsen.util 10290 :invoke :cas [10 [34 21]] INFO [2019-12-17 12:36:26,955] jepsen worker 87 - jepsen.util 10167 :ok :read [29 6] INFO [2019-12-17 12:36:26,955] jepsen worker 87 - jepsen.util 10167 :invoke :read [29 nil] INFO [2019-12-17 12:36:26,963] jepsen worker 86 - jepsen.util 15566 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:36:26,963] jepsen worker 86 - jepsen.util 15566 :invoke :write [28 20] INFO [2019-12-17 12:36:26,964] jepsen worker 10 - jepsen.util 10630 :fail :cas [3 [2 9]] :ValueNotSwapFrom INFO [2019-12-17 12:36:26,964] jepsen worker 10 - jepsen.util 10630 :invoke :cas [3 [27 45]] INFO [2019-12-17 12:36:26,993] jepsen worker 29 - jepsen.util 9209 :ok :read [9 41] INFO [2019-12-17 12:36:26,994] jepsen worker 29 - jepsen.util 9209 :invoke :write [9 34] INFO [2019-12-17 12:36:27,005] jepsen worker 10 - jepsen.util 10630 :fail :cas [3 [27 45]] :ValueNotSwapFrom INFO [2019-12-17 12:36:27,005] jepsen worker 10 - jepsen.util 10630 :invoke :write [3 13] INFO [2019-12-17 12:36:27,008] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":62627,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:36:27,008] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":62627,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:36:27,008] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":62631,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:36:27,008] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":62631,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:36:27,017] jepsen worker 73 - jepsen.util 10063 :info :write [24 47] :RequestCanceledException INFO [2019-12-17 12:36:27,017] jepsen worker 73 - jepsen.util 10153 :invoke :write [24 28] INFO [2019-12-17 12:36:27,032] jepsen worker 43 - jepsen.util 14713 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:36:27,033] jepsen worker 43 - jepsen.util 14713 :invoke :write [14 7] INFO [2019-12-17 12:36:27,036] jepsen worker 30 - jepsen.util 10290 :fail :cas [10 [34 21]] :ValueNotSwapFrom INFO [2019-12-17 12:36:27,037] jepsen worker 30 - jepsen.util 10290 :invoke :read [10 nil] INFO [2019-12-17 12:36:27,055] jepsen worker 80 - jepsen.util 10520 :ok :read [26 30] INFO [2019-12-17 12:36:27,056] jepsen worker 80 - jepsen.util 10520 :invoke :write [26 7] INFO [2019-12-17 12:36:27,085] jepsen worker 53 - jepsen.util 9953 :fail :cas [17 [25 10]] :ValueNotSwapFrom INFO [2019-12-17 12:36:27,085] jepsen worker 53 - jepsen.util 9953 :invoke :write [17 20] INFO [2019-12-17 12:36:27,095] jepsen worker 27 - jepsen.util 10197 :ok :write [9 16] INFO [2019-12-17 12:36:27,095] jepsen worker 27 - jepsen.util 10197 :invoke :write [9 46] INFO [2019-12-17 12:36:27,099] jepsen worker 50 - jepsen.util 15170 :info :cas [16 [28 35]] :RequestCanceledException INFO [2019-12-17 12:36:27,100] jepsen worker 50 - jepsen.util 15260 :invoke :cas [16 [17 16]] INFO [2019-12-17 12:36:27,158] jepsen worker 89 - jepsen.util 10979 :ok :read [29 6] INFO [2019-12-17 12:36:27,158] jepsen worker 89 - jepsen.util 10979 :invoke :read [29 nil] INFO [2019-12-17 12:36:27,163] jepsen worker 10 - jepsen.util 10630 :ok :write [3 13] INFO [2019-12-17 12:36:27,163] jepsen worker 10 - jepsen.util 10630 :invoke :cas [3 [4 13]] INFO [2019-12-17 12:36:27,163] jepsen worker 57 - jepsen.util 14457 :info :write [19 5] :RequestCanceledException INFO [2019-12-17 12:36:27,164] jepsen worker 57 - jepsen.util 14547 :invoke :cas [19 [36 34]] INFO [2019-12-17 12:36:27,170] jepsen worker 87 - jepsen.util 10167 :ok :read [29 6] INFO [2019-12-17 12:36:27,171] jepsen worker 87 - jepsen.util 10167 :invoke :cas [29 [7 8]] INFO [2019-12-17 12:36:27,184] jepsen worker 10 - jepsen.util 10630 :fail :cas [3 [4 13]] :ValueNotSwapFrom INFO [2019-12-17 12:36:27,184] jepsen worker 10 - jepsen.util 10630 :invoke :write [3 17] INFO [2019-12-17 12:36:27,201] jepsen worker 27 - jepsen.util 10197 :ok :write [9 46] INFO [2019-12-17 12:36:27,201] jepsen worker 27 - jepsen.util 10197 :invoke :write [9 38] INFO [2019-12-17 12:36:27,216] jepsen worker 2 - jepsen.util 14402 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:36:27,216] jepsen worker 2 - jepsen.util 14402 :invoke :read [0 nil] INFO [2019-12-17 12:36:27,232] jepsen worker 15 - jepsen.util 13605 :info :cas [5 [4 18]] :RequestCanceledException INFO [2019-12-17 12:36:27,232] jepsen worker 24 - jepsen.util 13344 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:36:27,232] jepsen worker 24 - jepsen.util 13344 :invoke :read [8 nil] INFO [2019-12-17 12:36:27,233] jepsen worker 15 - jepsen.util 13695 :invoke :read [5 nil] INFO [2019-12-17 12:36:27,269] jepsen worker 27 - jepsen.util 10197 :ok :write [9 38] INFO [2019-12-17 12:36:27,269] jepsen worker 27 - jepsen.util 10197 :invoke :write [9 14] INFO [2019-12-17 12:36:27,281] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":62657,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:36:27,281] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":62657,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:36:27,281] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":62659,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:36:27,281] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":62659,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:36:27,284] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":62646,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:36:27,284] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":62646,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:36:27,284] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":62647,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:36:27,285] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":62647,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:36:27,316] jepsen worker 27 - jepsen.util 10197 :ok :write [9 14] INFO [2019-12-17 12:36:27,316] jepsen worker 27 - jepsen.util 10197 :invoke :read [9 nil] INFO [2019-12-17 12:36:27,330] jepsen worker 41 - jepsen.util 15521 :info :cas [13 [44 23]] :RequestCanceledException INFO [2019-12-17 12:36:27,330] jepsen worker 76 - jepsen.util 15016 :info :cas [25 [23 4]] :RequestCanceledException INFO [2019-12-17 12:36:27,331] jepsen worker 41 - jepsen.util 15611 :invoke :write [13 23] INFO [2019-12-17 12:36:27,331] jepsen worker 76 - jepsen.util 15106 :invoke :cas [25 [13 37]] INFO [2019-12-17 12:36:27,344] jepsen worker 27 - jepsen.util 10197 :ok :read [9 14] INFO [2019-12-17 12:36:27,344] jepsen worker 27 - jepsen.util 10197 :invoke :write [9 34] INFO [2019-12-17 12:36:27,363] jepsen worker 60 - jepsen.util 15540 :info :cas [20 [43 5]] :RequestCanceledException INFO [2019-12-17 12:36:27,363] jepsen worker 60 - jepsen.util 15630 :invoke :cas [20 [35 31]] INFO [2019-12-17 12:36:27,387] jepsen worker 27 - jepsen.util 10197 :ok :write [9 34] INFO [2019-12-17 12:36:27,387] jepsen worker 27 - jepsen.util 10197 :invoke :write [9 34] INFO [2019-12-17 12:36:27,396] jepsen worker 38 - jepsen.util 11288 :ok :read [12 37] INFO [2019-12-17 12:36:27,396] jepsen worker 38 - jepsen.util 11288 :invoke :read [12 nil] INFO [2019-12-17 12:36:27,399] jepsen worker 42 - jepsen.util 15342 :info :cas [14 [41 26]] :RequestCanceledException INFO [2019-12-17 12:36:27,404] jepsen worker 42 - jepsen.util 15432 :invoke :cas [14 [4 32]] INFO [2019-12-17 12:36:27,416] jepsen worker 17 - jepsen.util 14507 :info :cas [5 [3 44]] :RequestCanceledException INFO [2019-12-17 12:36:27,417] jepsen worker 17 - jepsen.util 14597 :invoke :cas [5 [8 26]] INFO [2019-12-17 12:36:27,418] jepsen worker 89 - jepsen.util 10979 :ok :read [29 6] INFO [2019-12-17 12:36:27,418] jepsen worker 89 - jepsen.util 10979 :invoke :write [29 16] INFO [2019-12-17 12:36:27,420] jepsen worker 10 - jepsen.util 10630 :ok :write [3 17] INFO [2019-12-17 12:36:27,420] jepsen worker 10 - jepsen.util 10630 :invoke :read [3 nil] INFO [2019-12-17 12:36:27,421] jepsen worker 73 - jepsen.util 10153 :ok :write [24 28] INFO [2019-12-17 12:36:27,421] jepsen worker 73 - jepsen.util 10153 :invoke :cas [24 [21 32]] INFO [2019-12-17 12:36:27,429] jepsen worker 87 - jepsen.util 10167 :fail :cas [29 [7 8]] :ValueNotSwapFrom INFO [2019-12-17 12:36:27,429] jepsen worker 87 - jepsen.util 10167 :invoke :read [29 nil] INFO [2019-12-17 12:36:27,432] jepsen worker 20 - jepsen.util 10550 :info :write [6 23] :RequestCanceledException INFO [2019-12-17 12:36:27,433] jepsen worker 20 - jepsen.util 10640 :invoke :cas [6 [4 41]] INFO [2019-12-17 12:36:27,446] jepsen worker 6 - jepsen.util 14946 :info :write [2 29] :RequestCanceledException INFO [2019-12-17 12:36:27,446] jepsen worker 6 - jepsen.util 15036 :invoke :cas [2 [26 31]] INFO [2019-12-17 12:36:27,456] jepsen worker 27 - jepsen.util 10197 :ok :write [9 34] INFO [2019-12-17 12:36:27,456] jepsen worker 27 - jepsen.util 10197 :invoke :read [9 nil] INFO [2019-12-17 12:36:27,460] jepsen worker 10 - jepsen.util 10630 :ok :read [3 17] INFO [2019-12-17 12:36:27,461] jepsen worker 10 - jepsen.util 10630 :invoke :read [3 nil] INFO [2019-12-17 12:36:27,492] jepsen worker 20 - jepsen.util 10640 :fail :cas [6 [4 41]] :ValueNotSwapFrom INFO [2019-12-17 12:36:27,492] jepsen worker 20 - jepsen.util 10640 :invoke :read [6 nil] INFO [2019-12-17 12:36:27,508] jepsen worker 73 - jepsen.util 10153 :fail :cas [24 [21 32]] :ValueNotSwapFrom INFO [2019-12-17 12:36:27,509] jepsen worker 73 - jepsen.util 10153 :invoke :write [24 43] INFO [2019-12-17 12:36:27,511] jepsen worker 10 - jepsen.util 10630 :ok :read [3 17] INFO [2019-12-17 12:36:27,511] jepsen worker 10 - jepsen.util 10630 :invoke :write [3 1] INFO [2019-12-17 12:36:27,521] jepsen worker 27 - jepsen.util 10197 :ok :read [9 34] INFO [2019-12-17 12:36:27,521] jepsen worker 27 - jepsen.util 10197 :invoke :read [9 nil] INFO [2019-12-17 12:36:27,532] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":62677,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:36:27,532] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":62677,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:36:27,532] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":62681,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:36:27,532] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":62681,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:36:27,545] jepsen worker 20 - jepsen.util 10640 :ok :read [6 38] INFO [2019-12-17 12:36:27,545] jepsen worker 20 - jepsen.util 10640 :invoke :write [6 28] INFO [2019-12-17 12:36:27,550] jepsen worker 87 - jepsen.util 10167 :ok :read [29 6] INFO [2019-12-17 12:36:27,550] jepsen worker 87 - jepsen.util 10167 :invoke :write [29 40] INFO [2019-12-17 12:36:27,565] jepsen worker 27 - jepsen.util 10197 :ok :read [9 34] INFO [2019-12-17 12:36:27,565] jepsen worker 27 - jepsen.util 10197 :invoke :write [9 24] INFO [2019-12-17 12:36:27,594] jepsen worker 35 - jepsen.util 14705 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:36:27,594] jepsen worker 35 - jepsen.util 14705 :invoke :read [11 nil] INFO [2019-12-17 12:36:27,616] jepsen worker 56 - jepsen.util 11306 :info :write [18 43] :RequestCanceledException INFO [2019-12-17 12:36:27,616] jepsen worker 56 - jepsen.util 11396 :invoke :write [18 31] INFO [2019-12-17 12:36:27,630] jepsen worker 65 - jepsen.util 9245 :fail :read [21 nil] :RequestTimeoutException INFO [2019-12-17 12:36:27,630] jepsen worker 18 - jepsen.util 10908 :info :cas [6 [14 8]] :RequestCanceledException INFO [2019-12-17 12:36:27,630] jepsen worker 65 - jepsen.util 9245 :invoke :write [21 3] INFO [2019-12-17 12:36:27,631] jepsen worker 18 - jepsen.util 10998 :invoke :read [6 nil] INFO [2019-12-17 12:36:27,650] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":62700,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:36:27,650] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":62700,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:36:27,650] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":62702,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:36:27,650] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":62702,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:36:27,663] jepsen worker 11 - jepsen.util 10451 :info :write [3 7] :RequestCanceledException INFO [2019-12-17 12:36:27,663] jepsen worker 11 - jepsen.util 10541 :invoke :read [3 nil] INFO [2019-12-17 12:36:27,675] jepsen worker 11 - jepsen.util 10541 :ok :read [3 17] INFO [2019-12-17 12:36:27,675] jepsen worker 11 - jepsen.util 10541 :invoke :cas [3 [23 8]] INFO [2019-12-17 12:36:27,684] jepsen worker 11 - jepsen.util 10541 :fail :cas [3 [23 8]] :ValueNotSwapFrom INFO [2019-12-17 12:36:27,685] jepsen worker 11 - jepsen.util 10541 :invoke :cas [3 [10 3]] INFO [2019-12-17 12:36:27,688] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:36:27,688] jepsen nemesis - couchbase.util Rebalance check count: 488 INFO [2019-12-17 12:36:27,688] jepsen nemesis - couchbase.util Rebalance stuck count: 469 INFO [2019-12-17 12:36:27,699] jepsen worker 9 - jepsen.util 11619 :fail :read [3 nil] :RequestTimeoutException INFO [2019-12-17 12:36:27,699] jepsen worker 9 - jepsen.util 11619 :invoke :cas [3 [44 47]] INFO [2019-12-17 12:36:27,701] jepsen worker 11 - jepsen.util 10541 :fail :cas [3 [10 3]] :ValueNotSwapFrom INFO [2019-12-17 12:36:27,701] jepsen worker 11 - jepsen.util 10541 :invoke :cas [3 [22 11]] INFO [2019-12-17 12:36:27,717] jepsen worker 44 - jepsen.util 15074 :info :write [14 26] :RequestCanceledException INFO [2019-12-17 12:36:27,717] jepsen worker 55 - jepsen.util 11305 :info :write [18 3] :RequestCanceledException INFO [2019-12-17 12:36:27,717] jepsen worker 55 - jepsen.util 11395 :invoke :write [18 9] INFO [2019-12-17 12:36:27,721] jepsen worker 44 - jepsen.util 15164 :invoke :write [14 14] INFO [2019-12-17 12:36:27,731] jepsen worker 11 - jepsen.util 10541 :fail :cas [3 [22 11]] :ValueNotSwapFrom INFO [2019-12-17 12:36:27,732] jepsen worker 11 - jepsen.util 10541 :invoke :read [3 nil] INFO [2019-12-17 12:36:27,745] jepsen worker 62 - jepsen.util 15002 :info :cas [20 [13 0]] :RequestCanceledException INFO [2019-12-17 12:36:27,747] jepsen worker 62 - jepsen.util 15092 :invoke :cas [20 [17 29]] INFO [2019-12-17 12:36:27,760] jepsen worker 11 - jepsen.util 10541 :ok :read [3 17] INFO [2019-12-17 12:36:27,760] jepsen worker 11 - jepsen.util 10541 :invoke :cas [3 [36 3]] INFO [2019-12-17 12:36:27,763] jepsen worker 85 - jepsen.util 15565 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:36:27,763] jepsen worker 85 - jepsen.util 15565 :invoke :write [28 26] INFO [2019-12-17 12:36:27,774] jepsen worker 11 - jepsen.util 10541 :fail :cas [3 [36 3]] :ValueNotSwapFrom INFO [2019-12-17 12:36:27,774] jepsen worker 11 - jepsen.util 10541 :invoke :read [3 nil] INFO [2019-12-17 12:36:27,811] jepsen worker 11 - jepsen.util 10541 :ok :read [3 17] INFO [2019-12-17 12:36:27,811] jepsen worker 11 - jepsen.util 10541 :invoke :cas [3 [0 47]] INFO [2019-12-17 12:36:27,816] jepsen worker 23 - jepsen.util 14513 :info :write [7 15] :RequestCanceledException INFO [2019-12-17 12:36:27,816] jepsen worker 37 - jepsen.util 10297 :info :write [12 25] :RequestCanceledException INFO [2019-12-17 12:36:27,821] jepsen worker 37 - jepsen.util 10387 :invoke :cas [12 [1 46]] INFO [2019-12-17 12:36:27,822] jepsen worker 23 - jepsen.util 14603 :invoke :write [7 7] INFO [2019-12-17 12:36:27,833] jepsen worker 40 - jepsen.util 14980 :info :write [13 19] :RequestCanceledException INFO [2019-12-17 12:36:27,838] jepsen worker 40 - jepsen.util 15070 :invoke :read [13 nil] INFO [2019-12-17 12:36:27,857] jepsen worker 11 - jepsen.util 10541 :fail :cas [3 [0 47]] :ValueNotSwapFrom INFO [2019-12-17 12:36:27,857] jepsen worker 11 - jepsen.util 10541 :invoke :read [3 nil] INFO [2019-12-17 12:36:27,866] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":62720,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:36:27,867] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":62720,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:36:27,867] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":62722,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:36:27,867] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":62722,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:36:27,912] jepsen worker 38 - jepsen.util 11288 :ok :read [12 37] INFO [2019-12-17 12:36:27,912] jepsen worker 38 - jepsen.util 11288 :invoke :write [12 48] INFO [2019-12-17 12:36:27,924] jepsen worker 11 - jepsen.util 10541 :ok :read [3 17] INFO [2019-12-17 12:36:27,924] jepsen worker 11 - jepsen.util 10541 :invoke :write [3 44] INFO [2019-12-17 12:36:27,932] jepsen worker 66 - jepsen.util 14466 :info :cas [22 [48 8]] :RequestCanceledException INFO [2019-12-17 12:36:27,937] jepsen worker 66 - jepsen.util 14556 :invoke :write [22 29] INFO [2019-12-17 12:36:27,962] jepsen worker 14 - jepsen.util 10544 :info :write [4 32] :RequestCanceledException INFO [2019-12-17 12:36:27,965] jepsen worker 14 - jepsen.util 10634 :invoke :cas [4 [22 16]] INFO [2019-12-17 12:36:27,998] jepsen worker 69 - jepsen.util 10869 :info :write [23 31] :RequestCanceledException INFO [2019-12-17 12:36:27,998] jepsen worker 7 - jepsen.util 14677 :info :cas [2 [34 40]] :RequestCanceledException INFO [2019-12-17 12:36:28,001] jepsen worker 69 - jepsen.util 10959 :invoke :read [23 nil] INFO [2019-12-17 12:36:28,004] jepsen worker 7 - jepsen.util 14767 :invoke :read [2 nil] INFO [2019-12-17 12:36:28,017] jepsen worker 54 - jepsen.util 11574 :info :write [18 11] :RequestCanceledException INFO [2019-12-17 12:36:28,017] jepsen worker 78 - jepsen.util 10878 :info :cas [26 [30 46]] :RequestCanceledException INFO [2019-12-17 12:36:28,017] jepsen worker 54 - jepsen.util 11664 :invoke :write [18 41] INFO [2019-12-17 12:36:28,018] jepsen worker 78 - jepsen.util 10968 :invoke :write [26 0] INFO [2019-12-17 12:36:28,018] jepsen worker 20 - jepsen.util 10640 :ok :write [6 28] INFO [2019-12-17 12:36:28,018] jepsen worker 73 - jepsen.util 10153 :ok :write [24 43] INFO [2019-12-17 12:36:28,018] jepsen worker 10 - jepsen.util 10630 :ok :write [3 1] INFO [2019-12-17 12:36:28,018] jepsen worker 20 - jepsen.util 10640 :invoke :read [6 nil] INFO [2019-12-17 12:36:28,019] jepsen worker 73 - jepsen.util 10153 :invoke :write [24 29] INFO [2019-12-17 12:36:28,019] jepsen worker 10 - jepsen.util 10630 :invoke :cas [3 [33 13]] INFO [2019-12-17 12:36:28,031] jepsen worker 0 - jepsen.util 15570 :info :cas [0 [40 5]] :RequestCanceledException INFO [2019-12-17 12:36:28,032] jepsen worker 83 - jepsen.util 14843 :info :cas [27 [46 0]] :RequestCanceledException INFO [2019-12-17 12:36:28,039] jepsen worker 83 - jepsen.util 14933 :invoke :write [27 29] INFO [2019-12-17 12:36:28,042] jepsen worker 0 - jepsen.util 15660 :invoke :write [0 43] INFO [2019-12-17 12:36:28,046] jepsen worker 49 - jepsen.util 14449 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:36:28,046] jepsen worker 88 - jepsen.util 10978 :info :write [29 19] :RequestCanceledException INFO [2019-12-17 12:36:28,046] jepsen worker 61 - jepsen.util 15541 :info :write [20 4] :RequestCanceledException INFO [2019-12-17 12:36:28,046] jepsen worker 49 - jepsen.util 14449 :invoke :write [16 7] INFO [2019-12-17 12:36:28,054] jepsen worker 61 - jepsen.util 15631 :invoke :read [20 nil] INFO [2019-12-17 12:36:28,054] jepsen worker 88 - jepsen.util 11068 :invoke :read [29 nil] INFO [2019-12-17 12:36:28,064] jepsen worker 1 - jepsen.util 14761 :info :write [0 3] :RequestCanceledException INFO [2019-12-17 12:36:28,072] jepsen worker 1 - jepsen.util 14851 :invoke :cas [0 [31 16]] INFO [2019-12-17 12:36:28,102] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":62730,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:36:28,102] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":62730,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:36:28,102] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":62731,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:36:28,102] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":62731,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:36:28,129] jepsen worker 26 - jepsen.util 15326 :info :write [8 36] :RequestCanceledException INFO [2019-12-17 12:36:28,132] jepsen worker 27 - jepsen.util 10197 :ok :write [9 24] INFO [2019-12-17 12:36:28,132] jepsen worker 27 - jepsen.util 10197 :invoke :write [9 44] INFO [2019-12-17 12:36:28,135] jepsen worker 18 - jepsen.util 10998 :ok :read [6 28] INFO [2019-12-17 12:36:28,138] jepsen worker 18 - jepsen.util 10998 :invoke :read [6 nil] INFO [2019-12-17 12:36:28,142] jepsen worker 9 - jepsen.util 11619 :fail :cas [3 [44 47]] :ValueNotSwapFrom INFO [2019-12-17 12:36:28,143] jepsen worker 26 - jepsen.util 15416 :invoke :read [8 nil] INFO [2019-12-17 12:36:28,143] jepsen worker 9 - jepsen.util 11619 :invoke :write [3 24] INFO [2019-12-17 12:36:28,144] jepsen worker 10 - jepsen.util 10630 :fail :cas [3 [33 13]] :ValueNotSwapFrom INFO [2019-12-17 12:36:28,144] jepsen worker 10 - jepsen.util 10630 :invoke :cas [3 [4 47]] INFO [2019-12-17 12:36:28,145] jepsen worker 32 - jepsen.util 11012 :info :write [10 11] :RequestCanceledException INFO [2019-12-17 12:36:28,147] jepsen worker 32 - jepsen.util 11102 :invoke :read [10 nil] INFO [2019-12-17 12:36:28,158] jepsen worker 5 - jepsen.util 10265 :info :write [1 49] :RequestCanceledException INFO [2019-12-17 12:36:28,164] jepsen worker 5 - jepsen.util 10355 :invoke :write [1 19] INFO [2019-12-17 12:36:28,199] jepsen worker 20 - jepsen.util 10640 :ok :read [6 28] INFO [2019-12-17 12:36:28,199] jepsen worker 20 - jepsen.util 10640 :invoke :write [6 10] INFO [2019-12-17 12:36:28,217] jepsen worker 58 - jepsen.util 15178 :info :cas [19 [21 12]] :RequestCanceledException INFO [2019-12-17 12:36:28,220] jepsen worker 58 - jepsen.util 15268 :invoke :read [19 nil] INFO [2019-12-17 12:36:28,232] jepsen worker 75 - jepsen.util 14295 :info :write [25 44] :RequestCanceledException INFO [2019-12-17 12:36:28,232] jepsen worker 19 - jepsen.util 10999 :info :write [6 15] :RequestCanceledException INFO [2019-12-17 12:36:28,232] jepsen worker 4 - jepsen.util 9904 :info :write [1 4] :RequestCanceledException INFO [2019-12-17 12:36:28,233] jepsen worker 4 - jepsen.util 9994 :invoke :write [1 11] INFO [2019-12-17 12:36:28,239] jepsen worker 75 - jepsen.util 14385 :invoke :cas [25 [8 46]] INFO [2019-12-17 12:36:28,240] jepsen worker 19 - jepsen.util 11089 :invoke :write [6 24] INFO [2019-12-17 12:36:28,246] jepsen worker 72 - jepsen.util 10962 :info :write [24 35] :RequestCanceledException INFO [2019-12-17 12:36:28,246] jepsen worker 32 - jepsen.util 11102 :ok :read [10 49] INFO [2019-12-17 12:36:28,246] jepsen worker 71 - jepsen.util 11771 :info :cas [23 [8 28]] :RequestCanceledException INFO [2019-12-17 12:36:28,247] jepsen worker 32 - jepsen.util 11102 :invoke :cas [10 [37 4]] INFO [2019-12-17 12:36:28,252] jepsen worker 72 - jepsen.util 11052 :invoke :write [24 23] INFO [2019-12-17 12:36:28,254] jepsen worker 71 - jepsen.util 11861 :invoke :read [23 nil] INFO [2019-12-17 12:36:28,263] jepsen worker 28 - jepsen.util 8938 :info :write [9 28] :RequestCanceledException INFO [2019-12-17 12:36:28,265] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":62764,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:36:28,265] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":62764,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:36:28,265] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":62767,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:36:28,265] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":62756,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:36:28,266] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":62767,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:36:28,266] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":62756,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:36:28,266] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":62768,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:36:28,267] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":62757,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:36:28,267] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":62757,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:36:28,267] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":62758,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:36:28,267] jepsen worker 10 - jepsen.util 10630 :fail :cas [3 [4 47]] :ValueNotSwapFrom INFO [2019-12-17 12:36:28,267] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":62768,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:36:28,268] jepsen worker 10 - jepsen.util 10630 :invoke :write [3 12] INFO [2019-12-17 12:36:28,269] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":62758,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:36:28,269] jepsen worker 28 - jepsen.util 9028 :invoke :read [9 nil] INFO [2019-12-17 12:36:28,271] jepsen worker 27 - jepsen.util 10197 :ok :write [9 44] INFO [2019-12-17 12:36:28,271] jepsen worker 27 - jepsen.util 10197 :invoke :write [9 28] INFO [2019-12-17 12:36:28,302] jepsen worker 70 - jepsen.util 11410 :ok :read [23 8] INFO [2019-12-17 12:36:28,302] jepsen worker 70 - jepsen.util 11410 :invoke :write [23 19] INFO [2019-12-17 12:36:28,330] jepsen worker 32 - jepsen.util 11102 :fail :cas [10 [37 4]] :ValueNotSwapFrom INFO [2019-12-17 12:36:28,331] jepsen worker 32 - jepsen.util 11102 :invoke :cas [10 [22 12]] INFO [2019-12-17 12:36:28,332] jepsen worker 52 - jepsen.util 10582 :info :write [17 10] :RequestCanceledException INFO [2019-12-17 12:36:28,333] jepsen worker 52 - jepsen.util 10672 :invoke :cas [17 [48 43]] INFO [2019-12-17 12:36:28,346] jepsen worker 74 - jepsen.util 10784 :fail :read [24 nil] :RequestTimeoutException INFO [2019-12-17 12:36:28,347] jepsen worker 74 - jepsen.util 10784 :invoke :read [24 nil] INFO [2019-12-17 12:36:28,372] jepsen worker 37 - jepsen.util 10387 :fail :cas [12 [1 46]] :ValueNotSwapFrom INFO [2019-12-17 12:36:28,372] jepsen worker 37 - jepsen.util 10387 :invoke :write [12 36] INFO [2019-12-17 12:36:28,394] jepsen worker 82 - jepsen.util 14842 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:36:28,394] jepsen worker 82 - jepsen.util 14842 :invoke :write [27 20] INFO [2019-12-17 12:36:28,403] jepsen worker 74 - jepsen.util 10784 :ok :read [24 43] INFO [2019-12-17 12:36:28,403] jepsen worker 74 - jepsen.util 10784 :invoke :write [24 9] INFO [2019-12-17 12:36:28,404] jepsen worker 9 - jepsen.util 11619 :ok :write [3 24] INFO [2019-12-17 12:36:28,404] jepsen worker 9 - jepsen.util 11619 :invoke :cas [3 [22 22]] INFO [2019-12-17 12:36:28,404] jepsen worker 73 - jepsen.util 10153 :ok :write [24 29] INFO [2019-12-17 12:36:28,404] jepsen worker 73 - jepsen.util 10153 :invoke :cas [24 [25 37]] INFO [2019-12-17 12:36:28,446] jepsen worker 25 - jepsen.util 15685 :info :cas [8 [34 15]] :RequestCanceledException INFO [2019-12-17 12:36:28,446] jepsen worker 68 - jepsen.util 13928 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:36:28,446] jepsen worker 68 - jepsen.util 13928 :invoke :write [22 3] INFO [2019-12-17 12:36:28,455] jepsen worker 25 - jepsen.util 15775 :invoke :read [8 nil] INFO [2019-12-17 12:36:28,471] jepsen worker 28 - jepsen.util 9028 :ok :read [9 44] INFO [2019-12-17 12:36:28,471] jepsen worker 28 - jepsen.util 9028 :invoke :cas [9 [47 11]] INFO [2019-12-17 12:36:28,499] jepsen worker 47 - jepsen.util 11567 :fail :read [15 nil] :RequestTimeoutException INFO [2019-12-17 12:36:28,499] jepsen worker 47 - jepsen.util 11567 :invoke :cas [15 [33 24]] INFO [2019-12-17 12:36:28,513] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":62788,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:36:28,513] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":62788,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:36:28,513] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":62793,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:36:28,513] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":62793,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:36:28,516] jepsen worker 3 - jepsen.util 10353 :info :write [1 11] :RequestCanceledException INFO [2019-12-17 12:36:28,517] jepsen worker 3 - jepsen.util 10443 :invoke :write [1 4] INFO [2019-12-17 12:36:28,519] jepsen worker 18 - jepsen.util 10998 :ok :read [6 28] INFO [2019-12-17 12:36:28,519] jepsen worker 18 - jepsen.util 10998 :invoke :write [6 27] INFO [2019-12-17 12:36:28,532] jepsen worker 63 - jepsen.util 9783 :info :write [21 17] :RequestCanceledException INFO [2019-12-17 12:36:28,534] jepsen worker 63 - jepsen.util 9873 :invoke :write [21 2] INFO [2019-12-17 12:36:28,538] jepsen worker 73 - jepsen.util 10153 :fail :cas [24 [25 37]] :ValueNotSwapFrom INFO [2019-12-17 12:36:28,538] jepsen worker 73 - jepsen.util 10153 :invoke :read [24 nil] INFO [2019-12-17 12:36:28,546] jepsen worker 51 - jepsen.util 10131 :info :write [17 3] :RequestCanceledException INFO [2019-12-17 12:36:28,557] jepsen worker 51 - jepsen.util 10221 :invoke :write [17 5] INFO [2019-12-17 12:36:28,558] jepsen worker 69 - jepsen.util 10959 :ok :read [23 8] INFO [2019-12-17 12:36:28,558] jepsen worker 69 - jepsen.util 10959 :invoke :cas [23 [47 34]] INFO [2019-12-17 12:36:28,589] jepsen worker 14 - jepsen.util 10634 :fail :cas [4 [22 16]] :ValueNotSwapFrom INFO [2019-12-17 12:36:28,590] jepsen worker 14 - jepsen.util 10634 :invoke :cas [4 [18 37]] INFO [2019-12-17 12:36:28,597] jepsen worker 73 - jepsen.util 10153 :ok :read [24 29] INFO [2019-12-17 12:36:28,597] jepsen worker 73 - jepsen.util 10153 :invoke :write [24 3] INFO [2019-12-17 12:36:28,598] jepsen worker 48 - jepsen.util 13728 :info :cas [16 [45 33]] :RequestCanceledException INFO [2019-12-17 12:36:28,600] jepsen worker 48 - jepsen.util 13818 :invoke :read [16 nil] INFO [2019-12-17 12:36:28,617] jepsen worker 45 - jepsen.util 10935 :info :write [15 31] :RequestCanceledException INFO [2019-12-17 12:36:28,620] jepsen worker 45 - jepsen.util 11025 :invoke :write [15 24] INFO [2019-12-17 12:36:28,632] jepsen worker 33 - jepsen.util 14703 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:36:28,632] jepsen worker 33 - jepsen.util 14703 :invoke :cas [11 [47 48]] INFO [2019-12-17 12:36:28,650] jepsen worker 63 - jepsen.util 9873 :ok :write [21 2] INFO [2019-12-17 12:36:28,652] jepsen worker 63 - jepsen.util 9873 :invoke :read [21 nil] INFO [2019-12-17 12:36:28,663] jepsen worker 64 - jepsen.util 8434 :fail :read [21 nil] :RequestTimeoutException INFO [2019-12-17 12:36:28,663] jepsen worker 13 - jepsen.util 10273 :info :write [4 14] :RequestCanceledException INFO [2019-12-17 12:36:28,663] jepsen worker 64 - jepsen.util 8434 :invoke :write [21 0] INFO [2019-12-17 12:36:28,664] jepsen worker 13 - jepsen.util 10363 :invoke :cas [4 [49 35]] INFO [2019-12-17 12:36:28,679] jepsen worker 88 - jepsen.util 11068 :ok :read [29 6] INFO [2019-12-17 12:36:28,679] jepsen worker 88 - jepsen.util 11068 :invoke :write [29 17] INFO [2019-12-17 12:36:28,704] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:36:28,704] jepsen nemesis - couchbase.util Rebalance check count: 489 INFO [2019-12-17 12:36:28,704] jepsen nemesis - couchbase.util Rebalance stuck count: 470 INFO [2019-12-17 12:36:28,707] jepsen worker 63 - jepsen.util 9873 :ok :read [21 2] INFO [2019-12-17 12:36:28,708] jepsen worker 63 - jepsen.util 9873 :invoke :read [21 nil] INFO [2019-12-17 12:36:28,732] jepsen worker 36 - jepsen.util 11556 :info :write [12 9] :RequestCanceledException INFO [2019-12-17 12:36:28,733] jepsen worker 36 - jepsen.util 11646 :invoke :write [12 26] INFO [2019-12-17 12:36:28,737] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":62810,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:36:28,738] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":62810,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:36:28,738] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":62813,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:36:28,738] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":62813,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:36:28,738] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":62814,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:36:28,738] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":62814,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:36:28,746] jepsen worker 81 - jepsen.util 14301 :info :write [27 12] :RequestCanceledException INFO [2019-12-17 12:36:28,746] jepsen worker 39 - jepsen.util 14349 :info :write [13 36] :RequestCanceledException INFO [2019-12-17 12:36:28,751] jepsen worker 81 - jepsen.util 14391 :invoke :write [27 19] INFO [2019-12-17 12:36:28,752] jepsen worker 39 - jepsen.util 14439 :invoke :read [13 nil] INFO [2019-12-17 12:36:28,758] jepsen worker 64 - jepsen.util 8434 :ok :write [21 0] INFO [2019-12-17 12:36:28,758] jepsen worker 64 - jepsen.util 8434 :invoke :read [21 nil] INFO [2019-12-17 12:36:28,762] jepsen worker 8 - jepsen.util 14498 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:36:28,763] jepsen worker 8 - jepsen.util 14498 :invoke :cas [2 [49 2]] INFO [2019-12-17 12:36:28,770] jepsen worker 71 - jepsen.util 11861 :ok :read [23 8] INFO [2019-12-17 12:36:28,770] jepsen worker 71 - jepsen.util 11861 :invoke :write [23 37] INFO [2019-12-17 12:36:28,779] jepsen worker 13 - jepsen.util 10363 :fail :cas [4 [49 35]] :ValueNotSwapFrom INFO [2019-12-17 12:36:28,779] jepsen worker 13 - jepsen.util 10363 :invoke :cas [4 [20 32]] INFO [2019-12-17 12:36:28,800] jepsen worker 59 - jepsen.util 13739 :info :write [19 26] :RequestCanceledException INFO [2019-12-17 12:36:28,803] jepsen worker 59 - jepsen.util 13829 :invoke :cas [19 [44 12]] INFO [2019-12-17 12:36:28,803] jepsen worker 63 - jepsen.util 9873 :ok :read [21 0] INFO [2019-12-17 12:36:28,803] jepsen worker 63 - jepsen.util 9873 :invoke :cas [21 [49 19]] INFO [2019-12-17 12:36:28,832] jepsen worker 77 - jepsen.util 15377 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:36:28,832] jepsen worker 46 - jepsen.util 10576 :info :write [15 39] :RequestCanceledException INFO [2019-12-17 12:36:28,832] jepsen worker 77 - jepsen.util 15377 :invoke :cas [25 [45 28]] INFO [2019-12-17 12:36:28,835] jepsen worker 64 - jepsen.util 8434 :ok :read [21 0] INFO [2019-12-17 12:36:28,835] jepsen worker 64 - jepsen.util 8434 :invoke :cas [21 [47 9]] INFO [2019-12-17 12:36:28,839] jepsen worker 46 - jepsen.util 10666 :invoke :read [15 nil] INFO [2019-12-17 12:36:28,855] jepsen worker 63 - jepsen.util 9873 :fail :cas [21 [49 19]] :ValueNotSwapFrom INFO [2019-12-17 12:36:28,855] jepsen worker 63 - jepsen.util 9873 :invoke :cas [21 [49 39]] INFO [2019-12-17 12:36:28,872] jepsen worker 69 - jepsen.util 10959 :fail :cas [23 [47 34]] :ValueNotSwapFrom INFO [2019-12-17 12:36:28,872] jepsen worker 69 - jepsen.util 10959 :invoke :read [23 nil] INFO [2019-12-17 12:36:28,898] jepsen worker 31 - jepsen.util 9391 :fail :read [10 nil] :RequestTimeoutException INFO [2019-12-17 12:36:28,899] jepsen worker 31 - jepsen.util 9391 :invoke :read [10 nil] INFO [2019-12-17 12:36:28,904] jepsen worker 64 - jepsen.util 8434 :fail :cas [21 [47 9]] :ValueNotSwapFrom INFO [2019-12-17 12:36:28,904] jepsen worker 64 - jepsen.util 8434 :invoke :write [21 6] INFO [2019-12-17 12:36:28,916] jepsen worker 63 - jepsen.util 9873 :fail :cas [21 [49 39]] :ValueNotSwapFrom INFO [2019-12-17 12:36:28,916] jepsen worker 63 - jepsen.util 9873 :invoke :read [21 nil] INFO [2019-12-17 12:36:28,933] jepsen worker 34 - jepsen.util 15064 :info :write [11 1] :RequestCanceledException INFO [2019-12-17 12:36:28,933] jepsen worker 79 - jepsen.util 11059 :info :write [26 33] :RequestCanceledException INFO [2019-12-17 12:36:28,933] jepsen worker 21 - jepsen.util 14151 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:36:28,933] jepsen worker 21 - jepsen.util 14151 :invoke :cas [7 [24 18]] INFO [2019-12-17 12:36:28,937] jepsen worker 79 - jepsen.util 11149 :invoke :read [26 nil] INFO [2019-12-17 12:36:28,945] jepsen worker 34 - jepsen.util 15154 :invoke :cas [11 [1 33]] INFO [2019-12-17 12:36:28,946] jepsen worker 16 - jepsen.util 16126 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:36:28,946] jepsen worker 16 - jepsen.util 16126 :invoke :write [5 31] INFO [2019-12-17 12:36:28,961] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":62836,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:36:28,961] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":62836,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:36:28,963] jepsen worker 22 - jepsen.util 14602 :info :cas [7 [37 12]] :RequestCanceledException INFO [2019-12-17 12:36:28,971] jepsen worker 22 - jepsen.util 14692 :invoke :write [7 40] INFO [2019-12-17 12:36:28,980] jepsen worker 52 - jepsen.util 10672 :fail :cas [17 [48 43]] :ValueNotSwapFrom INFO [2019-12-17 12:36:28,980] jepsen worker 52 - jepsen.util 10672 :invoke :read [17 nil] INFO [2019-12-17 12:36:28,985] jepsen worker 64 - jepsen.util 8434 :ok :write [21 6] INFO [2019-12-17 12:36:28,985] jepsen worker 64 - jepsen.util 8434 :invoke :write [21 34] INFO [2019-12-17 12:36:28,990] jepsen worker 84 - jepsen.util 14394 :info :cas [28 [23 18]] :RequestCanceledException INFO [2019-12-17 12:36:28,994] jepsen worker 14 - jepsen.util 10634 :fail :cas [4 [18 37]] :ValueNotSwapFrom INFO [2019-12-17 12:36:28,994] jepsen worker 14 - jepsen.util 10634 :invoke :read [4 nil] INFO [2019-12-17 12:36:28,996] jepsen worker 63 - jepsen.util 9873 :ok :read [21 0] INFO [2019-12-17 12:36:28,996] jepsen worker 63 - jepsen.util 9873 :invoke :cas [21 [25 27]] INFO [2019-12-17 12:36:29,006] jepsen worker 84 - jepsen.util 14484 :invoke :read [28 nil] INFO [2019-12-17 12:36:29,017] jepsen worker 12 - jepsen.util 11082 :info :write [4 34] :RequestCanceledException INFO [2019-12-17 12:36:29,018] jepsen worker 12 - jepsen.util 11172 :invoke :write [4 16] INFO [2019-12-17 12:36:29,018] jepsen worker 13 - jepsen.util 10363 :fail :cas [4 [20 32]] :ValueNotSwapFrom INFO [2019-12-17 12:36:29,018] jepsen worker 13 - jepsen.util 10363 :invoke :write [4 28] INFO [2019-12-17 12:36:29,046] jepsen worker 67 - jepsen.util 14827 :info :cas [22 [35 41]] :RequestCanceledException INFO [2019-12-17 12:36:29,052] jepsen worker 67 - jepsen.util 14917 :invoke :read [22 nil] INFO [2019-12-17 12:36:29,059] jepsen worker 86 - jepsen.util 15566 :info :write [28 20] :RequestCanceledException INFO [2019-12-17 12:36:29,059] jepsen worker 29 - jepsen.util 9209 :info :write [9 34] :RequestCanceledException INFO [2019-12-17 12:36:29,059] jepsen worker 30 - jepsen.util 10290 :fail :read [10 nil] :RequestTimeoutException INFO [2019-12-17 12:36:29,060] jepsen worker 30 - jepsen.util 10290 :invoke :read [10 nil] INFO [2019-12-17 12:36:29,065] jepsen worker 86 - jepsen.util 15656 :invoke :write [28 44] INFO [2019-12-17 12:36:29,068] jepsen worker 29 - jepsen.util 9299 :invoke :read [9 nil] INFO [2019-12-17 12:36:29,077] jepsen worker 64 - jepsen.util 8434 :ok :write [21 34] INFO [2019-12-17 12:36:29,077] jepsen worker 64 - jepsen.util 8434 :invoke :cas [21 [19 19]] INFO [2019-12-17 12:36:29,084] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":62850,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:36:29,085] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":62850,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:36:29,085] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":62855,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:36:29,085] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":62855,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:36:29,088] jepsen worker 63 - jepsen.util 9873 :fail :cas [21 [25 27]] :ValueNotSwapFrom INFO [2019-12-17 12:36:29,088] jepsen worker 63 - jepsen.util 9873 :invoke :write [21 6] INFO [2019-12-17 12:36:29,131] jepsen worker 43 - jepsen.util 14713 :info :write [14 7] :RequestCanceledException INFO [2019-12-17 12:36:29,131] jepsen worker 53 - jepsen.util 9953 :info :write [17 20] :RequestCanceledException INFO [2019-12-17 12:36:29,137] jepsen worker 53 - jepsen.util 10043 :invoke :read [17 nil] INFO [2019-12-17 12:36:29,139] jepsen worker 43 - jepsen.util 14803 :invoke :cas [14 [2 34]] INFO [2019-12-17 12:36:29,146] jepsen worker 80 - jepsen.util 10520 :info :write [26 7] :RequestCanceledException INFO [2019-12-17 12:36:29,146] jepsen worker 50 - jepsen.util 15260 :info :cas [16 [17 16]] :RequestCanceledException INFO [2019-12-17 12:36:29,148] jepsen worker 80 - jepsen.util 10610 :invoke :read [26 nil] INFO [2019-12-17 12:36:29,152] jepsen worker 50 - jepsen.util 15350 :invoke :write [16 18] INFO [2019-12-17 12:36:29,155] jepsen worker 69 - jepsen.util 10959 :ok :read [23 8] INFO [2019-12-17 12:36:29,156] jepsen worker 69 - jepsen.util 10959 :invoke :read [23 nil] INFO [2019-12-17 12:36:29,157] jepsen worker 79 - jepsen.util 11149 :ok :read [26 30] INFO [2019-12-17 12:36:29,157] jepsen worker 79 - jepsen.util 11149 :invoke :read [26 nil] INFO [2019-12-17 12:36:29,181] jepsen worker 64 - jepsen.util 8434 :fail :cas [21 [19 19]] :ValueNotSwapFrom INFO [2019-12-17 12:36:29,181] jepsen worker 64 - jepsen.util 8434 :invoke :write [21 5] INFO [2019-12-17 12:36:29,188] jepsen worker 30 - jepsen.util 10290 :ok :read [10 49] INFO [2019-12-17 12:36:29,188] jepsen worker 30 - jepsen.util 10290 :invoke :cas [10 [9 30]] INFO [2019-12-17 12:36:29,196] jepsen worker 63 - jepsen.util 9873 :ok :write [21 6] INFO [2019-12-17 12:36:29,196] jepsen worker 63 - jepsen.util 9873 :invoke :cas [21 [30 46]] INFO [2019-12-17 12:36:29,257] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":62877,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:36:29,257] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":62877,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:36:29,257] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":62878,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:36:29,257] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":62878,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:36:29,257] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":62885,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:36:29,257] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":62885,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:36:29,261] jepsen worker 30 - jepsen.util 10290 :fail :cas [10 [9 30]] :ValueNotSwapFrom INFO [2019-12-17 12:36:29,261] jepsen worker 30 - jepsen.util 10290 :invoke :cas [10 [0 17]] INFO [2019-12-17 12:36:29,263] jepsen worker 64 - jepsen.util 8434 :ok :write [21 5] INFO [2019-12-17 12:36:29,263] jepsen worker 57 - jepsen.util 14547 :info :cas [19 [36 34]] :RequestCanceledException INFO [2019-12-17 12:36:29,263] jepsen worker 64 - jepsen.util 8434 :invoke :cas [21 [3 22]] INFO [2019-12-17 12:36:29,269] jepsen worker 57 - jepsen.util 14637 :invoke :write [19 8] INFO [2019-12-17 12:36:29,294] jepsen worker 63 - jepsen.util 9873 :fail :cas [21 [30 46]] :ValueNotSwapFrom INFO [2019-12-17 12:36:29,294] jepsen worker 63 - jepsen.util 9873 :invoke :read [21 nil] INFO [2019-12-17 12:36:29,317] jepsen worker 80 - jepsen.util 10610 :ok :read [26 30] INFO [2019-12-17 12:36:29,317] jepsen worker 2 - jepsen.util 14402 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:36:29,317] jepsen worker 80 - jepsen.util 10610 :invoke :read [26 nil] INFO [2019-12-17 12:36:29,317] jepsen worker 2 - jepsen.util 14402 :invoke :read [0 nil] INFO [2019-12-17 12:36:29,331] jepsen worker 15 - jepsen.util 13695 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:36:29,331] jepsen worker 24 - jepsen.util 13344 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:36:29,331] jepsen worker 15 - jepsen.util 13695 :invoke :cas [5 [43 39]] INFO [2019-12-17 12:36:29,332] jepsen worker 24 - jepsen.util 13344 :invoke :write [8 5] INFO [2019-12-17 12:36:29,360] jepsen worker 30 - jepsen.util 10290 :fail :cas [10 [0 17]] :ValueNotSwapFrom INFO [2019-12-17 12:36:29,360] jepsen worker 30 - jepsen.util 10290 :invoke :write [10 43] INFO [2019-12-17 12:36:29,365] jepsen worker 64 - jepsen.util 8434 :fail :cas [21 [3 22]] :ValueNotSwapFrom INFO [2019-12-17 12:36:29,365] jepsen worker 64 - jepsen.util 8434 :invoke :write [21 20] INFO [2019-12-17 12:36:29,387] jepsen worker 53 - jepsen.util 10043 :ok :read [17 5] INFO [2019-12-17 12:36:29,387] jepsen worker 53 - jepsen.util 10043 :invoke :read [17 nil] INFO [2019-12-17 12:36:29,392] jepsen worker 63 - jepsen.util 9873 :ok :read [21 5] INFO [2019-12-17 12:36:29,392] jepsen worker 63 - jepsen.util 9873 :invoke :read [21 nil] INFO [2019-12-17 12:36:29,398] jepsen worker 41 - jepsen.util 15611 :info :write [13 23] :RequestCanceledException INFO [2019-12-17 12:36:29,402] jepsen worker 41 - jepsen.util 15701 :invoke :write [13 2] INFO [2019-12-17 12:36:29,415] jepsen worker 76 - jepsen.util 15106 :info :cas [25 [13 37]] :RequestCanceledException INFO [2019-12-17 12:36:29,418] jepsen worker 76 - jepsen.util 15196 :invoke :write [25 33] INFO [2019-12-17 12:36:29,418] jepsen worker 69 - jepsen.util 10959 :ok :read [23 8] INFO [2019-12-17 12:36:29,418] jepsen worker 69 - jepsen.util 10959 :invoke :read [23 nil] INFO [2019-12-17 12:36:29,432] jepsen worker 60 - jepsen.util 15630 :info :cas [20 [35 31]] :RequestCanceledException INFO [2019-12-17 12:36:29,434] jepsen worker 60 - jepsen.util 15720 :invoke :write [20 3] INFO [2019-12-17 12:36:29,446] jepsen worker 42 - jepsen.util 15432 :info :cas [14 [4 32]] :RequestCanceledException INFO [2019-12-17 12:36:29,449] jepsen worker 42 - jepsen.util 15522 :invoke :cas [14 [49 42]] INFO [2019-12-17 12:36:29,463] jepsen worker 17 - jepsen.util 14597 :info :cas [5 [8 26]] :RequestCanceledException INFO [2019-12-17 12:36:29,466] jepsen worker 17 - jepsen.util 14687 :invoke :cas [5 [2 23]] INFO [2019-12-17 12:36:29,487] jepsen worker 79 - jepsen.util 11149 :ok :read [26 30] INFO [2019-12-17 12:36:29,487] jepsen worker 79 - jepsen.util 11149 :invoke :cas [26 [37 26]] INFO [2019-12-17 12:36:29,499] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":62906,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:36:29,499] jepsen worker 6 - jepsen.util 15036 :info :cas [2 [26 31]] :RequestCanceledException INFO [2019-12-17 12:36:29,499] jepsen worker 89 - jepsen.util 10979 :info :write [29 16] :RequestCanceledException INFO [2019-12-17 12:36:29,499] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":62906,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:36:29,499] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":62908,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:36:29,499] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":62908,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:36:29,503] jepsen worker 89 - jepsen.util 11069 :invoke :cas [29 [3 36]] INFO [2019-12-17 12:36:29,503] jepsen worker 6 - jepsen.util 15126 :invoke :write [2 6] INFO [2019-12-17 12:36:29,526] jepsen worker 69 - jepsen.util 10959 :ok :read [23 8] INFO [2019-12-17 12:36:29,526] jepsen worker 69 - jepsen.util 10959 :invoke :read [23 nil] INFO [2019-12-17 12:36:29,563] jepsen worker 30 - jepsen.util 10290 :ok :write [10 43] INFO [2019-12-17 12:36:29,563] jepsen worker 10 - jepsen.util 10630 :ok :write [3 12] INFO [2019-12-17 12:36:29,563] jepsen worker 10 - jepsen.util 10630 :invoke :read [3 nil] INFO [2019-12-17 12:36:29,563] jepsen worker 30 - jepsen.util 10290 :invoke :cas [10 [48 1]] INFO [2019-12-17 12:36:29,566] jepsen worker 53 - jepsen.util 10043 :ok :read [17 5] INFO [2019-12-17 12:36:29,566] jepsen worker 53 - jepsen.util 10043 :invoke :write [17 2] INFO [2019-12-17 12:36:29,591] jepsen worker 74 - jepsen.util 10784 :ok :write [24 9] INFO [2019-12-17 12:36:29,591] jepsen worker 74 - jepsen.util 10784 :invoke :read [24 nil] INFO [2019-12-17 12:36:29,594] jepsen worker 45 - jepsen.util 11025 :ok :write [15 24] INFO [2019-12-17 12:36:29,594] jepsen worker 45 - jepsen.util 11025 :invoke :write [15 8] INFO [2019-12-17 12:36:29,595] jepsen worker 20 - jepsen.util 10640 :ok :write [6 10] INFO [2019-12-17 12:36:29,595] jepsen worker 20 - jepsen.util 10640 :invoke :cas [6 [1 35]] INFO [2019-12-17 12:36:29,599] jepsen worker 87 - jepsen.util 10167 :info :write [29 40] :RequestCanceledException INFO [2019-12-17 12:36:29,600] jepsen worker 87 - jepsen.util 10257 :invoke :write [29 26] INFO [2019-12-17 12:36:29,703] jepsen worker 63 - jepsen.util 9873 :ok :read [21 5] INFO [2019-12-17 12:36:29,703] jepsen worker 63 - jepsen.util 9873 :invoke :read [21 nil] INFO [2019-12-17 12:36:29,703] jepsen worker 80 - jepsen.util 10610 :ok :read [26 30] INFO [2019-12-17 12:36:29,703] jepsen worker 80 - jepsen.util 10610 :invoke :cas [26 [28 36]] INFO [2019-12-17 12:36:29,703] jepsen worker 14 - jepsen.util 10634 :ok :read [4 7] INFO [2019-12-17 12:36:29,704] jepsen worker 14 - jepsen.util 10634 :invoke :write [4 16] INFO [2019-12-17 12:36:29,704] jepsen worker 35 - jepsen.util 14705 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:36:29,704] jepsen worker 35 - jepsen.util 14705 :invoke :write [11 2] INFO [2019-12-17 12:36:29,708] jepsen worker 56 - jepsen.util 11396 :info :write [18 31] :RequestCanceledException INFO [2019-12-17 12:36:29,712] jepsen worker 56 - jepsen.util 11486 :invoke :write [18 8] INFO [2019-12-17 12:36:29,725] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":62926,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:36:29,726] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":62926,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:36:29,726] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":62929,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:36:29,726] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":62929,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:36:29,731] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:36:29,731] jepsen nemesis - couchbase.util Rebalance check count: 490 INFO [2019-12-17 12:36:29,731] jepsen nemesis - couchbase.util Rebalance stuck count: 471 INFO [2019-12-17 12:36:29,731] jepsen worker 65 - jepsen.util 9245 :info :write [21 3] :RequestCanceledException INFO [2019-12-17 12:36:29,733] jepsen worker 65 - jepsen.util 9335 :invoke :read [21 nil] INFO [2019-12-17 12:36:29,735] jepsen worker 27 - jepsen.util 10197 :ok :write [9 28] INFO [2019-12-17 12:36:29,736] jepsen worker 27 - jepsen.util 10197 :invoke :write [9 28] INFO [2019-12-17 12:36:29,758] jepsen worker 64 - jepsen.util 8434 :ok :write [21 20] INFO [2019-12-17 12:36:29,758] jepsen worker 64 - jepsen.util 8434 :invoke :read [21 nil] INFO [2019-12-17 12:36:29,761] jepsen worker 29 - jepsen.util 9299 :ok :read [9 44] INFO [2019-12-17 12:36:29,761] jepsen worker 29 - jepsen.util 9299 :invoke :read [9 nil] INFO [2019-12-17 12:36:29,762] jepsen worker 55 - jepsen.util 11395 :info :write [18 9] :RequestCanceledException INFO [2019-12-17 12:36:29,768] jepsen worker 55 - jepsen.util 11485 :invoke :write [18 31] INFO [2019-12-17 12:36:29,790] jepsen worker 46 - jepsen.util 10666 :ok :read [15 24] INFO [2019-12-17 12:36:29,790] jepsen worker 46 - jepsen.util 10666 :invoke :write [15 19] INFO [2019-12-17 12:36:29,797] jepsen worker 62 - jepsen.util 15092 :info :cas [20 [17 29]] :RequestCanceledException INFO [2019-12-17 12:36:29,806] jepsen worker 62 - jepsen.util 15182 :invoke :cas [20 [7 29]] INFO [2019-12-17 12:36:29,816] jepsen worker 28 - jepsen.util 9028 :fail :cas [9 [47 11]] :ValueNotSwapFrom INFO [2019-12-17 12:36:29,816] jepsen worker 28 - jepsen.util 9028 :invoke :read [9 nil] INFO [2019-12-17 12:36:29,831] jepsen worker 44 - jepsen.util 15164 :info :write [14 14] :RequestCanceledException INFO [2019-12-17 12:36:29,837] jepsen worker 74 - jepsen.util 10784 :ok :read [24 9] INFO [2019-12-17 12:36:29,837] jepsen worker 74 - jepsen.util 10784 :invoke :read [24 nil] INFO [2019-12-17 12:36:29,839] jepsen worker 44 - jepsen.util 15254 :invoke :cas [14 [43 9]] INFO [2019-12-17 12:36:29,856] jepsen worker 47 - jepsen.util 11567 :fail :cas [15 [33 24]] :ValueNotSwapFrom INFO [2019-12-17 12:36:29,856] jepsen worker 47 - jepsen.util 11567 :invoke :read [15 nil] INFO [2019-12-17 12:36:29,863] jepsen worker 79 - jepsen.util 11149 :fail :cas [26 [37 26]] :ValueNotSwapFrom INFO [2019-12-17 12:36:29,863] jepsen worker 85 - jepsen.util 15565 :info :write [28 26] :RequestCanceledException INFO [2019-12-17 12:36:29,863] jepsen worker 79 - jepsen.util 11149 :invoke :write [26 7] INFO [2019-12-17 12:36:29,870] jepsen worker 31 - jepsen.util 9391 :ok :read [10 43] INFO [2019-12-17 12:36:29,871] jepsen worker 31 - jepsen.util 9391 :invoke :read [10 nil] INFO [2019-12-17 12:36:29,875] jepsen worker 85 - jepsen.util 15655 :invoke :read [28 nil] INFO [2019-12-17 12:36:29,922] jepsen worker 89 - jepsen.util 11069 :fail :cas [29 [3 36]] :ValueNotSwapFrom INFO [2019-12-17 12:36:29,922] jepsen worker 89 - jepsen.util 11069 :invoke :cas [29 [11 15]] INFO [2019-12-17 12:36:29,932] jepsen worker 23 - jepsen.util 14603 :info :write [7 7] :RequestCanceledException INFO [2019-12-17 12:36:29,939] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":62946,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:36:29,939] jepsen worker 38 - jepsen.util 11288 :info :write [12 48] :RequestCanceledException INFO [2019-12-17 12:36:29,939] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":62946,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:36:29,939] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":62950,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:36:29,940] jepsen worker 40 - jepsen.util 15070 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:36:29,940] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":62950,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:36:29,940] jepsen worker 40 - jepsen.util 15070 :invoke :write [13 14] INFO [2019-12-17 12:36:29,941] jepsen worker 23 - jepsen.util 14693 :invoke :read [7 nil] INFO [2019-12-17 12:36:29,946] jepsen worker 38 - jepsen.util 11378 :invoke :write [12 45] INFO [2019-12-17 12:36:29,948] jepsen worker 63 - jepsen.util 9873 :ok :read [21 20] INFO [2019-12-17 12:36:29,949] jepsen worker 63 - jepsen.util 9873 :invoke :read [21 nil] INFO [2019-12-17 12:36:29,955] jepsen worker 52 - jepsen.util 10672 :ok :read [17 5] INFO [2019-12-17 12:36:29,955] jepsen worker 52 - jepsen.util 10672 :invoke :read [17 nil] INFO [2019-12-17 12:36:29,973] jepsen worker 64 - jepsen.util 8434 :ok :read [21 20] INFO [2019-12-17 12:36:29,973] jepsen worker 64 - jepsen.util 8434 :invoke :read [21 nil] INFO [2019-12-17 12:36:29,977] jepsen worker 20 - jepsen.util 10640 :fail :cas [6 [1 35]] :ValueNotSwapFrom INFO [2019-12-17 12:36:29,977] jepsen worker 20 - jepsen.util 10640 :invoke :cas [6 [36 44]] INFO [2019-12-17 12:36:29,994] jepsen worker 29 - jepsen.util 9299 :ok :read [9 28] INFO [2019-12-17 12:36:30,005] jepsen worker 78 - jepsen.util 10968 :ok :write [26 0] INFO [2019-12-17 12:36:30,006] jepsen worker 10 - jepsen.util 10630 :ok :read [3 12] INFO [2019-12-17 12:36:30,014] jepsen worker 65 - jepsen.util 9335 :ok :read [21 20] INFO [2019-12-17 12:36:30,017] jepsen worker 7 - jepsen.util 14767 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:36:30,019] jepsen worker 29 - jepsen.util 9299 :invoke :read [9 nil] INFO [2019-12-17 12:36:30,044] jepsen worker 78 - jepsen.util 10968 :invoke :write [26 14] INFO [2019-12-17 12:36:30,044] jepsen worker 10 - jepsen.util 10630 :invoke :write [3 4] INFO [2019-12-17 12:36:30,044] jepsen worker 65 - jepsen.util 9335 :invoke :write [21 21] INFO [2019-12-17 12:36:30,044] jepsen worker 7 - jepsen.util 14767 :invoke :write [2 37] INFO [2019-12-17 12:36:30,045] jepsen worker 11 - jepsen.util 10541 :info :write [3 44] :RequestCanceledException INFO [2019-12-17 12:36:30,047] jepsen worker 11 - jepsen.util 10631 :invoke :cas [3 [27 31]] INFO [2019-12-17 12:36:30,054] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":62965,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:36:30,054] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":62965,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:36:30,054] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":62967,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:36:30,054] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":62967,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:36:30,064] jepsen worker 66 - jepsen.util 14556 :info :write [22 29] :RequestCanceledException INFO [2019-12-17 12:36:30,083] jepsen worker 27 - jepsen.util 10197 :ok :write [9 28] INFO [2019-12-17 12:36:30,084] jepsen worker 27 - jepsen.util 10197 :invoke :read [9 nil] INFO [2019-12-17 12:36:30,100] jepsen worker 61 - jepsen.util 15631 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:36:30,101] jepsen worker 61 - jepsen.util 15631 :invoke :write [20 43] INFO [2019-12-17 12:36:30,104] jepsen worker 66 - jepsen.util 14646 :invoke :write [22 13] INFO [2019-12-17 12:36:30,137] jepsen worker 54 - jepsen.util 11664 :info :write [18 41] :RequestCanceledException INFO [2019-12-17 12:36:30,154] jepsen worker 45 - jepsen.util 11025 :ok :write [15 8] INFO [2019-12-17 12:36:30,155] jepsen worker 0 - jepsen.util 15660 :info :write [0 43] :RequestCanceledException INFO [2019-12-17 12:36:30,156] jepsen worker 1 - jepsen.util 14851 :info :cas [0 [31 16]] :RequestCanceledException INFO [2019-12-17 12:36:30,160] jepsen worker 45 - jepsen.util 11025 :invoke :read [15 nil] INFO [2019-12-17 12:36:30,161] jepsen worker 49 - jepsen.util 14449 :info :write [16 7] :RequestCanceledException INFO [2019-12-17 12:36:30,169] jepsen worker 83 - jepsen.util 14933 :info :write [27 29] :RequestCanceledException INFO [2019-12-17 12:36:30,194] jepsen worker 12 - jepsen.util 11172 :ok :write [4 16] INFO [2019-12-17 12:36:30,194] jepsen worker 12 - jepsen.util 11172 :invoke :write [4 0] INFO [2019-12-17 12:36:30,232] jepsen worker 5 - jepsen.util 10355 :info :write [1 19] :RequestCanceledException INFO [2019-12-17 12:36:30,254] jepsen worker 26 - jepsen.util 15416 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:36:30,254] jepsen worker 26 - jepsen.util 15416 :invoke :cas [8 [31 0]] INFO [2019-12-17 12:36:30,277] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":62973,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:36:30,277] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":62973,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:36:30,280] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":62978,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:36:30,288] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":62975,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:36:30,292] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":62978,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:36:30,294] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":62975,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:36:30,296] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":62981,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:36:30,300] jepsen worker 46 - jepsen.util 10666 :ok :write [15 19] INFO [2019-12-17 12:36:30,300] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":62983,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:36:30,302] jepsen worker 46 - jepsen.util 10666 :invoke :write [15 20] INFO [2019-12-17 12:36:30,302] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":62981,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:36:30,302] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":62983,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:36:30,302] jepsen worker 58 - jepsen.util 15268 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:36:30,303] jepsen worker 58 - jepsen.util 15268 :invoke :cas [19 [25 28]] INFO [2019-12-17 12:36:30,314] jepsen worker 0 - jepsen.util 15750 :invoke :read [0 nil] INFO [2019-12-17 12:36:30,329] jepsen worker 4 - jepsen.util 9994 :info :write [1 11] :RequestCanceledException INFO [2019-12-17 12:36:30,329] jepsen worker 54 - jepsen.util 11754 :invoke :cas [18 [1 35]] INFO [2019-12-17 12:36:30,331] jepsen worker 1 - jepsen.util 14941 :invoke :read [0 nil] INFO [2019-12-17 12:36:30,331] jepsen worker 4 - jepsen.util 10084 :invoke :cas [1 [5 44]] INFO [2019-12-17 12:36:30,337] jepsen worker 75 - jepsen.util 14385 :info :cas [25 [8 46]] :RequestCanceledException INFO [2019-12-17 12:36:30,351] jepsen worker 83 - jepsen.util 15023 :invoke :read [27 nil] INFO [2019-12-17 12:36:30,355] jepsen worker 19 - jepsen.util 11089 :info :write [6 24] :RequestCanceledException INFO [2019-12-17 12:36:30,358] jepsen worker 5 - jepsen.util 10445 :invoke :cas [1 [14 27]] INFO [2019-12-17 12:36:30,358] jepsen worker 49 - jepsen.util 14539 :invoke :write [16 44] INFO [2019-12-17 12:36:30,361] jepsen worker 72 - jepsen.util 11052 :info :write [24 23] :RequestCanceledException INFO [2019-12-17 12:36:30,361] jepsen worker 32 - jepsen.util 11102 :info :cas [10 [22 12]] :RequestCanceledException INFO [2019-12-17 12:36:30,367] jepsen worker 75 - jepsen.util 14475 :invoke :read [25 nil] INFO [2019-12-17 12:36:30,368] jepsen worker 72 - jepsen.util 11142 :invoke :cas [24 [40 29]] INFO [2019-12-17 12:36:30,378] jepsen worker 19 - jepsen.util 11179 :invoke :write [6 11] INFO [2019-12-17 12:36:30,381] jepsen worker 32 - jepsen.util 11192 :invoke :read [10 nil] INFO [2019-12-17 12:36:30,398] jepsen worker 47 - jepsen.util 11567 :ok :read [15 24] INFO [2019-12-17 12:36:30,399] jepsen worker 47 - jepsen.util 11567 :invoke :write [15 7] INFO [2019-12-17 12:36:30,399] jepsen worker 74 - jepsen.util 10784 :ok :read [24 9] INFO [2019-12-17 12:36:30,399] jepsen worker 74 - jepsen.util 10784 :invoke :cas [24 [1 48]] INFO [2019-12-17 12:36:30,407] jepsen worker 28 - jepsen.util 9028 :ok :read [9 28] INFO [2019-12-17 12:36:30,408] jepsen worker 28 - jepsen.util 9028 :invoke :write [9 5] INFO [2019-12-17 12:36:30,420] jepsen worker 70 - jepsen.util 11410 :info :write [23 19] :RequestCanceledException INFO [2019-12-17 12:36:30,422] jepsen worker 37 - jepsen.util 10387 :info :write [12 36] :RequestCanceledException INFO [2019-12-17 12:36:30,495] jepsen worker 37 - jepsen.util 10477 :invoke :cas [12 [31 0]] INFO [2019-12-17 12:36:30,499] jepsen worker 70 - jepsen.util 11500 :invoke :write [23 16] INFO [2019-12-17 12:36:30,514] jepsen worker 9 - jepsen.util 11619 :info :cas [3 [22 22]] :RequestCanceledException INFO [2019-12-17 12:36:30,517] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63001,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:36:30,523] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63001,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:36:30,524] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63005,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:36:30,527] jepsen worker 82 - jepsen.util 14842 :info :write [27 20] :RequestCanceledException INFO [2019-12-17 12:36:30,527] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63005,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:36:30,531] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63006,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:36:30,531] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63006,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:36:30,539] jepsen worker 25 - jepsen.util 15775 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:36:30,543] jepsen worker 25 - jepsen.util 15775 :invoke :write [8 17] INFO [2019-12-17 12:36:30,548] jepsen worker 68 - jepsen.util 13928 :info :write [22 3] :RequestCanceledException INFO [2019-12-17 12:36:30,553] jepsen worker 3 - jepsen.util 10443 :info :write [1 4] :RequestCanceledException INFO [2019-12-17 12:36:30,560] jepsen worker 9 - jepsen.util 11709 :invoke :read [3 nil] INFO [2019-12-17 12:36:30,604] jepsen worker 68 - jepsen.util 14018 :invoke :write [22 40] INFO [2019-12-17 12:36:30,608] jepsen worker 3 - jepsen.util 10533 :invoke :read [1 nil] INFO [2019-12-17 12:36:30,629] jepsen worker 82 - jepsen.util 14932 :invoke :cas [27 [1 20]] INFO [2019-12-17 12:36:30,633] jepsen worker 51 - jepsen.util 10221 :info :write [17 5] :RequestCanceledException INFO [2019-12-17 12:36:30,634] jepsen worker 18 - jepsen.util 10998 :info :write [6 27] :RequestCanceledException INFO [2019-12-17 12:36:30,660] jepsen worker 73 - jepsen.util 10153 :info :write [24 3] :RequestCanceledException INFO [2019-12-17 12:36:30,671] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63016,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:36:30,675] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63016,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:36:30,678] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63019,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:36:30,684] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63019,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:36:30,687] jepsen worker 18 - jepsen.util 11088 :invoke :read [6 nil] INFO [2019-12-17 12:36:30,702] jepsen worker 48 - jepsen.util 13818 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:36:30,702] jepsen worker 48 - jepsen.util 13818 :invoke :write [16 36] INFO [2019-12-17 12:36:30,722] jepsen worker 88 - jepsen.util 11068 :info :write [29 17] :RequestCanceledException INFO [2019-12-17 12:36:30,733] jepsen worker 33 - jepsen.util 14703 :info :cas [11 [47 48]] :RequestCanceledException INFO [2019-12-17 12:36:30,745] jepsen worker 51 - jepsen.util 10311 :invoke :write [17 46] INFO [2019-12-17 12:36:30,757] jepsen worker 73 - jepsen.util 10243 :invoke :write [24 40] INFO [2019-12-17 12:36:30,766] jepsen worker 87 - jepsen.util 10257 :ok :write [29 26] INFO [2019-12-17 12:36:30,774] jepsen worker 87 - jepsen.util 10257 :invoke :cas [29 [31 48]] INFO [2019-12-17 12:36:30,823] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:36:30,823] jepsen worker 88 - jepsen.util 11158 :invoke :write [29 35] INFO [2019-12-17 12:36:30,830] jepsen nemesis - couchbase.util Rebalance check count: 491 INFO [2019-12-17 12:36:30,830] jepsen nemesis - couchbase.util Rebalance stuck count: 472 INFO [2019-12-17 12:36:30,846] jepsen worker 71 - jepsen.util 11861 :info :write [23 37] :RequestCanceledException INFO [2019-12-17 12:36:30,858] jepsen worker 39 - jepsen.util 14439 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:36:30,860] jepsen worker 36 - jepsen.util 11646 :info :write [12 26] :RequestCanceledException INFO [2019-12-17 12:36:30,871] jepsen worker 8 - jepsen.util 14498 :info :cas [2 [49 2]] :RequestCanceledException INFO [2019-12-17 12:36:30,872] jepsen worker 39 - jepsen.util 14439 :invoke :cas [13 [46 43]] INFO [2019-12-17 12:36:30,873] jepsen worker 81 - jepsen.util 14391 :info :write [27 19] :RequestCanceledException INFO [2019-12-17 12:36:30,910] jepsen worker 59 - jepsen.util 13829 :info :cas [19 [44 12]] :RequestCanceledException INFO [2019-12-17 12:36:30,914] jepsen worker 33 - jepsen.util 14793 :invoke :write [11 34] INFO [2019-12-17 12:36:30,936] jepsen worker 77 - jepsen.util 15377 :info :cas [25 [45 28]] :RequestCanceledException INFO [2019-12-17 12:36:30,956] jepsen worker 34 - jepsen.util 15154 :info :cas [11 [1 33]] :RequestCanceledException INFO [2019-12-17 12:36:30,967] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63027,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:36:30,997] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63027,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:36:30,998] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63028,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:36:31,001] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63028,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:36:31,001] jepsen worker 81 - jepsen.util 14481 :invoke :read [27 nil] INFO [2019-12-17 12:36:31,003] jepsen worker 8 - jepsen.util 14588 :invoke :read [2 nil] INFO [2019-12-17 12:36:31,003] jepsen worker 36 - jepsen.util 11736 :invoke :cas [12 [31 12]] INFO [2019-12-17 12:36:31,027] jepsen worker 71 - jepsen.util 11951 :invoke :read [23 nil] INFO [2019-12-17 12:36:31,036] jepsen worker 21 - jepsen.util 14151 :info :cas [7 [24 18]] :RequestCanceledException INFO [2019-12-17 12:36:31,037] jepsen worker 84 - jepsen.util 14484 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:36:31,039] jepsen worker 84 - jepsen.util 14484 :invoke :cas [28 [8 30]] INFO [2019-12-17 12:36:31,039] jepsen worker 59 - jepsen.util 13919 :invoke :write [19 22] INFO [2019-12-17 12:36:31,041] jepsen worker 77 - jepsen.util 15467 :invoke :cas [25 [29 15]] INFO [2019-12-17 12:36:31,043] jepsen worker 34 - jepsen.util 15244 :invoke :write [11 38] INFO [2019-12-17 12:36:31,057] jepsen worker 79 - jepsen.util 11149 :ok :write [26 7] INFO [2019-12-17 12:36:31,058] jepsen worker 16 - jepsen.util 16126 :info :write [5 31] :RequestCanceledException INFO [2019-12-17 12:36:31,059] jepsen worker 79 - jepsen.util 11149 :invoke :cas [26 [6 9]] INFO [2019-12-17 12:36:31,073] jepsen worker 22 - jepsen.util 14692 :info :write [7 40] :RequestCanceledException INFO [2019-12-17 12:36:31,100] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63031,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:36:31,101] jepsen worker 13 - jepsen.util 10363 :ok :write [4 28] INFO [2019-12-17 12:36:31,101] jepsen worker 13 - jepsen.util 10363 :invoke :cas [4 [46 25]] INFO [2019-12-17 12:36:31,102] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63031,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:36:31,106] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63036,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:36:31,125] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63036,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:36:31,125] jepsen worker 67 - jepsen.util 14917 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:36:31,127] jepsen worker 67 - jepsen.util 14917 :invoke :write [22 33] INFO [2019-12-17 12:36:31,143] jepsen worker 86 - jepsen.util 15656 :info :write [28 44] :RequestCanceledException INFO [2019-12-17 12:36:31,144] jepsen worker 21 - jepsen.util 14241 :invoke :write [7 48] INFO [2019-12-17 12:36:31,218] jepsen worker 16 - jepsen.util 16216 :invoke :read [5 nil] INFO [2019-12-17 12:36:31,225] jepsen worker 50 - jepsen.util 15350 :info :write [16 18] :RequestCanceledException INFO [2019-12-17 12:36:31,260] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63049,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:36:31,261] jepsen worker 43 - jepsen.util 14803 :info :cas [14 [2 34]] :RequestCanceledException INFO [2019-12-17 12:36:31,263] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63053,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:36:31,272] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63053,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:36:31,272] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63049,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:36:31,291] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63056,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:36:31,317] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63056,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:36:31,333] jepsen worker 22 - jepsen.util 14782 :invoke :write [7 45] INFO [2019-12-17 12:36:31,333] jepsen worker 57 - jepsen.util 14637 :info :write [19 8] :RequestCanceledException INFO [2019-12-17 12:36:31,426] jepsen worker 2 - jepsen.util 14402 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:36:31,458] jepsen worker 15 - jepsen.util 13695 :info :cas [5 [43 39]] :RequestCanceledException INFO [2019-12-17 12:36:31,461] jepsen worker 24 - jepsen.util 13344 :info :write [8 5] :RequestCanceledException INFO [2019-12-17 12:36:31,471] jepsen worker 41 - jepsen.util 15701 :info :write [13 2] :RequestCanceledException INFO [2019-12-17 12:36:31,486] jepsen worker 76 - jepsen.util 15196 :info :write [25 33] :RequestCanceledException INFO [2019-12-17 12:36:31,487] jepsen worker 86 - jepsen.util 15746 :invoke :read [28 nil] INFO [2019-12-17 12:36:31,491] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63063,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:36:31,494] jepsen worker 2 - jepsen.util 14402 :invoke :read [0 nil] INFO [2019-12-17 12:36:31,509] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63063,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:36:31,539] jepsen worker 42 - jepsen.util 15522 :info :cas [14 [49 42]] :RequestCanceledException INFO [2019-12-17 12:36:31,548] jepsen worker 43 - jepsen.util 14893 :invoke :cas [14 [15 44]] INFO [2019-12-17 12:36:31,549] jepsen worker 17 - jepsen.util 14687 :info :cas [5 [2 23]] :RequestCanceledException INFO [2019-12-17 12:36:31,550] jepsen worker 60 - jepsen.util 15720 :info :write [20 3] :RequestCanceledException INFO [2019-12-17 12:36:31,615] jepsen worker 50 - jepsen.util 15440 :invoke :write [16 2] INFO [2019-12-17 12:36:31,649] jepsen worker 57 - jepsen.util 14727 :invoke :cas [19 [49 2]] INFO [2019-12-17 12:36:31,680] jepsen worker 6 - jepsen.util 15126 :info :write [2 6] :RequestCanceledException INFO [2019-12-17 12:36:31,681] jepsen worker 30 - jepsen.util 10290 :info :cas [10 [48 1]] :RequestCanceledException INFO [2019-12-17 12:36:31,682] jepsen worker 53 - jepsen.util 10043 :info :write [17 2] :RequestCanceledException INFO [2019-12-17 12:36:31,697] jepsen worker 42 - jepsen.util 15612 :invoke :read [14 nil] INFO [2019-12-17 12:36:31,703] jepsen worker 15 - jepsen.util 13785 :invoke :write [5 9] INFO [2019-12-17 12:36:31,711] jepsen worker 24 - jepsen.util 13434 :invoke :read [8 nil] INFO [2019-12-17 12:36:31,897] jepsen worker 17 - jepsen.util 14777 :invoke :write [5 9] INFO [2019-12-17 12:36:31,903] jepsen worker 64 - jepsen.util 8434 :ok :read [21 20] INFO [2019-12-17 12:36:31,906] jepsen worker 31 - jepsen.util 9391 :ok :read [10 43] INFO [2019-12-17 12:36:31,906] jepsen worker 31 - jepsen.util 9391 :invoke :write [10 48] INFO [2019-12-17 12:36:31,907] jepsen worker 27 - jepsen.util 10197 :ok :read [9 28] INFO [2019-12-17 12:36:31,907] jepsen worker 30 - jepsen.util 10380 :invoke :cas [10 [41 37]] INFO [2019-12-17 12:36:31,917] jepsen worker 63 - jepsen.util 9873 :ok :read [21 20] INFO [2019-12-17 12:36:31,919] jepsen worker 64 - jepsen.util 8434 :invoke :write [21 28] INFO [2019-12-17 12:36:31,919] jepsen worker 76 - jepsen.util 15286 :invoke :write [25 26] INFO [2019-12-17 12:36:31,920] jepsen worker 38 - jepsen.util 11378 :ok :write [12 45] INFO [2019-12-17 12:36:31,921] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63090,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:36:31,922] jepsen worker 27 - jepsen.util 10197 :invoke :read [9 nil] INFO [2019-12-17 12:36:31,922] jepsen worker 63 - jepsen.util 9873 :invoke :read [21 nil] INFO [2019-12-17 12:36:31,930] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63073,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:36:31,937] jepsen worker 41 - jepsen.util 15791 :invoke :write [13 20] INFO [2019-12-17 12:36:31,942] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63090,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:36:31,944] jepsen worker 85 - jepsen.util 15655 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:36:31,944] jepsen worker 14 - jepsen.util 10634 :info :write [4 16] :RequestCanceledException INFO [2019-12-17 12:36:31,948] jepsen worker 38 - jepsen.util 11378 :invoke :read [12 nil] INFO [2019-12-17 12:36:31,958] jepsen worker 35 - jepsen.util 14705 :info :write [11 2] :RequestCanceledException INFO [2019-12-17 12:36:31,960] jepsen worker 44 - jepsen.util 15254 :info :cas [14 [43 9]] :RequestCanceledException INFO [2019-12-17 12:36:31,963] jepsen worker 55 - jepsen.util 11485 :info :write [18 31] :RequestCanceledException INFO [2019-12-17 12:36:31,965] jepsen worker 56 - jepsen.util 11486 :info :write [18 8] :RequestCanceledException INFO [2019-12-17 12:36:31,992] jepsen worker 62 - jepsen.util 15182 :info :cas [20 [7 29]] :RequestCanceledException INFO [2019-12-17 12:36:32,005] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63073,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:36:32,008] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63092,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:36:32,018] jepsen worker 52 - jepsen.util 10672 :fail :read [17 nil] :RequestTimeoutException INFO [2019-12-17 12:36:32,022] jepsen worker 85 - jepsen.util 15655 :invoke :write [28 5] INFO [2019-12-17 12:36:32,022] jepsen worker 46 - jepsen.util 10666 :ok :write [15 20] INFO [2019-12-17 12:36:32,051] jepsen worker 60 - jepsen.util 15810 :invoke :write [20 40] INFO [2019-12-17 12:36:32,062] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63078,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:36:32,067] jepsen worker 52 - jepsen.util 10672 :invoke :read [17 nil] INFO [2019-12-17 12:36:32,077] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63092,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:36:32,091] jepsen worker 23 - jepsen.util 14693 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:36:32,091] jepsen worker 46 - jepsen.util 10666 :invoke :write [15 14] INFO [2019-12-17 12:36:32,112] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63078,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:36:32,117] jepsen worker 40 - jepsen.util 15070 :info :write [13 14] :RequestCanceledException INFO [2019-12-17 12:36:32,118] jepsen worker 6 - jepsen.util 15216 :invoke :write [2 0] INFO [2019-12-17 12:36:32,132] jepsen worker 23 - jepsen.util 14693 :invoke :cas [7 [33 13]] INFO [2019-12-17 12:36:32,135] jepsen worker 11 - jepsen.util 10631 :info :cas [3 [27 31]] :RequestCanceledException INFO [2019-12-17 12:36:32,135] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63099,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:36:32,154] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63099,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:36:32,157] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63102,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:36:32,166] jepsen worker 78 - jepsen.util 10968 :info :write [26 14] :RequestCanceledException INFO [2019-12-17 12:36:32,197] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:36:32,197] jepsen nemesis - couchbase.util Rebalance check count: 492 INFO [2019-12-17 12:36:32,210] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63105,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:36:32,211] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63102,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:36:32,211] jepsen nemesis - couchbase.util Rebalance stuck count: 473 INFO [2019-12-17 12:36:32,214] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63105,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:36:32,222] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63110,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:36:32,233] jepsen worker 45 - jepsen.util 11025 :fail :read [15 nil] :RequestTimeoutException INFO [2019-12-17 12:36:32,252] jepsen worker 45 - jepsen.util 11025 :invoke :read [15 nil] INFO [2019-12-17 12:36:32,253] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63110,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:36:32,264] jepsen worker 55 - jepsen.util 11575 :invoke :read [18 nil] INFO [2019-12-17 12:36:32,264] jepsen worker 11 - jepsen.util 10721 :invoke :cas [3 [42 15]] INFO [2019-12-17 12:36:32,273] jepsen worker 53 - jepsen.util 10133 :invoke :write [17 45] INFO [2019-12-17 12:36:32,309] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63114,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:36:32,322] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63114,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:36:32,367] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63115,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:36:32,386] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63115,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:36:32,392] jepsen worker 26 - jepsen.util 15416 :info :cas [8 [31 0]] :RequestCanceledException INFO [2019-12-17 12:36:32,394] jepsen worker 14 - jepsen.util 10724 :invoke :write [4 6] INFO [2019-12-17 12:36:32,394] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63116,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:36:32,397] jepsen worker 0 - jepsen.util 15750 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:36:32,411] jepsen worker 58 - jepsen.util 15268 :info :cas [19 [25 28]] :RequestCanceledException INFO [2019-12-17 12:36:32,418] jepsen worker 35 - jepsen.util 14795 :invoke :read [11 nil] INFO [2019-12-17 12:36:32,418] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63116,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:36:32,420] jepsen worker 4 - jepsen.util 10084 :info :cas [1 [5 44]] :RequestCanceledException INFO [2019-12-17 12:36:32,420] jepsen worker 0 - jepsen.util 15750 :invoke :cas [0 [26 12]] INFO [2019-12-17 12:36:32,483] jepsen worker 4 - jepsen.util 10174 :invoke :cas [1 [8 5]] INFO [2019-12-17 12:36:32,492] jepsen worker 5 - jepsen.util 10445 :info :cas [1 [14 27]] :RequestCanceledException INFO [2019-12-17 12:36:32,500] jepsen worker 62 - jepsen.util 15272 :invoke :cas [20 [13 26]] INFO [2019-12-17 12:36:32,501] jepsen worker 78 - jepsen.util 11058 :invoke :write [26 33] INFO [2019-12-17 12:36:32,506] jepsen worker 75 - jepsen.util 14475 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:36:32,508] jepsen worker 75 - jepsen.util 14475 :invoke :write [25 34] INFO [2019-12-17 12:36:32,510] jepsen worker 1 - jepsen.util 14941 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:36:32,521] jepsen worker 65 - jepsen.util 9335 :info :write [21 21] :RequestCanceledException INFO [2019-12-17 12:36:32,525] jepsen worker 56 - jepsen.util 11576 :invoke :cas [18 [15 38]] INFO [2019-12-17 12:36:32,529] jepsen worker 10 - jepsen.util 10630 :info :write [3 4] :RequestCanceledException INFO [2019-12-17 12:36:32,532] jepsen worker 1 - jepsen.util 14941 :invoke :read [0 nil] INFO [2019-12-17 12:36:32,533] jepsen worker 61 - jepsen.util 15631 :info :write [20 43] :RequestCanceledException INFO [2019-12-17 12:36:32,540] jepsen worker 12 - jepsen.util 11172 :info :write [4 0] :RequestCanceledException INFO [2019-12-17 12:36:32,544] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63123,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:36:32,558] jepsen worker 7 - jepsen.util 14767 :info :write [2 37] :RequestCanceledException INFO [2019-12-17 12:36:32,598] jepsen worker 37 - jepsen.util 10477 :info :cas [12 [31 0]] :RequestCanceledException INFO [2019-12-17 12:36:32,599] jepsen worker 83 - jepsen.util 15023 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:36:32,601] jepsen worker 83 - jepsen.util 15023 :invoke :read [27 nil] INFO [2019-12-17 12:36:32,614] jepsen worker 66 - jepsen.util 14646 :info :write [22 13] :RequestCanceledException INFO [2019-12-17 12:36:32,625] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63123,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:36:32,641] jepsen worker 44 - jepsen.util 15344 :invoke :read [14 nil] INFO [2019-12-17 12:36:32,650] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63128,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:36:32,654] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63128,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:36:32,659] jepsen worker 40 - jepsen.util 15160 :invoke :write [13 38] INFO [2019-12-17 12:36:32,717] jepsen worker 3 - jepsen.util 10533 :fail :read [1 nil] :RequestTimeoutException INFO [2019-12-17 12:36:32,721] jepsen worker 3 - jepsen.util 10533 :invoke :read [1 nil] INFO [2019-12-17 12:36:32,733] jepsen worker 82 - jepsen.util 14932 :info :cas [27 [1 20]] :RequestCanceledException INFO [2019-12-17 12:36:32,780] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63133,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:36:32,821] jepsen worker 19 - jepsen.util 11179 :info :write [6 11] :RequestCanceledException INFO [2019-12-17 12:36:32,853] jepsen worker 5 - jepsen.util 10535 :invoke :cas [1 [21 20]] INFO [2019-12-17 12:36:32,858] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63133,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:36:32,864] jepsen worker 65 - jepsen.util 9425 :invoke :cas [21 [6 44]] INFO [2019-12-17 12:36:32,872] jepsen worker 7 - jepsen.util 14857 :invoke :read [2 nil] INFO [2019-12-17 12:36:32,874] jepsen worker 49 - jepsen.util 14539 :info :write [16 44] :RequestCanceledException INFO [2019-12-17 12:36:32,894] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63138,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:36:32,916] jepsen worker 87 - jepsen.util 10257 :info :cas [29 [31 48]] :RequestCanceledException INFO [2019-12-17 12:36:32,931] jepsen worker 26 - jepsen.util 15506 :invoke :write [8 29] INFO [2019-12-17 12:36:32,932] jepsen worker 37 - jepsen.util 10567 :invoke :write [12 5] INFO [2019-12-17 12:36:32,947] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63148,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:36:32,948] jepsen worker 39 - jepsen.util 14439 :info :cas [13 [46 43]] :RequestCanceledException INFO [2019-12-17 12:36:32,949] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63138,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:36:32,977] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63139,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:36:32,990] jepsen worker 12 - jepsen.util 11262 :invoke :cas [4 [11 20]] INFO [2019-12-17 12:36:32,992] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63148,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:36:32,996] jepsen worker 58 - jepsen.util 15358 :invoke :cas [19 [10 44]] INFO [2019-12-17 12:36:32,996] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63139,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:36:33,001] jepsen worker 19 - jepsen.util 11269 :invoke :write [6 25] INFO [2019-12-17 12:36:33,003] jepsen worker 10 - jepsen.util 10720 :invoke :read [3 nil] INFO [2019-12-17 12:36:33,004] jepsen worker 61 - jepsen.util 15721 :invoke :write [20 45] INFO [2019-12-17 12:36:33,029] jepsen worker 66 - jepsen.util 14736 :invoke :write [22 40] INFO [2019-12-17 12:36:33,029] jepsen worker 8 - jepsen.util 14588 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:36:33,031] jepsen worker 8 - jepsen.util 14588 :invoke :read [2 nil] INFO [2019-12-17 12:36:33,031] jepsen worker 82 - jepsen.util 15022 :invoke :read [27 nil] INFO [2019-12-17 12:36:33,047] jepsen worker 69 - jepsen.util 10959 :ok :read [23 8] INFO [2019-12-17 12:36:33,048] jepsen worker 69 - jepsen.util 10959 :invoke :write [23 21] INFO [2019-12-17 12:36:33,050] jepsen worker 36 - jepsen.util 11736 :info :cas [12 [31 12]] :RequestCanceledException INFO [2019-12-17 12:36:33,056] jepsen worker 9 - jepsen.util 11709 :ok :read [3 12] INFO [2019-12-17 12:36:33,056] jepsen worker 9 - jepsen.util 11709 :invoke :cas [3 [22 12]] INFO [2019-12-17 12:36:33,067] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63154,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:36:33,069] jepsen worker 87 - jepsen.util 10347 :invoke :cas [29 [32 17]] INFO [2019-12-17 12:36:33,069] jepsen worker 36 - jepsen.util 11826 :invoke :write [12 43] INFO [2019-12-17 12:36:33,098] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63154,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:36:33,105] jepsen worker 77 - jepsen.util 15467 :info :cas [25 [29 15]] :RequestCanceledException INFO [2019-12-17 12:36:33,105] jepsen worker 49 - jepsen.util 14629 :invoke :write [16 33] INFO [2019-12-17 12:36:33,120] jepsen worker 39 - jepsen.util 14529 :invoke :write [13 5] INFO [2019-12-17 12:36:33,113] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63158,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:36:33,129] jepsen worker 81 - jepsen.util 14481 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:36:33,135] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63158,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:36:33,137] jepsen worker 84 - jepsen.util 14484 :info :cas [28 [8 30]] :RequestCanceledException INFO [2019-12-17 12:36:33,140] jepsen worker 79 - jepsen.util 11149 :info :cas [26 [6 9]] :RequestCanceledException INFO [2019-12-17 12:36:33,142] jepsen worker 81 - jepsen.util 14481 :invoke :write [27 32] INFO [2019-12-17 12:36:33,142] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63159,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:36:33,146] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63159,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:36:33,170] jepsen worker 77 - jepsen.util 15557 :invoke :read [25 nil] INFO [2019-12-17 12:36:33,172] jepsen worker 79 - jepsen.util 11239 :invoke :write [26 22] INFO [2019-12-17 12:36:33,175] jepsen worker 84 - jepsen.util 14574 :invoke :read [28 nil] INFO [2019-12-17 12:36:33,193] jepsen worker 72 - jepsen.util 11142 :fail :cas [24 [40 29]] :ValueNotSwapFrom INFO [2019-12-17 12:36:33,195] jepsen worker 72 - jepsen.util 11142 :invoke :cas [24 [38 18]] INFO [2019-12-17 12:36:33,200] jepsen worker 20 - jepsen.util 10640 :fail :cas [6 [36 44]] :ValueNotSwapFrom INFO [2019-12-17 12:36:33,200] jepsen worker 18 - jepsen.util 11088 :ok :read [6 10] INFO [2019-12-17 12:36:33,202] jepsen worker 20 - jepsen.util 10640 :invoke :cas [6 [40 20]] INFO [2019-12-17 12:36:33,209] jepsen worker 18 - jepsen.util 11088 :invoke :cas [6 [24 27]] INFO [2019-12-17 12:36:33,213] jepsen worker 28 - jepsen.util 9028 :info :write [9 5] :RequestCanceledException INFO [2019-12-17 12:36:33,213] jepsen worker 47 - jepsen.util 11567 :info :write [15 7] :RequestCanceledException INFO [2019-12-17 12:36:33,228] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63179,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:36:33,237] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63179,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:36:33,238] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63185,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:36:33,239] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63185,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:36:33,270] jepsen worker 25 - jepsen.util 15775 :info :write [8 17] :RequestCanceledException INFO [2019-12-17 12:36:33,283] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63173,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:36:33,287] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63173,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:36:33,288] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63180,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:36:33,289] jepsen worker 80 - jepsen.util 10610 :fail :cas [26 [28 36]] :ValueNotSwapFrom INFO [2019-12-17 12:36:33,291] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63180,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:36:33,291] jepsen worker 29 - jepsen.util 9299 :ok :read [9 28] INFO [2019-12-17 12:36:33,295] jepsen worker 80 - jepsen.util 10610 :invoke :write [26 40] INFO [2019-12-17 12:36:33,295] jepsen worker 29 - jepsen.util 9299 :invoke :write [9 33] INFO [2019-12-17 12:36:33,295] jepsen worker 28 - jepsen.util 9118 :invoke :read [9 nil] INFO [2019-12-17 12:36:33,312] jepsen worker 25 - jepsen.util 15865 :invoke :cas [8 [26 46]] INFO [2019-12-17 12:36:33,314] jepsen worker 47 - jepsen.util 11657 :invoke :write [15 5] INFO [2019-12-17 12:36:33,356] jepsen worker 16 - jepsen.util 16216 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:36:33,356] jepsen worker 16 - jepsen.util 16216 :invoke :cas [5 [27 6]] INFO [2019-12-17 12:36:33,452] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63208,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:36:33,459] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63208,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:36:33,473] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:36:33,475] jepsen nemesis - couchbase.util Rebalance check count: 493 INFO [2019-12-17 12:36:33,476] jepsen nemesis - couchbase.util Rebalance stuck count: 474 INFO [2019-12-17 12:36:33,488] jepsen worker 74 - jepsen.util 10784 :fail :cas [24 [1 48]] :ValueNotSwapFrom INFO [2019-12-17 12:36:33,489] jepsen worker 74 - jepsen.util 10784 :invoke :read [24 nil] INFO [2019-12-17 12:36:33,523] jepsen worker 89 - jepsen.util 11069 :fail :cas [29 [11 15]] :ValueNotSwapFrom INFO [2019-12-17 12:36:33,524] jepsen worker 89 - jepsen.util 11069 :invoke :read [29 nil] INFO [2019-12-17 12:36:33,532] jepsen worker 86 - jepsen.util 15746 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:36:33,532] jepsen worker 68 - jepsen.util 14018 :info :write [22 40] :RequestCanceledException INFO [2019-12-17 12:36:33,533] jepsen worker 86 - jepsen.util 15746 :invoke :read [28 nil] INFO [2019-12-17 12:36:33,540] jepsen worker 70 - jepsen.util 11500 :info :write [23 16] :RequestCanceledException INFO [2019-12-17 12:36:33,543] jepsen worker 68 - jepsen.util 14108 :invoke :write [22 26] INFO [2019-12-17 12:36:33,544] jepsen worker 70 - jepsen.util 11590 :invoke :write [23 28] INFO [2019-12-17 12:36:33,577] jepsen worker 32 - jepsen.util 11192 :ok :read [10 43] INFO [2019-12-17 12:36:33,578] jepsen worker 32 - jepsen.util 11192 :invoke :read [10 nil] INFO [2019-12-17 12:36:33,601] jepsen worker 54 - jepsen.util 11754 :fail :cas [18 [1 35]] :ValueNotSwapFrom INFO [2019-12-17 12:36:33,604] jepsen worker 54 - jepsen.util 11754 :invoke :read [18 nil] INFO [2019-12-17 12:36:33,604] jepsen worker 43 - jepsen.util 14893 :info :cas [14 [15 44]] :RequestCanceledException INFO [2019-12-17 12:36:33,619] jepsen worker 2 - jepsen.util 14402 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:36:33,620] jepsen worker 2 - jepsen.util 14402 :invoke :read [0 nil] INFO [2019-12-17 12:36:33,738] jepsen worker 79 - jepsen.util 11239 :ok :write [26 22] INFO [2019-12-17 12:36:33,746] jepsen worker 79 - jepsen.util 11239 :invoke :write [26 13] INFO [2019-12-17 12:36:33,747] jepsen worker 73 - jepsen.util 10243 :ok :write [24 40] INFO [2019-12-17 12:36:33,747] jepsen worker 29 - jepsen.util 9299 :ok :write [9 33] INFO [2019-12-17 12:36:33,747] jepsen worker 73 - jepsen.util 10243 :invoke :read [24 nil] INFO [2019-12-17 12:36:33,747] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63232,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:36:33,747] jepsen worker 57 - jepsen.util 14727 :info :cas [19 [49 2]] :RequestCanceledException INFO [2019-12-17 12:36:33,747] jepsen worker 24 - jepsen.util 13434 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:36:33,748] jepsen worker 42 - jepsen.util 15612 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:36:33,750] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63232,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:36:33,750] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63233,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:36:33,750] jepsen worker 42 - jepsen.util 15612 :invoke :read [14 nil] INFO [2019-12-17 12:36:33,750] jepsen worker 24 - jepsen.util 13434 :invoke :write [8 5] INFO [2019-12-17 12:36:33,752] jepsen worker 29 - jepsen.util 9299 :invoke :cas [9 [23 4]] INFO [2019-12-17 12:36:33,755] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63233,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:36:33,783] jepsen worker 43 - jepsen.util 14983 :invoke :write [14 10] INFO [2019-12-17 12:36:33,819] jepsen worker 48 - jepsen.util 13818 :info :write [16 36] :RequestCanceledException INFO [2019-12-17 12:36:33,827] jepsen worker 13 - jepsen.util 10363 :fail :cas [4 [46 25]] :ValueNotSwapFrom INFO [2019-12-17 12:36:33,834] jepsen worker 57 - jepsen.util 14817 :invoke :write [19 45] INFO [2019-12-17 12:36:33,849] jepsen worker 13 - jepsen.util 10363 :invoke :cas [4 [9 47]] INFO [2019-12-17 12:36:33,897] jepsen worker 48 - jepsen.util 13908 :invoke :write [16 43] INFO [2019-12-17 12:36:34,099] jepsen worker 71 - jepsen.util 11951 :ok :read [23 8] INFO [2019-12-17 12:36:34,100] jepsen worker 71 - jepsen.util 11951 :invoke :read [23 nil] INFO [2019-12-17 12:36:34,102] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63249,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:36:34,102] jepsen worker 88 - jepsen.util 11158 :info :write [29 35] :RequestCanceledException INFO [2019-12-17 12:36:34,106] jepsen worker 33 - jepsen.util 14793 :info :write [11 34] :RequestCanceledException INFO [2019-12-17 12:36:34,107] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63249,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:36:34,107] jepsen worker 51 - jepsen.util 10311 :info :write [17 46] :RequestCanceledException INFO [2019-12-17 12:36:34,110] jepsen worker 27 - jepsen.util 10197 :ok :read [9 28] INFO [2019-12-17 12:36:34,112] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63242,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:36:34,112] jepsen worker 45 - jepsen.util 11025 :ok :read [15 20] INFO [2019-12-17 12:36:34,112] jepsen worker 11 - jepsen.util 10721 :fail :cas [3 [42 15]] :ValueNotSwapFrom INFO [2019-12-17 12:36:34,116] jepsen worker 38 - jepsen.util 11378 :fail :read [12 nil] :RequestTimeoutException INFO [2019-12-17 12:36:34,117] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63251,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:36:34,117] jepsen worker 30 - jepsen.util 10380 :info :cas [10 [41 37]] :RequestCanceledException INFO [2019-12-17 12:36:34,119] jepsen worker 59 - jepsen.util 13919 :info :write [19 22] :RequestCanceledException INFO [2019-12-17 12:36:34,121] jepsen worker 63 - jepsen.util 9873 :fail :read [21 nil] :RequestTimeoutException INFO [2019-12-17 12:36:34,121] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63242,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:36:34,121] jepsen worker 27 - jepsen.util 10197 :invoke :write [9 14] INFO [2019-12-17 12:36:34,121] jepsen worker 38 - jepsen.util 11378 :invoke :write [12 37] INFO [2019-12-17 12:36:34,123] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63251,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:36:34,126] jepsen worker 11 - jepsen.util 10721 :invoke :read [3 nil] INFO [2019-12-17 12:36:34,129] jepsen worker 63 - jepsen.util 9873 :invoke :cas [21 [46 27]] INFO [2019-12-17 12:36:34,129] jepsen worker 45 - jepsen.util 11025 :invoke :write [15 47] INFO [2019-12-17 12:36:34,130] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63243,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:36:34,131] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63253,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:36:34,144] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63243,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:36:34,146] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63253,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:36:34,153] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63244,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:36:34,171] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63244,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:36:34,172] jepsen worker 88 - jepsen.util 11248 :invoke :cas [29 [13 0]] INFO [2019-12-17 12:36:34,178] jepsen worker 30 - jepsen.util 10470 :invoke :write [10 31] INFO [2019-12-17 12:36:34,191] jepsen worker 51 - jepsen.util 10401 :invoke :read [17 nil] INFO [2019-12-17 12:36:34,219] jepsen worker 47 - jepsen.util 11657 :ok :write [15 5] INFO [2019-12-17 12:36:34,221] jepsen worker 47 - jepsen.util 11657 :invoke :read [15 nil] INFO [2019-12-17 12:36:34,227] jepsen worker 52 - jepsen.util 10672 :fail :read [17 nil] :RequestTimeoutException INFO [2019-12-17 12:36:34,227] jepsen worker 52 - jepsen.util 10672 :invoke :read [17 nil] INFO [2019-12-17 12:36:34,240] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63260,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:36:34,241] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63262,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:36:34,241] jepsen worker 23 - jepsen.util 14693 :info :cas [7 [33 13]] :RequestCanceledException INFO [2019-12-17 12:36:34,262] jepsen worker 33 - jepsen.util 14883 :invoke :read [11 nil] INFO [2019-12-17 12:36:34,264] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63260,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:36:34,284] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63262,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:36:34,291] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63264,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:36:34,292] jepsen worker 59 - jepsen.util 14009 :invoke :cas [19 [39 19]] INFO [2019-12-17 12:36:34,296] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63272,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:36:34,310] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63264,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:36:34,317] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63272,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:36:34,317] jepsen worker 55 - jepsen.util 11575 :fail :read [18 nil] :RequestTimeoutException INFO [2019-12-17 12:36:34,324] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63270,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:36:34,325] jepsen worker 55 - jepsen.util 11575 :invoke :read [18 nil] INFO [2019-12-17 12:36:34,325] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63273,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:36:34,331] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63270,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:36:34,335] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63273,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:36:34,374] jepsen worker 23 - jepsen.util 14783 :invoke :write [7 23] INFO [2019-12-17 12:36:34,448] jepsen worker 46 - jepsen.util 10666 :ok :write [15 14] INFO [2019-12-17 12:36:34,448] jepsen worker 46 - jepsen.util 10666 :invoke :read [15 nil] INFO [2019-12-17 12:36:34,459] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63279,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:36:34,460] jepsen worker 35 - jepsen.util 14795 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:36:34,461] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63279,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:36:34,462] jepsen worker 35 - jepsen.util 14795 :invoke :cas [11 [4 27]] INFO [2019-12-17 12:36:34,472] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63282,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:36:34,478] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63282,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:36:34,480] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63283,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:36:34,487] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63283,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:36:34,565] jepsen worker 0 - jepsen.util 15750 :info :cas [0 [26 12]] :RequestCanceledException INFO [2019-12-17 12:36:34,567] jepsen worker 4 - jepsen.util 10174 :info :cas [1 [8 5]] :RequestCanceledException INFO [2019-12-17 12:36:34,586] jepsen worker 64 - jepsen.util 8434 :ok :write [21 28] INFO [2019-12-17 12:36:34,586] jepsen worker 64 - jepsen.util 8434 :invoke :cas [21 [49 5]] INFO [2019-12-17 12:36:34,618] jepsen worker 62 - jepsen.util 15272 :info :cas [20 [13 26]] :RequestCanceledException INFO [2019-12-17 12:36:34,635] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestTimeoutException: CarrierBucketConfigRequest {"retried":0,"reason":"TIMEOUT","requestId":63098,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:36:34,648] jepsen worker 10 - jepsen.util 10720 :ok :read [3 12] INFO [2019-12-17 12:36:34,650] jepsen worker 56 - jepsen.util 11576 :info :cas [18 [15 38]] :RequestCanceledException INFO [2019-12-17 12:36:34,653] jepsen worker 10 - jepsen.util 10720 :invoke :read [3 nil] INFO [2019-12-17 12:36:34,655] jepsen worker 1 - jepsen.util 14941 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:36:34,669] jepsen worker 1 - jepsen.util 14941 :invoke :read [0 nil] INFO [2019-12-17 12:36:34,689] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63292,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:36:34,691] jepsen worker 0 - jepsen.util 15840 :invoke :read [0 nil] INFO [2019-12-17 12:36:34,699] jepsen worker 18 - jepsen.util 11088 :fail :cas [6 [24 27]] :ValueNotSwapFrom INFO [2019-12-17 12:36:34,719] jepsen worker 18 - jepsen.util 11088 :invoke :cas [6 [45 35]] INFO [2019-12-17 12:36:34,745] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63292,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:36:34,752] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:36:34,752] jepsen worker 83 - jepsen.util 15023 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:36:34,752] jepsen worker 34 - jepsen.util 15244 :info :write [11 38] :RequestCanceledException INFO [2019-12-17 12:36:34,760] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63297,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:36:34,768] jepsen nemesis - couchbase.util Rebalance check count: 494 INFO [2019-12-17 12:36:34,775] jepsen nemesis - couchbase.util Rebalance stuck count: 475 INFO [2019-12-17 12:36:34,783] jepsen worker 83 - jepsen.util 15023 :invoke :write [27 31] INFO [2019-12-17 12:36:34,789] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63297,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:36:34,790] jepsen worker 44 - jepsen.util 15344 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:36:34,797] jepsen worker 44 - jepsen.util 15344 :invoke :cas [14 [33 28]] INFO [2019-12-17 12:36:34,798] jepsen worker 72 - jepsen.util 11142 :fail :cas [24 [38 18]] :ValueNotSwapFrom INFO [2019-12-17 12:36:34,799] jepsen worker 62 - jepsen.util 15362 :invoke :cas [20 [10 28]] INFO [2019-12-17 12:36:34,800] jepsen worker 3 - jepsen.util 10533 :fail :read [1 nil] :RequestTimeoutException INFO [2019-12-17 12:36:34,804] jepsen worker 72 - jepsen.util 11142 :invoke :write [24 8] INFO [2019-12-17 12:36:34,810] jepsen worker 3 - jepsen.util 10533 :invoke :read [1 nil] INFO [2019-12-17 12:36:34,814] jepsen worker 56 - jepsen.util 11666 :invoke :cas [18 [27 35]] INFO [2019-12-17 12:36:34,813] jepsen worker 20 - jepsen.util 10640 :fail :cas [6 [40 20]] :ValueNotSwapFrom INFO [2019-12-17 12:36:34,821] jepsen worker 20 - jepsen.util 10640 :invoke :cas [6 [3 48]] INFO [2019-12-17 12:36:34,824] jepsen worker 4 - jepsen.util 10264 :invoke :cas [1 [49 28]] INFO [2019-12-17 12:36:34,829] jepsen worker 67 - jepsen.util 14917 :info :write [22 33] :RequestCanceledException INFO [2019-12-17 12:36:34,935] jepsen worker 7 - jepsen.util 14857 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:36:34,937] jepsen worker 65 - jepsen.util 9425 :info :cas [21 [6 44]] :RequestCanceledException INFO [2019-12-17 12:36:34,939] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63312,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:36:34,945] jepsen worker 7 - jepsen.util 14857 :invoke :cas [2 [44 41]] INFO [2019-12-17 12:36:34,952] jepsen worker 5 - jepsen.util 10535 :info :cas [1 [21 20]] :RequestCanceledException INFO [2019-12-17 12:36:34,966] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63312,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:36:34,996] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63313,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:36:35,010] jepsen worker 34 - jepsen.util 15334 :invoke :write [11 5] INFO [2019-12-17 12:36:35,023] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63313,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:36:35,053] jepsen worker 8 - jepsen.util 14588 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:36:35,055] jepsen worker 58 - jepsen.util 15358 :info :cas [19 [10 44]] :RequestCanceledException INFO [2019-12-17 12:36:35,075] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63316,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:36:35,086] jepsen worker 8 - jepsen.util 14588 :invoke :read [2 nil] INFO [2019-12-17 12:36:35,115] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63316,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:36:35,115] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63320,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:36:35,115] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63320,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:36:35,115] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63321,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:36:35,115] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63321,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:36:35,125] jepsen worker 21 - jepsen.util 14241 :info :write [7 48] :RequestCanceledException INFO [2019-12-17 12:36:35,135] jepsen worker 82 - jepsen.util 15022 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:36:35,147] jepsen worker 82 - jepsen.util 15022 :invoke :cas [27 [44 27]] INFO [2019-12-17 12:36:35,160] jepsen worker 12 - jepsen.util 11262 :fail :cas [4 [11 20]] :ValueNotSwapFrom INFO [2019-12-17 12:36:35,160] jepsen worker 5 - jepsen.util 10625 :invoke :cas [1 [27 29]] INFO [2019-12-17 12:36:35,160] jepsen worker 87 - jepsen.util 10347 :info :cas [29 [32 17]] :RequestCanceledException INFO [2019-12-17 12:36:35,165] jepsen worker 53 - jepsen.util 10133 :info :write [17 45] :RequestCanceledException INFO [2019-12-17 12:36:35,174] jepsen worker 12 - jepsen.util 11262 :invoke :cas [4 [43 33]] INFO [2019-12-17 12:36:35,205] jepsen worker 65 - jepsen.util 9515 :invoke :read [21 nil] INFO [2019-12-17 12:36:35,230] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63326,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:36:35,235] jepsen worker 50 - jepsen.util 15440 :info :write [16 2] :RequestCanceledException INFO [2019-12-17 12:36:35,237] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63326,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:36:35,252] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63335,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:36:35,256] jepsen worker 67 - jepsen.util 15007 :invoke :read [22 nil] INFO [2019-12-17 12:36:35,257] jepsen worker 84 - jepsen.util 14574 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:36:35,265] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63335,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:36:35,267] jepsen worker 84 - jepsen.util 14574 :invoke :read [28 nil] INFO [2019-12-17 12:36:35,277] jepsen worker 77 - jepsen.util 15557 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:36:35,288] jepsen worker 22 - jepsen.util 14782 :info :write [7 45] :RequestCanceledException INFO [2019-12-17 12:36:35,288] jepsen worker 60 - jepsen.util 15810 :info :write [20 40] :RequestCanceledException INFO [2019-12-17 12:36:35,293] jepsen worker 77 - jepsen.util 15557 :invoke :write [25 42] INFO [2019-12-17 12:36:35,296] jepsen worker 15 - jepsen.util 13785 :info :write [5 9] :RequestCanceledException INFO [2019-12-17 12:36:35,324] jepsen worker 87 - jepsen.util 10437 :invoke :cas [29 [43 36]] INFO [2019-12-17 12:36:35,345] jepsen worker 79 - jepsen.util 11239 :ok :write [26 13] INFO [2019-12-17 12:36:35,352] jepsen worker 79 - jepsen.util 11239 :invoke :read [26 nil] INFO [2019-12-17 12:36:35,366] jepsen worker 6 - jepsen.util 15216 :info :write [2 0] :RequestCanceledException INFO [2019-12-17 12:36:35,370] jepsen worker 14 - jepsen.util 10724 :info :write [4 6] :RequestCanceledException INFO [2019-12-17 12:36:35,373] jepsen worker 58 - jepsen.util 15448 :invoke :read [19 nil] INFO [2019-12-17 12:36:35,376] jepsen worker 85 - jepsen.util 15655 :info :write [28 5] :RequestCanceledException INFO [2019-12-17 12:36:35,405] jepsen worker 28 - jepsen.util 9118 :fail :read [9 nil] :RequestTimeoutException INFO [2019-12-17 12:36:35,408] jepsen worker 28 - jepsen.util 9118 :invoke :cas [9 [18 0]] INFO [2019-12-17 12:36:35,408] jepsen worker 21 - jepsen.util 14331 :invoke :cas [7 [13 44]] INFO [2019-12-17 12:36:35,415] jepsen worker 70 - jepsen.util 11590 :ok :write [23 28] INFO [2019-12-17 12:36:35,418] jepsen worker 70 - jepsen.util 11590 :invoke :read [23 nil] INFO [2019-12-17 12:36:35,424] jepsen worker 17 - jepsen.util 14777 :info :write [5 9] :RequestCanceledException INFO [2019-12-17 12:36:35,425] jepsen worker 53 - jepsen.util 10223 :invoke :cas [17 [4 2]] INFO [2019-12-17 12:36:35,425] jepsen worker 78 - jepsen.util 11058 :info :write [26 33] :RequestCanceledException INFO [2019-12-17 12:36:35,432] jepsen worker 19 - jepsen.util 11269 :info :write [6 25] :RequestCanceledException INFO [2019-12-17 12:36:35,433] jepsen worker 25 - jepsen.util 15865 :info :cas [8 [26 46]] :RequestCanceledException INFO [2019-12-17 12:36:35,441] jepsen worker 31 - jepsen.util 9391 :info :write [10 48] :RequestCanceledException INFO [2019-12-17 12:36:35,442] jepsen worker 36 - jepsen.util 11826 :info :write [12 43] :RequestCanceledException INFO [2019-12-17 12:36:35,456] jepsen worker 15 - jepsen.util 13875 :invoke :read [5 nil] INFO [2019-12-17 12:36:35,462] jepsen worker 37 - jepsen.util 10567 :info :write [12 5] :RequestCanceledException INFO [2019-12-17 12:36:35,463] jepsen worker 16 - jepsen.util 16216 :info :cas [5 [27 6]] :RequestCanceledException INFO [2019-12-17 12:36:35,475] jepsen worker 76 - jepsen.util 15286 :info :write [25 26] :RequestCanceledException INFO [2019-12-17 12:36:35,491] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63352,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:36:35,492] jepsen worker 14 - jepsen.util 10814 :invoke :read [4 nil] INFO [2019-12-17 12:36:35,497] jepsen worker 41 - jepsen.util 15791 :info :write [13 20] :RequestCanceledException INFO [2019-12-17 12:36:35,500] jepsen worker 50 - jepsen.util 15530 :invoke :read [16 nil] INFO [2019-12-17 12:36:35,503] jepsen worker 66 - jepsen.util 14736 :info :write [22 40] :RequestCanceledException INFO [2019-12-17 12:36:35,515] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63352,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:36:35,517] jepsen worker 22 - jepsen.util 14872 :invoke :read [7 nil] INFO [2019-12-17 12:36:35,521] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63353,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:36:35,523] jepsen worker 69 - jepsen.util 10959 :info :write [23 21] :RequestCanceledException INFO [2019-12-17 12:36:35,530] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63353,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:36:35,536] jepsen worker 60 - jepsen.util 15900 :invoke :write [20 14] INFO [2019-12-17 12:36:35,551] jepsen worker 26 - jepsen.util 15506 :info :write [8 29] :RequestCanceledException INFO [2019-12-17 12:36:35,552] jepsen worker 6 - jepsen.util 15306 :invoke :cas [2 [3 0]] INFO [2019-12-17 12:36:35,553] jepsen worker 39 - jepsen.util 14529 :info :write [13 5] :RequestCanceledException INFO [2019-12-17 12:36:35,553] jepsen worker 61 - jepsen.util 15721 :info :write [20 45] :RequestCanceledException INFO [2019-12-17 12:36:35,558] jepsen worker 85 - jepsen.util 15745 :invoke :read [28 nil] INFO [2019-12-17 12:36:35,564] jepsen worker 37 - jepsen.util 10657 :invoke :write [12 29] INFO [2019-12-17 12:36:35,571] jepsen worker 75 - jepsen.util 14475 :info :write [25 34] :RequestCanceledException INFO [2019-12-17 12:36:35,571] jepsen worker 40 - jepsen.util 15160 :info :write [13 38] :RequestCanceledException INFO [2019-12-17 12:36:35,598] jepsen worker 19 - jepsen.util 11359 :invoke :read [6 nil] INFO [2019-12-17 12:36:35,622] jepsen worker 80 - jepsen.util 10610 :info :write [26 40] :RequestCanceledException INFO [2019-12-17 12:36:35,628] jepsen worker 49 - jepsen.util 14629 :info :write [16 33] :RequestCanceledException INFO [2019-12-17 12:36:35,646] jepsen worker 76 - jepsen.util 15376 :invoke :read [25 nil] INFO [2019-12-17 12:36:35,655] jepsen worker 81 - jepsen.util 14481 :info :write [27 32] :RequestCanceledException INFO [2019-12-17 12:36:35,665] jepsen worker 78 - jepsen.util 11148 :invoke :read [26 nil] INFO [2019-12-17 12:36:35,666] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63367,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:36:35,672] jepsen worker 86 - jepsen.util 15746 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:36:35,673] jepsen worker 89 - jepsen.util 11069 :fail :read [29 nil] :RequestTimeoutException INFO [2019-12-17 12:36:35,673] jepsen worker 36 - jepsen.util 11916 :invoke :write [12 14] INFO [2019-12-17 12:36:35,683] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63367,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:36:35,683] jepsen worker 31 - jepsen.util 9481 :invoke :write [10 0] INFO [2019-12-17 12:36:35,696] jepsen worker 86 - jepsen.util 15746 :invoke :read [28 nil] INFO [2019-12-17 12:36:35,697] jepsen worker 89 - jepsen.util 11069 :invoke :write [29 35] INFO [2019-12-17 12:36:35,710] jepsen worker 16 - jepsen.util 16306 :invoke :write [5 36] INFO [2019-12-17 12:36:35,713] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63368,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:36:35,718] jepsen worker 41 - jepsen.util 15881 :invoke :cas [13 [12 39]] INFO [2019-12-17 12:36:35,724] jepsen worker 17 - jepsen.util 14867 :invoke :write [5 45] INFO [2019-12-17 12:36:35,725] jepsen worker 25 - jepsen.util 15955 :invoke :cas [8 [17 22]] INFO [2019-12-17 12:36:35,725] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63368,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:36:35,725] jepsen worker 2 - jepsen.util 14402 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:36:35,726] jepsen worker 69 - jepsen.util 11049 :invoke :write [23 45] INFO [2019-12-17 12:36:35,750] jepsen worker 2 - jepsen.util 14402 :invoke :write [0 32] INFO [2019-12-17 12:36:35,756] jepsen worker 30 - jepsen.util 10470 :ok :write [10 31] INFO [2019-12-17 12:36:35,757] jepsen worker 66 - jepsen.util 14826 :invoke :write [22 1] INFO [2019-12-17 12:36:35,762] jepsen worker 30 - jepsen.util 10470 :invoke :read [10 nil] INFO [2019-12-17 12:36:35,765] jepsen worker 73 - jepsen.util 10243 :fail :read [24 nil] :RequestTimeoutException INFO [2019-12-17 12:36:35,766] jepsen worker 73 - jepsen.util 10243 :invoke :read [24 nil] INFO [2019-12-17 12:36:35,789] jepsen worker 61 - jepsen.util 15811 :invoke :cas [20 [4 16]] INFO [2019-12-17 12:36:35,805] jepsen worker 39 - jepsen.util 14619 :invoke :write [13 13] INFO [2019-12-17 12:36:35,808] jepsen worker 80 - jepsen.util 10700 :invoke :write [26 11] INFO [2019-12-17 12:36:35,821] jepsen worker 26 - jepsen.util 15596 :invoke :cas [8 [15 27]] INFO [2019-12-17 12:36:35,827] jepsen worker 75 - jepsen.util 14565 :invoke :read [25 nil] INFO [2019-12-17 12:36:35,856] jepsen worker 42 - jepsen.util 15612 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:36:35,882] jepsen worker 42 - jepsen.util 15612 :invoke :read [14 nil] INFO [2019-12-17 12:36:35,889] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63379,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:36:35,890] jepsen worker 40 - jepsen.util 15250 :invoke :write [13 42] INFO [2019-12-17 12:36:35,896] jepsen worker 49 - jepsen.util 14719 :invoke :write [16 3] INFO [2019-12-17 12:36:35,908] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:36:35,922] jepsen worker 81 - jepsen.util 14571 :invoke :cas [27 [1 14]] INFO [2019-12-17 12:36:35,928] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63379,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:36:35,930] jepsen nemesis - couchbase.util Rebalance check count: 495 INFO [2019-12-17 12:36:35,948] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63385,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:36:35,951] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63385,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:36:35,955] jepsen nemesis - couchbase.util Rebalance stuck count: 476 INFO [2019-12-17 12:36:35,957] jepsen worker 13 - jepsen.util 10363 :info :cas [4 [9 47]] :RequestCanceledException INFO [2019-12-17 12:36:36,103] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63389,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:36:36,514] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63389,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:36:36,514] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63392,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:36:36,514] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63392,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:36:36,518] jepsen worker 54 - jepsen.util 11754 :ok :read [18 9] INFO [2019-12-17 12:36:36,518] jepsen worker 54 - jepsen.util 11754 :invoke :read [18 nil] INFO [2019-12-17 12:36:36,518] jepsen worker 13 - jepsen.util 10453 :invoke :read [4 nil] INFO [2019-12-17 12:36:36,521] jepsen worker 29 - jepsen.util 9299 :fail :cas [9 [23 4]] :ValueNotSwapFrom INFO [2019-12-17 12:36:36,524] jepsen worker 29 - jepsen.util 9299 :invoke :read [9 nil] INFO [2019-12-17 12:36:36,527] jepsen worker 74 - jepsen.util 10784 :ok :read [24 9] INFO [2019-12-17 12:36:36,528] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63401,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:36:36,529] jepsen worker 59 - jepsen.util 14009 :info :cas [19 [39 19]] :RequestCanceledException INFO [2019-12-17 12:36:36,531] jepsen worker 88 - jepsen.util 11248 :info :cas [29 [13 0]] :RequestCanceledException INFO [2019-12-17 12:36:36,531] jepsen worker 35 - jepsen.util 14795 :info :cas [11 [4 27]] :RequestCanceledException INFO [2019-12-17 12:36:36,537] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63401,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:36:36,538] jepsen worker 24 - jepsen.util 13434 :info :write [8 5] :RequestCanceledException INFO [2019-12-17 12:36:36,538] jepsen worker 68 - jepsen.util 14108 :info :write [22 26] :RequestCanceledException INFO [2019-12-17 12:36:36,531] jepsen worker 33 - jepsen.util 14883 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:36:36,539] jepsen worker 74 - jepsen.util 10784 :invoke :cas [24 [47 21]] INFO [2019-12-17 12:36:36,551] jepsen worker 55 - jepsen.util 11575 :fail :read [18 nil] :RequestTimeoutException INFO [2019-12-17 12:36:36,552] jepsen worker 11 - jepsen.util 10721 :fail :read [3 nil] :RequestTimeoutException INFO [2019-12-17 12:36:36,554] jepsen worker 43 - jepsen.util 14983 :info :write [14 10] :RequestCanceledException INFO [2019-12-17 12:36:36,588] jepsen worker 11 - jepsen.util 10721 :invoke :read [3 nil] INFO [2019-12-17 12:36:36,591] jepsen worker 48 - jepsen.util 13908 :info :write [16 43] :RequestCanceledException INFO [2019-12-17 12:36:36,594] jepsen worker 33 - jepsen.util 14883 :invoke :read [11 nil] INFO [2019-12-17 12:36:36,598] jepsen worker 57 - jepsen.util 14817 :info :write [19 45] :RequestCanceledException INFO [2019-12-17 12:36:36,599] jepsen worker 55 - jepsen.util 11575 :invoke :read [18 nil] INFO [2019-12-17 12:36:36,624] jepsen worker 38 - jepsen.util 11378 :info :write [12 37] :RequestCanceledException INFO [2019-12-17 12:36:36,633] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63407,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:36:36,653] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63405,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:36:36,663] jepsen worker 46 - jepsen.util 10666 :fail :read [15 nil] :RequestTimeoutException INFO [2019-12-17 12:36:36,665] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63407,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:36:36,670] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63405,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:36:36,674] jepsen worker 46 - jepsen.util 10666 :invoke :read [15 nil] INFO [2019-12-17 12:36:36,683] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63413,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:36:36,688] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63409,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:36:36,695] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63413,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:36:36,710] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63421,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:36:36,711] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63409,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:36:36,714] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63411,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:36:36,733] jepsen worker 45 - jepsen.util 11025 :info :write [15 47] :RequestCanceledException INFO [2019-12-17 12:36:36,743] jepsen worker 10 - jepsen.util 10720 :fail :read [3 nil] :RequestTimeoutException INFO [2019-12-17 12:36:36,751] jepsen worker 1 - jepsen.util 14941 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:36:36,772] jepsen worker 0 - jepsen.util 15840 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:36:36,791] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63421,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:36:36,797] jepsen worker 10 - jepsen.util 10720 :invoke :write [3 41] INFO [2019-12-17 12:36:36,798] jepsen worker 0 - jepsen.util 15840 :invoke :read [0 nil] INFO [2019-12-17 12:36:36,811] jepsen worker 1 - jepsen.util 14941 :invoke :cas [0 [40 27]] INFO [2019-12-17 12:36:36,812] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63411,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:36:36,845] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63425,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:36:36,852] jepsen worker 88 - jepsen.util 11338 :invoke :read [29 nil] INFO [2019-12-17 12:36:36,863] jepsen worker 44 - jepsen.util 15344 :info :cas [14 [33 28]] :RequestCanceledException INFO [2019-12-17 12:36:36,867] jepsen worker 56 - jepsen.util 11666 :info :cas [18 [27 35]] :RequestCanceledException INFO [2019-12-17 12:36:36,876] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63425,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:36:36,886] jepsen worker 62 - jepsen.util 15362 :info :cas [20 [10 28]] :RequestCanceledException INFO [2019-12-17 12:36:36,898] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63433,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:36:36,900] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63427,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:36:36,920] jepsen worker 3 - jepsen.util 10533 :fail :read [1 nil] :RequestTimeoutException INFO [2019-12-17 12:36:36,921] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63433,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:36:36,923] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63427,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:36:36,924] jepsen worker 38 - jepsen.util 11468 :invoke :write [12 31] INFO [2019-12-17 12:36:36,929] jepsen worker 3 - jepsen.util 10533 :invoke :write [1 27] INFO [2019-12-17 12:36:36,935] jepsen worker 35 - jepsen.util 14885 :invoke :read [11 nil] INFO [2019-12-17 12:36:37,052] jepsen worker 27 - jepsen.util 10197 :info :write [9 14] :RequestCanceledException INFO [2019-12-17 12:36:37,076] jepsen worker 59 - jepsen.util 14099 :invoke :read [19 nil] INFO [2019-12-17 12:36:37,098] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63439,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:36:37,098] jepsen worker 56 - jepsen.util 11756 :invoke :write [18 6] INFO [2019-12-17 12:36:37,101] jepsen worker 7 - jepsen.util 14857 :info :cas [2 [44 41]] :RequestCanceledException INFO [2019-12-17 12:36:37,111] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63439,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:36:37,121] jepsen worker 68 - jepsen.util 14198 :invoke :cas [22 [22 43]] INFO [2019-12-17 12:36:37,137] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63443,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:36:37,138] jepsen worker 8 - jepsen.util 14588 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:36:37,140] jepsen worker 48 - jepsen.util 13998 :invoke :read [16 nil] INFO [2019-12-17 12:36:37,157] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63443,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:36:37,168] jepsen worker 8 - jepsen.util 14588 :invoke :write [2 10] INFO [2019-12-17 12:36:37,177] jepsen worker 45 - jepsen.util 11115 :invoke :read [15 nil] INFO [2019-12-17 12:36:37,178] jepsen worker 82 - jepsen.util 15022 :info :cas [27 [44 27]] :RequestCanceledException INFO [2019-12-17 12:36:37,196] jepsen worker 57 - jepsen.util 14907 :invoke :write [19 29] INFO [2019-12-17 12:36:37,216] jepsen worker 62 - jepsen.util 15452 :invoke :read [20 nil] INFO [2019-12-17 12:36:37,223] jepsen worker 65 - jepsen.util 9515 :fail :read [21 nil] :RequestTimeoutException INFO [2019-12-17 12:36:37,228] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63451,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:36:37,230] jepsen worker 65 - jepsen.util 9515 :invoke :read [21 nil] INFO [2019-12-17 12:36:37,238] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63451,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:36:37,246] jepsen worker 5 - jepsen.util 10625 :info :cas [1 [27 29]] :RequestCanceledException INFO [2019-12-17 12:36:37,281] jepsen worker 43 - jepsen.util 15073 :invoke :cas [14 [8 41]] INFO [2019-12-17 12:36:37,285] jepsen worker 12 - jepsen.util 11262 :info :cas [4 [43 33]] :RequestCanceledException INFO [2019-12-17 12:36:37,297] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:36:37,300] jepsen nemesis - couchbase.util Rebalance check count: 496 INFO [2019-12-17 12:36:37,300] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63458,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:36:37,307] jepsen nemesis - couchbase.util Rebalance stuck count: 477 INFO [2019-12-17 12:36:37,351] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63458,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:36:37,352] jepsen worker 84 - jepsen.util 14574 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:36:37,370] jepsen worker 84 - jepsen.util 14574 :invoke :write [28 2] INFO [2019-12-17 12:36:37,372] jepsen worker 87 - jepsen.util 10437 :info :cas [29 [43 36]] :RequestCanceledException INFO [2019-12-17 12:36:37,377] jepsen worker 24 - jepsen.util 13524 :invoke :write [8 4] INFO [2019-12-17 12:36:37,390] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63459,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:36:37,391] jepsen worker 79 - jepsen.util 11239 :fail :read [26 nil] :RequestTimeoutException INFO [2019-12-17 12:36:37,399] jepsen worker 67 - jepsen.util 15007 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:36:37,403] jepsen worker 79 - jepsen.util 11239 :invoke :write [26 17] INFO [2019-12-17 12:36:37,409] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63459,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:36:37,416] jepsen worker 44 - jepsen.util 15434 :invoke :cas [14 [12 7]] INFO [2019-12-17 12:36:37,418] jepsen worker 67 - jepsen.util 15007 :invoke :cas [22 [21 48]] INFO [2019-12-17 12:36:37,426] jepsen worker 23 - jepsen.util 14783 :info :write [7 23] :RequestCanceledException INFO [2019-12-17 12:36:37,426] jepsen worker 5 - jepsen.util 10715 :invoke :read [1 nil] INFO [2019-12-17 12:36:37,426] jepsen worker 27 - jepsen.util 10287 :invoke :cas [9 [35 4]] INFO [2019-12-17 12:36:37,475] jepsen worker 58 - jepsen.util 15448 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:36:37,482] jepsen worker 70 - jepsen.util 11590 :fail :read [23 nil] :RequestTimeoutException INFO [2019-12-17 12:36:37,489] jepsen worker 58 - jepsen.util 15448 :invoke :read [19 nil] INFO [2019-12-17 12:36:37,509] jepsen worker 70 - jepsen.util 11590 :invoke :write [23 5] INFO [2019-12-17 12:36:37,525] jepsen worker 82 - jepsen.util 15112 :invoke :write [27 15] INFO [2019-12-17 12:36:37,903] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63470,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:36:37,904] jepsen worker 53 - jepsen.util 10223 :info :cas [17 [4 2]] :RequestCanceledException INFO [2019-12-17 12:36:37,905] jepsen worker 75 - jepsen.util 14565 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:36:37,907] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63490,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:36:37,907] jepsen worker 6 - jepsen.util 15306 :info :cas [2 [3 0]] :RequestCanceledException INFO [2019-12-17 12:36:37,909] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63470,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:36:37,910] jepsen worker 9 - jepsen.util 11709 :fail :cas [3 [22 12]] :ValueNotSwapFrom INFO [2019-12-17 12:36:37,910] jepsen worker 21 - jepsen.util 14331 :info :cas [7 [13 44]] :RequestCanceledException INFO [2019-12-17 12:36:37,910] jepsen worker 26 - jepsen.util 15596 :info :cas [8 [15 27]] :RequestCanceledException INFO [2019-12-17 12:36:37,910] jepsen worker 47 - jepsen.util 11657 :ok :read [15 5] INFO [2019-12-17 12:36:37,914] jepsen worker 50 - jepsen.util 15530 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:36:37,916] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63492,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:36:37,916] jepsen worker 42 - jepsen.util 15612 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:36:37,916] jepsen worker 12 - jepsen.util 11352 :invoke :write [4 23] INFO [2019-12-17 12:36:37,916] jepsen worker 15 - jepsen.util 13875 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:36:37,917] jepsen worker 78 - jepsen.util 11148 :fail :read [26 nil] :RequestTimeoutException INFO [2019-12-17 12:36:37,917] jepsen worker 32 - jepsen.util 11192 :ok :read [10 43] INFO [2019-12-17 12:36:37,917] jepsen worker 41 - jepsen.util 15881 :info :cas [13 [12 39]] :RequestCanceledException INFO [2019-12-17 12:36:37,917] jepsen worker 22 - jepsen.util 14872 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:36:37,918] jepsen worker 85 - jepsen.util 15745 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:36:37,919] jepsen worker 72 - jepsen.util 11142 :info :write [24 8] :RequestCanceledException INFO [2019-12-17 12:36:37,919] jepsen worker 76 - jepsen.util 15376 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:36:37,919] jepsen worker 75 - jepsen.util 14565 :invoke :read [25 nil] INFO [2019-12-17 12:36:37,923] jepsen worker 86 - jepsen.util 15746 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:36:37,924] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63490,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:36:37,926] jepsen worker 9 - jepsen.util 11709 :invoke :read [3 nil] INFO [2019-12-17 12:36:37,927] jepsen worker 61 - jepsen.util 15811 :info :cas [20 [4 16]] :RequestCanceledException INFO [2019-12-17 12:36:37,928] jepsen worker 25 - jepsen.util 15955 :info :cas [8 [17 22]] :RequestCanceledException INFO [2019-12-17 12:36:37,930] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63471,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:36:37,932] jepsen worker 50 - jepsen.util 15530 :invoke :read [16 nil] INFO [2019-12-17 12:36:37,935] jepsen worker 42 - jepsen.util 15612 :invoke :cas [14 [26 33]] INFO [2019-12-17 12:36:37,936] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63492,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:36:37,936] jepsen worker 15 - jepsen.util 13875 :invoke :cas [5 [26 31]] INFO [2019-12-17 12:36:37,937] jepsen worker 32 - jepsen.util 11192 :invoke :read [10 nil] INFO [2019-12-17 12:36:37,937] jepsen worker 47 - jepsen.util 11657 :invoke :cas [15 [36 34]] INFO [2019-12-17 12:36:37,940] jepsen worker 85 - jepsen.util 15745 :invoke :write [28 17] INFO [2019-12-17 12:36:37,942] jepsen worker 76 - jepsen.util 15376 :invoke :read [25 nil] INFO [2019-12-17 12:36:37,943] jepsen worker 86 - jepsen.util 15746 :invoke :read [28 nil] INFO [2019-12-17 12:36:37,946] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63497,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:36:37,946] jepsen worker 78 - jepsen.util 11148 :invoke :write [26 13] INFO [2019-12-17 12:36:37,947] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63471,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:36:37,949] jepsen worker 22 - jepsen.util 14872 :invoke :write [7 19] INFO [2019-12-17 12:36:37,949] jepsen worker 7 - jepsen.util 14947 :invoke :write [2 22] INFO [2019-12-17 12:36:37,956] jepsen worker 23 - jepsen.util 14873 :invoke :write [7 23] INFO [2019-12-17 12:36:37,964] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63497,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:36:37,969] jepsen worker 87 - jepsen.util 10527 :invoke :read [29 nil] INFO [2019-12-17 12:36:38,009] jepsen worker 53 - jepsen.util 10313 :invoke :cas [17 [7 35]] INFO [2019-12-17 12:36:38,032] jepsen worker 83 - jepsen.util 15023 :info :write [27 31] :RequestCanceledException INFO [2019-12-17 12:36:38,037] jepsen worker 21 - jepsen.util 14421 :invoke :cas [7 [34 10]] INFO [2019-12-17 12:36:38,039] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestTimeoutException: CarrierBucketConfigRequest {"retried":0,"reason":"TIMEOUT","requestId":63332,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:36:38,040] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63517,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:36:38,055] jepsen worker 81 - jepsen.util 14571 :info :cas [27 [1 14]] :RequestCanceledException INFO [2019-12-17 12:36:38,056] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63517,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:36:38,059] jepsen worker 6 - jepsen.util 15396 :invoke :cas [2 [37 26]] INFO [2019-12-17 12:36:38,070] jepsen worker 26 - jepsen.util 15686 :invoke :read [8 nil] INFO [2019-12-17 12:36:38,073] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63519,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:36:38,081] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63519,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:36:38,095] jepsen worker 72 - jepsen.util 11232 :invoke :write [24 12] INFO [2019-12-17 12:36:38,117] jepsen worker 41 - jepsen.util 15971 :invoke :cas [13 [28 24]] INFO [2019-12-17 12:36:38,127] jepsen worker 25 - jepsen.util 16045 :invoke :cas [8 [29 33]] INFO [2019-12-17 12:36:38,185] jepsen worker 81 - jepsen.util 14661 :invoke :read [27 nil] INFO [2019-12-17 12:36:38,185] jepsen worker 61 - jepsen.util 15901 :invoke :cas [20 [40 49]] INFO [2019-12-17 12:36:38,248] jepsen worker 83 - jepsen.util 15113 :invoke :cas [27 [0 14]] INFO [2019-12-17 12:36:38,272] jepsen worker 31 - jepsen.util 9481 :ok :write [10 0] INFO [2019-12-17 12:36:38,274] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63529,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:36:38,274] jepsen worker 31 - jepsen.util 9481 :invoke :cas [10 [35 14]] INFO [2019-12-17 12:36:38,282] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63529,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:36:38,287] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63532,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:36:38,290] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63532,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:36:38,291] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63537,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:36:38,305] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63537,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:36:38,314] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63540,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:36:38,317] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63540,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:36:38,365] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:36:38,366] jepsen nemesis - couchbase.util Rebalance check count: 497 INFO [2019-12-17 12:36:38,369] jepsen nemesis - couchbase.util Rebalance stuck count: 478 INFO [2019-12-17 12:36:38,467] jepsen worker 20 - jepsen.util 10640 :fail :cas [6 [3 48]] :ValueNotSwapFrom INFO [2019-12-17 12:36:38,469] jepsen worker 20 - jepsen.util 10640 :invoke :write [6 24] INFO [2019-12-17 12:36:38,479] jepsen worker 37 - jepsen.util 10657 :ok :write [12 29] INFO [2019-12-17 12:36:38,481] jepsen worker 37 - jepsen.util 10657 :invoke :write [12 25] INFO [2019-12-17 12:36:38,504] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63545,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:36:38,518] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63545,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:36:38,539] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63549,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:36:38,541] jepsen worker 13 - jepsen.util 10453 :fail :read [4 nil] :RequestTimeoutException INFO [2019-12-17 12:36:38,547] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63549,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:36:38,547] jepsen worker 13 - jepsen.util 10453 :invoke :read [4 nil] INFO [2019-12-17 12:36:38,555] jepsen worker 77 - jepsen.util 15557 :info :write [25 42] :RequestCanceledException INFO [2019-12-17 12:36:38,571] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestTimeoutException: CarrierBucketConfigRequest {"retried":0,"reason":"TIMEOUT","requestId":63384,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:36:38,571] jepsen worker 54 - jepsen.util 11754 :fail :read [18 nil] :RequestTimeoutException INFO [2019-12-17 12:36:38,583] jepsen worker 54 - jepsen.util 11754 :invoke :cas [18 [20 24]] INFO [2019-12-17 12:36:38,601] jepsen worker 34 - jepsen.util 15334 :info :write [11 5] :RequestCanceledException INFO [2019-12-17 12:36:38,626] jepsen worker 11 - jepsen.util 10721 :fail :read [3 nil] :RequestTimeoutException INFO [2019-12-17 12:36:38,626] jepsen worker 11 - jepsen.util 10721 :invoke :cas [3 [4 29]] INFO [2019-12-17 12:36:38,631] jepsen worker 63 - jepsen.util 9873 :fail :cas [21 [46 27]] :ValueNotSwapFrom INFO [2019-12-17 12:36:38,632] jepsen worker 18 - jepsen.util 11088 :fail :cas [6 [45 35]] :ValueNotSwapFrom INFO [2019-12-17 12:36:38,633] jepsen worker 63 - jepsen.util 9873 :invoke :read [21 nil] INFO [2019-12-17 12:36:38,634] jepsen worker 18 - jepsen.util 11088 :invoke :write [6 24] INFO [2019-12-17 12:36:38,640] jepsen worker 33 - jepsen.util 14883 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:36:38,641] jepsen worker 33 - jepsen.util 14883 :invoke :write [11 21] INFO [2019-12-17 12:36:38,695] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63560,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:36:38,702] jepsen worker 55 - jepsen.util 11575 :fail :read [18 nil] :RequestTimeoutException INFO [2019-12-17 12:36:38,716] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63560,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:36:38,719] jepsen worker 55 - jepsen.util 11575 :invoke :write [18 42] INFO [2019-12-17 12:36:38,724] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63564,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:36:38,732] jepsen worker 46 - jepsen.util 10666 :fail :read [15 nil] :RequestTimeoutException INFO [2019-12-17 12:36:38,732] jepsen worker 34 - jepsen.util 15424 :invoke :write [11 30] INFO [2019-12-17 12:36:38,743] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63564,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:36:38,756] jepsen worker 46 - jepsen.util 10666 :invoke :write [15 32] INFO [2019-12-17 12:36:38,822] jepsen worker 77 - jepsen.util 15647 :invoke :read [25 nil] INFO [2019-12-17 12:36:38,843] jepsen worker 1 - jepsen.util 14941 :info :cas [0 [40 27]] :RequestCanceledException INFO [2019-12-17 12:36:38,848] jepsen worker 0 - jepsen.util 15840 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:36:38,860] jepsen worker 0 - jepsen.util 15840 :invoke :cas [0 [13 4]] INFO [2019-12-17 12:36:38,914] jepsen worker 88 - jepsen.util 11338 :fail :read [29 nil] :RequestTimeoutException INFO [2019-12-17 12:36:38,924] jepsen worker 88 - jepsen.util 11338 :invoke :read [29 nil] INFO [2019-12-17 12:36:38,929] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63573,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:36:38,944] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63573,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:36:39,050] jepsen worker 35 - jepsen.util 14885 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:36:39,060] jepsen worker 35 - jepsen.util 14885 :invoke :write [11 0] INFO [2019-12-17 12:36:39,068] jepsen worker 1 - jepsen.util 15031 :invoke :read [0 nil] INFO [2019-12-17 12:36:39,072] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63581,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:36:39,112] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63581,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:36:39,125] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63582,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:36:39,126] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63582,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:36:39,136] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63583,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:36:39,150] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63583,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:36:39,151] jepsen worker 59 - jepsen.util 14099 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:36:39,155] jepsen worker 59 - jepsen.util 14099 :invoke :write [19 47] INFO [2019-12-17 12:36:39,183] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestTimeoutException: CarrierBucketConfigRequest {"retried":0,"reason":"TIMEOUT","requestId":63406,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:36:39,201] jepsen worker 48 - jepsen.util 13998 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:36:39,202] jepsen worker 48 - jepsen.util 13998 :invoke :read [16 nil] INFO [2019-12-17 12:36:39,218] jepsen worker 45 - jepsen.util 11115 :fail :read [15 nil] :RequestTimeoutException INFO [2019-12-17 12:36:39,218] jepsen worker 45 - jepsen.util 11115 :invoke :cas [15 [10 36]] INFO [2019-12-17 12:36:39,235] jepsen worker 68 - jepsen.util 14198 :info :cas [22 [22 43]] :RequestCanceledException INFO [2019-12-17 12:36:39,249] jepsen worker 62 - jepsen.util 15452 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:36:39,251] jepsen worker 62 - jepsen.util 15452 :invoke :cas [20 [12 19]] INFO [2019-12-17 12:36:39,273] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63598,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:36:39,283] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63598,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:36:39,293] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63602,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:36:39,306] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63602,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:36:39,314] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63590,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:36:39,320] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63590,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:36:39,322] jepsen worker 65 - jepsen.util 9515 :fail :read [21 nil] :RequestTimeoutException INFO [2019-12-17 12:36:39,323] jepsen worker 65 - jepsen.util 9515 :invoke :write [21 37] INFO [2019-12-17 12:36:39,328] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63591,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:36:39,333] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63591,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:36:39,344] jepsen worker 68 - jepsen.util 14288 :invoke :write [22 41] INFO [2019-12-17 12:36:39,370] jepsen worker 43 - jepsen.util 15073 :info :cas [14 [8 41]] :RequestCanceledException INFO [2019-12-17 12:36:39,402] jepsen worker 69 - jepsen.util 11049 :info :write [23 45] :RequestCanceledException INFO [2019-12-17 12:36:39,444] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:36:39,444] jepsen worker 64 - jepsen.util 8434 :fail :cas [21 [49 5]] :ValueNotSwapFrom INFO [2019-12-17 12:36:39,444] jepsen nemesis - couchbase.util Rebalance check count: 498 INFO [2019-12-17 12:36:39,445] jepsen nemesis - couchbase.util Rebalance stuck count: 479 INFO [2019-12-17 12:36:39,446] jepsen worker 64 - jepsen.util 8434 :invoke :write [21 44] INFO [2019-12-17 12:36:39,448] jepsen worker 69 - jepsen.util 11139 :invoke :write [23 6] INFO [2019-12-17 12:36:39,460] jepsen worker 43 - jepsen.util 15163 :invoke :read [14 nil] INFO [2019-12-17 12:36:39,468] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63609,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:36:39,491] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63609,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:36:39,501] jepsen worker 5 - jepsen.util 10715 :fail :read [1 nil] :RequestTimeoutException INFO [2019-12-17 12:36:39,502] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63610,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:36:39,502] jepsen worker 5 - jepsen.util 10715 :invoke :cas [1 [16 45]] INFO [2019-12-17 12:36:39,509] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63610,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:36:39,528] jepsen worker 27 - jepsen.util 10287 :info :cas [9 [35 4]] :RequestCanceledException INFO [2019-12-17 12:36:39,530] jepsen worker 28 - jepsen.util 9118 :fail :cas [9 [18 0]] :ValueNotSwapFrom INFO [2019-12-17 12:36:39,533] jepsen worker 36 - jepsen.util 11916 :info :write [12 14] :RequestCanceledException INFO [2019-12-17 12:36:39,541] jepsen worker 28 - jepsen.util 9118 :invoke :write [9 45] INFO [2019-12-17 12:36:39,543] jepsen worker 58 - jepsen.util 15448 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:36:39,553] jepsen worker 44 - jepsen.util 15434 :info :cas [14 [12 7]] :RequestCanceledException INFO [2019-12-17 12:36:39,554] jepsen worker 80 - jepsen.util 10700 :ok :write [26 11] INFO [2019-12-17 12:36:39,554] jepsen worker 58 - jepsen.util 15448 :invoke :read [19 nil] INFO [2019-12-17 12:36:39,562] jepsen worker 80 - jepsen.util 10700 :invoke :read [26 nil] INFO [2019-12-17 12:36:39,562] jepsen worker 67 - jepsen.util 15007 :info :cas [22 [21 48]] :RequestCanceledException INFO [2019-12-17 12:36:39,593] jepsen worker 71 - jepsen.util 11951 :ok :read [23 28] INFO [2019-12-17 12:36:39,602] jepsen worker 71 - jepsen.util 11951 :invoke :cas [23 [14 40]] INFO [2019-12-17 12:36:39,648] jepsen worker 52 - jepsen.util 10672 :ok :read [17 5] INFO [2019-12-17 12:36:39,649] jepsen worker 52 - jepsen.util 10672 :invoke :read [17 nil] INFO [2019-12-17 12:36:39,692] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63626,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:36:39,718] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63626,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:36:39,735] jepsen worker 27 - jepsen.util 10377 :invoke :write [9 28] INFO [2019-12-17 12:36:39,736] jepsen worker 4 - jepsen.util 10264 :fail :cas [1 [49 28]] :ValueNotSwapFrom INFO [2019-12-17 12:36:39,737] jepsen worker 4 - jepsen.util 10264 :invoke :write [1 20] INFO [2019-12-17 12:36:39,756] jepsen worker 30 - jepsen.util 10470 :ok :read [10 31] INFO [2019-12-17 12:36:39,765] jepsen worker 30 - jepsen.util 10470 :invoke :read [10 nil] INFO [2019-12-17 12:36:39,780] jepsen worker 36 - jepsen.util 12006 :invoke :write [12 20] INFO [2019-12-17 12:36:39,821] jepsen worker 67 - jepsen.util 15097 :invoke :write [22 12] INFO [2019-12-17 12:36:39,824] jepsen worker 44 - jepsen.util 15524 :invoke :cas [14 [33 39]] INFO [2019-12-17 12:36:39,901] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63634,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:36:39,906] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63634,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:36:39,906] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63637,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:36:39,912] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63637,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:36:39,929] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63638,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:36:39,937] jepsen worker 89 - jepsen.util 11069 :info :write [29 35] :RequestCanceledException INFO [2019-12-17 12:36:39,939] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63638,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:36:39,942] jepsen worker 2 - jepsen.util 14402 :info :write [0 32] :RequestCanceledException INFO [2019-12-17 12:36:39,943] jepsen worker 16 - jepsen.util 16306 :info :write [5 36] :RequestCanceledException INFO [2019-12-17 12:36:39,944] jepsen worker 66 - jepsen.util 14826 :info :write [22 1] :RequestCanceledException INFO [2019-12-17 12:36:39,952] jepsen worker 17 - jepsen.util 14867 :info :write [5 45] :RequestCanceledException INFO [2019-12-17 12:36:39,952] jepsen worker 10 - jepsen.util 10720 :info :write [3 41] :RequestCanceledException INFO [2019-12-17 12:36:39,953] jepsen worker 49 - jepsen.util 14719 :info :write [16 3] :RequestCanceledException INFO [2019-12-17 12:36:39,955] jepsen worker 19 - jepsen.util 11359 :ok :read [6 11] INFO [2019-12-17 12:36:39,962] jepsen worker 19 - jepsen.util 11359 :invoke :read [6 nil] INFO [2019-12-17 12:36:39,988] jepsen worker 39 - jepsen.util 14619 :info :write [13 13] :RequestCanceledException INFO [2019-12-17 12:36:39,988] jepsen worker 40 - jepsen.util 15250 :info :write [13 42] :RequestCanceledException INFO [2019-12-17 12:36:39,988] jepsen worker 50 - jepsen.util 15530 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:36:39,998] jepsen worker 50 - jepsen.util 15530 :invoke :write [16 16] INFO [2019-12-17 12:36:40,014] jepsen worker 60 - jepsen.util 15900 :info :write [20 14] :RequestCanceledException INFO [2019-12-17 12:36:40,022] jepsen worker 3 - jepsen.util 10533 :info :write [1 27] :RequestCanceledException INFO [2019-12-17 12:36:40,026] jepsen worker 75 - jepsen.util 14565 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:36:40,027] jepsen worker 87 - jepsen.util 10527 :fail :read [29 nil] :RequestTimeoutException INFO [2019-12-17 12:36:40,037] jepsen worker 75 - jepsen.util 14565 :invoke :read [25 nil] INFO [2019-12-17 12:36:40,046] jepsen worker 87 - jepsen.util 10527 :invoke :cas [29 [10 18]] INFO [2019-12-17 12:36:40,062] jepsen worker 15 - jepsen.util 13875 :info :cas [5 [26 31]] :RequestCanceledException INFO [2019-12-17 12:36:40,070] jepsen worker 86 - jepsen.util 15746 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:36:40,073] jepsen worker 76 - jepsen.util 15376 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:36:40,075] jepsen worker 86 - jepsen.util 15746 :invoke :read [28 nil] INFO [2019-12-17 12:36:40,077] jepsen worker 72 - jepsen.util 11232 :ok :write [24 12] INFO [2019-12-17 12:36:40,079] jepsen worker 53 - jepsen.util 10313 :info :cas [17 [7 35]] :RequestCanceledException INFO [2019-12-17 12:36:40,080] jepsen worker 42 - jepsen.util 15612 :info :cas [14 [26 33]] :RequestCanceledException INFO [2019-12-17 12:36:40,088] jepsen worker 10 - jepsen.util 10810 :invoke :read [3 nil] INFO [2019-12-17 12:36:40,088] jepsen worker 76 - jepsen.util 15376 :invoke :cas [25 [17 27]] INFO [2019-12-17 12:36:40,091] jepsen worker 56 - jepsen.util 11756 :info :write [18 6] :RequestCanceledException INFO [2019-12-17 12:36:40,097] jepsen worker 72 - jepsen.util 11232 :invoke :write [24 44] INFO [2019-12-17 12:36:40,111] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63644,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:36:40,140] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63644,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:36:40,156] jepsen worker 26 - jepsen.util 15686 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:36:40,162] jepsen worker 26 - jepsen.util 15686 :invoke :write [8 13] INFO [2019-12-17 12:36:40,166] jepsen worker 2 - jepsen.util 14492 :invoke :write [0 18] INFO [2019-12-17 12:36:40,173] jepsen worker 6 - jepsen.util 15396 :info :cas [2 [37 26]] :RequestCanceledException INFO [2019-12-17 12:36:40,175] jepsen worker 21 - jepsen.util 14421 :info :cas [7 [34 10]] :RequestCanceledException INFO [2019-12-17 12:36:40,194] jepsen worker 49 - jepsen.util 14809 :invoke :cas [16 [8 32]] INFO [2019-12-17 12:36:40,197] jepsen worker 89 - jepsen.util 11159 :invoke :read [29 nil] INFO [2019-12-17 12:36:40,210] jepsen worker 17 - jepsen.util 14957 :invoke :write [5 20] INFO [2019-12-17 12:36:40,212] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63651,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:36:40,219] jepsen worker 16 - jepsen.util 16396 :invoke :write [5 22] INFO [2019-12-17 12:36:40,222] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63651,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:36:40,226] jepsen worker 29 - jepsen.util 9299 :ok :read [9 5] INFO [2019-12-17 12:36:40,228] jepsen worker 41 - jepsen.util 15971 :info :cas [13 [28 24]] :RequestCanceledException INFO [2019-12-17 12:36:40,235] jepsen worker 53 - jepsen.util 10403 :invoke :write [17 19] INFO [2019-12-17 12:36:40,235] jepsen worker 38 - jepsen.util 11468 :info :write [12 31] :RequestCanceledException INFO [2019-12-17 12:36:40,235] jepsen worker 29 - jepsen.util 9299 :invoke :read [9 nil] INFO [2019-12-17 12:36:40,237] jepsen worker 78 - jepsen.util 11148 :info :write [26 13] :RequestCanceledException INFO [2019-12-17 12:36:40,237] jepsen worker 66 - jepsen.util 14916 :invoke :read [22 nil] INFO [2019-12-17 12:36:40,243] jepsen worker 25 - jepsen.util 16045 :info :cas [8 [29 33]] :RequestCanceledException INFO [2019-12-17 12:36:40,244] jepsen worker 40 - jepsen.util 15340 :invoke :read [13 nil] INFO [2019-12-17 12:36:40,250] jepsen worker 39 - jepsen.util 14709 :invoke :write [13 27] INFO [2019-12-17 12:36:40,250] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63657,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:36:40,258] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63657,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:36:40,260] jepsen worker 81 - jepsen.util 14661 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:36:40,281] jepsen worker 61 - jepsen.util 15901 :info :cas [20 [40 49]] :RequestCanceledException INFO [2019-12-17 12:36:40,297] jepsen worker 81 - jepsen.util 14661 :invoke :write [27 5] INFO [2019-12-17 12:36:40,304] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63660,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:36:40,338] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63660,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:36:40,345] jepsen worker 83 - jepsen.util 15113 :info :cas [27 [0 14]] :RequestCanceledException INFO [2019-12-17 12:36:40,346] jepsen worker 56 - jepsen.util 11846 :invoke :read [18 nil] INFO [2019-12-17 12:36:40,353] jepsen worker 3 - jepsen.util 10623 :invoke :read [1 nil] INFO [2019-12-17 12:36:40,357] jepsen worker 60 - jepsen.util 15990 :invoke :read [20 nil] INFO [2019-12-17 12:36:40,365] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63663,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:36:40,366] jepsen worker 15 - jepsen.util 13965 :invoke :write [5 5] INFO [2019-12-17 12:36:40,371] jepsen worker 21 - jepsen.util 14511 :invoke :write [7 8] INFO [2019-12-17 12:36:40,383] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63663,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:36:40,389] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63666,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:36:40,407] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63666,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:36:40,412] jepsen worker 42 - jepsen.util 15702 :invoke :read [14 nil] INFO [2019-12-17 12:36:40,412] jepsen worker 6 - jepsen.util 15486 :invoke :read [2 nil] INFO [2019-12-17 12:36:40,440] jepsen worker 78 - jepsen.util 11238 :invoke :write [26 43] INFO [2019-12-17 12:36:40,450] jepsen worker 24 - jepsen.util 13524 :info :write [8 4] :RequestCanceledException INFO [2019-12-17 12:36:40,450] jepsen worker 38 - jepsen.util 11558 :invoke :read [12 nil] INFO [2019-12-17 12:36:40,551] jepsen worker 8 - jepsen.util 14588 :info :write [2 10] :RequestCanceledException INFO [2019-12-17 12:36:40,558] jepsen worker 73 - jepsen.util 10243 :ok :read [24 40] INFO [2019-12-17 12:36:40,569] jepsen worker 57 - jepsen.util 14907 :info :write [19 29] :RequestCanceledException INFO [2019-12-17 12:36:40,570] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63678,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:36:40,593] jepsen worker 79 - jepsen.util 11239 :info :write [26 17] :RequestCanceledException INFO [2019-12-17 12:36:40,608] jepsen worker 73 - jepsen.util 10243 :invoke :cas [24 [18 25]] INFO [2019-12-17 12:36:40,605] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63678,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:36:40,628] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63682,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:36:40,629] jepsen worker 37 - jepsen.util 10657 :ok :write [12 25] INFO [2019-12-17 12:36:40,641] jepsen worker 37 - jepsen.util 10657 :invoke :write [12 11] INFO [2019-12-17 12:36:40,648] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63682,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:36:40,659] jepsen worker 82 - jepsen.util 15112 :info :write [27 15] :RequestCanceledException INFO [2019-12-17 12:36:40,669] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:36:40,695] jepsen worker 54 - jepsen.util 11754 :info :cas [18 [20 24]] :RequestCanceledException INFO [2019-12-17 12:36:40,695] jepsen nemesis - couchbase.util Rebalance check count: 499 INFO [2019-12-17 12:36:40,696] jepsen worker 70 - jepsen.util 11590 :info :write [23 5] :RequestCanceledException INFO [2019-12-17 12:36:40,702] jepsen nemesis - couchbase.util Rebalance stuck count: 480 INFO [2019-12-17 12:36:40,703] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63685,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:36:40,703] jepsen worker 25 - jepsen.util 16135 :invoke :cas [8 [2 15]] INFO [2019-12-17 12:36:40,725] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63685,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:36:40,764] jepsen worker 24 - jepsen.util 13614 :invoke :write [8 44] INFO [2019-12-17 12:36:40,784] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63689,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:36:40,808] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63689,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:36:40,841] jepsen worker 84 - jepsen.util 14574 :info :write [28 2] :RequestCanceledException INFO [2019-12-17 12:36:40,858] jepsen worker 85 - jepsen.util 15745 :info :write [28 17] :RequestCanceledException INFO [2019-12-17 12:36:40,897] jepsen worker 41 - jepsen.util 16061 :invoke :read [13 nil] INFO [2019-12-17 12:36:40,902] jepsen worker 12 - jepsen.util 11352 :info :write [4 23] :RequestCanceledException INFO [2019-12-17 12:36:40,913] jepsen worker 54 - jepsen.util 11844 :invoke :read [18 nil] INFO [2019-12-17 12:36:40,915] jepsen worker 83 - jepsen.util 15203 :invoke :read [27 nil] INFO [2019-12-17 12:36:40,946] jepsen worker 61 - jepsen.util 15991 :invoke :cas [20 [37 43]] INFO [2019-12-17 12:36:40,951] jepsen worker 0 - jepsen.util 15840 :info :cas [0 [13 4]] :RequestCanceledException INFO [2019-12-17 12:36:40,951] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63692,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:36:40,953] jepsen worker 77 - jepsen.util 15647 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:36:40,957] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63692,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:36:40,960] jepsen worker 77 - jepsen.util 15647 :invoke :write [25 3] INFO [2019-12-17 12:36:40,960] jepsen worker 22 - jepsen.util 14872 :info :write [7 19] :RequestCanceledException INFO [2019-12-17 12:36:40,961] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63696,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:36:40,980] jepsen worker 57 - jepsen.util 14997 :invoke :write [19 29] INFO [2019-12-17 12:36:40,997] jepsen worker 70 - jepsen.util 11680 :invoke :write [23 26] INFO [2019-12-17 12:36:40,998] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63696,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:36:41,005] jepsen worker 51 - jepsen.util 10401 :ok :read [17 5] INFO [2019-12-17 12:36:41,007] jepsen worker 88 - jepsen.util 11338 :fail :read [29 nil] :RequestTimeoutException INFO [2019-12-17 12:36:41,007] jepsen worker 51 - jepsen.util 10401 :invoke :write [17 29] INFO [2019-12-17 12:36:41,014] jepsen worker 8 - jepsen.util 14678 :invoke :cas [2 [7 16]] INFO [2019-12-17 12:36:41,015] jepsen worker 88 - jepsen.util 11338 :invoke :write [29 5] INFO [2019-12-17 12:36:41,016] jepsen worker 31 - jepsen.util 9481 :fail :cas [10 [35 14]] :ValueNotSwapFrom INFO [2019-12-17 12:36:41,024] jepsen worker 31 - jepsen.util 9481 :invoke :write [10 4] INFO [2019-12-17 12:36:41,031] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63704,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:36:41,032] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63704,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:36:41,033] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63708,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:36:41,042] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63708,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:36:41,053] jepsen worker 0 - jepsen.util 15930 :invoke :read [0 nil] INFO [2019-12-17 12:36:41,074] jepsen worker 47 - jepsen.util 11657 :fail :cas [15 [36 34]] :ValueNotSwapFrom INFO [2019-12-17 12:36:41,081] jepsen worker 47 - jepsen.util 11657 :invoke :read [15 nil] INFO [2019-12-17 12:36:41,091] jepsen worker 79 - jepsen.util 11329 :invoke :read [26 nil] INFO [2019-12-17 12:36:41,091] jepsen worker 82 - jepsen.util 15202 :invoke :read [27 nil] INFO [2019-12-17 12:36:41,101] jepsen worker 22 - jepsen.util 14962 :invoke :write [7 40] INFO [2019-12-17 12:36:41,110] jepsen worker 74 - jepsen.util 10784 :fail :cas [24 [47 21]] :ValueNotSwapFrom INFO [2019-12-17 12:36:41,121] jepsen worker 74 - jepsen.util 10784 :invoke :cas [24 [1 7]] INFO [2019-12-17 12:36:41,144] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestTimeoutException: CarrierBucketConfigRequest {"retried":0,"reason":"TIMEOUT","requestId":63548,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:36:41,144] jepsen worker 23 - jepsen.util 14873 :info :write [7 23] :RequestCanceledException INFO [2019-12-17 12:36:41,146] jepsen worker 1 - jepsen.util 15031 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:36:41,155] jepsen worker 85 - jepsen.util 15835 :invoke :cas [28 [44 16]] INFO [2019-12-17 12:36:41,162] jepsen worker 1 - jepsen.util 15031 :invoke :write [0 39] INFO [2019-12-17 12:36:41,172] jepsen worker 12 - jepsen.util 11442 :invoke :cas [4 [25 40]] INFO [2019-12-17 12:36:41,172] jepsen worker 84 - jepsen.util 14664 :invoke :cas [28 [42 43]] INFO [2019-12-17 12:36:41,190] jepsen worker 14 - jepsen.util 10814 :ok :read [4 6] INFO [2019-12-17 12:36:41,195] jepsen worker 14 - jepsen.util 10814 :invoke :cas [4 [12 32]] INFO [2019-12-17 12:36:41,231] jepsen worker 7 - jepsen.util 14947 :info :write [2 22] :RequestCanceledException INFO [2019-12-17 12:36:41,239] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63730,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:36:41,239] jepsen worker 46 - jepsen.util 10666 :info :write [15 32] :RequestCanceledException INFO [2019-12-17 12:36:41,250] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63730,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:36:41,259] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63733,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:36:41,263] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63733,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:36:41,286] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63723,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:36:41,291] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63723,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:36:41,310] jepsen worker 48 - jepsen.util 13998 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:36:41,312] jepsen worker 48 - jepsen.util 13998 :invoke :read [16 nil] INFO [2019-12-17 12:36:41,329] jepsen worker 45 - jepsen.util 11115 :info :cas [15 [10 36]] :RequestCanceledException INFO [2019-12-17 12:36:41,363] jepsen worker 62 - jepsen.util 15452 :info :cas [20 [12 19]] :RequestCanceledException INFO [2019-12-17 12:36:41,374] jepsen worker 20 - jepsen.util 10640 :ok :write [6 24] INFO [2019-12-17 12:36:41,374] jepsen worker 46 - jepsen.util 10756 :invoke :cas [15 [23 46]] INFO [2019-12-17 12:36:41,385] jepsen worker 20 - jepsen.util 10640 :invoke :write [6 31] INFO [2019-12-17 12:36:41,385] jepsen worker 23 - jepsen.util 14963 :invoke :read [7 nil] INFO [2019-12-17 12:36:41,419] jepsen worker 7 - jepsen.util 15037 :invoke :read [2 nil] INFO [2019-12-17 12:36:41,477] jepsen worker 45 - jepsen.util 11205 :invoke :read [15 nil] INFO [2019-12-17 12:36:41,502] jepsen worker 55 - jepsen.util 11575 :info :write [18 42] :RequestCanceledException INFO [2019-12-17 12:36:41,507] jepsen worker 62 - jepsen.util 15542 :invoke :cas [20 [23 47]] INFO [2019-12-17 12:36:41,507] jepsen worker 34 - jepsen.util 15424 :info :write [11 30] :RequestCanceledException INFO [2019-12-17 12:36:41,537] jepsen worker 18 - jepsen.util 11088 :info :write [6 24] :RequestCanceledException INFO [2019-12-17 12:36:41,537] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63742,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:36:41,539] jepsen worker 43 - jepsen.util 15163 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:36:41,546] jepsen worker 43 - jepsen.util 15163 :invoke :read [14 nil] INFO [2019-12-17 12:36:41,547] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63742,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:36:41,630] jepsen worker 5 - jepsen.util 10715 :info :cas [1 [16 45]] :RequestCanceledException INFO [2019-12-17 12:36:41,632] jepsen worker 18 - jepsen.util 11178 :invoke :write [6 13] INFO [2019-12-17 12:36:41,648] jepsen worker 35 - jepsen.util 14885 :info :write [11 0] :RequestCanceledException INFO [2019-12-17 12:36:41,649] jepsen worker 80 - jepsen.util 10700 :fail :read [26 nil] :RequestTimeoutException INFO [2019-12-17 12:36:41,655] jepsen worker 80 - jepsen.util 10700 :invoke :write [26 24] INFO [2019-12-17 12:36:41,661] jepsen worker 58 - jepsen.util 15448 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:36:41,663] jepsen worker 59 - jepsen.util 14099 :info :write [19 47] :RequestCanceledException INFO [2019-12-17 12:36:41,665] jepsen worker 71 - jepsen.util 11951 :info :cas [23 [14 40]] :RequestCanceledException INFO [2019-12-17 12:36:41,667] jepsen worker 58 - jepsen.util 15448 :invoke :cas [19 [7 43]] INFO [2019-12-17 12:36:41,668] jepsen worker 33 - jepsen.util 14883 :info :write [11 21] :RequestCanceledException INFO [2019-12-17 12:36:41,688] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63761,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:36:41,700] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63761,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:36:41,764] jepsen worker 34 - jepsen.util 15514 :invoke :write [11 13] INFO [2019-12-17 12:36:41,777] jepsen worker 55 - jepsen.util 11665 :invoke :write [18 33] INFO [2019-12-17 12:36:41,786] jepsen worker 5 - jepsen.util 10805 :invoke :cas [1 [16 5]] INFO [2019-12-17 12:36:41,860] jepsen worker 44 - jepsen.util 15524 :info :cas [14 [33 39]] :RequestCanceledException INFO [2019-12-17 12:36:41,881] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63769,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:36:41,891] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:36:41,897] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63769,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:36:41,903] jepsen nemesis - couchbase.util Rebalance check count: 500 INFO [2019-12-17 12:36:41,913] jepsen nemesis - couchbase.util Rebalance stuck count: 481 INFO [2019-12-17 12:36:41,920] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63774,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:36:41,923] jepsen worker 65 - jepsen.util 9515 :info :write [21 37] :RequestCanceledException INFO [2019-12-17 12:36:41,945] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63774,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:36:41,991] jepsen worker 63 - jepsen.util 9873 :ok :read [21 21] INFO [2019-12-17 12:36:41,993] jepsen worker 65 - jepsen.util 9605 :invoke :read [21 nil] INFO [2019-12-17 12:36:41,995] jepsen worker 63 - jepsen.util 9873 :invoke :cas [21 [30 39]] INFO [2019-12-17 12:36:42,010] jepsen worker 59 - jepsen.util 14189 :invoke :cas [19 [48 31]] INFO [2019-12-17 12:36:42,012] jepsen worker 33 - jepsen.util 14973 :invoke :read [11 nil] INFO [2019-12-17 12:36:42,014] jepsen worker 35 - jepsen.util 14975 :invoke :write [11 26] INFO [2019-12-17 12:36:42,025] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63780,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:36:42,032] jepsen worker 36 - jepsen.util 12006 :ok :write [12 20] INFO [2019-12-17 12:36:42,035] jepsen worker 36 - jepsen.util 12006 :invoke :cas [12 [6 25]] INFO [2019-12-17 12:36:42,036] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63780,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:36:42,037] jepsen worker 71 - jepsen.util 12041 :invoke :write [23 14] INFO [2019-12-17 12:36:42,053] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63784,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:36:42,060] jepsen worker 32 - jepsen.util 11192 :ok :read [10 31] INFO [2019-12-17 12:36:42,063] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63784,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:36:42,066] jepsen worker 32 - jepsen.util 11192 :invoke :cas [10 [34 12]] INFO [2019-12-17 12:36:42,074] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63787,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:36:42,085] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63787,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:36:42,106] jepsen worker 87 - jepsen.util 10527 :info :cas [29 [10 18]] :RequestCanceledException INFO [2019-12-17 12:36:42,126] jepsen worker 75 - jepsen.util 14565 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:36:42,128] jepsen worker 75 - jepsen.util 14565 :invoke :read [25 nil] INFO [2019-12-17 12:36:42,138] jepsen worker 44 - jepsen.util 15614 :invoke :read [14 nil] INFO [2019-12-17 12:36:42,153] jepsen worker 86 - jepsen.util 15746 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:36:42,156] jepsen worker 86 - jepsen.util 15746 :invoke :write [28 31] INFO [2019-12-17 12:36:42,158] jepsen worker 76 - jepsen.util 15376 :info :cas [25 [17 27]] :RequestCanceledException INFO [2019-12-17 12:36:42,175] jepsen worker 10 - jepsen.util 10810 :fail :read [3 nil] :RequestTimeoutException INFO [2019-12-17 12:36:42,177] jepsen worker 10 - jepsen.util 10810 :invoke :read [3 nil] INFO [2019-12-17 12:36:42,226] jepsen worker 87 - jepsen.util 10617 :invoke :write [29 18] INFO [2019-12-17 12:36:42,268] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63803,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:36:42,275] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63796,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:36:42,276] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63803,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:36:42,278] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63796,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:36:42,288] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63807,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:36:42,289] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63797,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:36:42,293] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63807,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:36:42,294] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63797,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:36:42,303] jepsen worker 76 - jepsen.util 15466 :invoke :cas [25 [34 9]] INFO [2019-12-17 12:36:42,303] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63798,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:36:42,308] jepsen worker 49 - jepsen.util 14809 :info :cas [16 [8 32]] :RequestCanceledException INFO [2019-12-17 12:36:42,313] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63798,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:36:42,314] jepsen worker 40 - jepsen.util 15340 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:36:42,317] jepsen worker 40 - jepsen.util 15340 :invoke :write [13 20] INFO [2019-12-17 12:36:42,334] jepsen worker 66 - jepsen.util 14916 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:36:42,335] jepsen worker 66 - jepsen.util 14916 :invoke :cas [22 [26 33]] INFO [2019-12-17 12:36:42,366] jepsen worker 69 - jepsen.util 11139 :info :write [23 6] :RequestCanceledException INFO [2019-12-17 12:36:42,402] jepsen worker 49 - jepsen.util 14899 :invoke :cas [16 [8 27]] INFO [2019-12-17 12:36:42,403] jepsen worker 9 - jepsen.util 11709 :ok :read [3 4] INFO [2019-12-17 12:36:42,404] jepsen worker 9 - jepsen.util 11709 :invoke :cas [3 [42 37]] INFO [2019-12-17 12:36:42,433] jepsen worker 69 - jepsen.util 11229 :invoke :read [23 nil] INFO [2019-12-17 12:36:42,447] jepsen worker 68 - jepsen.util 14288 :info :write [22 41] :RequestCanceledException INFO [2019-12-17 12:36:42,457] jepsen worker 60 - jepsen.util 15990 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:36:42,457] jepsen worker 60 - jepsen.util 15990 :invoke :write [20 39] INFO [2019-12-17 12:36:42,459] jepsen worker 38 - jepsen.util 11558 :fail :read [12 nil] :RequestTimeoutException INFO [2019-12-17 12:36:42,459] jepsen worker 64 - jepsen.util 8434 :info :write [21 44] :RequestCanceledException INFO [2019-12-17 12:36:42,460] jepsen worker 38 - jepsen.util 11558 :invoke :cas [12 [45 1]] INFO [2019-12-17 12:36:42,469] jepsen worker 68 - jepsen.util 14378 :invoke :cas [22 [43 26]] INFO [2019-12-17 12:36:42,475] jepsen worker 64 - jepsen.util 8524 :invoke :read [21 nil] INFO [2019-12-17 12:36:42,486] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63819,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:36:42,495] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63819,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:36:42,497] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63822,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:36:42,500] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63822,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:36:42,519] jepsen worker 6 - jepsen.util 15486 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:36:42,520] jepsen worker 6 - jepsen.util 15486 :invoke :read [2 nil] INFO [2019-12-17 12:36:42,548] jepsen worker 42 - jepsen.util 15702 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:36:42,549] jepsen worker 42 - jepsen.util 15702 :invoke :write [14 26] INFO [2019-12-17 12:36:42,588] jepsen worker 4 - jepsen.util 10264 :ok :write [1 20] INFO [2019-12-17 12:36:42,588] jepsen worker 4 - jepsen.util 10264 :invoke :write [1 20] INFO [2019-12-17 12:36:42,619] jepsen worker 29 - jepsen.util 9299 :ok :read [9 14] INFO [2019-12-17 12:36:42,619] jepsen worker 29 - jepsen.util 9299 :invoke :cas [9 [27 4]] INFO [2019-12-17 12:36:42,750] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63840,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:36:42,750] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63840,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:36:42,750] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63842,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:36:42,751] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63842,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:36:42,752] jepsen worker 13 - jepsen.util 10453 :ok :read [4 6] INFO [2019-12-17 12:36:42,753] jepsen worker 13 - jepsen.util 10453 :invoke :cas [4 [19 36]] INFO [2019-12-17 12:36:42,787] jepsen worker 19 - jepsen.util 11359 :ok :read [6 11] INFO [2019-12-17 12:36:42,788] jepsen worker 19 - jepsen.util 11359 :invoke :cas [6 [11 8]] INFO [2019-12-17 12:36:42,791] jepsen worker 52 - jepsen.util 10672 :ok :read [17 45] INFO [2019-12-17 12:36:42,792] jepsen worker 52 - jepsen.util 10672 :invoke :read [17 nil] INFO [2019-12-17 12:36:42,792] jepsen worker 30 - jepsen.util 10470 :ok :read [10 0] INFO [2019-12-17 12:36:42,793] jepsen worker 30 - jepsen.util 10470 :invoke :read [10 nil] INFO [2019-12-17 12:36:42,798] jepsen worker 74 - jepsen.util 10784 :fail :cas [24 [1 7]] :ValueNotSwapFrom INFO [2019-12-17 12:36:42,798] jepsen worker 74 - jepsen.util 10784 :invoke :cas [24 [41 39]] INFO [2019-12-17 12:36:42,813] jepsen worker 56 - jepsen.util 11846 :ok :read [18 8] INFO [2019-12-17 12:36:42,813] jepsen worker 56 - jepsen.util 11846 :invoke :write [18 46] INFO [2019-12-17 12:36:42,831] jepsen worker 89 - jepsen.util 11159 :ok :read [29 17] INFO [2019-12-17 12:36:42,831] jepsen worker 89 - jepsen.util 11159 :invoke :read [29 nil] INFO [2019-12-17 12:36:42,834] jepsen worker 25 - jepsen.util 16135 :info :cas [8 [2 15]] :RequestCanceledException INFO [2019-12-17 12:36:42,845] jepsen worker 46 - jepsen.util 10756 :fail :cas [15 [23 46]] :ValueNotSwapFrom INFO [2019-12-17 12:36:42,847] jepsen worker 46 - jepsen.util 10756 :invoke :cas [15 [24 36]] INFO [2019-12-17 12:36:42,861] jepsen worker 25 - jepsen.util 16225 :invoke :read [8 nil] INFO [2019-12-17 12:36:42,883] jepsen worker 12 - jepsen.util 11442 :fail :cas [4 [25 40]] :ValueNotSwapFrom INFO [2019-12-17 12:36:42,884] jepsen worker 12 - jepsen.util 11442 :invoke :cas [4 [37 43]] INFO [2019-12-17 12:36:42,893] jepsen worker 3 - jepsen.util 10623 :ok :read [1 4] INFO [2019-12-17 12:36:42,894] jepsen worker 3 - jepsen.util 10623 :invoke :read [1 nil] INFO [2019-12-17 12:36:42,901] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63856,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:36:42,902] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63856,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:36:42,902] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63859,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:36:42,903] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63859,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:36:42,947] jepsen worker 83 - jepsen.util 15203 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:36:42,948] jepsen worker 83 - jepsen.util 15203 :invoke :write [27 46] INFO [2019-12-17 12:36:42,965] jepsen worker 61 - jepsen.util 15991 :info :cas [20 [37 43]] :RequestCanceledException INFO [2019-12-17 12:36:42,973] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:36:42,973] jepsen nemesis - couchbase.util Rebalance check count: 501 INFO [2019-12-17 12:36:42,974] jepsen nemesis - couchbase.util Rebalance stuck count: 482 INFO [2019-12-17 12:36:43,012] jepsen worker 61 - jepsen.util 16081 :invoke :write [20 10] INFO [2019-12-17 12:36:43,017] jepsen worker 41 - jepsen.util 16061 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:36:43,020] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63864,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:36:43,020] jepsen worker 41 - jepsen.util 16061 :invoke :write [13 26] INFO [2019-12-17 12:36:43,020] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63864,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:36:43,021] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63867,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:36:43,021] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63867,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:36:43,022] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63868,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:36:43,024] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63868,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:36:43,049] jepsen worker 47 - jepsen.util 11657 :ok :read [15 47] INFO [2019-12-17 12:36:43,050] jepsen worker 47 - jepsen.util 11657 :invoke :write [15 8] INFO [2019-12-17 12:36:43,077] jepsen worker 28 - jepsen.util 9118 :ok :write [9 45] INFO [2019-12-17 12:36:43,078] jepsen worker 28 - jepsen.util 9118 :invoke :cas [9 [15 38]] INFO [2019-12-17 12:36:43,100] jepsen worker 27 - jepsen.util 10377 :info :write [9 28] :RequestCanceledException INFO [2019-12-17 12:36:43,126] jepsen worker 8 - jepsen.util 14678 :info :cas [2 [7 16]] :RequestCanceledException INFO [2019-12-17 12:36:43,134] jepsen worker 0 - jepsen.util 15930 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:36:43,135] jepsen worker 67 - jepsen.util 15097 :info :write [22 12] :RequestCanceledException INFO [2019-12-17 12:36:43,140] jepsen worker 27 - jepsen.util 10467 :invoke :read [9 nil] INFO [2019-12-17 12:36:43,163] jepsen worker 82 - jepsen.util 15202 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:36:43,165] jepsen worker 37 - jepsen.util 10657 :ok :write [12 11] INFO [2019-12-17 12:36:43,199] jepsen worker 85 - jepsen.util 15835 :info :cas [28 [44 16]] :RequestCanceledException INFO [2019-12-17 12:36:43,208] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63871,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:36:43,211] jepsen worker 0 - jepsen.util 15930 :invoke :read [0 nil] INFO [2019-12-17 12:36:43,213] jepsen worker 82 - jepsen.util 15202 :invoke :read [27 nil] INFO [2019-12-17 12:36:43,213] jepsen worker 37 - jepsen.util 10657 :invoke :cas [12 [21 13]] INFO [2019-12-17 12:36:43,214] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63871,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:36:43,238] jepsen worker 84 - jepsen.util 14664 :info :cas [28 [42 43]] :RequestCanceledException INFO [2019-12-17 12:36:43,266] jepsen worker 20 - jepsen.util 10640 :ok :write [6 31] INFO [2019-12-17 12:36:43,267] jepsen worker 20 - jepsen.util 10640 :invoke :cas [6 [7 8]] INFO [2019-12-17 12:36:43,276] jepsen worker 80 - jepsen.util 10700 :ok :write [26 24] INFO [2019-12-17 12:36:43,276] jepsen worker 80 - jepsen.util 10700 :invoke :cas [26 [14 11]] INFO [2019-12-17 12:36:43,291] jepsen worker 8 - jepsen.util 14768 :invoke :write [2 25] INFO [2019-12-17 12:36:43,293] jepsen worker 67 - jepsen.util 15187 :invoke :read [22 nil] INFO [2019-12-17 12:36:43,293] jepsen worker 84 - jepsen.util 14754 :invoke :cas [28 [31 26]] INFO [2019-12-17 12:36:43,302] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63884,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:36:43,303] jepsen worker 85 - jepsen.util 15925 :invoke :cas [28 [23 24]] INFO [2019-12-17 12:36:43,307] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63884,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:36:43,400] jepsen worker 48 - jepsen.util 13998 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:36:43,400] jepsen worker 72 - jepsen.util 11232 :info :write [24 44] :RequestCanceledException INFO [2019-12-17 12:36:43,400] jepsen worker 48 - jepsen.util 13998 :invoke :write [16 38] INFO [2019-12-17 12:36:43,417] jepsen worker 72 - jepsen.util 11322 :invoke :read [24 nil] INFO [2019-12-17 12:36:43,446] jepsen worker 36 - jepsen.util 12006 :fail :cas [12 [6 25]] :ValueNotSwapFrom INFO [2019-12-17 12:36:43,448] jepsen worker 36 - jepsen.util 12006 :invoke :write [12 34] INFO [2019-12-17 12:36:43,468] jepsen worker 23 - jepsen.util 14963 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:36:43,469] jepsen worker 50 - jepsen.util 15530 :info :write [16 16] :RequestCanceledException INFO [2019-12-17 12:36:43,469] jepsen worker 23 - jepsen.util 14963 :invoke :cas [7 [43 18]] INFO [2019-12-17 12:36:43,476] jepsen worker 79 - jepsen.util 11329 :ok :read [26 13] INFO [2019-12-17 12:36:43,478] jepsen worker 79 - jepsen.util 11329 :invoke :write [26 28] INFO [2019-12-17 12:36:43,484] jepsen worker 73 - jepsen.util 10243 :fail :cas [24 [18 25]] :ValueNotSwapFrom INFO [2019-12-17 12:36:43,484] jepsen worker 5 - jepsen.util 10805 :fail :cas [1 [16 5]] :ValueNotSwapFrom INFO [2019-12-17 12:36:43,485] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63897,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:36:43,485] jepsen worker 73 - jepsen.util 10243 :invoke :cas [24 [12 38]] INFO [2019-12-17 12:36:43,486] jepsen worker 5 - jepsen.util 10805 :invoke :write [1 40] INFO [2019-12-17 12:36:43,488] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63897,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:36:43,496] jepsen worker 45 - jepsen.util 11205 :fail :read [15 nil] :RequestTimeoutException INFO [2019-12-17 12:36:43,497] jepsen worker 45 - jepsen.util 11205 :invoke :cas [15 [35 21]] INFO [2019-12-17 12:36:43,501] jepsen worker 63 - jepsen.util 9873 :fail :cas [21 [30 39]] :ValueNotSwapFrom INFO [2019-12-17 12:36:43,502] jepsen worker 63 - jepsen.util 9873 :invoke :cas [21 [18 1]] INFO [2019-12-17 12:36:43,511] jepsen worker 10 - jepsen.util 10810 :ok :read [3 4] INFO [2019-12-17 12:36:43,512] jepsen worker 50 - jepsen.util 15620 :invoke :read [16 nil] INFO [2019-12-17 12:36:43,512] jepsen worker 10 - jepsen.util 10810 :invoke :read [3 nil] INFO [2019-12-17 12:36:43,512] jepsen worker 54 - jepsen.util 11844 :ok :read [18 8] INFO [2019-12-17 12:36:43,513] jepsen worker 54 - jepsen.util 11844 :invoke :cas [18 [5 19]] INFO [2019-12-17 12:36:43,513] jepsen worker 78 - jepsen.util 11238 :ok :write [26 43] INFO [2019-12-17 12:36:43,513] jepsen worker 78 - jepsen.util 11238 :invoke :cas [26 [5 35]] INFO [2019-12-17 12:36:43,533] jepsen worker 7 - jepsen.util 15037 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:36:43,533] jepsen worker 2 - jepsen.util 14492 :info :write [0 18] :RequestCanceledException INFO [2019-12-17 12:36:43,533] jepsen worker 7 - jepsen.util 15037 :invoke :read [2 nil] INFO [2019-12-17 12:36:43,542] jepsen worker 29 - jepsen.util 9299 :fail :cas [9 [27 4]] :ValueNotSwapFrom INFO [2019-12-17 12:36:43,543] jepsen worker 29 - jepsen.util 9299 :invoke :cas [9 [4 24]] INFO [2019-12-17 12:36:43,546] jepsen worker 16 - jepsen.util 16396 :info :write [5 22] :RequestCanceledException INFO [2019-12-17 12:36:43,547] jepsen worker 81 - jepsen.util 14661 :info :write [27 5] :RequestCanceledException INFO [2019-12-17 12:36:43,551] jepsen worker 2 - jepsen.util 14582 :invoke :cas [0 [24 45]] INFO [2019-12-17 12:36:43,563] jepsen worker 46 - jepsen.util 10756 :fail :cas [15 [24 36]] :ValueNotSwapFrom INFO [2019-12-17 12:36:43,564] jepsen worker 53 - jepsen.util 10403 :info :write [17 19] :RequestCanceledException INFO [2019-12-17 12:36:43,564] jepsen worker 46 - jepsen.util 10756 :invoke :read [15 nil] INFO [2019-12-17 12:36:43,567] jepsen worker 16 - jepsen.util 16486 :invoke :write [5 38] INFO [2019-12-17 12:36:43,570] jepsen worker 81 - jepsen.util 14751 :invoke :read [27 nil] INFO [2019-12-17 12:36:43,573] jepsen worker 53 - jepsen.util 10493 :invoke :cas [17 [48 8]] INFO [2019-12-17 12:36:43,579] jepsen worker 14 - jepsen.util 10814 :fail :cas [4 [12 32]] :ValueNotSwapFrom INFO [2019-12-17 12:36:43,580] jepsen worker 14 - jepsen.util 10814 :invoke :cas [4 [26 22]] INFO [2019-12-17 12:36:43,618] jepsen worker 62 - jepsen.util 15542 :info :cas [20 [23 47]] :RequestCanceledException INFO [2019-12-17 12:36:43,618] jepsen worker 39 - jepsen.util 14709 :info :write [13 27] :RequestCanceledException INFO [2019-12-17 12:36:43,632] jepsen worker 43 - jepsen.util 15163 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:36:43,632] jepsen worker 15 - jepsen.util 13965 :info :write [5 5] :RequestCanceledException INFO [2019-12-17 12:36:43,633] jepsen worker 43 - jepsen.util 15163 :invoke :read [14 nil] INFO [2019-12-17 12:36:43,634] jepsen worker 17 - jepsen.util 14957 :info :write [5 20] :RequestCanceledException INFO [2019-12-17 12:36:43,638] jepsen worker 62 - jepsen.util 15632 :invoke :cas [20 [1 40]] INFO [2019-12-17 12:36:43,647] jepsen worker 39 - jepsen.util 14799 :invoke :write [13 1] INFO [2019-12-17 12:36:43,647] jepsen worker 17 - jepsen.util 15047 :invoke :read [5 nil] INFO [2019-12-17 12:36:43,651] jepsen worker 15 - jepsen.util 14055 :invoke :cas [5 [6 3]] INFO [2019-12-17 12:36:43,653] jepsen worker 4 - jepsen.util 10264 :ok :write [1 20] INFO [2019-12-17 12:36:43,653] jepsen worker 4 - jepsen.util 10264 :invoke :write [1 0] INFO [2019-12-17 12:36:43,699] jepsen worker 88 - jepsen.util 11338 :info :write [29 5] :RequestCanceledException INFO [2019-12-17 12:36:43,699] jepsen worker 21 - jepsen.util 14511 :info :write [7 8] :RequestCanceledException INFO [2019-12-17 12:36:43,701] jepsen worker 89 - jepsen.util 11159 :ok :read [29 35] INFO [2019-12-17 12:36:43,702] jepsen worker 89 - jepsen.util 11159 :invoke :read [29 nil] INFO [2019-12-17 12:36:43,715] jepsen worker 88 - jepsen.util 11428 :invoke :cas [29 [43 39]] INFO [2019-12-17 12:36:43,715] jepsen worker 74 - jepsen.util 10784 :fail :cas [24 [41 39]] :ValueNotSwapFrom INFO [2019-12-17 12:36:43,717] jepsen worker 74 - jepsen.util 10784 :invoke :read [24 nil] INFO [2019-12-17 12:36:43,726] jepsen worker 21 - jepsen.util 14601 :invoke :cas [7 [11 31]] INFO [2019-12-17 12:36:43,733] jepsen worker 58 - jepsen.util 15448 :info :cas [19 [7 43]] :RequestCanceledException INFO [2019-12-17 12:36:43,746] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63925,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:36:43,747] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63925,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:36:43,747] jepsen worker 30 - jepsen.util 10470 :ok :read [10 0] INFO [2019-12-17 12:36:43,749] jepsen worker 30 - jepsen.util 10470 :invoke :write [10 10] INFO [2019-12-17 12:36:43,757] jepsen worker 58 - jepsen.util 15538 :invoke :read [19 nil] INFO [2019-12-17 12:36:43,763] jepsen worker 3 - jepsen.util 10623 :ok :read [1 20] INFO [2019-12-17 12:36:43,764] jepsen worker 3 - jepsen.util 10623 :invoke :cas [1 [15 28]] INFO [2019-12-17 12:36:43,770] jepsen worker 52 - jepsen.util 10672 :ok :read [17 45] INFO [2019-12-17 12:36:43,771] jepsen worker 52 - jepsen.util 10672 :invoke :read [17 nil] INFO [2019-12-17 12:36:43,796] jepsen worker 9 - jepsen.util 11709 :fail :cas [3 [42 37]] :ValueNotSwapFrom INFO [2019-12-17 12:36:43,796] jepsen worker 9 - jepsen.util 11709 :invoke :read [3 nil] INFO [2019-12-17 12:36:43,828] jepsen worker 69 - jepsen.util 11229 :ok :read [23 21] INFO [2019-12-17 12:36:43,829] jepsen worker 69 - jepsen.util 11229 :invoke :write [23 42] INFO [2019-12-17 12:36:43,832] jepsen worker 13 - jepsen.util 10453 :fail :cas [4 [19 36]] :ValueNotSwapFrom INFO [2019-12-17 12:36:43,833] jepsen worker 13 - jepsen.util 10453 :invoke :cas [4 [39 21]] INFO [2019-12-17 12:36:43,834] jepsen worker 26 - jepsen.util 15686 :info :write [8 13] :RequestCanceledException INFO [2019-12-17 12:36:43,862] jepsen worker 26 - jepsen.util 15776 :invoke :write [8 36] INFO [2019-12-17 12:36:43,879] jepsen worker 12 - jepsen.util 11442 :fail :cas [4 [37 43]] :ValueNotSwapFrom INFO [2019-12-17 12:36:43,879] jepsen worker 12 - jepsen.util 11442 :invoke :cas [4 [28 39]] INFO [2019-12-17 12:36:43,880] jepsen worker 20 - jepsen.util 10640 :fail :cas [6 [7 8]] :ValueNotSwapFrom INFO [2019-12-17 12:36:43,880] jepsen worker 20 - jepsen.util 10640 :invoke :read [6 nil] INFO [2019-12-17 12:36:43,884] jepsen worker 19 - jepsen.util 11359 :fail :cas [6 [11 8]] :ValueNotSwapFrom INFO [2019-12-17 12:36:43,885] jepsen worker 19 - jepsen.util 11359 :invoke :read [6 nil] INFO [2019-12-17 12:36:43,890] jepsen worker 64 - jepsen.util 8524 :ok :read [21 44] INFO [2019-12-17 12:36:43,890] jepsen worker 64 - jepsen.util 8524 :invoke :read [21 nil] INFO [2019-12-17 12:36:43,922] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63947,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:36:43,922] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63947,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:36:43,923] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63949,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:36:43,924] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63949,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:36:43,964] jepsen worker 51 - jepsen.util 10401 :info :write [17 29] :RequestCanceledException INFO [2019-12-17 12:36:43,987] jepsen worker 51 - jepsen.util 10491 :invoke :cas [17 [28 45]] INFO [2019-12-17 12:36:43,995] jepsen worker 31 - jepsen.util 9481 :info :write [10 4] :RequestCanceledException INFO [2019-12-17 12:36:43,997] jepsen worker 31 - jepsen.util 9571 :invoke :cas [10 [38 38]] INFO [2019-12-17 12:36:44,001] jepsen worker 37 - jepsen.util 10657 :fail :cas [12 [21 13]] :ValueNotSwapFrom INFO [2019-12-17 12:36:44,003] jepsen worker 37 - jepsen.util 10657 :invoke :cas [12 [43 43]] INFO [2019-12-17 12:36:44,026] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:36:44,027] jepsen nemesis - couchbase.util Rebalance check count: 502 INFO [2019-12-17 12:36:44,027] jepsen nemesis - couchbase.util Rebalance stuck count: 483 INFO [2019-12-17 12:36:44,029] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63960,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:36:44,029] jepsen worker 33 - jepsen.util 14973 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:36:44,030] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63960,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:36:44,031] jepsen worker 33 - jepsen.util 14973 :invoke :cas [11 [14 34]] INFO [2019-12-17 12:36:44,031] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63963,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:36:44,031] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63963,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:36:44,099] jepsen worker 56 - jepsen.util 11846 :ok :write [18 46] INFO [2019-12-17 12:36:44,100] jepsen worker 56 - jepsen.util 11846 :invoke :read [18 nil] INFO [2019-12-17 12:36:44,101] jepsen worker 70 - jepsen.util 11680 :info :write [23 26] :RequestCanceledException INFO [2019-12-17 12:36:44,101] jepsen worker 24 - jepsen.util 13614 :info :write [8 44] :RequestCanceledException INFO [2019-12-17 12:36:44,103] jepsen worker 65 - jepsen.util 9605 :fail :read [21 nil] :RequestTimeoutException INFO [2019-12-17 12:36:44,103] jepsen worker 65 - jepsen.util 9605 :invoke :cas [21 [9 26]] INFO [2019-12-17 12:36:44,117] jepsen worker 59 - jepsen.util 14189 :info :cas [19 [48 31]] :RequestCanceledException INFO [2019-12-17 12:36:44,117] jepsen worker 32 - jepsen.util 11192 :info :cas [10 [34 12]] :RequestCanceledException INFO [2019-12-17 12:36:44,117] jepsen worker 77 - jepsen.util 15647 :info :write [25 3] :RequestCanceledException INFO [2019-12-17 12:36:44,119] jepsen worker 70 - jepsen.util 11770 :invoke :read [23 nil] INFO [2019-12-17 12:36:44,123] jepsen worker 24 - jepsen.util 13704 :invoke :read [8 nil] INFO [2019-12-17 12:36:44,132] jepsen worker 57 - jepsen.util 14997 :info :write [19 29] :RequestCanceledException INFO [2019-12-17 12:36:44,133] jepsen worker 77 - jepsen.util 15737 :invoke :cas [25 [27 42]] INFO [2019-12-17 12:36:44,138] jepsen worker 10 - jepsen.util 10810 :ok :read [3 4] INFO [2019-12-17 12:36:44,139] jepsen worker 10 - jepsen.util 10810 :invoke :write [3 1] INFO [2019-12-17 12:36:44,141] jepsen worker 59 - jepsen.util 14279 :invoke :write [19 2] INFO [2019-12-17 12:36:44,156] jepsen worker 57 - jepsen.util 15087 :invoke :read [19 nil] INFO [2019-12-17 12:36:44,158] jepsen worker 32 - jepsen.util 11282 :invoke :write [10 2] INFO [2019-12-17 12:36:44,190] jepsen worker 63 - jepsen.util 9873 :fail :cas [21 [18 1]] :ValueNotSwapFrom INFO [2019-12-17 12:36:44,191] jepsen worker 63 - jepsen.util 9873 :invoke :write [21 31] INFO [2019-12-17 12:36:44,204] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63973,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:36:44,204] jepsen worker 29 - jepsen.util 9299 :fail :cas [9 [4 24]] :ValueNotSwapFrom INFO [2019-12-17 12:36:44,205] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63973,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:36:44,206] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63975,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:36:44,206] jepsen worker 29 - jepsen.util 9299 :invoke :write [9 5] INFO [2019-12-17 12:36:44,207] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63975,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:36:44,211] jepsen worker 89 - jepsen.util 11159 :ok :read [29 35] INFO [2019-12-17 12:36:44,213] jepsen worker 89 - jepsen.util 11159 :invoke :read [29 nil] INFO [2019-12-17 12:36:44,218] jepsen worker 75 - jepsen.util 14565 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:36:44,218] jepsen worker 75 - jepsen.util 14565 :invoke :cas [25 [41 47]] INFO [2019-12-17 12:36:44,232] jepsen worker 44 - jepsen.util 15614 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:36:44,233] jepsen worker 44 - jepsen.util 15614 :invoke :read [14 nil] INFO [2019-12-17 12:36:44,240] jepsen worker 46 - jepsen.util 10756 :ok :read [15 32] INFO [2019-12-17 12:36:44,241] jepsen worker 46 - jepsen.util 10756 :invoke :read [15 nil] INFO [2019-12-17 12:36:44,258] jepsen worker 74 - jepsen.util 10784 :ok :read [24 12] INFO [2019-12-17 12:36:44,259] jepsen worker 74 - jepsen.util 10784 :invoke :cas [24 [16 27]] INFO [2019-12-17 12:36:44,284] jepsen worker 14 - jepsen.util 10814 :fail :cas [4 [26 22]] :ValueNotSwapFrom INFO [2019-12-17 12:36:44,285] jepsen worker 14 - jepsen.util 10814 :invoke :write [4 22] INFO [2019-12-17 12:36:44,301] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63981,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:36:44,302] jepsen worker 9 - jepsen.util 11709 :ok :read [3 4] INFO [2019-12-17 12:36:44,303] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63981,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:36:44,303] jepsen worker 9 - jepsen.util 11709 :invoke :write [3 1] INFO [2019-12-17 12:36:44,304] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63986,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:36:44,305] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":63986,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:36:44,315] jepsen worker 19 - jepsen.util 11359 :ok :read [6 31] INFO [2019-12-17 12:36:44,316] jepsen worker 19 - jepsen.util 11359 :invoke :cas [6 [4 9]] INFO [2019-12-17 12:36:44,317] jepsen worker 76 - jepsen.util 15466 :info :cas [25 [34 9]] :RequestCanceledException INFO [2019-12-17 12:36:44,320] jepsen worker 20 - jepsen.util 10640 :ok :read [6 31] INFO [2019-12-17 12:36:44,321] jepsen worker 20 - jepsen.util 10640 :invoke :read [6 nil] INFO [2019-12-17 12:36:44,327] jepsen worker 76 - jepsen.util 15556 :invoke :read [25 nil] INFO [2019-12-17 12:36:44,331] jepsen worker 31 - jepsen.util 9571 :fail :cas [10 [38 38]] :ValueNotSwapFrom INFO [2019-12-17 12:36:44,332] jepsen worker 31 - jepsen.util 9571 :invoke :read [10 nil] INFO [2019-12-17 12:36:44,332] jepsen worker 22 - jepsen.util 14962 :info :write [7 40] :RequestCanceledException INFO [2019-12-17 12:36:44,348] jepsen worker 22 - jepsen.util 15052 :invoke :write [7 44] INFO [2019-12-17 12:36:44,386] jepsen worker 11 - jepsen.util 10721 :ok :cas [3 [4 29]] INFO [2019-12-17 12:36:44,386] jepsen worker 11 - jepsen.util 10721 :invoke :cas [3 [4 1]] INFO [2019-12-17 12:36:44,412] jepsen worker 3 - jepsen.util 10623 :fail :cas [1 [15 28]] :ValueNotSwapFrom INFO [2019-12-17 12:36:44,412] jepsen worker 3 - jepsen.util 10623 :invoke :cas [1 [32 7]] INFO [2019-12-17 12:36:44,433] jepsen worker 18 - jepsen.util 11178 :info :write [6 13] :RequestCanceledException INFO [2019-12-17 12:36:44,433] jepsen worker 66 - jepsen.util 14916 :info :cas [22 [26 33]] :RequestCanceledException INFO [2019-12-17 12:36:44,433] jepsen worker 1 - jepsen.util 15031 :info :write [0 39] :RequestCanceledException INFO [2019-12-17 12:36:44,434] jepsen worker 49 - jepsen.util 14899 :info :cas [16 [8 27]] :RequestCanceledException INFO [2019-12-17 12:36:44,443] jepsen worker 18 - jepsen.util 11268 :invoke :cas [6 [39 2]] INFO [2019-12-17 12:36:44,453] jepsen worker 49 - jepsen.util 14989 :invoke :write [16 37] INFO [2019-12-17 12:36:44,460] jepsen worker 66 - jepsen.util 15006 :invoke :cas [22 [28 18]] INFO [2019-12-17 12:36:44,474] jepsen worker 1 - jepsen.util 15121 :invoke :read [0 nil] INFO [2019-12-17 12:36:44,484] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64004,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:36:44,484] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64004,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:36:44,484] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64008,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:36:44,484] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64008,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:36:44,485] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64009,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:36:44,486] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64009,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:36:44,517] jepsen worker 64 - jepsen.util 8524 :ok :read [21 44] INFO [2019-12-17 12:36:44,517] jepsen worker 64 - jepsen.util 8524 :invoke :write [21 28] INFO [2019-12-17 12:36:44,563] jepsen worker 80 - jepsen.util 10700 :fail :cas [26 [14 11]] :ValueNotSwapFrom INFO [2019-12-17 12:36:44,563] jepsen worker 80 - jepsen.util 10700 :invoke :write [26 11] INFO [2019-12-17 12:36:44,564] jepsen worker 38 - jepsen.util 11558 :info :cas [12 [45 1]] :RequestCanceledException INFO [2019-12-17 12:36:44,564] jepsen worker 68 - jepsen.util 14378 :info :cas [22 [43 26]] :RequestCanceledException INFO [2019-12-17 12:36:44,568] jepsen worker 68 - jepsen.util 14468 :invoke :read [22 nil] INFO [2019-12-17 12:36:44,576] jepsen worker 38 - jepsen.util 11648 :invoke :cas [12 [42 33]] INFO [2019-12-17 12:36:44,578] jepsen worker 72 - jepsen.util 11322 :ok :read [24 44] INFO [2019-12-17 12:36:44,578] jepsen worker 72 - jepsen.util 11322 :invoke :cas [24 [10 33]] INFO [2019-12-17 12:36:44,586] jepsen worker 78 - jepsen.util 11238 :fail :cas [26 [5 35]] :ValueNotSwapFrom INFO [2019-12-17 12:36:44,586] jepsen worker 78 - jepsen.util 11238 :invoke :write [26 40] INFO [2019-12-17 12:36:44,586] jepsen worker 54 - jepsen.util 11844 :fail :cas [18 [5 19]] :ValueNotSwapFrom INFO [2019-12-17 12:36:44,587] jepsen worker 54 - jepsen.util 11844 :invoke :cas [18 [33 15]] INFO [2019-12-17 12:36:44,590] jepsen worker 40 - jepsen.util 15340 :info :write [13 20] :RequestCanceledException INFO [2019-12-17 12:36:44,591] jepsen worker 87 - jepsen.util 10617 :info :write [29 18] :RequestCanceledException INFO [2019-12-17 12:36:44,599] jepsen worker 87 - jepsen.util 10707 :invoke :read [29 nil] INFO [2019-12-17 12:36:44,600] jepsen worker 40 - jepsen.util 15430 :invoke :write [13 30] INFO [2019-12-17 12:36:44,607] jepsen worker 20 - jepsen.util 10640 :ok :read [6 31] INFO [2019-12-17 12:36:44,607] jepsen worker 20 - jepsen.util 10640 :invoke :cas [6 [0 33]] INFO [2019-12-17 12:36:44,613] jepsen worker 11 - jepsen.util 10721 :fail :cas [3 [4 1]] :ValueNotSwapFrom INFO [2019-12-17 12:36:44,614] jepsen worker 11 - jepsen.util 10721 :invoke :cas [3 [7 47]] INFO [2019-12-17 12:36:44,616] jepsen worker 56 - jepsen.util 11846 :ok :read [18 46] INFO [2019-12-17 12:36:44,616] jepsen worker 56 - jepsen.util 11846 :invoke :cas [18 [8 12]] INFO [2019-12-17 12:36:44,617] jepsen worker 6 - jepsen.util 15486 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:36:44,617] jepsen worker 6 - jepsen.util 15486 :invoke :write [2 36] INFO [2019-12-17 12:36:44,633] jepsen worker 86 - jepsen.util 15746 :info :write [28 31] :RequestCanceledException INFO [2019-12-17 12:36:44,638] jepsen worker 65 - jepsen.util 9605 :fail :cas [21 [9 26]] :ValueNotSwapFrom INFO [2019-12-17 12:36:44,639] jepsen worker 65 - jepsen.util 9605 :invoke :write [21 23] INFO [2019-12-17 12:36:44,642] jepsen worker 86 - jepsen.util 15836 :invoke :read [28 nil] INFO [2019-12-17 12:36:44,648] jepsen worker 34 - jepsen.util 15514 :info :write [11 13] :RequestCanceledException INFO [2019-12-17 12:36:44,649] jepsen worker 35 - jepsen.util 14975 :info :write [11 26] :RequestCanceledException INFO [2019-12-17 12:36:44,650] jepsen worker 45 - jepsen.util 11205 :fail :cas [15 [35 21]] :ValueNotSwapFrom INFO [2019-12-17 12:36:44,652] jepsen worker 45 - jepsen.util 11205 :invoke :read [15 nil] INFO [2019-12-17 12:36:44,657] jepsen worker 34 - jepsen.util 15604 :invoke :cas [11 [33 2]] INFO [2019-12-17 12:36:44,658] jepsen worker 35 - jepsen.util 15065 :invoke :read [11 nil] INFO [2019-12-17 12:36:44,664] jepsen worker 55 - jepsen.util 11665 :info :write [18 33] :RequestCanceledException INFO [2019-12-17 12:36:44,667] jepsen worker 60 - jepsen.util 15990 :info :write [20 39] :RequestCanceledException INFO [2019-12-17 12:36:44,667] jepsen worker 71 - jepsen.util 12041 :info :write [23 14] :RequestCanceledException INFO [2019-12-17 12:36:44,669] jepsen worker 55 - jepsen.util 11755 :invoke :write [18 36] INFO [2019-12-17 12:36:44,677] jepsen worker 60 - jepsen.util 16080 :invoke :write [20 20] INFO [2019-12-17 12:36:44,682] jepsen worker 71 - jepsen.util 12131 :invoke :read [23 nil] INFO [2019-12-17 12:36:44,727] jepsen worker 18 - jepsen.util 11268 :fail :cas [6 [39 2]] :ValueNotSwapFrom INFO [2019-12-17 12:36:44,727] jepsen worker 18 - jepsen.util 11268 :invoke :cas [6 [6 25]] INFO [2019-12-17 12:36:44,743] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64029,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:36:44,744] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64029,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:36:44,744] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64032,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:36:44,744] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64032,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:36:44,744] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64033,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:36:44,744] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64033,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:36:44,746] jepsen worker 42 - jepsen.util 15702 :info :write [14 26] :RequestCanceledException INFO [2019-12-17 12:36:44,776] jepsen worker 42 - jepsen.util 15792 :invoke :write [14 39] INFO [2019-12-17 12:36:44,829] jepsen worker 72 - jepsen.util 11322 :fail :cas [24 [10 33]] :ValueNotSwapFrom INFO [2019-12-17 12:36:44,831] jepsen worker 72 - jepsen.util 11322 :invoke :cas [24 [9 8]] INFO [2019-12-17 12:36:44,900] jepsen worker 25 - jepsen.util 16225 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:36:44,900] jepsen worker 25 - jepsen.util 16225 :invoke :read [8 nil] INFO [2019-12-17 12:36:44,922] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64046,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:36:44,923] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64046,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:36:44,924] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64049,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:36:44,924] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64049,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:36:44,924] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64050,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:36:44,924] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64050,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:36:44,953] jepsen worker 45 - jepsen.util 11205 :ok :read [15 32] INFO [2019-12-17 12:36:44,953] jepsen worker 45 - jepsen.util 11205 :invoke :read [15 nil] INFO [2019-12-17 12:36:44,967] jepsen worker 27 - jepsen.util 10467 :ok :read [9 28] INFO [2019-12-17 12:36:44,972] jepsen worker 27 - jepsen.util 10467 :invoke :write [9 35] INFO [2019-12-17 12:36:45,024] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64056,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:36:45,024] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64056,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:36:45,024] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64058,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:36:45,024] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64058,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:36:45,024] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64059,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:36:45,024] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64059,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:36:45,085] jepsen worker 18 - jepsen.util 11268 :fail :cas [6 [6 25]] :ValueNotSwapFrom INFO [2019-12-17 12:36:45,085] jepsen worker 18 - jepsen.util 11268 :invoke :read [6 nil] INFO [2019-12-17 12:36:45,085] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:36:45,085] jepsen nemesis - couchbase.util Rebalance check count: 503 INFO [2019-12-17 12:36:45,085] jepsen nemesis - couchbase.util Rebalance stuck count: 484 INFO [2019-12-17 12:36:45,095] jepsen worker 11 - jepsen.util 10721 :fail :cas [3 [7 47]] :ValueNotSwapFrom INFO [2019-12-17 12:36:45,096] jepsen worker 11 - jepsen.util 10721 :invoke :write [3 2] INFO [2019-12-17 12:36:45,098] jepsen worker 28 - jepsen.util 9118 :info :cas [9 [15 38]] :RequestCanceledException INFO [2019-12-17 12:36:45,123] jepsen worker 72 - jepsen.util 11322 :fail :cas [24 [9 8]] :ValueNotSwapFrom INFO [2019-12-17 12:36:45,123] jepsen worker 72 - jepsen.util 11322 :invoke :cas [24 [37 27]] INFO [2019-12-17 12:36:45,125] jepsen worker 28 - jepsen.util 9208 :invoke :write [9 13] INFO [2019-12-17 12:36:45,178] jepsen worker 79 - jepsen.util 11329 :ok :write [26 28] INFO [2019-12-17 12:36:45,178] jepsen worker 79 - jepsen.util 11329 :invoke :write [26 38] INFO [2019-12-17 12:36:45,190] jepsen worker 56 - jepsen.util 11846 :fail :cas [18 [8 12]] :ValueNotSwapFrom INFO [2019-12-17 12:36:45,190] jepsen worker 56 - jepsen.util 11846 :invoke :write [18 34] INFO [2019-12-17 12:36:45,197] jepsen worker 36 - jepsen.util 12006 :ok :write [12 34] INFO [2019-12-17 12:36:45,202] jepsen worker 36 - jepsen.util 12006 :invoke :read [12 nil] INFO [2019-12-17 12:36:45,216] jepsen worker 4 - jepsen.util 10264 :ok :write [1 0] INFO [2019-12-17 12:36:45,216] jepsen worker 4 - jepsen.util 10264 :invoke :cas [1 [21 31]] INFO [2019-12-17 12:36:45,229] jepsen worker 0 - jepsen.util 15930 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:36:45,229] jepsen worker 0 - jepsen.util 15930 :invoke :cas [0 [36 8]] INFO [2019-12-17 12:36:45,234] jepsen worker 54 - jepsen.util 11844 :fail :cas [18 [33 15]] :ValueNotSwapFrom INFO [2019-12-17 12:36:45,234] jepsen worker 54 - jepsen.util 11844 :invoke :cas [18 [28 2]] INFO [2019-12-17 12:36:45,264] jepsen worker 82 - jepsen.util 15202 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:36:45,264] jepsen worker 82 - jepsen.util 15202 :invoke :cas [27 [48 43]] INFO [2019-12-17 12:36:45,280] jepsen worker 18 - jepsen.util 11268 :ok :read [6 13] INFO [2019-12-17 12:36:45,280] jepsen worker 18 - jepsen.util 11268 :invoke :read [6 nil] INFO [2019-12-17 12:36:45,286] jepsen worker 30 - jepsen.util 10470 :ok :write [10 10] INFO [2019-12-17 12:36:45,286] jepsen worker 30 - jepsen.util 10470 :invoke :read [10 nil] INFO [2019-12-17 12:36:45,287] jepsen worker 64 - jepsen.util 8524 :ok :write [21 28] INFO [2019-12-17 12:36:45,288] jepsen worker 64 - jepsen.util 8524 :invoke :cas [21 [40 35]] INFO [2019-12-17 12:36:45,294] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64064,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:36:45,294] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64064,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:36:45,294] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64065,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:36:45,294] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64074,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:36:45,294] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64065,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:36:45,295] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64074,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:36:45,330] jepsen worker 84 - jepsen.util 14754 :info :cas [28 [31 26]] :RequestCanceledException INFO [2019-12-17 12:36:45,335] jepsen worker 84 - jepsen.util 14844 :invoke :cas [28 [40 45]] INFO [2019-12-17 12:36:45,346] jepsen worker 83 - jepsen.util 15203 :info :write [27 46] :RequestCanceledException INFO [2019-12-17 12:36:45,357] jepsen worker 67 - jepsen.util 15187 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:36:45,358] jepsen worker 67 - jepsen.util 15187 :invoke :read [22 nil] INFO [2019-12-17 12:36:45,366] jepsen worker 83 - jepsen.util 15293 :invoke :write [27 36] INFO [2019-12-17 12:36:45,396] jepsen worker 85 - jepsen.util 15925 :info :cas [28 [23 24]] :RequestCanceledException INFO [2019-12-17 12:36:45,400] jepsen worker 85 - jepsen.util 16015 :invoke :read [28 nil] INFO [2019-12-17 12:36:45,417] jepsen worker 61 - jepsen.util 16081 :info :write [20 10] :RequestCanceledException INFO [2019-12-17 12:36:45,417] jepsen worker 41 - jepsen.util 16061 :info :write [13 26] :RequestCanceledException INFO [2019-12-17 12:36:45,420] jepsen worker 20 - jepsen.util 10640 :fail :cas [6 [0 33]] :ValueNotSwapFrom INFO [2019-12-17 12:36:45,421] jepsen worker 20 - jepsen.util 10640 :invoke :read [6 nil] INFO [2019-12-17 12:36:45,427] jepsen worker 72 - jepsen.util 11322 :fail :cas [24 [37 27]] :ValueNotSwapFrom INFO [2019-12-17 12:36:45,427] jepsen worker 72 - jepsen.util 11322 :invoke :cas [24 [1 27]] INFO [2019-12-17 12:36:45,435] jepsen worker 61 - jepsen.util 16171 :invoke :write [20 45] INFO [2019-12-17 12:36:45,436] jepsen worker 41 - jepsen.util 16151 :invoke :write [13 41] INFO [2019-12-17 12:36:45,443] jepsen worker 8 - jepsen.util 14768 :info :write [2 25] :RequestCanceledException INFO [2019-12-17 12:36:45,460] jepsen worker 71 - jepsen.util 12131 :ok :read [23 5] INFO [2019-12-17 12:36:45,460] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64094,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:36:45,460] jepsen worker 71 - jepsen.util 12131 :invoke :cas [23 [48 24]] INFO [2019-12-17 12:36:45,461] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64094,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:36:45,462] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64096,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:36:45,466] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64096,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:36:45,467] jepsen worker 8 - jepsen.util 14858 :invoke :write [2 25] INFO [2019-12-17 12:36:45,470] jepsen worker 65 - jepsen.util 9605 :ok :write [21 23] INFO [2019-12-17 12:36:45,471] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64097,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:36:45,472] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64097,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:36:45,472] jepsen worker 65 - jepsen.util 9605 :invoke :write [21 6] INFO [2019-12-17 12:36:45,478] jepsen worker 46 - jepsen.util 10756 :ok :read [15 32] INFO [2019-12-17 12:36:45,478] jepsen worker 46 - jepsen.util 10756 :invoke :write [15 35] INFO [2019-12-17 12:36:45,495] jepsen worker 88 - jepsen.util 11428 :fail :cas [29 [43 39]] :ValueNotSwapFrom INFO [2019-12-17 12:36:45,495] jepsen worker 88 - jepsen.util 11428 :invoke :cas [29 [15 9]] INFO [2019-12-17 12:36:45,522] jepsen worker 74 - jepsen.util 10784 :fail :cas [24 [16 27]] :ValueNotSwapFrom INFO [2019-12-17 12:36:45,523] jepsen worker 74 - jepsen.util 10784 :invoke :read [24 nil] INFO [2019-12-17 12:36:45,529] jepsen worker 64 - jepsen.util 8524 :fail :cas [21 [40 35]] :ValueNotSwapFrom INFO [2019-12-17 12:36:45,529] jepsen worker 64 - jepsen.util 8524 :invoke :read [21 nil] INFO [2019-12-17 12:36:45,532] jepsen worker 47 - jepsen.util 11657 :info :write [15 8] :RequestCanceledException INFO [2019-12-17 12:36:45,532] jepsen worker 50 - jepsen.util 15620 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:36:45,533] jepsen worker 50 - jepsen.util 15620 :invoke :read [16 nil] INFO [2019-12-17 12:36:45,535] jepsen worker 47 - jepsen.util 11747 :invoke :read [15 nil] INFO [2019-12-17 12:36:45,561] jepsen worker 23 - jepsen.util 14963 :info :cas [7 [43 18]] :RequestCanceledException INFO [2019-12-17 12:36:45,564] jepsen worker 23 - jepsen.util 15053 :invoke :write [7 19] INFO [2019-12-17 12:36:45,599] jepsen worker 48 - jepsen.util 13998 :info :write [16 38] :RequestCanceledException INFO [2019-12-17 12:36:45,599] jepsen worker 53 - jepsen.util 10493 :info :cas [17 [48 8]] :RequestCanceledException INFO [2019-12-17 12:36:45,601] jepsen worker 48 - jepsen.util 14088 :invoke :read [16 nil] INFO [2019-12-17 12:36:45,603] jepsen worker 53 - jepsen.util 10583 :invoke :cas [17 [3 9]] INFO [2019-12-17 12:36:45,627] jepsen worker 81 - jepsen.util 14751 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:36:45,627] jepsen worker 7 - jepsen.util 15037 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:36:45,627] jepsen worker 81 - jepsen.util 14751 :invoke :write [27 11] INFO [2019-12-17 12:36:45,627] jepsen worker 7 - jepsen.util 15037 :invoke :cas [2 [26 9]] INFO [2019-12-17 12:36:45,631] jepsen worker 18 - jepsen.util 11268 :ok :read [6 13] INFO [2019-12-17 12:36:45,631] jepsen worker 18 - jepsen.util 11268 :invoke :cas [6 [38 34]] INFO [2019-12-17 12:36:45,648] jepsen worker 2 - jepsen.util 14582 :info :cas [0 [24 45]] :RequestCanceledException INFO [2019-12-17 12:36:45,665] jepsen worker 15 - jepsen.util 14055 :info :cas [5 [6 3]] :RequestCanceledException INFO [2019-12-17 12:36:45,666] jepsen worker 2 - jepsen.util 14672 :invoke :read [0 nil] INFO [2019-12-17 12:36:45,672] jepsen worker 15 - jepsen.util 14145 :invoke :write [5 16] INFO [2019-12-17 12:36:45,695] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64116,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:36:45,697] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64116,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:36:45,697] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64119,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:36:45,697] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64119,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:36:45,698] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64120,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:36:45,699] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64120,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:36:45,714] jepsen worker 13 - jepsen.util 10453 :fail :cas [4 [39 21]] :ValueNotSwapFrom INFO [2019-12-17 12:36:45,715] jepsen worker 13 - jepsen.util 10453 :invoke :read [4 nil] INFO [2019-12-17 12:36:45,715] jepsen worker 62 - jepsen.util 15632 :info :cas [20 [1 40]] :RequestCanceledException INFO [2019-12-17 12:36:45,725] jepsen worker 62 - jepsen.util 15722 :invoke :write [20 12] INFO [2019-12-17 12:36:45,732] jepsen worker 43 - jepsen.util 15163 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:36:45,735] jepsen worker 43 - jepsen.util 15163 :invoke :cas [14 [12 24]] INFO [2019-12-17 12:36:45,745] jepsen worker 20 - jepsen.util 10640 :ok :read [6 13] INFO [2019-12-17 12:36:45,746] jepsen worker 20 - jepsen.util 10640 :invoke :cas [6 [13 34]] INFO [2019-12-17 12:36:45,747] jepsen worker 17 - jepsen.util 15047 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:36:45,747] jepsen worker 17 - jepsen.util 15047 :invoke :cas [5 [49 43]] INFO [2019-12-17 12:36:45,764] jepsen worker 16 - jepsen.util 16486 :info :write [5 38] :RequestCanceledException INFO [2019-12-17 12:36:45,767] jepsen worker 16 - jepsen.util 16576 :invoke :write [5 49] INFO [2019-12-17 12:36:45,798] jepsen worker 21 - jepsen.util 14601 :info :cas [7 [11 31]] :RequestCanceledException INFO [2019-12-17 12:36:45,805] jepsen worker 29 - jepsen.util 9299 :ok :write [9 5] INFO [2019-12-17 12:36:45,805] jepsen worker 29 - jepsen.util 9299 :invoke :cas [9 [15 5]] INFO [2019-12-17 12:36:45,807] jepsen worker 72 - jepsen.util 11322 :fail :cas [24 [1 27]] :ValueNotSwapFrom INFO [2019-12-17 12:36:45,807] jepsen worker 72 - jepsen.util 11322 :invoke :write [24 7] INFO [2019-12-17 12:36:45,809] jepsen worker 58 - jepsen.util 15538 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:36:45,810] jepsen worker 21 - jepsen.util 14691 :invoke :write [7 30] INFO [2019-12-17 12:36:45,810] jepsen worker 58 - jepsen.util 15538 :invoke :read [19 nil] INFO [2019-12-17 12:36:45,812] jepsen worker 64 - jepsen.util 8524 :ok :read [21 23] INFO [2019-12-17 12:36:45,821] jepsen worker 64 - jepsen.util 8524 :invoke :write [21 23] INFO [2019-12-17 12:36:45,830] jepsen worker 5 - jepsen.util 10805 :info :write [1 40] :RequestCanceledException INFO [2019-12-17 12:36:45,830] jepsen worker 39 - jepsen.util 14799 :info :write [13 1] :RequestCanceledException INFO [2019-12-17 12:36:45,841] jepsen worker 5 - jepsen.util 10895 :invoke :read [1 nil] INFO [2019-12-17 12:36:45,841] jepsen worker 39 - jepsen.util 14889 :invoke :read [13 nil] INFO [2019-12-17 12:36:45,872] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64135,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:36:45,873] jepsen worker 52 - jepsen.util 10672 :ok :read [17 45] INFO [2019-12-17 12:36:45,873] jepsen worker 52 - jepsen.util 10672 :invoke :read [17 nil] INFO [2019-12-17 12:36:45,873] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64135,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:36:45,875] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64138,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:36:45,876] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64138,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:36:45,882] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64139,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:36:45,882] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64139,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:36:45,910] jepsen worker 47 - jepsen.util 11747 :ok :read [15 32] INFO [2019-12-17 12:36:45,911] jepsen worker 47 - jepsen.util 11747 :invoke :write [15 49] INFO [2019-12-17 12:36:45,948] jepsen worker 69 - jepsen.util 11229 :info :write [23 42] :RequestCanceledException INFO [2019-12-17 12:36:45,956] jepsen worker 69 - jepsen.util 11319 :invoke :read [23 nil] INFO [2019-12-17 12:36:45,958] jepsen worker 30 - jepsen.util 10470 :ok :read [10 10] INFO [2019-12-17 12:36:45,959] jepsen worker 30 - jepsen.util 10470 :invoke :cas [10 [19 39]] INFO [2019-12-17 12:36:45,962] jepsen worker 12 - jepsen.util 11442 :fail :cas [4 [28 39]] :ValueNotSwapFrom INFO [2019-12-17 12:36:45,962] jepsen worker 12 - jepsen.util 11442 :invoke :write [4 25] INFO [2019-12-17 12:36:45,971] jepsen worker 18 - jepsen.util 11268 :fail :cas [6 [38 34]] :ValueNotSwapFrom INFO [2019-12-17 12:36:45,971] jepsen worker 18 - jepsen.util 11268 :invoke :read [6 nil] INFO [2019-12-17 12:36:45,972] jepsen worker 4 - jepsen.util 10264 :fail :cas [1 [21 31]] :ValueNotSwapFrom INFO [2019-12-17 12:36:45,972] jepsen worker 4 - jepsen.util 10264 :invoke :cas [1 [21 36]] INFO [2019-12-17 12:36:46,017] jepsen worker 65 - jepsen.util 9605 :ok :write [21 6] INFO [2019-12-17 12:36:46,018] jepsen worker 65 - jepsen.util 9605 :invoke :write [21 26] INFO [2019-12-17 12:36:46,032] jepsen worker 26 - jepsen.util 15776 :info :write [8 36] :RequestCanceledException INFO [2019-12-17 12:36:46,036] jepsen worker 26 - jepsen.util 15866 :invoke :write [8 5] INFO [2019-12-17 12:36:46,039] jepsen worker 10 - jepsen.util 10810 :ok :write [3 1] INFO [2019-12-17 12:36:46,043] jepsen worker 10 - jepsen.util 10810 :invoke :cas [3 [13 14]] INFO [2019-12-17 12:36:46,045] jepsen worker 51 - jepsen.util 10491 :info :cas [17 [28 45]] :RequestCanceledException INFO [2019-12-17 12:36:46,045] jepsen worker 37 - jepsen.util 10657 :info :cas [12 [43 43]] :RequestCanceledException INFO [2019-12-17 12:36:46,056] jepsen worker 37 - jepsen.util 10747 :invoke :write [12 49] INFO [2019-12-17 12:36:46,061] jepsen worker 51 - jepsen.util 10581 :invoke :read [17 nil] INFO [2019-12-17 12:36:46,099] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:36:46,099] jepsen nemesis - couchbase.util Rebalance check count: 504 INFO [2019-12-17 12:36:46,099] jepsen nemesis - couchbase.util Rebalance stuck count: 485 INFO [2019-12-17 12:36:46,104] jepsen worker 27 - jepsen.util 10467 :ok :write [9 35] INFO [2019-12-17 12:36:46,104] jepsen worker 27 - jepsen.util 10467 :invoke :read [9 nil] INFO [2019-12-17 12:36:46,154] jepsen worker 88 - jepsen.util 11428 :fail :cas [29 [15 9]] :ValueNotSwapFrom INFO [2019-12-17 12:36:46,155] jepsen worker 88 - jepsen.util 11428 :invoke :write [29 11] INFO [2019-12-17 12:36:46,155] jepsen worker 70 - jepsen.util 11770 :fail :read [23 nil] :RequestTimeoutException INFO [2019-12-17 12:36:46,155] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64151,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:36:46,155] jepsen worker 70 - jepsen.util 11770 :invoke :read [23 nil] INFO [2019-12-17 12:36:46,156] jepsen worker 33 - jepsen.util 14973 :info :cas [11 [14 34]] :RequestCanceledException INFO [2019-12-17 12:36:46,156] jepsen worker 73 - jepsen.util 10243 :fail :cas [24 [12 38]] :CASMismatchException INFO [2019-12-17 12:36:46,157] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64151,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:36:46,158] jepsen worker 73 - jepsen.util 10243 :invoke :cas [24 [11 23]] INFO [2019-12-17 12:36:46,161] jepsen worker 33 - jepsen.util 15063 :invoke :cas [11 [42 42]] INFO [2019-12-17 12:36:46,198] jepsen worker 24 - jepsen.util 13704 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:36:46,199] jepsen worker 24 - jepsen.util 13704 :invoke :cas [8 [22 36]] INFO [2019-12-17 12:36:46,207] jepsen worker 63 - jepsen.util 9873 :ok :write [21 31] INFO [2019-12-17 12:36:46,207] jepsen worker 63 - jepsen.util 9873 :invoke :cas [21 [35 42]] INFO [2019-12-17 12:36:46,209] jepsen worker 77 - jepsen.util 15737 :info :cas [25 [27 42]] :RequestCanceledException INFO [2019-12-17 12:36:46,213] jepsen worker 77 - jepsen.util 15827 :invoke :write [25 37] INFO [2019-12-17 12:36:46,214] jepsen worker 19 - jepsen.util 11359 :fail :cas [6 [4 9]] :ValueNotSwapFrom INFO [2019-12-17 12:36:46,214] jepsen worker 19 - jepsen.util 11359 :invoke :write [6 48] INFO [2019-12-17 12:36:46,218] jepsen worker 89 - jepsen.util 11159 :ok :read [29 5] INFO [2019-12-17 12:36:46,218] jepsen worker 89 - jepsen.util 11159 :invoke :read [29 nil] INFO [2019-12-17 12:36:46,248] jepsen worker 57 - jepsen.util 15087 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:36:46,248] jepsen worker 57 - jepsen.util 15087 :invoke :read [19 nil] INFO [2019-12-17 12:36:46,250] jepsen worker 27 - jepsen.util 10467 :ok :read [9 35] INFO [2019-12-17 12:36:46,250] jepsen worker 27 - jepsen.util 10467 :invoke :read [9 nil] INFO [2019-12-17 12:36:46,259] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64166,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:36:46,259] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64174,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:36:46,259] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64166,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:36:46,260] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64174,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:36:46,260] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64168,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:36:46,260] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64179,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:36:46,260] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64168,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:36:46,260] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64179,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:36:46,260] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64169,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:36:46,260] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64169,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:36:46,282] jepsen worker 64 - jepsen.util 8524 :ok :write [21 23] INFO [2019-12-17 12:36:46,283] jepsen worker 64 - jepsen.util 8524 :invoke :cas [21 [2 23]] INFO [2019-12-17 12:36:46,287] jepsen worker 31 - jepsen.util 9571 :ok :read [10 10] INFO [2019-12-17 12:36:46,287] jepsen worker 31 - jepsen.util 9571 :invoke :read [10 nil] INFO [2019-12-17 12:36:46,309] jepsen worker 75 - jepsen.util 14565 :info :cas [25 [41 47]] :RequestCanceledException INFO [2019-12-17 12:36:46,315] jepsen worker 75 - jepsen.util 14655 :invoke :read [25 nil] INFO [2019-12-17 12:36:46,333] jepsen worker 44 - jepsen.util 15614 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:36:46,333] jepsen worker 59 - jepsen.util 14279 :info :write [19 2] :RequestCanceledException INFO [2019-12-17 12:36:46,333] jepsen worker 44 - jepsen.util 15614 :invoke :write [14 49] INFO [2019-12-17 12:36:46,337] jepsen worker 59 - jepsen.util 14369 :invoke :cas [19 [11 28]] INFO [2019-12-17 12:36:46,354] jepsen worker 4 - jepsen.util 10264 :fail :cas [1 [21 36]] :ValueNotSwapFrom INFO [2019-12-17 12:36:46,354] jepsen worker 4 - jepsen.util 10264 :invoke :cas [1 [33 10]] INFO [2019-12-17 12:36:46,365] jepsen worker 32 - jepsen.util 11282 :info :write [10 2] :RequestCanceledException INFO [2019-12-17 12:36:46,370] jepsen worker 32 - jepsen.util 11372 :invoke :write [10 4] INFO [2019-12-17 12:36:46,396] jepsen worker 87 - jepsen.util 10707 :ok :read [29 5] INFO [2019-12-17 12:36:46,396] jepsen worker 87 - jepsen.util 10707 :invoke :write [29 45] INFO [2019-12-17 12:36:46,397] jepsen worker 65 - jepsen.util 9605 :ok :write [21 26] INFO [2019-12-17 12:36:46,397] jepsen worker 65 - jepsen.util 9605 :invoke :cas [21 [11 3]] INFO [2019-12-17 12:36:46,405] jepsen worker 27 - jepsen.util 10467 :ok :read [9 35] INFO [2019-12-17 12:36:46,405] jepsen worker 27 - jepsen.util 10467 :invoke :cas [9 [24 15]] INFO [2019-12-17 12:36:46,430] jepsen worker 76 - jepsen.util 15556 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:36:46,431] jepsen worker 76 - jepsen.util 15556 :invoke :cas [25 [46 18]] INFO [2019-12-17 12:36:46,438] jepsen worker 14 - jepsen.util 10814 :info :write [4 22] :RequestCanceledException INFO [2019-12-17 12:36:46,438] jepsen worker 3 - jepsen.util 10623 :info :cas [1 [32 7]] :RequestCanceledException INFO [2019-12-17 12:36:46,442] jepsen worker 3 - jepsen.util 10713 :invoke :read [1 nil] INFO [2019-12-17 12:36:46,443] jepsen worker 14 - jepsen.util 10904 :invoke :write [4 17] INFO [2019-12-17 12:36:46,457] jepsen worker 9 - jepsen.util 11709 :info :write [3 1] :RequestCanceledException INFO [2019-12-17 12:36:46,457] jepsen worker 36 - jepsen.util 12006 :ok :read [12 34] INFO [2019-12-17 12:36:46,457] jepsen worker 36 - jepsen.util 12006 :invoke :read [12 nil] INFO [2019-12-17 12:36:46,457] jepsen worker 64 - jepsen.util 8524 :fail :cas [21 [2 23]] :ValueNotSwapFrom INFO [2019-12-17 12:36:46,459] jepsen worker 9 - jepsen.util 11799 :invoke :read [3 nil] INFO [2019-12-17 12:36:46,475] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64195,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:36:46,481] jepsen worker 46 - jepsen.util 10756 :ok :write [15 35] INFO [2019-12-17 12:36:46,494] jepsen worker 22 - jepsen.util 15052 :info :write [7 44] :RequestCanceledException INFO [2019-12-17 12:36:46,533] jepsen worker 74 - jepsen.util 10784 :ok :read [24 44] INFO [2019-12-17 12:36:46,543] jepsen worker 66 - jepsen.util 15006 :info :cas [22 [28 18]] :RequestCanceledException INFO [2019-12-17 12:36:46,553] jepsen worker 72 - jepsen.util 11322 :ok :write [24 7] INFO [2019-12-17 12:36:46,553] jepsen worker 29 - jepsen.util 9299 :fail :cas [9 [15 5]] :ValueNotSwapFrom INFO [2019-12-17 12:36:46,561] jepsen worker 38 - jepsen.util 11648 :fail :cas [12 [42 33]] :ValueNotSwapFrom INFO [2019-12-17 12:36:46,564] jepsen worker 1 - jepsen.util 15121 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:36:46,571] jepsen worker 64 - jepsen.util 8524 :invoke :read [21 nil] INFO [2019-12-17 12:36:46,573] jepsen worker 46 - jepsen.util 10756 :invoke :read [15 nil] INFO [2019-12-17 12:36:46,574] jepsen worker 38 - jepsen.util 11648 :invoke :write [12 36] INFO [2019-12-17 12:36:46,574] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64195,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:36:46,574] jepsen worker 74 - jepsen.util 10784 :invoke :cas [24 [21 13]] INFO [2019-12-17 12:36:46,574] jepsen worker 72 - jepsen.util 11322 :invoke :read [24 nil] INFO [2019-12-17 12:36:46,574] jepsen worker 29 - jepsen.util 9299 :invoke :write [9 24] INFO [2019-12-17 12:36:46,574] jepsen worker 1 - jepsen.util 15121 :invoke :read [0 nil] INFO [2019-12-17 12:36:46,575] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64198,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:36:46,576] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64198,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:36:46,577] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64199,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:36:46,578] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64199,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:36:46,583] jepsen worker 27 - jepsen.util 10467 :fail :cas [9 [24 15]] :ValueNotSwapFrom INFO [2019-12-17 12:36:46,584] jepsen worker 27 - jepsen.util 10467 :invoke :write [9 36] INFO [2019-12-17 12:36:46,584] jepsen worker 22 - jepsen.util 15142 :invoke :read [7 nil] INFO [2019-12-17 12:36:46,584] jepsen worker 66 - jepsen.util 15096 :invoke :write [22 22] INFO [2019-12-17 12:36:46,599] jepsen worker 65 - jepsen.util 9605 :fail :cas [21 [11 3]] :ValueNotSwapFrom INFO [2019-12-17 12:36:46,599] jepsen worker 65 - jepsen.util 9605 :invoke :read [21 nil] INFO [2019-12-17 12:36:46,629] jepsen worker 49 - jepsen.util 14989 :info :write [16 37] :RequestCanceledException INFO [2019-12-17 12:36:46,629] jepsen worker 68 - jepsen.util 14468 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:36:46,629] jepsen worker 78 - jepsen.util 11238 :info :write [26 40] :RequestCanceledException INFO [2019-12-17 12:36:46,629] jepsen worker 80 - jepsen.util 10700 :info :write [26 11] :RequestCanceledException INFO [2019-12-17 12:36:46,658] jepsen worker 9 - jepsen.util 11799 :ok :read [3 1] INFO [2019-12-17 12:36:46,662] jepsen worker 63 - jepsen.util 9873 :fail :cas [21 [35 42]] :ValueNotSwapFrom INFO [2019-12-17 12:36:46,673] jepsen worker 46 - jepsen.util 10756 :ok :read [15 35] INFO [2019-12-17 12:36:46,674] jepsen worker 30 - jepsen.util 10470 :fail :cas [10 [19 39]] :ValueNotSwapFrom INFO [2019-12-17 12:36:46,691] jepsen worker 68 - jepsen.util 14468 :invoke :cas [22 [28 45]] INFO [2019-12-17 12:36:46,691] jepsen worker 9 - jepsen.util 11799 :invoke :cas [3 [38 3]] INFO [2019-12-17 12:36:46,691] jepsen worker 46 - jepsen.util 10756 :invoke :cas [15 [36 3]] INFO [2019-12-17 12:36:46,691] jepsen worker 30 - jepsen.util 10470 :invoke :read [10 nil] INFO [2019-12-17 12:36:46,692] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64217,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:36:46,692] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64217,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:36:46,692] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64220,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:36:46,692] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64220,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:36:46,692] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64221,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:36:46,692] jepsen worker 63 - jepsen.util 9873 :invoke :cas [21 [10 41]] INFO [2019-12-17 12:36:46,693] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64221,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:36:46,693] jepsen worker 78 - jepsen.util 11328 :invoke :read [26 nil] INFO [2019-12-17 12:36:46,696] jepsen worker 49 - jepsen.util 15079 :invoke :cas [16 [9 5]] INFO [2019-12-17 12:36:46,703] jepsen worker 80 - jepsen.util 10790 :invoke :write [26 45] INFO [2019-12-17 12:36:46,706] jepsen worker 69 - jepsen.util 11319 :ok :read [23 5] INFO [2019-12-17 12:36:46,706] jepsen worker 69 - jepsen.util 11319 :invoke :write [23 42] INFO [2019-12-17 12:36:46,716] jepsen worker 6 - jepsen.util 15486 :info :write [2 36] :RequestCanceledException INFO [2019-12-17 12:36:46,719] jepsen worker 6 - jepsen.util 15576 :invoke :cas [2 [47 41]] INFO [2019-12-17 12:36:46,728] jepsen worker 35 - jepsen.util 15065 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:36:46,728] jepsen worker 40 - jepsen.util 15430 :info :write [13 30] :RequestCanceledException INFO [2019-12-17 12:36:46,728] jepsen worker 35 - jepsen.util 15065 :invoke :cas [11 [26 22]] INFO [2019-12-17 12:36:46,734] jepsen worker 74 - jepsen.util 10784 :fail :cas [24 [21 13]] :ValueNotSwapFrom INFO [2019-12-17 12:36:46,734] jepsen worker 74 - jepsen.util 10784 :invoke :write [24 35] INFO [2019-12-17 12:36:46,746] jepsen worker 86 - jepsen.util 15836 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:36:46,747] jepsen worker 86 - jepsen.util 15836 :invoke :read [28 nil] INFO [2019-12-17 12:36:46,759] jepsen worker 34 - jepsen.util 15604 :info :cas [11 [33 2]] :RequestCanceledException INFO [2019-12-17 12:36:46,765] jepsen worker 34 - jepsen.util 15694 :invoke :read [11 nil] INFO [2019-12-17 12:36:46,768] jepsen worker 40 - jepsen.util 15520 :invoke :cas [13 [43 1]] INFO [2019-12-17 12:36:46,771] jepsen worker 27 - jepsen.util 10467 :ok :write [9 36] INFO [2019-12-17 12:36:46,772] jepsen worker 27 - jepsen.util 10467 :invoke :write [9 29] INFO [2019-12-17 12:36:46,794] jepsen worker 55 - jepsen.util 11755 :info :write [18 36] :RequestCanceledException INFO [2019-12-17 12:36:46,801] jepsen worker 9 - jepsen.util 11799 :fail :cas [3 [38 3]] :ValueNotSwapFrom INFO [2019-12-17 12:36:46,801] jepsen worker 9 - jepsen.util 11799 :invoke :read [3 nil] INFO [2019-12-17 12:36:46,802] jepsen worker 55 - jepsen.util 11845 :invoke :read [18 nil] INFO [2019-12-17 12:36:46,809] jepsen worker 60 - jepsen.util 16080 :info :write [20 20] :RequestCanceledException INFO [2019-12-17 12:36:46,814] jepsen worker 60 - jepsen.util 16170 :invoke :write [20 24] INFO [2019-12-17 12:36:46,840] jepsen worker 42 - jepsen.util 15792 :info :write [14 39] :RequestCanceledException INFO [2019-12-17 12:36:46,848] jepsen worker 42 - jepsen.util 15882 :invoke :write [14 33] INFO [2019-12-17 12:36:46,869] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64243,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:36:46,869] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64243,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:36:46,870] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64245,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:36:46,870] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64245,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:36:46,875] jepsen worker 46 - jepsen.util 10756 :fail :cas [15 [36 3]] :ValueNotSwapFrom INFO [2019-12-17 12:36:46,875] jepsen worker 46 - jepsen.util 10756 :invoke :write [15 49] INFO [2019-12-17 12:36:46,906] jepsen worker 30 - jepsen.util 10470 :ok :read [10 10] INFO [2019-12-17 12:36:46,907] jepsen worker 30 - jepsen.util 10470 :invoke :read [10 nil] INFO [2019-12-17 12:36:46,956] jepsen worker 64 - jepsen.util 8524 :ok :read [21 26] INFO [2019-12-17 12:36:46,957] jepsen worker 64 - jepsen.util 8524 :invoke :read [21 nil] INFO [2019-12-17 12:36:46,994] jepsen worker 45 - jepsen.util 11205 :fail :read [15 nil] :RequestTimeoutException INFO [2019-12-17 12:36:46,994] jepsen worker 25 - jepsen.util 16225 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:36:46,994] jepsen worker 45 - jepsen.util 11205 :invoke :write [15 22] INFO [2019-12-17 12:36:46,994] jepsen worker 72 - jepsen.util 11322 :ok :read [24 7] INFO [2019-12-17 12:36:46,994] jepsen worker 25 - jepsen.util 16225 :invoke :read [8 nil] INFO [2019-12-17 12:36:46,995] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64253,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:36:46,995] jepsen worker 72 - jepsen.util 11322 :invoke :cas [24 [25 35]] INFO [2019-12-17 12:36:46,995] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64253,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:36:46,995] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64256,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:36:46,995] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64256,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:36:46,995] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64257,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:36:46,995] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64257,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:36:47,026] jepsen worker 27 - jepsen.util 10467 :ok :write [9 29] INFO [2019-12-17 12:36:47,027] jepsen worker 27 - jepsen.util 10467 :invoke :write [9 7] INFO [2019-12-17 12:36:47,052] jepsen worker 9 - jepsen.util 11799 :ok :read [3 1] INFO [2019-12-17 12:36:47,052] jepsen worker 9 - jepsen.util 11799 :invoke :write [3 49] INFO [2019-12-17 12:36:47,094] jepsen worker 64 - jepsen.util 8524 :ok :read [21 26] INFO [2019-12-17 12:36:47,095] jepsen worker 64 - jepsen.util 8524 :invoke :cas [21 [49 23]] INFO [2019-12-17 12:36:47,096] jepsen worker 78 - jepsen.util 11328 :ok :read [26 28] INFO [2019-12-17 12:36:47,096] jepsen worker 78 - jepsen.util 11328 :invoke :read [26 nil] INFO [2019-12-17 12:36:47,123] jepsen worker 11 - jepsen.util 10721 :info :write [3 2] :RequestCanceledException INFO [2019-12-17 12:36:47,159] jepsen worker 11 - jepsen.util 10811 :invoke :read [3 nil] INFO [2019-12-17 12:36:47,160] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:36:47,160] jepsen nemesis - couchbase.util Rebalance check count: 505 INFO [2019-12-17 12:36:47,160] jepsen nemesis - couchbase.util Rebalance stuck count: 486 INFO [2019-12-17 12:36:47,172] jepsen worker 27 - jepsen.util 10467 :ok :write [9 7] INFO [2019-12-17 12:36:47,174] jepsen worker 27 - jepsen.util 10467 :invoke :write [9 12] INFO [2019-12-17 12:36:47,232] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64265,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:36:47,234] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64265,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:36:47,237] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64270,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:36:47,237] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64270,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:36:47,272] jepsen worker 28 - jepsen.util 9208 :info :write [9 13] :RequestCanceledException INFO [2019-12-17 12:36:47,335] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64276,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:36:47,336] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64276,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:36:47,338] jepsen worker 0 - jepsen.util 15930 :info :cas [0 [36 8]] :RequestCanceledException INFO [2019-12-17 12:36:47,346] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64277,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:36:47,350] jepsen worker 79 - jepsen.util 11329 :info :write [26 38] :RequestCanceledException INFO [2019-12-17 12:36:47,353] jepsen worker 56 - jepsen.util 11846 :info :write [18 34] :RequestCanceledException INFO [2019-12-17 12:36:47,367] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64277,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:36:47,386] jepsen worker 82 - jepsen.util 15202 :info :cas [27 [48 43]] :RequestCanceledException INFO [2019-12-17 12:36:47,413] jepsen worker 28 - jepsen.util 9298 :invoke :cas [9 [49 32]] INFO [2019-12-17 12:36:47,424] jepsen worker 85 - jepsen.util 16015 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:36:47,425] jepsen worker 85 - jepsen.util 16015 :invoke :read [28 nil] INFO [2019-12-17 12:36:47,438] jepsen worker 84 - jepsen.util 14844 :info :cas [28 [40 45]] :RequestCanceledException INFO [2019-12-17 12:36:47,446] jepsen worker 79 - jepsen.util 11419 :invoke :read [26 nil] INFO [2019-12-17 12:36:47,450] jepsen worker 0 - jepsen.util 16020 :invoke :read [0 nil] INFO [2019-12-17 12:36:47,452] jepsen worker 56 - jepsen.util 11936 :invoke :cas [18 [11 38]] INFO [2019-12-17 12:36:47,457] jepsen worker 82 - jepsen.util 15292 :invoke :write [27 24] INFO [2019-12-17 12:36:47,479] jepsen worker 67 - jepsen.util 15187 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:36:47,499] jepsen worker 83 - jepsen.util 15293 :info :write [27 36] :RequestCanceledException INFO [2019-12-17 12:36:47,512] jepsen worker 84 - jepsen.util 14934 :invoke :write [28 18] INFO [2019-12-17 12:36:47,538] jepsen worker 61 - jepsen.util 16171 :info :write [20 45] :RequestCanceledException INFO [2019-12-17 12:36:47,539] jepsen worker 71 - jepsen.util 12131 :info :cas [23 [48 24]] :RequestCanceledException INFO [2019-12-17 12:36:47,558] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64283,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:36:47,559] jepsen worker 41 - jepsen.util 16151 :info :write [13 41] :RequestCanceledException INFO [2019-12-17 12:36:47,573] jepsen worker 8 - jepsen.util 14858 :info :write [2 25] :RequestCanceledException INFO [2019-12-17 12:36:47,605] jepsen worker 23 - jepsen.util 15053 :info :write [7 19] :RequestCanceledException INFO [2019-12-17 12:36:47,617] jepsen worker 48 - jepsen.util 14088 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:36:47,634] jepsen worker 50 - jepsen.util 15620 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:36:47,660] jepsen worker 67 - jepsen.util 15187 :invoke :cas [22 [40 0]] INFO [2019-12-17 12:36:47,667] jepsen worker 54 - jepsen.util 11844 :fail :cas [18 [28 2]] :ValueNotSwapFrom INFO [2019-12-17 12:36:47,682] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64283,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:36:47,683] jepsen worker 48 - jepsen.util 14088 :invoke :read [16 nil] INFO [2019-12-17 12:36:47,683] jepsen worker 54 - jepsen.util 11844 :invoke :read [18 nil] INFO [2019-12-17 12:36:47,688] jepsen worker 50 - jepsen.util 15620 :invoke :read [16 nil] INFO [2019-12-17 12:36:47,706] jepsen worker 71 - jepsen.util 12221 :invoke :write [23 7] INFO [2019-12-17 12:36:47,710] jepsen worker 8 - jepsen.util 14948 :invoke :cas [2 [28 47]] INFO [2019-12-17 12:36:47,720] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64291,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:36:47,722] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64291,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:36:47,728] jepsen worker 83 - jepsen.util 15383 :invoke :cas [27 [22 42]] INFO [2019-12-17 12:36:47,728] jepsen worker 41 - jepsen.util 16241 :invoke :write [13 41] INFO [2019-12-17 12:36:47,735] jepsen worker 7 - jepsen.util 15037 :info :cas [2 [26 9]] :RequestCanceledException INFO [2019-12-17 12:36:47,736] jepsen worker 81 - jepsen.util 14751 :info :write [27 11] :RequestCanceledException INFO [2019-12-17 12:36:47,738] jepsen worker 23 - jepsen.util 15143 :invoke :write [7 40] INFO [2019-12-17 12:36:47,744] jepsen worker 61 - jepsen.util 16261 :invoke :cas [20 [13 48]] INFO [2019-12-17 12:36:47,748] jepsen worker 2 - jepsen.util 14672 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:36:47,749] jepsen worker 2 - jepsen.util 14672 :invoke :write [0 21] INFO [2019-12-17 12:36:47,758] jepsen worker 53 - jepsen.util 10583 :fail :cas [17 [3 9]] :ValueNotSwapFrom INFO [2019-12-17 12:36:47,758] jepsen worker 53 - jepsen.util 10583 :invoke :read [17 nil] INFO [2019-12-17 12:36:47,761] jepsen worker 15 - jepsen.util 14145 :info :write [5 16] :RequestCanceledException INFO [2019-12-17 12:36:47,769] jepsen worker 5 - jepsen.util 10895 :ok :read [1 0] INFO [2019-12-17 12:36:47,770] jepsen worker 5 - jepsen.util 10895 :invoke :cas [1 [9 8]] INFO [2019-12-17 12:36:47,774] jepsen worker 81 - jepsen.util 14841 :invoke :write [27 6] INFO [2019-12-17 12:36:47,808] jepsen worker 7 - jepsen.util 15127 :invoke :cas [2 [30 15]] INFO [2019-12-17 12:36:47,831] jepsen worker 62 - jepsen.util 15722 :info :write [20 12] :RequestCanceledException INFO [2019-12-17 12:36:47,833] jepsen worker 43 - jepsen.util 15163 :info :cas [14 [12 24]] :RequestCanceledException INFO [2019-12-17 12:36:47,833] jepsen worker 15 - jepsen.util 14235 :invoke :read [5 nil] INFO [2019-12-17 12:36:47,847] jepsen worker 17 - jepsen.util 15047 :info :cas [5 [49 43]] :RequestCanceledException INFO [2019-12-17 12:36:47,891] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64311,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:36:47,896] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64311,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:36:47,901] jepsen worker 16 - jepsen.util 16576 :info :write [5 49] :RequestCanceledException INFO [2019-12-17 12:36:47,902] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64313,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:36:47,904] jepsen worker 19 - jepsen.util 11359 :ok :write [6 48] INFO [2019-12-17 12:36:47,904] jepsen worker 32 - jepsen.util 11372 :ok :write [10 4] INFO [2019-12-17 12:36:47,905] jepsen worker 74 - jepsen.util 10784 :ok :write [24 35] INFO [2019-12-17 12:36:47,905] jepsen worker 19 - jepsen.util 11359 :invoke :cas [6 [15 13]] INFO [2019-12-17 12:36:47,906] jepsen worker 74 - jepsen.util 10784 :invoke :read [24 nil] INFO [2019-12-17 12:36:47,906] jepsen worker 32 - jepsen.util 11372 :invoke :cas [10 [44 10]] INFO [2019-12-17 12:36:47,909] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64313,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:36:47,917] jepsen worker 58 - jepsen.util 15538 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:36:47,917] jepsen worker 58 - jepsen.util 15538 :invoke :read [19 nil] INFO [2019-12-17 12:36:47,918] jepsen worker 21 - jepsen.util 14691 :info :write [7 30] :RequestCanceledException INFO [2019-12-17 12:36:47,926] jepsen worker 17 - jepsen.util 15137 :invoke :cas [5 [28 38]] INFO [2019-12-17 12:36:47,930] jepsen worker 43 - jepsen.util 15253 :invoke :cas [14 [7 2]] INFO [2019-12-17 12:36:47,937] jepsen worker 62 - jepsen.util 15812 :invoke :cas [20 [9 10]] INFO [2019-12-17 12:36:47,951] jepsen worker 39 - jepsen.util 14889 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:36:47,951] jepsen worker 39 - jepsen.util 14889 :invoke :read [13 nil] INFO [2019-12-17 12:36:47,959] jepsen worker 16 - jepsen.util 16666 :invoke :cas [5 [38 11]] INFO [2019-12-17 12:36:47,963] jepsen worker 21 - jepsen.util 14781 :invoke :cas [7 [42 1]] INFO [2019-12-17 12:36:48,002] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64327,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:36:48,003] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64327,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:36:48,004] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64331,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:36:48,004] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64331,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:36:48,012] jepsen worker 12 - jepsen.util 11442 :info :write [4 25] :RequestCanceledException INFO [2019-12-17 12:36:48,017] jepsen worker 18 - jepsen.util 11268 :fail :read [6 nil] :RequestTimeoutException INFO [2019-12-17 12:36:48,022] jepsen worker 18 - jepsen.util 11268 :invoke :cas [6 [33 18]] INFO [2019-12-17 12:36:48,025] jepsen worker 12 - jepsen.util 11532 :invoke :read [4 nil] INFO [2019-12-17 12:36:48,032] jepsen worker 47 - jepsen.util 11747 :info :write [15 49] :RequestCanceledException INFO [2019-12-17 12:36:48,033] jepsen worker 72 - jepsen.util 11322 :fail :cas [24 [25 35]] :ValueNotSwapFrom INFO [2019-12-17 12:36:48,033] jepsen worker 20 - jepsen.util 10640 :info :cas [6 [13 34]] :RequestCanceledException INFO [2019-12-17 12:36:48,035] jepsen worker 72 - jepsen.util 11322 :invoke :cas [24 [10 27]] INFO [2019-12-17 12:36:48,043] jepsen worker 20 - jepsen.util 10730 :invoke :read [6 nil] INFO [2019-12-17 12:36:48,066] jepsen worker 10 - jepsen.util 10810 :info :cas [3 [13 14]] :RequestCanceledException INFO [2019-12-17 12:36:48,066] jepsen worker 47 - jepsen.util 11837 :invoke :cas [15 [37 29]] INFO [2019-12-17 12:36:48,082] jepsen worker 51 - jepsen.util 10581 :ok :read [17 45] INFO [2019-12-17 12:36:48,084] jepsen worker 51 - jepsen.util 10581 :invoke :write [17 18] INFO [2019-12-17 12:36:48,088] jepsen worker 70 - jepsen.util 11770 :ok :read [23 5] INFO [2019-12-17 12:36:48,089] jepsen worker 70 - jepsen.util 11770 :invoke :cas [23 [2 21]] INFO [2019-12-17 12:36:48,089] jepsen worker 52 - jepsen.util 10672 :ok :read [17 45] INFO [2019-12-17 12:36:48,089] jepsen worker 52 - jepsen.util 10672 :invoke :read [17 nil] INFO [2019-12-17 12:36:48,091] jepsen worker 10 - jepsen.util 10900 :invoke :cas [3 [47 47]] INFO [2019-12-17 12:36:48,093] jepsen worker 89 - jepsen.util 11159 :ok :read [29 5] INFO [2019-12-17 12:36:48,096] jepsen worker 89 - jepsen.util 11159 :invoke :write [29 47] INFO [2019-12-17 12:36:48,097] jepsen worker 37 - jepsen.util 10747 :info :write [12 49] :RequestCanceledException INFO [2019-12-17 12:36:48,105] jepsen worker 37 - jepsen.util 10837 :invoke :read [12 nil] INFO [2019-12-17 12:36:48,106] jepsen worker 13 - jepsen.util 10453 :ok :read [4 23] INFO [2019-12-17 12:36:48,108] jepsen worker 13 - jepsen.util 10453 :invoke :read [4 nil] INFO [2019-12-17 12:36:48,135] jepsen worker 26 - jepsen.util 15866 :info :write [8 5] :RequestCanceledException INFO [2019-12-17 12:36:48,139] jepsen worker 64 - jepsen.util 8524 :fail :cas [21 [49 23]] :ValueNotSwapFrom INFO [2019-12-17 12:36:48,139] jepsen worker 64 - jepsen.util 8524 :invoke :write [21 16] INFO [2019-12-17 12:36:48,155] jepsen worker 3 - jepsen.util 10713 :ok :read [1 0] INFO [2019-12-17 12:36:48,156] jepsen worker 3 - jepsen.util 10713 :invoke :cas [1 [46 39]] INFO [2019-12-17 12:36:48,172] jepsen worker 26 - jepsen.util 15956 :invoke :write [8 0] INFO [2019-12-17 12:36:48,189] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:36:48,190] jepsen nemesis - couchbase.util Rebalance check count: 506 INFO [2019-12-17 12:36:48,190] jepsen nemesis - couchbase.util Rebalance stuck count: 487 INFO [2019-12-17 12:36:48,214] jepsen worker 78 - jepsen.util 11328 :ok :read [26 28] INFO [2019-12-17 12:36:48,214] jepsen worker 78 - jepsen.util 11328 :invoke :write [26 49] INFO [2019-12-17 12:36:48,220] jepsen worker 4 - jepsen.util 10264 :fail :cas [1 [33 10]] :ValueNotSwapFrom INFO [2019-12-17 12:36:48,220] jepsen worker 4 - jepsen.util 10264 :invoke :cas [1 [8 41]] INFO [2019-12-17 12:36:48,224] jepsen worker 33 - jepsen.util 15063 :info :cas [11 [42 42]] :RequestCanceledException INFO [2019-12-17 12:36:48,224] jepsen worker 88 - jepsen.util 11428 :info :write [29 11] :RequestCanceledException INFO [2019-12-17 12:36:48,231] jepsen worker 33 - jepsen.util 15153 :invoke :read [11 nil] INFO [2019-12-17 12:36:48,232] jepsen worker 65 - jepsen.util 9605 :ok :read [21 26] INFO [2019-12-17 12:36:48,232] jepsen worker 65 - jepsen.util 9605 :invoke :write [21 32] INFO [2019-12-17 12:36:48,237] jepsen worker 88 - jepsen.util 11518 :invoke :read [29 nil] INFO [2019-12-17 12:36:48,250] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64350,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:36:48,250] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64350,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:36:48,250] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64352,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:36:48,251] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64352,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:36:48,261] jepsen worker 73 - jepsen.util 10243 :info :cas [24 [11 23]] :RequestCanceledException INFO [2019-12-17 12:36:48,261] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64357,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:36:48,261] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64357,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:36:48,262] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64360,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:36:48,263] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64360,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:36:48,263] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64361,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:36:48,263] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64361,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:36:48,270] jepsen worker 36 - jepsen.util 12006 :ok :read [12 34] INFO [2019-12-17 12:36:48,275] jepsen worker 73 - jepsen.util 10333 :invoke :write [24 8] INFO [2019-12-17 12:36:48,276] jepsen worker 36 - jepsen.util 12006 :invoke :cas [12 [31 18]] INFO [2019-12-17 12:36:48,287] jepsen worker 27 - jepsen.util 10467 :ok :write [9 12] INFO [2019-12-17 12:36:48,287] jepsen worker 27 - jepsen.util 10467 :invoke :cas [9 [43 27]] INFO [2019-12-17 12:36:48,291] jepsen worker 18 - jepsen.util 11268 :fail :cas [6 [33 18]] :ValueNotSwapFrom INFO [2019-12-17 12:36:48,292] jepsen worker 18 - jepsen.util 11268 :invoke :write [6 37] INFO [2019-12-17 12:36:48,298] jepsen worker 24 - jepsen.util 13704 :info :cas [8 [22 36]] :RequestCanceledException INFO [2019-12-17 12:36:48,305] jepsen worker 79 - jepsen.util 11419 :ok :read [26 28] INFO [2019-12-17 12:36:48,305] jepsen worker 79 - jepsen.util 11419 :invoke :cas [26 [3 32]] INFO [2019-12-17 12:36:48,310] jepsen worker 24 - jepsen.util 13794 :invoke :cas [8 [0 10]] INFO [2019-12-17 12:36:48,332] jepsen worker 63 - jepsen.util 9873 :fail :cas [21 [10 41]] :ValueNotSwapFrom INFO [2019-12-17 12:36:48,332] jepsen worker 63 - jepsen.util 9873 :invoke :cas [21 [42 36]] INFO [2019-12-17 12:36:48,338] jepsen worker 19 - jepsen.util 11359 :fail :cas [6 [15 13]] :ValueNotSwapFrom INFO [2019-12-17 12:36:48,338] jepsen worker 19 - jepsen.util 11359 :invoke :read [6 nil] INFO [2019-12-17 12:36:48,346] jepsen worker 57 - jepsen.util 15087 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:36:48,346] jepsen worker 77 - jepsen.util 15827 :info :write [25 37] :RequestCanceledException INFO [2019-12-17 12:36:48,346] jepsen worker 57 - jepsen.util 15087 :invoke :read [19 nil] INFO [2019-12-17 12:36:48,351] jepsen worker 77 - jepsen.util 15917 :invoke :write [25 27] INFO [2019-12-17 12:36:48,362] jepsen worker 75 - jepsen.util 14655 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:36:48,362] jepsen worker 75 - jepsen.util 14655 :invoke :write [25 38] INFO [2019-12-17 12:36:48,362] jepsen worker 28 - jepsen.util 9298 :fail :cas [9 [49 32]] :ValueNotSwapFrom INFO [2019-12-17 12:36:48,363] jepsen worker 28 - jepsen.util 9298 :invoke :read [9 nil] INFO [2019-12-17 12:36:48,365] jepsen worker 31 - jepsen.util 9571 :fail :read [10 nil] :RequestTimeoutException INFO [2019-12-17 12:36:48,367] jepsen worker 31 - jepsen.util 9571 :invoke :write [10 30] INFO [2019-12-17 12:36:48,389] jepsen worker 32 - jepsen.util 11372 :fail :cas [10 [44 10]] :ValueNotSwapFrom INFO [2019-12-17 12:36:48,389] jepsen worker 32 - jepsen.util 11372 :invoke :read [10 nil] INFO [2019-12-17 12:36:48,411] jepsen worker 72 - jepsen.util 11322 :fail :cas [24 [10 27]] :ValueNotSwapFrom INFO [2019-12-17 12:36:48,411] jepsen worker 72 - jepsen.util 11322 :invoke :read [24 nil] INFO [2019-12-17 12:36:48,432] jepsen worker 59 - jepsen.util 14369 :info :cas [19 [11 28]] :RequestCanceledException INFO [2019-12-17 12:36:48,432] jepsen worker 44 - jepsen.util 15614 :info :write [14 49] :RequestCanceledException INFO [2019-12-17 12:36:48,437] jepsen worker 59 - jepsen.util 14459 :invoke :cas [19 [12 39]] INFO [2019-12-17 12:36:48,438] jepsen worker 44 - jepsen.util 15704 :invoke :cas [14 [26 12]] INFO [2019-12-17 12:36:48,448] jepsen worker 20 - jepsen.util 10730 :ok :read [6 48] INFO [2019-12-17 12:36:48,448] jepsen worker 20 - jepsen.util 10730 :invoke :read [6 nil] INFO [2019-12-17 12:36:48,452] jepsen worker 64 - jepsen.util 8524 :ok :write [21 16] INFO [2019-12-17 12:36:48,453] jepsen worker 64 - jepsen.util 8524 :invoke :write [21 48] INFO [2019-12-17 12:36:48,463] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64384,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:36:48,463] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64384,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:36:48,463] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64387,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:36:48,463] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64387,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:36:48,464] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64388,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:36:48,464] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64388,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:36:48,509] jepsen worker 19 - jepsen.util 11359 :ok :read [6 48] INFO [2019-12-17 12:36:48,510] jepsen worker 56 - jepsen.util 11936 :fail :cas [18 [11 38]] :ValueNotSwapFrom INFO [2019-12-17 12:36:48,513] jepsen worker 87 - jepsen.util 10707 :info :write [29 45] :RequestCanceledException INFO [2019-12-17 12:36:48,513] jepsen worker 56 - jepsen.util 11936 :invoke :cas [18 [33 4]] INFO [2019-12-17 12:36:48,513] jepsen worker 19 - jepsen.util 11359 :invoke :write [6 45] INFO [2019-12-17 12:36:48,516] jepsen worker 65 - jepsen.util 9605 :ok :write [21 32] INFO [2019-12-17 12:36:48,521] jepsen worker 65 - jepsen.util 9605 :invoke :write [21 7] INFO [2019-12-17 12:36:48,532] jepsen worker 76 - jepsen.util 15556 :info :cas [25 [46 18]] :RequestCanceledException INFO [2019-12-17 12:36:48,533] jepsen worker 14 - jepsen.util 10904 :info :write [4 17] :RequestCanceledException INFO [2019-12-17 12:36:48,534] jepsen worker 55 - jepsen.util 11845 :ok :read [18 46] INFO [2019-12-17 12:36:48,534] jepsen worker 55 - jepsen.util 11845 :invoke :read [18 nil] INFO [2019-12-17 12:36:48,550] jepsen worker 87 - jepsen.util 10797 :invoke :cas [29 [41 13]] INFO [2019-12-17 12:36:48,555] jepsen worker 76 - jepsen.util 15646 :invoke :read [25 nil] INFO [2019-12-17 12:36:48,560] jepsen worker 63 - jepsen.util 9873 :fail :cas [21 [42 36]] :ValueNotSwapFrom INFO [2019-12-17 12:36:48,561] jepsen worker 63 - jepsen.util 9873 :invoke :write [21 20] INFO [2019-12-17 12:36:48,562] jepsen worker 14 - jepsen.util 10994 :invoke :read [4 nil] INFO [2019-12-17 12:36:48,591] jepsen worker 27 - jepsen.util 10467 :fail :cas [9 [43 27]] :ValueNotSwapFrom INFO [2019-12-17 12:36:48,591] jepsen worker 27 - jepsen.util 10467 :invoke :read [9 nil] INFO [2019-12-17 12:36:48,607] jepsen worker 54 - jepsen.util 11844 :ok :read [18 46] INFO [2019-12-17 12:36:48,607] jepsen worker 54 - jepsen.util 11844 :invoke :cas [18 [11 10]] INFO [2019-12-17 12:36:48,619] jepsen worker 70 - jepsen.util 11770 :fail :cas [23 [2 21]] :ValueNotSwapFrom INFO [2019-12-17 12:36:48,620] jepsen worker 70 - jepsen.util 11770 :invoke :write [23 3] INFO [2019-12-17 12:36:48,632] jepsen worker 66 - jepsen.util 15096 :info :write [22 22] :RequestCanceledException INFO [2019-12-17 12:36:48,646] jepsen worker 29 - jepsen.util 9299 :info :write [9 24] :RequestCanceledException INFO [2019-12-17 12:36:48,667] jepsen worker 1 - jepsen.util 15121 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:36:48,667] jepsen worker 1 - jepsen.util 15121 :invoke :read [0 nil] INFO [2019-12-17 12:36:48,669] jepsen worker 22 - jepsen.util 15142 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:36:48,675] jepsen worker 22 - jepsen.util 15142 :invoke :cas [7 [0 4]] INFO [2019-12-17 12:36:48,676] jepsen worker 29 - jepsen.util 9389 :invoke :cas [9 [22 14]] INFO [2019-12-17 12:36:48,682] jepsen worker 66 - jepsen.util 15186 :invoke :read [22 nil] INFO [2019-12-17 12:36:48,682] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64404,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:36:48,682] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64404,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:36:48,683] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64408,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:36:48,685] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64408,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:36:48,698] jepsen worker 38 - jepsen.util 11648 :info :write [12 36] :RequestCanceledException INFO [2019-12-17 12:36:48,701] jepsen worker 38 - jepsen.util 11738 :invoke :read [12 nil] INFO [2019-12-17 12:36:48,717] jepsen worker 49 - jepsen.util 15079 :info :cas [16 [9 5]] :RequestCanceledException INFO [2019-12-17 12:36:48,724] jepsen worker 68 - jepsen.util 14468 :info :cas [22 [28 45]] :RequestCanceledException INFO [2019-12-17 12:36:48,724] jepsen worker 80 - jepsen.util 10790 :info :write [26 45] :RequestCanceledException INFO [2019-12-17 12:36:48,726] jepsen worker 49 - jepsen.util 15169 :invoke :write [16 29] INFO [2019-12-17 12:36:48,726] jepsen worker 80 - jepsen.util 10880 :invoke :write [26 41] INFO [2019-12-17 12:36:48,732] jepsen worker 68 - jepsen.util 14558 :invoke :cas [22 [18 19]] INFO [2019-12-17 12:36:48,746] jepsen worker 6 - jepsen.util 15576 :info :cas [2 [47 41]] :RequestCanceledException INFO [2019-12-17 12:36:48,749] jepsen worker 13 - jepsen.util 10453 :ok :read [4 23] INFO [2019-12-17 12:36:48,749] jepsen worker 13 - jepsen.util 10453 :invoke :cas [4 [15 44]] INFO [2019-12-17 12:36:48,752] jepsen worker 6 - jepsen.util 15666 :invoke :write [2 4] INFO [2019-12-17 12:36:48,754] jepsen worker 14 - jepsen.util 10994 :ok :read [4 23] INFO [2019-12-17 12:36:48,754] jepsen worker 14 - jepsen.util 10994 :invoke :read [4 nil] INFO [2019-12-17 12:36:48,764] jepsen worker 69 - jepsen.util 11319 :info :write [23 42] :RequestCanceledException INFO [2019-12-17 12:36:48,772] jepsen worker 69 - jepsen.util 11409 :invoke :read [23 nil] INFO [2019-12-17 12:36:48,782] jepsen worker 64 - jepsen.util 8524 :ok :write [21 48] INFO [2019-12-17 12:36:48,783] jepsen worker 64 - jepsen.util 8524 :invoke :cas [21 [41 10]] INFO [2019-12-17 12:36:48,798] jepsen worker 4 - jepsen.util 10264 :fail :cas [1 [8 41]] :ValueNotSwapFrom INFO [2019-12-17 12:36:48,799] jepsen worker 4 - jepsen.util 10264 :invoke :write [1 22] INFO [2019-12-17 12:36:48,808] jepsen worker 28 - jepsen.util 9298 :ok :read [9 12] INFO [2019-12-17 12:36:48,808] jepsen worker 28 - jepsen.util 9298 :invoke :read [9 nil] INFO [2019-12-17 12:36:48,808] jepsen worker 53 - jepsen.util 10583 :ok :read [17 45] INFO [2019-12-17 12:36:48,808] jepsen worker 53 - jepsen.util 10583 :invoke :cas [17 [9 13]] INFO [2019-12-17 12:36:48,836] jepsen worker 35 - jepsen.util 15065 :info :cas [11 [26 22]] :RequestCanceledException INFO [2019-12-17 12:36:48,836] jepsen worker 40 - jepsen.util 15520 :info :cas [13 [43 1]] :RequestCanceledException INFO [2019-12-17 12:36:48,845] jepsen worker 86 - jepsen.util 15836 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:36:48,845] jepsen worker 86 - jepsen.util 15836 :invoke :cas [28 [37 28]] INFO [2019-12-17 12:36:48,852] jepsen worker 40 - jepsen.util 15610 :invoke :write [13 5] INFO [2019-12-17 12:36:48,853] jepsen worker 35 - jepsen.util 15155 :invoke :cas [11 [48 35]] INFO [2019-12-17 12:36:48,857] jepsen worker 32 - jepsen.util 11372 :ok :read [10 4] INFO [2019-12-17 12:36:48,857] jepsen worker 32 - jepsen.util 11372 :invoke :read [10 nil] INFO [2019-12-17 12:36:48,862] jepsen worker 34 - jepsen.util 15694 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:36:48,862] jepsen worker 34 - jepsen.util 15694 :invoke :read [11 nil] INFO [2019-12-17 12:36:48,866] jepsen worker 5 - jepsen.util 10895 :fail :cas [1 [9 8]] :ValueNotSwapFrom INFO [2019-12-17 12:36:48,867] jepsen worker 5 - jepsen.util 10895 :invoke :read [1 nil] INFO [2019-12-17 12:36:48,876] jepsen worker 79 - jepsen.util 11419 :fail :cas [26 [3 32]] :ValueNotSwapFrom INFO [2019-12-17 12:36:48,877] jepsen worker 79 - jepsen.util 11419 :invoke :cas [26 [16 39]] INFO [2019-12-17 12:36:48,888] jepsen worker 65 - jepsen.util 9605 :ok :write [21 7] INFO [2019-12-17 12:36:48,888] jepsen worker 65 - jepsen.util 9605 :invoke :write [21 8] INFO [2019-12-17 12:36:48,893] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64429,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:36:48,894] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64429,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:36:48,894] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64432,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:36:48,900] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64432,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:36:48,907] jepsen worker 64 - jepsen.util 8524 :fail :cas [21 [41 10]] :ValueNotSwapFrom INFO [2019-12-17 12:36:48,907] jepsen worker 64 - jepsen.util 8524 :invoke :read [21 nil] INFO [2019-12-17 12:36:48,917] jepsen worker 60 - jepsen.util 16170 :info :write [20 24] :RequestCanceledException INFO [2019-12-17 12:36:48,927] jepsen worker 14 - jepsen.util 10994 :ok :read [4 17] INFO [2019-12-17 12:36:48,927] jepsen worker 14 - jepsen.util 10994 :invoke :read [4 nil] INFO [2019-12-17 12:36:48,933] jepsen worker 42 - jepsen.util 15882 :info :write [14 33] :RequestCanceledException INFO [2019-12-17 12:36:48,941] jepsen worker 60 - jepsen.util 16260 :invoke :write [20 47] INFO [2019-12-17 12:36:48,948] jepsen worker 46 - jepsen.util 10756 :info :write [15 49] :RequestCanceledException INFO [2019-12-17 12:36:48,950] jepsen worker 30 - jepsen.util 10470 :fail :read [10 nil] :RequestTimeoutException INFO [2019-12-17 12:36:48,950] jepsen worker 30 - jepsen.util 10470 :invoke :cas [10 [20 24]] INFO [2019-12-17 12:36:48,977] jepsen worker 42 - jepsen.util 15972 :invoke :write [14 25] INFO [2019-12-17 12:36:49,009] jepsen worker 46 - jepsen.util 10846 :invoke :write [15 6] INFO [2019-12-17 12:36:49,012] jepsen worker 64 - jepsen.util 8524 :ok :read [21 7] INFO [2019-12-17 12:36:49,012] jepsen worker 64 - jepsen.util 8524 :invoke :read [21 nil] INFO [2019-12-17 12:36:49,060] jepsen worker 18 - jepsen.util 11268 :ok :write [6 37] INFO [2019-12-17 12:36:49,061] jepsen worker 18 - jepsen.util 11268 :invoke :read [6 nil] INFO [2019-12-17 12:36:49,063] jepsen worker 12 - jepsen.util 11532 :ok :read [4 17] INFO [2019-12-17 12:36:49,064] jepsen worker 12 - jepsen.util 11532 :invoke :read [4 nil] INFO [2019-12-17 12:36:49,066] jepsen worker 14 - jepsen.util 10994 :ok :read [4 17] INFO [2019-12-17 12:36:49,066] jepsen worker 14 - jepsen.util 10994 :invoke :write [4 25] INFO [2019-12-17 12:36:49,072] jepsen worker 32 - jepsen.util 11372 :ok :read [10 4] INFO [2019-12-17 12:36:49,072] jepsen worker 32 - jepsen.util 11372 :invoke :write [10 17] INFO [2019-12-17 12:36:49,086] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64445,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:36:49,086] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64445,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:36:49,086] jepsen worker 70 - jepsen.util 11770 :ok :write [23 3] INFO [2019-12-17 12:36:49,086] jepsen worker 70 - jepsen.util 11770 :invoke :write [23 37] INFO [2019-12-17 12:36:49,088] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64449,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:36:49,088] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64449,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:36:49,099] jepsen worker 25 - jepsen.util 16225 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:36:49,099] jepsen worker 25 - jepsen.util 16225 :invoke :write [8 8] INFO [2019-12-17 12:36:49,102] jepsen worker 45 - jepsen.util 11205 :info :write [15 22] :RequestCanceledException INFO [2019-12-17 12:36:49,107] jepsen worker 45 - jepsen.util 11295 :invoke :write [15 21] INFO [2019-12-17 12:36:49,124] jepsen worker 65 - jepsen.util 9605 :ok :write [21 8] INFO [2019-12-17 12:36:49,124] jepsen worker 65 - jepsen.util 9605 :invoke :write [21 7] INFO [2019-12-17 12:36:49,145] jepsen worker 64 - jepsen.util 8524 :ok :read [21 7] INFO [2019-12-17 12:36:49,145] jepsen worker 64 - jepsen.util 8524 :invoke :write [21 40] INFO [2019-12-17 12:36:49,174] jepsen worker 78 - jepsen.util 11328 :ok :write [26 49] INFO [2019-12-17 12:36:49,175] jepsen worker 78 - jepsen.util 11328 :invoke :cas [26 [15 42]] INFO [2019-12-17 12:36:49,194] jepsen worker 73 - jepsen.util 10333 :ok :write [24 8] INFO [2019-12-17 12:36:49,195] jepsen worker 73 - jepsen.util 10333 :invoke :cas [24 [43 47]] INFO [2019-12-17 12:36:49,208] jepsen worker 69 - jepsen.util 11409 :ok :read [23 3] INFO [2019-12-17 12:36:49,208] jepsen worker 69 - jepsen.util 11409 :invoke :cas [23 [23 11]] INFO [2019-12-17 12:36:49,219] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:36:49,219] jepsen nemesis - couchbase.util Rebalance check count: 507 INFO [2019-12-17 12:36:49,219] jepsen nemesis - couchbase.util Rebalance stuck count: 488 INFO [2019-12-17 12:36:49,224] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64465,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:36:49,224] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64465,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:36:49,246] jepsen worker 11 - jepsen.util 10811 :fail :read [3 nil] :RequestTimeoutException INFO [2019-12-17 12:36:49,246] jepsen worker 9 - jepsen.util 11799 :info :write [3 49] :RequestCanceledException INFO [2019-12-17 12:36:49,247] jepsen worker 11 - jepsen.util 10811 :invoke :write [3 49] INFO [2019-12-17 12:36:49,247] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64472,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:36:49,248] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64472,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:36:49,248] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64475,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:36:49,248] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64475,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:36:49,254] jepsen worker 9 - jepsen.util 11889 :invoke :read [3 nil] INFO [2019-12-17 12:36:49,268] jepsen worker 72 - jepsen.util 11322 :ok :read [24 35] INFO [2019-12-17 12:36:49,268] jepsen worker 72 - jepsen.util 11322 :invoke :write [24 15] INFO [2019-12-17 12:36:49,277] jepsen worker 18 - jepsen.util 11268 :ok :read [6 37] INFO [2019-12-17 12:36:49,277] jepsen worker 18 - jepsen.util 11268 :invoke :write [6 9] INFO [2019-12-17 12:36:49,301] jepsen worker 37 - jepsen.util 10837 :ok :read [12 34] INFO [2019-12-17 12:36:49,302] jepsen worker 37 - jepsen.util 10837 :invoke :write [12 3] INFO [2019-12-17 12:36:49,314] jepsen worker 27 - jepsen.util 10467 :ok :read [9 12] INFO [2019-12-17 12:36:49,315] jepsen worker 27 - jepsen.util 10467 :invoke :read [9 nil] INFO [2019-12-17 12:36:49,331] jepsen worker 73 - jepsen.util 10333 :fail :cas [24 [43 47]] :ValueNotSwapFrom INFO [2019-12-17 12:36:49,331] jepsen worker 73 - jepsen.util 10333 :invoke :cas [24 [34 1]] INFO [2019-12-17 12:36:49,364] jepsen worker 13 - jepsen.util 10453 :fail :cas [4 [15 44]] :ValueNotSwapFrom INFO [2019-12-17 12:36:49,364] jepsen worker 13 - jepsen.util 10453 :invoke :write [4 26] INFO [2019-12-17 12:36:49,439] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64488,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:36:49,440] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64488,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:36:49,477] jepsen worker 65 - jepsen.util 9605 :ok :write [21 7] INFO [2019-12-17 12:36:49,478] jepsen worker 65 - jepsen.util 9605 :invoke :read [21 nil] INFO [2019-12-17 12:36:49,500] jepsen worker 0 - jepsen.util 16020 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:36:49,500] jepsen worker 0 - jepsen.util 16020 :invoke :read [0 nil] INFO [2019-12-17 12:36:49,510] jepsen worker 54 - jepsen.util 11844 :fail :cas [18 [11 10]] :ValueNotSwapFrom INFO [2019-12-17 12:36:49,510] jepsen worker 54 - jepsen.util 11844 :invoke :read [18 nil] INFO [2019-12-17 12:36:49,516] jepsen worker 85 - jepsen.util 16015 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:36:49,516] jepsen worker 85 - jepsen.util 16015 :invoke :write [28 13] INFO [2019-12-17 12:36:49,555] jepsen worker 55 - jepsen.util 11845 :ok :read [18 46] INFO [2019-12-17 12:36:49,555] jepsen worker 55 - jepsen.util 11845 :invoke :write [18 6] INFO [2019-12-17 12:36:49,594] jepsen worker 38 - jepsen.util 11738 :ok :read [12 34] INFO [2019-12-17 12:36:49,594] jepsen worker 38 - jepsen.util 11738 :invoke :read [12 nil] INFO [2019-12-17 12:36:49,605] jepsen worker 12 - jepsen.util 11532 :ok :read [4 25] INFO [2019-12-17 12:36:49,605] jepsen worker 12 - jepsen.util 11532 :invoke :read [4 nil] INFO [2019-12-17 12:36:49,621] jepsen worker 65 - jepsen.util 9605 :ok :read [21 7] INFO [2019-12-17 12:36:49,622] jepsen worker 65 - jepsen.util 9605 :invoke :write [21 37] INFO [2019-12-17 12:36:49,637] jepsen worker 53 - jepsen.util 10583 :fail :cas [17 [9 13]] :ValueNotSwapFrom INFO [2019-12-17 12:36:49,637] jepsen worker 53 - jepsen.util 10583 :invoke :cas [17 [34 11]] INFO [2019-12-17 12:36:49,637] jepsen worker 79 - jepsen.util 11419 :fail :cas [26 [16 39]] :ValueNotSwapFrom INFO [2019-12-17 12:36:49,637] jepsen worker 79 - jepsen.util 11419 :invoke :read [26 nil] INFO [2019-12-17 12:36:49,666] jepsen worker 73 - jepsen.util 10333 :fail :cas [24 [34 1]] :ValueNotSwapFrom INFO [2019-12-17 12:36:49,667] jepsen worker 73 - jepsen.util 10333 :invoke :write [24 47] INFO [2019-12-17 12:36:49,671] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64501,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:36:49,671] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64501,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:36:49,671] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64504,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:36:49,671] jepsen worker 27 - jepsen.util 10467 :ok :read [9 12] INFO [2019-12-17 12:36:49,671] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64504,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:36:49,671] jepsen worker 27 - jepsen.util 10467 :invoke :write [9 39] INFO [2019-12-17 12:36:49,689] jepsen worker 78 - jepsen.util 11328 :fail :cas [26 [15 42]] :ValueNotSwapFrom INFO [2019-12-17 12:36:49,689] jepsen worker 78 - jepsen.util 11328 :invoke :cas [26 [12 36]] INFO [2019-12-17 12:36:49,701] jepsen worker 5 - jepsen.util 10895 :ok :read [1 0] INFO [2019-12-17 12:36:49,702] jepsen worker 5 - jepsen.util 10895 :invoke :cas [1 [29 29]] INFO [2019-12-17 12:36:49,718] jepsen worker 48 - jepsen.util 14088 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:36:49,726] jepsen worker 50 - jepsen.util 15620 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:36:49,726] jepsen worker 8 - jepsen.util 14948 :info :cas [2 [28 47]] :RequestCanceledException INFO [2019-12-17 12:36:49,732] jepsen worker 69 - jepsen.util 11409 :fail :cas [23 [23 11]] :ValueNotSwapFrom INFO [2019-12-17 12:36:49,751] jepsen worker 65 - jepsen.util 9605 :ok :write [21 37] INFO [2019-12-17 12:36:49,764] jepsen worker 67 - jepsen.util 15187 :info :cas [22 [40 0]] :RequestCanceledException INFO [2019-12-17 12:36:49,765] jepsen worker 31 - jepsen.util 9571 :ok :write [10 30] INFO [2019-12-17 12:36:49,765] jepsen worker 45 - jepsen.util 11295 :ok :write [15 21] INFO [2019-12-17 12:36:49,797] jepsen worker 83 - jepsen.util 15383 :info :cas [27 [22 42]] :RequestCanceledException INFO [2019-12-17 12:36:49,833] jepsen worker 82 - jepsen.util 15292 :info :write [27 24] :RequestCanceledException INFO [2019-12-17 12:36:49,833] jepsen worker 7 - jepsen.util 15127 :info :cas [2 [30 15]] :RequestCanceledException INFO [2019-12-17 12:36:49,845] jepsen worker 48 - jepsen.util 14088 :invoke :read [16 nil] INFO [2019-12-17 12:36:49,845] jepsen worker 50 - jepsen.util 15620 :invoke :write [16 12] INFO [2019-12-17 12:36:49,845] jepsen worker 65 - jepsen.util 9605 :invoke :write [21 37] INFO [2019-12-17 12:36:49,845] jepsen worker 45 - jepsen.util 11295 :invoke :read [15 nil] INFO [2019-12-17 12:36:49,845] jepsen worker 31 - jepsen.util 9571 :invoke :read [10 nil] INFO [2019-12-17 12:36:49,845] jepsen worker 69 - jepsen.util 11409 :invoke :cas [23 [22 19]] INFO [2019-12-17 12:36:49,846] jepsen worker 61 - jepsen.util 16261 :info :cas [20 [13 48]] :RequestCanceledException INFO [2019-12-17 12:36:49,867] jepsen worker 7 - jepsen.util 15217 :invoke :cas [2 [7 16]] INFO [2019-12-17 12:36:49,869] jepsen worker 61 - jepsen.util 16351 :invoke :cas [20 [1 36]] INFO [2019-12-17 12:36:49,874] jepsen worker 67 - jepsen.util 15277 :invoke :cas [22 [5 44]] INFO [2019-12-17 12:36:49,874] jepsen worker 83 - jepsen.util 15473 :invoke :write [27 1] INFO [2019-12-17 12:36:49,875] jepsen worker 8 - jepsen.util 15038 :invoke :cas [2 [27 11]] INFO [2019-12-17 12:36:49,875] jepsen worker 82 - jepsen.util 15382 :invoke :write [27 29] INFO [2019-12-17 12:36:49,883] jepsen worker 64 - jepsen.util 8524 :ok :write [21 40] INFO [2019-12-17 12:36:49,884] jepsen worker 64 - jepsen.util 8524 :invoke :write [21 35] INFO [2019-12-17 12:36:49,888] jepsen worker 18 - jepsen.util 11268 :ok :write [6 9] INFO [2019-12-17 12:36:49,888] jepsen worker 18 - jepsen.util 11268 :invoke :read [6 nil] INFO [2019-12-17 12:36:49,891] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64515,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:36:49,893] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64515,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:36:49,896] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64520,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:36:49,897] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64520,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:36:49,899] jepsen worker 15 - jepsen.util 14235 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:36:49,901] jepsen worker 15 - jepsen.util 14235 :invoke :cas [5 [9 8]] INFO [2019-12-17 12:36:49,966] jepsen worker 71 - jepsen.util 12221 :info :write [23 7] :RequestCanceledException INFO [2019-12-17 12:36:49,994] jepsen worker 71 - jepsen.util 12311 :invoke :read [23 nil] INFO [2019-12-17 12:36:50,020] jepsen worker 17 - jepsen.util 15137 :info :cas [5 [28 38]] :RequestCanceledException INFO [2019-12-17 12:36:50,023] jepsen worker 58 - jepsen.util 15538 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:36:50,024] jepsen worker 58 - jepsen.util 15538 :invoke :read [19 nil] INFO [2019-12-17 12:36:50,032] jepsen worker 27 - jepsen.util 10467 :ok :write [9 39] INFO [2019-12-17 12:36:50,035] jepsen worker 27 - jepsen.util 10467 :invoke :read [9 nil] INFO [2019-12-17 12:36:50,036] jepsen worker 21 - jepsen.util 14781 :info :cas [7 [42 1]] :RequestCanceledException INFO [2019-12-17 12:36:50,036] jepsen worker 23 - jepsen.util 15143 :info :write [7 40] :RequestCanceledException INFO [2019-12-17 12:36:50,036] jepsen worker 43 - jepsen.util 15253 :info :cas [14 [7 2]] :RequestCanceledException INFO [2019-12-17 12:36:50,054] jepsen worker 39 - jepsen.util 14889 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:36:50,055] jepsen worker 84 - jepsen.util 14934 :info :write [28 18] :RequestCanceledException INFO [2019-12-17 12:36:50,059] jepsen worker 39 - jepsen.util 14889 :invoke :write [13 12] INFO [2019-12-17 12:36:50,059] jepsen worker 2 - jepsen.util 14672 :info :write [0 21] :RequestCanceledException INFO [2019-12-17 12:36:50,060] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64533,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:36:50,060] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64533,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:36:50,060] jepsen worker 62 - jepsen.util 15812 :info :cas [20 [9 10]] :RequestCanceledException INFO [2019-12-17 12:36:50,066] jepsen worker 16 - jepsen.util 16666 :info :cas [5 [38 11]] :RequestCanceledException INFO [2019-12-17 12:36:50,081] jepsen worker 17 - jepsen.util 15227 :invoke :cas [5 [46 44]] INFO [2019-12-17 12:36:50,093] jepsen worker 43 - jepsen.util 15343 :invoke :cas [14 [14 12]] INFO [2019-12-17 12:36:50,099] jepsen worker 21 - jepsen.util 14871 :invoke :read [7 nil] INFO [2019-12-17 12:36:50,099] jepsen worker 23 - jepsen.util 15233 :invoke :cas [7 [48 11]] INFO [2019-12-17 12:36:50,102] jepsen worker 84 - jepsen.util 15024 :invoke :read [28 nil] INFO [2019-12-17 12:36:50,106] jepsen worker 2 - jepsen.util 14762 :invoke :cas [0 [19 39]] INFO [2019-12-17 12:36:50,112] jepsen worker 16 - jepsen.util 16756 :invoke :write [5 38] INFO [2019-12-17 12:36:50,116] jepsen worker 41 - jepsen.util 16241 :info :write [13 41] :RequestCanceledException INFO [2019-12-17 12:36:50,116] jepsen worker 62 - jepsen.util 15902 :invoke :cas [20 [46 26]] INFO [2019-12-17 12:36:50,116] jepsen worker 52 - jepsen.util 10672 :fail :read [17 nil] :RequestTimeoutException INFO [2019-12-17 12:36:50,119] jepsen worker 52 - jepsen.util 10672 :invoke :read [17 nil] INFO [2019-12-17 12:36:50,134] jepsen worker 47 - jepsen.util 11837 :info :cas [15 [37 29]] :RequestCanceledException INFO [2019-12-17 12:36:50,142] jepsen worker 46 - jepsen.util 10846 :ok :write [15 6] INFO [2019-12-17 12:36:50,143] jepsen worker 11 - jepsen.util 10811 :ok :write [3 49] INFO [2019-12-17 12:36:50,143] jepsen worker 11 - jepsen.util 10811 :invoke :write [3 37] INFO [2019-12-17 12:36:50,144] jepsen worker 46 - jepsen.util 10846 :invoke :read [15 nil] INFO [2019-12-17 12:36:50,884] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64549,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:36:50,884] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64549,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:36:50,884] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64550,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:36:50,884] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64550,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:36:50,884] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64554,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:36:50,884] jepsen worker 76 - jepsen.util 15646 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:36:50,884] jepsen worker 26 - jepsen.util 15956 :info :write [8 0] :RequestCanceledException INFO [2019-12-17 12:36:50,885] jepsen worker 76 - jepsen.util 15646 :invoke :cas [25 [33 37]] INFO [2019-12-17 12:36:50,885] jepsen worker 59 - jepsen.util 14459 :info :cas [19 [12 39]] :RequestCanceledException INFO [2019-12-17 12:36:50,885] jepsen worker 66 - jepsen.util 15186 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:36:50,885] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64554,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:36:50,885] jepsen worker 66 - jepsen.util 15186 :invoke :write [22 21] INFO [2019-12-17 12:36:50,885] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64551,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:36:50,885] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64551,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:36:50,886] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64556,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:36:50,886] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64556,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:36:50,886] jepsen worker 89 - jepsen.util 11159 :info :write [29 47] :RequestCanceledException INFO [2019-12-17 12:36:50,886] jepsen worker 88 - jepsen.util 11518 :fail :read [29 nil] :RequestTimeoutException INFO [2019-12-17 12:36:50,887] jepsen worker 88 - jepsen.util 11518 :invoke :write [29 23] INFO [2019-12-17 12:36:50,888] jepsen worker 36 - jepsen.util 12006 :info :cas [12 [31 18]] :RequestCanceledException INFO [2019-12-17 12:36:50,888] jepsen worker 51 - jepsen.util 10581 :info :write [17 18] :RequestCanceledException INFO [2019-12-17 12:36:50,890] jepsen worker 56 - jepsen.util 11936 :info :cas [18 [33 4]] :RequestCanceledException INFO [2019-12-17 12:36:50,890] jepsen worker 49 - jepsen.util 15169 :info :write [16 29] :RequestCanceledException INFO [2019-12-17 12:36:50,890] jepsen worker 68 - jepsen.util 14558 :info :cas [22 [18 19]] :RequestCanceledException INFO [2019-12-17 12:36:50,891] jepsen worker 57 - jepsen.util 15087 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:36:50,891] jepsen worker 20 - jepsen.util 10730 :fail :read [6 nil] :RequestTimeoutException INFO [2019-12-17 12:36:50,891] jepsen worker 87 - jepsen.util 10797 :info :cas [29 [41 13]] :RequestCanceledException INFO [2019-12-17 12:36:50,891] jepsen worker 24 - jepsen.util 13794 :info :cas [8 [0 10]] :RequestCanceledException INFO [2019-12-17 12:36:50,891] jepsen worker 35 - jepsen.util 15155 :info :cas [11 [48 35]] :RequestCanceledException INFO [2019-12-17 12:36:50,892] jepsen worker 3 - jepsen.util 10713 :fail :cas [1 [46 39]] :ValueNotSwapFrom INFO [2019-12-17 12:36:50,892] jepsen worker 57 - jepsen.util 15087 :invoke :read [19 nil] INFO [2019-12-17 12:36:50,894] jepsen worker 20 - jepsen.util 10730 :invoke :cas [6 [24 44]] INFO [2019-12-17 12:36:50,894] jepsen worker 86 - jepsen.util 15836 :info :cas [28 [37 28]] :RequestCanceledException INFO [2019-12-17 12:36:50,895] jepsen worker 54 - jepsen.util 11844 :ok :read [18 46] INFO [2019-12-17 12:36:50,896] jepsen worker 3 - jepsen.util 10713 :invoke :write [1 34] INFO [2019-12-17 12:36:50,897] jepsen worker 72 - jepsen.util 11322 :ok :write [24 15] INFO [2019-12-17 12:36:50,897] jepsen worker 33 - jepsen.util 15153 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:36:50,898] jepsen worker 54 - jepsen.util 11844 :invoke :read [18 nil] INFO [2019-12-17 12:36:50,898] jepsen worker 81 - jepsen.util 14841 :info :write [27 6] :RequestCanceledException INFO [2019-12-17 12:36:50,898] jepsen worker 33 - jepsen.util 15153 :invoke :write [11 5] INFO [2019-12-17 12:36:50,899] jepsen worker 72 - jepsen.util 11322 :invoke :read [24 nil] INFO [2019-12-17 12:36:50,900] jepsen worker 44 - jepsen.util 15704 :info :cas [14 [26 12]] :RequestCanceledException INFO [2019-12-17 12:36:50,900] jepsen worker 34 - jepsen.util 15694 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:36:50,900] jepsen worker 40 - jepsen.util 15610 :info :write [13 5] :RequestCanceledException INFO [2019-12-17 12:36:50,900] jepsen worker 77 - jepsen.util 15917 :info :write [25 27] :RequestCanceledException INFO [2019-12-17 12:36:50,900] jepsen worker 26 - jepsen.util 16046 :invoke :read [8 nil] INFO [2019-12-17 12:36:50,901] jepsen worker 34 - jepsen.util 15694 :invoke :write [11 5] INFO [2019-12-17 12:36:50,903] jepsen worker 75 - jepsen.util 14655 :info :write [25 38] :RequestCanceledException INFO [2019-12-17 12:36:50,903] jepsen worker 59 - jepsen.util 14549 :invoke :cas [19 [44 29]] INFO [2019-12-17 12:36:50,905] jepsen worker 89 - jepsen.util 11249 :invoke :write [29 42] INFO [2019-12-17 12:36:50,905] jepsen worker 63 - jepsen.util 9873 :info :write [21 20] :RequestCanceledException INFO [2019-12-17 12:36:50,905] jepsen worker 19 - jepsen.util 11359 :info :write [6 45] :RequestCanceledException INFO [2019-12-17 12:36:50,907] jepsen worker 1 - jepsen.util 15121 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:36:50,907] jepsen worker 1 - jepsen.util 15121 :invoke :read [0 nil] INFO [2019-12-17 12:36:50,908] jepsen worker 41 - jepsen.util 16331 :invoke :read [13 nil] INFO [2019-12-17 12:36:50,909] jepsen worker 29 - jepsen.util 9389 :info :cas [9 [22 14]] :RequestCanceledException INFO [2019-12-17 12:36:50,909] jepsen worker 47 - jepsen.util 11927 :invoke :cas [15 [3 26]] INFO [2019-12-17 12:36:50,909] jepsen worker 22 - jepsen.util 15142 :info :cas [7 [0 4]] :RequestCanceledException INFO [2019-12-17 12:36:50,912] jepsen worker 6 - jepsen.util 15666 :info :write [2 4] :RequestCanceledException INFO [2019-12-17 12:36:50,915] jepsen worker 49 - jepsen.util 15259 :invoke :write [16 4] INFO [2019-12-17 12:36:50,915] jepsen worker 68 - jepsen.util 14648 :invoke :cas [22 [22 3]] INFO [2019-12-17 12:36:50,916] jepsen worker 35 - jepsen.util 15245 :invoke :read [11 nil] INFO [2019-12-17 12:36:50,916] jepsen worker 56 - jepsen.util 12026 :invoke :cas [18 [23 5]] INFO [2019-12-17 12:36:50,922] jepsen worker 36 - jepsen.util 12096 :invoke :write [12 22] INFO [2019-12-17 12:36:50,925] jepsen worker 87 - jepsen.util 10887 :invoke :read [29 nil] INFO [2019-12-17 12:36:50,925] jepsen worker 51 - jepsen.util 10671 :invoke :write [17 40] INFO [2019-12-17 12:36:50,929] jepsen worker 63 - jepsen.util 9963 :invoke :write [21 2] INFO [2019-12-17 12:36:50,932] jepsen worker 40 - jepsen.util 15700 :invoke :cas [13 [14 23]] INFO [2019-12-17 12:36:50,933] jepsen worker 44 - jepsen.util 15794 :invoke :cas [14 [0 38]] INFO [2019-12-17 12:36:50,933] jepsen worker 4 - jepsen.util 10264 :info :write [1 22] :RequestCanceledException INFO [2019-12-17 12:36:50,933] jepsen worker 80 - jepsen.util 10880 :info :write [26 41] :RequestCanceledException INFO [2019-12-17 12:36:50,933] jepsen worker 24 - jepsen.util 13884 :invoke :write [8 12] INFO [2019-12-17 12:36:50,936] jepsen worker 19 - jepsen.util 11449 :invoke :write [6 47] INFO [2019-12-17 12:36:50,937] jepsen worker 86 - jepsen.util 15926 :invoke :write [28 12] INFO [2019-12-17 12:36:50,937] jepsen worker 77 - jepsen.util 16007 :invoke :cas [25 [31 49]] INFO [2019-12-17 12:36:50,939] jepsen worker 81 - jepsen.util 14931 :invoke :read [27 nil] INFO [2019-12-17 12:36:50,941] jepsen worker 75 - jepsen.util 14745 :invoke :cas [25 [34 18]] INFO [2019-12-17 12:36:50,943] jepsen worker 6 - jepsen.util 15756 :invoke :read [2 nil] INFO [2019-12-17 12:36:50,945] jepsen worker 29 - jepsen.util 9479 :invoke :read [9 nil] INFO [2019-12-17 12:36:50,947] jepsen worker 22 - jepsen.util 15232 :invoke :cas [7 [49 3]] INFO [2019-12-17 12:36:50,960] jepsen worker 80 - jepsen.util 10970 :invoke :read [26 nil] INFO [2019-12-17 12:36:50,964] jepsen worker 4 - jepsen.util 10354 :invoke :read [1 nil] INFO [2019-12-17 12:36:50,991] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64578,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:36:50,991] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64560,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:36:50,991] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64578,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:36:50,992] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64582,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:36:50,992] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64560,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:36:50,993] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64582,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:36:50,994] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64569,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:36:50,995] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64569,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:36:51,000] jepsen worker 10 - jepsen.util 10900 :fail :cas [3 [47 47]] :ValueNotSwapFrom INFO [2019-12-17 12:36:51,000] jepsen worker 10 - jepsen.util 10900 :invoke :cas [3 [42 36]] INFO [2019-12-17 12:36:51,006] jepsen worker 28 - jepsen.util 9298 :ok :read [9 12] INFO [2019-12-17 12:36:51,007] jepsen worker 28 - jepsen.util 9298 :invoke :read [9 nil] INFO [2019-12-17 12:36:51,009] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64612,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:36:51,010] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64612,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:36:51,011] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64613,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:36:51,011] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64613,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:36:51,011] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64614,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:36:51,011] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64614,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:36:51,015] jepsen worker 74 - jepsen.util 10784 :ok :read [24 8] INFO [2019-12-17 12:36:51,016] jepsen worker 74 - jepsen.util 10784 :invoke :write [24 11] INFO [2019-12-17 12:36:51,027] jepsen worker 60 - jepsen.util 16260 :info :write [20 47] :RequestCanceledException INFO [2019-12-17 12:36:51,030] jepsen worker 60 - jepsen.util 16350 :invoke :write [20 41] INFO [2019-12-17 12:36:51,039] jepsen worker 42 - jepsen.util 15972 :info :write [14 25] :RequestCanceledException INFO [2019-12-17 12:36:51,043] jepsen worker 42 - jepsen.util 16062 :invoke :write [14 48] INFO [2019-12-17 12:36:51,062] jepsen worker 5 - jepsen.util 10895 :fail :cas [1 [29 29]] :ValueNotSwapFrom INFO [2019-12-17 12:36:51,063] jepsen worker 5 - jepsen.util 10895 :invoke :cas [1 [18 1]] INFO [2019-12-17 12:36:51,064] jepsen worker 31 - jepsen.util 9571 :ok :read [10 30] INFO [2019-12-17 12:36:51,065] jepsen worker 31 - jepsen.util 9571 :invoke :read [10 nil] INFO [2019-12-17 12:36:51,070] jepsen worker 30 - jepsen.util 10470 :fail :cas [10 [20 24]] :ValueNotSwapFrom INFO [2019-12-17 12:36:51,071] jepsen worker 30 - jepsen.util 10470 :invoke :read [10 nil] INFO [2019-12-17 12:36:51,077] jepsen worker 53 - jepsen.util 10583 :fail :cas [17 [34 11]] :ValueNotSwapFrom INFO [2019-12-17 12:36:51,077] jepsen worker 53 - jepsen.util 10583 :invoke :write [17 7] INFO [2019-12-17 12:36:51,077] jepsen worker 27 - jepsen.util 10467 :ok :read [9 39] INFO [2019-12-17 12:36:51,078] jepsen worker 27 - jepsen.util 10467 :invoke :write [9 19] INFO [2019-12-17 12:36:51,080] jepsen worker 45 - jepsen.util 11295 :ok :read [15 21] INFO [2019-12-17 12:36:51,081] jepsen worker 45 - jepsen.util 11295 :invoke :write [15 39] INFO [2019-12-17 12:36:51,087] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:36:51,088] jepsen nemesis - couchbase.util Rebalance check count: 508 INFO [2019-12-17 12:36:51,088] jepsen nemesis - couchbase.util Rebalance stuck count: 489 INFO [2019-12-17 12:36:51,129] jepsen worker 46 - jepsen.util 10846 :ok :read [15 6] INFO [2019-12-17 12:36:51,129] jepsen worker 46 - jepsen.util 10846 :invoke :write [15 30] INFO [2019-12-17 12:36:51,132] jepsen worker 70 - jepsen.util 11770 :info :write [23 37] :RequestCanceledException INFO [2019-12-17 12:36:51,143] jepsen worker 70 - jepsen.util 11860 :invoke :write [23 30] INFO [2019-12-17 12:36:51,149] jepsen worker 14 - jepsen.util 10994 :info :write [4 25] :RequestCanceledException INFO [2019-12-17 12:36:51,154] jepsen worker 14 - jepsen.util 11084 :invoke :cas [4 [17 7]] INFO [2019-12-17 12:36:51,174] jepsen worker 20 - jepsen.util 10730 :fail :cas [6 [24 44]] :ValueNotSwapFrom INFO [2019-12-17 12:36:51,175] jepsen worker 20 - jepsen.util 10730 :invoke :write [6 46] INFO [2019-12-17 12:36:51,192] jepsen worker 79 - jepsen.util 11419 :ok :read [26 49] INFO [2019-12-17 12:36:51,192] jepsen worker 79 - jepsen.util 11419 :invoke :write [26 1] INFO [2019-12-17 12:36:51,199] jepsen worker 32 - jepsen.util 11372 :info :write [10 17] :RequestCanceledException INFO [2019-12-17 12:36:51,199] jepsen worker 25 - jepsen.util 16225 :info :write [8 8] :RequestCanceledException INFO [2019-12-17 12:36:51,200] jepsen worker 78 - jepsen.util 11328 :fail :cas [26 [12 36]] :ValueNotSwapFrom INFO [2019-12-17 12:36:51,200] jepsen worker 78 - jepsen.util 11328 :invoke :read [26 nil] INFO [2019-12-17 12:36:51,204] jepsen worker 32 - jepsen.util 11462 :invoke :write [10 42] INFO [2019-12-17 12:36:51,204] jepsen worker 25 - jepsen.util 16315 :invoke :read [8 nil] INFO [2019-12-17 12:36:51,209] jepsen worker 72 - jepsen.util 11322 :ok :read [24 15] INFO [2019-12-17 12:36:51,209] jepsen worker 72 - jepsen.util 11322 :invoke :cas [24 [48 30]] INFO [2019-12-17 12:36:51,214] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64642,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:36:51,215] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64642,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:36:51,215] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64652,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:36:51,215] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64643,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:36:51,215] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64652,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:36:51,215] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64643,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:36:51,215] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64644,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:36:51,215] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64656,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:36:51,215] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64644,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:36:51,215] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64656,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:36:51,216] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64657,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:36:51,216] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64657,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:36:51,218] jepsen worker 28 - jepsen.util 9298 :ok :read [9 39] INFO [2019-12-17 12:36:51,241] jepsen worker 47 - jepsen.util 11927 :fail :cas [15 [3 26]] :ValueNotSwapFrom INFO [2019-12-17 12:36:51,266] jepsen worker 29 - jepsen.util 9479 :ok :read [9 39] INFO [2019-12-17 12:36:51,281] jepsen worker 65 - jepsen.util 9605 :ok :write [21 37] INFO [2019-12-17 12:36:51,299] jepsen worker 9 - jepsen.util 11889 :fail :read [3 nil] :RequestTimeoutException INFO [2019-12-17 12:36:51,325] jepsen worker 63 - jepsen.util 9963 :ok :write [21 2] INFO [2019-12-17 12:36:51,331] jepsen worker 69 - jepsen.util 11409 :fail :cas [23 [22 19]] :ValueNotSwapFrom INFO [2019-12-17 12:36:51,352] jepsen worker 11 - jepsen.util 10811 :ok :write [3 37] INFO [2019-12-17 12:36:51,366] jepsen worker 55 - jepsen.util 11845 :ok :write [18 6] INFO [2019-12-17 12:36:51,387] jepsen worker 37 - jepsen.util 10837 :ok :write [12 3] INFO [2019-12-17 12:36:51,388] jepsen worker 10 - jepsen.util 10900 :fail :cas [3 [42 36]] :ValueNotSwapFrom INFO [2019-12-17 12:36:51,391] jepsen worker 20 - jepsen.util 10730 :ok :write [6 46] INFO [2019-12-17 12:36:51,437] jepsen worker 38 - jepsen.util 11738 :ok :read [12 3] INFO [2019-12-17 12:36:51,448] jepsen worker 12 - jepsen.util 11532 :ok :read [4 25] INFO [2019-12-17 12:36:51,455] jepsen worker 71 - jepsen.util 12311 :ok :read [23 3] INFO [2019-12-17 12:36:51,504] jepsen worker 47 - jepsen.util 11927 :invoke :write [15 32] INFO [2019-12-17 12:36:51,504] jepsen worker 28 - jepsen.util 9298 :invoke :write [9 2] INFO [2019-12-17 12:36:51,504] jepsen worker 29 - jepsen.util 9479 :invoke :read [9 nil] INFO [2019-12-17 12:36:51,504] jepsen worker 65 - jepsen.util 9605 :invoke :read [21 nil] INFO [2019-12-17 12:36:51,504] jepsen worker 9 - jepsen.util 11889 :invoke :write [3 38] INFO [2019-12-17 12:36:51,504] jepsen worker 63 - jepsen.util 9963 :invoke :cas [21 [44 44]] INFO [2019-12-17 12:36:51,504] jepsen worker 69 - jepsen.util 11409 :invoke :write [23 12] INFO [2019-12-17 12:36:51,504] jepsen worker 11 - jepsen.util 10811 :invoke :write [3 27] INFO [2019-12-17 12:36:51,504] jepsen worker 55 - jepsen.util 11845 :invoke :write [18 10] INFO [2019-12-17 12:36:51,504] jepsen worker 37 - jepsen.util 10837 :invoke :write [12 17] INFO [2019-12-17 12:36:51,504] jepsen worker 10 - jepsen.util 10900 :invoke :read [3 nil] INFO [2019-12-17 12:36:51,504] jepsen worker 20 - jepsen.util 10730 :invoke :write [6 34] INFO [2019-12-17 12:36:51,504] jepsen worker 38 - jepsen.util 11738 :invoke :cas [12 [40 44]] INFO [2019-12-17 12:36:51,504] jepsen worker 12 - jepsen.util 11532 :invoke :write [4 8] INFO [2019-12-17 12:36:51,504] jepsen worker 71 - jepsen.util 12311 :invoke :cas [23 [28 15]] INFO [2019-12-17 12:36:51,525] jepsen worker 13 - jepsen.util 10453 :info :write [4 26] :RequestCanceledException INFO [2019-12-17 12:36:51,532] jepsen worker 13 - jepsen.util 10543 :invoke :read [4 nil] INFO [2019-12-17 12:36:51,542] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64660,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:36:51,543] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64660,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:36:51,543] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64662,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:36:51,543] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64662,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:36:51,543] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64663,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:36:51,544] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64663,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:36:51,577] jepsen worker 52 - jepsen.util 10672 :ok :read [17 45] INFO [2019-12-17 12:36:51,577] jepsen worker 52 - jepsen.util 10672 :invoke :write [17 22] INFO [2019-12-17 12:36:51,582] jepsen worker 63 - jepsen.util 9963 :fail :cas [21 [44 44]] :ValueNotSwapFrom INFO [2019-12-17 12:36:51,583] jepsen worker 63 - jepsen.util 9963 :invoke :read [21 nil] INFO [2019-12-17 12:36:51,604] jepsen worker 0 - jepsen.util 16020 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:36:51,604] jepsen worker 0 - jepsen.util 16020 :invoke :read [0 nil] INFO [2019-12-17 12:36:51,616] jepsen worker 85 - jepsen.util 16015 :info :write [28 13] :RequestCanceledException INFO [2019-12-17 12:36:51,622] jepsen worker 85 - jepsen.util 16105 :invoke :read [28 nil] INFO [2019-12-17 12:36:51,638] jepsen worker 65 - jepsen.util 9605 :ok :read [21 2] INFO [2019-12-17 12:36:51,638] jepsen worker 65 - jepsen.util 9605 :invoke :write [21 23] INFO [2019-12-17 12:36:51,651] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64685,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:36:51,651] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64685,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:36:51,651] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64688,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:36:51,651] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64688,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:36:51,652] jepsen worker 56 - jepsen.util 12026 :fail :cas [18 [23 5]] :ValueNotSwapFrom INFO [2019-12-17 12:36:51,652] jepsen worker 56 - jepsen.util 12026 :invoke :read [18 nil] INFO [2019-12-17 12:36:51,662] jepsen worker 80 - jepsen.util 10970 :ok :read [26 49] INFO [2019-12-17 12:36:51,662] jepsen worker 80 - jepsen.util 10970 :invoke :cas [26 [10 27]] INFO [2019-12-17 12:36:51,665] jepsen worker 10 - jepsen.util 10900 :ok :read [3 37] INFO [2019-12-17 12:36:51,665] jepsen worker 10 - jepsen.util 10900 :invoke :write [3 36] INFO [2019-12-17 12:36:51,672] jepsen worker 54 - jepsen.util 11844 :ok :read [18 6] INFO [2019-12-17 12:36:51,672] jepsen worker 54 - jepsen.util 11844 :invoke :read [18 nil] INFO [2019-12-17 12:36:51,717] jepsen worker 51 - jepsen.util 10671 :ok :write [17 40] INFO [2019-12-17 12:36:51,717] jepsen worker 51 - jepsen.util 10671 :invoke :read [17 nil] INFO [2019-12-17 12:36:51,718] jepsen worker 28 - jepsen.util 9298 :ok :write [9 2] INFO [2019-12-17 12:36:51,718] jepsen worker 28 - jepsen.util 9298 :invoke :cas [9 [20 5]] INFO [2019-12-17 12:36:51,720] jepsen worker 19 - jepsen.util 11449 :ok :write [6 47] INFO [2019-12-17 12:36:51,720] jepsen worker 19 - jepsen.util 11449 :invoke :cas [6 [9 21]] INFO [2019-12-17 12:36:51,726] jepsen worker 73 - jepsen.util 10333 :info :write [24 47] :RequestCanceledException INFO [2019-12-17 12:36:51,729] jepsen worker 73 - jepsen.util 10423 :invoke :read [24 nil] INFO [2019-12-17 12:36:51,796] jepsen worker 28 - jepsen.util 9298 :fail :cas [9 [20 5]] :ValueNotSwapFrom INFO [2019-12-17 12:36:51,796] jepsen worker 28 - jepsen.util 9298 :invoke :read [9 nil] INFO [2019-12-17 12:36:51,798] jepsen worker 14 - jepsen.util 11084 :fail :cas [4 [17 7]] :ValueNotSwapFrom INFO [2019-12-17 12:36:51,798] jepsen worker 14 - jepsen.util 11084 :invoke :cas [4 [48 17]] INFO [2019-12-17 12:36:51,799] jepsen worker 29 - jepsen.util 9479 :ok :read [9 2] INFO [2019-12-17 12:36:51,799] jepsen worker 29 - jepsen.util 9479 :invoke :cas [9 [11 9]] INFO [2019-12-17 12:36:51,813] jepsen worker 89 - jepsen.util 11249 :ok :write [29 42] INFO [2019-12-17 12:36:51,848] jepsen worker 28 - jepsen.util 9298 :ok :read [9 2] INFO [2019-12-17 12:36:51,872] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64703,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:36:51,884] jepsen worker 51 - jepsen.util 10671 :ok :read [17 40] INFO [2019-12-17 12:36:51,919] jepsen worker 7 - jepsen.util 15217 :info :cas [2 [7 16]] :RequestCanceledException INFO [2019-12-17 12:36:51,919] jepsen worker 18 - jepsen.util 11268 :fail :read [6 nil] :RequestTimeoutException INFO [2019-12-17 12:36:51,925] jepsen worker 48 - jepsen.util 14088 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:36:51,927] jepsen worker 78 - jepsen.util 11328 :ok :read [26 41] INFO [2019-12-17 12:36:51,932] jepsen worker 61 - jepsen.util 16351 :info :cas [20 [1 36]] :RequestCanceledException INFO [2019-12-17 12:36:51,932] jepsen worker 8 - jepsen.util 15038 :info :cas [2 [27 11]] :RequestCanceledException INFO [2019-12-17 12:36:51,953] jepsen worker 89 - jepsen.util 11249 :invoke :read [29 nil] INFO [2019-12-17 12:36:51,953] jepsen worker 28 - jepsen.util 9298 :invoke :read [9 nil] INFO [2019-12-17 12:36:51,953] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64703,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:36:51,953] jepsen worker 51 - jepsen.util 10671 :invoke :cas [17 [9 33]] INFO [2019-12-17 12:36:51,953] jepsen worker 48 - jepsen.util 14088 :invoke :cas [16 [25 6]] INFO [2019-12-17 12:36:51,953] jepsen worker 18 - jepsen.util 11268 :invoke :write [6 27] INFO [2019-12-17 12:36:51,954] jepsen worker 78 - jepsen.util 11328 :invoke :cas [26 [28 42]] INFO [2019-12-17 12:36:51,954] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64707,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:36:51,954] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64707,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:36:51,956] jepsen worker 7 - jepsen.util 15307 :invoke :write [2 33] INFO [2019-12-17 12:36:51,956] jepsen worker 61 - jepsen.util 16441 :invoke :cas [20 [24 28]] INFO [2019-12-17 12:36:51,956] jepsen worker 8 - jepsen.util 15128 :invoke :cas [2 [28 33]] INFO [2019-12-17 12:36:51,961] jepsen worker 67 - jepsen.util 15277 :info :cas [22 [5 44]] :RequestCanceledException INFO [2019-12-17 12:36:51,966] jepsen worker 67 - jepsen.util 15367 :invoke :cas [22 [28 39]] INFO [2019-12-17 12:36:51,978] jepsen worker 3 - jepsen.util 10713 :ok :write [1 34] INFO [2019-12-17 12:36:51,978] jepsen worker 3 - jepsen.util 10713 :invoke :cas [1 [38 8]] INFO [2019-12-17 12:36:51,999] jepsen worker 15 - jepsen.util 14235 :info :cas [5 [9 8]] :RequestCanceledException INFO [2019-12-17 12:36:52,000] jepsen worker 28 - jepsen.util 9298 :ok :read [9 2] INFO [2019-12-17 12:36:52,026] jepsen worker 38 - jepsen.util 11738 :fail :cas [12 [40 44]] :ValueNotSwapFrom INFO [2019-12-17 12:36:52,032] jepsen worker 64 - jepsen.util 8524 :info :write [21 35] :RequestCanceledException INFO [2019-12-17 12:36:52,039] jepsen worker 87 - jepsen.util 10887 :ok :read [29 42] INFO [2019-12-17 12:36:52,051] jepsen worker 71 - jepsen.util 12311 :fail :cas [23 [28 15]] :ValueNotSwapFrom INFO [2019-12-17 12:36:52,059] jepsen worker 9 - jepsen.util 11889 :ok :write [3 38] INFO [2019-12-17 12:36:52,060] jepsen worker 28 - jepsen.util 9298 :invoke :write [9 47] INFO [2019-12-17 12:36:52,060] jepsen worker 38 - jepsen.util 11738 :invoke :cas [12 [35 14]] INFO [2019-12-17 12:36:52,060] jepsen worker 87 - jepsen.util 10887 :invoke :cas [29 [36 17]] INFO [2019-12-17 12:36:52,060] jepsen worker 71 - jepsen.util 12311 :invoke :read [23 nil] INFO [2019-12-17 12:36:52,060] jepsen worker 9 - jepsen.util 11889 :invoke :cas [3 [14 42]] INFO [2019-12-17 12:36:52,062] jepsen worker 36 - jepsen.util 12096 :ok :write [12 22] INFO [2019-12-17 12:36:52,062] jepsen worker 36 - jepsen.util 12096 :invoke :cas [12 [13 32]] INFO [2019-12-17 12:36:52,062] jepsen worker 64 - jepsen.util 8614 :invoke :read [21 nil] INFO [2019-12-17 12:36:52,073] jepsen worker 15 - jepsen.util 14325 :invoke :read [5 nil] INFO [2019-12-17 12:36:52,088] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64719,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:36:52,088] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64719,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:36:52,088] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64720,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:36:52,088] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64720,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:36:52,106] jepsen worker 9 - jepsen.util 11889 :fail :cas [3 [14 42]] :ValueNotSwapFrom INFO [2019-12-17 12:36:52,106] jepsen worker 9 - jepsen.util 11889 :invoke :cas [3 [19 18]] INFO [2019-12-17 12:36:52,106] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:36:52,106] jepsen nemesis - couchbase.util Rebalance check count: 509 INFO [2019-12-17 12:36:52,106] jepsen nemesis - couchbase.util Rebalance stuck count: 490 INFO [2019-12-17 12:36:52,117] jepsen worker 58 - jepsen.util 15538 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:36:52,117] jepsen worker 2 - jepsen.util 14762 :info :cas [0 [19 39]] :RequestCanceledException INFO [2019-12-17 12:36:52,117] jepsen worker 5 - jepsen.util 10895 :fail :cas [1 [18 1]] :ValueNotSwapFrom INFO [2019-12-17 12:36:52,118] jepsen worker 58 - jepsen.util 15538 :invoke :cas [19 [8 26]] INFO [2019-12-17 12:36:52,118] jepsen worker 5 - jepsen.util 10895 :invoke :write [1 0] INFO [2019-12-17 12:36:52,121] jepsen worker 2 - jepsen.util 14852 :invoke :cas [0 [42 0]] INFO [2019-12-17 12:36:52,132] jepsen worker 23 - jepsen.util 15233 :info :cas [7 [48 11]] :RequestCanceledException INFO [2019-12-17 12:36:52,133] jepsen worker 50 - jepsen.util 15620 :info :write [16 12] :RequestCanceledException INFO [2019-12-17 12:36:52,133] jepsen worker 17 - jepsen.util 15227 :info :cas [5 [46 44]] :RequestCanceledException INFO [2019-12-17 12:36:52,136] jepsen worker 23 - jepsen.util 15323 :invoke :read [7 nil] INFO [2019-12-17 12:36:52,137] jepsen worker 50 - jepsen.util 15710 :invoke :write [16 24] INFO [2019-12-17 12:36:52,137] jepsen worker 17 - jepsen.util 15317 :invoke :write [5 1] INFO [2019-12-17 12:36:52,140] jepsen worker 14 - jepsen.util 11084 :fail :cas [4 [48 17]] :ValueNotSwapFrom INFO [2019-12-17 12:36:52,140] jepsen worker 14 - jepsen.util 11084 :invoke :write [4 36] INFO [2019-12-17 12:36:52,147] jepsen worker 43 - jepsen.util 15343 :info :cas [14 [14 12]] :RequestCanceledException INFO [2019-12-17 12:36:52,147] jepsen worker 62 - jepsen.util 15902 :info :cas [20 [46 26]] :RequestCanceledException INFO [2019-12-17 12:36:52,152] jepsen worker 62 - jepsen.util 15992 :invoke :write [20 5] INFO [2019-12-17 12:36:52,152] jepsen worker 43 - jepsen.util 15433 :invoke :cas [14 [24 45]] INFO [2019-12-17 12:36:52,164] jepsen worker 21 - jepsen.util 14871 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:36:52,165] jepsen worker 21 - jepsen.util 14871 :invoke :read [7 nil] INFO [2019-12-17 12:36:52,170] jepsen worker 9 - jepsen.util 11889 :fail :cas [3 [19 18]] :ValueNotSwapFrom INFO [2019-12-17 12:36:52,170] jepsen worker 9 - jepsen.util 11889 :invoke :read [3 nil] INFO [2019-12-17 12:36:52,175] jepsen worker 47 - jepsen.util 11927 :ok :write [15 32] INFO [2019-12-17 12:36:52,175] jepsen worker 47 - jepsen.util 11927 :invoke :write [15 11] INFO [2019-12-17 12:36:52,195] jepsen worker 84 - jepsen.util 15024 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:36:52,196] jepsen worker 84 - jepsen.util 15024 :invoke :cas [28 [18 48]] INFO [2019-12-17 12:36:52,206] jepsen worker 4 - jepsen.util 10354 :ok :read [1 34] INFO [2019-12-17 12:36:52,207] jepsen worker 4 - jepsen.util 10354 :invoke :read [1 nil] INFO [2019-12-17 12:36:52,213] jepsen worker 9 - jepsen.util 11889 :ok :read [3 38] INFO [2019-12-17 12:36:52,213] jepsen worker 9 - jepsen.util 11889 :invoke :cas [3 [38 46]] INFO [2019-12-17 12:36:52,227] jepsen worker 52 - jepsen.util 10672 :ok :write [17 22] INFO [2019-12-17 12:36:52,228] jepsen worker 52 - jepsen.util 10672 :invoke :cas [17 [17 15]] INFO [2019-12-17 12:36:52,232] jepsen worker 28 - jepsen.util 9298 :ok :write [9 47] INFO [2019-12-17 12:36:52,233] jepsen worker 28 - jepsen.util 9298 :invoke :write [9 17] INFO [2019-12-17 12:36:52,236] jepsen worker 18 - jepsen.util 11268 :ok :write [6 27] INFO [2019-12-17 12:36:52,236] jepsen worker 18 - jepsen.util 11268 :invoke :read [6 nil] INFO [2019-12-17 12:36:52,240] jepsen worker 29 - jepsen.util 9479 :fail :cas [9 [11 9]] :ValueNotSwapFrom INFO [2019-12-17 12:36:52,240] jepsen worker 29 - jepsen.util 9479 :invoke :write [9 9] INFO [2019-12-17 12:36:52,271] jepsen worker 12 - jepsen.util 11532 :ok :write [4 8] INFO [2019-12-17 12:36:52,271] jepsen worker 12 - jepsen.util 11532 :invoke :write [4 39] INFO [2019-12-17 12:36:52,297] jepsen worker 79 - jepsen.util 11419 :ok :write [26 1] INFO [2019-12-17 12:36:52,298] jepsen worker 79 - jepsen.util 11419 :invoke :read [26 nil] INFO [2019-12-17 12:36:52,298] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64747,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:36:52,299] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64747,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:36:52,300] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64749,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:36:52,300] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64755,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:36:52,300] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64749,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:36:52,301] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64755,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:36:52,301] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64759,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:36:52,302] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64759,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:36:52,302] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64760,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:36:52,303] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64760,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:36:52,335] jepsen worker 64 - jepsen.util 8614 :ok :read [21 2] INFO [2019-12-17 12:36:52,335] jepsen worker 64 - jepsen.util 8614 :invoke :read [21 nil] INFO [2019-12-17 12:36:52,372] jepsen worker 18 - jepsen.util 11268 :ok :read [6 27] INFO [2019-12-17 12:36:52,373] jepsen worker 18 - jepsen.util 11268 :invoke :cas [6 [36 8]] INFO [2019-12-17 12:36:52,381] jepsen worker 4 - jepsen.util 10354 :ok :read [1 34] INFO [2019-12-17 12:36:52,381] jepsen worker 4 - jepsen.util 10354 :invoke :cas [1 [13 1]] INFO [2019-12-17 12:36:52,393] jepsen worker 37 - jepsen.util 10837 :ok :write [12 17] INFO [2019-12-17 12:36:52,393] jepsen worker 37 - jepsen.util 10837 :invoke :read [12 nil] INFO [2019-12-17 12:36:52,406] jepsen worker 13 - jepsen.util 10543 :ok :read [4 8] INFO [2019-12-17 12:36:52,406] jepsen worker 13 - jepsen.util 10543 :invoke :write [4 11] INFO [2019-12-17 12:36:52,410] jepsen worker 79 - jepsen.util 11419 :ok :read [26 1] INFO [2019-12-17 12:36:52,410] jepsen worker 79 - jepsen.util 11419 :invoke :cas [26 [34 3]] INFO [2019-12-17 12:36:52,422] jepsen worker 64 - jepsen.util 8614 :ok :read [21 2] INFO [2019-12-17 12:36:52,422] jepsen worker 64 - jepsen.util 8614 :invoke :write [21 43] INFO [2019-12-17 12:36:52,465] jepsen worker 80 - jepsen.util 10970 :fail :cas [26 [10 27]] :ValueNotSwapFrom INFO [2019-12-17 12:36:52,465] jepsen worker 80 - jepsen.util 10970 :invoke :read [26 nil] INFO [2019-12-17 12:36:52,495] jepsen worker 18 - jepsen.util 11268 :fail :cas [6 [36 8]] :ValueNotSwapFrom INFO [2019-12-17 12:36:52,495] jepsen worker 18 - jepsen.util 11268 :invoke :write [6 40] INFO [2019-12-17 12:36:52,516] jepsen worker 47 - jepsen.util 11927 :ok :write [15 11] INFO [2019-12-17 12:36:52,516] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64775,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:36:52,517] jepsen worker 47 - jepsen.util 11927 :invoke :cas [15 [32 47]] INFO [2019-12-17 12:36:52,517] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64775,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:36:52,517] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64778,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:36:52,517] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64778,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:36:52,517] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64779,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:36:52,518] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64779,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:36:52,526] jepsen worker 88 - jepsen.util 11518 :ok :write [29 23] INFO [2019-12-17 12:36:52,526] jepsen worker 88 - jepsen.util 11518 :invoke :read [29 nil] INFO [2019-12-17 12:36:52,543] jepsen worker 28 - jepsen.util 9298 :ok :write [9 17] INFO [2019-12-17 12:36:52,543] jepsen worker 28 - jepsen.util 9298 :invoke :cas [9 [36 27]] INFO [2019-12-17 12:36:52,544] jepsen worker 37 - jepsen.util 10837 :ok :read [12 17] INFO [2019-12-17 12:36:52,544] jepsen worker 37 - jepsen.util 10837 :invoke :cas [12 [0 25]] INFO [2019-12-17 12:36:52,546] jepsen worker 9 - jepsen.util 11889 :ok :cas [3 [38 46]] INFO [2019-12-17 12:36:52,546] jepsen worker 9 - jepsen.util 11889 :invoke :cas [3 [8 49]] INFO [2019-12-17 12:36:52,565] jepsen worker 80 - jepsen.util 10970 :ok :read [26 1] INFO [2019-12-17 12:36:52,565] jepsen worker 80 - jepsen.util 10970 :invoke :read [26 nil] INFO [2019-12-17 12:36:52,601] jepsen worker 28 - jepsen.util 9298 :fail :cas [9 [36 27]] :ValueNotSwapFrom INFO [2019-12-17 12:36:52,601] jepsen worker 28 - jepsen.util 9298 :invoke :cas [9 [36 39]] INFO [2019-12-17 12:36:52,621] jepsen worker 29 - jepsen.util 9479 :ok :write [9 9] INFO [2019-12-17 12:36:52,622] jepsen worker 29 - jepsen.util 9479 :invoke :write [9 21] INFO [2019-12-17 12:36:52,633] jepsen worker 64 - jepsen.util 8614 :ok :write [21 43] INFO [2019-12-17 12:36:52,633] jepsen worker 64 - jepsen.util 8614 :invoke :read [21 nil] INFO [2019-12-17 12:36:52,640] jepsen worker 51 - jepsen.util 10671 :fail :cas [17 [9 33]] :ValueNotSwapFrom INFO [2019-12-17 12:36:52,640] jepsen worker 51 - jepsen.util 10671 :invoke :read [17 nil] INFO [2019-12-17 12:36:52,683] jepsen worker 9 - jepsen.util 11889 :fail :cas [3 [8 49]] :ValueNotSwapFrom INFO [2019-12-17 12:36:52,683] jepsen worker 9 - jepsen.util 11889 :invoke :read [3 nil] INFO [2019-12-17 12:36:52,694] jepsen worker 37 - jepsen.util 10837 :fail :cas [12 [0 25]] :ValueNotSwapFrom INFO [2019-12-17 12:36:52,694] jepsen worker 37 - jepsen.util 10837 :invoke :write [12 21] INFO [2019-12-17 12:36:52,700] jepsen worker 3 - jepsen.util 10713 :fail :cas [1 [38 8]] :ValueNotSwapFrom INFO [2019-12-17 12:36:52,700] jepsen worker 3 - jepsen.util 10713 :invoke :cas [1 [36 48]] INFO [2019-12-17 12:36:52,707] jepsen worker 28 - jepsen.util 9298 :fail :cas [9 [36 39]] :ValueNotSwapFrom INFO [2019-12-17 12:36:52,707] jepsen worker 28 - jepsen.util 9298 :invoke :cas [9 [16 43]] INFO [2019-12-17 12:36:52,712] jepsen worker 80 - jepsen.util 10970 :ok :read [26 1] INFO [2019-12-17 12:36:52,712] jepsen worker 80 - jepsen.util 10970 :invoke :read [26 nil] INFO [2019-12-17 12:36:52,741] jepsen worker 14 - jepsen.util 11084 :ok :write [4 36] INFO [2019-12-17 12:36:52,741] jepsen worker 14 - jepsen.util 11084 :invoke :write [4 45] INFO [2019-12-17 12:36:52,743] jepsen worker 4 - jepsen.util 10354 :fail :cas [1 [13 1]] :ValueNotSwapFrom INFO [2019-12-17 12:36:52,743] jepsen worker 4 - jepsen.util 10354 :invoke :write [1 23] INFO [2019-12-17 12:36:52,753] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64795,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:36:52,754] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64795,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:36:52,754] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64797,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:36:52,754] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64797,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:36:52,763] jepsen worker 70 - jepsen.util 11860 :ok :write [23 30] INFO [2019-12-17 12:36:52,764] jepsen worker 70 - jepsen.util 11860 :invoke :read [23 nil] INFO [2019-12-17 12:36:52,779] jepsen worker 64 - jepsen.util 8614 :ok :read [21 43] INFO [2019-12-17 12:36:52,779] jepsen worker 64 - jepsen.util 8614 :invoke :cas [21 [35 22]] INFO [2019-12-17 12:36:52,785] jepsen worker 79 - jepsen.util 11419 :fail :cas [26 [34 3]] :ValueNotSwapFrom INFO [2019-12-17 12:36:52,786] jepsen worker 79 - jepsen.util 11419 :invoke :cas [26 [16 19]] INFO [2019-12-17 12:36:52,808] jepsen worker 9 - jepsen.util 11889 :ok :read [3 46] INFO [2019-12-17 12:36:52,809] jepsen worker 9 - jepsen.util 11889 :invoke :cas [3 [27 29]] INFO [2019-12-17 12:36:52,812] jepsen worker 87 - jepsen.util 10887 :fail :cas [29 [36 17]] :ValueNotSwapFrom INFO [2019-12-17 12:36:52,812] jepsen worker 87 - jepsen.util 10887 :invoke :read [29 nil] INFO [2019-12-17 12:36:52,817] jepsen worker 55 - jepsen.util 11845 :ok :write [18 10] INFO [2019-12-17 12:36:52,817] jepsen worker 55 - jepsen.util 11845 :invoke :cas [18 [6 1]] INFO [2019-12-17 12:36:52,820] jepsen worker 80 - jepsen.util 10970 :ok :read [26 1] INFO [2019-12-17 12:36:52,821] jepsen worker 80 - jepsen.util 10970 :invoke :cas [26 [44 10]] INFO [2019-12-17 12:36:52,833] jepsen worker 28 - jepsen.util 9298 :fail :cas [9 [16 43]] :ValueNotSwapFrom INFO [2019-12-17 12:36:52,833] jepsen worker 28 - jepsen.util 9298 :invoke :read [9 nil] INFO [2019-12-17 12:36:52,858] jepsen worker 51 - jepsen.util 10671 :ok :read [17 22] INFO [2019-12-17 12:36:52,858] jepsen worker 51 - jepsen.util 10671 :invoke :cas [17 [21 40]] INFO [2019-12-17 12:36:52,863] jepsen worker 64 - jepsen.util 8614 :fail :cas [21 [35 22]] :ValueNotSwapFrom INFO [2019-12-17 12:36:52,863] jepsen worker 64 - jepsen.util 8614 :invoke :read [21 nil] INFO [2019-12-17 12:36:52,892] jepsen worker 53 - jepsen.util 10583 :ok :write [17 7] INFO [2019-12-17 12:36:52,892] jepsen worker 53 - jepsen.util 10583 :invoke :write [17 3] INFO [2019-12-17 12:36:52,902] jepsen worker 71 - jepsen.util 12311 :ok :read [23 30] INFO [2019-12-17 12:36:52,909] jepsen worker 71 - jepsen.util 12311 :invoke :write [23 32] INFO [2019-12-17 12:36:52,930] jepsen worker 29 - jepsen.util 9479 :ok :write [9 21] INFO [2019-12-17 12:36:52,933] jepsen worker 76 - jepsen.util 15646 :info :cas [25 [33 37]] :RequestCanceledException INFO [2019-12-17 12:36:52,933] jepsen worker 59 - jepsen.util 14549 :info :cas [19 [44 29]] :RequestCanceledException INFO [2019-12-17 12:36:52,934] jepsen worker 35 - jepsen.util 15245 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:36:52,940] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64817,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:36:52,942] jepsen worker 29 - jepsen.util 9479 :invoke :write [9 18] INFO [2019-12-17 12:36:52,942] jepsen worker 35 - jepsen.util 15245 :invoke :write [11 0] INFO [2019-12-17 12:36:52,944] jepsen worker 83 - jepsen.util 15473 :info :write [27 1] :RequestCanceledException INFO [2019-12-17 12:36:52,945] jepsen worker 9 - jepsen.util 11889 :fail :cas [3 [27 29]] :ValueNotSwapFrom INFO [2019-12-17 12:36:52,945] jepsen worker 57 - jepsen.util 15087 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:36:52,954] jepsen worker 59 - jepsen.util 14639 :invoke :read [19 nil] INFO [2019-12-17 12:36:52,954] jepsen worker 76 - jepsen.util 15736 :invoke :cas [25 [32 6]] INFO [2019-12-17 12:36:52,961] jepsen worker 52 - jepsen.util 10672 :fail :cas [17 [17 15]] :ValueNotSwapFrom INFO [2019-12-17 12:36:52,961] jepsen worker 28 - jepsen.util 9298 :ok :read [9 9] INFO [2019-12-17 12:36:52,964] jepsen worker 26 - jepsen.util 16046 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:36:52,964] jepsen worker 1 - jepsen.util 15121 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:36:52,964] jepsen worker 68 - jepsen.util 14648 :info :cas [22 [22 3]] :RequestCanceledException INFO [2019-12-17 12:36:52,964] jepsen worker 77 - jepsen.util 16007 :info :cas [25 [31 49]] :RequestCanceledException INFO [2019-12-17 12:36:52,972] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64817,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:36:52,973] jepsen worker 9 - jepsen.util 11889 :invoke :read [3 nil] INFO [2019-12-17 12:36:52,973] jepsen worker 57 - jepsen.util 15087 :invoke :cas [19 [31 6]] INFO [2019-12-17 12:36:52,973] jepsen worker 52 - jepsen.util 10672 :invoke :read [17 nil] INFO [2019-12-17 12:36:52,973] jepsen worker 28 - jepsen.util 9298 :invoke :write [9 3] INFO [2019-12-17 12:36:52,973] jepsen worker 1 - jepsen.util 15121 :invoke :cas [0 [7 15]] INFO [2019-12-17 12:36:52,974] jepsen worker 26 - jepsen.util 16046 :invoke :read [8 nil] INFO [2019-12-17 12:36:52,981] jepsen worker 83 - jepsen.util 15563 :invoke :read [27 nil] INFO [2019-12-17 12:36:52,981] jepsen worker 68 - jepsen.util 14738 :invoke :read [22 nil] INFO [2019-12-17 12:36:52,982] jepsen worker 64 - jepsen.util 8614 :ok :read [21 43] INFO [2019-12-17 12:36:52,983] jepsen worker 77 - jepsen.util 16097 :invoke :cas [25 [48 2]] INFO [2019-12-17 12:36:52,984] jepsen worker 88 - jepsen.util 11518 :ok :read [29 23] INFO [2019-12-17 12:36:53,000] jepsen worker 44 - jepsen.util 15794 :info :cas [14 [0 38]] :RequestCanceledException INFO [2019-12-17 12:36:53,001] jepsen worker 82 - jepsen.util 15382 :info :write [27 29] :RequestCanceledException INFO [2019-12-17 12:36:53,001] jepsen worker 75 - jepsen.util 14745 :info :cas [25 [34 18]] :RequestCanceledException INFO [2019-12-17 12:36:53,016] jepsen worker 41 - jepsen.util 16331 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:36:53,016] jepsen worker 6 - jepsen.util 15756 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:36:53,016] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64820,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:36:53,017] jepsen worker 64 - jepsen.util 8614 :invoke :write [21 45] INFO [2019-12-17 12:36:53,017] jepsen worker 88 - jepsen.util 11518 :invoke :write [29 3] INFO [2019-12-17 12:36:53,018] jepsen worker 6 - jepsen.util 15756 :invoke :read [2 nil] INFO [2019-12-17 12:36:53,018] jepsen worker 41 - jepsen.util 16331 :invoke :cas [13 [0 11]] INFO [2019-12-17 12:36:53,018] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64820,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:36:53,030] jepsen worker 75 - jepsen.util 14835 :invoke :write [25 6] INFO [2019-12-17 12:36:53,032] jepsen worker 44 - jepsen.util 15884 :invoke :read [14 nil] INFO [2019-12-17 12:36:53,033] jepsen worker 16 - jepsen.util 16756 :info :write [5 38] :RequestCanceledException INFO [2019-12-17 12:36:53,033] jepsen worker 40 - jepsen.util 15700 :info :cas [13 [14 23]] :RequestCanceledException INFO [2019-12-17 12:36:53,033] jepsen worker 82 - jepsen.util 15472 :invoke :cas [27 [47 9]] INFO [2019-12-17 12:36:53,034] jepsen worker 81 - jepsen.util 14931 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:36:53,034] jepsen worker 66 - jepsen.util 15186 :info :write [22 21] :RequestCanceledException INFO [2019-12-17 12:36:53,037] jepsen worker 81 - jepsen.util 14931 :invoke :write [27 37] INFO [2019-12-17 12:36:53,046] jepsen worker 16 - jepsen.util 16846 :invoke :cas [5 [6 8]] INFO [2019-12-17 12:36:53,046] jepsen worker 40 - jepsen.util 15790 :invoke :write [13 46] INFO [2019-12-17 12:36:53,047] jepsen worker 66 - jepsen.util 15276 :invoke :write [22 44] INFO [2019-12-17 12:36:53,048] jepsen worker 22 - jepsen.util 15232 :info :cas [7 [49 3]] :RequestCanceledException INFO [2019-12-17 12:36:53,049] jepsen worker 39 - jepsen.util 14889 :info :write [13 12] :RequestCanceledException INFO [2019-12-17 12:36:53,049] jepsen worker 49 - jepsen.util 15259 :info :write [16 4] :RequestCanceledException INFO [2019-12-17 12:36:53,049] jepsen worker 86 - jepsen.util 15926 :info :write [28 12] :RequestCanceledException INFO [2019-12-17 12:36:53,056] jepsen worker 3 - jepsen.util 10713 :fail :cas [1 [36 48]] :ValueNotSwapFrom INFO [2019-12-17 12:36:53,066] jepsen worker 34 - jepsen.util 15694 :info :write [11 5] :RequestCanceledException INFO [2019-12-17 12:36:53,077] jepsen worker 4 - jepsen.util 10354 :ok :write [1 23] INFO [2019-12-17 12:36:53,078] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64828,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:36:53,080] jepsen worker 3 - jepsen.util 10713 :invoke :cas [1 [25 1]] INFO [2019-12-17 12:36:53,081] jepsen worker 4 - jepsen.util 10354 :invoke :cas [1 [33 42]] INFO [2019-12-17 12:36:53,088] jepsen worker 39 - jepsen.util 14979 :invoke :write [13 40] INFO [2019-12-17 12:36:53,089] jepsen worker 49 - jepsen.util 15349 :invoke :write [16 40] INFO [2019-12-17 12:36:53,090] jepsen worker 22 - jepsen.util 15322 :invoke :cas [7 [22 46]] INFO [2019-12-17 12:36:53,091] jepsen worker 34 - jepsen.util 15784 :invoke :read [11 nil] INFO [2019-12-17 12:36:53,092] jepsen worker 86 - jepsen.util 16016 :invoke :write [28 7] INFO [2019-12-17 12:36:53,094] jepsen worker 12 - jepsen.util 11532 :ok :write [4 39] INFO [2019-12-17 12:36:53,099] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64828,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:36:53,100] jepsen worker 12 - jepsen.util 11532 :invoke :cas [4 [29 8]] INFO [2019-12-17 12:36:53,100] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64833,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:36:53,100] jepsen worker 33 - jepsen.util 15153 :info :write [11 5] :RequestCanceledException INFO [2019-12-17 12:36:53,102] jepsen worker 60 - jepsen.util 16350 :info :write [20 41] :RequestCanceledException INFO [2019-12-17 12:36:53,102] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64833,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:36:53,111] jepsen worker 60 - jepsen.util 16440 :invoke :read [20 nil] INFO [2019-12-17 12:36:53,112] jepsen worker 33 - jepsen.util 15243 :invoke :write [11 8] INFO [2019-12-17 12:36:53,115] jepsen worker 9 - jepsen.util 11889 :ok :read [3 46] INFO [2019-12-17 12:36:53,116] jepsen worker 24 - jepsen.util 13884 :info :write [8 12] :RequestCanceledException INFO [2019-12-17 12:36:53,116] jepsen worker 42 - jepsen.util 16062 :info :write [14 48] :RequestCanceledException INFO [2019-12-17 12:36:53,333] jepsen worker 14 - jepsen.util 11084 :ok :write [4 45] INFO [2019-12-17 12:36:53,333] jepsen worker 37 - jepsen.util 10837 :ok :write [12 21] INFO [2019-12-17 12:36:53,333] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64868,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:36:53,333] jepsen worker 72 - jepsen.util 11322 :info :cas [24 [48 30]] :RequestCanceledException INFO [2019-12-17 12:36:53,333] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64860,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:36:53,333] jepsen worker 31 - jepsen.util 9571 :fail :read [10 nil] :RequestTimeoutException INFO [2019-12-17 12:36:53,333] jepsen worker 29 - jepsen.util 9479 :ok :write [9 18] INFO [2019-12-17 12:36:53,333] jepsen worker 46 - jepsen.util 10846 :info :write [15 30] :RequestCanceledException INFO [2019-12-17 12:36:53,333] jepsen worker 32 - jepsen.util 11462 :info :write [10 42] :RequestCanceledException INFO [2019-12-17 12:36:53,333] jepsen worker 74 - jepsen.util 10784 :info :write [24 11] :RequestCanceledException INFO [2019-12-17 12:36:53,334] jepsen worker 30 - jepsen.util 10470 :fail :read [10 nil] :RequestTimeoutException INFO [2019-12-17 12:36:53,334] jepsen worker 45 - jepsen.util 11295 :info :write [15 39] :RequestCanceledException INFO [2019-12-17 12:36:53,334] jepsen worker 27 - jepsen.util 10467 :info :write [9 19] :RequestCanceledException INFO [2019-12-17 12:36:53,334] jepsen worker 25 - jepsen.util 16315 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:36:53,335] jepsen worker 14 - jepsen.util 11084 :invoke :read [4 nil] INFO [2019-12-17 12:36:53,335] jepsen worker 37 - jepsen.util 10837 :invoke :write [12 46] INFO [2019-12-17 12:36:53,335] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64868,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:36:53,335] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64860,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:36:53,335] jepsen worker 9 - jepsen.util 11889 :invoke :read [3 nil] INFO [2019-12-17 12:36:53,335] jepsen worker 31 - jepsen.util 9571 :invoke :cas [10 [10 7]] INFO [2019-12-17 12:36:53,335] jepsen worker 29 - jepsen.util 9479 :invoke :write [9 46] INFO [2019-12-17 12:36:53,336] jepsen worker 30 - jepsen.util 10470 :invoke :cas [10 [31 25]] INFO [2019-12-17 12:36:53,336] jepsen worker 25 - jepsen.util 16315 :invoke :read [8 nil] INFO [2019-12-17 12:36:53,337] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64870,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:36:53,337] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64866,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:36:53,337] jepsen worker 46 - jepsen.util 10936 :invoke :write [15 24] INFO [2019-12-17 12:36:53,337] jepsen worker 24 - jepsen.util 13974 :invoke :read [8 nil] INFO [2019-12-17 12:36:53,337] jepsen worker 74 - jepsen.util 10874 :invoke :cas [24 [45 7]] INFO [2019-12-17 12:36:53,337] jepsen worker 32 - jepsen.util 11552 :invoke :read [10 nil] INFO [2019-12-17 12:36:53,338] jepsen worker 42 - jepsen.util 16152 :invoke :cas [14 [28 21]] INFO [2019-12-17 12:36:53,338] jepsen worker 45 - jepsen.util 11385 :invoke :cas [15 [10 48]] INFO [2019-12-17 12:36:53,338] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64870,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:36:53,338] jepsen worker 27 - jepsen.util 10557 :invoke :cas [9 [43 13]] INFO [2019-12-17 12:36:53,338] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64866,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:36:53,339] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64869,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:36:53,339] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64869,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:36:53,346] jepsen worker 72 - jepsen.util 11412 :invoke :cas [24 [13 0]] INFO [2019-12-17 12:36:53,350] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:36:53,350] jepsen nemesis - couchbase.util Rebalance check count: 510 INFO [2019-12-17 12:36:53,350] jepsen nemesis - couchbase.util Rebalance stuck count: 491 INFO [2019-12-17 12:36:53,355] jepsen worker 87 - jepsen.util 10887 :ok :read [29 23] INFO [2019-12-17 12:36:53,356] jepsen worker 87 - jepsen.util 10887 :invoke :write [29 21] INFO [2019-12-17 12:36:53,360] jepsen worker 51 - jepsen.util 10671 :fail :cas [17 [21 40]] :ValueNotSwapFrom INFO [2019-12-17 12:36:53,360] jepsen worker 51 - jepsen.util 10671 :invoke :write [17 35] INFO [2019-12-17 12:36:53,362] jepsen worker 54 - jepsen.util 11844 :ok :read [18 10] INFO [2019-12-17 12:36:53,362] jepsen worker 56 - jepsen.util 12026 :ok :read [18 10] INFO [2019-12-17 12:36:53,363] jepsen worker 54 - jepsen.util 11844 :invoke :read [18 nil] INFO [2019-12-17 12:36:53,363] jepsen worker 56 - jepsen.util 12026 :invoke :read [18 nil] INFO [2019-12-17 12:36:53,367] jepsen worker 79 - jepsen.util 11419 :fail :cas [26 [16 19]] :ValueNotSwapFrom INFO [2019-12-17 12:36:53,367] jepsen worker 79 - jepsen.util 11419 :invoke :write [26 37] INFO [2019-12-17 12:36:53,376] jepsen worker 36 - jepsen.util 12096 :fail :cas [12 [13 32]] :ValueNotSwapFrom INFO [2019-12-17 12:36:53,376] jepsen worker 36 - jepsen.util 12096 :invoke :cas [12 [34 33]] INFO [2019-12-17 12:36:53,409] jepsen worker 32 - jepsen.util 11552 :ok :read [10 30] INFO [2019-12-17 12:36:53,409] jepsen worker 32 - jepsen.util 11552 :invoke :write [10 25] INFO [2019-12-17 12:36:53,413] jepsen worker 53 - jepsen.util 10583 :ok :write [17 3] INFO [2019-12-17 12:36:53,413] jepsen worker 53 - jepsen.util 10583 :invoke :read [17 nil] INFO [2019-12-17 12:36:53,419] jepsen worker 9 - jepsen.util 11889 :ok :read [3 46] INFO [2019-12-17 12:36:53,419] jepsen worker 9 - jepsen.util 11889 :invoke :write [3 25] INFO [2019-12-17 12:36:53,422] jepsen worker 80 - jepsen.util 10970 :fail :cas [26 [44 10]] :ValueNotSwapFrom INFO [2019-12-17 12:36:53,422] jepsen worker 80 - jepsen.util 10970 :invoke :read [26 nil] INFO [2019-12-17 12:36:53,428] jepsen worker 74 - jepsen.util 10874 :fail :cas [24 [45 7]] :ValueNotSwapFrom INFO [2019-12-17 12:36:53,428] jepsen worker 74 - jepsen.util 10874 :invoke :cas [24 [34 15]] INFO [2019-12-17 12:36:53,436] jepsen worker 30 - jepsen.util 10470 :fail :cas [10 [31 25]] :ValueNotSwapFrom INFO [2019-12-17 12:36:53,436] jepsen worker 30 - jepsen.util 10470 :invoke :write [10 14] INFO [2019-12-17 12:36:53,443] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64904,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:36:53,443] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64904,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:36:53,443] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64907,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:36:53,443] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64907,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:36:53,446] jepsen worker 14 - jepsen.util 11084 :ok :read [4 45] INFO [2019-12-17 12:36:53,446] jepsen worker 14 - jepsen.util 11084 :invoke :read [4 nil] INFO [2019-12-17 12:36:53,451] jepsen worker 27 - jepsen.util 10557 :fail :cas [9 [43 13]] :ValueNotSwapFrom INFO [2019-12-17 12:36:53,451] jepsen worker 27 - jepsen.util 10557 :invoke :read [9 nil] INFO [2019-12-17 12:36:53,461] jepsen worker 89 - jepsen.util 11249 :ok :read [29 23] INFO [2019-12-17 12:36:53,462] jepsen worker 89 - jepsen.util 11249 :invoke :read [29 nil] INFO [2019-12-17 12:36:53,466] jepsen worker 78 - jepsen.util 11328 :fail :cas [26 [28 42]] :ValueNotSwapFrom INFO [2019-12-17 12:36:53,466] jepsen worker 78 - jepsen.util 11328 :invoke :read [26 nil] INFO [2019-12-17 12:36:53,487] jepsen worker 14 - jepsen.util 11084 :ok :read [4 45] INFO [2019-12-17 12:36:53,487] jepsen worker 14 - jepsen.util 11084 :invoke :cas [4 [41 49]] INFO [2019-12-17 12:36:53,495] jepsen worker 38 - jepsen.util 11738 :fail :cas [12 [35 14]] :ValueNotSwapFrom INFO [2019-12-17 12:36:53,495] jepsen worker 38 - jepsen.util 11738 :invoke :read [12 nil] INFO [2019-12-17 12:36:53,499] jepsen worker 29 - jepsen.util 9479 :ok :write [9 46] INFO [2019-12-17 12:36:53,499] jepsen worker 29 - jepsen.util 9479 :invoke :cas [9 [7 15]] INFO [2019-12-17 12:36:53,511] jepsen worker 74 - jepsen.util 10874 :fail :cas [24 [34 15]] :ValueNotSwapFrom INFO [2019-12-17 12:36:53,511] jepsen worker 74 - jepsen.util 10874 :invoke :cas [24 [1 22]] INFO [2019-12-17 12:36:53,526] jepsen worker 29 - jepsen.util 9479 :fail :cas [9 [7 15]] :ValueNotSwapFrom INFO [2019-12-17 12:36:53,526] jepsen worker 29 - jepsen.util 9479 :invoke :write [9 31] INFO [2019-12-17 12:36:53,547] jepsen worker 11 - jepsen.util 10811 :info :write [3 27] :RequestCanceledException INFO [2019-12-17 12:36:53,549] jepsen worker 11 - jepsen.util 10901 :invoke :read [3 nil] INFO [2019-12-17 12:36:53,553] jepsen worker 14 - jepsen.util 11084 :fail :cas [4 [41 49]] :ValueNotSwapFrom INFO [2019-12-17 12:36:53,553] jepsen worker 14 - jepsen.util 11084 :invoke :write [4 39] INFO [2019-12-17 12:36:53,565] jepsen worker 55 - jepsen.util 11845 :fail :cas [18 [6 1]] :ValueNotSwapFrom INFO [2019-12-17 12:36:53,565] jepsen worker 55 - jepsen.util 11845 :invoke :write [18 17] INFO [2019-12-17 12:36:53,568] jepsen worker 74 - jepsen.util 10874 :fail :cas [24 [1 22]] :ValueNotSwapFrom INFO [2019-12-17 12:36:53,568] jepsen worker 74 - jepsen.util 10874 :invoke :cas [24 [6 0]] INFO [2019-12-17 12:36:53,605] jepsen worker 12 - jepsen.util 11532 :fail :cas [4 [29 8]] :ValueNotSwapFrom INFO [2019-12-17 12:36:53,605] jepsen worker 12 - jepsen.util 11532 :invoke :cas [4 [41 43]] INFO [2019-12-17 12:36:53,605] jepsen worker 64 - jepsen.util 8614 :ok :write [21 45] INFO [2019-12-17 12:36:53,606] jepsen worker 64 - jepsen.util 8614 :invoke :write [21 0] INFO [2019-12-17 12:36:53,617] jepsen worker 20 - jepsen.util 10730 :info :write [6 34] :RequestCanceledException INFO [2019-12-17 12:36:53,619] jepsen worker 20 - jepsen.util 10820 :invoke :write [6 3] INFO [2019-12-17 12:36:53,630] jepsen worker 69 - jepsen.util 11409 :info :write [23 12] :RequestCanceledException INFO [2019-12-17 12:36:53,631] jepsen worker 69 - jepsen.util 11499 :invoke :cas [23 [39 36]] INFO [2019-12-17 12:36:53,645] jepsen worker 29 - jepsen.util 9479 :ok :write [9 31] INFO [2019-12-17 12:36:53,645] jepsen worker 29 - jepsen.util 9479 :invoke :read [9 nil] INFO [2019-12-17 12:36:53,657] jepsen worker 56 - jepsen.util 12026 :ok :read [18 10] INFO [2019-12-17 12:36:53,657] jepsen worker 56 - jepsen.util 12026 :invoke :read [18 nil] INFO [2019-12-17 12:36:53,661] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64928,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:36:53,661] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64928,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:36:53,661] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64931,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:36:53,661] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64931,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:36:53,664] jepsen worker 63 - jepsen.util 9963 :fail :read [21 nil] :RequestTimeoutException INFO [2019-12-17 12:36:53,664] jepsen worker 63 - jepsen.util 9963 :invoke :write [21 18] INFO [2019-12-17 12:36:53,672] jepsen worker 70 - jepsen.util 11860 :ok :read [23 30] INFO [2019-12-17 12:36:53,672] jepsen worker 70 - jepsen.util 11860 :invoke :write [23 40] INFO [2019-12-17 12:36:53,682] jepsen worker 29 - jepsen.util 9479 :ok :read [9 31] INFO [2019-12-17 12:36:53,682] jepsen worker 29 - jepsen.util 9479 :invoke :cas [9 [44 47]] INFO [2019-12-17 12:36:53,699] jepsen worker 65 - jepsen.util 9605 :info :write [21 23] :RequestCanceledException INFO [2019-12-17 12:36:53,699] jepsen worker 0 - jepsen.util 16020 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:36:53,699] jepsen worker 0 - jepsen.util 16020 :invoke :cas [0 [2 5]] INFO [2019-12-17 12:36:53,700] jepsen worker 65 - jepsen.util 9695 :invoke :read [21 nil] INFO [2019-12-17 12:36:53,712] jepsen worker 29 - jepsen.util 9479 :fail :cas [9 [44 47]] :ValueNotSwapFrom INFO [2019-12-17 12:36:53,712] jepsen worker 29 - jepsen.util 9479 :invoke :cas [9 [3 1]] INFO [2019-12-17 12:36:53,717] jepsen worker 85 - jepsen.util 16105 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:36:53,717] jepsen worker 85 - jepsen.util 16105 :invoke :cas [28 [6 29]] INFO [2019-12-17 12:36:53,724] jepsen worker 65 - jepsen.util 9695 :ok :read [21 45] INFO [2019-12-17 12:36:53,724] jepsen worker 65 - jepsen.util 9695 :invoke :cas [21 [32 32]] INFO [2019-12-17 12:36:53,732] jepsen worker 19 - jepsen.util 11449 :info :cas [6 [9 21]] :RequestCanceledException INFO [2019-12-17 12:36:53,734] jepsen worker 19 - jepsen.util 11539 :invoke :cas [6 [48 46]] INFO [2019-12-17 12:36:53,747] jepsen worker 10 - jepsen.util 10900 :info :write [3 36] :RequestCanceledException INFO [2019-12-17 12:36:53,748] jepsen worker 10 - jepsen.util 10990 :invoke :cas [3 [45 37]] INFO [2019-12-17 12:36:53,756] jepsen worker 89 - jepsen.util 11249 :ok :read [29 23] INFO [2019-12-17 12:36:53,756] jepsen worker 89 - jepsen.util 11249 :invoke :cas [29 [30 13]] INFO [2019-12-17 12:36:53,778] jepsen worker 64 - jepsen.util 8614 :ok :write [21 0] INFO [2019-12-17 12:36:53,779] jepsen worker 64 - jepsen.util 8614 :invoke :cas [21 [47 9]] INFO [2019-12-17 12:36:53,803] jepsen worker 80 - jepsen.util 10970 :ok :read [26 1] INFO [2019-12-17 12:36:53,803] jepsen worker 80 - jepsen.util 10970 :invoke :cas [26 [44 36]] INFO [2019-12-17 12:36:53,806] jepsen worker 78 - jepsen.util 11328 :ok :read [26 1] INFO [2019-12-17 12:36:53,806] jepsen worker 78 - jepsen.util 11328 :invoke :write [26 12] INFO [2019-12-17 12:36:53,814] jepsen worker 38 - jepsen.util 11738 :ok :read [12 21] INFO [2019-12-17 12:36:53,814] jepsen worker 38 - jepsen.util 11738 :invoke :read [12 nil] INFO [2019-12-17 12:36:53,826] jepsen worker 9 - jepsen.util 11889 :ok :write [3 25] INFO [2019-12-17 12:36:53,827] jepsen worker 9 - jepsen.util 11889 :invoke :cas [3 [12 33]] INFO [2019-12-17 12:36:53,828] jepsen worker 29 - jepsen.util 9479 :fail :cas [9 [3 1]] :ValueNotSwapFrom INFO [2019-12-17 12:36:53,828] jepsen worker 29 - jepsen.util 9479 :invoke :write [9 27] INFO [2019-12-17 12:36:53,832] jepsen worker 73 - jepsen.util 10423 :fail :read [24 nil] :RequestTimeoutException INFO [2019-12-17 12:36:53,833] jepsen worker 73 - jepsen.util 10423 :invoke :write [24 17] INFO [2019-12-17 12:36:53,842] jepsen worker 46 - jepsen.util 10936 :ok :write [15 24] INFO [2019-12-17 12:36:53,842] jepsen worker 46 - jepsen.util 10936 :invoke :cas [15 [45 35]] INFO [2019-12-17 12:36:53,846] jepsen worker 65 - jepsen.util 9695 :fail :cas [21 [32 32]] :ValueNotSwapFrom INFO [2019-12-17 12:36:53,847] jepsen worker 65 - jepsen.util 9695 :invoke :cas [21 [26 2]] INFO [2019-12-17 12:36:53,858] jepsen worker 64 - jepsen.util 8614 :fail :cas [21 [47 9]] :ValueNotSwapFrom INFO [2019-12-17 12:36:53,858] jepsen worker 64 - jepsen.util 8614 :invoke :read [21 nil] INFO [2019-12-17 12:36:53,865] jepsen worker 27 - jepsen.util 10557 :ok :read [9 31] INFO [2019-12-17 12:36:53,866] jepsen worker 27 - jepsen.util 10557 :invoke :write [9 31] INFO [2019-12-17 12:36:53,868] jepsen worker 12 - jepsen.util 11532 :fail :cas [4 [41 43]] :ValueNotSwapFrom INFO [2019-12-17 12:36:53,868] jepsen worker 12 - jepsen.util 11532 :invoke :read [4 nil] INFO [2019-12-17 12:36:53,872] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64956,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:36:53,873] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64956,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:36:53,873] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64960,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:36:53,873] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64960,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:36:53,878] jepsen worker 4 - jepsen.util 10354 :fail :cas [1 [33 42]] :ValueNotSwapFrom INFO [2019-12-17 12:36:53,878] jepsen worker 4 - jepsen.util 10354 :invoke :write [1 13] INFO [2019-12-17 12:36:53,901] jepsen worker 19 - jepsen.util 11539 :fail :cas [6 [48 46]] :ValueNotSwapFrom INFO [2019-12-17 12:36:53,901] jepsen worker 19 - jepsen.util 11539 :invoke :read [6 nil] INFO [2019-12-17 12:36:53,906] jepsen worker 56 - jepsen.util 12026 :ok :read [18 10] INFO [2019-12-17 12:36:53,906] jepsen worker 56 - jepsen.util 12026 :invoke :cas [18 [11 18]] INFO [2019-12-17 12:36:53,909] jepsen worker 32 - jepsen.util 11552 :ok :write [10 25] INFO [2019-12-17 12:36:53,909] jepsen worker 32 - jepsen.util 11552 :invoke :cas [10 [17 14]] INFO [2019-12-17 12:36:53,988] jepsen worker 46 - jepsen.util 10936 :fail :cas [15 [45 35]] :ValueNotSwapFrom INFO [2019-12-17 12:36:53,988] jepsen worker 46 - jepsen.util 10936 :invoke :cas [15 [27 6]] INFO [2019-12-17 12:36:53,989] jepsen worker 63 - jepsen.util 9963 :ok :write [21 18] INFO [2019-12-17 12:36:53,989] jepsen worker 63 - jepsen.util 9963 :invoke :write [21 37] INFO [2019-12-17 12:36:53,999] jepsen worker 67 - jepsen.util 15367 :info :cas [22 [28 39]] :RequestCanceledException INFO [2019-12-17 12:36:54,000] jepsen worker 67 - jepsen.util 15457 :invoke :read [22 nil] INFO [2019-12-17 12:36:54,009] jepsen worker 89 - jepsen.util 11249 :fail :cas [29 [30 13]] :ValueNotSwapFrom INFO [2019-12-17 12:36:54,009] jepsen worker 89 - jepsen.util 11249 :invoke :cas [29 [1 36]] INFO [2019-12-17 12:36:54,012] jepsen worker 52 - jepsen.util 10672 :ok :read [17 3] INFO [2019-12-17 12:36:54,012] jepsen worker 52 - jepsen.util 10672 :invoke :cas [17 [17 13]] INFO [2019-12-17 12:36:54,017] jepsen worker 48 - jepsen.util 14088 :info :cas [16 [25 6]] :RequestCanceledException INFO [2019-12-17 12:36:54,019] jepsen worker 48 - jepsen.util 14178 :invoke :cas [16 [28 14]] INFO [2019-12-17 12:36:54,032] jepsen worker 8 - jepsen.util 15128 :info :cas [2 [28 33]] :RequestCanceledException INFO [2019-12-17 12:36:54,033] jepsen worker 8 - jepsen.util 15218 :invoke :cas [2 [1 45]] INFO [2019-12-17 12:36:54,047] jepsen worker 7 - jepsen.util 15307 :info :write [2 33] :RequestCanceledException INFO [2019-12-17 12:36:54,048] jepsen worker 7 - jepsen.util 15397 :invoke :write [2 47] INFO [2019-12-17 12:36:54,056] jepsen worker 10 - jepsen.util 10990 :fail :cas [3 [45 37]] :ValueNotSwapFrom INFO [2019-12-17 12:36:54,056] jepsen worker 10 - jepsen.util 10990 :invoke :read [3 nil] INFO [2019-12-17 12:36:54,057] jepsen worker 29 - jepsen.util 9479 :ok :write [9 27] INFO [2019-12-17 12:36:54,057] jepsen worker 29 - jepsen.util 9479 :invoke :cas [9 [41 48]] INFO [2019-12-17 12:36:54,064] jepsen worker 61 - jepsen.util 16441 :info :cas [20 [24 28]] :RequestCanceledException INFO [2019-12-17 12:36:54,065] jepsen worker 61 - jepsen.util 16531 :invoke :write [20 30] INFO [2019-12-17 12:36:54,082] jepsen worker 12 - jepsen.util 11532 :ok :read [4 45] INFO [2019-12-17 12:36:54,083] jepsen worker 12 - jepsen.util 11532 :invoke :cas [4 [17 3]] INFO [2019-12-17 12:36:54,085] jepsen worker 64 - jepsen.util 8614 :ok :read [21 0] INFO [2019-12-17 12:36:54,085] jepsen worker 64 - jepsen.util 8614 :invoke :read [21 nil] INFO [2019-12-17 12:36:54,092] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64971,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:36:54,092] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64971,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:36:54,094] jepsen worker 29 - jepsen.util 9479 :fail :cas [9 [41 48]] :ValueNotSwapFrom INFO [2019-12-17 12:36:54,095] jepsen worker 29 - jepsen.util 9479 :invoke :cas [9 [31 10]] INFO [2019-12-17 12:36:54,116] jepsen worker 11 - jepsen.util 10901 :ok :read [3 25] INFO [2019-12-17 12:36:54,116] jepsen worker 11 - jepsen.util 10901 :invoke :cas [3 [22 14]] INFO [2019-12-17 12:36:54,119] jepsen worker 3 - jepsen.util 10713 :fail :cas [1 [25 1]] :ValueNotSwapFrom INFO [2019-12-17 12:36:54,120] jepsen worker 3 - jepsen.util 10713 :invoke :cas [1 [37 21]] INFO [2019-12-17 12:36:54,132] jepsen worker 15 - jepsen.util 14325 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:36:54,133] jepsen worker 15 - jepsen.util 14325 :invoke :cas [5 [36 3]] INFO [2019-12-17 12:36:54,148] jepsen worker 2 - jepsen.util 14852 :info :cas [0 [42 0]] :RequestCanceledException INFO [2019-12-17 12:36:54,150] jepsen worker 2 - jepsen.util 14942 :invoke :read [0 nil] INFO [2019-12-17 12:36:54,157] jepsen worker 64 - jepsen.util 8614 :ok :read [21 18] INFO [2019-12-17 12:36:54,157] jepsen worker 32 - jepsen.util 11552 :fail :cas [10 [17 14]] :ValueNotSwapFrom INFO [2019-12-17 12:36:54,158] jepsen worker 64 - jepsen.util 8614 :invoke :write [21 22] INFO [2019-12-17 12:36:54,158] jepsen worker 32 - jepsen.util 11552 :invoke :read [10 nil] INFO [2019-12-17 12:36:54,164] jepsen worker 23 - jepsen.util 15323 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:36:54,164] jepsen worker 23 - jepsen.util 15323 :invoke :write [7 44] INFO [2019-12-17 12:36:54,184] jepsen worker 52 - jepsen.util 10672 :fail :cas [17 [17 13]] :ValueNotSwapFrom INFO [2019-12-17 12:36:54,184] jepsen worker 52 - jepsen.util 10672 :invoke :write [17 44] INFO [2019-12-17 12:36:54,199] jepsen worker 17 - jepsen.util 15317 :info :write [5 1] :RequestCanceledException INFO [2019-12-17 12:36:54,199] jepsen worker 84 - jepsen.util 15024 :info :cas [28 [18 48]] :RequestCanceledException INFO [2019-12-17 12:36:54,200] jepsen worker 17 - jepsen.util 15407 :invoke :read [5 nil] INFO [2019-12-17 12:36:54,200] jepsen worker 84 - jepsen.util 15114 :invoke :cas [28 [11 15]] INFO [2019-12-17 12:36:54,203] jepsen worker 11 - jepsen.util 10901 :fail :cas [3 [22 14]] :ValueNotSwapFrom INFO [2019-12-17 12:36:54,204] jepsen worker 11 - jepsen.util 10901 :invoke :write [3 10] INFO [2019-12-17 12:36:54,216] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64997,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:36:54,216] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64997,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:36:54,216] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64998,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:36:54,216] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64998,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:36:54,216] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64999,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:36:54,216] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":64999,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:36:54,217] jepsen worker 62 - jepsen.util 15992 :info :write [20 5] :RequestCanceledException INFO [2019-12-17 12:36:54,217] jepsen worker 58 - jepsen.util 15538 :info :cas [19 [8 26]] :RequestCanceledException INFO [2019-12-17 12:36:54,217] jepsen worker 58 - jepsen.util 15628 :invoke :write [19 24] INFO [2019-12-17 12:36:54,217] jepsen worker 62 - jepsen.util 16082 :invoke :read [20 nil] INFO [2019-12-17 12:36:54,233] jepsen worker 5 - jepsen.util 10895 :info :write [1 0] :RequestCanceledException INFO [2019-12-17 12:36:54,233] jepsen worker 43 - jepsen.util 15433 :info :cas [14 [24 45]] :RequestCanceledException INFO [2019-12-17 12:36:54,233] jepsen worker 50 - jepsen.util 15710 :info :write [16 24] :RequestCanceledException INFO [2019-12-17 12:36:54,234] jepsen worker 43 - jepsen.util 15523 :invoke :cas [14 [45 24]] INFO [2019-12-17 12:36:54,234] jepsen worker 50 - jepsen.util 15800 :invoke :cas [16 [47 11]] INFO [2019-12-17 12:36:54,234] jepsen worker 5 - jepsen.util 10985 :invoke :cas [1 [28 34]] INFO [2019-12-17 12:36:54,237] jepsen worker 20 - jepsen.util 10820 :ok :write [6 3] INFO [2019-12-17 12:36:54,237] jepsen worker 20 - jepsen.util 10820 :invoke :cas [6 [37 18]] INFO [2019-12-17 12:36:54,269] jepsen worker 38 - jepsen.util 11738 :ok :read [12 21] INFO [2019-12-17 12:36:54,269] jepsen worker 21 - jepsen.util 14871 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:36:54,269] jepsen worker 38 - jepsen.util 11738 :invoke :read [12 nil] INFO [2019-12-17 12:36:54,269] jepsen worker 21 - jepsen.util 14871 :invoke :read [7 nil] INFO [2019-12-17 12:36:54,287] jepsen worker 69 - jepsen.util 11499 :fail :cas [23 [39 36]] :ValueNotSwapFrom INFO [2019-12-17 12:36:54,287] jepsen worker 69 - jepsen.util 11499 :invoke :cas [23 [30 40]] INFO [2019-12-17 12:36:54,291] jepsen worker 29 - jepsen.util 9479 :fail :cas [9 [31 10]] :ValueNotSwapFrom INFO [2019-12-17 12:36:54,291] jepsen worker 29 - jepsen.util 9479 :invoke :read [9 nil] INFO [2019-12-17 12:36:54,308] jepsen worker 9 - jepsen.util 11889 :fail :cas [3 [12 33]] :ValueNotSwapFrom INFO [2019-12-17 12:36:54,309] jepsen worker 9 - jepsen.util 11889 :invoke :cas [3 [48 16]] INFO [2019-12-17 12:36:54,326] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65005,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:36:54,327] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65005,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:36:54,327] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65006,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:36:54,327] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65006,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:36:54,327] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65007,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:36:54,327] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65007,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:36:54,328] jepsen worker 27 - jepsen.util 10557 :ok :write [9 31] INFO [2019-12-17 12:36:54,328] jepsen worker 32 - jepsen.util 11552 :ok :read [10 25] INFO [2019-12-17 12:36:54,328] jepsen worker 27 - jepsen.util 10557 :invoke :read [9 nil] INFO [2019-12-17 12:36:54,328] jepsen worker 32 - jepsen.util 11552 :invoke :write [10 23] INFO [2019-12-17 12:36:54,337] jepsen worker 56 - jepsen.util 12026 :fail :cas [18 [11 18]] :ValueNotSwapFrom INFO [2019-12-17 12:36:54,337] jepsen worker 56 - jepsen.util 12026 :invoke :cas [18 [47 37]] INFO [2019-12-17 12:36:54,375] jepsen worker 63 - jepsen.util 9963 :ok :write [21 37] INFO [2019-12-17 12:36:54,375] jepsen worker 63 - jepsen.util 9963 :invoke :read [21 nil] INFO [2019-12-17 12:36:54,396] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:36:54,396] jepsen nemesis - couchbase.util Rebalance check count: 511 INFO [2019-12-17 12:36:54,396] jepsen nemesis - couchbase.util Rebalance stuck count: 492 INFO [2019-12-17 12:36:54,405] jepsen worker 12 - jepsen.util 11532 :fail :cas [4 [17 3]] :ValueNotSwapFrom INFO [2019-12-17 12:36:54,405] jepsen worker 12 - jepsen.util 11532 :invoke :write [4 15] INFO [2019-12-17 12:36:54,422] jepsen worker 80 - jepsen.util 10970 :fail :cas [26 [44 36]] :ValueNotSwapFrom INFO [2019-12-17 12:36:54,422] jepsen worker 80 - jepsen.util 10970 :invoke :write [26 23] INFO [2019-12-17 12:36:54,423] jepsen worker 54 - jepsen.util 11844 :ok :read [18 10] INFO [2019-12-17 12:36:54,423] jepsen worker 54 - jepsen.util 11844 :invoke :write [18 22] INFO [2019-12-17 12:36:54,446] jepsen worker 36 - jepsen.util 12096 :fail :cas [12 [34 33]] :ValueNotSwapFrom INFO [2019-12-17 12:36:54,446] jepsen worker 36 - jepsen.util 12096 :invoke :cas [12 [48 2]] INFO [2019-12-17 12:36:54,469] jepsen worker 89 - jepsen.util 11249 :fail :cas [29 [1 36]] :ValueNotSwapFrom INFO [2019-12-17 12:36:54,469] jepsen worker 89 - jepsen.util 11249 :invoke :read [29 nil] INFO [2019-12-17 12:36:54,489] jepsen worker 53 - jepsen.util 10583 :ok :read [17 3] INFO [2019-12-17 12:36:54,498] jepsen worker 13 - jepsen.util 10543 :info :write [4 11] :RequestCanceledException INFO [2019-12-17 12:36:54,517] jepsen worker 18 - jepsen.util 11268 :info :write [6 40] :RequestCanceledException INFO [2019-12-17 12:36:54,530] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65029,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:36:54,533] jepsen worker 47 - jepsen.util 11927 :info :cas [15 [32 47]] :RequestCanceledException INFO [2019-12-17 12:36:54,535] jepsen worker 29 - jepsen.util 9479 :ok :read [9 27] INFO [2019-12-17 12:36:54,538] jepsen worker 27 - jepsen.util 10557 :ok :read [9 31] INFO [2019-12-17 12:36:54,555] jepsen worker 5 - jepsen.util 10985 :fail :cas [1 [28 34]] :ValueNotSwapFrom INFO [2019-12-17 12:36:54,559] jepsen worker 20 - jepsen.util 10820 :fail :cas [6 [37 18]] :ValueNotSwapFrom INFO [2019-12-17 12:36:54,560] jepsen worker 63 - jepsen.util 9963 :ok :read [21 37] INFO [2019-12-17 12:36:54,565] jepsen worker 78 - jepsen.util 11328 :ok :write [26 12] INFO [2019-12-17 12:36:54,595] jepsen worker 38 - jepsen.util 11738 :ok :read [12 21] INFO [2019-12-17 12:36:54,607] jepsen worker 9 - jepsen.util 11889 :fail :cas [3 [48 16]] :ValueNotSwapFrom INFO [2019-12-17 12:36:54,620] jepsen worker 37 - jepsen.util 10837 :ok :write [12 46] INFO [2019-12-17 12:36:54,626] jepsen worker 64 - jepsen.util 8614 :ok :write [21 22] INFO [2019-12-17 12:36:54,651] jepsen worker 11 - jepsen.util 10901 :ok :write [3 10] INFO [2019-12-17 12:36:54,654] jepsen worker 71 - jepsen.util 12311 :ok :write [23 32] INFO [2019-12-17 12:36:54,656] jepsen worker 14 - jepsen.util 11084 :ok :write [4 39] INFO [2019-12-17 12:36:54,669] jepsen worker 55 - jepsen.util 11845 :ok :write [18 17] INFO [2019-12-17 12:36:54,688] jepsen worker 56 - jepsen.util 12026 :fail :cas [18 [47 37]] :ValueNotSwapFrom INFO [2019-12-17 12:36:54,715] jepsen worker 89 - jepsen.util 11249 :ok :read [29 23] INFO [2019-12-17 12:36:54,749] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65037,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:36:54,767] jepsen worker 3 - jepsen.util 10713 :fail :cas [1 [37 21]] :ValueNotSwapFrom INFO [2019-12-17 12:36:54,806] jepsen worker 12 - jepsen.util 11532 :ok :write [4 15] INFO [2019-12-17 12:36:54,818] jepsen worker 79 - jepsen.util 11419 :ok :write [26 37] INFO [2019-12-17 12:36:54,838] jepsen worker 52 - jepsen.util 10672 :ok :write [17 44] INFO [2019-12-17 12:36:54,879] jepsen worker 69 - jepsen.util 11499 :fail :cas [23 [30 40]] :ValueNotSwapFrom INFO [2019-12-17 12:36:54,909] jepsen worker 80 - jepsen.util 10970 :ok :write [26 23] INFO [2019-12-17 12:36:54,928] jepsen worker 88 - jepsen.util 11518 :ok :write [29 3] INFO [2019-12-17 12:36:54,950] jepsen worker 4 - jepsen.util 10354 :ok :write [1 13] INFO [2019-12-17 12:36:54,969] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65044,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:36:54,974] jepsen worker 87 - jepsen.util 10887 :ok :write [29 21] INFO [2019-12-17 12:36:54,987] jepsen worker 70 - jepsen.util 11860 :ok :write [23 40] INFO [2019-12-17 12:36:55,025] jepsen worker 54 - jepsen.util 11844 :ok :write [18 22] INFO [2019-12-17 12:36:55,033] jepsen worker 35 - jepsen.util 15245 :info :write [11 0] :RequestCanceledException INFO [2019-12-17 12:36:55,033] jepsen worker 83 - jepsen.util 15563 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:36:55,047] jepsen worker 59 - jepsen.util 14639 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:36:55,047] jepsen worker 57 - jepsen.util 15087 :info :cas [19 [31 6]] :RequestCanceledException INFO [2019-12-17 12:36:55,047] jepsen worker 44 - jepsen.util 15884 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:36:55,064] jepsen worker 82 - jepsen.util 15472 :info :cas [27 [47 9]] :RequestCanceledException INFO [2019-12-17 12:36:55,064] jepsen worker 1 - jepsen.util 15121 :info :cas [0 [7 15]] :RequestCanceledException INFO [2019-12-17 12:36:55,064] jepsen worker 76 - jepsen.util 15736 :info :cas [25 [32 6]] :RequestCanceledException INFO [2019-12-17 12:36:55,064] jepsen worker 26 - jepsen.util 16046 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:36:55,064] jepsen worker 28 - jepsen.util 9298 :info :write [9 3] :RequestCanceledException INFO [2019-12-17 12:36:55,065] jepsen worker 36 - jepsen.util 12096 :fail :cas [12 [48 2]] :ValueNotSwapFrom INFO [2019-12-17 12:36:55,078] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65049,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:36:55,079] jepsen worker 51 - jepsen.util 10671 :ok :write [17 35] INFO [2019-12-17 12:36:55,099] jepsen worker 68 - jepsen.util 14738 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:36:55,116] jepsen worker 41 - jepsen.util 16331 :info :cas [13 [0 11]] :RequestCanceledException INFO [2019-12-17 12:36:55,116] jepsen worker 77 - jepsen.util 16097 :info :cas [25 [48 2]] :RequestCanceledException INFO [2019-12-17 12:36:55,116] jepsen worker 66 - jepsen.util 15276 :info :write [22 44] :RequestCanceledException INFO [2019-12-17 12:36:55,116] jepsen worker 6 - jepsen.util 15756 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:36:55,132] jepsen worker 16 - jepsen.util 16846 :info :cas [5 [6 8]] :RequestCanceledException INFO [2019-12-17 12:36:55,132] jepsen worker 34 - jepsen.util 15784 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:36:55,132] jepsen worker 75 - jepsen.util 14835 :info :write [25 6] :RequestCanceledException INFO [2019-12-17 12:36:55,132] jepsen worker 81 - jepsen.util 14931 :info :write [27 37] :RequestCanceledException INFO [2019-12-17 12:36:55,163] jepsen worker 22 - jepsen.util 15322 :info :cas [7 [22 46]] :RequestCanceledException INFO [2019-12-17 12:36:55,199] jepsen worker 40 - jepsen.util 15790 :info :write [13 46] :RequestCanceledException INFO [2019-12-17 12:36:55,201] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65055,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:36:55,259] jepsen worker 53 - jepsen.util 10583 :invoke :cas [17 [28 7]] INFO [2019-12-17 12:36:55,259] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65029,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:36:55,259] jepsen worker 29 - jepsen.util 9479 :invoke :cas [9 [13 2]] INFO [2019-12-17 12:36:55,259] jepsen worker 27 - jepsen.util 10557 :invoke :write [9 15] INFO [2019-12-17 12:36:55,259] jepsen worker 5 - jepsen.util 10985 :invoke :write [1 45] INFO [2019-12-17 12:36:55,259] jepsen worker 20 - jepsen.util 10820 :invoke :write [6 32] INFO [2019-12-17 12:36:55,260] jepsen worker 63 - jepsen.util 9963 :invoke :write [21 31] INFO [2019-12-17 12:36:55,260] jepsen worker 78 - jepsen.util 11328 :invoke :write [26 12] INFO [2019-12-17 12:36:55,260] jepsen worker 38 - jepsen.util 11738 :invoke :read [12 nil] INFO [2019-12-17 12:36:55,260] jepsen worker 9 - jepsen.util 11889 :invoke :cas [3 [42 35]] INFO [2019-12-17 12:36:55,260] jepsen worker 37 - jepsen.util 10837 :invoke :write [12 22] INFO [2019-12-17 12:36:55,260] jepsen worker 64 - jepsen.util 8614 :invoke :write [21 33] INFO [2019-12-17 12:36:55,260] jepsen worker 18 - jepsen.util 11358 :invoke :write [6 25] INFO [2019-12-17 12:36:55,260] jepsen worker 13 - jepsen.util 10633 :invoke :cas [4 [40 44]] INFO [2019-12-17 12:36:55,260] jepsen worker 11 - jepsen.util 10901 :invoke :read [3 nil] INFO [2019-12-17 12:36:55,260] jepsen worker 71 - jepsen.util 12311 :invoke :read [23 nil] INFO [2019-12-17 12:36:55,260] jepsen worker 14 - jepsen.util 11084 :invoke :write [4 35] INFO [2019-12-17 12:36:55,260] jepsen worker 47 - jepsen.util 12017 :invoke :cas [15 [21 27]] INFO [2019-12-17 12:36:55,260] jepsen worker 55 - jepsen.util 11845 :invoke :write [18 28] INFO [2019-12-17 12:36:55,260] jepsen worker 56 - jepsen.util 12026 :invoke :read [18 nil] INFO [2019-12-17 12:36:55,260] jepsen worker 89 - jepsen.util 11249 :invoke :cas [29 [8 10]] INFO [2019-12-17 12:36:55,260] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65037,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:36:55,260] jepsen worker 3 - jepsen.util 10713 :invoke :read [1 nil] INFO [2019-12-17 12:36:55,260] jepsen worker 12 - jepsen.util 11532 :invoke :cas [4 [23 19]] INFO [2019-12-17 12:36:55,261] jepsen worker 79 - jepsen.util 11419 :invoke :cas [26 [14 32]] INFO [2019-12-17 12:36:55,261] jepsen worker 52 - jepsen.util 10672 :invoke :read [17 nil] INFO [2019-12-17 12:36:55,261] jepsen worker 69 - jepsen.util 11499 :invoke :cas [23 [43 38]] INFO [2019-12-17 12:36:55,261] jepsen worker 80 - jepsen.util 10970 :invoke :cas [26 [34 29]] INFO [2019-12-17 12:36:55,261] jepsen worker 88 - jepsen.util 11518 :invoke :read [29 nil] INFO [2019-12-17 12:36:55,261] jepsen worker 4 - jepsen.util 10354 :invoke :write [1 1] INFO [2019-12-17 12:36:55,261] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65044,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:36:55,261] jepsen worker 87 - jepsen.util 10887 :invoke :write [29 19] INFO [2019-12-17 12:36:55,261] jepsen worker 70 - jepsen.util 11860 :invoke :cas [23 [30 29]] INFO [2019-12-17 12:36:55,261] jepsen worker 54 - jepsen.util 11844 :invoke :read [18 nil] INFO [2019-12-17 12:36:55,262] jepsen worker 83 - jepsen.util 15563 :invoke :read [27 nil] INFO [2019-12-17 12:36:55,262] jepsen worker 59 - jepsen.util 14639 :invoke :write [19 41] INFO [2019-12-17 12:36:55,262] jepsen worker 44 - jepsen.util 15884 :invoke :cas [14 [22 5]] INFO [2019-12-17 12:36:55,262] jepsen worker 26 - jepsen.util 16046 :invoke :write [8 4] INFO [2019-12-17 12:36:55,262] jepsen worker 36 - jepsen.util 12096 :invoke :cas [12 [46 41]] INFO [2019-12-17 12:36:55,262] jepsen worker 51 - jepsen.util 10671 :invoke :cas [17 [44 38]] INFO [2019-12-17 12:36:55,263] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65049,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:36:55,263] jepsen worker 68 - jepsen.util 14738 :invoke :cas [22 [31 36]] INFO [2019-12-17 12:36:55,263] jepsen worker 28 - jepsen.util 9388 :invoke :read [9 nil] INFO [2019-12-17 12:36:55,264] jepsen worker 34 - jepsen.util 15784 :invoke :cas [11 [40 16]] INFO [2019-12-17 12:36:55,264] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65055,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:36:55,265] jepsen worker 6 - jepsen.util 15756 :invoke :cas [2 [9 0]] INFO [2019-12-17 12:36:55,265] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65032,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:36:55,267] jepsen worker 35 - jepsen.util 15335 :invoke :cas [11 [1 12]] INFO [2019-12-17 12:36:55,267] jepsen worker 1 - jepsen.util 15211 :invoke :write [0 47] INFO [2019-12-17 12:36:55,267] jepsen worker 76 - jepsen.util 15826 :invoke :read [25 nil] INFO [2019-12-17 12:36:55,267] jepsen worker 75 - jepsen.util 14925 :invoke :cas [25 [13 25]] INFO [2019-12-17 12:36:55,267] jepsen worker 16 - jepsen.util 16936 :invoke :write [5 41] INFO [2019-12-17 12:36:55,267] jepsen worker 41 - jepsen.util 16421 :invoke :write [13 24] INFO [2019-12-17 12:36:55,267] jepsen worker 57 - jepsen.util 15177 :invoke :read [19 nil] INFO [2019-12-17 12:36:55,267] jepsen worker 82 - jepsen.util 15562 :invoke :write [27 31] INFO [2019-12-17 12:36:55,268] jepsen worker 22 - jepsen.util 15412 :invoke :write [7 11] INFO [2019-12-17 12:36:55,268] jepsen worker 81 - jepsen.util 15021 :invoke :write [27 31] INFO [2019-12-17 12:36:55,268] jepsen worker 66 - jepsen.util 15366 :invoke :read [22 nil] INFO [2019-12-17 12:36:55,268] jepsen worker 40 - jepsen.util 15880 :invoke :write [13 46] INFO [2019-12-17 12:36:55,269] jepsen worker 77 - jepsen.util 16187 :invoke :read [25 nil] INFO [2019-12-17 12:36:55,269] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65041,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:36:55,279] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65053,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:36:55,288] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65032,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:36:55,289] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65041,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:36:55,289] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65053,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:36:55,289] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65033,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:36:55,289] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65033,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:36:55,314] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65063,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:36:55,314] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65063,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:36:55,337] jepsen worker 49 - jepsen.util 15349 :info :write [16 40] :RequestCanceledException INFO [2019-12-17 12:36:55,338] jepsen worker 49 - jepsen.util 15439 :invoke :write [16 28] INFO [2019-12-17 12:36:55,380] jepsen worker 11 - jepsen.util 10901 :ok :read [3 10] INFO [2019-12-17 12:36:55,381] jepsen worker 11 - jepsen.util 10901 :invoke :cas [3 [17 25]] INFO [2019-12-17 12:36:55,382] jepsen worker 31 - jepsen.util 9571 :info :cas [10 [10 7]] :RequestCanceledException INFO [2019-12-17 12:36:55,382] jepsen worker 24 - jepsen.util 13974 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:36:55,382] jepsen worker 72 - jepsen.util 11412 :info :cas [24 [13 0]] :RequestCanceledException INFO [2019-12-17 12:36:55,386] jepsen worker 24 - jepsen.util 13974 :invoke :cas [8 [21 33]] INFO [2019-12-17 12:36:55,393] jepsen worker 31 - jepsen.util 9661 :invoke :cas [10 [24 43]] INFO [2019-12-17 12:36:55,399] jepsen worker 86 - jepsen.util 16016 :info :write [28 7] :RequestCanceledException INFO [2019-12-17 12:36:55,399] jepsen worker 42 - jepsen.util 16152 :info :cas [14 [28 21]] :RequestCanceledException INFO [2019-12-17 12:36:55,399] jepsen worker 72 - jepsen.util 11502 :invoke :write [24 0] INFO [2019-12-17 12:36:55,414] jepsen worker 42 - jepsen.util 16242 :invoke :cas [14 [7 3]] INFO [2019-12-17 12:36:55,416] jepsen worker 86 - jepsen.util 16106 :invoke :read [28 nil] INFO [2019-12-17 12:36:55,419] jepsen worker 60 - jepsen.util 16440 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:36:55,419] jepsen worker 45 - jepsen.util 11385 :info :cas [15 [10 48]] :RequestCanceledException INFO [2019-12-17 12:36:55,419] jepsen worker 60 - jepsen.util 16440 :invoke :write [20 38] INFO [2019-12-17 12:36:55,424] jepsen worker 79 - jepsen.util 11419 :fail :cas [26 [14 32]] :ValueNotSwapFrom INFO [2019-12-17 12:36:55,424] jepsen worker 79 - jepsen.util 11419 :invoke :cas [26 [44 25]] INFO [2019-12-17 12:36:55,427] jepsen worker 45 - jepsen.util 11475 :invoke :cas [15 [36 29]] INFO [2019-12-17 12:36:55,432] jepsen worker 39 - jepsen.util 14979 :info :write [13 40] :RequestCanceledException INFO [2019-12-17 12:36:55,432] jepsen worker 33 - jepsen.util 15243 :info :write [11 8] :RequestCanceledException INFO [2019-12-17 12:36:55,432] jepsen worker 25 - jepsen.util 16315 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:36:55,433] jepsen worker 25 - jepsen.util 16315 :invoke :write [8 26] INFO [2019-12-17 12:36:55,446] jepsen worker 39 - jepsen.util 15069 :invoke :read [13 nil] INFO [2019-12-17 12:36:55,446] jepsen worker 33 - jepsen.util 15333 :invoke :write [11 41] INFO [2019-12-17 12:36:55,483] jepsen worker 29 - jepsen.util 9479 :fail :cas [9 [13 2]] :ValueNotSwapFrom INFO [2019-12-17 12:36:55,483] jepsen worker 29 - jepsen.util 9479 :invoke :cas [9 [10 4]] INFO [2019-12-17 12:36:55,707] jepsen worker 14 - jepsen.util 11084 :ok :write [4 35] INFO [2019-12-17 12:36:55,707] jepsen worker 37 - jepsen.util 10837 :ok :write [12 22] INFO [2019-12-17 12:36:55,707] jepsen worker 20 - jepsen.util 10820 :ok :write [6 32] INFO [2019-12-17 12:36:55,707] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65132,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:36:55,707] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65139,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:36:55,707] jepsen worker 63 - jepsen.util 9963 :ok :write [21 31] INFO [2019-12-17 12:36:55,707] jepsen worker 74 - jepsen.util 10874 :info :cas [24 [6 0]] :RequestCanceledException INFO [2019-12-17 12:36:55,707] jepsen worker 14 - jepsen.util 11084 :invoke :cas [4 [21 11]] INFO [2019-12-17 12:36:55,707] jepsen worker 30 - jepsen.util 10470 :info :write [10 14] :RequestCanceledException INFO [2019-12-17 12:36:55,707] jepsen worker 37 - jepsen.util 10837 :invoke :read [12 nil] INFO [2019-12-17 12:36:55,708] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65132,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:36:55,708] jepsen worker 20 - jepsen.util 10820 :invoke :cas [6 [6 37]] INFO [2019-12-17 12:36:55,708] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65139,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:36:55,708] jepsen worker 63 - jepsen.util 9963 :invoke :write [21 4] INFO [2019-12-17 12:36:55,708] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65142,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:36:55,708] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65142,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:36:55,709] jepsen worker 74 - jepsen.util 10964 :invoke :write [24 45] INFO [2019-12-17 12:36:55,709] jepsen worker 53 - jepsen.util 10583 :fail :cas [17 [28 7]] :ValueNotSwapFrom INFO [2019-12-17 12:36:55,709] jepsen worker 53 - jepsen.util 10583 :invoke :read [17 nil] INFO [2019-12-17 12:36:55,710] jepsen worker 30 - jepsen.util 10560 :invoke :read [10 nil] INFO [2019-12-17 12:36:55,710] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:36:55,710] jepsen nemesis - couchbase.util Rebalance check count: 512 INFO [2019-12-17 12:36:55,710] jepsen nemesis - couchbase.util Rebalance stuck count: 493 INFO [2019-12-17 12:36:55,715] jepsen worker 13 - jepsen.util 10633 :fail :cas [4 [40 44]] :ValueNotSwapFrom INFO [2019-12-17 12:36:55,716] jepsen worker 13 - jepsen.util 10633 :invoke :write [4 33] INFO [2019-12-17 12:36:55,717] jepsen worker 71 - jepsen.util 12311 :ok :read [23 40] INFO [2019-12-17 12:36:55,717] jepsen worker 71 - jepsen.util 12311 :invoke :write [23 30] INFO [2019-12-17 12:36:55,719] jepsen worker 38 - jepsen.util 11738 :ok :read [12 46] INFO [2019-12-17 12:36:55,719] jepsen worker 3 - jepsen.util 10713 :ok :read [1 13] INFO [2019-12-17 12:36:55,719] jepsen worker 38 - jepsen.util 11738 :invoke :read [12 nil] INFO [2019-12-17 12:36:55,719] jepsen worker 3 - jepsen.util 10713 :invoke :read [1 nil] INFO [2019-12-17 12:36:55,716] jepsen worker 47 - jepsen.util 12017 :fail :cas [15 [21 27]] :ValueNotSwapFrom INFO [2019-12-17 12:36:55,719] jepsen worker 80 - jepsen.util 10970 :fail :cas [26 [34 29]] :ValueNotSwapFrom INFO [2019-12-17 12:36:55,720] jepsen worker 47 - jepsen.util 12017 :invoke :write [15 19] INFO [2019-12-17 12:36:55,722] jepsen worker 80 - jepsen.util 10970 :invoke :write [26 47] INFO [2019-12-17 12:36:55,724] jepsen worker 70 - jepsen.util 11860 :fail :cas [23 [30 29]] :ValueNotSwapFrom INFO [2019-12-17 12:36:55,725] jepsen worker 70 - jepsen.util 11860 :invoke :write [23 13] INFO [2019-12-17 12:36:55,725] jepsen worker 54 - jepsen.util 11844 :ok :read [18 22] INFO [2019-12-17 12:36:55,725] jepsen worker 54 - jepsen.util 11844 :invoke :write [18 34] INFO [2019-12-17 12:36:55,726] jepsen worker 11 - jepsen.util 10901 :fail :cas [3 [17 25]] :ValueNotSwapFrom INFO [2019-12-17 12:36:55,727] jepsen worker 11 - jepsen.util 10901 :invoke :cas [3 [6 38]] INFO [2019-12-17 12:36:55,727] jepsen worker 31 - jepsen.util 9661 :fail :cas [10 [24 43]] :ValueNotSwapFrom INFO [2019-12-17 12:36:55,727] jepsen worker 31 - jepsen.util 9661 :invoke :read [10 nil] INFO [2019-12-17 12:36:55,727] jepsen worker 9 - jepsen.util 11889 :fail :cas [3 [42 35]] :ValueNotSwapFrom INFO [2019-12-17 12:36:55,727] jepsen worker 9 - jepsen.util 11889 :invoke :write [3 11] INFO [2019-12-17 12:36:55,728] jepsen worker 0 - jepsen.util 16020 :info :cas [0 [2 5]] :RequestCanceledException INFO [2019-12-17 12:36:55,729] jepsen worker 52 - jepsen.util 10672 :ok :read [17 35] INFO [2019-12-17 12:36:55,730] jepsen worker 52 - jepsen.util 10672 :invoke :cas [17 [46 14]] INFO [2019-12-17 12:36:55,731] jepsen worker 51 - jepsen.util 10671 :fail :cas [17 [44 38]] :ValueNotSwapFrom INFO [2019-12-17 12:36:55,731] jepsen worker 51 - jepsen.util 10671 :invoke :read [17 nil] INFO [2019-12-17 12:36:55,732] jepsen worker 0 - jepsen.util 16110 :invoke :write [0 27] INFO [2019-12-17 12:36:55,734] jepsen worker 56 - jepsen.util 12026 :ok :read [18 22] INFO [2019-12-17 12:36:55,734] jepsen worker 56 - jepsen.util 12026 :invoke :write [18 5] INFO [2019-12-17 12:36:55,735] jepsen worker 45 - jepsen.util 11475 :fail :cas [15 [36 29]] :ValueNotSwapFrom INFO [2019-12-17 12:36:55,735] jepsen worker 45 - jepsen.util 11475 :invoke :cas [15 [6 9]] INFO [2019-12-17 12:36:55,740] jepsen worker 89 - jepsen.util 11249 :fail :cas [29 [8 10]] :ValueNotSwapFrom INFO [2019-12-17 12:36:55,740] jepsen worker 89 - jepsen.util 11249 :invoke :cas [29 [20 48]] INFO [2019-12-17 12:36:55,746] jepsen worker 29 - jepsen.util 9479 :fail :cas [9 [10 4]] :ValueNotSwapFrom INFO [2019-12-17 12:36:55,746] jepsen worker 29 - jepsen.util 9479 :invoke :read [9 nil] INFO [2019-12-17 12:36:55,758] jepsen worker 69 - jepsen.util 11499 :fail :cas [23 [43 38]] :ValueNotSwapFrom INFO [2019-12-17 12:36:55,758] jepsen worker 69 - jepsen.util 11499 :invoke :read [23 nil] INFO [2019-12-17 12:36:55,810] jepsen worker 20 - jepsen.util 10820 :fail :cas [6 [6 37]] :ValueNotSwapFrom INFO [2019-12-17 12:36:55,810] jepsen worker 20 - jepsen.util 10820 :invoke :cas [6 [42 21]] INFO [2019-12-17 12:36:55,814] jepsen worker 85 - jepsen.util 16105 :info :cas [28 [6 29]] :RequestCanceledException INFO [2019-12-17 12:36:55,815] jepsen worker 85 - jepsen.util 16195 :invoke :write [28 1] INFO [2019-12-17 12:36:55,820] jepsen worker 30 - jepsen.util 10560 :ok :read [10 25] INFO [2019-12-17 12:36:55,820] jepsen worker 30 - jepsen.util 10560 :invoke :read [10 nil] INFO [2019-12-17 12:36:55,846] jepsen worker 12 - jepsen.util 11532 :fail :cas [4 [23 19]] :ValueNotSwapFrom INFO [2019-12-17 12:36:55,846] jepsen worker 12 - jepsen.util 11532 :invoke :read [4 nil] INFO [2019-12-17 12:36:55,871] jepsen worker 14 - jepsen.util 11084 :fail :cas [4 [21 11]] :ValueNotSwapFrom INFO [2019-12-17 12:36:55,872] jepsen worker 14 - jepsen.util 11084 :invoke :write [4 6] INFO [2019-12-17 12:36:55,879] jepsen worker 64 - jepsen.util 8614 :ok :write [21 33] INFO [2019-12-17 12:36:55,879] jepsen worker 64 - jepsen.util 8614 :invoke :cas [21 [45 18]] INFO [2019-12-17 12:36:55,885] jepsen worker 11 - jepsen.util 10901 :fail :cas [3 [6 38]] :ValueNotSwapFrom INFO [2019-12-17 12:36:55,885] jepsen worker 11 - jepsen.util 10901 :invoke :write [3 27] INFO [2019-12-17 12:36:55,901] jepsen worker 53 - jepsen.util 10583 :ok :read [17 35] INFO [2019-12-17 12:36:55,902] jepsen worker 53 - jepsen.util 10583 :invoke :cas [17 [10 7]] INFO [2019-12-17 12:36:55,902] jepsen worker 88 - jepsen.util 11518 :ok :read [29 21] INFO [2019-12-17 12:36:55,902] jepsen worker 88 - jepsen.util 11518 :invoke :write [29 7] INFO [2019-12-17 12:36:55,902] jepsen worker 37 - jepsen.util 10837 :ok :read [12 22] INFO [2019-12-17 12:36:55,902] jepsen worker 37 - jepsen.util 10837 :invoke :write [12 23] INFO [2019-12-17 12:36:55,904] jepsen worker 31 - jepsen.util 9661 :ok :read [10 25] INFO [2019-12-17 12:36:55,904] jepsen worker 31 - jepsen.util 9661 :invoke :read [10 nil] INFO [2019-12-17 12:36:55,924] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65178,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:36:55,924] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65178,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:36:55,924] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65181,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:36:55,924] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65181,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:36:55,924] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65182,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:36:55,924] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65182,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:36:55,928] jepsen worker 27 - jepsen.util 10557 :ok :write [9 15] INFO [2019-12-17 12:36:55,928] jepsen worker 27 - jepsen.util 10557 :invoke :cas [9 [29 3]] INFO [2019-12-17 12:36:55,931] jepsen worker 73 - jepsen.util 10423 :info :write [24 17] :RequestCanceledException INFO [2019-12-17 12:36:55,931] jepsen worker 65 - jepsen.util 9695 :info :cas [21 [26 2]] :RequestCanceledException INFO [2019-12-17 12:36:55,932] jepsen worker 65 - jepsen.util 9785 :invoke :write [21 30] INFO [2019-12-17 12:36:55,932] jepsen worker 73 - jepsen.util 10513 :invoke :cas [24 [4 27]] INFO [2019-12-17 12:36:55,947] jepsen worker 19 - jepsen.util 11539 :fail :read [6 nil] :RequestTimeoutException INFO [2019-12-17 12:36:55,947] jepsen worker 19 - jepsen.util 11539 :invoke :write [6 29] INFO [2019-12-17 12:36:55,964] jepsen worker 27 - jepsen.util 10557 :fail :cas [9 [29 3]] :ValueNotSwapFrom INFO [2019-12-17 12:36:55,964] jepsen worker 27 - jepsen.util 10557 :invoke :read [9 nil] INFO [2019-12-17 12:36:55,969] jepsen worker 51 - jepsen.util 10671 :ok :read [17 35] INFO [2019-12-17 12:36:55,969] jepsen worker 51 - jepsen.util 10671 :invoke :write [17 48] INFO [2019-12-17 12:36:56,009] jepsen worker 5 - jepsen.util 10985 :ok :write [1 45] INFO [2019-12-17 12:36:56,009] jepsen worker 5 - jepsen.util 10985 :invoke :cas [1 [22 24]] INFO [2019-12-17 12:36:56,033] jepsen worker 67 - jepsen.util 15457 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:36:56,033] jepsen worker 67 - jepsen.util 15457 :invoke :cas [22 [31 12]] INFO [2019-12-17 12:36:56,034] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65196,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:36:56,034] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65196,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:36:56,034] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65200,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:36:56,034] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65200,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:36:56,047] jepsen worker 46 - jepsen.util 10936 :info :cas [15 [27 6]] :RequestCanceledException INFO [2019-12-17 12:36:56,048] jepsen worker 46 - jepsen.util 11026 :invoke :read [15 nil] INFO [2019-12-17 12:36:56,071] jepsen worker 79 - jepsen.util 11419 :fail :cas [26 [44 25]] :ValueNotSwapFrom INFO [2019-12-17 12:36:56,072] jepsen worker 79 - jepsen.util 11419 :invoke :cas [26 [32 30]] INFO [2019-12-17 12:36:56,088] jepsen worker 87 - jepsen.util 10887 :ok :write [29 19] INFO [2019-12-17 12:36:56,088] jepsen worker 87 - jepsen.util 10887 :invoke :read [29 nil] INFO [2019-12-17 12:36:56,090] jepsen worker 78 - jepsen.util 11328 :ok :write [26 12] INFO [2019-12-17 12:36:56,090] jepsen worker 78 - jepsen.util 11328 :invoke :read [26 nil] INFO [2019-12-17 12:36:56,097] jepsen worker 48 - jepsen.util 14178 :info :cas [16 [28 14]] :RequestCanceledException INFO [2019-12-17 12:36:56,099] jepsen worker 48 - jepsen.util 14268 :invoke :cas [16 [38 31]] INFO [2019-12-17 12:36:56,117] jepsen worker 8 - jepsen.util 15218 :info :cas [2 [1 45]] :RequestCanceledException INFO [2019-12-17 12:36:56,118] jepsen worker 8 - jepsen.util 15308 :invoke :read [2 nil] INFO [2019-12-17 12:36:56,122] jepsen worker 3 - jepsen.util 10713 :ok :read [1 45] INFO [2019-12-17 12:36:56,122] jepsen worker 3 - jepsen.util 10713 :invoke :cas [1 [3 49]] INFO [2019-12-17 12:36:56,132] jepsen worker 7 - jepsen.util 15397 :info :write [2 47] :RequestCanceledException INFO [2019-12-17 12:36:56,133] jepsen worker 7 - jepsen.util 15487 :invoke :read [2 nil] INFO [2019-12-17 12:36:56,147] jepsen worker 61 - jepsen.util 16531 :info :write [20 30] :RequestCanceledException INFO [2019-12-17 12:36:56,148] jepsen worker 61 - jepsen.util 16621 :invoke :read [20 nil] INFO [2019-12-17 12:36:56,150] jepsen worker 18 - jepsen.util 11358 :ok :write [6 25] INFO [2019-12-17 12:36:56,150] jepsen worker 47 - jepsen.util 12017 :ok :write [15 19] INFO [2019-12-17 12:36:56,150] jepsen worker 18 - jepsen.util 11358 :invoke :cas [6 [38 33]] INFO [2019-12-17 12:36:56,150] jepsen worker 47 - jepsen.util 12017 :invoke :read [15 nil] INFO [2019-12-17 12:36:56,164] jepsen worker 2 - jepsen.util 14942 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:36:56,164] jepsen worker 10 - jepsen.util 10990 :fail :read [3 nil] :RequestTimeoutException INFO [2019-12-17 12:36:56,164] jepsen worker 2 - jepsen.util 14942 :invoke :read [0 nil] INFO [2019-12-17 12:36:56,164] jepsen worker 10 - jepsen.util 10990 :invoke :cas [3 [23 40]] INFO [2019-12-17 12:36:56,169] jepsen worker 47 - jepsen.util 12017 :ok :read [15 19] INFO [2019-12-17 12:36:56,170] jepsen worker 47 - jepsen.util 12017 :invoke :read [15 nil] INFO [2019-12-17 12:36:56,175] jepsen worker 52 - jepsen.util 10672 :fail :cas [17 [46 14]] :ValueNotSwapFrom INFO [2019-12-17 12:36:56,175] jepsen worker 52 - jepsen.util 10672 :invoke :write [17 11] INFO [2019-12-17 12:36:56,179] jepsen worker 9 - jepsen.util 11889 :ok :write [3 11] INFO [2019-12-17 12:36:56,179] jepsen worker 9 - jepsen.util 11889 :invoke :write [3 20] INFO [2019-12-17 12:36:56,183] jepsen worker 80 - jepsen.util 10970 :ok :write [26 47] INFO [2019-12-17 12:36:56,183] jepsen worker 80 - jepsen.util 10970 :invoke :write [26 20] INFO [2019-12-17 12:36:56,199] jepsen worker 10 - jepsen.util 10990 :fail :cas [3 [23 40]] :ValueNotSwapFrom INFO [2019-12-17 12:36:56,199] jepsen worker 10 - jepsen.util 10990 :invoke :read [3 nil] INFO [2019-12-17 12:36:56,204] jepsen worker 47 - jepsen.util 12017 :ok :read [15 19] INFO [2019-12-17 12:36:56,204] jepsen worker 47 - jepsen.util 12017 :invoke :cas [15 [39 0]] INFO [2019-12-17 12:36:56,212] jepsen worker 73 - jepsen.util 10513 :fail :cas [24 [4 27]] :ValueNotSwapFrom INFO [2019-12-17 12:36:56,212] jepsen worker 73 - jepsen.util 10513 :invoke :write [24 16] INFO [2019-12-17 12:36:56,228] jepsen worker 4 - jepsen.util 10354 :ok :write [1 1] INFO [2019-12-17 12:36:56,228] jepsen worker 4 - jepsen.util 10354 :invoke :read [1 nil] INFO [2019-12-17 12:36:56,228] jepsen worker 14 - jepsen.util 11084 :ok :write [4 6] INFO [2019-12-17 12:36:56,229] jepsen worker 14 - jepsen.util 11084 :invoke :cas [4 [34 2]] INFO [2019-12-17 12:36:56,231] jepsen worker 58 - jepsen.util 15628 :info :write [19 24] :RequestCanceledException INFO [2019-12-17 12:36:56,231] jepsen worker 17 - jepsen.util 15407 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:36:56,231] jepsen worker 15 - jepsen.util 14325 :info :cas [5 [36 3]] :RequestCanceledException INFO [2019-12-17 12:36:56,231] jepsen worker 46 - jepsen.util 11026 :ok :read [15 19] INFO [2019-12-17 12:36:56,231] jepsen worker 17 - jepsen.util 15407 :invoke :cas [5 [42 46]] INFO [2019-12-17 12:36:56,231] jepsen worker 46 - jepsen.util 11026 :invoke :write [15 24] INFO [2019-12-17 12:36:56,232] jepsen worker 15 - jepsen.util 14415 :invoke :read [5 nil] INFO [2019-12-17 12:36:56,232] jepsen worker 58 - jepsen.util 15718 :invoke :read [19 nil] INFO [2019-12-17 12:36:56,239] jepsen worker 69 - jepsen.util 11499 :ok :read [23 40] INFO [2019-12-17 12:36:56,239] jepsen worker 69 - jepsen.util 11499 :invoke :read [23 nil] INFO [2019-12-17 12:36:56,246] jepsen worker 43 - jepsen.util 15523 :info :cas [14 [45 24]] :RequestCanceledException INFO [2019-12-17 12:36:56,247] jepsen worker 43 - jepsen.util 15613 :invoke :write [14 11] INFO [2019-12-17 12:36:56,254] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65230,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:36:56,254] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65230,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:36:56,254] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65234,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:36:56,254] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65234,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:36:56,254] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65235,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:36:56,254] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65235,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:36:56,263] jepsen worker 50 - jepsen.util 15800 :info :cas [16 [47 11]] :RequestCanceledException INFO [2019-12-17 12:36:56,263] jepsen worker 23 - jepsen.util 15323 :info :write [7 44] :RequestCanceledException INFO [2019-12-17 12:36:56,264] jepsen worker 50 - jepsen.util 15890 :invoke :cas [16 [48 30]] INFO [2019-12-17 12:36:56,264] jepsen worker 23 - jepsen.util 15413 :invoke :cas [7 [2 32]] INFO [2019-12-17 12:36:56,272] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65221,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:36:56,272] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65221,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:36:56,273] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65223,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:36:56,273] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65223,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:36:56,273] jepsen worker 10 - jepsen.util 10990 :ok :read [3 11] INFO [2019-12-17 12:36:56,273] jepsen worker 10 - jepsen.util 10990 :invoke :read [3 nil] INFO [2019-12-17 12:36:56,286] jepsen worker 18 - jepsen.util 11358 :fail :cas [6 [38 33]] :ValueNotSwapFrom INFO [2019-12-17 12:36:56,286] jepsen worker 18 - jepsen.util 11358 :invoke :read [6 nil] INFO [2019-12-17 12:36:56,293] jepsen worker 47 - jepsen.util 12017 :fail :cas [15 [39 0]] :ValueNotSwapFrom INFO [2019-12-17 12:36:56,293] jepsen worker 47 - jepsen.util 12017 :invoke :cas [15 [14 37]] INFO [2019-12-17 12:36:56,299] jepsen worker 37 - jepsen.util 10837 :ok :write [12 23] INFO [2019-12-17 12:36:56,299] jepsen worker 37 - jepsen.util 10837 :invoke :cas [12 [44 30]] INFO [2019-12-17 12:36:56,299] jepsen worker 84 - jepsen.util 15114 :info :cas [28 [11 15]] :RequestCanceledException INFO [2019-12-17 12:36:56,301] jepsen worker 84 - jepsen.util 15204 :invoke :read [28 nil] INFO [2019-12-17 12:36:56,313] jepsen worker 62 - jepsen.util 16082 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:36:56,313] jepsen worker 62 - jepsen.util 16082 :invoke :read [20 nil] INFO [2019-12-17 12:36:56,324] jepsen worker 47 - jepsen.util 12017 :fail :cas [15 [14 37]] :ValueNotSwapFrom INFO [2019-12-17 12:36:56,324] jepsen worker 47 - jepsen.util 12017 :invoke :write [15 35] INFO [2019-12-17 12:36:56,327] jepsen worker 10 - jepsen.util 10990 :ok :read [3 11] INFO [2019-12-17 12:36:56,328] jepsen worker 10 - jepsen.util 10990 :invoke :write [3 13] INFO [2019-12-17 12:36:56,364] jepsen worker 21 - jepsen.util 14871 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:36:56,364] jepsen worker 21 - jepsen.util 14871 :invoke :write [7 46] INFO [2019-12-17 12:36:56,385] jepsen worker 5 - jepsen.util 10985 :fail :cas [1 [22 24]] :ValueNotSwapFrom INFO [2019-12-17 12:36:56,385] jepsen worker 5 - jepsen.util 10985 :invoke :read [1 nil] INFO [2019-12-17 12:36:56,391] jepsen worker 70 - jepsen.util 11860 :ok :write [23 13] INFO [2019-12-17 12:36:56,391] jepsen worker 70 - jepsen.util 11860 :invoke :cas [23 [14 46]] INFO [2019-12-17 12:36:56,432] jepsen worker 32 - jepsen.util 11552 :info :write [10 23] :RequestCanceledException INFO [2019-12-17 12:36:56,433] jepsen worker 32 - jepsen.util 11642 :invoke :cas [10 [38 44]] INFO [2019-12-17 12:36:56,437] jepsen worker 51 - jepsen.util 10671 :ok :write [17 48] INFO [2019-12-17 12:36:56,437] jepsen worker 88 - jepsen.util 11518 :ok :write [29 7] INFO [2019-12-17 12:36:56,437] jepsen worker 51 - jepsen.util 10671 :invoke :read [17 nil] INFO [2019-12-17 12:36:56,437] jepsen worker 88 - jepsen.util 11518 :invoke :read [29 nil] INFO [2019-12-17 12:36:56,442] jepsen worker 9 - jepsen.util 11889 :ok :write [3 20] INFO [2019-12-17 12:36:56,443] jepsen worker 9 - jepsen.util 11889 :invoke :cas [3 [41 24]] INFO [2019-12-17 12:36:56,460] jepsen worker 32 - jepsen.util 11642 :fail :cas [10 [38 44]] :ValueNotSwapFrom INFO [2019-12-17 12:36:56,460] jepsen worker 32 - jepsen.util 11642 :invoke :write [10 42] INFO [2019-12-17 12:36:56,469] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65258,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:36:56,469] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65258,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:36:56,469] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65262,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:36:56,469] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65262,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:36:56,482] jepsen worker 79 - jepsen.util 11419 :fail :cas [26 [32 30]] :ValueNotSwapFrom INFO [2019-12-17 12:36:56,482] jepsen worker 79 - jepsen.util 11419 :invoke :cas [26 [44 30]] INFO [2019-12-17 12:36:56,496] jepsen worker 9 - jepsen.util 11889 :fail :cas [3 [41 24]] :ValueNotSwapFrom INFO [2019-12-17 12:36:56,496] jepsen worker 9 - jepsen.util 11889 :invoke :read [3 nil] INFO [2019-12-17 12:36:56,507] jepsen worker 80 - jepsen.util 10970 :ok :write [26 20] INFO [2019-12-17 12:36:56,507] jepsen worker 80 - jepsen.util 10970 :invoke :write [26 23] INFO [2019-12-17 12:36:56,554] jepsen worker 87 - jepsen.util 10887 :ok :read [29 7] INFO [2019-12-17 12:36:56,554] jepsen worker 87 - jepsen.util 10887 :invoke :cas [29 [15 1]] INFO [2019-12-17 12:36:56,561] jepsen worker 46 - jepsen.util 11026 :ok :write [15 24] INFO [2019-12-17 12:36:56,561] jepsen worker 46 - jepsen.util 11026 :invoke :write [15 43] INFO [2019-12-17 12:36:56,567] jepsen worker 32 - jepsen.util 11642 :ok :write [10 42] INFO [2019-12-17 12:36:56,568] jepsen worker 32 - jepsen.util 11642 :invoke :cas [10 [1 23]] INFO [2019-12-17 12:36:56,574] jepsen worker 55 - jepsen.util 11845 :ok :write [18 28] INFO [2019-12-17 12:36:56,574] jepsen worker 55 - jepsen.util 11845 :invoke :cas [18 [24 25]] INFO [2019-12-17 12:36:56,589] jepsen worker 32 - jepsen.util 11642 :fail :cas [10 [1 23]] :ValueNotSwapFrom INFO [2019-12-17 12:36:56,589] jepsen worker 32 - jepsen.util 11642 :invoke :read [10 nil] INFO [2019-12-17 12:36:56,606] jepsen worker 32 - jepsen.util 11642 :ok :read [10 42] INFO [2019-12-17 12:36:56,606] jepsen worker 32 - jepsen.util 11642 :invoke :write [10 15] INFO [2019-12-17 12:36:56,645] jepsen worker 3 - jepsen.util 10713 :fail :cas [1 [3 49]] :ValueNotSwapFrom INFO [2019-12-17 12:36:56,645] jepsen worker 3 - jepsen.util 10713 :invoke :read [1 nil] INFO [2019-12-17 12:36:56,689] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65278,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:36:56,689] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65278,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:36:56,689] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65282,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:36:56,690] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65282,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:36:56,690] jepsen worker 36 - jepsen.util 12096 :fail :cas [12 [46 41]] :CASMismatchException INFO [2019-12-17 12:36:56,690] jepsen worker 36 - jepsen.util 12096 :invoke :read [12 nil] INFO [2019-12-17 12:36:56,718] jepsen worker 78 - jepsen.util 11328 :ok :read [26 20] INFO [2019-12-17 12:36:56,718] jepsen worker 78 - jepsen.util 11328 :invoke :read [26 nil] INFO [2019-12-17 12:36:56,723] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:36:56,723] jepsen nemesis - couchbase.util Rebalance check count: 513 INFO [2019-12-17 12:36:56,723] jepsen nemesis - couchbase.util Rebalance stuck count: 494 INFO [2019-12-17 12:36:56,760] jepsen worker 37 - jepsen.util 10837 :fail :cas [12 [44 30]] :ValueNotSwapFrom INFO [2019-12-17 12:36:56,760] jepsen worker 37 - jepsen.util 10837 :invoke :cas [12 [32 24]] INFO [2019-12-17 12:36:56,816] jepsen worker 4 - jepsen.util 10354 :ok :read [1 1] INFO [2019-12-17 12:36:56,816] jepsen worker 4 - jepsen.util 10354 :invoke :cas [1 [30 6]] INFO [2019-12-17 12:36:56,845] jepsen worker 38 - jepsen.util 11738 :ok :read [12 23] INFO [2019-12-17 12:36:56,845] jepsen worker 38 - jepsen.util 11738 :invoke :read [12 nil] INFO [2019-12-17 12:36:56,847] jepsen worker 14 - jepsen.util 11084 :fail :cas [4 [34 2]] :ValueNotSwapFrom INFO [2019-12-17 12:36:56,847] jepsen worker 14 - jepsen.util 11084 :invoke :write [4 30] INFO [2019-12-17 12:36:56,857] jepsen worker 53 - jepsen.util 10583 :fail :cas [17 [10 7]] :ValueNotSwapFrom INFO [2019-12-17 12:36:56,857] jepsen worker 53 - jepsen.util 10583 :invoke :write [17 39] INFO [2019-12-17 12:36:56,881] jepsen worker 5 - jepsen.util 10985 :ok :read [1 1] INFO [2019-12-17 12:36:56,881] jepsen worker 5 - jepsen.util 10985 :invoke :cas [1 [20 37]] INFO [2019-12-17 12:36:56,886] jepsen worker 46 - jepsen.util 11026 :ok :write [15 43] INFO [2019-12-17 12:36:56,886] jepsen worker 46 - jepsen.util 11026 :invoke :write [15 22] INFO [2019-12-17 12:36:56,906] jepsen worker 70 - jepsen.util 11860 :fail :cas [23 [14 46]] :ValueNotSwapFrom INFO [2019-12-17 12:36:56,907] jepsen worker 70 - jepsen.util 11860 :invoke :write [23 5] INFO [2019-12-17 12:36:56,909] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65291,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:36:56,909] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65291,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:36:56,909] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65294,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:36:56,909] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65294,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:36:56,977] jepsen worker 51 - jepsen.util 10671 :ok :read [17 48] INFO [2019-12-17 12:36:56,977] jepsen worker 51 - jepsen.util 10671 :invoke :read [17 nil] INFO [2019-12-17 12:36:56,979] jepsen worker 4 - jepsen.util 10354 :fail :cas [1 [30 6]] :ValueNotSwapFrom INFO [2019-12-17 12:36:56,979] jepsen worker 4 - jepsen.util 10354 :invoke :cas [1 [8 49]] INFO [2019-12-17 12:36:56,995] jepsen worker 32 - jepsen.util 11642 :ok :write [10 15] INFO [2019-12-17 12:36:56,995] jepsen worker 32 - jepsen.util 11642 :invoke :read [10 nil] INFO [2019-12-17 12:36:56,995] jepsen worker 46 - jepsen.util 11026 :ok :write [15 22] INFO [2019-12-17 12:36:56,995] jepsen worker 46 - jepsen.util 11026 :invoke :write [15 2] INFO [2019-12-17 12:36:57,007] jepsen worker 13 - jepsen.util 10633 :ok :write [4 33] INFO [2019-12-17 12:36:57,007] jepsen worker 13 - jepsen.util 10633 :invoke :write [4 39] INFO [2019-12-17 12:36:57,016] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65305,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:36:57,016] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65305,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:36:57,016] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65309,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:36:57,016] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65309,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:36:57,025] jepsen worker 32 - jepsen.util 11642 :ok :read [10 15] INFO [2019-12-17 12:36:57,025] jepsen worker 32 - jepsen.util 11642 :invoke :cas [10 [13 8]] INFO [2019-12-17 12:36:57,027] jepsen worker 36 - jepsen.util 12096 :ok :read [12 23] INFO [2019-12-17 12:36:57,027] jepsen worker 36 - jepsen.util 12096 :invoke :write [12 36] INFO [2019-12-17 12:36:57,045] jepsen worker 52 - jepsen.util 10672 :ok :write [17 11] INFO [2019-12-17 12:36:57,045] jepsen worker 52 - jepsen.util 10672 :invoke :cas [17 [40 4]] INFO [2019-12-17 12:36:57,048] jepsen worker 32 - jepsen.util 11642 :fail :cas [10 [13 8]] :ValueNotSwapFrom INFO [2019-12-17 12:36:57,048] jepsen worker 32 - jepsen.util 11642 :invoke :write [10 22] INFO [2019-12-17 12:36:57,071] jepsen worker 79 - jepsen.util 11419 :fail :cas [26 [44 30]] :ValueNotSwapFrom INFO [2019-12-17 12:36:57,079] jepsen worker 79 - jepsen.util 11419 :invoke :cas [26 [35 22]] INFO [2019-12-17 12:36:57,092] jepsen worker 51 - jepsen.util 10671 :ok :read [17 11] INFO [2019-12-17 12:36:57,092] jepsen worker 51 - jepsen.util 10671 :invoke :cas [17 [25 5]] INFO [2019-12-17 12:36:57,100] jepsen worker 89 - jepsen.util 11249 :fail :cas [29 [20 48]] :ValueNotSwapFrom INFO [2019-12-17 12:36:57,100] jepsen worker 89 - jepsen.util 11249 :invoke :write [29 46] INFO [2019-12-17 12:36:57,132] jepsen worker 5 - jepsen.util 10985 :fail :cas [1 [20 37]] :ValueNotSwapFrom INFO [2019-12-17 12:36:57,132] jepsen worker 5 - jepsen.util 10985 :invoke :cas [1 [38 15]] INFO [2019-12-17 12:36:57,206] jepsen worker 4 - jepsen.util 10354 :fail :cas [1 [8 49]] :ValueNotSwapFrom INFO [2019-12-17 12:36:57,207] jepsen worker 4 - jepsen.util 10354 :invoke :read [1 nil] INFO [2019-12-17 12:36:57,213] jepsen worker 37 - jepsen.util 10837 :fail :cas [12 [32 24]] :ValueNotSwapFrom INFO [2019-12-17 12:36:57,213] jepsen worker 37 - jepsen.util 10837 :invoke :write [12 41] INFO [2019-12-17 12:36:57,231] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65328,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:36:57,232] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65328,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:36:57,252] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65320,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:36:57,252] jepsen worker 71 - jepsen.util 12311 :ok :write [23 30] INFO [2019-12-17 12:36:57,252] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65320,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:36:57,252] jepsen worker 71 - jepsen.util 12311 :invoke :read [23 nil] INFO [2019-12-17 12:36:57,252] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65321,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:36:57,253] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65321,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:36:57,258] jepsen worker 80 - jepsen.util 10970 :ok :write [26 23] INFO [2019-12-17 12:36:57,258] jepsen worker 56 - jepsen.util 12026 :ok :write [18 5] INFO [2019-12-17 12:36:57,258] jepsen worker 80 - jepsen.util 10970 :invoke :cas [26 [35 27]] INFO [2019-12-17 12:36:57,259] jepsen worker 56 - jepsen.util 12026 :invoke :write [18 46] INFO [2019-12-17 12:36:57,300] jepsen worker 68 - jepsen.util 14738 :info :cas [22 [31 36]] :RequestCanceledException INFO [2019-12-17 12:36:57,301] jepsen worker 68 - jepsen.util 14828 :invoke :read [22 nil] INFO [2019-12-17 12:36:57,317] jepsen worker 81 - jepsen.util 15021 :info :write [27 31] :RequestCanceledException INFO [2019-12-17 12:36:57,317] jepsen worker 75 - jepsen.util 14925 :info :cas [25 [13 25]] :RequestCanceledException INFO [2019-12-17 12:36:57,317] jepsen worker 6 - jepsen.util 15756 :info :cas [2 [9 0]] :RequestCanceledException INFO [2019-12-17 12:36:57,318] jepsen worker 81 - jepsen.util 15111 :invoke :read [27 nil] INFO [2019-12-17 12:36:57,318] jepsen worker 75 - jepsen.util 15015 :invoke :read [25 nil] INFO [2019-12-17 12:36:57,318] jepsen worker 6 - jepsen.util 15846 :invoke :cas [2 [47 43]] INFO [2019-12-17 12:36:57,327] jepsen worker 12 - jepsen.util 11532 :ok :read [4 33] INFO [2019-12-17 12:36:57,327] jepsen worker 12 - jepsen.util 11532 :invoke :write [4 38] INFO [2019-12-17 12:36:57,332] jepsen worker 83 - jepsen.util 15563 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:36:57,332] jepsen worker 66 - jepsen.util 15366 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:36:57,332] jepsen worker 16 - jepsen.util 16936 :info :write [5 41] :RequestCanceledException INFO [2019-12-17 12:36:57,332] jepsen worker 34 - jepsen.util 15784 :info :cas [11 [40 16]] :RequestCanceledException INFO [2019-12-17 12:36:57,332] jepsen worker 76 - jepsen.util 15826 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:36:57,332] jepsen worker 57 - jepsen.util 15177 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:36:57,332] jepsen worker 83 - jepsen.util 15563 :invoke :read [27 nil] INFO [2019-12-17 12:36:57,332] jepsen worker 66 - jepsen.util 15366 :invoke :cas [22 [43 33]] INFO [2019-12-17 12:36:57,332] jepsen worker 76 - jepsen.util 15826 :invoke :write [25 24] INFO [2019-12-17 12:36:57,332] jepsen worker 57 - jepsen.util 15177 :invoke :read [19 nil] INFO [2019-12-17 12:36:57,333] jepsen worker 16 - jepsen.util 17026 :invoke :write [5 2] INFO [2019-12-17 12:36:57,333] jepsen worker 34 - jepsen.util 15874 :invoke :cas [11 [47 17]] INFO [2019-12-17 12:36:57,346] jepsen worker 59 - jepsen.util 14639 :info :write [19 41] :RequestCanceledException INFO [2019-12-17 12:36:57,346] jepsen worker 28 - jepsen.util 9388 :fail :read [9 nil] :RequestTimeoutException INFO [2019-12-17 12:36:57,346] jepsen worker 44 - jepsen.util 15884 :info :cas [14 [22 5]] :RequestCanceledException INFO [2019-12-17 12:36:57,346] jepsen worker 41 - jepsen.util 16421 :info :write [13 24] :RequestCanceledException INFO [2019-12-17 12:36:57,346] jepsen worker 40 - jepsen.util 15880 :info :write [13 46] :RequestCanceledException INFO [2019-12-17 12:36:57,346] jepsen worker 28 - jepsen.util 9388 :invoke :read [9 nil] INFO [2019-12-17 12:36:57,348] jepsen worker 40 - jepsen.util 15970 :invoke :read [13 nil] INFO [2019-12-17 12:36:57,349] jepsen worker 41 - jepsen.util 16511 :invoke :write [13 37] INFO [2019-12-17 12:36:57,350] jepsen worker 59 - jepsen.util 14729 :invoke :read [19 nil] INFO [2019-12-17 12:36:57,350] jepsen worker 44 - jepsen.util 15974 :invoke :cas [14 [17 17]] INFO [2019-12-17 12:36:57,358] jepsen worker 80 - jepsen.util 10970 :fail :cas [26 [35 27]] :ValueNotSwapFrom INFO [2019-12-17 12:36:57,358] jepsen worker 80 - jepsen.util 10970 :invoke :write [26 6] INFO [2019-12-17 12:36:57,358] jepsen worker 28 - jepsen.util 9388 :ok :read [9 15] INFO [2019-12-17 12:36:57,358] jepsen worker 28 - jepsen.util 9388 :invoke :cas [9 [33 28]] INFO [2019-12-17 12:36:57,364] jepsen worker 26 - jepsen.util 16046 :info :write [8 4] :RequestCanceledException INFO [2019-12-17 12:36:57,364] jepsen worker 35 - jepsen.util 15335 :info :cas [11 [1 12]] :RequestCanceledException INFO [2019-12-17 12:36:57,364] jepsen worker 77 - jepsen.util 16187 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:36:57,365] jepsen worker 82 - jepsen.util 15562 :info :write [27 31] :RequestCanceledException INFO [2019-12-17 12:36:57,365] jepsen worker 77 - jepsen.util 16187 :invoke :write [25 31] INFO [2019-12-17 12:36:57,367] jepsen worker 35 - jepsen.util 15425 :invoke :write [11 37] INFO [2019-12-17 12:36:57,367] jepsen worker 82 - jepsen.util 15652 :invoke :read [27 nil] INFO [2019-12-17 12:36:57,367] jepsen worker 26 - jepsen.util 16136 :invoke :write [8 23] INFO [2019-12-17 12:36:57,369] jepsen worker 69 - jepsen.util 11499 :ok :read [23 30] INFO [2019-12-17 12:36:57,369] jepsen worker 69 - jepsen.util 11499 :invoke :write [23 4] INFO [2019-12-17 12:36:57,374] jepsen worker 28 - jepsen.util 9388 :fail :cas [9 [33 28]] :ValueNotSwapFrom INFO [2019-12-17 12:36:57,374] jepsen worker 28 - jepsen.util 9388 :invoke :write [9 48] INFO [2019-12-17 12:36:57,387] jepsen worker 46 - jepsen.util 11026 :ok :write [15 2] INFO [2019-12-17 12:36:57,387] jepsen worker 46 - jepsen.util 11026 :invoke :cas [15 [4 36]] INFO [2019-12-17 12:36:57,395] jepsen worker 55 - jepsen.util 11845 :fail :cas [18 [24 25]] :ValueNotSwapFrom INFO [2019-12-17 12:36:57,395] jepsen worker 55 - jepsen.util 11845 :invoke :cas [18 [13 5]] INFO [2019-12-17 12:36:57,396] jepsen worker 32 - jepsen.util 11642 :ok :write [10 22] INFO [2019-12-17 12:36:57,397] jepsen worker 32 - jepsen.util 11642 :invoke :cas [10 [34 38]] INFO [2019-12-17 12:36:57,398] jepsen worker 1 - jepsen.util 15211 :info :write [0 47] :RequestCanceledException INFO [2019-12-17 12:36:57,398] jepsen worker 22 - jepsen.util 15412 :info :write [7 11] :RequestCanceledException INFO [2019-12-17 12:36:57,399] jepsen worker 1 - jepsen.util 15301 :invoke :cas [0 [44 47]] INFO [2019-12-17 12:36:57,399] jepsen worker 22 - jepsen.util 15502 :invoke :write [7 11] INFO [2019-12-17 12:36:57,407] jepsen worker 78 - jepsen.util 11328 :ok :read [26 23] INFO [2019-12-17 12:36:57,407] jepsen worker 78 - jepsen.util 11328 :invoke :read [26 nil] INFO [2019-12-17 12:36:57,415] jepsen worker 88 - jepsen.util 11518 :ok :read [29 7] INFO [2019-12-17 12:36:57,416] jepsen worker 88 - jepsen.util 11518 :invoke :read [29 nil] INFO [2019-12-17 12:36:57,416] jepsen worker 38 - jepsen.util 11738 :ok :read [12 23] INFO [2019-12-17 12:36:57,416] jepsen worker 38 - jepsen.util 11738 :invoke :write [12 34] INFO [2019-12-17 12:36:57,433] jepsen worker 42 - jepsen.util 16242 :info :cas [14 [7 3]] :RequestCanceledException INFO [2019-12-17 12:36:57,433] jepsen worker 49 - jepsen.util 15439 :info :write [16 28] :RequestCanceledException INFO [2019-12-17 12:36:57,434] jepsen worker 42 - jepsen.util 16332 :invoke :write [14 11] INFO [2019-12-17 12:36:57,434] jepsen worker 49 - jepsen.util 15529 :invoke :read [16 nil] INFO [2019-12-17 12:36:57,436] jepsen worker 32 - jepsen.util 11642 :fail :cas [10 [34 38]] :ValueNotSwapFrom INFO [2019-12-17 12:36:57,437] jepsen worker 32 - jepsen.util 11642 :invoke :read [10 nil] INFO [2019-12-17 12:36:57,441] jepsen worker 46 - jepsen.util 11026 :fail :cas [15 [4 36]] :ValueNotSwapFrom INFO [2019-12-17 12:36:57,441] jepsen worker 46 - jepsen.util 11026 :invoke :read [15 nil] INFO [2019-12-17 12:36:57,447] jepsen worker 86 - jepsen.util 16106 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:36:57,447] jepsen worker 86 - jepsen.util 16106 :invoke :cas [28 [9 7]] INFO [2019-12-17 12:36:57,451] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65372,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:36:57,452] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65372,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:36:57,452] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65374,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:36:57,452] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65374,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:36:57,453] jepsen worker 87 - jepsen.util 10887 :fail :cas [29 [15 1]] :ValueNotSwapFrom INFO [2019-12-17 12:36:57,453] jepsen worker 87 - jepsen.util 10887 :invoke :read [29 nil] INFO [2019-12-17 12:36:57,463] jepsen worker 24 - jepsen.util 13974 :info :cas [8 [21 33]] :RequestCanceledException INFO [2019-12-17 12:36:57,464] jepsen worker 24 - jepsen.util 14064 :invoke :write [8 31] INFO [2019-12-17 12:36:57,476] jepsen worker 5 - jepsen.util 10985 :fail :cas [1 [38 15]] :ValueNotSwapFrom INFO [2019-12-17 12:36:57,476] jepsen worker 5 - jepsen.util 10985 :invoke :read [1 nil] INFO [2019-12-17 12:36:57,481] jepsen worker 32 - jepsen.util 11642 :ok :read [10 22] INFO [2019-12-17 12:36:57,481] jepsen worker 32 - jepsen.util 11642 :invoke :read [10 nil] INFO [2019-12-17 12:36:57,503] jepsen worker 46 - jepsen.util 11026 :ok :read [15 2] INFO [2019-12-17 12:36:57,503] jepsen worker 46 - jepsen.util 11026 :invoke :read [15 nil] INFO [2019-12-17 12:36:57,513] jepsen worker 37 - jepsen.util 10837 :ok :write [12 41] INFO [2019-12-17 12:36:57,514] jepsen worker 37 - jepsen.util 10837 :invoke :write [12 7] INFO [2019-12-17 12:36:57,518] jepsen worker 60 - jepsen.util 16440 :info :write [20 38] :RequestCanceledException INFO [2019-12-17 12:36:57,518] jepsen worker 72 - jepsen.util 11502 :info :write [24 0] :RequestCanceledException INFO [2019-12-17 12:36:57,518] jepsen worker 32 - jepsen.util 11642 :ok :read [10 22] INFO [2019-12-17 12:36:57,519] jepsen worker 32 - jepsen.util 11642 :invoke :write [10 45] INFO [2019-12-17 12:36:57,519] jepsen worker 60 - jepsen.util 16530 :invoke :cas [20 [30 37]] INFO [2019-12-17 12:36:57,519] jepsen worker 72 - jepsen.util 11592 :invoke :cas [24 [45 25]] INFO [2019-12-17 12:36:57,526] jepsen worker 39 - jepsen.util 15069 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:36:57,526] jepsen worker 39 - jepsen.util 15069 :invoke :read [13 nil] INFO [2019-12-17 12:36:57,558] jepsen worker 46 - jepsen.util 11026 :ok :read [15 2] INFO [2019-12-17 12:36:57,558] jepsen worker 46 - jepsen.util 11026 :invoke :read [15 nil] INFO [2019-12-17 12:36:57,579] jepsen worker 54 - jepsen.util 11844 :ok :write [18 34] INFO [2019-12-17 12:36:57,579] jepsen worker 54 - jepsen.util 11844 :invoke :read [18 nil] INFO [2019-12-17 12:36:57,583] jepsen worker 79 - jepsen.util 11419 :fail :cas [26 [35 22]] :ValueNotSwapFrom INFO [2019-12-17 12:36:57,583] jepsen worker 79 - jepsen.util 11419 :invoke :write [26 6] INFO [2019-12-17 12:36:57,585] jepsen worker 13 - jepsen.util 10633 :ok :write [4 39] INFO [2019-12-17 12:36:57,585] jepsen worker 13 - jepsen.util 10633 :invoke :cas [4 [20 48]] INFO [2019-12-17 12:36:57,591] jepsen worker 3 - jepsen.util 10713 :ok :read [1 1] INFO [2019-12-17 12:36:57,591] jepsen worker 3 - jepsen.util 10713 :invoke :read [1 nil] INFO [2019-12-17 12:36:57,620] jepsen worker 46 - jepsen.util 11026 :ok :read [15 2] INFO [2019-12-17 12:36:57,620] jepsen worker 46 - jepsen.util 11026 :invoke :read [15 nil] INFO [2019-12-17 12:36:57,621] jepsen worker 28 - jepsen.util 9388 :ok :write [9 48] INFO [2019-12-17 12:36:57,621] jepsen worker 28 - jepsen.util 9388 :invoke :read [9 nil] INFO [2019-12-17 12:36:57,625] jepsen worker 72 - jepsen.util 11592 :fail :cas [24 [45 25]] :ValueNotSwapFrom INFO [2019-12-17 12:36:57,625] jepsen worker 72 - jepsen.util 11592 :invoke :write [24 38] INFO [2019-12-17 12:36:57,637] jepsen worker 4 - jepsen.util 10354 :ok :read [1 1] INFO [2019-12-17 12:36:57,637] jepsen worker 4 - jepsen.util 10354 :invoke :write [1 10] INFO [2019-12-17 12:36:57,656] jepsen worker 55 - jepsen.util 11845 :fail :cas [18 [13 5]] :ValueNotSwapFrom INFO [2019-12-17 12:36:57,657] jepsen worker 55 - jepsen.util 11845 :invoke :write [18 36] INFO [2019-12-17 12:36:57,693] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65400,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:36:57,693] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65400,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:36:57,701] jepsen worker 5 - jepsen.util 10985 :ok :read [1 1] INFO [2019-12-17 12:36:57,701] jepsen worker 52 - jepsen.util 10672 :fail :cas [17 [40 4]] :ValueNotSwapFrom INFO [2019-12-17 12:36:57,701] jepsen worker 5 - jepsen.util 10985 :invoke :write [1 24] INFO [2019-12-17 12:36:57,701] jepsen worker 52 - jepsen.util 10672 :invoke :cas [17 [10 31]] INFO [2019-12-17 12:36:57,709] jepsen worker 71 - jepsen.util 12311 :ok :read [23 30] INFO [2019-12-17 12:36:57,709] jepsen worker 71 - jepsen.util 12311 :invoke :write [23 13] INFO [2019-12-17 12:36:57,713] jepsen worker 28 - jepsen.util 9388 :ok :read [9 48] INFO [2019-12-17 12:36:57,713] jepsen worker 28 - jepsen.util 9388 :invoke :cas [9 [18 2]] INFO [2019-12-17 12:36:57,731] jepsen worker 46 - jepsen.util 11026 :ok :read [15 2] INFO [2019-12-17 12:36:57,731] jepsen worker 46 - jepsen.util 11026 :invoke :read [15 nil] INFO [2019-12-17 12:36:57,732] jepsen worker 25 - jepsen.util 16315 :info :write [8 26] :RequestCanceledException INFO [2019-12-17 12:36:57,734] jepsen worker 25 - jepsen.util 16405 :invoke :write [8 33] INFO [2019-12-17 12:36:57,737] jepsen worker 14 - jepsen.util 11084 :ok :write [4 30] INFO [2019-12-17 12:36:57,737] jepsen worker 14 - jepsen.util 11084 :invoke :write [4 11] INFO [2019-12-17 12:36:57,743] jepsen worker 80 - jepsen.util 10970 :ok :write [26 6] INFO [2019-12-17 12:36:57,743] jepsen worker 80 - jepsen.util 10970 :invoke :cas [26 [0 28]] INFO [2019-12-17 12:36:57,757] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:36:57,757] jepsen nemesis - couchbase.util Rebalance check count: 514 INFO [2019-12-17 12:36:57,757] jepsen nemesis - couchbase.util Rebalance stuck count: 495 INFO [2019-12-17 12:36:57,759] jepsen worker 63 - jepsen.util 9963 :info :write [21 4] :RequestCanceledException INFO [2019-12-17 12:36:57,759] jepsen worker 45 - jepsen.util 11475 :info :cas [15 [6 9]] :RequestCanceledException INFO [2019-12-17 12:36:57,760] jepsen worker 63 - jepsen.util 10053 :invoke :write [21 37] INFO [2019-12-17 12:36:57,760] jepsen worker 45 - jepsen.util 11565 :invoke :write [15 5] INFO [2019-12-17 12:36:57,768] jepsen worker 28 - jepsen.util 9388 :fail :cas [9 [18 2]] :ValueNotSwapFrom INFO [2019-12-17 12:36:57,768] jepsen worker 28 - jepsen.util 9388 :invoke :write [9 0] INFO [2019-12-17 12:36:57,784] jepsen worker 46 - jepsen.util 11026 :ok :read [15 2] INFO [2019-12-17 12:36:57,785] jepsen worker 46 - jepsen.util 11026 :invoke :read [15 nil] INFO [2019-12-17 12:36:57,796] jepsen worker 33 - jepsen.util 15333 :info :write [11 41] :RequestCanceledException INFO [2019-12-17 12:36:57,798] jepsen worker 33 - jepsen.util 15423 :invoke :read [11 nil] INFO [2019-12-17 12:36:57,817] jepsen worker 74 - jepsen.util 10964 :info :write [24 45] :RequestCanceledException INFO [2019-12-17 12:36:57,819] jepsen worker 74 - jepsen.util 11054 :invoke :cas [24 [0 31]] INFO [2019-12-17 12:36:57,833] jepsen worker 29 - jepsen.util 9479 :fail :read [9 nil] :RequestTimeoutException INFO [2019-12-17 12:36:57,833] jepsen worker 30 - jepsen.util 10560 :fail :read [10 nil] :RequestTimeoutException INFO [2019-12-17 12:36:57,833] jepsen worker 29 - jepsen.util 9479 :invoke :read [9 nil] INFO [2019-12-17 12:36:57,833] jepsen worker 30 - jepsen.util 10560 :invoke :read [10 nil] INFO [2019-12-17 12:36:57,839] jepsen worker 0 - jepsen.util 16110 :info :write [0 27] :RequestCanceledException INFO [2019-12-17 12:36:57,840] jepsen worker 0 - jepsen.util 16200 :invoke :write [0 30] INFO [2019-12-17 12:36:57,860] jepsen worker 46 - jepsen.util 11026 :ok :read [15 2] INFO [2019-12-17 12:36:57,860] jepsen worker 46 - jepsen.util 11026 :invoke :read [15 nil] INFO [2019-12-17 12:36:57,862] jepsen worker 85 - jepsen.util 16195 :info :write [28 1] :RequestCanceledException INFO [2019-12-17 12:36:57,863] jepsen worker 85 - jepsen.util 16285 :invoke :read [28 nil] INFO [2019-12-17 12:36:57,872] jepsen worker 74 - jepsen.util 11054 :fail :cas [24 [0 31]] :ValueNotSwapFrom INFO [2019-12-17 12:36:57,872] jepsen worker 74 - jepsen.util 11054 :invoke :cas [24 [46 5]] INFO [2019-12-17 12:36:57,899] jepsen worker 20 - jepsen.util 10820 :info :cas [6 [42 21]] :RequestCanceledException INFO [2019-12-17 12:36:57,900] jepsen worker 20 - jepsen.util 10910 :invoke :write [6 14] INFO [2019-12-17 12:36:57,904] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65424,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:36:57,905] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65424,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:36:57,905] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65427,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:36:57,905] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65427,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:36:57,905] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65428,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:36:57,905] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65428,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:36:57,905] jepsen worker 46 - jepsen.util 11026 :ok :read [15 2] INFO [2019-12-17 12:36:57,906] jepsen worker 46 - jepsen.util 11026 :invoke :read [15 nil] INFO [2019-12-17 12:36:57,915] jepsen worker 64 - jepsen.util 8614 :info :cas [21 [45 18]] :RequestCanceledException INFO [2019-12-17 12:36:57,916] jepsen worker 64 - jepsen.util 8704 :invoke :read [21 nil] INFO [2019-12-17 12:36:57,924] jepsen worker 74 - jepsen.util 11054 :fail :cas [24 [46 5]] :ValueNotSwapFrom INFO [2019-12-17 12:36:57,924] jepsen worker 74 - jepsen.util 11054 :invoke :cas [24 [46 44]] INFO [2019-12-17 12:36:57,933] jepsen worker 11 - jepsen.util 10901 :info :write [3 27] :RequestCanceledException INFO [2019-12-17 12:36:57,933] jepsen worker 11 - jepsen.util 10991 :invoke :write [3 27] INFO [2019-12-17 12:36:57,939] jepsen worker 79 - jepsen.util 11419 :ok :write [26 6] INFO [2019-12-17 12:36:57,939] jepsen worker 79 - jepsen.util 11419 :invoke :write [26 47] INFO [2019-12-17 12:36:57,944] jepsen worker 51 - jepsen.util 10671 :fail :cas [17 [25 5]] :ValueNotSwapFrom INFO [2019-12-17 12:36:57,944] jepsen worker 51 - jepsen.util 10671 :invoke :read [17 nil] INFO [2019-12-17 12:36:57,949] jepsen worker 46 - jepsen.util 11026 :ok :read [15 2] INFO [2019-12-17 12:36:57,949] jepsen worker 46 - jepsen.util 11026 :invoke :write [15 29] INFO [2019-12-17 12:36:57,977] jepsen worker 69 - jepsen.util 11499 :ok :write [23 4] INFO [2019-12-17 12:36:57,977] jepsen worker 69 - jepsen.util 11499 :invoke :read [23 nil] INFO [2019-12-17 12:36:57,991] jepsen worker 74 - jepsen.util 11054 :fail :cas [24 [46 44]] :ValueNotSwapFrom INFO [2019-12-17 12:36:57,991] jepsen worker 74 - jepsen.util 11054 :invoke :cas [24 [36 6]] INFO [2019-12-17 12:36:57,998] jepsen worker 31 - jepsen.util 9661 :fail :read [10 nil] :RequestTimeoutException INFO [2019-12-17 12:36:57,999] jepsen worker 31 - jepsen.util 9661 :invoke :cas [10 [39 40]] INFO [2019-12-17 12:36:58,002] jepsen worker 63 - jepsen.util 10053 :ok :write [21 37] INFO [2019-12-17 12:36:58,002] jepsen worker 63 - jepsen.util 10053 :invoke :write [21 47] INFO [2019-12-17 12:36:58,011] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65443,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:36:58,011] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65443,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:36:58,011] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65446,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:36:58,011] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65446,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:36:58,011] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65447,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:36:58,011] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65447,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:36:58,033] jepsen worker 65 - jepsen.util 9785 :info :write [21 30] :RequestCanceledException INFO [2019-12-17 12:36:58,034] jepsen worker 65 - jepsen.util 9875 :invoke :cas [21 [37 37]] INFO [2019-12-17 12:36:58,038] jepsen worker 87 - jepsen.util 10887 :ok :read [29 7] INFO [2019-12-17 12:36:58,038] jepsen worker 87 - jepsen.util 10887 :invoke :write [29 5] INFO [2019-12-17 12:36:58,047] jepsen worker 27 - jepsen.util 10557 :fail :read [9 nil] :RequestTimeoutException INFO [2019-12-17 12:36:58,047] jepsen worker 19 - jepsen.util 11539 :info :write [6 29] :RequestCanceledException INFO [2019-12-17 12:36:58,047] jepsen worker 27 - jepsen.util 10557 :invoke :write [9 3] INFO [2019-12-17 12:36:58,048] jepsen worker 19 - jepsen.util 11629 :invoke :write [6 29] INFO [2019-12-17 12:36:58,050] jepsen worker 14 - jepsen.util 11084 :ok :write [4 11] INFO [2019-12-17 12:36:58,050] jepsen worker 14 - jepsen.util 11084 :invoke :read [4 nil] INFO [2019-12-17 12:36:58,054] jepsen worker 88 - jepsen.util 11518 :ok :read [29 7] INFO [2019-12-17 12:36:58,054] jepsen worker 88 - jepsen.util 11518 :invoke :cas [29 [33 20]] INFO [2019-12-17 12:36:58,081] jepsen worker 78 - jepsen.util 11328 :ok :read [26 6] INFO [2019-12-17 12:36:58,081] jepsen worker 78 - jepsen.util 11328 :invoke :read [26 nil] INFO [2019-12-17 12:36:58,087] jepsen worker 3 - jepsen.util 10713 :ok :read [1 1] INFO [2019-12-17 12:36:58,087] jepsen worker 3 - jepsen.util 10713 :invoke :write [1 6] INFO [2019-12-17 12:36:58,101] jepsen worker 52 - jepsen.util 10672 :fail :cas [17 [10 31]] :ValueNotSwapFrom INFO [2019-12-17 12:36:58,101] jepsen worker 52 - jepsen.util 10672 :invoke :cas [17 [17 14]] INFO [2019-12-17 12:36:58,115] jepsen worker 28 - jepsen.util 9388 :ok :write [9 0] INFO [2019-12-17 12:36:58,115] jepsen worker 28 - jepsen.util 9388 :invoke :write [9 42] INFO [2019-12-17 12:36:58,132] jepsen worker 67 - jepsen.util 15457 :info :cas [22 [31 12]] :RequestCanceledException INFO [2019-12-17 12:36:58,132] jepsen worker 48 - jepsen.util 14268 :info :cas [16 [38 31]] :RequestCanceledException INFO [2019-12-17 12:36:58,133] jepsen worker 48 - jepsen.util 14358 :invoke :read [16 nil] INFO [2019-12-17 12:36:58,134] jepsen worker 67 - jepsen.util 15547 :invoke :cas [22 [8 44]] INFO [2019-12-17 12:36:58,145] jepsen worker 8 - jepsen.util 15308 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:36:58,145] jepsen worker 8 - jepsen.util 15308 :invoke :cas [2 [27 31]] INFO [2019-12-17 12:36:58,164] jepsen worker 7 - jepsen.util 15487 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:36:58,164] jepsen worker 7 - jepsen.util 15487 :invoke :write [2 45] INFO [2019-12-17 12:36:58,164] jepsen worker 27 - jepsen.util 10557 :ok :write [9 3] INFO [2019-12-17 12:36:58,164] jepsen worker 27 - jepsen.util 10557 :invoke :read [9 nil] INFO [2019-12-17 12:36:58,171] jepsen worker 45 - jepsen.util 11565 :ok :write [15 5] INFO [2019-12-17 12:36:58,171] jepsen worker 45 - jepsen.util 11565 :invoke :read [15 nil] INFO [2019-12-17 12:36:58,177] jepsen worker 78 - jepsen.util 11328 :ok :read [26 6] INFO [2019-12-17 12:36:58,177] jepsen worker 78 - jepsen.util 11328 :invoke :read [26 nil] INFO [2019-12-17 12:36:58,179] jepsen worker 32 - jepsen.util 11642 :ok :write [10 45] INFO [2019-12-17 12:36:58,179] jepsen worker 32 - jepsen.util 11642 :invoke :write [10 38] INFO [2019-12-17 12:36:58,179] jepsen worker 72 - jepsen.util 11592 :ok :write [24 38] INFO [2019-12-17 12:36:58,180] jepsen worker 72 - jepsen.util 11592 :invoke :cas [24 [0 20]] INFO [2019-12-17 12:36:58,204] jepsen worker 51 - jepsen.util 10671 :ok :read [17 11] INFO [2019-12-17 12:36:58,204] jepsen worker 51 - jepsen.util 10671 :invoke :read [17 nil] INFO [2019-12-17 12:36:58,208] jepsen worker 27 - jepsen.util 10557 :ok :read [9 3] INFO [2019-12-17 12:36:58,209] jepsen worker 27 - jepsen.util 10557 :invoke :cas [9 [6 3]] INFO [2019-12-17 12:36:58,213] jepsen worker 64 - jepsen.util 8704 :ok :read [21 37] INFO [2019-12-17 12:36:58,213] jepsen worker 64 - jepsen.util 8704 :invoke :read [21 nil] INFO [2019-12-17 12:36:58,231] jepsen worker 14 - jepsen.util 11084 :ok :read [4 11] INFO [2019-12-17 12:36:58,231] jepsen worker 14 - jepsen.util 11084 :invoke :cas [4 [23 49]] INFO [2019-12-17 12:36:58,233] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65468,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:36:58,233] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65478,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:36:58,233] jepsen worker 61 - jepsen.util 16621 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:36:58,233] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65468,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:36:58,233] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65478,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:36:58,233] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65469,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:36:58,233] jepsen worker 61 - jepsen.util 16621 :invoke :read [20 nil] INFO [2019-12-17 12:36:58,233] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65482,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:36:58,234] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65469,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:36:58,234] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65482,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:36:58,234] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65470,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:36:58,234] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65470,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:36:58,258] jepsen worker 29 - jepsen.util 9479 :ok :read [9 3] INFO [2019-12-17 12:36:58,258] jepsen worker 13 - jepsen.util 10633 :fail :cas [4 [20 48]] :ValueNotSwapFrom INFO [2019-12-17 12:36:58,258] jepsen worker 29 - jepsen.util 9479 :invoke :write [9 15] INFO [2019-12-17 12:36:58,258] jepsen worker 13 - jepsen.util 10633 :invoke :read [4 nil] INFO [2019-12-17 12:36:58,263] jepsen worker 2 - jepsen.util 14942 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:36:58,263] jepsen worker 2 - jepsen.util 14942 :invoke :cas [0 [1 20]] INFO [2019-12-17 12:36:58,291] jepsen worker 51 - jepsen.util 10671 :ok :read [17 11] INFO [2019-12-17 12:36:58,292] jepsen worker 51 - jepsen.util 10671 :invoke :cas [17 [16 19]] INFO [2019-12-17 12:36:58,295] jepsen worker 54 - jepsen.util 11844 :ok :read [18 34] INFO [2019-12-17 12:36:58,295] jepsen worker 54 - jepsen.util 11844 :invoke :cas [18 [9 30]] INFO [2019-12-17 12:36:58,300] jepsen worker 73 - jepsen.util 10513 :info :write [24 16] :RequestCanceledException INFO [2019-12-17 12:36:58,300] jepsen worker 15 - jepsen.util 14415 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:36:58,300] jepsen worker 80 - jepsen.util 10970 :fail :cas [26 [0 28]] :ValueNotSwapFrom INFO [2019-12-17 12:36:58,300] jepsen worker 15 - jepsen.util 14415 :invoke :read [5 nil] INFO [2019-12-17 12:36:58,300] jepsen worker 80 - jepsen.util 10970 :invoke :cas [26 [49 38]] INFO [2019-12-17 12:36:58,301] jepsen worker 73 - jepsen.util 10603 :invoke :write [24 44] INFO [2019-12-17 12:36:58,305] jepsen worker 30 - jepsen.util 10560 :ok :read [10 45] INFO [2019-12-17 12:36:58,306] jepsen worker 30 - jepsen.util 10560 :invoke :read [10 nil] INFO [2019-12-17 12:36:58,306] jepsen worker 27 - jepsen.util 10557 :fail :cas [9 [6 3]] :ValueNotSwapFrom INFO [2019-12-17 12:36:58,306] jepsen worker 27 - jepsen.util 10557 :invoke :write [9 37] INFO [2019-12-17 12:36:58,306] jepsen worker 45 - jepsen.util 11565 :ok :read [15 5] INFO [2019-12-17 12:36:58,306] jepsen worker 45 - jepsen.util 11565 :invoke :read [15 nil] INFO [2019-12-17 12:36:58,314] jepsen worker 64 - jepsen.util 8704 :ok :read [21 37] INFO [2019-12-17 12:36:58,314] jepsen worker 64 - jepsen.util 8704 :invoke :read [21 nil] INFO [2019-12-17 12:36:58,318] jepsen worker 58 - jepsen.util 15718 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:36:58,318] jepsen worker 18 - jepsen.util 11358 :fail :read [6 nil] :RequestTimeoutException INFO [2019-12-17 12:36:58,318] jepsen worker 18 - jepsen.util 11358 :invoke :read [6 nil] INFO [2019-12-17 12:36:58,318] jepsen worker 58 - jepsen.util 15718 :invoke :write [19 18] INFO [2019-12-17 12:36:58,333] jepsen worker 43 - jepsen.util 15613 :info :write [14 11] :RequestCanceledException INFO [2019-12-17 12:36:58,333] jepsen worker 17 - jepsen.util 15407 :info :cas [5 [42 46]] :RequestCanceledException INFO [2019-12-17 12:36:58,333] jepsen worker 84 - jepsen.util 15204 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:36:58,333] jepsen worker 84 - jepsen.util 15204 :invoke :cas [28 [4 30]] INFO [2019-12-17 12:36:58,334] jepsen worker 43 - jepsen.util 15703 :invoke :read [14 nil] INFO [2019-12-17 12:36:58,334] jepsen worker 17 - jepsen.util 15497 :invoke :read [5 nil] INFO [2019-12-17 12:36:58,338] jepsen worker 20 - jepsen.util 10910 :ok :write [6 14] INFO [2019-12-17 12:36:58,338] jepsen worker 20 - jepsen.util 10910 :invoke :read [6 nil] INFO [2019-12-17 12:36:58,347] jepsen worker 18 - jepsen.util 11358 :ok :read [6 25] INFO [2019-12-17 12:36:58,347] jepsen worker 50 - jepsen.util 15890 :info :cas [16 [48 30]] :RequestCanceledException INFO [2019-12-17 12:36:58,347] jepsen worker 18 - jepsen.util 11358 :invoke :write [6 41] INFO [2019-12-17 12:36:58,348] jepsen worker 50 - jepsen.util 15980 :invoke :write [16 49] INFO [2019-12-17 12:36:58,363] jepsen worker 23 - jepsen.util 15413 :info :cas [7 [2 32]] :RequestCanceledException INFO [2019-12-17 12:36:58,363] jepsen worker 10 - jepsen.util 10990 :info :write [3 13] :RequestCanceledException INFO [2019-12-17 12:36:58,365] jepsen worker 10 - jepsen.util 11080 :invoke :read [3 nil] INFO [2019-12-17 12:36:58,365] jepsen worker 23 - jepsen.util 15503 :invoke :read [7 nil] INFO [2019-12-17 12:36:58,368] jepsen worker 69 - jepsen.util 11499 :ok :read [23 4] INFO [2019-12-17 12:36:58,368] jepsen worker 69 - jepsen.util 11499 :invoke :read [23 nil] INFO [2019-12-17 12:36:58,380] jepsen worker 20 - jepsen.util 10910 :ok :read [6 14] INFO [2019-12-17 12:36:58,380] jepsen worker 20 - jepsen.util 10910 :invoke :read [6 nil] INFO [2019-12-17 12:36:58,382] jepsen worker 80 - jepsen.util 10970 :fail :cas [26 [49 38]] :ValueNotSwapFrom INFO [2019-12-17 12:36:58,382] jepsen worker 80 - jepsen.util 10970 :invoke :cas [26 [19 22]] INFO [2019-12-17 12:36:58,384] jepsen worker 14 - jepsen.util 11084 :fail :cas [4 [23 49]] :ValueNotSwapFrom INFO [2019-12-17 12:36:58,384] jepsen worker 14 - jepsen.util 11084 :invoke :read [4 nil] INFO [2019-12-17 12:36:58,408] jepsen worker 27 - jepsen.util 10557 :ok :write [9 37] INFO [2019-12-17 12:36:58,408] jepsen worker 27 - jepsen.util 10557 :invoke :write [9 44] INFO [2019-12-17 12:36:58,412] jepsen worker 20 - jepsen.util 10910 :ok :read [6 14] INFO [2019-12-17 12:36:58,413] jepsen worker 20 - jepsen.util 10910 :invoke :cas [6 [10 23]] INFO [2019-12-17 12:36:58,417] jepsen worker 62 - jepsen.util 16082 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:36:58,418] jepsen worker 62 - jepsen.util 16082 :invoke :read [20 nil] INFO [2019-12-17 12:36:58,432] jepsen worker 47 - jepsen.util 12017 :info :write [15 35] :RequestCanceledException INFO [2019-12-17 12:36:58,433] jepsen worker 47 - jepsen.util 12107 :invoke :read [15 nil] INFO [2019-12-17 12:36:58,449] jepsen worker 10 - jepsen.util 11080 :ok :read [3 20] INFO [2019-12-17 12:36:58,450] jepsen worker 10 - jepsen.util 11080 :invoke :read [3 nil] INFO [2019-12-17 12:36:58,453] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65516,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:36:58,453] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65516,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:36:58,453] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65519,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:36:58,453] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65519,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:36:58,453] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65520,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:36:58,453] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65520,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:36:58,464] jepsen worker 21 - jepsen.util 14871 :info :write [7 46] :RequestCanceledException INFO [2019-12-17 12:36:58,465] jepsen worker 21 - jepsen.util 14961 :invoke :write [7 17] INFO [2019-12-17 12:36:58,482] jepsen worker 51 - jepsen.util 10671 :fail :cas [17 [16 19]] :ValueNotSwapFrom INFO [2019-12-17 12:36:58,482] jepsen worker 51 - jepsen.util 10671 :invoke :read [17 nil] INFO [2019-12-17 12:36:58,487] jepsen worker 20 - jepsen.util 10910 :fail :cas [6 [10 23]] :ValueNotSwapFrom INFO [2019-12-17 12:36:58,487] jepsen worker 20 - jepsen.util 10910 :invoke :cas [6 [0 36]] INFO [2019-12-17 12:36:58,500] jepsen worker 88 - jepsen.util 11518 :fail :cas [29 [33 20]] :ValueNotSwapFrom INFO [2019-12-17 12:36:58,501] jepsen worker 88 - jepsen.util 11518 :invoke :cas [29 [49 23]] INFO [2019-12-17 12:36:58,504] jepsen worker 20 - jepsen.util 10910 :fail :cas [6 [0 36]] :ValueNotSwapFrom INFO [2019-12-17 12:36:58,505] jepsen worker 20 - jepsen.util 10910 :invoke :cas [6 [13 44]] INFO [2019-12-17 12:36:58,515] jepsen worker 20 - jepsen.util 10910 :fail :cas [6 [13 44]] :ValueNotSwapFrom INFO [2019-12-17 12:36:58,515] jepsen worker 20 - jepsen.util 10910 :invoke :cas [6 [21 11]] INFO [2019-12-17 12:36:58,527] jepsen worker 80 - jepsen.util 10970 :fail :cas [26 [19 22]] :ValueNotSwapFrom INFO [2019-12-17 12:36:58,527] jepsen worker 80 - jepsen.util 10970 :invoke :read [26 nil] INFO [2019-12-17 12:36:58,532] jepsen worker 20 - jepsen.util 10910 :fail :cas [6 [21 11]] :ValueNotSwapFrom INFO [2019-12-17 12:36:58,533] jepsen worker 20 - jepsen.util 10910 :invoke :cas [6 [25 7]] INFO [2019-12-17 12:36:58,534] jepsen worker 14 - jepsen.util 11084 :ok :read [4 11] INFO [2019-12-17 12:36:58,534] jepsen worker 14 - jepsen.util 11084 :invoke :read [4 nil] INFO [2019-12-17 12:36:58,562] jepsen worker 51 - jepsen.util 10671 :ok :read [17 11] INFO [2019-12-17 12:36:58,562] jepsen worker 51 - jepsen.util 10671 :invoke :write [17 46] INFO [2019-12-17 12:36:58,564] jepsen worker 20 - jepsen.util 10910 :fail :cas [6 [25 7]] :ValueNotSwapFrom INFO [2019-12-17 12:36:58,564] jepsen worker 20 - jepsen.util 10910 :invoke :write [6 32] INFO [2019-12-17 12:36:58,574] jepsen worker 52 - jepsen.util 10672 :fail :cas [17 [17 14]] :ValueNotSwapFrom INFO [2019-12-17 12:36:58,574] jepsen worker 52 - jepsen.util 10672 :invoke :read [17 nil] INFO [2019-12-17 12:36:58,579] jepsen worker 27 - jepsen.util 10557 :ok :write [9 44] INFO [2019-12-17 12:36:58,579] jepsen worker 27 - jepsen.util 10557 :invoke :cas [9 [43 45]] INFO [2019-12-17 12:36:58,600] jepsen worker 9 - jepsen.util 11889 :fail :read [3 nil] :RequestTimeoutException INFO [2019-12-17 12:36:58,600] jepsen worker 9 - jepsen.util 11889 :invoke :read [3 nil] INFO [2019-12-17 12:36:58,605] jepsen worker 80 - jepsen.util 10970 :ok :read [26 6] INFO [2019-12-17 12:36:58,605] jepsen worker 27 - jepsen.util 10557 :fail :cas [9 [43 45]] :ValueNotSwapFrom INFO [2019-12-17 12:36:58,605] jepsen worker 80 - jepsen.util 10970 :invoke :read [26 nil] INFO [2019-12-17 12:36:58,605] jepsen worker 27 - jepsen.util 10557 :invoke :read [9 nil] INFO [2019-12-17 12:36:58,616] jepsen worker 27 - jepsen.util 10557 :ok :read [9 44] INFO [2019-12-17 12:36:58,616] jepsen worker 27 - jepsen.util 10557 :invoke :write [9 16] INFO [2019-12-17 12:36:58,635] jepsen worker 14 - jepsen.util 11084 :ok :read [4 11] INFO [2019-12-17 12:36:58,635] jepsen worker 14 - jepsen.util 11084 :invoke :cas [4 [30 22]] INFO [2019-12-17 12:36:58,641] jepsen worker 52 - jepsen.util 10672 :ok :read [17 11] INFO [2019-12-17 12:36:58,641] jepsen worker 52 - jepsen.util 10672 :invoke :write [17 28] INFO [2019-12-17 12:36:58,659] jepsen worker 27 - jepsen.util 10557 :ok :write [9 16] INFO [2019-12-17 12:36:58,659] jepsen worker 27 - jepsen.util 10557 :invoke :cas [9 [4 49]] INFO [2019-12-17 12:36:58,676] jepsen worker 27 - jepsen.util 10557 :fail :cas [9 [4 49]] :ValueNotSwapFrom INFO [2019-12-17 12:36:58,676] jepsen worker 27 - jepsen.util 10557 :invoke :read [9 nil] INFO [2019-12-17 12:36:58,679] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65541,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:36:58,679] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65541,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:36:58,679] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65543,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:36:58,679] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65543,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:36:58,688] jepsen worker 27 - jepsen.util 10557 :ok :read [9 16] INFO [2019-12-17 12:36:58,688] jepsen worker 27 - jepsen.util 10557 :invoke :read [9 nil] INFO [2019-12-17 12:36:58,697] jepsen worker 27 - jepsen.util 10557 :ok :read [9 16] INFO [2019-12-17 12:36:58,697] jepsen worker 27 - jepsen.util 10557 :invoke :read [9 nil] INFO [2019-12-17 12:36:58,708] jepsen worker 27 - jepsen.util 10557 :ok :read [9 16] INFO [2019-12-17 12:36:58,708] jepsen worker 27 - jepsen.util 10557 :invoke :write [9 21] INFO [2019-12-17 12:36:58,727] jepsen worker 69 - jepsen.util 11499 :ok :read [23 4] INFO [2019-12-17 12:36:58,727] jepsen worker 69 - jepsen.util 11499 :invoke :write [23 40] INFO [2019-12-17 12:36:58,738] jepsen worker 54 - jepsen.util 11844 :fail :cas [18 [9 30]] :ValueNotSwapFrom INFO [2019-12-17 12:36:58,739] jepsen worker 54 - jepsen.util 11844 :invoke :cas [18 [11 5]] INFO [2019-12-17 12:36:58,751] jepsen worker 27 - jepsen.util 10557 :ok :write [9 21] INFO [2019-12-17 12:36:58,752] jepsen worker 27 - jepsen.util 10557 :invoke :cas [9 [25 7]] INFO [2019-12-17 12:36:58,765] jepsen worker 88 - jepsen.util 11518 :fail :cas [29 [49 23]] :ValueNotSwapFrom INFO [2019-12-17 12:36:58,765] jepsen worker 88 - jepsen.util 11518 :invoke :cas [29 [15 27]] INFO [2019-12-17 12:36:58,774] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:36:58,774] jepsen nemesis - couchbase.util Rebalance check count: 515 INFO [2019-12-17 12:36:58,774] jepsen nemesis - couchbase.util Rebalance stuck count: 496 INFO [2019-12-17 12:36:58,777] jepsen worker 27 - jepsen.util 10557 :fail :cas [9 [25 7]] :ValueNotSwapFrom INFO [2019-12-17 12:36:58,777] jepsen worker 27 - jepsen.util 10557 :invoke :read [9 nil] INFO [2019-12-17 12:36:58,786] jepsen worker 27 - jepsen.util 10557 :ok :read [9 21] INFO [2019-12-17 12:36:58,787] jepsen worker 27 - jepsen.util 10557 :invoke :read [9 nil] INFO [2019-12-17 12:36:58,797] jepsen worker 14 - jepsen.util 11084 :fail :cas [4 [30 22]] :ValueNotSwapFrom INFO [2019-12-17 12:36:58,797] jepsen worker 14 - jepsen.util 11084 :invoke :write [4 1] INFO [2019-12-17 12:36:58,799] jepsen worker 27 - jepsen.util 10557 :ok :read [9 21] INFO [2019-12-17 12:36:58,800] jepsen worker 27 - jepsen.util 10557 :invoke :cas [9 [22 38]] INFO [2019-12-17 12:36:58,808] jepsen worker 11 - jepsen.util 10991 :ok :write [3 27] INFO [2019-12-17 12:36:58,809] jepsen worker 11 - jepsen.util 10991 :invoke :cas [3 [15 49]] INFO [2019-12-17 12:36:58,812] jepsen worker 27 - jepsen.util 10557 :fail :cas [9 [22 38]] :ValueNotSwapFrom INFO [2019-12-17 12:36:58,812] jepsen worker 27 - jepsen.util 10557 :invoke :cas [9 [45 41]] INFO [2019-12-17 12:36:58,829] jepsen worker 80 - jepsen.util 10970 :ok :read [26 6] INFO [2019-12-17 12:36:58,830] jepsen worker 80 - jepsen.util 10970 :invoke :cas [26 [29 0]] INFO [2019-12-17 12:36:58,834] jepsen worker 27 - jepsen.util 10557 :fail :cas [9 [45 41]] :ValueNotSwapFrom INFO [2019-12-17 12:36:58,834] jepsen worker 27 - jepsen.util 10557 :invoke :read [9 nil] INFO [2019-12-17 12:36:58,835] jepsen worker 19 - jepsen.util 11629 :ok :write [6 29] INFO [2019-12-17 12:36:58,835] jepsen worker 19 - jepsen.util 11629 :invoke :cas [6 [15 46]] INFO [2019-12-17 12:36:58,849] jepsen worker 73 - jepsen.util 10603 :ok :write [24 44] INFO [2019-12-17 12:36:58,849] jepsen worker 73 - jepsen.util 10603 :invoke :read [24 nil] INFO [2019-12-17 12:36:58,865] jepsen worker 27 - jepsen.util 10557 :ok :read [9 21] INFO [2019-12-17 12:36:58,865] jepsen worker 27 - jepsen.util 10557 :invoke :read [9 nil] INFO [2019-12-17 12:36:58,873] jepsen worker 45 - jepsen.util 11565 :ok :read [15 5] INFO [2019-12-17 12:36:58,873] jepsen worker 45 - jepsen.util 11565 :invoke :read [15 nil] INFO [2019-12-17 12:36:58,890] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65571,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:36:58,891] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65571,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:36:58,891] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65572,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:36:58,891] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65572,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:36:58,910] jepsen worker 27 - jepsen.util 10557 :ok :read [9 21] INFO [2019-12-17 12:36:58,910] jepsen worker 27 - jepsen.util 10557 :invoke :read [9 nil] INFO [2019-12-17 12:36:58,911] jepsen worker 73 - jepsen.util 10603 :ok :read [24 44] INFO [2019-12-17 12:36:58,911] jepsen worker 73 - jepsen.util 10603 :invoke :read [24 nil] INFO [2019-12-17 12:36:58,930] jepsen worker 70 - jepsen.util 11860 :info :write [23 5] :RequestCanceledException INFO [2019-12-17 12:36:58,930] jepsen worker 53 - jepsen.util 10583 :info :write [17 39] :RequestCanceledException INFO [2019-12-17 12:36:58,932] jepsen worker 70 - jepsen.util 11950 :invoke :read [23 nil] INFO [2019-12-17 12:36:58,932] jepsen worker 53 - jepsen.util 10673 :invoke :write [17 43] INFO [2019-12-17 12:36:58,937] jepsen worker 45 - jepsen.util 11565 :ok :read [15 5] INFO [2019-12-17 12:36:58,937] jepsen worker 45 - jepsen.util 11565 :invoke :cas [15 [9 3]] INFO [2019-12-17 12:36:58,938] jepsen worker 19 - jepsen.util 11629 :fail :cas [6 [15 46]] :ValueNotSwapFrom INFO [2019-12-17 12:36:58,938] jepsen worker 19 - jepsen.util 11629 :invoke :read [6 nil] INFO [2019-12-17 12:36:58,939] jepsen worker 80 - jepsen.util 10970 :fail :cas [26 [29 0]] :ValueNotSwapFrom INFO [2019-12-17 12:36:58,940] jepsen worker 80 - jepsen.util 10970 :invoke :write [26 26] INFO [2019-12-17 12:36:58,977] jepsen worker 19 - jepsen.util 11629 :ok :read [6 29] INFO [2019-12-17 12:36:58,977] jepsen worker 19 - jepsen.util 11629 :invoke :write [6 22] INFO [2019-12-17 12:36:58,984] jepsen worker 27 - jepsen.util 10557 :ok :read [9 21] INFO [2019-12-17 12:36:58,984] jepsen worker 27 - jepsen.util 10557 :invoke :read [9 nil] INFO [2019-12-17 12:36:58,986] jepsen worker 73 - jepsen.util 10603 :ok :read [24 44] INFO [2019-12-17 12:36:58,986] jepsen worker 73 - jepsen.util 10603 :invoke :read [24 nil] INFO [2019-12-17 12:36:59,000] jepsen worker 45 - jepsen.util 11565 :fail :cas [15 [9 3]] :ValueNotSwapFrom INFO [2019-12-17 12:36:59,000] jepsen worker 45 - jepsen.util 11565 :invoke :cas [15 [43 3]] INFO [2019-12-17 12:36:59,042] jepsen worker 27 - jepsen.util 10557 :ok :read [9 21] INFO [2019-12-17 12:36:59,043] jepsen worker 27 - jepsen.util 10557 :invoke :cas [9 [3 41]] INFO [2019-12-17 12:36:59,047] jepsen worker 73 - jepsen.util 10603 :ok :read [24 44] INFO [2019-12-17 12:36:59,047] jepsen worker 73 - jepsen.util 10603 :invoke :write [24 14] INFO [2019-12-17 12:36:59,073] jepsen worker 5 - jepsen.util 10985 :ok :write [1 24] INFO [2019-12-17 12:36:59,073] jepsen worker 5 - jepsen.util 10985 :invoke :write [1 42] INFO [2019-12-17 12:36:59,076] jepsen worker 27 - jepsen.util 10557 :fail :cas [9 [3 41]] :ValueNotSwapFrom INFO [2019-12-17 12:36:59,076] jepsen worker 27 - jepsen.util 10557 :invoke :cas [9 [48 27]] INFO [2019-12-17 12:36:59,078] jepsen worker 45 - jepsen.util 11565 :fail :cas [15 [43 3]] :ValueNotSwapFrom INFO [2019-12-17 12:36:59,078] jepsen worker 45 - jepsen.util 11565 :invoke :write [15 27] INFO [2019-12-17 12:36:59,100] jepsen worker 32 - jepsen.util 11642 :ok :write [10 38] INFO [2019-12-17 12:36:59,100] jepsen worker 32 - jepsen.util 11642 :invoke :read [10 nil] INFO [2019-12-17 12:36:59,104] jepsen worker 27 - jepsen.util 10557 :fail :cas [9 [48 27]] :ValueNotSwapFrom INFO [2019-12-17 12:36:59,104] jepsen worker 27 - jepsen.util 10557 :invoke :write [9 7] INFO [2019-12-17 12:36:59,105] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65586,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:36:59,105] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65586,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:36:59,105] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65589,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:36:59,105] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65589,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:36:59,105] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65590,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:36:59,105] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65590,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:36:59,117] jepsen worker 19 - jepsen.util 11629 :ok :write [6 22] INFO [2019-12-17 12:36:59,117] jepsen worker 19 - jepsen.util 11629 :invoke :cas [6 [15 30]] INFO [2019-12-17 12:36:59,120] jepsen worker 9 - jepsen.util 11889 :ok :read [3 27] INFO [2019-12-17 12:36:59,120] jepsen worker 9 - jepsen.util 11889 :invoke :cas [3 [34 0]] INFO [2019-12-17 12:36:59,133] jepsen worker 36 - jepsen.util 12096 :info :write [12 36] :RequestCanceledException INFO [2019-12-17 12:36:59,134] jepsen worker 36 - jepsen.util 12186 :invoke :read [12 nil] INFO [2019-12-17 12:36:59,147] jepsen worker 11 - jepsen.util 10991 :fail :cas [3 [15 49]] :ValueNotSwapFrom INFO [2019-12-17 12:36:59,147] jepsen worker 11 - jepsen.util 10991 :invoke :cas [3 [26 24]] INFO [2019-12-17 12:36:59,175] jepsen worker 32 - jepsen.util 11642 :ok :read [10 38] INFO [2019-12-17 12:36:59,175] jepsen worker 32 - jepsen.util 11642 :invoke :cas [10 [28 5]] INFO [2019-12-17 12:36:59,177] jepsen worker 19 - jepsen.util 11629 :fail :cas [6 [15 30]] :ValueNotSwapFrom INFO [2019-12-17 12:36:59,177] jepsen worker 19 - jepsen.util 11629 :invoke :read [6 nil] INFO [2019-12-17 12:36:59,191] jepsen worker 71 - jepsen.util 12311 :ok :write [23 13] INFO [2019-12-17 12:36:59,191] jepsen worker 71 - jepsen.util 12311 :invoke :read [23 nil] INFO [2019-12-17 12:36:59,192] jepsen worker 54 - jepsen.util 11844 :fail :cas [18 [11 5]] :ValueNotSwapFrom INFO [2019-12-17 12:36:59,192] jepsen worker 54 - jepsen.util 11844 :invoke :write [18 5] INFO [2019-12-17 12:36:59,199] jepsen worker 89 - jepsen.util 11249 :info :write [29 46] :RequestCanceledException INFO [2019-12-17 12:36:59,200] jepsen worker 89 - jepsen.util 11339 :invoke :write [29 13] INFO [2019-12-17 12:36:59,214] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65606,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:36:59,215] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65606,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:36:59,215] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65607,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:36:59,216] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65607,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:36:59,222] jepsen worker 9 - jepsen.util 11889 :fail :cas [3 [34 0]] :ValueNotSwapFrom INFO [2019-12-17 12:36:59,222] jepsen worker 9 - jepsen.util 11889 :invoke :cas [3 [49 2]] INFO [2019-12-17 12:36:59,236] jepsen worker 19 - jepsen.util 11629 :ok :read [6 22] INFO [2019-12-17 12:36:59,237] jepsen worker 19 - jepsen.util 11629 :invoke :cas [6 [13 15]] INFO [2019-12-17 12:36:59,239] jepsen worker 11 - jepsen.util 10991 :fail :cas [3 [26 24]] :ValueNotSwapFrom INFO [2019-12-17 12:36:59,240] jepsen worker 11 - jepsen.util 10991 :invoke :read [3 nil] INFO [2019-12-17 12:36:59,287] jepsen worker 52 - jepsen.util 10672 :ok :write [17 28] INFO [2019-12-17 12:36:59,287] jepsen worker 52 - jepsen.util 10672 :invoke :read [17 nil] INFO [2019-12-17 12:36:59,288] jepsen worker 32 - jepsen.util 11642 :fail :cas [10 [28 5]] :ValueNotSwapFrom INFO [2019-12-17 12:36:59,288] jepsen worker 32 - jepsen.util 11642 :invoke :cas [10 [45 45]] INFO [2019-12-17 12:36:59,296] jepsen worker 27 - jepsen.util 10557 :ok :write [9 7] INFO [2019-12-17 12:36:59,296] jepsen worker 19 - jepsen.util 11629 :fail :cas [6 [13 15]] :ValueNotSwapFrom INFO [2019-12-17 12:36:59,296] jepsen worker 27 - jepsen.util 10557 :invoke :write [9 8] INFO [2019-12-17 12:36:59,296] jepsen worker 19 - jepsen.util 11629 :invoke :write [6 40] INFO [2019-12-17 12:36:59,297] jepsen worker 56 - jepsen.util 12026 :info :write [18 46] :RequestCanceledException INFO [2019-12-17 12:36:59,298] jepsen worker 56 - jepsen.util 12116 :invoke :read [18 nil] INFO [2019-12-17 12:36:59,301] jepsen worker 9 - jepsen.util 11889 :fail :cas [3 [49 2]] :ValueNotSwapFrom INFO [2019-12-17 12:36:59,301] jepsen worker 9 - jepsen.util 11889 :invoke :read [3 nil] INFO [2019-12-17 12:36:59,317] jepsen worker 68 - jepsen.util 14828 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:36:59,317] jepsen worker 68 - jepsen.util 14828 :invoke :cas [22 [23 34]] INFO [2019-12-17 12:36:59,328] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65617,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:36:59,328] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65617,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:36:59,328] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65618,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:36:59,328] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65618,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:36:59,333] jepsen worker 81 - jepsen.util 15111 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:36:59,334] jepsen worker 81 - jepsen.util 15111 :invoke :read [27 nil] INFO [2019-12-17 12:36:59,338] jepsen worker 37 - jepsen.util 10837 :ok :write [12 7] INFO [2019-12-17 12:36:59,338] jepsen worker 37 - jepsen.util 10837 :invoke :cas [12 [32 30]] INFO [2019-12-17 12:36:59,338] jepsen worker 11 - jepsen.util 10991 :ok :read [3 27] INFO [2019-12-17 12:36:59,338] jepsen worker 11 - jepsen.util 10991 :invoke :read [3 nil] INFO [2019-12-17 12:36:59,346] jepsen worker 75 - jepsen.util 15015 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:36:59,346] jepsen worker 75 - jepsen.util 15015 :invoke :write [25 1] INFO [2019-12-17 12:36:59,354] jepsen worker 32 - jepsen.util 11642 :fail :cas [10 [45 45]] :ValueNotSwapFrom INFO [2019-12-17 12:36:59,354] jepsen worker 32 - jepsen.util 11642 :invoke :write [10 48] INFO [2019-12-17 12:36:59,363] jepsen worker 44 - jepsen.util 15974 :info :cas [14 [17 17]] :RequestCanceledException INFO [2019-12-17 12:36:59,363] jepsen worker 6 - jepsen.util 15846 :info :cas [2 [47 43]] :RequestCanceledException INFO [2019-12-17 12:36:59,365] jepsen worker 44 - jepsen.util 16064 :invoke :cas [14 [0 34]] INFO [2019-12-17 12:36:59,366] jepsen worker 6 - jepsen.util 15936 :invoke :cas [2 [4 42]] INFO [2019-12-17 12:36:59,384] jepsen worker 9 - jepsen.util 11889 :ok :read [3 27] INFO [2019-12-17 12:36:59,384] jepsen worker 9 - jepsen.util 11889 :invoke :write [3 18] INFO [2019-12-17 12:36:59,400] jepsen worker 82 - jepsen.util 15652 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:36:59,400] jepsen worker 12 - jepsen.util 11532 :info :write [4 38] :RequestCanceledException INFO [2019-12-17 12:36:59,400] jepsen worker 34 - jepsen.util 15874 :info :cas [11 [47 17]] :RequestCanceledException INFO [2019-12-17 12:36:59,400] jepsen worker 82 - jepsen.util 15652 :invoke :write [27 11] INFO [2019-12-17 12:36:59,401] jepsen worker 12 - jepsen.util 11622 :invoke :read [4 nil] INFO [2019-12-17 12:36:59,401] jepsen worker 34 - jepsen.util 15964 :invoke :write [11 27] INFO [2019-12-17 12:36:59,403] jepsen worker 11 - jepsen.util 10991 :ok :read [3 27] INFO [2019-12-17 12:36:59,403] jepsen worker 11 - jepsen.util 10991 :invoke :cas [3 [27 48]] INFO [2019-12-17 12:36:59,411] jepsen worker 27 - jepsen.util 10557 :ok :write [9 8] INFO [2019-12-17 12:36:59,411] jepsen worker 27 - jepsen.util 10557 :invoke :read [9 nil] INFO [2019-12-17 12:36:59,415] jepsen worker 40 - jepsen.util 15970 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:36:59,415] jepsen worker 26 - jepsen.util 16136 :info :write [8 23] :RequestCanceledException INFO [2019-12-17 12:36:59,415] jepsen worker 38 - jepsen.util 11738 :ok :write [12 34] INFO [2019-12-17 12:36:59,415] jepsen worker 40 - jepsen.util 15970 :invoke :write [13 49] INFO [2019-12-17 12:36:59,415] jepsen worker 38 - jepsen.util 11738 :invoke :read [12 nil] INFO [2019-12-17 12:36:59,416] jepsen worker 26 - jepsen.util 16226 :invoke :cas [8 [45 40]] INFO [2019-12-17 12:36:59,424] jepsen worker 73 - jepsen.util 10603 :ok :write [24 14] INFO [2019-12-17 12:36:59,424] jepsen worker 73 - jepsen.util 10603 :invoke :read [24 nil] INFO [2019-12-17 12:36:59,424] jepsen worker 27 - jepsen.util 10557 :ok :read [9 8] INFO [2019-12-17 12:36:59,424] jepsen worker 27 - jepsen.util 10557 :invoke :write [9 32] INFO [2019-12-17 12:36:59,434] jepsen worker 83 - jepsen.util 15563 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:36:59,434] jepsen worker 66 - jepsen.util 15366 :info :cas [22 [43 33]] :RequestCanceledException INFO [2019-12-17 12:36:59,434] jepsen worker 41 - jepsen.util 16511 :info :write [13 37] :RequestCanceledException INFO [2019-12-17 12:36:59,434] jepsen worker 1 - jepsen.util 15301 :info :cas [0 [44 47]] :RequestCanceledException INFO [2019-12-17 12:36:59,434] jepsen worker 57 - jepsen.util 15177 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:36:59,434] jepsen worker 76 - jepsen.util 15826 :info :write [25 24] :RequestCanceledException INFO [2019-12-17 12:36:59,434] jepsen worker 16 - jepsen.util 17026 :info :write [5 2] :RequestCanceledException INFO [2019-12-17 12:36:59,434] jepsen worker 35 - jepsen.util 15425 :info :write [11 37] :RequestCanceledException INFO [2019-12-17 12:36:59,434] jepsen worker 83 - jepsen.util 15563 :invoke :write [27 36] INFO [2019-12-17 12:36:59,434] jepsen worker 57 - jepsen.util 15177 :invoke :cas [19 [39 19]] INFO [2019-12-17 12:36:59,435] jepsen worker 41 - jepsen.util 16601 :invoke :cas [13 [24 19]] INFO [2019-12-17 12:36:59,435] jepsen worker 66 - jepsen.util 15456 :invoke :cas [22 [16 20]] INFO [2019-12-17 12:36:59,435] jepsen worker 1 - jepsen.util 15391 :invoke :cas [0 [0 9]] INFO [2019-12-17 12:36:59,435] jepsen worker 76 - jepsen.util 15916 :invoke :read [25 nil] INFO [2019-12-17 12:36:59,435] jepsen worker 16 - jepsen.util 17116 :invoke :cas [5 [48 26]] INFO [2019-12-17 12:36:59,436] jepsen worker 35 - jepsen.util 15515 :invoke :read [11 nil] INFO [2019-12-17 12:36:59,447] jepsen worker 59 - jepsen.util 14729 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:36:59,447] jepsen worker 22 - jepsen.util 15502 :info :write [7 11] :RequestCanceledException INFO [2019-12-17 12:36:59,447] jepsen worker 59 - jepsen.util 14729 :invoke :cas [19 [10 11]] INFO [2019-12-17 12:36:59,449] jepsen worker 22 - jepsen.util 15592 :invoke :read [7 nil] INFO [2019-12-17 12:36:59,450] jepsen worker 73 - jepsen.util 10603 :ok :read [24 14] INFO [2019-12-17 12:36:59,450] jepsen worker 73 - jepsen.util 10603 :invoke :read [24 nil] INFO [2019-12-17 12:36:59,462] jepsen worker 77 - jepsen.util 16187 :info :write [25 31] :RequestCanceledException INFO [2019-12-17 12:36:59,462] jepsen worker 42 - jepsen.util 16332 :info :write [14 11] :RequestCanceledException INFO [2019-12-17 12:36:59,464] jepsen worker 42 - jepsen.util 16422 :invoke :read [14 nil] INFO [2019-12-17 12:36:59,464] jepsen worker 77 - jepsen.util 16277 :invoke :cas [25 [16 43]] INFO [2019-12-17 12:36:59,475] jepsen worker 78 - jepsen.util 11328 :ok :read [26 6] INFO [2019-12-17 12:36:59,475] jepsen worker 78 - jepsen.util 11328 :invoke :write [26 16] INFO [2019-12-17 12:36:59,481] jepsen worker 73 - jepsen.util 10603 :ok :read [24 14] INFO [2019-12-17 12:36:59,481] jepsen worker 73 - jepsen.util 10603 :invoke :cas [24 [47 41]] INFO [2019-12-17 12:36:59,496] jepsen worker 80 - jepsen.util 10970 :ok :write [26 26] INFO [2019-12-17 12:36:59,496] jepsen worker 80 - jepsen.util 10970 :invoke :cas [26 [12 35]] INFO [2019-12-17 12:36:59,496] jepsen worker 24 - jepsen.util 14064 :info :write [8 31] :RequestCanceledException INFO [2019-12-17 12:36:59,497] jepsen worker 24 - jepsen.util 14154 :invoke :cas [8 [38 3]] INFO [2019-12-17 12:36:59,501] jepsen worker 13 - jepsen.util 10633 :ok :read [4 11] INFO [2019-12-17 12:36:59,501] jepsen worker 13 - jepsen.util 10633 :invoke :read [4 nil] INFO [2019-12-17 12:36:59,507] jepsen worker 73 - jepsen.util 10603 :fail :cas [24 [47 41]] :ValueNotSwapFrom INFO [2019-12-17 12:36:59,508] jepsen worker 73 - jepsen.util 10603 :invoke :read [24 nil] INFO [2019-12-17 12:36:59,508] jepsen worker 27 - jepsen.util 10557 :ok :write [9 32] INFO [2019-12-17 12:36:59,508] jepsen worker 27 - jepsen.util 10557 :invoke :read [9 nil] INFO [2019-12-17 12:36:59,513] jepsen worker 51 - jepsen.util 10671 :ok :write [17 46] INFO [2019-12-17 12:36:59,514] jepsen worker 51 - jepsen.util 10671 :invoke :write [17 16] INFO [2019-12-17 12:36:59,532] jepsen worker 49 - jepsen.util 15529 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:36:59,532] jepsen worker 49 - jepsen.util 15529 :invoke :cas [16 [42 8]] INFO [2019-12-17 12:36:59,543] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65647,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:36:59,544] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65647,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:36:59,544] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65651,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:36:59,544] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65651,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:36:59,546] jepsen worker 86 - jepsen.util 16106 :info :cas [28 [9 7]] :RequestCanceledException INFO [2019-12-17 12:36:59,547] jepsen worker 86 - jepsen.util 16196 :invoke :read [28 nil] INFO [2019-12-17 12:36:59,555] jepsen worker 27 - jepsen.util 10557 :ok :read [9 32] INFO [2019-12-17 12:36:59,555] jepsen worker 27 - jepsen.util 10557 :invoke :write [9 2] INFO [2019-12-17 12:36:59,555] jepsen worker 73 - jepsen.util 10603 :ok :read [24 14] INFO [2019-12-17 12:36:59,555] jepsen worker 73 - jepsen.util 10603 :invoke :cas [24 [19 38]] INFO [2019-12-17 12:36:59,576] jepsen worker 73 - jepsen.util 10603 :fail :cas [24 [19 38]] :ValueNotSwapFrom INFO [2019-12-17 12:36:59,576] jepsen worker 73 - jepsen.util 10603 :invoke :cas [24 [45 7]] INFO [2019-12-17 12:36:59,583] jepsen worker 32 - jepsen.util 11642 :ok :write [10 48] INFO [2019-12-17 12:36:59,583] jepsen worker 32 - jepsen.util 11642 :invoke :cas [10 [38 13]] INFO [2019-12-17 12:36:59,594] jepsen worker 19 - jepsen.util 11629 :ok :write [6 40] INFO [2019-12-17 12:36:59,594] jepsen worker 19 - jepsen.util 11629 :invoke :write [6 15] INFO [2019-12-17 12:36:59,617] jepsen worker 60 - jepsen.util 16530 :info :cas [20 [30 37]] :RequestCanceledException INFO [2019-12-17 12:36:59,618] jepsen worker 60 - jepsen.util 16620 :invoke :cas [20 [13 5]] INFO [2019-12-17 12:36:59,622] jepsen worker 73 - jepsen.util 10603 :fail :cas [24 [45 7]] :ValueNotSwapFrom INFO [2019-12-17 12:36:59,622] jepsen worker 73 - jepsen.util 10603 :invoke :read [24 nil] INFO [2019-12-17 12:36:59,627] jepsen worker 37 - jepsen.util 10837 :fail :cas [12 [32 30]] :ValueNotSwapFrom INFO [2019-12-17 12:36:59,627] jepsen worker 37 - jepsen.util 10837 :invoke :read [12 nil] INFO [2019-12-17 12:36:59,633] jepsen worker 39 - jepsen.util 15069 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:36:59,634] jepsen worker 39 - jepsen.util 15069 :invoke :cas [13 [21 40]] INFO [2019-12-17 12:36:59,637] jepsen worker 88 - jepsen.util 11518 :fail :cas [29 [15 27]] :ValueNotSwapFrom INFO [2019-12-17 12:36:59,638] jepsen worker 88 - jepsen.util 11518 :invoke :write [29 22] INFO [2019-12-17 12:36:59,644] jepsen worker 27 - jepsen.util 10557 :ok :write [9 2] INFO [2019-12-17 12:36:59,644] jepsen worker 27 - jepsen.util 10557 :invoke :read [9 nil] INFO [2019-12-17 12:36:59,648] jepsen worker 73 - jepsen.util 10603 :ok :read [24 14] INFO [2019-12-17 12:36:59,649] jepsen worker 73 - jepsen.util 10603 :invoke :cas [24 [46 4]] INFO [2019-12-17 12:36:59,659] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65690,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:36:59,659] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65690,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:36:59,663] jepsen worker 27 - jepsen.util 10557 :ok :read [9 2] INFO [2019-12-17 12:36:59,663] jepsen worker 27 - jepsen.util 10557 :invoke :read [9 nil] INFO [2019-12-17 12:36:59,664] jepsen worker 80 - jepsen.util 10970 :fail :cas [26 [12 35]] :ValueNotSwapFrom INFO [2019-12-17 12:36:59,664] jepsen worker 80 - jepsen.util 10970 :invoke :write [26 38] INFO [2019-12-17 12:36:59,667] jepsen worker 38 - jepsen.util 11738 :ok :read [12 34] INFO [2019-12-17 12:36:59,667] jepsen worker 38 - jepsen.util 11738 :invoke :cas [12 [40 8]] INFO [2019-12-17 12:36:59,677] jepsen worker 73 - jepsen.util 10603 :fail :cas [24 [46 4]] :ValueNotSwapFrom INFO [2019-12-17 12:36:59,677] jepsen worker 73 - jepsen.util 10603 :invoke :write [24 36] INFO [2019-12-17 12:36:59,688] jepsen worker 27 - jepsen.util 10557 :ok :read [9 2] INFO [2019-12-17 12:36:59,688] jepsen worker 27 - jepsen.util 10557 :invoke :cas [9 [15 29]] INFO [2019-12-17 12:36:59,698] jepsen worker 55 - jepsen.util 11845 :info :write [18 36] :RequestCanceledException INFO [2019-12-17 12:36:59,699] jepsen worker 55 - jepsen.util 11935 :invoke :write [18 46] INFO [2019-12-17 12:36:59,702] jepsen worker 37 - jepsen.util 10837 :ok :read [12 34] INFO [2019-12-17 12:36:59,702] jepsen worker 37 - jepsen.util 10837 :invoke :read [12 nil] INFO [2019-12-17 12:36:59,713] jepsen worker 36 - jepsen.util 12186 :ok :read [12 34] INFO [2019-12-17 12:36:59,713] jepsen worker 36 - jepsen.util 12186 :invoke :read [12 nil] INFO [2019-12-17 12:36:59,720] jepsen worker 9 - jepsen.util 11889 :ok :write [3 18] INFO [2019-12-17 12:36:59,720] jepsen worker 9 - jepsen.util 11889 :invoke :write [3 6] INFO [2019-12-17 12:36:59,722] jepsen worker 27 - jepsen.util 10557 :fail :cas [9 [15 29]] :ValueNotSwapFrom INFO [2019-12-17 12:36:59,723] jepsen worker 27 - jepsen.util 10557 :invoke :write [9 49] INFO [2019-12-17 12:36:59,733] jepsen worker 4 - jepsen.util 10354 :info :write [1 10] :RequestCanceledException INFO [2019-12-17 12:36:59,734] jepsen worker 4 - jepsen.util 10444 :invoke :read [1 nil] INFO [2019-12-17 12:36:59,750] jepsen worker 32 - jepsen.util 11642 :fail :cas [10 [38 13]] :ValueNotSwapFrom INFO [2019-12-17 12:36:59,750] jepsen worker 32 - jepsen.util 11642 :invoke :write [10 41] INFO [2019-12-17 12:36:59,781] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:36:59,781] jepsen nemesis - couchbase.util Rebalance check count: 516 INFO [2019-12-17 12:36:59,781] jepsen nemesis - couchbase.util Rebalance stuck count: 497 INFO [2019-12-17 12:36:59,800] jepsen worker 27 - jepsen.util 10557 :ok :write [9 49] INFO [2019-12-17 12:36:59,800] jepsen worker 27 - jepsen.util 10557 :invoke :read [9 nil] INFO [2019-12-17 12:36:59,814] jepsen worker 36 - jepsen.util 12186 :ok :read [12 34] INFO [2019-12-17 12:36:59,815] jepsen worker 36 - jepsen.util 12186 :invoke :cas [12 [9 41]] INFO [2019-12-17 12:36:59,826] jepsen worker 27 - jepsen.util 10557 :ok :read [9 49] INFO [2019-12-17 12:36:59,827] jepsen worker 27 - jepsen.util 10557 :invoke :cas [9 [36 20]] INFO [2019-12-17 12:36:59,846] jepsen worker 25 - jepsen.util 16405 :info :write [8 33] :RequestCanceledException INFO [2019-12-17 12:36:59,847] jepsen worker 25 - jepsen.util 16495 :invoke :cas [8 [46 16]] INFO [2019-12-17 12:36:59,852] jepsen worker 27 - jepsen.util 10557 :fail :cas [9 [36 20]] :ValueNotSwapFrom INFO [2019-12-17 12:36:59,852] jepsen worker 27 - jepsen.util 10557 :invoke :read [9 nil] INFO [2019-12-17 12:36:59,872] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65712,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:36:59,873] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65712,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:36:59,873] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65715,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:36:59,873] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65715,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:36:59,873] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65716,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:36:59,873] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65716,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:36:59,875] jepsen worker 27 - jepsen.util 10557 :ok :read [9 49] INFO [2019-12-17 12:36:59,875] jepsen worker 27 - jepsen.util 10557 :invoke :cas [9 [15 12]] INFO [2019-12-17 12:36:59,883] jepsen worker 13 - jepsen.util 10633 :ok :read [4 11] INFO [2019-12-17 12:36:59,883] jepsen worker 13 - jepsen.util 10633 :invoke :cas [4 [8 48]] INFO [2019-12-17 12:36:59,889] jepsen worker 27 - jepsen.util 10557 :fail :cas [9 [15 12]] :ValueNotSwapFrom INFO [2019-12-17 12:36:59,889] jepsen worker 27 - jepsen.util 10557 :invoke :write [9 30] INFO [2019-12-17 12:36:59,895] jepsen worker 33 - jepsen.util 15423 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:36:59,895] jepsen worker 33 - jepsen.util 15423 :invoke :cas [11 [16 8]] INFO [2019-12-17 12:36:59,931] jepsen worker 37 - jepsen.util 10837 :ok :read [12 34] INFO [2019-12-17 12:36:59,931] jepsen worker 37 - jepsen.util 10837 :invoke :cas [12 [1 40]] INFO [2019-12-17 12:36:59,931] jepsen worker 0 - jepsen.util 16200 :info :write [0 30] :RequestCanceledException INFO [2019-12-17 12:36:59,932] jepsen worker 0 - jepsen.util 16290 :invoke :write [0 23] INFO [2019-12-17 12:36:59,937] jepsen worker 27 - jepsen.util 10557 :ok :write [9 30] INFO [2019-12-17 12:36:59,937] jepsen worker 27 - jepsen.util 10557 :invoke :write [9 48] INFO [2019-12-17 12:36:59,946] jepsen worker 85 - jepsen.util 16285 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:36:59,946] jepsen worker 85 - jepsen.util 16285 :invoke :write [28 37] INFO [2019-12-17 12:36:59,956] jepsen worker 38 - jepsen.util 11738 :fail :cas [12 [40 8]] :ValueNotSwapFrom INFO [2019-12-17 12:36:59,956] jepsen worker 38 - jepsen.util 11738 :invoke :write [12 13] INFO [2019-12-17 12:36:59,980] jepsen worker 12 - jepsen.util 11622 :ok :read [4 11] INFO [2019-12-17 12:36:59,980] jepsen worker 12 - jepsen.util 11622 :invoke :cas [4 [19 28]] INFO [2019-12-17 12:36:59,983] jepsen worker 27 - jepsen.util 10557 :ok :write [9 48] INFO [2019-12-17 12:36:59,983] jepsen worker 27 - jepsen.util 10557 :invoke :write [9 41] INFO [2019-12-17 12:37:00,001] jepsen worker 70 - jepsen.util 11950 :ok :read [23 13] INFO [2019-12-17 12:37:00,001] jepsen worker 4 - jepsen.util 10444 :ok :read [1 24] INFO [2019-12-17 12:37:00,001] jepsen worker 4 - jepsen.util 10444 :invoke :write [1 8] INFO [2019-12-17 12:37:00,001] jepsen worker 70 - jepsen.util 11950 :invoke :write [23 6] INFO [2019-12-17 12:37:00,018] jepsen worker 46 - jepsen.util 11026 :info :write [15 29] :RequestCanceledException INFO [2019-12-17 12:37:00,018] jepsen worker 74 - jepsen.util 11054 :info :cas [24 [36 6]] :RequestCanceledException INFO [2019-12-17 12:37:00,019] jepsen worker 46 - jepsen.util 11116 :invoke :read [15 nil] INFO [2019-12-17 12:37:00,019] jepsen worker 74 - jepsen.util 11144 :invoke :cas [24 [4 14]] INFO [2019-12-17 12:37:00,031] jepsen worker 79 - jepsen.util 11419 :info :write [26 47] :RequestCanceledException INFO [2019-12-17 12:37:00,033] jepsen worker 79 - jepsen.util 11509 :invoke :read [26 nil] INFO [2019-12-17 12:37:00,039] jepsen worker 71 - jepsen.util 12311 :ok :read [23 13] INFO [2019-12-17 12:37:00,039] jepsen worker 71 - jepsen.util 12311 :invoke :cas [23 [29 5]] INFO [2019-12-17 12:37:00,043] jepsen worker 74 - jepsen.util 11144 :fail :cas [24 [4 14]] :ValueNotSwapFrom INFO [2019-12-17 12:37:00,044] jepsen worker 74 - jepsen.util 11144 :invoke :cas [24 [20 27]] INFO [2019-12-17 12:37:00,047] jepsen worker 27 - jepsen.util 10557 :ok :write [9 41] INFO [2019-12-17 12:37:00,047] jepsen worker 27 - jepsen.util 10557 :invoke :cas [9 [20 31]] INFO [2019-12-17 12:37:00,059] jepsen worker 74 - jepsen.util 11144 :fail :cas [24 [20 27]] :ValueNotSwapFrom INFO [2019-12-17 12:37:00,060] jepsen worker 74 - jepsen.util 11144 :invoke :read [24 nil] INFO [2019-12-17 12:37:00,064] jepsen worker 63 - jepsen.util 10053 :info :write [21 47] :RequestCanceledException INFO [2019-12-17 12:37:00,065] jepsen worker 63 - jepsen.util 10143 :invoke :cas [21 [43 44]] INFO [2019-12-17 12:37:00,077] jepsen worker 46 - jepsen.util 11116 :ok :read [15 5] INFO [2019-12-17 12:37:00,078] jepsen worker 46 - jepsen.util 11116 :invoke :cas [15 [42 6]] INFO [2019-12-17 12:37:00,092] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65729,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:37:00,092] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65729,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:37:00,092] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65732,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:37:00,092] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65732,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:37:00,097] jepsen worker 27 - jepsen.util 10557 :fail :cas [9 [20 31]] :ValueNotSwapFrom INFO [2019-12-17 12:37:00,098] jepsen worker 27 - jepsen.util 10557 :invoke :read [9 nil] INFO [2019-12-17 12:37:00,099] jepsen worker 31 - jepsen.util 9661 :info :cas [10 [39 40]] :RequestCanceledException INFO [2019-12-17 12:37:00,100] jepsen worker 31 - jepsen.util 9751 :invoke :cas [10 [22 43]] INFO [2019-12-17 12:37:00,102] jepsen worker 74 - jepsen.util 11144 :ok :read [24 14] INFO [2019-12-17 12:37:00,102] jepsen worker 74 - jepsen.util 11144 :invoke :cas [24 [24 36]] INFO [2019-12-17 12:37:00,111] jepsen worker 46 - jepsen.util 11116 :fail :cas [15 [42 6]] :ValueNotSwapFrom INFO [2019-12-17 12:37:00,111] jepsen worker 46 - jepsen.util 11116 :invoke :cas [15 [12 21]] INFO [2019-12-17 12:37:00,118] jepsen worker 87 - jepsen.util 10887 :info :write [29 5] :RequestCanceledException INFO [2019-12-17 12:37:00,118] jepsen worker 65 - jepsen.util 9875 :info :cas [21 [37 37]] :RequestCanceledException INFO [2019-12-17 12:37:00,118] jepsen worker 3 - jepsen.util 10713 :info :write [1 6] :RequestCanceledException INFO [2019-12-17 12:37:00,119] jepsen worker 65 - jepsen.util 9965 :invoke :cas [21 [38 7]] INFO [2019-12-17 12:37:00,119] jepsen worker 3 - jepsen.util 10803 :invoke :cas [1 [3 45]] INFO [2019-12-17 12:37:00,119] jepsen worker 87 - jepsen.util 10977 :invoke :read [29 nil] INFO [2019-12-17 12:37:00,129] jepsen worker 52 - jepsen.util 10672 :ok :read [17 46] INFO [2019-12-17 12:37:00,129] jepsen worker 52 - jepsen.util 10672 :invoke :cas [17 [33 28]] INFO [2019-12-17 12:37:00,132] jepsen worker 27 - jepsen.util 10557 :ok :read [9 41] INFO [2019-12-17 12:37:00,132] jepsen worker 27 - jepsen.util 10557 :invoke :write [9 23] INFO [2019-12-17 12:37:00,140] jepsen worker 74 - jepsen.util 11144 :fail :cas [24 [24 36]] :ValueNotSwapFrom INFO [2019-12-17 12:37:00,140] jepsen worker 74 - jepsen.util 11144 :invoke :cas [24 [38 27]] INFO [2019-12-17 12:37:00,147] jepsen worker 28 - jepsen.util 9388 :info :write [9 42] :RequestCanceledException INFO [2019-12-17 12:37:00,147] jepsen worker 48 - jepsen.util 14358 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:37:00,147] jepsen worker 48 - jepsen.util 14358 :invoke :read [16 nil] INFO [2019-12-17 12:37:00,148] jepsen worker 28 - jepsen.util 9478 :invoke :write [9 1] INFO [2019-12-17 12:37:00,159] jepsen worker 31 - jepsen.util 9751 :fail :cas [10 [22 43]] :ValueNotSwapFrom INFO [2019-12-17 12:37:00,159] jepsen worker 31 - jepsen.util 9751 :invoke :read [10 nil] INFO [2019-12-17 12:37:00,164] jepsen worker 67 - jepsen.util 15547 :info :cas [22 [8 44]] :RequestCanceledException INFO [2019-12-17 12:37:00,165] jepsen worker 67 - jepsen.util 15637 :invoke :cas [22 [16 26]] INFO [2019-12-17 12:37:00,166] jepsen worker 12 - jepsen.util 11622 :fail :cas [4 [19 28]] :ValueNotSwapFrom INFO [2019-12-17 12:37:00,167] jepsen worker 12 - jepsen.util 11622 :invoke :write [4 42] INFO [2019-12-17 12:37:00,169] jepsen worker 46 - jepsen.util 11116 :fail :cas [15 [12 21]] :ValueNotSwapFrom INFO [2019-12-17 12:37:00,169] jepsen worker 46 - jepsen.util 11116 :invoke :write [15 34] INFO [2019-12-17 12:37:00,194] jepsen worker 74 - jepsen.util 11144 :fail :cas [24 [38 27]] :ValueNotSwapFrom INFO [2019-12-17 12:37:00,194] jepsen worker 74 - jepsen.util 11144 :invoke :read [24 nil] INFO [2019-12-17 12:37:00,201] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65762,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:37:00,202] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65762,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:37:00,202] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65763,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:37:00,202] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65763,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:37:00,202] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65764,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:37:00,202] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65764,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:37:00,208] jepsen worker 31 - jepsen.util 9751 :ok :read [10 48] INFO [2019-12-17 12:37:00,208] jepsen worker 31 - jepsen.util 9751 :invoke :read [10 nil] INFO [2019-12-17 12:37:00,210] jepsen worker 74 - jepsen.util 11144 :ok :read [24 14] INFO [2019-12-17 12:37:00,210] jepsen worker 74 - jepsen.util 11144 :invoke :read [24 nil] INFO [2019-12-17 12:37:00,231] jepsen worker 36 - jepsen.util 12186 :fail :cas [12 [9 41]] :ValueNotSwapFrom INFO [2019-12-17 12:37:00,231] jepsen worker 36 - jepsen.util 12186 :invoke :cas [12 [44 40]] INFO [2019-12-17 12:37:00,232] jepsen worker 72 - jepsen.util 11592 :info :cas [24 [0 20]] :RequestCanceledException INFO [2019-12-17 12:37:00,233] jepsen worker 9 - jepsen.util 11889 :ok :write [3 6] INFO [2019-12-17 12:37:00,233] jepsen worker 19 - jepsen.util 11629 :ok :write [6 15] INFO [2019-12-17 12:37:00,233] jepsen worker 9 - jepsen.util 11889 :invoke :read [3 nil] INFO [2019-12-17 12:37:00,233] jepsen worker 19 - jepsen.util 11629 :invoke :write [6 37] INFO [2019-12-17 12:37:00,234] jepsen worker 72 - jepsen.util 11682 :invoke :read [24 nil] INFO [2019-12-17 12:37:00,236] jepsen worker 52 - jepsen.util 10672 :fail :cas [17 [33 28]] :ValueNotSwapFrom INFO [2019-12-17 12:37:00,236] jepsen worker 52 - jepsen.util 10672 :invoke :write [17 25] INFO [2019-12-17 12:37:00,245] jepsen worker 73 - jepsen.util 10603 :ok :write [24 36] INFO [2019-12-17 12:37:00,245] jepsen worker 73 - jepsen.util 10603 :invoke :write [24 37] INFO [2019-12-17 12:37:00,246] jepsen worker 8 - jepsen.util 15308 :info :cas [2 [27 31]] :RequestCanceledException INFO [2019-12-17 12:37:00,247] jepsen worker 8 - jepsen.util 15398 :invoke :read [2 nil] INFO [2019-12-17 12:37:00,262] jepsen worker 7 - jepsen.util 15487 :info :write [2 45] :RequestCanceledException INFO [2019-12-17 12:37:00,263] jepsen worker 31 - jepsen.util 9751 :ok :read [10 48] INFO [2019-12-17 12:37:00,263] jepsen worker 31 - jepsen.util 9751 :invoke :read [10 nil] INFO [2019-12-17 12:37:00,264] jepsen worker 7 - jepsen.util 15577 :invoke :cas [2 [18 1]] INFO [2019-12-17 12:37:00,267] jepsen worker 3 - jepsen.util 10803 :fail :cas [1 [3 45]] :ValueNotSwapFrom INFO [2019-12-17 12:37:00,267] jepsen worker 3 - jepsen.util 10803 :invoke :write [1 35] INFO [2019-12-17 12:37:00,285] jepsen worker 74 - jepsen.util 11144 :ok :read [24 14] INFO [2019-12-17 12:37:00,285] jepsen worker 74 - jepsen.util 11144 :invoke :write [24 8] INFO [2019-12-17 12:37:00,296] jepsen worker 72 - jepsen.util 11682 :ok :read [24 36] INFO [2019-12-17 12:37:00,297] jepsen worker 72 - jepsen.util 11682 :invoke :cas [24 [26 42]] INFO [2019-12-17 12:37:00,301] jepsen worker 37 - jepsen.util 10837 :fail :cas [12 [1 40]] :ValueNotSwapFrom INFO [2019-12-17 12:37:00,301] jepsen worker 37 - jepsen.util 10837 :invoke :cas [12 [16 41]] INFO [2019-12-17 12:37:00,305] jepsen worker 27 - jepsen.util 10557 :ok :write [9 23] INFO [2019-12-17 12:37:00,305] jepsen worker 27 - jepsen.util 10557 :invoke :write [9 45] INFO [2019-12-17 12:37:00,311] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65771,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:37:00,311] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65771,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:37:00,311] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65774,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:37:00,311] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65774,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:37:00,311] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65775,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:37:00,311] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65775,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:37:00,331] jepsen worker 65 - jepsen.util 9965 :fail :cas [21 [38 7]] :ValueNotSwapFrom INFO [2019-12-17 12:37:00,331] jepsen worker 65 - jepsen.util 9965 :invoke :read [21 nil] INFO [2019-12-17 12:37:00,333] jepsen worker 64 - jepsen.util 8704 :fail :read [21 nil] :RequestTimeoutException INFO [2019-12-17 12:37:00,333] jepsen worker 61 - jepsen.util 16621 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:37:00,333] jepsen worker 30 - jepsen.util 10560 :fail :read [10 nil] :RequestTimeoutException INFO [2019-12-17 12:37:00,333] jepsen worker 29 - jepsen.util 9479 :info :write [9 15] :RequestCanceledException INFO [2019-12-17 12:37:00,333] jepsen worker 64 - jepsen.util 8704 :invoke :cas [21 [39 19]] INFO [2019-12-17 12:37:00,333] jepsen worker 61 - jepsen.util 16621 :invoke :cas [20 [14 16]] INFO [2019-12-17 12:37:00,333] jepsen worker 30 - jepsen.util 10560 :invoke :write [10 15] INFO [2019-12-17 12:37:00,334] jepsen worker 29 - jepsen.util 9569 :invoke :read [9 nil] INFO [2019-12-17 12:37:00,361] jepsen worker 2 - jepsen.util 14942 :info :cas [0 [1 20]] :RequestCanceledException INFO [2019-12-17 12:37:00,362] jepsen worker 2 - jepsen.util 15032 :invoke :write [0 23] INFO [2019-12-17 12:37:00,373] jepsen worker 87 - jepsen.util 10977 :ok :read [29 7] INFO [2019-12-17 12:37:00,373] jepsen worker 87 - jepsen.util 10977 :invoke :write [29 24] INFO [2019-12-17 12:37:00,395] jepsen worker 31 - jepsen.util 9751 :ok :read [10 48] INFO [2019-12-17 12:37:00,395] jepsen worker 31 - jepsen.util 9751 :invoke :read [10 nil] INFO [2019-12-17 12:37:00,398] jepsen worker 15 - jepsen.util 14415 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:37:00,398] jepsen worker 43 - jepsen.util 15703 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:37:00,398] jepsen worker 15 - jepsen.util 14415 :invoke :cas [5 [29 12]] INFO [2019-12-17 12:37:00,399] jepsen worker 43 - jepsen.util 15703 :invoke :write [14 20] INFO [2019-12-17 12:37:00,408] jepsen worker 72 - jepsen.util 11682 :fail :cas [24 [26 42]] :ValueNotSwapFrom INFO [2019-12-17 12:37:00,408] jepsen worker 72 - jepsen.util 11682 :invoke :write [24 19] INFO [2019-12-17 12:37:00,417] jepsen worker 17 - jepsen.util 15497 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:37:00,417] jepsen worker 58 - jepsen.util 15718 :info :write [19 18] :RequestCanceledException INFO [2019-12-17 12:37:00,417] jepsen worker 18 - jepsen.util 11358 :info :write [6 41] :RequestCanceledException INFO [2019-12-17 12:37:00,417] jepsen worker 17 - jepsen.util 15497 :invoke :cas [5 [23 44]] INFO [2019-12-17 12:37:00,418] jepsen worker 18 - jepsen.util 11448 :invoke :write [6 17] INFO [2019-12-17 12:37:00,419] jepsen worker 58 - jepsen.util 15808 :invoke :write [19 11] INFO [2019-12-17 12:37:00,427] jepsen worker 28 - jepsen.util 9478 :ok :write [9 1] INFO [2019-12-17 12:37:00,427] jepsen worker 28 - jepsen.util 9478 :invoke :read [9 nil] INFO [2019-12-17 12:37:00,433] jepsen worker 50 - jepsen.util 15980 :info :write [16 49] :RequestCanceledException INFO [2019-12-17 12:37:00,433] jepsen worker 84 - jepsen.util 15204 :info :cas [28 [4 30]] :RequestCanceledException INFO [2019-12-17 12:37:00,434] jepsen worker 84 - jepsen.util 15294 :invoke :read [28 nil] INFO [2019-12-17 12:37:00,435] jepsen worker 50 - jepsen.util 16070 :invoke :write [16 0] INFO [2019-12-17 12:37:00,436] jepsen worker 31 - jepsen.util 9751 :ok :read [10 48] INFO [2019-12-17 12:37:00,436] jepsen worker 31 - jepsen.util 9751 :invoke :read [10 nil] INFO [2019-12-17 12:37:00,463] jepsen worker 64 - jepsen.util 8704 :fail :cas [21 [39 19]] :ValueNotSwapFrom INFO [2019-12-17 12:37:00,463] jepsen worker 64 - jepsen.util 8704 :invoke :write [21 30] INFO [2019-12-17 12:37:00,464] jepsen worker 23 - jepsen.util 15503 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:37:00,464] jepsen worker 23 - jepsen.util 15503 :invoke :cas [7 [42 49]] INFO [2019-12-17 12:37:00,500] jepsen worker 21 - jepsen.util 14961 :info :write [7 17] :RequestCanceledException INFO [2019-12-17 12:37:00,500] jepsen worker 28 - jepsen.util 9478 :ok :read [9 1] INFO [2019-12-17 12:37:00,501] jepsen worker 21 - jepsen.util 15051 :invoke :cas [7 [5 16]] INFO [2019-12-17 12:37:00,501] jepsen worker 28 - jepsen.util 9478 :invoke :read [9 nil] INFO [2019-12-17 12:37:00,517] jepsen worker 56 - jepsen.util 12116 :ok :read [18 46] INFO [2019-12-17 12:37:00,517] jepsen worker 56 - jepsen.util 12116 :invoke :read [18 nil] INFO [2019-12-17 12:37:00,517] jepsen worker 62 - jepsen.util 16082 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:37:00,517] jepsen worker 62 - jepsen.util 16082 :invoke :cas [20 [49 15]] INFO [2019-12-17 12:37:00,528] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65806,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:37:00,529] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65806,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:37:00,529] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65810,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:37:00,529] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65810,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:37:00,529] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65811,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:37:00,529] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65811,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:37:00,532] jepsen worker 47 - jepsen.util 12107 :fail :read [15 nil] :RequestTimeoutException INFO [2019-12-17 12:37:00,532] jepsen worker 47 - jepsen.util 12107 :invoke :read [15 nil] INFO [2019-12-17 12:37:00,547] jepsen worker 10 - jepsen.util 11080 :fail :read [3 nil] :RequestTimeoutException INFO [2019-12-17 12:37:00,547] jepsen worker 10 - jepsen.util 11080 :invoke :cas [3 [47 24]] INFO [2019-12-17 12:37:00,565] jepsen worker 71 - jepsen.util 12311 :fail :cas [23 [29 5]] :ValueNotSwapFrom INFO [2019-12-17 12:37:00,565] jepsen worker 71 - jepsen.util 12311 :invoke :read [23 nil] INFO [2019-12-17 12:37:00,568] jepsen worker 64 - jepsen.util 8704 :ok :write [21 30] INFO [2019-12-17 12:37:00,568] jepsen worker 36 - jepsen.util 12186 :fail :cas [12 [44 40]] :ValueNotSwapFrom INFO [2019-12-17 12:37:00,569] jepsen worker 64 - jepsen.util 8704 :invoke :read [21 nil] INFO [2019-12-17 12:37:00,569] jepsen worker 36 - jepsen.util 12186 :invoke :cas [12 [33 11]] INFO [2019-12-17 12:37:00,581] jepsen worker 56 - jepsen.util 12116 :ok :read [18 46] INFO [2019-12-17 12:37:00,581] jepsen worker 56 - jepsen.util 12116 :invoke :cas [18 [29 1]] INFO [2019-12-17 12:37:00,586] jepsen worker 47 - jepsen.util 12107 :ok :read [15 5] INFO [2019-12-17 12:37:00,587] jepsen worker 47 - jepsen.util 12107 :invoke :read [15 nil] INFO [2019-12-17 12:37:00,610] jepsen worker 79 - jepsen.util 11509 :ok :read [26 26] INFO [2019-12-17 12:37:00,610] jepsen worker 79 - jepsen.util 11509 :invoke :write [26 4] INFO [2019-12-17 12:37:00,614] jepsen worker 64 - jepsen.util 8704 :ok :read [21 30] INFO [2019-12-17 12:37:00,614] jepsen worker 64 - jepsen.util 8704 :invoke :cas [21 [45 31]] INFO [2019-12-17 12:37:00,624] jepsen worker 47 - jepsen.util 12107 :ok :read [15 5] INFO [2019-12-17 12:37:00,625] jepsen worker 47 - jepsen.util 12107 :invoke :cas [15 [1 1]] INFO [2019-12-17 12:37:00,634] jepsen worker 64 - jepsen.util 8704 :fail :cas [21 [45 31]] :ValueNotSwapFrom INFO [2019-12-17 12:37:00,634] jepsen worker 64 - jepsen.util 8704 :invoke :write [21 19] INFO [2019-12-17 12:37:00,641] jepsen worker 47 - jepsen.util 12107 :fail :cas [15 [1 1]] :ValueNotSwapFrom INFO [2019-12-17 12:37:00,642] jepsen worker 47 - jepsen.util 12107 :invoke :cas [15 [0 32]] INFO [2019-12-17 12:37:00,664] jepsen worker 13 - jepsen.util 10633 :fail :cas [4 [8 48]] :ValueNotSwapFrom INFO [2019-12-17 12:37:00,664] jepsen worker 20 - jepsen.util 10910 :info :write [6 32] :RequestCanceledException INFO [2019-12-17 12:37:00,664] jepsen worker 13 - jepsen.util 10633 :invoke :write [4 30] INFO [2019-12-17 12:37:00,665] jepsen worker 20 - jepsen.util 11000 :invoke :write [6 15] INFO [2019-12-17 12:37:00,674] jepsen worker 56 - jepsen.util 12116 :fail :cas [18 [29 1]] :ValueNotSwapFrom INFO [2019-12-17 12:37:00,674] jepsen worker 56 - jepsen.util 12116 :invoke :read [18 nil] INFO [2019-12-17 12:37:00,692] jepsen worker 64 - jepsen.util 8704 :ok :write [21 19] INFO [2019-12-17 12:37:00,692] jepsen worker 64 - jepsen.util 8704 :invoke :cas [21 [7 41]] INFO [2019-12-17 12:37:00,709] jepsen worker 32 - jepsen.util 11642 :ok :write [10 41] INFO [2019-12-17 12:37:00,709] jepsen worker 32 - jepsen.util 11642 :invoke :write [10 17] INFO [2019-12-17 12:37:00,709] jepsen worker 19 - jepsen.util 11629 :ok :write [6 37] INFO [2019-12-17 12:37:00,709] jepsen worker 19 - jepsen.util 11629 :invoke :write [6 37] INFO [2019-12-17 12:37:00,710] jepsen worker 74 - jepsen.util 11144 :ok :write [24 8] INFO [2019-12-17 12:37:00,710] jepsen worker 74 - jepsen.util 11144 :invoke :cas [24 [2 44]] INFO [2019-12-17 12:37:00,752] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65833,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:37:00,752] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65833,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:37:00,752] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65836,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:37:00,752] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65836,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:37:00,752] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65837,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:37:00,752] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65837,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:37:00,777] jepsen worker 46 - jepsen.util 11116 :ok :write [15 34] INFO [2019-12-17 12:37:00,777] jepsen worker 46 - jepsen.util 11116 :invoke :read [15 nil] INFO [2019-12-17 12:37:00,803] jepsen worker 65 - jepsen.util 9965 :ok :read [21 19] INFO [2019-12-17 12:37:00,803] jepsen worker 65 - jepsen.util 9965 :invoke :read [21 nil] INFO [2019-12-17 12:37:00,803] jepsen worker 63 - jepsen.util 10143 :fail :cas [21 [43 44]] :ValueNotSwapFrom INFO [2019-12-17 12:37:00,804] jepsen worker 63 - jepsen.util 10143 :invoke :read [21 nil] INFO [2019-12-17 12:37:00,805] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:37:00,805] jepsen nemesis - couchbase.util Rebalance check count: 517 INFO [2019-12-17 12:37:00,805] jepsen nemesis - couchbase.util Rebalance stuck count: 498 INFO [2019-12-17 12:37:00,814] jepsen worker 10 - jepsen.util 11080 :fail :cas [3 [47 24]] :ValueNotSwapFrom INFO [2019-12-17 12:37:00,815] jepsen worker 10 - jepsen.util 11080 :invoke :read [3 nil] INFO [2019-12-17 12:37:00,815] jepsen worker 69 - jepsen.util 11499 :info :write [23 40] :RequestCanceledException INFO [2019-12-17 12:37:00,816] jepsen worker 69 - jepsen.util 11589 :invoke :write [23 13] INFO [2019-12-17 12:37:00,822] jepsen worker 46 - jepsen.util 11116 :ok :read [15 34] INFO [2019-12-17 12:37:00,822] jepsen worker 46 - jepsen.util 11116 :invoke :read [15 nil] INFO [2019-12-17 12:37:00,840] jepsen worker 56 - jepsen.util 12116 :ok :read [18 46] INFO [2019-12-17 12:37:00,840] jepsen worker 56 - jepsen.util 12116 :invoke :write [18 5] INFO [2019-12-17 12:37:00,841] jepsen worker 74 - jepsen.util 11144 :fail :cas [24 [2 44]] :ValueNotSwapFrom INFO [2019-12-17 12:37:00,841] jepsen worker 74 - jepsen.util 11144 :invoke :cas [24 [20 10]] INFO [2019-12-17 12:37:00,847] jepsen worker 14 - jepsen.util 11084 :info :write [4 1] :RequestCanceledException INFO [2019-12-17 12:37:00,848] jepsen worker 10 - jepsen.util 11080 :ok :read [3 6] INFO [2019-12-17 12:37:00,848] jepsen worker 14 - jepsen.util 11174 :invoke :read [4 nil] INFO [2019-12-17 12:37:00,848] jepsen worker 10 - jepsen.util 11080 :invoke :write [3 41] INFO [2019-12-17 12:37:00,881] jepsen worker 9 - jepsen.util 11889 :ok :read [3 6] INFO [2019-12-17 12:37:00,881] jepsen worker 9 - jepsen.util 11889 :invoke :cas [3 [14 4]] INFO [2019-12-17 12:37:00,896] jepsen worker 46 - jepsen.util 11116 :ok :read [15 34] INFO [2019-12-17 12:37:00,896] jepsen worker 46 - jepsen.util 11116 :invoke :read [15 nil] INFO [2019-12-17 12:37:00,935] jepsen worker 63 - jepsen.util 10143 :ok :read [21 19] INFO [2019-12-17 12:37:00,935] jepsen worker 63 - jepsen.util 10143 :invoke :write [21 44] INFO [2019-12-17 12:37:00,937] jepsen worker 9 - jepsen.util 11889 :fail :cas [3 [14 4]] :ValueNotSwapFrom INFO [2019-12-17 12:37:00,937] jepsen worker 9 - jepsen.util 11889 :invoke :cas [3 [4 15]] INFO [2019-12-17 12:37:00,954] jepsen worker 37 - jepsen.util 10837 :fail :cas [12 [16 41]] :ValueNotSwapFrom INFO [2019-12-17 12:37:00,954] jepsen worker 37 - jepsen.util 10837 :invoke :write [12 3] INFO [2019-12-17 12:37:00,956] jepsen worker 74 - jepsen.util 11144 :fail :cas [24 [20 10]] :ValueNotSwapFrom INFO [2019-12-17 12:37:00,956] jepsen worker 74 - jepsen.util 11144 :invoke :cas [24 [49 5]] INFO [2019-12-17 12:37:00,967] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65859,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:37:00,967] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65859,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:37:00,992] jepsen worker 51 - jepsen.util 10671 :ok :write [17 16] INFO [2019-12-17 12:37:00,992] jepsen worker 51 - jepsen.util 10671 :invoke :write [17 17] INFO [2019-12-17 12:37:00,995] jepsen worker 9 - jepsen.util 11889 :fail :cas [3 [4 15]] :ValueNotSwapFrom INFO [2019-12-17 12:37:00,995] jepsen worker 9 - jepsen.util 11889 :invoke :write [3 49] INFO [2019-12-17 12:37:01,015] jepsen worker 74 - jepsen.util 11144 :fail :cas [24 [49 5]] :ValueNotSwapFrom INFO [2019-12-17 12:37:01,015] jepsen worker 74 - jepsen.util 11144 :invoke :cas [24 [12 10]] INFO [2019-12-17 12:37:01,026] jepsen worker 36 - jepsen.util 12186 :fail :cas [12 [33 11]] :ValueNotSwapFrom INFO [2019-12-17 12:37:01,026] jepsen worker 36 - jepsen.util 12186 :invoke :cas [12 [20 32]] INFO [2019-12-17 12:37:01,032] jepsen worker 79 - jepsen.util 11509 :ok :write [26 4] INFO [2019-12-17 12:37:01,033] jepsen worker 79 - jepsen.util 11509 :invoke :write [26 23] INFO [2019-12-17 12:37:01,033] jepsen worker 63 - jepsen.util 10143 :ok :write [21 44] INFO [2019-12-17 12:37:01,033] jepsen worker 63 - jepsen.util 10143 :invoke :write [21 13] INFO [2019-12-17 12:37:01,034] jepsen worker 53 - jepsen.util 10673 :info :write [17 43] :RequestCanceledException INFO [2019-12-17 12:37:01,038] jepsen worker 53 - jepsen.util 10763 :invoke :write [17 4] INFO [2019-12-17 12:37:01,039] jepsen worker 74 - jepsen.util 11144 :fail :cas [24 [12 10]] :ValueNotSwapFrom INFO [2019-12-17 12:37:01,039] jepsen worker 74 - jepsen.util 11144 :invoke :write [24 49] INFO [2019-12-17 12:37:01,076] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65869,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:37:01,076] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65869,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:37:01,076] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65871,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:37:01,077] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65871,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:37:01,117] jepsen worker 12 - jepsen.util 11622 :ok :write [4 42] INFO [2019-12-17 12:37:01,117] jepsen worker 12 - jepsen.util 11622 :invoke :cas [4 [24 37]] INFO [2019-12-17 12:37:01,128] jepsen worker 63 - jepsen.util 10143 :ok :write [21 13] INFO [2019-12-17 12:37:01,128] jepsen worker 63 - jepsen.util 10143 :invoke :cas [21 [18 42]] INFO [2019-12-17 12:37:01,133] jepsen worker 45 - jepsen.util 11565 :info :write [15 27] :RequestCanceledException INFO [2019-12-17 12:37:01,133] jepsen worker 5 - jepsen.util 10985 :info :write [1 42] :RequestCanceledException INFO [2019-12-17 12:37:01,137] jepsen worker 5 - jepsen.util 11075 :invoke :cas [1 [28 4]] INFO [2019-12-17 12:37:01,139] jepsen worker 45 - jepsen.util 11655 :invoke :write [15 12] INFO [2019-12-17 12:37:01,176] jepsen worker 20 - jepsen.util 11000 :ok :write [6 15] INFO [2019-12-17 12:37:01,179] jepsen worker 30 - jepsen.util 10560 :ok :write [10 15] INFO [2019-12-17 12:37:01,182] jepsen worker 63 - jepsen.util 10143 :fail :cas [21 [18 42]] :ValueNotSwapFrom INFO [2019-12-17 12:37:01,184] jepsen worker 10 - jepsen.util 11080 :ok :write [3 41] INFO [2019-12-17 12:37:01,235] jepsen worker 54 - jepsen.util 11844 :info :write [18 5] :RequestCanceledException INFO [2019-12-17 12:37:01,236] jepsen worker 20 - jepsen.util 11000 :invoke :write [6 19] INFO [2019-12-17 12:37:01,236] jepsen worker 30 - jepsen.util 10560 :invoke :read [10 nil] INFO [2019-12-17 12:37:01,237] jepsen worker 10 - jepsen.util 11080 :invoke :cas [3 [36 29]] INFO [2019-12-17 12:37:01,237] jepsen worker 63 - jepsen.util 10143 :invoke :cas [21 [29 35]] INFO [2019-12-17 12:37:01,244] jepsen worker 54 - jepsen.util 11934 :invoke :cas [18 [37 43]] INFO [2019-12-17 12:37:01,261] jepsen worker 89 - jepsen.util 11339 :info :write [29 13] :RequestCanceledException INFO [2019-12-17 12:37:01,272] jepsen worker 89 - jepsen.util 11429 :invoke :write [29 34] INFO [2019-12-17 12:37:01,296] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65886,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:37:01,296] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65886,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:37:01,297] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65891,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:37:01,298] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65891,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:37:01,299] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65895,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:37:01,300] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65895,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:37:01,320] jepsen worker 46 - jepsen.util 11116 :ok :read [15 34] INFO [2019-12-17 12:37:01,320] jepsen worker 46 - jepsen.util 11116 :invoke :read [15 nil] INFO [2019-12-17 12:37:01,339] jepsen worker 63 - jepsen.util 10143 :fail :cas [21 [29 35]] :ValueNotSwapFrom INFO [2019-12-17 12:37:01,339] jepsen worker 63 - jepsen.util 10143 :invoke :write [21 21] INFO [2019-12-17 12:37:01,342] jepsen worker 36 - jepsen.util 12186 :fail :cas [12 [20 32]] :ValueNotSwapFrom INFO [2019-12-17 12:37:01,342] jepsen worker 36 - jepsen.util 12186 :invoke :read [12 nil] INFO [2019-12-17 12:37:01,389] jepsen worker 10 - jepsen.util 11080 :fail :cas [3 [36 29]] :ValueNotSwapFrom INFO [2019-12-17 12:37:01,390] jepsen worker 10 - jepsen.util 11080 :invoke :read [3 nil] INFO [2019-12-17 12:37:01,401] jepsen worker 44 - jepsen.util 16064 :info :cas [14 [0 34]] :RequestCanceledException INFO [2019-12-17 12:37:01,411] jepsen worker 44 - jepsen.util 16154 :invoke :read [14 nil] INFO [2019-12-17 12:37:01,417] jepsen worker 68 - jepsen.util 14828 :info :cas [22 [23 34]] :RequestCanceledException INFO [2019-12-17 12:37:01,418] jepsen worker 6 - jepsen.util 15936 :info :cas [2 [4 42]] :RequestCanceledException INFO [2019-12-17 12:37:01,421] jepsen worker 19 - jepsen.util 11629 :ok :write [6 37] INFO [2019-12-17 12:37:01,422] jepsen worker 74 - jepsen.util 11144 :ok :write [24 49] INFO [2019-12-17 12:37:01,422] jepsen worker 74 - jepsen.util 11144 :invoke :write [24 23] INFO [2019-12-17 12:37:01,422] jepsen worker 19 - jepsen.util 11629 :invoke :read [6 nil] INFO [2019-12-17 12:37:01,428] jepsen worker 46 - jepsen.util 11116 :ok :read [15 34] INFO [2019-12-17 12:37:01,429] jepsen worker 46 - jepsen.util 11116 :invoke :read [15 nil] INFO [2019-12-17 12:37:01,429] jepsen worker 6 - jepsen.util 16026 :invoke :read [2 nil] INFO [2019-12-17 12:37:01,432] jepsen worker 68 - jepsen.util 14918 :invoke :cas [22 [12 7]] INFO [2019-12-17 12:37:01,435] jepsen worker 81 - jepsen.util 15111 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:37:01,436] jepsen worker 81 - jepsen.util 15111 :invoke :write [27 32] INFO [2019-12-17 12:37:01,447] jepsen worker 75 - jepsen.util 15015 :info :write [25 1] :RequestCanceledException INFO [2019-12-17 12:37:01,447] jepsen worker 34 - jepsen.util 15964 :info :write [11 27] :RequestCanceledException INFO [2019-12-17 12:37:01,449] jepsen worker 16 - jepsen.util 17116 :info :cas [5 [48 26]] :RequestCanceledException INFO [2019-12-17 12:37:01,461] jepsen worker 26 - jepsen.util 16226 :info :cas [8 [45 40]] :RequestCanceledException INFO [2019-12-17 12:37:01,463] jepsen worker 35 - jepsen.util 15515 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:37:01,463] jepsen worker 35 - jepsen.util 15515 :invoke :read [11 nil] INFO [2019-12-17 12:37:01,477] jepsen worker 71 - jepsen.util 12311 :ok :read [23 13] INFO [2019-12-17 12:37:01,477] jepsen worker 71 - jepsen.util 12311 :invoke :cas [23 [5 37]] INFO [2019-12-17 12:37:01,477] jepsen worker 75 - jepsen.util 15105 :invoke :cas [25 [9 5]] INFO [2019-12-17 12:37:01,479] jepsen worker 16 - jepsen.util 17206 :invoke :cas [5 [49 45]] INFO [2019-12-17 12:37:01,479] jepsen worker 34 - jepsen.util 16054 :invoke :write [11 14] INFO [2019-12-17 12:37:01,486] jepsen worker 26 - jepsen.util 16316 :invoke :cas [8 [30 39]] INFO [2019-12-17 12:37:01,488] jepsen worker 63 - jepsen.util 10143 :ok :write [21 21] INFO [2019-12-17 12:37:01,489] jepsen worker 63 - jepsen.util 10143 :invoke :cas [21 [10 6]] INFO [2019-12-17 12:37:01,497] jepsen worker 11 - jepsen.util 10991 :info :cas [3 [27 48]] :RequestCanceledException INFO [2019-12-17 12:37:01,497] jepsen worker 82 - jepsen.util 15652 :info :write [27 11] :RequestCanceledException INFO [2019-12-17 12:37:01,497] jepsen worker 66 - jepsen.util 15456 :info :cas [22 [16 20]] :RequestCanceledException INFO [2019-12-17 12:37:01,498] jepsen worker 42 - jepsen.util 16422 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:37:01,498] jepsen worker 78 - jepsen.util 11328 :info :write [26 16] :RequestCanceledException INFO [2019-12-17 12:37:01,498] jepsen worker 42 - jepsen.util 16422 :invoke :write [14 18] INFO [2019-12-17 12:37:01,499] jepsen worker 5 - jepsen.util 11075 :fail :cas [1 [28 4]] :ValueNotSwapFrom INFO [2019-12-17 12:37:01,500] jepsen worker 5 - jepsen.util 11075 :invoke :write [1 6] INFO [2019-12-17 12:37:01,520] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65914,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:37:01,524] jepsen worker 1 - jepsen.util 15391 :info :cas [0 [0 9]] :RequestCanceledException INFO [2019-12-17 12:37:01,525] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65914,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:37:01,527] jepsen worker 77 - jepsen.util 16277 :info :cas [25 [16 43]] :RequestCanceledException INFO [2019-12-17 12:37:01,527] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65917,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:37:01,530] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65917,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:37:01,533] jepsen worker 76 - jepsen.util 15916 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:37:01,533] jepsen worker 76 - jepsen.util 15916 :invoke :write [25 32] INFO [2019-12-17 12:37:01,534] jepsen worker 24 - jepsen.util 14154 :info :cas [8 [38 3]] :RequestCanceledException INFO [2019-12-17 12:37:01,535] jepsen worker 57 - jepsen.util 15177 :info :cas [19 [39 19]] :RequestCanceledException INFO [2019-12-17 12:37:01,536] jepsen worker 41 - jepsen.util 16601 :info :cas [13 [24 19]] :RequestCanceledException INFO [2019-12-17 12:37:01,540] jepsen worker 40 - jepsen.util 15970 :info :write [13 49] :RequestCanceledException INFO [2019-12-17 12:37:01,542] jepsen worker 82 - jepsen.util 15742 :invoke :write [27 40] INFO [2019-12-17 12:37:01,544] jepsen worker 22 - jepsen.util 15592 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:37:01,544] jepsen worker 22 - jepsen.util 15592 :invoke :write [7 49] INFO [2019-12-17 12:37:01,545] jepsen worker 83 - jepsen.util 15563 :info :write [27 36] :RequestCanceledException INFO [2019-12-17 12:37:01,546] jepsen worker 78 - jepsen.util 11418 :invoke :read [26 nil] INFO [2019-12-17 12:37:01,548] jepsen worker 59 - jepsen.util 14729 :info :cas [19 [10 11]] :RequestCanceledException INFO [2019-12-17 12:37:01,572] jepsen worker 11 - jepsen.util 11081 :invoke :read [3 nil] INFO [2019-12-17 12:37:01,589] jepsen worker 66 - jepsen.util 15546 :invoke :read [22 nil] INFO [2019-12-17 12:37:01,595] jepsen worker 77 - jepsen.util 16367 :invoke :cas [25 [31 43]] INFO [2019-12-17 12:37:01,597] jepsen worker 1 - jepsen.util 15481 :invoke :write [0 44] INFO [2019-12-17 12:37:01,610] jepsen worker 24 - jepsen.util 14244 :invoke :cas [8 [41 0]] INFO [2019-12-17 12:37:01,611] jepsen worker 40 - jepsen.util 16060 :invoke :cas [13 [15 29]] INFO [2019-12-17 12:37:01,611] jepsen worker 41 - jepsen.util 16691 :invoke :read [13 nil] INFO [2019-12-17 12:37:01,613] jepsen worker 59 - jepsen.util 14819 :invoke :cas [19 [32 40]] INFO [2019-12-17 12:37:01,614] jepsen worker 83 - jepsen.util 15653 :invoke :cas [27 [46 6]] INFO [2019-12-17 12:37:01,614] jepsen worker 57 - jepsen.util 15267 :invoke :cas [19 [13 2]] INFO [2019-12-17 12:37:01,634] jepsen worker 49 - jepsen.util 15529 :info :cas [16 [42 8]] :RequestCanceledException INFO [2019-12-17 12:37:01,646] jepsen worker 86 - jepsen.util 16196 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:37:01,647] jepsen worker 86 - jepsen.util 16196 :invoke :cas [28 [32 20]] INFO [2019-12-17 12:37:01,664] jepsen worker 60 - jepsen.util 16620 :info :cas [20 [13 5]] :RequestCanceledException INFO [2019-12-17 12:37:01,677] jepsen worker 49 - jepsen.util 15619 :invoke :write [16 29] INFO [2019-12-17 12:37:01,695] jepsen worker 60 - jepsen.util 16710 :invoke :cas [20 [41 25]] INFO [2019-12-17 12:37:01,719] jepsen worker 88 - jepsen.util 11518 :info :write [29 22] :RequestCanceledException INFO [2019-12-17 12:37:01,729] jepsen worker 10 - jepsen.util 11080 :ok :read [3 41] INFO [2019-12-17 12:37:01,730] jepsen worker 10 - jepsen.util 11080 :invoke :cas [3 [7 43]] INFO [2019-12-17 12:37:01,743] jepsen worker 55 - jepsen.util 11935 :info :write [18 46] :RequestCanceledException INFO [2019-12-17 12:37:01,749] jepsen worker 39 - jepsen.util 15069 :info :cas [13 [21 40]] :RequestCanceledException INFO [2019-12-17 12:37:01,757] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65937,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:37:01,757] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65937,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:37:01,758] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65941,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:37:01,762] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65941,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:37:01,766] jepsen worker 80 - jepsen.util 10970 :info :write [26 38] :RequestCanceledException INFO [2019-12-17 12:37:01,773] jepsen worker 88 - jepsen.util 11608 :invoke :cas [29 [13 0]] INFO [2019-12-17 12:37:01,777] jepsen worker 39 - jepsen.util 15159 :invoke :read [13 nil] INFO [2019-12-17 12:37:01,782] jepsen worker 55 - jepsen.util 12025 :invoke :write [18 44] INFO [2019-12-17 12:37:01,792] jepsen worker 80 - jepsen.util 11060 :invoke :write [26 39] INFO [2019-12-17 12:37:01,846] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:37:01,848] jepsen nemesis - couchbase.util Rebalance check count: 518 INFO [2019-12-17 12:37:01,850] jepsen nemesis - couchbase.util Rebalance stuck count: 499 INFO [2019-12-17 12:37:01,916] jepsen worker 25 - jepsen.util 16495 :info :cas [8 [46 16]] :RequestCanceledException INFO [2019-12-17 12:37:01,975] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65956,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:37:01,978] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65956,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:37:01,981] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65959,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:37:01,995] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65959,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:37:01,998] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65960,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:37:02,001] jepsen worker 25 - jepsen.util 16585 :invoke :write [8 31] INFO [2019-12-17 12:37:02,001] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65960,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:37:02,004] jepsen worker 33 - jepsen.util 15423 :info :cas [11 [16 8]] :RequestCanceledException INFO [2019-12-17 12:37:02,005] jepsen worker 38 - jepsen.util 11738 :info :write [12 13] :RequestCanceledException INFO [2019-12-17 12:37:02,019] jepsen worker 70 - jepsen.util 11950 :info :write [23 6] :RequestCanceledException INFO [2019-12-17 12:37:02,039] jepsen worker 0 - jepsen.util 16290 :info :write [0 23] :RequestCanceledException INFO [2019-12-17 12:37:02,041] jepsen worker 30 - jepsen.util 10560 :ok :read [10 15] INFO [2019-12-17 12:37:02,044] jepsen worker 30 - jepsen.util 10560 :invoke :read [10 nil] INFO [2019-12-17 12:37:02,058] jepsen worker 85 - jepsen.util 16285 :info :write [28 37] :RequestCanceledException INFO [2019-12-17 12:37:02,101] jepsen worker 33 - jepsen.util 15513 :invoke :read [11 nil] INFO [2019-12-17 12:37:02,101] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65966,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:37:02,101] jepsen worker 4 - jepsen.util 10444 :info :write [1 8] :RequestCanceledException INFO [2019-12-17 12:37:02,102] jepsen worker 38 - jepsen.util 11828 :invoke :read [12 nil] INFO [2019-12-17 12:37:02,104] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65966,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:37:02,105] jepsen worker 70 - jepsen.util 12040 :invoke :cas [23 [12 37]] INFO [2019-12-17 12:37:02,106] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65968,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:37:02,111] jepsen worker 0 - jepsen.util 16380 :invoke :read [0 nil] INFO [2019-12-17 12:37:02,111] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65968,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:37:02,113] jepsen worker 85 - jepsen.util 16375 :invoke :read [28 nil] INFO [2019-12-17 12:37:02,117] jepsen worker 4 - jepsen.util 10534 :invoke :read [1 nil] INFO [2019-12-17 12:37:02,119] jepsen worker 14 - jepsen.util 11174 :ok :read [4 42] INFO [2019-12-17 12:37:02,119] jepsen worker 14 - jepsen.util 11174 :invoke :read [4 nil] INFO [2019-12-17 12:37:02,122] jepsen worker 19 - jepsen.util 11629 :ok :read [6 37] INFO [2019-12-17 12:37:02,123] jepsen worker 19 - jepsen.util 11629 :invoke :write [6 39] INFO [2019-12-17 12:37:02,146] jepsen worker 46 - jepsen.util 11116 :ok :read [15 34] INFO [2019-12-17 12:37:02,146] jepsen worker 46 - jepsen.util 11116 :invoke :cas [15 [3 39]] INFO [2019-12-17 12:37:02,171] jepsen worker 63 - jepsen.util 10143 :fail :cas [21 [10 6]] :ValueNotSwapFrom INFO [2019-12-17 12:37:02,172] jepsen worker 63 - jepsen.util 10143 :invoke :cas [21 [28 22]] INFO [2019-12-17 12:37:02,203] jepsen worker 67 - jepsen.util 15637 :info :cas [22 [16 26]] :RequestCanceledException INFO [2019-12-17 12:37:02,220] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65980,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:37:02,221] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65980,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:37:02,222] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65981,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:37:02,222] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65981,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:37:02,224] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65982,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:37:02,228] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65982,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:37:02,240] jepsen worker 67 - jepsen.util 15727 :invoke :read [22 nil] INFO [2019-12-17 12:37:02,247] jepsen worker 48 - jepsen.util 14358 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:37:02,249] jepsen worker 48 - jepsen.util 14358 :invoke :write [16 13] INFO [2019-12-17 12:37:02,259] jepsen worker 12 - jepsen.util 11622 :fail :cas [4 [24 37]] :ValueNotSwapFrom INFO [2019-12-17 12:37:02,260] jepsen worker 12 - jepsen.util 11622 :invoke :cas [4 [17 48]] INFO [2019-12-17 12:37:02,271] jepsen worker 45 - jepsen.util 11655 :ok :write [15 12] INFO [2019-12-17 12:37:02,272] jepsen worker 45 - jepsen.util 11655 :invoke :write [15 44] INFO [2019-12-17 12:37:02,318] jepsen worker 52 - jepsen.util 10672 :info :write [17 25] :RequestCanceledException INFO [2019-12-17 12:37:02,367] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65987,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:37:02,368] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65987,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:37:02,368] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65991,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:37:02,370] jepsen worker 29 - jepsen.util 9569 :fail :read [9 nil] :RequestTimeoutException INFO [2019-12-17 12:37:02,371] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":65991,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:37:02,371] jepsen worker 10 - jepsen.util 11080 :fail :cas [3 [7 43]] :ValueNotSwapFrom INFO [2019-12-17 12:37:02,371] jepsen worker 29 - jepsen.util 9569 :invoke :write [9 34] INFO [2019-12-17 12:37:02,373] jepsen worker 7 - jepsen.util 15577 :info :cas [2 [18 1]] :RequestCanceledException INFO [2019-12-17 12:37:02,373] jepsen worker 10 - jepsen.util 11080 :invoke :write [3 20] INFO [2019-12-17 12:37:02,375] jepsen worker 73 - jepsen.util 10603 :info :write [24 37] :RequestCanceledException INFO [2019-12-17 12:37:02,375] jepsen worker 3 - jepsen.util 10803 :info :write [1 35] :RequestCanceledException INFO [2019-12-17 12:37:02,378] jepsen worker 8 - jepsen.util 15398 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:37:02,379] jepsen worker 52 - jepsen.util 10762 :invoke :write [17 23] INFO [2019-12-17 12:37:02,379] jepsen worker 27 - jepsen.util 10557 :info :write [9 45] :RequestCanceledException INFO [2019-12-17 12:37:02,380] jepsen worker 8 - jepsen.util 15398 :invoke :cas [2 [16 48]] INFO [2019-12-17 12:37:02,399] jepsen worker 27 - jepsen.util 10647 :invoke :cas [9 [15 25]] INFO [2019-12-17 12:37:02,408] jepsen worker 73 - jepsen.util 10693 :invoke :read [24 nil] INFO [2019-12-17 12:37:02,408] jepsen worker 3 - jepsen.util 10893 :invoke :cas [1 [44 22]] INFO [2019-12-17 12:37:02,413] jepsen worker 7 - jepsen.util 15667 :invoke :write [2 26] INFO [2019-12-17 12:37:02,416] jepsen worker 72 - jepsen.util 11682 :info :write [24 19] :RequestCanceledException INFO [2019-12-17 12:37:02,433] jepsen worker 61 - jepsen.util 16621 :info :cas [20 [14 16]] :RequestCanceledException INFO [2019-12-17 12:37:02,433] jepsen worker 2 - jepsen.util 15032 :info :write [0 23] :RequestCanceledException INFO [2019-12-17 12:37:02,444] jepsen worker 72 - jepsen.util 11772 :invoke :write [24 24] INFO [2019-12-17 12:37:02,463] jepsen worker 87 - jepsen.util 10977 :info :write [29 24] :RequestCanceledException INFO [2019-12-17 12:37:02,465] jepsen worker 61 - jepsen.util 16711 :invoke :write [20 47] INFO [2019-12-17 12:37:02,472] jepsen worker 2 - jepsen.util 15122 :invoke :cas [0 [15 0]] INFO [2019-12-17 12:37:02,479] jepsen worker 87 - jepsen.util 11067 :invoke :read [29 nil] INFO [2019-12-17 12:37:02,488] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66003,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:37:02,489] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66003,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:37:02,494] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66006,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:37:02,496] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66006,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:37:02,503] jepsen worker 15 - jepsen.util 14415 :info :cas [5 [29 12]] :RequestCanceledException INFO [2019-12-17 12:37:02,504] jepsen worker 43 - jepsen.util 15703 :info :write [14 20] :RequestCanceledException INFO [2019-12-17 12:37:02,505] jepsen worker 18 - jepsen.util 11448 :info :write [6 17] :RequestCanceledException INFO [2019-12-17 12:37:02,513] jepsen worker 30 - jepsen.util 10560 :ok :read [10 15] INFO [2019-12-17 12:37:02,513] jepsen worker 30 - jepsen.util 10560 :invoke :read [10 nil] INFO [2019-12-17 12:37:02,523] jepsen worker 58 - jepsen.util 15808 :info :write [19 11] :RequestCanceledException INFO [2019-12-17 12:37:02,524] jepsen worker 17 - jepsen.util 15497 :info :cas [5 [23 44]] :RequestCanceledException INFO [2019-12-17 12:37:02,526] jepsen worker 46 - jepsen.util 11116 :fail :cas [15 [3 39]] :ValueNotSwapFrom INFO [2019-12-17 12:37:02,527] jepsen worker 46 - jepsen.util 11116 :invoke :write [15 30] INFO [2019-12-17 12:37:02,529] jepsen worker 31 - jepsen.util 9751 :fail :read [10 nil] :RequestTimeoutException INFO [2019-12-17 12:37:02,529] jepsen worker 31 - jepsen.util 9751 :invoke :write [10 35] INFO [2019-12-17 12:37:02,533] jepsen worker 28 - jepsen.util 9478 :fail :read [9 nil] :RequestTimeoutException INFO [2019-12-17 12:37:02,534] jepsen worker 28 - jepsen.util 9478 :invoke :read [9 nil] INFO [2019-12-17 12:37:02,536] jepsen worker 84 - jepsen.util 15294 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:37:02,536] jepsen worker 84 - jepsen.util 15294 :invoke :write [28 44] INFO [2019-12-17 12:37:02,541] jepsen worker 18 - jepsen.util 11538 :invoke :read [6 nil] INFO [2019-12-17 12:37:02,544] jepsen worker 43 - jepsen.util 15793 :invoke :read [14 nil] INFO [2019-12-17 12:37:02,546] jepsen worker 17 - jepsen.util 15587 :invoke :write [5 14] INFO [2019-12-17 12:37:02,546] jepsen worker 58 - jepsen.util 15898 :invoke :cas [19 [8 11]] INFO [2019-12-17 12:37:02,547] jepsen worker 50 - jepsen.util 16070 :info :write [16 0] :RequestCanceledException INFO [2019-12-17 12:37:02,552] jepsen worker 15 - jepsen.util 14505 :invoke :read [5 nil] INFO [2019-12-17 12:37:02,564] jepsen worker 23 - jepsen.util 15503 :info :cas [7 [42 49]] :RequestCanceledException INFO [2019-12-17 12:37:02,569] jepsen worker 21 - jepsen.util 15051 :info :cas [7 [5 16]] :RequestCanceledException INFO [2019-12-17 12:37:02,578] jepsen worker 50 - jepsen.util 16160 :invoke :read [16 nil] INFO [2019-12-17 12:37:02,607] jepsen worker 23 - jepsen.util 15593 :invoke :cas [7 [43 35]] INFO [2019-12-17 12:37:02,613] jepsen worker 21 - jepsen.util 15141 :invoke :read [7 nil] INFO [2019-12-17 12:37:02,623] jepsen worker 62 - jepsen.util 16082 :info :cas [20 [49 15]] :RequestCanceledException INFO [2019-12-17 12:37:02,697] jepsen worker 62 - jepsen.util 16172 :invoke :cas [20 [24 33]] INFO [2019-12-17 12:37:02,709] jepsen worker 13 - jepsen.util 10633 :info :write [4 30] :RequestCanceledException INFO [2019-12-17 12:37:02,716] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66025,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:37:02,722] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66025,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:37:02,725] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66029,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:37:02,725] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66029,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:37:02,739] jepsen worker 47 - jepsen.util 12107 :info :cas [15 [0 32]] :RequestCanceledException INFO [2019-12-17 12:37:02,739] jepsen worker 64 - jepsen.util 8704 :info :cas [21 [7 41]] :RequestCanceledException INFO [2019-12-17 12:37:02,740] jepsen worker 64 - jepsen.util 8794 :invoke :write [21 19] INFO [2019-12-17 12:37:02,745] jepsen worker 47 - jepsen.util 12197 :invoke :write [15 43] INFO [2019-12-17 12:37:02,753] jepsen worker 74 - jepsen.util 11144 :ok :write [24 23] INFO [2019-12-17 12:37:02,755] jepsen worker 19 - jepsen.util 11629 :ok :write [6 39] INFO [2019-12-17 12:37:02,758] jepsen worker 19 - jepsen.util 11629 :invoke :read [6 nil] INFO [2019-12-17 12:37:02,759] jepsen worker 74 - jepsen.util 11144 :invoke :cas [24 [39 48]] INFO [2019-12-17 12:37:02,787] jepsen worker 13 - jepsen.util 10723 :invoke :read [4 nil] INFO [2019-12-17 12:37:02,808] jepsen worker 32 - jepsen.util 11642 :info :write [10 17] :RequestCanceledException INFO [2019-12-17 12:37:02,837] jepsen worker 65 - jepsen.util 9965 :fail :read [21 nil] :RequestTimeoutException INFO [2019-12-17 12:37:02,838] jepsen worker 65 - jepsen.util 9965 :invoke :write [21 5] INFO [2019-12-17 12:37:02,880] jepsen worker 63 - jepsen.util 10143 :fail :cas [21 [28 22]] :ValueNotSwapFrom INFO [2019-12-17 12:37:02,886] jepsen worker 63 - jepsen.util 10143 :invoke :read [21 nil] INFO [2019-12-17 12:37:02,897] jepsen worker 69 - jepsen.util 11589 :info :write [23 13] :RequestCanceledException INFO [2019-12-17 12:37:02,912] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:37:02,918] jepsen nemesis - couchbase.util Rebalance check count: 519 INFO [2019-12-17 12:37:02,935] jepsen nemesis - couchbase.util Rebalance stuck count: 500 INFO [2019-12-17 12:37:02,936] jepsen worker 56 - jepsen.util 12116 :info :write [18 5] :RequestCanceledException INFO [2019-12-17 12:37:02,954] jepsen worker 32 - jepsen.util 11732 :invoke :write [10 31] INFO [2019-12-17 12:37:02,962] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66040,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:37:02,963] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66040,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:37:02,975] jepsen worker 78 - jepsen.util 11418 :ok :read [26 4] INFO [2019-12-17 12:37:02,976] jepsen worker 78 - jepsen.util 11418 :invoke :cas [26 [21 44]] INFO [2019-12-17 12:37:03,001] jepsen worker 69 - jepsen.util 11679 :invoke :cas [23 [36 7]] INFO [2019-12-17 12:37:03,010] jepsen worker 56 - jepsen.util 12206 :invoke :write [18 4] INFO [2019-12-17 12:37:03,023] jepsen worker 45 - jepsen.util 11655 :ok :write [15 44] INFO [2019-12-17 12:37:03,024] jepsen worker 45 - jepsen.util 11655 :invoke :read [15 nil] INFO [2019-12-17 12:37:03,056] jepsen worker 51 - jepsen.util 10671 :info :write [17 17] :RequestCanceledException INFO [2019-12-17 12:37:03,067] jepsen worker 37 - jepsen.util 10837 :info :write [12 3] :RequestCanceledException INFO [2019-12-17 12:37:03,095] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66046,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:37:03,098] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66046,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:37:03,105] jepsen worker 36 - jepsen.util 12186 :ok :read [12 34] INFO [2019-12-17 12:37:03,107] jepsen worker 36 - jepsen.util 12186 :invoke :read [12 nil] INFO [2019-12-17 12:37:03,110] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66051,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:37:03,112] jepsen worker 9 - jepsen.util 11889 :info :write [3 49] :RequestCanceledException INFO [2019-12-17 12:37:03,113] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66051,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:37:03,126] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66052,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:37:03,134] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66052,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:37:03,136] jepsen worker 79 - jepsen.util 11509 :info :write [26 23] :RequestCanceledException INFO [2019-12-17 12:37:03,136] jepsen worker 54 - jepsen.util 11934 :fail :cas [18 [37 43]] :ValueNotSwapFrom INFO [2019-12-17 12:37:03,138] jepsen worker 53 - jepsen.util 10763 :info :write [17 4] :RequestCanceledException INFO [2019-12-17 12:37:03,139] jepsen worker 54 - jepsen.util 11934 :invoke :read [18 nil] INFO [2019-12-17 12:37:03,152] jepsen worker 51 - jepsen.util 10761 :invoke :read [17 nil] INFO [2019-12-17 12:37:03,162] jepsen worker 71 - jepsen.util 12311 :fail :cas [23 [5 37]] :ValueNotSwapFrom INFO [2019-12-17 12:37:03,162] jepsen worker 71 - jepsen.util 12311 :invoke :read [23 nil] INFO [2019-12-17 12:37:03,178] jepsen worker 27 - jepsen.util 10647 :fail :cas [9 [15 25]] :ValueNotSwapFrom INFO [2019-12-17 12:37:03,178] jepsen worker 27 - jepsen.util 10647 :invoke :cas [9 [23 2]] INFO [2019-12-17 12:37:03,181] jepsen worker 37 - jepsen.util 10927 :invoke :write [12 33] INFO [2019-12-17 12:37:03,211] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66061,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:37:03,219] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66061,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:37:03,221] jepsen worker 9 - jepsen.util 11979 :invoke :write [3 13] INFO [2019-12-17 12:37:03,268] jepsen worker 79 - jepsen.util 11599 :invoke :write [26 18] INFO [2019-12-17 12:37:03,269] jepsen worker 53 - jepsen.util 10853 :invoke :write [17 25] INFO [2019-12-17 12:37:03,304] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66068,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:37:03,313] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66068,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:37:03,329] jepsen worker 31 - jepsen.util 9751 :ok :write [10 35] INFO [2019-12-17 12:37:03,330] jepsen worker 31 - jepsen.util 9751 :invoke :read [10 nil] INFO [2019-12-17 12:37:03,335] jepsen worker 20 - jepsen.util 11000 :info :write [6 19] :RequestCanceledException INFO [2019-12-17 12:37:03,376] jepsen worker 20 - jepsen.util 11090 :invoke :read [6 nil] INFO [2019-12-17 12:37:03,405] jepsen worker 89 - jepsen.util 11429 :info :write [29 34] :RequestCanceledException INFO [2019-12-17 12:37:03,469] jepsen worker 68 - jepsen.util 14918 :info :cas [22 [12 7]] :RequestCanceledException INFO [2019-12-17 12:37:03,474] jepsen worker 73 - jepsen.util 10693 :ok :read [24 49] INFO [2019-12-17 12:37:03,475] jepsen worker 73 - jepsen.util 10693 :invoke :write [24 29] INFO [2019-12-17 12:37:03,476] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66082,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:37:03,485] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66082,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:37:03,500] jepsen worker 10 - jepsen.util 11080 :ok :write [3 20] INFO [2019-12-17 12:37:03,501] jepsen worker 10 - jepsen.util 11080 :invoke :write [3 6] INFO [2019-12-17 12:37:03,509] jepsen worker 89 - jepsen.util 11519 :invoke :read [29 nil] INFO [2019-12-17 12:37:03,528] jepsen worker 44 - jepsen.util 16154 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:37:03,529] jepsen worker 44 - jepsen.util 16154 :invoke :cas [14 [26 39]] INFO [2019-12-17 12:37:03,534] jepsen worker 16 - jepsen.util 17206 :info :cas [5 [49 45]] :RequestCanceledException INFO [2019-12-17 12:37:03,534] jepsen worker 26 - jepsen.util 16316 :info :cas [8 [30 39]] :RequestCanceledException INFO [2019-12-17 12:37:03,534] jepsen worker 6 - jepsen.util 16026 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:37:03,535] jepsen worker 81 - jepsen.util 15111 :info :write [27 32] :RequestCanceledException INFO [2019-12-17 12:37:03,536] jepsen worker 6 - jepsen.util 16026 :invoke :cas [2 [39 44]] INFO [2019-12-17 12:37:03,571] jepsen worker 68 - jepsen.util 15008 :invoke :read [22 nil] INFO [2019-12-17 12:37:03,582] jepsen worker 75 - jepsen.util 15105 :info :cas [25 [9 5]] :RequestCanceledException INFO [2019-12-17 12:37:03,586] jepsen worker 35 - jepsen.util 15515 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:37:03,589] jepsen worker 35 - jepsen.util 15515 :invoke :cas [11 [47 21]] INFO [2019-12-17 12:37:03,592] jepsen worker 16 - jepsen.util 17296 :invoke :cas [5 [1 19]] INFO [2019-12-17 12:37:03,606] jepsen worker 81 - jepsen.util 15201 :invoke :cas [27 [5 37]] INFO [2019-12-17 12:37:03,610] jepsen worker 26 - jepsen.util 16406 :invoke :write [8 19] INFO [2019-12-17 12:37:03,639] jepsen worker 66 - jepsen.util 15546 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:37:03,640] jepsen worker 24 - jepsen.util 14244 :info :cas [8 [41 0]] :RequestCanceledException INFO [2019-12-17 12:37:03,644] jepsen worker 66 - jepsen.util 15546 :invoke :write [22 0] INFO [2019-12-17 12:37:03,645] jepsen worker 59 - jepsen.util 14819 :info :cas [19 [32 40]] :RequestCanceledException INFO [2019-12-17 12:37:03,648] jepsen worker 40 - jepsen.util 16060 :info :cas [13 [15 29]] :RequestCanceledException INFO [2019-12-17 12:37:03,651] jepsen worker 86 - jepsen.util 16196 :info :cas [28 [32 20]] :RequestCanceledException INFO [2019-12-17 12:37:03,663] jepsen worker 4 - jepsen.util 10534 :ok :read [1 24] INFO [2019-12-17 12:37:03,663] jepsen worker 4 - jepsen.util 10534 :invoke :cas [1 [38 2]] INFO [2019-12-17 12:37:03,680] jepsen worker 41 - jepsen.util 16691 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:37:03,681] jepsen worker 41 - jepsen.util 16691 :invoke :read [13 nil] INFO [2019-12-17 12:37:03,688] jepsen worker 72 - jepsen.util 11772 :ok :write [24 24] INFO [2019-12-17 12:37:03,690] jepsen worker 72 - jepsen.util 11772 :invoke :cas [24 [37 49]] INFO [2019-12-17 12:37:03,704] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66095,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:37:03,712] jepsen worker 77 - jepsen.util 16367 :info :cas [25 [31 43]] :RequestCanceledException INFO [2019-12-17 12:37:03,713] jepsen worker 34 - jepsen.util 16054 :info :write [11 14] :RequestCanceledException INFO [2019-12-17 12:37:03,713] jepsen worker 83 - jepsen.util 15653 :info :cas [27 [46 6]] :RequestCanceledException INFO [2019-12-17 12:37:03,715] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66095,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:37:03,723] jepsen worker 75 - jepsen.util 15195 :invoke :read [25 nil] INFO [2019-12-17 12:37:03,728] jepsen worker 57 - jepsen.util 15267 :info :cas [19 [13 2]] :RequestCanceledException INFO [2019-12-17 12:37:03,741] jepsen worker 1 - jepsen.util 15481 :info :write [0 44] :RequestCanceledException INFO [2019-12-17 12:37:03,750] jepsen worker 22 - jepsen.util 15592 :info :write [7 49] :RequestCanceledException INFO [2019-12-17 12:37:03,751] jepsen worker 76 - jepsen.util 15916 :info :write [25 32] :RequestCanceledException INFO [2019-12-17 12:37:03,752] jepsen worker 82 - jepsen.util 15742 :info :write [27 40] :RequestCanceledException INFO [2019-12-17 12:37:03,753] jepsen worker 60 - jepsen.util 16710 :info :cas [20 [41 25]] :RequestCanceledException INFO [2019-12-17 12:37:03,763] jepsen worker 5 - jepsen.util 11075 :info :write [1 6] :RequestCanceledException INFO [2019-12-17 12:37:03,779] jepsen worker 40 - jepsen.util 16150 :invoke :cas [13 [0 17]] INFO [2019-12-17 12:37:03,795] jepsen worker 59 - jepsen.util 14909 :invoke :write [19 42] INFO [2019-12-17 12:37:03,806] jepsen worker 28 - jepsen.util 9478 :ok :read [9 1] INFO [2019-12-17 12:37:03,807] jepsen worker 28 - jepsen.util 9478 :invoke :read [9 nil] INFO [2019-12-17 12:37:03,809] jepsen worker 42 - jepsen.util 16422 :info :write [14 18] :RequestCanceledException INFO [2019-12-17 12:37:03,811] jepsen worker 24 - jepsen.util 14334 :invoke :read [8 nil] INFO [2019-12-17 12:37:03,833] jepsen worker 77 - jepsen.util 16457 :invoke :write [25 11] INFO [2019-12-17 12:37:03,834] jepsen worker 86 - jepsen.util 16286 :invoke :write [28 48] INFO [2019-12-17 12:37:03,835] jepsen worker 49 - jepsen.util 15619 :info :write [16 29] :RequestCanceledException INFO [2019-12-17 12:37:03,840] jepsen worker 39 - jepsen.util 15159 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:37:03,843] jepsen worker 39 - jepsen.util 15159 :invoke :read [13 nil] INFO [2019-12-17 12:37:03,881] jepsen worker 5 - jepsen.util 11165 :invoke :cas [1 [47 7]] INFO [2019-12-17 12:37:03,881] jepsen worker 34 - jepsen.util 16144 :invoke :read [11 nil] INFO [2019-12-17 12:37:03,885] jepsen worker 76 - jepsen.util 16006 :invoke :read [25 nil] INFO [2019-12-17 12:37:03,896] jepsen worker 82 - jepsen.util 15832 :invoke :write [27 37] INFO [2019-12-17 12:37:03,897] jepsen worker 22 - jepsen.util 15682 :invoke :read [7 nil] INFO [2019-12-17 12:37:03,900] jepsen worker 60 - jepsen.util 16800 :invoke :cas [20 [25 16]] INFO [2019-12-17 12:37:03,900] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66110,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:37:03,901] jepsen worker 1 - jepsen.util 15571 :invoke :cas [0 [32 25]] INFO [2019-12-17 12:37:03,901] jepsen worker 83 - jepsen.util 15743 :invoke :cas [27 [32 15]] INFO [2019-12-17 12:37:03,902] jepsen worker 57 - jepsen.util 15357 :invoke :cas [19 [7 27]] INFO [2019-12-17 12:37:03,903] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66110,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:37:03,906] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66114,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:37:03,909] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66114,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:37:03,910] jepsen worker 42 - jepsen.util 16512 :invoke :cas [14 [23 38]] INFO [2019-12-17 12:37:03,929] jepsen worker 49 - jepsen.util 15709 :invoke :read [16 nil] INFO [2019-12-17 12:37:03,954] jepsen worker 88 - jepsen.util 11608 :fail :cas [29 [13 0]] :ValueNotSwapFrom INFO [2019-12-17 12:37:03,956] jepsen worker 88 - jepsen.util 11608 :invoke :write [29 31] INFO [2019-12-17 12:37:03,964] jepsen worker 65 - jepsen.util 9965 :ok :write [21 5] INFO [2019-12-17 12:37:03,965] jepsen worker 65 - jepsen.util 9965 :invoke :read [21 nil] INFO [2019-12-17 12:37:04,023] jepsen worker 80 - jepsen.util 11060 :info :write [26 39] :RequestCanceledException INFO [2019-12-17 12:37:04,030] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:37:04,034] jepsen worker 18 - jepsen.util 11538 :ok :read [6 37] INFO [2019-12-17 12:37:04,036] jepsen worker 18 - jepsen.util 11538 :invoke :read [6 nil] INFO [2019-12-17 12:37:04,045] jepsen nemesis - couchbase.util Rebalance check count: 520 INFO [2019-12-17 12:37:04,045] jepsen nemesis - couchbase.util Rebalance stuck count: 501 INFO [2019-12-17 12:37:04,067] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66128,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:37:04,088] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66128,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:37:04,089] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66131,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:37:04,094] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66131,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:37:04,122] jepsen worker 55 - jepsen.util 12025 :info :write [18 44] :RequestCanceledException INFO [2019-12-17 12:37:04,124] jepsen worker 80 - jepsen.util 11150 :invoke :read [26 nil] INFO [2019-12-17 12:37:04,134] jepsen worker 47 - jepsen.util 12197 :ok :write [15 43] INFO [2019-12-17 12:37:04,135] jepsen worker 47 - jepsen.util 12197 :invoke :read [15 nil] INFO [2019-12-17 12:37:04,150] jepsen worker 33 - jepsen.util 15513 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:37:04,153] jepsen worker 33 - jepsen.util 15513 :invoke :cas [11 [41 7]] INFO [2019-12-17 12:37:04,896] jepsen worker 70 - jepsen.util 12040 :fail :cas [23 [12 37]] :ValueNotSwapFrom INFO [2019-12-17 12:37:04,897] jepsen worker 70 - jepsen.util 12040 :invoke :write [23 32] INFO [2019-12-17 12:37:04,899] jepsen worker 19 - jepsen.util 11629 :ok :read [6 39] INFO [2019-12-17 12:37:04,899] jepsen worker 55 - jepsen.util 12115 :invoke :write [18 17] INFO [2019-12-17 12:37:04,899] jepsen worker 27 - jepsen.util 10647 :fail :cas [9 [23 2]] :ValueNotSwapFrom INFO [2019-12-17 12:37:04,899] jepsen worker 69 - jepsen.util 11679 :fail :cas [23 [36 7]] :ValueNotSwapFrom INFO [2019-12-17 12:37:04,899] jepsen worker 19 - jepsen.util 11629 :invoke :cas [6 [18 9]] INFO [2019-12-17 12:37:04,899] jepsen worker 27 - jepsen.util 10647 :invoke :write [9 11] INFO [2019-12-17 12:37:04,899] jepsen worker 64 - jepsen.util 8794 :ok :write [21 19] INFO [2019-12-17 12:37:04,899] jepsen worker 38 - jepsen.util 11828 :ok :read [12 34] INFO [2019-12-17 12:37:04,899] jepsen worker 69 - jepsen.util 11679 :invoke :read [23 nil] INFO [2019-12-17 12:37:04,904] jepsen worker 64 - jepsen.util 8794 :invoke :cas [21 [28 18]] INFO [2019-12-17 12:37:04,905] jepsen worker 38 - jepsen.util 11828 :invoke :write [12 26] INFO [2019-12-17 12:37:04,905] jepsen worker 3 - jepsen.util 10893 :fail :cas [1 [44 22]] :ValueNotSwapFrom INFO [2019-12-17 12:37:04,907] jepsen worker 14 - jepsen.util 11174 :ok :read [4 42] INFO [2019-12-17 12:37:04,908] jepsen worker 3 - jepsen.util 10893 :invoke :read [1 nil] INFO [2019-12-17 12:37:04,908] jepsen worker 14 - jepsen.util 11174 :invoke :cas [4 [48 26]] INFO [2019-12-17 12:37:04,912] jepsen worker 12 - jepsen.util 11622 :fail :cas [4 [17 48]] :ValueNotSwapFrom INFO [2019-12-17 12:37:04,912] jepsen worker 12 - jepsen.util 11622 :invoke :read [4 nil] INFO [2019-12-17 12:37:04,912] jepsen worker 74 - jepsen.util 11144 :fail :cas [24 [39 48]] :ValueNotSwapFrom INFO [2019-12-17 12:37:04,913] jepsen worker 74 - jepsen.util 11144 :invoke :read [24 nil] INFO [2019-12-17 12:37:04,913] jepsen worker 63 - jepsen.util 10143 :ok :read [21 21] INFO [2019-12-17 12:37:04,914] jepsen worker 63 - jepsen.util 10143 :invoke :read [21 nil] INFO [2019-12-17 12:37:04,914] jepsen worker 71 - jepsen.util 12311 :ok :read [23 13] INFO [2019-12-17 12:37:04,914] jepsen worker 71 - jepsen.util 12311 :invoke :cas [23 [3 34]] INFO [2019-12-17 12:37:04,916] jepsen worker 72 - jepsen.util 11772 :fail :cas [24 [37 49]] :ValueNotSwapFrom INFO [2019-12-17 12:37:04,916] jepsen worker 72 - jepsen.util 11772 :invoke :cas [24 [0 31]] INFO [2019-12-17 12:37:04,918] jepsen worker 4 - jepsen.util 10534 :fail :cas [1 [38 2]] :ValueNotSwapFrom INFO [2019-12-17 12:37:04,918] jepsen worker 4 - jepsen.util 10534 :invoke :read [1 nil] INFO [2019-12-17 12:37:04,924] jepsen worker 11 - jepsen.util 11081 :ok :read [3 41] INFO [2019-12-17 12:37:04,924] jepsen worker 11 - jepsen.util 11081 :invoke :cas [3 [16 39]] INFO [2019-12-17 12:37:04,924] jepsen worker 30 - jepsen.util 10560 :ok :read [10 15] INFO [2019-12-17 12:37:04,924] jepsen worker 30 - jepsen.util 10560 :invoke :read [10 nil] INFO [2019-12-17 12:37:04,926] jepsen worker 58 - jepsen.util 15898 :info :cas [19 [8 11]] :RequestCanceledException INFO [2019-12-17 12:37:04,926] jepsen worker 61 - jepsen.util 16711 :info :write [20 47] :RequestCanceledException INFO [2019-12-17 12:37:04,926] jepsen worker 29 - jepsen.util 9569 :info :write [9 34] :RequestCanceledException INFO [2019-12-17 12:37:04,926] jepsen worker 52 - jepsen.util 10762 :info :write [17 23] :RequestCanceledException INFO [2019-12-17 12:37:04,926] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66138,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:37:04,927] jepsen worker 31 - jepsen.util 9751 :ok :read [10 35] INFO [2019-12-17 12:37:04,927] jepsen worker 58 - jepsen.util 15988 :invoke :read [19 nil] INFO [2019-12-17 12:37:04,928] jepsen worker 29 - jepsen.util 9659 :invoke :cas [9 [5 13]] INFO [2019-12-17 12:37:04,928] jepsen worker 52 - jepsen.util 10852 :invoke :cas [17 [20 48]] INFO [2019-12-17 12:37:04,928] jepsen worker 61 - jepsen.util 16801 :invoke :cas [20 [19 48]] INFO [2019-12-17 12:37:04,928] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66138,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:37:04,928] jepsen worker 31 - jepsen.util 9751 :invoke :read [10 nil] INFO [2019-12-17 12:37:04,928] jepsen worker 48 - jepsen.util 14358 :info :write [16 13] :RequestCanceledException INFO [2019-12-17 12:37:04,928] jepsen worker 67 - jepsen.util 15727 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:37:04,928] jepsen worker 46 - jepsen.util 11116 :info :write [15 30] :RequestCanceledException INFO [2019-12-17 12:37:04,928] jepsen worker 17 - jepsen.util 15587 :info :write [5 14] :RequestCanceledException INFO [2019-12-17 12:37:04,928] jepsen worker 62 - jepsen.util 16172 :info :cas [20 [24 33]] :RequestCanceledException INFO [2019-12-17 12:37:04,929] jepsen worker 25 - jepsen.util 16585 :info :write [8 31] :RequestCanceledException INFO [2019-12-17 12:37:04,929] jepsen worker 2 - jepsen.util 15122 :info :cas [0 [15 0]] :RequestCanceledException INFO [2019-12-17 12:37:04,929] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66142,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:37:04,929] jepsen worker 15 - jepsen.util 14505 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:37:04,929] jepsen worker 28 - jepsen.util 9478 :ok :read [9 1] INFO [2019-12-17 12:37:04,929] jepsen worker 8 - jepsen.util 15398 :info :cas [2 [16 48]] :RequestCanceledException INFO [2019-12-17 12:37:04,929] jepsen worker 7 - jepsen.util 15667 :info :write [2 26] :RequestCanceledException INFO [2019-12-17 12:37:04,929] jepsen worker 43 - jepsen.util 15793 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:37:04,929] jepsen worker 84 - jepsen.util 15294 :info :write [28 44] :RequestCanceledException INFO [2019-12-17 12:37:04,929] jepsen worker 21 - jepsen.util 15141 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:37:04,930] jepsen worker 23 - jepsen.util 15593 :info :cas [7 [43 35]] :RequestCanceledException INFO [2019-12-17 12:37:04,930] jepsen worker 85 - jepsen.util 16375 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:37:04,930] jepsen worker 67 - jepsen.util 15727 :invoke :write [22 47] INFO [2019-12-17 12:37:04,930] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66140,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:37:04,931] jepsen worker 0 - jepsen.util 16380 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:37:04,931] jepsen worker 87 - jepsen.util 11067 :fail :read [29 nil] :RequestTimeoutException INFO [2019-12-17 12:37:04,931] jepsen worker 48 - jepsen.util 14448 :invoke :cas [16 [43 2]] INFO [2019-12-17 12:37:04,931] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66139,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:37:04,931] jepsen worker 13 - jepsen.util 10723 :fail :read [4 nil] :RequestTimeoutException INFO [2019-12-17 12:37:04,931] jepsen worker 17 - jepsen.util 15677 :invoke :read [5 nil] INFO [2019-12-17 12:37:04,931] jepsen worker 46 - jepsen.util 11206 :invoke :read [15 nil] INFO [2019-12-17 12:37:04,931] jepsen worker 65 - jepsen.util 9965 :ok :read [21 5] INFO [2019-12-17 12:37:04,932] jepsen worker 50 - jepsen.util 16160 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:37:04,932] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66142,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:37:04,932] jepsen worker 15 - jepsen.util 14505 :invoke :cas [5 [31 7]] INFO [2019-12-17 12:37:04,932] jepsen worker 28 - jepsen.util 9478 :invoke :read [9 nil] INFO [2019-12-17 12:37:04,932] jepsen worker 62 - jepsen.util 16262 :invoke :write [20 22] INFO [2019-12-17 12:37:04,932] jepsen worker 43 - jepsen.util 15793 :invoke :read [14 nil] INFO [2019-12-17 12:37:04,932] jepsen worker 25 - jepsen.util 16675 :invoke :read [8 nil] INFO [2019-12-17 12:37:04,933] jepsen worker 2 - jepsen.util 15212 :invoke :read [0 nil] INFO [2019-12-17 12:37:04,933] jepsen worker 21 - jepsen.util 15141 :invoke :cas [7 [23 42]] INFO [2019-12-17 12:37:04,933] jepsen worker 85 - jepsen.util 16375 :invoke :read [28 nil] INFO [2019-12-17 12:37:04,933] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66140,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:37:04,933] jepsen worker 0 - jepsen.util 16380 :invoke :cas [0 [41 5]] INFO [2019-12-17 12:37:04,933] jepsen worker 87 - jepsen.util 11067 :invoke :write [29 14] INFO [2019-12-17 12:37:04,934] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66139,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:37:04,934] jepsen worker 13 - jepsen.util 10723 :invoke :write [4 15] INFO [2019-12-17 12:37:04,934] jepsen worker 65 - jepsen.util 9965 :invoke :write [21 4] INFO [2019-12-17 12:37:04,934] jepsen worker 8 - jepsen.util 15488 :invoke :read [2 nil] INFO [2019-12-17 12:37:04,934] jepsen worker 7 - jepsen.util 15757 :invoke :read [2 nil] INFO [2019-12-17 12:37:04,934] jepsen worker 50 - jepsen.util 16160 :invoke :read [16 nil] INFO [2019-12-17 12:37:04,935] jepsen worker 23 - jepsen.util 15683 :invoke :cas [7 [39 19]] INFO [2019-12-17 12:37:04,935] jepsen worker 84 - jepsen.util 15384 :invoke :cas [28 [0 38]] INFO [2019-12-17 12:37:04,935] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66144,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:37:04,935] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66141,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:37:04,938] jepsen worker 47 - jepsen.util 12197 :ok :read [15 43] INFO [2019-12-17 12:37:04,938] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66144,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:37:04,938] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66141,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:37:04,939] jepsen worker 47 - jepsen.util 12197 :invoke :read [15 nil] INFO [2019-12-17 12:37:04,939] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66145,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:37:04,939] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66145,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:37:04,942] jepsen worker 78 - jepsen.util 11418 :fail :cas [26 [21 44]] :ValueNotSwapFrom INFO [2019-12-17 12:37:04,942] jepsen worker 78 - jepsen.util 11418 :invoke :cas [26 [31 33]] INFO [2019-12-17 12:37:04,943] jepsen worker 45 - jepsen.util 11655 :ok :read [15 44] INFO [2019-12-17 12:37:04,943] jepsen worker 51 - jepsen.util 10761 :ok :read [17 4] INFO [2019-12-17 12:37:04,943] jepsen worker 45 - jepsen.util 11655 :invoke :cas [15 [6 44]] INFO [2019-12-17 12:37:04,944] jepsen worker 51 - jepsen.util 10761 :invoke :write [17 44] INFO [2019-12-17 12:37:04,974] jepsen worker 64 - jepsen.util 8794 :fail :cas [21 [28 18]] :ValueNotSwapFrom INFO [2019-12-17 12:37:04,974] jepsen worker 64 - jepsen.util 8794 :invoke :write [21 37] INFO [2019-12-17 12:37:04,995] jepsen worker 36 - jepsen.util 12186 :ok :read [12 34] INFO [2019-12-17 12:37:04,995] jepsen worker 36 - jepsen.util 12186 :invoke :read [12 nil] INFO [2019-12-17 12:37:05,035] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66193,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:37:05,036] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66193,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:37:05,036] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66194,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:37:05,036] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66194,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:37:05,036] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66195,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:37:05,036] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66195,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:37:05,039] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66181,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:37:05,039] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66181,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:37:05,039] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66185,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:37:05,039] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66185,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:37:05,039] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66186,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:37:05,039] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66186,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:37:05,040] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66224,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:37:05,040] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66224,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:37:05,040] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66225,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:37:05,040] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66225,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:37:05,040] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66227,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:37:05,040] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66227,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:37:05,045] jepsen worker 74 - jepsen.util 11144 :ok :read [24 24] INFO [2019-12-17 12:37:05,045] jepsen worker 74 - jepsen.util 11144 :invoke :cas [24 [38 8]] INFO [2019-12-17 12:37:05,052] jepsen worker 79 - jepsen.util 11599 :ok :write [26 18] INFO [2019-12-17 12:37:05,052] jepsen worker 79 - jepsen.util 11599 :invoke :cas [26 [37 12]] INFO [2019-12-17 12:37:05,066] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:37:05,066] jepsen nemesis - couchbase.util Rebalance check count: 521 INFO [2019-12-17 12:37:05,066] jepsen nemesis - couchbase.util Rebalance stuck count: 502 INFO [2019-12-17 12:37:05,074] jepsen worker 11 - jepsen.util 11081 :fail :cas [3 [16 39]] :ValueNotSwapFrom INFO [2019-12-17 12:37:05,075] jepsen worker 11 - jepsen.util 11081 :invoke :write [3 19] INFO [2019-12-17 12:37:05,086] jepsen worker 19 - jepsen.util 11629 :fail :cas [6 [18 9]] :ValueNotSwapFrom INFO [2019-12-17 12:37:05,086] jepsen worker 19 - jepsen.util 11629 :invoke :write [6 29] INFO [2019-12-17 12:37:05,100] jepsen worker 27 - jepsen.util 10647 :ok :write [9 11] INFO [2019-12-17 12:37:05,101] jepsen worker 27 - jepsen.util 10647 :invoke :cas [9 [1 38]] INFO [2019-12-17 12:37:05,112] jepsen worker 46 - jepsen.util 11206 :ok :read [15 43] INFO [2019-12-17 12:37:05,112] jepsen worker 46 - jepsen.util 11206 :invoke :write [15 47] INFO [2019-12-17 12:37:05,145] jepsen worker 27 - jepsen.util 10647 :fail :cas [9 [1 38]] :ValueNotSwapFrom INFO [2019-12-17 12:37:05,145] jepsen worker 27 - jepsen.util 10647 :invoke :read [9 nil] INFO [2019-12-17 12:37:05,174] jepsen worker 89 - jepsen.util 11519 :ok :read [29 7] INFO [2019-12-17 12:37:05,174] jepsen worker 89 - jepsen.util 11519 :invoke :read [29 nil] INFO [2019-12-17 12:37:05,194] jepsen worker 3 - jepsen.util 10893 :ok :read [1 24] INFO [2019-12-17 12:37:05,194] jepsen worker 3 - jepsen.util 10893 :invoke :read [1 nil] INFO [2019-12-17 12:37:05,201] jepsen worker 27 - jepsen.util 10647 :ok :read [9 11] INFO [2019-12-17 12:37:05,201] jepsen worker 30 - jepsen.util 10560 :ok :read [10 35] INFO [2019-12-17 12:37:05,201] jepsen worker 27 - jepsen.util 10647 :invoke :read [9 nil] INFO [2019-12-17 12:37:05,201] jepsen worker 30 - jepsen.util 10560 :invoke :write [10 40] INFO [2019-12-17 12:37:05,203] jepsen worker 45 - jepsen.util 11655 :fail :cas [15 [6 44]] :ValueNotSwapFrom INFO [2019-12-17 12:37:05,204] jepsen worker 45 - jepsen.util 11655 :invoke :cas [15 [26 8]] INFO [2019-12-17 12:37:05,221] jepsen worker 27 - jepsen.util 10647 :ok :read [9 11] INFO [2019-12-17 12:37:05,221] jepsen worker 27 - jepsen.util 10647 :invoke :write [9 10] INFO [2019-12-17 12:37:05,232] jepsen worker 54 - jepsen.util 11934 :fail :read [18 nil] :RequestTimeoutException INFO [2019-12-17 12:37:05,233] jepsen worker 54 - jepsen.util 11934 :invoke :read [18 nil] INFO [2019-12-17 12:37:05,239] jepsen worker 14 - jepsen.util 11174 :fail :cas [4 [48 26]] :ValueNotSwapFrom INFO [2019-12-17 12:37:05,239] jepsen worker 14 - jepsen.util 11174 :invoke :read [4 nil] INFO [2019-12-17 12:37:05,255] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66240,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:37:05,255] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66240,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:37:05,255] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66241,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:37:05,255] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66241,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:37:05,257] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66251,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:37:05,257] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66251,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:37:05,259] jepsen worker 71 - jepsen.util 12311 :fail :cas [23 [3 34]] :ValueNotSwapFrom INFO [2019-12-17 12:37:05,259] jepsen worker 71 - jepsen.util 12311 :invoke :cas [23 [5 37]] INFO [2019-12-17 12:37:05,310] jepsen worker 79 - jepsen.util 11599 :fail :cas [26 [37 12]] :ValueNotSwapFrom INFO [2019-12-17 12:37:05,310] jepsen worker 79 - jepsen.util 11599 :invoke :read [26 nil] INFO [2019-12-17 12:37:05,324] jepsen worker 88 - jepsen.util 11608 :ok :write [29 31] INFO [2019-12-17 12:37:05,324] jepsen worker 88 - jepsen.util 11608 :invoke :read [29 nil] INFO [2019-12-17 12:37:05,329] jepsen worker 64 - jepsen.util 8794 :ok :write [21 37] INFO [2019-12-17 12:37:05,329] jepsen worker 64 - jepsen.util 8794 :invoke :write [21 19] INFO [2019-12-17 12:37:05,341] jepsen worker 27 - jepsen.util 10647 :ok :write [9 10] INFO [2019-12-17 12:37:05,341] jepsen worker 27 - jepsen.util 10647 :invoke :write [9 45] INFO [2019-12-17 12:37:05,341] jepsen worker 3 - jepsen.util 10893 :ok :read [1 24] INFO [2019-12-17 12:37:05,342] jepsen worker 3 - jepsen.util 10893 :invoke :read [1 nil] INFO [2019-12-17 12:37:05,359] jepsen worker 14 - jepsen.util 11174 :ok :read [4 30] INFO [2019-12-17 12:37:05,359] jepsen worker 14 - jepsen.util 11174 :invoke :read [4 nil] INFO [2019-12-17 12:37:05,385] jepsen worker 45 - jepsen.util 11655 :fail :cas [15 [26 8]] :ValueNotSwapFrom INFO [2019-12-17 12:37:05,385] jepsen worker 45 - jepsen.util 11655 :invoke :write [15 28] INFO [2019-12-17 12:37:05,397] jepsen worker 27 - jepsen.util 10647 :ok :write [9 45] INFO [2019-12-17 12:37:05,398] jepsen worker 27 - jepsen.util 10647 :invoke :cas [9 [16 38]] INFO [2019-12-17 12:37:05,417] jepsen worker 32 - jepsen.util 11732 :info :write [10 31] :RequestCanceledException INFO [2019-12-17 12:37:05,418] jepsen worker 32 - jepsen.util 11822 :invoke :write [10 2] INFO [2019-12-17 12:37:05,422] jepsen worker 27 - jepsen.util 10647 :fail :cas [9 [16 38]] :ValueNotSwapFrom INFO [2019-12-17 12:37:05,422] jepsen worker 27 - jepsen.util 10647 :invoke :cas [9 [43 12]] INFO [2019-12-17 12:37:05,432] jepsen worker 71 - jepsen.util 12311 :fail :cas [23 [5 37]] :ValueNotSwapFrom INFO [2019-12-17 12:37:05,432] jepsen worker 71 - jepsen.util 12311 :invoke :cas [23 [21 31]] INFO [2019-12-17 12:37:05,447] jepsen worker 56 - jepsen.util 12206 :info :write [18 4] :RequestCanceledException INFO [2019-12-17 12:37:05,447] jepsen worker 20 - jepsen.util 11090 :fail :read [6 nil] :RequestTimeoutException INFO [2019-12-17 12:37:05,447] jepsen worker 20 - jepsen.util 11090 :invoke :read [6 nil] INFO [2019-12-17 12:37:05,448] jepsen worker 56 - jepsen.util 12296 :invoke :write [18 26] INFO [2019-12-17 12:37:05,450] jepsen worker 27 - jepsen.util 10647 :fail :cas [9 [43 12]] :ValueNotSwapFrom INFO [2019-12-17 12:37:05,450] jepsen worker 27 - jepsen.util 10647 :invoke :write [9 47] INFO [2019-12-17 12:37:05,456] jepsen worker 79 - jepsen.util 11599 :ok :read [26 39] INFO [2019-12-17 12:37:05,456] jepsen worker 79 - jepsen.util 11599 :invoke :read [26 nil] INFO [2019-12-17 12:37:05,476] jepsen worker 3 - jepsen.util 10893 :ok :read [1 24] INFO [2019-12-17 12:37:05,476] jepsen worker 3 - jepsen.util 10893 :invoke :write [1 24] INFO [2019-12-17 12:37:05,480] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66272,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:37:05,480] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66272,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:37:05,480] jepsen worker 64 - jepsen.util 8794 :ok :write [21 19] INFO [2019-12-17 12:37:05,480] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66274,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:37:05,480] jepsen worker 64 - jepsen.util 8794 :invoke :read [21 nil] INFO [2019-12-17 12:37:05,480] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66274,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:37:05,540] jepsen worker 27 - jepsen.util 10647 :ok :write [9 47] INFO [2019-12-17 12:37:05,540] jepsen worker 27 - jepsen.util 10647 :invoke :read [9 nil] INFO [2019-12-17 12:37:05,560] jepsen worker 27 - jepsen.util 10647 :ok :read [9 47] INFO [2019-12-17 12:37:05,560] jepsen worker 27 - jepsen.util 10647 :invoke :cas [9 [35 20]] INFO [2019-12-17 12:37:05,562] jepsen worker 14 - jepsen.util 11174 :ok :read [4 30] INFO [2019-12-17 12:37:05,562] jepsen worker 14 - jepsen.util 11174 :invoke :cas [4 [49 12]] INFO [2019-12-17 12:37:05,585] jepsen worker 27 - jepsen.util 10647 :fail :cas [9 [35 20]] :ValueNotSwapFrom INFO [2019-12-17 12:37:05,585] jepsen worker 27 - jepsen.util 10647 :invoke :cas [9 [20 46]] INFO [2019-12-17 12:37:05,599] jepsen worker 16 - jepsen.util 17296 :info :cas [5 [1 19]] :RequestCanceledException INFO [2019-12-17 12:37:05,600] jepsen worker 16 - jepsen.util 17386 :invoke :read [5 nil] INFO [2019-12-17 12:37:05,605] jepsen worker 27 - jepsen.util 10647 :fail :cas [9 [20 46]] :ValueNotSwapFrom INFO [2019-12-17 12:37:05,605] jepsen worker 27 - jepsen.util 10647 :invoke :write [9 2] INFO [2019-12-17 12:37:05,618] jepsen worker 44 - jepsen.util 16154 :info :cas [14 [26 39]] :RequestCanceledException INFO [2019-12-17 12:37:05,618] jepsen worker 81 - jepsen.util 15201 :info :cas [27 [5 37]] :RequestCanceledException INFO [2019-12-17 12:37:05,619] jepsen worker 81 - jepsen.util 15291 :invoke :write [27 21] INFO [2019-12-17 12:37:05,619] jepsen worker 44 - jepsen.util 16244 :invoke :cas [14 [13 37]] INFO [2019-12-17 12:37:05,633] jepsen worker 68 - jepsen.util 15008 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:37:05,633] jepsen worker 68 - jepsen.util 15008 :invoke :read [22 nil] INFO [2019-12-17 12:37:05,633] jepsen worker 6 - jepsen.util 16026 :info :cas [2 [39 44]] :RequestCanceledException INFO [2019-12-17 12:37:05,634] jepsen worker 6 - jepsen.util 16116 :invoke :read [2 nil] INFO [2019-12-17 12:37:05,649] jepsen worker 71 - jepsen.util 12311 :fail :cas [23 [21 31]] :ValueNotSwapFrom INFO [2019-12-17 12:37:05,649] jepsen worker 71 - jepsen.util 12311 :invoke :read [23 nil] INFO [2019-12-17 12:37:05,664] jepsen worker 35 - jepsen.util 15515 :info :cas [11 [47 21]] :RequestCanceledException INFO [2019-12-17 12:37:05,666] jepsen worker 35 - jepsen.util 15605 :invoke :read [11 nil] INFO [2019-12-17 12:37:05,686] jepsen worker 27 - jepsen.util 10647 :ok :write [9 2] INFO [2019-12-17 12:37:05,686] jepsen worker 27 - jepsen.util 10647 :invoke :read [9 nil] INFO [2019-12-17 12:37:05,699] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66292,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:37:05,699] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66292,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:37:05,699] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66295,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:37:05,699] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66295,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:37:05,705] jepsen worker 27 - jepsen.util 10647 :ok :read [9 2] INFO [2019-12-17 12:37:05,705] jepsen worker 27 - jepsen.util 10647 :invoke :cas [9 [34 18]] INFO [2019-12-17 12:37:05,732] jepsen worker 27 - jepsen.util 10647 :fail :cas [9 [34 18]] :ValueNotSwapFrom INFO [2019-12-17 12:37:05,732] jepsen worker 53 - jepsen.util 10853 :info :write [17 25] :RequestCanceledException INFO [2019-12-17 12:37:05,732] jepsen worker 27 - jepsen.util 10647 :invoke :write [9 14] INFO [2019-12-17 12:37:05,734] jepsen worker 53 - jepsen.util 10943 :invoke :read [17 nil] INFO [2019-12-17 12:37:05,734] jepsen worker 37 - jepsen.util 10927 :info :write [12 33] :RequestCanceledException INFO [2019-12-17 12:37:05,735] jepsen worker 37 - jepsen.util 11017 :invoke :write [12 24] INFO [2019-12-17 12:37:05,741] jepsen worker 38 - jepsen.util 11828 :ok :write [12 26] INFO [2019-12-17 12:37:05,741] jepsen worker 38 - jepsen.util 11828 :invoke :read [12 nil] INFO [2019-12-17 12:37:05,747] jepsen worker 75 - jepsen.util 15195 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:37:05,747] jepsen worker 75 - jepsen.util 15195 :invoke :write [25 23] INFO [2019-12-17 12:37:05,764] jepsen worker 41 - jepsen.util 16691 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:37:05,764] jepsen worker 41 - jepsen.util 16691 :invoke :cas [13 [45 1]] INFO [2019-12-17 12:37:05,784] jepsen worker 73 - jepsen.util 10693 :ok :write [24 29] INFO [2019-12-17 12:37:05,784] jepsen worker 73 - jepsen.util 10693 :invoke :write [24 14] INFO [2019-12-17 12:37:05,794] jepsen worker 9 - jepsen.util 11979 :info :write [3 13] :RequestCanceledException INFO [2019-12-17 12:37:05,796] jepsen worker 9 - jepsen.util 12069 :invoke :cas [3 [19 6]] INFO [2019-12-17 12:37:05,796] jepsen worker 10 - jepsen.util 11080 :ok :write [3 6] INFO [2019-12-17 12:37:05,796] jepsen worker 10 - jepsen.util 11080 :invoke :cas [3 [25 15]] INFO [2019-12-17 12:37:05,816] jepsen worker 63 - jepsen.util 10143 :ok :read [21 19] INFO [2019-12-17 12:37:05,816] jepsen worker 63 - jepsen.util 10143 :invoke :read [21 nil] INFO [2019-12-17 12:37:05,819] jepsen worker 56 - jepsen.util 12296 :ok :write [18 26] INFO [2019-12-17 12:37:05,819] jepsen worker 56 - jepsen.util 12296 :invoke :cas [18 [1 25]] INFO [2019-12-17 12:37:05,827] jepsen worker 72 - jepsen.util 11772 :fail :cas [24 [0 31]] :ValueNotSwapFrom INFO [2019-12-17 12:37:05,827] jepsen worker 72 - jepsen.util 11772 :invoke :cas [24 [10 41]] INFO [2019-12-17 12:37:05,852] jepsen worker 20 - jepsen.util 11090 :ok :read [6 39] INFO [2019-12-17 12:37:05,853] jepsen worker 20 - jepsen.util 11090 :invoke :write [6 10] INFO [2019-12-17 12:37:05,863] jepsen worker 40 - jepsen.util 16150 :info :cas [13 [0 17]] :RequestCanceledException INFO [2019-12-17 12:37:05,864] jepsen worker 40 - jepsen.util 16240 :invoke :cas [13 [3 12]] INFO [2019-12-17 12:37:05,885] jepsen worker 9 - jepsen.util 12069 :fail :cas [3 [19 6]] :ValueNotSwapFrom INFO [2019-12-17 12:37:05,885] jepsen worker 9 - jepsen.util 12069 :invoke :write [3 48] INFO [2019-12-17 12:37:05,886] jepsen worker 27 - jepsen.util 10647 :ok :write [9 14] INFO [2019-12-17 12:37:05,886] jepsen worker 27 - jepsen.util 10647 :invoke :cas [9 [0 17]] INFO [2019-12-17 12:37:05,899] jepsen worker 24 - jepsen.util 14334 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:37:05,899] jepsen worker 24 - jepsen.util 14334 :invoke :cas [8 [8 10]] INFO [2019-12-17 12:37:05,902] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66317,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:37:05,902] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66317,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:37:05,903] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66320,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:37:05,903] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66320,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:37:05,915] jepsen worker 22 - jepsen.util 15682 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:37:05,915] jepsen worker 22 - jepsen.util 15682 :invoke :cas [7 [7 28]] INFO [2019-12-17 12:37:05,921] jepsen worker 79 - jepsen.util 11599 :ok :read [26 23] INFO [2019-12-17 12:37:05,921] jepsen worker 79 - jepsen.util 11599 :invoke :write [26 6] INFO [2019-12-17 12:37:05,923] jepsen worker 39 - jepsen.util 15159 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:37:05,923] jepsen worker 76 - jepsen.util 16006 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:37:05,923] jepsen worker 39 - jepsen.util 15159 :invoke :cas [13 [32 40]] INFO [2019-12-17 12:37:05,923] jepsen worker 57 - jepsen.util 15357 :info :cas [19 [7 27]] :RequestCanceledException INFO [2019-12-17 12:37:05,923] jepsen worker 76 - jepsen.util 16006 :invoke :read [25 nil] INFO [2019-12-17 12:37:05,925] jepsen worker 57 - jepsen.util 15447 :invoke :write [19 46] INFO [2019-12-17 12:37:05,927] jepsen worker 60 - jepsen.util 16800 :info :cas [20 [25 16]] :RequestCanceledException INFO [2019-12-17 12:37:05,928] jepsen worker 60 - jepsen.util 16890 :invoke :write [20 5] INFO [2019-12-17 12:37:05,940] jepsen worker 5 - jepsen.util 11165 :info :cas [1 [47 7]] :RequestCanceledException INFO [2019-12-17 12:37:05,940] jepsen worker 1 - jepsen.util 15571 :info :cas [0 [32 25]] :RequestCanceledException INFO [2019-12-17 12:37:05,941] jepsen worker 5 - jepsen.util 11255 :invoke :write [1 35] INFO [2019-12-17 12:37:05,941] jepsen worker 1 - jepsen.util 15661 :invoke :cas [0 [23 41]] INFO [2019-12-17 12:37:05,946] jepsen worker 27 - jepsen.util 10647 :fail :cas [9 [0 17]] :ValueNotSwapFrom INFO [2019-12-17 12:37:05,946] jepsen worker 27 - jepsen.util 10647 :invoke :read [9 nil] INFO [2019-12-17 12:37:05,948] jepsen worker 38 - jepsen.util 11828 :ok :read [12 26] INFO [2019-12-17 12:37:05,949] jepsen worker 38 - jepsen.util 11828 :invoke :write [12 0] INFO [2019-12-17 12:37:05,954] jepsen worker 10 - jepsen.util 11080 :fail :cas [3 [25 15]] :ValueNotSwapFrom INFO [2019-12-17 12:37:05,954] jepsen worker 10 - jepsen.util 11080 :invoke :cas [3 [6 34]] INFO [2019-12-17 12:37:05,960] jepsen worker 34 - jepsen.util 16144 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:37:05,960] jepsen worker 83 - jepsen.util 15743 :info :cas [27 [32 15]] :RequestCanceledException INFO [2019-12-17 12:37:05,961] jepsen worker 34 - jepsen.util 16144 :invoke :read [11 nil] INFO [2019-12-17 12:37:05,962] jepsen worker 83 - jepsen.util 15833 :invoke :read [27 nil] INFO [2019-12-17 12:37:05,996] jepsen worker 72 - jepsen.util 11772 :fail :cas [24 [10 41]] :ValueNotSwapFrom INFO [2019-12-17 12:37:05,996] jepsen worker 72 - jepsen.util 11772 :invoke :read [24 nil] INFO [2019-12-17 12:37:06,000] jepsen worker 42 - jepsen.util 16512 :info :cas [14 [23 38]] :RequestCanceledException INFO [2019-12-17 12:37:06,001] jepsen worker 42 - jepsen.util 16602 :invoke :write [14 43] INFO [2019-12-17 12:37:06,010] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66339,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:37:06,010] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66339,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:37:06,010] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66340,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:37:06,010] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66340,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:37:06,010] jepsen worker 80 - jepsen.util 11150 :ok :read [26 23] INFO [2019-12-17 12:37:06,011] jepsen worker 80 - jepsen.util 11150 :invoke :read [26 nil] INFO [2019-12-17 12:37:06,016] jepsen worker 49 - jepsen.util 15709 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:37:06,016] jepsen worker 49 - jepsen.util 15709 :invoke :read [16 nil] INFO [2019-12-17 12:37:06,017] jepsen worker 27 - jepsen.util 10647 :ok :read [9 14] INFO [2019-12-17 12:37:06,017] jepsen worker 27 - jepsen.util 10647 :invoke :read [9 nil] INFO [2019-12-17 12:37:06,031] jepsen worker 27 - jepsen.util 10647 :ok :read [9 14] INFO [2019-12-17 12:37:06,032] jepsen worker 27 - jepsen.util 10647 :invoke :cas [9 [49 1]] INFO [2019-12-17 12:37:06,043] jepsen worker 89 - jepsen.util 11519 :ok :read [29 31] INFO [2019-12-17 12:37:06,044] jepsen worker 89 - jepsen.util 11519 :invoke :cas [29 [27 19]] INFO [2019-12-17 12:37:06,047] jepsen worker 27 - jepsen.util 10647 :fail :cas [9 [49 1]] :ValueNotSwapFrom INFO [2019-12-17 12:37:06,047] jepsen worker 27 - jepsen.util 10647 :invoke :cas [9 [3 28]] INFO [2019-12-17 12:37:06,060] jepsen worker 27 - jepsen.util 10647 :fail :cas [9 [3 28]] :ValueNotSwapFrom INFO [2019-12-17 12:37:06,060] jepsen worker 27 - jepsen.util 10647 :invoke :read [9 nil] INFO [2019-12-17 12:37:06,077] jepsen worker 27 - jepsen.util 10647 :ok :read [9 14] INFO [2019-12-17 12:37:06,077] jepsen worker 27 - jepsen.util 10647 :invoke :write [9 15] INFO [2019-12-17 12:37:06,080] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:37:06,080] jepsen nemesis - couchbase.util Rebalance check count: 522 INFO [2019-12-17 12:37:06,080] jepsen nemesis - couchbase.util Rebalance stuck count: 503 INFO [2019-12-17 12:37:06,112] jepsen worker 69 - jepsen.util 11679 :ok :read [23 13] INFO [2019-12-17 12:37:06,112] jepsen worker 69 - jepsen.util 11679 :invoke :read [23 nil] INFO [2019-12-17 12:37:06,116] jepsen worker 18 - jepsen.util 11538 :fail :read [6 nil] :RequestTimeoutException INFO [2019-12-17 12:37:06,116] jepsen worker 18 - jepsen.util 11538 :invoke :write [6 19] INFO [2019-12-17 12:37:06,133] jepsen worker 27 - jepsen.util 10647 :ok :write [9 15] INFO [2019-12-17 12:37:06,133] jepsen worker 27 - jepsen.util 10647 :invoke :cas [9 [48 23]] INFO [2019-12-17 12:37:06,160] jepsen worker 27 - jepsen.util 10647 :fail :cas [9 [48 23]] :ValueNotSwapFrom INFO [2019-12-17 12:37:06,160] jepsen worker 27 - jepsen.util 10647 :invoke :cas [9 [36 43]] INFO [2019-12-17 12:37:06,173] jepsen worker 27 - jepsen.util 10647 :fail :cas [9 [36 43]] :ValueNotSwapFrom INFO [2019-12-17 12:37:06,174] jepsen worker 27 - jepsen.util 10647 :invoke :cas [9 [38 15]] INFO [2019-12-17 12:37:06,184] jepsen worker 27 - jepsen.util 10647 :fail :cas [9 [38 15]] :ValueNotSwapFrom INFO [2019-12-17 12:37:06,185] jepsen worker 27 - jepsen.util 10647 :invoke :write [9 28] INFO [2019-12-17 12:37:06,220] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66359,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:37:06,220] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66359,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:37:06,220] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66361,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:37:06,220] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66361,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:37:06,232] jepsen worker 27 - jepsen.util 10647 :ok :write [9 28] INFO [2019-12-17 12:37:06,232] jepsen worker 27 - jepsen.util 10647 :invoke :write [9 16] INFO [2019-12-17 12:37:06,235] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66368,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:37:06,235] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66368,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:37:06,235] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66370,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:37:06,235] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66370,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:37:06,245] jepsen worker 33 - jepsen.util 15513 :info :cas [11 [41 7]] :RequestCanceledException INFO [2019-12-17 12:37:06,249] jepsen worker 33 - jepsen.util 15603 :invoke :read [11 nil] INFO [2019-12-17 12:37:06,256] jepsen worker 12 - jepsen.util 11622 :ok :read [4 30] INFO [2019-12-17 12:37:06,256] jepsen worker 88 - jepsen.util 11608 :ok :read [29 31] INFO [2019-12-17 12:37:06,256] jepsen worker 12 - jepsen.util 11622 :invoke :write [4 37] INFO [2019-12-17 12:37:06,256] jepsen worker 88 - jepsen.util 11608 :invoke :cas [29 [15 25]] INFO [2019-12-17 12:37:06,259] jepsen worker 45 - jepsen.util 11655 :ok :write [15 28] INFO [2019-12-17 12:37:06,259] jepsen worker 45 - jepsen.util 11655 :invoke :write [15 4] INFO [2019-12-17 12:37:06,281] jepsen worker 4 - jepsen.util 10534 :ok :read [1 6] INFO [2019-12-17 12:37:06,282] jepsen worker 4 - jepsen.util 10534 :invoke :cas [1 [19 39]] INFO [2019-12-17 12:37:06,302] jepsen worker 27 - jepsen.util 10647 :ok :write [9 16] INFO [2019-12-17 12:37:06,302] jepsen worker 27 - jepsen.util 10647 :invoke :read [9 nil] INFO [2019-12-17 12:37:06,317] jepsen worker 27 - jepsen.util 10647 :ok :read [9 16] INFO [2019-12-17 12:37:06,317] jepsen worker 27 - jepsen.util 10647 :invoke :write [9 33] INFO [2019-12-17 12:37:06,324] jepsen worker 52 - jepsen.util 10852 :fail :cas [17 [20 48]] :ValueNotSwapFrom INFO [2019-12-17 12:37:06,324] jepsen worker 52 - jepsen.util 10852 :invoke :write [17 16] INFO [2019-12-17 12:37:06,335] jepsen worker 3 - jepsen.util 10893 :ok :write [1 24] INFO [2019-12-17 12:37:06,335] jepsen worker 3 - jepsen.util 10893 :invoke :read [1 nil] INFO [2019-12-17 12:37:06,375] jepsen worker 14 - jepsen.util 11174 :fail :cas [4 [49 12]] :ValueNotSwapFrom INFO [2019-12-17 12:37:06,375] jepsen worker 14 - jepsen.util 11174 :invoke :cas [4 [44 43]] INFO [2019-12-17 12:37:06,381] jepsen worker 27 - jepsen.util 10647 :ok :write [9 33] INFO [2019-12-17 12:37:06,381] jepsen worker 27 - jepsen.util 10647 :invoke :read [9 nil] INFO [2019-12-17 12:37:06,400] jepsen worker 27 - jepsen.util 10647 :ok :read [9 33] INFO [2019-12-17 12:37:06,400] jepsen worker 27 - jepsen.util 10647 :invoke :write [9 36] INFO [2019-12-17 12:37:06,407] jepsen worker 71 - jepsen.util 12311 :ok :read [23 13] INFO [2019-12-17 12:37:06,407] jepsen worker 71 - jepsen.util 12311 :invoke :write [23 8] INFO [2019-12-17 12:37:06,430] jepsen worker 51 - jepsen.util 10761 :ok :write [17 44] INFO [2019-12-17 12:37:06,430] jepsen worker 51 - jepsen.util 10761 :invoke :cas [17 [18 30]] INFO [2019-12-17 12:37:06,449] jepsen worker 32 - jepsen.util 11822 :ok :write [10 2] INFO [2019-12-17 12:37:06,451] jepsen worker 32 - jepsen.util 11822 :invoke :read [10 nil] INFO [2019-12-17 12:37:06,459] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66387,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:37:06,459] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66387,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:37:06,459] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66390,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:37:06,459] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66390,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:37:06,459] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66391,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:37:06,459] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66391,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:37:06,467] jepsen worker 27 - jepsen.util 10647 :ok :write [9 36] INFO [2019-12-17 12:37:06,467] jepsen worker 27 - jepsen.util 10647 :invoke :read [9 nil] INFO [2019-12-17 12:37:06,469] jepsen worker 70 - jepsen.util 12040 :ok :write [23 32] INFO [2019-12-17 12:37:06,469] jepsen worker 70 - jepsen.util 12040 :invoke :write [23 20] INFO [2019-12-17 12:37:06,488] jepsen worker 27 - jepsen.util 10647 :ok :read [9 36] INFO [2019-12-17 12:37:06,488] jepsen worker 27 - jepsen.util 10647 :invoke :cas [9 [6 19]] INFO [2019-12-17 12:37:06,517] jepsen worker 27 - jepsen.util 10647 :fail :cas [9 [6 19]] :ValueNotSwapFrom INFO [2019-12-17 12:37:06,517] jepsen worker 27 - jepsen.util 10647 :invoke :read [9 nil] INFO [2019-12-17 12:37:06,531] jepsen worker 54 - jepsen.util 11934 :ok :read [18 4] INFO [2019-12-17 12:37:06,531] jepsen worker 54 - jepsen.util 11934 :invoke :read [18 nil] INFO [2019-12-17 12:37:06,540] jepsen worker 27 - jepsen.util 10647 :ok :read [9 36] INFO [2019-12-17 12:37:06,540] jepsen worker 27 - jepsen.util 10647 :invoke :cas [9 [43 9]] INFO [2019-12-17 12:37:06,552] jepsen worker 5 - jepsen.util 11255 :ok :write [1 35] INFO [2019-12-17 12:37:06,552] jepsen worker 5 - jepsen.util 11255 :invoke :write [1 27] INFO [2019-12-17 12:37:06,561] jepsen worker 27 - jepsen.util 10647 :fail :cas [9 [43 9]] :ValueNotSwapFrom INFO [2019-12-17 12:37:06,561] jepsen worker 27 - jepsen.util 10647 :invoke :read [9 nil] INFO [2019-12-17 12:37:06,585] jepsen worker 27 - jepsen.util 10647 :ok :read [9 36] INFO [2019-12-17 12:37:06,585] jepsen worker 27 - jepsen.util 10647 :invoke :write [9 11] INFO [2019-12-17 12:37:06,603] jepsen worker 20 - jepsen.util 11090 :ok :write [6 10] INFO [2019-12-17 12:37:06,604] jepsen worker 20 - jepsen.util 11090 :invoke :cas [6 [1 15]] INFO [2019-12-17 12:37:06,611] jepsen worker 14 - jepsen.util 11174 :fail :cas [4 [44 43]] :ValueNotSwapFrom INFO [2019-12-17 12:37:06,611] jepsen worker 14 - jepsen.util 11174 :invoke :read [4 nil] INFO [2019-12-17 12:37:06,672] jepsen worker 72 - jepsen.util 11772 :ok :read [24 29] INFO [2019-12-17 12:37:06,672] jepsen worker 72 - jepsen.util 11772 :invoke :cas [24 [14 43]] INFO [2019-12-17 12:37:06,675] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66405,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:37:06,676] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66405,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:37:06,676] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66409,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:37:06,676] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66409,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:37:06,682] jepsen worker 54 - jepsen.util 11934 :ok :read [18 4] INFO [2019-12-17 12:37:06,683] jepsen worker 54 - jepsen.util 11934 :invoke :read [18 nil] INFO [2019-12-17 12:37:06,691] jepsen worker 27 - jepsen.util 10647 :ok :write [9 11] INFO [2019-12-17 12:37:06,691] jepsen worker 27 - jepsen.util 10647 :invoke :cas [9 [44 48]] INFO [2019-12-17 12:37:06,695] jepsen worker 20 - jepsen.util 11090 :fail :cas [6 [1 15]] :ValueNotSwapFrom INFO [2019-12-17 12:37:06,696] jepsen worker 20 - jepsen.util 11090 :invoke :write [6 16] INFO [2019-12-17 12:37:06,722] jepsen worker 38 - jepsen.util 11828 :ok :write [12 0] INFO [2019-12-17 12:37:06,722] jepsen worker 38 - jepsen.util 11828 :invoke :read [12 nil] INFO [2019-12-17 12:37:06,739] jepsen worker 27 - jepsen.util 10647 :fail :cas [9 [44 48]] :ValueNotSwapFrom INFO [2019-12-17 12:37:06,739] jepsen worker 27 - jepsen.util 10647 :invoke :write [9 49] INFO [2019-12-17 12:37:06,749] jepsen worker 87 - jepsen.util 11067 :ok :write [29 14] INFO [2019-12-17 12:37:06,749] jepsen worker 87 - jepsen.util 11067 :invoke :read [29 nil] INFO [2019-12-17 12:37:06,791] jepsen worker 27 - jepsen.util 10647 :ok :write [9 49] INFO [2019-12-17 12:37:06,791] jepsen worker 27 - jepsen.util 10647 :invoke :read [9 nil] INFO [2019-12-17 12:37:06,806] jepsen worker 27 - jepsen.util 10647 :ok :read [9 49] INFO [2019-12-17 12:37:06,806] jepsen worker 27 - jepsen.util 10647 :invoke :write [9 17] INFO [2019-12-17 12:37:06,824] jepsen worker 45 - jepsen.util 11655 :ok :write [15 4] INFO [2019-12-17 12:37:06,824] jepsen worker 45 - jepsen.util 11655 :invoke :write [15 45] INFO [2019-12-17 12:37:06,825] jepsen worker 54 - jepsen.util 11934 :ok :read [18 4] INFO [2019-12-17 12:37:06,825] jepsen worker 54 - jepsen.util 11934 :invoke :read [18 nil] INFO [2019-12-17 12:37:06,871] jepsen worker 89 - jepsen.util 11519 :fail :cas [29 [27 19]] :ValueNotSwapFrom INFO [2019-12-17 12:37:06,871] jepsen worker 89 - jepsen.util 11519 :invoke :read [29 nil] INFO [2019-12-17 12:37:06,873] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66425,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:37:06,873] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66425,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:37:06,897] jepsen worker 55 - jepsen.util 12115 :ok :write [18 17] INFO [2019-12-17 12:37:06,898] jepsen worker 55 - jepsen.util 12115 :invoke :read [18 nil] INFO [2019-12-17 12:37:06,917] jepsen worker 77 - jepsen.util 16457 :info :write [25 11] :RequestCanceledException INFO [2019-12-17 12:37:06,919] jepsen worker 77 - jepsen.util 16547 :invoke :cas [25 [20 9]] INFO [2019-12-17 12:37:06,920] jepsen worker 27 - jepsen.util 10647 :ok :write [9 17] INFO [2019-12-17 12:37:06,920] jepsen worker 27 - jepsen.util 10647 :invoke :write [9 31] INFO [2019-12-17 12:37:06,926] jepsen worker 73 - jepsen.util 10693 :ok :write [24 14] INFO [2019-12-17 12:37:06,926] jepsen worker 73 - jepsen.util 10693 :invoke :write [24 3] INFO [2019-12-17 12:37:06,927] jepsen worker 88 - jepsen.util 11608 :fail :cas [29 [15 25]] :ValueNotSwapFrom INFO [2019-12-17 12:37:06,928] jepsen worker 88 - jepsen.util 11608 :invoke :write [29 23] INFO [2019-12-17 12:37:06,930] jepsen worker 71 - jepsen.util 12311 :ok :write [23 8] INFO [2019-12-17 12:37:06,931] jepsen worker 71 - jepsen.util 12311 :invoke :cas [23 [27 26]] INFO [2019-12-17 12:37:06,932] jepsen worker 86 - jepsen.util 16286 :info :write [28 48] :RequestCanceledException INFO [2019-12-17 12:37:06,932] jepsen worker 66 - jepsen.util 15546 :info :write [22 0] :RequestCanceledException INFO [2019-12-17 12:37:06,932] jepsen worker 26 - jepsen.util 16406 :info :write [8 19] :RequestCanceledException INFO [2019-12-17 12:37:06,933] jepsen worker 26 - jepsen.util 16496 :invoke :read [8 nil] INFO [2019-12-17 12:37:06,933] jepsen worker 66 - jepsen.util 15636 :invoke :read [22 nil] INFO [2019-12-17 12:37:06,933] jepsen worker 86 - jepsen.util 16376 :invoke :cas [28 [25 28]] INFO [2019-12-17 12:37:06,947] jepsen worker 17 - jepsen.util 15677 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:37:06,947] jepsen worker 7 - jepsen.util 15757 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:37:06,947] jepsen worker 17 - jepsen.util 15677 :invoke :cas [5 [15 29]] INFO [2019-12-17 12:37:06,947] jepsen worker 7 - jepsen.util 15757 :invoke :write [2 4] INFO [2019-12-17 12:37:06,960] jepsen worker 23 - jepsen.util 15683 :info :cas [7 [39 19]] :RequestCanceledException INFO [2019-12-17 12:37:06,960] jepsen worker 58 - jepsen.util 15988 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:37:06,960] jepsen worker 58 - jepsen.util 15988 :invoke :cas [19 [44 4]] INFO [2019-12-17 12:37:06,961] jepsen worker 23 - jepsen.util 15773 :invoke :cas [7 [21 32]] INFO [2019-12-17 12:37:06,964] jepsen worker 13 - jepsen.util 10723 :ok :write [4 15] INFO [2019-12-17 12:37:06,964] jepsen worker 13 - jepsen.util 10723 :invoke :write [4 11] INFO [2019-12-17 12:37:06,999] jepsen worker 82 - jepsen.util 15832 :info :write [27 37] :RequestCanceledException INFO [2019-12-17 12:37:07,000] jepsen worker 2 - jepsen.util 15212 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:37:07,000] jepsen worker 0 - jepsen.util 16380 :info :cas [0 [41 5]] :RequestCanceledException INFO [2019-12-17 12:37:07,000] jepsen worker 50 - jepsen.util 16160 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:37:07,000] jepsen worker 2 - jepsen.util 15212 :invoke :cas [0 [31 20]] INFO [2019-12-17 12:37:07,000] jepsen worker 50 - jepsen.util 16160 :invoke :read [16 nil] INFO [2019-12-17 12:37:07,001] jepsen worker 82 - jepsen.util 15922 :invoke :write [27 32] INFO [2019-12-17 12:37:07,001] jepsen worker 0 - jepsen.util 16470 :invoke :write [0 36] INFO [2019-12-17 12:37:07,002] jepsen worker 78 - jepsen.util 11418 :fail :cas [26 [31 33]] :ValueNotSwapFrom INFO [2019-12-17 12:37:07,002] jepsen worker 78 - jepsen.util 11418 :invoke :cas [26 [0 8]] INFO [2019-12-17 12:37:07,013] jepsen worker 54 - jepsen.util 11934 :ok :read [18 17] INFO [2019-12-17 12:37:07,013] jepsen worker 54 - jepsen.util 11934 :invoke :write [18 9] INFO [2019-12-17 12:37:07,014] jepsen worker 14 - jepsen.util 11174 :ok :read [4 15] INFO [2019-12-17 12:37:07,014] jepsen worker 14 - jepsen.util 11174 :invoke :cas [4 [11 16]] INFO [2019-12-17 12:37:07,015] jepsen worker 61 - jepsen.util 16801 :info :cas [20 [19 48]] :RequestCanceledException INFO [2019-12-17 12:37:07,015] jepsen worker 31 - jepsen.util 9751 :fail :read [10 nil] :RequestTimeoutException INFO [2019-12-17 12:37:07,015] jepsen worker 85 - jepsen.util 16375 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:37:07,015] jepsen worker 62 - jepsen.util 16262 :info :write [20 22] :RequestCanceledException INFO [2019-12-17 12:37:07,016] jepsen worker 31 - jepsen.util 9751 :invoke :write [10 21] INFO [2019-12-17 12:37:07,016] jepsen worker 85 - jepsen.util 16375 :invoke :write [28 21] INFO [2019-12-17 12:37:07,016] jepsen worker 63 - jepsen.util 10143 :ok :read [21 19] INFO [2019-12-17 12:37:07,016] jepsen worker 63 - jepsen.util 10143 :invoke :read [21 nil] INFO [2019-12-17 12:37:07,016] jepsen worker 61 - jepsen.util 16891 :invoke :cas [20 [48 0]] INFO [2019-12-17 12:37:07,017] jepsen worker 62 - jepsen.util 16352 :invoke :cas [20 [30 25]] INFO [2019-12-17 12:37:07,032] jepsen worker 59 - jepsen.util 14909 :info :write [19 42] :RequestCanceledException INFO [2019-12-17 12:37:07,032] jepsen worker 43 - jepsen.util 15793 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:37:07,032] jepsen worker 48 - jepsen.util 14448 :info :cas [16 [43 2]] :RequestCanceledException INFO [2019-12-17 12:37:07,032] jepsen worker 29 - jepsen.util 9659 :info :cas [9 [5 13]] :RequestCanceledException INFO [2019-12-17 12:37:07,032] jepsen worker 21 - jepsen.util 15141 :info :cas [7 [23 42]] :RequestCanceledException INFO [2019-12-17 12:37:07,032] jepsen worker 8 - jepsen.util 15488 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:37:07,032] jepsen worker 15 - jepsen.util 14505 :info :cas [5 [31 7]] :RequestCanceledException INFO [2019-12-17 12:37:07,032] jepsen worker 28 - jepsen.util 9478 :fail :read [9 nil] :RequestTimeoutException INFO [2019-12-17 12:37:07,032] jepsen worker 47 - jepsen.util 12197 :fail :read [15 nil] :RequestTimeoutException INFO [2019-12-17 12:37:07,032] jepsen worker 65 - jepsen.util 9965 :info :write [21 4] :RequestCanceledException INFO [2019-12-17 12:37:07,032] jepsen worker 25 - jepsen.util 16675 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:37:07,032] jepsen worker 84 - jepsen.util 15384 :info :cas [28 [0 38]] :RequestCanceledException INFO [2019-12-17 12:37:07,032] jepsen worker 43 - jepsen.util 15793 :invoke :cas [14 [8 47]] INFO [2019-12-17 12:37:07,032] jepsen worker 8 - jepsen.util 15488 :invoke :read [2 nil] INFO [2019-12-17 12:37:07,033] jepsen worker 28 - jepsen.util 9478 :invoke :cas [9 [17 49]] INFO [2019-12-17 12:37:07,033] jepsen worker 47 - jepsen.util 12197 :invoke :read [15 nil] INFO [2019-12-17 12:37:07,033] jepsen worker 25 - jepsen.util 16675 :invoke :cas [8 [32 42]] INFO [2019-12-17 12:37:07,033] jepsen worker 29 - jepsen.util 9749 :invoke :read [9 nil] INFO [2019-12-17 12:37:07,033] jepsen worker 48 - jepsen.util 14538 :invoke :read [16 nil] INFO [2019-12-17 12:37:07,033] jepsen worker 21 - jepsen.util 15231 :invoke :cas [7 [31 11]] INFO [2019-12-17 12:37:07,033] jepsen worker 15 - jepsen.util 14595 :invoke :read [5 nil] INFO [2019-12-17 12:37:07,034] jepsen worker 59 - jepsen.util 14999 :invoke :cas [19 [5 4]] INFO [2019-12-17 12:37:07,034] jepsen worker 65 - jepsen.util 10055 :invoke :write [21 11] INFO [2019-12-17 12:37:07,034] jepsen worker 84 - jepsen.util 15474 :invoke :cas [28 [25 18]] INFO [2019-12-17 12:37:07,047] jepsen worker 67 - jepsen.util 15727 :info :write [22 47] :RequestCanceledException INFO [2019-12-17 12:37:07,047] jepsen worker 36 - jepsen.util 12186 :fail :read [12 nil] :RequestTimeoutException INFO [2019-12-17 12:37:07,047] jepsen worker 36 - jepsen.util 12186 :invoke :write [12 37] INFO [2019-12-17 12:37:07,048] jepsen worker 67 - jepsen.util 15817 :invoke :cas [22 [37 35]] INFO [2019-12-17 12:37:07,064] jepsen worker 74 - jepsen.util 11144 :info :cas [24 [38 8]] :RequestCanceledException INFO [2019-12-17 12:37:07,065] jepsen worker 74 - jepsen.util 11234 :invoke :write [24 48] INFO [2019-12-17 12:37:07,071] jepsen worker 51 - jepsen.util 10761 :fail :cas [17 [18 30]] :ValueNotSwapFrom INFO [2019-12-17 12:37:07,071] jepsen worker 51 - jepsen.util 10761 :invoke :write [17 47] INFO [2019-12-17 12:37:07,073] jepsen worker 32 - jepsen.util 11822 :ok :read [10 2] INFO [2019-12-17 12:37:07,073] jepsen worker 32 - jepsen.util 11822 :invoke :write [10 17] INFO [2019-12-17 12:37:07,090] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66446,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:37:07,090] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66446,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:37:07,090] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66449,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:37:07,090] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66449,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:37:07,090] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66450,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:37:07,090] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66450,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:37:07,105] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:37:07,105] jepsen nemesis - couchbase.util Rebalance check count: 523 INFO [2019-12-17 12:37:07,105] jepsen nemesis - couchbase.util Rebalance stuck count: 504 INFO [2019-12-17 12:37:07,119] jepsen worker 38 - jepsen.util 11828 :ok :read [12 0] INFO [2019-12-17 12:37:07,119] jepsen worker 38 - jepsen.util 11828 :invoke :write [12 16] INFO [2019-12-17 12:37:07,125] jepsen worker 27 - jepsen.util 10647 :ok :write [9 31] INFO [2019-12-17 12:37:07,126] jepsen worker 27 - jepsen.util 10647 :invoke :cas [9 [49 36]] INFO [2019-12-17 12:37:07,133] jepsen worker 19 - jepsen.util 11629 :info :write [6 29] :RequestCanceledException INFO [2019-12-17 12:37:07,134] jepsen worker 19 - jepsen.util 11719 :invoke :cas [6 [45 28]] INFO [2019-12-17 12:37:07,145] jepsen worker 56 - jepsen.util 12296 :fail :cas [18 [1 25]] :ValueNotSwapFrom INFO [2019-12-17 12:37:07,146] jepsen worker 56 - jepsen.util 12296 :invoke :write [18 7] INFO [2019-12-17 12:37:07,152] jepsen worker 28 - jepsen.util 9478 :fail :cas [9 [17 49]] :ValueNotSwapFrom INFO [2019-12-17 12:37:07,153] jepsen worker 28 - jepsen.util 9478 :invoke :cas [9 [8 24]] INFO [2019-12-17 12:37:07,160] jepsen worker 46 - jepsen.util 11206 :info :write [15 47] :RequestCanceledException INFO [2019-12-17 12:37:07,160] jepsen worker 11 - jepsen.util 11081 :info :write [3 19] :RequestCanceledException INFO [2019-12-17 12:37:07,161] jepsen worker 46 - jepsen.util 11296 :invoke :cas [15 [37 22]] INFO [2019-12-17 12:37:07,161] jepsen worker 11 - jepsen.util 11171 :invoke :read [3 nil] INFO [2019-12-17 12:37:07,181] jepsen worker 55 - jepsen.util 12115 :ok :read [18 17] INFO [2019-12-17 12:37:07,181] jepsen worker 55 - jepsen.util 12115 :invoke :read [18 nil] INFO [2019-12-17 12:37:07,213] jepsen worker 47 - jepsen.util 12197 :ok :read [15 4] INFO [2019-12-17 12:37:07,213] jepsen worker 47 - jepsen.util 12197 :invoke :cas [15 [34 23]] INFO [2019-12-17 12:37:07,233] jepsen worker 30 - jepsen.util 10560 :info :write [10 40] :RequestCanceledException INFO [2019-12-17 12:37:07,234] jepsen worker 30 - jepsen.util 10650 :invoke :cas [10 [39 3]] INFO [2019-12-17 12:37:07,261] jepsen worker 11 - jepsen.util 11171 :ok :read [3 6] INFO [2019-12-17 12:37:07,261] jepsen worker 11 - jepsen.util 11171 :invoke :write [3 47] INFO [2019-12-17 12:37:07,294] jepsen worker 3 - jepsen.util 10893 :ok :read [1 35] INFO [2019-12-17 12:37:07,294] jepsen worker 3 - jepsen.util 10893 :invoke :write [1 23] INFO [2019-12-17 12:37:07,308] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66486,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:37:07,309] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66486,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:37:07,309] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66487,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:37:07,309] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66487,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:37:07,309] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66488,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:37:07,309] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66488,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:37:07,320] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66496,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:37:07,320] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66496,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:37:07,320] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66498,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:37:07,320] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66498,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:37:07,327] jepsen worker 78 - jepsen.util 11418 :fail :cas [26 [0 8]] :ValueNotSwapFrom INFO [2019-12-17 12:37:07,327] jepsen worker 78 - jepsen.util 11418 :invoke :cas [26 [49 20]] INFO [2019-12-17 12:37:07,359] jepsen worker 89 - jepsen.util 11519 :ok :read [29 14] INFO [2019-12-17 12:37:07,359] jepsen worker 89 - jepsen.util 11519 :invoke :write [29 4] INFO [2019-12-17 12:37:07,370] jepsen worker 70 - jepsen.util 12040 :ok :write [23 20] INFO [2019-12-17 12:37:07,370] jepsen worker 70 - jepsen.util 12040 :invoke :write [23 10] INFO [2019-12-17 12:37:07,424] jepsen worker 80 - jepsen.util 11150 :ok :read [26 23] INFO [2019-12-17 12:37:07,424] jepsen worker 80 - jepsen.util 11150 :invoke :cas [26 [37 39]] INFO [2019-12-17 12:37:07,434] jepsen worker 31 - jepsen.util 9751 :ok :write [10 21] INFO [2019-12-17 12:37:07,434] jepsen worker 31 - jepsen.util 9751 :invoke :write [10 27] INFO [2019-12-17 12:37:07,434] jepsen worker 10 - jepsen.util 11080 :ok :cas [3 [6 34]] INFO [2019-12-17 12:37:07,435] jepsen worker 10 - jepsen.util 11080 :invoke :read [3 nil] INFO [2019-12-17 12:37:07,439] jepsen worker 45 - jepsen.util 11655 :ok :write [15 45] INFO [2019-12-17 12:37:07,439] jepsen worker 45 - jepsen.util 11655 :invoke :read [15 nil] INFO [2019-12-17 12:37:07,453] jepsen worker 69 - jepsen.util 11679 :ok :read [23 20] INFO [2019-12-17 12:37:07,453] jepsen worker 69 - jepsen.util 11679 :invoke :cas [23 [9 25]] INFO [2019-12-17 12:37:07,490] jepsen worker 14 - jepsen.util 11174 :fail :cas [4 [11 16]] :ValueNotSwapFrom INFO [2019-12-17 12:37:07,490] jepsen worker 14 - jepsen.util 11174 :invoke :cas [4 [0 22]] INFO [2019-12-17 12:37:07,493] jepsen worker 45 - jepsen.util 11655 :ok :read [15 45] INFO [2019-12-17 12:37:07,493] jepsen worker 45 - jepsen.util 11655 :invoke :read [15 nil] INFO [2019-12-17 12:37:07,523] jepsen worker 45 - jepsen.util 11655 :ok :read [15 45] INFO [2019-12-17 12:37:07,523] jepsen worker 45 - jepsen.util 11655 :invoke :read [15 nil] INFO [2019-12-17 12:37:07,525] jepsen worker 46 - jepsen.util 11296 :fail :cas [15 [37 22]] :ValueNotSwapFrom INFO [2019-12-17 12:37:07,525] jepsen worker 46 - jepsen.util 11296 :invoke :read [15 nil] INFO [2019-12-17 12:37:07,541] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66508,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:37:07,541] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66508,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:37:07,543] jepsen worker 45 - jepsen.util 11655 :ok :read [15 45] INFO [2019-12-17 12:37:07,543] jepsen worker 45 - jepsen.util 11655 :invoke :write [15 30] INFO [2019-12-17 12:37:07,543] jepsen worker 71 - jepsen.util 12311 :fail :cas [23 [27 26]] :ValueNotSwapFrom INFO [2019-12-17 12:37:07,543] jepsen worker 71 - jepsen.util 12311 :invoke :read [23 nil] INFO [2019-12-17 12:37:07,545] jepsen worker 4 - jepsen.util 10534 :fail :cas [1 [19 39]] :ValueNotSwapFrom INFO [2019-12-17 12:37:07,545] jepsen worker 4 - jepsen.util 10534 :invoke :read [1 nil] INFO [2019-12-17 12:37:07,564] jepsen worker 64 - jepsen.util 8794 :fail :read [21 nil] :RequestTimeoutException INFO [2019-12-17 12:37:07,564] jepsen worker 64 - jepsen.util 8794 :invoke :cas [21 [32 10]] INFO [2019-12-17 12:37:07,606] jepsen worker 54 - jepsen.util 11934 :ok :write [18 9] INFO [2019-12-17 12:37:07,606] jepsen worker 54 - jepsen.util 11934 :invoke :cas [18 [26 37]] INFO [2019-12-17 12:37:07,606] jepsen worker 73 - jepsen.util 10693 :ok :write [24 3] INFO [2019-12-17 12:37:07,607] jepsen worker 73 - jepsen.util 10693 :invoke :write [24 0] INFO [2019-12-17 12:37:07,609] jepsen worker 80 - jepsen.util 11150 :fail :cas [26 [37 39]] :ValueNotSwapFrom INFO [2019-12-17 12:37:07,609] jepsen worker 80 - jepsen.util 11150 :invoke :read [26 nil] INFO [2019-12-17 12:37:07,612] jepsen worker 55 - jepsen.util 12115 :ok :read [18 17] INFO [2019-12-17 12:37:07,613] jepsen worker 55 - jepsen.util 12115 :invoke :read [18 nil] INFO [2019-12-17 12:37:07,632] jepsen worker 16 - jepsen.util 17386 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:37:07,632] jepsen worker 16 - jepsen.util 17386 :invoke :cas [5 [19 28]] INFO [2019-12-17 12:37:07,637] jepsen worker 19 - jepsen.util 11719 :fail :cas [6 [45 28]] :ValueNotSwapFrom INFO [2019-12-17 12:37:07,637] jepsen worker 19 - jepsen.util 11719 :invoke :cas [6 [28 4]] INFO [2019-12-17 12:37:07,647] jepsen worker 81 - jepsen.util 15291 :info :write [27 21] :RequestCanceledException INFO [2019-12-17 12:37:07,649] jepsen worker 81 - jepsen.util 15381 :invoke :read [27 nil] INFO [2019-12-17 12:37:07,650] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66533,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:37:07,651] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66533,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:37:07,651] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66535,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:37:07,651] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66535,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:37:07,664] jepsen worker 44 - jepsen.util 16244 :info :cas [14 [13 37]] :RequestCanceledException INFO [2019-12-17 12:37:07,665] jepsen worker 44 - jepsen.util 16334 :invoke :read [14 nil] INFO [2019-12-17 12:37:07,693] jepsen worker 20 - jepsen.util 11090 :ok :write [6 16] INFO [2019-12-17 12:37:07,694] jepsen worker 20 - jepsen.util 11090 :invoke :read [6 nil] INFO [2019-12-17 12:37:07,696] jepsen worker 35 - jepsen.util 15605 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:37:07,696] jepsen worker 35 - jepsen.util 15605 :invoke :write [11 37] INFO [2019-12-17 12:37:07,697] jepsen worker 31 - jepsen.util 9751 :ok :write [10 27] INFO [2019-12-17 12:37:07,697] jepsen worker 31 - jepsen.util 9751 :invoke :read [10 nil] INFO [2019-12-17 12:37:07,723] jepsen worker 29 - jepsen.util 9749 :ok :read [9 31] INFO [2019-12-17 12:37:07,724] jepsen worker 29 - jepsen.util 9749 :invoke :read [9 nil] INFO [2019-12-17 12:37:07,724] jepsen worker 46 - jepsen.util 11296 :ok :read [15 45] INFO [2019-12-17 12:37:07,724] jepsen worker 46 - jepsen.util 11296 :invoke :write [15 28] INFO [2019-12-17 12:37:07,731] jepsen worker 68 - jepsen.util 15008 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:37:07,731] jepsen worker 6 - jepsen.util 16116 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:37:07,731] jepsen worker 68 - jepsen.util 15008 :invoke :write [22 43] INFO [2019-12-17 12:37:07,732] jepsen worker 6 - jepsen.util 16116 :invoke :write [2 24] INFO [2019-12-17 12:37:07,742] jepsen worker 31 - jepsen.util 9751 :ok :read [10 27] INFO [2019-12-17 12:37:07,742] jepsen worker 31 - jepsen.util 9751 :invoke :write [10 10] INFO [2019-12-17 12:37:07,749] jepsen worker 30 - jepsen.util 10650 :fail :cas [10 [39 3]] :ValueNotSwapFrom INFO [2019-12-17 12:37:07,749] jepsen worker 30 - jepsen.util 10650 :invoke :read [10 nil] INFO [2019-12-17 12:37:07,761] jepsen worker 45 - jepsen.util 11655 :ok :write [15 30] INFO [2019-12-17 12:37:07,761] jepsen worker 45 - jepsen.util 11655 :invoke :cas [15 [23 42]] INFO [2019-12-17 12:37:07,764] jepsen worker 10 - jepsen.util 11080 :ok :read [3 34] INFO [2019-12-17 12:37:07,764] jepsen worker 10 - jepsen.util 11080 :invoke :read [3 nil] INFO [2019-12-17 12:37:07,778] jepsen worker 53 - jepsen.util 10943 :ok :read [17 44] INFO [2019-12-17 12:37:07,778] jepsen worker 53 - jepsen.util 10943 :invoke :write [17 42] INFO [2019-12-17 12:37:07,806] jepsen worker 20 - jepsen.util 11090 :ok :read [6 16] INFO [2019-12-17 12:37:07,806] jepsen worker 20 - jepsen.util 11090 :invoke :cas [6 [8 47]] INFO [2019-12-17 12:37:07,812] jepsen worker 45 - jepsen.util 11655 :fail :cas [15 [23 42]] :ValueNotSwapFrom INFO [2019-12-17 12:37:07,812] jepsen worker 45 - jepsen.util 11655 :invoke :cas [15 [26 0]] INFO [2019-12-17 12:37:07,829] jepsen worker 45 - jepsen.util 11655 :fail :cas [15 [26 0]] :ValueNotSwapFrom INFO [2019-12-17 12:37:07,829] jepsen worker 45 - jepsen.util 11655 :invoke :cas [15 [42 38]] INFO [2019-12-17 12:37:07,833] jepsen worker 37 - jepsen.util 11017 :info :write [12 24] :RequestCanceledException INFO [2019-12-17 12:37:07,834] jepsen worker 37 - jepsen.util 11107 :invoke :cas [12 [30 23]] INFO [2019-12-17 12:37:07,847] jepsen worker 69 - jepsen.util 11679 :fail :cas [23 [9 25]] :ValueNotSwapFrom INFO [2019-12-17 12:37:07,847] jepsen worker 69 - jepsen.util 11679 :invoke :write [23 31] INFO [2019-12-17 12:37:07,847] jepsen worker 75 - jepsen.util 15195 :info :write [25 23] :RequestCanceledException INFO [2019-12-17 12:37:07,848] jepsen worker 75 - jepsen.util 15285 :invoke :cas [25 [34 27]] INFO [2019-12-17 12:37:07,849] jepsen worker 29 - jepsen.util 9749 :ok :read [9 31] INFO [2019-12-17 12:37:07,849] jepsen worker 29 - jepsen.util 9749 :invoke :write [9 33] INFO [2019-12-17 12:37:07,860] jepsen worker 45 - jepsen.util 11655 :fail :cas [15 [42 38]] :ValueNotSwapFrom INFO [2019-12-17 12:37:07,862] jepsen worker 45 - jepsen.util 11655 :invoke :cas [15 [25 14]] INFO [2019-12-17 12:37:07,863] jepsen worker 30 - jepsen.util 10650 :ok :read [10 27] INFO [2019-12-17 12:37:07,864] jepsen worker 30 - jepsen.util 10650 :invoke :read [10 nil] INFO [2019-12-17 12:37:07,864] jepsen worker 41 - jepsen.util 16691 :info :cas [13 [45 1]] :RequestCanceledException INFO [2019-12-17 12:37:07,865] jepsen worker 41 - jepsen.util 16781 :invoke :write [13 9] INFO [2019-12-17 12:37:07,888] jepsen worker 14 - jepsen.util 11174 :fail :cas [4 [0 22]] :ValueNotSwapFrom INFO [2019-12-17 12:37:07,889] jepsen worker 14 - jepsen.util 11174 :invoke :cas [4 [17 9]] INFO [2019-12-17 12:37:07,891] jepsen worker 54 - jepsen.util 11934 :fail :cas [18 [26 37]] :ValueNotSwapFrom INFO [2019-12-17 12:37:07,891] jepsen worker 54 - jepsen.util 11934 :invoke :write [18 22] INFO [2019-12-17 12:37:07,891] jepsen worker 10 - jepsen.util 11080 :ok :read [3 34] INFO [2019-12-17 12:37:07,891] jepsen worker 10 - jepsen.util 11080 :invoke :cas [3 [29 37]] INFO [2019-12-17 12:37:07,907] jepsen worker 20 - jepsen.util 11090 :fail :cas [6 [8 47]] :ValueNotSwapFrom INFO [2019-12-17 12:37:07,907] jepsen worker 20 - jepsen.util 11090 :invoke :write [6 24] INFO [2019-12-17 12:37:07,916] jepsen worker 45 - jepsen.util 11655 :fail :cas [15 [25 14]] :ValueNotSwapFrom INFO [2019-12-17 12:37:07,916] jepsen worker 45 - jepsen.util 11655 :invoke :cas [15 [32 9]] INFO [2019-12-17 12:37:07,931] jepsen worker 45 - jepsen.util 11655 :fail :cas [15 [32 9]] :ValueNotSwapFrom INFO [2019-12-17 12:37:07,931] jepsen worker 45 - jepsen.util 11655 :invoke :read [15 nil] INFO [2019-12-17 12:37:07,947] jepsen worker 9 - jepsen.util 12069 :info :write [3 48] :RequestCanceledException INFO [2019-12-17 12:37:07,948] jepsen worker 9 - jepsen.util 12159 :invoke :cas [3 [19 4]] INFO [2019-12-17 12:37:07,948] jepsen worker 78 - jepsen.util 11418 :fail :cas [26 [49 20]] :ValueNotSwapFrom INFO [2019-12-17 12:37:07,948] jepsen worker 78 - jepsen.util 11418 :invoke :read [26 nil] INFO [2019-12-17 12:37:07,957] jepsen worker 45 - jepsen.util 11655 :ok :read [15 30] INFO [2019-12-17 12:37:07,957] jepsen worker 45 - jepsen.util 11655 :invoke :cas [15 [7 22]] INFO [2019-12-17 12:37:07,959] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66560,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:37:07,959] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66560,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:37:07,959] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66563,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:37:07,960] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66563,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:37:07,960] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66564,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:37:07,960] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66564,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:37:07,960] jepsen worker 71 - jepsen.util 12311 :ok :read [23 20] INFO [2019-12-17 12:37:07,960] jepsen worker 71 - jepsen.util 12311 :invoke :read [23 nil] INFO [2019-12-17 12:37:07,964] jepsen worker 40 - jepsen.util 16240 :info :cas [13 [3 12]] :RequestCanceledException INFO [2019-12-17 12:37:07,965] jepsen worker 40 - jepsen.util 16330 :invoke :read [13 nil] INFO [2019-12-17 12:37:07,977] jepsen worker 29 - jepsen.util 9749 :ok :write [9 33] INFO [2019-12-17 12:37:07,977] jepsen worker 29 - jepsen.util 9749 :invoke :write [9 15] INFO [2019-12-17 12:37:07,979] jepsen worker 45 - jepsen.util 11655 :fail :cas [15 [7 22]] :ValueNotSwapFrom INFO [2019-12-17 12:37:07,979] jepsen worker 45 - jepsen.util 11655 :invoke :read [15 nil] INFO [2019-12-17 12:37:08,000] jepsen worker 60 - jepsen.util 16890 :info :write [20 5] :RequestCanceledException INFO [2019-12-17 12:37:08,000] jepsen worker 24 - jepsen.util 14334 :info :cas [8 [8 10]] :RequestCanceledException INFO [2019-12-17 12:37:08,001] jepsen worker 24 - jepsen.util 14424 :invoke :cas [8 [8 36]] INFO [2019-12-17 12:37:08,002] jepsen worker 60 - jepsen.util 16980 :invoke :write [20 5] INFO [2019-12-17 12:37:08,017] jepsen worker 79 - jepsen.util 11599 :info :write [26 6] :RequestCanceledException INFO [2019-12-17 12:37:08,017] jepsen worker 22 - jepsen.util 15682 :info :cas [7 [7 28]] :RequestCanceledException INFO [2019-12-17 12:37:08,018] jepsen worker 79 - jepsen.util 11689 :invoke :write [26 21] INFO [2019-12-17 12:37:08,018] jepsen worker 22 - jepsen.util 15772 :invoke :write [7 5] INFO [2019-12-17 12:37:08,025] jepsen worker 39 - jepsen.util 15159 :info :cas [13 [32 40]] :RequestCanceledException INFO [2019-12-17 12:37:08,025] jepsen worker 76 - jepsen.util 16006 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:37:08,025] jepsen worker 1 - jepsen.util 15661 :info :cas [0 [23 41]] :RequestCanceledException INFO [2019-12-17 12:37:08,025] jepsen worker 57 - jepsen.util 15447 :info :write [19 46] :RequestCanceledException INFO [2019-12-17 12:37:08,025] jepsen worker 76 - jepsen.util 16006 :invoke :read [25 nil] INFO [2019-12-17 12:37:08,027] jepsen worker 39 - jepsen.util 15249 :invoke :read [13 nil] INFO [2019-12-17 12:37:08,027] jepsen worker 1 - jepsen.util 15751 :invoke :cas [0 [25 37]] INFO [2019-12-17 12:37:08,028] jepsen worker 57 - jepsen.util 15537 :invoke :write [19 27] INFO [2019-12-17 12:37:08,033] jepsen worker 10 - jepsen.util 11080 :fail :cas [3 [29 37]] :ValueNotSwapFrom INFO [2019-12-17 12:37:08,033] jepsen worker 10 - jepsen.util 11080 :invoke :cas [3 [26 11]] INFO [2019-12-17 12:37:08,034] jepsen worker 73 - jepsen.util 10693 :ok :write [24 0] INFO [2019-12-17 12:37:08,034] jepsen worker 73 - jepsen.util 10693 :invoke :write [24 15] INFO [2019-12-17 12:37:08,036] jepsen worker 46 - jepsen.util 11296 :ok :write [15 28] INFO [2019-12-17 12:37:08,036] jepsen worker 46 - jepsen.util 11296 :invoke :cas [15 [4 4]] INFO [2019-12-17 12:37:08,040] jepsen worker 45 - jepsen.util 11655 :ok :read [15 30] INFO [2019-12-17 12:37:08,040] jepsen worker 45 - jepsen.util 11655 :invoke :write [15 8] INFO [2019-12-17 12:37:08,046] jepsen worker 83 - jepsen.util 15833 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:37:08,046] jepsen worker 83 - jepsen.util 15833 :invoke :write [27 48] INFO [2019-12-17 12:37:08,064] jepsen worker 34 - jepsen.util 16144 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:37:08,064] jepsen worker 42 - jepsen.util 16602 :info :write [14 43] :RequestCanceledException INFO [2019-12-17 12:37:08,064] jepsen worker 34 - jepsen.util 16144 :invoke :cas [11 [13 46]] INFO [2019-12-17 12:37:08,065] jepsen worker 42 - jepsen.util 16692 :invoke :cas [14 [43 22]] INFO [2019-12-17 12:37:08,069] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66580,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:37:08,069] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66580,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:37:08,069] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66583,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:37:08,069] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66583,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:37:08,101] jepsen worker 46 - jepsen.util 11296 :fail :cas [15 [4 4]] :ValueNotSwapFrom INFO [2019-12-17 12:37:08,102] jepsen worker 46 - jepsen.util 11296 :invoke :write [15 22] INFO [2019-12-17 12:37:08,117] jepsen worker 49 - jepsen.util 15709 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:37:08,117] jepsen worker 49 - jepsen.util 15709 :invoke :write [16 21] INFO [2019-12-17 12:37:08,134] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:37:08,134] jepsen nemesis - couchbase.util Rebalance check count: 524 INFO [2019-12-17 12:37:08,134] jepsen nemesis - couchbase.util Rebalance stuck count: 505 INFO [2019-12-17 12:37:08,148] jepsen worker 4 - jepsen.util 10534 :ok :read [1 35] INFO [2019-12-17 12:37:08,148] jepsen worker 4 - jepsen.util 10534 :invoke :write [1 40] INFO [2019-12-17 12:37:08,165] jepsen worker 55 - jepsen.util 12115 :ok :read [18 9] INFO [2019-12-17 12:37:08,165] jepsen worker 55 - jepsen.util 12115 :invoke :cas [18 [46 3]] INFO [2019-12-17 12:37:08,218] jepsen worker 18 - jepsen.util 11538 :info :write [6 19] :RequestCanceledException INFO [2019-12-17 12:37:08,219] jepsen worker 18 - jepsen.util 11628 :invoke :write [6 1] INFO [2019-12-17 12:37:08,233] jepsen worker 80 - jepsen.util 11150 :ok :read [26 23] INFO [2019-12-17 12:37:08,233] jepsen worker 80 - jepsen.util 11150 :invoke :write [26 20] INFO [2019-12-17 12:37:08,238] jepsen worker 78 - jepsen.util 11418 :ok :read [26 23] INFO [2019-12-17 12:37:08,238] jepsen worker 78 - jepsen.util 11418 :invoke :write [26 15] INFO [2019-12-17 12:37:08,288] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66604,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:37:08,288] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66604,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:37:08,288] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66606,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:37:08,288] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66606,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:37:08,303] jepsen worker 45 - jepsen.util 11655 :ok :write [15 8] INFO [2019-12-17 12:37:08,303] jepsen worker 73 - jepsen.util 10693 :ok :write [24 15] INFO [2019-12-17 12:37:08,304] jepsen worker 45 - jepsen.util 11655 :invoke :cas [15 [35 25]] INFO [2019-12-17 12:37:08,304] jepsen worker 73 - jepsen.util 10693 :invoke :cas [24 [9 39]] INFO [2019-12-17 12:37:08,305] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66611,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:37:08,305] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66611,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:37:08,306] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66614,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:37:08,306] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66614,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:37:08,306] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66615,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:37:08,306] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66615,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:37:08,313] jepsen worker 20 - jepsen.util 11090 :ok :write [6 24] INFO [2019-12-17 12:37:08,313] jepsen worker 20 - jepsen.util 11090 :invoke :write [6 22] INFO [2019-12-17 12:37:08,333] jepsen worker 33 - jepsen.util 15603 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:37:08,333] jepsen worker 12 - jepsen.util 11622 :info :write [4 37] :RequestCanceledException INFO [2019-12-17 12:37:08,333] jepsen worker 33 - jepsen.util 15603 :invoke :read [11 nil] INFO [2019-12-17 12:37:08,334] jepsen worker 12 - jepsen.util 11712 :invoke :read [4 nil] INFO [2019-12-17 12:37:08,384] jepsen worker 10 - jepsen.util 11080 :fail :cas [3 [26 11]] :ValueNotSwapFrom INFO [2019-12-17 12:37:08,384] jepsen worker 10 - jepsen.util 11080 :invoke :write [3 27] INFO [2019-12-17 12:37:08,396] jepsen worker 45 - jepsen.util 11655 :fail :cas [15 [35 25]] :ValueNotSwapFrom INFO [2019-12-17 12:37:08,397] jepsen worker 45 - jepsen.util 11655 :invoke :cas [15 [25 9]] INFO [2019-12-17 12:37:08,400] jepsen worker 52 - jepsen.util 10852 :info :write [17 16] :RequestCanceledException INFO [2019-12-17 12:37:08,402] jepsen worker 52 - jepsen.util 10942 :invoke :write [17 37] INFO [2019-12-17 12:37:08,405] jepsen worker 73 - jepsen.util 10693 :fail :cas [24 [9 39]] :ValueNotSwapFrom INFO [2019-12-17 12:37:08,406] jepsen worker 73 - jepsen.util 10693 :invoke :read [24 nil] INFO [2019-12-17 12:37:08,413] jepsen worker 45 - jepsen.util 11655 :fail :cas [15 [25 9]] :ValueNotSwapFrom INFO [2019-12-17 12:37:08,414] jepsen worker 45 - jepsen.util 11655 :invoke :write [15 35] INFO [2019-12-17 12:37:08,420] jepsen worker 73 - jepsen.util 10693 :ok :read [24 15] INFO [2019-12-17 12:37:08,421] jepsen worker 73 - jepsen.util 10693 :invoke :cas [24 [40 6]] INFO [2019-12-17 12:37:08,447] jepsen worker 73 - jepsen.util 10693 :fail :cas [24 [40 6]] :ValueNotSwapFrom INFO [2019-12-17 12:37:08,447] jepsen worker 73 - jepsen.util 10693 :invoke :read [24 nil] INFO [2019-12-17 12:37:08,476] jepsen worker 20 - jepsen.util 11090 :ok :write [6 22] INFO [2019-12-17 12:37:08,476] jepsen worker 20 - jepsen.util 11090 :invoke :write [6 13] INFO [2019-12-17 12:37:08,483] jepsen worker 73 - jepsen.util 10693 :ok :read [24 15] INFO [2019-12-17 12:37:08,483] jepsen worker 73 - jepsen.util 10693 :invoke :cas [24 [17 44]] INFO [2019-12-17 12:37:08,496] jepsen worker 73 - jepsen.util 10693 :fail :cas [24 [17 44]] :ValueNotSwapFrom INFO [2019-12-17 12:37:08,496] jepsen worker 73 - jepsen.util 10693 :invoke :write [24 25] INFO [2019-12-17 12:37:08,521] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66631,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:37:08,521] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66631,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:37:08,521] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66635,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:37:08,521] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66635,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:37:08,617] jepsen worker 5 - jepsen.util 11255 :info :write [1 27] :RequestCanceledException INFO [2019-12-17 12:37:08,618] jepsen worker 5 - jepsen.util 11345 :invoke :read [1 nil] INFO [2019-12-17 12:37:08,679] jepsen worker 37 - jepsen.util 11107 :fail :cas [12 [30 23]] :ValueNotSwapFrom INFO [2019-12-17 12:37:08,679] jepsen worker 37 - jepsen.util 11107 :invoke :cas [12 [42 46]] INFO [2019-12-17 12:37:08,728] jepsen worker 37 - jepsen.util 11107 :fail :cas [12 [42 46]] :ValueNotSwapFrom INFO [2019-12-17 12:37:08,728] jepsen worker 37 - jepsen.util 11107 :invoke :read [12 nil] INFO [2019-12-17 12:37:08,740] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66641,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:37:08,740] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66641,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:37:08,740] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66645,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:37:08,740] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66645,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:37:08,740] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66646,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:37:08,740] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66646,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:37:08,747] jepsen worker 72 - jepsen.util 11772 :info :cas [24 [14 43]] :RequestCanceledException INFO [2019-12-17 12:37:08,748] jepsen worker 72 - jepsen.util 11862 :invoke :write [24 39] INFO [2019-12-17 12:37:08,768] jepsen worker 37 - jepsen.util 11107 :ok :read [12 0] INFO [2019-12-17 12:37:08,769] jepsen worker 37 - jepsen.util 11107 :invoke :cas [12 [28 26]] INFO [2019-12-17 12:37:08,770] jepsen worker 5 - jepsen.util 11345 :ok :read [1 35] INFO [2019-12-17 12:37:08,770] jepsen worker 5 - jepsen.util 11345 :invoke :write [1 4] INFO [2019-12-17 12:37:08,799] jepsen worker 87 - jepsen.util 11067 :fail :read [29 nil] :RequestTimeoutException INFO [2019-12-17 12:37:08,799] jepsen worker 87 - jepsen.util 11067 :invoke :write [29 19] INFO [2019-12-17 12:37:08,810] jepsen worker 55 - jepsen.util 12115 :fail :cas [18 [46 3]] :ValueNotSwapFrom INFO [2019-12-17 12:37:08,810] jepsen worker 55 - jepsen.util 12115 :invoke :read [18 nil] INFO [2019-12-17 12:37:08,843] jepsen worker 37 - jepsen.util 11107 :fail :cas [12 [28 26]] :ValueNotSwapFrom INFO [2019-12-17 12:37:08,843] jepsen worker 37 - jepsen.util 11107 :invoke :cas [12 [28 11]] INFO [2019-12-17 12:37:08,855] jepsen worker 14 - jepsen.util 11174 :fail :cas [4 [17 9]] :ValueNotSwapFrom INFO [2019-12-17 12:37:08,855] jepsen worker 14 - jepsen.util 11174 :invoke :write [4 49] INFO [2019-12-17 12:37:08,863] jepsen worker 71 - jepsen.util 12311 :ok :read [23 20] INFO [2019-12-17 12:37:08,863] jepsen worker 71 - jepsen.util 12311 :invoke :read [23 nil] INFO [2019-12-17 12:37:08,942] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66658,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:37:08,942] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66658,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:37:08,942] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66661,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:37:08,942] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66661,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:37:08,947] jepsen worker 66 - jepsen.util 15636 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:37:08,947] jepsen worker 66 - jepsen.util 15636 :invoke :write [22 43] INFO [2019-12-17 12:37:08,964] jepsen worker 88 - jepsen.util 11608 :info :write [29 23] :RequestCanceledException INFO [2019-12-17 12:37:08,965] jepsen worker 88 - jepsen.util 11698 :invoke :read [29 nil] INFO [2019-12-17 12:37:08,999] jepsen worker 13 - jepsen.util 10723 :info :write [4 11] :RequestCanceledException INFO [2019-12-17 12:37:08,999] jepsen worker 77 - jepsen.util 16547 :info :cas [25 [20 9]] :RequestCanceledException INFO [2019-12-17 12:37:09,000] jepsen worker 13 - jepsen.util 10813 :invoke :write [4 36] INFO [2019-12-17 12:37:09,000] jepsen worker 77 - jepsen.util 16637 :invoke :read [25 nil] INFO [2019-12-17 12:37:09,007] jepsen worker 12 - jepsen.util 11712 :ok :read [4 15] INFO [2019-12-17 12:37:09,007] jepsen worker 12 - jepsen.util 11712 :invoke :cas [4 [29 3]] INFO [2019-12-17 12:37:09,018] jepsen worker 26 - jepsen.util 16496 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:37:09,018] jepsen worker 26 - jepsen.util 16496 :invoke :write [8 7] INFO [2019-12-17 12:37:09,021] jepsen worker 10 - jepsen.util 11080 :ok :write [3 27] INFO [2019-12-17 12:37:09,021] jepsen worker 10 - jepsen.util 11080 :invoke :read [3 nil] INFO [2019-12-17 12:37:09,021] jepsen worker 45 - jepsen.util 11655 :ok :write [15 35] INFO [2019-12-17 12:37:09,021] jepsen worker 45 - jepsen.util 11655 :invoke :read [15 nil] INFO [2019-12-17 12:37:09,022] jepsen worker 73 - jepsen.util 10693 :ok :write [24 25] INFO [2019-12-17 12:37:09,024] jepsen worker 73 - jepsen.util 10693 :invoke :read [24 nil] INFO [2019-12-17 12:37:09,033] jepsen worker 82 - jepsen.util 15922 :info :write [27 32] :RequestCanceledException INFO [2019-12-17 12:37:09,033] jepsen worker 86 - jepsen.util 16376 :info :cas [28 [25 28]] :RequestCanceledException INFO [2019-12-17 12:37:09,033] jepsen worker 62 - jepsen.util 16352 :info :cas [20 [30 25]] :RequestCanceledException INFO [2019-12-17 12:37:09,034] jepsen worker 82 - jepsen.util 16012 :invoke :cas [27 [43 17]] INFO [2019-12-17 12:37:09,034] jepsen worker 62 - jepsen.util 16442 :invoke :cas [20 [19 41]] INFO [2019-12-17 12:37:09,034] jepsen worker 86 - jepsen.util 16466 :invoke :write [28 38] INFO [2019-12-17 12:37:09,042] jepsen worker 17 - jepsen.util 15677 :info :cas [5 [15 29]] :RequestCanceledException INFO [2019-12-17 12:37:09,042] jepsen worker 7 - jepsen.util 15757 :info :write [2 4] :RequestCanceledException INFO [2019-12-17 12:37:09,042] jepsen worker 84 - jepsen.util 15474 :info :cas [28 [25 18]] :RequestCanceledException INFO [2019-12-17 12:37:09,043] jepsen worker 17 - jepsen.util 15767 :invoke :cas [5 [20 16]] INFO [2019-12-17 12:37:09,043] jepsen worker 84 - jepsen.util 15564 :invoke :write [28 48] INFO [2019-12-17 12:37:09,043] jepsen worker 7 - jepsen.util 15847 :invoke :read [2 nil] INFO [2019-12-17 12:37:09,047] jepsen worker 45 - jepsen.util 11655 :ok :read [15 35] INFO [2019-12-17 12:37:09,047] jepsen worker 45 - jepsen.util 11655 :invoke :cas [15 [23 41]] INFO [2019-12-17 12:37:09,051] jepsen worker 73 - jepsen.util 10693 :ok :read [24 25] INFO [2019-12-17 12:37:09,051] jepsen worker 73 - jepsen.util 10693 :invoke :cas [24 [31 23]] INFO [2019-12-17 12:37:09,051] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66667,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:37:09,052] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66667,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:37:09,052] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66671,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:37:09,052] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66671,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:37:09,064] jepsen worker 67 - jepsen.util 15817 :info :cas [22 [37 35]] :RequestCanceledException INFO [2019-12-17 12:37:09,064] jepsen worker 23 - jepsen.util 15773 :info :cas [7 [21 32]] :RequestCanceledException INFO [2019-12-17 12:37:09,064] jepsen worker 48 - jepsen.util 14538 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:37:09,064] jepsen worker 58 - jepsen.util 15988 :info :cas [19 [44 4]] :RequestCanceledException INFO [2019-12-17 12:37:09,064] jepsen worker 48 - jepsen.util 14538 :invoke :write [16 45] INFO [2019-12-17 12:37:09,065] jepsen worker 23 - jepsen.util 15863 :invoke :write [7 6] INFO [2019-12-17 12:37:09,065] jepsen worker 58 - jepsen.util 16078 :invoke :write [19 30] INFO [2019-12-17 12:37:09,066] jepsen worker 67 - jepsen.util 15907 :invoke :read [22 nil] INFO [2019-12-17 12:37:09,082] jepsen worker 73 - jepsen.util 10693 :fail :cas [24 [31 23]] :ValueNotSwapFrom INFO [2019-12-17 12:37:09,082] jepsen worker 73 - jepsen.util 10693 :invoke :write [24 15] INFO [2019-12-17 12:37:09,093] jepsen worker 45 - jepsen.util 11655 :fail :cas [15 [23 41]] :ValueNotSwapFrom INFO [2019-12-17 12:37:09,093] jepsen worker 45 - jepsen.util 11655 :invoke :read [15 nil] INFO [2019-12-17 12:37:09,097] jepsen worker 2 - jepsen.util 15212 :info :cas [0 [31 20]] :RequestCanceledException INFO [2019-12-17 12:37:09,097] jepsen worker 50 - jepsen.util 16160 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:37:09,097] jepsen worker 0 - jepsen.util 16470 :info :write [0 36] :RequestCanceledException INFO [2019-12-17 12:37:09,097] jepsen worker 63 - jepsen.util 10143 :fail :read [21 nil] :RequestTimeoutException INFO [2019-12-17 12:37:09,097] jepsen worker 65 - jepsen.util 10055 :info :write [21 11] :RequestCanceledException INFO [2019-12-17 12:37:09,097] jepsen worker 32 - jepsen.util 11822 :info :write [10 17] :RequestCanceledException INFO [2019-12-17 12:37:09,097] jepsen worker 50 - jepsen.util 16160 :invoke :read [16 nil] INFO [2019-12-17 12:37:09,098] jepsen worker 63 - jepsen.util 10143 :invoke :read [21 nil] INFO [2019-12-17 12:37:09,098] jepsen worker 0 - jepsen.util 16560 :invoke :read [0 nil] INFO [2019-12-17 12:37:09,099] jepsen worker 65 - jepsen.util 10145 :invoke :write [21 47] INFO [2019-12-17 12:37:09,099] jepsen worker 2 - jepsen.util 15302 :invoke :cas [0 [47 21]] INFO [2019-12-17 12:37:09,099] jepsen worker 32 - jepsen.util 11912 :invoke :write [10 44] INFO [2019-12-17 12:37:09,105] jepsen worker 37 - jepsen.util 11107 :fail :cas [12 [28 11]] :ValueNotSwapFrom INFO [2019-12-17 12:37:09,105] jepsen worker 37 - jepsen.util 11107 :invoke :cas [12 [14 21]] INFO [2019-12-17 12:37:09,118] jepsen worker 61 - jepsen.util 16891 :info :cas [20 [48 0]] :RequestCanceledException INFO [2019-12-17 12:37:09,118] jepsen worker 85 - jepsen.util 16375 :info :write [28 21] :RequestCanceledException INFO [2019-12-17 12:37:09,118] jepsen worker 21 - jepsen.util 15231 :info :cas [7 [31 11]] :RequestCanceledException INFO [2019-12-17 12:37:09,119] jepsen worker 21 - jepsen.util 15321 :invoke :read [7 nil] INFO [2019-12-17 12:37:09,119] jepsen worker 61 - jepsen.util 16981 :invoke :write [20 27] INFO [2019-12-17 12:37:09,120] jepsen worker 85 - jepsen.util 16465 :invoke :read [28 nil] INFO [2019-12-17 12:37:09,133] jepsen worker 25 - jepsen.util 16675 :info :cas [8 [32 42]] :RequestCanceledException INFO [2019-12-17 12:37:09,133] jepsen worker 74 - jepsen.util 11234 :info :write [24 48] :RequestCanceledException INFO [2019-12-17 12:37:09,133] jepsen worker 59 - jepsen.util 14999 :info :cas [19 [5 4]] :RequestCanceledException INFO [2019-12-17 12:37:09,133] jepsen worker 43 - jepsen.util 15793 :info :cas [14 [8 47]] :RequestCanceledException INFO [2019-12-17 12:37:09,133] jepsen worker 15 - jepsen.util 14595 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:37:09,133] jepsen worker 8 - jepsen.util 15488 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:37:09,135] jepsen worker 15 - jepsen.util 14595 :invoke :cas [5 [19 28]] INFO [2019-12-17 12:37:09,135] jepsen worker 89 - jepsen.util 11519 :ok :write [29 4] INFO [2019-12-17 12:37:09,135] jepsen worker 59 - jepsen.util 15089 :invoke :read [19 nil] INFO [2019-12-17 12:37:09,135] jepsen worker 8 - jepsen.util 15488 :invoke :cas [2 [42 15]] INFO [2019-12-17 12:37:09,135] jepsen worker 43 - jepsen.util 15883 :invoke :write [14 3] INFO [2019-12-17 12:37:09,135] jepsen worker 89 - jepsen.util 11519 :invoke :cas [29 [0 47]] INFO [2019-12-17 12:37:09,135] jepsen worker 25 - jepsen.util 16765 :invoke :cas [8 [30 12]] INFO [2019-12-17 12:37:09,135] jepsen worker 74 - jepsen.util 11324 :invoke :cas [24 [36 49]] INFO [2019-12-17 12:37:09,143] jepsen worker 20 - jepsen.util 11090 :ok :write [6 13] INFO [2019-12-17 12:37:09,143] jepsen worker 20 - jepsen.util 11090 :invoke :cas [6 [36 8]] INFO [2019-12-17 12:37:09,147] jepsen worker 36 - jepsen.util 12186 :info :write [12 37] :RequestCanceledException INFO [2019-12-17 12:37:09,148] jepsen worker 36 - jepsen.util 12276 :invoke :cas [12 [39 31]] INFO [2019-12-17 12:37:09,149] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:37:09,149] jepsen nemesis - couchbase.util Rebalance check count: 525 INFO [2019-12-17 12:37:09,149] jepsen nemesis - couchbase.util Rebalance stuck count: 506 INFO [2019-12-17 12:37:09,153] jepsen worker 45 - jepsen.util 11655 :ok :read [15 35] INFO [2019-12-17 12:37:09,153] jepsen worker 45 - jepsen.util 11655 :invoke :write [15 40] INFO [2019-12-17 12:37:09,162] jepsen worker 88 - jepsen.util 11698 :ok :read [29 14] INFO [2019-12-17 12:37:09,162] jepsen worker 88 - jepsen.util 11698 :invoke :cas [29 [7 23]] INFO [2019-12-17 12:37:09,164] jepsen worker 51 - jepsen.util 10761 :info :write [17 47] :RequestCanceledException INFO [2019-12-17 12:37:09,164] jepsen worker 38 - jepsen.util 11828 :info :write [12 16] :RequestCanceledException INFO [2019-12-17 12:37:09,193] jepsen worker 38 - jepsen.util 11918 :invoke :write [12 18] INFO [2019-12-17 12:37:09,193] jepsen worker 51 - jepsen.util 10851 :invoke :cas [17 [39 9]] INFO [2019-12-17 12:37:09,217] jepsen worker 56 - jepsen.util 12296 :info :write [18 7] :RequestCanceledException INFO [2019-12-17 12:37:09,218] jepsen worker 56 - jepsen.util 12386 :invoke :write [18 10] INFO [2019-12-17 12:37:09,226] jepsen worker 74 - jepsen.util 11324 :fail :cas [24 [36 49]] :ValueNotSwapFrom INFO [2019-12-17 12:37:09,226] jepsen worker 10 - jepsen.util 11080 :ok :read [3 27] INFO [2019-12-17 12:37:09,226] jepsen worker 74 - jepsen.util 11324 :invoke :read [24 nil] INFO [2019-12-17 12:37:09,226] jepsen worker 10 - jepsen.util 11080 :invoke :cas [3 [21 32]] INFO [2019-12-17 12:37:09,232] jepsen worker 27 - jepsen.util 10647 :info :cas [9 [49 36]] :RequestCanceledException INFO [2019-12-17 12:37:09,232] jepsen worker 28 - jepsen.util 9478 :info :cas [9 [8 24]] :RequestCanceledException INFO [2019-12-17 12:37:09,232] jepsen worker 47 - jepsen.util 12197 :info :cas [15 [34 23]] :RequestCanceledException INFO [2019-12-17 12:37:09,233] jepsen worker 47 - jepsen.util 12287 :invoke :write [15 8] INFO [2019-12-17 12:37:09,233] jepsen worker 28 - jepsen.util 9568 :invoke :read [9 nil] INFO [2019-12-17 12:37:09,233] jepsen worker 27 - jepsen.util 10737 :invoke :cas [9 [21 17]] INFO [2019-12-17 12:37:09,262] jepsen worker 27 - jepsen.util 10737 :fail :cas [9 [21 17]] :ValueNotSwapFrom INFO [2019-12-17 12:37:09,262] jepsen worker 27 - jepsen.util 10737 :invoke :read [9 nil] INFO [2019-12-17 12:37:09,271] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66716,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:37:09,271] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66716,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:37:09,271] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66717,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:37:09,271] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66717,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:37:09,281] jepsen worker 70 - jepsen.util 12040 :ok :write [23 10] INFO [2019-12-17 12:37:09,281] jepsen worker 70 - jepsen.util 12040 :invoke :write [23 41] INFO [2019-12-17 12:37:09,283] jepsen worker 71 - jepsen.util 12311 :ok :read [23 20] INFO [2019-12-17 12:37:09,283] jepsen worker 71 - jepsen.util 12311 :invoke :read [23 nil] INFO [2019-12-17 12:37:09,290] jepsen worker 73 - jepsen.util 10693 :ok :write [24 15] INFO [2019-12-17 12:37:09,290] jepsen worker 73 - jepsen.util 10693 :invoke :write [24 49] INFO [2019-12-17 12:37:09,301] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66722,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:37:09,301] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66722,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:37:09,301] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66725,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:37:09,302] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66725,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:37:09,302] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66726,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:37:09,302] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66726,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:37:09,317] jepsen worker 3 - jepsen.util 10893 :info :write [1 23] :RequestCanceledException INFO [2019-12-17 12:37:09,318] jepsen worker 3 - jepsen.util 10983 :invoke :cas [1 [6 8]] INFO [2019-12-17 12:37:09,330] jepsen worker 27 - jepsen.util 10737 :ok :read [9 33] INFO [2019-12-17 12:37:09,331] jepsen worker 27 - jepsen.util 10737 :invoke :read [9 nil] INFO [2019-12-17 12:37:09,333] jepsen worker 11 - jepsen.util 11171 :info :write [3 47] :RequestCanceledException INFO [2019-12-17 12:37:09,334] jepsen worker 11 - jepsen.util 11261 :invoke :write [3 49] INFO [2019-12-17 12:37:09,370] jepsen worker 27 - jepsen.util 10737 :ok :read [9 33] INFO [2019-12-17 12:37:09,370] jepsen worker 27 - jepsen.util 10737 :invoke :cas [9 [49 20]] INFO [2019-12-17 12:37:09,384] jepsen worker 65 - jepsen.util 10145 :ok :write [21 47] INFO [2019-12-17 12:37:09,384] jepsen worker 65 - jepsen.util 10145 :invoke :cas [21 [30 1]] INFO [2019-12-17 12:37:09,389] jepsen worker 36 - jepsen.util 12276 :fail :cas [12 [39 31]] :ValueNotSwapFrom INFO [2019-12-17 12:37:09,389] jepsen worker 36 - jepsen.util 12276 :invoke :write [12 7] INFO [2019-12-17 12:37:09,390] jepsen worker 45 - jepsen.util 11655 :ok :write [15 40] INFO [2019-12-17 12:37:09,390] jepsen worker 45 - jepsen.util 11655 :invoke :read [15 nil] INFO [2019-12-17 12:37:09,410] jepsen worker 89 - jepsen.util 11519 :fail :cas [29 [0 47]] :ValueNotSwapFrom INFO [2019-12-17 12:37:09,410] jepsen worker 89 - jepsen.util 11519 :invoke :read [29 nil] INFO [2019-12-17 12:37:09,425] jepsen worker 27 - jepsen.util 10737 :fail :cas [9 [49 20]] :ValueNotSwapFrom INFO [2019-12-17 12:37:09,425] jepsen worker 27 - jepsen.util 10737 :invoke :write [9 33] INFO [2019-12-17 12:37:09,432] jepsen worker 74 - jepsen.util 11324 :ok :read [24 15] INFO [2019-12-17 12:37:09,432] jepsen worker 74 - jepsen.util 11324 :invoke :read [24 nil] INFO [2019-12-17 12:37:09,439] jepsen worker 55 - jepsen.util 12115 :ok :read [18 9] INFO [2019-12-17 12:37:09,439] jepsen worker 55 - jepsen.util 12115 :invoke :read [18 nil] INFO [2019-12-17 12:37:09,467] jepsen worker 28 - jepsen.util 9568 :ok :read [9 33] INFO [2019-12-17 12:37:09,467] jepsen worker 28 - jepsen.util 9568 :invoke :read [9 nil] INFO [2019-12-17 12:37:09,472] jepsen worker 37 - jepsen.util 11107 :fail :cas [12 [14 21]] :ValueNotSwapFrom INFO [2019-12-17 12:37:09,472] jepsen worker 37 - jepsen.util 11107 :invoke :cas [12 [7 43]] INFO [2019-12-17 12:37:09,489] jepsen worker 45 - jepsen.util 11655 :ok :read [15 40] INFO [2019-12-17 12:37:09,489] jepsen worker 45 - jepsen.util 11655 :invoke :write [15 22] INFO [2019-12-17 12:37:09,497] jepsen worker 74 - jepsen.util 11324 :ok :read [24 15] INFO [2019-12-17 12:37:09,497] jepsen worker 74 - jepsen.util 11324 :invoke :write [24 13] INFO [2019-12-17 12:37:09,500] jepsen worker 89 - jepsen.util 11519 :ok :read [29 4] INFO [2019-12-17 12:37:09,500] jepsen worker 89 - jepsen.util 11519 :invoke :read [29 nil] INFO [2019-12-17 12:37:09,521] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66750,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:37:09,521] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66750,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:37:09,521] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66752,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:37:09,521] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66752,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:37:09,522] jepsen worker 28 - jepsen.util 9568 :ok :read [9 33] INFO [2019-12-17 12:37:09,522] jepsen worker 28 - jepsen.util 9568 :invoke :write [9 43] INFO [2019-12-17 12:37:09,535] jepsen worker 27 - jepsen.util 10737 :ok :write [9 33] INFO [2019-12-17 12:37:09,536] jepsen worker 27 - jepsen.util 10737 :invoke :write [9 1] INFO [2019-12-17 12:37:09,566] jepsen worker 73 - jepsen.util 10693 :ok :write [24 49] INFO [2019-12-17 12:37:09,567] jepsen worker 73 - jepsen.util 10693 :invoke :cas [24 [31 29]] INFO [2019-12-17 12:37:09,567] jepsen worker 32 - jepsen.util 11912 :ok :write [10 44] INFO [2019-12-17 12:37:09,567] jepsen worker 32 - jepsen.util 11912 :invoke :write [10 9] INFO [2019-12-17 12:37:09,610] jepsen worker 27 - jepsen.util 10737 :ok :write [9 1] INFO [2019-12-17 12:37:09,610] jepsen worker 27 - jepsen.util 10737 :invoke :write [9 32] INFO [2019-12-17 12:37:09,620] jepsen worker 73 - jepsen.util 10693 :fail :cas [24 [31 29]] :ValueNotSwapFrom INFO [2019-12-17 12:37:09,620] jepsen worker 73 - jepsen.util 10693 :invoke :read [24 nil] INFO [2019-12-17 12:37:09,632] jepsen worker 73 - jepsen.util 10693 :ok :read [24 49] INFO [2019-12-17 12:37:09,632] jepsen worker 73 - jepsen.util 10693 :invoke :cas [24 [18 28]] INFO [2019-12-17 12:37:09,637] jepsen worker 88 - jepsen.util 11698 :fail :cas [29 [7 23]] :ValueNotSwapFrom INFO [2019-12-17 12:37:09,637] jepsen worker 88 - jepsen.util 11698 :invoke :read [29 nil] INFO [2019-12-17 12:37:09,645] jepsen worker 55 - jepsen.util 12115 :ok :read [18 9] INFO [2019-12-17 12:37:09,645] jepsen worker 55 - jepsen.util 12115 :invoke :cas [18 [23 20]] INFO [2019-12-17 12:37:09,647] jepsen worker 51 - jepsen.util 10851 :fail :cas [17 [39 9]] :ValueNotSwapFrom INFO [2019-12-17 12:37:09,647] jepsen worker 51 - jepsen.util 10851 :invoke :write [17 1] INFO [2019-12-17 12:37:09,664] jepsen worker 64 - jepsen.util 8794 :info :cas [21 [32 10]] :RequestCanceledException INFO [2019-12-17 12:37:09,664] jepsen worker 81 - jepsen.util 15381 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:37:09,664] jepsen worker 81 - jepsen.util 15381 :invoke :write [27 30] INFO [2019-12-17 12:37:09,665] jepsen worker 64 - jepsen.util 8884 :invoke :write [21 49] INFO [2019-12-17 12:37:09,680] jepsen worker 12 - jepsen.util 11712 :fail :cas [4 [29 3]] :ValueNotSwapFrom INFO [2019-12-17 12:37:09,681] jepsen worker 12 - jepsen.util 11712 :invoke :read [4 nil] INFO [2019-12-17 12:37:09,682] jepsen worker 89 - jepsen.util 11519 :ok :read [29 4] INFO [2019-12-17 12:37:09,682] jepsen worker 89 - jepsen.util 11519 :invoke :cas [29 [23 14]] INFO [2019-12-17 12:37:09,688] jepsen worker 73 - jepsen.util 10693 :fail :cas [24 [18 28]] :ValueNotSwapFrom INFO [2019-12-17 12:37:09,688] jepsen worker 73 - jepsen.util 10693 :invoke :write [24 31] INFO [2019-12-17 12:37:09,698] jepsen worker 11 - jepsen.util 11261 :ok :write [3 49] INFO [2019-12-17 12:37:09,698] jepsen worker 11 - jepsen.util 11261 :invoke :read [3 nil] INFO [2019-12-17 12:37:09,700] jepsen worker 19 - jepsen.util 11719 :info :cas [6 [28 4]] :RequestCanceledException INFO [2019-12-17 12:37:09,703] jepsen worker 19 - jepsen.util 11809 :invoke :read [6 nil] INFO [2019-12-17 12:37:09,720] jepsen worker 11 - jepsen.util 11261 :ok :read [3 49] INFO [2019-12-17 12:37:09,720] jepsen worker 11 - jepsen.util 11261 :invoke :read [3 nil] INFO [2019-12-17 12:37:09,724] jepsen worker 3 - jepsen.util 10983 :fail :cas [1 [6 8]] :ValueNotSwapFrom INFO [2019-12-17 12:37:09,724] jepsen worker 3 - jepsen.util 10983 :invoke :read [1 nil] INFO [2019-12-17 12:37:09,732] jepsen worker 44 - jepsen.util 16334 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:37:09,732] jepsen worker 16 - jepsen.util 17386 :info :cas [5 [19 28]] :RequestCanceledException INFO [2019-12-17 12:37:09,732] jepsen worker 44 - jepsen.util 16334 :invoke :cas [14 [11 39]] INFO [2019-12-17 12:37:09,733] jepsen worker 16 - jepsen.util 17476 :invoke :cas [5 [18 26]] INFO [2019-12-17 12:37:09,740] jepsen worker 27 - jepsen.util 10737 :ok :write [9 32] INFO [2019-12-17 12:37:09,740] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66769,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:37:09,740] jepsen worker 27 - jepsen.util 10737 :invoke :write [9 18] INFO [2019-12-17 12:37:09,740] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66769,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:37:09,740] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66772,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:37:09,740] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66772,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:37:09,746] jepsen worker 11 - jepsen.util 11261 :ok :read [3 49] INFO [2019-12-17 12:37:09,746] jepsen worker 11 - jepsen.util 11261 :invoke :write [3 33] INFO [2019-12-17 12:37:09,767] jepsen worker 45 - jepsen.util 11655 :ok :write [15 22] INFO [2019-12-17 12:37:09,767] jepsen worker 45 - jepsen.util 11655 :invoke :write [15 30] INFO [2019-12-17 12:37:09,778] jepsen worker 37 - jepsen.util 11107 :fail :cas [12 [7 43]] :ValueNotSwapFrom INFO [2019-12-17 12:37:09,778] jepsen worker 37 - jepsen.util 11107 :invoke :cas [12 [43 21]] INFO [2019-12-17 12:37:09,798] jepsen worker 64 - jepsen.util 8884 :ok :write [21 49] INFO [2019-12-17 12:37:09,798] jepsen worker 64 - jepsen.util 8884 :invoke :read [21 nil] INFO [2019-12-17 12:37:09,799] jepsen worker 35 - jepsen.util 15605 :info :write [11 37] :RequestCanceledException INFO [2019-12-17 12:37:09,800] jepsen worker 35 - jepsen.util 15695 :invoke :cas [11 [12 40]] INFO [2019-12-17 12:37:09,803] jepsen worker 32 - jepsen.util 11912 :ok :write [10 9] INFO [2019-12-17 12:37:09,803] jepsen worker 32 - jepsen.util 11912 :invoke :read [10 nil] INFO [2019-12-17 12:37:09,815] jepsen worker 31 - jepsen.util 9751 :info :write [10 10] :RequestCanceledException INFO [2019-12-17 12:37:09,815] jepsen worker 53 - jepsen.util 10943 :info :write [17 42] :RequestCanceledException INFO [2019-12-17 12:37:09,817] jepsen worker 31 - jepsen.util 9841 :invoke :read [10 nil] INFO [2019-12-17 12:37:09,817] jepsen worker 53 - jepsen.util 11033 :invoke :read [17 nil] INFO [2019-12-17 12:37:09,820] jepsen worker 19 - jepsen.util 11809 :ok :read [6 13] INFO [2019-12-17 12:37:09,820] jepsen worker 19 - jepsen.util 11809 :invoke :read [6 nil] INFO [2019-12-17 12:37:09,829] jepsen worker 5 - jepsen.util 11345 :ok :write [1 4] INFO [2019-12-17 12:37:09,829] jepsen worker 5 - jepsen.util 11345 :invoke :cas [1 [23 40]] INFO [2019-12-17 12:37:09,832] jepsen worker 68 - jepsen.util 15008 :info :write [22 43] :RequestCanceledException INFO [2019-12-17 12:37:09,832] jepsen worker 6 - jepsen.util 16116 :info :write [2 24] :RequestCanceledException INFO [2019-12-17 12:37:09,833] jepsen worker 68 - jepsen.util 15098 :invoke :write [22 43] INFO [2019-12-17 12:37:09,833] jepsen worker 6 - jepsen.util 16206 :invoke :write [2 14] INFO [2019-12-17 12:37:09,835] jepsen worker 88 - jepsen.util 11698 :ok :read [29 4] INFO [2019-12-17 12:37:09,835] jepsen worker 88 - jepsen.util 11698 :invoke :read [29 nil] INFO [2019-12-17 12:37:09,837] jepsen worker 64 - jepsen.util 8884 :ok :read [21 49] INFO [2019-12-17 12:37:09,838] jepsen worker 64 - jepsen.util 8884 :invoke :read [21 nil] INFO [2019-12-17 12:37:09,858] jepsen worker 89 - jepsen.util 11519 :fail :cas [29 [23 14]] :ValueNotSwapFrom INFO [2019-12-17 12:37:09,858] jepsen worker 89 - jepsen.util 11519 :invoke :cas [29 [12 45]] INFO [2019-12-17 12:37:09,862] jepsen worker 52 - jepsen.util 10942 :ok :write [17 37] INFO [2019-12-17 12:37:09,862] jepsen worker 52 - jepsen.util 10942 :invoke :read [17 nil] INFO [2019-12-17 12:37:09,887] jepsen worker 32 - jepsen.util 11912 :ok :read [10 9] INFO [2019-12-17 12:37:09,887] jepsen worker 32 - jepsen.util 11912 :invoke :read [10 nil] INFO [2019-12-17 12:37:09,902] jepsen worker 64 - jepsen.util 8884 :ok :read [21 49] INFO [2019-12-17 12:37:09,902] jepsen worker 64 - jepsen.util 8884 :invoke :read [21 nil] INFO [2019-12-17 12:37:09,905] jepsen worker 19 - jepsen.util 11809 :ok :read [6 13] INFO [2019-12-17 12:37:09,905] jepsen worker 19 - jepsen.util 11809 :invoke :read [6 nil] INFO [2019-12-17 12:37:09,910] jepsen worker 71 - jepsen.util 12311 :ok :read [23 10] INFO [2019-12-17 12:37:09,911] jepsen worker 71 - jepsen.util 12311 :invoke :read [23 nil] INFO [2019-12-17 12:37:09,912] jepsen worker 27 - jepsen.util 10737 :ok :write [9 18] INFO [2019-12-17 12:37:09,912] jepsen worker 27 - jepsen.util 10737 :invoke :cas [9 [14 32]] INFO [2019-12-17 12:37:09,916] jepsen worker 5 - jepsen.util 11345 :fail :cas [1 [23 40]] :ValueNotSwapFrom INFO [2019-12-17 12:37:09,916] jepsen worker 5 - jepsen.util 11345 :invoke :read [1 nil] INFO [2019-12-17 12:37:09,916] jepsen worker 75 - jepsen.util 15285 :info :cas [25 [34 27]] :RequestCanceledException INFO [2019-12-17 12:37:09,918] jepsen worker 75 - jepsen.util 15375 :invoke :write [25 43] INFO [2019-12-17 12:37:09,919] jepsen worker 32 - jepsen.util 11912 :ok :read [10 9] INFO [2019-12-17 12:37:09,919] jepsen worker 32 - jepsen.util 11912 :invoke :cas [10 [24 8]] INFO [2019-12-17 12:37:09,925] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66804,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:37:09,925] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66804,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:37:09,925] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66807,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:37:09,925] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66807,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:37:09,925] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66808,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:37:09,925] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66808,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:37:09,931] jepsen worker 55 - jepsen.util 12115 :fail :cas [18 [23 20]] :ValueNotSwapFrom INFO [2019-12-17 12:37:09,931] jepsen worker 55 - jepsen.util 12115 :invoke :cas [18 [40 11]] INFO [2019-12-17 12:37:09,933] jepsen worker 69 - jepsen.util 11679 :info :write [23 31] :RequestCanceledException INFO [2019-12-17 12:37:09,933] jepsen worker 41 - jepsen.util 16781 :info :write [13 9] :RequestCanceledException INFO [2019-12-17 12:37:09,933] jepsen worker 54 - jepsen.util 11934 :info :write [18 22] :RequestCanceledException INFO [2019-12-17 12:37:09,934] jepsen worker 69 - jepsen.util 11769 :invoke :cas [23 [15 1]] INFO [2019-12-17 12:37:09,934] jepsen worker 41 - jepsen.util 16871 :invoke :write [13 11] INFO [2019-12-17 12:37:09,934] jepsen worker 54 - jepsen.util 12024 :invoke :cas [18 [25 15]] INFO [2019-12-17 12:37:09,947] jepsen worker 30 - jepsen.util 10650 :fail :read [10 nil] :RequestTimeoutException INFO [2019-12-17 12:37:09,947] jepsen worker 30 - jepsen.util 10650 :invoke :write [10 2] INFO [2019-12-17 12:37:09,949] jepsen worker 64 - jepsen.util 8884 :ok :read [21 49] INFO [2019-12-17 12:37:09,949] jepsen worker 64 - jepsen.util 8884 :invoke :write [21 44] INFO [2019-12-17 12:37:09,974] jepsen worker 19 - jepsen.util 11809 :ok :read [6 13] INFO [2019-12-17 12:37:09,974] jepsen worker 19 - jepsen.util 11809 :invoke :cas [6 [15 18]] INFO [2019-12-17 12:37:09,998] jepsen worker 27 - jepsen.util 10737 :fail :cas [9 [14 32]] :ValueNotSwapFrom INFO [2019-12-17 12:37:09,998] jepsen worker 27 - jepsen.util 10737 :invoke :read [9 nil] INFO [2019-12-17 12:37:10,007] jepsen worker 52 - jepsen.util 10942 :ok :read [17 37] INFO [2019-12-17 12:37:10,007] jepsen worker 52 - jepsen.util 10942 :invoke :read [17 nil] INFO [2019-12-17 12:37:10,012] jepsen worker 89 - jepsen.util 11519 :fail :cas [29 [12 45]] :ValueNotSwapFrom INFO [2019-12-17 12:37:10,012] jepsen worker 89 - jepsen.util 11519 :invoke :read [29 nil] INFO [2019-12-17 12:37:10,023] jepsen worker 5 - jepsen.util 11345 :ok :read [1 4] INFO [2019-12-17 12:37:10,023] jepsen worker 5 - jepsen.util 11345 :invoke :write [1 20] INFO [2019-12-17 12:37:10,033] jepsen worker 24 - jepsen.util 14424 :info :cas [8 [8 36]] :RequestCanceledException INFO [2019-12-17 12:37:10,034] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66828,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:37:10,035] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66828,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:37:10,035] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66830,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:37:10,035] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66830,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:37:10,036] jepsen worker 32 - jepsen.util 11912 :fail :cas [10 [24 8]] :ValueNotSwapFrom INFO [2019-12-17 12:37:10,036] jepsen worker 32 - jepsen.util 11912 :invoke :read [10 nil] INFO [2019-12-17 12:37:10,037] jepsen worker 24 - jepsen.util 14514 :invoke :write [8 33] INFO [2019-12-17 12:37:10,047] jepsen worker 29 - jepsen.util 9749 :info :write [9 15] :RequestCanceledException INFO [2019-12-17 12:37:10,047] jepsen worker 9 - jepsen.util 12159 :info :cas [3 [19 4]] :RequestCanceledException INFO [2019-12-17 12:37:10,047] jepsen worker 39 - jepsen.util 15249 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:37:10,047] jepsen worker 39 - jepsen.util 15249 :invoke :cas [13 [31 45]] INFO [2019-12-17 12:37:10,048] jepsen worker 9 - jepsen.util 12249 :invoke :cas [3 [26 3]] INFO [2019-12-17 12:37:10,049] jepsen worker 29 - jepsen.util 9839 :invoke :cas [9 [1 16]] INFO [2019-12-17 12:37:10,060] jepsen worker 55 - jepsen.util 12115 :fail :cas [18 [40 11]] :ValueNotSwapFrom INFO [2019-12-17 12:37:10,060] jepsen worker 55 - jepsen.util 12115 :invoke :write [18 16] INFO [2019-12-17 12:37:10,062] jepsen worker 27 - jepsen.util 10737 :ok :read [9 18] INFO [2019-12-17 12:37:10,063] jepsen worker 27 - jepsen.util 10737 :invoke :write [9 41] INFO [2019-12-17 12:37:10,064] jepsen worker 40 - jepsen.util 16330 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:37:10,064] jepsen worker 1 - jepsen.util 15751 :info :cas [0 [25 37]] :RequestCanceledException INFO [2019-12-17 12:37:10,064] jepsen worker 40 - jepsen.util 16330 :invoke :write [13 47] INFO [2019-12-17 12:37:10,065] jepsen worker 1 - jepsen.util 15841 :invoke :read [0 nil] INFO [2019-12-17 12:37:10,099] jepsen worker 60 - jepsen.util 16980 :info :write [20 5] :RequestCanceledException INFO [2019-12-17 12:37:10,099] jepsen worker 42 - jepsen.util 16692 :info :cas [14 [43 22]] :RequestCanceledException INFO [2019-12-17 12:37:10,101] jepsen worker 60 - jepsen.util 17070 :invoke :write [20 7] INFO [2019-12-17 12:37:10,101] jepsen worker 42 - jepsen.util 16782 :invoke :read [14 nil] INFO [2019-12-17 12:37:10,118] jepsen worker 79 - jepsen.util 11689 :info :write [26 21] :RequestCanceledException INFO [2019-12-17 12:37:10,118] jepsen worker 46 - jepsen.util 11296 :info :write [15 22] :RequestCanceledException INFO [2019-12-17 12:37:10,119] jepsen worker 79 - jepsen.util 11779 :invoke :cas [26 [19 34]] INFO [2019-12-17 12:37:10,119] jepsen worker 46 - jepsen.util 11386 :invoke :read [15 nil] INFO [2019-12-17 12:37:10,124] jepsen worker 27 - jepsen.util 10737 :ok :write [9 41] INFO [2019-12-17 12:37:10,124] jepsen worker 27 - jepsen.util 10737 :invoke :read [9 nil] INFO [2019-12-17 12:37:10,134] jepsen worker 22 - jepsen.util 15772 :info :write [7 5] :RequestCanceledException INFO [2019-12-17 12:37:10,134] jepsen worker 57 - jepsen.util 15537 :info :write [19 27] :RequestCanceledException INFO [2019-12-17 12:37:10,134] jepsen worker 76 - jepsen.util 16006 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:37:10,134] jepsen worker 76 - jepsen.util 16006 :invoke :cas [25 [16 29]] INFO [2019-12-17 12:37:10,135] jepsen worker 22 - jepsen.util 15862 :invoke :read [7 nil] INFO [2019-12-17 12:37:10,135] jepsen worker 57 - jepsen.util 15627 :invoke :write [19 19] INFO [2019-12-17 12:37:10,141] jepsen worker 64 - jepsen.util 8884 :ok :write [21 44] INFO [2019-12-17 12:37:10,141] jepsen worker 64 - jepsen.util 8884 :invoke :write [21 42] INFO [2019-12-17 12:37:10,146] jepsen worker 83 - jepsen.util 15833 :info :write [27 48] :RequestCanceledException INFO [2019-12-17 12:37:10,148] jepsen worker 27 - jepsen.util 10737 :ok :read [9 41] INFO [2019-12-17 12:37:10,148] jepsen worker 83 - jepsen.util 15923 :invoke :cas [27 [26 26]] INFO [2019-12-17 12:37:10,148] jepsen worker 27 - jepsen.util 10737 :invoke :cas [9 [5 12]] INFO [2019-12-17 12:37:10,160] jepsen worker 12 - jepsen.util 11712 :ok :read [4 15] INFO [2019-12-17 12:37:10,160] jepsen worker 12 - jepsen.util 11712 :invoke :write [4 0] INFO [2019-12-17 12:37:10,165] jepsen worker 34 - jepsen.util 16144 :info :cas [11 [13 46]] :RequestCanceledException INFO [2019-12-17 12:37:10,166] jepsen worker 34 - jepsen.util 16234 :invoke :cas [11 [49 6]] INFO [2019-12-17 12:37:10,170] jepsen worker 89 - jepsen.util 11519 :ok :read [29 4] INFO [2019-12-17 12:37:10,170] jepsen worker 89 - jepsen.util 11519 :invoke :read [29 nil] INFO [2019-12-17 12:37:10,175] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:37:10,175] jepsen nemesis - couchbase.util Rebalance check count: 526 INFO [2019-12-17 12:37:10,175] jepsen nemesis - couchbase.util Rebalance stuck count: 507 INFO [2019-12-17 12:37:10,181] jepsen worker 27 - jepsen.util 10737 :fail :cas [9 [5 12]] :ValueNotSwapFrom INFO [2019-12-17 12:37:10,181] jepsen worker 27 - jepsen.util 10737 :invoke :cas [9 [40 14]] INFO [2019-12-17 12:37:10,214] jepsen worker 27 - jepsen.util 10737 :fail :cas [9 [40 14]] :ValueNotSwapFrom INFO [2019-12-17 12:37:10,215] jepsen worker 27 - jepsen.util 10737 :invoke :write [9 25] INFO [2019-12-17 12:37:10,217] jepsen worker 49 - jepsen.util 15709 :info :write [16 21] :RequestCanceledException INFO [2019-12-17 12:37:10,218] jepsen worker 49 - jepsen.util 15799 :invoke :read [16 nil] INFO [2019-12-17 12:37:10,233] jepsen worker 64 - jepsen.util 8884 :ok :write [21 42] INFO [2019-12-17 12:37:10,233] jepsen worker 4 - jepsen.util 10534 :info :write [1 40] :RequestCanceledException INFO [2019-12-17 12:37:10,233] jepsen worker 64 - jepsen.util 8884 :invoke :write [21 45] INFO [2019-12-17 12:37:10,234] jepsen worker 4 - jepsen.util 10624 :invoke :write [1 45] INFO [2019-12-17 12:37:10,253] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66858,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:37:10,253] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66858,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:37:10,253] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66859,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:37:10,253] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66859,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:37:10,253] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66860,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:37:10,253] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66860,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:37:10,263] jepsen worker 52 - jepsen.util 10942 :ok :read [17 37] INFO [2019-12-17 12:37:10,263] jepsen worker 52 - jepsen.util 10942 :invoke :read [17 nil] INFO [2019-12-17 12:37:10,265] jepsen worker 79 - jepsen.util 11779 :fail :cas [26 [19 34]] :ValueNotSwapFrom INFO [2019-12-17 12:37:10,265] jepsen worker 79 - jepsen.util 11779 :invoke :read [26 nil] INFO [2019-12-17 12:37:10,280] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66865,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:37:10,280] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66865,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:37:10,280] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66869,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:37:10,280] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66869,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:37:10,297] jepsen worker 27 - jepsen.util 10737 :ok :write [9 25] INFO [2019-12-17 12:37:10,297] jepsen worker 27 - jepsen.util 10737 :invoke :read [9 nil] INFO [2019-12-17 12:37:10,309] jepsen worker 51 - jepsen.util 10851 :ok :write [17 1] INFO [2019-12-17 12:37:10,309] jepsen worker 51 - jepsen.util 10851 :invoke :read [17 nil] INFO [2019-12-17 12:37:10,311] jepsen worker 87 - jepsen.util 11067 :ok :write [29 19] INFO [2019-12-17 12:37:10,312] jepsen worker 87 - jepsen.util 11067 :invoke :cas [29 [30 26]] INFO [2019-12-17 12:37:10,317] jepsen worker 18 - jepsen.util 11628 :info :write [6 1] :RequestCanceledException INFO [2019-12-17 12:37:10,321] jepsen worker 18 - jepsen.util 11718 :invoke :cas [6 [23 3]] INFO [2019-12-17 12:37:10,321] jepsen worker 70 - jepsen.util 12040 :ok :write [23 41] INFO [2019-12-17 12:37:10,321] jepsen worker 70 - jepsen.util 12040 :invoke :write [23 31] INFO [2019-12-17 12:37:10,333] jepsen worker 80 - jepsen.util 11150 :info :write [26 20] :RequestCanceledException INFO [2019-12-17 12:37:10,334] jepsen worker 80 - jepsen.util 11240 :invoke :read [26 nil] INFO [2019-12-17 12:37:10,344] jepsen worker 27 - jepsen.util 10737 :ok :read [9 25] INFO [2019-12-17 12:37:10,344] jepsen worker 27 - jepsen.util 10737 :invoke :cas [9 [31 11]] INFO [2019-12-17 12:37:10,345] jepsen worker 78 - jepsen.util 11418 :info :write [26 15] :RequestCanceledException INFO [2019-12-17 12:37:10,346] jepsen worker 3 - jepsen.util 10983 :ok :read [1 4] INFO [2019-12-17 12:37:10,346] jepsen worker 3 - jepsen.util 10983 :invoke :write [1 48] INFO [2019-12-17 12:37:10,346] jepsen worker 78 - jepsen.util 11508 :invoke :cas [26 [17 25]] INFO [2019-12-17 12:37:10,354] jepsen worker 53 - jepsen.util 11033 :ok :read [17 1] INFO [2019-12-17 12:37:10,355] jepsen worker 53 - jepsen.util 11033 :invoke :write [17 41] INFO [2019-12-17 12:37:10,373] jepsen worker 64 - jepsen.util 8884 :ok :write [21 45] INFO [2019-12-17 12:37:10,373] jepsen worker 64 - jepsen.util 8884 :invoke :read [21 nil] INFO [2019-12-17 12:37:10,373] jepsen worker 27 - jepsen.util 10737 :fail :cas [9 [31 11]] :ValueNotSwapFrom INFO [2019-12-17 12:37:10,373] jepsen worker 27 - jepsen.util 10737 :invoke :cas [9 [14 6]] INFO [2019-12-17 12:37:10,377] jepsen worker 30 - jepsen.util 10650 :ok :write [10 2] INFO [2019-12-17 12:37:10,377] jepsen worker 30 - jepsen.util 10650 :invoke :write [10 31] INFO [2019-12-17 12:37:10,392] jepsen worker 69 - jepsen.util 11769 :fail :cas [23 [15 1]] :ValueNotSwapFrom INFO [2019-12-17 12:37:10,392] jepsen worker 69 - jepsen.util 11769 :invoke :read [23 nil] INFO [2019-12-17 12:37:10,396] jepsen worker 71 - jepsen.util 12311 :ok :read [23 41] INFO [2019-12-17 12:37:10,396] jepsen worker 71 - jepsen.util 12311 :invoke :read [23 nil] INFO [2019-12-17 12:37:10,420] jepsen worker 32 - jepsen.util 11912 :ok :read [10 2] INFO [2019-12-17 12:37:10,420] jepsen worker 32 - jepsen.util 11912 :invoke :cas [10 [46 15]] INFO [2019-12-17 12:37:10,423] jepsen worker 37 - jepsen.util 11107 :fail :cas [12 [43 21]] :ValueNotSwapFrom INFO [2019-12-17 12:37:10,423] jepsen worker 37 - jepsen.util 11107 :invoke :read [12 nil] INFO [2019-12-17 12:37:10,425] jepsen worker 73 - jepsen.util 10693 :ok :write [24 31] INFO [2019-12-17 12:37:10,425] jepsen worker 73 - jepsen.util 10693 :invoke :read [24 nil] INFO [2019-12-17 12:37:10,425] jepsen worker 45 - jepsen.util 11655 :ok :write [15 30] INFO [2019-12-17 12:37:10,425] jepsen worker 45 - jepsen.util 11655 :invoke :cas [15 [3 18]] INFO [2019-12-17 12:37:10,432] jepsen worker 33 - jepsen.util 15603 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:37:10,433] jepsen worker 33 - jepsen.util 15603 :invoke :cas [11 [47 38]] INFO [2019-12-17 12:37:10,444] jepsen worker 54 - jepsen.util 12024 :fail :cas [18 [25 15]] :ValueNotSwapFrom INFO [2019-12-17 12:37:10,444] jepsen worker 54 - jepsen.util 12024 :invoke :read [18 nil] INFO [2019-12-17 12:37:10,459] jepsen worker 88 - jepsen.util 11698 :ok :read [29 19] INFO [2019-12-17 12:37:10,460] jepsen worker 88 - jepsen.util 11698 :invoke :cas [29 [4 34]] INFO [2019-12-17 12:37:10,464] jepsen worker 11 - jepsen.util 11261 :ok :write [3 33] INFO [2019-12-17 12:37:10,464] jepsen worker 11 - jepsen.util 11261 :invoke :read [3 nil] INFO [2019-12-17 12:37:10,483] jepsen worker 64 - jepsen.util 8884 :ok :read [21 45] INFO [2019-12-17 12:37:10,483] jepsen worker 64 - jepsen.util 8884 :invoke :write [21 34] INFO [2019-12-17 12:37:10,496] jepsen worker 78 - jepsen.util 11508 :fail :cas [26 [17 25]] :ValueNotSwapFrom INFO [2019-12-17 12:37:10,496] jepsen worker 78 - jepsen.util 11508 :invoke :cas [26 [23 46]] INFO [2019-12-17 12:37:10,499] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66897,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:37:10,499] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66897,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:37:10,500] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66901,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:37:10,500] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66901,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:37:10,534] jepsen worker 51 - jepsen.util 10851 :ok :read [17 1] INFO [2019-12-17 12:37:10,534] jepsen worker 51 - jepsen.util 10851 :invoke :read [17 nil] INFO [2019-12-17 12:37:10,537] jepsen worker 89 - jepsen.util 11519 :ok :read [29 19] INFO [2019-12-17 12:37:10,538] jepsen worker 89 - jepsen.util 11519 :invoke :write [29 9] INFO [2019-12-17 12:37:10,542] jepsen worker 9 - jepsen.util 12249 :fail :cas [3 [26 3]] :ValueNotSwapFrom INFO [2019-12-17 12:37:10,542] jepsen worker 9 - jepsen.util 12249 :invoke :read [3 nil] INFO [2019-12-17 12:37:10,548] jepsen worker 88 - jepsen.util 11698 :fail :cas [29 [4 34]] :ValueNotSwapFrom INFO [2019-12-17 12:37:10,549] jepsen worker 88 - jepsen.util 11698 :invoke :write [29 20] INFO [2019-12-17 12:37:10,555] jepsen worker 27 - jepsen.util 10737 :fail :cas [9 [14 6]] :ValueNotSwapFrom INFO [2019-12-17 12:37:10,555] jepsen worker 27 - jepsen.util 10737 :invoke :cas [9 [25 5]] INFO [2019-12-17 12:37:10,575] jepsen worker 71 - jepsen.util 12311 :ok :read [23 41] INFO [2019-12-17 12:37:10,575] jepsen worker 71 - jepsen.util 12311 :invoke :cas [23 [4 47]] INFO [2019-12-17 12:37:10,594] jepsen worker 54 - jepsen.util 12024 :ok :read [18 9] INFO [2019-12-17 12:37:10,594] jepsen worker 54 - jepsen.util 12024 :invoke :write [18 44] INFO [2019-12-17 12:37:10,622] jepsen worker 29 - jepsen.util 9839 :fail :cas [9 [1 16]] :ValueNotSwapFrom INFO [2019-12-17 12:37:10,623] jepsen worker 29 - jepsen.util 9839 :invoke :cas [9 [33 19]] INFO [2019-12-17 12:37:10,638] jepsen worker 46 - jepsen.util 11386 :ok :read [15 30] INFO [2019-12-17 12:37:10,638] jepsen worker 46 - jepsen.util 11386 :invoke :write [15 43] INFO [2019-12-17 12:37:10,642] jepsen worker 31 - jepsen.util 9841 :ok :read [10 2] INFO [2019-12-17 12:37:10,642] jepsen worker 31 - jepsen.util 9841 :invoke :write [10 40] INFO [2019-12-17 12:37:10,660] jepsen worker 52 - jepsen.util 10942 :ok :read [17 1] INFO [2019-12-17 12:37:10,660] jepsen worker 52 - jepsen.util 10942 :invoke :read [17 nil] INFO [2019-12-17 12:37:10,662] jepsen worker 71 - jepsen.util 12311 :fail :cas [23 [4 47]] :ValueNotSwapFrom INFO [2019-12-17 12:37:10,662] jepsen worker 71 - jepsen.util 12311 :invoke :read [23 nil] INFO [2019-12-17 12:37:10,683] jepsen worker 11 - jepsen.util 11261 :ok :read [3 33] INFO [2019-12-17 12:37:10,683] jepsen worker 11 - jepsen.util 11261 :invoke :cas [3 [19 19]] INFO [2019-12-17 12:37:10,688] jepsen worker 87 - jepsen.util 11067 :fail :cas [29 [30 26]] :ValueNotSwapFrom INFO [2019-12-17 12:37:10,688] jepsen worker 87 - jepsen.util 11067 :invoke :write [29 22] INFO [2019-12-17 12:37:10,695] jepsen worker 18 - jepsen.util 11718 :fail :cas [6 [23 3]] :ValueNotSwapFrom INFO [2019-12-17 12:37:10,695] jepsen worker 18 - jepsen.util 11718 :invoke :write [6 46] INFO [2019-12-17 12:37:10,727] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66919,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:37:10,727] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66919,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:37:10,727] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66922,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:37:10,727] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66922,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:37:10,727] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66923,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:37:10,727] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66923,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:37:10,727] jepsen worker 37 - jepsen.util 11107 :ok :read [12 0] INFO [2019-12-17 12:37:10,727] jepsen worker 37 - jepsen.util 11107 :invoke :write [12 29] INFO [2019-12-17 12:37:10,769] jepsen worker 73 - jepsen.util 10693 :ok :read [24 31] INFO [2019-12-17 12:37:10,769] jepsen worker 73 - jepsen.util 10693 :invoke :cas [24 [28 28]] INFO [2019-12-17 12:37:10,775] jepsen worker 45 - jepsen.util 11655 :fail :cas [15 [3 18]] :ValueNotSwapFrom INFO [2019-12-17 12:37:10,775] jepsen worker 45 - jepsen.util 11655 :invoke :write [15 49] INFO [2019-12-17 12:37:10,782] jepsen worker 11 - jepsen.util 11261 :fail :cas [3 [19 19]] :ValueNotSwapFrom INFO [2019-12-17 12:37:10,782] jepsen worker 11 - jepsen.util 11261 :invoke :write [3 28] INFO [2019-12-17 12:37:10,800] jepsen worker 52 - jepsen.util 10942 :ok :read [17 1] INFO [2019-12-17 12:37:10,801] jepsen worker 52 - jepsen.util 10942 :invoke :read [17 nil] INFO [2019-12-17 12:37:10,815] jepsen worker 80 - jepsen.util 11240 :ok :read [26 23] INFO [2019-12-17 12:37:10,815] jepsen worker 80 - jepsen.util 11240 :invoke :cas [26 [15 44]] INFO [2019-12-17 12:37:10,818] jepsen worker 29 - jepsen.util 9839 :fail :cas [9 [33 19]] :ValueNotSwapFrom INFO [2019-12-17 12:37:10,818] jepsen worker 29 - jepsen.util 9839 :invoke :write [9 0] INFO [2019-12-17 12:37:10,833] jepsen worker 79 - jepsen.util 11779 :ok :read [26 23] INFO [2019-12-17 12:37:10,833] jepsen worker 79 - jepsen.util 11779 :invoke :write [26 35] INFO [2019-12-17 12:37:10,834] jepsen worker 72 - jepsen.util 11862 :info :write [24 39] :RequestCanceledException INFO [2019-12-17 12:37:10,835] jepsen worker 72 - jepsen.util 11952 :invoke :read [24 nil] INFO [2019-12-17 12:37:10,846] jepsen worker 64 - jepsen.util 8884 :ok :write [21 34] INFO [2019-12-17 12:37:10,847] jepsen worker 64 - jepsen.util 8884 :invoke :cas [21 [39 0]] INFO [2019-12-17 12:37:10,905] jepsen worker 64 - jepsen.util 8884 :fail :cas [21 [39 0]] :ValueNotSwapFrom INFO [2019-12-17 12:37:10,905] jepsen worker 64 - jepsen.util 8884 :invoke :cas [21 [23 11]] INFO [2019-12-17 12:37:10,909] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66942,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:37:10,909] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66942,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:37:10,909] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66945,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:37:10,909] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66945,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:37:10,911] jepsen worker 14 - jepsen.util 11174 :info :write [4 49] :RequestCanceledException INFO [2019-12-17 12:37:10,912] jepsen worker 14 - jepsen.util 11264 :invoke :read [4 nil] INFO [2019-12-17 12:37:10,918] jepsen worker 73 - jepsen.util 10693 :fail :cas [24 [28 28]] :ValueNotSwapFrom INFO [2019-12-17 12:37:10,918] jepsen worker 73 - jepsen.util 10693 :invoke :cas [24 [15 13]] INFO [2019-12-17 12:37:10,925] jepsen worker 64 - jepsen.util 8884 :fail :cas [21 [23 11]] :ValueNotSwapFrom INFO [2019-12-17 12:37:10,925] jepsen worker 64 - jepsen.util 8884 :invoke :write [21 39] INFO [2019-12-17 12:37:10,937] jepsen worker 73 - jepsen.util 10693 :fail :cas [24 [15 13]] :ValueNotSwapFrom INFO [2019-12-17 12:37:10,937] jepsen worker 73 - jepsen.util 10693 :invoke :cas [24 [36 22]] INFO [2019-12-17 12:37:10,940] jepsen worker 37 - jepsen.util 11107 :ok :write [12 29] INFO [2019-12-17 12:37:10,940] jepsen worker 37 - jepsen.util 11107 :invoke :write [12 24] INFO [2019-12-17 12:37:10,961] jepsen worker 69 - jepsen.util 11769 :ok :read [23 41] INFO [2019-12-17 12:37:10,961] jepsen worker 69 - jepsen.util 11769 :invoke :write [23 40] INFO [2019-12-17 12:37:10,970] jepsen worker 51 - jepsen.util 10851 :ok :read [17 1] INFO [2019-12-17 12:37:10,971] jepsen worker 51 - jepsen.util 10851 :invoke :read [17 nil] INFO [2019-12-17 12:37:10,991] jepsen worker 73 - jepsen.util 10693 :fail :cas [24 [36 22]] :ValueNotSwapFrom INFO [2019-12-17 12:37:10,992] jepsen worker 73 - jepsen.util 10693 :invoke :read [24 nil] INFO [2019-12-17 12:37:10,995] jepsen worker 71 - jepsen.util 12311 :ok :read [23 41] INFO [2019-12-17 12:37:10,996] jepsen worker 71 - jepsen.util 12311 :invoke :cas [23 [3 16]] INFO [2019-12-17 12:37:11,011] jepsen worker 64 - jepsen.util 8884 :ok :write [21 39] INFO [2019-12-17 12:37:11,011] jepsen worker 64 - jepsen.util 8884 :invoke :write [21 14] INFO [2019-12-17 12:37:11,017] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66954,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:37:11,018] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66954,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:37:11,018] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66959,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:37:11,018] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66959,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:37:11,018] jepsen worker 13 - jepsen.util 10813 :info :write [4 36] :RequestCanceledException INFO [2019-12-17 12:37:11,018] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66960,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:37:11,018] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66960,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:37:11,019] jepsen worker 13 - jepsen.util 10903 :invoke :read [4 nil] INFO [2019-12-17 12:37:11,023] jepsen worker 73 - jepsen.util 10693 :ok :read [24 31] INFO [2019-12-17 12:37:11,023] jepsen worker 73 - jepsen.util 10693 :invoke :cas [24 [46 26]] INFO [2019-12-17 12:37:11,028] jepsen worker 77 - jepsen.util 16637 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:37:11,028] jepsen worker 77 - jepsen.util 16637 :invoke :cas [25 [46 21]] INFO [2019-12-17 12:37:11,047] jepsen worker 82 - jepsen.util 16012 :info :cas [27 [43 17]] :RequestCanceledException INFO [2019-12-17 12:37:11,047] jepsen worker 66 - jepsen.util 15636 :info :write [22 43] :RequestCanceledException INFO [2019-12-17 12:37:11,048] jepsen worker 66 - jepsen.util 15726 :invoke :read [22 nil] INFO [2019-12-17 12:37:11,048] jepsen worker 82 - jepsen.util 16102 :invoke :write [27 27] INFO [2019-12-17 12:37:11,064] jepsen worker 62 - jepsen.util 16442 :info :cas [20 [19 41]] :RequestCanceledException INFO [2019-12-17 12:37:11,065] jepsen worker 62 - jepsen.util 16532 :invoke :cas [20 [33 41]] INFO [2019-12-17 12:37:11,079] jepsen worker 87 - jepsen.util 11067 :ok :write [29 22] INFO [2019-12-17 12:37:11,079] jepsen worker 87 - jepsen.util 11067 :invoke :write [29 14] INFO [2019-12-17 12:37:11,100] jepsen worker 17 - jepsen.util 15767 :info :cas [5 [20 16]] :RequestCanceledException INFO [2019-12-17 12:37:11,102] jepsen worker 17 - jepsen.util 15857 :invoke :cas [5 [12 4]] INFO [2019-12-17 12:37:11,105] jepsen worker 73 - jepsen.util 10693 :fail :cas [24 [46 26]] :ValueNotSwapFrom INFO [2019-12-17 12:37:11,105] jepsen worker 73 - jepsen.util 10693 :invoke :cas [24 [37 31]] INFO [2019-12-17 12:37:11,117] jepsen worker 26 - jepsen.util 16496 :info :write [8 7] :RequestCanceledException INFO [2019-12-17 12:37:11,117] jepsen worker 84 - jepsen.util 15564 :info :write [28 48] :RequestCanceledException INFO [2019-12-17 12:37:11,118] jepsen worker 26 - jepsen.util 16586 :invoke :read [8 nil] INFO [2019-12-17 12:37:11,118] jepsen worker 84 - jepsen.util 15654 :invoke :read [28 nil] INFO [2019-12-17 12:37:11,132] jepsen worker 73 - jepsen.util 10693 :fail :cas [24 [37 31]] :ValueNotSwapFrom INFO [2019-12-17 12:37:11,133] jepsen worker 73 - jepsen.util 10693 :invoke :read [24 nil] INFO [2019-12-17 12:37:11,133] jepsen worker 7 - jepsen.util 15847 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:37:11,133] jepsen worker 86 - jepsen.util 16466 :info :write [28 38] :RequestCanceledException INFO [2019-12-17 12:37:11,133] jepsen worker 2 - jepsen.util 15302 :info :cas [0 [47 21]] :RequestCanceledException INFO [2019-12-17 12:37:11,133] jepsen worker 67 - jepsen.util 15907 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:37:11,133] jepsen worker 7 - jepsen.util 15847 :invoke :write [2 36] INFO [2019-12-17 12:37:11,133] jepsen worker 67 - jepsen.util 15907 :invoke :read [22 nil] INFO [2019-12-17 12:37:11,134] jepsen worker 2 - jepsen.util 15392 :invoke :read [0 nil] INFO [2019-12-17 12:37:11,135] jepsen worker 86 - jepsen.util 16556 :invoke :cas [28 [39 19]] INFO [2019-12-17 12:37:11,147] jepsen worker 23 - jepsen.util 15863 :info :write [7 6] :RequestCanceledException INFO [2019-12-17 12:37:11,147] jepsen worker 25 - jepsen.util 16765 :info :cas [8 [30 12]] :RequestCanceledException INFO [2019-12-17 12:37:11,148] jepsen worker 23 - jepsen.util 15953 :invoke :cas [7 [23 49]] INFO [2019-12-17 12:37:11,148] jepsen worker 25 - jepsen.util 16855 :invoke :cas [8 [30 11]] INFO [2019-12-17 12:37:11,149] jepsen worker 18 - jepsen.util 11718 :ok :write [6 46] INFO [2019-12-17 12:37:11,149] jepsen worker 18 - jepsen.util 11718 :invoke :read [6 nil] INFO [2019-12-17 12:37:11,152] jepsen worker 30 - jepsen.util 10650 :ok :write [10 31] INFO [2019-12-17 12:37:11,152] jepsen worker 30 - jepsen.util 10650 :invoke :read [10 nil] INFO [2019-12-17 12:37:11,154] jepsen worker 5 - jepsen.util 11345 :ok :write [1 20] INFO [2019-12-17 12:37:11,154] jepsen worker 5 - jepsen.util 11345 :invoke :cas [1 [15 10]] INFO [2019-12-17 12:37:11,155] jepsen worker 11 - jepsen.util 11261 :ok :write [3 28] INFO [2019-12-17 12:37:11,155] jepsen worker 11 - jepsen.util 11261 :invoke :read [3 nil] INFO [2019-12-17 12:37:11,158] jepsen worker 12 - jepsen.util 11712 :ok :write [4 0] INFO [2019-12-17 12:37:11,158] jepsen worker 12 - jepsen.util 11712 :invoke :cas [4 [22 49]] INFO [2019-12-17 12:37:11,164] jepsen worker 48 - jepsen.util 14538 :info :write [16 45] :RequestCanceledException INFO [2019-12-17 12:37:11,164] jepsen worker 58 - jepsen.util 16078 :info :write [19 30] :RequestCanceledException INFO [2019-12-17 12:37:11,164] jepsen worker 21 - jepsen.util 15321 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:37:11,165] jepsen worker 21 - jepsen.util 15321 :invoke :read [7 nil] INFO [2019-12-17 12:37:11,165] jepsen worker 48 - jepsen.util 14628 :invoke :write [16 45] INFO [2019-12-17 12:37:11,165] jepsen worker 58 - jepsen.util 16168 :invoke :cas [19 [7 37]] INFO [2019-12-17 12:37:11,175] jepsen worker 73 - jepsen.util 10693 :ok :read [24 31] INFO [2019-12-17 12:37:11,175] jepsen worker 73 - jepsen.util 10693 :invoke :read [24 nil] INFO [2019-12-17 12:37:11,198] jepsen worker 85 - jepsen.util 16465 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:37:11,198] jepsen worker 0 - jepsen.util 16560 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:37:11,198] jepsen worker 50 - jepsen.util 16160 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:37:11,198] jepsen worker 63 - jepsen.util 10143 :fail :read [21 nil] :RequestTimeoutException INFO [2019-12-17 12:37:11,199] jepsen worker 85 - jepsen.util 16465 :invoke :write [28 10] INFO [2019-12-17 12:37:11,199] jepsen worker 0 - jepsen.util 16560 :invoke :read [0 nil] INFO [2019-12-17 12:37:11,199] jepsen worker 50 - jepsen.util 16160 :invoke :cas [16 [19 3]] INFO [2019-12-17 12:37:11,199] jepsen worker 63 - jepsen.util 10143 :invoke :read [21 nil] INFO [2019-12-17 12:37:11,199] jepsen worker 18 - jepsen.util 11718 :ok :read [6 46] INFO [2019-12-17 12:37:11,199] jepsen worker 18 - jepsen.util 11718 :invoke :write [6 0] INFO [2019-12-17 12:37:11,207] jepsen worker 27 - jepsen.util 10737 :ok :cas [9 [25 5]] INFO [2019-12-17 12:37:11,208] jepsen worker 27 - jepsen.util 10737 :invoke :cas [9 [0 49]] INFO [2019-12-17 12:37:11,217] jepsen worker 59 - jepsen.util 15089 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:37:11,218] jepsen worker 59 - jepsen.util 15089 :invoke :cas [19 [47 18]] INFO [2019-12-17 12:37:11,222] jepsen worker 78 - jepsen.util 11508 :ok :cas [26 [23 46]] INFO [2019-12-17 12:37:11,222] jepsen worker 78 - jepsen.util 11508 :invoke :cas [26 [27 1]] INFO [2019-12-17 12:37:11,228] jepsen worker 36 - jepsen.util 12276 :ok :write [12 7] INFO [2019-12-17 12:37:11,228] jepsen worker 36 - jepsen.util 12276 :invoke :write [12 13] INFO [2019-12-17 12:37:11,231] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:37:11,231] jepsen nemesis - couchbase.util Rebalance check count: 527 INFO [2019-12-17 12:37:11,231] jepsen nemesis - couchbase.util Rebalance stuck count: 508 INFO [2019-12-17 12:37:11,233] jepsen worker 8 - jepsen.util 15488 :info :cas [2 [42 15]] :RequestCanceledException INFO [2019-12-17 12:37:11,233] jepsen worker 61 - jepsen.util 16981 :info :write [20 27] :RequestCanceledException INFO [2019-12-17 12:37:11,233] jepsen worker 15 - jepsen.util 14595 :info :cas [5 [19 28]] :RequestCanceledException INFO [2019-12-17 12:37:11,233] jepsen worker 43 - jepsen.util 15883 :info :write [14 3] :RequestCanceledException INFO [2019-12-17 12:37:11,233] jepsen worker 56 - jepsen.util 12386 :info :write [18 10] :RequestCanceledException INFO [2019-12-17 12:37:11,234] jepsen worker 56 - jepsen.util 12476 :invoke :cas [18 [33 9]] INFO [2019-12-17 12:37:11,235] jepsen worker 61 - jepsen.util 17071 :invoke :read [20 nil] INFO [2019-12-17 12:37:11,236] jepsen worker 43 - jepsen.util 15973 :invoke :cas [14 [36 28]] INFO [2019-12-17 12:37:11,236] jepsen worker 8 - jepsen.util 15578 :invoke :read [2 nil] INFO [2019-12-17 12:37:11,236] jepsen worker 15 - jepsen.util 14685 :invoke :write [5 14] INFO [2019-12-17 12:37:11,238] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66993,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:37:11,238] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66993,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:37:11,238] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66994,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:37:11,238] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":66994,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:37:11,247] jepsen worker 10 - jepsen.util 11080 :info :cas [3 [21 32]] :RequestCanceledException INFO [2019-12-17 12:37:11,247] jepsen worker 20 - jepsen.util 11090 :info :cas [6 [36 8]] :RequestCanceledException INFO [2019-12-17 12:37:11,247] jepsen worker 20 - jepsen.util 11180 :invoke :read [6 nil] INFO [2019-12-17 12:37:11,247] jepsen worker 10 - jepsen.util 11170 :invoke :write [3 34] INFO [2019-12-17 12:37:11,264] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":67005,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:37:11,264] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":67005,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:37:11,264] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":67008,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:37:11,264] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":67008,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:37:11,264] jepsen worker 79 - jepsen.util 11779 :ok :write [26 35] INFO [2019-12-17 12:37:11,265] jepsen worker 79 - jepsen.util 11779 :invoke :read [26 nil] INFO [2019-12-17 12:37:11,275] jepsen worker 9 - jepsen.util 12249 :ok :read [3 33] INFO [2019-12-17 12:37:11,275] jepsen worker 9 - jepsen.util 12249 :invoke :read [3 nil] INFO [2019-12-17 12:37:11,287] jepsen worker 89 - jepsen.util 11519 :ok :write [29 9] INFO [2019-12-17 12:37:11,288] jepsen worker 89 - jepsen.util 11519 :invoke :read [29 nil] INFO [2019-12-17 12:37:11,292] jepsen worker 3 - jepsen.util 10983 :ok :write [1 48] INFO [2019-12-17 12:37:11,293] jepsen worker 3 - jepsen.util 10983 :invoke :cas [1 [39 28]] INFO [2019-12-17 12:37:11,293] jepsen worker 14 - jepsen.util 11264 :ok :read [4 0] INFO [2019-12-17 12:37:11,293] jepsen worker 14 - jepsen.util 11264 :invoke :cas [4 [33 33]] INFO [2019-12-17 12:37:11,295] jepsen worker 38 - jepsen.util 11918 :info :write [12 18] :RequestCanceledException INFO [2019-12-17 12:37:11,301] jepsen worker 38 - jepsen.util 12008 :invoke :write [12 29] INFO [2019-12-17 12:37:11,326] jepsen worker 11 - jepsen.util 11261 :ok :read [3 28] INFO [2019-12-17 12:37:11,326] jepsen worker 11 - jepsen.util 11261 :invoke :cas [3 [5 7]] INFO [2019-12-17 12:37:11,327] jepsen worker 64 - jepsen.util 8884 :ok :write [21 14] INFO [2019-12-17 12:37:11,327] jepsen worker 64 - jepsen.util 8884 :invoke :read [21 nil] INFO [2019-12-17 12:37:11,328] jepsen worker 27 - jepsen.util 10737 :fail :cas [9 [0 49]] :ValueNotSwapFrom INFO [2019-12-17 12:37:11,328] jepsen worker 27 - jepsen.util 10737 :invoke :write [9 42] INFO [2019-12-17 12:37:11,329] jepsen worker 12 - jepsen.util 11712 :fail :cas [4 [22 49]] :ValueNotSwapFrom INFO [2019-12-17 12:37:11,329] jepsen worker 12 - jepsen.util 11712 :invoke :read [4 nil] INFO [2019-12-17 12:37:11,347] jepsen worker 47 - jepsen.util 12287 :info :write [15 8] :RequestCanceledException INFO [2019-12-17 12:37:11,349] jepsen worker 47 - jepsen.util 12377 :invoke :read [15 nil] INFO [2019-12-17 12:37:11,368] jepsen worker 11 - jepsen.util 11261 :fail :cas [3 [5 7]] :ValueNotSwapFrom INFO [2019-12-17 12:37:11,368] jepsen worker 11 - jepsen.util 11261 :invoke :cas [3 [39 45]] INFO [2019-12-17 12:37:11,374] jepsen worker 51 - jepsen.util 10851 :ok :read [17 1] INFO [2019-12-17 12:37:11,374] jepsen worker 51 - jepsen.util 10851 :invoke :read [17 nil] INFO [2019-12-17 12:37:11,376] jepsen worker 73 - jepsen.util 10693 :ok :read [24 31] INFO [2019-12-17 12:37:11,376] jepsen worker 73 - jepsen.util 10693 :invoke :read [24 nil] INFO [2019-12-17 12:37:11,392] jepsen worker 79 - jepsen.util 11779 :ok :read [26 35] INFO [2019-12-17 12:37:11,392] jepsen worker 79 - jepsen.util 11779 :invoke :cas [26 [18 20]] INFO [2019-12-17 12:37:11,397] jepsen worker 56 - jepsen.util 12476 :fail :cas [18 [33 9]] :ValueNotSwapFrom INFO [2019-12-17 12:37:11,397] jepsen worker 56 - jepsen.util 12476 :invoke :cas [18 [3 25]] INFO [2019-12-17 12:37:11,399] jepsen worker 11 - jepsen.util 11261 :fail :cas [3 [39 45]] :ValueNotSwapFrom INFO [2019-12-17 12:37:11,399] jepsen worker 11 - jepsen.util 11261 :invoke :write [3 24] INFO [2019-12-17 12:37:11,401] jepsen worker 71 - jepsen.util 12311 :fail :cas [23 [3 16]] :ValueNotSwapFrom INFO [2019-12-17 12:37:11,401] jepsen worker 71 - jepsen.util 12311 :invoke :cas [23 [27 38]] INFO [2019-12-17 12:37:11,418] jepsen worker 65 - jepsen.util 10145 :info :cas [21 [30 1]] :RequestCanceledException INFO [2019-12-17 12:37:11,420] jepsen worker 65 - jepsen.util 10235 :invoke :cas [21 [33 47]] INFO [2019-12-17 12:37:11,427] jepsen worker 14 - jepsen.util 11264 :fail :cas [4 [33 33]] :ValueNotSwapFrom INFO [2019-12-17 12:37:11,427] jepsen worker 14 - jepsen.util 11264 :invoke :read [4 nil] INFO [2019-12-17 12:37:11,429] jepsen worker 27 - jepsen.util 10737 :ok :write [9 42] INFO [2019-12-17 12:37:11,430] jepsen worker 27 - jepsen.util 10737 :invoke :write [9 40] INFO [2019-12-17 12:37:11,431] jepsen worker 55 - jepsen.util 12115 :ok :write [18 16] INFO [2019-12-17 12:37:11,431] jepsen worker 55 - jepsen.util 12115 :invoke :write [18 33] INFO [2019-12-17 12:37:11,452] jepsen worker 12 - jepsen.util 11712 :ok :read [4 0] INFO [2019-12-17 12:37:11,452] jepsen worker 12 - jepsen.util 11712 :invoke :read [4 nil] INFO [2019-12-17 12:37:11,468] jepsen worker 52 - jepsen.util 10942 :ok :read [17 1] INFO [2019-12-17 12:37:11,468] jepsen worker 52 - jepsen.util 10942 :invoke :read [17 nil] INFO [2019-12-17 12:37:11,476] jepsen worker 64 - jepsen.util 8884 :ok :read [21 14] INFO [2019-12-17 12:37:11,476] jepsen worker 64 - jepsen.util 8884 :invoke :write [21 35] INFO [2019-12-17 12:37:11,483] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":67042,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:37:11,483] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":67042,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:37:11,484] jepsen worker 79 - jepsen.util 11779 :fail :cas [26 [18 20]] :ValueNotSwapFrom INFO [2019-12-17 12:37:11,484] jepsen worker 79 - jepsen.util 11779 :invoke :read [26 nil] INFO [2019-12-17 12:37:11,491] jepsen worker 5 - jepsen.util 11345 :fail :cas [1 [15 10]] :ValueNotSwapFrom INFO [2019-12-17 12:37:11,492] jepsen worker 5 - jepsen.util 11345 :invoke :read [1 nil] INFO [2019-12-17 12:37:11,509] jepsen worker 63 - jepsen.util 10143 :ok :read [21 14] INFO [2019-12-17 12:37:11,509] jepsen worker 63 - jepsen.util 10143 :invoke :write [21 5] INFO [2019-12-17 12:37:11,519] jepsen worker 51 - jepsen.util 10851 :ok :read [17 1] INFO [2019-12-17 12:37:11,519] jepsen worker 51 - jepsen.util 10851 :invoke :write [17 20] INFO [2019-12-17 12:37:11,533] jepsen worker 14 - jepsen.util 11264 :ok :read [4 0] INFO [2019-12-17 12:37:11,533] jepsen worker 14 - jepsen.util 11264 :invoke :cas [4 [42 37]] INFO [2019-12-17 12:37:11,552] jepsen worker 13 - jepsen.util 10903 :ok :read [4 0] INFO [2019-12-17 12:37:11,552] jepsen worker 13 - jepsen.util 10903 :invoke :read [4 nil] INFO [2019-12-17 12:37:11,560] jepsen worker 47 - jepsen.util 12377 :ok :read [15 30] INFO [2019-12-17 12:37:11,560] jepsen worker 47 - jepsen.util 12377 :invoke :write [15 15] INFO [2019-12-17 12:37:11,562] jepsen worker 28 - jepsen.util 9568 :info :write [9 43] :RequestCanceledException INFO [2019-12-17 12:37:11,562] jepsen worker 74 - jepsen.util 11324 :info :write [24 13] :RequestCanceledException INFO [2019-12-17 12:37:11,563] jepsen worker 28 - jepsen.util 9658 :invoke :read [9 nil] INFO [2019-12-17 12:37:11,563] jepsen worker 74 - jepsen.util 11414 :invoke :cas [24 [27 9]] INFO [2019-12-17 12:37:11,566] jepsen worker 12 - jepsen.util 11712 :ok :read [4 0] INFO [2019-12-17 12:37:11,567] jepsen worker 12 - jepsen.util 11712 :invoke :write [4 9] INFO [2019-12-17 12:37:11,567] jepsen worker 73 - jepsen.util 10693 :ok :read [24 31] INFO [2019-12-17 12:37:11,568] jepsen worker 73 - jepsen.util 10693 :invoke :write [24 26] INFO [2019-12-17 12:37:11,594] jepsen worker 27 - jepsen.util 10737 :ok :write [9 40] INFO [2019-12-17 12:37:11,594] jepsen worker 27 - jepsen.util 10737 :invoke :read [9 nil] INFO [2019-12-17 12:37:11,596] jepsen worker 64 - jepsen.util 8884 :ok :write [21 35] INFO [2019-12-17 12:37:11,596] jepsen worker 64 - jepsen.util 8884 :invoke :write [21 36] INFO [2019-12-17 12:37:11,632] jepsen worker 27 - jepsen.util 10737 :ok :read [9 40] INFO [2019-12-17 12:37:11,632] jepsen worker 27 - jepsen.util 10737 :invoke :read [9 nil] INFO [2019-12-17 12:37:11,632] jepsen worker 52 - jepsen.util 10942 :ok :read [17 1] INFO [2019-12-17 12:37:11,632] jepsen worker 52 - jepsen.util 10942 :invoke :read [17 nil] INFO [2019-12-17 12:37:11,643] jepsen worker 28 - jepsen.util 9658 :ok :read [9 40] INFO [2019-12-17 12:37:11,643] jepsen worker 28 - jepsen.util 9658 :invoke :cas [9 [13 29]] INFO [2019-12-17 12:37:11,652] jepsen worker 27 - jepsen.util 10737 :ok :read [9 40] INFO [2019-12-17 12:37:11,652] jepsen worker 27 - jepsen.util 10737 :invoke :read [9 nil] INFO [2019-12-17 12:37:11,673] jepsen worker 5 - jepsen.util 11345 :ok :read [1 48] INFO [2019-12-17 12:37:11,673] jepsen worker 5 - jepsen.util 11345 :invoke :cas [1 [14 22]] INFO [2019-12-17 12:37:11,674] jepsen worker 27 - jepsen.util 10737 :ok :read [9 40] INFO [2019-12-17 12:37:11,674] jepsen worker 27 - jepsen.util 10737 :invoke :write [9 13] INFO [2019-12-17 12:37:11,686] jepsen worker 64 - jepsen.util 8884 :ok :write [21 36] INFO [2019-12-17 12:37:11,686] jepsen worker 64 - jepsen.util 8884 :invoke :cas [21 [48 43]] INFO [2019-12-17 12:37:11,687] jepsen worker 71 - jepsen.util 12311 :fail :cas [23 [27 38]] :ValueNotSwapFrom INFO [2019-12-17 12:37:11,687] jepsen worker 71 - jepsen.util 12311 :invoke :write [23 47] INFO [2019-12-17 12:37:11,700] jepsen worker 89 - jepsen.util 11519 :ok :read [29 9] INFO [2019-12-17 12:37:11,700] jepsen worker 89 - jepsen.util 11519 :invoke :write [29 39] INFO [2019-12-17 12:37:11,709] jepsen worker 64 - jepsen.util 8884 :fail :cas [21 [48 43]] :ValueNotSwapFrom INFO [2019-12-17 12:37:11,710] jepsen worker 64 - jepsen.util 8884 :invoke :write [21 40] INFO [2019-12-17 12:37:11,711] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":67068,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:37:11,711] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":67068,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:37:11,716] jepsen worker 79 - jepsen.util 11779 :ok :read [26 35] INFO [2019-12-17 12:37:11,716] jepsen worker 79 - jepsen.util 11779 :invoke :read [26 nil] INFO [2019-12-17 12:37:11,732] jepsen worker 27 - jepsen.util 10737 :ok :write [9 13] INFO [2019-12-17 12:37:11,732] jepsen worker 27 - jepsen.util 10737 :invoke :cas [9 [16 5]] INFO [2019-12-17 12:37:11,739] jepsen worker 3 - jepsen.util 10983 :fail :cas [1 [39 28]] :ValueNotSwapFrom INFO [2019-12-17 12:37:11,739] jepsen worker 3 - jepsen.util 10983 :invoke :read [1 nil] INFO [2019-12-17 12:37:11,758] jepsen worker 27 - jepsen.util 10737 :fail :cas [9 [16 5]] :ValueNotSwapFrom INFO [2019-12-17 12:37:11,758] jepsen worker 27 - jepsen.util 10737 :invoke :cas [9 [1 27]] INFO [2019-12-17 12:37:11,763] jepsen worker 16 - jepsen.util 17476 :info :cas [5 [18 26]] :RequestCanceledException INFO [2019-12-17 12:37:11,763] jepsen worker 81 - jepsen.util 15381 :info :write [27 30] :RequestCanceledException INFO [2019-12-17 12:37:11,764] jepsen worker 16 - jepsen.util 17566 :invoke :cas [5 [21 27]] INFO [2019-12-17 12:37:11,764] jepsen worker 81 - jepsen.util 15471 :invoke :cas [27 [39 23]] INFO [2019-12-17 12:37:11,789] jepsen worker 13 - jepsen.util 10903 :ok :read [4 0] INFO [2019-12-17 12:37:11,789] jepsen worker 13 - jepsen.util 10903 :invoke :cas [4 [38 14]] INFO [2019-12-17 12:37:11,795] jepsen worker 27 - jepsen.util 10737 :fail :cas [9 [1 27]] :ValueNotSwapFrom INFO [2019-12-17 12:37:11,795] jepsen worker 27 - jepsen.util 10737 :invoke :cas [9 [3 14]] INFO [2019-12-17 12:37:11,825] jepsen worker 27 - jepsen.util 10737 :fail :cas [9 [3 14]] :ValueNotSwapFrom INFO [2019-12-17 12:37:11,825] jepsen worker 27 - jepsen.util 10737 :invoke :write [9 17] INFO [2019-12-17 12:37:11,833] jepsen worker 44 - jepsen.util 16334 :info :cas [14 [11 39]] :RequestCanceledException INFO [2019-12-17 12:37:11,833] jepsen worker 35 - jepsen.util 15695 :info :cas [11 [12 40]] :RequestCanceledException INFO [2019-12-17 12:37:11,835] jepsen worker 44 - jepsen.util 16424 :invoke :read [14 nil] INFO [2019-12-17 12:37:11,835] jepsen worker 35 - jepsen.util 15785 :invoke :read [11 nil] INFO [2019-12-17 12:37:11,835] jepsen worker 52 - jepsen.util 10942 :ok :read [17 1] INFO [2019-12-17 12:37:11,835] jepsen worker 52 - jepsen.util 10942 :invoke :read [17 nil] INFO [2019-12-17 12:37:11,850] jepsen worker 80 - jepsen.util 11240 :fail :cas [26 [15 44]] :ValueNotSwapFrom INFO [2019-12-17 12:37:11,850] jepsen worker 80 - jepsen.util 11240 :invoke :read [26 nil] INFO [2019-12-17 12:37:11,851] jepsen worker 11 - jepsen.util 11261 :ok :write [3 24] INFO [2019-12-17 12:37:11,851] jepsen worker 11 - jepsen.util 11261 :invoke :read [3 nil] INFO [2019-12-17 12:37:11,853] jepsen worker 46 - jepsen.util 11386 :ok :write [15 43] INFO [2019-12-17 12:37:11,853] jepsen worker 46 - jepsen.util 11386 :invoke :read [15 nil] INFO [2019-12-17 12:37:11,858] jepsen worker 18 - jepsen.util 11718 :ok :write [6 0] INFO [2019-12-17 12:37:11,858] jepsen worker 18 - jepsen.util 11718 :invoke :read [6 nil] INFO [2019-12-17 12:37:11,872] jepsen worker 64 - jepsen.util 8884 :ok :write [21 40] INFO [2019-12-17 12:37:11,872] jepsen worker 64 - jepsen.util 8884 :invoke :read [21 nil] INFO [2019-12-17 12:37:11,891] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":67095,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:37:11,891] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":67095,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:37:11,891] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":67098,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:37:11,891] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":67098,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:37:11,891] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":67099,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:37:11,891] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":67099,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:37:11,899] jepsen worker 18 - jepsen.util 11718 :ok :read [6 0] INFO [2019-12-17 12:37:11,899] jepsen worker 18 - jepsen.util 11718 :invoke :cas [6 [30 22]] INFO [2019-12-17 12:37:11,923] jepsen worker 56 - jepsen.util 12476 :fail :cas [18 [3 25]] :ValueNotSwapFrom INFO [2019-12-17 12:37:11,923] jepsen worker 56 - jepsen.util 12476 :invoke :write [18 40] INFO [2019-12-17 12:37:11,931] jepsen worker 64 - jepsen.util 8884 :ok :read [21 40] INFO [2019-12-17 12:37:11,932] jepsen worker 64 - jepsen.util 8884 :invoke :write [21 12] INFO [2019-12-17 12:37:11,933] jepsen worker 68 - jepsen.util 15098 :info :write [22 43] :RequestCanceledException INFO [2019-12-17 12:37:11,935] jepsen worker 68 - jepsen.util 15188 :invoke :write [22 29] INFO [2019-12-17 12:37:11,947] jepsen worker 6 - jepsen.util 16206 :info :write [2 14] :RequestCanceledException INFO [2019-12-17 12:37:11,948] jepsen worker 6 - jepsen.util 16296 :invoke :write [2 41] INFO [2019-12-17 12:37:11,960] jepsen worker 14 - jepsen.util 11264 :fail :cas [4 [42 37]] :ValueNotSwapFrom INFO [2019-12-17 12:37:11,961] jepsen worker 14 - jepsen.util 11264 :invoke :write [4 7] INFO [2019-12-17 12:37:11,961] jepsen worker 18 - jepsen.util 11718 :fail :cas [6 [30 22]] :ValueNotSwapFrom INFO [2019-12-17 12:37:11,961] jepsen worker 18 - jepsen.util 11718 :invoke :read [6 nil] INFO [2019-12-17 12:37:11,962] jepsen worker 78 - jepsen.util 11508 :fail :cas [26 [27 1]] :ValueNotSwapFrom INFO [2019-12-17 12:37:11,962] jepsen worker 78 - jepsen.util 11508 :invoke :read [26 nil] INFO [2019-12-17 12:37:11,963] jepsen worker 75 - jepsen.util 15375 :info :write [25 43] :RequestCanceledException INFO [2019-12-17 12:37:11,964] jepsen worker 75 - jepsen.util 15465 :invoke :write [25 26] INFO [2019-12-17 12:37:12,000] jepsen worker 41 - jepsen.util 16871 :info :write [13 11] :RequestCanceledException INFO [2019-12-17 12:37:12,001] jepsen worker 41 - jepsen.util 16961 :invoke :cas [13 [5 2]] INFO [2019-12-17 12:37:12,019] jepsen worker 79 - jepsen.util 11779 :ok :read [26 35] INFO [2019-12-17 12:37:12,019] jepsen worker 79 - jepsen.util 11779 :invoke :write [26 26] INFO [2019-12-17 12:37:12,022] jepsen worker 73 - jepsen.util 10693 :ok :write [24 26] INFO [2019-12-17 12:37:12,022] jepsen worker 73 - jepsen.util 10693 :invoke :write [24 8] INFO [2019-12-17 12:37:12,032] jepsen worker 18 - jepsen.util 11718 :ok :read [6 0] INFO [2019-12-17 12:37:12,032] jepsen worker 18 - jepsen.util 11718 :invoke :cas [6 [41 36]] INFO [2019-12-17 12:37:12,043] jepsen worker 11 - jepsen.util 11261 :ok :read [3 24] INFO [2019-12-17 12:37:12,043] jepsen worker 11 - jepsen.util 11261 :invoke :cas [3 [44 47]] INFO [2019-12-17 12:37:12,046] jepsen worker 19 - jepsen.util 11809 :info :cas [6 [15 18]] :RequestCanceledException INFO [2019-12-17 12:37:12,047] jepsen worker 19 - jepsen.util 11899 :invoke :write [6 30] INFO [2019-12-17 12:37:12,050] jepsen worker 74 - jepsen.util 11414 :fail :cas [24 [27 9]] :ValueNotSwapFrom INFO [2019-12-17 12:37:12,050] jepsen worker 74 - jepsen.util 11414 :invoke :read [24 nil] INFO [2019-12-17 12:37:12,105] jepsen worker 46 - jepsen.util 11386 :ok :read [15 43] INFO [2019-12-17 12:37:12,105] jepsen worker 46 - jepsen.util 11386 :invoke :write [15 43] INFO [2019-12-17 12:37:12,110] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":67111,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:37:12,110] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":67111,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:37:12,110] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":67114,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:37:12,110] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":67114,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:37:12,117] jepsen worker 42 - jepsen.util 16782 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:37:12,117] jepsen worker 42 - jepsen.util 16782 :invoke :cas [14 [15 17]] INFO [2019-12-17 12:37:12,117] jepsen worker 5 - jepsen.util 11345 :fail :cas [1 [14 22]] :ValueNotSwapFrom INFO [2019-12-17 12:37:12,117] jepsen worker 5 - jepsen.util 11345 :invoke :write [1 24] INFO [2019-12-17 12:37:12,126] jepsen worker 80 - jepsen.util 11240 :ok :read [26 35] INFO [2019-12-17 12:37:12,126] jepsen worker 80 - jepsen.util 11240 :invoke :write [26 16] INFO [2019-12-17 12:37:12,132] jepsen worker 1 - jepsen.util 15841 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:37:12,133] jepsen worker 1 - jepsen.util 15841 :invoke :cas [0 [0 5]] INFO [2019-12-17 12:37:12,133] jepsen worker 24 - jepsen.util 14514 :info :write [8 33] :RequestCanceledException INFO [2019-12-17 12:37:12,135] jepsen worker 24 - jepsen.util 14604 :invoke :write [8 7] INFO [2019-12-17 12:37:12,135] jepsen worker 11 - jepsen.util 11261 :fail :cas [3 [44 47]] :ValueNotSwapFrom INFO [2019-12-17 12:37:12,135] jepsen worker 11 - jepsen.util 11261 :invoke :write [3 13] INFO [2019-12-17 12:37:12,144] jepsen worker 74 - jepsen.util 11414 :ok :read [24 26] INFO [2019-12-17 12:37:12,144] jepsen worker 74 - jepsen.util 11414 :invoke :cas [24 [12 21]] INFO [2019-12-17 12:37:12,147] jepsen worker 39 - jepsen.util 15249 :info :cas [13 [31 45]] :RequestCanceledException INFO [2019-12-17 12:37:12,148] jepsen worker 39 - jepsen.util 15339 :invoke :read [13 nil] INFO [2019-12-17 12:37:12,150] jepsen worker 18 - jepsen.util 11718 :fail :cas [6 [41 36]] :ValueNotSwapFrom INFO [2019-12-17 12:37:12,150] jepsen worker 18 - jepsen.util 11718 :invoke :cas [6 [18 16]] INFO [2019-12-17 12:37:12,164] jepsen worker 40 - jepsen.util 16330 :info :write [13 47] :RequestCanceledException INFO [2019-12-17 12:37:12,164] jepsen worker 22 - jepsen.util 15862 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:37:12,164] jepsen worker 22 - jepsen.util 15862 :invoke :write [7 27] INFO [2019-12-17 12:37:12,165] jepsen worker 40 - jepsen.util 16420 :invoke :cas [13 [3 13]] INFO [2019-12-17 12:37:12,197] jepsen worker 27 - jepsen.util 10737 :ok :write [9 17] INFO [2019-12-17 12:37:12,198] jepsen worker 27 - jepsen.util 10737 :invoke :cas [9 [36 29]] INFO [2019-12-17 12:37:12,200] jepsen worker 83 - jepsen.util 15923 :info :cas [27 [26 26]] :RequestCanceledException INFO [2019-12-17 12:37:12,200] jepsen worker 60 - jepsen.util 17070 :info :write [20 7] :RequestCanceledException INFO [2019-12-17 12:37:12,202] jepsen worker 60 - jepsen.util 17160 :invoke :read [20 nil] INFO [2019-12-17 12:37:12,202] jepsen worker 83 - jepsen.util 16013 :invoke :cas [27 [49 17]] INFO [2019-12-17 12:37:12,207] jepsen worker 64 - jepsen.util 8884 :ok :write [21 12] INFO [2019-12-17 12:37:12,207] jepsen worker 64 - jepsen.util 8884 :invoke :write [21 34] INFO [2019-12-17 12:37:12,210] jepsen worker 18 - jepsen.util 11718 :fail :cas [6 [18 16]] :ValueNotSwapFrom INFO [2019-12-17 12:37:12,210] jepsen worker 18 - jepsen.util 11718 :invoke :cas [6 [5 40]] INFO [2019-12-17 12:37:12,218] jepsen worker 34 - jepsen.util 16234 :info :cas [11 [49 6]] :RequestCanceledException INFO [2019-12-17 12:37:12,219] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":67136,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:37:12,219] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":67136,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:37:12,219] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":67138,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:37:12,219] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":67138,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:37:12,229] jepsen worker 34 - jepsen.util 16324 :invoke :cas [11 [29 3]] INFO [2019-12-17 12:37:12,233] jepsen worker 49 - jepsen.util 15799 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:37:12,233] jepsen worker 49 - jepsen.util 15799 :invoke :cas [16 [8 48]] INFO [2019-12-17 12:37:12,233] jepsen worker 76 - jepsen.util 16006 :info :cas [25 [16 29]] :RequestCanceledException INFO [2019-12-17 12:37:12,233] jepsen worker 57 - jepsen.util 15627 :info :write [19 19] :RequestCanceledException INFO [2019-12-17 12:37:12,236] jepsen worker 76 - jepsen.util 16096 :invoke :cas [25 [40 1]] INFO [2019-12-17 12:37:12,236] jepsen worker 57 - jepsen.util 15717 :invoke :write [19 7] INFO [2019-12-17 12:37:12,248] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":67149,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:37:12,248] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":67149,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:37:12,249] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":67150,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:37:12,249] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":67150,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:37:12,249] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":67151,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:37:12,249] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":67151,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:37:12,252] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:37:12,252] jepsen nemesis - couchbase.util Rebalance check count: 528 INFO [2019-12-17 12:37:12,252] jepsen nemesis - couchbase.util Rebalance stuck count: 509 INFO [2019-12-17 12:37:12,264] jepsen worker 18 - jepsen.util 11718 :fail :cas [6 [5 40]] :ValueNotSwapFrom INFO [2019-12-17 12:37:12,264] jepsen worker 18 - jepsen.util 11718 :invoke :cas [6 [13 34]] INFO [2019-12-17 12:37:12,267] jepsen worker 27 - jepsen.util 10737 :fail :cas [9 [36 29]] :ValueNotSwapFrom INFO [2019-12-17 12:37:12,267] jepsen worker 27 - jepsen.util 10737 :invoke :cas [9 [26 41]] INFO [2019-12-17 12:37:12,318] jepsen worker 27 - jepsen.util 10737 :fail :cas [9 [26 41]] :ValueNotSwapFrom INFO [2019-12-17 12:37:12,319] jepsen worker 27 - jepsen.util 10737 :invoke :cas [9 [18 27]] INFO [2019-12-17 12:37:12,347] jepsen worker 4 - jepsen.util 10624 :info :write [1 45] :RequestCanceledException INFO [2019-12-17 12:37:12,348] jepsen worker 4 - jepsen.util 10714 :invoke :read [1 nil] INFO [2019-12-17 12:37:12,352] jepsen worker 64 - jepsen.util 8884 :ok :write [21 34] INFO [2019-12-17 12:37:12,352] jepsen worker 64 - jepsen.util 8884 :invoke :cas [21 [49 8]] INFO [2019-12-17 12:37:12,360] jepsen worker 18 - jepsen.util 11718 :fail :cas [6 [13 34]] :ValueNotSwapFrom INFO [2019-12-17 12:37:12,360] jepsen worker 18 - jepsen.util 11718 :invoke :cas [6 [32 8]] INFO [2019-12-17 12:37:12,364] jepsen worker 27 - jepsen.util 10737 :fail :cas [9 [18 27]] :ValueNotSwapFrom INFO [2019-12-17 12:37:12,365] jepsen worker 27 - jepsen.util 10737 :invoke :read [9 nil] INFO [2019-12-17 12:37:12,398] jepsen worker 46 - jepsen.util 11386 :ok :write [15 43] INFO [2019-12-17 12:37:12,398] jepsen worker 73 - jepsen.util 10693 :ok :write [24 8] INFO [2019-12-17 12:37:12,399] jepsen worker 46 - jepsen.util 11386 :invoke :read [15 nil] INFO [2019-12-17 12:37:12,399] jepsen worker 73 - jepsen.util 10693 :invoke :cas [24 [42 3]] INFO [2019-12-17 12:37:12,417] jepsen worker 11 - jepsen.util 11261 :ok :write [3 13] INFO [2019-12-17 12:37:12,417] jepsen worker 11 - jepsen.util 11261 :invoke :write [3 16] INFO [2019-12-17 12:37:12,418] jepsen worker 53 - jepsen.util 11033 :info :write [17 41] :RequestCanceledException INFO [2019-12-17 12:37:12,421] jepsen worker 53 - jepsen.util 11123 :invoke :write [17 1] INFO [2019-12-17 12:37:12,433] jepsen worker 70 - jepsen.util 12040 :info :write [23 31] :RequestCanceledException INFO [2019-12-17 12:37:12,436] jepsen worker 3 - jepsen.util 10983 :ok :read [1 48] INFO [2019-12-17 12:37:12,436] jepsen worker 3 - jepsen.util 10983 :invoke :cas [1 [32 13]] INFO [2019-12-17 12:37:12,448] jepsen worker 27 - jepsen.util 10737 :ok :read [9 17] INFO [2019-12-17 12:37:12,448] jepsen worker 32 - jepsen.util 11912 :info :cas [10 [46 15]] :RequestCanceledException INFO [2019-12-17 12:37:12,448] jepsen worker 27 - jepsen.util 10737 :invoke :read [9 nil] INFO [2019-12-17 12:37:12,448] jepsen worker 70 - jepsen.util 12130 :invoke :read [23 nil] INFO [2019-12-17 12:37:12,451] jepsen worker 32 - jepsen.util 12002 :invoke :read [10 nil] INFO [2019-12-17 12:37:12,455] jepsen worker 64 - jepsen.util 8884 :fail :cas [21 [49 8]] :ValueNotSwapFrom INFO [2019-12-17 12:37:12,455] jepsen worker 64 - jepsen.util 8884 :invoke :write [21 25] INFO [2019-12-17 12:37:12,464] jepsen worker 18 - jepsen.util 11718 :fail :cas [6 [32 8]] :ValueNotSwapFrom INFO [2019-12-17 12:37:12,465] jepsen worker 18 - jepsen.util 11718 :invoke :read [6 nil] INFO [2019-12-17 12:37:12,468] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":67166,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:37:12,468] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":67166,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:37:12,468] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":67170,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:37:12,469] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":67170,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:37:12,495] jepsen worker 73 - jepsen.util 10693 :fail :cas [24 [42 3]] :ValueNotSwapFrom INFO [2019-12-17 12:37:12,495] jepsen worker 73 - jepsen.util 10693 :invoke :write [24 13] INFO [2019-12-17 12:37:12,497] jepsen worker 46 - jepsen.util 11386 :ok :read [15 43] INFO [2019-12-17 12:37:12,498] jepsen worker 46 - jepsen.util 11386 :invoke :read [15 nil] INFO [2019-12-17 12:37:12,520] jepsen worker 27 - jepsen.util 10737 :ok :read [9 17] INFO [2019-12-17 12:37:12,520] jepsen worker 27 - jepsen.util 10737 :invoke :read [9 nil] INFO [2019-12-17 12:37:12,534] jepsen worker 33 - jepsen.util 15603 :info :cas [11 [47 38]] :RequestCanceledException INFO [2019-12-17 12:37:12,536] jepsen worker 13 - jepsen.util 10903 :fail :cas [4 [38 14]] :ValueNotSwapFrom INFO [2019-12-17 12:37:12,536] jepsen worker 13 - jepsen.util 10903 :invoke :cas [4 [41 13]] INFO [2019-12-17 12:37:12,542] jepsen worker 33 - jepsen.util 15693 :invoke :read [11 nil] INFO [2019-12-17 12:37:12,547] jepsen worker 32 - jepsen.util 12002 :ok :read [10 31] INFO [2019-12-17 12:37:12,547] jepsen worker 32 - jepsen.util 12002 :invoke :write [10 30] INFO [2019-12-17 12:37:12,571] jepsen worker 64 - jepsen.util 8884 :ok :write [21 25] INFO [2019-12-17 12:37:12,572] jepsen worker 64 - jepsen.util 8884 :invoke :cas [21 [33 10]] INFO [2019-12-17 12:37:12,585] jepsen worker 18 - jepsen.util 11718 :ok :read [6 0] INFO [2019-12-17 12:37:12,585] jepsen worker 18 - jepsen.util 11718 :invoke :cas [6 [15 4]] INFO [2019-12-17 12:37:12,600] jepsen worker 88 - jepsen.util 11698 :info :write [29 20] :RequestCanceledException INFO [2019-12-17 12:37:12,601] jepsen worker 88 - jepsen.util 11788 :invoke :read [29 nil] INFO [2019-12-17 12:37:12,618] jepsen worker 54 - jepsen.util 12024 :info :write [18 44] :RequestCanceledException INFO [2019-12-17 12:37:12,629] jepsen worker 27 - jepsen.util 10737 :ok :read [9 17] INFO [2019-12-17 12:37:12,629] jepsen worker 27 - jepsen.util 10737 :invoke :cas [9 [3 29]] INFO [2019-12-17 12:37:12,630] jepsen worker 54 - jepsen.util 12114 :invoke :cas [18 [4 48]] INFO [2019-12-17 12:37:12,654] jepsen worker 46 - jepsen.util 11386 :ok :read [15 43] INFO [2019-12-17 12:37:12,654] jepsen worker 46 - jepsen.util 11386 :invoke :cas [15 [37 13]] INFO [2019-12-17 12:37:12,690] jepsen worker 52 - jepsen.util 10942 :ok :read [17 1] INFO [2019-12-17 12:37:12,691] jepsen worker 52 - jepsen.util 10942 :invoke :cas [17 [0 30]] INFO [2019-12-17 12:37:12,698] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":67191,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:37:12,699] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":67191,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:37:12,699] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":67194,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:37:12,699] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":67194,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:37:12,699] jepsen worker 31 - jepsen.util 9841 :info :write [10 40] :RequestCanceledException INFO [2019-12-17 12:37:12,699] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":67195,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:37:12,700] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":67195,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:37:12,706] jepsen worker 31 - jepsen.util 9931 :invoke :cas [10 [7 12]] INFO [2019-12-17 12:37:12,716] jepsen worker 78 - jepsen.util 11508 :ok :read [26 35] INFO [2019-12-17 12:37:12,716] jepsen worker 78 - jepsen.util 11508 :invoke :cas [26 [13 36]] INFO [2019-12-17 12:37:12,720] jepsen worker 64 - jepsen.util 8884 :fail :cas [21 [33 10]] :ValueNotSwapFrom INFO [2019-12-17 12:37:12,720] jepsen worker 64 - jepsen.util 8884 :invoke :cas [21 [41 30]] INFO [2019-12-17 12:37:12,725] jepsen worker 18 - jepsen.util 11718 :fail :cas [6 [15 4]] :ValueNotSwapFrom INFO [2019-12-17 12:37:12,725] jepsen worker 18 - jepsen.util 11718 :invoke :cas [6 [38 8]] INFO [2019-12-17 12:37:12,742] jepsen worker 27 - jepsen.util 10737 :fail :cas [9 [3 29]] :ValueNotSwapFrom INFO [2019-12-17 12:37:12,742] jepsen worker 27 - jepsen.util 10737 :invoke :cas [9 [30 22]] INFO [2019-12-17 12:37:12,745] jepsen worker 46 - jepsen.util 11386 :fail :cas [15 [37 13]] :ValueNotSwapFrom INFO [2019-12-17 12:37:12,745] jepsen worker 46 - jepsen.util 11386 :invoke :cas [15 [9 48]] INFO [2019-12-17 12:37:12,768] jepsen worker 3 - jepsen.util 10983 :fail :cas [1 [32 13]] :ValueNotSwapFrom INFO [2019-12-17 12:37:12,768] jepsen worker 3 - jepsen.util 10983 :invoke :read [1 nil] INFO [2019-12-17 12:37:12,769] jepsen worker 31 - jepsen.util 9931 :fail :cas [10 [7 12]] :ValueNotSwapFrom INFO [2019-12-17 12:37:12,769] jepsen worker 31 - jepsen.util 9931 :invoke :write [10 21] INFO [2019-12-17 12:37:12,779] jepsen worker 78 - jepsen.util 11508 :fail :cas [26 [13 36]] :ValueNotSwapFrom INFO [2019-12-17 12:37:12,779] jepsen worker 78 - jepsen.util 11508 :invoke :read [26 nil] INFO [2019-12-17 12:37:12,837] jepsen worker 3 - jepsen.util 10983 :ok :read [1 48] INFO [2019-12-17 12:37:12,838] jepsen worker 3 - jepsen.util 10983 :invoke :cas [1 [16 28]] INFO [2019-12-17 12:37:12,846] jepsen worker 64 - jepsen.util 8884 :fail :cas [21 [41 30]] :ValueNotSwapFrom INFO [2019-12-17 12:37:12,846] jepsen worker 64 - jepsen.util 8884 :invoke :read [21 nil] INFO [2019-12-17 12:37:12,848] jepsen worker 18 - jepsen.util 11718 :fail :cas [6 [38 8]] :ValueNotSwapFrom INFO [2019-12-17 12:37:12,848] jepsen worker 18 - jepsen.util 11718 :invoke :read [6 nil] INFO [2019-12-17 12:37:12,862] jepsen worker 78 - jepsen.util 11508 :ok :read [26 35] INFO [2019-12-17 12:37:12,862] jepsen worker 78 - jepsen.util 11508 :invoke :read [26 nil] INFO [2019-12-17 12:37:12,863] jepsen worker 45 - jepsen.util 11655 :info :write [15 49] :RequestCanceledException INFO [2019-12-17 12:37:12,863] jepsen worker 29 - jepsen.util 9839 :info :write [9 0] :RequestCanceledException INFO [2019-12-17 12:37:12,864] jepsen worker 45 - jepsen.util 11745 :invoke :write [15 46] INFO [2019-12-17 12:37:12,866] jepsen worker 29 - jepsen.util 9929 :invoke :write [9 47] INFO [2019-12-17 12:37:12,878] jepsen worker 27 - jepsen.util 10737 :fail :cas [9 [30 22]] :ValueNotSwapFrom INFO [2019-12-17 12:37:12,878] jepsen worker 27 - jepsen.util 10737 :invoke :write [9 0] INFO [2019-12-17 12:37:12,879] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":67212,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:37:12,880] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":67212,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:37:12,880] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":67215,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:37:12,880] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":67215,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:37:12,900] jepsen worker 46 - jepsen.util 11386 :fail :cas [15 [9 48]] :ValueNotSwapFrom INFO [2019-12-17 12:37:12,900] jepsen worker 46 - jepsen.util 11386 :invoke :write [15 25] INFO [2019-12-17 12:37:12,910] jepsen worker 80 - jepsen.util 11240 :ok :write [26 16] INFO [2019-12-17 12:37:12,910] jepsen worker 80 - jepsen.util 11240 :invoke :cas [26 [8 16]] INFO [2019-12-17 12:37:12,915] jepsen worker 72 - jepsen.util 11952 :fail :read [24 nil] :RequestTimeoutException INFO [2019-12-17 12:37:12,924] jepsen worker 64 - jepsen.util 8884 :ok :read [21 25] INFO [2019-12-17 12:37:12,944] jepsen worker 4 - jepsen.util 10714 :ok :read [1 48] INFO [2019-12-17 12:37:12,946] jepsen worker 11 - jepsen.util 11261 :ok :write [3 16] INFO [2019-12-17 12:37:12,982] jepsen worker 70 - jepsen.util 12130 :ok :read [23 41] INFO [2019-12-17 12:37:13,000] jepsen worker 37 - jepsen.util 11107 :info :write [12 24] :RequestCanceledException INFO [2019-12-17 12:37:13,008] jepsen worker 80 - jepsen.util 11240 :fail :cas [26 [8 16]] :ValueNotSwapFrom INFO [2019-12-17 12:37:13,012] jepsen worker 72 - jepsen.util 11952 :invoke :cas [24 [38 13]] INFO [2019-12-17 12:37:13,012] jepsen worker 64 - jepsen.util 8884 :invoke :read [21 nil] INFO [2019-12-17 12:37:13,012] jepsen worker 4 - jepsen.util 10714 :invoke :write [1 24] INFO [2019-12-17 12:37:13,012] jepsen worker 11 - jepsen.util 11261 :invoke :write [3 20] INFO [2019-12-17 12:37:13,012] jepsen worker 70 - jepsen.util 12130 :invoke :cas [23 [23 12]] INFO [2019-12-17 12:37:13,012] jepsen worker 80 - jepsen.util 11240 :invoke :write [26 38] INFO [2019-12-17 12:37:13,013] jepsen worker 13 - jepsen.util 10903 :fail :cas [4 [41 13]] :ValueNotSwapFrom INFO [2019-12-17 12:37:13,013] jepsen worker 13 - jepsen.util 10903 :invoke :read [4 nil] INFO [2019-12-17 12:37:13,015] jepsen worker 37 - jepsen.util 11197 :invoke :read [12 nil] INFO [2019-12-17 12:37:13,033] jepsen worker 69 - jepsen.util 11769 :info :write [23 40] :RequestCanceledException INFO [2019-12-17 12:37:13,039] jepsen worker 69 - jepsen.util 11859 :invoke :write [23 46] INFO [2019-12-17 12:37:13,064] jepsen worker 66 - jepsen.util 15726 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:37:13,064] jepsen worker 66 - jepsen.util 15726 :invoke :write [22 20] INFO [2019-12-17 12:37:13,083] jepsen worker 31 - jepsen.util 9931 :ok :write [10 21] INFO [2019-12-17 12:37:13,083] jepsen worker 31 - jepsen.util 9931 :invoke :read [10 nil] INFO [2019-12-17 12:37:13,084] jepsen worker 73 - jepsen.util 10693 :ok :write [24 13] INFO [2019-12-17 12:37:13,084] jepsen worker 73 - jepsen.util 10693 :invoke :read [24 nil] INFO [2019-12-17 12:37:13,090] jepsen worker 54 - jepsen.util 12114 :fail :cas [18 [4 48]] :ValueNotSwapFrom INFO [2019-12-17 12:37:13,090] jepsen worker 54 - jepsen.util 12114 :invoke :read [18 nil] INFO [2019-12-17 12:37:13,094] jepsen worker 64 - jepsen.util 8884 :ok :read [21 25] INFO [2019-12-17 12:37:13,094] jepsen worker 64 - jepsen.util 8884 :invoke :read [21 nil] INFO [2019-12-17 12:37:13,100] jepsen worker 62 - jepsen.util 16532 :info :cas [20 [33 41]] :RequestCanceledException INFO [2019-12-17 12:37:13,100] jepsen worker 87 - jepsen.util 11067 :info :write [29 14] :RequestCanceledException INFO [2019-12-17 12:37:13,100] jepsen worker 72 - jepsen.util 11952 :fail :cas [24 [38 13]] :ValueNotSwapFrom INFO [2019-12-17 12:37:13,100] jepsen worker 72 - jepsen.util 11952 :invoke :read [24 nil] INFO [2019-12-17 12:37:13,100] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":67223,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:37:13,100] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":67223,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:37:13,101] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":67226,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:37:13,101] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":67226,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:37:13,101] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":67227,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:37:13,101] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":67227,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:37:13,102] jepsen worker 87 - jepsen.util 11157 :invoke :cas [29 [9 17]] INFO [2019-12-17 12:37:13,103] jepsen worker 62 - jepsen.util 16622 :invoke :write [20 2] INFO [2019-12-17 12:37:13,108] jepsen worker 52 - jepsen.util 10942 :fail :cas [17 [0 30]] :ValueNotSwapFrom INFO [2019-12-17 12:37:13,108] jepsen worker 52 - jepsen.util 10942 :invoke :read [17 nil] INFO [2019-12-17 12:37:13,117] jepsen worker 17 - jepsen.util 15857 :info :cas [5 [12 4]] :RequestCanceledException INFO [2019-12-17 12:37:13,119] jepsen worker 17 - jepsen.util 15947 :invoke :read [5 nil] INFO [2019-12-17 12:37:13,132] jepsen worker 82 - jepsen.util 16102 :info :write [27 27] :RequestCanceledException INFO [2019-12-17 12:37:13,132] jepsen worker 77 - jepsen.util 16637 :info :cas [25 [46 21]] :RequestCanceledException INFO [2019-12-17 12:37:13,132] jepsen worker 26 - jepsen.util 16586 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:37:13,132] jepsen worker 26 - jepsen.util 16586 :invoke :write [8 7] INFO [2019-12-17 12:37:13,135] jepsen worker 82 - jepsen.util 16192 :invoke :write [27 27] INFO [2019-12-17 12:37:13,136] jepsen worker 77 - jepsen.util 16727 :invoke :cas [25 [28 11]] INFO [2019-12-17 12:37:13,146] jepsen worker 84 - jepsen.util 15654 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:37:13,146] jepsen worker 84 - jepsen.util 15654 :invoke :read [28 nil] INFO [2019-12-17 12:37:13,160] jepsen worker 18 - jepsen.util 11718 :ok :read [6 0] INFO [2019-12-17 12:37:13,160] jepsen worker 18 - jepsen.util 11718 :invoke :write [6 28] INFO [2019-12-17 12:37:13,164] jepsen worker 2 - jepsen.util 15392 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:37:13,164] jepsen worker 2 - jepsen.util 15392 :invoke :cas [0 [18 40]] INFO [2019-12-17 12:37:13,186] jepsen worker 64 - jepsen.util 8884 :ok :read [21 25] INFO [2019-12-17 12:37:13,187] jepsen worker 64 - jepsen.util 8884 :invoke :read [21 nil] INFO [2019-12-17 12:37:13,192] jepsen worker 37 - jepsen.util 11197 :ok :read [12 7] INFO [2019-12-17 12:37:13,192] jepsen worker 37 - jepsen.util 11197 :invoke :cas [12 [1 11]] INFO [2019-12-17 12:37:13,193] jepsen worker 31 - jepsen.util 9931 :ok :read [10 21] INFO [2019-12-17 12:37:13,193] jepsen worker 31 - jepsen.util 9931 :invoke :write [10 34] INFO [2019-12-17 12:37:13,199] jepsen worker 23 - jepsen.util 15953 :info :cas [7 [23 49]] :RequestCanceledException INFO [2019-12-17 12:37:13,201] jepsen worker 23 - jepsen.util 16043 :invoke :cas [7 [39 16]] INFO [2019-12-17 12:37:13,214] jepsen worker 70 - jepsen.util 12130 :fail :cas [23 [23 12]] :ValueNotSwapFrom INFO [2019-12-17 12:37:13,215] jepsen worker 70 - jepsen.util 12130 :invoke :write [23 17] INFO [2019-12-17 12:37:13,217] jepsen worker 25 - jepsen.util 16855 :info :cas [8 [30 11]] :RequestCanceledException INFO [2019-12-17 12:37:13,219] jepsen worker 25 - jepsen.util 16945 :invoke :write [8 41] INFO [2019-12-17 12:37:13,232] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":67267,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:37:13,232] jepsen worker 67 - jepsen.util 15907 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:37:13,232] jepsen worker 7 - jepsen.util 15847 :info :write [2 36] :RequestCanceledException INFO [2019-12-17 12:37:13,232] jepsen worker 86 - jepsen.util 16556 :info :cas [28 [39 19]] :RequestCanceledException INFO [2019-12-17 12:37:13,232] jepsen worker 48 - jepsen.util 14628 :info :write [16 45] :RequestCanceledException INFO [2019-12-17 12:37:13,232] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":67267,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:37:13,232] jepsen worker 67 - jepsen.util 15907 :invoke :read [22 nil] INFO [2019-12-17 12:37:13,234] jepsen worker 48 - jepsen.util 14718 :invoke :cas [16 [44 47]] INFO [2019-12-17 12:37:13,234] jepsen worker 7 - jepsen.util 15937 :invoke :write [2 36] INFO [2019-12-17 12:37:13,234] jepsen worker 86 - jepsen.util 16646 :invoke :write [28 22] INFO [2019-12-17 12:37:13,247] jepsen worker 30 - jepsen.util 10650 :fail :read [10 nil] :RequestTimeoutException INFO [2019-12-17 12:37:13,247] jepsen worker 30 - jepsen.util 10650 :invoke :read [10 nil] INFO [2019-12-17 12:37:13,247] jepsen worker 58 - jepsen.util 16168 :info :cas [19 [7 37]] :RequestCanceledException INFO [2019-12-17 12:37:13,249] jepsen worker 58 - jepsen.util 16258 :invoke :write [19 46] INFO [2019-12-17 12:37:13,257] jepsen worker 64 - jepsen.util 8884 :ok :read [21 25] INFO [2019-12-17 12:37:13,257] jepsen worker 64 - jepsen.util 8884 :invoke :write [21 44] INFO [2019-12-17 12:37:13,264] jepsen worker 21 - jepsen.util 15321 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:37:13,264] jepsen worker 10 - jepsen.util 11170 :info :write [3 34] :RequestCanceledException INFO [2019-12-17 12:37:13,264] jepsen worker 21 - jepsen.util 15321 :invoke :cas [7 [25 27]] INFO [2019-12-17 12:37:13,266] jepsen worker 10 - jepsen.util 11260 :invoke :read [3 nil] INFO [2019-12-17 12:37:13,269] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:37:13,269] jepsen nemesis - couchbase.util Rebalance check count: 529 INFO [2019-12-17 12:37:13,269] jepsen nemesis - couchbase.util Rebalance stuck count: 510 INFO [2019-12-17 12:37:13,270] jepsen worker 11 - jepsen.util 11261 :ok :write [3 20] INFO [2019-12-17 12:37:13,271] jepsen worker 11 - jepsen.util 11261 :invoke :write [3 7] INFO [2019-12-17 12:37:13,271] jepsen worker 45 - jepsen.util 11745 :ok :write [15 46] INFO [2019-12-17 12:37:13,271] jepsen worker 45 - jepsen.util 11745 :invoke :cas [15 [16 30]] INFO [2019-12-17 12:37:13,295] jepsen worker 30 - jepsen.util 10650 :ok :read [10 21] INFO [2019-12-17 12:37:13,296] jepsen worker 30 - jepsen.util 10650 :invoke :write [10 37] INFO [2019-12-17 12:37:13,300] jepsen worker 0 - jepsen.util 16560 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:37:13,300] jepsen worker 50 - jepsen.util 16160 :info :cas [16 [19 3]] :RequestCanceledException INFO [2019-12-17 12:37:13,300] jepsen worker 0 - jepsen.util 16560 :invoke :cas [0 [27 38]] INFO [2019-12-17 12:37:13,300] jepsen worker 85 - jepsen.util 16465 :info :write [28 10] :RequestCanceledException INFO [2019-12-17 12:37:13,300] jepsen worker 43 - jepsen.util 15973 :info :cas [14 [36 28]] :RequestCanceledException INFO [2019-12-17 12:37:13,305] jepsen worker 85 - jepsen.util 16555 :invoke :cas [28 [17 40]] INFO [2019-12-17 12:37:13,306] jepsen worker 50 - jepsen.util 16250 :invoke :read [16 nil] INFO [2019-12-17 12:37:13,306] jepsen worker 43 - jepsen.util 16063 :invoke :cas [14 [24 20]] INFO [2019-12-17 12:37:13,309] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":67257,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:37:13,309] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":67257,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:37:13,309] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":67258,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:37:13,310] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":67258,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:37:13,318] jepsen worker 59 - jepsen.util 15089 :info :cas [19 [47 18]] :RequestCanceledException INFO [2019-12-17 12:37:13,318] jepsen worker 8 - jepsen.util 15578 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:37:13,318] jepsen worker 8 - jepsen.util 15578 :invoke :write [2 30] INFO [2019-12-17 12:37:13,320] jepsen worker 3 - jepsen.util 10983 :fail :cas [1 [16 28]] :ValueNotSwapFrom INFO [2019-12-17 12:37:13,320] jepsen worker 3 - jepsen.util 10983 :invoke :write [1 26] INFO [2019-12-17 12:37:13,326] jepsen worker 59 - jepsen.util 15179 :invoke :cas [19 [33 24]] INFO [2019-12-17 12:37:13,333] jepsen worker 36 - jepsen.util 12276 :info :write [12 13] :RequestCanceledException INFO [2019-12-17 12:37:13,333] jepsen worker 61 - jepsen.util 17071 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:37:13,333] jepsen worker 61 - jepsen.util 17071 :invoke :write [20 39] INFO [2019-12-17 12:37:13,334] jepsen worker 38 - jepsen.util 12008 :info :write [12 29] :RequestCanceledException INFO [2019-12-17 12:37:13,334] jepsen worker 15 - jepsen.util 14685 :info :write [5 14] :RequestCanceledException INFO [2019-12-17 12:37:13,341] jepsen worker 36 - jepsen.util 12366 :invoke :write [12 16] INFO [2019-12-17 12:37:13,346] jepsen worker 38 - jepsen.util 12098 :invoke :read [12 nil] INFO [2019-12-17 12:37:13,346] jepsen worker 20 - jepsen.util 11180 :fail :read [6 nil] :RequestTimeoutException INFO [2019-12-17 12:37:13,347] jepsen worker 9 - jepsen.util 12249 :fail :read [3 nil] :RequestTimeoutException INFO [2019-12-17 12:37:13,347] jepsen worker 20 - jepsen.util 11180 :invoke :write [6 34] INFO [2019-12-17 12:37:13,347] jepsen worker 9 - jepsen.util 12249 :invoke :read [3 nil] INFO [2019-12-17 12:37:13,347] jepsen worker 15 - jepsen.util 14775 :invoke :write [5 37] INFO [2019-12-17 12:37:13,379] jepsen worker 13 - jepsen.util 10903 :ok :read [4 0] INFO [2019-12-17 12:37:13,379] jepsen worker 13 - jepsen.util 10903 :invoke :read [4 nil] INFO [2019-12-17 12:37:13,408] jepsen worker 37 - jepsen.util 11197 :fail :cas [12 [1 11]] :ValueNotSwapFrom INFO [2019-12-17 12:37:13,408] jepsen worker 37 - jepsen.util 11197 :invoke :write [12 15] INFO [2019-12-17 12:37:13,410] jepsen worker 72 - jepsen.util 11952 :ok :read [24 13] INFO [2019-12-17 12:37:13,410] jepsen worker 72 - jepsen.util 11952 :invoke :read [24 nil] INFO [2019-12-17 12:37:13,410] jepsen worker 10 - jepsen.util 11260 :ok :read [3 20] INFO [2019-12-17 12:37:13,411] jepsen worker 10 - jepsen.util 11260 :invoke :write [3 14] INFO [2019-12-17 12:37:13,433] jepsen worker 45 - jepsen.util 11745 :fail :cas [15 [16 30]] :ValueNotSwapFrom INFO [2019-12-17 12:37:13,433] jepsen worker 45 - jepsen.util 11745 :invoke :cas [15 [6 22]] INFO [2019-12-17 12:37:13,447] jepsen worker 55 - jepsen.util 12115 :info :write [18 33] :RequestCanceledException INFO [2019-12-17 12:37:13,448] jepsen worker 55 - jepsen.util 12205 :invoke :write [18 14] INFO [2019-12-17 12:37:13,456] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":67303,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:37:13,456] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":67303,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:37:13,457] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":67305,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:37:13,457] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":67305,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:37:13,457] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":67306,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:37:13,457] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":67306,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:37:13,465] jepsen worker 64 - jepsen.util 8884 :ok :write [21 44] INFO [2019-12-17 12:37:13,465] jepsen worker 64 - jepsen.util 8884 :invoke :read [21 nil] INFO [2019-12-17 12:37:13,490] jepsen worker 78 - jepsen.util 11508 :ok :read [26 16] INFO [2019-12-17 12:37:13,490] jepsen worker 78 - jepsen.util 11508 :invoke :write [26 44] INFO [2019-12-17 12:37:13,518] jepsen worker 65 - jepsen.util 10235 :info :cas [21 [33 47]] :RequestCanceledException INFO [2019-12-17 12:37:13,520] jepsen worker 65 - jepsen.util 10325 :invoke :cas [21 [0 35]] INFO [2019-12-17 12:37:13,534] jepsen worker 54 - jepsen.util 12114 :ok :read [18 16] INFO [2019-12-17 12:37:13,534] jepsen worker 54 - jepsen.util 12114 :invoke :write [18 6] INFO [2019-12-17 12:37:13,551] jepsen worker 88 - jepsen.util 11788 :ok :read [29 9] INFO [2019-12-17 12:37:13,551] jepsen worker 88 - jepsen.util 11788 :invoke :read [29 nil] INFO [2019-12-17 12:37:13,590] jepsen worker 9 - jepsen.util 12249 :ok :read [3 20] INFO [2019-12-17 12:37:13,591] jepsen worker 9 - jepsen.util 12249 :invoke :write [3 31] INFO [2019-12-17 12:37:13,596] jepsen worker 45 - jepsen.util 11745 :fail :cas [15 [6 22]] :ValueNotSwapFrom INFO [2019-12-17 12:37:13,597] jepsen worker 45 - jepsen.util 11745 :invoke :write [15 19] INFO [2019-12-17 12:37:13,599] jepsen worker 63 - jepsen.util 10143 :info :write [21 5] :RequestCanceledException INFO [2019-12-17 12:37:13,600] jepsen worker 47 - jepsen.util 12377 :info :write [15 15] :RequestCanceledException INFO [2019-12-17 12:37:13,605] jepsen worker 47 - jepsen.util 12467 :invoke :write [15 20] INFO [2019-12-17 12:37:13,605] jepsen worker 63 - jepsen.util 10233 :invoke :cas [21 [37 3]] INFO [2019-12-17 12:37:13,615] jepsen worker 65 - jepsen.util 10325 :fail :cas [21 [0 35]] :ValueNotSwapFrom INFO [2019-12-17 12:37:13,615] jepsen worker 65 - jepsen.util 10325 :invoke :read [21 nil] INFO [2019-12-17 12:37:13,618] jepsen worker 51 - jepsen.util 10851 :info :write [17 20] :RequestCanceledException INFO [2019-12-17 12:37:13,625] jepsen worker 51 - jepsen.util 10941 :invoke :cas [17 [10 31]] INFO [2019-12-17 12:37:13,634] jepsen worker 12 - jepsen.util 11712 :info :write [4 9] :RequestCanceledException INFO [2019-12-17 12:37:13,639] jepsen worker 12 - jepsen.util 11802 :invoke :read [4 nil] INFO [2019-12-17 12:37:13,669] jepsen worker 88 - jepsen.util 11788 :ok :read [29 9] INFO [2019-12-17 12:37:13,670] jepsen worker 88 - jepsen.util 11788 :invoke :write [29 36] INFO [2019-12-17 12:37:13,680] jepsen worker 52 - jepsen.util 10942 :ok :read [17 1] INFO [2019-12-17 12:37:13,681] jepsen worker 52 - jepsen.util 10942 :invoke :write [17 35] INFO [2019-12-17 12:37:13,685] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":67323,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:37:13,685] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":67323,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:37:13,709] jepsen worker 51 - jepsen.util 10941 :fail :cas [17 [10 31]] :ValueNotSwapFrom INFO [2019-12-17 12:37:13,709] jepsen worker 51 - jepsen.util 10941 :invoke :read [17 nil] INFO [2019-12-17 12:37:13,717] jepsen worker 71 - jepsen.util 12311 :info :write [23 47] :RequestCanceledException INFO [2019-12-17 12:37:13,719] jepsen worker 71 - jepsen.util 12401 :invoke :read [23 nil] INFO [2019-12-17 12:37:13,733] jepsen worker 28 - jepsen.util 9658 :info :cas [9 [13 29]] :RequestCanceledException INFO [2019-12-17 12:37:13,734] jepsen worker 28 - jepsen.util 9748 :invoke :read [9 nil] INFO [2019-12-17 12:37:13,764] jepsen worker 89 - jepsen.util 11519 :info :write [29 39] :RequestCanceledException INFO [2019-12-17 12:37:13,765] jepsen worker 89 - jepsen.util 11609 :invoke :write [29 23] INFO [2019-12-17 12:37:13,813] jepsen worker 13 - jepsen.util 10903 :ok :read [4 0] INFO [2019-12-17 12:37:13,813] jepsen worker 13 - jepsen.util 10903 :invoke :write [4 34] INFO [2019-12-17 12:37:13,819] jepsen worker 12 - jepsen.util 11802 :ok :read [4 0] INFO [2019-12-17 12:37:13,819] jepsen worker 12 - jepsen.util 11802 :invoke :write [4 17] INFO [2019-12-17 12:37:13,832] jepsen worker 81 - jepsen.util 15471 :info :cas [27 [39 23]] :RequestCanceledException INFO [2019-12-17 12:37:13,834] jepsen worker 81 - jepsen.util 15561 :invoke :read [27 nil] INFO [2019-12-17 12:37:13,864] jepsen worker 16 - jepsen.util 17566 :info :cas [5 [21 27]] :RequestCanceledException INFO [2019-12-17 12:37:13,867] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":67337,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:37:13,868] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":67337,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:37:13,868] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":67341,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:37:13,869] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":67341,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:37:13,871] jepsen worker 16 - jepsen.util 17656 :invoke :cas [5 [46 5]] INFO [2019-12-17 12:37:13,877] jepsen worker 38 - jepsen.util 12098 :ok :read [12 7] INFO [2019-12-17 12:37:13,878] jepsen worker 38 - jepsen.util 12098 :invoke :cas [12 [16 39]] INFO [2019-12-17 12:37:13,899] jepsen worker 44 - jepsen.util 16424 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:37:13,899] jepsen worker 44 - jepsen.util 16424 :invoke :write [14 24] INFO [2019-12-17 12:37:13,918] jepsen worker 35 - jepsen.util 15785 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:37:13,918] jepsen worker 35 - jepsen.util 15785 :invoke :read [11 nil] INFO [2019-12-17 12:37:13,964] jepsen worker 56 - jepsen.util 12476 :info :write [18 40] :RequestCanceledException INFO [2019-12-17 12:37:13,965] jepsen worker 56 - jepsen.util 12566 :invoke :write [18 19] INFO [2019-12-17 12:37:14,032] jepsen worker 14 - jepsen.util 11264 :info :write [4 7] :RequestCanceledException INFO [2019-12-17 12:37:14,032] jepsen worker 68 - jepsen.util 15188 :info :write [22 29] :RequestCanceledException INFO [2019-12-17 12:37:14,032] jepsen worker 41 - jepsen.util 16961 :info :cas [13 [5 2]] :RequestCanceledException INFO [2019-12-17 12:37:14,032] jepsen worker 79 - jepsen.util 11779 :info :write [26 26] :RequestCanceledException INFO [2019-12-17 12:37:14,033] jepsen worker 14 - jepsen.util 11354 :invoke :cas [4 [9 2]] INFO [2019-12-17 12:37:14,035] jepsen worker 41 - jepsen.util 17051 :invoke :write [13 34] INFO [2019-12-17 12:37:14,035] jepsen worker 79 - jepsen.util 11869 :invoke :read [26 nil] INFO [2019-12-17 12:37:14,036] jepsen worker 68 - jepsen.util 15278 :invoke :cas [22 [20 15]] INFO [2019-12-17 12:37:14,188] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":67349,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:37:14,189] jepsen worker 19 - jepsen.util 11899 :info :write [6 30] :RequestCanceledException INFO [2019-12-17 12:37:14,189] jepsen worker 75 - jepsen.util 15465 :info :write [25 26] :RequestCanceledException INFO [2019-12-17 12:37:14,189] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":67349,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:37:14,189] jepsen worker 74 - jepsen.util 11414 :info :cas [24 [12 21]] :RequestCanceledException INFO [2019-12-17 12:37:14,189] jepsen worker 6 - jepsen.util 16296 :info :write [2 41] :RequestCanceledException INFO [2019-12-17 12:37:14,189] jepsen worker 22 - jepsen.util 15862 :info :write [7 27] :RequestCanceledException INFO [2019-12-17 12:37:14,189] jepsen worker 40 - jepsen.util 16420 :info :cas [13 [3 13]] :RequestCanceledException INFO [2019-12-17 12:37:14,189] jepsen worker 5 - jepsen.util 11345 :info :write [1 24] :RequestCanceledException INFO [2019-12-17 12:37:14,189] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":67352,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:37:14,190] jepsen worker 19 - jepsen.util 11989 :invoke :read [6 nil] INFO [2019-12-17 12:37:14,190] jepsen worker 74 - jepsen.util 11504 :invoke :read [24 nil] INFO [2019-12-17 12:37:14,191] jepsen worker 75 - jepsen.util 15555 :invoke :write [25 47] INFO [2019-12-17 12:37:14,191] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":67352,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:37:14,191] jepsen worker 6 - jepsen.util 16386 :invoke :read [2 nil] INFO [2019-12-17 12:37:14,191] jepsen worker 22 - jepsen.util 15952 :invoke :cas [7 [9 36]] INFO [2019-12-17 12:37:14,191] jepsen worker 40 - jepsen.util 16510 :invoke :read [13 nil] INFO [2019-12-17 12:37:14,192] jepsen worker 5 - jepsen.util 11435 :invoke :read [1 nil] INFO [2019-12-17 12:37:14,200] jepsen worker 14 - jepsen.util 11354 :fail :cas [4 [9 2]] :ValueNotSwapFrom INFO [2019-12-17 12:37:14,200] jepsen worker 14 - jepsen.util 11354 :invoke :cas [4 [47 18]] INFO [2019-12-17 12:37:14,209] jepsen worker 79 - jepsen.util 11869 :ok :read [26 16] INFO [2019-12-17 12:37:14,209] jepsen worker 79 - jepsen.util 11869 :invoke :cas [26 [48 29]] INFO [2019-12-17 12:37:14,216] jepsen worker 42 - jepsen.util 16782 :info :cas [14 [15 17]] :RequestCanceledException INFO [2019-12-17 12:37:14,216] jepsen worker 24 - jepsen.util 14604 :info :write [8 7] :RequestCanceledException INFO [2019-12-17 12:37:14,217] jepsen worker 24 - jepsen.util 14694 :invoke :write [8 31] INFO [2019-12-17 12:37:14,217] jepsen worker 42 - jepsen.util 16872 :invoke :read [14 nil] INFO [2019-12-17 12:37:14,220] jepsen worker 74 - jepsen.util 11504 :ok :read [24 13] INFO [2019-12-17 12:37:14,220] jepsen worker 19 - jepsen.util 11989 :ok :read [6 0] INFO [2019-12-17 12:37:14,220] jepsen worker 74 - jepsen.util 11504 :invoke :write [24 47] INFO [2019-12-17 12:37:14,220] jepsen worker 19 - jepsen.util 11989 :invoke :cas [6 [44 19]] INFO [2019-12-17 12:37:14,233] jepsen worker 83 - jepsen.util 16013 :info :cas [27 [49 17]] :RequestCanceledException INFO [2019-12-17 12:37:14,233] jepsen worker 39 - jepsen.util 15339 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:37:14,233] jepsen worker 1 - jepsen.util 15841 :info :cas [0 [0 5]] :RequestCanceledException INFO [2019-12-17 12:37:14,234] jepsen worker 39 - jepsen.util 15339 :invoke :cas [13 [32 6]] INFO [2019-12-17 12:37:14,234] jepsen worker 1 - jepsen.util 15931 :invoke :cas [0 [39 40]] INFO [2019-12-17 12:37:14,234] jepsen worker 83 - jepsen.util 16103 :invoke :write [27 41] INFO [2019-12-17 12:37:14,235] jepsen worker 19 - jepsen.util 11989 :fail :cas [6 [44 19]] :ValueNotSwapFrom INFO [2019-12-17 12:37:14,235] jepsen worker 19 - jepsen.util 11989 :invoke :read [6 nil] INFO [2019-12-17 12:37:14,250] jepsen worker 19 - jepsen.util 11989 :ok :read [6 0] INFO [2019-12-17 12:37:14,251] jepsen worker 19 - jepsen.util 11989 :invoke :read [6 nil] INFO [2019-12-17 12:37:14,264] jepsen worker 60 - jepsen.util 17160 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:37:14,264] jepsen worker 60 - jepsen.util 17160 :invoke :write [20 20] INFO [2019-12-17 12:37:14,265] jepsen worker 19 - jepsen.util 11989 :ok :read [6 0] INFO [2019-12-17 12:37:14,265] jepsen worker 19 - jepsen.util 11989 :invoke :cas [6 [25 9]] INFO [2019-12-17 12:37:14,278] jepsen worker 19 - jepsen.util 11989 :fail :cas [6 [25 9]] :ValueNotSwapFrom INFO [2019-12-17 12:37:14,279] jepsen worker 19 - jepsen.util 11989 :invoke :cas [6 [46 36]] INFO [2019-12-17 12:37:14,285] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:37:14,285] jepsen nemesis - couchbase.util Rebalance check count: 530 INFO [2019-12-17 12:37:14,285] jepsen nemesis - couchbase.util Rebalance stuck count: 511 INFO [2019-12-17 12:37:14,292] jepsen worker 19 - jepsen.util 11989 :fail :cas [6 [46 36]] :ValueNotSwapFrom INFO [2019-12-17 12:37:14,292] jepsen worker 19 - jepsen.util 11989 :invoke :write [6 32] INFO [2019-12-17 12:37:14,298] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":67363,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:37:14,298] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":67374,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:37:14,298] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":67363,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:37:14,298] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":67374,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:37:14,298] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":67364,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:37:14,298] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":67378,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:37:14,298] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":67364,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:37:14,298] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":67378,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:37:14,298] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":67365,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:37:14,299] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":67365,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:37:14,300] jepsen worker 34 - jepsen.util 16324 :info :cas [11 [29 3]] :RequestCanceledException INFO [2019-12-17 12:37:14,300] jepsen worker 34 - jepsen.util 16414 :invoke :write [11 32] INFO [2019-12-17 12:37:14,318] jepsen worker 76 - jepsen.util 16096 :info :cas [25 [40 1]] :RequestCanceledException INFO [2019-12-17 12:37:14,318] jepsen worker 76 - jepsen.util 16186 :invoke :read [25 nil] INFO [2019-12-17 12:37:14,322] jepsen worker 71 - jepsen.util 12401 :ok :read [23 41] INFO [2019-12-17 12:37:14,322] jepsen worker 71 - jepsen.util 12401 :invoke :write [23 8] INFO [2019-12-17 12:37:14,331] jepsen worker 49 - jepsen.util 15799 :info :cas [16 [8 48]] :RequestCanceledException INFO [2019-12-17 12:37:14,331] jepsen worker 57 - jepsen.util 15717 :info :write [19 7] :RequestCanceledException INFO [2019-12-17 12:37:14,332] jepsen worker 49 - jepsen.util 15889 :invoke :write [16 15] INFO [2019-12-17 12:37:14,333] jepsen worker 57 - jepsen.util 15807 :invoke :read [19 nil] INFO [2019-12-17 12:37:14,463] jepsen worker 53 - jepsen.util 11123 :info :write [17 1] :RequestCanceledException INFO [2019-12-17 12:37:14,464] jepsen worker 53 - jepsen.util 11213 :invoke :write [17 44] INFO [2019-12-17 12:37:14,482] jepsen worker 51 - jepsen.util 10941 :ok :read [17 1] INFO [2019-12-17 12:37:14,482] jepsen worker 51 - jepsen.util 10941 :invoke :read [17 nil] INFO [2019-12-17 12:37:14,516] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":67397,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:37:14,516] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":67397,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:37:14,516] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":67400,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:37:14,516] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":67400,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:37:14,516] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":67401,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:37:14,516] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":67401,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:37:14,540] jepsen worker 38 - jepsen.util 12098 :fail :cas [12 [16 39]] :ValueNotSwapFrom INFO [2019-12-17 12:37:14,541] jepsen worker 38 - jepsen.util 12098 :invoke :write [12 2] INFO [2019-12-17 12:37:14,576] jepsen worker 79 - jepsen.util 11869 :fail :cas [26 [48 29]] :ValueNotSwapFrom INFO [2019-12-17 12:37:14,577] jepsen worker 79 - jepsen.util 11869 :invoke :cas [26 [11 3]] INFO [2019-12-17 12:37:14,593] jepsen worker 32 - jepsen.util 12002 :ok :write [10 30] INFO [2019-12-17 12:37:14,593] jepsen worker 32 - jepsen.util 12002 :invoke :cas [10 [43 5]] INFO [2019-12-17 12:37:14,614] jepsen worker 29 - jepsen.util 9929 :ok :write [9 47] INFO [2019-12-17 12:37:14,614] jepsen worker 29 - jepsen.util 9929 :invoke :read [9 nil] INFO [2019-12-17 12:37:14,617] jepsen worker 33 - jepsen.util 15693 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:37:14,617] jepsen worker 33 - jepsen.util 15693 :invoke :cas [11 [25 32]] INFO [2019-12-17 12:37:14,628] jepsen worker 32 - jepsen.util 12002 :fail :cas [10 [43 5]] :ValueNotSwapFrom INFO [2019-12-17 12:37:14,628] jepsen worker 32 - jepsen.util 12002 :invoke :write [10 36] INFO [2019-12-17 12:37:14,636] jepsen worker 29 - jepsen.util 9929 :ok :read [9 47] INFO [2019-12-17 12:37:14,636] jepsen worker 29 - jepsen.util 9929 :invoke :cas [9 [48 28]] INFO [2019-12-17 12:37:14,640] jepsen worker 11 - jepsen.util 11261 :ok :write [3 7] INFO [2019-12-17 12:37:14,640] jepsen worker 11 - jepsen.util 11261 :invoke :read [3 nil] INFO [2019-12-17 12:37:14,640] jepsen worker 45 - jepsen.util 11745 :ok :write [15 19] INFO [2019-12-17 12:37:14,640] jepsen worker 45 - jepsen.util 11745 :invoke :read [15 nil] INFO [2019-12-17 12:37:14,646] jepsen worker 18 - jepsen.util 11718 :ok :write [6 28] INFO [2019-12-17 12:37:14,646] jepsen worker 18 - jepsen.util 11718 :invoke :cas [6 [2 13]] INFO [2019-12-17 12:37:14,658] jepsen worker 74 - jepsen.util 11504 :ok :write [24 47] INFO [2019-12-17 12:37:14,658] jepsen worker 74 - jepsen.util 11504 :invoke :write [24 27] INFO [2019-12-17 12:37:14,659] jepsen worker 5 - jepsen.util 11435 :ok :read [1 48] INFO [2019-12-17 12:37:14,659] jepsen worker 5 - jepsen.util 11435 :invoke :read [1 nil] INFO [2019-12-17 12:37:14,708] jepsen worker 65 - jepsen.util 10325 :ok :read [21 44] INFO [2019-12-17 12:37:14,708] jepsen worker 65 - jepsen.util 10325 :invoke :write [21 49] INFO [2019-12-17 12:37:14,718] jepsen worker 18 - jepsen.util 11718 :fail :cas [6 [2 13]] :ValueNotSwapFrom INFO [2019-12-17 12:37:14,718] jepsen worker 18 - jepsen.util 11718 :invoke :write [6 24] INFO [2019-12-17 12:37:14,736] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":67415,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:37:14,736] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":67415,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:37:14,736] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":67419,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:37:14,736] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":67419,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:37:14,742] jepsen worker 51 - jepsen.util 10941 :ok :read [17 1] INFO [2019-12-17 12:37:14,742] jepsen worker 51 - jepsen.util 10941 :invoke :cas [17 [8 13]] INFO [2019-12-17 12:37:14,752] jepsen worker 63 - jepsen.util 10233 :fail :cas [21 [37 3]] :ValueNotSwapFrom INFO [2019-12-17 12:37:14,752] jepsen worker 63 - jepsen.util 10233 :invoke :write [21 39] INFO [2019-12-17 12:37:14,763] jepsen worker 79 - jepsen.util 11869 :fail :cas [26 [11 3]] :ValueNotSwapFrom INFO [2019-12-17 12:37:14,763] jepsen worker 79 - jepsen.util 11869 :invoke :cas [26 [42 2]] INFO [2019-12-17 12:37:14,772] jepsen worker 28 - jepsen.util 9748 :ok :read [9 47] INFO [2019-12-17 12:37:14,772] jepsen worker 28 - jepsen.util 9748 :invoke :read [9 nil] INFO [2019-12-17 12:37:14,785] jepsen worker 5 - jepsen.util 11435 :ok :read [1 48] INFO [2019-12-17 12:37:14,785] jepsen worker 5 - jepsen.util 11435 :invoke :cas [1 [15 15]] INFO [2019-12-17 12:37:14,804] jepsen worker 11 - jepsen.util 11261 :ok :read [3 7] INFO [2019-12-17 12:37:14,804] jepsen worker 11 - jepsen.util 11261 :invoke :read [3 nil] INFO [2019-12-17 12:37:14,811] jepsen worker 45 - jepsen.util 11745 :ok :read [15 19] INFO [2019-12-17 12:37:14,811] jepsen worker 45 - jepsen.util 11745 :invoke :write [15 31] INFO [2019-12-17 12:37:14,847] jepsen worker 78 - jepsen.util 11508 :ok :write [26 44] INFO [2019-12-17 12:37:14,847] jepsen worker 78 - jepsen.util 11508 :invoke :cas [26 [31 10]] INFO [2019-12-17 12:37:14,854] jepsen worker 63 - jepsen.util 10233 :ok :write [21 39] INFO [2019-12-17 12:37:14,854] jepsen worker 63 - jepsen.util 10233 :invoke :write [21 15] INFO [2019-12-17 12:37:14,889] jepsen worker 74 - jepsen.util 11504 :ok :write [24 27] INFO [2019-12-17 12:37:14,889] jepsen worker 74 - jepsen.util 11504 :invoke :read [24 nil] INFO [2019-12-17 12:37:14,890] jepsen worker 65 - jepsen.util 10325 :ok :write [21 49] INFO [2019-12-17 12:37:14,890] jepsen worker 65 - jepsen.util 10325 :invoke :cas [21 [17 33]] INFO [2019-12-17 12:37:14,903] jepsen worker 28 - jepsen.util 9748 :ok :read [9 47] INFO [2019-12-17 12:37:14,903] jepsen worker 28 - jepsen.util 9748 :invoke :write [9 20] INFO [2019-12-17 12:37:14,908] jepsen worker 53 - jepsen.util 11213 :ok :write [17 44] INFO [2019-12-17 12:37:14,908] jepsen worker 53 - jepsen.util 11213 :invoke :write [17 31] INFO [2019-12-17 12:37:14,923] jepsen worker 74 - jepsen.util 11504 :ok :read [24 27] INFO [2019-12-17 12:37:14,923] jepsen worker 74 - jepsen.util 11504 :invoke :read [24 nil] INFO [2019-12-17 12:37:14,930] jepsen worker 32 - jepsen.util 12002 :ok :write [10 36] INFO [2019-12-17 12:37:14,930] jepsen worker 32 - jepsen.util 12002 :invoke :cas [10 [22 49]] INFO [2019-12-17 12:37:14,933] jepsen worker 27 - jepsen.util 10737 :info :write [9 0] :RequestCanceledException INFO [2019-12-17 12:37:14,934] jepsen worker 27 - jepsen.util 10827 :invoke :write [9 25] INFO [2019-12-17 12:37:14,947] jepsen worker 46 - jepsen.util 11386 :info :write [15 25] :RequestCanceledException INFO [2019-12-17 12:37:14,948] jepsen worker 46 - jepsen.util 11476 :invoke :read [15 nil] INFO [2019-12-17 12:37:14,949] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":67436,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:37:14,949] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":67436,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:37:14,951] jepsen worker 63 - jepsen.util 10233 :ok :write [21 15] INFO [2019-12-17 12:37:14,951] jepsen worker 63 - jepsen.util 10233 :invoke :cas [21 [47 45]] INFO [2019-12-17 12:37:14,964] jepsen worker 38 - jepsen.util 12098 :ok :write [12 2] INFO [2019-12-17 12:37:14,964] jepsen worker 38 - jepsen.util 12098 :invoke :read [12 nil] INFO [2019-12-17 12:37:14,964] jepsen worker 18 - jepsen.util 11718 :ok :write [6 24] INFO [2019-12-17 12:37:14,964] jepsen worker 18 - jepsen.util 11718 :invoke :cas [6 [19 40]] INFO [2019-12-17 12:37:14,995] jepsen worker 14 - jepsen.util 11354 :fail :cas [4 [47 18]] :ValueNotSwapFrom INFO [2019-12-17 12:37:14,995] jepsen worker 14 - jepsen.util 11354 :invoke :write [4 8] INFO [2019-12-17 12:37:14,996] jepsen worker 74 - jepsen.util 11504 :ok :read [24 27] INFO [2019-12-17 12:37:14,996] jepsen worker 74 - jepsen.util 11504 :invoke :read [24 nil] INFO [2019-12-17 12:37:15,002] jepsen worker 32 - jepsen.util 12002 :fail :cas [10 [22 49]] :ValueNotSwapFrom INFO [2019-12-17 12:37:15,002] jepsen worker 32 - jepsen.util 12002 :invoke :read [10 nil] INFO [2019-12-17 12:37:15,020] jepsen worker 46 - jepsen.util 11476 :ok :read [15 19] INFO [2019-12-17 12:37:15,021] jepsen worker 46 - jepsen.util 11476 :invoke :write [15 3] INFO [2019-12-17 12:37:15,030] jepsen worker 74 - jepsen.util 11504 :ok :read [24 27] INFO [2019-12-17 12:37:15,030] jepsen worker 74 - jepsen.util 11504 :invoke :write [24 45] INFO [2019-12-17 12:37:15,036] jepsen worker 18 - jepsen.util 11718 :fail :cas [6 [19 40]] :ValueNotSwapFrom INFO [2019-12-17 12:37:15,036] jepsen worker 18 - jepsen.util 11718 :invoke :cas [6 [45 4]] INFO [2019-12-17 12:37:15,048] jepsen worker 4 - jepsen.util 10714 :info :write [1 24] :RequestCanceledException INFO [2019-12-17 12:37:15,049] jepsen worker 4 - jepsen.util 10804 :invoke :cas [1 [37 1]] INFO [2019-12-17 12:37:15,066] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":67452,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:37:15,066] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":67452,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:37:15,066] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":67457,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:37:15,066] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":67457,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:37:15,068] jepsen worker 79 - jepsen.util 11869 :fail :cas [26 [42 2]] :ValueNotSwapFrom INFO [2019-12-17 12:37:15,068] jepsen worker 79 - jepsen.util 11869 :invoke :write [26 1] INFO [2019-12-17 12:37:15,080] jepsen worker 63 - jepsen.util 10233 :fail :cas [21 [47 45]] :ValueNotSwapFrom INFO [2019-12-17 12:37:15,080] jepsen worker 63 - jepsen.util 10233 :invoke :read [21 nil] INFO [2019-12-17 12:37:15,098] jepsen worker 18 - jepsen.util 11718 :fail :cas [6 [45 4]] :ValueNotSwapFrom INFO [2019-12-17 12:37:15,098] jepsen worker 73 - jepsen.util 10693 :fail :read [24 nil] :RequestTimeoutException INFO [2019-12-17 12:37:15,098] jepsen worker 18 - jepsen.util 11718 :invoke :read [6 nil] INFO [2019-12-17 12:37:15,099] jepsen worker 73 - jepsen.util 10693 :invoke :cas [24 [33 5]] INFO [2019-12-17 12:37:15,101] jepsen worker 78 - jepsen.util 11508 :fail :cas [26 [31 10]] :ValueNotSwapFrom INFO [2019-12-17 12:37:15,101] jepsen worker 78 - jepsen.util 11508 :invoke :read [26 nil] INFO [2019-12-17 12:37:15,120] jepsen worker 18 - jepsen.util 11718 :ok :read [6 24] INFO [2019-12-17 12:37:15,120] jepsen worker 18 - jepsen.util 11718 :invoke :write [6 37] INFO [2019-12-17 12:37:15,123] jepsen worker 27 - jepsen.util 10827 :ok :write [9 25] INFO [2019-12-17 12:37:15,123] jepsen worker 27 - jepsen.util 10827 :invoke :cas [9 [25 4]] INFO [2019-12-17 12:37:15,133] jepsen worker 69 - jepsen.util 11859 :info :write [23 46] :RequestCanceledException INFO [2019-12-17 12:37:15,133] jepsen worker 80 - jepsen.util 11240 :info :write [26 38] :RequestCanceledException INFO [2019-12-17 12:37:15,133] jepsen worker 17 - jepsen.util 15947 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:37:15,134] jepsen worker 17 - jepsen.util 15947 :invoke :cas [5 [15 47]] INFO [2019-12-17 12:37:15,134] jepsen worker 80 - jepsen.util 11330 :invoke :write [26 16] INFO [2019-12-17 12:37:15,134] jepsen worker 69 - jepsen.util 11949 :invoke :write [23 47] INFO [2019-12-17 12:37:15,147] jepsen worker 84 - jepsen.util 15654 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:37:15,147] jepsen worker 84 - jepsen.util 15654 :invoke :write [28 11] INFO [2019-12-17 12:37:15,165] jepsen worker 62 - jepsen.util 16622 :info :write [20 2] :RequestCanceledException INFO [2019-12-17 12:37:15,165] jepsen worker 66 - jepsen.util 15726 :info :write [22 20] :RequestCanceledException INFO [2019-12-17 12:37:15,165] jepsen worker 62 - jepsen.util 16712 :invoke :write [20 38] INFO [2019-12-17 12:37:15,165] jepsen worker 66 - jepsen.util 15816 :invoke :write [22 6] INFO [2019-12-17 12:37:15,172] jepsen worker 32 - jepsen.util 12002 :ok :read [10 36] INFO [2019-12-17 12:37:15,172] jepsen worker 32 - jepsen.util 12002 :invoke :read [10 nil] INFO [2019-12-17 12:37:15,200] jepsen worker 82 - jepsen.util 16192 :info :write [27 27] :RequestCanceledException INFO [2019-12-17 12:37:15,201] jepsen worker 82 - jepsen.util 16282 :invoke :cas [27 [40 39]] INFO [2019-12-17 12:37:15,210] jepsen worker 73 - jepsen.util 10693 :fail :cas [24 [33 5]] :ValueNotSwapFrom INFO [2019-12-17 12:37:15,210] jepsen worker 73 - jepsen.util 10693 :invoke :read [24 nil] INFO [2019-12-17 12:37:15,218] jepsen worker 77 - jepsen.util 16727 :info :cas [25 [28 11]] :RequestCanceledException INFO [2019-12-17 12:37:15,218] jepsen worker 77 - jepsen.util 16817 :invoke :cas [25 [30 26]] INFO [2019-12-17 12:37:15,223] jepsen worker 74 - jepsen.util 11504 :ok :write [24 45] INFO [2019-12-17 12:37:15,223] jepsen worker 74 - jepsen.util 11504 :invoke :read [24 nil] INFO [2019-12-17 12:37:15,233] jepsen worker 26 - jepsen.util 16586 :info :write [8 7] :RequestCanceledException INFO [2019-12-17 12:37:15,233] jepsen worker 23 - jepsen.util 16043 :info :cas [7 [39 16]] :RequestCanceledException INFO [2019-12-17 12:37:15,233] jepsen worker 70 - jepsen.util 12130 :info :write [23 17] :RequestCanceledException INFO [2019-12-17 12:37:15,234] jepsen worker 26 - jepsen.util 16676 :invoke :cas [8 [9 30]] INFO [2019-12-17 12:37:15,234] jepsen worker 70 - jepsen.util 12220 :invoke :write [23 41] INFO [2019-12-17 12:37:15,234] jepsen worker 23 - jepsen.util 16133 :invoke :cas [7 [6 41]] INFO [2019-12-17 12:37:15,241] jepsen worker 25 - jepsen.util 16945 :info :write [8 41] :RequestCanceledException INFO [2019-12-17 12:37:15,242] jepsen worker 25 - jepsen.util 17035 :invoke :read [8 nil] INFO [2019-12-17 12:37:15,243] jepsen worker 74 - jepsen.util 11504 :ok :read [24 45] INFO [2019-12-17 12:37:15,243] jepsen worker 74 - jepsen.util 11504 :invoke :cas [24 [30 3]] INFO [2019-12-17 12:37:15,259] jepsen worker 54 - jepsen.util 12114 :ok :write [18 6] INFO [2019-12-17 12:37:15,259] jepsen worker 54 - jepsen.util 12114 :invoke :write [18 13] INFO [2019-12-17 12:37:15,264] jepsen worker 31 - jepsen.util 9931 :info :write [10 34] :RequestCanceledException INFO [2019-12-17 12:37:15,264] jepsen worker 2 - jepsen.util 15392 :info :cas [0 [18 40]] :RequestCanceledException INFO [2019-12-17 12:37:15,264] jepsen worker 48 - jepsen.util 14718 :info :cas [16 [44 47]] :RequestCanceledException INFO [2019-12-17 12:37:15,265] jepsen worker 31 - jepsen.util 10021 :invoke :read [10 nil] INFO [2019-12-17 12:37:15,265] jepsen worker 48 - jepsen.util 14808 :invoke :write [16 44] INFO [2019-12-17 12:37:15,265] jepsen worker 2 - jepsen.util 15482 :invoke :cas [0 [21 44]] INFO [2019-12-17 12:37:15,278] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":67478,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:37:15,278] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":67486,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:37:15,278] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":67478,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:37:15,278] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":67486,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:37:15,284] jepsen worker 74 - jepsen.util 11504 :fail :cas [24 [30 3]] :ValueNotSwapFrom INFO [2019-12-17 12:37:15,285] jepsen worker 74 - jepsen.util 11504 :invoke :read [24 nil] INFO [2019-12-17 12:37:15,287] jepsen worker 78 - jepsen.util 11508 :ok :read [26 44] INFO [2019-12-17 12:37:15,287] jepsen worker 78 - jepsen.util 11508 :invoke :cas [26 [17 41]] INFO [2019-12-17 12:37:15,300] jepsen worker 86 - jepsen.util 16646 :info :write [28 22] :RequestCanceledException INFO [2019-12-17 12:37:15,301] jepsen worker 86 - jepsen.util 16736 :invoke :cas [28 [46 17]] INFO [2019-12-17 12:37:15,312] jepsen worker 32 - jepsen.util 12002 :ok :read [10 36] INFO [2019-12-17 12:37:15,312] jepsen worker 32 - jepsen.util 12002 :invoke :cas [10 [49 49]] INFO [2019-12-17 12:37:15,312] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:37:15,312] jepsen nemesis - couchbase.util Rebalance check count: 531 INFO [2019-12-17 12:37:15,312] jepsen nemesis - couchbase.util Rebalance stuck count: 512 INFO [2019-12-17 12:37:15,317] jepsen worker 58 - jepsen.util 16258 :info :write [19 46] :RequestCanceledException INFO [2019-12-17 12:37:15,318] jepsen worker 58 - jepsen.util 16348 :invoke :read [19 nil] INFO [2019-12-17 12:37:15,333] jepsen worker 67 - jepsen.util 15907 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:37:15,333] jepsen worker 7 - jepsen.util 15937 :info :write [2 36] :RequestCanceledException INFO [2019-12-17 12:37:15,333] jepsen worker 43 - jepsen.util 16063 :info :cas [14 [24 20]] :RequestCanceledException INFO [2019-12-17 12:37:15,333] jepsen worker 67 - jepsen.util 15907 :invoke :cas [22 [28 0]] INFO [2019-12-17 12:37:15,334] jepsen worker 43 - jepsen.util 16153 :invoke :cas [14 [20 41]] INFO [2019-12-17 12:37:15,334] jepsen worker 7 - jepsen.util 16027 :invoke :read [2 nil] INFO [2019-12-17 12:37:15,339] jepsen worker 73 - jepsen.util 10693 :ok :read [24 45] INFO [2019-12-17 12:37:15,339] jepsen worker 73 - jepsen.util 10693 :invoke :cas [24 [7 0]] INFO [2019-12-17 12:37:15,341] jepsen worker 74 - jepsen.util 11504 :ok :read [24 45] INFO [2019-12-17 12:37:15,341] jepsen worker 74 - jepsen.util 11504 :invoke :write [24 49] INFO [2019-12-17 12:37:15,347] jepsen worker 30 - jepsen.util 10650 :info :write [10 37] :RequestCanceledException INFO [2019-12-17 12:37:15,347] jepsen worker 85 - jepsen.util 16555 :info :cas [28 [17 40]] :RequestCanceledException INFO [2019-12-17 12:37:15,348] jepsen worker 30 - jepsen.util 10740 :invoke :cas [10 [36 18]] INFO [2019-12-17 12:37:15,348] jepsen worker 85 - jepsen.util 16645 :invoke :cas [28 [28 8]] INFO [2019-12-17 12:37:15,364] jepsen worker 50 - jepsen.util 16250 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:37:15,364] jepsen worker 21 - jepsen.util 15321 :info :cas [7 [25 27]] :RequestCanceledException INFO [2019-12-17 12:37:15,364] jepsen worker 50 - jepsen.util 16250 :invoke :read [16 nil] INFO [2019-12-17 12:37:15,365] jepsen worker 21 - jepsen.util 15411 :invoke :write [7 10] INFO [2019-12-17 12:37:15,380] jepsen worker 27 - jepsen.util 10827 :ok :cas [9 [25 4]] INFO [2019-12-17 12:37:15,380] jepsen worker 27 - jepsen.util 10827 :invoke :write [9 45] INFO [2019-12-17 12:37:15,400] jepsen worker 0 - jepsen.util 16560 :info :cas [0 [27 38]] :RequestCanceledException INFO [2019-12-17 12:37:15,400] jepsen worker 59 - jepsen.util 15179 :info :cas [19 [33 24]] :RequestCanceledException INFO [2019-12-17 12:37:15,400] jepsen worker 3 - jepsen.util 10983 :info :write [1 26] :RequestCanceledException INFO [2019-12-17 12:37:15,401] jepsen worker 3 - jepsen.util 11073 :invoke :read [1 nil] INFO [2019-12-17 12:37:15,401] jepsen worker 0 - jepsen.util 16650 :invoke :read [0 nil] INFO [2019-12-17 12:37:15,401] jepsen worker 59 - jepsen.util 15269 :invoke :read [19 nil] INFO [2019-12-17 12:37:15,404] jepsen worker 51 - jepsen.util 10941 :fail :cas [17 [8 13]] :ValueNotSwapFrom INFO [2019-12-17 12:37:15,404] jepsen worker 51 - jepsen.util 10941 :invoke :write [17 16] INFO [2019-12-17 12:37:15,405] jepsen worker 78 - jepsen.util 11508 :fail :cas [26 [17 41]] :ValueNotSwapFrom INFO [2019-12-17 12:37:15,406] jepsen worker 78 - jepsen.util 11508 :invoke :cas [26 [49 12]] INFO [2019-12-17 12:37:15,416] jepsen worker 8 - jepsen.util 15578 :info :write [2 30] :RequestCanceledException INFO [2019-12-17 12:37:15,416] jepsen worker 36 - jepsen.util 12366 :info :write [12 16] :RequestCanceledException INFO [2019-12-17 12:37:15,416] jepsen worker 72 - jepsen.util 11952 :fail :read [24 nil] :RequestTimeoutException INFO [2019-12-17 12:37:15,417] jepsen worker 72 - jepsen.util 11952 :invoke :cas [24 [41 6]] INFO [2019-12-17 12:37:15,417] jepsen worker 36 - jepsen.util 12456 :invoke :read [12 nil] INFO [2019-12-17 12:37:15,417] jepsen worker 8 - jepsen.util 15668 :invoke :cas [2 [34 48]] INFO [2019-12-17 12:37:15,432] jepsen worker 10 - jepsen.util 11260 :info :write [3 14] :RequestCanceledException INFO [2019-12-17 12:37:15,433] jepsen worker 61 - jepsen.util 17071 :info :write [20 39] :RequestCanceledException INFO [2019-12-17 12:37:15,433] jepsen worker 10 - jepsen.util 11350 :invoke :cas [3 [8 1]] INFO [2019-12-17 12:37:15,435] jepsen worker 61 - jepsen.util 17161 :invoke :write [20 10] INFO [2019-12-17 12:37:15,435] jepsen worker 5 - jepsen.util 11435 :fail :cas [1 [15 15]] :ValueNotSwapFrom INFO [2019-12-17 12:37:15,435] jepsen worker 5 - jepsen.util 11435 :invoke :read [1 nil] INFO [2019-12-17 12:37:15,447] jepsen worker 20 - jepsen.util 11180 :info :write [6 34] :RequestCanceledException INFO [2019-12-17 12:37:15,447] jepsen worker 15 - jepsen.util 14775 :info :write [5 37] :RequestCanceledException INFO [2019-12-17 12:37:15,448] jepsen worker 20 - jepsen.util 11270 :invoke :write [6 42] INFO [2019-12-17 12:37:15,448] jepsen worker 15 - jepsen.util 14865 :invoke :read [5 nil] INFO [2019-12-17 12:37:15,452] jepsen worker 38 - jepsen.util 12098 :ok :read [12 2] INFO [2019-12-17 12:37:15,453] jepsen worker 38 - jepsen.util 12098 :invoke :read [12 nil] INFO [2019-12-17 12:37:15,477] jepsen worker 4 - jepsen.util 10804 :fail :cas [1 [37 1]] :ValueNotSwapFrom INFO [2019-12-17 12:37:15,477] jepsen worker 4 - jepsen.util 10804 :invoke :read [1 nil] INFO [2019-12-17 12:37:15,492] jepsen worker 3 - jepsen.util 11073 :ok :read [1 48] INFO [2019-12-17 12:37:15,492] jepsen worker 3 - jepsen.util 11073 :invoke :cas [1 [12 39]] INFO [2019-12-17 12:37:15,497] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":67529,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:37:15,497] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":67529,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:37:15,497] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":67532,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:37:15,497] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":67532,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:37:15,505] jepsen worker 74 - jepsen.util 11504 :ok :write [24 49] INFO [2019-12-17 12:37:15,505] jepsen worker 74 - jepsen.util 11504 :invoke :write [24 39] INFO [2019-12-17 12:37:15,506] jepsen worker 46 - jepsen.util 11476 :ok :write [15 3] INFO [2019-12-17 12:37:15,507] jepsen worker 46 - jepsen.util 11476 :invoke :read [15 nil] INFO [2019-12-17 12:37:15,517] jepsen worker 37 - jepsen.util 11197 :info :write [12 15] :RequestCanceledException INFO [2019-12-17 12:37:15,518] jepsen worker 37 - jepsen.util 11287 :invoke :write [12 15] INFO [2019-12-17 12:37:15,522] jepsen worker 38 - jepsen.util 12098 :ok :read [12 2] INFO [2019-12-17 12:37:15,522] jepsen worker 38 - jepsen.util 12098 :invoke :read [12 nil] INFO [2019-12-17 12:37:15,527] jepsen worker 27 - jepsen.util 10827 :ok :write [9 45] INFO [2019-12-17 12:37:15,527] jepsen worker 27 - jepsen.util 10827 :invoke :cas [9 [1 42]] INFO [2019-12-17 12:37:15,533] jepsen worker 73 - jepsen.util 10693 :fail :cas [24 [7 0]] :ValueNotSwapFrom INFO [2019-12-17 12:37:15,533] jepsen worker 64 - jepsen.util 8884 :fail :read [21 nil] :RequestTimeoutException INFO [2019-12-17 12:37:15,533] jepsen worker 64 - jepsen.util 8884 :invoke :read [21 nil] INFO [2019-12-17 12:37:15,533] jepsen worker 73 - jepsen.util 10693 :invoke :write [24 8] INFO [2019-12-17 12:37:15,541] jepsen worker 18 - jepsen.util 11718 :ok :write [6 37] INFO [2019-12-17 12:37:15,542] jepsen worker 18 - jepsen.util 11718 :invoke :cas [6 [13 18]] INFO [2019-12-17 12:37:15,560] jepsen worker 55 - jepsen.util 12205 :info :write [18 14] :RequestCanceledException INFO [2019-12-17 12:37:15,562] jepsen worker 55 - jepsen.util 12295 :invoke :cas [18 [25 9]] INFO [2019-12-17 12:37:15,567] jepsen worker 4 - jepsen.util 10804 :ok :read [1 48] INFO [2019-12-17 12:37:15,568] jepsen worker 4 - jepsen.util 10804 :invoke :read [1 nil] INFO [2019-12-17 12:37:15,568] jepsen worker 3 - jepsen.util 11073 :fail :cas [1 [12 39]] :ValueNotSwapFrom INFO [2019-12-17 12:37:15,568] jepsen worker 3 - jepsen.util 11073 :invoke :read [1 nil] INFO [2019-12-17 12:37:15,570] jepsen worker 31 - jepsen.util 10021 :ok :read [10 36] INFO [2019-12-17 12:37:15,570] jepsen worker 31 - jepsen.util 10021 :invoke :cas [10 [15 25]] INFO [2019-12-17 12:37:15,587] jepsen worker 38 - jepsen.util 12098 :ok :read [12 2] INFO [2019-12-17 12:37:15,587] jepsen worker 38 - jepsen.util 12098 :invoke :cas [12 [1 23]] INFO [2019-12-17 12:37:15,595] jepsen worker 4 - jepsen.util 10804 :ok :read [1 48] INFO [2019-12-17 12:37:15,595] jepsen worker 4 - jepsen.util 10804 :invoke :write [1 32] INFO [2019-12-17 12:37:15,609] jepsen worker 10 - jepsen.util 11350 :fail :cas [3 [8 1]] :ValueNotSwapFrom INFO [2019-12-17 12:37:15,609] jepsen worker 10 - jepsen.util 11350 :invoke :read [3 nil] INFO [2019-12-17 12:37:15,617] jepsen worker 72 - jepsen.util 11952 :fail :cas [24 [41 6]] :ValueNotSwapFrom INFO [2019-12-17 12:37:15,617] jepsen worker 72 - jepsen.util 11952 :invoke :read [24 nil] INFO [2019-12-17 12:37:15,627] jepsen worker 55 - jepsen.util 12295 :fail :cas [18 [25 9]] :ValueNotSwapFrom INFO [2019-12-17 12:37:15,627] jepsen worker 55 - jepsen.util 12295 :invoke :write [18 34] INFO [2019-12-17 12:37:15,630] jepsen worker 18 - jepsen.util 11718 :fail :cas [6 [13 18]] :ValueNotSwapFrom INFO [2019-12-17 12:37:15,631] jepsen worker 18 - jepsen.util 11718 :invoke :read [6 nil] INFO [2019-12-17 12:37:15,636] jepsen worker 38 - jepsen.util 12098 :fail :cas [12 [1 23]] :ValueNotSwapFrom INFO [2019-12-17 12:37:15,636] jepsen worker 38 - jepsen.util 12098 :invoke :cas [12 [18 4]] INFO [2019-12-17 12:37:15,647] jepsen worker 9 - jepsen.util 12249 :info :write [3 31] :RequestCanceledException INFO [2019-12-17 12:37:15,648] jepsen worker 9 - jepsen.util 12339 :invoke :read [3 nil] INFO [2019-12-17 12:37:15,653] jepsen worker 10 - jepsen.util 11350 :ok :read [3 7] INFO [2019-12-17 12:37:15,653] jepsen worker 10 - jepsen.util 11350 :invoke :write [3 48] INFO [2019-12-17 12:37:15,661] jepsen worker 18 - jepsen.util 11718 :ok :read [6 37] INFO [2019-12-17 12:37:15,661] jepsen worker 18 - jepsen.util 11718 :invoke :cas [6 [23 23]] INFO [2019-12-17 12:37:15,670] jepsen worker 9 - jepsen.util 12339 :ok :read [3 7] INFO [2019-12-17 12:37:15,670] jepsen worker 9 - jepsen.util 12339 :invoke :write [3 1] INFO [2019-12-17 12:37:15,681] jepsen worker 46 - jepsen.util 11476 :ok :read [15 3] INFO [2019-12-17 12:37:15,681] jepsen worker 46 - jepsen.util 11476 :invoke :write [15 41] INFO [2019-12-17 12:37:15,685] jepsen worker 3 - jepsen.util 11073 :ok :read [1 48] INFO [2019-12-17 12:37:15,685] jepsen worker 3 - jepsen.util 11073 :invoke :read [1 nil] INFO [2019-12-17 12:37:15,695] jepsen worker 31 - jepsen.util 10021 :fail :cas [10 [15 25]] :ValueNotSwapFrom INFO [2019-12-17 12:37:15,695] jepsen worker 30 - jepsen.util 10740 :ok :cas [10 [36 18]] INFO [2019-12-17 12:37:15,695] jepsen worker 31 - jepsen.util 10021 :invoke :read [10 nil] INFO [2019-12-17 12:37:15,695] jepsen worker 30 - jepsen.util 10740 :invoke :write [10 6] INFO [2019-12-17 12:37:15,696] jepsen worker 47 - jepsen.util 12467 :info :write [15 20] :RequestCanceledException INFO [2019-12-17 12:37:15,697] jepsen worker 47 - jepsen.util 12557 :invoke :cas [15 [49 20]] INFO [2019-12-17 12:37:15,704] jepsen worker 72 - jepsen.util 11952 :ok :read [24 49] INFO [2019-12-17 12:37:15,704] jepsen worker 72 - jepsen.util 11952 :invoke :read [24 nil] INFO [2019-12-17 12:37:15,716] jepsen worker 3 - jepsen.util 11073 :ok :read [1 48] INFO [2019-12-17 12:37:15,716] jepsen worker 3 - jepsen.util 11073 :invoke :write [1 26] INFO [2019-12-17 12:37:15,716] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":67563,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:37:15,716] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":67563,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:37:15,717] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":67566,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:37:15,717] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":67566,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:37:15,729] jepsen worker 64 - jepsen.util 8884 :ok :read [21 15] INFO [2019-12-17 12:37:15,729] jepsen worker 64 - jepsen.util 8884 :invoke :write [21 29] INFO [2019-12-17 12:37:15,733] jepsen worker 88 - jepsen.util 11788 :info :write [29 36] :RequestCanceledException INFO [2019-12-17 12:37:15,734] jepsen worker 88 - jepsen.util 11878 :invoke :read [29 nil] INFO [2019-12-17 12:37:15,747] jepsen worker 87 - jepsen.util 11157 :info :cas [29 [9 17]] :RequestCanceledException INFO [2019-12-17 12:37:15,747] jepsen worker 52 - jepsen.util 10942 :info :write [17 35] :RequestCanceledException INFO [2019-12-17 12:37:15,749] jepsen worker 52 - jepsen.util 11032 :invoke :read [17 nil] INFO [2019-12-17 12:37:15,749] jepsen worker 87 - jepsen.util 11247 :invoke :read [29 nil] INFO [2019-12-17 12:37:15,769] jepsen worker 47 - jepsen.util 12557 :fail :cas [15 [49 20]] :ValueNotSwapFrom INFO [2019-12-17 12:37:15,769] jepsen worker 47 - jepsen.util 12557 :invoke :cas [15 [17 2]] INFO [2019-12-17 12:37:15,774] jepsen worker 36 - jepsen.util 12456 :ok :read [12 2] INFO [2019-12-17 12:37:15,774] jepsen worker 36 - jepsen.util 12456 :invoke :write [12 29] INFO [2019-12-17 12:37:15,778] jepsen worker 78 - jepsen.util 11508 :fail :cas [26 [49 12]] :ValueNotSwapFrom INFO [2019-12-17 12:37:15,778] jepsen worker 78 - jepsen.util 11508 :invoke :cas [26 [44 31]] INFO [2019-12-17 12:37:15,800] jepsen worker 20 - jepsen.util 11270 :ok :write [6 42] INFO [2019-12-17 12:37:15,800] jepsen worker 20 - jepsen.util 11270 :invoke :cas [6 [46 49]] INFO [2019-12-17 12:37:15,800] jepsen worker 89 - jepsen.util 11609 :info :write [29 23] :RequestCanceledException INFO [2019-12-17 12:37:15,801] jepsen worker 89 - jepsen.util 11699 :invoke :read [29 nil] INFO [2019-12-17 12:37:15,803] jepsen worker 47 - jepsen.util 12557 :fail :cas [15 [17 2]] :ValueNotSwapFrom INFO [2019-12-17 12:37:15,803] jepsen worker 47 - jepsen.util 12557 :invoke :write [15 43] INFO [2019-12-17 12:37:15,825] jepsen worker 27 - jepsen.util 10827 :fail :cas [9 [1 42]] :ValueNotSwapFrom INFO [2019-12-17 12:37:15,825] jepsen worker 27 - jepsen.util 10827 :invoke :write [9 27] INFO [2019-12-17 12:37:15,832] jepsen worker 52 - jepsen.util 11032 :ok :read [17 44] INFO [2019-12-17 12:37:15,832] jepsen worker 52 - jepsen.util 11032 :invoke :read [17 nil] INFO [2019-12-17 12:37:15,836] jepsen worker 5 - jepsen.util 11435 :ok :read [1 48] INFO [2019-12-17 12:37:15,836] jepsen worker 5 - jepsen.util 11435 :invoke :write [1 46] INFO [2019-12-17 12:37:15,847] jepsen worker 12 - jepsen.util 11802 :info :write [4 17] :RequestCanceledException INFO [2019-12-17 12:37:15,847] jepsen worker 13 - jepsen.util 10903 :info :write [4 34] :RequestCanceledException INFO [2019-12-17 12:37:15,848] jepsen worker 52 - jepsen.util 11032 :ok :read [17 44] INFO [2019-12-17 12:37:15,848] jepsen worker 52 - jepsen.util 11032 :invoke :write [17 40] INFO [2019-12-17 12:37:15,848] jepsen worker 13 - jepsen.util 10993 :invoke :cas [4 [17 4]] INFO [2019-12-17 12:37:15,848] jepsen worker 12 - jepsen.util 11892 :invoke :read [4 nil] INFO [2019-12-17 12:37:15,881] jepsen worker 46 - jepsen.util 11476 :ok :write [15 41] INFO [2019-12-17 12:37:15,881] jepsen worker 74 - jepsen.util 11504 :ok :write [24 39] INFO [2019-12-17 12:37:15,881] jepsen worker 46 - jepsen.util 11476 :invoke :cas [15 [20 29]] INFO [2019-12-17 12:37:15,881] jepsen worker 74 - jepsen.util 11504 :invoke :write [24 19] INFO [2019-12-17 12:37:15,886] jepsen worker 10 - jepsen.util 11350 :ok :write [3 48] INFO [2019-12-17 12:37:15,887] jepsen worker 10 - jepsen.util 11350 :invoke :write [3 47] INFO [2019-12-17 12:37:15,887] jepsen worker 30 - jepsen.util 10740 :ok :write [10 6] INFO [2019-12-17 12:37:15,888] jepsen worker 30 - jepsen.util 10740 :invoke :cas [10 [35 18]] INFO [2019-12-17 12:37:15,896] jepsen worker 88 - jepsen.util 11878 :ok :read [29 9] INFO [2019-12-17 12:37:15,896] jepsen worker 88 - jepsen.util 11878 :invoke :cas [29 [2 3]] INFO [2019-12-17 12:37:15,903] jepsen worker 46 - jepsen.util 11476 :fail :cas [15 [20 29]] :ValueNotSwapFrom INFO [2019-12-17 12:37:15,903] jepsen worker 46 - jepsen.util 11476 :invoke :read [15 nil] INFO [2019-12-17 12:37:15,918] jepsen worker 81 - jepsen.util 15561 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:37:15,918] jepsen worker 81 - jepsen.util 15561 :invoke :read [27 nil] INFO [2019-12-17 12:37:15,923] jepsen worker 46 - jepsen.util 11476 :ok :read [15 41] INFO [2019-12-17 12:37:15,923] jepsen worker 46 - jepsen.util 11476 :invoke :write [15 16] INFO [2019-12-17 12:37:15,930] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":67594,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:37:15,931] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":67594,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:37:15,931] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":67597,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:37:15,931] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":67597,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:37:15,931] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":67598,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:37:15,931] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":67598,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:37:15,931] jepsen worker 16 - jepsen.util 17656 :info :cas [5 [46 5]] :RequestCanceledException INFO [2019-12-17 12:37:15,932] jepsen worker 16 - jepsen.util 17746 :invoke :cas [5 [19 8]] INFO [2019-12-17 12:37:15,999] jepsen worker 44 - jepsen.util 16424 :info :write [14 24] :RequestCanceledException INFO [2019-12-17 12:37:15,999] jepsen worker 64 - jepsen.util 8884 :ok :write [21 29] INFO [2019-12-17 12:37:16,000] jepsen worker 64 - jepsen.util 8884 :invoke :write [21 0] INFO [2019-12-17 12:37:16,001] jepsen worker 44 - jepsen.util 16514 :invoke :write [14 33] INFO [2019-12-17 12:37:16,018] jepsen worker 35 - jepsen.util 15785 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:37:16,018] jepsen worker 35 - jepsen.util 15785 :invoke :read [11 nil] INFO [2019-12-17 12:37:16,039] jepsen worker 64 - jepsen.util 8884 :ok :write [21 0] INFO [2019-12-17 12:37:16,040] jepsen worker 64 - jepsen.util 8884 :invoke :write [21 11] INFO [2019-12-17 12:37:16,042] jepsen worker 9 - jepsen.util 12339 :ok :write [3 1] INFO [2019-12-17 12:37:16,042] jepsen worker 9 - jepsen.util 12339 :invoke :write [3 2] INFO [2019-12-17 12:37:16,047] jepsen worker 56 - jepsen.util 12566 :info :write [18 19] :RequestCanceledException INFO [2019-12-17 12:37:16,048] jepsen worker 56 - jepsen.util 12656 :invoke :read [18 nil] INFO [2019-12-17 12:37:16,051] jepsen worker 12 - jepsen.util 11892 :ok :read [4 0] INFO [2019-12-17 12:37:16,051] jepsen worker 12 - jepsen.util 11892 :invoke :read [4 nil] INFO [2019-12-17 12:37:16,053] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":67609,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:37:16,053] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":67609,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:37:16,053] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":67613,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:37:16,053] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":67613,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:37:16,063] jepsen worker 38 - jepsen.util 12098 :fail :cas [12 [18 4]] :ValueNotSwapFrom INFO [2019-12-17 12:37:16,063] jepsen worker 38 - jepsen.util 12098 :invoke :write [12 19] INFO [2019-12-17 12:37:16,063] jepsen worker 74 - jepsen.util 11504 :ok :write [24 19] INFO [2019-12-17 12:37:16,063] jepsen worker 46 - jepsen.util 11476 :ok :write [15 16] INFO [2019-12-17 12:37:16,063] jepsen worker 74 - jepsen.util 11504 :invoke :write [24 43] INFO [2019-12-17 12:37:16,063] jepsen worker 46 - jepsen.util 11476 :invoke :cas [15 [28 26]] INFO [2019-12-17 12:37:16,081] jepsen worker 46 - jepsen.util 11476 :fail :cas [15 [28 26]] :ValueNotSwapFrom INFO [2019-12-17 12:37:16,081] jepsen worker 46 - jepsen.util 11476 :invoke :write [15 4] INFO [2019-12-17 12:37:16,099] jepsen worker 10 - jepsen.util 11350 :ok :write [3 47] INFO [2019-12-17 12:37:16,100] jepsen worker 10 - jepsen.util 11350 :invoke :cas [3 [38 27]] INFO [2019-12-17 12:37:16,100] jepsen worker 64 - jepsen.util 8884 :ok :write [21 11] INFO [2019-12-17 12:37:16,100] jepsen worker 64 - jepsen.util 8884 :invoke :cas [21 [30 41]] INFO [2019-12-17 12:37:16,118] jepsen worker 68 - jepsen.util 15278 :info :cas [22 [20 15]] :RequestCanceledException INFO [2019-12-17 12:37:16,119] jepsen worker 68 - jepsen.util 15368 :invoke :read [22 nil] INFO [2019-12-17 12:37:16,121] jepsen worker 64 - jepsen.util 8884 :fail :cas [21 [30 41]] :ValueNotSwapFrom INFO [2019-12-17 12:37:16,122] jepsen worker 64 - jepsen.util 8884 :invoke :cas [21 [2 35]] INFO [2019-12-17 12:37:16,147] jepsen worker 64 - jepsen.util 8884 :fail :cas [21 [2 35]] :ValueNotSwapFrom INFO [2019-12-17 12:37:16,148] jepsen worker 64 - jepsen.util 8884 :invoke :write [21 24] INFO [2019-12-17 12:37:16,164] jepsen worker 88 - jepsen.util 11878 :fail :cas [29 [2 3]] :ValueNotSwapFrom INFO [2019-12-17 12:37:16,164] jepsen worker 88 - jepsen.util 11878 :invoke :read [29 nil] INFO [2019-12-17 12:37:16,176] jepsen worker 54 - jepsen.util 12114 :ok :write [18 13] INFO [2019-12-17 12:37:16,176] jepsen worker 54 - jepsen.util 12114 :invoke :write [18 35] INFO [2019-12-17 12:37:16,186] jepsen worker 74 - jepsen.util 11504 :ok :write [24 43] INFO [2019-12-17 12:37:16,186] jepsen worker 74 - jepsen.util 11504 :invoke :write [24 5] INFO [2019-12-17 12:37:16,195] jepsen worker 64 - jepsen.util 8884 :ok :write [21 24] INFO [2019-12-17 12:37:16,195] jepsen worker 64 - jepsen.util 8884 :invoke :cas [21 [39 27]] INFO [2019-12-17 12:37:16,217] jepsen worker 40 - jepsen.util 16510 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:37:16,218] jepsen worker 40 - jepsen.util 16510 :invoke :cas [13 [33 9]] INFO [2019-12-17 12:37:16,221] jepsen worker 64 - jepsen.util 8884 :fail :cas [21 [39 27]] :ValueNotSwapFrom INFO [2019-12-17 12:37:16,221] jepsen worker 64 - jepsen.util 8884 :invoke :cas [21 [26 36]] INFO [2019-12-17 12:37:16,224] jepsen worker 9 - jepsen.util 12339 :ok :write [3 2] INFO [2019-12-17 12:37:16,224] jepsen worker 9 - jepsen.util 12339 :invoke :write [3 5] INFO [2019-12-17 12:37:16,227] jepsen worker 87 - jepsen.util 11247 :ok :read [29 9] INFO [2019-12-17 12:37:16,227] jepsen worker 87 - jepsen.util 11247 :invoke :read [29 nil] INFO [2019-12-17 12:37:16,233] jepsen worker 41 - jepsen.util 17051 :info :write [13 34] :RequestCanceledException INFO [2019-12-17 12:37:16,233] jepsen worker 6 - jepsen.util 16386 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:37:16,233] jepsen worker 6 - jepsen.util 16386 :invoke :cas [2 [27 28]] INFO [2019-12-17 12:37:16,234] jepsen worker 41 - jepsen.util 17141 :invoke :write [13 4] INFO [2019-12-17 12:37:16,247] jepsen worker 24 - jepsen.util 14694 :info :write [8 31] :RequestCanceledException INFO [2019-12-17 12:37:16,248] jepsen worker 24 - jepsen.util 14784 :invoke :cas [8 [4 47]] INFO [2019-12-17 12:37:16,249] jepsen worker 70 - jepsen.util 12220 :ok :write [23 41] INFO [2019-12-17 12:37:16,249] jepsen worker 70 - jepsen.util 12220 :invoke :read [23 nil] INFO [2019-12-17 12:37:16,259] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":67635,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:37:16,259] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":67642,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:37:16,259] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":67635,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:37:16,259] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":67637,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:37:16,259] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":67642,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:37:16,259] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":67637,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:37:16,259] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":67644,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:37:16,260] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":67644,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:37:16,264] jepsen worker 75 - jepsen.util 15555 :info :write [25 47] :RequestCanceledException INFO [2019-12-17 12:37:16,264] jepsen worker 42 - jepsen.util 16872 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:37:16,264] jepsen worker 42 - jepsen.util 16872 :invoke :write [14 19] INFO [2019-12-17 12:37:16,266] jepsen worker 75 - jepsen.util 15645 :invoke :read [25 nil] INFO [2019-12-17 12:37:16,268] jepsen worker 64 - jepsen.util 8884 :fail :cas [21 [26 36]] :ValueNotSwapFrom INFO [2019-12-17 12:37:16,268] jepsen worker 64 - jepsen.util 8884 :invoke :cas [21 [3 19]] INFO [2019-12-17 12:37:16,283] jepsen worker 13 - jepsen.util 10993 :fail :cas [4 [17 4]] :ValueNotSwapFrom INFO [2019-12-17 12:37:16,283] jepsen worker 13 - jepsen.util 10993 :invoke :write [4 49] INFO [2019-12-17 12:37:16,293] jepsen worker 64 - jepsen.util 8884 :fail :cas [21 [3 19]] :ValueNotSwapFrom INFO [2019-12-17 12:37:16,293] jepsen worker 64 - jepsen.util 8884 :invoke :write [21 20] INFO [2019-12-17 12:37:16,300] jepsen worker 22 - jepsen.util 15952 :info :cas [7 [9 36]] :RequestCanceledException INFO [2019-12-17 12:37:16,300] jepsen worker 83 - jepsen.util 16103 :info :write [27 41] :RequestCanceledException INFO [2019-12-17 12:37:16,301] jepsen worker 22 - jepsen.util 16042 :invoke :read [7 nil] INFO [2019-12-17 12:37:16,301] jepsen worker 83 - jepsen.util 16193 :invoke :cas [27 [4 11]] INFO [2019-12-17 12:37:16,304] jepsen worker 12 - jepsen.util 11892 :ok :read [4 0] INFO [2019-12-17 12:37:16,304] jepsen worker 12 - jepsen.util 11892 :invoke :write [4 8] INFO [2019-12-17 12:37:16,317] jepsen worker 34 - jepsen.util 16414 :info :write [11 32] :RequestCanceledException INFO [2019-12-17 12:37:16,318] jepsen worker 34 - jepsen.util 16504 :invoke :read [11 nil] INFO [2019-12-17 12:37:16,325] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:37:16,325] jepsen nemesis - couchbase.util Rebalance check count: 532 INFO [2019-12-17 12:37:16,325] jepsen nemesis - couchbase.util Rebalance stuck count: 513 INFO [2019-12-17 12:37:16,333] jepsen worker 39 - jepsen.util 15339 :info :cas [13 [32 6]] :RequestCanceledException INFO [2019-12-17 12:37:16,333] jepsen worker 1 - jepsen.util 15931 :info :cas [0 [39 40]] :RequestCanceledException INFO [2019-12-17 12:37:16,333] jepsen worker 19 - jepsen.util 11989 :info :write [6 32] :RequestCanceledException INFO [2019-12-17 12:37:16,333] jepsen worker 76 - jepsen.util 16186 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:37:16,333] jepsen worker 76 - jepsen.util 16186 :invoke :read [25 nil] INFO [2019-12-17 12:37:16,334] jepsen worker 19 - jepsen.util 12079 :invoke :write [6 41] INFO [2019-12-17 12:37:16,334] jepsen worker 1 - jepsen.util 16021 :invoke :write [0 8] INFO [2019-12-17 12:37:16,334] jepsen worker 39 - jepsen.util 15429 :invoke :cas [13 [17 26]] INFO [2019-12-17 12:37:16,340] jepsen worker 64 - jepsen.util 8884 :ok :write [21 20] INFO [2019-12-17 12:37:16,340] jepsen worker 64 - jepsen.util 8884 :invoke :write [21 5] INFO [2019-12-17 12:37:16,347] jepsen worker 49 - jepsen.util 15889 :info :write [16 15] :RequestCanceledException INFO [2019-12-17 12:37:16,348] jepsen worker 49 - jepsen.util 15979 :invoke :cas [16 [49 15]] INFO [2019-12-17 12:37:16,362] jepsen worker 71 - jepsen.util 12401 :info :write [23 8] :RequestCanceledException INFO [2019-12-17 12:37:16,362] jepsen worker 57 - jepsen.util 15807 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:37:16,362] jepsen worker 60 - jepsen.util 17160 :info :write [20 20] :RequestCanceledException INFO [2019-12-17 12:37:16,362] jepsen worker 57 - jepsen.util 15807 :invoke :write [19 1] INFO [2019-12-17 12:37:16,363] jepsen worker 71 - jepsen.util 12491 :invoke :read [23 nil] INFO [2019-12-17 12:37:16,363] jepsen worker 60 - jepsen.util 17250 :invoke :read [20 nil] INFO [2019-12-17 12:37:16,373] jepsen worker 46 - jepsen.util 11476 :ok :write [15 4] INFO [2019-12-17 12:37:16,373] jepsen worker 46 - jepsen.util 11476 :invoke :read [15 nil] INFO [2019-12-17 12:37:16,396] jepsen worker 46 - jepsen.util 11476 :ok :read [15 4] INFO [2019-12-17 12:37:16,396] jepsen worker 46 - jepsen.util 11476 :invoke :cas [15 [37 32]] INFO [2019-12-17 12:37:16,396] jepsen worker 64 - jepsen.util 8884 :ok :write [21 5] INFO [2019-12-17 12:37:16,397] jepsen worker 64 - jepsen.util 8884 :invoke :write [21 31] INFO [2019-12-17 12:37:16,411] jepsen worker 89 - jepsen.util 11699 :ok :read [29 9] INFO [2019-12-17 12:37:16,411] jepsen worker 89 - jepsen.util 11699 :invoke :write [29 3] INFO [2019-12-17 12:37:16,419] jepsen worker 46 - jepsen.util 11476 :fail :cas [15 [37 32]] :ValueNotSwapFrom INFO [2019-12-17 12:37:16,419] jepsen worker 46 - jepsen.util 11476 :invoke :read [15 nil] INFO [2019-12-17 12:37:16,433] jepsen worker 46 - jepsen.util 11476 :ok :read [15 4] INFO [2019-12-17 12:37:16,433] jepsen worker 46 - jepsen.util 11476 :invoke :cas [15 [15 37]] INFO [2019-12-17 12:37:16,451] jepsen worker 46 - jepsen.util 11476 :fail :cas [15 [15 37]] :ValueNotSwapFrom INFO [2019-12-17 12:37:16,451] jepsen worker 46 - jepsen.util 11476 :invoke :write [15 38] INFO [2019-12-17 12:37:16,461] jepsen worker 64 - jepsen.util 8884 :ok :write [21 31] INFO [2019-12-17 12:37:16,465] jepsen worker 64 - jepsen.util 8884 :invoke :read [21 nil] INFO [2019-12-17 12:37:16,478] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":67680,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:37:16,478] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":67680,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:37:16,479] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":67684,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:37:16,479] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":67684,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:37:16,488] jepsen worker 64 - jepsen.util 8884 :ok :read [21 31] INFO [2019-12-17 12:37:16,488] jepsen worker 64 - jepsen.util 8884 :invoke :cas [21 [29 23]] INFO [2019-12-17 12:37:16,502] jepsen worker 70 - jepsen.util 12220 :ok :read [23 41] INFO [2019-12-17 12:37:16,502] jepsen worker 70 - jepsen.util 12220 :invoke :cas [23 [27 29]] INFO [2019-12-17 12:37:16,508] jepsen worker 64 - jepsen.util 8884 :fail :cas [21 [29 23]] :ValueNotSwapFrom INFO [2019-12-17 12:37:16,508] jepsen worker 64 - jepsen.util 8884 :invoke :cas [21 [41 10]] INFO [2019-12-17 12:37:16,525] jepsen worker 64 - jepsen.util 8884 :fail :cas [21 [41 10]] :ValueNotSwapFrom INFO [2019-12-17 12:37:16,525] jepsen worker 64 - jepsen.util 8884 :invoke :write [21 45] INFO [2019-12-17 12:37:16,568] jepsen worker 64 - jepsen.util 8884 :ok :write [21 45] INFO [2019-12-17 12:37:16,569] jepsen worker 64 - jepsen.util 8884 :invoke :cas [21 [3 16]] INFO [2019-12-17 12:37:16,588] jepsen worker 64 - jepsen.util 8884 :fail :cas [21 [3 16]] :ValueNotSwapFrom INFO [2019-12-17 12:37:16,588] jepsen worker 64 - jepsen.util 8884 :invoke :write [21 5] INFO [2019-12-17 12:37:16,617] jepsen worker 88 - jepsen.util 11878 :ok :read [29 9] INFO [2019-12-17 12:37:16,617] jepsen worker 88 - jepsen.util 11878 :invoke :write [29 3] INFO [2019-12-17 12:37:16,631] jepsen worker 64 - jepsen.util 8884 :ok :write [21 5] INFO [2019-12-17 12:37:16,631] jepsen worker 64 - jepsen.util 8884 :invoke :write [21 40] INFO [2019-12-17 12:37:16,656] jepsen worker 74 - jepsen.util 11504 :ok :write [24 5] INFO [2019-12-17 12:37:16,656] jepsen worker 74 - jepsen.util 11504 :invoke :cas [24 [13 39]] INFO [2019-12-17 12:37:16,656] jepsen worker 9 - jepsen.util 12339 :ok :write [3 5] INFO [2019-12-17 12:37:16,657] jepsen worker 9 - jepsen.util 12339 :invoke :cas [3 [41 11]] INFO [2019-12-17 12:37:16,657] jepsen worker 46 - jepsen.util 11476 :ok :write [15 38] INFO [2019-12-17 12:37:16,657] jepsen worker 46 - jepsen.util 11476 :invoke :write [15 5] INFO [2019-12-17 12:37:16,671] jepsen worker 74 - jepsen.util 11504 :fail :cas [24 [13 39]] :ValueNotSwapFrom INFO [2019-12-17 12:37:16,672] jepsen worker 74 - jepsen.util 11504 :invoke :write [24 16] INFO [2019-12-17 12:37:16,698] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":67696,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:37:16,698] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":67696,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:37:16,698] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":67699,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:37:16,698] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":67699,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:37:16,698] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":67700,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:37:16,698] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":67700,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:37:16,699] jepsen worker 29 - jepsen.util 9929 :info :cas [9 [48 28]] :RequestCanceledException INFO [2019-12-17 12:37:16,700] jepsen worker 29 - jepsen.util 10019 :invoke :read [9 nil] INFO [2019-12-17 12:37:16,715] jepsen worker 64 - jepsen.util 8884 :ok :write [21 40] INFO [2019-12-17 12:37:16,715] jepsen worker 64 - jepsen.util 8884 :invoke :write [21 4] INFO [2019-12-17 12:37:16,718] jepsen worker 33 - jepsen.util 15693 :info :cas [11 [25 32]] :RequestCanceledException INFO [2019-12-17 12:37:16,719] jepsen worker 33 - jepsen.util 15783 :invoke :write [11 2] INFO [2019-12-17 12:37:16,764] jepsen worker 64 - jepsen.util 8884 :ok :write [21 4] INFO [2019-12-17 12:37:16,767] jepsen worker 64 - jepsen.util 8884 :invoke :write [21 35] INFO [2019-12-17 12:37:16,817] jepsen worker 11 - jepsen.util 11261 :fail :read [3 nil] :RequestTimeoutException INFO [2019-12-17 12:37:16,818] jepsen worker 11 - jepsen.util 11261 :invoke :cas [3 [25 13]] INFO [2019-12-17 12:37:16,820] jepsen worker 64 - jepsen.util 8884 :ok :write [21 35] INFO [2019-12-17 12:37:16,820] jepsen worker 64 - jepsen.util 8884 :invoke :write [21 9] INFO [2019-12-17 12:37:16,833] jepsen worker 45 - jepsen.util 11745 :info :write [15 31] :RequestCanceledException INFO [2019-12-17 12:37:16,834] jepsen worker 45 - jepsen.util 11835 :invoke :write [15 15] INFO [2019-12-17 12:37:16,855] jepsen worker 87 - jepsen.util 11247 :ok :read [29 9] INFO [2019-12-17 12:37:16,855] jepsen worker 87 - jepsen.util 11247 :invoke :read [29 nil] INFO [2019-12-17 12:37:16,892] jepsen worker 56 - jepsen.util 12656 :ok :read [18 13] INFO [2019-12-17 12:37:16,892] jepsen worker 56 - jepsen.util 12656 :invoke :cas [18 [9 42]] INFO [2019-12-17 12:37:16,894] jepsen worker 19 - jepsen.util 12079 :ok :write [6 41] INFO [2019-12-17 12:37:16,894] jepsen worker 19 - jepsen.util 12079 :invoke :read [6 nil] INFO [2019-12-17 12:37:16,916] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":67714,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:37:16,916] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":67714,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:37:16,916] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":67717,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:37:16,916] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":67717,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:37:16,933] jepsen worker 65 - jepsen.util 10325 :info :cas [21 [17 33]] :RequestCanceledException INFO [2019-12-17 12:37:16,933] jepsen worker 53 - jepsen.util 11213 :info :write [17 31] :RequestCanceledException INFO [2019-12-17 12:37:16,933] jepsen worker 28 - jepsen.util 9748 :info :write [9 20] :RequestCanceledException INFO [2019-12-17 12:37:16,934] jepsen worker 53 - jepsen.util 11303 :invoke :cas [17 [17 10]] INFO [2019-12-17 12:37:16,935] jepsen worker 74 - jepsen.util 11504 :ok :write [24 16] INFO [2019-12-17 12:37:16,935] jepsen worker 46 - jepsen.util 11476 :ok :write [15 5] INFO [2019-12-17 12:37:16,935] jepsen worker 74 - jepsen.util 11504 :invoke :write [24 29] INFO [2019-12-17 12:37:16,935] jepsen worker 46 - jepsen.util 11476 :invoke :write [15 34] INFO [2019-12-17 12:37:16,937] jepsen worker 65 - jepsen.util 10415 :invoke :write [21 26] INFO [2019-12-17 12:37:16,947] jepsen worker 28 - jepsen.util 9838 :invoke :write [9 1] INFO [2019-12-17 12:37:16,970] jepsen worker 64 - jepsen.util 8884 :ok :write [21 9] INFO [2019-12-17 12:37:16,970] jepsen worker 64 - jepsen.util 8884 :invoke :cas [21 [8 47]] INFO [2019-12-17 12:37:16,988] jepsen worker 71 - jepsen.util 12491 :ok :read [23 41] INFO [2019-12-17 12:37:16,989] jepsen worker 71 - jepsen.util 12491 :invoke :read [23 nil] INFO [2019-12-17 12:37:17,027] jepsen worker 11 - jepsen.util 11261 :fail :cas [3 [25 13]] :ValueNotSwapFrom INFO [2019-12-17 12:37:17,027] jepsen worker 11 - jepsen.util 11261 :invoke :read [3 nil] INFO [2019-12-17 12:37:17,040] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":67728,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:37:17,040] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":67728,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:37:17,040] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":67730,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:37:17,040] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":67730,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:37:17,046] jepsen worker 9 - jepsen.util 12339 :fail :cas [3 [41 11]] :ValueNotSwapFrom INFO [2019-12-17 12:37:17,046] jepsen worker 9 - jepsen.util 12339 :invoke :read [3 nil] INFO [2019-12-17 12:37:17,050] jepsen worker 29 - jepsen.util 10019 :ok :read [9 45] INFO [2019-12-17 12:37:17,050] jepsen worker 29 - jepsen.util 10019 :invoke :write [9 36] INFO [2019-12-17 12:37:17,064] jepsen worker 14 - jepsen.util 11354 :info :write [4 8] :RequestCanceledException INFO [2019-12-17 12:37:17,065] jepsen worker 14 - jepsen.util 11444 :invoke :read [4 nil] INFO [2019-12-17 12:37:17,079] jepsen worker 19 - jepsen.util 12079 :ok :read [6 41] INFO [2019-12-17 12:37:17,079] jepsen worker 19 - jepsen.util 12079 :invoke :read [6 nil] INFO [2019-12-17 12:37:17,099] jepsen worker 79 - jepsen.util 11869 :info :write [26 1] :RequestCanceledException INFO [2019-12-17 12:37:17,100] jepsen worker 79 - jepsen.util 11959 :invoke :write [26 34] INFO [2019-12-17 12:37:17,118] jepsen worker 63 - jepsen.util 10233 :fail :read [21 nil] :RequestTimeoutException INFO [2019-12-17 12:37:17,118] jepsen worker 63 - jepsen.util 10233 :invoke :read [21 nil] INFO [2019-12-17 12:37:17,121] jepsen worker 9 - jepsen.util 12339 :ok :read [3 5] INFO [2019-12-17 12:37:17,121] jepsen worker 9 - jepsen.util 12339 :invoke :cas [3 [3 49]] INFO [2019-12-17 12:37:17,164] jepsen worker 69 - jepsen.util 11949 :info :write [23 47] :RequestCanceledException INFO [2019-12-17 12:37:17,165] jepsen worker 69 - jepsen.util 12039 :invoke :cas [23 [1 25]] INFO [2019-12-17 12:37:17,170] jepsen worker 53 - jepsen.util 11303 :fail :cas [17 [17 10]] :ValueNotSwapFrom INFO [2019-12-17 12:37:17,170] jepsen worker 53 - jepsen.util 11303 :invoke :write [17 15] INFO [2019-12-17 12:37:17,190] jepsen worker 65 - jepsen.util 10415 :ok :write [21 26] INFO [2019-12-17 12:37:17,190] jepsen worker 65 - jepsen.util 10415 :invoke :cas [21 [45 20]] INFO [2019-12-17 12:37:17,200] jepsen worker 66 - jepsen.util 15816 :info :write [22 6] :RequestCanceledException INFO [2019-12-17 12:37:17,201] jepsen worker 66 - jepsen.util 15906 :invoke :write [22 27] INFO [2019-12-17 12:37:17,213] jepsen worker 65 - jepsen.util 10415 :fail :cas [21 [45 20]] :ValueNotSwapFrom INFO [2019-12-17 12:37:17,213] jepsen worker 65 - jepsen.util 10415 :invoke :read [21 nil] INFO [2019-12-17 12:37:17,217] jepsen worker 82 - jepsen.util 16282 :info :cas [27 [40 39]] :RequestCanceledException INFO [2019-12-17 12:37:17,218] jepsen worker 82 - jepsen.util 16372 :invoke :write [27 15] INFO [2019-12-17 12:37:17,233] jepsen worker 62 - jepsen.util 16712 :info :write [20 38] :RequestCanceledException INFO [2019-12-17 12:37:17,233] jepsen worker 17 - jepsen.util 15947 :info :cas [5 [15 47]] :RequestCanceledException INFO [2019-12-17 12:37:17,233] jepsen worker 77 - jepsen.util 16817 :info :cas [25 [30 26]] :RequestCanceledException INFO [2019-12-17 12:37:17,234] jepsen worker 62 - jepsen.util 16802 :invoke :read [20 nil] INFO [2019-12-17 12:37:17,234] jepsen worker 17 - jepsen.util 16037 :invoke :cas [5 [30 27]] INFO [2019-12-17 12:37:17,234] jepsen worker 77 - jepsen.util 16907 :invoke :write [25 26] INFO [2019-12-17 12:37:17,241] jepsen worker 29 - jepsen.util 10019 :ok :write [9 36] INFO [2019-12-17 12:37:17,241] jepsen worker 29 - jepsen.util 10019 :invoke :read [9 nil] INFO [2019-12-17 12:37:17,244] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":67746,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:37:17,244] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":67746,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:37:17,246] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":67753,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:37:17,246] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":67753,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:37:17,247] jepsen worker 26 - jepsen.util 16676 :info :cas [8 [9 30]] :RequestCanceledException INFO [2019-12-17 12:37:17,247] jepsen worker 80 - jepsen.util 11330 :info :write [26 16] :RequestCanceledException INFO [2019-12-17 12:37:17,247] jepsen worker 84 - jepsen.util 15654 :info :write [28 11] :RequestCanceledException INFO [2019-12-17 12:37:17,247] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":67754,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:37:17,247] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":67754,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:37:17,248] jepsen worker 84 - jepsen.util 15744 :invoke :cas [28 [18 25]] INFO [2019-12-17 12:37:17,250] jepsen worker 80 - jepsen.util 11420 :invoke :read [26 nil] INFO [2019-12-17 12:37:17,251] jepsen worker 65 - jepsen.util 10415 :ok :read [21 26] INFO [2019-12-17 12:37:17,251] jepsen worker 65 - jepsen.util 10415 :invoke :read [21 nil] INFO [2019-12-17 12:37:17,254] jepsen worker 26 - jepsen.util 16766 :invoke :cas [8 [32 36]] INFO [2019-12-17 12:37:17,256] jepsen worker 74 - jepsen.util 11504 :ok :write [24 29] INFO [2019-12-17 12:37:17,256] jepsen worker 74 - jepsen.util 11504 :invoke :write [24 18] INFO [2019-12-17 12:37:17,257] jepsen worker 29 - jepsen.util 10019 :ok :read [9 36] INFO [2019-12-17 12:37:17,257] jepsen worker 29 - jepsen.util 10019 :invoke :read [9 nil] INFO [2019-12-17 12:37:17,277] jepsen worker 65 - jepsen.util 10415 :ok :read [21 26] INFO [2019-12-17 12:37:17,277] jepsen worker 65 - jepsen.util 10415 :invoke :cas [21 [39 47]] INFO [2019-12-17 12:37:17,281] jepsen worker 69 - jepsen.util 12039 :fail :cas [23 [1 25]] :ValueNotSwapFrom INFO [2019-12-17 12:37:17,281] jepsen worker 69 - jepsen.util 12039 :invoke :cas [23 [14 34]] INFO [2019-12-17 12:37:17,299] jepsen worker 25 - jepsen.util 17035 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:37:17,299] jepsen worker 25 - jepsen.util 17035 :invoke :read [8 nil] INFO [2019-12-17 12:37:17,313] jepsen worker 46 - jepsen.util 11476 :ok :write [15 34] INFO [2019-12-17 12:37:17,313] jepsen worker 46 - jepsen.util 11476 :invoke :read [15 nil] INFO [2019-12-17 12:37:17,323] jepsen worker 29 - jepsen.util 10019 :ok :read [9 36] INFO [2019-12-17 12:37:17,323] jepsen worker 29 - jepsen.util 10019 :invoke :write [9 9] INFO [2019-12-17 12:37:17,333] jepsen worker 48 - jepsen.util 14808 :info :write [16 44] :RequestCanceledException INFO [2019-12-17 12:37:17,333] jepsen worker 23 - jepsen.util 16133 :info :cas [7 [6 41]] :RequestCanceledException INFO [2019-12-17 12:37:17,333] jepsen worker 58 - jepsen.util 16348 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:37:17,333] jepsen worker 58 - jepsen.util 16348 :invoke :write [19 18] INFO [2019-12-17 12:37:17,334] jepsen worker 48 - jepsen.util 14898 :invoke :read [16 nil] INFO [2019-12-17 12:37:17,334] jepsen worker 23 - jepsen.util 16223 :invoke :cas [7 [15 3]] INFO [2019-12-17 12:37:17,343] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:37:17,343] jepsen nemesis - couchbase.util Rebalance check count: 533 INFO [2019-12-17 12:37:17,343] jepsen nemesis - couchbase.util Rebalance stuck count: 514 INFO [2019-12-17 12:37:17,347] jepsen worker 65 - jepsen.util 10415 :fail :cas [21 [39 47]] :ValueNotSwapFrom INFO [2019-12-17 12:37:17,347] jepsen worker 65 - jepsen.util 10415 :invoke :cas [21 [21 17]] INFO [2019-12-17 12:37:17,347] jepsen worker 2 - jepsen.util 15482 :info :cas [0 [21 44]] :RequestCanceledException INFO [2019-12-17 12:37:17,348] jepsen worker 2 - jepsen.util 15572 :invoke :read [0 nil] INFO [2019-12-17 12:37:17,350] jepsen worker 63 - jepsen.util 10233 :ok :read [21 26] INFO [2019-12-17 12:37:17,350] jepsen worker 63 - jepsen.util 10233 :invoke :cas [21 [5 37]] INFO [2019-12-17 12:37:17,360] jepsen worker 46 - jepsen.util 11476 :ok :read [15 34] INFO [2019-12-17 12:37:17,360] jepsen worker 46 - jepsen.util 11476 :invoke :cas [15 [34 4]] INFO [2019-12-17 12:37:17,364] jepsen worker 86 - jepsen.util 16736 :info :cas [28 [46 17]] :RequestCanceledException INFO [2019-12-17 12:37:17,365] jepsen worker 86 - jepsen.util 16826 :invoke :cas [28 [33 46]] INFO [2019-12-17 12:37:17,373] jepsen worker 63 - jepsen.util 10233 :fail :cas [21 [5 37]] :ValueNotSwapFrom INFO [2019-12-17 12:37:17,373] jepsen worker 63 - jepsen.util 10233 :invoke :cas [21 [11 21]] INFO [2019-12-17 12:37:17,397] jepsen worker 65 - jepsen.util 10415 :fail :cas [21 [21 17]] :ValueNotSwapFrom INFO [2019-12-17 12:37:17,397] jepsen worker 65 - jepsen.util 10415 :invoke :cas [21 [6 21]] INFO [2019-12-17 12:37:17,399] jepsen worker 32 - jepsen.util 12002 :info :cas [10 [49 49]] :RequestCanceledException INFO [2019-12-17 12:37:17,399] jepsen worker 43 - jepsen.util 16153 :info :cas [14 [20 41]] :RequestCanceledException INFO [2019-12-17 12:37:17,400] jepsen worker 63 - jepsen.util 10233 :fail :cas [21 [11 21]] :ValueNotSwapFrom INFO [2019-12-17 12:37:17,400] jepsen worker 63 - jepsen.util 10233 :invoke :cas [21 [34 14]] INFO [2019-12-17 12:37:17,400] jepsen worker 43 - jepsen.util 16243 :invoke :read [14 nil] INFO [2019-12-17 12:37:17,400] jepsen worker 32 - jepsen.util 12092 :invoke :write [10 45] INFO [2019-12-17 12:37:17,413] jepsen worker 65 - jepsen.util 10415 :fail :cas [21 [6 21]] :ValueNotSwapFrom INFO [2019-12-17 12:37:17,414] jepsen worker 65 - jepsen.util 10415 :invoke :read [21 nil] INFO [2019-12-17 12:37:17,418] jepsen worker 59 - jepsen.util 15269 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:37:17,418] jepsen worker 7 - jepsen.util 16027 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:37:17,418] jepsen worker 59 - jepsen.util 15269 :invoke :write [19 40] INFO [2019-12-17 12:37:17,418] jepsen worker 7 - jepsen.util 16027 :invoke :write [2 48] INFO [2019-12-17 12:37:17,420] jepsen worker 70 - jepsen.util 12220 :fail :cas [23 [27 29]] :ValueNotSwapFrom INFO [2019-12-17 12:37:17,420] jepsen worker 70 - jepsen.util 12220 :invoke :read [23 nil] INFO [2019-12-17 12:37:17,433] jepsen worker 51 - jepsen.util 10941 :info :write [17 16] :RequestCanceledException INFO [2019-12-17 12:37:17,433] jepsen worker 67 - jepsen.util 15907 :info :cas [22 [28 0]] :RequestCanceledException INFO [2019-12-17 12:37:17,434] jepsen worker 51 - jepsen.util 11031 :invoke :write [17 3] INFO [2019-12-17 12:37:17,434] jepsen worker 67 - jepsen.util 15997 :invoke :write [22 8] INFO [2019-12-17 12:37:17,438] jepsen worker 29 - jepsen.util 10019 :ok :write [9 9] INFO [2019-12-17 12:37:17,438] jepsen worker 29 - jepsen.util 10019 :invoke :write [9 8] INFO [2019-12-17 12:37:17,447] jepsen worker 8 - jepsen.util 15668 :info :cas [2 [34 48]] :RequestCanceledException INFO [2019-12-17 12:37:17,447] jepsen worker 85 - jepsen.util 16645 :info :cas [28 [28 8]] :RequestCanceledException INFO [2019-12-17 12:37:17,448] jepsen worker 8 - jepsen.util 15758 :invoke :read [2 nil] INFO [2019-12-17 12:37:17,448] jepsen worker 85 - jepsen.util 16735 :invoke :cas [28 [39 14]] INFO [2019-12-17 12:37:17,464] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":67795,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:37:17,464] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":67795,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:37:17,464] jepsen worker 21 - jepsen.util 15411 :info :write [7 10] :RequestCanceledException INFO [2019-12-17 12:37:17,464] jepsen worker 50 - jepsen.util 16250 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:37:17,464] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":67798,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:37:17,464] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":67798,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:37:17,464] jepsen worker 50 - jepsen.util 16250 :invoke :read [16 nil] INFO [2019-12-17 12:37:17,465] jepsen worker 21 - jepsen.util 15501 :invoke :write [7 8] INFO [2019-12-17 12:37:17,476] jepsen worker 29 - jepsen.util 10019 :ok :write [9 8] INFO [2019-12-17 12:37:17,476] jepsen worker 29 - jepsen.util 10019 :invoke :read [9 nil] INFO [2019-12-17 12:37:17,491] jepsen worker 29 - jepsen.util 10019 :ok :read [9 8] INFO [2019-12-17 12:37:17,491] jepsen worker 29 - jepsen.util 10019 :invoke :write [9 34] INFO [2019-12-17 12:37:17,500] jepsen worker 0 - jepsen.util 16650 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:37:17,500] jepsen worker 0 - jepsen.util 16650 :invoke :cas [0 [18 2]] INFO [2019-12-17 12:37:17,505] jepsen worker 80 - jepsen.util 11420 :ok :read [26 44] INFO [2019-12-17 12:37:17,505] jepsen worker 80 - jepsen.util 11420 :invoke :cas [26 [16 14]] INFO [2019-12-17 12:37:17,513] jepsen worker 69 - jepsen.util 12039 :fail :cas [23 [14 34]] :ValueNotSwapFrom INFO [2019-12-17 12:37:17,514] jepsen worker 69 - jepsen.util 12039 :invoke :cas [23 [20 0]] INFO [2019-12-17 12:37:17,517] jepsen worker 15 - jepsen.util 14865 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:37:17,518] jepsen worker 15 - jepsen.util 14865 :invoke :read [5 nil] INFO [2019-12-17 12:37:17,533] jepsen worker 37 - jepsen.util 11287 :info :write [12 15] :RequestCanceledException INFO [2019-12-17 12:37:17,533] jepsen worker 61 - jepsen.util 17161 :info :write [20 10] :RequestCanceledException INFO [2019-12-17 12:37:17,534] jepsen worker 61 - jepsen.util 17251 :invoke :write [20 36] INFO [2019-12-17 12:37:17,534] jepsen worker 37 - jepsen.util 11377 :invoke :cas [12 [18 29]] INFO [2019-12-17 12:37:17,535] jepsen worker 29 - jepsen.util 10019 :ok :write [9 34] INFO [2019-12-17 12:37:17,536] jepsen worker 29 - jepsen.util 10019 :invoke :cas [9 [31 2]] INFO [2019-12-17 12:37:17,550] jepsen worker 29 - jepsen.util 10019 :fail :cas [9 [31 2]] :ValueNotSwapFrom INFO [2019-12-17 12:37:17,550] jepsen worker 29 - jepsen.util 10019 :invoke :write [9 10] INFO [2019-12-17 12:37:17,592] jepsen worker 29 - jepsen.util 10019 :ok :write [9 10] INFO [2019-12-17 12:37:17,592] jepsen worker 29 - jepsen.util 10019 :invoke :cas [9 [32 39]] INFO [2019-12-17 12:37:17,595] jepsen worker 87 - jepsen.util 11247 :ok :read [29 9] INFO [2019-12-17 12:37:17,596] jepsen worker 87 - jepsen.util 11247 :invoke :cas [29 [10 3]] INFO [2019-12-17 12:37:17,599] jepsen worker 73 - jepsen.util 10693 :info :write [24 8] :RequestCanceledException INFO [2019-12-17 12:37:17,600] jepsen worker 73 - jepsen.util 10783 :invoke :cas [24 [30 36]] INFO [2019-12-17 12:37:17,609] jepsen worker 29 - jepsen.util 10019 :fail :cas [9 [32 39]] :ValueNotSwapFrom INFO [2019-12-17 12:37:17,609] jepsen worker 29 - jepsen.util 10019 :invoke :read [9 nil] INFO [2019-12-17 12:37:17,616] jepsen worker 73 - jepsen.util 10783 :fail :cas [24 [30 36]] :ValueNotSwapFrom INFO [2019-12-17 12:37:17,616] jepsen worker 73 - jepsen.util 10783 :invoke :write [24 5] INFO [2019-12-17 12:37:17,624] jepsen worker 29 - jepsen.util 10019 :ok :read [9 10] INFO [2019-12-17 12:37:17,624] jepsen worker 29 - jepsen.util 10019 :invoke :cas [9 [20 36]] INFO [2019-12-17 12:37:17,633] jepsen worker 4 - jepsen.util 10804 :info :write [1 32] :RequestCanceledException INFO [2019-12-17 12:37:17,634] jepsen worker 4 - jepsen.util 10894 :invoke :cas [1 [30 34]] INFO [2019-12-17 12:37:17,647] jepsen worker 55 - jepsen.util 12295 :info :write [18 34] :RequestCanceledException INFO [2019-12-17 12:37:17,652] jepsen worker 55 - jepsen.util 12385 :invoke :read [18 nil] INFO [2019-12-17 12:37:17,665] jepsen worker 56 - jepsen.util 12656 :fail :cas [18 [9 42]] :ValueNotSwapFrom INFO [2019-12-17 12:37:17,666] jepsen worker 56 - jepsen.util 12656 :invoke :cas [18 [9 43]] INFO [2019-12-17 12:37:17,666] jepsen worker 69 - jepsen.util 12039 :fail :cas [23 [20 0]] :ValueNotSwapFrom INFO [2019-12-17 12:37:17,666] jepsen worker 69 - jepsen.util 12039 :invoke :write [23 3] INFO [2019-12-17 12:37:17,682] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":67823,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:37:17,682] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":67823,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:37:17,682] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":67827,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:37:17,682] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":67827,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:37:17,712] jepsen worker 37 - jepsen.util 11377 :fail :cas [12 [18 29]] :ValueNotSwapFrom INFO [2019-12-17 12:37:17,712] jepsen worker 37 - jepsen.util 11377 :invoke :cas [12 [35 42]] INFO [2019-12-17 12:37:17,717] jepsen worker 31 - jepsen.util 10021 :fail :read [10 nil] :RequestTimeoutException INFO [2019-12-17 12:37:17,717] jepsen worker 72 - jepsen.util 11952 :fail :read [24 nil] :RequestTimeoutException INFO [2019-12-17 12:37:17,717] jepsen worker 31 - jepsen.util 10021 :invoke :read [10 nil] INFO [2019-12-17 12:37:17,717] jepsen worker 72 - jepsen.util 11952 :invoke :cas [24 [26 23]] INFO [2019-12-17 12:37:17,729] jepsen worker 55 - jepsen.util 12385 :ok :read [18 13] INFO [2019-12-17 12:37:17,729] jepsen worker 55 - jepsen.util 12385 :invoke :read [18 nil] INFO [2019-12-17 12:37:17,733] jepsen worker 3 - jepsen.util 11073 :info :write [1 26] :RequestCanceledException INFO [2019-12-17 12:37:17,733] jepsen worker 3 - jepsen.util 11163 :invoke :write [1 25] INFO [2019-12-17 12:37:17,750] jepsen worker 37 - jepsen.util 11377 :fail :cas [12 [35 42]] :ValueNotSwapFrom INFO [2019-12-17 12:37:17,750] jepsen worker 37 - jepsen.util 11377 :invoke :read [12 nil] INFO [2019-12-17 12:37:17,760] jepsen worker 71 - jepsen.util 12491 :ok :read [23 41] INFO [2019-12-17 12:37:17,763] jepsen worker 71 - jepsen.util 12491 :invoke :cas [23 [33 44]] INFO [2019-12-17 12:37:17,764] jepsen worker 18 - jepsen.util 11718 :info :cas [6 [23 23]] :RequestCanceledException INFO [2019-12-17 12:37:17,765] jepsen worker 18 - jepsen.util 11808 :invoke :write [6 15] INFO [2019-12-17 12:37:17,833] jepsen worker 36 - jepsen.util 12456 :info :write [12 29] :RequestCanceledException INFO [2019-12-17 12:37:17,833] jepsen worker 47 - jepsen.util 12557 :info :write [15 43] :RequestCanceledException INFO [2019-12-17 12:37:17,834] jepsen worker 36 - jepsen.util 12546 :invoke :read [12 nil] INFO [2019-12-17 12:37:17,834] jepsen worker 47 - jepsen.util 12647 :invoke :cas [15 [30 0]] INFO [2019-12-17 12:37:17,838] jepsen worker 80 - jepsen.util 11420 :fail :cas [26 [16 14]] :ValueNotSwapFrom INFO [2019-12-17 12:37:17,838] jepsen worker 80 - jepsen.util 11420 :invoke :cas [26 [45 49]] INFO [2019-12-17 12:37:17,840] jepsen worker 4 - jepsen.util 10894 :fail :cas [1 [30 34]] :ValueNotSwapFrom INFO [2019-12-17 12:37:17,841] jepsen worker 4 - jepsen.util 10894 :invoke :write [1 30] INFO [2019-12-17 12:37:17,866] jepsen worker 87 - jepsen.util 11247 :fail :cas [29 [10 3]] :ValueNotSwapFrom INFO [2019-12-17 12:37:17,878] jepsen worker 47 - jepsen.util 12647 :fail :cas [15 [30 0]] :ValueNotSwapFrom INFO [2019-12-17 12:37:17,895] jepsen worker 55 - jepsen.util 12385 :ok :read [18 13] INFO [2019-12-17 12:37:17,899] jepsen worker 20 - jepsen.util 11270 :info :cas [6 [46 49]] :RequestCanceledException INFO [2019-12-17 12:37:17,899] jepsen worker 27 - jepsen.util 10827 :info :write [9 27] :RequestCanceledException INFO [2019-12-17 12:37:17,901] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":67845,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:37:17,918] jepsen worker 52 - jepsen.util 11032 :info :write [17 40] :RequestCanceledException INFO [2019-12-17 12:37:17,933] jepsen worker 5 - jepsen.util 11435 :info :write [1 46] :RequestCanceledException INFO [2019-12-17 12:37:17,933] jepsen worker 30 - jepsen.util 10740 :info :cas [10 [35 18]] :RequestCanceledException INFO [2019-12-17 12:37:17,971] jepsen worker 47 - jepsen.util 12647 :invoke :cas [15 [5 26]] INFO [2019-12-17 12:37:17,971] jepsen worker 87 - jepsen.util 11247 :invoke :cas [29 [13 30]] INFO [2019-12-17 12:37:17,972] jepsen worker 55 - jepsen.util 12385 :invoke :read [18 nil] INFO [2019-12-17 12:37:17,972] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":67845,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:37:17,972] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":67849,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:37:17,972] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":67849,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:37:17,973] jepsen worker 27 - jepsen.util 10917 :invoke :cas [9 [0 6]] INFO [2019-12-17 12:37:17,973] jepsen worker 20 - jepsen.util 11360 :invoke :read [6 nil] INFO [2019-12-17 12:37:17,973] jepsen worker 52 - jepsen.util 11122 :invoke :write [17 22] INFO [2019-12-17 12:37:17,973] jepsen worker 5 - jepsen.util 11525 :invoke :cas [1 [42 20]] INFO [2019-12-17 12:37:17,974] jepsen worker 30 - jepsen.util 10830 :invoke :read [10 nil] INFO [2019-12-17 12:37:17,982] jepsen worker 37 - jepsen.util 11377 :ok :read [12 2] INFO [2019-12-17 12:37:17,982] jepsen worker 37 - jepsen.util 11377 :invoke :read [12 nil] INFO [2019-12-17 12:37:17,996] jepsen worker 27 - jepsen.util 10917 :fail :cas [9 [0 6]] :ValueNotSwapFrom INFO [2019-12-17 12:37:17,996] jepsen worker 47 - jepsen.util 12647 :fail :cas [15 [5 26]] :ValueNotSwapFrom INFO [2019-12-17 12:37:17,996] jepsen worker 27 - jepsen.util 10917 :invoke :cas [9 [1 49]] INFO [2019-12-17 12:37:17,996] jepsen worker 47 - jepsen.util 12647 :invoke :write [15 23] INFO [2019-12-17 12:37:17,999] jepsen worker 16 - jepsen.util 17746 :info :cas [5 [19 8]] :RequestCanceledException INFO [2019-12-17 12:37:17,999] jepsen worker 78 - jepsen.util 11508 :info :cas [26 [44 31]] :RequestCanceledException INFO [2019-12-17 12:37:17,999] jepsen worker 20 - jepsen.util 11360 :ok :read [6 41] INFO [2019-12-17 12:37:17,999] jepsen worker 20 - jepsen.util 11360 :invoke :write [6 30] INFO [2019-12-17 12:37:18,000] jepsen worker 78 - jepsen.util 11598 :invoke :cas [26 [30 6]] INFO [2019-12-17 12:37:18,000] jepsen worker 16 - jepsen.util 17836 :invoke :write [5 43] INFO [2019-12-17 12:37:18,017] jepsen worker 44 - jepsen.util 16514 :info :write [14 33] :RequestCanceledException INFO [2019-12-17 12:37:18,017] jepsen worker 81 - jepsen.util 15561 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:37:18,017] jepsen worker 81 - jepsen.util 15561 :invoke :write [27 2] INFO [2019-12-17 12:37:18,018] jepsen worker 44 - jepsen.util 16604 :invoke :write [14 42] INFO [2019-12-17 12:37:18,025] jepsen worker 27 - jepsen.util 10917 :fail :cas [9 [1 49]] :ValueNotSwapFrom INFO [2019-12-17 12:37:18,025] jepsen worker 27 - jepsen.util 10917 :invoke :read [9 nil] INFO [2019-12-17 12:37:18,035] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":67865,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:37:18,036] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":67865,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:37:18,036] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":67867,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:37:18,036] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":67867,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:37:18,038] jepsen worker 36 - jepsen.util 12546 :ok :read [12 2] INFO [2019-12-17 12:37:18,038] jepsen worker 36 - jepsen.util 12546 :invoke :write [12 49] INFO [2019-12-17 12:37:18,048] jepsen worker 27 - jepsen.util 10917 :ok :read [9 10] INFO [2019-12-17 12:37:18,049] jepsen worker 27 - jepsen.util 10917 :invoke :read [9 nil] INFO [2019-12-17 12:37:18,053] jepsen worker 55 - jepsen.util 12385 :ok :read [18 13] INFO [2019-12-17 12:37:18,053] jepsen worker 55 - jepsen.util 12385 :invoke :cas [18 [36 42]] INFO [2019-12-17 12:37:18,061] jepsen worker 27 - jepsen.util 10917 :ok :read [9 10] INFO [2019-12-17 12:37:18,062] jepsen worker 27 - jepsen.util 10917 :invoke :read [9 nil] INFO [2019-12-17 12:37:18,077] jepsen worker 74 - jepsen.util 11504 :ok :write [24 18] INFO [2019-12-17 12:37:18,078] jepsen worker 74 - jepsen.util 11504 :invoke :read [24 nil] INFO [2019-12-17 12:37:18,078] jepsen worker 27 - jepsen.util 10917 :ok :read [9 10] INFO [2019-12-17 12:37:18,079] jepsen worker 27 - jepsen.util 10917 :invoke :cas [9 [35 40]] INFO [2019-12-17 12:37:18,080] jepsen worker 46 - jepsen.util 11476 :ok :cas [15 [34 4]] INFO [2019-12-17 12:37:18,080] jepsen worker 46 - jepsen.util 11476 :invoke :read [15 nil] INFO [2019-12-17 12:37:18,084] jepsen worker 32 - jepsen.util 12092 :ok :write [10 45] INFO [2019-12-17 12:37:18,085] jepsen worker 32 - jepsen.util 12092 :invoke :cas [10 [32 17]] INFO [2019-12-17 12:37:18,095] jepsen worker 20 - jepsen.util 11360 :ok :write [6 30] INFO [2019-12-17 12:37:18,096] jepsen worker 20 - jepsen.util 11360 :invoke :write [6 0] INFO [2019-12-17 12:37:18,114] jepsen worker 30 - jepsen.util 10830 :ok :read [10 45] INFO [2019-12-17 12:37:18,114] jepsen worker 30 - jepsen.util 10830 :invoke :cas [10 [26 34]] INFO [2019-12-17 12:37:18,116] jepsen worker 27 - jepsen.util 10917 :fail :cas [9 [35 40]] :ValueNotSwapFrom INFO [2019-12-17 12:37:18,117] jepsen worker 27 - jepsen.util 10917 :invoke :write [9 29] INFO [2019-12-17 12:37:18,118] jepsen worker 35 - jepsen.util 15785 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:37:18,118] jepsen worker 35 - jepsen.util 15785 :invoke :write [11 15] INFO [2019-12-17 12:37:18,125] jepsen worker 63 - jepsen.util 10233 :fail :cas [21 [34 14]] :ValueNotSwapFrom INFO [2019-12-17 12:37:18,125] jepsen worker 63 - jepsen.util 10233 :invoke :read [21 nil] INFO [2019-12-17 12:37:18,129] jepsen worker 38 - jepsen.util 12098 :info :write [12 19] :RequestCanceledException INFO [2019-12-17 12:37:18,131] jepsen worker 38 - jepsen.util 12188 :invoke :cas [12 [32 35]] INFO [2019-12-17 12:37:18,147] jepsen worker 68 - jepsen.util 15368 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:37:18,148] jepsen worker 68 - jepsen.util 15368 :invoke :cas [22 [48 44]] INFO [2019-12-17 12:37:18,154] jepsen worker 74 - jepsen.util 11504 :ok :read [24 18] INFO [2019-12-17 12:37:18,154] jepsen worker 74 - jepsen.util 11504 :invoke :read [24 nil] INFO [2019-12-17 12:37:18,161] jepsen worker 10 - jepsen.util 11350 :info :cas [3 [38 27]] :RequestCanceledException INFO [2019-12-17 12:37:18,161] jepsen worker 31 - jepsen.util 10021 :ok :read [10 45] INFO [2019-12-17 12:37:18,161] jepsen worker 31 - jepsen.util 10021 :invoke :cas [10 [33 19]] INFO [2019-12-17 12:37:18,162] jepsen worker 10 - jepsen.util 11440 :invoke :read [3 nil] INFO [2019-12-17 12:37:18,169] jepsen worker 56 - jepsen.util 12656 :fail :cas [18 [9 43]] :ValueNotSwapFrom INFO [2019-12-17 12:37:18,169] jepsen worker 56 - jepsen.util 12656 :invoke :read [18 nil] INFO [2019-12-17 12:37:18,194] jepsen worker 55 - jepsen.util 12385 :fail :cas [18 [36 42]] :ValueNotSwapFrom INFO [2019-12-17 12:37:18,194] jepsen worker 55 - jepsen.util 12385 :invoke :read [18 nil] INFO [2019-12-17 12:37:18,208] jepsen worker 30 - jepsen.util 10830 :fail :cas [10 [26 34]] :ValueNotSwapFrom INFO [2019-12-17 12:37:18,209] jepsen worker 30 - jepsen.util 10830 :invoke :cas [10 [16 12]] INFO [2019-12-17 12:37:18,209] jepsen worker 52 - jepsen.util 11122 :ok :write [17 22] INFO [2019-12-17 12:37:18,210] jepsen worker 52 - jepsen.util 11122 :invoke :read [17 nil] INFO [2019-12-17 12:37:18,223] jepsen worker 72 - jepsen.util 11952 :fail :cas [24 [26 23]] :ValueNotSwapFrom INFO [2019-12-17 12:37:18,223] jepsen worker 72 - jepsen.util 11952 :invoke :read [24 nil] INFO [2019-12-17 12:37:18,225] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":67893,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:37:18,225] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":67902,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:37:18,225] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":67893,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:37:18,225] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":67902,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:37:18,225] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":67895,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:37:18,226] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":67906,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:37:18,226] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":67895,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:37:18,226] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":67906,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:37:18,229] jepsen worker 14 - jepsen.util 11444 :ok :read [4 0] INFO [2019-12-17 12:37:18,229] jepsen worker 14 - jepsen.util 11444 :invoke :write [4 0] INFO [2019-12-17 12:37:18,242] jepsen worker 74 - jepsen.util 11504 :ok :read [24 18] INFO [2019-12-17 12:37:18,242] jepsen worker 74 - jepsen.util 11504 :invoke :read [24 nil] INFO [2019-12-17 12:37:18,247] jepsen worker 54 - jepsen.util 12114 :info :write [18 35] :RequestCanceledException INFO [2019-12-17 12:37:18,248] jepsen worker 54 - jepsen.util 12204 :invoke :read [18 nil] INFO [2019-12-17 12:37:18,255] jepsen worker 38 - jepsen.util 12188 :fail :cas [12 [32 35]] :ValueNotSwapFrom INFO [2019-12-17 12:37:18,255] jepsen worker 38 - jepsen.util 12188 :invoke :read [12 nil] INFO [2019-12-17 12:37:18,264] jepsen worker 41 - jepsen.util 17141 :info :write [13 4] :RequestCanceledException INFO [2019-12-17 12:37:18,265] jepsen worker 41 - jepsen.util 17231 :invoke :write [13 22] INFO [2019-12-17 12:37:18,274] jepsen worker 46 - jepsen.util 11476 :ok :read [15 4] INFO [2019-12-17 12:37:18,275] jepsen worker 46 - jepsen.util 11476 :invoke :read [15 nil] INFO [2019-12-17 12:37:18,282] jepsen worker 52 - jepsen.util 11122 :ok :read [17 22] INFO [2019-12-17 12:37:18,282] jepsen worker 52 - jepsen.util 11122 :invoke :write [17 9] INFO [2019-12-17 12:37:18,295] jepsen worker 30 - jepsen.util 10830 :fail :cas [10 [16 12]] :ValueNotSwapFrom INFO [2019-12-17 12:37:18,295] jepsen worker 75 - jepsen.util 15645 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:37:18,295] jepsen worker 30 - jepsen.util 10830 :invoke :cas [10 [44 46]] INFO [2019-12-17 12:37:18,296] jepsen worker 75 - jepsen.util 15645 :invoke :cas [25 [21 35]] INFO [2019-12-17 12:37:18,305] jepsen worker 63 - jepsen.util 10233 :ok :read [21 26] INFO [2019-12-17 12:37:18,305] jepsen worker 63 - jepsen.util 10233 :invoke :read [21 nil] INFO [2019-12-17 12:37:18,317] jepsen worker 22 - jepsen.util 16042 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:37:18,317] jepsen worker 40 - jepsen.util 16510 :info :cas [13 [33 9]] :RequestCanceledException INFO [2019-12-17 12:37:18,317] jepsen worker 22 - jepsen.util 16042 :invoke :read [7 nil] INFO [2019-12-17 12:37:18,319] jepsen worker 40 - jepsen.util 16600 :invoke :cas [13 [35 26]] INFO [2019-12-17 12:37:18,332] jepsen worker 83 - jepsen.util 16193 :info :cas [27 [4 11]] :RequestCanceledException INFO [2019-12-17 12:37:18,332] jepsen worker 24 - jepsen.util 14784 :info :cas [8 [4 47]] :RequestCanceledException INFO [2019-12-17 12:37:18,332] jepsen worker 6 - jepsen.util 16386 :info :cas [2 [27 28]] :RequestCanceledException INFO [2019-12-17 12:37:18,332] jepsen worker 12 - jepsen.util 11892 :info :write [4 8] :RequestCanceledException INFO [2019-12-17 12:37:18,333] jepsen worker 12 - jepsen.util 11982 :invoke :write [4 2] INFO [2019-12-17 12:37:18,333] jepsen worker 71 - jepsen.util 12491 :fail :cas [23 [33 44]] :ValueNotSwapFrom INFO [2019-12-17 12:37:18,333] jepsen worker 71 - jepsen.util 12491 :invoke :write [23 13] INFO [2019-12-17 12:37:18,333] jepsen worker 6 - jepsen.util 16476 :invoke :write [2 25] INFO [2019-12-17 12:37:18,334] jepsen worker 24 - jepsen.util 14874 :invoke :cas [8 [34 36]] INFO [2019-12-17 12:37:18,334] jepsen worker 83 - jepsen.util 16283 :invoke :write [27 43] INFO [2019-12-17 12:37:18,338] jepsen worker 34 - jepsen.util 16504 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:37:18,338] jepsen worker 34 - jepsen.util 16504 :invoke :write [11 24] INFO [2019-12-17 12:37:18,340] jepsen worker 70 - jepsen.util 12220 :ok :read [23 41] INFO [2019-12-17 12:37:18,340] jepsen worker 70 - jepsen.util 12220 :invoke :cas [23 [49 10]] INFO [2019-12-17 12:37:18,357] jepsen worker 88 - jepsen.util 11878 :ok :write [29 3] INFO [2019-12-17 12:37:18,358] jepsen worker 88 - jepsen.util 11878 :invoke :cas [29 [16 3]] INFO [2019-12-17 12:37:18,361] jepsen worker 74 - jepsen.util 11504 :ok :read [24 18] INFO [2019-12-17 12:37:18,362] jepsen worker 74 - jepsen.util 11504 :invoke :read [24 nil] INFO [2019-12-17 12:37:18,364] jepsen worker 42 - jepsen.util 16872 :info :write [14 19] :RequestCanceledException INFO [2019-12-17 12:37:18,364] jepsen worker 13 - jepsen.util 10993 :info :write [4 49] :RequestCanceledException INFO [2019-12-17 12:37:18,365] jepsen worker 13 - jepsen.util 11083 :invoke :cas [4 [20 8]] INFO [2019-12-17 12:37:18,365] jepsen worker 42 - jepsen.util 16962 :invoke :write [14 30] INFO [2019-12-17 12:37:18,372] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:37:18,372] jepsen nemesis - couchbase.util Rebalance check count: 534 INFO [2019-12-17 12:37:18,372] jepsen nemesis - couchbase.util Rebalance stuck count: 515 INFO [2019-12-17 12:37:18,400] jepsen worker 39 - jepsen.util 15429 :info :cas [13 [17 26]] :RequestCanceledException INFO [2019-12-17 12:37:18,405] jepsen worker 39 - jepsen.util 15519 :invoke :cas [13 [6 34]] INFO [2019-12-17 12:37:18,406] jepsen worker 31 - jepsen.util 10021 :fail :cas [10 [33 19]] :ValueNotSwapFrom INFO [2019-12-17 12:37:18,406] jepsen worker 31 - jepsen.util 10021 :invoke :read [10 nil] INFO [2019-12-17 12:37:18,415] jepsen worker 32 - jepsen.util 12092 :fail :cas [10 [32 17]] :ValueNotSwapFrom INFO [2019-12-17 12:37:18,415] jepsen worker 32 - jepsen.util 12092 :invoke :read [10 nil] INFO [2019-12-17 12:37:18,415] jepsen worker 30 - jepsen.util 10830 :fail :cas [10 [44 46]] :ValueNotSwapFrom INFO [2019-12-17 12:37:18,415] jepsen worker 30 - jepsen.util 10830 :invoke :read [10 nil] INFO [2019-12-17 12:37:18,418] jepsen worker 49 - jepsen.util 15979 :info :cas [16 [49 15]] :RequestCanceledException INFO [2019-12-17 12:37:18,419] jepsen worker 49 - jepsen.util 16069 :invoke :cas [16 [24 9]] INFO [2019-12-17 12:37:18,433] jepsen worker 1 - jepsen.util 16021 :info :write [0 8] :RequestCanceledException INFO [2019-12-17 12:37:18,433] jepsen worker 76 - jepsen.util 16186 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:37:18,433] jepsen worker 76 - jepsen.util 16186 :invoke :cas [25 [36 29]] INFO [2019-12-17 12:37:18,434] jepsen worker 1 - jepsen.util 16111 :invoke :write [0 30] INFO [2019-12-17 12:37:18,438] jepsen worker 10 - jepsen.util 11440 :ok :read [3 5] INFO [2019-12-17 12:37:18,438] jepsen worker 10 - jepsen.util 11440 :invoke :cas [3 [3 17]] INFO [2019-12-17 12:37:18,443] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":67938,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:37:18,443] jepsen worker 72 - jepsen.util 11952 :ok :read [24 18] INFO [2019-12-17 12:37:18,443] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":67938,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:37:18,443] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":67941,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:37:18,443] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":67941,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:37:18,443] jepsen worker 72 - jepsen.util 11952 :invoke :write [24 47] INFO [2019-12-17 12:37:18,443] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":67942,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:37:18,443] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":67942,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:37:18,443] jepsen worker 74 - jepsen.util 11504 :ok :read [24 18] INFO [2019-12-17 12:37:18,443] jepsen worker 74 - jepsen.util 11504 :invoke :write [24 0] INFO [2019-12-17 12:37:18,446] jepsen worker 60 - jepsen.util 17250 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:37:18,446] jepsen worker 89 - jepsen.util 11699 :info :write [29 3] :RequestCanceledException INFO [2019-12-17 12:37:18,446] jepsen worker 60 - jepsen.util 17250 :invoke :read [20 nil] INFO [2019-12-17 12:37:18,447] jepsen worker 89 - jepsen.util 11789 :invoke :write [29 34] INFO [2019-12-17 12:37:18,452] jepsen worker 87 - jepsen.util 11247 :fail :cas [29 [13 30]] :ValueNotSwapFrom INFO [2019-12-17 12:37:18,452] jepsen worker 87 - jepsen.util 11247 :invoke :read [29 nil] INFO [2019-12-17 12:37:18,464] jepsen worker 57 - jepsen.util 15807 :info :write [19 1] :RequestCanceledException INFO [2019-12-17 12:37:18,465] jepsen worker 57 - jepsen.util 15897 :invoke :write [19 2] INFO [2019-12-17 12:37:18,468] jepsen worker 78 - jepsen.util 11598 :fail :cas [26 [30 6]] :ValueNotSwapFrom INFO [2019-12-17 12:37:18,468] jepsen worker 78 - jepsen.util 11598 :invoke :cas [26 [39 17]] INFO [2019-12-17 12:37:18,490] jepsen worker 46 - jepsen.util 11476 :ok :read [15 4] INFO [2019-12-17 12:37:18,490] jepsen worker 46 - jepsen.util 11476 :invoke :write [15 37] INFO [2019-12-17 12:37:18,492] jepsen worker 32 - jepsen.util 12092 :ok :read [10 45] INFO [2019-12-17 12:37:18,492] jepsen worker 32 - jepsen.util 12092 :invoke :write [10 11] INFO [2019-12-17 12:37:18,493] jepsen worker 5 - jepsen.util 11525 :fail :cas [1 [42 20]] :ValueNotSwapFrom INFO [2019-12-17 12:37:18,493] jepsen worker 5 - jepsen.util 11525 :invoke :write [1 22] INFO [2019-12-17 12:37:18,493] jepsen worker 30 - jepsen.util 10830 :ok :read [10 45] INFO [2019-12-17 12:37:18,494] jepsen worker 30 - jepsen.util 10830 :invoke :read [10 nil] INFO [2019-12-17 12:37:18,504] jepsen worker 4 - jepsen.util 10894 :ok :write [1 30] INFO [2019-12-17 12:37:18,505] jepsen worker 4 - jepsen.util 10894 :invoke :cas [1 [2 25]] INFO [2019-12-17 12:37:18,519] jepsen worker 55 - jepsen.util 12385 :ok :read [18 13] INFO [2019-12-17 12:37:18,519] jepsen worker 55 - jepsen.util 12385 :invoke :cas [18 [29 15]] INFO [2019-12-17 12:37:18,543] jepsen worker 56 - jepsen.util 12656 :ok :read [18 13] INFO [2019-12-17 12:37:18,543] jepsen worker 56 - jepsen.util 12656 :invoke :read [18 nil] INFO [2019-12-17 12:37:18,545] jepsen worker 63 - jepsen.util 10233 :ok :read [21 26] INFO [2019-12-17 12:37:18,545] jepsen worker 63 - jepsen.util 10233 :invoke :read [21 nil] INFO [2019-12-17 12:37:18,545] jepsen worker 36 - jepsen.util 12546 :ok :write [12 49] INFO [2019-12-17 12:37:18,545] jepsen worker 36 - jepsen.util 12546 :invoke :read [12 nil] INFO [2019-12-17 12:37:18,549] jepsen worker 10 - jepsen.util 11440 :fail :cas [3 [3 17]] :ValueNotSwapFrom INFO [2019-12-17 12:37:18,549] jepsen worker 10 - jepsen.util 11440 :invoke :write [3 12] INFO [2019-12-17 12:37:18,575] jepsen worker 31 - jepsen.util 10021 :ok :read [10 45] INFO [2019-12-17 12:37:18,575] jepsen worker 31 - jepsen.util 10021 :invoke :write [10 14] INFO [2019-12-17 12:37:18,588] jepsen worker 27 - jepsen.util 10917 :ok :write [9 29] INFO [2019-12-17 12:37:18,588] jepsen worker 27 - jepsen.util 10917 :invoke :cas [9 [37 20]] INFO [2019-12-17 12:37:18,592] jepsen worker 37 - jepsen.util 11377 :ok :read [12 49] INFO [2019-12-17 12:37:18,592] jepsen worker 37 - jepsen.util 11377 :invoke :cas [12 [30 12]] INFO [2019-12-17 12:37:18,605] jepsen worker 56 - jepsen.util 12656 :ok :read [18 13] INFO [2019-12-17 12:37:18,606] jepsen worker 56 - jepsen.util 12656 :invoke :write [18 47] INFO [2019-12-17 12:37:18,633] jepsen worker 79 - jepsen.util 11959 :ok :write [26 34] INFO [2019-12-17 12:37:18,633] jepsen worker 79 - jepsen.util 11959 :invoke :cas [26 [10 19]] INFO [2019-12-17 12:37:18,636] jepsen worker 53 - jepsen.util 11303 :ok :write [17 15] INFO [2019-12-17 12:37:18,636] jepsen worker 53 - jepsen.util 11303 :invoke :cas [17 [28 37]] INFO [2019-12-17 12:37:18,637] jepsen worker 27 - jepsen.util 10917 :fail :cas [9 [37 20]] :ValueNotSwapFrom INFO [2019-12-17 12:37:18,637] jepsen worker 27 - jepsen.util 10917 :invoke :read [9 nil] INFO [2019-12-17 12:37:18,662] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":67970,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:37:18,662] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":67970,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:37:18,662] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":67973,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:37:18,664] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":67973,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:37:18,664] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":67974,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:37:18,664] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":67974,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:37:18,665] jepsen worker 4 - jepsen.util 10894 :fail :cas [1 [2 25]] :ValueNotSwapFrom INFO [2019-12-17 12:37:18,665] jepsen worker 4 - jepsen.util 10894 :invoke :read [1 nil] INFO [2019-12-17 12:37:18,675] jepsen worker 27 - jepsen.util 10917 :ok :read [9 29] INFO [2019-12-17 12:37:18,675] jepsen worker 27 - jepsen.util 10917 :invoke :write [9 20] INFO [2019-12-17 12:37:18,681] jepsen worker 79 - jepsen.util 11959 :fail :cas [26 [10 19]] :ValueNotSwapFrom INFO [2019-12-17 12:37:18,682] jepsen worker 79 - jepsen.util 11959 :invoke :write [26 35] INFO [2019-12-17 12:37:18,722] jepsen worker 4 - jepsen.util 10894 :ok :read [1 30] INFO [2019-12-17 12:37:18,722] jepsen worker 4 - jepsen.util 10894 :invoke :read [1 nil] INFO [2019-12-17 12:37:18,792] jepsen worker 69 - jepsen.util 12039 :ok :write [23 3] INFO [2019-12-17 12:37:18,792] jepsen worker 69 - jepsen.util 12039 :invoke :cas [23 [7 15]] INFO [2019-12-17 12:37:18,808] jepsen worker 27 - jepsen.util 10917 :ok :write [9 20] INFO [2019-12-17 12:37:18,808] jepsen worker 27 - jepsen.util 10917 :invoke :read [9 nil] INFO [2019-12-17 12:37:18,827] jepsen worker 88 - jepsen.util 11878 :fail :cas [29 [16 3]] :ValueNotSwapFrom INFO [2019-12-17 12:37:18,827] jepsen worker 88 - jepsen.util 11878 :invoke :write [29 0] INFO [2019-12-17 12:37:18,829] jepsen worker 27 - jepsen.util 10917 :ok :read [9 20] INFO [2019-12-17 12:37:18,830] jepsen worker 27 - jepsen.util 10917 :invoke :write [9 0] INFO [2019-12-17 12:37:18,831] jepsen worker 51 - jepsen.util 11031 :ok :write [17 3] INFO [2019-12-17 12:37:18,831] jepsen worker 51 - jepsen.util 11031 :invoke :read [17 nil] INFO [2019-12-17 12:37:18,833] jepsen worker 33 - jepsen.util 15783 :info :write [11 2] :RequestCanceledException INFO [2019-12-17 12:37:18,834] jepsen worker 33 - jepsen.util 15873 :invoke :write [11 18] INFO [2019-12-17 12:37:18,845] jepsen worker 56 - jepsen.util 12656 :ok :write [18 47] INFO [2019-12-17 12:37:18,845] jepsen worker 56 - jepsen.util 12656 :invoke :write [18 21] INFO [2019-12-17 12:37:18,895] jepsen worker 27 - jepsen.util 10917 :ok :write [9 0] INFO [2019-12-17 12:37:18,895] jepsen worker 27 - jepsen.util 10917 :invoke :read [9 nil] INFO [2019-12-17 12:37:18,899] jepsen worker 45 - jepsen.util 11835 :info :write [15 15] :RequestCanceledException INFO [2019-12-17 12:37:18,900] jepsen worker 45 - jepsen.util 11925 :invoke :write [15 38] INFO [2019-12-17 12:37:18,906] jepsen worker 20 - jepsen.util 11360 :ok :write [6 0] INFO [2019-12-17 12:37:18,906] jepsen worker 20 - jepsen.util 11360 :invoke :cas [6 [46 7]] INFO [2019-12-17 12:37:18,913] jepsen worker 27 - jepsen.util 10917 :ok :read [9 0] INFO [2019-12-17 12:37:18,913] jepsen worker 27 - jepsen.util 10917 :invoke :read [9 nil] INFO [2019-12-17 12:37:18,930] jepsen worker 72 - jepsen.util 11952 :ok :write [24 47] INFO [2019-12-17 12:37:18,930] jepsen worker 72 - jepsen.util 11952 :invoke :read [24 nil] INFO [2019-12-17 12:37:18,947] jepsen worker 87 - jepsen.util 11247 :ok :read [29 3] INFO [2019-12-17 12:37:18,947] jepsen worker 87 - jepsen.util 11247 :invoke :read [29 nil] INFO [2019-12-17 12:37:18,951] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":67988,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:37:18,951] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":67988,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:37:18,955] jepsen worker 36 - jepsen.util 12546 :ok :read [12 49] INFO [2019-12-17 12:37:18,955] jepsen worker 36 - jepsen.util 12546 :invoke :write [12 9] INFO [2019-12-17 12:37:18,976] jepsen worker 54 - jepsen.util 12204 :ok :read [18 47] INFO [2019-12-17 12:37:18,976] jepsen worker 54 - jepsen.util 12204 :invoke :read [18 nil] INFO [2019-12-17 12:37:18,984] jepsen worker 63 - jepsen.util 10233 :ok :read [21 26] INFO [2019-12-17 12:37:18,984] jepsen worker 63 - jepsen.util 10233 :invoke :write [21 43] INFO [2019-12-17 12:37:18,984] jepsen worker 20 - jepsen.util 11360 :fail :cas [6 [46 7]] :ValueNotSwapFrom INFO [2019-12-17 12:37:18,984] jepsen worker 20 - jepsen.util 11360 :invoke :cas [6 [13 12]] INFO [2019-12-17 12:37:19,012] jepsen worker 14 - jepsen.util 11444 :ok :write [4 0] INFO [2019-12-17 12:37:19,012] jepsen worker 14 - jepsen.util 11444 :invoke :cas [4 [9 24]] INFO [2019-12-17 12:37:19,020] jepsen worker 27 - jepsen.util 10917 :ok :read [9 0] INFO [2019-12-17 12:37:19,021] jepsen worker 27 - jepsen.util 10917 :invoke :cas [9 [4 18]] INFO [2019-12-17 12:37:19,021] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68002,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:37:19,022] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68002,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:37:19,022] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68003,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:37:19,022] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68003,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:37:19,022] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68007,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:37:19,022] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68007,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:37:19,032] jepsen worker 79 - jepsen.util 11959 :ok :write [26 35] INFO [2019-12-17 12:37:19,032] jepsen worker 79 - jepsen.util 11959 :invoke :write [26 29] INFO [2019-12-17 12:37:19,034] jepsen worker 64 - jepsen.util 8884 :info :cas [21 [8 47]] :RequestCanceledException INFO [2019-12-17 12:37:19,035] jepsen worker 64 - jepsen.util 8974 :invoke :read [21 nil] INFO [2019-12-17 12:37:19,040] jepsen worker 3 - jepsen.util 11163 :ok :write [1 25] INFO [2019-12-17 12:37:19,040] jepsen worker 3 - jepsen.util 11163 :invoke :cas [1 [10 34]] INFO [2019-12-17 12:37:19,047] jepsen worker 28 - jepsen.util 9838 :info :write [9 1] :RequestCanceledException INFO [2019-12-17 12:37:19,049] jepsen worker 72 - jepsen.util 11952 :ok :read [24 47] INFO [2019-12-17 12:37:19,049] jepsen worker 72 - jepsen.util 11952 :invoke :read [24 nil] INFO [2019-12-17 12:37:19,049] jepsen worker 28 - jepsen.util 9928 :invoke :read [9 nil] INFO [2019-12-17 12:37:19,074] jepsen worker 4 - jepsen.util 10894 :ok :read [1 25] INFO [2019-12-17 12:37:19,075] jepsen worker 4 - jepsen.util 10894 :invoke :read [1 nil] INFO [2019-12-17 12:37:19,081] jepsen worker 27 - jepsen.util 10917 :fail :cas [9 [4 18]] :ValueNotSwapFrom INFO [2019-12-17 12:37:19,081] jepsen worker 27 - jepsen.util 10917 :invoke :write [9 25] INFO [2019-12-17 12:37:19,089] jepsen worker 63 - jepsen.util 10233 :ok :write [21 43] INFO [2019-12-17 12:37:19,089] jepsen worker 63 - jepsen.util 10233 :invoke :read [21 nil] INFO [2019-12-17 12:37:19,106] jepsen worker 72 - jepsen.util 11952 :ok :read [24 47] INFO [2019-12-17 12:37:19,106] jepsen worker 72 - jepsen.util 11952 :invoke :cas [24 [48 33]] INFO [2019-12-17 12:37:19,118] jepsen worker 11 - jepsen.util 11261 :fail :read [3 nil] :RequestTimeoutException INFO [2019-12-17 12:37:19,118] jepsen worker 11 - jepsen.util 11261 :invoke :cas [3 [39 49]] INFO [2019-12-17 12:37:19,125] jepsen worker 70 - jepsen.util 12220 :fail :cas [23 [49 10]] :ValueNotSwapFrom INFO [2019-12-17 12:37:19,125] jepsen worker 70 - jepsen.util 12220 :invoke :write [23 6] INFO [2019-12-17 12:37:19,130] jepsen worker 80 - jepsen.util 11420 :fail :cas [26 [45 49]] :ValueNotSwapFrom INFO [2019-12-17 12:37:19,131] jepsen worker 80 - jepsen.util 11420 :invoke :read [26 nil] INFO [2019-12-17 12:37:19,133] jepsen worker 63 - jepsen.util 10233 :ok :read [21 43] INFO [2019-12-17 12:37:19,133] jepsen worker 63 - jepsen.util 10233 :invoke :write [21 25] INFO [2019-12-17 12:37:19,148] jepsen worker 13 - jepsen.util 11083 :fail :cas [4 [20 8]] :ValueNotSwapFrom INFO [2019-12-17 12:37:19,148] jepsen worker 13 - jepsen.util 11083 :invoke :read [4 nil] INFO [2019-12-17 12:37:19,161] jepsen worker 19 - jepsen.util 12079 :fail :read [6 nil] :RequestTimeoutException INFO [2019-12-17 12:37:19,161] jepsen worker 9 - jepsen.util 12339 :info :cas [3 [3 49]] :RequestCanceledException INFO [2019-12-17 12:37:19,161] jepsen worker 19 - jepsen.util 12079 :invoke :cas [6 [9 24]] INFO [2019-12-17 12:37:19,223] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68026,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:37:19,224] jepsen worker 66 - jepsen.util 15906 :info :write [22 27] :RequestCanceledException INFO [2019-12-17 12:37:19,224] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68026,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:37:19,224] jepsen worker 82 - jepsen.util 16372 :info :write [27 15] :RequestCanceledException INFO [2019-12-17 12:37:19,224] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68030,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:37:19,224] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68030,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:37:19,224] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68034,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:37:19,224] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68034,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:37:19,224] jepsen worker 5 - jepsen.util 11525 :ok :write [1 22] INFO [2019-12-17 12:37:19,224] jepsen worker 5 - jepsen.util 11525 :invoke :read [1 nil] INFO [2019-12-17 12:37:19,225] jepsen worker 9 - jepsen.util 12429 :invoke :read [3 nil] INFO [2019-12-17 12:37:19,225] jepsen worker 66 - jepsen.util 15996 :invoke :write [22 38] INFO [2019-12-17 12:37:19,225] jepsen worker 82 - jepsen.util 16462 :invoke :write [27 0] INFO [2019-12-17 12:37:19,230] jepsen worker 72 - jepsen.util 11952 :fail :cas [24 [48 33]] :ValueNotSwapFrom INFO [2019-12-17 12:37:19,230] jepsen worker 72 - jepsen.util 11952 :invoke :cas [24 [47 36]] INFO [2019-12-17 12:37:19,231] jepsen worker 38 - jepsen.util 12188 :ok :read [12 49] INFO [2019-12-17 12:37:19,231] jepsen worker 38 - jepsen.util 12188 :invoke :write [12 4] INFO [2019-12-17 12:37:19,233] jepsen worker 78 - jepsen.util 11598 :fail :cas [26 [39 17]] :ValueNotSwapFrom INFO [2019-12-17 12:37:19,234] jepsen worker 78 - jepsen.util 11598 :invoke :read [26 nil] INFO [2019-12-17 12:37:19,243] jepsen worker 11 - jepsen.util 11261 :fail :cas [3 [39 49]] :ValueNotSwapFrom INFO [2019-12-17 12:37:19,243] jepsen worker 11 - jepsen.util 11261 :invoke :cas [3 [14 43]] INFO [2019-12-17 12:37:19,247] jepsen worker 62 - jepsen.util 16802 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:37:19,248] jepsen worker 62 - jepsen.util 16802 :invoke :write [20 26] INFO [2019-12-17 12:37:19,257] jepsen worker 19 - jepsen.util 12079 :fail :cas [6 [9 24]] :ValueNotSwapFrom INFO [2019-12-17 12:37:19,257] jepsen worker 19 - jepsen.util 12079 :invoke :write [6 49] INFO [2019-12-17 12:37:19,264] jepsen worker 17 - jepsen.util 16037 :info :cas [5 [30 27]] :RequestCanceledException INFO [2019-12-17 12:37:19,265] jepsen worker 17 - jepsen.util 16127 :invoke :cas [5 [14 5]] INFO [2019-12-17 12:37:19,280] jepsen worker 9 - jepsen.util 12429 :ok :read [3 5] INFO [2019-12-17 12:37:19,280] jepsen worker 9 - jepsen.util 12429 :invoke :write [3 32] INFO [2019-12-17 12:37:19,300] jepsen worker 84 - jepsen.util 15744 :info :cas [28 [18 25]] :RequestCanceledException INFO [2019-12-17 12:37:19,301] jepsen worker 84 - jepsen.util 15834 :invoke :write [28 48] INFO [2019-12-17 12:37:19,307] jepsen worker 69 - jepsen.util 12039 :fail :cas [23 [7 15]] :ValueNotSwapFrom INFO [2019-12-17 12:37:19,307] jepsen worker 69 - jepsen.util 12039 :invoke :cas [23 [39 15]] INFO [2019-12-17 12:37:19,308] jepsen worker 14 - jepsen.util 11444 :fail :cas [4 [9 24]] :ValueNotSwapFrom INFO [2019-12-17 12:37:19,308] jepsen worker 14 - jepsen.util 11444 :invoke :read [4 nil] INFO [2019-12-17 12:37:19,311] jepsen worker 89 - jepsen.util 11789 :ok :write [29 34] INFO [2019-12-17 12:37:19,311] jepsen worker 89 - jepsen.util 11789 :invoke :write [29 18] INFO [2019-12-17 12:37:19,316] jepsen worker 32 - jepsen.util 12092 :ok :write [10 11] INFO [2019-12-17 12:37:19,316] jepsen worker 32 - jepsen.util 12092 :invoke :write [10 16] INFO [2019-12-17 12:37:19,333] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68035,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:37:19,333] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68035,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:37:19,333] jepsen worker 77 - jepsen.util 16907 :info :write [25 26] :RequestCanceledException INFO [2019-12-17 12:37:19,334] jepsen worker 26 - jepsen.util 16766 :info :cas [8 [32 36]] :RequestCanceledException INFO [2019-12-17 12:37:19,334] jepsen worker 77 - jepsen.util 16997 :invoke :cas [25 [5 35]] INFO [2019-12-17 12:37:19,334] jepsen worker 26 - jepsen.util 16856 :invoke :cas [8 [27 42]] INFO [2019-12-17 12:37:19,341] jepsen worker 63 - jepsen.util 10233 :ok :write [21 25] INFO [2019-12-17 12:37:19,341] jepsen worker 63 - jepsen.util 10233 :invoke :cas [21 [36 26]] INFO [2019-12-17 12:37:19,346] jepsen worker 28 - jepsen.util 9928 :ok :read [9 25] INFO [2019-12-17 12:37:19,346] jepsen worker 28 - jepsen.util 9928 :invoke :cas [9 [10 26]] INFO [2019-12-17 12:37:19,346] jepsen worker 27 - jepsen.util 10917 :ok :write [9 25] INFO [2019-12-17 12:37:19,346] jepsen worker 27 - jepsen.util 10917 :invoke :cas [9 [19 11]] INFO [2019-12-17 12:37:19,347] jepsen worker 37 - jepsen.util 11377 :fail :cas [12 [30 12]] :ValueNotSwapFrom INFO [2019-12-17 12:37:19,347] jepsen worker 37 - jepsen.util 11377 :invoke :write [12 30] INFO [2019-12-17 12:37:19,347] jepsen worker 48 - jepsen.util 14898 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:37:19,347] jepsen worker 48 - jepsen.util 14898 :invoke :write [16 15] INFO [2019-12-17 12:37:19,364] jepsen worker 23 - jepsen.util 16223 :info :cas [7 [15 3]] :RequestCanceledException INFO [2019-12-17 12:37:19,365] jepsen worker 23 - jepsen.util 16313 :invoke :write [7 1] INFO [2019-12-17 12:37:19,389] jepsen worker 27 - jepsen.util 10917 :fail :cas [9 [19 11]] :ValueNotSwapFrom INFO [2019-12-17 12:37:19,389] jepsen worker 27 - jepsen.util 10917 :invoke :read [9 nil] INFO [2019-12-17 12:37:19,400] jepsen worker 25 - jepsen.util 17035 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:37:19,400] jepsen worker 86 - jepsen.util 16826 :info :cas [28 [33 46]] :RequestCanceledException INFO [2019-12-17 12:37:19,400] jepsen worker 25 - jepsen.util 17035 :invoke :cas [8 [39 9]] INFO [2019-12-17 12:37:19,401] jepsen worker 86 - jepsen.util 16916 :invoke :cas [28 [36 37]] INFO [2019-12-17 12:37:19,402] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:37:19,402] jepsen nemesis - couchbase.util Rebalance check count: 535 INFO [2019-12-17 12:37:19,402] jepsen nemesis - couchbase.util Rebalance stuck count: 516 INFO [2019-12-17 12:37:19,416] jepsen worker 11 - jepsen.util 11261 :fail :cas [3 [14 43]] :ValueNotSwapFrom INFO [2019-12-17 12:37:19,416] jepsen worker 11 - jepsen.util 11261 :invoke :write [3 17] INFO [2019-12-17 12:37:19,417] jepsen worker 51 - jepsen.util 11031 :ok :read [17 3] INFO [2019-12-17 12:37:19,417] jepsen worker 51 - jepsen.util 11031 :invoke :write [17 28] INFO [2019-12-17 12:37:19,418] jepsen worker 43 - jepsen.util 16243 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:37:19,418] jepsen worker 43 - jepsen.util 16243 :invoke :write [14 8] INFO [2019-12-17 12:37:19,434] jepsen worker 58 - jepsen.util 16348 :info :write [19 18] :RequestCanceledException INFO [2019-12-17 12:37:19,434] jepsen worker 65 - jepsen.util 10415 :fail :read [21 nil] :RequestTimeoutException INFO [2019-12-17 12:37:19,434] jepsen worker 2 - jepsen.util 15572 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:37:19,434] jepsen worker 65 - jepsen.util 10415 :invoke :read [21 nil] INFO [2019-12-17 12:37:19,434] jepsen worker 2 - jepsen.util 15572 :invoke :write [0 49] INFO [2019-12-17 12:37:19,434] jepsen worker 58 - jepsen.util 16438 :invoke :cas [19 [1 40]] INFO [2019-12-17 12:37:19,436] jepsen worker 55 - jepsen.util 12385 :fail :cas [18 [29 15]] :ValueNotSwapFrom INFO [2019-12-17 12:37:19,437] jepsen worker 55 - jepsen.util 12385 :invoke :cas [18 [3 37]] INFO [2019-12-17 12:37:19,442] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68071,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:37:19,442] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68071,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:37:19,442] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68075,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:37:19,442] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68075,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:37:19,450] jepsen worker 63 - jepsen.util 10233 :fail :cas [21 [36 26]] :ValueNotSwapFrom INFO [2019-12-17 12:37:19,451] jepsen worker 63 - jepsen.util 10233 :invoke :read [21 nil] INFO [2019-12-17 12:37:19,451] jepsen worker 27 - jepsen.util 10917 :ok :read [9 25] INFO [2019-12-17 12:37:19,452] jepsen worker 27 - jepsen.util 10917 :invoke :read [9 nil] INFO [2019-12-17 12:37:19,464] jepsen worker 67 - jepsen.util 15997 :info :write [22 8] :RequestCanceledException INFO [2019-12-17 12:37:19,464] jepsen worker 52 - jepsen.util 11122 :ok :write [17 9] INFO [2019-12-17 12:37:19,464] jepsen worker 52 - jepsen.util 11122 :invoke :read [17 nil] INFO [2019-12-17 12:37:19,465] jepsen worker 67 - jepsen.util 16087 :invoke :cas [22 [49 28]] INFO [2019-12-17 12:37:19,470] jepsen worker 79 - jepsen.util 11959 :ok :write [26 29] INFO [2019-12-17 12:37:19,470] jepsen worker 79 - jepsen.util 11959 :invoke :write [26 8] INFO [2019-12-17 12:37:19,473] jepsen worker 27 - jepsen.util 10917 :ok :read [9 25] INFO [2019-12-17 12:37:19,473] jepsen worker 27 - jepsen.util 10917 :invoke :write [9 18] INFO [2019-12-17 12:37:19,498] jepsen worker 53 - jepsen.util 11303 :fail :cas [17 [28 37]] :ValueNotSwapFrom INFO [2019-12-17 12:37:19,498] jepsen worker 53 - jepsen.util 11303 :invoke :write [17 27] INFO [2019-12-17 12:37:19,500] jepsen worker 85 - jepsen.util 16735 :info :cas [28 [39 14]] :RequestCanceledException INFO [2019-12-17 12:37:19,500] jepsen worker 85 - jepsen.util 16825 :invoke :read [28 nil] INFO [2019-12-17 12:37:19,518] jepsen worker 21 - jepsen.util 15501 :info :write [7 8] :RequestCanceledException INFO [2019-12-17 12:37:19,518] jepsen worker 7 - jepsen.util 16027 :info :write [2 48] :RequestCanceledException INFO [2019-12-17 12:37:19,518] jepsen worker 59 - jepsen.util 15269 :info :write [19 40] :RequestCanceledException INFO [2019-12-17 12:37:19,519] jepsen worker 21 - jepsen.util 15591 :invoke :cas [7 [29 3]] INFO [2019-12-17 12:37:19,519] jepsen worker 7 - jepsen.util 16117 :invoke :read [2 nil] INFO [2019-12-17 12:37:19,519] jepsen worker 59 - jepsen.util 15359 :invoke :write [19 5] INFO [2019-12-17 12:37:19,533] jepsen worker 8 - jepsen.util 15758 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:37:19,534] jepsen worker 8 - jepsen.util 15758 :invoke :read [2 nil] INFO [2019-12-17 12:37:19,535] jepsen worker 4 - jepsen.util 10894 :ok :read [1 22] INFO [2019-12-17 12:37:19,535] jepsen worker 4 - jepsen.util 10894 :invoke :read [1 nil] INFO [2019-12-17 12:37:19,564] jepsen worker 50 - jepsen.util 16250 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:37:19,564] jepsen worker 50 - jepsen.util 16250 :invoke :cas [16 [5 26]] INFO [2019-12-17 12:37:19,569] jepsen worker 27 - jepsen.util 10917 :ok :write [9 18] INFO [2019-12-17 12:37:19,569] jepsen worker 27 - jepsen.util 10917 :invoke :cas [9 [46 35]] INFO [2019-12-17 12:37:19,578] jepsen worker 36 - jepsen.util 12546 :ok :write [12 9] INFO [2019-12-17 12:37:19,578] jepsen worker 36 - jepsen.util 12546 :invoke :read [12 nil] INFO [2019-12-17 12:37:19,584] jepsen worker 27 - jepsen.util 10917 :fail :cas [9 [46 35]] :ValueNotSwapFrom INFO [2019-12-17 12:37:19,584] jepsen worker 27 - jepsen.util 10917 :invoke :write [9 6] INFO [2019-12-17 12:37:19,588] jepsen worker 9 - jepsen.util 12429 :ok :write [3 32] INFO [2019-12-17 12:37:19,588] jepsen worker 45 - jepsen.util 11925 :ok :write [15 38] INFO [2019-12-17 12:37:19,588] jepsen worker 9 - jepsen.util 12429 :invoke :write [3 43] INFO [2019-12-17 12:37:19,588] jepsen worker 45 - jepsen.util 11925 :invoke :cas [15 [38 45]] INFO [2019-12-17 12:37:19,589] jepsen worker 19 - jepsen.util 12079 :ok :write [6 49] INFO [2019-12-17 12:37:19,589] jepsen worker 19 - jepsen.util 12079 :invoke :write [6 30] INFO [2019-12-17 12:37:19,599] jepsen worker 0 - jepsen.util 16650 :info :cas [0 [18 2]] :RequestCanceledException INFO [2019-12-17 12:37:19,600] jepsen worker 70 - jepsen.util 12220 :ok :write [23 6] INFO [2019-12-17 12:37:19,600] jepsen worker 70 - jepsen.util 12220 :invoke :write [23 33] INFO [2019-12-17 12:37:19,600] jepsen worker 0 - jepsen.util 16740 :invoke :read [0 nil] INFO [2019-12-17 12:37:19,617] jepsen worker 15 - jepsen.util 14865 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:37:19,617] jepsen worker 15 - jepsen.util 14865 :invoke :cas [5 [20 11]] INFO [2019-12-17 12:37:19,631] jepsen worker 3 - jepsen.util 11163 :fail :cas [1 [10 34]] :ValueNotSwapFrom INFO [2019-12-17 12:37:19,631] jepsen worker 3 - jepsen.util 11163 :invoke :write [1 15] INFO [2019-12-17 12:37:19,633] jepsen worker 61 - jepsen.util 17251 :info :write [20 36] :RequestCanceledException INFO [2019-12-17 12:37:19,634] jepsen worker 20 - jepsen.util 11360 :fail :cas [6 [13 12]] :ValueNotSwapFrom INFO [2019-12-17 12:37:19,634] jepsen worker 61 - jepsen.util 17341 :invoke :cas [20 [5 23]] INFO [2019-12-17 12:37:19,634] jepsen worker 20 - jepsen.util 11360 :invoke :write [6 39] INFO [2019-12-17 12:37:19,658] jepsen worker 13 - jepsen.util 11083 :ok :read [4 0] INFO [2019-12-17 12:37:19,658] jepsen worker 13 - jepsen.util 11083 :invoke :cas [4 [0 14]] INFO [2019-12-17 12:37:19,661] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68105,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:37:19,661] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68105,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:37:19,661] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68106,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:37:19,662] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68106,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:37:19,662] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68110,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:37:19,662] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68110,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:37:19,664] jepsen worker 29 - jepsen.util 10019 :info :cas [9 [20 36]] :RequestCanceledException INFO [2019-12-17 12:37:19,664] jepsen worker 73 - jepsen.util 10783 :info :write [24 5] :RequestCanceledException INFO [2019-12-17 12:37:19,665] jepsen worker 29 - jepsen.util 10109 :invoke :write [9 41] INFO [2019-12-17 12:37:19,667] jepsen worker 73 - jepsen.util 10873 :invoke :write [24 19] INFO [2019-12-17 12:37:19,675] jepsen worker 80 - jepsen.util 11420 :ok :read [26 29] INFO [2019-12-17 12:37:19,675] jepsen worker 80 - jepsen.util 11420 :invoke :write [26 31] INFO [2019-12-17 12:37:19,696] jepsen worker 27 - jepsen.util 10917 :ok :write [9 6] INFO [2019-12-17 12:37:19,696] jepsen worker 27 - jepsen.util 10917 :invoke :write [9 36] INFO [2019-12-17 12:37:19,717] jepsen worker 78 - jepsen.util 11598 :ok :read [26 29] INFO [2019-12-17 12:37:19,718] jepsen worker 78 - jepsen.util 11598 :invoke :write [26 27] INFO [2019-12-17 12:37:19,751] jepsen worker 89 - jepsen.util 11789 :ok :write [29 18] INFO [2019-12-17 12:37:19,751] jepsen worker 89 - jepsen.util 11789 :invoke :read [29 nil] INFO [2019-12-17 12:37:19,756] jepsen worker 32 - jepsen.util 12092 :ok :write [10 16] INFO [2019-12-17 12:37:19,756] jepsen worker 32 - jepsen.util 12092 :invoke :write [10 11] INFO [2019-12-17 12:37:19,757] jepsen worker 14 - jepsen.util 11444 :ok :read [4 0] INFO [2019-12-17 12:37:19,758] jepsen worker 14 - jepsen.util 11444 :invoke :cas [4 [39 16]] INFO [2019-12-17 12:37:19,773] jepsen worker 38 - jepsen.util 12188 :ok :write [12 4] INFO [2019-12-17 12:37:19,773] jepsen worker 38 - jepsen.util 12188 :invoke :cas [12 [30 7]] INFO [2019-12-17 12:37:19,792] jepsen worker 65 - jepsen.util 10415 :ok :read [21 25] INFO [2019-12-17 12:37:19,792] jepsen worker 65 - jepsen.util 10415 :invoke :cas [21 [9 39]] INFO [2019-12-17 12:37:19,809] jepsen worker 79 - jepsen.util 11959 :ok :write [26 8] INFO [2019-12-17 12:37:19,809] jepsen worker 79 - jepsen.util 11959 :invoke :cas [26 [9 39]] INFO [2019-12-17 12:37:19,831] jepsen worker 65 - jepsen.util 10415 :fail :cas [21 [9 39]] :ValueNotSwapFrom INFO [2019-12-17 12:37:19,831] jepsen worker 65 - jepsen.util 10415 :invoke :write [21 1] INFO [2019-12-17 12:37:19,832] jepsen worker 27 - jepsen.util 10917 :ok :write [9 36] INFO [2019-12-17 12:37:19,832] jepsen worker 27 - jepsen.util 10917 :invoke :cas [9 [29 42]] INFO [2019-12-17 12:37:19,833] jepsen worker 18 - jepsen.util 11808 :info :write [6 15] :RequestCanceledException INFO [2019-12-17 12:37:19,834] jepsen worker 18 - jepsen.util 11898 :invoke :cas [6 [2 23]] INFO [2019-12-17 12:37:19,872] jepsen worker 52 - jepsen.util 11122 :ok :read [17 9] INFO [2019-12-17 12:37:19,872] jepsen worker 52 - jepsen.util 11122 :invoke :read [17 nil] INFO [2019-12-17 12:37:19,880] jepsen worker 79 - jepsen.util 11959 :fail :cas [26 [9 39]] :ValueNotSwapFrom INFO [2019-12-17 12:37:19,881] jepsen worker 79 - jepsen.util 11959 :invoke :write [26 33] INFO [2019-12-17 12:37:19,881] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68128,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:37:19,882] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68128,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:37:19,882] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68131,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:37:19,882] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68131,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:37:19,882] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68132,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:37:19,882] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68132,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:37:19,891] jepsen worker 4 - jepsen.util 10894 :ok :read [1 22] INFO [2019-12-17 12:37:19,892] jepsen worker 4 - jepsen.util 10894 :invoke :write [1 32] INFO [2019-12-17 12:37:19,900] jepsen worker 12 - jepsen.util 11982 :ok :write [4 2] INFO [2019-12-17 12:37:19,900] jepsen worker 12 - jepsen.util 11982 :invoke :read [4 nil] INFO [2019-12-17 12:37:19,913] jepsen worker 51 - jepsen.util 11031 :ok :write [17 28] INFO [2019-12-17 12:37:19,913] jepsen worker 51 - jepsen.util 11031 :invoke :write [17 36] INFO [2019-12-17 12:37:19,914] jepsen worker 18 - jepsen.util 11898 :fail :cas [6 [2 23]] :ValueNotSwapFrom INFO [2019-12-17 12:37:19,914] jepsen worker 18 - jepsen.util 11898 :invoke :write [6 40] INFO [2019-12-17 12:37:19,930] jepsen worker 27 - jepsen.util 10917 :fail :cas [9 [29 42]] :ValueNotSwapFrom INFO [2019-12-17 12:37:19,931] jepsen worker 27 - jepsen.util 10917 :invoke :cas [9 [30 43]] INFO [2019-12-17 12:37:19,962] jepsen worker 27 - jepsen.util 10917 :fail :cas [9 [30 43]] :ValueNotSwapFrom INFO [2019-12-17 12:37:19,962] jepsen worker 27 - jepsen.util 10917 :invoke :read [9 nil] INFO [2019-12-17 12:37:19,976] jepsen worker 88 - jepsen.util 11878 :ok :write [29 0] INFO [2019-12-17 12:37:19,976] jepsen worker 88 - jepsen.util 11878 :invoke :cas [29 [34 6]] INFO [2019-12-17 12:37:19,988] jepsen worker 69 - jepsen.util 12039 :fail :cas [23 [39 15]] :ValueNotSwapFrom INFO [2019-12-17 12:37:19,988] jepsen worker 69 - jepsen.util 12039 :invoke :read [23 nil] INFO [2019-12-17 12:37:19,995] jepsen worker 27 - jepsen.util 10917 :ok :read [9 36] INFO [2019-12-17 12:37:19,995] jepsen worker 27 - jepsen.util 10917 :invoke :read [9 nil] INFO [2019-12-17 12:37:19,997] jepsen worker 55 - jepsen.util 12385 :fail :cas [18 [3 37]] :ValueNotSwapFrom INFO [2019-12-17 12:37:19,997] jepsen worker 55 - jepsen.util 12385 :invoke :read [18 nil] INFO [2019-12-17 12:37:20,008] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68148,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:37:20,009] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68148,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:37:20,012] jepsen worker 88 - jepsen.util 11878 :fail :cas [29 [34 6]] :ValueNotSwapFrom INFO [2019-12-17 12:37:20,012] jepsen worker 88 - jepsen.util 11878 :invoke :cas [29 [28 33]] INFO [2019-12-17 12:37:20,033] jepsen worker 16 - jepsen.util 17836 :info :write [5 43] :RequestCanceledException INFO [2019-12-17 12:37:20,034] jepsen worker 69 - jepsen.util 12039 :ok :read [23 6] INFO [2019-12-17 12:37:20,035] jepsen worker 69 - jepsen.util 12039 :invoke :read [23 nil] INFO [2019-12-17 12:37:20,035] jepsen worker 16 - jepsen.util 17926 :invoke :cas [5 [43 40]] INFO [2019-12-17 12:37:20,038] jepsen worker 27 - jepsen.util 10917 :ok :read [9 36] INFO [2019-12-17 12:37:20,039] jepsen worker 27 - jepsen.util 10917 :invoke :cas [9 [3 1]] INFO [2019-12-17 12:37:20,046] jepsen worker 55 - jepsen.util 12385 :ok :read [18 47] INFO [2019-12-17 12:37:20,046] jepsen worker 55 - jepsen.util 12385 :invoke :cas [18 [33 23]] INFO [2019-12-17 12:37:20,055] jepsen worker 27 - jepsen.util 10917 :fail :cas [9 [3 1]] :ValueNotSwapFrom INFO [2019-12-17 12:37:20,055] jepsen worker 27 - jepsen.util 10917 :invoke :write [9 24] INFO [2019-12-17 12:37:20,056] jepsen worker 12 - jepsen.util 11982 :ok :read [4 2] INFO [2019-12-17 12:37:20,056] jepsen worker 12 - jepsen.util 11982 :invoke :read [4 nil] INFO [2019-12-17 12:37:20,062] jepsen worker 47 - jepsen.util 12647 :info :write [15 23] :RequestCanceledException INFO [2019-12-17 12:37:20,063] jepsen worker 47 - jepsen.util 12737 :invoke :cas [15 [30 31]] INFO [2019-12-17 12:37:20,072] jepsen worker 88 - jepsen.util 11878 :fail :cas [29 [28 33]] :ValueNotSwapFrom INFO [2019-12-17 12:37:20,073] jepsen worker 88 - jepsen.util 11878 :invoke :write [29 7] INFO [2019-12-17 12:37:20,081] jepsen worker 69 - jepsen.util 12039 :ok :read [23 6] INFO [2019-12-17 12:37:20,082] jepsen worker 69 - jepsen.util 12039 :invoke :cas [23 [6 29]] INFO [2019-12-17 12:37:20,099] jepsen worker 44 - jepsen.util 16604 :info :write [14 42] :RequestCanceledException INFO [2019-12-17 12:37:20,099] jepsen worker 65 - jepsen.util 10415 :ok :write [21 1] INFO [2019-12-17 12:37:20,099] jepsen worker 65 - jepsen.util 10415 :invoke :write [21 49] INFO [2019-12-17 12:37:20,100] jepsen worker 44 - jepsen.util 16694 :invoke :cas [14 [22 38]] INFO [2019-12-17 12:37:20,107] jepsen worker 55 - jepsen.util 12385 :fail :cas [18 [33 23]] :ValueNotSwapFrom INFO [2019-12-17 12:37:20,107] jepsen worker 55 - jepsen.util 12385 :invoke :write [18 11] INFO [2019-12-17 12:37:20,118] jepsen worker 81 - jepsen.util 15561 :info :write [27 2] :RequestCanceledException INFO [2019-12-17 12:37:20,120] jepsen worker 81 - jepsen.util 15651 :invoke :write [27 33] INFO [2019-12-17 12:37:20,125] jepsen worker 36 - jepsen.util 12546 :ok :read [12 4] INFO [2019-12-17 12:37:20,125] jepsen worker 36 - jepsen.util 12546 :invoke :cas [12 [29 3]] INFO [2019-12-17 12:37:20,155] jepsen worker 27 - jepsen.util 10917 :ok :write [9 24] INFO [2019-12-17 12:37:20,155] jepsen worker 27 - jepsen.util 10917 :invoke :write [9 14] INFO [2019-12-17 12:37:20,160] jepsen worker 89 - jepsen.util 11789 :ok :read [29 0] INFO [2019-12-17 12:37:20,160] jepsen worker 89 - jepsen.util 11789 :invoke :write [29 17] INFO [2019-12-17 12:37:20,168] jepsen worker 12 - jepsen.util 11982 :ok :read [4 2] INFO [2019-12-17 12:37:20,168] jepsen worker 12 - jepsen.util 11982 :invoke :write [4 7] INFO [2019-12-17 12:37:20,174] jepsen worker 65 - jepsen.util 10415 :ok :write [21 49] INFO [2019-12-17 12:37:20,174] jepsen worker 65 - jepsen.util 10415 :invoke :read [21 nil] INFO [2019-12-17 12:37:20,192] jepsen worker 20 - jepsen.util 11360 :ok :write [6 39] INFO [2019-12-17 12:37:20,192] jepsen worker 20 - jepsen.util 11360 :invoke :read [6 nil] INFO [2019-12-17 12:37:20,199] jepsen worker 72 - jepsen.util 11952 :ok :cas [24 [47 36]] INFO [2019-12-17 12:37:20,199] jepsen worker 72 - jepsen.util 11952 :invoke :read [24 nil] INFO [2019-12-17 12:37:20,207] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68172,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:37:20,208] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68172,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:37:20,208] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68174,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:37:20,208] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68174,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:37:20,218] jepsen worker 35 - jepsen.util 15785 :info :write [11 15] :RequestCanceledException INFO [2019-12-17 12:37:20,219] jepsen worker 35 - jepsen.util 15875 :invoke :cas [11 [3 38]] INFO [2019-12-17 12:37:20,229] jepsen worker 65 - jepsen.util 10415 :ok :read [21 49] INFO [2019-12-17 12:37:20,229] jepsen worker 65 - jepsen.util 10415 :invoke :write [21 5] INFO [2019-12-17 12:37:20,234] jepsen worker 72 - jepsen.util 11952 :ok :read [24 36] INFO [2019-12-17 12:37:20,234] jepsen worker 72 - jepsen.util 11952 :invoke :read [24 nil] INFO [2019-12-17 12:37:20,248] jepsen worker 68 - jepsen.util 15368 :info :cas [22 [48 44]] :RequestCanceledException INFO [2019-12-17 12:37:20,248] jepsen worker 68 - jepsen.util 15458 :invoke :read [22 nil] INFO [2019-12-17 12:37:20,249] jepsen worker 51 - jepsen.util 11031 :ok :write [17 36] INFO [2019-12-17 12:37:20,250] jepsen worker 51 - jepsen.util 11031 :invoke :read [17 nil] INFO [2019-12-17 12:37:20,250] jepsen worker 70 - jepsen.util 12220 :ok :write [23 33] INFO [2019-12-17 12:37:20,250] jepsen worker 70 - jepsen.util 12220 :invoke :cas [23 [1 39]] INFO [2019-12-17 12:37:20,251] jepsen worker 56 - jepsen.util 12656 :ok :write [18 21] INFO [2019-12-17 12:37:20,251] jepsen worker 56 - jepsen.util 12656 :invoke :cas [18 [16 1]] INFO [2019-12-17 12:37:20,264] jepsen worker 72 - jepsen.util 11952 :ok :read [24 36] INFO [2019-12-17 12:37:20,264] jepsen worker 72 - jepsen.util 11952 :invoke :cas [24 [39 1]] INFO [2019-12-17 12:37:20,264] jepsen worker 27 - jepsen.util 10917 :ok :write [9 14] INFO [2019-12-17 12:37:20,264] jepsen worker 27 - jepsen.util 10917 :invoke :cas [9 [37 22]] INFO [2019-12-17 12:37:20,307] jepsen worker 14 - jepsen.util 11444 :fail :cas [4 [39 16]] :ValueNotSwapFrom INFO [2019-12-17 12:37:20,307] jepsen worker 14 - jepsen.util 11444 :invoke :cas [4 [48 29]] INFO [2019-12-17 12:37:20,309] jepsen worker 72 - jepsen.util 11952 :fail :cas [24 [39 1]] :ValueNotSwapFrom INFO [2019-12-17 12:37:20,309] jepsen worker 72 - jepsen.util 11952 :invoke :cas [24 [24 0]] INFO [2019-12-17 12:37:20,312] jepsen worker 27 - jepsen.util 10917 :fail :cas [9 [37 22]] :ValueNotSwapFrom INFO [2019-12-17 12:37:20,312] jepsen worker 27 - jepsen.util 10917 :invoke :cas [9 [46 28]] INFO [2019-12-17 12:37:20,317] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68179,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:37:20,317] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68179,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:37:20,317] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68182,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:37:20,317] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68182,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:37:20,318] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68184,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:37:20,318] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68184,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:37:20,330] jepsen worker 72 - jepsen.util 11952 :fail :cas [24 [24 0]] :ValueNotSwapFrom INFO [2019-12-17 12:37:20,331] jepsen worker 72 - jepsen.util 11952 :invoke :write [24 38] INFO [2019-12-17 12:37:20,333] jepsen worker 36 - jepsen.util 12546 :fail :cas [12 [29 3]] :ValueNotSwapFrom INFO [2019-12-17 12:37:20,333] jepsen worker 36 - jepsen.util 12546 :invoke :read [12 nil] INFO [2019-12-17 12:37:20,333] jepsen worker 40 - jepsen.util 16600 :info :cas [13 [35 26]] :RequestCanceledException INFO [2019-12-17 12:37:20,334] jepsen worker 40 - jepsen.util 16690 :invoke :write [13 46] INFO [2019-12-17 12:37:20,342] jepsen worker 27 - jepsen.util 10917 :fail :cas [9 [46 28]] :ValueNotSwapFrom INFO [2019-12-17 12:37:20,342] jepsen worker 27 - jepsen.util 10917 :invoke :cas [9 [43 38]] INFO [2019-12-17 12:37:20,346] jepsen worker 13 - jepsen.util 11083 :fail :cas [4 [0 14]] :ValueNotSwapFrom INFO [2019-12-17 12:37:20,346] jepsen worker 13 - jepsen.util 11083 :invoke :write [4 27] INFO [2019-12-17 12:37:20,365] jepsen worker 41 - jepsen.util 17231 :info :write [13 22] :RequestCanceledException INFO [2019-12-17 12:37:20,365] jepsen worker 65 - jepsen.util 10415 :ok :write [21 5] INFO [2019-12-17 12:37:20,365] jepsen worker 65 - jepsen.util 10415 :invoke :cas [21 [28 4]] INFO [2019-12-17 12:37:20,366] jepsen worker 41 - jepsen.util 17321 :invoke :write [13 10] INFO [2019-12-17 12:37:20,372] jepsen worker 27 - jepsen.util 10917 :fail :cas [9 [43 38]] :ValueNotSwapFrom INFO [2019-12-17 12:37:20,372] jepsen worker 87 - jepsen.util 11247 :ok :read [29 0] INFO [2019-12-17 12:37:20,372] jepsen worker 27 - jepsen.util 10917 :invoke :read [9 nil] INFO [2019-12-17 12:37:20,373] jepsen worker 87 - jepsen.util 11247 :invoke :cas [29 [1 25]] INFO [2019-12-17 12:37:20,377] jepsen worker 9 - jepsen.util 12429 :ok :write [3 43] INFO [2019-12-17 12:37:20,377] jepsen worker 9 - jepsen.util 12429 :invoke :write [3 48] INFO [2019-12-17 12:37:20,391] jepsen worker 65 - jepsen.util 10415 :fail :cas [21 [28 4]] :ValueNotSwapFrom INFO [2019-12-17 12:37:20,391] jepsen worker 65 - jepsen.util 10415 :invoke :write [21 6] INFO [2019-12-17 12:37:20,391] jepsen worker 12 - jepsen.util 11982 :ok :write [4 7] INFO [2019-12-17 12:37:20,392] jepsen worker 12 - jepsen.util 11982 :invoke :write [4 11] INFO [2019-12-17 12:37:20,400] jepsen worker 75 - jepsen.util 15645 :info :cas [25 [21 35]] :RequestCanceledException INFO [2019-12-17 12:37:20,400] jepsen worker 75 - jepsen.util 15735 :invoke :read [25 nil] INFO [2019-12-17 12:37:20,418] jepsen worker 6 - jepsen.util 16476 :info :write [2 25] :RequestCanceledException INFO [2019-12-17 12:37:20,418] jepsen worker 22 - jepsen.util 16042 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:37:20,418] jepsen worker 22 - jepsen.util 16042 :invoke :cas [7 [49 25]] INFO [2019-12-17 12:37:20,419] jepsen worker 6 - jepsen.util 16566 :invoke :write [2 9] INFO [2019-12-17 12:37:20,420] jepsen worker 70 - jepsen.util 12220 :fail :cas [23 [1 39]] :ValueNotSwapFrom INFO [2019-12-17 12:37:20,421] jepsen worker 70 - jepsen.util 12220 :invoke :cas [23 [19 25]] INFO [2019-12-17 12:37:20,429] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:37:20,429] jepsen nemesis - couchbase.util Rebalance check count: 536 INFO [2019-12-17 12:37:20,429] jepsen nemesis - couchbase.util Rebalance stuck count: 517 INFO [2019-12-17 12:37:20,433] jepsen worker 42 - jepsen.util 16962 :info :write [14 30] :RequestCanceledException INFO [2019-12-17 12:37:20,434] jepsen worker 42 - jepsen.util 17052 :invoke :cas [14 [19 9]] INFO [2019-12-17 12:37:20,434] jepsen worker 24 - jepsen.util 14874 :info :cas [8 [34 36]] :RequestCanceledException INFO [2019-12-17 12:37:20,434] jepsen worker 71 - jepsen.util 12491 :info :write [23 13] :RequestCanceledException INFO [2019-12-17 12:37:20,435] jepsen worker 71 - jepsen.util 12581 :invoke :cas [23 [48 10]] INFO [2019-12-17 12:37:20,435] jepsen worker 24 - jepsen.util 14964 :invoke :cas [8 [22 1]] INFO [2019-12-17 12:37:20,438] jepsen worker 5 - jepsen.util 11525 :ok :read [1 32] INFO [2019-12-17 12:37:20,438] jepsen worker 5 - jepsen.util 11525 :invoke :read [1 nil] INFO [2019-12-17 12:37:20,447] jepsen worker 34 - jepsen.util 16504 :info :write [11 24] :RequestCanceledException INFO [2019-12-17 12:37:20,447] jepsen worker 83 - jepsen.util 16283 :info :write [27 43] :RequestCanceledException INFO [2019-12-17 12:37:20,447] jepsen worker 45 - jepsen.util 11925 :ok :cas [15 [38 45]] INFO [2019-12-17 12:37:20,447] jepsen worker 45 - jepsen.util 11925 :invoke :cas [15 [0 18]] INFO [2019-12-17 12:37:20,448] jepsen worker 83 - jepsen.util 16373 :invoke :cas [27 [2 25]] INFO [2019-12-17 12:37:20,448] jepsen worker 34 - jepsen.util 16594 :invoke :write [11 35] INFO [2019-12-17 12:37:20,451] jepsen worker 27 - jepsen.util 10917 :ok :read [9 14] INFO [2019-12-17 12:37:20,451] jepsen worker 27 - jepsen.util 10917 :invoke :read [9 nil] INFO [2019-12-17 12:37:20,468] jepsen worker 37 - jepsen.util 11377 :ok :write [12 30] INFO [2019-12-17 12:37:20,468] jepsen worker 4 - jepsen.util 10894 :ok :write [1 32] INFO [2019-12-17 12:37:20,468] jepsen worker 37 - jepsen.util 11377 :invoke :cas [12 [18 37]] INFO [2019-12-17 12:37:20,468] jepsen worker 4 - jepsen.util 10894 :invoke :cas [1 [47 31]] INFO [2019-12-17 12:37:20,499] jepsen worker 39 - jepsen.util 15519 :info :cas [13 [6 34]] :RequestCanceledException INFO [2019-12-17 12:37:20,500] jepsen worker 39 - jepsen.util 15609 :invoke :cas [13 [32 16]] INFO [2019-12-17 12:37:20,503] jepsen worker 14 - jepsen.util 11444 :fail :cas [4 [48 29]] :ValueNotSwapFrom INFO [2019-12-17 12:37:20,503] jepsen worker 14 - jepsen.util 11444 :invoke :read [4 nil] INFO [2019-12-17 12:37:20,517] jepsen worker 69 - jepsen.util 12039 :fail :cas [23 [6 29]] :ValueNotSwapFrom INFO [2019-12-17 12:37:20,517] jepsen worker 69 - jepsen.util 12039 :invoke :write [23 32] INFO [2019-12-17 12:37:20,518] jepsen worker 46 - jepsen.util 11476 :info :write [15 37] :RequestCanceledException INFO [2019-12-17 12:37:20,518] jepsen worker 49 - jepsen.util 16069 :info :cas [16 [24 9]] :RequestCanceledException INFO [2019-12-17 12:37:20,519] jepsen worker 49 - jepsen.util 16159 :invoke :cas [16 [29 43]] INFO [2019-12-17 12:37:20,520] jepsen worker 46 - jepsen.util 11566 :invoke :write [15 48] INFO [2019-12-17 12:37:20,526] jepsen worker 47 - jepsen.util 12737 :fail :cas [15 [30 31]] :ValueNotSwapFrom INFO [2019-12-17 12:37:20,526] jepsen worker 47 - jepsen.util 12737 :invoke :read [15 nil] INFO [2019-12-17 12:37:20,533] jepsen worker 76 - jepsen.util 16186 :info :cas [25 [36 29]] :RequestCanceledException INFO [2019-12-17 12:37:20,533] jepsen worker 1 - jepsen.util 16111 :info :write [0 30] :RequestCanceledException INFO [2019-12-17 12:37:20,535] jepsen worker 76 - jepsen.util 16276 :invoke :read [25 nil] INFO [2019-12-17 12:37:20,535] jepsen worker 1 - jepsen.util 16201 :invoke :cas [0 [17 5]] INFO [2019-12-17 12:37:20,536] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68214,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:37:20,536] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68214,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:37:20,536] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68217,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:37:20,536] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68217,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:37:20,536] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68218,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:37:20,536] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68218,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:37:20,547] jepsen worker 30 - jepsen.util 10830 :fail :read [10 nil] :RequestTimeoutException INFO [2019-12-17 12:37:20,547] jepsen worker 60 - jepsen.util 17250 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:37:20,547] jepsen worker 74 - jepsen.util 11504 :info :write [24 0] :RequestCanceledException INFO [2019-12-17 12:37:20,547] jepsen worker 30 - jepsen.util 10830 :invoke :cas [10 [30 33]] INFO [2019-12-17 12:37:20,547] jepsen worker 60 - jepsen.util 17250 :invoke :write [20 1] INFO [2019-12-17 12:37:20,548] jepsen worker 74 - jepsen.util 11594 :invoke :write [24 28] INFO [2019-12-17 12:37:20,564] jepsen worker 57 - jepsen.util 15897 :info :write [19 2] :RequestCanceledException INFO [2019-12-17 12:37:20,565] jepsen worker 57 - jepsen.util 15987 :invoke :cas [19 [40 10]] INFO [2019-12-17 12:37:20,565] jepsen worker 13 - jepsen.util 11083 :ok :write [4 27] INFO [2019-12-17 12:37:20,565] jepsen worker 13 - jepsen.util 11083 :invoke :read [4 nil] INFO [2019-12-17 12:37:20,576] jepsen worker 20 - jepsen.util 11360 :ok :read [6 39] INFO [2019-12-17 12:37:20,576] jepsen worker 20 - jepsen.util 11360 :invoke :write [6 13] INFO [2019-12-17 12:37:20,590] jepsen worker 70 - jepsen.util 12220 :fail :cas [23 [19 25]] :ValueNotSwapFrom INFO [2019-12-17 12:37:20,590] jepsen worker 70 - jepsen.util 12220 :invoke :cas [23 [32 8]] INFO [2019-12-17 12:37:20,598] jepsen worker 71 - jepsen.util 12581 :fail :cas [23 [48 10]] :ValueNotSwapFrom INFO [2019-12-17 12:37:20,599] jepsen worker 71 - jepsen.util 12581 :invoke :read [23 nil] INFO [2019-12-17 12:37:20,602] jepsen worker 27 - jepsen.util 10917 :ok :read [9 14] INFO [2019-12-17 12:37:20,602] jepsen worker 27 - jepsen.util 10917 :invoke :read [9 nil] INFO [2019-12-17 12:37:20,617] jepsen worker 31 - jepsen.util 10021 :info :write [10 14] :RequestCanceledException INFO [2019-12-17 12:37:20,618] jepsen worker 65 - jepsen.util 10415 :ok :write [21 6] INFO [2019-12-17 12:37:20,618] jepsen worker 65 - jepsen.util 10415 :invoke :cas [21 [35 22]] INFO [2019-12-17 12:37:20,619] jepsen worker 31 - jepsen.util 10111 :invoke :read [10 nil] INFO [2019-12-17 12:37:20,619] jepsen worker 51 - jepsen.util 11031 :ok :read [17 36] INFO [2019-12-17 12:37:20,619] jepsen worker 51 - jepsen.util 11031 :invoke :cas [17 [7 1]] INFO [2019-12-17 12:37:20,633] jepsen worker 10 - jepsen.util 11440 :info :write [3 12] :RequestCanceledException INFO [2019-12-17 12:37:20,634] jepsen worker 10 - jepsen.util 11530 :invoke :read [3 nil] INFO [2019-12-17 12:37:20,640] jepsen worker 45 - jepsen.util 11925 :fail :cas [15 [0 18]] :ValueNotSwapFrom INFO [2019-12-17 12:37:20,641] jepsen worker 45 - jepsen.util 11925 :invoke :read [15 nil] INFO [2019-12-17 12:37:20,654] jepsen worker 27 - jepsen.util 10917 :ok :read [9 14] INFO [2019-12-17 12:37:20,654] jepsen worker 27 - jepsen.util 10917 :invoke :read [9 nil] INFO [2019-12-17 12:37:20,669] jepsen worker 70 - jepsen.util 12220 :fail :cas [23 [32 8]] :ValueNotSwapFrom INFO [2019-12-17 12:37:20,669] jepsen worker 70 - jepsen.util 12220 :invoke :read [23 nil] INFO [2019-12-17 12:37:20,675] jepsen worker 65 - jepsen.util 10415 :fail :cas [21 [35 22]] :ValueNotSwapFrom INFO [2019-12-17 12:37:20,675] jepsen worker 65 - jepsen.util 10415 :invoke :write [21 16] INFO [2019-12-17 12:37:20,689] jepsen worker 27 - jepsen.util 10917 :ok :read [9 14] INFO [2019-12-17 12:37:20,689] jepsen worker 27 - jepsen.util 10917 :invoke :write [9 10] INFO [2019-12-17 12:37:20,720] jepsen worker 47 - jepsen.util 12737 :ok :read [15 45] INFO [2019-12-17 12:37:20,720] jepsen worker 47 - jepsen.util 12737 :invoke :cas [15 [1 42]] INFO [2019-12-17 12:37:20,725] jepsen worker 36 - jepsen.util 12546 :ok :read [12 30] INFO [2019-12-17 12:37:20,725] jepsen worker 36 - jepsen.util 12546 :invoke :cas [12 [44 31]] INFO [2019-12-17 12:37:20,725] jepsen worker 79 - jepsen.util 11959 :ok :write [26 33] INFO [2019-12-17 12:37:20,725] jepsen worker 79 - jepsen.util 11959 :invoke :write [26 49] INFO [2019-12-17 12:37:20,755] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68251,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:37:20,756] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68251,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:37:20,756] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68255,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:37:20,756] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68255,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:37:20,760] jepsen worker 71 - jepsen.util 12581 :ok :read [23 33] INFO [2019-12-17 12:37:20,760] jepsen worker 71 - jepsen.util 12581 :invoke :cas [23 [35 36]] INFO [2019-12-17 12:37:20,821] jepsen worker 65 - jepsen.util 10415 :ok :write [21 16] INFO [2019-12-17 12:37:20,821] jepsen worker 65 - jepsen.util 10415 :invoke :cas [21 [0 12]] INFO [2019-12-17 12:37:20,835] jepsen worker 37 - jepsen.util 11377 :fail :cas [12 [18 37]] :ValueNotSwapFrom INFO [2019-12-17 12:37:20,835] jepsen worker 37 - jepsen.util 11377 :invoke :write [12 49] INFO [2019-12-17 12:37:20,836] jepsen worker 52 - jepsen.util 11122 :ok :read [17 36] INFO [2019-12-17 12:37:20,836] jepsen worker 52 - jepsen.util 11122 :invoke :cas [17 [42 3]] INFO [2019-12-17 12:37:20,869] jepsen worker 65 - jepsen.util 10415 :fail :cas [21 [0 12]] :ValueNotSwapFrom INFO [2019-12-17 12:37:20,869] jepsen worker 27 - jepsen.util 10917 :ok :write [9 10] INFO [2019-12-17 12:37:20,869] jepsen worker 65 - jepsen.util 10415 :invoke :cas [21 [32 49]] INFO [2019-12-17 12:37:20,869] jepsen worker 27 - jepsen.util 10917 :invoke :read [9 nil] INFO [2019-12-17 12:37:20,884] jepsen worker 4 - jepsen.util 10894 :fail :cas [1 [47 31]] :ValueNotSwapFrom INFO [2019-12-17 12:37:20,884] jepsen worker 4 - jepsen.util 10894 :invoke :cas [1 [11 18]] INFO [2019-12-17 12:37:20,914] jepsen worker 65 - jepsen.util 10415 :fail :cas [21 [32 49]] :ValueNotSwapFrom INFO [2019-12-17 12:37:20,915] jepsen worker 65 - jepsen.util 10415 :invoke :read [21 nil] INFO [2019-12-17 12:37:20,916] jepsen worker 27 - jepsen.util 10917 :ok :read [9 10] INFO [2019-12-17 12:37:20,916] jepsen worker 27 - jepsen.util 10917 :invoke :cas [9 [7 36]] INFO [2019-12-17 12:37:20,927] jepsen worker 65 - jepsen.util 10415 :ok :read [21 16] INFO [2019-12-17 12:37:20,927] jepsen worker 65 - jepsen.util 10415 :invoke :cas [21 [6 39]] INFO [2019-12-17 12:37:20,933] jepsen worker 33 - jepsen.util 15873 :info :write [11 18] :RequestCanceledException INFO [2019-12-17 12:37:20,935] jepsen worker 33 - jepsen.util 15963 :invoke :read [11 nil] INFO [2019-12-17 12:37:20,936] jepsen worker 14 - jepsen.util 11444 :ok :read [4 27] INFO [2019-12-17 12:37:20,936] jepsen worker 14 - jepsen.util 11444 :invoke :write [4 47] INFO [2019-12-17 12:37:20,936] jepsen worker 27 - jepsen.util 10917 :fail :cas [9 [7 36]] :ValueNotSwapFrom INFO [2019-12-17 12:37:20,937] jepsen worker 27 - jepsen.util 10917 :invoke :read [9 nil] INFO [2019-12-17 12:37:20,957] jepsen worker 65 - jepsen.util 10415 :fail :cas [21 [6 39]] :ValueNotSwapFrom INFO [2019-12-17 12:37:20,957] jepsen worker 65 - jepsen.util 10415 :invoke :read [21 nil] INFO [2019-12-17 12:37:20,965] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68269,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:37:20,966] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68269,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:37:20,966] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68272,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:37:20,966] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68272,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:37:20,970] jepsen worker 51 - jepsen.util 11031 :fail :cas [17 [7 1]] :ValueNotSwapFrom INFO [2019-12-17 12:37:20,970] jepsen worker 51 - jepsen.util 11031 :invoke :write [17 16] INFO [2019-12-17 12:37:20,970] jepsen worker 27 - jepsen.util 10917 :ok :read [9 10] INFO [2019-12-17 12:37:20,970] jepsen worker 27 - jepsen.util 10917 :invoke :read [9 nil] INFO [2019-12-17 12:37:20,973] jepsen worker 65 - jepsen.util 10415 :ok :read [21 16] INFO [2019-12-17 12:37:20,973] jepsen worker 65 - jepsen.util 10415 :invoke :read [21 nil] INFO [2019-12-17 12:37:20,976] jepsen worker 56 - jepsen.util 12656 :fail :cas [18 [16 1]] :ValueNotSwapFrom INFO [2019-12-17 12:37:20,976] jepsen worker 56 - jepsen.util 12656 :invoke :read [18 nil] INFO [2019-12-17 12:37:21,000] jepsen worker 27 - jepsen.util 10917 :ok :read [9 10] INFO [2019-12-17 12:37:21,000] jepsen worker 27 - jepsen.util 10917 :invoke :cas [9 [32 20]] INFO [2019-12-17 12:37:21,001] jepsen worker 13 - jepsen.util 11083 :ok :read [4 27] INFO [2019-12-17 12:37:21,001] jepsen worker 13 - jepsen.util 11083 :invoke :read [4 nil] INFO [2019-12-17 12:37:21,010] jepsen worker 65 - jepsen.util 10415 :ok :read [21 16] INFO [2019-12-17 12:37:21,010] jepsen worker 65 - jepsen.util 10415 :invoke :read [21 nil] INFO [2019-12-17 12:37:21,020] jepsen worker 27 - jepsen.util 10917 :fail :cas [9 [32 20]] :ValueNotSwapFrom INFO [2019-12-17 12:37:21,021] jepsen worker 27 - jepsen.util 10917 :invoke :read [9 nil] INFO [2019-12-17 12:37:21,022] jepsen worker 3 - jepsen.util 11163 :ok :write [1 15] INFO [2019-12-17 12:37:21,023] jepsen worker 3 - jepsen.util 11163 :invoke :cas [1 [39 45]] INFO [2019-12-17 12:37:21,029] jepsen worker 88 - jepsen.util 11878 :ok :write [29 7] INFO [2019-12-17 12:37:21,029] jepsen worker 88 - jepsen.util 11878 :invoke :cas [29 [40 32]] INFO [2019-12-17 12:37:21,035] jepsen worker 65 - jepsen.util 10415 :ok :read [21 16] INFO [2019-12-17 12:37:21,035] jepsen worker 65 - jepsen.util 10415 :invoke :cas [21 [20 23]] INFO [2019-12-17 12:37:21,048] jepsen worker 54 - jepsen.util 12204 :fail :read [18 nil] :RequestTimeoutException INFO [2019-12-17 12:37:21,048] jepsen worker 54 - jepsen.util 12204 :invoke :write [18 47] INFO [2019-12-17 12:37:21,048] jepsen worker 87 - jepsen.util 11247 :fail :cas [29 [1 25]] :ValueNotSwapFrom INFO [2019-12-17 12:37:21,049] jepsen worker 87 - jepsen.util 11247 :invoke :cas [29 [16 9]] INFO [2019-12-17 12:37:21,074] jepsen worker 27 - jepsen.util 10917 :ok :read [9 10] INFO [2019-12-17 12:37:21,074] jepsen worker 27 - jepsen.util 10917 :invoke :cas [9 [1 43]] INFO [2019-12-17 12:37:21,076] jepsen worker 89 - jepsen.util 11789 :ok :write [29 17] INFO [2019-12-17 12:37:21,076] jepsen worker 89 - jepsen.util 11789 :invoke :write [29 6] INFO [2019-12-17 12:37:21,084] jepsen worker 5 - jepsen.util 11525 :ok :read [1 15] INFO [2019-12-17 12:37:21,084] jepsen worker 5 - jepsen.util 11525 :invoke :read [1 nil] INFO [2019-12-17 12:37:21,089] jepsen worker 65 - jepsen.util 10415 :fail :cas [21 [20 23]] :ValueNotSwapFrom INFO [2019-12-17 12:37:21,089] jepsen worker 65 - jepsen.util 10415 :invoke :cas [21 [7 36]] INFO [2019-12-17 12:37:21,098] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68290,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:37:21,098] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68290,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:37:21,098] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68292,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:37:21,098] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68292,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:37:21,102] jepsen worker 27 - jepsen.util 10917 :fail :cas [9 [1 43]] :ValueNotSwapFrom INFO [2019-12-17 12:37:21,103] jepsen worker 27 - jepsen.util 10917 :invoke :cas [9 [28 2]] INFO [2019-12-17 12:37:21,111] jepsen worker 65 - jepsen.util 10415 :fail :cas [21 [7 36]] :ValueNotSwapFrom INFO [2019-12-17 12:37:21,111] jepsen worker 65 - jepsen.util 10415 :invoke :write [21 46] INFO [2019-12-17 12:37:21,118] jepsen worker 64 - jepsen.util 8974 :fail :read [21 nil] :RequestTimeoutException INFO [2019-12-17 12:37:21,118] jepsen worker 64 - jepsen.util 8974 :invoke :cas [21 [33 9]] INFO [2019-12-17 12:37:21,122] jepsen worker 79 - jepsen.util 11959 :ok :write [26 49] INFO [2019-12-17 12:37:21,122] jepsen worker 79 - jepsen.util 11959 :invoke :read [26 nil] INFO [2019-12-17 12:37:21,122] jepsen worker 27 - jepsen.util 10917 :fail :cas [9 [28 2]] :ValueNotSwapFrom INFO [2019-12-17 12:37:21,122] jepsen worker 27 - jepsen.util 10917 :invoke :cas [9 [7 30]] INFO [2019-12-17 12:37:21,122] jepsen worker 38 - jepsen.util 12188 :ok :cas [12 [30 7]] INFO [2019-12-17 12:37:21,123] jepsen worker 38 - jepsen.util 12188 :invoke :read [12 nil] INFO [2019-12-17 12:37:21,139] jepsen worker 27 - jepsen.util 10917 :fail :cas [9 [7 30]] :ValueNotSwapFrom INFO [2019-12-17 12:37:21,140] jepsen worker 27 - jepsen.util 10917 :invoke :write [9 19] INFO [2019-12-17 12:37:21,186] jepsen worker 37 - jepsen.util 11377 :ok :write [12 49] INFO [2019-12-17 12:37:21,187] jepsen worker 37 - jepsen.util 11377 :invoke :cas [12 [49 21]] INFO [2019-12-17 12:37:21,189] jepsen worker 87 - jepsen.util 11247 :fail :cas [29 [16 9]] :ValueNotSwapFrom INFO [2019-12-17 12:37:21,189] jepsen worker 87 - jepsen.util 11247 :invoke :write [29 1] INFO [2019-12-17 12:37:21,196] jepsen worker 65 - jepsen.util 10415 :ok :write [21 46] INFO [2019-12-17 12:37:21,196] jepsen worker 65 - jepsen.util 10415 :invoke :cas [21 [37 42]] INFO [2019-12-17 12:37:21,224] jepsen worker 65 - jepsen.util 10415 :fail :cas [21 [37 42]] :ValueNotSwapFrom INFO [2019-12-17 12:37:21,224] jepsen worker 65 - jepsen.util 10415 :invoke :read [21 nil] INFO [2019-12-17 12:37:21,227] jepsen worker 9 - jepsen.util 12429 :ok :write [3 48] INFO [2019-12-17 12:37:21,227] jepsen worker 9 - jepsen.util 12429 :invoke :write [3 22] INFO [2019-12-17 12:37:21,227] jepsen worker 72 - jepsen.util 11952 :ok :write [24 38] INFO [2019-12-17 12:37:21,227] jepsen worker 72 - jepsen.util 11952 :invoke :cas [24 [47 42]] INFO [2019-12-17 12:37:21,234] jepsen worker 36 - jepsen.util 12546 :fail :cas [12 [44 31]] :ValueNotSwapFrom INFO [2019-12-17 12:37:21,234] jepsen worker 36 - jepsen.util 12546 :invoke :cas [12 [26 37]] INFO [2019-12-17 12:37:21,238] jepsen worker 79 - jepsen.util 11959 :ok :read [26 49] INFO [2019-12-17 12:37:21,238] jepsen worker 79 - jepsen.util 11959 :invoke :cas [26 [47 35]] INFO [2019-12-17 12:37:21,240] jepsen worker 80 - jepsen.util 11420 :ok :write [26 31] INFO [2019-12-17 12:37:21,240] jepsen worker 80 - jepsen.util 11420 :invoke :cas [26 [16 24]] INFO [2019-12-17 12:37:21,242] jepsen worker 5 - jepsen.util 11525 :ok :read [1 15] INFO [2019-12-17 12:37:21,242] jepsen worker 5 - jepsen.util 11525 :invoke :write [1 23] INFO [2019-12-17 12:37:21,250] jepsen worker 46 - jepsen.util 11566 :ok :write [15 48] INFO [2019-12-17 12:37:21,250] jepsen worker 20 - jepsen.util 11360 :ok :write [6 13] INFO [2019-12-17 12:37:21,250] jepsen worker 46 - jepsen.util 11566 :invoke :cas [15 [39 21]] INFO [2019-12-17 12:37:21,250] jepsen worker 20 - jepsen.util 11360 :invoke :cas [6 [49 48]] INFO [2019-12-17 12:37:21,265] jepsen worker 66 - jepsen.util 15996 :info :write [22 38] :RequestCanceledException INFO [2019-12-17 12:37:21,266] jepsen worker 66 - jepsen.util 16086 :invoke :write [22 22] INFO [2019-12-17 12:37:21,294] jepsen worker 30 - jepsen.util 10830 :fail :cas [10 [30 33]] :ValueNotSwapFrom INFO [2019-12-17 12:37:21,294] jepsen worker 30 - jepsen.util 10830 :invoke :cas [10 [16 46]] INFO [2019-12-17 12:37:21,294] jepsen worker 65 - jepsen.util 10415 :ok :read [21 46] INFO [2019-12-17 12:37:21,295] jepsen worker 65 - jepsen.util 10415 :invoke :read [21 nil] INFO [2019-12-17 12:37:21,295] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68324,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:37:21,295] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68318,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:37:21,295] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68324,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:37:21,295] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68318,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:37:21,295] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68327,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:37:21,295] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68327,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:37:21,300] jepsen worker 17 - jepsen.util 16127 :info :cas [5 [14 5]] :RequestCanceledException INFO [2019-12-17 12:37:21,301] jepsen worker 17 - jepsen.util 16217 :invoke :write [5 23] INFO [2019-12-17 12:37:21,310] jepsen worker 84 - jepsen.util 15834 :info :write [28 48] :RequestCanceledException INFO [2019-12-17 12:37:21,311] jepsen worker 84 - jepsen.util 15924 :invoke :read [28 nil] INFO [2019-12-17 12:37:21,333] jepsen worker 82 - jepsen.util 16462 :info :write [27 0] :RequestCanceledException INFO [2019-12-17 12:37:21,335] jepsen worker 82 - jepsen.util 16552 :invoke :write [27 41] INFO [2019-12-17 12:37:21,338] jepsen worker 27 - jepsen.util 10917 :ok :write [9 19] INFO [2019-12-17 12:37:21,339] jepsen worker 27 - jepsen.util 10917 :invoke :read [9 nil] INFO [2019-12-17 12:37:21,348] jepsen worker 26 - jepsen.util 16856 :info :cas [8 [27 42]] :RequestCanceledException INFO [2019-12-17 12:37:21,348] jepsen worker 62 - jepsen.util 16802 :info :write [20 26] :RequestCanceledException INFO [2019-12-17 12:37:21,349] jepsen worker 62 - jepsen.util 16892 :invoke :read [20 nil] INFO [2019-12-17 12:37:21,350] jepsen worker 26 - jepsen.util 16946 :invoke :write [8 25] INFO [2019-12-17 12:37:21,351] jepsen worker 31 - jepsen.util 10111 :ok :read [10 16] INFO [2019-12-17 12:37:21,351] jepsen worker 31 - jepsen.util 10111 :invoke :read [10 nil] INFO [2019-12-17 12:37:21,363] jepsen worker 45 - jepsen.util 11925 :ok :read [15 48] INFO [2019-12-17 12:37:21,363] jepsen worker 45 - jepsen.util 11925 :invoke :read [15 nil] INFO [2019-12-17 12:37:21,364] jepsen worker 65 - jepsen.util 10415 :ok :read [21 46] INFO [2019-12-17 12:37:21,365] jepsen worker 65 - jepsen.util 10415 :invoke :write [21 35] INFO [2019-12-17 12:37:21,374] jepsen worker 71 - jepsen.util 12581 :fail :cas [23 [35 36]] :ValueNotSwapFrom INFO [2019-12-17 12:37:21,374] jepsen worker 71 - jepsen.util 12581 :invoke :read [23 nil] INFO [2019-12-17 12:37:21,377] jepsen worker 4 - jepsen.util 10894 :fail :cas [1 [11 18]] :ValueNotSwapFrom INFO [2019-12-17 12:37:21,377] jepsen worker 4 - jepsen.util 10894 :invoke :write [1 10] INFO [2019-12-17 12:37:21,378] jepsen worker 10 - jepsen.util 11530 :ok :read [3 48] INFO [2019-12-17 12:37:21,378] jepsen worker 10 - jepsen.util 11530 :invoke :write [3 29] INFO [2019-12-17 12:37:21,386] jepsen worker 56 - jepsen.util 12656 :ok :read [18 21] INFO [2019-12-17 12:37:21,386] jepsen worker 56 - jepsen.util 12656 :invoke :cas [18 [23 16]] INFO [2019-12-17 12:37:21,401] jepsen worker 27 - jepsen.util 10917 :ok :read [9 19] INFO [2019-12-17 12:37:21,401] jepsen worker 27 - jepsen.util 10917 :invoke :read [9 nil] INFO [2019-12-17 12:37:21,410] jepsen worker 53 - jepsen.util 11303 :ok :write [17 27] INFO [2019-12-17 12:37:21,411] jepsen worker 53 - jepsen.util 11303 :invoke :cas [17 [47 19]] INFO [2019-12-17 12:37:21,417] jepsen worker 79 - jepsen.util 11959 :fail :cas [26 [47 35]] :ValueNotSwapFrom INFO [2019-12-17 12:37:21,417] jepsen worker 79 - jepsen.util 11959 :invoke :cas [26 [44 5]] INFO [2019-12-17 12:37:21,432] jepsen worker 86 - jepsen.util 16916 :info :cas [28 [36 37]] :RequestCanceledException INFO [2019-12-17 12:37:21,432] jepsen worker 77 - jepsen.util 16997 :info :cas [25 [5 35]] :RequestCanceledException INFO [2019-12-17 12:37:21,432] jepsen worker 28 - jepsen.util 9928 :info :cas [9 [10 26]] :RequestCanceledException INFO [2019-12-17 12:37:21,434] jepsen worker 86 - jepsen.util 17006 :invoke :read [28 nil] INFO [2019-12-17 12:37:21,434] jepsen worker 77 - jepsen.util 17087 :invoke :cas [25 [9 18]] INFO [2019-12-17 12:37:21,434] jepsen worker 28 - jepsen.util 10018 :invoke :read [9 nil] INFO [2019-12-17 12:37:21,447] jepsen worker 48 - jepsen.util 14898 :info :write [16 15] :RequestCanceledException INFO [2019-12-17 12:37:21,448] jepsen worker 48 - jepsen.util 14988 :invoke :read [16 nil] INFO [2019-12-17 12:37:21,454] jepsen worker 51 - jepsen.util 11031 :ok :write [17 16] INFO [2019-12-17 12:37:21,454] jepsen worker 51 - jepsen.util 11031 :invoke :write [17 43] INFO [2019-12-17 12:37:21,460] jepsen worker 52 - jepsen.util 11122 :fail :cas [17 [42 3]] :ValueNotSwapFrom INFO [2019-12-17 12:37:21,460] jepsen worker 52 - jepsen.util 11122 :invoke :read [17 nil] INFO [2019-12-17 12:37:21,464] jepsen worker 23 - jepsen.util 16313 :info :write [7 1] :RequestCanceledException INFO [2019-12-17 12:37:21,466] jepsen worker 23 - jepsen.util 16403 :invoke :write [7 26] INFO [2019-12-17 12:37:21,468] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:37:21,468] jepsen nemesis - couchbase.util Rebalance check count: 537 INFO [2019-12-17 12:37:21,469] jepsen nemesis - couchbase.util Rebalance stuck count: 518 INFO [2019-12-17 12:37:21,478] jepsen worker 70 - jepsen.util 12220 :ok :read [23 13] INFO [2019-12-17 12:37:21,479] jepsen worker 70 - jepsen.util 12220 :invoke :write [23 31] INFO [2019-12-17 12:37:21,479] jepsen worker 46 - jepsen.util 11566 :fail :cas [15 [39 21]] :ValueNotSwapFrom INFO [2019-12-17 12:37:21,479] jepsen worker 46 - jepsen.util 11566 :invoke :read [15 nil] INFO [2019-12-17 12:37:21,479] jepsen worker 47 - jepsen.util 12737 :fail :cas [15 [1 42]] :ValueNotSwapFrom INFO [2019-12-17 12:37:21,480] jepsen worker 47 - jepsen.util 12737 :invoke :read [15 nil] INFO [2019-12-17 12:37:21,482] jepsen worker 27 - jepsen.util 10917 :ok :read [9 19] INFO [2019-12-17 12:37:21,482] jepsen worker 27 - jepsen.util 10917 :invoke :read [9 nil] INFO [2019-12-17 12:37:21,500] jepsen worker 58 - jepsen.util 16438 :info :cas [19 [1 40]] :RequestCanceledException INFO [2019-12-17 12:37:21,500] jepsen worker 25 - jepsen.util 17035 :info :cas [8 [39 9]] :RequestCanceledException INFO [2019-12-17 12:37:21,501] jepsen worker 58 - jepsen.util 16528 :invoke :read [19 nil] INFO [2019-12-17 12:37:21,501] jepsen worker 25 - jepsen.util 17125 :invoke :cas [8 [28 47]] INFO [2019-12-17 12:37:21,509] jepsen worker 56 - jepsen.util 12656 :fail :cas [18 [23 16]] :ValueNotSwapFrom INFO [2019-12-17 12:37:21,509] jepsen worker 56 - jepsen.util 12656 :invoke :cas [18 [46 6]] INFO [2019-12-17 12:37:21,514] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68361,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:37:21,514] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68361,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:37:21,514] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68364,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:37:21,514] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68364,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:37:21,514] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68365,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:37:21,514] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68365,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:37:21,517] jepsen worker 43 - jepsen.util 16243 :info :write [14 8] :RequestCanceledException INFO [2019-12-17 12:37:21,517] jepsen worker 11 - jepsen.util 11261 :info :write [3 17] :RequestCanceledException INFO [2019-12-17 12:37:21,518] jepsen worker 67 - jepsen.util 16087 :info :cas [22 [49 28]] :RequestCanceledException INFO [2019-12-17 12:37:21,518] jepsen worker 63 - jepsen.util 10233 :fail :read [21 nil] :RequestTimeoutException INFO [2019-12-17 12:37:21,518] jepsen worker 63 - jepsen.util 10233 :invoke :read [21 nil] INFO [2019-12-17 12:37:21,519] jepsen worker 43 - jepsen.util 16333 :invoke :write [14 12] INFO [2019-12-17 12:37:21,519] jepsen worker 11 - jepsen.util 11351 :invoke :read [3 nil] INFO [2019-12-17 12:37:21,519] jepsen worker 67 - jepsen.util 16177 :invoke :read [22 nil] INFO [2019-12-17 12:37:21,528] jepsen worker 36 - jepsen.util 12546 :fail :cas [12 [26 37]] :ValueNotSwapFrom INFO [2019-12-17 12:37:21,528] jepsen worker 36 - jepsen.util 12546 :invoke :cas [12 [18 40]] INFO [2019-12-17 12:37:21,531] jepsen worker 3 - jepsen.util 11163 :fail :cas [1 [39 45]] :ValueNotSwapFrom INFO [2019-12-17 12:37:21,532] jepsen worker 3 - jepsen.util 11163 :invoke :cas [1 [13 21]] INFO [2019-12-17 12:37:21,534] jepsen worker 2 - jepsen.util 15572 :info :write [0 49] :RequestCanceledException INFO [2019-12-17 12:37:21,534] jepsen worker 85 - jepsen.util 16825 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:37:21,534] jepsen worker 85 - jepsen.util 16825 :invoke :cas [28 [49 41]] INFO [2019-12-17 12:37:21,535] jepsen worker 2 - jepsen.util 15662 :invoke :cas [0 [41 6]] INFO [2019-12-17 12:37:21,547] jepsen worker 21 - jepsen.util 15591 :info :cas [7 [29 3]] :RequestCanceledException INFO [2019-12-17 12:37:21,548] jepsen worker 21 - jepsen.util 15681 :invoke :read [7 nil] INFO [2019-12-17 12:37:21,550] jepsen worker 38 - jepsen.util 12188 :ok :read [12 49] INFO [2019-12-17 12:37:21,550] jepsen worker 38 - jepsen.util 12188 :invoke :cas [12 [26 34]] INFO [2019-12-17 12:37:21,562] jepsen worker 7 - jepsen.util 16117 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:37:21,562] jepsen worker 7 - jepsen.util 16117 :invoke :write [2 9] INFO [2019-12-17 12:37:21,564] jepsen worker 13 - jepsen.util 11083 :ok :read [4 27] INFO [2019-12-17 12:37:21,564] jepsen worker 13 - jepsen.util 11083 :invoke :write [4 17] INFO [2019-12-17 12:37:21,566] jepsen worker 27 - jepsen.util 10917 :ok :read [9 19] INFO [2019-12-17 12:37:21,566] jepsen worker 27 - jepsen.util 10917 :invoke :read [9 nil] INFO [2019-12-17 12:37:21,570] jepsen worker 46 - jepsen.util 11566 :ok :read [15 48] INFO [2019-12-17 12:37:21,570] jepsen worker 46 - jepsen.util 11566 :invoke :cas [15 [46 38]] INFO [2019-12-17 12:37:21,575] jepsen worker 20 - jepsen.util 11360 :fail :cas [6 [49 48]] :ValueNotSwapFrom INFO [2019-12-17 12:37:21,575] jepsen worker 20 - jepsen.util 11360 :invoke :read [6 nil] INFO [2019-12-17 12:37:21,622] jepsen worker 80 - jepsen.util 11420 :fail :cas [26 [16 24]] :ValueNotSwapFrom INFO [2019-12-17 12:37:21,622] jepsen worker 80 - jepsen.util 11420 :invoke :write [26 48] INFO [2019-12-17 12:37:21,633] jepsen worker 59 - jepsen.util 15359 :info :write [19 5] :RequestCanceledException INFO [2019-12-17 12:37:21,633] jepsen worker 8 - jepsen.util 15758 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:37:21,634] jepsen worker 8 - jepsen.util 15758 :invoke :cas [2 [8 12]] INFO [2019-12-17 12:37:21,635] jepsen worker 59 - jepsen.util 15449 :invoke :read [19 nil] INFO [2019-12-17 12:37:21,639] jepsen worker 53 - jepsen.util 11303 :fail :cas [17 [47 19]] :ValueNotSwapFrom INFO [2019-12-17 12:37:21,639] jepsen worker 53 - jepsen.util 11303 :invoke :read [17 nil] INFO [2019-12-17 12:37:21,647] jepsen worker 27 - jepsen.util 10917 :ok :read [9 19] INFO [2019-12-17 12:37:21,647] jepsen worker 27 - jepsen.util 10917 :invoke :write [9 19] INFO [2019-12-17 12:37:21,650] jepsen worker 87 - jepsen.util 11247 :ok :write [29 1] INFO [2019-12-17 12:37:21,650] jepsen worker 87 - jepsen.util 11247 :invoke :read [29 nil] INFO [2019-12-17 12:37:21,658] jepsen worker 46 - jepsen.util 11566 :fail :cas [15 [46 38]] :ValueNotSwapFrom INFO [2019-12-17 12:37:21,658] jepsen worker 46 - jepsen.util 11566 :invoke :cas [15 [19 42]] INFO [2019-12-17 12:37:21,662] jepsen worker 50 - jepsen.util 16250 :info :cas [16 [5 26]] :RequestCanceledException INFO [2019-12-17 12:37:21,662] jepsen worker 19 - jepsen.util 12079 :info :write [6 30] :RequestCanceledException INFO [2019-12-17 12:37:21,664] jepsen worker 19 - jepsen.util 12169 :invoke :write [6 37] INFO [2019-12-17 12:37:21,666] jepsen worker 50 - jepsen.util 16340 :invoke :read [16 nil] INFO [2019-12-17 12:37:21,667] jepsen worker 52 - jepsen.util 11122 :ok :read [17 27] INFO [2019-12-17 12:37:21,667] jepsen worker 52 - jepsen.util 11122 :invoke :read [17 nil] INFO [2019-12-17 12:37:21,673] jepsen worker 45 - jepsen.util 11925 :ok :read [15 48] INFO [2019-12-17 12:37:21,673] jepsen worker 45 - jepsen.util 11925 :invoke :read [15 nil] INFO [2019-12-17 12:37:21,696] jepsen worker 0 - jepsen.util 16740 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:37:21,696] jepsen worker 4 - jepsen.util 10894 :ok :write [1 10] INFO [2019-12-17 12:37:21,696] jepsen worker 0 - jepsen.util 16740 :invoke :read [0 nil] INFO [2019-12-17 12:37:21,697] jepsen worker 4 - jepsen.util 10894 :invoke :read [1 nil] INFO [2019-12-17 12:37:21,701] jepsen worker 3 - jepsen.util 11163 :fail :cas [1 [13 21]] :ValueNotSwapFrom INFO [2019-12-17 12:37:21,701] jepsen worker 3 - jepsen.util 11163 :invoke :read [1 nil] INFO [2019-12-17 12:37:21,707] jepsen worker 65 - jepsen.util 10415 :ok :write [21 35] INFO [2019-12-17 12:37:21,707] jepsen worker 65 - jepsen.util 10415 :invoke :read [21 nil] INFO [2019-12-17 12:37:21,708] jepsen worker 78 - jepsen.util 11598 :ok :write [26 27] INFO [2019-12-17 12:37:21,708] jepsen worker 78 - jepsen.util 11598 :invoke :write [26 0] INFO [2019-12-17 12:37:21,712] jepsen worker 61 - jepsen.util 17341 :info :cas [20 [5 23]] :RequestCanceledException INFO [2019-12-17 12:37:21,712] jepsen worker 15 - jepsen.util 14865 :info :cas [5 [20 11]] :RequestCanceledException INFO [2019-12-17 12:37:21,714] jepsen worker 15 - jepsen.util 14955 :invoke :write [5 42] INFO [2019-12-17 12:37:21,714] jepsen worker 61 - jepsen.util 17431 :invoke :write [20 33] INFO [2019-12-17 12:37:21,715] jepsen worker 46 - jepsen.util 11566 :fail :cas [15 [19 42]] :ValueNotSwapFrom INFO [2019-12-17 12:37:21,715] jepsen worker 47 - jepsen.util 12737 :ok :read [15 48] INFO [2019-12-17 12:37:21,715] jepsen worker 46 - jepsen.util 11566 :invoke :read [15 nil] INFO [2019-12-17 12:37:21,715] jepsen worker 47 - jepsen.util 12737 :invoke :write [15 23] INFO [2019-12-17 12:37:21,733] jepsen worker 29 - jepsen.util 10109 :info :write [9 41] :RequestCanceledException INFO [2019-12-17 12:37:21,733] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68397,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:37:21,733] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68397,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:37:21,733] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68400,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:37:21,734] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68400,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:37:21,734] jepsen worker 29 - jepsen.util 10199 :invoke :read [9 nil] INFO [2019-12-17 12:37:21,747] jepsen worker 73 - jepsen.util 10873 :info :write [24 19] :RequestCanceledException INFO [2019-12-17 12:37:21,748] jepsen worker 73 - jepsen.util 10963 :invoke :write [24 6] INFO [2019-12-17 12:37:21,752] jepsen worker 71 - jepsen.util 12581 :ok :read [23 13] INFO [2019-12-17 12:37:21,752] jepsen worker 71 - jepsen.util 12581 :invoke :read [23 nil] INFO [2019-12-17 12:37:21,757] jepsen worker 79 - jepsen.util 11959 :fail :cas [26 [44 5]] :ValueNotSwapFrom INFO [2019-12-17 12:37:21,757] jepsen worker 79 - jepsen.util 11959 :invoke :cas [26 [5 48]] INFO [2019-12-17 12:37:21,766] jepsen worker 4 - jepsen.util 10894 :ok :read [1 10] INFO [2019-12-17 12:37:21,766] jepsen worker 4 - jepsen.util 10894 :invoke :write [1 46] INFO [2019-12-17 12:37:21,774] jepsen worker 38 - jepsen.util 12188 :fail :cas [12 [26 34]] :ValueNotSwapFrom INFO [2019-12-17 12:37:21,774] jepsen worker 38 - jepsen.util 12188 :invoke :read [12 nil] INFO [2019-12-17 12:37:21,807] jepsen worker 79 - jepsen.util 11959 :fail :cas [26 [5 48]] :ValueNotSwapFrom INFO [2019-12-17 12:37:21,807] jepsen worker 79 - jepsen.util 11959 :invoke :write [26 1] INFO [2019-12-17 12:37:21,807] jepsen worker 52 - jepsen.util 11122 :ok :read [17 27] INFO [2019-12-17 12:37:21,807] jepsen worker 52 - jepsen.util 11122 :invoke :write [17 47] INFO [2019-12-17 12:37:21,817] jepsen worker 65 - jepsen.util 10415 :ok :read [21 35] INFO [2019-12-17 12:37:21,818] jepsen worker 65 - jepsen.util 10415 :invoke :write [21 37] INFO [2019-12-17 12:37:21,831] jepsen worker 20 - jepsen.util 11360 :ok :read [6 13] INFO [2019-12-17 12:37:21,833] jepsen worker 32 - jepsen.util 12092 :info :write [10 11] :RequestCanceledException INFO [2019-12-17 12:37:21,834] jepsen worker 20 - jepsen.util 11360 :invoke :write [6 37] INFO [2019-12-17 12:37:21,836] jepsen worker 32 - jepsen.util 12182 :invoke :write [10 47] INFO [2019-12-17 12:37:21,841] jepsen worker 56 - jepsen.util 12656 :fail :cas [18 [46 6]] :ValueNotSwapFrom INFO [2019-12-17 12:37:21,841] jepsen worker 56 - jepsen.util 12656 :invoke :cas [18 [41 15]] INFO [2019-12-17 12:37:21,846] jepsen worker 3 - jepsen.util 11163 :ok :read [1 10] INFO [2019-12-17 12:37:21,846] jepsen worker 3 - jepsen.util 11163 :invoke :cas [1 [28 19]] INFO [2019-12-17 12:37:21,846] jepsen worker 46 - jepsen.util 11566 :ok :read [15 48] INFO [2019-12-17 12:37:21,846] jepsen worker 46 - jepsen.util 11566 :invoke :cas [15 [0 1]] INFO [2019-12-17 12:37:21,883] jepsen worker 87 - jepsen.util 11247 :ok :read [29 1] INFO [2019-12-17 12:37:21,884] jepsen worker 87 - jepsen.util 11247 :invoke :write [29 13] INFO [2019-12-17 12:37:21,887] jepsen worker 45 - jepsen.util 11925 :ok :read [15 48] INFO [2019-12-17 12:37:21,887] jepsen worker 45 - jepsen.util 11925 :invoke :read [15 nil] INFO [2019-12-17 12:37:21,888] jepsen worker 53 - jepsen.util 11303 :ok :read [17 27] INFO [2019-12-17 12:37:21,888] jepsen worker 53 - jepsen.util 11303 :invoke :write [17 17] INFO [2019-12-17 12:37:21,898] jepsen worker 27 - jepsen.util 10917 :ok :write [9 19] INFO [2019-12-17 12:37:21,898] jepsen worker 27 - jepsen.util 10917 :invoke :cas [9 [36 13]] INFO [2019-12-17 12:37:21,902] jepsen worker 38 - jepsen.util 12188 :ok :read [12 49] INFO [2019-12-17 12:37:21,902] jepsen worker 38 - jepsen.util 12188 :invoke :write [12 11] INFO [2019-12-17 12:37:21,924] jepsen worker 29 - jepsen.util 10199 :ok :read [9 19] INFO [2019-12-17 12:37:21,924] jepsen worker 29 - jepsen.util 10199 :invoke :read [9 nil] INFO [2019-12-17 12:37:21,938] jepsen worker 46 - jepsen.util 11566 :fail :cas [15 [0 1]] :ValueNotSwapFrom INFO [2019-12-17 12:37:21,938] jepsen worker 46 - jepsen.util 11566 :invoke :write [15 17] INFO [2019-12-17 12:37:21,947] jepsen worker 27 - jepsen.util 10917 :fail :cas [9 [36 13]] :ValueNotSwapFrom INFO [2019-12-17 12:37:21,947] jepsen worker 27 - jepsen.util 10917 :invoke :write [9 41] INFO [2019-12-17 12:37:21,950] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68433,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:37:21,951] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68433,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:37:21,951] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68437,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:37:21,951] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68437,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:37:21,958] jepsen worker 74 - jepsen.util 11594 :ok :write [24 28] INFO [2019-12-17 12:37:21,958] jepsen worker 74 - jepsen.util 11594 :invoke :cas [24 [19 8]] INFO [2019-12-17 12:37:21,964] jepsen worker 18 - jepsen.util 11898 :info :write [6 40] :RequestCanceledException INFO [2019-12-17 12:37:21,965] jepsen worker 18 - jepsen.util 11988 :invoke :read [6 nil] INFO [2019-12-17 12:37:21,971] jepsen worker 5 - jepsen.util 11525 :ok :write [1 23] INFO [2019-12-17 12:37:21,972] jepsen worker 5 - jepsen.util 11525 :invoke :read [1 nil] INFO [2019-12-17 12:37:21,983] jepsen worker 52 - jepsen.util 11122 :ok :write [17 47] INFO [2019-12-17 12:37:21,983] jepsen worker 52 - jepsen.util 11122 :invoke :write [17 48] INFO [2019-12-17 12:37:22,001] jepsen worker 56 - jepsen.util 12656 :fail :cas [18 [41 15]] :ValueNotSwapFrom INFO [2019-12-17 12:37:22,001] jepsen worker 56 - jepsen.util 12656 :invoke :cas [18 [7 41]] INFO [2019-12-17 12:37:22,008] jepsen worker 29 - jepsen.util 10199 :ok :read [9 19] INFO [2019-12-17 12:37:22,008] jepsen worker 29 - jepsen.util 10199 :invoke :read [9 nil] INFO [2019-12-17 12:37:22,022] jepsen worker 64 - jepsen.util 8974 :fail :cas [21 [33 9]] :ValueNotSwapFrom INFO [2019-12-17 12:37:22,022] jepsen worker 64 - jepsen.util 8974 :invoke :read [21 nil] INFO [2019-12-17 12:37:22,044] jepsen worker 55 - jepsen.util 12385 :ok :write [18 11] INFO [2019-12-17 12:37:22,044] jepsen worker 55 - jepsen.util 12385 :invoke :read [18 nil] INFO [2019-12-17 12:37:22,056] jepsen worker 72 - jepsen.util 11952 :fail :cas [24 [47 42]] :ValueNotSwapFrom INFO [2019-12-17 12:37:22,058] jepsen worker 72 - jepsen.util 11952 :invoke :cas [24 [13 23]] INFO [2019-12-17 12:37:22,071] jepsen worker 63 - jepsen.util 10233 :ok :read [21 35] INFO [2019-12-17 12:37:22,072] jepsen worker 63 - jepsen.util 10233 :invoke :write [21 1] INFO [2019-12-17 12:37:22,077] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68451,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:37:22,077] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68451,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:37:22,077] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68454,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:37:22,077] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68454,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:37:22,077] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68455,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:37:22,077] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68455,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:37:22,097] jepsen worker 56 - jepsen.util 12656 :fail :cas [18 [7 41]] :ValueNotSwapFrom INFO [2019-12-17 12:37:22,097] jepsen worker 56 - jepsen.util 12656 :invoke :write [18 4] INFO [2019-12-17 12:37:22,098] jepsen worker 65 - jepsen.util 10415 :ok :write [21 37] INFO [2019-12-17 12:37:22,098] jepsen worker 65 - jepsen.util 10415 :invoke :read [21 nil] INFO [2019-12-17 12:37:22,111] jepsen worker 18 - jepsen.util 11988 :ok :read [6 13] INFO [2019-12-17 12:37:22,111] jepsen worker 18 - jepsen.util 11988 :invoke :cas [6 [27 31]] INFO [2019-12-17 12:37:22,117] jepsen worker 44 - jepsen.util 16694 :info :cas [14 [22 38]] :RequestCanceledException INFO [2019-12-17 12:37:22,119] jepsen worker 44 - jepsen.util 16784 :invoke :write [14 3] INFO [2019-12-17 12:37:22,133] jepsen worker 16 - jepsen.util 17926 :info :cas [5 [43 40]] :RequestCanceledException INFO [2019-12-17 12:37:22,134] jepsen worker 16 - jepsen.util 18016 :invoke :cas [5 [14 19]] INFO [2019-12-17 12:37:22,135] jepsen worker 88 - jepsen.util 11878 :fail :cas [29 [40 32]] :ValueNotSwapFrom INFO [2019-12-17 12:37:22,135] jepsen worker 88 - jepsen.util 11878 :invoke :write [29 36] INFO [2019-12-17 12:37:22,149] jepsen worker 65 - jepsen.util 10415 :ok :read [21 37] INFO [2019-12-17 12:37:22,149] jepsen worker 65 - jepsen.util 10415 :invoke :write [21 13] INFO [2019-12-17 12:37:22,157] jepsen worker 53 - jepsen.util 11303 :ok :write [17 17] INFO [2019-12-17 12:37:22,158] jepsen worker 53 - jepsen.util 11303 :invoke :cas [17 [38 23]] INFO [2019-12-17 12:37:22,158] jepsen worker 89 - jepsen.util 11789 :ok :write [29 6] INFO [2019-12-17 12:37:22,158] jepsen worker 89 - jepsen.util 11789 :invoke :cas [29 [38 38]] INFO [2019-12-17 12:37:22,190] jepsen worker 27 - jepsen.util 10917 :ok :write [9 41] INFO [2019-12-17 12:37:22,190] jepsen worker 27 - jepsen.util 10917 :invoke :write [9 38] INFO [2019-12-17 12:37:22,206] jepsen worker 18 - jepsen.util 11988 :fail :cas [6 [27 31]] :ValueNotSwapFrom INFO [2019-12-17 12:37:22,206] jepsen worker 18 - jepsen.util 11988 :invoke :cas [6 [46 0]] INFO [2019-12-17 12:37:22,230] jepsen worker 5 - jepsen.util 11525 :ok :read [1 23] INFO [2019-12-17 12:37:22,230] jepsen worker 5 - jepsen.util 11525 :invoke :cas [1 [31 37]] INFO [2019-12-17 12:37:22,233] jepsen worker 81 - jepsen.util 15651 :info :write [27 33] :RequestCanceledException INFO [2019-12-17 12:37:22,234] jepsen worker 81 - jepsen.util 15741 :invoke :write [27 36] INFO [2019-12-17 12:37:22,247] jepsen worker 35 - jepsen.util 15875 :info :cas [11 [3 38]] :RequestCanceledException INFO [2019-12-17 12:37:22,248] jepsen worker 35 - jepsen.util 15965 :invoke :cas [11 [49 10]] INFO [2019-12-17 12:37:22,262] jepsen worker 68 - jepsen.util 15458 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:37:22,262] jepsen worker 68 - jepsen.util 15458 :invoke :write [22 4] INFO [2019-12-17 12:37:22,269] jepsen worker 9 - jepsen.util 12429 :ok :write [3 22] INFO [2019-12-17 12:37:22,269] jepsen worker 9 - jepsen.util 12429 :invoke :write [3 40] INFO [2019-12-17 12:37:22,278] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68473,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:37:22,279] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68480,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:37:22,279] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68473,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:37:22,279] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68480,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:37:22,279] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68475,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:37:22,279] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68481,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:37:22,279] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68475,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:37:22,279] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68481,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:37:22,279] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68482,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:37:22,279] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68482,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:37:22,303] jepsen worker 4 - jepsen.util 10894 :ok :write [1 46] INFO [2019-12-17 12:37:22,304] jepsen worker 4 - jepsen.util 10894 :invoke :write [1 32] INFO [2019-12-17 12:37:22,308] jepsen worker 70 - jepsen.util 12220 :ok :write [23 31] INFO [2019-12-17 12:37:22,308] jepsen worker 70 - jepsen.util 12220 :invoke :cas [23 [23 19]] INFO [2019-12-17 12:37:22,310] jepsen worker 31 - jepsen.util 10111 :ok :read [10 16] INFO [2019-12-17 12:37:22,310] jepsen worker 31 - jepsen.util 10111 :invoke :write [10 33] INFO [2019-12-17 12:37:22,318] jepsen worker 36 - jepsen.util 12546 :fail :cas [12 [18 40]] :ValueNotSwapFrom INFO [2019-12-17 12:37:22,318] jepsen worker 36 - jepsen.util 12546 :invoke :read [12 nil] INFO [2019-12-17 12:37:22,323] jepsen worker 65 - jepsen.util 10415 :ok :write [21 13] INFO [2019-12-17 12:37:22,323] jepsen worker 65 - jepsen.util 10415 :invoke :cas [21 [40 35]] INFO [2019-12-17 12:37:22,328] jepsen worker 55 - jepsen.util 12385 :ok :read [18 11] INFO [2019-12-17 12:37:22,329] jepsen worker 55 - jepsen.util 12385 :invoke :cas [18 [5 41]] INFO [2019-12-17 12:37:22,332] jepsen worker 27 - jepsen.util 10917 :ok :write [9 38] INFO [2019-12-17 12:37:22,332] jepsen worker 27 - jepsen.util 10917 :invoke :write [9 7] INFO [2019-12-17 12:37:22,374] jepsen worker 65 - jepsen.util 10415 :fail :cas [21 [40 35]] :ValueNotSwapFrom INFO [2019-12-17 12:37:22,375] jepsen worker 65 - jepsen.util 10415 :invoke :cas [21 [13 31]] INFO [2019-12-17 12:37:22,379] jepsen worker 5 - jepsen.util 11525 :fail :cas [1 [31 37]] :ValueNotSwapFrom INFO [2019-12-17 12:37:22,379] jepsen worker 5 - jepsen.util 11525 :invoke :read [1 nil] INFO [2019-12-17 12:37:22,400] jepsen worker 41 - jepsen.util 17321 :info :write [13 10] :RequestCanceledException INFO [2019-12-17 12:37:22,401] jepsen worker 41 - jepsen.util 17411 :invoke :write [13 0] INFO [2019-12-17 12:37:22,418] jepsen worker 75 - jepsen.util 15735 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:37:22,418] jepsen worker 75 - jepsen.util 15735 :invoke :write [25 35] INFO [2019-12-17 12:37:22,426] jepsen worker 88 - jepsen.util 11878 :ok :write [29 36] INFO [2019-12-17 12:37:22,426] jepsen worker 88 - jepsen.util 11878 :invoke :cas [29 [18 5]] INFO [2019-12-17 12:37:22,433] jepsen worker 40 - jepsen.util 16690 :info :write [13 46] :RequestCanceledException INFO [2019-12-17 12:37:22,434] jepsen worker 40 - jepsen.util 16780 :invoke :read [13 nil] INFO [2019-12-17 12:37:22,436] jepsen worker 80 - jepsen.util 11420 :ok :write [26 48] INFO [2019-12-17 12:37:22,437] jepsen worker 80 - jepsen.util 11420 :invoke :write [26 23] INFO [2019-12-17 12:37:22,448] jepsen worker 42 - jepsen.util 17052 :info :cas [14 [19 9]] :RequestCanceledException INFO [2019-12-17 12:37:22,449] jepsen worker 42 - jepsen.util 17142 :invoke :read [14 nil] INFO [2019-12-17 12:37:22,463] jepsen worker 19 - jepsen.util 12169 :ok :write [6 37] INFO [2019-12-17 12:37:22,463] jepsen worker 19 - jepsen.util 12169 :invoke :write [6 33] INFO [2019-12-17 12:37:22,478] jepsen worker 27 - jepsen.util 10917 :ok :write [9 7] INFO [2019-12-17 12:37:22,478] jepsen worker 27 - jepsen.util 10917 :invoke :write [9 27] INFO [2019-12-17 12:37:22,482] jepsen worker 47 - jepsen.util 12737 :ok :write [15 23] INFO [2019-12-17 12:37:22,483] jepsen worker 47 - jepsen.util 12737 :invoke :write [15 0] INFO [2019-12-17 12:37:22,498] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68507,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:37:22,498] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68507,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:37:22,498] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68511,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:37:22,498] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68511,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:37:22,499] jepsen worker 83 - jepsen.util 16373 :info :cas [27 [2 25]] :RequestCanceledException INFO [2019-12-17 12:37:22,499] jepsen worker 12 - jepsen.util 11982 :info :write [4 11] :RequestCanceledException INFO [2019-12-17 12:37:22,500] jepsen worker 83 - jepsen.util 16463 :invoke :read [27 nil] INFO [2019-12-17 12:37:22,500] jepsen worker 12 - jepsen.util 12072 :invoke :read [4 nil] INFO [2019-12-17 12:37:22,507] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:37:22,507] jepsen worker 89 - jepsen.util 11789 :fail :cas [29 [38 38]] :ValueNotSwapFrom INFO [2019-12-17 12:37:22,507] jepsen nemesis - couchbase.util Rebalance check count: 538 INFO [2019-12-17 12:37:22,507] jepsen worker 89 - jepsen.util 11789 :invoke :write [29 28] INFO [2019-12-17 12:37:22,507] jepsen nemesis - couchbase.util Rebalance stuck count: 519 INFO [2019-12-17 12:37:22,517] jepsen worker 34 - jepsen.util 16594 :info :write [11 35] :RequestCanceledException INFO [2019-12-17 12:37:22,517] jepsen worker 22 - jepsen.util 16042 :info :cas [7 [49 25]] :RequestCanceledException INFO [2019-12-17 12:37:22,518] jepsen worker 34 - jepsen.util 16684 :invoke :write [11 44] INFO [2019-12-17 12:37:22,518] jepsen worker 22 - jepsen.util 16132 :invoke :cas [7 [3 20]] INFO [2019-12-17 12:37:22,529] jepsen worker 3 - jepsen.util 11163 :fail :cas [1 [28 19]] :ValueNotSwapFrom INFO [2019-12-17 12:37:22,529] jepsen worker 3 - jepsen.util 11163 :invoke :read [1 nil] INFO [2019-12-17 12:37:22,533] jepsen worker 6 - jepsen.util 16566 :info :write [2 9] :RequestCanceledException INFO [2019-12-17 12:37:22,533] jepsen worker 39 - jepsen.util 15609 :info :cas [13 [32 16]] :RequestCanceledException INFO [2019-12-17 12:37:22,533] jepsen worker 24 - jepsen.util 14964 :info :cas [8 [22 1]] :RequestCanceledException INFO [2019-12-17 12:37:22,534] jepsen worker 39 - jepsen.util 15699 :invoke :cas [13 [30 7]] INFO [2019-12-17 12:37:22,534] jepsen worker 6 - jepsen.util 16656 :invoke :read [2 nil] INFO [2019-12-17 12:37:22,534] jepsen worker 24 - jepsen.util 15054 :invoke :cas [8 [34 4]] INFO [2019-12-17 12:37:22,548] jepsen worker 49 - jepsen.util 16159 :info :cas [16 [29 43]] :RequestCanceledException INFO [2019-12-17 12:37:22,548] jepsen worker 64 - jepsen.util 8974 :ok :read [21 13] INFO [2019-12-17 12:37:22,549] jepsen worker 64 - jepsen.util 8974 :invoke :cas [21 [14 30]] INFO [2019-12-17 12:37:22,549] jepsen worker 49 - jepsen.util 16249 :invoke :write [16 44] INFO [2019-12-17 12:37:22,554] jepsen worker 65 - jepsen.util 10415 :ok :cas [21 [13 31]] INFO [2019-12-17 12:37:22,554] jepsen worker 65 - jepsen.util 10415 :invoke :read [21 nil] INFO [2019-12-17 12:37:22,557] jepsen worker 72 - jepsen.util 11952 :fail :cas [24 [13 23]] :ValueNotSwapFrom INFO [2019-12-17 12:37:22,557] jepsen worker 72 - jepsen.util 11952 :invoke :write [24 30] INFO [2019-12-17 12:37:22,568] jepsen worker 65 - jepsen.util 10415 :ok :read [21 31] INFO [2019-12-17 12:37:22,568] jepsen worker 65 - jepsen.util 10415 :invoke :read [21 nil] INFO [2019-12-17 12:37:22,588] jepsen worker 65 - jepsen.util 10415 :ok :read [21 31] INFO [2019-12-17 12:37:22,588] jepsen worker 65 - jepsen.util 10415 :invoke :read [21 nil] INFO [2019-12-17 12:37:22,600] jepsen worker 69 - jepsen.util 12039 :info :write [23 32] :RequestCanceledException INFO [2019-12-17 12:37:22,600] jepsen worker 1 - jepsen.util 16201 :info :cas [0 [17 5]] :RequestCanceledException INFO [2019-12-17 12:37:22,601] jepsen worker 69 - jepsen.util 12129 :invoke :read [23 nil] INFO [2019-12-17 12:37:22,601] jepsen worker 1 - jepsen.util 16291 :invoke :write [0 9] INFO [2019-12-17 12:37:22,610] jepsen worker 65 - jepsen.util 10415 :ok :read [21 31] INFO [2019-12-17 12:37:22,610] jepsen worker 65 - jepsen.util 10415 :invoke :write [21 46] INFO [2019-12-17 12:37:22,619] jepsen worker 5 - jepsen.util 11525 :ok :read [1 46] INFO [2019-12-17 12:37:22,619] jepsen worker 5 - jepsen.util 11525 :invoke :write [1 34] INFO [2019-12-17 12:37:22,623] jepsen worker 12 - jepsen.util 12072 :ok :read [4 27] INFO [2019-12-17 12:37:22,624] jepsen worker 12 - jepsen.util 12072 :invoke :cas [4 [48 21]] INFO [2019-12-17 12:37:22,633] jepsen worker 57 - jepsen.util 15987 :info :cas [19 [40 10]] :RequestCanceledException INFO [2019-12-17 12:37:22,633] jepsen worker 76 - jepsen.util 16276 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:37:22,633] jepsen worker 76 - jepsen.util 16276 :invoke :write [25 3] INFO [2019-12-17 12:37:22,639] jepsen worker 57 - jepsen.util 16077 :invoke :read [19 nil] INFO [2019-12-17 12:37:22,648] jepsen worker 60 - jepsen.util 17250 :info :write [20 1] :RequestCanceledException INFO [2019-12-17 12:37:22,649] jepsen worker 60 - jepsen.util 17340 :invoke :cas [20 [38 40]] INFO [2019-12-17 12:37:22,663] jepsen worker 63 - jepsen.util 10233 :ok :write [21 1] INFO [2019-12-17 12:37:22,663] jepsen worker 63 - jepsen.util 10233 :invoke :read [21 nil] INFO [2019-12-17 12:37:22,673] jepsen worker 20 - jepsen.util 11360 :ok :write [6 37] INFO [2019-12-17 12:37:22,673] jepsen worker 20 - jepsen.util 11360 :invoke :read [6 nil] INFO [2019-12-17 12:37:22,684] jepsen worker 9 - jepsen.util 12429 :ok :write [3 40] INFO [2019-12-17 12:37:22,684] jepsen worker 9 - jepsen.util 12429 :invoke :read [3 nil] INFO [2019-12-17 12:37:22,684] jepsen worker 20 - jepsen.util 11360 :ok :read [6 37] INFO [2019-12-17 12:37:22,684] jepsen worker 20 - jepsen.util 11360 :invoke :cas [6 [44 16]] INFO [2019-12-17 12:37:22,709] jepsen worker 37 - jepsen.util 11377 :ok :cas [12 [49 21]] INFO [2019-12-17 12:37:22,709] jepsen worker 37 - jepsen.util 11377 :invoke :write [12 27] INFO [2019-12-17 12:37:22,709] jepsen worker 20 - jepsen.util 11360 :fail :cas [6 [44 16]] :ValueNotSwapFrom INFO [2019-12-17 12:37:22,709] jepsen worker 20 - jepsen.util 11360 :invoke :cas [6 [10 47]] INFO [2019-12-17 12:37:22,717] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68540,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:37:22,717] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68540,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:37:22,718] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68544,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:37:22,718] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68544,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:37:22,718] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68545,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:37:22,718] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68545,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:37:22,722] jepsen worker 65 - jepsen.util 10415 :ok :write [21 46] INFO [2019-12-17 12:37:22,722] jepsen worker 65 - jepsen.util 10415 :invoke :read [21 nil] INFO [2019-12-17 12:37:22,729] jepsen worker 20 - jepsen.util 11360 :fail :cas [6 [10 47]] :ValueNotSwapFrom INFO [2019-12-17 12:37:22,729] jepsen worker 20 - jepsen.util 11360 :invoke :write [6 26] INFO [2019-12-17 12:37:22,739] jepsen worker 65 - jepsen.util 10415 :ok :read [21 46] INFO [2019-12-17 12:37:22,739] jepsen worker 65 - jepsen.util 10415 :invoke :write [21 32] INFO [2019-12-17 12:37:22,760] jepsen worker 27 - jepsen.util 10917 :ok :write [9 27] INFO [2019-12-17 12:37:22,760] jepsen worker 27 - jepsen.util 10917 :invoke :write [9 44] INFO [2019-12-17 12:37:22,789] jepsen worker 4 - jepsen.util 10894 :ok :write [1 32] INFO [2019-12-17 12:37:22,789] jepsen worker 4 - jepsen.util 10894 :invoke :write [1 16] INFO [2019-12-17 12:37:22,801] jepsen worker 19 - jepsen.util 12169 :ok :write [6 33] INFO [2019-12-17 12:37:22,802] jepsen worker 19 - jepsen.util 12169 :invoke :read [6 nil] INFO [2019-12-17 12:37:22,830] jepsen worker 31 - jepsen.util 10111 :ok :write [10 33] INFO [2019-12-17 12:37:22,830] jepsen worker 31 - jepsen.util 10111 :invoke :cas [10 [33 14]] INFO [2019-12-17 12:37:22,834] jepsen worker 55 - jepsen.util 12385 :fail :cas [18 [5 41]] :ValueNotSwapFrom INFO [2019-12-17 12:37:22,834] jepsen worker 55 - jepsen.util 12385 :invoke :cas [18 [7 31]] INFO [2019-12-17 12:37:22,846] jepsen worker 88 - jepsen.util 11878 :fail :cas [29 [18 5]] :ValueNotSwapFrom INFO [2019-12-17 12:37:22,846] jepsen worker 88 - jepsen.util 11878 :invoke :write [29 16] INFO [2019-12-17 12:37:22,847] jepsen worker 19 - jepsen.util 12169 :ok :read [6 33] INFO [2019-12-17 12:37:22,847] jepsen worker 19 - jepsen.util 12169 :invoke :read [6 nil] INFO [2019-12-17 12:37:22,879] jepsen worker 69 - jepsen.util 12129 :ok :read [23 32] INFO [2019-12-17 12:37:22,879] jepsen worker 69 - jepsen.util 12129 :invoke :write [23 37] INFO [2019-12-17 12:37:22,883] jepsen worker 19 - jepsen.util 12169 :ok :read [6 33] INFO [2019-12-17 12:37:22,883] jepsen worker 19 - jepsen.util 12169 :invoke :read [6 nil] INFO [2019-12-17 12:37:22,900] jepsen worker 65 - jepsen.util 10415 :ok :write [21 32] INFO [2019-12-17 12:37:22,900] jepsen worker 65 - jepsen.util 10415 :invoke :cas [21 [38 32]] INFO [2019-12-17 12:37:22,917] jepsen worker 65 - jepsen.util 10415 :fail :cas [21 [38 32]] :ValueNotSwapFrom INFO [2019-12-17 12:37:22,917] jepsen worker 65 - jepsen.util 10415 :invoke :write [21 39] INFO [2019-12-17 12:37:22,936] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68564,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:37:22,936] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68564,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:37:22,960] jepsen worker 27 - jepsen.util 10917 :ok :write [9 44] INFO [2019-12-17 12:37:22,960] jepsen worker 27 - jepsen.util 10917 :invoke :write [9 18] INFO [2019-12-17 12:37:22,962] jepsen worker 14 - jepsen.util 11444 :info :write [4 47] :RequestCanceledException INFO [2019-12-17 12:37:22,963] jepsen worker 14 - jepsen.util 11534 :invoke :cas [4 [32 12]] INFO [2019-12-17 12:37:22,966] jepsen worker 19 - jepsen.util 12169 :ok :read [6 33] INFO [2019-12-17 12:37:22,967] jepsen worker 19 - jepsen.util 12169 :invoke :cas [6 [45 44]] INFO [2019-12-17 12:37:22,970] jepsen worker 55 - jepsen.util 12385 :fail :cas [18 [7 31]] :ValueNotSwapFrom INFO [2019-12-17 12:37:22,970] jepsen worker 55 - jepsen.util 12385 :invoke :write [18 48] INFO [2019-12-17 12:37:22,990] jepsen worker 5 - jepsen.util 11525 :ok :write [1 34] INFO [2019-12-17 12:37:22,990] jepsen worker 5 - jepsen.util 11525 :invoke :write [1 15] INFO [2019-12-17 12:37:23,000] jepsen worker 19 - jepsen.util 12169 :fail :cas [6 [45 44]] :ValueNotSwapFrom INFO [2019-12-17 12:37:23,000] jepsen worker 19 - jepsen.util 12169 :invoke :write [6 4] INFO [2019-12-17 12:37:23,017] jepsen worker 12 - jepsen.util 12072 :fail :cas [4 [48 21]] :ValueNotSwapFrom INFO [2019-12-17 12:37:23,017] jepsen worker 12 - jepsen.util 12072 :invoke :read [4 nil] INFO [2019-12-17 12:37:23,021] jepsen worker 51 - jepsen.util 11031 :ok :write [17 43] INFO [2019-12-17 12:37:23,021] jepsen worker 51 - jepsen.util 11031 :invoke :write [17 44] INFO [2019-12-17 12:37:23,032] jepsen worker 33 - jepsen.util 15963 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:37:23,032] jepsen worker 33 - jepsen.util 15963 :invoke :cas [11 [13 19]] INFO [2019-12-17 12:37:23,032] jepsen worker 65 - jepsen.util 10415 :ok :write [21 39] INFO [2019-12-17 12:37:23,032] jepsen worker 65 - jepsen.util 10415 :invoke :cas [21 [16 28]] INFO [2019-12-17 12:37:23,055] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68579,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:37:23,055] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68579,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:37:23,055] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68582,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:37:23,055] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68582,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:37:23,055] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68583,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:37:23,055] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68583,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:37:23,066] jepsen worker 65 - jepsen.util 10415 :fail :cas [21 [16 28]] :ValueNotSwapFrom INFO [2019-12-17 12:37:23,066] jepsen worker 65 - jepsen.util 10415 :invoke :cas [21 [9 11]] INFO [2019-12-17 12:37:23,074] jepsen worker 31 - jepsen.util 10111 :ok :cas [10 [33 14]] INFO [2019-12-17 12:37:23,074] jepsen worker 31 - jepsen.util 10111 :invoke :read [10 nil] INFO [2019-12-17 12:37:23,096] jepsen worker 65 - jepsen.util 10415 :fail :cas [21 [9 11]] :ValueNotSwapFrom INFO [2019-12-17 12:37:23,096] jepsen worker 65 - jepsen.util 10415 :invoke :read [21 nil] INFO [2019-12-17 12:37:23,103] jepsen worker 70 - jepsen.util 12220 :fail :cas [23 [23 19]] :ValueNotSwapFrom INFO [2019-12-17 12:37:23,103] jepsen worker 70 - jepsen.util 12220 :invoke :write [23 37] INFO [2019-12-17 12:37:23,128] jepsen worker 36 - jepsen.util 12546 :ok :read [12 21] INFO [2019-12-17 12:37:23,128] jepsen worker 36 - jepsen.util 12546 :invoke :read [12 nil] INFO [2019-12-17 12:37:23,137] jepsen worker 65 - jepsen.util 10415 :ok :read [21 39] INFO [2019-12-17 12:37:23,137] jepsen worker 65 - jepsen.util 10415 :invoke :cas [21 [37 5]] INFO [2019-12-17 12:37:23,145] jepsen worker 31 - jepsen.util 10111 :ok :read [10 14] INFO [2019-12-17 12:37:23,146] jepsen worker 31 - jepsen.util 10111 :invoke :cas [10 [48 16]] INFO [2019-12-17 12:37:23,147] jepsen worker 54 - jepsen.util 12204 :info :write [18 47] :RequestCanceledException INFO [2019-12-17 12:37:23,148] jepsen worker 54 - jepsen.util 12294 :invoke :read [18 nil] INFO [2019-12-17 12:37:23,172] jepsen worker 65 - jepsen.util 10415 :fail :cas [21 [37 5]] :ValueNotSwapFrom INFO [2019-12-17 12:37:23,173] jepsen worker 65 - jepsen.util 10415 :invoke :read [21 nil] INFO [2019-12-17 12:37:23,180] jepsen worker 36 - jepsen.util 12546 :ok :read [12 21] INFO [2019-12-17 12:37:23,181] jepsen worker 36 - jepsen.util 12546 :invoke :cas [12 [16 31]] INFO [2019-12-17 12:37:23,186] jepsen worker 65 - jepsen.util 10415 :ok :read [21 39] INFO [2019-12-17 12:37:23,186] jepsen worker 65 - jepsen.util 10415 :invoke :cas [21 [15 28]] INFO [2019-12-17 12:37:23,194] jepsen worker 27 - jepsen.util 10917 :ok :write [9 18] INFO [2019-12-17 12:37:23,194] jepsen worker 27 - jepsen.util 10917 :invoke :cas [9 [4 36]] INFO [2019-12-17 12:37:23,198] jepsen worker 65 - jepsen.util 10415 :fail :cas [21 [15 28]] :ValueNotSwapFrom INFO [2019-12-17 12:37:23,198] jepsen worker 65 - jepsen.util 10415 :invoke :write [21 48] INFO [2019-12-17 12:37:23,208] jepsen worker 31 - jepsen.util 10111 :fail :cas [10 [48 16]] :ValueNotSwapFrom INFO [2019-12-17 12:37:23,208] jepsen worker 31 - jepsen.util 10111 :invoke :write [10 36] INFO [2019-12-17 12:37:23,230] jepsen worker 27 - jepsen.util 10917 :fail :cas [9 [4 36]] :ValueNotSwapFrom INFO [2019-12-17 12:37:23,230] jepsen worker 27 - jepsen.util 10917 :invoke :write [9 48] INFO [2019-12-17 12:37:23,253] jepsen worker 19 - jepsen.util 12169 :ok :write [6 4] INFO [2019-12-17 12:37:23,253] jepsen worker 19 - jepsen.util 12169 :invoke :write [6 24] INFO [2019-12-17 12:37:23,254] jepsen worker 65 - jepsen.util 10415 :ok :write [21 48] INFO [2019-12-17 12:37:23,254] jepsen worker 65 - jepsen.util 10415 :invoke :write [21 24] INFO [2019-12-17 12:37:23,264] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68602,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:37:23,264] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68611,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:37:23,264] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68602,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:37:23,264] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68611,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:37:23,264] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68614,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:37:23,264] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68614,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:37:23,271] jepsen worker 14 - jepsen.util 11534 :fail :cas [4 [32 12]] :ValueNotSwapFrom INFO [2019-12-17 12:37:23,271] jepsen worker 14 - jepsen.util 11534 :invoke :read [4 nil] INFO [2019-12-17 12:37:23,299] jepsen worker 66 - jepsen.util 16086 :info :write [22 22] :RequestCanceledException INFO [2019-12-17 12:37:23,300] jepsen worker 66 - jepsen.util 16176 :invoke :read [22 nil] INFO [2019-12-17 12:37:23,309] jepsen worker 31 - jepsen.util 10111 :ok :write [10 36] INFO [2019-12-17 12:37:23,309] jepsen worker 31 - jepsen.util 10111 :invoke :write [10 4] INFO [2019-12-17 12:37:23,313] jepsen worker 3 - jepsen.util 11163 :ok :read [1 34] INFO [2019-12-17 12:37:23,313] jepsen worker 3 - jepsen.util 11163 :invoke :cas [1 [0 13]] INFO [2019-12-17 12:37:23,318] jepsen worker 17 - jepsen.util 16217 :info :write [5 23] :RequestCanceledException INFO [2019-12-17 12:37:23,319] jepsen worker 17 - jepsen.util 16307 :invoke :write [5 1] INFO [2019-12-17 12:37:23,331] jepsen worker 27 - jepsen.util 10917 :ok :write [9 48] INFO [2019-12-17 12:37:23,332] jepsen worker 27 - jepsen.util 10917 :invoke :cas [9 [8 20]] INFO [2019-12-17 12:37:23,333] jepsen worker 84 - jepsen.util 15924 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:37:23,334] jepsen worker 84 - jepsen.util 15924 :invoke :cas [28 [19 1]] INFO [2019-12-17 12:37:23,360] jepsen worker 14 - jepsen.util 11534 :ok :read [4 11] INFO [2019-12-17 12:37:23,361] jepsen worker 14 - jepsen.util 11534 :invoke :write [4 33] INFO [2019-12-17 12:37:23,365] jepsen worker 62 - jepsen.util 16892 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:37:23,365] jepsen worker 62 - jepsen.util 16892 :invoke :cas [20 [19 30]] INFO [2019-12-17 12:37:23,381] jepsen worker 27 - jepsen.util 10917 :fail :cas [9 [8 20]] :ValueNotSwapFrom INFO [2019-12-17 12:37:23,381] jepsen worker 27 - jepsen.util 10917 :invoke :read [9 nil] INFO [2019-12-17 12:37:23,401] jepsen worker 31 - jepsen.util 10111 :ok :write [10 4] INFO [2019-12-17 12:37:23,401] jepsen worker 31 - jepsen.util 10111 :invoke :read [10 nil] INFO [2019-12-17 12:37:23,409] jepsen worker 27 - jepsen.util 10917 :ok :read [9 48] INFO [2019-12-17 12:37:23,409] jepsen worker 27 - jepsen.util 10917 :invoke :cas [9 [20 2]] INFO [2019-12-17 12:37:23,416] jepsen worker 31 - jepsen.util 10111 :ok :read [10 4] INFO [2019-12-17 12:37:23,416] jepsen worker 31 - jepsen.util 10111 :invoke :write [10 7] INFO [2019-12-17 12:37:23,433] jepsen worker 26 - jepsen.util 16946 :info :write [8 25] :RequestCanceledException INFO [2019-12-17 12:37:23,434] jepsen worker 26 - jepsen.util 17036 :invoke :read [8 nil] INFO [2019-12-17 12:37:23,442] jepsen worker 27 - jepsen.util 10917 :fail :cas [9 [20 2]] :ValueNotSwapFrom INFO [2019-12-17 12:37:23,442] jepsen worker 27 - jepsen.util 10917 :invoke :write [9 34] INFO [2019-12-17 12:37:23,448] jepsen worker 82 - jepsen.util 16552 :info :write [27 41] :RequestCanceledException INFO [2019-12-17 12:37:23,448] jepsen worker 82 - jepsen.util 16642 :invoke :cas [27 [21 48]] INFO [2019-12-17 12:37:23,464] jepsen worker 10 - jepsen.util 11530 :info :write [3 29] :RequestCanceledException INFO [2019-12-17 12:37:23,465] jepsen worker 10 - jepsen.util 11620 :invoke :write [3 27] INFO [2019-12-17 12:37:23,483] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68633,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:37:23,483] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68633,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:37:23,483] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68636,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:37:23,483] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68636,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:37:23,483] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68637,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:37:23,483] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68637,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:37:23,485] jepsen worker 19 - jepsen.util 12169 :ok :write [6 24] INFO [2019-12-17 12:37:23,485] jepsen worker 19 - jepsen.util 12169 :invoke :cas [6 [3 45]] INFO [2019-12-17 12:37:23,499] jepsen worker 19 - jepsen.util 12169 :fail :cas [6 [3 45]] :ValueNotSwapFrom INFO [2019-12-17 12:37:23,499] jepsen worker 19 - jepsen.util 12169 :invoke :cas [6 [29 7]] INFO [2019-12-17 12:37:23,500] jepsen worker 86 - jepsen.util 17006 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:37:23,500] jepsen worker 86 - jepsen.util 17006 :invoke :write [28 44] INFO [2019-12-17 12:37:23,518] jepsen worker 77 - jepsen.util 17087 :info :cas [25 [9 18]] :RequestCanceledException INFO [2019-12-17 12:37:23,519] jepsen worker 77 - jepsen.util 17177 :invoke :write [25 22] INFO [2019-12-17 12:37:23,522] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:37:23,522] jepsen nemesis - couchbase.util Rebalance check count: 539 INFO [2019-12-17 12:37:23,522] jepsen nemesis - couchbase.util Rebalance stuck count: 520 INFO [2019-12-17 12:37:23,527] jepsen worker 19 - jepsen.util 12169 :fail :cas [6 [29 7]] :ValueNotSwapFrom INFO [2019-12-17 12:37:23,527] jepsen worker 19 - jepsen.util 12169 :invoke :write [6 40] INFO [2019-12-17 12:37:23,533] jepsen worker 28 - jepsen.util 10018 :fail :read [9 nil] :RequestTimeoutException INFO [2019-12-17 12:37:23,533] jepsen worker 11 - jepsen.util 11351 :fail :read [3 nil] :RequestTimeoutException INFO [2019-12-17 12:37:23,533] jepsen worker 28 - jepsen.util 10018 :invoke :cas [9 [47 48]] INFO [2019-12-17 12:37:23,533] jepsen worker 11 - jepsen.util 11351 :invoke :write [3 4] INFO [2019-12-17 12:37:23,534] jepsen worker 58 - jepsen.util 16528 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:37:23,534] jepsen worker 58 - jepsen.util 16528 :invoke :read [19 nil] INFO [2019-12-17 12:37:23,536] jepsen worker 53 - jepsen.util 11303 :fail :cas [17 [38 23]] :ValueNotSwapFrom INFO [2019-12-17 12:37:23,536] jepsen worker 53 - jepsen.util 11303 :invoke :read [17 nil] INFO [2019-12-17 12:37:23,547] jepsen worker 67 - jepsen.util 16177 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:37:23,547] jepsen worker 48 - jepsen.util 14988 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:37:23,547] jepsen worker 67 - jepsen.util 16177 :invoke :cas [22 [13 38]] INFO [2019-12-17 12:37:23,547] jepsen worker 48 - jepsen.util 14988 :invoke :read [16 nil] INFO [2019-12-17 12:37:23,564] jepsen worker 2 - jepsen.util 15662 :info :cas [0 [41 6]] :RequestCanceledException INFO [2019-12-17 12:37:23,564] jepsen worker 23 - jepsen.util 16403 :info :write [7 26] :RequestCanceledException INFO [2019-12-17 12:37:23,567] jepsen worker 2 - jepsen.util 15752 :invoke :read [0 nil] INFO [2019-12-17 12:37:23,567] jepsen worker 23 - jepsen.util 16493 :invoke :write [7 5] INFO [2019-12-17 12:37:23,581] jepsen worker 27 - jepsen.util 10917 :ok :write [9 34] INFO [2019-12-17 12:37:23,581] jepsen worker 27 - jepsen.util 10917 :invoke :cas [9 [15 27]] INFO [2019-12-17 12:37:23,584] jepsen worker 12 - jepsen.util 12072 :ok :read [4 11] INFO [2019-12-17 12:37:23,584] jepsen worker 12 - jepsen.util 12072 :invoke :read [4 nil] INFO [2019-12-17 12:37:23,585] jepsen worker 3 - jepsen.util 11163 :fail :cas [1 [0 13]] :ValueNotSwapFrom INFO [2019-12-17 12:37:23,585] jepsen worker 3 - jepsen.util 11163 :invoke :read [1 nil] INFO [2019-12-17 12:37:23,594] jepsen worker 27 - jepsen.util 10917 :fail :cas [9 [15 27]] :ValueNotSwapFrom INFO [2019-12-17 12:37:23,594] jepsen worker 27 - jepsen.util 10917 :invoke :write [9 22] INFO [2019-12-17 12:37:23,599] jepsen worker 25 - jepsen.util 17125 :info :cas [8 [28 47]] :RequestCanceledException INFO [2019-12-17 12:37:23,600] jepsen worker 25 - jepsen.util 17215 :invoke :cas [8 [31 38]] INFO [2019-12-17 12:37:23,618] jepsen worker 43 - jepsen.util 16333 :info :write [14 12] :RequestCanceledException INFO [2019-12-17 12:37:23,618] jepsen worker 43 - jepsen.util 16423 :invoke :read [14 nil] INFO [2019-12-17 12:37:23,633] jepsen worker 85 - jepsen.util 16825 :info :cas [28 [49 41]] :RequestCanceledException INFO [2019-12-17 12:37:23,634] jepsen worker 21 - jepsen.util 15681 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:37:23,634] jepsen worker 21 - jepsen.util 15681 :invoke :read [7 nil] INFO [2019-12-17 12:37:23,634] jepsen worker 85 - jepsen.util 16915 :invoke :cas [28 [26 15]] INFO [2019-12-17 12:37:23,649] jepsen worker 54 - jepsen.util 12294 :ok :read [18 11] INFO [2019-12-17 12:37:23,649] jepsen worker 54 - jepsen.util 12294 :invoke :cas [18 [31 39]] INFO [2019-12-17 12:37:23,657] jepsen worker 27 - jepsen.util 10917 :ok :write [9 22] INFO [2019-12-17 12:37:23,657] jepsen worker 27 - jepsen.util 10917 :invoke :write [9 23] INFO [2019-12-17 12:37:23,664] jepsen worker 13 - jepsen.util 11083 :info :write [4 17] :RequestCanceledException INFO [2019-12-17 12:37:23,664] jepsen worker 7 - jepsen.util 16117 :info :write [2 9] :RequestCanceledException INFO [2019-12-17 12:37:23,666] jepsen worker 13 - jepsen.util 11173 :invoke :cas [4 [3 24]] INFO [2019-12-17 12:37:23,666] jepsen worker 7 - jepsen.util 16207 :invoke :read [2 nil] INFO [2019-12-17 12:37:23,700] jepsen worker 59 - jepsen.util 15449 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:37:23,701] jepsen worker 59 - jepsen.util 15449 :invoke :cas [19 [44 44]] INFO [2019-12-17 12:37:23,702] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68665,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:37:23,703] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68665,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:37:23,703] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68667,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:37:23,703] jepsen worker 65 - jepsen.util 10415 :ok :write [21 24] INFO [2019-12-17 12:37:23,703] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68667,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:37:23,703] jepsen worker 65 - jepsen.util 10415 :invoke :write [21 19] INFO [2019-12-17 12:37:23,719] jepsen worker 71 - jepsen.util 12581 :ok :read [23 32] INFO [2019-12-17 12:37:23,719] jepsen worker 71 - jepsen.util 12581 :invoke :cas [23 [8 5]] INFO [2019-12-17 12:37:23,731] jepsen worker 8 - jepsen.util 15758 :info :cas [2 [8 12]] :RequestCanceledException INFO [2019-12-17 12:37:23,733] jepsen worker 50 - jepsen.util 16340 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:37:23,733] jepsen worker 50 - jepsen.util 16340 :invoke :cas [16 [49 30]] INFO [2019-12-17 12:37:23,734] jepsen worker 8 - jepsen.util 15848 :invoke :write [2 22] INFO [2019-12-17 12:37:23,748] jepsen worker 15 - jepsen.util 14955 :info :write [5 42] :RequestCanceledException INFO [2019-12-17 12:37:23,749] jepsen worker 15 - jepsen.util 15045 :invoke :read [5 nil] INFO [2019-12-17 12:37:23,755] jepsen worker 27 - jepsen.util 10917 :ok :write [9 23] INFO [2019-12-17 12:37:23,755] jepsen worker 27 - jepsen.util 10917 :invoke :cas [9 [15 16]] INFO [2019-12-17 12:37:23,761] jepsen worker 78 - jepsen.util 11598 :info :write [26 0] :RequestCanceledException INFO [2019-12-17 12:37:23,761] jepsen worker 61 - jepsen.util 17431 :info :write [20 33] :RequestCanceledException INFO [2019-12-17 12:37:23,762] jepsen worker 54 - jepsen.util 12294 :fail :cas [18 [31 39]] :ValueNotSwapFrom INFO [2019-12-17 12:37:23,762] jepsen worker 54 - jepsen.util 12294 :invoke :write [18 32] INFO [2019-12-17 12:37:23,763] jepsen worker 78 - jepsen.util 11688 :invoke :cas [26 [37 36]] INFO [2019-12-17 12:37:23,763] jepsen worker 61 - jepsen.util 17521 :invoke :cas [20 [3 33]] INFO [2019-12-17 12:37:23,771] jepsen worker 65 - jepsen.util 10415 :ok :write [21 19] INFO [2019-12-17 12:37:23,771] jepsen worker 65 - jepsen.util 10415 :invoke :read [21 nil] INFO [2019-12-17 12:37:23,778] jepsen worker 27 - jepsen.util 10917 :fail :cas [9 [15 16]] :ValueNotSwapFrom INFO [2019-12-17 12:37:23,778] jepsen worker 27 - jepsen.util 10917 :invoke :read [9 nil] INFO [2019-12-17 12:37:23,791] jepsen worker 13 - jepsen.util 11173 :fail :cas [4 [3 24]] :ValueNotSwapFrom INFO [2019-12-17 12:37:23,792] jepsen worker 13 - jepsen.util 11173 :invoke :write [4 40] INFO [2019-12-17 12:37:23,800] jepsen worker 0 - jepsen.util 16740 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:37:23,800] jepsen worker 73 - jepsen.util 10963 :info :write [24 6] :RequestCanceledException INFO [2019-12-17 12:37:23,801] jepsen worker 0 - jepsen.util 16740 :invoke :write [0 9] INFO [2019-12-17 12:37:23,802] jepsen worker 73 - jepsen.util 11053 :invoke :write [24 30] INFO [2019-12-17 12:37:23,808] jepsen worker 65 - jepsen.util 10415 :ok :read [21 19] INFO [2019-12-17 12:37:23,808] jepsen worker 65 - jepsen.util 10415 :invoke :read [21 nil] INFO [2019-12-17 12:37:23,817] jepsen worker 12 - jepsen.util 12072 :ok :read [4 11] INFO [2019-12-17 12:37:23,817] jepsen worker 12 - jepsen.util 12072 :invoke :cas [4 [4 29]] INFO [2019-12-17 12:37:23,827] jepsen worker 27 - jepsen.util 10917 :ok :read [9 23] INFO [2019-12-17 12:37:23,827] jepsen worker 27 - jepsen.util 10917 :invoke :cas [9 [5 28]] INFO [2019-12-17 12:37:23,832] jepsen worker 71 - jepsen.util 12581 :fail :cas [23 [8 5]] :ValueNotSwapFrom INFO [2019-12-17 12:37:23,832] jepsen worker 71 - jepsen.util 12581 :invoke :write [23 27] INFO [2019-12-17 12:37:23,833] jepsen worker 79 - jepsen.util 11959 :info :write [26 1] :RequestCanceledException INFO [2019-12-17 12:37:23,834] jepsen worker 79 - jepsen.util 12049 :invoke :write [26 49] INFO [2019-12-17 12:37:23,843] jepsen worker 65 - jepsen.util 10415 :ok :read [21 19] INFO [2019-12-17 12:37:23,843] jepsen worker 65 - jepsen.util 10415 :invoke :cas [21 [40 7]] INFO [2019-12-17 12:37:23,852] jepsen worker 65 - jepsen.util 10415 :fail :cas [21 [40 7]] :ValueNotSwapFrom INFO [2019-12-17 12:37:23,853] jepsen worker 65 - jepsen.util 10415 :invoke :read [21 nil] INFO [2019-12-17 12:37:23,869] jepsen worker 65 - jepsen.util 10415 :ok :read [21 19] INFO [2019-12-17 12:37:23,869] jepsen worker 65 - jepsen.util 10415 :invoke :cas [21 [5 15]] INFO [2019-12-17 12:37:23,889] jepsen worker 65 - jepsen.util 10415 :fail :cas [21 [5 15]] :ValueNotSwapFrom INFO [2019-12-17 12:37:23,889] jepsen worker 65 - jepsen.util 10415 :invoke :read [21 nil] INFO [2019-12-17 12:37:23,900] jepsen worker 45 - jepsen.util 11925 :fail :read [15 nil] :RequestTimeoutException INFO [2019-12-17 12:37:23,900] jepsen worker 45 - jepsen.util 11925 :invoke :cas [15 [46 1]] INFO [2019-12-17 12:37:23,907] jepsen worker 65 - jepsen.util 10415 :ok :read [21 19] INFO [2019-12-17 12:37:23,907] jepsen worker 65 - jepsen.util 10415 :invoke :read [21 nil] INFO [2019-12-17 12:37:23,916] jepsen worker 38 - jepsen.util 12188 :info :write [12 11] :RequestCanceledException INFO [2019-12-17 12:37:23,916] jepsen worker 32 - jepsen.util 12182 :info :write [10 47] :RequestCanceledException INFO [2019-12-17 12:37:23,916] jepsen worker 32 - jepsen.util 12272 :invoke :read [10 nil] INFO [2019-12-17 12:37:23,916] jepsen worker 38 - jepsen.util 12278 :invoke :write [12 32] INFO [2019-12-17 12:37:23,921] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68695,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:37:23,921] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68695,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:37:23,921] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68698,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:37:23,921] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68698,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:37:23,921] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68699,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:37:23,921] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68699,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:37:23,924] jepsen worker 45 - jepsen.util 11925 :fail :cas [15 [46 1]] :ValueNotSwapFrom INFO [2019-12-17 12:37:23,924] jepsen worker 45 - jepsen.util 11925 :invoke :read [15 nil] INFO [2019-12-17 12:37:23,934] jepsen worker 87 - jepsen.util 11247 :info :write [29 13] :RequestCanceledException INFO [2019-12-17 12:37:23,938] jepsen worker 65 - jepsen.util 10415 :ok :read [21 19] INFO [2019-12-17 12:37:23,938] jepsen worker 65 - jepsen.util 10415 :invoke :write [21 1] INFO [2019-12-17 12:37:23,939] jepsen worker 87 - jepsen.util 11337 :invoke :read [29 nil] INFO [2019-12-17 12:37:23,943] jepsen worker 53 - jepsen.util 11303 :ok :read [17 43] INFO [2019-12-17 12:37:23,943] jepsen worker 53 - jepsen.util 11303 :invoke :write [17 15] INFO [2019-12-17 12:37:23,964] jepsen worker 46 - jepsen.util 11566 :info :write [15 17] :RequestCanceledException INFO [2019-12-17 12:37:23,964] jepsen worker 19 - jepsen.util 12169 :ok :write [6 40] INFO [2019-12-17 12:37:23,964] jepsen worker 19 - jepsen.util 12169 :invoke :cas [6 [12 18]] INFO [2019-12-17 12:37:23,967] jepsen worker 46 - jepsen.util 11656 :invoke :cas [15 [33 49]] INFO [2019-12-17 12:37:23,967] jepsen worker 45 - jepsen.util 11925 :ok :read [15 23] INFO [2019-12-17 12:37:23,967] jepsen worker 31 - jepsen.util 10111 :ok :write [10 7] INFO [2019-12-17 12:37:23,967] jepsen worker 45 - jepsen.util 11925 :invoke :write [15 8] INFO [2019-12-17 12:37:23,967] jepsen worker 31 - jepsen.util 10111 :invoke :cas [10 [28 2]] INFO [2019-12-17 12:37:23,991] jepsen worker 3 - jepsen.util 11163 :ok :read [1 34] INFO [2019-12-17 12:37:23,991] jepsen worker 3 - jepsen.util 11163 :invoke :write [1 5] INFO [2019-12-17 12:37:24,017] jepsen worker 74 - jepsen.util 11594 :info :cas [24 [19 8]] :RequestCanceledException INFO [2019-12-17 12:37:24,018] jepsen worker 52 - jepsen.util 11122 :info :write [17 48] :RequestCanceledException INFO [2019-12-17 12:37:24,018] jepsen worker 74 - jepsen.util 11684 :invoke :cas [24 [4 36]] INFO [2019-12-17 12:37:24,022] jepsen worker 52 - jepsen.util 11212 :invoke :write [17 26] INFO [2019-12-17 12:37:24,034] jepsen worker 29 - jepsen.util 10199 :fail :read [9 nil] :RequestTimeoutException INFO [2019-12-17 12:37:24,034] jepsen worker 29 - jepsen.util 10199 :invoke :write [9 11] INFO [2019-12-17 12:37:24,037] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68715,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:37:24,038] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68715,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:37:24,038] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68718,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:37:24,038] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68718,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:37:24,038] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68719,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:37:24,039] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68719,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:37:24,043] jepsen worker 10 - jepsen.util 11620 :ok :write [3 27] INFO [2019-12-17 12:37:24,043] jepsen worker 10 - jepsen.util 11620 :invoke :read [3 nil] INFO [2019-12-17 12:37:24,048] jepsen worker 36 - jepsen.util 12546 :fail :cas [12 [16 31]] :ValueNotSwapFrom INFO [2019-12-17 12:37:24,048] jepsen worker 36 - jepsen.util 12546 :invoke :cas [12 [36 38]] INFO [2019-12-17 12:37:24,074] jepsen worker 73 - jepsen.util 11053 :ok :write [24 30] INFO [2019-12-17 12:37:24,074] jepsen worker 73 - jepsen.util 11053 :invoke :cas [24 [17 28]] INFO [2019-12-17 12:37:24,084] jepsen worker 32 - jepsen.util 12272 :ok :read [10 4] INFO [2019-12-17 12:37:24,084] jepsen worker 32 - jepsen.util 12272 :invoke :write [10 28] INFO [2019-12-17 12:37:24,106] jepsen worker 65 - jepsen.util 10415 :ok :write [21 1] INFO [2019-12-17 12:37:24,106] jepsen worker 65 - jepsen.util 10415 :invoke :cas [21 [45 27]] INFO [2019-12-17 12:37:24,115] jepsen worker 31 - jepsen.util 10111 :fail :cas [10 [28 2]] :ValueNotSwapFrom INFO [2019-12-17 12:37:24,115] jepsen worker 31 - jepsen.util 10111 :invoke :write [10 32] INFO [2019-12-17 12:37:24,132] jepsen worker 56 - jepsen.util 12656 :info :write [18 4] :RequestCanceledException INFO [2019-12-17 12:37:24,137] jepsen worker 56 - jepsen.util 12746 :invoke :cas [18 [1 20]] INFO [2019-12-17 12:37:24,147] jepsen worker 44 - jepsen.util 16784 :info :write [14 3] :RequestCanceledException INFO [2019-12-17 12:37:24,151] jepsen worker 44 - jepsen.util 16874 :invoke :cas [14 [36 27]] INFO [2019-12-17 12:37:24,152] jepsen worker 46 - jepsen.util 11656 :fail :cas [15 [33 49]] :ValueNotSwapFrom INFO [2019-12-17 12:37:24,152] jepsen worker 46 - jepsen.util 11656 :invoke :read [15 nil] INFO [2019-12-17 12:37:24,164] jepsen worker 16 - jepsen.util 18016 :info :cas [5 [14 19]] :RequestCanceledException INFO [2019-12-17 12:37:24,169] jepsen worker 16 - jepsen.util 18106 :invoke :read [5 nil] INFO [2019-12-17 12:37:24,199] jepsen worker 74 - jepsen.util 11684 :fail :cas [24 [4 36]] :ValueNotSwapFrom INFO [2019-12-17 12:37:24,199] jepsen worker 74 - jepsen.util 11684 :invoke :write [24 38] INFO [2019-12-17 12:37:24,201] jepsen worker 19 - jepsen.util 12169 :fail :cas [6 [12 18]] :ValueNotSwapFrom INFO [2019-12-17 12:37:24,201] jepsen worker 19 - jepsen.util 12169 :invoke :write [6 34] INFO [2019-12-17 12:37:24,231] jepsen worker 78 - jepsen.util 11688 :fail :cas [26 [37 36]] :ValueNotSwapFrom INFO [2019-12-17 12:37:24,231] jepsen worker 78 - jepsen.util 11688 :invoke :cas [26 [43 46]] INFO [2019-12-17 12:37:24,233] jepsen worker 18 - jepsen.util 11988 :info :cas [6 [46 0]] :RequestCanceledException INFO [2019-12-17 12:37:24,236] jepsen worker 18 - jepsen.util 12078 :invoke :write [6 39] INFO [2019-12-17 12:37:24,249] jepsen worker 10 - jepsen.util 11620 :ok :read [3 27] INFO [2019-12-17 12:37:24,249] jepsen worker 10 - jepsen.util 11620 :invoke :write [3 37] INFO [2019-12-17 12:37:24,250] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68742,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:37:24,250] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68742,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:37:24,250] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68734,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:37:24,251] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68734,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:37:24,251] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68745,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:37:24,251] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68736,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:37:24,251] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68736,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:37:24,251] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68745,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:37:24,251] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68746,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:37:24,251] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68746,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:37:24,255] jepsen worker 65 - jepsen.util 10415 :fail :cas [21 [45 27]] :ValueNotSwapFrom INFO [2019-12-17 12:37:24,255] jepsen worker 65 - jepsen.util 10415 :invoke :write [21 17] INFO [2019-12-17 12:37:24,258] jepsen worker 46 - jepsen.util 11656 :ok :read [15 23] INFO [2019-12-17 12:37:24,259] jepsen worker 46 - jepsen.util 11656 :invoke :write [15 1] INFO [2019-12-17 12:37:24,300] jepsen worker 35 - jepsen.util 15965 :info :cas [11 [49 10]] :RequestCanceledException INFO [2019-12-17 12:37:24,301] jepsen worker 35 - jepsen.util 16055 :invoke :write [11 40] INFO [2019-12-17 12:37:24,302] jepsen worker 29 - jepsen.util 10199 :ok :write [9 11] INFO [2019-12-17 12:37:24,302] jepsen worker 29 - jepsen.util 10199 :invoke :cas [9 [0 38]] INFO [2019-12-17 12:37:24,309] jepsen worker 45 - jepsen.util 11925 :ok :write [15 8] INFO [2019-12-17 12:37:24,310] jepsen worker 45 - jepsen.util 11925 :invoke :cas [15 [38 30]] INFO [2019-12-17 12:37:24,331] jepsen worker 81 - jepsen.util 15741 :info :write [27 36] :RequestCanceledException INFO [2019-12-17 12:37:24,332] jepsen worker 81 - jepsen.util 15831 :invoke :read [27 nil] INFO [2019-12-17 12:37:24,346] jepsen worker 30 - jepsen.util 10830 :info :cas [10 [16 46]] :RequestCanceledException INFO [2019-12-17 12:37:24,349] jepsen worker 30 - jepsen.util 10920 :invoke :write [10 44] INFO [2019-12-17 12:37:24,362] jepsen worker 87 - jepsen.util 11337 :ok :read [29 36] INFO [2019-12-17 12:37:24,363] jepsen worker 87 - jepsen.util 11337 :invoke :write [29 17] INFO [2019-12-17 12:37:24,364] jepsen worker 68 - jepsen.util 15458 :info :write [22 4] :RequestCanceledException INFO [2019-12-17 12:37:24,368] jepsen worker 68 - jepsen.util 15548 :invoke :cas [22 [3 24]] INFO [2019-12-17 12:37:24,414] jepsen worker 45 - jepsen.util 11925 :fail :cas [15 [38 30]] :ValueNotSwapFrom INFO [2019-12-17 12:37:24,415] jepsen worker 45 - jepsen.util 11925 :invoke :read [15 nil] INFO [2019-12-17 12:37:24,418] jepsen worker 41 - jepsen.util 17411 :info :write [13 0] :RequestCanceledException INFO [2019-12-17 12:37:24,420] jepsen worker 41 - jepsen.util 17501 :invoke :write [13 18] INFO [2019-12-17 12:37:24,448] jepsen worker 65 - jepsen.util 10415 :ok :write [21 17] INFO [2019-12-17 12:37:24,448] jepsen worker 65 - jepsen.util 10415 :invoke :write [21 47] INFO [2019-12-17 12:37:24,471] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68762,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:37:24,471] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68762,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:37:24,472] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68766,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:37:24,472] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68766,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:37:24,480] jepsen worker 45 - jepsen.util 11925 :ok :read [15 8] INFO [2019-12-17 12:37:24,480] jepsen worker 45 - jepsen.util 11925 :invoke :write [15 31] INFO [2019-12-17 12:37:24,500] jepsen worker 47 - jepsen.util 12737 :info :write [15 0] :RequestCanceledException INFO [2019-12-17 12:37:24,512] jepsen worker 47 - jepsen.util 12827 :invoke :read [15 nil] INFO [2019-12-17 12:37:24,518] jepsen worker 80 - jepsen.util 11420 :info :write [26 23] :RequestCanceledException INFO [2019-12-17 12:37:24,518] jepsen worker 75 - jepsen.util 15735 :info :write [25 35] :RequestCanceledException INFO [2019-12-17 12:37:24,522] jepsen worker 80 - jepsen.util 11510 :invoke :write [26 38] INFO [2019-12-17 12:37:24,526] jepsen worker 75 - jepsen.util 15825 :invoke :write [25 42] INFO [2019-12-17 12:37:24,533] jepsen worker 40 - jepsen.util 16780 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:37:24,533] jepsen worker 40 - jepsen.util 16780 :invoke :cas [13 [45 48]] INFO [2019-12-17 12:37:24,537] jepsen worker 65 - jepsen.util 10415 :ok :write [21 47] INFO [2019-12-17 12:37:24,537] jepsen worker 65 - jepsen.util 10415 :invoke :read [21 nil] INFO [2019-12-17 12:37:24,540] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:37:24,540] jepsen nemesis - couchbase.util Rebalance check count: 540 INFO [2019-12-17 12:37:24,540] jepsen nemesis - couchbase.util Rebalance stuck count: 521 INFO [2019-12-17 12:37:24,548] jepsen worker 89 - jepsen.util 11789 :info :write [29 28] :RequestCanceledException INFO [2019-12-17 12:37:24,548] jepsen worker 6 - jepsen.util 16656 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:37:24,548] jepsen worker 42 - jepsen.util 17142 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:37:24,548] jepsen worker 6 - jepsen.util 16656 :invoke :read [2 nil] INFO [2019-12-17 12:37:24,548] jepsen worker 42 - jepsen.util 17142 :invoke :cas [14 [21 39]] INFO [2019-12-17 12:37:24,552] jepsen worker 89 - jepsen.util 11879 :invoke :cas [29 [3 15]] INFO [2019-12-17 12:37:24,555] jepsen worker 18 - jepsen.util 12078 :ok :write [6 39] INFO [2019-12-17 12:37:24,555] jepsen worker 18 - jepsen.util 12078 :invoke :write [6 45] INFO [2019-12-17 12:37:24,556] jepsen worker 31 - jepsen.util 10111 :ok :write [10 32] INFO [2019-12-17 12:37:24,557] jepsen worker 31 - jepsen.util 10111 :invoke :read [10 nil] INFO [2019-12-17 12:37:24,563] jepsen worker 36 - jepsen.util 12546 :fail :cas [12 [36 38]] :ValueNotSwapFrom INFO [2019-12-17 12:37:24,563] jepsen worker 36 - jepsen.util 12546 :invoke :cas [12 [36 30]] INFO [2019-12-17 12:37:24,564] jepsen worker 83 - jepsen.util 16463 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:37:24,564] jepsen worker 24 - jepsen.util 15054 :info :cas [8 [34 4]] :RequestCanceledException INFO [2019-12-17 12:37:24,565] jepsen worker 83 - jepsen.util 16463 :invoke :write [27 7] INFO [2019-12-17 12:37:24,571] jepsen worker 24 - jepsen.util 15144 :invoke :cas [8 [46 19]] INFO [2019-12-17 12:37:24,600] jepsen worker 34 - jepsen.util 16684 :info :write [11 44] :RequestCanceledException INFO [2019-12-17 12:37:24,602] jepsen worker 34 - jepsen.util 16774 :invoke :write [11 42] INFO [2019-12-17 12:37:24,615] jepsen worker 65 - jepsen.util 10415 :ok :read [21 47] INFO [2019-12-17 12:37:24,615] jepsen worker 65 - jepsen.util 10415 :invoke :cas [21 [6 49]] INFO [2019-12-17 12:37:24,618] jepsen worker 22 - jepsen.util 16132 :info :cas [7 [3 20]] :RequestCanceledException INFO [2019-12-17 12:37:24,618] jepsen worker 64 - jepsen.util 8974 :info :cas [21 [14 30]] :RequestCanceledException INFO [2019-12-17 12:37:24,619] jepsen worker 72 - jepsen.util 11952 :info :write [24 30] :RequestCanceledException INFO [2019-12-17 12:37:24,619] jepsen worker 1 - jepsen.util 16291 :info :write [0 9] :RequestCanceledException INFO [2019-12-17 12:37:24,623] jepsen worker 64 - jepsen.util 9064 :invoke :write [21 9] INFO [2019-12-17 12:37:24,623] jepsen worker 22 - jepsen.util 16222 :invoke :write [7 10] INFO [2019-12-17 12:37:24,626] jepsen worker 72 - jepsen.util 12042 :invoke :cas [24 [36 42]] INFO [2019-12-17 12:37:24,627] jepsen worker 1 - jepsen.util 16381 :invoke :write [0 20] INFO [2019-12-17 12:37:24,632] jepsen worker 31 - jepsen.util 10111 :ok :read [10 32] INFO [2019-12-17 12:37:24,632] jepsen worker 31 - jepsen.util 10111 :invoke :read [10 nil] INFO [2019-12-17 12:37:24,633] jepsen worker 49 - jepsen.util 16249 :info :write [16 44] :RequestCanceledException INFO [2019-12-17 12:37:24,633] jepsen worker 39 - jepsen.util 15699 :info :cas [13 [30 7]] :RequestCanceledException INFO [2019-12-17 12:37:24,640] jepsen worker 49 - jepsen.util 16339 :invoke :write [16 33] INFO [2019-12-17 12:37:24,645] jepsen worker 39 - jepsen.util 15789 :invoke :cas [13 [0 45]] INFO [2019-12-17 12:37:24,655] jepsen worker 74 - jepsen.util 11684 :ok :write [24 38] INFO [2019-12-17 12:37:24,655] jepsen worker 74 - jepsen.util 11684 :invoke :write [24 27] INFO [2019-12-17 12:37:24,680] jepsen worker 12 - jepsen.util 12072 :fail :cas [4 [4 29]] :ValueNotSwapFrom INFO [2019-12-17 12:37:24,680] jepsen worker 12 - jepsen.util 12072 :invoke :write [4 33] INFO [2019-12-17 12:37:24,692] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68788,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:37:24,692] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68788,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:37:24,692] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68791,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:37:24,693] jepsen worker 78 - jepsen.util 11688 :fail :cas [26 [43 46]] :ValueNotSwapFrom INFO [2019-12-17 12:37:24,693] jepsen worker 78 - jepsen.util 11688 :invoke :cas [26 [26 49]] INFO [2019-12-17 12:37:24,693] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68791,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:37:24,693] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68792,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:37:24,694] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68792,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:37:24,713] jepsen worker 45 - jepsen.util 11925 :ok :write [15 31] INFO [2019-12-17 12:37:24,713] jepsen worker 45 - jepsen.util 11925 :invoke :write [15 42] INFO [2019-12-17 12:37:24,717] jepsen worker 63 - jepsen.util 10233 :fail :read [21 nil] :RequestTimeoutException INFO [2019-12-17 12:37:24,718] jepsen worker 63 - jepsen.util 10233 :invoke :write [21 24] INFO [2019-12-17 12:37:24,724] jepsen worker 31 - jepsen.util 10111 :ok :read [10 32] INFO [2019-12-17 12:37:24,724] jepsen worker 31 - jepsen.util 10111 :invoke :write [10 29] INFO [2019-12-17 12:37:24,724] jepsen worker 10 - jepsen.util 11620 :ok :write [3 37] INFO [2019-12-17 12:37:24,724] jepsen worker 10 - jepsen.util 11620 :invoke :read [3 nil] INFO [2019-12-17 12:37:24,726] jepsen worker 57 - jepsen.util 16077 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:37:24,726] jepsen worker 57 - jepsen.util 16077 :invoke :read [19 nil] INFO [2019-12-17 12:37:24,728] jepsen worker 76 - jepsen.util 16276 :info :write [25 3] :RequestCanceledException INFO [2019-12-17 12:37:24,736] jepsen worker 76 - jepsen.util 16366 :invoke :cas [25 [3 39]] INFO [2019-12-17 12:37:24,739] jepsen worker 65 - jepsen.util 10415 :fail :cas [21 [6 49]] :ValueNotSwapFrom INFO [2019-12-17 12:37:24,739] jepsen worker 65 - jepsen.util 10415 :invoke :cas [21 [47 43]] INFO [2019-12-17 12:37:24,745] jepsen worker 9 - jepsen.util 12429 :fail :read [3 nil] :RequestTimeoutException INFO [2019-12-17 12:37:24,745] jepsen worker 60 - jepsen.util 17340 :info :cas [20 [38 40]] :RequestCanceledException INFO [2019-12-17 12:37:24,745] jepsen worker 9 - jepsen.util 12429 :invoke :read [3 nil] INFO [2019-12-17 12:37:24,746] jepsen worker 37 - jepsen.util 11377 :info :write [12 27] :RequestCanceledException INFO [2019-12-17 12:37:24,754] jepsen worker 60 - jepsen.util 17430 :invoke :read [20 nil] INFO [2019-12-17 12:37:24,756] jepsen worker 37 - jepsen.util 11467 :invoke :write [12 26] INFO [2019-12-17 12:37:24,792] jepsen worker 72 - jepsen.util 12042 :fail :cas [24 [36 42]] :ValueNotSwapFrom INFO [2019-12-17 12:37:24,797] jepsen worker 72 - jepsen.util 12042 :invoke :cas [24 [20 31]] INFO [2019-12-17 12:37:24,801] jepsen worker 20 - jepsen.util 11360 :info :write [6 26] :RequestCanceledException INFO [2019-12-17 12:37:24,811] jepsen worker 20 - jepsen.util 11450 :invoke :write [6 15] INFO [2019-12-17 12:37:24,833] jepsen worker 4 - jepsen.util 10894 :info :write [1 16] :RequestCanceledException INFO [2019-12-17 12:37:24,842] jepsen worker 4 - jepsen.util 10984 :invoke :write [1 34] INFO [2019-12-17 12:37:24,857] jepsen worker 9 - jepsen.util 12429 :ok :read [3 37] INFO [2019-12-17 12:37:24,883] jepsen worker 9 - jepsen.util 12429 :invoke :write [3 47] INFO [2019-12-17 12:37:24,900] jepsen worker 88 - jepsen.util 11878 :info :write [29 16] :RequestCanceledException INFO [2019-12-17 12:37:24,901] jepsen worker 69 - jepsen.util 12129 :info :write [23 37] :RequestCanceledException INFO [2019-12-17 12:37:24,908] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68812,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:37:24,909] jepsen worker 64 - jepsen.util 9064 :ok :write [21 9] INFO [2019-12-17 12:37:24,928] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68812,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:37:24,928] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68817,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:37:24,928] jepsen worker 64 - jepsen.util 9064 :invoke :cas [21 [22 14]] INFO [2019-12-17 12:37:24,928] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68817,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:37:24,931] jepsen worker 88 - jepsen.util 11968 :invoke :read [29 nil] INFO [2019-12-17 12:37:24,932] jepsen worker 69 - jepsen.util 12219 :invoke :write [23 35] INFO [2019-12-17 12:37:24,958] jepsen worker 36 - jepsen.util 12546 :fail :cas [12 [36 30]] :ValueNotSwapFrom INFO [2019-12-17 12:37:24,958] jepsen worker 36 - jepsen.util 12546 :invoke :read [12 nil] INFO [2019-12-17 12:37:24,982] jepsen worker 63 - jepsen.util 10233 :ok :write [21 24] INFO [2019-12-17 12:37:24,982] jepsen worker 63 - jepsen.util 10233 :invoke :read [21 nil] INFO [2019-12-17 12:37:24,998] jepsen worker 78 - jepsen.util 11688 :fail :cas [26 [26 49]] :ValueNotSwapFrom INFO [2019-12-17 12:37:24,998] jepsen worker 78 - jepsen.util 11688 :invoke :read [26 nil] INFO [2019-12-17 12:37:25,001] jepsen worker 55 - jepsen.util 12385 :info :write [18 48] :RequestCanceledException INFO [2019-12-17 12:37:25,002] jepsen worker 55 - jepsen.util 12475 :invoke :read [18 nil] INFO [2019-12-17 12:37:25,023] jepsen worker 64 - jepsen.util 9064 :fail :cas [21 [22 14]] :ValueNotSwapFrom INFO [2019-12-17 12:37:25,023] jepsen worker 64 - jepsen.util 9064 :invoke :write [21 35] INFO [2019-12-17 12:37:25,027] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68831,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:37:25,027] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68831,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:37:25,027] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68834,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:37:25,027] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68834,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:37:25,033] jepsen worker 5 - jepsen.util 11525 :info :write [1 15] :RequestCanceledException INFO [2019-12-17 12:37:25,037] jepsen worker 5 - jepsen.util 11615 :invoke :read [1 nil] INFO [2019-12-17 12:37:25,047] jepsen worker 51 - jepsen.util 11031 :info :write [17 44] :RequestCanceledException INFO [2019-12-17 12:37:25,049] jepsen worker 51 - jepsen.util 11121 :invoke :cas [17 [40 45]] INFO [2019-12-17 12:37:25,052] jepsen worker 63 - jepsen.util 10233 :ok :read [21 24] INFO [2019-12-17 12:37:25,052] jepsen worker 63 - jepsen.util 10233 :invoke :cas [21 [35 6]] INFO [2019-12-17 12:37:25,078] jepsen worker 74 - jepsen.util 11684 :ok :write [24 27] INFO [2019-12-17 12:37:25,079] jepsen worker 74 - jepsen.util 11684 :invoke :cas [24 [19 16]] INFO [2019-12-17 12:37:25,079] jepsen worker 31 - jepsen.util 10111 :ok :write [10 29] INFO [2019-12-17 12:37:25,079] jepsen worker 31 - jepsen.util 10111 :invoke :write [10 5] INFO [2019-12-17 12:37:25,095] jepsen worker 45 - jepsen.util 11925 :ok :write [15 42] INFO [2019-12-17 12:37:25,095] jepsen worker 45 - jepsen.util 11925 :invoke :cas [15 [15 7]] INFO [2019-12-17 12:37:25,115] jepsen worker 63 - jepsen.util 10233 :fail :cas [21 [35 6]] :ValueNotSwapFrom INFO [2019-12-17 12:37:25,116] jepsen worker 63 - jepsen.util 10233 :invoke :write [21 36] INFO [2019-12-17 12:37:25,133] jepsen worker 33 - jepsen.util 15963 :info :cas [11 [13 19]] :RequestCanceledException INFO [2019-12-17 12:37:25,140] jepsen worker 33 - jepsen.util 16053 :invoke :cas [11 [18 14]] INFO [2019-12-17 12:37:25,161] jepsen worker 74 - jepsen.util 11684 :fail :cas [24 [19 16]] :ValueNotSwapFrom INFO [2019-12-17 12:37:25,161] jepsen worker 74 - jepsen.util 11684 :invoke :cas [24 [15 13]] INFO [2019-12-17 12:37:25,165] jepsen worker 70 - jepsen.util 12220 :info :write [23 37] :RequestCanceledException INFO [2019-12-17 12:37:25,168] jepsen worker 70 - jepsen.util 12310 :invoke :read [23 nil] INFO [2019-12-17 12:37:25,225] jepsen worker 64 - jepsen.util 9064 :ok :write [21 35] INFO [2019-12-17 12:37:25,226] jepsen worker 64 - jepsen.util 9064 :invoke :cas [21 [14 49]] INFO [2019-12-17 12:37:25,232] jepsen worker 45 - jepsen.util 11925 :fail :cas [15 [15 7]] :ValueNotSwapFrom INFO [2019-12-17 12:37:25,232] jepsen worker 45 - jepsen.util 11925 :invoke :write [15 32] INFO [2019-12-17 12:37:25,236] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68853,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:37:25,237] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68853,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:37:25,237] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68856,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:37:25,237] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68856,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:37:25,237] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68857,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:37:25,237] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68847,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:37:25,238] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68857,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:37:25,238] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68847,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:37:25,261] jepsen worker 74 - jepsen.util 11684 :fail :cas [24 [15 13]] :ValueNotSwapFrom INFO [2019-12-17 12:37:25,261] jepsen worker 74 - jepsen.util 11684 :invoke :cas [24 [4 6]] INFO [2019-12-17 12:37:25,263] jepsen worker 9 - jepsen.util 12429 :ok :write [3 47] INFO [2019-12-17 12:37:25,263] jepsen worker 9 - jepsen.util 12429 :invoke :read [3 nil] INFO [2019-12-17 12:37:25,291] jepsen worker 88 - jepsen.util 11968 :ok :read [29 36] INFO [2019-12-17 12:37:25,291] jepsen worker 88 - jepsen.util 11968 :invoke :read [29 nil] INFO [2019-12-17 12:37:25,303] jepsen worker 64 - jepsen.util 9064 :fail :cas [21 [14 49]] :ValueNotSwapFrom INFO [2019-12-17 12:37:25,303] jepsen worker 64 - jepsen.util 9064 :invoke :cas [21 [19 13]] INFO [2019-12-17 12:37:25,329] jepsen worker 63 - jepsen.util 10233 :ok :write [21 36] INFO [2019-12-17 12:37:25,329] jepsen worker 63 - jepsen.util 10233 :invoke :write [21 6] INFO [2019-12-17 12:37:25,365] jepsen worker 74 - jepsen.util 11684 :fail :cas [24 [4 6]] :ValueNotSwapFrom INFO [2019-12-17 12:37:25,365] jepsen worker 74 - jepsen.util 11684 :invoke :write [24 48] INFO [2019-12-17 12:37:25,370] jepsen worker 36 - jepsen.util 12546 :ok :read [12 21] INFO [2019-12-17 12:37:25,370] jepsen worker 36 - jepsen.util 12546 :invoke :read [12 nil] INFO [2019-12-17 12:37:25,384] jepsen worker 64 - jepsen.util 9064 :fail :cas [21 [19 13]] :ValueNotSwapFrom INFO [2019-12-17 12:37:25,384] jepsen worker 64 - jepsen.util 9064 :invoke :read [21 nil] INFO [2019-12-17 12:37:25,400] jepsen worker 66 - jepsen.util 16176 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:37:25,400] jepsen worker 66 - jepsen.util 16176 :invoke :cas [22 [28 27]] INFO [2019-12-17 12:37:25,418] jepsen worker 17 - jepsen.util 16307 :info :write [5 1] :RequestCanceledException INFO [2019-12-17 12:37:25,423] jepsen worker 17 - jepsen.util 16397 :invoke :write [5 7] INFO [2019-12-17 12:37:25,434] jepsen worker 84 - jepsen.util 15924 :info :cas [28 [19 1]] :RequestCanceledException INFO [2019-12-17 12:37:25,436] jepsen worker 78 - jepsen.util 11688 :ok :read [26 48] INFO [2019-12-17 12:37:25,436] jepsen worker 78 - jepsen.util 11688 :invoke :read [26 nil] INFO [2019-12-17 12:37:25,441] jepsen worker 84 - jepsen.util 16014 :invoke :cas [28 [46 34]] INFO [2019-12-17 12:37:25,453] jepsen worker 64 - jepsen.util 9064 :ok :read [21 36] INFO [2019-12-17 12:37:25,454] jepsen worker 64 - jepsen.util 9064 :invoke :read [21 nil] INFO [2019-12-17 12:37:25,457] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68873,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:37:25,457] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68873,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:37:25,457] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68874,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:37:25,457] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68874,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:37:25,464] jepsen worker 62 - jepsen.util 16892 :info :cas [20 [19 30]] :RequestCanceledException INFO [2019-12-17 12:37:25,464] jepsen worker 14 - jepsen.util 11534 :info :write [4 33] :RequestCanceledException INFO [2019-12-17 12:37:25,466] jepsen worker 14 - jepsen.util 11624 :invoke :write [4 26] INFO [2019-12-17 12:37:25,468] jepsen worker 62 - jepsen.util 16982 :invoke :write [20 27] INFO [2019-12-17 12:37:25,505] jepsen worker 63 - jepsen.util 10233 :ok :write [21 6] INFO [2019-12-17 12:37:25,505] jepsen worker 63 - jepsen.util 10233 :invoke :write [21 40] INFO [2019-12-17 12:37:25,535] jepsen worker 26 - jepsen.util 17036 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:37:25,539] jepsen worker 26 - jepsen.util 17036 :invoke :read [8 nil] INFO [2019-12-17 12:37:25,548] jepsen worker 82 - jepsen.util 16642 :info :cas [27 [21 48]] :RequestCanceledException INFO [2019-12-17 12:37:25,572] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:37:25,574] jepsen nemesis - couchbase.util Rebalance check count: 541 INFO [2019-12-17 12:37:25,575] jepsen nemesis - couchbase.util Rebalance stuck count: 522 INFO [2019-12-17 12:37:25,580] jepsen worker 82 - jepsen.util 16732 :invoke :read [27 nil] INFO [2019-12-17 12:37:25,601] jepsen worker 86 - jepsen.util 17006 :info :write [28 44] :RequestCanceledException INFO [2019-12-17 12:37:25,604] jepsen worker 2 - jepsen.util 15752 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:37:25,604] jepsen worker 2 - jepsen.util 15752 :invoke :write [0 48] INFO [2019-12-17 12:37:25,616] jepsen worker 86 - jepsen.util 17096 :invoke :read [28 nil] INFO [2019-12-17 12:37:25,618] jepsen worker 23 - jepsen.util 16493 :info :write [7 5] :RequestCanceledException INFO [2019-12-17 12:37:25,634] jepsen worker 77 - jepsen.util 17177 :info :write [25 22] :RequestCanceledException INFO [2019-12-17 12:37:25,634] jepsen worker 28 - jepsen.util 10018 :info :cas [9 [47 48]] :RequestCanceledException INFO [2019-12-17 12:37:25,635] jepsen worker 11 - jepsen.util 11351 :info :write [3 4] :RequestCanceledException INFO [2019-12-17 12:37:25,635] jepsen worker 58 - jepsen.util 16528 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:37:25,635] jepsen worker 58 - jepsen.util 16528 :invoke :write [19 26] INFO [2019-12-17 12:37:25,637] jepsen worker 28 - jepsen.util 10108 :invoke :cas [9 [27 43]] INFO [2019-12-17 12:37:25,640] jepsen worker 25 - jepsen.util 17215 :info :cas [8 [31 38]] :RequestCanceledException INFO [2019-12-17 12:37:25,644] jepsen worker 23 - jepsen.util 16583 :invoke :cas [7 [49 31]] INFO [2019-12-17 12:37:25,648] jepsen worker 48 - jepsen.util 14988 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:37:25,648] jepsen worker 43 - jepsen.util 16423 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:37:25,649] jepsen worker 48 - jepsen.util 14988 :invoke :read [16 nil] INFO [2019-12-17 12:37:25,649] jepsen worker 43 - jepsen.util 16423 :invoke :cas [14 [34 9]] INFO [2019-12-17 12:37:25,650] jepsen worker 67 - jepsen.util 16177 :info :cas [22 [13 38]] :RequestCanceledException INFO [2019-12-17 12:37:25,666] jepsen worker 85 - jepsen.util 16915 :info :cas [28 [26 15]] :RequestCanceledException INFO [2019-12-17 12:37:25,679] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68890,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:37:25,686] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68890,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:37:25,687] jepsen worker 77 - jepsen.util 17267 :invoke :write [25 13] INFO [2019-12-17 12:37:25,697] jepsen worker 11 - jepsen.util 11441 :invoke :read [3 nil] INFO [2019-12-17 12:37:25,702] jepsen worker 7 - jepsen.util 16207 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:37:25,702] jepsen worker 7 - jepsen.util 16207 :invoke :write [2 25] INFO [2019-12-17 12:37:25,704] jepsen worker 25 - jepsen.util 17305 :invoke :write [8 38] INFO [2019-12-17 12:37:25,712] jepsen worker 67 - jepsen.util 16267 :invoke :cas [22 [0 34]] INFO [2019-12-17 12:37:25,727] jepsen worker 85 - jepsen.util 17005 :invoke :cas [28 [27 47]] INFO [2019-12-17 12:37:25,734] jepsen worker 21 - jepsen.util 15681 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:37:25,734] jepsen worker 21 - jepsen.util 15681 :invoke :read [7 nil] INFO [2019-12-17 12:37:25,745] jepsen worker 45 - jepsen.util 11925 :ok :write [15 32] INFO [2019-12-17 12:37:25,745] jepsen worker 74 - jepsen.util 11684 :ok :write [24 48] INFO [2019-12-17 12:37:25,745] jepsen worker 45 - jepsen.util 11925 :invoke :read [15 nil] INFO [2019-12-17 12:37:25,746] jepsen worker 31 - jepsen.util 10111 :ok :write [10 5] INFO [2019-12-17 12:37:25,746] jepsen worker 74 - jepsen.util 11684 :invoke :write [24 17] INFO [2019-12-17 12:37:25,750] jepsen worker 31 - jepsen.util 10111 :invoke :write [10 6] INFO [2019-12-17 12:37:25,763] jepsen worker 64 - jepsen.util 9064 :ok :read [21 36] INFO [2019-12-17 12:37:25,764] jepsen worker 64 - jepsen.util 9064 :invoke :cas [21 [22 42]] INFO [2019-12-17 12:37:25,800] jepsen worker 59 - jepsen.util 15449 :info :cas [19 [44 44]] :RequestCanceledException INFO [2019-12-17 12:37:25,822] jepsen worker 8 - jepsen.util 15848 :info :write [2 22] :RequestCanceledException INFO [2019-12-17 12:37:25,834] jepsen worker 54 - jepsen.util 12294 :info :write [18 32] :RequestCanceledException INFO [2019-12-17 12:37:25,835] jepsen worker 13 - jepsen.util 11173 :info :write [4 40] :RequestCanceledException INFO [2019-12-17 12:37:25,839] jepsen worker 50 - jepsen.util 16340 :info :cas [16 [49 30]] :RequestCanceledException INFO [2019-12-17 12:37:25,841] jepsen worker 15 - jepsen.util 15045 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:37:25,842] jepsen worker 15 - jepsen.util 15045 :invoke :cas [5 [44 33]] INFO [2019-12-17 12:37:25,850] jepsen worker 59 - jepsen.util 15539 :invoke :cas [19 [13 45]] INFO [2019-12-17 12:37:25,856] jepsen worker 8 - jepsen.util 15938 :invoke :write [2 30] INFO [2019-12-17 12:37:25,858] jepsen worker 50 - jepsen.util 16430 :invoke :read [16 nil] INFO [2019-12-17 12:37:25,865] jepsen worker 61 - jepsen.util 17521 :info :cas [20 [3 33]] :RequestCanceledException INFO [2019-12-17 12:37:25,865] jepsen worker 71 - jepsen.util 12581 :info :write [23 27] :RequestCanceledException INFO [2019-12-17 12:37:25,867] jepsen worker 54 - jepsen.util 12384 :invoke :cas [18 [40 16]] INFO [2019-12-17 12:37:25,869] jepsen worker 71 - jepsen.util 12671 :invoke :read [23 nil] INFO [2019-12-17 12:37:25,876] jepsen worker 13 - jepsen.util 11263 :invoke :write [4 9] INFO [2019-12-17 12:37:25,900] jepsen worker 79 - jepsen.util 12049 :info :write [26 49] :RequestCanceledException INFO [2019-12-17 12:37:25,900] jepsen worker 0 - jepsen.util 16740 :info :write [0 9] :RequestCanceledException INFO [2019-12-17 12:37:25,902] jepsen worker 61 - jepsen.util 17611 :invoke :write [20 8] INFO [2019-12-17 12:37:25,912] jepsen worker 79 - jepsen.util 12139 :invoke :write [26 13] INFO [2019-12-17 12:37:25,914] jepsen worker 0 - jepsen.util 16830 :invoke :read [0 nil] INFO [2019-12-17 12:37:25,925] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68908,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:37:25,933] jepsen worker 27 - jepsen.util 10917 :info :cas [9 [5 28]] :RequestCanceledException INFO [2019-12-17 12:37:25,935] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68908,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:37:25,937] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68910,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:37:25,944] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68910,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:37:25,953] jepsen worker 55 - jepsen.util 12475 :ok :read [18 11] INFO [2019-12-17 12:37:25,953] jepsen worker 55 - jepsen.util 12475 :invoke :cas [18 [5 2]] INFO [2019-12-17 12:37:25,994] jepsen worker 27 - jepsen.util 11007 :invoke :cas [9 [47 26]] INFO [2019-12-17 12:37:26,019] jepsen worker 53 - jepsen.util 11303 :info :write [17 15] :RequestCanceledException INFO [2019-12-17 12:37:26,022] jepsen worker 3 - jepsen.util 11163 :info :write [1 5] :RequestCanceledException INFO [2019-12-17 12:37:26,022] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68924,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:37:26,029] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68924,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:37:26,033] jepsen worker 38 - jepsen.util 12278 :info :write [12 32] :RequestCanceledException INFO [2019-12-17 12:37:26,035] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68925,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:37:26,039] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68925,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:37:26,090] jepsen worker 63 - jepsen.util 10233 :ok :write [21 40] INFO [2019-12-17 12:37:26,090] jepsen worker 63 - jepsen.util 10233 :invoke :write [21 25] INFO [2019-12-17 12:37:26,102] jepsen worker 52 - jepsen.util 11212 :info :write [17 26] :RequestCanceledException INFO [2019-12-17 12:37:26,118] jepsen worker 32 - jepsen.util 12272 :info :write [10 28] :RequestCanceledException INFO [2019-12-17 12:37:26,135] jepsen worker 73 - jepsen.util 11053 :info :cas [24 [17 28]] :RequestCanceledException INFO [2019-12-17 12:37:26,137] jepsen worker 73 - jepsen.util 11143 :invoke :write [24 48] INFO [2019-12-17 12:37:26,138] jepsen worker 3 - jepsen.util 11253 :invoke :read [1 nil] INFO [2019-12-17 12:37:26,138] jepsen worker 38 - jepsen.util 12368 :invoke :read [12 nil] INFO [2019-12-17 12:37:26,177] jepsen worker 53 - jepsen.util 11393 :invoke :cas [17 [22 1]] INFO [2019-12-17 12:37:26,185] jepsen worker 52 - jepsen.util 11302 :invoke :read [17 nil] INFO [2019-12-17 12:37:26,187] jepsen worker 32 - jepsen.util 12362 :invoke :write [10 31] INFO [2019-12-17 12:37:26,194] jepsen worker 69 - jepsen.util 12219 :ok :write [23 35] INFO [2019-12-17 12:37:26,194] jepsen worker 69 - jepsen.util 12219 :invoke :write [23 29] INFO [2019-12-17 12:37:26,220] jepsen worker 56 - jepsen.util 12746 :info :cas [18 [1 20]] :RequestCanceledException INFO [2019-12-17 12:37:26,225] jepsen worker 44 - jepsen.util 16874 :info :cas [14 [36 27]] :RequestCanceledException INFO [2019-12-17 12:37:26,233] jepsen worker 56 - jepsen.util 12836 :invoke :write [18 44] INFO [2019-12-17 12:37:26,245] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68942,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:37:26,250] jepsen worker 16 - jepsen.util 18106 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:37:26,251] jepsen worker 16 - jepsen.util 18106 :invoke :read [5 nil] INFO [2019-12-17 12:37:26,253] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68942,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:37:26,253] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68946,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:37:26,254] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68946,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:37:26,259] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68932,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:37:26,265] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68932,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:37:26,270] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68933,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:37:26,277] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68933,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:37:26,285] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68934,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:37:26,294] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68934,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:37:26,319] jepsen worker 19 - jepsen.util 12169 :info :write [6 34] :RequestCanceledException INFO [2019-12-17 12:37:26,331] jepsen worker 44 - jepsen.util 16964 :invoke :cas [14 [49 25]] INFO [2019-12-17 12:37:26,333] jepsen worker 29 - jepsen.util 10199 :info :cas [9 [0 38]] :RequestCanceledException INFO [2019-12-17 12:37:26,334] jepsen worker 19 - jepsen.util 12259 :invoke :write [6 15] INFO [2019-12-17 12:37:26,335] jepsen worker 35 - jepsen.util 16055 :info :write [11 40] :RequestCanceledException INFO [2019-12-17 12:37:26,364] jepsen worker 46 - jepsen.util 11656 :info :write [15 1] :RequestCanceledException INFO [2019-12-17 12:37:26,378] jepsen worker 4 - jepsen.util 10984 :ok :write [1 34] INFO [2019-12-17 12:37:26,379] jepsen worker 4 - jepsen.util 10984 :invoke :cas [1 [21 46]] INFO [2019-12-17 12:37:26,390] jepsen worker 35 - jepsen.util 16145 :invoke :read [11 nil] INFO [2019-12-17 12:37:26,401] jepsen worker 81 - jepsen.util 15831 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:37:26,401] jepsen worker 81 - jepsen.util 15831 :invoke :cas [27 [23 15]] INFO [2019-12-17 12:37:26,409] jepsen worker 29 - jepsen.util 10289 :invoke :cas [9 [44 37]] INFO [2019-12-17 12:37:26,413] jepsen worker 46 - jepsen.util 11746 :invoke :read [15 nil] INFO [2019-12-17 12:37:26,419] jepsen worker 30 - jepsen.util 10920 :info :write [10 44] :RequestCanceledException INFO [2019-12-17 12:37:26,434] jepsen worker 68 - jepsen.util 15548 :info :cas [22 [3 24]] :RequestCanceledException INFO [2019-12-17 12:37:26,437] jepsen worker 30 - jepsen.util 11010 :invoke :cas [10 [25 40]] INFO [2019-12-17 12:37:26,448] jepsen worker 87 - jepsen.util 11337 :info :write [29 17] :RequestCanceledException INFO [2019-12-17 12:37:26,465] jepsen worker 87 - jepsen.util 11427 :invoke :read [29 nil] INFO [2019-12-17 12:37:26,481] jepsen worker 68 - jepsen.util 15638 :invoke :write [22 31] INFO [2019-12-17 12:37:26,513] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68961,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:37:26,515] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68961,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:37:26,515] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68964,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:37:26,518] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68964,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:37:26,550] jepsen worker 41 - jepsen.util 17501 :info :write [13 18] :RequestCanceledException INFO [2019-12-17 12:37:26,563] jepsen worker 74 - jepsen.util 11684 :ok :write [24 17] INFO [2019-12-17 12:37:26,568] jepsen worker 74 - jepsen.util 11684 :invoke :cas [24 [17 29]] INFO [2019-12-17 12:37:26,570] jepsen worker 47 - jepsen.util 12827 :fail :read [15 nil] :RequestTimeoutException INFO [2019-12-17 12:37:26,570] jepsen worker 47 - jepsen.util 12827 :invoke :write [15 32] INFO [2019-12-17 12:37:26,626] jepsen worker 75 - jepsen.util 15825 :info :write [25 42] :RequestCanceledException INFO [2019-12-17 12:37:26,651] jepsen worker 80 - jepsen.util 11510 :info :write [26 38] :RequestCanceledException INFO [2019-12-17 12:37:26,651] jepsen worker 40 - jepsen.util 16780 :info :cas [13 [45 48]] :RequestCanceledException INFO [2019-12-17 12:37:26,660] jepsen worker 14 - jepsen.util 11624 :ok :write [4 26] INFO [2019-12-17 12:37:26,661] jepsen worker 6 - jepsen.util 16656 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:37:26,662] jepsen worker 6 - jepsen.util 16656 :invoke :write [2 1] INFO [2019-12-17 12:37:26,663] jepsen worker 14 - jepsen.util 11624 :invoke :read [4 nil] INFO [2019-12-17 12:37:26,673] jepsen worker 24 - jepsen.util 15144 :info :cas [8 [46 19]] :RequestCanceledException INFO [2019-12-17 12:37:26,686] jepsen worker 18 - jepsen.util 12078 :info :write [6 45] :RequestCanceledException INFO [2019-12-17 12:37:26,687] jepsen worker 42 - jepsen.util 17142 :info :cas [14 [21 39]] :RequestCanceledException INFO [2019-12-17 12:37:26,692] jepsen worker 34 - jepsen.util 16774 :info :write [11 42] :RequestCanceledException INFO [2019-12-17 12:37:26,692] jepsen worker 83 - jepsen.util 16463 :info :write [27 7] :RequestCanceledException INFO [2019-12-17 12:37:26,715] jepsen worker 89 - jepsen.util 11879 :fail :cas [29 [3 15]] :ValueNotSwapFrom INFO [2019-12-17 12:37:26,718] jepsen worker 89 - jepsen.util 11879 :invoke :read [29 nil] INFO [2019-12-17 12:37:26,730] jepsen worker 41 - jepsen.util 17591 :invoke :write [13 6] INFO [2019-12-17 12:37:26,730] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68970,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:37:26,730] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:37:26,735] jepsen worker 1 - jepsen.util 16381 :info :write [0 20] :RequestCanceledException INFO [2019-12-17 12:37:26,735] jepsen nemesis - couchbase.util Rebalance check count: 542 INFO [2019-12-17 12:37:26,736] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68970,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:37:26,736] jepsen worker 49 - jepsen.util 16339 :info :write [16 33] :RequestCanceledException INFO [2019-12-17 12:37:26,736] jepsen nemesis - couchbase.util Rebalance stuck count: 523 INFO [2019-12-17 12:37:26,736] jepsen worker 22 - jepsen.util 16222 :info :write [7 10] :RequestCanceledException INFO [2019-12-17 12:37:26,739] jepsen worker 12 - jepsen.util 12072 :info :write [4 33] :RequestCanceledException INFO [2019-12-17 12:37:26,739] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68975,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:37:26,771] jepsen worker 10 - jepsen.util 11620 :fail :read [3 nil] :RequestTimeoutException INFO [2019-12-17 12:37:26,772] jepsen worker 10 - jepsen.util 11620 :invoke :cas [3 [3 36]] INFO [2019-12-17 12:37:26,773] jepsen worker 39 - jepsen.util 15789 :info :cas [13 [0 45]] :RequestCanceledException INFO [2019-12-17 12:37:26,779] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68975,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:37:26,801] jepsen worker 78 - jepsen.util 11688 :ok :read [26 48] INFO [2019-12-17 12:37:26,802] jepsen worker 78 - jepsen.util 11688 :invoke :write [26 25] INFO [2019-12-17 12:37:26,802] jepsen worker 76 - jepsen.util 16366 :info :cas [25 [3 39]] :RequestCanceledException INFO [2019-12-17 12:37:26,818] jepsen worker 75 - jepsen.util 15915 :invoke :write [25 42] INFO [2019-12-17 12:37:26,834] jepsen worker 42 - jepsen.util 17232 :invoke :cas [14 [23 49]] INFO [2019-12-17 12:37:26,835] jepsen worker 40 - jepsen.util 16870 :invoke :write [13 20] INFO [2019-12-17 12:37:26,835] jepsen worker 57 - jepsen.util 16077 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:37:26,836] jepsen worker 57 - jepsen.util 16077 :invoke :read [19 nil] INFO [2019-12-17 12:37:26,839] jepsen worker 18 - jepsen.util 12168 :invoke :cas [6 [24 7]] INFO [2019-12-17 12:37:26,841] jepsen worker 60 - jepsen.util 17430 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:37:26,851] jepsen worker 60 - jepsen.util 17430 :invoke :write [20 39] INFO [2019-12-17 12:37:26,889] jepsen worker 83 - jepsen.util 16553 :invoke :read [27 nil] INFO [2019-12-17 12:37:26,891] jepsen worker 24 - jepsen.util 15234 :invoke :write [8 47] INFO [2019-12-17 12:37:26,902] jepsen worker 31 - jepsen.util 10111 :ok :write [10 6] INFO [2019-12-17 12:37:26,904] jepsen worker 31 - jepsen.util 10111 :invoke :read [10 nil] INFO [2019-12-17 12:37:26,904] jepsen worker 37 - jepsen.util 11467 :info :write [12 26] :RequestCanceledException INFO [2019-12-17 12:37:26,906] jepsen worker 80 - jepsen.util 11600 :invoke :write [26 29] INFO [2019-12-17 12:37:26,908] jepsen worker 72 - jepsen.util 12042 :info :cas [24 [20 31]] :RequestCanceledException INFO [2019-12-17 12:37:26,908] jepsen worker 12 - jepsen.util 12162 :invoke :cas [4 [21 27]] INFO [2019-12-17 12:37:26,910] jepsen worker 34 - jepsen.util 16864 :invoke :write [11 24] INFO [2019-12-17 12:37:26,914] jepsen worker 49 - jepsen.util 16429 :invoke :cas [16 [47 15]] INFO [2019-12-17 12:37:26,914] jepsen worker 20 - jepsen.util 11450 :info :write [6 15] :RequestCanceledException INFO [2019-12-17 12:37:26,936] jepsen worker 65 - jepsen.util 10415 :info :cas [21 [47 43]] :RequestCanceledException INFO [2019-12-17 12:37:26,952] jepsen worker 1 - jepsen.util 16471 :invoke :read [0 nil] INFO [2019-12-17 12:37:26,954] jepsen worker 22 - jepsen.util 16312 :invoke :cas [7 [5 12]] INFO [2019-12-17 12:37:26,958] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68985,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:37:26,962] jepsen worker 37 - jepsen.util 11557 :invoke :read [12 nil] INFO [2019-12-17 12:37:26,962] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":68985,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:37:26,976] jepsen worker 39 - jepsen.util 15879 :invoke :cas [13 [26 13]] INFO [2019-12-17 12:37:26,983] jepsen worker 76 - jepsen.util 16456 :invoke :cas [25 [36 22]] INFO [2019-12-17 12:37:27,025] jepsen worker 72 - jepsen.util 12132 :invoke :read [24 nil] INFO [2019-12-17 12:37:27,031] jepsen worker 20 - jepsen.util 11540 :invoke :write [6 28] INFO [2019-12-17 12:37:27,040] jepsen worker 65 - jepsen.util 10505 :invoke :cas [21 [26 2]] INFO [2019-12-17 12:37:27,059] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69000,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:37:27,060] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69000,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:37:27,061] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69002,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:37:27,063] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69002,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:37:27,164] jepsen worker 45 - jepsen.util 11925 :ok :read [15 32] INFO [2019-12-17 12:37:27,166] jepsen worker 45 - jepsen.util 11925 :invoke :read [15 nil] INFO [2019-12-17 12:37:27,191] jepsen worker 28 - jepsen.util 10108 :fail :cas [9 [27 43]] :ValueNotSwapFrom INFO [2019-12-17 12:37:27,192] jepsen worker 28 - jepsen.util 10108 :invoke :write [9 9] INFO [2019-12-17 12:37:27,203] jepsen worker 51 - jepsen.util 11121 :fail :cas [17 [40 45]] :ValueNotSwapFrom INFO [2019-12-17 12:37:27,204] jepsen worker 64 - jepsen.util 9064 :fail :cas [21 [22 42]] :ValueNotSwapFrom INFO [2019-12-17 12:37:27,210] jepsen worker 51 - jepsen.util 11121 :invoke :write [17 19] INFO [2019-12-17 12:37:27,221] jepsen worker 64 - jepsen.util 9064 :invoke :cas [21 [6 15]] INFO [2019-12-17 12:37:27,239] jepsen worker 33 - jepsen.util 16053 :info :cas [11 [18 14]] :RequestCanceledException INFO [2019-12-17 12:37:27,242] jepsen worker 73 - jepsen.util 11143 :ok :write [24 48] INFO [2019-12-17 12:37:27,243] jepsen worker 73 - jepsen.util 11143 :invoke :read [24 nil] INFO [2019-12-17 12:37:27,244] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69008,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:37:27,246] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69008,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:37:27,246] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69010,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:37:27,247] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69010,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:37:27,265] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69017,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:37:27,271] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69017,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:37:27,286] jepsen worker 33 - jepsen.util 16143 :invoke :write [11 47] INFO [2019-12-17 12:37:27,286] jepsen worker 27 - jepsen.util 11007 :fail :cas [9 [47 26]] :ValueNotSwapFrom INFO [2019-12-17 12:37:27,289] jepsen worker 27 - jepsen.util 11007 :invoke :read [9 nil] INFO [2019-12-17 12:37:27,303] jepsen worker 70 - jepsen.util 12310 :ok :read [23 32] INFO [2019-12-17 12:37:27,308] jepsen worker 70 - jepsen.util 12310 :invoke :cas [23 [47 7]] INFO [2019-12-17 12:37:27,351] jepsen worker 9 - jepsen.util 12429 :fail :read [3 nil] :RequestTimeoutException INFO [2019-12-17 12:37:27,352] jepsen worker 9 - jepsen.util 12429 :invoke :write [3 43] INFO [2019-12-17 12:37:27,362] jepsen worker 63 - jepsen.util 10233 :ok :write [21 25] INFO [2019-12-17 12:37:27,364] jepsen worker 63 - jepsen.util 10233 :invoke :read [21 nil] INFO [2019-12-17 12:37:27,461] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69038,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:37:27,465] jepsen worker 17 - jepsen.util 16397 :info :write [5 7] :RequestCanceledException INFO [2019-12-17 12:37:27,468] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69038,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:37:27,506] jepsen worker 66 - jepsen.util 16176 :info :cas [22 [28 27]] :RequestCanceledException INFO [2019-12-17 12:37:27,523] jepsen worker 62 - jepsen.util 16982 :info :write [20 27] :RequestCanceledException INFO [2019-12-17 12:37:27,535] jepsen worker 84 - jepsen.util 16014 :info :cas [28 [46 34]] :RequestCanceledException INFO [2019-12-17 12:37:27,556] jepsen worker 17 - jepsen.util 16487 :invoke :cas [5 [38 21]] INFO [2019-12-17 12:37:27,567] jepsen worker 32 - jepsen.util 12362 :ok :write [10 31] INFO [2019-12-17 12:37:27,568] jepsen worker 32 - jepsen.util 12362 :invoke :write [10 33] INFO [2019-12-17 12:37:27,571] jepsen worker 66 - jepsen.util 16266 :invoke :read [22 nil] INFO [2019-12-17 12:37:27,572] jepsen worker 62 - jepsen.util 17072 :invoke :read [20 nil] INFO [2019-12-17 12:37:27,576] jepsen worker 84 - jepsen.util 16104 :invoke :read [28 nil] INFO [2019-12-17 12:37:27,642] jepsen worker 26 - jepsen.util 17036 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:37:27,643] jepsen worker 82 - jepsen.util 16732 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:37:27,643] jepsen worker 26 - jepsen.util 17036 :invoke :read [8 nil] INFO [2019-12-17 12:37:27,643] jepsen worker 82 - jepsen.util 16732 :invoke :read [27 nil] INFO [2019-12-17 12:37:27,648] jepsen worker 86 - jepsen.util 17096 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:37:27,648] jepsen worker 86 - jepsen.util 17096 :invoke :read [28 nil] INFO [2019-12-17 12:37:27,694] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69052,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:37:27,698] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69052,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:37:27,698] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69053,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:37:27,699] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69053,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:37:27,739] jepsen worker 23 - jepsen.util 16583 :info :cas [7 [49 31]] :RequestCanceledException INFO [2019-12-17 12:37:27,753] jepsen worker 48 - jepsen.util 14988 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:37:27,754] jepsen worker 48 - jepsen.util 14988 :invoke :write [16 11] INFO [2019-12-17 12:37:27,757] jepsen worker 43 - jepsen.util 16423 :info :cas [14 [34 9]] :RequestCanceledException INFO [2019-12-17 12:37:27,772] jepsen worker 67 - jepsen.util 16267 :info :cas [22 [0 34]] :RequestCanceledException INFO [2019-12-17 12:37:27,772] jepsen worker 85 - jepsen.util 17005 :info :cas [28 [27 47]] :RequestCanceledException INFO [2019-12-17 12:37:27,811] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:37:27,813] jepsen nemesis - couchbase.util Rebalance check count: 543 INFO [2019-12-17 12:37:27,820] jepsen nemesis - couchbase.util Rebalance stuck count: 524 INFO [2019-12-17 12:37:27,835] jepsen worker 21 - jepsen.util 15681 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:37:27,843] jepsen worker 21 - jepsen.util 15681 :invoke :cas [7 [7 5]] INFO [2019-12-17 12:37:27,844] jepsen worker 23 - jepsen.util 16673 :invoke :cas [7 [48 7]] INFO [2019-12-17 12:37:27,848] jepsen worker 11 - jepsen.util 11441 :ok :read [3 47] INFO [2019-12-17 12:37:27,854] jepsen worker 85 - jepsen.util 17095 :invoke :read [28 nil] INFO [2019-12-17 12:37:27,855] jepsen worker 11 - jepsen.util 11441 :invoke :write [3 36] INFO [2019-12-17 12:37:27,860] jepsen worker 43 - jepsen.util 16513 :invoke :write [14 46] INFO [2019-12-17 12:37:27,860] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69061,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:37:27,867] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69061,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:37:27,870] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69065,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:37:27,873] jepsen worker 67 - jepsen.util 16357 :invoke :cas [22 [27 25]] INFO [2019-12-17 12:37:27,908] jepsen worker 7 - jepsen.util 16207 :info :write [2 25] :RequestCanceledException INFO [2019-12-17 12:37:27,913] jepsen worker 2 - jepsen.util 15752 :info :write [0 48] :RequestCanceledException INFO [2019-12-17 12:37:27,928] jepsen worker 50 - jepsen.util 16430 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:37:27,929] jepsen worker 50 - jepsen.util 16430 :invoke :write [16 39] INFO [2019-12-17 12:37:27,938] jepsen worker 15 - jepsen.util 15045 :info :cas [5 [44 33]] :RequestCanceledException INFO [2019-12-17 12:37:27,938] jepsen worker 59 - jepsen.util 15539 :info :cas [19 [13 45]] :RequestCanceledException INFO [2019-12-17 12:37:27,939] jepsen worker 5 - jepsen.util 11615 :ok :read [1 34] INFO [2019-12-17 12:37:27,940] jepsen worker 52 - jepsen.util 11302 :ok :read [17 15] INFO [2019-12-17 12:37:27,941] jepsen worker 5 - jepsen.util 11615 :invoke :cas [1 [20 23]] INFO [2019-12-17 12:37:27,941] jepsen worker 52 - jepsen.util 11302 :invoke :read [17 nil] INFO [2019-12-17 12:37:27,948] jepsen worker 71 - jepsen.util 12671 :fail :read [23 nil] :RequestTimeoutException INFO [2019-12-17 12:37:27,949] jepsen worker 71 - jepsen.util 12671 :invoke :cas [23 [15 19]] INFO [2019-12-17 12:37:27,953] jepsen worker 88 - jepsen.util 11968 :ok :read [29 36] INFO [2019-12-17 12:37:27,954] jepsen worker 88 - jepsen.util 11968 :invoke :write [29 36] INFO [2019-12-17 12:37:27,988] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69065,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:37:28,004] jepsen worker 79 - jepsen.util 12139 :ok :write [26 13] INFO [2019-12-17 12:37:28,006] jepsen worker 79 - jepsen.util 12139 :invoke :cas [26 [48 37]] INFO [2019-12-17 12:37:28,012] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69067,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:37:28,017] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69067,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:37:28,040] jepsen worker 58 - jepsen.util 16528 :info :write [19 26] :RequestCanceledException INFO [2019-12-17 12:37:28,051] jepsen worker 55 - jepsen.util 12475 :info :cas [18 [5 2]] :RequestCanceledException INFO [2019-12-17 12:37:28,053] jepsen worker 0 - jepsen.util 16830 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:37:28,059] jepsen worker 0 - jepsen.util 16830 :invoke :read [0 nil] INFO [2019-12-17 12:37:28,087] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69077,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:37:28,087] jepsen worker 7 - jepsen.util 16297 :invoke :write [2 48] INFO [2019-12-17 12:37:28,091] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69077,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:37:28,094] jepsen worker 15 - jepsen.util 15135 :invoke :write [5 44] INFO [2019-12-17 12:37:28,112] jepsen worker 2 - jepsen.util 15842 :invoke :cas [0 [1 4]] INFO [2019-12-17 12:37:28,174] jepsen worker 3 - jepsen.util 11253 :fail :read [1 nil] :RequestTimeoutException INFO [2019-12-17 12:37:28,176] jepsen worker 3 - jepsen.util 11253 :invoke :write [1 19] INFO [2019-12-17 12:37:28,176] jepsen worker 59 - jepsen.util 15629 :invoke :cas [19 [16 18]] INFO [2019-12-17 12:37:28,185] jepsen worker 58 - jepsen.util 16618 :invoke :write [19 8] INFO [2019-12-17 12:37:28,211] jepsen worker 77 - jepsen.util 17267 :info :write [25 13] :RequestCanceledException INFO [2019-12-17 12:37:28,213] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69082,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:37:28,228] jepsen worker 55 - jepsen.util 12565 :invoke :read [18 nil] INFO [2019-12-17 12:37:28,246] jepsen worker 25 - jepsen.util 17305 :info :write [8 38] :RequestCanceledException INFO [2019-12-17 12:37:28,248] jepsen worker 38 - jepsen.util 12368 :fail :read [12 nil] :RequestTimeoutException INFO [2019-12-17 12:37:28,254] jepsen worker 38 - jepsen.util 12368 :invoke :cas [12 [44 48]] INFO [2019-12-17 12:37:28,269] jepsen worker 13 - jepsen.util 11263 :info :write [4 9] :RequestCanceledException INFO [2019-12-17 12:37:28,299] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69087,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:37:28,305] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69087,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:37:28,306] jepsen worker 8 - jepsen.util 15938 :info :write [2 30] :RequestCanceledException INFO [2019-12-17 12:37:28,307] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69090,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:37:28,308] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69090,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:37:28,327] jepsen worker 77 - jepsen.util 17357 :invoke :read [25 nil] INFO [2019-12-17 12:37:28,336] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69082,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:37:28,342] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69088,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:37:28,360] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69088,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:37:28,362] jepsen worker 16 - jepsen.util 18106 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:37:28,363] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69094,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:37:28,363] jepsen worker 25 - jepsen.util 17395 :invoke :read [8 nil] INFO [2019-12-17 12:37:28,363] jepsen worker 13 - jepsen.util 11353 :invoke :write [4 8] INFO [2019-12-17 12:37:28,364] jepsen worker 16 - jepsen.util 18106 :invoke :cas [5 [36 20]] INFO [2019-12-17 12:37:28,370] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69094,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:37:28,371] jepsen worker 44 - jepsen.util 16964 :info :cas [14 [49 25]] :RequestCanceledException INFO [2019-12-17 12:37:28,402] jepsen worker 54 - jepsen.util 12384 :fail :cas [18 [40 16]] :ValueNotSwapFrom INFO [2019-12-17 12:37:28,411] jepsen worker 54 - jepsen.util 12384 :invoke :write [18 35] INFO [2019-12-17 12:37:28,416] jepsen worker 69 - jepsen.util 12219 :ok :write [23 29] INFO [2019-12-17 12:37:28,417] jepsen worker 69 - jepsen.util 12219 :invoke :cas [23 [7 49]] INFO [2019-12-17 12:37:28,442] jepsen worker 8 - jepsen.util 16028 :invoke :write [2 5] INFO [2019-12-17 12:37:28,454] jepsen worker 61 - jepsen.util 17611 :info :write [20 8] :RequestCanceledException INFO [2019-12-17 12:37:28,481] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69101,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:37:28,489] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69101,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:37:28,492] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69104,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:37:28,497] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69104,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:37:28,507] jepsen worker 35 - jepsen.util 16145 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:37:28,508] jepsen worker 35 - jepsen.util 16145 :invoke :cas [11 [36 22]] INFO [2019-12-17 12:37:28,509] jepsen worker 81 - jepsen.util 15831 :info :cas [27 [23 15]] :RequestCanceledException INFO [2019-12-17 12:37:28,533] jepsen worker 44 - jepsen.util 17054 :invoke :cas [14 [4 12]] INFO [2019-12-17 12:37:28,538] jepsen worker 61 - jepsen.util 17701 :invoke :read [20 nil] INFO [2019-12-17 12:37:28,545] jepsen worker 81 - jepsen.util 15921 :invoke :write [27 38] INFO [2019-12-17 12:37:28,551] jepsen worker 56 - jepsen.util 12836 :info :write [18 44] :RequestCanceledException INFO [2019-12-17 12:37:28,563] jepsen worker 87 - jepsen.util 11427 :fail :read [29 nil] :RequestTimeoutException INFO [2019-12-17 12:37:28,564] jepsen worker 87 - jepsen.util 11427 :invoke :cas [29 [20 14]] INFO [2019-12-17 12:37:28,565] jepsen worker 56 - jepsen.util 12926 :invoke :cas [18 [4 29]] INFO [2019-12-17 12:37:28,584] jepsen worker 36 - jepsen.util 12546 :ok :read [12 21] INFO [2019-12-17 12:37:28,585] jepsen worker 36 - jepsen.util 12546 :invoke :cas [12 [46 19]] INFO [2019-12-17 12:37:28,631] jepsen worker 10 - jepsen.util 11620 :fail :cas [3 [3 36]] :ValueNotSwapFrom INFO [2019-12-17 12:37:28,631] jepsen worker 10 - jepsen.util 11620 :invoke :write [3 9] INFO [2019-12-17 12:37:28,638] jepsen worker 18 - jepsen.util 12168 :fail :cas [6 [24 7]] :ValueNotSwapFrom INFO [2019-12-17 12:37:28,638] jepsen worker 18 - jepsen.util 12168 :invoke :cas [6 [42 36]] INFO [2019-12-17 12:37:28,650] jepsen worker 46 - jepsen.util 11746 :ok :read [15 32] INFO [2019-12-17 12:37:28,651] jepsen worker 4 - jepsen.util 10984 :fail :cas [1 [21 46]] :ValueNotSwapFrom INFO [2019-12-17 12:37:28,651] jepsen worker 46 - jepsen.util 11746 :invoke :write [15 44] INFO [2019-12-17 12:37:28,651] jepsen worker 4 - jepsen.util 10984 :invoke :cas [1 [41 24]] INFO [2019-12-17 12:37:28,652] jepsen worker 89 - jepsen.util 11879 :ok :read [29 36] INFO [2019-12-17 12:37:28,653] jepsen worker 89 - jepsen.util 11879 :invoke :cas [29 [8 47]] INFO [2019-12-17 12:37:28,655] jepsen worker 30 - jepsen.util 11010 :fail :cas [10 [25 40]] :ValueNotSwapFrom INFO [2019-12-17 12:37:28,656] jepsen worker 30 - jepsen.util 11010 :invoke :read [10 nil] INFO [2019-12-17 12:37:28,697] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69121,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:37:28,698] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69121,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:37:28,698] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69124,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:37:28,698] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69124,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:37:28,699] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69125,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:37:28,701] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69125,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:37:28,725] jepsen worker 37 - jepsen.util 11557 :ok :read [12 26] INFO [2019-12-17 12:37:28,725] jepsen worker 37 - jepsen.util 11557 :invoke :read [12 nil] INFO [2019-12-17 12:37:28,745] jepsen worker 65 - jepsen.util 10505 :fail :cas [21 [26 2]] :ValueNotSwapFrom INFO [2019-12-17 12:37:28,746] jepsen worker 65 - jepsen.util 10505 :invoke :read [21 nil] INFO [2019-12-17 12:37:28,797] jepsen worker 14 - jepsen.util 11624 :ok :read [4 26] INFO [2019-12-17 12:37:28,799] jepsen worker 14 - jepsen.util 11624 :invoke :read [4 nil] INFO [2019-12-17 12:37:28,807] jepsen worker 29 - jepsen.util 10289 :fail :cas [9 [44 37]] :ValueNotSwapFrom INFO [2019-12-17 12:37:28,807] jepsen worker 29 - jepsen.util 10289 :invoke :write [9 24] INFO [2019-12-17 12:37:28,816] jepsen worker 12 - jepsen.util 12162 :fail :cas [4 [21 27]] :ValueNotSwapFrom INFO [2019-12-17 12:37:28,817] jepsen worker 12 - jepsen.util 12162 :invoke :cas [4 [40 25]] INFO [2019-12-17 12:37:28,823] jepsen worker 73 - jepsen.util 11143 :ok :read [24 48] INFO [2019-12-17 12:37:28,823] jepsen worker 73 - jepsen.util 11143 :invoke :write [24 15] INFO [2019-12-17 12:37:28,834] jepsen worker 45 - jepsen.util 11925 :ok :read [15 32] INFO [2019-12-17 12:37:28,834] jepsen worker 45 - jepsen.util 11925 :invoke :read [15 nil] INFO [2019-12-17 12:37:28,854] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:37:28,854] jepsen nemesis - couchbase.util Rebalance check count: 544 INFO [2019-12-17 12:37:28,854] jepsen nemesis - couchbase.util Rebalance stuck count: 525 INFO [2019-12-17 12:37:28,867] jepsen worker 53 - jepsen.util 11393 :fail :cas [17 [22 1]] :ValueNotSwapFrom INFO [2019-12-17 12:37:28,868] jepsen worker 53 - jepsen.util 11393 :invoke :write [17 28] INFO [2019-12-17 12:37:28,868] jepsen worker 63 - jepsen.util 10233 :ok :read [21 25] INFO [2019-12-17 12:37:28,869] jepsen worker 63 - jepsen.util 10233 :invoke :cas [21 [12 46]] INFO [2019-12-17 12:37:28,914] jepsen worker 56 - jepsen.util 12926 :fail :cas [18 [4 29]] :ValueNotSwapFrom INFO [2019-12-17 12:37:28,914] jepsen worker 56 - jepsen.util 12926 :invoke :cas [18 [14 2]] INFO [2019-12-17 12:37:28,917] jepsen worker 72 - jepsen.util 12132 :ok :read [24 48] INFO [2019-12-17 12:37:28,918] jepsen worker 72 - jepsen.util 12132 :invoke :read [24 nil] INFO [2019-12-17 12:37:28,921] jepsen worker 38 - jepsen.util 12368 :fail :cas [12 [44 48]] :ValueNotSwapFrom INFO [2019-12-17 12:37:28,922] jepsen worker 38 - jepsen.util 12368 :invoke :cas [12 [39 23]] INFO [2019-12-17 12:37:28,923] jepsen worker 79 - jepsen.util 12139 :fail :cas [26 [48 37]] :ValueNotSwapFrom INFO [2019-12-17 12:37:28,923] jepsen worker 79 - jepsen.util 12139 :invoke :write [26 37] INFO [2019-12-17 12:37:28,927] jepsen worker 5 - jepsen.util 11615 :fail :cas [1 [20 23]] :ValueNotSwapFrom INFO [2019-12-17 12:37:28,927] jepsen worker 5 - jepsen.util 11615 :invoke :cas [1 [47 44]] INFO [2019-12-17 12:37:28,933] jepsen worker 83 - jepsen.util 16553 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:37:28,933] jepsen worker 57 - jepsen.util 16077 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:37:28,933] jepsen worker 42 - jepsen.util 17232 :info :cas [14 [23 49]] :RequestCanceledException INFO [2019-12-17 12:37:28,933] jepsen worker 57 - jepsen.util 16077 :invoke :read [19 nil] INFO [2019-12-17 12:37:28,933] jepsen worker 83 - jepsen.util 16553 :invoke :write [27 12] INFO [2019-12-17 12:37:28,934] jepsen worker 19 - jepsen.util 12259 :info :write [6 15] :RequestCanceledException INFO [2019-12-17 12:37:28,937] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69143,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:37:28,937] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69143,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:37:28,938] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69147,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:37:28,940] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69147,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:37:28,944] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69148,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:37:28,947] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69148,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:37:28,948] jepsen worker 31 - jepsen.util 10111 :fail :read [10 nil] :RequestTimeoutException INFO [2019-12-17 12:37:28,953] jepsen worker 31 - jepsen.util 10111 :invoke :read [10 nil] INFO [2019-12-17 12:37:28,953] jepsen worker 19 - jepsen.util 12349 :invoke :write [6 20] INFO [2019-12-17 12:37:28,963] jepsen worker 49 - jepsen.util 16429 :info :cas [16 [47 15]] :RequestCanceledException INFO [2019-12-17 12:37:28,967] jepsen worker 42 - jepsen.util 17322 :invoke :read [14 nil] INFO [2019-12-17 12:37:28,993] jepsen worker 49 - jepsen.util 16519 :invoke :write [16 8] INFO [2019-12-17 12:37:28,996] jepsen worker 4 - jepsen.util 10984 :fail :cas [1 [41 24]] :ValueNotSwapFrom INFO [2019-12-17 12:37:29,001] jepsen worker 22 - jepsen.util 16312 :info :cas [7 [5 12]] :RequestCanceledException INFO [2019-12-17 12:37:29,002] jepsen worker 4 - jepsen.util 10984 :invoke :write [1 30] INFO [2019-12-17 12:37:29,025] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69161,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:37:29,026] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69161,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:37:29,027] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69165,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:37:29,028] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69165,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:37:29,035] jepsen worker 39 - jepsen.util 15879 :info :cas [13 [26 13]] :RequestCanceledException INFO [2019-12-17 12:37:29,035] jepsen worker 1 - jepsen.util 16471 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:37:29,035] jepsen worker 1 - jepsen.util 16471 :invoke :read [0 nil] INFO [2019-12-17 12:37:29,058] jepsen worker 76 - jepsen.util 16456 :info :cas [25 [36 22]] :RequestCanceledException INFO [2019-12-17 12:37:29,078] jepsen worker 22 - jepsen.util 16402 :invoke :read [7 nil] INFO [2019-12-17 12:37:29,109] jepsen worker 80 - jepsen.util 11600 :ok :write [26 29] INFO [2019-12-17 12:37:29,110] jepsen worker 80 - jepsen.util 11600 :invoke :cas [26 [5 2]] INFO [2019-12-17 12:37:29,124] jepsen worker 39 - jepsen.util 15969 :invoke :cas [13 [4 36]] INFO [2019-12-17 12:37:29,128] jepsen worker 76 - jepsen.util 16546 :invoke :cas [25 [24 31]] INFO [2019-12-17 12:37:29,170] jepsen worker 47 - jepsen.util 12827 :info :write [15 32] :RequestCanceledException INFO [2019-12-17 12:37:29,204] jepsen worker 47 - jepsen.util 12917 :invoke :read [15 nil] INFO [2019-12-17 12:37:29,226] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69182,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:37:29,227] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69182,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:37:29,235] jepsen worker 68 - jepsen.util 15638 :info :write [22 31] :RequestCanceledException INFO [2019-12-17 12:37:29,246] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69175,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:37:29,247] jepsen worker 6 - jepsen.util 16656 :info :write [2 1] :RequestCanceledException INFO [2019-12-17 12:37:29,248] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69175,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:37:29,255] jepsen worker 68 - jepsen.util 15728 :invoke :cas [22 [16 20]] INFO [2019-12-17 12:37:29,270] jepsen worker 64 - jepsen.util 9064 :info :cas [21 [6 15]] :RequestCanceledException INFO [2019-12-17 12:37:29,286] jepsen worker 9 - jepsen.util 12429 :ok :write [3 43] INFO [2019-12-17 12:37:29,286] jepsen worker 32 - jepsen.util 12362 :ok :write [10 33] INFO [2019-12-17 12:37:29,289] jepsen worker 9 - jepsen.util 12429 :invoke :cas [3 [24 6]] INFO [2019-12-17 12:37:29,289] jepsen worker 32 - jepsen.util 12362 :invoke :cas [10 [27 35]] INFO [2019-12-17 12:37:29,295] jepsen worker 64 - jepsen.util 9154 :invoke :read [21 nil] INFO [2019-12-17 12:37:29,298] jepsen worker 6 - jepsen.util 16746 :invoke :cas [2 [4 41]] INFO [2019-12-17 12:37:29,340] jepsen worker 20 - jepsen.util 11540 :info :write [6 28] :RequestCanceledException INFO [2019-12-17 12:37:29,343] jepsen worker 27 - jepsen.util 11007 :fail :read [9 nil] :RequestTimeoutException INFO [2019-12-17 12:37:29,343] jepsen worker 27 - jepsen.util 11007 :invoke :read [9 nil] INFO [2019-12-17 12:37:29,348] jepsen worker 78 - jepsen.util 11688 :info :write [26 25] :RequestCanceledException INFO [2019-12-17 12:37:29,356] jepsen worker 40 - jepsen.util 16870 :info :write [13 20] :RequestCanceledException INFO [2019-12-17 12:37:29,357] jepsen worker 70 - jepsen.util 12310 :info :cas [23 [47 7]] :RequestCanceledException INFO [2019-12-17 12:37:29,363] jepsen worker 51 - jepsen.util 11121 :ok :write [17 19] INFO [2019-12-17 12:37:29,365] jepsen worker 51 - jepsen.util 11121 :invoke :write [17 8] INFO [2019-12-17 12:37:29,378] jepsen worker 78 - jepsen.util 11778 :invoke :write [26 5] INFO [2019-12-17 12:37:29,390] jepsen worker 70 - jepsen.util 12400 :invoke :read [23 nil] INFO [2019-12-17 12:37:29,399] jepsen worker 20 - jepsen.util 11630 :invoke :read [6 nil] INFO [2019-12-17 12:37:29,414] jepsen worker 41 - jepsen.util 17591 :info :write [13 6] :RequestCanceledException INFO [2019-12-17 12:37:29,414] jepsen worker 24 - jepsen.util 15234 :info :write [8 47] :RequestCanceledException INFO [2019-12-17 12:37:29,431] jepsen worker 75 - jepsen.util 15915 :info :write [25 42] :RequestCanceledException INFO [2019-12-17 12:37:29,432] jepsen worker 40 - jepsen.util 16960 :invoke :cas [13 [19 42]] INFO [2019-12-17 12:37:29,447] jepsen worker 60 - jepsen.util 17430 :info :write [20 39] :RequestCanceledException INFO [2019-12-17 12:37:29,447] jepsen worker 34 - jepsen.util 16864 :info :write [11 24] :RequestCanceledException INFO [2019-12-17 12:37:29,491] jepsen worker 73 - jepsen.util 11143 :ok :write [24 15] INFO [2019-12-17 12:37:29,491] jepsen worker 73 - jepsen.util 11143 :invoke :write [24 29] INFO [2019-12-17 12:37:29,496] jepsen worker 69 - jepsen.util 12219 :fail :cas [23 [7 49]] :ValueNotSwapFrom INFO [2019-12-17 12:37:29,510] jepsen worker 69 - jepsen.util 12219 :invoke :cas [23 [39 17]] INFO [2019-12-17 12:37:29,511] jepsen worker 41 - jepsen.util 17681 :invoke :cas [13 [45 11]] INFO [2019-12-17 12:37:29,514] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69197,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:37:29,514] jepsen worker 34 - jepsen.util 16954 :invoke :cas [11 [33 12]] INFO [2019-12-17 12:37:29,515] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69197,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:37:29,516] jepsen worker 24 - jepsen.util 15324 :invoke :cas [8 [40 39]] INFO [2019-12-17 12:37:29,517] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69202,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:37:29,533] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69202,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:37:29,534] jepsen worker 60 - jepsen.util 17520 :invoke :read [20 nil] INFO [2019-12-17 12:37:29,536] jepsen worker 75 - jepsen.util 16005 :invoke :write [25 48] INFO [2019-12-17 12:37:29,598] jepsen worker 28 - jepsen.util 10108 :ok :write [9 9] INFO [2019-12-17 12:37:29,602] jepsen worker 28 - jepsen.util 10108 :invoke :write [9 36] INFO [2019-12-17 12:37:29,637] jepsen worker 17 - jepsen.util 16487 :info :cas [5 [38 21]] :RequestCanceledException INFO [2019-12-17 12:37:29,637] jepsen worker 84 - jepsen.util 16104 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:37:29,642] jepsen worker 84 - jepsen.util 16104 :invoke :write [28 2] INFO [2019-12-17 12:37:29,649] jepsen worker 66 - jepsen.util 16266 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:37:29,650] jepsen worker 66 - jepsen.util 16266 :invoke :write [22 30] INFO [2019-12-17 12:37:29,665] jepsen worker 62 - jepsen.util 17072 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:37:29,666] jepsen worker 62 - jepsen.util 17072 :invoke :cas [20 [20 31]] INFO [2019-12-17 12:37:29,705] jepsen worker 17 - jepsen.util 16577 :invoke :read [5 nil] INFO [2019-12-17 12:37:29,719] jepsen worker 33 - jepsen.util 16143 :info :write [11 47] :RequestCanceledException INFO [2019-12-17 12:37:29,720] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69211,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:37:29,724] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69211,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:37:29,725] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69215,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:37:29,735] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69215,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:37:29,736] jepsen worker 26 - jepsen.util 17036 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:37:29,736] jepsen worker 82 - jepsen.util 16732 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:37:29,740] jepsen worker 82 - jepsen.util 16732 :invoke :write [27 20] INFO [2019-12-17 12:37:29,741] jepsen worker 26 - jepsen.util 17036 :invoke :cas [8 [42 13]] INFO [2019-12-17 12:37:29,749] jepsen worker 86 - jepsen.util 17096 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:37:29,750] jepsen worker 86 - jepsen.util 17096 :invoke :read [28 nil] INFO [2019-12-17 12:37:29,767] jepsen worker 33 - jepsen.util 16233 :invoke :write [11 47] INFO [2019-12-17 12:37:29,826] jepsen worker 19 - jepsen.util 12349 :ok :write [6 20] INFO [2019-12-17 12:37:29,827] jepsen worker 19 - jepsen.util 12349 :invoke :write [6 45] INFO [2019-12-17 12:37:29,876] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69223,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:37:29,884] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69223,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:37:29,886] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69228,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:37:29,888] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69228,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:37:29,897] jepsen worker 23 - jepsen.util 16673 :info :cas [7 [48 7]] :RequestCanceledException INFO [2019-12-17 12:37:29,901] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:37:29,903] jepsen nemesis - couchbase.util Rebalance check count: 545 INFO [2019-12-17 12:37:29,903] jepsen nemesis - couchbase.util Rebalance stuck count: 526 INFO [2019-12-17 12:37:29,914] jepsen worker 85 - jepsen.util 17095 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:37:29,914] jepsen worker 85 - jepsen.util 17095 :invoke :cas [28 [33 25]] INFO [2019-12-17 12:37:29,918] jepsen worker 23 - jepsen.util 16763 :invoke :write [7 18] INFO [2019-12-17 12:37:29,934] jepsen worker 21 - jepsen.util 15681 :info :cas [7 [7 5]] :RequestCanceledException INFO [2019-12-17 12:37:29,946] jepsen worker 13 - jepsen.util 11353 :ok :write [4 8] INFO [2019-12-17 12:37:29,947] jepsen worker 13 - jepsen.util 11353 :invoke :cas [4 [2 8]] INFO [2019-12-17 12:37:29,948] jepsen worker 67 - jepsen.util 16357 :info :cas [22 [27 25]] :RequestCanceledException INFO [2019-12-17 12:37:29,957] jepsen worker 65 - jepsen.util 10505 :ok :read [21 25] INFO [2019-12-17 12:37:29,958] jepsen worker 65 - jepsen.util 10505 :invoke :write [21 33] INFO [2019-12-17 12:37:29,968] jepsen worker 21 - jepsen.util 15771 :invoke :write [7 14] INFO [2019-12-17 12:37:29,974] jepsen worker 67 - jepsen.util 16447 :invoke :cas [22 [30 32]] INFO [2019-12-17 12:37:29,978] jepsen worker 87 - jepsen.util 11427 :fail :cas [29 [20 14]] :ValueNotSwapFrom INFO [2019-12-17 12:37:29,979] jepsen worker 87 - jepsen.util 11427 :invoke :read [29 nil] INFO [2019-12-17 12:37:29,988] jepsen worker 56 - jepsen.util 12926 :fail :cas [18 [14 2]] :ValueNotSwapFrom INFO [2019-12-17 12:37:29,999] jepsen worker 45 - jepsen.util 11925 :ok :read [15 32] INFO [2019-12-17 12:37:30,010] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69244,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:37:30,016] jepsen worker 3 - jepsen.util 11253 :ok :write [1 19] INFO [2019-12-17 12:37:30,016] jepsen worker 56 - jepsen.util 12926 :invoke :write [18 3] INFO [2019-12-17 12:37:30,016] jepsen worker 3 - jepsen.util 11253 :invoke :read [1 nil] INFO [2019-12-17 12:37:30,016] jepsen worker 45 - jepsen.util 11925 :invoke :write [15 21] INFO [2019-12-17 12:37:30,020] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69244,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:37:30,020] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69245,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:37:30,023] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69245,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:37:30,046] jepsen worker 71 - jepsen.util 12671 :info :cas [23 [15 19]] :RequestCanceledException INFO [2019-12-17 12:37:30,049] jepsen worker 71 - jepsen.util 12761 :invoke :read [23 nil] INFO [2019-12-17 12:37:30,052] jepsen worker 12 - jepsen.util 12162 :fail :cas [4 [40 25]] :ValueNotSwapFrom INFO [2019-12-17 12:37:30,052] jepsen worker 12 - jepsen.util 12162 :invoke :cas [4 [48 13]] INFO [2019-12-17 12:37:30,054] jepsen worker 64 - jepsen.util 9154 :ok :read [21 25] INFO [2019-12-17 12:37:30,054] jepsen worker 64 - jepsen.util 9154 :invoke :read [21 nil] INFO [2019-12-17 12:37:30,058] jepsen worker 27 - jepsen.util 11007 :ok :read [9 9] INFO [2019-12-17 12:37:30,059] jepsen worker 27 - jepsen.util 11007 :invoke :read [9 nil] INFO [2019-12-17 12:37:30,062] jepsen worker 38 - jepsen.util 12368 :fail :cas [12 [39 23]] :ValueNotSwapFrom INFO [2019-12-17 12:37:30,063] jepsen worker 38 - jepsen.util 12368 :invoke :write [12 25] INFO [2019-12-17 12:37:30,065] jepsen worker 52 - jepsen.util 11302 :ok :read [17 19] INFO [2019-12-17 12:37:30,066] jepsen worker 52 - jepsen.util 11302 :invoke :write [17 9] INFO [2019-12-17 12:37:30,066] jepsen worker 9 - jepsen.util 12429 :fail :cas [3 [24 6]] :ValueNotSwapFrom INFO [2019-12-17 12:37:30,066] jepsen worker 9 - jepsen.util 12429 :invoke :read [3 nil] INFO [2019-12-17 12:37:30,071] jepsen worker 70 - jepsen.util 12400 :ok :read [23 29] INFO [2019-12-17 12:37:30,072] jepsen worker 70 - jepsen.util 12400 :invoke :write [23 45] INFO [2019-12-17 12:37:30,081] jepsen worker 32 - jepsen.util 12362 :fail :cas [10 [27 35]] :ValueNotSwapFrom INFO [2019-12-17 12:37:30,082] jepsen worker 32 - jepsen.util 12362 :invoke :write [10 42] INFO [2019-12-17 12:37:30,107] jepsen worker 36 - jepsen.util 12546 :fail :cas [12 [46 19]] :ValueNotSwapFrom INFO [2019-12-17 12:37:30,108] jepsen worker 36 - jepsen.util 12546 :invoke :cas [12 [25 37]] INFO [2019-12-17 12:37:30,110] jepsen worker 14 - jepsen.util 11624 :ok :read [4 8] INFO [2019-12-17 12:37:30,111] jepsen worker 14 - jepsen.util 11624 :invoke :write [4 44] INFO [2019-12-17 12:37:30,117] jepsen worker 0 - jepsen.util 16830 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:37:30,117] jepsen worker 0 - jepsen.util 16830 :invoke :write [0 32] INFO [2019-12-17 12:37:30,123] jepsen worker 47 - jepsen.util 12917 :ok :read [15 32] INFO [2019-12-17 12:37:30,123] jepsen worker 47 - jepsen.util 12917 :invoke :read [15 nil] INFO [2019-12-17 12:37:30,126] jepsen worker 89 - jepsen.util 11879 :fail :cas [29 [8 47]] :ValueNotSwapFrom INFO [2019-12-17 12:37:30,126] jepsen worker 89 - jepsen.util 11879 :invoke :read [29 nil] INFO [2019-12-17 12:37:30,131] jepsen worker 20 - jepsen.util 11630 :ok :read [6 20] INFO [2019-12-17 12:37:30,132] jepsen worker 20 - jepsen.util 11630 :invoke :write [6 40] INFO [2019-12-17 12:37:30,204] jepsen worker 2 - jepsen.util 15842 :info :cas [0 [1 4]] :RequestCanceledException INFO [2019-12-17 12:37:30,215] jepsen worker 59 - jepsen.util 15629 :info :cas [19 [16 18]] :RequestCanceledException INFO [2019-12-17 12:37:30,221] jepsen worker 2 - jepsen.util 15932 :invoke :cas [0 [37 7]] INFO [2019-12-17 12:37:30,221] jepsen worker 59 - jepsen.util 15719 :invoke :read [19 nil] INFO [2019-12-17 12:37:30,236] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69272,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:37:30,236] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69272,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:37:30,239] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69276,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:37:30,243] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69276,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:37:30,254] jepsen worker 64 - jepsen.util 9154 :ok :read [21 25] INFO [2019-12-17 12:37:30,255] jepsen worker 64 - jepsen.util 9154 :invoke :cas [21 [42 47]] INFO [2019-12-17 12:37:30,257] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69265,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:37:30,257] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69265,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:37:30,257] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69266,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:37:30,257] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69266,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:37:30,259] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69267,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:37:30,259] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69267,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:37:30,261] jepsen worker 37 - jepsen.util 11557 :ok :read [12 26] INFO [2019-12-17 12:37:30,262] jepsen worker 37 - jepsen.util 11557 :invoke :cas [12 [10 10]] INFO [2019-12-17 12:37:30,286] jepsen worker 27 - jepsen.util 11007 :ok :read [9 9] INFO [2019-12-17 12:37:30,287] jepsen worker 27 - jepsen.util 11007 :invoke :write [9 10] INFO [2019-12-17 12:37:30,296] jepsen worker 28 - jepsen.util 10108 :ok :write [9 36] INFO [2019-12-17 12:37:30,297] jepsen worker 28 - jepsen.util 10108 :invoke :write [9 26] INFO [2019-12-17 12:37:30,328] jepsen worker 79 - jepsen.util 12139 :ok :write [26 37] INFO [2019-12-17 12:37:30,329] jepsen worker 79 - jepsen.util 12139 :invoke :cas [26 [41 19]] INFO [2019-12-17 12:37:30,347] jepsen worker 55 - jepsen.util 12565 :fail :read [18 nil] :RequestTimeoutException INFO [2019-12-17 12:37:30,348] jepsen worker 55 - jepsen.util 12565 :invoke :cas [18 [44 16]] INFO [2019-12-17 12:37:30,361] jepsen worker 77 - jepsen.util 17357 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:37:30,361] jepsen worker 77 - jepsen.util 17357 :invoke :write [25 10] INFO [2019-12-17 12:37:30,419] jepsen worker 32 - jepsen.util 12362 :ok :write [10 42] INFO [2019-12-17 12:37:30,419] jepsen worker 32 - jepsen.util 12362 :invoke :cas [10 [18 17]] INFO [2019-12-17 12:37:30,420] jepsen worker 73 - jepsen.util 11143 :ok :write [24 29] INFO [2019-12-17 12:37:30,420] jepsen worker 73 - jepsen.util 11143 :invoke :write [24 29] INFO [2019-12-17 12:37:30,433] jepsen worker 25 - jepsen.util 17395 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:37:30,433] jepsen worker 25 - jepsen.util 17395 :invoke :write [8 17] INFO [2019-12-17 12:37:30,441] jepsen worker 56 - jepsen.util 12926 :ok :write [18 3] INFO [2019-12-17 12:37:30,441] jepsen worker 56 - jepsen.util 12926 :invoke :read [18 nil] INFO [2019-12-17 12:37:30,447] jepsen worker 16 - jepsen.util 18106 :info :cas [5 [36 20]] :RequestCanceledException INFO [2019-12-17 12:37:30,467] jepsen worker 16 - jepsen.util 18196 :invoke :cas [5 [40 30]] INFO [2019-12-17 12:37:30,474] jepsen worker 5 - jepsen.util 11615 :fail :cas [1 [47 44]] :ValueNotSwapFrom INFO [2019-12-17 12:37:30,474] jepsen worker 5 - jepsen.util 11615 :invoke :write [1 21] INFO [2019-12-17 12:37:30,512] jepsen worker 28 - jepsen.util 10108 :ok :write [9 26] INFO [2019-12-17 12:37:30,512] jepsen worker 28 - jepsen.util 10108 :invoke :cas [9 [9 3]] INFO [2019-12-17 12:37:30,515] jepsen worker 69 - jepsen.util 12219 :fail :cas [23 [39 17]] :ValueNotSwapFrom INFO [2019-12-17 12:37:30,517] jepsen worker 69 - jepsen.util 12219 :invoke :read [23 nil] INFO [2019-12-17 12:37:30,529] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69288,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:37:30,529] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69288,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:37:30,530] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69292,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:37:30,532] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69292,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:37:30,535] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69294,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:37:30,537] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69294,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:37:30,541] jepsen worker 80 - jepsen.util 11600 :fail :cas [26 [5 2]] :ValueNotSwapFrom INFO [2019-12-17 12:37:30,541] jepsen worker 80 - jepsen.util 11600 :invoke :cas [26 [0 26]] INFO [2019-12-17 12:37:30,542] jepsen worker 27 - jepsen.util 11007 :ok :write [9 10] INFO [2019-12-17 12:37:30,543] jepsen worker 27 - jepsen.util 11007 :invoke :cas [9 [20 33]] INFO [2019-12-17 12:37:30,547] jepsen worker 48 - jepsen.util 14988 :info :write [16 11] :RequestCanceledException INFO [2019-12-17 12:37:30,555] jepsen worker 48 - jepsen.util 15078 :invoke :read [16 nil] INFO [2019-12-17 12:37:30,567] jepsen worker 64 - jepsen.util 9154 :fail :cas [21 [42 47]] :ValueNotSwapFrom INFO [2019-12-17 12:37:30,567] jepsen worker 64 - jepsen.util 9154 :invoke :cas [21 [17 39]] INFO [2019-12-17 12:37:30,602] jepsen worker 35 - jepsen.util 16145 :info :cas [11 [36 22]] :RequestCanceledException INFO [2019-12-17 12:37:30,608] jepsen worker 19 - jepsen.util 12349 :ok :write [6 45] INFO [2019-12-17 12:37:30,608] jepsen worker 19 - jepsen.util 12349 :invoke :cas [6 [33 7]] INFO [2019-12-17 12:37:30,616] jepsen worker 8 - jepsen.util 16028 :info :write [2 5] :RequestCanceledException INFO [2019-12-17 12:37:30,618] jepsen worker 28 - jepsen.util 10108 :fail :cas [9 [9 3]] :ValueNotSwapFrom INFO [2019-12-17 12:37:30,618] jepsen worker 28 - jepsen.util 10108 :invoke :cas [9 [31 41]] INFO [2019-12-17 12:37:30,619] jepsen worker 35 - jepsen.util 16235 :invoke :cas [11 [42 18]] INFO [2019-12-17 12:37:30,635] jepsen worker 44 - jepsen.util 17054 :info :cas [14 [4 12]] :RequestCanceledException INFO [2019-12-17 12:37:30,635] jepsen worker 43 - jepsen.util 16513 :info :write [14 46] :RequestCanceledException INFO [2019-12-17 12:37:30,640] jepsen worker 8 - jepsen.util 16118 :invoke :write [2 12] INFO [2019-12-17 12:37:30,647] jepsen worker 61 - jepsen.util 17701 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:37:30,649] jepsen worker 61 - jepsen.util 17701 :invoke :write [20 11] INFO [2019-12-17 12:37:30,649] jepsen worker 44 - jepsen.util 17144 :invoke :read [14 nil] INFO [2019-12-17 12:37:30,651] jepsen worker 43 - jepsen.util 16603 :invoke :write [14 2] INFO [2019-12-17 12:37:30,658] jepsen worker 81 - jepsen.util 15921 :info :write [27 38] :RequestCanceledException INFO [2019-12-17 12:37:30,660] jepsen worker 18 - jepsen.util 12168 :info :cas [6 [42 36]] :RequestCanceledException INFO [2019-12-17 12:37:30,661] jepsen worker 27 - jepsen.util 11007 :fail :cas [9 [20 33]] :ValueNotSwapFrom INFO [2019-12-17 12:37:30,662] jepsen worker 27 - jepsen.util 11007 :invoke :read [9 nil] INFO [2019-12-17 12:37:30,667] jepsen worker 81 - jepsen.util 16011 :invoke :cas [27 [10 12]] INFO [2019-12-17 12:37:30,671] jepsen worker 18 - jepsen.util 12258 :invoke :write [6 44] INFO [2019-12-17 12:37:30,674] jepsen worker 73 - jepsen.util 11143 :ok :write [24 29] INFO [2019-12-17 12:37:30,674] jepsen worker 73 - jepsen.util 11143 :invoke :write [24 23] INFO [2019-12-17 12:37:30,684] jepsen worker 64 - jepsen.util 9154 :fail :cas [21 [17 39]] :ValueNotSwapFrom INFO [2019-12-17 12:37:30,685] jepsen worker 64 - jepsen.util 9154 :invoke :read [21 nil] INFO [2019-12-17 12:37:30,706] jepsen worker 65 - jepsen.util 10505 :ok :write [21 33] INFO [2019-12-17 12:37:30,707] jepsen worker 65 - jepsen.util 10505 :invoke :write [21 5] INFO [2019-12-17 12:37:30,718] jepsen worker 11 - jepsen.util 11441 :info :write [3 36] :RequestCanceledException INFO [2019-12-17 12:37:30,725] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69309,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:37:30,725] jepsen worker 50 - jepsen.util 16430 :info :write [16 39] :RequestCanceledException INFO [2019-12-17 12:37:30,725] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69309,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:37:30,725] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69314,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:37:30,729] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69314,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:37:30,730] jepsen worker 15 - jepsen.util 15135 :info :write [5 44] :RequestCanceledException INFO [2019-12-17 12:37:30,730] jepsen worker 58 - jepsen.util 16618 :info :write [19 8] :RequestCanceledException INFO [2019-12-17 12:37:30,734] jepsen worker 11 - jepsen.util 11531 :invoke :cas [3 [33 14]] INFO [2019-12-17 12:37:30,742] jepsen worker 15 - jepsen.util 15225 :invoke :write [5 18] INFO [2019-12-17 12:37:30,748] jepsen worker 58 - jepsen.util 16708 :invoke :cas [19 [45 33]] INFO [2019-12-17 12:37:30,750] jepsen worker 88 - jepsen.util 11968 :info :write [29 36] :RequestCanceledException INFO [2019-12-17 12:37:30,750] jepsen worker 30 - jepsen.util 11010 :fail :read [10 nil] :RequestTimeoutException INFO [2019-12-17 12:37:30,750] jepsen worker 30 - jepsen.util 11010 :invoke :read [10 nil] INFO [2019-12-17 12:37:30,754] jepsen worker 50 - jepsen.util 16520 :invoke :read [16 nil] INFO [2019-12-17 12:37:30,755] jepsen worker 69 - jepsen.util 12219 :ok :read [23 29] INFO [2019-12-17 12:37:30,756] jepsen worker 69 - jepsen.util 12219 :invoke :read [23 nil] INFO [2019-12-17 12:37:30,763] jepsen worker 7 - jepsen.util 16297 :info :write [2 48] :RequestCanceledException INFO [2019-12-17 12:37:30,765] jepsen worker 88 - jepsen.util 12058 :invoke :write [29 6] INFO [2019-12-17 12:37:30,776] jepsen worker 7 - jepsen.util 16387 :invoke :write [2 45] INFO [2019-12-17 12:37:30,803] jepsen worker 28 - jepsen.util 10108 :fail :cas [9 [31 41]] :ValueNotSwapFrom INFO [2019-12-17 12:37:30,803] jepsen worker 28 - jepsen.util 10108 :invoke :cas [9 [26 32]] INFO [2019-12-17 12:37:30,820] jepsen worker 27 - jepsen.util 11007 :ok :read [9 10] INFO [2019-12-17 12:37:30,820] jepsen worker 27 - jepsen.util 11007 :invoke :write [9 33] INFO [2019-12-17 12:37:30,826] jepsen worker 19 - jepsen.util 12349 :fail :cas [6 [33 7]] :ValueNotSwapFrom INFO [2019-12-17 12:37:30,826] jepsen worker 19 - jepsen.util 12349 :invoke :cas [6 [19 3]] INFO [2019-12-17 12:37:30,830] jepsen worker 64 - jepsen.util 9154 :ok :read [21 25] INFO [2019-12-17 12:37:30,831] jepsen worker 64 - jepsen.util 9154 :invoke :write [21 23] INFO [2019-12-17 12:37:30,833] jepsen worker 74 - jepsen.util 11684 :info :cas [24 [17 29]] :RequestCanceledException INFO [2019-12-17 12:37:30,833] jepsen worker 54 - jepsen.util 12384 :info :write [18 35] :RequestCanceledException INFO [2019-12-17 12:37:30,838] jepsen worker 74 - jepsen.util 11774 :invoke :cas [24 [48 21]] INFO [2019-12-17 12:37:30,838] jepsen worker 54 - jepsen.util 12474 :invoke :cas [18 [9 18]] INFO [2019-12-17 12:37:30,857] jepsen worker 11 - jepsen.util 11531 :fail :cas [3 [33 14]] :ValueNotSwapFrom INFO [2019-12-17 12:37:30,857] jepsen worker 11 - jepsen.util 11531 :invoke :read [3 nil] INFO [2019-12-17 12:37:30,861] jepsen worker 10 - jepsen.util 11620 :info :write [3 9] :RequestCanceledException INFO [2019-12-17 12:37:30,864] jepsen worker 10 - jepsen.util 11710 :invoke :write [3 45] INFO [2019-12-17 12:37:30,873] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69342,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:37:30,873] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69342,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:37:30,873] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69343,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:37:30,873] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69343,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:37:30,874] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69344,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:37:30,874] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69344,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:37:30,903] jepsen worker 13 - jepsen.util 11353 :fail :cas [4 [2 8]] :ValueNotSwapFrom INFO [2019-12-17 12:37:30,903] jepsen worker 13 - jepsen.util 11353 :invoke :read [4 nil] INFO [2019-12-17 12:37:30,912] jepsen worker 71 - jepsen.util 12761 :ok :read [23 29] INFO [2019-12-17 12:37:30,912] jepsen worker 71 - jepsen.util 12761 :invoke :write [23 28] INFO [2019-12-17 12:37:30,916] jepsen worker 63 - jepsen.util 10233 :info :cas [21 [12 46]] :RequestCanceledException INFO [2019-12-17 12:37:30,916] jepsen worker 29 - jepsen.util 10289 :info :write [9 24] :RequestCanceledException INFO [2019-12-17 12:37:30,918] jepsen worker 63 - jepsen.util 10323 :invoke :cas [21 [38 10]] INFO [2019-12-17 12:37:30,920] jepsen worker 29 - jepsen.util 10379 :invoke :cas [9 [41 34]] INFO [2019-12-17 12:37:30,920] jepsen worker 74 - jepsen.util 11774 :fail :cas [24 [48 21]] :ValueNotSwapFrom INFO [2019-12-17 12:37:30,920] jepsen worker 74 - jepsen.util 11774 :invoke :read [24 nil] INFO [2019-12-17 12:37:30,923] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:37:30,923] jepsen nemesis - couchbase.util Rebalance check count: 546 INFO [2019-12-17 12:37:30,923] jepsen nemesis - couchbase.util Rebalance stuck count: 527 INFO [2019-12-17 12:37:30,924] jepsen worker 46 - jepsen.util 11746 :info :write [15 44] :RequestCanceledException INFO [2019-12-17 12:37:30,929] jepsen worker 46 - jepsen.util 11836 :invoke :write [15 19] INFO [2019-12-17 12:37:30,955] jepsen worker 36 - jepsen.util 12546 :fail :cas [12 [25 37]] :ValueNotSwapFrom INFO [2019-12-17 12:37:30,956] jepsen worker 36 - jepsen.util 12546 :invoke :cas [12 [9 27]] INFO [2019-12-17 12:37:30,960] jepsen worker 65 - jepsen.util 10505 :ok :write [21 5] INFO [2019-12-17 12:37:30,960] jepsen worker 72 - jepsen.util 12132 :fail :read [24 nil] :RequestTimeoutException INFO [2019-12-17 12:37:30,960] jepsen worker 65 - jepsen.util 10505 :invoke :read [21 nil] INFO [2019-12-17 12:37:30,960] jepsen worker 72 - jepsen.util 12132 :invoke :write [24 38] INFO [2019-12-17 12:37:30,971] jepsen worker 11 - jepsen.util 11531 :ok :read [3 43] INFO [2019-12-17 12:37:30,971] jepsen worker 11 - jepsen.util 11531 :invoke :write [3 32] INFO [2019-12-17 12:37:30,992] jepsen worker 74 - jepsen.util 11774 :ok :read [24 29] INFO [2019-12-17 12:37:30,993] jepsen worker 74 - jepsen.util 11774 :invoke :read [24 nil] INFO [2019-12-17 12:37:31,001] jepsen worker 53 - jepsen.util 11393 :info :write [17 28] :RequestCanceledException INFO [2019-12-17 12:37:31,003] jepsen worker 53 - jepsen.util 11483 :invoke :read [17 nil] INFO [2019-12-17 12:37:31,007] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69357,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:37:31,007] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69357,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:37:31,007] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69358,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:37:31,007] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69358,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:37:31,007] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69359,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:37:31,007] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69359,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:37:31,012] jepsen worker 42 - jepsen.util 17322 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:37:31,012] jepsen worker 42 - jepsen.util 17322 :invoke :read [14 nil] INFO [2019-12-17 12:37:31,033] jepsen worker 57 - jepsen.util 16077 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:37:31,033] jepsen worker 57 - jepsen.util 16077 :invoke :cas [19 [16 29]] INFO [2019-12-17 12:37:31,035] jepsen worker 79 - jepsen.util 12139 :fail :cas [26 [41 19]] :ValueNotSwapFrom INFO [2019-12-17 12:37:31,035] jepsen worker 79 - jepsen.util 12139 :invoke :read [26 nil] INFO [2019-12-17 12:37:31,047] jepsen worker 31 - jepsen.util 10111 :fail :read [10 nil] :RequestTimeoutException INFO [2019-12-17 12:37:31,047] jepsen worker 31 - jepsen.util 10111 :invoke :read [10 nil] INFO [2019-12-17 12:37:31,053] jepsen worker 65 - jepsen.util 10505 :ok :read [21 5] INFO [2019-12-17 12:37:31,053] jepsen worker 65 - jepsen.util 10505 :invoke :write [21 8] INFO [2019-12-17 12:37:31,067] jepsen worker 27 - jepsen.util 11007 :ok :write [9 33] INFO [2019-12-17 12:37:31,068] jepsen worker 27 - jepsen.util 11007 :invoke :read [9 nil] INFO [2019-12-17 12:37:31,094] jepsen worker 56 - jepsen.util 12926 :ok :read [18 3] INFO [2019-12-17 12:37:31,094] jepsen worker 56 - jepsen.util 12926 :invoke :cas [18 [15 40]] INFO [2019-12-17 12:37:31,133] jepsen worker 1 - jepsen.util 16471 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:37:31,133] jepsen worker 83 - jepsen.util 16553 :info :write [27 12] :RequestCanceledException INFO [2019-12-17 12:37:31,134] jepsen worker 1 - jepsen.util 16471 :invoke :cas [0 [13 5]] INFO [2019-12-17 12:37:31,136] jepsen worker 83 - jepsen.util 16643 :invoke :cas [27 [22 18]] INFO [2019-12-17 12:37:31,143] jepsen worker 55 - jepsen.util 12565 :fail :cas [18 [44 16]] :ValueNotSwapFrom INFO [2019-12-17 12:37:31,144] jepsen worker 55 - jepsen.util 12565 :invoke :write [18 16] INFO [2019-12-17 12:37:31,145] jepsen worker 22 - jepsen.util 16402 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:37:31,145] jepsen worker 22 - jepsen.util 16402 :invoke :cas [7 [18 49]] INFO [2019-12-17 12:37:31,151] jepsen worker 65 - jepsen.util 10505 :ok :write [21 8] INFO [2019-12-17 12:37:31,151] jepsen worker 65 - jepsen.util 10505 :invoke :read [21 nil] INFO [2019-12-17 12:37:31,166] jepsen worker 39 - jepsen.util 15969 :info :cas [13 [4 36]] :RequestCanceledException INFO [2019-12-17 12:37:31,169] jepsen worker 39 - jepsen.util 16059 :invoke :read [13 nil] INFO [2019-12-17 12:37:31,186] jepsen worker 65 - jepsen.util 10505 :ok :read [21 8] INFO [2019-12-17 12:37:31,186] jepsen worker 65 - jepsen.util 10505 :invoke :cas [21 [36 20]] INFO [2019-12-17 12:37:31,219] jepsen worker 65 - jepsen.util 10505 :fail :cas [21 [36 20]] :ValueNotSwapFrom INFO [2019-12-17 12:37:31,219] jepsen worker 65 - jepsen.util 10505 :invoke :read [21 nil] INFO [2019-12-17 12:37:31,221] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69378,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:37:31,222] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69378,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:37:31,222] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69379,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:37:31,222] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69379,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:37:31,222] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69389,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:37:31,222] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69389,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:37:31,223] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69390,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:37:31,223] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69390,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:37:31,233] jepsen worker 76 - jepsen.util 16546 :info :cas [25 [24 31]] :RequestCanceledException INFO [2019-12-17 12:37:31,235] jepsen worker 89 - jepsen.util 11879 :ok :read [29 17] INFO [2019-12-17 12:37:31,236] jepsen worker 89 - jepsen.util 11879 :invoke :read [29 nil] INFO [2019-12-17 12:37:31,238] jepsen worker 76 - jepsen.util 16636 :invoke :cas [25 [7 8]] INFO [2019-12-17 12:37:31,243] jepsen worker 53 - jepsen.util 11483 :ok :read [17 19] INFO [2019-12-17 12:37:31,243] jepsen worker 53 - jepsen.util 11483 :invoke :write [17 6] INFO [2019-12-17 12:37:31,261] jepsen worker 65 - jepsen.util 10505 :ok :read [21 8] INFO [2019-12-17 12:37:31,261] jepsen worker 65 - jepsen.util 10505 :invoke :write [21 25] INFO [2019-12-17 12:37:31,313] jepsen worker 68 - jepsen.util 15728 :info :cas [22 [16 20]] :RequestCanceledException INFO [2019-12-17 12:37:31,315] jepsen worker 87 - jepsen.util 11427 :ok :read [29 17] INFO [2019-12-17 12:37:31,316] jepsen worker 87 - jepsen.util 11427 :invoke :write [29 4] INFO [2019-12-17 12:37:31,318] jepsen worker 68 - jepsen.util 15818 :invoke :write [22 23] INFO [2019-12-17 12:37:31,328] jepsen worker 49 - jepsen.util 16519 :info :write [16 8] :RequestCanceledException INFO [2019-12-17 12:37:31,333] jepsen worker 49 - jepsen.util 16609 :invoke :read [16 nil] INFO [2019-12-17 12:37:31,344] jepsen worker 79 - jepsen.util 12139 :ok :read [26 37] INFO [2019-12-17 12:37:31,344] jepsen worker 79 - jepsen.util 12139 :invoke :cas [26 [15 41]] INFO [2019-12-17 12:37:31,346] jepsen worker 6 - jepsen.util 16746 :info :cas [2 [4 41]] :RequestCanceledException INFO [2019-12-17 12:37:31,349] jepsen worker 65 - jepsen.util 10505 :ok :write [21 25] INFO [2019-12-17 12:37:31,349] jepsen worker 6 - jepsen.util 16836 :invoke :read [2 nil] INFO [2019-12-17 12:37:31,349] jepsen worker 65 - jepsen.util 10505 :invoke :read [21 nil] INFO [2019-12-17 12:37:31,361] jepsen worker 3 - jepsen.util 11253 :ok :read [1 19] INFO [2019-12-17 12:37:31,361] jepsen worker 3 - jepsen.util 11253 :invoke :write [1 14] INFO [2019-12-17 12:37:31,396] jepsen worker 65 - jepsen.util 10505 :ok :read [21 25] INFO [2019-12-17 12:37:31,396] jepsen worker 65 - jepsen.util 10505 :invoke :write [21 8] INFO [2019-12-17 12:37:31,400] jepsen worker 73 - jepsen.util 11143 :ok :write [24 23] INFO [2019-12-17 12:37:31,401] jepsen worker 73 - jepsen.util 11143 :invoke :read [24 nil] INFO [2019-12-17 12:37:31,401] jepsen worker 18 - jepsen.util 12258 :ok :write [6 44] INFO [2019-12-17 12:37:31,402] jepsen worker 18 - jepsen.util 12258 :invoke :read [6 nil] INFO [2019-12-17 12:37:31,425] jepsen worker 54 - jepsen.util 12474 :fail :cas [18 [9 18]] :ValueNotSwapFrom INFO [2019-12-17 12:37:31,426] jepsen worker 54 - jepsen.util 12474 :invoke :read [18 nil] INFO [2019-12-17 12:37:31,452] jepsen worker 30 - jepsen.util 11010 :ok :read [10 42] INFO [2019-12-17 12:37:31,453] jepsen worker 30 - jepsen.util 11010 :invoke :write [10 34] INFO [2019-12-17 12:37:31,489] jepsen worker 29 - jepsen.util 10379 :fail :cas [9 [41 34]] :ValueNotSwapFrom INFO [2019-12-17 12:37:31,489] jepsen worker 29 - jepsen.util 10379 :invoke :write [9 48] INFO [2019-12-17 12:37:31,506] jepsen worker 69 - jepsen.util 12219 :ok :read [23 29] INFO [2019-12-17 12:37:31,506] jepsen worker 69 - jepsen.util 12219 :invoke :cas [23 [36 20]] INFO [2019-12-17 12:37:31,515] jepsen worker 37 - jepsen.util 11557 :fail :cas [12 [10 10]] :ValueNotSwapFrom INFO [2019-12-17 12:37:31,515] jepsen worker 37 - jepsen.util 11557 :invoke :cas [12 [46 49]] INFO [2019-12-17 12:37:31,516] jepsen worker 40 - jepsen.util 16960 :info :cas [13 [19 42]] :RequestCanceledException INFO [2019-12-17 12:37:31,516] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69407,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:37:31,516] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69407,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:37:31,516] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69410,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:37:31,517] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69410,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:37:31,519] jepsen worker 40 - jepsen.util 17050 :invoke :write [13 3] INFO [2019-12-17 12:37:31,519] jepsen worker 31 - jepsen.util 10111 :ok :read [10 42] INFO [2019-12-17 12:37:31,519] jepsen worker 31 - jepsen.util 10111 :invoke :read [10 nil] INFO [2019-12-17 12:37:31,520] jepsen worker 36 - jepsen.util 12546 :fail :cas [12 [9 27]] :ValueNotSwapFrom INFO [2019-12-17 12:37:31,520] jepsen worker 36 - jepsen.util 12546 :invoke :read [12 nil] INFO [2019-12-17 12:37:31,533] jepsen worker 4 - jepsen.util 10984 :info :write [1 30] :RequestCanceledException INFO [2019-12-17 12:37:31,533] jepsen worker 34 - jepsen.util 16954 :info :cas [11 [33 12]] :RequestCanceledException INFO [2019-12-17 12:37:31,535] jepsen worker 34 - jepsen.util 17044 :invoke :cas [11 [20 0]] INFO [2019-12-17 12:37:31,535] jepsen worker 4 - jepsen.util 11074 :invoke :cas [1 [2 9]] INFO [2019-12-17 12:37:31,545] jepsen worker 41 - jepsen.util 17681 :info :cas [13 [45 11]] :RequestCanceledException INFO [2019-12-17 12:37:31,545] jepsen worker 65 - jepsen.util 10505 :ok :write [21 8] INFO [2019-12-17 12:37:31,545] jepsen worker 65 - jepsen.util 10505 :invoke :cas [21 [7 42]] INFO [2019-12-17 12:37:31,548] jepsen worker 41 - jepsen.util 17771 :invoke :read [13 nil] INFO [2019-12-17 12:37:31,565] jepsen worker 24 - jepsen.util 15324 :info :cas [8 [40 39]] :RequestCanceledException INFO [2019-12-17 12:37:31,568] jepsen worker 73 - jepsen.util 11143 :ok :read [24 23] INFO [2019-12-17 12:37:31,568] jepsen worker 73 - jepsen.util 11143 :invoke :write [24 19] INFO [2019-12-17 12:37:31,569] jepsen worker 24 - jepsen.util 15414 :invoke :write [8 40] INFO [2019-12-17 12:37:31,571] jepsen worker 80 - jepsen.util 11600 :fail :cas [26 [0 26]] :ValueNotSwapFrom INFO [2019-12-17 12:37:31,571] jepsen worker 80 - jepsen.util 11600 :invoke :cas [26 [36 10]] INFO [2019-12-17 12:37:31,585] jepsen worker 10 - jepsen.util 11710 :ok :write [3 45] INFO [2019-12-17 12:37:31,585] jepsen worker 10 - jepsen.util 11710 :invoke :cas [3 [6 29]] INFO [2019-12-17 12:37:31,593] jepsen worker 65 - jepsen.util 10505 :fail :cas [21 [7 42]] :ValueNotSwapFrom INFO [2019-12-17 12:37:31,594] jepsen worker 65 - jepsen.util 10505 :invoke :read [21 nil] INFO [2019-12-17 12:37:31,633] jepsen worker 36 - jepsen.util 12546 :ok :read [12 26] INFO [2019-12-17 12:37:31,633] jepsen worker 60 - jepsen.util 17520 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:37:31,636] jepsen worker 46 - jepsen.util 11836 :ok :write [15 19] INFO [2019-12-17 12:37:31,644] jepsen worker 30 - jepsen.util 11010 :ok :write [10 34] INFO [2019-12-17 12:37:31,671] jepsen worker 78 - jepsen.util 11778 :info :write [26 5] :RequestCanceledException INFO [2019-12-17 12:37:31,684] jepsen worker 10 - jepsen.util 11710 :fail :cas [3 [6 29]] :ValueNotSwapFrom INFO [2019-12-17 12:37:31,716] jepsen worker 73 - jepsen.util 11143 :ok :write [24 19] INFO [2019-12-17 12:37:31,716] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69431,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:37:31,718] jepsen worker 51 - jepsen.util 11121 :info :write [17 8] :RequestCanceledException INFO [2019-12-17 12:37:31,718] jepsen worker 17 - jepsen.util 16577 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:37:31,742] jepsen worker 36 - jepsen.util 12546 :invoke :cas [12 [21 3]] INFO [2019-12-17 12:37:31,742] jepsen worker 60 - jepsen.util 17520 :invoke :cas [20 [7 29]] INFO [2019-12-17 12:37:31,742] jepsen worker 46 - jepsen.util 11836 :invoke :write [15 45] INFO [2019-12-17 12:37:31,742] jepsen worker 30 - jepsen.util 11010 :invoke :cas [10 [6 14]] INFO [2019-12-17 12:37:31,742] jepsen worker 10 - jepsen.util 11710 :invoke :cas [3 [12 31]] INFO [2019-12-17 12:37:31,743] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69431,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:37:31,743] jepsen worker 73 - jepsen.util 11143 :invoke :write [24 2] INFO [2019-12-17 12:37:31,743] jepsen worker 17 - jepsen.util 16577 :invoke :read [5 nil] INFO [2019-12-17 12:37:31,743] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69433,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:37:31,743] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69433,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:37:31,751] jepsen worker 18 - jepsen.util 12258 :ok :read [6 44] INFO [2019-12-17 12:37:31,752] jepsen worker 18 - jepsen.util 12258 :invoke :read [6 nil] INFO [2019-12-17 12:37:31,758] jepsen worker 78 - jepsen.util 11868 :invoke :write [26 12] INFO [2019-12-17 12:37:31,764] jepsen worker 62 - jepsen.util 17072 :info :cas [20 [20 31]] :RequestCanceledException INFO [2019-12-17 12:37:31,764] jepsen worker 51 - jepsen.util 11211 :invoke :read [17 nil] INFO [2019-12-17 12:37:31,765] jepsen worker 65 - jepsen.util 10505 :ok :read [21 8] INFO [2019-12-17 12:37:31,765] jepsen worker 65 - jepsen.util 10505 :invoke :read [21 nil] INFO [2019-12-17 12:37:31,768] jepsen worker 62 - jepsen.util 17162 :invoke :read [20 nil] INFO [2019-12-17 12:37:31,787] jepsen worker 13 - jepsen.util 11353 :ok :read [4 8] INFO [2019-12-17 12:37:31,787] jepsen worker 13 - jepsen.util 11353 :invoke :read [4 nil] INFO [2019-12-17 12:37:31,834] jepsen worker 26 - jepsen.util 17036 :info :cas [8 [42 13]] :RequestCanceledException INFO [2019-12-17 12:37:31,840] jepsen worker 26 - jepsen.util 17126 :invoke :read [8 nil] INFO [2019-12-17 12:37:31,847] jepsen worker 86 - jepsen.util 17096 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:37:31,847] jepsen worker 86 - jepsen.util 17096 :invoke :write [28 1] INFO [2019-12-17 12:37:31,856] jepsen worker 10 - jepsen.util 11710 :fail :cas [3 [12 31]] :ValueNotSwapFrom INFO [2019-12-17 12:37:31,856] jepsen worker 10 - jepsen.util 11710 :invoke :write [3 3] INFO [2019-12-17 12:37:31,865] jepsen worker 69 - jepsen.util 12219 :fail :cas [23 [36 20]] :ValueNotSwapFrom INFO [2019-12-17 12:37:31,865] jepsen worker 69 - jepsen.util 12219 :invoke :cas [23 [4 45]] INFO [2019-12-17 12:37:31,876] jepsen worker 30 - jepsen.util 11010 :fail :cas [10 [6 14]] :ValueNotSwapFrom INFO [2019-12-17 12:37:31,876] jepsen worker 30 - jepsen.util 11010 :invoke :cas [10 [20 14]] INFO [2019-12-17 12:37:31,886] jepsen worker 65 - jepsen.util 10505 :ok :read [21 8] INFO [2019-12-17 12:37:31,886] jepsen worker 65 - jepsen.util 10505 :invoke :cas [21 [15 48]] INFO [2019-12-17 12:37:31,894] jepsen worker 18 - jepsen.util 12258 :ok :read [6 44] INFO [2019-12-17 12:37:31,895] jepsen worker 18 - jepsen.util 12258 :invoke :write [6 48] INFO [2019-12-17 12:37:31,899] jepsen worker 75 - jepsen.util 16005 :info :write [25 48] :RequestCanceledException INFO [2019-12-17 12:37:31,901] jepsen worker 75 - jepsen.util 16095 :invoke :cas [25 [40 8]] INFO [2019-12-17 12:37:31,950] jepsen worker 4 - jepsen.util 11074 :fail :cas [1 [2 9]] :ValueNotSwapFrom INFO [2019-12-17 12:37:31,950] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:37:31,950] jepsen worker 4 - jepsen.util 11074 :invoke :write [1 17] INFO [2019-12-17 12:37:31,951] jepsen nemesis - couchbase.util Rebalance check count: 547 INFO [2019-12-17 12:37:31,952] jepsen nemesis - couchbase.util Rebalance stuck count: 528 INFO [2019-12-17 12:37:31,960] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69451,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:37:31,960] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69451,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:37:31,960] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69453,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:37:31,960] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69453,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:37:31,960] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69454,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:37:31,960] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69454,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:37:31,986] jepsen worker 65 - jepsen.util 10505 :fail :cas [21 [15 48]] :ValueNotSwapFrom INFO [2019-12-17 12:37:31,987] jepsen worker 65 - jepsen.util 10505 :invoke :write [21 37] INFO [2019-12-17 12:37:32,007] jepsen worker 73 - jepsen.util 11143 :ok :write [24 2] INFO [2019-12-17 12:37:32,007] jepsen worker 73 - jepsen.util 11143 :invoke :read [24 nil] INFO [2019-12-17 12:37:32,011] jepsen worker 46 - jepsen.util 11836 :ok :write [15 45] INFO [2019-12-17 12:37:32,011] jepsen worker 46 - jepsen.util 11836 :invoke :read [15 nil] INFO [2019-12-17 12:37:32,017] jepsen worker 85 - jepsen.util 17095 :info :cas [28 [33 25]] :RequestCanceledException INFO [2019-12-17 12:37:32,021] jepsen worker 85 - jepsen.util 17185 :invoke :write [28 47] INFO [2019-12-17 12:37:32,024] jepsen worker 36 - jepsen.util 12546 :fail :cas [12 [21 3]] :ValueNotSwapFrom INFO [2019-12-17 12:37:32,024] jepsen worker 36 - jepsen.util 12546 :invoke :read [12 nil] INFO [2019-12-17 12:37:32,033] jepsen worker 84 - jepsen.util 16104 :info :write [28 2] :RequestCanceledException INFO [2019-12-17 12:37:32,033] jepsen worker 82 - jepsen.util 16732 :info :write [27 20] :RequestCanceledException INFO [2019-12-17 12:37:32,034] jepsen worker 67 - jepsen.util 16447 :info :cas [22 [30 32]] :RequestCanceledException INFO [2019-12-17 12:37:32,034] jepsen worker 33 - jepsen.util 16233 :info :write [11 47] :RequestCanceledException INFO [2019-12-17 12:37:32,044] jepsen worker 84 - jepsen.util 16194 :invoke :read [28 nil] INFO [2019-12-17 12:37:32,044] jepsen worker 33 - jepsen.util 16323 :invoke :read [11 nil] INFO [2019-12-17 12:37:32,045] jepsen worker 67 - jepsen.util 16537 :invoke :read [22 nil] INFO [2019-12-17 12:37:32,045] jepsen worker 82 - jepsen.util 16822 :invoke :read [27 nil] INFO [2019-12-17 12:37:32,047] jepsen worker 23 - jepsen.util 16763 :info :write [7 18] :RequestCanceledException INFO [2019-12-17 12:37:32,047] jepsen worker 66 - jepsen.util 16266 :info :write [22 30] :RequestCanceledException INFO [2019-12-17 12:37:32,049] jepsen worker 23 - jepsen.util 16853 :invoke :write [7 41] INFO [2019-12-17 12:37:32,049] jepsen worker 66 - jepsen.util 16356 :invoke :cas [22 [3 14]] INFO [2019-12-17 12:37:32,076] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69463,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:37:32,076] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69463,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:37:32,083] jepsen worker 73 - jepsen.util 11143 :ok :read [24 2] INFO [2019-12-17 12:37:32,083] jepsen worker 73 - jepsen.util 11143 :invoke :read [24 nil] INFO [2019-12-17 12:37:32,098] jepsen worker 30 - jepsen.util 11010 :fail :cas [10 [20 14]] :ValueNotSwapFrom INFO [2019-12-17 12:37:32,099] jepsen worker 30 - jepsen.util 11010 :invoke :read [10 nil] INFO [2019-12-17 12:37:32,100] jepsen worker 45 - jepsen.util 11925 :info :write [15 21] :RequestCanceledException INFO [2019-12-17 12:37:32,109] jepsen worker 45 - jepsen.util 12015 :invoke :write [15 18] INFO [2019-12-17 12:37:32,122] jepsen worker 65 - jepsen.util 10505 :ok :write [21 37] INFO [2019-12-17 12:37:32,123] jepsen worker 65 - jepsen.util 10505 :invoke :cas [21 [22 35]] INFO [2019-12-17 12:37:32,133] jepsen worker 12 - jepsen.util 12162 :info :cas [4 [48 13]] :RequestCanceledException INFO [2019-12-17 12:37:32,133] jepsen worker 38 - jepsen.util 12368 :info :write [12 25] :RequestCanceledException INFO [2019-12-17 12:37:32,136] jepsen worker 38 - jepsen.util 12458 :invoke :write [12 31] INFO [2019-12-17 12:37:32,136] jepsen worker 12 - jepsen.util 12252 :invoke :cas [4 [46 4]] INFO [2019-12-17 12:37:32,147] jepsen worker 9 - jepsen.util 12429 :fail :read [3 nil] :RequestTimeoutException INFO [2019-12-17 12:37:32,147] jepsen worker 9 - jepsen.util 12429 :invoke :cas [3 [13 39]] INFO [2019-12-17 12:37:32,161] jepsen worker 21 - jepsen.util 15771 :info :write [7 14] :RequestCanceledException INFO [2019-12-17 12:37:32,167] jepsen worker 21 - jepsen.util 15861 :invoke :read [7 nil] INFO [2019-12-17 12:37:32,200] jepsen worker 20 - jepsen.util 11630 :info :write [6 40] :RequestCanceledException INFO [2019-12-17 12:37:32,200] jepsen worker 47 - jepsen.util 12917 :fail :read [15 nil] :RequestTimeoutException INFO [2019-12-17 12:37:32,200] jepsen worker 47 - jepsen.util 12917 :invoke :read [15 nil] INFO [2019-12-17 12:37:32,205] jepsen worker 54 - jepsen.util 12474 :ok :read [18 3] INFO [2019-12-17 12:37:32,205] jepsen worker 54 - jepsen.util 12474 :invoke :read [18 nil] INFO [2019-12-17 12:37:32,207] jepsen worker 20 - jepsen.util 11720 :invoke :cas [6 [18 6]] INFO [2019-12-17 12:37:32,217] jepsen worker 52 - jepsen.util 11302 :info :write [17 9] :RequestCanceledException INFO [2019-12-17 12:37:32,217] jepsen worker 0 - jepsen.util 16830 :info :write [0 32] :RequestCanceledException INFO [2019-12-17 12:37:32,225] jepsen worker 0 - jepsen.util 16920 :invoke :write [0 40] INFO [2019-12-17 12:37:32,227] jepsen worker 52 - jepsen.util 11392 :invoke :read [17 nil] INFO [2019-12-17 12:37:32,233] jepsen worker 73 - jepsen.util 11143 :ok :read [24 2] INFO [2019-12-17 12:37:32,233] jepsen worker 70 - jepsen.util 12400 :info :write [23 45] :RequestCanceledException INFO [2019-12-17 12:37:32,234] jepsen worker 59 - jepsen.util 15719 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:37:32,234] jepsen worker 56 - jepsen.util 12926 :fail :cas [18 [15 40]] :ValueNotSwapFrom INFO [2019-12-17 12:37:32,243] jepsen worker 79 - jepsen.util 12139 :fail :cas [26 [15 41]] :ValueNotSwapFrom INFO [2019-12-17 12:37:32,261] jepsen worker 9 - jepsen.util 12429 :fail :cas [3 [13 39]] :ValueNotSwapFrom INFO [2019-12-17 12:37:32,264] jepsen worker 30 - jepsen.util 11010 :ok :read [10 34] INFO [2019-12-17 12:37:32,280] jepsen worker 69 - jepsen.util 12219 :fail :cas [23 [4 45]] :ValueNotSwapFrom INFO [2019-12-17 12:37:32,280] jepsen worker 89 - jepsen.util 11879 :ok :read [29 17] INFO [2019-12-17 12:37:32,281] jepsen worker 65 - jepsen.util 10505 :fail :cas [21 [22 35]] :ValueNotSwapFrom INFO [2019-12-17 12:37:32,283] jepsen worker 73 - jepsen.util 11143 :invoke :write [24 46] INFO [2019-12-17 12:37:32,283] jepsen worker 59 - jepsen.util 15719 :invoke :read [19 nil] INFO [2019-12-17 12:37:32,283] jepsen worker 56 - jepsen.util 12926 :invoke :cas [18 [4 14]] INFO [2019-12-17 12:37:32,283] jepsen worker 79 - jepsen.util 12139 :invoke :write [26 46] INFO [2019-12-17 12:37:32,283] jepsen worker 9 - jepsen.util 12429 :invoke :read [3 nil] INFO [2019-12-17 12:37:32,283] jepsen worker 30 - jepsen.util 11010 :invoke :cas [10 [32 42]] INFO [2019-12-17 12:37:32,283] jepsen worker 69 - jepsen.util 12219 :invoke :read [23 nil] INFO [2019-12-17 12:37:32,283] jepsen worker 89 - jepsen.util 11879 :invoke :cas [29 [39 18]] INFO [2019-12-17 12:37:32,283] jepsen worker 65 - jepsen.util 10505 :invoke :read [21 nil] INFO [2019-12-17 12:37:32,287] jepsen worker 70 - jepsen.util 12490 :invoke :write [23 21] INFO [2019-12-17 12:37:32,295] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69488,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:37:32,295] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69488,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:37:32,295] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69490,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:37:32,295] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69490,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:37:32,295] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69497,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:37:32,295] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69497,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:37:32,295] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69501,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:37:32,296] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69501,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:37:32,301] jepsen worker 14 - jepsen.util 11624 :info :write [4 44] :RequestCanceledException INFO [2019-12-17 12:37:32,308] jepsen worker 14 - jepsen.util 11714 :invoke :write [4 14] INFO [2019-12-17 12:37:32,315] jepsen worker 2 - jepsen.util 15932 :info :cas [0 [37 7]] :RequestCanceledException INFO [2019-12-17 12:37:32,323] jepsen worker 2 - jepsen.util 16022 :invoke :read [0 nil] INFO [2019-12-17 12:37:32,324] jepsen worker 47 - jepsen.util 12917 :ok :read [15 45] INFO [2019-12-17 12:37:32,325] jepsen worker 47 - jepsen.util 12917 :invoke :write [15 31] INFO [2019-12-17 12:37:32,326] jepsen worker 20 - jepsen.util 11720 :fail :cas [6 [18 6]] :ValueNotSwapFrom INFO [2019-12-17 12:37:32,327] jepsen worker 20 - jepsen.util 11720 :invoke :cas [6 [7 48]] INFO [2019-12-17 12:37:32,328] jepsen worker 51 - jepsen.util 11211 :ok :read [17 19] INFO [2019-12-17 12:37:32,328] jepsen worker 51 - jepsen.util 11211 :invoke :read [17 nil] INFO [2019-12-17 12:37:32,365] jepsen worker 30 - jepsen.util 11010 :fail :cas [10 [32 42]] :ValueNotSwapFrom INFO [2019-12-17 12:37:32,365] jepsen worker 30 - jepsen.util 11010 :invoke :write [10 5] INFO [2019-12-17 12:37:32,371] jepsen worker 9 - jepsen.util 12429 :ok :read [3 45] INFO [2019-12-17 12:37:32,371] jepsen worker 9 - jepsen.util 12429 :invoke :cas [3 [30 7]] INFO [2019-12-17 12:37:32,385] jepsen worker 13 - jepsen.util 11353 :ok :read [4 8] INFO [2019-12-17 12:37:32,385] jepsen worker 13 - jepsen.util 11353 :invoke :write [4 47] INFO [2019-12-17 12:37:32,392] jepsen worker 65 - jepsen.util 10505 :ok :read [21 37] INFO [2019-12-17 12:37:32,392] jepsen worker 65 - jepsen.util 10505 :invoke :read [21 nil] INFO [2019-12-17 12:37:32,433] jepsen worker 32 - jepsen.util 12362 :info :cas [10 [18 17]] :RequestCanceledException INFO [2019-12-17 12:37:32,437] jepsen worker 36 - jepsen.util 12546 :ok :read [12 26] INFO [2019-12-17 12:37:32,437] jepsen worker 36 - jepsen.util 12546 :invoke :write [12 13] INFO [2019-12-17 12:37:32,441] jepsen worker 32 - jepsen.util 12452 :invoke :cas [10 [9 48]] INFO [2019-12-17 12:37:32,442] jepsen worker 65 - jepsen.util 10505 :ok :read [21 37] INFO [2019-12-17 12:37:32,442] jepsen worker 65 - jepsen.util 10505 :invoke :cas [21 [2 18]] INFO [2019-12-17 12:37:32,461] jepsen worker 77 - jepsen.util 17357 :info :write [25 10] :RequestCanceledException INFO [2019-12-17 12:37:32,496] jepsen worker 77 - jepsen.util 17447 :invoke :cas [25 [46 14]] INFO [2019-12-17 12:37:32,504] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69524,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:37:32,519] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69524,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:37:32,520] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69528,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:37:32,521] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69528,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:37:32,537] jepsen worker 25 - jepsen.util 17395 :info :write [8 17] :RequestCanceledException INFO [2019-12-17 12:37:32,547] jepsen worker 25 - jepsen.util 17485 :invoke :cas [8 [33 27]] INFO [2019-12-17 12:37:32,547] jepsen worker 16 - jepsen.util 18196 :info :cas [5 [40 30]] :RequestCanceledException INFO [2019-12-17 12:37:32,559] jepsen worker 16 - jepsen.util 18286 :invoke :write [5 25] INFO [2019-12-17 12:37:32,561] jepsen worker 12 - jepsen.util 12252 :fail :cas [4 [46 4]] :ValueNotSwapFrom INFO [2019-12-17 12:37:32,561] jepsen worker 12 - jepsen.util 12252 :invoke :read [4 nil] INFO [2019-12-17 12:37:32,597] jepsen worker 20 - jepsen.util 11720 :fail :cas [6 [7 48]] :ValueNotSwapFrom INFO [2019-12-17 12:37:32,598] jepsen worker 20 - jepsen.util 11720 :invoke :write [6 19] INFO [2019-12-17 12:37:32,598] jepsen worker 37 - jepsen.util 11557 :fail :cas [12 [46 49]] :ValueNotSwapFrom INFO [2019-12-17 12:37:32,598] jepsen worker 37 - jepsen.util 11557 :invoke :read [12 nil] INFO [2019-12-17 12:37:32,600] jepsen worker 5 - jepsen.util 11615 :info :write [1 21] :RequestCanceledException INFO [2019-12-17 12:37:32,604] jepsen worker 5 - jepsen.util 11705 :invoke :write [1 33] INFO [2019-12-17 12:37:32,616] jepsen worker 80 - jepsen.util 11600 :fail :cas [26 [36 10]] :ValueNotSwapFrom INFO [2019-12-17 12:37:32,616] jepsen worker 80 - jepsen.util 11600 :invoke :read [26 nil] INFO [2019-12-17 12:37:32,627] jepsen worker 65 - jepsen.util 10505 :fail :cas [21 [2 18]] :ValueNotSwapFrom INFO [2019-12-17 12:37:32,628] jepsen worker 65 - jepsen.util 10505 :invoke :write [21 16] INFO [2019-12-17 12:37:32,634] jepsen worker 35 - jepsen.util 16235 :info :cas [11 [42 18]] :RequestCanceledException INFO [2019-12-17 12:37:32,637] jepsen worker 35 - jepsen.util 16325 :invoke :write [11 24] INFO [2019-12-17 12:37:32,664] jepsen worker 48 - jepsen.util 15078 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:37:32,665] jepsen worker 48 - jepsen.util 15078 :invoke :cas [16 [17 17]] INFO [2019-12-17 12:37:32,679] jepsen worker 65 - jepsen.util 10505 :ok :write [21 16] INFO [2019-12-17 12:37:32,680] jepsen worker 65 - jepsen.util 10505 :invoke :cas [21 [12 17]] INFO [2019-12-17 12:37:32,699] jepsen worker 54 - jepsen.util 12474 :ok :read [18 3] INFO [2019-12-17 12:37:32,699] jepsen worker 54 - jepsen.util 12474 :invoke :read [18 nil] INFO [2019-12-17 12:37:32,701] jepsen worker 8 - jepsen.util 16118 :info :write [2 12] :RequestCanceledException INFO [2019-12-17 12:37:32,704] jepsen worker 8 - jepsen.util 16208 :invoke :read [2 nil] INFO [2019-12-17 12:37:32,718] jepsen worker 44 - jepsen.util 17144 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:37:32,719] jepsen worker 44 - jepsen.util 17144 :invoke :write [14 19] INFO [2019-12-17 12:37:32,725] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69542,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:37:32,725] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69542,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:37:32,726] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69543,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:37:32,726] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69543,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:37:32,726] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69544,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:37:32,727] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69544,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:37:32,733] jepsen worker 43 - jepsen.util 16603 :info :write [14 2] :RequestCanceledException INFO [2019-12-17 12:37:32,736] jepsen worker 43 - jepsen.util 16693 :invoke :write [14 21] INFO [2019-12-17 12:37:32,741] jepsen worker 65 - jepsen.util 10505 :fail :cas [21 [12 17]] :ValueNotSwapFrom INFO [2019-12-17 12:37:32,742] jepsen worker 65 - jepsen.util 10505 :invoke :write [21 39] INFO [2019-12-17 12:37:32,746] jepsen worker 61 - jepsen.util 17701 :info :write [20 11] :RequestCanceledException INFO [2019-12-17 12:37:32,746] jepsen worker 81 - jepsen.util 16011 :info :cas [27 [10 12]] :RequestCanceledException INFO [2019-12-17 12:37:32,748] jepsen worker 61 - jepsen.util 17791 :invoke :write [20 23] INFO [2019-12-17 12:37:32,749] jepsen worker 81 - jepsen.util 16101 :invoke :cas [27 [33 2]] INFO [2019-12-17 12:37:32,799] jepsen worker 15 - jepsen.util 15225 :info :write [5 18] :RequestCanceledException INFO [2019-12-17 12:37:32,803] jepsen worker 65 - jepsen.util 10505 :ok :write [21 39] INFO [2019-12-17 12:37:32,804] jepsen worker 65 - jepsen.util 10505 :invoke :write [21 15] INFO [2019-12-17 12:37:32,819] jepsen worker 58 - jepsen.util 16708 :info :cas [19 [45 33]] :RequestCanceledException INFO [2019-12-17 12:37:32,823] jepsen worker 15 - jepsen.util 15315 :invoke :cas [5 [21 31]] INFO [2019-12-17 12:37:32,836] jepsen worker 50 - jepsen.util 16520 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:37:32,836] jepsen worker 58 - jepsen.util 16798 :invoke :read [19 nil] INFO [2019-12-17 12:37:32,836] jepsen worker 50 - jepsen.util 16520 :invoke :read [16 nil] INFO [2019-12-17 12:37:32,848] jepsen worker 88 - jepsen.util 12058 :info :write [29 6] :RequestCanceledException INFO [2019-12-17 12:37:32,862] jepsen worker 7 - jepsen.util 16387 :info :write [2 45] :RequestCanceledException INFO [2019-12-17 12:37:32,862] jepsen worker 28 - jepsen.util 10108 :info :cas [9 [26 32]] :RequestCanceledException INFO [2019-12-17 12:37:32,863] jepsen worker 89 - jepsen.util 11879 :fail :cas [29 [39 18]] :ValueNotSwapFrom INFO [2019-12-17 12:37:32,863] jepsen worker 89 - jepsen.util 11879 :invoke :read [29 nil] INFO [2019-12-17 12:37:32,864] jepsen worker 28 - jepsen.util 10198 :invoke :read [9 nil] INFO [2019-12-17 12:37:32,866] jepsen worker 7 - jepsen.util 16477 :invoke :cas [2 [48 4]] INFO [2019-12-17 12:37:32,874] jepsen worker 88 - jepsen.util 12148 :invoke :read [29 nil] INFO [2019-12-17 12:37:33,136] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69565,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:37:33,136] jepsen worker 72 - jepsen.util 12132 :info :write [24 38] :RequestCanceledException INFO [2019-12-17 12:37:33,136] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69565,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:37:33,137] jepsen worker 63 - jepsen.util 10323 :info :cas [21 [38 10]] :RequestCanceledException INFO [2019-12-17 12:37:33,137] jepsen worker 64 - jepsen.util 9154 :info :write [21 23] :RequestCanceledException INFO [2019-12-17 12:37:33,137] jepsen worker 27 - jepsen.util 11007 :fail :read [9 nil] :RequestTimeoutException INFO [2019-12-17 12:37:33,137] jepsen worker 52 - jepsen.util 11392 :ok :read [17 19] INFO [2019-12-17 12:37:33,137] jepsen worker 74 - jepsen.util 11774 :fail :read [24 nil] :RequestTimeoutException INFO [2019-12-17 12:37:33,137] jepsen worker 11 - jepsen.util 11531 :info :write [3 32] :RequestCanceledException INFO [2019-12-17 12:37:33,137] jepsen worker 1 - jepsen.util 16471 :info :cas [0 [13 5]] :RequestCanceledException INFO [2019-12-17 12:37:33,137] jepsen worker 57 - jepsen.util 16077 :info :cas [19 [16 29]] :RequestCanceledException INFO [2019-12-17 12:37:33,137] jepsen worker 42 - jepsen.util 17322 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:37:33,138] jepsen worker 27 - jepsen.util 11007 :invoke :read [9 nil] INFO [2019-12-17 12:37:33,138] jepsen worker 72 - jepsen.util 12222 :invoke :write [24 14] INFO [2019-12-17 12:37:33,138] jepsen worker 52 - jepsen.util 11392 :invoke :write [17 21] INFO [2019-12-17 12:37:33,138] jepsen worker 74 - jepsen.util 11774 :invoke :read [24 nil] INFO [2019-12-17 12:37:33,138] jepsen worker 19 - jepsen.util 12349 :info :cas [6 [19 3]] :RequestCanceledException INFO [2019-12-17 12:37:33,138] jepsen worker 71 - jepsen.util 12761 :info :write [23 28] :RequestCanceledException INFO [2019-12-17 12:37:33,138] jepsen worker 83 - jepsen.util 16643 :info :cas [27 [22 18]] :RequestCanceledException INFO [2019-12-17 12:37:33,139] jepsen worker 42 - jepsen.util 17322 :invoke :write [14 20] INFO [2019-12-17 12:37:33,139] jepsen worker 63 - jepsen.util 10413 :invoke :write [21 36] INFO [2019-12-17 12:37:33,140] jepsen worker 1 - jepsen.util 16561 :invoke :cas [0 [23 35]] INFO [2019-12-17 12:37:33,140] jepsen worker 64 - jepsen.util 9244 :invoke :cas [21 [41 38]] INFO [2019-12-17 12:37:33,140] jepsen worker 57 - jepsen.util 16167 :invoke :read [19 nil] INFO [2019-12-17 12:37:33,140] jepsen worker 56 - jepsen.util 12926 :fail :cas [18 [4 14]] :ValueNotSwapFrom INFO [2019-12-17 12:37:33,141] jepsen worker 19 - jepsen.util 12439 :invoke :cas [6 [6 8]] INFO [2019-12-17 12:37:33,141] jepsen worker 71 - jepsen.util 12851 :invoke :cas [23 [9 25]] INFO [2019-12-17 12:37:33,141] jepsen worker 11 - jepsen.util 11621 :invoke :write [3 7] INFO [2019-12-17 12:37:33,142] jepsen worker 83 - jepsen.util 16733 :invoke :read [27 nil] INFO [2019-12-17 12:37:33,142] jepsen worker 56 - jepsen.util 12926 :invoke :write [18 49] INFO [2019-12-17 12:37:33,155] jepsen worker 80 - jepsen.util 11600 :ok :read [26 37] INFO [2019-12-17 12:37:33,155] jepsen worker 80 - jepsen.util 11600 :invoke :cas [26 [24 6]] INFO [2019-12-17 12:37:33,160] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:37:33,160] jepsen nemesis - couchbase.util Rebalance check count: 548 INFO [2019-12-17 12:37:33,160] jepsen nemesis - couchbase.util Rebalance stuck count: 529 INFO [2019-12-17 12:37:33,166] jepsen worker 27 - jepsen.util 11007 :ok :read [9 33] INFO [2019-12-17 12:37:33,166] jepsen worker 27 - jepsen.util 11007 :invoke :write [9 9] INFO [2019-12-17 12:37:33,192] jepsen worker 10 - jepsen.util 11710 :ok :write [3 3] INFO [2019-12-17 12:37:33,192] jepsen worker 10 - jepsen.util 11710 :invoke :cas [3 [20 12]] INFO [2019-12-17 12:37:33,194] jepsen worker 18 - jepsen.util 12258 :ok :write [6 48] INFO [2019-12-17 12:37:33,194] jepsen worker 18 - jepsen.util 12258 :invoke :read [6 nil] INFO [2019-12-17 12:37:33,215] jepsen worker 39 - jepsen.util 16059 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:37:33,215] jepsen worker 37 - jepsen.util 11557 :ok :read [12 26] INFO [2019-12-17 12:37:33,215] jepsen worker 39 - jepsen.util 16059 :invoke :cas [13 [44 23]] INFO [2019-12-17 12:37:33,215] jepsen worker 37 - jepsen.util 11557 :invoke :cas [12 [17 46]] INFO [2019-12-17 12:37:33,219] jepsen worker 45 - jepsen.util 12015 :ok :write [15 18] INFO [2019-12-17 12:37:33,219] jepsen worker 73 - jepsen.util 11143 :ok :write [24 46] INFO [2019-12-17 12:37:33,219] jepsen worker 45 - jepsen.util 12015 :invoke :cas [15 [13 34]] INFO [2019-12-17 12:37:33,219] jepsen worker 73 - jepsen.util 11143 :invoke :write [24 11] INFO [2019-12-17 12:37:33,234] jepsen worker 18 - jepsen.util 12258 :ok :read [6 48] INFO [2019-12-17 12:37:33,234] jepsen worker 18 - jepsen.util 12258 :invoke :write [6 47] INFO [2019-12-17 12:37:33,245] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69572,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:37:33,245] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69596,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:37:33,245] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69602,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:37:33,245] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69572,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:37:33,245] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69596,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:37:33,245] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69602,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:37:33,245] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69576,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:37:33,246] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69597,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:37:33,246] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69605,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:37:33,246] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69576,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:37:33,246] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69597,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:37:33,246] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69605,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:37:33,246] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69606,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:37:33,246] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69606,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:37:33,247] jepsen worker 22 - jepsen.util 16402 :info :cas [7 [18 49]] :RequestCanceledException INFO [2019-12-17 12:37:33,247] jepsen worker 55 - jepsen.util 12565 :info :write [18 16] :RequestCanceledException INFO [2019-12-17 12:37:33,249] jepsen worker 55 - jepsen.util 12655 :invoke :write [18 47] INFO [2019-12-17 12:37:33,249] jepsen worker 22 - jepsen.util 16492 :invoke :cas [7 [3 14]] INFO [2019-12-17 12:37:33,265] jepsen worker 65 - jepsen.util 10505 :ok :write [21 15] INFO [2019-12-17 12:37:33,265] jepsen worker 65 - jepsen.util 10505 :invoke :read [21 nil] INFO [2019-12-17 12:37:33,287] jepsen worker 74 - jepsen.util 11774 :ok :read [24 46] INFO [2019-12-17 12:37:33,287] jepsen worker 74 - jepsen.util 11774 :invoke :cas [24 [49 22]] INFO [2019-12-17 12:37:33,290] jepsen worker 51 - jepsen.util 11211 :ok :read [17 19] INFO [2019-12-17 12:37:33,290] jepsen worker 51 - jepsen.util 11211 :invoke :write [17 40] INFO [2019-12-17 12:37:33,316] jepsen worker 12 - jepsen.util 12252 :ok :read [4 8] INFO [2019-12-17 12:37:33,316] jepsen worker 12 - jepsen.util 12252 :invoke :cas [4 [42 48]] INFO [2019-12-17 12:37:33,333] jepsen worker 76 - jepsen.util 16636 :info :cas [25 [7 8]] :RequestCanceledException INFO [2019-12-17 12:37:33,333] jepsen worker 53 - jepsen.util 11483 :info :write [17 6] :RequestCanceledException INFO [2019-12-17 12:37:33,335] jepsen worker 53 - jepsen.util 11573 :invoke :cas [17 [2 5]] INFO [2019-12-17 12:37:33,335] jepsen worker 76 - jepsen.util 16726 :invoke :read [25 nil] INFO [2019-12-17 12:37:33,346] jepsen worker 27 - jepsen.util 11007 :ok :write [9 9] INFO [2019-12-17 12:37:33,346] jepsen worker 27 - jepsen.util 11007 :invoke :cas [9 [43 39]] INFO [2019-12-17 12:37:33,347] jepsen worker 68 - jepsen.util 15818 :info :write [22 23] :RequestCanceledException INFO [2019-12-17 12:37:33,352] jepsen worker 68 - jepsen.util 15908 :invoke :read [22 nil] INFO [2019-12-17 12:37:33,357] jepsen worker 65 - jepsen.util 10505 :ok :read [21 15] INFO [2019-12-17 12:37:33,357] jepsen worker 65 - jepsen.util 10505 :invoke :cas [21 [11 33]] INFO [2019-12-17 12:37:33,362] jepsen worker 49 - jepsen.util 16609 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:37:33,362] jepsen worker 87 - jepsen.util 11427 :info :write [29 4] :RequestCanceledException INFO [2019-12-17 12:37:33,362] jepsen worker 49 - jepsen.util 16609 :invoke :read [16 nil] INFO [2019-12-17 12:37:33,363] jepsen worker 87 - jepsen.util 11517 :invoke :cas [29 [3 19]] INFO [2019-12-17 12:37:33,372] jepsen worker 88 - jepsen.util 12148 :ok :read [29 17] INFO [2019-12-17 12:37:33,372] jepsen worker 88 - jepsen.util 12148 :invoke :read [29 nil] INFO [2019-12-17 12:37:33,402] jepsen worker 19 - jepsen.util 12439 :fail :cas [6 [6 8]] :ValueNotSwapFrom INFO [2019-12-17 12:37:33,402] jepsen worker 19 - jepsen.util 12439 :invoke :write [6 33] INFO [2019-12-17 12:37:33,424] jepsen worker 87 - jepsen.util 11517 :fail :cas [29 [3 19]] :ValueNotSwapFrom INFO [2019-12-17 12:37:33,424] jepsen worker 87 - jepsen.util 11517 :invoke :write [29 25] INFO [2019-12-17 12:37:33,434] jepsen worker 6 - jepsen.util 16836 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:37:33,434] jepsen worker 6 - jepsen.util 16836 :invoke :cas [2 [14 7]] INFO [2019-12-17 12:37:33,453] jepsen worker 45 - jepsen.util 12015 :fail :cas [15 [13 34]] :ValueNotSwapFrom INFO [2019-12-17 12:37:33,453] jepsen worker 45 - jepsen.util 12015 :invoke :cas [15 [45 28]] INFO [2019-12-17 12:37:33,455] jepsen worker 27 - jepsen.util 11007 :fail :cas [9 [43 39]] :ValueNotSwapFrom INFO [2019-12-17 12:37:33,455] jepsen worker 27 - jepsen.util 11007 :invoke :read [9 nil] INFO [2019-12-17 12:37:33,459] jepsen worker 69 - jepsen.util 12219 :ok :read [23 29] INFO [2019-12-17 12:37:33,459] jepsen worker 69 - jepsen.util 12219 :invoke :write [23 21] INFO [2019-12-17 12:37:33,464] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69630,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:37:33,464] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69630,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:37:33,464] jepsen worker 3 - jepsen.util 11253 :info :write [1 14] :RequestCanceledException INFO [2019-12-17 12:37:33,464] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69634,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:37:33,464] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69634,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:37:33,465] jepsen worker 3 - jepsen.util 11343 :invoke :read [1 nil] INFO [2019-12-17 12:37:33,478] jepsen worker 54 - jepsen.util 12474 :ok :read [18 3] INFO [2019-12-17 12:37:33,478] jepsen worker 54 - jepsen.util 12474 :invoke :write [18 30] INFO [2019-12-17 12:37:33,521] jepsen worker 65 - jepsen.util 10505 :fail :cas [21 [11 33]] :ValueNotSwapFrom INFO [2019-12-17 12:37:33,522] jepsen worker 65 - jepsen.util 10505 :invoke :write [21 15] INFO [2019-12-17 12:37:33,537] jepsen worker 27 - jepsen.util 11007 :ok :read [9 9] INFO [2019-12-17 12:37:33,537] jepsen worker 27 - jepsen.util 11007 :invoke :cas [9 [12 43]] INFO [2019-12-17 12:37:33,539] jepsen worker 31 - jepsen.util 10111 :fail :read [10 nil] :RequestTimeoutException INFO [2019-12-17 12:37:33,539] jepsen worker 29 - jepsen.util 10379 :info :write [9 48] :RequestCanceledException INFO [2019-12-17 12:37:33,540] jepsen worker 31 - jepsen.util 10111 :invoke :write [10 42] INFO [2019-12-17 12:37:33,540] jepsen worker 29 - jepsen.util 10469 :invoke :cas [9 [14 39]] INFO [2019-12-17 12:37:33,544] jepsen worker 20 - jepsen.util 11720 :ok :write [6 19] INFO [2019-12-17 12:37:33,544] jepsen worker 20 - jepsen.util 11720 :invoke :cas [6 [42 44]] INFO [2019-12-17 12:37:33,554] jepsen worker 74 - jepsen.util 11774 :fail :cas [24 [49 22]] :ValueNotSwapFrom INFO [2019-12-17 12:37:33,554] jepsen worker 74 - jepsen.util 11774 :invoke :read [24 nil] INFO [2019-12-17 12:37:33,567] jepsen worker 53 - jepsen.util 11573 :fail :cas [17 [2 5]] :ValueNotSwapFrom INFO [2019-12-17 12:37:33,567] jepsen worker 53 - jepsen.util 11573 :invoke :cas [17 [47 35]] INFO [2019-12-17 12:37:33,591] jepsen worker 63 - jepsen.util 10413 :ok :write [21 36] INFO [2019-12-17 12:37:33,591] jepsen worker 63 - jepsen.util 10413 :invoke :cas [21 [47 19]] INFO [2019-12-17 12:37:33,600] jepsen worker 40 - jepsen.util 17050 :info :write [13 3] :RequestCanceledException INFO [2019-12-17 12:37:33,601] jepsen worker 40 - jepsen.util 17140 :invoke :write [13 25] INFO [2019-12-17 12:37:33,603] jepsen worker 71 - jepsen.util 12851 :fail :cas [23 [9 25]] :ValueNotSwapFrom INFO [2019-12-17 12:37:33,603] jepsen worker 71 - jepsen.util 12851 :invoke :cas [23 [13 33]] INFO [2019-12-17 12:37:33,618] jepsen worker 34 - jepsen.util 17044 :info :cas [11 [20 0]] :RequestCanceledException INFO [2019-12-17 12:37:33,620] jepsen worker 34 - jepsen.util 17134 :invoke :cas [11 [21 49]] INFO [2019-12-17 12:37:33,630] jepsen worker 27 - jepsen.util 11007 :fail :cas [9 [12 43]] :ValueNotSwapFrom INFO [2019-12-17 12:37:33,630] jepsen worker 27 - jepsen.util 11007 :invoke :read [9 nil] INFO [2019-12-17 12:37:33,647] jepsen worker 41 - jepsen.util 17771 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:37:33,647] jepsen worker 41 - jepsen.util 17771 :invoke :cas [13 [23 25]] INFO [2019-12-17 12:37:33,651] jepsen worker 28 - jepsen.util 10198 :ok :read [9 9] INFO [2019-12-17 12:37:33,651] jepsen worker 28 - jepsen.util 10198 :invoke :read [9 nil] INFO [2019-12-17 12:37:33,664] jepsen worker 24 - jepsen.util 15414 :info :write [8 40] :RequestCanceledException INFO [2019-12-17 12:37:33,665] jepsen worker 24 - jepsen.util 15504 :invoke :write [8 27] INFO [2019-12-17 12:37:33,665] jepsen worker 29 - jepsen.util 10469 :fail :cas [9 [14 39]] :ValueNotSwapFrom INFO [2019-12-17 12:37:33,666] jepsen worker 29 - jepsen.util 10469 :invoke :read [9 nil] INFO [2019-12-17 12:37:33,680] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69654,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:37:33,680] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69654,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:37:33,680] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69657,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:37:33,680] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69657,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:37:33,680] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69658,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:37:33,680] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69658,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:37:33,693] jepsen worker 64 - jepsen.util 9244 :fail :cas [21 [41 38]] :ValueNotSwapFrom INFO [2019-12-17 12:37:33,694] jepsen worker 64 - jepsen.util 9244 :invoke :read [21 nil] INFO [2019-12-17 12:37:33,709] jepsen worker 27 - jepsen.util 11007 :ok :read [9 9] INFO [2019-12-17 12:37:33,709] jepsen worker 27 - jepsen.util 11007 :invoke :cas [9 [0 13]] INFO [2019-12-17 12:37:33,724] jepsen worker 65 - jepsen.util 10505 :ok :write [21 15] INFO [2019-12-17 12:37:33,724] jepsen worker 65 - jepsen.util 10505 :invoke :write [21 10] INFO [2019-12-17 12:37:33,727] jepsen worker 29 - jepsen.util 10469 :ok :read [9 9] INFO [2019-12-17 12:37:33,727] jepsen worker 29 - jepsen.util 10469 :invoke :cas [9 [24 40]] INFO [2019-12-17 12:37:33,753] jepsen worker 87 - jepsen.util 11517 :ok :write [29 25] INFO [2019-12-17 12:37:33,753] jepsen worker 87 - jepsen.util 11517 :invoke :read [29 nil] INFO [2019-12-17 12:37:33,759] jepsen worker 27 - jepsen.util 11007 :fail :cas [9 [0 13]] :ValueNotSwapFrom INFO [2019-12-17 12:37:33,760] jepsen worker 27 - jepsen.util 11007 :invoke :cas [9 [16 26]] INFO [2019-12-17 12:37:33,774] jepsen worker 29 - jepsen.util 10469 :fail :cas [9 [24 40]] :ValueNotSwapFrom INFO [2019-12-17 12:37:33,774] jepsen worker 29 - jepsen.util 10469 :invoke :write [9 9] INFO [2019-12-17 12:37:33,784] jepsen worker 12 - jepsen.util 12252 :fail :cas [4 [42 48]] :ValueNotSwapFrom INFO [2019-12-17 12:37:33,784] jepsen worker 12 - jepsen.util 12252 :invoke :write [4 37] INFO [2019-12-17 12:37:33,801] jepsen worker 27 - jepsen.util 11007 :fail :cas [9 [16 26]] :ValueNotSwapFrom INFO [2019-12-17 12:37:33,801] jepsen worker 27 - jepsen.util 11007 :invoke :read [9 nil] INFO [2019-12-17 12:37:33,816] jepsen worker 17 - jepsen.util 16577 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:37:33,816] jepsen worker 62 - jepsen.util 17162 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:37:33,817] jepsen worker 17 - jepsen.util 16577 :invoke :write [5 41] INFO [2019-12-17 12:37:33,817] jepsen worker 62 - jepsen.util 17162 :invoke :write [20 0] INFO [2019-12-17 12:37:33,829] jepsen worker 11 - jepsen.util 11621 :ok :write [3 7] INFO [2019-12-17 12:37:33,829] jepsen worker 11 - jepsen.util 11621 :invoke :cas [3 [41 19]] INFO [2019-12-17 12:37:33,831] jepsen worker 60 - jepsen.util 17520 :info :cas [20 [7 29]] :RequestCanceledException INFO [2019-12-17 12:37:33,831] jepsen worker 78 - jepsen.util 11868 :info :write [26 12] :RequestCanceledException INFO [2019-12-17 12:37:33,832] jepsen worker 78 - jepsen.util 11958 :invoke :read [26 nil] INFO [2019-12-17 12:37:33,833] jepsen worker 27 - jepsen.util 11007 :ok :read [9 9] INFO [2019-12-17 12:37:33,833] jepsen worker 27 - jepsen.util 11007 :invoke :write [9 2] INFO [2019-12-17 12:37:33,833] jepsen worker 60 - jepsen.util 17610 :invoke :read [20 nil] INFO [2019-12-17 12:37:33,852] jepsen worker 63 - jepsen.util 10413 :fail :cas [21 [47 19]] :ValueNotSwapFrom INFO [2019-12-17 12:37:33,852] jepsen worker 63 - jepsen.util 10413 :invoke :cas [21 [1 46]] INFO [2019-12-17 12:37:33,879] jepsen worker 65 - jepsen.util 10505 :ok :write [21 10] INFO [2019-12-17 12:37:33,879] jepsen worker 65 - jepsen.util 10505 :invoke :cas [21 [26 0]] INFO [2019-12-17 12:37:33,896] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69679,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:37:33,896] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69679,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:37:33,896] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69682,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:37:33,896] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69682,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:37:33,896] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69683,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:37:33,897] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69683,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:37:33,909] jepsen worker 11 - jepsen.util 11621 :fail :cas [3 [41 19]] :ValueNotSwapFrom INFO [2019-12-17 12:37:33,909] jepsen worker 11 - jepsen.util 11621 :invoke :read [3 nil] INFO [2019-12-17 12:37:33,920] jepsen worker 63 - jepsen.util 10413 :fail :cas [21 [1 46]] :ValueNotSwapFrom INFO [2019-12-17 12:37:33,920] jepsen worker 63 - jepsen.util 10413 :invoke :cas [21 [20 30]] INFO [2019-12-17 12:37:33,934] jepsen worker 26 - jepsen.util 17126 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:37:33,934] jepsen worker 26 - jepsen.util 17126 :invoke :cas [8 [38 4]] INFO [2019-12-17 12:37:33,941] jepsen worker 11 - jepsen.util 11621 :ok :read [3 7] INFO [2019-12-17 12:37:33,941] jepsen worker 11 - jepsen.util 11621 :invoke :cas [3 [1 32]] INFO [2019-12-17 12:37:33,948] jepsen worker 75 - jepsen.util 16095 :info :cas [25 [40 8]] :RequestCanceledException INFO [2019-12-17 12:37:33,948] jepsen worker 86 - jepsen.util 17096 :info :write [28 1] :RequestCanceledException INFO [2019-12-17 12:37:33,949] jepsen worker 86 - jepsen.util 17186 :invoke :write [28 34] INFO [2019-12-17 12:37:33,949] jepsen worker 75 - jepsen.util 16185 :invoke :read [25 nil] INFO [2019-12-17 12:37:33,954] jepsen worker 63 - jepsen.util 10413 :fail :cas [21 [20 30]] :ValueNotSwapFrom INFO [2019-12-17 12:37:33,954] jepsen worker 63 - jepsen.util 10413 :invoke :cas [21 [34 6]] INFO [2019-12-17 12:37:33,987] jepsen worker 11 - jepsen.util 11621 :fail :cas [3 [1 32]] :ValueNotSwapFrom INFO [2019-12-17 12:37:33,987] jepsen worker 11 - jepsen.util 11621 :invoke :write [3 38] INFO [2019-12-17 12:37:34,003] jepsen worker 88 - jepsen.util 12148 :ok :read [29 25] INFO [2019-12-17 12:37:34,003] jepsen worker 88 - jepsen.util 12148 :invoke :cas [29 [3 6]] INFO [2019-12-17 12:37:34,006] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69696,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:37:34,006] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69696,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:37:34,006] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69698,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:37:34,006] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69698,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:37:34,028] jepsen worker 63 - jepsen.util 10413 :fail :cas [21 [34 6]] :ValueNotSwapFrom INFO [2019-12-17 12:37:34,028] jepsen worker 63 - jepsen.util 10413 :invoke :cas [21 [5 33]] INFO [2019-12-17 12:37:34,031] jepsen worker 87 - jepsen.util 11517 :ok :read [29 25] INFO [2019-12-17 12:37:34,032] jepsen worker 87 - jepsen.util 11517 :invoke :write [29 49] INFO [2019-12-17 12:37:34,034] jepsen worker 4 - jepsen.util 11074 :info :write [1 17] :RequestCanceledException INFO [2019-12-17 12:37:34,035] jepsen worker 4 - jepsen.util 11164 :invoke :write [1 10] INFO [2019-12-17 12:37:34,038] jepsen worker 65 - jepsen.util 10505 :fail :cas [21 [26 0]] :ValueNotSwapFrom INFO [2019-12-17 12:37:34,039] jepsen worker 65 - jepsen.util 10505 :invoke :write [21 32] INFO [2019-12-17 12:37:34,056] jepsen worker 53 - jepsen.util 11573 :fail :cas [17 [47 35]] :ValueNotSwapFrom INFO [2019-12-17 12:37:34,056] jepsen worker 53 - jepsen.util 11573 :invoke :cas [17 [18 20]] INFO [2019-12-17 12:37:34,058] jepsen worker 29 - jepsen.util 10469 :ok :write [9 9] INFO [2019-12-17 12:37:34,059] jepsen worker 29 - jepsen.util 10469 :invoke :cas [9 [45 39]] INFO [2019-12-17 12:37:34,064] jepsen worker 46 - jepsen.util 11836 :fail :read [15 nil] :RequestTimeoutException INFO [2019-12-17 12:37:34,064] jepsen worker 46 - jepsen.util 11836 :invoke :read [15 nil] INFO [2019-12-17 12:37:34,064] jepsen worker 66 - jepsen.util 16356 :info :cas [22 [3 14]] :RequestCanceledException INFO [2019-12-17 12:37:34,065] jepsen worker 85 - jepsen.util 17185 :info :write [28 47] :RequestCanceledException INFO [2019-12-17 12:37:34,067] jepsen worker 85 - jepsen.util 17275 :invoke :read [28 nil] INFO [2019-12-17 12:37:34,068] jepsen worker 66 - jepsen.util 16446 :invoke :write [22 6] INFO [2019-12-17 12:37:34,082] jepsen worker 29 - jepsen.util 10469 :fail :cas [9 [45 39]] :ValueNotSwapFrom INFO [2019-12-17 12:37:34,082] jepsen worker 29 - jepsen.util 10469 :invoke :cas [9 [43 24]] INFO [2019-12-17 12:37:34,100] jepsen worker 46 - jepsen.util 11836 :ok :read [15 18] INFO [2019-12-17 12:37:34,100] jepsen worker 33 - jepsen.util 16323 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:37:34,100] jepsen worker 46 - jepsen.util 11836 :invoke :write [15 37] INFO [2019-12-17 12:37:34,100] jepsen worker 33 - jepsen.util 16323 :invoke :cas [11 [36 47]] INFO [2019-12-17 12:37:34,101] jepsen worker 78 - jepsen.util 11958 :ok :read [26 37] INFO [2019-12-17 12:37:34,101] jepsen worker 78 - jepsen.util 11958 :invoke :read [26 nil] INFO [2019-12-17 12:37:34,113] jepsen worker 67 - jepsen.util 16537 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:37:34,113] jepsen worker 67 - jepsen.util 16537 :invoke :write [22 14] INFO [2019-12-17 12:37:34,133] jepsen worker 82 - jepsen.util 16822 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:37:34,133] jepsen worker 84 - jepsen.util 16194 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:37:34,133] jepsen worker 82 - jepsen.util 16822 :invoke :cas [27 [28 24]] INFO [2019-12-17 12:37:34,134] jepsen worker 84 - jepsen.util 16194 :invoke :write [28 7] INFO [2019-12-17 12:37:34,134] jepsen worker 73 - jepsen.util 11143 :ok :write [24 11] INFO [2019-12-17 12:37:34,134] jepsen worker 73 - jepsen.util 11143 :invoke :write [24 25] INFO [2019-12-17 12:37:34,140] jepsen worker 23 - jepsen.util 16853 :info :write [7 41] :RequestCanceledException INFO [2019-12-17 12:37:34,141] jepsen worker 29 - jepsen.util 10469 :fail :cas [9 [43 24]] :ValueNotSwapFrom INFO [2019-12-17 12:37:34,141] jepsen worker 29 - jepsen.util 10469 :invoke :write [9 49] INFO [2019-12-17 12:37:34,142] jepsen worker 23 - jepsen.util 16943 :invoke :cas [7 [19 18]] INFO [2019-12-17 12:37:34,150] jepsen worker 80 - jepsen.util 11600 :fail :cas [26 [24 6]] :ValueNotSwapFrom INFO [2019-12-17 12:37:34,150] jepsen worker 80 - jepsen.util 11600 :invoke :read [26 nil] INFO [2019-12-17 12:37:34,179] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:37:34,179] jepsen nemesis - couchbase.util Rebalance check count: 549 INFO [2019-12-17 12:37:34,179] jepsen nemesis - couchbase.util Rebalance stuck count: 530 INFO [2019-12-17 12:37:34,186] jepsen worker 64 - jepsen.util 9244 :ok :read [21 10] INFO [2019-12-17 12:37:34,186] jepsen worker 64 - jepsen.util 9244 :invoke :read [21 nil] INFO [2019-12-17 12:37:34,193] jepsen worker 37 - jepsen.util 11557 :fail :cas [12 [17 46]] :ValueNotSwapFrom INFO [2019-12-17 12:37:34,193] jepsen worker 37 - jepsen.util 11557 :invoke :read [12 nil] INFO [2019-12-17 12:37:34,195] jepsen worker 65 - jepsen.util 10505 :ok :write [21 32] INFO [2019-12-17 12:37:34,195] jepsen worker 65 - jepsen.util 10505 :invoke :cas [21 [38 38]] INFO [2019-12-17 12:37:34,211] jepsen worker 38 - jepsen.util 12458 :info :write [12 31] :RequestCanceledException INFO [2019-12-17 12:37:34,214] jepsen worker 38 - jepsen.util 12548 :invoke :cas [12 [26 6]] INFO [2019-12-17 12:37:34,219] jepsen worker 65 - jepsen.util 10505 :fail :cas [21 [38 38]] :ValueNotSwapFrom INFO [2019-12-17 12:37:34,219] jepsen worker 65 - jepsen.util 10505 :invoke :write [21 19] INFO [2019-12-17 12:37:34,222] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69723,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:37:34,222] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69731,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:37:34,222] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69723,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:37:34,223] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69731,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:37:34,223] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69724,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:37:34,223] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69735,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:37:34,223] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69724,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:37:34,223] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69735,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:37:34,223] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69725,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:37:34,223] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69725,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:37:34,234] jepsen worker 21 - jepsen.util 15861 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:37:34,234] jepsen worker 21 - jepsen.util 15861 :invoke :write [7 1] INFO [2019-12-17 12:37:34,273] jepsen worker 65 - jepsen.util 10505 :ok :write [21 19] INFO [2019-12-17 12:37:34,274] jepsen worker 65 - jepsen.util 10505 :invoke :read [21 nil] INFO [2019-12-17 12:37:34,286] jepsen worker 71 - jepsen.util 12851 :fail :cas [23 [13 33]] :ValueNotSwapFrom INFO [2019-12-17 12:37:34,286] jepsen worker 71 - jepsen.util 12851 :invoke :write [23 3] INFO [2019-12-17 12:37:34,295] jepsen worker 65 - jepsen.util 10505 :ok :read [21 19] INFO [2019-12-17 12:37:34,295] jepsen worker 65 - jepsen.util 10505 :invoke :write [21 32] INFO [2019-12-17 12:37:34,334] jepsen worker 59 - jepsen.util 15719 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:37:34,334] jepsen worker 59 - jepsen.util 15719 :invoke :read [19 nil] INFO [2019-12-17 12:37:34,337] jepsen worker 19 - jepsen.util 12439 :ok :write [6 33] INFO [2019-12-17 12:37:34,337] jepsen worker 73 - jepsen.util 11143 :ok :write [24 25] INFO [2019-12-17 12:37:34,337] jepsen worker 19 - jepsen.util 12439 :invoke :write [6 19] INFO [2019-12-17 12:37:34,337] jepsen worker 73 - jepsen.util 11143 :invoke :cas [24 [19 17]] INFO [2019-12-17 12:37:34,339] jepsen worker 11 - jepsen.util 11621 :ok :write [3 38] INFO [2019-12-17 12:37:34,339] jepsen worker 11 - jepsen.util 11621 :invoke :write [3 9] INFO [2019-12-17 12:37:34,352] jepsen worker 64 - jepsen.util 9244 :ok :read [21 19] INFO [2019-12-17 12:37:34,352] jepsen worker 64 - jepsen.util 9244 :invoke :write [21 9] INFO [2019-12-17 12:37:34,363] jepsen worker 78 - jepsen.util 11958 :ok :read [26 37] INFO [2019-12-17 12:37:34,363] jepsen worker 78 - jepsen.util 11958 :invoke :read [26 nil] INFO [2019-12-17 12:37:34,364] jepsen worker 0 - jepsen.util 16920 :info :write [0 40] :RequestCanceledException INFO [2019-12-17 12:37:34,365] jepsen worker 65 - jepsen.util 10505 :ok :write [21 32] INFO [2019-12-17 12:37:34,365] jepsen worker 65 - jepsen.util 10505 :invoke :write [21 21] INFO [2019-12-17 12:37:34,365] jepsen worker 0 - jepsen.util 17010 :invoke :cas [0 [14 19]] INFO [2019-12-17 12:37:34,395] jepsen worker 73 - jepsen.util 11143 :fail :cas [24 [19 17]] :ValueNotSwapFrom INFO [2019-12-17 12:37:34,395] jepsen worker 73 - jepsen.util 11143 :invoke :read [24 nil] INFO [2019-12-17 12:37:34,400] jepsen worker 70 - jepsen.util 12490 :info :write [23 21] :RequestCanceledException INFO [2019-12-17 12:37:34,400] jepsen worker 79 - jepsen.util 12139 :info :write [26 46] :RequestCanceledException INFO [2019-12-17 12:37:34,400] jepsen worker 47 - jepsen.util 12917 :info :write [15 31] :RequestCanceledException INFO [2019-12-17 12:37:34,402] jepsen worker 79 - jepsen.util 12229 :invoke :read [26 nil] INFO [2019-12-17 12:37:34,402] jepsen worker 47 - jepsen.util 13007 :invoke :read [15 nil] INFO [2019-12-17 12:37:34,404] jepsen worker 70 - jepsen.util 12580 :invoke :read [23 nil] INFO [2019-12-17 12:37:34,415] jepsen worker 14 - jepsen.util 11714 :info :write [4 14] :RequestCanceledException INFO [2019-12-17 12:37:34,416] jepsen worker 14 - jepsen.util 11804 :invoke :cas [4 [35 36]] INFO [2019-12-17 12:37:34,420] jepsen worker 89 - jepsen.util 11879 :ok :read [29 25] INFO [2019-12-17 12:37:34,420] jepsen worker 89 - jepsen.util 11879 :invoke :read [29 nil] INFO [2019-12-17 12:37:34,433] jepsen worker 2 - jepsen.util 16022 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:37:34,434] jepsen worker 2 - jepsen.util 16022 :invoke :read [0 nil] INFO [2019-12-17 12:37:34,442] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69756,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:37:34,442] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69756,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:37:34,442] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69760,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:37:34,442] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69760,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:37:34,442] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69761,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:37:34,442] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69761,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:37:34,447] jepsen worker 9 - jepsen.util 12429 :info :cas [3 [30 7]] :RequestCanceledException INFO [2019-12-17 12:37:34,447] jepsen worker 30 - jepsen.util 11010 :info :write [10 5] :RequestCanceledException INFO [2019-12-17 12:37:34,448] jepsen worker 32 - jepsen.util 12452 :info :cas [10 [9 48]] :RequestCanceledException INFO [2019-12-17 12:37:34,448] jepsen worker 65 - jepsen.util 10505 :ok :write [21 21] INFO [2019-12-17 12:37:34,448] jepsen worker 65 - jepsen.util 10505 :invoke :read [21 nil] INFO [2019-12-17 12:37:34,449] jepsen worker 32 - jepsen.util 12542 :invoke :read [10 nil] INFO [2019-12-17 12:37:34,449] jepsen worker 9 - jepsen.util 12519 :invoke :write [3 48] INFO [2019-12-17 12:37:34,449] jepsen worker 30 - jepsen.util 11100 :invoke :read [10 nil] INFO [2019-12-17 12:37:34,461] jepsen worker 78 - jepsen.util 11958 :ok :read [26 37] INFO [2019-12-17 12:37:34,462] jepsen worker 78 - jepsen.util 11958 :invoke :read [26 nil] INFO [2019-12-17 12:37:34,462] jepsen worker 88 - jepsen.util 12148 :fail :cas [29 [3 6]] :ValueNotSwapFrom INFO [2019-12-17 12:37:34,462] jepsen worker 88 - jepsen.util 12148 :invoke :cas [29 [32 39]] INFO [2019-12-17 12:37:34,476] jepsen worker 53 - jepsen.util 11573 :fail :cas [17 [18 20]] :ValueNotSwapFrom INFO [2019-12-17 12:37:34,476] jepsen worker 53 - jepsen.util 11573 :invoke :write [17 46] INFO [2019-12-17 12:37:34,485] jepsen worker 65 - jepsen.util 10505 :ok :read [21 21] INFO [2019-12-17 12:37:34,485] jepsen worker 65 - jepsen.util 10505 :invoke :cas [21 [14 19]] INFO [2019-12-17 12:37:34,500] jepsen worker 13 - jepsen.util 11353 :info :write [4 47] :RequestCanceledException INFO [2019-12-17 12:37:34,501] jepsen worker 13 - jepsen.util 11443 :invoke :write [4 38] INFO [2019-12-17 12:37:34,539] jepsen worker 88 - jepsen.util 12148 :fail :cas [29 [32 39]] :ValueNotSwapFrom INFO [2019-12-17 12:37:34,539] jepsen worker 88 - jepsen.util 12148 :invoke :write [29 18] INFO [2019-12-17 12:37:34,547] jepsen worker 36 - jepsen.util 12546 :info :write [12 13] :RequestCanceledException INFO [2019-12-17 12:37:34,548] jepsen worker 36 - jepsen.util 12636 :invoke :read [12 nil] INFO [2019-12-17 12:37:34,551] jepsen worker 3 - jepsen.util 11343 :ok :read [1 19] INFO [2019-12-17 12:37:34,551] jepsen worker 3 - jepsen.util 11343 :invoke :write [1 43] INFO [2019-12-17 12:37:34,564] jepsen worker 77 - jepsen.util 17447 :info :cas [25 [46 14]] :RequestCanceledException INFO [2019-12-17 12:37:34,565] jepsen worker 77 - jepsen.util 17537 :invoke :write [25 12] INFO [2019-12-17 12:37:34,590] jepsen worker 70 - jepsen.util 12580 :ok :read [23 29] INFO [2019-12-17 12:37:34,590] jepsen worker 70 - jepsen.util 12580 :invoke :write [23 9] INFO [2019-12-17 12:37:34,633] jepsen worker 25 - jepsen.util 17485 :info :cas [8 [33 27]] :RequestCanceledException INFO [2019-12-17 12:37:34,636] jepsen worker 25 - jepsen.util 17575 :invoke :write [8 13] INFO [2019-12-17 12:37:34,660] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69779,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:37:34,660] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69779,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:37:34,660] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69783,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:37:34,660] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69783,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:37:34,680] jepsen worker 78 - jepsen.util 11958 :ok :read [26 37] INFO [2019-12-17 12:37:34,680] jepsen worker 78 - jepsen.util 11958 :invoke :cas [26 [31 47]] INFO [2019-12-17 12:37:34,700] jepsen worker 16 - jepsen.util 18286 :info :write [5 25] :RequestCanceledException INFO [2019-12-17 12:37:34,701] jepsen worker 16 - jepsen.util 18376 :invoke :cas [5 [29 39]] INFO [2019-12-17 12:37:34,704] jepsen worker 36 - jepsen.util 12636 :ok :read [12 26] INFO [2019-12-17 12:37:34,704] jepsen worker 36 - jepsen.util 12636 :invoke :read [12 nil] INFO [2019-12-17 12:37:34,718] jepsen worker 5 - jepsen.util 11705 :info :write [1 33] :RequestCanceledException INFO [2019-12-17 12:37:34,719] jepsen worker 5 - jepsen.util 11795 :invoke :read [1 nil] INFO [2019-12-17 12:37:34,734] jepsen worker 35 - jepsen.util 16325 :info :write [11 24] :RequestCanceledException INFO [2019-12-17 12:37:34,736] jepsen worker 35 - jepsen.util 16415 :invoke :write [11 48] INFO [2019-12-17 12:37:34,739] jepsen worker 80 - jepsen.util 11600 :ok :read [26 37] INFO [2019-12-17 12:37:34,739] jepsen worker 80 - jepsen.util 11600 :invoke :cas [26 [0 33]] INFO [2019-12-17 12:37:34,741] jepsen worker 8 - jepsen.util 16208 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:37:34,741] jepsen worker 8 - jepsen.util 16208 :invoke :write [2 21] INFO [2019-12-17 12:37:34,760] jepsen worker 43 - jepsen.util 16693 :info :write [14 21] :RequestCanceledException INFO [2019-12-17 12:37:34,761] jepsen worker 48 - jepsen.util 15078 :info :cas [16 [17 17]] :RequestCanceledException INFO [2019-12-17 12:37:34,762] jepsen worker 48 - jepsen.util 15168 :invoke :cas [16 [32 19]] INFO [2019-12-17 12:37:34,762] jepsen worker 43 - jepsen.util 16783 :invoke :read [14 nil] INFO [2019-12-17 12:37:34,781] jepsen worker 14 - jepsen.util 11804 :fail :cas [4 [35 36]] :ValueNotSwapFrom INFO [2019-12-17 12:37:34,781] jepsen worker 14 - jepsen.util 11804 :invoke :cas [4 [19 10]] INFO [2019-12-17 12:37:34,784] jepsen worker 37 - jepsen.util 11557 :ok :read [12 26] INFO [2019-12-17 12:37:34,785] jepsen worker 37 - jepsen.util 11557 :invoke :write [12 11] INFO [2019-12-17 12:37:34,800] jepsen worker 81 - jepsen.util 16101 :info :cas [27 [33 2]] :RequestCanceledException INFO [2019-12-17 12:37:34,803] jepsen worker 81 - jepsen.util 16191 :invoke :write [27 45] INFO [2019-12-17 12:37:34,818] jepsen worker 44 - jepsen.util 17144 :info :write [14 19] :RequestCanceledException INFO [2019-12-17 12:37:34,821] jepsen worker 44 - jepsen.util 17234 :invoke :read [14 nil] INFO [2019-12-17 12:37:34,830] jepsen worker 78 - jepsen.util 11958 :fail :cas [26 [31 47]] :ValueNotSwapFrom INFO [2019-12-17 12:37:34,830] jepsen worker 78 - jepsen.util 11958 :invoke :read [26 nil] INFO [2019-12-17 12:37:34,833] jepsen worker 61 - jepsen.util 17791 :info :write [20 23] :RequestCanceledException INFO [2019-12-17 12:37:34,835] jepsen worker 61 - jepsen.util 17881 :invoke :write [20 0] INFO [2019-12-17 12:37:34,876] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69800,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:37:34,876] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69800,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:37:34,876] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69804,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:37:34,876] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69804,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:37:34,877] jepsen worker 36 - jepsen.util 12636 :ok :read [12 26] INFO [2019-12-17 12:37:34,877] jepsen worker 36 - jepsen.util 12636 :invoke :write [12 5] INFO [2019-12-17 12:37:34,918] jepsen worker 15 - jepsen.util 15315 :info :cas [5 [21 31]] :RequestCanceledException INFO [2019-12-17 12:37:34,920] jepsen worker 15 - jepsen.util 15405 :invoke :write [5 16] INFO [2019-12-17 12:37:34,934] jepsen worker 50 - jepsen.util 16520 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:37:34,934] jepsen worker 58 - jepsen.util 16798 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:37:34,934] jepsen worker 58 - jepsen.util 16798 :invoke :read [19 nil] INFO [2019-12-17 12:37:34,934] jepsen worker 50 - jepsen.util 16520 :invoke :write [16 43] INFO [2019-12-17 12:37:34,960] jepsen worker 5 - jepsen.util 11795 :ok :read [1 19] INFO [2019-12-17 12:37:34,960] jepsen worker 5 - jepsen.util 11795 :invoke :read [1 nil] INFO [2019-12-17 12:37:34,965] jepsen worker 7 - jepsen.util 16477 :info :cas [2 [48 4]] :RequestCanceledException INFO [2019-12-17 12:37:34,966] jepsen worker 7 - jepsen.util 16567 :invoke :write [2 2] INFO [2019-12-17 12:37:35,001] jepsen worker 89 - jepsen.util 11879 :ok :read [29 25] INFO [2019-12-17 12:37:35,001] jepsen worker 89 - jepsen.util 11879 :invoke :cas [29 [4 49]] INFO [2019-12-17 12:37:35,073] jepsen worker 9 - jepsen.util 12519 :ok :write [3 48] INFO [2019-12-17 12:37:35,073] jepsen worker 9 - jepsen.util 12519 :invoke :write [3 38] INFO [2019-12-17 12:37:35,089] jepsen worker 46 - jepsen.util 11836 :ok :write [15 37] INFO [2019-12-17 12:37:35,089] jepsen worker 46 - jepsen.util 11836 :invoke :cas [15 [32 15]] INFO [2019-12-17 12:37:35,093] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69813,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:37:35,094] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69813,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:37:35,094] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69816,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:37:35,094] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69816,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:37:35,094] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69817,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:37:35,094] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69817,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:37:35,108] jepsen worker 78 - jepsen.util 11958 :ok :read [26 37] INFO [2019-12-17 12:37:35,108] jepsen worker 78 - jepsen.util 11958 :invoke :cas [26 [41 8]] INFO [2019-12-17 12:37:35,112] jepsen worker 65 - jepsen.util 10505 :fail :cas [21 [14 19]] :ValueNotSwapFrom INFO [2019-12-17 12:37:35,113] jepsen worker 65 - jepsen.util 10505 :invoke :read [21 nil] INFO [2019-12-17 12:37:35,139] jepsen worker 32 - jepsen.util 12542 :ok :read [10 34] INFO [2019-12-17 12:37:35,139] jepsen worker 32 - jepsen.util 12542 :invoke :cas [10 [35 42]] INFO [2019-12-17 12:37:35,146] jepsen worker 46 - jepsen.util 11836 :fail :cas [15 [32 15]] :ValueNotSwapFrom INFO [2019-12-17 12:37:35,146] jepsen worker 46 - jepsen.util 11836 :invoke :cas [15 [17 26]] INFO [2019-12-17 12:37:35,165] jepsen worker 57 - jepsen.util 16167 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:37:35,165] jepsen worker 57 - jepsen.util 16167 :invoke :write [19 9] INFO [2019-12-17 12:37:35,170] jepsen worker 32 - jepsen.util 12542 :fail :cas [10 [35 42]] :ValueNotSwapFrom INFO [2019-12-17 12:37:35,170] jepsen worker 32 - jepsen.util 12542 :invoke :read [10 nil] INFO [2019-12-17 12:37:35,176] jepsen worker 46 - jepsen.util 11836 :fail :cas [15 [17 26]] :ValueNotSwapFrom INFO [2019-12-17 12:37:35,176] jepsen worker 46 - jepsen.util 11836 :invoke :read [15 nil] INFO [2019-12-17 12:37:35,193] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:37:35,194] jepsen nemesis - couchbase.util Rebalance check count: 550 INFO [2019-12-17 12:37:35,194] jepsen nemesis - couchbase.util Rebalance stuck count: 531 INFO [2019-12-17 12:37:35,200] jepsen worker 72 - jepsen.util 12222 :info :write [24 14] :RequestCanceledException INFO [2019-12-17 12:37:35,203] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69827,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:37:35,203] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69827,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:37:35,203] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69828,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:37:35,203] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69828,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:37:35,204] jepsen worker 72 - jepsen.util 12312 :invoke :cas [24 [17 21]] INFO [2019-12-17 12:37:35,206] jepsen worker 32 - jepsen.util 12542 :ok :read [10 34] INFO [2019-12-17 12:37:35,206] jepsen worker 32 - jepsen.util 12542 :invoke :cas [10 [5 34]] INFO [2019-12-17 12:37:35,212] jepsen worker 14 - jepsen.util 11804 :fail :cas [4 [19 10]] :ValueNotSwapFrom INFO [2019-12-17 12:37:35,212] jepsen worker 14 - jepsen.util 11804 :invoke :cas [4 [8 36]] INFO [2019-12-17 12:37:35,213] jepsen worker 46 - jepsen.util 11836 :ok :read [15 37] INFO [2019-12-17 12:37:35,213] jepsen worker 46 - jepsen.util 11836 :invoke :cas [15 [5 40]] INFO [2019-12-17 12:37:35,218] jepsen worker 10 - jepsen.util 11710 :info :cas [3 [20 12]] :RequestCanceledException INFO [2019-12-17 12:37:35,218] jepsen worker 42 - jepsen.util 17322 :info :write [14 20] :RequestCanceledException INFO [2019-12-17 12:37:35,219] jepsen worker 10 - jepsen.util 11800 :invoke :cas [3 [49 39]] INFO [2019-12-17 12:37:35,219] jepsen worker 42 - jepsen.util 17412 :invoke :write [14 36] INFO [2019-12-17 12:37:35,234] jepsen worker 52 - jepsen.util 11392 :info :write [17 21] :RequestCanceledException INFO [2019-12-17 12:37:35,234] jepsen worker 83 - jepsen.util 16733 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:37:35,234] jepsen worker 56 - jepsen.util 12926 :info :write [18 49] :RequestCanceledException INFO [2019-12-17 12:37:35,234] jepsen worker 1 - jepsen.util 16561 :info :cas [0 [23 35]] :RequestCanceledException INFO [2019-12-17 12:37:35,234] jepsen worker 83 - jepsen.util 16733 :invoke :cas [27 [46 27]] INFO [2019-12-17 12:37:35,235] jepsen worker 52 - jepsen.util 11482 :invoke :cas [17 [7 46]] INFO [2019-12-17 12:37:35,237] jepsen worker 56 - jepsen.util 13016 :invoke :read [18 nil] INFO [2019-12-17 12:37:35,238] jepsen worker 1 - jepsen.util 16651 :invoke :cas [0 [20 17]] INFO [2019-12-17 12:37:35,262] jepsen worker 32 - jepsen.util 12542 :fail :cas [10 [5 34]] :ValueNotSwapFrom INFO [2019-12-17 12:37:35,263] jepsen worker 32 - jepsen.util 12542 :invoke :cas [10 [11 21]] INFO [2019-12-17 12:37:35,265] jepsen worker 18 - jepsen.util 12258 :info :write [6 47] :RequestCanceledException INFO [2019-12-17 12:37:35,265] jepsen worker 22 - jepsen.util 16492 :info :cas [7 [3 14]] :RequestCanceledException INFO [2019-12-17 12:37:35,271] jepsen worker 22 - jepsen.util 16582 :invoke :cas [7 [3 18]] INFO [2019-12-17 12:37:35,272] jepsen worker 18 - jepsen.util 12348 :invoke :write [6 37] INFO [2019-12-17 12:37:35,277] jepsen worker 72 - jepsen.util 12312 :fail :cas [24 [17 21]] :ValueNotSwapFrom INFO [2019-12-17 12:37:35,278] jepsen worker 72 - jepsen.util 12312 :invoke :write [24 43] INFO [2019-12-17 12:37:35,287] jepsen worker 46 - jepsen.util 11836 :fail :cas [15 [5 40]] :ValueNotSwapFrom INFO [2019-12-17 12:37:35,287] jepsen worker 46 - jepsen.util 11836 :invoke :write [15 41] INFO [2019-12-17 12:37:35,298] jepsen worker 10 - jepsen.util 11800 :fail :cas [3 [49 39]] :ValueNotSwapFrom INFO [2019-12-17 12:37:35,298] jepsen worker 10 - jepsen.util 11800 :invoke :cas [3 [6 25]] INFO [2019-12-17 12:37:35,300] jepsen worker 51 - jepsen.util 11211 :info :write [17 40] :RequestCanceledException INFO [2019-12-17 12:37:35,307] jepsen worker 51 - jepsen.util 11301 :invoke :read [17 nil] INFO [2019-12-17 12:37:35,313] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69839,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:37:35,313] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69839,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:37:35,314] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69840,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:37:35,314] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69840,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:37:35,318] jepsen worker 39 - jepsen.util 16059 :info :cas [13 [44 23]] :RequestCanceledException INFO [2019-12-17 12:37:35,324] jepsen worker 32 - jepsen.util 12542 :fail :cas [10 [11 21]] :ValueNotSwapFrom INFO [2019-12-17 12:37:35,324] jepsen worker 32 - jepsen.util 12542 :invoke :write [10 26] INFO [2019-12-17 12:37:35,325] jepsen worker 39 - jepsen.util 16149 :invoke :read [13 nil] INFO [2019-12-17 12:37:35,340] jepsen worker 80 - jepsen.util 11600 :fail :cas [26 [0 33]] :ValueNotSwapFrom INFO [2019-12-17 12:37:35,340] jepsen worker 80 - jepsen.util 11600 :invoke :cas [26 [38 25]] INFO [2019-12-17 12:37:35,347] jepsen worker 55 - jepsen.util 12655 :info :write [18 47] :RequestCanceledException INFO [2019-12-17 12:37:35,349] jepsen worker 55 - jepsen.util 12745 :invoke :write [18 38] INFO [2019-12-17 12:37:35,367] jepsen worker 79 - jepsen.util 12229 :ok :read [26 37] INFO [2019-12-17 12:37:35,367] jepsen worker 79 - jepsen.util 12229 :invoke :write [26 20] INFO [2019-12-17 12:37:35,388] jepsen worker 19 - jepsen.util 12439 :ok :write [6 19] INFO [2019-12-17 12:37:35,388] jepsen worker 19 - jepsen.util 12439 :invoke :cas [6 [2 23]] INFO [2019-12-17 12:37:35,390] jepsen worker 9 - jepsen.util 12519 :ok :write [3 38] INFO [2019-12-17 12:37:35,390] jepsen worker 9 - jepsen.util 12519 :invoke :cas [3 [44 36]] INFO [2019-12-17 12:37:35,400] jepsen worker 76 - jepsen.util 16726 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:37:35,400] jepsen worker 76 - jepsen.util 16726 :invoke :cas [25 [47 37]] INFO [2019-12-17 12:37:35,402] jepsen worker 10 - jepsen.util 11800 :fail :cas [3 [6 25]] :ValueNotSwapFrom INFO [2019-12-17 12:37:35,402] jepsen worker 10 - jepsen.util 11800 :invoke :cas [3 [15 33]] INFO [2019-12-17 12:37:35,418] jepsen worker 68 - jepsen.util 15908 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:37:35,418] jepsen worker 68 - jepsen.util 15908 :invoke :write [22 31] INFO [2019-12-17 12:37:35,462] jepsen worker 49 - jepsen.util 16609 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:37:35,462] jepsen worker 72 - jepsen.util 12312 :ok :write [24 43] INFO [2019-12-17 12:37:35,462] jepsen worker 49 - jepsen.util 16609 :invoke :write [16 1] INFO [2019-12-17 12:37:35,462] jepsen worker 72 - jepsen.util 12312 :invoke :cas [24 [29 15]] INFO [2019-12-17 12:37:35,468] jepsen worker 30 - jepsen.util 11100 :ok :read [10 34] INFO [2019-12-17 12:37:35,468] jepsen worker 30 - jepsen.util 11100 :invoke :read [10 nil] INFO [2019-12-17 12:37:35,474] jepsen worker 65 - jepsen.util 10505 :ok :read [21 21] INFO [2019-12-17 12:37:35,474] jepsen worker 65 - jepsen.util 10505 :invoke :write [21 40] INFO [2019-12-17 12:37:35,478] jepsen worker 46 - jepsen.util 11836 :ok :write [15 41] INFO [2019-12-17 12:37:35,478] jepsen worker 46 - jepsen.util 11836 :invoke :cas [15 [15 11]] INFO [2019-12-17 12:37:35,495] jepsen worker 9 - jepsen.util 12519 :fail :cas [3 [44 36]] :ValueNotSwapFrom INFO [2019-12-17 12:37:35,495] jepsen worker 9 - jepsen.util 12519 :invoke :write [3 2] INFO [2019-12-17 12:37:35,500] jepsen worker 54 - jepsen.util 12474 :info :write [18 30] :RequestCanceledException INFO [2019-12-17 12:37:35,508] jepsen worker 54 - jepsen.util 12564 :invoke :write [18 36] INFO [2019-12-17 12:37:35,528] jepsen worker 37 - jepsen.util 11557 :ok :write [12 11] INFO [2019-12-17 12:37:35,529] jepsen worker 37 - jepsen.util 11557 :invoke :write [12 13] INFO [2019-12-17 12:37:35,535] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69866,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:37:35,535] jepsen worker 6 - jepsen.util 16836 :info :cas [2 [14 7]] :RequestCanceledException INFO [2019-12-17 12:37:35,536] jepsen worker 45 - jepsen.util 12015 :info :cas [15 [45 28]] :RequestCanceledException INFO [2019-12-17 12:37:35,538] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69866,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:37:35,538] jepsen worker 19 - jepsen.util 12439 :fail :cas [6 [2 23]] :ValueNotSwapFrom INFO [2019-12-17 12:37:35,539] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69870,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:37:35,539] jepsen worker 19 - jepsen.util 12439 :invoke :write [6 0] INFO [2019-12-17 12:37:35,539] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69870,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:37:35,542] jepsen worker 45 - jepsen.util 12105 :invoke :cas [15 [20 38]] INFO [2019-12-17 12:37:35,545] jepsen worker 6 - jepsen.util 16926 :invoke :read [2 nil] INFO [2019-12-17 12:37:35,564] jepsen worker 69 - jepsen.util 12219 :info :write [23 21] :RequestCanceledException INFO [2019-12-17 12:37:35,573] jepsen worker 72 - jepsen.util 12312 :fail :cas [24 [29 15]] :ValueNotSwapFrom INFO [2019-12-17 12:37:35,573] jepsen worker 72 - jepsen.util 12312 :invoke :read [24 nil] INFO [2019-12-17 12:37:35,573] jepsen worker 69 - jepsen.util 12309 :invoke :cas [23 [0 36]] INFO [2019-12-17 12:37:35,581] jepsen worker 30 - jepsen.util 11100 :ok :read [10 34] INFO [2019-12-17 12:37:35,581] jepsen worker 30 - jepsen.util 11100 :invoke :write [10 17] INFO [2019-12-17 12:37:35,588] jepsen worker 10 - jepsen.util 11800 :fail :cas [3 [15 33]] :ValueNotSwapFrom INFO [2019-12-17 12:37:35,589] jepsen worker 10 - jepsen.util 11800 :invoke :cas [3 [43 31]] INFO [2019-12-17 12:37:35,600] jepsen worker 46 - jepsen.util 11836 :fail :cas [15 [15 11]] :ValueNotSwapFrom INFO [2019-12-17 12:37:35,600] jepsen worker 46 - jepsen.util 11836 :invoke :write [15 3] INFO [2019-12-17 12:37:35,613] jepsen worker 45 - jepsen.util 12105 :fail :cas [15 [20 38]] :ValueNotSwapFrom INFO [2019-12-17 12:37:35,613] jepsen worker 45 - jepsen.util 12105 :invoke :read [15 nil] INFO [2019-12-17 12:37:35,627] jepsen worker 72 - jepsen.util 12312 :ok :read [24 43] INFO [2019-12-17 12:37:35,627] jepsen worker 72 - jepsen.util 12312 :invoke :write [24 44] INFO [2019-12-17 12:37:35,633] jepsen worker 74 - jepsen.util 11774 :fail :read [24 nil] :RequestTimeoutException INFO [2019-12-17 12:37:35,633] jepsen worker 40 - jepsen.util 17140 :info :write [13 25] :RequestCanceledException INFO [2019-12-17 12:37:35,634] jepsen worker 74 - jepsen.util 11774 :invoke :write [24 3] INFO [2019-12-17 12:37:35,637] jepsen worker 40 - jepsen.util 17230 :invoke :write [13 33] INFO [2019-12-17 12:37:35,648] jepsen worker 31 - jepsen.util 10111 :info :write [10 42] :RequestCanceledException INFO [2019-12-17 12:37:35,648] jepsen worker 20 - jepsen.util 11720 :info :cas [6 [42 44]] :RequestCanceledException INFO [2019-12-17 12:37:35,649] jepsen worker 20 - jepsen.util 11810 :invoke :read [6 nil] INFO [2019-12-17 12:37:35,653] jepsen worker 31 - jepsen.util 10201 :invoke :cas [10 [33 33]] INFO [2019-12-17 12:37:35,668] jepsen worker 51 - jepsen.util 11301 :ok :read [17 19] INFO [2019-12-17 12:37:35,668] jepsen worker 51 - jepsen.util 11301 :invoke :write [17 26] INFO [2019-12-17 12:37:35,685] jepsen worker 45 - jepsen.util 12105 :ok :read [15 41] INFO [2019-12-17 12:37:35,685] jepsen worker 45 - jepsen.util 12105 :invoke :read [15 nil] INFO [2019-12-17 12:37:35,697] jepsen worker 10 - jepsen.util 11800 :fail :cas [3 [43 31]] :ValueNotSwapFrom INFO [2019-12-17 12:37:35,697] jepsen worker 10 - jepsen.util 11800 :invoke :read [3 nil] INFO [2019-12-17 12:37:35,700] jepsen worker 34 - jepsen.util 17134 :info :cas [11 [21 49]] :RequestCanceledException INFO [2019-12-17 12:37:35,702] jepsen worker 34 - jepsen.util 17224 :invoke :read [11 nil] INFO [2019-12-17 12:37:35,715] jepsen worker 24 - jepsen.util 15504 :info :write [8 27] :RequestCanceledException INFO [2019-12-17 12:37:35,721] jepsen worker 24 - jepsen.util 15594 :invoke :cas [8 [14 41]] INFO [2019-12-17 12:37:35,727] jepsen worker 65 - jepsen.util 10505 :ok :write [21 40] INFO [2019-12-17 12:37:35,727] jepsen worker 65 - jepsen.util 10505 :invoke :write [21 46] INFO [2019-12-17 12:37:35,748] jepsen worker 41 - jepsen.util 17771 :info :cas [13 [23 25]] :RequestCanceledException INFO [2019-12-17 12:37:35,748] jepsen worker 28 - jepsen.util 10198 :fail :read [9 nil] :RequestTimeoutException INFO [2019-12-17 12:37:35,748] jepsen worker 28 - jepsen.util 10198 :invoke :write [9 6] INFO [2019-12-17 12:37:35,750] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69893,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:37:35,750] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69893,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:37:35,750] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69896,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:37:35,751] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69896,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:37:35,754] jepsen worker 41 - jepsen.util 17861 :invoke :cas [13 [31 49]] INFO [2019-12-17 12:37:35,764] jepsen worker 31 - jepsen.util 10201 :fail :cas [10 [33 33]] :ValueNotSwapFrom INFO [2019-12-17 12:37:35,764] jepsen worker 31 - jepsen.util 10201 :invoke :cas [10 [32 17]] INFO [2019-12-17 12:37:35,775] jepsen worker 20 - jepsen.util 11810 :ok :read [6 19] INFO [2019-12-17 12:37:35,775] jepsen worker 20 - jepsen.util 11810 :invoke :read [6 nil] INFO [2019-12-17 12:37:35,796] jepsen worker 12 - jepsen.util 12252 :info :write [4 37] :RequestCanceledException INFO [2019-12-17 12:37:35,799] jepsen worker 12 - jepsen.util 12342 :invoke :read [4 nil] INFO [2019-12-17 12:37:35,817] jepsen worker 20 - jepsen.util 11810 :ok :read [6 19] INFO [2019-12-17 12:37:35,818] jepsen worker 20 - jepsen.util 11810 :invoke :cas [6 [0 24]] INFO [2019-12-17 12:37:35,828] jepsen worker 5 - jepsen.util 11795 :ok :read [1 19] INFO [2019-12-17 12:37:35,828] jepsen worker 5 - jepsen.util 11795 :invoke :cas [1 [24 29]] INFO [2019-12-17 12:37:35,835] jepsen worker 31 - jepsen.util 10201 :fail :cas [10 [32 17]] :ValueNotSwapFrom INFO [2019-12-17 12:37:35,835] jepsen worker 31 - jepsen.util 10201 :invoke :read [10 nil] INFO [2019-12-17 12:37:35,848] jepsen worker 60 - jepsen.util 17610 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:37:35,848] jepsen worker 60 - jepsen.util 17610 :invoke :read [20 nil] INFO [2019-12-17 12:37:35,879] jepsen worker 20 - jepsen.util 11810 :fail :cas [6 [0 24]] :ValueNotSwapFrom INFO [2019-12-17 12:37:35,879] jepsen worker 20 - jepsen.util 11810 :invoke :write [6 46] INFO [2019-12-17 12:37:35,882] jepsen worker 78 - jepsen.util 11958 :fail :cas [26 [41 8]] :ValueNotSwapFrom INFO [2019-12-17 12:37:35,883] jepsen worker 78 - jepsen.util 11958 :invoke :cas [26 [35 33]] INFO [2019-12-17 12:37:35,895] jepsen worker 31 - jepsen.util 10201 :ok :read [10 34] INFO [2019-12-17 12:37:35,896] jepsen worker 31 - jepsen.util 10201 :invoke :write [10 48] INFO [2019-12-17 12:37:35,915] jepsen worker 62 - jepsen.util 17162 :info :write [20 0] :RequestCanceledException INFO [2019-12-17 12:37:35,915] jepsen worker 17 - jepsen.util 16577 :info :write [5 41] :RequestCanceledException INFO [2019-12-17 12:37:35,917] jepsen worker 17 - jepsen.util 16667 :invoke :write [5 19] INFO [2019-12-17 12:37:35,918] jepsen worker 62 - jepsen.util 17252 :invoke :write [20 4] INFO [2019-12-17 12:37:35,933] jepsen worker 27 - jepsen.util 11007 :info :write [9 2] :RequestCanceledException INFO [2019-12-17 12:37:35,935] jepsen worker 27 - jepsen.util 11097 :invoke :read [9 nil] INFO [2019-12-17 12:37:35,949] jepsen worker 19 - jepsen.util 12439 :ok :write [6 0] INFO [2019-12-17 12:37:35,949] jepsen worker 19 - jepsen.util 12439 :invoke :write [6 5] INFO [2019-12-17 12:37:35,961] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69916,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:37:35,962] jepsen worker 75 - jepsen.util 16185 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:37:35,962] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69916,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:37:35,962] jepsen worker 75 - jepsen.util 16185 :invoke :cas [25 [47 36]] INFO [2019-12-17 12:37:35,962] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69921,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:37:35,962] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69921,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:37:35,989] jepsen worker 52 - jepsen.util 11482 :fail :cas [17 [7 46]] :ValueNotSwapFrom INFO [2019-12-17 12:37:35,989] jepsen worker 52 - jepsen.util 11482 :invoke :read [17 nil] INFO [2019-12-17 12:37:36,017] jepsen worker 69 - jepsen.util 12309 :fail :cas [23 [0 36]] :ValueNotSwapFrom INFO [2019-12-17 12:37:36,017] jepsen worker 69 - jepsen.util 12309 :invoke :read [23 nil] INFO [2019-12-17 12:37:36,033] jepsen worker 86 - jepsen.util 17186 :info :write [28 34] :RequestCanceledException INFO [2019-12-17 12:37:36,033] jepsen worker 26 - jepsen.util 17126 :info :cas [8 [38 4]] :RequestCanceledException INFO [2019-12-17 12:37:36,036] jepsen worker 86 - jepsen.util 17276 :invoke :read [28 nil] INFO [2019-12-17 12:37:36,036] jepsen worker 26 - jepsen.util 17216 :invoke :cas [8 [30 35]] INFO [2019-12-17 12:37:36,051] jepsen worker 89 - jepsen.util 11879 :fail :cas [29 [4 49]] :ValueNotSwapFrom INFO [2019-12-17 12:37:36,051] jepsen worker 89 - jepsen.util 11879 :invoke :write [29 49] INFO [2019-12-17 12:37:36,083] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69935,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:37:36,083] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69935,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:37:36,084] jepsen worker 56 - jepsen.util 13016 :ok :read [18 3] INFO [2019-12-17 12:37:36,084] jepsen worker 56 - jepsen.util 13016 :invoke :cas [18 [41 18]] INFO [2019-12-17 12:37:36,100] jepsen worker 4 - jepsen.util 11164 :info :write [1 10] :RequestCanceledException INFO [2019-12-17 12:37:36,103] jepsen worker 4 - jepsen.util 11254 :invoke :cas [1 [17 38]] INFO [2019-12-17 12:37:36,114] jepsen worker 12 - jepsen.util 12342 :ok :read [4 8] INFO [2019-12-17 12:37:36,114] jepsen worker 12 - jepsen.util 12342 :invoke :read [4 nil] INFO [2019-12-17 12:37:36,118] jepsen worker 85 - jepsen.util 17275 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:37:36,118] jepsen worker 63 - jepsen.util 10413 :info :cas [21 [5 33]] :RequestCanceledException INFO [2019-12-17 12:37:36,118] jepsen worker 85 - jepsen.util 17275 :invoke :read [28 nil] INFO [2019-12-17 12:37:36,120] jepsen worker 63 - jepsen.util 10503 :invoke :read [21 nil] INFO [2019-12-17 12:37:36,133] jepsen worker 87 - jepsen.util 11517 :info :write [29 49] :RequestCanceledException INFO [2019-12-17 12:37:36,133] jepsen worker 66 - jepsen.util 16446 :info :write [22 6] :RequestCanceledException INFO [2019-12-17 12:37:36,136] jepsen worker 66 - jepsen.util 16536 :invoke :read [22 nil] INFO [2019-12-17 12:37:36,137] jepsen worker 87 - jepsen.util 11607 :invoke :read [29 nil] INFO [2019-12-17 12:37:36,152] jepsen worker 5 - jepsen.util 11795 :fail :cas [1 [24 29]] :ValueNotSwapFrom INFO [2019-12-17 12:37:36,153] jepsen worker 5 - jepsen.util 11795 :invoke :write [1 19] INFO [2019-12-17 12:37:36,155] jepsen worker 63 - jepsen.util 10503 :ok :read [21 40] INFO [2019-12-17 12:37:36,155] jepsen worker 63 - jepsen.util 10503 :invoke :read [21 nil] INFO [2019-12-17 12:37:36,165] jepsen worker 29 - jepsen.util 10469 :info :write [9 49] :RequestCanceledException INFO [2019-12-17 12:37:36,167] jepsen worker 29 - jepsen.util 10559 :invoke :write [9 32] INFO [2019-12-17 12:37:36,189] jepsen worker 63 - jepsen.util 10503 :ok :read [21 40] INFO [2019-12-17 12:37:36,190] jepsen worker 63 - jepsen.util 10503 :invoke :cas [21 [17 1]] INFO [2019-12-17 12:37:36,200] jepsen worker 33 - jepsen.util 16323 :info :cas [11 [36 47]] :RequestCanceledException INFO [2019-12-17 12:37:36,203] jepsen worker 33 - jepsen.util 16413 :invoke :write [11 24] INFO [2019-12-17 12:37:36,210] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:37:36,210] jepsen nemesis - couchbase.util Rebalance check count: 551 INFO [2019-12-17 12:37:36,210] jepsen nemesis - couchbase.util Rebalance stuck count: 532 INFO [2019-12-17 12:37:36,218] jepsen worker 67 - jepsen.util 16537 :info :write [22 14] :RequestCanceledException INFO [2019-12-17 12:37:36,223] jepsen worker 67 - jepsen.util 16627 :invoke :cas [22 [24 21]] INFO [2019-12-17 12:37:36,231] jepsen worker 52 - jepsen.util 11482 :ok :read [17 19] INFO [2019-12-17 12:37:36,231] jepsen worker 52 - jepsen.util 11482 :invoke :cas [17 [31 43]] INFO [2019-12-17 12:37:36,233] jepsen worker 84 - jepsen.util 16194 :info :write [28 7] :RequestCanceledException INFO [2019-12-17 12:37:36,233] jepsen worker 82 - jepsen.util 16822 :info :cas [27 [28 24]] :RequestCanceledException INFO [2019-12-17 12:37:36,234] jepsen worker 63 - jepsen.util 10503 :fail :cas [21 [17 1]] :ValueNotSwapFrom INFO [2019-12-17 12:37:36,234] jepsen worker 63 - jepsen.util 10503 :invoke :write [21 36] INFO [2019-12-17 12:37:36,236] jepsen worker 82 - jepsen.util 16912 :invoke :write [27 15] INFO [2019-12-17 12:37:36,237] jepsen worker 84 - jepsen.util 16284 :invoke :read [28 nil] INFO [2019-12-17 12:37:36,247] jepsen worker 23 - jepsen.util 16943 :info :cas [7 [19 18]] :RequestCanceledException INFO [2019-12-17 12:37:36,247] jepsen worker 80 - jepsen.util 11600 :fail :cas [26 [38 25]] :ValueNotSwapFrom INFO [2019-12-17 12:37:36,247] jepsen worker 80 - jepsen.util 11600 :invoke :read [26 nil] INFO [2019-12-17 12:37:36,251] jepsen worker 23 - jepsen.util 17033 :invoke :read [7 nil] INFO [2019-12-17 12:37:36,300] jepsen worker 63 - jepsen.util 10503 :ok :write [21 36] INFO [2019-12-17 12:37:36,300] jepsen worker 71 - jepsen.util 12851 :info :write [23 3] :RequestCanceledException INFO [2019-12-17 12:37:36,301] jepsen worker 63 - jepsen.util 10503 :invoke :write [21 33] INFO [2019-12-17 12:37:36,303] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69952,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:37:36,303] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69952,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:37:36,303] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69960,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:37:36,304] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69954,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:37:36,304] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69960,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:37:36,304] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69954,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:37:36,304] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69963,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:37:36,304] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69963,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:37:36,305] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69964,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:37:36,305] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69964,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:37:36,305] jepsen worker 71 - jepsen.util 12941 :invoke :read [23 nil] INFO [2019-12-17 12:37:36,333] jepsen worker 21 - jepsen.util 15861 :info :write [7 1] :RequestCanceledException INFO [2019-12-17 12:37:36,334] jepsen worker 87 - jepsen.util 11607 :ok :read [29 25] INFO [2019-12-17 12:37:36,334] jepsen worker 87 - jepsen.util 11607 :invoke :write [29 2] INFO [2019-12-17 12:37:36,336] jepsen worker 21 - jepsen.util 15951 :invoke :write [7 16] INFO [2019-12-17 12:37:36,364] jepsen worker 4 - jepsen.util 11254 :fail :cas [1 [17 38]] :ValueNotSwapFrom INFO [2019-12-17 12:37:36,364] jepsen worker 4 - jepsen.util 11254 :invoke :write [1 43] INFO [2019-12-17 12:37:36,380] jepsen worker 63 - jepsen.util 10503 :ok :write [21 33] INFO [2019-12-17 12:37:36,381] jepsen worker 63 - jepsen.util 10503 :invoke :cas [21 [48 20]] INFO [2019-12-17 12:37:36,414] jepsen worker 63 - jepsen.util 10503 :fail :cas [21 [48 20]] :ValueNotSwapFrom INFO [2019-12-17 12:37:36,414] jepsen worker 63 - jepsen.util 10503 :invoke :read [21 nil] INFO [2019-12-17 12:37:36,416] jepsen worker 47 - jepsen.util 13007 :fail :read [15 nil] :RequestTimeoutException INFO [2019-12-17 12:37:36,416] jepsen worker 11 - jepsen.util 11621 :info :write [3 9] :RequestCanceledException INFO [2019-12-17 12:37:36,417] jepsen worker 47 - jepsen.util 13007 :invoke :write [15 43] INFO [2019-12-17 12:37:36,420] jepsen worker 11 - jepsen.util 11711 :invoke :write [3 19] INFO [2019-12-17 12:37:36,423] jepsen worker 12 - jepsen.util 12342 :ok :read [4 8] INFO [2019-12-17 12:37:36,423] jepsen worker 12 - jepsen.util 12342 :invoke :cas [4 [7 47]] INFO [2019-12-17 12:37:36,434] jepsen worker 0 - jepsen.util 17010 :info :cas [0 [14 19]] :RequestCanceledException INFO [2019-12-17 12:37:36,434] jepsen worker 59 - jepsen.util 15719 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:37:36,434] jepsen worker 59 - jepsen.util 15719 :invoke :read [19 nil] INFO [2019-12-17 12:37:36,440] jepsen worker 0 - jepsen.util 17100 :invoke :write [0 30] INFO [2019-12-17 12:37:36,448] jepsen worker 64 - jepsen.util 9244 :info :write [21 9] :RequestCanceledException INFO [2019-12-17 12:37:36,448] jepsen worker 73 - jepsen.util 11143 :fail :read [24 nil] :RequestTimeoutException INFO [2019-12-17 12:37:36,448] jepsen worker 73 - jepsen.util 11143 :invoke :read [24 nil] INFO [2019-12-17 12:37:36,452] jepsen worker 64 - jepsen.util 9334 :invoke :write [21 17] INFO [2019-12-17 12:37:36,465] jepsen worker 63 - jepsen.util 10503 :ok :read [21 33] INFO [2019-12-17 12:37:36,465] jepsen worker 63 - jepsen.util 10503 :invoke :read [21 nil] INFO [2019-12-17 12:37:36,496] jepsen worker 63 - jepsen.util 10503 :ok :read [21 33] INFO [2019-12-17 12:37:36,496] jepsen worker 63 - jepsen.util 10503 :invoke :read [21 nil] INFO [2019-12-17 12:37:36,505] jepsen worker 52 - jepsen.util 11482 :fail :cas [17 [31 43]] :ValueNotSwapFrom INFO [2019-12-17 12:37:36,505] jepsen worker 52 - jepsen.util 11482 :invoke :write [17 43] INFO [2019-12-17 12:37:36,518] jepsen worker 13 - jepsen.util 11443 :info :write [4 38] :RequestCanceledException INFO [2019-12-17 12:37:36,521] jepsen worker 13 - jepsen.util 11533 :invoke :cas [4 [25 11]] INFO [2019-12-17 12:37:36,521] jepsen worker 63 - jepsen.util 10503 :ok :read [21 33] INFO [2019-12-17 12:37:36,522] jepsen worker 63 - jepsen.util 10503 :invoke :write [21 29] INFO [2019-12-17 12:37:36,522] jepsen worker 31 - jepsen.util 10201 :ok :write [10 48] INFO [2019-12-17 12:37:36,522] jepsen worker 31 - jepsen.util 10201 :invoke :write [10 10] INFO [2019-12-17 12:37:36,523] jepsen worker 9 - jepsen.util 12519 :ok :write [3 2] INFO [2019-12-17 12:37:36,523] jepsen worker 9 - jepsen.util 12519 :invoke :write [3 29] INFO [2019-12-17 12:37:36,523] jepsen worker 46 - jepsen.util 11836 :ok :write [15 3] INFO [2019-12-17 12:37:36,523] jepsen worker 46 - jepsen.util 11836 :invoke :read [15 nil] INFO [2019-12-17 12:37:36,526] jepsen worker 72 - jepsen.util 12312 :ok :write [24 44] INFO [2019-12-17 12:37:36,526] jepsen worker 72 - jepsen.util 12312 :invoke :read [24 nil] INFO [2019-12-17 12:37:36,528] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69984,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:37:36,529] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69984,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:37:36,529] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69988,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:37:36,529] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":69988,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:37:36,534] jepsen worker 53 - jepsen.util 11573 :info :write [17 46] :RequestCanceledException INFO [2019-12-17 12:37:36,534] jepsen worker 2 - jepsen.util 16022 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:37:36,534] jepsen worker 2 - jepsen.util 16022 :invoke :read [0 nil] INFO [2019-12-17 12:37:36,538] jepsen worker 53 - jepsen.util 11663 :invoke :write [17 5] INFO [2019-12-17 12:37:36,613] jepsen worker 28 - jepsen.util 10198 :ok :write [9 6] INFO [2019-12-17 12:37:36,613] jepsen worker 28 - jepsen.util 10198 :invoke :cas [9 [44 13]] INFO [2019-12-17 12:37:36,620] jepsen worker 78 - jepsen.util 11958 :fail :cas [26 [35 33]] :ValueNotSwapFrom INFO [2019-12-17 12:37:36,620] jepsen worker 78 - jepsen.util 11958 :invoke :cas [26 [0 45]] INFO [2019-12-17 12:37:36,627] jepsen worker 65 - jepsen.util 10505 :ok :write [21 46] INFO [2019-12-17 12:37:36,627] jepsen worker 65 - jepsen.util 10505 :invoke :read [21 nil] INFO [2019-12-17 12:37:36,633] jepsen worker 70 - jepsen.util 12580 :info :write [23 9] :RequestCanceledException INFO [2019-12-17 12:37:36,633] jepsen worker 88 - jepsen.util 12148 :info :write [29 18] :RequestCanceledException INFO [2019-12-17 12:37:36,636] jepsen worker 88 - jepsen.util 12238 :invoke :write [29 6] INFO [2019-12-17 12:37:36,636] jepsen worker 70 - jepsen.util 12670 :invoke :cas [23 [3 43]] INFO [2019-12-17 12:37:36,645] jepsen worker 3 - jepsen.util 11343 :info :write [1 43] :RequestCanceledException INFO [2019-12-17 12:37:36,645] jepsen worker 77 - jepsen.util 17537 :info :write [25 12] :RequestCanceledException INFO [2019-12-17 12:37:36,648] jepsen worker 3 - jepsen.util 11433 :invoke :cas [1 [6 37]] INFO [2019-12-17 12:37:36,649] jepsen worker 77 - jepsen.util 17627 :invoke :cas [25 [16 4]] INFO [2019-12-17 12:37:36,654] jepsen worker 73 - jepsen.util 11143 :ok :read [24 44] INFO [2019-12-17 12:37:36,655] jepsen worker 73 - jepsen.util 11143 :invoke :write [24 34] INFO [2019-12-17 12:37:36,661] jepsen worker 38 - jepsen.util 12548 :info :cas [12 [26 6]] :RequestCanceledException INFO [2019-12-17 12:37:36,661] jepsen worker 25 - jepsen.util 17575 :info :write [8 13] :RequestCanceledException INFO [2019-12-17 12:37:36,665] jepsen worker 38 - jepsen.util 12638 :invoke :cas [12 [1 40]] INFO [2019-12-17 12:37:36,666] jepsen worker 25 - jepsen.util 17665 :invoke :write [8 4] INFO [2019-12-17 12:37:36,687] jepsen worker 72 - jepsen.util 12312 :ok :read [24 44] INFO [2019-12-17 12:37:36,687] jepsen worker 72 - jepsen.util 12312 :invoke :write [24 36] INFO [2019-12-17 12:37:36,698] jepsen worker 87 - jepsen.util 11607 :ok :write [29 2] INFO [2019-12-17 12:37:36,698] jepsen worker 87 - jepsen.util 11607 :invoke :write [29 36] INFO [2019-12-17 12:37:36,707] jepsen worker 79 - jepsen.util 12229 :ok :write [26 20] INFO [2019-12-17 12:37:36,707] jepsen worker 79 - jepsen.util 12229 :invoke :cas [26 [24 13]] INFO [2019-12-17 12:37:36,708] jepsen worker 29 - jepsen.util 10559 :ok :write [9 32] INFO [2019-12-17 12:37:36,709] jepsen worker 29 - jepsen.util 10559 :invoke :cas [9 [35 29]] INFO [2019-12-17 12:37:36,714] jepsen worker 28 - jepsen.util 10198 :fail :cas [9 [44 13]] :ValueNotSwapFrom INFO [2019-12-17 12:37:36,715] jepsen worker 28 - jepsen.util 10198 :invoke :read [9 nil] INFO [2019-12-17 12:37:36,726] jepsen worker 16 - jepsen.util 18376 :info :cas [5 [29 39]] :RequestCanceledException INFO [2019-12-17 12:37:36,731] jepsen worker 16 - jepsen.util 18466 :invoke :read [5 nil] INFO [2019-12-17 12:37:36,736] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":70008,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:37:36,737] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":70008,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:37:36,737] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":70011,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:37:36,737] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":70011,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:37:36,737] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":70012,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:37:36,738] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":70012,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:37:36,752] jepsen worker 29 - jepsen.util 10559 :fail :cas [9 [35 29]] :ValueNotSwapFrom INFO [2019-12-17 12:37:36,752] jepsen worker 29 - jepsen.util 10559 :invoke :cas [9 [18 43]] INFO [2019-12-17 12:37:36,772] jepsen worker 69 - jepsen.util 12309 :ok :read [23 29] INFO [2019-12-17 12:37:36,772] jepsen worker 69 - jepsen.util 12309 :invoke :write [23 9] INFO [2019-12-17 12:37:36,786] jepsen worker 29 - jepsen.util 10559 :fail :cas [9 [18 43]] :ValueNotSwapFrom INFO [2019-12-17 12:37:36,786] jepsen worker 29 - jepsen.util 10559 :invoke :cas [9 [33 5]] INFO [2019-12-17 12:37:36,807] jepsen worker 29 - jepsen.util 10559 :fail :cas [9 [33 5]] :ValueNotSwapFrom INFO [2019-12-17 12:37:36,807] jepsen worker 29 - jepsen.util 10559 :invoke :write [9 18] INFO [2019-12-17 12:37:36,812] jepsen worker 11 - jepsen.util 11711 :ok :write [3 19] INFO [2019-12-17 12:37:36,812] jepsen worker 11 - jepsen.util 11711 :invoke :write [3 6] INFO [2019-12-17 12:37:36,815] jepsen worker 35 - jepsen.util 16415 :info :write [11 48] :RequestCanceledException INFO [2019-12-17 12:37:36,817] jepsen worker 35 - jepsen.util 16505 :invoke :read [11 nil] INFO [2019-12-17 12:37:36,833] jepsen worker 44 - jepsen.util 17234 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:37:36,833] jepsen worker 44 - jepsen.util 17234 :invoke :read [14 nil] INFO [2019-12-17 12:37:36,834] jepsen worker 48 - jepsen.util 15168 :info :cas [16 [32 19]] :RequestCanceledException INFO [2019-12-17 12:37:36,838] jepsen worker 48 - jepsen.util 15258 :invoke :write [16 14] INFO [2019-12-17 12:37:36,838] jepsen worker 73 - jepsen.util 11143 :ok :write [24 34] INFO [2019-12-17 12:37:36,838] jepsen worker 73 - jepsen.util 11143 :invoke :cas [24 [16 18]] INFO [2019-12-17 12:37:36,848] jepsen worker 8 - jepsen.util 16208 :info :write [2 21] :RequestCanceledException INFO [2019-12-17 12:37:36,848] jepsen worker 43 - jepsen.util 16783 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:37:36,848] jepsen worker 43 - jepsen.util 16783 :invoke :read [14 nil] INFO [2019-12-17 12:37:36,855] jepsen worker 8 - jepsen.util 16298 :invoke :cas [2 [22 20]] INFO [2019-12-17 12:37:36,865] jepsen worker 81 - jepsen.util 16191 :info :write [27 45] :RequestCanceledException INFO [2019-12-17 12:37:36,871] jepsen worker 81 - jepsen.util 16281 :invoke :read [27 nil] INFO [2019-12-17 12:37:36,879] jepsen worker 65 - jepsen.util 10505 :ok :read [21 46] INFO [2019-12-17 12:37:36,879] jepsen worker 65 - jepsen.util 10505 :invoke :write [21 25] INFO [2019-12-17 12:37:36,901] jepsen worker 61 - jepsen.util 17881 :info :write [20 0] :RequestCanceledException INFO [2019-12-17 12:37:36,906] jepsen worker 61 - jepsen.util 17971 :invoke :cas [20 [35 39]] INFO [2019-12-17 12:37:36,907] jepsen worker 28 - jepsen.util 10198 :ok :read [9 32] INFO [2019-12-17 12:37:36,908] jepsen worker 28 - jepsen.util 10198 :invoke :write [9 33] INFO [2019-12-17 12:37:36,916] jepsen worker 13 - jepsen.util 11533 :fail :cas [4 [25 11]] :ValueNotSwapFrom INFO [2019-12-17 12:37:36,916] jepsen worker 13 - jepsen.util 11533 :invoke :write [4 38] INFO [2019-12-17 12:37:36,921] jepsen worker 73 - jepsen.util 11143 :fail :cas [24 [16 18]] :ValueNotSwapFrom INFO [2019-12-17 12:37:36,921] jepsen worker 73 - jepsen.util 11143 :invoke :write [24 21] INFO [2019-12-17 12:37:36,933] jepsen worker 36 - jepsen.util 12636 :info :write [12 5] :RequestCanceledException INFO [2019-12-17 12:37:36,940] jepsen worker 36 - jepsen.util 12726 :invoke :cas [12 [15 40]] INFO [2019-12-17 12:37:36,942] jepsen worker 29 - jepsen.util 10559 :ok :write [9 18] INFO [2019-12-17 12:37:36,943] jepsen worker 29 - jepsen.util 10559 :invoke :write [9 7] INFO [2019-12-17 12:37:36,946] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":70036,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:37:36,947] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":70036,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:37:36,947] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":70040,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:37:36,947] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":70040,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:37:36,947] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":70041,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:37:36,948] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":70041,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:37:36,962] jepsen worker 56 - jepsen.util 13016 :fail :cas [18 [41 18]] :ValueNotSwapFrom INFO [2019-12-17 12:37:36,962] jepsen worker 56 - jepsen.util 13016 :invoke :read [18 nil] INFO [2019-12-17 12:37:36,980] jepsen worker 80 - jepsen.util 11600 :ok :read [26 20] INFO [2019-12-17 12:37:36,980] jepsen worker 80 - jepsen.util 11600 :invoke :write [26 35] INFO [2019-12-17 12:37:36,989] jepsen worker 78 - jepsen.util 11958 :fail :cas [26 [0 45]] :ValueNotSwapFrom INFO [2019-12-17 12:37:36,989] jepsen worker 78 - jepsen.util 11958 :invoke :read [26 nil] INFO [2019-12-17 12:37:37,003] jepsen worker 11 - jepsen.util 11711 :ok :write [3 6] INFO [2019-12-17 12:37:37,003] jepsen worker 11 - jepsen.util 11711 :invoke :cas [3 [16 46]] INFO [2019-12-17 12:37:37,018] jepsen worker 15 - jepsen.util 15405 :info :write [5 16] :RequestCanceledException INFO [2019-12-17 12:37:37,019] jepsen worker 15 - jepsen.util 15495 :invoke :read [5 nil] INFO [2019-12-17 12:37:37,023] jepsen worker 28 - jepsen.util 10198 :ok :write [9 33] INFO [2019-12-17 12:37:37,024] jepsen worker 28 - jepsen.util 10198 :invoke :cas [9 [24 20]] INFO [2019-12-17 12:37:37,033] jepsen worker 58 - jepsen.util 16798 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:37:37,033] jepsen worker 50 - jepsen.util 16520 :info :write [16 43] :RequestCanceledException INFO [2019-12-17 12:37:37,033] jepsen worker 58 - jepsen.util 16798 :invoke :write [19 5] INFO [2019-12-17 12:37:37,033] jepsen worker 7 - jepsen.util 16567 :info :write [2 2] :RequestCanceledException INFO [2019-12-17 12:37:37,039] jepsen worker 7 - jepsen.util 16657 :invoke :write [2 48] INFO [2019-12-17 12:37:37,039] jepsen worker 50 - jepsen.util 16610 :invoke :read [16 nil] INFO [2019-12-17 12:37:37,062] jepsen worker 38 - jepsen.util 12638 :fail :cas [12 [1 40]] :ValueNotSwapFrom INFO [2019-12-17 12:37:37,062] jepsen worker 38 - jepsen.util 12638 :invoke :read [12 nil] INFO [2019-12-17 12:37:37,070] jepsen worker 11 - jepsen.util 11711 :fail :cas [3 [16 46]] :ValueNotSwapFrom INFO [2019-12-17 12:37:37,070] jepsen worker 11 - jepsen.util 11711 :invoke :write [3 31] INFO [2019-12-17 12:37:37,072] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":70054,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:37:37,072] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":70054,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:37:37,072] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":70058,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:37:37,073] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":70058,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:37:37,078] jepsen worker 73 - jepsen.util 11143 :ok :write [24 21] INFO [2019-12-17 12:37:37,078] jepsen worker 73 - jepsen.util 11143 :invoke :cas [24 [5 41]] INFO [2019-12-17 12:37:37,106] jepsen worker 78 - jepsen.util 11958 :ok :read [26 20] INFO [2019-12-17 12:37:37,106] jepsen worker 78 - jepsen.util 11958 :invoke :read [26 nil] INFO [2019-12-17 12:37:37,125] jepsen worker 73 - jepsen.util 11143 :fail :cas [24 [5 41]] :ValueNotSwapFrom INFO [2019-12-17 12:37:37,125] jepsen worker 73 - jepsen.util 11143 :invoke :cas [24 [7 42]] INFO [2019-12-17 12:37:37,146] jepsen worker 11 - jepsen.util 11711 :ok :write [3 31] INFO [2019-12-17 12:37:37,146] jepsen worker 11 - jepsen.util 11711 :invoke :cas [3 [24 25]] INFO [2019-12-17 12:37:37,149] jepsen worker 73 - jepsen.util 11143 :fail :cas [24 [7 42]] :ValueNotSwapFrom INFO [2019-12-17 12:37:37,149] jepsen worker 73 - jepsen.util 11143 :invoke :cas [24 [28 42]] INFO [2019-12-17 12:37:37,183] jepsen worker 11 - jepsen.util 11711 :fail :cas [3 [24 25]] :ValueNotSwapFrom INFO [2019-12-17 12:37:37,183] jepsen worker 11 - jepsen.util 11711 :invoke :cas [3 [1 10]] INFO [2019-12-17 12:37:37,186] jepsen worker 73 - jepsen.util 11143 :fail :cas [24 [28 42]] :ValueNotSwapFrom INFO [2019-12-17 12:37:37,186] jepsen worker 73 - jepsen.util 11143 :invoke :read [24 nil] INFO [2019-12-17 12:37:37,191] jepsen worker 78 - jepsen.util 11958 :ok :read [26 20] INFO [2019-12-17 12:37:37,192] jepsen worker 78 - jepsen.util 11958 :invoke :read [26 nil] INFO [2019-12-17 12:37:37,213] jepsen worker 73 - jepsen.util 11143 :ok :read [24 21] INFO [2019-12-17 12:37:37,213] jepsen worker 73 - jepsen.util 11143 :invoke :read [24 nil] INFO [2019-12-17 12:37:37,214] jepsen worker 11 - jepsen.util 11711 :fail :cas [3 [1 10]] :ValueNotSwapFrom INFO [2019-12-17 12:37:37,215] jepsen worker 11 - jepsen.util 11711 :invoke :cas [3 [29 10]] INFO [2019-12-17 12:37:37,224] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:37:37,224] jepsen nemesis - couchbase.util Rebalance check count: 552 INFO [2019-12-17 12:37:37,224] jepsen nemesis - couchbase.util Rebalance stuck count: 533 INFO [2019-12-17 12:37:37,247] jepsen worker 73 - jepsen.util 11143 :ok :read [24 21] INFO [2019-12-17 12:37:37,247] jepsen worker 73 - jepsen.util 11143 :invoke :cas [24 [18 11]] INFO [2019-12-17 12:37:37,251] jepsen worker 11 - jepsen.util 11711 :fail :cas [3 [29 10]] :ValueNotSwapFrom INFO [2019-12-17 12:37:37,251] jepsen worker 11 - jepsen.util 11711 :invoke :write [3 11] INFO [2019-12-17 12:37:37,265] jepsen worker 57 - jepsen.util 16167 :info :write [19 9] :RequestCanceledException INFO [2019-12-17 12:37:37,270] jepsen worker 57 - jepsen.util 16257 :invoke :cas [19 [43 22]] INFO [2019-12-17 12:37:37,276] jepsen worker 3 - jepsen.util 11433 :fail :cas [1 [6 37]] :ValueNotSwapFrom INFO [2019-12-17 12:37:37,276] jepsen worker 3 - jepsen.util 11433 :invoke :write [1 19] INFO [2019-12-17 12:37:37,287] jepsen worker 73 - jepsen.util 11143 :fail :cas [24 [18 11]] :ValueNotSwapFrom INFO [2019-12-17 12:37:37,287] jepsen worker 73 - jepsen.util 11143 :invoke :read [24 nil] INFO [2019-12-17 12:37:37,293] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":70076,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:37:37,293] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":70076,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:37:37,293] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":70077,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:37:37,293] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":70077,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:37:37,294] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":70084,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:37:37,294] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":70084,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:37:37,294] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":70087,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:37:37,294] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":70087,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:37:37,328] jepsen worker 73 - jepsen.util 11143 :ok :read [24 21] INFO [2019-12-17 12:37:37,328] jepsen worker 73 - jepsen.util 11143 :invoke :cas [24 [45 23]] INFO [2019-12-17 12:37:37,329] jepsen worker 78 - jepsen.util 11958 :ok :read [26 20] INFO [2019-12-17 12:37:37,329] jepsen worker 78 - jepsen.util 11958 :invoke :cas [26 [6 36]] INFO [2019-12-17 12:37:37,333] jepsen worker 42 - jepsen.util 17412 :info :write [14 36] :RequestCanceledException INFO [2019-12-17 12:37:37,333] jepsen worker 83 - jepsen.util 16733 :info :cas [27 [46 27]] :RequestCanceledException INFO [2019-12-17 12:37:37,334] jepsen worker 1 - jepsen.util 16651 :info :cas [0 [20 17]] :RequestCanceledException INFO [2019-12-17 12:37:37,338] jepsen worker 42 - jepsen.util 17502 :invoke :read [14 nil] INFO [2019-12-17 12:37:37,338] jepsen worker 1 - jepsen.util 16741 :invoke :cas [0 [40 44]] INFO [2019-12-17 12:37:37,339] jepsen worker 83 - jepsen.util 16823 :invoke :read [27 nil] INFO [2019-12-17 12:37:37,346] jepsen worker 11 - jepsen.util 11711 :ok :write [3 11] INFO [2019-12-17 12:37:37,346] jepsen worker 11 - jepsen.util 11711 :invoke :cas [3 [14 16]] INFO [2019-12-17 12:37:37,346] jepsen worker 12 - jepsen.util 12342 :fail :cas [4 [7 47]] :ValueNotSwapFrom INFO [2019-12-17 12:37:37,347] jepsen worker 12 - jepsen.util 12342 :invoke :read [4 nil] INFO [2019-12-17 12:37:37,347] jepsen worker 22 - jepsen.util 16582 :info :cas [7 [3 18]] :RequestCanceledException INFO [2019-12-17 12:37:37,351] jepsen worker 22 - jepsen.util 16672 :invoke :cas [7 [42 25]] INFO [2019-12-17 12:37:37,364] jepsen worker 32 - jepsen.util 12542 :info :write [10 26] :RequestCanceledException INFO [2019-12-17 12:37:37,364] jepsen worker 18 - jepsen.util 12348 :info :write [6 37] :RequestCanceledException INFO [2019-12-17 12:37:37,367] jepsen worker 32 - jepsen.util 12632 :invoke :cas [10 [7 33]] INFO [2019-12-17 12:37:37,368] jepsen worker 18 - jepsen.util 12438 :invoke :cas [6 [49 39]] INFO [2019-12-17 12:37:37,368] jepsen worker 73 - jepsen.util 11143 :fail :cas [24 [45 23]] :ValueNotSwapFrom INFO [2019-12-17 12:37:37,368] jepsen worker 73 - jepsen.util 11143 :invoke :write [24 4] INFO [2019-12-17 12:37:37,383] jepsen worker 11 - jepsen.util 11711 :fail :cas [3 [14 16]] :ValueNotSwapFrom INFO [2019-12-17 12:37:37,383] jepsen worker 11 - jepsen.util 11711 :invoke :write [3 13] INFO [2019-12-17 12:37:37,400] jepsen worker 39 - jepsen.util 16149 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:37:37,400] jepsen worker 39 - jepsen.util 16149 :invoke :cas [13 [11 5]] INFO [2019-12-17 12:37:37,409] jepsen worker 32 - jepsen.util 12632 :fail :cas [10 [7 33]] :ValueNotSwapFrom INFO [2019-12-17 12:37:37,410] jepsen worker 32 - jepsen.util 12632 :invoke :cas [10 [28 30]] INFO [2019-12-17 12:37:37,414] jepsen worker 18 - jepsen.util 12438 :fail :cas [6 [49 39]] :ValueNotSwapFrom INFO [2019-12-17 12:37:37,414] jepsen worker 18 - jepsen.util 12438 :invoke :read [6 nil] INFO [2019-12-17 12:37:37,418] jepsen worker 55 - jepsen.util 12745 :info :write [18 38] :RequestCanceledException INFO [2019-12-17 12:37:37,422] jepsen worker 55 - jepsen.util 12835 :invoke :cas [18 [5 5]] INFO [2019-12-17 12:37:37,445] jepsen worker 32 - jepsen.util 12632 :fail :cas [10 [28 30]] :ValueNotSwapFrom INFO [2019-12-17 12:37:37,445] jepsen worker 32 - jepsen.util 12632 :invoke :cas [10 [3 19]] INFO [2019-12-17 12:37:37,477] jepsen worker 78 - jepsen.util 11958 :fail :cas [26 [6 36]] :ValueNotSwapFrom INFO [2019-12-17 12:37:37,477] jepsen worker 78 - jepsen.util 11958 :invoke :write [26 44] INFO [2019-12-17 12:37:37,487] jepsen worker 32 - jepsen.util 12632 :fail :cas [10 [3 19]] :ValueNotSwapFrom INFO [2019-12-17 12:37:37,487] jepsen worker 32 - jepsen.util 12632 :invoke :read [10 nil] INFO [2019-12-17 12:37:37,491] jepsen worker 70 - jepsen.util 12670 :fail :cas [23 [3 43]] :ValueNotSwapFrom INFO [2019-12-17 12:37:37,491] jepsen worker 70 - jepsen.util 12670 :invoke :read [23 nil] INFO [2019-12-17 12:37:37,500] jepsen worker 76 - jepsen.util 16726 :info :cas [25 [47 37]] :RequestCanceledException INFO [2019-12-17 12:37:37,503] jepsen worker 76 - jepsen.util 16816 :invoke :read [25 nil] INFO [2019-12-17 12:37:37,517] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":70112,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:37:37,517] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":70112,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:37:37,517] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":70115,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:37:37,517] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":70115,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:37:37,518] jepsen worker 68 - jepsen.util 15908 :info :write [22 31] :RequestCanceledException INFO [2019-12-17 12:37:37,521] jepsen worker 68 - jepsen.util 15998 :invoke :read [22 nil] INFO [2019-12-17 12:37:37,521] jepsen worker 32 - jepsen.util 12632 :ok :read [10 48] INFO [2019-12-17 12:37:37,521] jepsen worker 32 - jepsen.util 12632 :invoke :read [10 nil] INFO [2019-12-17 12:37:37,532] jepsen worker 79 - jepsen.util 12229 :fail :cas [26 [24 13]] :ValueNotSwapFrom INFO [2019-12-17 12:37:37,532] jepsen worker 79 - jepsen.util 12229 :invoke :write [26 34] INFO [2019-12-17 12:37:37,547] jepsen worker 32 - jepsen.util 12632 :ok :read [10 48] INFO [2019-12-17 12:37:37,547] jepsen worker 32 - jepsen.util 12632 :invoke :write [10 16] INFO [2019-12-17 12:37:37,565] jepsen worker 49 - jepsen.util 16609 :info :write [16 1] :RequestCanceledException INFO [2019-12-17 12:37:37,565] jepsen worker 6 - jepsen.util 16926 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:37:37,565] jepsen worker 6 - jepsen.util 16926 :invoke :cas [2 [19 32]] INFO [2019-12-17 12:37:37,567] jepsen worker 49 - jepsen.util 16699 :invoke :read [16 nil] INFO [2019-12-17 12:37:37,569] jepsen worker 36 - jepsen.util 12726 :fail :cas [12 [15 40]] :ValueNotSwapFrom INFO [2019-12-17 12:37:37,569] jepsen worker 36 - jepsen.util 12726 :invoke :cas [12 [29 17]] INFO [2019-12-17 12:37:37,595] jepsen worker 73 - jepsen.util 11143 :ok :write [24 4] INFO [2019-12-17 12:37:37,596] jepsen worker 73 - jepsen.util 11143 :invoke :read [24 nil] INFO [2019-12-17 12:37:37,618] jepsen worker 37 - jepsen.util 11557 :info :write [12 13] :RequestCanceledException INFO [2019-12-17 12:37:37,622] jepsen worker 37 - jepsen.util 11647 :invoke :read [12 nil] INFO [2019-12-17 12:37:37,631] jepsen worker 18 - jepsen.util 12438 :ok :read [6 0] INFO [2019-12-17 12:37:37,631] jepsen worker 18 - jepsen.util 12438 :invoke :write [6 8] INFO [2019-12-17 12:37:37,633] jepsen worker 30 - jepsen.util 11100 :info :write [10 17] :RequestCanceledException INFO [2019-12-17 12:37:37,633] jepsen worker 54 - jepsen.util 12564 :info :write [18 36] :RequestCanceledException INFO [2019-12-17 12:37:37,638] jepsen worker 54 - jepsen.util 12654 :invoke :cas [18 [38 32]] INFO [2019-12-17 12:37:37,639] jepsen worker 30 - jepsen.util 11190 :invoke :cas [10 [18 41]] INFO [2019-12-17 12:37:37,646] jepsen worker 36 - jepsen.util 12726 :fail :cas [12 [29 17]] :ValueNotSwapFrom INFO [2019-12-17 12:37:37,647] jepsen worker 36 - jepsen.util 12726 :invoke :cas [12 [40 19]] INFO [2019-12-17 12:37:37,666] jepsen worker 11 - jepsen.util 11711 :ok :write [3 13] INFO [2019-12-17 12:37:37,666] jepsen worker 11 - jepsen.util 11711 :invoke :cas [3 [21 46]] INFO [2019-12-17 12:37:37,678] jepsen worker 38 - jepsen.util 12638 :ok :read [12 11] INFO [2019-12-17 12:37:37,679] jepsen worker 38 - jepsen.util 12638 :invoke :cas [12 [7 35]] INFO [2019-12-17 12:37:37,687] jepsen worker 73 - jepsen.util 11143 :ok :read [24 4] INFO [2019-12-17 12:37:37,687] jepsen worker 73 - jepsen.util 11143 :invoke :read [24 nil] INFO [2019-12-17 12:37:37,698] jepsen worker 30 - jepsen.util 11190 :fail :cas [10 [18 41]] :ValueNotSwapFrom INFO [2019-12-17 12:37:37,698] jepsen worker 30 - jepsen.util 11190 :invoke :write [10 16] INFO [2019-12-17 12:37:37,700] jepsen worker 40 - jepsen.util 17230 :info :write [13 33] :RequestCanceledException INFO [2019-12-17 12:37:37,706] jepsen worker 40 - jepsen.util 17320 :invoke :write [13 15] INFO [2019-12-17 12:37:37,726] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":70137,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:37:37,726] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":70137,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:37:37,730] jepsen worker 11 - jepsen.util 11711 :fail :cas [3 [21 46]] :ValueNotSwapFrom INFO [2019-12-17 12:37:37,730] jepsen worker 11 - jepsen.util 11711 :invoke :write [3 49] INFO [2019-12-17 12:37:37,731] jepsen worker 74 - jepsen.util 11774 :info :write [24 3] :RequestCanceledException INFO [2019-12-17 12:37:37,731] jepsen worker 51 - jepsen.util 11301 :info :write [17 26] :RequestCanceledException INFO [2019-12-17 12:37:37,734] jepsen worker 74 - jepsen.util 11864 :invoke :cas [24 [24 41]] INFO [2019-12-17 12:37:37,741] jepsen worker 51 - jepsen.util 11391 :invoke :read [17 nil] INFO [2019-12-17 12:37:37,744] jepsen worker 54 - jepsen.util 12654 :fail :cas [18 [38 32]] :ValueNotSwapFrom INFO [2019-12-17 12:37:37,744] jepsen worker 54 - jepsen.util 12654 :invoke :cas [18 [45 27]] INFO [2019-12-17 12:37:37,747] jepsen worker 34 - jepsen.util 17224 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:37:37,747] jepsen worker 45 - jepsen.util 12105 :fail :read [15 nil] :RequestTimeoutException INFO [2019-12-17 12:37:37,748] jepsen worker 45 - jepsen.util 12105 :invoke :write [15 43] INFO [2019-12-17 12:37:37,748] jepsen worker 34 - jepsen.util 17224 :invoke :write [11 35] INFO [2019-12-17 12:37:37,750] jepsen worker 70 - jepsen.util 12670 :ok :read [23 29] INFO [2019-12-17 12:37:37,750] jepsen worker 70 - jepsen.util 12670 :invoke :read [23 nil] INFO [2019-12-17 12:37:37,765] jepsen worker 10 - jepsen.util 11800 :fail :read [3 nil] :RequestTimeoutException INFO [2019-12-17 12:37:37,765] jepsen worker 24 - jepsen.util 15594 :info :cas [8 [14 41]] :RequestCanceledException INFO [2019-12-17 12:37:37,765] jepsen worker 10 - jepsen.util 11800 :invoke :read [3 nil] INFO [2019-12-17 12:37:37,767] jepsen worker 24 - jepsen.util 15684 :invoke :cas [8 [16 41]] INFO [2019-12-17 12:37:37,784] jepsen worker 74 - jepsen.util 11864 :fail :cas [24 [24 41]] :ValueNotSwapFrom INFO [2019-12-17 12:37:37,785] jepsen worker 74 - jepsen.util 11864 :invoke :read [24 nil] INFO [2019-12-17 12:37:37,815] jepsen worker 10 - jepsen.util 11800 :ok :read [3 13] INFO [2019-12-17 12:37:37,815] jepsen worker 10 - jepsen.util 11800 :invoke :cas [3 [19 16]] INFO [2019-12-17 12:37:37,833] jepsen worker 41 - jepsen.util 17861 :info :cas [13 [31 49]] :RequestCanceledException INFO [2019-12-17 12:37:37,835] jepsen worker 41 - jepsen.util 17951 :invoke :cas [13 [38 26]] INFO [2019-12-17 12:37:37,840] jepsen worker 10 - jepsen.util 11800 :fail :cas [3 [19 16]] :ValueNotSwapFrom INFO [2019-12-17 12:37:37,840] jepsen worker 10 - jepsen.util 11800 :invoke :write [3 0] INFO [2019-12-17 12:37:37,847] jepsen worker 14 - jepsen.util 11804 :info :cas [4 [8 36]] :RequestCanceledException INFO [2019-12-17 12:37:37,849] jepsen worker 14 - jepsen.util 11894 :invoke :cas [4 [11 36]] INFO [2019-12-17 12:37:37,857] jepsen worker 71 - jepsen.util 12941 :ok :read [23 29] INFO [2019-12-17 12:37:37,857] jepsen worker 71 - jepsen.util 12941 :invoke :write [23 26] INFO [2019-12-17 12:37:37,885] jepsen worker 56 - jepsen.util 13016 :ok :read [18 3] INFO [2019-12-17 12:37:37,885] jepsen worker 56 - jepsen.util 13016 :invoke :write [18 22] INFO [2019-12-17 12:37:37,918] jepsen worker 20 - jepsen.util 11810 :info :write [6 46] :RequestCanceledException INFO [2019-12-17 12:37:37,919] jepsen worker 20 - jepsen.util 11900 :invoke :read [6 nil] INFO [2019-12-17 12:37:37,936] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":70161,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:37:37,936] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":70161,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:37:37,937] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":70165,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:37:37,937] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":70165,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:37:37,948] jepsen worker 60 - jepsen.util 17610 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:37:37,948] jepsen worker 60 - jepsen.util 17610 :invoke :cas [20 [10 49]] INFO [2019-12-17 12:37:37,948] jepsen worker 27 - jepsen.util 11097 :fail :read [9 nil] :RequestTimeoutException INFO [2019-12-17 12:37:37,948] jepsen worker 27 - jepsen.util 11097 :invoke :cas [9 [35 6]] INFO [2019-12-17 12:37:37,950] jepsen worker 20 - jepsen.util 11900 :ok :read [6 0] INFO [2019-12-17 12:37:37,950] jepsen worker 20 - jepsen.util 11900 :invoke :read [6 nil] INFO [2019-12-17 12:37:37,962] jepsen worker 14 - jepsen.util 11894 :fail :cas [4 [11 36]] :ValueNotSwapFrom INFO [2019-12-17 12:37:37,962] jepsen worker 14 - jepsen.util 11894 :invoke :cas [4 [38 34]] INFO [2019-12-17 12:37:37,975] jepsen worker 20 - jepsen.util 11900 :ok :read [6 0] INFO [2019-12-17 12:37:37,975] jepsen worker 20 - jepsen.util 11900 :invoke :cas [6 [14 18]] INFO [2019-12-17 12:37:37,988] jepsen worker 32 - jepsen.util 12632 :ok :write [10 16] INFO [2019-12-17 12:37:37,989] jepsen worker 32 - jepsen.util 12632 :invoke :write [10 40] INFO [2019-12-17 12:37:38,001] jepsen worker 11 - jepsen.util 11711 :ok :write [3 49] INFO [2019-12-17 12:37:38,001] jepsen worker 11 - jepsen.util 11711 :invoke :write [3 3] INFO [2019-12-17 12:37:38,007] jepsen worker 18 - jepsen.util 12438 :ok :write [6 8] INFO [2019-12-17 12:37:38,007] jepsen worker 18 - jepsen.util 12438 :invoke :cas [6 [14 32]] INFO [2019-12-17 12:37:38,008] jepsen worker 20 - jepsen.util 11900 :fail :cas [6 [14 18]] :ValueNotSwapFrom INFO [2019-12-17 12:37:38,008] jepsen worker 20 - jepsen.util 11900 :invoke :write [6 6] INFO [2019-12-17 12:37:38,018] jepsen worker 17 - jepsen.util 16667 :info :write [5 19] :RequestCanceledException INFO [2019-12-17 12:37:38,024] jepsen worker 17 - jepsen.util 16757 :invoke :write [5 8] INFO [2019-12-17 12:37:38,032] jepsen worker 62 - jepsen.util 17252 :info :write [20 4] :RequestCanceledException INFO [2019-12-17 12:37:38,032] jepsen worker 19 - jepsen.util 12439 :info :write [6 5] :RequestCanceledException INFO [2019-12-17 12:37:38,038] jepsen worker 36 - jepsen.util 12726 :fail :cas [12 [40 19]] :ValueNotSwapFrom INFO [2019-12-17 12:37:38,038] jepsen worker 19 - jepsen.util 12529 :invoke :read [6 nil] INFO [2019-12-17 12:37:38,038] jepsen worker 36 - jepsen.util 12726 :invoke :cas [12 [3 4]] INFO [2019-12-17 12:37:38,038] jepsen worker 62 - jepsen.util 17342 :invoke :cas [20 [19 32]] INFO [2019-12-17 12:37:38,060] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":70175,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:37:38,060] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":70175,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:37:38,065] jepsen worker 75 - jepsen.util 16185 :info :cas [25 [47 36]] :RequestCanceledException INFO [2019-12-17 12:37:38,065] jepsen worker 86 - jepsen.util 17276 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:37:38,065] jepsen worker 86 - jepsen.util 17276 :invoke :cas [28 [20 32]] INFO [2019-12-17 12:37:38,069] jepsen worker 75 - jepsen.util 16275 :invoke :write [25 39] INFO [2019-12-17 12:37:38,072] jepsen worker 73 - jepsen.util 11143 :ok :read [24 4] INFO [2019-12-17 12:37:38,072] jepsen worker 73 - jepsen.util 11143 :invoke :read [24 nil] INFO [2019-12-17 12:37:38,072] jepsen worker 74 - jepsen.util 11864 :ok :read [24 4] INFO [2019-12-17 12:37:38,072] jepsen worker 74 - jepsen.util 11864 :invoke :cas [24 [43 17]] INFO [2019-12-17 12:37:38,106] jepsen worker 19 - jepsen.util 12529 :ok :read [6 8] INFO [2019-12-17 12:37:38,107] jepsen worker 19 - jepsen.util 12529 :invoke :write [6 28] INFO [2019-12-17 12:37:38,118] jepsen worker 89 - jepsen.util 11879 :info :write [29 49] :RequestCanceledException INFO [2019-12-17 12:37:38,119] jepsen worker 89 - jepsen.util 11969 :invoke :cas [29 [4 28]] INFO [2019-12-17 12:37:38,134] jepsen worker 26 - jepsen.util 17216 :info :cas [8 [30 35]] :RequestCanceledException INFO [2019-12-17 12:37:38,137] jepsen worker 26 - jepsen.util 17306 :invoke :cas [8 [36 24]] INFO [2019-12-17 12:37:38,149] jepsen worker 12 - jepsen.util 12342 :ok :read [4 8] INFO [2019-12-17 12:37:38,150] jepsen worker 12 - jepsen.util 12342 :invoke :cas [4 [29 38]] INFO [2019-12-17 12:37:38,190] jepsen worker 13 - jepsen.util 11533 :ok :write [4 38] INFO [2019-12-17 12:37:38,190] jepsen worker 13 - jepsen.util 11533 :invoke :read [4 nil] INFO [2019-12-17 12:37:38,194] jepsen worker 78 - jepsen.util 11958 :ok :write [26 44] INFO [2019-12-17 12:37:38,194] jepsen worker 78 - jepsen.util 11958 :invoke :read [26 nil] INFO [2019-12-17 12:37:38,200] jepsen worker 5 - jepsen.util 11795 :info :write [1 19] :RequestCanceledException INFO [2019-12-17 12:37:38,202] jepsen worker 5 - jepsen.util 11885 :invoke :cas [1 [47 12]] INFO [2019-12-17 12:37:38,218] jepsen worker 85 - jepsen.util 17275 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:37:38,218] jepsen worker 85 - jepsen.util 17275 :invoke :read [28 nil] INFO [2019-12-17 12:37:38,233] jepsen worker 33 - jepsen.util 16413 :info :write [11 24] :RequestCanceledException INFO [2019-12-17 12:37:38,233] jepsen worker 66 - jepsen.util 16536 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:37:38,234] jepsen worker 66 - jepsen.util 16536 :invoke :write [22 43] INFO [2019-12-17 12:37:38,236] jepsen worker 33 - jepsen.util 16503 :invoke :write [11 42] INFO [2019-12-17 12:37:38,238] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:37:38,239] jepsen nemesis - couchbase.util Rebalance check count: 553 INFO [2019-12-17 12:37:38,239] jepsen nemesis - couchbase.util Rebalance stuck count: 534 INFO [2019-12-17 12:37:38,260] jepsen worker 36 - jepsen.util 12726 :fail :cas [12 [3 4]] :ValueNotSwapFrom INFO [2019-12-17 12:37:38,260] jepsen worker 36 - jepsen.util 12726 :invoke :write [12 36] INFO [2019-12-17 12:37:38,279] jepsen worker 45 - jepsen.util 12105 :ok :write [15 43] INFO [2019-12-17 12:37:38,279] jepsen worker 20 - jepsen.util 11900 :ok :write [6 6] INFO [2019-12-17 12:37:38,279] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":70197,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:37:38,279] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":70205,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:37:38,279] jepsen worker 45 - jepsen.util 12105 :invoke :cas [15 [12 3]] INFO [2019-12-17 12:37:38,279] jepsen worker 20 - jepsen.util 11900 :invoke :read [6 nil] INFO [2019-12-17 12:37:38,279] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":70197,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:37:38,279] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":70205,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:37:38,279] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":70199,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:37:38,279] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":70210,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:37:38,279] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":70199,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:37:38,279] jepsen worker 11 - jepsen.util 11711 :ok :write [3 3] INFO [2019-12-17 12:37:38,279] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":70210,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:37:38,279] jepsen worker 11 - jepsen.util 11711 :invoke :write [3 27] INFO [2019-12-17 12:37:38,300] jepsen worker 67 - jepsen.util 16627 :info :cas [22 [24 21]] :RequestCanceledException INFO [2019-12-17 12:37:38,303] jepsen worker 67 - jepsen.util 16717 :invoke :cas [22 [48 21]] INFO [2019-12-17 12:37:38,306] jepsen worker 37 - jepsen.util 11647 :ok :read [12 11] INFO [2019-12-17 12:37:38,306] jepsen worker 37 - jepsen.util 11647 :invoke :read [12 nil] INFO [2019-12-17 12:37:38,317] jepsen worker 82 - jepsen.util 16912 :info :write [27 15] :RequestCanceledException INFO [2019-12-17 12:37:38,320] jepsen worker 82 - jepsen.util 17002 :invoke :write [27 32] INFO [2019-12-17 12:37:38,321] jepsen worker 32 - jepsen.util 12632 :ok :write [10 40] INFO [2019-12-17 12:37:38,321] jepsen worker 32 - jepsen.util 12632 :invoke :write [10 20] INFO [2019-12-17 12:37:38,334] jepsen worker 84 - jepsen.util 16284 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:37:38,334] jepsen worker 84 - jepsen.util 16284 :invoke :write [28 2] INFO [2019-12-17 12:37:38,348] jepsen worker 23 - jepsen.util 17033 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:37:38,348] jepsen worker 23 - jepsen.util 17033 :invoke :read [7 nil] INFO [2019-12-17 12:37:38,363] jepsen worker 45 - jepsen.util 12105 :fail :cas [15 [12 3]] :ValueNotSwapFrom INFO [2019-12-17 12:37:38,363] jepsen worker 45 - jepsen.util 12105 :invoke :read [15 nil] INFO [2019-12-17 12:37:38,372] jepsen worker 20 - jepsen.util 11900 :ok :read [6 6] INFO [2019-12-17 12:37:38,372] jepsen worker 20 - jepsen.util 11900 :invoke :read [6 nil] INFO [2019-12-17 12:37:38,374] jepsen worker 74 - jepsen.util 11864 :fail :cas [24 [43 17]] :ValueNotSwapFrom INFO [2019-12-17 12:37:38,374] jepsen worker 74 - jepsen.util 11864 :invoke :cas [24 [36 42]] INFO [2019-12-17 12:37:38,397] jepsen worker 45 - jepsen.util 12105 :ok :read [15 43] INFO [2019-12-17 12:37:38,397] jepsen worker 45 - jepsen.util 12105 :invoke :read [15 nil] INFO [2019-12-17 12:37:38,400] jepsen worker 4 - jepsen.util 11254 :info :write [1 43] :RequestCanceledException INFO [2019-12-17 12:37:38,402] jepsen worker 20 - jepsen.util 11900 :ok :read [6 6] INFO [2019-12-17 12:37:38,402] jepsen worker 20 - jepsen.util 11900 :invoke :read [6 nil] INFO [2019-12-17 12:37:38,403] jepsen worker 4 - jepsen.util 11344 :invoke :read [1 nil] INFO [2019-12-17 12:37:38,425] jepsen worker 55 - jepsen.util 12835 :fail :cas [18 [5 5]] :ValueNotSwapFrom INFO [2019-12-17 12:37:38,425] jepsen worker 55 - jepsen.util 12835 :invoke :cas [18 [6 27]] INFO [2019-12-17 12:37:38,429] jepsen worker 45 - jepsen.util 12105 :ok :read [15 43] INFO [2019-12-17 12:37:38,429] jepsen worker 45 - jepsen.util 12105 :invoke :write [15 26] INFO [2019-12-17 12:37:38,433] jepsen worker 21 - jepsen.util 15951 :info :write [7 16] :RequestCanceledException INFO [2019-12-17 12:37:38,438] jepsen worker 21 - jepsen.util 16041 :invoke :cas [7 [26 25]] INFO [2019-12-17 12:37:38,441] jepsen worker 11 - jepsen.util 11711 :ok :write [3 27] INFO [2019-12-17 12:37:38,442] jepsen worker 11 - jepsen.util 11711 :invoke :read [3 nil] INFO [2019-12-17 12:37:38,445] jepsen worker 20 - jepsen.util 11900 :ok :read [6 6] INFO [2019-12-17 12:37:38,445] jepsen worker 20 - jepsen.util 11900 :invoke :write [6 38] INFO [2019-12-17 12:37:38,449] jepsen worker 32 - jepsen.util 12632 :ok :write [10 20] INFO [2019-12-17 12:37:38,449] jepsen worker 32 - jepsen.util 12632 :invoke :write [10 13] INFO [2019-12-17 12:37:38,479] jepsen worker 54 - jepsen.util 12654 :fail :cas [18 [45 27]] :ValueNotSwapFrom INFO [2019-12-17 12:37:38,479] jepsen worker 54 - jepsen.util 12654 :invoke :write [18 39] INFO [2019-12-17 12:37:38,501] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":70231,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:37:38,501] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":70231,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:37:38,501] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":70235,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:37:38,502] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":70235,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:37:38,503] jepsen worker 74 - jepsen.util 11864 :fail :cas [24 [36 42]] :ValueNotSwapFrom INFO [2019-12-17 12:37:38,504] jepsen worker 74 - jepsen.util 11864 :invoke :read [24 nil] INFO [2019-12-17 12:37:38,518] jepsen worker 47 - jepsen.util 13007 :info :write [15 43] :RequestCanceledException INFO [2019-12-17 12:37:38,518] jepsen worker 0 - jepsen.util 17100 :info :write [0 30] :RequestCanceledException INFO [2019-12-17 12:37:38,523] jepsen worker 11 - jepsen.util 11711 :ok :read [3 27] INFO [2019-12-17 12:37:38,523] jepsen worker 11 - jepsen.util 11711 :invoke :write [3 41] INFO [2019-12-17 12:37:38,524] jepsen worker 47 - jepsen.util 13097 :invoke :write [15 4] INFO [2019-12-17 12:37:38,525] jepsen worker 0 - jepsen.util 17190 :invoke :write [0 41] INFO [2019-12-17 12:37:38,534] jepsen worker 59 - jepsen.util 15719 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:37:38,534] jepsen worker 64 - jepsen.util 9334 :info :write [21 17] :RequestCanceledException INFO [2019-12-17 12:37:38,535] jepsen worker 59 - jepsen.util 15719 :invoke :cas [19 [48 20]] INFO [2019-12-17 12:37:38,540] jepsen worker 64 - jepsen.util 9424 :invoke :write [21 25] INFO [2019-12-17 12:37:38,555] jepsen worker 38 - jepsen.util 12638 :fail :cas [12 [7 35]] :ValueNotSwapFrom INFO [2019-12-17 12:37:38,556] jepsen worker 38 - jepsen.util 12638 :invoke :cas [12 [29 20]] INFO [2019-12-17 12:37:38,565] jepsen worker 46 - jepsen.util 11836 :fail :read [15 nil] :RequestTimeoutException INFO [2019-12-17 12:37:38,565] jepsen worker 53 - jepsen.util 11663 :info :write [17 5] :RequestCanceledException INFO [2019-12-17 12:37:38,565] jepsen worker 46 - jepsen.util 11836 :invoke :cas [15 [38 15]] INFO [2019-12-17 12:37:38,572] jepsen worker 53 - jepsen.util 11753 :invoke :read [17 nil] INFO [2019-12-17 12:37:38,583] jepsen worker 13 - jepsen.util 11533 :ok :read [4 38] INFO [2019-12-17 12:37:38,584] jepsen worker 13 - jepsen.util 11533 :invoke :write [4 46] INFO [2019-12-17 12:37:38,591] jepsen worker 78 - jepsen.util 11958 :ok :read [26 44] INFO [2019-12-17 12:37:38,592] jepsen worker 78 - jepsen.util 11958 :invoke :read [26 nil] INFO [2019-12-17 12:37:38,600] jepsen worker 52 - jepsen.util 11482 :info :write [17 43] :RequestCanceledException INFO [2019-12-17 12:37:38,600] jepsen worker 9 - jepsen.util 12519 :info :write [3 29] :RequestCanceledException INFO [2019-12-17 12:37:38,603] jepsen worker 9 - jepsen.util 12609 :invoke :cas [3 [26 5]] INFO [2019-12-17 12:37:38,603] jepsen worker 74 - jepsen.util 11864 :ok :read [24 4] INFO [2019-12-17 12:37:38,603] jepsen worker 74 - jepsen.util 11864 :invoke :write [24 10] INFO [2019-12-17 12:37:38,604] jepsen worker 52 - jepsen.util 11572 :invoke :cas [17 [28 23]] INFO [2019-12-17 12:37:38,613] jepsen worker 46 - jepsen.util 11836 :fail :cas [15 [38 15]] :ValueNotSwapFrom INFO [2019-12-17 12:37:38,613] jepsen worker 46 - jepsen.util 11836 :invoke :cas [15 [18 47]] INFO [2019-12-17 12:37:38,618] jepsen worker 63 - jepsen.util 10503 :info :write [21 29] :RequestCanceledException INFO [2019-12-17 12:37:38,619] jepsen worker 63 - jepsen.util 10593 :invoke :cas [21 [46 20]] INFO [2019-12-17 12:37:38,634] jepsen worker 31 - jepsen.util 10201 :info :write [10 10] :RequestCanceledException INFO [2019-12-17 12:37:38,634] jepsen worker 2 - jepsen.util 16022 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:37:38,634] jepsen worker 2 - jepsen.util 16022 :invoke :cas [0 [26 9]] INFO [2019-12-17 12:37:38,636] jepsen worker 9 - jepsen.util 12609 :fail :cas [3 [26 5]] :ValueNotSwapFrom INFO [2019-12-17 12:37:38,636] jepsen worker 9 - jepsen.util 12609 :invoke :write [3 0] INFO [2019-12-17 12:37:38,636] jepsen worker 31 - jepsen.util 10291 :invoke :write [10 19] INFO [2019-12-17 12:37:38,669] jepsen worker 37 - jepsen.util 11647 :ok :read [12 11] INFO [2019-12-17 12:37:38,670] jepsen worker 37 - jepsen.util 11647 :invoke :write [12 29] INFO [2019-12-17 12:37:38,686] jepsen worker 46 - jepsen.util 11836 :fail :cas [15 [18 47]] :ValueNotSwapFrom INFO [2019-12-17 12:37:38,686] jepsen worker 46 - jepsen.util 11836 :invoke :write [15 34] INFO [2019-12-17 12:37:38,713] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":70263,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:37:38,713] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":70263,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:37:38,713] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":70267,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:37:38,713] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":70267,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:37:38,719] jepsen worker 70 - jepsen.util 12670 :ok :read [23 29] INFO [2019-12-17 12:37:38,719] jepsen worker 70 - jepsen.util 12670 :invoke :write [23 35] INFO [2019-12-17 12:37:38,725] jepsen worker 78 - jepsen.util 11958 :ok :read [26 44] INFO [2019-12-17 12:37:38,725] jepsen worker 78 - jepsen.util 11958 :invoke :cas [26 [40 21]] INFO [2019-12-17 12:37:38,733] jepsen worker 88 - jepsen.util 12238 :info :write [29 6] :RequestCanceledException INFO [2019-12-17 12:37:38,733] jepsen worker 77 - jepsen.util 17627 :info :cas [25 [16 4]] :RequestCanceledException INFO [2019-12-17 12:37:38,735] jepsen worker 88 - jepsen.util 12328 :invoke :cas [29 [1 33]] INFO [2019-12-17 12:37:38,735] jepsen worker 77 - jepsen.util 17717 :invoke :cas [25 [38 38]] INFO [2019-12-17 12:37:38,747] jepsen worker 72 - jepsen.util 12312 :info :write [24 36] :RequestCanceledException INFO [2019-12-17 12:37:38,747] jepsen worker 87 - jepsen.util 11607 :info :write [29 36] :RequestCanceledException INFO [2019-12-17 12:37:38,750] jepsen worker 87 - jepsen.util 11697 :invoke :write [29 35] INFO [2019-12-17 12:37:38,754] jepsen worker 72 - jepsen.util 12402 :invoke :write [24 16] INFO [2019-12-17 12:37:38,765] jepsen worker 25 - jepsen.util 17665 :info :write [8 4] :RequestCanceledException INFO [2019-12-17 12:37:38,767] jepsen worker 25 - jepsen.util 17755 :invoke :read [8 nil] INFO [2019-12-17 12:37:38,819] jepsen worker 12 - jepsen.util 12342 :fail :cas [4 [29 38]] :ValueNotSwapFrom INFO [2019-12-17 12:37:38,819] jepsen worker 12 - jepsen.util 12342 :invoke :write [4 21] INFO [2019-12-17 12:37:38,833] jepsen worker 16 - jepsen.util 18466 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:37:38,833] jepsen worker 69 - jepsen.util 12309 :info :write [23 9] :RequestCanceledException INFO [2019-12-17 12:37:38,833] jepsen worker 16 - jepsen.util 18466 :invoke :read [5 nil] INFO [2019-12-17 12:37:38,835] jepsen worker 69 - jepsen.util 12399 :invoke :cas [23 [23 35]] INFO [2019-12-17 12:37:38,851] jepsen worker 5 - jepsen.util 11885 :fail :cas [1 [47 12]] :ValueNotSwapFrom INFO [2019-12-17 12:37:38,852] jepsen worker 5 - jepsen.util 11885 :invoke :cas [1 [5 14]] INFO [2019-12-17 12:37:38,900] jepsen worker 35 - jepsen.util 16505 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:37:38,900] jepsen worker 35 - jepsen.util 16505 :invoke :write [11 41] INFO [2019-12-17 12:37:38,918] jepsen worker 48 - jepsen.util 15258 :info :write [16 14] :RequestCanceledException INFO [2019-12-17 12:37:38,920] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":70284,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:37:38,920] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":70284,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:37:38,920] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":70286,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:37:38,920] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":70286,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:37:38,921] jepsen worker 48 - jepsen.util 15348 :invoke :write [16 42] INFO [2019-12-17 12:37:38,933] jepsen worker 8 - jepsen.util 16298 :info :cas [2 [22 20]] :RequestCanceledException INFO [2019-12-17 12:37:38,934] jepsen worker 44 - jepsen.util 17234 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:37:38,934] jepsen worker 44 - jepsen.util 17234 :invoke :read [14 nil] INFO [2019-12-17 12:37:38,935] jepsen worker 8 - jepsen.util 16388 :invoke :read [2 nil] INFO [2019-12-17 12:37:38,947] jepsen worker 43 - jepsen.util 16783 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:37:38,947] jepsen worker 81 - jepsen.util 16281 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:37:38,947] jepsen worker 43 - jepsen.util 16783 :invoke :write [14 32] INFO [2019-12-17 12:37:38,947] jepsen worker 81 - jepsen.util 16281 :invoke :write [27 23] INFO [2019-12-17 12:37:38,955] jepsen worker 38 - jepsen.util 12638 :fail :cas [12 [29 20]] :ValueNotSwapFrom INFO [2019-12-17 12:37:38,955] jepsen worker 38 - jepsen.util 12638 :invoke :write [12 13] INFO [2019-12-17 12:37:38,961] jepsen worker 61 - jepsen.util 17971 :info :cas [20 [35 39]] :RequestCanceledException INFO [2019-12-17 12:37:38,963] jepsen worker 61 - jepsen.util 18061 :invoke :cas [20 [9 3]] INFO [2019-12-17 12:37:39,001] jepsen worker 65 - jepsen.util 10505 :info :write [21 25] :RequestCanceledException INFO [2019-12-17 12:37:39,003] jepsen worker 65 - jepsen.util 10595 :invoke :cas [21 [14 32]] INFO [2019-12-17 12:37:39,018] jepsen worker 80 - jepsen.util 11600 :info :write [26 35] :RequestCanceledException INFO [2019-12-17 12:37:39,020] jepsen worker 80 - jepsen.util 11690 :invoke :cas [26 [42 47]] INFO [2019-12-17 12:37:39,038] jepsen worker 52 - jepsen.util 11572 :fail :cas [17 [28 23]] :ValueNotSwapFrom INFO [2019-12-17 12:37:39,038] jepsen worker 52 - jepsen.util 11572 :invoke :cas [17 [29 27]] INFO [2019-12-17 12:37:39,048] jepsen worker 28 - jepsen.util 10198 :info :cas [9 [24 20]] :RequestCanceledException INFO [2019-12-17 12:37:39,050] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":70295,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:37:39,050] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":70295,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:37:39,051] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":70299,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:37:39,051] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":70299,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:37:39,051] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":70300,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:37:39,052] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":70300,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:37:39,055] jepsen worker 28 - jepsen.util 10288 :invoke :read [9 nil] INFO [2019-12-17 12:37:39,065] jepsen worker 29 - jepsen.util 10559 :info :write [9 7] :RequestCanceledException INFO [2019-12-17 12:37:39,067] jepsen worker 29 - jepsen.util 10649 :invoke :write [9 35] INFO [2019-12-17 12:37:39,090] jepsen worker 28 - jepsen.util 10288 :ok :read [9 33] INFO [2019-12-17 12:37:39,090] jepsen worker 28 - jepsen.util 10288 :invoke :cas [9 [7 38]] INFO [2019-12-17 12:37:39,096] jepsen worker 53 - jepsen.util 11753 :ok :read [17 19] INFO [2019-12-17 12:37:39,096] jepsen worker 53 - jepsen.util 11753 :invoke :write [17 37] INFO [2019-12-17 12:37:39,101] jepsen worker 15 - jepsen.util 15495 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:37:39,101] jepsen worker 15 - jepsen.util 15495 :invoke :cas [5 [2 42]] INFO [2019-12-17 12:37:39,118] jepsen worker 7 - jepsen.util 16657 :info :write [2 48] :RequestCanceledException INFO [2019-12-17 12:37:39,122] jepsen worker 7 - jepsen.util 16747 :invoke :read [2 nil] INFO [2019-12-17 12:37:39,134] jepsen worker 50 - jepsen.util 16610 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:37:39,134] jepsen worker 58 - jepsen.util 16798 :info :write [19 5] :RequestCanceledException INFO [2019-12-17 12:37:39,134] jepsen worker 50 - jepsen.util 16610 :invoke :write [16 13] INFO [2019-12-17 12:37:39,138] jepsen worker 51 - jepsen.util 11391 :ok :read [17 19] INFO [2019-12-17 12:37:39,138] jepsen worker 51 - jepsen.util 11391 :invoke :read [17 nil] INFO [2019-12-17 12:37:39,140] jepsen worker 28 - jepsen.util 10288 :fail :cas [9 [7 38]] :ValueNotSwapFrom INFO [2019-12-17 12:37:39,140] jepsen worker 28 - jepsen.util 10288 :invoke :write [9 44] INFO [2019-12-17 12:37:39,141] jepsen worker 58 - jepsen.util 16888 :invoke :read [19 nil] INFO [2019-12-17 12:37:39,168] jepsen worker 5 - jepsen.util 11885 :fail :cas [1 [5 14]] :ValueNotSwapFrom INFO [2019-12-17 12:37:39,168] jepsen worker 5 - jepsen.util 11885 :invoke :read [1 nil] INFO [2019-12-17 12:37:39,254] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:37:39,255] jepsen nemesis - couchbase.util Rebalance check count: 554 INFO [2019-12-17 12:37:39,255] jepsen nemesis - couchbase.util Rebalance stuck count: 535 INFO [2019-12-17 12:37:39,261] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":70316,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:37:39,261] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":70324,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:37:39,261] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":70316,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:37:39,262] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":70324,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:37:39,262] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":70326,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:37:39,262] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":70326,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:37:39,267] jepsen worker 89 - jepsen.util 11969 :fail :cas [29 [4 28]] :ValueNotSwapFrom INFO [2019-12-17 12:37:39,268] jepsen worker 89 - jepsen.util 11969 :invoke :cas [29 [12 35]] INFO [2019-12-17 12:37:39,276] jepsen worker 28 - jepsen.util 10288 :ok :write [9 44] INFO [2019-12-17 12:37:39,276] jepsen worker 28 - jepsen.util 10288 :invoke :read [9 nil] INFO [2019-12-17 12:37:39,318] jepsen worker 80 - jepsen.util 11690 :fail :cas [26 [42 47]] :ValueNotSwapFrom INFO [2019-12-17 12:37:39,318] jepsen worker 80 - jepsen.util 11690 :invoke :read [26 nil] INFO [2019-12-17 12:37:39,318] jepsen worker 3 - jepsen.util 11433 :info :write [1 19] :RequestCanceledException INFO [2019-12-17 12:37:39,321] jepsen worker 3 - jepsen.util 11523 :invoke :write [1 24] INFO [2019-12-17 12:37:39,321] jepsen worker 28 - jepsen.util 10288 :ok :read [9 44] INFO [2019-12-17 12:37:39,322] jepsen worker 28 - jepsen.util 10288 :invoke :read [9 nil] INFO [2019-12-17 12:37:39,346] jepsen worker 28 - jepsen.util 10288 :ok :read [9 44] INFO [2019-12-17 12:37:39,346] jepsen worker 28 - jepsen.util 10288 :invoke :cas [9 [5 1]] INFO [2019-12-17 12:37:39,347] jepsen worker 57 - jepsen.util 16257 :info :cas [19 [43 22]] :RequestCanceledException INFO [2019-12-17 12:37:39,349] jepsen worker 57 - jepsen.util 16347 :invoke :cas [19 [5 2]] INFO [2019-12-17 12:37:39,365] jepsen worker 42 - jepsen.util 17502 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:37:39,366] jepsen worker 42 - jepsen.util 17502 :invoke :cas [14 [7 23]] INFO [2019-12-17 12:37:39,381] jepsen worker 28 - jepsen.util 10288 :fail :cas [9 [5 1]] :ValueNotSwapFrom INFO [2019-12-17 12:37:39,381] jepsen worker 28 - jepsen.util 10288 :invoke :cas [9 [22 7]] INFO [2019-12-17 12:37:39,400] jepsen worker 83 - jepsen.util 16823 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:37:39,401] jepsen worker 83 - jepsen.util 16823 :invoke :read [27 nil] INFO [2019-12-17 12:37:39,410] jepsen worker 28 - jepsen.util 10288 :fail :cas [9 [22 7]] :ValueNotSwapFrom INFO [2019-12-17 12:37:39,410] jepsen worker 28 - jepsen.util 10288 :invoke :write [9 1] INFO [2019-12-17 12:37:39,419] jepsen worker 22 - jepsen.util 16672 :info :cas [7 [42 25]] :RequestCanceledException INFO [2019-12-17 12:37:39,427] jepsen worker 22 - jepsen.util 16762 :invoke :write [7 13] INFO [2019-12-17 12:37:39,434] jepsen worker 1 - jepsen.util 16741 :info :cas [0 [40 44]] :RequestCanceledException INFO [2019-12-17 12:37:39,437] jepsen worker 1 - jepsen.util 16831 :invoke :write [0 34] INFO [2019-12-17 12:37:39,459] jepsen worker 28 - jepsen.util 10288 :ok :write [9 1] INFO [2019-12-17 12:37:39,459] jepsen worker 28 - jepsen.util 10288 :invoke :write [9 3] INFO [2019-12-17 12:37:39,469] jepsen worker 52 - jepsen.util 11572 :fail :cas [17 [29 27]] :ValueNotSwapFrom INFO [2019-12-17 12:37:39,469] jepsen worker 52 - jepsen.util 11572 :invoke :read [17 nil] INFO [2019-12-17 12:37:39,490] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":70338,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:37:39,491] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":70338,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:37:39,491] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":70342,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:37:39,491] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":70342,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:37:39,493] jepsen worker 69 - jepsen.util 12399 :fail :cas [23 [23 35]] :ValueNotSwapFrom INFO [2019-12-17 12:37:39,493] jepsen worker 69 - jepsen.util 12399 :invoke :cas [23 [45 2]] INFO [2019-12-17 12:37:39,500] jepsen worker 39 - jepsen.util 16149 :info :cas [13 [11 5]] :RequestCanceledException INFO [2019-12-17 12:37:39,502] jepsen worker 39 - jepsen.util 16239 :invoke :write [13 5] INFO [2019-12-17 12:37:39,533] jepsen worker 28 - jepsen.util 10288 :ok :write [9 3] INFO [2019-12-17 12:37:39,533] jepsen worker 76 - jepsen.util 16816 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:37:39,533] jepsen worker 28 - jepsen.util 10288 :invoke :read [9 nil] INFO [2019-12-17 12:37:39,533] jepsen worker 76 - jepsen.util 16816 :invoke :write [25 40] INFO [2019-12-17 12:37:39,550] jepsen worker 51 - jepsen.util 11391 :ok :read [17 19] INFO [2019-12-17 12:37:39,551] jepsen worker 51 - jepsen.util 11391 :invoke :cas [17 [32 12]] INFO [2019-12-17 12:37:39,593] jepsen worker 28 - jepsen.util 10288 :ok :read [9 3] INFO [2019-12-17 12:37:39,593] jepsen worker 28 - jepsen.util 10288 :invoke :cas [9 [28 5]] INFO [2019-12-17 12:37:39,601] jepsen worker 68 - jepsen.util 15998 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:37:39,601] jepsen worker 79 - jepsen.util 12229 :info :write [26 34] :RequestCanceledException INFO [2019-12-17 12:37:39,601] jepsen worker 68 - jepsen.util 15998 :invoke :read [22 nil] INFO [2019-12-17 12:37:39,611] jepsen worker 79 - jepsen.util 12319 :invoke :write [26 46] INFO [2019-12-17 12:37:39,619] jepsen worker 49 - jepsen.util 16699 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:37:39,619] jepsen worker 49 - jepsen.util 16699 :invoke :write [16 45] INFO [2019-12-17 12:37:39,622] jepsen worker 69 - jepsen.util 12399 :fail :cas [23 [45 2]] :ValueNotSwapFrom INFO [2019-12-17 12:37:39,622] jepsen worker 69 - jepsen.util 12399 :invoke :cas [23 [4 10]] INFO [2019-12-17 12:37:39,634] jepsen worker 28 - jepsen.util 10288 :fail :cas [9 [28 5]] :ValueNotSwapFrom INFO [2019-12-17 12:37:39,634] jepsen worker 28 - jepsen.util 10288 :invoke :write [9 48] INFO [2019-12-17 12:37:39,657] jepsen worker 4 - jepsen.util 11344 :ok :read [1 19] INFO [2019-12-17 12:37:39,657] jepsen worker 4 - jepsen.util 11344 :invoke :read [1 nil] INFO [2019-12-17 12:37:39,662] jepsen worker 6 - jepsen.util 16926 :info :cas [2 [19 32]] :RequestCanceledException INFO [2019-12-17 12:37:39,664] jepsen worker 6 - jepsen.util 17016 :invoke :read [2 nil] INFO [2019-12-17 12:37:39,683] jepsen worker 55 - jepsen.util 12835 :fail :cas [18 [6 27]] :ValueNotSwapFrom INFO [2019-12-17 12:37:39,683] jepsen worker 55 - jepsen.util 12835 :invoke :write [18 20] INFO [2019-12-17 12:37:39,700] jepsen worker 28 - jepsen.util 10288 :ok :write [9 48] INFO [2019-12-17 12:37:39,701] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":70362,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:37:39,701] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":70362,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:37:39,701] jepsen worker 28 - jepsen.util 10288 :invoke :cas [9 [34 49]] INFO [2019-12-17 12:37:39,701] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":70364,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:37:39,701] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":70364,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:37:39,715] jepsen worker 28 - jepsen.util 10288 :fail :cas [9 [34 49]] :ValueNotSwapFrom INFO [2019-12-17 12:37:39,716] jepsen worker 28 - jepsen.util 10288 :invoke :read [9 nil] INFO [2019-12-17 12:37:39,740] jepsen worker 28 - jepsen.util 10288 :ok :read [9 48] INFO [2019-12-17 12:37:39,740] jepsen worker 28 - jepsen.util 10288 :invoke :cas [9 [0 43]] INFO [2019-12-17 12:37:39,753] jepsen worker 28 - jepsen.util 10288 :fail :cas [9 [0 43]] :ValueNotSwapFrom INFO [2019-12-17 12:37:39,753] jepsen worker 28 - jepsen.util 10288 :invoke :write [9 43] INFO [2019-12-17 12:37:39,763] jepsen worker 32 - jepsen.util 12632 :ok :write [10 13] INFO [2019-12-17 12:37:39,763] jepsen worker 32 - jepsen.util 12632 :invoke :cas [10 [44 41]] INFO [2019-12-17 12:37:39,764] jepsen worker 11 - jepsen.util 11711 :ok :write [3 41] INFO [2019-12-17 12:37:39,764] jepsen worker 11 - jepsen.util 11711 :invoke :write [3 21] INFO [2019-12-17 12:37:39,765] jepsen worker 30 - jepsen.util 11190 :info :write [10 16] :RequestCanceledException INFO [2019-12-17 12:37:39,766] jepsen worker 52 - jepsen.util 11572 :ok :read [17 19] INFO [2019-12-17 12:37:39,766] jepsen worker 30 - jepsen.util 11280 :invoke :read [10 nil] INFO [2019-12-17 12:37:39,766] jepsen worker 52 - jepsen.util 11572 :invoke :read [17 nil] INFO [2019-12-17 12:37:39,772] jepsen worker 20 - jepsen.util 11900 :ok :write [6 38] INFO [2019-12-17 12:37:39,772] jepsen worker 20 - jepsen.util 11900 :invoke :write [6 36] INFO [2019-12-17 12:37:39,820] jepsen worker 45 - jepsen.util 12105 :ok :write [15 26] INFO [2019-12-17 12:37:39,820] jepsen worker 45 - jepsen.util 12105 :invoke :write [15 44] INFO [2019-12-17 12:37:39,834] jepsen worker 24 - jepsen.util 15684 :info :cas [8 [16 41]] :RequestCanceledException INFO [2019-12-17 12:37:39,834] jepsen worker 40 - jepsen.util 17320 :info :write [13 15] :RequestCanceledException INFO [2019-12-17 12:37:39,835] jepsen worker 24 - jepsen.util 15774 :invoke :write [8 6] INFO [2019-12-17 12:37:39,835] jepsen worker 40 - jepsen.util 17410 :invoke :cas [13 [38 42]] INFO [2019-12-17 12:37:39,848] jepsen worker 41 - jepsen.util 17951 :info :cas [13 [38 26]] :RequestCanceledException INFO [2019-12-17 12:37:39,848] jepsen worker 34 - jepsen.util 17224 :info :write [11 35] :RequestCanceledException INFO [2019-12-17 12:37:39,849] jepsen worker 34 - jepsen.util 17314 :invoke :cas [11 [34 14]] INFO [2019-12-17 12:37:39,850] jepsen worker 41 - jepsen.util 18041 :invoke :read [13 nil] INFO [2019-12-17 12:37:39,865] jepsen worker 10 - jepsen.util 11800 :info :write [3 0] :RequestCanceledException INFO [2019-12-17 12:37:39,866] jepsen worker 10 - jepsen.util 11890 :invoke :cas [3 [25 11]] INFO [2019-12-17 12:37:39,889] jepsen worker 78 - jepsen.util 11958 :fail :cas [26 [40 21]] :ValueNotSwapFrom INFO [2019-12-17 12:37:39,890] jepsen worker 78 - jepsen.util 11958 :invoke :write [26 33] INFO [2019-12-17 12:37:39,893] jepsen worker 64 - jepsen.util 9424 :ok :write [21 25] INFO [2019-12-17 12:37:39,893] jepsen worker 64 - jepsen.util 9424 :invoke :write [21 38] INFO [2019-12-17 12:37:39,897] jepsen worker 28 - jepsen.util 10288 :ok :write [9 43] INFO [2019-12-17 12:37:39,897] jepsen worker 28 - jepsen.util 10288 :invoke :write [9 47] INFO [2019-12-17 12:37:39,901] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":70383,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:37:39,902] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":70383,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:37:39,902] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":70384,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:37:39,902] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":70384,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:37:39,902] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":70385,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:37:39,902] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":70385,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:37:39,912] jepsen worker 5 - jepsen.util 11885 :ok :read [1 19] INFO [2019-12-17 12:37:39,912] jepsen worker 5 - jepsen.util 11885 :invoke :cas [1 [8 12]] INFO [2019-12-17 12:37:39,912] jepsen worker 56 - jepsen.util 13016 :info :write [18 22] :RequestCanceledException INFO [2019-12-17 12:37:39,913] jepsen worker 56 - jepsen.util 13106 :invoke :write [18 5] INFO [2019-12-17 12:37:39,934] jepsen worker 88 - jepsen.util 12328 :fail :cas [29 [1 33]] :ValueNotSwapFrom INFO [2019-12-17 12:37:39,935] jepsen worker 88 - jepsen.util 12328 :invoke :cas [29 [44 49]] INFO [2019-12-17 12:37:39,949] jepsen worker 29 - jepsen.util 10649 :ok :write [9 35] INFO [2019-12-17 12:37:39,950] jepsen worker 29 - jepsen.util 10649 :invoke :cas [9 [10 25]] INFO [2019-12-17 12:37:39,951] jepsen worker 80 - jepsen.util 11690 :ok :read [26 44] INFO [2019-12-17 12:37:39,951] jepsen worker 80 - jepsen.util 11690 :invoke :cas [26 [22 48]] INFO [2019-12-17 12:37:39,965] jepsen worker 71 - jepsen.util 12941 :info :write [23 26] :RequestCanceledException INFO [2019-12-17 12:37:39,967] jepsen worker 71 - jepsen.util 13031 :invoke :cas [23 [44 10]] INFO [2019-12-17 12:37:39,969] jepsen worker 32 - jepsen.util 12632 :fail :cas [10 [44 41]] :ValueNotSwapFrom INFO [2019-12-17 12:37:39,969] jepsen worker 32 - jepsen.util 12632 :invoke :write [10 46] INFO [2019-12-17 12:37:40,005] jepsen worker 64 - jepsen.util 9424 :ok :write [21 38] INFO [2019-12-17 12:37:40,005] jepsen worker 64 - jepsen.util 9424 :invoke :read [21 nil] INFO [2019-12-17 12:37:40,018] jepsen worker 51 - jepsen.util 11391 :fail :cas [17 [32 12]] :ValueNotSwapFrom INFO [2019-12-17 12:37:40,018] jepsen worker 51 - jepsen.util 11391 :invoke :read [17 nil] INFO [2019-12-17 12:37:40,028] jepsen worker 30 - jepsen.util 11280 :ok :read [10 13] INFO [2019-12-17 12:37:40,028] jepsen worker 30 - jepsen.util 11280 :invoke :cas [10 [13 47]] INFO [2019-12-17 12:37:40,036] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":70400,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:37:40,036] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":70400,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:37:40,047] jepsen worker 27 - jepsen.util 11097 :info :cas [9 [35 6]] :RequestCanceledException INFO [2019-12-17 12:37:40,047] jepsen worker 60 - jepsen.util 17610 :info :cas [20 [10 49]] :RequestCanceledException INFO [2019-12-17 12:37:40,047] jepsen worker 18 - jepsen.util 12438 :info :cas [6 [14 32]] :RequestCanceledException INFO [2019-12-17 12:37:40,048] jepsen worker 18 - jepsen.util 12528 :invoke :read [6 nil] INFO [2019-12-17 12:37:40,048] jepsen worker 27 - jepsen.util 11187 :invoke :read [9 nil] INFO [2019-12-17 12:37:40,048] jepsen worker 60 - jepsen.util 17700 :invoke :write [20 42] INFO [2019-12-17 12:37:40,100] jepsen worker 17 - jepsen.util 16757 :info :write [5 8] :RequestCanceledException INFO [2019-12-17 12:37:40,101] jepsen worker 17 - jepsen.util 16847 :invoke :write [5 23] INFO [2019-12-17 12:37:40,119] jepsen worker 11 - jepsen.util 11711 :ok :write [3 21] INFO [2019-12-17 12:37:40,119] jepsen worker 11 - jepsen.util 11711 :invoke :cas [3 [2 7]] INFO [2019-12-17 12:37:40,121] jepsen worker 45 - jepsen.util 12105 :ok :write [15 44] INFO [2019-12-17 12:37:40,122] jepsen worker 72 - jepsen.util 12402 :ok :write [24 16] INFO [2019-12-17 12:37:40,122] jepsen worker 45 - jepsen.util 12105 :invoke :write [15 7] INFO [2019-12-17 12:37:40,122] jepsen worker 72 - jepsen.util 12402 :invoke :write [24 41] INFO [2019-12-17 12:37:40,134] jepsen worker 62 - jepsen.util 17342 :info :cas [20 [19 32]] :RequestCanceledException INFO [2019-12-17 12:37:40,136] jepsen worker 62 - jepsen.util 17432 :invoke :write [20 14] INFO [2019-12-17 12:37:40,140] jepsen worker 3 - jepsen.util 11523 :ok :write [1 24] INFO [2019-12-17 12:37:40,140] jepsen worker 3 - jepsen.util 11523 :invoke :cas [1 [31 8]] INFO [2019-12-17 12:37:40,147] jepsen worker 73 - jepsen.util 11143 :fail :read [24 nil] :RequestTimeoutException INFO [2019-12-17 12:37:40,147] jepsen worker 75 - jepsen.util 16275 :info :write [25 39] :RequestCanceledException INFO [2019-12-17 12:37:40,148] jepsen worker 73 - jepsen.util 11143 :invoke :cas [24 [2 35]] INFO [2019-12-17 12:37:40,148] jepsen worker 37 - jepsen.util 11647 :ok :write [12 29] INFO [2019-12-17 12:37:40,148] jepsen worker 12 - jepsen.util 12342 :ok :write [4 21] INFO [2019-12-17 12:37:40,148] jepsen worker 37 - jepsen.util 11647 :invoke :read [12 nil] INFO [2019-12-17 12:37:40,148] jepsen worker 12 - jepsen.util 12342 :invoke :read [4 nil] INFO [2019-12-17 12:37:40,149] jepsen worker 75 - jepsen.util 16365 :invoke :read [25 nil] INFO [2019-12-17 12:37:40,165] jepsen worker 86 - jepsen.util 17276 :info :cas [28 [20 32]] :RequestCanceledException INFO [2019-12-17 12:37:40,166] jepsen worker 86 - jepsen.util 17366 :invoke :write [28 7] INFO [2019-12-17 12:37:40,178] jepsen worker 69 - jepsen.util 12399 :fail :cas [23 [4 10]] :ValueNotSwapFrom INFO [2019-12-17 12:37:40,178] jepsen worker 69 - jepsen.util 12399 :invoke :cas [23 [16 1]] INFO [2019-12-17 12:37:40,200] jepsen worker 52 - jepsen.util 11572 :ok :read [17 19] INFO [2019-12-17 12:37:40,200] jepsen worker 52 - jepsen.util 11572 :invoke :cas [17 [35 28]] INFO [2019-12-17 12:37:40,218] jepsen worker 19 - jepsen.util 12529 :info :write [6 28] :RequestCanceledException INFO [2019-12-17 12:37:40,219] jepsen worker 19 - jepsen.util 12619 :invoke :read [6 nil] INFO [2019-12-17 12:37:40,246] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":70424,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:37:40,246] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":70431,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:37:40,246] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":70424,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:37:40,246] jepsen worker 26 - jepsen.util 17306 :info :cas [8 [36 24]] :RequestCanceledException INFO [2019-12-17 12:37:40,246] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":70431,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:37:40,247] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":70425,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:37:40,247] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":70433,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:37:40,247] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":70425,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:37:40,247] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":70433,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:37:40,247] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":70426,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:37:40,247] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":70426,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:37:40,248] jepsen worker 26 - jepsen.util 17396 :invoke :read [8 nil] INFO [2019-12-17 12:37:40,251] jepsen worker 27 - jepsen.util 11187 :ok :read [9 35] INFO [2019-12-17 12:37:40,251] jepsen worker 27 - jepsen.util 11187 :invoke :write [9 35] INFO [2019-12-17 12:37:40,252] jepsen worker 11 - jepsen.util 11711 :fail :cas [3 [2 7]] :ValueNotSwapFrom INFO [2019-12-17 12:37:40,252] jepsen worker 11 - jepsen.util 11711 :invoke :write [3 47] INFO [2019-12-17 12:37:40,255] jepsen worker 65 - jepsen.util 10595 :fail :cas [21 [14 32]] :ValueNotSwapFrom INFO [2019-12-17 12:37:40,255] jepsen worker 65 - jepsen.util 10595 :invoke :write [21 26] INFO [2019-12-17 12:37:40,265] jepsen worker 32 - jepsen.util 12632 :ok :write [10 46] INFO [2019-12-17 12:37:40,265] jepsen worker 32 - jepsen.util 12632 :invoke :cas [10 [46 29]] INFO [2019-12-17 12:37:40,274] jepsen worker 71 - jepsen.util 13031 :fail :cas [23 [44 10]] :ValueNotSwapFrom INFO [2019-12-17 12:37:40,274] jepsen worker 71 - jepsen.util 13031 :invoke :write [23 20] INFO [2019-12-17 12:37:40,282] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:37:40,282] jepsen nemesis - couchbase.util Rebalance check count: 555 INFO [2019-12-17 12:37:40,282] jepsen nemesis - couchbase.util Rebalance stuck count: 536 INFO [2019-12-17 12:37:40,293] jepsen worker 51 - jepsen.util 11391 :ok :read [17 19] INFO [2019-12-17 12:37:40,293] jepsen worker 51 - jepsen.util 11391 :invoke :read [17 nil] INFO [2019-12-17 12:37:40,300] jepsen worker 18 - jepsen.util 12528 :ok :read [6 38] INFO [2019-12-17 12:37:40,300] jepsen worker 18 - jepsen.util 12528 :invoke :write [6 42] INFO [2019-12-17 12:37:40,319] jepsen worker 85 - jepsen.util 17275 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:37:40,319] jepsen worker 33 - jepsen.util 16503 :info :write [11 42] :RequestCanceledException INFO [2019-12-17 12:37:40,319] jepsen worker 85 - jepsen.util 17275 :invoke :cas [28 [5 39]] INFO [2019-12-17 12:37:40,320] jepsen worker 33 - jepsen.util 16593 :invoke :read [11 nil] INFO [2019-12-17 12:37:40,321] jepsen worker 69 - jepsen.util 12399 :fail :cas [23 [16 1]] :ValueNotSwapFrom INFO [2019-12-17 12:37:40,321] jepsen worker 69 - jepsen.util 12399 :invoke :write [23 15] INFO [2019-12-17 12:37:40,333] jepsen worker 36 - jepsen.util 12726 :info :write [12 36] :RequestCanceledException INFO [2019-12-17 12:37:40,333] jepsen worker 66 - jepsen.util 16536 :info :write [22 43] :RequestCanceledException INFO [2019-12-17 12:37:40,333] jepsen worker 67 - jepsen.util 16717 :info :cas [22 [48 21]] :RequestCanceledException INFO [2019-12-17 12:37:40,334] jepsen worker 66 - jepsen.util 16626 :invoke :write [22 31] INFO [2019-12-17 12:37:40,334] jepsen worker 67 - jepsen.util 16807 :invoke :write [22 38] INFO [2019-12-17 12:37:40,334] jepsen worker 36 - jepsen.util 12816 :invoke :cas [12 [27 27]] INFO [2019-12-17 12:37:40,341] jepsen worker 70 - jepsen.util 12670 :ok :write [23 35] INFO [2019-12-17 12:37:40,342] jepsen worker 70 - jepsen.util 12670 :invoke :write [23 2] INFO [2019-12-17 12:37:40,348] jepsen worker 82 - jepsen.util 17002 :info :write [27 32] :RequestCanceledException INFO [2019-12-17 12:37:40,349] jepsen worker 82 - jepsen.util 17092 :invoke :read [27 nil] INFO [2019-12-17 12:37:40,351] jepsen worker 30 - jepsen.util 11280 :fail :cas [10 [13 47]] :CASMismatchException INFO [2019-12-17 12:37:40,351] jepsen worker 30 - jepsen.util 11280 :invoke :cas [10 [42 32]] INFO [2019-12-17 12:37:40,369] jepsen worker 53 - jepsen.util 11753 :ok :write [17 37] INFO [2019-12-17 12:37:40,369] jepsen worker 53 - jepsen.util 11753 :invoke :write [17 23] INFO [2019-12-17 12:37:40,382] jepsen worker 30 - jepsen.util 11280 :fail :cas [10 [42 32]] :ValueNotSwapFrom INFO [2019-12-17 12:37:40,382] jepsen worker 30 - jepsen.util 11280 :invoke :cas [10 [20 19]] INFO [2019-12-17 12:37:40,396] jepsen worker 30 - jepsen.util 11280 :fail :cas [10 [20 19]] :ValueNotSwapFrom INFO [2019-12-17 12:37:40,396] jepsen worker 30 - jepsen.util 11280 :invoke :cas [10 [44 16]] INFO [2019-12-17 12:37:40,401] jepsen worker 80 - jepsen.util 11690 :fail :cas [26 [22 48]] :ValueNotSwapFrom INFO [2019-12-17 12:37:40,401] jepsen worker 80 - jepsen.util 11690 :invoke :write [26 26] INFO [2019-12-17 12:37:40,411] jepsen worker 30 - jepsen.util 11280 :fail :cas [10 [44 16]] :ValueNotSwapFrom INFO [2019-12-17 12:37:40,411] jepsen worker 30 - jepsen.util 11280 :invoke :cas [10 [7 36]] INFO [2019-12-17 12:37:40,433] jepsen worker 84 - jepsen.util 16284 :info :write [28 2] :RequestCanceledException INFO [2019-12-17 12:37:40,435] jepsen worker 84 - jepsen.util 16374 :invoke :read [28 nil] INFO [2019-12-17 12:37:40,436] jepsen worker 89 - jepsen.util 11969 :fail :cas [29 [12 35]] :ValueNotSwapFrom INFO [2019-12-17 12:37:40,436] jepsen worker 89 - jepsen.util 11969 :invoke :read [29 nil] INFO [2019-12-17 12:37:40,444] jepsen worker 30 - jepsen.util 11280 :fail :cas [10 [7 36]] :ValueNotSwapFrom INFO [2019-12-17 12:37:40,444] jepsen worker 30 - jepsen.util 11280 :invoke :write [10 33] INFO [2019-12-17 12:37:40,448] jepsen worker 23 - jepsen.util 17033 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:37:40,448] jepsen worker 23 - jepsen.util 17033 :invoke :cas [7 [47 25]] INFO [2019-12-17 12:37:40,464] jepsen worker 51 - jepsen.util 11391 :ok :read [17 37] INFO [2019-12-17 12:37:40,464] jepsen worker 51 - jepsen.util 11391 :invoke :cas [17 [5 0]] INFO [2019-12-17 12:37:40,468] jepsen worker 5 - jepsen.util 11885 :fail :cas [1 [8 12]] :ValueNotSwapFrom INFO [2019-12-17 12:37:40,468] jepsen worker 5 - jepsen.util 11885 :invoke :cas [1 [10 16]] INFO [2019-12-17 12:37:40,474] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":70462,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:37:40,475] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":70462,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:37:40,475] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":70467,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:37:40,475] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":70467,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:37:40,507] jepsen worker 20 - jepsen.util 11900 :ok :write [6 36] INFO [2019-12-17 12:37:40,507] jepsen worker 20 - jepsen.util 11900 :invoke :write [6 30] INFO [2019-12-17 12:37:40,522] jepsen worker 78 - jepsen.util 11958 :ok :write [26 33] INFO [2019-12-17 12:37:40,522] jepsen worker 78 - jepsen.util 11958 :invoke :read [26 nil] INFO [2019-12-17 12:37:40,534] jepsen worker 21 - jepsen.util 16041 :info :cas [7 [26 25]] :RequestCanceledException INFO [2019-12-17 12:37:40,535] jepsen worker 21 - jepsen.util 16131 :invoke :write [7 45] INFO [2019-12-17 12:37:40,548] jepsen worker 54 - jepsen.util 12654 :info :write [18 39] :RequestCanceledException INFO [2019-12-17 12:37:40,549] jepsen worker 54 - jepsen.util 12744 :invoke :write [18 27] INFO [2019-12-17 12:37:40,554] jepsen worker 12 - jepsen.util 12342 :ok :read [4 21] INFO [2019-12-17 12:37:40,554] jepsen worker 12 - jepsen.util 12342 :invoke :write [4 46] INFO [2019-12-17 12:37:40,569] jepsen worker 45 - jepsen.util 12105 :ok :write [15 7] INFO [2019-12-17 12:37:40,569] jepsen worker 45 - jepsen.util 12105 :invoke :write [15 8] INFO [2019-12-17 12:37:40,598] jepsen worker 88 - jepsen.util 12328 :fail :cas [29 [44 49]] :ValueNotSwapFrom INFO [2019-12-17 12:37:40,598] jepsen worker 88 - jepsen.util 12328 :invoke :read [29 nil] INFO [2019-12-17 12:37:40,600] jepsen worker 47 - jepsen.util 13097 :info :write [15 4] :RequestCanceledException INFO [2019-12-17 12:37:40,601] jepsen worker 47 - jepsen.util 13187 :invoke :cas [15 [41 39]] INFO [2019-12-17 12:37:40,612] jepsen worker 69 - jepsen.util 12399 :ok :write [23 15] INFO [2019-12-17 12:37:40,612] jepsen worker 69 - jepsen.util 12399 :invoke :read [23 nil] INFO [2019-12-17 12:37:40,617] jepsen worker 0 - jepsen.util 17190 :info :write [0 41] :RequestCanceledException INFO [2019-12-17 12:37:40,618] jepsen worker 0 - jepsen.util 17280 :invoke :cas [0 [21 11]] INFO [2019-12-17 12:37:40,620] jepsen worker 30 - jepsen.util 11280 :ok :write [10 33] INFO [2019-12-17 12:37:40,621] jepsen worker 30 - jepsen.util 11280 :invoke :read [10 nil] INFO [2019-12-17 12:37:40,634] jepsen worker 59 - jepsen.util 15719 :info :cas [19 [48 20]] :RequestCanceledException INFO [2019-12-17 12:37:40,635] jepsen worker 59 - jepsen.util 15809 :invoke :cas [19 [28 37]] INFO [2019-12-17 12:37:40,636] jepsen worker 30 - jepsen.util 11280 :ok :read [10 33] INFO [2019-12-17 12:37:40,636] jepsen worker 30 - jepsen.util 11280 :invoke :write [10 14] INFO [2019-12-17 12:37:40,644] jepsen worker 51 - jepsen.util 11391 :fail :cas [17 [5 0]] :ValueNotSwapFrom INFO [2019-12-17 12:37:40,645] jepsen worker 51 - jepsen.util 11391 :invoke :read [17 nil] INFO [2019-12-17 12:37:40,647] jepsen worker 13 - jepsen.util 11533 :info :write [4 46] :RequestCanceledException INFO [2019-12-17 12:37:40,648] jepsen worker 13 - jepsen.util 11623 :invoke :write [4 12] INFO [2019-12-17 12:37:40,663] jepsen worker 14 - jepsen.util 11894 :fail :cas [4 [38 34]] :CASMismatchException INFO [2019-12-17 12:37:40,663] jepsen worker 14 - jepsen.util 11894 :invoke :cas [4 [3 5]] INFO [2019-12-17 12:37:40,665] jepsen worker 9 - jepsen.util 12609 :info :write [3 0] :RequestCanceledException INFO [2019-12-17 12:37:40,666] jepsen worker 9 - jepsen.util 12699 :invoke :read [3 nil] INFO [2019-12-17 12:37:40,678] jepsen worker 47 - jepsen.util 13187 :fail :cas [15 [41 39]] :ValueNotSwapFrom INFO [2019-12-17 12:37:40,678] jepsen worker 47 - jepsen.util 13187 :invoke :read [15 nil] INFO [2019-12-17 12:37:40,685] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":70488,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:37:40,685] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":70488,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:37:40,685] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":70491,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:37:40,685] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":70491,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:37:40,685] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":70492,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:37:40,685] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":70492,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:37:40,692] jepsen worker 47 - jepsen.util 13187 :ok :read [15 7] INFO [2019-12-17 12:37:40,692] jepsen worker 47 - jepsen.util 13187 :invoke :read [15 nil] INFO [2019-12-17 12:37:40,700] jepsen worker 74 - jepsen.util 11864 :info :write [24 10] :RequestCanceledException INFO [2019-12-17 12:37:40,700] jepsen worker 63 - jepsen.util 10593 :info :cas [21 [46 20]] :RequestCanceledException INFO [2019-12-17 12:37:40,701] jepsen worker 63 - jepsen.util 10683 :invoke :read [21 nil] INFO [2019-12-17 12:37:40,701] jepsen worker 74 - jepsen.util 11954 :invoke :write [24 19] INFO [2019-12-17 12:37:40,703] jepsen worker 37 - jepsen.util 11647 :ok :read [12 29] INFO [2019-12-17 12:37:40,704] jepsen worker 37 - jepsen.util 11647 :invoke :write [12 17] INFO [2019-12-17 12:37:40,709] jepsen worker 4 - jepsen.util 11344 :ok :read [1 24] INFO [2019-12-17 12:37:40,710] jepsen worker 4 - jepsen.util 11344 :invoke :cas [1 [8 35]] INFO [2019-12-17 12:37:40,718] jepsen worker 31 - jepsen.util 10291 :info :write [10 19] :RequestCanceledException INFO [2019-12-17 12:37:40,719] jepsen worker 31 - jepsen.util 10381 :invoke :cas [10 [4 4]] INFO [2019-12-17 12:37:40,725] jepsen worker 3 - jepsen.util 11523 :fail :cas [1 [31 8]] :ValueNotSwapFrom INFO [2019-12-17 12:37:40,725] jepsen worker 3 - jepsen.util 11523 :invoke :cas [1 [39 6]] INFO [2019-12-17 12:37:40,733] jepsen worker 2 - jepsen.util 16022 :info :cas [0 [26 9]] :RequestCanceledException INFO [2019-12-17 12:37:40,734] jepsen worker 2 - jepsen.util 16112 :invoke :read [0 nil] INFO [2019-12-17 12:37:40,735] jepsen worker 52 - jepsen.util 11572 :fail :cas [17 [35 28]] :ValueNotSwapFrom INFO [2019-12-17 12:37:40,735] jepsen worker 52 - jepsen.util 11572 :invoke :write [17 0] INFO [2019-12-17 12:37:40,743] jepsen worker 3 - jepsen.util 11523 :fail :cas [1 [39 6]] :ValueNotSwapFrom INFO [2019-12-17 12:37:40,743] jepsen worker 3 - jepsen.util 11523 :invoke :cas [1 [16 21]] INFO [2019-12-17 12:37:40,748] jepsen worker 77 - jepsen.util 17717 :info :cas [25 [38 38]] :RequestCanceledException INFO [2019-12-17 12:37:40,749] jepsen worker 77 - jepsen.util 17807 :invoke :write [25 17] INFO [2019-12-17 12:37:40,765] jepsen worker 46 - jepsen.util 11836 :info :write [15 34] :RequestCanceledException INFO [2019-12-17 12:37:40,768] jepsen worker 46 - jepsen.util 11926 :invoke :cas [15 [17 3]] INFO [2019-12-17 12:37:40,770] jepsen worker 36 - jepsen.util 12816 :fail :cas [12 [27 27]] :ValueNotSwapFrom INFO [2019-12-17 12:37:40,770] jepsen worker 36 - jepsen.util 12816 :invoke :write [12 11] INFO [2019-12-17 12:37:40,780] jepsen worker 47 - jepsen.util 13187 :ok :read [15 7] INFO [2019-12-17 12:37:40,780] jepsen worker 47 - jepsen.util 13187 :invoke :read [15 nil] INFO [2019-12-17 12:37:40,783] jepsen worker 31 - jepsen.util 10381 :fail :cas [10 [4 4]] :ValueNotSwapFrom INFO [2019-12-17 12:37:40,783] jepsen worker 31 - jepsen.util 10381 :invoke :write [10 3] INFO [2019-12-17 12:37:40,801] jepsen worker 25 - jepsen.util 17755 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:37:40,801] jepsen worker 25 - jepsen.util 17755 :invoke :read [8 nil] INFO [2019-12-17 12:37:40,804] jepsen worker 72 - jepsen.util 12402 :ok :write [24 41] INFO [2019-12-17 12:37:40,804] jepsen worker 72 - jepsen.util 12402 :invoke :cas [24 [31 22]] INFO [2019-12-17 12:37:40,834] jepsen worker 9 - jepsen.util 12699 :ok :read [3 21] INFO [2019-12-17 12:37:40,834] jepsen worker 9 - jepsen.util 12699 :invoke :write [3 27] INFO [2019-12-17 12:37:40,847] jepsen worker 46 - jepsen.util 11926 :fail :cas [15 [17 3]] :ValueNotSwapFrom INFO [2019-12-17 12:37:40,847] jepsen worker 46 - jepsen.util 11926 :invoke :read [15 nil] INFO [2019-12-17 12:37:40,858] jepsen worker 88 - jepsen.util 12328 :ok :read [29 2] INFO [2019-12-17 12:37:40,858] jepsen worker 88 - jepsen.util 12328 :invoke :read [29 nil] INFO [2019-12-17 12:37:40,863] jepsen worker 87 - jepsen.util 11697 :info :write [29 35] :RequestCanceledException INFO [2019-12-17 12:37:40,865] jepsen worker 87 - jepsen.util 11787 :invoke :read [29 nil] INFO [2019-12-17 12:37:40,869] jepsen worker 72 - jepsen.util 12402 :fail :cas [24 [31 22]] :ValueNotSwapFrom INFO [2019-12-17 12:37:40,869] jepsen worker 72 - jepsen.util 12402 :invoke :read [24 nil] INFO [2019-12-17 12:37:40,884] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":70519,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:37:40,884] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":70519,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:37:40,884] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":70521,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:37:40,884] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":70521,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:37:40,913] jepsen worker 78 - jepsen.util 11958 :ok :read [26 33] INFO [2019-12-17 12:37:40,913] jepsen worker 78 - jepsen.util 11958 :invoke :write [26 1] INFO [2019-12-17 12:37:40,920] jepsen worker 45 - jepsen.util 12105 :ok :write [15 8] INFO [2019-12-17 12:37:40,920] jepsen worker 45 - jepsen.util 12105 :invoke :cas [15 [32 49]] INFO [2019-12-17 12:37:40,934] jepsen worker 16 - jepsen.util 18466 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:37:40,934] jepsen worker 16 - jepsen.util 18466 :invoke :read [5 nil] INFO [2019-12-17 12:37:40,938] jepsen worker 71 - jepsen.util 13031 :ok :write [23 20] INFO [2019-12-17 12:37:40,938] jepsen worker 38 - jepsen.util 12638 :ok :write [12 13] INFO [2019-12-17 12:37:40,938] jepsen worker 71 - jepsen.util 13031 :invoke :write [23 9] INFO [2019-12-17 12:37:40,938] jepsen worker 38 - jepsen.util 12638 :invoke :read [12 nil] INFO [2019-12-17 12:37:40,942] jepsen worker 45 - jepsen.util 12105 :fail :cas [15 [32 49]] :ValueNotSwapFrom INFO [2019-12-17 12:37:40,942] jepsen worker 45 - jepsen.util 12105 :invoke :read [15 nil] INFO [2019-12-17 12:37:40,948] jepsen worker 8 - jepsen.util 16388 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:37:40,948] jepsen worker 8 - jepsen.util 16388 :invoke :read [2 nil] INFO [2019-12-17 12:37:40,965] jepsen worker 45 - jepsen.util 12105 :ok :read [15 8] INFO [2019-12-17 12:37:40,966] jepsen worker 45 - jepsen.util 12105 :invoke :write [15 4] INFO [2019-12-17 12:37:40,978] jepsen worker 79 - jepsen.util 12319 :ok :write [26 46] INFO [2019-12-17 12:37:40,978] jepsen worker 79 - jepsen.util 12319 :invoke :write [26 3] INFO [2019-12-17 12:37:40,988] jepsen worker 51 - jepsen.util 11391 :ok :read [17 37] INFO [2019-12-17 12:37:40,989] jepsen worker 51 - jepsen.util 11391 :invoke :read [17 nil] INFO [2019-12-17 12:37:40,995] jepsen worker 3 - jepsen.util 11523 :fail :cas [1 [16 21]] :ValueNotSwapFrom INFO [2019-12-17 12:37:40,995] jepsen worker 3 - jepsen.util 11523 :invoke :read [1 nil] INFO [2019-12-17 12:37:41,000] jepsen worker 35 - jepsen.util 16505 :info :write [11 41] :RequestCanceledException INFO [2019-12-17 12:37:41,001] jepsen worker 35 - jepsen.util 16595 :invoke :cas [11 [41 2]] INFO [2019-12-17 12:37:41,022] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":70538,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:37:41,022] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":70538,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:37:41,023] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":70542,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:37:41,023] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":70542,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:37:41,033] jepsen worker 44 - jepsen.util 17234 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:37:41,033] jepsen worker 44 - jepsen.util 17234 :invoke :write [14 36] INFO [2019-12-17 12:37:41,034] jepsen worker 48 - jepsen.util 15348 :info :write [16 42] :RequestCanceledException INFO [2019-12-17 12:37:41,035] jepsen worker 48 - jepsen.util 15438 :invoke :read [16 nil] INFO [2019-12-17 12:37:41,048] jepsen worker 43 - jepsen.util 16783 :info :write [14 32] :RequestCanceledException INFO [2019-12-17 12:37:41,049] jepsen worker 81 - jepsen.util 16281 :info :write [27 23] :RequestCanceledException INFO [2019-12-17 12:37:41,049] jepsen worker 43 - jepsen.util 16873 :invoke :cas [14 [0 12]] INFO [2019-12-17 12:37:41,051] jepsen worker 81 - jepsen.util 16371 :invoke :write [27 34] INFO [2019-12-17 12:37:41,065] jepsen worker 61 - jepsen.util 18061 :info :cas [20 [9 3]] :RequestCanceledException INFO [2019-12-17 12:37:41,066] jepsen worker 61 - jepsen.util 18151 :invoke :write [20 14] INFO [2019-12-17 12:37:41,097] jepsen worker 37 - jepsen.util 11647 :ok :write [12 17] INFO [2019-12-17 12:37:41,098] jepsen worker 37 - jepsen.util 11647 :invoke :cas [12 [21 13]] INFO [2019-12-17 12:37:41,102] jepsen worker 69 - jepsen.util 12399 :ok :read [23 20] INFO [2019-12-17 12:37:41,102] jepsen worker 69 - jepsen.util 12399 :invoke :cas [23 [38 19]] INFO [2019-12-17 12:37:41,120] jepsen worker 9 - jepsen.util 12699 :ok :write [3 27] INFO [2019-12-17 12:37:41,120] jepsen worker 30 - jepsen.util 11280 :ok :write [10 14] INFO [2019-12-17 12:37:41,120] jepsen worker 9 - jepsen.util 12699 :invoke :read [3 nil] INFO [2019-12-17 12:37:41,120] jepsen worker 30 - jepsen.util 11280 :invoke :cas [10 [30 2]] INFO [2019-12-17 12:37:41,148] jepsen worker 7 - jepsen.util 16747 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:37:41,148] jepsen worker 7 - jepsen.util 16747 :invoke :cas [2 [13 43]] INFO [2019-12-17 12:37:41,150] jepsen worker 80 - jepsen.util 11690 :ok :write [26 26] INFO [2019-12-17 12:37:41,150] jepsen worker 80 - jepsen.util 11690 :invoke :write [26 20] INFO [2019-12-17 12:37:41,162] jepsen worker 58 - jepsen.util 16888 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:37:41,162] jepsen worker 58 - jepsen.util 16888 :invoke :read [19 nil] INFO [2019-12-17 12:37:41,166] jepsen worker 9 - jepsen.util 12699 :ok :read [3 27] INFO [2019-12-17 12:37:41,166] jepsen worker 9 - jepsen.util 12699 :invoke :read [3 nil] INFO [2019-12-17 12:37:41,167] jepsen worker 63 - jepsen.util 10683 :ok :read [21 38] INFO [2019-12-17 12:37:41,167] jepsen worker 63 - jepsen.util 10683 :invoke :write [21 13] INFO [2019-12-17 12:37:41,176] jepsen worker 30 - jepsen.util 11280 :fail :cas [10 [30 2]] :ValueNotSwapFrom INFO [2019-12-17 12:37:41,176] jepsen worker 30 - jepsen.util 11280 :invoke :cas [10 [45 46]] INFO [2019-12-17 12:37:41,181] jepsen worker 5 - jepsen.util 11885 :fail :cas [1 [10 16]] :ValueNotSwapFrom INFO [2019-12-17 12:37:41,181] jepsen worker 5 - jepsen.util 11885 :invoke :read [1 nil] INFO [2019-12-17 12:37:41,194] jepsen worker 51 - jepsen.util 11391 :ok :read [17 37] INFO [2019-12-17 12:37:41,194] jepsen worker 51 - jepsen.util 11391 :invoke :write [17 13] INFO [2019-12-17 12:37:41,198] jepsen worker 15 - jepsen.util 15495 :info :cas [5 [2 42]] :RequestCanceledException INFO [2019-12-17 12:37:41,200] jepsen worker 15 - jepsen.util 15585 :invoke :cas [5 [6 47]] INFO [2019-12-17 12:37:41,204] jepsen worker 69 - jepsen.util 12399 :fail :cas [23 [38 19]] :ValueNotSwapFrom INFO [2019-12-17 12:37:41,204] jepsen worker 69 - jepsen.util 12399 :invoke :read [23 nil] INFO [2019-12-17 12:37:41,207] jepsen worker 3 - jepsen.util 11523 :ok :read [1 24] INFO [2019-12-17 12:37:41,207] jepsen worker 3 - jepsen.util 11523 :invoke :cas [1 [1 21]] INFO [2019-12-17 12:37:41,221] jepsen worker 9 - jepsen.util 12699 :ok :read [3 27] INFO [2019-12-17 12:37:41,222] jepsen worker 9 - jepsen.util 12699 :invoke :write [3 24] INFO [2019-12-17 12:37:41,233] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":70570,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:37:41,233] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":70560,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:37:41,233] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":70570,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:37:41,233] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":70560,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:37:41,233] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":70573,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:37:41,233] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":70561,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:37:41,233] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":70573,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:37:41,234] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":70561,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:37:41,234] jepsen worker 50 - jepsen.util 16610 :info :write [16 13] :RequestCanceledException INFO [2019-12-17 12:37:41,238] jepsen worker 50 - jepsen.util 16700 :invoke :write [16 24] INFO [2019-12-17 12:37:41,245] jepsen worker 45 - jepsen.util 12105 :ok :write [15 4] INFO [2019-12-17 12:37:41,246] jepsen worker 45 - jepsen.util 12105 :invoke :read [15 nil] INFO [2019-12-17 12:37:41,246] jepsen worker 74 - jepsen.util 11954 :ok :write [24 19] INFO [2019-12-17 12:37:41,246] jepsen worker 74 - jepsen.util 11954 :invoke :write [24 19] INFO [2019-12-17 12:37:41,293] jepsen worker 38 - jepsen.util 12638 :ok :read [12 17] INFO [2019-12-17 12:37:41,293] jepsen worker 38 - jepsen.util 12638 :invoke :write [12 20] INFO [2019-12-17 12:37:41,313] jepsen worker 47 - jepsen.util 13187 :ok :read [15 4] INFO [2019-12-17 12:37:41,313] jepsen worker 47 - jepsen.util 13187 :invoke :write [15 30] INFO [2019-12-17 12:37:41,322] jepsen worker 45 - jepsen.util 12105 :ok :read [15 4] INFO [2019-12-17 12:37:41,322] jepsen worker 45 - jepsen.util 12105 :invoke :write [15 29] INFO [2019-12-17 12:37:41,325] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:37:41,325] jepsen nemesis - couchbase.util Rebalance check count: 556 INFO [2019-12-17 12:37:41,325] jepsen nemesis - couchbase.util Rebalance stuck count: 537 INFO [2019-12-17 12:37:41,340] jepsen worker 63 - jepsen.util 10683 :ok :write [21 13] INFO [2019-12-17 12:37:41,340] jepsen worker 63 - jepsen.util 10683 :invoke :cas [21 [13 12]] INFO [2019-12-17 12:37:41,384] jepsen worker 3 - jepsen.util 11523 :fail :cas [1 [1 21]] :ValueNotSwapFrom INFO [2019-12-17 12:37:41,384] jepsen worker 3 - jepsen.util 11523 :invoke :write [1 5] INFO [2019-12-17 12:37:41,400] jepsen worker 57 - jepsen.util 16347 :info :cas [19 [5 2]] :RequestCanceledException INFO [2019-12-17 12:37:41,401] jepsen worker 57 - jepsen.util 16437 :invoke :read [19 nil] INFO [2019-12-17 12:37:41,426] jepsen worker 9 - jepsen.util 12699 :ok :write [3 24] INFO [2019-12-17 12:37:41,427] jepsen worker 9 - jepsen.util 12699 :invoke :write [3 49] INFO [2019-12-17 12:37:41,431] jepsen worker 14 - jepsen.util 11894 :fail :cas [4 [3 5]] :ValueNotSwapFrom INFO [2019-12-17 12:37:41,431] jepsen worker 14 - jepsen.util 11894 :invoke :read [4 nil] INFO [2019-12-17 12:37:41,437] jepsen worker 69 - jepsen.util 12399 :ok :read [23 20] INFO [2019-12-17 12:37:41,437] jepsen worker 69 - jepsen.util 12399 :invoke :write [23 37] INFO [2019-12-17 12:37:41,446] jepsen worker 63 - jepsen.util 10683 :ok :cas [21 [13 12]] INFO [2019-12-17 12:37:41,446] jepsen worker 63 - jepsen.util 10683 :invoke :write [21 48] INFO [2019-12-17 12:37:41,459] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":70589,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:37:41,459] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":70589,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:37:41,460] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":70593,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:37:41,460] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":70593,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:37:41,461] jepsen worker 42 - jepsen.util 17502 :info :cas [14 [7 23]] :RequestCanceledException INFO [2019-12-17 12:37:41,462] jepsen worker 42 - jepsen.util 17592 :invoke :read [14 nil] INFO [2019-12-17 12:37:41,500] jepsen worker 83 - jepsen.util 16823 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:37:41,500] jepsen worker 83 - jepsen.util 16823 :invoke :cas [27 [20 2]] INFO [2019-12-17 12:37:41,508] jepsen worker 87 - jepsen.util 11787 :ok :read [29 2] INFO [2019-12-17 12:37:41,508] jepsen worker 87 - jepsen.util 11787 :invoke :read [29 nil] INFO [2019-12-17 12:37:41,510] jepsen worker 74 - jepsen.util 11954 :ok :write [24 19] INFO [2019-12-17 12:37:41,510] jepsen worker 74 - jepsen.util 11954 :invoke :read [24 nil] INFO [2019-12-17 12:37:41,518] jepsen worker 22 - jepsen.util 16762 :info :write [7 13] :RequestCanceledException INFO [2019-12-17 12:37:41,519] jepsen worker 22 - jepsen.util 16852 :invoke :cas [7 [0 43]] INFO [2019-12-17 12:37:41,520] jepsen worker 63 - jepsen.util 10683 :ok :write [21 48] INFO [2019-12-17 12:37:41,520] jepsen worker 63 - jepsen.util 10683 :invoke :cas [21 [7 40]] INFO [2019-12-17 12:37:41,534] jepsen worker 1 - jepsen.util 16831 :info :write [0 34] :RequestCanceledException INFO [2019-12-17 12:37:41,535] jepsen worker 63 - jepsen.util 10683 :fail :cas [21 [7 40]] :ValueNotSwapFrom INFO [2019-12-17 12:37:41,536] jepsen worker 1 - jepsen.util 16921 :invoke :cas [0 [11 31]] INFO [2019-12-17 12:37:41,536] jepsen worker 63 - jepsen.util 10683 :invoke :write [21 9] INFO [2019-12-17 12:37:41,548] jepsen worker 39 - jepsen.util 16239 :info :write [13 5] :RequestCanceledException INFO [2019-12-17 12:37:41,549] jepsen worker 39 - jepsen.util 16329 :invoke :read [13 nil] INFO [2019-12-17 12:37:41,569] jepsen worker 5 - jepsen.util 11885 :ok :read [1 24] INFO [2019-12-17 12:37:41,569] jepsen worker 5 - jepsen.util 11885 :invoke :cas [1 [3 35]] INFO [2019-12-17 12:37:41,603] jepsen worker 74 - jepsen.util 11954 :ok :read [24 19] INFO [2019-12-17 12:37:41,604] jepsen worker 74 - jepsen.util 11954 :invoke :write [24 25] INFO [2019-12-17 12:37:41,634] jepsen worker 76 - jepsen.util 16816 :info :write [25 40] :RequestCanceledException INFO [2019-12-17 12:37:41,635] jepsen worker 76 - jepsen.util 16906 :invoke :read [25 nil] INFO [2019-12-17 12:37:41,648] jepsen worker 45 - jepsen.util 12105 :ok :write [15 29] INFO [2019-12-17 12:37:41,649] jepsen worker 45 - jepsen.util 12105 :invoke :read [15 nil] INFO [2019-12-17 12:37:41,649] jepsen worker 63 - jepsen.util 10683 :ok :write [21 9] INFO [2019-12-17 12:37:41,649] jepsen worker 63 - jepsen.util 10683 :invoke :cas [21 [45 17]] INFO [2019-12-17 12:37:41,657] jepsen worker 9 - jepsen.util 12699 :ok :write [3 49] INFO [2019-12-17 12:37:41,657] jepsen worker 9 - jepsen.util 12699 :invoke :write [3 30] INFO [2019-12-17 12:37:41,672] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":70609,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:37:41,673] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":70609,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:37:41,673] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":70613,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:37:41,673] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":70613,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:37:41,673] jepsen worker 63 - jepsen.util 10683 :fail :cas [21 [45 17]] :ValueNotSwapFrom INFO [2019-12-17 12:37:41,673] jepsen worker 63 - jepsen.util 10683 :invoke :write [21 11] INFO [2019-12-17 12:37:41,701] jepsen worker 68 - jepsen.util 15998 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:37:41,701] jepsen worker 68 - jepsen.util 15998 :invoke :read [22 nil] INFO [2019-12-17 12:37:41,711] jepsen worker 45 - jepsen.util 12105 :ok :read [15 29] INFO [2019-12-17 12:37:41,711] jepsen worker 45 - jepsen.util 12105 :invoke :cas [15 [9 23]] INFO [2019-12-17 12:37:41,718] jepsen worker 49 - jepsen.util 16699 :info :write [16 45] :RequestCanceledException INFO [2019-12-17 12:37:41,719] jepsen worker 49 - jepsen.util 16789 :invoke :read [16 nil] INFO [2019-12-17 12:37:41,733] jepsen worker 6 - jepsen.util 17016 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:37:41,733] jepsen worker 6 - jepsen.util 17016 :invoke :write [2 18] INFO [2019-12-17 12:37:41,747] jepsen worker 87 - jepsen.util 11787 :ok :read [29 2] INFO [2019-12-17 12:37:41,747] jepsen worker 87 - jepsen.util 11787 :invoke :write [29 38] INFO [2019-12-17 12:37:41,765] jepsen worker 55 - jepsen.util 12835 :info :write [18 20] :RequestCanceledException INFO [2019-12-17 12:37:41,766] jepsen worker 55 - jepsen.util 12925 :invoke :cas [18 [4 20]] INFO [2019-12-17 12:37:41,804] jepsen worker 45 - jepsen.util 12105 :fail :cas [15 [9 23]] :ValueNotSwapFrom INFO [2019-12-17 12:37:41,804] jepsen worker 45 - jepsen.util 12105 :invoke :read [15 nil] INFO [2019-12-17 12:37:41,843] jepsen worker 45 - jepsen.util 12105 :ok :read [15 29] INFO [2019-12-17 12:37:41,843] jepsen worker 45 - jepsen.util 12105 :invoke :cas [15 [17 36]] INFO [2019-12-17 12:37:41,853] jepsen worker 63 - jepsen.util 10683 :ok :write [21 11] INFO [2019-12-17 12:37:41,854] jepsen worker 63 - jepsen.util 10683 :invoke :cas [21 [12 18]] INFO [2019-12-17 12:37:41,859] jepsen worker 4 - jepsen.util 11344 :fail :cas [1 [8 35]] :ValueNotSwapFrom INFO [2019-12-17 12:37:41,859] jepsen worker 4 - jepsen.util 11344 :invoke :cas [1 [26 6]] INFO [2019-12-17 12:37:41,865] jepsen worker 41 - jepsen.util 18041 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:37:41,865] jepsen worker 41 - jepsen.util 18041 :invoke :write [13 0] INFO [2019-12-17 12:37:41,868] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":70628,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:37:41,868] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":70628,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:37:41,868] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":70630,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:37:41,868] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":70630,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:37:41,873] jepsen worker 5 - jepsen.util 11885 :fail :cas [1 [3 35]] :ValueNotSwapFrom INFO [2019-12-17 12:37:41,874] jepsen worker 5 - jepsen.util 11885 :invoke :cas [1 [25 0]] INFO [2019-12-17 12:37:41,881] jepsen worker 45 - jepsen.util 12105 :fail :cas [15 [17 36]] :ValueNotSwapFrom INFO [2019-12-17 12:37:41,881] jepsen worker 45 - jepsen.util 12105 :invoke :read [15 nil] INFO [2019-12-17 12:37:41,899] jepsen worker 45 - jepsen.util 12105 :ok :read [15 29] INFO [2019-12-17 12:37:41,899] jepsen worker 45 - jepsen.util 12105 :invoke :cas [15 [33 47]] INFO [2019-12-17 12:37:41,918] jepsen worker 28 - jepsen.util 10288 :info :write [9 47] :RequestCanceledException INFO [2019-12-17 12:37:41,918] jepsen worker 24 - jepsen.util 15774 :info :write [8 6] :RequestCanceledException INFO [2019-12-17 12:37:41,919] jepsen worker 24 - jepsen.util 15864 :invoke :cas [8 [14 40]] INFO [2019-12-17 12:37:41,920] jepsen worker 28 - jepsen.util 10378 :invoke :cas [9 [28 41]] INFO [2019-12-17 12:37:41,926] jepsen worker 63 - jepsen.util 10683 :fail :cas [21 [12 18]] :ValueNotSwapFrom INFO [2019-12-17 12:37:41,926] jepsen worker 63 - jepsen.util 10683 :invoke :cas [21 [10 45]] INFO [2019-12-17 12:37:41,929] jepsen worker 45 - jepsen.util 12105 :fail :cas [15 [33 47]] :ValueNotSwapFrom INFO [2019-12-17 12:37:41,929] jepsen worker 45 - jepsen.util 12105 :invoke :read [15 nil] INFO [2019-12-17 12:37:41,935] jepsen worker 10 - jepsen.util 11890 :info :cas [3 [25 11]] :RequestCanceledException INFO [2019-12-17 12:37:41,935] jepsen worker 40 - jepsen.util 17410 :info :cas [13 [38 42]] :RequestCanceledException INFO [2019-12-17 12:37:41,936] jepsen worker 10 - jepsen.util 11980 :invoke :cas [3 [49 30]] INFO [2019-12-17 12:37:41,937] jepsen worker 40 - jepsen.util 17500 :invoke :write [13 49] INFO [2019-12-17 12:37:41,948] jepsen worker 34 - jepsen.util 17314 :info :cas [11 [34 14]] :RequestCanceledException INFO [2019-12-17 12:37:41,949] jepsen worker 34 - jepsen.util 17404 :invoke :cas [11 [17 3]] INFO [2019-12-17 12:37:41,966] jepsen worker 45 - jepsen.util 12105 :ok :read [15 29] INFO [2019-12-17 12:37:41,966] jepsen worker 45 - jepsen.util 12105 :invoke :cas [15 [33 23]] INFO [2019-12-17 12:37:41,968] jepsen worker 63 - jepsen.util 10683 :fail :cas [21 [10 45]] :ValueNotSwapFrom INFO [2019-12-17 12:37:41,968] jepsen worker 63 - jepsen.util 10683 :invoke :cas [21 [29 43]] INFO [2019-12-17 12:37:41,992] jepsen worker 45 - jepsen.util 12105 :fail :cas [15 [33 23]] :ValueNotSwapFrom INFO [2019-12-17 12:37:41,992] jepsen worker 45 - jepsen.util 12105 :invoke :cas [15 [4 19]] INFO [2019-12-17 12:37:41,997] jepsen worker 63 - jepsen.util 10683 :fail :cas [21 [29 43]] :ValueNotSwapFrom INFO [2019-12-17 12:37:41,997] jepsen worker 63 - jepsen.util 10683 :invoke :cas [21 [33 17]] INFO [2019-12-17 12:37:42,001] jepsen worker 56 - jepsen.util 13106 :info :write [18 5] :RequestCanceledException INFO [2019-12-17 12:37:42,002] jepsen worker 56 - jepsen.util 13196 :invoke :read [18 nil] INFO [2019-12-17 12:37:42,006] jepsen worker 45 - jepsen.util 12105 :fail :cas [15 [4 19]] :ValueNotSwapFrom INFO [2019-12-17 12:37:42,006] jepsen worker 45 - jepsen.util 12105 :invoke :read [15 nil] INFO [2019-12-17 12:37:42,024] jepsen worker 45 - jepsen.util 12105 :ok :read [15 29] INFO [2019-12-17 12:37:42,025] jepsen worker 45 - jepsen.util 12105 :invoke :write [15 25] INFO [2019-12-17 12:37:42,028] jepsen worker 55 - jepsen.util 12925 :fail :cas [18 [4 20]] :ValueNotSwapFrom INFO [2019-12-17 12:37:42,028] jepsen worker 55 - jepsen.util 12925 :invoke :write [18 3] INFO [2019-12-17 12:37:42,033] jepsen worker 29 - jepsen.util 10649 :info :cas [9 [10 25]] :RequestCanceledException INFO [2019-12-17 12:37:42,033] jepsen worker 64 - jepsen.util 9424 :fail :read [21 nil] :RequestTimeoutException INFO [2019-12-17 12:37:42,033] jepsen worker 64 - jepsen.util 9424 :invoke :write [21 9] INFO [2019-12-17 12:37:42,034] jepsen worker 29 - jepsen.util 10739 :invoke :read [9 nil] INFO [2019-12-17 12:37:42,052] jepsen worker 56 - jepsen.util 13196 :ok :read [18 39] INFO [2019-12-17 12:37:42,052] jepsen worker 56 - jepsen.util 13196 :invoke :cas [18 [13 39]] INFO [2019-12-17 12:37:42,056] jepsen worker 29 - jepsen.util 10739 :ok :read [9 35] INFO [2019-12-17 12:37:42,056] jepsen worker 29 - jepsen.util 10739 :invoke :write [9 8] INFO [2019-12-17 12:37:42,083] jepsen worker 4 - jepsen.util 11344 :fail :cas [1 [26 6]] :ValueNotSwapFrom INFO [2019-12-17 12:37:42,083] jepsen worker 4 - jepsen.util 11344 :invoke :cas [1 [30 5]] INFO [2019-12-17 12:37:42,102] jepsen worker 14 - jepsen.util 11894 :ok :read [4 21] INFO [2019-12-17 12:37:42,102] jepsen worker 14 - jepsen.util 11894 :invoke :write [4 12] INFO [2019-12-17 12:37:42,108] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":70650,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:37:42,108] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":70650,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:37:42,108] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":70652,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:37:42,108] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":70652,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:37:42,108] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":70655,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:37:42,108] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":70655,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:37:42,124] jepsen worker 64 - jepsen.util 9424 :ok :write [21 9] INFO [2019-12-17 12:37:42,124] jepsen worker 64 - jepsen.util 9424 :invoke :read [21 nil] INFO [2019-12-17 12:37:42,132] jepsen worker 17 - jepsen.util 16847 :info :write [5 23] :RequestCanceledException INFO [2019-12-17 12:37:42,133] jepsen worker 17 - jepsen.util 16937 :invoke :write [5 28] INFO [2019-12-17 12:37:42,151] jepsen worker 64 - jepsen.util 9424 :ok :read [21 9] INFO [2019-12-17 12:37:42,151] jepsen worker 64 - jepsen.util 9424 :invoke :cas [21 [38 21]] INFO [2019-12-17 12:37:42,159] jepsen worker 29 - jepsen.util 10739 :ok :write [9 8] INFO [2019-12-17 12:37:42,159] jepsen worker 29 - jepsen.util 10739 :invoke :write [9 48] INFO [2019-12-17 12:37:42,165] jepsen worker 60 - jepsen.util 17700 :info :write [20 42] :RequestCanceledException INFO [2019-12-17 12:37:42,166] jepsen worker 60 - jepsen.util 17790 :invoke :read [20 nil] INFO [2019-12-17 12:37:42,170] jepsen worker 64 - jepsen.util 9424 :fail :cas [21 [38 21]] :ValueNotSwapFrom INFO [2019-12-17 12:37:42,170] jepsen worker 64 - jepsen.util 9424 :invoke :cas [21 [23 49]] INFO [2019-12-17 12:37:42,184] jepsen worker 64 - jepsen.util 9424 :fail :cas [21 [23 49]] :ValueNotSwapFrom INFO [2019-12-17 12:37:42,184] jepsen worker 64 - jepsen.util 9424 :invoke :write [21 6] INFO [2019-12-17 12:37:42,200] jepsen worker 62 - jepsen.util 17432 :info :write [20 14] :RequestCanceledException INFO [2019-12-17 12:37:42,201] jepsen worker 62 - jepsen.util 17522 :invoke :read [20 nil] INFO [2019-12-17 12:37:42,215] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":70674,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:37:42,215] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":70674,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:37:42,215] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":70675,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:37:42,215] jepsen worker 75 - jepsen.util 16365 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:37:42,215] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":70675,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:37:42,215] jepsen worker 75 - jepsen.util 16365 :invoke :read [25 nil] INFO [2019-12-17 12:37:42,215] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":70676,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:37:42,215] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":70676,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:37:42,225] jepsen worker 29 - jepsen.util 10739 :ok :write [9 48] INFO [2019-12-17 12:37:42,225] jepsen worker 29 - jepsen.util 10739 :invoke :read [9 nil] INFO [2019-12-17 12:37:42,233] jepsen worker 86 - jepsen.util 17366 :info :write [28 7] :RequestCanceledException INFO [2019-12-17 12:37:42,235] jepsen worker 86 - jepsen.util 17456 :invoke :cas [28 [44 37]] INFO [2019-12-17 12:37:42,240] jepsen worker 29 - jepsen.util 10739 :ok :read [9 48] INFO [2019-12-17 12:37:42,240] jepsen worker 29 - jepsen.util 10739 :invoke :cas [9 [13 47]] INFO [2019-12-17 12:37:42,248] jepsen worker 73 - jepsen.util 11143 :info :cas [24 [2 35]] :RequestCanceledException INFO [2019-12-17 12:37:42,248] jepsen worker 19 - jepsen.util 12619 :fail :read [6 nil] :RequestTimeoutException INFO [2019-12-17 12:37:42,248] jepsen worker 19 - jepsen.util 12619 :invoke :read [6 nil] INFO [2019-12-17 12:37:42,249] jepsen worker 73 - jepsen.util 11233 :invoke :cas [24 [0 42]] INFO [2019-12-17 12:37:42,260] jepsen worker 64 - jepsen.util 9424 :ok :write [21 6] INFO [2019-12-17 12:37:42,260] jepsen worker 64 - jepsen.util 9424 :invoke :write [21 1] INFO [2019-12-17 12:37:42,261] jepsen worker 29 - jepsen.util 10739 :fail :cas [9 [13 47]] :ValueNotSwapFrom INFO [2019-12-17 12:37:42,261] jepsen worker 29 - jepsen.util 10739 :invoke :write [9 20] INFO [2019-12-17 12:37:42,265] jepsen worker 26 - jepsen.util 17396 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:37:42,265] jepsen worker 26 - jepsen.util 17396 :invoke :write [8 36] INFO [2019-12-17 12:37:42,273] jepsen worker 88 - jepsen.util 12328 :ok :read [29 2] INFO [2019-12-17 12:37:42,273] jepsen worker 88 - jepsen.util 12328 :invoke :write [29 19] INFO [2019-12-17 12:37:42,299] jepsen worker 4 - jepsen.util 11344 :fail :cas [1 [30 5]] :ValueNotSwapFrom INFO [2019-12-17 12:37:42,299] jepsen worker 4 - jepsen.util 11344 :invoke :read [1 nil] INFO [2019-12-17 12:37:42,300] jepsen worker 11 - jepsen.util 11711 :info :write [3 47] :RequestCanceledException INFO [2019-12-17 12:37:42,301] jepsen worker 11 - jepsen.util 11801 :invoke :read [3 nil] INFO [2019-12-17 12:37:42,303] jepsen worker 45 - jepsen.util 12105 :ok :write [15 25] INFO [2019-12-17 12:37:42,303] jepsen worker 45 - jepsen.util 12105 :invoke :write [15 3] INFO [2019-12-17 12:37:42,306] jepsen worker 74 - jepsen.util 11954 :ok :write [24 25] INFO [2019-12-17 12:37:42,306] jepsen worker 74 - jepsen.util 11954 :invoke :write [24 30] INFO [2019-12-17 12:37:42,306] jepsen worker 9 - jepsen.util 12699 :ok :write [3 30] INFO [2019-12-17 12:37:42,306] jepsen worker 9 - jepsen.util 12699 :invoke :read [3 nil] INFO [2019-12-17 12:37:42,324] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":70681,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:37:42,324] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":70681,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:37:42,324] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":70685,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:37:42,325] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":70685,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:37:42,333] jepsen worker 65 - jepsen.util 10595 :info :write [21 26] :RequestCanceledException INFO [2019-12-17 12:37:42,333] jepsen worker 33 - jepsen.util 16593 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:37:42,333] jepsen worker 33 - jepsen.util 16593 :invoke :cas [11 [22 36]] INFO [2019-12-17 12:37:42,334] jepsen worker 65 - jepsen.util 10685 :invoke :write [21 42] INFO [2019-12-17 12:37:42,334] jepsen worker 18 - jepsen.util 12528 :info :write [6 42] :RequestCanceledException INFO [2019-12-17 12:37:42,335] jepsen worker 18 - jepsen.util 12618 :invoke :read [6 nil] INFO [2019-12-17 12:37:42,340] jepsen worker 27 - jepsen.util 11187 :info :write [9 35] :RequestCanceledException INFO [2019-12-17 12:37:42,341] jepsen worker 27 - jepsen.util 11277 :invoke :read [9 nil] INFO [2019-12-17 12:37:42,351] jepsen worker 19 - jepsen.util 12619 :ok :read [6 36] INFO [2019-12-17 12:37:42,352] jepsen worker 19 - jepsen.util 12619 :invoke :read [6 nil] INFO [2019-12-17 12:37:42,367] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:37:42,367] jepsen nemesis - couchbase.util Rebalance check count: 557 INFO [2019-12-17 12:37:42,367] jepsen nemesis - couchbase.util Rebalance stuck count: 538 INFO [2019-12-17 12:37:42,373] jepsen worker 28 - jepsen.util 10378 :fail :cas [9 [28 41]] :ValueNotSwapFrom INFO [2019-12-17 12:37:42,373] jepsen worker 28 - jepsen.util 10378 :invoke :cas [9 [34 47]] INFO [2019-12-17 12:37:42,378] jepsen worker 73 - jepsen.util 11233 :fail :cas [24 [0 42]] :ValueNotSwapFrom INFO [2019-12-17 12:37:42,378] jepsen worker 73 - jepsen.util 11233 :invoke :write [24 44] INFO [2019-12-17 12:37:42,401] jepsen worker 66 - jepsen.util 16626 :info :write [22 31] :RequestCanceledException INFO [2019-12-17 12:37:42,401] jepsen worker 70 - jepsen.util 12670 :info :write [23 2] :RequestCanceledException INFO [2019-12-17 12:37:42,401] jepsen worker 56 - jepsen.util 13196 :fail :cas [18 [13 39]] :ValueNotSwapFrom INFO [2019-12-17 12:37:42,402] jepsen worker 56 - jepsen.util 13196 :invoke :read [18 nil] INFO [2019-12-17 12:37:42,402] jepsen worker 70 - jepsen.util 12760 :invoke :cas [23 [16 9]] INFO [2019-12-17 12:37:42,403] jepsen worker 66 - jepsen.util 16716 :invoke :cas [22 [32 11]] INFO [2019-12-17 12:37:42,409] jepsen worker 18 - jepsen.util 12618 :ok :read [6 36] INFO [2019-12-17 12:37:42,409] jepsen worker 18 - jepsen.util 12618 :invoke :write [6 27] INFO [2019-12-17 12:37:42,413] jepsen worker 9 - jepsen.util 12699 :ok :read [3 30] INFO [2019-12-17 12:37:42,413] jepsen worker 9 - jepsen.util 12699 :invoke :write [3 16] INFO [2019-12-17 12:37:42,418] jepsen worker 85 - jepsen.util 17275 :info :cas [28 [5 39]] :RequestCanceledException INFO [2019-12-17 12:37:42,418] jepsen worker 4 - jepsen.util 11344 :ok :read [1 24] INFO [2019-12-17 12:37:42,418] jepsen worker 67 - jepsen.util 16807 :info :write [22 38] :RequestCanceledException INFO [2019-12-17 12:37:42,419] jepsen worker 4 - jepsen.util 11344 :invoke :write [1 5] INFO [2019-12-17 12:37:42,420] jepsen worker 67 - jepsen.util 16897 :invoke :cas [22 [10 6]] INFO [2019-12-17 12:37:42,420] jepsen worker 85 - jepsen.util 17365 :invoke :read [28 nil] INFO [2019-12-17 12:37:42,434] jepsen worker 32 - jepsen.util 12632 :info :cas [10 [46 29]] :RequestCanceledException INFO [2019-12-17 12:37:42,435] jepsen worker 32 - jepsen.util 12722 :invoke :read [10 nil] INFO [2019-12-17 12:37:42,447] jepsen worker 82 - jepsen.util 17092 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:37:42,447] jepsen worker 53 - jepsen.util 11753 :info :write [17 23] :RequestCanceledException INFO [2019-12-17 12:37:42,448] jepsen worker 82 - jepsen.util 17092 :invoke :cas [27 [19 17]] INFO [2019-12-17 12:37:42,448] jepsen worker 53 - jepsen.util 11843 :invoke :write [17 48] INFO [2019-12-17 12:37:42,457] jepsen worker 28 - jepsen.util 10378 :fail :cas [9 [34 47]] :ValueNotSwapFrom INFO [2019-12-17 12:37:42,457] jepsen worker 28 - jepsen.util 10378 :invoke :cas [9 [45 1]] INFO [2019-12-17 12:37:42,460] jepsen worker 37 - jepsen.util 11647 :fail :cas [12 [21 13]] :ValueNotSwapFrom INFO [2019-12-17 12:37:42,460] jepsen worker 37 - jepsen.util 11647 :invoke :read [12 nil] INFO [2019-12-17 12:37:42,465] jepsen worker 89 - jepsen.util 11969 :fail :read [29 nil] :RequestTimeoutException INFO [2019-12-17 12:37:42,465] jepsen worker 84 - jepsen.util 16374 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:37:42,465] jepsen worker 89 - jepsen.util 11969 :invoke :write [29 17] INFO [2019-12-17 12:37:42,465] jepsen worker 84 - jepsen.util 16374 :invoke :read [28 nil] INFO [2019-12-17 12:37:42,472] jepsen worker 64 - jepsen.util 9424 :ok :write [21 1] INFO [2019-12-17 12:37:42,473] jepsen worker 64 - jepsen.util 9424 :invoke :read [21 nil] INFO [2019-12-17 12:37:42,491] jepsen worker 64 - jepsen.util 9424 :ok :read [21 1] INFO [2019-12-17 12:37:42,491] jepsen worker 64 - jepsen.util 9424 :invoke :cas [21 [48 36]] INFO [2019-12-17 12:37:42,521] jepsen worker 64 - jepsen.util 9424 :fail :cas [21 [48 36]] :ValueNotSwapFrom INFO [2019-12-17 12:37:42,521] jepsen worker 64 - jepsen.util 9424 :invoke :write [21 6] INFO [2019-12-17 12:37:42,533] jepsen worker 20 - jepsen.util 11900 :info :write [6 30] :RequestCanceledException INFO [2019-12-17 12:37:42,534] jepsen worker 20 - jepsen.util 11990 :invoke :write [6 8] INFO [2019-12-17 12:37:42,544] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":70717,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:37:42,544] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":70717,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:37:42,544] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":70720,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:37:42,544] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":70720,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:37:42,544] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":70721,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:37:42,544] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":70721,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:37:42,547] jepsen worker 23 - jepsen.util 17033 :info :cas [7 [47 25]] :RequestCanceledException INFO [2019-12-17 12:37:42,548] jepsen worker 23 - jepsen.util 17123 :invoke :write [7 35] INFO [2019-12-17 12:37:42,600] jepsen worker 12 - jepsen.util 12342 :info :write [4 46] :RequestCanceledException INFO [2019-12-17 12:37:42,600] jepsen worker 54 - jepsen.util 12744 :info :write [18 27] :RequestCanceledException INFO [2019-12-17 12:37:42,601] jepsen worker 54 - jepsen.util 12834 :invoke :cas [18 [36 44]] INFO [2019-12-17 12:37:42,601] jepsen worker 12 - jepsen.util 12432 :invoke :write [4 16] INFO [2019-12-17 12:37:42,604] jepsen worker 29 - jepsen.util 10739 :ok :write [9 20] INFO [2019-12-17 12:37:42,604] jepsen worker 29 - jepsen.util 10739 :invoke :read [9 nil] INFO [2019-12-17 12:37:42,620] jepsen worker 29 - jepsen.util 10739 :ok :read [9 20] INFO [2019-12-17 12:37:42,620] jepsen worker 29 - jepsen.util 10739 :invoke :write [9 28] INFO [2019-12-17 12:37:42,621] jepsen worker 64 - jepsen.util 9424 :ok :write [21 6] INFO [2019-12-17 12:37:42,621] jepsen worker 64 - jepsen.util 9424 :invoke :read [21 nil] INFO [2019-12-17 12:37:42,634] jepsen worker 0 - jepsen.util 17280 :info :cas [0 [21 11]] :RequestCanceledException INFO [2019-12-17 12:37:42,634] jepsen worker 21 - jepsen.util 16131 :info :write [7 45] :RequestCanceledException INFO [2019-12-17 12:37:42,635] jepsen worker 0 - jepsen.util 17370 :invoke :read [0 nil] INFO [2019-12-17 12:37:42,635] jepsen worker 21 - jepsen.util 16221 :invoke :write [7 12] INFO [2019-12-17 12:37:42,647] jepsen worker 59 - jepsen.util 15809 :info :cas [19 [28 37]] :RequestCanceledException INFO [2019-12-17 12:37:42,648] jepsen worker 59 - jepsen.util 15899 :invoke :cas [19 [45 37]] INFO [2019-12-17 12:37:42,651] jepsen worker 64 - jepsen.util 9424 :ok :read [21 6] INFO [2019-12-17 12:37:42,652] jepsen worker 64 - jepsen.util 9424 :invoke :read [21 nil] INFO [2019-12-17 12:37:42,653] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":70742,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:37:42,653] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":70742,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:37:42,653] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":70747,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:37:42,653] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":70747,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:37:42,665] jepsen worker 13 - jepsen.util 11623 :info :write [4 12] :RequestCanceledException INFO [2019-12-17 12:37:42,666] jepsen worker 13 - jepsen.util 11713 :invoke :read [4 nil] INFO [2019-12-17 12:37:42,677] jepsen worker 64 - jepsen.util 9424 :ok :read [21 6] INFO [2019-12-17 12:37:42,678] jepsen worker 64 - jepsen.util 9424 :invoke :write [21 17] INFO [2019-12-17 12:37:42,709] jepsen worker 78 - jepsen.util 11958 :ok :write [26 1] INFO [2019-12-17 12:37:42,709] jepsen worker 78 - jepsen.util 11958 :invoke :cas [26 [20 44]] INFO [2019-12-17 12:37:42,717] jepsen worker 29 - jepsen.util 10739 :ok :write [9 28] INFO [2019-12-17 12:37:42,717] jepsen worker 29 - jepsen.util 10739 :invoke :write [9 41] INFO [2019-12-17 12:37:42,747] jepsen worker 2 - jepsen.util 16112 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:37:42,748] jepsen worker 2 - jepsen.util 16112 :invoke :read [0 nil] INFO [2019-12-17 12:37:42,764] jepsen worker 77 - jepsen.util 17807 :info :write [25 17] :RequestCanceledException INFO [2019-12-17 12:37:42,766] jepsen worker 77 - jepsen.util 17897 :invoke :cas [25 [48 24]] INFO [2019-12-17 12:37:42,769] jepsen worker 64 - jepsen.util 9424 :ok :write [21 17] INFO [2019-12-17 12:37:42,769] jepsen worker 64 - jepsen.util 9424 :invoke :cas [21 [23 13]] INFO [2019-12-17 12:37:42,790] jepsen worker 64 - jepsen.util 9424 :fail :cas [21 [23 13]] :ValueNotSwapFrom INFO [2019-12-17 12:37:42,790] jepsen worker 64 - jepsen.util 9424 :invoke :write [21 3] INFO [2019-12-17 12:37:42,792] jepsen worker 45 - jepsen.util 12105 :ok :write [15 3] INFO [2019-12-17 12:37:42,792] jepsen worker 45 - jepsen.util 12105 :invoke :read [15 nil] INFO [2019-12-17 12:37:42,815] jepsen worker 56 - jepsen.util 13196 :ok :read [18 39] INFO [2019-12-17 12:37:42,816] jepsen worker 56 - jepsen.util 13196 :invoke :write [18 23] INFO [2019-12-17 12:37:42,833] jepsen worker 52 - jepsen.util 11572 :info :write [17 0] :RequestCanceledException INFO [2019-12-17 12:37:42,834] jepsen worker 36 - jepsen.util 12816 :info :write [12 11] :RequestCanceledException INFO [2019-12-17 12:37:42,834] jepsen worker 31 - jepsen.util 10381 :info :write [10 3] :RequestCanceledException INFO [2019-12-17 12:37:42,835] jepsen worker 36 - jepsen.util 12906 :invoke :write [12 40] INFO [2019-12-17 12:37:42,835] jepsen worker 31 - jepsen.util 10471 :invoke :cas [10 [21 42]] INFO [2019-12-17 12:37:42,835] jepsen worker 52 - jepsen.util 11662 :invoke :read [17 nil] INFO [2019-12-17 12:37:42,853] jepsen worker 29 - jepsen.util 10739 :ok :write [9 41] INFO [2019-12-17 12:37:42,853] jepsen worker 29 - jepsen.util 10739 :invoke :cas [9 [4 36]] INFO [2019-12-17 12:37:42,860] jepsen worker 45 - jepsen.util 12105 :ok :read [15 3] INFO [2019-12-17 12:37:42,860] jepsen worker 45 - jepsen.util 12105 :invoke :read [15 nil] INFO [2019-12-17 12:37:42,880] jepsen worker 70 - jepsen.util 12760 :fail :cas [23 [16 9]] :ValueNotSwapFrom INFO [2019-12-17 12:37:42,881] jepsen worker 70 - jepsen.util 12760 :invoke :read [23 nil] INFO [2019-12-17 12:37:42,888] jepsen worker 5 - jepsen.util 11885 :fail :cas [1 [25 0]] :ValueNotSwapFrom INFO [2019-12-17 12:37:42,889] jepsen worker 5 - jepsen.util 11885 :invoke :read [1 nil] INFO [2019-12-17 12:37:42,901] jepsen worker 25 - jepsen.util 17755 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:37:42,901] jepsen worker 25 - jepsen.util 17755 :invoke :write [8 18] INFO [2019-12-17 12:37:42,916] jepsen worker 13 - jepsen.util 11713 :ok :read [4 21] INFO [2019-12-17 12:37:42,916] jepsen worker 13 - jepsen.util 11713 :invoke :cas [4 [14 36]] INFO [2019-12-17 12:37:42,928] jepsen worker 29 - jepsen.util 10739 :fail :cas [9 [4 36]] :ValueNotSwapFrom INFO [2019-12-17 12:37:42,929] jepsen worker 29 - jepsen.util 10739 :invoke :read [9 nil] INFO [2019-12-17 12:37:42,934] jepsen worker 46 - jepsen.util 11926 :fail :read [15 nil] :RequestTimeoutException INFO [2019-12-17 12:37:42,934] jepsen worker 72 - jepsen.util 12402 :fail :read [24 nil] :RequestTimeoutException INFO [2019-12-17 12:37:42,934] jepsen worker 46 - jepsen.util 11926 :invoke :cas [15 [3 10]] INFO [2019-12-17 12:37:42,934] jepsen worker 72 - jepsen.util 12402 :invoke :write [24 46] INFO [2019-12-17 12:37:42,937] jepsen worker 45 - jepsen.util 12105 :ok :read [15 3] INFO [2019-12-17 12:37:42,937] jepsen worker 45 - jepsen.util 12105 :invoke :cas [15 [39 49]] INFO [2019-12-17 12:37:42,952] jepsen worker 64 - jepsen.util 9424 :ok :write [21 3] INFO [2019-12-17 12:37:42,953] jepsen worker 64 - jepsen.util 9424 :invoke :write [21 33] INFO [2019-12-17 12:37:42,957] jepsen worker 29 - jepsen.util 10739 :ok :read [9 41] INFO [2019-12-17 12:37:42,957] jepsen worker 29 - jepsen.util 10739 :invoke :read [9 nil] INFO [2019-12-17 12:37:42,960] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":70766,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:37:42,961] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":70766,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:37:42,961] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":70769,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:37:42,961] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":70769,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:37:42,964] jepsen worker 45 - jepsen.util 12105 :fail :cas [15 [39 49]] :ValueNotSwapFrom INFO [2019-12-17 12:37:42,964] jepsen worker 45 - jepsen.util 12105 :invoke :read [15 nil] INFO [2019-12-17 12:37:42,983] jepsen worker 37 - jepsen.util 11647 :ok :read [12 17] INFO [2019-12-17 12:37:42,983] jepsen worker 37 - jepsen.util 11647 :invoke :write [12 49] INFO [2019-12-17 12:37:43,006] jepsen worker 29 - jepsen.util 10739 :ok :read [9 41] INFO [2019-12-17 12:37:43,006] jepsen worker 29 - jepsen.util 10739 :invoke :cas [9 [38 15]] INFO [2019-12-17 12:37:43,011] jepsen worker 45 - jepsen.util 12105 :ok :read [15 3] INFO [2019-12-17 12:37:43,011] jepsen worker 45 - jepsen.util 12105 :invoke :read [15 nil] INFO [2019-12-17 12:37:43,018] jepsen worker 71 - jepsen.util 13031 :info :write [23 9] :RequestCanceledException INFO [2019-12-17 12:37:43,018] jepsen worker 35 - jepsen.util 16595 :info :cas [11 [41 2]] :RequestCanceledException INFO [2019-12-17 12:37:43,019] jepsen worker 35 - jepsen.util 16685 :invoke :write [11 31] INFO [2019-12-17 12:37:43,019] jepsen worker 71 - jepsen.util 13121 :invoke :cas [23 [10 49]] INFO [2019-12-17 12:37:43,025] jepsen worker 45 - jepsen.util 12105 :ok :read [15 3] INFO [2019-12-17 12:37:43,025] jepsen worker 45 - jepsen.util 12105 :invoke :cas [15 [36 20]] INFO [2019-12-17 12:37:43,034] jepsen worker 16 - jepsen.util 18466 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:37:43,034] jepsen worker 16 - jepsen.util 18466 :invoke :read [5 nil] INFO [2019-12-17 12:37:43,047] jepsen worker 45 - jepsen.util 12105 :fail :cas [15 [36 20]] :ValueNotSwapFrom INFO [2019-12-17 12:37:43,047] jepsen worker 45 - jepsen.util 12105 :invoke :read [15 nil] INFO [2019-12-17 12:37:43,048] jepsen worker 8 - jepsen.util 16388 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:37:43,048] jepsen worker 8 - jepsen.util 16388 :invoke :cas [2 [23 43]] INFO [2019-12-17 12:37:43,050] jepsen worker 29 - jepsen.util 10739 :fail :cas [9 [38 15]] :ValueNotSwapFrom INFO [2019-12-17 12:37:43,051] jepsen worker 29 - jepsen.util 10739 :invoke :cas [9 [10 15]] INFO [2019-12-17 12:37:43,053] jepsen worker 13 - jepsen.util 11713 :fail :cas [4 [14 36]] :ValueNotSwapFrom INFO [2019-12-17 12:37:43,053] jepsen worker 13 - jepsen.util 11713 :invoke :read [4 nil] INFO [2019-12-17 12:37:43,064] jepsen worker 79 - jepsen.util 12319 :info :write [26 3] :RequestCanceledException INFO [2019-12-17 12:37:43,068] jepsen worker 79 - jepsen.util 12409 :invoke :cas [26 [47 31]] INFO [2019-12-17 12:37:43,079] jepsen worker 45 - jepsen.util 12105 :ok :read [15 3] INFO [2019-12-17 12:37:43,079] jepsen worker 45 - jepsen.util 12105 :invoke :write [15 49] INFO [2019-12-17 12:37:43,080] jepsen worker 64 - jepsen.util 9424 :ok :write [21 33] INFO [2019-12-17 12:37:43,081] jepsen worker 64 - jepsen.util 9424 :invoke :write [21 3] INFO [2019-12-17 12:37:43,083] jepsen worker 29 - jepsen.util 10739 :fail :cas [9 [10 15]] :ValueNotSwapFrom INFO [2019-12-17 12:37:43,083] jepsen worker 29 - jepsen.util 10739 :invoke :read [9 nil] INFO [2019-12-17 12:37:43,091] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":70783,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:37:43,091] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":70783,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:37:43,092] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":70786,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:37:43,092] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":70786,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:37:43,092] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":70787,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:37:43,092] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":70787,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:37:43,094] jepsen worker 70 - jepsen.util 12760 :ok :read [23 20] INFO [2019-12-17 12:37:43,094] jepsen worker 70 - jepsen.util 12760 :invoke :read [23 nil] INFO [2019-12-17 12:37:43,114] jepsen worker 9 - jepsen.util 12699 :ok :write [3 16] INFO [2019-12-17 12:37:43,115] jepsen worker 9 - jepsen.util 12699 :invoke :cas [3 [26 15]] INFO [2019-12-17 12:37:43,118] jepsen worker 18 - jepsen.util 12618 :ok :write [6 27] INFO [2019-12-17 12:37:43,118] jepsen worker 18 - jepsen.util 12618 :invoke :read [6 nil] INFO [2019-12-17 12:37:43,120] jepsen worker 29 - jepsen.util 10739 :ok :read [9 41] INFO [2019-12-17 12:37:43,120] jepsen worker 29 - jepsen.util 10739 :invoke :read [9 nil] INFO [2019-12-17 12:37:43,122] jepsen worker 74 - jepsen.util 11954 :ok :write [24 30] INFO [2019-12-17 12:37:43,122] jepsen worker 74 - jepsen.util 11954 :invoke :write [24 43] INFO [2019-12-17 12:37:43,133] jepsen worker 61 - jepsen.util 18151 :info :write [20 14] :RequestCanceledException INFO [2019-12-17 12:37:43,133] jepsen worker 44 - jepsen.util 17234 :info :write [14 36] :RequestCanceledException INFO [2019-12-17 12:37:43,134] jepsen worker 48 - jepsen.util 15438 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:37:43,134] jepsen worker 48 - jepsen.util 15438 :invoke :cas [16 [38 39]] INFO [2019-12-17 12:37:43,134] jepsen worker 61 - jepsen.util 18241 :invoke :write [20 38] INFO [2019-12-17 12:37:43,134] jepsen worker 44 - jepsen.util 17324 :invoke :read [14 nil] INFO [2019-12-17 12:37:43,147] jepsen worker 43 - jepsen.util 16873 :info :cas [14 [0 12]] :RequestCanceledException INFO [2019-12-17 12:37:43,148] jepsen worker 43 - jepsen.util 16963 :invoke :write [14 32] INFO [2019-12-17 12:37:43,165] jepsen worker 81 - jepsen.util 16371 :info :write [27 34] :RequestCanceledException INFO [2019-12-17 12:37:43,166] jepsen worker 81 - jepsen.util 16461 :invoke :read [27 nil] INFO [2019-12-17 12:37:43,171] jepsen worker 29 - jepsen.util 10739 :ok :read [9 41] INFO [2019-12-17 12:37:43,171] jepsen worker 29 - jepsen.util 10739 :invoke :read [9 nil] INFO [2019-12-17 12:37:43,172] jepsen worker 31 - jepsen.util 10471 :fail :cas [10 [21 42]] :ValueNotSwapFrom INFO [2019-12-17 12:37:43,173] jepsen worker 31 - jepsen.util 10471 :invoke :cas [10 [33 31]] INFO [2019-12-17 12:37:43,193] jepsen worker 18 - jepsen.util 12618 :ok :read [6 27] INFO [2019-12-17 12:37:43,193] jepsen worker 18 - jepsen.util 12618 :invoke :cas [6 [26 14]] INFO [2019-12-17 12:37:43,200] jepsen worker 80 - jepsen.util 11690 :info :write [26 20] :RequestCanceledException INFO [2019-12-17 12:37:43,200] jepsen worker 30 - jepsen.util 11280 :info :cas [10 [45 46]] :RequestCanceledException INFO [2019-12-17 12:37:43,201] jepsen worker 80 - jepsen.util 11780 :invoke :read [26 nil] INFO [2019-12-17 12:37:43,201] jepsen worker 30 - jepsen.util 11370 :invoke :cas [10 [10 21]] INFO [2019-12-17 12:37:43,219] jepsen worker 51 - jepsen.util 11391 :info :write [17 13] :RequestCanceledException INFO [2019-12-17 12:37:43,219] jepsen worker 51 - jepsen.util 11481 :invoke :read [17 nil] INFO [2019-12-17 12:37:43,232] jepsen worker 78 - jepsen.util 11958 :fail :cas [26 [20 44]] :ValueNotSwapFrom INFO [2019-12-17 12:37:43,232] jepsen worker 78 - jepsen.util 11958 :invoke :read [26 nil] INFO [2019-12-17 12:37:43,248] jepsen worker 7 - jepsen.util 16747 :info :cas [2 [13 43]] :RequestCanceledException INFO [2019-12-17 12:37:43,248] jepsen worker 7 - jepsen.util 16837 :invoke :read [2 nil] INFO [2019-12-17 12:37:43,263] jepsen worker 5 - jepsen.util 11885 :ok :read [1 24] INFO [2019-12-17 12:37:43,264] jepsen worker 5 - jepsen.util 11885 :invoke :cas [1 [10 39]] INFO [2019-12-17 12:37:43,264] jepsen worker 58 - jepsen.util 16888 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:37:43,264] jepsen worker 58 - jepsen.util 16888 :invoke :cas [19 [15 1]] INFO [2019-12-17 12:37:43,265] jepsen worker 29 - jepsen.util 10739 :ok :read [9 41] INFO [2019-12-17 12:37:43,265] jepsen worker 29 - jepsen.util 10739 :invoke :write [9 13] INFO [2019-12-17 12:37:43,301] jepsen worker 15 - jepsen.util 15585 :info :cas [5 [6 47]] :RequestCanceledException INFO [2019-12-17 12:37:43,302] jepsen worker 15 - jepsen.util 15675 :invoke :write [5 0] INFO [2019-12-17 12:37:43,304] jepsen worker 30 - jepsen.util 11370 :fail :cas [10 [10 21]] :ValueNotSwapFrom INFO [2019-12-17 12:37:43,304] jepsen worker 30 - jepsen.util 11370 :invoke :cas [10 [40 17]] INFO [2019-12-17 12:37:43,305] jepsen worker 64 - jepsen.util 9424 :ok :write [21 3] INFO [2019-12-17 12:37:43,305] jepsen worker 64 - jepsen.util 9424 :invoke :read [21 nil] INFO [2019-12-17 12:37:43,311] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":70818,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:37:43,311] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":70825,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:37:43,311] jepsen worker 31 - jepsen.util 10471 :fail :cas [10 [33 31]] :ValueNotSwapFrom INFO [2019-12-17 12:37:43,311] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":70818,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:37:43,311] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":70825,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:37:43,311] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":70819,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:37:43,311] jepsen worker 31 - jepsen.util 10471 :invoke :read [10 nil] INFO [2019-12-17 12:37:43,311] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":70828,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:37:43,311] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":70819,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:37:43,311] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":70828,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:37:43,311] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":70820,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:37:43,311] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":70820,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:37:43,318] jepsen worker 50 - jepsen.util 16700 :info :write [16 24] :RequestCanceledException INFO [2019-12-17 12:37:43,320] jepsen worker 50 - jepsen.util 16790 :invoke :read [16 nil] INFO [2019-12-17 12:37:43,343] jepsen worker 64 - jepsen.util 9424 :ok :read [21 3] INFO [2019-12-17 12:37:43,343] jepsen worker 64 - jepsen.util 9424 :invoke :cas [21 [14 34]] INFO [2019-12-17 12:37:43,344] jepsen worker 52 - jepsen.util 11662 :ok :read [17 37] INFO [2019-12-17 12:37:43,345] jepsen worker 52 - jepsen.util 11662 :invoke :cas [17 [6 16]] INFO [2019-12-17 12:37:43,346] jepsen worker 38 - jepsen.util 12638 :info :write [12 20] :RequestCanceledException INFO [2019-12-17 12:37:43,347] jepsen worker 38 - jepsen.util 12728 :invoke :cas [12 [23 18]] INFO [2019-12-17 12:37:43,350] jepsen worker 18 - jepsen.util 12618 :fail :cas [6 [26 14]] :ValueNotSwapFrom INFO [2019-12-17 12:37:43,350] jepsen worker 18 - jepsen.util 12618 :invoke :read [6 nil] INFO [2019-12-17 12:37:43,368] jepsen worker 70 - jepsen.util 12760 :ok :read [23 20] INFO [2019-12-17 12:37:43,368] jepsen worker 70 - jepsen.util 12760 :invoke :read [23 nil] INFO [2019-12-17 12:37:43,383] jepsen worker 31 - jepsen.util 10471 :ok :read [10 14] INFO [2019-12-17 12:37:43,384] jepsen worker 31 - jepsen.util 10471 :invoke :write [10 9] INFO [2019-12-17 12:37:43,388] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:37:43,388] jepsen nemesis - couchbase.util Rebalance check count: 558 INFO [2019-12-17 12:37:43,388] jepsen nemesis - couchbase.util Rebalance stuck count: 539 INFO [2019-12-17 12:37:43,418] jepsen worker 47 - jepsen.util 13187 :info :write [15 30] :RequestCanceledException INFO [2019-12-17 12:37:43,419] jepsen worker 18 - jepsen.util 12618 :ok :read [6 27] INFO [2019-12-17 12:37:43,419] jepsen worker 18 - jepsen.util 12618 :invoke :write [6 29] INFO [2019-12-17 12:37:43,419] jepsen worker 47 - jepsen.util 13277 :invoke :cas [15 [26 43]] INFO [2019-12-17 12:37:43,434] jepsen worker 29 - jepsen.util 10739 :ok :write [9 13] INFO [2019-12-17 12:37:43,434] jepsen worker 57 - jepsen.util 16437 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:37:43,434] jepsen worker 29 - jepsen.util 10739 :invoke :cas [9 [26 29]] INFO [2019-12-17 12:37:43,434] jepsen worker 57 - jepsen.util 16437 :invoke :write [19 41] INFO [2019-12-17 12:37:43,434] jepsen worker 3 - jepsen.util 11523 :info :write [1 5] :RequestCanceledException INFO [2019-12-17 12:37:43,435] jepsen worker 3 - jepsen.util 11613 :invoke :write [1 12] INFO [2019-12-17 12:37:43,438] jepsen worker 78 - jepsen.util 11958 :ok :read [26 1] INFO [2019-12-17 12:37:43,438] jepsen worker 78 - jepsen.util 11958 :invoke :cas [26 [38 31]] INFO [2019-12-17 12:37:43,439] jepsen worker 64 - jepsen.util 9424 :fail :cas [21 [14 34]] :ValueNotSwapFrom INFO [2019-12-17 12:37:43,440] jepsen worker 64 - jepsen.util 9424 :invoke :read [21 nil] INFO [2019-12-17 12:37:43,472] jepsen worker 64 - jepsen.util 9424 :ok :read [21 3] INFO [2019-12-17 12:37:43,473] jepsen worker 64 - jepsen.util 9424 :invoke :cas [21 [34 37]] INFO [2019-12-17 12:37:43,482] jepsen worker 47 - jepsen.util 13277 :fail :cas [15 [26 43]] :ValueNotSwapFrom INFO [2019-12-17 12:37:43,482] jepsen worker 47 - jepsen.util 13277 :invoke :read [15 nil] INFO [2019-12-17 12:37:43,491] jepsen worker 45 - jepsen.util 12105 :ok :write [15 49] INFO [2019-12-17 12:37:43,492] jepsen worker 45 - jepsen.util 12105 :invoke :write [15 48] INFO [2019-12-17 12:37:43,503] jepsen worker 29 - jepsen.util 10739 :fail :cas [9 [26 29]] :ValueNotSwapFrom INFO [2019-12-17 12:37:43,503] jepsen worker 29 - jepsen.util 10739 :invoke :cas [9 [44 39]] INFO [2019-12-17 12:37:43,513] jepsen worker 70 - jepsen.util 12760 :ok :read [23 20] INFO [2019-12-17 12:37:43,513] jepsen worker 70 - jepsen.util 12760 :invoke :cas [23 [26 42]] INFO [2019-12-17 12:37:43,517] jepsen worker 69 - jepsen.util 12399 :info :write [23 37] :RequestCanceledException INFO [2019-12-17 12:37:43,518] jepsen worker 72 - jepsen.util 12402 :ok :write [24 46] INFO [2019-12-17 12:37:43,518] jepsen worker 72 - jepsen.util 12402 :invoke :read [24 nil] INFO [2019-12-17 12:37:43,521] jepsen worker 69 - jepsen.util 12489 :invoke :write [23 19] INFO [2019-12-17 12:37:43,522] jepsen worker 64 - jepsen.util 9424 :fail :cas [21 [34 37]] :ValueNotSwapFrom INFO [2019-12-17 12:37:43,522] jepsen worker 64 - jepsen.util 9424 :invoke :cas [21 [28 10]] INFO [2019-12-17 12:37:43,524] jepsen worker 47 - jepsen.util 13277 :ok :read [15 49] INFO [2019-12-17 12:37:43,524] jepsen worker 47 - jepsen.util 13277 :invoke :read [15 nil] INFO [2019-12-17 12:37:43,530] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":70852,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:37:43,530] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":70852,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:37:43,548] jepsen worker 42 - jepsen.util 17592 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:37:43,548] jepsen worker 42 - jepsen.util 17592 :invoke :cas [14 [23 8]] INFO [2019-12-17 12:37:43,565] jepsen worker 22 - jepsen.util 16852 :info :cas [7 [0 43]] :RequestCanceledException INFO [2019-12-17 12:37:43,568] jepsen worker 22 - jepsen.util 16942 :invoke :read [7 nil] INFO [2019-12-17 12:37:43,583] jepsen worker 70 - jepsen.util 12760 :fail :cas [23 [26 42]] :ValueNotSwapFrom INFO [2019-12-17 12:37:43,583] jepsen worker 70 - jepsen.util 12760 :invoke :cas [23 [39 46]] INFO [2019-12-17 12:37:43,601] jepsen worker 83 - jepsen.util 16823 :info :cas [27 [20 2]] :RequestCanceledException INFO [2019-12-17 12:37:43,601] jepsen worker 39 - jepsen.util 16329 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:37:43,601] jepsen worker 39 - jepsen.util 16329 :invoke :write [13 25] INFO [2019-12-17 12:37:43,602] jepsen worker 83 - jepsen.util 16913 :invoke :read [27 nil] INFO [2019-12-17 12:37:43,616] jepsen worker 78 - jepsen.util 11958 :fail :cas [26 [38 31]] :ValueNotSwapFrom INFO [2019-12-17 12:37:43,617] jepsen worker 78 - jepsen.util 11958 :invoke :read [26 nil] INFO [2019-12-17 12:37:43,618] jepsen worker 29 - jepsen.util 10739 :fail :cas [9 [44 39]] :ValueNotSwapFrom INFO [2019-12-17 12:37:43,618] jepsen worker 29 - jepsen.util 10739 :invoke :write [9 35] INFO [2019-12-17 12:37:43,623] jepsen worker 9 - jepsen.util 12699 :fail :cas [3 [26 15]] :ValueNotSwapFrom INFO [2019-12-17 12:37:43,623] jepsen worker 9 - jepsen.util 12699 :invoke :write [3 2] INFO [2019-12-17 12:37:43,623] jepsen worker 64 - jepsen.util 9424 :fail :cas [21 [28 10]] :ValueNotSwapFrom INFO [2019-12-17 12:37:43,623] jepsen worker 64 - jepsen.util 9424 :invoke :read [21 nil] INFO [2019-12-17 12:37:43,634] jepsen worker 1 - jepsen.util 16921 :info :cas [0 [11 31]] :RequestCanceledException INFO [2019-12-17 12:37:43,635] jepsen worker 1 - jepsen.util 17011 :invoke :cas [0 [34 49]] INFO [2019-12-17 12:37:43,653] jepsen worker 64 - jepsen.util 9424 :ok :read [21 3] INFO [2019-12-17 12:37:43,653] jepsen worker 64 - jepsen.util 9424 :invoke :cas [21 [15 9]] INFO [2019-12-17 12:37:43,662] jepsen worker 47 - jepsen.util 13277 :ok :read [15 49] INFO [2019-12-17 12:37:43,662] jepsen worker 47 - jepsen.util 13277 :invoke :read [15 nil] INFO [2019-12-17 12:37:43,685] jepsen worker 64 - jepsen.util 9424 :fail :cas [21 [15 9]] :ValueNotSwapFrom INFO [2019-12-17 12:37:43,685] jepsen worker 64 - jepsen.util 9424 :invoke :read [21 nil] INFO [2019-12-17 12:37:43,694] jepsen worker 53 - jepsen.util 11843 :ok :write [17 48] INFO [2019-12-17 12:37:43,694] jepsen worker 53 - jepsen.util 11843 :invoke :write [17 1] INFO [2019-12-17 12:37:43,713] jepsen worker 64 - jepsen.util 9424 :ok :read [21 3] INFO [2019-12-17 12:37:43,713] jepsen worker 64 - jepsen.util 9424 :invoke :read [21 nil] INFO [2019-12-17 12:37:43,718] jepsen worker 76 - jepsen.util 16906 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:37:43,718] jepsen worker 76 - jepsen.util 16906 :invoke :write [25 5] INFO [2019-12-17 12:37:43,728] jepsen worker 29 - jepsen.util 10739 :ok :write [9 35] INFO [2019-12-17 12:37:43,728] jepsen worker 29 - jepsen.util 10739 :invoke :write [9 23] INFO [2019-12-17 12:37:43,729] jepsen worker 45 - jepsen.util 12105 :ok :write [15 48] INFO [2019-12-17 12:37:43,729] jepsen worker 45 - jepsen.util 12105 :invoke :write [15 6] INFO [2019-12-17 12:37:43,733] jepsen worker 49 - jepsen.util 16789 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:37:43,733] jepsen worker 49 - jepsen.util 16789 :invoke :cas [16 [0 9]] INFO [2019-12-17 12:37:43,735] jepsen worker 64 - jepsen.util 9424 :ok :read [21 3] INFO [2019-12-17 12:37:43,735] jepsen worker 64 - jepsen.util 9424 :invoke :write [21 44] INFO [2019-12-17 12:37:43,749] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":70883,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:37:43,749] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":70883,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:37:43,749] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":70884,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:37:43,749] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":70884,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:37:43,763] jepsen worker 71 - jepsen.util 13121 :fail :cas [23 [10 49]] :ValueNotSwapFrom INFO [2019-12-17 12:37:43,763] jepsen worker 71 - jepsen.util 13121 :invoke :read [23 nil] INFO [2019-12-17 12:37:43,778] jepsen worker 31 - jepsen.util 10471 :ok :write [10 9] INFO [2019-12-17 12:37:43,778] jepsen worker 31 - jepsen.util 10471 :invoke :read [10 nil] INFO [2019-12-17 12:37:43,778] jepsen worker 52 - jepsen.util 11662 :fail :cas [17 [6 16]] :ValueNotSwapFrom INFO [2019-12-17 12:37:43,778] jepsen worker 52 - jepsen.util 11662 :invoke :cas [17 [3 18]] INFO [2019-12-17 12:37:43,794] jepsen worker 51 - jepsen.util 11481 :ok :read [17 48] INFO [2019-12-17 12:37:43,794] jepsen worker 51 - jepsen.util 11481 :invoke :write [17 40] INFO [2019-12-17 12:37:43,800] jepsen worker 87 - jepsen.util 11787 :info :write [29 38] :RequestCanceledException INFO [2019-12-17 12:37:43,800] jepsen worker 68 - jepsen.util 15998 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:37:43,800] jepsen worker 68 - jepsen.util 15998 :invoke :cas [22 [33 27]] INFO [2019-12-17 12:37:43,801] jepsen worker 87 - jepsen.util 11877 :invoke :cas [29 [5 10]] INFO [2019-12-17 12:37:43,823] jepsen worker 31 - jepsen.util 10471 :ok :read [10 9] INFO [2019-12-17 12:37:43,823] jepsen worker 31 - jepsen.util 10471 :invoke :write [10 31] INFO [2019-12-17 12:37:43,834] jepsen worker 6 - jepsen.util 17016 :info :write [2 18] :RequestCanceledException INFO [2019-12-17 12:37:43,835] jepsen worker 6 - jepsen.util 17106 :invoke :cas [2 [6 9]] INFO [2019-12-17 12:37:43,843] jepsen worker 13 - jepsen.util 11713 :ok :read [4 21] INFO [2019-12-17 12:37:43,843] jepsen worker 13 - jepsen.util 11713 :invoke :read [4 nil] INFO [2019-12-17 12:37:43,856] jepsen worker 70 - jepsen.util 12760 :fail :cas [23 [39 46]] :ValueNotSwapFrom INFO [2019-12-17 12:37:43,857] jepsen worker 70 - jepsen.util 12760 :invoke :write [23 37] INFO [2019-12-17 12:37:43,857] jepsen worker 29 - jepsen.util 10739 :ok :write [9 23] INFO [2019-12-17 12:37:43,857] jepsen worker 29 - jepsen.util 10739 :invoke :cas [9 [34 19]] INFO [2019-12-17 12:37:43,884] jepsen worker 29 - jepsen.util 10739 :fail :cas [9 [34 19]] :ValueNotSwapFrom INFO [2019-12-17 12:37:43,884] jepsen worker 29 - jepsen.util 10739 :invoke :cas [9 [27 34]] INFO [2019-12-17 12:37:43,887] jepsen worker 80 - jepsen.util 11780 :ok :read [26 1] INFO [2019-12-17 12:37:43,887] jepsen worker 80 - jepsen.util 11780 :invoke :cas [26 [5 21]] INFO [2019-12-17 12:37:43,906] jepsen worker 29 - jepsen.util 10739 :fail :cas [9 [27 34]] :ValueNotSwapFrom INFO [2019-12-17 12:37:43,906] jepsen worker 29 - jepsen.util 10739 :invoke :write [9 1] INFO [2019-12-17 12:37:43,916] jepsen worker 38 - jepsen.util 12728 :fail :cas [12 [23 18]] :ValueNotSwapFrom INFO [2019-12-17 12:37:43,916] jepsen worker 38 - jepsen.util 12728 :invoke :cas [12 [46 36]] INFO [2019-12-17 12:37:43,917] jepsen worker 64 - jepsen.util 9424 :ok :write [21 44] INFO [2019-12-17 12:37:43,917] jepsen worker 64 - jepsen.util 9424 :invoke :write [21 21] INFO [2019-12-17 12:37:43,944] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":70907,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:37:43,944] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":70907,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:37:43,944] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":70910,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:37:43,944] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":70910,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:37:43,944] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":70911,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:37:43,944] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":70911,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:37:43,952] jepsen worker 78 - jepsen.util 11958 :ok :read [26 1] INFO [2019-12-17 12:37:43,953] jepsen worker 78 - jepsen.util 11958 :invoke :read [26 nil] INFO [2019-12-17 12:37:43,965] jepsen worker 24 - jepsen.util 15864 :info :cas [8 [14 40]] :RequestCanceledException INFO [2019-12-17 12:37:43,965] jepsen worker 41 - jepsen.util 18041 :info :write [13 0] :RequestCanceledException INFO [2019-12-17 12:37:43,966] jepsen worker 41 - jepsen.util 18131 :invoke :cas [13 [1 3]] INFO [2019-12-17 12:37:43,966] jepsen worker 24 - jepsen.util 15954 :invoke :write [8 0] INFO [2019-12-17 12:37:43,973] jepsen worker 45 - jepsen.util 12105 :ok :write [15 6] INFO [2019-12-17 12:37:43,974] jepsen worker 45 - jepsen.util 12105 :invoke :read [15 nil] INFO [2019-12-17 12:37:44,002] jepsen worker 71 - jepsen.util 13121 :ok :read [23 20] INFO [2019-12-17 12:37:44,002] jepsen worker 71 - jepsen.util 13121 :invoke :cas [23 [30 6]] INFO [2019-12-17 12:37:44,008] jepsen worker 31 - jepsen.util 10471 :ok :write [10 31] INFO [2019-12-17 12:37:44,009] jepsen worker 31 - jepsen.util 10471 :invoke :cas [10 [32 1]] INFO [2019-12-17 12:37:44,018] jepsen worker 40 - jepsen.util 17500 :info :write [13 49] :RequestCanceledException INFO [2019-12-17 12:37:44,019] jepsen worker 40 - jepsen.util 17590 :invoke :cas [13 [12 16]] INFO [2019-12-17 12:37:44,022] jepsen worker 45 - jepsen.util 12105 :ok :read [15 6] INFO [2019-12-17 12:37:44,022] jepsen worker 45 - jepsen.util 12105 :invoke :write [15 41] INFO [2019-12-17 12:37:44,034] jepsen worker 34 - jepsen.util 17404 :info :cas [11 [17 3]] :RequestCanceledException INFO [2019-12-17 12:37:44,036] jepsen worker 34 - jepsen.util 17494 :invoke :cas [11 [42 38]] INFO [2019-12-17 12:37:44,047] jepsen worker 29 - jepsen.util 10739 :ok :write [9 1] INFO [2019-12-17 12:37:44,047] jepsen worker 29 - jepsen.util 10739 :invoke :read [9 nil] INFO [2019-12-17 12:37:44,048] jepsen worker 55 - jepsen.util 12925 :info :write [18 3] :RequestCanceledException INFO [2019-12-17 12:37:44,054] jepsen worker 31 - jepsen.util 10471 :fail :cas [10 [32 1]] :ValueNotSwapFrom INFO [2019-12-17 12:37:44,054] jepsen worker 31 - jepsen.util 10471 :invoke :read [10 nil] INFO [2019-12-17 12:37:44,056] jepsen worker 55 - jepsen.util 13015 :invoke :write [18 37] INFO [2019-12-17 12:37:44,072] jepsen worker 29 - jepsen.util 10739 :ok :read [9 1] INFO [2019-12-17 12:37:44,072] jepsen worker 29 - jepsen.util 10739 :invoke :write [9 38] INFO [2019-12-17 12:37:44,078] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":70924,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:37:44,078] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":70924,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:37:44,088] jepsen worker 52 - jepsen.util 11662 :fail :cas [17 [3 18]] :ValueNotSwapFrom INFO [2019-12-17 12:37:44,091] jepsen worker 52 - jepsen.util 11662 :invoke :read [17 nil] INFO [2019-12-17 12:37:44,094] jepsen worker 5 - jepsen.util 11885 :fail :cas [1 [10 39]] :ValueNotSwapFrom INFO [2019-12-17 12:37:44,094] jepsen worker 5 - jepsen.util 11885 :invoke :read [1 nil] INFO [2019-12-17 12:37:44,101] jepsen worker 63 - jepsen.util 10683 :info :cas [21 [33 17]] :RequestCanceledException INFO [2019-12-17 12:37:44,101] jepsen worker 10 - jepsen.util 11980 :info :cas [3 [49 30]] :RequestCanceledException INFO [2019-12-17 12:37:44,103] jepsen worker 13 - jepsen.util 11713 :ok :read [4 21] INFO [2019-12-17 12:37:44,103] jepsen worker 13 - jepsen.util 11713 :invoke :read [4 nil] INFO [2019-12-17 12:37:44,104] jepsen worker 63 - jepsen.util 10773 :invoke :write [21 28] INFO [2019-12-17 12:37:44,105] jepsen worker 10 - jepsen.util 12070 :invoke :read [3 nil] INFO [2019-12-17 12:37:44,114] jepsen worker 31 - jepsen.util 10471 :ok :read [10 31] INFO [2019-12-17 12:37:44,114] jepsen worker 31 - jepsen.util 10471 :invoke :write [10 29] INFO [2019-12-17 12:37:44,115] jepsen worker 64 - jepsen.util 9424 :ok :write [21 21] INFO [2019-12-17 12:37:44,116] jepsen worker 64 - jepsen.util 9424 :invoke :write [21 2] INFO [2019-12-17 12:37:44,148] jepsen worker 38 - jepsen.util 12728 :fail :cas [12 [46 36]] :ValueNotSwapFrom INFO [2019-12-17 12:37:44,149] jepsen worker 38 - jepsen.util 12728 :invoke :read [12 nil] INFO [2019-12-17 12:37:44,159] jepsen worker 54 - jepsen.util 12834 :fail :cas [18 [36 44]] :ValueNotSwapFrom INFO [2019-12-17 12:37:44,160] jepsen worker 54 - jepsen.util 12834 :invoke :read [18 nil] INFO [2019-12-17 12:37:44,163] jepsen worker 10 - jepsen.util 12070 :ok :read [3 16] INFO [2019-12-17 12:37:44,164] jepsen worker 10 - jepsen.util 12070 :invoke :cas [3 [47 4]] INFO [2019-12-17 12:37:44,165] jepsen worker 14 - jepsen.util 11894 :info :write [4 12] :RequestCanceledException INFO [2019-12-17 12:37:44,167] jepsen worker 14 - jepsen.util 11984 :invoke :cas [4 [2 2]] INFO [2019-12-17 12:37:44,183] jepsen worker 80 - jepsen.util 11780 :fail :cas [26 [5 21]] :ValueNotSwapFrom INFO [2019-12-17 12:37:44,196] jepsen worker 29 - jepsen.util 10739 :ok :write [9 38] INFO [2019-12-17 12:37:44,201] jepsen worker 60 - jepsen.util 17790 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:37:44,218] jepsen worker 62 - jepsen.util 17522 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:37:44,220] jepsen worker 63 - jepsen.util 10773 :ok :write [21 28] INFO [2019-12-17 12:37:44,233] jepsen worker 80 - jepsen.util 11780 :invoke :cas [26 [10 25]] INFO [2019-12-17 12:37:44,233] jepsen worker 17 - jepsen.util 16937 :info :write [5 28] :RequestCanceledException INFO [2019-12-17 12:37:44,234] jepsen worker 60 - jepsen.util 17790 :invoke :write [20 46] INFO [2019-12-17 12:37:44,234] jepsen worker 62 - jepsen.util 17522 :invoke :read [20 nil] INFO [2019-12-17 12:37:44,234] jepsen worker 29 - jepsen.util 10739 :invoke :cas [9 [17 17]] INFO [2019-12-17 12:37:44,234] jepsen worker 63 - jepsen.util 10773 :invoke :cas [21 [37 40]] INFO [2019-12-17 12:37:44,235] jepsen worker 17 - jepsen.util 17027 :invoke :read [5 nil] INFO [2019-12-17 12:37:44,254] jepsen worker 63 - jepsen.util 10773 :fail :cas [21 [37 40]] :ValueNotSwapFrom INFO [2019-12-17 12:37:44,254] jepsen worker 63 - jepsen.util 10773 :invoke :read [21 nil] INFO [2019-12-17 12:37:44,256] jepsen worker 5 - jepsen.util 11885 :ok :read [1 24] INFO [2019-12-17 12:37:44,256] jepsen worker 5 - jepsen.util 11885 :invoke :read [1 nil] INFO [2019-12-17 12:37:44,285] jepsen worker 63 - jepsen.util 10773 :ok :read [21 28] INFO [2019-12-17 12:37:44,285] jepsen worker 63 - jepsen.util 10773 :invoke :write [21 6] INFO [2019-12-17 12:37:44,290] jepsen worker 54 - jepsen.util 12834 :ok :read [18 39] INFO [2019-12-17 12:37:44,290] jepsen worker 54 - jepsen.util 12834 :invoke :write [18 47] INFO [2019-12-17 12:37:44,291] jepsen worker 29 - jepsen.util 10739 :fail :cas [9 [17 17]] :ValueNotSwapFrom INFO [2019-12-17 12:37:44,291] jepsen worker 29 - jepsen.util 10739 :invoke :write [9 25] INFO [2019-12-17 12:37:44,297] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":70954,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:37:44,298] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":70954,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:37:44,298] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":70957,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:37:44,298] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":70957,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:37:44,298] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":70958,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:37:44,298] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":70958,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:37:44,298] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":70947,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:37:44,298] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":70947,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:37:44,298] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":70949,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:37:44,298] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":70949,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:37:44,302] jepsen worker 52 - jepsen.util 11662 :ok :read [17 48] INFO [2019-12-17 12:37:44,302] jepsen worker 52 - jepsen.util 11662 :invoke :write [17 27] INFO [2019-12-17 12:37:44,318] jepsen worker 75 - jepsen.util 16365 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:37:44,318] jepsen worker 75 - jepsen.util 16365 :invoke :write [25 18] INFO [2019-12-17 12:37:44,321] jepsen worker 64 - jepsen.util 9424 :ok :write [21 2] INFO [2019-12-17 12:37:44,321] jepsen worker 64 - jepsen.util 9424 :invoke :read [21 nil] INFO [2019-12-17 12:37:44,331] jepsen worker 71 - jepsen.util 13121 :fail :cas [23 [30 6]] :ValueNotSwapFrom INFO [2019-12-17 12:37:44,331] jepsen worker 71 - jepsen.util 13121 :invoke :cas [23 [26 48]] INFO [2019-12-17 12:37:44,334] jepsen worker 78 - jepsen.util 11958 :ok :read [26 1] INFO [2019-12-17 12:37:44,334] jepsen worker 78 - jepsen.util 11958 :invoke :read [26 nil] INFO [2019-12-17 12:37:44,334] jepsen worker 86 - jepsen.util 17456 :info :cas [28 [44 37]] :RequestCanceledException INFO [2019-12-17 12:37:44,334] jepsen worker 88 - jepsen.util 12328 :info :write [29 19] :RequestCanceledException INFO [2019-12-17 12:37:44,335] jepsen worker 88 - jepsen.util 12418 :invoke :write [29 10] INFO [2019-12-17 12:37:44,336] jepsen worker 86 - jepsen.util 17546 :invoke :read [28 nil] INFO [2019-12-17 12:37:44,354] jepsen worker 64 - jepsen.util 9424 :ok :read [21 2] INFO [2019-12-17 12:37:44,354] jepsen worker 64 - jepsen.util 9424 :invoke :write [21 1] INFO [2019-12-17 12:37:44,364] jepsen worker 26 - jepsen.util 17396 :info :write [8 36] :RequestCanceledException INFO [2019-12-17 12:37:44,364] jepsen worker 11 - jepsen.util 11801 :fail :read [3 nil] :RequestTimeoutException INFO [2019-12-17 12:37:44,365] jepsen worker 11 - jepsen.util 11801 :invoke :read [3 nil] INFO [2019-12-17 12:37:44,366] jepsen worker 26 - jepsen.util 17486 :invoke :write [8 33] INFO [2019-12-17 12:37:44,392] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:37:44,392] jepsen nemesis - couchbase.util Rebalance check count: 559 INFO [2019-12-17 12:37:44,392] jepsen nemesis - couchbase.util Rebalance stuck count: 540 INFO [2019-12-17 12:37:44,394] jepsen worker 63 - jepsen.util 10773 :ok :write [21 6] INFO [2019-12-17 12:37:44,394] jepsen worker 63 - jepsen.util 10773 :invoke :read [21 nil] INFO [2019-12-17 12:37:44,408] jepsen worker 13 - jepsen.util 11713 :ok :read [4 21] INFO [2019-12-17 12:37:44,409] jepsen worker 13 - jepsen.util 11713 :invoke :read [4 nil] INFO [2019-12-17 12:37:44,409] jepsen worker 29 - jepsen.util 10739 :ok :write [9 25] INFO [2019-12-17 12:37:44,409] jepsen worker 29 - jepsen.util 10739 :invoke :cas [9 [9 12]] INFO [2019-12-17 12:37:44,409] jepsen worker 31 - jepsen.util 10471 :ok :write [10 29] INFO [2019-12-17 12:37:44,409] jepsen worker 31 - jepsen.util 10471 :invoke :cas [10 [5 4]] INFO [2019-12-17 12:37:44,418] jepsen worker 45 - jepsen.util 12105 :ok :write [15 41] INFO [2019-12-17 12:37:44,418] jepsen worker 45 - jepsen.util 12105 :invoke :cas [15 [9 29]] INFO [2019-12-17 12:37:44,419] jepsen worker 27 - jepsen.util 11277 :fail :read [9 nil] :RequestTimeoutException INFO [2019-12-17 12:37:44,419] jepsen worker 27 - jepsen.util 11277 :invoke :write [9 36] INFO [2019-12-17 12:37:44,434] jepsen worker 33 - jepsen.util 16593 :info :cas [11 [22 36]] :RequestCanceledException INFO [2019-12-17 12:37:44,434] jepsen worker 65 - jepsen.util 10685 :info :write [21 42] :RequestCanceledException INFO [2019-12-17 12:37:44,434] jepsen worker 85 - jepsen.util 17365 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:37:44,434] jepsen worker 85 - jepsen.util 17365 :invoke :write [28 11] INFO [2019-12-17 12:37:44,435] jepsen worker 65 - jepsen.util 10775 :invoke :read [21 nil] INFO [2019-12-17 12:37:44,435] jepsen worker 33 - jepsen.util 16683 :invoke :cas [11 [42 19]] INFO [2019-12-17 12:37:44,440] jepsen worker 63 - jepsen.util 10773 :ok :read [21 6] INFO [2019-12-17 12:37:44,440] jepsen worker 63 - jepsen.util 10773 :invoke :write [21 16] INFO [2019-12-17 12:37:44,442] jepsen worker 73 - jepsen.util 11233 :info :write [24 44] :RequestCanceledException INFO [2019-12-17 12:37:44,442] jepsen worker 66 - jepsen.util 16716 :info :cas [22 [32 11]] :RequestCanceledException INFO [2019-12-17 12:37:44,442] jepsen worker 19 - jepsen.util 12619 :fail :read [6 nil] :RequestTimeoutException INFO [2019-12-17 12:37:44,442] jepsen worker 19 - jepsen.util 12619 :invoke :read [6 nil] INFO [2019-12-17 12:37:44,443] jepsen worker 73 - jepsen.util 11323 :invoke :read [24 nil] INFO [2019-12-17 12:37:44,444] jepsen worker 66 - jepsen.util 16806 :invoke :write [22 14] INFO [2019-12-17 12:37:44,465] jepsen worker 4 - jepsen.util 11344 :info :write [1 5] :RequestCanceledException INFO [2019-12-17 12:37:44,465] jepsen worker 67 - jepsen.util 16897 :info :cas [22 [10 6]] :RequestCanceledException INFO [2019-12-17 12:37:44,467] jepsen worker 67 - jepsen.util 16987 :invoke :cas [22 [38 19]] INFO [2019-12-17 12:37:44,467] jepsen worker 4 - jepsen.util 11434 :invoke :cas [1 [48 39]] INFO [2019-12-17 12:37:44,468] jepsen worker 38 - jepsen.util 12728 :ok :read [12 11] INFO [2019-12-17 12:37:44,468] jepsen worker 38 - jepsen.util 12728 :invoke :cas [12 [47 8]] INFO [2019-12-17 12:37:44,472] jepsen worker 78 - jepsen.util 11958 :ok :read [26 1] INFO [2019-12-17 12:37:44,472] jepsen worker 78 - jepsen.util 11958 :invoke :cas [26 [2 19]] INFO [2019-12-17 12:37:44,500] jepsen worker 32 - jepsen.util 12722 :fail :read [10 nil] :RequestTimeoutException INFO [2019-12-17 12:37:44,500] jepsen worker 32 - jepsen.util 12722 :invoke :cas [10 [10 35]] INFO [2019-12-17 12:37:44,517] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":70987,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:37:44,517] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":70987,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:37:44,518] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":70993,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:37:44,518] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":70993,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:37:44,533] jepsen worker 11 - jepsen.util 11801 :ok :read [3 16] INFO [2019-12-17 12:37:44,533] jepsen worker 11 - jepsen.util 11801 :invoke :read [3 nil] INFO [2019-12-17 12:37:44,533] jepsen worker 28 - jepsen.util 10378 :info :cas [9 [45 1]] :RequestCanceledException INFO [2019-12-17 12:37:44,535] jepsen worker 28 - jepsen.util 10468 :invoke :read [9 nil] INFO [2019-12-17 12:37:44,537] jepsen worker 80 - jepsen.util 11780 :fail :cas [26 [10 25]] :ValueNotSwapFrom INFO [2019-12-17 12:37:44,537] jepsen worker 80 - jepsen.util 11780 :invoke :read [26 nil] INFO [2019-12-17 12:37:44,547] jepsen worker 82 - jepsen.util 17092 :info :cas [27 [19 17]] :RequestCanceledException INFO [2019-12-17 12:37:44,551] jepsen worker 82 - jepsen.util 17182 :invoke :write [27 42] INFO [2019-12-17 12:37:44,565] jepsen worker 84 - jepsen.util 16374 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:37:44,565] jepsen worker 89 - jepsen.util 11969 :info :write [29 17] :RequestCanceledException INFO [2019-12-17 12:37:44,565] jepsen worker 84 - jepsen.util 16374 :invoke :cas [28 [25 5]] INFO [2019-12-17 12:37:44,566] jepsen worker 89 - jepsen.util 12059 :invoke :cas [29 [10 28]] INFO [2019-12-17 12:37:44,569] jepsen worker 10 - jepsen.util 12070 :fail :cas [3 [47 4]] :ValueNotSwapFrom INFO [2019-12-17 12:37:44,569] jepsen worker 10 - jepsen.util 12070 :invoke :read [3 nil] INFO [2019-12-17 12:37:44,573] jepsen worker 52 - jepsen.util 11662 :ok :write [17 27] INFO [2019-12-17 12:37:44,573] jepsen worker 52 - jepsen.util 11662 :invoke :write [17 34] INFO [2019-12-17 12:37:44,598] jepsen worker 73 - jepsen.util 11323 :ok :read [24 46] INFO [2019-12-17 12:37:44,598] jepsen worker 73 - jepsen.util 11323 :invoke :write [24 23] INFO [2019-12-17 12:37:44,598] jepsen worker 45 - jepsen.util 12105 :fail :cas [15 [9 29]] :ValueNotSwapFrom INFO [2019-12-17 12:37:44,598] jepsen worker 45 - jepsen.util 12105 :invoke :write [15 8] INFO [2019-12-17 12:37:44,600] jepsen worker 78 - jepsen.util 11958 :fail :cas [26 [2 19]] :ValueNotSwapFrom INFO [2019-12-17 12:37:44,600] jepsen worker 78 - jepsen.util 11958 :invoke :cas [26 [16 21]] INFO [2019-12-17 12:37:44,629] jepsen worker 71 - jepsen.util 13121 :fail :cas [23 [26 48]] :ValueNotSwapFrom INFO [2019-12-17 12:37:44,629] jepsen worker 71 - jepsen.util 13121 :invoke :read [23 nil] INFO [2019-12-17 12:37:44,634] jepsen worker 20 - jepsen.util 11990 :info :write [6 8] :RequestCanceledException INFO [2019-12-17 12:37:44,634] jepsen worker 12 - jepsen.util 12432 :info :write [4 16] :RequestCanceledException INFO [2019-12-17 12:37:44,635] jepsen worker 12 - jepsen.util 12522 :invoke :write [4 13] INFO [2019-12-17 12:37:44,635] jepsen worker 20 - jepsen.util 12080 :invoke :write [6 4] INFO [2019-12-17 12:37:44,643] jepsen worker 87 - jepsen.util 11877 :fail :cas [29 [5 10]] :ValueNotSwapFrom INFO [2019-12-17 12:37:44,643] jepsen worker 87 - jepsen.util 11877 :invoke :cas [29 [11 3]] INFO [2019-12-17 12:37:44,647] jepsen worker 23 - jepsen.util 17123 :info :write [7 35] :RequestCanceledException INFO [2019-12-17 12:37:44,648] jepsen worker 23 - jepsen.util 17213 :invoke :cas [7 [35 41]] INFO [2019-12-17 12:37:44,656] jepsen worker 29 - jepsen.util 10739 :fail :cas [9 [9 12]] :ValueNotSwapFrom INFO [2019-12-17 12:37:44,656] jepsen worker 29 - jepsen.util 10739 :invoke :read [9 nil] INFO [2019-12-17 12:37:44,659] jepsen worker 31 - jepsen.util 10471 :fail :cas [10 [5 4]] :ValueNotSwapFrom INFO [2019-12-17 12:37:44,660] jepsen worker 31 - jepsen.util 10471 :invoke :read [10 nil] INFO [2019-12-17 12:37:44,670] jepsen worker 14 - jepsen.util 11984 :fail :cas [4 [2 2]] :ValueNotSwapFrom INFO [2019-12-17 12:37:44,670] jepsen worker 14 - jepsen.util 11984 :invoke :write [4 37] INFO [2019-12-17 12:37:44,701] jepsen worker 0 - jepsen.util 17370 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:37:44,701] jepsen worker 0 - jepsen.util 17370 :invoke :write [0 24] INFO [2019-12-17 12:37:44,712] jepsen worker 11 - jepsen.util 11801 :ok :read [3 16] INFO [2019-12-17 12:37:44,712] jepsen worker 11 - jepsen.util 11801 :invoke :cas [3 [43 16]] INFO [2019-12-17 12:37:44,715] jepsen worker 19 - jepsen.util 12619 :ok :read [6 27] INFO [2019-12-17 12:37:44,715] jepsen worker 19 - jepsen.util 12619 :invoke :cas [6 [44 20]] INFO [2019-12-17 12:37:44,718] jepsen worker 21 - jepsen.util 16221 :info :write [7 12] :RequestCanceledException INFO [2019-12-17 12:37:44,719] jepsen worker 21 - jepsen.util 16311 :invoke :cas [7 [23 0]] INFO [2019-12-17 12:37:44,734] jepsen worker 59 - jepsen.util 15899 :info :cas [19 [45 37]] :RequestCanceledException INFO [2019-12-17 12:37:44,735] jepsen worker 59 - jepsen.util 15989 :invoke :write [19 45] INFO [2019-12-17 12:37:44,736] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71021,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:37:44,737] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71021,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:37:44,737] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71024,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:37:44,737] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71024,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:37:44,737] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71025,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:37:44,737] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71025,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:37:44,749] jepsen worker 65 - jepsen.util 10775 :ok :read [21 6] INFO [2019-12-17 12:37:44,750] jepsen worker 65 - jepsen.util 10775 :invoke :read [21 nil] INFO [2019-12-17 12:37:44,756] jepsen worker 13 - jepsen.util 11713 :ok :read [4 21] INFO [2019-12-17 12:37:44,756] jepsen worker 13 - jepsen.util 11713 :invoke :write [4 3] INFO [2019-12-17 12:37:44,759] jepsen worker 63 - jepsen.util 10773 :ok :write [21 16] INFO [2019-12-17 12:37:44,760] jepsen worker 63 - jepsen.util 10773 :invoke :read [21 nil] INFO [2019-12-17 12:37:44,786] jepsen worker 32 - jepsen.util 12722 :fail :cas [10 [10 35]] :ValueNotSwapFrom INFO [2019-12-17 12:37:44,786] jepsen worker 32 - jepsen.util 12722 :invoke :read [10 nil] INFO [2019-12-17 12:37:44,793] jepsen worker 28 - jepsen.util 10468 :ok :read [9 25] INFO [2019-12-17 12:37:44,794] jepsen worker 28 - jepsen.util 10468 :invoke :write [9 18] INFO [2019-12-17 12:37:44,803] jepsen worker 63 - jepsen.util 10773 :ok :read [21 16] INFO [2019-12-17 12:37:44,803] jepsen worker 63 - jepsen.util 10773 :invoke :cas [21 [18 19]] INFO [2019-12-17 12:37:44,805] jepsen worker 10 - jepsen.util 12070 :ok :read [3 16] INFO [2019-12-17 12:37:44,805] jepsen worker 10 - jepsen.util 12070 :invoke :write [3 48] INFO [2019-12-17 12:37:44,824] jepsen worker 80 - jepsen.util 11780 :ok :read [26 1] INFO [2019-12-17 12:37:44,824] jepsen worker 80 - jepsen.util 11780 :invoke :read [26 nil] INFO [2019-12-17 12:37:44,827] jepsen worker 89 - jepsen.util 12059 :fail :cas [29 [10 28]] :ValueNotSwapFrom INFO [2019-12-17 12:37:44,827] jepsen worker 89 - jepsen.util 12059 :invoke :cas [29 [8 5]] INFO [2019-12-17 12:37:44,832] jepsen worker 69 - jepsen.util 12489 :ok :write [23 19] INFO [2019-12-17 12:37:44,832] jepsen worker 78 - jepsen.util 11958 :fail :cas [26 [16 21]] :ValueNotSwapFrom INFO [2019-12-17 12:37:44,832] jepsen worker 69 - jepsen.util 12489 :invoke :write [23 8] INFO [2019-12-17 12:37:44,832] jepsen worker 78 - jepsen.util 11958 :invoke :cas [26 [18 39]] INFO [2019-12-17 12:37:44,843] jepsen worker 2 - jepsen.util 16112 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:37:44,843] jepsen worker 56 - jepsen.util 13196 :info :write [18 23] :RequestCanceledException INFO [2019-12-17 12:37:44,843] jepsen worker 2 - jepsen.util 16112 :invoke :write [0 24] INFO [2019-12-17 12:37:44,844] jepsen worker 56 - jepsen.util 13286 :invoke :read [18 nil] INFO [2019-12-17 12:37:44,857] jepsen worker 63 - jepsen.util 10773 :fail :cas [21 [18 19]] :ValueNotSwapFrom INFO [2019-12-17 12:37:44,857] jepsen worker 63 - jepsen.util 10773 :invoke :cas [21 [27 8]] INFO [2019-12-17 12:37:44,858] jepsen worker 31 - jepsen.util 10471 :ok :read [10 29] INFO [2019-12-17 12:37:44,858] jepsen worker 31 - jepsen.util 10471 :invoke :cas [10 [18 21]] INFO [2019-12-17 12:37:44,860] jepsen worker 5 - jepsen.util 11885 :ok :read [1 24] INFO [2019-12-17 12:37:44,863] jepsen worker 77 - jepsen.util 17897 :info :cas [25 [48 24]] :RequestCanceledException INFO [2019-12-17 12:37:44,864] jepsen worker 5 - jepsen.util 11885 :invoke :write [1 31] INFO [2019-12-17 12:37:44,864] jepsen worker 77 - jepsen.util 17987 :invoke :cas [25 [10 26]] INFO [2019-12-17 12:37:44,905] jepsen worker 63 - jepsen.util 10773 :fail :cas [21 [27 8]] :ValueNotSwapFrom INFO [2019-12-17 12:37:44,905] jepsen worker 63 - jepsen.util 10773 :invoke :write [21 17] INFO [2019-12-17 12:37:44,911] jepsen worker 65 - jepsen.util 10775 :ok :read [21 16] INFO [2019-12-17 12:37:44,911] jepsen worker 65 - jepsen.util 10775 :invoke :cas [21 [15 10]] INFO [2019-12-17 12:37:44,914] jepsen worker 38 - jepsen.util 12728 :fail :cas [12 [47 8]] :ValueNotSwapFrom INFO [2019-12-17 12:37:44,915] jepsen worker 38 - jepsen.util 12728 :invoke :write [12 0] INFO [2019-12-17 12:37:44,917] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71051,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:37:44,917] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71051,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:37:44,917] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71054,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:37:44,917] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71054,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:37:44,929] jepsen worker 71 - jepsen.util 13121 :ok :read [23 19] INFO [2019-12-17 12:37:44,929] jepsen worker 71 - jepsen.util 13121 :invoke :read [23 nil] INFO [2019-12-17 12:37:44,934] jepsen worker 36 - jepsen.util 12906 :info :write [12 40] :RequestCanceledException INFO [2019-12-17 12:37:44,935] jepsen worker 36 - jepsen.util 12996 :invoke :read [12 nil] INFO [2019-12-17 12:37:44,980] jepsen worker 32 - jepsen.util 12722 :ok :read [10 29] INFO [2019-12-17 12:37:44,981] jepsen worker 32 - jepsen.util 12722 :invoke :cas [10 [18 22]] INFO [2019-12-17 12:37:44,997] jepsen worker 25 - jepsen.util 17755 :info :write [8 18] :RequestCanceledException INFO [2019-12-17 12:37:44,998] jepsen worker 25 - jepsen.util 17845 :invoke :write [8 0] INFO [2019-12-17 12:37:45,006] jepsen worker 63 - jepsen.util 10773 :ok :write [21 17] INFO [2019-12-17 12:37:45,007] jepsen worker 63 - jepsen.util 10773 :invoke :read [21 nil] INFO [2019-12-17 12:37:45,013] jepsen worker 4 - jepsen.util 11434 :fail :cas [1 [48 39]] :ValueNotSwapFrom INFO [2019-12-17 12:37:45,014] jepsen worker 4 - jepsen.util 11434 :invoke :read [1 nil] INFO [2019-12-17 12:37:45,016] jepsen worker 28 - jepsen.util 10468 :ok :write [9 18] INFO [2019-12-17 12:37:45,017] jepsen worker 28 - jepsen.util 10468 :invoke :cas [9 [7 4]] INFO [2019-12-17 12:37:45,034] jepsen worker 37 - jepsen.util 11647 :info :write [12 49] :RequestCanceledException INFO [2019-12-17 12:37:45,035] jepsen worker 54 - jepsen.util 12834 :ok :write [18 47] INFO [2019-12-17 12:37:45,035] jepsen worker 54 - jepsen.util 12834 :invoke :write [18 46] INFO [2019-12-17 12:37:45,047] jepsen worker 37 - jepsen.util 11737 :invoke :cas [12 [5 27]] INFO [2019-12-17 12:37:45,050] jepsen worker 65 - jepsen.util 10775 :fail :cas [21 [15 10]] :ValueNotSwapFrom INFO [2019-12-17 12:37:45,050] jepsen worker 65 - jepsen.util 10775 :invoke :read [21 nil] INFO [2019-12-17 12:37:45,051] jepsen worker 63 - jepsen.util 10773 :ok :read [21 17] INFO [2019-12-17 12:37:45,051] jepsen worker 63 - jepsen.util 10773 :invoke :cas [21 [11 16]] INFO [2019-12-17 12:37:45,062] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71064,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:37:45,062] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71064,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:37:45,062] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71069,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:37:45,062] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71069,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:37:45,066] jepsen worker 32 - jepsen.util 12722 :fail :cas [10 [18 22]] :ValueNotSwapFrom INFO [2019-12-17 12:37:45,066] jepsen worker 32 - jepsen.util 12722 :invoke :cas [10 [8 9]] INFO [2019-12-17 12:37:45,082] jepsen worker 87 - jepsen.util 11877 :fail :cas [29 [11 3]] :ValueNotSwapFrom INFO [2019-12-17 12:37:45,082] jepsen worker 87 - jepsen.util 11877 :invoke :write [29 35] INFO [2019-12-17 12:37:45,087] jepsen worker 51 - jepsen.util 11481 :ok :write [17 40] INFO [2019-12-17 12:37:45,087] jepsen worker 51 - jepsen.util 11481 :invoke :cas [17 [21 31]] INFO [2019-12-17 12:37:45,096] jepsen worker 28 - jepsen.util 10468 :fail :cas [9 [7 4]] :ValueNotSwapFrom INFO [2019-12-17 12:37:45,096] jepsen worker 28 - jepsen.util 10468 :invoke :write [9 4] INFO [2019-12-17 12:37:45,126] jepsen worker 4 - jepsen.util 11434 :ok :read [1 24] INFO [2019-12-17 12:37:45,126] jepsen worker 4 - jepsen.util 11434 :invoke :cas [1 [18 28]] INFO [2019-12-17 12:37:45,134] jepsen worker 35 - jepsen.util 16685 :info :write [11 31] :RequestCanceledException INFO [2019-12-17 12:37:45,134] jepsen worker 16 - jepsen.util 18466 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:37:45,134] jepsen worker 16 - jepsen.util 18466 :invoke :cas [5 [7 35]] INFO [2019-12-17 12:37:45,134] jepsen worker 35 - jepsen.util 16775 :invoke :cas [11 [39 4]] INFO [2019-12-17 12:37:45,147] jepsen worker 8 - jepsen.util 16388 :info :cas [2 [23 43]] :RequestCanceledException INFO [2019-12-17 12:37:45,148] jepsen worker 8 - jepsen.util 16478 :invoke :write [2 5] INFO [2019-12-17 12:37:45,153] jepsen worker 65 - jepsen.util 10775 :ok :read [21 17] INFO [2019-12-17 12:37:45,154] jepsen worker 65 - jepsen.util 10775 :invoke :write [21 37] INFO [2019-12-17 12:37:45,165] jepsen worker 79 - jepsen.util 12409 :info :cas [26 [47 31]] :RequestCanceledException INFO [2019-12-17 12:37:45,166] jepsen worker 79 - jepsen.util 12499 :invoke :write [26 1] INFO [2019-12-17 12:37:45,186] jepsen worker 63 - jepsen.util 10773 :fail :cas [21 [11 16]] :ValueNotSwapFrom INFO [2019-12-17 12:37:45,186] jepsen worker 63 - jepsen.util 10773 :invoke :read [21 nil] INFO [2019-12-17 12:37:45,189] jepsen worker 3 - jepsen.util 11613 :ok :write [1 12] INFO [2019-12-17 12:37:45,189] jepsen worker 3 - jepsen.util 11613 :invoke :cas [1 [12 46]] INFO [2019-12-17 12:37:45,200] jepsen worker 44 - jepsen.util 17324 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:37:45,200] jepsen worker 32 - jepsen.util 12722 :fail :cas [10 [8 9]] :ValueNotSwapFrom INFO [2019-12-17 12:37:45,200] jepsen worker 44 - jepsen.util 17324 :invoke :cas [14 [22 47]] INFO [2019-12-17 12:37:45,200] jepsen worker 32 - jepsen.util 12722 :invoke :read [10 nil] INFO [2019-12-17 12:37:45,216] jepsen worker 63 - jepsen.util 10773 :ok :read [21 17] INFO [2019-12-17 12:37:45,216] jepsen worker 63 - jepsen.util 10773 :invoke :read [21 nil] INFO [2019-12-17 12:37:45,218] jepsen worker 43 - jepsen.util 16963 :info :write [14 32] :RequestCanceledException INFO [2019-12-17 12:37:45,218] jepsen worker 74 - jepsen.util 11954 :info :write [24 43] :RequestCanceledException INFO [2019-12-17 12:37:45,219] jepsen worker 74 - jepsen.util 12044 :invoke :write [24 30] INFO [2019-12-17 12:37:45,219] jepsen worker 43 - jepsen.util 17053 :invoke :read [14 nil] INFO [2019-12-17 12:37:45,223] jepsen worker 32 - jepsen.util 12722 :ok :read [10 29] INFO [2019-12-17 12:37:45,223] jepsen worker 32 - jepsen.util 12722 :invoke :cas [10 [22 30]] INFO [2019-12-17 12:37:45,223] jepsen worker 4 - jepsen.util 11434 :fail :cas [1 [18 28]] :ValueNotSwapFrom INFO [2019-12-17 12:37:45,224] jepsen worker 4 - jepsen.util 11434 :invoke :cas [1 [43 3]] INFO [2019-12-17 12:37:45,233] jepsen worker 61 - jepsen.util 18241 :info :write [20 38] :RequestCanceledException INFO [2019-12-17 12:37:45,233] jepsen worker 46 - jepsen.util 11926 :info :cas [15 [3 10]] :RequestCanceledException INFO [2019-12-17 12:37:45,234] jepsen worker 61 - jepsen.util 18331 :invoke :cas [20 [7 5]] INFO [2019-12-17 12:37:45,236] jepsen worker 48 - jepsen.util 15438 :info :cas [16 [38 39]] :RequestCanceledException INFO [2019-12-17 12:37:45,236] jepsen worker 81 - jepsen.util 16461 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:37:45,236] jepsen worker 81 - jepsen.util 16461 :invoke :cas [27 [43 3]] INFO [2019-12-17 12:37:45,236] jepsen worker 46 - jepsen.util 12016 :invoke :read [15 nil] INFO [2019-12-17 12:37:45,237] jepsen worker 48 - jepsen.util 15528 :invoke :write [16 47] INFO [2019-12-17 12:37:45,248] jepsen worker 12 - jepsen.util 12522 :ok :write [4 13] INFO [2019-12-17 12:37:45,249] jepsen worker 12 - jepsen.util 12522 :invoke :cas [4 [35 1]] INFO [2019-12-17 12:37:45,268] jepsen worker 27 - jepsen.util 11277 :ok :write [9 36] INFO [2019-12-17 12:37:45,269] jepsen worker 27 - jepsen.util 11277 :invoke :cas [9 [32 39]] INFO [2019-12-17 12:37:45,276] jepsen worker 51 - jepsen.util 11481 :fail :cas [17 [21 31]] :ValueNotSwapFrom INFO [2019-12-17 12:37:45,276] jepsen worker 51 - jepsen.util 11481 :invoke :read [17 nil] INFO [2019-12-17 12:37:45,281] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71092,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:37:45,281] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71100,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:37:45,281] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71092,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:37:45,281] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71100,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:37:45,282] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71103,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:37:45,282] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71103,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:37:45,282] jepsen worker 32 - jepsen.util 12722 :fail :cas [10 [22 30]] :ValueNotSwapFrom INFO [2019-12-17 12:37:45,282] jepsen worker 32 - jepsen.util 12722 :invoke :read [10 nil] INFO [2019-12-17 12:37:45,301] jepsen worker 7 - jepsen.util 16837 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:37:45,301] jepsen worker 7 - jepsen.util 16837 :invoke :read [2 nil] INFO [2019-12-17 12:37:45,302] jepsen worker 63 - jepsen.util 10773 :ok :read [21 17] INFO [2019-12-17 12:37:45,302] jepsen worker 63 - jepsen.util 10773 :invoke :cas [21 [44 37]] INFO [2019-12-17 12:37:45,318] jepsen worker 15 - jepsen.util 15675 :info :write [5 0] :RequestCanceledException INFO [2019-12-17 12:37:45,324] jepsen worker 15 - jepsen.util 15765 :invoke :write [5 22] INFO [2019-12-17 12:37:45,334] jepsen worker 50 - jepsen.util 16790 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:37:45,334] jepsen worker 50 - jepsen.util 16790 :invoke :write [16 15] INFO [2019-12-17 12:37:45,343] jepsen worker 4 - jepsen.util 11434 :fail :cas [1 [43 3]] :ValueNotSwapFrom INFO [2019-12-17 12:37:45,343] jepsen worker 4 - jepsen.util 11434 :invoke :write [1 40] INFO [2019-12-17 12:37:45,354] jepsen worker 32 - jepsen.util 12722 :ok :read [10 29] INFO [2019-12-17 12:37:45,354] jepsen worker 32 - jepsen.util 12722 :invoke :read [10 nil] INFO [2019-12-17 12:37:45,364] jepsen worker 58 - jepsen.util 16888 :info :cas [19 [15 1]] :RequestCanceledException INFO [2019-12-17 12:37:45,364] jepsen worker 30 - jepsen.util 11370 :info :cas [10 [40 17]] :RequestCanceledException INFO [2019-12-17 12:37:45,366] jepsen worker 30 - jepsen.util 11460 :invoke :read [10 nil] INFO [2019-12-17 12:37:45,366] jepsen worker 58 - jepsen.util 16978 :invoke :write [19 23] INFO [2019-12-17 12:37:45,369] jepsen worker 46 - jepsen.util 12016 :ok :read [15 41] INFO [2019-12-17 12:37:45,370] jepsen worker 46 - jepsen.util 12016 :invoke :write [15 20] INFO [2019-12-17 12:37:45,371] jepsen worker 14 - jepsen.util 11984 :ok :write [4 37] INFO [2019-12-17 12:37:45,371] jepsen worker 14 - jepsen.util 11984 :invoke :read [4 nil] INFO [2019-12-17 12:37:45,372] jepsen worker 63 - jepsen.util 10773 :fail :cas [21 [44 37]] :ValueNotSwapFrom INFO [2019-12-17 12:37:45,372] jepsen worker 63 - jepsen.util 10773 :invoke :read [21 nil] INFO [2019-12-17 12:37:45,390] jepsen worker 51 - jepsen.util 11481 :ok :read [17 40] INFO [2019-12-17 12:37:45,390] jepsen worker 51 - jepsen.util 11481 :invoke :read [17 nil] INFO [2019-12-17 12:37:45,394] jepsen worker 5 - jepsen.util 11885 :ok :write [1 31] INFO [2019-12-17 12:37:45,396] jepsen worker 5 - jepsen.util 11885 :invoke :read [1 nil] INFO [2019-12-17 12:37:45,399] jepsen worker 32 - jepsen.util 12722 :ok :read [10 29] INFO [2019-12-17 12:37:45,400] jepsen worker 32 - jepsen.util 12722 :invoke :cas [10 [41 3]] INFO [2019-12-17 12:37:45,412] jepsen worker 5 - jepsen.util 11885 :ok :read [1 31] INFO [2019-12-17 12:37:45,412] jepsen worker 5 - jepsen.util 11885 :invoke :write [1 26] INFO [2019-12-17 12:37:45,418] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:37:45,418] jepsen nemesis - couchbase.util Rebalance check count: 560 INFO [2019-12-17 12:37:45,418] jepsen nemesis - couchbase.util Rebalance stuck count: 541 INFO [2019-12-17 12:37:45,424] jepsen worker 27 - jepsen.util 11277 :fail :cas [9 [32 39]] :ValueNotSwapFrom INFO [2019-12-17 12:37:45,425] jepsen worker 27 - jepsen.util 11277 :invoke :write [9 40] INFO [2019-12-17 12:37:45,431] jepsen worker 3 - jepsen.util 11613 :fail :cas [1 [12 46]] :CASMismatchException INFO [2019-12-17 12:37:45,431] jepsen worker 3 - jepsen.util 11613 :invoke :write [1 46] INFO [2019-12-17 12:37:45,452] jepsen worker 51 - jepsen.util 11481 :ok :read [17 40] INFO [2019-12-17 12:37:45,452] jepsen worker 51 - jepsen.util 11481 :invoke :write [17 42] INFO [2019-12-17 12:37:45,454] jepsen worker 63 - jepsen.util 10773 :ok :read [21 17] INFO [2019-12-17 12:37:45,455] jepsen worker 63 - jepsen.util 10773 :invoke :cas [21 [27 32]] INFO [2019-12-17 12:37:45,457] jepsen worker 45 - jepsen.util 12105 :ok :write [15 8] INFO [2019-12-17 12:37:45,457] jepsen worker 45 - jepsen.util 12105 :invoke :read [15 nil] INFO [2019-12-17 12:37:45,458] jepsen worker 32 - jepsen.util 12722 :fail :cas [10 [41 3]] :ValueNotSwapFrom INFO [2019-12-17 12:37:45,458] jepsen worker 32 - jepsen.util 12722 :invoke :write [10 14] INFO [2019-12-17 12:37:45,469] jepsen worker 20 - jepsen.util 12080 :ok :write [6 4] INFO [2019-12-17 12:37:45,469] jepsen worker 20 - jepsen.util 12080 :invoke :cas [6 [0 17]] INFO [2019-12-17 12:37:45,469] jepsen worker 73 - jepsen.util 11323 :ok :write [24 23] INFO [2019-12-17 12:37:45,469] jepsen worker 73 - jepsen.util 11323 :invoke :write [24 47] INFO [2019-12-17 12:37:45,500] jepsen worker 18 - jepsen.util 12618 :info :write [6 29] :RequestCanceledException INFO [2019-12-17 12:37:45,501] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71136,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:37:45,501] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71136,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:37:45,506] jepsen worker 18 - jepsen.util 12708 :invoke :cas [6 [38 35]] INFO [2019-12-17 12:37:45,511] jepsen worker 4 - jepsen.util 11434 :ok :write [1 40] INFO [2019-12-17 12:37:45,512] jepsen worker 4 - jepsen.util 11434 :invoke :write [1 38] INFO [2019-12-17 12:37:45,513] jepsen worker 37 - jepsen.util 11737 :fail :cas [12 [5 27]] :ValueNotSwapFrom INFO [2019-12-17 12:37:45,513] jepsen worker 63 - jepsen.util 10773 :fail :cas [21 [27 32]] :ValueNotSwapFrom INFO [2019-12-17 12:37:45,513] jepsen worker 37 - jepsen.util 11737 :invoke :read [12 nil] INFO [2019-12-17 12:37:45,513] jepsen worker 63 - jepsen.util 10773 :invoke :cas [21 [6 9]] INFO [2019-12-17 12:37:45,533] jepsen worker 72 - jepsen.util 12402 :fail :read [24 nil] :RequestTimeoutException INFO [2019-12-17 12:37:45,533] jepsen worker 57 - jepsen.util 16437 :info :write [19 41] :RequestCanceledException INFO [2019-12-17 12:37:45,534] jepsen worker 72 - jepsen.util 12402 :invoke :cas [24 [27 40]] INFO [2019-12-17 12:37:45,534] jepsen worker 57 - jepsen.util 16527 :invoke :cas [19 [32 26]] INFO [2019-12-17 12:37:45,565] jepsen worker 14 - jepsen.util 11984 :ok :read [4 37] INFO [2019-12-17 12:37:45,566] jepsen worker 14 - jepsen.util 11984 :invoke :read [4 nil] INFO [2019-12-17 12:37:45,579] jepsen worker 45 - jepsen.util 12105 :ok :read [15 8] INFO [2019-12-17 12:37:45,579] jepsen worker 45 - jepsen.util 12105 :invoke :read [15 nil] INFO [2019-12-17 12:37:45,581] jepsen worker 11 - jepsen.util 11801 :fail :cas [3 [43 16]] :ValueNotSwapFrom INFO [2019-12-17 12:37:45,581] jepsen worker 11 - jepsen.util 11801 :invoke :write [3 2] INFO [2019-12-17 12:37:45,584] jepsen worker 70 - jepsen.util 12760 :ok :write [23 37] INFO [2019-12-17 12:37:45,585] jepsen worker 70 - jepsen.util 12760 :invoke :read [23 nil] INFO [2019-12-17 12:37:45,588] jepsen worker 14 - jepsen.util 11984 :ok :read [4 37] INFO [2019-12-17 12:37:45,588] jepsen worker 14 - jepsen.util 11984 :invoke :read [4 nil] INFO [2019-12-17 12:37:45,618] jepsen worker 22 - jepsen.util 16942 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:37:45,618] jepsen worker 22 - jepsen.util 16942 :invoke :read [7 nil] INFO [2019-12-17 12:37:45,631] jepsen worker 45 - jepsen.util 12105 :ok :read [15 8] INFO [2019-12-17 12:37:45,631] jepsen worker 45 - jepsen.util 12105 :invoke :cas [15 [28 46]] INFO [2019-12-17 12:37:45,634] jepsen worker 83 - jepsen.util 16913 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:37:45,634] jepsen worker 83 - jepsen.util 16913 :invoke :read [27 nil] INFO [2019-12-17 12:37:45,636] jepsen worker 63 - jepsen.util 10773 :fail :cas [21 [6 9]] :ValueNotSwapFrom INFO [2019-12-17 12:37:45,636] jepsen worker 63 - jepsen.util 10773 :invoke :read [21 nil] INFO [2019-12-17 12:37:45,645] jepsen worker 14 - jepsen.util 11984 :ok :read [4 37] INFO [2019-12-17 12:37:45,647] jepsen worker 14 - jepsen.util 11984 :invoke :cas [4 [41 3]] INFO [2019-12-17 12:37:45,648] jepsen worker 42 - jepsen.util 17592 :info :cas [14 [23 8]] :RequestCanceledException INFO [2019-12-17 12:37:45,648] jepsen worker 1 - jepsen.util 17011 :info :cas [0 [34 49]] :RequestCanceledException INFO [2019-12-17 12:37:45,649] jepsen worker 42 - jepsen.util 17682 :invoke :read [14 nil] INFO [2019-12-17 12:37:45,650] jepsen worker 1 - jepsen.util 17101 :invoke :cas [0 [3 30]] INFO [2019-12-17 12:37:45,667] jepsen worker 72 - jepsen.util 12402 :fail :cas [24 [27 40]] :ValueNotSwapFrom INFO [2019-12-17 12:37:45,667] jepsen worker 72 - jepsen.util 12402 :invoke :cas [24 [30 10]] INFO [2019-12-17 12:37:45,672] jepsen worker 45 - jepsen.util 12105 :fail :cas [15 [28 46]] :ValueNotSwapFrom INFO [2019-12-17 12:37:45,672] jepsen worker 45 - jepsen.util 12105 :invoke :write [15 34] INFO [2019-12-17 12:37:45,691] jepsen worker 63 - jepsen.util 10773 :ok :read [21 17] INFO [2019-12-17 12:37:45,691] jepsen worker 63 - jepsen.util 10773 :invoke :cas [21 [22 23]] INFO [2019-12-17 12:37:45,700] jepsen worker 39 - jepsen.util 16329 :info :write [13 25] :RequestCanceledException INFO [2019-12-17 12:37:45,701] jepsen worker 39 - jepsen.util 16419 :invoke :write [13 18] INFO [2019-12-17 12:37:45,717] jepsen worker 53 - jepsen.util 11843 :info :write [17 1] :RequestCanceledException INFO [2019-12-17 12:37:45,718] jepsen worker 53 - jepsen.util 11933 :invoke :read [17 nil] INFO [2019-12-17 12:37:45,720] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71163,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:37:45,721] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71163,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:37:45,721] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71166,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:37:45,721] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71166,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:37:45,725] jepsen worker 72 - jepsen.util 12402 :fail :cas [24 [30 10]] :ValueNotSwapFrom INFO [2019-12-17 12:37:45,726] jepsen worker 37 - jepsen.util 11737 :ok :read [12 11] INFO [2019-12-17 12:37:45,726] jepsen worker 72 - jepsen.util 12402 :invoke :read [24 nil] INFO [2019-12-17 12:37:45,726] jepsen worker 37 - jepsen.util 11737 :invoke :write [12 19] INFO [2019-12-17 12:37:45,734] jepsen worker 9 - jepsen.util 12699 :info :write [3 2] :RequestCanceledException INFO [2019-12-17 12:37:45,736] jepsen worker 9 - jepsen.util 12789 :invoke :cas [3 [11 28]] INFO [2019-12-17 12:37:45,749] jepsen worker 63 - jepsen.util 10773 :fail :cas [21 [22 23]] :ValueNotSwapFrom INFO [2019-12-17 12:37:45,750] jepsen worker 63 - jepsen.util 10773 :invoke :cas [21 [19 29]] INFO [2019-12-17 12:37:45,757] jepsen worker 55 - jepsen.util 13015 :ok :write [18 37] INFO [2019-12-17 12:37:45,758] jepsen worker 55 - jepsen.util 13015 :invoke :write [18 15] INFO [2019-12-17 12:37:45,765] jepsen worker 47 - jepsen.util 13277 :fail :read [15 nil] :RequestTimeoutException INFO [2019-12-17 12:37:45,766] jepsen worker 47 - jepsen.util 13277 :invoke :write [15 31] INFO [2019-12-17 12:37:45,769] jepsen worker 72 - jepsen.util 12402 :ok :read [24 23] INFO [2019-12-17 12:37:45,770] jepsen worker 72 - jepsen.util 12402 :invoke :cas [24 [20 37]] INFO [2019-12-17 12:37:45,783] jepsen worker 63 - jepsen.util 10773 :fail :cas [21 [19 29]] :ValueNotSwapFrom INFO [2019-12-17 12:37:45,784] jepsen worker 63 - jepsen.util 10773 :invoke :read [21 nil] INFO [2019-12-17 12:37:45,797] jepsen worker 80 - jepsen.util 11780 :ok :read [26 1] INFO [2019-12-17 12:37:45,797] jepsen worker 72 - jepsen.util 12402 :fail :cas [24 [20 37]] :ValueNotSwapFrom INFO [2019-12-17 12:37:45,798] jepsen worker 80 - jepsen.util 11780 :invoke :write [26 20] INFO [2019-12-17 12:37:45,798] jepsen worker 72 - jepsen.util 12402 :invoke :cas [24 [46 8]] INFO [2019-12-17 12:37:45,800] jepsen worker 36 - jepsen.util 12996 :ok :read [12 11] INFO [2019-12-17 12:37:45,800] jepsen worker 36 - jepsen.util 12996 :invoke :write [12 11] INFO [2019-12-17 12:37:45,817] jepsen worker 76 - jepsen.util 16906 :info :write [25 5] :RequestCanceledException INFO [2019-12-17 12:37:45,818] jepsen worker 76 - jepsen.util 16996 :invoke :cas [25 [11 46]] INFO [2019-12-17 12:37:45,820] jepsen worker 63 - jepsen.util 10773 :ok :read [21 17] INFO [2019-12-17 12:37:45,821] jepsen worker 63 - jepsen.util 10773 :invoke :read [21 nil] INFO [2019-12-17 12:37:45,831] jepsen worker 72 - jepsen.util 12402 :fail :cas [24 [46 8]] :ValueNotSwapFrom INFO [2019-12-17 12:37:45,831] jepsen worker 72 - jepsen.util 12402 :invoke :read [24 nil] INFO [2019-12-17 12:37:45,834] jepsen worker 49 - jepsen.util 16789 :info :cas [16 [0 9]] :RequestCanceledException INFO [2019-12-17 12:37:45,835] jepsen worker 49 - jepsen.util 16879 :invoke :write [16 47] INFO [2019-12-17 12:37:45,847] jepsen worker 14 - jepsen.util 11984 :fail :cas [4 [41 3]] :ValueNotSwapFrom INFO [2019-12-17 12:37:45,848] jepsen worker 14 - jepsen.util 11984 :invoke :cas [4 [47 19]] INFO [2019-12-17 12:37:45,860] jepsen worker 63 - jepsen.util 10773 :ok :read [21 17] INFO [2019-12-17 12:37:45,860] jepsen worker 63 - jepsen.util 10773 :invoke :write [21 11] INFO [2019-12-17 12:37:45,895] jepsen worker 72 - jepsen.util 12402 :ok :read [24 23] INFO [2019-12-17 12:37:45,895] jepsen worker 72 - jepsen.util 12402 :invoke :write [24 19] INFO [2019-12-17 12:37:45,900] jepsen worker 68 - jepsen.util 15998 :info :cas [22 [33 27]] :RequestCanceledException INFO [2019-12-17 12:37:45,901] jepsen worker 68 - jepsen.util 16088 :invoke :read [22 nil] INFO [2019-12-17 12:37:45,902] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71192,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:37:45,902] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71192,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:37:45,902] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71193,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:37:45,902] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71193,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:37:45,903] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71197,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:37:45,903] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71197,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:37:45,923] jepsen worker 63 - jepsen.util 10773 :ok :write [21 11] INFO [2019-12-17 12:37:45,924] jepsen worker 63 - jepsen.util 10773 :invoke :cas [21 [0 26]] INFO [2019-12-17 12:37:45,933] jepsen worker 6 - jepsen.util 17106 :info :cas [2 [6 9]] :RequestCanceledException INFO [2019-12-17 12:37:45,934] jepsen worker 6 - jepsen.util 17196 :invoke :read [2 nil] INFO [2019-12-17 12:37:45,944] jepsen worker 14 - jepsen.util 11984 :fail :cas [4 [47 19]] :ValueNotSwapFrom INFO [2019-12-17 12:37:45,944] jepsen worker 14 - jepsen.util 11984 :invoke :cas [4 [45 2]] INFO [2019-12-17 12:37:45,990] jepsen worker 71 - jepsen.util 13121 :ok :read [23 37] INFO [2019-12-17 12:37:45,990] jepsen worker 71 - jepsen.util 13121 :invoke :read [23 nil] INFO [2019-12-17 12:37:45,996] jepsen worker 87 - jepsen.util 11877 :ok :write [29 35] INFO [2019-12-17 12:37:45,996] jepsen worker 87 - jepsen.util 11877 :invoke :cas [29 [23 5]] INFO [2019-12-17 12:37:46,000] jepsen worker 41 - jepsen.util 18131 :info :cas [13 [1 3]] :RequestCanceledException INFO [2019-12-17 12:37:46,002] jepsen worker 41 - jepsen.util 18221 :invoke :read [13 nil] INFO [2019-12-17 12:37:46,003] jepsen worker 89 - jepsen.util 12059 :fail :cas [29 [8 5]] :ValueNotSwapFrom INFO [2019-12-17 12:37:46,003] jepsen worker 89 - jepsen.util 12059 :invoke :read [29 nil] INFO [2019-12-17 12:37:46,018] jepsen worker 24 - jepsen.util 15954 :info :write [8 0] :RequestCanceledException INFO [2019-12-17 12:37:46,019] jepsen worker 24 - jepsen.util 16044 :invoke :read [8 nil] INFO [2019-12-17 12:37:46,021] jepsen worker 14 - jepsen.util 11984 :fail :cas [4 [45 2]] :ValueNotSwapFrom INFO [2019-12-17 12:37:46,021] jepsen worker 14 - jepsen.util 11984 :invoke :write [4 26] INFO [2019-12-17 12:37:46,033] jepsen worker 40 - jepsen.util 17590 :info :cas [13 [12 16]] :RequestCanceledException INFO [2019-12-17 12:37:46,043] jepsen worker 40 - jepsen.util 17680 :invoke :read [13 nil] INFO [2019-12-17 12:37:46,048] jepsen worker 34 - jepsen.util 17494 :info :cas [11 [42 38]] :RequestCanceledException INFO [2019-12-17 12:37:46,050] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71207,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:37:46,050] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71207,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:37:46,050] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71211,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:37:46,050] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71211,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:37:46,050] jepsen worker 34 - jepsen.util 17584 :invoke :write [11 29] INFO [2019-12-17 12:37:46,051] jepsen worker 78 - jepsen.util 11958 :fail :cas [26 [18 39]] :ValueNotSwapFrom INFO [2019-12-17 12:37:46,051] jepsen worker 78 - jepsen.util 11958 :invoke :cas [26 [10 10]] INFO [2019-12-17 12:37:46,087] jepsen worker 56 - jepsen.util 13286 :ok :read [18 37] INFO [2019-12-17 12:37:46,087] jepsen worker 56 - jepsen.util 13286 :invoke :cas [18 [20 17]] INFO [2019-12-17 12:37:46,099] jepsen worker 74 - jepsen.util 12044 :ok :write [24 30] INFO [2019-12-17 12:37:46,099] jepsen worker 74 - jepsen.util 12044 :invoke :read [24 nil] INFO [2019-12-17 12:37:46,100] jepsen worker 46 - jepsen.util 12016 :ok :write [15 20] INFO [2019-12-17 12:37:46,100] jepsen worker 46 - jepsen.util 12016 :invoke :cas [15 [11 12]] INFO [2019-12-17 12:37:46,114] jepsen worker 70 - jepsen.util 12760 :ok :read [23 37] INFO [2019-12-17 12:37:46,114] jepsen worker 70 - jepsen.util 12760 :invoke :cas [23 [18 23]] INFO [2019-12-17 12:37:46,126] jepsen worker 46 - jepsen.util 12016 :fail :cas [15 [11 12]] :ValueNotSwapFrom INFO [2019-12-17 12:37:46,126] jepsen worker 46 - jepsen.util 12016 :invoke :read [15 nil] INFO [2019-12-17 12:37:46,129] jepsen worker 20 - jepsen.util 12080 :fail :cas [6 [0 17]] :ValueNotSwapFrom INFO [2019-12-17 12:37:46,129] jepsen worker 20 - jepsen.util 12080 :invoke :write [6 27] INFO [2019-12-17 12:37:46,130] jepsen worker 36 - jepsen.util 12996 :ok :write [12 11] INFO [2019-12-17 12:37:46,131] jepsen worker 36 - jepsen.util 12996 :invoke :cas [12 [42 34]] INFO [2019-12-17 12:37:46,144] jepsen worker 46 - jepsen.util 12016 :ok :read [15 20] INFO [2019-12-17 12:37:46,144] jepsen worker 46 - jepsen.util 12016 :invoke :write [15 20] INFO [2019-12-17 12:37:46,145] jepsen worker 89 - jepsen.util 12059 :ok :read [29 35] INFO [2019-12-17 12:37:46,145] jepsen worker 89 - jepsen.util 12059 :invoke :write [29 0] INFO [2019-12-17 12:37:46,187] jepsen worker 32 - jepsen.util 12722 :ok :write [10 14] INFO [2019-12-17 12:37:46,187] jepsen worker 32 - jepsen.util 12722 :invoke :read [10 nil] INFO [2019-12-17 12:37:46,201] jepsen worker 9 - jepsen.util 12789 :fail :cas [3 [11 28]] :ValueNotSwapFrom INFO [2019-12-17 12:37:46,201] jepsen worker 9 - jepsen.util 12789 :invoke :write [3 28] INFO [2019-12-17 12:37:46,212] jepsen worker 32 - jepsen.util 12722 :ok :read [10 14] INFO [2019-12-17 12:37:46,213] jepsen worker 32 - jepsen.util 12722 :invoke :read [10 nil] INFO [2019-12-17 12:37:46,222] jepsen worker 36 - jepsen.util 12996 :fail :cas [12 [42 34]] :ValueNotSwapFrom INFO [2019-12-17 12:37:46,222] jepsen worker 36 - jepsen.util 12996 :invoke :write [12 44] INFO [2019-12-17 12:37:46,245] jepsen worker 78 - jepsen.util 11958 :fail :cas [26 [10 10]] :ValueNotSwapFrom INFO [2019-12-17 12:37:46,245] jepsen worker 78 - jepsen.util 11958 :invoke :write [26 33] INFO [2019-12-17 12:37:46,268] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71228,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:37:46,268] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71228,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:37:46,268] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71229,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:37:46,268] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71229,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:37:46,268] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71230,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:37:46,268] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71230,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:37:46,269] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71237,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:37:46,269] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71237,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:37:46,269] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71241,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:37:46,269] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71241,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:37:46,273] jepsen worker 79 - jepsen.util 12499 :ok :write [26 1] INFO [2019-12-17 12:37:46,274] jepsen worker 79 - jepsen.util 12499 :invoke :write [26 6] INFO [2019-12-17 12:37:46,300] jepsen worker 60 - jepsen.util 17790 :info :write [20 46] :RequestCanceledException INFO [2019-12-17 12:37:46,301] jepsen worker 11 - jepsen.util 11801 :ok :write [3 2] INFO [2019-12-17 12:37:46,301] jepsen worker 11 - jepsen.util 11801 :invoke :write [3 43] INFO [2019-12-17 12:37:46,306] jepsen worker 60 - jepsen.util 17880 :invoke :read [20 nil] INFO [2019-12-17 12:37:46,319] jepsen worker 62 - jepsen.util 17522 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:37:46,319] jepsen worker 62 - jepsen.util 17522 :invoke :write [20 49] INFO [2019-12-17 12:37:46,322] jepsen worker 12 - jepsen.util 12522 :fail :cas [4 [35 1]] :ValueNotSwapFrom INFO [2019-12-17 12:37:46,322] jepsen worker 12 - jepsen.util 12522 :invoke :cas [4 [15 7]] INFO [2019-12-17 12:37:46,326] jepsen worker 46 - jepsen.util 12016 :ok :write [15 20] INFO [2019-12-17 12:37:46,326] jepsen worker 46 - jepsen.util 12016 :invoke :read [15 nil] INFO [2019-12-17 12:37:46,334] jepsen worker 17 - jepsen.util 17027 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:37:46,334] jepsen worker 17 - jepsen.util 17027 :invoke :read [5 nil] INFO [2019-12-17 12:37:46,342] jepsen worker 56 - jepsen.util 13286 :fail :cas [18 [20 17]] :ValueNotSwapFrom INFO [2019-12-17 12:37:46,342] jepsen worker 56 - jepsen.util 13286 :invoke :read [18 nil] INFO [2019-12-17 12:37:46,353] jepsen worker 74 - jepsen.util 12044 :ok :read [24 30] INFO [2019-12-17 12:37:46,353] jepsen worker 74 - jepsen.util 12044 :invoke :cas [24 [0 20]] INFO [2019-12-17 12:37:46,365] jepsen worker 86 - jepsen.util 17546 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:37:46,365] jepsen worker 86 - jepsen.util 17546 :invoke :read [28 nil] INFO [2019-12-17 12:37:46,371] jepsen worker 46 - jepsen.util 12016 :ok :read [15 20] INFO [2019-12-17 12:37:46,371] jepsen worker 46 - jepsen.util 12016 :invoke :cas [15 [43 21]] INFO [2019-12-17 12:37:46,383] jepsen worker 87 - jepsen.util 11877 :fail :cas [29 [23 5]] :ValueNotSwapFrom INFO [2019-12-17 12:37:46,383] jepsen worker 87 - jepsen.util 11877 :invoke :write [29 46] INFO [2019-12-17 12:37:46,386] jepsen worker 70 - jepsen.util 12760 :fail :cas [23 [18 23]] :ValueNotSwapFrom INFO [2019-12-17 12:37:46,386] jepsen worker 46 - jepsen.util 12016 :fail :cas [15 [43 21]] :ValueNotSwapFrom INFO [2019-12-17 12:37:46,386] jepsen worker 70 - jepsen.util 12760 :invoke :cas [23 [14 29]] INFO [2019-12-17 12:37:46,386] jepsen worker 46 - jepsen.util 12016 :invoke :write [15 1] INFO [2019-12-17 12:37:46,401] jepsen worker 20 - jepsen.util 12080 :ok :write [6 27] INFO [2019-12-17 12:37:46,401] jepsen worker 20 - jepsen.util 12080 :invoke :cas [6 [29 23]] INFO [2019-12-17 12:37:46,412] jepsen worker 53 - jepsen.util 11933 :ok :read [17 40] INFO [2019-12-17 12:37:46,412] jepsen worker 53 - jepsen.util 11933 :invoke :read [17 nil] INFO [2019-12-17 12:37:46,418] jepsen worker 75 - jepsen.util 16365 :info :write [25 18] :RequestCanceledException INFO [2019-12-17 12:37:46,419] jepsen worker 75 - jepsen.util 16455 :invoke :read [25 nil] INFO [2019-12-17 12:37:46,431] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:37:46,432] jepsen nemesis - couchbase.util Rebalance check count: 561 INFO [2019-12-17 12:37:46,432] jepsen nemesis - couchbase.util Rebalance stuck count: 542 INFO [2019-12-17 12:37:46,433] jepsen worker 26 - jepsen.util 17486 :info :write [8 33] :RequestCanceledException INFO [2019-12-17 12:37:46,433] jepsen worker 64 - jepsen.util 9424 :info :write [21 1] :RequestCanceledException INFO [2019-12-17 12:37:46,435] jepsen worker 64 - jepsen.util 9514 :invoke :cas [21 [40 16]] INFO [2019-12-17 12:37:46,437] jepsen worker 26 - jepsen.util 17576 :invoke :cas [8 [16 21]] INFO [2019-12-17 12:37:46,448] jepsen worker 88 - jepsen.util 12418 :info :write [29 10] :RequestCanceledException INFO [2019-12-17 12:37:46,448] jepsen worker 88 - jepsen.util 12508 :invoke :cas [29 [2 29]] INFO [2019-12-17 12:37:46,452] jepsen worker 64 - jepsen.util 9514 :fail :cas [21 [40 16]] :ValueNotSwapFrom INFO [2019-12-17 12:37:46,452] jepsen worker 64 - jepsen.util 9514 :invoke :read [21 nil] INFO [2019-12-17 12:37:46,467] jepsen worker 46 - jepsen.util 12016 :ok :write [15 1] INFO [2019-12-17 12:37:46,485] jepsen worker 64 - jepsen.util 9514 :ok :read [21 11] INFO [2019-12-17 12:37:46,488] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71262,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:37:46,497] jepsen worker 53 - jepsen.util 11933 :ok :read [17 40] INFO [2019-12-17 12:37:46,512] jepsen worker 11 - jepsen.util 11801 :ok :write [3 43] INFO [2019-12-17 12:37:46,519] jepsen worker 33 - jepsen.util 16683 :info :cas [11 [42 19]] :RequestCanceledException INFO [2019-12-17 12:37:46,525] jepsen worker 74 - jepsen.util 12044 :fail :cas [24 [0 20]] :ValueNotSwapFrom INFO [2019-12-17 12:37:46,533] jepsen worker 85 - jepsen.util 17365 :info :write [28 11] :RequestCanceledException INFO [2019-12-17 12:37:46,536] jepsen worker 20 - jepsen.util 12080 :fail :cas [6 [29 23]] :ValueNotSwapFrom INFO [2019-12-17 12:37:46,545] jepsen worker 66 - jepsen.util 16806 :info :write [22 14] :RequestCanceledException INFO [2019-12-17 12:37:46,565] jepsen worker 67 - jepsen.util 16987 :info :cas [22 [38 19]] :RequestCanceledException INFO [2019-12-17 12:37:46,618] jepsen worker 82 - jepsen.util 17182 :info :write [27 42] :RequestCanceledException INFO [2019-12-17 12:37:46,619] jepsen worker 52 - jepsen.util 11662 :info :write [17 34] :RequestCanceledException INFO [2019-12-17 12:37:46,623] jepsen worker 71 - jepsen.util 13121 :ok :read [23 37] INFO [2019-12-17 12:37:46,653] jepsen worker 88 - jepsen.util 12508 :fail :cas [29 [2 29]] :ValueNotSwapFrom INFO [2019-12-17 12:37:46,654] jepsen worker 46 - jepsen.util 12016 :invoke :cas [15 [36 28]] INFO [2019-12-17 12:37:46,654] jepsen worker 64 - jepsen.util 9514 :invoke :read [21 nil] INFO [2019-12-17 12:37:46,654] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71262,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:37:46,654] jepsen worker 53 - jepsen.util 11933 :invoke :read [17 nil] INFO [2019-12-17 12:37:46,654] jepsen worker 11 - jepsen.util 11801 :invoke :cas [3 [27 19]] INFO [2019-12-17 12:37:46,655] jepsen worker 74 - jepsen.util 12044 :invoke :read [24 nil] INFO [2019-12-17 12:37:46,655] jepsen worker 20 - jepsen.util 12080 :invoke :cas [6 [44 35]] INFO [2019-12-17 12:37:46,655] jepsen worker 33 - jepsen.util 16773 :invoke :read [11 nil] INFO [2019-12-17 12:37:46,655] jepsen worker 71 - jepsen.util 13121 :invoke :write [23 48] INFO [2019-12-17 12:37:46,655] jepsen worker 88 - jepsen.util 12508 :invoke :read [29 nil] INFO [2019-12-17 12:37:46,656] jepsen worker 85 - jepsen.util 17455 :invoke :cas [28 [38 29]] INFO [2019-12-17 12:37:46,656] jepsen worker 66 - jepsen.util 16896 :invoke :cas [22 [31 6]] INFO [2019-12-17 12:37:46,656] jepsen worker 67 - jepsen.util 17077 :invoke :cas [22 [29 38]] INFO [2019-12-17 12:37:46,656] jepsen worker 82 - jepsen.util 17272 :invoke :read [27 nil] INFO [2019-12-17 12:37:46,656] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71265,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:37:46,657] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71265,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:37:46,657] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71266,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:37:46,657] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71266,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:37:46,664] jepsen worker 52 - jepsen.util 11752 :invoke :cas [17 [31 22]] INFO [2019-12-17 12:37:46,665] jepsen worker 84 - jepsen.util 16374 :info :cas [28 [25 5]] :RequestCanceledException INFO [2019-12-17 12:37:46,666] jepsen worker 84 - jepsen.util 16464 :invoke :read [28 nil] INFO [2019-12-17 12:37:46,688] jepsen worker 11 - jepsen.util 11801 :fail :cas [3 [27 19]] :ValueNotSwapFrom INFO [2019-12-17 12:37:46,688] jepsen worker 11 - jepsen.util 11801 :invoke :cas [3 [39 6]] INFO [2019-12-17 12:37:46,692] jepsen worker 64 - jepsen.util 9514 :ok :read [21 11] INFO [2019-12-17 12:37:46,692] jepsen worker 64 - jepsen.util 9514 :invoke :cas [21 [44 8]] INFO [2019-12-17 12:37:46,700] jepsen worker 74 - jepsen.util 12044 :ok :read [24 30] INFO [2019-12-17 12:37:46,700] jepsen worker 74 - jepsen.util 12044 :invoke :write [24 15] INFO [2019-12-17 12:37:46,709] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71272,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:37:46,710] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71272,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:37:46,710] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71276,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:37:46,710] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71276,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:37:46,712] jepsen worker 20 - jepsen.util 12080 :fail :cas [6 [44 35]] :ValueNotSwapFrom INFO [2019-12-17 12:37:46,712] jepsen worker 20 - jepsen.util 12080 :invoke :write [6 43] INFO [2019-12-17 12:37:46,734] jepsen worker 23 - jepsen.util 17213 :info :cas [7 [35 41]] :RequestCanceledException INFO [2019-12-17 12:37:46,735] jepsen worker 23 - jepsen.util 17303 :invoke :cas [7 [7 45]] INFO [2019-12-17 12:37:46,740] jepsen worker 88 - jepsen.util 12508 :ok :read [29 35] INFO [2019-12-17 12:37:46,740] jepsen worker 88 - jepsen.util 12508 :invoke :write [29 34] INFO [2019-12-17 12:37:46,747] jepsen worker 19 - jepsen.util 12619 :info :cas [6 [44 20]] :RequestCanceledException INFO [2019-12-17 12:37:46,749] jepsen worker 19 - jepsen.util 12709 :invoke :write [6 25] INFO [2019-12-17 12:37:46,749] jepsen worker 64 - jepsen.util 9514 :fail :cas [21 [44 8]] :ValueNotSwapFrom INFO [2019-12-17 12:37:46,750] jepsen worker 64 - jepsen.util 9514 :invoke :cas [21 [38 34]] INFO [2019-12-17 12:37:46,750] jepsen worker 11 - jepsen.util 11801 :fail :cas [3 [39 6]] :ValueNotSwapFrom INFO [2019-12-17 12:37:46,751] jepsen worker 11 - jepsen.util 11801 :invoke :cas [3 [48 38]] INFO [2019-12-17 12:37:46,752] jepsen worker 46 - jepsen.util 12016 :fail :cas [15 [36 28]] :ValueNotSwapFrom INFO [2019-12-17 12:37:46,753] jepsen worker 46 - jepsen.util 12016 :invoke :write [15 28] INFO [2019-12-17 12:37:46,765] jepsen worker 29 - jepsen.util 10739 :fail :read [9 nil] :RequestTimeoutException INFO [2019-12-17 12:37:46,765] jepsen worker 29 - jepsen.util 10739 :invoke :cas [9 [6 49]] INFO [2019-12-17 12:37:46,775] jepsen worker 64 - jepsen.util 9514 :fail :cas [21 [38 34]] :ValueNotSwapFrom INFO [2019-12-17 12:37:46,776] jepsen worker 64 - jepsen.util 9514 :invoke :write [21 37] INFO [2019-12-17 12:37:46,783] jepsen worker 52 - jepsen.util 11752 :fail :cas [17 [31 22]] :ValueNotSwapFrom INFO [2019-12-17 12:37:46,784] jepsen worker 52 - jepsen.util 11752 :invoke :write [17 9] INFO [2019-12-17 12:37:46,800] jepsen worker 21 - jepsen.util 16311 :info :cas [7 [23 0]] :RequestCanceledException INFO [2019-12-17 12:37:46,800] jepsen worker 0 - jepsen.util 17370 :info :write [0 24] :RequestCanceledException INFO [2019-12-17 12:37:46,801] jepsen worker 0 - jepsen.util 17460 :invoke :cas [0 [26 0]] INFO [2019-12-17 12:37:46,802] jepsen worker 21 - jepsen.util 16401 :invoke :read [7 nil] INFO [2019-12-17 12:37:46,817] jepsen worker 59 - jepsen.util 15989 :info :write [19 45] :RequestCanceledException INFO [2019-12-17 12:37:46,834] jepsen worker 74 - jepsen.util 12044 :ok :write [24 15] INFO [2019-12-17 12:37:46,848] jepsen worker 13 - jepsen.util 11713 :info :write [4 3] :RequestCanceledException INFO [2019-12-17 12:37:46,862] jepsen worker 64 - jepsen.util 9514 :ok :write [21 37] INFO [2019-12-17 12:37:46,869] jepsen worker 11 - jepsen.util 11801 :fail :cas [3 [48 38]] :ValueNotSwapFrom INFO [2019-12-17 12:37:46,889] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71311,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:37:46,900] jepsen worker 10 - jepsen.util 12070 :info :write [3 48] :RequestCanceledException INFO [2019-12-17 12:37:46,900] jepsen worker 69 - jepsen.util 12489 :info :write [23 8] :RequestCanceledException INFO [2019-12-17 12:37:46,900] jepsen worker 31 - jepsen.util 10471 :info :cas [10 [18 21]] :RequestCanceledException INFO [2019-12-17 12:37:46,923] jepsen worker 56 - jepsen.util 13286 :ok :read [18 37] INFO [2019-12-17 12:37:46,948] jepsen worker 38 - jepsen.util 12728 :info :write [12 0] :RequestCanceledException INFO [2019-12-17 12:37:46,948] jepsen worker 77 - jepsen.util 17987 :info :cas [25 [10 26]] :RequestCanceledException INFO [2019-12-17 12:37:46,948] jepsen worker 2 - jepsen.util 16112 :info :write [0 24] :RequestCanceledException INFO [2019-12-17 12:37:46,951] jepsen worker 51 - jepsen.util 11481 :ok :write [17 42] INFO [2019-12-17 12:37:46,985] jepsen worker 46 - jepsen.util 12016 :ok :write [15 28] INFO [2019-12-17 12:37:46,988] jepsen worker 19 - jepsen.util 12709 :ok :write [6 25] INFO [2019-12-17 12:37:46,998] jepsen worker 12 - jepsen.util 12522 :fail :cas [4 [15 7]] :ValueNotSwapFrom INFO [2019-12-17 12:37:47,013] jepsen worker 70 - jepsen.util 12760 :fail :cas [23 [14 29]] :ValueNotSwapFrom INFO [2019-12-17 12:37:47,033] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71314,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:37:47,034] jepsen worker 25 - jepsen.util 17845 :info :write [8 0] :RequestCanceledException INFO [2019-12-17 12:37:47,048] jepsen worker 74 - jepsen.util 12044 :invoke :cas [24 [36 39]] INFO [2019-12-17 12:37:47,049] jepsen worker 11 - jepsen.util 11801 :invoke :cas [3 [42 4]] INFO [2019-12-17 12:37:47,049] jepsen worker 64 - jepsen.util 9514 :invoke :write [21 17] INFO [2019-12-17 12:37:47,049] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71311,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:37:47,049] jepsen worker 56 - jepsen.util 13286 :invoke :read [18 nil] INFO [2019-12-17 12:37:47,049] jepsen worker 59 - jepsen.util 16079 :invoke :cas [19 [6 38]] INFO [2019-12-17 12:37:47,050] jepsen worker 69 - jepsen.util 12579 :invoke :read [23 nil] INFO [2019-12-17 12:37:47,050] jepsen worker 10 - jepsen.util 12160 :invoke :write [3 45] INFO [2019-12-17 12:37:47,050] jepsen worker 38 - jepsen.util 12818 :invoke :cas [12 [14 10]] INFO [2019-12-17 12:37:47,050] jepsen worker 51 - jepsen.util 11481 :invoke :read [17 nil] INFO [2019-12-17 12:37:47,050] jepsen worker 13 - jepsen.util 11803 :invoke :cas [4 [25 11]] INFO [2019-12-17 12:37:47,050] jepsen worker 77 - jepsen.util 18077 :invoke :write [25 34] INFO [2019-12-17 12:37:47,050] jepsen worker 46 - jepsen.util 12016 :invoke :cas [15 [8 34]] INFO [2019-12-17 12:37:47,050] jepsen worker 19 - jepsen.util 12709 :invoke :write [6 23] INFO [2019-12-17 12:37:47,050] jepsen worker 12 - jepsen.util 12522 :invoke :read [4 nil] INFO [2019-12-17 12:37:47,050] jepsen worker 70 - jepsen.util 12760 :invoke :cas [23 [8 8]] INFO [2019-12-17 12:37:47,050] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71314,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:37:47,051] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71312,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:37:47,051] jepsen worker 25 - jepsen.util 17935 :invoke :write [8 30] INFO [2019-12-17 12:37:47,053] jepsen worker 31 - jepsen.util 10561 :invoke :read [10 nil] INFO [2019-12-17 12:37:47,053] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71317,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:37:47,053] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71312,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:37:47,053] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71317,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:37:47,053] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71318,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:37:47,053] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71318,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:37:47,054] jepsen worker 2 - jepsen.util 16202 :invoke :read [0 nil] INFO [2019-12-17 12:37:47,057] jepsen worker 54 - jepsen.util 12834 :info :write [18 46] :RequestCanceledException INFO [2019-12-17 12:37:47,064] jepsen worker 54 - jepsen.util 12924 :invoke :write [18 9] INFO [2019-12-17 12:37:47,094] jepsen worker 74 - jepsen.util 12044 :fail :cas [24 [36 39]] :ValueNotSwapFrom INFO [2019-12-17 12:37:47,094] jepsen worker 74 - jepsen.util 12044 :invoke :cas [24 [27 4]] INFO [2019-12-17 12:37:47,110] jepsen worker 46 - jepsen.util 12016 :fail :cas [15 [8 34]] :ValueNotSwapFrom INFO [2019-12-17 12:37:47,110] jepsen worker 46 - jepsen.util 12016 :invoke :cas [15 [8 4]] INFO [2019-12-17 12:37:47,114] jepsen worker 53 - jepsen.util 11933 :ok :read [17 42] INFO [2019-12-17 12:37:47,114] jepsen worker 53 - jepsen.util 11933 :invoke :write [17 25] INFO [2019-12-17 12:37:47,117] jepsen worker 38 - jepsen.util 12818 :fail :cas [12 [14 10]] :ValueNotSwapFrom INFO [2019-12-17 12:37:47,117] jepsen worker 38 - jepsen.util 12818 :invoke :write [12 26] INFO [2019-12-17 12:37:47,158] jepsen worker 31 - jepsen.util 10561 :ok :read [10 14] INFO [2019-12-17 12:37:47,158] jepsen worker 31 - jepsen.util 10561 :invoke :cas [10 [8 36]] INFO [2019-12-17 12:37:47,173] jepsen worker 11 - jepsen.util 11801 :fail :cas [3 [42 4]] :ValueNotSwapFrom INFO [2019-12-17 12:37:47,173] jepsen worker 11 - jepsen.util 11801 :invoke :read [3 nil] INFO [2019-12-17 12:37:47,184] jepsen worker 31 - jepsen.util 10561 :fail :cas [10 [8 36]] :ValueNotSwapFrom INFO [2019-12-17 12:37:47,184] jepsen worker 31 - jepsen.util 10561 :invoke :read [10 nil] INFO [2019-12-17 12:37:47,190] jepsen worker 46 - jepsen.util 12016 :fail :cas [15 [8 4]] :ValueNotSwapFrom INFO [2019-12-17 12:37:47,191] jepsen worker 46 - jepsen.util 12016 :invoke :cas [15 [20 7]] INFO [2019-12-17 12:37:47,201] jepsen worker 65 - jepsen.util 10775 :info :write [21 37] :RequestCanceledException INFO [2019-12-17 12:37:47,201] jepsen worker 28 - jepsen.util 10468 :info :write [9 4] :RequestCanceledException INFO [2019-12-17 12:37:47,202] jepsen worker 28 - jepsen.util 10558 :invoke :cas [9 [6 12]] INFO [2019-12-17 12:37:47,202] jepsen worker 65 - jepsen.util 10865 :invoke :write [21 7] INFO [2019-12-17 12:37:47,202] jepsen worker 31 - jepsen.util 10561 :ok :read [10 14] INFO [2019-12-17 12:37:47,202] jepsen worker 51 - jepsen.util 11481 :ok :read [17 42] INFO [2019-12-17 12:37:47,203] jepsen worker 31 - jepsen.util 10561 :invoke :read [10 nil] INFO [2019-12-17 12:37:47,203] jepsen worker 51 - jepsen.util 11481 :invoke :write [17 29] INFO [2019-12-17 12:37:47,218] jepsen worker 35 - jepsen.util 16775 :info :cas [11 [39 4]] :RequestCanceledException INFO [2019-12-17 12:37:47,221] jepsen worker 35 - jepsen.util 16865 :invoke :cas [11 [32 19]] INFO [2019-12-17 12:37:47,228] jepsen worker 13 - jepsen.util 11803 :fail :cas [4 [25 11]] :ValueNotSwapFrom INFO [2019-12-17 12:37:47,228] jepsen worker 13 - jepsen.util 11803 :invoke :cas [4 [15 31]] INFO [2019-12-17 12:37:47,231] jepsen worker 31 - jepsen.util 10561 :ok :read [10 14] INFO [2019-12-17 12:37:47,231] jepsen worker 31 - jepsen.util 10561 :invoke :cas [10 [0 19]] INFO [2019-12-17 12:37:47,234] jepsen worker 43 - jepsen.util 17053 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:37:47,234] jepsen worker 16 - jepsen.util 18466 :info :cas [5 [7 35]] :RequestCanceledException INFO [2019-12-17 12:37:47,234] jepsen worker 8 - jepsen.util 16478 :info :write [2 5] :RequestCanceledException INFO [2019-12-17 12:37:47,234] jepsen worker 43 - jepsen.util 17053 :invoke :cas [14 [44 33]] INFO [2019-12-17 12:37:47,235] jepsen worker 8 - jepsen.util 16568 :invoke :read [2 nil] INFO [2019-12-17 12:37:47,235] jepsen worker 16 - jepsen.util 18556 :invoke :read [5 nil] INFO [2019-12-17 12:37:47,245] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71346,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:37:47,245] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71346,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:37:47,245] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71347,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:37:47,245] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71347,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:37:47,245] jepsen worker 64 - jepsen.util 9514 :ok :write [21 17] INFO [2019-12-17 12:37:47,245] jepsen worker 64 - jepsen.util 9514 :invoke :read [21 nil] INFO [2019-12-17 12:37:47,252] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71359,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:37:47,252] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71359,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:37:47,252] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71361,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:37:47,252] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71361,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:37:47,258] jepsen worker 31 - jepsen.util 10561 :fail :cas [10 [0 19]] :ValueNotSwapFrom INFO [2019-12-17 12:37:47,258] jepsen worker 31 - jepsen.util 10561 :invoke :write [10 44] INFO [2019-12-17 12:37:47,277] jepsen worker 19 - jepsen.util 12709 :ok :write [6 23] INFO [2019-12-17 12:37:47,277] jepsen worker 19 - jepsen.util 12709 :invoke :cas [6 [4 6]] INFO [2019-12-17 12:37:47,280] jepsen worker 46 - jepsen.util 12016 :fail :cas [15 [20 7]] :ValueNotSwapFrom INFO [2019-12-17 12:37:47,280] jepsen worker 46 - jepsen.util 12016 :invoke :write [15 4] INFO [2019-12-17 12:37:47,301] jepsen worker 61 - jepsen.util 18331 :info :cas [20 [7 5]] :RequestCanceledException INFO [2019-12-17 12:37:47,301] jepsen worker 44 - jepsen.util 17324 :info :cas [14 [22 47]] :RequestCanceledException INFO [2019-12-17 12:37:47,302] jepsen worker 61 - jepsen.util 18421 :invoke :read [20 nil] INFO [2019-12-17 12:37:47,302] jepsen worker 44 - jepsen.util 17414 :invoke :cas [14 [31 5]] INFO [2019-12-17 12:37:47,314] jepsen worker 28 - jepsen.util 10558 :fail :cas [9 [6 12]] :ValueNotSwapFrom INFO [2019-12-17 12:37:47,314] jepsen worker 28 - jepsen.util 10558 :invoke :read [9 nil] INFO [2019-12-17 12:37:47,334] jepsen worker 81 - jepsen.util 16461 :info :cas [27 [43 3]] :RequestCanceledException INFO [2019-12-17 12:37:47,334] jepsen worker 48 - jepsen.util 15528 :info :write [16 47] :RequestCanceledException INFO [2019-12-17 12:37:47,335] jepsen worker 81 - jepsen.util 16551 :invoke :write [27 34] INFO [2019-12-17 12:37:47,335] jepsen worker 48 - jepsen.util 15618 :invoke :read [16 nil] INFO [2019-12-17 12:37:47,346] jepsen worker 64 - jepsen.util 9514 :ok :read [21 17] INFO [2019-12-17 12:37:47,347] jepsen worker 64 - jepsen.util 9514 :invoke :cas [21 [35 25]] INFO [2019-12-17 12:37:47,348] jepsen worker 15 - jepsen.util 15765 :info :write [5 22] :RequestCanceledException INFO [2019-12-17 12:37:47,349] jepsen worker 15 - jepsen.util 15855 :invoke :write [5 36] INFO [2019-12-17 12:37:47,357] jepsen worker 13 - jepsen.util 11803 :fail :cas [4 [15 31]] :ValueNotSwapFrom INFO [2019-12-17 12:37:47,358] jepsen worker 13 - jepsen.util 11803 :invoke :cas [4 [21 21]] INFO [2019-12-17 12:37:47,367] jepsen worker 28 - jepsen.util 10558 :ok :read [9 36] INFO [2019-12-17 12:37:47,367] jepsen worker 28 - jepsen.util 10558 :invoke :read [9 nil] INFO [2019-12-17 12:37:47,381] jepsen worker 11 - jepsen.util 11801 :ok :read [3 43] INFO [2019-12-17 12:37:47,381] jepsen worker 11 - jepsen.util 11801 :invoke :write [3 27] INFO [2019-12-17 12:37:47,382] jepsen worker 64 - jepsen.util 9514 :fail :cas [21 [35 25]] :ValueNotSwapFrom INFO [2019-12-17 12:37:47,382] jepsen worker 64 - jepsen.util 9514 :invoke :read [21 nil] INFO [2019-12-17 12:37:47,396] jepsen worker 7 - jepsen.util 16837 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:37:47,396] jepsen worker 7 - jepsen.util 16837 :invoke :cas [2 [9 39]] INFO [2019-12-17 12:37:47,398] jepsen worker 10 - jepsen.util 12160 :ok :write [3 45] INFO [2019-12-17 12:37:47,398] jepsen worker 10 - jepsen.util 12160 :invoke :read [3 nil] INFO [2019-12-17 12:37:47,409] jepsen worker 28 - jepsen.util 10558 :ok :read [9 36] INFO [2019-12-17 12:37:47,410] jepsen worker 28 - jepsen.util 10558 :invoke :read [9 nil] INFO [2019-12-17 12:37:47,434] jepsen worker 58 - jepsen.util 16978 :info :write [19 23] :RequestCanceledException INFO [2019-12-17 12:37:47,434] jepsen worker 50 - jepsen.util 16790 :info :write [16 15] :RequestCanceledException INFO [2019-12-17 12:37:47,435] jepsen worker 58 - jepsen.util 17068 :invoke :read [19 nil] INFO [2019-12-17 12:37:47,435] jepsen worker 50 - jepsen.util 16880 :invoke :write [16 44] INFO [2019-12-17 12:37:47,457] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:37:47,457] jepsen nemesis - couchbase.util Rebalance check count: 562 INFO [2019-12-17 12:37:47,457] jepsen nemesis - couchbase.util Rebalance stuck count: 543 INFO [2019-12-17 12:37:47,461] jepsen worker 30 - jepsen.util 11460 :fail :read [10 nil] :RequestTimeoutException INFO [2019-12-17 12:37:47,461] jepsen worker 30 - jepsen.util 11460 :invoke :write [10 17] INFO [2019-12-17 12:37:47,470] jepsen worker 64 - jepsen.util 9514 :ok :read [21 17] INFO [2019-12-17 12:37:47,470] jepsen worker 64 - jepsen.util 9514 :invoke :read [21 nil] INFO [2019-12-17 12:37:47,478] jepsen worker 19 - jepsen.util 12709 :fail :cas [6 [4 6]] :ValueNotSwapFrom INFO [2019-12-17 12:37:47,478] jepsen worker 19 - jepsen.util 12709 :invoke :write [6 1] INFO [2019-12-17 12:37:47,483] jepsen worker 10 - jepsen.util 12160 :ok :read [3 45] INFO [2019-12-17 12:37:47,483] jepsen worker 10 - jepsen.util 12160 :invoke :write [3 24] INFO [2019-12-17 12:37:47,501] jepsen worker 5 - jepsen.util 11885 :info :write [1 26] :RequestCanceledException INFO [2019-12-17 12:37:47,502] jepsen worker 5 - jepsen.util 11975 :invoke :read [1 nil] INFO [2019-12-17 12:37:47,520] jepsen worker 27 - jepsen.util 11277 :info :write [9 40] :RequestCanceledException INFO [2019-12-17 12:37:47,521] jepsen worker 27 - jepsen.util 11367 :invoke :write [9 12] INFO [2019-12-17 12:37:47,524] jepsen worker 65 - jepsen.util 10865 :ok :write [21 7] INFO [2019-12-17 12:37:47,525] jepsen worker 65 - jepsen.util 10865 :invoke :read [21 nil] INFO [2019-12-17 12:37:47,526] jepsen worker 28 - jepsen.util 10558 :ok :read [9 36] INFO [2019-12-17 12:37:47,526] jepsen worker 28 - jepsen.util 10558 :invoke :write [9 0] INFO [2019-12-17 12:37:47,533] jepsen worker 73 - jepsen.util 11323 :info :write [24 47] :RequestCanceledException INFO [2019-12-17 12:37:47,533] jepsen worker 3 - jepsen.util 11613 :info :write [1 46] :RequestCanceledException INFO [2019-12-17 12:37:47,534] jepsen worker 4 - jepsen.util 11434 :info :write [1 38] :RequestCanceledException INFO [2019-12-17 12:37:47,534] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71388,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:37:47,535] jepsen worker 73 - jepsen.util 11413 :invoke :write [24 4] INFO [2019-12-17 12:37:47,535] jepsen worker 4 - jepsen.util 11524 :invoke :write [1 10] INFO [2019-12-17 12:37:47,536] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71388,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:37:47,536] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71391,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:37:47,536] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71391,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:37:47,536] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71392,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:37:47,537] jepsen worker 64 - jepsen.util 9514 :ok :read [21 17] INFO [2019-12-17 12:37:47,537] jepsen worker 3 - jepsen.util 11703 :invoke :read [1 nil] INFO [2019-12-17 12:37:47,537] jepsen worker 64 - jepsen.util 9514 :invoke :read [21 nil] INFO [2019-12-17 12:37:47,537] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71392,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:37:47,544] jepsen worker 5 - jepsen.util 11975 :ok :read [1 40] INFO [2019-12-17 12:37:47,544] jepsen worker 5 - jepsen.util 11975 :invoke :read [1 nil] INFO [2019-12-17 12:37:47,572] jepsen worker 13 - jepsen.util 11803 :fail :cas [4 [21 21]] :ValueNotSwapFrom INFO [2019-12-17 12:37:47,572] jepsen worker 13 - jepsen.util 11803 :invoke :cas [4 [18 44]] INFO [2019-12-17 12:37:47,595] jepsen worker 65 - jepsen.util 10865 :ok :read [21 7] INFO [2019-12-17 12:37:47,596] jepsen worker 65 - jepsen.util 10865 :invoke :read [21 nil] INFO [2019-12-17 12:37:47,600] jepsen worker 18 - jepsen.util 12708 :info :cas [6 [38 35]] :RequestCanceledException INFO [2019-12-17 12:37:47,601] jepsen worker 18 - jepsen.util 12798 :invoke :write [6 17] INFO [2019-12-17 12:37:47,618] jepsen worker 57 - jepsen.util 16527 :info :cas [19 [32 26]] :RequestCanceledException INFO [2019-12-17 12:37:47,619] jepsen worker 57 - jepsen.util 16617 :invoke :cas [19 [25 6]] INFO [2019-12-17 12:37:47,629] jepsen worker 56 - jepsen.util 13286 :ok :read [18 37] INFO [2019-12-17 12:37:47,629] jepsen worker 56 - jepsen.util 13286 :invoke :cas [18 [23 12]] INFO [2019-12-17 12:37:47,658] jepsen worker 27 - jepsen.util 11367 :ok :write [9 12] INFO [2019-12-17 12:37:47,658] jepsen worker 27 - jepsen.util 11367 :invoke :read [9 nil] INFO [2019-12-17 12:37:47,665] jepsen worker 64 - jepsen.util 9514 :ok :read [21 7] INFO [2019-12-17 12:37:47,665] jepsen worker 64 - jepsen.util 9514 :invoke :write [21 20] INFO [2019-12-17 12:37:47,681] jepsen worker 65 - jepsen.util 10865 :ok :read [21 7] INFO [2019-12-17 12:37:47,683] jepsen worker 65 - jepsen.util 10865 :invoke :write [21 18] INFO [2019-12-17 12:37:47,688] jepsen worker 69 - jepsen.util 12579 :ok :read [23 37] INFO [2019-12-17 12:37:47,689] jepsen worker 69 - jepsen.util 12579 :invoke :write [23 39] INFO [2019-12-17 12:37:47,689] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71414,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:37:47,689] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71414,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:37:47,710] jepsen worker 27 - jepsen.util 11367 :ok :read [9 12] INFO [2019-12-17 12:37:47,710] jepsen worker 11 - jepsen.util 11801 :ok :write [3 27] INFO [2019-12-17 12:37:47,710] jepsen worker 27 - jepsen.util 11367 :invoke :cas [9 [32 18]] INFO [2019-12-17 12:37:47,710] jepsen worker 11 - jepsen.util 11801 :invoke :write [3 45] INFO [2019-12-17 12:37:47,718] jepsen worker 22 - jepsen.util 16942 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:37:47,718] jepsen worker 22 - jepsen.util 16942 :invoke :cas [7 [34 12]] INFO [2019-12-17 12:37:47,725] jepsen worker 31 - jepsen.util 10561 :ok :write [10 44] INFO [2019-12-17 12:37:47,725] jepsen worker 31 - jepsen.util 10561 :invoke :read [10 nil] INFO [2019-12-17 12:37:47,732] jepsen worker 42 - jepsen.util 17682 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:37:47,732] jepsen worker 83 - jepsen.util 16913 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:37:47,734] jepsen worker 83 - jepsen.util 16913 :invoke :read [27 nil] INFO [2019-12-17 12:37:47,734] jepsen worker 42 - jepsen.util 17682 :invoke :write [14 24] INFO [2019-12-17 12:37:47,745] jepsen worker 64 - jepsen.util 9514 :ok :write [21 20] INFO [2019-12-17 12:37:47,745] jepsen worker 64 - jepsen.util 9514 :invoke :write [21 36] INFO [2019-12-17 12:37:47,748] jepsen worker 45 - jepsen.util 12105 :info :write [15 34] :RequestCanceledException INFO [2019-12-17 12:37:47,748] jepsen worker 1 - jepsen.util 17101 :info :cas [0 [3 30]] :RequestCanceledException INFO [2019-12-17 12:37:47,749] jepsen worker 45 - jepsen.util 12195 :invoke :read [15 nil] INFO [2019-12-17 12:37:47,752] jepsen worker 1 - jepsen.util 17191 :invoke :write [0 5] INFO [2019-12-17 12:37:47,757] jepsen worker 70 - jepsen.util 12760 :fail :cas [23 [8 8]] :ValueNotSwapFrom INFO [2019-12-17 12:37:47,757] jepsen worker 70 - jepsen.util 12760 :invoke :write [23 17] INFO [2019-12-17 12:37:47,765] jepsen worker 39 - jepsen.util 16419 :info :write [13 18] :RequestCanceledException INFO [2019-12-17 12:37:47,766] jepsen worker 39 - jepsen.util 16509 :invoke :read [13 nil] INFO [2019-12-17 12:37:47,774] jepsen worker 31 - jepsen.util 10561 :ok :read [10 44] INFO [2019-12-17 12:37:47,774] jepsen worker 31 - jepsen.util 10561 :invoke :write [10 39] INFO [2019-12-17 12:37:47,794] jepsen worker 64 - jepsen.util 9514 :ok :write [21 36] INFO [2019-12-17 12:37:47,794] jepsen worker 64 - jepsen.util 9514 :invoke :read [21 nil] INFO [2019-12-17 12:37:47,800] jepsen worker 37 - jepsen.util 11737 :info :write [12 19] :RequestCanceledException INFO [2019-12-17 12:37:47,801] jepsen worker 37 - jepsen.util 11827 :invoke :write [12 15] INFO [2019-12-17 12:37:47,842] jepsen worker 64 - jepsen.util 9514 :ok :read [21 36] INFO [2019-12-17 12:37:47,842] jepsen worker 64 - jepsen.util 9514 :invoke :cas [21 [45 20]] INFO [2019-12-17 12:37:47,847] jepsen worker 5 - jepsen.util 11975 :ok :read [1 40] INFO [2019-12-17 12:37:47,847] jepsen worker 80 - jepsen.util 11780 :info :write [26 20] :RequestCanceledException INFO [2019-12-17 12:37:47,847] jepsen worker 5 - jepsen.util 11975 :invoke :cas [1 [40 33]] INFO [2019-12-17 12:37:47,851] jepsen worker 80 - jepsen.util 11870 :invoke :write [26 36] INFO [2019-12-17 12:37:47,865] jepsen worker 55 - jepsen.util 13015 :info :write [18 15] :RequestCanceledException INFO [2019-12-17 12:37:47,865] jepsen worker 47 - jepsen.util 13277 :info :write [15 31] :RequestCanceledException INFO [2019-12-17 12:37:47,866] jepsen worker 47 - jepsen.util 13367 :invoke :write [15 37] INFO [2019-12-17 12:37:47,866] jepsen worker 55 - jepsen.util 13105 :invoke :write [18 39] INFO [2019-12-17 12:37:47,887] jepsen worker 13 - jepsen.util 11803 :fail :cas [4 [18 44]] :ValueNotSwapFrom INFO [2019-12-17 12:37:47,887] jepsen worker 13 - jepsen.util 11803 :invoke :cas [4 [36 47]] INFO [2019-12-17 12:37:47,902] jepsen worker 64 - jepsen.util 9514 :fail :cas [21 [45 20]] :ValueNotSwapFrom INFO [2019-12-17 12:37:47,902] jepsen worker 64 - jepsen.util 9514 :invoke :cas [21 [7 9]] INFO [2019-12-17 12:37:47,919] jepsen worker 76 - jepsen.util 16996 :info :cas [25 [11 46]] :RequestCanceledException INFO [2019-12-17 12:37:47,920] jepsen worker 76 - jepsen.util 17086 :invoke :read [25 nil] INFO [2019-12-17 12:37:47,928] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71439,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:37:47,928] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71439,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:37:47,928] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71442,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:37:47,928] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71442,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:37:47,928] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71443,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:37:47,928] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71443,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:37:47,934] jepsen worker 49 - jepsen.util 16879 :info :write [16 47] :RequestCanceledException INFO [2019-12-17 12:37:47,934] jepsen worker 72 - jepsen.util 12402 :info :write [24 19] :RequestCanceledException INFO [2019-12-17 12:37:47,935] jepsen worker 49 - jepsen.util 16969 :invoke :read [16 nil] INFO [2019-12-17 12:37:47,935] jepsen worker 72 - jepsen.util 12492 :invoke :read [24 nil] INFO [2019-12-17 12:37:47,943] jepsen worker 12 - jepsen.util 12522 :ok :read [4 37] INFO [2019-12-17 12:37:47,943] jepsen worker 12 - jepsen.util 12522 :invoke :cas [4 [15 33]] INFO [2019-12-17 12:37:47,947] jepsen worker 72 - jepsen.util 12492 :ok :read [24 15] INFO [2019-12-17 12:37:47,947] jepsen worker 72 - jepsen.util 12492 :invoke :read [24 nil] INFO [2019-12-17 12:37:47,948] jepsen worker 68 - jepsen.util 16088 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:37:47,948] jepsen worker 68 - jepsen.util 16088 :invoke :read [22 nil] INFO [2019-12-17 12:37:47,959] jepsen worker 72 - jepsen.util 12492 :ok :read [24 15] INFO [2019-12-17 12:37:47,959] jepsen worker 72 - jepsen.util 12492 :invoke :cas [24 [31 5]] INFO [2019-12-17 12:37:47,965] jepsen worker 6 - jepsen.util 17196 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:37:47,965] jepsen worker 6 - jepsen.util 17196 :invoke :write [2 21] INFO [2019-12-17 12:37:47,974] jepsen worker 72 - jepsen.util 12492 :fail :cas [24 [31 5]] :ValueNotSwapFrom INFO [2019-12-17 12:37:47,974] jepsen worker 72 - jepsen.util 12492 :invoke :read [24 nil] INFO [2019-12-17 12:37:47,992] jepsen worker 72 - jepsen.util 12492 :ok :read [24 15] INFO [2019-12-17 12:37:47,992] jepsen worker 72 - jepsen.util 12492 :invoke :cas [24 [27 44]] INFO [2019-12-17 12:37:48,014] jepsen worker 72 - jepsen.util 12492 :fail :cas [24 [27 44]] :ValueNotSwapFrom INFO [2019-12-17 12:37:48,015] jepsen worker 72 - jepsen.util 12492 :invoke :cas [24 [41 31]] INFO [2019-12-17 12:37:48,026] jepsen worker 72 - jepsen.util 12492 :fail :cas [24 [41 31]] :ValueNotSwapFrom INFO [2019-12-17 12:37:48,026] jepsen worker 72 - jepsen.util 12492 :invoke :cas [24 [40 41]] INFO [2019-12-17 12:37:48,034] jepsen worker 63 - jepsen.util 10773 :info :cas [21 [0 26]] :RequestCanceledException INFO [2019-12-17 12:37:48,035] jepsen worker 41 - jepsen.util 18221 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:37:48,035] jepsen worker 41 - jepsen.util 18221 :invoke :cas [13 [36 36]] INFO [2019-12-17 12:37:48,035] jepsen worker 63 - jepsen.util 10863 :invoke :write [21 11] INFO [2019-12-17 12:37:48,037] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71458,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:37:48,038] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71458,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:37:48,038] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71462,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:37:48,038] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71462,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:37:48,040] jepsen worker 72 - jepsen.util 12492 :fail :cas [24 [40 41]] :ValueNotSwapFrom INFO [2019-12-17 12:37:48,040] jepsen worker 72 - jepsen.util 12492 :invoke :read [24 nil] INFO [2019-12-17 12:37:48,064] jepsen worker 72 - jepsen.util 12492 :ok :read [24 15] INFO [2019-12-17 12:37:48,065] jepsen worker 72 - jepsen.util 12492 :invoke :write [24 6] INFO [2019-12-17 12:37:48,100] jepsen worker 24 - jepsen.util 16044 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:37:48,101] jepsen worker 24 - jepsen.util 16044 :invoke :cas [8 [13 17]] INFO [2019-12-17 12:37:48,118] jepsen worker 40 - jepsen.util 17680 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:37:48,118] jepsen worker 40 - jepsen.util 17680 :invoke :cas [13 [35 29]] INFO [2019-12-17 12:37:48,120] jepsen worker 14 - jepsen.util 11984 :info :write [4 26] :RequestCanceledException INFO [2019-12-17 12:37:48,132] jepsen worker 14 - jepsen.util 12074 :invoke :cas [4 [44 40]] INFO [2019-12-17 12:37:48,134] jepsen worker 34 - jepsen.util 17584 :info :write [11 29] :RequestCanceledException INFO [2019-12-17 12:37:48,135] jepsen worker 34 - jepsen.util 17674 :invoke :cas [11 [1 43]] INFO [2019-12-17 12:37:48,177] jepsen worker 56 - jepsen.util 13286 :fail :cas [18 [23 12]] :ValueNotSwapFrom INFO [2019-12-17 12:37:48,177] jepsen worker 56 - jepsen.util 13286 :invoke :read [18 nil] INFO [2019-12-17 12:37:48,228] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71475,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:37:48,228] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71475,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:37:48,228] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71477,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:37:48,228] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71477,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:37:48,234] jepsen worker 89 - jepsen.util 12059 :info :write [29 0] :RequestCanceledException INFO [2019-12-17 12:37:48,238] jepsen worker 89 - jepsen.util 12149 :invoke :write [29 22] INFO [2019-12-17 12:37:48,239] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71482,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:37:48,239] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71482,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:37:48,239] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71485,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:37:48,239] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71485,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:37:48,239] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71486,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:37:48,239] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71486,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:37:48,244] jepsen worker 13 - jepsen.util 11803 :fail :cas [4 [36 47]] :ValueNotSwapFrom INFO [2019-12-17 12:37:48,245] jepsen worker 13 - jepsen.util 11803 :invoke :write [4 5] INFO [2019-12-17 12:37:48,265] jepsen worker 36 - jepsen.util 12996 :info :write [12 44] :RequestCanceledException INFO [2019-12-17 12:37:48,299] jepsen worker 9 - jepsen.util 12789 :info :write [3 28] :RequestCanceledException INFO [2019-12-17 12:37:48,299] jepsen worker 32 - jepsen.util 12722 :fail :read [10 nil] :RequestTimeoutException INFO [2019-12-17 12:37:48,334] jepsen worker 78 - jepsen.util 11958 :info :write [26 33] :RequestCanceledException INFO [2019-12-17 12:37:48,340] jepsen worker 12 - jepsen.util 12522 :fail :cas [4 [15 33]] :ValueNotSwapFrom INFO [2019-12-17 12:37:48,347] jepsen worker 79 - jepsen.util 12499 :info :write [26 6] :RequestCanceledException INFO [2019-12-17 12:37:48,347] jepsen worker 60 - jepsen.util 17880 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:37:48,418] jepsen worker 62 - jepsen.util 17522 :info :write [20 49] :RequestCanceledException INFO [2019-12-17 12:37:48,434] jepsen worker 17 - jepsen.util 17027 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:37:48,439] jepsen worker 3 - jepsen.util 11703 :ok :read [1 40] INFO [2019-12-17 12:37:48,465] jepsen worker 86 - jepsen.util 17546 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:37:48,465] jepsen worker 75 - jepsen.util 16455 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:37:48,465] jepsen worker 87 - jepsen.util 11877 :info :write [29 46] :RequestCanceledException INFO [2019-12-17 12:37:48,469] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:37:48,478] jepsen worker 32 - jepsen.util 12722 :invoke :cas [10 [29 11]] INFO [2019-12-17 12:37:48,479] jepsen worker 12 - jepsen.util 12522 :invoke :write [4 18] INFO [2019-12-17 12:37:48,479] jepsen worker 60 - jepsen.util 17880 :invoke :write [20 47] INFO [2019-12-17 12:37:48,479] jepsen worker 17 - jepsen.util 17027 :invoke :cas [5 [19 22]] INFO [2019-12-17 12:37:48,479] jepsen worker 3 - jepsen.util 11703 :invoke :read [1 nil] INFO [2019-12-17 12:37:48,479] jepsen worker 86 - jepsen.util 17546 :invoke :write [28 12] INFO [2019-12-17 12:37:48,479] jepsen worker 75 - jepsen.util 16455 :invoke :write [25 44] INFO [2019-12-17 12:37:48,479] jepsen worker 9 - jepsen.util 12879 :invoke :write [3 9] INFO [2019-12-17 12:37:48,479] jepsen nemesis - couchbase.util Rebalance check count: 563 INFO [2019-12-17 12:37:48,480] jepsen worker 78 - jepsen.util 12048 :invoke :read [26 nil] INFO [2019-12-17 12:37:48,480] jepsen worker 79 - jepsen.util 12589 :invoke :cas [26 [24 47]] INFO [2019-12-17 12:37:48,482] jepsen nemesis - couchbase.util Rebalance stuck count: 544 INFO [2019-12-17 12:37:48,482] jepsen worker 36 - jepsen.util 13086 :invoke :cas [12 [27 46]] INFO [2019-12-17 12:37:48,486] jepsen worker 87 - jepsen.util 11967 :invoke :write [29 32] INFO [2019-12-17 12:37:48,499] jepsen worker 26 - jepsen.util 17576 :info :cas [8 [16 21]] :RequestCanceledException INFO [2019-12-17 12:37:48,500] jepsen worker 32 - jepsen.util 12722 :fail :cas [10 [29 11]] :ValueNotSwapFrom INFO [2019-12-17 12:37:48,501] jepsen worker 62 - jepsen.util 17612 :invoke :write [20 46] INFO [2019-12-17 12:37:48,508] jepsen worker 78 - jepsen.util 12048 :ok :read [26 1] INFO [2019-12-17 12:37:48,511] jepsen worker 32 - jepsen.util 12722 :invoke :read [10 nil] INFO [2019-12-17 12:37:48,511] jepsen worker 78 - jepsen.util 12048 :invoke :write [26 49] INFO [2019-12-17 12:37:48,518] jepsen worker 26 - jepsen.util 17666 :invoke :read [8 nil] INFO [2019-12-17 12:37:48,526] jepsen worker 32 - jepsen.util 12722 :ok :read [10 44] INFO [2019-12-17 12:37:48,526] jepsen worker 32 - jepsen.util 12722 :invoke :read [10 nil] INFO [2019-12-17 12:37:48,536] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71491,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:37:48,536] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71491,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:37:48,538] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71495,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:37:48,538] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71495,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:37:48,561] jepsen worker 46 - jepsen.util 12016 :ok :write [15 4] INFO [2019-12-17 12:37:48,561] jepsen worker 46 - jepsen.util 12016 :invoke :write [15 21] INFO [2019-12-17 12:37:48,561] jepsen worker 72 - jepsen.util 12492 :ok :write [24 6] INFO [2019-12-17 12:37:48,562] jepsen worker 19 - jepsen.util 12709 :ok :write [6 1] INFO [2019-12-17 12:37:48,563] jepsen worker 31 - jepsen.util 10561 :ok :write [10 39] INFO [2019-12-17 12:37:48,563] jepsen worker 72 - jepsen.util 12492 :invoke :cas [24 [1 43]] INFO [2019-12-17 12:37:48,563] jepsen worker 19 - jepsen.util 12709 :invoke :cas [6 [7 13]] INFO [2019-12-17 12:37:48,563] jepsen worker 31 - jepsen.util 10561 :invoke :read [10 nil] INFO [2019-12-17 12:37:48,576] jepsen worker 72 - jepsen.util 12492 :fail :cas [24 [1 43]] :ValueNotSwapFrom INFO [2019-12-17 12:37:48,577] jepsen worker 72 - jepsen.util 12492 :invoke :read [24 nil] INFO [2019-12-17 12:37:48,578] jepsen worker 32 - jepsen.util 12722 :ok :read [10 44] INFO [2019-12-17 12:37:48,578] jepsen worker 32 - jepsen.util 12722 :invoke :cas [10 [46 49]] INFO [2019-12-17 12:37:48,623] jepsen worker 31 - jepsen.util 10561 :ok :read [10 39] INFO [2019-12-17 12:37:48,623] jepsen worker 31 - jepsen.util 10561 :invoke :write [10 48] INFO [2019-12-17 12:37:48,638] jepsen worker 72 - jepsen.util 12492 :ok :read [24 6] INFO [2019-12-17 12:37:48,638] jepsen worker 72 - jepsen.util 12492 :invoke :cas [24 [10 39]] INFO [2019-12-17 12:37:48,642] jepsen worker 32 - jepsen.util 12722 :fail :cas [10 [46 49]] :ValueNotSwapFrom INFO [2019-12-17 12:37:48,642] jepsen worker 32 - jepsen.util 12722 :invoke :read [10 nil] INFO [2019-12-17 12:37:48,675] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71522,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:37:48,680] jepsen worker 14 - jepsen.util 12074 :fail :cas [4 [44 40]] :ValueNotSwapFrom INFO [2019-12-17 12:37:48,685] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71522,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:37:48,685] jepsen worker 14 - jepsen.util 12074 :invoke :cas [4 [2 3]] INFO [2019-12-17 12:37:48,685] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71526,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:37:48,685] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71526,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:37:48,685] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71527,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:37:48,685] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71527,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:37:48,699] jepsen worker 82 - jepsen.util 17272 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:37:48,699] jepsen worker 82 - jepsen.util 17272 :invoke :cas [27 [23 19]] INFO [2019-12-17 12:37:48,703] jepsen worker 72 - jepsen.util 12492 :fail :cas [24 [10 39]] :ValueNotSwapFrom INFO [2019-12-17 12:37:48,703] jepsen worker 72 - jepsen.util 12492 :invoke :cas [24 [28 9]] INFO [2019-12-17 12:37:48,711] jepsen worker 32 - jepsen.util 12722 :ok :read [10 39] INFO [2019-12-17 12:37:48,711] jepsen worker 32 - jepsen.util 12722 :invoke :cas [10 [20 36]] INFO [2019-12-17 12:37:48,718] jepsen worker 63 - jepsen.util 10863 :ok :write [21 11] INFO [2019-12-17 12:37:48,718] jepsen worker 63 - jepsen.util 10863 :invoke :write [21 27] INFO [2019-12-17 12:37:48,720] jepsen worker 72 - jepsen.util 12492 :fail :cas [24 [28 9]] :ValueNotSwapFrom INFO [2019-12-17 12:37:48,721] jepsen worker 72 - jepsen.util 12492 :invoke :read [24 nil] INFO [2019-12-17 12:37:48,729] jepsen worker 56 - jepsen.util 13286 :ok :read [18 37] INFO [2019-12-17 12:37:48,729] jepsen worker 56 - jepsen.util 13286 :invoke :read [18 nil] INFO [2019-12-17 12:37:48,734] jepsen worker 33 - jepsen.util 16773 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:37:48,734] jepsen worker 66 - jepsen.util 16896 :info :cas [22 [31 6]] :RequestCanceledException INFO [2019-12-17 12:37:48,734] jepsen worker 84 - jepsen.util 16464 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:37:48,734] jepsen worker 33 - jepsen.util 16773 :invoke :read [11 nil] INFO [2019-12-17 12:37:48,734] jepsen worker 84 - jepsen.util 16464 :invoke :read [28 nil] INFO [2019-12-17 12:37:48,735] jepsen worker 66 - jepsen.util 16986 :invoke :cas [22 [22 25]] INFO [2019-12-17 12:37:48,748] jepsen worker 71 - jepsen.util 13121 :info :write [23 48] :RequestCanceledException INFO [2019-12-17 12:37:48,748] jepsen worker 23 - jepsen.util 17303 :info :cas [7 [7 45]] :RequestCanceledException INFO [2019-12-17 12:37:48,748] jepsen worker 85 - jepsen.util 17455 :info :cas [28 [38 29]] :RequestCanceledException INFO [2019-12-17 12:37:48,749] jepsen worker 71 - jepsen.util 13211 :invoke :read [23 nil] INFO [2019-12-17 12:37:48,754] jepsen worker 23 - jepsen.util 17393 :invoke :read [7 nil] INFO [2019-12-17 12:37:48,759] jepsen worker 85 - jepsen.util 17545 :invoke :read [28 nil] INFO [2019-12-17 12:37:48,764] jepsen worker 72 - jepsen.util 12492 :ok :read [24 6] INFO [2019-12-17 12:37:48,764] jepsen worker 72 - jepsen.util 12492 :invoke :write [24 21] INFO [2019-12-17 12:37:48,764] jepsen worker 20 - jepsen.util 12080 :info :write [6 43] :RequestCanceledException INFO [2019-12-17 12:37:48,764] jepsen worker 67 - jepsen.util 17077 :info :cas [22 [29 38]] :RequestCanceledException INFO [2019-12-17 12:37:48,765] jepsen worker 20 - jepsen.util 12170 :invoke :read [6 nil] INFO [2019-12-17 12:37:48,765] jepsen worker 67 - jepsen.util 17167 :invoke :read [22 nil] INFO [2019-12-17 12:37:48,779] jepsen worker 32 - jepsen.util 12722 :fail :cas [10 [20 36]] :ValueNotSwapFrom INFO [2019-12-17 12:37:48,779] jepsen worker 32 - jepsen.util 12722 :invoke :write [10 19] INFO [2019-12-17 12:37:48,789] jepsen worker 19 - jepsen.util 12709 :fail :cas [6 [7 13]] :ValueNotSwapFrom INFO [2019-12-17 12:37:48,790] jepsen worker 19 - jepsen.util 12709 :invoke :cas [6 [46 28]] INFO [2019-12-17 12:37:48,818] jepsen worker 88 - jepsen.util 12508 :info :write [29 34] :RequestCanceledException INFO [2019-12-17 12:37:48,818] jepsen worker 52 - jepsen.util 11752 :info :write [17 9] :RequestCanceledException INFO [2019-12-17 12:37:48,822] jepsen worker 3 - jepsen.util 11703 :ok :read [1 40] INFO [2019-12-17 12:37:48,824] jepsen worker 3 - jepsen.util 11703 :invoke :cas [1 [8 34]] INFO [2019-12-17 12:37:48,824] jepsen worker 88 - jepsen.util 12598 :invoke :read [29 nil] INFO [2019-12-17 12:37:48,824] jepsen worker 52 - jepsen.util 11842 :invoke :read [17 nil] INFO [2019-12-17 12:37:48,834] jepsen worker 0 - jepsen.util 17460 :info :cas [0 [26 0]] :RequestCanceledException INFO [2019-12-17 12:37:48,835] jepsen worker 0 - jepsen.util 17550 :invoke :read [0 nil] INFO [2019-12-17 12:37:48,865] jepsen worker 29 - jepsen.util 10739 :info :cas [9 [6 49]] :RequestCanceledException INFO [2019-12-17 12:37:48,867] jepsen worker 29 - jepsen.util 10829 :invoke :cas [9 [49 25]] INFO [2019-12-17 12:37:48,885] jepsen worker 79 - jepsen.util 12589 :fail :cas [26 [24 47]] :ValueNotSwapFrom INFO [2019-12-17 12:37:48,885] jepsen worker 79 - jepsen.util 12589 :invoke :read [26 nil] INFO [2019-12-17 12:37:48,900] jepsen worker 21 - jepsen.util 16401 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:37:48,900] jepsen worker 21 - jepsen.util 16401 :invoke :cas [7 [33 32]] INFO [2019-12-17 12:37:48,916] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71551,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:37:48,917] jepsen worker 52 - jepsen.util 11842 :ok :read [17 42] INFO [2019-12-17 12:37:48,925] jepsen worker 63 - jepsen.util 10863 :ok :write [21 27] INFO [2019-12-17 12:37:48,933] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71551,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:37:48,933] jepsen worker 52 - jepsen.util 11842 :invoke :write [17 25] INFO [2019-12-17 12:37:48,933] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71555,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:37:48,933] jepsen worker 63 - jepsen.util 10863 :invoke :write [21 47] INFO [2019-12-17 12:37:48,934] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71555,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:37:48,987] jepsen worker 53 - jepsen.util 11933 :ok :write [17 25] INFO [2019-12-17 12:37:48,987] jepsen worker 53 - jepsen.util 11933 :invoke :read [17 nil] INFO [2019-12-17 12:37:48,995] jepsen worker 63 - jepsen.util 10863 :ok :write [21 47] INFO [2019-12-17 12:37:48,995] jepsen worker 63 - jepsen.util 10863 :invoke :read [21 nil] INFO [2019-12-17 12:37:49,000] jepsen worker 54 - jepsen.util 12924 :ok :write [18 9] INFO [2019-12-17 12:37:49,000] jepsen worker 54 - jepsen.util 12924 :invoke :cas [18 [43 49]] INFO [2019-12-17 12:37:49,020] jepsen worker 46 - jepsen.util 12016 :ok :write [15 21] INFO [2019-12-17 12:37:49,020] jepsen worker 63 - jepsen.util 10863 :ok :read [21 47] INFO [2019-12-17 12:37:49,023] jepsen worker 63 - jepsen.util 10863 :invoke :cas [21 [3 2]] INFO [2019-12-17 12:37:49,023] jepsen worker 46 - jepsen.util 12016 :invoke :cas [15 [38 20]] INFO [2019-12-17 12:37:49,024] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71564,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:37:49,024] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71564,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:37:49,024] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71566,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:37:49,024] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71566,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:37:49,043] jepsen worker 20 - jepsen.util 12170 :ok :read [6 1] INFO [2019-12-17 12:37:49,043] jepsen worker 20 - jepsen.util 12170 :invoke :write [6 44] INFO [2019-12-17 12:37:49,062] jepsen worker 46 - jepsen.util 12016 :fail :cas [15 [38 20]] :ValueNotSwapFrom INFO [2019-12-17 12:37:49,062] jepsen worker 46 - jepsen.util 12016 :invoke :write [15 6] INFO [2019-12-17 12:37:49,079] jepsen worker 31 - jepsen.util 10561 :ok :write [10 48] INFO [2019-12-17 12:37:49,079] jepsen worker 31 - jepsen.util 10561 :invoke :cas [10 [43 11]] INFO [2019-12-17 12:37:49,082] jepsen worker 3 - jepsen.util 11703 :fail :cas [1 [8 34]] :ValueNotSwapFrom INFO [2019-12-17 12:37:49,082] jepsen worker 3 - jepsen.util 11703 :invoke :cas [1 [2 48]] INFO [2019-12-17 12:37:49,089] jepsen worker 63 - jepsen.util 10863 :fail :cas [21 [3 2]] :ValueNotSwapFrom INFO [2019-12-17 12:37:49,089] jepsen worker 63 - jepsen.util 10863 :invoke :write [21 42] INFO [2019-12-17 12:37:49,095] jepsen worker 31 - jepsen.util 10561 :fail :cas [10 [43 11]] :ValueNotSwapFrom INFO [2019-12-17 12:37:49,095] jepsen worker 31 - jepsen.util 10561 :invoke :cas [10 [14 15]] INFO [2019-12-17 12:37:49,100] jepsen worker 77 - jepsen.util 18077 :info :write [25 34] :RequestCanceledException INFO [2019-12-17 12:37:49,118] jepsen worker 31 - jepsen.util 10561 :fail :cas [10 [14 15]] :ValueNotSwapFrom INFO [2019-12-17 12:37:49,118] jepsen worker 59 - jepsen.util 16079 :info :cas [19 [6 38]] :RequestCanceledException INFO [2019-12-17 12:37:49,118] jepsen worker 25 - jepsen.util 17935 :info :write [8 30] :RequestCanceledException INFO [2019-12-17 12:37:49,127] jepsen worker 31 - jepsen.util 10561 :invoke :read [10 nil] INFO [2019-12-17 12:37:49,128] jepsen worker 59 - jepsen.util 16169 :invoke :read [19 nil] INFO [2019-12-17 12:37:49,128] jepsen worker 77 - jepsen.util 18167 :invoke :read [25 nil] INFO [2019-12-17 12:37:49,131] jepsen worker 88 - jepsen.util 12598 :ok :read [29 35] INFO [2019-12-17 12:37:49,131] jepsen worker 88 - jepsen.util 12598 :invoke :read [29 nil] INFO [2019-12-17 12:37:49,139] jepsen worker 25 - jepsen.util 18025 :invoke :write [8 43] INFO [2019-12-17 12:37:49,148] jepsen worker 2 - jepsen.util 16202 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:37:49,148] jepsen worker 2 - jepsen.util 16202 :invoke :cas [0 [7 42]] INFO [2019-12-17 12:37:49,167] jepsen worker 74 - jepsen.util 12044 :info :cas [24 [27 4]] :RequestCanceledException INFO [2019-12-17 12:37:49,168] jepsen worker 74 - jepsen.util 12134 :invoke :read [24 nil] INFO [2019-12-17 12:37:49,172] jepsen worker 71 - jepsen.util 13211 :ok :read [23 37] INFO [2019-12-17 12:37:49,172] jepsen worker 71 - jepsen.util 13211 :invoke :read [23 nil] INFO [2019-12-17 12:37:49,172] jepsen worker 38 - jepsen.util 12818 :info :write [12 26] :RequestCanceledException INFO [2019-12-17 12:37:49,176] jepsen worker 63 - jepsen.util 10863 :ok :write [21 42] INFO [2019-12-17 12:37:49,176] jepsen worker 63 - jepsen.util 10863 :invoke :write [21 26] INFO [2019-12-17 12:37:49,185] jepsen worker 38 - jepsen.util 12908 :invoke :cas [12 [39 33]] INFO [2019-12-17 12:37:49,200] jepsen worker 74 - jepsen.util 12134 :ok :read [24 6] INFO [2019-12-17 12:37:49,200] jepsen worker 74 - jepsen.util 12134 :invoke :read [24 nil] INFO [2019-12-17 12:37:49,203] jepsen worker 72 - jepsen.util 12492 :ok :write [24 21] INFO [2019-12-17 12:37:49,203] jepsen worker 72 - jepsen.util 12492 :invoke :write [24 24] INFO [2019-12-17 12:37:49,204] jepsen worker 20 - jepsen.util 12170 :ok :write [6 44] INFO [2019-12-17 12:37:49,205] jepsen worker 20 - jepsen.util 12170 :invoke :read [6 nil] INFO [2019-12-17 12:37:49,206] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71590,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:37:49,206] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71590,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:37:49,206] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71591,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:37:49,206] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71591,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:37:49,213] jepsen worker 31 - jepsen.util 10561 :ok :read [10 48] INFO [2019-12-17 12:37:49,213] jepsen worker 31 - jepsen.util 10561 :invoke :write [10 35] INFO [2019-12-17 12:37:49,214] jepsen worker 79 - jepsen.util 12589 :ok :read [26 1] INFO [2019-12-17 12:37:49,214] jepsen worker 79 - jepsen.util 12589 :invoke :read [26 nil] INFO [2019-12-17 12:37:49,222] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71600,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:37:49,222] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71600,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:37:49,222] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71603,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:37:49,222] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71603,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:37:49,234] jepsen worker 51 - jepsen.util 11481 :info :write [17 29] :RequestCanceledException INFO [2019-12-17 12:37:49,235] jepsen worker 20 - jepsen.util 12170 :ok :read [6 44] INFO [2019-12-17 12:37:49,237] jepsen worker 20 - jepsen.util 12170 :invoke :cas [6 [48 12]] INFO [2019-12-17 12:37:49,238] jepsen worker 51 - jepsen.util 11571 :invoke :read [17 nil] INFO [2019-12-17 12:37:49,245] jepsen worker 88 - jepsen.util 12598 :ok :read [29 35] INFO [2019-12-17 12:37:49,245] jepsen worker 88 - jepsen.util 12598 :invoke :read [29 nil] INFO [2019-12-17 12:37:49,248] jepsen worker 16 - jepsen.util 18556 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:37:49,250] jepsen worker 16 - jepsen.util 18556 :invoke :cas [5 [22 29]] INFO [2019-12-17 12:37:49,265] jepsen worker 63 - jepsen.util 10863 :ok :write [21 26] INFO [2019-12-17 12:37:49,265] jepsen worker 63 - jepsen.util 10863 :invoke :cas [21 [45 13]] INFO [2019-12-17 12:37:49,279] jepsen worker 20 - jepsen.util 12170 :fail :cas [6 [48 12]] :ValueNotSwapFrom INFO [2019-12-17 12:37:49,279] jepsen worker 20 - jepsen.util 12170 :invoke :cas [6 [19 26]] INFO [2019-12-17 12:37:49,295] jepsen worker 63 - jepsen.util 10863 :fail :cas [21 [45 13]] :ValueNotSwapFrom INFO [2019-12-17 12:37:49,295] jepsen worker 63 - jepsen.util 10863 :invoke :write [21 12] INFO [2019-12-17 12:37:49,308] jepsen worker 20 - jepsen.util 12170 :fail :cas [6 [19 26]] :ValueNotSwapFrom INFO [2019-12-17 12:37:49,308] jepsen worker 20 - jepsen.util 12170 :invoke :read [6 nil] INFO [2019-12-17 12:37:49,318] jepsen worker 35 - jepsen.util 16865 :info :cas [11 [32 19]] :RequestCanceledException INFO [2019-12-17 12:37:49,319] jepsen worker 35 - jepsen.util 16955 :invoke :cas [11 [30 13]] INFO [2019-12-17 12:37:49,322] jepsen worker 72 - jepsen.util 12492 :ok :write [24 24] INFO [2019-12-17 12:37:49,322] jepsen worker 72 - jepsen.util 12492 :invoke :read [24 nil] INFO [2019-12-17 12:37:49,330] jepsen worker 20 - jepsen.util 12170 :ok :read [6 44] INFO [2019-12-17 12:37:49,330] jepsen worker 20 - jepsen.util 12170 :invoke :read [6 nil] INFO [2019-12-17 12:37:49,331] jepsen worker 36 - jepsen.util 13086 :fail :cas [12 [27 46]] :ValueNotSwapFrom INFO [2019-12-17 12:37:49,333] jepsen worker 44 - jepsen.util 17414 :info :cas [14 [31 5]] :RequestCanceledException INFO [2019-12-17 12:37:49,333] jepsen worker 43 - jepsen.util 17053 :info :cas [14 [44 33]] :RequestCanceledException INFO [2019-12-17 12:37:49,334] jepsen worker 8 - jepsen.util 16568 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:37:49,334] jepsen worker 8 - jepsen.util 16568 :invoke :cas [2 [25 12]] INFO [2019-12-17 12:37:49,334] jepsen worker 44 - jepsen.util 17504 :invoke :write [14 44] INFO [2019-12-17 12:37:49,336] jepsen worker 36 - jepsen.util 13086 :invoke :cas [12 [38 21]] INFO [2019-12-17 12:37:49,336] jepsen worker 43 - jepsen.util 17143 :invoke :write [14 25] INFO [2019-12-17 12:37:49,365] jepsen worker 61 - jepsen.util 18421 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:37:49,366] jepsen worker 61 - jepsen.util 18421 :invoke :write [20 12] INFO [2019-12-17 12:37:49,379] jepsen worker 14 - jepsen.util 12074 :fail :cas [4 [2 3]] :ValueNotSwapFrom INFO [2019-12-17 12:37:49,379] jepsen worker 14 - jepsen.util 12074 :invoke :write [4 12] INFO [2019-12-17 12:37:49,401] jepsen worker 81 - jepsen.util 16551 :info :write [27 34] :RequestCanceledException INFO [2019-12-17 12:37:49,402] jepsen worker 81 - jepsen.util 16641 :invoke :cas [27 [47 49]] INFO [2019-12-17 12:37:49,411] jepsen worker 3 - jepsen.util 11703 :fail :cas [1 [2 48]] :ValueNotSwapFrom INFO [2019-12-17 12:37:49,411] jepsen worker 3 - jepsen.util 11703 :invoke :write [1 12] INFO [2019-12-17 12:37:49,413] jepsen worker 72 - jepsen.util 12492 :ok :read [24 24] INFO [2019-12-17 12:37:49,414] jepsen worker 72 - jepsen.util 12492 :invoke :cas [24 [32 33]] INFO [2019-12-17 12:37:49,418] jepsen worker 48 - jepsen.util 15618 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:37:49,419] jepsen worker 48 - jepsen.util 15618 :invoke :write [16 23] INFO [2019-12-17 12:37:49,421] jepsen worker 20 - jepsen.util 12170 :ok :read [6 44] INFO [2019-12-17 12:37:49,421] jepsen worker 20 - jepsen.util 12170 :invoke :read [6 nil] INFO [2019-12-17 12:37:49,434] jepsen worker 53 - jepsen.util 11933 :ok :read [17 25] INFO [2019-12-17 12:37:49,434] jepsen worker 15 - jepsen.util 15855 :info :write [5 36] :RequestCanceledException INFO [2019-12-17 12:37:49,446] jepsen worker 63 - jepsen.util 10863 :ok :write [21 12] INFO [2019-12-17 12:37:49,447] jepsen worker 72 - jepsen.util 12492 :fail :cas [24 [32 33]] :ValueNotSwapFrom INFO [2019-12-17 12:37:49,448] jepsen worker 58 - jepsen.util 17068 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:37:49,462] jepsen worker 88 - jepsen.util 12598 :ok :read [29 35] INFO [2019-12-17 12:37:49,464] jepsen worker 20 - jepsen.util 12170 :ok :read [6 44] INFO [2019-12-17 12:37:49,465] jepsen worker 50 - jepsen.util 16880 :info :write [16 44] :RequestCanceledException INFO [2019-12-17 12:37:49,469] jepsen worker 53 - jepsen.util 11933 :invoke :cas [17 [42 41]] INFO [2019-12-17 12:37:49,469] jepsen worker 63 - jepsen.util 10863 :invoke :write [21 27] INFO [2019-12-17 12:37:49,469] jepsen worker 72 - jepsen.util 12492 :invoke :cas [24 [18 35]] INFO [2019-12-17 12:37:49,469] jepsen worker 58 - jepsen.util 17068 :invoke :cas [19 [7 6]] INFO [2019-12-17 12:37:49,469] jepsen worker 88 - jepsen.util 12598 :invoke :cas [29 [45 7]] INFO [2019-12-17 12:37:49,469] jepsen worker 20 - jepsen.util 12170 :invoke :cas [6 [3 23]] INFO [2019-12-17 12:37:49,473] jepsen worker 50 - jepsen.util 16970 :invoke :write [16 0] INFO [2019-12-17 12:37:49,477] jepsen worker 15 - jepsen.util 15945 :invoke :cas [5 [23 42]] INFO [2019-12-17 12:37:49,477] jepsen worker 31 - jepsen.util 10561 :ok :write [10 35] INFO [2019-12-17 12:37:49,477] jepsen worker 31 - jepsen.util 10561 :invoke :write [10 6] INFO [2019-12-17 12:37:49,488] jepsen worker 38 - jepsen.util 12908 :fail :cas [12 [39 33]] :ValueNotSwapFrom INFO [2019-12-17 12:37:49,489] jepsen worker 38 - jepsen.util 12908 :invoke :cas [12 [1 24]] INFO [2019-12-17 12:37:49,501] jepsen worker 7 - jepsen.util 16837 :info :cas [2 [9 39]] :RequestCanceledException INFO [2019-12-17 12:37:49,509] jepsen worker 72 - jepsen.util 12492 :fail :cas [24 [18 35]] :ValueNotSwapFrom INFO [2019-12-17 12:37:49,523] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:37:49,524] jepsen worker 51 - jepsen.util 11571 :ok :read [17 25] INFO [2019-12-17 12:37:49,527] jepsen worker 20 - jepsen.util 12170 :fail :cas [6 [3 23]] :ValueNotSwapFrom INFO [2019-12-17 12:37:49,530] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71630,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:37:49,532] jepsen worker 72 - jepsen.util 12492 :invoke :read [24 nil] INFO [2019-12-17 12:37:49,532] jepsen nemesis - couchbase.util Rebalance check count: 564 INFO [2019-12-17 12:37:49,532] jepsen worker 51 - jepsen.util 11571 :invoke :read [17 nil] INFO [2019-12-17 12:37:49,533] jepsen worker 20 - jepsen.util 12170 :invoke :cas [6 [47 35]] INFO [2019-12-17 12:37:49,533] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71630,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:37:49,533] jepsen nemesis - couchbase.util Rebalance stuck count: 545 INFO [2019-12-17 12:37:49,533] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71632,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:37:49,533] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71632,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:37:49,533] jepsen worker 7 - jepsen.util 16927 :invoke :write [2 14] INFO [2019-12-17 12:37:49,533] jepsen worker 28 - jepsen.util 10558 :info :write [9 0] :RequestCanceledException INFO [2019-12-17 12:37:49,535] jepsen worker 28 - jepsen.util 10648 :invoke :cas [9 [44 32]] INFO [2019-12-17 12:37:49,543] jepsen worker 10 - jepsen.util 12160 :info :write [3 24] :RequestCanceledException INFO [2019-12-17 12:37:49,547] jepsen worker 13 - jepsen.util 11803 :ok :write [4 5] INFO [2019-12-17 12:37:49,547] jepsen worker 13 - jepsen.util 11803 :invoke :write [4 48] INFO [2019-12-17 12:37:49,550] jepsen worker 10 - jepsen.util 12250 :invoke :write [3 48] INFO [2019-12-17 12:37:49,556] jepsen worker 20 - jepsen.util 12170 :fail :cas [6 [47 35]] :ValueNotSwapFrom INFO [2019-12-17 12:37:49,556] jepsen worker 20 - jepsen.util 12170 :invoke :write [6 8] INFO [2019-12-17 12:37:49,563] jepsen worker 30 - jepsen.util 11460 :info :write [10 17] :RequestCanceledException INFO [2019-12-17 12:37:49,564] jepsen worker 30 - jepsen.util 11550 :invoke :write [10 42] INFO [2019-12-17 12:37:49,573] jepsen worker 72 - jepsen.util 12492 :ok :read [24 24] INFO [2019-12-17 12:37:49,576] jepsen worker 72 - jepsen.util 12492 :invoke :read [24 nil] INFO [2019-12-17 12:37:49,588] jepsen worker 12 - jepsen.util 12522 :ok :write [4 18] INFO [2019-12-17 12:37:49,588] jepsen worker 12 - jepsen.util 12522 :invoke :write [4 4] INFO [2019-12-17 12:37:49,592] jepsen worker 51 - jepsen.util 11571 :ok :read [17 25] INFO [2019-12-17 12:37:49,592] jepsen worker 51 - jepsen.util 11571 :invoke :read [17 nil] INFO [2019-12-17 12:37:49,598] jepsen worker 63 - jepsen.util 10863 :ok :write [21 27] INFO [2019-12-17 12:37:49,600] jepsen worker 63 - jepsen.util 10863 :invoke :cas [21 [38 33]] INFO [2019-12-17 12:37:49,609] jepsen worker 51 - jepsen.util 11571 :ok :read [17 25] INFO [2019-12-17 12:37:49,615] jepsen worker 51 - jepsen.util 11571 :invoke :read [17 nil] INFO [2019-12-17 12:37:49,618] jepsen worker 73 - jepsen.util 11413 :info :write [24 4] :RequestCanceledException INFO [2019-12-17 12:37:49,619] jepsen worker 73 - jepsen.util 11503 :invoke :write [24 37] INFO [2019-12-17 12:37:49,619] jepsen worker 31 - jepsen.util 10561 :ok :write [10 6] INFO [2019-12-17 12:37:49,619] jepsen worker 31 - jepsen.util 10561 :invoke :write [10 29] INFO [2019-12-17 12:37:49,634] jepsen worker 28 - jepsen.util 10648 :fail :cas [9 [44 32]] :ValueNotSwapFrom INFO [2019-12-17 12:37:49,634] jepsen worker 57 - jepsen.util 16617 :info :cas [19 [25 6]] :RequestCanceledException INFO [2019-12-17 12:37:49,634] jepsen worker 4 - jepsen.util 11524 :info :write [1 10] :RequestCanceledException INFO [2019-12-17 12:37:49,636] jepsen worker 28 - jepsen.util 10648 :invoke :read [9 nil] INFO [2019-12-17 12:37:49,637] jepsen worker 57 - jepsen.util 16707 :invoke :write [19 28] INFO [2019-12-17 12:37:49,637] jepsen worker 4 - jepsen.util 11614 :invoke :write [1 45] INFO [2019-12-17 12:37:49,664] jepsen worker 89 - jepsen.util 12149 :ok :write [29 22] INFO [2019-12-17 12:37:49,664] jepsen worker 89 - jepsen.util 12149 :invoke :cas [29 [19 46]] INFO [2019-12-17 12:37:49,665] jepsen worker 18 - jepsen.util 12798 :info :write [6 17] :RequestCanceledException INFO [2019-12-17 12:37:49,667] jepsen worker 18 - jepsen.util 12888 :invoke :write [6 10] INFO [2019-12-17 12:37:49,667] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71662,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:37:49,669] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71662,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:37:49,670] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71666,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:37:49,670] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71666,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:37:49,707] jepsen worker 51 - jepsen.util 11571 :ok :read [17 25] INFO [2019-12-17 12:37:49,710] jepsen worker 51 - jepsen.util 11571 :invoke :read [17 nil] INFO [2019-12-17 12:37:49,734] jepsen worker 69 - jepsen.util 12579 :info :write [23 39] :RequestCanceledException INFO [2019-12-17 12:37:49,735] jepsen worker 69 - jepsen.util 12669 :invoke :read [23 nil] INFO [2019-12-17 12:37:49,746] jepsen worker 36 - jepsen.util 13086 :fail :cas [12 [38 21]] :ValueNotSwapFrom INFO [2019-12-17 12:37:49,749] jepsen worker 79 - jepsen.util 12589 :ok :read [26 1] INFO [2019-12-17 12:37:49,750] jepsen worker 36 - jepsen.util 13086 :invoke :write [12 5] INFO [2019-12-17 12:37:49,750] jepsen worker 79 - jepsen.util 12589 :invoke :cas [26 [28 34]] INFO [2019-12-17 12:37:49,765] jepsen worker 11 - jepsen.util 11801 :info :write [3 45] :RequestCanceledException INFO [2019-12-17 12:37:49,767] jepsen worker 88 - jepsen.util 12598 :fail :cas [29 [45 7]] :ValueNotSwapFrom INFO [2019-12-17 12:37:49,770] jepsen worker 88 - jepsen.util 12598 :invoke :cas [29 [33 42]] INFO [2019-12-17 12:37:49,771] jepsen worker 11 - jepsen.util 11891 :invoke :read [3 nil] INFO [2019-12-17 12:37:49,791] jepsen worker 11 - jepsen.util 11891 :ok :read [3 27] INFO [2019-12-17 12:37:49,795] jepsen worker 11 - jepsen.util 11891 :invoke :write [3 26] INFO [2019-12-17 12:37:49,801] jepsen worker 27 - jepsen.util 11367 :info :cas [9 [32 18]] :RequestCanceledException INFO [2019-12-17 12:37:49,801] jepsen worker 65 - jepsen.util 10865 :info :write [21 18] :RequestCanceledException INFO [2019-12-17 12:37:49,801] jepsen worker 45 - jepsen.util 12195 :fail :read [15 nil] :RequestTimeoutException INFO [2019-12-17 12:37:49,804] jepsen worker 27 - jepsen.util 11457 :invoke :cas [9 [23 36]] INFO [2019-12-17 12:37:49,805] jepsen worker 45 - jepsen.util 12195 :invoke :write [15 14] INFO [2019-12-17 12:37:49,811] jepsen worker 65 - jepsen.util 10955 :invoke :cas [21 [26 17]] INFO [2019-12-17 12:37:49,818] jepsen worker 22 - jepsen.util 16942 :info :cas [7 [34 12]] :RequestCanceledException INFO [2019-12-17 12:37:49,818] jepsen worker 1 - jepsen.util 17191 :info :write [0 5] :RequestCanceledException INFO [2019-12-17 12:37:49,823] jepsen worker 1 - jepsen.util 17281 :invoke :read [0 nil] INFO [2019-12-17 12:37:49,832] jepsen worker 22 - jepsen.util 17032 :invoke :write [7 3] INFO [2019-12-17 12:37:49,833] jepsen worker 83 - jepsen.util 16913 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:37:49,833] jepsen worker 42 - jepsen.util 17682 :info :write [14 24] :RequestCanceledException INFO [2019-12-17 12:37:49,833] jepsen worker 39 - jepsen.util 16509 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:37:49,833] jepsen worker 70 - jepsen.util 12760 :info :write [23 17] :RequestCanceledException INFO [2019-12-17 12:37:49,838] jepsen worker 83 - jepsen.util 16913 :invoke :read [27 nil] INFO [2019-12-17 12:37:49,840] jepsen worker 39 - jepsen.util 16509 :invoke :write [13 38] INFO [2019-12-17 12:37:49,841] jepsen worker 70 - jepsen.util 12850 :invoke :cas [23 [25 44]] INFO [2019-12-17 12:37:49,843] jepsen worker 65 - jepsen.util 10955 :fail :cas [21 [26 17]] :ValueNotSwapFrom INFO [2019-12-17 12:37:49,844] jepsen worker 65 - jepsen.util 10955 :invoke :write [21 6] INFO [2019-12-17 12:37:49,845] jepsen worker 42 - jepsen.util 17772 :invoke :write [14 34] INFO [2019-12-17 12:37:49,848] jepsen worker 37 - jepsen.util 11827 :info :write [12 15] :RequestCanceledException INFO [2019-12-17 12:37:49,850] jepsen worker 37 - jepsen.util 11917 :invoke :cas [12 [23 37]] INFO [2019-12-17 12:37:49,871] jepsen worker 20 - jepsen.util 12170 :ok :write [6 8] INFO [2019-12-17 12:37:49,871] jepsen worker 10 - jepsen.util 12250 :ok :write [3 48] INFO [2019-12-17 12:37:49,872] jepsen worker 20 - jepsen.util 12170 :invoke :write [6 31] INFO [2019-12-17 12:37:49,872] jepsen worker 10 - jepsen.util 12250 :invoke :read [3 nil] INFO [2019-12-17 12:37:49,895] jepsen worker 56 - jepsen.util 13286 :ok :read [18 9] INFO [2019-12-17 12:37:49,896] jepsen worker 56 - jepsen.util 13286 :invoke :read [18 nil] INFO [2019-12-17 12:37:49,896] jepsen worker 55 - jepsen.util 13105 :info :write [18 39] :RequestCanceledException INFO [2019-12-17 12:37:49,903] jepsen worker 10 - jepsen.util 12250 :ok :read [3 48] INFO [2019-12-17 12:37:49,903] jepsen worker 10 - jepsen.util 12250 :invoke :cas [3 [32 4]] INFO [2019-12-17 12:37:49,915] jepsen worker 55 - jepsen.util 13195 :invoke :write [18 32] INFO [2019-12-17 12:37:49,925] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71689,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:37:49,925] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71689,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:37:49,925] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71693,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:37:49,925] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71693,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:37:49,934] jepsen worker 47 - jepsen.util 13367 :info :write [15 37] :RequestCanceledException INFO [2019-12-17 12:37:49,934] jepsen worker 64 - jepsen.util 9514 :info :cas [21 [7 9]] :RequestCanceledException INFO [2019-12-17 12:37:49,935] jepsen worker 47 - jepsen.util 13457 :invoke :read [15 nil] INFO [2019-12-17 12:37:49,935] jepsen worker 64 - jepsen.util 9604 :invoke :write [21 11] INFO [2019-12-17 12:37:49,941] jepsen worker 10 - jepsen.util 12250 :fail :cas [3 [32 4]] :ValueNotSwapFrom INFO [2019-12-17 12:37:49,941] jepsen worker 10 - jepsen.util 12250 :invoke :read [3 nil] INFO [2019-12-17 12:37:49,964] jepsen worker 80 - jepsen.util 11870 :info :write [26 36] :RequestCanceledException INFO [2019-12-17 12:37:49,965] jepsen worker 80 - jepsen.util 11960 :invoke :cas [26 [13 39]] INFO [2019-12-17 12:37:49,980] jepsen worker 10 - jepsen.util 12250 :ok :read [3 48] INFO [2019-12-17 12:37:49,985] jepsen worker 31 - jepsen.util 10561 :ok :write [10 29] INFO [2019-12-17 12:37:49,990] jepsen worker 73 - jepsen.util 11503 :ok :write [24 37] INFO [2019-12-17 12:37:49,992] jepsen worker 37 - jepsen.util 11917 :fail :cas [12 [23 37]] :ValueNotSwapFrom INFO [2019-12-17 12:37:49,993] jepsen worker 70 - jepsen.util 12850 :fail :cas [23 [25 44]] :ValueNotSwapFrom INFO [2019-12-17 12:37:49,995] jepsen worker 10 - jepsen.util 12250 :invoke :write [3 20] INFO [2019-12-17 12:37:49,995] jepsen worker 31 - jepsen.util 10561 :invoke :read [10 nil] INFO [2019-12-17 12:37:49,995] jepsen worker 73 - jepsen.util 11503 :invoke :write [24 3] INFO [2019-12-17 12:37:49,995] jepsen worker 37 - jepsen.util 11917 :invoke :cas [12 [44 17]] INFO [2019-12-17 12:37:49,995] jepsen worker 70 - jepsen.util 12850 :invoke :read [23 nil] INFO [2019-12-17 12:37:50,006] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71708,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:37:50,006] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71708,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:37:50,016] jepsen worker 76 - jepsen.util 17086 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:37:50,016] jepsen worker 76 - jepsen.util 17086 :invoke :write [25 2] INFO [2019-12-17 12:37:50,023] jepsen worker 28 - jepsen.util 10648 :ok :read [9 12] INFO [2019-12-17 12:37:50,023] jepsen worker 28 - jepsen.util 10648 :invoke :read [9 nil] INFO [2019-12-17 12:37:50,033] jepsen worker 49 - jepsen.util 16969 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:37:50,034] jepsen worker 49 - jepsen.util 16969 :invoke :read [16 nil] INFO [2019-12-17 12:37:50,037] jepsen worker 4 - jepsen.util 11614 :ok :write [1 45] INFO [2019-12-17 12:37:50,037] jepsen worker 4 - jepsen.util 11614 :invoke :read [1 nil] INFO [2019-12-17 12:37:50,047] jepsen worker 68 - jepsen.util 16088 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:37:50,047] jepsen worker 68 - jepsen.util 16088 :invoke :cas [22 [10 2]] INFO [2019-12-17 12:37:50,053] jepsen worker 27 - jepsen.util 11457 :fail :cas [9 [23 36]] :ValueNotSwapFrom INFO [2019-12-17 12:37:50,054] jepsen worker 27 - jepsen.util 11457 :invoke :write [9 41] INFO [2019-12-17 12:37:50,065] jepsen worker 6 - jepsen.util 17196 :info :write [2 21] :RequestCanceledException INFO [2019-12-17 12:37:50,066] jepsen worker 6 - jepsen.util 17286 :invoke :cas [2 [30 4]] INFO [2019-12-17 12:37:50,068] jepsen worker 31 - jepsen.util 10561 :ok :read [10 29] INFO [2019-12-17 12:37:50,068] jepsen worker 31 - jepsen.util 10561 :invoke :write [10 22] INFO [2019-12-17 12:37:50,081] jepsen worker 28 - jepsen.util 10648 :ok :read [9 12] INFO [2019-12-17 12:37:50,082] jepsen worker 28 - jepsen.util 10648 :invoke :read [9 nil] INFO [2019-12-17 12:37:50,082] jepsen worker 45 - jepsen.util 12195 :ok :write [15 14] INFO [2019-12-17 12:37:50,085] jepsen worker 45 - jepsen.util 12195 :invoke :write [15 46] INFO [2019-12-17 12:37:50,091] jepsen worker 12 - jepsen.util 12522 :ok :write [4 4] INFO [2019-12-17 12:37:50,092] jepsen worker 12 - jepsen.util 12522 :invoke :cas [4 [21 37]] INFO [2019-12-17 12:37:50,105] jepsen worker 28 - jepsen.util 10648 :ok :read [9 12] INFO [2019-12-17 12:37:50,105] jepsen worker 28 - jepsen.util 10648 :invoke :read [9 nil] INFO [2019-12-17 12:37:50,110] jepsen worker 20 - jepsen.util 12170 :ok :write [6 31] INFO [2019-12-17 12:37:50,110] jepsen worker 20 - jepsen.util 12170 :invoke :write [6 4] INFO [2019-12-17 12:37:50,117] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71712,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:37:50,117] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71712,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:37:50,118] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71713,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:37:50,118] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71713,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:37:50,127] jepsen worker 65 - jepsen.util 10955 :ok :write [21 6] INFO [2019-12-17 12:37:50,127] jepsen worker 65 - jepsen.util 10955 :invoke :read [21 nil] INFO [2019-12-17 12:37:50,132] jepsen worker 89 - jepsen.util 12149 :fail :cas [29 [19 46]] :ValueNotSwapFrom INFO [2019-12-17 12:37:50,134] jepsen worker 41 - jepsen.util 18221 :info :cas [13 [36 36]] :RequestCanceledException INFO [2019-12-17 12:37:50,135] jepsen worker 89 - jepsen.util 12149 :invoke :cas [29 [46 47]] INFO [2019-12-17 12:37:50,149] jepsen worker 41 - jepsen.util 18311 :invoke :cas [13 [45 14]] INFO [2019-12-17 12:37:50,173] jepsen worker 47 - jepsen.util 13457 :ok :read [15 14] INFO [2019-12-17 12:37:50,173] jepsen worker 47 - jepsen.util 13457 :invoke :write [15 21] INFO [2019-12-17 12:37:50,180] jepsen worker 53 - jepsen.util 11933 :fail :cas [17 [42 41]] :ValueNotSwapFrom INFO [2019-12-17 12:37:50,180] jepsen worker 53 - jepsen.util 11933 :invoke :read [17 nil] INFO [2019-12-17 12:37:50,200] jepsen worker 24 - jepsen.util 16044 :info :cas [8 [13 17]] :RequestCanceledException INFO [2019-12-17 12:37:50,200] jepsen worker 34 - jepsen.util 17674 :info :cas [11 [1 43]] :RequestCanceledException INFO [2019-12-17 12:37:50,201] jepsen worker 24 - jepsen.util 16134 :invoke :cas [8 [42 35]] INFO [2019-12-17 12:37:50,201] jepsen worker 34 - jepsen.util 17764 :invoke :write [11 3] INFO [2019-12-17 12:37:50,217] jepsen worker 40 - jepsen.util 17680 :info :cas [13 [35 29]] :RequestCanceledException INFO [2019-12-17 12:37:50,218] jepsen worker 40 - jepsen.util 17770 :invoke :cas [13 [25 44]] INFO [2019-12-17 12:37:50,234] jepsen worker 69 - jepsen.util 12669 :ok :read [23 39] INFO [2019-12-17 12:37:50,234] jepsen worker 69 - jepsen.util 12669 :invoke :write [23 47] INFO [2019-12-17 12:37:50,241] jepsen worker 4 - jepsen.util 11614 :ok :read [1 45] INFO [2019-12-17 12:37:50,241] jepsen worker 4 - jepsen.util 11614 :invoke :cas [1 [13 32]] INFO [2019-12-17 12:37:50,260] jepsen worker 38 - jepsen.util 12908 :fail :cas [12 [1 24]] :ValueNotSwapFrom INFO [2019-12-17 12:37:50,260] jepsen worker 38 - jepsen.util 12908 :invoke :write [12 44] INFO [2019-12-17 12:37:50,299] jepsen worker 56 - jepsen.util 13286 :ok :read [18 9] INFO [2019-12-17 12:37:50,299] jepsen worker 56 - jepsen.util 13286 :invoke :cas [18 [45 34]] INFO [2019-12-17 12:37:50,301] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71733,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:37:50,301] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71733,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:37:50,301] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71734,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:37:50,301] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71734,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:37:50,301] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71735,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:37:50,302] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71735,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:37:50,307] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71740,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:37:50,307] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71740,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:37:50,307] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71743,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:37:50,307] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71743,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:37:50,307] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71744,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:37:50,307] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71744,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:37:50,334] jepsen worker 5 - jepsen.util 11975 :info :cas [1 [40 33]] :RequestCanceledException INFO [2019-12-17 12:37:50,337] jepsen worker 5 - jepsen.util 12065 :invoke :cas [1 [27 4]] INFO [2019-12-17 12:37:50,344] jepsen worker 79 - jepsen.util 12589 :fail :cas [26 [28 34]] :ValueNotSwapFrom INFO [2019-12-17 12:37:50,344] jepsen worker 79 - jepsen.util 12589 :invoke :write [26 17] INFO [2019-12-17 12:37:50,351] jepsen worker 10 - jepsen.util 12250 :ok :write [3 20] INFO [2019-12-17 12:37:50,351] jepsen worker 73 - jepsen.util 11503 :ok :write [24 3] INFO [2019-12-17 12:37:50,351] jepsen worker 10 - jepsen.util 12250 :invoke :cas [3 [6 21]] INFO [2019-12-17 12:37:50,351] jepsen worker 73 - jepsen.util 11503 :invoke :read [24 nil] INFO [2019-12-17 12:37:50,366] jepsen worker 89 - jepsen.util 12149 :fail :cas [29 [46 47]] :ValueNotSwapFrom INFO [2019-12-17 12:37:50,367] jepsen worker 89 - jepsen.util 12149 :invoke :read [29 nil] INFO [2019-12-17 12:37:50,397] jepsen worker 51 - jepsen.util 11571 :ok :read [17 25] INFO [2019-12-17 12:37:50,397] jepsen worker 51 - jepsen.util 11571 :invoke :write [17 22] INFO [2019-12-17 12:37:50,414] jepsen worker 88 - jepsen.util 12598 :fail :cas [29 [33 42]] :ValueNotSwapFrom INFO [2019-12-17 12:37:50,414] jepsen worker 88 - jepsen.util 12598 :invoke :read [29 nil] INFO [2019-12-17 12:37:50,462] jepsen worker 10 - jepsen.util 12250 :fail :cas [3 [6 21]] :ValueNotSwapFrom INFO [2019-12-17 12:37:50,462] jepsen worker 10 - jepsen.util 12250 :invoke :cas [3 [1 25]] INFO [2019-12-17 12:37:50,463] jepsen worker 45 - jepsen.util 12195 :ok :write [15 46] INFO [2019-12-17 12:37:50,463] jepsen worker 45 - jepsen.util 12195 :invoke :write [15 46] INFO [2019-12-17 12:37:50,463] jepsen worker 31 - jepsen.util 10561 :ok :write [10 22] INFO [2019-12-17 12:37:50,463] jepsen worker 31 - jepsen.util 10561 :invoke :read [10 nil] INFO [2019-12-17 12:37:50,465] jepsen worker 20 - jepsen.util 12170 :ok :write [6 4] INFO [2019-12-17 12:37:50,466] jepsen worker 20 - jepsen.util 12170 :invoke :read [6 nil] INFO [2019-12-17 12:37:50,483] jepsen worker 4 - jepsen.util 11614 :fail :cas [1 [13 32]] :ValueNotSwapFrom INFO [2019-12-17 12:37:50,483] jepsen worker 4 - jepsen.util 11614 :invoke :read [1 nil] INFO [2019-12-17 12:37:50,526] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71759,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:37:50,526] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71759,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:37:50,526] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71763,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:37:50,529] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71763,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:37:50,534] jepsen worker 47 - jepsen.util 13457 :ok :write [15 21] INFO [2019-12-17 12:37:50,535] jepsen worker 47 - jepsen.util 13457 :invoke :cas [15 [37 8]] INFO [2019-12-17 12:37:50,537] jepsen worker 17 - jepsen.util 17027 :info :cas [5 [19 22]] :RequestCanceledException INFO [2019-12-17 12:37:50,537] jepsen worker 26 - jepsen.util 17666 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:37:50,537] jepsen worker 78 - jepsen.util 12048 :info :write [26 49] :RequestCanceledException INFO [2019-12-17 12:37:50,537] jepsen worker 26 - jepsen.util 17666 :invoke :write [8 11] INFO [2019-12-17 12:37:50,545] jepsen worker 73 - jepsen.util 11503 :ok :read [24 3] INFO [2019-12-17 12:37:50,548] jepsen worker 60 - jepsen.util 17880 :info :write [20 47] :RequestCanceledException INFO [2019-12-17 12:37:50,549] jepsen worker 73 - jepsen.util 11503 :invoke :write [24 19] INFO [2019-12-17 12:37:50,556] jepsen worker 78 - jepsen.util 12138 :invoke :cas [26 [18 46]] INFO [2019-12-17 12:37:50,560] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:37:50,561] jepsen nemesis - couchbase.util Rebalance check count: 565 INFO [2019-12-17 12:37:50,561] jepsen nemesis - couchbase.util Rebalance stuck count: 546 INFO [2019-12-17 12:37:50,568] jepsen worker 86 - jepsen.util 17546 :info :write [28 12] :RequestCanceledException INFO [2019-12-17 12:37:50,568] jepsen worker 17 - jepsen.util 17117 :invoke :read [5 nil] INFO [2019-12-17 12:37:50,569] jepsen worker 89 - jepsen.util 12149 :ok :read [29 22] INFO [2019-12-17 12:37:50,569] jepsen worker 75 - jepsen.util 16455 :info :write [25 44] :RequestCanceledException INFO [2019-12-17 12:37:50,569] jepsen worker 89 - jepsen.util 12149 :invoke :write [29 1] INFO [2019-12-17 12:37:50,569] jepsen worker 9 - jepsen.util 12879 :info :write [3 9] :RequestCanceledException INFO [2019-12-17 12:37:50,573] jepsen worker 56 - jepsen.util 13286 :fail :cas [18 [45 34]] :ValueNotSwapFrom INFO [2019-12-17 12:37:50,573] jepsen worker 56 - jepsen.util 13286 :invoke :read [18 nil] INFO [2019-12-17 12:37:50,573] jepsen worker 60 - jepsen.util 17970 :invoke :read [20 nil] INFO [2019-12-17 12:37:50,577] jepsen worker 86 - jepsen.util 17636 :invoke :write [28 15] INFO [2019-12-17 12:37:50,579] jepsen worker 9 - jepsen.util 12969 :invoke :cas [3 [45 32]] INFO [2019-12-17 12:37:50,588] jepsen worker 75 - jepsen.util 16545 :invoke :read [25 nil] INFO [2019-12-17 12:37:50,600] jepsen worker 87 - jepsen.util 11967 :info :write [29 32] :RequestCanceledException INFO [2019-12-17 12:37:50,619] jepsen worker 62 - jepsen.util 17612 :info :write [20 46] :RequestCanceledException INFO [2019-12-17 12:37:50,630] jepsen worker 62 - jepsen.util 17702 :invoke :cas [20 [23 6]] INFO [2019-12-17 12:37:50,630] jepsen worker 87 - jepsen.util 12057 :invoke :cas [29 [31 0]] INFO [2019-12-17 12:37:50,638] jepsen worker 70 - jepsen.util 12850 :ok :read [23 39] INFO [2019-12-17 12:37:50,638] jepsen worker 70 - jepsen.util 12850 :invoke :write [23 23] INFO [2019-12-17 12:37:50,644] jepsen worker 80 - jepsen.util 11960 :fail :cas [26 [13 39]] :ValueNotSwapFrom INFO [2019-12-17 12:37:50,644] jepsen worker 80 - jepsen.util 11960 :invoke :read [26 nil] INFO [2019-12-17 12:37:50,647] jepsen worker 71 - jepsen.util 13211 :ok :read [23 39] INFO [2019-12-17 12:37:50,647] jepsen worker 71 - jepsen.util 13211 :invoke :cas [23 [7 11]] INFO [2019-12-17 12:37:50,657] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71783,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:37:50,657] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71783,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:37:50,657] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71788,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:37:50,657] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71788,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:37:50,665] jepsen worker 54 - jepsen.util 12924 :fail :cas [18 [43 49]] :ValueNotSwapFrom INFO [2019-12-17 12:37:50,665] jepsen worker 54 - jepsen.util 12924 :invoke :read [18 nil] INFO [2019-12-17 12:37:50,674] jepsen worker 31 - jepsen.util 10561 :ok :read [10 22] INFO [2019-12-17 12:37:50,675] jepsen worker 31 - jepsen.util 10561 :invoke :write [10 40] INFO [2019-12-17 12:37:50,677] jepsen worker 20 - jepsen.util 12170 :ok :read [6 4] INFO [2019-12-17 12:37:50,677] jepsen worker 20 - jepsen.util 12170 :invoke :cas [6 [42 49]] INFO [2019-12-17 12:37:50,694] jepsen worker 10 - jepsen.util 12250 :fail :cas [3 [1 25]] :ValueNotSwapFrom INFO [2019-12-17 12:37:50,694] jepsen worker 10 - jepsen.util 12250 :invoke :write [3 48] INFO [2019-12-17 12:37:50,720] jepsen worker 47 - jepsen.util 13457 :fail :cas [15 [37 8]] :ValueNotSwapFrom INFO [2019-12-17 12:37:50,720] jepsen worker 47 - jepsen.util 13457 :invoke :write [15 27] INFO [2019-12-17 12:37:50,739] jepsen worker 9 - jepsen.util 12969 :fail :cas [3 [45 32]] :ValueNotSwapFrom INFO [2019-12-17 12:37:50,739] jepsen worker 9 - jepsen.util 12969 :invoke :cas [3 [16 17]] INFO [2019-12-17 12:37:50,748] jepsen worker 66 - jepsen.util 16986 :info :cas [22 [22 25]] :RequestCanceledException INFO [2019-12-17 12:37:50,750] jepsen worker 4 - jepsen.util 11614 :ok :read [1 45] INFO [2019-12-17 12:37:50,750] jepsen worker 4 - jepsen.util 11614 :invoke :read [1 nil] INFO [2019-12-17 12:37:50,753] jepsen worker 66 - jepsen.util 17076 :invoke :cas [22 [33 16]] INFO [2019-12-17 12:37:50,801] jepsen worker 85 - jepsen.util 17545 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:37:50,801] jepsen worker 82 - jepsen.util 17272 :info :cas [27 [23 19]] :RequestCanceledException INFO [2019-12-17 12:37:50,802] jepsen worker 32 - jepsen.util 12722 :info :write [10 19] :RequestCanceledException INFO [2019-12-17 12:37:50,804] jepsen worker 85 - jepsen.util 17545 :invoke :write [28 38] INFO [2019-12-17 12:37:50,812] jepsen worker 5 - jepsen.util 12065 :fail :cas [1 [27 4]] :ValueNotSwapFrom INFO [2019-12-17 12:37:50,813] jepsen worker 5 - jepsen.util 12065 :invoke :read [1 nil] INFO [2019-12-17 12:37:50,825] jepsen worker 32 - jepsen.util 12812 :invoke :cas [10 [22 24]] INFO [2019-12-17 12:37:50,827] jepsen worker 37 - jepsen.util 11917 :fail :cas [12 [44 17]] :ValueNotSwapFrom INFO [2019-12-17 12:37:50,827] jepsen worker 37 - jepsen.util 11917 :invoke :write [12 42] INFO [2019-12-17 12:37:50,827] jepsen worker 82 - jepsen.util 17362 :invoke :write [27 26] INFO [2019-12-17 12:37:50,832] jepsen worker 33 - jepsen.util 16773 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:37:50,832] jepsen worker 23 - jepsen.util 17393 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:37:50,832] jepsen worker 84 - jepsen.util 16464 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:37:50,836] jepsen worker 71 - jepsen.util 13211 :fail :cas [23 [7 11]] :ValueNotSwapFrom INFO [2019-12-17 12:37:50,836] jepsen worker 67 - jepsen.util 17167 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:37:50,837] jepsen worker 23 - jepsen.util 17393 :invoke :cas [7 [35 10]] INFO [2019-12-17 12:37:50,837] jepsen worker 84 - jepsen.util 16464 :invoke :cas [28 [46 17]] INFO [2019-12-17 12:37:50,837] jepsen worker 71 - jepsen.util 13211 :invoke :write [23 11] INFO [2019-12-17 12:37:50,839] jepsen worker 33 - jepsen.util 16773 :invoke :cas [11 [16 46]] INFO [2019-12-17 12:37:50,840] jepsen worker 67 - jepsen.util 17167 :invoke :cas [22 [11 26]] INFO [2019-12-17 12:37:50,850] jepsen worker 12 - jepsen.util 12522 :fail :cas [4 [21 37]] :ValueNotSwapFrom INFO [2019-12-17 12:37:50,851] jepsen worker 12 - jepsen.util 12522 :invoke :write [4 46] INFO [2019-12-17 12:37:50,862] jepsen worker 19 - jepsen.util 12709 :info :cas [6 [46 28]] :RequestCanceledException INFO [2019-12-17 12:37:50,866] jepsen worker 19 - jepsen.util 12799 :invoke :read [6 nil] INFO [2019-12-17 12:37:50,867] jepsen worker 20 - jepsen.util 12170 :fail :cas [6 [42 49]] :ValueNotSwapFrom INFO [2019-12-17 12:37:50,867] jepsen worker 20 - jepsen.util 12170 :invoke :cas [6 [5 25]] INFO [2019-12-17 12:37:50,896] jepsen worker 29 - jepsen.util 10829 :info :cas [9 [49 25]] :RequestCanceledException INFO [2019-12-17 12:37:50,912] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71809,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:37:50,930] jepsen worker 29 - jepsen.util 10919 :invoke :cas [9 [38 33]] INFO [2019-12-17 12:37:50,934] jepsen worker 0 - jepsen.util 17550 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:37:50,934] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71809,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:37:50,934] jepsen worker 0 - jepsen.util 17550 :invoke :write [0 5] INFO [2019-12-17 12:37:50,938] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71810,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:37:50,939] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71810,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:37:50,963] jepsen worker 52 - jepsen.util 11842 :info :write [17 25] :RequestCanceledException INFO [2019-12-17 12:37:50,965] jepsen worker 52 - jepsen.util 11932 :invoke :write [17 9] INFO [2019-12-17 12:37:50,986] jepsen worker 56 - jepsen.util 13286 :ok :read [18 9] INFO [2019-12-17 12:37:50,986] jepsen worker 56 - jepsen.util 13286 :invoke :cas [18 [36 26]] INFO [2019-12-17 12:37:50,999] jepsen worker 21 - jepsen.util 16401 :info :cas [7 [33 32]] :RequestCanceledException INFO [2019-12-17 12:37:51,013] jepsen worker 21 - jepsen.util 16491 :invoke :read [7 nil] INFO [2019-12-17 12:37:51,050] jepsen worker 78 - jepsen.util 12138 :fail :cas [26 [18 46]] :ValueNotSwapFrom INFO [2019-12-17 12:37:51,050] jepsen worker 78 - jepsen.util 12138 :invoke :cas [26 [12 28]] INFO [2019-12-17 12:37:51,056] jepsen worker 20 - jepsen.util 12170 :fail :cas [6 [5 25]] :ValueNotSwapFrom INFO [2019-12-17 12:37:51,057] jepsen worker 20 - jepsen.util 12170 :invoke :cas [6 [13 29]] INFO [2019-12-17 12:37:51,066] jepsen worker 29 - jepsen.util 10919 :fail :cas [9 [38 33]] :ValueNotSwapFrom INFO [2019-12-17 12:37:51,066] jepsen worker 29 - jepsen.util 10919 :invoke :read [9 nil] INFO [2019-12-17 12:37:51,089] jepsen worker 31 - jepsen.util 10561 :ok :write [10 40] INFO [2019-12-17 12:37:51,089] jepsen worker 45 - jepsen.util 12195 :ok :write [15 46] INFO [2019-12-17 12:37:51,090] jepsen worker 31 - jepsen.util 10561 :invoke :cas [10 [18 21]] INFO [2019-12-17 12:37:51,090] jepsen worker 45 - jepsen.util 12195 :invoke :read [15 nil] INFO [2019-12-17 12:37:51,099] jepsen worker 53 - jepsen.util 11933 :ok :read [17 25] INFO [2019-12-17 12:37:51,100] jepsen worker 53 - jepsen.util 11933 :invoke :cas [17 [24 26]] INFO [2019-12-17 12:37:51,103] jepsen worker 73 - jepsen.util 11503 :ok :write [24 19] INFO [2019-12-17 12:37:51,104] jepsen worker 73 - jepsen.util 11503 :invoke :read [24 nil] INFO [2019-12-17 12:37:51,104] jepsen worker 10 - jepsen.util 12250 :ok :write [3 48] INFO [2019-12-17 12:37:51,104] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71821,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:37:51,104] jepsen worker 10 - jepsen.util 12250 :invoke :read [3 nil] INFO [2019-12-17 12:37:51,105] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71821,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:37:51,105] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71825,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:37:51,105] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71825,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:37:51,117] jepsen worker 46 - jepsen.util 12016 :info :write [15 6] :RequestCanceledException INFO [2019-12-17 12:37:51,121] jepsen worker 88 - jepsen.util 12598 :ok :read [29 22] INFO [2019-12-17 12:37:51,122] jepsen worker 88 - jepsen.util 12598 :invoke :read [29 nil] INFO [2019-12-17 12:37:51,122] jepsen worker 46 - jepsen.util 12106 :invoke :write [15 38] INFO [2019-12-17 12:37:51,129] jepsen worker 54 - jepsen.util 12924 :ok :read [18 9] INFO [2019-12-17 12:37:51,129] jepsen worker 54 - jepsen.util 12924 :invoke :read [18 nil] INFO [2019-12-17 12:37:51,144] jepsen worker 80 - jepsen.util 11960 :ok :read [26 1] INFO [2019-12-17 12:37:51,144] jepsen worker 80 - jepsen.util 11960 :invoke :read [26 nil] INFO [2019-12-17 12:37:51,176] jepsen worker 56 - jepsen.util 13286 :fail :cas [18 [36 26]] :ValueNotSwapFrom INFO [2019-12-17 12:37:51,215] jepsen worker 32 - jepsen.util 12812 :fail :cas [10 [22 24]] :CASMismatchException INFO [2019-12-17 12:37:51,215] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71843,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:37:51,216] jepsen worker 59 - jepsen.util 16169 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:37:51,218] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71848,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:37:51,242] jepsen worker 77 - jepsen.util 18167 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:37:51,246] jepsen worker 29 - jepsen.util 10919 :ok :read [9 12] INFO [2019-12-17 12:37:51,246] jepsen worker 2 - jepsen.util 16202 :info :cas [0 [7 42]] :RequestCanceledException INFO [2019-12-17 12:37:51,246] jepsen worker 25 - jepsen.util 18025 :info :write [8 43] :RequestCanceledException INFO [2019-12-17 12:37:51,261] jepsen worker 74 - jepsen.util 12134 :fail :read [24 nil] :RequestTimeoutException INFO [2019-12-17 12:37:51,278] jepsen worker 46 - jepsen.util 12106 :ok :write [15 38] INFO [2019-12-17 12:37:51,283] jepsen worker 55 - jepsen.util 13195 :ok :write [18 32] INFO [2019-12-17 12:37:51,298] jepsen worker 45 - jepsen.util 12195 :ok :read [15 46] INFO [2019-12-17 12:37:51,334] jepsen worker 9 - jepsen.util 12969 :fail :cas [3 [16 17]] :ValueNotSwapFrom INFO [2019-12-17 12:37:51,348] jepsen worker 16 - jepsen.util 18556 :info :cas [5 [22 29]] :RequestCanceledException INFO [2019-12-17 12:37:51,355] jepsen worker 19 - jepsen.util 12799 :ok :read [6 4] INFO [2019-12-17 12:37:51,361] jepsen worker 73 - jepsen.util 11503 :ok :read [24 19] INFO [2019-12-17 12:37:51,362] jepsen worker 31 - jepsen.util 10561 :fail :cas [10 [18 21]] :ValueNotSwapFrom INFO [2019-12-17 12:37:51,370] jepsen worker 5 - jepsen.util 12065 :ok :read [1 45] INFO [2019-12-17 12:37:51,372] jepsen worker 78 - jepsen.util 12138 :fail :cas [26 [12 28]] :ValueNotSwapFrom INFO [2019-12-17 12:37:51,374] jepsen worker 4 - jepsen.util 11614 :ok :read [1 45] INFO [2019-12-17 12:37:51,375] jepsen worker 20 - jepsen.util 12170 :fail :cas [6 [13 29]] :ValueNotSwapFrom INFO [2019-12-17 12:37:51,380] jepsen worker 10 - jepsen.util 12250 :ok :read [3 48] INFO [2019-12-17 12:37:51,418] jepsen worker 35 - jepsen.util 16955 :info :cas [11 [30 13]] :RequestCanceledException INFO [2019-12-17 12:37:51,434] jepsen worker 14 - jepsen.util 12074 :info :write [4 12] :RequestCanceledException INFO [2019-12-17 12:37:51,435] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71857,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:37:51,435] jepsen worker 8 - jepsen.util 16568 :info :cas [2 [25 12]] :RequestCanceledException INFO [2019-12-17 12:37:51,438] jepsen worker 44 - jepsen.util 17504 :info :write [14 44] :RequestCanceledException INFO [2019-12-17 12:37:51,447] jepsen worker 43 - jepsen.util 17143 :info :write [14 25] :RequestCanceledException INFO [2019-12-17 12:37:51,447] jepsen worker 3 - jepsen.util 11703 :info :write [1 12] :RequestCanceledException INFO [2019-12-17 12:37:51,465] jepsen worker 61 - jepsen.util 18421 :info :write [20 12] :RequestCanceledException INFO [2019-12-17 12:37:51,468] jepsen worker 81 - jepsen.util 16641 :info :cas [27 [47 49]] :RequestCanceledException INFO [2019-12-17 12:37:51,495] jepsen worker 54 - jepsen.util 12924 :ok :read [18 9] INFO [2019-12-17 12:37:51,500] jepsen worker 15 - jepsen.util 15945 :info :cas [5 [23 42]] :RequestCanceledException INFO [2019-12-17 12:37:51,515] jepsen worker 48 - jepsen.util 15618 :info :write [16 23] :RequestCanceledException INFO [2019-12-17 12:37:51,534] jepsen worker 50 - jepsen.util 16970 :info :write [16 0] :RequestCanceledException INFO [2019-12-17 12:37:51,535] jepsen worker 87 - jepsen.util 12057 :fail :cas [29 [31 0]] :ValueNotSwapFrom INFO [2019-12-17 12:37:51,547] jepsen worker 58 - jepsen.util 17068 :info :cas [19 [7 6]] :RequestCanceledException INFO [2019-12-17 12:37:51,577] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:37:51,588] jepsen worker 56 - jepsen.util 13286 :invoke :cas [18 [29 17]] INFO [2019-12-17 12:37:51,588] jepsen worker 32 - jepsen.util 12812 :invoke :write [10 44] INFO [2019-12-17 12:37:51,588] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71843,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:37:51,588] jepsen worker 59 - jepsen.util 16169 :invoke :write [19 45] INFO [2019-12-17 12:37:51,588] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71848,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:37:51,588] jepsen worker 77 - jepsen.util 18167 :invoke :read [25 nil] INFO [2019-12-17 12:37:51,588] jepsen worker 29 - jepsen.util 10919 :invoke :read [9 nil] INFO [2019-12-17 12:37:51,589] jepsen worker 74 - jepsen.util 12134 :invoke :read [24 nil] INFO [2019-12-17 12:37:51,589] jepsen worker 46 - jepsen.util 12106 :invoke :read [15 nil] INFO [2019-12-17 12:37:51,589] jepsen worker 55 - jepsen.util 13195 :invoke :cas [18 [44 24]] INFO [2019-12-17 12:37:51,589] jepsen worker 45 - jepsen.util 12195 :invoke :write [15 44] INFO [2019-12-17 12:37:51,589] jepsen worker 9 - jepsen.util 12969 :invoke :write [3 46] INFO [2019-12-17 12:37:51,590] jepsen worker 19 - jepsen.util 12799 :invoke :write [6 31] INFO [2019-12-17 12:37:51,590] jepsen worker 73 - jepsen.util 11503 :invoke :read [24 nil] INFO [2019-12-17 12:37:51,590] jepsen worker 31 - jepsen.util 10561 :invoke :write [10 37] INFO [2019-12-17 12:37:51,591] jepsen worker 5 - jepsen.util 12065 :invoke :read [1 nil] INFO [2019-12-17 12:37:51,591] jepsen worker 78 - jepsen.util 12138 :invoke :read [26 nil] INFO [2019-12-17 12:37:51,591] jepsen worker 4 - jepsen.util 11614 :invoke :read [1 nil] INFO [2019-12-17 12:37:51,591] jepsen worker 20 - jepsen.util 12170 :invoke :read [6 nil] INFO [2019-12-17 12:37:51,591] jepsen worker 10 - jepsen.util 12250 :invoke :cas [3 [20 35]] INFO [2019-12-17 12:37:51,591] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71857,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:37:51,592] jepsen worker 54 - jepsen.util 12924 :invoke :cas [18 [47 0]] INFO [2019-12-17 12:37:51,592] jepsen worker 14 - jepsen.util 12164 :invoke :write [4 17] INFO [2019-12-17 12:37:51,592] jepsen worker 87 - jepsen.util 12057 :invoke :write [29 8] INFO [2019-12-17 12:37:51,593] jepsen nemesis - couchbase.util Rebalance check count: 566 INFO [2019-12-17 12:37:51,593] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71846,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:37:51,593] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71852,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:37:51,600] jepsen worker 25 - jepsen.util 18115 :invoke :write [8 21] INFO [2019-12-17 12:37:51,603] jepsen worker 61 - jepsen.util 18511 :invoke :read [20 nil] INFO [2019-12-17 12:37:51,603] jepsen worker 35 - jepsen.util 17045 :invoke :write [11 49] INFO [2019-12-17 12:37:51,604] jepsen worker 3 - jepsen.util 11793 :invoke :cas [1 [33 9]] INFO [2019-12-17 12:37:51,604] jepsen nemesis - couchbase.util Rebalance stuck count: 547 INFO [2019-12-17 12:37:51,604] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71846,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:37:51,604] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71852,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:37:51,607] jepsen worker 8 - jepsen.util 16658 :invoke :write [2 23] INFO [2019-12-17 12:37:51,607] jepsen worker 16 - jepsen.util 18646 :invoke :read [5 nil] INFO [2019-12-17 12:37:51,608] jepsen worker 15 - jepsen.util 16035 :invoke :write [5 38] INFO [2019-12-17 12:37:51,610] jepsen worker 44 - jepsen.util 17594 :invoke :write [14 31] INFO [2019-12-17 12:37:51,613] jepsen worker 50 - jepsen.util 17060 :invoke :read [16 nil] INFO [2019-12-17 12:37:51,614] jepsen worker 81 - jepsen.util 16731 :invoke :read [27 nil] INFO [2019-12-17 12:37:51,616] jepsen worker 43 - jepsen.util 17233 :invoke :cas [14 [24 33]] INFO [2019-12-17 12:37:51,618] jepsen worker 48 - jepsen.util 15708 :invoke :write [16 40] INFO [2019-12-17 12:37:51,619] jepsen worker 7 - jepsen.util 16927 :info :write [2 14] :RequestCanceledException INFO [2019-12-17 12:37:51,619] jepsen worker 58 - jepsen.util 17158 :invoke :write [19 30] INFO [2019-12-17 12:37:51,620] jepsen worker 2 - jepsen.util 16292 :invoke :cas [0 [17 2]] INFO [2019-12-17 12:37:51,632] jepsen worker 13 - jepsen.util 11803 :info :write [4 48] :RequestCanceledException INFO [2019-12-17 12:37:51,639] jepsen worker 7 - jepsen.util 17017 :invoke :write [2 29] INFO [2019-12-17 12:37:51,646] jepsen worker 13 - jepsen.util 11893 :invoke :read [4 nil] INFO [2019-12-17 12:37:51,647] jepsen worker 72 - jepsen.util 12492 :fail :read [24 nil] :RequestTimeoutException INFO [2019-12-17 12:37:51,647] jepsen worker 72 - jepsen.util 12492 :invoke :write [24 24] INFO [2019-12-17 12:37:51,651] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71884,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:37:51,651] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71884,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:37:51,651] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71888,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:37:51,659] jepsen worker 30 - jepsen.util 11550 :info :write [10 42] :RequestCanceledException INFO [2019-12-17 12:37:51,659] jepsen worker 63 - jepsen.util 10863 :info :cas [21 [38 33]] :RequestCanceledException INFO [2019-12-17 12:37:51,673] jepsen worker 30 - jepsen.util 11640 :invoke :read [10 nil] INFO [2019-12-17 12:37:51,673] jepsen worker 63 - jepsen.util 10953 :invoke :cas [21 [49 31]] INFO [2019-12-17 12:37:51,701] jepsen worker 57 - jepsen.util 16707 :info :write [19 28] :RequestCanceledException INFO [2019-12-17 12:37:51,704] jepsen worker 53 - jepsen.util 11933 :fail :cas [17 [24 26]] :ValueNotSwapFrom INFO [2019-12-17 12:37:51,705] jepsen worker 53 - jepsen.util 11933 :invoke :write [17 8] INFO [2019-12-17 12:37:51,706] jepsen worker 57 - jepsen.util 16797 :invoke :read [19 nil] INFO [2019-12-17 12:37:51,734] jepsen worker 18 - jepsen.util 12888 :info :write [6 10] :RequestCanceledException INFO [2019-12-17 12:37:51,735] jepsen worker 18 - jepsen.util 12978 :invoke :cas [6 [40 30]] INFO [2019-12-17 12:37:51,760] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71888,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:37:51,787] jepsen worker 4 - jepsen.util 11614 :ok :read [1 45] INFO [2019-12-17 12:37:51,787] jepsen worker 4 - jepsen.util 11614 :invoke :cas [1 [27 42]] INFO [2019-12-17 12:37:51,793] jepsen worker 29 - jepsen.util 10919 :ok :read [9 12] INFO [2019-12-17 12:37:51,793] jepsen worker 29 - jepsen.util 10919 :invoke :cas [9 [45 40]] INFO [2019-12-17 12:37:51,802] jepsen worker 56 - jepsen.util 13286 :fail :cas [18 [29 17]] :ValueNotSwapFrom INFO [2019-12-17 12:37:51,803] jepsen worker 56 - jepsen.util 13286 :invoke :cas [18 [5 35]] INFO [2019-12-17 12:37:51,813] jepsen worker 55 - jepsen.util 13195 :fail :cas [18 [44 24]] :ValueNotSwapFrom INFO [2019-12-17 12:37:51,813] jepsen worker 55 - jepsen.util 13195 :invoke :cas [18 [24 25]] INFO [2019-12-17 12:37:51,833] jepsen worker 36 - jepsen.util 13086 :info :write [12 5] :RequestCanceledException INFO [2019-12-17 12:37:51,841] jepsen worker 46 - jepsen.util 12106 :ok :read [15 38] INFO [2019-12-17 12:37:51,842] jepsen worker 10 - jepsen.util 12250 :fail :cas [3 [20 35]] :ValueNotSwapFrom INFO [2019-12-17 12:37:51,842] jepsen worker 46 - jepsen.util 12106 :invoke :read [15 nil] INFO [2019-12-17 12:37:51,842] jepsen worker 10 - jepsen.util 12250 :invoke :cas [3 [35 37]] INFO [2019-12-17 12:37:51,842] jepsen worker 36 - jepsen.util 13176 :invoke :cas [12 [36 39]] INFO [2019-12-17 12:37:51,865] jepsen worker 42 - jepsen.util 17772 :info :write [14 34] :RequestCanceledException INFO [2019-12-17 12:37:51,865] jepsen worker 11 - jepsen.util 11891 :info :write [3 26] :RequestCanceledException INFO [2019-12-17 12:37:51,866] jepsen worker 11 - jepsen.util 11981 :invoke :cas [3 [40 32]] INFO [2019-12-17 12:37:51,871] jepsen worker 3 - jepsen.util 11793 :fail :cas [1 [33 9]] :ValueNotSwapFrom INFO [2019-12-17 12:37:51,872] jepsen worker 3 - jepsen.util 11793 :invoke :read [1 nil] INFO [2019-12-17 12:37:51,873] jepsen worker 42 - jepsen.util 17862 :invoke :write [14 26] INFO [2019-12-17 12:37:51,903] jepsen worker 73 - jepsen.util 11503 :ok :read [24 19] INFO [2019-12-17 12:37:51,903] jepsen worker 73 - jepsen.util 11503 :invoke :read [24 nil] INFO [2019-12-17 12:37:51,918] jepsen worker 1 - jepsen.util 17281 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:37:51,918] jepsen worker 1 - jepsen.util 17281 :invoke :cas [0 [11 46]] INFO [2019-12-17 12:37:51,931] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71918,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:37:51,932] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71918,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:37:51,932] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71922,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:37:51,932] jepsen worker 83 - jepsen.util 16913 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:37:51,932] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71922,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:37:51,932] jepsen worker 83 - jepsen.util 16913 :invoke :write [27 9] INFO [2019-12-17 12:37:51,932] jepsen worker 22 - jepsen.util 17032 :info :write [7 3] :RequestCanceledException INFO [2019-12-17 12:37:51,932] jepsen worker 39 - jepsen.util 16509 :info :write [13 38] :RequestCanceledException INFO [2019-12-17 12:37:51,940] jepsen worker 22 - jepsen.util 17122 :invoke :write [7 47] INFO [2019-12-17 12:37:51,941] jepsen worker 39 - jepsen.util 16599 :invoke :read [13 nil] INFO [2019-12-17 12:37:51,970] jepsen worker 74 - jepsen.util 12134 :ok :read [24 19] INFO [2019-12-17 12:37:51,970] jepsen worker 74 - jepsen.util 12134 :invoke :write [24 28] INFO [2019-12-17 12:37:51,986] jepsen worker 13 - jepsen.util 11893 :ok :read [4 4] INFO [2019-12-17 12:37:51,986] jepsen worker 13 - jepsen.util 11893 :invoke :cas [4 [33 35]] INFO [2019-12-17 12:37:52,003] jepsen worker 78 - jepsen.util 12138 :ok :read [26 1] INFO [2019-12-17 12:37:52,003] jepsen worker 78 - jepsen.util 12138 :invoke :cas [26 [37 40]] INFO [2019-12-17 12:37:52,007] jepsen worker 18 - jepsen.util 12978 :fail :cas [6 [40 30]] :ValueNotSwapFrom INFO [2019-12-17 12:37:52,007] jepsen worker 18 - jepsen.util 12978 :invoke :cas [6 [34 19]] INFO [2019-12-17 12:37:52,016] jepsen worker 80 - jepsen.util 11960 :ok :read [26 1] INFO [2019-12-17 12:37:52,016] jepsen worker 80 - jepsen.util 11960 :invoke :read [26 nil] INFO [2019-12-17 12:37:52,034] jepsen worker 64 - jepsen.util 9604 :info :write [21 11] :RequestCanceledException INFO [2019-12-17 12:37:52,036] jepsen worker 64 - jepsen.util 9694 :invoke :cas [21 [49 28]] INFO [2019-12-17 12:37:52,049] jepsen worker 20 - jepsen.util 12170 :ok :read [6 4] INFO [2019-12-17 12:37:52,049] jepsen worker 20 - jepsen.util 12170 :invoke :write [6 15] INFO [2019-12-17 12:37:52,050] jepsen worker 4 - jepsen.util 11614 :fail :cas [1 [27 42]] :ValueNotSwapFrom INFO [2019-12-17 12:37:52,050] jepsen worker 4 - jepsen.util 11614 :invoke :read [1 nil] INFO [2019-12-17 12:37:52,060] jepsen worker 29 - jepsen.util 10919 :fail :cas [9 [45 40]] :ValueNotSwapFrom INFO [2019-12-17 12:37:52,060] jepsen worker 29 - jepsen.util 10919 :invoke :read [9 nil] INFO [2019-12-17 12:37:52,075] jepsen worker 73 - jepsen.util 11503 :ok :read [24 19] INFO [2019-12-17 12:37:52,075] jepsen worker 73 - jepsen.util 11503 :invoke :cas [24 [42 49]] INFO [2019-12-17 12:37:52,075] jepsen worker 88 - jepsen.util 12598 :ok :read [29 22] INFO [2019-12-17 12:37:52,075] jepsen worker 88 - jepsen.util 12598 :invoke :read [29 nil] INFO [2019-12-17 12:37:52,087] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71933,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:37:52,087] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71933,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:37:52,098] jepsen worker 54 - jepsen.util 12924 :fail :cas [18 [47 0]] :ValueNotSwapFrom INFO [2019-12-17 12:37:52,098] jepsen worker 54 - jepsen.util 12924 :invoke :cas [18 [47 24]] INFO [2019-12-17 12:37:52,104] jepsen worker 56 - jepsen.util 13286 :fail :cas [18 [5 35]] :ValueNotSwapFrom INFO [2019-12-17 12:37:52,104] jepsen worker 56 - jepsen.util 13286 :invoke :write [18 16] INFO [2019-12-17 12:37:52,110] jepsen worker 55 - jepsen.util 13195 :fail :cas [18 [24 25]] :ValueNotSwapFrom INFO [2019-12-17 12:37:52,110] jepsen worker 55 - jepsen.util 13195 :invoke :read [18 nil] INFO [2019-12-17 12:37:52,118] jepsen worker 76 - jepsen.util 17086 :info :write [25 2] :RequestCanceledException INFO [2019-12-17 12:37:52,120] jepsen worker 76 - jepsen.util 17176 :invoke :read [25 nil] INFO [2019-12-17 12:37:52,133] jepsen worker 49 - jepsen.util 16969 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:37:52,133] jepsen worker 27 - jepsen.util 11457 :info :write [9 41] :RequestCanceledException INFO [2019-12-17 12:37:52,133] jepsen worker 28 - jepsen.util 10648 :fail :read [9 nil] :RequestTimeoutException INFO [2019-12-17 12:37:52,134] jepsen worker 49 - jepsen.util 16969 :invoke :cas [16 [6 36]] INFO [2019-12-17 12:37:52,134] jepsen worker 28 - jepsen.util 10648 :invoke :cas [9 [7 2]] INFO [2019-12-17 12:37:52,136] jepsen worker 27 - jepsen.util 11547 :invoke :write [9 21] INFO [2019-12-17 12:37:52,147] jepsen worker 68 - jepsen.util 16088 :info :cas [22 [10 2]] :RequestCanceledException INFO [2019-12-17 12:37:52,153] jepsen worker 73 - jepsen.util 11503 :fail :cas [24 [42 49]] :ValueNotSwapFrom INFO [2019-12-17 12:37:52,153] jepsen worker 73 - jepsen.util 11503 :invoke :cas [24 [26 41]] INFO [2019-12-17 12:37:52,153] jepsen worker 68 - jepsen.util 16178 :invoke :write [22 26] INFO [2019-12-17 12:37:52,165] jepsen worker 6 - jepsen.util 17286 :info :cas [2 [30 4]] :RequestCanceledException INFO [2019-12-17 12:37:52,172] jepsen worker 6 - jepsen.util 17376 :invoke :write [2 31] INFO [2019-12-17 12:37:52,195] jepsen worker 3 - jepsen.util 11793 :ok :read [1 45] INFO [2019-12-17 12:37:52,195] jepsen worker 3 - jepsen.util 11793 :invoke :read [1 nil] INFO [2019-12-17 12:37:52,199] jepsen worker 65 - jepsen.util 10955 :fail :read [21 nil] :RequestTimeoutException INFO [2019-12-17 12:37:52,199] jepsen worker 65 - jepsen.util 10955 :invoke :read [21 nil] INFO [2019-12-17 12:37:52,210] jepsen worker 73 - jepsen.util 11503 :fail :cas [24 [26 41]] :ValueNotSwapFrom INFO [2019-12-17 12:37:52,210] jepsen worker 73 - jepsen.util 11503 :invoke :read [24 nil] INFO [2019-12-17 12:37:52,234] jepsen worker 40 - jepsen.util 17770 :info :cas [13 [25 44]] :RequestCanceledException INFO [2019-12-17 12:37:52,234] jepsen worker 41 - jepsen.util 18311 :info :cas [13 [45 14]] :RequestCanceledException INFO [2019-12-17 12:37:52,237] jepsen worker 40 - jepsen.util 17860 :invoke :write [13 28] INFO [2019-12-17 12:37:52,239] jepsen worker 41 - jepsen.util 18401 :invoke :write [13 20] INFO [2019-12-17 12:37:52,270] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71959,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:37:52,270] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71959,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:37:52,271] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71961,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:37:52,271] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71961,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:37:52,271] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71968,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:37:52,272] jepsen worker 19 - jepsen.util 12799 :ok :write [6 31] INFO [2019-12-17 12:37:52,272] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71968,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:37:52,272] jepsen worker 19 - jepsen.util 12799 :invoke :cas [6 [20 39]] INFO [2019-12-17 12:37:52,273] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71971,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:37:52,273] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71971,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:37:52,289] jepsen worker 73 - jepsen.util 11503 :ok :read [24 19] INFO [2019-12-17 12:37:52,289] jepsen worker 73 - jepsen.util 11503 :invoke :write [24 29] INFO [2019-12-17 12:37:52,300] jepsen worker 24 - jepsen.util 16134 :info :cas [8 [42 35]] :RequestCanceledException INFO [2019-12-17 12:37:52,302] jepsen worker 13 - jepsen.util 11893 :fail :cas [4 [33 35]] :ValueNotSwapFrom INFO [2019-12-17 12:37:52,303] jepsen worker 13 - jepsen.util 11893 :invoke :write [4 4] INFO [2019-12-17 12:37:52,310] jepsen worker 24 - jepsen.util 16224 :invoke :cas [8 [22 0]] INFO [2019-12-17 12:37:52,312] jepsen worker 4 - jepsen.util 11614 :ok :read [1 45] INFO [2019-12-17 12:37:52,312] jepsen worker 4 - jepsen.util 11614 :invoke :cas [1 [28 20]] INFO [2019-12-17 12:37:52,317] jepsen worker 34 - jepsen.util 17764 :info :write [11 3] :RequestCanceledException INFO [2019-12-17 12:37:52,329] jepsen worker 34 - jepsen.util 17854 :invoke :write [11 2] INFO [2019-12-17 12:37:52,334] jepsen worker 38 - jepsen.util 12908 :info :write [12 44] :RequestCanceledException INFO [2019-12-17 12:37:52,337] jepsen worker 38 - jepsen.util 12998 :invoke :write [12 26] INFO [2019-12-17 12:37:52,349] jepsen worker 69 - jepsen.util 12669 :info :write [23 47] :RequestCanceledException INFO [2019-12-17 12:37:52,357] jepsen worker 69 - jepsen.util 12759 :invoke :cas [23 [29 18]] INFO [2019-12-17 12:37:52,367] jepsen worker 3 - jepsen.util 11793 :ok :read [1 45] INFO [2019-12-17 12:37:52,367] jepsen worker 3 - jepsen.util 11793 :invoke :write [1 6] INFO [2019-12-17 12:37:52,377] jepsen worker 55 - jepsen.util 13195 :ok :read [18 32] INFO [2019-12-17 12:37:52,377] jepsen worker 55 - jepsen.util 13195 :invoke :cas [18 [43 38]] INFO [2019-12-17 12:37:52,384] jepsen worker 5 - jepsen.util 12065 :ok :read [1 45] INFO [2019-12-17 12:37:52,384] jepsen worker 5 - jepsen.util 12065 :invoke :read [1 nil] INFO [2019-12-17 12:37:52,412] jepsen worker 19 - jepsen.util 12799 :fail :cas [6 [20 39]] :ValueNotSwapFrom INFO [2019-12-17 12:37:52,412] jepsen worker 19 - jepsen.util 12799 :invoke :cas [6 [34 42]] INFO [2019-12-17 12:37:52,433] jepsen worker 78 - jepsen.util 12138 :fail :cas [26 [37 40]] :ValueNotSwapFrom INFO [2019-12-17 12:37:52,433] jepsen worker 78 - jepsen.util 12138 :invoke :cas [26 [5 13]] INFO [2019-12-17 12:37:52,436] jepsen worker 54 - jepsen.util 12924 :fail :cas [18 [47 24]] :ValueNotSwapFrom INFO [2019-12-17 12:37:52,436] jepsen worker 54 - jepsen.util 12924 :invoke :cas [18 [15 41]] INFO [2019-12-17 12:37:52,448] jepsen worker 79 - jepsen.util 12589 :info :write [26 17] :RequestCanceledException INFO [2019-12-17 12:37:52,448] jepsen worker 19 - jepsen.util 12799 :fail :cas [6 [34 42]] :ValueNotSwapFrom INFO [2019-12-17 12:37:52,449] jepsen worker 19 - jepsen.util 12799 :invoke :read [6 nil] INFO [2019-12-17 12:37:52,450] jepsen worker 79 - jepsen.util 12679 :invoke :write [26 3] INFO [2019-12-17 12:37:52,476] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71988,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:37:52,476] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71988,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:37:52,476] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71991,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:37:52,477] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71991,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:37:52,477] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71992,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:37:52,477] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":71992,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:37:52,501] jepsen worker 51 - jepsen.util 11571 :info :write [17 22] :RequestCanceledException INFO [2019-12-17 12:37:52,502] jepsen worker 80 - jepsen.util 11960 :ok :read [26 1] INFO [2019-12-17 12:37:52,502] jepsen worker 80 - jepsen.util 11960 :invoke :read [26 nil] INFO [2019-12-17 12:37:52,506] jepsen worker 51 - jepsen.util 11661 :invoke :write [17 21] INFO [2019-12-17 12:37:52,524] jepsen worker 78 - jepsen.util 12138 :fail :cas [26 [5 13]] :ValueNotSwapFrom INFO [2019-12-17 12:37:52,524] jepsen worker 78 - jepsen.util 12138 :invoke :read [26 nil] INFO [2019-12-17 12:37:52,530] jepsen worker 54 - jepsen.util 12924 :fail :cas [18 [15 41]] :ValueNotSwapFrom INFO [2019-12-17 12:37:52,531] jepsen worker 54 - jepsen.util 12924 :invoke :write [18 44] INFO [2019-12-17 12:37:52,550] jepsen worker 19 - jepsen.util 12799 :ok :read [6 31] INFO [2019-12-17 12:37:52,550] jepsen worker 19 - jepsen.util 12799 :invoke :cas [6 [14 44]] INFO [2019-12-17 12:37:52,557] jepsen worker 88 - jepsen.util 12598 :ok :read [29 22] INFO [2019-12-17 12:37:52,557] jepsen worker 88 - jepsen.util 12598 :invoke :write [29 49] INFO [2019-12-17 12:37:52,600] jepsen worker 60 - jepsen.util 17970 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:37:52,600] jepsen worker 60 - jepsen.util 17970 :invoke :cas [20 [47 20]] INFO [2019-12-17 12:37:52,600] jepsen worker 19 - jepsen.util 12799 :fail :cas [6 [14 44]] :ValueNotSwapFrom INFO [2019-12-17 12:37:52,600] jepsen worker 19 - jepsen.util 12799 :invoke :read [6 nil] INFO [2019-12-17 12:37:52,618] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:37:52,618] jepsen nemesis - couchbase.util Rebalance check count: 567 INFO [2019-12-17 12:37:52,618] jepsen nemesis - couchbase.util Rebalance stuck count: 548 INFO [2019-12-17 12:37:52,633] jepsen worker 26 - jepsen.util 17666 :info :write [8 11] :RequestCanceledException INFO [2019-12-17 12:37:52,633] jepsen worker 17 - jepsen.util 17117 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:37:52,633] jepsen worker 17 - jepsen.util 17117 :invoke :cas [5 [30 3]] INFO [2019-12-17 12:37:52,637] jepsen worker 26 - jepsen.util 17756 :invoke :write [8 6] INFO [2019-12-17 12:37:52,648] jepsen worker 75 - jepsen.util 16545 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:37:52,648] jepsen worker 75 - jepsen.util 16545 :invoke :cas [25 [44 3]] INFO [2019-12-17 12:37:52,648] jepsen worker 19 - jepsen.util 12799 :ok :read [6 31] INFO [2019-12-17 12:37:52,649] jepsen worker 19 - jepsen.util 12799 :invoke :write [6 45] INFO [2019-12-17 12:37:52,665] jepsen worker 62 - jepsen.util 17702 :info :cas [20 [23 6]] :RequestCanceledException INFO [2019-12-17 12:37:52,672] jepsen worker 62 - jepsen.util 17792 :invoke :read [20 nil] INFO [2019-12-17 12:37:52,718] jepsen worker 89 - jepsen.util 12149 :info :write [29 1] :RequestCanceledException INFO [2019-12-17 12:37:52,720] jepsen worker 89 - jepsen.util 12239 :invoke :write [29 40] INFO [2019-12-17 12:37:52,720] jepsen worker 86 - jepsen.util 17636 :info :write [28 15] :RequestCanceledException INFO [2019-12-17 12:37:52,723] jepsen worker 86 - jepsen.util 17726 :invoke :cas [28 [26 25]] INFO [2019-12-17 12:37:52,747] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72007,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:37:52,747] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72007,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:37:52,748] jepsen worker 70 - jepsen.util 12850 :info :write [23 23] :RequestCanceledException INFO [2019-12-17 12:37:52,750] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72010,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:37:52,750] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72010,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:37:52,750] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72011,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:37:52,751] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72011,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:37:52,756] jepsen worker 70 - jepsen.util 12940 :invoke :cas [23 [25 43]] INFO [2019-12-17 12:37:52,801] jepsen worker 66 - jepsen.util 17076 :info :cas [22 [33 16]] :RequestCanceledException INFO [2019-12-17 12:37:52,803] jepsen worker 66 - jepsen.util 17166 :invoke :cas [22 [5 20]] INFO [2019-12-17 12:37:52,812] jepsen worker 55 - jepsen.util 13195 :fail :cas [18 [43 38]] :ValueNotSwapFrom INFO [2019-12-17 12:37:52,813] jepsen worker 55 - jepsen.util 13195 :invoke :read [18 nil] INFO [2019-12-17 12:37:52,819] jepsen worker 47 - jepsen.util 13457 :info :write [15 27] :RequestCanceledException INFO [2019-12-17 12:37:52,821] jepsen worker 47 - jepsen.util 13547 :invoke :read [15 nil] INFO [2019-12-17 12:37:52,844] jepsen worker 4 - jepsen.util 11614 :fail :cas [1 [28 20]] :ValueNotSwapFrom INFO [2019-12-17 12:37:52,845] jepsen worker 4 - jepsen.util 11614 :invoke :write [1 48] INFO [2019-12-17 12:37:52,889] jepsen worker 3 - jepsen.util 11793 :ok :write [1 6] INFO [2019-12-17 12:37:52,889] jepsen worker 3 - jepsen.util 11793 :invoke :write [1 21] INFO [2019-12-17 12:37:52,893] jepsen worker 47 - jepsen.util 13547 :ok :read [15 38] INFO [2019-12-17 12:37:52,893] jepsen worker 47 - jepsen.util 13547 :invoke :cas [15 [20 34]] INFO [2019-12-17 12:37:52,901] jepsen worker 85 - jepsen.util 17545 :info :write [28 38] :RequestCanceledException INFO [2019-12-17 12:37:52,906] jepsen worker 85 - jepsen.util 17635 :invoke :cas [28 [20 6]] INFO [2019-12-17 12:37:52,918] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72023,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:37:52,919] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72023,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:37:52,920] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72026,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:37:52,920] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72026,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:37:52,928] jepsen worker 69 - jepsen.util 12759 :fail :cas [23 [29 18]] :ValueNotSwapFrom INFO [2019-12-17 12:37:52,928] jepsen worker 69 - jepsen.util 12759 :invoke :read [23 nil] INFO [2019-12-17 12:37:52,933] jepsen worker 37 - jepsen.util 11917 :info :write [12 42] :RequestCanceledException INFO [2019-12-17 12:37:52,933] jepsen worker 23 - jepsen.util 17393 :info :cas [7 [35 10]] :RequestCanceledException INFO [2019-12-17 12:37:52,934] jepsen worker 84 - jepsen.util 16464 :info :cas [28 [46 17]] :RequestCanceledException INFO [2019-12-17 12:37:52,934] jepsen worker 33 - jepsen.util 16773 :info :cas [11 [16 46]] :RequestCanceledException INFO [2019-12-17 12:37:52,935] jepsen worker 67 - jepsen.util 17167 :info :cas [22 [11 26]] :RequestCanceledException INFO [2019-12-17 12:37:52,935] jepsen worker 82 - jepsen.util 17362 :info :write [27 26] :RequestCanceledException INFO [2019-12-17 12:37:52,935] jepsen worker 37 - jepsen.util 12007 :invoke :cas [12 [25 9]] INFO [2019-12-17 12:37:52,937] jepsen worker 33 - jepsen.util 16863 :invoke :read [11 nil] INFO [2019-12-17 12:37:52,938] jepsen worker 82 - jepsen.util 17452 :invoke :read [27 nil] INFO [2019-12-17 12:37:52,940] jepsen worker 23 - jepsen.util 17483 :invoke :write [7 24] INFO [2019-12-17 12:37:52,940] jepsen worker 84 - jepsen.util 16554 :invoke :write [28 8] INFO [2019-12-17 12:37:52,941] jepsen worker 67 - jepsen.util 17257 :invoke :write [22 33] INFO [2019-12-17 12:37:52,948] jepsen worker 12 - jepsen.util 12522 :info :write [4 46] :RequestCanceledException INFO [2019-12-17 12:37:52,958] jepsen worker 12 - jepsen.util 12612 :invoke :read [4 nil] INFO [2019-12-17 12:37:52,959] jepsen worker 47 - jepsen.util 13547 :fail :cas [15 [20 34]] :ValueNotSwapFrom INFO [2019-12-17 12:37:52,959] jepsen worker 47 - jepsen.util 13547 :invoke :read [15 nil] INFO [2019-12-17 12:37:52,962] jepsen worker 71 - jepsen.util 13211 :info :write [23 11] :RequestCanceledException INFO [2019-12-17 12:37:52,967] jepsen worker 71 - jepsen.util 13301 :invoke :read [23 nil] INFO [2019-12-17 12:37:52,988] jepsen worker 51 - jepsen.util 11661 :ok :write [17 21] INFO [2019-12-17 12:37:52,989] jepsen worker 51 - jepsen.util 11661 :invoke :write [17 36] INFO [2019-12-17 12:37:53,009] jepsen worker 47 - jepsen.util 13547 :ok :read [15 38] INFO [2019-12-17 12:37:53,010] jepsen worker 47 - jepsen.util 13547 :invoke :read [15 nil] INFO [2019-12-17 12:37:53,032] jepsen worker 56 - jepsen.util 13286 :ok :write [18 16] INFO [2019-12-17 12:37:53,032] jepsen worker 56 - jepsen.util 13286 :invoke :cas [18 [44 24]] INFO [2019-12-17 12:37:53,034] jepsen worker 52 - jepsen.util 11932 :info :write [17 9] :RequestCanceledException INFO [2019-12-17 12:37:53,035] jepsen worker 53 - jepsen.util 11933 :ok :write [17 8] INFO [2019-12-17 12:37:53,035] jepsen worker 53 - jepsen.util 11933 :invoke :cas [17 [29 26]] INFO [2019-12-17 12:37:53,037] jepsen worker 0 - jepsen.util 17550 :info :write [0 5] :RequestCanceledException INFO [2019-12-17 12:37:53,048] jepsen worker 52 - jepsen.util 12022 :invoke :cas [17 [21 13]] INFO [2019-12-17 12:37:53,050] jepsen worker 0 - jepsen.util 17640 :invoke :read [0 nil] INFO [2019-12-17 12:37:53,071] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72041,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:37:53,071] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72041,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:37:53,071] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72046,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:37:53,071] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72046,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:37:53,078] jepsen worker 5 - jepsen.util 12065 :ok :read [1 6] INFO [2019-12-17 12:37:53,079] jepsen worker 5 - jepsen.util 12065 :invoke :write [1 43] INFO [2019-12-17 12:37:53,085] jepsen worker 47 - jepsen.util 13547 :ok :read [15 38] INFO [2019-12-17 12:37:53,085] jepsen worker 47 - jepsen.util 13547 :invoke :read [15 nil] INFO [2019-12-17 12:37:53,101] jepsen worker 21 - jepsen.util 16491 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:37:53,101] jepsen worker 21 - jepsen.util 16491 :invoke :cas [7 [14 5]] INFO [2019-12-17 12:37:53,132] jepsen worker 55 - jepsen.util 13195 :ok :read [18 16] INFO [2019-12-17 12:37:53,132] jepsen worker 55 - jepsen.util 13195 :invoke :read [18 nil] INFO [2019-12-17 12:37:53,148] jepsen worker 78 - jepsen.util 12138 :ok :read [26 1] INFO [2019-12-17 12:37:53,148] jepsen worker 78 - jepsen.util 12138 :invoke :read [26 nil] INFO [2019-12-17 12:37:53,173] jepsen worker 71 - jepsen.util 13301 :ok :read [23 39] INFO [2019-12-17 12:37:53,173] jepsen worker 71 - jepsen.util 13301 :invoke :write [23 40] INFO [2019-12-17 12:37:53,184] jepsen worker 47 - jepsen.util 13547 :ok :read [15 38] INFO [2019-12-17 12:37:53,184] jepsen worker 47 - jepsen.util 13547 :invoke :write [15 17] INFO [2019-12-17 12:37:53,200] jepsen worker 56 - jepsen.util 13286 :fail :cas [18 [44 24]] :ValueNotSwapFrom INFO [2019-12-17 12:37:53,201] jepsen worker 56 - jepsen.util 13286 :invoke :cas [18 [10 12]] INFO [2019-12-17 12:37:53,209] jepsen worker 80 - jepsen.util 11960 :ok :read [26 1] INFO [2019-12-17 12:37:53,210] jepsen worker 80 - jepsen.util 11960 :invoke :write [26 12] INFO [2019-12-17 12:37:53,224] jepsen worker 69 - jepsen.util 12759 :ok :read [23 39] INFO [2019-12-17 12:37:53,224] jepsen worker 69 - jepsen.util 12759 :invoke :write [23 11] INFO [2019-12-17 12:37:53,264] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72060,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:37:53,264] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72060,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:37:53,264] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72061,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:37:53,264] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72061,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:37:53,264] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72067,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:37:53,265] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72067,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:37:53,265] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72071,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:37:53,265] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72071,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:37:53,267] jepsen worker 70 - jepsen.util 12940 :fail :cas [23 [25 43]] :ValueNotSwapFrom INFO [2019-12-17 12:37:53,267] jepsen worker 70 - jepsen.util 12940 :invoke :cas [23 [46 10]] INFO [2019-12-17 12:37:53,308] jepsen worker 56 - jepsen.util 13286 :fail :cas [18 [10 12]] :ValueNotSwapFrom INFO [2019-12-17 12:37:53,308] jepsen worker 56 - jepsen.util 13286 :invoke :cas [18 [3 20]] INFO [2019-12-17 12:37:53,357] jepsen worker 79 - jepsen.util 12679 :ok :write [26 3] INFO [2019-12-17 12:37:53,357] jepsen worker 79 - jepsen.util 12679 :invoke :read [26 nil] INFO [2019-12-17 12:37:53,392] jepsen worker 55 - jepsen.util 13195 :ok :read [18 16] INFO [2019-12-17 12:37:53,392] jepsen worker 55 - jepsen.util 13195 :invoke :read [18 nil] INFO [2019-12-17 12:37:53,446] jepsen worker 52 - jepsen.util 12022 :fail :cas [17 [21 13]] :ValueNotSwapFrom INFO [2019-12-17 12:37:53,446] jepsen worker 52 - jepsen.util 12022 :invoke :cas [17 [27 5]] INFO [2019-12-17 12:37:53,454] jepsen worker 54 - jepsen.util 12924 :ok :write [18 44] INFO [2019-12-17 12:37:53,454] jepsen worker 54 - jepsen.util 12924 :invoke :read [18 nil] INFO [2019-12-17 12:37:53,459] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72082,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:37:53,460] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72082,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:37:53,460] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72084,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:37:53,460] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72084,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:37:53,495] jepsen worker 5 - jepsen.util 12065 :ok :write [1 43] INFO [2019-12-17 12:37:53,496] jepsen worker 5 - jepsen.util 12065 :invoke :write [1 28] INFO [2019-12-17 12:37:53,501] jepsen worker 55 - jepsen.util 13195 :ok :read [18 16] INFO [2019-12-17 12:37:53,502] jepsen worker 55 - jepsen.util 13195 :invoke :write [18 33] INFO [2019-12-17 12:37:53,516] jepsen worker 53 - jepsen.util 11933 :fail :cas [17 [29 26]] :ValueNotSwapFrom INFO [2019-12-17 12:37:53,517] jepsen worker 53 - jepsen.util 11933 :invoke :write [17 8] INFO [2019-12-17 12:37:53,553] jepsen worker 51 - jepsen.util 11661 :ok :write [17 36] INFO [2019-12-17 12:37:53,554] jepsen worker 51 - jepsen.util 11661 :invoke :write [17 9] INFO [2019-12-17 12:37:53,574] jepsen worker 87 - jepsen.util 12057 :ok :write [29 8] INFO [2019-12-17 12:37:53,575] jepsen worker 87 - jepsen.util 12057 :invoke :write [29 4] INFO [2019-12-17 12:37:53,597] jepsen worker 52 - jepsen.util 12022 :fail :cas [17 [27 5]] :ValueNotSwapFrom INFO [2019-12-17 12:37:53,597] jepsen worker 52 - jepsen.util 12022 :invoke :read [17 nil] INFO [2019-12-17 12:37:53,630] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:37:53,630] jepsen nemesis - couchbase.util Rebalance check count: 568 INFO [2019-12-17 12:37:53,630] jepsen nemesis - couchbase.util Rebalance stuck count: 549 INFO [2019-12-17 12:37:53,633] jepsen worker 78 - jepsen.util 12138 :ok :read [26 3] INFO [2019-12-17 12:37:53,633] jepsen worker 54 - jepsen.util 12924 :ok :read [18 44] INFO [2019-12-17 12:37:53,633] jepsen worker 78 - jepsen.util 12138 :invoke :read [26 nil] INFO [2019-12-17 12:37:53,633] jepsen worker 54 - jepsen.util 12924 :invoke :write [18 14] INFO [2019-12-17 12:37:53,634] jepsen worker 50 - jepsen.util 17060 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:37:53,634] jepsen worker 77 - jepsen.util 18167 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:37:53,635] jepsen worker 16 - jepsen.util 18646 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:37:53,635] jepsen worker 50 - jepsen.util 17060 :invoke :cas [16 [49 33]] INFO [2019-12-17 12:37:53,635] jepsen worker 77 - jepsen.util 18167 :invoke :cas [25 [25 31]] INFO [2019-12-17 12:37:53,635] jepsen worker 16 - jepsen.util 18646 :invoke :write [5 13] INFO [2019-12-17 12:37:53,661] jepsen worker 61 - jepsen.util 18511 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:37:53,661] jepsen worker 61 - jepsen.util 18511 :invoke :cas [20 [46 3]] INFO [2019-12-17 12:37:53,661] jepsen worker 2 - jepsen.util 16292 :info :cas [0 [17 2]] :RequestCanceledException INFO [2019-12-17 12:37:53,665] jepsen worker 2 - jepsen.util 16382 :invoke :cas [0 [8 19]] INFO [2019-12-17 12:37:53,676] jepsen worker 88 - jepsen.util 12598 :ok :write [29 49] INFO [2019-12-17 12:37:53,676] jepsen worker 88 - jepsen.util 12598 :invoke :cas [29 [13 30]] INFO [2019-12-17 12:37:53,700] jepsen worker 14 - jepsen.util 12164 :ok :write [4 17] INFO [2019-12-17 12:37:53,700] jepsen worker 45 - jepsen.util 12195 :info :write [15 44] :RequestCanceledException INFO [2019-12-17 12:37:53,700] jepsen worker 81 - jepsen.util 16731 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:37:53,700] jepsen worker 14 - jepsen.util 12164 :invoke :read [4 nil] INFO [2019-12-17 12:37:53,700] jepsen worker 81 - jepsen.util 16731 :invoke :read [27 nil] INFO [2019-12-17 12:37:53,702] jepsen worker 45 - jepsen.util 12285 :invoke :write [15 45] INFO [2019-12-17 12:37:53,718] jepsen worker 59 - jepsen.util 16169 :info :write [19 45] :RequestCanceledException INFO [2019-12-17 12:37:53,718] jepsen worker 43 - jepsen.util 17233 :info :cas [14 [24 33]] :RequestCanceledException INFO [2019-12-17 12:37:53,718] jepsen worker 32 - jepsen.util 12812 :info :write [10 44] :RequestCanceledException INFO [2019-12-17 12:37:53,718] jepsen worker 8 - jepsen.util 16658 :info :write [2 23] :RequestCanceledException INFO [2019-12-17 12:37:53,718] jepsen worker 30 - jepsen.util 11640 :fail :read [10 nil] :RequestTimeoutException INFO [2019-12-17 12:37:53,719] jepsen worker 30 - jepsen.util 11640 :invoke :write [10 9] INFO [2019-12-17 12:37:53,725] jepsen worker 59 - jepsen.util 16259 :invoke :read [19 nil] INFO [2019-12-17 12:37:53,725] jepsen worker 43 - jepsen.util 17323 :invoke :cas [14 [44 21]] INFO [2019-12-17 12:37:53,727] jepsen worker 8 - jepsen.util 16748 :invoke :read [2 nil] INFO [2019-12-17 12:37:53,729] jepsen worker 32 - jepsen.util 12902 :invoke :cas [10 [32 44]] INFO [2019-12-17 12:37:53,734] jepsen worker 7 - jepsen.util 17017 :info :write [2 29] :RequestCanceledException INFO [2019-12-17 12:37:53,734] jepsen worker 35 - jepsen.util 17045 :info :write [11 49] :RequestCanceledException INFO [2019-12-17 12:37:53,734] jepsen worker 9 - jepsen.util 12969 :info :write [3 46] :RequestCanceledException INFO [2019-12-17 12:37:53,734] jepsen worker 31 - jepsen.util 10561 :info :write [10 37] :RequestCanceledException INFO [2019-12-17 12:37:53,734] jepsen worker 48 - jepsen.util 15708 :info :write [16 40] :RequestCanceledException INFO [2019-12-17 12:37:53,734] jepsen worker 63 - jepsen.util 10953 :info :cas [21 [49 31]] :RequestCanceledException INFO [2019-12-17 12:37:53,735] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72097,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:37:53,735] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72097,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:37:53,735] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72100,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:37:53,735] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72100,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:37:53,737] jepsen worker 63 - jepsen.util 11043 :invoke :write [21 6] INFO [2019-12-17 12:37:53,739] jepsen worker 31 - jepsen.util 10651 :invoke :cas [10 [35 3]] INFO [2019-12-17 12:37:53,739] jepsen worker 7 - jepsen.util 17107 :invoke :cas [2 [16 32]] INFO [2019-12-17 12:37:53,740] jepsen worker 9 - jepsen.util 13059 :invoke :write [3 30] INFO [2019-12-17 12:37:53,742] jepsen worker 48 - jepsen.util 15798 :invoke :cas [16 [36 7]] INFO [2019-12-17 12:37:53,743] jepsen worker 35 - jepsen.util 17135 :invoke :cas [11 [37 26]] INFO [2019-12-17 12:37:53,748] jepsen worker 15 - jepsen.util 16035 :info :write [5 38] :RequestCanceledException INFO [2019-12-17 12:37:53,748] jepsen worker 25 - jepsen.util 18115 :info :write [8 21] :RequestCanceledException INFO [2019-12-17 12:37:53,749] jepsen worker 58 - jepsen.util 17158 :info :write [19 30] :RequestCanceledException INFO [2019-12-17 12:37:53,749] jepsen worker 72 - jepsen.util 12492 :info :write [24 24] :RequestCanceledException INFO [2019-12-17 12:37:53,749] jepsen worker 57 - jepsen.util 16797 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:37:53,749] jepsen worker 57 - jepsen.util 16797 :invoke :cas [19 [18 41]] INFO [2019-12-17 12:37:53,754] jepsen worker 58 - jepsen.util 17248 :invoke :write [19 28] INFO [2019-12-17 12:37:53,758] jepsen worker 25 - jepsen.util 18205 :invoke :read [8 nil] INFO [2019-12-17 12:37:53,758] jepsen worker 15 - jepsen.util 16125 :invoke :write [5 45] INFO [2019-12-17 12:37:53,765] jepsen worker 37 - jepsen.util 12007 :fail :cas [12 [25 9]] :ValueNotSwapFrom INFO [2019-12-17 12:37:53,765] jepsen worker 37 - jepsen.util 12007 :invoke :cas [12 [46 29]] INFO [2019-12-17 12:37:53,765] jepsen worker 44 - jepsen.util 17594 :info :write [14 31] :RequestCanceledException INFO [2019-12-17 12:37:53,767] jepsen worker 72 - jepsen.util 12582 :invoke :read [24 nil] INFO [2019-12-17 12:37:53,773] jepsen worker 44 - jepsen.util 17684 :invoke :cas [14 [6 22]] INFO [2019-12-17 12:37:53,844] jepsen worker 12 - jepsen.util 12612 :ok :read [4 17] INFO [2019-12-17 12:37:53,845] jepsen worker 12 - jepsen.util 12612 :invoke :read [4 nil] INFO [2019-12-17 12:37:53,872] jepsen worker 70 - jepsen.util 12940 :fail :cas [23 [46 10]] :ValueNotSwapFrom INFO [2019-12-17 12:37:53,872] jepsen worker 70 - jepsen.util 12940 :invoke :write [23 16] INFO [2019-12-17 12:37:53,895] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72130,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:37:53,895] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72130,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:37:53,895] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72134,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:37:53,895] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72134,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:37:53,901] jepsen worker 11 - jepsen.util 11981 :info :cas [3 [40 32]] :RequestCanceledException INFO [2019-12-17 12:37:53,902] jepsen worker 63 - jepsen.util 11043 :ok :write [21 6] INFO [2019-12-17 12:37:53,902] jepsen worker 63 - jepsen.util 11043 :invoke :cas [21 [36 45]] INFO [2019-12-17 12:37:53,904] jepsen worker 11 - jepsen.util 12071 :invoke :cas [3 [4 11]] INFO [2019-12-17 12:37:53,918] jepsen worker 46 - jepsen.util 12106 :fail :read [15 nil] :RequestTimeoutException INFO [2019-12-17 12:37:53,918] jepsen worker 42 - jepsen.util 17862 :info :write [14 26] :RequestCanceledException INFO [2019-12-17 12:37:53,919] jepsen worker 46 - jepsen.util 12106 :invoke :write [15 27] INFO [2019-12-17 12:37:53,924] jepsen worker 42 - jepsen.util 17952 :invoke :read [14 nil] INFO [2019-12-17 12:37:53,938] jepsen worker 36 - jepsen.util 13176 :info :cas [12 [36 39]] :RequestCanceledException INFO [2019-12-17 12:37:53,939] jepsen worker 36 - jepsen.util 13266 :invoke :cas [12 [17 14]] INFO [2019-12-17 12:37:53,943] jepsen worker 27 - jepsen.util 11547 :ok :write [9 21] INFO [2019-12-17 12:37:53,943] jepsen worker 27 - jepsen.util 11547 :invoke :cas [9 [28 25]] INFO [2019-12-17 12:37:53,955] jepsen worker 56 - jepsen.util 13286 :fail :cas [18 [3 20]] :ValueNotSwapFrom INFO [2019-12-17 12:37:53,955] jepsen worker 10 - jepsen.util 12250 :info :cas [3 [35 37]] :RequestCanceledException INFO [2019-12-17 12:37:53,956] jepsen worker 56 - jepsen.util 13286 :invoke :read [18 nil] INFO [2019-12-17 12:37:53,956] jepsen worker 74 - jepsen.util 12134 :info :write [24 28] :SyncWriteAmbiguous INFO [2019-12-17 12:37:53,958] jepsen worker 74 - jepsen.util 12224 :invoke :write [24 49] INFO [2019-12-17 12:37:53,961] jepsen worker 10 - jepsen.util 12340 :invoke :read [3 nil] INFO [2019-12-17 12:37:53,972] jepsen worker 79 - jepsen.util 12679 :ok :read [26 3] INFO [2019-12-17 12:37:53,972] jepsen worker 79 - jepsen.util 12679 :invoke :read [26 nil] INFO [2019-12-17 12:37:53,997] jepsen worker 63 - jepsen.util 11043 :fail :cas [21 [36 45]] :ValueNotSwapFrom INFO [2019-12-17 12:37:53,997] jepsen worker 63 - jepsen.util 11043 :invoke :cas [21 [24 11]] INFO [2019-12-17 12:37:54,005] jepsen worker 37 - jepsen.util 12007 :fail :cas [12 [46 29]] :ValueNotSwapFrom INFO [2019-12-17 12:37:54,005] jepsen worker 37 - jepsen.util 12007 :invoke :cas [12 [28 26]] INFO [2019-12-17 12:37:54,018] jepsen worker 1 - jepsen.util 17281 :info :cas [0 [11 46]] :RequestCanceledException INFO [2019-12-17 12:37:54,023] jepsen worker 1 - jepsen.util 17371 :invoke :read [0 nil] INFO [2019-12-17 12:37:54,024] jepsen worker 54 - jepsen.util 12924 :ok :write [18 14] INFO [2019-12-17 12:37:54,024] jepsen worker 54 - jepsen.util 12924 :invoke :write [18 18] INFO [2019-12-17 12:37:54,033] jepsen worker 83 - jepsen.util 16913 :info :write [27 9] :RequestCanceledException INFO [2019-12-17 12:37:54,033] jepsen worker 22 - jepsen.util 17122 :info :write [7 47] :RequestCanceledException INFO [2019-12-17 12:37:54,049] jepsen worker 83 - jepsen.util 17003 :invoke :write [27 5] INFO [2019-12-17 12:37:54,049] jepsen worker 22 - jepsen.util 17212 :invoke :write [7 38] INFO [2019-12-17 12:37:54,049] jepsen worker 39 - jepsen.util 16599 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:37:54,049] jepsen worker 39 - jepsen.util 16599 :invoke :cas [13 [13 19]] INFO [2019-12-17 12:37:54,053] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72150,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:37:54,054] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72150,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:37:54,056] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72152,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:37:54,056] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72152,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:37:54,062] jepsen worker 64 - jepsen.util 9694 :info :cas [21 [49 28]] :RequestCanceledException INFO [2019-12-17 12:37:54,067] jepsen worker 64 - jepsen.util 9784 :invoke :read [21 nil] INFO [2019-12-17 12:37:54,068] jepsen worker 88 - jepsen.util 12598 :fail :cas [29 [13 30]] :ValueNotSwapFrom INFO [2019-12-17 12:37:54,069] jepsen worker 88 - jepsen.util 12598 :invoke :read [29 nil] INFO [2019-12-17 12:37:54,088] jepsen worker 52 - jepsen.util 12022 :ok :read [17 36] INFO [2019-12-17 12:37:54,088] jepsen worker 52 - jepsen.util 12022 :invoke :cas [17 [7 18]] INFO [2019-12-17 12:37:54,106] jepsen worker 63 - jepsen.util 11043 :fail :cas [21 [24 11]] :ValueNotSwapFrom INFO [2019-12-17 12:37:54,106] jepsen worker 63 - jepsen.util 11043 :invoke :cas [21 [33 17]] INFO [2019-12-17 12:37:54,111] jepsen worker 32 - jepsen.util 12902 :fail :cas [10 [32 44]] :ValueNotSwapFrom INFO [2019-12-17 12:37:54,112] jepsen worker 32 - jepsen.util 12902 :invoke :write [10 48] INFO [2019-12-17 12:37:54,115] jepsen worker 18 - jepsen.util 12978 :fail :cas [6 [34 19]] :ValueNotSwapFrom INFO [2019-12-17 12:37:54,115] jepsen worker 18 - jepsen.util 12978 :invoke :write [6 8] INFO [2019-12-17 12:37:54,118] jepsen worker 20 - jepsen.util 12170 :info :write [6 15] :RequestCanceledException INFO [2019-12-17 12:37:54,123] jepsen worker 20 - jepsen.util 12260 :invoke :cas [6 [16 11]] INFO [2019-12-17 12:37:54,131] jepsen worker 14 - jepsen.util 12164 :ok :read [4 17] INFO [2019-12-17 12:37:54,131] jepsen worker 14 - jepsen.util 12164 :invoke :cas [4 [10 15]] INFO [2019-12-17 12:37:54,134] jepsen worker 76 - jepsen.util 17176 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:37:54,134] jepsen worker 76 - jepsen.util 17176 :invoke :read [25 nil] INFO [2019-12-17 12:37:54,165] jepsen worker 29 - jepsen.util 10919 :fail :read [9 nil] :RequestTimeoutException INFO [2019-12-17 12:37:54,165] jepsen worker 29 - jepsen.util 10919 :invoke :cas [9 [16 25]] INFO [2019-12-17 12:37:54,177] jepsen worker 78 - jepsen.util 12138 :ok :read [26 3] INFO [2019-12-17 12:37:54,177] jepsen worker 78 - jepsen.util 12138 :invoke :write [26 7] INFO [2019-12-17 12:37:54,188] jepsen worker 11 - jepsen.util 12071 :fail :cas [3 [4 11]] :ValueNotSwapFrom INFO [2019-12-17 12:37:54,188] jepsen worker 11 - jepsen.util 12071 :invoke :write [3 29] INFO [2019-12-17 12:37:54,199] jepsen worker 80 - jepsen.util 11960 :ok :write [26 12] INFO [2019-12-17 12:37:54,199] jepsen worker 80 - jepsen.util 11960 :invoke :cas [26 [37 21]] INFO [2019-12-17 12:37:54,204] jepsen worker 63 - jepsen.util 11043 :fail :cas [21 [33 17]] :ValueNotSwapFrom INFO [2019-12-17 12:37:54,204] jepsen worker 63 - jepsen.util 11043 :invoke :read [21 nil] INFO [2019-12-17 12:37:54,218] jepsen worker 68 - jepsen.util 16178 :info :write [22 26] :RequestCanceledException INFO [2019-12-17 12:37:54,225] jepsen worker 68 - jepsen.util 16268 :invoke :cas [22 [28 10]] INFO [2019-12-17 12:37:54,235] jepsen worker 28 - jepsen.util 10648 :info :cas [9 [7 2]] :RequestCanceledException INFO [2019-12-17 12:37:54,235] jepsen worker 49 - jepsen.util 16969 :info :cas [16 [6 36]] :RequestCanceledException INFO [2019-12-17 12:37:54,235] jepsen worker 6 - jepsen.util 17376 :info :write [2 31] :RequestCanceledException INFO [2019-12-17 12:37:54,237] jepsen worker 28 - jepsen.util 10738 :invoke :write [9 7] INFO [2019-12-17 12:37:54,237] jepsen worker 49 - jepsen.util 17059 :invoke :write [16 34] INFO [2019-12-17 12:37:54,241] jepsen worker 6 - jepsen.util 17466 :invoke :read [2 nil] INFO [2019-12-17 12:37:54,253] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72170,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:37:54,253] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72170,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:37:54,253] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72180,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:37:54,253] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72171,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:37:54,253] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72180,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:37:54,253] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72171,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:37:54,253] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72183,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:37:54,254] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72172,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:37:54,254] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72172,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:37:54,254] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72183,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:37:54,255] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72184,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:37:54,256] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72184,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:37:54,292] jepsen worker 63 - jepsen.util 11043 :ok :read [21 6] INFO [2019-12-17 12:37:54,292] jepsen worker 63 - jepsen.util 11043 :invoke :write [21 8] INFO [2019-12-17 12:37:54,301] jepsen worker 65 - jepsen.util 10955 :fail :read [21 nil] :RequestTimeoutException INFO [2019-12-17 12:37:54,301] jepsen worker 65 - jepsen.util 10955 :invoke :read [21 nil] INFO [2019-12-17 12:37:54,317] jepsen worker 79 - jepsen.util 12679 :ok :read [26 12] INFO [2019-12-17 12:37:54,318] jepsen worker 79 - jepsen.util 12679 :invoke :write [26 9] INFO [2019-12-17 12:37:54,334] jepsen worker 24 - jepsen.util 16224 :info :cas [8 [22 0]] :RequestCanceledException INFO [2019-12-17 12:37:54,334] jepsen worker 73 - jepsen.util 11503 :info :write [24 29] :RequestCanceledException INFO [2019-12-17 12:37:54,340] jepsen worker 73 - jepsen.util 11593 :invoke :write [24 0] INFO [2019-12-17 12:37:54,344] jepsen worker 24 - jepsen.util 16314 :invoke :cas [8 [12 1]] INFO [2019-12-17 12:37:54,348] jepsen worker 40 - jepsen.util 17860 :info :write [13 28] :RequestCanceledException INFO [2019-12-17 12:37:54,349] jepsen worker 56 - jepsen.util 13286 :ok :read [18 14] INFO [2019-12-17 12:37:54,350] jepsen worker 56 - jepsen.util 13286 :invoke :read [18 nil] INFO [2019-12-17 12:37:54,354] jepsen worker 40 - jepsen.util 17950 :invoke :write [13 28] INFO [2019-12-17 12:37:54,366] jepsen worker 41 - jepsen.util 18401 :info :write [13 20] :RequestCanceledException INFO [2019-12-17 12:37:54,366] jepsen worker 14 - jepsen.util 12164 :fail :cas [4 [10 15]] :ValueNotSwapFrom INFO [2019-12-17 12:37:54,366] jepsen worker 14 - jepsen.util 12164 :invoke :write [4 37] INFO [2019-12-17 12:37:54,374] jepsen worker 41 - jepsen.util 18491 :invoke :write [13 43] INFO [2019-12-17 12:37:54,388] jepsen worker 63 - jepsen.util 11043 :ok :write [21 8] INFO [2019-12-17 12:37:54,388] jepsen worker 63 - jepsen.util 11043 :invoke :cas [21 [15 13]] INFO [2019-12-17 12:37:54,401] jepsen worker 13 - jepsen.util 11893 :info :write [4 4] :RequestCanceledException INFO [2019-12-17 12:37:54,401] jepsen worker 34 - jepsen.util 17854 :info :write [11 2] :RequestCanceledException INFO [2019-12-17 12:37:54,407] jepsen worker 13 - jepsen.util 11983 :invoke :write [4 21] INFO [2019-12-17 12:37:54,410] jepsen worker 34 - jepsen.util 17944 :invoke :write [11 3] INFO [2019-12-17 12:37:54,419] jepsen worker 38 - jepsen.util 12998 :info :write [12 26] :RequestCanceledException INFO [2019-12-17 12:37:54,421] jepsen worker 38 - jepsen.util 13088 :invoke :cas [12 [40 19]] INFO [2019-12-17 12:37:54,440] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72201,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:37:54,441] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72201,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:37:54,441] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72206,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:37:54,441] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72206,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:37:54,467] jepsen worker 63 - jepsen.util 11043 :fail :cas [21 [15 13]] :ValueNotSwapFrom INFO [2019-12-17 12:37:54,467] jepsen worker 63 - jepsen.util 11043 :invoke :cas [21 [45 41]] INFO [2019-12-17 12:37:54,473] jepsen worker 88 - jepsen.util 12598 :ok :read [29 49] INFO [2019-12-17 12:37:54,473] jepsen worker 88 - jepsen.util 12598 :invoke :read [29 nil] INFO [2019-12-17 12:37:54,519] jepsen worker 63 - jepsen.util 11043 :fail :cas [21 [45 41]] :ValueNotSwapFrom INFO [2019-12-17 12:37:54,519] jepsen worker 19 - jepsen.util 12799 :info :write [6 45] :SyncWriteAmbiguous INFO [2019-12-17 12:37:54,519] jepsen worker 63 - jepsen.util 11043 :invoke :cas [21 [27 16]] INFO [2019-12-17 12:37:54,523] jepsen worker 19 - jepsen.util 12889 :invoke :read [6 nil] INFO [2019-12-17 12:37:54,565] jepsen worker 12 - jepsen.util 12612 :ok :read [4 17] INFO [2019-12-17 12:37:54,566] jepsen worker 12 - jepsen.util 12612 :invoke :write [4 38] INFO [2019-12-17 12:37:54,582] jepsen worker 36 - jepsen.util 13266 :fail :cas [12 [17 14]] :ValueNotSwapFrom INFO [2019-12-17 12:37:54,582] jepsen worker 36 - jepsen.util 13266 :invoke :cas [12 [13 39]] INFO [2019-12-17 12:37:54,597] jepsen worker 28 - jepsen.util 10738 :ok :write [9 7] INFO [2019-12-17 12:37:54,597] jepsen worker 28 - jepsen.util 10738 :invoke :cas [9 [13 40]] INFO [2019-12-17 12:37:54,613] jepsen worker 63 - jepsen.util 11043 :fail :cas [21 [27 16]] :ValueNotSwapFrom INFO [2019-12-17 12:37:54,613] jepsen worker 63 - jepsen.util 11043 :invoke :read [21 nil] INFO [2019-12-17 12:37:54,645] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:37:54,645] jepsen nemesis - couchbase.util Rebalance check count: 569 INFO [2019-12-17 12:37:54,645] jepsen nemesis - couchbase.util Rebalance stuck count: 550 INFO [2019-12-17 12:37:54,651] jepsen worker 52 - jepsen.util 12022 :fail :cas [17 [7 18]] :ValueNotSwapFrom INFO [2019-12-17 12:37:54,651] jepsen worker 52 - jepsen.util 12022 :invoke :read [17 nil] INFO [2019-12-17 12:37:54,658] jepsen worker 28 - jepsen.util 10738 :fail :cas [9 [13 40]] :ValueNotSwapFrom INFO [2019-12-17 12:37:54,659] jepsen worker 28 - jepsen.util 10738 :invoke :write [9 10] INFO [2019-12-17 12:37:54,664] jepsen worker 37 - jepsen.util 12007 :fail :cas [12 [28 26]] :ValueNotSwapFrom INFO [2019-12-17 12:37:54,664] jepsen worker 37 - jepsen.util 12007 :invoke :read [12 nil] INFO [2019-12-17 12:37:54,675] jepsen worker 63 - jepsen.util 11043 :ok :read [21 8] INFO [2019-12-17 12:37:54,676] jepsen worker 63 - jepsen.util 11043 :invoke :write [21 20] INFO [2019-12-17 12:37:54,693] jepsen worker 36 - jepsen.util 13266 :fail :cas [12 [13 39]] :ValueNotSwapFrom INFO [2019-12-17 12:37:54,693] jepsen worker 36 - jepsen.util 13266 :invoke :cas [12 [17 17]] INFO [2019-12-17 12:37:54,698] jepsen worker 62 - jepsen.util 17792 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:37:54,698] jepsen worker 60 - jepsen.util 17970 :info :cas [20 [47 20]] :RequestCanceledException INFO [2019-12-17 12:37:54,698] jepsen worker 62 - jepsen.util 17792 :invoke :read [20 nil] INFO [2019-12-17 12:37:54,703] jepsen worker 60 - jepsen.util 18060 :invoke :cas [20 [49 38]] INFO [2019-12-17 12:37:54,721] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72217,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:37:54,721] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72217,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:37:54,723] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72221,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:37:54,723] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72221,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:37:54,733] jepsen worker 26 - jepsen.util 17756 :info :write [8 6] :RequestCanceledException INFO [2019-12-17 12:37:54,733] jepsen worker 17 - jepsen.util 17117 :info :cas [5 [30 3]] :RequestCanceledException INFO [2019-12-17 12:37:54,734] jepsen worker 86 - jepsen.util 17726 :info :cas [28 [26 25]] :RequestCanceledException INFO [2019-12-17 12:37:54,739] jepsen worker 17 - jepsen.util 17207 :invoke :cas [5 [0 13]] INFO [2019-12-17 12:37:54,740] jepsen worker 86 - jepsen.util 17816 :invoke :read [28 nil] INFO [2019-12-17 12:37:54,741] jepsen worker 26 - jepsen.util 17846 :invoke :write [8 4] INFO [2019-12-17 12:37:54,749] jepsen worker 75 - jepsen.util 16545 :info :cas [25 [44 3]] :RequestCanceledException INFO [2019-12-17 12:37:54,756] jepsen worker 75 - jepsen.util 16635 :invoke :read [25 nil] INFO [2019-12-17 12:37:54,768] jepsen worker 71 - jepsen.util 13301 :ok :write [23 40] INFO [2019-12-17 12:37:54,769] jepsen worker 71 - jepsen.util 13301 :invoke :write [23 37] INFO [2019-12-17 12:37:54,770] jepsen worker 38 - jepsen.util 13088 :fail :cas [12 [40 19]] :ValueNotSwapFrom INFO [2019-12-17 12:37:54,770] jepsen worker 38 - jepsen.util 13088 :invoke :read [12 nil] INFO [2019-12-17 12:37:54,778] jepsen worker 36 - jepsen.util 13266 :fail :cas [12 [17 17]] :ValueNotSwapFrom INFO [2019-12-17 12:37:54,778] jepsen worker 36 - jepsen.util 13266 :invoke :cas [12 [37 29]] INFO [2019-12-17 12:37:54,787] jepsen worker 28 - jepsen.util 10738 :ok :write [9 10] INFO [2019-12-17 12:37:54,788] jepsen worker 28 - jepsen.util 10738 :invoke :cas [9 [33 48]] INFO [2019-12-17 12:37:54,792] jepsen worker 80 - jepsen.util 11960 :fail :cas [26 [37 21]] :ValueNotSwapFrom INFO [2019-12-17 12:37:54,792] jepsen worker 80 - jepsen.util 11960 :invoke :read [26 nil] INFO [2019-12-17 12:37:54,817] jepsen worker 89 - jepsen.util 12239 :info :write [29 40] :RequestCanceledException INFO [2019-12-17 12:37:54,825] jepsen worker 89 - jepsen.util 12329 :invoke :write [29 26] INFO [2019-12-17 12:37:54,826] jepsen worker 63 - jepsen.util 11043 :ok :write [21 20] INFO [2019-12-17 12:37:54,826] jepsen worker 63 - jepsen.util 11043 :invoke :read [21 nil] INFO [2019-12-17 12:37:54,846] jepsen worker 28 - jepsen.util 10738 :fail :cas [9 [33 48]] :ValueNotSwapFrom INFO [2019-12-17 12:37:54,846] jepsen worker 28 - jepsen.util 10738 :invoke :write [9 18] INFO [2019-12-17 12:37:54,862] jepsen worker 66 - jepsen.util 17166 :info :cas [22 [5 20]] :RequestCanceledException INFO [2019-12-17 12:37:54,866] jepsen worker 66 - jepsen.util 17256 :invoke :cas [22 [21 1]] INFO [2019-12-17 12:37:54,881] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72241,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:37:54,882] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72241,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:37:54,882] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72246,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:37:54,882] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72246,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:37:54,890] jepsen worker 63 - jepsen.util 11043 :ok :read [21 20] INFO [2019-12-17 12:37:54,892] jepsen worker 63 - jepsen.util 11043 :invoke :write [21 49] INFO [2019-12-17 12:37:54,917] jepsen worker 4 - jepsen.util 11614 :info :write [1 48] :RequestCanceledException INFO [2019-12-17 12:37:54,924] jepsen worker 4 - jepsen.util 11704 :invoke :write [1 20] INFO [2019-12-17 12:37:54,940] jepsen worker 88 - jepsen.util 12598 :ok :read [29 49] INFO [2019-12-17 12:37:54,940] jepsen worker 88 - jepsen.util 12598 :invoke :read [29 nil] INFO [2019-12-17 12:37:54,942] jepsen worker 56 - jepsen.util 13286 :ok :read [18 14] INFO [2019-12-17 12:37:54,942] jepsen worker 56 - jepsen.util 13286 :invoke :read [18 nil] INFO [2019-12-17 12:37:54,949] jepsen worker 28 - jepsen.util 10738 :ok :write [9 18] INFO [2019-12-17 12:37:54,949] jepsen worker 28 - jepsen.util 10738 :invoke :read [9 nil] INFO [2019-12-17 12:37:54,972] jepsen worker 55 - jepsen.util 13195 :ok :write [18 33] INFO [2019-12-17 12:37:54,972] jepsen worker 55 - jepsen.util 13195 :invoke :read [18 nil] INFO [2019-12-17 12:37:54,998] jepsen worker 63 - jepsen.util 11043 :ok :write [21 49] INFO [2019-12-17 12:37:54,998] jepsen worker 63 - jepsen.util 11043 :invoke :cas [21 [42 37]] INFO [2019-12-17 12:37:55,001] jepsen worker 3 - jepsen.util 11793 :info :write [1 21] :RequestCanceledException INFO [2019-12-17 12:37:55,002] jepsen worker 85 - jepsen.util 17635 :info :cas [28 [20 6]] :RequestCanceledException INFO [2019-12-17 12:37:55,002] jepsen worker 67 - jepsen.util 17257 :info :write [22 33] :RequestCanceledException INFO [2019-12-17 12:37:55,003] jepsen worker 3 - jepsen.util 11883 :invoke :read [1 nil] INFO [2019-12-17 12:37:55,006] jepsen worker 85 - jepsen.util 17725 :invoke :cas [28 [7 38]] INFO [2019-12-17 12:37:55,007] jepsen worker 67 - jepsen.util 17347 :invoke :read [22 nil] INFO [2019-12-17 12:37:55,017] jepsen worker 53 - jepsen.util 11933 :ok :write [17 8] INFO [2019-12-17 12:37:55,017] jepsen worker 53 - jepsen.util 11933 :invoke :cas [17 [2 1]] INFO [2019-12-17 12:37:55,032] jepsen worker 28 - jepsen.util 10738 :ok :read [9 18] INFO [2019-12-17 12:37:55,032] jepsen worker 28 - jepsen.util 10738 :invoke :read [9 nil] INFO [2019-12-17 12:37:55,034] jepsen worker 84 - jepsen.util 16554 :info :write [28 8] :RequestCanceledException INFO [2019-12-17 12:37:55,034] jepsen worker 33 - jepsen.util 16863 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:37:55,034] jepsen worker 33 - jepsen.util 16863 :invoke :write [11 0] INFO [2019-12-17 12:37:55,039] jepsen worker 84 - jepsen.util 16644 :invoke :write [28 39] INFO [2019-12-17 12:37:55,041] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72255,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:37:55,041] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72255,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:37:55,041] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72259,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:37:55,041] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72259,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:37:55,041] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72260,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:37:55,042] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72260,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:37:55,048] jepsen worker 82 - jepsen.util 17452 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:37:55,048] jepsen worker 82 - jepsen.util 17452 :invoke :write [27 32] INFO [2019-12-17 12:37:55,065] jepsen worker 23 - jepsen.util 17483 :info :write [7 24] :RequestCanceledException INFO [2019-12-17 12:37:55,065] jepsen worker 0 - jepsen.util 17640 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:37:55,065] jepsen worker 0 - jepsen.util 17640 :invoke :read [0 nil] INFO [2019-12-17 12:37:55,066] jepsen worker 63 - jepsen.util 11043 :fail :cas [21 [42 37]] :ValueNotSwapFrom INFO [2019-12-17 12:37:55,067] jepsen worker 69 - jepsen.util 12759 :ok :write [23 11] INFO [2019-12-17 12:37:55,067] jepsen worker 69 - jepsen.util 12759 :invoke :cas [23 [13 24]] INFO [2019-12-17 12:37:55,069] jepsen worker 63 - jepsen.util 11043 :invoke :cas [21 [39 33]] INFO [2019-12-17 12:37:55,075] jepsen worker 23 - jepsen.util 17573 :invoke :cas [7 [13 15]] INFO [2019-12-17 12:37:55,122] jepsen worker 88 - jepsen.util 12598 :ok :read [29 49] INFO [2019-12-17 12:37:55,122] jepsen worker 88 - jepsen.util 12598 :invoke :write [29 23] INFO [2019-12-17 12:37:55,122] jepsen worker 4 - jepsen.util 11704 :ok :write [1 20] INFO [2019-12-17 12:37:55,123] jepsen worker 4 - jepsen.util 11704 :invoke :write [1 45] INFO [2019-12-17 12:37:55,124] jepsen worker 28 - jepsen.util 10738 :ok :read [9 18] INFO [2019-12-17 12:37:55,124] jepsen worker 28 - jepsen.util 10738 :invoke :cas [9 [22 33]] INFO [2019-12-17 12:37:55,134] jepsen worker 63 - jepsen.util 11043 :fail :cas [21 [39 33]] :ValueNotSwapFrom INFO [2019-12-17 12:37:55,135] jepsen worker 63 - jepsen.util 11043 :invoke :write [21 3] INFO [2019-12-17 12:37:55,185] jepsen worker 14 - jepsen.util 12164 :ok :write [4 37] INFO [2019-12-17 12:37:55,185] jepsen worker 14 - jepsen.util 12164 :invoke :write [4 16] INFO [2019-12-17 12:37:55,192] jepsen worker 28 - jepsen.util 10738 :fail :cas [9 [22 33]] :ValueNotSwapFrom INFO [2019-12-17 12:37:55,192] jepsen worker 28 - jepsen.util 10738 :invoke :cas [9 [16 10]] INFO [2019-12-17 12:37:55,200] jepsen worker 21 - jepsen.util 16491 :info :cas [7 [14 5]] :RequestCanceledException INFO [2019-12-17 12:37:55,203] jepsen worker 21 - jepsen.util 16581 :invoke :read [7 nil] INFO [2019-12-17 12:37:55,210] jepsen worker 63 - jepsen.util 11043 :ok :write [21 3] INFO [2019-12-17 12:37:55,210] jepsen worker 63 - jepsen.util 11043 :invoke :cas [21 [40 6]] INFO [2019-12-17 12:37:55,234] jepsen worker 47 - jepsen.util 13547 :info :write [15 17] :RequestCanceledException INFO [2019-12-17 12:37:55,236] jepsen worker 47 - jepsen.util 13637 :invoke :cas [15 [37 47]] INFO [2019-12-17 12:37:55,237] jepsen worker 28 - jepsen.util 10738 :fail :cas [9 [16 10]] :ValueNotSwapFrom INFO [2019-12-17 12:37:55,237] jepsen worker 28 - jepsen.util 10738 :invoke :cas [9 [36 24]] INFO [2019-12-17 12:37:55,238] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72290,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:37:55,239] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72278,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:37:55,239] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72290,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:37:55,239] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72278,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:37:55,239] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72292,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:37:55,239] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72279,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:37:55,239] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72292,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:37:55,239] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72279,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:37:55,272] jepsen worker 63 - jepsen.util 11043 :fail :cas [21 [40 6]] :ValueNotSwapFrom INFO [2019-12-17 12:37:55,272] jepsen worker 63 - jepsen.util 11043 :invoke :read [21 nil] INFO [2019-12-17 12:37:55,301] jepsen worker 47 - jepsen.util 13637 :fail :cas [15 [37 47]] :ValueNotSwapFrom INFO [2019-12-17 12:37:55,302] jepsen worker 47 - jepsen.util 13637 :invoke :cas [15 [21 43]] INFO [2019-12-17 12:37:55,490] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72300,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:37:55,490] jepsen worker 28 - jepsen.util 10738 :fail :cas [9 [36 24]] :ValueNotSwapFrom INFO [2019-12-17 12:37:55,490] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72300,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:37:55,490] jepsen worker 28 - jepsen.util 10738 :invoke :write [9 40] INFO [2019-12-17 12:37:55,495] jepsen worker 63 - jepsen.util 11043 :ok :read [21 3] INFO [2019-12-17 12:37:55,496] jepsen worker 56 - jepsen.util 13286 :ok :read [18 33] INFO [2019-12-17 12:37:55,496] jepsen worker 63 - jepsen.util 11043 :invoke :read [21 nil] INFO [2019-12-17 12:37:55,496] jepsen worker 56 - jepsen.util 13286 :invoke :cas [18 [43 34]] INFO [2019-12-17 12:37:55,496] jepsen worker 47 - jepsen.util 13637 :fail :cas [15 [21 43]] :ValueNotSwapFrom INFO [2019-12-17 12:37:55,496] jepsen worker 47 - jepsen.util 13637 :invoke :cas [15 [41 0]] INFO [2019-12-17 12:37:55,500] jepsen worker 69 - jepsen.util 12759 :fail :cas [23 [13 24]] :ValueNotSwapFrom INFO [2019-12-17 12:37:55,500] jepsen worker 69 - jepsen.util 12759 :invoke :cas [23 [7 39]] INFO [2019-12-17 12:37:55,501] jepsen worker 53 - jepsen.util 11933 :fail :cas [17 [2 1]] :ValueNotSwapFrom INFO [2019-12-17 12:37:55,501] jepsen worker 53 - jepsen.util 11933 :invoke :read [17 nil] INFO [2019-12-17 12:37:55,505] jepsen worker 52 - jepsen.util 12022 :ok :read [17 8] INFO [2019-12-17 12:37:55,505] jepsen worker 52 - jepsen.util 12022 :invoke :cas [17 [3 9]] INFO [2019-12-17 12:37:55,524] jepsen worker 80 - jepsen.util 11960 :ok :read [26 12] INFO [2019-12-17 12:37:55,524] jepsen worker 80 - jepsen.util 11960 :invoke :cas [26 [2 12]] INFO [2019-12-17 12:37:55,529] jepsen worker 47 - jepsen.util 13637 :fail :cas [15 [41 0]] :ValueNotSwapFrom INFO [2019-12-17 12:37:55,529] jepsen worker 47 - jepsen.util 13637 :invoke :read [15 nil] INFO [2019-12-17 12:37:55,534] jepsen worker 63 - jepsen.util 11043 :ok :read [21 3] INFO [2019-12-17 12:37:55,534] jepsen worker 63 - jepsen.util 11043 :invoke :cas [21 [35 21]] INFO [2019-12-17 12:37:55,548] jepsen worker 5 - jepsen.util 12065 :info :write [1 28] :RequestCanceledException INFO [2019-12-17 12:37:55,549] jepsen worker 5 - jepsen.util 12155 :invoke :write [1 11] INFO [2019-12-17 12:37:55,550] jepsen worker 47 - jepsen.util 13637 :ok :read [15 38] INFO [2019-12-17 12:37:55,550] jepsen worker 47 - jepsen.util 13637 :invoke :write [15 5] INFO [2019-12-17 12:37:55,579] jepsen worker 63 - jepsen.util 11043 :fail :cas [21 [35 21]] :ValueNotSwapFrom INFO [2019-12-17 12:37:55,580] jepsen worker 63 - jepsen.util 11043 :invoke :write [21 16] INFO [2019-12-17 12:37:55,610] jepsen worker 12 - jepsen.util 12612 :ok :write [4 38] INFO [2019-12-17 12:37:55,610] jepsen worker 12 - jepsen.util 12612 :invoke :cas [4 [39 22]] INFO [2019-12-17 12:37:55,615] jepsen worker 28 - jepsen.util 10738 :ok :write [9 40] INFO [2019-12-17 12:37:55,616] jepsen worker 28 - jepsen.util 10738 :invoke :cas [9 [14 2]] INFO [2019-12-17 12:37:55,633] jepsen worker 28 - jepsen.util 10738 :fail :cas [9 [14 2]] :ValueNotSwapFrom INFO [2019-12-17 12:37:55,633] jepsen worker 28 - jepsen.util 10738 :invoke :cas [9 [9 37]] INFO [2019-12-17 12:37:55,634] jepsen worker 87 - jepsen.util 12057 :info :write [29 4] :RequestCanceledException INFO [2019-12-17 12:37:55,634] jepsen worker 87 - jepsen.util 12147 :invoke :read [29 nil] INFO [2019-12-17 12:37:55,651] jepsen worker 28 - jepsen.util 10738 :fail :cas [9 [9 37]] :ValueNotSwapFrom INFO [2019-12-17 12:37:55,652] jepsen worker 28 - jepsen.util 10738 :invoke :cas [9 [37 19]] INFO [2019-12-17 12:37:55,652] jepsen worker 88 - jepsen.util 12598 :ok :write [29 23] INFO [2019-12-17 12:37:55,653] jepsen worker 88 - jepsen.util 12598 :invoke :cas [29 [49 47]] INFO [2019-12-17 12:37:55,654] jepsen worker 63 - jepsen.util 11043 :ok :write [21 16] INFO [2019-12-17 12:37:55,655] jepsen worker 63 - jepsen.util 11043 :invoke :write [21 1] INFO [2019-12-17 12:37:55,661] jepsen worker 51 - jepsen.util 11661 :info :write [17 9] :RequestCanceledException INFO [2019-12-17 12:37:55,665] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:37:55,665] jepsen nemesis - couchbase.util Rebalance check count: 570 INFO [2019-12-17 12:37:55,665] jepsen nemesis - couchbase.util Rebalance stuck count: 551 INFO [2019-12-17 12:37:55,665] jepsen worker 51 - jepsen.util 11751 :invoke :cas [17 [38 20]] INFO [2019-12-17 12:37:55,676] jepsen worker 28 - jepsen.util 10738 :fail :cas [9 [37 19]] :ValueNotSwapFrom INFO [2019-12-17 12:37:55,676] jepsen worker 28 - jepsen.util 10738 :invoke :write [9 13] INFO [2019-12-17 12:37:55,703] jepsen worker 37 - jepsen.util 12007 :ok :read [12 15] INFO [2019-12-17 12:37:55,704] jepsen worker 37 - jepsen.util 12007 :invoke :write [12 4] INFO [2019-12-17 12:37:55,709] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72323,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:37:55,709] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72323,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:37:55,709] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72325,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:37:55,709] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72325,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:37:55,726] jepsen worker 30 - jepsen.util 11640 :info :write [10 9] :SyncWriteAmbiguous INFO [2019-12-17 12:37:55,727] jepsen worker 30 - jepsen.util 11730 :invoke :read [10 nil] INFO [2019-12-17 12:37:55,734] jepsen worker 77 - jepsen.util 18167 :info :cas [25 [25 31]] :RequestCanceledException INFO [2019-12-17 12:37:55,734] jepsen worker 16 - jepsen.util 18646 :info :write [5 13] :RequestCanceledException INFO [2019-12-17 12:37:55,734] jepsen worker 50 - jepsen.util 17060 :info :cas [16 [49 33]] :RequestCanceledException INFO [2019-12-17 12:37:55,734] jepsen worker 2 - jepsen.util 16382 :info :cas [0 [8 19]] :RequestCanceledException INFO [2019-12-17 12:37:55,735] jepsen worker 16 - jepsen.util 18736 :invoke :read [5 nil] INFO [2019-12-17 12:37:55,735] jepsen worker 77 - jepsen.util 18257 :invoke :cas [25 [7 36]] INFO [2019-12-17 12:37:55,736] jepsen worker 50 - jepsen.util 17150 :invoke :read [16 nil] INFO [2019-12-17 12:37:55,736] jepsen worker 2 - jepsen.util 16472 :invoke :read [0 nil] INFO [2019-12-17 12:37:55,748] jepsen worker 8 - jepsen.util 16748 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:37:55,748] jepsen worker 8 - jepsen.util 16748 :invoke :read [2 nil] INFO [2019-12-17 12:37:55,761] jepsen worker 63 - jepsen.util 11043 :ok :write [21 1] INFO [2019-12-17 12:37:55,761] jepsen worker 63 - jepsen.util 11043 :invoke :write [21 45] INFO [2019-12-17 12:37:55,764] jepsen worker 19 - jepsen.util 12889 :ok :read [6 31] INFO [2019-12-17 12:37:55,764] jepsen worker 61 - jepsen.util 18511 :info :cas [20 [46 3]] :RequestCanceledException INFO [2019-12-17 12:37:55,764] jepsen worker 35 - jepsen.util 17135 :info :cas [11 [37 26]] :RequestCanceledException INFO [2019-12-17 12:37:55,764] jepsen worker 19 - jepsen.util 12889 :invoke :cas [6 [14 43]] INFO [2019-12-17 12:37:55,765] jepsen worker 35 - jepsen.util 17225 :invoke :read [11 nil] INFO [2019-12-17 12:37:55,765] jepsen worker 61 - jepsen.util 18601 :invoke :read [20 nil] INFO [2019-12-17 12:37:55,770] jepsen worker 38 - jepsen.util 13088 :ok :read [12 15] INFO [2019-12-17 12:37:55,770] jepsen worker 38 - jepsen.util 13088 :invoke :read [12 nil] INFO [2019-12-17 12:37:55,789] jepsen worker 28 - jepsen.util 10738 :ok :write [9 13] INFO [2019-12-17 12:37:55,789] jepsen worker 28 - jepsen.util 10738 :invoke :cas [9 [48 39]] INFO [2019-12-17 12:37:55,791] jepsen worker 45 - jepsen.util 12285 :info :write [15 45] :RequestCanceledException INFO [2019-12-17 12:37:55,791] jepsen worker 81 - jepsen.util 16731 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:37:55,791] jepsen worker 31 - jepsen.util 10651 :info :cas [10 [35 3]] :RequestCanceledException INFO [2019-12-17 12:37:55,791] jepsen worker 25 - jepsen.util 18205 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:37:55,792] jepsen worker 81 - jepsen.util 16731 :invoke :cas [27 [14 7]] INFO [2019-12-17 12:37:55,792] jepsen worker 25 - jepsen.util 18205 :invoke :write [8 38] INFO [2019-12-17 12:37:55,792] jepsen worker 45 - jepsen.util 12375 :invoke :cas [15 [49 19]] INFO [2019-12-17 12:37:55,792] jepsen worker 31 - jepsen.util 10741 :invoke :cas [10 [29 10]] INFO [2019-12-17 12:37:55,796] jepsen worker 36 - jepsen.util 13266 :fail :cas [12 [37 29]] :ValueNotSwapFrom INFO [2019-12-17 12:37:55,796] jepsen worker 36 - jepsen.util 13266 :invoke :write [12 7] INFO [2019-12-17 12:37:55,819] jepsen worker 59 - jepsen.util 16259 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:37:55,819] jepsen worker 7 - jepsen.util 17107 :info :cas [2 [16 32]] :RequestCanceledException INFO [2019-12-17 12:37:55,819] jepsen worker 15 - jepsen.util 16125 :info :write [5 45] :RequestCanceledException INFO [2019-12-17 12:37:55,819] jepsen worker 59 - jepsen.util 16259 :invoke :read [19 nil] INFO [2019-12-17 12:37:55,820] jepsen worker 7 - jepsen.util 17197 :invoke :read [2 nil] INFO [2019-12-17 12:37:55,820] jepsen worker 15 - jepsen.util 16215 :invoke :cas [5 [37 9]] INFO [2019-12-17 12:37:55,823] jepsen worker 19 - jepsen.util 12889 :fail :cas [6 [14 43]] :ValueNotSwapFrom INFO [2019-12-17 12:37:55,823] jepsen worker 19 - jepsen.util 12889 :invoke :read [6 nil] INFO [2019-12-17 12:37:55,827] jepsen worker 55 - jepsen.util 13195 :ok :read [18 33] INFO [2019-12-17 12:37:55,827] jepsen worker 55 - jepsen.util 13195 :invoke :cas [18 [37 35]] INFO [2019-12-17 12:37:55,834] jepsen worker 43 - jepsen.util 17323 :info :cas [14 [44 21]] :RequestCanceledException INFO [2019-12-17 12:37:55,834] jepsen worker 9 - jepsen.util 13059 :info :write [3 30] :RequestCanceledException INFO [2019-12-17 12:37:55,834] jepsen worker 72 - jepsen.util 12582 :fail :read [24 nil] :RequestTimeoutException INFO [2019-12-17 12:37:55,834] jepsen worker 72 - jepsen.util 12582 :invoke :read [24 nil] INFO [2019-12-17 12:37:55,834] jepsen worker 58 - jepsen.util 17248 :info :write [19 28] :RequestCanceledException INFO [2019-12-17 12:37:55,835] jepsen worker 43 - jepsen.util 17413 :invoke :write [14 2] INFO [2019-12-17 12:37:55,835] jepsen worker 9 - jepsen.util 13149 :invoke :write [3 20] INFO [2019-12-17 12:37:55,835] jepsen worker 58 - jepsen.util 17338 :invoke :write [19 3] INFO [2019-12-17 12:37:55,843] jepsen worker 57 - jepsen.util 16797 :info :cas [19 [18 41]] :RequestCanceledException INFO [2019-12-17 12:37:55,843] jepsen worker 44 - jepsen.util 17684 :info :cas [14 [6 22]] :RequestCanceledException INFO [2019-12-17 12:37:55,843] jepsen worker 48 - jepsen.util 15798 :info :cas [16 [36 7]] :RequestCanceledException INFO [2019-12-17 12:37:55,844] jepsen worker 57 - jepsen.util 16887 :invoke :cas [19 [43 6]] INFO [2019-12-17 12:37:55,844] jepsen worker 48 - jepsen.util 15888 :invoke :read [16 nil] INFO [2019-12-17 12:37:55,844] jepsen worker 44 - jepsen.util 17774 :invoke :cas [14 [33 13]] INFO [2019-12-17 12:37:55,856] jepsen worker 31 - jepsen.util 10741 :fail :cas [10 [29 10]] :ValueNotSwapFrom INFO [2019-12-17 12:37:55,856] jepsen worker 31 - jepsen.util 10741 :invoke :read [10 nil] INFO [2019-12-17 12:37:55,858] jepsen worker 87 - jepsen.util 12147 :ok :read [29 23] INFO [2019-12-17 12:37:55,858] jepsen worker 87 - jepsen.util 12147 :invoke :read [29 nil] INFO [2019-12-17 12:37:55,872] jepsen worker 89 - jepsen.util 12329 :ok :write [29 26] INFO [2019-12-17 12:37:55,872] jepsen worker 89 - jepsen.util 12329 :invoke :cas [29 [9 39]] INFO [2019-12-17 12:37:55,879] jepsen worker 28 - jepsen.util 10738 :fail :cas [9 [48 39]] :ValueNotSwapFrom INFO [2019-12-17 12:37:55,879] jepsen worker 28 - jepsen.util 10738 :invoke :read [9 nil] INFO [2019-12-17 12:37:55,911] jepsen worker 19 - jepsen.util 12889 :ok :read [6 31] INFO [2019-12-17 12:37:55,912] jepsen worker 19 - jepsen.util 12889 :invoke :cas [6 [23 34]] INFO [2019-12-17 12:37:55,915] jepsen worker 79 - jepsen.util 12679 :ok :write [26 9] INFO [2019-12-17 12:37:55,915] jepsen worker 79 - jepsen.util 12679 :invoke :cas [26 [36 31]] INFO [2019-12-17 12:37:55,921] jepsen worker 38 - jepsen.util 13088 :ok :read [12 15] INFO [2019-12-17 12:37:55,921] jepsen worker 38 - jepsen.util 13088 :invoke :read [12 nil] INFO [2019-12-17 12:37:55,928] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72364,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:37:55,928] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72364,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:37:55,928] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72367,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:37:55,928] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72367,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:37:55,928] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72368,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:37:55,928] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72368,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:37:55,931] jepsen worker 31 - jepsen.util 10741 :ok :read [10 40] INFO [2019-12-17 12:37:55,931] jepsen worker 31 - jepsen.util 10741 :invoke :cas [10 [36 40]] INFO [2019-12-17 12:37:55,936] jepsen worker 64 - jepsen.util 9784 :ok :read [21 1] INFO [2019-12-17 12:37:55,936] jepsen worker 64 - jepsen.util 9784 :invoke :read [21 nil] INFO [2019-12-17 12:37:55,947] jepsen worker 70 - jepsen.util 12940 :info :write [23 16] :RequestCanceledException INFO [2019-12-17 12:37:55,948] jepsen worker 70 - jepsen.util 13030 :invoke :read [23 nil] INFO [2019-12-17 12:37:55,966] jepsen worker 45 - jepsen.util 12375 :fail :cas [15 [49 19]] :ValueNotSwapFrom INFO [2019-12-17 12:37:55,966] jepsen worker 45 - jepsen.util 12375 :invoke :cas [15 [40 1]] INFO [2019-12-17 12:37:55,977] jepsen worker 37 - jepsen.util 12007 :ok :write [12 4] INFO [2019-12-17 12:37:55,978] jepsen worker 37 - jepsen.util 12007 :invoke :read [12 nil] INFO [2019-12-17 12:37:56,001] jepsen worker 27 - jepsen.util 11547 :info :cas [9 [28 25]] :RequestCanceledException INFO [2019-12-17 12:37:56,002] jepsen worker 27 - jepsen.util 11637 :invoke :write [9 13] INFO [2019-12-17 12:37:56,004] jepsen worker 31 - jepsen.util 10741 :fail :cas [10 [36 40]] :ValueNotSwapFrom INFO [2019-12-17 12:37:56,004] jepsen worker 31 - jepsen.util 10741 :invoke :write [10 35] INFO [2019-12-17 12:37:56,012] jepsen worker 65 - jepsen.util 10955 :ok :read [21 1] INFO [2019-12-17 12:37:56,012] jepsen worker 65 - jepsen.util 10955 :invoke :cas [21 [45 32]] INFO [2019-12-17 12:37:56,013] jepsen worker 55 - jepsen.util 13195 :fail :cas [18 [37 35]] :ValueNotSwapFrom INFO [2019-12-17 12:37:56,014] jepsen worker 55 - jepsen.util 13195 :invoke :write [18 13] INFO [2019-12-17 12:37:56,018] jepsen worker 46 - jepsen.util 12106 :info :write [15 27] :RequestCanceledException INFO [2019-12-17 12:37:56,018] jepsen worker 74 - jepsen.util 12224 :info :write [24 49] :RequestCanceledException INFO [2019-12-17 12:37:56,019] jepsen worker 74 - jepsen.util 12314 :invoke :read [24 nil] INFO [2019-12-17 12:37:56,022] jepsen worker 46 - jepsen.util 12196 :invoke :cas [15 [25 9]] INFO [2019-12-17 12:37:56,032] jepsen worker 19 - jepsen.util 12889 :fail :cas [6 [23 34]] :ValueNotSwapFrom INFO [2019-12-17 12:37:56,032] jepsen worker 19 - jepsen.util 12889 :invoke :write [6 47] INFO [2019-12-17 12:37:56,034] jepsen worker 10 - jepsen.util 12340 :fail :read [3 nil] :RequestTimeoutException INFO [2019-12-17 12:37:56,034] jepsen worker 42 - jepsen.util 17952 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:37:56,034] jepsen worker 10 - jepsen.util 12340 :invoke :cas [3 [23 25]] INFO [2019-12-17 12:37:56,034] jepsen worker 42 - jepsen.util 17952 :invoke :write [14 6] INFO [2019-12-17 12:37:56,037] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72382,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:37:56,037] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72382,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:37:56,042] jepsen worker 28 - jepsen.util 10738 :ok :read [9 13] INFO [2019-12-17 12:37:56,042] jepsen worker 28 - jepsen.util 10738 :invoke :write [9 0] INFO [2019-12-17 12:37:56,048] jepsen worker 1 - jepsen.util 17371 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:37:56,048] jepsen worker 1 - jepsen.util 17371 :invoke :write [0 22] INFO [2019-12-17 12:37:56,066] jepsen worker 12 - jepsen.util 12612 :fail :cas [4 [39 22]] :ValueNotSwapFrom INFO [2019-12-17 12:37:56,066] jepsen worker 12 - jepsen.util 12612 :invoke :write [4 0] INFO [2019-12-17 12:37:56,095] jepsen worker 30 - jepsen.util 11730 :ok :read [10 40] INFO [2019-12-17 12:37:56,095] jepsen worker 30 - jepsen.util 11730 :invoke :write [10 12] INFO [2019-12-17 12:37:56,095] jepsen worker 45 - jepsen.util 12375 :fail :cas [15 [40 1]] :ValueNotSwapFrom INFO [2019-12-17 12:37:56,095] jepsen worker 45 - jepsen.util 12375 :invoke :read [15 nil] INFO [2019-12-17 12:37:56,109] jepsen worker 63 - jepsen.util 11043 :ok :write [21 45] INFO [2019-12-17 12:37:56,109] jepsen worker 63 - jepsen.util 11043 :invoke :cas [21 [34 19]] INFO [2019-12-17 12:37:56,121] jepsen worker 87 - jepsen.util 12147 :ok :read [29 26] INFO [2019-12-17 12:37:56,121] jepsen worker 87 - jepsen.util 12147 :invoke :write [29 1] INFO [2019-12-17 12:37:56,126] jepsen worker 71 - jepsen.util 13301 :ok :write [23 37] INFO [2019-12-17 12:37:56,127] jepsen worker 71 - jepsen.util 13301 :invoke :read [23 nil] INFO [2019-12-17 12:37:56,131] jepsen worker 63 - jepsen.util 11043 :fail :cas [21 [34 19]] :ValueNotSwapFrom INFO [2019-12-17 12:37:56,131] jepsen worker 63 - jepsen.util 11043 :invoke :read [21 nil] INFO [2019-12-17 12:37:56,132] jepsen worker 22 - jepsen.util 17212 :info :write [7 38] :RequestCanceledException INFO [2019-12-17 12:37:56,134] jepsen worker 22 - jepsen.util 17302 :invoke :write [7 0] INFO [2019-12-17 12:37:56,135] jepsen worker 51 - jepsen.util 11751 :fail :cas [17 [38 20]] :ValueNotSwapFrom INFO [2019-12-17 12:37:56,135] jepsen worker 51 - jepsen.util 11751 :invoke :cas [17 [34 41]] INFO [2019-12-17 12:37:56,140] jepsen worker 3 - jepsen.util 11883 :ok :read [1 21] INFO [2019-12-17 12:37:56,140] jepsen worker 3 - jepsen.util 11883 :invoke :cas [1 [28 26]] INFO [2019-12-17 12:37:56,148] jepsen worker 39 - jepsen.util 16599 :info :cas [13 [13 19]] :RequestCanceledException INFO [2019-12-17 12:37:56,150] jepsen worker 39 - jepsen.util 16689 :invoke :cas [13 [47 28]] INFO [2019-12-17 12:37:56,153] jepsen worker 63 - jepsen.util 11043 :ok :read [21 45] INFO [2019-12-17 12:37:56,153] jepsen worker 63 - jepsen.util 11043 :invoke :write [21 18] INFO [2019-12-17 12:37:56,159] jepsen worker 89 - jepsen.util 12329 :fail :cas [29 [9 39]] :ValueNotSwapFrom INFO [2019-12-17 12:37:56,159] jepsen worker 89 - jepsen.util 12329 :invoke :write [29 7] INFO [2019-12-17 12:37:56,164] jepsen worker 54 - jepsen.util 12924 :info :write [18 18] :RequestCanceledException INFO [2019-12-17 12:37:56,164] jepsen worker 83 - jepsen.util 17003 :info :write [27 5] :RequestCanceledException INFO [2019-12-17 12:37:56,164] jepsen worker 32 - jepsen.util 12902 :info :write [10 48] :RequestCanceledException INFO [2019-12-17 12:37:56,164] jepsen worker 18 - jepsen.util 12978 :info :write [6 8] :RequestCanceledException INFO [2019-12-17 12:37:56,165] jepsen worker 32 - jepsen.util 12992 :invoke :read [10 nil] INFO [2019-12-17 12:37:56,165] jepsen worker 54 - jepsen.util 13014 :invoke :cas [18 [44 41]] INFO [2019-12-17 12:37:56,168] jepsen worker 18 - jepsen.util 13068 :invoke :read [6 nil] INFO [2019-12-17 12:37:56,168] jepsen worker 83 - jepsen.util 17093 :invoke :write [27 39] INFO [2019-12-17 12:37:56,173] jepsen worker 70 - jepsen.util 13030 :ok :read [23 37] INFO [2019-12-17 12:37:56,173] jepsen worker 70 - jepsen.util 13030 :invoke :cas [23 [45 41]] INFO [2019-12-17 12:37:56,199] jepsen worker 38 - jepsen.util 13088 :ok :read [12 4] INFO [2019-12-17 12:37:56,199] jepsen worker 38 - jepsen.util 13088 :invoke :write [12 17] INFO [2019-12-17 12:37:56,209] jepsen worker 63 - jepsen.util 11043 :ok :write [21 18] INFO [2019-12-17 12:37:56,209] jepsen worker 63 - jepsen.util 11043 :invoke :cas [21 [12 20]] INFO [2019-12-17 12:37:56,214] jepsen worker 4 - jepsen.util 11704 :ok :write [1 45] INFO [2019-12-17 12:37:56,214] jepsen worker 4 - jepsen.util 11704 :invoke :read [1 nil] INFO [2019-12-17 12:37:56,218] jepsen worker 20 - jepsen.util 12260 :info :cas [6 [16 11]] :RequestCanceledException INFO [2019-12-17 12:37:56,219] jepsen worker 20 - jepsen.util 12350 :invoke :write [6 6] INFO [2019-12-17 12:37:56,228] jepsen worker 63 - jepsen.util 11043 :fail :cas [21 [12 20]] :ValueNotSwapFrom INFO [2019-12-17 12:37:56,228] jepsen worker 63 - jepsen.util 11043 :invoke :write [21 18] INFO [2019-12-17 12:37:56,234] jepsen worker 76 - jepsen.util 17176 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:37:56,235] jepsen worker 76 - jepsen.util 17176 :invoke :write [25 42] INFO [2019-12-17 12:37:56,251] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72414,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:37:56,251] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72424,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:37:56,251] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72414,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:37:56,251] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72424,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:37:56,251] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72416,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:37:56,251] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72427,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:37:56,251] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72416,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:37:56,251] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72427,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:37:56,251] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72428,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:37:56,251] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72428,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:37:56,263] jepsen worker 29 - jepsen.util 10919 :info :cas [9 [16 25]] :RequestCanceledException INFO [2019-12-17 12:37:56,263] jepsen worker 78 - jepsen.util 12138 :info :write [26 7] :RequestCanceledException INFO [2019-12-17 12:37:56,263] jepsen worker 11 - jepsen.util 12071 :info :write [3 29] :RequestCanceledException INFO [2019-12-17 12:37:56,264] jepsen worker 29 - jepsen.util 11009 :invoke :cas [9 [13 9]] INFO [2019-12-17 12:37:56,264] jepsen worker 78 - jepsen.util 12228 :invoke :write [26 18] INFO [2019-12-17 12:37:56,265] jepsen worker 11 - jepsen.util 12161 :invoke :cas [3 [37 39]] INFO [2019-12-17 12:37:56,281] jepsen worker 52 - jepsen.util 12022 :fail :cas [17 [3 9]] :ValueNotSwapFrom INFO [2019-12-17 12:37:56,281] jepsen worker 52 - jepsen.util 12022 :invoke :cas [17 [14 25]] INFO [2019-12-17 12:37:56,295] jepsen worker 37 - jepsen.util 12007 :ok :read [12 4] INFO [2019-12-17 12:37:56,326] jepsen worker 74 - jepsen.util 12314 :ok :read [24 19] INFO [2019-12-17 12:37:56,333] jepsen worker 72 - jepsen.util 12582 :ok :read [24 19] INFO [2019-12-17 12:37:56,334] jepsen worker 6 - jepsen.util 17466 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:37:56,334] jepsen worker 68 - jepsen.util 16268 :info :cas [22 [28 10]] :RequestCanceledException INFO [2019-12-17 12:37:56,365] jepsen worker 80 - jepsen.util 11960 :fail :cas [26 [2 12]] :ValueNotSwapFrom INFO [2019-12-17 12:37:56,365] jepsen worker 49 - jepsen.util 17059 :info :write [16 34] :RequestCanceledException INFO [2019-12-17 12:37:56,372] jepsen worker 63 - jepsen.util 11043 :ok :write [21 18] INFO [2019-12-17 12:37:56,373] jepsen worker 88 - jepsen.util 12598 :fail :cas [29 [49 47]] :ValueNotSwapFrom INFO [2019-12-17 12:37:56,385] jepsen worker 79 - jepsen.util 12679 :fail :cas [26 [36 31]] :ValueNotSwapFrom INFO [2019-12-17 12:37:56,398] jepsen worker 14 - jepsen.util 12164 :ok :write [4 16] INFO [2019-12-17 12:37:56,400] jepsen worker 73 - jepsen.util 11593 :info :write [24 0] :RequestCanceledException INFO [2019-12-17 12:37:56,405] jepsen worker 74 - jepsen.util 12314 :invoke :write [24 22] INFO [2019-12-17 12:37:56,405] jepsen worker 37 - jepsen.util 12007 :invoke :write [12 22] INFO [2019-12-17 12:37:56,405] jepsen worker 72 - jepsen.util 12582 :invoke :write [24 25] INFO [2019-12-17 12:37:56,405] jepsen worker 6 - jepsen.util 17466 :invoke :cas [2 [39 32]] INFO [2019-12-17 12:37:56,405] jepsen worker 80 - jepsen.util 11960 :invoke :read [26 nil] INFO [2019-12-17 12:37:56,405] jepsen worker 63 - jepsen.util 11043 :invoke :write [21 44] INFO [2019-12-17 12:37:56,405] jepsen worker 88 - jepsen.util 12598 :invoke :read [29 nil] INFO [2019-12-17 12:37:56,405] jepsen worker 79 - jepsen.util 12679 :invoke :read [26 nil] INFO [2019-12-17 12:37:56,405] jepsen worker 14 - jepsen.util 12164 :invoke :write [4 35] INFO [2019-12-17 12:37:56,406] jepsen worker 68 - jepsen.util 16358 :invoke :cas [22 [39 46]] INFO [2019-12-17 12:37:56,406] jepsen worker 49 - jepsen.util 17149 :invoke :cas [16 [37 9]] INFO [2019-12-17 12:37:56,406] jepsen worker 73 - jepsen.util 11683 :invoke :read [24 nil] INFO [2019-12-17 12:37:56,416] jepsen worker 24 - jepsen.util 16314 :info :cas [8 [12 1]] :RequestCanceledException INFO [2019-12-17 12:37:56,418] jepsen worker 24 - jepsen.util 16404 :invoke :cas [8 [13 34]] INFO [2019-12-17 12:37:56,434] jepsen worker 40 - jepsen.util 17950 :info :write [13 28] :RequestCanceledException INFO [2019-12-17 12:37:56,435] jepsen worker 40 - jepsen.util 18040 :invoke :write [13 46] INFO [2019-12-17 12:37:56,440] jepsen worker 36 - jepsen.util 13266 :ok :write [12 7] INFO [2019-12-17 12:37:56,440] jepsen worker 36 - jepsen.util 13266 :invoke :write [12 44] INFO [2019-12-17 12:37:56,448] jepsen worker 41 - jepsen.util 18491 :info :write [13 43] :RequestCanceledException INFO [2019-12-17 12:37:56,449] jepsen worker 41 - jepsen.util 18581 :invoke :cas [13 [13 13]] INFO [2019-12-17 12:37:56,450] jepsen worker 73 - jepsen.util 11683 :ok :read [24 19] INFO [2019-12-17 12:37:56,450] jepsen worker 73 - jepsen.util 11683 :invoke :cas [24 [3 42]] INFO [2019-12-17 12:37:56,460] jepsen worker 4 - jepsen.util 11704 :ok :read [1 45] INFO [2019-12-17 12:37:56,460] jepsen worker 4 - jepsen.util 11704 :invoke :cas [1 [32 45]] INFO [2019-12-17 12:37:56,464] jepsen worker 13 - jepsen.util 11983 :info :write [4 21] :RequestCanceledException INFO [2019-12-17 12:37:56,466] jepsen worker 13 - jepsen.util 12073 :invoke :cas [4 [47 36]] INFO [2019-12-17 12:37:56,470] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72450,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:37:56,470] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72450,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:37:56,470] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72453,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:37:56,470] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72453,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:37:56,471] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72454,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:37:56,471] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72454,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:37:56,475] jepsen worker 56 - jepsen.util 13286 :fail :cas [18 [43 34]] :ValueNotSwapFrom INFO [2019-12-17 12:37:56,476] jepsen worker 56 - jepsen.util 13286 :invoke :write [18 31] INFO [2019-12-17 12:37:56,534] jepsen worker 34 - jepsen.util 17944 :info :write [11 3] :RequestCanceledException INFO [2019-12-17 12:37:56,534] jepsen worker 63 - jepsen.util 11043 :ok :write [21 44] INFO [2019-12-17 12:37:56,534] jepsen worker 63 - jepsen.util 11043 :invoke :cas [21 [34 8]] INFO [2019-12-17 12:37:56,535] jepsen worker 34 - jepsen.util 18034 :invoke :cas [11 [42 3]] INFO [2019-12-17 12:37:56,558] jepsen worker 69 - jepsen.util 12759 :fail :cas [23 [7 39]] :ValueNotSwapFrom INFO [2019-12-17 12:37:56,558] jepsen worker 69 - jepsen.util 12759 :invoke :write [23 1] INFO [2019-12-17 12:37:56,559] jepsen worker 52 - jepsen.util 12022 :fail :cas [17 [14 25]] :ValueNotSwapFrom INFO [2019-12-17 12:37:56,559] jepsen worker 52 - jepsen.util 12022 :invoke :read [17 nil] INFO [2019-12-17 12:37:56,571] jepsen worker 63 - jepsen.util 11043 :fail :cas [21 [34 8]] :ValueNotSwapFrom INFO [2019-12-17 12:37:56,571] jepsen worker 63 - jepsen.util 11043 :invoke :write [21 21] INFO [2019-12-17 12:37:56,591] jepsen worker 53 - jepsen.util 11933 :ok :read [17 9] INFO [2019-12-17 12:37:56,591] jepsen worker 53 - jepsen.util 11933 :invoke :write [17 37] INFO [2019-12-17 12:37:56,629] jepsen worker 54 - jepsen.util 13014 :fail :cas [18 [44 41]] :ValueNotSwapFrom INFO [2019-12-17 12:37:56,630] jepsen worker 54 - jepsen.util 13014 :invoke :read [18 nil] INFO [2019-12-17 12:37:56,649] jepsen worker 63 - jepsen.util 11043 :ok :write [21 21] INFO [2019-12-17 12:37:56,649] jepsen worker 63 - jepsen.util 11043 :invoke :cas [21 [10 22]] INFO [2019-12-17 12:37:56,657] jepsen worker 80 - jepsen.util 11960 :ok :read [26 7] INFO [2019-12-17 12:37:56,657] jepsen worker 80 - jepsen.util 11960 :invoke :cas [26 [3 7]] INFO [2019-12-17 12:37:56,661] jepsen worker 63 - jepsen.util 11043 :fail :cas [21 [10 22]] :ValueNotSwapFrom INFO [2019-12-17 12:37:56,661] jepsen worker 63 - jepsen.util 11043 :invoke :read [21 nil] INFO [2019-12-17 12:37:56,686] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72471,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:37:56,686] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:37:56,687] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72471,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:37:56,687] jepsen nemesis - couchbase.util Rebalance check count: 571 INFO [2019-12-17 12:37:56,687] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72475,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:37:56,687] jepsen nemesis - couchbase.util Rebalance stuck count: 552 INFO [2019-12-17 12:37:56,687] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72475,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:37:56,691] jepsen worker 63 - jepsen.util 11043 :ok :read [21 21] INFO [2019-12-17 12:37:56,691] jepsen worker 63 - jepsen.util 11043 :invoke :cas [21 [17 26]] INFO [2019-12-17 12:37:56,709] jepsen worker 88 - jepsen.util 12598 :ok :read [29 4] INFO [2019-12-17 12:37:56,710] jepsen worker 88 - jepsen.util 12598 :invoke :write [29 2] INFO [2019-12-17 12:37:56,712] jepsen worker 63 - jepsen.util 11043 :fail :cas [21 [17 26]] :ValueNotSwapFrom INFO [2019-12-17 12:37:56,712] jepsen worker 63 - jepsen.util 11043 :invoke :write [21 15] INFO [2019-12-17 12:37:56,734] jepsen worker 60 - jepsen.util 18060 :info :cas [20 [49 38]] :RequestCanceledException INFO [2019-12-17 12:37:56,735] jepsen worker 60 - jepsen.util 18150 :invoke :read [20 nil] INFO [2019-12-17 12:37:56,739] jepsen worker 71 - jepsen.util 13301 :ok :read [23 37] INFO [2019-12-17 12:37:56,740] jepsen worker 71 - jepsen.util 13301 :invoke :read [23 nil] INFO [2019-12-17 12:37:56,748] jepsen worker 17 - jepsen.util 17207 :info :cas [5 [0 13]] :RequestCanceledException INFO [2019-12-17 12:37:56,749] jepsen worker 17 - jepsen.util 17297 :invoke :cas [5 [46 11]] INFO [2019-12-17 12:37:56,751] jepsen worker 79 - jepsen.util 12679 :ok :read [26 7] INFO [2019-12-17 12:37:56,751] jepsen worker 79 - jepsen.util 12679 :invoke :write [26 46] INFO [2019-12-17 12:37:56,765] jepsen worker 86 - jepsen.util 17816 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:37:56,766] jepsen worker 86 - jepsen.util 17816 :invoke :write [28 47] INFO [2019-12-17 12:37:56,789] jepsen worker 4 - jepsen.util 11704 :fail :cas [1 [32 45]] :ValueNotSwapFrom INFO [2019-12-17 12:37:56,789] jepsen worker 4 - jepsen.util 11704 :invoke :write [1 26] INFO [2019-12-17 12:37:56,801] jepsen worker 62 - jepsen.util 17792 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:37:56,801] jepsen worker 75 - jepsen.util 16635 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:37:56,801] jepsen worker 62 - jepsen.util 17792 :invoke :cas [20 [48 38]] INFO [2019-12-17 12:37:56,801] jepsen worker 75 - jepsen.util 16635 :invoke :cas [25 [37 0]] INFO [2019-12-17 12:37:56,809] jepsen worker 63 - jepsen.util 11043 :ok :write [21 15] INFO [2019-12-17 12:37:56,809] jepsen worker 63 - jepsen.util 11043 :invoke :write [21 49] INFO [2019-12-17 12:37:56,831] jepsen worker 13 - jepsen.util 12073 :fail :cas [4 [47 36]] :ValueNotSwapFrom INFO [2019-12-17 12:37:56,831] jepsen worker 26 - jepsen.util 17846 :info :write [8 4] :RequestCanceledException INFO [2019-12-17 12:37:56,831] jepsen worker 13 - jepsen.util 12073 :invoke :cas [4 [27 1]] INFO [2019-12-17 12:37:56,832] jepsen worker 26 - jepsen.util 17936 :invoke :read [8 nil] INFO [2019-12-17 12:37:56,866] jepsen worker 52 - jepsen.util 12022 :ok :read [17 9] INFO [2019-12-17 12:37:56,866] jepsen worker 52 - jepsen.util 12022 :invoke :read [17 nil] INFO [2019-12-17 12:37:56,900] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72494,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:37:56,900] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72494,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:37:56,925] jepsen worker 87 - jepsen.util 12147 :ok :write [29 1] INFO [2019-12-17 12:37:56,925] jepsen worker 87 - jepsen.util 12147 :invoke :read [29 nil] INFO [2019-12-17 12:37:56,930] jepsen worker 3 - jepsen.util 11883 :fail :cas [1 [28 26]] :ValueNotSwapFrom INFO [2019-12-17 12:37:56,930] jepsen worker 3 - jepsen.util 11883 :invoke :read [1 nil] INFO [2019-12-17 12:37:56,935] jepsen worker 66 - jepsen.util 17256 :info :cas [22 [21 1]] :RequestCanceledException INFO [2019-12-17 12:37:56,936] jepsen worker 66 - jepsen.util 17346 :invoke :read [22 nil] INFO [2019-12-17 12:37:56,941] jepsen worker 80 - jepsen.util 11960 :fail :cas [26 [3 7]] :ValueNotSwapFrom INFO [2019-12-17 12:37:56,941] jepsen worker 80 - jepsen.util 11960 :invoke :cas [26 [40 49]] INFO [2019-12-17 12:37:56,941] jepsen worker 63 - jepsen.util 11043 :ok :write [21 49] INFO [2019-12-17 12:37:56,941] jepsen worker 63 - jepsen.util 11043 :invoke :cas [21 [29 33]] INFO [2019-12-17 12:37:56,963] jepsen worker 63 - jepsen.util 11043 :fail :cas [21 [29 33]] :ValueNotSwapFrom INFO [2019-12-17 12:37:56,964] jepsen worker 63 - jepsen.util 11043 :invoke :cas [21 [47 41]] INFO [2019-12-17 12:37:56,968] jepsen worker 37 - jepsen.util 12007 :ok :write [12 22] INFO [2019-12-17 12:37:56,968] jepsen worker 37 - jepsen.util 12007 :invoke :read [12 nil] INFO [2019-12-17 12:37:56,980] jepsen worker 3 - jepsen.util 11883 :ok :read [1 45] INFO [2019-12-17 12:37:56,980] jepsen worker 3 - jepsen.util 11883 :invoke :cas [1 [35 42]] INFO [2019-12-17 12:37:56,989] jepsen worker 63 - jepsen.util 11043 :fail :cas [21 [47 41]] :ValueNotSwapFrom INFO [2019-12-17 12:37:56,989] jepsen worker 63 - jepsen.util 11043 :invoke :write [21 21] INFO [2019-12-17 12:37:57,010] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72509,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:37:57,010] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72509,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:37:57,010] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72513,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:37:57,010] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72513,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:37:57,034] jepsen worker 85 - jepsen.util 17725 :info :cas [28 [7 38]] :RequestCanceledException INFO [2019-12-17 12:37:57,035] jepsen worker 85 - jepsen.util 17815 :invoke :read [28 nil] INFO [2019-12-17 12:37:57,052] jepsen worker 51 - jepsen.util 11751 :fail :cas [17 [34 41]] :ValueNotSwapFrom INFO [2019-12-17 12:37:57,052] jepsen worker 51 - jepsen.util 11751 :invoke :read [17 nil] INFO [2019-12-17 12:37:57,057] jepsen worker 63 - jepsen.util 11043 :ok :write [21 21] INFO [2019-12-17 12:37:57,057] jepsen worker 63 - jepsen.util 11043 :invoke :write [21 14] INFO [2019-12-17 12:37:57,068] jepsen worker 3 - jepsen.util 11883 :fail :cas [1 [35 42]] :ValueNotSwapFrom INFO [2019-12-17 12:37:57,068] jepsen worker 3 - jepsen.util 11883 :invoke :write [1 25] INFO [2019-12-17 12:37:57,079] jepsen worker 14 - jepsen.util 12164 :ok :write [4 35] INFO [2019-12-17 12:37:57,079] jepsen worker 14 - jepsen.util 12164 :invoke :write [4 19] INFO [2019-12-17 12:37:57,098] jepsen worker 70 - jepsen.util 13030 :fail :cas [23 [45 41]] :ValueNotSwapFrom INFO [2019-12-17 12:37:57,099] jepsen worker 70 - jepsen.util 13030 :invoke :write [23 5] INFO [2019-12-17 12:37:57,100] jepsen worker 67 - jepsen.util 17347 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:37:57,101] jepsen worker 67 - jepsen.util 17347 :invoke :write [22 38] INFO [2019-12-17 12:37:57,114] jepsen worker 88 - jepsen.util 12598 :ok :write [29 2] INFO [2019-12-17 12:37:57,114] jepsen worker 88 - jepsen.util 12598 :invoke :read [29 nil] INFO [2019-12-17 12:37:57,119] jepsen worker 84 - jepsen.util 16644 :info :write [28 39] :RequestCanceledException INFO [2019-12-17 12:37:57,120] jepsen worker 84 - jepsen.util 16734 :invoke :read [28 nil] INFO [2019-12-17 12:37:57,134] jepsen worker 23 - jepsen.util 17573 :info :cas [7 [13 15]] :RequestCanceledException INFO [2019-12-17 12:37:57,134] jepsen worker 33 - jepsen.util 16863 :info :write [11 0] :RequestCanceledException INFO [2019-12-17 12:37:57,135] jepsen worker 23 - jepsen.util 17663 :invoke :read [7 nil] INFO [2019-12-17 12:37:57,136] jepsen worker 33 - jepsen.util 16953 :invoke :cas [11 [36 40]] INFO [2019-12-17 12:37:57,149] jepsen worker 82 - jepsen.util 17452 :info :write [27 32] :RequestCanceledException INFO [2019-12-17 12:37:57,150] jepsen worker 82 - jepsen.util 17542 :invoke :cas [27 [42 42]] INFO [2019-12-17 12:37:57,155] jepsen worker 80 - jepsen.util 11960 :fail :cas [26 [40 49]] :ValueNotSwapFrom INFO [2019-12-17 12:37:57,155] jepsen worker 80 - jepsen.util 11960 :invoke :write [26 4] INFO [2019-12-17 12:37:57,165] jepsen worker 0 - jepsen.util 17640 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:37:57,165] jepsen worker 0 - jepsen.util 17640 :invoke :read [0 nil] INFO [2019-12-17 12:37:57,183] jepsen worker 69 - jepsen.util 12759 :ok :write [23 1] INFO [2019-12-17 12:37:57,183] jepsen worker 69 - jepsen.util 12759 :invoke :write [23 1] INFO [2019-12-17 12:37:57,185] jepsen worker 63 - jepsen.util 11043 :ok :write [21 14] INFO [2019-12-17 12:37:57,186] jepsen worker 63 - jepsen.util 11043 :invoke :write [21 18] INFO [2019-12-17 12:37:57,228] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72530,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:37:57,228] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72537,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:37:57,228] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72530,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:37:57,229] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72537,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:37:57,229] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72531,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:37:57,229] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72540,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:37:57,229] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72531,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:37:57,229] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72540,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:37:57,229] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72532,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:37:57,229] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72532,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:37:57,248] jepsen worker 21 - jepsen.util 16581 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:37:57,248] jepsen worker 21 - jepsen.util 16581 :invoke :cas [7 [5 17]] INFO [2019-12-17 12:37:57,286] jepsen worker 63 - jepsen.util 11043 :ok :write [21 18] INFO [2019-12-17 12:37:57,287] jepsen worker 63 - jepsen.util 11043 :invoke :cas [21 [38 12]] INFO [2019-12-17 12:37:57,306] jepsen worker 63 - jepsen.util 11043 :fail :cas [21 [38 12]] :ValueNotSwapFrom INFO [2019-12-17 12:37:57,307] jepsen worker 63 - jepsen.util 11043 :invoke :read [21 nil] INFO [2019-12-17 12:37:57,313] jepsen worker 78 - jepsen.util 12228 :ok :write [26 18] INFO [2019-12-17 12:37:57,313] jepsen worker 78 - jepsen.util 12228 :invoke :write [26 8] INFO [2019-12-17 12:37:57,326] jepsen worker 63 - jepsen.util 11043 :ok :read [21 18] INFO [2019-12-17 12:37:57,326] jepsen worker 63 - jepsen.util 11043 :invoke :cas [21 [6 33]] INFO [2019-12-17 12:37:57,341] jepsen worker 38 - jepsen.util 13088 :ok :write [12 17] INFO [2019-12-17 12:37:57,341] jepsen worker 38 - jepsen.util 13088 :invoke :write [12 1] INFO [2019-12-17 12:37:57,344] jepsen worker 63 - jepsen.util 11043 :fail :cas [21 [6 33]] :ValueNotSwapFrom INFO [2019-12-17 12:37:57,344] jepsen worker 63 - jepsen.util 11043 :invoke :write [21 37] INFO [2019-12-17 12:37:57,347] jepsen worker 56 - jepsen.util 13286 :ok :write [18 31] INFO [2019-12-17 12:37:57,347] jepsen worker 56 - jepsen.util 13286 :invoke :cas [18 [9 33]] INFO [2019-12-17 12:37:57,380] jepsen worker 5 - jepsen.util 12155 :ok :write [1 11] INFO [2019-12-17 12:37:57,380] jepsen worker 5 - jepsen.util 12155 :invoke :read [1 nil] INFO [2019-12-17 12:37:57,428] jepsen worker 63 - jepsen.util 11043 :ok :write [21 37] INFO [2019-12-17 12:37:57,428] jepsen worker 63 - jepsen.util 11043 :invoke :read [21 nil] INFO [2019-12-17 12:37:57,448] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72554,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:37:57,448] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72554,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:37:57,448] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72557,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:37:57,448] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72557,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:37:57,472] jepsen worker 63 - jepsen.util 11043 :ok :read [21 37] INFO [2019-12-17 12:37:57,472] jepsen worker 63 - jepsen.util 11043 :invoke :write [21 41] INFO [2019-12-17 12:37:57,475] jepsen worker 5 - jepsen.util 12155 :ok :read [1 11] INFO [2019-12-17 12:37:57,475] jepsen worker 5 - jepsen.util 12155 :invoke :write [1 41] INFO [2019-12-17 12:37:57,475] jepsen worker 12 - jepsen.util 12612 :ok :write [4 0] INFO [2019-12-17 12:37:57,475] jepsen worker 12 - jepsen.util 12612 :invoke :cas [4 [12 26]] INFO [2019-12-17 12:37:57,478] jepsen worker 64 - jepsen.util 9784 :ok :read [21 18] INFO [2019-12-17 12:37:57,478] jepsen worker 64 - jepsen.util 9784 :invoke :write [21 26] INFO [2019-12-17 12:37:57,486] jepsen worker 4 - jepsen.util 11704 :ok :write [1 26] INFO [2019-12-17 12:37:57,486] jepsen worker 4 - jepsen.util 11704 :invoke :write [1 7] INFO [2019-12-17 12:37:57,489] jepsen worker 36 - jepsen.util 13266 :ok :write [12 44] INFO [2019-12-17 12:37:57,489] jepsen worker 36 - jepsen.util 13266 :invoke :write [12 32] INFO [2019-12-17 12:37:57,495] jepsen worker 54 - jepsen.util 13014 :ok :read [18 31] INFO [2019-12-17 12:37:57,495] jepsen worker 54 - jepsen.util 13014 :invoke :write [18 9] INFO [2019-12-17 12:37:57,504] jepsen worker 65 - jepsen.util 10955 :fail :cas [21 [45 32]] :ValueNotSwapFrom INFO [2019-12-17 12:37:57,504] jepsen worker 65 - jepsen.util 10955 :invoke :read [21 nil] INFO [2019-12-17 12:37:57,524] jepsen worker 71 - jepsen.util 13301 :ok :read [23 1] INFO [2019-12-17 12:37:57,524] jepsen worker 71 - jepsen.util 13301 :invoke :cas [23 [44 20]] INFO [2019-12-17 12:37:57,544] jepsen worker 13 - jepsen.util 12073 :fail :cas [4 [27 1]] :ValueNotSwapFrom INFO [2019-12-17 12:37:57,544] jepsen worker 13 - jepsen.util 12073 :invoke :read [4 nil] INFO [2019-12-17 12:37:57,563] jepsen worker 47 - jepsen.util 13637 :info :write [15 5] :RequestCanceledException INFO [2019-12-17 12:37:57,564] jepsen worker 47 - jepsen.util 13727 :invoke :read [15 nil] INFO [2019-12-17 12:37:57,569] jepsen worker 52 - jepsen.util 12022 :ok :read [17 9] INFO [2019-12-17 12:37:57,569] jepsen worker 52 - jepsen.util 12022 :invoke :write [17 39] INFO [2019-12-17 12:37:57,592] jepsen worker 47 - jepsen.util 13727 :ok :read [15 38] INFO [2019-12-17 12:37:57,592] jepsen worker 47 - jepsen.util 13727 :invoke :read [15 nil] INFO [2019-12-17 12:37:57,595] jepsen worker 88 - jepsen.util 12598 :ok :read [29 2] INFO [2019-12-17 12:37:57,595] jepsen worker 88 - jepsen.util 12598 :invoke :write [29 22] INFO [2019-12-17 12:37:57,625] jepsen worker 47 - jepsen.util 13727 :ok :read [15 38] INFO [2019-12-17 12:37:57,625] jepsen worker 47 - jepsen.util 13727 :invoke :cas [15 [13 35]] INFO [2019-12-17 12:37:57,644] jepsen worker 47 - jepsen.util 13727 :fail :cas [15 [13 35]] :ValueNotSwapFrom INFO [2019-12-17 12:37:57,644] jepsen worker 47 - jepsen.util 13727 :invoke :cas [15 [40 2]] INFO [2019-12-17 12:37:57,664] jepsen worker 64 - jepsen.util 9784 :ok :write [21 26] INFO [2019-12-17 12:37:57,664] jepsen worker 47 - jepsen.util 13727 :fail :cas [15 [40 2]] :ValueNotSwapFrom INFO [2019-12-17 12:37:57,664] jepsen worker 64 - jepsen.util 9784 :invoke :read [21 nil] INFO [2019-12-17 12:37:57,664] jepsen worker 47 - jepsen.util 13727 :invoke :write [15 34] INFO [2019-12-17 12:37:57,666] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72576,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:37:57,666] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72576,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:37:57,666] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72579,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:37:57,666] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72579,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:37:57,690] jepsen worker 71 - jepsen.util 13301 :fail :cas [23 [44 20]] :ValueNotSwapFrom INFO [2019-12-17 12:37:57,690] jepsen worker 71 - jepsen.util 13301 :invoke :write [23 23] INFO [2019-12-17 12:37:57,703] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:37:57,703] jepsen nemesis - couchbase.util Rebalance check count: 572 INFO [2019-12-17 12:37:57,703] jepsen nemesis - couchbase.util Rebalance stuck count: 553 INFO [2019-12-17 12:37:57,747] jepsen worker 16 - jepsen.util 18736 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:37:57,748] jepsen worker 16 - jepsen.util 18736 :invoke :write [5 8] INFO [2019-12-17 12:37:57,765] jepsen worker 77 - jepsen.util 18257 :info :cas [25 [7 36]] :RequestCanceledException INFO [2019-12-17 12:37:57,766] jepsen worker 77 - jepsen.util 18347 :invoke :read [25 nil] INFO [2019-12-17 12:37:57,768] jepsen worker 29 - jepsen.util 11009 :ok :cas [9 [13 9]] INFO [2019-12-17 12:37:57,768] jepsen worker 29 - jepsen.util 11009 :invoke :write [9 34] INFO [2019-12-17 12:37:57,783] jepsen worker 56 - jepsen.util 13286 :fail :cas [18 [9 33]] :ValueNotSwapFrom INFO [2019-12-17 12:37:57,783] jepsen worker 56 - jepsen.util 13286 :invoke :cas [18 [10 43]] INFO [2019-12-17 12:37:57,801] jepsen worker 2 - jepsen.util 16472 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:37:57,801] jepsen worker 2 - jepsen.util 16472 :invoke :read [0 nil] INFO [2019-12-17 12:37:57,810] jepsen worker 47 - jepsen.util 13727 :ok :write [15 34] INFO [2019-12-17 12:37:57,810] jepsen worker 47 - jepsen.util 13727 :invoke :cas [15 [31 35]] INFO [2019-12-17 12:37:57,818] jepsen worker 35 - jepsen.util 17225 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:37:57,818] jepsen worker 35 - jepsen.util 17225 :invoke :cas [11 [18 48]] INFO [2019-12-17 12:37:57,829] jepsen worker 47 - jepsen.util 13727 :fail :cas [15 [31 35]] :ValueNotSwapFrom INFO [2019-12-17 12:37:57,829] jepsen worker 47 - jepsen.util 13727 :invoke :write [15 7] INFO [2019-12-17 12:37:57,834] jepsen worker 61 - jepsen.util 18601 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:37:57,834] jepsen worker 50 - jepsen.util 17150 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:37:57,834] jepsen worker 7 - jepsen.util 17197 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:37:57,835] jepsen worker 61 - jepsen.util 18601 :invoke :read [20 nil] INFO [2019-12-17 12:37:57,835] jepsen worker 50 - jepsen.util 17150 :invoke :cas [16 [3 22]] INFO [2019-12-17 12:37:57,835] jepsen worker 7 - jepsen.util 17197 :invoke :cas [2 [37 28]] INFO [2019-12-17 12:37:57,848] jepsen worker 8 - jepsen.util 16748 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:37:57,848] jepsen worker 8 - jepsen.util 16748 :invoke :read [2 nil] INFO [2019-12-17 12:37:57,861] jepsen worker 57 - jepsen.util 16887 :info :cas [19 [43 6]] :RequestCanceledException INFO [2019-12-17 12:37:57,863] jepsen worker 57 - jepsen.util 16977 :invoke :write [19 30] INFO [2019-12-17 12:37:57,865] jepsen worker 29 - jepsen.util 11009 :ok :write [9 34] INFO [2019-12-17 12:37:57,865] jepsen worker 29 - jepsen.util 11009 :invoke :cas [9 [26 36]] INFO [2019-12-17 12:37:57,872] jepsen worker 55 - jepsen.util 13195 :ok :write [18 13] INFO [2019-12-17 12:37:57,872] jepsen worker 55 - jepsen.util 13195 :invoke :read [18 nil] INFO [2019-12-17 12:37:57,879] jepsen worker 29 - jepsen.util 11009 :fail :cas [9 [26 36]] :ValueNotSwapFrom INFO [2019-12-17 12:37:57,880] jepsen worker 29 - jepsen.util 11009 :invoke :write [9 18] INFO [2019-12-17 12:37:57,885] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72598,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:37:57,885] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72598,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:37:57,885] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72602,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:37:57,885] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72602,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:37:57,900] jepsen worker 81 - jepsen.util 16731 :info :cas [27 [14 7]] :RequestCanceledException INFO [2019-12-17 12:37:57,900] jepsen worker 25 - jepsen.util 18205 :info :write [8 38] :RequestCanceledException INFO [2019-12-17 12:37:57,900] jepsen worker 15 - jepsen.util 16215 :info :cas [5 [37 9]] :RequestCanceledException INFO [2019-12-17 12:37:57,900] jepsen worker 44 - jepsen.util 17774 :info :cas [14 [33 13]] :RequestCanceledException INFO [2019-12-17 12:37:57,901] jepsen worker 44 - jepsen.util 17864 :invoke :read [14 nil] INFO [2019-12-17 12:37:57,901] jepsen worker 81 - jepsen.util 16821 :invoke :cas [27 [16 18]] INFO [2019-12-17 12:37:57,901] jepsen worker 15 - jepsen.util 16305 :invoke :write [5 26] INFO [2019-12-17 12:37:57,902] jepsen worker 37 - jepsen.util 12007 :ok :read [12 44] INFO [2019-12-17 12:37:57,902] jepsen worker 37 - jepsen.util 12007 :invoke :cas [12 [43 8]] INFO [2019-12-17 12:37:57,904] jepsen worker 25 - jepsen.util 18295 :invoke :cas [8 [28 28]] INFO [2019-12-17 12:37:57,919] jepsen worker 59 - jepsen.util 16259 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:37:57,919] jepsen worker 43 - jepsen.util 17413 :info :write [14 2] :RequestCanceledException INFO [2019-12-17 12:37:57,919] jepsen worker 59 - jepsen.util 16259 :invoke :cas [19 [0 3]] INFO [2019-12-17 12:37:57,920] jepsen worker 43 - jepsen.util 17503 :invoke :read [14 nil] INFO [2019-12-17 12:37:57,933] jepsen worker 31 - jepsen.util 10741 :info :write [10 35] :SyncWriteAmbiguous INFO [2019-12-17 12:37:57,934] jepsen worker 9 - jepsen.util 13149 :info :write [3 20] :RequestCanceledException INFO [2019-12-17 12:37:57,934] jepsen worker 48 - jepsen.util 15888 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:37:57,934] jepsen worker 48 - jepsen.util 15888 :invoke :cas [16 [4 31]] INFO [2019-12-17 12:37:57,935] jepsen worker 31 - jepsen.util 10831 :invoke :write [10 45] INFO [2019-12-17 12:37:57,935] jepsen worker 9 - jepsen.util 13239 :invoke :write [3 7] INFO [2019-12-17 12:37:57,942] jepsen worker 29 - jepsen.util 11009 :ok :write [9 18] INFO [2019-12-17 12:37:57,942] jepsen worker 29 - jepsen.util 11009 :invoke :cas [9 [6 17]] INFO [2019-12-17 12:37:57,948] jepsen worker 58 - jepsen.util 17338 :info :write [19 3] :RequestCanceledException INFO [2019-12-17 12:37:57,949] jepsen worker 58 - jepsen.util 17428 :invoke :read [19 nil] INFO [2019-12-17 12:37:57,972] jepsen worker 29 - jepsen.util 11009 :fail :cas [9 [6 17]] :ValueNotSwapFrom INFO [2019-12-17 12:37:57,973] jepsen worker 29 - jepsen.util 11009 :invoke :write [9 37] INFO [2019-12-17 12:37:57,984] jepsen worker 19 - jepsen.util 12889 :info :write [6 47] :SyncWriteAmbiguous INFO [2019-12-17 12:37:57,986] jepsen worker 19 - jepsen.util 12979 :invoke :read [6 nil] INFO [2019-12-17 12:37:58,001] jepsen worker 51 - jepsen.util 11751 :ok :read [17 9] INFO [2019-12-17 12:37:58,001] jepsen worker 51 - jepsen.util 11751 :invoke :cas [17 [34 17]] INFO [2019-12-17 12:37:58,013] jepsen worker 32 - jepsen.util 12992 :ok :read [10 40] INFO [2019-12-17 12:37:58,013] jepsen worker 32 - jepsen.util 12992 :invoke :cas [10 [38 9]] INFO [2019-12-17 12:37:58,033] jepsen worker 27 - jepsen.util 11637 :info :write [9 13] :RequestCanceledException INFO [2019-12-17 12:37:58,033] jepsen worker 12 - jepsen.util 12612 :fail :cas [4 [12 26]] :ValueNotSwapFrom INFO [2019-12-17 12:37:58,034] jepsen worker 12 - jepsen.util 12612 :invoke :cas [4 [0 48]] INFO [2019-12-17 12:37:58,034] jepsen worker 27 - jepsen.util 11727 :invoke :cas [9 [29 28]] INFO [2019-12-17 12:37:58,043] jepsen worker 19 - jepsen.util 12979 :ok :read [6 31] INFO [2019-12-17 12:37:58,043] jepsen worker 19 - jepsen.util 12979 :invoke :read [6 nil] INFO [2019-12-17 12:37:58,058] jepsen worker 32 - jepsen.util 12992 :fail :cas [10 [38 9]] :ValueNotSwapFrom INFO [2019-12-17 12:37:58,058] jepsen worker 32 - jepsen.util 12992 :invoke :read [10 nil] INFO [2019-12-17 12:37:58,063] jepsen worker 46 - jepsen.util 12196 :info :cas [15 [25 9]] :RequestCanceledException INFO [2019-12-17 12:37:58,064] jepsen worker 19 - jepsen.util 12979 :ok :read [6 31] INFO [2019-12-17 12:37:58,064] jepsen worker 46 - jepsen.util 12286 :invoke :write [15 28] INFO [2019-12-17 12:37:58,064] jepsen worker 19 - jepsen.util 12979 :invoke :cas [6 [13 6]] INFO [2019-12-17 12:37:58,093] jepsen worker 32 - jepsen.util 12992 :ok :read [10 40] INFO [2019-12-17 12:37:58,093] jepsen worker 32 - jepsen.util 12992 :invoke :read [10 nil] INFO [2019-12-17 12:37:58,104] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72622,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:37:58,104] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72622,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:37:58,104] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72625,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:37:58,105] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72625,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:37:58,105] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72626,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:37:58,105] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72626,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:37:58,110] jepsen worker 19 - jepsen.util 12979 :fail :cas [6 [13 6]] :ValueNotSwapFrom INFO [2019-12-17 12:37:58,110] jepsen worker 19 - jepsen.util 12979 :invoke :write [6 29] INFO [2019-12-17 12:37:58,126] jepsen worker 30 - jepsen.util 11730 :info :write [10 12] :RequestCanceledException INFO [2019-12-17 12:37:58,126] jepsen worker 10 - jepsen.util 12340 :info :cas [3 [23 25]] :RequestCanceledException INFO [2019-12-17 12:37:58,126] jepsen worker 42 - jepsen.util 17952 :info :write [14 6] :RequestCanceledException INFO [2019-12-17 12:37:58,127] jepsen worker 42 - jepsen.util 18042 :invoke :write [14 40] INFO [2019-12-17 12:37:58,127] jepsen worker 10 - jepsen.util 12430 :invoke :write [3 48] INFO [2019-12-17 12:37:58,128] jepsen worker 30 - jepsen.util 11820 :invoke :read [10 nil] INFO [2019-12-17 12:37:58,145] jepsen worker 29 - jepsen.util 11009 :ok :write [9 37] INFO [2019-12-17 12:37:58,145] jepsen worker 29 - jepsen.util 11009 :invoke :write [9 23] INFO [2019-12-17 12:37:58,146] jepsen worker 45 - jepsen.util 12375 :fail :read [15 nil] :RequestTimeoutException INFO [2019-12-17 12:37:58,146] jepsen worker 1 - jepsen.util 17371 :info :write [0 22] :RequestCanceledException INFO [2019-12-17 12:37:58,146] jepsen worker 28 - jepsen.util 10738 :info :write [9 0] :RequestCanceledException INFO [2019-12-17 12:37:58,146] jepsen worker 45 - jepsen.util 12375 :invoke :write [15 34] INFO [2019-12-17 12:37:58,147] jepsen worker 1 - jepsen.util 17461 :invoke :read [0 nil] INFO [2019-12-17 12:37:58,148] jepsen worker 28 - jepsen.util 10828 :invoke :cas [9 [28 39]] INFO [2019-12-17 12:37:58,149] jepsen worker 87 - jepsen.util 12147 :ok :read [29 2] INFO [2019-12-17 12:37:58,149] jepsen worker 87 - jepsen.util 12147 :invoke :read [29 nil] INFO [2019-12-17 12:37:58,169] jepsen worker 30 - jepsen.util 11820 :ok :read [10 40] INFO [2019-12-17 12:37:58,169] jepsen worker 30 - jepsen.util 11820 :invoke :read [10 nil] INFO [2019-12-17 12:37:58,196] jepsen worker 39 - jepsen.util 16689 :info :cas [13 [47 28]] :RequestCanceledException INFO [2019-12-17 12:37:58,196] jepsen worker 32 - jepsen.util 12992 :ok :read [10 40] INFO [2019-12-17 12:37:58,196] jepsen worker 32 - jepsen.util 12992 :invoke :write [10 7] INFO [2019-12-17 12:37:58,197] jepsen worker 39 - jepsen.util 16779 :invoke :write [13 48] INFO [2019-12-17 12:37:58,214] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72645,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:37:58,216] jepsen worker 89 - jepsen.util 12329 :info :write [29 7] :RequestCanceledException INFO [2019-12-17 12:37:58,217] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72645,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:37:58,217] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72646,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:37:58,217] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72646,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:37:58,217] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72647,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:37:58,217] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72647,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:37:58,219] jepsen worker 89 - jepsen.util 12419 :invoke :read [29 nil] INFO [2019-12-17 12:37:58,234] jepsen worker 22 - jepsen.util 17302 :info :write [7 0] :RequestCanceledException INFO [2019-12-17 12:37:58,234] jepsen worker 20 - jepsen.util 12350 :info :write [6 6] :RequestCanceledException INFO [2019-12-17 12:37:58,234] jepsen worker 29 - jepsen.util 11009 :ok :write [9 23] INFO [2019-12-17 12:37:58,234] jepsen worker 29 - jepsen.util 11009 :invoke :read [9 nil] INFO [2019-12-17 12:37:58,235] jepsen worker 22 - jepsen.util 17392 :invoke :read [7 nil] INFO [2019-12-17 12:37:58,236] jepsen worker 20 - jepsen.util 12440 :invoke :cas [6 [15 22]] INFO [2019-12-17 12:37:58,248] jepsen worker 18 - jepsen.util 13068 :fail :read [6 nil] :RequestTimeoutException INFO [2019-12-17 12:37:58,248] jepsen worker 18 - jepsen.util 13068 :invoke :write [6 3] INFO [2019-12-17 12:37:58,261] jepsen worker 83 - jepsen.util 17093 :info :write [27 39] :RequestCanceledException INFO [2019-12-17 12:37:58,262] jepsen worker 83 - jepsen.util 17183 :invoke :cas [27 [20 33]] INFO [2019-12-17 12:37:58,275] jepsen worker 13 - jepsen.util 12073 :ok :read [4 0] INFO [2019-12-17 12:37:58,275] jepsen worker 13 - jepsen.util 12073 :invoke :read [4 nil] INFO [2019-12-17 12:37:58,283] jepsen worker 30 - jepsen.util 11820 :ok :read [10 40] INFO [2019-12-17 12:37:58,284] jepsen worker 30 - jepsen.util 11820 :invoke :read [10 nil] INFO [2019-12-17 12:37:58,316] jepsen worker 29 - jepsen.util 11009 :ok :read [9 23] INFO [2019-12-17 12:37:58,317] jepsen worker 29 - jepsen.util 11009 :invoke :cas [9 [31 48]] INFO [2019-12-17 12:37:58,324] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72655,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:37:58,324] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72655,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:37:58,324] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72659,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:37:58,324] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72659,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:37:58,334] jepsen worker 76 - jepsen.util 17176 :info :write [25 42] :RequestCanceledException INFO [2019-12-17 12:37:58,334] jepsen worker 11 - jepsen.util 12161 :info :cas [3 [37 39]] :RequestCanceledException INFO [2019-12-17 12:37:58,335] jepsen worker 76 - jepsen.util 17266 :invoke :read [25 nil] INFO [2019-12-17 12:37:58,335] jepsen worker 11 - jepsen.util 12251 :invoke :write [3 12] INFO [2019-12-17 12:37:58,336] jepsen worker 20 - jepsen.util 12440 :fail :cas [6 [15 22]] :ValueNotSwapFrom INFO [2019-12-17 12:37:58,336] jepsen worker 20 - jepsen.util 12440 :invoke :cas [6 [36 23]] INFO [2019-12-17 12:37:58,357] jepsen worker 55 - jepsen.util 13195 :ok :read [18 13] INFO [2019-12-17 12:37:58,357] jepsen worker 55 - jepsen.util 13195 :invoke :cas [18 [18 22]] INFO [2019-12-17 12:37:58,362] jepsen worker 30 - jepsen.util 11820 :ok :read [10 40] INFO [2019-12-17 12:37:58,362] jepsen worker 30 - jepsen.util 11820 :invoke :write [10 16] INFO [2019-12-17 12:37:58,386] jepsen worker 29 - jepsen.util 11009 :fail :cas [9 [31 48]] :ValueNotSwapFrom INFO [2019-12-17 12:37:58,386] jepsen worker 29 - jepsen.util 11009 :invoke :read [9 nil] INFO [2019-12-17 12:37:58,391] jepsen worker 51 - jepsen.util 11751 :fail :cas [17 [34 17]] :ValueNotSwapFrom INFO [2019-12-17 12:37:58,391] jepsen worker 51 - jepsen.util 11751 :invoke :cas [17 [8 29]] INFO [2019-12-17 12:37:58,396] jepsen worker 56 - jepsen.util 13286 :fail :cas [18 [10 43]] :ValueNotSwapFrom INFO [2019-12-17 12:37:58,396] jepsen worker 56 - jepsen.util 13286 :invoke :cas [18 [0 10]] INFO [2019-12-17 12:37:58,403] jepsen worker 20 - jepsen.util 12440 :fail :cas [6 [36 23]] :ValueNotSwapFrom INFO [2019-12-17 12:37:58,403] jepsen worker 20 - jepsen.util 12440 :invoke :cas [6 [19 29]] INFO [2019-12-17 12:37:58,409] jepsen worker 87 - jepsen.util 12147 :ok :read [29 2] INFO [2019-12-17 12:37:58,409] jepsen worker 87 - jepsen.util 12147 :invoke :read [29 nil] INFO [2019-12-17 12:37:58,425] jepsen worker 20 - jepsen.util 12440 :fail :cas [6 [19 29]] :ValueNotSwapFrom INFO [2019-12-17 12:37:58,426] jepsen worker 20 - jepsen.util 12440 :invoke :read [6 nil] INFO [2019-12-17 12:37:58,434] jepsen worker 72 - jepsen.util 12582 :info :write [24 25] :RequestCanceledException INFO [2019-12-17 12:37:58,434] jepsen worker 6 - jepsen.util 17466 :info :cas [2 [39 32]] :RequestCanceledException INFO [2019-12-17 12:37:58,435] jepsen worker 6 - jepsen.util 17556 :invoke :write [2 31] INFO [2019-12-17 12:37:58,437] jepsen worker 72 - jepsen.util 12672 :invoke :cas [24 [12 40]] INFO [2019-12-17 12:37:58,442] jepsen worker 89 - jepsen.util 12419 :ok :read [29 2] INFO [2019-12-17 12:37:58,442] jepsen worker 89 - jepsen.util 12419 :invoke :read [29 nil] INFO [2019-12-17 12:37:58,447] jepsen worker 20 - jepsen.util 12440 :ok :read [6 31] INFO [2019-12-17 12:37:58,447] jepsen worker 20 - jepsen.util 12440 :invoke :write [6 36] INFO [2019-12-17 12:37:58,447] jepsen worker 68 - jepsen.util 16358 :info :cas [22 [39 46]] :RequestCanceledException INFO [2019-12-17 12:37:58,447] jepsen worker 74 - jepsen.util 12314 :info :write [24 22] :RequestCanceledException INFO [2019-12-17 12:37:58,448] jepsen worker 68 - jepsen.util 16448 :invoke :write [22 5] INFO [2019-12-17 12:37:58,448] jepsen worker 74 - jepsen.util 12404 :invoke :read [24 nil] INFO [2019-12-17 12:37:58,465] jepsen worker 49 - jepsen.util 17149 :info :cas [16 [37 9]] :RequestCanceledException INFO [2019-12-17 12:37:58,466] jepsen worker 49 - jepsen.util 17239 :invoke :read [16 nil] INFO [2019-12-17 12:37:58,500] jepsen worker 24 - jepsen.util 16404 :info :cas [8 [13 34]] :RequestCanceledException INFO [2019-12-17 12:37:58,501] jepsen worker 24 - jepsen.util 16494 :invoke :read [8 nil] INFO [2019-12-17 12:37:58,514] jepsen worker 40 - jepsen.util 18040 :info :write [13 46] :RequestCanceledException INFO [2019-12-17 12:37:58,515] jepsen worker 40 - jepsen.util 18130 :invoke :read [13 nil] INFO [2019-12-17 12:37:58,535] jepsen worker 73 - jepsen.util 11683 :info :cas [24 [3 42]] :RequestCanceledException INFO [2019-12-17 12:37:58,535] jepsen worker 41 - jepsen.util 18581 :info :cas [13 [13 13]] :RequestCanceledException INFO [2019-12-17 12:37:58,536] jepsen worker 41 - jepsen.util 18671 :invoke :read [13 nil] INFO [2019-12-17 12:37:58,537] jepsen worker 73 - jepsen.util 11773 :invoke :cas [24 [45 28]] INFO [2019-12-17 12:37:58,543] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72680,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:37:58,543] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72680,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:37:58,543] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72683,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:37:58,543] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72683,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:37:58,543] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72684,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:37:58,543] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72684,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:37:58,554] jepsen worker 73 - jepsen.util 11773 :fail :cas [24 [45 28]] :ValueNotSwapFrom INFO [2019-12-17 12:37:58,554] jepsen worker 73 - jepsen.util 11773 :invoke :read [24 nil] INFO [2019-12-17 12:37:58,565] jepsen worker 34 - jepsen.util 18034 :info :cas [11 [42 3]] :RequestCanceledException INFO [2019-12-17 12:37:58,567] jepsen worker 34 - jepsen.util 18124 :invoke :read [11 nil] INFO [2019-12-17 12:37:58,572] jepsen worker 73 - jepsen.util 11773 :ok :read [24 0] INFO [2019-12-17 12:37:58,572] jepsen worker 73 - jepsen.util 11773 :invoke :read [24 nil] INFO [2019-12-17 12:37:58,586] jepsen worker 73 - jepsen.util 11773 :ok :read [24 0] INFO [2019-12-17 12:37:58,586] jepsen worker 73 - jepsen.util 11773 :invoke :write [24 19] INFO [2019-12-17 12:37:58,609] jepsen worker 89 - jepsen.util 12419 :ok :read [29 2] INFO [2019-12-17 12:37:58,610] jepsen worker 89 - jepsen.util 12419 :invoke :cas [29 [32 15]] INFO [2019-12-17 12:37:58,635] jepsen worker 53 - jepsen.util 11933 :info :write [17 37] :RequestCanceledException INFO [2019-12-17 12:37:58,636] jepsen worker 53 - jepsen.util 12023 :invoke :read [17 nil] INFO [2019-12-17 12:37:58,652] jepsen worker 55 - jepsen.util 13195 :fail :cas [18 [18 22]] :ValueNotSwapFrom INFO [2019-12-17 12:37:58,652] jepsen worker 55 - jepsen.util 13195 :invoke :cas [18 [49 17]] INFO [2019-12-17 12:37:58,653] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72703,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:37:58,653] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72703,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:37:58,653] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72707,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:37:58,653] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72707,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:37:58,673] jepsen worker 13 - jepsen.util 12073 :ok :read [4 0] INFO [2019-12-17 12:37:58,673] jepsen worker 13 - jepsen.util 12073 :invoke :cas [4 [7 31]] INFO [2019-12-17 12:37:58,685] jepsen worker 87 - jepsen.util 12147 :ok :read [29 2] INFO [2019-12-17 12:37:58,685] jepsen worker 87 - jepsen.util 12147 :invoke :cas [29 [13 26]] INFO [2019-12-17 12:37:58,717] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:37:58,717] jepsen nemesis - couchbase.util Rebalance check count: 573 INFO [2019-12-17 12:37:58,717] jepsen nemesis - couchbase.util Rebalance stuck count: 554 INFO [2019-12-17 12:37:58,732] jepsen worker 89 - jepsen.util 12419 :fail :cas [29 [32 15]] :ValueNotSwapFrom INFO [2019-12-17 12:37:58,732] jepsen worker 89 - jepsen.util 12419 :invoke :read [29 nil] INFO [2019-12-17 12:37:58,775] jepsen worker 47 - jepsen.util 13727 :ok :write [15 7] INFO [2019-12-17 12:37:58,775] jepsen worker 73 - jepsen.util 11773 :ok :write [24 19] INFO [2019-12-17 12:37:58,775] jepsen worker 9 - jepsen.util 13239 :ok :write [3 7] INFO [2019-12-17 12:37:58,778] jepsen worker 37 - jepsen.util 12007 :fail :cas [12 [43 8]] :ValueNotSwapFrom INFO [2019-12-17 12:37:58,783] jepsen worker 47 - jepsen.util 13727 :invoke :cas [15 [5 23]] INFO [2019-12-17 12:37:58,783] jepsen worker 9 - jepsen.util 13239 :invoke :cas [3 [16 30]] INFO [2019-12-17 12:37:58,783] jepsen worker 73 - jepsen.util 11773 :invoke :cas [24 [7 10]] INFO [2019-12-17 12:37:58,783] jepsen worker 37 - jepsen.util 12007 :invoke :cas [12 [28 10]] INFO [2019-12-17 12:37:58,794] jepsen worker 27 - jepsen.util 11727 :fail :cas [9 [29 28]] :ValueNotSwapFrom INFO [2019-12-17 12:37:58,794] jepsen worker 27 - jepsen.util 11727 :invoke :write [9 34] INFO [2019-12-17 12:37:58,801] jepsen worker 17 - jepsen.util 17297 :info :cas [5 [46 11]] :RequestCanceledException INFO [2019-12-17 12:37:58,804] jepsen worker 17 - jepsen.util 17387 :invoke :read [5 nil] INFO [2019-12-17 12:37:58,828] jepsen worker 60 - jepsen.util 18150 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:37:58,828] jepsen worker 60 - jepsen.util 18150 :invoke :cas [20 [32 33]] INFO [2019-12-17 12:37:58,843] jepsen worker 28 - jepsen.util 10828 :fail :cas [9 [28 39]] :ValueNotSwapFrom INFO [2019-12-17 12:37:58,844] jepsen worker 28 - jepsen.util 10828 :invoke :read [9 nil] INFO [2019-12-17 12:37:58,849] jepsen worker 79 - jepsen.util 12679 :info :write [26 46] :RequestCanceledException INFO [2019-12-17 12:37:58,850] jepsen worker 79 - jepsen.util 12769 :invoke :write [26 46] INFO [2019-12-17 12:37:58,856] jepsen worker 73 - jepsen.util 11773 :fail :cas [24 [7 10]] :ValueNotSwapFrom INFO [2019-12-17 12:37:58,856] jepsen worker 73 - jepsen.util 11773 :invoke :read [24 nil] INFO [2019-12-17 12:37:58,863] jepsen worker 86 - jepsen.util 17816 :info :write [28 47] :RequestCanceledException INFO [2019-12-17 12:37:58,864] jepsen worker 86 - jepsen.util 17906 :invoke :cas [28 [8 44]] INFO [2019-12-17 12:37:58,867] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72726,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:37:58,867] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72726,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:37:58,867] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72728,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:37:58,867] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72728,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:37:58,886] jepsen worker 47 - jepsen.util 13727 :fail :cas [15 [5 23]] :ValueNotSwapFrom INFO [2019-12-17 12:37:58,886] jepsen worker 47 - jepsen.util 13727 :invoke :write [15 19] INFO [2019-12-17 12:37:58,898] jepsen worker 62 - jepsen.util 17792 :info :cas [20 [48 38]] :RequestCanceledException INFO [2019-12-17 12:37:58,898] jepsen worker 75 - jepsen.util 16635 :info :cas [25 [37 0]] :RequestCanceledException INFO [2019-12-17 12:37:58,899] jepsen worker 75 - jepsen.util 16725 :invoke :write [25 14] INFO [2019-12-17 12:37:58,899] jepsen worker 62 - jepsen.util 17882 :invoke :write [20 1] INFO [2019-12-17 12:37:58,903] jepsen worker 73 - jepsen.util 11773 :ok :read [24 19] INFO [2019-12-17 12:37:58,903] jepsen worker 73 - jepsen.util 11773 :invoke :read [24 nil] INFO [2019-12-17 12:37:58,917] jepsen worker 26 - jepsen.util 17936 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:37:58,917] jepsen worker 26 - jepsen.util 17936 :invoke :write [8 47] INFO [2019-12-17 12:37:58,941] jepsen worker 73 - jepsen.util 11773 :ok :read [24 19] INFO [2019-12-17 12:37:58,941] jepsen worker 73 - jepsen.util 11773 :invoke :write [24 24] INFO [2019-12-17 12:37:58,957] jepsen worker 27 - jepsen.util 11727 :ok :write [9 34] INFO [2019-12-17 12:37:58,957] jepsen worker 27 - jepsen.util 11727 :invoke :read [9 nil] INFO [2019-12-17 12:37:59,034] jepsen worker 66 - jepsen.util 17346 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:37:59,034] jepsen worker 66 - jepsen.util 17346 :invoke :cas [22 [35 2]] INFO [2019-12-17 12:37:59,048] jepsen worker 85 - jepsen.util 17815 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:37:59,048] jepsen worker 85 - jepsen.util 17815 :invoke :write [28 40] INFO [2019-12-17 12:37:59,070] jepsen worker 89 - jepsen.util 12419 :ok :read [29 2] INFO [2019-12-17 12:37:59,070] jepsen worker 89 - jepsen.util 12419 :invoke :read [29 nil] INFO [2019-12-17 12:37:59,086] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72736,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:37:59,086] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72736,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:37:59,086] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72740,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:37:59,086] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72740,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:37:59,086] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72741,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:37:59,086] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72741,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:37:59,117] jepsen worker 70 - jepsen.util 13030 :info :write [23 5] :RequestCanceledException INFO [2019-12-17 12:37:59,120] jepsen worker 56 - jepsen.util 13286 :fail :cas [18 [0 10]] :ValueNotSwapFrom INFO [2019-12-17 12:37:59,121] jepsen worker 56 - jepsen.util 13286 :invoke :write [18 26] INFO [2019-12-17 12:37:59,122] jepsen worker 70 - jepsen.util 13120 :invoke :read [23 nil] INFO [2019-12-17 12:37:59,134] jepsen worker 14 - jepsen.util 12164 :info :write [4 19] :RequestCanceledException INFO [2019-12-17 12:37:59,135] jepsen worker 14 - jepsen.util 12254 :invoke :cas [4 [4 10]] INFO [2019-12-17 12:37:59,165] jepsen worker 3 - jepsen.util 11883 :info :write [1 25] :RequestCanceledException INFO [2019-12-17 12:37:59,165] jepsen worker 84 - jepsen.util 16734 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:37:59,165] jepsen worker 84 - jepsen.util 16734 :invoke :write [28 21] INFO [2019-12-17 12:37:59,167] jepsen worker 3 - jepsen.util 11973 :invoke :write [1 43] INFO [2019-12-17 12:37:59,201] jepsen worker 67 - jepsen.util 17347 :info :write [22 38] :RequestCanceledException INFO [2019-12-17 12:37:59,201] jepsen worker 33 - jepsen.util 16953 :info :cas [11 [36 40]] :RequestCanceledException INFO [2019-12-17 12:37:59,201] jepsen worker 67 - jepsen.util 17437 :invoke :read [22 nil] INFO [2019-12-17 12:37:59,201] jepsen worker 33 - jepsen.util 17043 :invoke :cas [11 [17 29]] INFO [2019-12-17 12:37:59,212] jepsen worker 82 - jepsen.util 17542 :info :cas [27 [42 42]] :RequestCanceledException INFO [2019-12-17 12:37:59,213] jepsen worker 82 - jepsen.util 17632 :invoke :read [27 nil] INFO [2019-12-17 12:37:59,232] jepsen worker 10 - jepsen.util 12430 :ok :write [3 48] INFO [2019-12-17 12:37:59,232] jepsen worker 73 - jepsen.util 11773 :ok :write [24 24] INFO [2019-12-17 12:37:59,232] jepsen worker 10 - jepsen.util 12430 :invoke :cas [3 [9 34]] INFO [2019-12-17 12:37:59,232] jepsen worker 73 - jepsen.util 11773 :invoke :write [24 5] INFO [2019-12-17 12:37:59,234] jepsen worker 23 - jepsen.util 17663 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:37:59,234] jepsen worker 69 - jepsen.util 12759 :info :write [23 1] :RequestCanceledException INFO [2019-12-17 12:37:59,234] jepsen worker 23 - jepsen.util 17663 :invoke :write [7 2] INFO [2019-12-17 12:37:59,235] jepsen worker 69 - jepsen.util 12849 :invoke :cas [23 [46 45]] INFO [2019-12-17 12:37:59,237] jepsen worker 47 - jepsen.util 13727 :ok :write [15 19] INFO [2019-12-17 12:37:59,237] jepsen worker 47 - jepsen.util 13727 :invoke :write [15 25] INFO [2019-12-17 12:37:59,248] jepsen worker 80 - jepsen.util 11960 :info :write [26 4] :RequestCanceledException INFO [2019-12-17 12:37:59,249] jepsen worker 80 - jepsen.util 12050 :invoke :read [26 nil] INFO [2019-12-17 12:37:59,255] jepsen worker 72 - jepsen.util 12672 :fail :cas [24 [12 40]] :ValueNotSwapFrom INFO [2019-12-17 12:37:59,255] jepsen worker 72 - jepsen.util 12672 :invoke :cas [24 [36 37]] INFO [2019-12-17 12:37:59,265] jepsen worker 0 - jepsen.util 17640 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:37:59,265] jepsen worker 0 - jepsen.util 17640 :invoke :read [0 nil] INFO [2019-12-17 12:37:59,292] jepsen worker 89 - jepsen.util 12419 :ok :read [29 2] INFO [2019-12-17 12:37:59,293] jepsen worker 89 - jepsen.util 12419 :invoke :cas [29 [44 11]] INFO [2019-12-17 12:37:59,300] jepsen worker 9 - jepsen.util 13239 :fail :cas [3 [16 30]] :ValueNotSwapFrom INFO [2019-12-17 12:37:59,300] jepsen worker 9 - jepsen.util 13239 :invoke :read [3 nil] INFO [2019-12-17 12:37:59,305] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72752,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:37:59,305] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72752,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:37:59,305] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72762,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:37:59,305] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72753,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:37:59,305] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72762,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:37:59,305] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72753,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:37:59,305] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72764,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:37:59,306] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72764,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:37:59,322] jepsen worker 10 - jepsen.util 12430 :fail :cas [3 [9 34]] :ValueNotSwapFrom INFO [2019-12-17 12:37:59,322] jepsen worker 10 - jepsen.util 12430 :invoke :cas [3 [31 38]] INFO [2019-12-17 12:37:59,330] jepsen worker 72 - jepsen.util 12672 :fail :cas [24 [36 37]] :ValueNotSwapFrom INFO [2019-12-17 12:37:59,331] jepsen worker 72 - jepsen.util 12672 :invoke :cas [24 [44 42]] INFO [2019-12-17 12:37:59,338] jepsen worker 9 - jepsen.util 13239 :ok :read [3 48] INFO [2019-12-17 12:37:59,338] jepsen worker 9 - jepsen.util 13239 :invoke :cas [3 [24 49]] INFO [2019-12-17 12:37:59,348] jepsen worker 21 - jepsen.util 16581 :info :cas [7 [5 17]] :RequestCanceledException INFO [2019-12-17 12:37:59,350] jepsen worker 21 - jepsen.util 16671 :invoke :cas [7 [30 36]] INFO [2019-12-17 12:37:59,351] jepsen worker 14 - jepsen.util 12254 :fail :cas [4 [4 10]] :ValueNotSwapFrom INFO [2019-12-17 12:37:59,351] jepsen worker 14 - jepsen.util 12254 :invoke :write [4 39] INFO [2019-12-17 12:37:59,355] jepsen worker 10 - jepsen.util 12430 :fail :cas [3 [31 38]] :ValueNotSwapFrom INFO [2019-12-17 12:37:59,355] jepsen worker 10 - jepsen.util 12430 :invoke :read [3 nil] INFO [2019-12-17 12:37:59,363] jepsen worker 72 - jepsen.util 12672 :fail :cas [24 [44 42]] :ValueNotSwapFrom INFO [2019-12-17 12:37:59,364] jepsen worker 72 - jepsen.util 12672 :invoke :write [24 34] INFO [2019-12-17 12:37:59,384] jepsen worker 9 - jepsen.util 13239 :fail :cas [3 [24 49]] :ValueNotSwapFrom INFO [2019-12-17 12:37:59,384] jepsen worker 9 - jepsen.util 13239 :invoke :read [3 nil] INFO [2019-12-17 12:37:59,399] jepsen worker 10 - jepsen.util 12430 :ok :read [3 48] INFO [2019-12-17 12:37:59,399] jepsen worker 10 - jepsen.util 12430 :invoke :read [3 nil] INFO [2019-12-17 12:37:59,400] jepsen worker 38 - jepsen.util 13088 :info :write [12 1] :RequestCanceledException INFO [2019-12-17 12:37:59,401] jepsen worker 38 - jepsen.util 13178 :invoke :cas [12 [41 1]] INFO [2019-12-17 12:37:59,416] jepsen worker 78 - jepsen.util 12228 :info :write [26 8] :RequestCanceledException INFO [2019-12-17 12:37:59,418] jepsen worker 78 - jepsen.util 12318 :invoke :read [26 nil] INFO [2019-12-17 12:37:59,447] jepsen worker 53 - jepsen.util 12023 :ok :read [17 9] INFO [2019-12-17 12:37:59,448] jepsen worker 53 - jepsen.util 12023 :invoke :write [17 4] INFO [2019-12-17 12:37:59,461] jepsen worker 73 - jepsen.util 11773 :ok :write [24 5] INFO [2019-12-17 12:37:59,461] jepsen worker 73 - jepsen.util 11773 :invoke :read [24 nil] INFO [2019-12-17 12:37:59,486] jepsen worker 73 - jepsen.util 11773 :ok :read [24 5] INFO [2019-12-17 12:37:59,487] jepsen worker 73 - jepsen.util 11773 :invoke :write [24 35] INFO [2019-12-17 12:37:59,489] jepsen worker 51 - jepsen.util 11751 :fail :cas [17 [8 29]] :ValueNotSwapFrom INFO [2019-12-17 12:37:59,489] jepsen worker 51 - jepsen.util 11751 :invoke :read [17 nil] INFO [2019-12-17 12:37:59,490] jepsen worker 13 - jepsen.util 12073 :fail :cas [4 [7 31]] :ValueNotSwapFrom INFO [2019-12-17 12:37:59,491] jepsen worker 13 - jepsen.util 12073 :invoke :cas [4 [36 20]] INFO [2019-12-17 12:37:59,501] jepsen worker 36 - jepsen.util 13266 :info :write [12 32] :RequestCanceledException INFO [2019-12-17 12:37:59,502] jepsen worker 36 - jepsen.util 13356 :invoke :cas [12 [45 35]] INFO [2019-12-17 12:37:59,507] jepsen worker 55 - jepsen.util 13195 :fail :cas [18 [49 17]] :ValueNotSwapFrom INFO [2019-12-17 12:37:59,507] jepsen worker 55 - jepsen.util 13195 :invoke :write [18 18] INFO [2019-12-17 12:37:59,524] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72788,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:37:59,524] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72788,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:37:59,524] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72791,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:37:59,524] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72791,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:37:59,534] jepsen worker 5 - jepsen.util 12155 :info :write [1 41] :RequestCanceledException INFO [2019-12-17 12:37:59,534] jepsen worker 54 - jepsen.util 13014 :info :write [18 9] :RequestCanceledException INFO [2019-12-17 12:37:59,534] jepsen worker 63 - jepsen.util 11043 :info :write [21 41] :RequestCanceledException INFO [2019-12-17 12:37:59,535] jepsen worker 63 - jepsen.util 11133 :invoke :read [21 nil] INFO [2019-12-17 12:37:59,535] jepsen worker 54 - jepsen.util 13104 :invoke :write [18 36] INFO [2019-12-17 12:37:59,535] jepsen worker 5 - jepsen.util 12245 :invoke :write [1 21] INFO [2019-12-17 12:37:59,548] jepsen worker 4 - jepsen.util 11704 :info :write [1 7] :RequestCanceledException INFO [2019-12-17 12:37:59,548] jepsen worker 4 - jepsen.util 11794 :invoke :read [1 nil] INFO [2019-12-17 12:37:59,584] jepsen worker 87 - jepsen.util 12147 :fail :cas [29 [13 26]] :ValueNotSwapFrom INFO [2019-12-17 12:37:59,585] jepsen worker 87 - jepsen.util 12147 :invoke :read [29 nil] INFO [2019-12-17 12:37:59,601] jepsen worker 65 - jepsen.util 10955 :fail :read [21 nil] :RequestTimeoutException INFO [2019-12-17 12:37:59,601] jepsen worker 65 - jepsen.util 10955 :invoke :write [21 24] INFO [2019-12-17 12:37:59,615] jepsen worker 73 - jepsen.util 11773 :ok :write [24 35] INFO [2019-12-17 12:37:59,616] jepsen worker 73 - jepsen.util 11773 :invoke :read [24 nil] INFO [2019-12-17 12:37:59,616] jepsen worker 47 - jepsen.util 13727 :ok :write [15 25] INFO [2019-12-17 12:37:59,616] jepsen worker 47 - jepsen.util 13727 :invoke :write [15 26] INFO [2019-12-17 12:37:59,632] jepsen worker 89 - jepsen.util 12419 :fail :cas [29 [44 11]] :ValueNotSwapFrom INFO [2019-12-17 12:37:59,632] jepsen worker 89 - jepsen.util 12419 :invoke :cas [29 [27 23]] INFO [2019-12-17 12:37:59,647] jepsen worker 63 - jepsen.util 11133 :ok :read [21 26] INFO [2019-12-17 12:37:59,648] jepsen worker 63 - jepsen.util 11133 :invoke :cas [21 [39 44]] INFO [2019-12-17 12:37:59,648] jepsen worker 88 - jepsen.util 12598 :info :write [29 22] :RequestCanceledException INFO [2019-12-17 12:37:59,648] jepsen worker 52 - jepsen.util 12022 :info :write [17 39] :RequestCanceledException INFO [2019-12-17 12:37:59,649] jepsen worker 52 - jepsen.util 12112 :invoke :cas [17 [21 13]] INFO [2019-12-17 12:37:59,650] jepsen worker 88 - jepsen.util 12688 :invoke :read [29 nil] INFO [2019-12-17 12:37:59,659] jepsen worker 37 - jepsen.util 12007 :fail :cas [12 [28 10]] :ValueNotSwapFrom INFO [2019-12-17 12:37:59,659] jepsen worker 37 - jepsen.util 12007 :invoke :read [12 nil] INFO [2019-12-17 12:37:59,687] jepsen worker 65 - jepsen.util 10955 :ok :write [21 24] INFO [2019-12-17 12:37:59,688] jepsen worker 65 - jepsen.util 10955 :invoke :cas [21 [29 41]] INFO [2019-12-17 12:37:59,697] jepsen worker 64 - jepsen.util 9784 :fail :read [21 nil] :RequestTimeoutException INFO [2019-12-17 12:37:59,698] jepsen worker 64 - jepsen.util 9784 :invoke :read [21 nil] INFO [2019-12-17 12:37:59,717] jepsen worker 65 - jepsen.util 10955 :fail :cas [21 [29 41]] :ValueNotSwapFrom INFO [2019-12-17 12:37:59,717] jepsen worker 65 - jepsen.util 10955 :invoke :read [21 nil] INFO [2019-12-17 12:37:59,728] jepsen worker 64 - jepsen.util 9784 :ok :read [21 24] INFO [2019-12-17 12:37:59,728] jepsen worker 64 - jepsen.util 9784 :invoke :cas [21 [45 12]] INFO [2019-12-17 12:37:59,731] jepsen worker 87 - jepsen.util 12147 :ok :read [29 2] INFO [2019-12-17 12:37:59,731] jepsen worker 87 - jepsen.util 12147 :invoke :read [29 nil] INFO [2019-12-17 12:37:59,732] jepsen worker 71 - jepsen.util 13301 :info :write [23 23] :RequestCanceledException INFO [2019-12-17 12:37:59,734] jepsen worker 89 - jepsen.util 12419 :fail :cas [29 [27 23]] :ValueNotSwapFrom INFO [2019-12-17 12:37:59,734] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72814,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:37:59,734] jepsen worker 71 - jepsen.util 13391 :invoke :write [23 47] INFO [2019-12-17 12:37:59,734] jepsen worker 89 - jepsen.util 12419 :invoke :cas [29 [4 27]] INFO [2019-12-17 12:37:59,734] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72814,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:37:59,734] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72816,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:37:59,734] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72816,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:37:59,743] jepsen worker 65 - jepsen.util 10955 :ok :read [21 24] INFO [2019-12-17 12:37:59,743] jepsen worker 65 - jepsen.util 10955 :invoke :read [21 nil] INFO [2019-12-17 12:37:59,743] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:37:59,744] jepsen nemesis - couchbase.util Rebalance check count: 574 INFO [2019-12-17 12:37:59,744] jepsen nemesis - couchbase.util Rebalance stuck count: 555 INFO [2019-12-17 12:37:59,766] jepsen worker 64 - jepsen.util 9784 :fail :cas [21 [45 12]] :ValueNotSwapFrom INFO [2019-12-17 12:37:59,768] jepsen worker 64 - jepsen.util 9784 :invoke :cas [21 [8 30]] INFO [2019-12-17 12:37:59,774] jepsen worker 65 - jepsen.util 10955 :ok :read [21 24] INFO [2019-12-17 12:37:59,774] jepsen worker 65 - jepsen.util 10955 :invoke :read [21 nil] INFO [2019-12-17 12:37:59,782] jepsen worker 80 - jepsen.util 12050 :ok :read [26 18] INFO [2019-12-17 12:37:59,782] jepsen worker 80 - jepsen.util 12050 :invoke :read [26 nil] INFO [2019-12-17 12:37:59,794] jepsen worker 64 - jepsen.util 9784 :fail :cas [21 [8 30]] :ValueNotSwapFrom INFO [2019-12-17 12:37:59,795] jepsen worker 64 - jepsen.util 9784 :invoke :read [21 nil] INFO [2019-12-17 12:37:59,799] jepsen worker 65 - jepsen.util 10955 :ok :read [21 24] INFO [2019-12-17 12:37:59,799] jepsen worker 65 - jepsen.util 10955 :invoke :write [21 12] INFO [2019-12-17 12:37:59,801] jepsen worker 38 - jepsen.util 13178 :fail :cas [12 [41 1]] :ValueNotSwapFrom INFO [2019-12-17 12:37:59,801] jepsen worker 38 - jepsen.util 13178 :invoke :read [12 nil] INFO [2019-12-17 12:37:59,811] jepsen worker 64 - jepsen.util 9784 :ok :read [21 24] INFO [2019-12-17 12:37:59,812] jepsen worker 64 - jepsen.util 9784 :invoke :cas [21 [30 32]] INFO [2019-12-17 12:37:59,817] jepsen worker 78 - jepsen.util 12318 :ok :read [26 18] INFO [2019-12-17 12:37:59,817] jepsen worker 78 - jepsen.util 12318 :invoke :read [26 nil] INFO [2019-12-17 12:37:59,827] jepsen worker 47 - jepsen.util 13727 :ok :write [15 26] INFO [2019-12-17 12:37:59,828] jepsen worker 47 - jepsen.util 13727 :invoke :read [15 nil] INFO [2019-12-17 12:37:59,848] jepsen worker 31 - jepsen.util 10831 :info :write [10 45] :SyncWriteAmbiguous INFO [2019-12-17 12:37:59,848] jepsen worker 16 - jepsen.util 18736 :info :write [5 8] :RequestCanceledException INFO [2019-12-17 12:37:59,849] jepsen worker 77 - jepsen.util 18347 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:37:59,849] jepsen worker 77 - jepsen.util 18347 :invoke :read [25 nil] INFO [2019-12-17 12:37:59,850] jepsen worker 16 - jepsen.util 18826 :invoke :read [5 nil] INFO [2019-12-17 12:37:59,850] jepsen worker 31 - jepsen.util 10921 :invoke :write [10 36] INFO [2019-12-17 12:37:59,852] jepsen worker 38 - jepsen.util 13178 :ok :read [12 1] INFO [2019-12-17 12:37:59,852] jepsen worker 38 - jepsen.util 13178 :invoke :write [12 21] INFO [2019-12-17 12:37:59,861] jepsen worker 47 - jepsen.util 13727 :ok :read [15 26] INFO [2019-12-17 12:37:59,861] jepsen worker 47 - jepsen.util 13727 :invoke :write [15 8] INFO [2019-12-17 12:37:59,878] jepsen worker 51 - jepsen.util 11751 :ok :read [17 9] INFO [2019-12-17 12:37:59,879] jepsen worker 51 - jepsen.util 11751 :invoke :read [17 nil] INFO [2019-12-17 12:37:59,900] jepsen worker 2 - jepsen.util 16472 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:37:59,901] jepsen worker 2 - jepsen.util 16472 :invoke :write [0 1] INFO [2019-12-17 12:37:59,901] jepsen worker 65 - jepsen.util 10955 :ok :write [21 12] INFO [2019-12-17 12:37:59,902] jepsen worker 65 - jepsen.util 10955 :invoke :read [21 nil] INFO [2019-12-17 12:37:59,910] jepsen worker 36 - jepsen.util 13356 :fail :cas [12 [45 35]] :ValueNotSwapFrom INFO [2019-12-17 12:37:59,911] jepsen worker 36 - jepsen.util 13356 :invoke :read [12 nil] INFO [2019-12-17 12:37:59,916] jepsen worker 35 - jepsen.util 17225 :info :cas [11 [18 48]] :RequestCanceledException INFO [2019-12-17 12:37:59,917] jepsen worker 15 - jepsen.util 16305 :info :write [5 26] :RequestCanceledException INFO [2019-12-17 12:37:59,918] jepsen worker 35 - jepsen.util 17315 :invoke :write [11 27] INFO [2019-12-17 12:37:59,918] jepsen worker 15 - jepsen.util 16395 :invoke :read [5 nil] INFO [2019-12-17 12:37:59,919] jepsen worker 70 - jepsen.util 13120 :ok :read [23 1] INFO [2019-12-17 12:37:59,919] jepsen worker 70 - jepsen.util 13120 :invoke :read [23 nil] INFO [2019-12-17 12:37:59,930] jepsen worker 4 - jepsen.util 11794 :ok :read [1 26] INFO [2019-12-17 12:37:59,930] jepsen worker 4 - jepsen.util 11794 :invoke :read [1 nil] INFO [2019-12-17 12:37:59,934] jepsen worker 7 - jepsen.util 17197 :info :cas [2 [37 28]] :RequestCanceledException INFO [2019-12-17 12:37:59,934] jepsen worker 50 - jepsen.util 17150 :info :cas [16 [3 22]] :RequestCanceledException INFO [2019-12-17 12:37:59,934] jepsen worker 44 - jepsen.util 17864 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:37:59,934] jepsen worker 61 - jepsen.util 18601 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:37:59,934] jepsen worker 25 - jepsen.util 18295 :info :cas [8 [28 28]] :RequestCanceledException INFO [2019-12-17 12:37:59,934] jepsen worker 44 - jepsen.util 17864 :invoke :cas [14 [37 30]] INFO [2019-12-17 12:37:59,934] jepsen worker 61 - jepsen.util 18601 :invoke :read [20 nil] INFO [2019-12-17 12:37:59,935] jepsen worker 50 - jepsen.util 17240 :invoke :cas [16 [12 31]] INFO [2019-12-17 12:37:59,935] jepsen worker 7 - jepsen.util 17287 :invoke :cas [2 [48 45]] INFO [2019-12-17 12:37:59,937] jepsen worker 65 - jepsen.util 10955 :ok :read [21 12] INFO [2019-12-17 12:37:59,938] jepsen worker 65 - jepsen.util 10955 :invoke :cas [21 [45 15]] INFO [2019-12-17 12:37:59,945] jepsen worker 88 - jepsen.util 12688 :ok :read [29 2] INFO [2019-12-17 12:37:59,945] jepsen worker 88 - jepsen.util 12688 :invoke :cas [29 [5 20]] INFO [2019-12-17 12:37:59,948] jepsen worker 8 - jepsen.util 16748 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:37:59,948] jepsen worker 43 - jepsen.util 17503 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:37:59,949] jepsen worker 8 - jepsen.util 16748 :invoke :cas [2 [34 40]] INFO [2019-12-17 12:37:59,949] jepsen worker 43 - jepsen.util 17503 :invoke :write [14 6] INFO [2019-12-17 12:37:59,949] jepsen worker 65 - jepsen.util 10955 :fail :cas [21 [45 15]] :ValueNotSwapFrom INFO [2019-12-17 12:37:59,950] jepsen worker 65 - jepsen.util 10955 :invoke :read [21 nil] INFO [2019-12-17 12:37:59,950] jepsen worker 25 - jepsen.util 18385 :invoke :read [8 nil] INFO [2019-12-17 12:37:59,950] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72841,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:37:59,950] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72841,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:37:59,950] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72844,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:37:59,950] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72844,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:37:59,951] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72845,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:37:59,951] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72845,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:37:59,962] jepsen worker 57 - jepsen.util 16977 :info :write [19 30] :RequestCanceledException INFO [2019-12-17 12:37:59,967] jepsen worker 65 - jepsen.util 10955 :ok :read [21 12] INFO [2019-12-17 12:37:59,967] jepsen worker 65 - jepsen.util 10955 :invoke :cas [21 [7 0]] INFO [2019-12-17 12:37:59,968] jepsen worker 57 - jepsen.util 17067 :invoke :write [19 35] INFO [2019-12-17 12:37:59,972] jepsen worker 37 - jepsen.util 12007 :ok :read [12 1] INFO [2019-12-17 12:37:59,972] jepsen worker 37 - jepsen.util 12007 :invoke :cas [12 [22 4]] INFO [2019-12-17 12:37:59,983] jepsen worker 65 - jepsen.util 10955 :fail :cas [21 [7 0]] :ValueNotSwapFrom INFO [2019-12-17 12:37:59,983] jepsen worker 65 - jepsen.util 10955 :invoke :write [21 46] INFO [2019-12-17 12:38:00,001] jepsen worker 58 - jepsen.util 17428 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:38:00,001] jepsen worker 81 - jepsen.util 16821 :info :cas [27 [16 18]] :RequestCanceledException INFO [2019-12-17 12:38:00,002] jepsen worker 58 - jepsen.util 17428 :invoke :write [19 26] INFO [2019-12-17 12:38:00,003] jepsen worker 81 - jepsen.util 16911 :invoke :read [27 nil] INFO [2019-12-17 12:38:00,018] jepsen worker 59 - jepsen.util 16259 :info :cas [19 [0 3]] :RequestCanceledException INFO [2019-12-17 12:38:00,020] jepsen worker 59 - jepsen.util 16349 :invoke :cas [19 [2 20]] INFO [2019-12-17 12:38:00,027] jepsen worker 65 - jepsen.util 10955 :ok :write [21 46] INFO [2019-12-17 12:38:00,028] jepsen worker 65 - jepsen.util 10955 :invoke :cas [21 [49 18]] INFO [2019-12-17 12:38:00,028] jepsen worker 69 - jepsen.util 12849 :fail :cas [23 [46 45]] :ValueNotSwapFrom INFO [2019-12-17 12:38:00,028] jepsen worker 69 - jepsen.util 12849 :invoke :cas [23 [16 11]] INFO [2019-12-17 12:38:00,034] jepsen worker 48 - jepsen.util 15888 :info :cas [16 [4 31]] :RequestCanceledException INFO [2019-12-17 12:38:00,035] jepsen worker 48 - jepsen.util 15978 :invoke :cas [16 [17 4]] INFO [2019-12-17 12:38:00,051] jepsen worker 36 - jepsen.util 13356 :ok :read [12 1] INFO [2019-12-17 12:38:00,051] jepsen worker 36 - jepsen.util 13356 :invoke :cas [12 [30 7]] INFO [2019-12-17 12:38:00,052] jepsen worker 65 - jepsen.util 10955 :fail :cas [21 [49 18]] :ValueNotSwapFrom INFO [2019-12-17 12:38:00,052] jepsen worker 65 - jepsen.util 10955 :invoke :cas [21 [34 17]] INFO [2019-12-17 12:38:00,059] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72871,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:38:00,059] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72871,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:38:00,063] jepsen worker 52 - jepsen.util 12112 :fail :cas [17 [21 13]] :ValueNotSwapFrom INFO [2019-12-17 12:38:00,063] jepsen worker 52 - jepsen.util 12112 :invoke :cas [17 [16 19]] INFO [2019-12-17 12:38:00,067] jepsen worker 19 - jepsen.util 12979 :info :write [6 29] :SyncWriteAmbiguous INFO [2019-12-17 12:38:00,068] jepsen worker 19 - jepsen.util 13069 :invoke :write [6 25] INFO [2019-12-17 12:38:00,075] jepsen worker 65 - jepsen.util 10955 :fail :cas [21 [34 17]] :ValueNotSwapFrom INFO [2019-12-17 12:38:00,075] jepsen worker 65 - jepsen.util 10955 :invoke :read [21 nil] INFO [2019-12-17 12:38:00,094] jepsen worker 74 - jepsen.util 12404 :ok :read [24 35] INFO [2019-12-17 12:38:00,095] jepsen worker 74 - jepsen.util 12404 :invoke :read [24 nil] INFO [2019-12-17 12:38:00,102] jepsen worker 65 - jepsen.util 10955 :ok :read [21 46] INFO [2019-12-17 12:38:00,102] jepsen worker 65 - jepsen.util 10955 :invoke :write [21 16] INFO [2019-12-17 12:38:00,111] jepsen worker 74 - jepsen.util 12404 :ok :read [24 35] INFO [2019-12-17 12:38:00,112] jepsen worker 74 - jepsen.util 12404 :invoke :cas [24 [16 8]] INFO [2019-12-17 12:38:00,126] jepsen worker 87 - jepsen.util 12147 :ok :read [29 2] INFO [2019-12-17 12:38:00,126] jepsen worker 87 - jepsen.util 12147 :invoke :read [29 nil] INFO [2019-12-17 12:38:00,134] jepsen worker 69 - jepsen.util 12849 :fail :cas [23 [16 11]] :ValueNotSwapFrom INFO [2019-12-17 12:38:00,134] jepsen worker 69 - jepsen.util 12849 :invoke :write [23 32] INFO [2019-12-17 12:38:00,139] jepsen worker 74 - jepsen.util 12404 :fail :cas [24 [16 8]] :ValueNotSwapFrom INFO [2019-12-17 12:38:00,139] jepsen worker 74 - jepsen.util 12404 :invoke :cas [24 [14 23]] INFO [2019-12-17 12:38:00,148] jepsen worker 46 - jepsen.util 12286 :info :write [15 28] :RequestCanceledException INFO [2019-12-17 12:38:00,150] jepsen worker 46 - jepsen.util 12376 :invoke :cas [15 [7 27]] INFO [2019-12-17 12:38:00,152] jepsen worker 47 - jepsen.util 13727 :ok :write [15 8] INFO [2019-12-17 12:38:00,152] jepsen worker 47 - jepsen.util 13727 :invoke :read [15 nil] INFO [2019-12-17 12:38:00,152] jepsen worker 51 - jepsen.util 11751 :ok :read [17 9] INFO [2019-12-17 12:38:00,152] jepsen worker 51 - jepsen.util 11751 :invoke :write [17 40] INFO [2019-12-17 12:38:00,155] jepsen worker 74 - jepsen.util 12404 :fail :cas [24 [14 23]] :ValueNotSwapFrom INFO [2019-12-17 12:38:00,155] jepsen worker 74 - jepsen.util 12404 :invoke :cas [24 [3 37]] INFO [2019-12-17 12:38:00,163] jepsen worker 42 - jepsen.util 18042 :info :write [14 40] :RequestCanceledException INFO [2019-12-17 12:38:00,164] jepsen worker 42 - jepsen.util 18132 :invoke :cas [14 [10 3]] INFO [2019-12-17 12:38:00,172] jepsen worker 88 - jepsen.util 12688 :fail :cas [29 [5 20]] :ValueNotSwapFrom INFO [2019-12-17 12:38:00,172] jepsen worker 88 - jepsen.util 12688 :invoke :read [29 nil] INFO [2019-12-17 12:38:00,175] jepsen worker 4 - jepsen.util 11794 :ok :read [1 26] INFO [2019-12-17 12:38:00,175] jepsen worker 4 - jepsen.util 11794 :invoke :read [1 nil] INFO [2019-12-17 12:38:00,196] jepsen worker 37 - jepsen.util 12007 :fail :cas [12 [22 4]] :ValueNotSwapFrom INFO [2019-12-17 12:38:00,196] jepsen worker 37 - jepsen.util 12007 :invoke :read [12 nil] INFO [2019-12-17 12:38:00,218] jepsen worker 1 - jepsen.util 17461 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:38:00,219] jepsen worker 1 - jepsen.util 17461 :invoke :write [0 47] INFO [2019-12-17 12:38:00,222] jepsen worker 32 - jepsen.util 12992 :info :write [10 7] :RequestCanceledException INFO [2019-12-17 12:38:00,223] jepsen worker 32 - jepsen.util 13082 :invoke :write [10 31] INFO [2019-12-17 12:38:00,232] jepsen worker 74 - jepsen.util 12404 :fail :cas [24 [3 37]] :ValueNotSwapFrom INFO [2019-12-17 12:38:00,232] jepsen worker 74 - jepsen.util 12404 :invoke :cas [24 [12 34]] INFO [2019-12-17 12:38:00,244] jepsen worker 36 - jepsen.util 13356 :fail :cas [12 [30 7]] :ValueNotSwapFrom INFO [2019-12-17 12:38:00,244] jepsen worker 36 - jepsen.util 13356 :invoke :write [12 37] INFO [2019-12-17 12:38:00,245] jepsen worker 46 - jepsen.util 12376 :fail :cas [15 [7 27]] :ValueNotSwapFrom INFO [2019-12-17 12:38:00,245] jepsen worker 46 - jepsen.util 12376 :invoke :write [15 48] INFO [2019-12-17 12:38:00,246] jepsen worker 78 - jepsen.util 12318 :ok :read [26 18] INFO [2019-12-17 12:38:00,246] jepsen worker 78 - jepsen.util 12318 :invoke :read [26 nil] INFO [2019-12-17 12:38:00,248] jepsen worker 39 - jepsen.util 16779 :info :write [13 48] :RequestCanceledException INFO [2019-12-17 12:38:00,248] jepsen worker 45 - jepsen.util 12375 :info :write [15 34] :RequestCanceledException INFO [2019-12-17 12:38:00,249] jepsen worker 39 - jepsen.util 16869 :invoke :cas [13 [32 22]] INFO [2019-12-17 12:38:00,249] jepsen worker 45 - jepsen.util 12465 :invoke :read [15 nil] INFO [2019-12-17 12:38:00,269] jepsen worker 74 - jepsen.util 12404 :fail :cas [24 [12 34]] :ValueNotSwapFrom INFO [2019-12-17 12:38:00,269] jepsen worker 74 - jepsen.util 12404 :invoke :read [24 nil] INFO [2019-12-17 12:38:00,271] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72907,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:38:00,271] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72900,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:38:00,271] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72907,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:38:00,271] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72900,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:38:00,271] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72911,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:38:00,271] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72902,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:38:00,271] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72911,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:38:00,271] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72902,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:38:00,277] jepsen worker 65 - jepsen.util 10955 :ok :write [21 16] INFO [2019-12-17 12:38:00,277] jepsen worker 65 - jepsen.util 10955 :invoke :write [21 2] INFO [2019-12-17 12:38:00,298] jepsen worker 4 - jepsen.util 11794 :ok :read [1 26] INFO [2019-12-17 12:38:00,299] jepsen worker 4 - jepsen.util 11794 :invoke :write [1 2] INFO [2019-12-17 12:38:00,313] jepsen worker 45 - jepsen.util 12465 :ok :read [15 8] INFO [2019-12-17 12:38:00,314] jepsen worker 45 - jepsen.util 12465 :invoke :cas [15 [1 24]] INFO [2019-12-17 12:38:00,316] jepsen worker 83 - jepsen.util 17183 :info :cas [27 [20 33]] :RequestCanceledException INFO [2019-12-17 12:38:00,318] jepsen worker 83 - jepsen.util 17273 :invoke :cas [27 [5 21]] INFO [2019-12-17 12:38:00,319] jepsen worker 37 - jepsen.util 12007 :ok :read [12 1] INFO [2019-12-17 12:38:00,319] jepsen worker 37 - jepsen.util 12007 :invoke :cas [12 [1 3]] INFO [2019-12-17 12:38:00,334] jepsen worker 22 - jepsen.util 17392 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:38:00,334] jepsen worker 22 - jepsen.util 17392 :invoke :cas [7 [21 0]] INFO [2019-12-17 12:38:00,337] jepsen worker 88 - jepsen.util 12688 :ok :read [29 2] INFO [2019-12-17 12:38:00,337] jepsen worker 88 - jepsen.util 12688 :invoke :read [29 nil] INFO [2019-12-17 12:38:00,344] jepsen worker 74 - jepsen.util 12404 :ok :read [24 35] INFO [2019-12-17 12:38:00,344] jepsen worker 74 - jepsen.util 12404 :invoke :write [24 15] INFO [2019-12-17 12:38:00,348] jepsen worker 18 - jepsen.util 13068 :info :write [6 3] :RequestCanceledException INFO [2019-12-17 12:38:00,349] jepsen worker 18 - jepsen.util 13158 :invoke :write [6 5] INFO [2019-12-17 12:38:00,359] jepsen worker 45 - jepsen.util 12465 :fail :cas [15 [1 24]] :ValueNotSwapFrom INFO [2019-12-17 12:38:00,359] jepsen worker 45 - jepsen.util 12465 :invoke :read [15 nil] INFO [2019-12-17 12:38:00,363] jepsen worker 52 - jepsen.util 12112 :fail :cas [17 [16 19]] :ValueNotSwapFrom INFO [2019-12-17 12:38:00,363] jepsen worker 52 - jepsen.util 12112 :invoke :write [17 11] INFO [2019-12-17 12:38:00,373] jepsen worker 65 - jepsen.util 10955 :ok :write [21 2] INFO [2019-12-17 12:38:00,373] jepsen worker 65 - jepsen.util 10955 :invoke :write [21 13] INFO [2019-12-17 12:38:00,401] jepsen worker 30 - jepsen.util 11820 :info :write [10 16] :RequestCanceledException INFO [2019-12-17 12:38:00,402] jepsen worker 29 - jepsen.util 11009 :fail :read [9 nil] :RequestTimeoutException INFO [2019-12-17 12:38:00,402] jepsen worker 29 - jepsen.util 11009 :invoke :read [9 nil] INFO [2019-12-17 12:38:00,403] jepsen worker 30 - jepsen.util 11910 :invoke :read [10 nil] INFO [2019-12-17 12:38:00,403] jepsen worker 13 - jepsen.util 12073 :fail :cas [4 [36 20]] :ValueNotSwapFrom INFO [2019-12-17 12:38:00,403] jepsen worker 13 - jepsen.util 12073 :invoke :write [4 5] INFO [2019-12-17 12:38:00,411] jepsen worker 87 - jepsen.util 12147 :ok :read [29 2] INFO [2019-12-17 12:38:00,412] jepsen worker 87 - jepsen.util 12147 :invoke :read [29 nil] INFO [2019-12-17 12:38:00,434] jepsen worker 76 - jepsen.util 17266 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:38:00,434] jepsen worker 76 - jepsen.util 17266 :invoke :write [25 34] INFO [2019-12-17 12:38:00,438] jepsen worker 65 - jepsen.util 10955 :ok :write [21 13] INFO [2019-12-17 12:38:00,438] jepsen worker 65 - jepsen.util 10955 :invoke :cas [21 [6 26]] INFO [2019-12-17 12:38:00,448] jepsen worker 11 - jepsen.util 12251 :info :write [3 12] :RequestCanceledException INFO [2019-12-17 12:38:00,450] jepsen worker 70 - jepsen.util 13120 :ok :read [23 1] INFO [2019-12-17 12:38:00,450] jepsen worker 11 - jepsen.util 12341 :invoke :read [3 nil] INFO [2019-12-17 12:38:00,450] jepsen worker 70 - jepsen.util 13120 :invoke :read [23 nil] INFO [2019-12-17 12:38:00,465] jepsen worker 6 - jepsen.util 17556 :info :write [2 31] :RequestCanceledException INFO [2019-12-17 12:38:00,466] jepsen worker 6 - jepsen.util 17646 :invoke :cas [2 [4 23]] INFO [2019-12-17 12:38:00,471] jepsen worker 11 - jepsen.util 12341 :ok :read [3 48] INFO [2019-12-17 12:38:00,472] jepsen worker 11 - jepsen.util 12341 :invoke :cas [3 [33 22]] INFO [2019-12-17 12:38:00,479] jepsen worker 29 - jepsen.util 11009 :ok :read [9 34] INFO [2019-12-17 12:38:00,480] jepsen worker 29 - jepsen.util 11009 :invoke :cas [9 [49 21]] INFO [2019-12-17 12:38:00,490] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72938,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:38:00,490] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72938,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:38:00,490] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72942,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:38:00,490] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72942,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:38:00,495] jepsen worker 80 - jepsen.util 12050 :ok :read [26 18] INFO [2019-12-17 12:38:00,495] jepsen worker 80 - jepsen.util 12050 :invoke :read [26 nil] INFO [2019-12-17 12:38:00,501] jepsen worker 20 - jepsen.util 12440 :info :write [6 36] :RequestCanceledException INFO [2019-12-17 12:38:00,501] jepsen worker 68 - jepsen.util 16448 :info :write [22 5] :RequestCanceledException INFO [2019-12-17 12:38:00,502] jepsen worker 68 - jepsen.util 16538 :invoke :read [22 nil] INFO [2019-12-17 12:38:00,503] jepsen worker 78 - jepsen.util 12318 :ok :read [26 18] INFO [2019-12-17 12:38:00,503] jepsen worker 20 - jepsen.util 12530 :invoke :write [6 27] INFO [2019-12-17 12:38:00,503] jepsen worker 78 - jepsen.util 12318 :invoke :cas [26 [33 29]] INFO [2019-12-17 12:38:00,516] jepsen worker 11 - jepsen.util 12341 :fail :cas [3 [33 22]] :ValueNotSwapFrom INFO [2019-12-17 12:38:00,516] jepsen worker 11 - jepsen.util 12341 :invoke :cas [3 [24 35]] INFO [2019-12-17 12:38:00,520] jepsen worker 65 - jepsen.util 10955 :fail :cas [21 [6 26]] :ValueNotSwapFrom INFO [2019-12-17 12:38:00,520] jepsen worker 65 - jepsen.util 10955 :invoke :read [21 nil] INFO [2019-12-17 12:38:00,531] jepsen worker 29 - jepsen.util 11009 :fail :cas [9 [49 21]] :ValueNotSwapFrom INFO [2019-12-17 12:38:00,531] jepsen worker 29 - jepsen.util 11009 :invoke :cas [9 [21 27]] INFO [2019-12-17 12:38:00,534] jepsen worker 49 - jepsen.util 17239 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:38:00,535] jepsen worker 49 - jepsen.util 17239 :invoke :cas [16 [31 45]] INFO [2019-12-17 12:38:00,546] jepsen worker 24 - jepsen.util 16494 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:38:00,546] jepsen worker 24 - jepsen.util 16494 :invoke :read [8 nil] INFO [2019-12-17 12:38:00,555] jepsen worker 11 - jepsen.util 12341 :fail :cas [3 [24 35]] :ValueNotSwapFrom INFO [2019-12-17 12:38:00,555] jepsen worker 11 - jepsen.util 12341 :invoke :cas [3 [12 28]] INFO [2019-12-17 12:38:00,556] jepsen worker 65 - jepsen.util 10955 :ok :read [21 13] INFO [2019-12-17 12:38:00,556] jepsen worker 65 - jepsen.util 10955 :invoke :cas [21 [2 9]] INFO [2019-12-17 12:38:00,566] jepsen worker 40 - jepsen.util 18130 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:38:00,566] jepsen worker 40 - jepsen.util 18130 :invoke :read [13 nil] INFO [2019-12-17 12:38:00,573] jepsen worker 89 - jepsen.util 12419 :fail :cas [29 [4 27]] :ValueNotSwapFrom INFO [2019-12-17 12:38:00,573] jepsen worker 89 - jepsen.util 12419 :invoke :write [29 32] INFO [2019-12-17 12:38:00,581] jepsen worker 29 - jepsen.util 11009 :fail :cas [9 [21 27]] :ValueNotSwapFrom INFO [2019-12-17 12:38:00,581] jepsen worker 29 - jepsen.util 11009 :invoke :read [9 nil] INFO [2019-12-17 12:38:00,587] jepsen worker 11 - jepsen.util 12341 :fail :cas [3 [12 28]] :ValueNotSwapFrom INFO [2019-12-17 12:38:00,587] jepsen worker 11 - jepsen.util 12341 :invoke :cas [3 [43 33]] INFO [2019-12-17 12:38:00,601] jepsen worker 65 - jepsen.util 10955 :fail :cas [21 [2 9]] :ValueNotSwapFrom INFO [2019-12-17 12:38:00,601] jepsen worker 65 - jepsen.util 10955 :invoke :read [21 nil] INFO [2019-12-17 12:38:00,615] jepsen worker 11 - jepsen.util 12341 :fail :cas [3 [43 33]] :ValueNotSwapFrom INFO [2019-12-17 12:38:00,615] jepsen worker 11 - jepsen.util 12341 :invoke :write [3 31] INFO [2019-12-17 12:38:00,617] jepsen worker 34 - jepsen.util 18124 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:38:00,617] jepsen worker 34 - jepsen.util 18124 :invoke :write [11 10] INFO [2019-12-17 12:38:00,620] jepsen worker 29 - jepsen.util 11009 :ok :read [9 34] INFO [2019-12-17 12:38:00,620] jepsen worker 29 - jepsen.util 11009 :invoke :cas [9 [8 48]] INFO [2019-12-17 12:38:00,634] jepsen worker 41 - jepsen.util 18671 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:38:00,634] jepsen worker 41 - jepsen.util 18671 :invoke :cas [13 [25 40]] INFO [2019-12-17 12:38:00,640] jepsen worker 65 - jepsen.util 10955 :ok :read [21 13] INFO [2019-12-17 12:38:00,641] jepsen worker 65 - jepsen.util 10955 :invoke :read [21 nil] INFO [2019-12-17 12:38:00,671] jepsen worker 29 - jepsen.util 11009 :fail :cas [9 [8 48]] :ValueNotSwapFrom INFO [2019-12-17 12:38:00,671] jepsen worker 29 - jepsen.util 11009 :invoke :cas [9 [17 43]] INFO [2019-12-17 12:38:00,681] jepsen worker 65 - jepsen.util 10955 :ok :read [21 13] INFO [2019-12-17 12:38:00,681] jepsen worker 65 - jepsen.util 10955 :invoke :cas [21 [29 24]] INFO [2019-12-17 12:38:00,696] jepsen worker 29 - jepsen.util 11009 :fail :cas [9 [17 43]] :ValueNotSwapFrom INFO [2019-12-17 12:38:00,697] jepsen worker 29 - jepsen.util 11009 :invoke :write [9 23] INFO [2019-12-17 12:38:00,704] jepsen worker 65 - jepsen.util 10955 :fail :cas [21 [29 24]] :ValueNotSwapFrom INFO [2019-12-17 12:38:00,704] jepsen worker 65 - jepsen.util 10955 :invoke :write [21 33] INFO [2019-12-17 12:38:00,708] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72972,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:38:00,708] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72972,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:38:00,708] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72975,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:38:00,708] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72975,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:38:00,708] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72976,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:38:00,709] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72976,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:38:00,735] jepsen worker 74 - jepsen.util 12404 :ok :write [24 15] INFO [2019-12-17 12:38:00,736] jepsen worker 74 - jepsen.util 12404 :invoke :write [24 44] INFO [2019-12-17 12:38:00,749] jepsen worker 29 - jepsen.util 11009 :ok :write [9 23] INFO [2019-12-17 12:38:00,749] jepsen worker 29 - jepsen.util 11009 :invoke :write [9 49] INFO [2019-12-17 12:38:00,769] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:38:00,770] jepsen nemesis - couchbase.util Rebalance check count: 575 INFO [2019-12-17 12:38:00,770] jepsen nemesis - couchbase.util Rebalance stuck count: 556 INFO [2019-12-17 12:38:00,785] jepsen worker 46 - jepsen.util 12376 :ok :write [15 48] INFO [2019-12-17 12:38:00,785] jepsen worker 46 - jepsen.util 12376 :invoke :write [15 44] INFO [2019-12-17 12:38:00,831] jepsen worker 29 - jepsen.util 11009 :ok :write [9 49] INFO [2019-12-17 12:38:00,831] jepsen worker 29 - jepsen.util 11009 :invoke :cas [9 [37 2]] INFO [2019-12-17 12:38:00,845] jepsen worker 29 - jepsen.util 11009 :fail :cas [9 [37 2]] :ValueNotSwapFrom INFO [2019-12-17 12:38:00,845] jepsen worker 29 - jepsen.util 11009 :invoke :read [9 nil] INFO [2019-12-17 12:38:00,848] jepsen worker 17 - jepsen.util 17387 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:38:00,849] jepsen worker 17 - jepsen.util 17387 :invoke :read [5 nil] INFO [2019-12-17 12:38:00,859] jepsen worker 70 - jepsen.util 13120 :ok :read [23 1] INFO [2019-12-17 12:38:00,860] jepsen worker 70 - jepsen.util 13120 :invoke :read [23 nil] INFO [2019-12-17 12:38:00,862] jepsen worker 79 - jepsen.util 12769 :info :write [26 46] :RequestCanceledException INFO [2019-12-17 12:38:00,863] jepsen worker 29 - jepsen.util 11009 :ok :read [9 49] INFO [2019-12-17 12:38:00,863] jepsen worker 29 - jepsen.util 11009 :invoke :write [9 2] INFO [2019-12-17 12:38:00,863] jepsen worker 79 - jepsen.util 12859 :invoke :read [26 nil] INFO [2019-12-17 12:38:00,918] jepsen worker 62 - jepsen.util 17882 :info :write [20 1] :RequestCanceledException INFO [2019-12-17 12:38:00,919] jepsen worker 29 - jepsen.util 11009 :ok :write [9 2] INFO [2019-12-17 12:38:00,919] jepsen worker 29 - jepsen.util 11009 :invoke :write [9 40] INFO [2019-12-17 12:38:00,919] jepsen worker 62 - jepsen.util 17972 :invoke :cas [20 [46 5]] INFO [2019-12-17 12:38:00,928] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72991,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:38:00,928] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72991,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:38:00,928] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72994,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:38:00,928] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":72994,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:38:00,934] jepsen worker 28 - jepsen.util 10828 :fail :read [9 nil] :RequestTimeoutException INFO [2019-12-17 12:38:00,934] jepsen worker 60 - jepsen.util 18150 :info :cas [20 [32 33]] :RequestCanceledException INFO [2019-12-17 12:38:00,934] jepsen worker 86 - jepsen.util 17906 :info :cas [28 [8 44]] :RequestCanceledException INFO [2019-12-17 12:38:00,934] jepsen worker 28 - jepsen.util 10828 :invoke :read [9 nil] INFO [2019-12-17 12:38:00,935] jepsen worker 60 - jepsen.util 18240 :invoke :cas [20 [5 27]] INFO [2019-12-17 12:38:00,938] jepsen worker 86 - jepsen.util 17996 :invoke :cas [28 [9 6]] INFO [2019-12-17 12:38:00,966] jepsen worker 29 - jepsen.util 11009 :ok :write [9 40] INFO [2019-12-17 12:38:00,966] jepsen worker 29 - jepsen.util 11009 :invoke :write [9 43] INFO [2019-12-17 12:38:00,979] jepsen worker 80 - jepsen.util 12050 :ok :read [26 18] INFO [2019-12-17 12:38:00,979] jepsen worker 80 - jepsen.util 12050 :invoke :read [26 nil] INFO [2019-12-17 12:38:01,000] jepsen worker 75 - jepsen.util 16725 :info :write [25 14] :RequestCanceledException INFO [2019-12-17 12:38:01,001] jepsen worker 75 - jepsen.util 16815 :invoke :read [25 nil] INFO [2019-12-17 12:38:01,018] jepsen worker 26 - jepsen.util 17936 :info :write [8 47] :RequestCanceledException INFO [2019-12-17 12:38:01,020] jepsen worker 26 - jepsen.util 18026 :invoke :cas [8 [38 30]] INFO [2019-12-17 12:38:01,022] jepsen worker 29 - jepsen.util 11009 :ok :write [9 43] INFO [2019-12-17 12:38:01,022] jepsen worker 29 - jepsen.util 11009 :invoke :read [9 nil] INFO [2019-12-17 12:38:01,034] jepsen worker 27 - jepsen.util 11727 :fail :read [9 nil] :RequestTimeoutException INFO [2019-12-17 12:38:01,035] jepsen worker 27 - jepsen.util 11727 :invoke :write [9 25] INFO [2019-12-17 12:38:01,035] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73005,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:38:01,035] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73005,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:38:01,035] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73009,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:38:01,036] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73009,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:38:01,042] jepsen worker 29 - jepsen.util 11009 :ok :read [9 43] INFO [2019-12-17 12:38:01,042] jepsen worker 29 - jepsen.util 11009 :invoke :write [9 28] INFO [2019-12-17 12:38:01,066] jepsen worker 87 - jepsen.util 12147 :ok :read [29 2] INFO [2019-12-17 12:38:01,067] jepsen worker 87 - jepsen.util 12147 :invoke :cas [29 [38 29]] INFO [2019-12-17 12:38:01,088] jepsen worker 29 - jepsen.util 11009 :ok :write [9 28] INFO [2019-12-17 12:38:01,088] jepsen worker 29 - jepsen.util 11009 :invoke :cas [9 [31 31]] INFO [2019-12-17 12:38:01,104] jepsen worker 29 - jepsen.util 11009 :fail :cas [9 [31 31]] :ValueNotSwapFrom INFO [2019-12-17 12:38:01,104] jepsen worker 29 - jepsen.util 11009 :invoke :cas [9 [18 3]] INFO [2019-12-17 12:38:01,115] jepsen worker 78 - jepsen.util 12318 :fail :cas [26 [33 29]] :ValueNotSwapFrom INFO [2019-12-17 12:38:01,115] jepsen worker 78 - jepsen.util 12318 :invoke :write [26 16] INFO [2019-12-17 12:38:01,118] jepsen worker 29 - jepsen.util 11009 :fail :cas [9 [18 3]] :ValueNotSwapFrom INFO [2019-12-17 12:38:01,118] jepsen worker 29 - jepsen.util 11009 :invoke :read [9 nil] INFO [2019-12-17 12:38:01,130] jepsen worker 29 - jepsen.util 11009 :ok :read [9 28] INFO [2019-12-17 12:38:01,130] jepsen worker 29 - jepsen.util 11009 :invoke :read [9 nil] INFO [2019-12-17 12:38:01,134] jepsen worker 66 - jepsen.util 17346 :info :cas [22 [35 2]] :RequestCanceledException INFO [2019-12-17 12:38:01,134] jepsen worker 56 - jepsen.util 13286 :info :write [18 26] :RequestCanceledException INFO [2019-12-17 12:38:01,135] jepsen worker 56 - jepsen.util 13376 :invoke :read [18 nil] INFO [2019-12-17 12:38:01,136] jepsen worker 66 - jepsen.util 17436 :invoke :cas [22 [36 4]] INFO [2019-12-17 12:38:01,140] jepsen worker 46 - jepsen.util 12376 :ok :write [15 44] INFO [2019-12-17 12:38:01,141] jepsen worker 46 - jepsen.util 12376 :invoke :cas [15 [43 30]] INFO [2019-12-17 12:38:01,141] jepsen worker 11 - jepsen.util 12341 :ok :write [3 31] INFO [2019-12-17 12:38:01,141] jepsen worker 74 - jepsen.util 12404 :ok :write [24 44] INFO [2019-12-17 12:38:01,141] jepsen worker 11 - jepsen.util 12341 :invoke :write [3 5] INFO [2019-12-17 12:38:01,141] jepsen worker 74 - jepsen.util 12404 :invoke :cas [24 [10 29]] INFO [2019-12-17 12:38:01,141] jepsen worker 29 - jepsen.util 11009 :ok :read [9 28] INFO [2019-12-17 12:38:01,141] jepsen worker 29 - jepsen.util 11009 :invoke :cas [9 [17 14]] INFO [2019-12-17 12:38:01,146] jepsen worker 85 - jepsen.util 17815 :info :write [28 40] :RequestCanceledException INFO [2019-12-17 12:38:01,148] jepsen worker 85 - jepsen.util 17905 :invoke :write [28 23] INFO [2019-12-17 12:38:01,196] jepsen worker 74 - jepsen.util 12404 :fail :cas [24 [10 29]] :ValueNotSwapFrom INFO [2019-12-17 12:38:01,196] jepsen worker 74 - jepsen.util 12404 :invoke :cas [24 [2 15]] INFO [2019-12-17 12:38:01,199] jepsen worker 46 - jepsen.util 12376 :fail :cas [15 [43 30]] :ValueNotSwapFrom INFO [2019-12-17 12:38:01,199] jepsen worker 46 - jepsen.util 12376 :invoke :read [15 nil] INFO [2019-12-17 12:38:01,210] jepsen worker 29 - jepsen.util 11009 :fail :cas [9 [17 14]] :ValueNotSwapFrom INFO [2019-12-17 12:38:01,210] jepsen worker 29 - jepsen.util 11009 :invoke :cas [9 [17 22]] INFO [2019-12-17 12:38:01,232] jepsen worker 67 - jepsen.util 17437 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:38:01,232] jepsen worker 67 - jepsen.util 17437 :invoke :write [22 27] INFO [2019-12-17 12:38:01,248] jepsen worker 74 - jepsen.util 12404 :fail :cas [24 [2 15]] :ValueNotSwapFrom INFO [2019-12-17 12:38:01,248] jepsen worker 74 - jepsen.util 12404 :invoke :cas [24 [46 18]] INFO [2019-12-17 12:38:01,250] jepsen worker 46 - jepsen.util 12376 :ok :read [15 44] INFO [2019-12-17 12:38:01,251] jepsen worker 46 - jepsen.util 12376 :invoke :read [15 nil] INFO [2019-12-17 12:38:01,251] jepsen worker 29 - jepsen.util 11009 :fail :cas [9 [17 22]] :ValueNotSwapFrom INFO [2019-12-17 12:38:01,251] jepsen worker 29 - jepsen.util 11009 :invoke :write [9 19] INFO [2019-12-17 12:38:01,251] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73029,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:38:01,251] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73037,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:38:01,251] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73029,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:38:01,251] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73037,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:38:01,251] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73030,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:38:01,251] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73040,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:38:01,251] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73030,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:38:01,251] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73040,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:38:01,251] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73031,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:38:01,252] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73031,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:38:01,265] jepsen worker 3 - jepsen.util 11973 :info :write [1 43] :RequestCanceledException INFO [2019-12-17 12:38:01,265] jepsen worker 84 - jepsen.util 16734 :info :write [28 21] :RequestCanceledException INFO [2019-12-17 12:38:01,266] jepsen worker 3 - jepsen.util 12063 :invoke :write [1 46] INFO [2019-12-17 12:38:01,266] jepsen worker 84 - jepsen.util 16824 :invoke :read [28 nil] INFO [2019-12-17 12:38:01,291] jepsen worker 88 - jepsen.util 12688 :ok :read [29 2] INFO [2019-12-17 12:38:01,292] jepsen worker 88 - jepsen.util 12688 :invoke :write [29 42] INFO [2019-12-17 12:38:01,301] jepsen worker 33 - jepsen.util 17043 :info :cas [11 [17 29]] :RequestCanceledException INFO [2019-12-17 12:38:01,302] jepsen worker 33 - jepsen.util 17133 :invoke :write [11 39] INFO [2019-12-17 12:38:01,316] jepsen worker 82 - jepsen.util 17632 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:38:01,316] jepsen worker 12 - jepsen.util 12612 :info :cas [4 [0 48]] :RequestCanceledException INFO [2019-12-17 12:38:01,316] jepsen worker 82 - jepsen.util 17632 :invoke :read [27 nil] INFO [2019-12-17 12:38:01,317] jepsen worker 12 - jepsen.util 12702 :invoke :cas [4 [17 35]] INFO [2019-12-17 12:38:01,319] jepsen worker 74 - jepsen.util 12404 :fail :cas [24 [46 18]] :ValueNotSwapFrom INFO [2019-12-17 12:38:01,319] jepsen worker 74 - jepsen.util 12404 :invoke :write [24 2] INFO [2019-12-17 12:38:01,320] jepsen worker 46 - jepsen.util 12376 :ok :read [15 44] INFO [2019-12-17 12:38:01,320] jepsen worker 46 - jepsen.util 12376 :invoke :cas [15 [46 10]] INFO [2019-12-17 12:38:01,334] jepsen worker 23 - jepsen.util 17663 :info :write [7 2] :RequestCanceledException INFO [2019-12-17 12:38:01,335] jepsen worker 23 - jepsen.util 17753 :invoke :cas [7 [10 23]] INFO [2019-12-17 12:38:01,362] jepsen worker 21 - jepsen.util 16671 :info :cas [7 [30 36]] :RequestCanceledException INFO [2019-12-17 12:38:01,362] jepsen worker 0 - jepsen.util 17640 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:38:01,362] jepsen worker 0 - jepsen.util 17640 :invoke :write [0 30] INFO [2019-12-17 12:38:01,363] jepsen worker 21 - jepsen.util 16761 :invoke :cas [7 [22 35]] INFO [2019-12-17 12:38:01,372] jepsen worker 5 - jepsen.util 12245 :ok :write [1 21] INFO [2019-12-17 12:38:01,372] jepsen worker 5 - jepsen.util 12245 :invoke :read [1 nil] INFO [2019-12-17 12:38:01,393] jepsen worker 46 - jepsen.util 12376 :fail :cas [15 [46 10]] :ValueNotSwapFrom INFO [2019-12-17 12:38:01,393] jepsen worker 46 - jepsen.util 12376 :invoke :read [15 nil] INFO [2019-12-17 12:38:01,399] jepsen worker 14 - jepsen.util 12254 :ok :write [4 39] INFO [2019-12-17 12:38:01,399] jepsen worker 14 - jepsen.util 12254 :invoke :write [4 4] INFO [2019-12-17 12:38:01,425] jepsen worker 12 - jepsen.util 12702 :fail :cas [4 [17 35]] :ValueNotSwapFrom INFO [2019-12-17 12:38:01,425] jepsen worker 12 - jepsen.util 12702 :invoke :cas [4 [16 9]] INFO [2019-12-17 12:38:01,427] jepsen worker 46 - jepsen.util 12376 :ok :read [15 44] INFO [2019-12-17 12:38:01,427] jepsen worker 46 - jepsen.util 12376 :invoke :cas [15 [39 2]] INFO [2019-12-17 12:38:01,434] jepsen worker 72 - jepsen.util 12672 :info :write [24 34] :RequestCanceledException INFO [2019-12-17 12:38:01,434] jepsen worker 9 - jepsen.util 13239 :fail :read [3 nil] :RequestTimeoutException INFO [2019-12-17 12:38:01,434] jepsen worker 10 - jepsen.util 12430 :fail :read [3 nil] :RequestTimeoutException INFO [2019-12-17 12:38:01,434] jepsen worker 9 - jepsen.util 13239 :invoke :write [3 35] INFO [2019-12-17 12:38:01,434] jepsen worker 10 - jepsen.util 12430 :invoke :read [3 nil] INFO [2019-12-17 12:38:01,435] jepsen worker 72 - jepsen.util 12762 :invoke :cas [24 [45 9]] INFO [2019-12-17 12:38:01,452] jepsen worker 46 - jepsen.util 12376 :fail :cas [15 [39 2]] :ValueNotSwapFrom INFO [2019-12-17 12:38:01,452] jepsen worker 46 - jepsen.util 12376 :invoke :read [15 nil] INFO [2019-12-17 12:38:01,461] jepsen worker 29 - jepsen.util 11009 :ok :write [9 19] INFO [2019-12-17 12:38:01,462] jepsen worker 79 - jepsen.util 12859 :ok :read [26 18] INFO [2019-12-17 12:38:01,462] jepsen worker 79 - jepsen.util 12859 :invoke :read [26 nil] INFO [2019-12-17 12:38:01,462] jepsen worker 29 - jepsen.util 11009 :invoke :write [9 1] INFO [2019-12-17 12:38:01,462] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73065,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:38:01,462] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73065,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:38:01,462] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73068,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:38:01,462] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73068,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:38:01,462] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73070,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:38:01,462] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73070,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:38:01,473] jepsen worker 80 - jepsen.util 12050 :ok :read [26 18] INFO [2019-12-17 12:38:01,473] jepsen worker 80 - jepsen.util 12050 :invoke :cas [26 [47 24]] INFO [2019-12-17 12:38:01,491] jepsen worker 56 - jepsen.util 13376 :ok :read [18 13] INFO [2019-12-17 12:38:01,492] jepsen worker 56 - jepsen.util 13376 :invoke :cas [18 [17 18]] INFO [2019-12-17 12:38:01,492] jepsen worker 46 - jepsen.util 12376 :ok :read [15 44] INFO [2019-12-17 12:38:01,492] jepsen worker 46 - jepsen.util 12376 :invoke :cas [15 [45 40]] INFO [2019-12-17 12:38:01,515] jepsen worker 46 - jepsen.util 12376 :fail :cas [15 [45 40]] :ValueNotSwapFrom INFO [2019-12-17 12:38:01,516] jepsen worker 46 - jepsen.util 12376 :invoke :cas [15 [37 29]] INFO [2019-12-17 12:38:01,516] jepsen worker 72 - jepsen.util 12762 :fail :cas [24 [45 9]] :ValueNotSwapFrom INFO [2019-12-17 12:38:01,516] jepsen worker 72 - jepsen.util 12762 :invoke :read [24 nil] INFO [2019-12-17 12:38:01,528] jepsen worker 46 - jepsen.util 12376 :fail :cas [15 [37 29]] :ValueNotSwapFrom INFO [2019-12-17 12:38:01,528] jepsen worker 46 - jepsen.util 12376 :invoke :read [15 nil] INFO [2019-12-17 12:38:01,534] jepsen worker 53 - jepsen.util 12023 :info :write [17 4] :RequestCanceledException INFO [2019-12-17 12:38:01,535] jepsen worker 53 - jepsen.util 12113 :invoke :cas [17 [39 48]] INFO [2019-12-17 12:38:01,541] jepsen worker 46 - jepsen.util 12376 :ok :read [15 44] INFO [2019-12-17 12:38:01,541] jepsen worker 46 - jepsen.util 12376 :invoke :read [15 nil] INFO [2019-12-17 12:38:01,549] jepsen worker 54 - jepsen.util 13104 :info :write [18 36] :RequestCanceledException INFO [2019-12-17 12:38:01,550] jepsen worker 54 - jepsen.util 13194 :invoke :read [18 nil] INFO [2019-12-17 12:38:01,562] jepsen worker 29 - jepsen.util 11009 :ok :write [9 1] INFO [2019-12-17 12:38:01,562] jepsen worker 29 - jepsen.util 11009 :invoke :read [9 nil] INFO [2019-12-17 12:38:01,563] jepsen worker 46 - jepsen.util 12376 :ok :read [15 44] INFO [2019-12-17 12:38:01,563] jepsen worker 46 - jepsen.util 12376 :invoke :read [15 nil] INFO [2019-12-17 12:38:01,577] jepsen worker 12 - jepsen.util 12702 :fail :cas [4 [16 9]] :ValueNotSwapFrom INFO [2019-12-17 12:38:01,578] jepsen worker 12 - jepsen.util 12702 :invoke :cas [4 [15 39]] INFO [2019-12-17 12:38:01,601] jepsen worker 5 - jepsen.util 12245 :ok :read [1 21] INFO [2019-12-17 12:38:01,601] jepsen worker 5 - jepsen.util 12245 :invoke :cas [1 [7 26]] INFO [2019-12-17 12:38:01,601] jepsen worker 55 - jepsen.util 13195 :info :write [18 18] :RequestCanceledException INFO [2019-12-17 12:38:01,602] jepsen worker 55 - jepsen.util 13285 :invoke :read [18 nil] INFO [2019-12-17 12:38:01,616] jepsen worker 72 - jepsen.util 12762 :ok :read [24 44] INFO [2019-12-17 12:38:01,616] jepsen worker 72 - jepsen.util 12762 :invoke :read [24 nil] INFO [2019-12-17 12:38:01,621] jepsen worker 46 - jepsen.util 12376 :ok :read [15 44] INFO [2019-12-17 12:38:01,622] jepsen worker 46 - jepsen.util 12376 :invoke :write [15 38] INFO [2019-12-17 12:38:01,628] jepsen worker 29 - jepsen.util 11009 :ok :read [9 1] INFO [2019-12-17 12:38:01,628] jepsen worker 29 - jepsen.util 11009 :invoke :read [9 nil] INFO [2019-12-17 12:38:01,651] jepsen worker 72 - jepsen.util 12762 :ok :read [24 44] INFO [2019-12-17 12:38:01,651] jepsen worker 72 - jepsen.util 12762 :invoke :write [24 14] INFO [2019-12-17 12:38:01,652] jepsen worker 11 - jepsen.util 12341 :ok :write [3 5] INFO [2019-12-17 12:38:01,652] jepsen worker 11 - jepsen.util 12341 :invoke :write [3 22] INFO [2019-12-17 12:38:01,677] jepsen worker 87 - jepsen.util 12147 :fail :cas [29 [38 29]] :ValueNotSwapFrom INFO [2019-12-17 12:38:01,678] jepsen worker 87 - jepsen.util 12147 :invoke :cas [29 [48 9]] INFO [2019-12-17 12:38:01,681] jepsen worker 14 - jepsen.util 12254 :ok :write [4 4] INFO [2019-12-17 12:38:01,681] jepsen worker 14 - jepsen.util 12254 :invoke :cas [4 [34 11]] INFO [2019-12-17 12:38:01,681] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73093,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:38:01,681] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73093,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:38:01,681] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73096,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:38:01,681] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73096,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:38:01,681] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73097,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:38:01,681] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73097,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:38:01,688] jepsen worker 29 - jepsen.util 11009 :ok :read [9 1] INFO [2019-12-17 12:38:01,688] jepsen worker 29 - jepsen.util 11009 :invoke :write [9 25] INFO [2019-12-17 12:38:01,695] jepsen worker 10 - jepsen.util 12430 :ok :read [3 5] INFO [2019-12-17 12:38:01,695] jepsen worker 10 - jepsen.util 12430 :invoke :read [3 nil] INFO [2019-12-17 12:38:01,700] jepsen worker 73 - jepsen.util 11773 :fail :read [24 nil] :RequestTimeoutException INFO [2019-12-17 12:38:01,701] jepsen worker 73 - jepsen.util 11773 :invoke :write [24 30] INFO [2019-12-17 12:38:01,727] jepsen worker 74 - jepsen.util 12404 :ok :write [24 2] INFO [2019-12-17 12:38:01,727] jepsen worker 74 - jepsen.util 12404 :invoke :read [24 nil] INFO [2019-12-17 12:38:01,730] jepsen worker 31 - jepsen.util 10921 :info :write [10 36] :SyncWriteAmbiguous INFO [2019-12-17 12:38:01,731] jepsen worker 31 - jepsen.util 11011 :invoke :cas [10 [42 18]] INFO [2019-12-17 12:38:01,734] jepsen worker 63 - jepsen.util 11133 :info :cas [21 [39 44]] :RequestCanceledException INFO [2019-12-17 12:38:01,736] jepsen worker 63 - jepsen.util 11223 :invoke :cas [21 [9 9]] INFO [2019-12-17 12:38:01,744] jepsen worker 30 - jepsen.util 11910 :ok :read [10 40] INFO [2019-12-17 12:38:01,744] jepsen worker 30 - jepsen.util 11910 :invoke :write [10 22] INFO [2019-12-17 12:38:01,753] jepsen worker 79 - jepsen.util 12859 :ok :read [26 18] INFO [2019-12-17 12:38:01,753] jepsen worker 79 - jepsen.util 12859 :invoke :cas [26 [11 6]] INFO [2019-12-17 12:38:01,776] jepsen worker 31 - jepsen.util 11011 :fail :cas [10 [42 18]] :ValueNotSwapFrom INFO [2019-12-17 12:38:01,776] jepsen worker 31 - jepsen.util 11011 :invoke :write [10 31] INFO [2019-12-17 12:38:01,785] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:38:01,785] jepsen nemesis - couchbase.util Rebalance check count: 576 INFO [2019-12-17 12:38:01,785] jepsen nemesis - couchbase.util Rebalance stuck count: 557 INFO [2019-12-17 12:38:01,788] jepsen worker 80 - jepsen.util 12050 :fail :cas [26 [47 24]] :ValueNotSwapFrom INFO [2019-12-17 12:38:01,788] jepsen worker 80 - jepsen.util 12050 :invoke :write [26 5] INFO [2019-12-17 12:38:01,789] jepsen worker 74 - jepsen.util 12404 :ok :read [24 2] INFO [2019-12-17 12:38:01,789] jepsen worker 74 - jepsen.util 12404 :invoke :write [24 37] INFO [2019-12-17 12:38:01,823] jepsen worker 56 - jepsen.util 13376 :fail :cas [18 [17 18]] :ValueNotSwapFrom INFO [2019-12-17 12:38:01,823] jepsen worker 56 - jepsen.util 13376 :invoke :read [18 nil] INFO [2019-12-17 12:38:01,831] jepsen worker 9 - jepsen.util 13239 :ok :write [3 35] INFO [2019-12-17 12:38:01,831] jepsen worker 9 - jepsen.util 13239 :invoke :read [3 nil] INFO [2019-12-17 12:38:01,834] jepsen worker 71 - jepsen.util 13391 :info :write [23 47] :RequestCanceledException INFO [2019-12-17 12:38:01,835] jepsen worker 71 - jepsen.util 13481 :invoke :write [23 36] INFO [2019-12-17 12:38:01,849] jepsen worker 10 - jepsen.util 12430 :ok :read [3 35] INFO [2019-12-17 12:38:01,849] jepsen worker 10 - jepsen.util 12430 :invoke :read [3 nil] INFO [2019-12-17 12:38:01,881] jepsen worker 63 - jepsen.util 11223 :fail :cas [21 [9 9]] :ValueNotSwapFrom INFO [2019-12-17 12:38:01,881] jepsen worker 63 - jepsen.util 11223 :invoke :read [21 nil] INFO [2019-12-17 12:38:01,900] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73117,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:38:01,900] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73117,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:38:01,900] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73121,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:38:01,900] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73121,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:38:01,900] jepsen worker 64 - jepsen.util 9784 :info :cas [21 [30 32]] :RequestCanceledException INFO [2019-12-17 12:38:01,900] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73122,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:38:01,900] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73122,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:38:01,902] jepsen worker 64 - jepsen.util 9874 :invoke :write [21 43] INFO [2019-12-17 12:38:01,904] jepsen worker 9 - jepsen.util 13239 :ok :read [3 35] INFO [2019-12-17 12:38:01,904] jepsen worker 9 - jepsen.util 13239 :invoke :cas [3 [27 15]] INFO [2019-12-17 12:38:01,906] jepsen worker 12 - jepsen.util 12702 :fail :cas [4 [15 39]] :ValueNotSwapFrom INFO [2019-12-17 12:38:01,906] jepsen worker 12 - jepsen.util 12702 :invoke :write [4 17] INFO [2019-12-17 12:38:01,909] jepsen worker 5 - jepsen.util 12245 :fail :cas [1 [7 26]] :ValueNotSwapFrom INFO [2019-12-17 12:38:01,909] jepsen worker 5 - jepsen.util 12245 :invoke :cas [1 [38 35]] INFO [2019-12-17 12:38:01,934] jepsen worker 35 - jepsen.util 17315 :info :write [11 27] :RequestCanceledException INFO [2019-12-17 12:38:01,934] jepsen worker 38 - jepsen.util 13178 :info :write [12 21] :RequestCanceledException INFO [2019-12-17 12:38:01,934] jepsen worker 10 - jepsen.util 12430 :ok :read [3 35] INFO [2019-12-17 12:38:01,934] jepsen worker 10 - jepsen.util 12430 :invoke :read [3 nil] INFO [2019-12-17 12:38:01,935] jepsen worker 35 - jepsen.util 17405 :invoke :read [11 nil] INFO [2019-12-17 12:38:01,935] jepsen worker 38 - jepsen.util 13268 :invoke :cas [12 [31 21]] INFO [2019-12-17 12:38:01,949] jepsen worker 77 - jepsen.util 18347 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:38:01,949] jepsen worker 16 - jepsen.util 18826 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:38:01,949] jepsen worker 16 - jepsen.util 18826 :invoke :read [5 nil] INFO [2019-12-17 12:38:01,949] jepsen worker 77 - jepsen.util 18347 :invoke :write [25 0] INFO [2019-12-17 12:38:01,950] jepsen worker 63 - jepsen.util 11223 :ok :read [21 13] INFO [2019-12-17 12:38:01,950] jepsen worker 63 - jepsen.util 11223 :invoke :cas [21 [3 47]] INFO [2019-12-17 12:38:01,961] jepsen worker 9 - jepsen.util 13239 :fail :cas [3 [27 15]] :ValueNotSwapFrom INFO [2019-12-17 12:38:01,962] jepsen worker 9 - jepsen.util 13239 :invoke :read [3 nil] INFO [2019-12-17 12:38:01,973] jepsen worker 10 - jepsen.util 12430 :ok :read [3 35] INFO [2019-12-17 12:38:01,973] jepsen worker 10 - jepsen.util 12430 :invoke :read [3 nil] INFO [2019-12-17 12:38:01,987] jepsen worker 63 - jepsen.util 11223 :fail :cas [21 [3 47]] :ValueNotSwapFrom INFO [2019-12-17 12:38:01,987] jepsen worker 63 - jepsen.util 11223 :invoke :read [21 nil] INFO [2019-12-17 12:38:01,989] jepsen worker 72 - jepsen.util 12762 :ok :write [24 14] INFO [2019-12-17 12:38:01,989] jepsen worker 72 - jepsen.util 12762 :invoke :read [24 nil] INFO [2019-12-17 12:38:02,000] jepsen worker 15 - jepsen.util 16395 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:38:02,000] jepsen worker 2 - jepsen.util 16472 :info :write [0 1] :RequestCanceledException INFO [2019-12-17 12:38:02,001] jepsen worker 15 - jepsen.util 16395 :invoke :cas [5 [9 38]] INFO [2019-12-17 12:38:02,005] jepsen worker 2 - jepsen.util 16562 :invoke :cas [0 [25 4]] INFO [2019-12-17 12:38:02,009] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73141,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:38:02,009] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73141,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:38:02,019] jepsen worker 50 - jepsen.util 17240 :info :cas [16 [12 31]] :RequestCanceledException INFO [2019-12-17 12:38:02,023] jepsen worker 55 - jepsen.util 13285 :ok :read [18 13] INFO [2019-12-17 12:38:02,023] jepsen worker 55 - jepsen.util 13285 :invoke :write [18 18] INFO [2019-12-17 12:38:02,024] jepsen worker 50 - jepsen.util 17330 :invoke :cas [16 [15 5]] INFO [2019-12-17 12:38:02,028] jepsen worker 72 - jepsen.util 12762 :ok :read [24 14] INFO [2019-12-17 12:38:02,028] jepsen worker 72 - jepsen.util 12762 :invoke :write [24 5] INFO [2019-12-17 12:38:02,031] jepsen worker 46 - jepsen.util 12376 :ok :write [15 38] INFO [2019-12-17 12:38:02,031] jepsen worker 9 - jepsen.util 13239 :ok :read [3 35] INFO [2019-12-17 12:38:02,031] jepsen worker 46 - jepsen.util 12376 :invoke :write [15 40] INFO [2019-12-17 12:38:02,031] jepsen worker 9 - jepsen.util 13239 :invoke :write [3 0] INFO [2019-12-17 12:38:02,034] jepsen worker 81 - jepsen.util 16911 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:38:02,034] jepsen worker 44 - jepsen.util 17864 :info :cas [14 [37 30]] :RequestCanceledException INFO [2019-12-17 12:38:02,034] jepsen worker 81 - jepsen.util 16911 :invoke :read [27 nil] INFO [2019-12-17 12:38:02,034] jepsen worker 61 - jepsen.util 18601 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:38:02,034] jepsen worker 7 - jepsen.util 17287 :info :cas [2 [48 45]] :RequestCanceledException INFO [2019-12-17 12:38:02,035] jepsen worker 61 - jepsen.util 18601 :invoke :read [20 nil] INFO [2019-12-17 12:38:02,036] jepsen worker 7 - jepsen.util 17377 :invoke :read [2 nil] INFO [2019-12-17 12:38:02,037] jepsen worker 44 - jepsen.util 17954 :invoke :write [14 46] INFO [2019-12-17 12:38:02,048] jepsen worker 8 - jepsen.util 16748 :info :cas [2 [34 40]] :RequestCanceledException INFO [2019-12-17 12:38:02,048] jepsen worker 25 - jepsen.util 18385 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:38:02,048] jepsen worker 43 - jepsen.util 17503 :info :write [14 6] :RequestCanceledException INFO [2019-12-17 12:38:02,065] jepsen worker 57 - jepsen.util 17067 :info :write [19 35] :RequestCanceledException INFO [2019-12-17 12:38:02,089] jepsen worker 87 - jepsen.util 12147 :fail :cas [29 [48 9]] :ValueNotSwapFrom INFO [2019-12-17 12:38:02,090] jepsen worker 10 - jepsen.util 12430 :ok :read [3 35] INFO [2019-12-17 12:38:02,091] jepsen worker 63 - jepsen.util 11223 :ok :read [21 13] INFO [2019-12-17 12:38:02,093] jepsen worker 19 - jepsen.util 13069 :info :write [6 25] :SyncWriteAmbiguous INFO [2019-12-17 12:38:02,101] jepsen worker 59 - jepsen.util 16349 :info :cas [19 [2 20]] :RequestCanceledException INFO [2019-12-17 12:38:02,101] jepsen worker 58 - jepsen.util 17428 :info :write [19 26] :RequestCanceledException INFO [2019-12-17 12:38:02,110] jepsen worker 5 - jepsen.util 12245 :fail :cas [1 [38 35]] :ValueNotSwapFrom INFO [2019-12-17 12:38:02,119] jepsen worker 48 - jepsen.util 15978 :info :cas [16 [17 4]] :RequestCanceledException INFO [2019-12-17 12:38:02,121] jepsen worker 25 - jepsen.util 18385 :invoke :cas [8 [7 21]] INFO [2019-12-17 12:38:02,121] jepsen worker 87 - jepsen.util 12147 :invoke :write [29 13] INFO [2019-12-17 12:38:02,121] jepsen worker 10 - jepsen.util 12430 :invoke :cas [3 [45 37]] INFO [2019-12-17 12:38:02,122] jepsen worker 63 - jepsen.util 11223 :invoke :write [21 25] INFO [2019-12-17 12:38:02,122] jepsen worker 57 - jepsen.util 17157 :invoke :read [19 nil] INFO [2019-12-17 12:38:02,122] jepsen worker 8 - jepsen.util 16838 :invoke :write [2 23] INFO [2019-12-17 12:38:02,122] jepsen worker 5 - jepsen.util 12245 :invoke :write [1 20] INFO [2019-12-17 12:38:02,122] jepsen worker 43 - jepsen.util 17593 :invoke :read [14 nil] INFO [2019-12-17 12:38:02,123] jepsen worker 19 - jepsen.util 13159 :invoke :cas [6 [47 48]] INFO [2019-12-17 12:38:02,123] jepsen worker 59 - jepsen.util 16439 :invoke :write [19 17] INFO [2019-12-17 12:38:02,123] jepsen worker 58 - jepsen.util 17518 :invoke :write [19 26] INFO [2019-12-17 12:38:02,123] jepsen worker 48 - jepsen.util 16068 :invoke :cas [16 [49 33]] INFO [2019-12-17 12:38:02,130] jepsen worker 70 - jepsen.util 13120 :ok :read [23 1] INFO [2019-12-17 12:38:02,130] jepsen worker 70 - jepsen.util 13120 :invoke :cas [23 [34 9]] INFO [2019-12-17 12:38:02,147] jepsen worker 14 - jepsen.util 12254 :fail :cas [4 [34 11]] :ValueNotSwapFrom INFO [2019-12-17 12:38:02,147] jepsen worker 14 - jepsen.util 12254 :invoke :write [4 36] INFO [2019-12-17 12:38:02,164] jepsen worker 64 - jepsen.util 9874 :ok :write [21 43] INFO [2019-12-17 12:38:02,165] jepsen worker 64 - jepsen.util 9874 :invoke :read [21 nil] INFO [2019-12-17 12:38:02,188] jepsen worker 64 - jepsen.util 9874 :ok :read [21 43] INFO [2019-12-17 12:38:02,188] jepsen worker 64 - jepsen.util 9874 :invoke :cas [21 [38 2]] INFO [2019-12-17 12:38:02,210] jepsen worker 64 - jepsen.util 9874 :fail :cas [21 [38 2]] :ValueNotSwapFrom INFO [2019-12-17 12:38:02,210] jepsen worker 64 - jepsen.util 9874 :invoke :read [21 nil] INFO [2019-12-17 12:38:02,210] jepsen worker 72 - jepsen.util 12762 :ok :write [24 5] INFO [2019-12-17 12:38:02,210] jepsen worker 72 - jepsen.util 12762 :invoke :read [24 nil] INFO [2019-12-17 12:38:02,219] jepsen worker 51 - jepsen.util 11751 :info :write [17 40] :RequestCanceledException INFO [2019-12-17 12:38:02,220] jepsen worker 51 - jepsen.util 11841 :invoke :cas [17 [25 15]] INFO [2019-12-17 12:38:02,229] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73171,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:38:02,229] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73180,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:38:02,229] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73171,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:38:02,229] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73180,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:38:02,229] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73173,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:38:02,229] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73184,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:38:02,229] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73173,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:38:02,229] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73184,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:38:02,232] jepsen worker 64 - jepsen.util 9874 :ok :read [21 43] INFO [2019-12-17 12:38:02,232] jepsen worker 64 - jepsen.util 9874 :invoke :cas [21 [36 6]] INFO [2019-12-17 12:38:02,234] jepsen worker 69 - jepsen.util 12849 :info :write [23 32] :RequestCanceledException INFO [2019-12-17 12:38:02,234] jepsen worker 47 - jepsen.util 13727 :fail :read [15 nil] :RequestTimeoutException INFO [2019-12-17 12:38:02,234] jepsen worker 47 - jepsen.util 13727 :invoke :read [15 nil] INFO [2019-12-17 12:38:02,235] jepsen worker 69 - jepsen.util 12939 :invoke :cas [23 [7 42]] INFO [2019-12-17 12:38:02,238] jepsen worker 72 - jepsen.util 12762 :ok :read [24 5] INFO [2019-12-17 12:38:02,238] jepsen worker 72 - jepsen.util 12762 :invoke :read [24 nil] INFO [2019-12-17 12:38:02,263] jepsen worker 42 - jepsen.util 18132 :info :cas [14 [10 3]] :RequestCanceledException INFO [2019-12-17 12:38:02,265] jepsen worker 42 - jepsen.util 18222 :invoke :write [14 39] INFO [2019-12-17 12:38:02,266] jepsen worker 64 - jepsen.util 9874 :fail :cas [21 [36 6]] :ValueNotSwapFrom INFO [2019-12-17 12:38:02,266] jepsen worker 64 - jepsen.util 9874 :invoke :write [21 30] INFO [2019-12-17 12:38:02,274] jepsen worker 72 - jepsen.util 12762 :ok :read [24 5] INFO [2019-12-17 12:38:02,274] jepsen worker 72 - jepsen.util 12762 :invoke :cas [24 [18 43]] INFO [2019-12-17 12:38:02,296] jepsen worker 54 - jepsen.util 13194 :ok :read [18 13] INFO [2019-12-17 12:38:02,297] jepsen worker 54 - jepsen.util 13194 :invoke :write [18 16] INFO [2019-12-17 12:38:02,300] jepsen worker 39 - jepsen.util 16869 :info :cas [13 [32 22]] :RequestCanceledException INFO [2019-12-17 12:38:02,302] jepsen worker 39 - jepsen.util 16959 :invoke :read [13 nil] INFO [2019-12-17 12:38:02,308] jepsen worker 79 - jepsen.util 12859 :fail :cas [26 [11 6]] :ValueNotSwapFrom INFO [2019-12-17 12:38:02,308] jepsen worker 79 - jepsen.util 12859 :invoke :write [26 14] INFO [2019-12-17 12:38:02,313] jepsen worker 72 - jepsen.util 12762 :fail :cas [24 [18 43]] :ValueNotSwapFrom INFO [2019-12-17 12:38:02,314] jepsen worker 72 - jepsen.util 12762 :invoke :cas [24 [2 10]] INFO [2019-12-17 12:38:02,319] jepsen worker 36 - jepsen.util 13356 :info :write [12 37] :RequestCanceledException INFO [2019-12-17 12:38:02,319] jepsen worker 1 - jepsen.util 17461 :info :write [0 47] :RequestCanceledException INFO [2019-12-17 12:38:02,320] jepsen worker 1 - jepsen.util 17551 :invoke :cas [0 [19 45]] INFO [2019-12-17 12:38:02,320] jepsen worker 36 - jepsen.util 13446 :invoke :write [12 30] INFO [2019-12-17 12:38:02,329] jepsen worker 72 - jepsen.util 12762 :fail :cas [24 [2 10]] :ValueNotSwapFrom INFO [2019-12-17 12:38:02,329] jepsen worker 72 - jepsen.util 12762 :invoke :cas [24 [40 27]] INFO [2019-12-17 12:38:02,334] jepsen worker 32 - jepsen.util 13082 :info :write [10 31] :RequestCanceledException INFO [2019-12-17 12:38:02,334] jepsen worker 4 - jepsen.util 11794 :info :write [1 2] :RequestCanceledException INFO [2019-12-17 12:38:02,334] jepsen worker 83 - jepsen.util 17273 :info :cas [27 [5 21]] :RequestCanceledException INFO [2019-12-17 12:38:02,335] jepsen worker 4 - jepsen.util 11884 :invoke :write [1 38] INFO [2019-12-17 12:38:02,335] jepsen worker 83 - jepsen.util 17363 :invoke :read [27 nil] INFO [2019-12-17 12:38:02,337] jepsen worker 32 - jepsen.util 13172 :invoke :write [10 41] INFO [2019-12-17 12:38:02,359] jepsen worker 72 - jepsen.util 12762 :fail :cas [24 [40 27]] :ValueNotSwapFrom INFO [2019-12-17 12:38:02,359] jepsen worker 72 - jepsen.util 12762 :invoke :cas [24 [29 46]] INFO [2019-12-17 12:38:02,370] jepsen worker 72 - jepsen.util 12762 :fail :cas [24 [29 46]] :ValueNotSwapFrom INFO [2019-12-17 12:38:02,370] jepsen worker 72 - jepsen.util 12762 :invoke :write [24 39] INFO [2019-12-17 12:38:02,373] jepsen worker 64 - jepsen.util 9874 :ok :write [21 30] INFO [2019-12-17 12:38:02,373] jepsen worker 64 - jepsen.util 9874 :invoke :cas [21 [19 28]] INFO [2019-12-17 12:38:02,398] jepsen worker 64 - jepsen.util 9874 :fail :cas [21 [19 28]] :ValueNotSwapFrom INFO [2019-12-17 12:38:02,398] jepsen worker 64 - jepsen.util 9874 :invoke :write [21 39] INFO [2019-12-17 12:38:02,401] jepsen worker 52 - jepsen.util 12112 :info :write [17 11] :RequestCanceledException INFO [2019-12-17 12:38:02,403] jepsen worker 52 - jepsen.util 12202 :invoke :cas [17 [18 40]] INFO [2019-12-17 12:38:02,404] jepsen worker 70 - jepsen.util 13120 :fail :cas [23 [34 9]] :ValueNotSwapFrom INFO [2019-12-17 12:38:02,404] jepsen worker 70 - jepsen.util 13120 :invoke :read [23 nil] INFO [2019-12-17 12:38:02,418] jepsen worker 45 - jepsen.util 12465 :fail :read [15 nil] :RequestTimeoutException INFO [2019-12-17 12:38:02,418] jepsen worker 45 - jepsen.util 12465 :invoke :read [15 nil] INFO [2019-12-17 12:38:02,428] jepsen worker 56 - jepsen.util 13376 :ok :read [18 13] INFO [2019-12-17 12:38:02,429] jepsen worker 56 - jepsen.util 13376 :invoke :write [18 15] INFO [2019-12-17 12:38:02,434] jepsen worker 22 - jepsen.util 17392 :info :cas [7 [21 0]] :RequestCanceledException INFO [2019-12-17 12:38:02,436] jepsen worker 22 - jepsen.util 17482 :invoke :write [7 48] INFO [2019-12-17 12:38:02,447] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73212,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:38:02,447] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73212,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:38:02,447] jepsen worker 13 - jepsen.util 12073 :info :write [4 5] :RequestCanceledException INFO [2019-12-17 12:38:02,447] jepsen worker 18 - jepsen.util 13158 :info :write [6 5] :RequestCanceledException INFO [2019-12-17 12:38:02,447] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73215,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:38:02,447] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73215,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:38:02,448] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73216,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:38:02,448] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73216,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:38:02,448] jepsen worker 18 - jepsen.util 13248 :invoke :cas [6 [22 40]] INFO [2019-12-17 12:38:02,448] jepsen worker 13 - jepsen.util 12163 :invoke :cas [4 [29 47]] INFO [2019-12-17 12:38:02,450] jepsen worker 64 - jepsen.util 9874 :ok :write [21 39] INFO [2019-12-17 12:38:02,450] jepsen worker 64 - jepsen.util 9874 :invoke :cas [21 [36 12]] INFO [2019-12-17 12:38:02,471] jepsen worker 69 - jepsen.util 12939 :fail :cas [23 [7 42]] :ValueNotSwapFrom INFO [2019-12-17 12:38:02,471] jepsen worker 69 - jepsen.util 12939 :invoke :write [23 5] INFO [2019-12-17 12:38:02,478] jepsen worker 45 - jepsen.util 12465 :ok :read [15 38] INFO [2019-12-17 12:38:02,479] jepsen worker 45 - jepsen.util 12465 :invoke :cas [15 [4 27]] INFO [2019-12-17 12:38:02,487] jepsen worker 64 - jepsen.util 9874 :fail :cas [21 [36 12]] :ValueNotSwapFrom INFO [2019-12-17 12:38:02,487] jepsen worker 64 - jepsen.util 9874 :invoke :cas [21 [8 7]] INFO [2019-12-17 12:38:02,499] jepsen worker 45 - jepsen.util 12465 :fail :cas [15 [4 27]] :ValueNotSwapFrom INFO [2019-12-17 12:38:02,499] jepsen worker 45 - jepsen.util 12465 :invoke :write [15 44] INFO [2019-12-17 12:38:02,501] jepsen worker 6 - jepsen.util 17646 :info :cas [2 [4 23]] :RequestCanceledException INFO [2019-12-17 12:38:02,501] jepsen worker 6 - jepsen.util 17736 :invoke :cas [2 [27 46]] INFO [2019-12-17 12:38:02,513] jepsen worker 68 - jepsen.util 16538 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:38:02,513] jepsen worker 68 - jepsen.util 16538 :invoke :read [22 nil] INFO [2019-12-17 12:38:02,516] jepsen worker 64 - jepsen.util 9874 :fail :cas [21 [8 7]] :ValueNotSwapFrom INFO [2019-12-17 12:38:02,517] jepsen worker 64 - jepsen.util 9874 :invoke :read [21 nil] INFO [2019-12-17 12:38:02,535] jepsen worker 76 - jepsen.util 17266 :info :write [25 34] :RequestCanceledException INFO [2019-12-17 12:38:02,535] jepsen worker 20 - jepsen.util 12530 :info :write [6 27] :RequestCanceledException INFO [2019-12-17 12:38:02,536] jepsen worker 20 - jepsen.util 12620 :invoke :cas [6 [17 6]] INFO [2019-12-17 12:38:02,536] jepsen worker 76 - jepsen.util 17356 :invoke :cas [25 [35 15]] INFO [2019-12-17 12:38:02,563] jepsen worker 38 - jepsen.util 13268 :fail :cas [12 [31 21]] :ValueNotSwapFrom INFO [2019-12-17 12:38:02,563] jepsen worker 38 - jepsen.util 13268 :invoke :read [12 nil] INFO [2019-12-17 12:38:02,634] jepsen worker 49 - jepsen.util 17239 :info :cas [16 [31 45]] :RequestCanceledException INFO [2019-12-17 12:38:02,635] jepsen worker 49 - jepsen.util 17329 :invoke :read [16 nil] INFO [2019-12-17 12:38:02,647] jepsen worker 53 - jepsen.util 12113 :fail :cas [17 [39 48]] :ValueNotSwapFrom INFO [2019-12-17 12:38:02,647] jepsen worker 53 - jepsen.util 12113 :invoke :write [17 2] INFO [2019-12-17 12:38:02,648] jepsen worker 24 - jepsen.util 16494 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:38:02,648] jepsen worker 24 - jepsen.util 16494 :invoke :read [8 nil] INFO [2019-12-17 12:38:02,666] jepsen worker 89 - jepsen.util 12419 :info :write [29 32] :RequestCanceledException INFO [2019-12-17 12:38:02,666] jepsen worker 40 - jepsen.util 18130 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:38:02,666] jepsen worker 40 - jepsen.util 18130 :invoke :read [13 nil] INFO [2019-12-17 12:38:02,666] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73234,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:38:02,666] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73234,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:38:02,666] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73238,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:38:02,666] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73238,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:38:02,668] jepsen worker 89 - jepsen.util 12509 :invoke :write [29 25] INFO [2019-12-17 12:38:02,680] jepsen worker 52 - jepsen.util 12202 :fail :cas [17 [18 40]] :ValueNotSwapFrom INFO [2019-12-17 12:38:02,681] jepsen worker 52 - jepsen.util 12202 :invoke :cas [17 [2 44]] INFO [2019-12-17 12:38:02,718] jepsen worker 34 - jepsen.util 18124 :info :write [11 10] :RequestCanceledException INFO [2019-12-17 12:38:02,719] jepsen worker 34 - jepsen.util 18214 :invoke :write [11 6] INFO [2019-12-17 12:38:02,731] jepsen worker 41 - jepsen.util 18671 :info :cas [13 [25 40]] :RequestCanceledException INFO [2019-12-17 12:38:02,732] jepsen worker 41 - jepsen.util 18761 :invoke :read [13 nil] INFO [2019-12-17 12:38:02,743] jepsen worker 30 - jepsen.util 11910 :ok :write [10 22] INFO [2019-12-17 12:38:02,743] jepsen worker 30 - jepsen.util 11910 :invoke :read [10 nil] INFO [2019-12-17 12:38:02,796] jepsen worker 52 - jepsen.util 12202 :fail :cas [17 [2 44]] :ValueNotSwapFrom INFO [2019-12-17 12:38:02,796] jepsen worker 52 - jepsen.util 12202 :invoke :write [17 15] INFO [2019-12-17 12:38:02,799] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:38:02,799] jepsen nemesis - couchbase.util Rebalance check count: 577 INFO [2019-12-17 12:38:02,799] jepsen nemesis - couchbase.util Rebalance stuck count: 558 INFO [2019-12-17 12:38:02,800] jepsen worker 65 - jepsen.util 10955 :info :write [21 33] :RequestCanceledException INFO [2019-12-17 12:38:02,810] jepsen worker 65 - jepsen.util 11045 :invoke :write [21 33] INFO [2019-12-17 12:38:02,885] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73251,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:38:02,885] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73251,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:38:02,886] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73253,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:38:02,886] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73253,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:38:02,898] jepsen worker 51 - jepsen.util 11841 :fail :cas [17 [25 15]] :ValueNotSwapFrom INFO [2019-12-17 12:38:02,899] jepsen worker 51 - jepsen.util 11841 :invoke :write [17 14] INFO [2019-12-17 12:38:02,948] jepsen worker 17 - jepsen.util 17387 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:38:02,948] jepsen worker 17 - jepsen.util 17387 :invoke :read [5 nil] INFO [2019-12-17 12:38:02,964] jepsen worker 62 - jepsen.util 17972 :info :cas [20 [46 5]] :RequestCanceledException INFO [2019-12-17 12:38:02,965] jepsen worker 62 - jepsen.util 18062 :invoke :cas [20 [40 26]] INFO [2019-12-17 12:38:02,972] jepsen worker 70 - jepsen.util 13120 :ok :read [23 1] INFO [2019-12-17 12:38:02,972] jepsen worker 70 - jepsen.util 13120 :invoke :read [23 nil] INFO [2019-12-17 12:38:03,001] jepsen worker 86 - jepsen.util 17996 :info :cas [28 [9 6]] :RequestCanceledException INFO [2019-12-17 12:38:03,002] jepsen worker 86 - jepsen.util 18086 :invoke :cas [28 [14 20]] INFO [2019-12-17 12:38:03,018] jepsen worker 75 - jepsen.util 16815 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:38:03,018] jepsen worker 75 - jepsen.util 16815 :invoke :cas [25 [30 16]] INFO [2019-12-17 12:38:03,034] jepsen worker 60 - jepsen.util 18240 :info :cas [20 [5 27]] :RequestCanceledException INFO [2019-12-17 12:38:03,034] jepsen worker 28 - jepsen.util 10828 :fail :read [9 nil] :RequestTimeoutException INFO [2019-12-17 12:38:03,034] jepsen worker 26 - jepsen.util 18026 :info :cas [8 [38 30]] :RequestCanceledException INFO [2019-12-17 12:38:03,034] jepsen worker 28 - jepsen.util 10828 :invoke :read [9 nil] INFO [2019-12-17 12:38:03,035] jepsen worker 60 - jepsen.util 18330 :invoke :read [20 nil] INFO [2019-12-17 12:38:03,035] jepsen worker 26 - jepsen.util 18116 :invoke :write [8 17] INFO [2019-12-17 12:38:03,066] jepsen worker 70 - jepsen.util 13120 :ok :read [23 1] INFO [2019-12-17 12:38:03,066] jepsen worker 70 - jepsen.util 13120 :invoke :read [23 nil] INFO [2019-12-17 12:38:03,105] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73259,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:38:03,105] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73259,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:38:03,105] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73263,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:38:03,105] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73263,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:38:03,135] jepsen worker 27 - jepsen.util 11727 :info :write [9 25] :RequestCanceledException INFO [2019-12-17 12:38:03,136] jepsen worker 27 - jepsen.util 11817 :invoke :read [9 nil] INFO [2019-12-17 12:38:03,165] jepsen worker 66 - jepsen.util 17436 :info :cas [22 [36 4]] :RequestCanceledException INFO [2019-12-17 12:38:03,167] jepsen worker 66 - jepsen.util 17526 :invoke :read [22 nil] INFO [2019-12-17 12:38:03,200] jepsen worker 78 - jepsen.util 12318 :info :write [26 16] :RequestCanceledException INFO [2019-12-17 12:38:03,202] jepsen worker 78 - jepsen.util 12408 :invoke :cas [26 [35 41]] INFO [2019-12-17 12:38:03,215] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73273,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:38:03,215] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73273,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:38:03,215] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73274,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:38:03,215] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73274,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:38:03,234] jepsen worker 85 - jepsen.util 17905 :info :write [28 23] :RequestCanceledException INFO [2019-12-17 12:38:03,235] jepsen worker 85 - jepsen.util 17995 :invoke :read [28 nil] INFO [2019-12-17 12:38:03,240] jepsen worker 13 - jepsen.util 12163 :fail :cas [4 [29 47]] :ValueNotSwapFrom INFO [2019-12-17 12:38:03,240] jepsen worker 13 - jepsen.util 12163 :invoke :write [4 15] INFO [2019-12-17 12:38:03,301] jepsen worker 3 - jepsen.util 12063 :info :write [1 46] :RequestCanceledException INFO [2019-12-17 12:38:03,302] jepsen worker 3 - jepsen.util 12153 :invoke :cas [1 [13 1]] INFO [2019-12-17 12:38:03,318] jepsen worker 37 - jepsen.util 12007 :info :cas [12 [1 3]] :RequestCanceledException INFO [2019-12-17 12:38:03,318] jepsen worker 88 - jepsen.util 12688 :info :write [29 42] :RequestCanceledException INFO [2019-12-17 12:38:03,318] jepsen worker 84 - jepsen.util 16824 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:38:03,318] jepsen worker 84 - jepsen.util 16824 :invoke :write [28 39] INFO [2019-12-17 12:38:03,319] jepsen worker 37 - jepsen.util 12097 :invoke :read [12 nil] INFO [2019-12-17 12:38:03,324] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73282,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:38:03,324] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73282,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:38:03,325] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73284,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:38:03,325] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73284,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:38:03,330] jepsen worker 70 - jepsen.util 13120 :ok :read [23 1] INFO [2019-12-17 12:38:03,330] jepsen worker 70 - jepsen.util 13120 :invoke :read [23 nil] INFO [2019-12-17 12:38:03,334] jepsen worker 67 - jepsen.util 17437 :info :write [22 27] :RequestCanceledException INFO [2019-12-17 12:38:03,334] jepsen worker 33 - jepsen.util 17133 :info :write [11 39] :RequestCanceledException INFO [2019-12-17 12:38:03,335] jepsen worker 67 - jepsen.util 17527 :invoke :cas [22 [31 22]] INFO [2019-12-17 12:38:03,335] jepsen worker 33 - jepsen.util 17223 :invoke :cas [11 [14 20]] INFO [2019-12-17 12:38:03,339] jepsen worker 88 - jepsen.util 12778 :invoke :cas [29 [29 35]] INFO [2019-12-17 12:38:03,343] jepsen worker 38 - jepsen.util 13268 :ok :read [12 1] INFO [2019-12-17 12:38:03,343] jepsen worker 38 - jepsen.util 13268 :invoke :write [12 33] INFO [2019-12-17 12:38:03,365] jepsen worker 23 - jepsen.util 17753 :info :cas [7 [10 23]] :RequestCanceledException INFO [2019-12-17 12:38:03,366] jepsen worker 23 - jepsen.util 17843 :invoke :read [7 nil] INFO [2019-12-17 12:38:03,419] jepsen worker 82 - jepsen.util 17632 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:38:03,419] jepsen worker 82 - jepsen.util 17632 :invoke :write [27 14] INFO [2019-12-17 12:38:03,434] jepsen worker 21 - jepsen.util 16761 :info :cas [7 [22 35]] :RequestCanceledException INFO [2019-12-17 12:38:03,435] jepsen worker 21 - jepsen.util 16851 :invoke :cas [7 [30 29]] INFO [2019-12-17 12:38:03,465] jepsen worker 0 - jepsen.util 17640 :info :write [0 30] :RequestCanceledException INFO [2019-12-17 12:38:03,466] jepsen worker 0 - jepsen.util 17730 :invoke :write [0 24] INFO [2019-12-17 12:38:03,505] jepsen worker 72 - jepsen.util 12762 :ok :write [24 39] INFO [2019-12-17 12:38:03,506] jepsen worker 72 - jepsen.util 12762 :invoke :read [24 nil] INFO [2019-12-17 12:38:03,519] jepsen worker 9 - jepsen.util 13239 :ok :write [3 0] INFO [2019-12-17 12:38:03,519] jepsen worker 9 - jepsen.util 13239 :invoke :read [3 nil] INFO [2019-12-17 12:38:03,519] jepsen worker 46 - jepsen.util 12376 :ok :write [15 40] INFO [2019-12-17 12:38:03,519] jepsen worker 46 - jepsen.util 12376 :invoke :read [15 nil] INFO [2019-12-17 12:38:03,520] jepsen worker 31 - jepsen.util 11011 :ok :write [10 31] INFO [2019-12-17 12:38:03,520] jepsen worker 31 - jepsen.util 11011 :invoke :cas [10 [34 30]] INFO [2019-12-17 12:38:03,525] jepsen worker 72 - jepsen.util 12762 :ok :read [24 39] INFO [2019-12-17 12:38:03,526] jepsen worker 72 - jepsen.util 12762 :invoke :write [24 46] INFO [2019-12-17 12:38:03,534] jepsen worker 30 - jepsen.util 11910 :ok :read [10 31] INFO [2019-12-17 12:38:03,535] jepsen worker 30 - jepsen.util 11910 :invoke :read [10 nil] INFO [2019-12-17 12:38:03,544] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73299,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:38:03,544] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73299,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:38:03,544] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73302,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:38:03,544] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73302,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:38:03,551] jepsen worker 78 - jepsen.util 12408 :fail :cas [26 [35 41]] :ValueNotSwapFrom INFO [2019-12-17 12:38:03,551] jepsen worker 78 - jepsen.util 12408 :invoke :read [26 nil] INFO [2019-12-17 12:38:03,555] jepsen worker 46 - jepsen.util 12376 :ok :read [15 40] INFO [2019-12-17 12:38:03,555] jepsen worker 46 - jepsen.util 12376 :invoke :read [15 nil] INFO [2019-12-17 12:38:03,558] jepsen worker 27 - jepsen.util 11817 :ok :read [9 1] INFO [2019-12-17 12:38:03,558] jepsen worker 27 - jepsen.util 11817 :invoke :cas [9 [31 26]] INFO [2019-12-17 12:38:03,589] jepsen worker 46 - jepsen.util 12376 :ok :read [15 40] INFO [2019-12-17 12:38:03,589] jepsen worker 46 - jepsen.util 12376 :invoke :cas [15 [23 19]] INFO [2019-12-17 12:38:03,593] jepsen worker 31 - jepsen.util 11011 :fail :cas [10 [34 30]] :ValueNotSwapFrom INFO [2019-12-17 12:38:03,593] jepsen worker 31 - jepsen.util 11011 :invoke :write [10 22] INFO [2019-12-17 12:38:03,638] jepsen worker 46 - jepsen.util 12376 :fail :cas [15 [23 19]] :ValueNotSwapFrom INFO [2019-12-17 12:38:03,638] jepsen worker 46 - jepsen.util 12376 :invoke :read [15 nil] INFO [2019-12-17 12:38:03,639] jepsen worker 30 - jepsen.util 11910 :ok :read [10 31] INFO [2019-12-17 12:38:03,639] jepsen worker 30 - jepsen.util 11910 :invoke :write [10 32] INFO [2019-12-17 12:38:03,654] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73322,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:38:03,657] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73322,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:38:03,657] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73323,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:38:03,657] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73323,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:38:03,665] jepsen worker 27 - jepsen.util 11817 :fail :cas [9 [31 26]] :ValueNotSwapFrom INFO [2019-12-17 12:38:03,666] jepsen worker 27 - jepsen.util 11817 :invoke :write [9 8] INFO [2019-12-17 12:38:03,686] jepsen worker 5 - jepsen.util 12245 :ok :write [1 20] INFO [2019-12-17 12:38:03,686] jepsen worker 5 - jepsen.util 12245 :invoke :cas [1 [29 8]] INFO [2019-12-17 12:38:03,686] jepsen worker 46 - jepsen.util 12376 :ok :read [15 40] INFO [2019-12-17 12:38:03,687] jepsen worker 46 - jepsen.util 12376 :invoke :write [15 1] INFO [2019-12-17 12:38:03,724] jepsen worker 72 - jepsen.util 12762 :ok :write [24 46] INFO [2019-12-17 12:38:03,724] jepsen worker 72 - jepsen.util 12762 :invoke :read [24 nil] INFO [2019-12-17 12:38:03,735] jepsen worker 11 - jepsen.util 12341 :info :write [3 22] :RequestCanceledException INFO [2019-12-17 12:38:03,739] jepsen worker 11 - jepsen.util 12431 :invoke :cas [3 [43 14]] INFO [2019-12-17 12:38:03,742] jepsen worker 51 - jepsen.util 11841 :ok :write [17 14] INFO [2019-12-17 12:38:03,742] jepsen worker 51 - jepsen.util 11841 :invoke :write [17 1] INFO [2019-12-17 12:38:03,749] jepsen worker 72 - jepsen.util 12762 :ok :read [24 46] INFO [2019-12-17 12:38:03,750] jepsen worker 72 - jepsen.util 12762 :invoke :write [24 6] INFO [2019-12-17 12:38:03,750] jepsen worker 32 - jepsen.util 13172 :ok :write [10 41] INFO [2019-12-17 12:38:03,750] jepsen worker 32 - jepsen.util 13172 :invoke :read [10 nil] INFO [2019-12-17 12:38:03,766] jepsen worker 71 - jepsen.util 13481 :ok :write [23 36] INFO [2019-12-17 12:38:03,766] jepsen worker 71 - jepsen.util 13481 :invoke :write [23 24] INFO [2019-12-17 12:38:03,780] jepsen worker 9 - jepsen.util 13239 :ok :read [3 0] INFO [2019-12-17 12:38:03,780] jepsen worker 9 - jepsen.util 13239 :invoke :cas [3 [15 29]] INFO [2019-12-17 12:38:03,797] jepsen worker 27 - jepsen.util 11817 :ok :write [9 8] INFO [2019-12-17 12:38:03,797] jepsen worker 27 - jepsen.util 11817 :invoke :cas [9 [8 27]] INFO [2019-12-17 12:38:03,801] jepsen worker 29 - jepsen.util 11009 :info :write [9 25] :RequestCanceledException INFO [2019-12-17 12:38:03,801] jepsen worker 73 - jepsen.util 11773 :info :write [24 30] :RequestCanceledException INFO [2019-12-17 12:38:03,801] jepsen worker 32 - jepsen.util 13172 :ok :read [10 41] INFO [2019-12-17 12:38:03,801] jepsen worker 32 - jepsen.util 13172 :invoke :read [10 nil] INFO [2019-12-17 12:38:03,802] jepsen worker 73 - jepsen.util 11863 :invoke :read [24 nil] INFO [2019-12-17 12:38:03,802] jepsen worker 29 - jepsen.util 11099 :invoke :cas [9 [14 9]] INFO [2019-12-17 12:38:03,816] jepsen worker 74 - jepsen.util 12404 :info :write [24 37] :RequestCanceledException INFO [2019-12-17 12:38:03,818] jepsen worker 88 - jepsen.util 12778 :fail :cas [29 [29 35]] :ValueNotSwapFrom INFO [2019-12-17 12:38:03,818] jepsen worker 88 - jepsen.util 12778 :invoke :write [29 4] INFO [2019-12-17 12:38:03,820] jepsen worker 74 - jepsen.util 12494 :invoke :cas [24 [9 49]] INFO [2019-12-17 12:38:03,829] jepsen worker 9 - jepsen.util 13239 :fail :cas [3 [15 29]] :ValueNotSwapFrom INFO [2019-12-17 12:38:03,829] jepsen worker 9 - jepsen.util 13239 :invoke :write [3 35] INFO [2019-12-17 12:38:03,840] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:38:03,840] jepsen nemesis - couchbase.util Rebalance check count: 578 INFO [2019-12-17 12:38:03,840] jepsen nemesis - couchbase.util Rebalance stuck count: 559 INFO [2019-12-17 12:38:03,848] jepsen worker 80 - jepsen.util 12050 :info :write [26 5] :RequestCanceledException INFO [2019-12-17 12:38:03,849] jepsen worker 80 - jepsen.util 12140 :invoke :read [26 nil] INFO [2019-12-17 12:38:03,866] jepsen worker 11 - jepsen.util 12431 :fail :cas [3 [43 14]] :ValueNotSwapFrom INFO [2019-12-17 12:38:03,866] jepsen worker 11 - jepsen.util 12431 :invoke :write [3 14] INFO [2019-12-17 12:38:03,872] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73343,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:38:03,872] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73343,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:38:03,872] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73346,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:38:03,872] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73346,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:38:03,872] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73347,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:38:03,872] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73347,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:38:03,889] jepsen worker 32 - jepsen.util 13172 :ok :read [10 41] INFO [2019-12-17 12:38:03,889] jepsen worker 32 - jepsen.util 13172 :invoke :read [10 nil] INFO [2019-12-17 12:38:03,935] jepsen worker 74 - jepsen.util 12494 :fail :cas [24 [9 49]] :ValueNotSwapFrom INFO [2019-12-17 12:38:03,935] jepsen worker 74 - jepsen.util 12494 :invoke :cas [24 [17 21]] INFO [2019-12-17 12:38:03,941] jepsen worker 73 - jepsen.util 11863 :ok :read [24 46] INFO [2019-12-17 12:38:03,941] jepsen worker 73 - jepsen.util 11863 :invoke :write [24 24] INFO [2019-12-17 12:38:03,949] jepsen worker 12 - jepsen.util 12702 :info :write [4 17] :RequestCanceledException INFO [2019-12-17 12:38:03,950] jepsen worker 12 - jepsen.util 12792 :invoke :write [4 4] INFO [2019-12-17 12:38:03,960] jepsen worker 74 - jepsen.util 12494 :fail :cas [24 [17 21]] :ValueNotSwapFrom INFO [2019-12-17 12:38:03,960] jepsen worker 74 - jepsen.util 12494 :invoke :write [24 10] INFO [2019-12-17 12:38:04,018] jepsen worker 47 - jepsen.util 13727 :ok :read [15 40] INFO [2019-12-17 12:38:04,018] jepsen worker 47 - jepsen.util 13727 :invoke :cas [15 [19 8]] INFO [2019-12-17 12:38:04,026] jepsen worker 18 - jepsen.util 13248 :fail :cas [6 [22 40]] :ValueNotSwapFrom INFO [2019-12-17 12:38:04,026] jepsen worker 18 - jepsen.util 13248 :invoke :write [6 24] INFO [2019-12-17 12:38:04,034] jepsen worker 35 - jepsen.util 17405 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:38:04,034] jepsen worker 35 - jepsen.util 17405 :invoke :write [11 28] INFO [2019-12-17 12:38:04,039] jepsen worker 28 - jepsen.util 10828 :ok :read [9 25] INFO [2019-12-17 12:38:04,039] jepsen worker 28 - jepsen.util 10828 :invoke :cas [9 [19 28]] INFO [2019-12-17 12:38:04,046] jepsen worker 5 - jepsen.util 12245 :fail :cas [1 [29 8]] :ValueNotSwapFrom INFO [2019-12-17 12:38:04,048] jepsen worker 16 - jepsen.util 18826 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:38:04,048] jepsen worker 77 - jepsen.util 18347 :info :write [25 0] :RequestCanceledException INFO [2019-12-17 12:38:04,048] jepsen worker 55 - jepsen.util 13285 :info :write [18 18] :RequestCanceledException INFO [2019-12-17 12:38:04,048] jepsen worker 16 - jepsen.util 18826 :invoke :cas [5 [39 44]] INFO [2019-12-17 12:38:04,050] jepsen worker 77 - jepsen.util 18437 :invoke :cas [25 [23 36]] INFO [2019-12-17 12:38:04,050] jepsen worker 55 - jepsen.util 13375 :invoke :write [18 24] INFO [2019-12-17 12:38:04,050] jepsen worker 5 - jepsen.util 12245 :invoke :write [1 46] INFO [2019-12-17 12:38:04,053] jepsen worker 46 - jepsen.util 12376 :ok :write [15 1] INFO [2019-12-17 12:38:04,053] jepsen worker 46 - jepsen.util 12376 :invoke :cas [15 [23 49]] INFO [2019-12-17 12:38:04,057] jepsen worker 47 - jepsen.util 13727 :fail :cas [15 [19 8]] :ValueNotSwapFrom INFO [2019-12-17 12:38:04,057] jepsen worker 47 - jepsen.util 13727 :invoke :cas [15 [29 18]] INFO [2019-12-17 12:38:04,066] jepsen worker 2 - jepsen.util 16562 :info :cas [0 [25 4]] :RequestCanceledException INFO [2019-12-17 12:38:04,067] jepsen worker 2 - jepsen.util 16652 :invoke :write [0 18] INFO [2019-12-17 12:38:04,093] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73356,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:38:04,093] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73356,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:38:04,093] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73359,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:38:04,093] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73359,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:38:04,096] jepsen worker 72 - jepsen.util 12762 :ok :write [24 6] INFO [2019-12-17 12:38:04,096] jepsen worker 72 - jepsen.util 12762 :invoke :write [24 8] INFO [2019-12-17 12:38:04,100] jepsen worker 15 - jepsen.util 16395 :info :cas [5 [9 38]] :RequestCanceledException INFO [2019-12-17 12:38:04,101] jepsen worker 7 - jepsen.util 17377 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:38:04,101] jepsen worker 46 - jepsen.util 12376 :fail :cas [15 [23 49]] :ValueNotSwapFrom INFO [2019-12-17 12:38:04,101] jepsen worker 7 - jepsen.util 17377 :invoke :read [2 nil] INFO [2019-12-17 12:38:04,101] jepsen worker 46 - jepsen.util 12376 :invoke :cas [15 [16 48]] INFO [2019-12-17 12:38:04,102] jepsen worker 37 - jepsen.util 12097 :ok :read [12 1] INFO [2019-12-17 12:38:04,102] jepsen worker 37 - jepsen.util 12097 :invoke :write [12 9] INFO [2019-12-17 12:38:04,104] jepsen worker 15 - jepsen.util 16485 :invoke :cas [5 [43 16]] INFO [2019-12-17 12:38:04,113] jepsen worker 44 - jepsen.util 17954 :info :write [14 46] :RequestCanceledException INFO [2019-12-17 12:38:04,114] jepsen worker 44 - jepsen.util 18044 :invoke :cas [14 [27 42]] INFO [2019-12-17 12:38:04,132] jepsen worker 50 - jepsen.util 17330 :info :cas [16 [15 5]] :RequestCanceledException INFO [2019-12-17 12:38:04,132] jepsen worker 57 - jepsen.util 17157 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:38:04,132] jepsen worker 61 - jepsen.util 18601 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:38:04,132] jepsen worker 81 - jepsen.util 16911 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:38:04,133] jepsen worker 48 - jepsen.util 16068 :info :cas [16 [49 33]] :RequestCanceledException INFO [2019-12-17 12:38:04,133] jepsen worker 10 - jepsen.util 12430 :info :cas [3 [45 37]] :RequestCanceledException INFO [2019-12-17 12:38:04,133] jepsen worker 19 - jepsen.util 13159 :info :cas [6 [47 48]] :RequestCanceledException INFO [2019-12-17 12:38:04,133] jepsen worker 57 - jepsen.util 17157 :invoke :read [19 nil] INFO [2019-12-17 12:38:04,133] jepsen worker 61 - jepsen.util 18601 :invoke :read [20 nil] INFO [2019-12-17 12:38:04,133] jepsen worker 81 - jepsen.util 16911 :invoke :read [27 nil] INFO [2019-12-17 12:38:04,134] jepsen worker 50 - jepsen.util 17420 :invoke :read [16 nil] INFO [2019-12-17 12:38:04,134] jepsen worker 48 - jepsen.util 16158 :invoke :cas [16 [33 2]] INFO [2019-12-17 12:38:04,137] jepsen worker 29 - jepsen.util 11099 :fail :cas [9 [14 9]] :ValueNotSwapFrom INFO [2019-12-17 12:38:04,138] jepsen worker 29 - jepsen.util 11099 :invoke :cas [9 [16 13]] INFO [2019-12-17 12:38:04,138] jepsen worker 19 - jepsen.util 13249 :invoke :write [6 26] INFO [2019-12-17 12:38:04,138] jepsen worker 10 - jepsen.util 12520 :invoke :write [3 34] INFO [2019-12-17 12:38:04,148] jepsen worker 8 - jepsen.util 16838 :info :write [2 23] :RequestCanceledException INFO [2019-12-17 12:38:04,148] jepsen worker 25 - jepsen.util 18385 :info :cas [8 [7 21]] :RequestCanceledException INFO [2019-12-17 12:38:04,149] jepsen worker 8 - jepsen.util 16928 :invoke :cas [2 [17 45]] INFO [2019-12-17 12:38:04,149] jepsen worker 25 - jepsen.util 18475 :invoke :cas [8 [39 31]] INFO [2019-12-17 12:38:04,164] jepsen worker 46 - jepsen.util 12376 :fail :cas [15 [16 48]] :ValueNotSwapFrom INFO [2019-12-17 12:38:04,164] jepsen worker 46 - jepsen.util 12376 :invoke :cas [15 [4 15]] INFO [2019-12-17 12:38:04,166] jepsen worker 43 - jepsen.util 17593 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:38:04,166] jepsen worker 43 - jepsen.util 17593 :invoke :read [14 nil] INFO [2019-12-17 12:38:04,188] jepsen worker 46 - jepsen.util 12376 :fail :cas [15 [4 15]] :ValueNotSwapFrom INFO [2019-12-17 12:38:04,189] jepsen worker 70 - jepsen.util 13120 :ok :read [23 36] INFO [2019-12-17 12:38:04,189] jepsen worker 46 - jepsen.util 12376 :invoke :write [15 3] INFO [2019-12-17 12:38:04,189] jepsen worker 70 - jepsen.util 13120 :invoke :cas [23 [15 35]] INFO [2019-12-17 12:38:04,202] jepsen worker 59 - jepsen.util 16439 :info :write [19 17] :RequestCanceledException INFO [2019-12-17 12:38:04,202] jepsen worker 87 - jepsen.util 12147 :info :write [29 13] :RequestCanceledException INFO [2019-12-17 12:38:04,203] jepsen worker 87 - jepsen.util 12237 :invoke :cas [29 [34 9]] INFO [2019-12-17 12:38:04,203] jepsen worker 59 - jepsen.util 16529 :invoke :read [19 nil] INFO [2019-12-17 12:38:04,219] jepsen worker 58 - jepsen.util 17518 :info :write [19 26] :RequestCanceledException INFO [2019-12-17 12:38:04,220] jepsen worker 58 - jepsen.util 17608 :invoke :cas [19 [33 40]] INFO [2019-12-17 12:38:04,233] jepsen worker 9 - jepsen.util 13239 :ok :write [3 35] INFO [2019-12-17 12:38:04,233] jepsen worker 9 - jepsen.util 13239 :invoke :write [3 7] INFO [2019-12-17 12:38:04,235] jepsen worker 63 - jepsen.util 11223 :info :write [21 25] :RequestCanceledException INFO [2019-12-17 12:38:04,236] jepsen worker 63 - jepsen.util 11313 :invoke :write [21 7] INFO [2019-12-17 12:38:04,241] jepsen worker 14 - jepsen.util 12254 :info :write [4 36] :RequestCanceledException INFO [2019-12-17 12:38:04,243] jepsen worker 14 - jepsen.util 12344 :invoke :cas [4 [41 13]] INFO [2019-12-17 12:38:04,278] jepsen worker 14 - jepsen.util 12344 :fail :cas [4 [41 13]] :ValueNotSwapFrom INFO [2019-12-17 12:38:04,279] jepsen worker 14 - jepsen.util 12344 :invoke :cas [4 [32 4]] INFO [2019-12-17 12:38:04,310] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73392,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:38:04,310] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73401,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:38:04,310] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73392,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:38:04,310] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73401,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:38:04,310] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73393,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:38:04,310] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73405,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:38:04,310] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73393,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:38:04,310] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73405,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:38:04,310] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73394,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:38:04,310] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73394,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:38:04,332] jepsen worker 54 - jepsen.util 13194 :info :write [18 16] :RequestCanceledException INFO [2019-12-17 12:38:04,332] jepsen worker 42 - jepsen.util 18222 :info :write [14 39] :RequestCanceledException INFO [2019-12-17 12:38:04,333] jepsen worker 42 - jepsen.util 18312 :invoke :write [14 37] INFO [2019-12-17 12:38:04,333] jepsen worker 80 - jepsen.util 12140 :ok :read [26 18] INFO [2019-12-17 12:38:04,333] jepsen worker 80 - jepsen.util 12140 :invoke :read [26 nil] INFO [2019-12-17 12:38:04,334] jepsen worker 54 - jepsen.util 13284 :invoke :cas [18 [44 25]] INFO [2019-12-17 12:38:04,345] jepsen worker 18 - jepsen.util 13248 :ok :write [6 24] INFO [2019-12-17 12:38:04,345] jepsen worker 18 - jepsen.util 13248 :invoke :read [6 nil] INFO [2019-12-17 12:38:04,345] jepsen worker 72 - jepsen.util 12762 :ok :write [24 8] INFO [2019-12-17 12:38:04,345] jepsen worker 72 - jepsen.util 12762 :invoke :write [24 47] INFO [2019-12-17 12:38:04,348] jepsen worker 79 - jepsen.util 12859 :info :write [26 14] :RequestCanceledException INFO [2019-12-17 12:38:04,352] jepsen worker 79 - jepsen.util 12949 :invoke :read [26 nil] INFO [2019-12-17 12:38:04,364] jepsen worker 83 - jepsen.util 17363 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:38:04,364] jepsen worker 83 - jepsen.util 17363 :invoke :cas [27 [5 34]] INFO [2019-12-17 12:38:04,364] jepsen worker 18 - jepsen.util 13248 :ok :read [6 24] INFO [2019-12-17 12:38:04,365] jepsen worker 18 - jepsen.util 13248 :invoke :write [6 40] INFO [2019-12-17 12:38:04,384] jepsen worker 78 - jepsen.util 12408 :ok :read [26 18] INFO [2019-12-17 12:38:04,384] jepsen worker 78 - jepsen.util 12408 :invoke :read [26 nil] INFO [2019-12-17 12:38:04,401] jepsen worker 39 - jepsen.util 16959 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:38:04,401] jepsen worker 39 - jepsen.util 16959 :invoke :read [13 nil] INFO [2019-12-17 12:38:04,404] jepsen worker 63 - jepsen.util 11313 :ok :write [21 7] INFO [2019-12-17 12:38:04,404] jepsen worker 63 - jepsen.util 11313 :invoke :read [21 nil] INFO [2019-12-17 12:38:04,405] jepsen worker 14 - jepsen.util 12344 :fail :cas [4 [32 4]] :ValueNotSwapFrom INFO [2019-12-17 12:38:04,405] jepsen worker 46 - jepsen.util 12376 :ok :write [15 3] INFO [2019-12-17 12:38:04,405] jepsen worker 46 - jepsen.util 12376 :invoke :read [15 nil] INFO [2019-12-17 12:38:04,407] jepsen worker 14 - jepsen.util 12344 :invoke :read [4 nil] INFO [2019-12-17 12:38:04,418] jepsen worker 1 - jepsen.util 17551 :info :cas [0 [19 45]] :RequestCanceledException INFO [2019-12-17 12:38:04,425] jepsen worker 1 - jepsen.util 17641 :invoke :cas [0 [5 13]] INFO [2019-12-17 12:38:04,428] jepsen worker 46 - jepsen.util 12376 :ok :read [15 3] INFO [2019-12-17 12:38:04,428] jepsen worker 46 - jepsen.util 12376 :invoke :cas [15 [42 1]] INFO [2019-12-17 12:38:04,434] jepsen worker 36 - jepsen.util 13446 :info :write [12 30] :RequestCanceledException INFO [2019-12-17 12:38:04,435] jepsen worker 36 - jepsen.util 13536 :invoke :write [12 11] INFO [2019-12-17 12:38:04,438] jepsen worker 63 - jepsen.util 11313 :ok :read [21 7] INFO [2019-12-17 12:38:04,438] jepsen worker 63 - jepsen.util 11313 :invoke :cas [21 [7 24]] INFO [2019-12-17 12:38:04,448] jepsen worker 56 - jepsen.util 13376 :info :write [18 15] :RequestCanceledException INFO [2019-12-17 12:38:04,448] jepsen worker 4 - jepsen.util 11884 :info :write [1 38] :RequestCanceledException INFO [2019-12-17 12:38:04,449] jepsen worker 4 - jepsen.util 11974 :invoke :cas [1 [1 13]] INFO [2019-12-17 12:38:04,449] jepsen worker 56 - jepsen.util 13466 :invoke :cas [18 [20 27]] INFO [2019-12-17 12:38:04,462] jepsen worker 14 - jepsen.util 12344 :ok :read [4 4] INFO [2019-12-17 12:38:04,462] jepsen worker 14 - jepsen.util 12344 :invoke :cas [4 [7 12]] INFO [2019-12-17 12:38:04,467] jepsen worker 46 - jepsen.util 12376 :fail :cas [15 [42 1]] :ValueNotSwapFrom INFO [2019-12-17 12:38:04,467] jepsen worker 46 - jepsen.util 12376 :invoke :write [15 48] INFO [2019-12-17 12:38:04,488] jepsen worker 70 - jepsen.util 13120 :fail :cas [23 [15 35]] :ValueNotSwapFrom INFO [2019-12-17 12:38:04,488] jepsen worker 70 - jepsen.util 13120 :invoke :cas [23 [1 7]] INFO [2019-12-17 12:38:04,516] jepsen worker 22 - jepsen.util 17482 :info :write [7 48] :RequestCanceledException INFO [2019-12-17 12:38:04,516] jepsen worker 45 - jepsen.util 12465 :info :write [15 44] :RequestCanceledException INFO [2019-12-17 12:38:04,517] jepsen worker 45 - jepsen.util 12555 :invoke :read [15 nil] INFO [2019-12-17 12:38:04,517] jepsen worker 63 - jepsen.util 11313 :ok :cas [21 [7 24]] INFO [2019-12-17 12:38:04,517] jepsen worker 63 - jepsen.util 11313 :invoke :write [21 46] INFO [2019-12-17 12:38:04,520] jepsen worker 80 - jepsen.util 12140 :ok :read [26 18] INFO [2019-12-17 12:38:04,520] jepsen worker 22 - jepsen.util 17572 :invoke :write [7 36] INFO [2019-12-17 12:38:04,520] jepsen worker 80 - jepsen.util 12140 :invoke :write [26 1] INFO [2019-12-17 12:38:04,531] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73427,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:38:04,531] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73427,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:38:04,531] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73431,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:38:04,531] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73431,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:38:04,563] jepsen worker 69 - jepsen.util 12939 :info :write [23 5] :RequestCanceledException INFO [2019-12-17 12:38:04,563] jepsen worker 6 - jepsen.util 17736 :info :cas [2 [27 46]] :RequestCanceledException INFO [2019-12-17 12:38:04,564] jepsen worker 69 - jepsen.util 13029 :invoke :write [23 14] INFO [2019-12-17 12:38:04,568] jepsen worker 63 - jepsen.util 11313 :ok :write [21 46] INFO [2019-12-17 12:38:04,569] jepsen worker 63 - jepsen.util 11313 :invoke :cas [21 [24 25]] INFO [2019-12-17 12:38:04,580] jepsen worker 6 - jepsen.util 17826 :invoke :write [2 15] INFO [2019-12-17 12:38:04,582] jepsen worker 63 - jepsen.util 11313 :fail :cas [21 [24 25]] :ValueNotSwapFrom INFO [2019-12-17 12:38:04,582] jepsen worker 63 - jepsen.util 11313 :invoke :write [21 19] INFO [2019-12-17 12:38:04,601] jepsen worker 76 - jepsen.util 17356 :info :cas [25 [35 15]] :RequestCanceledException INFO [2019-12-17 12:38:04,602] jepsen worker 76 - jepsen.util 17446 :invoke :read [25 nil] INFO [2019-12-17 12:38:04,611] jepsen worker 56 - jepsen.util 13466 :fail :cas [18 [20 27]] :ValueNotSwapFrom INFO [2019-12-17 12:38:04,611] jepsen worker 56 - jepsen.util 13466 :invoke :cas [18 [31 7]] INFO [2019-12-17 12:38:04,618] jepsen worker 64 - jepsen.util 9874 :fail :read [21 nil] :RequestTimeoutException INFO [2019-12-17 12:38:04,618] jepsen worker 68 - jepsen.util 16538 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:38:04,618] jepsen worker 64 - jepsen.util 9874 :invoke :write [21 44] INFO [2019-12-17 12:38:04,618] jepsen worker 68 - jepsen.util 16538 :invoke :cas [22 [48 44]] INFO [2019-12-17 12:38:04,629] jepsen worker 63 - jepsen.util 11313 :ok :write [21 19] INFO [2019-12-17 12:38:04,629] jepsen worker 63 - jepsen.util 11313 :invoke :cas [21 [41 4]] INFO [2019-12-17 12:38:04,634] jepsen worker 20 - jepsen.util 12620 :info :cas [6 [17 6]] :RequestCanceledException INFO [2019-12-17 12:38:04,635] jepsen worker 20 - jepsen.util 12710 :invoke :cas [6 [44 48]] INFO [2019-12-17 12:38:04,645] jepsen worker 70 - jepsen.util 13120 :fail :cas [23 [1 7]] :ValueNotSwapFrom INFO [2019-12-17 12:38:04,645] jepsen worker 70 - jepsen.util 13120 :invoke :cas [23 [45 27]] INFO [2019-12-17 12:38:04,654] jepsen worker 63 - jepsen.util 11313 :fail :cas [21 [41 4]] :ValueNotSwapFrom INFO [2019-12-17 12:38:04,655] jepsen worker 63 - jepsen.util 11313 :invoke :write [21 3] INFO [2019-12-17 12:38:04,692] jepsen worker 64 - jepsen.util 9874 :ok :write [21 44] INFO [2019-12-17 12:38:04,692] jepsen worker 64 - jepsen.util 9874 :invoke :write [21 14] INFO [2019-12-17 12:38:04,696] jepsen worker 4 - jepsen.util 11974 :fail :cas [1 [1 13]] :ValueNotSwapFrom INFO [2019-12-17 12:38:04,697] jepsen worker 4 - jepsen.util 11974 :invoke :cas [1 [30 48]] INFO [2019-12-17 12:38:04,716] jepsen worker 3 - jepsen.util 12153 :fail :cas [1 [13 1]] :ValueNotSwapFrom INFO [2019-12-17 12:38:04,716] jepsen worker 3 - jepsen.util 12153 :invoke :cas [1 [36 44]] INFO [2019-12-17 12:38:04,719] jepsen worker 49 - jepsen.util 17329 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:38:04,719] jepsen worker 53 - jepsen.util 12113 :info :write [17 2] :RequestCanceledException INFO [2019-12-17 12:38:04,719] jepsen worker 49 - jepsen.util 17329 :invoke :cas [16 [41 44]] INFO [2019-12-17 12:38:04,731] jepsen worker 14 - jepsen.util 12344 :fail :cas [4 [7 12]] :ValueNotSwapFrom INFO [2019-12-17 12:38:04,731] jepsen worker 14 - jepsen.util 12344 :invoke :read [4 nil] INFO [2019-12-17 12:38:04,732] jepsen worker 53 - jepsen.util 12203 :invoke :write [17 38] INFO [2019-12-17 12:38:04,734] jepsen worker 89 - jepsen.util 12509 :info :write [29 25] :RequestCanceledException INFO [2019-12-17 12:38:04,735] jepsen worker 89 - jepsen.util 12599 :invoke :write [29 29] INFO [2019-12-17 12:38:04,739] jepsen worker 64 - jepsen.util 9874 :ok :write [21 14] INFO [2019-12-17 12:38:04,739] jepsen worker 64 - jepsen.util 9874 :invoke :read [21 nil] INFO [2019-12-17 12:38:04,748] jepsen worker 24 - jepsen.util 16494 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:38:04,748] jepsen worker 34 - jepsen.util 18214 :info :write [11 6] :RequestCanceledException INFO [2019-12-17 12:38:04,749] jepsen worker 24 - jepsen.util 16494 :invoke :read [8 nil] INFO [2019-12-17 12:38:04,752] jepsen worker 34 - jepsen.util 18304 :invoke :read [11 nil] INFO [2019-12-17 12:38:04,753] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73456,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:38:04,753] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73456,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:38:04,753] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73459,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:38:04,753] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73459,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:38:04,754] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73460,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:38:04,754] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73460,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:38:04,756] jepsen worker 64 - jepsen.util 9874 :ok :read [21 14] INFO [2019-12-17 12:38:04,756] jepsen worker 64 - jepsen.util 9874 :invoke :cas [21 [49 24]] INFO [2019-12-17 12:38:04,762] jepsen worker 56 - jepsen.util 13466 :fail :cas [18 [31 7]] :ValueNotSwapFrom INFO [2019-12-17 12:38:04,762] jepsen worker 56 - jepsen.util 13466 :invoke :write [18 22] INFO [2019-12-17 12:38:04,765] jepsen worker 40 - jepsen.util 18130 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:38:04,765] jepsen worker 41 - jepsen.util 18761 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:38:04,765] jepsen worker 40 - jepsen.util 18130 :invoke :cas [13 [21 39]] INFO [2019-12-17 12:38:04,765] jepsen worker 41 - jepsen.util 18761 :invoke :read [13 nil] INFO [2019-12-17 12:38:04,771] jepsen worker 64 - jepsen.util 9874 :fail :cas [21 [49 24]] :ValueNotSwapFrom INFO [2019-12-17 12:38:04,771] jepsen worker 64 - jepsen.util 9874 :invoke :read [21 nil] INFO [2019-12-17 12:38:04,785] jepsen worker 4 - jepsen.util 11974 :fail :cas [1 [30 48]] :ValueNotSwapFrom INFO [2019-12-17 12:38:04,785] jepsen worker 4 - jepsen.util 11974 :invoke :write [1 36] INFO [2019-12-17 12:38:04,788] jepsen worker 64 - jepsen.util 9874 :ok :read [21 14] INFO [2019-12-17 12:38:04,788] jepsen worker 64 - jepsen.util 9874 :invoke :write [21 33] INFO [2019-12-17 12:38:04,822] jepsen worker 70 - jepsen.util 13120 :fail :cas [23 [45 27]] :ValueNotSwapFrom INFO [2019-12-17 12:38:04,822] jepsen worker 70 - jepsen.util 13120 :invoke :cas [23 [14 25]] INFO [2019-12-17 12:38:04,831] jepsen worker 64 - jepsen.util 9874 :ok :write [21 33] INFO [2019-12-17 12:38:04,831] jepsen worker 64 - jepsen.util 9874 :invoke :cas [21 [44 4]] INFO [2019-12-17 12:38:04,834] jepsen worker 65 - jepsen.util 11045 :info :write [21 33] :RequestCanceledException INFO [2019-12-17 12:38:04,835] jepsen worker 65 - jepsen.util 11135 :invoke :cas [21 [6 13]] INFO [2019-12-17 12:38:04,859] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:38:04,859] jepsen nemesis - couchbase.util Rebalance check count: 579 INFO [2019-12-17 12:38:04,859] jepsen nemesis - couchbase.util Rebalance stuck count: 560 INFO [2019-12-17 12:38:04,863] jepsen worker 3 - jepsen.util 12153 :fail :cas [1 [36 44]] :ValueNotSwapFrom INFO [2019-12-17 12:38:04,863] jepsen worker 3 - jepsen.util 12153 :invoke :read [1 nil] INFO [2019-12-17 12:38:04,866] jepsen worker 65 - jepsen.util 11135 :fail :cas [21 [6 13]] :ValueNotSwapFrom INFO [2019-12-17 12:38:04,866] jepsen worker 65 - jepsen.util 11135 :invoke :read [21 nil] INFO [2019-12-17 12:38:04,868] jepsen worker 64 - jepsen.util 9874 :fail :cas [21 [44 4]] :ValueNotSwapFrom INFO [2019-12-17 12:38:04,868] jepsen worker 64 - jepsen.util 9874 :invoke :cas [21 [47 33]] INFO [2019-12-17 12:38:04,890] jepsen worker 64 - jepsen.util 9874 :fail :cas [21 [47 33]] :ValueNotSwapFrom INFO [2019-12-17 12:38:04,890] jepsen worker 64 - jepsen.util 9874 :invoke :cas [21 [7 31]] INFO [2019-12-17 12:38:04,894] jepsen worker 65 - jepsen.util 11135 :ok :read [21 33] INFO [2019-12-17 12:38:04,894] jepsen worker 65 - jepsen.util 11135 :invoke :read [21 nil] INFO [2019-12-17 12:38:04,895] jepsen worker 14 - jepsen.util 12344 :ok :read [4 4] INFO [2019-12-17 12:38:04,895] jepsen worker 14 - jepsen.util 12344 :invoke :read [4 nil] INFO [2019-12-17 12:38:04,901] jepsen worker 52 - jepsen.util 12202 :info :write [17 15] :RequestCanceledException INFO [2019-12-17 12:38:04,901] jepsen worker 52 - jepsen.util 12292 :invoke :cas [17 [24 42]] INFO [2019-12-17 12:38:04,917] jepsen worker 65 - jepsen.util 11135 :ok :read [21 33] INFO [2019-12-17 12:38:04,917] jepsen worker 65 - jepsen.util 11135 :invoke :write [21 28] INFO [2019-12-17 12:38:04,924] jepsen worker 64 - jepsen.util 9874 :fail :cas [21 [7 31]] :ValueNotSwapFrom INFO [2019-12-17 12:38:04,924] jepsen worker 64 - jepsen.util 9874 :invoke :read [21 nil] INFO [2019-12-17 12:38:04,937] jepsen worker 64 - jepsen.util 9874 :ok :read [21 33] INFO [2019-12-17 12:38:04,937] jepsen worker 64 - jepsen.util 9874 :invoke :write [21 45] INFO [2019-12-17 12:38:04,944] jepsen worker 3 - jepsen.util 12153 :ok :read [1 20] INFO [2019-12-17 12:38:04,944] jepsen worker 3 - jepsen.util 12153 :invoke :write [1 5] INFO [2019-12-17 12:38:04,966] jepsen worker 65 - jepsen.util 11135 :ok :write [21 28] INFO [2019-12-17 12:38:04,966] jepsen worker 65 - jepsen.util 11135 :invoke :read [21 nil] INFO [2019-12-17 12:38:04,969] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73485,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:38:04,969] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73485,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:38:04,969] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73488,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:38:04,969] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73488,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:38:04,979] jepsen worker 65 - jepsen.util 11135 :ok :read [21 28] INFO [2019-12-17 12:38:04,979] jepsen worker 65 - jepsen.util 11135 :invoke :write [21 20] INFO [2019-12-17 12:38:05,000] jepsen worker 62 - jepsen.util 18062 :info :cas [20 [40 26]] :RequestCanceledException INFO [2019-12-17 12:38:05,000] jepsen worker 64 - jepsen.util 9874 :ok :write [21 45] INFO [2019-12-17 12:38:05,001] jepsen worker 64 - jepsen.util 9874 :invoke :cas [21 [9 9]] INFO [2019-12-17 12:38:05,001] jepsen worker 62 - jepsen.util 18152 :invoke :read [20 nil] INFO [2019-12-17 12:38:05,019] jepsen worker 86 - jepsen.util 18086 :info :cas [28 [14 20]] :RequestCanceledException INFO [2019-12-17 12:38:05,020] jepsen worker 86 - jepsen.util 18176 :invoke :cas [28 [4 36]] INFO [2019-12-17 12:38:05,037] jepsen worker 65 - jepsen.util 11135 :ok :write [21 20] INFO [2019-12-17 12:38:05,037] jepsen worker 65 - jepsen.util 11135 :invoke :cas [21 [29 44]] INFO [2019-12-17 12:38:05,039] jepsen worker 78 - jepsen.util 12408 :ok :read [26 18] INFO [2019-12-17 12:38:05,039] jepsen worker 78 - jepsen.util 12408 :invoke :cas [26 [21 0]] INFO [2019-12-17 12:38:05,040] jepsen worker 64 - jepsen.util 9874 :fail :cas [21 [9 9]] :ValueNotSwapFrom INFO [2019-12-17 12:38:05,040] jepsen worker 64 - jepsen.util 9874 :invoke :write [21 14] INFO [2019-12-17 12:38:05,048] jepsen worker 17 - jepsen.util 17387 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:38:05,049] jepsen worker 17 - jepsen.util 17387 :invoke :write [5 15] INFO [2019-12-17 12:38:05,078] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73503,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:38:05,078] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73503,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:38:05,079] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73505,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:38:05,079] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73505,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:38:05,081] jepsen worker 65 - jepsen.util 11135 :fail :cas [21 [29 44]] :ValueNotSwapFrom INFO [2019-12-17 12:38:05,081] jepsen worker 65 - jepsen.util 11135 :invoke :write [21 46] INFO [2019-12-17 12:38:05,095] jepsen worker 64 - jepsen.util 9874 :ok :write [21 14] INFO [2019-12-17 12:38:05,095] jepsen worker 64 - jepsen.util 9874 :invoke :write [21 28] INFO [2019-12-17 12:38:05,103] jepsen worker 70 - jepsen.util 13120 :fail :cas [23 [14 25]] :ValueNotSwapFrom INFO [2019-12-17 12:38:05,103] jepsen worker 70 - jepsen.util 13120 :invoke :cas [23 [49 35]] INFO [2019-12-17 12:38:05,119] jepsen worker 75 - jepsen.util 16815 :info :cas [25 [30 16]] :RequestCanceledException INFO [2019-12-17 12:38:05,119] jepsen worker 75 - jepsen.util 16905 :invoke :cas [25 [23 13]] INFO [2019-12-17 12:38:05,128] jepsen worker 87 - jepsen.util 12237 :fail :cas [29 [34 9]] :ValueNotSwapFrom INFO [2019-12-17 12:38:05,128] jepsen worker 87 - jepsen.util 12237 :invoke :write [29 43] INFO [2019-12-17 12:38:05,134] jepsen worker 60 - jepsen.util 18330 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:38:05,135] jepsen worker 60 - jepsen.util 18330 :invoke :cas [20 [6 1]] INFO [2019-12-17 12:38:05,148] jepsen worker 26 - jepsen.util 18116 :info :write [8 17] :RequestCanceledException INFO [2019-12-17 12:38:05,151] jepsen worker 65 - jepsen.util 11135 :ok :write [21 46] INFO [2019-12-17 12:38:05,152] jepsen worker 65 - jepsen.util 11135 :invoke :write [21 19] INFO [2019-12-17 12:38:05,152] jepsen worker 14 - jepsen.util 12344 :ok :read [4 4] INFO [2019-12-17 12:38:05,153] jepsen worker 14 - jepsen.util 12344 :invoke :read [4 nil] INFO [2019-12-17 12:38:05,154] jepsen worker 26 - jepsen.util 18206 :invoke :cas [8 [20 24]] INFO [2019-12-17 12:38:05,162] jepsen worker 54 - jepsen.util 13284 :fail :cas [18 [44 25]] :ValueNotSwapFrom INFO [2019-12-17 12:38:05,162] jepsen worker 54 - jepsen.util 13284 :invoke :read [18 nil] INFO [2019-12-17 12:38:05,176] jepsen worker 52 - jepsen.util 12292 :fail :cas [17 [24 42]] :ValueNotSwapFrom INFO [2019-12-17 12:38:05,176] jepsen worker 52 - jepsen.util 12292 :invoke :read [17 nil] INFO [2019-12-17 12:38:05,186] jepsen worker 70 - jepsen.util 13120 :fail :cas [23 [49 35]] :ValueNotSwapFrom INFO [2019-12-17 12:38:05,187] jepsen worker 70 - jepsen.util 13120 :invoke :write [23 35] INFO [2019-12-17 12:38:05,198] jepsen worker 65 - jepsen.util 11135 :ok :write [21 19] INFO [2019-12-17 12:38:05,199] jepsen worker 65 - jepsen.util 11135 :invoke :read [21 nil] INFO [2019-12-17 12:38:05,214] jepsen worker 65 - jepsen.util 11135 :ok :read [21 19] INFO [2019-12-17 12:38:05,214] jepsen worker 65 - jepsen.util 11135 :invoke :read [21 nil] INFO [2019-12-17 12:38:05,231] jepsen worker 65 - jepsen.util 11135 :ok :read [21 19] INFO [2019-12-17 12:38:05,231] jepsen worker 65 - jepsen.util 11135 :invoke :cas [21 [3 35]] INFO [2019-12-17 12:38:05,234] jepsen worker 66 - jepsen.util 17526 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:38:05,234] jepsen worker 66 - jepsen.util 17526 :invoke :read [22 nil] INFO [2019-12-17 12:38:05,236] jepsen worker 14 - jepsen.util 12344 :ok :read [4 4] INFO [2019-12-17 12:38:05,236] jepsen worker 14 - jepsen.util 12344 :invoke :read [4 nil] INFO [2019-12-17 12:38:05,245] jepsen worker 65 - jepsen.util 11135 :fail :cas [21 [3 35]] :ValueNotSwapFrom INFO [2019-12-17 12:38:05,245] jepsen worker 65 - jepsen.util 11135 :invoke :cas [21 [11 39]] INFO [2019-12-17 12:38:05,259] jepsen worker 65 - jepsen.util 11135 :fail :cas [21 [11 39]] :ValueNotSwapFrom INFO [2019-12-17 12:38:05,259] jepsen worker 65 - jepsen.util 11135 :invoke :cas [21 [43 14]] INFO [2019-12-17 12:38:05,273] jepsen worker 65 - jepsen.util 11135 :fail :cas [21 [43 14]] :ValueNotSwapFrom INFO [2019-12-17 12:38:05,273] jepsen worker 65 - jepsen.util 11135 :invoke :read [21 nil] INFO [2019-12-17 12:38:05,289] jepsen worker 65 - jepsen.util 11135 :ok :read [21 19] INFO [2019-12-17 12:38:05,289] jepsen worker 65 - jepsen.util 11135 :invoke :write [21 43] INFO [2019-12-17 12:38:05,296] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73528,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:38:05,296] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73535,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:38:05,296] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73528,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:38:05,296] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73535,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:38:05,296] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73529,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:38:05,296] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73538,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:38:05,296] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73529,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:38:05,296] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73538,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:38:05,306] jepsen worker 52 - jepsen.util 12292 :ok :read [17 14] INFO [2019-12-17 12:38:05,306] jepsen worker 52 - jepsen.util 12292 :invoke :read [17 nil] INFO [2019-12-17 12:38:05,319] jepsen worker 85 - jepsen.util 17995 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:38:05,319] jepsen worker 85 - jepsen.util 17995 :invoke :read [28 nil] INFO [2019-12-17 12:38:05,337] jepsen worker 65 - jepsen.util 11135 :ok :write [21 43] INFO [2019-12-17 12:38:05,337] jepsen worker 65 - jepsen.util 11135 :invoke :read [21 nil] INFO [2019-12-17 12:38:05,346] jepsen worker 13 - jepsen.util 12163 :info :write [4 15] :RequestCanceledException INFO [2019-12-17 12:38:05,347] jepsen worker 13 - jepsen.util 12253 :invoke :read [4 nil] INFO [2019-12-17 12:38:05,351] jepsen worker 65 - jepsen.util 11135 :ok :read [21 43] INFO [2019-12-17 12:38:05,352] jepsen worker 65 - jepsen.util 11135 :invoke :read [21 nil] INFO [2019-12-17 12:38:05,362] jepsen worker 52 - jepsen.util 12292 :ok :read [17 14] INFO [2019-12-17 12:38:05,362] jepsen worker 52 - jepsen.util 12292 :invoke :read [17 nil] INFO [2019-12-17 12:38:05,366] jepsen worker 67 - jepsen.util 17527 :info :cas [22 [31 22]] :RequestCanceledException INFO [2019-12-17 12:38:05,366] jepsen worker 67 - jepsen.util 17617 :invoke :cas [22 [37 22]] INFO [2019-12-17 12:38:05,378] jepsen worker 65 - jepsen.util 11135 :ok :read [21 43] INFO [2019-12-17 12:38:05,378] jepsen worker 65 - jepsen.util 11135 :invoke :cas [21 [16 28]] INFO [2019-12-17 12:38:05,383] jepsen worker 14 - jepsen.util 12344 :ok :read [4 4] INFO [2019-12-17 12:38:05,384] jepsen worker 14 - jepsen.util 12344 :invoke :read [4 nil] INFO [2019-12-17 12:38:05,393] jepsen worker 65 - jepsen.util 11135 :fail :cas [21 [16 28]] :ValueNotSwapFrom INFO [2019-12-17 12:38:05,393] jepsen worker 65 - jepsen.util 11135 :invoke :read [21 nil] INFO [2019-12-17 12:38:05,397] jepsen worker 52 - jepsen.util 12292 :ok :read [17 14] INFO [2019-12-17 12:38:05,397] jepsen worker 52 - jepsen.util 12292 :invoke :write [17 44] INFO [2019-12-17 12:38:05,410] jepsen worker 65 - jepsen.util 11135 :ok :read [21 43] INFO [2019-12-17 12:38:05,410] jepsen worker 65 - jepsen.util 11135 :invoke :cas [21 [0 2]] INFO [2019-12-17 12:38:05,417] jepsen worker 78 - jepsen.util 12408 :fail :cas [26 [21 0]] :ValueNotSwapFrom INFO [2019-12-17 12:38:05,417] jepsen worker 78 - jepsen.util 12408 :invoke :cas [26 [37 19]] INFO [2019-12-17 12:38:05,419] jepsen worker 84 - jepsen.util 16824 :info :write [28 39] :RequestCanceledException INFO [2019-12-17 12:38:05,419] jepsen worker 23 - jepsen.util 17843 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:38:05,419] jepsen worker 23 - jepsen.util 17843 :invoke :read [7 nil] INFO [2019-12-17 12:38:05,420] jepsen worker 84 - jepsen.util 16914 :invoke :read [28 nil] INFO [2019-12-17 12:38:05,424] jepsen worker 65 - jepsen.util 11135 :fail :cas [21 [0 2]] :ValueNotSwapFrom INFO [2019-12-17 12:38:05,424] jepsen worker 65 - jepsen.util 11135 :invoke :cas [21 [45 23]] INFO [2019-12-17 12:38:05,433] jepsen worker 14 - jepsen.util 12344 :ok :read [4 4] INFO [2019-12-17 12:38:05,433] jepsen worker 14 - jepsen.util 12344 :invoke :read [4 nil] INFO [2019-12-17 12:38:05,434] jepsen worker 33 - jepsen.util 17223 :info :cas [11 [14 20]] :RequestCanceledException INFO [2019-12-17 12:38:05,435] jepsen worker 33 - jepsen.util 17313 :invoke :read [11 nil] INFO [2019-12-17 12:38:05,438] jepsen worker 65 - jepsen.util 11135 :fail :cas [21 [45 23]] :ValueNotSwapFrom INFO [2019-12-17 12:38:05,438] jepsen worker 65 - jepsen.util 11135 :invoke :cas [21 [10 25]] INFO [2019-12-17 12:38:05,448] jepsen worker 38 - jepsen.util 13268 :info :write [12 33] :RequestCanceledException INFO [2019-12-17 12:38:05,449] jepsen worker 38 - jepsen.util 13358 :invoke :write [12 29] INFO [2019-12-17 12:38:05,452] jepsen worker 65 - jepsen.util 11135 :fail :cas [21 [10 25]] :ValueNotSwapFrom INFO [2019-12-17 12:38:05,452] jepsen worker 65 - jepsen.util 11135 :invoke :write [21 43] INFO [2019-12-17 12:38:05,465] jepsen worker 13 - jepsen.util 12253 :ok :read [4 4] INFO [2019-12-17 12:38:05,465] jepsen worker 13 - jepsen.util 12253 :invoke :write [4 32] INFO [2019-12-17 12:38:05,499] jepsen worker 65 - jepsen.util 11135 :ok :write [21 43] INFO [2019-12-17 12:38:05,499] jepsen worker 65 - jepsen.util 11135 :invoke :write [21 49] INFO [2019-12-17 12:38:05,516] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73567,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:38:05,516] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73567,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:38:05,516] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73570,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:38:05,516] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73570,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:38:05,519] jepsen worker 82 - jepsen.util 17632 :info :write [27 14] :RequestCanceledException INFO [2019-12-17 12:38:05,520] jepsen worker 82 - jepsen.util 17722 :invoke :write [27 26] INFO [2019-12-17 12:38:05,523] jepsen worker 79 - jepsen.util 12949 :ok :read [26 18] INFO [2019-12-17 12:38:05,524] jepsen worker 79 - jepsen.util 12949 :invoke :write [26 6] INFO [2019-12-17 12:38:05,535] jepsen worker 21 - jepsen.util 16851 :info :cas [7 [30 29]] :RequestCanceledException INFO [2019-12-17 12:38:05,536] jepsen worker 21 - jepsen.util 16941 :invoke :cas [7 [5 18]] INFO [2019-12-17 12:38:05,544] jepsen worker 65 - jepsen.util 11135 :ok :write [21 49] INFO [2019-12-17 12:38:05,545] jepsen worker 65 - jepsen.util 11135 :invoke :cas [21 [27 16]] INFO [2019-12-17 12:38:05,548] jepsen worker 0 - jepsen.util 17730 :info :write [0 24] :RequestCanceledException INFO [2019-12-17 12:38:05,549] jepsen worker 0 - jepsen.util 17820 :invoke :read [0 nil] INFO [2019-12-17 12:38:05,565] jepsen worker 65 - jepsen.util 11135 :fail :cas [21 [27 16]] :ValueNotSwapFrom INFO [2019-12-17 12:38:05,565] jepsen worker 65 - jepsen.util 11135 :invoke :read [21 nil] INFO [2019-12-17 12:38:05,575] jepsen worker 65 - jepsen.util 11135 :ok :read [21 49] INFO [2019-12-17 12:38:05,575] jepsen worker 65 - jepsen.util 11135 :invoke :read [21 nil] INFO [2019-12-17 12:38:05,588] jepsen worker 65 - jepsen.util 11135 :ok :read [21 49] INFO [2019-12-17 12:38:05,588] jepsen worker 65 - jepsen.util 11135 :invoke :write [21 29] INFO [2019-12-17 12:38:05,631] jepsen worker 65 - jepsen.util 11135 :ok :write [21 29] INFO [2019-12-17 12:38:05,631] jepsen worker 65 - jepsen.util 11135 :invoke :cas [21 [47 24]] INFO [2019-12-17 12:38:05,649] jepsen worker 65 - jepsen.util 11135 :fail :cas [21 [47 24]] :ValueNotSwapFrom INFO [2019-12-17 12:38:05,650] jepsen worker 65 - jepsen.util 11135 :invoke :cas [21 [40 15]] INFO [2019-12-17 12:38:05,662] jepsen worker 65 - jepsen.util 11135 :fail :cas [21 [40 15]] :ValueNotSwapFrom INFO [2019-12-17 12:38:05,662] jepsen worker 65 - jepsen.util 11135 :invoke :read [21 nil] INFO [2019-12-17 12:38:05,665] jepsen worker 31 - jepsen.util 11011 :info :write [10 22] :RequestCanceledException INFO [2019-12-17 12:38:05,665] jepsen worker 30 - jepsen.util 11910 :info :write [10 32] :RequestCanceledException INFO [2019-12-17 12:38:05,666] jepsen worker 30 - jepsen.util 12000 :invoke :read [10 nil] INFO [2019-12-17 12:38:05,666] jepsen worker 31 - jepsen.util 11101 :invoke :read [10 nil] INFO [2019-12-17 12:38:05,678] jepsen worker 65 - jepsen.util 11135 :ok :read [21 29] INFO [2019-12-17 12:38:05,678] jepsen worker 65 - jepsen.util 11135 :invoke :write [21 34] INFO [2019-12-17 12:38:05,698] jepsen worker 14 - jepsen.util 12344 :ok :read [4 4] INFO [2019-12-17 12:38:05,698] jepsen worker 14 - jepsen.util 12344 :invoke :read [4 nil] INFO [2019-12-17 12:38:05,714] jepsen worker 65 - jepsen.util 11135 :ok :write [21 34] INFO [2019-12-17 12:38:05,714] jepsen worker 65 - jepsen.util 11135 :invoke :write [21 24] INFO [2019-12-17 12:38:05,735] jepsen worker 14 - jepsen.util 12344 :ok :read [4 4] INFO [2019-12-17 12:38:05,735] jepsen worker 14 - jepsen.util 12344 :invoke :write [4 0] INFO [2019-12-17 12:38:05,740] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73586,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:38:05,740] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73586,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:38:05,740] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73589,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:38:05,740] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73589,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:38:05,749] jepsen worker 51 - jepsen.util 11841 :info :write [17 1] :RequestCanceledException INFO [2019-12-17 12:38:05,750] jepsen worker 51 - jepsen.util 11931 :invoke :write [17 30] INFO [2019-12-17 12:38:05,757] jepsen worker 65 - jepsen.util 11135 :ok :write [21 24] INFO [2019-12-17 12:38:05,757] jepsen worker 65 - jepsen.util 11135 :invoke :read [21 nil] INFO [2019-12-17 12:38:05,773] jepsen worker 65 - jepsen.util 11135 :ok :read [21 24] INFO [2019-12-17 12:38:05,773] jepsen worker 65 - jepsen.util 11135 :invoke :cas [21 [4 28]] INFO [2019-12-17 12:38:05,785] jepsen worker 65 - jepsen.util 11135 :fail :cas [21 [4 28]] :ValueNotSwapFrom INFO [2019-12-17 12:38:05,785] jepsen worker 65 - jepsen.util 11135 :invoke :read [21 nil] INFO [2019-12-17 12:38:05,797] jepsen worker 65 - jepsen.util 11135 :ok :read [21 24] INFO [2019-12-17 12:38:05,797] jepsen worker 65 - jepsen.util 11135 :invoke :read [21 nil] INFO [2019-12-17 12:38:05,802] jepsen worker 71 - jepsen.util 13481 :info :write [23 24] :RequestCanceledException INFO [2019-12-17 12:38:05,813] jepsen worker 65 - jepsen.util 11135 :ok :read [21 24] INFO [2019-12-17 12:38:05,813] jepsen worker 65 - jepsen.util 11135 :invoke :cas [21 [33 9]] INFO [2019-12-17 12:38:05,829] jepsen worker 71 - jepsen.util 13571 :invoke :read [23 nil] INFO [2019-12-17 12:38:05,833] jepsen worker 65 - jepsen.util 11135 :fail :cas [21 [33 9]] :ValueNotSwapFrom INFO [2019-12-17 12:38:05,833] jepsen worker 65 - jepsen.util 11135 :invoke :cas [21 [15 13]] INFO [2019-12-17 12:38:05,858] jepsen worker 65 - jepsen.util 11135 :fail :cas [21 [15 13]] :ValueNotSwapFrom INFO [2019-12-17 12:38:05,858] jepsen worker 65 - jepsen.util 11135 :invoke :read [21 nil] INFO [2019-12-17 12:38:05,874] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:38:05,874] jepsen nemesis - couchbase.util Rebalance check count: 580 INFO [2019-12-17 12:38:05,874] jepsen nemesis - couchbase.util Rebalance stuck count: 561 INFO [2019-12-17 12:38:05,879] jepsen worker 65 - jepsen.util 11135 :ok :read [21 24] INFO [2019-12-17 12:38:05,879] jepsen worker 65 - jepsen.util 11135 :invoke :write [21 11] INFO [2019-12-17 12:38:05,901] jepsen worker 88 - jepsen.util 12778 :info :write [29 4] :RequestCanceledException INFO [2019-12-17 12:38:05,902] jepsen worker 88 - jepsen.util 12868 :invoke :cas [29 [25 1]] INFO [2019-12-17 12:38:05,926] jepsen worker 65 - jepsen.util 11135 :ok :write [21 11] INFO [2019-12-17 12:38:05,926] jepsen worker 65 - jepsen.util 11135 :invoke :cas [21 [18 24]] INFO [2019-12-17 12:38:05,934] jepsen worker 32 - jepsen.util 13172 :fail :read [10 nil] :RequestTimeoutException INFO [2019-12-17 12:38:05,934] jepsen worker 32 - jepsen.util 13172 :invoke :cas [10 [5 28]] INFO [2019-12-17 12:38:05,948] jepsen worker 65 - jepsen.util 11135 :fail :cas [21 [18 24]] :ValueNotSwapFrom INFO [2019-12-17 12:38:05,949] jepsen worker 65 - jepsen.util 11135 :invoke :cas [21 [22 25]] INFO [2019-12-17 12:38:05,952] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73608,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:38:05,952] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73608,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:38:05,952] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73613,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:38:05,952] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73613,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:38:05,963] jepsen worker 65 - jepsen.util 11135 :fail :cas [21 [22 25]] :ValueNotSwapFrom INFO [2019-12-17 12:38:05,963] jepsen worker 65 - jepsen.util 11135 :invoke :read [21 nil] INFO [2019-12-17 12:38:05,965] jepsen worker 27 - jepsen.util 11817 :info :cas [9 [8 27]] :RequestCanceledException INFO [2019-12-17 12:38:05,965] jepsen worker 11 - jepsen.util 12431 :info :write [3 14] :RequestCanceledException INFO [2019-12-17 12:38:05,966] jepsen worker 27 - jepsen.util 11907 :invoke :cas [9 [21 30]] INFO [2019-12-17 12:38:05,966] jepsen worker 11 - jepsen.util 12521 :invoke :read [3 nil] INFO [2019-12-17 12:38:05,977] jepsen worker 65 - jepsen.util 11135 :ok :read [21 11] INFO [2019-12-17 12:38:05,977] jepsen worker 65 - jepsen.util 11135 :invoke :write [21 44] INFO [2019-12-17 12:38:05,988] jepsen worker 27 - jepsen.util 11907 :fail :cas [9 [21 30]] :ValueNotSwapFrom INFO [2019-12-17 12:38:05,988] jepsen worker 27 - jepsen.util 11907 :invoke :cas [9 [25 28]] INFO [2019-12-17 12:38:06,003] jepsen worker 78 - jepsen.util 12408 :fail :cas [26 [37 19]] :ValueNotSwapFrom INFO [2019-12-17 12:38:06,003] jepsen worker 78 - jepsen.util 12408 :invoke :write [26 7] INFO [2019-12-17 12:38:06,018] jepsen worker 74 - jepsen.util 12494 :info :write [24 10] :RequestCanceledException INFO [2019-12-17 12:38:06,019] jepsen worker 74 - jepsen.util 12584 :invoke :write [24 0] INFO [2019-12-17 12:38:06,034] jepsen worker 65 - jepsen.util 11135 :ok :write [21 44] INFO [2019-12-17 12:38:06,034] jepsen worker 65 - jepsen.util 11135 :invoke :read [21 nil] INFO [2019-12-17 12:38:06,035] jepsen worker 73 - jepsen.util 11863 :info :write [24 24] :RequestCanceledException INFO [2019-12-17 12:38:06,036] jepsen worker 73 - jepsen.util 11953 :invoke :read [24 nil] INFO [2019-12-17 12:38:06,039] jepsen worker 71 - jepsen.util 13571 :ok :read [23 36] INFO [2019-12-17 12:38:06,040] jepsen worker 71 - jepsen.util 13571 :invoke :write [23 44] INFO [2019-12-17 12:38:06,048] jepsen worker 12 - jepsen.util 12792 :info :write [4 4] :RequestCanceledException INFO [2019-12-17 12:38:06,049] jepsen worker 12 - jepsen.util 12882 :invoke :write [4 35] INFO [2019-12-17 12:38:06,059] jepsen worker 65 - jepsen.util 11135 :ok :read [21 44] INFO [2019-12-17 12:38:06,059] jepsen worker 65 - jepsen.util 11135 :invoke :cas [21 [25 28]] INFO [2019-12-17 12:38:06,061] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73626,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:38:06,061] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73626,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:38:06,061] jepsen worker 77 - jepsen.util 18437 :info :cas [25 [23 36]] :RequestCanceledException INFO [2019-12-17 12:38:06,062] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73629,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:38:06,062] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73629,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:38:06,063] jepsen worker 77 - jepsen.util 18527 :invoke :cas [25 [13 12]] INFO [2019-12-17 12:38:06,063] jepsen worker 5 - jepsen.util 12245 :info :write [1 46] :RequestCanceledException INFO [2019-12-17 12:38:06,065] jepsen worker 5 - jepsen.util 12335 :invoke :read [1 nil] INFO [2019-12-17 12:38:06,066] jepsen worker 73 - jepsen.util 11953 :ok :read [24 8] INFO [2019-12-17 12:38:06,066] jepsen worker 73 - jepsen.util 11953 :invoke :write [24 25] INFO [2019-12-17 12:38:06,077] jepsen worker 18 - jepsen.util 13248 :ok :write [6 40] INFO [2019-12-17 12:38:06,077] jepsen worker 18 - jepsen.util 13248 :invoke :write [6 27] INFO [2019-12-17 12:38:06,091] jepsen worker 65 - jepsen.util 11135 :fail :cas [21 [25 28]] :ValueNotSwapFrom INFO [2019-12-17 12:38:06,091] jepsen worker 65 - jepsen.util 11135 :invoke :cas [21 [32 17]] INFO [2019-12-17 12:38:06,098] jepsen worker 20 - jepsen.util 12710 :fail :cas [6 [44 48]] :ValueNotSwapFrom INFO [2019-12-17 12:38:06,098] jepsen worker 20 - jepsen.util 12710 :invoke :cas [6 [16 41]] INFO [2019-12-17 12:38:06,101] jepsen worker 46 - jepsen.util 12376 :ok :write [15 48] INFO [2019-12-17 12:38:06,101] jepsen worker 2 - jepsen.util 16652 :info :write [0 18] :RequestCanceledException INFO [2019-12-17 12:38:06,101] jepsen worker 46 - jepsen.util 12376 :invoke :write [15 28] INFO [2019-12-17 12:38:06,102] jepsen worker 2 - jepsen.util 16742 :invoke :cas [0 [33 31]] INFO [2019-12-17 12:38:06,103] jepsen worker 9 - jepsen.util 13239 :ok :write [3 7] INFO [2019-12-17 12:38:06,103] jepsen worker 9 - jepsen.util 13239 :invoke :read [3 nil] INFO [2019-12-17 12:38:06,119] jepsen worker 15 - jepsen.util 16485 :info :cas [5 [43 16]] :RequestCanceledException INFO [2019-12-17 12:38:06,120] jepsen worker 15 - jepsen.util 16575 :invoke :cas [5 [29 47]] INFO [2019-12-17 12:38:06,135] jepsen worker 55 - jepsen.util 13375 :info :write [18 24] :RequestCanceledException INFO [2019-12-17 12:38:06,135] jepsen worker 28 - jepsen.util 10828 :info :cas [9 [19 28]] :RequestCanceledException INFO [2019-12-17 12:38:06,135] jepsen worker 35 - jepsen.util 17405 :info :write [11 28] :RequestCanceledException INFO [2019-12-17 12:38:06,135] jepsen worker 47 - jepsen.util 13727 :info :cas [15 [29 18]] :RequestCanceledException INFO [2019-12-17 12:38:06,135] jepsen worker 44 - jepsen.util 18044 :info :cas [14 [27 42]] :RequestCanceledException INFO [2019-12-17 12:38:06,136] jepsen worker 55 - jepsen.util 13465 :invoke :read [18 nil] INFO [2019-12-17 12:38:06,137] jepsen worker 35 - jepsen.util 17495 :invoke :write [11 28] INFO [2019-12-17 12:38:06,137] jepsen worker 47 - jepsen.util 13817 :invoke :cas [15 [41 35]] INFO [2019-12-17 12:38:06,137] jepsen worker 44 - jepsen.util 18134 :invoke :cas [14 [28 45]] INFO [2019-12-17 12:38:06,137] jepsen worker 28 - jepsen.util 10918 :invoke :read [9 nil] INFO [2019-12-17 12:38:06,148] jepsen worker 50 - jepsen.util 17420 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:38:06,148] jepsen worker 16 - jepsen.util 18826 :info :cas [5 [39 44]] :RequestCanceledException INFO [2019-12-17 12:38:06,148] jepsen worker 37 - jepsen.util 12097 :info :write [12 9] :RequestCanceledException INFO [2019-12-17 12:38:06,149] jepsen worker 65 - jepsen.util 11135 :fail :cas [21 [32 17]] :ValueNotSwapFrom INFO [2019-12-17 12:38:06,150] jepsen worker 30 - jepsen.util 12000 :ok :read [10 41] INFO [2019-12-17 12:38:06,151] jepsen worker 50 - jepsen.util 17420 :invoke :write [16 0] INFO [2019-12-17 12:38:06,153] jepsen worker 16 - jepsen.util 18916 :invoke :cas [5 [31 16]] INFO [2019-12-17 12:38:06,155] jepsen worker 65 - jepsen.util 11135 :invoke :write [21 41] INFO [2019-12-17 12:38:06,155] jepsen worker 30 - jepsen.util 12000 :invoke :cas [10 [47 23]] INFO [2019-12-17 12:38:06,164] jepsen worker 37 - jepsen.util 12187 :invoke :write [12 11] INFO [2019-12-17 12:38:06,166] jepsen worker 48 - jepsen.util 16158 :info :cas [16 [33 2]] :RequestCanceledException INFO [2019-12-17 12:38:06,168] jepsen worker 48 - jepsen.util 16248 :invoke :write [16 39] INFO [2019-12-17 12:38:06,201] jepsen worker 10 - jepsen.util 12520 :info :write [3 34] :RequestCanceledException INFO [2019-12-17 12:38:06,201] jepsen worker 29 - jepsen.util 11099 :info :cas [9 [16 13]] :RequestCanceledException INFO [2019-12-17 12:38:06,201] jepsen worker 7 - jepsen.util 17377 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:38:06,205] jepsen worker 10 - jepsen.util 12610 :invoke :read [3 nil] INFO [2019-12-17 12:38:06,207] jepsen worker 7 - jepsen.util 17377 :invoke :cas [2 [2 27]] INFO [2019-12-17 12:38:06,211] jepsen worker 54 - jepsen.util 13284 :ok :read [18 13] INFO [2019-12-17 12:38:06,214] jepsen worker 54 - jepsen.util 13284 :invoke :cas [18 [11 16]] INFO [2019-12-17 12:38:06,215] jepsen worker 29 - jepsen.util 11189 :invoke :write [9 31] INFO [2019-12-17 12:38:06,217] jepsen worker 8 - jepsen.util 16928 :info :cas [2 [17 45]] :RequestCanceledException INFO [2019-12-17 12:38:06,218] jepsen worker 11 - jepsen.util 12521 :ok :read [3 7] INFO [2019-12-17 12:38:06,221] jepsen worker 11 - jepsen.util 12521 :invoke :read [3 nil] INFO [2019-12-17 12:38:06,223] jepsen worker 47 - jepsen.util 13817 :fail :cas [15 [41 35]] :ValueNotSwapFrom INFO [2019-12-17 12:38:06,225] jepsen worker 47 - jepsen.util 13817 :invoke :cas [15 [45 9]] INFO [2019-12-17 12:38:06,225] jepsen worker 45 - jepsen.util 12555 :ok :read [15 48] INFO [2019-12-17 12:38:06,227] jepsen worker 45 - jepsen.util 12555 :invoke :read [15 nil] INFO [2019-12-17 12:38:06,228] jepsen worker 8 - jepsen.util 17018 :invoke :read [2 nil] INFO [2019-12-17 12:38:06,232] jepsen worker 32 - jepsen.util 13172 :fail :cas [10 [5 28]] :ValueNotSwapFrom INFO [2019-12-17 12:38:06,234] jepsen worker 61 - jepsen.util 18601 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:38:06,234] jepsen worker 57 - jepsen.util 17157 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:38:06,234] jepsen worker 81 - jepsen.util 16911 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:38:06,234] jepsen worker 19 - jepsen.util 13249 :info :write [6 26] :RequestCanceledException INFO [2019-12-17 12:38:06,234] jepsen worker 25 - jepsen.util 18475 :info :cas [8 [39 31]] :RequestCanceledException INFO [2019-12-17 12:38:06,234] jepsen worker 58 - jepsen.util 17608 :info :cas [19 [33 40]] :RequestCanceledException INFO [2019-12-17 12:38:06,235] jepsen worker 32 - jepsen.util 13172 :invoke :write [10 13] INFO [2019-12-17 12:38:06,235] jepsen worker 61 - jepsen.util 18601 :invoke :read [20 nil] INFO [2019-12-17 12:38:06,235] jepsen worker 57 - jepsen.util 17157 :invoke :write [19 9] INFO [2019-12-17 12:38:06,235] jepsen worker 81 - jepsen.util 16911 :invoke :cas [27 [41 28]] INFO [2019-12-17 12:38:06,237] jepsen worker 19 - jepsen.util 13339 :invoke :cas [6 [20 4]] INFO [2019-12-17 12:38:06,238] jepsen worker 58 - jepsen.util 17698 :invoke :write [19 43] INFO [2019-12-17 12:38:06,239] jepsen worker 25 - jepsen.util 18565 :invoke :write [8 30] INFO [2019-12-17 12:38:06,243] jepsen worker 31 - jepsen.util 11101 :ok :read [10 41] INFO [2019-12-17 12:38:06,253] jepsen worker 31 - jepsen.util 11101 :invoke :cas [10 [47 18]] INFO [2019-12-17 12:38:06,261] jepsen worker 9 - jepsen.util 13239 :ok :read [3 7] INFO [2019-12-17 12:38:06,265] jepsen worker 43 - jepsen.util 17593 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:38:06,265] jepsen worker 59 - jepsen.util 16529 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:38:06,265] jepsen worker 9 - jepsen.util 13239 :invoke :cas [3 [13 21]] INFO [2019-12-17 12:38:06,265] jepsen worker 43 - jepsen.util 17593 :invoke :write [14 38] INFO [2019-12-17 12:38:06,265] jepsen worker 59 - jepsen.util 16529 :invoke :write [19 31] INFO [2019-12-17 12:38:06,270] jepsen worker 27 - jepsen.util 11907 :ok :cas [9 [25 28]] INFO [2019-12-17 12:38:06,271] jepsen worker 27 - jepsen.util 11907 :invoke :read [9 nil] INFO [2019-12-17 12:38:06,281] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73668,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:38:06,281] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73659,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:38:06,283] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73659,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:38:06,283] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73668,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:38:06,285] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73660,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:38:06,287] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73672,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:38:06,287] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73660,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:38:06,289] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73672,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:38:06,289] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73661,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:38:06,289] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73661,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:38:06,290] jepsen worker 45 - jepsen.util 12555 :ok :read [15 48] INFO [2019-12-17 12:38:06,292] jepsen worker 45 - jepsen.util 12555 :invoke :cas [15 [8 6]] INFO [2019-12-17 12:38:06,335] jepsen worker 19 - jepsen.util 13339 :fail :cas [6 [20 4]] :ValueNotSwapFrom INFO [2019-12-17 12:38:06,336] jepsen worker 19 - jepsen.util 13339 :invoke :cas [6 [29 30]] INFO [2019-12-17 12:38:06,348] jepsen worker 5 - jepsen.util 12335 :ok :read [1 20] INFO [2019-12-17 12:38:06,348] jepsen worker 5 - jepsen.util 12335 :invoke :write [1 18] INFO [2019-12-17 12:38:06,350] jepsen worker 54 - jepsen.util 13284 :fail :cas [18 [11 16]] :ValueNotSwapFrom INFO [2019-12-17 12:38:06,351] jepsen worker 54 - jepsen.util 13284 :invoke :read [18 nil] INFO [2019-12-17 12:38:06,358] jepsen worker 65 - jepsen.util 11135 :ok :write [21 41] INFO [2019-12-17 12:38:06,358] jepsen worker 65 - jepsen.util 11135 :invoke :write [21 6] INFO [2019-12-17 12:38:06,393] jepsen worker 69 - jepsen.util 13029 :ok :write [23 14] INFO [2019-12-17 12:38:06,393] jepsen worker 69 - jepsen.util 13029 :invoke :write [23 46] INFO [2019-12-17 12:38:06,401] jepsen worker 72 - jepsen.util 12762 :info :write [24 47] :RequestCanceledException INFO [2019-12-17 12:38:06,402] jepsen worker 72 - jepsen.util 12852 :invoke :write [24 1] INFO [2019-12-17 12:38:06,406] jepsen worker 52 - jepsen.util 12292 :ok :write [17 44] INFO [2019-12-17 12:38:06,407] jepsen worker 52 - jepsen.util 12292 :invoke :write [17 49] INFO [2019-12-17 12:38:06,415] jepsen worker 45 - jepsen.util 12555 :fail :cas [15 [8 6]] :ValueNotSwapFrom INFO [2019-12-17 12:38:06,415] jepsen worker 45 - jepsen.util 12555 :invoke :read [15 nil] INFO [2019-12-17 12:38:06,419] jepsen worker 31 - jepsen.util 11101 :fail :cas [10 [47 18]] :ValueNotSwapFrom INFO [2019-12-17 12:38:06,419] jepsen worker 31 - jepsen.util 11101 :invoke :cas [10 [20 27]] INFO [2019-12-17 12:38:06,432] jepsen worker 42 - jepsen.util 18312 :info :write [14 37] :RequestCanceledException INFO [2019-12-17 12:38:06,433] jepsen worker 42 - jepsen.util 18402 :invoke :read [14 nil] INFO [2019-12-17 12:38:06,433] jepsen worker 19 - jepsen.util 13339 :fail :cas [6 [29 30]] :ValueNotSwapFrom INFO [2019-12-17 12:38:06,434] jepsen worker 19 - jepsen.util 13339 :invoke :read [6 nil] INFO [2019-12-17 12:38:06,454] jepsen worker 38 - jepsen.util 13358 :ok :write [12 29] INFO [2019-12-17 12:38:06,454] jepsen worker 38 - jepsen.util 13358 :invoke :write [12 39] INFO [2019-12-17 12:38:06,465] jepsen worker 83 - jepsen.util 17363 :info :cas [27 [5 34]] :RequestCanceledException INFO [2019-12-17 12:38:06,491] jepsen worker 36 - jepsen.util 13536 :ok :write [12 11] INFO [2019-12-17 12:38:06,491] jepsen worker 36 - jepsen.util 13536 :invoke :cas [12 [32 7]] INFO [2019-12-17 12:38:06,500] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73701,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:38:06,500] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73701,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:38:06,500] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73704,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:38:06,500] jepsen worker 14 - jepsen.util 12344 :ok :write [4 0] INFO [2019-12-17 12:38:06,500] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73704,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:38:06,501] jepsen worker 14 - jepsen.util 12344 :invoke :write [4 13] INFO [2019-12-17 12:38:06,501] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73705,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:38:06,501] jepsen worker 39 - jepsen.util 16959 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:38:06,501] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73705,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:38:06,501] jepsen worker 39 - jepsen.util 16959 :invoke :read [13 nil] INFO [2019-12-17 12:38:06,502] jepsen worker 45 - jepsen.util 12555 :ok :read [15 48] INFO [2019-12-17 12:38:06,502] jepsen worker 45 - jepsen.util 12555 :invoke :cas [15 [27 7]] INFO [2019-12-17 12:38:06,503] jepsen worker 31 - jepsen.util 11101 :fail :cas [10 [20 27]] :ValueNotSwapFrom INFO [2019-12-17 12:38:06,504] jepsen worker 31 - jepsen.util 11101 :invoke :cas [10 [22 3]] INFO [2019-12-17 12:38:06,504] jepsen worker 83 - jepsen.util 17453 :invoke :cas [27 [23 35]] INFO [2019-12-17 12:38:06,527] jepsen worker 19 - jepsen.util 13339 :ok :read [6 40] INFO [2019-12-17 12:38:06,528] jepsen worker 19 - jepsen.util 13339 :invoke :read [6 nil] INFO [2019-12-17 12:38:06,535] jepsen worker 1 - jepsen.util 17641 :info :cas [0 [5 13]] :RequestCanceledException INFO [2019-12-17 12:38:06,535] jepsen worker 1 - jepsen.util 17731 :invoke :write [0 1] INFO [2019-12-17 12:38:06,566] jepsen worker 36 - jepsen.util 13536 :fail :cas [12 [32 7]] :ValueNotSwapFrom INFO [2019-12-17 12:38:06,566] jepsen worker 36 - jepsen.util 13536 :invoke :cas [12 [33 6]] INFO [2019-12-17 12:38:06,566] jepsen worker 22 - jepsen.util 17572 :info :write [7 36] :RequestCanceledException INFO [2019-12-17 12:38:06,567] jepsen worker 22 - jepsen.util 17662 :invoke :write [7 22] INFO [2019-12-17 12:38:06,568] jepsen worker 45 - jepsen.util 12555 :fail :cas [15 [27 7]] :ValueNotSwapFrom INFO [2019-12-17 12:38:06,568] jepsen worker 45 - jepsen.util 12555 :invoke :read [15 nil] INFO [2019-12-17 12:38:06,572] jepsen worker 4 - jepsen.util 11974 :ok :write [1 36] INFO [2019-12-17 12:38:06,572] jepsen worker 4 - jepsen.util 11974 :invoke :write [1 31] INFO [2019-12-17 12:38:06,591] jepsen worker 65 - jepsen.util 11135 :ok :write [21 6] INFO [2019-12-17 12:38:06,591] jepsen worker 65 - jepsen.util 11135 :invoke :read [21 nil] INFO [2019-12-17 12:38:06,601] jepsen worker 6 - jepsen.util 17826 :info :write [2 15] :RequestCanceledException INFO [2019-12-17 12:38:06,615] jepsen worker 6 - jepsen.util 17916 :invoke :write [2 5] INFO [2019-12-17 12:38:06,619] jepsen worker 76 - jepsen.util 17446 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:38:06,619] jepsen worker 76 - jepsen.util 17446 :invoke :cas [25 [31 15]] INFO [2019-12-17 12:38:06,622] jepsen worker 87 - jepsen.util 12237 :ok :write [29 43] INFO [2019-12-17 12:38:06,622] jepsen worker 87 - jepsen.util 12237 :invoke :read [29 nil] INFO [2019-12-17 12:38:06,631] jepsen worker 31 - jepsen.util 11101 :fail :cas [10 [22 3]] :ValueNotSwapFrom INFO [2019-12-17 12:38:06,631] jepsen worker 31 - jepsen.util 11101 :invoke :read [10 nil] INFO [2019-12-17 12:38:06,634] jepsen worker 80 - jepsen.util 12140 :info :write [26 1] :RequestCanceledException INFO [2019-12-17 12:38:06,635] jepsen worker 80 - jepsen.util 12230 :invoke :read [26 nil] INFO [2019-12-17 12:38:06,638] jepsen worker 29 - jepsen.util 11189 :ok :write [9 31] INFO [2019-12-17 12:38:06,639] jepsen worker 29 - jepsen.util 11189 :invoke :read [9 nil] INFO [2019-12-17 12:38:06,656] jepsen worker 18 - jepsen.util 13248 :ok :write [6 27] INFO [2019-12-17 12:38:06,656] jepsen worker 18 - jepsen.util 13248 :invoke :read [6 nil] INFO [2019-12-17 12:38:06,657] jepsen worker 73 - jepsen.util 11953 :ok :write [24 25] INFO [2019-12-17 12:38:06,657] jepsen worker 73 - jepsen.util 11953 :invoke :read [24 nil] INFO [2019-12-17 12:38:06,669] jepsen worker 54 - jepsen.util 13284 :ok :read [18 13] INFO [2019-12-17 12:38:06,670] jepsen worker 54 - jepsen.util 13284 :invoke :cas [18 [26 11]] INFO [2019-12-17 12:38:06,674] jepsen worker 29 - jepsen.util 11189 :ok :read [9 31] INFO [2019-12-17 12:38:06,674] jepsen worker 29 - jepsen.util 11189 :invoke :cas [9 [2 0]] INFO [2019-12-17 12:38:06,700] jepsen worker 14 - jepsen.util 12344 :ok :write [4 13] INFO [2019-12-17 12:38:06,701] jepsen worker 14 - jepsen.util 12344 :invoke :write [4 45] INFO [2019-12-17 12:38:06,701] jepsen worker 63 - jepsen.util 11313 :info :write [21 3] :RequestCanceledException INFO [2019-12-17 12:38:06,702] jepsen worker 63 - jepsen.util 11403 :invoke :cas [21 [46 20]] INFO [2019-12-17 12:38:06,706] jepsen worker 88 - jepsen.util 12868 :fail :cas [29 [25 1]] :ValueNotSwapFrom INFO [2019-12-17 12:38:06,706] jepsen worker 88 - jepsen.util 12868 :invoke :cas [29 [37 38]] INFO [2019-12-17 12:38:06,711] jepsen worker 87 - jepsen.util 12237 :ok :read [29 43] INFO [2019-12-17 12:38:06,712] jepsen worker 87 - jepsen.util 12237 :invoke :write [29 6] INFO [2019-12-17 12:38:06,714] jepsen worker 54 - jepsen.util 13284 :fail :cas [18 [26 11]] :ValueNotSwapFrom INFO [2019-12-17 12:38:06,714] jepsen worker 54 - jepsen.util 13284 :invoke :read [18 nil] INFO [2019-12-17 12:38:06,717] jepsen worker 68 - jepsen.util 16538 :info :cas [22 [48 44]] :RequestCanceledException INFO [2019-12-17 12:38:06,718] jepsen worker 68 - jepsen.util 16628 :invoke :cas [22 [48 45]] INFO [2019-12-17 12:38:06,727] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73730,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:38:06,727] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73730,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:38:06,727] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73734,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:38:06,728] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73734,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:38:06,743] jepsen worker 28 - jepsen.util 10918 :ok :read [9 31] INFO [2019-12-17 12:38:06,743] jepsen worker 28 - jepsen.util 10918 :invoke :cas [9 [28 21]] INFO [2019-12-17 12:38:06,761] jepsen worker 19 - jepsen.util 13339 :ok :read [6 40] INFO [2019-12-17 12:38:06,761] jepsen worker 19 - jepsen.util 13339 :invoke :read [6 nil] INFO [2019-12-17 12:38:06,764] jepsen worker 89 - jepsen.util 12599 :info :write [29 29] :RequestCanceledException INFO [2019-12-17 12:38:06,765] jepsen worker 89 - jepsen.util 12689 :invoke :read [29 nil] INFO [2019-12-17 12:38:06,768] jepsen worker 56 - jepsen.util 13466 :ok :write [18 22] INFO [2019-12-17 12:38:06,768] jepsen worker 56 - jepsen.util 13466 :invoke :write [18 23] INFO [2019-12-17 12:38:06,819] jepsen worker 49 - jepsen.util 17329 :info :cas [16 [41 44]] :RequestCanceledException INFO [2019-12-17 12:38:06,820] jepsen worker 49 - jepsen.util 17419 :invoke :read [16 nil] INFO [2019-12-17 12:38:06,833] jepsen worker 38 - jepsen.util 13358 :ok :write [12 39] INFO [2019-12-17 12:38:06,833] jepsen worker 38 - jepsen.util 13358 :invoke :read [12 nil] INFO [2019-12-17 12:38:06,834] jepsen worker 34 - jepsen.util 18304 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:38:06,835] jepsen worker 34 - jepsen.util 18304 :invoke :write [11 14] INFO [2019-12-17 12:38:06,838] jepsen worker 54 - jepsen.util 13284 :ok :read [18 22] INFO [2019-12-17 12:38:06,838] jepsen worker 54 - jepsen.util 13284 :invoke :cas [18 [24 3]] INFO [2019-12-17 12:38:06,848] jepsen worker 53 - jepsen.util 12203 :info :write [17 38] :RequestCanceledException INFO [2019-12-17 12:38:06,848] jepsen worker 24 - jepsen.util 16494 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:38:06,848] jepsen worker 24 - jepsen.util 16494 :invoke :read [8 nil] INFO [2019-12-17 12:38:06,849] jepsen worker 53 - jepsen.util 12293 :invoke :read [17 nil] INFO [2019-12-17 12:38:06,865] jepsen worker 40 - jepsen.util 18130 :info :cas [13 [21 39]] :RequestCanceledException INFO [2019-12-17 12:38:06,865] jepsen worker 41 - jepsen.util 18761 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:38:06,865] jepsen worker 41 - jepsen.util 18761 :invoke :write [13 46] INFO [2019-12-17 12:38:06,866] jepsen worker 40 - jepsen.util 18220 :invoke :read [13 nil] INFO [2019-12-17 12:38:06,868] jepsen worker 30 - jepsen.util 12000 :fail :cas [10 [47 23]] :ValueNotSwapFrom INFO [2019-12-17 12:38:06,868] jepsen worker 30 - jepsen.util 12000 :invoke :write [10 7] INFO [2019-12-17 12:38:06,869] jepsen worker 55 - jepsen.util 13465 :ok :read [18 22] INFO [2019-12-17 12:38:06,869] jepsen worker 55 - jepsen.util 13465 :invoke :cas [18 [9 37]] INFO [2019-12-17 12:38:06,871] jepsen worker 32 - jepsen.util 13172 :ok :write [10 13] INFO [2019-12-17 12:38:06,871] jepsen worker 32 - jepsen.util 13172 :invoke :write [10 24] INFO [2019-12-17 12:38:06,886] jepsen worker 73 - jepsen.util 11953 :ok :read [24 25] INFO [2019-12-17 12:38:06,886] jepsen worker 73 - jepsen.util 11953 :invoke :read [24 nil] INFO [2019-12-17 12:38:06,905] jepsen worker 29 - jepsen.util 11189 :fail :cas [9 [2 0]] :ValueNotSwapFrom INFO [2019-12-17 12:38:06,905] jepsen worker 29 - jepsen.util 11189 :invoke :cas [9 [33 43]] INFO [2019-12-17 12:38:06,911] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:38:06,911] jepsen nemesis - couchbase.util Rebalance check count: 581 INFO [2019-12-17 12:38:06,911] jepsen nemesis - couchbase.util Rebalance stuck count: 562 INFO [2019-12-17 12:38:06,921] jepsen worker 45 - jepsen.util 12555 :ok :read [15 48] INFO [2019-12-17 12:38:06,922] jepsen worker 45 - jepsen.util 12555 :invoke :read [15 nil] INFO [2019-12-17 12:38:06,936] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73757,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:38:06,936] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73757,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:38:06,936] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73761,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:38:06,936] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73761,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:38:06,948] jepsen worker 87 - jepsen.util 12237 :ok :write [29 6] INFO [2019-12-17 12:38:06,948] jepsen worker 87 - jepsen.util 12237 :invoke :write [29 31] INFO [2019-12-17 12:38:06,954] jepsen worker 11 - jepsen.util 12521 :ok :read [3 7] INFO [2019-12-17 12:38:06,954] jepsen worker 11 - jepsen.util 12521 :invoke :write [3 27] INFO [2019-12-17 12:38:06,959] jepsen worker 63 - jepsen.util 11403 :fail :cas [21 [46 20]] :ValueNotSwapFrom INFO [2019-12-17 12:38:06,959] jepsen worker 63 - jepsen.util 11403 :invoke :cas [21 [29 3]] INFO [2019-12-17 12:38:06,969] jepsen worker 73 - jepsen.util 11953 :ok :read [24 25] INFO [2019-12-17 12:38:06,970] jepsen worker 73 - jepsen.util 11953 :invoke :write [24 22] INFO [2019-12-17 12:38:06,986] jepsen worker 31 - jepsen.util 11101 :ok :read [10 13] INFO [2019-12-17 12:38:06,986] jepsen worker 31 - jepsen.util 11101 :invoke :write [10 3] INFO [2019-12-17 12:38:07,000] jepsen worker 54 - jepsen.util 13284 :fail :cas [18 [24 3]] :ValueNotSwapFrom INFO [2019-12-17 12:38:07,000] jepsen worker 54 - jepsen.util 13284 :invoke :write [18 19] INFO [2019-12-17 12:38:07,007] jepsen worker 18 - jepsen.util 13248 :ok :read [6 27] INFO [2019-12-17 12:38:07,008] jepsen worker 18 - jepsen.util 13248 :invoke :cas [6 [28 18]] INFO [2019-12-17 12:38:07,025] jepsen worker 89 - jepsen.util 12689 :ok :read [29 6] INFO [2019-12-17 12:38:07,026] jepsen worker 89 - jepsen.util 12689 :invoke :write [29 1] INFO [2019-12-17 12:38:07,035] jepsen worker 3 - jepsen.util 12153 :info :write [1 5] :RequestCanceledException INFO [2019-12-17 12:38:07,035] jepsen worker 62 - jepsen.util 18152 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:38:07,035] jepsen worker 62 - jepsen.util 18152 :invoke :cas [20 [16 44]] INFO [2019-12-17 12:38:07,036] jepsen worker 3 - jepsen.util 12243 :invoke :write [1 49] INFO [2019-12-17 12:38:07,045] jepsen worker 29 - jepsen.util 11189 :fail :cas [9 [33 43]] :ValueNotSwapFrom INFO [2019-12-17 12:38:07,045] jepsen worker 29 - jepsen.util 11189 :invoke :read [9 nil] INFO [2019-12-17 12:38:07,046] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73775,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:38:07,046] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73775,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:38:07,046] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73778,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:38:07,046] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73778,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:38:07,046] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73779,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:38:07,046] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73779,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:38:07,048] jepsen worker 86 - jepsen.util 18176 :info :cas [28 [4 36]] :RequestCanceledException INFO [2019-12-17 12:38:07,050] jepsen worker 86 - jepsen.util 18266 :invoke :read [28 nil] INFO [2019-12-17 12:38:07,058] jepsen worker 19 - jepsen.util 13339 :ok :read [6 27] INFO [2019-12-17 12:38:07,058] jepsen worker 19 - jepsen.util 13339 :invoke :cas [6 [2 39]] INFO [2019-12-17 12:38:07,072] jepsen worker 28 - jepsen.util 10918 :fail :cas [9 [28 21]] :ValueNotSwapFrom INFO [2019-12-17 12:38:07,072] jepsen worker 28 - jepsen.util 10918 :invoke :cas [9 [2 39]] INFO [2019-12-17 12:38:07,076] jepsen worker 71 - jepsen.util 13571 :ok :write [23 44] INFO [2019-12-17 12:38:07,077] jepsen worker 71 - jepsen.util 13571 :invoke :write [23 27] INFO [2019-12-17 12:38:07,095] jepsen worker 63 - jepsen.util 11403 :fail :cas [21 [29 3]] :ValueNotSwapFrom INFO [2019-12-17 12:38:07,095] jepsen worker 63 - jepsen.util 11403 :invoke :read [21 nil] INFO [2019-12-17 12:38:07,113] jepsen worker 45 - jepsen.util 12555 :ok :read [15 48] INFO [2019-12-17 12:38:07,114] jepsen worker 45 - jepsen.util 12555 :invoke :read [15 nil] INFO [2019-12-17 12:38:07,117] jepsen worker 29 - jepsen.util 11189 :ok :read [9 31] INFO [2019-12-17 12:38:07,117] jepsen worker 29 - jepsen.util 11189 :invoke :cas [9 [31 39]] INFO [2019-12-17 12:38:07,119] jepsen worker 64 - jepsen.util 9874 :info :write [21 28] :RequestCanceledException INFO [2019-12-17 12:38:07,122] jepsen worker 64 - jepsen.util 9964 :invoke :read [21 nil] INFO [2019-12-17 12:38:07,142] jepsen worker 38 - jepsen.util 13358 :ok :read [12 39] INFO [2019-12-17 12:38:07,142] jepsen worker 38 - jepsen.util 13358 :invoke :cas [12 [33 6]] INFO [2019-12-17 12:38:07,145] jepsen worker 63 - jepsen.util 11403 :ok :read [21 6] INFO [2019-12-17 12:38:07,148] jepsen worker 63 - jepsen.util 11403 :invoke :cas [21 [46 46]] INFO [2019-12-17 12:38:07,148] jepsen worker 17 - jepsen.util 17387 :info :write [5 15] :RequestCanceledException INFO [2019-12-17 12:38:07,149] jepsen worker 17 - jepsen.util 17477 :invoke :write [5 47] INFO [2019-12-17 12:38:07,165] jepsen worker 75 - jepsen.util 16905 :info :cas [25 [23 13]] :RequestCanceledException INFO [2019-12-17 12:38:07,166] jepsen worker 75 - jepsen.util 16995 :invoke :write [25 2] INFO [2019-12-17 12:38:07,230] jepsen worker 73 - jepsen.util 11953 :ok :write [24 22] INFO [2019-12-17 12:38:07,230] jepsen worker 73 - jepsen.util 11953 :invoke :read [24 nil] INFO [2019-12-17 12:38:07,232] jepsen worker 63 - jepsen.util 11403 :fail :cas [21 [46 46]] :ValueNotSwapFrom INFO [2019-12-17 12:38:07,232] jepsen worker 63 - jepsen.util 11403 :invoke :read [21 nil] INFO [2019-12-17 12:38:07,234] jepsen worker 60 - jepsen.util 18330 :info :cas [20 [6 1]] :RequestCanceledException INFO [2019-12-17 12:38:07,234] jepsen worker 70 - jepsen.util 13120 :info :write [23 35] :RequestCanceledException INFO [2019-12-17 12:38:07,234] jepsen worker 26 - jepsen.util 18206 :info :cas [8 [20 24]] :RequestCanceledException INFO [2019-12-17 12:38:07,236] jepsen worker 26 - jepsen.util 18296 :invoke :write [8 22] INFO [2019-12-17 12:38:07,236] jepsen worker 60 - jepsen.util 18420 :invoke :cas [20 [23 26]] INFO [2019-12-17 12:38:07,236] jepsen worker 70 - jepsen.util 13210 :invoke :write [23 33] INFO [2019-12-17 12:38:07,243] jepsen worker 55 - jepsen.util 13465 :fail :cas [18 [9 37]] :ValueNotSwapFrom INFO [2019-12-17 12:38:07,243] jepsen worker 55 - jepsen.util 13465 :invoke :cas [18 [49 33]] INFO [2019-12-17 12:38:07,267] jepsen worker 73 - jepsen.util 11953 :ok :read [24 22] INFO [2019-12-17 12:38:07,267] jepsen worker 73 - jepsen.util 11953 :invoke :cas [24 [45 20]] INFO [2019-12-17 12:38:07,277] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73805,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:38:07,277] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73799,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:38:07,277] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73805,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:38:07,277] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73799,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:38:07,277] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73806,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:38:07,277] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73806,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:38:07,277] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73807,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:38:07,277] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73807,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:38:07,284] jepsen worker 73 - jepsen.util 11953 :fail :cas [24 [45 20]] :ValueNotSwapFrom INFO [2019-12-17 12:38:07,284] jepsen worker 73 - jepsen.util 11953 :invoke :read [24 nil] INFO [2019-12-17 12:38:07,305] jepsen worker 38 - jepsen.util 13358 :fail :cas [12 [33 6]] :ValueNotSwapFrom INFO [2019-12-17 12:38:07,305] jepsen worker 38 - jepsen.util 13358 :invoke :read [12 nil] INFO [2019-12-17 12:38:07,313] jepsen worker 73 - jepsen.util 11953 :ok :read [24 22] INFO [2019-12-17 12:38:07,313] jepsen worker 73 - jepsen.util 11953 :invoke :write [24 24] INFO [2019-12-17 12:38:07,334] jepsen worker 66 - jepsen.util 17526 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:38:07,334] jepsen worker 66 - jepsen.util 17526 :invoke :read [22 nil] INFO [2019-12-17 12:38:07,348] jepsen worker 56 - jepsen.util 13466 :ok :write [18 23] INFO [2019-12-17 12:38:07,348] jepsen worker 56 - jepsen.util 13466 :invoke :read [18 nil] INFO [2019-12-17 12:38:07,349] jepsen worker 53 - jepsen.util 12293 :ok :read [17 44] INFO [2019-12-17 12:38:07,349] jepsen worker 53 - jepsen.util 12293 :invoke :read [17 nil] INFO [2019-12-17 12:38:07,360] jepsen worker 31 - jepsen.util 11101 :ok :write [10 3] INFO [2019-12-17 12:38:07,360] jepsen worker 32 - jepsen.util 13172 :ok :write [10 24] INFO [2019-12-17 12:38:07,360] jepsen worker 31 - jepsen.util 11101 :invoke :cas [10 [31 36]] INFO [2019-12-17 12:38:07,360] jepsen worker 32 - jepsen.util 13172 :invoke :write [10 20] INFO [2019-12-17 12:38:07,365] jepsen worker 36 - jepsen.util 13536 :fail :cas [12 [33 6]] :ValueNotSwapFrom INFO [2019-12-17 12:38:07,365] jepsen worker 36 - jepsen.util 13536 :invoke :write [12 9] INFO [2019-12-17 12:38:07,419] jepsen worker 85 - jepsen.util 17995 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:38:07,419] jepsen worker 67 - jepsen.util 17617 :info :cas [22 [37 22]] :RequestCanceledException INFO [2019-12-17 12:38:07,419] jepsen worker 85 - jepsen.util 17995 :invoke :write [28 2] INFO [2019-12-17 12:38:07,423] jepsen worker 67 - jepsen.util 17707 :invoke :cas [22 [21 12]] INFO [2019-12-17 12:38:07,429] jepsen worker 55 - jepsen.util 13465 :fail :cas [18 [49 33]] :ValueNotSwapFrom INFO [2019-12-17 12:38:07,429] jepsen worker 55 - jepsen.util 13465 :invoke :write [18 43] INFO [2019-12-17 12:38:07,432] jepsen worker 84 - jepsen.util 16914 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:38:07,432] jepsen worker 84 - jepsen.util 16914 :invoke :write [28 47] INFO [2019-12-17 12:38:07,447] jepsen worker 29 - jepsen.util 11189 :ok :cas [9 [31 39]] INFO [2019-12-17 12:38:07,448] jepsen worker 29 - jepsen.util 11189 :invoke :cas [9 [35 48]] INFO [2019-12-17 12:38:07,448] jepsen worker 33 - jepsen.util 17313 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:38:07,448] jepsen worker 33 - jepsen.util 17313 :invoke :write [11 8] INFO [2019-12-17 12:38:07,455] jepsen worker 31 - jepsen.util 11101 :fail :cas [10 [31 36]] :ValueNotSwapFrom INFO [2019-12-17 12:38:07,455] jepsen worker 31 - jepsen.util 11101 :invoke :cas [10 [12 28]] INFO [2019-12-17 12:38:07,468] jepsen worker 29 - jepsen.util 11189 :fail :cas [9 [35 48]] :ValueNotSwapFrom INFO [2019-12-17 12:38:07,469] jepsen worker 29 - jepsen.util 11189 :invoke :write [9 41] INFO [2019-12-17 12:38:07,472] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73831,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:38:07,472] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73831,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:38:07,472] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73834,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:38:07,472] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73834,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:38:07,501] jepsen worker 13 - jepsen.util 12253 :info :write [4 32] :RequestCanceledException INFO [2019-12-17 12:38:07,502] jepsen worker 13 - jepsen.util 12343 :invoke :write [4 22] INFO [2019-12-17 12:38:07,503] jepsen worker 31 - jepsen.util 11101 :fail :cas [10 [12 28]] :ValueNotSwapFrom INFO [2019-12-17 12:38:07,503] jepsen worker 31 - jepsen.util 11101 :invoke :read [10 nil] INFO [2019-12-17 12:38:07,508] jepsen worker 80 - jepsen.util 12230 :ok :read [26 18] INFO [2019-12-17 12:38:07,508] jepsen worker 80 - jepsen.util 12230 :invoke :cas [26 [3 44]] INFO [2019-12-17 12:38:07,519] jepsen worker 23 - jepsen.util 17843 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:38:07,519] jepsen worker 23 - jepsen.util 17843 :invoke :cas [7 [23 46]] INFO [2019-12-17 12:38:07,525] jepsen worker 88 - jepsen.util 12868 :fail :cas [29 [37 38]] :ValueNotSwapFrom INFO [2019-12-17 12:38:07,525] jepsen worker 88 - jepsen.util 12868 :invoke :read [29 nil] INFO [2019-12-17 12:38:07,566] jepsen worker 79 - jepsen.util 12949 :info :write [26 6] :RequestCanceledException INFO [2019-12-17 12:38:07,567] jepsen worker 79 - jepsen.util 13039 :invoke :read [26 nil] INFO [2019-12-17 12:38:07,586] jepsen worker 31 - jepsen.util 11101 :ok :read [10 3] INFO [2019-12-17 12:38:07,586] jepsen worker 31 - jepsen.util 11101 :invoke :read [10 nil] INFO [2019-12-17 12:38:07,602] jepsen worker 21 - jepsen.util 16941 :info :cas [7 [5 18]] :RequestCanceledException INFO [2019-12-17 12:38:07,602] jepsen worker 29 - jepsen.util 11189 :ok :write [9 41] INFO [2019-12-17 12:38:07,602] jepsen worker 29 - jepsen.util 11189 :invoke :read [9 nil] INFO [2019-12-17 12:38:07,603] jepsen worker 21 - jepsen.util 17031 :invoke :cas [7 [40 16]] INFO [2019-12-17 12:38:07,613] jepsen worker 89 - jepsen.util 12689 :ok :write [29 1] INFO [2019-12-17 12:38:07,613] jepsen worker 89 - jepsen.util 12689 :invoke :cas [29 [47 26]] INFO [2019-12-17 12:38:07,616] jepsen worker 0 - jepsen.util 17820 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:38:07,616] jepsen worker 0 - jepsen.util 17820 :invoke :write [0 22] INFO [2019-12-17 12:38:07,622] jepsen worker 73 - jepsen.util 11953 :ok :write [24 24] INFO [2019-12-17 12:38:07,622] jepsen worker 73 - jepsen.util 11953 :invoke :read [24 nil] INFO [2019-12-17 12:38:07,624] jepsen worker 37 - jepsen.util 12187 :ok :write [12 11] INFO [2019-12-17 12:38:07,624] jepsen worker 37 - jepsen.util 12187 :invoke :write [12 0] INFO [2019-12-17 12:38:07,624] jepsen worker 31 - jepsen.util 11101 :ok :read [10 3] INFO [2019-12-17 12:38:07,624] jepsen worker 31 - jepsen.util 11101 :invoke :cas [10 [2 38]] INFO [2019-12-17 12:38:07,635] jepsen worker 82 - jepsen.util 17722 :info :write [27 26] :RequestCanceledException INFO [2019-12-17 12:38:07,636] jepsen worker 82 - jepsen.util 17812 :invoke :read [27 nil] INFO [2019-12-17 12:38:07,637] jepsen worker 78 - jepsen.util 12408 :ok :write [26 7] INFO [2019-12-17 12:38:07,637] jepsen worker 78 - jepsen.util 12408 :invoke :write [26 48] INFO [2019-12-17 12:38:07,642] jepsen worker 38 - jepsen.util 13358 :ok :read [12 11] INFO [2019-12-17 12:38:07,643] jepsen worker 38 - jepsen.util 13358 :invoke :write [12 45] INFO [2019-12-17 12:38:07,675] jepsen worker 29 - jepsen.util 11189 :ok :read [9 41] INFO [2019-12-17 12:38:07,675] jepsen worker 29 - jepsen.util 11189 :invoke :write [9 47] INFO [2019-12-17 12:38:07,712] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73860,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:38:07,712] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73860,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:38:07,718] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73862,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:38:07,719] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73862,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:38:07,741] jepsen worker 31 - jepsen.util 11101 :fail :cas [10 [2 38]] :ValueNotSwapFrom INFO [2019-12-17 12:38:07,741] jepsen worker 31 - jepsen.util 11101 :invoke :write [10 22] INFO [2019-12-17 12:38:07,743] jepsen worker 73 - jepsen.util 11953 :ok :read [24 24] INFO [2019-12-17 12:38:07,743] jepsen worker 73 - jepsen.util 11953 :invoke :cas [24 [35 30]] INFO [2019-12-17 12:38:07,761] jepsen worker 51 - jepsen.util 11931 :info :write [17 30] :RequestCanceledException INFO [2019-12-17 12:38:07,763] jepsen worker 51 - jepsen.util 12021 :invoke :write [17 20] INFO [2019-12-17 12:38:07,773] jepsen worker 73 - jepsen.util 11953 :fail :cas [24 [35 30]] :ValueNotSwapFrom INFO [2019-12-17 12:38:07,773] jepsen worker 73 - jepsen.util 11953 :invoke :cas [24 [34 16]] INFO [2019-12-17 12:38:07,796] jepsen worker 32 - jepsen.util 13172 :ok :write [10 20] INFO [2019-12-17 12:38:07,796] jepsen worker 32 - jepsen.util 13172 :invoke :read [10 nil] INFO [2019-12-17 12:38:07,800] jepsen worker 89 - jepsen.util 12689 :fail :cas [29 [47 26]] :ValueNotSwapFrom INFO [2019-12-17 12:38:07,800] jepsen worker 89 - jepsen.util 12689 :invoke :write [29 36] INFO [2019-12-17 12:38:07,811] jepsen worker 73 - jepsen.util 11953 :fail :cas [24 [34 16]] :ValueNotSwapFrom INFO [2019-12-17 12:38:07,811] jepsen worker 73 - jepsen.util 11953 :invoke :cas [24 [34 16]] INFO [2019-12-17 12:38:07,867] jepsen worker 4 - jepsen.util 11974 :ok :write [1 31] INFO [2019-12-17 12:38:07,868] jepsen worker 4 - jepsen.util 11974 :invoke :cas [1 [25 2]] INFO [2019-12-17 12:38:07,889] jepsen worker 80 - jepsen.util 12230 :fail :cas [26 [3 44]] :ValueNotSwapFrom INFO [2019-12-17 12:38:07,889] jepsen worker 80 - jepsen.util 12230 :invoke :cas [26 [15 31]] INFO [2019-12-17 12:38:07,899] jepsen worker 73 - jepsen.util 11953 :fail :cas [24 [34 16]] :ValueNotSwapFrom INFO [2019-12-17 12:38:07,899] jepsen worker 73 - jepsen.util 11953 :invoke :read [24 nil] INFO [2019-12-17 12:38:07,908] jepsen worker 32 - jepsen.util 13172 :ok :read [10 20] INFO [2019-12-17 12:38:07,908] jepsen worker 32 - jepsen.util 13172 :invoke :cas [10 [5 23]] INFO [2019-12-17 12:38:07,910] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73873,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:38:07,910] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73873,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:38:07,910] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73876,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:38:07,911] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73876,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:38:07,911] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73877,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:38:07,911] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73877,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:38:07,928] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:38:07,928] jepsen nemesis - couchbase.util Rebalance check count: 582 INFO [2019-12-17 12:38:07,928] jepsen nemesis - couchbase.util Rebalance stuck count: 563 INFO [2019-12-17 12:38:07,931] jepsen worker 87 - jepsen.util 12237 :ok :write [29 31] INFO [2019-12-17 12:38:07,931] jepsen worker 87 - jepsen.util 12237 :invoke :write [29 17] INFO [2019-12-17 12:38:07,938] jepsen worker 73 - jepsen.util 11953 :ok :read [24 24] INFO [2019-12-17 12:38:07,938] jepsen worker 73 - jepsen.util 11953 :invoke :read [24 nil] INFO [2019-12-17 12:38:07,942] jepsen worker 29 - jepsen.util 11189 :ok :write [9 47] INFO [2019-12-17 12:38:07,942] jepsen worker 29 - jepsen.util 11189 :invoke :write [9 7] INFO [2019-12-17 12:38:07,955] jepsen worker 32 - jepsen.util 13172 :fail :cas [10 [5 23]] :ValueNotSwapFrom INFO [2019-12-17 12:38:07,955] jepsen worker 32 - jepsen.util 13172 :invoke :read [10 nil] INFO [2019-12-17 12:38:07,957] jepsen worker 73 - jepsen.util 11953 :ok :read [24 24] INFO [2019-12-17 12:38:07,957] jepsen worker 73 - jepsen.util 11953 :invoke :cas [24 [41 31]] INFO [2019-12-17 12:38:07,970] jepsen worker 56 - jepsen.util 13466 :ok :read [18 23] INFO [2019-12-17 12:38:07,970] jepsen worker 56 - jepsen.util 13466 :invoke :read [18 nil] INFO [2019-12-17 12:38:07,999] jepsen worker 73 - jepsen.util 11953 :fail :cas [24 [41 31]] :ValueNotSwapFrom INFO [2019-12-17 12:38:07,999] jepsen worker 73 - jepsen.util 11953 :invoke :write [24 19] INFO [2019-12-17 12:38:08,006] jepsen worker 32 - jepsen.util 13172 :ok :read [10 20] INFO [2019-12-17 12:38:08,006] jepsen worker 32 - jepsen.util 13172 :invoke :read [10 nil] INFO [2019-12-17 12:38:08,026] jepsen worker 14 - jepsen.util 12344 :ok :write [4 45] INFO [2019-12-17 12:38:08,026] jepsen worker 14 - jepsen.util 12344 :invoke :write [4 9] INFO [2019-12-17 12:38:08,031] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73888,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:38:08,031] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73888,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:38:08,031] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73889,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:38:08,031] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73889,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:38:08,048] jepsen worker 74 - jepsen.util 12584 :info :write [24 0] :RequestCanceledException INFO [2019-12-17 12:38:08,049] jepsen worker 74 - jepsen.util 12674 :invoke :cas [24 [36 40]] INFO [2019-12-17 12:38:08,071] jepsen worker 88 - jepsen.util 12868 :ok :read [29 1] INFO [2019-12-17 12:38:08,071] jepsen worker 88 - jepsen.util 12868 :invoke :cas [29 [11 44]] INFO [2019-12-17 12:38:08,097] jepsen worker 29 - jepsen.util 11189 :ok :write [9 7] INFO [2019-12-17 12:38:08,097] jepsen worker 29 - jepsen.util 11189 :invoke :cas [9 [46 45]] INFO [2019-12-17 12:38:08,097] jepsen worker 31 - jepsen.util 11101 :ok :write [10 22] INFO [2019-12-17 12:38:08,097] jepsen worker 31 - jepsen.util 11101 :invoke :read [10 nil] INFO [2019-12-17 12:38:08,097] jepsen worker 73 - jepsen.util 11953 :ok :write [24 19] INFO [2019-12-17 12:38:08,097] jepsen worker 73 - jepsen.util 11953 :invoke :read [24 nil] INFO [2019-12-17 12:38:08,100] jepsen worker 77 - jepsen.util 18527 :info :cas [25 [13 12]] :RequestCanceledException INFO [2019-12-17 12:38:08,101] jepsen worker 77 - jepsen.util 18617 :invoke :write [25 25] INFO [2019-12-17 12:38:08,103] jepsen worker 32 - jepsen.util 13172 :ok :read [10 22] INFO [2019-12-17 12:38:08,103] jepsen worker 32 - jepsen.util 13172 :invoke :cas [10 [44 44]] INFO [2019-12-17 12:38:08,119] jepsen worker 71 - jepsen.util 13571 :ok :write [23 27] INFO [2019-12-17 12:38:08,120] jepsen worker 71 - jepsen.util 13571 :invoke :read [23 nil] INFO [2019-12-17 12:38:08,134] jepsen worker 2 - jepsen.util 16742 :info :cas [0 [33 31]] :RequestCanceledException INFO [2019-12-17 12:38:08,134] jepsen worker 12 - jepsen.util 12882 :info :write [4 35] :RequestCanceledException INFO [2019-12-17 12:38:08,134] jepsen worker 20 - jepsen.util 12710 :info :cas [6 [16 41]] :RequestCanceledException INFO [2019-12-17 12:38:08,135] jepsen worker 20 - jepsen.util 12800 :invoke :cas [6 [23 5]] INFO [2019-12-17 12:38:08,135] jepsen worker 12 - jepsen.util 12972 :invoke :read [4 nil] INFO [2019-12-17 12:38:08,135] jepsen worker 2 - jepsen.util 16832 :invoke :read [0 nil] INFO [2019-12-17 12:38:08,138] jepsen worker 73 - jepsen.util 11953 :ok :read [24 19] INFO [2019-12-17 12:38:08,138] jepsen worker 73 - jepsen.util 11953 :invoke :cas [24 [10 21]] INFO [2019-12-17 12:38:08,149] jepsen worker 55 - jepsen.util 13465 :ok :write [18 43] INFO [2019-12-17 12:38:08,149] jepsen worker 46 - jepsen.util 12376 :info :write [15 28] :RequestCanceledException INFO [2019-12-17 12:38:08,149] jepsen worker 15 - jepsen.util 16575 :info :cas [5 [29 47]] :RequestCanceledException INFO [2019-12-17 12:38:08,149] jepsen worker 55 - jepsen.util 13465 :invoke :read [18 nil] INFO [2019-12-17 12:38:08,150] jepsen worker 15 - jepsen.util 16665 :invoke :write [5 29] INFO [2019-12-17 12:38:08,150] jepsen worker 46 - jepsen.util 12466 :invoke :write [15 0] INFO [2019-12-17 12:38:08,173] jepsen worker 32 - jepsen.util 13172 :fail :cas [10 [44 44]] :ValueNotSwapFrom INFO [2019-12-17 12:38:08,173] jepsen worker 32 - jepsen.util 13172 :invoke :cas [10 [33 1]] INFO [2019-12-17 12:38:08,199] jepsen worker 73 - jepsen.util 11953 :fail :cas [24 [10 21]] :ValueNotSwapFrom INFO [2019-12-17 12:38:08,200] jepsen worker 73 - jepsen.util 11953 :invoke :read [24 nil] INFO [2019-12-17 12:38:08,215] jepsen worker 44 - jepsen.util 18134 :info :cas [14 [28 45]] :RequestCanceledException INFO [2019-12-17 12:38:08,217] jepsen worker 44 - jepsen.util 18224 :invoke :read [14 nil] INFO [2019-12-17 12:38:08,231] jepsen worker 80 - jepsen.util 12230 :fail :cas [26 [15 31]] :ValueNotSwapFrom INFO [2019-12-17 12:38:08,231] jepsen worker 80 - jepsen.util 12230 :invoke :cas [26 [37 48]] INFO [2019-12-17 12:38:08,231] jepsen worker 35 - jepsen.util 17495 :info :write [11 28] :RequestCanceledException INFO [2019-12-17 12:38:08,234] jepsen worker 48 - jepsen.util 16248 :info :write [16 39] :RequestCanceledException INFO [2019-12-17 12:38:08,238] jepsen worker 48 - jepsen.util 16338 :invoke :read [16 nil] INFO [2019-12-17 12:38:08,238] jepsen worker 35 - jepsen.util 17585 :invoke :cas [11 [19 45]] INFO [2019-12-17 12:38:08,241] jepsen worker 29 - jepsen.util 11189 :fail :cas [9 [46 45]] :ValueNotSwapFrom INFO [2019-12-17 12:38:08,241] jepsen worker 29 - jepsen.util 11189 :invoke :read [9 nil] INFO [2019-12-17 12:38:08,246] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73921,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:38:08,246] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73921,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:38:08,246] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73925,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:38:08,246] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73925,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:38:08,246] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73914,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:38:08,246] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73914,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:38:08,246] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73916,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:38:08,247] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73916,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:38:08,247] jepsen worker 50 - jepsen.util 17420 :info :write [16 0] :RequestCanceledException INFO [2019-12-17 12:38:08,247] jepsen worker 16 - jepsen.util 18916 :info :cas [5 [31 16]] :RequestCanceledException INFO [2019-12-17 12:38:08,248] jepsen worker 50 - jepsen.util 17510 :invoke :write [16 13] INFO [2019-12-17 12:38:08,255] jepsen worker 16 - jepsen.util 19006 :invoke :read [5 nil] INFO [2019-12-17 12:38:08,258] jepsen worker 74 - jepsen.util 12674 :fail :cas [24 [36 40]] :ValueNotSwapFrom INFO [2019-12-17 12:38:08,258] jepsen worker 74 - jepsen.util 12674 :invoke :write [24 24] INFO [2019-12-17 12:38:08,265] jepsen worker 58 - jepsen.util 17698 :info :write [19 43] :RequestCanceledException INFO [2019-12-17 12:38:08,269] jepsen worker 58 - jepsen.util 17788 :invoke :write [19 10] INFO [2019-12-17 12:38:08,298] jepsen worker 10 - jepsen.util 12610 :fail :read [3 nil] :RequestTimeoutException INFO [2019-12-17 12:38:08,298] jepsen worker 7 - jepsen.util 17377 :info :cas [2 [2 27]] :RequestCanceledException INFO [2019-12-17 12:38:08,298] jepsen worker 47 - jepsen.util 13817 :info :cas [15 [45 9]] :RequestCanceledException INFO [2019-12-17 12:38:08,298] jepsen worker 10 - jepsen.util 12610 :invoke :read [3 nil] INFO [2019-12-17 12:38:08,299] jepsen worker 7 - jepsen.util 17467 :invoke :read [2 nil] INFO [2019-12-17 12:38:08,299] jepsen worker 47 - jepsen.util 13907 :invoke :write [15 36] INFO [2019-12-17 12:38:08,301] jepsen worker 20 - jepsen.util 12800 :fail :cas [6 [23 5]] :ValueNotSwapFrom INFO [2019-12-17 12:38:08,301] jepsen worker 20 - jepsen.util 12800 :invoke :read [6 nil] INFO [2019-12-17 12:38:08,302] jepsen worker 73 - jepsen.util 11953 :ok :read [24 19] INFO [2019-12-17 12:38:08,302] jepsen worker 73 - jepsen.util 11953 :invoke :write [24 22] INFO [2019-12-17 12:38:08,319] jepsen worker 27 - jepsen.util 11907 :fail :read [9 nil] :RequestTimeoutException INFO [2019-12-17 12:38:08,319] jepsen worker 27 - jepsen.util 11907 :invoke :cas [9 [47 34]] INFO [2019-12-17 12:38:08,332] jepsen worker 12 - jepsen.util 12972 :ok :read [4 45] INFO [2019-12-17 12:38:08,332] jepsen worker 12 - jepsen.util 12972 :invoke :cas [4 [22 49]] INFO [2019-12-17 12:38:08,332] jepsen worker 8 - jepsen.util 17018 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:38:08,332] jepsen worker 61 - jepsen.util 18601 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:38:08,332] jepsen worker 57 - jepsen.util 17157 :info :write [19 9] :RequestCanceledException INFO [2019-12-17 12:38:08,332] jepsen worker 81 - jepsen.util 16911 :info :cas [27 [41 28]] :RequestCanceledException INFO [2019-12-17 12:38:08,333] jepsen worker 8 - jepsen.util 17018 :invoke :cas [2 [26 14]] INFO [2019-12-17 12:38:08,333] jepsen worker 61 - jepsen.util 18601 :invoke :read [20 nil] INFO [2019-12-17 12:38:08,333] jepsen worker 25 - jepsen.util 18565 :info :write [8 30] :RequestCanceledException INFO [2019-12-17 12:38:08,333] jepsen worker 9 - jepsen.util 13239 :info :cas [3 [13 21]] :RequestCanceledException INFO [2019-12-17 12:38:08,334] jepsen worker 81 - jepsen.util 17001 :invoke :read [27 nil] INFO [2019-12-17 12:38:08,334] jepsen worker 9 - jepsen.util 13329 :invoke :write [3 43] INFO [2019-12-17 12:38:08,334] jepsen worker 57 - jepsen.util 17247 :invoke :write [19 29] INFO [2019-12-17 12:38:08,334] jepsen worker 25 - jepsen.util 18655 :invoke :read [8 nil] INFO [2019-12-17 12:38:08,345] jepsen worker 4 - jepsen.util 11974 :fail :cas [1 [25 2]] :ValueNotSwapFrom INFO [2019-12-17 12:38:08,347] jepsen worker 4 - jepsen.util 11974 :invoke :cas [1 [3 0]] INFO [2019-12-17 12:38:08,365] jepsen worker 59 - jepsen.util 16529 :info :write [19 31] :RequestCanceledException INFO [2019-12-17 12:38:08,365] jepsen worker 43 - jepsen.util 17593 :info :write [14 38] :RequestCanceledException INFO [2019-12-17 12:38:08,366] jepsen worker 43 - jepsen.util 17683 :invoke :write [14 30] INFO [2019-12-17 12:38:08,366] jepsen worker 59 - jepsen.util 16619 :invoke :cas [19 [26 10]] INFO [2019-12-17 12:38:08,376] jepsen worker 53 - jepsen.util 12293 :ok :read [17 44] INFO [2019-12-17 12:38:08,377] jepsen worker 53 - jepsen.util 12293 :invoke :write [17 8] INFO [2019-12-17 12:38:08,384] jepsen worker 56 - jepsen.util 13466 :ok :read [18 43] INFO [2019-12-17 12:38:08,384] jepsen worker 56 - jepsen.util 13466 :invoke :write [18 37] INFO [2019-12-17 12:38:08,402] jepsen worker 88 - jepsen.util 12868 :fail :cas [29 [11 44]] :ValueNotSwapFrom INFO [2019-12-17 12:38:08,402] jepsen worker 88 - jepsen.util 12868 :invoke :cas [29 [39 21]] INFO [2019-12-17 12:38:08,418] jepsen worker 79 - jepsen.util 13039 :ok :read [26 7] INFO [2019-12-17 12:38:08,418] jepsen worker 79 - jepsen.util 13039 :invoke :write [26 48] INFO [2019-12-17 12:38:08,419] jepsen worker 5 - jepsen.util 12335 :info :write [1 18] :RequestCanceledException INFO [2019-12-17 12:38:08,420] jepsen worker 5 - jepsen.util 12425 :invoke :write [1 18] INFO [2019-12-17 12:38:08,428] jepsen worker 31 - jepsen.util 11101 :ok :read [10 22] INFO [2019-12-17 12:38:08,428] jepsen worker 31 - jepsen.util 11101 :invoke :write [10 22] INFO [2019-12-17 12:38:08,433] jepsen worker 69 - jepsen.util 13029 :info :write [23 46] :RequestCanceledException INFO [2019-12-17 12:38:08,437] jepsen worker 69 - jepsen.util 13119 :invoke :read [23 nil] INFO [2019-12-17 12:38:08,449] jepsen worker 71 - jepsen.util 13571 :ok :read [23 27] INFO [2019-12-17 12:38:08,451] jepsen worker 29 - jepsen.util 11189 :ok :read [9 7] INFO [2019-12-17 12:38:08,452] jepsen worker 71 - jepsen.util 13571 :invoke :write [23 11] INFO [2019-12-17 12:38:08,452] jepsen worker 29 - jepsen.util 11189 :invoke :read [9 nil] INFO [2019-12-17 12:38:08,458] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73958,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:38:08,458] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73958,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:38:08,458] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73962,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:38:08,459] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73962,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:38:08,460] jepsen worker 32 - jepsen.util 13172 :fail :cas [10 [33 1]] :ValueNotSwapFrom INFO [2019-12-17 12:38:08,460] jepsen worker 32 - jepsen.util 13172 :invoke :write [10 49] INFO [2019-12-17 12:38:08,481] jepsen worker 36 - jepsen.util 13536 :ok :write [12 9] INFO [2019-12-17 12:38:08,482] jepsen worker 36 - jepsen.util 13536 :invoke :cas [12 [44 46]] INFO [2019-12-17 12:38:08,501] jepsen worker 72 - jepsen.util 12852 :info :write [24 1] :RequestCanceledException INFO [2019-12-17 12:38:08,502] jepsen worker 72 - jepsen.util 12942 :invoke :write [24 33] INFO [2019-12-17 12:38:08,511] jepsen worker 52 - jepsen.util 12292 :info :write [17 49] :RequestCanceledException INFO [2019-12-17 12:38:08,511] jepsen worker 42 - jepsen.util 18402 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:38:08,512] jepsen worker 42 - jepsen.util 18402 :invoke :read [14 nil] INFO [2019-12-17 12:38:08,515] jepsen worker 52 - jepsen.util 12382 :invoke :read [17 nil] INFO [2019-12-17 12:38:08,527] jepsen worker 53 - jepsen.util 12293 :ok :write [17 8] INFO [2019-12-17 12:38:08,527] jepsen worker 53 - jepsen.util 12293 :invoke :write [17 43] INFO [2019-12-17 12:38:08,534] jepsen worker 83 - jepsen.util 17453 :info :cas [27 [23 35]] :RequestCanceledException INFO [2019-12-17 12:38:08,536] jepsen worker 83 - jepsen.util 17543 :invoke :read [27 nil] INFO [2019-12-17 12:38:08,552] jepsen worker 87 - jepsen.util 12237 :ok :write [29 17] INFO [2019-12-17 12:38:08,552] jepsen worker 87 - jepsen.util 12237 :invoke :write [29 42] INFO [2019-12-17 12:38:08,601] jepsen worker 39 - jepsen.util 16959 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:38:08,601] jepsen worker 39 - jepsen.util 16959 :invoke :cas [13 [22 34]] INFO [2019-12-17 12:38:08,609] jepsen worker 10 - jepsen.util 12610 :ok :read [3 7] INFO [2019-12-17 12:38:08,609] jepsen worker 10 - jepsen.util 12610 :invoke :cas [3 [45 2]] INFO [2019-12-17 12:38:08,612] jepsen worker 20 - jepsen.util 12800 :ok :read [6 27] INFO [2019-12-17 12:38:08,612] jepsen worker 20 - jepsen.util 12800 :invoke :write [6 3] INFO [2019-12-17 12:38:08,624] jepsen worker 27 - jepsen.util 11907 :fail :cas [9 [47 34]] :ValueNotSwapFrom INFO [2019-12-17 12:38:08,624] jepsen worker 27 - jepsen.util 11907 :invoke :cas [9 [28 2]] INFO [2019-12-17 12:38:08,634] jepsen worker 6 - jepsen.util 17916 :info :write [2 5] :RequestCanceledException INFO [2019-12-17 12:38:08,647] jepsen worker 6 - jepsen.util 18006 :invoke :read [2 nil] INFO [2019-12-17 12:38:08,650] jepsen worker 1 - jepsen.util 17731 :info :write [0 1] :RequestCanceledException INFO [2019-12-17 12:38:08,655] jepsen worker 1 - jepsen.util 17821 :invoke :write [0 7] INFO [2019-12-17 12:38:08,659] jepsen worker 80 - jepsen.util 12230 :fail :cas [26 [37 48]] :ValueNotSwapFrom INFO [2019-12-17 12:38:08,659] jepsen worker 80 - jepsen.util 12230 :invoke :cas [26 [45 10]] INFO [2019-12-17 12:38:08,662] jepsen worker 22 - jepsen.util 17662 :info :write [7 22] :RequestCanceledException INFO [2019-12-17 12:38:08,663] jepsen worker 22 - jepsen.util 17752 :invoke :cas [7 [5 27]] INFO [2019-12-17 12:38:08,678] jepsen worker 12 - jepsen.util 12972 :fail :cas [4 [22 49]] :ValueNotSwapFrom INFO [2019-12-17 12:38:08,679] jepsen worker 12 - jepsen.util 12972 :invoke :read [4 nil] INFO [2019-12-17 12:38:08,697] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73979,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:38:08,697] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73979,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:38:08,697] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73980,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:38:08,697] jepsen worker 65 - jepsen.util 11135 :fail :read [21 nil] :RequestTimeoutException INFO [2019-12-17 12:38:08,697] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73980,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:38:08,697] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73981,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:38:08,697] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":73981,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:38:08,697] jepsen worker 65 - jepsen.util 11135 :invoke :cas [21 [24 10]] INFO [2019-12-17 12:38:08,700] jepsen worker 88 - jepsen.util 12868 :fail :cas [29 [39 21]] :ValueNotSwapFrom INFO [2019-12-17 12:38:08,700] jepsen worker 88 - jepsen.util 12868 :invoke :cas [29 [0 45]] INFO [2019-12-17 12:38:08,710] jepsen worker 10 - jepsen.util 12610 :fail :cas [3 [45 2]] :ValueNotSwapFrom INFO [2019-12-17 12:38:08,710] jepsen worker 10 - jepsen.util 12610 :invoke :write [3 48] INFO [2019-12-17 12:38:08,719] jepsen worker 76 - jepsen.util 17446 :info :cas [25 [31 15]] :RequestCanceledException INFO [2019-12-17 12:38:08,730] jepsen worker 76 - jepsen.util 17536 :invoke :cas [25 [23 46]] INFO [2019-12-17 12:38:08,734] jepsen worker 68 - jepsen.util 16628 :info :cas [22 [48 45]] :RequestCanceledException INFO [2019-12-17 12:38:08,735] jepsen worker 68 - jepsen.util 16718 :invoke :read [22 nil] INFO [2019-12-17 12:38:08,772] jepsen worker 36 - jepsen.util 13536 :fail :cas [12 [44 46]] :ValueNotSwapFrom INFO [2019-12-17 12:38:08,772] jepsen worker 36 - jepsen.util 13536 :invoke :read [12 nil] INFO [2019-12-17 12:38:08,813] jepsen worker 12 - jepsen.util 12972 :ok :read [4 45] INFO [2019-12-17 12:38:08,813] jepsen worker 12 - jepsen.util 12972 :invoke :write [4 13] INFO [2019-12-17 12:38:08,828] jepsen worker 32 - jepsen.util 13172 :ok :write [10 49] INFO [2019-12-17 12:38:08,828] jepsen worker 32 - jepsen.util 13172 :invoke :cas [10 [1 20]] INFO [2019-12-17 12:38:08,829] jepsen worker 46 - jepsen.util 12466 :ok :write [15 0] INFO [2019-12-17 12:38:08,829] jepsen worker 46 - jepsen.util 12466 :invoke :read [15 nil] INFO [2019-12-17 12:38:08,848] jepsen worker 65 - jepsen.util 11135 :fail :cas [21 [24 10]] :ValueNotSwapFrom INFO [2019-12-17 12:38:08,848] jepsen worker 65 - jepsen.util 11135 :invoke :read [21 nil] INFO [2019-12-17 12:38:08,866] jepsen worker 49 - jepsen.util 17419 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:38:08,866] jepsen worker 49 - jepsen.util 17419 :invoke :cas [16 [28 6]] INFO [2019-12-17 12:38:08,880] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":74000,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:38:08,880] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":74000,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:38:08,880] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":74003,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:38:08,880] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":74003,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:38:08,901] jepsen worker 40 - jepsen.util 18220 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:38:08,904] jepsen worker 40 - jepsen.util 18220 :invoke :cas [13 [47 28]] INFO [2019-12-17 12:38:08,924] jepsen worker 80 - jepsen.util 12230 :fail :cas [26 [45 10]] :ValueNotSwapFrom INFO [2019-12-17 12:38:08,924] jepsen worker 80 - jepsen.util 12230 :invoke :read [26 nil] INFO [2019-12-17 12:38:08,926] jepsen worker 88 - jepsen.util 12868 :fail :cas [29 [0 45]] :ValueNotSwapFrom INFO [2019-12-17 12:38:08,926] jepsen worker 88 - jepsen.util 12868 :invoke :read [29 nil] INFO [2019-12-17 12:38:08,934] jepsen worker 34 - jepsen.util 18304 :info :write [11 14] :RequestCanceledException INFO [2019-12-17 12:38:08,934] jepsen worker 34 - jepsen.util 18394 :invoke :cas [11 [24 37]] INFO [2019-12-17 12:38:08,935] jepsen worker 30 - jepsen.util 12000 :info :write [10 7] :RequestCanceledException INFO [2019-12-17 12:38:08,937] jepsen worker 30 - jepsen.util 12090 :invoke :write [10 18] INFO [2019-12-17 12:38:08,941] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:38:08,941] jepsen nemesis - couchbase.util Rebalance check count: 583 INFO [2019-12-17 12:38:08,941] jepsen nemesis - couchbase.util Rebalance stuck count: 564 INFO [2019-12-17 12:38:08,948] jepsen worker 24 - jepsen.util 16494 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:38:08,948] jepsen worker 24 - jepsen.util 16494 :invoke :cas [8 [23 16]] INFO [2019-12-17 12:38:08,955] jepsen worker 20 - jepsen.util 12800 :ok :write [6 3] INFO [2019-12-17 12:38:08,955] jepsen worker 20 - jepsen.util 12800 :invoke :read [6 nil] INFO [2019-12-17 12:38:08,956] jepsen worker 9 - jepsen.util 13329 :ok :write [3 43] INFO [2019-12-17 12:38:08,956] jepsen worker 9 - jepsen.util 13329 :invoke :write [3 23] INFO [2019-12-17 12:38:08,961] jepsen worker 73 - jepsen.util 11953 :ok :write [24 22] INFO [2019-12-17 12:38:08,961] jepsen worker 73 - jepsen.util 11953 :invoke :read [24 nil] INFO [2019-12-17 12:38:08,965] jepsen worker 41 - jepsen.util 18761 :info :write [13 46] :RequestCanceledException INFO [2019-12-17 12:38:08,966] jepsen worker 41 - jepsen.util 18851 :invoke :write [13 20] INFO [2019-12-17 12:38:09,001] jepsen worker 4 - jepsen.util 11974 :fail :cas [1 [3 0]] :ValueNotSwapFrom INFO [2019-12-17 12:38:09,003] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":74017,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:38:09,004] jepsen worker 4 - jepsen.util 11974 :invoke :read [1 nil] INFO [2019-12-17 12:38:09,004] jepsen worker 69 - jepsen.util 13119 :ok :read [23 27] INFO [2019-12-17 12:38:09,006] jepsen worker 69 - jepsen.util 13119 :invoke :write [23 1] INFO [2019-12-17 12:38:09,008] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":74017,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:38:09,008] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":74021,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:38:09,008] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":74021,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:38:09,035] jepsen worker 18 - jepsen.util 13248 :info :cas [6 [28 18]] :RequestCanceledException INFO [2019-12-17 12:38:09,035] jepsen worker 11 - jepsen.util 12521 :info :write [3 27] :RequestCanceledException INFO [2019-12-17 12:38:09,035] jepsen worker 11 - jepsen.util 12611 :invoke :read [3 nil] INFO [2019-12-17 12:38:09,036] jepsen worker 18 - jepsen.util 13338 :invoke :read [6 nil] INFO [2019-12-17 12:38:09,038] jepsen worker 73 - jepsen.util 11953 :ok :read [24 22] INFO [2019-12-17 12:38:09,038] jepsen worker 73 - jepsen.util 11953 :invoke :cas [24 [16 0]] INFO [2019-12-17 12:38:09,046] jepsen worker 31 - jepsen.util 11101 :ok :write [10 22] INFO [2019-12-17 12:38:09,046] jepsen worker 31 - jepsen.util 11101 :invoke :write [10 27] INFO [2019-12-17 12:38:09,046] jepsen worker 54 - jepsen.util 13284 :info :write [18 19] :RequestCanceledException INFO [2019-12-17 12:38:09,047] jepsen worker 54 - jepsen.util 13374 :invoke :read [18 nil] INFO [2019-12-17 12:38:09,077] jepsen worker 36 - jepsen.util 13536 :ok :read [12 9] INFO [2019-12-17 12:38:09,077] jepsen worker 36 - jepsen.util 13536 :invoke :write [12 8] INFO [2019-12-17 12:38:09,117] jepsen worker 18 - jepsen.util 13338 :ok :read [6 3] INFO [2019-12-17 12:38:09,117] jepsen worker 18 - jepsen.util 13338 :invoke :write [6 34] INFO [2019-12-17 12:38:09,118] jepsen worker 73 - jepsen.util 11953 :fail :cas [24 [16 0]] :ValueNotSwapFrom INFO [2019-12-17 12:38:09,118] jepsen worker 3 - jepsen.util 12243 :info :write [1 49] :RequestCanceledException INFO [2019-12-17 12:38:09,118] jepsen worker 73 - jepsen.util 11953 :invoke :cas [24 [43 1]] INFO [2019-12-17 12:38:09,119] jepsen worker 3 - jepsen.util 12333 :invoke :write [1 36] INFO [2019-12-17 12:38:09,119] jepsen worker 80 - jepsen.util 12230 :ok :read [26 7] INFO [2019-12-17 12:38:09,119] jepsen worker 80 - jepsen.util 12230 :invoke :read [26 nil] INFO [2019-12-17 12:38:09,123] jepsen worker 4 - jepsen.util 11974 :ok :read [1 31] INFO [2019-12-17 12:38:09,124] jepsen worker 4 - jepsen.util 11974 :invoke :cas [1 [21 42]] INFO [2019-12-17 12:38:09,124] jepsen worker 46 - jepsen.util 12466 :ok :read [15 0] INFO [2019-12-17 12:38:09,124] jepsen worker 46 - jepsen.util 12466 :invoke :cas [15 [41 26]] INFO [2019-12-17 12:38:09,134] jepsen worker 62 - jepsen.util 18152 :info :cas [20 [16 44]] :RequestCanceledException INFO [2019-12-17 12:38:09,134] jepsen worker 28 - jepsen.util 10918 :info :cas [9 [2 39]] :RequestCanceledException INFO [2019-12-17 12:38:09,134] jepsen worker 86 - jepsen.util 18266 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:38:09,135] jepsen worker 86 - jepsen.util 18266 :invoke :cas [28 [5 22]] INFO [2019-12-17 12:38:09,135] jepsen worker 28 - jepsen.util 11008 :invoke :read [9 nil] INFO [2019-12-17 12:38:09,135] jepsen worker 62 - jepsen.util 18242 :invoke :cas [20 [10 7]] INFO [2019-12-17 12:38:09,141] jepsen worker 73 - jepsen.util 11953 :fail :cas [24 [43 1]] :ValueNotSwapFrom INFO [2019-12-17 12:38:09,141] jepsen worker 73 - jepsen.util 11953 :invoke :write [24 14] INFO [2019-12-17 12:38:09,148] jepsen worker 45 - jepsen.util 12555 :fail :read [15 nil] :RequestTimeoutException INFO [2019-12-17 12:38:09,148] jepsen worker 19 - jepsen.util 13339 :info :cas [6 [2 39]] :RequestCanceledException INFO [2019-12-17 12:38:09,148] jepsen worker 64 - jepsen.util 9964 :fail :read [21 nil] :RequestTimeoutException INFO [2019-12-17 12:38:09,148] jepsen worker 45 - jepsen.util 12555 :invoke :read [15 nil] INFO [2019-12-17 12:38:09,148] jepsen worker 64 - jepsen.util 9964 :invoke :read [21 nil] INFO [2019-12-17 12:38:09,149] jepsen worker 19 - jepsen.util 13429 :invoke :cas [6 [9 5]] INFO [2019-12-17 12:38:09,181] jepsen worker 28 - jepsen.util 11008 :ok :read [9 7] INFO [2019-12-17 12:38:09,181] jepsen worker 28 - jepsen.util 11008 :invoke :cas [9 [42 21]] INFO [2019-12-17 12:38:09,196] jepsen worker 28 - jepsen.util 11008 :fail :cas [9 [42 21]] :ValueNotSwapFrom INFO [2019-12-17 12:38:09,196] jepsen worker 28 - jepsen.util 11008 :invoke :cas [9 [9 25]] INFO [2019-12-17 12:38:09,220] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":74046,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:38:09,220] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":74046,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:38:09,220] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":74047,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:38:09,220] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":74047,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:38:09,233] jepsen worker 75 - jepsen.util 16995 :info :write [25 2] :RequestCanceledException INFO [2019-12-17 12:38:09,243] jepsen worker 75 - jepsen.util 17085 :invoke :write [25 3] INFO [2019-12-17 12:38:09,247] jepsen worker 28 - jepsen.util 11008 :fail :cas [9 [9 25]] :ValueNotSwapFrom INFO [2019-12-17 12:38:09,247] jepsen worker 28 - jepsen.util 11008 :invoke :cas [9 [7 48]] INFO [2019-12-17 12:38:09,255] jepsen worker 55 - jepsen.util 13465 :ok :read [18 43] INFO [2019-12-17 12:38:09,255] jepsen worker 55 - jepsen.util 13465 :invoke :read [18 nil] INFO [2019-12-17 12:38:09,265] jepsen worker 17 - jepsen.util 17477 :info :write [5 47] :RequestCanceledException INFO [2019-12-17 12:38:09,266] jepsen worker 17 - jepsen.util 17567 :invoke :read [5 nil] INFO [2019-12-17 12:38:09,275] jepsen worker 18 - jepsen.util 13338 :ok :write [6 34] INFO [2019-12-17 12:38:09,275] jepsen worker 18 - jepsen.util 13338 :invoke :write [6 39] INFO [2019-12-17 12:38:09,275] jepsen worker 31 - jepsen.util 11101 :ok :write [10 27] INFO [2019-12-17 12:38:09,275] jepsen worker 31 - jepsen.util 11101 :invoke :cas [10 [22 39]] INFO [2019-12-17 12:38:09,276] jepsen worker 9 - jepsen.util 13329 :ok :write [3 23] INFO [2019-12-17 12:38:09,277] jepsen worker 9 - jepsen.util 13329 :invoke :read [3 nil] INFO [2019-12-17 12:38:09,285] jepsen worker 73 - jepsen.util 11953 :ok :write [24 14] INFO [2019-12-17 12:38:09,285] jepsen worker 73 - jepsen.util 11953 :invoke :read [24 nil] INFO [2019-12-17 12:38:09,300] jepsen worker 26 - jepsen.util 18296 :info :write [8 22] :RequestCanceledException INFO [2019-12-17 12:38:09,301] jepsen worker 26 - jepsen.util 18386 :invoke :cas [8 [12 33]] INFO [2019-12-17 12:38:09,318] jepsen worker 88 - jepsen.util 12868 :ok :read [29 17] INFO [2019-12-17 12:38:09,318] jepsen worker 88 - jepsen.util 12868 :invoke :read [29 nil] INFO [2019-12-17 12:38:09,319] jepsen worker 60 - jepsen.util 18420 :info :cas [20 [23 26]] :RequestCanceledException INFO [2019-12-17 12:38:09,319] jepsen worker 63 - jepsen.util 11403 :fail :read [21 nil] :RequestTimeoutException INFO [2019-12-17 12:38:09,319] jepsen worker 63 - jepsen.util 11403 :invoke :read [21 nil] INFO [2019-12-17 12:38:09,320] jepsen worker 60 - jepsen.util 18510 :invoke :read [20 nil] INFO [2019-12-17 12:38:09,329] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":74052,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:38:09,329] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":74052,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:38:09,329] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":74055,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:38:09,329] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":74055,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:38:09,334] jepsen worker 70 - jepsen.util 13210 :info :write [23 33] :RequestCanceledException INFO [2019-12-17 12:38:09,335] jepsen worker 70 - jepsen.util 13300 :invoke :cas [23 [40 24]] INFO [2019-12-17 12:38:09,374] jepsen worker 14 - jepsen.util 12344 :ok :write [4 9] INFO [2019-12-17 12:38:09,375] jepsen worker 14 - jepsen.util 12344 :invoke :write [4 0] INFO [2019-12-17 12:38:09,377] jepsen worker 73 - jepsen.util 11953 :ok :read [24 14] INFO [2019-12-17 12:38:09,377] jepsen worker 73 - jepsen.util 11953 :invoke :cas [24 [38 25]] INFO [2019-12-17 12:38:09,378] jepsen worker 46 - jepsen.util 12466 :fail :cas [15 [41 26]] :ValueNotSwapFrom INFO [2019-12-17 12:38:09,378] jepsen worker 46 - jepsen.util 12466 :invoke :cas [15 [27 9]] INFO [2019-12-17 12:38:09,427] jepsen worker 11 - jepsen.util 12611 :ok :read [3 23] INFO [2019-12-17 12:38:09,428] jepsen worker 11 - jepsen.util 12611 :invoke :write [3 0] INFO [2019-12-17 12:38:09,432] jepsen worker 66 - jepsen.util 17526 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:38:09,432] jepsen worker 66 - jepsen.util 17526 :invoke :read [22 nil] INFO [2019-12-17 12:38:09,433] jepsen worker 45 - jepsen.util 12555 :ok :read [15 0] INFO [2019-12-17 12:38:09,433] jepsen worker 45 - jepsen.util 12555 :invoke :write [15 10] INFO [2019-12-17 12:38:09,440] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":74077,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:38:09,440] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":74077,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:38:09,444] jepsen worker 52 - jepsen.util 12382 :ok :read [17 8] INFO [2019-12-17 12:38:09,445] jepsen worker 52 - jepsen.util 12382 :invoke :write [17 29] INFO [2019-12-17 12:38:09,448] jepsen worker 67 - jepsen.util 17707 :info :cas [22 [21 12]] :RequestCanceledException INFO [2019-12-17 12:38:09,450] jepsen worker 67 - jepsen.util 17797 :invoke :read [22 nil] INFO [2019-12-17 12:38:09,452] jepsen worker 73 - jepsen.util 11953 :fail :cas [24 [38 25]] :ValueNotSwapFrom INFO [2019-12-17 12:38:09,453] jepsen worker 73 - jepsen.util 11953 :invoke :cas [24 [44 49]] INFO [2019-12-17 12:38:09,455] jepsen worker 4 - jepsen.util 11974 :fail :cas [1 [21 42]] :ValueNotSwapFrom INFO [2019-12-17 12:38:09,455] jepsen worker 4 - jepsen.util 11974 :invoke :read [1 nil] INFO [2019-12-17 12:38:09,490] jepsen worker 51 - jepsen.util 12021 :ok :write [17 20] INFO [2019-12-17 12:38:09,490] jepsen worker 51 - jepsen.util 12021 :invoke :write [17 34] INFO [2019-12-17 12:38:09,490] jepsen worker 12 - jepsen.util 12972 :ok :write [4 13] INFO [2019-12-17 12:38:09,490] jepsen worker 12 - jepsen.util 12972 :invoke :read [4 nil] INFO [2019-12-17 12:38:09,504] jepsen worker 64 - jepsen.util 9964 :ok :read [21 6] INFO [2019-12-17 12:38:09,504] jepsen worker 64 - jepsen.util 9964 :invoke :read [21 nil] INFO [2019-12-17 12:38:09,519] jepsen worker 85 - jepsen.util 17995 :info :write [28 2] :RequestCanceledException INFO [2019-12-17 12:38:09,521] jepsen worker 85 - jepsen.util 18085 :invoke :cas [28 [3 36]] INFO [2019-12-17 12:38:09,524] jepsen worker 55 - jepsen.util 13465 :ok :read [18 43] INFO [2019-12-17 12:38:09,525] jepsen worker 55 - jepsen.util 13465 :invoke :read [18 nil] INFO [2019-12-17 12:38:09,530] jepsen worker 19 - jepsen.util 13429 :fail :cas [6 [9 5]] :ValueNotSwapFrom INFO [2019-12-17 12:38:09,530] jepsen worker 19 - jepsen.util 13429 :invoke :write [6 17] INFO [2019-12-17 12:38:09,535] jepsen worker 84 - jepsen.util 16914 :info :write [28 47] :RequestCanceledException INFO [2019-12-17 12:38:09,536] jepsen worker 84 - jepsen.util 17004 :invoke :write [28 1] INFO [2019-12-17 12:38:09,548] jepsen worker 33 - jepsen.util 17313 :info :write [11 8] :RequestCanceledException INFO [2019-12-17 12:38:09,549] jepsen worker 33 - jepsen.util 17403 :invoke :write [11 32] INFO [2019-12-17 12:38:09,565] jepsen worker 13 - jepsen.util 12343 :info :write [4 22] :RequestCanceledException INFO [2019-12-17 12:38:09,566] jepsen worker 13 - jepsen.util 12433 :invoke :read [4 nil] INFO [2019-12-17 12:38:09,593] jepsen worker 73 - jepsen.util 11953 :fail :cas [24 [44 49]] :ValueNotSwapFrom INFO [2019-12-17 12:38:09,593] jepsen worker 73 - jepsen.util 11953 :invoke :cas [24 [34 7]] INFO [2019-12-17 12:38:09,602] jepsen worker 87 - jepsen.util 12237 :ok :write [29 42] INFO [2019-12-17 12:38:09,602] jepsen worker 87 - jepsen.util 12237 :invoke :write [29 25] INFO [2019-12-17 12:38:09,604] jepsen worker 13 - jepsen.util 12433 :ok :read [4 13] INFO [2019-12-17 12:38:09,604] jepsen worker 13 - jepsen.util 12433 :invoke :write [4 29] INFO [2019-12-17 12:38:09,619] jepsen worker 23 - jepsen.util 17843 :info :cas [7 [23 46]] :RequestCanceledException INFO [2019-12-17 12:38:09,620] jepsen worker 31 - jepsen.util 11101 :fail :cas [10 [22 39]] :ValueNotSwapFrom INFO [2019-12-17 12:38:09,620] jepsen worker 31 - jepsen.util 11101 :invoke :cas [10 [7 29]] INFO [2019-12-17 12:38:09,620] jepsen worker 23 - jepsen.util 17933 :invoke :read [7 nil] INFO [2019-12-17 12:38:09,660] jepsen worker 9 - jepsen.util 13329 :ok :read [3 23] INFO [2019-12-17 12:38:09,661] jepsen worker 9 - jepsen.util 13329 :invoke :read [3 nil] INFO [2019-12-17 12:38:09,662] jepsen worker 18 - jepsen.util 13338 :ok :write [6 39] INFO [2019-12-17 12:38:09,662] jepsen worker 18 - jepsen.util 13338 :invoke :read [6 nil] INFO [2019-12-17 12:38:09,663] jepsen worker 46 - jepsen.util 12466 :fail :cas [15 [27 9]] :ValueNotSwapFrom INFO [2019-12-17 12:38:09,663] jepsen worker 46 - jepsen.util 12466 :invoke :cas [15 [8 17]] INFO [2019-12-17 12:38:09,666] jepsen worker 37 - jepsen.util 12187 :info :write [12 0] :RequestCanceledException INFO [2019-12-17 12:38:09,666] jepsen worker 38 - jepsen.util 13358 :info :write [12 45] :RequestCanceledException INFO [2019-12-17 12:38:09,667] jepsen worker 37 - jepsen.util 12277 :invoke :write [12 49] INFO [2019-12-17 12:38:09,667] jepsen worker 38 - jepsen.util 13448 :invoke :write [12 25] INFO [2019-12-17 12:38:09,668] jepsen worker 73 - jepsen.util 11953 :fail :cas [24 [34 7]] :ValueNotSwapFrom INFO [2019-12-17 12:38:09,668] jepsen worker 73 - jepsen.util 11953 :invoke :write [24 43] INFO [2019-12-17 12:38:09,681] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":74102,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:38:09,681] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":74102,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:38:09,681] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":74105,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:38:09,681] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":74105,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:38:09,701] jepsen worker 21 - jepsen.util 17031 :info :cas [7 [40 16]] :RequestCanceledException INFO [2019-12-17 12:38:09,701] jepsen worker 78 - jepsen.util 12408 :info :write [26 48] :RequestCanceledException INFO [2019-12-17 12:38:09,702] jepsen worker 21 - jepsen.util 17121 :invoke :write [7 34] INFO [2019-12-17 12:38:09,702] jepsen worker 63 - jepsen.util 11403 :ok :read [21 6] INFO [2019-12-17 12:38:09,702] jepsen worker 63 - jepsen.util 11403 :invoke :read [21 nil] INFO [2019-12-17 12:38:09,704] jepsen worker 78 - jepsen.util 12498 :invoke :cas [26 [41 49]] INFO [2019-12-17 12:38:09,718] jepsen worker 82 - jepsen.util 17812 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:38:09,718] jepsen worker 0 - jepsen.util 17820 :info :write [0 22] :RequestCanceledException INFO [2019-12-17 12:38:09,719] jepsen worker 82 - jepsen.util 17812 :invoke :read [27 nil] INFO [2019-12-17 12:38:09,720] jepsen worker 0 - jepsen.util 17910 :invoke :cas [0 [18 1]] INFO [2019-12-17 12:38:09,725] jepsen worker 18 - jepsen.util 13338 :ok :read [6 39] INFO [2019-12-17 12:38:09,725] jepsen worker 18 - jepsen.util 13338 :invoke :cas [6 [8 22]] INFO [2019-12-17 12:38:09,754] jepsen worker 55 - jepsen.util 13465 :ok :read [18 43] INFO [2019-12-17 12:38:09,754] jepsen worker 55 - jepsen.util 13465 :invoke :write [18 37] INFO [2019-12-17 12:38:09,778] jepsen worker 18 - jepsen.util 13338 :fail :cas [6 [8 22]] :ValueNotSwapFrom INFO [2019-12-17 12:38:09,778] jepsen worker 18 - jepsen.util 13338 :invoke :read [6 nil] INFO [2019-12-17 12:38:09,787] jepsen worker 28 - jepsen.util 11008 :ok :cas [9 [7 48]] INFO [2019-12-17 12:38:09,787] jepsen worker 28 - jepsen.util 11008 :invoke :read [9 nil] INFO [2019-12-17 12:38:09,806] jepsen worker 4 - jepsen.util 11974 :ok :read [1 31] INFO [2019-12-17 12:38:09,807] jepsen worker 4 - jepsen.util 11974 :invoke :cas [1 [26 7]] INFO [2019-12-17 12:38:09,808] jepsen worker 28 - jepsen.util 11008 :ok :read [9 48] INFO [2019-12-17 12:38:09,808] jepsen worker 28 - jepsen.util 11008 :invoke :read [9 nil] INFO [2019-12-17 12:38:09,822] jepsen worker 54 - jepsen.util 13374 :ok :read [18 43] INFO [2019-12-17 12:38:09,822] jepsen worker 54 - jepsen.util 13374 :invoke :cas [18 [13 26]] INFO [2019-12-17 12:38:09,845] jepsen worker 12 - jepsen.util 12972 :ok :read [4 13] INFO [2019-12-17 12:38:09,845] jepsen worker 12 - jepsen.util 12972 :invoke :cas [4 [29 31]] INFO [2019-12-17 12:38:09,845] jepsen worker 28 - jepsen.util 11008 :ok :read [9 48] INFO [2019-12-17 12:38:09,845] jepsen worker 28 - jepsen.util 11008 :invoke :cas [9 [40 22]] INFO [2019-12-17 12:38:09,845] jepsen worker 18 - jepsen.util 13338 :ok :read [6 39] INFO [2019-12-17 12:38:09,845] jepsen worker 18 - jepsen.util 13338 :invoke :cas [6 [24 28]] INFO [2019-12-17 12:38:09,848] jepsen worker 89 - jepsen.util 12689 :info :write [29 36] :RequestCanceledException INFO [2019-12-17 12:38:09,849] jepsen worker 89 - jepsen.util 12779 :invoke :cas [29 [30 18]] INFO [2019-12-17 12:38:09,893] jepsen worker 28 - jepsen.util 11008 :fail :cas [9 [40 22]] :ValueNotSwapFrom INFO [2019-12-17 12:38:09,893] jepsen worker 28 - jepsen.util 11008 :invoke :read [9 nil] INFO [2019-12-17 12:38:09,906] jepsen worker 18 - jepsen.util 13338 :fail :cas [6 [24 28]] :ValueNotSwapFrom INFO [2019-12-17 12:38:09,906] jepsen worker 18 - jepsen.util 13338 :invoke :read [6 nil] INFO [2019-12-17 12:38:09,909] jepsen worker 28 - jepsen.util 11008 :ok :read [9 48] INFO [2019-12-17 12:38:09,910] jepsen worker 28 - jepsen.util 11008 :invoke :read [9 nil] INFO [2019-12-17 12:38:09,939] jepsen worker 28 - jepsen.util 11008 :ok :read [9 48] INFO [2019-12-17 12:38:09,940] jepsen worker 28 - jepsen.util 11008 :invoke :cas [9 [44 30]] INFO [2019-12-17 12:38:09,948] jepsen worker 18 - jepsen.util 13338 :ok :read [6 39] INFO [2019-12-17 12:38:09,948] jepsen worker 18 - jepsen.util 13338 :invoke :write [6 37] INFO [2019-12-17 12:38:09,959] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:38:09,959] jepsen nemesis - couchbase.util Rebalance check count: 584 INFO [2019-12-17 12:38:09,960] jepsen nemesis - couchbase.util Rebalance stuck count: 565 INFO [2019-12-17 12:38:09,965] jepsen worker 28 - jepsen.util 11008 :fail :cas [9 [44 30]] :ValueNotSwapFrom INFO [2019-12-17 12:38:09,966] jepsen worker 28 - jepsen.util 11008 :invoke :cas [9 [43 11]] INFO [2019-12-17 12:38:09,974] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":74130,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:38:09,974] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":74130,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:38:09,974] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":74134,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:38:09,974] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":74134,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:38:09,974] jepsen worker 11 - jepsen.util 12611 :ok :write [3 0] INFO [2019-12-17 12:38:09,974] jepsen worker 45 - jepsen.util 12555 :ok :write [15 10] INFO [2019-12-17 12:38:09,975] jepsen worker 45 - jepsen.util 12555 :invoke :read [15 nil] INFO [2019-12-17 12:38:09,975] jepsen worker 11 - jepsen.util 12611 :invoke :cas [3 [11 18]] INFO [2019-12-17 12:38:10,008] jepsen worker 73 - jepsen.util 11953 :ok :write [24 43] INFO [2019-12-17 12:38:10,008] jepsen worker 73 - jepsen.util 11953 :invoke :cas [24 [13 18]] INFO [2019-12-17 12:38:10,019] jepsen worker 31 - jepsen.util 11101 :fail :cas [10 [7 29]] :ValueNotSwapFrom INFO [2019-12-17 12:38:10,020] jepsen worker 31 - jepsen.util 11101 :invoke :write [10 16] INFO [2019-12-17 12:38:10,022] jepsen worker 28 - jepsen.util 11008 :fail :cas [9 [43 11]] :ValueNotSwapFrom INFO [2019-12-17 12:38:10,023] jepsen worker 28 - jepsen.util 11008 :invoke :cas [9 [37 2]] INFO [2019-12-17 12:38:10,029] jepsen worker 63 - jepsen.util 11403 :ok :read [21 6] INFO [2019-12-17 12:38:10,029] jepsen worker 63 - jepsen.util 11403 :invoke :cas [21 [28 14]] INFO [2019-12-17 12:38:10,078] jepsen worker 46 - jepsen.util 12466 :fail :cas [15 [8 17]] :ValueNotSwapFrom INFO [2019-12-17 12:38:10,079] jepsen worker 46 - jepsen.util 12466 :invoke :read [15 nil] INFO [2019-12-17 12:38:10,108] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":74144,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:38:10,108] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":74144,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:38:10,108] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":74148,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:38:10,108] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":74148,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:38:10,116] jepsen worker 80 - jepsen.util 12230 :ok :read [26 7] INFO [2019-12-17 12:38:10,116] jepsen worker 80 - jepsen.util 12230 :invoke :cas [26 [1 37]] INFO [2019-12-17 12:38:10,124] jepsen worker 88 - jepsen.util 12868 :ok :read [29 42] INFO [2019-12-17 12:38:10,124] jepsen worker 88 - jepsen.util 12868 :invoke :write [29 1] INFO [2019-12-17 12:38:10,151] jepsen worker 28 - jepsen.util 11008 :fail :cas [9 [37 2]] :ValueNotSwapFrom INFO [2019-12-17 12:38:10,151] jepsen worker 28 - jepsen.util 11008 :invoke :cas [9 [25 40]] INFO [2019-12-17 12:38:10,153] jepsen worker 4 - jepsen.util 11974 :fail :cas [1 [26 7]] :ValueNotSwapFrom INFO [2019-12-17 12:38:10,153] jepsen worker 4 - jepsen.util 11974 :invoke :read [1 nil] INFO [2019-12-17 12:38:10,165] jepsen worker 77 - jepsen.util 18617 :info :write [25 25] :RequestCanceledException INFO [2019-12-17 12:38:10,166] jepsen worker 77 - jepsen.util 18707 :invoke :read [25 nil] INFO [2019-12-17 12:38:10,168] jepsen worker 45 - jepsen.util 12555 :ok :read [15 10] INFO [2019-12-17 12:38:10,168] jepsen worker 45 - jepsen.util 12555 :invoke :read [15 nil] INFO [2019-12-17 12:38:10,172] jepsen worker 11 - jepsen.util 12611 :fail :cas [3 [11 18]] :ValueNotSwapFrom INFO [2019-12-17 12:38:10,172] jepsen worker 11 - jepsen.util 12611 :invoke :write [3 24] INFO [2019-12-17 12:38:10,181] jepsen worker 54 - jepsen.util 13374 :fail :cas [18 [13 26]] :ValueNotSwapFrom INFO [2019-12-17 12:38:10,181] jepsen worker 54 - jepsen.util 13374 :invoke :write [18 28] INFO [2019-12-17 12:38:10,181] jepsen worker 73 - jepsen.util 11953 :fail :cas [24 [13 18]] :ValueNotSwapFrom INFO [2019-12-17 12:38:10,181] jepsen worker 73 - jepsen.util 11953 :invoke :cas [24 [5 33]] INFO [2019-12-17 12:38:10,183] jepsen worker 63 - jepsen.util 11403 :fail :cas [21 [28 14]] :ValueNotSwapFrom INFO [2019-12-17 12:38:10,184] jepsen worker 63 - jepsen.util 11403 :invoke :write [21 13] INFO [2019-12-17 12:38:10,185] jepsen worker 28 - jepsen.util 11008 :fail :cas [9 [25 40]] :ValueNotSwapFrom INFO [2019-12-17 12:38:10,186] jepsen worker 28 - jepsen.util 11008 :invoke :write [9 48] INFO [2019-12-17 12:38:10,204] jepsen worker 73 - jepsen.util 11953 :fail :cas [24 [5 33]] :ValueNotSwapFrom INFO [2019-12-17 12:38:10,204] jepsen worker 73 - jepsen.util 11953 :invoke :read [24 nil] INFO [2019-12-17 12:38:10,219] jepsen worker 2 - jepsen.util 16832 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:38:10,220] jepsen worker 2 - jepsen.util 16832 :invoke :write [0 30] INFO [2019-12-17 12:38:10,231] jepsen worker 73 - jepsen.util 11953 :ok :read [24 43] INFO [2019-12-17 12:38:10,231] jepsen worker 73 - jepsen.util 11953 :invoke :cas [24 [34 11]] INFO [2019-12-17 12:38:10,235] jepsen worker 15 - jepsen.util 16665 :info :write [5 29] :RequestCanceledException INFO [2019-12-17 12:38:10,244] jepsen worker 15 - jepsen.util 16755 :invoke :cas [5 [36 47]] INFO [2019-12-17 12:38:10,250] jepsen worker 28 - jepsen.util 11008 :ok :write [9 48] INFO [2019-12-17 12:38:10,250] jepsen worker 28 - jepsen.util 11008 :invoke :read [9 nil] INFO [2019-12-17 12:38:10,255] jepsen worker 70 - jepsen.util 13300 :fail :cas [23 [40 24]] :ValueNotSwapFrom INFO [2019-12-17 12:38:10,255] jepsen worker 70 - jepsen.util 13300 :invoke :read [23 nil] INFO [2019-12-17 12:38:10,265] jepsen worker 44 - jepsen.util 18224 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:38:10,265] jepsen worker 44 - jepsen.util 18224 :invoke :cas [14 [33 24]] INFO [2019-12-17 12:38:10,301] jepsen worker 35 - jepsen.util 17585 :info :cas [11 [19 45]] :RequestCanceledException INFO [2019-12-17 12:38:10,305] jepsen worker 28 - jepsen.util 11008 :ok :read [9 48] INFO [2019-12-17 12:38:10,305] jepsen worker 28 - jepsen.util 11008 :invoke :read [9 nil] INFO [2019-12-17 12:38:10,309] jepsen worker 35 - jepsen.util 17675 :invoke :cas [11 [16 23]] INFO [2019-12-17 12:38:10,311] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":74165,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:38:10,312] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":74165,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:38:10,312] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":74167,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:38:10,312] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":74167,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:38:10,317] jepsen worker 18 - jepsen.util 13338 :ok :write [6 37] INFO [2019-12-17 12:38:10,317] jepsen worker 18 - jepsen.util 13338 :invoke :write [6 4] INFO [2019-12-17 12:38:10,318] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":74173,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:38:10,318] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":74173,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:38:10,319] jepsen worker 50 - jepsen.util 17510 :info :write [16 13] :RequestCanceledException INFO [2019-12-17 12:38:10,319] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":74176,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:38:10,320] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":74176,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:38:10,322] jepsen worker 73 - jepsen.util 11953 :fail :cas [24 [34 11]] :ValueNotSwapFrom INFO [2019-12-17 12:38:10,323] jepsen worker 73 - jepsen.util 11953 :invoke :cas [24 [19 46]] INFO [2019-12-17 12:38:10,325] jepsen worker 50 - jepsen.util 17600 :invoke :read [16 nil] INFO [2019-12-17 12:38:10,329] jepsen worker 80 - jepsen.util 12230 :fail :cas [26 [1 37]] :ValueNotSwapFrom INFO [2019-12-17 12:38:10,329] jepsen worker 80 - jepsen.util 12230 :invoke :read [26 nil] INFO [2019-12-17 12:38:10,334] jepsen worker 16 - jepsen.util 19006 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:38:10,335] jepsen worker 16 - jepsen.util 19006 :invoke :write [5 17] INFO [2019-12-17 12:38:10,335] jepsen worker 48 - jepsen.util 16338 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:38:10,335] jepsen worker 47 - jepsen.util 13907 :info :write [15 36] :RequestCanceledException INFO [2019-12-17 12:38:10,335] jepsen worker 48 - jepsen.util 16338 :invoke :cas [16 [44 44]] INFO [2019-12-17 12:38:10,337] jepsen worker 47 - jepsen.util 13997 :invoke :write [15 22] INFO [2019-12-17 12:38:10,349] jepsen worker 58 - jepsen.util 17788 :info :write [19 10] :RequestCanceledException INFO [2019-12-17 12:38:10,349] jepsen worker 74 - jepsen.util 12674 :info :write [24 24] :RequestCanceledException INFO [2019-12-17 12:38:10,349] jepsen worker 25 - jepsen.util 18655 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:38:10,349] jepsen worker 25 - jepsen.util 18655 :invoke :write [8 21] INFO [2019-12-17 12:38:10,354] jepsen worker 58 - jepsen.util 17878 :invoke :write [19 20] INFO [2019-12-17 12:38:10,358] jepsen worker 74 - jepsen.util 12764 :invoke :cas [24 [4 48]] INFO [2019-12-17 12:38:10,362] jepsen worker 7 - jepsen.util 17467 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:38:10,362] jepsen worker 7 - jepsen.util 17467 :invoke :read [2 nil] INFO [2019-12-17 12:38:10,380] jepsen worker 73 - jepsen.util 11953 :fail :cas [24 [19 46]] :ValueNotSwapFrom INFO [2019-12-17 12:38:10,380] jepsen worker 73 - jepsen.util 11953 :invoke :write [24 20] INFO [2019-12-17 12:38:10,381] jepsen worker 28 - jepsen.util 11008 :ok :read [9 48] INFO [2019-12-17 12:38:10,381] jepsen worker 28 - jepsen.util 11008 :invoke :write [9 26] INFO [2019-12-17 12:38:10,393] jepsen worker 81 - jepsen.util 17001 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:38:10,393] jepsen worker 81 - jepsen.util 17001 :invoke :cas [27 [7 2]] INFO [2019-12-17 12:38:10,434] jepsen worker 8 - jepsen.util 17018 :info :cas [2 [26 14]] :RequestCanceledException INFO [2019-12-17 12:38:10,434] jepsen worker 61 - jepsen.util 18601 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:38:10,434] jepsen worker 57 - jepsen.util 17247 :info :write [19 29] :RequestCanceledException INFO [2019-12-17 12:38:10,434] jepsen worker 59 - jepsen.util 16619 :info :cas [19 [26 10]] :RequestCanceledException INFO [2019-12-17 12:38:10,434] jepsen worker 61 - jepsen.util 18601 :invoke :cas [20 [49 47]] INFO [2019-12-17 12:38:10,434] jepsen worker 56 - jepsen.util 13466 :info :write [18 37] :RequestCanceledException INFO [2019-12-17 12:38:10,435] jepsen worker 79 - jepsen.util 13039 :info :write [26 48] :RequestCanceledException INFO [2019-12-17 12:38:10,435] jepsen worker 56 - jepsen.util 13556 :invoke :read [18 nil] INFO [2019-12-17 12:38:10,437] jepsen worker 57 - jepsen.util 17337 :invoke :read [19 nil] INFO [2019-12-17 12:38:10,438] jepsen worker 8 - jepsen.util 17108 :invoke :read [2 nil] INFO [2019-12-17 12:38:10,439] jepsen worker 59 - jepsen.util 16709 :invoke :cas [19 [23 42]] INFO [2019-12-17 12:38:10,439] jepsen worker 79 - jepsen.util 13129 :invoke :read [26 nil] INFO [2019-12-17 12:38:10,441] jepsen worker 70 - jepsen.util 13300 :ok :read [23 27] INFO [2019-12-17 12:38:10,441] jepsen worker 70 - jepsen.util 13300 :invoke :write [23 27] INFO [2019-12-17 12:38:10,466] jepsen worker 43 - jepsen.util 17683 :info :write [14 30] :RequestCanceledException INFO [2019-12-17 12:38:10,473] jepsen worker 43 - jepsen.util 17773 :invoke :write [14 20] INFO [2019-12-17 12:38:10,501] jepsen worker 5 - jepsen.util 12425 :info :write [1 18] :RequestCanceledException INFO [2019-12-17 12:38:10,502] jepsen worker 12 - jepsen.util 12972 :fail :cas [4 [29 31]] :ValueNotSwapFrom INFO [2019-12-17 12:38:10,502] jepsen worker 12 - jepsen.util 12972 :invoke :read [4 nil] INFO [2019-12-17 12:38:10,502] jepsen worker 5 - jepsen.util 12515 :invoke :read [1 nil] INFO [2019-12-17 12:38:10,519] jepsen worker 29 - jepsen.util 11189 :fail :read [9 nil] :RequestTimeoutException INFO [2019-12-17 12:38:10,519] jepsen worker 29 - jepsen.util 11189 :invoke :read [9 nil] INFO [2019-12-17 12:38:10,533] jepsen worker 89 - jepsen.util 12779 :fail :cas [29 [30 18]] :ValueNotSwapFrom INFO [2019-12-17 12:38:10,533] jepsen worker 89 - jepsen.util 12779 :invoke :cas [29 [38 49]] INFO [2019-12-17 12:38:10,535] jepsen worker 72 - jepsen.util 12942 :info :write [24 33] :RequestCanceledException INFO [2019-12-17 12:38:10,535] jepsen worker 71 - jepsen.util 13571 :info :write [23 11] :RequestCanceledException INFO [2019-12-17 12:38:10,536] jepsen worker 72 - jepsen.util 13032 :invoke :cas [24 [40 16]] INFO [2019-12-17 12:38:10,537] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":74203,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:38:10,538] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":74203,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:38:10,538] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":74206,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:38:10,538] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":74206,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:38:10,538] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":74208,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:38:10,538] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":74208,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:38:10,543] jepsen worker 71 - jepsen.util 13661 :invoke :cas [23 [47 33]] INFO [2019-12-17 12:38:10,565] jepsen worker 83 - jepsen.util 17543 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:38:10,566] jepsen worker 83 - jepsen.util 17543 :invoke :read [27 nil] INFO [2019-12-17 12:38:10,567] jepsen worker 4 - jepsen.util 11974 :ok :read [1 31] INFO [2019-12-17 12:38:10,568] jepsen worker 4 - jepsen.util 11974 :invoke :read [1 nil] INFO [2019-12-17 12:38:10,619] jepsen worker 42 - jepsen.util 18402 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:38:10,620] jepsen worker 42 - jepsen.util 18402 :invoke :cas [14 [28 43]] INFO [2019-12-17 12:38:10,635] jepsen worker 53 - jepsen.util 12293 :info :write [17 43] :RequestCanceledException INFO [2019-12-17 12:38:10,636] jepsen worker 53 - jepsen.util 12383 :invoke :cas [17 [37 28]] INFO [2019-12-17 12:38:10,637] jepsen worker 4 - jepsen.util 11974 :ok :read [1 31] INFO [2019-12-17 12:38:10,637] jepsen worker 4 - jepsen.util 11974 :invoke :read [1 nil] INFO [2019-12-17 12:38:10,646] jepsen worker 78 - jepsen.util 12498 :fail :cas [26 [41 49]] :ValueNotSwapFrom INFO [2019-12-17 12:38:10,646] jepsen worker 78 - jepsen.util 12498 :invoke :read [26 nil] INFO [2019-12-17 12:38:10,666] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":74227,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:38:10,666] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":74227,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:38:10,666] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":74231,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:38:10,666] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":74231,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:38:10,701] jepsen worker 39 - jepsen.util 16959 :info :cas [13 [22 34]] :RequestCanceledException INFO [2019-12-17 12:38:10,701] jepsen worker 1 - jepsen.util 17821 :info :write [0 7] :RequestCanceledException INFO [2019-12-17 12:38:10,705] jepsen worker 1 - jepsen.util 17911 :invoke :read [0 nil] INFO [2019-12-17 12:38:10,705] jepsen worker 39 - jepsen.util 17049 :invoke :cas [13 [34 45]] INFO [2019-12-17 12:38:10,719] jepsen worker 22 - jepsen.util 17752 :info :cas [7 [5 27]] :RequestCanceledException INFO [2019-12-17 12:38:10,719] jepsen worker 27 - jepsen.util 11907 :info :cas [9 [28 2]] :RequestCanceledException INFO [2019-12-17 12:38:10,723] jepsen worker 27 - jepsen.util 11997 :invoke :read [9 nil] INFO [2019-12-17 12:38:10,724] jepsen worker 22 - jepsen.util 17842 :invoke :write [7 15] INFO [2019-12-17 12:38:10,735] jepsen worker 6 - jepsen.util 18006 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:38:10,735] jepsen worker 6 - jepsen.util 18006 :invoke :read [2 nil] INFO [2019-12-17 12:38:10,749] jepsen worker 68 - jepsen.util 16718 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:38:10,749] jepsen worker 68 - jepsen.util 16718 :invoke :cas [22 [28 8]] INFO [2019-12-17 12:38:10,759] jepsen worker 31 - jepsen.util 11101 :ok :write [10 16] INFO [2019-12-17 12:38:10,759] jepsen worker 11 - jepsen.util 12611 :ok :write [3 24] INFO [2019-12-17 12:38:10,759] jepsen worker 31 - jepsen.util 11101 :invoke :read [10 nil] INFO [2019-12-17 12:38:10,759] jepsen worker 11 - jepsen.util 12611 :invoke :cas [3 [38 40]] INFO [2019-12-17 12:38:10,766] jepsen worker 73 - jepsen.util 11953 :ok :write [24 20] INFO [2019-12-17 12:38:10,767] jepsen worker 47 - jepsen.util 13997 :ok :write [15 22] INFO [2019-12-17 12:38:10,767] jepsen worker 47 - jepsen.util 13997 :invoke :cas [15 [13 28]] INFO [2019-12-17 12:38:10,767] jepsen worker 73 - jepsen.util 11953 :invoke :cas [24 [17 21]] INFO [2019-12-17 12:38:10,771] jepsen worker 18 - jepsen.util 13338 :ok :write [6 4] INFO [2019-12-17 12:38:10,771] jepsen worker 18 - jepsen.util 13338 :invoke :cas [6 [49 28]] INFO [2019-12-17 12:38:10,780] jepsen worker 78 - jepsen.util 12498 :ok :read [26 7] INFO [2019-12-17 12:38:10,781] jepsen worker 78 - jepsen.util 12498 :invoke :write [26 46] INFO [2019-12-17 12:38:10,801] jepsen worker 10 - jepsen.util 12610 :info :write [3 48] :RequestCanceledException INFO [2019-12-17 12:38:10,804] jepsen worker 10 - jepsen.util 12700 :invoke :read [3 nil] INFO [2019-12-17 12:38:10,819] jepsen worker 27 - jepsen.util 11997 :ok :read [9 48] INFO [2019-12-17 12:38:10,819] jepsen worker 27 - jepsen.util 11997 :invoke :read [9 nil] INFO [2019-12-17 12:38:10,821] jepsen worker 73 - jepsen.util 11953 :fail :cas [24 [17 21]] :ValueNotSwapFrom INFO [2019-12-17 12:38:10,822] jepsen worker 73 - jepsen.util 11953 :invoke :cas [24 [30 26]] INFO [2019-12-17 12:38:10,834] jepsen worker 76 - jepsen.util 17536 :info :cas [25 [23 46]] :RequestCanceledException INFO [2019-12-17 12:38:10,840] jepsen worker 18 - jepsen.util 13338 :fail :cas [6 [49 28]] :ValueNotSwapFrom INFO [2019-12-17 12:38:10,840] jepsen worker 18 - jepsen.util 13338 :invoke :read [6 nil] INFO [2019-12-17 12:38:10,841] jepsen worker 76 - jepsen.util 17626 :invoke :cas [25 [24 42]] INFO [2019-12-17 12:38:10,864] jepsen worker 45 - jepsen.util 12555 :ok :read [15 10] INFO [2019-12-17 12:38:10,864] jepsen worker 45 - jepsen.util 12555 :invoke :cas [15 [37 22]] INFO [2019-12-17 12:38:10,890] jepsen worker 53 - jepsen.util 12383 :fail :cas [17 [37 28]] :ValueNotSwapFrom INFO [2019-12-17 12:38:10,890] jepsen worker 53 - jepsen.util 12383 :invoke :write [17 4] INFO [2019-12-17 12:38:10,897] jepsen worker 65 - jepsen.util 11135 :fail :read [21 nil] :RequestTimeoutException INFO [2019-12-17 12:38:10,897] jepsen worker 65 - jepsen.util 11135 :invoke :write [21 11] INFO [2019-12-17 12:38:10,900] jepsen worker 74 - jepsen.util 12764 :fail :cas [24 [4 48]] :ValueNotSwapFrom INFO [2019-12-17 12:38:10,900] jepsen worker 74 - jepsen.util 12764 :invoke :read [24 nil] INFO [2019-12-17 12:38:10,925] jepsen worker 32 - jepsen.util 13172 :info :cas [10 [1 20]] :RequestCanceledException INFO [2019-12-17 12:38:10,926] jepsen worker 32 - jepsen.util 13262 :invoke :read [10 nil] INFO [2019-12-17 12:38:10,952] jepsen worker 72 - jepsen.util 13032 :fail :cas [24 [40 16]] :ValueNotSwapFrom INFO [2019-12-17 12:38:10,952] jepsen worker 72 - jepsen.util 13032 :invoke :read [24 nil] INFO [2019-12-17 12:38:10,961] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":74254,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:38:10,961] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":74254,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:38:10,962] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":74255,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:38:10,962] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":74255,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:38:10,965] jepsen worker 49 - jepsen.util 17419 :info :cas [16 [28 6]] :RequestCanceledException INFO [2019-12-17 12:38:10,965] jepsen worker 34 - jepsen.util 18394 :info :cas [11 [24 37]] :RequestCanceledException INFO [2019-12-17 12:38:10,972] jepsen worker 29 - jepsen.util 11189 :ok :read [9 48] INFO [2019-12-17 12:38:10,972] jepsen worker 29 - jepsen.util 11189 :invoke :write [9 23] INFO [2019-12-17 12:38:10,972] jepsen worker 34 - jepsen.util 18484 :invoke :cas [11 [43 37]] INFO [2019-12-17 12:38:10,972] jepsen worker 49 - jepsen.util 17509 :invoke :write [16 6] INFO [2019-12-17 12:38:10,980] jepsen worker 31 - jepsen.util 11101 :ok :read [10 16] INFO [2019-12-17 12:38:10,980] jepsen worker 31 - jepsen.util 11101 :invoke :write [10 18] INFO [2019-12-17 12:38:10,987] jepsen worker 11 - jepsen.util 12611 :fail :cas [3 [38 40]] :ValueNotSwapFrom INFO [2019-12-17 12:38:10,987] jepsen worker 11 - jepsen.util 12611 :invoke :read [3 nil] INFO [2019-12-17 12:38:11,001] jepsen worker 41 - jepsen.util 18851 :info :write [13 20] :RequestCanceledException INFO [2019-12-17 12:38:11,001] jepsen worker 40 - jepsen.util 18220 :info :cas [13 [47 28]] :RequestCanceledException INFO [2019-12-17 12:38:11,008] jepsen worker 40 - jepsen.util 18310 :invoke :cas [13 [25 14]] INFO [2019-12-17 12:38:11,008] jepsen worker 56 - jepsen.util 13556 :ok :read [18 43] INFO [2019-12-17 12:38:11,009] jepsen worker 56 - jepsen.util 13556 :invoke :write [18 20] INFO [2019-12-17 12:38:11,009] jepsen worker 41 - jepsen.util 18941 :invoke :cas [13 [49 32]] INFO [2019-12-17 12:38:11,022] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:38:11,022] jepsen nemesis - couchbase.util Rebalance check count: 585 INFO [2019-12-17 12:38:11,023] jepsen nemesis - couchbase.util Rebalance stuck count: 566 INFO [2019-12-17 12:38:11,034] jepsen worker 30 - jepsen.util 12090 :info :write [10 18] :RequestCanceledException INFO [2019-12-17 12:38:11,034] jepsen worker 20 - jepsen.util 12800 :fail :read [6 nil] :RequestTimeoutException INFO [2019-12-17 12:38:11,034] jepsen worker 20 - jepsen.util 12800 :invoke :write [6 40] INFO [2019-12-17 12:38:11,036] jepsen worker 30 - jepsen.util 12180 :invoke :write [10 31] INFO [2019-12-17 12:38:11,048] jepsen worker 24 - jepsen.util 16494 :info :cas [8 [23 16]] :RequestCanceledException INFO [2019-12-17 12:38:11,056] jepsen worker 24 - jepsen.util 16584 :invoke :cas [8 [31 37]] INFO [2019-12-17 12:38:11,074] jepsen worker 4 - jepsen.util 11974 :ok :read [1 31] INFO [2019-12-17 12:38:11,075] jepsen worker 4 - jepsen.util 11974 :invoke :write [1 29] INFO [2019-12-17 12:38:11,091] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":74266,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:38:11,091] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":74266,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:38:11,091] jepsen worker 27 - jepsen.util 11997 :ok :read [9 48] INFO [2019-12-17 12:38:11,092] jepsen worker 27 - jepsen.util 11997 :invoke :cas [9 [6 27]] INFO [2019-12-17 12:38:11,092] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":74270,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:38:11,092] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":74270,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:38:11,098] jepsen worker 74 - jepsen.util 12764 :ok :read [24 20] INFO [2019-12-17 12:38:11,098] jepsen worker 74 - jepsen.util 12764 :invoke :write [24 14] INFO [2019-12-17 12:38:11,101] jepsen worker 36 - jepsen.util 13536 :info :write [12 8] :RequestCanceledException INFO [2019-12-17 12:38:11,102] jepsen worker 10 - jepsen.util 12700 :ok :read [3 24] INFO [2019-12-17 12:38:11,102] jepsen worker 10 - jepsen.util 12700 :invoke :write [3 14] INFO [2019-12-17 12:38:11,109] jepsen worker 45 - jepsen.util 12555 :fail :cas [15 [37 22]] :ValueNotSwapFrom INFO [2019-12-17 12:38:11,110] jepsen worker 45 - jepsen.util 12555 :invoke :cas [15 [17 22]] INFO [2019-12-17 12:38:11,110] jepsen worker 28 - jepsen.util 11008 :ok :write [9 26] INFO [2019-12-17 12:38:11,110] jepsen worker 28 - jepsen.util 11008 :invoke :read [9 nil] INFO [2019-12-17 12:38:11,111] jepsen worker 36 - jepsen.util 13626 :invoke :write [12 46] INFO [2019-12-17 12:38:11,119] jepsen worker 69 - jepsen.util 13119 :info :write [23 1] :RequestCanceledException INFO [2019-12-17 12:38:11,126] jepsen worker 69 - jepsen.util 13209 :invoke :read [23 nil] INFO [2019-12-17 12:38:11,154] jepsen worker 11 - jepsen.util 12611 :ok :read [3 24] INFO [2019-12-17 12:38:11,154] jepsen worker 18 - jepsen.util 13338 :ok :read [6 4] INFO [2019-12-17 12:38:11,154] jepsen worker 11 - jepsen.util 12611 :invoke :write [3 5] INFO [2019-12-17 12:38:11,154] jepsen worker 18 - jepsen.util 13338 :invoke :write [6 13] INFO [2019-12-17 12:38:11,165] jepsen worker 3 - jepsen.util 12333 :info :write [1 36] :RequestCanceledException INFO [2019-12-17 12:38:11,168] jepsen worker 3 - jepsen.util 12423 :invoke :write [1 0] INFO [2019-12-17 12:38:11,181] jepsen worker 5 - jepsen.util 12515 :ok :read [1 31] INFO [2019-12-17 12:38:11,181] jepsen worker 5 - jepsen.util 12515 :invoke :cas [1 [47 11]] INFO [2019-12-17 12:38:11,201] jepsen worker 62 - jepsen.util 18242 :info :cas [20 [10 7]] :RequestCanceledException INFO [2019-12-17 12:38:11,210] jepsen worker 62 - jepsen.util 18332 :invoke :read [20 nil] INFO [2019-12-17 12:38:11,211] jepsen worker 72 - jepsen.util 13032 :ok :read [24 20] INFO [2019-12-17 12:38:11,212] jepsen worker 72 - jepsen.util 13032 :invoke :write [24 6] INFO [2019-12-17 12:38:11,224] jepsen worker 73 - jepsen.util 11953 :fail :cas [24 [30 26]] :ValueNotSwapFrom INFO [2019-12-17 12:38:11,224] jepsen worker 73 - jepsen.util 11953 :invoke :read [24 nil] INFO [2019-12-17 12:38:11,234] jepsen worker 86 - jepsen.util 18266 :info :cas [28 [5 22]] :RequestCanceledException INFO [2019-12-17 12:38:11,240] jepsen worker 86 - jepsen.util 18356 :invoke :read [28 nil] INFO [2019-12-17 12:38:11,255] jepsen worker 65 - jepsen.util 11135 :ok :write [21 11] INFO [2019-12-17 12:38:11,256] jepsen worker 65 - jepsen.util 11135 :invoke :write [21 32] INFO [2019-12-17 12:38:11,259] jepsen worker 80 - jepsen.util 12230 :ok :read [26 7] INFO [2019-12-17 12:38:11,259] jepsen worker 80 - jepsen.util 12230 :invoke :cas [26 [8 31]] INFO [2019-12-17 12:38:11,267] jepsen worker 47 - jepsen.util 13997 :fail :cas [15 [13 28]] :ValueNotSwapFrom INFO [2019-12-17 12:38:11,268] jepsen worker 47 - jepsen.util 13997 :invoke :write [15 4] INFO [2019-12-17 12:38:11,293] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":74290,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:38:11,294] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":74290,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:38:11,294] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":74291,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:38:11,294] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":74291,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:38:11,295] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":74292,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:38:11,295] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":74292,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:38:11,301] jepsen worker 27 - jepsen.util 11997 :fail :cas [9 [6 27]] :ValueNotSwapFrom INFO [2019-12-17 12:38:11,301] jepsen worker 27 - jepsen.util 11997 :invoke :write [9 0] INFO [2019-12-17 12:38:11,307] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":74298,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:38:11,307] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":74298,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:38:11,307] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":74301,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:38:11,307] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":74301,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:38:11,308] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":74302,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:38:11,308] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":74302,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:38:11,334] jepsen worker 75 - jepsen.util 17085 :info :write [25 3] :RequestCanceledException INFO [2019-12-17 12:38:11,334] jepsen worker 60 - jepsen.util 18510 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:38:11,334] jepsen worker 60 - jepsen.util 18510 :invoke :write [20 41] INFO [2019-12-17 12:38:11,344] jepsen worker 75 - jepsen.util 17175 :invoke :cas [25 [11 28]] INFO [2019-12-17 12:38:11,345] jepsen worker 79 - jepsen.util 13129 :ok :read [26 7] INFO [2019-12-17 12:38:11,345] jepsen worker 79 - jepsen.util 13129 :invoke :write [26 46] INFO [2019-12-17 12:38:11,347] jepsen worker 29 - jepsen.util 11189 :ok :write [9 23] INFO [2019-12-17 12:38:11,347] jepsen worker 29 - jepsen.util 11189 :invoke :write [9 2] INFO [2019-12-17 12:38:11,348] jepsen worker 17 - jepsen.util 17567 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:38:11,349] jepsen worker 17 - jepsen.util 17567 :invoke :cas [5 [14 19]] INFO [2019-12-17 12:38:11,365] jepsen worker 26 - jepsen.util 18386 :info :cas [8 [12 33]] :RequestCanceledException INFO [2019-12-17 12:38:11,372] jepsen worker 5 - jepsen.util 12515 :fail :cas [1 [47 11]] :ValueNotSwapFrom INFO [2019-12-17 12:38:11,372] jepsen worker 5 - jepsen.util 12515 :invoke :read [1 nil] INFO [2019-12-17 12:38:11,373] jepsen worker 26 - jepsen.util 18476 :invoke :write [8 27] INFO [2019-12-17 12:38:11,388] jepsen worker 32 - jepsen.util 13262 :ok :read [10 16] INFO [2019-12-17 12:38:11,388] jepsen worker 32 - jepsen.util 13262 :invoke :read [10 nil] INFO [2019-12-17 12:38:11,392] jepsen worker 71 - jepsen.util 13661 :fail :cas [23 [47 33]] :ValueNotSwapFrom INFO [2019-12-17 12:38:11,392] jepsen worker 71 - jepsen.util 13661 :invoke :read [23 nil] INFO [2019-12-17 12:38:11,411] jepsen worker 69 - jepsen.util 13209 :ok :read [23 27] INFO [2019-12-17 12:38:11,411] jepsen worker 69 - jepsen.util 13209 :invoke :write [23 17] INFO [2019-12-17 12:38:11,419] jepsen worker 14 - jepsen.util 12344 :info :write [4 0] :RequestCanceledException INFO [2019-12-17 12:38:11,423] jepsen worker 14 - jepsen.util 12434 :invoke :write [4 36] INFO [2019-12-17 12:38:11,456] jepsen worker 27 - jepsen.util 11997 :ok :write [9 0] INFO [2019-12-17 12:38:11,457] jepsen worker 27 - jepsen.util 11997 :invoke :write [9 33] INFO [2019-12-17 12:38:11,500] jepsen worker 12 - jepsen.util 12972 :ok :read [4 13] INFO [2019-12-17 12:38:11,501] jepsen worker 12 - jepsen.util 12972 :invoke :cas [4 [23 7]] INFO [2019-12-17 12:38:11,506] jepsen worker 29 - jepsen.util 11189 :ok :write [9 2] INFO [2019-12-17 12:38:11,506] jepsen worker 29 - jepsen.util 11189 :invoke :read [9 nil] INFO [2019-12-17 12:38:11,519] jepsen worker 67 - jepsen.util 17797 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:38:11,519] jepsen worker 67 - jepsen.util 17797 :invoke :cas [22 [38 49]] INFO [2019-12-17 12:38:11,521] jepsen worker 65 - jepsen.util 11135 :ok :write [21 32] INFO [2019-12-17 12:38:11,522] jepsen worker 65 - jepsen.util 11135 :invoke :cas [21 [11 39]] INFO [2019-12-17 12:38:11,527] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":74323,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:38:11,527] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":74323,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:38:11,527] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":74324,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:38:11,527] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":74324,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:38:11,528] jepsen worker 89 - jepsen.util 12779 :fail :cas [29 [38 49]] :ValueNotSwapFrom INFO [2019-12-17 12:38:11,528] jepsen worker 89 - jepsen.util 12779 :invoke :read [29 nil] INFO [2019-12-17 12:38:11,535] jepsen worker 51 - jepsen.util 12021 :info :write [17 34] :RequestCanceledException INFO [2019-12-17 12:38:11,535] jepsen worker 66 - jepsen.util 17526 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:38:11,535] jepsen worker 85 - jepsen.util 18085 :info :cas [28 [3 36]] :RequestCanceledException INFO [2019-12-17 12:38:11,535] jepsen worker 66 - jepsen.util 17526 :invoke :cas [22 [30 33]] INFO [2019-12-17 12:38:11,538] jepsen worker 51 - jepsen.util 12111 :invoke :write [17 20] INFO [2019-12-17 12:38:11,541] jepsen worker 85 - jepsen.util 18175 :invoke :read [28 nil] INFO [2019-12-17 12:38:11,549] jepsen worker 52 - jepsen.util 12382 :info :write [17 29] :RequestCanceledException INFO [2019-12-17 12:38:11,549] jepsen worker 64 - jepsen.util 9964 :fail :read [21 nil] :RequestTimeoutException INFO [2019-12-17 12:38:11,549] jepsen worker 64 - jepsen.util 9964 :invoke :write [21 38] INFO [2019-12-17 12:38:11,554] jepsen worker 52 - jepsen.util 12472 :invoke :cas [17 [43 19]] INFO [2019-12-17 12:38:11,565] jepsen worker 33 - jepsen.util 17403 :info :write [11 32] :RequestCanceledException INFO [2019-12-17 12:38:11,573] jepsen worker 33 - jepsen.util 17493 :invoke :write [11 13] INFO [2019-12-17 12:38:11,582] jepsen worker 27 - jepsen.util 11997 :ok :write [9 33] INFO [2019-12-17 12:38:11,582] jepsen worker 27 - jepsen.util 11997 :invoke :write [9 45] INFO [2019-12-17 12:38:11,583] jepsen worker 20 - jepsen.util 12800 :ok :write [6 40] INFO [2019-12-17 12:38:11,584] jepsen worker 20 - jepsen.util 12800 :invoke :write [6 10] INFO [2019-12-17 12:38:11,602] jepsen worker 31 - jepsen.util 11101 :ok :write [10 18] INFO [2019-12-17 12:38:11,602] jepsen worker 31 - jepsen.util 11101 :invoke :read [10 nil] INFO [2019-12-17 12:38:11,617] jepsen worker 29 - jepsen.util 11189 :ok :read [9 2] INFO [2019-12-17 12:38:11,617] jepsen worker 29 - jepsen.util 11189 :invoke :cas [9 [5 25]] INFO [2019-12-17 12:38:11,618] jepsen worker 19 - jepsen.util 13429 :info :write [6 17] :RequestCanceledException INFO [2019-12-17 12:38:11,619] jepsen worker 65 - jepsen.util 11135 :fail :cas [21 [11 39]] :ValueNotSwapFrom INFO [2019-12-17 12:38:11,619] jepsen worker 65 - jepsen.util 11135 :invoke :cas [21 [4 19]] INFO [2019-12-17 12:38:11,620] jepsen worker 19 - jepsen.util 13519 :invoke :cas [6 [45 10]] INFO [2019-12-17 12:38:11,633] jepsen worker 13 - jepsen.util 12433 :info :write [4 29] :RequestCanceledException INFO [2019-12-17 12:38:11,634] jepsen worker 13 - jepsen.util 12523 :invoke :cas [4 [46 4]] INFO [2019-12-17 12:38:11,648] jepsen worker 87 - jepsen.util 12237 :info :write [29 25] :RequestCanceledException INFO [2019-12-17 12:38:11,648] jepsen worker 84 - jepsen.util 17004 :info :write [28 1] :RequestCanceledException INFO [2019-12-17 12:38:11,650] jepsen worker 87 - jepsen.util 12327 :invoke :cas [29 [3 12]] INFO [2019-12-17 12:38:11,655] jepsen worker 84 - jepsen.util 17094 :invoke :read [28 nil] INFO [2019-12-17 12:38:11,796] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":74347,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:38:11,796] jepsen worker 10 - jepsen.util 12700 :ok :write [3 14] INFO [2019-12-17 12:38:11,796] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":74347,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:38:11,796] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":74350,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:38:11,796] jepsen worker 21 - jepsen.util 17121 :info :write [7 34] :RequestCanceledException INFO [2019-12-17 12:38:11,796] jepsen worker 10 - jepsen.util 12700 :invoke :write [3 40] INFO [2019-12-17 12:38:11,796] jepsen worker 0 - jepsen.util 17910 :info :cas [0 [18 1]] :RequestCanceledException INFO [2019-12-17 12:38:11,796] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":74350,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:38:11,796] jepsen worker 38 - jepsen.util 13448 :info :write [12 25] :RequestCanceledException INFO [2019-12-17 12:38:11,796] jepsen worker 9 - jepsen.util 13329 :fail :read [3 nil] :RequestTimeoutException INFO [2019-12-17 12:38:11,796] jepsen worker 37 - jepsen.util 12277 :info :write [12 49] :RequestCanceledException INFO [2019-12-17 12:38:11,796] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":74351,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:38:11,796] jepsen worker 23 - jepsen.util 17933 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:38:11,797] jepsen worker 9 - jepsen.util 13329 :invoke :write [3 19] INFO [2019-12-17 12:38:11,797] jepsen worker 21 - jepsen.util 17211 :invoke :read [7 nil] INFO [2019-12-17 12:38:11,797] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":74351,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:38:11,797] jepsen worker 30 - jepsen.util 12180 :ok :write [10 31] INFO [2019-12-17 12:38:11,797] jepsen worker 23 - jepsen.util 17933 :invoke :read [7 nil] INFO [2019-12-17 12:38:11,797] jepsen worker 38 - jepsen.util 13538 :invoke :cas [12 [44 19]] INFO [2019-12-17 12:38:11,798] jepsen worker 30 - jepsen.util 12180 :invoke :write [10 28] INFO [2019-12-17 12:38:11,798] jepsen worker 37 - jepsen.util 12367 :invoke :write [12 46] INFO [2019-12-17 12:38:11,798] jepsen worker 0 - jepsen.util 18000 :invoke :write [0 8] INFO [2019-12-17 12:38:11,805] jepsen worker 65 - jepsen.util 11135 :fail :cas [21 [4 19]] :ValueNotSwapFrom INFO [2019-12-17 12:38:11,805] jepsen worker 65 - jepsen.util 11135 :invoke :cas [21 [47 44]] INFO [2019-12-17 12:38:11,807] jepsen worker 29 - jepsen.util 11189 :fail :cas [9 [5 25]] :ValueNotSwapFrom INFO [2019-12-17 12:38:11,807] jepsen worker 29 - jepsen.util 11189 :invoke :read [9 nil] INFO [2019-12-17 12:38:11,809] jepsen worker 19 - jepsen.util 13519 :fail :cas [6 [45 10]] :ValueNotSwapFrom INFO [2019-12-17 12:38:11,809] jepsen worker 19 - jepsen.util 13519 :invoke :write [6 34] INFO [2019-12-17 12:38:11,810] jepsen worker 5 - jepsen.util 12515 :ok :read [1 31] INFO [2019-12-17 12:38:11,810] jepsen worker 5 - jepsen.util 12515 :invoke :write [1 33] INFO [2019-12-17 12:38:11,816] jepsen worker 82 - jepsen.util 17812 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:38:11,816] jepsen worker 82 - jepsen.util 17812 :invoke :read [27 nil] INFO [2019-12-17 12:38:11,865] jepsen worker 55 - jepsen.util 13465 :info :write [18 37] :RequestCanceledException INFO [2019-12-17 12:38:11,869] jepsen worker 55 - jepsen.util 13555 :invoke :write [18 17] INFO [2019-12-17 12:38:11,882] jepsen worker 65 - jepsen.util 11135 :fail :cas [21 [47 44]] :ValueNotSwapFrom INFO [2019-12-17 12:38:11,882] jepsen worker 65 - jepsen.util 11135 :invoke :read [21 nil] INFO [2019-12-17 12:38:11,883] jepsen worker 72 - jepsen.util 13032 :ok :write [24 6] INFO [2019-12-17 12:38:11,883] jepsen worker 72 - jepsen.util 13032 :invoke :cas [24 [5 19]] INFO [2019-12-17 12:38:11,906] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":74368,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:38:11,906] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":74368,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:38:11,906] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":74372,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:38:11,906] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":74372,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:38:11,917] jepsen worker 4 - jepsen.util 11974 :ok :write [1 29] INFO [2019-12-17 12:38:11,918] jepsen worker 4 - jepsen.util 11974 :invoke :read [1 nil] INFO [2019-12-17 12:38:11,942] jepsen worker 53 - jepsen.util 12383 :ok :write [17 4] INFO [2019-12-17 12:38:11,943] jepsen worker 53 - jepsen.util 12383 :invoke :write [17 16] INFO [2019-12-17 12:38:11,951] jepsen worker 65 - jepsen.util 11135 :ok :read [21 32] INFO [2019-12-17 12:38:11,951] jepsen worker 65 - jepsen.util 11135 :invoke :write [21 40] INFO [2019-12-17 12:38:11,977] jepsen worker 70 - jepsen.util 13300 :ok :write [23 27] INFO [2019-12-17 12:38:11,977] jepsen worker 70 - jepsen.util 13300 :invoke :cas [23 [28 34]] INFO [2019-12-17 12:38:11,987] jepsen worker 71 - jepsen.util 13661 :ok :read [23 27] INFO [2019-12-17 12:38:11,988] jepsen worker 71 - jepsen.util 13661 :invoke :read [23 nil] INFO [2019-12-17 12:38:11,991] jepsen worker 64 - jepsen.util 9964 :ok :write [21 38] INFO [2019-12-17 12:38:11,991] jepsen worker 64 - jepsen.util 9964 :invoke :write [21 17] INFO [2019-12-17 12:38:12,002] jepsen worker 27 - jepsen.util 11997 :ok :write [9 45] INFO [2019-12-17 12:38:12,002] jepsen worker 27 - jepsen.util 11997 :invoke :read [9 nil] INFO [2019-12-17 12:38:12,008] jepsen worker 72 - jepsen.util 13032 :fail :cas [24 [5 19]] :ValueNotSwapFrom INFO [2019-12-17 12:38:12,008] jepsen worker 72 - jepsen.util 13032 :invoke :read [24 nil] INFO [2019-12-17 12:38:12,016] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":74384,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:38:12,016] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":74384,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:38:12,016] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":74387,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:38:12,016] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":74387,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:38:12,026] jepsen worker 80 - jepsen.util 12230 :fail :cas [26 [8 31]] :ValueNotSwapFrom INFO [2019-12-17 12:38:12,026] jepsen worker 80 - jepsen.util 12230 :invoke :cas [26 [27 46]] INFO [2019-12-17 12:38:12,039] jepsen worker 27 - jepsen.util 11997 :ok :read [9 45] INFO [2019-12-17 12:38:12,039] jepsen worker 27 - jepsen.util 11997 :invoke :write [9 7] INFO [2019-12-17 12:38:12,039] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:38:12,039] jepsen nemesis - couchbase.util Rebalance check count: 586 INFO [2019-12-17 12:38:12,039] jepsen nemesis - couchbase.util Rebalance stuck count: 567 INFO [2019-12-17 12:38:12,061] jepsen worker 52 - jepsen.util 12472 :fail :cas [17 [43 19]] :ValueNotSwapFrom INFO [2019-12-17 12:38:12,061] jepsen worker 52 - jepsen.util 12472 :invoke :read [17 nil] INFO [2019-12-17 12:38:12,083] jepsen worker 9 - jepsen.util 13329 :ok :write [3 19] INFO [2019-12-17 12:38:12,084] jepsen worker 9 - jepsen.util 13329 :invoke :write [3 23] INFO [2019-12-17 12:38:12,092] jepsen worker 12 - jepsen.util 12972 :fail :cas [4 [23 7]] :ValueNotSwapFrom INFO [2019-12-17 12:38:12,092] jepsen worker 12 - jepsen.util 12972 :invoke :write [4 0] INFO [2019-12-17 12:38:12,104] jepsen worker 20 - jepsen.util 12800 :ok :write [6 10] INFO [2019-12-17 12:38:12,104] jepsen worker 20 - jepsen.util 12800 :invoke :read [6 nil] INFO [2019-12-17 12:38:12,108] jepsen worker 29 - jepsen.util 11189 :ok :read [9 45] INFO [2019-12-17 12:38:12,108] jepsen worker 29 - jepsen.util 11189 :invoke :read [9 nil] INFO [2019-12-17 12:38:12,114] jepsen worker 89 - jepsen.util 12779 :ok :read [29 42] INFO [2019-12-17 12:38:12,114] jepsen worker 89 - jepsen.util 12779 :invoke :cas [29 [19 4]] INFO [2019-12-17 12:38:12,133] jepsen worker 52 - jepsen.util 12472 :ok :read [17 4] INFO [2019-12-17 12:38:12,133] jepsen worker 52 - jepsen.util 12472 :invoke :write [17 20] INFO [2019-12-17 12:38:12,139] jepsen worker 20 - jepsen.util 12800 :ok :read [6 10] INFO [2019-12-17 12:38:12,139] jepsen worker 20 - jepsen.util 12800 :invoke :read [6 nil] INFO [2019-12-17 12:38:12,148] jepsen worker 46 - jepsen.util 12466 :fail :read [15 nil] :RequestTimeoutException INFO [2019-12-17 12:38:12,148] jepsen worker 46 - jepsen.util 12466 :invoke :read [15 nil] INFO [2019-12-17 12:38:12,153] jepsen worker 27 - jepsen.util 11997 :ok :write [9 7] INFO [2019-12-17 12:38:12,153] jepsen worker 27 - jepsen.util 11997 :invoke :write [9 14] INFO [2019-12-17 12:38:12,173] jepsen worker 20 - jepsen.util 12800 :ok :read [6 10] INFO [2019-12-17 12:38:12,173] jepsen worker 20 - jepsen.util 12800 :invoke :read [6 nil] INFO [2019-12-17 12:38:12,174] jepsen worker 64 - jepsen.util 9964 :ok :write [21 17] INFO [2019-12-17 12:38:12,174] jepsen worker 64 - jepsen.util 9964 :invoke :read [21 nil] INFO [2019-12-17 12:38:12,174] jepsen worker 88 - jepsen.util 12868 :ok :write [29 1] INFO [2019-12-17 12:38:12,174] jepsen worker 88 - jepsen.util 12868 :invoke :write [29 32] INFO [2019-12-17 12:38:12,196] jepsen worker 46 - jepsen.util 12466 :ok :read [15 22] INFO [2019-12-17 12:38:12,197] jepsen worker 46 - jepsen.util 12466 :invoke :write [15 23] INFO [2019-12-17 12:38:12,198] jepsen worker 20 - jepsen.util 12800 :ok :read [6 10] INFO [2019-12-17 12:38:12,198] jepsen worker 20 - jepsen.util 12800 :invoke :read [6 nil] INFO [2019-12-17 12:38:12,209] jepsen worker 64 - jepsen.util 9964 :ok :read [21 17] INFO [2019-12-17 12:38:12,210] jepsen worker 64 - jepsen.util 9964 :invoke :cas [21 [43 7]] INFO [2019-12-17 12:38:12,211] jepsen worker 38 - jepsen.util 13538 :fail :cas [12 [44 19]] :ValueNotSwapFrom INFO [2019-12-17 12:38:12,211] jepsen worker 38 - jepsen.util 13538 :invoke :read [12 nil] INFO [2019-12-17 12:38:12,218] jepsen worker 63 - jepsen.util 11403 :info :write [21 13] :RequestCanceledException INFO [2019-12-17 12:38:12,218] jepsen worker 77 - jepsen.util 18707 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:38:12,218] jepsen worker 77 - jepsen.util 18707 :invoke :write [25 39] INFO [2019-12-17 12:38:12,219] jepsen worker 63 - jepsen.util 11493 :invoke :write [21 31] INFO [2019-12-17 12:38:12,222] jepsen worker 69 - jepsen.util 13209 :ok :write [23 17] INFO [2019-12-17 12:38:12,222] jepsen worker 69 - jepsen.util 13209 :invoke :write [23 28] INFO [2019-12-17 12:38:12,227] jepsen worker 20 - jepsen.util 12800 :ok :read [6 10] INFO [2019-12-17 12:38:12,227] jepsen worker 20 - jepsen.util 12800 :invoke :cas [6 [23 7]] INFO [2019-12-17 12:38:12,233] jepsen worker 9 - jepsen.util 13329 :ok :write [3 23] INFO [2019-12-17 12:38:12,234] jepsen worker 9 - jepsen.util 13329 :invoke :read [3 nil] INFO [2019-12-17 12:38:12,235] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":74415,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:38:12,235] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":74407,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:38:12,235] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":74415,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:38:12,235] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":74407,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:38:12,235] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":74416,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:38:12,235] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":74409,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:38:12,235] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":74416,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:38:12,235] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":74409,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:38:12,235] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":74417,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:38:12,235] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":74417,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:38:12,248] jepsen worker 54 - jepsen.util 13374 :info :write [18 28] :RequestCanceledException INFO [2019-12-17 12:38:12,249] jepsen worker 54 - jepsen.util 13464 :invoke :read [18 nil] INFO [2019-12-17 12:38:12,260] jepsen worker 27 - jepsen.util 11997 :ok :write [9 14] INFO [2019-12-17 12:38:12,260] jepsen worker 27 - jepsen.util 11997 :invoke :read [9 nil] INFO [2019-12-17 12:38:12,267] jepsen worker 4 - jepsen.util 11974 :ok :read [1 29] INFO [2019-12-17 12:38:12,267] jepsen worker 4 - jepsen.util 11974 :invoke :read [1 nil] INFO [2019-12-17 12:38:12,285] jepsen worker 70 - jepsen.util 13300 :fail :cas [23 [28 34]] :ValueNotSwapFrom INFO [2019-12-17 12:38:12,286] jepsen worker 70 - jepsen.util 13300 :invoke :write [23 19] INFO [2019-12-17 12:38:12,294] jepsen worker 71 - jepsen.util 13661 :ok :read [23 17] INFO [2019-12-17 12:38:12,294] jepsen worker 71 - jepsen.util 13661 :invoke :write [23 13] INFO [2019-12-17 12:38:12,303] jepsen worker 20 - jepsen.util 12800 :fail :cas [6 [23 7]] :ValueNotSwapFrom INFO [2019-12-17 12:38:12,303] jepsen worker 20 - jepsen.util 12800 :invoke :cas [6 [2 39]] INFO [2019-12-17 12:38:12,303] jepsen worker 27 - jepsen.util 11997 :ok :read [9 14] INFO [2019-12-17 12:38:12,303] jepsen worker 27 - jepsen.util 11997 :invoke :write [9 4] INFO [2019-12-17 12:38:12,306] jepsen worker 9 - jepsen.util 13329 :ok :read [3 23] INFO [2019-12-17 12:38:12,306] jepsen worker 9 - jepsen.util 13329 :invoke :write [3 21] INFO [2019-12-17 12:38:12,319] jepsen worker 2 - jepsen.util 16832 :info :write [0 30] :RequestCanceledException INFO [2019-12-17 12:38:12,320] jepsen worker 2 - jepsen.util 16922 :invoke :cas [0 [36 38]] INFO [2019-12-17 12:38:12,330] jepsen worker 80 - jepsen.util 12230 :fail :cas [26 [27 46]] :ValueNotSwapFrom INFO [2019-12-17 12:38:12,330] jepsen worker 80 - jepsen.util 12230 :invoke :read [26 nil] INFO [2019-12-17 12:38:12,334] jepsen worker 15 - jepsen.util 16755 :info :cas [5 [36 47]] :RequestCanceledException INFO [2019-12-17 12:38:12,335] jepsen worker 15 - jepsen.util 16845 :invoke :read [5 nil] INFO [2019-12-17 12:38:12,348] jepsen worker 35 - jepsen.util 17675 :info :cas [11 [16 23]] :RequestCanceledException INFO [2019-12-17 12:38:12,349] jepsen worker 35 - jepsen.util 17765 :invoke :cas [11 [32 37]] INFO [2019-12-17 12:38:12,365] jepsen worker 50 - jepsen.util 17600 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:38:12,365] jepsen worker 44 - jepsen.util 18224 :info :cas [14 [33 24]] :RequestCanceledException INFO [2019-12-17 12:38:12,366] jepsen worker 50 - jepsen.util 17600 :invoke :read [16 nil] INFO [2019-12-17 12:38:12,367] jepsen worker 44 - jepsen.util 18314 :invoke :cas [14 [0 23]] INFO [2019-12-17 12:38:12,378] jepsen worker 20 - jepsen.util 12800 :fail :cas [6 [2 39]] :ValueNotSwapFrom INFO [2019-12-17 12:38:12,378] jepsen worker 20 - jepsen.util 12800 :invoke :write [6 46] INFO [2019-12-17 12:38:12,401] jepsen worker 58 - jepsen.util 17878 :info :write [19 20] :RequestCanceledException INFO [2019-12-17 12:38:12,401] jepsen worker 81 - jepsen.util 17001 :info :cas [27 [7 2]] :RequestCanceledException INFO [2019-12-17 12:38:12,402] jepsen worker 58 - jepsen.util 17968 :invoke :read [19 nil] INFO [2019-12-17 12:38:12,402] jepsen worker 81 - jepsen.util 17091 :invoke :cas [27 [3 30]] INFO [2019-12-17 12:38:12,418] jepsen worker 79 - jepsen.util 13129 :ok :write [26 46] INFO [2019-12-17 12:38:12,418] jepsen worker 79 - jepsen.util 13129 :invoke :cas [26 [16 9]] INFO [2019-12-17 12:38:12,421] jepsen worker 63 - jepsen.util 11493 :ok :write [21 31] INFO [2019-12-17 12:38:12,432] jepsen worker 16 - jepsen.util 19006 :info :write [5 17] :RequestCanceledException INFO [2019-12-17 12:38:12,432] jepsen worker 48 - jepsen.util 16338 :info :cas [16 [44 44]] :RequestCanceledException INFO [2019-12-17 12:38:12,449] jepsen worker 57 - jepsen.util 17337 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:38:12,449] jepsen worker 25 - jepsen.util 18655 :info :write [8 21] :RequestCanceledException INFO [2019-12-17 12:38:12,454] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":74445,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:38:12,456] jepsen worker 27 - jepsen.util 11997 :ok :write [9 4] INFO [2019-12-17 12:38:12,465] jepsen worker 8 - jepsen.util 17108 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:38:12,465] jepsen worker 7 - jepsen.util 17467 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:38:12,467] jepsen worker 9 - jepsen.util 13329 :ok :write [3 21] INFO [2019-12-17 12:38:12,468] jepsen worker 89 - jepsen.util 12779 :fail :cas [29 [19 4]] :ValueNotSwapFrom INFO [2019-12-17 12:38:12,477] jepsen worker 63 - jepsen.util 11493 :invoke :write [21 28] INFO [2019-12-17 12:38:12,477] jepsen worker 57 - jepsen.util 17337 :invoke :read [19 nil] INFO [2019-12-17 12:38:12,477] jepsen worker 27 - jepsen.util 11997 :invoke :write [9 20] INFO [2019-12-17 12:38:12,477] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":74445,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:38:12,477] jepsen worker 8 - jepsen.util 17108 :invoke :write [2 7] INFO [2019-12-17 12:38:12,477] jepsen worker 7 - jepsen.util 17467 :invoke :read [2 nil] INFO [2019-12-17 12:38:12,477] jepsen worker 9 - jepsen.util 13329 :invoke :cas [3 [43 48]] INFO [2019-12-17 12:38:12,478] jepsen worker 89 - jepsen.util 12779 :invoke :read [29 nil] INFO [2019-12-17 12:38:12,478] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":74449,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:38:12,478] jepsen worker 48 - jepsen.util 16428 :invoke :cas [16 [40 15]] INFO [2019-12-17 12:38:12,478] jepsen worker 16 - jepsen.util 19096 :invoke :cas [5 [17 33]] INFO [2019-12-17 12:38:12,478] jepsen worker 25 - jepsen.util 18745 :invoke :read [8 nil] INFO [2019-12-17 12:38:12,478] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":74449,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:38:12,507] jepsen worker 36 - jepsen.util 13626 :ok :write [12 46] INFO [2019-12-17 12:38:12,507] jepsen worker 36 - jepsen.util 13626 :invoke :read [12 nil] INFO [2019-12-17 12:38:12,517] jepsen worker 87 - jepsen.util 12327 :fail :cas [29 [3 12]] :ValueNotSwapFrom INFO [2019-12-17 12:38:12,517] jepsen worker 87 - jepsen.util 12327 :invoke :cas [29 [13 45]] INFO [2019-12-17 12:38:12,519] jepsen worker 43 - jepsen.util 17773 :info :write [14 20] :RequestCanceledException INFO [2019-12-17 12:38:12,520] jepsen worker 43 - jepsen.util 17863 :invoke :cas [14 [42 30]] INFO [2019-12-17 12:38:12,522] jepsen worker 9 - jepsen.util 13329 :fail :cas [3 [43 48]] :ValueNotSwapFrom INFO [2019-12-17 12:38:12,522] jepsen worker 9 - jepsen.util 13329 :invoke :read [3 nil] INFO [2019-12-17 12:38:12,531] jepsen worker 80 - jepsen.util 12230 :ok :read [26 46] INFO [2019-12-17 12:38:12,531] jepsen worker 80 - jepsen.util 12230 :invoke :read [26 nil] INFO [2019-12-17 12:38:12,534] jepsen worker 61 - jepsen.util 18601 :info :cas [20 [49 47]] :RequestCanceledException INFO [2019-12-17 12:38:12,535] jepsen worker 59 - jepsen.util 16709 :info :cas [19 [23 42]] :RequestCanceledException INFO [2019-12-17 12:38:12,535] jepsen worker 61 - jepsen.util 18691 :invoke :cas [20 [35 22]] INFO [2019-12-17 12:38:12,536] jepsen worker 59 - jepsen.util 16799 :invoke :write [19 48] INFO [2019-12-17 12:38:12,545] jepsen worker 9 - jepsen.util 13329 :ok :read [3 21] INFO [2019-12-17 12:38:12,545] jepsen worker 9 - jepsen.util 13329 :invoke :cas [3 [10 12]] INFO [2019-12-17 12:38:12,547] jepsen worker 38 - jepsen.util 13538 :ok :read [12 46] INFO [2019-12-17 12:38:12,547] jepsen worker 38 - jepsen.util 13538 :invoke :write [12 29] INFO [2019-12-17 12:38:12,562] jepsen worker 9 - jepsen.util 13329 :fail :cas [3 [10 12]] :ValueNotSwapFrom INFO [2019-12-17 12:38:12,562] jepsen worker 9 - jepsen.util 13329 :invoke :cas [3 [37 36]] INFO [2019-12-17 12:38:12,596] jepsen worker 46 - jepsen.util 12466 :ok :write [15 23] INFO [2019-12-17 12:38:12,596] jepsen worker 46 - jepsen.util 12466 :invoke :write [15 8] INFO [2019-12-17 12:38:12,599] jepsen worker 9 - jepsen.util 13329 :fail :cas [3 [37 36]] :ValueNotSwapFrom INFO [2019-12-17 12:38:12,599] jepsen worker 9 - jepsen.util 13329 :invoke :write [3 21] INFO [2019-12-17 12:38:12,628] jepsen worker 63 - jepsen.util 11493 :ok :write [21 28] INFO [2019-12-17 12:38:12,628] jepsen worker 63 - jepsen.util 11493 :invoke :cas [21 [44 16]] INFO [2019-12-17 12:38:12,633] jepsen worker 87 - jepsen.util 12327 :fail :cas [29 [13 45]] :ValueNotSwapFrom INFO [2019-12-17 12:38:12,633] jepsen worker 87 - jepsen.util 12327 :invoke :write [29 8] INFO [2019-12-17 12:38:12,645] jepsen worker 20 - jepsen.util 12800 :ok :write [6 46] INFO [2019-12-17 12:38:12,645] jepsen worker 20 - jepsen.util 12800 :invoke :cas [6 [9 14]] INFO [2019-12-17 12:38:12,650] jepsen worker 27 - jepsen.util 11997 :ok :write [9 20] INFO [2019-12-17 12:38:12,650] jepsen worker 27 - jepsen.util 11997 :invoke :cas [9 [14 30]] INFO [2019-12-17 12:38:12,665] jepsen worker 4 - jepsen.util 11974 :ok :read [1 29] INFO [2019-12-17 12:38:12,665] jepsen worker 4 - jepsen.util 11974 :invoke :cas [1 [46 38]] INFO [2019-12-17 12:38:12,666] jepsen worker 83 - jepsen.util 17543 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:38:12,666] jepsen worker 83 - jepsen.util 17543 :invoke :write [27 33] INFO [2019-12-17 12:38:12,669] jepsen worker 63 - jepsen.util 11493 :fail :cas [21 [44 16]] :ValueNotSwapFrom INFO [2019-12-17 12:38:12,669] jepsen worker 63 - jepsen.util 11493 :invoke :write [21 44] INFO [2019-12-17 12:38:12,676] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":74476,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:38:12,676] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":74476,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:38:12,676] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":74479,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:38:12,676] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":74479,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:38:12,676] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":74480,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:38:12,676] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":74480,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:38:12,678] jepsen worker 27 - jepsen.util 11997 :fail :cas [9 [14 30]] :ValueNotSwapFrom INFO [2019-12-17 12:38:12,678] jepsen worker 27 - jepsen.util 11997 :invoke :read [9 nil] INFO [2019-12-17 12:38:12,693] jepsen worker 80 - jepsen.util 12230 :ok :read [26 46] INFO [2019-12-17 12:38:12,693] jepsen worker 80 - jepsen.util 12230 :invoke :write [26 25] INFO [2019-12-17 12:38:12,699] jepsen worker 27 - jepsen.util 11997 :ok :read [9 20] INFO [2019-12-17 12:38:12,699] jepsen worker 27 - jepsen.util 11997 :invoke :write [9 2] INFO [2019-12-17 12:38:12,705] jepsen worker 54 - jepsen.util 13464 :ok :read [18 43] INFO [2019-12-17 12:38:12,705] jepsen worker 54 - jepsen.util 13464 :invoke :cas [18 [39 17]] INFO [2019-12-17 12:38:12,710] jepsen worker 36 - jepsen.util 13626 :ok :read [12 46] INFO [2019-12-17 12:38:12,710] jepsen worker 36 - jepsen.util 13626 :invoke :write [12 35] INFO [2019-12-17 12:38:12,720] jepsen worker 42 - jepsen.util 18402 :info :cas [14 [28 43]] :RequestCanceledException INFO [2019-12-17 12:38:12,720] jepsen worker 39 - jepsen.util 17049 :info :cas [13 [34 45]] :RequestCanceledException INFO [2019-12-17 12:38:12,721] jepsen worker 42 - jepsen.util 18492 :invoke :cas [14 [48 44]] INFO [2019-12-17 12:38:12,721] jepsen worker 39 - jepsen.util 17139 :invoke :cas [13 [34 38]] INFO [2019-12-17 12:38:12,722] jepsen worker 20 - jepsen.util 12800 :fail :cas [6 [9 14]] :ValueNotSwapFrom INFO [2019-12-17 12:38:12,722] jepsen worker 20 - jepsen.util 12800 :invoke :write [6 12] INFO [2019-12-17 12:38:12,762] jepsen worker 4 - jepsen.util 11974 :fail :cas [1 [46 38]] :ValueNotSwapFrom INFO [2019-12-17 12:38:12,762] jepsen worker 4 - jepsen.util 11974 :invoke :read [1 nil] INFO [2019-12-17 12:38:12,763] jepsen worker 63 - jepsen.util 11493 :ok :write [21 44] INFO [2019-12-17 12:38:12,763] jepsen worker 63 - jepsen.util 11493 :invoke :write [21 1] INFO [2019-12-17 12:38:12,766] jepsen worker 13 - jepsen.util 12523 :fail :cas [4 [46 4]] :ValueNotSwapFrom INFO [2019-12-17 12:38:12,766] jepsen worker 13 - jepsen.util 12523 :invoke :cas [4 [34 47]] INFO [2019-12-17 12:38:12,780] jepsen worker 27 - jepsen.util 11997 :ok :write [9 2] INFO [2019-12-17 12:38:12,780] jepsen worker 27 - jepsen.util 11997 :invoke :read [9 nil] INFO [2019-12-17 12:38:12,801] jepsen worker 1 - jepsen.util 17911 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:38:12,801] jepsen worker 1 - jepsen.util 17911 :invoke :write [0 5] INFO [2019-12-17 12:38:12,812] jepsen worker 27 - jepsen.util 11997 :ok :read [9 2] INFO [2019-12-17 12:38:12,813] jepsen worker 27 - jepsen.util 11997 :invoke :write [9 45] INFO [2019-12-17 12:38:12,819] jepsen worker 9 - jepsen.util 13329 :ok :write [3 21] INFO [2019-12-17 12:38:12,819] jepsen worker 9 - jepsen.util 13329 :invoke :cas [3 [35 25]] INFO [2019-12-17 12:38:12,831] jepsen worker 6 - jepsen.util 18006 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:38:12,831] jepsen worker 6 - jepsen.util 18006 :invoke :cas [2 [19 9]] INFO [2019-12-17 12:38:12,848] jepsen worker 22 - jepsen.util 17842 :info :write [7 15] :RequestCanceledException INFO [2019-12-17 12:38:12,848] jepsen worker 68 - jepsen.util 16718 :info :cas [22 [28 8]] :RequestCanceledException INFO [2019-12-17 12:38:12,849] jepsen worker 22 - jepsen.util 17932 :invoke :write [7 33] INFO [2019-12-17 12:38:12,849] jepsen worker 68 - jepsen.util 16808 :invoke :cas [22 [42 10]] INFO [2019-12-17 12:38:12,858] jepsen worker 9 - jepsen.util 13329 :fail :cas [3 [35 25]] :ValueNotSwapFrom INFO [2019-12-17 12:38:12,859] jepsen worker 9 - jepsen.util 13329 :invoke :cas [3 [24 45]] INFO [2019-12-17 12:38:12,863] jepsen worker 78 - jepsen.util 12498 :info :write [26 46] :RequestCanceledException INFO [2019-12-17 12:38:12,864] jepsen worker 78 - jepsen.util 12588 :invoke :cas [26 [40 15]] INFO [2019-12-17 12:38:12,881] jepsen worker 9 - jepsen.util 13329 :fail :cas [3 [24 45]] :ValueNotSwapFrom INFO [2019-12-17 12:38:12,882] jepsen worker 9 - jepsen.util 13329 :invoke :write [3 42] INFO [2019-12-17 12:38:12,885] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":74506,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:38:12,885] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":74506,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:38:12,885] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":74509,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:38:12,885] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":74509,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:38:12,911] jepsen worker 52 - jepsen.util 12472 :ok :write [17 20] INFO [2019-12-17 12:38:12,911] jepsen worker 52 - jepsen.util 12472 :invoke :write [17 8] INFO [2019-12-17 12:38:12,922] jepsen worker 20 - jepsen.util 12800 :ok :write [6 12] INFO [2019-12-17 12:38:12,922] jepsen worker 46 - jepsen.util 12466 :ok :write [15 8] INFO [2019-12-17 12:38:12,922] jepsen worker 20 - jepsen.util 12800 :invoke :write [6 40] INFO [2019-12-17 12:38:12,922] jepsen worker 46 - jepsen.util 12466 :invoke :read [15 nil] INFO [2019-12-17 12:38:12,934] jepsen worker 76 - jepsen.util 17626 :info :cas [25 [24 42]] :RequestCanceledException INFO [2019-12-17 12:38:12,935] jepsen worker 76 - jepsen.util 17716 :invoke :write [25 9] INFO [2019-12-17 12:38:12,963] jepsen worker 46 - jepsen.util 12466 :ok :read [15 8] INFO [2019-12-17 12:38:12,963] jepsen worker 46 - jepsen.util 12466 :invoke :write [15 31] INFO [2019-12-17 12:38:12,984] jepsen worker 79 - jepsen.util 13129 :fail :cas [26 [16 9]] :ValueNotSwapFrom INFO [2019-12-17 12:38:12,984] jepsen worker 79 - jepsen.util 13129 :invoke :cas [26 [37 49]] INFO [2019-12-17 12:38:13,012] jepsen worker 63 - jepsen.util 11493 :ok :write [21 1] INFO [2019-12-17 12:38:13,012] jepsen worker 63 - jepsen.util 11493 :invoke :write [21 25] INFO [2019-12-17 12:38:13,014] jepsen worker 89 - jepsen.util 12779 :ok :read [29 1] INFO [2019-12-17 12:38:13,014] jepsen worker 89 - jepsen.util 12779 :invoke :write [29 14] INFO [2019-12-17 12:38:13,018] jepsen worker 34 - jepsen.util 18484 :info :cas [11 [43 37]] :RequestCanceledException INFO [2019-12-17 12:38:13,021] jepsen worker 34 - jepsen.util 18574 :invoke :cas [11 [24 46]] INFO [2019-12-17 12:38:13,035] jepsen worker 49 - jepsen.util 17509 :info :write [16 6] :RequestCanceledException INFO [2019-12-17 12:38:13,036] jepsen worker 49 - jepsen.util 17599 :invoke :write [16 45] INFO [2019-12-17 12:38:13,036] jepsen worker 54 - jepsen.util 13464 :fail :cas [18 [39 17]] :ValueNotSwapFrom INFO [2019-12-17 12:38:13,036] jepsen worker 54 - jepsen.util 13464 :invoke :write [18 13] INFO [2019-12-17 12:38:13,048] jepsen worker 40 - jepsen.util 18310 :info :cas [13 [25 14]] :RequestCanceledException INFO [2019-12-17 12:38:13,049] jepsen worker 40 - jepsen.util 18400 :invoke :read [13 nil] INFO [2019-12-17 12:38:13,049] jepsen worker 27 - jepsen.util 11997 :ok :write [9 45] INFO [2019-12-17 12:38:13,049] jepsen worker 27 - jepsen.util 11997 :invoke :write [9 14] INFO [2019-12-17 12:38:13,054] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:38:13,055] jepsen nemesis - couchbase.util Rebalance check count: 587 INFO [2019-12-17 12:38:13,055] jepsen nemesis - couchbase.util Rebalance stuck count: 568 INFO [2019-12-17 12:38:13,065] jepsen worker 41 - jepsen.util 18941 :info :cas [13 [49 32]] :RequestCanceledException INFO [2019-12-17 12:38:13,066] jepsen worker 41 - jepsen.util 19031 :invoke :read [13 nil] INFO [2019-12-17 12:38:13,081] jepsen worker 36 - jepsen.util 13626 :ok :write [12 35] INFO [2019-12-17 12:38:13,082] jepsen worker 36 - jepsen.util 13626 :invoke :read [12 nil] INFO [2019-12-17 12:38:13,090] jepsen worker 63 - jepsen.util 11493 :ok :write [21 25] INFO [2019-12-17 12:38:13,090] jepsen worker 63 - jepsen.util 11493 :invoke :write [21 13] INFO [2019-12-17 12:38:13,101] jepsen worker 24 - jepsen.util 16584 :info :cas [8 [31 37]] :RequestCanceledException INFO [2019-12-17 12:38:13,103] jepsen worker 24 - jepsen.util 16674 :invoke :write [8 36] INFO [2019-12-17 12:38:13,107] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":74519,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:38:13,107] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":74519,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:38:13,107] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":74523,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:38:13,107] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":74523,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:38:13,111] jepsen worker 20 - jepsen.util 12800 :ok :write [6 40] INFO [2019-12-17 12:38:13,111] jepsen worker 20 - jepsen.util 12800 :invoke :write [6 29] INFO [2019-12-17 12:38:13,112] jepsen worker 9 - jepsen.util 13329 :ok :write [3 42] INFO [2019-12-17 12:38:13,112] jepsen worker 9 - jepsen.util 13329 :invoke :write [3 13] INFO [2019-12-17 12:38:13,129] jepsen worker 46 - jepsen.util 12466 :ok :write [15 31] INFO [2019-12-17 12:38:13,130] jepsen worker 46 - jepsen.util 12466 :invoke :read [15 nil] INFO [2019-12-17 12:38:13,134] jepsen worker 28 - jepsen.util 11008 :fail :read [9 nil] :RequestTimeoutException INFO [2019-12-17 12:38:13,135] jepsen worker 56 - jepsen.util 13556 :info :write [18 20] :RequestCanceledException INFO [2019-12-17 12:38:13,135] jepsen worker 28 - jepsen.util 11008 :invoke :read [9 nil] INFO [2019-12-17 12:38:13,136] jepsen worker 56 - jepsen.util 13646 :invoke :read [18 nil] INFO [2019-12-17 12:38:13,145] jepsen worker 4 - jepsen.util 11974 :ok :read [1 29] INFO [2019-12-17 12:38:13,145] jepsen worker 4 - jepsen.util 11974 :invoke :write [1 39] INFO [2019-12-17 12:38:13,148] jepsen worker 45 - jepsen.util 12555 :info :cas [15 [17 22]] :RequestCanceledException INFO [2019-12-17 12:38:13,149] jepsen worker 45 - jepsen.util 12645 :invoke :write [15 40] INFO [2019-12-17 12:38:13,157] jepsen worker 46 - jepsen.util 12466 :ok :read [15 31] INFO [2019-12-17 12:38:13,157] jepsen worker 46 - jepsen.util 12466 :invoke :write [15 6] INFO [2019-12-17 12:38:13,159] jepsen worker 27 - jepsen.util 11997 :ok :write [9 14] INFO [2019-12-17 12:38:13,159] jepsen worker 27 - jepsen.util 11997 :invoke :write [9 13] INFO [2019-12-17 12:38:13,180] jepsen worker 78 - jepsen.util 12588 :fail :cas [26 [40 15]] :ValueNotSwapFrom INFO [2019-12-17 12:38:13,180] jepsen worker 78 - jepsen.util 12588 :invoke :write [26 4] INFO [2019-12-17 12:38:13,187] jepsen worker 36 - jepsen.util 13626 :ok :read [12 35] INFO [2019-12-17 12:38:13,187] jepsen worker 36 - jepsen.util 13626 :invoke :read [12 nil] INFO [2019-12-17 12:38:13,188] jepsen worker 63 - jepsen.util 11493 :ok :write [21 13] INFO [2019-12-17 12:38:13,188] jepsen worker 63 - jepsen.util 11493 :invoke :read [21 nil] INFO [2019-12-17 12:38:13,216] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":74547,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:38:13,217] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":74547,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:38:13,217] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":74548,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:38:13,217] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":74548,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:38:13,219] jepsen worker 74 - jepsen.util 12764 :info :write [24 14] :RequestCanceledException INFO [2019-12-17 12:38:13,219] jepsen worker 11 - jepsen.util 12611 :info :write [3 5] :RequestCanceledException INFO [2019-12-17 12:38:13,220] jepsen worker 74 - jepsen.util 12854 :invoke :cas [24 [36 21]] INFO [2019-12-17 12:38:13,220] jepsen worker 11 - jepsen.util 12701 :invoke :write [3 43] INFO [2019-12-17 12:38:13,234] jepsen worker 18 - jepsen.util 13338 :info :write [6 13] :RequestCanceledException INFO [2019-12-17 12:38:13,235] jepsen worker 18 - jepsen.util 13428 :invoke :write [6 29] INFO [2019-12-17 12:38:13,238] jepsen worker 63 - jepsen.util 11493 :ok :read [21 13] INFO [2019-12-17 12:38:13,238] jepsen worker 63 - jepsen.util 11493 :invoke :read [21 nil] INFO [2019-12-17 12:38:13,240] jepsen worker 36 - jepsen.util 13626 :ok :read [12 35] INFO [2019-12-17 12:38:13,241] jepsen worker 36 - jepsen.util 13626 :invoke :write [12 5] INFO [2019-12-17 12:38:13,241] jepsen worker 3 - jepsen.util 12423 :info :write [1 0] :RequestCanceledException INFO [2019-12-17 12:38:13,243] jepsen worker 3 - jepsen.util 12513 :invoke :read [1 nil] INFO [2019-12-17 12:38:13,265] jepsen worker 73 - jepsen.util 11953 :fail :read [24 nil] :RequestTimeoutException INFO [2019-12-17 12:38:13,265] jepsen worker 62 - jepsen.util 18332 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:38:13,265] jepsen worker 73 - jepsen.util 11953 :invoke :write [24 25] INFO [2019-12-17 12:38:13,265] jepsen worker 62 - jepsen.util 18332 :invoke :read [20 nil] INFO [2019-12-17 12:38:13,288] jepsen worker 63 - jepsen.util 11493 :ok :read [21 13] INFO [2019-12-17 12:38:13,288] jepsen worker 63 - jepsen.util 11493 :invoke :read [21 nil] INFO [2019-12-17 12:38:13,291] jepsen worker 27 - jepsen.util 11997 :ok :write [9 13] INFO [2019-12-17 12:38:13,291] jepsen worker 27 - jepsen.util 11997 :invoke :read [9 nil] INFO [2019-12-17 12:38:13,326] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":74558,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:38:13,326] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":74558,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:38:13,334] jepsen worker 86 - jepsen.util 18356 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:38:13,334] jepsen worker 47 - jepsen.util 13997 :info :write [15 4] :RequestCanceledException INFO [2019-12-17 12:38:13,334] jepsen worker 86 - jepsen.util 18356 :invoke :write [28 28] INFO [2019-12-17 12:38:13,335] jepsen worker 47 - jepsen.util 14087 :invoke :read [15 nil] INFO [2019-12-17 12:38:13,342] jepsen worker 13 - jepsen.util 12523 :fail :cas [4 [34 47]] :ValueNotSwapFrom INFO [2019-12-17 12:38:13,342] jepsen worker 13 - jepsen.util 12523 :invoke :write [4 13] INFO [2019-12-17 12:38:13,400] jepsen worker 75 - jepsen.util 17175 :info :cas [25 [11 28]] :RequestCanceledException INFO [2019-12-17 12:38:13,400] jepsen worker 32 - jepsen.util 13262 :fail :read [10 nil] :RequestTimeoutException INFO [2019-12-17 12:38:13,400] jepsen worker 32 - jepsen.util 13262 :invoke :read [10 nil] INFO [2019-12-17 12:38:13,401] jepsen worker 75 - jepsen.util 17265 :invoke :cas [25 [38 18]] INFO [2019-12-17 12:38:13,419] jepsen worker 26 - jepsen.util 18476 :info :write [8 27] :RequestCanceledException INFO [2019-12-17 12:38:13,420] jepsen worker 26 - jepsen.util 18566 :invoke :cas [8 [34 26]] INFO [2019-12-17 12:38:13,429] jepsen worker 3 - jepsen.util 12513 :ok :read [1 29] INFO [2019-12-17 12:38:13,429] jepsen worker 3 - jepsen.util 12513 :invoke :read [1 nil] INFO [2019-12-17 12:38:13,435] jepsen worker 60 - jepsen.util 18510 :info :write [20 41] :RequestCanceledException INFO [2019-12-17 12:38:13,436] jepsen worker 60 - jepsen.util 18600 :invoke :write [20 37] INFO [2019-12-17 12:38:13,446] jepsen worker 17 - jepsen.util 17567 :info :cas [5 [14 19]] :RequestCanceledException INFO [2019-12-17 12:38:13,447] jepsen worker 17 - jepsen.util 17657 :invoke :cas [5 [39 26]] INFO [2019-12-17 12:38:13,460] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":74579,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:38:13,460] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":74579,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:38:13,460] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":74581,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:38:13,460] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":74581,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:38:13,521] jepsen worker 79 - jepsen.util 13129 :fail :cas [26 [37 49]] :ValueNotSwapFrom INFO [2019-12-17 12:38:13,521] jepsen worker 79 - jepsen.util 13129 :invoke :read [26 nil] INFO [2019-12-17 12:38:13,532] jepsen worker 14 - jepsen.util 12434 :info :write [4 36] :RequestCanceledException INFO [2019-12-17 12:38:13,533] jepsen worker 14 - jepsen.util 12524 :invoke :read [4 nil] INFO [2019-12-17 12:38:13,549] jepsen worker 56 - jepsen.util 13646 :ok :read [18 43] INFO [2019-12-17 12:38:13,549] jepsen worker 56 - jepsen.util 13646 :invoke :write [18 9] INFO [2019-12-17 12:38:13,552] jepsen worker 14 - jepsen.util 12524 :ok :read [4 13] INFO [2019-12-17 12:38:13,552] jepsen worker 14 - jepsen.util 12524 :invoke :write [4 24] INFO [2019-12-17 12:38:13,566] jepsen worker 85 - jepsen.util 18175 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:38:13,566] jepsen worker 85 - jepsen.util 18175 :invoke :cas [28 [29 24]] INFO [2019-12-17 12:38:13,598] jepsen worker 9 - jepsen.util 13329 :ok :write [3 13] INFO [2019-12-17 12:38:13,598] jepsen worker 20 - jepsen.util 12800 :ok :write [6 29] INFO [2019-12-17 12:38:13,598] jepsen worker 9 - jepsen.util 13329 :invoke :read [3 nil] INFO [2019-12-17 12:38:13,598] jepsen worker 20 - jepsen.util 12800 :invoke :cas [6 [11 0]] INFO [2019-12-17 12:38:13,601] jepsen worker 33 - jepsen.util 17493 :info :write [11 13] :RequestCanceledException INFO [2019-12-17 12:38:13,602] jepsen worker 33 - jepsen.util 17583 :invoke :write [11 43] INFO [2019-12-17 12:38:13,617] jepsen worker 67 - jepsen.util 17797 :info :cas [22 [38 49]] :RequestCanceledException INFO [2019-12-17 12:38:13,619] jepsen worker 67 - jepsen.util 17887 :invoke :read [22 nil] INFO [2019-12-17 12:38:13,625] jepsen worker 74 - jepsen.util 12854 :fail :cas [24 [36 21]] :ValueNotSwapFrom INFO [2019-12-17 12:38:13,625] jepsen worker 74 - jepsen.util 12854 :invoke :read [24 nil] INFO [2019-12-17 12:38:13,627] jepsen worker 45 - jepsen.util 12645 :ok :write [15 40] INFO [2019-12-17 12:38:13,627] jepsen worker 73 - jepsen.util 11953 :ok :write [24 25] INFO [2019-12-17 12:38:13,627] jepsen worker 45 - jepsen.util 12645 :invoke :write [15 35] INFO [2019-12-17 12:38:13,627] jepsen worker 73 - jepsen.util 11953 :invoke :read [24 nil] INFO [2019-12-17 12:38:13,634] jepsen worker 66 - jepsen.util 17526 :info :cas [22 [30 33]] :RequestCanceledException INFO [2019-12-17 12:38:13,636] jepsen worker 66 - jepsen.util 17616 :invoke :cas [22 [43 13]] INFO [2019-12-17 12:38:13,639] jepsen worker 47 - jepsen.util 14087 :ok :read [15 40] INFO [2019-12-17 12:38:13,639] jepsen worker 47 - jepsen.util 14087 :invoke :cas [15 [24 5]] INFO [2019-12-17 12:38:13,648] jepsen worker 31 - jepsen.util 11101 :fail :read [10 nil] :RequestTimeoutException INFO [2019-12-17 12:38:13,648] jepsen worker 51 - jepsen.util 12111 :info :write [17 20] :RequestCanceledException INFO [2019-12-17 12:38:13,648] jepsen worker 31 - jepsen.util 11101 :invoke :write [10 45] INFO [2019-12-17 12:38:13,649] jepsen worker 51 - jepsen.util 12201 :invoke :cas [17 [37 17]] INFO [2019-12-17 12:38:13,655] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":74601,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:38:13,655] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":74601,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:38:13,689] jepsen worker 3 - jepsen.util 12513 :ok :read [1 29] INFO [2019-12-17 12:38:13,689] jepsen worker 3 - jepsen.util 12513 :invoke :cas [1 [41 48]] INFO [2019-12-17 12:38:13,733] jepsen worker 9 - jepsen.util 13329 :ok :read [3 13] INFO [2019-12-17 12:38:13,733] jepsen worker 9 - jepsen.util 13329 :invoke :cas [3 [14 10]] INFO [2019-12-17 12:38:13,740] jepsen worker 13 - jepsen.util 12523 :ok :write [4 13] INFO [2019-12-17 12:38:13,740] jepsen worker 13 - jepsen.util 12523 :invoke :cas [4 [0 41]] INFO [2019-12-17 12:38:13,746] jepsen worker 73 - jepsen.util 11953 :ok :read [24 25] INFO [2019-12-17 12:38:13,747] jepsen worker 73 - jepsen.util 11953 :invoke :write [24 21] INFO [2019-12-17 12:38:13,765] jepsen worker 84 - jepsen.util 17094 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:38:13,766] jepsen worker 84 - jepsen.util 17094 :invoke :write [28 30] INFO [2019-12-17 12:38:13,772] jepsen worker 20 - jepsen.util 12800 :fail :cas [6 [11 0]] :ValueNotSwapFrom INFO [2019-12-17 12:38:13,772] jepsen worker 20 - jepsen.util 12800 :invoke :write [6 9] INFO [2019-12-17 12:38:13,773] jepsen worker 32 - jepsen.util 13262 :ok :read [10 31] INFO [2019-12-17 12:38:13,773] jepsen worker 32 - jepsen.util 13262 :invoke :read [10 nil] INFO [2019-12-17 12:38:13,817] jepsen worker 37 - jepsen.util 12367 :info :write [12 46] :RequestCanceledException INFO [2019-12-17 12:38:13,818] jepsen worker 37 - jepsen.util 12457 :invoke :cas [12 [23 41]] INFO [2019-12-17 12:38:13,821] jepsen worker 74 - jepsen.util 12854 :ok :read [24 25] INFO [2019-12-17 12:38:13,821] jepsen worker 74 - jepsen.util 12854 :invoke :cas [24 [8 5]] INFO [2019-12-17 12:38:13,826] jepsen worker 47 - jepsen.util 14087 :fail :cas [15 [24 5]] :ValueNotSwapFrom INFO [2019-12-17 12:38:13,826] jepsen worker 47 - jepsen.util 14087 :invoke :cas [15 [38 20]] INFO [2019-12-17 12:38:13,831] jepsen worker 78 - jepsen.util 12588 :ok :write [26 4] INFO [2019-12-17 12:38:13,831] jepsen worker 78 - jepsen.util 12588 :invoke :write [26 48] INFO [2019-12-17 12:38:13,834] jepsen worker 5 - jepsen.util 12515 :info :write [1 33] :RequestCanceledException INFO [2019-12-17 12:38:13,834] jepsen worker 0 - jepsen.util 18000 :info :write [0 8] :RequestCanceledException INFO [2019-12-17 12:38:13,834] jepsen worker 21 - jepsen.util 17211 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:38:13,834] jepsen worker 30 - jepsen.util 12180 :info :write [10 28] :RequestCanceledException INFO [2019-12-17 12:38:13,834] jepsen worker 21 - jepsen.util 17211 :invoke :write [7 16] INFO [2019-12-17 12:38:13,835] jepsen worker 0 - jepsen.util 18090 :invoke :cas [0 [15 22]] INFO [2019-12-17 12:38:13,835] jepsen worker 5 - jepsen.util 12605 :invoke :write [1 25] INFO [2019-12-17 12:38:13,835] jepsen worker 30 - jepsen.util 12270 :invoke :write [10 0] INFO [2019-12-17 12:38:13,853] jepsen worker 9 - jepsen.util 13329 :fail :cas [3 [14 10]] :ValueNotSwapFrom INFO [2019-12-17 12:38:13,853] jepsen worker 9 - jepsen.util 13329 :invoke :write [3 47] INFO [2019-12-17 12:38:13,865] jepsen worker 10 - jepsen.util 12700 :info :write [3 40] :RequestCanceledException INFO [2019-12-17 12:38:13,866] jepsen worker 10 - jepsen.util 12790 :invoke :cas [3 [2 11]] INFO [2019-12-17 12:38:13,873] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":74624,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:38:13,873] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":74624,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:38:13,873] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":74628,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:38:13,873] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":74628,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:38:13,877] jepsen worker 74 - jepsen.util 12854 :fail :cas [24 [8 5]] :ValueNotSwapFrom INFO [2019-12-17 12:38:13,877] jepsen worker 74 - jepsen.util 12854 :invoke :cas [24 [34 18]] INFO [2019-12-17 12:38:13,900] jepsen worker 23 - jepsen.util 17933 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:38:13,901] jepsen worker 23 - jepsen.util 17933 :invoke :read [7 nil] INFO [2019-12-17 12:38:13,918] jepsen worker 82 - jepsen.util 17812 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:38:13,918] jepsen worker 19 - jepsen.util 13519 :info :write [6 34] :RequestCanceledException INFO [2019-12-17 12:38:13,918] jepsen worker 82 - jepsen.util 17812 :invoke :read [27 nil] INFO [2019-12-17 12:38:13,919] jepsen worker 19 - jepsen.util 13609 :invoke :read [6 nil] INFO [2019-12-17 12:38:13,949] jepsen worker 55 - jepsen.util 13555 :info :write [18 17] :RequestCanceledException INFO [2019-12-17 12:38:13,950] jepsen worker 55 - jepsen.util 13645 :invoke :cas [18 [33 30]] INFO [2019-12-17 12:38:13,951] jepsen worker 19 - jepsen.util 13609 :ok :read [6 29] INFO [2019-12-17 12:38:13,951] jepsen worker 19 - jepsen.util 13609 :invoke :read [6 nil] INFO [2019-12-17 12:38:13,952] jepsen worker 51 - jepsen.util 12201 :fail :cas [17 [37 17]] :ValueNotSwapFrom INFO [2019-12-17 12:38:13,952] jepsen worker 51 - jepsen.util 12201 :invoke :write [17 23] INFO [2019-12-17 12:38:13,955] jepsen worker 79 - jepsen.util 13129 :ok :read [26 4] INFO [2019-12-17 12:38:13,955] jepsen worker 79 - jepsen.util 13129 :invoke :read [26 nil] INFO [2019-12-17 12:38:13,980] jepsen worker 19 - jepsen.util 13609 :ok :read [6 29] INFO [2019-12-17 12:38:13,980] jepsen worker 19 - jepsen.util 13609 :invoke :read [6 nil] INFO [2019-12-17 12:38:13,996] jepsen worker 45 - jepsen.util 12645 :ok :write [15 35] INFO [2019-12-17 12:38:13,997] jepsen worker 45 - jepsen.util 12645 :invoke :read [15 nil] INFO [2019-12-17 12:38:13,997] jepsen worker 73 - jepsen.util 11953 :ok :write [24 21] INFO [2019-12-17 12:38:13,997] jepsen worker 73 - jepsen.util 11953 :invoke :write [24 33] INFO [2019-12-17 12:38:13,998] jepsen worker 19 - jepsen.util 13609 :ok :read [6 29] INFO [2019-12-17 12:38:13,998] jepsen worker 19 - jepsen.util 13609 :invoke :cas [6 [20 13]] INFO [2019-12-17 12:38:14,001] jepsen worker 65 - jepsen.util 11135 :info :write [21 40] :RequestCanceledException INFO [2019-12-17 12:38:14,003] jepsen worker 65 - jepsen.util 11225 :invoke :cas [21 [45 44]] INFO [2019-12-17 12:38:14,028] jepsen worker 19 - jepsen.util 13609 :fail :cas [6 [20 13]] :ValueNotSwapFrom INFO [2019-12-17 12:38:14,028] jepsen worker 19 - jepsen.util 13609 :invoke :write [6 38] INFO [2019-12-17 12:38:14,028] jepsen worker 32 - jepsen.util 13262 :ok :read [10 31] INFO [2019-12-17 12:38:14,028] jepsen worker 32 - jepsen.util 13262 :invoke :cas [10 [18 38]] INFO [2019-12-17 12:38:14,034] jepsen worker 53 - jepsen.util 12383 :info :write [17 16] :RequestCanceledException INFO [2019-12-17 12:38:14,035] jepsen worker 53 - jepsen.util 12473 :invoke :write [17 49] INFO [2019-12-17 12:38:14,048] jepsen worker 72 - jepsen.util 13032 :fail :read [24 nil] :RequestTimeoutException INFO [2019-12-17 12:38:14,048] jepsen worker 72 - jepsen.util 13032 :invoke :read [24 nil] INFO [2019-12-17 12:38:14,056] jepsen worker 32 - jepsen.util 13262 :fail :cas [10 [18 38]] :ValueNotSwapFrom INFO [2019-12-17 12:38:14,056] jepsen worker 32 - jepsen.util 13262 :invoke :cas [10 [41 38]] INFO [2019-12-17 12:38:14,067] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:38:14,067] jepsen nemesis - couchbase.util Rebalance check count: 588 INFO [2019-12-17 12:38:14,067] jepsen nemesis - couchbase.util Rebalance stuck count: 569 INFO [2019-12-17 12:38:14,093] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":74644,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:38:14,093] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":74644,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:38:14,093] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":74646,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:38:14,093] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":74646,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:38:14,114] jepsen worker 37 - jepsen.util 12457 :fail :cas [12 [23 41]] :ValueNotSwapFrom INFO [2019-12-17 12:38:14,114] jepsen worker 37 - jepsen.util 12457 :invoke :read [12 nil] INFO [2019-12-17 12:38:14,115] jepsen worker 32 - jepsen.util 13262 :fail :cas [10 [41 38]] :ValueNotSwapFrom INFO [2019-12-17 12:38:14,115] jepsen worker 32 - jepsen.util 13262 :invoke :read [10 nil] INFO [2019-12-17 12:38:14,118] jepsen worker 29 - jepsen.util 11189 :fail :read [9 nil] :RequestTimeoutException INFO [2019-12-17 12:38:14,118] jepsen worker 29 - jepsen.util 11189 :invoke :cas [9 [15 41]] INFO [2019-12-17 12:38:14,140] jepsen worker 13 - jepsen.util 12523 :fail :cas [4 [0 41]] :ValueNotSwapFrom INFO [2019-12-17 12:38:14,140] jepsen worker 13 - jepsen.util 12523 :invoke :read [4 nil] INFO [2019-12-17 12:38:14,141] jepsen worker 32 - jepsen.util 13262 :ok :read [10 31] INFO [2019-12-17 12:38:14,141] jepsen worker 32 - jepsen.util 13262 :invoke :read [10 nil] INFO [2019-12-17 12:38:14,150] jepsen worker 32 - jepsen.util 13262 :ok :read [10 31] INFO [2019-12-17 12:38:14,151] jepsen worker 32 - jepsen.util 13262 :invoke :read [10 nil] INFO [2019-12-17 12:38:14,160] jepsen worker 32 - jepsen.util 13262 :ok :read [10 31] INFO [2019-12-17 12:38:14,161] jepsen worker 32 - jepsen.util 13262 :invoke :cas [10 [11 17]] INFO [2019-12-17 12:38:14,173] jepsen worker 32 - jepsen.util 13262 :fail :cas [10 [11 17]] :ValueNotSwapFrom INFO [2019-12-17 12:38:14,173] jepsen worker 32 - jepsen.util 13262 :invoke :read [10 nil] INFO [2019-12-17 12:38:14,185] jepsen worker 32 - jepsen.util 13262 :ok :read [10 31] INFO [2019-12-17 12:38:14,185] jepsen worker 32 - jepsen.util 13262 :invoke :read [10 nil] INFO [2019-12-17 12:38:14,196] jepsen worker 32 - jepsen.util 13262 :ok :read [10 31] INFO [2019-12-17 12:38:14,196] jepsen worker 32 - jepsen.util 13262 :invoke :read [10 nil] INFO [2019-12-17 12:38:14,201] jepsen worker 12 - jepsen.util 12972 :info :write [4 0] :RequestCanceledException INFO [2019-12-17 12:38:14,201] jepsen worker 88 - jepsen.util 12868 :info :write [29 32] :RequestCanceledException INFO [2019-12-17 12:38:14,202] jepsen worker 12 - jepsen.util 13062 :invoke :write [4 29] INFO [2019-12-17 12:38:14,202] jepsen worker 88 - jepsen.util 12958 :invoke :write [29 33] INFO [2019-12-17 12:38:14,202] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":74663,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:38:14,202] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":74663,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:38:14,202] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":74664,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:38:14,202] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":74664,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:38:14,203] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":74665,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:38:14,203] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":74665,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:38:14,212] jepsen worker 32 - jepsen.util 13262 :ok :read [10 31] INFO [2019-12-17 12:38:14,212] jepsen worker 32 - jepsen.util 13262 :invoke :write [10 32] INFO [2019-12-17 12:38:14,231] jepsen worker 55 - jepsen.util 13645 :fail :cas [18 [33 30]] :ValueNotSwapFrom INFO [2019-12-17 12:38:14,231] jepsen worker 55 - jepsen.util 13645 :invoke :write [18 47] INFO [2019-12-17 12:38:14,248] jepsen worker 64 - jepsen.util 9964 :info :cas [21 [43 7]] :RequestCanceledException INFO [2019-12-17 12:38:14,249] jepsen worker 64 - jepsen.util 10054 :invoke :read [21 nil] INFO [2019-12-17 12:38:14,262] jepsen worker 64 - jepsen.util 10054 :ok :read [21 13] INFO [2019-12-17 12:38:14,263] jepsen worker 64 - jepsen.util 10054 :invoke :write [21 8] INFO [2019-12-17 12:38:14,271] jepsen worker 3 - jepsen.util 12513 :fail :cas [1 [41 48]] :ValueNotSwapFrom INFO [2019-12-17 12:38:14,271] jepsen worker 3 - jepsen.util 12513 :invoke :cas [1 [19 31]] INFO [2019-12-17 12:38:14,273] jepsen worker 79 - jepsen.util 13129 :ok :read [26 4] INFO [2019-12-17 12:38:14,273] jepsen worker 79 - jepsen.util 13129 :invoke :write [26 37] INFO [2019-12-17 12:38:14,278] jepsen worker 37 - jepsen.util 12457 :ok :read [12 35] INFO [2019-12-17 12:38:14,279] jepsen worker 37 - jepsen.util 12457 :invoke :write [12 37] INFO [2019-12-17 12:38:14,301] jepsen worker 70 - jepsen.util 13300 :info :write [23 19] :RequestCanceledException INFO [2019-12-17 12:38:14,302] jepsen worker 70 - jepsen.util 13390 :invoke :read [23 nil] INFO [2019-12-17 12:38:14,305] jepsen worker 20 - jepsen.util 12800 :ok :write [6 9] INFO [2019-12-17 12:38:14,305] jepsen worker 20 - jepsen.util 12800 :invoke :cas [6 [43 41]] INFO [2019-12-17 12:38:14,306] jepsen worker 31 - jepsen.util 11101 :ok :write [10 45] INFO [2019-12-17 12:38:14,306] jepsen worker 31 - jepsen.util 11101 :invoke :read [10 nil] INFO [2019-12-17 12:38:14,306] jepsen worker 9 - jepsen.util 13329 :ok :write [3 47] INFO [2019-12-17 12:38:14,306] jepsen worker 9 - jepsen.util 13329 :invoke :read [3 nil] INFO [2019-12-17 12:38:14,309] jepsen worker 73 - jepsen.util 11953 :ok :write [24 33] INFO [2019-12-17 12:38:14,309] jepsen worker 73 - jepsen.util 11953 :invoke :write [24 26] INFO [2019-12-17 12:38:14,312] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":74673,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:38:14,312] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":74673,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:38:14,312] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":74677,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:38:14,312] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":74677,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:38:14,315] jepsen worker 13 - jepsen.util 12523 :ok :read [4 13] INFO [2019-12-17 12:38:14,315] jepsen worker 13 - jepsen.util 12523 :invoke :write [4 17] INFO [2019-12-17 12:38:14,319] jepsen worker 77 - jepsen.util 18707 :info :write [25 39] :RequestCanceledException INFO [2019-12-17 12:38:14,320] jepsen worker 77 - jepsen.util 18797 :invoke :write [25 47] INFO [2019-12-17 12:38:14,334] jepsen worker 69 - jepsen.util 13209 :info :write [23 28] :RequestCanceledException INFO [2019-12-17 12:38:14,335] jepsen worker 69 - jepsen.util 13299 :invoke :write [23 7] INFO [2019-12-17 12:38:14,347] jepsen worker 65 - jepsen.util 11225 :fail :cas [21 [45 44]] :ValueNotSwapFrom INFO [2019-12-17 12:38:14,347] jepsen worker 65 - jepsen.util 11225 :invoke :cas [21 [46 14]] INFO [2019-12-17 12:38:14,355] jepsen worker 72 - jepsen.util 13032 :ok :read [24 21] INFO [2019-12-17 12:38:14,355] jepsen worker 72 - jepsen.util 13032 :invoke :write [24 19] INFO [2019-12-17 12:38:14,363] jepsen worker 64 - jepsen.util 10054 :ok :write [21 8] INFO [2019-12-17 12:38:14,363] jepsen worker 64 - jepsen.util 10054 :invoke :write [21 33] INFO [2019-12-17 12:38:14,366] jepsen worker 71 - jepsen.util 13661 :info :write [23 13] :RequestCanceledException INFO [2019-12-17 12:38:14,367] jepsen worker 71 - jepsen.util 13751 :invoke :write [23 31] INFO [2019-12-17 12:38:14,401] jepsen worker 2 - jepsen.util 16922 :info :cas [0 [36 38]] :RequestCanceledException INFO [2019-12-17 12:38:14,402] jepsen worker 2 - jepsen.util 17012 :invoke :write [0 27] INFO [2019-12-17 12:38:14,408] jepsen worker 29 - jepsen.util 11189 :fail :cas [9 [15 41]] :ValueNotSwapFrom INFO [2019-12-17 12:38:14,408] jepsen worker 20 - jepsen.util 12800 :fail :cas [6 [43 41]] :ValueNotSwapFrom INFO [2019-12-17 12:38:14,408] jepsen worker 29 - jepsen.util 11189 :invoke :read [9 nil] INFO [2019-12-17 12:38:14,408] jepsen worker 20 - jepsen.util 12800 :invoke :read [6 nil] INFO [2019-12-17 12:38:14,412] jepsen worker 31 - jepsen.util 11101 :ok :read [10 45] INFO [2019-12-17 12:38:14,412] jepsen worker 31 - jepsen.util 11101 :invoke :cas [10 [40 29]] INFO [2019-12-17 12:38:14,416] jepsen worker 15 - jepsen.util 16845 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:38:14,417] jepsen worker 15 - jepsen.util 16845 :invoke :cas [5 [44 39]] INFO [2019-12-17 12:38:14,420] jepsen worker 9 - jepsen.util 13329 :ok :read [3 47] INFO [2019-12-17 12:38:14,421] jepsen worker 9 - jepsen.util 13329 :invoke :cas [3 [6 33]] INFO [2019-12-17 12:38:14,435] jepsen worker 81 - jepsen.util 17091 :info :cas [27 [3 30]] :RequestCanceledException INFO [2019-12-17 12:38:14,435] jepsen worker 35 - jepsen.util 17765 :info :cas [11 [32 37]] :RequestCanceledException INFO [2019-12-17 12:38:14,436] jepsen worker 81 - jepsen.util 17181 :invoke :read [27 nil] INFO [2019-12-17 12:38:14,436] jepsen worker 35 - jepsen.util 17855 :invoke :read [11 nil] INFO [2019-12-17 12:38:14,441] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":74703,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:38:14,441] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":74703,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:38:14,441] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":74709,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:38:14,441] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":74709,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:38:14,448] jepsen worker 44 - jepsen.util 18314 :info :cas [14 [0 23]] :RequestCanceledException INFO [2019-12-17 12:38:14,449] jepsen worker 44 - jepsen.util 18404 :invoke :cas [14 [2 42]] INFO [2019-12-17 12:38:14,449] jepsen worker 20 - jepsen.util 12800 :ok :read [6 9] INFO [2019-12-17 12:38:14,449] jepsen worker 20 - jepsen.util 12800 :invoke :write [6 2] INFO [2019-12-17 12:38:14,454] jepsen worker 29 - jepsen.util 11189 :ok :read [9 13] INFO [2019-12-17 12:38:14,454] jepsen worker 29 - jepsen.util 11189 :invoke :write [9 13] INFO [2019-12-17 12:38:14,465] jepsen worker 58 - jepsen.util 17968 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:38:14,465] jepsen worker 50 - jepsen.util 17600 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:38:14,465] jepsen worker 58 - jepsen.util 17968 :invoke :cas [19 [28 31]] INFO [2019-12-17 12:38:14,465] jepsen worker 50 - jepsen.util 17600 :invoke :read [16 nil] INFO [2019-12-17 12:38:14,483] jepsen worker 9 - jepsen.util 13329 :fail :cas [3 [6 33]] :ValueNotSwapFrom INFO [2019-12-17 12:38:14,483] jepsen worker 9 - jepsen.util 13329 :invoke :cas [3 [41 24]] INFO [2019-12-17 12:38:14,488] jepsen worker 64 - jepsen.util 10054 :ok :write [21 33] INFO [2019-12-17 12:38:14,488] jepsen worker 64 - jepsen.util 10054 :invoke :cas [21 [11 38]] INFO [2019-12-17 12:38:14,492] jepsen worker 3 - jepsen.util 12513 :fail :cas [1 [19 31]] :ValueNotSwapFrom INFO [2019-12-17 12:38:14,492] jepsen worker 3 - jepsen.util 12513 :invoke :read [1 nil] INFO [2019-12-17 12:38:14,501] jepsen worker 48 - jepsen.util 16428 :info :cas [16 [40 15]] :RequestCanceledException INFO [2019-12-17 12:38:14,502] jepsen worker 48 - jepsen.util 16518 :invoke :read [16 nil] INFO [2019-12-17 12:38:14,505] jepsen worker 73 - jepsen.util 11953 :ok :write [24 26] INFO [2019-12-17 12:38:14,505] jepsen worker 73 - jepsen.util 11953 :invoke :cas [24 [41 1]] INFO [2019-12-17 12:38:14,506] jepsen worker 64 - jepsen.util 10054 :fail :cas [21 [11 38]] :ValueNotSwapFrom INFO [2019-12-17 12:38:14,506] jepsen worker 64 - jepsen.util 10054 :invoke :write [21 25] INFO [2019-12-17 12:38:14,518] jepsen worker 73 - jepsen.util 11953 :fail :cas [24 [41 1]] :ValueNotSwapFrom INFO [2019-12-17 12:38:14,518] jepsen worker 73 - jepsen.util 11953 :invoke :write [24 1] INFO [2019-12-17 12:38:14,519] jepsen worker 16 - jepsen.util 19096 :info :cas [5 [17 33]] :RequestCanceledException INFO [2019-12-17 12:38:14,520] jepsen worker 16 - jepsen.util 19186 :invoke :write [5 17] INFO [2019-12-17 12:38:14,534] jepsen worker 25 - jepsen.util 18745 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:38:14,534] jepsen worker 25 - jepsen.util 18745 :invoke :write [8 3] INFO [2019-12-17 12:38:14,547] jepsen worker 29 - jepsen.util 11189 :ok :write [9 13] INFO [2019-12-17 12:38:14,547] jepsen worker 29 - jepsen.util 11189 :invoke :cas [9 [39 28]] INFO [2019-12-17 12:38:14,548] jepsen worker 43 - jepsen.util 17863 :info :cas [14 [42 30]] :RequestCanceledException INFO [2019-12-17 12:38:14,548] jepsen worker 57 - jepsen.util 17337 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:38:14,549] jepsen worker 57 - jepsen.util 17337 :invoke :write [19 9] INFO [2019-12-17 12:38:14,550] jepsen worker 43 - jepsen.util 17953 :invoke :cas [14 [38 38]] INFO [2019-12-17 12:38:14,566] jepsen worker 8 - jepsen.util 17108 :info :write [2 7] :RequestCanceledException INFO [2019-12-17 12:38:14,566] jepsen worker 7 - jepsen.util 17467 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:38:14,566] jepsen worker 61 - jepsen.util 18691 :info :cas [20 [35 22]] :RequestCanceledException INFO [2019-12-17 12:38:14,566] jepsen worker 7 - jepsen.util 17467 :invoke :read [2 nil] INFO [2019-12-17 12:38:14,567] jepsen worker 61 - jepsen.util 18781 :invoke :cas [20 [1 9]] INFO [2019-12-17 12:38:14,567] jepsen worker 8 - jepsen.util 17198 :invoke :write [2 12] INFO [2019-12-17 12:38:14,594] jepsen worker 64 - jepsen.util 10054 :ok :write [21 25] INFO [2019-12-17 12:38:14,595] jepsen worker 64 - jepsen.util 10054 :invoke :read [21 nil] INFO [2019-12-17 12:38:14,595] jepsen worker 29 - jepsen.util 11189 :fail :cas [9 [39 28]] :ValueNotSwapFrom INFO [2019-12-17 12:38:14,596] jepsen worker 29 - jepsen.util 11189 :invoke :cas [9 [39 41]] INFO [2019-12-17 12:38:14,599] jepsen worker 38 - jepsen.util 13538 :info :write [12 29] :RequestCanceledException INFO [2019-12-17 12:38:14,600] jepsen worker 38 - jepsen.util 13628 :invoke :cas [12 [32 34]] INFO [2019-12-17 12:38:14,610] jepsen worker 3 - jepsen.util 12513 :ok :read [1 29] INFO [2019-12-17 12:38:14,610] jepsen worker 3 - jepsen.util 12513 :invoke :write [1 21] INFO [2019-12-17 12:38:14,635] jepsen worker 59 - jepsen.util 16799 :info :write [19 48] :RequestCanceledException INFO [2019-12-17 12:38:14,636] jepsen worker 59 - jepsen.util 16889 :invoke :read [19 nil] INFO [2019-12-17 12:38:14,637] jepsen worker 29 - jepsen.util 11189 :fail :cas [9 [39 41]] :ValueNotSwapFrom INFO [2019-12-17 12:38:14,637] jepsen worker 29 - jepsen.util 11189 :invoke :cas [9 [48 33]] INFO [2019-12-17 12:38:14,638] jepsen worker 64 - jepsen.util 10054 :ok :read [21 25] INFO [2019-12-17 12:38:14,638] jepsen worker 64 - jepsen.util 10054 :invoke :write [21 12] INFO [2019-12-17 12:38:14,647] jepsen worker 87 - jepsen.util 12327 :info :write [29 8] :RequestCanceledException INFO [2019-12-17 12:38:14,649] jepsen worker 87 - jepsen.util 12417 :invoke :write [29 39] INFO [2019-12-17 12:38:14,685] jepsen worker 29 - jepsen.util 11189 :fail :cas [9 [48 33]] :ValueNotSwapFrom INFO [2019-12-17 12:38:14,685] jepsen worker 29 - jepsen.util 11189 :invoke :cas [9 [19 23]] INFO [2019-12-17 12:38:14,704] jepsen worker 29 - jepsen.util 11189 :fail :cas [9 [19 23]] :ValueNotSwapFrom INFO [2019-12-17 12:38:14,705] jepsen worker 29 - jepsen.util 11189 :invoke :write [9 42] INFO [2019-12-17 12:38:14,709] jepsen worker 20 - jepsen.util 12800 :ok :write [6 2] INFO [2019-12-17 12:38:14,709] jepsen worker 20 - jepsen.util 12800 :invoke :write [6 8] INFO [2019-12-17 12:38:14,714] jepsen worker 72 - jepsen.util 13032 :ok :write [24 19] INFO [2019-12-17 12:38:14,714] jepsen worker 72 - jepsen.util 13032 :invoke :write [24 4] INFO [2019-12-17 12:38:14,749] jepsen worker 64 - jepsen.util 10054 :ok :write [21 12] INFO [2019-12-17 12:38:14,749] jepsen worker 64 - jepsen.util 10054 :invoke :write [21 47] INFO [2019-12-17 12:38:14,750] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":74738,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:38:14,750] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":74738,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:38:14,750] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":74742,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:38:14,750] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":74742,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:38:14,766] jepsen worker 83 - jepsen.util 17543 :info :write [27 33] :RequestCanceledException INFO [2019-12-17 12:38:14,771] jepsen worker 83 - jepsen.util 17633 :invoke :write [27 44] INFO [2019-12-17 12:38:14,785] jepsen worker 29 - jepsen.util 11189 :ok :write [9 42] INFO [2019-12-17 12:38:14,786] jepsen worker 29 - jepsen.util 11189 :invoke :read [9 nil] INFO [2019-12-17 12:38:14,801] jepsen worker 80 - jepsen.util 12230 :info :write [26 25] :RequestCanceledException INFO [2019-12-17 12:38:14,801] jepsen worker 42 - jepsen.util 18492 :info :cas [14 [48 44]] :RequestCanceledException INFO [2019-12-17 12:38:14,802] jepsen worker 42 - jepsen.util 18582 :invoke :read [14 nil] INFO [2019-12-17 12:38:14,802] jepsen worker 80 - jepsen.util 12320 :invoke :read [26 nil] INFO [2019-12-17 12:38:14,805] jepsen worker 29 - jepsen.util 11189 :ok :read [9 42] INFO [2019-12-17 12:38:14,805] jepsen worker 29 - jepsen.util 11189 :invoke :read [9 nil] INFO [2019-12-17 12:38:14,819] jepsen worker 39 - jepsen.util 17139 :info :cas [13 [34 38]] :RequestCanceledException INFO [2019-12-17 12:38:14,821] jepsen worker 39 - jepsen.util 17229 :invoke :write [13 41] INFO [2019-12-17 12:38:14,830] jepsen worker 29 - jepsen.util 11189 :ok :read [9 42] INFO [2019-12-17 12:38:14,831] jepsen worker 29 - jepsen.util 11189 :invoke :read [9 nil] INFO [2019-12-17 12:38:14,845] jepsen worker 64 - jepsen.util 10054 :ok :write [21 47] INFO [2019-12-17 12:38:14,845] jepsen worker 64 - jepsen.util 10054 :invoke :write [21 39] INFO [2019-12-17 12:38:14,858] jepsen worker 29 - jepsen.util 11189 :ok :read [9 42] INFO [2019-12-17 12:38:14,858] jepsen worker 29 - jepsen.util 11189 :invoke :write [9 39] INFO [2019-12-17 12:38:14,894] jepsen worker 20 - jepsen.util 12800 :ok :write [6 8] INFO [2019-12-17 12:38:14,894] jepsen worker 20 - jepsen.util 12800 :invoke :write [6 4] INFO [2019-12-17 12:38:14,898] jepsen worker 1 - jepsen.util 17911 :info :write [0 5] :RequestCanceledException INFO [2019-12-17 12:38:14,899] jepsen worker 1 - jepsen.util 18001 :invoke :cas [0 [10 16]] INFO [2019-12-17 12:38:14,933] jepsen worker 6 - jepsen.util 18006 :info :cas [2 [19 9]] :RequestCanceledException INFO [2019-12-17 12:38:14,933] jepsen worker 22 - jepsen.util 17932 :info :write [7 33] :RequestCanceledException INFO [2019-12-17 12:38:14,933] jepsen worker 52 - jepsen.util 12472 :info :write [17 8] :RequestCanceledException INFO [2019-12-17 12:38:14,934] jepsen worker 22 - jepsen.util 18022 :invoke :cas [7 [1 20]] INFO [2019-12-17 12:38:14,934] jepsen worker 6 - jepsen.util 18096 :invoke :read [2 nil] INFO [2019-12-17 12:38:14,935] jepsen worker 52 - jepsen.util 12562 :invoke :write [17 27] INFO [2019-12-17 12:38:14,938] jepsen worker 29 - jepsen.util 11189 :ok :write [9 39] INFO [2019-12-17 12:38:14,938] jepsen worker 29 - jepsen.util 11189 :invoke :read [9 nil] INFO [2019-12-17 12:38:14,948] jepsen worker 68 - jepsen.util 16808 :info :cas [22 [42 10]] :RequestCanceledException INFO [2019-12-17 12:38:14,950] jepsen worker 68 - jepsen.util 16898 :invoke :write [22 23] INFO [2019-12-17 12:38:14,966] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":74761,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:38:14,966] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":74761,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:38:14,966] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":74762,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:38:14,966] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":74762,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:38:14,974] jepsen worker 29 - jepsen.util 11189 :ok :read [9 39] INFO [2019-12-17 12:38:14,974] jepsen worker 29 - jepsen.util 11189 :invoke :write [9 8] INFO [2019-12-17 12:38:14,984] jepsen worker 80 - jepsen.util 12320 :ok :read [26 4] INFO [2019-12-17 12:38:14,984] jepsen worker 80 - jepsen.util 12320 :invoke :write [26 17] INFO [2019-12-17 12:38:15,011] jepsen worker 64 - jepsen.util 10054 :ok :write [21 39] INFO [2019-12-17 12:38:15,011] jepsen worker 64 - jepsen.util 10054 :invoke :read [21 nil] INFO [2019-12-17 12:38:15,033] jepsen worker 38 - jepsen.util 13628 :fail :cas [12 [32 34]] :ValueNotSwapFrom INFO [2019-12-17 12:38:15,033] jepsen worker 38 - jepsen.util 13628 :invoke :write [12 48] INFO [2019-12-17 12:38:15,034] jepsen worker 76 - jepsen.util 17716 :info :write [25 9] :RequestCanceledException INFO [2019-12-17 12:38:15,035] jepsen worker 76 - jepsen.util 17806 :invoke :cas [25 [34 6]] INFO [2019-12-17 12:38:15,040] jepsen worker 29 - jepsen.util 11189 :ok :write [9 8] INFO [2019-12-17 12:38:15,040] jepsen worker 29 - jepsen.util 11189 :invoke :cas [9 [11 20]] INFO [2019-12-17 12:38:15,042] jepsen worker 64 - jepsen.util 10054 :ok :read [21 39] INFO [2019-12-17 12:38:15,043] jepsen worker 64 - jepsen.util 10054 :invoke :read [21 nil] INFO [2019-12-17 12:38:15,054] jepsen worker 29 - jepsen.util 11189 :fail :cas [9 [11 20]] :ValueNotSwapFrom INFO [2019-12-17 12:38:15,055] jepsen worker 29 - jepsen.util 11189 :invoke :read [9 nil] INFO [2019-12-17 12:38:15,075] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":74773,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:38:15,076] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":74773,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:38:15,076] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":74774,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:38:15,076] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":74774,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:38:15,076] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":74775,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:38:15,076] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":74775,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:38:15,080] jepsen worker 64 - jepsen.util 10054 :ok :read [21 39] INFO [2019-12-17 12:38:15,080] jepsen worker 64 - jepsen.util 10054 :invoke :read [21 nil] INFO [2019-12-17 12:38:15,081] jepsen worker 29 - jepsen.util 11189 :ok :read [9 8] INFO [2019-12-17 12:38:15,081] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:38:15,081] jepsen worker 29 - jepsen.util 11189 :invoke :read [9 nil] INFO [2019-12-17 12:38:15,081] jepsen nemesis - couchbase.util Rebalance check count: 589 INFO [2019-12-17 12:38:15,081] jepsen nemesis - couchbase.util Rebalance stuck count: 570 INFO [2019-12-17 12:38:15,100] jepsen worker 89 - jepsen.util 12779 :info :write [29 14] :RequestCanceledException INFO [2019-12-17 12:38:15,100] jepsen worker 34 - jepsen.util 18574 :info :cas [11 [24 46]] :RequestCanceledException INFO [2019-12-17 12:38:15,101] jepsen worker 89 - jepsen.util 12869 :invoke :write [29 46] INFO [2019-12-17 12:38:15,101] jepsen worker 34 - jepsen.util 18664 :invoke :write [11 11] INFO [2019-12-17 12:38:15,111] jepsen worker 29 - jepsen.util 11189 :ok :read [9 8] INFO [2019-12-17 12:38:15,111] jepsen worker 29 - jepsen.util 11189 :invoke :cas [9 [11 28]] INFO [2019-12-17 12:38:15,117] jepsen worker 64 - jepsen.util 10054 :ok :read [21 39] INFO [2019-12-17 12:38:15,117] jepsen worker 64 - jepsen.util 10054 :invoke :read [21 nil] INFO [2019-12-17 12:38:15,119] jepsen worker 49 - jepsen.util 17599 :info :write [16 45] :RequestCanceledException INFO [2019-12-17 12:38:15,120] jepsen worker 49 - jepsen.util 17689 :invoke :cas [16 [10 30]] INFO [2019-12-17 12:38:15,134] jepsen worker 54 - jepsen.util 13464 :info :write [18 13] :RequestCanceledException INFO [2019-12-17 12:38:15,134] jepsen worker 40 - jepsen.util 18400 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:38:15,134] jepsen worker 40 - jepsen.util 18400 :invoke :write [13 47] INFO [2019-12-17 12:38:15,135] jepsen worker 54 - jepsen.util 13554 :invoke :cas [18 [27 30]] INFO [2019-12-17 12:38:15,140] jepsen worker 64 - jepsen.util 10054 :ok :read [21 39] INFO [2019-12-17 12:38:15,140] jepsen worker 64 - jepsen.util 10054 :invoke :write [21 11] INFO [2019-12-17 12:38:15,141] jepsen worker 29 - jepsen.util 11189 :fail :cas [9 [11 28]] :ValueNotSwapFrom INFO [2019-12-17 12:38:15,142] jepsen worker 29 - jepsen.util 11189 :invoke :read [9 nil] INFO [2019-12-17 12:38:15,149] jepsen worker 41 - jepsen.util 19031 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:38:15,149] jepsen worker 41 - jepsen.util 19031 :invoke :cas [13 [8 37]] INFO [2019-12-17 12:38:15,153] jepsen worker 70 - jepsen.util 13390 :ok :read [23 17] INFO [2019-12-17 12:38:15,153] jepsen worker 70 - jepsen.util 13390 :invoke :write [23 49] INFO [2019-12-17 12:38:15,158] jepsen worker 29 - jepsen.util 11189 :ok :read [9 8] INFO [2019-12-17 12:38:15,158] jepsen worker 29 - jepsen.util 11189 :invoke :write [9 1] INFO [2019-12-17 12:38:15,166] jepsen worker 24 - jepsen.util 16674 :info :write [8 36] :RequestCanceledException INFO [2019-12-17 12:38:15,167] jepsen worker 24 - jepsen.util 16764 :invoke :cas [8 [35 29]] INFO [2019-12-17 12:38:15,198] jepsen worker 64 - jepsen.util 10054 :ok :write [21 11] INFO [2019-12-17 12:38:15,198] jepsen worker 64 - jepsen.util 10054 :invoke :write [21 20] INFO [2019-12-17 12:38:15,212] jepsen worker 29 - jepsen.util 11189 :ok :write [9 1] INFO [2019-12-17 12:38:15,214] jepsen worker 29 - jepsen.util 11189 :invoke :read [9 nil] INFO [2019-12-17 12:38:15,219] jepsen worker 4 - jepsen.util 11974 :info :write [1 39] :RequestCanceledException INFO [2019-12-17 12:38:15,220] jepsen worker 4 - jepsen.util 12064 :invoke :read [1 nil] INFO [2019-12-17 12:38:15,229] jepsen worker 29 - jepsen.util 11189 :ok :read [9 1] INFO [2019-12-17 12:38:15,229] jepsen worker 29 - jepsen.util 11189 :invoke :write [9 15] INFO [2019-12-17 12:38:15,234] jepsen worker 11 - jepsen.util 12701 :info :write [3 43] :RequestCanceledException INFO [2019-12-17 12:38:15,234] jepsen worker 28 - jepsen.util 11008 :fail :read [9 nil] :RequestTimeoutException INFO [2019-12-17 12:38:15,234] jepsen worker 28 - jepsen.util 11008 :invoke :read [9 nil] INFO [2019-12-17 12:38:15,235] jepsen worker 11 - jepsen.util 12791 :invoke :cas [3 [25 7]] INFO [2019-12-17 12:38:15,248] jepsen worker 46 - jepsen.util 12466 :info :write [15 6] :RequestCanceledException INFO [2019-12-17 12:38:15,248] jepsen worker 18 - jepsen.util 13428 :info :write [6 29] :RequestCanceledException INFO [2019-12-17 12:38:15,249] jepsen worker 18 - jepsen.util 13518 :invoke :write [6 16] INFO [2019-12-17 12:38:15,249] jepsen worker 46 - jepsen.util 12556 :invoke :write [15 35] INFO [2019-12-17 12:38:15,273] jepsen worker 64 - jepsen.util 10054 :ok :write [21 20] INFO [2019-12-17 12:38:15,273] jepsen worker 64 - jepsen.util 10054 :invoke :read [21 nil] INFO [2019-12-17 12:38:15,291] jepsen worker 72 - jepsen.util 13032 :ok :write [24 4] INFO [2019-12-17 12:38:15,291] jepsen worker 20 - jepsen.util 12800 :ok :write [6 4] INFO [2019-12-17 12:38:15,291] jepsen worker 72 - jepsen.util 13032 :invoke :read [24 nil] INFO [2019-12-17 12:38:15,291] jepsen worker 20 - jepsen.util 12800 :invoke :write [6 47] INFO [2019-12-17 12:38:15,295] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":74812,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:38:15,295] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":74802,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:38:15,295] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":74812,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:38:15,295] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":74802,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:38:15,295] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":74814,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:38:15,295] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":74814,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:38:15,297] jepsen worker 64 - jepsen.util 10054 :ok :read [21 20] INFO [2019-12-17 12:38:15,298] jepsen worker 64 - jepsen.util 10054 :invoke :write [21 35] INFO [2019-12-17 12:38:15,319] jepsen worker 36 - jepsen.util 13626 :info :write [12 5] :RequestCanceledException INFO [2019-12-17 12:38:15,320] jepsen worker 36 - jepsen.util 13716 :invoke :cas [12 [13 8]] INFO [2019-12-17 12:38:15,334] jepsen worker 27 - jepsen.util 11997 :fail :read [9 nil] :RequestTimeoutException INFO [2019-12-17 12:38:15,334] jepsen worker 27 - jepsen.util 11997 :invoke :cas [9 [14 8]] INFO [2019-12-17 12:38:15,337] jepsen worker 28 - jepsen.util 11008 :ok :read [9 1] INFO [2019-12-17 12:38:15,337] jepsen worker 28 - jepsen.util 11008 :invoke :read [9 nil] INFO [2019-12-17 12:38:15,348] jepsen worker 29 - jepsen.util 11189 :ok :write [9 15] INFO [2019-12-17 12:38:15,348] jepsen worker 29 - jepsen.util 11189 :invoke :write [9 13] INFO [2019-12-17 12:38:15,360] jepsen worker 72 - jepsen.util 13032 :ok :read [24 4] INFO [2019-12-17 12:38:15,360] jepsen worker 72 - jepsen.util 13032 :invoke :read [24 nil] INFO [2019-12-17 12:38:15,362] jepsen worker 63 - jepsen.util 11493 :fail :read [21 nil] :RequestTimeoutException INFO [2019-12-17 12:38:15,362] jepsen worker 62 - jepsen.util 18332 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:38:15,362] jepsen worker 63 - jepsen.util 11493 :invoke :write [21 12] INFO [2019-12-17 12:38:15,362] jepsen worker 62 - jepsen.util 18332 :invoke :cas [20 [2 44]] INFO [2019-12-17 12:38:15,365] jepsen worker 11 - jepsen.util 12791 :fail :cas [3 [25 7]] :ValueNotSwapFrom INFO [2019-12-17 12:38:15,365] jepsen worker 11 - jepsen.util 12791 :invoke :cas [3 [7 16]] INFO [2019-12-17 12:38:15,374] jepsen worker 52 - jepsen.util 12562 :ok :write [17 27] INFO [2019-12-17 12:38:15,374] jepsen worker 52 - jepsen.util 12562 :invoke :cas [17 [25 26]] INFO [2019-12-17 12:38:15,379] jepsen worker 4 - jepsen.util 12064 :ok :read [1 29] INFO [2019-12-17 12:38:15,379] jepsen worker 4 - jepsen.util 12064 :invoke :cas [1 [2 43]] INFO [2019-12-17 12:38:15,395] jepsen worker 36 - jepsen.util 13716 :fail :cas [12 [13 8]] :ValueNotSwapFrom INFO [2019-12-17 12:38:15,396] jepsen worker 36 - jepsen.util 13716 :invoke :write [12 15] INFO [2019-12-17 12:38:15,409] jepsen worker 64 - jepsen.util 10054 :ok :write [21 35] INFO [2019-12-17 12:38:15,409] jepsen worker 64 - jepsen.util 10054 :invoke :write [21 23] INFO [2019-12-17 12:38:15,416] jepsen worker 72 - jepsen.util 13032 :ok :read [24 4] INFO [2019-12-17 12:38:15,416] jepsen worker 72 - jepsen.util 13032 :invoke :cas [24 [0 10]] INFO [2019-12-17 12:38:15,434] jepsen worker 27 - jepsen.util 11997 :fail :cas [9 [14 8]] :ValueNotSwapFrom INFO [2019-12-17 12:38:15,434] jepsen worker 27 - jepsen.util 11997 :invoke :cas [9 [26 24]] INFO [2019-12-17 12:38:15,434] jepsen worker 86 - jepsen.util 18356 :info :write [28 28] :RequestCanceledException INFO [2019-12-17 12:38:15,435] jepsen worker 86 - jepsen.util 18446 :invoke :cas [28 [35 32]] INFO [2019-12-17 12:38:15,453] jepsen worker 13 - jepsen.util 12523 :ok :write [4 17] INFO [2019-12-17 12:38:15,453] jepsen worker 13 - jepsen.util 12523 :invoke :write [4 34] INFO [2019-12-17 12:38:15,465] jepsen worker 72 - jepsen.util 13032 :fail :cas [24 [0 10]] :ValueNotSwapFrom INFO [2019-12-17 12:38:15,465] jepsen worker 72 - jepsen.util 13032 :invoke :cas [24 [47 3]] INFO [2019-12-17 12:38:15,491] jepsen worker 71 - jepsen.util 13751 :ok :write [23 31] INFO [2019-12-17 12:38:15,491] jepsen worker 71 - jepsen.util 13751 :invoke :write [23 2] INFO [2019-12-17 12:38:15,498] jepsen worker 75 - jepsen.util 17265 :info :cas [25 [38 18]] :RequestCanceledException INFO [2019-12-17 12:38:15,498] jepsen worker 54 - jepsen.util 13554 :fail :cas [18 [27 30]] :ValueNotSwapFrom INFO [2019-12-17 12:38:15,499] jepsen worker 54 - jepsen.util 13554 :invoke :write [18 31] INFO [2019-12-17 12:38:15,499] jepsen worker 75 - jepsen.util 17355 :invoke :cas [25 [31 35]] INFO [2019-12-17 12:38:15,500] jepsen worker 63 - jepsen.util 11493 :ok :write [21 12] INFO [2019-12-17 12:38:15,500] jepsen worker 63 - jepsen.util 11493 :invoke :write [21 14] INFO [2019-12-17 12:38:15,501] jepsen worker 72 - jepsen.util 13032 :fail :cas [24 [47 3]] :ValueNotSwapFrom INFO [2019-12-17 12:38:15,502] jepsen worker 72 - jepsen.util 13032 :invoke :read [24 nil] INFO [2019-12-17 12:38:15,503] jepsen worker 46 - jepsen.util 12556 :ok :write [15 35] INFO [2019-12-17 12:38:15,503] jepsen worker 46 - jepsen.util 12556 :invoke :write [15 18] INFO [2019-12-17 12:38:15,517] jepsen worker 26 - jepsen.util 18566 :info :cas [8 [34 26]] :RequestCanceledException INFO [2019-12-17 12:38:15,518] jepsen worker 26 - jepsen.util 18656 :invoke :read [8 nil] INFO [2019-12-17 12:38:15,534] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":74840,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:38:15,534] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":74840,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:38:15,534] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":74843,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:38:15,534] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":74843,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:38:15,534] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":74844,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:38:15,534] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":74844,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:38:15,534] jepsen worker 60 - jepsen.util 18600 :info :write [20 37] :RequestCanceledException INFO [2019-12-17 12:38:15,535] jepsen worker 60 - jepsen.util 18690 :invoke :write [20 30] INFO [2019-12-17 12:38:15,548] jepsen worker 17 - jepsen.util 17657 :info :cas [5 [39 26]] :RequestCanceledException INFO [2019-12-17 12:38:15,550] jepsen worker 17 - jepsen.util 17747 :invoke :cas [5 [14 38]] INFO [2019-12-17 12:38:15,567] jepsen worker 72 - jepsen.util 13032 :ok :read [24 4] INFO [2019-12-17 12:38:15,567] jepsen worker 72 - jepsen.util 13032 :invoke :cas [24 [45 17]] INFO [2019-12-17 12:38:15,571] jepsen worker 51 - jepsen.util 12201 :ok :write [17 23] INFO [2019-12-17 12:38:15,571] jepsen worker 51 - jepsen.util 12201 :invoke :write [17 30] INFO [2019-12-17 12:38:15,588] jepsen worker 69 - jepsen.util 13299 :ok :write [23 7] INFO [2019-12-17 12:38:15,588] jepsen worker 52 - jepsen.util 12562 :fail :cas [17 [25 26]] :ValueNotSwapFrom INFO [2019-12-17 12:38:15,588] jepsen worker 52 - jepsen.util 12562 :invoke :read [17 nil] INFO [2019-12-17 12:38:15,588] jepsen worker 69 - jepsen.util 13299 :invoke :write [23 32] INFO [2019-12-17 12:38:15,596] jepsen worker 72 - jepsen.util 13032 :fail :cas [24 [45 17]] :ValueNotSwapFrom INFO [2019-12-17 12:38:15,596] jepsen worker 72 - jepsen.util 13032 :invoke :read [24 nil] INFO [2019-12-17 12:38:15,613] jepsen worker 38 - jepsen.util 13628 :ok :write [12 48] INFO [2019-12-17 12:38:15,613] jepsen worker 38 - jepsen.util 13628 :invoke :read [12 nil] INFO [2019-12-17 12:38:15,624] jepsen worker 63 - jepsen.util 11493 :ok :write [21 14] INFO [2019-12-17 12:38:15,624] jepsen worker 63 - jepsen.util 11493 :invoke :cas [21 [34 10]] INFO [2019-12-17 12:38:15,627] jepsen worker 88 - jepsen.util 12958 :ok :write [29 33] INFO [2019-12-17 12:38:15,628] jepsen worker 88 - jepsen.util 12958 :invoke :read [29 nil] INFO [2019-12-17 12:38:15,634] jepsen worker 56 - jepsen.util 13646 :info :write [18 9] :RequestCanceledException INFO [2019-12-17 12:38:15,634] jepsen worker 33 - jepsen.util 17583 :info :write [11 43] :RequestCanceledException INFO [2019-12-17 12:38:15,635] jepsen worker 33 - jepsen.util 17673 :invoke :cas [11 [47 21]] INFO [2019-12-17 12:38:15,635] jepsen worker 56 - jepsen.util 13736 :invoke :read [18 nil] INFO [2019-12-17 12:38:15,641] jepsen worker 72 - jepsen.util 13032 :ok :read [24 4] INFO [2019-12-17 12:38:15,641] jepsen worker 72 - jepsen.util 13032 :invoke :cas [24 [30 45]] INFO [2019-12-17 12:38:15,659] jepsen worker 63 - jepsen.util 11493 :fail :cas [21 [34 10]] :ValueNotSwapFrom INFO [2019-12-17 12:38:15,659] jepsen worker 63 - jepsen.util 11493 :invoke :write [21 7] INFO [2019-12-17 12:38:15,662] jepsen worker 85 - jepsen.util 18175 :info :cas [28 [29 24]] :RequestCanceledException INFO [2019-12-17 12:38:15,662] jepsen worker 14 - jepsen.util 12524 :info :write [4 24] :RequestCanceledException INFO [2019-12-17 12:38:15,663] jepsen worker 14 - jepsen.util 12614 :invoke :cas [4 [0 36]] INFO [2019-12-17 12:38:15,663] jepsen worker 85 - jepsen.util 18265 :invoke :write [28 43] INFO [2019-12-17 12:38:15,669] jepsen worker 72 - jepsen.util 13032 :fail :cas [24 [30 45]] :ValueNotSwapFrom INFO [2019-12-17 12:38:15,669] jepsen worker 72 - jepsen.util 13032 :invoke :read [24 nil] INFO [2019-12-17 12:38:15,700] jepsen worker 67 - jepsen.util 17887 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:38:15,700] jepsen worker 67 - jepsen.util 17887 :invoke :write [22 43] INFO [2019-12-17 12:38:15,701] jepsen worker 72 - jepsen.util 13032 :ok :read [24 4] INFO [2019-12-17 12:38:15,701] jepsen worker 72 - jepsen.util 13032 :invoke :read [24 nil] INFO [2019-12-17 12:38:15,717] jepsen worker 66 - jepsen.util 17616 :info :cas [22 [43 13]] :RequestCanceledException INFO [2019-12-17 12:38:15,718] jepsen worker 66 - jepsen.util 17706 :invoke :write [22 34] INFO [2019-12-17 12:38:15,725] jepsen worker 88 - jepsen.util 12958 :ok :read [29 33] INFO [2019-12-17 12:38:15,725] jepsen worker 88 - jepsen.util 12958 :invoke :write [29 11] INFO [2019-12-17 12:38:15,726] jepsen worker 72 - jepsen.util 13032 :ok :read [24 4] INFO [2019-12-17 12:38:15,726] jepsen worker 72 - jepsen.util 13032 :invoke :write [24 16] INFO [2019-12-17 12:38:15,733] jepsen worker 52 - jepsen.util 12562 :ok :read [17 23] INFO [2019-12-17 12:38:15,734] jepsen worker 52 - jepsen.util 12562 :invoke :cas [17 [11 30]] INFO [2019-12-17 12:38:15,734] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":74870,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:38:15,734] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":74870,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:38:15,734] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":74872,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:38:15,734] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":74872,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:38:15,741] jepsen worker 63 - jepsen.util 11493 :ok :write [21 7] INFO [2019-12-17 12:38:15,741] jepsen worker 63 - jepsen.util 11493 :invoke :cas [21 [36 3]] INFO [2019-12-17 12:38:15,757] jepsen worker 63 - jepsen.util 11493 :fail :cas [21 [36 3]] :ValueNotSwapFrom INFO [2019-12-17 12:38:15,757] jepsen worker 63 - jepsen.util 11493 :invoke :write [21 6] INFO [2019-12-17 12:38:15,794] jepsen worker 13 - jepsen.util 12523 :ok :write [4 34] INFO [2019-12-17 12:38:15,794] jepsen worker 13 - jepsen.util 12523 :invoke :cas [4 [6 20]] INFO [2019-12-17 12:38:15,798] jepsen worker 20 - jepsen.util 12800 :ok :write [6 47] INFO [2019-12-17 12:38:15,798] jepsen worker 20 - jepsen.util 12800 :invoke :write [6 23] INFO [2019-12-17 12:38:15,835] jepsen worker 63 - jepsen.util 11493 :ok :write [21 6] INFO [2019-12-17 12:38:15,836] jepsen worker 63 - jepsen.util 11493 :invoke :read [21 nil] INFO [2019-12-17 12:38:15,838] jepsen worker 53 - jepsen.util 12473 :ok :write [17 49] INFO [2019-12-17 12:38:15,838] jepsen worker 53 - jepsen.util 12473 :invoke :cas [17 [1 3]] INFO [2019-12-17 12:38:15,851] jepsen worker 46 - jepsen.util 12556 :ok :write [15 18] INFO [2019-12-17 12:38:15,851] jepsen worker 46 - jepsen.util 12556 :invoke :read [15 nil] INFO [2019-12-17 12:38:15,858] jepsen worker 63 - jepsen.util 11493 :ok :read [21 6] INFO [2019-12-17 12:38:15,858] jepsen worker 63 - jepsen.util 11493 :invoke :write [21 15] INFO [2019-12-17 12:38:15,864] jepsen worker 84 - jepsen.util 17094 :info :write [28 30] :RequestCanceledException INFO [2019-12-17 12:38:15,864] jepsen worker 78 - jepsen.util 12588 :info :write [26 48] :RequestCanceledException INFO [2019-12-17 12:38:15,864] jepsen worker 0 - jepsen.util 18090 :info :cas [0 [15 22]] :RequestCanceledException INFO [2019-12-17 12:38:15,866] jepsen worker 78 - jepsen.util 12678 :invoke :read [26 nil] INFO [2019-12-17 12:38:15,866] jepsen worker 84 - jepsen.util 17184 :invoke :read [28 nil] INFO [2019-12-17 12:38:15,866] jepsen worker 0 - jepsen.util 18180 :invoke :write [0 48] INFO [2019-12-17 12:38:15,879] jepsen worker 46 - jepsen.util 12556 :ok :read [15 18] INFO [2019-12-17 12:38:15,879] jepsen worker 46 - jepsen.util 12556 :invoke :write [15 27] INFO [2019-12-17 12:38:15,901] jepsen worker 30 - jepsen.util 12270 :info :write [10 0] :RequestCanceledException INFO [2019-12-17 12:38:15,901] jepsen worker 30 - jepsen.util 12360 :invoke :read [10 nil] INFO [2019-12-17 12:38:15,919] jepsen worker 74 - jepsen.util 12854 :info :cas [24 [34 18]] :RequestCanceledException INFO [2019-12-17 12:38:15,919] jepsen worker 10 - jepsen.util 12790 :info :cas [3 [2 11]] :RequestCanceledException INFO [2019-12-17 12:38:15,920] jepsen worker 10 - jepsen.util 12880 :invoke :cas [3 [45 10]] INFO [2019-12-17 12:38:15,920] jepsen worker 74 - jepsen.util 12944 :invoke :cas [24 [34 38]] INFO [2019-12-17 12:38:15,931] jepsen worker 78 - jepsen.util 12678 :ok :read [26 4] INFO [2019-12-17 12:38:15,931] jepsen worker 78 - jepsen.util 12678 :invoke :write [26 11] INFO [2019-12-17 12:38:15,935] jepsen worker 5 - jepsen.util 12605 :info :write [1 25] :RequestCanceledException INFO [2019-12-17 12:38:15,935] jepsen worker 21 - jepsen.util 17211 :info :write [7 16] :RequestCanceledException INFO [2019-12-17 12:38:15,935] jepsen worker 47 - jepsen.util 14087 :info :cas [15 [38 20]] :RequestCanceledException INFO [2019-12-17 12:38:15,936] jepsen worker 21 - jepsen.util 17301 :invoke :cas [7 [2 35]] INFO [2019-12-17 12:38:15,937] jepsen worker 5 - jepsen.util 12695 :invoke :write [1 28] INFO [2019-12-17 12:38:15,937] jepsen worker 47 - jepsen.util 14177 :invoke :read [15 nil] INFO [2019-12-17 12:38:15,952] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":74892,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:38:15,953] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":74892,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:38:15,953] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":74895,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:38:15,953] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":74895,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:38:15,953] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":74896,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:38:15,953] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":74896,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:38:15,954] jepsen worker 72 - jepsen.util 13032 :ok :write [24 16] INFO [2019-12-17 12:38:15,954] jepsen worker 72 - jepsen.util 13032 :invoke :read [24 nil] INFO [2019-12-17 12:38:15,965] jepsen worker 74 - jepsen.util 12944 :fail :cas [24 [34 38]] :ValueNotSwapFrom INFO [2019-12-17 12:38:15,966] jepsen worker 74 - jepsen.util 12944 :invoke :write [24 29] INFO [2019-12-17 12:38:15,971] jepsen worker 47 - jepsen.util 14177 :ok :read [15 18] INFO [2019-12-17 12:38:15,971] jepsen worker 47 - jepsen.util 14177 :invoke :read [15 nil] INFO [2019-12-17 12:38:16,002] jepsen worker 23 - jepsen.util 17933 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:38:16,002] jepsen worker 23 - jepsen.util 17933 :invoke :cas [7 [34 0]] INFO [2019-12-17 12:38:16,008] jepsen worker 52 - jepsen.util 12562 :fail :cas [17 [11 30]] :ValueNotSwapFrom INFO [2019-12-17 12:38:16,008] jepsen worker 52 - jepsen.util 12562 :invoke :write [17 47] INFO [2019-12-17 12:38:16,019] jepsen worker 82 - jepsen.util 17812 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:38:16,019] jepsen worker 82 - jepsen.util 17812 :invoke :read [27 nil] INFO [2019-12-17 12:38:16,021] jepsen worker 72 - jepsen.util 13032 :ok :read [24 16] INFO [2019-12-17 12:38:16,021] jepsen worker 72 - jepsen.util 13032 :invoke :cas [24 [26 34]] INFO [2019-12-17 12:38:16,032] jepsen worker 47 - jepsen.util 14177 :ok :read [15 18] INFO [2019-12-17 12:38:16,032] jepsen worker 47 - jepsen.util 14177 :invoke :cas [15 [19 9]] INFO [2019-12-17 12:38:16,039] jepsen worker 63 - jepsen.util 11493 :ok :write [21 15] INFO [2019-12-17 12:38:16,039] jepsen worker 63 - jepsen.util 11493 :invoke :read [21 nil] INFO [2019-12-17 12:38:16,044] jepsen worker 72 - jepsen.util 13032 :fail :cas [24 [26 34]] :ValueNotSwapFrom INFO [2019-12-17 12:38:16,044] jepsen worker 72 - jepsen.util 13032 :invoke :read [24 nil] INFO [2019-12-17 12:38:16,053] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":74912,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:38:16,053] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":74912,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:38:16,053] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":74916,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:38:16,053] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":74916,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:38:16,054] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":74917,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:38:16,054] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":74917,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:38:16,067] jepsen worker 47 - jepsen.util 14177 :fail :cas [15 [19 9]] :ValueNotSwapFrom INFO [2019-12-17 12:38:16,067] jepsen worker 47 - jepsen.util 14177 :invoke :read [15 nil] INFO [2019-12-17 12:38:16,087] jepsen worker 63 - jepsen.util 11493 :ok :read [21 15] INFO [2019-12-17 12:38:16,087] jepsen worker 63 - jepsen.util 11493 :invoke :cas [21 [7 30]] INFO [2019-12-17 12:38:16,087] jepsen worker 4 - jepsen.util 12064 :fail :cas [1 [2 43]] :ValueNotSwapFrom INFO [2019-12-17 12:38:16,088] jepsen worker 4 - jepsen.util 12064 :invoke :write [1 45] INFO [2019-12-17 12:38:16,102] jepsen worker 45 - jepsen.util 12645 :fail :read [15 nil] :RequestTimeoutException INFO [2019-12-17 12:38:16,102] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:38:16,102] jepsen nemesis - couchbase.util Rebalance check count: 590 INFO [2019-12-17 12:38:16,102] jepsen worker 45 - jepsen.util 12645 :invoke :cas [15 [21 7]] INFO [2019-12-17 12:38:16,102] jepsen nemesis - couchbase.util Rebalance stuck count: 571 INFO [2019-12-17 12:38:16,119] jepsen worker 72 - jepsen.util 13032 :ok :read [24 16] INFO [2019-12-17 12:38:16,119] jepsen worker 72 - jepsen.util 13032 :invoke :write [24 32] INFO [2019-12-17 12:38:16,128] jepsen worker 13 - jepsen.util 12523 :fail :cas [4 [6 20]] :ValueNotSwapFrom INFO [2019-12-17 12:38:16,128] jepsen worker 13 - jepsen.util 12523 :invoke :cas [4 [40 19]] INFO [2019-12-17 12:38:16,134] jepsen worker 19 - jepsen.util 13609 :info :write [6 38] :RequestCanceledException INFO [2019-12-17 12:38:16,137] jepsen worker 19 - jepsen.util 13699 :invoke :read [6 nil] INFO [2019-12-17 12:38:16,169] jepsen worker 38 - jepsen.util 13628 :ok :read [12 48] INFO [2019-12-17 12:38:16,169] jepsen worker 38 - jepsen.util 13628 :invoke :write [12 16] INFO [2019-12-17 12:38:16,175] jepsen worker 63 - jepsen.util 11493 :fail :cas [21 [7 30]] :ValueNotSwapFrom INFO [2019-12-17 12:38:16,175] jepsen worker 63 - jepsen.util 11493 :invoke :read [21 nil] INFO [2019-12-17 12:38:16,185] jepsen worker 47 - jepsen.util 14177 :ok :read [15 18] INFO [2019-12-17 12:38:16,185] jepsen worker 47 - jepsen.util 14177 :invoke :cas [15 [40 9]] INFO [2019-12-17 12:38:16,188] jepsen worker 19 - jepsen.util 13699 :ok :read [6 47] INFO [2019-12-17 12:38:16,189] jepsen worker 19 - jepsen.util 13699 :invoke :cas [6 [7 5]] INFO [2019-12-17 12:38:16,197] jepsen worker 56 - jepsen.util 13736 :ok :read [18 43] INFO [2019-12-17 12:38:16,197] jepsen worker 56 - jepsen.util 13736 :invoke :read [18 nil] INFO [2019-12-17 12:38:16,210] jepsen worker 14 - jepsen.util 12614 :fail :cas [4 [0 36]] :ValueNotSwapFrom INFO [2019-12-17 12:38:16,210] jepsen worker 14 - jepsen.util 12614 :invoke :cas [4 [3 20]] INFO [2019-12-17 12:38:16,217] jepsen worker 63 - jepsen.util 11493 :ok :read [21 15] INFO [2019-12-17 12:38:16,217] jepsen worker 63 - jepsen.util 11493 :invoke :write [21 43] INFO [2019-12-17 12:38:16,248] jepsen worker 55 - jepsen.util 13645 :info :write [18 47] :RequestCanceledException INFO [2019-12-17 12:38:16,249] jepsen worker 55 - jepsen.util 13735 :invoke :cas [18 [36 27]] INFO [2019-12-17 12:38:16,258] jepsen worker 53 - jepsen.util 12473 :fail :cas [17 [1 3]] :ValueNotSwapFrom INFO [2019-12-17 12:38:16,259] jepsen worker 53 - jepsen.util 12473 :invoke :cas [17 [37 8]] INFO [2019-12-17 12:38:16,263] jepsen worker 19 - jepsen.util 13699 :fail :cas [6 [7 5]] :ValueNotSwapFrom INFO [2019-12-17 12:38:16,263] jepsen worker 19 - jepsen.util 13699 :invoke :cas [6 [43 9]] INFO [2019-12-17 12:38:16,272] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":74936,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:38:16,272] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":74944,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:38:16,272] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":74936,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:38:16,272] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":74944,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:38:16,272] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":74938,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:38:16,272] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":74947,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:38:16,272] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":74938,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:38:16,273] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":74947,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:38:16,273] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":74948,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:38:16,273] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":74948,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:38:16,279] jepsen worker 46 - jepsen.util 12556 :ok :write [15 27] INFO [2019-12-17 12:38:16,279] jepsen worker 20 - jepsen.util 12800 :ok :write [6 23] INFO [2019-12-17 12:38:16,279] jepsen worker 46 - jepsen.util 12556 :invoke :read [15 nil] INFO [2019-12-17 12:38:16,279] jepsen worker 20 - jepsen.util 12800 :invoke :read [6 nil] INFO [2019-12-17 12:38:16,299] jepsen worker 74 - jepsen.util 12944 :ok :write [24 29] INFO [2019-12-17 12:38:16,300] jepsen worker 74 - jepsen.util 12944 :invoke :cas [24 [2 7]] INFO [2019-12-17 12:38:16,301] jepsen worker 32 - jepsen.util 13262 :info :write [10 32] :RequestCanceledException INFO [2019-12-17 12:38:16,301] jepsen worker 12 - jepsen.util 13062 :info :write [4 29] :RequestCanceledException INFO [2019-12-17 12:38:16,301] jepsen worker 79 - jepsen.util 13129 :info :write [26 37] :RequestCanceledException INFO [2019-12-17 12:38:16,302] jepsen worker 32 - jepsen.util 13352 :invoke :read [10 nil] INFO [2019-12-17 12:38:16,303] jepsen worker 12 - jepsen.util 13152 :invoke :write [4 7] INFO [2019-12-17 12:38:16,303] jepsen worker 79 - jepsen.util 13219 :invoke :cas [26 [21 46]] INFO [2019-12-17 12:38:16,311] jepsen worker 10 - jepsen.util 12880 :fail :cas [3 [45 10]] :ValueNotSwapFrom INFO [2019-12-17 12:38:16,311] jepsen worker 10 - jepsen.util 12880 :invoke :read [3 nil] INFO [2019-12-17 12:38:16,313] jepsen worker 46 - jepsen.util 12556 :ok :read [15 27] INFO [2019-12-17 12:38:16,313] jepsen worker 46 - jepsen.util 12556 :invoke :write [15 8] INFO [2019-12-17 12:38:16,323] jepsen worker 45 - jepsen.util 12645 :fail :cas [15 [21 7]] :ValueNotSwapFrom INFO [2019-12-17 12:38:16,323] jepsen worker 45 - jepsen.util 12645 :invoke :cas [15 [35 34]] INFO [2019-12-17 12:38:16,337] jepsen worker 13 - jepsen.util 12523 :fail :cas [4 [40 19]] :ValueNotSwapFrom INFO [2019-12-17 12:38:16,337] jepsen worker 13 - jepsen.util 12523 :invoke :write [4 18] INFO [2019-12-17 12:38:16,348] jepsen worker 37 - jepsen.util 12457 :info :write [12 37] :RequestCanceledException INFO [2019-12-17 12:38:16,349] jepsen worker 37 - jepsen.util 12547 :invoke :cas [12 [19 45]] INFO [2019-12-17 12:38:16,357] jepsen worker 14 - jepsen.util 12614 :fail :cas [4 [3 20]] :ValueNotSwapFrom INFO [2019-12-17 12:38:16,357] jepsen worker 14 - jepsen.util 12614 :invoke :cas [4 [38 38]] INFO [2019-12-17 12:38:16,365] jepsen worker 65 - jepsen.util 11225 :info :cas [21 [46 14]] :RequestCanceledException INFO [2019-12-17 12:38:16,365] jepsen worker 77 - jepsen.util 18797 :info :write [25 47] :RequestCanceledException INFO [2019-12-17 12:38:16,366] jepsen worker 65 - jepsen.util 11315 :invoke :cas [21 [9 22]] INFO [2019-12-17 12:38:16,366] jepsen worker 77 - jepsen.util 18887 :invoke :cas [25 [1 35]] INFO [2019-12-17 12:38:16,384] jepsen worker 20 - jepsen.util 12800 :ok :read [6 23] INFO [2019-12-17 12:38:16,384] jepsen worker 20 - jepsen.util 12800 :invoke :read [6 nil] INFO [2019-12-17 12:38:16,402] jepsen worker 74 - jepsen.util 12944 :fail :cas [24 [2 7]] :ValueNotSwapFrom INFO [2019-12-17 12:38:16,402] jepsen worker 74 - jepsen.util 12944 :invoke :read [24 nil] INFO [2019-12-17 12:38:16,413] jepsen worker 63 - jepsen.util 11493 :ok :write [21 43] INFO [2019-12-17 12:38:16,413] jepsen worker 63 - jepsen.util 11493 :invoke :read [21 nil] INFO [2019-12-17 12:38:16,419] jepsen worker 32 - jepsen.util 13352 :ok :read [10 45] INFO [2019-12-17 12:38:16,419] jepsen worker 2 - jepsen.util 17012 :info :write [0 27] :RequestCanceledException INFO [2019-12-17 12:38:16,419] jepsen worker 32 - jepsen.util 13352 :invoke :write [10 17] INFO [2019-12-17 12:38:16,420] jepsen worker 2 - jepsen.util 17102 :invoke :read [0 nil] INFO [2019-12-17 12:38:16,442] jepsen worker 45 - jepsen.util 12645 :fail :cas [15 [35 34]] :ValueNotSwapFrom INFO [2019-12-17 12:38:16,443] jepsen worker 45 - jepsen.util 12645 :invoke :read [15 nil] INFO [2019-12-17 12:38:16,448] jepsen worker 31 - jepsen.util 11101 :info :cas [10 [40 29]] :RequestCanceledException INFO [2019-12-17 12:38:16,448] jepsen worker 35 - jepsen.util 17855 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:38:16,448] jepsen worker 35 - jepsen.util 17855 :invoke :write [11 42] INFO [2019-12-17 12:38:16,449] jepsen worker 31 - jepsen.util 11191 :invoke :write [10 35] INFO [2019-12-17 12:38:16,465] jepsen worker 44 - jepsen.util 18404 :info :cas [14 [2 42]] :RequestCanceledException INFO [2019-12-17 12:38:16,471] jepsen worker 19 - jepsen.util 13699 :fail :cas [6 [43 9]] :ValueNotSwapFrom INFO [2019-12-17 12:38:16,478] jepsen worker 44 - jepsen.util 18494 :invoke :read [14 nil] INFO [2019-12-17 12:38:16,479] jepsen worker 19 - jepsen.util 13699 :invoke :cas [6 [33 42]] INFO [2019-12-17 12:38:16,502] jepsen worker 10 - jepsen.util 12880 :ok :read [3 47] INFO [2019-12-17 12:38:16,502] jepsen worker 10 - jepsen.util 12880 :invoke :cas [3 [25 27]] INFO [2019-12-17 12:38:16,507] jepsen worker 74 - jepsen.util 12944 :ok :read [24 29] INFO [2019-12-17 12:38:16,507] jepsen worker 74 - jepsen.util 12944 :invoke :write [24 34] INFO [2019-12-17 12:38:16,512] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":74973,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:38:16,512] jepsen worker 15 - jepsen.util 16845 :info :cas [5 [44 39]] :RequestCanceledException INFO [2019-12-17 12:38:16,512] jepsen worker 9 - jepsen.util 13329 :info :cas [3 [41 24]] :RequestCanceledException INFO [2019-12-17 12:38:16,512] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":74973,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:38:16,512] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":74977,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:38:16,512] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":74977,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:38:16,513] jepsen worker 15 - jepsen.util 16935 :invoke :read [5 nil] INFO [2019-12-17 12:38:16,513] jepsen worker 9 - jepsen.util 13419 :invoke :cas [3 [3 17]] INFO [2019-12-17 12:38:16,515] jepsen worker 20 - jepsen.util 12800 :ok :read [6 23] INFO [2019-12-17 12:38:16,516] jepsen worker 20 - jepsen.util 12800 :invoke :read [6 nil] INFO [2019-12-17 12:38:16,526] jepsen worker 65 - jepsen.util 11315 :fail :cas [21 [9 22]] :ValueNotSwapFrom INFO [2019-12-17 12:38:16,526] jepsen worker 65 - jepsen.util 11315 :invoke :cas [21 [33 39]] INFO [2019-12-17 12:38:16,534] jepsen worker 81 - jepsen.util 17181 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:38:16,534] jepsen worker 48 - jepsen.util 16518 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:38:16,534] jepsen worker 81 - jepsen.util 17181 :invoke :write [27 38] INFO [2019-12-17 12:38:16,534] jepsen worker 48 - jepsen.util 16518 :invoke :cas [16 [37 8]] INFO [2019-12-17 12:38:16,535] jepsen worker 10 - jepsen.util 12880 :fail :cas [3 [25 27]] :ValueNotSwapFrom INFO [2019-12-17 12:38:16,535] jepsen worker 10 - jepsen.util 12880 :invoke :cas [3 [18 29]] INFO [2019-12-17 12:38:16,543] jepsen worker 63 - jepsen.util 11493 :ok :read [21 43] INFO [2019-12-17 12:38:16,543] jepsen worker 63 - jepsen.util 11493 :invoke :cas [21 [20 33]] INFO [2019-12-17 12:38:16,549] jepsen worker 20 - jepsen.util 12800 :ok :read [6 23] INFO [2019-12-17 12:38:16,550] jepsen worker 20 - jepsen.util 12800 :invoke :write [6 35] INFO [2019-12-17 12:38:16,556] jepsen worker 19 - jepsen.util 13699 :fail :cas [6 [33 42]] :ValueNotSwapFrom INFO [2019-12-17 12:38:16,556] jepsen worker 19 - jepsen.util 13699 :invoke :write [6 27] INFO [2019-12-17 12:38:16,563] jepsen worker 10 - jepsen.util 12880 :fail :cas [3 [18 29]] :ValueNotSwapFrom INFO [2019-12-17 12:38:16,563] jepsen worker 10 - jepsen.util 12880 :invoke :cas [3 [5 49]] INFO [2019-12-17 12:38:16,564] jepsen worker 58 - jepsen.util 17968 :info :cas [19 [28 31]] :RequestCanceledException INFO [2019-12-17 12:38:16,564] jepsen worker 50 - jepsen.util 17600 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:38:16,564] jepsen worker 73 - jepsen.util 11953 :info :write [24 1] :RequestCanceledException INFO [2019-12-17 12:38:16,564] jepsen worker 50 - jepsen.util 17600 :invoke :cas [16 [3 39]] INFO [2019-12-17 12:38:16,565] jepsen worker 73 - jepsen.util 12043 :invoke :write [24 33] INFO [2019-12-17 12:38:16,565] jepsen worker 58 - jepsen.util 18058 :invoke :write [19 28] INFO [2019-12-17 12:38:16,571] jepsen worker 9 - jepsen.util 13419 :fail :cas [3 [3 17]] :ValueNotSwapFrom INFO [2019-12-17 12:38:16,571] jepsen worker 9 - jepsen.util 13419 :invoke :read [3 nil] INFO [2019-12-17 12:38:16,601] jepsen worker 16 - jepsen.util 19186 :info :write [5 17] :RequestCanceledException INFO [2019-12-17 12:38:16,603] jepsen worker 65 - jepsen.util 11315 :fail :cas [21 [33 39]] :ValueNotSwapFrom INFO [2019-12-17 12:38:16,603] jepsen worker 65 - jepsen.util 11315 :invoke :cas [21 [23 41]] INFO [2019-12-17 12:38:16,604] jepsen worker 16 - jepsen.util 19276 :invoke :write [5 7] INFO [2019-12-17 12:38:16,619] jepsen worker 43 - jepsen.util 17953 :info :cas [14 [38 38]] :RequestCanceledException INFO [2019-12-17 12:38:16,621] jepsen worker 43 - jepsen.util 18043 :invoke :read [14 nil] INFO [2019-12-17 12:38:16,635] jepsen worker 61 - jepsen.util 18781 :info :cas [20 [1 9]] :RequestCanceledException INFO [2019-12-17 12:38:16,635] jepsen worker 25 - jepsen.util 18745 :info :write [8 3] :RequestCanceledException INFO [2019-12-17 12:38:16,636] jepsen worker 61 - jepsen.util 18871 :invoke :write [20 30] INFO [2019-12-17 12:38:16,636] jepsen worker 25 - jepsen.util 18835 :invoke :write [8 45] INFO [2019-12-17 12:38:16,638] jepsen worker 63 - jepsen.util 11493 :fail :cas [21 [20 33]] :ValueNotSwapFrom INFO [2019-12-17 12:38:16,638] jepsen worker 63 - jepsen.util 11493 :invoke :cas [21 [38 31]] INFO [2019-12-17 12:38:16,640] jepsen worker 56 - jepsen.util 13736 :ok :read [18 43] INFO [2019-12-17 12:38:16,640] jepsen worker 56 - jepsen.util 13736 :invoke :read [18 nil] INFO [2019-12-17 12:38:16,641] jepsen worker 57 - jepsen.util 17337 :info :write [19 9] :RequestCanceledException INFO [2019-12-17 12:38:16,641] jepsen worker 8 - jepsen.util 17198 :info :write [2 12] :RequestCanceledException INFO [2019-12-17 12:38:16,642] jepsen worker 8 - jepsen.util 17288 :invoke :write [2 12] INFO [2019-12-17 12:38:16,642] jepsen worker 57 - jepsen.util 17427 :invoke :read [19 nil] INFO [2019-12-17 12:38:16,655] jepsen worker 55 - jepsen.util 13735 :fail :cas [18 [36 27]] :ValueNotSwapFrom INFO [2019-12-17 12:38:16,655] jepsen worker 55 - jepsen.util 13735 :invoke :read [18 nil] INFO [2019-12-17 12:38:16,666] jepsen worker 7 - jepsen.util 17467 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:38:16,666] jepsen worker 3 - jepsen.util 12513 :info :write [1 21] :RequestCanceledException INFO [2019-12-17 12:38:16,666] jepsen worker 7 - jepsen.util 17467 :invoke :read [2 nil] INFO [2019-12-17 12:38:16,667] jepsen worker 3 - jepsen.util 12603 :invoke :cas [1 [49 27]] INFO [2019-12-17 12:38:16,680] jepsen worker 37 - jepsen.util 12547 :fail :cas [12 [19 45]] :ValueNotSwapFrom INFO [2019-12-17 12:38:16,680] jepsen worker 37 - jepsen.util 12547 :invoke :cas [12 [26 4]] INFO [2019-12-17 12:38:16,686] jepsen worker 53 - jepsen.util 12473 :fail :cas [17 [37 8]] :ValueNotSwapFrom INFO [2019-12-17 12:38:16,687] jepsen worker 53 - jepsen.util 12473 :invoke :write [17 26] INFO [2019-12-17 12:38:16,700] jepsen worker 87 - jepsen.util 12417 :info :write [29 39] :RequestCanceledException INFO [2019-12-17 12:38:16,701] jepsen worker 87 - jepsen.util 12507 :invoke :read [29 nil] INFO [2019-12-17 12:38:16,716] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":75010,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:38:16,716] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":75010,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:38:16,716] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":75013,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:38:16,716] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":75013,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:38:16,716] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":75014,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:38:16,716] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":75014,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:38:16,734] jepsen worker 59 - jepsen.util 16889 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:38:16,734] jepsen worker 59 - jepsen.util 16889 :invoke :read [19 nil] INFO [2019-12-17 12:38:16,738] jepsen worker 87 - jepsen.util 12507 :ok :read [29 33] INFO [2019-12-17 12:38:16,738] jepsen worker 87 - jepsen.util 12507 :invoke :cas [29 [1 46]] INFO [2019-12-17 12:38:16,809] jepsen worker 14 - jepsen.util 12614 :fail :cas [4 [38 38]] :ValueNotSwapFrom INFO [2019-12-17 12:38:16,810] jepsen worker 14 - jepsen.util 12614 :invoke :write [4 10] INFO [2019-12-17 12:38:16,819] jepsen worker 83 - jepsen.util 17633 :info :write [27 44] :RequestCanceledException INFO [2019-12-17 12:38:16,820] jepsen worker 83 - jepsen.util 17723 :invoke :cas [27 [13 35]] INFO [2019-12-17 12:38:16,829] jepsen worker 46 - jepsen.util 12556 :ok :write [15 8] INFO [2019-12-17 12:38:16,829] jepsen worker 46 - jepsen.util 12556 :invoke :cas [15 [27 11]] INFO [2019-12-17 12:38:16,830] jepsen worker 32 - jepsen.util 13352 :ok :write [10 17] INFO [2019-12-17 12:38:16,830] jepsen worker 32 - jepsen.util 13352 :invoke :write [10 35] INFO [2019-12-17 12:38:16,834] jepsen worker 42 - jepsen.util 18582 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:38:16,835] jepsen worker 42 - jepsen.util 18582 :invoke :cas [14 [35 32]] INFO [2019-12-17 12:38:16,863] jepsen worker 39 - jepsen.util 17229 :info :write [13 41] :RequestCanceledException INFO [2019-12-17 12:38:16,865] jepsen worker 39 - jepsen.util 17319 :invoke :cas [13 [0 41]] INFO [2019-12-17 12:38:16,924] jepsen worker 79 - jepsen.util 13219 :fail :cas [26 [21 46]] :ValueNotSwapFrom INFO [2019-12-17 12:38:16,925] jepsen worker 79 - jepsen.util 13219 :invoke :cas [26 [47 32]] INFO [2019-12-17 12:38:16,934] jepsen worker 1 - jepsen.util 18001 :info :cas [0 [10 16]] :RequestCanceledException INFO [2019-12-17 12:38:16,935] jepsen worker 1 - jepsen.util 18091 :invoke :write [0 18] INFO [2019-12-17 12:38:16,935] jepsen worker 55 - jepsen.util 13735 :ok :read [18 43] INFO [2019-12-17 12:38:16,935] jepsen worker 55 - jepsen.util 13735 :invoke :cas [18 [36 4]] INFO [2019-12-17 12:38:16,936] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":75030,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:38:16,936] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":75030,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:38:16,936] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":75033,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:38:16,936] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":75033,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:38:16,946] jepsen worker 56 - jepsen.util 13736 :ok :read [18 43] INFO [2019-12-17 12:38:16,946] jepsen worker 56 - jepsen.util 13736 :invoke :read [18 nil] INFO [2019-12-17 12:38:16,952] jepsen worker 37 - jepsen.util 12547 :fail :cas [12 [26 4]] :ValueNotSwapFrom INFO [2019-12-17 12:38:16,952] jepsen worker 37 - jepsen.util 12547 :invoke :write [12 8] INFO [2019-12-17 12:38:17,001] jepsen worker 22 - jepsen.util 18022 :info :cas [7 [1 20]] :RequestCanceledException INFO [2019-12-17 12:38:17,002] jepsen worker 22 - jepsen.util 18112 :invoke :cas [7 [12 47]] INFO [2019-12-17 12:38:17,016] jepsen worker 56 - jepsen.util 13736 :ok :read [18 43] INFO [2019-12-17 12:38:17,016] jepsen worker 56 - jepsen.util 13736 :invoke :cas [18 [16 28]] INFO [2019-12-17 12:38:17,018] jepsen worker 6 - jepsen.util 18096 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:38:17,019] jepsen worker 6 - jepsen.util 18096 :invoke :cas [2 [42 49]] INFO [2019-12-17 12:38:17,040] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":75041,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:38:17,040] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":75041,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:38:17,040] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":75045,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:38:17,040] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":75045,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:38:17,049] jepsen worker 80 - jepsen.util 12320 :info :write [26 17] :RequestCanceledException INFO [2019-12-17 12:38:17,049] jepsen worker 68 - jepsen.util 16898 :info :write [22 23] :RequestCanceledException INFO [2019-12-17 12:38:17,049] jepsen worker 68 - jepsen.util 16988 :invoke :write [22 25] INFO [2019-12-17 12:38:17,049] jepsen worker 80 - jepsen.util 12410 :invoke :write [26 30] INFO [2019-12-17 12:38:17,065] jepsen worker 76 - jepsen.util 17806 :info :cas [25 [34 6]] :RequestCanceledException INFO [2019-12-17 12:38:17,066] jepsen worker 76 - jepsen.util 17896 :invoke :write [25 13] INFO [2019-12-17 12:38:17,089] jepsen worker 55 - jepsen.util 13735 :fail :cas [18 [36 4]] :ValueNotSwapFrom INFO [2019-12-17 12:38:17,090] jepsen worker 55 - jepsen.util 13735 :invoke :cas [18 [23 46]] INFO [2019-12-17 12:38:17,114] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:38:17,114] jepsen nemesis - couchbase.util Rebalance check count: 591 INFO [2019-12-17 12:38:17,114] jepsen nemesis - couchbase.util Rebalance stuck count: 572 INFO [2019-12-17 12:38:17,134] jepsen worker 89 - jepsen.util 12869 :info :write [29 46] :RequestCanceledException INFO [2019-12-17 12:38:17,135] jepsen worker 89 - jepsen.util 12959 :invoke :read [29 nil] INFO [2019-12-17 12:38:17,201] jepsen worker 34 - jepsen.util 18664 :info :write [11 11] :RequestCanceledException INFO [2019-12-17 12:38:17,202] jepsen worker 34 - jepsen.util 18754 :invoke :write [11 9] INFO [2019-12-17 12:38:17,203] jepsen worker 89 - jepsen.util 12959 :ok :read [29 33] INFO [2019-12-17 12:38:17,203] jepsen worker 89 - jepsen.util 12959 :invoke :read [29 nil] INFO [2019-12-17 12:38:17,219] jepsen worker 49 - jepsen.util 17689 :info :cas [16 [10 30]] :RequestCanceledException INFO [2019-12-17 12:38:17,219] jepsen worker 31 - jepsen.util 11191 :ok :write [10 35] INFO [2019-12-17 12:38:17,219] jepsen worker 20 - jepsen.util 12800 :ok :write [6 35] INFO [2019-12-17 12:38:17,219] jepsen worker 31 - jepsen.util 11191 :invoke :read [10 nil] INFO [2019-12-17 12:38:17,220] jepsen worker 20 - jepsen.util 12800 :invoke :write [6 0] INFO [2019-12-17 12:38:17,221] jepsen worker 49 - jepsen.util 17779 :invoke :write [16 17] INFO [2019-12-17 12:38:17,222] jepsen worker 56 - jepsen.util 13736 :fail :cas [18 [16 28]] :ValueNotSwapFrom INFO [2019-12-17 12:38:17,222] jepsen worker 56 - jepsen.util 13736 :invoke :cas [18 [41 2]] INFO [2019-12-17 12:38:17,235] jepsen worker 40 - jepsen.util 18400 :info :write [13 47] :RequestCanceledException INFO [2019-12-17 12:38:17,236] jepsen worker 40 - jepsen.util 18490 :invoke :write [13 20] INFO [2019-12-17 12:38:17,248] jepsen worker 41 - jepsen.util 19031 :info :cas [13 [8 37]] :RequestCanceledException INFO [2019-12-17 12:38:17,248] jepsen worker 24 - jepsen.util 16764 :info :cas [8 [35 29]] :RequestCanceledException INFO [2019-12-17 12:38:17,248] jepsen worker 70 - jepsen.util 13390 :info :write [23 49] :RequestCanceledException INFO [2019-12-17 12:38:17,249] jepsen worker 41 - jepsen.util 19121 :invoke :read [13 nil] INFO [2019-12-17 12:38:17,249] jepsen worker 24 - jepsen.util 16854 :invoke :read [8 nil] INFO [2019-12-17 12:38:17,249] jepsen worker 70 - jepsen.util 13480 :invoke :read [23 nil] INFO [2019-12-17 12:38:17,252] jepsen worker 31 - jepsen.util 11191 :ok :read [10 35] INFO [2019-12-17 12:38:17,252] jepsen worker 31 - jepsen.util 11191 :invoke :cas [10 [18 16]] INFO [2019-12-17 12:38:17,259] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":75055,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:38:17,259] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":75055,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:38:17,259] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":75056,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:38:17,259] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":75056,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:38:17,259] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":75065,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:38:17,259] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":75065,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:38:17,266] jepsen worker 74 - jepsen.util 12944 :ok :write [24 34] INFO [2019-12-17 12:38:17,266] jepsen worker 74 - jepsen.util 12944 :invoke :write [24 22] INFO [2019-12-17 12:38:17,278] jepsen worker 31 - jepsen.util 11191 :fail :cas [10 [18 16]] :ValueNotSwapFrom INFO [2019-12-17 12:38:17,279] jepsen worker 31 - jepsen.util 11191 :invoke :read [10 nil] INFO [2019-12-17 12:38:17,298] jepsen worker 31 - jepsen.util 11191 :ok :read [10 35] INFO [2019-12-17 12:38:17,298] jepsen worker 31 - jepsen.util 11191 :invoke :write [10 45] INFO [2019-12-17 12:38:17,300] jepsen worker 18 - jepsen.util 13518 :info :write [6 16] :RequestCanceledException INFO [2019-12-17 12:38:17,304] jepsen worker 18 - jepsen.util 13608 :invoke :read [6 nil] INFO [2019-12-17 12:38:17,314] jepsen worker 56 - jepsen.util 13736 :fail :cas [18 [41 2]] :ValueNotSwapFrom INFO [2019-12-17 12:38:17,314] jepsen worker 56 - jepsen.util 13736 :invoke :read [18 nil] INFO [2019-12-17 12:38:17,339] jepsen worker 56 - jepsen.util 13736 :ok :read [18 43] INFO [2019-12-17 12:38:17,339] jepsen worker 56 - jepsen.util 13736 :invoke :read [18 nil] INFO [2019-12-17 12:38:17,371] jepsen worker 56 - jepsen.util 13736 :ok :read [18 43] INFO [2019-12-17 12:38:17,371] jepsen worker 56 - jepsen.util 13736 :invoke :cas [18 [39 34]] INFO [2019-12-17 12:38:17,399] jepsen worker 89 - jepsen.util 12959 :ok :read [29 33] INFO [2019-12-17 12:38:17,399] jepsen worker 20 - jepsen.util 12800 :ok :write [6 0] INFO [2019-12-17 12:38:17,411] jepsen worker 89 - jepsen.util 12959 :invoke :cas [29 [16 5]] INFO [2019-12-17 12:38:17,411] jepsen worker 20 - jepsen.util 12800 :invoke :write [6 15] INFO [2019-12-17 12:38:17,413] jepsen worker 18 - jepsen.util 13608 :ok :read [6 0] INFO [2019-12-17 12:38:17,413] jepsen worker 18 - jepsen.util 13608 :invoke :read [6 nil] INFO [2019-12-17 12:38:17,419] jepsen worker 29 - jepsen.util 11189 :info :write [9 13] :RequestCanceledException INFO [2019-12-17 12:38:17,419] jepsen worker 29 - jepsen.util 11279 :invoke :write [9 4] INFO [2019-12-17 12:38:17,427] jepsen worker 55 - jepsen.util 13735 :fail :cas [18 [23 46]] :ValueNotSwapFrom INFO [2019-12-17 12:38:17,428] jepsen worker 55 - jepsen.util 13735 :invoke :read [18 nil] INFO [2019-12-17 12:38:17,432] jepsen worker 36 - jepsen.util 13716 :info :write [12 15] :RequestCanceledException INFO [2019-12-17 12:38:17,432] jepsen worker 28 - jepsen.util 11008 :fail :read [9 nil] :RequestTimeoutException INFO [2019-12-17 12:38:17,432] jepsen worker 64 - jepsen.util 10054 :info :write [21 23] :RequestCanceledException INFO [2019-12-17 12:38:17,432] jepsen worker 11 - jepsen.util 12791 :info :cas [3 [7 16]] :RequestCanceledException INFO [2019-12-17 12:38:17,432] jepsen worker 28 - jepsen.util 11008 :invoke :write [9 13] INFO [2019-12-17 12:38:17,433] jepsen worker 36 - jepsen.util 13806 :invoke :cas [12 [3 13]] INFO [2019-12-17 12:38:17,433] jepsen worker 64 - jepsen.util 10144 :invoke :write [21 14] INFO [2019-12-17 12:38:17,434] jepsen worker 11 - jepsen.util 12881 :invoke :write [3 16] INFO [2019-12-17 12:38:17,436] jepsen worker 18 - jepsen.util 13608 :ok :read [6 0] INFO [2019-12-17 12:38:17,436] jepsen worker 18 - jepsen.util 13608 :invoke :cas [6 [17 2]] INFO [2019-12-17 12:38:17,449] jepsen worker 86 - jepsen.util 18446 :info :cas [28 [35 32]] :RequestCanceledException INFO [2019-12-17 12:38:17,450] jepsen worker 86 - jepsen.util 18536 :invoke :read [28 nil] INFO [2019-12-17 12:38:17,465] jepsen worker 62 - jepsen.util 18332 :info :cas [20 [2 44]] :RequestCanceledException INFO [2019-12-17 12:38:17,466] jepsen worker 62 - jepsen.util 18422 :invoke :read [20 nil] INFO [2019-12-17 12:38:17,484] jepsen worker 64 - jepsen.util 10144 :ok :write [21 14] INFO [2019-12-17 12:38:17,485] jepsen worker 64 - jepsen.util 10144 :invoke :write [21 1] INFO [2019-12-17 12:38:17,497] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":75092,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:38:17,497] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":75092,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:38:17,497] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":75094,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:38:17,497] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":75094,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:38:17,527] jepsen worker 64 - jepsen.util 10144 :ok :write [21 1] INFO [2019-12-17 12:38:17,527] jepsen worker 64 - jepsen.util 10144 :invoke :read [21 nil] INFO [2019-12-17 12:38:17,530] jepsen worker 70 - jepsen.util 13480 :ok :read [23 7] INFO [2019-12-17 12:38:17,531] jepsen worker 70 - jepsen.util 13480 :invoke :write [23 34] INFO [2019-12-17 12:38:17,536] jepsen worker 27 - jepsen.util 11997 :info :cas [9 [26 24]] :RequestCanceledException INFO [2019-12-17 12:38:17,536] jepsen worker 26 - jepsen.util 18656 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:38:17,536] jepsen worker 54 - jepsen.util 13554 :info :write [18 31] :RequestCanceledException INFO [2019-12-17 12:38:17,538] jepsen worker 27 - jepsen.util 12087 :invoke :read [9 nil] INFO [2019-12-17 12:38:17,538] jepsen worker 26 - jepsen.util 18656 :invoke :write [8 15] INFO [2019-12-17 12:38:17,539] jepsen worker 54 - jepsen.util 13644 :invoke :cas [18 [15 26]] INFO [2019-12-17 12:38:17,562] jepsen worker 75 - jepsen.util 17355 :info :cas [25 [31 35]] :RequestCanceledException INFO [2019-12-17 12:38:17,565] jepsen worker 64 - jepsen.util 10144 :ok :read [21 1] INFO [2019-12-17 12:38:17,565] jepsen worker 64 - jepsen.util 10144 :invoke :write [21 45] INFO [2019-12-17 12:38:17,567] jepsen worker 75 - jepsen.util 17445 :invoke :cas [25 [42 25]] INFO [2019-12-17 12:38:17,577] jepsen worker 55 - jepsen.util 13735 :ok :read [18 43] INFO [2019-12-17 12:38:17,578] jepsen worker 55 - jepsen.util 13735 :invoke :cas [18 [11 12]] INFO [2019-12-17 12:38:17,582] jepsen worker 56 - jepsen.util 13736 :fail :cas [18 [39 34]] :ValueNotSwapFrom INFO [2019-12-17 12:38:17,582] jepsen worker 56 - jepsen.util 13736 :invoke :cas [18 [37 39]] INFO [2019-12-17 12:38:17,601] jepsen worker 71 - jepsen.util 13751 :info :write [23 2] :RequestCanceledException INFO [2019-12-17 12:38:17,601] jepsen worker 60 - jepsen.util 18690 :info :write [20 30] :RequestCanceledException INFO [2019-12-17 12:38:17,602] jepsen worker 60 - jepsen.util 18780 :invoke :read [20 nil] INFO [2019-12-17 12:38:17,603] jepsen worker 71 - jepsen.util 13841 :invoke :read [23 nil] INFO [2019-12-17 12:38:17,604] jepsen worker 3 - jepsen.util 12603 :fail :cas [1 [49 27]] :ValueNotSwapFrom INFO [2019-12-17 12:38:17,604] jepsen worker 3 - jepsen.util 12603 :invoke :read [1 nil] INFO [2019-12-17 12:38:17,607] jepsen worker 31 - jepsen.util 11191 :ok :write [10 45] INFO [2019-12-17 12:38:17,607] jepsen worker 11 - jepsen.util 12881 :ok :write [3 16] INFO [2019-12-17 12:38:17,607] jepsen worker 31 - jepsen.util 11191 :invoke :write [10 40] INFO [2019-12-17 12:38:17,607] jepsen worker 11 - jepsen.util 12881 :invoke :read [3 nil] INFO [2019-12-17 12:38:17,608] jepsen worker 74 - jepsen.util 12944 :ok :write [24 22] INFO [2019-12-17 12:38:17,608] jepsen worker 74 - jepsen.util 12944 :invoke :write [24 17] INFO [2019-12-17 12:38:17,619] jepsen worker 17 - jepsen.util 17747 :info :cas [5 [14 38]] :RequestCanceledException INFO [2019-12-17 12:38:17,620] jepsen worker 17 - jepsen.util 17837 :invoke :cas [5 [20 12]] INFO [2019-12-17 12:38:17,634] jepsen worker 51 - jepsen.util 12201 :info :write [17 30] :RequestCanceledException INFO [2019-12-17 12:38:17,634] jepsen worker 69 - jepsen.util 13299 :info :write [23 32] :RequestCanceledException INFO [2019-12-17 12:38:17,635] jepsen worker 51 - jepsen.util 12291 :invoke :read [17 nil] INFO [2019-12-17 12:38:17,635] jepsen worker 69 - jepsen.util 13389 :invoke :write [23 15] INFO [2019-12-17 12:38:17,647] jepsen worker 11 - jepsen.util 12881 :ok :read [3 16] INFO [2019-12-17 12:38:17,647] jepsen worker 11 - jepsen.util 12881 :invoke :write [3 39] INFO [2019-12-17 12:38:17,650] jepsen worker 89 - jepsen.util 12959 :fail :cas [29 [16 5]] :ValueNotSwapFrom INFO [2019-12-17 12:38:17,650] jepsen worker 89 - jepsen.util 12959 :invoke :cas [29 [22 38]] INFO [2019-12-17 12:38:17,650] jepsen worker 64 - jepsen.util 10144 :ok :write [21 45] INFO [2019-12-17 12:38:17,651] jepsen worker 64 - jepsen.util 10144 :invoke :cas [21 [19 3]] INFO [2019-12-17 12:38:17,677] jepsen worker 20 - jepsen.util 12800 :ok :write [6 15] INFO [2019-12-17 12:38:17,677] jepsen worker 20 - jepsen.util 12800 :invoke :read [6 nil] INFO [2019-12-17 12:38:17,688] jepsen worker 56 - jepsen.util 13736 :fail :cas [18 [37 39]] :ValueNotSwapFrom INFO [2019-12-17 12:38:17,688] jepsen worker 56 - jepsen.util 13736 :invoke :cas [18 [40 49]] INFO [2019-12-17 12:38:17,703] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":75123,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:38:17,703] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":75123,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:38:17,703] jepsen worker 64 - jepsen.util 10144 :fail :cas [21 [19 3]] :ValueNotSwapFrom INFO [2019-12-17 12:38:17,704] jepsen worker 64 - jepsen.util 10144 :invoke :read [21 nil] INFO [2019-12-17 12:38:17,731] jepsen worker 56 - jepsen.util 13736 :fail :cas [18 [40 49]] :ValueNotSwapFrom INFO [2019-12-17 12:38:17,731] jepsen worker 56 - jepsen.util 13736 :invoke :read [18 nil] INFO [2019-12-17 12:38:17,734] jepsen worker 85 - jepsen.util 18265 :info :write [28 43] :RequestCanceledException INFO [2019-12-17 12:38:17,735] jepsen worker 33 - jepsen.util 17673 :info :cas [11 [47 21]] :RequestCanceledException INFO [2019-12-17 12:38:17,735] jepsen worker 85 - jepsen.util 18355 :invoke :read [28 nil] INFO [2019-12-17 12:38:17,736] jepsen worker 33 - jepsen.util 17763 :invoke :read [11 nil] INFO [2019-12-17 12:38:17,767] jepsen worker 36 - jepsen.util 13806 :fail :cas [12 [3 13]] :ValueNotSwapFrom INFO [2019-12-17 12:38:17,768] jepsen worker 36 - jepsen.util 13806 :invoke :write [12 17] INFO [2019-12-17 12:38:17,798] jepsen worker 67 - jepsen.util 17887 :info :write [22 43] :RequestCanceledException INFO [2019-12-17 12:38:17,798] jepsen worker 66 - jepsen.util 17706 :info :write [22 34] :RequestCanceledException INFO [2019-12-17 12:38:17,800] jepsen worker 67 - jepsen.util 17977 :invoke :cas [22 [42 21]] INFO [2019-12-17 12:38:17,800] jepsen worker 66 - jepsen.util 17796 :invoke :write [22 19] INFO [2019-12-17 12:38:17,800] jepsen worker 54 - jepsen.util 13644 :fail :cas [18 [15 26]] :ValueNotSwapFrom INFO [2019-12-17 12:38:17,800] jepsen worker 54 - jepsen.util 13644 :invoke :cas [18 [35 5]] INFO [2019-12-17 12:38:17,804] jepsen worker 18 - jepsen.util 13608 :fail :cas [6 [17 2]] :ValueNotSwapFrom INFO [2019-12-17 12:38:17,805] jepsen worker 18 - jepsen.util 13608 :invoke :write [6 36] INFO [2019-12-17 12:38:17,805] jepsen worker 64 - jepsen.util 10144 :ok :read [21 45] INFO [2019-12-17 12:38:17,805] jepsen worker 64 - jepsen.util 10144 :invoke :read [21 nil] INFO [2019-12-17 12:38:17,819] jepsen worker 88 - jepsen.util 12958 :info :write [29 11] :RequestCanceledException INFO [2019-12-17 12:38:17,820] jepsen worker 55 - jepsen.util 13735 :fail :cas [18 [11 12]] :ValueNotSwapFrom INFO [2019-12-17 12:38:17,820] jepsen worker 55 - jepsen.util 13735 :invoke :read [18 nil] INFO [2019-12-17 12:38:17,820] jepsen worker 88 - jepsen.util 13048 :invoke :write [29 11] INFO [2019-12-17 12:38:17,824] jepsen worker 56 - jepsen.util 13736 :ok :read [18 43] INFO [2019-12-17 12:38:17,824] jepsen worker 56 - jepsen.util 13736 :invoke :read [18 nil] INFO [2019-12-17 12:38:17,832] jepsen worker 64 - jepsen.util 10144 :ok :read [21 45] INFO [2019-12-17 12:38:17,832] jepsen worker 64 - jepsen.util 10144 :invoke :cas [21 [49 40]] INFO [2019-12-17 12:38:17,849] jepsen worker 29 - jepsen.util 11279 :ok :write [9 4] INFO [2019-12-17 12:38:17,849] jepsen worker 29 - jepsen.util 11279 :invoke :read [9 nil] INFO [2019-12-17 12:38:17,852] jepsen worker 3 - jepsen.util 12603 :ok :read [1 29] INFO [2019-12-17 12:38:17,852] jepsen worker 3 - jepsen.util 12603 :invoke :write [1 39] INFO [2019-12-17 12:38:17,859] jepsen worker 5 - jepsen.util 12695 :ok :write [1 28] INFO [2019-12-17 12:38:17,859] jepsen worker 5 - jepsen.util 12695 :invoke :write [1 35] INFO [2019-12-17 12:38:17,887] jepsen worker 64 - jepsen.util 10144 :fail :cas [21 [49 40]] :ValueNotSwapFrom INFO [2019-12-17 12:38:17,887] jepsen worker 64 - jepsen.util 10144 :invoke :write [21 33] INFO [2019-12-17 12:38:17,921] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":75150,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:38:17,921] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":75150,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:38:17,921] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":75154,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:38:17,921] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":75154,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:38:17,930] jepsen worker 64 - jepsen.util 10144 :ok :write [21 33] INFO [2019-12-17 12:38:17,930] jepsen worker 64 - jepsen.util 10144 :invoke :read [21 nil] INFO [2019-12-17 12:38:17,935] jepsen worker 56 - jepsen.util 13736 :ok :read [18 43] INFO [2019-12-17 12:38:17,935] jepsen worker 56 - jepsen.util 13736 :invoke :read [18 nil] INFO [2019-12-17 12:38:17,935] jepsen worker 84 - jepsen.util 17184 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:38:17,935] jepsen worker 84 - jepsen.util 17184 :invoke :write [28 7] INFO [2019-12-17 12:38:17,943] jepsen worker 14 - jepsen.util 12614 :ok :write [4 10] INFO [2019-12-17 12:38:17,943] jepsen worker 14 - jepsen.util 12614 :invoke :cas [4 [43 38]] INFO [2019-12-17 12:38:17,948] jepsen worker 64 - jepsen.util 10144 :ok :read [21 33] INFO [2019-12-17 12:38:17,948] jepsen worker 64 - jepsen.util 10144 :invoke :write [21 44] INFO [2019-12-17 12:38:17,948] jepsen worker 79 - jepsen.util 13219 :fail :cas [26 [47 32]] :ValueNotSwapFrom INFO [2019-12-17 12:38:17,948] jepsen worker 79 - jepsen.util 13219 :invoke :read [26 nil] INFO [2019-12-17 12:38:17,949] jepsen worker 0 - jepsen.util 18180 :info :write [0 48] :RequestCanceledException INFO [2019-12-17 12:38:17,950] jepsen worker 0 - jepsen.util 18270 :invoke :read [0 nil] INFO [2019-12-17 12:38:17,966] jepsen worker 30 - jepsen.util 12360 :fail :read [10 nil] :RequestTimeoutException INFO [2019-12-17 12:38:17,966] jepsen worker 30 - jepsen.util 12360 :invoke :cas [10 [32 38]] INFO [2019-12-17 12:38:17,977] jepsen worker 71 - jepsen.util 13841 :ok :read [23 7] INFO [2019-12-17 12:38:17,978] jepsen worker 71 - jepsen.util 13841 :invoke :read [23 nil] INFO [2019-12-17 12:38:17,989] jepsen worker 30 - jepsen.util 12360 :fail :cas [10 [32 38]] :ValueNotSwapFrom INFO [2019-12-17 12:38:17,989] jepsen worker 30 - jepsen.util 12360 :invoke :cas [10 [49 0]] INFO [2019-12-17 12:38:18,004] jepsen worker 30 - jepsen.util 12360 :fail :cas [10 [49 0]] :ValueNotSwapFrom INFO [2019-12-17 12:38:18,004] jepsen worker 30 - jepsen.util 12360 :invoke :read [10 nil] INFO [2019-12-17 12:38:18,006] jepsen worker 64 - jepsen.util 10144 :ok :write [21 44] INFO [2019-12-17 12:38:18,006] jepsen worker 64 - jepsen.util 10144 :invoke :read [21 nil] INFO [2019-12-17 12:38:18,015] jepsen worker 89 - jepsen.util 12959 :fail :cas [29 [22 38]] :ValueNotSwapFrom INFO [2019-12-17 12:38:18,015] jepsen worker 89 - jepsen.util 12959 :invoke :cas [29 [36 8]] INFO [2019-12-17 12:38:18,018] jepsen worker 78 - jepsen.util 12678 :info :write [26 11] :RequestCanceledException INFO [2019-12-17 12:38:18,018] jepsen worker 21 - jepsen.util 17301 :info :cas [7 [2 35]] :RequestCanceledException INFO [2019-12-17 12:38:18,019] jepsen worker 78 - jepsen.util 12768 :invoke :read [26 nil] INFO [2019-12-17 12:38:18,019] jepsen worker 21 - jepsen.util 17391 :invoke :write [7 14] INFO [2019-12-17 12:38:18,027] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":75168,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:38:18,027] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":75168,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:38:18,027] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":75171,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:38:18,027] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":75171,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:38:18,035] jepsen worker 52 - jepsen.util 12562 :info :write [17 47] :RequestCanceledException INFO [2019-12-17 12:38:18,038] jepsen worker 30 - jepsen.util 12360 :ok :read [10 45] INFO [2019-12-17 12:38:18,038] jepsen worker 64 - jepsen.util 10144 :ok :read [21 44] INFO [2019-12-17 12:38:18,038] jepsen worker 30 - jepsen.util 12360 :invoke :write [10 9] INFO [2019-12-17 12:38:18,038] jepsen worker 64 - jepsen.util 10144 :invoke :write [21 10] INFO [2019-12-17 12:38:18,038] jepsen worker 52 - jepsen.util 12652 :invoke :read [17 nil] INFO [2019-12-17 12:38:18,054] jepsen worker 56 - jepsen.util 13736 :ok :read [18 43] INFO [2019-12-17 12:38:18,054] jepsen worker 56 - jepsen.util 13736 :invoke :read [18 nil] INFO [2019-12-17 12:38:18,084] jepsen worker 64 - jepsen.util 10144 :ok :write [21 10] INFO [2019-12-17 12:38:18,084] jepsen worker 31 - jepsen.util 11191 :ok :write [10 40] INFO [2019-12-17 12:38:18,084] jepsen worker 64 - jepsen.util 10144 :invoke :write [21 6] INFO [2019-12-17 12:38:18,084] jepsen worker 31 - jepsen.util 11191 :invoke :cas [10 [24 38]] INFO [2019-12-17 12:38:18,101] jepsen worker 23 - jepsen.util 17933 :info :cas [7 [34 0]] :RequestCanceledException INFO [2019-12-17 12:38:18,102] jepsen worker 23 - jepsen.util 18023 :invoke :read [7 nil] INFO [2019-12-17 12:38:18,104] jepsen worker 31 - jepsen.util 11191 :fail :cas [10 [24 38]] :ValueNotSwapFrom INFO [2019-12-17 12:38:18,104] jepsen worker 31 - jepsen.util 11191 :invoke :read [10 nil] INFO [2019-12-17 12:38:18,119] jepsen worker 82 - jepsen.util 17812 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:38:18,119] jepsen worker 82 - jepsen.util 17812 :invoke :read [27 nil] INFO [2019-12-17 12:38:18,123] jepsen worker 74 - jepsen.util 12944 :ok :write [24 17] INFO [2019-12-17 12:38:18,124] jepsen worker 11 - jepsen.util 12881 :ok :write [3 39] INFO [2019-12-17 12:38:18,124] jepsen worker 74 - jepsen.util 12944 :invoke :read [24 nil] INFO [2019-12-17 12:38:18,124] jepsen worker 11 - jepsen.util 12881 :invoke :cas [3 [26 20]] INFO [2019-12-17 12:38:18,137] jepsen worker 31 - jepsen.util 11191 :ok :read [10 40] INFO [2019-12-17 12:38:18,137] jepsen worker 31 - jepsen.util 11191 :invoke :write [10 16] INFO [2019-12-17 12:38:18,145] jepsen worker 11 - jepsen.util 12881 :fail :cas [3 [26 20]] :ValueNotSwapFrom INFO [2019-12-17 12:38:18,146] jepsen worker 11 - jepsen.util 12881 :invoke :write [3 37] INFO [2019-12-17 12:38:18,148] jepsen worker 72 - jepsen.util 13032 :info :write [24 32] :RequestCanceledException INFO [2019-12-17 12:38:18,149] jepsen worker 72 - jepsen.util 13122 :invoke :read [24 nil] INFO [2019-12-17 12:38:18,155] jepsen worker 29 - jepsen.util 11279 :ok :read [9 4] INFO [2019-12-17 12:38:18,155] jepsen worker 29 - jepsen.util 11279 :invoke :read [9 nil] INFO [2019-12-17 12:38:18,159] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:38:18,159] jepsen nemesis - couchbase.util Rebalance check count: 592 INFO [2019-12-17 12:38:18,159] jepsen nemesis - couchbase.util Rebalance stuck count: 573 INFO [2019-12-17 12:38:18,166] jepsen worker 4 - jepsen.util 12064 :info :write [1 45] :RequestCanceledException INFO [2019-12-17 12:38:18,167] jepsen worker 4 - jepsen.util 12154 :invoke :read [1 nil] INFO [2019-12-17 12:38:18,173] jepsen worker 64 - jepsen.util 10144 :ok :write [21 6] INFO [2019-12-17 12:38:18,173] jepsen worker 64 - jepsen.util 10144 :invoke :write [21 36] INFO [2019-12-17 12:38:18,177] jepsen worker 74 - jepsen.util 12944 :ok :read [24 17] INFO [2019-12-17 12:38:18,177] jepsen worker 74 - jepsen.util 12944 :invoke :write [24 49] INFO [2019-12-17 12:38:18,189] jepsen worker 72 - jepsen.util 13122 :ok :read [24 17] INFO [2019-12-17 12:38:18,190] jepsen worker 72 - jepsen.util 13122 :invoke :cas [24 [17 4]] INFO [2019-12-17 12:38:18,194] jepsen worker 55 - jepsen.util 13735 :ok :read [18 43] INFO [2019-12-17 12:38:18,194] jepsen worker 55 - jepsen.util 13735 :invoke :read [18 nil] INFO [2019-12-17 12:38:18,208] jepsen worker 56 - jepsen.util 13736 :ok :read [18 43] INFO [2019-12-17 12:38:18,208] jepsen worker 56 - jepsen.util 13736 :invoke :read [18 nil] INFO [2019-12-17 12:38:18,226] jepsen worker 29 - jepsen.util 11279 :ok :read [9 4] INFO [2019-12-17 12:38:18,226] jepsen worker 29 - jepsen.util 11279 :invoke :cas [9 [42 34]] INFO [2019-12-17 12:38:18,228] jepsen worker 51 - jepsen.util 12291 :ok :read [17 49] INFO [2019-12-17 12:38:18,228] jepsen worker 51 - jepsen.util 12291 :invoke :read [17 nil] INFO [2019-12-17 12:38:18,246] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":75205,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:38:18,246] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":75205,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:38:18,246] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":75209,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:38:18,246] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":75209,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:38:18,246] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":75197,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:38:18,246] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":75197,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:38:18,246] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":75198,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:38:18,246] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":75198,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:38:18,247] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":75199,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:38:18,247] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":75199,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:38:18,248] jepsen worker 47 - jepsen.util 14177 :info :cas [15 [40 9]] :RequestCanceledException INFO [2019-12-17 12:38:18,249] jepsen worker 47 - jepsen.util 14267 :invoke :read [15 nil] INFO [2019-12-17 12:38:18,266] jepsen worker 38 - jepsen.util 13628 :info :write [12 16] :RequestCanceledException INFO [2019-12-17 12:38:18,267] jepsen worker 38 - jepsen.util 13718 :invoke :cas [12 [45 34]] INFO [2019-12-17 12:38:18,284] jepsen worker 64 - jepsen.util 10144 :ok :write [21 36] INFO [2019-12-17 12:38:18,284] jepsen worker 64 - jepsen.util 10144 :invoke :read [21 nil] INFO [2019-12-17 12:38:18,289] jepsen worker 56 - jepsen.util 13736 :ok :read [18 43] INFO [2019-12-17 12:38:18,289] jepsen worker 56 - jepsen.util 13736 :invoke :cas [18 [24 3]] INFO [2019-12-17 12:38:18,302] jepsen worker 54 - jepsen.util 13644 :fail :cas [18 [35 5]] :ValueNotSwapFrom INFO [2019-12-17 12:38:18,302] jepsen worker 54 - jepsen.util 13644 :invoke :write [18 37] INFO [2019-12-17 12:38:18,304] jepsen worker 47 - jepsen.util 14267 :ok :read [15 8] INFO [2019-12-17 12:38:18,304] jepsen worker 47 - jepsen.util 14267 :invoke :write [15 20] INFO [2019-12-17 12:38:18,319] jepsen worker 12 - jepsen.util 13152 :info :write [4 7] :RequestCanceledException INFO [2019-12-17 12:38:18,323] jepsen worker 64 - jepsen.util 10144 :ok :read [21 36] INFO [2019-12-17 12:38:18,323] jepsen worker 64 - jepsen.util 10144 :invoke :write [21 30] INFO [2019-12-17 12:38:18,330] jepsen worker 12 - jepsen.util 13242 :invoke :read [4 nil] INFO [2019-12-17 12:38:18,355] jepsen worker 18 - jepsen.util 13608 :ok :write [6 36] INFO [2019-12-17 12:38:18,355] jepsen worker 18 - jepsen.util 13608 :invoke :cas [6 [42 5]] INFO [2019-12-17 12:38:18,374] jepsen worker 18 - jepsen.util 13608 :fail :cas [6 [42 5]] :ValueNotSwapFrom INFO [2019-12-17 12:38:18,374] jepsen worker 18 - jepsen.util 13608 :invoke :read [6 nil] INFO [2019-12-17 12:38:18,386] jepsen worker 18 - jepsen.util 13608 :ok :read [6 36] INFO [2019-12-17 12:38:18,386] jepsen worker 18 - jepsen.util 13608 :invoke :read [6 nil] INFO [2019-12-17 12:38:18,393] jepsen worker 14 - jepsen.util 12614 :fail :cas [4 [43 38]] :ValueNotSwapFrom INFO [2019-12-17 12:38:18,393] jepsen worker 14 - jepsen.util 12614 :invoke :cas [4 [11 46]] INFO [2019-12-17 12:38:18,399] jepsen worker 18 - jepsen.util 13608 :ok :read [6 36] INFO [2019-12-17 12:38:18,399] jepsen worker 18 - jepsen.util 13608 :invoke :write [6 44] INFO [2019-12-17 12:38:18,412] jepsen worker 12 - jepsen.util 13242 :ok :read [4 10] INFO [2019-12-17 12:38:18,412] jepsen worker 12 - jepsen.util 13242 :invoke :read [4 nil] INFO [2019-12-17 12:38:18,434] jepsen worker 77 - jepsen.util 18887 :info :cas [25 [1 35]] :RequestCanceledException INFO [2019-12-17 12:38:18,435] jepsen worker 77 - jepsen.util 18977 :invoke :cas [25 [40 8]] INFO [2019-12-17 12:38:18,435] jepsen worker 13 - jepsen.util 12523 :info :write [4 18] :RequestCanceledException INFO [2019-12-17 12:38:18,436] jepsen worker 13 - jepsen.util 12613 :invoke :read [4 nil] INFO [2019-12-17 12:38:18,469] jepsen worker 4 - jepsen.util 12154 :ok :read [1 28] INFO [2019-12-17 12:38:18,480] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":75227,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:38:18,483] jepsen worker 71 - jepsen.util 13841 :ok :read [23 7] INFO [2019-12-17 12:38:18,495] jepsen worker 55 - jepsen.util 13735 :ok :read [18 43] INFO [2019-12-17 12:38:18,501] jepsen worker 45 - jepsen.util 12645 :fail :read [15 nil] :RequestTimeoutException INFO [2019-12-17 12:38:18,501] jepsen worker 2 - jepsen.util 17102 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:38:18,516] jepsen worker 89 - jepsen.util 12959 :fail :cas [29 [36 8]] :ValueNotSwapFrom INFO [2019-12-17 12:38:18,521] jepsen worker 79 - jepsen.util 13219 :ok :read [26 4] INFO [2019-12-17 12:38:18,535] jepsen worker 44 - jepsen.util 18494 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:38:18,537] jepsen worker 78 - jepsen.util 12768 :ok :read [26 4] INFO [2019-12-17 12:38:18,548] jepsen worker 15 - jepsen.util 16935 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:38:18,548] jepsen worker 35 - jepsen.util 17855 :info :write [11 42] :RequestCanceledException INFO [2019-12-17 12:38:18,563] jepsen worker 19 - jepsen.util 13699 :info :write [6 27] :RequestCanceledException INFO [2019-12-17 12:38:18,585] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":75227,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:38:18,585] jepsen worker 4 - jepsen.util 12154 :invoke :read [1 nil] INFO [2019-12-17 12:38:18,585] jepsen worker 71 - jepsen.util 13841 :invoke :cas [23 [11 29]] INFO [2019-12-17 12:38:18,585] jepsen worker 45 - jepsen.util 12645 :invoke :cas [15 [19 41]] INFO [2019-12-17 12:38:18,585] jepsen worker 2 - jepsen.util 17102 :invoke :read [0 nil] INFO [2019-12-17 12:38:18,585] jepsen worker 89 - jepsen.util 12959 :invoke :cas [29 [45 45]] INFO [2019-12-17 12:38:18,585] jepsen worker 79 - jepsen.util 13219 :invoke :read [26 nil] INFO [2019-12-17 12:38:18,585] jepsen worker 44 - jepsen.util 18494 :invoke :write [14 26] INFO [2019-12-17 12:38:18,585] jepsen worker 78 - jepsen.util 12768 :invoke :cas [26 [6 5]] INFO [2019-12-17 12:38:18,585] jepsen worker 15 - jepsen.util 16935 :invoke :cas [5 [46 47]] INFO [2019-12-17 12:38:18,585] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":75232,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:38:18,585] jepsen worker 55 - jepsen.util 13735 :invoke :read [18 nil] INFO [2019-12-17 12:38:18,586] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":75232,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:38:18,586] jepsen worker 19 - jepsen.util 13789 :invoke :read [6 nil] INFO [2019-12-17 12:38:18,586] jepsen worker 35 - jepsen.util 17945 :invoke :read [11 nil] INFO [2019-12-17 12:38:18,597] jepsen worker 52 - jepsen.util 12652 :ok :read [17 49] INFO [2019-12-17 12:38:18,597] jepsen worker 52 - jepsen.util 12652 :invoke :write [17 40] INFO [2019-12-17 12:38:18,601] jepsen worker 58 - jepsen.util 18058 :info :write [19 28] :RequestCanceledException INFO [2019-12-17 12:38:18,603] jepsen worker 58 - jepsen.util 18148 :invoke :read [19 nil] INFO [2019-12-17 12:38:18,619] jepsen worker 16 - jepsen.util 19276 :info :write [5 7] :RequestCanceledException INFO [2019-12-17 12:38:18,620] jepsen worker 16 - jepsen.util 19366 :invoke :cas [5 [4 10]] INFO [2019-12-17 12:38:18,629] jepsen worker 55 - jepsen.util 13735 :ok :read [18 43] INFO [2019-12-17 12:38:18,629] jepsen worker 55 - jepsen.util 13735 :invoke :write [18 12] INFO [2019-12-17 12:38:18,634] jepsen worker 73 - jepsen.util 12043 :info :write [24 33] :RequestCanceledException INFO [2019-12-17 12:38:18,634] jepsen worker 81 - jepsen.util 17181 :info :write [27 38] :RequestCanceledException INFO [2019-12-17 12:38:18,634] jepsen worker 43 - jepsen.util 18043 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:38:18,634] jepsen worker 48 - jepsen.util 16518 :info :cas [16 [37 8]] :RequestCanceledException INFO [2019-12-17 12:38:18,634] jepsen worker 10 - jepsen.util 12880 :info :cas [3 [5 49]] :RequestCanceledException INFO [2019-12-17 12:38:18,635] jepsen worker 43 - jepsen.util 18043 :invoke :write [14 35] INFO [2019-12-17 12:38:18,636] jepsen worker 73 - jepsen.util 12133 :invoke :write [24 46] INFO [2019-12-17 12:38:18,636] jepsen worker 81 - jepsen.util 17271 :invoke :cas [27 [6 13]] INFO [2019-12-17 12:38:18,636] jepsen worker 10 - jepsen.util 12970 :invoke :cas [3 [18 35]] INFO [2019-12-17 12:38:18,636] jepsen worker 48 - jepsen.util 16608 :invoke :read [16 nil] INFO [2019-12-17 12:38:18,665] jepsen worker 50 - jepsen.util 17600 :info :cas [16 [3 39]] :RequestCanceledException INFO [2019-12-17 12:38:18,665] jepsen worker 65 - jepsen.util 11315 :info :cas [21 [23 41]] :RequestCanceledException INFO [2019-12-17 12:38:18,665] jepsen worker 9 - jepsen.util 13419 :fail :read [3 nil] :RequestTimeoutException INFO [2019-12-17 12:38:18,665] jepsen worker 63 - jepsen.util 11493 :info :cas [21 [38 31]] :RequestCanceledException INFO [2019-12-17 12:38:18,665] jepsen worker 25 - jepsen.util 18835 :info :write [8 45] :RequestCanceledException INFO [2019-12-17 12:38:18,665] jepsen worker 9 - jepsen.util 13419 :invoke :cas [3 [14 2]] INFO [2019-12-17 12:38:18,666] jepsen worker 65 - jepsen.util 11405 :invoke :write [21 22] INFO [2019-12-17 12:38:18,666] jepsen worker 25 - jepsen.util 18925 :invoke :write [8 12] INFO [2019-12-17 12:38:18,667] jepsen worker 50 - jepsen.util 17690 :invoke :cas [16 [25 24]] INFO [2019-12-17 12:38:18,673] jepsen worker 63 - jepsen.util 11583 :invoke :cas [21 [30 14]] INFO [2019-12-17 12:38:18,679] jepsen worker 9 - jepsen.util 13419 :fail :cas [3 [14 2]] :ValueNotSwapFrom INFO [2019-12-17 12:38:18,679] jepsen worker 9 - jepsen.util 13419 :invoke :write [3 36] INFO [2019-12-17 12:38:18,688] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":75256,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:38:18,688] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":75256,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:38:18,688] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":75260,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:38:18,688] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":75260,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:38:18,701] jepsen worker 57 - jepsen.util 17427 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:38:18,701] jepsen worker 57 - jepsen.util 17427 :invoke :read [19 nil] INFO [2019-12-17 12:38:18,707] jepsen worker 51 - jepsen.util 12291 :ok :read [17 49] INFO [2019-12-17 12:38:18,708] jepsen worker 51 - jepsen.util 12291 :invoke :read [17 nil] INFO [2019-12-17 12:38:18,712] jepsen worker 31 - jepsen.util 11191 :ok :write [10 16] INFO [2019-12-17 12:38:18,713] jepsen worker 31 - jepsen.util 11191 :invoke :write [10 4] INFO [2019-12-17 12:38:18,718] jepsen worker 74 - jepsen.util 12944 :ok :write [24 49] INFO [2019-12-17 12:38:18,718] jepsen worker 74 - jepsen.util 12944 :invoke :read [24 nil] INFO [2019-12-17 12:38:18,725] jepsen worker 47 - jepsen.util 14267 :ok :write [15 20] INFO [2019-12-17 12:38:18,725] jepsen worker 47 - jepsen.util 14267 :invoke :cas [15 [38 13]] INFO [2019-12-17 12:38:18,731] jepsen worker 18 - jepsen.util 13608 :ok :write [6 44] INFO [2019-12-17 12:38:18,731] jepsen worker 18 - jepsen.util 13608 :invoke :cas [6 [31 24]] INFO [2019-12-17 12:38:18,732] jepsen worker 11 - jepsen.util 12881 :ok :write [3 37] INFO [2019-12-17 12:38:18,732] jepsen worker 11 - jepsen.util 12881 :invoke :cas [3 [32 37]] INFO [2019-12-17 12:38:18,734] jepsen worker 53 - jepsen.util 12473 :info :write [17 26] :RequestCanceledException INFO [2019-12-17 12:38:18,734] jepsen worker 8 - jepsen.util 17288 :info :write [2 12] :RequestCanceledException INFO [2019-12-17 12:38:18,735] jepsen worker 8 - jepsen.util 17378 :invoke :read [2 nil] INFO [2019-12-17 12:38:18,736] jepsen worker 53 - jepsen.util 12563 :invoke :read [17 nil] INFO [2019-12-17 12:38:18,741] jepsen worker 19 - jepsen.util 13789 :ok :read [6 36] INFO [2019-12-17 12:38:18,741] jepsen worker 19 - jepsen.util 13789 :invoke :cas [6 [14 37]] INFO [2019-12-17 12:38:18,747] jepsen worker 61 - jepsen.util 18871 :info :write [20 30] :RequestCanceledException INFO [2019-12-17 12:38:18,749] jepsen worker 61 - jepsen.util 18961 :invoke :cas [20 [1 34]] INFO [2019-12-17 12:38:18,765] jepsen worker 7 - jepsen.util 17467 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:38:18,765] jepsen worker 7 - jepsen.util 17467 :invoke :write [2 46] INFO [2019-12-17 12:38:18,771] jepsen worker 45 - jepsen.util 12645 :fail :cas [15 [19 41]] :ValueNotSwapFrom INFO [2019-12-17 12:38:18,771] jepsen worker 45 - jepsen.util 12645 :invoke :write [15 27] INFO [2019-12-17 12:38:18,789] jepsen worker 63 - jepsen.util 11583 :fail :cas [21 [30 14]] :ValueNotSwapFrom INFO [2019-12-17 12:38:18,789] jepsen worker 63 - jepsen.util 11583 :invoke :read [21 nil] INFO [2019-12-17 12:38:18,796] jepsen worker 38 - jepsen.util 13718 :fail :cas [12 [45 34]] :ValueNotSwapFrom INFO [2019-12-17 12:38:18,796] jepsen worker 38 - jepsen.util 13718 :invoke :read [12 nil] INFO [2019-12-17 12:38:18,804] jepsen worker 74 - jepsen.util 12944 :ok :read [24 49] INFO [2019-12-17 12:38:18,804] jepsen worker 74 - jepsen.util 12944 :invoke :read [24 nil] INFO [2019-12-17 12:38:18,824] jepsen worker 56 - jepsen.util 13736 :fail :cas [18 [24 3]] :ValueNotSwapFrom INFO [2019-12-17 12:38:18,824] jepsen worker 56 - jepsen.util 13736 :invoke :cas [18 [24 48]] INFO [2019-12-17 12:38:18,825] jepsen worker 59 - jepsen.util 16889 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:38:18,825] jepsen worker 59 - jepsen.util 16889 :invoke :write [19 37] INFO [2019-12-17 12:38:18,826] jepsen worker 87 - jepsen.util 12507 :info :cas [29 [1 46]] :RequestCanceledException INFO [2019-12-17 12:38:18,827] jepsen worker 87 - jepsen.util 12597 :invoke :cas [29 [4 43]] INFO [2019-12-17 12:38:18,829] jepsen worker 14 - jepsen.util 12614 :fail :cas [4 [11 46]] :ValueNotSwapFrom INFO [2019-12-17 12:38:18,829] jepsen worker 14 - jepsen.util 12614 :invoke :write [4 11] INFO [2019-12-17 12:38:18,837] jepsen worker 64 - jepsen.util 10144 :ok :write [21 30] INFO [2019-12-17 12:38:18,837] jepsen worker 64 - jepsen.util 10144 :invoke :cas [21 [31 3]] INFO [2019-12-17 12:38:18,839] jepsen worker 69 - jepsen.util 13389 :ok :write [23 15] INFO [2019-12-17 12:38:18,839] jepsen worker 69 - jepsen.util 13389 :invoke :write [23 13] INFO [2019-12-17 12:38:18,848] jepsen worker 83 - jepsen.util 17723 :info :cas [27 [13 35]] :RequestCanceledException INFO [2019-12-17 12:38:18,849] jepsen worker 83 - jepsen.util 17813 :invoke :read [27 nil] INFO [2019-12-17 12:38:18,853] jepsen worker 12 - jepsen.util 13242 :ok :read [4 10] INFO [2019-12-17 12:38:18,854] jepsen worker 12 - jepsen.util 13242 :invoke :read [4 nil] INFO [2019-12-17 12:38:18,856] jepsen worker 64 - jepsen.util 10144 :fail :cas [21 [31 3]] :ValueNotSwapFrom INFO [2019-12-17 12:38:18,857] jepsen worker 64 - jepsen.util 10144 :invoke :write [21 33] INFO [2019-12-17 12:38:18,890] jepsen worker 71 - jepsen.util 13841 :fail :cas [23 [11 29]] :ValueNotSwapFrom INFO [2019-12-17 12:38:18,890] jepsen worker 71 - jepsen.util 13841 :invoke :write [23 36] INFO [2019-12-17 12:38:18,892] jepsen worker 65 - jepsen.util 11405 :ok :write [21 22] INFO [2019-12-17 12:38:18,892] jepsen worker 65 - jepsen.util 11405 :invoke :read [21 nil] INFO [2019-12-17 12:38:18,902] jepsen worker 32 - jepsen.util 13352 :info :write [10 35] :RequestCanceledException INFO [2019-12-17 12:38:18,903] jepsen worker 32 - jepsen.util 13442 :invoke :cas [10 [17 30]] INFO [2019-12-17 12:38:18,905] jepsen worker 65 - jepsen.util 11405 :ok :read [21 22] INFO [2019-12-17 12:38:18,905] jepsen worker 65 - jepsen.util 11405 :invoke :cas [21 [44 0]] INFO [2019-12-17 12:38:18,908] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":75295,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:38:18,908] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":75295,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:38:18,908] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":75297,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:38:18,909] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":75297,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:38:18,909] jepsen worker 79 - jepsen.util 13219 :ok :read [26 4] INFO [2019-12-17 12:38:18,909] jepsen worker 79 - jepsen.util 13219 :invoke :read [26 nil] INFO [2019-12-17 12:38:18,919] jepsen worker 46 - jepsen.util 12556 :info :cas [15 [27 11]] :RequestCanceledException INFO [2019-12-17 12:38:18,920] jepsen worker 46 - jepsen.util 12646 :invoke :cas [15 [29 2]] INFO [2019-12-17 12:38:18,922] jepsen worker 38 - jepsen.util 13718 :ok :read [12 48] INFO [2019-12-17 12:38:18,922] jepsen worker 38 - jepsen.util 13718 :invoke :read [12 nil] INFO [2019-12-17 12:38:18,935] jepsen worker 42 - jepsen.util 18582 :info :cas [14 [35 32]] :RequestCanceledException INFO [2019-12-17 12:38:18,936] jepsen worker 42 - jepsen.util 18672 :invoke :cas [14 [27 1]] INFO [2019-12-17 12:38:18,938] jepsen worker 65 - jepsen.util 11405 :fail :cas [21 [44 0]] :ValueNotSwapFrom INFO [2019-12-17 12:38:18,938] jepsen worker 65 - jepsen.util 11405 :invoke :read [21 nil] INFO [2019-12-17 12:38:18,940] jepsen worker 56 - jepsen.util 13736 :fail :cas [18 [24 48]] :ValueNotSwapFrom INFO [2019-12-17 12:38:18,941] jepsen worker 56 - jepsen.util 13736 :invoke :write [18 48] INFO [2019-12-17 12:38:18,943] jepsen worker 78 - jepsen.util 12768 :fail :cas [26 [6 5]] :ValueNotSwapFrom INFO [2019-12-17 12:38:18,943] jepsen worker 78 - jepsen.util 12768 :invoke :read [26 nil] INFO [2019-12-17 12:38:18,954] jepsen worker 73 - jepsen.util 12133 :ok :write [24 46] INFO [2019-12-17 12:38:18,954] jepsen worker 73 - jepsen.util 12133 :invoke :read [24 nil] INFO [2019-12-17 12:38:18,959] jepsen worker 13 - jepsen.util 12613 :ok :read [4 10] INFO [2019-12-17 12:38:18,959] jepsen worker 13 - jepsen.util 12613 :invoke :cas [4 [26 13]] INFO [2019-12-17 12:38:18,965] jepsen worker 39 - jepsen.util 17319 :info :cas [13 [0 41]] :RequestCanceledException INFO [2019-12-17 12:38:18,966] jepsen worker 39 - jepsen.util 17409 :invoke :write [13 9] INFO [2019-12-17 12:38:18,970] jepsen worker 10 - jepsen.util 12970 :fail :cas [3 [18 35]] :ValueNotSwapFrom INFO [2019-12-17 12:38:18,970] jepsen worker 10 - jepsen.util 12970 :invoke :write [3 21] INFO [2019-12-17 12:38:18,982] jepsen worker 53 - jepsen.util 12563 :ok :read [17 49] INFO [2019-12-17 12:38:18,982] jepsen worker 53 - jepsen.util 12563 :invoke :read [17 nil] INFO [2019-12-17 12:38:18,995] jepsen worker 65 - jepsen.util 11405 :ok :read [21 22] INFO [2019-12-17 12:38:18,996] jepsen worker 65 - jepsen.util 11405 :invoke :write [21 17] INFO [2019-12-17 12:38:19,006] jepsen worker 73 - jepsen.util 12133 :ok :read [24 46] INFO [2019-12-17 12:38:19,006] jepsen worker 73 - jepsen.util 12133 :invoke :write [24 48] INFO [2019-12-17 12:38:19,008] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":75316,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:38:19,008] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":75316,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:38:19,008] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":75319,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:38:19,008] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":75319,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:38:19,008] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":75320,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:38:19,008] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":75320,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:38:19,011] jepsen worker 4 - jepsen.util 12154 :ok :read [1 28] INFO [2019-12-17 12:38:19,012] jepsen worker 4 - jepsen.util 12154 :invoke :write [1 4] INFO [2019-12-17 12:38:19,013] jepsen worker 38 - jepsen.util 13718 :ok :read [12 48] INFO [2019-12-17 12:38:19,013] jepsen worker 38 - jepsen.util 13718 :invoke :cas [12 [12 19]] INFO [2019-12-17 12:38:19,025] jepsen worker 64 - jepsen.util 10144 :ok :write [21 33] INFO [2019-12-17 12:38:19,026] jepsen worker 64 - jepsen.util 10144 :invoke :write [21 42] INFO [2019-12-17 12:38:19,032] jepsen worker 89 - jepsen.util 12959 :fail :cas [29 [45 45]] :ValueNotSwapFrom INFO [2019-12-17 12:38:19,033] jepsen worker 89 - jepsen.util 12959 :invoke :cas [29 [38 29]] INFO [2019-12-17 12:38:19,034] jepsen worker 1 - jepsen.util 18091 :info :write [0 18] :RequestCanceledException INFO [2019-12-17 12:38:19,035] jepsen worker 1 - jepsen.util 18181 :invoke :cas [0 [5 12]] INFO [2019-12-17 12:38:19,048] jepsen worker 37 - jepsen.util 12547 :info :write [12 8] :RequestCanceledException INFO [2019-12-17 12:38:19,049] jepsen worker 37 - jepsen.util 12637 :invoke :write [12 5] INFO [2019-12-17 12:38:19,082] jepsen worker 36 - jepsen.util 13806 :ok :write [12 17] INFO [2019-12-17 12:38:19,082] jepsen worker 36 - jepsen.util 13806 :invoke :cas [12 [20 26]] INFO [2019-12-17 12:38:19,088] jepsen worker 65 - jepsen.util 11405 :ok :write [21 17] INFO [2019-12-17 12:38:19,088] jepsen worker 65 - jepsen.util 11405 :invoke :read [21 nil] INFO [2019-12-17 12:38:19,101] jepsen worker 22 - jepsen.util 18112 :info :cas [7 [12 47]] :RequestCanceledException INFO [2019-12-17 12:38:19,103] jepsen worker 22 - jepsen.util 18202 :invoke :write [7 9] INFO [2019-12-17 12:38:19,109] jepsen worker 45 - jepsen.util 12645 :ok :write [15 27] INFO [2019-12-17 12:38:19,109] jepsen worker 45 - jepsen.util 12645 :invoke :write [15 15] INFO [2019-12-17 12:38:19,109] jepsen worker 65 - jepsen.util 11405 :ok :read [21 17] INFO [2019-12-17 12:38:19,110] jepsen worker 65 - jepsen.util 11405 :invoke :read [21 nil] INFO [2019-12-17 12:38:19,117] jepsen worker 68 - jepsen.util 16988 :info :write [22 25] :RequestCanceledException INFO [2019-12-17 12:38:19,117] jepsen worker 6 - jepsen.util 18096 :info :cas [2 [42 49]] :RequestCanceledException INFO [2019-12-17 12:38:19,118] jepsen worker 6 - jepsen.util 18186 :invoke :read [2 nil] INFO [2019-12-17 12:38:19,119] jepsen worker 68 - jepsen.util 17078 :invoke :read [22 nil] INFO [2019-12-17 12:38:19,132] jepsen worker 56 - jepsen.util 13736 :ok :write [18 48] INFO [2019-12-17 12:38:19,132] jepsen worker 56 - jepsen.util 13736 :invoke :cas [18 [38 6]] INFO [2019-12-17 12:38:19,135] jepsen worker 80 - jepsen.util 12410 :info :write [26 30] :RequestCanceledException INFO [2019-12-17 12:38:19,136] jepsen worker 80 - jepsen.util 12500 :invoke :write [26 25] INFO [2019-12-17 12:38:19,148] jepsen worker 76 - jepsen.util 17896 :info :write [25 13] :RequestCanceledException INFO [2019-12-17 12:38:19,150] jepsen worker 76 - jepsen.util 17986 :invoke :cas [25 [16 27]] INFO [2019-12-17 12:38:19,151] jepsen worker 32 - jepsen.util 13442 :fail :cas [10 [17 30]] :ValueNotSwapFrom INFO [2019-12-17 12:38:19,151] jepsen worker 32 - jepsen.util 13442 :invoke :read [10 nil] INFO [2019-12-17 12:38:19,153] jepsen worker 51 - jepsen.util 12291 :ok :read [17 49] INFO [2019-12-17 12:38:19,154] jepsen worker 51 - jepsen.util 12291 :invoke :write [17 7] INFO [2019-12-17 12:38:19,159] jepsen worker 38 - jepsen.util 13718 :fail :cas [12 [12 19]] :ValueNotSwapFrom INFO [2019-12-17 12:38:19,160] jepsen worker 38 - jepsen.util 13718 :invoke :cas [12 [20 46]] INFO [2019-12-17 12:38:19,160] jepsen worker 52 - jepsen.util 12652 :ok :write [17 40] INFO [2019-12-17 12:38:19,161] jepsen worker 52 - jepsen.util 12652 :invoke :read [17 nil] INFO [2019-12-17 12:38:19,161] jepsen worker 65 - jepsen.util 11405 :ok :read [21 17] INFO [2019-12-17 12:38:19,161] jepsen worker 65 - jepsen.util 11405 :invoke :read [21 nil] INFO [2019-12-17 12:38:19,192] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:38:19,192] jepsen nemesis - couchbase.util Rebalance check count: 593 INFO [2019-12-17 12:38:19,192] jepsen nemesis - couchbase.util Rebalance stuck count: 574 INFO [2019-12-17 12:38:19,196] jepsen worker 65 - jepsen.util 11405 :ok :read [21 17] INFO [2019-12-17 12:38:19,196] jepsen worker 65 - jepsen.util 11405 :invoke :read [21 nil] INFO [2019-12-17 12:38:19,214] jepsen worker 65 - jepsen.util 11405 :ok :read [21 17] INFO [2019-12-17 12:38:19,214] jepsen worker 65 - jepsen.util 11405 :invoke :cas [21 [41 37]] INFO [2019-12-17 12:38:19,226] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":75352,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:38:19,227] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":75352,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:38:19,227] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":75355,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:38:19,227] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":75355,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:38:19,227] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":75344,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:38:19,227] jepsen worker 65 - jepsen.util 11405 :fail :cas [21 [41 37]] :ValueNotSwapFrom INFO [2019-12-17 12:38:19,227] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":75356,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:38:19,227] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":75344,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:38:19,227] jepsen worker 65 - jepsen.util 11405 :invoke :read [21 nil] INFO [2019-12-17 12:38:19,227] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":75356,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:38:19,227] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":75345,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:38:19,227] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":75345,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:38:19,234] jepsen worker 34 - jepsen.util 18754 :info :write [11 9] :RequestCanceledException INFO [2019-12-17 12:38:19,234] jepsen worker 56 - jepsen.util 13736 :fail :cas [18 [38 6]] :ValueNotSwapFrom INFO [2019-12-17 12:38:19,234] jepsen worker 56 - jepsen.util 13736 :invoke :write [18 8] INFO [2019-12-17 12:38:19,235] jepsen worker 34 - jepsen.util 18844 :invoke :cas [11 [39 9]] INFO [2019-12-17 12:38:19,242] jepsen worker 65 - jepsen.util 11405 :ok :read [21 17] INFO [2019-12-17 12:38:19,242] jepsen worker 65 - jepsen.util 11405 :invoke :cas [21 [47 18]] INFO [2019-12-17 12:38:19,259] jepsen worker 31 - jepsen.util 11191 :ok :write [10 4] INFO [2019-12-17 12:38:19,259] jepsen worker 31 - jepsen.util 11191 :invoke :read [10 nil] INFO [2019-12-17 12:38:19,265] jepsen worker 65 - jepsen.util 11405 :fail :cas [21 [47 18]] :ValueNotSwapFrom INFO [2019-12-17 12:38:19,265] jepsen worker 65 - jepsen.util 11405 :invoke :read [21 nil] INFO [2019-12-17 12:38:19,300] jepsen worker 49 - jepsen.util 17779 :info :write [16 17] :RequestCanceledException INFO [2019-12-17 12:38:19,301] jepsen worker 49 - jepsen.util 17869 :invoke :read [16 nil] INFO [2019-12-17 12:38:19,307] jepsen worker 52 - jepsen.util 12652 :ok :read [17 40] INFO [2019-12-17 12:38:19,307] jepsen worker 52 - jepsen.util 12652 :invoke :write [17 2] INFO [2019-12-17 12:38:19,308] jepsen worker 46 - jepsen.util 12646 :fail :cas [15 [29 2]] :ValueNotSwapFrom INFO [2019-12-17 12:38:19,308] jepsen worker 46 - jepsen.util 12646 :invoke :cas [15 [27 35]] INFO [2019-12-17 12:38:19,319] jepsen worker 40 - jepsen.util 18490 :info :write [13 20] :RequestCanceledException INFO [2019-12-17 12:38:19,320] jepsen worker 65 - jepsen.util 11405 :ok :read [21 17] INFO [2019-12-17 12:38:19,320] jepsen worker 65 - jepsen.util 11405 :invoke :write [21 31] INFO [2019-12-17 12:38:19,320] jepsen worker 40 - jepsen.util 18580 :invoke :cas [13 [24 31]] INFO [2019-12-17 12:38:19,335] jepsen worker 41 - jepsen.util 19121 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:38:19,335] jepsen worker 41 - jepsen.util 19121 :invoke :cas [13 [36 6]] INFO [2019-12-17 12:38:19,345] jepsen worker 73 - jepsen.util 12133 :ok :write [24 48] INFO [2019-12-17 12:38:19,345] jepsen worker 73 - jepsen.util 12133 :invoke :cas [24 [2 8]] INFO [2019-12-17 12:38:19,349] jepsen worker 24 - jepsen.util 16854 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:38:19,349] jepsen worker 24 - jepsen.util 16854 :invoke :write [8 24] INFO [2019-12-17 12:38:19,351] jepsen worker 87 - jepsen.util 12597 :fail :cas [29 [4 43]] :ValueNotSwapFrom INFO [2019-12-17 12:38:19,351] jepsen worker 87 - jepsen.util 12597 :invoke :read [29 nil] INFO [2019-12-17 12:38:19,372] jepsen worker 73 - jepsen.util 12133 :fail :cas [24 [2 8]] :ValueNotSwapFrom INFO [2019-12-17 12:38:19,372] jepsen worker 73 - jepsen.util 12133 :invoke :write [24 37] INFO [2019-12-17 12:38:19,373] jepsen worker 12 - jepsen.util 13242 :ok :read [4 10] INFO [2019-12-17 12:38:19,373] jepsen worker 12 - jepsen.util 13242 :invoke :write [4 34] INFO [2019-12-17 12:38:19,394] jepsen worker 65 - jepsen.util 11405 :ok :write [21 31] INFO [2019-12-17 12:38:19,395] jepsen worker 65 - jepsen.util 11405 :invoke :read [21 nil] INFO [2019-12-17 12:38:19,413] jepsen worker 65 - jepsen.util 11405 :ok :read [21 31] INFO [2019-12-17 12:38:19,413] jepsen worker 65 - jepsen.util 11405 :invoke :read [21 nil] INFO [2019-12-17 12:38:19,414] jepsen worker 38 - jepsen.util 13718 :fail :cas [12 [20 46]] :ValueNotSwapFrom INFO [2019-12-17 12:38:19,414] jepsen worker 38 - jepsen.util 13718 :invoke :write [12 37] INFO [2019-12-17 12:38:19,434] jepsen worker 65 - jepsen.util 11405 :ok :read [21 31] INFO [2019-12-17 12:38:19,434] jepsen worker 65 - jepsen.util 11405 :invoke :read [21 nil] INFO [2019-12-17 12:38:19,448] jepsen worker 89 - jepsen.util 12959 :fail :cas [29 [38 29]] :ValueNotSwapFrom INFO [2019-12-17 12:38:19,448] jepsen worker 89 - jepsen.util 12959 :invoke :cas [29 [40 6]] INFO [2019-12-17 12:38:19,450] jepsen worker 65 - jepsen.util 11405 :ok :read [21 31] INFO [2019-12-17 12:38:19,450] jepsen worker 65 - jepsen.util 11405 :invoke :write [21 1] INFO [2019-12-17 12:38:19,455] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":75379,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:38:19,455] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":75379,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:38:19,455] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":75382,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:38:19,455] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":75382,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:38:19,466] jepsen worker 86 - jepsen.util 18536 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:38:19,466] jepsen worker 86 - jepsen.util 18536 :invoke :cas [28 [39 40]] INFO [2019-12-17 12:38:19,477] jepsen worker 3 - jepsen.util 12603 :ok :write [1 39] INFO [2019-12-17 12:38:19,477] jepsen worker 3 - jepsen.util 12603 :invoke :write [1 37] INFO [2019-12-17 12:38:19,506] jepsen worker 65 - jepsen.util 11405 :ok :write [21 1] INFO [2019-12-17 12:38:19,507] jepsen worker 65 - jepsen.util 11405 :invoke :cas [21 [31 44]] INFO [2019-12-17 12:38:19,527] jepsen worker 89 - jepsen.util 12959 :fail :cas [29 [40 6]] :ValueNotSwapFrom INFO [2019-12-17 12:38:19,527] jepsen worker 89 - jepsen.util 12959 :invoke :cas [29 [21 37]] INFO [2019-12-17 12:38:19,532] jepsen worker 65 - jepsen.util 11405 :fail :cas [21 [31 44]] :ValueNotSwapFrom INFO [2019-12-17 12:38:19,533] jepsen worker 65 - jepsen.util 11405 :invoke :cas [21 [46 9]] INFO [2019-12-17 12:38:19,535] jepsen worker 28 - jepsen.util 11008 :info :write [9 13] :RequestCanceledException INFO [2019-12-17 12:38:19,535] jepsen worker 62 - jepsen.util 18422 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:38:19,535] jepsen worker 62 - jepsen.util 18422 :invoke :cas [20 [23 13]] INFO [2019-12-17 12:38:19,536] jepsen worker 28 - jepsen.util 11098 :invoke :write [9 23] INFO [2019-12-17 12:38:19,546] jepsen worker 65 - jepsen.util 11405 :fail :cas [21 [46 9]] :ValueNotSwapFrom INFO [2019-12-17 12:38:19,546] jepsen worker 65 - jepsen.util 11405 :invoke :write [21 27] INFO [2019-12-17 12:38:19,552] jepsen worker 4 - jepsen.util 12154 :ok :write [1 4] INFO [2019-12-17 12:38:19,552] jepsen worker 4 - jepsen.util 12154 :invoke :read [1 nil] INFO [2019-12-17 12:38:19,554] jepsen worker 37 - jepsen.util 12637 :ok :write [12 5] INFO [2019-12-17 12:38:19,555] jepsen worker 37 - jepsen.util 12637 :invoke :cas [12 [4 33]] INFO [2019-12-17 12:38:19,565] jepsen worker 70 - jepsen.util 13480 :info :write [23 34] :RequestCanceledException INFO [2019-12-17 12:38:19,567] jepsen worker 70 - jepsen.util 13570 :invoke :read [23 nil] INFO [2019-12-17 12:38:19,577] jepsen worker 36 - jepsen.util 13806 :fail :cas [12 [20 26]] :ValueNotSwapFrom INFO [2019-12-17 12:38:19,578] jepsen worker 36 - jepsen.util 13806 :invoke :cas [12 [4 48]] INFO [2019-12-17 12:38:19,601] jepsen worker 27 - jepsen.util 12087 :fail :read [9 nil] :RequestTimeoutException INFO [2019-12-17 12:38:19,601] jepsen worker 27 - jepsen.util 12087 :invoke :write [9 44] INFO [2019-12-17 12:38:19,626] jepsen worker 45 - jepsen.util 12645 :ok :write [15 15] INFO [2019-12-17 12:38:19,626] jepsen worker 45 - jepsen.util 12645 :invoke :cas [15 [13 12]] INFO [2019-12-17 12:38:19,634] jepsen worker 75 - jepsen.util 17445 :info :cas [25 [42 25]] :RequestCanceledException INFO [2019-12-17 12:38:19,634] jepsen worker 26 - jepsen.util 18656 :info :write [8 15] :RequestCanceledException INFO [2019-12-17 12:38:19,634] jepsen worker 17 - jepsen.util 17837 :info :cas [5 [20 12]] :RequestCanceledException INFO [2019-12-17 12:38:19,635] jepsen worker 75 - jepsen.util 17535 :invoke :cas [25 [36 24]] INFO [2019-12-17 12:38:19,635] jepsen worker 26 - jepsen.util 18746 :invoke :write [8 32] INFO [2019-12-17 12:38:19,635] jepsen worker 17 - jepsen.util 17927 :invoke :cas [5 [10 6]] INFO [2019-12-17 12:38:19,648] jepsen worker 60 - jepsen.util 18780 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:38:19,648] jepsen worker 60 - jepsen.util 18780 :invoke :write [20 6] INFO [2019-12-17 12:38:19,661] jepsen worker 28 - jepsen.util 11098 :ok :write [9 23] INFO [2019-12-17 12:38:19,661] jepsen worker 28 - jepsen.util 11098 :invoke :read [9 nil] INFO [2019-12-17 12:38:19,661] jepsen worker 10 - jepsen.util 12970 :ok :write [3 21] INFO [2019-12-17 12:38:19,661] jepsen worker 10 - jepsen.util 12970 :invoke :read [3 nil] INFO [2019-12-17 12:38:19,667] jepsen worker 14 - jepsen.util 12614 :ok :write [4 11] INFO [2019-12-17 12:38:19,668] jepsen worker 14 - jepsen.util 12614 :invoke :read [4 nil] INFO [2019-12-17 12:38:19,668] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":75406,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:38:19,668] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":75406,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:38:19,668] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":75408,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:38:19,668] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":75408,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:38:19,672] jepsen worker 5 - jepsen.util 12695 :ok :write [1 35] INFO [2019-12-17 12:38:19,672] jepsen worker 5 - jepsen.util 12695 :invoke :cas [1 [29 27]] INFO [2019-12-17 12:38:19,677] jepsen worker 69 - jepsen.util 13389 :ok :write [23 13] INFO [2019-12-17 12:38:19,677] jepsen worker 69 - jepsen.util 13389 :invoke :read [23 nil] INFO [2019-12-17 12:38:19,680] jepsen worker 80 - jepsen.util 12500 :ok :write [26 25] INFO [2019-12-17 12:38:19,680] jepsen worker 80 - jepsen.util 12500 :invoke :write [26 10] INFO [2019-12-17 12:38:19,702] jepsen worker 28 - jepsen.util 11098 :ok :read [9 23] INFO [2019-12-17 12:38:19,702] jepsen worker 28 - jepsen.util 11098 :invoke :read [9 nil] INFO [2019-12-17 12:38:19,703] jepsen worker 73 - jepsen.util 12133 :ok :write [24 37] INFO [2019-12-17 12:38:19,704] jepsen worker 73 - jepsen.util 12133 :invoke :write [24 41] INFO [2019-12-17 12:38:19,712] jepsen worker 79 - jepsen.util 13219 :ok :read [26 25] INFO [2019-12-17 12:38:19,712] jepsen worker 79 - jepsen.util 13219 :invoke :cas [26 [22 1]] INFO [2019-12-17 12:38:19,718] jepsen worker 37 - jepsen.util 12637 :fail :cas [12 [4 33]] :ValueNotSwapFrom INFO [2019-12-17 12:38:19,718] jepsen worker 37 - jepsen.util 12637 :invoke :cas [12 [33 48]] INFO [2019-12-17 12:38:19,722] jepsen worker 78 - jepsen.util 12768 :ok :read [26 25] INFO [2019-12-17 12:38:19,722] jepsen worker 78 - jepsen.util 12768 :invoke :write [26 36] INFO [2019-12-17 12:38:19,724] jepsen worker 65 - jepsen.util 11405 :ok :write [21 27] INFO [2019-12-17 12:38:19,724] jepsen worker 65 - jepsen.util 11405 :invoke :read [21 nil] INFO [2019-12-17 12:38:19,734] jepsen worker 88 - jepsen.util 13048 :ok :write [29 11] INFO [2019-12-17 12:38:19,734] jepsen worker 88 - jepsen.util 13048 :invoke :cas [29 [20 36]] INFO [2019-12-17 12:38:19,735] jepsen worker 20 - jepsen.util 12800 :fail :read [6 nil] :RequestTimeoutException INFO [2019-12-17 12:38:19,735] jepsen worker 20 - jepsen.util 12800 :invoke :write [6 6] INFO [2019-12-17 12:38:19,737] jepsen worker 36 - jepsen.util 13806 :fail :cas [12 [4 48]] :ValueNotSwapFrom INFO [2019-12-17 12:38:19,737] jepsen worker 36 - jepsen.util 13806 :invoke :write [12 21] INFO [2019-12-17 12:38:19,748] jepsen worker 33 - jepsen.util 17763 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:38:19,749] jepsen worker 33 - jepsen.util 17763 :invoke :read [11 nil] INFO [2019-12-17 12:38:19,749] jepsen worker 53 - jepsen.util 12563 :ok :read [17 40] INFO [2019-12-17 12:38:19,750] jepsen worker 53 - jepsen.util 12563 :invoke :cas [17 [31 22]] INFO [2019-12-17 12:38:19,754] jepsen worker 28 - jepsen.util 11098 :ok :read [9 23] INFO [2019-12-17 12:38:19,754] jepsen worker 28 - jepsen.util 11098 :invoke :cas [9 [20 23]] INFO [2019-12-17 12:38:19,787] jepsen worker 65 - jepsen.util 11405 :ok :read [21 27] INFO [2019-12-17 12:38:19,787] jepsen worker 65 - jepsen.util 11405 :invoke :read [21 nil] INFO [2019-12-17 12:38:19,792] jepsen worker 28 - jepsen.util 11098 :fail :cas [9 [20 23]] :ValueNotSwapFrom INFO [2019-12-17 12:38:19,793] jepsen worker 28 - jepsen.util 11098 :invoke :cas [9 [6 19]] INFO [2019-12-17 12:38:19,795] jepsen worker 46 - jepsen.util 12646 :fail :cas [15 [27 35]] :ValueNotSwapFrom INFO [2019-12-17 12:38:19,795] jepsen worker 46 - jepsen.util 12646 :invoke :read [15 nil] INFO [2019-12-17 12:38:19,795] jepsen worker 37 - jepsen.util 12637 :fail :cas [12 [33 48]] :ValueNotSwapFrom INFO [2019-12-17 12:38:19,796] jepsen worker 37 - jepsen.util 12637 :invoke :read [12 nil] INFO [2019-12-17 12:38:19,796] jepsen worker 5 - jepsen.util 12695 :fail :cas [1 [29 27]] :ValueNotSwapFrom INFO [2019-12-17 12:38:19,796] jepsen worker 5 - jepsen.util 12695 :invoke :cas [1 [27 4]] INFO [2019-12-17 12:38:19,824] jepsen worker 65 - jepsen.util 11405 :ok :read [21 27] INFO [2019-12-17 12:38:19,825] jepsen worker 65 - jepsen.util 11405 :invoke :read [21 nil] INFO [2019-12-17 12:38:19,828] jepsen worker 28 - jepsen.util 11098 :fail :cas [9 [6 19]] :ValueNotSwapFrom INFO [2019-12-17 12:38:19,829] jepsen worker 28 - jepsen.util 11098 :invoke :read [9 nil] INFO [2019-12-17 12:38:19,830] jepsen worker 79 - jepsen.util 13219 :fail :cas [26 [22 1]] :ValueNotSwapFrom INFO [2019-12-17 12:38:19,830] jepsen worker 79 - jepsen.util 13219 :invoke :read [26 nil] INFO [2019-12-17 12:38:19,832] jepsen worker 85 - jepsen.util 18355 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:38:19,832] jepsen worker 66 - jepsen.util 17796 :info :write [22 19] :RequestCanceledException INFO [2019-12-17 12:38:19,832] jepsen worker 85 - jepsen.util 18355 :invoke :write [28 14] INFO [2019-12-17 12:38:19,833] jepsen worker 66 - jepsen.util 17886 :invoke :write [22 3] INFO [2019-12-17 12:38:19,839] jepsen worker 10 - jepsen.util 12970 :ok :read [3 21] INFO [2019-12-17 12:38:19,840] jepsen worker 10 - jepsen.util 12970 :invoke :write [3 9] INFO [2019-12-17 12:38:19,845] jepsen worker 13 - jepsen.util 12613 :fail :cas [4 [26 13]] :ValueNotSwapFrom INFO [2019-12-17 12:38:19,845] jepsen worker 13 - jepsen.util 12613 :invoke :read [4 nil] INFO [2019-12-17 12:38:19,865] jepsen worker 67 - jepsen.util 17977 :info :cas [22 [42 21]] :RequestCanceledException INFO [2019-12-17 12:38:19,866] jepsen worker 67 - jepsen.util 18067 :invoke :read [22 nil] INFO [2019-12-17 12:38:19,885] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":75442,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:38:19,885] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":75442,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:38:19,885] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":75445,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:38:19,885] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":75445,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:38:19,885] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":75446,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:38:19,885] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":75446,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:38:19,900] jepsen worker 65 - jepsen.util 11405 :ok :read [21 27] INFO [2019-12-17 12:38:19,900] jepsen worker 65 - jepsen.util 11405 :invoke :cas [21 [15 25]] INFO [2019-12-17 12:38:19,907] jepsen worker 28 - jepsen.util 11098 :ok :read [9 23] INFO [2019-12-17 12:38:19,907] jepsen worker 28 - jepsen.util 11098 :invoke :read [9 nil] INFO [2019-12-17 12:38:19,921] jepsen worker 37 - jepsen.util 12637 :ok :read [12 5] INFO [2019-12-17 12:38:19,921] jepsen worker 37 - jepsen.util 12637 :invoke :write [12 1] INFO [2019-12-17 12:38:19,944] jepsen worker 46 - jepsen.util 12646 :ok :read [15 15] INFO [2019-12-17 12:38:19,944] jepsen worker 46 - jepsen.util 12646 :invoke :read [15 nil] INFO [2019-12-17 12:38:19,946] jepsen worker 65 - jepsen.util 11405 :fail :cas [21 [15 25]] :ValueNotSwapFrom INFO [2019-12-17 12:38:19,946] jepsen worker 65 - jepsen.util 11405 :invoke :write [21 20] INFO [2019-12-17 12:38:19,948] jepsen worker 28 - jepsen.util 11098 :ok :read [9 23] INFO [2019-12-17 12:38:19,948] jepsen worker 28 - jepsen.util 11098 :invoke :cas [9 [35 30]] INFO [2019-12-17 12:38:19,956] jepsen worker 87 - jepsen.util 12597 :ok :read [29 11] INFO [2019-12-17 12:38:19,956] jepsen worker 87 - jepsen.util 12597 :invoke :write [29 22] INFO [2019-12-17 12:38:19,989] jepsen worker 28 - jepsen.util 11098 :fail :cas [9 [35 30]] :ValueNotSwapFrom INFO [2019-12-17 12:38:19,989] jepsen worker 28 - jepsen.util 11098 :invoke :read [9 nil] INFO [2019-12-17 12:38:20,004] jepsen worker 28 - jepsen.util 11098 :ok :read [9 23] INFO [2019-12-17 12:38:20,004] jepsen worker 28 - jepsen.util 11098 :invoke :write [9 20] INFO [2019-12-17 12:38:20,019] jepsen worker 0 - jepsen.util 18270 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:38:20,019] jepsen worker 0 - jepsen.util 18270 :invoke :write [0 20] INFO [2019-12-17 12:38:20,033] jepsen worker 84 - jepsen.util 17184 :info :write [28 7] :RequestCanceledException INFO [2019-12-17 12:38:20,034] jepsen worker 84 - jepsen.util 17274 :invoke :write [28 34] INFO [2019-12-17 12:38:20,038] jepsen worker 65 - jepsen.util 11405 :ok :write [21 20] INFO [2019-12-17 12:38:20,038] jepsen worker 65 - jepsen.util 11405 :invoke :write [21 41] INFO [2019-12-17 12:38:20,041] jepsen worker 70 - jepsen.util 13570 :ok :read [23 13] INFO [2019-12-17 12:38:20,041] jepsen worker 70 - jepsen.util 13570 :invoke :write [23 40] INFO [2019-12-17 12:38:20,047] jepsen worker 21 - jepsen.util 17391 :info :write [7 14] :RequestCanceledException INFO [2019-12-17 12:38:20,048] jepsen worker 4 - jepsen.util 12154 :ok :read [1 35] INFO [2019-12-17 12:38:20,048] jepsen worker 4 - jepsen.util 12154 :invoke :read [1 nil] INFO [2019-12-17 12:38:20,049] jepsen worker 21 - jepsen.util 17481 :invoke :read [7 nil] INFO [2019-12-17 12:38:20,062] jepsen worker 30 - jepsen.util 12360 :info :write [10 9] :RequestCanceledException INFO [2019-12-17 12:38:20,063] jepsen worker 30 - jepsen.util 12450 :invoke :write [10 35] INFO [2019-12-17 12:38:20,097] jepsen worker 69 - jepsen.util 13389 :ok :read [23 13] INFO [2019-12-17 12:38:20,097] jepsen worker 69 - jepsen.util 13389 :invoke :cas [23 [13 17]] INFO [2019-12-17 12:38:20,097] jepsen worker 28 - jepsen.util 11098 :ok :write [9 20] INFO [2019-12-17 12:38:20,098] jepsen worker 28 - jepsen.util 11098 :invoke :cas [9 [23 2]] INFO [2019-12-17 12:38:20,104] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":75455,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:38:20,104] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":75455,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:38:20,104] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":75456,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:38:20,104] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":75456,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:38:20,116] jepsen worker 65 - jepsen.util 11405 :ok :write [21 41] INFO [2019-12-17 12:38:20,116] jepsen worker 65 - jepsen.util 11405 :invoke :cas [21 [28 24]] INFO [2019-12-17 12:38:20,121] jepsen worker 79 - jepsen.util 13219 :ok :read [26 25] INFO [2019-12-17 12:38:20,121] jepsen worker 79 - jepsen.util 13219 :invoke :read [26 nil] INFO [2019-12-17 12:38:20,123] jepsen worker 28 - jepsen.util 11098 :fail :cas [9 [23 2]] :ValueNotSwapFrom INFO [2019-12-17 12:38:20,124] jepsen worker 28 - jepsen.util 11098 :invoke :read [9 nil] INFO [2019-12-17 12:38:20,135] jepsen worker 23 - jepsen.util 18023 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:38:20,135] jepsen worker 23 - jepsen.util 18023 :invoke :write [7 47] INFO [2019-12-17 12:38:20,142] jepsen worker 53 - jepsen.util 12563 :fail :cas [17 [31 22]] :ValueNotSwapFrom INFO [2019-12-17 12:38:20,142] jepsen worker 53 - jepsen.util 12563 :invoke :read [17 nil] INFO [2019-12-17 12:38:20,155] jepsen worker 5 - jepsen.util 12695 :fail :cas [1 [27 4]] :ValueNotSwapFrom INFO [2019-12-17 12:38:20,155] jepsen worker 5 - jepsen.util 12695 :invoke :write [1 43] INFO [2019-12-17 12:38:20,176] jepsen worker 28 - jepsen.util 11098 :ok :read [9 20] INFO [2019-12-17 12:38:20,177] jepsen worker 28 - jepsen.util 11098 :invoke :read [9 nil] INFO [2019-12-17 12:38:20,192] jepsen worker 65 - jepsen.util 11405 :fail :cas [21 [28 24]] :ValueNotSwapFrom INFO [2019-12-17 12:38:20,192] jepsen worker 65 - jepsen.util 11405 :invoke :read [21 nil] INFO [2019-12-17 12:38:20,197] jepsen worker 28 - jepsen.util 11098 :ok :read [9 20] INFO [2019-12-17 12:38:20,197] jepsen worker 28 - jepsen.util 11098 :invoke :cas [9 [18 6]] INFO [2019-12-17 12:38:20,201] jepsen worker 72 - jepsen.util 13122 :info :cas [24 [17 4]] :RequestCanceledException INFO [2019-12-17 12:38:20,202] jepsen worker 88 - jepsen.util 13048 :fail :cas [29 [20 36]] :ValueNotSwapFrom INFO [2019-12-17 12:38:20,202] jepsen worker 88 - jepsen.util 13048 :invoke :cas [29 [34 22]] INFO [2019-12-17 12:38:20,202] jepsen worker 72 - jepsen.util 13212 :invoke :cas [24 [3 7]] INFO [2019-12-17 12:38:20,202] jepsen worker 4 - jepsen.util 12154 :ok :read [1 35] INFO [2019-12-17 12:38:20,202] jepsen worker 4 - jepsen.util 12154 :invoke :write [1 10] INFO [2019-12-17 12:38:20,214] jepsen worker 65 - jepsen.util 11405 :ok :read [21 41] INFO [2019-12-17 12:38:20,214] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":75482,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:38:20,214] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":75482,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:38:20,214] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":75483,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:38:20,214] jepsen worker 65 - jepsen.util 11405 :invoke :write [21 12] INFO [2019-12-17 12:38:20,214] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":75483,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:38:20,214] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":75484,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:38:20,215] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":75484,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:38:20,215] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:38:20,215] jepsen nemesis - couchbase.util Rebalance check count: 594 INFO [2019-12-17 12:38:20,215] jepsen nemesis - couchbase.util Rebalance stuck count: 575 INFO [2019-12-17 12:38:20,216] jepsen worker 82 - jepsen.util 17812 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:38:20,217] jepsen worker 82 - jepsen.util 17812 :invoke :cas [27 [24 4]] INFO [2019-12-17 12:38:20,228] jepsen worker 28 - jepsen.util 11098 :fail :cas [9 [18 6]] :ValueNotSwapFrom INFO [2019-12-17 12:38:20,228] jepsen worker 28 - jepsen.util 11098 :invoke :write [9 24] INFO [2019-12-17 12:38:20,235] jepsen worker 72 - jepsen.util 13212 :fail :cas [24 [3 7]] :ValueNotSwapFrom INFO [2019-12-17 12:38:20,235] jepsen worker 72 - jepsen.util 13212 :invoke :read [24 nil] INFO [2019-12-17 12:38:20,259] jepsen worker 72 - jepsen.util 13212 :ok :read [24 37] INFO [2019-12-17 12:38:20,260] jepsen worker 72 - jepsen.util 13212 :invoke :write [24 0] INFO [2019-12-17 12:38:20,264] jepsen worker 79 - jepsen.util 13219 :ok :read [26 25] INFO [2019-12-17 12:38:20,264] jepsen worker 79 - jepsen.util 13219 :invoke :read [26 nil] INFO [2019-12-17 12:38:20,288] jepsen worker 73 - jepsen.util 12133 :ok :write [24 41] INFO [2019-12-17 12:38:20,288] jepsen worker 73 - jepsen.util 12133 :invoke :read [24 nil] INFO [2019-12-17 12:38:20,288] jepsen worker 10 - jepsen.util 12970 :ok :write [3 9] INFO [2019-12-17 12:38:20,288] jepsen worker 10 - jepsen.util 12970 :invoke :read [3 nil] INFO [2019-12-17 12:38:20,290] jepsen worker 53 - jepsen.util 12563 :ok :read [17 40] INFO [2019-12-17 12:38:20,290] jepsen worker 53 - jepsen.util 12563 :invoke :read [17 nil] INFO [2019-12-17 12:38:20,302] jepsen worker 29 - jepsen.util 11279 :info :cas [9 [42 34]] :RequestCanceledException INFO [2019-12-17 12:38:20,303] jepsen worker 29 - jepsen.util 11369 :invoke :read [9 nil] INFO [2019-12-17 12:38:20,309] jepsen worker 46 - jepsen.util 12646 :ok :read [15 15] INFO [2019-12-17 12:38:20,309] jepsen worker 46 - jepsen.util 12646 :invoke :cas [15 [38 9]] INFO [2019-12-17 12:38:20,318] jepsen worker 72 - jepsen.util 13212 :ok :write [24 0] INFO [2019-12-17 12:38:20,318] jepsen worker 72 - jepsen.util 13212 :invoke :read [24 nil] INFO [2019-12-17 12:38:20,320] jepsen worker 54 - jepsen.util 13644 :info :write [18 37] :RequestCanceledException INFO [2019-12-17 12:38:20,321] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":75489,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:38:20,321] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":75489,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:38:20,321] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":75494,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:38:20,321] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":75494,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:38:20,321] jepsen worker 54 - jepsen.util 13734 :invoke :cas [18 [46 3]] INFO [2019-12-17 12:38:20,328] jepsen worker 28 - jepsen.util 11098 :ok :write [9 24] INFO [2019-12-17 12:38:20,328] jepsen worker 28 - jepsen.util 11098 :invoke :read [9 nil] INFO [2019-12-17 12:38:20,340] jepsen worker 72 - jepsen.util 13212 :ok :read [24 0] INFO [2019-12-17 12:38:20,341] jepsen worker 72 - jepsen.util 13212 :invoke :cas [24 [31 28]] INFO [2019-12-17 12:38:20,345] jepsen worker 53 - jepsen.util 12563 :ok :read [17 40] INFO [2019-12-17 12:38:20,345] jepsen worker 53 - jepsen.util 12563 :invoke :read [17 nil] INFO [2019-12-17 12:38:20,366] jepsen worker 28 - jepsen.util 11098 :ok :read [9 24] INFO [2019-12-17 12:38:20,366] jepsen worker 28 - jepsen.util 11098 :invoke :write [9 39] INFO [2019-12-17 12:38:20,368] jepsen worker 65 - jepsen.util 11405 :ok :write [21 12] INFO [2019-12-17 12:38:20,368] jepsen worker 65 - jepsen.util 11405 :invoke :read [21 nil] INFO [2019-12-17 12:38:20,369] jepsen worker 13 - jepsen.util 12613 :ok :read [4 11] INFO [2019-12-17 12:38:20,369] jepsen worker 13 - jepsen.util 12613 :invoke :cas [4 [44 26]] INFO [2019-12-17 12:38:20,378] jepsen worker 89 - jepsen.util 12959 :fail :cas [29 [21 37]] :ValueNotSwapFrom INFO [2019-12-17 12:38:20,378] jepsen worker 89 - jepsen.util 12959 :invoke :cas [29 [36 8]] INFO [2019-12-17 12:38:20,407] jepsen worker 54 - jepsen.util 13734 :fail :cas [18 [46 3]] :ValueNotSwapFrom INFO [2019-12-17 12:38:20,408] jepsen worker 54 - jepsen.util 13734 :invoke :cas [18 [15 41]] INFO [2019-12-17 12:38:20,420] jepsen worker 14 - jepsen.util 12614 :ok :read [4 11] INFO [2019-12-17 12:38:20,420] jepsen worker 14 - jepsen.util 12614 :invoke :write [4 34] INFO [2019-12-17 12:38:20,438] jepsen worker 72 - jepsen.util 13212 :fail :cas [24 [31 28]] :ValueNotSwapFrom INFO [2019-12-17 12:38:20,438] jepsen worker 72 - jepsen.util 13212 :invoke :read [24 nil] INFO [2019-12-17 12:38:20,440] jepsen worker 10 - jepsen.util 12970 :ok :read [3 9] INFO [2019-12-17 12:38:20,441] jepsen worker 10 - jepsen.util 12970 :invoke :cas [3 [42 18]] INFO [2019-12-17 12:38:20,448] jepsen worker 65 - jepsen.util 11405 :ok :read [21 12] INFO [2019-12-17 12:38:20,448] jepsen worker 65 - jepsen.util 11405 :invoke :read [21 nil] INFO [2019-12-17 12:38:20,460] jepsen worker 29 - jepsen.util 11369 :ok :read [9 24] INFO [2019-12-17 12:38:20,460] jepsen worker 29 - jepsen.util 11369 :invoke :cas [9 [37 9]] INFO [2019-12-17 12:38:20,476] jepsen worker 72 - jepsen.util 13212 :ok :read [24 0] INFO [2019-12-17 12:38:20,476] jepsen worker 72 - jepsen.util 13212 :invoke :cas [24 [12 42]] INFO [2019-12-17 12:38:20,481] jepsen worker 46 - jepsen.util 12646 :fail :cas [15 [38 9]] :ValueNotSwapFrom INFO [2019-12-17 12:38:20,481] jepsen worker 46 - jepsen.util 12646 :invoke :write [15 19] INFO [2019-12-17 12:38:20,523] jepsen worker 65 - jepsen.util 11405 :ok :read [21 12] INFO [2019-12-17 12:38:20,523] jepsen worker 72 - jepsen.util 13212 :fail :cas [24 [12 42]] :ValueNotSwapFrom INFO [2019-12-17 12:38:20,523] jepsen worker 65 - jepsen.util 11405 :invoke :read [21 nil] INFO [2019-12-17 12:38:20,523] jepsen worker 72 - jepsen.util 13212 :invoke :read [24 nil] INFO [2019-12-17 12:38:20,533] jepsen worker 79 - jepsen.util 13219 :ok :read [26 25] INFO [2019-12-17 12:38:20,533] jepsen worker 79 - jepsen.util 13219 :invoke :write [26 3] INFO [2019-12-17 12:38:20,534] jepsen worker 77 - jepsen.util 18977 :info :cas [25 [40 8]] :RequestCanceledException INFO [2019-12-17 12:38:20,537] jepsen worker 77 - jepsen.util 19067 :invoke :cas [25 [15 35]] INFO [2019-12-17 12:38:20,544] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":75519,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:38:20,544] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":75519,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:38:20,544] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":75523,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:38:20,545] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":75523,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:38:20,566] jepsen worker 30 - jepsen.util 12450 :ok :write [10 35] INFO [2019-12-17 12:38:20,566] jepsen worker 30 - jepsen.util 12450 :invoke :cas [10 [5 11]] INFO [2019-12-17 12:38:20,572] jepsen worker 72 - jepsen.util 13212 :ok :read [24 0] INFO [2019-12-17 12:38:20,572] jepsen worker 72 - jepsen.util 13212 :invoke :cas [24 [1 20]] INFO [2019-12-17 12:38:20,580] jepsen worker 65 - jepsen.util 11405 :ok :read [21 12] INFO [2019-12-17 12:38:20,580] jepsen worker 10 - jepsen.util 12970 :fail :cas [3 [42 18]] :ValueNotSwapFrom INFO [2019-12-17 12:38:20,580] jepsen worker 65 - jepsen.util 11405 :invoke :read [21 nil] INFO [2019-12-17 12:38:20,580] jepsen worker 10 - jepsen.util 12970 :invoke :read [3 nil] INFO [2019-12-17 12:38:20,594] jepsen worker 88 - jepsen.util 13048 :fail :cas [29 [34 22]] :ValueNotSwapFrom INFO [2019-12-17 12:38:20,594] jepsen worker 88 - jepsen.util 13048 :invoke :read [29 nil] INFO [2019-12-17 12:38:20,601] jepsen worker 2 - jepsen.util 17102 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:38:20,602] jepsen worker 2 - jepsen.util 17102 :invoke :read [0 nil] INFO [2019-12-17 12:38:20,605] jepsen worker 28 - jepsen.util 11098 :ok :write [9 39] INFO [2019-12-17 12:38:20,605] jepsen worker 28 - jepsen.util 11098 :invoke :cas [9 [30 19]] INFO [2019-12-17 12:38:20,613] jepsen worker 53 - jepsen.util 12563 :ok :read [17 40] INFO [2019-12-17 12:38:20,613] jepsen worker 53 - jepsen.util 12563 :invoke :write [17 40] INFO [2019-12-17 12:38:20,634] jepsen worker 13 - jepsen.util 12613 :fail :cas [4 [44 26]] :ValueNotSwapFrom INFO [2019-12-17 12:38:20,634] jepsen worker 72 - jepsen.util 13212 :fail :cas [24 [1 20]] :ValueNotSwapFrom INFO [2019-12-17 12:38:20,634] jepsen worker 13 - jepsen.util 12613 :invoke :cas [4 [36 12]] INFO [2019-12-17 12:38:20,634] jepsen worker 72 - jepsen.util 13212 :invoke :read [24 nil] INFO [2019-12-17 12:38:20,634] jepsen worker 35 - jepsen.util 17945 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:38:20,634] jepsen worker 44 - jepsen.util 18494 :info :write [14 26] :RequestCanceledException INFO [2019-12-17 12:38:20,634] jepsen worker 55 - jepsen.util 13735 :info :write [18 12] :RequestCanceledException INFO [2019-12-17 12:38:20,634] jepsen worker 35 - jepsen.util 17945 :invoke :cas [11 [25 46]] INFO [2019-12-17 12:38:20,635] jepsen worker 55 - jepsen.util 13825 :invoke :write [18 39] INFO [2019-12-17 12:38:20,635] jepsen worker 44 - jepsen.util 18584 :invoke :cas [14 [22 47]] INFO [2019-12-17 12:38:20,643] jepsen worker 54 - jepsen.util 13734 :fail :cas [18 [15 41]] :ValueNotSwapFrom INFO [2019-12-17 12:38:20,643] jepsen worker 54 - jepsen.util 13734 :invoke :write [18 45] INFO [2019-12-17 12:38:20,647] jepsen worker 58 - jepsen.util 18148 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:38:20,647] jepsen worker 15 - jepsen.util 16935 :info :cas [5 [46 47]] :RequestCanceledException INFO [2019-12-17 12:38:20,647] jepsen worker 58 - jepsen.util 18148 :invoke :read [19 nil] INFO [2019-12-17 12:38:20,648] jepsen worker 15 - jepsen.util 17025 :invoke :write [5 7] INFO [2019-12-17 12:38:20,654] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":75548,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:38:20,654] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":75548,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:38:20,654] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":75550,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:38:20,654] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":75550,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:38:20,654] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":75551,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:38:20,654] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":75551,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:38:20,664] jepsen worker 73 - jepsen.util 12133 :ok :read [24 0] INFO [2019-12-17 12:38:20,664] jepsen worker 73 - jepsen.util 12133 :invoke :read [24 nil] INFO [2019-12-17 12:38:20,666] jepsen worker 16 - jepsen.util 19366 :info :cas [5 [4 10]] :RequestCanceledException INFO [2019-12-17 12:38:20,667] jepsen worker 16 - jepsen.util 19456 :invoke :cas [5 [2 2]] INFO [2019-12-17 12:38:20,678] jepsen worker 65 - jepsen.util 11405 :ok :read [21 12] INFO [2019-12-17 12:38:20,678] jepsen worker 65 - jepsen.util 11405 :invoke :read [21 nil] INFO [2019-12-17 12:38:20,701] jepsen worker 88 - jepsen.util 13048 :ok :read [29 11] INFO [2019-12-17 12:38:20,701] jepsen worker 88 - jepsen.util 13048 :invoke :read [29 nil] INFO [2019-12-17 12:38:20,701] jepsen worker 81 - jepsen.util 17271 :info :cas [27 [6 13]] :RequestCanceledException INFO [2019-12-17 12:38:20,702] jepsen worker 81 - jepsen.util 17361 :invoke :write [27 39] INFO [2019-12-17 12:38:20,722] jepsen worker 53 - jepsen.util 12563 :ok :write [17 40] INFO [2019-12-17 12:38:20,722] jepsen worker 53 - jepsen.util 12563 :invoke :write [17 6] INFO [2019-12-17 12:38:20,735] jepsen worker 50 - jepsen.util 17690 :info :cas [16 [25 24]] :RequestCanceledException INFO [2019-12-17 12:38:20,735] jepsen worker 48 - jepsen.util 16608 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:38:20,735] jepsen worker 43 - jepsen.util 18043 :info :write [14 35] :RequestCanceledException INFO [2019-12-17 12:38:20,735] jepsen worker 48 - jepsen.util 16608 :invoke :read [16 nil] INFO [2019-12-17 12:38:20,736] jepsen worker 50 - jepsen.util 17780 :invoke :write [16 22] INFO [2019-12-17 12:38:20,736] jepsen worker 43 - jepsen.util 18133 :invoke :read [14 nil] INFO [2019-12-17 12:38:20,742] jepsen worker 29 - jepsen.util 11369 :fail :cas [9 [37 9]] :ValueNotSwapFrom INFO [2019-12-17 12:38:20,742] jepsen worker 29 - jepsen.util 11369 :invoke :write [9 28] INFO [2019-12-17 12:38:20,745] jepsen worker 72 - jepsen.util 13212 :ok :read [24 0] INFO [2019-12-17 12:38:20,745] jepsen worker 72 - jepsen.util 13212 :invoke :read [24 nil] INFO [2019-12-17 12:38:20,749] jepsen worker 11 - jepsen.util 12881 :info :cas [3 [32 37]] :RequestCanceledException INFO [2019-12-17 12:38:20,750] jepsen worker 11 - jepsen.util 12971 :invoke :cas [3 [49 22]] INFO [2019-12-17 12:38:20,766] jepsen worker 9 - jepsen.util 13419 :info :write [3 36] :RequestCanceledException INFO [2019-12-17 12:38:20,766] jepsen worker 25 - jepsen.util 18925 :info :write [8 12] :RequestCanceledException INFO [2019-12-17 12:38:20,767] jepsen worker 28 - jepsen.util 11098 :fail :cas [9 [30 19]] :ValueNotSwapFrom INFO [2019-12-17 12:38:20,767] jepsen worker 25 - jepsen.util 19015 :invoke :read [8 nil] INFO [2019-12-17 12:38:20,767] jepsen worker 28 - jepsen.util 11098 :invoke :write [9 14] INFO [2019-12-17 12:38:20,767] jepsen worker 9 - jepsen.util 13509 :invoke :read [3 nil] INFO [2019-12-17 12:38:20,783] jepsen worker 30 - jepsen.util 12450 :fail :cas [10 [5 11]] :ValueNotSwapFrom INFO [2019-12-17 12:38:20,783] jepsen worker 30 - jepsen.util 12450 :invoke :cas [10 [33 40]] INFO [2019-12-17 12:38:20,801] jepsen worker 57 - jepsen.util 17427 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:38:20,801] jepsen worker 63 - jepsen.util 11583 :fail :read [21 nil] :RequestTimeoutException INFO [2019-12-17 12:38:20,801] jepsen worker 57 - jepsen.util 17427 :invoke :cas [19 [4 23]] INFO [2019-12-17 12:38:20,801] jepsen worker 63 - jepsen.util 11583 :invoke :cas [21 [44 6]] INFO [2019-12-17 12:38:20,802] jepsen worker 51 - jepsen.util 12291 :ok :write [17 7] INFO [2019-12-17 12:38:20,802] jepsen worker 51 - jepsen.util 12291 :invoke :cas [17 [38 31]] INFO [2019-12-17 12:38:20,812] jepsen worker 65 - jepsen.util 11405 :ok :read [21 12] INFO [2019-12-17 12:38:20,812] jepsen worker 65 - jepsen.util 11405 :invoke :read [21 nil] INFO [2019-12-17 12:38:20,819] jepsen worker 8 - jepsen.util 17378 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:38:20,819] jepsen worker 19 - jepsen.util 13789 :info :cas [6 [14 37]] :RequestCanceledException INFO [2019-12-17 12:38:20,819] jepsen worker 8 - jepsen.util 17378 :invoke :write [2 36] INFO [2019-12-17 12:38:20,820] jepsen worker 19 - jepsen.util 13879 :invoke :cas [6 [14 22]] INFO [2019-12-17 12:38:20,824] jepsen worker 72 - jepsen.util 13212 :ok :read [24 0] INFO [2019-12-17 12:38:20,824] jepsen worker 72 - jepsen.util 13212 :invoke :read [24 nil] INFO [2019-12-17 12:38:20,825] jepsen worker 88 - jepsen.util 13048 :ok :read [29 11] INFO [2019-12-17 12:38:20,825] jepsen worker 88 - jepsen.util 13048 :invoke :write [29 12] INFO [2019-12-17 12:38:20,835] jepsen worker 18 - jepsen.util 13608 :info :cas [6 [31 24]] :RequestCanceledException INFO [2019-12-17 12:38:20,835] jepsen worker 47 - jepsen.util 14267 :info :cas [15 [38 13]] :RequestCanceledException INFO [2019-12-17 12:38:20,836] jepsen worker 18 - jepsen.util 13698 :invoke :read [6 nil] INFO [2019-12-17 12:38:20,836] jepsen worker 47 - jepsen.util 14357 :invoke :write [15 48] INFO [2019-12-17 12:38:20,840] jepsen worker 38 - jepsen.util 13718 :ok :write [12 37] INFO [2019-12-17 12:38:20,841] jepsen worker 38 - jepsen.util 13718 :invoke :cas [12 [48 34]] INFO [2019-12-17 12:38:20,848] jepsen worker 13 - jepsen.util 12613 :fail :cas [4 [36 12]] :ValueNotSwapFrom INFO [2019-12-17 12:38:20,848] jepsen worker 61 - jepsen.util 18961 :info :cas [20 [1 34]] :RequestCanceledException INFO [2019-12-17 12:38:20,848] jepsen worker 13 - jepsen.util 12613 :invoke :cas [4 [4 29]] INFO [2019-12-17 12:38:20,849] jepsen worker 61 - jepsen.util 19051 :invoke :cas [20 [34 3]] INFO [2019-12-17 12:38:20,865] jepsen worker 73 - jepsen.util 12133 :ok :read [24 0] INFO [2019-12-17 12:38:20,865] jepsen worker 7 - jepsen.util 17467 :info :write [2 46] :RequestCanceledException INFO [2019-12-17 12:38:20,865] jepsen worker 73 - jepsen.util 12133 :invoke :write [24 39] INFO [2019-12-17 12:38:20,866] jepsen worker 7 - jepsen.util 17557 :invoke :cas [2 [46 21]] INFO [2019-12-17 12:38:20,873] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":75586,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:38:20,873] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":75586,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:38:20,873] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":75590,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:38:20,873] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":75590,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:38:20,873] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":75591,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:38:20,873] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":75591,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:38:20,883] jepsen worker 51 - jepsen.util 12291 :fail :cas [17 [38 31]] :ValueNotSwapFrom INFO [2019-12-17 12:38:20,883] jepsen worker 51 - jepsen.util 12291 :invoke :write [17 46] INFO [2019-12-17 12:38:20,897] jepsen worker 63 - jepsen.util 11583 :fail :cas [21 [44 6]] :ValueNotSwapFrom INFO [2019-12-17 12:38:20,897] jepsen worker 63 - jepsen.util 11583 :invoke :cas [21 [8 47]] INFO [2019-12-17 12:38:20,901] jepsen worker 74 - jepsen.util 12944 :fail :read [24 nil] :RequestTimeoutException INFO [2019-12-17 12:38:20,901] jepsen worker 74 - jepsen.util 12944 :invoke :write [24 11] INFO [2019-12-17 12:38:20,901] jepsen worker 4 - jepsen.util 12154 :ok :write [1 10] INFO [2019-12-17 12:38:20,902] jepsen worker 4 - jepsen.util 12154 :invoke :cas [1 [23 2]] INFO [2019-12-17 12:38:20,919] jepsen worker 65 - jepsen.util 11405 :ok :read [21 12] INFO [2019-12-17 12:38:20,919] jepsen worker 65 - jepsen.util 11405 :invoke :read [21 nil] INFO [2019-12-17 12:38:20,932] jepsen worker 11 - jepsen.util 12971 :fail :cas [3 [49 22]] :ValueNotSwapFrom INFO [2019-12-17 12:38:20,933] jepsen worker 11 - jepsen.util 12971 :invoke :write [3 26] INFO [2019-12-17 12:38:20,934] jepsen worker 83 - jepsen.util 17813 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:38:20,934] jepsen worker 59 - jepsen.util 16889 :info :write [19 37] :RequestCanceledException INFO [2019-12-17 12:38:20,934] jepsen worker 83 - jepsen.util 17813 :invoke :cas [27 [42 38]] INFO [2019-12-17 12:38:20,935] jepsen worker 59 - jepsen.util 16979 :invoke :cas [19 [18 1]] INFO [2019-12-17 12:38:20,955] jepsen worker 72 - jepsen.util 13212 :ok :read [24 0] INFO [2019-12-17 12:38:20,955] jepsen worker 72 - jepsen.util 13212 :invoke :write [24 14] INFO [2019-12-17 12:38:20,966] jepsen worker 71 - jepsen.util 13841 :info :write [23 36] :RequestCanceledException INFO [2019-12-17 12:38:20,971] jepsen worker 71 - jepsen.util 13931 :invoke :write [23 37] INFO [2019-12-17 12:38:20,980] jepsen worker 18 - jepsen.util 13698 :ok :read [6 44] INFO [2019-12-17 12:38:20,980] jepsen worker 18 - jepsen.util 13698 :invoke :cas [6 [13 14]] INFO [2019-12-17 12:38:20,986] jepsen worker 36 - jepsen.util 13806 :ok :write [12 21] INFO [2019-12-17 12:38:20,986] jepsen worker 36 - jepsen.util 13806 :invoke :cas [12 [45 47]] INFO [2019-12-17 12:38:20,992] jepsen worker 30 - jepsen.util 12450 :fail :cas [10 [33 40]] :ValueNotSwapFrom INFO [2019-12-17 12:38:20,992] jepsen worker 30 - jepsen.util 12450 :invoke :cas [10 [6 22]] INFO [2019-12-17 12:38:20,994] jepsen worker 88 - jepsen.util 13048 :ok :write [29 12] INFO [2019-12-17 12:38:20,994] jepsen worker 88 - jepsen.util 13048 :invoke :write [29 28] INFO [2019-12-17 12:38:21,002] jepsen worker 63 - jepsen.util 11583 :fail :cas [21 [8 47]] :ValueNotSwapFrom INFO [2019-12-17 12:38:21,002] jepsen worker 63 - jepsen.util 11583 :invoke :cas [21 [0 2]] INFO [2019-12-17 12:38:21,019] jepsen worker 65 - jepsen.util 11405 :ok :read [21 12] INFO [2019-12-17 12:38:21,020] jepsen worker 65 - jepsen.util 11405 :invoke :cas [21 [4 24]] INFO [2019-12-17 12:38:21,035] jepsen worker 42 - jepsen.util 18672 :info :cas [14 [27 1]] :RequestCanceledException INFO [2019-12-17 12:38:21,035] jepsen worker 42 - jepsen.util 18762 :invoke :read [14 nil] INFO [2019-12-17 12:38:21,049] jepsen worker 39 - jepsen.util 17409 :info :write [13 9] :RequestCanceledException INFO [2019-12-17 12:38:21,050] jepsen worker 39 - jepsen.util 17499 :invoke :read [13 nil] INFO [2019-12-17 12:38:21,059] jepsen worker 18 - jepsen.util 13698 :fail :cas [6 [13 14]] :ValueNotSwapFrom INFO [2019-12-17 12:38:21,060] jepsen worker 18 - jepsen.util 13698 :invoke :write [6 45] INFO [2019-12-17 12:38:21,066] jepsen worker 1 - jepsen.util 18181 :info :cas [0 [5 12]] :RequestCanceledException INFO [2019-12-17 12:38:21,067] jepsen worker 1 - jepsen.util 18271 :invoke :cas [0 [11 6]] INFO [2019-12-17 12:38:21,085] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":75608,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:38:21,085] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":75608,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:38:21,085] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":75611,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:38:21,085] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":75611,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:38:21,085] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":75613,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:38:21,085] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":75613,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:38:21,097] jepsen worker 12 - jepsen.util 13242 :ok :write [4 34] INFO [2019-12-17 12:38:21,097] jepsen worker 12 - jepsen.util 13242 :invoke :read [4 nil] INFO [2019-12-17 12:38:21,111] jepsen worker 65 - jepsen.util 11405 :fail :cas [21 [4 24]] :ValueNotSwapFrom INFO [2019-12-17 12:38:21,111] jepsen worker 65 - jepsen.util 11405 :invoke :write [21 33] INFO [2019-12-17 12:38:21,124] jepsen worker 46 - jepsen.util 12646 :ok :write [15 19] INFO [2019-12-17 12:38:21,124] jepsen worker 46 - jepsen.util 12646 :invoke :write [15 14] INFO [2019-12-17 12:38:21,130] jepsen worker 74 - jepsen.util 12944 :ok :write [24 11] INFO [2019-12-17 12:38:21,130] jepsen worker 74 - jepsen.util 12944 :invoke :write [24 39] INFO [2019-12-17 12:38:21,134] jepsen worker 64 - jepsen.util 10144 :info :write [21 42] :RequestCanceledException INFO [2019-12-17 12:38:21,134] jepsen worker 68 - jepsen.util 17078 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:38:21,134] jepsen worker 68 - jepsen.util 17078 :invoke :read [22 nil] INFO [2019-12-17 12:38:21,135] jepsen worker 64 - jepsen.util 10234 :invoke :write [21 36] INFO [2019-12-17 12:38:21,151] jepsen worker 28 - jepsen.util 11098 :ok :write [9 14] INFO [2019-12-17 12:38:21,151] jepsen worker 28 - jepsen.util 11098 :invoke :read [9 nil] INFO [2019-12-17 12:38:21,165] jepsen worker 76 - jepsen.util 17986 :info :cas [25 [16 27]] :RequestCanceledException INFO [2019-12-17 12:38:21,166] jepsen worker 76 - jepsen.util 18076 :invoke :read [25 nil] INFO [2019-12-17 12:38:21,168] jepsen worker 10 - jepsen.util 12970 :ok :read [3 9] INFO [2019-12-17 12:38:21,168] jepsen worker 10 - jepsen.util 12970 :invoke :read [3 nil] INFO [2019-12-17 12:38:21,178] jepsen worker 4 - jepsen.util 12154 :fail :cas [1 [23 2]] :ValueNotSwapFrom INFO [2019-12-17 12:38:21,178] jepsen worker 4 - jepsen.util 12154 :invoke :read [1 nil] INFO [2019-12-17 12:38:21,183] jepsen worker 28 - jepsen.util 11098 :ok :read [9 14] INFO [2019-12-17 12:38:21,183] jepsen worker 28 - jepsen.util 11098 :invoke :write [9 49] INFO [2019-12-17 12:38:21,201] jepsen worker 36 - jepsen.util 13806 :fail :cas [12 [45 47]] :ValueNotSwapFrom INFO [2019-12-17 12:38:21,201] jepsen worker 36 - jepsen.util 13806 :invoke :write [12 46] INFO [2019-12-17 12:38:21,201] jepsen worker 22 - jepsen.util 18202 :info :write [7 9] :RequestCanceledException INFO [2019-12-17 12:38:21,201] jepsen worker 32 - jepsen.util 13442 :fail :read [10 nil] :RequestTimeoutException INFO [2019-12-17 12:38:21,201] jepsen worker 32 - jepsen.util 13442 :invoke :read [10 nil] INFO [2019-12-17 12:38:21,202] jepsen worker 22 - jepsen.util 18292 :invoke :write [7 3] INFO [2019-12-17 12:38:21,212] jepsen worker 89 - jepsen.util 12959 :fail :cas [29 [36 8]] :ValueNotSwapFrom INFO [2019-12-17 12:38:21,212] jepsen worker 89 - jepsen.util 12959 :invoke :write [29 13] INFO [2019-12-17 12:38:21,219] jepsen worker 6 - jepsen.util 18186 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:38:21,219] jepsen worker 6 - jepsen.util 18186 :invoke :read [2 nil] INFO [2019-12-17 12:38:21,226] jepsen worker 29 - jepsen.util 11369 :ok :write [9 28] INFO [2019-12-17 12:38:21,226] jepsen worker 29 - jepsen.util 11369 :invoke :cas [9 [47 40]] INFO [2019-12-17 12:38:21,230] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:38:21,230] jepsen nemesis - couchbase.util Rebalance check count: 595 INFO [2019-12-17 12:38:21,230] jepsen nemesis - couchbase.util Rebalance stuck count: 576 INFO [2019-12-17 12:38:21,248] jepsen worker 56 - jepsen.util 13736 :info :write [18 8] :RequestCanceledException INFO [2019-12-17 12:38:21,249] jepsen worker 56 - jepsen.util 13826 :invoke :read [18 nil] INFO [2019-12-17 12:38:21,255] jepsen worker 29 - jepsen.util 11369 :fail :cas [9 [47 40]] :ValueNotSwapFrom INFO [2019-12-17 12:38:21,255] jepsen worker 29 - jepsen.util 11369 :invoke :write [9 9] INFO [2019-12-17 12:38:21,279] jepsen worker 4 - jepsen.util 12154 :ok :read [1 10] INFO [2019-12-17 12:38:21,279] jepsen worker 4 - jepsen.util 12154 :invoke :read [1 nil] INFO [2019-12-17 12:38:21,280] jepsen worker 28 - jepsen.util 11098 :ok :write [9 49] INFO [2019-12-17 12:38:21,280] jepsen worker 28 - jepsen.util 11098 :invoke :cas [9 [37 15]] INFO [2019-12-17 12:38:21,299] jepsen worker 18 - jepsen.util 13698 :ok :write [6 45] INFO [2019-12-17 12:38:21,299] jepsen worker 18 - jepsen.util 13698 :invoke :cas [6 [14 18]] INFO [2019-12-17 12:38:21,302] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":75642,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:38:21,302] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":75642,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:38:21,302] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":75645,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:38:21,302] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":75645,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:38:21,306] jepsen worker 28 - jepsen.util 11098 :fail :cas [9 [37 15]] :ValueNotSwapFrom INFO [2019-12-17 12:38:21,306] jepsen worker 28 - jepsen.util 11098 :invoke :write [9 2] INFO [2019-12-17 12:38:21,311] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":75632,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:38:21,311] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":75632,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:38:21,318] jepsen worker 72 - jepsen.util 13212 :ok :write [24 14] INFO [2019-12-17 12:38:21,318] jepsen worker 72 - jepsen.util 13212 :invoke :cas [24 [24 8]] INFO [2019-12-17 12:38:21,319] jepsen worker 31 - jepsen.util 11191 :fail :read [10 nil] :RequestTimeoutException INFO [2019-12-17 12:38:21,319] jepsen worker 31 - jepsen.util 11191 :invoke :write [10 36] INFO [2019-12-17 12:38:21,334] jepsen worker 18 - jepsen.util 13698 :fail :cas [6 [14 18]] :ValueNotSwapFrom INFO [2019-12-17 12:38:21,334] jepsen worker 34 - jepsen.util 18844 :info :cas [11 [39 9]] :RequestCanceledException INFO [2019-12-17 12:38:21,334] jepsen worker 18 - jepsen.util 13698 :invoke :cas [6 [23 43]] INFO [2019-12-17 12:38:21,335] jepsen worker 34 - jepsen.util 18934 :invoke :cas [11 [39 15]] INFO [2019-12-17 12:38:21,337] jepsen worker 29 - jepsen.util 11369 :ok :write [9 9] INFO [2019-12-17 12:38:21,337] jepsen worker 29 - jepsen.util 11369 :invoke :read [9 nil] INFO [2019-12-17 12:38:21,354] jepsen worker 18 - jepsen.util 13698 :fail :cas [6 [23 43]] :ValueNotSwapFrom INFO [2019-12-17 12:38:21,354] jepsen worker 18 - jepsen.util 13698 :invoke :cas [6 [19 25]] INFO [2019-12-17 12:38:21,356] jepsen worker 13 - jepsen.util 12613 :fail :cas [4 [4 29]] :ValueNotSwapFrom INFO [2019-12-17 12:38:21,356] jepsen worker 13 - jepsen.util 12613 :invoke :write [4 15] INFO [2019-12-17 12:38:21,366] jepsen worker 52 - jepsen.util 12652 :info :write [17 2] :RequestCanceledException INFO [2019-12-17 12:38:21,366] jepsen worker 52 - jepsen.util 12742 :invoke :read [17 nil] INFO [2019-12-17 12:38:21,380] jepsen worker 4 - jepsen.util 12154 :ok :read [1 10] INFO [2019-12-17 12:38:21,380] jepsen worker 4 - jepsen.util 12154 :invoke :read [1 nil] INFO [2019-12-17 12:38:21,387] jepsen worker 18 - jepsen.util 13698 :fail :cas [6 [19 25]] :ValueNotSwapFrom INFO [2019-12-17 12:38:21,388] jepsen worker 18 - jepsen.util 13698 :invoke :cas [6 [40 28]] INFO [2019-12-17 12:38:21,401] jepsen worker 49 - jepsen.util 17869 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:38:21,401] jepsen worker 49 - jepsen.util 17869 :invoke :cas [16 [43 45]] INFO [2019-12-17 12:38:21,418] jepsen worker 40 - jepsen.util 18580 :info :cas [13 [24 31]] :RequestCanceledException INFO [2019-12-17 12:38:21,420] jepsen worker 40 - jepsen.util 18670 :invoke :read [13 nil] INFO [2019-12-17 12:38:21,423] jepsen worker 18 - jepsen.util 13698 :fail :cas [6 [40 28]] :ValueNotSwapFrom INFO [2019-12-17 12:38:21,423] jepsen worker 18 - jepsen.util 13698 :invoke :write [6 40] INFO [2019-12-17 12:38:21,435] jepsen worker 41 - jepsen.util 19121 :info :cas [13 [36 6]] :RequestCanceledException INFO [2019-12-17 12:38:21,436] jepsen worker 41 - jepsen.util 19211 :invoke :read [13 nil] INFO [2019-12-17 12:38:21,438] jepsen worker 28 - jepsen.util 11098 :ok :write [9 2] INFO [2019-12-17 12:38:21,438] jepsen worker 28 - jepsen.util 11098 :invoke :write [9 49] INFO [2019-12-17 12:38:21,443] jepsen worker 38 - jepsen.util 13718 :fail :cas [12 [48 34]] :ValueNotSwapFrom INFO [2019-12-17 12:38:21,443] jepsen worker 38 - jepsen.util 13718 :invoke :cas [12 [30 44]] INFO [2019-12-17 12:38:21,449] jepsen worker 24 - jepsen.util 16854 :info :write [8 24] :RequestCanceledException INFO [2019-12-17 12:38:21,450] jepsen worker 24 - jepsen.util 16944 :invoke :write [8 14] INFO [2019-12-17 12:38:21,456] jepsen worker 4 - jepsen.util 12154 :ok :read [1 10] INFO [2019-12-17 12:38:21,456] jepsen worker 4 - jepsen.util 12154 :invoke :write [1 45] INFO [2019-12-17 12:38:21,488] jepsen worker 28 - jepsen.util 11098 :ok :write [9 49] INFO [2019-12-17 12:38:21,489] jepsen worker 28 - jepsen.util 11098 :invoke :cas [9 [49 13]] INFO [2019-12-17 12:38:21,519] jepsen worker 3 - jepsen.util 12603 :info :write [1 37] :RequestCanceledException INFO [2019-12-17 12:38:21,519] jepsen worker 3 - jepsen.util 12693 :invoke :read [1 nil] INFO [2019-12-17 12:38:21,530] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":75669,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:38:21,530] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":75669,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:38:21,530] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":75672,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:38:21,530] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":75672,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:38:21,541] jepsen worker 28 - jepsen.util 11098 :ok :cas [9 [49 13]] INFO [2019-12-17 12:38:21,541] jepsen worker 28 - jepsen.util 11098 :invoke :read [9 nil] INFO [2019-12-17 12:38:21,556] jepsen worker 28 - jepsen.util 11098 :ok :read [9 13] INFO [2019-12-17 12:38:21,556] jepsen worker 28 - jepsen.util 11098 :invoke :write [9 39] INFO [2019-12-17 12:38:21,565] jepsen worker 86 - jepsen.util 18536 :info :cas [28 [39 40]] :RequestCanceledException INFO [2019-12-17 12:38:21,567] jepsen worker 86 - jepsen.util 18626 :invoke :cas [28 [36 25]] INFO [2019-12-17 12:38:21,570] jepsen worker 56 - jepsen.util 13826 :ok :read [18 48] INFO [2019-12-17 12:38:21,570] jepsen worker 56 - jepsen.util 13826 :invoke :write [18 29] INFO [2019-12-17 12:38:21,589] jepsen worker 38 - jepsen.util 13718 :fail :cas [12 [30 44]] :ValueNotSwapFrom INFO [2019-12-17 12:38:21,589] jepsen worker 38 - jepsen.util 13718 :invoke :read [12 nil] INFO [2019-12-17 12:38:21,597] jepsen worker 28 - jepsen.util 11098 :ok :write [9 39] INFO [2019-12-17 12:38:21,597] jepsen worker 28 - jepsen.util 11098 :invoke :cas [9 [39 15]] INFO [2019-12-17 12:38:21,620] jepsen worker 52 - jepsen.util 12742 :ok :read [17 7] INFO [2019-12-17 12:38:21,620] jepsen worker 52 - jepsen.util 12742 :invoke :write [17 13] INFO [2019-12-17 12:38:21,633] jepsen worker 62 - jepsen.util 18422 :info :cas [20 [23 13]] :RequestCanceledException INFO [2019-12-17 12:38:21,634] jepsen worker 62 - jepsen.util 18512 :invoke :cas [20 [33 29]] INFO [2019-12-17 12:38:21,647] jepsen worker 28 - jepsen.util 11098 :ok :cas [9 [39 15]] INFO [2019-12-17 12:38:21,647] jepsen worker 28 - jepsen.util 11098 :invoke :cas [9 [0 23]] INFO [2019-12-17 12:38:21,666] jepsen worker 75 - jepsen.util 17535 :info :cas [25 [36 24]] :RequestCanceledException INFO [2019-12-17 12:38:21,667] jepsen worker 28 - jepsen.util 11098 :fail :cas [9 [0 23]] :ValueNotSwapFrom INFO [2019-12-17 12:38:21,668] jepsen worker 28 - jepsen.util 11098 :invoke :write [9 13] INFO [2019-12-17 12:38:21,668] jepsen worker 12 - jepsen.util 13242 :ok :read [4 34] INFO [2019-12-17 12:38:21,668] jepsen worker 12 - jepsen.util 13242 :invoke :write [4 35] INFO [2019-12-17 12:38:21,669] jepsen worker 75 - jepsen.util 17625 :invoke :write [25 47] INFO [2019-12-17 12:38:21,701] jepsen worker 27 - jepsen.util 12087 :info :write [9 44] :RequestCanceledException INFO [2019-12-17 12:38:21,701] jepsen worker 45 - jepsen.util 12645 :info :cas [15 [13 12]] :RequestCanceledException INFO [2019-12-17 12:38:21,701] jepsen worker 17 - jepsen.util 17927 :info :cas [5 [10 6]] :RequestCanceledException INFO [2019-12-17 12:38:21,702] jepsen worker 17 - jepsen.util 18017 :invoke :read [5 nil] INFO [2019-12-17 12:38:21,702] jepsen worker 27 - jepsen.util 12177 :invoke :cas [9 [19 45]] INFO [2019-12-17 12:38:21,703] jepsen worker 45 - jepsen.util 12735 :invoke :write [15 6] INFO [2019-12-17 12:38:21,709] jepsen worker 28 - jepsen.util 11098 :ok :write [9 13] INFO [2019-12-17 12:38:21,709] jepsen worker 28 - jepsen.util 11098 :invoke :cas [9 [48 21]] INFO [2019-12-17 12:38:21,727] jepsen worker 27 - jepsen.util 12177 :fail :cas [9 [19 45]] :ValueNotSwapFrom INFO [2019-12-17 12:38:21,727] jepsen worker 27 - jepsen.util 12177 :invoke :cas [9 [43 46]] INFO [2019-12-17 12:38:21,732] jepsen worker 28 - jepsen.util 11098 :fail :cas [9 [48 21]] :ValueNotSwapFrom INFO [2019-12-17 12:38:21,732] jepsen worker 28 - jepsen.util 11098 :invoke :cas [9 [11 5]] INFO [2019-12-17 12:38:21,735] jepsen worker 26 - jepsen.util 18746 :info :write [8 32] :RequestCanceledException INFO [2019-12-17 12:38:21,736] jepsen worker 26 - jepsen.util 18836 :invoke :cas [8 [36 10]] INFO [2019-12-17 12:38:21,744] jepsen worker 3 - jepsen.util 12693 :ok :read [1 10] INFO [2019-12-17 12:38:21,744] jepsen worker 3 - jepsen.util 12693 :invoke :read [1 nil] INFO [2019-12-17 12:38:21,748] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":75693,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:38:21,748] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":75693,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:38:21,749] jepsen worker 80 - jepsen.util 12500 :info :write [26 10] :RequestCanceledException INFO [2019-12-17 12:38:21,749] jepsen worker 60 - jepsen.util 18780 :info :write [20 6] :RequestCanceledException INFO [2019-12-17 12:38:21,750] jepsen worker 60 - jepsen.util 18870 :invoke :write [20 18] INFO [2019-12-17 12:38:21,750] jepsen worker 80 - jepsen.util 12590 :invoke :read [26 nil] INFO [2019-12-17 12:38:21,752] jepsen worker 27 - jepsen.util 12177 :fail :cas [9 [43 46]] :ValueNotSwapFrom INFO [2019-12-17 12:38:21,752] jepsen worker 27 - jepsen.util 12177 :invoke :write [9 35] INFO [2019-12-17 12:38:21,780] jepsen worker 28 - jepsen.util 11098 :fail :cas [9 [11 5]] :ValueNotSwapFrom INFO [2019-12-17 12:38:21,781] jepsen worker 28 - jepsen.util 11098 :invoke :write [9 22] INFO [2019-12-17 12:38:21,813] jepsen worker 27 - jepsen.util 12177 :ok :write [9 35] INFO [2019-12-17 12:38:21,813] jepsen worker 27 - jepsen.util 12177 :invoke :cas [9 [43 12]] INFO [2019-12-17 12:38:21,822] jepsen worker 38 - jepsen.util 13718 :ok :read [12 21] INFO [2019-12-17 12:38:21,822] jepsen worker 38 - jepsen.util 13718 :invoke :write [12 25] INFO [2019-12-17 12:38:21,832] jepsen worker 78 - jepsen.util 12768 :info :write [26 36] :RequestCanceledException INFO [2019-12-17 12:38:21,832] jepsen worker 20 - jepsen.util 12800 :info :write [6 6] :RequestCanceledException INFO [2019-12-17 12:38:21,835] jepsen worker 78 - jepsen.util 12858 :invoke :cas [26 [27 14]] INFO [2019-12-17 12:38:21,837] jepsen worker 20 - jepsen.util 12890 :invoke :read [6 nil] INFO [2019-12-17 12:38:21,846] jepsen worker 33 - jepsen.util 17763 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:38:21,846] jepsen worker 33 - jepsen.util 17763 :invoke :cas [11 [42 4]] INFO [2019-12-17 12:38:21,857] jepsen worker 28 - jepsen.util 11098 :ok :write [9 22] INFO [2019-12-17 12:38:21,857] jepsen worker 28 - jepsen.util 11098 :invoke :cas [9 [34 18]] INFO [2019-12-17 12:38:21,887] jepsen worker 20 - jepsen.util 12890 :ok :read [6 45] INFO [2019-12-17 12:38:21,887] jepsen worker 20 - jepsen.util 12890 :invoke :cas [6 [12 39]] INFO [2019-12-17 12:38:21,900] jepsen worker 28 - jepsen.util 11098 :fail :cas [9 [34 18]] :ValueNotSwapFrom INFO [2019-12-17 12:38:21,900] jepsen worker 28 - jepsen.util 11098 :invoke :cas [9 [21 21]] INFO [2019-12-17 12:38:21,904] jepsen worker 20 - jepsen.util 12890 :fail :cas [6 [12 39]] :ValueNotSwapFrom INFO [2019-12-17 12:38:21,905] jepsen worker 20 - jepsen.util 12890 :invoke :read [6 nil] INFO [2019-12-17 12:38:21,914] jepsen worker 28 - jepsen.util 11098 :fail :cas [9 [21 21]] :ValueNotSwapFrom INFO [2019-12-17 12:38:21,914] jepsen worker 28 - jepsen.util 11098 :invoke :read [9 nil] INFO [2019-12-17 12:38:21,919] jepsen worker 66 - jepsen.util 17886 :info :write [22 3] :RequestCanceledException INFO [2019-12-17 12:38:21,920] jepsen worker 66 - jepsen.util 17976 :invoke :write [22 15] INFO [2019-12-17 12:38:21,922] jepsen worker 20 - jepsen.util 12890 :ok :read [6 45] INFO [2019-12-17 12:38:21,922] jepsen worker 20 - jepsen.util 12890 :invoke :cas [6 [15 32]] INFO [2019-12-17 12:38:21,927] jepsen worker 28 - jepsen.util 11098 :ok :read [9 22] INFO [2019-12-17 12:38:21,927] jepsen worker 28 - jepsen.util 11098 :invoke :cas [9 [3 39]] INFO [2019-12-17 12:38:21,934] jepsen worker 37 - jepsen.util 12637 :info :write [12 1] :RequestCanceledException INFO [2019-12-17 12:38:21,935] jepsen worker 37 - jepsen.util 12727 :invoke :cas [12 [26 18]] INFO [2019-12-17 12:38:21,935] jepsen worker 20 - jepsen.util 12890 :fail :cas [6 [15 32]] :ValueNotSwapFrom INFO [2019-12-17 12:38:21,935] jepsen worker 20 - jepsen.util 12890 :invoke :cas [6 [28 26]] INFO [2019-12-17 12:38:21,935] jepsen worker 85 - jepsen.util 18355 :info :write [28 14] :RequestCanceledException INFO [2019-12-17 12:38:21,935] jepsen worker 67 - jepsen.util 18067 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:38:21,935] jepsen worker 67 - jepsen.util 18067 :invoke :write [22 0] INFO [2019-12-17 12:38:21,936] jepsen worker 85 - jepsen.util 18445 :invoke :read [28 nil] INFO [2019-12-17 12:38:21,942] jepsen worker 28 - jepsen.util 11098 :fail :cas [9 [3 39]] :ValueNotSwapFrom INFO [2019-12-17 12:38:21,942] jepsen worker 28 - jepsen.util 11098 :invoke :write [9 2] INFO [2019-12-17 12:38:21,949] jepsen worker 20 - jepsen.util 12890 :fail :cas [6 [28 26]] :ValueNotSwapFrom INFO [2019-12-17 12:38:21,949] jepsen worker 20 - jepsen.util 12890 :invoke :write [6 9] INFO [2019-12-17 12:38:21,967] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":75719,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:38:21,967] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":75719,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:38:21,967] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":75723,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:38:21,967] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":75723,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:38:21,981] jepsen worker 28 - jepsen.util 11098 :ok :write [9 2] INFO [2019-12-17 12:38:21,981] jepsen worker 28 - jepsen.util 11098 :invoke :write [9 1] INFO [2019-12-17 12:38:22,023] jepsen worker 28 - jepsen.util 11098 :ok :write [9 1] INFO [2019-12-17 12:38:22,023] jepsen worker 28 - jepsen.util 11098 :invoke :read [9 nil] INFO [2019-12-17 12:38:22,039] jepsen worker 28 - jepsen.util 11098 :ok :read [9 1] INFO [2019-12-17 12:38:22,040] jepsen worker 28 - jepsen.util 11098 :invoke :write [9 10] INFO [2019-12-17 12:38:22,065] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":75741,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:38:22,065] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":75741,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:38:22,065] jepsen worker 21 - jepsen.util 17481 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:38:22,065] jepsen worker 87 - jepsen.util 12597 :info :write [29 22] :RequestCanceledException INFO [2019-12-17 12:38:22,066] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":75743,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:38:22,066] jepsen worker 21 - jepsen.util 17481 :invoke :cas [7 [24 30]] INFO [2019-12-17 12:38:22,066] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":75743,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:38:22,066] jepsen worker 87 - jepsen.util 12687 :invoke :cas [29 [29 1]] INFO [2019-12-17 12:38:22,078] jepsen worker 28 - jepsen.util 11098 :ok :write [9 10] INFO [2019-12-17 12:38:22,078] jepsen worker 28 - jepsen.util 11098 :invoke :cas [9 [35 3]] INFO [2019-12-17 12:38:22,081] jepsen worker 3 - jepsen.util 12693 :ok :read [1 10] INFO [2019-12-17 12:38:22,081] jepsen worker 3 - jepsen.util 12693 :invoke :write [1 44] INFO [2019-12-17 12:38:22,098] jepsen worker 28 - jepsen.util 11098 :fail :cas [9 [35 3]] :ValueNotSwapFrom INFO [2019-12-17 12:38:22,098] jepsen worker 28 - jepsen.util 11098 :invoke :cas [9 [16 40]] INFO [2019-12-17 12:38:22,109] jepsen worker 28 - jepsen.util 11098 :fail :cas [9 [16 40]] :ValueNotSwapFrom INFO [2019-12-17 12:38:22,109] jepsen worker 28 - jepsen.util 11098 :invoke :cas [9 [27 47]] INFO [2019-12-17 12:38:22,120] jepsen worker 0 - jepsen.util 18270 :info :write [0 20] :RequestCanceledException INFO [2019-12-17 12:38:22,120] jepsen worker 28 - jepsen.util 11098 :fail :cas [9 [27 47]] :ValueNotSwapFrom INFO [2019-12-17 12:38:22,120] jepsen worker 28 - jepsen.util 11098 :invoke :cas [9 [35 44]] INFO [2019-12-17 12:38:22,121] jepsen worker 0 - jepsen.util 18360 :invoke :read [0 nil] INFO [2019-12-17 12:38:22,124] jepsen worker 80 - jepsen.util 12590 :ok :read [26 25] INFO [2019-12-17 12:38:22,124] jepsen worker 80 - jepsen.util 12590 :invoke :read [26 nil] INFO [2019-12-17 12:38:22,133] jepsen worker 28 - jepsen.util 11098 :fail :cas [9 [35 44]] :ValueNotSwapFrom INFO [2019-12-17 12:38:22,133] jepsen worker 28 - jepsen.util 11098 :invoke :read [9 nil] INFO [2019-12-17 12:38:22,147] jepsen worker 28 - jepsen.util 11098 :ok :read [9 10] INFO [2019-12-17 12:38:22,147] jepsen worker 28 - jepsen.util 11098 :invoke :read [9 nil] INFO [2019-12-17 12:38:22,148] jepsen worker 70 - jepsen.util 13570 :info :write [23 40] :RequestCanceledException INFO [2019-12-17 12:38:22,148] jepsen worker 84 - jepsen.util 17274 :info :write [28 34] :RequestCanceledException INFO [2019-12-17 12:38:22,149] jepsen worker 70 - jepsen.util 13660 :invoke :write [23 10] INFO [2019-12-17 12:38:22,149] jepsen worker 84 - jepsen.util 17364 :invoke :cas [28 [21 31]] INFO [2019-12-17 12:38:22,158] jepsen worker 28 - jepsen.util 11098 :ok :read [9 10] INFO [2019-12-17 12:38:22,158] jepsen worker 28 - jepsen.util 11098 :invoke :read [9 nil] INFO [2019-12-17 12:38:22,170] jepsen worker 28 - jepsen.util 11098 :ok :read [9 10] INFO [2019-12-17 12:38:22,170] jepsen worker 28 - jepsen.util 11098 :invoke :read [9 nil] INFO [2019-12-17 12:38:22,182] jepsen worker 28 - jepsen.util 11098 :ok :read [9 10] INFO [2019-12-17 12:38:22,182] jepsen worker 28 - jepsen.util 11098 :invoke :write [9 22] INFO [2019-12-17 12:38:22,191] jepsen worker 31 - jepsen.util 11191 :ok :write [10 36] INFO [2019-12-17 12:38:22,192] jepsen worker 31 - jepsen.util 11191 :invoke :read [10 nil] INFO [2019-12-17 12:38:22,207] jepsen worker 31 - jepsen.util 11191 :ok :read [10 36] INFO [2019-12-17 12:38:22,208] jepsen worker 31 - jepsen.util 11191 :invoke :read [10 nil] INFO [2019-12-17 12:38:22,222] jepsen worker 18 - jepsen.util 13698 :ok :write [6 40] INFO [2019-12-17 12:38:22,222] jepsen worker 18 - jepsen.util 13698 :invoke :cas [6 [26 40]] INFO [2019-12-17 12:38:22,226] jepsen worker 31 - jepsen.util 11191 :ok :read [10 36] INFO [2019-12-17 12:38:22,226] jepsen worker 31 - jepsen.util 11191 :invoke :write [10 32] INFO [2019-12-17 12:38:22,235] jepsen worker 5 - jepsen.util 12695 :info :write [1 43] :RequestCanceledException INFO [2019-12-17 12:38:22,235] jepsen worker 23 - jepsen.util 18023 :info :write [7 47] :RequestCanceledException INFO [2019-12-17 12:38:22,236] jepsen worker 5 - jepsen.util 12785 :invoke :cas [1 [34 35]] INFO [2019-12-17 12:38:22,236] jepsen worker 23 - jepsen.util 18113 :invoke :cas [7 [18 27]] INFO [2019-12-17 12:38:22,242] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:38:22,242] jepsen nemesis - couchbase.util Rebalance check count: 596 INFO [2019-12-17 12:38:22,242] jepsen nemesis - couchbase.util Rebalance stuck count: 577 INFO [2019-12-17 12:38:22,245] jepsen worker 28 - jepsen.util 11098 :ok :write [9 22] INFO [2019-12-17 12:38:22,245] jepsen worker 74 - jepsen.util 12944 :ok :write [24 39] INFO [2019-12-17 12:38:22,245] jepsen worker 28 - jepsen.util 11098 :invoke :write [9 26] INFO [2019-12-17 12:38:22,245] jepsen worker 74 - jepsen.util 12944 :invoke :cas [24 [25 33]] INFO [2019-12-17 12:38:22,268] jepsen worker 72 - jepsen.util 13212 :fail :cas [24 [24 8]] :ValueNotSwapFrom INFO [2019-12-17 12:38:22,268] jepsen worker 72 - jepsen.util 13212 :invoke :read [24 nil] INFO [2019-12-17 12:38:22,283] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":75770,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:38:22,283] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":75770,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:38:22,284] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":75772,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:38:22,284] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":75772,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:38:22,290] jepsen worker 74 - jepsen.util 12944 :fail :cas [24 [25 33]] :ValueNotSwapFrom INFO [2019-12-17 12:38:22,290] jepsen worker 74 - jepsen.util 12944 :invoke :read [24 nil] INFO [2019-12-17 12:38:22,294] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":75763,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:38:22,294] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":75763,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:38:22,295] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":75764,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:38:22,295] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":75764,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:38:22,295] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":75765,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:38:22,295] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":75765,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:38:22,298] jepsen worker 72 - jepsen.util 13212 :ok :read [24 39] INFO [2019-12-17 12:38:22,298] jepsen worker 72 - jepsen.util 13212 :invoke :cas [24 [33 21]] INFO [2019-12-17 12:38:22,304] jepsen worker 74 - jepsen.util 12944 :ok :read [24 39] INFO [2019-12-17 12:38:22,305] jepsen worker 74 - jepsen.util 12944 :invoke :write [24 41] INFO [2019-12-17 12:38:22,308] jepsen worker 37 - jepsen.util 12727 :fail :cas [12 [26 18]] :ValueNotSwapFrom INFO [2019-12-17 12:38:22,308] jepsen worker 37 - jepsen.util 12727 :invoke :write [12 3] INFO [2019-12-17 12:38:22,312] jepsen worker 72 - jepsen.util 13212 :fail :cas [24 [33 21]] :ValueNotSwapFrom INFO [2019-12-17 12:38:22,312] jepsen worker 72 - jepsen.util 13212 :invoke :write [24 39] INFO [2019-12-17 12:38:22,314] jepsen worker 45 - jepsen.util 12735 :ok :write [15 6] INFO [2019-12-17 12:38:22,315] jepsen worker 45 - jepsen.util 12735 :invoke :read [15 nil] INFO [2019-12-17 12:38:22,316] jepsen worker 28 - jepsen.util 11098 :ok :write [9 26] INFO [2019-12-17 12:38:22,316] jepsen worker 28 - jepsen.util 11098 :invoke :write [9 23] INFO [2019-12-17 12:38:22,317] jepsen worker 69 - jepsen.util 13389 :info :cas [23 [13 17]] :RequestCanceledException INFO [2019-12-17 12:38:22,317] jepsen worker 82 - jepsen.util 17812 :info :cas [27 [24 4]] :RequestCanceledException INFO [2019-12-17 12:38:22,318] jepsen worker 69 - jepsen.util 13479 :invoke :cas [23 [40 20]] INFO [2019-12-17 12:38:22,318] jepsen worker 82 - jepsen.util 17902 :invoke :write [27 16] INFO [2019-12-17 12:38:22,345] jepsen worker 18 - jepsen.util 13698 :fail :cas [6 [26 40]] :ValueNotSwapFrom INFO [2019-12-17 12:38:22,345] jepsen worker 18 - jepsen.util 13698 :invoke :cas [6 [0 17]] INFO [2019-12-17 12:38:22,347] jepsen worker 78 - jepsen.util 12858 :fail :cas [26 [27 14]] :ValueNotSwapFrom INFO [2019-12-17 12:38:22,347] jepsen worker 78 - jepsen.util 12858 :invoke :cas [26 [19 0]] INFO [2019-12-17 12:38:22,354] jepsen worker 45 - jepsen.util 12735 :ok :read [15 6] INFO [2019-12-17 12:38:22,354] jepsen worker 45 - jepsen.util 12735 :invoke :cas [15 [34 21]] INFO [2019-12-17 12:38:22,359] jepsen worker 18 - jepsen.util 13698 :fail :cas [6 [0 17]] :ValueNotSwapFrom INFO [2019-12-17 12:38:22,359] jepsen worker 18 - jepsen.util 13698 :invoke :write [6 3] INFO [2019-12-17 12:38:22,365] jepsen worker 45 - jepsen.util 12735 :fail :cas [15 [34 21]] :ValueNotSwapFrom INFO [2019-12-17 12:38:22,366] jepsen worker 45 - jepsen.util 12735 :invoke :write [15 22] INFO [2019-12-17 12:38:22,384] jepsen worker 28 - jepsen.util 11098 :ok :write [9 23] INFO [2019-12-17 12:38:22,385] jepsen worker 28 - jepsen.util 11098 :invoke :cas [9 [24 47]] INFO [2019-12-17 12:38:22,410] jepsen worker 28 - jepsen.util 11098 :fail :cas [9 [24 47]] :ValueNotSwapFrom INFO [2019-12-17 12:38:22,410] jepsen worker 28 - jepsen.util 11098 :invoke :read [9 nil] INFO [2019-12-17 12:38:22,437] jepsen worker 28 - jepsen.util 11098 :ok :read [9 23] INFO [2019-12-17 12:38:22,438] jepsen worker 28 - jepsen.util 11098 :invoke :read [9 nil] INFO [2019-12-17 12:38:22,452] jepsen worker 28 - jepsen.util 11098 :ok :read [9 23] INFO [2019-12-17 12:38:22,453] jepsen worker 28 - jepsen.util 11098 :invoke :write [9 4] INFO [2019-12-17 12:38:22,465] jepsen worker 14 - jepsen.util 12614 :info :write [4 34] :RequestCanceledException INFO [2019-12-17 12:38:22,466] jepsen worker 14 - jepsen.util 12704 :invoke :write [4 42] INFO [2019-12-17 12:38:22,514] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":75800,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:38:22,514] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":75800,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:38:22,514] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":75803,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:38:22,514] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":75803,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:38:22,514] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":75804,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:38:22,514] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":75804,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:38:22,515] jepsen worker 78 - jepsen.util 12858 :fail :cas [26 [19 0]] :ValueNotSwapFrom INFO [2019-12-17 12:38:22,515] jepsen worker 78 - jepsen.util 12858 :invoke :read [26 nil] INFO [2019-12-17 12:38:22,541] jepsen worker 28 - jepsen.util 11098 :ok :write [9 4] INFO [2019-12-17 12:38:22,542] jepsen worker 28 - jepsen.util 11098 :invoke :write [9 49] INFO [2019-12-17 12:38:22,544] jepsen worker 78 - jepsen.util 12858 :ok :read [26 25] INFO [2019-12-17 12:38:22,544] jepsen worker 78 - jepsen.util 12858 :invoke :write [26 37] INFO [2019-12-17 12:38:22,549] jepsen worker 77 - jepsen.util 19067 :info :cas [25 [15 35]] :RequestCanceledException INFO [2019-12-17 12:38:22,552] jepsen worker 77 - jepsen.util 19157 :invoke :cas [25 [32 41]] INFO [2019-12-17 12:38:22,584] jepsen worker 28 - jepsen.util 11098 :ok :write [9 49] INFO [2019-12-17 12:38:22,584] jepsen worker 28 - jepsen.util 11098 :invoke :read [9 nil] INFO [2019-12-17 12:38:22,628] jepsen worker 28 - jepsen.util 11098 :ok :read [9 49] INFO [2019-12-17 12:38:22,628] jepsen worker 28 - jepsen.util 11098 :invoke :read [9 nil] INFO [2019-12-17 12:38:22,634] jepsen worker 79 - jepsen.util 13219 :info :write [26 3] :RequestCanceledException INFO [2019-12-17 12:38:22,635] jepsen worker 52 - jepsen.util 12742 :ok :write [17 13] INFO [2019-12-17 12:38:22,635] jepsen worker 52 - jepsen.util 12742 :invoke :write [17 24] INFO [2019-12-17 12:38:22,636] jepsen worker 79 - jepsen.util 13309 :invoke :cas [26 [37 40]] INFO [2019-12-17 12:38:22,653] jepsen worker 28 - jepsen.util 11098 :ok :read [9 49] INFO [2019-12-17 12:38:22,654] jepsen worker 28 - jepsen.util 11098 :invoke :cas [9 [4 42]] INFO [2019-12-17 12:38:22,666] jepsen worker 55 - jepsen.util 13825 :info :write [18 39] :RequestCanceledException INFO [2019-12-17 12:38:22,667] jepsen worker 55 - jepsen.util 13915 :invoke :read [18 nil] INFO [2019-12-17 12:38:22,668] jepsen worker 28 - jepsen.util 11098 :fail :cas [9 [4 42]] :ValueNotSwapFrom INFO [2019-12-17 12:38:22,668] jepsen worker 28 - jepsen.util 11098 :invoke :cas [9 [42 41]] INFO [2019-12-17 12:38:22,686] jepsen worker 28 - jepsen.util 11098 :fail :cas [9 [42 41]] :ValueNotSwapFrom INFO [2019-12-17 12:38:22,686] jepsen worker 28 - jepsen.util 11098 :invoke :write [9 26] INFO [2019-12-17 12:38:22,701] jepsen worker 2 - jepsen.util 17102 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:38:22,701] jepsen worker 15 - jepsen.util 17025 :info :write [5 7] :RequestCanceledException INFO [2019-12-17 12:38:22,701] jepsen worker 2 - jepsen.util 17102 :invoke :cas [0 [13 22]] INFO [2019-12-17 12:38:22,703] jepsen worker 15 - jepsen.util 17115 :invoke :write [5 35] INFO [2019-12-17 12:38:22,719] jepsen worker 16 - jepsen.util 19456 :info :cas [5 [2 2]] :RequestCanceledException INFO [2019-12-17 12:38:22,720] jepsen worker 16 - jepsen.util 19546 :invoke :cas [5 [14 29]] INFO [2019-12-17 12:38:22,723] jepsen worker 5 - jepsen.util 12785 :fail :cas [1 [34 35]] :ValueNotSwapFrom INFO [2019-12-17 12:38:22,723] jepsen worker 5 - jepsen.util 12785 :invoke :cas [1 [35 2]] INFO [2019-12-17 12:38:22,730] jepsen worker 28 - jepsen.util 11098 :ok :write [9 26] INFO [2019-12-17 12:38:22,730] jepsen worker 28 - jepsen.util 11098 :invoke :read [9 nil] INFO [2019-12-17 12:38:22,734] jepsen worker 44 - jepsen.util 18584 :info :cas [14 [22 47]] :RequestCanceledException INFO [2019-12-17 12:38:22,734] jepsen worker 35 - jepsen.util 17945 :info :cas [11 [25 46]] :RequestCanceledException INFO [2019-12-17 12:38:22,734] jepsen worker 54 - jepsen.util 13734 :info :write [18 45] :RequestCanceledException INFO [2019-12-17 12:38:22,734] jepsen worker 81 - jepsen.util 17361 :info :write [27 39] :RequestCanceledException INFO [2019-12-17 12:38:22,734] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":75817,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:38:22,735] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":75817,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:38:22,735] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":75821,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:38:22,735] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":75821,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:38:22,735] jepsen worker 54 - jepsen.util 13824 :invoke :read [18 nil] INFO [2019-12-17 12:38:22,735] jepsen worker 44 - jepsen.util 18674 :invoke :write [14 37] INFO [2019-12-17 12:38:22,735] jepsen worker 35 - jepsen.util 18035 :invoke :cas [11 [9 32]] INFO [2019-12-17 12:38:22,735] jepsen worker 81 - jepsen.util 17451 :invoke :read [27 nil] INFO [2019-12-17 12:38:22,749] jepsen worker 58 - jepsen.util 18148 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:38:22,749] jepsen worker 58 - jepsen.util 18148 :invoke :cas [19 [28 29]] INFO [2019-12-17 12:38:22,752] jepsen worker 28 - jepsen.util 11098 :ok :read [9 26] INFO [2019-12-17 12:38:22,752] jepsen worker 28 - jepsen.util 11098 :invoke :write [9 18] INFO [2019-12-17 12:38:22,757] jepsen worker 31 - jepsen.util 11191 :ok :write [10 32] INFO [2019-12-17 12:38:22,757] jepsen worker 31 - jepsen.util 11191 :invoke :write [10 8] INFO [2019-12-17 12:38:22,758] jepsen worker 74 - jepsen.util 12944 :ok :write [24 41] INFO [2019-12-17 12:38:22,758] jepsen worker 74 - jepsen.util 12944 :invoke :read [24 nil] INFO [2019-12-17 12:38:22,765] jepsen worker 43 - jepsen.util 18133 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:38:22,766] jepsen worker 43 - jepsen.util 18133 :invoke :write [14 17] INFO [2019-12-17 12:38:22,789] jepsen worker 18 - jepsen.util 13698 :ok :write [6 3] INFO [2019-12-17 12:38:22,789] jepsen worker 18 - jepsen.util 13698 :invoke :read [6 nil] INFO [2019-12-17 12:38:22,801] jepsen worker 25 - jepsen.util 19015 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:38:22,801] jepsen worker 25 - jepsen.util 19015 :invoke :cas [8 [36 27]] INFO [2019-12-17 12:38:22,813] jepsen worker 28 - jepsen.util 11098 :ok :write [9 18] INFO [2019-12-17 12:38:22,813] jepsen worker 28 - jepsen.util 11098 :invoke :cas [9 [31 34]] INFO [2019-12-17 12:38:22,818] jepsen worker 9 - jepsen.util 13509 :fail :read [3 nil] :RequestTimeoutException INFO [2019-12-17 12:38:22,819] jepsen worker 9 - jepsen.util 13509 :invoke :cas [3 [23 17]] INFO [2019-12-17 12:38:22,824] jepsen worker 87 - jepsen.util 12687 :fail :cas [29 [29 1]] :ValueNotSwapFrom INFO [2019-12-17 12:38:22,824] jepsen worker 87 - jepsen.util 12687 :invoke :cas [29 [20 37]] INFO [2019-12-17 12:38:22,831] jepsen worker 28 - jepsen.util 11098 :fail :cas [9 [31 34]] :ValueNotSwapFrom INFO [2019-12-17 12:38:22,832] jepsen worker 28 - jepsen.util 11098 :invoke :read [9 nil] INFO [2019-12-17 12:38:22,835] jepsen worker 53 - jepsen.util 12563 :info :write [17 6] :RequestCanceledException INFO [2019-12-17 12:38:22,835] jepsen worker 48 - jepsen.util 16608 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:38:22,835] jepsen worker 19 - jepsen.util 13879 :info :cas [6 [14 22]] :RequestCanceledException INFO [2019-12-17 12:38:22,835] jepsen worker 48 - jepsen.util 16608 :invoke :write [16 21] INFO [2019-12-17 12:38:22,836] jepsen worker 53 - jepsen.util 12653 :invoke :write [17 3] INFO [2019-12-17 12:38:22,836] jepsen worker 19 - jepsen.util 13969 :invoke :read [6 nil] INFO [2019-12-17 12:38:22,845] jepsen worker 28 - jepsen.util 11098 :ok :read [9 18] INFO [2019-12-17 12:38:22,845] jepsen worker 28 - jepsen.util 11098 :invoke :cas [9 [25 15]] INFO [2019-12-17 12:38:22,848] jepsen worker 50 - jepsen.util 17780 :info :write [16 22] :RequestCanceledException INFO [2019-12-17 12:38:22,851] jepsen worker 50 - jepsen.util 17870 :invoke :read [16 nil] INFO [2019-12-17 12:38:22,866] jepsen worker 47 - jepsen.util 14357 :info :write [15 48] :RequestCanceledException INFO [2019-12-17 12:38:22,868] jepsen worker 28 - jepsen.util 11098 :fail :cas [9 [25 15]] :ValueNotSwapFrom INFO [2019-12-17 12:38:22,868] jepsen worker 28 - jepsen.util 11098 :invoke :read [9 nil] INFO [2019-12-17 12:38:22,870] jepsen worker 47 - jepsen.util 14447 :invoke :cas [15 [29 21]] INFO [2019-12-17 12:38:22,887] jepsen worker 28 - jepsen.util 11098 :ok :read [9 18] INFO [2019-12-17 12:38:22,887] jepsen worker 28 - jepsen.util 11098 :invoke :write [9 27] INFO [2019-12-17 12:38:22,901] jepsen worker 57 - jepsen.util 17427 :info :cas [19 [4 23]] :RequestCanceledException INFO [2019-12-17 12:38:22,901] jepsen worker 7 - jepsen.util 17557 :info :cas [2 [46 21]] :RequestCanceledException INFO [2019-12-17 12:38:22,901] jepsen worker 51 - jepsen.util 12291 :info :write [17 46] :RequestCanceledException INFO [2019-12-17 12:38:22,902] jepsen worker 51 - jepsen.util 12381 :invoke :read [17 nil] INFO [2019-12-17 12:38:22,902] jepsen worker 57 - jepsen.util 17517 :invoke :read [19 nil] INFO [2019-12-17 12:38:22,902] jepsen worker 7 - jepsen.util 17647 :invoke :cas [2 [14 18]] INFO [2019-12-17 12:38:22,919] jepsen worker 8 - jepsen.util 17378 :info :write [2 36] :RequestCanceledException INFO [2019-12-17 12:38:22,922] jepsen worker 8 - jepsen.util 17468 :invoke :cas [2 [41 38]] INFO [2019-12-17 12:38:22,934] jepsen worker 61 - jepsen.util 19051 :info :cas [20 [34 3]] :RequestCanceledException INFO [2019-12-17 12:38:22,934] jepsen worker 73 - jepsen.util 12133 :info :write [24 39] :RequestCanceledException INFO [2019-12-17 12:38:22,935] jepsen worker 61 - jepsen.util 19141 :invoke :write [20 39] INFO [2019-12-17 12:38:22,938] jepsen worker 73 - jepsen.util 12223 :invoke :write [24 2] INFO [2019-12-17 12:38:22,951] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":75855,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:38:22,951] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":75855,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:38:22,951] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":75857,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:38:22,952] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":75857,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:38:22,961] jepsen worker 28 - jepsen.util 11098 :ok :write [9 27] INFO [2019-12-17 12:38:22,962] jepsen worker 28 - jepsen.util 11098 :invoke :read [9 nil] INFO [2019-12-17 12:38:22,983] jepsen worker 31 - jepsen.util 11191 :ok :write [10 8] INFO [2019-12-17 12:38:22,984] jepsen worker 31 - jepsen.util 11191 :invoke :write [10 23] INFO [2019-12-17 12:38:22,988] jepsen worker 28 - jepsen.util 11098 :ok :read [9 27] INFO [2019-12-17 12:38:22,988] jepsen worker 28 - jepsen.util 11098 :invoke :cas [9 [35 7]] INFO [2019-12-17 12:38:23,019] jepsen worker 59 - jepsen.util 16979 :info :cas [19 [18 1]] :RequestCanceledException INFO [2019-12-17 12:38:23,019] jepsen worker 59 - jepsen.util 17069 :invoke :read [19 nil] INFO [2019-12-17 12:38:23,035] jepsen worker 11 - jepsen.util 12971 :info :write [3 26] :RequestCanceledException INFO [2019-12-17 12:38:23,035] jepsen worker 30 - jepsen.util 12450 :info :cas [10 [6 22]] :RequestCanceledException INFO [2019-12-17 12:38:23,035] jepsen worker 83 - jepsen.util 17813 :info :cas [27 [42 38]] :RequestCanceledException INFO [2019-12-17 12:38:23,035] jepsen worker 71 - jepsen.util 13931 :info :write [23 37] :RequestCanceledException INFO [2019-12-17 12:38:23,036] jepsen worker 11 - jepsen.util 13061 :invoke :read [3 nil] INFO [2019-12-17 12:38:23,036] jepsen worker 30 - jepsen.util 12540 :invoke :read [10 nil] INFO [2019-12-17 12:38:23,036] jepsen worker 83 - jepsen.util 17903 :invoke :write [27 6] INFO [2019-12-17 12:38:23,036] jepsen worker 71 - jepsen.util 14021 :invoke :cas [23 [3 44]] INFO [2019-12-17 12:38:23,037] jepsen worker 9 - jepsen.util 13509 :fail :cas [3 [23 17]] :ValueNotSwapFrom INFO [2019-12-17 12:38:23,037] jepsen worker 9 - jepsen.util 13509 :invoke :cas [3 [26 38]] INFO [2019-12-17 12:38:23,048] jepsen worker 42 - jepsen.util 18762 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:38:23,049] jepsen worker 42 - jepsen.util 18762 :invoke :write [14 28] INFO [2019-12-17 12:38:23,050] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":75870,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:38:23,050] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":75870,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:38:23,050] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":75873,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:38:23,050] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":75873,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:38:23,050] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":75874,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:38:23,050] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":75874,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:38:23,066] jepsen worker 39 - jepsen.util 17499 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:38:23,066] jepsen worker 39 - jepsen.util 17499 :invoke :read [13 nil] INFO [2019-12-17 12:38:23,070] jepsen worker 80 - jepsen.util 12590 :ok :read [26 25] INFO [2019-12-17 12:38:23,070] jepsen worker 80 - jepsen.util 12590 :invoke :cas [26 [9 5]] INFO [2019-12-17 12:38:23,086] jepsen worker 19 - jepsen.util 13969 :ok :read [6 3] INFO [2019-12-17 12:38:23,086] jepsen worker 19 - jepsen.util 13969 :invoke :read [6 nil] INFO [2019-12-17 12:38:23,087] jepsen worker 28 - jepsen.util 11098 :fail :cas [9 [35 7]] :ValueNotSwapFrom INFO [2019-12-17 12:38:23,087] jepsen worker 28 - jepsen.util 11098 :invoke :write [9 4] INFO [2019-12-17 12:38:23,096] jepsen worker 69 - jepsen.util 13479 :fail :cas [23 [40 20]] :ValueNotSwapFrom INFO [2019-12-17 12:38:23,096] jepsen worker 69 - jepsen.util 13479 :invoke :cas [23 [16 0]] INFO [2019-12-17 12:38:23,101] jepsen worker 63 - jepsen.util 11583 :info :cas [21 [0 2]] :RequestCanceledException INFO [2019-12-17 12:38:23,101] jepsen worker 88 - jepsen.util 13048 :info :write [29 28] :RequestCanceledException INFO [2019-12-17 12:38:23,102] jepsen worker 88 - jepsen.util 13138 :invoke :write [29 19] INFO [2019-12-17 12:38:23,106] jepsen worker 63 - jepsen.util 11673 :invoke :read [21 nil] INFO [2019-12-17 12:38:23,112] jepsen worker 11 - jepsen.util 13061 :ok :read [3 9] INFO [2019-12-17 12:38:23,113] jepsen worker 11 - jepsen.util 13061 :invoke :cas [3 [23 23]] INFO [2019-12-17 12:38:23,135] jepsen worker 1 - jepsen.util 18271 :info :cas [0 [11 6]] :RequestCanceledException INFO [2019-12-17 12:38:23,137] jepsen worker 1 - jepsen.util 18361 :invoke :cas [0 [37 19]] INFO [2019-12-17 12:38:23,139] jepsen worker 11 - jepsen.util 13061 :fail :cas [3 [23 23]] :ValueNotSwapFrom INFO [2019-12-17 12:38:23,140] jepsen worker 11 - jepsen.util 13061 :invoke :write [3 12] INFO [2019-12-17 12:38:23,144] jepsen worker 54 - jepsen.util 13824 :ok :read [18 48] INFO [2019-12-17 12:38:23,144] jepsen worker 54 - jepsen.util 13824 :invoke :read [18 nil] INFO [2019-12-17 12:38:23,148] jepsen worker 65 - jepsen.util 11405 :info :write [21 33] :RequestCanceledException INFO [2019-12-17 12:38:23,149] jepsen worker 65 - jepsen.util 11495 :invoke :cas [21 [23 28]] INFO [2019-12-17 12:38:23,168] jepsen worker 72 - jepsen.util 13212 :ok :write [24 39] INFO [2019-12-17 12:38:23,168] jepsen worker 72 - jepsen.util 13212 :invoke :write [24 48] INFO [2019-12-17 12:38:23,170] jepsen worker 45 - jepsen.util 12735 :ok :write [15 22] INFO [2019-12-17 12:38:23,170] jepsen worker 45 - jepsen.util 12735 :invoke :write [15 24] INFO [2019-12-17 12:38:23,176] jepsen worker 63 - jepsen.util 11673 :ok :read [21 12] INFO [2019-12-17 12:38:23,177] jepsen worker 63 - jepsen.util 11673 :invoke :write [21 43] INFO [2019-12-17 12:38:23,192] jepsen worker 31 - jepsen.util 11191 :ok :write [10 23] INFO [2019-12-17 12:38:23,192] jepsen worker 18 - jepsen.util 13698 :ok :read [6 3] INFO [2019-12-17 12:38:23,192] jepsen worker 31 - jepsen.util 11191 :invoke :read [10 nil] INFO [2019-12-17 12:38:23,192] jepsen worker 18 - jepsen.util 13698 :invoke :write [6 35] INFO [2019-12-17 12:38:23,199] jepsen worker 64 - jepsen.util 10234 :info :write [21 36] :RequestCanceledException INFO [2019-12-17 12:38:23,200] jepsen worker 64 - jepsen.util 10324 :invoke :write [21 31] INFO [2019-12-17 12:38:23,201] jepsen worker 36 - jepsen.util 13806 :ok :write [12 46] INFO [2019-12-17 12:38:23,201] jepsen worker 36 - jepsen.util 13806 :invoke :read [12 nil] INFO [2019-12-17 12:38:23,203] jepsen worker 69 - jepsen.util 13479 :fail :cas [23 [16 0]] :ValueNotSwapFrom INFO [2019-12-17 12:38:23,203] jepsen worker 69 - jepsen.util 13479 :invoke :write [23 9] INFO [2019-12-17 12:38:23,219] jepsen worker 46 - jepsen.util 12646 :info :write [15 14] :RequestCanceledException INFO [2019-12-17 12:38:23,219] jepsen worker 76 - jepsen.util 18076 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:38:23,219] jepsen worker 10 - jepsen.util 12970 :fail :read [3 nil] :RequestTimeoutException INFO [2019-12-17 12:38:23,219] jepsen worker 76 - jepsen.util 18076 :invoke :cas [25 [10 28]] INFO [2019-12-17 12:38:23,219] jepsen worker 10 - jepsen.util 12970 :invoke :cas [3 [2 27]] INFO [2019-12-17 12:38:23,221] jepsen worker 46 - jepsen.util 12736 :invoke :cas [15 [46 48]] INFO [2019-12-17 12:38:23,229] jepsen worker 47 - jepsen.util 14447 :fail :cas [15 [29 21]] :ValueNotSwapFrom INFO [2019-12-17 12:38:23,229] jepsen worker 47 - jepsen.util 14447 :invoke :read [15 nil] INFO [2019-12-17 12:38:23,234] jepsen worker 68 - jepsen.util 17078 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:38:23,234] jepsen worker 22 - jepsen.util 18292 :info :write [7 3] :RequestCanceledException INFO [2019-12-17 12:38:23,234] jepsen worker 68 - jepsen.util 17078 :invoke :cas [22 [18 27]] INFO [2019-12-17 12:38:23,235] jepsen worker 30 - jepsen.util 12540 :ok :read [10 8] INFO [2019-12-17 12:38:23,235] jepsen worker 30 - jepsen.util 12540 :invoke :read [10 nil] INFO [2019-12-17 12:38:23,235] jepsen worker 22 - jepsen.util 18382 :invoke :cas [7 [5 5]] INFO [2019-12-17 12:38:23,239] jepsen worker 28 - jepsen.util 11098 :ok :write [9 4] INFO [2019-12-17 12:38:23,239] jepsen worker 28 - jepsen.util 11098 :invoke :read [9 nil] INFO [2019-12-17 12:38:23,244] jepsen worker 12 - jepsen.util 13242 :ok :write [4 35] INFO [2019-12-17 12:38:23,244] jepsen worker 65 - jepsen.util 11495 :fail :cas [21 [23 28]] :ValueNotSwapFrom INFO [2019-12-17 12:38:23,244] jepsen worker 12 - jepsen.util 13242 :invoke :read [4 nil] INFO [2019-12-17 12:38:23,244] jepsen worker 65 - jepsen.util 11495 :invoke :write [21 24] INFO [2019-12-17 12:38:23,255] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:38:23,256] jepsen nemesis - couchbase.util Rebalance check count: 597 INFO [2019-12-17 12:38:23,256] jepsen nemesis - couchbase.util Rebalance stuck count: 578 INFO [2019-12-17 12:38:23,260] jepsen worker 46 - jepsen.util 12736 :fail :cas [15 [46 48]] :ValueNotSwapFrom INFO [2019-12-17 12:38:23,260] jepsen worker 46 - jepsen.util 12736 :invoke :cas [15 [36 3]] INFO [2019-12-17 12:38:23,266] jepsen worker 71 - jepsen.util 14021 :fail :cas [23 [3 44]] :ValueNotSwapFrom INFO [2019-12-17 12:38:23,266] jepsen worker 89 - jepsen.util 12959 :info :write [29 13] :RequestCanceledException INFO [2019-12-17 12:38:23,266] jepsen worker 71 - jepsen.util 14021 :invoke :read [23 nil] INFO [2019-12-17 12:38:23,267] jepsen worker 89 - jepsen.util 13049 :invoke :write [29 10] INFO [2019-12-17 12:38:23,269] jepsen worker 87 - jepsen.util 12687 :fail :cas [29 [20 37]] :ValueNotSwapFrom INFO [2019-12-17 12:38:23,269] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":75911,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:38:23,269] jepsen worker 87 - jepsen.util 12687 :invoke :cas [29 [5 29]] INFO [2019-12-17 12:38:23,269] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":75911,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:38:23,270] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":75914,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:38:23,270] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":75914,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:38:23,274] jepsen worker 28 - jepsen.util 11098 :ok :read [9 4] INFO [2019-12-17 12:38:23,274] jepsen worker 28 - jepsen.util 11098 :invoke :write [9 46] INFO [2019-12-17 12:38:23,274] jepsen worker 19 - jepsen.util 13969 :ok :read [6 3] INFO [2019-12-17 12:38:23,274] jepsen worker 19 - jepsen.util 13969 :invoke :write [6 45] INFO [2019-12-17 12:38:23,275] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":75900,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:38:23,275] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":75900,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:38:23,275] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":75901,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:38:23,275] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":75901,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:38:23,300] jepsen worker 32 - jepsen.util 13442 :fail :read [10 nil] :RequestTimeoutException INFO [2019-12-17 12:38:23,301] jepsen worker 32 - jepsen.util 13442 :invoke :write [10 33] INFO [2019-12-17 12:38:23,304] jepsen worker 79 - jepsen.util 13309 :fail :cas [26 [37 40]] :ValueNotSwapFrom INFO [2019-12-17 12:38:23,305] jepsen worker 79 - jepsen.util 13309 :invoke :write [26 19] INFO [2019-12-17 12:38:23,319] jepsen worker 6 - jepsen.util 18186 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:38:23,319] jepsen worker 6 - jepsen.util 18186 :invoke :write [2 31] INFO [2019-12-17 12:38:23,338] jepsen worker 5 - jepsen.util 12785 :fail :cas [1 [35 2]] :ValueNotSwapFrom INFO [2019-12-17 12:38:23,338] jepsen worker 5 - jepsen.util 12785 :invoke :cas [1 [40 20]] INFO [2019-12-17 12:38:23,345] jepsen worker 78 - jepsen.util 12858 :ok :write [26 37] INFO [2019-12-17 12:38:23,345] jepsen worker 31 - jepsen.util 11191 :ok :read [10 23] INFO [2019-12-17 12:38:23,345] jepsen worker 31 - jepsen.util 11191 :invoke :read [10 nil] INFO [2019-12-17 12:38:23,345] jepsen worker 78 - jepsen.util 12858 :invoke :write [26 23] INFO [2019-12-17 12:38:23,354] jepsen worker 46 - jepsen.util 12736 :fail :cas [15 [36 3]] :ValueNotSwapFrom INFO [2019-12-17 12:38:23,355] jepsen worker 46 - jepsen.util 12736 :invoke :cas [15 [48 18]] INFO [2019-12-17 12:38:23,366] jepsen worker 34 - jepsen.util 18934 :info :cas [11 [39 15]] :RequestCanceledException INFO [2019-12-17 12:38:23,369] jepsen worker 34 - jepsen.util 19024 :invoke :read [11 nil] INFO [2019-12-17 12:38:23,375] jepsen worker 36 - jepsen.util 13806 :ok :read [12 46] INFO [2019-12-17 12:38:23,375] jepsen worker 36 - jepsen.util 13806 :invoke :write [12 32] INFO [2019-12-17 12:38:23,379] jepsen worker 56 - jepsen.util 13826 :ok :write [18 29] INFO [2019-12-17 12:38:23,379] jepsen worker 56 - jepsen.util 13826 :invoke :read [18 nil] INFO [2019-12-17 12:38:23,382] jepsen worker 64 - jepsen.util 10324 :ok :write [21 31] INFO [2019-12-17 12:38:23,382] jepsen worker 64 - jepsen.util 10324 :invoke :cas [21 [1 42]] INFO [2019-12-17 12:38:23,401] jepsen worker 13 - jepsen.util 12613 :info :write [4 15] :RequestCanceledException INFO [2019-12-17 12:38:23,402] jepsen worker 13 - jepsen.util 12703 :invoke :read [4 nil] INFO [2019-12-17 12:38:23,420] jepsen worker 29 - jepsen.util 11369 :fail :read [9 nil] :RequestTimeoutException INFO [2019-12-17 12:38:23,420] jepsen worker 29 - jepsen.util 11369 :invoke :cas [9 [38 31]] INFO [2019-12-17 12:38:23,444] jepsen worker 37 - jepsen.util 12727 :ok :write [12 3] INFO [2019-12-17 12:38:23,444] jepsen worker 37 - jepsen.util 12727 :invoke :read [12 nil] INFO [2019-12-17 12:38:23,448] jepsen worker 63 - jepsen.util 11673 :ok :write [21 43] INFO [2019-12-17 12:38:23,448] jepsen worker 63 - jepsen.util 11673 :invoke :write [21 47] INFO [2019-12-17 12:38:23,452] jepsen worker 28 - jepsen.util 11098 :ok :write [9 46] INFO [2019-12-17 12:38:23,452] jepsen worker 28 - jepsen.util 11098 :invoke :write [9 32] INFO [2019-12-17 12:38:23,480] jepsen worker 80 - jepsen.util 12590 :fail :cas [26 [9 5]] :ValueNotSwapFrom INFO [2019-12-17 12:38:23,480] jepsen worker 80 - jepsen.util 12590 :invoke :read [26 nil] INFO [2019-12-17 12:38:23,481] jepsen worker 51 - jepsen.util 12381 :ok :read [17 13] INFO [2019-12-17 12:38:23,481] jepsen worker 51 - jepsen.util 12381 :invoke :read [17 nil] INFO [2019-12-17 12:38:23,494] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":75944,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:38:23,494] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":75944,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:38:23,495] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":75947,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:38:23,495] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":75947,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:38:23,502] jepsen worker 49 - jepsen.util 17869 :info :cas [16 [43 45]] :RequestCanceledException INFO [2019-12-17 12:38:23,502] jepsen worker 40 - jepsen.util 18670 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:38:23,502] jepsen worker 40 - jepsen.util 18670 :invoke :write [13 21] INFO [2019-12-17 12:38:23,503] jepsen worker 49 - jepsen.util 17959 :invoke :write [16 1] INFO [2019-12-17 12:38:23,519] jepsen worker 41 - jepsen.util 19211 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:38:23,519] jepsen worker 4 - jepsen.util 12154 :info :write [1 45] :RequestCanceledException INFO [2019-12-17 12:38:23,520] jepsen worker 41 - jepsen.util 19211 :invoke :cas [13 [10 15]] INFO [2019-12-17 12:38:23,520] jepsen worker 4 - jepsen.util 12244 :invoke :write [1 39] INFO [2019-12-17 12:38:23,535] jepsen worker 24 - jepsen.util 16944 :info :write [8 14] :RequestCanceledException INFO [2019-12-17 12:38:23,537] jepsen worker 24 - jepsen.util 17034 :invoke :write [8 47] INFO [2019-12-17 12:38:23,538] jepsen worker 12 - jepsen.util 13242 :ok :read [4 35] INFO [2019-12-17 12:38:23,538] jepsen worker 12 - jepsen.util 13242 :invoke :read [4 nil] INFO [2019-12-17 12:38:23,540] jepsen worker 63 - jepsen.util 11673 :ok :write [21 47] INFO [2019-12-17 12:38:23,540] jepsen worker 63 - jepsen.util 11673 :invoke :read [21 nil] INFO [2019-12-17 12:38:23,547] jepsen worker 55 - jepsen.util 13915 :ok :read [18 29] INFO [2019-12-17 12:38:23,547] jepsen worker 55 - jepsen.util 13915 :invoke :cas [18 [14 3]] INFO [2019-12-17 12:38:23,561] jepsen worker 63 - jepsen.util 11673 :ok :read [21 47] INFO [2019-12-17 12:38:23,561] jepsen worker 19 - jepsen.util 13969 :ok :write [6 45] INFO [2019-12-17 12:38:23,562] jepsen worker 11 - jepsen.util 13061 :ok :write [3 12] INFO [2019-12-17 12:38:23,562] jepsen worker 63 - jepsen.util 11673 :invoke :read [21 nil] INFO [2019-12-17 12:38:23,562] jepsen worker 19 - jepsen.util 13969 :invoke :read [6 nil] INFO [2019-12-17 12:38:23,562] jepsen worker 11 - jepsen.util 13061 :invoke :read [3 nil] INFO [2019-12-17 12:38:23,562] jepsen worker 73 - jepsen.util 12223 :ok :write [24 2] INFO [2019-12-17 12:38:23,562] jepsen worker 73 - jepsen.util 12223 :invoke :read [24 nil] INFO [2019-12-17 12:38:23,570] jepsen worker 28 - jepsen.util 11098 :ok :write [9 32] INFO [2019-12-17 12:38:23,570] jepsen worker 28 - jepsen.util 11098 :invoke :read [9 nil] INFO [2019-12-17 12:38:23,580] jepsen worker 63 - jepsen.util 11673 :ok :read [21 47] INFO [2019-12-17 12:38:23,581] jepsen worker 63 - jepsen.util 11673 :invoke :write [21 38] INFO [2019-12-17 12:38:23,582] jepsen worker 32 - jepsen.util 13442 :ok :write [10 33] INFO [2019-12-17 12:38:23,582] jepsen worker 32 - jepsen.util 13442 :invoke :read [10 nil] INFO [2019-12-17 12:38:23,593] jepsen worker 56 - jepsen.util 13826 :ok :read [18 29] INFO [2019-12-17 12:38:23,594] jepsen worker 56 - jepsen.util 13826 :invoke :cas [18 [40 49]] INFO [2019-12-17 12:38:23,639] jepsen worker 87 - jepsen.util 12687 :fail :cas [29 [5 29]] :ValueNotSwapFrom INFO [2019-12-17 12:38:23,639] jepsen worker 87 - jepsen.util 12687 :invoke :write [29 19] INFO [2019-12-17 12:38:23,646] jepsen worker 56 - jepsen.util 13826 :fail :cas [18 [40 49]] :ValueNotSwapFrom INFO [2019-12-17 12:38:23,646] jepsen worker 56 - jepsen.util 13826 :invoke :cas [18 [17 32]] INFO [2019-12-17 12:38:23,666] jepsen worker 86 - jepsen.util 18626 :info :cas [28 [36 25]] :RequestCanceledException INFO [2019-12-17 12:38:23,667] jepsen worker 86 - jepsen.util 18716 :invoke :write [28 22] INFO [2019-12-17 12:38:23,676] jepsen worker 13 - jepsen.util 12703 :ok :read [4 35] INFO [2019-12-17 12:38:23,677] jepsen worker 13 - jepsen.util 12703 :invoke :cas [4 [8 40]] INFO [2019-12-17 12:38:23,686] jepsen worker 56 - jepsen.util 13826 :fail :cas [18 [17 32]] :ValueNotSwapFrom INFO [2019-12-17 12:38:23,687] jepsen worker 56 - jepsen.util 13826 :invoke :read [18 nil] INFO [2019-12-17 12:38:23,701] jepsen worker 75 - jepsen.util 17625 :info :write [25 47] :RequestCanceledException INFO [2019-12-17 12:38:23,702] jepsen worker 75 - jepsen.util 17715 :invoke :write [25 6] INFO [2019-12-17 12:38:23,709] jepsen worker 28 - jepsen.util 11098 :ok :read [9 32] INFO [2019-12-17 12:38:23,710] jepsen worker 28 - jepsen.util 11098 :invoke :cas [9 [21 41]] INFO [2019-12-17 12:38:23,713] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":75974,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:38:23,714] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":75974,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:38:23,719] jepsen worker 17 - jepsen.util 18017 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:38:23,719] jepsen worker 17 - jepsen.util 18017 :invoke :read [5 nil] INFO [2019-12-17 12:38:23,719] jepsen worker 80 - jepsen.util 12590 :ok :read [26 37] INFO [2019-12-17 12:38:23,720] jepsen worker 80 - jepsen.util 12590 :invoke :read [26 nil] INFO [2019-12-17 12:38:23,727] jepsen worker 62 - jepsen.util 18512 :info :cas [20 [33 29]] :RequestCanceledException INFO [2019-12-17 12:38:23,728] jepsen worker 11 - jepsen.util 13061 :ok :read [3 12] INFO [2019-12-17 12:38:23,728] jepsen worker 11 - jepsen.util 13061 :invoke :write [3 49] INFO [2019-12-17 12:38:23,729] jepsen worker 62 - jepsen.util 18602 :invoke :write [20 31] INFO [2019-12-17 12:38:23,731] jepsen worker 29 - jepsen.util 11369 :fail :cas [9 [38 31]] :ValueNotSwapFrom INFO [2019-12-17 12:38:23,732] jepsen worker 29 - jepsen.util 11369 :invoke :write [9 44] INFO [2019-12-17 12:38:23,752] jepsen worker 32 - jepsen.util 13442 :ok :read [10 33] INFO [2019-12-17 12:38:23,752] jepsen worker 32 - jepsen.util 13442 :invoke :cas [10 [29 5]] INFO [2019-12-17 12:38:23,765] jepsen worker 26 - jepsen.util 18836 :info :cas [8 [36 10]] :RequestCanceledException INFO [2019-12-17 12:38:23,766] jepsen worker 26 - jepsen.util 18926 :invoke :write [8 1] INFO [2019-12-17 12:38:23,778] jepsen worker 5 - jepsen.util 12785 :fail :cas [1 [40 20]] :ValueNotSwapFrom INFO [2019-12-17 12:38:23,778] jepsen worker 5 - jepsen.util 12785 :invoke :cas [1 [1 0]] INFO [2019-12-17 12:38:23,798] jepsen worker 37 - jepsen.util 12727 :ok :read [12 3] INFO [2019-12-17 12:38:23,798] jepsen worker 37 - jepsen.util 12727 :invoke :cas [12 [12 17]] INFO [2019-12-17 12:38:23,808] jepsen worker 51 - jepsen.util 12381 :ok :read [17 13] INFO [2019-12-17 12:38:23,808] jepsen worker 51 - jepsen.util 12381 :invoke :cas [17 [36 36]] INFO [2019-12-17 12:38:23,833] jepsen worker 60 - jepsen.util 18870 :info :write [20 18] :RequestCanceledException INFO [2019-12-17 12:38:23,833] jepsen worker 27 - jepsen.util 12177 :info :cas [9 [43 12]] :RequestCanceledException INFO [2019-12-17 12:38:23,834] jepsen worker 27 - jepsen.util 12267 :invoke :read [9 nil] INFO [2019-12-17 12:38:23,836] jepsen worker 60 - jepsen.util 18960 :invoke :cas [20 [39 5]] INFO [2019-12-17 12:38:23,848] jepsen worker 38 - jepsen.util 13718 :info :write [12 25] :RequestCanceledException INFO [2019-12-17 12:38:23,849] jepsen worker 38 - jepsen.util 13808 :invoke :write [12 19] INFO [2019-12-17 12:38:23,878] jepsen worker 12 - jepsen.util 13242 :ok :read [4 35] INFO [2019-12-17 12:38:23,878] jepsen worker 12 - jepsen.util 13242 :invoke :read [4 nil] INFO [2019-12-17 12:38:23,890] jepsen worker 28 - jepsen.util 11098 :fail :cas [9 [21 41]] :ValueNotSwapFrom INFO [2019-12-17 12:38:23,891] jepsen worker 28 - jepsen.util 11098 :invoke :cas [9 [44 16]] INFO [2019-12-17 12:38:23,894] jepsen worker 56 - jepsen.util 13826 :ok :read [18 29] INFO [2019-12-17 12:38:23,895] jepsen worker 56 - jepsen.util 13826 :invoke :write [18 28] INFO [2019-12-17 12:38:23,901] jepsen worker 55 - jepsen.util 13915 :fail :cas [18 [14 3]] :ValueNotSwapFrom INFO [2019-12-17 12:38:23,901] jepsen worker 55 - jepsen.util 13915 :invoke :write [18 36] INFO [2019-12-17 12:38:23,903] jepsen worker 54 - jepsen.util 13824 :ok :read [18 29] INFO [2019-12-17 12:38:23,903] jepsen worker 54 - jepsen.util 13824 :invoke :read [18 nil] INFO [2019-12-17 12:38:23,930] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":75997,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:38:23,930] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":75997,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:38:23,930] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76000,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:38:23,930] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76000,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:38:23,930] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76001,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:38:23,931] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76001,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:38:23,935] jepsen worker 19 - jepsen.util 13969 :ok :read [6 45] INFO [2019-12-17 12:38:23,935] jepsen worker 19 - jepsen.util 13969 :invoke :cas [6 [22 15]] INFO [2019-12-17 12:38:23,949] jepsen worker 27 - jepsen.util 12267 :ok :read [9 32] INFO [2019-12-17 12:38:23,949] jepsen worker 66 - jepsen.util 17976 :info :write [22 15] :RequestCanceledException INFO [2019-12-17 12:38:23,949] jepsen worker 33 - jepsen.util 17763 :info :cas [11 [42 4]] :RequestCanceledException INFO [2019-12-17 12:38:23,949] jepsen worker 27 - jepsen.util 12267 :invoke :read [9 nil] INFO [2019-12-17 12:38:23,950] jepsen worker 33 - jepsen.util 17853 :invoke :cas [11 [48 15]] INFO [2019-12-17 12:38:23,950] jepsen worker 66 - jepsen.util 18066 :invoke :write [22 21] INFO [2019-12-17 12:38:23,950] jepsen worker 32 - jepsen.util 13442 :fail :cas [10 [29 5]] :ValueNotSwapFrom INFO [2019-12-17 12:38:23,951] jepsen worker 32 - jepsen.util 13442 :invoke :cas [10 [19 14]] INFO [2019-12-17 12:38:24,016] jepsen worker 28 - jepsen.util 11098 :fail :cas [9 [44 16]] :ValueNotSwapFrom INFO [2019-12-17 12:38:24,017] jepsen worker 28 - jepsen.util 11098 :invoke :cas [9 [16 36]] INFO [2019-12-17 12:38:24,034] jepsen worker 85 - jepsen.util 18445 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:38:24,034] jepsen worker 20 - jepsen.util 12890 :info :write [6 9] :RequestCanceledException INFO [2019-12-17 12:38:24,034] jepsen worker 67 - jepsen.util 18067 :info :write [22 0] :RequestCanceledException INFO [2019-12-17 12:38:24,034] jepsen worker 85 - jepsen.util 18445 :invoke :cas [28 [44 8]] INFO [2019-12-17 12:38:24,035] jepsen worker 20 - jepsen.util 12980 :invoke :write [6 48] INFO [2019-12-17 12:38:24,035] jepsen worker 67 - jepsen.util 18157 :invoke :write [22 2] INFO [2019-12-17 12:38:24,037] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76013,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:38:24,037] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76013,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:38:24,037] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76016,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:38:24,037] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76016,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:38:24,088] jepsen worker 5 - jepsen.util 12785 :fail :cas [1 [1 0]] :ValueNotSwapFrom INFO [2019-12-17 12:38:24,088] jepsen worker 5 - jepsen.util 12785 :invoke :cas [1 [18 25]] INFO [2019-12-17 12:38:24,094] jepsen worker 71 - jepsen.util 14021 :ok :read [23 13] INFO [2019-12-17 12:38:24,094] jepsen worker 71 - jepsen.util 14021 :invoke :cas [23 [25 43]] INFO [2019-12-17 12:38:24,107] jepsen worker 37 - jepsen.util 12727 :fail :cas [12 [12 17]] :ValueNotSwapFrom INFO [2019-12-17 12:38:24,107] jepsen worker 37 - jepsen.util 12727 :invoke :cas [12 [11 25]] INFO [2019-12-17 12:38:24,146] jepsen worker 27 - jepsen.util 12267 :ok :read [9 32] INFO [2019-12-17 12:38:24,146] jepsen worker 27 - jepsen.util 12267 :invoke :cas [9 [5 31]] INFO [2019-12-17 12:38:24,149] jepsen worker 3 - jepsen.util 12693 :info :write [1 44] :RequestCanceledException INFO [2019-12-17 12:38:24,150] jepsen worker 3 - jepsen.util 12783 :invoke :cas [1 [2 2]] INFO [2019-12-17 12:38:24,160] jepsen worker 36 - jepsen.util 13806 :ok :write [12 32] INFO [2019-12-17 12:38:24,161] jepsen worker 36 - jepsen.util 13806 :invoke :cas [12 [29 26]] INFO [2019-12-17 12:38:24,161] jepsen worker 51 - jepsen.util 12381 :fail :cas [17 [36 36]] :ValueNotSwapFrom INFO [2019-12-17 12:38:24,161] jepsen worker 51 - jepsen.util 12381 :invoke :cas [17 [39 19]] INFO [2019-12-17 12:38:24,165] jepsen worker 21 - jepsen.util 17481 :info :cas [7 [24 30]] :RequestCanceledException INFO [2019-12-17 12:38:24,166] jepsen worker 21 - jepsen.util 17571 :invoke :read [7 nil] INFO [2019-12-17 12:38:24,168] jepsen worker 32 - jepsen.util 13442 :fail :cas [10 [19 14]] :ValueNotSwapFrom INFO [2019-12-17 12:38:24,168] jepsen worker 32 - jepsen.util 13442 :invoke :write [10 25] INFO [2019-12-17 12:38:24,199] jepsen worker 28 - jepsen.util 11098 :fail :cas [9 [16 36]] :ValueNotSwapFrom INFO [2019-12-17 12:38:24,199] jepsen worker 28 - jepsen.util 11098 :invoke :cas [9 [45 21]] INFO [2019-12-17 12:38:24,219] jepsen worker 0 - jepsen.util 18360 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:38:24,219] jepsen worker 0 - jepsen.util 18360 :invoke :write [0 46] INFO [2019-12-17 12:38:24,230] jepsen worker 27 - jepsen.util 12267 :fail :cas [9 [5 31]] :ValueNotSwapFrom INFO [2019-12-17 12:38:24,230] jepsen worker 27 - jepsen.util 12267 :invoke :write [9 21] INFO [2019-12-17 12:38:24,234] jepsen worker 70 - jepsen.util 13660 :info :write [23 10] :RequestCanceledException INFO [2019-12-17 12:38:24,236] jepsen worker 70 - jepsen.util 13750 :invoke :cas [23 [40 10]] INFO [2019-12-17 12:38:24,237] jepsen worker 4 - jepsen.util 12244 :ok :write [1 39] INFO [2019-12-17 12:38:24,237] jepsen worker 4 - jepsen.util 12244 :invoke :write [1 29] INFO [2019-12-17 12:38:24,248] jepsen worker 29 - jepsen.util 11369 :ok :write [9 44] INFO [2019-12-17 12:38:24,248] jepsen worker 29 - jepsen.util 11369 :invoke :read [9 nil] INFO [2019-12-17 12:38:24,248] jepsen worker 84 - jepsen.util 17364 :info :cas [28 [21 31]] :RequestCanceledException INFO [2019-12-17 12:38:24,249] jepsen worker 84 - jepsen.util 17454 :invoke :read [28 nil] INFO [2019-12-17 12:38:24,253] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76039,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:38:24,253] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76039,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:38:24,253] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76043,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:38:24,253] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76043,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:38:24,256] jepsen worker 13 - jepsen.util 12703 :fail :cas [4 [8 40]] :ValueNotSwapFrom INFO [2019-12-17 12:38:24,256] jepsen worker 13 - jepsen.util 12703 :invoke :write [4 22] INFO [2019-12-17 12:38:24,262] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76032,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:38:24,262] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76032,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:38:24,262] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76034,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:38:24,262] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76034,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:38:24,298] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:38:24,298] jepsen nemesis - couchbase.util Rebalance check count: 598 INFO [2019-12-17 12:38:24,298] jepsen nemesis - couchbase.util Rebalance stuck count: 579 INFO [2019-12-17 12:38:24,305] jepsen worker 54 - jepsen.util 13824 :ok :read [18 29] INFO [2019-12-17 12:38:24,305] jepsen worker 54 - jepsen.util 13824 :invoke :read [18 nil] INFO [2019-12-17 12:38:24,311] jepsen worker 80 - jepsen.util 12590 :ok :read [26 37] INFO [2019-12-17 12:38:24,311] jepsen worker 80 - jepsen.util 12590 :invoke :write [26 26] INFO [2019-12-17 12:38:24,313] jepsen worker 88 - jepsen.util 13138 :ok :write [29 19] INFO [2019-12-17 12:38:24,313] jepsen worker 88 - jepsen.util 13138 :invoke :read [29 nil] INFO [2019-12-17 12:38:24,327] jepsen worker 63 - jepsen.util 11673 :ok :write [21 38] INFO [2019-12-17 12:38:24,327] jepsen worker 63 - jepsen.util 11673 :invoke :cas [21 [30 48]] INFO [2019-12-17 12:38:24,335] jepsen worker 23 - jepsen.util 18113 :info :cas [7 [18 27]] :RequestCanceledException INFO [2019-12-17 12:38:24,336] jepsen worker 23 - jepsen.util 18203 :invoke :read [7 nil] INFO [2019-12-17 12:38:24,365] jepsen worker 19 - jepsen.util 13969 :fail :cas [6 [22 15]] :ValueNotSwapFrom INFO [2019-12-17 12:38:24,377] jepsen worker 55 - jepsen.util 13915 :ok :write [18 36] INFO [2019-12-17 12:38:24,380] jepsen worker 71 - jepsen.util 14021 :fail :cas [23 [25 43]] :ValueNotSwapFrom INFO [2019-12-17 12:38:24,383] jepsen worker 78 - jepsen.util 12858 :ok :write [26 23] INFO [2019-12-17 12:38:24,393] jepsen worker 28 - jepsen.util 11098 :fail :cas [9 [45 21]] :ValueNotSwapFrom INFO [2019-12-17 12:38:24,396] jepsen worker 36 - jepsen.util 13806 :fail :cas [12 [29 26]] :ValueNotSwapFrom INFO [2019-12-17 12:38:24,406] jepsen worker 55 - jepsen.util 13915 :invoke :read [18 nil] INFO [2019-12-17 12:38:24,406] jepsen worker 19 - jepsen.util 13969 :invoke :write [6 8] INFO [2019-12-17 12:38:24,406] jepsen worker 71 - jepsen.util 14021 :invoke :write [23 19] INFO [2019-12-17 12:38:24,406] jepsen worker 78 - jepsen.util 12858 :invoke :write [26 18] INFO [2019-12-17 12:38:24,406] jepsen worker 28 - jepsen.util 11098 :invoke :read [9 nil] INFO [2019-12-17 12:38:24,406] jepsen worker 36 - jepsen.util 13806 :invoke :read [12 nil] INFO [2019-12-17 12:38:24,419] jepsen worker 82 - jepsen.util 17902 :info :write [27 16] :RequestCanceledException INFO [2019-12-17 12:38:24,420] jepsen worker 63 - jepsen.util 11673 :fail :cas [21 [30 48]] :ValueNotSwapFrom INFO [2019-12-17 12:38:24,420] jepsen worker 82 - jepsen.util 17992 :invoke :write [27 45] INFO [2019-12-17 12:38:24,420] jepsen worker 63 - jepsen.util 11673 :invoke :write [21 35] INFO [2019-12-17 12:38:24,448] jepsen worker 27 - jepsen.util 12267 :ok :write [9 21] INFO [2019-12-17 12:38:24,448] jepsen worker 27 - jepsen.util 12267 :invoke :cas [9 [3 5]] INFO [2019-12-17 12:38:24,478] jepsen worker 36 - jepsen.util 13806 :ok :read [12 32] INFO [2019-12-17 12:38:24,479] jepsen worker 36 - jepsen.util 13806 :invoke :write [12 38] INFO [2019-12-17 12:38:24,481] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76065,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:38:24,481] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76065,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:38:24,481] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76069,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:38:24,481] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76069,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:38:24,483] jepsen worker 38 - jepsen.util 13808 :ok :write [12 19] INFO [2019-12-17 12:38:24,483] jepsen worker 38 - jepsen.util 13808 :invoke :write [12 26] INFO [2019-12-17 12:38:24,505] jepsen worker 12 - jepsen.util 13242 :ok :read [4 35] INFO [2019-12-17 12:38:24,505] jepsen worker 12 - jepsen.util 13242 :invoke :write [4 39] INFO [2019-12-17 12:38:24,522] jepsen worker 28 - jepsen.util 11098 :ok :read [9 21] INFO [2019-12-17 12:38:24,522] jepsen worker 28 - jepsen.util 11098 :invoke :cas [9 [2 17]] INFO [2019-12-17 12:38:24,528] jepsen worker 51 - jepsen.util 12381 :fail :cas [17 [39 19]] :ValueNotSwapFrom INFO [2019-12-17 12:38:24,529] jepsen worker 51 - jepsen.util 12381 :invoke :cas [17 [38 34]] INFO [2019-12-17 12:38:24,531] jepsen worker 52 - jepsen.util 12742 :ok :write [17 24] INFO [2019-12-17 12:38:24,531] jepsen worker 52 - jepsen.util 12742 :invoke :read [17 nil] INFO [2019-12-17 12:38:24,534] jepsen worker 14 - jepsen.util 12704 :info :write [4 42] :RequestCanceledException INFO [2019-12-17 12:38:24,535] jepsen worker 14 - jepsen.util 12794 :invoke :read [4 nil] INFO [2019-12-17 12:38:24,541] jepsen worker 27 - jepsen.util 12267 :fail :cas [9 [3 5]] :ValueNotSwapFrom INFO [2019-12-17 12:38:24,541] jepsen worker 27 - jepsen.util 12267 :invoke :write [9 36] INFO [2019-12-17 12:38:24,549] jepsen worker 55 - jepsen.util 13915 :ok :read [18 36] INFO [2019-12-17 12:38:24,549] jepsen worker 55 - jepsen.util 13915 :invoke :write [18 2] INFO [2019-12-17 12:38:24,579] jepsen worker 70 - jepsen.util 13750 :fail :cas [23 [40 10]] :ValueNotSwapFrom INFO [2019-12-17 12:38:24,579] jepsen worker 70 - jepsen.util 13750 :invoke :cas [23 [18 36]] INFO [2019-12-17 12:38:24,585] jepsen worker 28 - jepsen.util 11098 :fail :cas [9 [2 17]] :ValueNotSwapFrom INFO [2019-12-17 12:38:24,586] jepsen worker 28 - jepsen.util 11098 :invoke :cas [9 [2 3]] INFO [2019-12-17 12:38:24,602] jepsen worker 52 - jepsen.util 12742 :ok :read [17 24] INFO [2019-12-17 12:38:24,602] jepsen worker 52 - jepsen.util 12742 :invoke :read [17 nil] INFO [2019-12-17 12:38:24,635] jepsen worker 63 - jepsen.util 11673 :ok :write [21 35] INFO [2019-12-17 12:38:24,635] jepsen worker 63 - jepsen.util 11673 :invoke :cas [21 [38 25]] INFO [2019-12-17 12:38:24,637] jepsen worker 78 - jepsen.util 12858 :ok :write [26 18] INFO [2019-12-17 12:38:24,637] jepsen worker 78 - jepsen.util 12858 :invoke :read [26 nil] INFO [2019-12-17 12:38:24,649] jepsen worker 77 - jepsen.util 19157 :info :cas [25 [32 41]] :RequestCanceledException INFO [2019-12-17 12:38:24,650] jepsen worker 77 - jepsen.util 19247 :invoke :write [25 29] INFO [2019-12-17 12:38:24,657] jepsen worker 70 - jepsen.util 13750 :fail :cas [23 [18 36]] :ValueNotSwapFrom INFO [2019-12-17 12:38:24,657] jepsen worker 70 - jepsen.util 13750 :invoke :write [23 39] INFO [2019-12-17 12:38:24,666] jepsen worker 89 - jepsen.util 13049 :ok :write [29 10] INFO [2019-12-17 12:38:24,666] jepsen worker 89 - jepsen.util 13049 :invoke :write [29 11] INFO [2019-12-17 12:38:24,669] jepsen worker 63 - jepsen.util 11673 :fail :cas [21 [38 25]] :ValueNotSwapFrom INFO [2019-12-17 12:38:24,669] jepsen worker 63 - jepsen.util 11673 :invoke :write [21 10] INFO [2019-12-17 12:38:24,689] jepsen worker 5 - jepsen.util 12785 :fail :cas [1 [18 25]] :ValueNotSwapFrom INFO [2019-12-17 12:38:24,689] jepsen worker 5 - jepsen.util 12785 :invoke :write [1 44] INFO [2019-12-17 12:38:24,700] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76086,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:38:24,700] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76086,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:38:24,700] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76090,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:38:24,700] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76090,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:38:24,701] jepsen worker 27 - jepsen.util 12267 :ok :write [9 36] INFO [2019-12-17 12:38:24,701] jepsen worker 27 - jepsen.util 12267 :invoke :write [9 14] INFO [2019-12-17 12:38:24,722] jepsen worker 37 - jepsen.util 12727 :fail :cas [12 [11 25]] :ValueNotSwapFrom INFO [2019-12-17 12:38:24,722] jepsen worker 37 - jepsen.util 12727 :invoke :cas [12 [4 14]] INFO [2019-12-17 12:38:24,739] jepsen worker 52 - jepsen.util 12742 :ok :read [17 24] INFO [2019-12-17 12:38:24,739] jepsen worker 52 - jepsen.util 12742 :invoke :write [17 33] INFO [2019-12-17 12:38:24,766] jepsen worker 35 - jepsen.util 18035 :info :cas [11 [9 32]] :RequestCanceledException INFO [2019-12-17 12:38:24,769] jepsen worker 35 - jepsen.util 18125 :invoke :write [11 36] INFO [2019-12-17 12:38:24,772] jepsen worker 4 - jepsen.util 12244 :ok :write [1 29] INFO [2019-12-17 12:38:24,772] jepsen worker 4 - jepsen.util 12244 :invoke :cas [1 [33 13]] INFO [2019-12-17 12:38:24,794] jepsen worker 36 - jepsen.util 13806 :ok :write [12 38] INFO [2019-12-17 12:38:24,794] jepsen worker 36 - jepsen.util 13806 :invoke :write [12 39] INFO [2019-12-17 12:38:24,801] jepsen worker 2 - jepsen.util 17102 :info :cas [0 [13 22]] :RequestCanceledException INFO [2019-12-17 12:38:24,801] jepsen worker 74 - jepsen.util 12944 :fail :read [24 nil] :RequestTimeoutException INFO [2019-12-17 12:38:24,801] jepsen worker 15 - jepsen.util 17115 :info :write [5 35] :RequestCanceledException INFO [2019-12-17 12:38:24,802] jepsen worker 74 - jepsen.util 12944 :invoke :write [24 26] INFO [2019-12-17 12:38:24,802] jepsen worker 2 - jepsen.util 17192 :invoke :write [0 24] INFO [2019-12-17 12:38:24,802] jepsen worker 15 - jepsen.util 17205 :invoke :read [5 nil] INFO [2019-12-17 12:38:24,819] jepsen worker 16 - jepsen.util 19546 :info :cas [5 [14 29]] :RequestCanceledException INFO [2019-12-17 12:38:24,820] jepsen worker 16 - jepsen.util 19636 :invoke :read [5 nil] INFO [2019-12-17 12:38:24,825] jepsen worker 28 - jepsen.util 11098 :fail :cas [9 [2 3]] :ValueNotSwapFrom INFO [2019-12-17 12:38:24,825] jepsen worker 28 - jepsen.util 11098 :invoke :write [9 37] INFO [2019-12-17 12:38:24,835] jepsen worker 81 - jepsen.util 17451 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:38:24,835] jepsen worker 81 - jepsen.util 17451 :invoke :cas [27 [2 8]] INFO [2019-12-17 12:38:24,841] jepsen worker 54 - jepsen.util 13824 :ok :read [18 36] INFO [2019-12-17 12:38:24,841] jepsen worker 54 - jepsen.util 13824 :invoke :write [18 3] INFO [2019-12-17 12:38:24,848] jepsen worker 79 - jepsen.util 13309 :ok :write [26 19] INFO [2019-12-17 12:38:24,848] jepsen worker 79 - jepsen.util 13309 :invoke :write [26 45] INFO [2019-12-17 12:38:24,848] jepsen worker 44 - jepsen.util 18674 :info :write [14 37] :RequestCanceledException INFO [2019-12-17 12:38:24,848] jepsen worker 58 - jepsen.util 18148 :info :cas [19 [28 29]] :RequestCanceledException INFO [2019-12-17 12:38:24,850] jepsen worker 44 - jepsen.util 18764 :invoke :cas [14 [11 29]] INFO [2019-12-17 12:38:24,851] jepsen worker 58 - jepsen.util 18238 :invoke :cas [19 [37 44]] INFO [2019-12-17 12:38:24,851] jepsen worker 78 - jepsen.util 12858 :ok :read [26 18] INFO [2019-12-17 12:38:24,852] jepsen worker 78 - jepsen.util 12858 :invoke :cas [26 [42 15]] INFO [2019-12-17 12:38:24,866] jepsen worker 43 - jepsen.util 18133 :info :write [14 17] :RequestCanceledException INFO [2019-12-17 12:38:24,867] jepsen worker 43 - jepsen.util 18223 :invoke :write [14 35] INFO [2019-12-17 12:38:24,879] jepsen worker 63 - jepsen.util 11673 :ok :write [21 10] INFO [2019-12-17 12:38:24,879] jepsen worker 63 - jepsen.util 11673 :invoke :read [21 nil] INFO [2019-12-17 12:38:24,896] jepsen worker 37 - jepsen.util 12727 :fail :cas [12 [4 14]] :ValueNotSwapFrom INFO [2019-12-17 12:38:24,896] jepsen worker 4 - jepsen.util 12244 :fail :cas [1 [33 13]] :ValueNotSwapFrom INFO [2019-12-17 12:38:24,896] jepsen worker 37 - jepsen.util 12727 :invoke :read [12 nil] INFO [2019-12-17 12:38:24,896] jepsen worker 4 - jepsen.util 12244 :invoke :write [1 20] INFO [2019-12-17 12:38:24,901] jepsen worker 53 - jepsen.util 12653 :info :write [17 3] :RequestCanceledException INFO [2019-12-17 12:38:24,901] jepsen worker 25 - jepsen.util 19015 :info :cas [8 [36 27]] :RequestCanceledException INFO [2019-12-17 12:38:24,902] jepsen worker 53 - jepsen.util 12743 :invoke :cas [17 [16 16]] INFO [2019-12-17 12:38:24,903] jepsen worker 25 - jepsen.util 19105 :invoke :cas [8 [4 45]] INFO [2019-12-17 12:38:24,913] jepsen worker 56 - jepsen.util 13826 :ok :write [18 28] INFO [2019-12-17 12:38:24,913] jepsen worker 56 - jepsen.util 13826 :invoke :cas [18 [17 26]] INFO [2019-12-17 12:38:24,917] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76114,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:38:24,917] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76114,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:38:24,917] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76117,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:38:24,917] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76117,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:38:24,925] jepsen worker 69 - jepsen.util 13479 :ok :write [23 9] INFO [2019-12-17 12:38:24,925] jepsen worker 69 - jepsen.util 13479 :invoke :write [23 0] INFO [2019-12-17 12:38:24,930] jepsen worker 63 - jepsen.util 11673 :ok :read [21 10] INFO [2019-12-17 12:38:24,931] jepsen worker 63 - jepsen.util 11673 :invoke :write [21 28] INFO [2019-12-17 12:38:24,934] jepsen worker 57 - jepsen.util 17517 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:38:24,934] jepsen worker 57 - jepsen.util 17517 :invoke :cas [19 [35 46]] INFO [2019-12-17 12:38:24,935] jepsen worker 48 - jepsen.util 16608 :info :write [16 21] :RequestCanceledException INFO [2019-12-17 12:38:24,935] jepsen worker 50 - jepsen.util 17870 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:38:24,935] jepsen worker 50 - jepsen.util 17870 :invoke :cas [16 [43 33]] INFO [2019-12-17 12:38:24,936] jepsen worker 48 - jepsen.util 16698 :invoke :read [16 nil] INFO [2019-12-17 12:38:24,962] jepsen worker 36 - jepsen.util 13806 :ok :write [12 39] INFO [2019-12-17 12:38:24,962] jepsen worker 36 - jepsen.util 13806 :invoke :cas [12 [47 18]] INFO [2019-12-17 12:38:24,979] jepsen worker 3 - jepsen.util 12783 :fail :cas [1 [2 2]] :ValueNotSwapFrom INFO [2019-12-17 12:38:24,979] jepsen worker 3 - jepsen.util 12783 :invoke :read [1 nil] INFO [2019-12-17 12:38:24,997] jepsen worker 7 - jepsen.util 17647 :info :cas [2 [14 18]] :RequestCanceledException INFO [2019-12-17 12:38:24,998] jepsen worker 7 - jepsen.util 17737 :invoke :write [2 7] INFO [2019-12-17 12:38:25,012] jepsen worker 88 - jepsen.util 13138 :ok :read [29 10] INFO [2019-12-17 12:38:25,012] jepsen worker 88 - jepsen.util 13138 :invoke :write [29 38] INFO [2019-12-17 12:38:25,014] jepsen worker 78 - jepsen.util 12858 :fail :cas [26 [42 15]] :ValueNotSwapFrom INFO [2019-12-17 12:38:25,014] jepsen worker 78 - jepsen.util 12858 :invoke :read [26 nil] INFO [2019-12-17 12:38:25,020] jepsen worker 8 - jepsen.util 17468 :info :cas [2 [41 38]] :RequestCanceledException INFO [2019-12-17 12:38:25,021] jepsen worker 8 - jepsen.util 17558 :invoke :write [2 10] INFO [2019-12-17 12:38:25,021] jepsen worker 27 - jepsen.util 12267 :ok :write [9 14] INFO [2019-12-17 12:38:25,021] jepsen worker 27 - jepsen.util 12267 :invoke :cas [9 [34 39]] INFO [2019-12-17 12:38:25,023] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76134,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:38:25,024] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76134,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:38:25,024] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76137,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:38:25,024] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76137,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:38:25,024] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76139,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:38:25,024] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76139,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:38:25,035] jepsen worker 61 - jepsen.util 19141 :info :write [20 39] :RequestCanceledException INFO [2019-12-17 12:38:25,036] jepsen worker 61 - jepsen.util 19231 :invoke :cas [20 [13 9]] INFO [2019-12-17 12:38:25,053] jepsen worker 27 - jepsen.util 12267 :fail :cas [9 [34 39]] :ValueNotSwapFrom INFO [2019-12-17 12:38:25,053] jepsen worker 27 - jepsen.util 12267 :invoke :cas [9 [7 4]] INFO [2019-12-17 12:38:25,057] jepsen worker 80 - jepsen.util 12590 :ok :write [26 26] INFO [2019-12-17 12:38:25,057] jepsen worker 80 - jepsen.util 12590 :invoke :write [26 28] INFO [2019-12-17 12:38:25,065] jepsen worker 59 - jepsen.util 17069 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:38:25,066] jepsen worker 59 - jepsen.util 17069 :invoke :cas [19 [2 41]] INFO [2019-12-17 12:38:25,085] jepsen worker 36 - jepsen.util 13806 :fail :cas [12 [47 18]] :ValueNotSwapFrom INFO [2019-12-17 12:38:25,085] jepsen worker 36 - jepsen.util 13806 :invoke :cas [12 [15 23]] INFO [2019-12-17 12:38:25,094] jepsen worker 14 - jepsen.util 12794 :ok :read [4 42] INFO [2019-12-17 12:38:25,094] jepsen worker 14 - jepsen.util 12794 :invoke :write [4 29] INFO [2019-12-17 12:38:25,096] jepsen worker 27 - jepsen.util 12267 :fail :cas [9 [7 4]] :ValueNotSwapFrom INFO [2019-12-17 12:38:25,097] jepsen worker 27 - jepsen.util 12267 :invoke :write [9 26] INFO [2019-12-17 12:38:25,106] jepsen worker 63 - jepsen.util 11673 :ok :write [21 28] INFO [2019-12-17 12:38:25,106] jepsen worker 63 - jepsen.util 11673 :invoke :cas [21 [15 5]] INFO [2019-12-17 12:38:25,119] jepsen worker 9 - jepsen.util 13509 :info :cas [3 [26 38]] :RequestCanceledException INFO [2019-12-17 12:38:25,119] jepsen worker 83 - jepsen.util 17903 :info :write [27 6] :RequestCanceledException INFO [2019-12-17 12:38:25,120] jepsen worker 9 - jepsen.util 13599 :invoke :read [3 nil] INFO [2019-12-17 12:38:25,121] jepsen worker 83 - jepsen.util 17993 :invoke :read [27 nil] INFO [2019-12-17 12:38:25,124] jepsen worker 56 - jepsen.util 13826 :fail :cas [18 [17 26]] :ValueNotSwapFrom INFO [2019-12-17 12:38:25,124] jepsen worker 56 - jepsen.util 13826 :invoke :write [18 26] INFO [2019-12-17 12:38:25,126] jepsen worker 78 - jepsen.util 12858 :ok :read [26 26] INFO [2019-12-17 12:38:25,127] jepsen worker 78 - jepsen.util 12858 :invoke :cas [26 [5 2]] INFO [2019-12-17 12:38:25,147] jepsen worker 42 - jepsen.util 18762 :info :write [14 28] :RequestCanceledException INFO [2019-12-17 12:38:25,148] jepsen worker 42 - jepsen.util 18852 :invoke :read [14 nil] INFO [2019-12-17 12:38:25,152] jepsen worker 3 - jepsen.util 12783 :ok :read [1 29] INFO [2019-12-17 12:38:25,152] jepsen worker 3 - jepsen.util 12783 :invoke :cas [1 [28 6]] INFO [2019-12-17 12:38:25,164] jepsen worker 45 - jepsen.util 12735 :ok :write [15 24] INFO [2019-12-17 12:38:25,164] jepsen worker 45 - jepsen.util 12735 :invoke :read [15 nil] INFO [2019-12-17 12:38:25,165] jepsen worker 11 - jepsen.util 13061 :ok :write [3 49] INFO [2019-12-17 12:38:25,165] jepsen worker 11 - jepsen.util 13061 :invoke :read [3 nil] INFO [2019-12-17 12:38:25,165] jepsen worker 32 - jepsen.util 13442 :ok :write [10 25] INFO [2019-12-17 12:38:25,165] jepsen worker 32 - jepsen.util 13442 :invoke :read [10 nil] INFO [2019-12-17 12:38:25,166] jepsen worker 39 - jepsen.util 17499 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:38:25,167] jepsen worker 39 - jepsen.util 17499 :invoke :write [13 26] INFO [2019-12-17 12:38:25,170] jepsen worker 52 - jepsen.util 12742 :ok :write [17 33] INFO [2019-12-17 12:38:25,171] jepsen worker 52 - jepsen.util 12742 :invoke :write [17 28] INFO [2019-12-17 12:38:25,172] jepsen worker 36 - jepsen.util 13806 :fail :cas [12 [15 23]] :ValueNotSwapFrom INFO [2019-12-17 12:38:25,172] jepsen worker 36 - jepsen.util 13806 :invoke :read [12 nil] INFO [2019-12-17 12:38:25,172] jepsen worker 63 - jepsen.util 11673 :fail :cas [21 [15 5]] :ValueNotSwapFrom INFO [2019-12-17 12:38:25,172] jepsen worker 63 - jepsen.util 11673 :invoke :write [21 8] INFO [2019-12-17 12:38:25,195] jepsen worker 47 - jepsen.util 14447 :ok :read [15 24] INFO [2019-12-17 12:38:25,195] jepsen worker 47 - jepsen.util 14447 :invoke :write [15 12] INFO [2019-12-17 12:38:25,198] jepsen worker 78 - jepsen.util 12858 :fail :cas [26 [5 2]] :ValueNotSwapFrom INFO [2019-12-17 12:38:25,198] jepsen worker 78 - jepsen.util 12858 :invoke :write [26 39] INFO [2019-12-17 12:38:25,200] jepsen worker 72 - jepsen.util 13212 :info :write [24 48] :RequestCanceledException INFO [2019-12-17 12:38:25,201] jepsen worker 72 - jepsen.util 13302 :invoke :read [24 nil] INFO [2019-12-17 12:38:25,204] jepsen worker 20 - jepsen.util 12980 :ok :write [6 48] INFO [2019-12-17 12:38:25,204] jepsen worker 20 - jepsen.util 12980 :invoke :read [6 nil] INFO [2019-12-17 12:38:25,208] jepsen worker 36 - jepsen.util 13806 :ok :read [12 39] INFO [2019-12-17 12:38:25,208] jepsen worker 36 - jepsen.util 13806 :invoke :write [12 14] INFO [2019-12-17 12:38:25,218] jepsen worker 11 - jepsen.util 13061 :ok :read [3 49] INFO [2019-12-17 12:38:25,218] jepsen worker 11 - jepsen.util 13061 :invoke :write [3 13] INFO [2019-12-17 12:38:25,221] jepsen worker 73 - jepsen.util 12223 :ok :read [24 2] INFO [2019-12-17 12:38:25,221] jepsen worker 73 - jepsen.util 12223 :invoke :read [24 nil] INFO [2019-12-17 12:38:25,233] jepsen worker 13 - jepsen.util 12703 :ok :write [4 22] INFO [2019-12-17 12:38:25,233] jepsen worker 13 - jepsen.util 12703 :invoke :write [4 7] INFO [2019-12-17 12:38:25,234] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76177,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:38:25,234] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76177,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:38:25,234] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76180,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:38:25,234] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76180,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:38:25,234] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76181,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:38:25,234] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76181,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:38:25,235] jepsen worker 1 - jepsen.util 18361 :info :cas [0 [37 19]] :RequestCanceledException INFO [2019-12-17 12:38:25,236] jepsen worker 1 - jepsen.util 18451 :invoke :write [0 18] INFO [2019-12-17 12:38:25,240] jepsen worker 45 - jepsen.util 12735 :ok :read [15 24] INFO [2019-12-17 12:38:25,240] jepsen worker 45 - jepsen.util 12735 :invoke :read [15 nil] INFO [2019-12-17 12:38:25,248] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76167,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:38:25,248] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76167,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:38:25,248] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76169,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:38:25,248] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76169,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:38:25,249] jepsen worker 18 - jepsen.util 13698 :info :write [6 35] :RequestCanceledException INFO [2019-12-17 12:38:25,249] jepsen worker 22 - jepsen.util 18382 :info :cas [7 [5 5]] :RequestCanceledException INFO [2019-12-17 12:38:25,250] jepsen worker 18 - jepsen.util 13788 :invoke :cas [6 [23 10]] INFO [2019-12-17 12:38:25,250] jepsen worker 22 - jepsen.util 18472 :invoke :write [7 3] INFO [2019-12-17 12:38:25,253] jepsen worker 27 - jepsen.util 12267 :ok :write [9 26] INFO [2019-12-17 12:38:25,253] jepsen worker 27 - jepsen.util 12267 :invoke :cas [9 [23 11]] INFO [2019-12-17 12:38:25,259] jepsen worker 53 - jepsen.util 12743 :fail :cas [17 [16 16]] :ValueNotSwapFrom INFO [2019-12-17 12:38:25,259] jepsen worker 53 - jepsen.util 12743 :invoke :write [17 9] INFO [2019-12-17 12:38:25,281] jepsen worker 27 - jepsen.util 12267 :fail :cas [9 [23 11]] :ValueNotSwapFrom INFO [2019-12-17 12:38:25,281] jepsen worker 27 - jepsen.util 12267 :invoke :write [9 6] INFO [2019-12-17 12:38:25,299] jepsen worker 63 - jepsen.util 11673 :ok :write [21 8] INFO [2019-12-17 12:38:25,299] jepsen worker 63 - jepsen.util 11673 :invoke :cas [21 [37 7]] INFO [2019-12-17 12:38:25,302] jepsen worker 30 - jepsen.util 12540 :fail :read [10 nil] :RequestTimeoutException INFO [2019-12-17 12:38:25,302] jepsen worker 65 - jepsen.util 11495 :info :write [21 24] :RequestCanceledException INFO [2019-12-17 12:38:25,302] jepsen worker 30 - jepsen.util 12540 :invoke :cas [10 [48 14]] INFO [2019-12-17 12:38:25,307] jepsen worker 65 - jepsen.util 11585 :invoke :read [21 nil] INFO [2019-12-17 12:38:25,319] jepsen worker 76 - jepsen.util 18076 :info :cas [25 [10 28]] :RequestCanceledException INFO [2019-12-17 12:38:25,319] jepsen worker 10 - jepsen.util 12970 :info :cas [3 [2 27]] :RequestCanceledException INFO [2019-12-17 12:38:25,320] jepsen worker 10 - jepsen.util 13060 :invoke :read [3 nil] INFO [2019-12-17 12:38:25,320] jepsen worker 76 - jepsen.util 18166 :invoke :cas [25 [13 5]] INFO [2019-12-17 12:38:25,324] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:38:25,324] jepsen nemesis - couchbase.util Rebalance check count: 599 INFO [2019-12-17 12:38:25,324] jepsen nemesis - couchbase.util Rebalance stuck count: 580 INFO [2019-12-17 12:38:25,335] jepsen worker 68 - jepsen.util 17078 :info :cas [22 [18 27]] :RequestCanceledException INFO [2019-12-17 12:38:25,336] jepsen worker 68 - jepsen.util 17168 :invoke :cas [22 [7 32]] INFO [2019-12-17 12:38:25,347] jepsen worker 63 - jepsen.util 11673 :fail :cas [21 [37 7]] :ValueNotSwapFrom INFO [2019-12-17 12:38:25,348] jepsen worker 63 - jepsen.util 11673 :invoke :write [21 29] INFO [2019-12-17 12:38:25,348] jepsen worker 54 - jepsen.util 13824 :ok :write [18 3] INFO [2019-12-17 12:38:25,348] jepsen worker 54 - jepsen.util 13824 :invoke :write [18 0] INFO [2019-12-17 12:38:25,355] jepsen worker 27 - jepsen.util 12267 :ok :write [9 6] INFO [2019-12-17 12:38:25,355] jepsen worker 27 - jepsen.util 12267 :invoke :read [9 nil] INFO [2019-12-17 12:38:25,360] jepsen worker 88 - jepsen.util 13138 :ok :write [29 38] INFO [2019-12-17 12:38:25,360] jepsen worker 88 - jepsen.util 13138 :invoke :read [29 nil] INFO [2019-12-17 12:38:25,370] jepsen worker 38 - jepsen.util 13808 :ok :write [12 26] INFO [2019-12-17 12:38:25,370] jepsen worker 38 - jepsen.util 13808 :invoke :cas [12 [21 5]] INFO [2019-12-17 12:38:25,388] jepsen worker 10 - jepsen.util 13060 :ok :read [3 49] INFO [2019-12-17 12:38:25,389] jepsen worker 10 - jepsen.util 13060 :invoke :cas [3 [46 5]] INFO [2019-12-17 12:38:25,393] jepsen worker 80 - jepsen.util 12590 :ok :write [26 28] INFO [2019-12-17 12:38:25,393] jepsen worker 80 - jepsen.util 12590 :invoke :read [26 nil] INFO [2019-12-17 12:38:25,420] jepsen worker 6 - jepsen.util 18186 :info :write [2 31] :RequestCanceledException INFO [2019-12-17 12:38:25,420] jepsen worker 64 - jepsen.util 10324 :info :cas [21 [1 42]] :RequestCanceledException INFO [2019-12-17 12:38:25,420] jepsen worker 31 - jepsen.util 11191 :fail :read [10 nil] :RequestTimeoutException INFO [2019-12-17 12:38:25,420] jepsen worker 31 - jepsen.util 11191 :invoke :read [10 nil] INFO [2019-12-17 12:38:25,421] jepsen worker 64 - jepsen.util 10414 :invoke :write [21 42] INFO [2019-12-17 12:38:25,424] jepsen worker 6 - jepsen.util 18276 :invoke :read [2 nil] INFO [2019-12-17 12:38:25,431] jepsen worker 27 - jepsen.util 12267 :ok :read [9 6] INFO [2019-12-17 12:38:25,432] jepsen worker 27 - jepsen.util 12267 :invoke :write [9 25] INFO [2019-12-17 12:38:25,435] jepsen worker 34 - jepsen.util 19024 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:38:25,436] jepsen worker 46 - jepsen.util 12736 :info :cas [15 [48 18]] :RequestCanceledException INFO [2019-12-17 12:38:25,436] jepsen worker 34 - jepsen.util 19024 :invoke :read [11 nil] INFO [2019-12-17 12:38:25,437] jepsen worker 46 - jepsen.util 12826 :invoke :write [15 16] INFO [2019-12-17 12:38:25,467] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76213,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:38:25,468] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76213,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:38:25,472] jepsen worker 10 - jepsen.util 13060 :fail :cas [3 [46 5]] :ValueNotSwapFrom INFO [2019-12-17 12:38:25,472] jepsen worker 10 - jepsen.util 13060 :invoke :write [3 20] INFO [2019-12-17 12:38:25,490] jepsen worker 63 - jepsen.util 11673 :ok :write [21 29] INFO [2019-12-17 12:38:25,491] jepsen worker 63 - jepsen.util 11673 :invoke :cas [21 [42 14]] INFO [2019-12-17 12:38:25,515] jepsen worker 80 - jepsen.util 12590 :ok :read [26 28] INFO [2019-12-17 12:38:25,515] jepsen worker 80 - jepsen.util 12590 :invoke :read [26 nil] INFO [2019-12-17 12:38:25,519] jepsen worker 49 - jepsen.util 17959 :info :write [16 1] :RequestCanceledException INFO [2019-12-17 12:38:25,520] jepsen worker 49 - jepsen.util 18049 :invoke :cas [16 [26 40]] INFO [2019-12-17 12:38:25,543] jepsen worker 51 - jepsen.util 12381 :fail :cas [17 [38 34]] :ValueNotSwapFrom INFO [2019-12-17 12:38:25,543] jepsen worker 51 - jepsen.util 12381 :invoke :cas [17 [21 38]] INFO [2019-12-17 12:38:25,559] jepsen worker 4 - jepsen.util 12244 :ok :write [1 20] INFO [2019-12-17 12:38:25,559] jepsen worker 4 - jepsen.util 12244 :invoke :write [1 42] INFO [2019-12-17 12:38:25,566] jepsen worker 63 - jepsen.util 11673 :fail :cas [21 [42 14]] :ValueNotSwapFrom INFO [2019-12-17 12:38:25,566] jepsen worker 63 - jepsen.util 11673 :invoke :write [21 37] INFO [2019-12-17 12:38:25,570] jepsen worker 27 - jepsen.util 12267 :ok :write [9 25] INFO [2019-12-17 12:38:25,570] jepsen worker 27 - jepsen.util 12267 :invoke :read [9 nil] INFO [2019-12-17 12:38:25,594] jepsen worker 14 - jepsen.util 12794 :ok :write [4 29] INFO [2019-12-17 12:38:25,595] jepsen worker 14 - jepsen.util 12794 :invoke :cas [4 [2 18]] INFO [2019-12-17 12:38:25,601] jepsen worker 40 - jepsen.util 18670 :info :write [13 21] :RequestCanceledException INFO [2019-12-17 12:38:25,602] jepsen worker 40 - jepsen.util 18760 :invoke :write [13 36] INFO [2019-12-17 12:38:25,604] jepsen worker 89 - jepsen.util 13049 :ok :write [29 11] INFO [2019-12-17 12:38:25,605] jepsen worker 89 - jepsen.util 13049 :invoke :read [29 nil] INFO [2019-12-17 12:38:25,619] jepsen worker 41 - jepsen.util 19211 :info :cas [13 [10 15]] :RequestCanceledException INFO [2019-12-17 12:38:25,621] jepsen worker 41 - jepsen.util 19301 :invoke :write [13 45] INFO [2019-12-17 12:38:25,649] jepsen worker 24 - jepsen.util 17034 :info :write [8 47] :RequestCanceledException INFO [2019-12-17 12:38:25,650] jepsen worker 24 - jepsen.util 17124 :invoke :write [8 36] INFO [2019-12-17 12:38:25,669] jepsen worker 36 - jepsen.util 13806 :ok :write [12 14] INFO [2019-12-17 12:38:25,669] jepsen worker 36 - jepsen.util 13806 :invoke :cas [12 [32 35]] INFO [2019-12-17 12:38:25,671] jepsen worker 70 - jepsen.util 13750 :ok :write [23 39] INFO [2019-12-17 12:38:25,671] jepsen worker 70 - jepsen.util 13750 :invoke :cas [23 [48 28]] INFO [2019-12-17 12:38:25,687] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76232,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:38:25,687] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76232,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:38:25,702] jepsen worker 87 - jepsen.util 12687 :info :write [29 19] :RequestCanceledException INFO [2019-12-17 12:38:25,703] jepsen worker 87 - jepsen.util 12777 :invoke :read [29 nil] INFO [2019-12-17 12:38:25,705] jepsen worker 74 - jepsen.util 12944 :ok :write [24 26] INFO [2019-12-17 12:38:25,705] jepsen worker 74 - jepsen.util 12944 :invoke :read [24 nil] INFO [2019-12-17 12:38:25,725] jepsen worker 13 - jepsen.util 12703 :ok :write [4 7] INFO [2019-12-17 12:38:25,726] jepsen worker 13 - jepsen.util 12703 :invoke :cas [4 [30 13]] INFO [2019-12-17 12:38:25,726] jepsen worker 19 - jepsen.util 13969 :ok :write [6 8] INFO [2019-12-17 12:38:25,727] jepsen worker 19 - jepsen.util 13969 :invoke :write [6 3] INFO [2019-12-17 12:38:25,735] jepsen worker 75 - jepsen.util 17715 :info :write [25 6] :RequestCanceledException INFO [2019-12-17 12:38:25,737] jepsen worker 75 - jepsen.util 17805 :invoke :write [25 23] INFO [2019-12-17 12:38:25,740] jepsen worker 63 - jepsen.util 11673 :ok :write [21 37] INFO [2019-12-17 12:38:25,740] jepsen worker 63 - jepsen.util 11673 :invoke :cas [21 [49 48]] INFO [2019-12-17 12:38:25,742] jepsen worker 9 - jepsen.util 13599 :ok :read [3 49] INFO [2019-12-17 12:38:25,742] jepsen worker 9 - jepsen.util 13599 :invoke :read [3 nil] INFO [2019-12-17 12:38:25,761] jepsen worker 47 - jepsen.util 14447 :ok :write [15 12] INFO [2019-12-17 12:38:25,761] jepsen worker 47 - jepsen.util 14447 :invoke :read [15 nil] INFO [2019-12-17 12:38:25,765] jepsen worker 78 - jepsen.util 12858 :ok :write [26 39] INFO [2019-12-17 12:38:25,766] jepsen worker 78 - jepsen.util 12858 :invoke :write [26 7] INFO [2019-12-17 12:38:25,766] jepsen worker 86 - jepsen.util 18716 :info :write [28 22] :RequestCanceledException INFO [2019-12-17 12:38:25,769] jepsen worker 5 - jepsen.util 12785 :ok :write [1 44] INFO [2019-12-17 12:38:25,769] jepsen worker 5 - jepsen.util 12785 :invoke :read [1 nil] INFO [2019-12-17 12:38:25,769] jepsen worker 86 - jepsen.util 18806 :invoke :write [28 46] INFO [2019-12-17 12:38:25,779] jepsen worker 11 - jepsen.util 13061 :ok :write [3 13] INFO [2019-12-17 12:38:25,779] jepsen worker 11 - jepsen.util 13061 :invoke :read [3 nil] INFO [2019-12-17 12:38:25,785] jepsen worker 32 - jepsen.util 13442 :ok :read [10 25] INFO [2019-12-17 12:38:25,785] jepsen worker 32 - jepsen.util 13442 :invoke :cas [10 [45 4]] INFO [2019-12-17 12:38:25,791] jepsen worker 37 - jepsen.util 12727 :ok :read [12 14] INFO [2019-12-17 12:38:25,791] jepsen worker 37 - jepsen.util 12727 :invoke :read [12 nil] INFO [2019-12-17 12:38:25,801] jepsen worker 62 - jepsen.util 18602 :info :write [20 31] :RequestCanceledException INFO [2019-12-17 12:38:25,802] jepsen worker 62 - jepsen.util 18692 :invoke :cas [20 [22 30]] INFO [2019-12-17 12:38:25,806] jepsen worker 38 - jepsen.util 13808 :fail :cas [12 [21 5]] :ValueNotSwapFrom INFO [2019-12-17 12:38:25,806] jepsen worker 38 - jepsen.util 13808 :invoke :cas [12 [13 28]] INFO [2019-12-17 12:38:25,819] jepsen worker 17 - jepsen.util 18017 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:38:25,819] jepsen worker 26 - jepsen.util 18926 :info :write [8 1] :RequestCanceledException INFO [2019-12-17 12:38:25,819] jepsen worker 17 - jepsen.util 18017 :invoke :read [5 nil] INFO [2019-12-17 12:38:25,820] jepsen worker 26 - jepsen.util 19016 :invoke :write [8 29] INFO [2019-12-17 12:38:25,826] jepsen worker 63 - jepsen.util 11673 :fail :cas [21 [49 48]] :ValueNotSwapFrom INFO [2019-12-17 12:38:25,827] jepsen worker 63 - jepsen.util 11673 :invoke :write [21 30] INFO [2019-12-17 12:38:25,849] jepsen worker 60 - jepsen.util 18960 :info :cas [20 [39 5]] :RequestCanceledException INFO [2019-12-17 12:38:25,849] jepsen worker 60 - jepsen.util 19050 :invoke :read [20 nil] INFO [2019-12-17 12:38:25,868] jepsen worker 72 - jepsen.util 13302 :ok :read [24 26] INFO [2019-12-17 12:38:25,869] jepsen worker 72 - jepsen.util 13302 :invoke :read [24 nil] INFO [2019-12-17 12:38:25,876] jepsen worker 80 - jepsen.util 12590 :ok :read [26 39] INFO [2019-12-17 12:38:25,876] jepsen worker 80 - jepsen.util 12590 :invoke :write [26 47] INFO [2019-12-17 12:38:25,883] jepsen worker 36 - jepsen.util 13806 :fail :cas [12 [32 35]] :ValueNotSwapFrom INFO [2019-12-17 12:38:25,884] jepsen worker 36 - jepsen.util 13806 :invoke :read [12 nil] INFO [2019-12-17 12:38:25,900] jepsen worker 12 - jepsen.util 13242 :ok :write [4 39] INFO [2019-12-17 12:38:25,900] jepsen worker 12 - jepsen.util 13242 :invoke :read [4 nil] INFO [2019-12-17 12:38:25,901] jepsen worker 87 - jepsen.util 12777 :ok :read [29 11] INFO [2019-12-17 12:38:25,901] jepsen worker 87 - jepsen.util 12777 :invoke :read [29 nil] INFO [2019-12-17 12:38:25,903] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76259,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:38:25,904] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76259,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:38:25,904] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76262,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:38:25,904] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76262,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:38:25,904] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76263,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:38:25,904] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76263,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:38:25,919] jepsen worker 13 - jepsen.util 12703 :fail :cas [4 [30 13]] :ValueNotSwapFrom INFO [2019-12-17 12:38:25,920] jepsen worker 13 - jepsen.util 12703 :invoke :cas [4 [1 48]] INFO [2019-12-17 12:38:25,921] jepsen worker 51 - jepsen.util 12381 :fail :cas [17 [21 38]] :ValueNotSwapFrom INFO [2019-12-17 12:38:25,921] jepsen worker 51 - jepsen.util 12381 :invoke :write [17 18] INFO [2019-12-17 12:38:25,934] jepsen worker 30 - jepsen.util 12540 :fail :cas [10 [48 14]] :ValueNotSwapFrom INFO [2019-12-17 12:38:25,934] jepsen worker 30 - jepsen.util 12540 :invoke :read [10 nil] INFO [2019-12-17 12:38:25,958] jepsen worker 45 - jepsen.util 12735 :ok :read [15 12] INFO [2019-12-17 12:38:25,958] jepsen worker 45 - jepsen.util 12735 :invoke :write [15 37] INFO [2019-12-17 12:38:25,964] jepsen worker 36 - jepsen.util 13806 :ok :read [12 14] INFO [2019-12-17 12:38:25,965] jepsen worker 36 - jepsen.util 13806 :invoke :cas [12 [25 26]] INFO [2019-12-17 12:38:25,993] jepsen worker 64 - jepsen.util 10414 :ok :write [21 42] INFO [2019-12-17 12:38:25,993] jepsen worker 64 - jepsen.util 10414 :invoke :cas [21 [14 30]] INFO [2019-12-17 12:38:25,998] jepsen worker 89 - jepsen.util 13049 :ok :read [29 11] INFO [2019-12-17 12:38:25,998] jepsen worker 89 - jepsen.util 13049 :invoke :cas [29 [37 24]] INFO [2019-12-17 12:38:26,001] jepsen worker 66 - jepsen.util 18066 :info :write [22 21] :RequestCanceledException INFO [2019-12-17 12:38:26,002] jepsen worker 66 - jepsen.util 18156 :invoke :write [22 48] INFO [2019-12-17 12:38:26,005] jepsen worker 74 - jepsen.util 12944 :ok :read [24 26] INFO [2019-12-17 12:38:26,005] jepsen worker 74 - jepsen.util 12944 :invoke :write [24 41] INFO [2019-12-17 12:38:26,010] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76277,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:38:26,010] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76277,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:38:26,010] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76278,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:38:26,010] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76278,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:38:26,018] jepsen worker 36 - jepsen.util 13806 :fail :cas [12 [25 26]] :ValueNotSwapFrom INFO [2019-12-17 12:38:26,018] jepsen worker 36 - jepsen.util 13806 :invoke :write [12 19] INFO [2019-12-17 12:38:26,020] jepsen worker 37 - jepsen.util 12727 :ok :read [12 14] INFO [2019-12-17 12:38:26,020] jepsen worker 37 - jepsen.util 12727 :invoke :write [12 26] INFO [2019-12-17 12:38:26,030] jepsen worker 53 - jepsen.util 12743 :ok :write [17 9] INFO [2019-12-17 12:38:26,031] jepsen worker 53 - jepsen.util 12743 :invoke :cas [17 [11 36]] INFO [2019-12-17 12:38:26,034] jepsen worker 33 - jepsen.util 17853 :info :cas [11 [48 15]] :RequestCanceledException INFO [2019-12-17 12:38:26,036] jepsen worker 33 - jepsen.util 17943 :invoke :cas [11 [13 11]] INFO [2019-12-17 12:38:26,037] jepsen worker 38 - jepsen.util 13808 :fail :cas [12 [13 28]] :ValueNotSwapFrom INFO [2019-12-17 12:38:26,037] jepsen worker 38 - jepsen.util 13808 :invoke :write [12 36] INFO [2019-12-17 12:38:26,067] jepsen worker 47 - jepsen.util 14447 :ok :read [15 12] INFO [2019-12-17 12:38:26,067] jepsen worker 47 - jepsen.util 14447 :invoke :read [15 nil] INFO [2019-12-17 12:38:26,070] jepsen worker 65 - jepsen.util 11585 :ok :read [21 37] INFO [2019-12-17 12:38:26,070] jepsen worker 65 - jepsen.util 11585 :invoke :read [21 nil] INFO [2019-12-17 12:38:26,083] jepsen worker 5 - jepsen.util 12785 :ok :read [1 44] INFO [2019-12-17 12:38:26,083] jepsen worker 5 - jepsen.util 12785 :invoke :read [1 nil] INFO [2019-12-17 12:38:26,096] jepsen worker 9 - jepsen.util 13599 :ok :read [3 13] INFO [2019-12-17 12:38:26,096] jepsen worker 9 - jepsen.util 13599 :invoke :read [3 nil] INFO [2019-12-17 12:38:26,099] jepsen worker 12 - jepsen.util 13242 :ok :read [4 39] INFO [2019-12-17 12:38:26,099] jepsen worker 12 - jepsen.util 13242 :invoke :write [4 31] INFO [2019-12-17 12:38:26,135] jepsen worker 67 - jepsen.util 18157 :info :write [22 2] :RequestCanceledException INFO [2019-12-17 12:38:26,135] jepsen worker 85 - jepsen.util 18445 :info :cas [28 [44 8]] :RequestCanceledException INFO [2019-12-17 12:38:26,135] jepsen worker 85 - jepsen.util 18535 :invoke :write [28 11] INFO [2019-12-17 12:38:26,136] jepsen worker 67 - jepsen.util 18247 :invoke :write [22 35] INFO [2019-12-17 12:38:26,159] jepsen worker 51 - jepsen.util 12381 :ok :write [17 18] INFO [2019-12-17 12:38:26,159] jepsen worker 51 - jepsen.util 12381 :invoke :cas [17 [34 12]] INFO [2019-12-17 12:38:26,186] jepsen worker 78 - jepsen.util 12858 :ok :write [26 7] INFO [2019-12-17 12:38:26,186] jepsen worker 78 - jepsen.util 12858 :invoke :write [26 37] INFO [2019-12-17 12:38:26,189] jepsen worker 70 - jepsen.util 13750 :fail :cas [23 [48 28]] :ValueNotSwapFrom INFO [2019-12-17 12:38:26,189] jepsen worker 70 - jepsen.util 13750 :invoke :read [23 nil] INFO [2019-12-17 12:38:26,195] jepsen worker 89 - jepsen.util 13049 :fail :cas [29 [37 24]] :ValueNotSwapFrom INFO [2019-12-17 12:38:26,195] jepsen worker 89 - jepsen.util 13049 :invoke :write [29 31] INFO [2019-12-17 12:38:26,216] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76307,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:38:26,216] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76307,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:38:26,216] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76311,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:38:26,216] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76311,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:38:26,235] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76300,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:38:26,235] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76300,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:38:26,235] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76302,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:38:26,235] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76302,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:38:26,240] jepsen worker 11 - jepsen.util 13061 :ok :read [3 13] INFO [2019-12-17 12:38:26,240] jepsen worker 11 - jepsen.util 13061 :invoke :read [3 nil] INFO [2019-12-17 12:38:26,251] jepsen worker 37 - jepsen.util 12727 :ok :write [12 26] INFO [2019-12-17 12:38:26,251] jepsen worker 37 - jepsen.util 12727 :invoke :read [12 nil] INFO [2019-12-17 12:38:26,263] jepsen worker 72 - jepsen.util 13302 :ok :read [24 26] INFO [2019-12-17 12:38:26,263] jepsen worker 72 - jepsen.util 13302 :invoke :read [24 nil] INFO [2019-12-17 12:38:26,266] jepsen worker 21 - jepsen.util 17571 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:38:26,266] jepsen worker 21 - jepsen.util 17571 :invoke :cas [7 [6 0]] INFO [2019-12-17 12:38:26,266] jepsen worker 32 - jepsen.util 13442 :fail :cas [10 [45 4]] :ValueNotSwapFrom INFO [2019-12-17 12:38:26,266] jepsen worker 32 - jepsen.util 13442 :invoke :cas [10 [47 27]] INFO [2019-12-17 12:38:26,283] jepsen worker 5 - jepsen.util 12785 :ok :read [1 44] INFO [2019-12-17 12:38:26,283] jepsen worker 5 - jepsen.util 12785 :invoke :cas [1 [20 1]] INFO [2019-12-17 12:38:26,283] jepsen worker 31 - jepsen.util 11191 :ok :read [10 25] INFO [2019-12-17 12:38:26,283] jepsen worker 31 - jepsen.util 11191 :invoke :read [10 nil] INFO [2019-12-17 12:38:26,301] jepsen worker 84 - jepsen.util 17454 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:38:26,301] jepsen worker 84 - jepsen.util 17454 :invoke :read [28 nil] INFO [2019-12-17 12:38:26,319] jepsen worker 0 - jepsen.util 18360 :info :write [0 46] :RequestCanceledException INFO [2019-12-17 12:38:26,319] jepsen worker 29 - jepsen.util 11369 :fail :read [9 nil] :RequestTimeoutException INFO [2019-12-17 12:38:26,319] jepsen worker 29 - jepsen.util 11369 :invoke :write [9 6] INFO [2019-12-17 12:38:26,321] jepsen worker 0 - jepsen.util 18450 :invoke :cas [0 [39 38]] INFO [2019-12-17 12:38:26,345] jepsen worker 87 - jepsen.util 12777 :ok :read [29 11] INFO [2019-12-17 12:38:26,345] jepsen worker 87 - jepsen.util 12777 :invoke :read [29 nil] INFO [2019-12-17 12:38:26,349] jepsen worker 56 - jepsen.util 13826 :ok :write [18 26] INFO [2019-12-17 12:38:26,349] jepsen worker 56 - jepsen.util 13826 :invoke :cas [18 [4 43]] INFO [2019-12-17 12:38:26,366] jepsen worker 64 - jepsen.util 10414 :fail :cas [21 [14 30]] :ValueNotSwapFrom INFO [2019-12-17 12:38:26,366] jepsen worker 64 - jepsen.util 10414 :invoke :write [21 4] INFO [2019-12-17 12:38:26,368] jepsen worker 13 - jepsen.util 12703 :fail :cas [4 [1 48]] :ValueNotSwapFrom INFO [2019-12-17 12:38:26,369] jepsen worker 13 - jepsen.util 12703 :invoke :cas [4 [20 0]] INFO [2019-12-17 12:38:26,386] jepsen worker 53 - jepsen.util 12743 :fail :cas [17 [11 36]] :ValueNotSwapFrom INFO [2019-12-17 12:38:26,387] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:38:26,387] jepsen worker 53 - jepsen.util 12743 :invoke :read [17 nil] INFO [2019-12-17 12:38:26,387] jepsen nemesis - couchbase.util Rebalance check count: 600 INFO [2019-12-17 12:38:26,387] jepsen nemesis - couchbase.util Rebalance stuck count: 581 INFO [2019-12-17 12:38:26,395] jepsen worker 63 - jepsen.util 11673 :ok :write [21 30] INFO [2019-12-17 12:38:26,395] jepsen worker 63 - jepsen.util 11673 :invoke :cas [21 [24 21]] INFO [2019-12-17 12:38:26,413] jepsen worker 3 - jepsen.util 12783 :fail :cas [1 [28 6]] :ValueNotSwapFrom INFO [2019-12-17 12:38:26,413] jepsen worker 3 - jepsen.util 12783 :invoke :write [1 38] INFO [2019-12-17 12:38:26,417] jepsen worker 23 - jepsen.util 18203 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:38:26,417] jepsen worker 23 - jepsen.util 18203 :invoke :write [7 5] INFO [2019-12-17 12:38:26,428] jepsen worker 11 - jepsen.util 13061 :ok :read [3 13] INFO [2019-12-17 12:38:26,428] jepsen worker 11 - jepsen.util 13061 :invoke :cas [3 [30 45]] INFO [2019-12-17 12:38:26,428] jepsen worker 30 - jepsen.util 12540 :ok :read [10 25] INFO [2019-12-17 12:38:26,428] jepsen worker 30 - jepsen.util 12540 :invoke :write [10 46] INFO [2019-12-17 12:38:26,434] jepsen worker 82 - jepsen.util 17992 :info :write [27 45] :RequestCanceledException INFO [2019-12-17 12:38:26,434] jepsen worker 71 - jepsen.util 14021 :info :write [23 19] :RequestCanceledException INFO [2019-12-17 12:38:26,435] jepsen worker 82 - jepsen.util 18082 :invoke :read [27 nil] INFO [2019-12-17 12:38:26,435] jepsen worker 71 - jepsen.util 14111 :invoke :cas [23 [33 14]] INFO [2019-12-17 12:38:26,438] jepsen worker 70 - jepsen.util 13750 :ok :read [23 39] INFO [2019-12-17 12:38:26,438] jepsen worker 70 - jepsen.util 13750 :invoke :cas [23 [12 33]] INFO [2019-12-17 12:38:26,440] jepsen worker 88 - jepsen.util 13138 :ok :read [29 11] INFO [2019-12-17 12:38:26,440] jepsen worker 88 - jepsen.util 13138 :invoke :read [29 nil] INFO [2019-12-17 12:38:26,445] jepsen worker 63 - jepsen.util 11673 :fail :cas [21 [24 21]] :ValueNotSwapFrom INFO [2019-12-17 12:38:26,445] jepsen worker 63 - jepsen.util 11673 :invoke :cas [21 [26 41]] INFO [2019-12-17 12:38:26,453] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76333,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:38:26,453] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76333,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:38:26,453] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76336,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:38:26,453] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76336,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:38:26,453] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76337,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:38:26,453] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76337,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:38:26,458] jepsen worker 63 - jepsen.util 11673 :fail :cas [21 [26 41]] :ValueNotSwapFrom INFO [2019-12-17 12:38:26,459] jepsen worker 63 - jepsen.util 11673 :invoke :write [21 34] INFO [2019-12-17 12:38:26,466] jepsen worker 5 - jepsen.util 12785 :fail :cas [1 [20 1]] :ValueNotSwapFrom INFO [2019-12-17 12:38:26,466] jepsen worker 5 - jepsen.util 12785 :invoke :read [1 nil] INFO [2019-12-17 12:38:26,478] jepsen worker 64 - jepsen.util 10414 :ok :write [21 4] INFO [2019-12-17 12:38:26,478] jepsen worker 64 - jepsen.util 10414 :invoke :cas [21 [12 38]] INFO [2019-12-17 12:38:26,484] jepsen worker 14 - jepsen.util 12794 :fail :cas [4 [2 18]] :ValueNotSwapFrom INFO [2019-12-17 12:38:26,484] jepsen worker 14 - jepsen.util 12794 :invoke :cas [4 [44 39]] INFO [2019-12-17 12:38:26,511] jepsen worker 74 - jepsen.util 12944 :ok :write [24 41] INFO [2019-12-17 12:38:26,511] jepsen worker 74 - jepsen.util 12944 :invoke :write [24 0] INFO [2019-12-17 12:38:26,513] jepsen worker 51 - jepsen.util 12381 :fail :cas [17 [34 12]] :ValueNotSwapFrom INFO [2019-12-17 12:38:26,513] jepsen worker 51 - jepsen.util 12381 :invoke :write [17 6] INFO [2019-12-17 12:38:26,517] jepsen worker 46 - jepsen.util 12826 :ok :write [15 16] INFO [2019-12-17 12:38:26,517] jepsen worker 46 - jepsen.util 12826 :invoke :cas [15 [5 34]] INFO [2019-12-17 12:38:26,518] jepsen worker 11 - jepsen.util 13061 :fail :cas [3 [30 45]] :ValueNotSwapFrom INFO [2019-12-17 12:38:26,519] jepsen worker 11 - jepsen.util 13061 :invoke :read [3 nil] INFO [2019-12-17 12:38:26,537] jepsen worker 56 - jepsen.util 13826 :fail :cas [18 [4 43]] :ValueNotSwapFrom INFO [2019-12-17 12:38:26,537] jepsen worker 56 - jepsen.util 13826 :invoke :write [18 28] INFO [2019-12-17 12:38:26,541] jepsen worker 64 - jepsen.util 10414 :fail :cas [21 [12 38]] :ValueNotSwapFrom INFO [2019-12-17 12:38:26,541] jepsen worker 64 - jepsen.util 10414 :invoke :cas [21 [37 3]] INFO [2019-12-17 12:38:26,569] jepsen worker 11 - jepsen.util 13061 :ok :read [3 13] INFO [2019-12-17 12:38:26,569] jepsen worker 11 - jepsen.util 13061 :invoke :write [3 18] INFO [2019-12-17 12:38:26,572] jepsen worker 9 - jepsen.util 13599 :ok :read [3 13] INFO [2019-12-17 12:38:26,572] jepsen worker 9 - jepsen.util 13599 :invoke :cas [3 [15 44]] INFO [2019-12-17 12:38:26,574] jepsen worker 65 - jepsen.util 11585 :ok :read [21 4] INFO [2019-12-17 12:38:26,574] jepsen worker 65 - jepsen.util 11585 :invoke :cas [21 [42 38]] INFO [2019-12-17 12:38:26,601] jepsen worker 87 - jepsen.util 12777 :ok :read [29 11] INFO [2019-12-17 12:38:26,601] jepsen worker 87 - jepsen.util 12777 :invoke :write [29 26] INFO [2019-12-17 12:38:26,606] jepsen worker 63 - jepsen.util 11673 :ok :write [21 34] INFO [2019-12-17 12:38:26,606] jepsen worker 63 - jepsen.util 11673 :invoke :write [21 29] INFO [2019-12-17 12:38:26,611] jepsen worker 13 - jepsen.util 12703 :fail :cas [4 [20 0]] :ValueNotSwapFrom INFO [2019-12-17 12:38:26,611] jepsen worker 13 - jepsen.util 12703 :invoke :cas [4 [1 39]] INFO [2019-12-17 12:38:26,617] jepsen worker 14 - jepsen.util 12794 :fail :cas [4 [44 39]] :ValueNotSwapFrom INFO [2019-12-17 12:38:26,617] jepsen worker 14 - jepsen.util 12794 :invoke :cas [4 [34 39]] INFO [2019-12-17 12:38:26,634] jepsen worker 55 - jepsen.util 13915 :info :write [18 2] :RequestCanceledException INFO [2019-12-17 12:38:26,635] jepsen worker 55 - jepsen.util 14005 :invoke :read [18 nil] INFO [2019-12-17 12:38:26,654] jepsen worker 32 - jepsen.util 13442 :fail :cas [10 [47 27]] :ValueNotSwapFrom INFO [2019-12-17 12:38:26,654] jepsen worker 32 - jepsen.util 13442 :invoke :write [10 39] INFO [2019-12-17 12:38:26,662] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76362,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:38:26,662] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76362,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:38:26,663] jepsen worker 77 - jepsen.util 19247 :info :write [25 29] :RequestCanceledException INFO [2019-12-17 12:38:26,663] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76366,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:38:26,663] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76366,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:38:26,670] jepsen worker 77 - jepsen.util 19337 :invoke :read [25 nil] INFO [2019-12-17 12:38:26,698] jepsen worker 64 - jepsen.util 10414 :fail :cas [21 [37 3]] :ValueNotSwapFrom INFO [2019-12-17 12:38:26,698] jepsen worker 64 - jepsen.util 10414 :invoke :cas [21 [8 21]] INFO [2019-12-17 12:38:26,717] jepsen worker 37 - jepsen.util 12727 :ok :read [12 26] INFO [2019-12-17 12:38:26,717] jepsen worker 37 - jepsen.util 12727 :invoke :read [12 nil] INFO [2019-12-17 12:38:26,722] jepsen worker 31 - jepsen.util 11191 :ok :read [10 25] INFO [2019-12-17 12:38:26,723] jepsen worker 31 - jepsen.util 11191 :invoke :write [10 28] INFO [2019-12-17 12:38:26,730] jepsen worker 63 - jepsen.util 11673 :ok :write [21 29] INFO [2019-12-17 12:38:26,730] jepsen worker 63 - jepsen.util 11673 :invoke :cas [21 [48 34]] INFO [2019-12-17 12:38:26,752] jepsen worker 14 - jepsen.util 12794 :fail :cas [4 [34 39]] :ValueNotSwapFrom INFO [2019-12-17 12:38:26,752] jepsen worker 14 - jepsen.util 12794 :invoke :write [4 17] INFO [2019-12-17 12:38:26,772] jepsen worker 29 - jepsen.util 11369 :ok :write [9 6] INFO [2019-12-17 12:38:26,772] jepsen worker 29 - jepsen.util 11369 :invoke :write [9 21] INFO [2019-12-17 12:38:26,781] jepsen worker 88 - jepsen.util 13138 :ok :read [29 11] INFO [2019-12-17 12:38:26,781] jepsen worker 88 - jepsen.util 13138 :invoke :cas [29 [43 2]] INFO [2019-12-17 12:38:26,806] jepsen worker 53 - jepsen.util 12743 :ok :read [17 18] INFO [2019-12-17 12:38:26,806] jepsen worker 53 - jepsen.util 12743 :invoke :write [17 29] INFO [2019-12-17 12:38:26,819] jepsen worker 15 - jepsen.util 17205 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:38:26,819] jepsen worker 15 - jepsen.util 17205 :invoke :read [5 nil] INFO [2019-12-17 12:38:26,821] jepsen worker 63 - jepsen.util 11673 :fail :cas [21 [48 34]] :ValueNotSwapFrom INFO [2019-12-17 12:38:26,821] jepsen worker 63 - jepsen.util 11673 :invoke :read [21 nil] INFO [2019-12-17 12:38:26,825] jepsen worker 30 - jepsen.util 12540 :ok :write [10 46] INFO [2019-12-17 12:38:26,825] jepsen worker 30 - jepsen.util 12540 :invoke :write [10 14] INFO [2019-12-17 12:38:26,835] jepsen worker 16 - jepsen.util 19636 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:38:26,835] jepsen worker 16 - jepsen.util 19636 :invoke :write [5 3] INFO [2019-12-17 12:38:26,836] jepsen worker 35 - jepsen.util 18125 :info :write [11 36] :RequestCanceledException INFO [2019-12-17 12:38:26,842] jepsen worker 35 - jepsen.util 18215 :invoke :cas [11 [10 4]] INFO [2019-12-17 12:38:26,842] jepsen worker 11 - jepsen.util 13061 :ok :write [3 18] INFO [2019-12-17 12:38:26,842] jepsen worker 11 - jepsen.util 13061 :invoke :cas [3 [22 12]] INFO [2019-12-17 12:38:26,865] jepsen worker 79 - jepsen.util 13309 :info :write [26 45] :RequestCanceledException INFO [2019-12-17 12:38:26,865] jepsen worker 58 - jepsen.util 18238 :info :cas [19 [37 44]] :RequestCanceledException INFO [2019-12-17 12:38:26,872] jepsen worker 58 - jepsen.util 18328 :invoke :read [19 nil] INFO [2019-12-17 12:38:26,875] jepsen worker 79 - jepsen.util 13399 :invoke :cas [26 [13 6]] INFO [2019-12-17 12:38:26,883] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76387,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:38:26,883] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76387,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:38:26,883] jepsen worker 71 - jepsen.util 14111 :fail :cas [23 [33 14]] :ValueNotSwapFrom INFO [2019-12-17 12:38:26,883] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76388,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:38:26,883] jepsen worker 71 - jepsen.util 14111 :invoke :read [23 nil] INFO [2019-12-17 12:38:26,883] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76388,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:38:26,883] jepsen worker 37 - jepsen.util 12727 :ok :read [12 26] INFO [2019-12-17 12:38:26,883] jepsen worker 37 - jepsen.util 12727 :invoke :write [12 21] INFO [2019-12-17 12:38:26,896] jepsen worker 2 - jepsen.util 17192 :info :write [0 24] :RequestCanceledException INFO [2019-12-17 12:38:26,901] jepsen worker 2 - jepsen.util 17282 :invoke :cas [0 [27 30]] INFO [2019-12-17 12:38:26,914] jepsen worker 64 - jepsen.util 10414 :fail :cas [21 [8 21]] :ValueNotSwapFrom INFO [2019-12-17 12:38:26,914] jepsen worker 64 - jepsen.util 10414 :invoke :cas [21 [23 18]] INFO [2019-12-17 12:38:26,918] jepsen worker 25 - jepsen.util 19105 :info :cas [8 [4 45]] :RequestCanceledException INFO [2019-12-17 12:38:26,920] jepsen worker 11 - jepsen.util 13061 :fail :cas [3 [22 12]] :ValueNotSwapFrom INFO [2019-12-17 12:38:26,920] jepsen worker 11 - jepsen.util 13061 :invoke :read [3 nil] INFO [2019-12-17 12:38:26,921] jepsen worker 25 - jepsen.util 19195 :invoke :write [8 46] INFO [2019-12-17 12:38:26,925] jepsen worker 9 - jepsen.util 13599 :fail :cas [3 [15 44]] :ValueNotSwapFrom INFO [2019-12-17 12:38:26,925] jepsen worker 9 - jepsen.util 13599 :invoke :read [3 nil] INFO [2019-12-17 12:38:26,930] jepsen worker 63 - jepsen.util 11673 :ok :read [21 29] INFO [2019-12-17 12:38:26,930] jepsen worker 63 - jepsen.util 11673 :invoke :cas [21 [30 22]] INFO [2019-12-17 12:38:26,934] jepsen worker 81 - jepsen.util 17451 :info :cas [27 [2 8]] :RequestCanceledException INFO [2019-12-17 12:38:26,934] jepsen worker 43 - jepsen.util 18223 :info :write [14 35] :RequestCanceledException INFO [2019-12-17 12:38:26,935] jepsen worker 28 - jepsen.util 11098 :info :write [9 37] :RequestCanceledException INFO [2019-12-17 12:38:26,936] jepsen worker 28 - jepsen.util 11188 :invoke :read [9 nil] INFO [2019-12-17 12:38:26,937] jepsen worker 81 - jepsen.util 17541 :invoke :cas [27 [17 2]] INFO [2019-12-17 12:38:26,937] jepsen worker 43 - jepsen.util 18313 :invoke :read [14 nil] INFO [2019-12-17 12:38:26,948] jepsen worker 44 - jepsen.util 18764 :info :cas [14 [11 29]] :RequestCanceledException INFO [2019-12-17 12:38:26,952] jepsen worker 44 - jepsen.util 18854 :invoke :cas [14 [14 16]] INFO [2019-12-17 12:38:26,976] jepsen worker 64 - jepsen.util 10414 :fail :cas [21 [23 18]] :ValueNotSwapFrom INFO [2019-12-17 12:38:26,977] jepsen worker 64 - jepsen.util 10414 :invoke :cas [21 [31 14]] INFO [2019-12-17 12:38:26,983] jepsen worker 32 - jepsen.util 13442 :ok :write [10 39] INFO [2019-12-17 12:38:26,983] jepsen worker 32 - jepsen.util 13442 :invoke :cas [10 [14 36]] INFO [2019-12-17 12:38:27,001] jepsen worker 69 - jepsen.util 13479 :info :write [23 0] :RequestCanceledException INFO [2019-12-17 12:38:27,005] jepsen worker 69 - jepsen.util 13569 :invoke :read [23 nil] INFO [2019-12-17 12:38:27,005] jepsen worker 11 - jepsen.util 13061 :ok :read [3 18] INFO [2019-12-17 12:38:27,005] jepsen worker 11 - jepsen.util 13061 :invoke :read [3 nil] INFO [2019-12-17 12:38:27,033] jepsen worker 48 - jepsen.util 16698 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:38:27,033] jepsen worker 57 - jepsen.util 17517 :info :cas [19 [35 46]] :RequestCanceledException INFO [2019-12-17 12:38:27,033] jepsen worker 48 - jepsen.util 16698 :invoke :cas [16 [27 35]] INFO [2019-12-17 12:38:27,034] jepsen worker 50 - jepsen.util 17870 :info :cas [16 [43 33]] :RequestCanceledException INFO [2019-12-17 12:38:27,039] jepsen worker 57 - jepsen.util 17607 :invoke :write [19 15] INFO [2019-12-17 12:38:27,044] jepsen worker 50 - jepsen.util 17960 :invoke :read [16 nil] INFO [2019-12-17 12:38:27,049] jepsen worker 13 - jepsen.util 12703 :fail :cas [4 [1 39]] :ValueNotSwapFrom INFO [2019-12-17 12:38:27,049] jepsen worker 13 - jepsen.util 12703 :invoke :write [4 46] INFO [2019-12-17 12:38:27,049] jepsen worker 7 - jepsen.util 17737 :info :write [2 7] :RequestCanceledException INFO [2019-12-17 12:38:27,057] jepsen worker 7 - jepsen.util 17827 :invoke :write [2 40] INFO [2019-12-17 12:38:27,067] jepsen worker 8 - jepsen.util 17558 :info :write [2 10] :RequestCanceledException INFO [2019-12-17 12:38:27,070] jepsen worker 88 - jepsen.util 13138 :fail :cas [29 [43 2]] :ValueNotSwapFrom INFO [2019-12-17 12:38:27,070] jepsen worker 88 - jepsen.util 13138 :invoke :cas [29 [3 39]] INFO [2019-12-17 12:38:27,076] jepsen worker 8 - jepsen.util 17648 :invoke :cas [2 [22 7]] INFO [2019-12-17 12:38:27,079] jepsen worker 63 - jepsen.util 11673 :fail :cas [21 [30 22]] :ValueNotSwapFrom INFO [2019-12-17 12:38:27,079] jepsen worker 63 - jepsen.util 11673 :invoke :cas [21 [42 36]] INFO [2019-12-17 12:38:27,080] jepsen worker 28 - jepsen.util 11188 :ok :read [9 6] INFO [2019-12-17 12:38:27,080] jepsen worker 28 - jepsen.util 11188 :invoke :cas [9 [42 45]] INFO [2019-12-17 12:38:27,101] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76405,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:38:27,101] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76405,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:38:27,102] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76410,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:38:27,102] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76410,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:38:27,109] jepsen worker 9 - jepsen.util 13599 :ok :read [3 18] INFO [2019-12-17 12:38:27,109] jepsen worker 9 - jepsen.util 13599 :invoke :write [3 17] INFO [2019-12-17 12:38:27,109] jepsen worker 64 - jepsen.util 10414 :fail :cas [21 [31 14]] :ValueNotSwapFrom INFO [2019-12-17 12:38:27,109] jepsen worker 64 - jepsen.util 10414 :invoke :cas [21 [1 18]] INFO [2019-12-17 12:38:27,118] jepsen worker 32 - jepsen.util 13442 :fail :cas [10 [14 36]] :ValueNotSwapFrom INFO [2019-12-17 12:38:27,118] jepsen worker 32 - jepsen.util 13442 :invoke :read [10 nil] INFO [2019-12-17 12:38:27,120] jepsen worker 11 - jepsen.util 13061 :ok :read [3 18] INFO [2019-12-17 12:38:27,120] jepsen worker 11 - jepsen.util 13061 :invoke :cas [3 [43 24]] INFO [2019-12-17 12:38:27,135] jepsen worker 61 - jepsen.util 19231 :info :cas [20 [13 9]] :RequestCanceledException INFO [2019-12-17 12:38:27,138] jepsen worker 61 - jepsen.util 19321 :invoke :write [20 30] INFO [2019-12-17 12:38:27,161] jepsen worker 28 - jepsen.util 11188 :fail :cas [9 [42 45]] :ValueNotSwapFrom INFO [2019-12-17 12:38:27,161] jepsen worker 28 - jepsen.util 11188 :invoke :read [9 nil] INFO [2019-12-17 12:38:27,166] jepsen worker 59 - jepsen.util 17069 :info :cas [19 [2 41]] :RequestCanceledException INFO [2019-12-17 12:38:27,172] jepsen worker 59 - jepsen.util 17159 :invoke :read [19 nil] INFO [2019-12-17 12:38:27,176] jepsen worker 63 - jepsen.util 11673 :fail :cas [21 [42 36]] :ValueNotSwapFrom INFO [2019-12-17 12:38:27,176] jepsen worker 63 - jepsen.util 11673 :invoke :read [21 nil] INFO [2019-12-17 12:38:27,190] jepsen worker 64 - jepsen.util 10414 :fail :cas [21 [1 18]] :ValueNotSwapFrom INFO [2019-12-17 12:38:27,190] jepsen worker 64 - jepsen.util 10414 :invoke :read [21 nil] INFO [2019-12-17 12:38:27,196] jepsen worker 32 - jepsen.util 13442 :ok :read [10 39] INFO [2019-12-17 12:38:27,197] jepsen worker 32 - jepsen.util 13442 :invoke :write [10 47] INFO [2019-12-17 12:38:27,210] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76432,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:38:27,210] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76432,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:38:27,218] jepsen worker 11 - jepsen.util 13061 :fail :cas [3 [43 24]] :ValueNotSwapFrom INFO [2019-12-17 12:38:27,218] jepsen worker 11 - jepsen.util 13061 :invoke :write [3 49] INFO [2019-12-17 12:38:27,219] jepsen worker 83 - jepsen.util 17993 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:38:27,219] jepsen worker 20 - jepsen.util 12980 :fail :read [6 nil] :RequestTimeoutException INFO [2019-12-17 12:38:27,219] jepsen worker 83 - jepsen.util 17993 :invoke :read [27 nil] INFO [2019-12-17 12:38:27,219] jepsen worker 20 - jepsen.util 12980 :invoke :write [6 32] INFO [2019-12-17 12:38:27,234] jepsen worker 52 - jepsen.util 12742 :info :write [17 28] :RequestCanceledException INFO [2019-12-17 12:38:27,234] jepsen worker 42 - jepsen.util 18852 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:38:27,235] jepsen worker 42 - jepsen.util 18852 :invoke :read [14 nil] INFO [2019-12-17 12:38:27,236] jepsen worker 52 - jepsen.util 12832 :invoke :write [17 44] INFO [2019-12-17 12:38:27,248] jepsen worker 73 - jepsen.util 12223 :fail :read [24 nil] :RequestTimeoutException INFO [2019-12-17 12:38:27,249] jepsen worker 73 - jepsen.util 12223 :invoke :read [24 nil] INFO [2019-12-17 12:38:27,259] jepsen worker 5 - jepsen.util 12785 :ok :read [1 44] INFO [2019-12-17 12:38:27,260] jepsen worker 5 - jepsen.util 12785 :invoke :cas [1 [12 21]] INFO [2019-12-17 12:38:27,266] jepsen worker 39 - jepsen.util 17499 :info :write [13 26] :RequestCanceledException INFO [2019-12-17 12:38:27,266] jepsen worker 1 - jepsen.util 18451 :info :write [0 18] :RequestCanceledException INFO [2019-12-17 12:38:27,269] jepsen worker 1 - jepsen.util 18541 :invoke :read [0 nil] INFO [2019-12-17 12:38:27,270] jepsen worker 39 - jepsen.util 17589 :invoke :write [13 49] INFO [2019-12-17 12:38:27,281] jepsen worker 28 - jepsen.util 11188 :ok :read [9 6] INFO [2019-12-17 12:38:27,282] jepsen worker 28 - jepsen.util 11188 :invoke :cas [9 [48 0]] INFO [2019-12-17 12:38:27,282] jepsen worker 63 - jepsen.util 11673 :ok :read [21 29] INFO [2019-12-17 12:38:27,282] jepsen worker 63 - jepsen.util 11673 :invoke :write [21 9] INFO [2019-12-17 12:38:27,290] jepsen worker 64 - jepsen.util 10414 :ok :read [21 29] INFO [2019-12-17 12:38:27,290] jepsen worker 64 - jepsen.util 10414 :invoke :cas [21 [10 43]] INFO [2019-12-17 12:38:27,299] jepsen worker 9 - jepsen.util 13599 :ok :write [3 17] INFO [2019-12-17 12:38:27,299] jepsen worker 9 - jepsen.util 13599 :invoke :cas [3 [7 20]] INFO [2019-12-17 12:38:27,302] jepsen worker 22 - jepsen.util 18472 :info :write [7 3] :RequestCanceledException INFO [2019-12-17 12:38:27,305] jepsen worker 22 - jepsen.util 18562 :invoke :read [7 nil] INFO [2019-12-17 12:38:27,312] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76442,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:38:27,312] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76442,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:38:27,312] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76443,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:38:27,312] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76443,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:38:27,319] jepsen worker 73 - jepsen.util 12223 :ok :read [24 41] INFO [2019-12-17 12:38:27,319] jepsen worker 73 - jepsen.util 12223 :invoke :cas [24 [48 32]] INFO [2019-12-17 12:38:27,325] jepsen worker 36 - jepsen.util 13806 :ok :write [12 19] INFO [2019-12-17 12:38:27,325] jepsen worker 36 - jepsen.util 13806 :invoke :read [12 nil] INFO [2019-12-17 12:38:27,334] jepsen worker 18 - jepsen.util 13788 :info :cas [6 [23 10]] :RequestCanceledException INFO [2019-12-17 12:38:27,335] jepsen worker 18 - jepsen.util 13878 :invoke :write [6 13] INFO [2019-12-17 12:38:27,346] jepsen worker 76 - jepsen.util 18166 :info :cas [25 [13 5]] :RequestCanceledException INFO [2019-12-17 12:38:27,352] jepsen worker 76 - jepsen.util 18256 :invoke :write [25 4] INFO [2019-12-17 12:38:27,353] jepsen worker 70 - jepsen.util 13750 :fail :cas [23 [12 33]] :ValueNotSwapFrom INFO [2019-12-17 12:38:27,353] jepsen worker 70 - jepsen.util 13750 :invoke :write [23 19] INFO [2019-12-17 12:38:27,356] jepsen worker 28 - jepsen.util 11188 :fail :cas [9 [48 0]] :ValueNotSwapFrom INFO [2019-12-17 12:38:27,356] jepsen worker 28 - jepsen.util 11188 :invoke :cas [9 [29 11]] INFO [2019-12-17 12:38:27,365] jepsen worker 68 - jepsen.util 17168 :info :cas [22 [7 32]] :RequestCanceledException INFO [2019-12-17 12:38:27,371] jepsen worker 68 - jepsen.util 17258 :invoke :write [22 8] INFO [2019-12-17 12:38:27,404] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:38:27,404] jepsen nemesis - couchbase.util Rebalance check count: 601 INFO [2019-12-17 12:38:27,404] jepsen nemesis - couchbase.util Rebalance stuck count: 582 INFO [2019-12-17 12:38:27,408] jepsen worker 79 - jepsen.util 13399 :fail :cas [26 [13 6]] :ValueNotSwapFrom INFO [2019-12-17 12:38:27,408] jepsen worker 79 - jepsen.util 13399 :invoke :cas [26 [47 30]] INFO [2019-12-17 12:38:27,420] jepsen worker 73 - jepsen.util 12223 :fail :cas [24 [48 32]] :ValueNotSwapFrom INFO [2019-12-17 12:38:27,420] jepsen worker 73 - jepsen.util 12223 :invoke :read [24 nil] INFO [2019-12-17 12:38:27,435] jepsen worker 54 - jepsen.util 13824 :info :write [18 0] :RequestCanceledException INFO [2019-12-17 12:38:27,437] jepsen worker 54 - jepsen.util 13914 :invoke :read [18 nil] INFO [2019-12-17 12:38:27,441] jepsen worker 28 - jepsen.util 11188 :fail :cas [9 [29 11]] :ValueNotSwapFrom INFO [2019-12-17 12:38:27,441] jepsen worker 28 - jepsen.util 11188 :invoke :read [9 nil] INFO [2019-12-17 12:38:27,491] jepsen worker 88 - jepsen.util 13138 :fail :cas [29 [3 39]] :ValueNotSwapFrom INFO [2019-12-17 12:38:27,491] jepsen worker 88 - jepsen.util 13138 :invoke :cas [29 [47 1]] INFO [2019-12-17 12:38:27,495] jepsen worker 73 - jepsen.util 12223 :ok :read [24 41] INFO [2019-12-17 12:38:27,495] jepsen worker 73 - jepsen.util 12223 :invoke :write [24 9] INFO [2019-12-17 12:38:27,501] jepsen worker 6 - jepsen.util 18276 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:38:27,501] jepsen worker 6 - jepsen.util 18276 :invoke :read [2 nil] INFO [2019-12-17 12:38:27,505] jepsen worker 63 - jepsen.util 11673 :ok :write [21 9] INFO [2019-12-17 12:38:27,506] jepsen worker 63 - jepsen.util 11673 :invoke :cas [21 [1 3]] INFO [2019-12-17 12:38:27,524] jepsen worker 11 - jepsen.util 13061 :ok :write [3 49] INFO [2019-12-17 12:38:27,524] jepsen worker 11 - jepsen.util 13061 :invoke :cas [3 [30 29]] INFO [2019-12-17 12:38:27,534] jepsen worker 34 - jepsen.util 19024 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:38:27,534] jepsen worker 10 - jepsen.util 13060 :info :write [3 20] :RequestCanceledException INFO [2019-12-17 12:38:27,534] jepsen worker 49 - jepsen.util 18049 :info :cas [16 [26 40]] :RequestCanceledException INFO [2019-12-17 12:38:27,534] jepsen worker 34 - jepsen.util 19024 :invoke :read [11 nil] INFO [2019-12-17 12:38:27,535] jepsen worker 28 - jepsen.util 11188 :ok :read [9 6] INFO [2019-12-17 12:38:27,535] jepsen worker 28 - jepsen.util 11188 :invoke :cas [9 [28 49]] INFO [2019-12-17 12:38:27,538] jepsen worker 10 - jepsen.util 13150 :invoke :read [3 nil] INFO [2019-12-17 12:38:27,540] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76468,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:38:27,540] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76468,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:38:27,541] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76471,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:38:27,541] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76471,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:38:27,542] jepsen worker 49 - jepsen.util 18139 :invoke :cas [16 [46 10]] INFO [2019-12-17 12:38:27,544] jepsen worker 71 - jepsen.util 14111 :ok :read [23 19] INFO [2019-12-17 12:38:27,544] jepsen worker 71 - jepsen.util 14111 :invoke :write [23 8] INFO [2019-12-17 12:38:27,594] jepsen worker 63 - jepsen.util 11673 :fail :cas [21 [1 3]] :ValueNotSwapFrom INFO [2019-12-17 12:38:27,595] jepsen worker 63 - jepsen.util 11673 :invoke :write [21 31] INFO [2019-12-17 12:38:27,606] jepsen worker 5 - jepsen.util 12785 :fail :cas [1 [12 21]] :ValueNotSwapFrom INFO [2019-12-17 12:38:27,606] jepsen worker 5 - jepsen.util 12785 :invoke :write [1 49] INFO [2019-12-17 12:38:27,620] jepsen worker 28 - jepsen.util 11188 :fail :cas [9 [28 49]] :ValueNotSwapFrom INFO [2019-12-17 12:38:27,621] jepsen worker 28 - jepsen.util 11188 :invoke :write [9 1] INFO [2019-12-17 12:38:27,634] jepsen worker 4 - jepsen.util 12244 :info :write [1 42] :RequestCanceledException INFO [2019-12-17 12:38:27,634] jepsen worker 27 - jepsen.util 12267 :fail :read [9 nil] :RequestTimeoutException INFO [2019-12-17 12:38:27,634] jepsen worker 27 - jepsen.util 12267 :invoke :read [9 nil] INFO [2019-12-17 12:38:27,640] jepsen worker 4 - jepsen.util 12334 :invoke :cas [1 [26 22]] INFO [2019-12-17 12:38:27,642] jepsen worker 69 - jepsen.util 13569 :ok :read [23 19] INFO [2019-12-17 12:38:27,642] jepsen worker 69 - jepsen.util 13569 :invoke :cas [23 [23 10]] INFO [2019-12-17 12:38:27,649] jepsen worker 40 - jepsen.util 18760 :info :write [13 36] :RequestCanceledException INFO [2019-12-17 12:38:27,651] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76494,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:38:27,652] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76494,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:38:27,653] jepsen worker 40 - jepsen.util 18850 :invoke :read [13 nil] INFO [2019-12-17 12:38:27,664] jepsen worker 41 - jepsen.util 19301 :info :write [13 45] :RequestCanceledException INFO [2019-12-17 12:38:27,669] jepsen worker 41 - jepsen.util 19391 :invoke :read [13 nil] INFO [2019-12-17 12:38:27,730] jepsen worker 63 - jepsen.util 11673 :ok :write [21 31] INFO [2019-12-17 12:38:27,730] jepsen worker 63 - jepsen.util 11673 :invoke :cas [21 [41 24]] INFO [2019-12-17 12:38:27,735] jepsen worker 24 - jepsen.util 17124 :info :write [8 36] :RequestCanceledException INFO [2019-12-17 12:38:27,740] jepsen worker 24 - jepsen.util 17214 :invoke :cas [8 [40 8]] INFO [2019-12-17 12:38:27,747] jepsen worker 28 - jepsen.util 11188 :ok :write [9 1] INFO [2019-12-17 12:38:27,747] jepsen worker 28 - jepsen.util 11188 :invoke :read [9 nil] INFO [2019-12-17 12:38:27,799] jepsen worker 27 - jepsen.util 12267 :ok :read [9 1] INFO [2019-12-17 12:38:27,799] jepsen worker 27 - jepsen.util 12267 :invoke :write [9 48] INFO [2019-12-17 12:38:27,803] jepsen worker 63 - jepsen.util 11673 :fail :cas [21 [41 24]] :ValueNotSwapFrom INFO [2019-12-17 12:38:27,803] jepsen worker 63 - jepsen.util 11673 :invoke :cas [21 [10 35]] INFO [2019-12-17 12:38:27,819] jepsen worker 19 - jepsen.util 13969 :info :write [6 3] :RequestCanceledException INFO [2019-12-17 12:38:27,819] jepsen worker 75 - jepsen.util 17805 :info :write [25 23] :RequestCanceledException INFO [2019-12-17 12:38:27,820] jepsen worker 19 - jepsen.util 14059 :invoke :cas [6 [11 19]] INFO [2019-12-17 12:38:27,825] jepsen worker 75 - jepsen.util 17895 :invoke :write [25 10] INFO [2019-12-17 12:38:27,832] jepsen worker 79 - jepsen.util 13399 :fail :cas [26 [47 30]] :ValueNotSwapFrom INFO [2019-12-17 12:38:27,832] jepsen worker 79 - jepsen.util 13399 :invoke :write [26 43] INFO [2019-12-17 12:38:27,834] jepsen worker 86 - jepsen.util 18806 :info :write [28 46] :RequestCanceledException INFO [2019-12-17 12:38:27,841] jepsen worker 86 - jepsen.util 18896 :invoke :write [28 47] INFO [2019-12-17 12:38:27,848] jepsen worker 62 - jepsen.util 18692 :info :cas [20 [22 30]] :RequestCanceledException INFO [2019-12-17 12:38:27,858] jepsen worker 62 - jepsen.util 18782 :invoke :cas [20 [5 12]] INFO [2019-12-17 12:38:27,860] jepsen worker 28 - jepsen.util 11188 :ok :read [9 1] INFO [2019-12-17 12:38:27,861] jepsen worker 28 - jepsen.util 11188 :invoke :write [9 0] INFO [2019-12-17 12:38:27,866] jepsen worker 26 - jepsen.util 19016 :info :write [8 29] :RequestCanceledException INFO [2019-12-17 12:38:27,867] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76510,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:38:27,868] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76510,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:38:27,868] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76511,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:38:27,868] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76511,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:38:27,868] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76512,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:38:27,869] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76512,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:38:27,873] jepsen worker 26 - jepsen.util 19106 :invoke :read [8 nil] INFO [2019-12-17 12:38:27,885] jepsen worker 88 - jepsen.util 13138 :fail :cas [29 [47 1]] :ValueNotSwapFrom INFO [2019-12-17 12:38:27,885] jepsen worker 88 - jepsen.util 13138 :invoke :read [29 nil] INFO [2019-12-17 12:38:27,902] jepsen worker 80 - jepsen.util 12590 :info :write [26 47] :RequestCanceledException INFO [2019-12-17 12:38:27,904] jepsen worker 80 - jepsen.util 12680 :invoke :cas [26 [28 14]] INFO [2019-12-17 12:38:27,910] jepsen worker 63 - jepsen.util 11673 :fail :cas [21 [10 35]] :ValueNotSwapFrom INFO [2019-12-17 12:38:27,910] jepsen worker 63 - jepsen.util 11673 :invoke :read [21 nil] INFO [2019-12-17 12:38:27,912] jepsen worker 27 - jepsen.util 12267 :ok :write [9 48] INFO [2019-12-17 12:38:27,912] jepsen worker 27 - jepsen.util 12267 :invoke :write [9 13] INFO [2019-12-17 12:38:27,919] jepsen worker 17 - jepsen.util 18017 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:38:27,919] jepsen worker 17 - jepsen.util 18017 :invoke :cas [5 [8 20]] INFO [2019-12-17 12:38:27,934] jepsen worker 60 - jepsen.util 19050 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:38:27,934] jepsen worker 60 - jepsen.util 19050 :invoke :cas [20 [8 40]] INFO [2019-12-17 12:38:27,970] jepsen worker 63 - jepsen.util 11673 :ok :read [21 31] INFO [2019-12-17 12:38:27,970] jepsen worker 63 - jepsen.util 11673 :invoke :write [21 49] INFO [2019-12-17 12:38:27,998] jepsen worker 28 - jepsen.util 11188 :ok :write [9 0] INFO [2019-12-17 12:38:27,998] jepsen worker 28 - jepsen.util 11188 :invoke :read [9 nil] INFO [2019-12-17 12:38:28,036] jepsen worker 4 - jepsen.util 12334 :fail :cas [1 [26 22]] :ValueNotSwapFrom INFO [2019-12-17 12:38:28,036] jepsen worker 4 - jepsen.util 12334 :invoke :read [1 nil] INFO [2019-12-17 12:38:28,042] jepsen worker 80 - jepsen.util 12680 :fail :cas [26 [28 14]] :ValueNotSwapFrom INFO [2019-12-17 12:38:28,042] jepsen worker 80 - jepsen.util 12680 :invoke :write [26 35] INFO [2019-12-17 12:38:28,047] jepsen worker 45 - jepsen.util 12735 :info :write [15 37] :RequestCanceledException INFO [2019-12-17 12:38:28,048] jepsen worker 88 - jepsen.util 13138 :ok :read [29 11] INFO [2019-12-17 12:38:28,048] jepsen worker 88 - jepsen.util 13138 :invoke :read [29 nil] INFO [2019-12-17 12:38:28,049] jepsen worker 45 - jepsen.util 12825 :invoke :cas [15 [26 21]] INFO [2019-12-17 12:38:28,066] jepsen worker 38 - jepsen.util 13808 :info :write [12 36] :RequestCanceledException INFO [2019-12-17 12:38:28,067] jepsen worker 38 - jepsen.util 13898 :invoke :cas [12 [34 34]] INFO [2019-12-17 12:38:28,087] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76526,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:38:28,087] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76526,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:38:28,087] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76529,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:38:28,087] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76529,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:38:28,092] jepsen worker 27 - jepsen.util 12267 :ok :write [9 13] INFO [2019-12-17 12:38:28,093] jepsen worker 27 - jepsen.util 12267 :invoke :read [9 nil] INFO [2019-12-17 12:38:28,099] jepsen worker 66 - jepsen.util 18156 :info :write [22 48] :RequestCanceledException INFO [2019-12-17 12:38:28,105] jepsen worker 66 - jepsen.util 18246 :invoke :cas [22 [33 24]] INFO [2019-12-17 12:38:28,108] jepsen worker 63 - jepsen.util 11673 :ok :write [21 49] INFO [2019-12-17 12:38:28,109] jepsen worker 63 - jepsen.util 11673 :invoke :read [21 nil] INFO [2019-12-17 12:38:28,118] jepsen worker 33 - jepsen.util 17943 :info :cas [11 [13 11]] :RequestCanceledException INFO [2019-12-17 12:38:28,121] jepsen worker 33 - jepsen.util 18033 :invoke :read [11 nil] INFO [2019-12-17 12:38:28,128] jepsen worker 54 - jepsen.util 13914 :ok :read [18 26] INFO [2019-12-17 12:38:28,129] jepsen worker 54 - jepsen.util 13914 :invoke :read [18 nil] INFO [2019-12-17 12:38:28,137] jepsen worker 27 - jepsen.util 12267 :ok :read [9 13] INFO [2019-12-17 12:38:28,138] jepsen worker 27 - jepsen.util 12267 :invoke :read [9 nil] INFO [2019-12-17 12:38:28,148] jepsen worker 47 - jepsen.util 14447 :fail :read [15 nil] :RequestTimeoutException INFO [2019-12-17 12:38:28,149] jepsen worker 47 - jepsen.util 14447 :invoke :write [15 34] INFO [2019-12-17 12:38:28,165] jepsen worker 12 - jepsen.util 13242 :info :write [4 31] :RequestCanceledException INFO [2019-12-17 12:38:28,167] jepsen worker 12 - jepsen.util 13332 :invoke :read [4 nil] INFO [2019-12-17 12:38:28,183] jepsen worker 27 - jepsen.util 12267 :ok :read [9 13] INFO [2019-12-17 12:38:28,184] jepsen worker 27 - jepsen.util 12267 :invoke :read [9 nil] INFO [2019-12-17 12:38:28,185] jepsen worker 63 - jepsen.util 11673 :ok :read [21 49] INFO [2019-12-17 12:38:28,185] jepsen worker 63 - jepsen.util 11673 :invoke :cas [21 [25 26]] INFO [2019-12-17 12:38:28,210] jepsen worker 32 - jepsen.util 13442 :ok :write [10 47] INFO [2019-12-17 12:38:28,210] jepsen worker 32 - jepsen.util 13442 :invoke :cas [10 [5 38]] INFO [2019-12-17 12:38:28,211] jepsen worker 20 - jepsen.util 12980 :ok :write [6 32] INFO [2019-12-17 12:38:28,211] jepsen worker 20 - jepsen.util 12980 :invoke :write [6 34] INFO [2019-12-17 12:38:28,215] jepsen worker 73 - jepsen.util 12223 :ok :write [24 9] INFO [2019-12-17 12:38:28,215] jepsen worker 73 - jepsen.util 12223 :invoke :write [24 35] INFO [2019-12-17 12:38:28,220] jepsen worker 78 - jepsen.util 12858 :info :write [26 37] :RequestCanceledException INFO [2019-12-17 12:38:28,221] jepsen worker 78 - jepsen.util 12948 :invoke :write [26 41] INFO [2019-12-17 12:38:28,235] jepsen worker 85 - jepsen.util 18535 :info :write [28 11] :RequestCanceledException INFO [2019-12-17 12:38:28,241] jepsen worker 85 - jepsen.util 18625 :invoke :read [28 nil] INFO [2019-12-17 12:38:28,248] jepsen worker 67 - jepsen.util 18247 :info :write [22 35] :RequestCanceledException INFO [2019-12-17 12:38:28,258] jepsen worker 67 - jepsen.util 18337 :invoke :write [22 31] INFO [2019-12-17 12:38:28,266] jepsen worker 89 - jepsen.util 13049 :info :write [29 31] :RequestCanceledException INFO [2019-12-17 12:38:28,270] jepsen worker 89 - jepsen.util 13139 :invoke :read [29 nil] INFO [2019-12-17 12:38:28,286] jepsen worker 27 - jepsen.util 12267 :ok :read [9 13] INFO [2019-12-17 12:38:28,286] jepsen worker 27 - jepsen.util 12267 :invoke :read [9 nil] INFO [2019-12-17 12:38:28,288] jepsen worker 32 - jepsen.util 13442 :fail :cas [10 [5 38]] :ValueNotSwapFrom INFO [2019-12-17 12:38:28,288] jepsen worker 32 - jepsen.util 13442 :invoke :write [10 34] INFO [2019-12-17 12:38:28,289] jepsen worker 63 - jepsen.util 11673 :fail :cas [21 [25 26]] :ValueNotSwapFrom INFO [2019-12-17 12:38:28,291] jepsen worker 63 - jepsen.util 11673 :invoke :write [21 34] INFO [2019-12-17 12:38:28,300] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76554,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:38:28,300] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76554,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:38:28,300] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76557,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:38:28,300] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76557,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:38:28,300] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76558,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:38:28,300] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76558,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:38:28,306] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76546,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:38:28,306] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76546,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:38:28,306] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76548,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:38:28,306] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76548,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:38:28,324] jepsen worker 19 - jepsen.util 14059 :fail :cas [6 [11 19]] :ValueNotSwapFrom INFO [2019-12-17 12:38:28,324] jepsen worker 19 - jepsen.util 14059 :invoke :read [6 nil] INFO [2019-12-17 12:38:28,349] jepsen worker 72 - jepsen.util 13302 :fail :read [24 nil] :RequestTimeoutException INFO [2019-12-17 12:38:28,349] jepsen worker 72 - jepsen.util 13302 :invoke :cas [24 [36 9]] INFO [2019-12-17 12:38:28,357] jepsen worker 45 - jepsen.util 12825 :fail :cas [15 [26 21]] :ValueNotSwapFrom INFO [2019-12-17 12:38:28,357] jepsen worker 45 - jepsen.util 12825 :invoke :write [15 32] INFO [2019-12-17 12:38:28,366] jepsen worker 21 - jepsen.util 17571 :info :cas [7 [6 0]] :RequestCanceledException INFO [2019-12-17 12:38:28,366] jepsen worker 0 - jepsen.util 18450 :info :cas [0 [39 38]] :RequestCanceledException INFO [2019-12-17 12:38:28,372] jepsen worker 21 - jepsen.util 17661 :invoke :read [7 nil] INFO [2019-12-17 12:38:28,374] jepsen worker 0 - jepsen.util 18540 :invoke :read [0 nil] INFO [2019-12-17 12:38:28,387] jepsen worker 27 - jepsen.util 12267 :ok :read [9 13] INFO [2019-12-17 12:38:28,387] jepsen worker 27 - jepsen.util 12267 :invoke :read [9 nil] INFO [2019-12-17 12:38:28,392] jepsen worker 54 - jepsen.util 13914 :ok :read [18 26] INFO [2019-12-17 12:38:28,393] jepsen worker 54 - jepsen.util 13914 :invoke :write [18 37] INFO [2019-12-17 12:38:28,400] jepsen worker 84 - jepsen.util 17454 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:38:28,400] jepsen worker 84 - jepsen.util 17454 :invoke :cas [28 [1 16]] INFO [2019-12-17 12:38:28,409] jepsen worker 88 - jepsen.util 13138 :ok :read [29 11] INFO [2019-12-17 12:38:28,409] jepsen worker 88 - jepsen.util 13138 :invoke :cas [29 [22 7]] INFO [2019-12-17 12:38:28,425] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:38:28,425] jepsen nemesis - couchbase.util Rebalance check count: 602 INFO [2019-12-17 12:38:28,426] jepsen nemesis - couchbase.util Rebalance stuck count: 583 INFO [2019-12-17 12:38:28,435] jepsen worker 63 - jepsen.util 11673 :ok :write [21 34] INFO [2019-12-17 12:38:28,436] jepsen worker 63 - jepsen.util 11673 :invoke :cas [21 [29 21]] INFO [2019-12-17 12:38:28,448] jepsen worker 3 - jepsen.util 12783 :info :write [1 38] :RequestCanceledException INFO [2019-12-17 12:38:28,452] jepsen worker 3 - jepsen.util 12873 :invoke :cas [1 [6 1]] INFO [2019-12-17 12:38:28,456] jepsen worker 20 - jepsen.util 12980 :ok :write [6 34] INFO [2019-12-17 12:38:28,456] jepsen worker 20 - jepsen.util 12980 :invoke :write [6 35] INFO [2019-12-17 12:38:28,478] jepsen worker 47 - jepsen.util 14447 :ok :write [15 34] INFO [2019-12-17 12:38:28,478] jepsen worker 47 - jepsen.util 14447 :invoke :read [15 nil] INFO [2019-12-17 12:38:28,486] jepsen worker 27 - jepsen.util 12267 :ok :read [9 13] INFO [2019-12-17 12:38:28,486] jepsen worker 27 - jepsen.util 12267 :invoke :cas [9 [6 32]] INFO [2019-12-17 12:38:28,520] jepsen worker 23 - jepsen.util 18203 :info :write [7 5] :RequestCanceledException INFO [2019-12-17 12:38:28,524] jepsen worker 32 - jepsen.util 13442 :ok :write [10 34] INFO [2019-12-17 12:38:28,524] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76578,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:38:28,524] jepsen worker 32 - jepsen.util 13442 :invoke :write [10 49] INFO [2019-12-17 12:38:28,525] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76578,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:38:28,525] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76583,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:38:28,525] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76583,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:38:28,528] jepsen worker 23 - jepsen.util 18293 :invoke :read [7 nil] INFO [2019-12-17 12:38:28,530] jepsen worker 4 - jepsen.util 12334 :ok :read [1 44] INFO [2019-12-17 12:38:28,530] jepsen worker 4 - jepsen.util 12334 :invoke :write [1 5] INFO [2019-12-17 12:38:28,535] jepsen worker 82 - jepsen.util 18082 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:38:28,535] jepsen worker 82 - jepsen.util 18082 :invoke :cas [27 [29 46]] INFO [2019-12-17 12:38:28,551] jepsen worker 63 - jepsen.util 11673 :fail :cas [21 [29 21]] :ValueNotSwapFrom INFO [2019-12-17 12:38:28,552] jepsen worker 63 - jepsen.util 11673 :invoke :cas [21 [37 22]] INFO [2019-12-17 12:38:28,565] jepsen worker 51 - jepsen.util 12381 :info :write [17 6] :RequestCanceledException INFO [2019-12-17 12:38:28,571] jepsen worker 51 - jepsen.util 12471 :invoke :write [17 32] INFO [2019-12-17 12:38:28,601] jepsen worker 74 - jepsen.util 12944 :info :write [24 0] :RequestCanceledException INFO [2019-12-17 12:38:28,602] jepsen worker 74 - jepsen.util 13034 :invoke :write [24 3] INFO [2019-12-17 12:38:28,619] jepsen worker 65 - jepsen.util 11585 :info :cas [21 [42 38]] :RequestCanceledException INFO [2019-12-17 12:38:28,619] jepsen worker 46 - jepsen.util 12826 :info :cas [15 [5 34]] :RequestCanceledException INFO [2019-12-17 12:38:28,621] jepsen worker 65 - jepsen.util 11675 :invoke :cas [21 [33 10]] INFO [2019-12-17 12:38:28,622] jepsen worker 46 - jepsen.util 12916 :invoke :write [15 33] INFO [2019-12-17 12:38:28,624] jepsen worker 88 - jepsen.util 13138 :fail :cas [29 [22 7]] :ValueNotSwapFrom INFO [2019-12-17 12:38:28,624] jepsen worker 88 - jepsen.util 13138 :invoke :read [29 nil] INFO [2019-12-17 12:38:28,632] jepsen worker 19 - jepsen.util 14059 :ok :read [6 34] INFO [2019-12-17 12:38:28,632] jepsen worker 19 - jepsen.util 14059 :invoke :read [6 nil] INFO [2019-12-17 12:38:28,635] jepsen worker 72 - jepsen.util 13302 :fail :cas [24 [36 9]] :ValueNotSwapFrom INFO [2019-12-17 12:38:28,635] jepsen worker 72 - jepsen.util 13302 :invoke :cas [24 [47 6]] INFO [2019-12-17 12:38:28,638] jepsen worker 27 - jepsen.util 12267 :fail :cas [9 [6 32]] :ValueNotSwapFrom INFO [2019-12-17 12:38:28,638] jepsen worker 27 - jepsen.util 12267 :invoke :read [9 nil] INFO [2019-12-17 12:38:28,646] jepsen worker 36 - jepsen.util 13806 :ok :read [12 19] INFO [2019-12-17 12:38:28,646] jepsen worker 36 - jepsen.util 13806 :invoke :cas [12 [11 14]] INFO [2019-12-17 12:38:28,649] jepsen worker 56 - jepsen.util 13826 :info :write [18 28] :RequestCanceledException INFO [2019-12-17 12:38:28,651] jepsen worker 56 - jepsen.util 13916 :invoke :write [18 42] INFO [2019-12-17 12:38:28,667] jepsen worker 63 - jepsen.util 11673 :fail :cas [21 [37 22]] :ValueNotSwapFrom INFO [2019-12-17 12:38:28,668] jepsen worker 63 - jepsen.util 11673 :invoke :write [21 7] INFO [2019-12-17 12:38:28,701] jepsen worker 87 - jepsen.util 12777 :info :write [29 26] :RequestCanceledException INFO [2019-12-17 12:38:28,711] jepsen worker 87 - jepsen.util 12867 :invoke :cas [29 [5 22]] INFO [2019-12-17 12:38:28,719] jepsen worker 55 - jepsen.util 14005 :fail :read [18 nil] :RequestTimeoutException INFO [2019-12-17 12:38:28,719] jepsen worker 55 - jepsen.util 14005 :invoke :cas [18 [31 8]] INFO [2019-12-17 12:38:28,719] jepsen worker 27 - jepsen.util 12267 :ok :read [9 13] INFO [2019-12-17 12:38:28,720] jepsen worker 27 - jepsen.util 12267 :invoke :write [9 26] INFO [2019-12-17 12:38:28,735] jepsen worker 77 - jepsen.util 19337 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:38:28,735] jepsen worker 77 - jepsen.util 19337 :invoke :write [25 18] INFO [2019-12-17 12:38:28,744] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76603,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:38:28,745] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76603,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:38:28,745] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76606,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:38:28,745] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76606,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:38:28,745] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76607,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:38:28,745] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76607,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:38:28,778] jepsen worker 38 - jepsen.util 13898 :fail :cas [12 [34 34]] :ValueNotSwapFrom INFO [2019-12-17 12:38:28,778] jepsen worker 38 - jepsen.util 13898 :invoke :read [12 nil] INFO [2019-12-17 12:38:28,784] jepsen worker 63 - jepsen.util 11673 :ok :write [21 7] INFO [2019-12-17 12:38:28,784] jepsen worker 63 - jepsen.util 11673 :invoke :cas [21 [6 5]] INFO [2019-12-17 12:38:28,819] jepsen worker 29 - jepsen.util 11369 :info :write [9 21] :RequestCanceledException INFO [2019-12-17 12:38:28,819] jepsen worker 31 - jepsen.util 11191 :info :write [10 28] :RequestCanceledException INFO [2019-12-17 12:38:28,822] jepsen worker 27 - jepsen.util 12267 :ok :write [9 26] INFO [2019-12-17 12:38:28,823] jepsen worker 27 - jepsen.util 12267 :invoke :cas [9 [17 19]] INFO [2019-12-17 12:38:28,823] jepsen worker 31 - jepsen.util 11281 :invoke :read [10 nil] INFO [2019-12-17 12:38:28,827] jepsen worker 29 - jepsen.util 11459 :invoke :write [9 36] INFO [2019-12-17 12:38:28,832] jepsen worker 89 - jepsen.util 13139 :ok :read [29 11] INFO [2019-12-17 12:38:28,833] jepsen worker 89 - jepsen.util 13139 :invoke :write [29 5] INFO [2019-12-17 12:38:28,849] jepsen worker 14 - jepsen.util 12794 :info :write [4 17] :RequestCanceledException INFO [2019-12-17 12:38:28,851] jepsen worker 14 - jepsen.util 12884 :invoke :write [4 1] INFO [2019-12-17 12:38:28,883] jepsen worker 63 - jepsen.util 11673 :fail :cas [21 [6 5]] :ValueNotSwapFrom INFO [2019-12-17 12:38:28,884] jepsen worker 63 - jepsen.util 11673 :invoke :read [21 nil] INFO [2019-12-17 12:38:28,901] jepsen worker 30 - jepsen.util 12540 :info :write [10 14] :RequestCanceledException INFO [2019-12-17 12:38:28,901] jepsen worker 53 - jepsen.util 12743 :info :write [17 29] :RequestCanceledException INFO [2019-12-17 12:38:28,904] jepsen worker 30 - jepsen.util 12630 :invoke :write [10 47] INFO [2019-12-17 12:38:28,915] jepsen worker 27 - jepsen.util 12267 :fail :cas [9 [17 19]] :ValueNotSwapFrom INFO [2019-12-17 12:38:28,915] jepsen worker 27 - jepsen.util 12267 :invoke :write [9 10] INFO [2019-12-17 12:38:28,916] jepsen worker 53 - jepsen.util 12833 :invoke :read [17 nil] INFO [2019-12-17 12:38:28,918] jepsen worker 15 - jepsen.util 17205 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:38:28,918] jepsen worker 15 - jepsen.util 17205 :invoke :read [5 nil] INFO [2019-12-17 12:38:28,933] jepsen worker 12 - jepsen.util 13332 :ok :read [4 39] INFO [2019-12-17 12:38:28,934] jepsen worker 12 - jepsen.util 13332 :invoke :read [4 nil] INFO [2019-12-17 12:38:28,935] jepsen worker 16 - jepsen.util 19636 :info :write [5 3] :RequestCanceledException INFO [2019-12-17 12:38:28,943] jepsen worker 16 - jepsen.util 19726 :invoke :write [5 17] INFO [2019-12-17 12:38:28,947] jepsen worker 31 - jepsen.util 11281 :ok :read [10 34] INFO [2019-12-17 12:38:28,947] jepsen worker 31 - jepsen.util 11281 :invoke :read [10 nil] INFO [2019-12-17 12:38:28,949] jepsen worker 63 - jepsen.util 11673 :ok :read [21 7] INFO [2019-12-17 12:38:28,949] jepsen worker 63 - jepsen.util 11673 :invoke :cas [21 [4 9]] INFO [2019-12-17 12:38:28,949] jepsen worker 35 - jepsen.util 18215 :info :cas [11 [10 4]] :RequestCanceledException INFO [2019-12-17 12:38:28,949] jepsen worker 81 - jepsen.util 17541 :info :cas [27 [17 2]] :RequestCanceledException INFO [2019-12-17 12:38:28,953] jepsen worker 81 - jepsen.util 17631 :invoke :read [27 nil] INFO [2019-12-17 12:38:28,972] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76620,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:38:28,972] jepsen worker 37 - jepsen.util 12727 :info :write [12 21] :RequestCanceledException INFO [2019-12-17 12:38:28,972] jepsen worker 58 - jepsen.util 18328 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:38:28,972] jepsen worker 43 - jepsen.util 18313 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:38:28,972] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76620,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:38:28,972] jepsen worker 58 - jepsen.util 18328 :invoke :read [19 nil] INFO [2019-12-17 12:38:28,972] jepsen worker 43 - jepsen.util 18313 :invoke :read [14 nil] INFO [2019-12-17 12:38:28,972] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76623,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:38:28,973] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76623,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:38:28,978] jepsen worker 35 - jepsen.util 18305 :invoke :write [11 32] INFO [2019-12-17 12:38:28,978] jepsen worker 37 - jepsen.util 12817 :invoke :read [12 nil] INFO [2019-12-17 12:38:29,001] jepsen worker 2 - jepsen.util 17282 :info :cas [0 [27 30]] :RequestCanceledException INFO [2019-12-17 12:38:29,011] jepsen worker 2 - jepsen.util 17372 :invoke :cas [0 [5 47]] INFO [2019-12-17 12:38:29,020] jepsen worker 25 - jepsen.util 19195 :info :write [8 46] :RequestCanceledException INFO [2019-12-17 12:38:29,022] jepsen worker 27 - jepsen.util 12267 :ok :write [9 10] INFO [2019-12-17 12:38:29,022] jepsen worker 27 - jepsen.util 12267 :invoke :write [9 34] INFO [2019-12-17 12:38:29,028] jepsen worker 25 - jepsen.util 19285 :invoke :read [8 nil] INFO [2019-12-17 12:38:29,034] jepsen worker 44 - jepsen.util 18854 :info :cas [14 [14 16]] :RequestCanceledException INFO [2019-12-17 12:38:29,048] jepsen worker 44 - jepsen.util 18944 :invoke :read [14 nil] INFO [2019-12-17 12:38:29,050] jepsen worker 45 - jepsen.util 12825 :ok :write [15 32] INFO [2019-12-17 12:38:29,051] jepsen worker 45 - jepsen.util 12825 :invoke :read [15 nil] INFO [2019-12-17 12:38:29,053] jepsen worker 36 - jepsen.util 13806 :fail :cas [12 [11 14]] :ValueNotSwapFrom INFO [2019-12-17 12:38:29,053] jepsen worker 36 - jepsen.util 13806 :invoke :cas [12 [20 5]] INFO [2019-12-17 12:38:29,078] jepsen worker 31 - jepsen.util 11281 :ok :read [10 34] INFO [2019-12-17 12:38:29,079] jepsen worker 31 - jepsen.util 11281 :invoke :cas [10 [44 41]] INFO [2019-12-17 12:38:29,081] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76641,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:38:29,081] jepsen worker 55 - jepsen.util 14005 :fail :cas [18 [31 8]] :ValueNotSwapFrom INFO [2019-12-17 12:38:29,082] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76641,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:38:29,082] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76645,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:38:29,082] jepsen worker 55 - jepsen.util 14005 :invoke :cas [18 [46 26]] INFO [2019-12-17 12:38:29,082] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76645,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:38:29,091] jepsen worker 88 - jepsen.util 13138 :ok :read [29 11] INFO [2019-12-17 12:38:29,091] jepsen worker 88 - jepsen.util 13138 :invoke :cas [29 [2 36]] INFO [2019-12-17 12:38:29,102] jepsen worker 13 - jepsen.util 12703 :info :write [4 46] :RequestCanceledException INFO [2019-12-17 12:38:29,110] jepsen worker 87 - jepsen.util 12867 :fail :cas [29 [5 22]] :ValueNotSwapFrom INFO [2019-12-17 12:38:29,110] jepsen worker 87 - jepsen.util 12867 :invoke :read [29 nil] INFO [2019-12-17 12:38:29,114] jepsen worker 13 - jepsen.util 12793 :invoke :cas [4 [24 2]] INFO [2019-12-17 12:38:29,119] jepsen worker 57 - jepsen.util 17607 :info :write [19 15] :RequestCanceledException INFO [2019-12-17 12:38:29,122] jepsen worker 63 - jepsen.util 11673 :fail :cas [21 [4 9]] :ValueNotSwapFrom INFO [2019-12-17 12:38:29,122] jepsen worker 63 - jepsen.util 11673 :invoke :read [21 nil] INFO [2019-12-17 12:38:29,127] jepsen worker 57 - jepsen.util 17697 :invoke :read [19 nil] INFO [2019-12-17 12:38:29,136] jepsen worker 50 - jepsen.util 17960 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:38:29,136] jepsen worker 48 - jepsen.util 16698 :info :cas [16 [27 35]] :RequestCanceledException INFO [2019-12-17 12:38:29,136] jepsen worker 50 - jepsen.util 17960 :invoke :write [16 28] INFO [2019-12-17 12:38:29,143] jepsen worker 48 - jepsen.util 16788 :invoke :cas [16 [30 15]] INFO [2019-12-17 12:38:29,149] jepsen worker 7 - jepsen.util 17827 :info :write [2 40] :RequestCanceledException INFO [2019-12-17 12:38:29,155] jepsen worker 7 - jepsen.util 17917 :invoke :write [2 38] INFO [2019-12-17 12:38:29,156] jepsen worker 27 - jepsen.util 12267 :ok :write [9 34] INFO [2019-12-17 12:38:29,156] jepsen worker 27 - jepsen.util 12267 :invoke :read [9 nil] INFO [2019-12-17 12:38:29,163] jepsen worker 8 - jepsen.util 17648 :info :cas [2 [22 7]] :RequestCanceledException INFO [2019-12-17 12:38:29,169] jepsen worker 19 - jepsen.util 14059 :ok :read [6 34] INFO [2019-12-17 12:38:29,169] jepsen worker 19 - jepsen.util 14059 :invoke :write [6 23] INFO [2019-12-17 12:38:29,169] jepsen worker 45 - jepsen.util 12825 :ok :read [15 32] INFO [2019-12-17 12:38:29,170] jepsen worker 45 - jepsen.util 12825 :invoke :read [15 nil] INFO [2019-12-17 12:38:29,172] jepsen worker 72 - jepsen.util 13302 :fail :cas [24 [47 6]] :ValueNotSwapFrom INFO [2019-12-17 12:38:29,172] jepsen worker 72 - jepsen.util 13302 :invoke :cas [24 [30 31]] INFO [2019-12-17 12:38:29,174] jepsen worker 8 - jepsen.util 17738 :invoke :read [2 nil] INFO [2019-12-17 12:38:29,182] jepsen worker 31 - jepsen.util 11281 :fail :cas [10 [44 41]] :ValueNotSwapFrom INFO [2019-12-17 12:38:29,183] jepsen worker 31 - jepsen.util 11281 :invoke :write [10 27] INFO [2019-12-17 12:38:29,202] jepsen worker 59 - jepsen.util 17159 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:38:29,203] jepsen worker 59 - jepsen.util 17159 :invoke :read [19 nil] INFO [2019-12-17 12:38:29,224] jepsen worker 63 - jepsen.util 11673 :ok :read [21 7] INFO [2019-12-17 12:38:29,224] jepsen worker 63 - jepsen.util 11673 :invoke :write [21 41] INFO [2019-12-17 12:38:29,235] jepsen worker 61 - jepsen.util 19321 :info :write [20 30] :RequestCanceledException INFO [2019-12-17 12:38:29,247] jepsen worker 61 - jepsen.util 19411 :invoke :write [20 42] INFO [2019-12-17 12:38:29,258] jepsen worker 88 - jepsen.util 13138 :fail :cas [29 [2 36]] :ValueNotSwapFrom INFO [2019-12-17 12:38:29,258] jepsen worker 88 - jepsen.util 13138 :invoke :cas [29 [46 16]] INFO [2019-12-17 12:38:29,283] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76676,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:38:29,284] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76676,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:38:29,284] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76680,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:38:29,284] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76680,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:38:29,293] jepsen worker 27 - jepsen.util 12267 :ok :read [9 34] INFO [2019-12-17 12:38:29,294] jepsen worker 27 - jepsen.util 12267 :invoke :write [9 44] INFO [2019-12-17 12:38:29,294] jepsen worker 3 - jepsen.util 12873 :fail :cas [1 [6 1]] :ValueNotSwapFrom INFO [2019-12-17 12:38:29,294] jepsen worker 3 - jepsen.util 12873 :invoke :read [1 nil] INFO [2019-12-17 12:38:29,301] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76669,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:38:29,301] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76669,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:38:29,301] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76671,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:38:29,301] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76671,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:38:29,302] jepsen worker 87 - jepsen.util 12867 :ok :read [29 11] INFO [2019-12-17 12:38:29,302] jepsen worker 87 - jepsen.util 12867 :invoke :read [29 nil] INFO [2019-12-17 12:38:29,303] jepsen worker 72 - jepsen.util 13302 :fail :cas [24 [30 31]] :ValueNotSwapFrom INFO [2019-12-17 12:38:29,303] jepsen worker 72 - jepsen.util 13302 :invoke :read [24 nil] INFO [2019-12-17 12:38:29,319] jepsen worker 83 - jepsen.util 17993 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:38:29,319] jepsen worker 83 - jepsen.util 17993 :invoke :write [27 45] INFO [2019-12-17 12:38:29,321] jepsen worker 52 - jepsen.util 12832 :info :write [17 44] :RequestCanceledException INFO [2019-12-17 12:38:29,327] jepsen worker 52 - jepsen.util 12922 :invoke :write [17 6] INFO [2019-12-17 12:38:29,334] jepsen worker 45 - jepsen.util 12825 :ok :read [15 32] INFO [2019-12-17 12:38:29,334] jepsen worker 45 - jepsen.util 12825 :invoke :read [15 nil] INFO [2019-12-17 12:38:29,335] jepsen worker 42 - jepsen.util 18852 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:38:29,335] jepsen worker 64 - jepsen.util 10414 :info :cas [21 [10 43]] :RequestCanceledException INFO [2019-12-17 12:38:29,335] jepsen worker 42 - jepsen.util 18852 :invoke :cas [14 [19 35]] INFO [2019-12-17 12:38:29,335] jepsen worker 1 - jepsen.util 18541 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:38:29,335] jepsen worker 1 - jepsen.util 18541 :invoke :write [0 39] INFO [2019-12-17 12:38:29,338] jepsen worker 64 - jepsen.util 10504 :invoke :write [21 18] INFO [2019-12-17 12:38:29,349] jepsen worker 39 - jepsen.util 17589 :info :write [13 49] :RequestCanceledException INFO [2019-12-17 12:38:29,355] jepsen worker 39 - jepsen.util 17679 :invoke :read [13 nil] INFO [2019-12-17 12:38:29,366] jepsen worker 22 - jepsen.util 18562 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:38:29,366] jepsen worker 22 - jepsen.util 18562 :invoke :cas [7 [2 39]] INFO [2019-12-17 12:38:29,370] jepsen worker 72 - jepsen.util 13302 :ok :read [24 9] INFO [2019-12-17 12:38:29,370] jepsen worker 72 - jepsen.util 13302 :invoke :cas [24 [19 7]] INFO [2019-12-17 12:38:29,402] jepsen worker 9 - jepsen.util 13599 :info :cas [3 [7 20]] :RequestCanceledException INFO [2019-12-17 12:38:29,403] jepsen worker 76 - jepsen.util 18256 :info :write [25 4] :RequestCanceledException INFO [2019-12-17 12:38:29,403] jepsen worker 45 - jepsen.util 12825 :ok :read [15 32] INFO [2019-12-17 12:38:29,404] jepsen worker 45 - jepsen.util 12825 :invoke :write [15 26] INFO [2019-12-17 12:38:29,406] jepsen worker 9 - jepsen.util 13689 :invoke :cas [3 [4 41]] INFO [2019-12-17 12:38:29,408] jepsen worker 76 - jepsen.util 18346 :invoke :cas [25 [18 38]] INFO [2019-12-17 12:38:29,419] jepsen worker 68 - jepsen.util 17258 :info :write [22 8] :RequestCanceledException INFO [2019-12-17 12:38:29,423] jepsen worker 63 - jepsen.util 11673 :ok :write [21 41] INFO [2019-12-17 12:38:29,424] jepsen worker 63 - jepsen.util 11673 :invoke :read [21 nil] INFO [2019-12-17 12:38:29,427] jepsen worker 68 - jepsen.util 17348 :invoke :read [22 nil] INFO [2019-12-17 12:38:29,432] jepsen worker 18 - jepsen.util 13878 :info :write [6 13] :RequestCanceledException INFO [2019-12-17 12:38:29,433] jepsen worker 70 - jepsen.util 13750 :info :write [23 19] :RequestCanceledException INFO [2019-12-17 12:38:29,434] jepsen worker 18 - jepsen.util 13968 :invoke :read [6 nil] INFO [2019-12-17 12:38:29,436] jepsen worker 70 - jepsen.util 13840 :invoke :cas [23 [48 47]] INFO [2019-12-17 12:38:29,438] jepsen worker 88 - jepsen.util 13138 :fail :cas [29 [46 16]] :ValueNotSwapFrom INFO [2019-12-17 12:38:29,438] jepsen worker 88 - jepsen.util 13138 :invoke :write [29 18] INFO [2019-12-17 12:38:29,444] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:38:29,444] jepsen nemesis - couchbase.util Rebalance check count: 603 INFO [2019-12-17 12:38:29,444] jepsen nemesis - couchbase.util Rebalance stuck count: 584 INFO [2019-12-17 12:38:29,477] jepsen worker 27 - jepsen.util 12267 :ok :write [9 44] INFO [2019-12-17 12:38:29,477] jepsen worker 27 - jepsen.util 12267 :invoke :read [9 nil] INFO [2019-12-17 12:38:29,486] jepsen worker 53 - jepsen.util 12833 :ok :read [17 18] INFO [2019-12-17 12:38:29,486] jepsen worker 53 - jepsen.util 12833 :invoke :write [17 24] INFO [2019-12-17 12:38:29,509] jepsen worker 12 - jepsen.util 13332 :ok :read [4 39] INFO [2019-12-17 12:38:29,509] jepsen worker 12 - jepsen.util 13332 :invoke :read [4 nil] INFO [2019-12-17 12:38:29,515] jepsen worker 72 - jepsen.util 13302 :fail :cas [24 [19 7]] :ValueNotSwapFrom INFO [2019-12-17 12:38:29,516] jepsen worker 72 - jepsen.util 13302 :invoke :write [24 22] INFO [2019-12-17 12:38:29,520] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76706,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:38:29,520] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76706,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:38:29,521] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76708,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:38:29,521] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76708,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:38:29,522] jepsen worker 87 - jepsen.util 12867 :ok :read [29 11] INFO [2019-12-17 12:38:29,523] jepsen worker 87 - jepsen.util 12867 :invoke :read [29 nil] INFO [2019-12-17 12:38:29,535] jepsen worker 11 - jepsen.util 13061 :info :cas [3 [30 29]] :RequestCanceledException INFO [2019-12-17 12:38:29,542] jepsen worker 63 - jepsen.util 11673 :ok :read [21 41] INFO [2019-12-17 12:38:29,543] jepsen worker 63 - jepsen.util 11673 :invoke :cas [21 [33 25]] INFO [2019-12-17 12:38:29,550] jepsen worker 11 - jepsen.util 13151 :invoke :write [3 1] INFO [2019-12-17 12:38:29,567] jepsen worker 49 - jepsen.util 18139 :info :cas [16 [46 10]] :RequestCanceledException INFO [2019-12-17 12:38:29,573] jepsen worker 49 - jepsen.util 18229 :invoke :read [16 nil] INFO [2019-12-17 12:38:29,574] jepsen worker 27 - jepsen.util 12267 :ok :read [9 44] INFO [2019-12-17 12:38:29,574] jepsen worker 27 - jepsen.util 12267 :invoke :read [9 nil] INFO [2019-12-17 12:38:29,575] jepsen worker 18 - jepsen.util 13968 :ok :read [6 34] INFO [2019-12-17 12:38:29,575] jepsen worker 18 - jepsen.util 13968 :invoke :read [6 nil] INFO [2019-12-17 12:38:29,593] jepsen worker 36 - jepsen.util 13806 :fail :cas [12 [20 5]] :ValueNotSwapFrom INFO [2019-12-17 12:38:29,593] jepsen worker 36 - jepsen.util 13806 :invoke :write [12 31] INFO [2019-12-17 12:38:29,601] jepsen worker 71 - jepsen.util 14111 :info :write [23 8] :RequestCanceledException INFO [2019-12-17 12:38:29,601] jepsen worker 6 - jepsen.util 18276 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:38:29,601] jepsen worker 6 - jepsen.util 18276 :invoke :cas [2 [39 20]] INFO [2019-12-17 12:38:29,605] jepsen worker 71 - jepsen.util 14201 :invoke :read [23 nil] INFO [2019-12-17 12:38:29,631] jepsen worker 63 - jepsen.util 11673 :fail :cas [21 [33 25]] :ValueNotSwapFrom INFO [2019-12-17 12:38:29,631] jepsen worker 63 - jepsen.util 11673 :invoke :cas [21 [32 49]] INFO [2019-12-17 12:38:29,634] jepsen worker 34 - jepsen.util 19024 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:38:29,634] jepsen worker 34 - jepsen.util 19024 :invoke :write [11 24] INFO [2019-12-17 12:38:29,640] jepsen worker 18 - jepsen.util 13968 :ok :read [6 34] INFO [2019-12-17 12:38:29,641] jepsen worker 18 - jepsen.util 13968 :invoke :write [6 15] INFO [2019-12-17 12:38:29,649] jepsen worker 10 - jepsen.util 13150 :fail :read [3 nil] :RequestTimeoutException INFO [2019-12-17 12:38:29,649] jepsen worker 10 - jepsen.util 13150 :invoke :read [3 nil] INFO [2019-12-17 12:38:29,662] jepsen worker 27 - jepsen.util 12267 :ok :read [9 44] INFO [2019-12-17 12:38:29,662] jepsen worker 27 - jepsen.util 12267 :invoke :read [9 nil] INFO [2019-12-17 12:38:29,664] jepsen worker 55 - jepsen.util 14005 :fail :cas [18 [46 26]] :ValueNotSwapFrom INFO [2019-12-17 12:38:29,664] jepsen worker 55 - jepsen.util 14005 :invoke :read [18 nil] INFO [2019-12-17 12:38:29,665] jepsen worker 5 - jepsen.util 12785 :info :write [1 49] :RequestCanceledException INFO [2019-12-17 12:38:29,667] jepsen worker 5 - jepsen.util 12875 :invoke :cas [1 [14 48]] INFO [2019-12-17 12:38:29,675] jepsen worker 20 - jepsen.util 12980 :ok :write [6 35] INFO [2019-12-17 12:38:29,675] jepsen worker 20 - jepsen.util 12980 :invoke :write [6 22] INFO [2019-12-17 12:38:29,676] jepsen worker 32 - jepsen.util 13442 :ok :write [10 49] INFO [2019-12-17 12:38:29,677] jepsen worker 32 - jepsen.util 13442 :invoke :cas [10 [46 19]] INFO [2019-12-17 12:38:29,701] jepsen worker 40 - jepsen.util 18850 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:38:29,702] jepsen worker 40 - jepsen.util 18850 :invoke :read [13 nil] INFO [2019-12-17 12:38:29,702] jepsen worker 69 - jepsen.util 13569 :info :cas [23 [23 10]] :RequestCanceledException INFO [2019-12-17 12:38:29,715] jepsen worker 69 - jepsen.util 13659 :invoke :write [23 18] INFO [2019-12-17 12:38:29,717] jepsen worker 74 - jepsen.util 13034 :ok :write [24 3] INFO [2019-12-17 12:38:29,717] jepsen worker 74 - jepsen.util 13034 :invoke :write [24 49] INFO [2019-12-17 12:38:29,718] jepsen worker 41 - jepsen.util 19391 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:38:29,718] jepsen worker 45 - jepsen.util 12825 :ok :write [15 26] INFO [2019-12-17 12:38:29,718] jepsen worker 41 - jepsen.util 19391 :invoke :write [13 11] INFO [2019-12-17 12:38:29,718] jepsen worker 45 - jepsen.util 12825 :invoke :read [15 nil] INFO [2019-12-17 12:38:29,736] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76728,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:38:29,736] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76728,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:38:29,736] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76731,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:38:29,737] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76731,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:38:29,737] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76732,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:38:29,737] jepsen worker 63 - jepsen.util 11673 :fail :cas [21 [32 49]] :ValueNotSwapFrom INFO [2019-12-17 12:38:29,737] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76732,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:38:29,737] jepsen worker 63 - jepsen.util 11673 :invoke :cas [21 [17 38]] INFO [2019-12-17 12:38:29,754] jepsen worker 27 - jepsen.util 12267 :ok :read [9 44] INFO [2019-12-17 12:38:29,754] jepsen worker 27 - jepsen.util 12267 :invoke :write [9 16] INFO [2019-12-17 12:38:29,781] jepsen worker 65 - jepsen.util 11675 :fail :cas [21 [33 10]] :ValueNotSwapFrom INFO [2019-12-17 12:38:29,781] jepsen worker 65 - jepsen.util 11675 :invoke :cas [21 [44 12]] INFO [2019-12-17 12:38:29,790] jepsen worker 37 - jepsen.util 12817 :ok :read [12 19] INFO [2019-12-17 12:38:29,790] jepsen worker 37 - jepsen.util 12817 :invoke :write [12 18] INFO [2019-12-17 12:38:29,830] jepsen worker 29 - jepsen.util 11459 :ok :write [9 36] INFO [2019-12-17 12:38:29,831] jepsen worker 29 - jepsen.util 11459 :invoke :write [9 29] INFO [2019-12-17 12:38:29,835] jepsen worker 24 - jepsen.util 17214 :info :cas [8 [40 8]] :RequestCanceledException INFO [2019-12-17 12:38:29,840] jepsen worker 24 - jepsen.util 17304 :invoke :cas [8 [22 16]] INFO [2019-12-17 12:38:29,872] jepsen worker 46 - jepsen.util 12916 :ok :write [15 33] INFO [2019-12-17 12:38:29,872] jepsen worker 46 - jepsen.util 12916 :invoke :cas [15 [18 35]] INFO [2019-12-17 12:38:29,882] jepsen worker 27 - jepsen.util 12267 :ok :write [9 16] INFO [2019-12-17 12:38:29,882] jepsen worker 27 - jepsen.util 12267 :invoke :cas [9 [4 9]] INFO [2019-12-17 12:38:29,901] jepsen worker 62 - jepsen.util 18782 :info :cas [20 [5 12]] :RequestCanceledException INFO [2019-12-17 12:38:29,908] jepsen worker 62 - jepsen.util 18872 :invoke :read [20 nil] INFO [2019-12-17 12:38:29,919] jepsen worker 26 - jepsen.util 19106 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:38:29,919] jepsen worker 26 - jepsen.util 19106 :invoke :read [8 nil] INFO [2019-12-17 12:38:29,932] jepsen worker 9 - jepsen.util 13689 :fail :cas [3 [4 41]] :ValueNotSwapFrom INFO [2019-12-17 12:38:29,932] jepsen worker 9 - jepsen.util 13689 :invoke :write [3 24] INFO [2019-12-17 12:38:29,933] jepsen worker 64 - jepsen.util 10504 :ok :write [21 18] INFO [2019-12-17 12:38:29,933] jepsen worker 64 - jepsen.util 10504 :invoke :cas [21 [35 5]] INFO [2019-12-17 12:38:29,935] jepsen worker 79 - jepsen.util 13399 :info :write [26 43] :RequestCanceledException INFO [2019-12-17 12:38:29,935] jepsen worker 86 - jepsen.util 18896 :info :write [28 47] :RequestCanceledException INFO [2019-12-17 12:38:29,943] jepsen worker 79 - jepsen.util 13489 :invoke :cas [26 [22 12]] INFO [2019-12-17 12:38:29,945] jepsen worker 86 - jepsen.util 18986 :invoke :read [28 nil] INFO [2019-12-17 12:38:29,952] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76754,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:38:29,952] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76754,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:38:29,953] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76756,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:38:29,953] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76756,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:38:29,965] jepsen worker 75 - jepsen.util 17895 :info :write [25 10] :RequestCanceledException INFO [2019-12-17 12:38:29,971] jepsen worker 75 - jepsen.util 17985 :invoke :read [25 nil] INFO [2019-12-17 12:38:29,985] jepsen worker 38 - jepsen.util 13898 :ok :read [12 19] INFO [2019-12-17 12:38:29,985] jepsen worker 38 - jepsen.util 13898 :invoke :write [12 43] INFO [2019-12-17 12:38:30,012] jepsen worker 27 - jepsen.util 12267 :fail :cas [9 [4 9]] :ValueNotSwapFrom INFO [2019-12-17 12:38:30,013] jepsen worker 27 - jepsen.util 12267 :invoke :write [9 29] INFO [2019-12-17 12:38:30,016] jepsen worker 12 - jepsen.util 13332 :ok :read [4 39] INFO [2019-12-17 12:38:30,016] jepsen worker 12 - jepsen.util 13332 :invoke :cas [4 [2 45]] INFO [2019-12-17 12:38:30,020] jepsen worker 17 - jepsen.util 18017 :info :cas [5 [8 20]] :RequestCanceledException INFO [2019-12-17 12:38:30,027] jepsen worker 17 - jepsen.util 18107 :invoke :read [5 nil] INFO [2019-12-17 12:38:30,035] jepsen worker 60 - jepsen.util 19050 :info :cas [20 [8 40]] :RequestCanceledException INFO [2019-12-17 12:38:30,035] jepsen worker 3 - jepsen.util 12873 :ok :read [1 44] INFO [2019-12-17 12:38:30,035] jepsen worker 3 - jepsen.util 12873 :invoke :cas [1 [48 42]] INFO [2019-12-17 12:38:30,035] jepsen worker 28 - jepsen.util 11188 :fail :read [9 nil] :RequestTimeoutException INFO [2019-12-17 12:38:30,035] jepsen worker 28 - jepsen.util 11188 :invoke :cas [9 [32 24]] INFO [2019-12-17 12:38:30,037] jepsen worker 60 - jepsen.util 19140 :invoke :write [20 1] INFO [2019-12-17 12:38:30,062] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76769,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:38:30,062] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76769,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:38:30,062] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76771,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:38:30,062] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76771,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:38:30,074] jepsen worker 64 - jepsen.util 10504 :fail :cas [21 [35 5]] :ValueNotSwapFrom INFO [2019-12-17 12:38:30,074] jepsen worker 64 - jepsen.util 10504 :invoke :read [21 nil] INFO [2019-12-17 12:38:30,078] jepsen worker 70 - jepsen.util 13840 :fail :cas [23 [48 47]] :ValueNotSwapFrom INFO [2019-12-17 12:38:30,079] jepsen worker 70 - jepsen.util 13840 :invoke :cas [23 [20 48]] INFO [2019-12-17 12:38:30,100] jepsen worker 13 - jepsen.util 12793 :fail :cas [4 [24 2]] :ValueNotSwapFrom INFO [2019-12-17 12:38:30,100] jepsen worker 13 - jepsen.util 12793 :invoke :write [4 18] INFO [2019-12-17 12:38:30,108] jepsen worker 72 - jepsen.util 13302 :ok :write [24 22] INFO [2019-12-17 12:38:30,109] jepsen worker 72 - jepsen.util 13302 :invoke :write [24 34] INFO [2019-12-17 12:38:30,109] jepsen worker 11 - jepsen.util 13151 :ok :write [3 1] INFO [2019-12-17 12:38:30,109] jepsen worker 18 - jepsen.util 13968 :ok :write [6 15] INFO [2019-12-17 12:38:30,109] jepsen worker 18 - jepsen.util 13968 :invoke :read [6 nil] INFO [2019-12-17 12:38:30,109] jepsen worker 11 - jepsen.util 13151 :invoke :cas [3 [33 4]] INFO [2019-12-17 12:38:30,126] jepsen worker 28 - jepsen.util 11188 :fail :cas [9 [32 24]] :ValueNotSwapFrom INFO [2019-12-17 12:38:30,126] jepsen worker 28 - jepsen.util 11188 :invoke :read [9 nil] INFO [2019-12-17 12:38:30,135] jepsen worker 80 - jepsen.util 12680 :info :write [26 35] :RequestCanceledException INFO [2019-12-17 12:38:30,135] jepsen worker 66 - jepsen.util 18246 :info :cas [22 [33 24]] :RequestCanceledException INFO [2019-12-17 12:38:30,138] jepsen worker 80 - jepsen.util 12770 :invoke :cas [26 [3 25]] INFO [2019-12-17 12:38:30,139] jepsen worker 66 - jepsen.util 18336 :invoke :write [22 13] INFO [2019-12-17 12:38:30,201] jepsen worker 33 - jepsen.util 18033 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:38:30,201] jepsen worker 33 - jepsen.util 18033 :invoke :cas [11 [16 47]] INFO [2019-12-17 12:38:30,202] jepsen worker 64 - jepsen.util 10504 :ok :read [21 18] INFO [2019-12-17 12:38:30,202] jepsen worker 64 - jepsen.util 10504 :invoke :cas [21 [22 15]] INFO [2019-12-17 12:38:30,214] jepsen worker 27 - jepsen.util 12267 :ok :write [9 29] INFO [2019-12-17 12:38:30,215] jepsen worker 27 - jepsen.util 12267 :invoke :read [9 nil] INFO [2019-12-17 12:38:30,226] jepsen worker 63 - jepsen.util 11673 :fail :cas [21 [17 38]] :ValueNotSwapFrom INFO [2019-12-17 12:38:30,226] jepsen worker 63 - jepsen.util 11673 :invoke :cas [21 [21 36]] INFO [2019-12-17 12:38:30,231] jepsen worker 55 - jepsen.util 14005 :ok :read [18 26] INFO [2019-12-17 12:38:30,231] jepsen worker 55 - jepsen.util 14005 :invoke :read [18 nil] INFO [2019-12-17 12:38:30,249] jepsen worker 73 - jepsen.util 12223 :info :write [24 35] :RequestCanceledException INFO [2019-12-17 12:38:30,251] jepsen worker 65 - jepsen.util 11675 :fail :cas [21 [44 12]] :ValueNotSwapFrom INFO [2019-12-17 12:38:30,251] jepsen worker 65 - jepsen.util 11675 :invoke :write [21 7] INFO [2019-12-17 12:38:30,257] jepsen worker 18 - jepsen.util 13968 :ok :read [6 15] INFO [2019-12-17 12:38:30,257] jepsen worker 18 - jepsen.util 13968 :invoke :write [6 15] INFO [2019-12-17 12:38:30,261] jepsen worker 73 - jepsen.util 12313 :invoke :write [24 32] INFO [2019-12-17 12:38:30,268] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76798,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:38:30,269] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76798,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:38:30,269] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76802,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:38:30,270] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76802,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:38:30,276] jepsen worker 12 - jepsen.util 13332 :fail :cas [4 [2 45]] :ValueNotSwapFrom INFO [2019-12-17 12:38:30,277] jepsen worker 12 - jepsen.util 13332 :invoke :write [4 36] INFO [2019-12-17 12:38:30,281] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76790,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:38:30,290] jepsen worker 11 - jepsen.util 13151 :fail :cas [3 [33 4]] :ValueNotSwapFrom INFO [2019-12-17 12:38:30,311] jepsen worker 71 - jepsen.util 14201 :ok :read [23 19] INFO [2019-12-17 12:38:30,331] jepsen worker 28 - jepsen.util 11188 :ok :read [9 29] INFO [2019-12-17 12:38:30,334] jepsen worker 78 - jepsen.util 12948 :info :write [26 41] :RequestCanceledException INFO [2019-12-17 12:38:30,337] jepsen worker 3 - jepsen.util 12873 :fail :cas [1 [48 42]] :ValueNotSwapFrom INFO [2019-12-17 12:38:30,338] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76790,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:38:30,339] jepsen worker 71 - jepsen.util 14201 :invoke :write [23 36] INFO [2019-12-17 12:38:30,339] jepsen worker 28 - jepsen.util 11188 :invoke :cas [9 [1 22]] INFO [2019-12-17 12:38:30,339] jepsen worker 11 - jepsen.util 13151 :invoke :cas [3 [12 32]] INFO [2019-12-17 12:38:30,339] jepsen worker 3 - jepsen.util 12873 :invoke :read [1 nil] INFO [2019-12-17 12:38:30,339] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76791,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:38:30,339] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76791,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:38:30,341] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76792,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:38:30,342] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76792,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:38:30,342] jepsen worker 78 - jepsen.util 13038 :invoke :write [26 36] INFO [2019-12-17 12:38:30,349] jepsen worker 85 - jepsen.util 18625 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:38:30,366] jepsen worker 67 - jepsen.util 18337 :info :write [22 31] :RequestCanceledException INFO [2019-12-17 12:38:30,374] jepsen worker 85 - jepsen.util 18625 :invoke :cas [28 [21 10]] INFO [2019-12-17 12:38:30,402] jepsen worker 21 - jepsen.util 17661 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:38:30,402] jepsen worker 21 - jepsen.util 17661 :invoke :cas [7 [20 15]] INFO [2019-12-17 12:38:30,418] jepsen worker 0 - jepsen.util 18540 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:38:30,419] jepsen worker 0 - jepsen.util 18540 :invoke :cas [0 [9 3]] INFO [2019-12-17 12:38:30,420] jepsen worker 65 - jepsen.util 11675 :ok :write [21 7] INFO [2019-12-17 12:38:30,420] jepsen worker 65 - jepsen.util 11675 :invoke :read [21 nil] INFO [2019-12-17 12:38:30,435] jepsen worker 54 - jepsen.util 13914 :info :write [18 37] :RequestCanceledException INFO [2019-12-17 12:38:30,449] jepsen worker 67 - jepsen.util 18427 :invoke :cas [22 [45 23]] INFO [2019-12-17 12:38:30,457] jepsen worker 72 - jepsen.util 13302 :ok :write [24 34] INFO [2019-12-17 12:38:30,458] jepsen worker 72 - jepsen.util 13302 :invoke :cas [24 [47 0]] INFO [2019-12-17 12:38:30,493] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:38:30,497] jepsen nemesis - couchbase.util Rebalance check count: 604 INFO [2019-12-17 12:38:30,498] jepsen nemesis - couchbase.util Rebalance stuck count: 585 INFO [2019-12-17 12:38:30,499] jepsen worker 54 - jepsen.util 14004 :invoke :read [18 nil] INFO [2019-12-17 12:38:30,502] jepsen worker 84 - jepsen.util 17454 :info :cas [28 [1 16]] :RequestCanceledException INFO [2019-12-17 12:38:30,508] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76818,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:38:30,509] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76818,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:38:30,511] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76821,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:38:30,511] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76821,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:38:30,511] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76822,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:38:30,512] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76822,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:38:30,523] jepsen worker 84 - jepsen.util 17544 :invoke :read [28 nil] INFO [2019-12-17 12:38:30,549] jepsen worker 47 - jepsen.util 14447 :fail :read [15 nil] :RequestTimeoutException INFO [2019-12-17 12:38:30,549] jepsen worker 23 - jepsen.util 18293 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:38:30,549] jepsen worker 47 - jepsen.util 14447 :invoke :read [15 nil] INFO [2019-12-17 12:38:30,549] jepsen worker 23 - jepsen.util 18293 :invoke :write [7 48] INFO [2019-12-17 12:38:30,635] jepsen worker 82 - jepsen.util 18082 :info :cas [27 [29 46]] :RequestCanceledException INFO [2019-12-17 12:38:30,645] jepsen worker 4 - jepsen.util 12334 :info :write [1 5] :RequestCanceledException INFO [2019-12-17 12:38:30,648] jepsen worker 4 - jepsen.util 12424 :invoke :write [1 32] INFO [2019-12-17 12:38:30,652] jepsen worker 82 - jepsen.util 18172 :invoke :write [27 14] INFO [2019-12-17 12:38:30,667] jepsen worker 51 - jepsen.util 12471 :info :write [17 32] :RequestCanceledException INFO [2019-12-17 12:38:30,673] jepsen worker 51 - jepsen.util 12561 :invoke :write [17 32] INFO [2019-12-17 12:38:30,681] jepsen worker 79 - jepsen.util 13489 :fail :cas [26 [22 12]] :ValueNotSwapFrom INFO [2019-12-17 12:38:30,682] jepsen worker 79 - jepsen.util 13489 :invoke :cas [26 [47 8]] INFO [2019-12-17 12:38:30,711] jepsen worker 87 - jepsen.util 12867 :ok :read [29 11] INFO [2019-12-17 12:38:30,712] jepsen worker 87 - jepsen.util 12867 :invoke :cas [29 [0 29]] INFO [2019-12-17 12:38:30,727] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76829,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:38:30,735] jepsen worker 56 - jepsen.util 13916 :info :write [18 42] :RequestCanceledException INFO [2019-12-17 12:38:30,743] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76829,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:38:30,744] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76834,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:38:30,748] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76834,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:38:30,761] jepsen worker 56 - jepsen.util 14006 :invoke :write [18 23] INFO [2019-12-17 12:38:30,836] jepsen worker 77 - jepsen.util 19337 :info :write [25 18] :RequestCanceledException INFO [2019-12-17 12:38:30,891] jepsen worker 77 - jepsen.util 19427 :invoke :read [25 nil] INFO [2019-12-17 12:38:30,902] jepsen worker 14 - jepsen.util 12884 :info :write [4 1] :RequestCanceledException INFO [2019-12-17 12:38:30,926] jepsen worker 64 - jepsen.util 10504 :fail :cas [21 [22 15]] :ValueNotSwapFrom INFO [2019-12-17 12:38:30,929] jepsen worker 64 - jepsen.util 10504 :invoke :read [21 nil] INFO [2019-12-17 12:38:30,935] jepsen worker 89 - jepsen.util 13139 :info :write [29 5] :RequestCanceledException INFO [2019-12-17 12:38:30,940] jepsen worker 14 - jepsen.util 12974 :invoke :read [4 nil] INFO [2019-12-17 12:38:30,948] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76843,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:38:30,949] jepsen worker 89 - jepsen.util 13229 :invoke :cas [29 [34 13]] INFO [2019-12-17 12:38:30,951] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76843,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:38:30,952] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76844,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:38:30,959] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76844,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:38:30,967] jepsen worker 81 - jepsen.util 17631 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:38:30,967] jepsen worker 81 - jepsen.util 17631 :invoke :cas [27 [35 34]] INFO [2019-12-17 12:38:30,986] jepsen worker 5 - jepsen.util 12875 :fail :cas [1 [14 48]] :ValueNotSwapFrom INFO [2019-12-17 12:38:30,987] jepsen worker 5 - jepsen.util 12875 :invoke :write [1 21] INFO [2019-12-17 12:38:31,010] jepsen worker 18 - jepsen.util 13968 :ok :write [6 15] INFO [2019-12-17 12:38:31,010] jepsen worker 18 - jepsen.util 13968 :invoke :write [6 31] INFO [2019-12-17 12:38:31,011] jepsen worker 73 - jepsen.util 12313 :ok :write [24 32] INFO [2019-12-17 12:38:31,013] jepsen worker 73 - jepsen.util 12313 :invoke :read [24 nil] INFO [2019-12-17 12:38:31,020] jepsen worker 15 - jepsen.util 17205 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:38:31,023] jepsen worker 30 - jepsen.util 12630 :info :write [10 47] :RequestCanceledException INFO [2019-12-17 12:38:31,023] jepsen worker 15 - jepsen.util 17205 :invoke :cas [5 [15 25]] INFO [2019-12-17 12:38:31,034] jepsen worker 35 - jepsen.util 18305 :info :write [11 32] :RequestCanceledException INFO [2019-12-17 12:38:31,054] jepsen worker 16 - jepsen.util 19726 :info :write [5 17] :RequestCanceledException INFO [2019-12-17 12:38:31,072] jepsen worker 43 - jepsen.util 18313 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:38:31,073] jepsen worker 43 - jepsen.util 18313 :invoke :write [14 32] INFO [2019-12-17 12:38:31,073] jepsen worker 58 - jepsen.util 18328 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:38:31,074] jepsen worker 58 - jepsen.util 18328 :invoke :read [19 nil] INFO [2019-12-17 12:38:31,086] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76853,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:38:31,088] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76853,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:38:31,089] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76856,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:38:31,090] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76856,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:38:31,103] jepsen worker 35 - jepsen.util 18395 :invoke :cas [11 [7 26]] INFO [2019-12-17 12:38:31,110] jepsen worker 30 - jepsen.util 12720 :invoke :read [10 nil] INFO [2019-12-17 12:38:31,115] jepsen worker 16 - jepsen.util 19816 :invoke :write [5 12] INFO [2019-12-17 12:38:31,124] jepsen worker 2 - jepsen.util 17372 :info :cas [0 [5 47]] :RequestCanceledException INFO [2019-12-17 12:38:31,133] jepsen worker 27 - jepsen.util 12267 :ok :read [9 29] INFO [2019-12-17 12:38:31,137] jepsen worker 25 - jepsen.util 19285 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:38:31,139] jepsen worker 25 - jepsen.util 19285 :invoke :cas [8 [18 12]] INFO [2019-12-17 12:38:31,140] jepsen worker 27 - jepsen.util 12267 :invoke :write [9 17] INFO [2019-12-17 12:38:31,151] jepsen worker 44 - jepsen.util 18944 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:38:31,152] jepsen worker 44 - jepsen.util 18944 :invoke :write [14 45] INFO [2019-12-17 12:38:31,204] jepsen worker 48 - jepsen.util 16788 :info :cas [16 [30 15]] :RequestCanceledException INFO [2019-12-17 12:38:31,226] jepsen worker 7 - jepsen.util 17917 :info :write [2 38] :RequestCanceledException INFO [2019-12-17 12:38:31,238] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76865,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:38:31,238] jepsen worker 8 - jepsen.util 17738 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:38:31,239] jepsen worker 50 - jepsen.util 17960 :info :write [16 28] :RequestCanceledException INFO [2019-12-17 12:38:31,239] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76865,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:38:31,239] jepsen worker 57 - jepsen.util 17697 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:38:31,239] jepsen worker 8 - jepsen.util 17738 :invoke :cas [2 [19 18]] INFO [2019-12-17 12:38:31,239] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76868,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:38:31,240] jepsen worker 57 - jepsen.util 17697 :invoke :write [19 42] INFO [2019-12-17 12:38:31,240] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76868,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:38:31,272] jepsen worker 31 - jepsen.util 11281 :info :write [10 27] :RequestCanceledException INFO [2019-12-17 12:38:31,272] jepsen worker 19 - jepsen.util 14059 :info :write [6 23] :RequestCanceledException INFO [2019-12-17 12:38:31,272] jepsen worker 2 - jepsen.util 17462 :invoke :cas [0 [30 10]] INFO [2019-12-17 12:38:31,282] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76874,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:38:31,304] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76874,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:38:31,305] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76876,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:38:31,305] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76876,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:38:31,308] jepsen worker 71 - jepsen.util 14201 :ok :write [23 36] INFO [2019-12-17 12:38:31,309] jepsen worker 71 - jepsen.util 14201 :invoke :read [23 nil] INFO [2019-12-17 12:38:31,313] jepsen worker 59 - jepsen.util 17159 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:38:31,314] jepsen worker 59 - jepsen.util 17159 :invoke :cas [19 [46 28]] INFO [2019-12-17 12:38:31,314] jepsen worker 48 - jepsen.util 16878 :invoke :cas [16 [48 29]] INFO [2019-12-17 12:38:31,316] jepsen worker 7 - jepsen.util 18007 :invoke :cas [2 [17 46]] INFO [2019-12-17 12:38:31,351] jepsen worker 61 - jepsen.util 19411 :info :write [20 42] :RequestCanceledException INFO [2019-12-17 12:38:31,358] jepsen worker 31 - jepsen.util 11371 :invoke :cas [10 [42 31]] INFO [2019-12-17 12:38:31,359] jepsen worker 19 - jepsen.util 14149 :invoke :write [6 39] INFO [2019-12-17 12:38:31,368] jepsen worker 50 - jepsen.util 18050 :invoke :read [16 nil] INFO [2019-12-17 12:38:31,404] jepsen worker 39 - jepsen.util 17679 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:38:31,405] jepsen worker 39 - jepsen.util 17679 :invoke :read [13 nil] INFO [2019-12-17 12:38:31,416] jepsen worker 72 - jepsen.util 13302 :fail :cas [24 [47 0]] :ValueNotSwapFrom INFO [2019-12-17 12:38:31,418] jepsen worker 72 - jepsen.util 13302 :invoke :read [24 nil] INFO [2019-12-17 12:38:31,420] jepsen worker 83 - jepsen.util 17993 :info :write [27 45] :RequestCanceledException INFO [2019-12-17 12:38:31,424] jepsen worker 61 - jepsen.util 19501 :invoke :cas [20 [9 28]] INFO [2019-12-17 12:38:31,443] jepsen worker 1 - jepsen.util 18541 :info :write [0 39] :RequestCanceledException INFO [2019-12-17 12:38:31,448] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76892,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:38:31,450] jepsen worker 42 - jepsen.util 18852 :info :cas [14 [19 35]] :RequestCanceledException INFO [2019-12-17 12:38:31,450] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76892,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:38:31,456] jepsen worker 68 - jepsen.util 17348 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:38:31,457] jepsen worker 76 - jepsen.util 18346 :info :cas [25 [18 38]] :RequestCanceledException INFO [2019-12-17 12:38:31,458] jepsen worker 65 - jepsen.util 11675 :ok :read [21 7] INFO [2019-12-17 12:38:31,460] jepsen worker 68 - jepsen.util 17348 :invoke :cas [22 [46 34]] INFO [2019-12-17 12:38:31,461] jepsen worker 65 - jepsen.util 11675 :invoke :cas [21 [8 43]] INFO [2019-12-17 12:38:31,500] jepsen worker 52 - jepsen.util 12922 :info :write [17 6] :RequestCanceledException INFO [2019-12-17 12:38:31,500] jepsen worker 22 - jepsen.util 18562 :info :cas [7 [2 39]] :RequestCanceledException INFO [2019-12-17 12:38:31,545] jepsen worker 53 - jepsen.util 12833 :info :write [17 24] :RequestCanceledException INFO [2019-12-17 12:38:31,548] jepsen worker 42 - jepsen.util 18942 :invoke :cas [14 [14 25]] INFO [2019-12-17 12:38:31,550] jepsen worker 88 - jepsen.util 13138 :info :write [29 18] :RequestCanceledException INFO [2019-12-17 12:38:31,550] jepsen worker 83 - jepsen.util 18083 :invoke :cas [27 [40 11]] INFO [2019-12-17 12:38:31,576] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:38:31,577] jepsen worker 88 - jepsen.util 13228 :invoke :cas [29 [40 41]] INFO [2019-12-17 12:38:31,578] jepsen nemesis - couchbase.util Rebalance check count: 605 INFO [2019-12-17 12:38:31,579] jepsen nemesis - couchbase.util Rebalance stuck count: 586 INFO [2019-12-17 12:38:31,584] jepsen worker 76 - jepsen.util 18436 :invoke :read [25 nil] INFO [2019-12-17 12:38:31,586] jepsen worker 1 - jepsen.util 18631 :invoke :cas [0 [11 15]] INFO [2019-12-17 12:38:31,588] jepsen worker 22 - jepsen.util 18652 :invoke :read [7 nil] INFO [2019-12-17 12:38:31,590] jepsen worker 52 - jepsen.util 13012 :invoke :cas [17 [32 21]] INFO [2019-12-17 12:38:31,600] jepsen worker 63 - jepsen.util 11673 :fail :cas [21 [21 36]] :ValueNotSwapFrom INFO [2019-12-17 12:38:31,602] jepsen worker 63 - jepsen.util 11673 :invoke :read [21 nil] INFO [2019-12-17 12:38:31,603] jepsen worker 53 - jepsen.util 12923 :invoke :cas [17 [1 6]] INFO [2019-12-17 12:38:31,633] jepsen worker 49 - jepsen.util 18229 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:38:31,643] jepsen worker 49 - jepsen.util 18229 :invoke :write [16 32] INFO [2019-12-17 12:38:31,643] jepsen worker 36 - jepsen.util 13806 :info :write [12 31] :RequestCanceledException INFO [2019-12-17 12:38:31,643] jepsen worker 47 - jepsen.util 14447 :ok :read [15 33] INFO [2019-12-17 12:38:31,693] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76912,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:38:31,700] jepsen worker 47 - jepsen.util 14447 :invoke :write [15 37] INFO [2019-12-17 12:38:31,737] jepsen worker 20 - jepsen.util 12980 :info :write [6 22] :RequestCanceledException INFO [2019-12-17 12:38:31,738] jepsen worker 6 - jepsen.util 18276 :info :cas [2 [39 20]] :RequestCanceledException INFO [2019-12-17 12:38:31,746] jepsen worker 32 - jepsen.util 13442 :info :cas [10 [46 19]] :RequestCanceledException INFO [2019-12-17 12:38:31,748] jepsen worker 34 - jepsen.util 19024 :info :write [11 24] :RequestCanceledException INFO [2019-12-17 12:38:31,760] jepsen worker 10 - jepsen.util 13150 :fail :read [3 nil] :RequestTimeoutException INFO [2019-12-17 12:38:31,770] jepsen worker 45 - jepsen.util 12825 :fail :read [15 nil] :RequestTimeoutException INFO [2019-12-17 12:38:31,771] jepsen worker 10 - jepsen.util 13150 :invoke :read [3 nil] INFO [2019-12-17 12:38:31,773] jepsen worker 36 - jepsen.util 13896 :invoke :cas [12 [13 12]] INFO [2019-12-17 12:38:31,775] jepsen worker 45 - jepsen.util 12825 :invoke :cas [15 [23 48]] INFO [2019-12-17 12:38:31,778] jepsen worker 69 - jepsen.util 13659 :info :write [23 18] :RequestCanceledException INFO [2019-12-17 12:38:31,779] jepsen worker 37 - jepsen.util 12817 :ok :write [12 18] INFO [2019-12-17 12:38:31,782] jepsen worker 37 - jepsen.util 12817 :invoke :read [12 nil] INFO [2019-12-17 12:38:31,798] jepsen worker 20 - jepsen.util 13070 :invoke :read [6 nil] INFO [2019-12-17 12:38:31,801] jepsen worker 32 - jepsen.util 13532 :invoke :cas [10 [32 20]] INFO [2019-12-17 12:38:31,810] jepsen worker 40 - jepsen.util 18850 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:38:31,810] jepsen worker 40 - jepsen.util 18850 :invoke :read [13 nil] INFO [2019-12-17 12:38:31,810] jepsen worker 69 - jepsen.util 13749 :invoke :write [23 19] INFO [2019-12-17 12:38:31,819] jepsen worker 6 - jepsen.util 18366 :invoke :read [2 nil] INFO [2019-12-17 12:38:31,823] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76912,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:38:31,823] jepsen worker 34 - jepsen.util 19114 :invoke :cas [11 [4 49]] INFO [2019-12-17 12:38:31,831] jepsen worker 41 - jepsen.util 19391 :info :write [13 11] :RequestCanceledException INFO [2019-12-17 12:38:31,838] jepsen worker 74 - jepsen.util 13034 :info :write [24 49] :RequestCanceledException INFO [2019-12-17 12:38:31,884] jepsen worker 74 - jepsen.util 13124 :invoke :cas [24 [43 37]] INFO [2019-12-17 12:38:31,895] jepsen worker 41 - jepsen.util 19481 :invoke :write [13 2] INFO [2019-12-17 12:38:31,927] jepsen worker 46 - jepsen.util 12916 :info :cas [15 [18 35]] :RequestCanceledException INFO [2019-12-17 12:38:31,929] jepsen worker 46 - jepsen.util 13006 :invoke :read [15 nil] INFO [2019-12-17 12:38:31,931] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76929,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:38:31,931] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76929,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:38:31,936] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76933,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:38:31,943] jepsen worker 29 - jepsen.util 11459 :info :write [9 29] :RequestCanceledException INFO [2019-12-17 12:38:31,944] jepsen worker 24 - jepsen.util 17304 :info :cas [8 [22 16]] :RequestCanceledException INFO [2019-12-17 12:38:31,945] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76933,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:38:31,956] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76934,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:38:31,968] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76934,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:38:31,979] jepsen worker 29 - jepsen.util 11549 :invoke :read [9 nil] INFO [2019-12-17 12:38:32,006] jepsen worker 62 - jepsen.util 18872 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:38:32,006] jepsen worker 62 - jepsen.util 18872 :invoke :write [20 43] INFO [2019-12-17 12:38:32,026] jepsen worker 24 - jepsen.util 17394 :invoke :read [8 nil] INFO [2019-12-17 12:38:32,028] jepsen worker 9 - jepsen.util 13689 :info :write [3 24] :RequestCanceledException INFO [2019-12-17 12:38:32,030] jepsen worker 86 - jepsen.util 18986 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:38:32,030] jepsen worker 26 - jepsen.util 19106 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:38:32,033] jepsen worker 86 - jepsen.util 18986 :invoke :write [28 9] INFO [2019-12-17 12:38:32,033] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76940,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:38:32,036] jepsen worker 26 - jepsen.util 19106 :invoke :read [8 nil] INFO [2019-12-17 12:38:32,043] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76940,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:38:32,055] jepsen worker 75 - jepsen.util 17985 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:38:32,056] jepsen worker 75 - jepsen.util 17985 :invoke :write [25 9] INFO [2019-12-17 12:38:32,075] jepsen worker 38 - jepsen.util 13898 :info :write [12 43] :RequestCanceledException INFO [2019-12-17 12:38:32,082] jepsen worker 17 - jepsen.util 18107 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:38:32,083] jepsen worker 17 - jepsen.util 18107 :invoke :read [5 nil] INFO [2019-12-17 12:38:32,088] jepsen worker 9 - jepsen.util 13779 :invoke :write [3 48] INFO [2019-12-17 12:38:32,124] jepsen worker 38 - jepsen.util 13988 :invoke :write [12 47] INFO [2019-12-17 12:38:32,144] jepsen worker 60 - jepsen.util 19140 :info :write [20 1] :RequestCanceledException INFO [2019-12-17 12:38:32,158] jepsen worker 54 - jepsen.util 14004 :ok :read [18 26] INFO [2019-12-17 12:38:32,159] jepsen worker 54 - jepsen.util 14004 :invoke :write [18 23] INFO [2019-12-17 12:38:32,166] jepsen worker 60 - jepsen.util 19230 :invoke :write [20 45] INFO [2019-12-17 12:38:32,169] jepsen worker 13 - jepsen.util 12793 :info :write [4 18] :RequestCanceledException INFO [2019-12-17 12:38:32,220] jepsen worker 13 - jepsen.util 12883 :invoke :cas [4 [14 14]] INFO [2019-12-17 12:38:32,229] jepsen worker 66 - jepsen.util 18336 :info :write [22 13] :RequestCanceledException INFO [2019-12-17 12:38:32,260] jepsen worker 27 - jepsen.util 12267 :ok :write [9 17] INFO [2019-12-17 12:38:32,262] jepsen worker 27 - jepsen.util 12267 :invoke :cas [9 [7 26]] INFO [2019-12-17 12:38:32,262] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76948,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:38:32,265] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76948,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:38:32,270] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76950,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:38:32,271] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76950,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:38:32,272] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76956,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:38:32,281] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76956,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:38:32,284] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76958,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:38:32,296] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76958,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:38:32,302] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76960,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:38:32,306] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76960,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:38:32,307] jepsen worker 33 - jepsen.util 18033 :info :cas [11 [16 47]] :RequestCanceledException INFO [2019-12-17 12:38:32,354] jepsen worker 33 - jepsen.util 18123 :invoke :read [11 nil] INFO [2019-12-17 12:38:32,382] jepsen worker 66 - jepsen.util 18426 :invoke :cas [22 [14 28]] INFO [2019-12-17 12:38:32,401] jepsen worker 11 - jepsen.util 13151 :info :cas [3 [12 32]] :RequestCanceledException INFO [2019-12-17 12:38:32,406] jepsen worker 3 - jepsen.util 12873 :ok :read [1 44] INFO [2019-12-17 12:38:32,407] jepsen worker 3 - jepsen.util 12873 :invoke :write [1 8] INFO [2019-12-17 12:38:32,407] jepsen worker 73 - jepsen.util 12313 :ok :read [24 32] INFO [2019-12-17 12:38:32,407] jepsen worker 73 - jepsen.util 12313 :invoke :read [24 nil] INFO [2019-12-17 12:38:32,408] jepsen worker 30 - jepsen.util 12720 :ok :read [10 49] INFO [2019-12-17 12:38:32,409] jepsen worker 30 - jepsen.util 12720 :invoke :cas [10 [13 49]] INFO [2019-12-17 12:38:32,416] jepsen worker 11 - jepsen.util 13241 :invoke :write [3 47] INFO [2019-12-17 12:38:32,421] jepsen worker 12 - jepsen.util 13332 :info :write [4 36] :RequestCanceledException INFO [2019-12-17 12:38:32,445] jepsen worker 28 - jepsen.util 11188 :info :cas [9 [1 22]] :RequestCanceledException INFO [2019-12-17 12:38:32,452] jepsen worker 78 - jepsen.util 13038 :info :write [26 36] :RequestCanceledException INFO [2019-12-17 12:38:32,456] jepsen worker 85 - jepsen.util 18625 :info :cas [28 [21 10]] :RequestCanceledException INFO [2019-12-17 12:38:32,463] jepsen worker 70 - jepsen.util 13840 :fail :cas [23 [20 48]] :ValueNotSwapFrom INFO [2019-12-17 12:38:32,464] jepsen worker 70 - jepsen.util 13840 :invoke :cas [23 [12 45]] INFO [2019-12-17 12:38:32,473] jepsen worker 78 - jepsen.util 13128 :invoke :cas [26 [3 9]] INFO [2019-12-17 12:38:32,473] jepsen worker 67 - jepsen.util 18427 :info :cas [22 [45 23]] :RequestCanceledException INFO [2019-12-17 12:38:32,474] jepsen worker 19 - jepsen.util 14149 :ok :write [6 39] INFO [2019-12-17 12:38:32,475] jepsen worker 19 - jepsen.util 14149 :invoke :read [6 nil] INFO [2019-12-17 12:38:32,478] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76969,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:38:32,478] jepsen worker 28 - jepsen.util 11278 :invoke :write [9 13] INFO [2019-12-17 12:38:32,479] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76969,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:38:32,483] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76973,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:38:32,484] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76973,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:38:32,499] jepsen worker 21 - jepsen.util 17661 :info :cas [7 [20 15]] :RequestCanceledException INFO [2019-12-17 12:38:32,502] jepsen worker 85 - jepsen.util 18715 :invoke :read [28 nil] INFO [2019-12-17 12:38:32,503] jepsen worker 12 - jepsen.util 13422 :invoke :write [4 3] INFO [2019-12-17 12:38:32,503] jepsen worker 67 - jepsen.util 18517 :invoke :cas [22 [37 28]] INFO [2019-12-17 12:38:32,523] jepsen worker 0 - jepsen.util 18540 :info :cas [0 [9 3]] :RequestCanceledException INFO [2019-12-17 12:38:32,530] jepsen worker 21 - jepsen.util 17751 :invoke :cas [7 [4 31]] INFO [2019-12-17 12:38:32,558] jepsen worker 0 - jepsen.util 18630 :invoke :cas [0 [33 34]] INFO [2019-12-17 12:38:32,599] jepsen worker 18 - jepsen.util 13968 :ok :write [6 31] INFO [2019-12-17 12:38:32,600] jepsen worker 18 - jepsen.util 13968 :invoke :cas [6 [26 7]] INFO [2019-12-17 12:38:32,609] jepsen worker 84 - jepsen.util 17544 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:38:32,609] jepsen worker 84 - jepsen.util 17544 :invoke :cas [28 [1 6]] INFO [2019-12-17 12:38:32,668] jepsen worker 31 - jepsen.util 11371 :fail :cas [10 [42 31]] :ValueNotSwapFrom INFO [2019-12-17 12:38:32,668] jepsen worker 31 - jepsen.util 11371 :invoke :write [10 20] INFO [2019-12-17 12:38:32,708] jepsen worker 80 - jepsen.util 12770 :fail :cas [26 [3 25]] :ValueNotSwapFrom INFO [2019-12-17 12:38:32,711] jepsen worker 80 - jepsen.util 12770 :invoke :write [26 5] INFO [2019-12-17 12:38:32,721] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76988,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:38:32,722] jepsen worker 79 - jepsen.util 13489 :info :cas [26 [47 8]] :RequestCanceledException INFO [2019-12-17 12:38:32,722] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76988,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:38:32,728] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76992,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:38:32,730] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76992,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:38:32,730] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76993,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:38:32,732] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":76993,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:38:32,751] jepsen worker 64 - jepsen.util 10504 :ok :read [21 7] INFO [2019-12-17 12:38:32,756] jepsen worker 79 - jepsen.util 13579 :invoke :read [26 nil] INFO [2019-12-17 12:38:32,757] jepsen worker 87 - jepsen.util 12867 :info :cas [29 [0 29]] :RequestCanceledException INFO [2019-12-17 12:38:32,758] jepsen worker 64 - jepsen.util 10504 :invoke :read [21 nil] INFO [2019-12-17 12:38:32,797] jepsen worker 87 - jepsen.util 12957 :invoke :cas [29 [15 40]] INFO [2019-12-17 12:38:32,823] jepsen worker 69 - jepsen.util 13749 :ok :write [23 19] INFO [2019-12-17 12:38:32,828] jepsen worker 69 - jepsen.util 13749 :invoke :read [23 nil] INFO [2019-12-17 12:38:32,866] jepsen worker 23 - jepsen.util 18293 :info :write [7 48] :RequestCanceledException INFO [2019-12-17 12:38:32,866] jepsen worker 51 - jepsen.util 12561 :info :write [17 32] :RequestCanceledException INFO [2019-12-17 12:38:32,883] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77003,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:38:32,884] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77003,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:38:32,884] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77004,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:38:32,886] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77004,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:38:32,908] jepsen worker 23 - jepsen.util 18383 :invoke :cas [7 [8 9]] INFO [2019-12-17 12:38:32,910] jepsen worker 51 - jepsen.util 12651 :invoke :read [17 nil] INFO [2019-12-17 12:38:32,929] jepsen worker 4 - jepsen.util 12424 :info :write [1 32] :RequestCanceledException INFO [2019-12-17 12:38:32,941] jepsen worker 77 - jepsen.util 19427 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:38:32,942] jepsen worker 77 - jepsen.util 19427 :invoke :read [25 nil] INFO [2019-12-17 12:38:32,968] jepsen worker 56 - jepsen.util 14006 :info :write [18 23] :RequestCanceledException INFO [2019-12-17 12:38:32,968] jepsen worker 4 - jepsen.util 12514 :invoke :cas [1 [10 42]] INFO [2019-12-17 12:38:32,984] jepsen worker 72 - jepsen.util 13302 :ok :read [24 32] INFO [2019-12-17 12:38:32,987] jepsen worker 72 - jepsen.util 13302 :invoke :write [24 38] INFO [2019-12-17 12:38:32,987] jepsen worker 56 - jepsen.util 14096 :invoke :cas [18 [48 20]] INFO [2019-12-17 12:38:33,023] jepsen worker 89 - jepsen.util 13229 :info :cas [29 [34 13]] :RequestCanceledException INFO [2019-12-17 12:38:33,047] jepsen worker 82 - jepsen.util 18172 :info :write [27 14] :RequestCanceledException INFO [2019-12-17 12:38:33,087] jepsen worker 81 - jepsen.util 17631 :info :cas [27 [35 34]] :RequestCanceledException INFO [2019-12-17 12:38:33,090] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77014,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:38:33,091] jepsen worker 89 - jepsen.util 13319 :invoke :cas [29 [7 41]] INFO [2019-12-17 12:38:33,103] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77014,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:38:33,112] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77018,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:38:33,117] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77018,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:38:33,134] jepsen worker 15 - jepsen.util 17205 :info :cas [5 [15 25]] :RequestCanceledException INFO [2019-12-17 12:38:33,141] jepsen worker 35 - jepsen.util 18395 :info :cas [11 [7 26]] :RequestCanceledException INFO [2019-12-17 12:38:33,177] jepsen worker 58 - jepsen.util 18328 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:38:33,178] jepsen worker 82 - jepsen.util 18262 :invoke :cas [27 [39 49]] INFO [2019-12-17 12:38:33,187] jepsen worker 58 - jepsen.util 18328 :invoke :cas [19 [2 23]] INFO [2019-12-17 12:38:33,187] jepsen worker 14 - jepsen.util 12974 :ok :read [4 39] INFO [2019-12-17 12:38:33,188] jepsen worker 14 - jepsen.util 12974 :invoke :read [4 nil] INFO [2019-12-17 12:38:33,216] jepsen worker 15 - jepsen.util 17295 :invoke :read [5 nil] INFO [2019-12-17 12:38:33,221] jepsen worker 65 - jepsen.util 11675 :fail :cas [21 [8 43]] :ValueNotSwapFrom INFO [2019-12-17 12:38:33,228] jepsen worker 65 - jepsen.util 11675 :invoke :read [21 nil] INFO [2019-12-17 12:38:33,239] jepsen worker 25 - jepsen.util 19285 :info :cas [8 [18 12]] :RequestCanceledException INFO [2019-12-17 12:38:33,245] jepsen worker 81 - jepsen.util 17721 :invoke :write [27 23] INFO [2019-12-17 12:38:33,249] jepsen worker 35 - jepsen.util 18485 :invoke :write [11 49] INFO [2019-12-17 12:38:33,267] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77034,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:38:33,275] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77034,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:38:33,275] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77039,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:38:33,291] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77039,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:38:33,310] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77026,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:38:33,317] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77026,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:38:33,348] jepsen worker 8 - jepsen.util 17738 :info :cas [2 [19 18]] :RequestCanceledException INFO [2019-12-17 12:38:33,357] jepsen worker 2 - jepsen.util 17462 :info :cas [0 [30 10]] :RequestCanceledException INFO [2019-12-17 12:38:33,394] jepsen worker 25 - jepsen.util 19375 :invoke :cas [8 [4 4]] INFO [2019-12-17 12:38:33,411] jepsen worker 59 - jepsen.util 17159 :info :cas [19 [46 28]] :RequestCanceledException INFO [2019-12-17 12:38:33,418] jepsen worker 48 - jepsen.util 16878 :info :cas [16 [48 29]] :RequestCanceledException INFO [2019-12-17 12:38:33,427] jepsen worker 7 - jepsen.util 18007 :info :cas [2 [17 46]] :RequestCanceledException INFO [2019-12-17 12:38:33,476] jepsen worker 50 - jepsen.util 18050 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:38:33,481] jepsen worker 50 - jepsen.util 18050 :invoke :cas [16 [31 38]] INFO [2019-12-17 12:38:33,487] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:38:33,496] jepsen worker 8 - jepsen.util 17828 :invoke :cas [2 [0 21]] INFO [2019-12-17 12:38:33,497] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77044,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:38:33,498] jepsen nemesis - couchbase.util Rebalance check count: 606 INFO [2019-12-17 12:38:33,499] jepsen worker 2 - jepsen.util 17552 :invoke :write [0 1] INFO [2019-12-17 12:38:33,500] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77044,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:38:33,502] jepsen nemesis - couchbase.util Rebalance stuck count: 587 INFO [2019-12-17 12:38:33,503] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77047,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:38:33,505] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77047,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:38:33,508] jepsen worker 63 - jepsen.util 11673 :ok :read [21 7] INFO [2019-12-17 12:38:33,508] jepsen worker 39 - jepsen.util 17679 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:38:33,515] jepsen worker 63 - jepsen.util 11673 :invoke :read [21 nil] INFO [2019-12-17 12:38:33,518] jepsen worker 39 - jepsen.util 17679 :invoke :cas [13 [4 30]] INFO [2019-12-17 12:38:33,528] jepsen worker 48 - jepsen.util 16968 :invoke :write [16 40] INFO [2019-12-17 12:38:33,537] jepsen worker 61 - jepsen.util 19501 :info :cas [20 [9 28]] :RequestCanceledException INFO [2019-12-17 12:38:33,545] jepsen worker 59 - jepsen.util 17249 :invoke :cas [19 [16 16]] INFO [2019-12-17 12:38:33,547] jepsen worker 7 - jepsen.util 18097 :invoke :write [2 40] INFO [2019-12-17 12:38:33,554] jepsen worker 68 - jepsen.util 17348 :info :cas [22 [46 34]] :RequestCanceledException INFO [2019-12-17 12:38:33,555] jepsen worker 5 - jepsen.util 12875 :info :write [1 21] :RequestCanceledException INFO [2019-12-17 12:38:33,576] jepsen worker 61 - jepsen.util 19591 :invoke :write [20 18] INFO [2019-12-17 12:38:33,578] jepsen worker 5 - jepsen.util 12965 :invoke :cas [1 [17 12]] INFO [2019-12-17 12:38:33,579] jepsen worker 43 - jepsen.util 18313 :info :write [14 32] :RequestCanceledException INFO [2019-12-17 12:38:33,611] jepsen worker 42 - jepsen.util 18942 :info :cas [14 [14 25]] :RequestCanceledException INFO [2019-12-17 12:38:33,628] jepsen worker 83 - jepsen.util 18083 :info :cas [27 [40 11]] :RequestCanceledException INFO [2019-12-17 12:38:33,628] jepsen worker 53 - jepsen.util 12923 :info :cas [17 [1 6]] :RequestCanceledException INFO [2019-12-17 12:38:33,632] jepsen worker 42 - jepsen.util 19032 :invoke :cas [14 [24 13]] INFO [2019-12-17 12:38:33,635] jepsen worker 68 - jepsen.util 17438 :invoke :read [22 nil] INFO [2019-12-17 12:38:33,635] jepsen worker 22 - jepsen.util 18652 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:38:33,636] jepsen worker 53 - jepsen.util 13013 :invoke :cas [17 [31 27]] INFO [2019-12-17 12:38:33,637] jepsen worker 22 - jepsen.util 18652 :invoke :write [7 25] INFO [2019-12-17 12:38:33,639] jepsen worker 43 - jepsen.util 18403 :invoke :cas [14 [43 11]] INFO [2019-12-17 12:38:33,649] jepsen worker 76 - jepsen.util 18436 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:38:33,649] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77060,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:38:33,649] jepsen worker 76 - jepsen.util 18436 :invoke :cas [25 [37 47]] INFO [2019-12-17 12:38:33,650] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77060,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:38:33,650] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77062,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:38:33,651] jepsen worker 44 - jepsen.util 18944 :info :write [14 45] :RequestCanceledException INFO [2019-12-17 12:38:33,652] jepsen worker 83 - jepsen.util 18173 :invoke :write [27 27] INFO [2019-12-17 12:38:33,652] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77062,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:38:33,654] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77064,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:38:33,654] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77064,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:38:33,667] jepsen worker 1 - jepsen.util 18631 :info :cas [0 [11 15]] :RequestCanceledException INFO [2019-12-17 12:38:33,693] jepsen worker 44 - jepsen.util 19034 :invoke :read [14 nil] INFO [2019-12-17 12:38:33,752] jepsen worker 57 - jepsen.util 17697 :info :write [19 42] :RequestCanceledException INFO [2019-12-17 12:38:33,766] jepsen worker 1 - jepsen.util 18721 :invoke :cas [0 [13 40]] INFO [2019-12-17 12:38:33,775] jepsen worker 16 - jepsen.util 19816 :info :write [5 12] :RequestCanceledException INFO [2019-12-17 12:38:33,831] jepsen worker 57 - jepsen.util 17787 :invoke :write [19 1] INFO [2019-12-17 12:38:33,841] jepsen worker 36 - jepsen.util 13896 :info :cas [12 [13 12]] :RequestCanceledException INFO [2019-12-17 12:38:33,855] jepsen worker 20 - jepsen.util 13070 :fail :read [6 nil] :RequestTimeoutException INFO [2019-12-17 12:38:33,862] jepsen worker 20 - jepsen.util 13070 :invoke :read [6 nil] INFO [2019-12-17 12:38:33,891] jepsen worker 16 - jepsen.util 19906 :invoke :write [5 26] INFO [2019-12-17 12:38:33,895] jepsen worker 36 - jepsen.util 13986 :invoke :read [12 nil] INFO [2019-12-17 12:38:33,896] jepsen worker 55 - jepsen.util 14005 :ok :read [18 26] INFO [2019-12-17 12:38:33,899] jepsen worker 55 - jepsen.util 14005 :invoke :read [18 nil] INFO [2019-12-17 12:38:33,903] jepsen worker 40 - jepsen.util 18850 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:38:33,904] jepsen worker 40 - jepsen.util 18850 :invoke :write [13 13] INFO [2019-12-17 12:38:33,912] jepsen worker 6 - jepsen.util 18366 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:38:33,914] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77069,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:38:33,927] jepsen worker 34 - jepsen.util 19114 :info :cas [11 [4 49]] :RequestCanceledException INFO [2019-12-17 12:38:33,929] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77069,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:38:33,929] jepsen worker 6 - jepsen.util 18366 :invoke :cas [2 [16 23]] INFO [2019-12-17 12:38:33,943] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77072,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:38:33,950] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77072,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:38:33,975] jepsen worker 34 - jepsen.util 19204 :invoke :cas [11 [8 7]] INFO [2019-12-17 12:38:34,041] jepsen worker 52 - jepsen.util 13012 :fail :cas [17 [32 21]] :ValueNotSwapFrom INFO [2019-12-17 12:38:34,043] jepsen worker 52 - jepsen.util 13012 :invoke :read [17 nil] INFO [2019-12-17 12:38:34,076] jepsen worker 9 - jepsen.util 13779 :ok :write [3 48] INFO [2019-12-17 12:38:34,076] jepsen worker 9 - jepsen.util 13779 :invoke :cas [3 [20 24]] INFO [2019-12-17 12:38:34,087] jepsen worker 71 - jepsen.util 14201 :ok :read [23 36] INFO [2019-12-17 12:38:34,088] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77083,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:38:34,088] jepsen worker 71 - jepsen.util 14201 :invoke :cas [23 [38 13]] INFO [2019-12-17 12:38:34,098] jepsen worker 29 - jepsen.util 11549 :ok :read [9 29] INFO [2019-12-17 12:38:34,099] jepsen worker 29 - jepsen.util 11549 :invoke :read [9 nil] INFO [2019-12-17 12:38:34,099] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77083,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:38:34,112] jepsen worker 24 - jepsen.util 17394 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:38:34,118] jepsen worker 24 - jepsen.util 17394 :invoke :write [8 47] INFO [2019-12-17 12:38:34,127] jepsen worker 26 - jepsen.util 19106 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:38:34,129] jepsen worker 26 - jepsen.util 19106 :invoke :cas [8 [49 14]] INFO [2019-12-17 12:38:34,129] jepsen worker 49 - jepsen.util 18229 :info :write [16 32] :RequestCanceledException INFO [2019-12-17 12:38:34,158] jepsen worker 88 - jepsen.util 13228 :fail :cas [29 [40 41]] :ValueNotSwapFrom INFO [2019-12-17 12:38:34,159] jepsen worker 88 - jepsen.util 13228 :invoke :write [29 39] INFO [2019-12-17 12:38:34,173] jepsen worker 17 - jepsen.util 18107 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:38:34,182] jepsen worker 17 - jepsen.util 18107 :invoke :write [5 6] INFO [2019-12-17 12:38:34,189] jepsen worker 11 - jepsen.util 13241 :ok :write [3 47] INFO [2019-12-17 12:38:34,191] jepsen worker 11 - jepsen.util 13241 :invoke :read [3 nil] INFO [2019-12-17 12:38:34,216] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77095,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:38:34,216] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77095,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:38:34,217] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77097,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:38:35,647] jepsen worker 49 - jepsen.util 18319 :invoke :read [16 nil] INFO [2019-12-17 12:38:35,647] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77097,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:38:35,647] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77110,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:38:35,647] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77110,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:38:35,648] jepsen worker 27 - jepsen.util 12267 :fail :cas [9 [7 26]] :ValueNotSwapFrom INFO [2019-12-17 12:38:35,648] jepsen worker 27 - jepsen.util 12267 :invoke :cas [9 [42 8]] INFO [2019-12-17 12:38:35,657] jepsen worker 74 - jepsen.util 13124 :fail :cas [24 [43 37]] :ValueNotSwapFrom INFO [2019-12-17 12:38:35,657] jepsen worker 64 - jepsen.util 10504 :ok :read [21 7] INFO [2019-12-17 12:38:35,658] jepsen worker 74 - jepsen.util 13124 :invoke :cas [24 [43 6]] INFO [2019-12-17 12:38:35,658] jepsen worker 64 - jepsen.util 10504 :invoke :write [21 24] INFO [2019-12-17 12:38:35,658] jepsen worker 32 - jepsen.util 13532 :fail :cas [10 [32 20]] :ValueNotSwapFrom INFO [2019-12-17 12:38:35,660] jepsen worker 32 - jepsen.util 13532 :invoke :write [10 23] INFO [2019-12-17 12:38:35,661] jepsen worker 10 - jepsen.util 13150 :ok :read [3 1] INFO [2019-12-17 12:38:35,662] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77108,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:38:35,662] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77117,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:38:35,662] jepsen worker 13 - jepsen.util 12883 :info :cas [4 [14 14]] :RequestCanceledException INFO [2019-12-17 12:38:35,663] jepsen worker 47 - jepsen.util 14447 :info :write [15 37] :RequestCanceledException INFO [2019-12-17 12:38:35,663] jepsen worker 10 - jepsen.util 13150 :invoke :write [3 32] INFO [2019-12-17 12:38:35,663] jepsen worker 66 - jepsen.util 18426 :info :cas [22 [14 28]] :RequestCanceledException INFO [2019-12-17 12:38:35,664] jepsen worker 85 - jepsen.util 18715 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:38:35,664] jepsen worker 62 - jepsen.util 18872 :info :write [20 43] :RequestCanceledException INFO [2019-12-17 12:38:35,664] jepsen worker 67 - jepsen.util 18517 :info :cas [22 [37 28]] :RequestCanceledException INFO [2019-12-17 12:38:35,664] jepsen worker 8 - jepsen.util 17828 :info :cas [2 [0 21]] :RequestCanceledException INFO [2019-12-17 12:38:35,664] jepsen worker 51 - jepsen.util 12651 :fail :read [17 nil] :RequestTimeoutException INFO [2019-12-17 12:38:35,664] jepsen worker 84 - jepsen.util 17544 :info :cas [28 [1 6]] :RequestCanceledException INFO [2019-12-17 12:38:35,664] jepsen worker 5 - jepsen.util 12965 :info :cas [1 [17 12]] :RequestCanceledException INFO [2019-12-17 12:38:35,664] jepsen worker 82 - jepsen.util 18262 :info :cas [27 [39 49]] :RequestCanceledException INFO [2019-12-17 12:38:35,664] jepsen worker 76 - jepsen.util 18436 :info :cas [25 [37 47]] :RequestCanceledException INFO [2019-12-17 12:38:35,664] jepsen worker 89 - jepsen.util 13319 :info :cas [29 [7 41]] :RequestCanceledException INFO [2019-12-17 12:38:35,664] jepsen worker 80 - jepsen.util 12770 :info :write [26 5] :RequestCanceledException INFO [2019-12-17 12:38:35,664] jepsen worker 30 - jepsen.util 12720 :info :cas [10 [13 49]] :RequestCanceledException INFO [2019-12-17 12:38:35,664] jepsen worker 87 - jepsen.util 12957 :info :cas [29 [15 40]] :RequestCanceledException INFO [2019-12-17 12:38:35,664] jepsen worker 75 - jepsen.util 17985 :info :write [25 9] :RequestCanceledException INFO [2019-12-17 12:38:35,664] jepsen worker 79 - jepsen.util 13579 :fail :read [26 nil] :RequestTimeoutException INFO [2019-12-17 12:38:35,664] jepsen worker 60 - jepsen.util 19230 :info :write [20 45] :RequestCanceledException INFO [2019-12-17 12:38:35,664] jepsen worker 28 - jepsen.util 11278 :info :write [9 13] :RequestCanceledException INFO [2019-12-17 12:38:35,665] jepsen worker 19 - jepsen.util 14149 :fail :read [6 nil] :RequestTimeoutException INFO [2019-12-17 12:38:35,665] jepsen worker 41 - jepsen.util 19481 :info :write [13 2] :RequestCanceledException INFO [2019-12-17 12:38:35,665] jepsen worker 68 - jepsen.util 17438 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:38:35,665] jepsen worker 14 - jepsen.util 12974 :fail :read [4 nil] :RequestTimeoutException INFO [2019-12-17 12:38:35,665] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77108,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:38:35,666] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77117,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:38:35,666] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77113,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:38:35,666] jepsen worker 50 - jepsen.util 18050 :info :cas [16 [31 38]] :RequestCanceledException INFO [2019-12-17 12:38:35,666] jepsen worker 39 - jepsen.util 17679 :info :cas [13 [4 30]] :RequestCanceledException INFO [2019-12-17 12:38:35,666] jepsen worker 86 - jepsen.util 18986 :info :write [28 9] :RequestCanceledException INFO [2019-12-17 12:38:35,666] jepsen worker 38 - jepsen.util 13988 :info :write [12 47] :RequestCanceledException INFO [2019-12-17 12:38:35,666] jepsen worker 25 - jepsen.util 19375 :info :cas [8 [4 4]] :RequestCanceledException INFO [2019-12-17 12:38:35,666] jepsen worker 33 - jepsen.util 18123 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:38:35,666] jepsen worker 77 - jepsen.util 19427 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:38:35,667] jepsen worker 0 - jepsen.util 18630 :info :cas [0 [33 34]] :RequestCanceledException INFO [2019-12-17 12:38:35,667] jepsen worker 15 - jepsen.util 17295 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:38:35,667] jepsen worker 23 - jepsen.util 18383 :info :cas [7 [8 9]] :RequestCanceledException INFO [2019-12-17 12:38:35,667] jepsen worker 78 - jepsen.util 13128 :info :cas [26 [3 9]] :RequestCanceledException INFO [2019-12-17 12:38:35,667] jepsen worker 58 - jepsen.util 18328 :info :cas [19 [2 23]] :RequestCanceledException INFO [2019-12-17 12:38:35,667] jepsen worker 56 - jepsen.util 14096 :info :cas [18 [48 20]] :RequestCanceledException INFO [2019-12-17 12:38:35,667] jepsen worker 69 - jepsen.util 13749 :fail :read [23 nil] :RequestTimeoutException INFO [2019-12-17 12:38:35,667] jepsen worker 3 - jepsen.util 12873 :info :write [1 8] :RequestCanceledException INFO [2019-12-17 12:38:35,667] jepsen worker 73 - jepsen.util 12313 :ok :read [24 32] INFO [2019-12-17 12:38:35,667] jepsen worker 54 - jepsen.util 14004 :info :write [18 23] :RequestCanceledException INFO [2019-12-17 12:38:35,667] jepsen worker 31 - jepsen.util 11371 :info :write [10 20] :RequestCanceledException INFO [2019-12-17 12:38:35,667] jepsen worker 12 - jepsen.util 13422 :info :write [4 3] :RequestCanceledException INFO [2019-12-17 12:38:35,667] jepsen worker 45 - jepsen.util 12825 :fail :cas [15 [23 48]] :ValueNotSwapFrom INFO [2019-12-17 12:38:35,669] jepsen worker 42 - jepsen.util 19032 :info :cas [14 [24 13]] :RequestCanceledException INFO [2019-12-17 12:38:35,669] jepsen worker 37 - jepsen.util 12817 :ok :read [12 18] INFO [2019-12-17 12:38:35,669] jepsen worker 85 - jepsen.util 18715 :invoke :write [28 24] INFO [2019-12-17 12:38:35,670] jepsen worker 51 - jepsen.util 12651 :invoke :cas [17 [25 38]] INFO [2019-12-17 12:38:35,672] jepsen worker 66 - jepsen.util 18516 :invoke :cas [22 [14 14]] INFO [2019-12-17 12:38:35,672] jepsen worker 67 - jepsen.util 18607 :invoke :cas [22 [34 34]] INFO [2019-12-17 12:38:35,675] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77157,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:38:35,675] jepsen worker 84 - jepsen.util 17634 :invoke :cas [28 [40 16]] INFO [2019-12-17 12:38:35,676] jepsen worker 62 - jepsen.util 18962 :invoke :write [20 16] INFO [2019-12-17 12:38:35,676] jepsen worker 13 - jepsen.util 12973 :invoke :write [4 44] INFO [2019-12-17 12:38:35,676] jepsen worker 5 - jepsen.util 13055 :invoke :write [1 37] INFO [2019-12-17 12:38:35,676] jepsen worker 82 - jepsen.util 18352 :invoke :write [27 42] INFO [2019-12-17 12:38:35,677] jepsen worker 47 - jepsen.util 14537 :invoke :write [15 11] INFO [2019-12-17 12:38:35,677] jepsen worker 8 - jepsen.util 17918 :invoke :write [2 19] INFO [2019-12-17 12:38:35,678] jepsen worker 4 - jepsen.util 12514 :info :cas [1 [10 42]] :RequestCanceledException INFO [2019-12-17 12:38:35,678] jepsen worker 21 - jepsen.util 17751 :info :cas [7 [4 31]] :RequestCanceledException INFO [2019-12-17 12:38:35,679] jepsen worker 76 - jepsen.util 18526 :invoke :cas [25 [46 22]] INFO [2019-12-17 12:38:35,680] jepsen worker 43 - jepsen.util 18403 :info :cas [14 [43 11]] :RequestCanceledException INFO [2019-12-17 12:38:35,680] jepsen worker 59 - jepsen.util 17249 :info :cas [19 [16 16]] :RequestCanceledException INFO [2019-12-17 12:38:35,681] jepsen worker 46 - jepsen.util 13006 :ok :read [15 33] INFO [2019-12-17 12:38:35,681] jepsen worker 87 - jepsen.util 13047 :invoke :cas [29 [49 22]] INFO [2019-12-17 12:38:35,681] jepsen worker 70 - jepsen.util 13840 :fail :cas [23 [12 45]] :ValueNotSwapFrom INFO [2019-12-17 12:38:35,682] jepsen worker 30 - jepsen.util 12810 :invoke :cas [10 [28 44]] INFO [2019-12-17 12:38:35,683] jepsen worker 79 - jepsen.util 13579 :invoke :cas [26 [27 14]] INFO [2019-12-17 12:38:35,683] jepsen worker 89 - jepsen.util 13409 :invoke :write [29 27] INFO [2019-12-17 12:38:35,684] jepsen worker 80 - jepsen.util 12860 :invoke :write [26 35] INFO [2019-12-17 12:38:35,685] jepsen worker 19 - jepsen.util 14149 :invoke :cas [6 [48 21]] INFO [2019-12-17 12:38:35,686] jepsen worker 28 - jepsen.util 11368 :invoke :write [9 23] INFO [2019-12-17 12:38:35,687] jepsen worker 68 - jepsen.util 17438 :invoke :cas [22 [13 4]] INFO [2019-12-17 12:38:35,687] jepsen worker 75 - jepsen.util 18075 :invoke :read [25 nil] INFO [2019-12-17 12:38:35,688] jepsen worker 14 - jepsen.util 12974 :invoke :cas [4 [27 12]] INFO [2019-12-17 12:38:35,689] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77112,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:38:35,690] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77122,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:38:35,690] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77113,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:38:35,690] jepsen worker 41 - jepsen.util 19571 :invoke :write [13 42] INFO [2019-12-17 12:38:35,692] jepsen worker 33 - jepsen.util 18123 :invoke :read [11 nil] INFO [2019-12-17 12:38:35,693] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:38:35,694] jepsen worker 15 - jepsen.util 17295 :invoke :write [5 27] INFO [2019-12-17 12:38:35,694] jepsen worker 77 - jepsen.util 19427 :invoke :read [25 nil] INFO [2019-12-17 12:38:35,696] jepsen worker 69 - jepsen.util 13749 :invoke :cas [23 [31 26]] INFO [2019-12-17 12:38:35,697] jepsen worker 73 - jepsen.util 12313 :invoke :read [24 nil] INFO [2019-12-17 12:38:35,697] jepsen worker 60 - jepsen.util 19320 :invoke :write [20 1] INFO [2019-12-17 12:38:35,697] jepsen worker 86 - jepsen.util 19076 :invoke :write [28 45] INFO [2019-12-17 12:38:35,698] jepsen worker 45 - jepsen.util 12825 :invoke :write [15 32] INFO [2019-12-17 12:38:35,699] jepsen worker 56 - jepsen.util 14186 :invoke :cas [18 [24 48]] INFO [2019-12-17 12:38:35,699] jepsen worker 23 - jepsen.util 18473 :invoke :write [7 2] INFO [2019-12-17 12:38:35,699] jepsen worker 37 - jepsen.util 12817 :invoke :cas [12 [9 43]] INFO [2019-12-17 12:38:35,700] jepsen worker 50 - jepsen.util 18140 :invoke :read [16 nil] INFO [2019-12-17 12:38:35,701] jepsen worker 3 - jepsen.util 12963 :invoke :write [1 36] INFO [2019-12-17 12:38:35,701] jepsen worker 12 - jepsen.util 13512 :invoke :cas [4 [11 33]] INFO [2019-12-17 12:38:35,701] jepsen worker 31 - jepsen.util 11461 :invoke :cas [10 [46 34]] INFO [2019-12-17 12:38:35,701] jepsen worker 78 - jepsen.util 13218 :invoke :cas [26 [10 11]] INFO [2019-12-17 12:38:35,701] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77157,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:38:35,701] jepsen worker 39 - jepsen.util 17769 :invoke :read [13 nil] INFO [2019-12-17 12:38:35,702] jepsen worker 0 - jepsen.util 18720 :invoke :cas [0 [21 40]] INFO [2019-12-17 12:38:35,702] jepsen worker 29 - jepsen.util 11549 :ok :read [9 17] INFO [2019-12-17 12:38:35,703] jepsen worker 54 - jepsen.util 14094 :invoke :cas [18 [48 45]] INFO [2019-12-17 12:38:35,704] jepsen worker 25 - jepsen.util 19465 :invoke :read [8 nil] INFO [2019-12-17 12:38:35,704] jepsen worker 11 - jepsen.util 13241 :ok :read [3 47] INFO [2019-12-17 12:38:35,706] jepsen worker 58 - jepsen.util 18418 :invoke :cas [19 [6 36]] INFO [2019-12-17 12:38:35,707] jepsen worker 42 - jepsen.util 19122 :invoke :write [14 7] INFO [2019-12-17 12:38:35,708] jepsen worker 38 - jepsen.util 14078 :invoke :cas [12 [29 28]] INFO [2019-12-17 12:38:35,708] jepsen worker 46 - jepsen.util 13006 :invoke :read [15 nil] INFO [2019-12-17 12:38:35,709] jepsen worker 21 - jepsen.util 17841 :invoke :write [7 36] INFO [2019-12-17 12:38:35,709] jepsen worker 4 - jepsen.util 12604 :invoke :cas [1 [27 46]] INFO [2019-12-17 12:38:35,709] jepsen worker 59 - jepsen.util 17339 :invoke :cas [19 [22 31]] INFO [2019-12-17 12:38:35,710] jepsen worker 70 - jepsen.util 13840 :invoke :cas [23 [24 29]] INFO [2019-12-17 12:38:35,710] jepsen worker 43 - jepsen.util 18493 :invoke :write [14 15] INFO [2019-12-17 12:38:35,714] jepsen worker 63 - jepsen.util 11673 :ok :read [21 7] INFO [2019-12-17 12:38:35,714] jepsen worker 18 - jepsen.util 13968 :fail :cas [6 [26 7]] :ValueNotSwapFrom INFO [2019-12-17 12:38:35,715] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77112,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:38:35,716] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77122,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:38:35,716] jepsen worker 65 - jepsen.util 11675 :ok :read [21 7] INFO [2019-12-17 12:38:35,716] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77115,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:38:35,717] jepsen nemesis - couchbase.util Rebalance check count: 607 INFO [2019-12-17 12:38:35,726] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77159,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:38:35,729] jepsen worker 29 - jepsen.util 11549 :invoke :write [9 26] INFO [2019-12-17 12:38:35,730] jepsen worker 11 - jepsen.util 13241 :invoke :write [3 25] INFO [2019-12-17 12:38:35,734] jepsen worker 18 - jepsen.util 13968 :invoke :write [6 3] INFO [2019-12-17 12:38:35,734] jepsen worker 63 - jepsen.util 11673 :invoke :write [21 42] INFO [2019-12-17 12:38:35,734] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77124,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:38:35,734] jepsen worker 65 - jepsen.util 11675 :invoke :read [21 nil] INFO [2019-12-17 12:38:35,735] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77115,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:38:35,735] jepsen nemesis - couchbase.util Rebalance stuck count: 588 INFO [2019-12-17 12:38:35,736] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77159,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:38:35,736] jepsen worker 53 - jepsen.util 13013 :info :cas [17 [31 27]] :RequestCanceledException INFO [2019-12-17 12:38:35,736] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77124,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:38:35,737] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77121,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:38:35,738] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77128,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:38:35,739] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77128,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:38:35,739] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77121,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:38:35,739] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77129,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:38:35,739] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77123,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:38:35,740] jepsen worker 44 - jepsen.util 19034 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:38:35,741] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77129,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:38:35,741] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77123,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:38:35,741] jepsen worker 53 - jepsen.util 13103 :invoke :cas [17 [2 19]] INFO [2019-12-17 12:38:35,741] jepsen worker 44 - jepsen.util 19034 :invoke :cas [14 [1 26]] INFO [2019-12-17 12:38:35,769] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77146,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:38:35,769] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77146,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:38:35,769] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77148,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:38:35,769] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77148,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:38:35,769] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77154,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:38:35,769] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77154,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:38:35,783] jepsen worker 36 - jepsen.util 13986 :ok :read [12 18] INFO [2019-12-17 12:38:35,783] jepsen worker 36 - jepsen.util 13986 :invoke :write [12 23] INFO [2019-12-17 12:38:35,810] jepsen worker 30 - jepsen.util 12810 :fail :cas [10 [28 44]] :ValueNotSwapFrom INFO [2019-12-17 12:38:35,811] jepsen worker 30 - jepsen.util 12810 :invoke :write [10 37] INFO [2019-12-17 12:38:35,818] jepsen worker 55 - jepsen.util 14005 :ok :read [18 26] INFO [2019-12-17 12:38:35,818] jepsen worker 55 - jepsen.util 14005 :invoke :read [18 nil] INFO [2019-12-17 12:38:35,840] jepsen worker 20 - jepsen.util 13070 :ok :read [6 31] INFO [2019-12-17 12:38:35,840] jepsen worker 20 - jepsen.util 13070 :invoke :cas [6 [14 4]] INFO [2019-12-17 12:38:35,864] jepsen worker 1 - jepsen.util 18721 :info :cas [0 [13 40]] :RequestCanceledException INFO [2019-12-17 12:38:35,867] jepsen worker 1 - jepsen.util 18811 :invoke :read [0 nil] INFO [2019-12-17 12:38:35,891] jepsen worker 65 - jepsen.util 11675 :ok :read [21 7] INFO [2019-12-17 12:38:35,891] jepsen worker 65 - jepsen.util 11675 :invoke :cas [21 [1 21]] INFO [2019-12-17 12:38:35,905] jepsen worker 52 - jepsen.util 13012 :ok :read [17 24] INFO [2019-12-17 12:38:35,905] jepsen worker 52 - jepsen.util 13012 :invoke :write [17 16] INFO [2019-12-17 12:38:35,932] jepsen worker 51 - jepsen.util 12651 :fail :cas [17 [25 38]] :ValueNotSwapFrom INFO [2019-12-17 12:38:35,932] jepsen worker 51 - jepsen.util 12651 :invoke :read [17 nil] INFO [2019-12-17 12:38:35,946] jepsen worker 14 - jepsen.util 12974 :fail :cas [4 [27 12]] :ValueNotSwapFrom INFO [2019-12-17 12:38:35,946] jepsen worker 14 - jepsen.util 12974 :invoke :cas [4 [39 37]] INFO [2019-12-17 12:38:35,949] jepsen worker 72 - jepsen.util 13302 :info :write [24 38] :RequestCanceledException INFO [2019-12-17 12:38:35,950] jepsen worker 72 - jepsen.util 13392 :invoke :read [24 nil] INFO [2019-12-17 12:38:35,951] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77235,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:38:35,951] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77235,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:38:35,951] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77238,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:38:35,951] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77238,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:38:35,951] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77239,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:38:35,951] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77239,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:38:35,965] jepsen worker 65 - jepsen.util 11675 :fail :cas [21 [1 21]] :ValueNotSwapFrom INFO [2019-12-17 12:38:35,965] jepsen worker 65 - jepsen.util 11675 :invoke :cas [21 [45 24]] INFO [2019-12-17 12:38:35,969] jepsen worker 64 - jepsen.util 10504 :ok :write [21 24] INFO [2019-12-17 12:38:35,969] jepsen worker 64 - jepsen.util 10504 :invoke :write [21 1] INFO [2019-12-17 12:38:35,990] jepsen worker 37 - jepsen.util 12817 :fail :cas [12 [9 43]] :ValueNotSwapFrom INFO [2019-12-17 12:38:35,990] jepsen worker 37 - jepsen.util 12817 :invoke :cas [12 [3 49]] INFO [2019-12-17 12:38:35,992] jepsen worker 13 - jepsen.util 12973 :ok :write [4 44] INFO [2019-12-17 12:38:35,992] jepsen worker 13 - jepsen.util 12973 :invoke :write [4 44] INFO [2019-12-17 12:38:36,001] jepsen worker 6 - jepsen.util 18366 :info :cas [2 [16 23]] :RequestCanceledException INFO [2019-12-17 12:38:36,003] jepsen worker 6 - jepsen.util 18456 :invoke :cas [2 [30 6]] INFO [2019-12-17 12:38:36,004] jepsen worker 65 - jepsen.util 11675 :fail :cas [21 [45 24]] :ValueNotSwapFrom INFO [2019-12-17 12:38:36,005] jepsen worker 65 - jepsen.util 11675 :invoke :read [21 nil] INFO [2019-12-17 12:38:36,020] jepsen worker 65 - jepsen.util 11675 :ok :read [21 24] INFO [2019-12-17 12:38:36,020] jepsen worker 65 - jepsen.util 11675 :invoke :write [21 23] INFO [2019-12-17 12:38:36,034] jepsen worker 34 - jepsen.util 19204 :info :cas [11 [8 7]] :RequestCanceledException INFO [2019-12-17 12:38:36,035] jepsen worker 34 - jepsen.util 19294 :invoke :write [11 14] INFO [2019-12-17 12:38:36,055] jepsen worker 54 - jepsen.util 14094 :fail :cas [18 [48 45]] :ValueNotSwapFrom INFO [2019-12-17 12:38:36,055] jepsen worker 54 - jepsen.util 14094 :invoke :read [18 nil] INFO [2019-12-17 12:38:36,063] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77251,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:38:36,070] jepsen worker 38 - jepsen.util 14078 :fail :cas [12 [29 28]] :ValueNotSwapFrom INFO [2019-12-17 12:38:36,071] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestTimeoutException: CarrierBucketConfigRequest {"retried":0,"reason":"TIMEOUT","requestId":77043,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:38:36,076] jepsen worker 64 - jepsen.util 10504 :ok :write [21 1] INFO [2019-12-17 12:38:36,101] jepsen worker 35 - jepsen.util 18485 :info :write [11 49] :RequestCanceledException INFO [2019-12-17 12:38:36,105] jepsen worker 65 - jepsen.util 11675 :ok :write [21 23] INFO [2019-12-17 12:38:36,105] jepsen worker 11 - jepsen.util 13241 :ok :write [3 25] INFO [2019-12-17 12:38:36,119] jepsen worker 81 - jepsen.util 17721 :info :write [27 23] :RequestCanceledException INFO [2019-12-17 12:38:36,119] jepsen worker 9 - jepsen.util 13779 :info :cas [3 [20 24]] :RequestCanceledException INFO [2019-12-17 12:38:36,125] jepsen worker 55 - jepsen.util 14005 :ok :read [18 23] INFO [2019-12-17 12:38:36,126] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77251,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:38:36,126] jepsen worker 38 - jepsen.util 14078 :invoke :cas [12 [30 47]] INFO [2019-12-17 12:38:36,126] jepsen worker 64 - jepsen.util 10504 :invoke :write [21 35] INFO [2019-12-17 12:38:36,126] jepsen worker 65 - jepsen.util 11675 :invoke :cas [21 [2 34]] INFO [2019-12-17 12:38:36,126] jepsen worker 11 - jepsen.util 13241 :invoke :read [3 nil] INFO [2019-12-17 12:38:36,126] jepsen worker 55 - jepsen.util 14005 :invoke :cas [18 [15 11]] INFO [2019-12-17 12:38:36,127] jepsen worker 9 - jepsen.util 13869 :invoke :read [3 nil] INFO [2019-12-17 12:38:36,127] jepsen worker 35 - jepsen.util 18575 :invoke :write [11 27] INFO [2019-12-17 12:38:36,127] jepsen worker 81 - jepsen.util 17811 :invoke :read [27 nil] INFO [2019-12-17 12:38:36,128] jepsen worker 52 - jepsen.util 13012 :ok :write [17 16] INFO [2019-12-17 12:38:36,128] jepsen worker 52 - jepsen.util 13012 :invoke :cas [17 [27 47]] INFO [2019-12-17 12:38:36,134] jepsen worker 61 - jepsen.util 19591 :info :write [20 18] :RequestCanceledException INFO [2019-12-17 12:38:36,134] jepsen worker 71 - jepsen.util 14201 :info :cas [23 [38 13]] :RequestCanceledException INFO [2019-12-17 12:38:36,135] jepsen worker 71 - jepsen.util 14291 :invoke :cas [23 [12 30]] INFO [2019-12-17 12:38:36,135] jepsen worker 61 - jepsen.util 19681 :invoke :read [20 nil] INFO [2019-12-17 12:38:36,141] jepsen worker 72 - jepsen.util 13392 :ok :read [24 32] INFO [2019-12-17 12:38:36,141] jepsen worker 72 - jepsen.util 13392 :invoke :cas [24 [22 37]] INFO [2019-12-17 12:38:36,141] jepsen worker 27 - jepsen.util 12267 :fail :cas [9 [42 8]] :ValueNotSwapFrom INFO [2019-12-17 12:38:36,141] jepsen worker 27 - jepsen.util 12267 :invoke :cas [9 [3 1]] INFO [2019-12-17 12:38:36,174] jepsen worker 65 - jepsen.util 11675 :fail :cas [21 [2 34]] :ValueNotSwapFrom INFO [2019-12-17 12:38:36,174] jepsen worker 65 - jepsen.util 11675 :invoke :cas [21 [34 13]] INFO [2019-12-17 12:38:36,196] jepsen worker 9 - jepsen.util 13869 :ok :read [3 25] INFO [2019-12-17 12:38:36,196] jepsen worker 9 - jepsen.util 13869 :invoke :cas [3 [43 30]] INFO [2019-12-17 12:38:36,219] jepsen worker 26 - jepsen.util 19106 :info :cas [8 [49 14]] :RequestCanceledException INFO [2019-12-17 12:38:36,222] jepsen worker 26 - jepsen.util 19196 :invoke :cas [8 [17 25]] INFO [2019-12-17 12:38:36,232] jepsen worker 27 - jepsen.util 12267 :fail :cas [9 [3 1]] :ValueNotSwapFrom INFO [2019-12-17 12:38:36,232] jepsen worker 27 - jepsen.util 12267 :invoke :write [9 25] INFO [2019-12-17 12:38:36,235] jepsen worker 83 - jepsen.util 18173 :info :write [27 27] :RequestCanceledException INFO [2019-12-17 12:38:36,236] jepsen worker 83 - jepsen.util 18263 :invoke :read [27 nil] INFO [2019-12-17 12:38:36,240] jepsen worker 54 - jepsen.util 14094 :ok :read [18 23] INFO [2019-12-17 12:38:36,240] jepsen worker 54 - jepsen.util 14094 :invoke :read [18 nil] INFO [2019-12-17 12:38:36,256] jepsen worker 11 - jepsen.util 13241 :ok :read [3 25] INFO [2019-12-17 12:38:36,256] jepsen worker 11 - jepsen.util 13241 :invoke :cas [3 [22 38]] INFO [2019-12-17 12:38:36,265] jepsen worker 14 - jepsen.util 12974 :fail :cas [4 [39 37]] :ValueNotSwapFrom INFO [2019-12-17 12:38:36,265] jepsen worker 14 - jepsen.util 12974 :invoke :read [4 nil] INFO [2019-12-17 12:38:36,273] jepsen worker 37 - jepsen.util 12817 :fail :cas [12 [3 49]] :ValueNotSwapFrom INFO [2019-12-17 12:38:36,273] jepsen worker 37 - jepsen.util 12817 :invoke :read [12 nil] INFO [2019-12-17 12:38:36,285] jepsen worker 9 - jepsen.util 13869 :fail :cas [3 [43 30]] :ValueNotSwapFrom INFO [2019-12-17 12:38:36,286] jepsen worker 9 - jepsen.util 13869 :invoke :write [3 45] INFO [2019-12-17 12:38:36,299] jepsen worker 65 - jepsen.util 11675 :fail :cas [21 [34 13]] :ValueNotSwapFrom INFO [2019-12-17 12:38:36,299] jepsen worker 65 - jepsen.util 11675 :invoke :read [21 nil] INFO [2019-12-17 12:38:36,300] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77276,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:38:36,300] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77276,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:38:36,300] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77277,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:38:36,300] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77277,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:38:36,300] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77278,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:38:36,301] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77278,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:38:36,312] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77283,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:38:36,312] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77283,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:38:36,312] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77286,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:38:36,312] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77286,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:38:36,312] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77287,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:38:36,312] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77287,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:38:36,314] jepsen worker 51 - jepsen.util 12651 :ok :read [17 16] INFO [2019-12-17 12:38:36,314] jepsen worker 51 - jepsen.util 12651 :invoke :cas [17 [38 4]] INFO [2019-12-17 12:38:36,339] jepsen worker 72 - jepsen.util 13392 :fail :cas [24 [22 37]] :ValueNotSwapFrom INFO [2019-12-17 12:38:36,340] jepsen worker 72 - jepsen.util 13392 :invoke :write [24 4] INFO [2019-12-17 12:38:36,363] jepsen worker 65 - jepsen.util 11675 :ok :read [21 23] INFO [2019-12-17 12:38:36,363] jepsen worker 65 - jepsen.util 11675 :invoke :cas [21 [0 39]] INFO [2019-12-17 12:38:36,368] jepsen worker 38 - jepsen.util 14078 :fail :cas [12 [30 47]] :ValueNotSwapFrom INFO [2019-12-17 12:38:36,368] jepsen worker 38 - jepsen.util 14078 :invoke :cas [12 [37 1]] INFO [2019-12-17 12:38:36,374] jepsen worker 11 - jepsen.util 13241 :fail :cas [3 [22 38]] :ValueNotSwapFrom INFO [2019-12-17 12:38:36,374] jepsen worker 11 - jepsen.util 13241 :invoke :cas [3 [48 21]] INFO [2019-12-17 12:38:36,387] jepsen worker 13 - jepsen.util 12973 :ok :write [4 44] INFO [2019-12-17 12:38:36,387] jepsen worker 13 - jepsen.util 12973 :invoke :cas [4 [42 25]] INFO [2019-12-17 12:38:36,387] jepsen worker 52 - jepsen.util 13012 :fail :cas [17 [27 47]] :ValueNotSwapFrom INFO [2019-12-17 12:38:36,388] jepsen worker 52 - jepsen.util 13012 :invoke :cas [17 [15 4]] INFO [2019-12-17 12:38:36,411] jepsen worker 14 - jepsen.util 12974 :ok :read [4 44] INFO [2019-12-17 12:38:36,411] jepsen worker 14 - jepsen.util 12974 :invoke :read [4 nil] INFO [2019-12-17 12:38:36,414] jepsen worker 65 - jepsen.util 11675 :fail :cas [21 [0 39]] :ValueNotSwapFrom INFO [2019-12-17 12:38:36,415] jepsen worker 65 - jepsen.util 11675 :invoke :read [21 nil] INFO [2019-12-17 12:38:36,434] jepsen worker 65 - jepsen.util 11675 :ok :read [21 23] INFO [2019-12-17 12:38:36,434] jepsen worker 65 - jepsen.util 11675 :invoke :cas [21 [15 14]] INFO [2019-12-17 12:38:36,442] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77308,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:38:36,442] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77308,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:38:36,442] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77312,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:38:36,442] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77312,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:38:36,449] jepsen worker 64 - jepsen.util 10504 :ok :write [21 35] INFO [2019-12-17 12:38:36,450] jepsen worker 64 - jepsen.util 10504 :invoke :read [21 nil] INFO [2019-12-17 12:38:36,454] jepsen worker 65 - jepsen.util 11675 :fail :cas [21 [15 14]] :ValueNotSwapFrom INFO [2019-12-17 12:38:36,454] jepsen worker 65 - jepsen.util 11675 :invoke :cas [21 [39 44]] INFO [2019-12-17 12:38:36,474] jepsen worker 64 - jepsen.util 10504 :ok :read [21 35] INFO [2019-12-17 12:38:36,474] jepsen worker 64 - jepsen.util 10504 :invoke :read [21 nil] INFO [2019-12-17 12:38:36,483] jepsen worker 27 - jepsen.util 12267 :ok :write [9 25] INFO [2019-12-17 12:38:36,483] jepsen worker 27 - jepsen.util 12267 :invoke :cas [9 [11 31]] INFO [2019-12-17 12:38:36,485] jepsen worker 37 - jepsen.util 12817 :ok :read [12 47] INFO [2019-12-17 12:38:36,485] jepsen worker 37 - jepsen.util 12817 :invoke :write [12 45] INFO [2019-12-17 12:38:36,497] jepsen worker 65 - jepsen.util 11675 :fail :cas [21 [39 44]] :ValueNotSwapFrom INFO [2019-12-17 12:38:36,497] jepsen worker 65 - jepsen.util 11675 :invoke :cas [21 [42 23]] INFO [2019-12-17 12:38:36,532] jepsen worker 51 - jepsen.util 12651 :fail :cas [17 [38 4]] :ValueNotSwapFrom INFO [2019-12-17 12:38:36,532] jepsen worker 51 - jepsen.util 12651 :invoke :read [17 nil] INFO [2019-12-17 12:38:36,548] jepsen worker 64 - jepsen.util 10504 :ok :read [21 35] INFO [2019-12-17 12:38:36,549] jepsen worker 64 - jepsen.util 10504 :invoke :cas [21 [11 25]] INFO [2019-12-17 12:38:36,551] jepsen worker 27 - jepsen.util 12267 :fail :cas [9 [11 31]] :ValueNotSwapFrom INFO [2019-12-17 12:38:36,551] jepsen worker 27 - jepsen.util 12267 :invoke :write [9 17] INFO [2019-12-17 12:38:36,585] jepsen worker 65 - jepsen.util 11675 :fail :cas [21 [42 23]] :ValueNotSwapFrom INFO [2019-12-17 12:38:36,585] jepsen worker 65 - jepsen.util 11675 :invoke :cas [21 [4 49]] INFO [2019-12-17 12:38:36,597] jepsen worker 9 - jepsen.util 13869 :ok :write [3 45] INFO [2019-12-17 12:38:36,597] jepsen worker 9 - jepsen.util 13869 :invoke :write [3 22] INFO [2019-12-17 12:38:36,625] jepsen worker 38 - jepsen.util 14078 :fail :cas [12 [37 1]] :ValueNotSwapFrom INFO [2019-12-17 12:38:36,625] jepsen worker 38 - jepsen.util 14078 :invoke :read [12 nil] INFO [2019-12-17 12:38:36,632] jepsen worker 64 - jepsen.util 10504 :fail :cas [21 [11 25]] :ValueNotSwapFrom INFO [2019-12-17 12:38:36,632] jepsen worker 64 - jepsen.util 10504 :invoke :read [21 nil] INFO [2019-12-17 12:38:36,665] jepsen worker 64 - jepsen.util 10504 :ok :read [21 35] INFO [2019-12-17 12:38:36,665] jepsen worker 64 - jepsen.util 10504 :invoke :cas [21 [49 17]] INFO [2019-12-17 12:38:36,678] jepsen worker 52 - jepsen.util 13012 :fail :cas [17 [15 4]] :ValueNotSwapFrom INFO [2019-12-17 12:38:36,679] jepsen worker 52 - jepsen.util 13012 :invoke :read [17 nil] INFO [2019-12-17 12:38:36,692] jepsen worker 72 - jepsen.util 13392 :ok :write [24 4] INFO [2019-12-17 12:38:36,692] jepsen worker 72 - jepsen.util 13392 :invoke :write [24 41] INFO [2019-12-17 12:38:36,696] jepsen worker 65 - jepsen.util 11675 :fail :cas [21 [4 49]] :ValueNotSwapFrom INFO [2019-12-17 12:38:36,697] jepsen worker 65 - jepsen.util 11675 :invoke :read [21 nil] INFO [2019-12-17 12:38:36,712] jepsen worker 64 - jepsen.util 10504 :fail :cas [21 [49 17]] :ValueNotSwapFrom INFO [2019-12-17 12:38:36,712] jepsen worker 64 - jepsen.util 10504 :invoke :write [21 25] INFO [2019-12-17 12:38:36,722] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77326,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:38:36,722] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77326,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:38:36,722] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77331,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:38:36,722] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77331,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:38:36,745] jepsen worker 11 - jepsen.util 13241 :fail :cas [3 [48 21]] :ValueNotSwapFrom INFO [2019-12-17 12:38:36,745] jepsen worker 11 - jepsen.util 13241 :invoke :cas [3 [43 9]] INFO [2019-12-17 12:38:36,751] jepsen worker 80 - jepsen.util 12860 :ok :write [26 35] INFO [2019-12-17 12:38:36,751] jepsen worker 80 - jepsen.util 12860 :invoke :cas [26 [24 33]] INFO [2019-12-17 12:38:36,753] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:38:36,753] jepsen nemesis - couchbase.util Rebalance check count: 608 INFO [2019-12-17 12:38:36,753] jepsen nemesis - couchbase.util Rebalance stuck count: 589 INFO [2019-12-17 12:38:36,771] jepsen worker 27 - jepsen.util 12267 :ok :write [9 17] INFO [2019-12-17 12:38:36,771] jepsen worker 27 - jepsen.util 12267 :invoke :read [9 nil] INFO [2019-12-17 12:38:36,829] jepsen worker 27 - jepsen.util 12267 :ok :read [9 17] INFO [2019-12-17 12:38:36,829] jepsen worker 65 - jepsen.util 11675 :ok :read [21 35] INFO [2019-12-17 12:38:36,856] jepsen worker 64 - jepsen.util 10504 :ok :write [21 25] INFO [2019-12-17 12:38:36,873] jepsen worker 9 - jepsen.util 13869 :ok :write [3 22] INFO [2019-12-17 12:38:36,894] jepsen worker 14 - jepsen.util 12974 :ok :read [4 3] INFO [2019-12-17 12:38:36,900] jepsen worker 72 - jepsen.util 13392 :ok :write [24 41] INFO [2019-12-17 12:38:36,926] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77341,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:38:36,933] jepsen worker 27 - jepsen.util 12267 :invoke :write [9 24] INFO [2019-12-17 12:38:36,933] jepsen worker 65 - jepsen.util 11675 :invoke :write [21 25] INFO [2019-12-17 12:38:36,933] jepsen worker 9 - jepsen.util 13869 :invoke :write [3 5] INFO [2019-12-17 12:38:36,933] jepsen worker 14 - jepsen.util 12974 :invoke :read [4 nil] INFO [2019-12-17 12:38:36,933] jepsen worker 64 - jepsen.util 10504 :invoke :read [21 nil] INFO [2019-12-17 12:38:36,934] jepsen worker 72 - jepsen.util 13392 :invoke :cas [24 [15 38]] INFO [2019-12-17 12:38:36,934] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77341,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:38:36,934] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77344,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:38:36,934] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77344,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:38:36,936] jepsen worker 11 - jepsen.util 13241 :fail :cas [3 [43 9]] :ValueNotSwapFrom INFO [2019-12-17 12:38:36,937] jepsen worker 11 - jepsen.util 13241 :invoke :read [3 nil] INFO [2019-12-17 12:38:36,953] jepsen worker 55 - jepsen.util 14005 :fail :cas [18 [15 11]] :ValueNotSwapFrom INFO [2019-12-17 12:38:36,953] jepsen worker 55 - jepsen.util 14005 :invoke :write [18 19] INFO [2019-12-17 12:38:36,978] jepsen worker 51 - jepsen.util 12651 :ok :read [17 16] INFO [2019-12-17 12:38:36,978] jepsen worker 51 - jepsen.util 12651 :invoke :write [17 42] INFO [2019-12-17 12:38:37,022] jepsen worker 71 - jepsen.util 14291 :fail :cas [23 [12 30]] :ValueNotSwapFrom INFO [2019-12-17 12:38:37,022] jepsen worker 71 - jepsen.util 14291 :invoke :read [23 nil] INFO [2019-12-17 12:38:37,037] jepsen worker 64 - jepsen.util 10504 :ok :read [21 25] INFO [2019-12-17 12:38:37,037] jepsen worker 64 - jepsen.util 10504 :invoke :cas [21 [8 48]] INFO [2019-12-17 12:38:37,040] jepsen worker 72 - jepsen.util 13392 :fail :cas [24 [15 38]] :ValueNotSwapFrom INFO [2019-12-17 12:38:37,040] jepsen worker 72 - jepsen.util 13392 :invoke :read [24 nil] INFO [2019-12-17 12:38:37,059] jepsen worker 11 - jepsen.util 13241 :ok :read [3 22] INFO [2019-12-17 12:38:37,060] jepsen worker 11 - jepsen.util 13241 :invoke :cas [3 [40 2]] INFO [2019-12-17 12:38:37,083] jepsen worker 38 - jepsen.util 14078 :ok :read [12 47] INFO [2019-12-17 12:38:37,083] jepsen worker 38 - jepsen.util 14078 :invoke :read [12 nil] INFO [2019-12-17 12:38:37,085] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77356,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:38:37,085] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77356,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:38:37,085] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77359,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:38:37,085] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77359,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:38:37,085] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77360,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:38:37,085] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77360,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:38:37,088] jepsen worker 64 - jepsen.util 10504 :fail :cas [21 [8 48]] :ValueNotSwapFrom INFO [2019-12-17 12:38:37,089] jepsen worker 64 - jepsen.util 10504 :invoke :cas [21 [49 18]] INFO [2019-12-17 12:38:37,093] jepsen worker 72 - jepsen.util 13392 :ok :read [24 41] INFO [2019-12-17 12:38:37,093] jepsen worker 72 - jepsen.util 13392 :invoke :write [24 32] INFO [2019-12-17 12:38:37,121] jepsen worker 11 - jepsen.util 13241 :fail :cas [3 [40 2]] :ValueNotSwapFrom INFO [2019-12-17 12:38:37,122] jepsen worker 11 - jepsen.util 13241 :invoke :write [3 15] INFO [2019-12-17 12:38:37,130] jepsen worker 64 - jepsen.util 10504 :fail :cas [21 [49 18]] :ValueNotSwapFrom INFO [2019-12-17 12:38:37,130] jepsen worker 64 - jepsen.util 10504 :invoke :read [21 nil] INFO [2019-12-17 12:38:37,169] jepsen worker 64 - jepsen.util 10504 :ok :read [21 25] INFO [2019-12-17 12:38:37,169] jepsen worker 64 - jepsen.util 10504 :invoke :write [21 14] INFO [2019-12-17 12:38:37,173] jepsen worker 79 - jepsen.util 13579 :fail :cas [26 [27 14]] :ValueNotSwapFrom INFO [2019-12-17 12:38:37,174] jepsen worker 79 - jepsen.util 13579 :invoke :cas [26 [39 7]] INFO [2019-12-17 12:38:37,194] jepsen worker 52 - jepsen.util 13012 :ok :read [17 16] INFO [2019-12-17 12:38:37,219] jepsen worker 65 - jepsen.util 11675 :ok :write [21 25] INFO [2019-12-17 12:38:37,219] jepsen worker 9 - jepsen.util 13869 :ok :write [3 5] INFO [2019-12-17 12:38:37,243] jepsen worker 79 - jepsen.util 13579 :fail :cas [26 [39 7]] :ValueNotSwapFrom INFO [2019-12-17 12:38:37,244] jepsen worker 27 - jepsen.util 12267 :ok :write [9 24] INFO [2019-12-17 12:38:37,259] jepsen worker 54 - jepsen.util 14094 :ok :read [18 23] INFO [2019-12-17 12:38:37,266] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77373,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:38:37,274] jepsen worker 70 - jepsen.util 13840 :fail :cas [23 [24 29]] :ValueNotSwapFrom INFO [2019-12-17 12:38:37,284] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77381,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:38:37,285] jepsen worker 52 - jepsen.util 13012 :invoke :read [17 nil] INFO [2019-12-17 12:38:37,285] jepsen worker 65 - jepsen.util 11675 :invoke :read [21 nil] INFO [2019-12-17 12:38:37,285] jepsen worker 79 - jepsen.util 13579 :invoke :read [26 nil] INFO [2019-12-17 12:38:37,285] jepsen worker 27 - jepsen.util 12267 :invoke :read [9 nil] INFO [2019-12-17 12:38:37,285] jepsen worker 9 - jepsen.util 13869 :invoke :write [3 49] INFO [2019-12-17 12:38:37,285] jepsen worker 54 - jepsen.util 14094 :invoke :read [18 nil] INFO [2019-12-17 12:38:37,285] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77373,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:38:37,286] jepsen worker 70 - jepsen.util 13840 :invoke :read [23 nil] INFO [2019-12-17 12:38:37,286] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77381,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:38:37,287] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77383,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:38:37,287] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77383,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:38:37,307] jepsen worker 69 - jepsen.util 13749 :fail :cas [23 [31 26]] :ValueNotSwapFrom INFO [2019-12-17 12:38:37,307] jepsen worker 69 - jepsen.util 13749 :invoke :cas [23 [22 0]] INFO [2019-12-17 12:38:37,327] jepsen worker 56 - jepsen.util 14186 :fail :cas [18 [24 48]] :ValueNotSwapFrom INFO [2019-12-17 12:38:37,327] jepsen worker 56 - jepsen.util 14186 :invoke :cas [18 [48 30]] INFO [2019-12-17 12:38:37,331] jepsen worker 72 - jepsen.util 13392 :ok :write [24 32] INFO [2019-12-17 12:38:37,331] jepsen worker 72 - jepsen.util 13392 :invoke :cas [24 [31 46]] INFO [2019-12-17 12:38:37,337] jepsen worker 65 - jepsen.util 11675 :ok :read [21 25] INFO [2019-12-17 12:38:37,338] jepsen worker 65 - jepsen.util 11675 :invoke :write [21 40] INFO [2019-12-17 12:38:37,346] jepsen worker 27 - jepsen.util 12267 :ok :read [9 24] INFO [2019-12-17 12:38:37,347] jepsen worker 27 - jepsen.util 12267 :invoke :write [9 13] INFO [2019-12-17 12:38:37,391] jepsen worker 53 - jepsen.util 13103 :fail :cas [17 [2 19]] :ValueNotSwapFrom INFO [2019-12-17 12:38:37,391] jepsen worker 53 - jepsen.util 13103 :invoke :write [17 27] INFO [2019-12-17 12:38:37,479] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77400,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:38:37,479] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77400,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:38:37,487] jepsen worker 80 - jepsen.util 12860 :fail :cas [26 [24 33]] :ValueNotSwapFrom INFO [2019-12-17 12:38:37,487] jepsen worker 12 - jepsen.util 13512 :fail :cas [4 [11 33]] :ValueNotSwapFrom INFO [2019-12-17 12:38:37,488] jepsen worker 80 - jepsen.util 12860 :invoke :read [26 nil] INFO [2019-12-17 12:38:37,488] jepsen worker 12 - jepsen.util 13512 :invoke :read [4 nil] INFO [2019-12-17 12:38:37,492] jepsen worker 79 - jepsen.util 13579 :ok :read [26 35] INFO [2019-12-17 12:38:37,492] jepsen worker 79 - jepsen.util 13579 :invoke :cas [26 [46 21]] INFO [2019-12-17 12:38:37,494] jepsen worker 13 - jepsen.util 12973 :fail :cas [4 [42 25]] :ValueNotSwapFrom INFO [2019-12-17 12:38:37,495] jepsen worker 13 - jepsen.util 12973 :invoke :cas [4 [21 22]] INFO [2019-12-17 12:38:37,495] jepsen worker 70 - jepsen.util 13840 :ok :read [23 18] INFO [2019-12-17 12:38:37,496] jepsen worker 70 - jepsen.util 13840 :invoke :cas [23 [48 30]] INFO [2019-12-17 12:38:37,506] jepsen worker 14 - jepsen.util 12974 :ok :read [4 3] INFO [2019-12-17 12:38:37,506] jepsen worker 14 - jepsen.util 12974 :invoke :write [4 11] INFO [2019-12-17 12:38:37,528] jepsen worker 27 - jepsen.util 12267 :ok :write [9 13] INFO [2019-12-17 12:38:37,528] jepsen worker 27 - jepsen.util 12267 :invoke :cas [9 [0 31]] INFO [2019-12-17 12:38:37,542] jepsen worker 65 - jepsen.util 11675 :ok :write [21 40] INFO [2019-12-17 12:38:37,543] jepsen worker 65 - jepsen.util 11675 :invoke :write [21 22] INFO [2019-12-17 12:38:37,551] jepsen worker 27 - jepsen.util 12267 :fail :cas [9 [0 31]] :ValueNotSwapFrom INFO [2019-12-17 12:38:37,551] jepsen worker 27 - jepsen.util 12267 :invoke :write [9 5] INFO [2019-12-17 12:38:37,563] jepsen worker 87 - jepsen.util 13047 :fail :cas [29 [49 22]] :ValueNotSwapFrom INFO [2019-12-17 12:38:37,563] jepsen worker 87 - jepsen.util 13047 :invoke :read [29 nil] INFO [2019-12-17 12:38:37,567] jepsen worker 69 - jepsen.util 13749 :fail :cas [23 [22 0]] :ValueNotSwapFrom INFO [2019-12-17 12:38:37,567] jepsen worker 69 - jepsen.util 13749 :invoke :write [23 20] INFO [2019-12-17 12:38:37,594] jepsen worker 38 - jepsen.util 14078 :ok :read [12 47] INFO [2019-12-17 12:38:37,595] jepsen worker 38 - jepsen.util 14078 :invoke :cas [12 [32 36]] INFO [2019-12-17 12:38:37,618] jepsen worker 65 - jepsen.util 11675 :ok :write [21 22] INFO [2019-12-17 12:38:37,618] jepsen worker 65 - jepsen.util 11675 :invoke :read [21 nil] INFO [2019-12-17 12:38:37,640] jepsen worker 65 - jepsen.util 11675 :ok :read [21 22] INFO [2019-12-17 12:38:37,640] jepsen worker 65 - jepsen.util 11675 :invoke :cas [21 [30 25]] INFO [2019-12-17 12:38:37,656] jepsen worker 65 - jepsen.util 11675 :fail :cas [21 [30 25]] :ValueNotSwapFrom INFO [2019-12-17 12:38:37,656] jepsen worker 65 - jepsen.util 11675 :invoke :read [21 nil] INFO [2019-12-17 12:38:37,660] jepsen worker 27 - jepsen.util 12267 :ok :write [9 5] INFO [2019-12-17 12:38:37,660] jepsen worker 27 - jepsen.util 12267 :invoke :cas [9 [30 8]] INFO [2019-12-17 12:38:37,666] jepsen worker 2 - jepsen.util 17552 :info :write [0 1] :RequestCanceledException INFO [2019-12-17 12:38:37,669] jepsen worker 4 - jepsen.util 12604 :fail :cas [1 [27 46]] :ValueNotSwapFrom INFO [2019-12-17 12:38:37,669] jepsen worker 4 - jepsen.util 12604 :invoke :cas [1 [46 21]] INFO [2019-12-17 12:38:37,669] jepsen worker 2 - jepsen.util 17642 :invoke :write [0 2] INFO [2019-12-17 12:38:37,679] jepsen worker 52 - jepsen.util 13012 :ok :read [17 16] INFO [2019-12-17 12:38:37,679] jepsen worker 52 - jepsen.util 13012 :invoke :read [17 nil] INFO [2019-12-17 12:38:37,688] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77417,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:38:37,689] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77417,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:38:37,689] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77418,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:38:37,689] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77418,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:38:37,689] jepsen worker 27 - jepsen.util 12267 :fail :cas [9 [30 8]] :ValueNotSwapFrom INFO [2019-12-17 12:38:37,689] jepsen worker 27 - jepsen.util 12267 :invoke :write [9 1] INFO [2019-12-17 12:38:37,690] jepsen worker 65 - jepsen.util 11675 :ok :read [21 22] INFO [2019-12-17 12:38:37,690] jepsen worker 65 - jepsen.util 11675 :invoke :read [21 nil] INFO [2019-12-17 12:38:37,692] jepsen worker 16 - jepsen.util 19906 :info :write [5 26] :RequestCanceledException INFO [2019-12-17 12:38:37,692] jepsen worker 7 - jepsen.util 18097 :info :write [2 40] :RequestCanceledException INFO [2019-12-17 12:38:37,692] jepsen worker 40 - jepsen.util 18850 :info :write [13 13] :RequestCanceledException INFO [2019-12-17 12:38:37,692] jepsen worker 24 - jepsen.util 17394 :info :write [8 47] :RequestCanceledException INFO [2019-12-17 12:38:37,693] jepsen worker 40 - jepsen.util 18940 :invoke :read [13 nil] INFO [2019-12-17 12:38:37,693] jepsen worker 16 - jepsen.util 19996 :invoke :write [5 2] INFO [2019-12-17 12:38:37,693] jepsen worker 24 - jepsen.util 17484 :invoke :write [8 1] INFO [2019-12-17 12:38:37,693] jepsen worker 7 - jepsen.util 18187 :invoke :write [2 2] INFO [2019-12-17 12:38:37,706] jepsen worker 11 - jepsen.util 13241 :ok :write [3 15] INFO [2019-12-17 12:38:37,706] jepsen worker 11 - jepsen.util 13241 :invoke :write [3 0] INFO [2019-12-17 12:38:37,709] jepsen worker 79 - jepsen.util 13579 :fail :cas [26 [46 21]] :ValueNotSwapFrom INFO [2019-12-17 12:38:37,710] jepsen worker 79 - jepsen.util 13579 :invoke :write [26 27] INFO [2019-12-17 12:38:37,711] jepsen worker 65 - jepsen.util 11675 :ok :read [21 22] INFO [2019-12-17 12:38:37,711] jepsen worker 65 - jepsen.util 11675 :invoke :write [21 7] INFO [2019-12-17 12:38:37,714] jepsen worker 76 - jepsen.util 18526 :info :cas [25 [46 22]] :RequestCanceledException INFO [2019-12-17 12:38:37,715] jepsen worker 76 - jepsen.util 18616 :invoke :cas [25 [45 24]] INFO [2019-12-17 12:38:37,718] jepsen worker 56 - jepsen.util 14186 :fail :cas [18 [48 30]] :ValueNotSwapFrom INFO [2019-12-17 12:38:37,718] jepsen worker 56 - jepsen.util 14186 :invoke :cas [18 [41 7]] INFO [2019-12-17 12:38:37,735] jepsen worker 74 - jepsen.util 13124 :info :cas [24 [43 6]] :RequestCanceledException INFO [2019-12-17 12:38:37,735] jepsen worker 5 - jepsen.util 13055 :info :write [1 37] :RequestCanceledException INFO [2019-12-17 12:38:37,735] jepsen worker 88 - jepsen.util 13228 :info :write [29 39] :RequestCanceledException INFO [2019-12-17 12:38:37,735] jepsen worker 73 - jepsen.util 12313 :fail :read [24 nil] :RequestTimeoutException INFO [2019-12-17 12:38:37,735] jepsen worker 73 - jepsen.util 12313 :invoke :cas [24 [40 30]] INFO [2019-12-17 12:38:37,735] jepsen worker 48 - jepsen.util 16968 :info :write [16 40] :RequestCanceledException INFO [2019-12-17 12:38:37,735] jepsen worker 22 - jepsen.util 18652 :info :write [7 25] :RequestCanceledException INFO [2019-12-17 12:38:37,735] jepsen worker 57 - jepsen.util 17787 :info :write [19 1] :RequestCanceledException INFO [2019-12-17 12:38:37,735] jepsen worker 67 - jepsen.util 18607 :info :cas [22 [34 34]] :RequestCanceledException INFO [2019-12-17 12:38:37,735] jepsen worker 33 - jepsen.util 18123 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:38:37,735] jepsen worker 77 - jepsen.util 19427 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:38:37,736] jepsen worker 33 - jepsen.util 18123 :invoke :read [11 nil] INFO [2019-12-17 12:38:37,736] jepsen worker 77 - jepsen.util 19427 :invoke :write [25 10] INFO [2019-12-17 12:38:37,736] jepsen worker 74 - jepsen.util 13214 :invoke :write [24 14] INFO [2019-12-17 12:38:37,736] jepsen worker 5 - jepsen.util 13145 :invoke :read [1 nil] INFO [2019-12-17 12:38:37,736] jepsen worker 88 - jepsen.util 13318 :invoke :write [29 34] INFO [2019-12-17 12:38:37,736] jepsen worker 48 - jepsen.util 17058 :invoke :read [16 nil] INFO [2019-12-17 12:38:37,736] jepsen worker 57 - jepsen.util 17877 :invoke :cas [19 [4 19]] INFO [2019-12-17 12:38:37,736] jepsen worker 22 - jepsen.util 18742 :invoke :write [7 40] INFO [2019-12-17 12:38:37,736] jepsen worker 67 - jepsen.util 18697 :invoke :write [22 42] INFO [2019-12-17 12:38:37,743] jepsen worker 72 - jepsen.util 13392 :fail :cas [24 [31 46]] :ValueNotSwapFrom INFO [2019-12-17 12:38:37,743] jepsen worker 72 - jepsen.util 13392 :invoke :cas [24 [45 12]] INFO [2019-12-17 12:38:37,749] jepsen worker 49 - jepsen.util 18319 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:38:37,749] jepsen worker 19 - jepsen.util 14149 :info :cas [6 [48 21]] :RequestCanceledException INFO [2019-12-17 12:38:37,749] jepsen worker 10 - jepsen.util 13150 :info :write [3 32] :RequestCanceledException INFO [2019-12-17 12:38:37,749] jepsen worker 82 - jepsen.util 18352 :info :write [27 42] :RequestCanceledException INFO [2019-12-17 12:38:37,749] jepsen worker 89 - jepsen.util 13409 :info :write [29 27] :RequestCanceledException INFO [2019-12-17 12:38:37,749] jepsen worker 31 - jepsen.util 11461 :info :cas [10 [46 34]] :RequestCanceledException INFO [2019-12-17 12:38:37,749] jepsen worker 45 - jepsen.util 12825 :info :write [15 32] :RequestCanceledException INFO [2019-12-17 12:38:37,749] jepsen worker 58 - jepsen.util 18418 :info :cas [19 [6 36]] :RequestCanceledException INFO [2019-12-17 12:38:37,749] jepsen worker 49 - jepsen.util 18319 :invoke :write [16 48] INFO [2019-12-17 12:38:37,749] jepsen worker 59 - jepsen.util 17339 :info :cas [19 [22 31]] :RequestCanceledException INFO [2019-12-17 12:38:37,749] jepsen worker 60 - jepsen.util 19320 :info :write [20 1] :RequestCanceledException INFO [2019-12-17 12:38:37,750] jepsen worker 19 - jepsen.util 14239 :invoke :read [6 nil] INFO [2019-12-17 12:38:37,750] jepsen worker 10 - jepsen.util 13240 :invoke :write [3 40] INFO [2019-12-17 12:38:37,750] jepsen worker 89 - jepsen.util 13499 :invoke :cas [29 [20 9]] INFO [2019-12-17 12:38:37,750] jepsen worker 82 - jepsen.util 18442 :invoke :read [27 nil] INFO [2019-12-17 12:38:37,750] jepsen worker 45 - jepsen.util 12915 :invoke :write [15 35] INFO [2019-12-17 12:38:37,750] jepsen worker 58 - jepsen.util 18508 :invoke :write [19 11] INFO [2019-12-17 12:38:37,750] jepsen worker 31 - jepsen.util 11551 :invoke :write [10 42] INFO [2019-12-17 12:38:37,750] jepsen worker 59 - jepsen.util 17429 :invoke :write [19 6] INFO [2019-12-17 12:38:37,750] jepsen worker 60 - jepsen.util 19410 :invoke :write [20 33] INFO [2019-12-17 12:38:37,765] jepsen worker 17 - jepsen.util 18107 :info :write [5 6] :RequestCanceledException INFO [2019-12-17 12:38:37,765] jepsen worker 32 - jepsen.util 13532 :info :write [10 23] :RequestCanceledException INFO [2019-12-17 12:38:37,765] jepsen worker 66 - jepsen.util 18516 :info :cas [22 [14 14]] :RequestCanceledException INFO [2019-12-17 12:38:37,765] jepsen worker 47 - jepsen.util 14537 :info :write [15 11] :RequestCanceledException INFO [2019-12-17 12:38:37,765] jepsen worker 78 - jepsen.util 13218 :info :cas [26 [10 11]] :RequestCanceledException INFO [2019-12-17 12:38:37,765] jepsen worker 85 - jepsen.util 18715 :info :write [28 24] :RequestCanceledException INFO [2019-12-17 12:38:37,766] jepsen worker 17 - jepsen.util 18197 :invoke :read [5 nil] INFO [2019-12-17 12:38:37,766] jepsen worker 78 - jepsen.util 13308 :invoke :write [26 23] INFO [2019-12-17 12:38:37,766] jepsen worker 32 - jepsen.util 13622 :invoke :write [10 35] INFO [2019-12-17 12:38:37,767] jepsen worker 63 - jepsen.util 11673 :info :write [21 42] :RequestCanceledException INFO [2019-12-17 12:38:37,767] jepsen worker 86 - jepsen.util 19076 :info :write [28 45] :RequestCanceledException INFO [2019-12-17 12:38:37,767] jepsen worker 43 - jepsen.util 18493 :info :write [14 15] :RequestCanceledException INFO [2019-12-17 12:38:37,767] jepsen worker 18 - jepsen.util 13968 :info :write [6 3] :RequestCanceledException INFO [2019-12-17 12:38:37,767] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:38:37,767] jepsen worker 46 - jepsen.util 13006 :fail :read [15 nil] :RequestTimeoutException INFO [2019-12-17 12:38:37,767] jepsen worker 28 - jepsen.util 11368 :info :write [9 23] :RequestCanceledException INFO [2019-12-17 12:38:37,767] jepsen worker 39 - jepsen.util 17769 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:38:37,767] jepsen nemesis - couchbase.util Rebalance check count: 609 INFO [2019-12-17 12:38:37,767] jepsen worker 46 - jepsen.util 13006 :invoke :cas [15 [17 42]] INFO [2019-12-17 12:38:37,767] jepsen worker 39 - jepsen.util 17769 :invoke :cas [13 [20 40]] INFO [2019-12-17 12:38:37,767] jepsen nemesis - couchbase.util Rebalance stuck count: 590 INFO [2019-12-17 12:38:37,768] jepsen worker 85 - jepsen.util 18805 :invoke :cas [28 [36 15]] INFO [2019-12-17 12:38:37,768] jepsen worker 66 - jepsen.util 18606 :invoke :write [22 45] INFO [2019-12-17 12:38:37,768] jepsen worker 63 - jepsen.util 11763 :invoke :cas [21 [16 4]] INFO [2019-12-17 12:38:37,768] jepsen worker 86 - jepsen.util 19166 :invoke :read [28 nil] INFO [2019-12-17 12:38:37,768] jepsen worker 43 - jepsen.util 18583 :invoke :read [14 nil] INFO [2019-12-17 12:38:37,768] jepsen worker 28 - jepsen.util 11458 :invoke :cas [9 [16 16]] INFO [2019-12-17 12:38:37,769] jepsen worker 18 - jepsen.util 14058 :invoke :cas [6 [13 21]] INFO [2019-12-17 12:38:37,770] jepsen worker 47 - jepsen.util 14627 :invoke :cas [15 [26 22]] INFO [2019-12-17 12:38:37,784] jepsen worker 71 - jepsen.util 14291 :ok :read [23 18] INFO [2019-12-17 12:38:37,784] jepsen worker 71 - jepsen.util 14291 :invoke :read [23 nil] INFO [2019-12-17 12:38:37,794] jepsen worker 46 - jepsen.util 13006 :fail :cas [15 [17 42]] :ValueNotSwapFrom INFO [2019-12-17 12:38:37,794] jepsen worker 46 - jepsen.util 13006 :invoke :read [15 nil] INFO [2019-12-17 12:38:37,802] jepsen worker 68 - jepsen.util 17438 :info :cas [22 [13 4]] :RequestCanceledException INFO [2019-12-17 12:38:37,802] jepsen worker 62 - jepsen.util 18962 :info :write [20 16] :RequestCanceledException INFO [2019-12-17 12:38:37,802] jepsen worker 0 - jepsen.util 18720 :info :cas [0 [21 40]] :RequestCanceledException INFO [2019-12-17 12:38:37,802] jepsen worker 23 - jepsen.util 18473 :info :write [7 2] :RequestCanceledException INFO [2019-12-17 12:38:37,803] jepsen worker 62 - jepsen.util 19052 :invoke :read [20 nil] INFO [2019-12-17 12:38:37,803] jepsen worker 23 - jepsen.util 18563 :invoke :cas [7 [41 49]] INFO [2019-12-17 12:38:37,805] jepsen worker 84 - jepsen.util 17634 :info :cas [28 [40 16]] :RequestCanceledException INFO [2019-12-17 12:38:37,805] jepsen worker 0 - jepsen.util 18810 :invoke :write [0 36] INFO [2019-12-17 12:38:37,806] jepsen worker 84 - jepsen.util 17724 :invoke :cas [28 [4 14]] INFO [2019-12-17 12:38:37,806] jepsen worker 27 - jepsen.util 12267 :ok :write [9 1] INFO [2019-12-17 12:38:37,806] jepsen worker 27 - jepsen.util 12267 :invoke :read [9 nil] INFO [2019-12-17 12:38:37,808] jepsen worker 68 - jepsen.util 17528 :invoke :read [22 nil] INFO [2019-12-17 12:38:37,818] jepsen worker 21 - jepsen.util 17841 :info :write [7 36] :RequestCanceledException INFO [2019-12-17 12:38:37,818] jepsen worker 75 - jepsen.util 18075 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:38:37,818] jepsen worker 36 - jepsen.util 13986 :info :write [12 23] :RequestCanceledException INFO [2019-12-17 12:38:37,818] jepsen worker 50 - jepsen.util 18140 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:38:37,818] jepsen worker 75 - jepsen.util 18075 :invoke :cas [25 [26 48]] INFO [2019-12-17 12:38:37,818] jepsen worker 50 - jepsen.util 18140 :invoke :read [16 nil] INFO [2019-12-17 12:38:37,819] jepsen worker 36 - jepsen.util 14076 :invoke :cas [12 [26 15]] INFO [2019-12-17 12:38:37,820] jepsen worker 21 - jepsen.util 17931 :invoke :write [7 0] INFO [2019-12-17 12:38:37,828] jepsen worker 54 - jepsen.util 14094 :ok :read [18 23] INFO [2019-12-17 12:38:37,828] jepsen worker 54 - jepsen.util 14094 :invoke :read [18 nil] INFO [2019-12-17 12:38:37,832] jepsen worker 41 - jepsen.util 19571 :info :write [13 42] :RequestCanceledException INFO [2019-12-17 12:38:37,832] jepsen worker 15 - jepsen.util 17295 :info :write [5 27] :RequestCanceledException INFO [2019-12-17 12:38:37,832] jepsen worker 8 - jepsen.util 17918 :info :write [2 19] :RequestCanceledException INFO [2019-12-17 12:38:37,832] jepsen worker 3 - jepsen.util 12963 :info :write [1 36] :RequestCanceledException INFO [2019-12-17 12:38:37,832] jepsen worker 42 - jepsen.util 19122 :info :write [14 7] :RequestCanceledException INFO [2019-12-17 12:38:37,833] jepsen worker 25 - jepsen.util 19465 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:38:37,833] jepsen worker 29 - jepsen.util 11549 :info :write [9 26] :RequestCanceledException INFO [2019-12-17 12:38:37,833] jepsen worker 25 - jepsen.util 19465 :invoke :cas [8 [40 32]] INFO [2019-12-17 12:38:37,833] jepsen worker 15 - jepsen.util 17385 :invoke :read [5 nil] INFO [2019-12-17 12:38:37,833] jepsen worker 8 - jepsen.util 18008 :invoke :read [2 nil] INFO [2019-12-17 12:38:37,833] jepsen worker 3 - jepsen.util 13053 :invoke :write [1 12] INFO [2019-12-17 12:38:37,834] jepsen worker 42 - jepsen.util 19212 :invoke :cas [14 [41 4]] INFO [2019-12-17 12:38:37,834] jepsen worker 41 - jepsen.util 19661 :invoke :read [13 nil] INFO [2019-12-17 12:38:37,834] jepsen worker 29 - jepsen.util 11639 :invoke :write [9 43] INFO [2019-12-17 12:38:37,847] jepsen worker 13 - jepsen.util 12973 :fail :cas [4 [21 22]] :ValueNotSwapFrom INFO [2019-12-17 12:38:37,847] jepsen worker 13 - jepsen.util 12973 :invoke :write [4 21] INFO [2019-12-17 12:38:37,849] jepsen worker 44 - jepsen.util 19034 :info :cas [14 [1 26]] :RequestCanceledException INFO [2019-12-17 12:38:37,850] jepsen worker 44 - jepsen.util 19124 :invoke :write [14 49] INFO [2019-12-17 12:38:37,862] jepsen worker 65 - jepsen.util 11675 :ok :write [21 7] INFO [2019-12-17 12:38:37,862] jepsen worker 65 - jepsen.util 11675 :invoke :cas [21 [23 44]] INFO [2019-12-17 12:38:37,865] jepsen worker 73 - jepsen.util 12313 :fail :cas [24 [40 30]] :ValueNotSwapFrom INFO [2019-12-17 12:38:37,865] jepsen worker 73 - jepsen.util 12313 :invoke :read [24 nil] INFO [2019-12-17 12:38:37,902] jepsen worker 1 - jepsen.util 18811 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:38:37,902] jepsen worker 1 - jepsen.util 18811 :invoke :read [0 nil] INFO [2019-12-17 12:38:37,904] jepsen worker 30 - jepsen.util 12810 :info :write [10 37] :RequestCanceledException INFO [2019-12-17 12:38:37,905] jepsen worker 30 - jepsen.util 12900 :invoke :read [10 nil] INFO [2019-12-17 12:38:37,908] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77498,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:38:37,908] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77498,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:38:37,908] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77502,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:38:37,908] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77502,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:38:37,922] jepsen worker 63 - jepsen.util 11763 :fail :cas [21 [16 4]] :ValueNotSwapFrom INFO [2019-12-17 12:38:37,922] jepsen worker 63 - jepsen.util 11763 :invoke :write [21 35] INFO [2019-12-17 12:38:37,923] jepsen worker 65 - jepsen.util 11675 :fail :cas [21 [23 44]] :ValueNotSwapFrom INFO [2019-12-17 12:38:37,923] jepsen worker 65 - jepsen.util 11675 :invoke :cas [21 [21 36]] INFO [2019-12-17 12:38:37,923] jepsen worker 38 - jepsen.util 14078 :fail :cas [12 [32 36]] :ValueNotSwapFrom INFO [2019-12-17 12:38:37,924] jepsen worker 38 - jepsen.util 14078 :invoke :cas [12 [6 34]] INFO [2019-12-17 12:38:37,925] jepsen worker 12 - jepsen.util 13512 :ok :read [4 3] INFO [2019-12-17 12:38:37,926] jepsen worker 12 - jepsen.util 13512 :invoke :cas [4 [34 23]] INFO [2019-12-17 12:38:37,941] jepsen worker 27 - jepsen.util 12267 :ok :read [9 1] INFO [2019-12-17 12:38:37,941] jepsen worker 27 - jepsen.util 12267 :invoke :write [9 47] INFO [2019-12-17 12:38:37,949] jepsen worker 20 - jepsen.util 13070 :info :cas [6 [14 4]] :RequestCanceledException INFO [2019-12-17 12:38:37,950] jepsen worker 20 - jepsen.util 13160 :invoke :write [6 46] INFO [2019-12-17 12:38:37,978] jepsen worker 87 - jepsen.util 13047 :ok :read [29 18] INFO [2019-12-17 12:38:37,979] jepsen worker 87 - jepsen.util 13047 :invoke :write [29 18] INFO [2019-12-17 12:38:37,980] jepsen worker 65 - jepsen.util 11675 :fail :cas [21 [21 36]] :ValueNotSwapFrom INFO [2019-12-17 12:38:37,980] jepsen worker 65 - jepsen.util 11675 :invoke :read [21 nil] INFO [2019-12-17 12:38:37,998] jepsen worker 65 - jepsen.util 11675 :ok :read [21 7] INFO [2019-12-17 12:38:37,998] jepsen worker 65 - jepsen.util 11675 :invoke :read [21 nil] INFO [2019-12-17 12:38:38,000] jepsen worker 29 - jepsen.util 11639 :ok :write [9 43] INFO [2019-12-17 12:38:38,001] jepsen worker 29 - jepsen.util 11639 :invoke :read [9 nil] INFO [2019-12-17 12:38:38,018] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77518,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:38:38,021] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77518,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:38:38,021] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77520,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:38:38,021] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77520,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:38:38,032] jepsen worker 65 - jepsen.util 11675 :ok :read [21 7] INFO [2019-12-17 12:38:38,033] jepsen worker 65 - jepsen.util 11675 :invoke :cas [21 [34 47]] INFO [2019-12-17 12:38:38,035] jepsen worker 6 - jepsen.util 18456 :info :cas [2 [30 6]] :RequestCanceledException INFO [2019-12-17 12:38:38,036] jepsen worker 6 - jepsen.util 18546 :invoke :read [2 nil] INFO [2019-12-17 12:38:38,060] jepsen worker 65 - jepsen.util 11675 :fail :cas [21 [34 47]] :ValueNotSwapFrom INFO [2019-12-17 12:38:38,060] jepsen worker 65 - jepsen.util 11675 :invoke :read [21 nil] INFO [2019-12-17 12:38:38,069] jepsen worker 63 - jepsen.util 11763 :ok :write [21 35] INFO [2019-12-17 12:38:38,069] jepsen worker 63 - jepsen.util 11763 :invoke :read [21 nil] INFO [2019-12-17 12:38:38,071] jepsen worker 65 - jepsen.util 11675 :ok :read [21 7] INFO [2019-12-17 12:38:38,071] jepsen worker 65 - jepsen.util 11675 :invoke :read [21 nil] INFO [2019-12-17 12:38:38,096] jepsen worker 63 - jepsen.util 11763 :ok :read [21 35] INFO [2019-12-17 12:38:38,096] jepsen worker 63 - jepsen.util 11763 :invoke :read [21 nil] INFO [2019-12-17 12:38:38,106] jepsen worker 65 - jepsen.util 11675 :ok :read [21 35] INFO [2019-12-17 12:38:38,106] jepsen worker 65 - jepsen.util 11675 :invoke :cas [21 [46 5]] INFO [2019-12-17 12:38:38,114] jepsen worker 63 - jepsen.util 11763 :ok :read [21 35] INFO [2019-12-17 12:38:38,115] jepsen worker 63 - jepsen.util 11763 :invoke :read [21 nil] INFO [2019-12-17 12:38:38,124] jepsen worker 65 - jepsen.util 11675 :fail :cas [21 [46 5]] :ValueNotSwapFrom INFO [2019-12-17 12:38:38,124] jepsen worker 65 - jepsen.util 11675 :invoke :write [21 43] INFO [2019-12-17 12:38:38,126] jepsen worker 74 - jepsen.util 13214 :ok :write [24 14] INFO [2019-12-17 12:38:38,126] jepsen worker 74 - jepsen.util 13214 :invoke :write [24 18] INFO [2019-12-17 12:38:38,146] jepsen worker 32 - jepsen.util 13622 :ok :write [10 35] INFO [2019-12-17 12:38:38,146] jepsen worker 32 - jepsen.util 13622 :invoke :cas [10 [41 46]] INFO [2019-12-17 12:38:38,146] jepsen worker 28 - jepsen.util 11458 :fail :cas [9 [16 16]] :ValueNotSwapFrom INFO [2019-12-17 12:38:38,146] jepsen worker 28 - jepsen.util 11458 :invoke :read [9 nil] INFO [2019-12-17 12:38:38,149] jepsen worker 34 - jepsen.util 19294 :info :write [11 14] :RequestCanceledException INFO [2019-12-17 12:38:38,150] jepsen worker 34 - jepsen.util 19384 :invoke :write [11 33] INFO [2019-12-17 12:38:38,152] jepsen worker 63 - jepsen.util 11763 :ok :read [21 35] INFO [2019-12-17 12:38:38,152] jepsen worker 63 - jepsen.util 11763 :invoke :write [21 6] INFO [2019-12-17 12:38:38,168] jepsen worker 46 - jepsen.util 13006 :ok :read [15 37] INFO [2019-12-17 12:38:38,168] jepsen worker 46 - jepsen.util 13006 :invoke :cas [15 [5 49]] INFO [2019-12-17 12:38:38,173] jepsen worker 11 - jepsen.util 13241 :ok :write [3 0] INFO [2019-12-17 12:38:38,174] jepsen worker 11 - jepsen.util 13241 :invoke :cas [3 [31 15]] INFO [2019-12-17 12:38:38,179] jepsen worker 30 - jepsen.util 12900 :ok :read [10 35] INFO [2019-12-17 12:38:38,179] jepsen worker 30 - jepsen.util 12900 :invoke :read [10 nil] INFO [2019-12-17 12:38:38,192] jepsen worker 11 - jepsen.util 13241 :fail :cas [3 [31 15]] :ValueNotSwapFrom INFO [2019-12-17 12:38:38,192] jepsen worker 11 - jepsen.util 13241 :invoke :cas [3 [3 1]] INFO [2019-12-17 12:38:38,193] jepsen worker 81 - jepsen.util 17811 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:38:38,193] jepsen worker 81 - jepsen.util 17811 :invoke :read [27 nil] INFO [2019-12-17 12:38:38,195] jepsen worker 32 - jepsen.util 13622 :fail :cas [10 [41 46]] :ValueNotSwapFrom INFO [2019-12-17 12:38:38,196] jepsen worker 32 - jepsen.util 13622 :invoke :read [10 nil] INFO [2019-12-17 12:38:38,199] jepsen worker 12 - jepsen.util 13512 :fail :cas [4 [34 23]] :ValueNotSwapFrom INFO [2019-12-17 12:38:38,199] jepsen worker 12 - jepsen.util 13512 :invoke :cas [4 [8 4]] INFO [2019-12-17 12:38:38,213] jepsen worker 11 - jepsen.util 13241 :fail :cas [3 [3 1]] :ValueNotSwapFrom INFO [2019-12-17 12:38:38,213] jepsen worker 11 - jepsen.util 13241 :invoke :cas [3 [35 18]] INFO [2019-12-17 12:38:38,218] jepsen worker 4 - jepsen.util 12604 :fail :cas [1 [46 21]] :ValueNotSwapFrom INFO [2019-12-17 12:38:38,218] jepsen worker 4 - jepsen.util 12604 :invoke :write [1 26] INFO [2019-12-17 12:38:38,235] jepsen worker 35 - jepsen.util 18575 :info :write [11 27] :RequestCanceledException INFO [2019-12-17 12:38:38,235] jepsen worker 61 - jepsen.util 19681 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:38:38,235] jepsen worker 61 - jepsen.util 19681 :invoke :write [20 41] INFO [2019-12-17 12:38:38,236] jepsen worker 35 - jepsen.util 18665 :invoke :read [11 nil] INFO [2019-12-17 12:38:38,237] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77550,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:38:38,237] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77540,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:38:38,237] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77550,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:38:38,237] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77540,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:38:38,237] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77554,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:38:38,237] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77541,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:38:38,238] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77554,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:38:38,238] jepsen worker 89 - jepsen.util 13499 :fail :cas [29 [20 9]] :ValueNotSwapFrom INFO [2019-12-17 12:38:38,238] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77541,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:38:38,238] jepsen worker 89 - jepsen.util 13499 :invoke :cas [29 [36 13]] INFO [2019-12-17 12:38:38,238] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77542,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:38:38,238] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77542,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:38:38,245] jepsen worker 46 - jepsen.util 13006 :fail :cas [15 [5 49]] :ValueNotSwapFrom INFO [2019-12-17 12:38:38,245] jepsen worker 46 - jepsen.util 13006 :invoke :read [15 nil] INFO [2019-12-17 12:38:38,249] jepsen worker 26 - jepsen.util 19196 :info :cas [8 [17 25]] :RequestCanceledException INFO [2019-12-17 12:38:38,250] jepsen worker 26 - jepsen.util 19286 :invoke :write [8 48] INFO [2019-12-17 12:38:38,266] jepsen worker 83 - jepsen.util 18263 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:38:38,266] jepsen worker 83 - jepsen.util 18263 :invoke :write [27 32] INFO [2019-12-17 12:38:38,273] jepsen worker 30 - jepsen.util 12900 :ok :read [10 35] INFO [2019-12-17 12:38:38,273] jepsen worker 11 - jepsen.util 13241 :fail :cas [3 [35 18]] :ValueNotSwapFrom INFO [2019-12-17 12:38:38,273] jepsen worker 11 - jepsen.util 13241 :invoke :cas [3 [32 40]] INFO [2019-12-17 12:38:38,273] jepsen worker 30 - jepsen.util 12900 :invoke :cas [10 [20 48]] INFO [2019-12-17 12:38:38,305] jepsen worker 32 - jepsen.util 13622 :ok :read [10 35] INFO [2019-12-17 12:38:38,305] jepsen worker 32 - jepsen.util 13622 :invoke :cas [10 [47 21]] INFO [2019-12-17 12:38:38,316] jepsen worker 11 - jepsen.util 13241 :fail :cas [3 [32 40]] :ValueNotSwapFrom INFO [2019-12-17 12:38:38,316] jepsen worker 11 - jepsen.util 13241 :invoke :cas [3 [11 30]] INFO [2019-12-17 12:38:38,336] jepsen worker 46 - jepsen.util 13006 :ok :read [15 37] INFO [2019-12-17 12:38:38,336] jepsen worker 46 - jepsen.util 13006 :invoke :read [15 nil] INFO [2019-12-17 12:38:38,374] jepsen worker 11 - jepsen.util 13241 :fail :cas [3 [11 30]] :ValueNotSwapFrom INFO [2019-12-17 12:38:38,374] jepsen worker 11 - jepsen.util 13241 :invoke :write [3 9] INFO [2019-12-17 12:38:38,376] jepsen worker 71 - jepsen.util 14291 :ok :read [23 18] INFO [2019-12-17 12:38:38,376] jepsen worker 71 - jepsen.util 14291 :invoke :cas [23 [3 12]] INFO [2019-12-17 12:38:38,393] jepsen worker 52 - jepsen.util 13012 :ok :read [17 16] INFO [2019-12-17 12:38:38,393] jepsen worker 52 - jepsen.util 13012 :invoke :cas [17 [15 14]] INFO [2019-12-17 12:38:38,425] jepsen worker 30 - jepsen.util 12900 :fail :cas [10 [20 48]] :ValueNotSwapFrom INFO [2019-12-17 12:38:38,425] jepsen worker 30 - jepsen.util 12900 :invoke :write [10 12] INFO [2019-12-17 12:38:38,456] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77572,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:38:38,456] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77572,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:38:38,456] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77575,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:38:38,456] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77575,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:38:38,457] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77576,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:38:38,457] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77576,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:38:38,462] jepsen worker 65 - jepsen.util 11675 :ok :write [21 43] INFO [2019-12-17 12:38:38,462] jepsen worker 65 - jepsen.util 11675 :invoke :cas [21 [36 41]] INFO [2019-12-17 12:38:38,470] jepsen worker 32 - jepsen.util 13622 :fail :cas [10 [47 21]] :ValueNotSwapFrom INFO [2019-12-17 12:38:38,471] jepsen worker 32 - jepsen.util 13622 :invoke :cas [10 [48 18]] INFO [2019-12-17 12:38:38,478] jepsen worker 80 - jepsen.util 12860 :ok :read [26 35] INFO [2019-12-17 12:38:38,479] jepsen worker 80 - jepsen.util 12860 :invoke :write [26 12] INFO [2019-12-17 12:38:38,482] jepsen worker 12 - jepsen.util 13512 :fail :cas [4 [8 4]] :ValueNotSwapFrom INFO [2019-12-17 12:38:38,482] jepsen worker 12 - jepsen.util 13512 :invoke :write [4 3] INFO [2019-12-17 12:38:38,485] jepsen worker 5 - jepsen.util 13145 :ok :read [1 21] INFO [2019-12-17 12:38:38,485] jepsen worker 5 - jepsen.util 13145 :invoke :cas [1 [4 21]] INFO [2019-12-17 12:38:38,494] jepsen worker 36 - jepsen.util 14076 :fail :cas [12 [26 15]] :ValueNotSwapFrom INFO [2019-12-17 12:38:38,494] jepsen worker 36 - jepsen.util 14076 :invoke :read [12 nil] INFO [2019-12-17 12:38:38,500] jepsen worker 37 - jepsen.util 12817 :info :write [12 45] :RequestCanceledException INFO [2019-12-17 12:38:38,501] jepsen worker 37 - jepsen.util 12907 :invoke :write [12 43] INFO [2019-12-17 12:38:38,516] jepsen worker 46 - jepsen.util 13006 :ok :read [15 37] INFO [2019-12-17 12:38:38,516] jepsen worker 46 - jepsen.util 13006 :invoke :write [15 38] INFO [2019-12-17 12:38:38,524] jepsen worker 89 - jepsen.util 13499 :fail :cas [29 [36 13]] :ValueNotSwapFrom INFO [2019-12-17 12:38:38,524] jepsen worker 89 - jepsen.util 13499 :invoke :cas [29 [16 5]] INFO [2019-12-17 12:38:38,546] jepsen worker 65 - jepsen.util 11675 :fail :cas [21 [36 41]] :ValueNotSwapFrom INFO [2019-12-17 12:38:38,546] jepsen worker 65 - jepsen.util 11675 :invoke :read [21 nil] INFO [2019-12-17 12:38:38,547] jepsen worker 63 - jepsen.util 11763 :ok :write [21 6] INFO [2019-12-17 12:38:38,547] jepsen worker 63 - jepsen.util 11763 :invoke :cas [21 [20 23]] INFO [2019-12-17 12:38:38,547] jepsen worker 54 - jepsen.util 14094 :ok :read [18 23] INFO [2019-12-17 12:38:38,547] jepsen worker 54 - jepsen.util 14094 :invoke :write [18 45] INFO [2019-12-17 12:38:38,564] jepsen worker 70 - jepsen.util 13840 :fail :cas [23 [48 30]] :ValueNotSwapFrom INFO [2019-12-17 12:38:38,564] jepsen worker 70 - jepsen.util 13840 :invoke :cas [23 [9 6]] INFO [2019-12-17 12:38:38,602] jepsen worker 20 - jepsen.util 13160 :ok :write [6 46] INFO [2019-12-17 12:38:38,603] jepsen worker 20 - jepsen.util 13160 :invoke :read [6 nil] INFO [2019-12-17 12:38:38,605] jepsen worker 65 - jepsen.util 11675 :ok :read [21 6] INFO [2019-12-17 12:38:38,605] jepsen worker 65 - jepsen.util 11675 :invoke :write [21 17] INFO [2019-12-17 12:38:38,606] jepsen worker 63 - jepsen.util 11763 :fail :cas [21 [20 23]] :ValueNotSwapFrom INFO [2019-12-17 12:38:38,606] jepsen worker 63 - jepsen.util 11763 :invoke :cas [21 [49 47]] INFO [2019-12-17 12:38:38,625] jepsen worker 56 - jepsen.util 14186 :fail :cas [18 [41 7]] :ValueNotSwapFrom INFO [2019-12-17 12:38:38,625] jepsen worker 56 - jepsen.util 14186 :invoke :write [18 37] INFO [2019-12-17 12:38:38,639] jepsen worker 63 - jepsen.util 11763 :fail :cas [21 [49 47]] :ValueNotSwapFrom INFO [2019-12-17 12:38:38,639] jepsen worker 63 - jepsen.util 11763 :invoke :cas [21 [35 1]] INFO [2019-12-17 12:38:38,674] jepsen worker 63 - jepsen.util 11763 :fail :cas [21 [35 1]] :ValueNotSwapFrom INFO [2019-12-17 12:38:38,674] jepsen worker 63 - jepsen.util 11763 :invoke :cas [21 [22 47]] INFO [2019-12-17 12:38:38,675] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77596,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:38:38,675] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77596,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:38:38,676] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77600,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:38:38,676] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77600,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:38:38,703] jepsen worker 63 - jepsen.util 11763 :fail :cas [21 [22 47]] :ValueNotSwapFrom INFO [2019-12-17 12:38:38,703] jepsen worker 63 - jepsen.util 11763 :invoke :read [21 nil] INFO [2019-12-17 12:38:38,715] jepsen worker 65 - jepsen.util 11675 :ok :write [21 17] INFO [2019-12-17 12:38:38,715] jepsen worker 65 - jepsen.util 11675 :invoke :read [21 nil] INFO [2019-12-17 12:38:38,719] jepsen worker 12 - jepsen.util 13512 :ok :write [4 3] INFO [2019-12-17 12:38:38,719] jepsen worker 12 - jepsen.util 13512 :invoke :write [4 44] INFO [2019-12-17 12:38:38,721] jepsen worker 63 - jepsen.util 11763 :ok :read [21 6] INFO [2019-12-17 12:38:38,721] jepsen worker 63 - jepsen.util 11763 :invoke :read [21 nil] INFO [2019-12-17 12:38:38,725] jepsen worker 11 - jepsen.util 13241 :ok :write [3 9] INFO [2019-12-17 12:38:38,725] jepsen worker 11 - jepsen.util 13241 :invoke :read [3 nil] INFO [2019-12-17 12:38:38,725] jepsen worker 30 - jepsen.util 12900 :ok :write [10 12] INFO [2019-12-17 12:38:38,725] jepsen worker 30 - jepsen.util 12900 :invoke :read [10 nil] INFO [2019-12-17 12:38:38,758] jepsen worker 11 - jepsen.util 13241 :ok :read [3 9] INFO [2019-12-17 12:38:38,758] jepsen worker 11 - jepsen.util 13241 :invoke :write [3 34] INFO [2019-12-17 12:38:38,801] jepsen worker 65 - jepsen.util 11675 :ok :read [21 17] INFO [2019-12-17 12:38:38,801] jepsen worker 65 - jepsen.util 11675 :invoke :read [21 nil] INFO [2019-12-17 12:38:38,802] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:38:38,802] jepsen nemesis - couchbase.util Rebalance check count: 610 INFO [2019-12-17 12:38:38,802] jepsen nemesis - couchbase.util Rebalance stuck count: 591 INFO [2019-12-17 12:38:38,808] jepsen worker 63 - jepsen.util 11763 :ok :read [21 17] INFO [2019-12-17 12:38:38,808] jepsen worker 63 - jepsen.util 11763 :invoke :read [21 nil] INFO [2019-12-17 12:38:38,816] jepsen worker 65 - jepsen.util 11675 :ok :read [21 17] INFO [2019-12-17 12:38:38,816] jepsen worker 65 - jepsen.util 11675 :invoke :read [21 nil] INFO [2019-12-17 12:38:38,851] jepsen worker 63 - jepsen.util 11763 :ok :read [21 17] INFO [2019-12-17 12:38:38,851] jepsen worker 63 - jepsen.util 11763 :invoke :read [21 nil] INFO [2019-12-17 12:38:38,873] jepsen worker 65 - jepsen.util 11675 :ok :read [21 17] INFO [2019-12-17 12:38:38,873] jepsen worker 65 - jepsen.util 11675 :invoke :read [21 nil] INFO [2019-12-17 12:38:38,882] jepsen worker 63 - jepsen.util 11763 :ok :read [21 17] INFO [2019-12-17 12:38:38,882] jepsen worker 63 - jepsen.util 11763 :invoke :cas [21 [36 38]] INFO [2019-12-17 12:38:38,895] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77616,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:38:38,895] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77616,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:38:38,895] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77618,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:38:38,895] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77618,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:38:38,935] jepsen worker 74 - jepsen.util 13214 :ok :write [24 18] INFO [2019-12-17 12:38:38,935] jepsen worker 74 - jepsen.util 13214 :invoke :read [24 nil] INFO [2019-12-17 12:38:38,954] jepsen worker 63 - jepsen.util 11763 :fail :cas [21 [36 38]] :ValueNotSwapFrom INFO [2019-12-17 12:38:38,954] jepsen worker 63 - jepsen.util 11763 :invoke :write [21 14] INFO [2019-12-17 12:38:38,957] jepsen worker 65 - jepsen.util 11675 :ok :read [21 17] INFO [2019-12-17 12:38:38,957] jepsen worker 65 - jepsen.util 11675 :invoke :read [21 nil] INFO [2019-12-17 12:38:38,959] jepsen worker 89 - jepsen.util 13499 :fail :cas [29 [16 5]] :ValueNotSwapFrom INFO [2019-12-17 12:38:38,959] jepsen worker 89 - jepsen.util 13499 :invoke :read [29 nil] INFO [2019-12-17 12:38:38,976] jepsen worker 3 - jepsen.util 13053 :ok :write [1 12] INFO [2019-12-17 12:38:38,976] jepsen worker 3 - jepsen.util 13053 :invoke :write [1 15] INFO [2019-12-17 12:38:39,002] jepsen worker 51 - jepsen.util 12651 :info :write [17 42] :RequestCanceledException INFO [2019-12-17 12:38:39,003] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77627,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:38:39,003] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77627,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:38:39,003] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77630,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:38:39,003] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77630,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:38:39,003] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77631,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:38:39,004] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77631,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:38:39,006] jepsen worker 51 - jepsen.util 12741 :invoke :cas [17 [27 0]] INFO [2019-12-17 12:38:39,007] jepsen worker 65 - jepsen.util 11675 :ok :read [21 17] INFO [2019-12-17 12:38:39,007] jepsen worker 65 - jepsen.util 11675 :invoke :write [21 36] INFO [2019-12-17 12:38:39,011] jepsen worker 20 - jepsen.util 13160 :ok :read [6 46] INFO [2019-12-17 12:38:39,011] jepsen worker 20 - jepsen.util 13160 :invoke :cas [6 [21 25]] INFO [2019-12-17 12:38:39,019] jepsen worker 55 - jepsen.util 14005 :info :write [18 19] :RequestCanceledException INFO [2019-12-17 12:38:39,025] jepsen worker 55 - jepsen.util 14095 :invoke :read [18 nil] INFO [2019-12-17 12:38:39,043] jepsen worker 74 - jepsen.util 13214 :ok :read [24 18] INFO [2019-12-17 12:38:39,043] jepsen worker 74 - jepsen.util 13214 :invoke :write [24 22] INFO [2019-12-17 12:38:39,064] jepsen worker 38 - jepsen.util 14078 :fail :cas [12 [6 34]] :ValueNotSwapFrom INFO [2019-12-17 12:38:39,064] jepsen worker 38 - jepsen.util 14078 :invoke :cas [12 [8 37]] INFO [2019-12-17 12:38:39,082] jepsen worker 20 - jepsen.util 13160 :fail :cas [6 [21 25]] :ValueNotSwapFrom INFO [2019-12-17 12:38:39,082] jepsen worker 20 - jepsen.util 13160 :invoke :write [6 10] INFO [2019-12-17 12:38:39,087] jepsen worker 88 - jepsen.util 13318 :ok :write [29 34] INFO [2019-12-17 12:38:39,088] jepsen worker 88 - jepsen.util 13318 :invoke :cas [29 [44 26]] INFO [2019-12-17 12:38:39,108] jepsen worker 63 - jepsen.util 11763 :ok :write [21 14] INFO [2019-12-17 12:38:39,108] jepsen worker 46 - jepsen.util 13006 :ok :write [15 38] INFO [2019-12-17 12:38:39,108] jepsen worker 46 - jepsen.util 13006 :invoke :write [15 2] INFO [2019-12-17 12:38:39,108] jepsen worker 63 - jepsen.util 11763 :invoke :cas [21 [28 5]] INFO [2019-12-17 12:38:39,213] jepsen worker 65 - jepsen.util 11675 :ok :write [21 36] INFO [2019-12-17 12:38:39,214] jepsen worker 65 - jepsen.util 11675 :invoke :cas [21 [34 46]] INFO [2019-12-17 12:38:39,218] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77642,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:38:39,218] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77642,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:38:39,218] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77643,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:38:39,218] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77643,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:38:39,222] jepsen worker 80 - jepsen.util 12860 :ok :write [26 12] INFO [2019-12-17 12:38:39,222] jepsen worker 80 - jepsen.util 12860 :invoke :read [26 nil] INFO [2019-12-17 12:38:39,223] jepsen worker 12 - jepsen.util 13512 :ok :write [4 44] INFO [2019-12-17 12:38:39,223] jepsen worker 12 - jepsen.util 13512 :invoke :read [4 nil] INFO [2019-12-17 12:38:39,231] jepsen worker 11 - jepsen.util 13241 :ok :write [3 34] INFO [2019-12-17 12:38:39,232] jepsen worker 11 - jepsen.util 13241 :invoke :read [3 nil] INFO [2019-12-17 12:38:39,236] jepsen worker 64 - jepsen.util 10504 :info :write [21 14] :RequestCanceledException INFO [2019-12-17 12:38:39,445] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77652,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:38:39,445] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77661,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:38:39,445] jepsen worker 46 - jepsen.util 13006 :ok :write [15 2] INFO [2019-12-17 12:38:39,445] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77652,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:38:39,445] jepsen worker 37 - jepsen.util 12907 :ok :write [12 43] INFO [2019-12-17 12:38:39,445] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77661,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:38:39,445] jepsen worker 74 - jepsen.util 13214 :ok :write [24 22] INFO [2019-12-17 12:38:39,445] jepsen worker 46 - jepsen.util 13006 :invoke :write [15 0] INFO [2019-12-17 12:38:39,445] jepsen worker 9 - jepsen.util 13869 :info :write [3 49] :RequestCanceledException INFO [2019-12-17 12:38:39,445] jepsen worker 37 - jepsen.util 12907 :invoke :read [12 nil] INFO [2019-12-17 12:38:39,446] jepsen worker 74 - jepsen.util 13214 :invoke :read [24 nil] INFO [2019-12-17 12:38:39,446] jepsen worker 64 - jepsen.util 10594 :invoke :write [21 39] INFO [2019-12-17 12:38:39,447] jepsen worker 9 - jepsen.util 13959 :invoke :read [3 nil] INFO [2019-12-17 12:38:39,455] jepsen worker 65 - jepsen.util 11675 :fail :cas [21 [34 46]] :ValueNotSwapFrom INFO [2019-12-17 12:38:39,455] jepsen worker 30 - jepsen.util 12900 :ok :read [10 12] INFO [2019-12-17 12:38:39,455] jepsen worker 65 - jepsen.util 11675 :invoke :cas [21 [38 49]] INFO [2019-12-17 12:38:39,455] jepsen worker 30 - jepsen.util 12900 :invoke :read [10 nil] INFO [2019-12-17 12:38:39,456] jepsen worker 63 - jepsen.util 11763 :fail :cas [21 [28 5]] :ValueNotSwapFrom INFO [2019-12-17 12:38:39,456] jepsen worker 63 - jepsen.util 11763 :invoke :write [21 15] INFO [2019-12-17 12:38:39,463] jepsen worker 11 - jepsen.util 13241 :ok :read [3 34] INFO [2019-12-17 12:38:39,464] jepsen worker 11 - jepsen.util 13241 :invoke :read [3 nil] INFO [2019-12-17 12:38:39,464] jepsen worker 52 - jepsen.util 13012 :fail :cas [17 [15 14]] :ValueNotSwapFrom INFO [2019-12-17 12:38:39,464] jepsen worker 52 - jepsen.util 13012 :invoke :read [17 nil] INFO [2019-12-17 12:38:39,504] jepsen worker 5 - jepsen.util 13145 :fail :cas [1 [4 21]] :ValueNotSwapFrom INFO [2019-12-17 12:38:39,504] jepsen worker 5 - jepsen.util 13145 :invoke :cas [1 [40 39]] INFO [2019-12-17 12:38:39,529] jepsen worker 9 - jepsen.util 13959 :ok :read [3 34] INFO [2019-12-17 12:38:39,529] jepsen worker 9 - jepsen.util 13959 :invoke :read [3 nil] INFO [2019-12-17 12:38:39,535] jepsen worker 53 - jepsen.util 13103 :info :write [17 27] :RequestCanceledException INFO [2019-12-17 12:38:39,538] jepsen worker 65 - jepsen.util 11675 :fail :cas [21 [38 49]] :ValueNotSwapFrom INFO [2019-12-17 12:38:39,538] jepsen worker 65 - jepsen.util 11675 :invoke :write [21 2] INFO [2019-12-17 12:38:39,539] jepsen worker 53 - jepsen.util 13193 :invoke :cas [17 [34 41]] INFO [2019-12-17 12:38:39,544] jepsen worker 74 - jepsen.util 13214 :ok :read [24 22] INFO [2019-12-17 12:38:39,545] jepsen worker 74 - jepsen.util 13214 :invoke :cas [24 [19 7]] INFO [2019-12-17 12:38:39,561] jepsen worker 74 - jepsen.util 13214 :fail :cas [24 [19 7]] :ValueNotSwapFrom INFO [2019-12-17 12:38:39,561] jepsen worker 89 - jepsen.util 13499 :ok :read [29 34] INFO [2019-12-17 12:38:39,561] jepsen worker 74 - jepsen.util 13214 :invoke :read [24 nil] INFO [2019-12-17 12:38:39,561] jepsen worker 89 - jepsen.util 13499 :invoke :write [29 24] INFO [2019-12-17 12:38:39,561] jepsen worker 37 - jepsen.util 12907 :ok :read [12 43] INFO [2019-12-17 12:38:39,561] jepsen worker 37 - jepsen.util 12907 :invoke :cas [12 [28 0]] INFO [2019-12-17 12:38:39,589] jepsen worker 74 - jepsen.util 13214 :ok :read [24 22] INFO [2019-12-17 12:38:39,589] jepsen worker 74 - jepsen.util 13214 :invoke :read [24 nil] INFO [2019-12-17 12:38:39,601] jepsen worker 14 - jepsen.util 12974 :info :write [4 11] :RequestCanceledException INFO [2019-12-17 12:38:39,602] jepsen worker 14 - jepsen.util 13064 :invoke :write [4 4] INFO [2019-12-17 12:38:39,631] jepsen worker 37 - jepsen.util 12907 :fail :cas [12 [28 0]] :ValueNotSwapFrom INFO [2019-12-17 12:38:39,631] jepsen worker 37 - jepsen.util 12907 :invoke :write [12 20] INFO [2019-12-17 12:38:39,632] jepsen worker 69 - jepsen.util 13749 :info :write [23 20] :RequestCanceledException INFO [2019-12-17 12:38:39,633] jepsen worker 69 - jepsen.util 13839 :invoke :write [23 19] INFO [2019-12-17 12:38:39,642] jepsen worker 63 - jepsen.util 11763 :ok :write [21 15] INFO [2019-12-17 12:38:39,642] jepsen worker 63 - jepsen.util 11763 :invoke :write [21 27] INFO [2019-12-17 12:38:39,643] jepsen worker 74 - jepsen.util 13214 :ok :read [24 22] INFO [2019-12-17 12:38:39,643] jepsen worker 74 - jepsen.util 13214 :invoke :read [24 nil] INFO [2019-12-17 12:38:39,663] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77691,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:38:39,663] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77691,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:38:39,663] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77693,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:38:39,663] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77693,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:38:39,686] jepsen worker 74 - jepsen.util 13214 :ok :read [24 22] INFO [2019-12-17 12:38:39,686] jepsen worker 74 - jepsen.util 13214 :invoke :read [24 nil] INFO [2019-12-17 12:38:39,706] jepsen worker 64 - jepsen.util 10594 :ok :write [21 39] INFO [2019-12-17 12:38:39,706] jepsen worker 64 - jepsen.util 10594 :invoke :cas [21 [36 8]] INFO [2019-12-17 12:38:39,714] jepsen worker 74 - jepsen.util 13214 :ok :read [24 22] INFO [2019-12-17 12:38:39,714] jepsen worker 74 - jepsen.util 13214 :invoke :read [24 nil] INFO [2019-12-17 12:38:39,717] jepsen worker 24 - jepsen.util 17484 :info :write [8 1] :RequestCanceledException INFO [2019-12-17 12:38:39,718] jepsen worker 24 - jepsen.util 17574 :invoke :read [8 nil] INFO [2019-12-17 12:38:39,734] jepsen worker 13 - jepsen.util 12973 :ok :write [4 21] INFO [2019-12-17 12:38:39,734] jepsen worker 13 - jepsen.util 12973 :invoke :cas [4 [16 5]] INFO [2019-12-17 12:38:39,735] jepsen worker 2 - jepsen.util 17642 :info :write [0 2] :RequestCanceledException INFO [2019-12-17 12:38:39,735] jepsen worker 7 - jepsen.util 18187 :info :write [2 2] :RequestCanceledException INFO [2019-12-17 12:38:39,736] jepsen worker 2 - jepsen.util 17732 :invoke :cas [0 [11 43]] INFO [2019-12-17 12:38:39,736] jepsen worker 7 - jepsen.util 18277 :invoke :cas [2 [39 43]] INFO [2019-12-17 12:38:39,749] jepsen worker 40 - jepsen.util 18940 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:38:39,749] jepsen worker 79 - jepsen.util 13579 :info :write [26 27] :RequestCanceledException INFO [2019-12-17 12:38:39,749] jepsen worker 57 - jepsen.util 17877 :info :cas [19 [4 19]] :RequestCanceledException INFO [2019-12-17 12:38:39,749] jepsen worker 40 - jepsen.util 18940 :invoke :cas [13 [8 2]] INFO [2019-12-17 12:38:39,750] jepsen worker 57 - jepsen.util 17967 :invoke :write [19 16] INFO [2019-12-17 12:38:39,750] jepsen worker 79 - jepsen.util 13669 :invoke :write [26 46] INFO [2019-12-17 12:38:39,766] jepsen worker 67 - jepsen.util 18697 :info :write [22 42] :RequestCanceledException INFO [2019-12-17 12:38:39,766] jepsen worker 76 - jepsen.util 18616 :info :cas [25 [45 24]] :RequestCanceledException INFO [2019-12-17 12:38:39,767] jepsen worker 76 - jepsen.util 18706 :invoke :read [25 nil] INFO [2019-12-17 12:38:39,767] jepsen worker 67 - jepsen.util 18787 :invoke :cas [22 [39 0]] INFO [2019-12-17 12:38:39,775] jepsen worker 14 - jepsen.util 13064 :ok :write [4 4] INFO [2019-12-17 12:38:39,775] jepsen worker 14 - jepsen.util 13064 :invoke :cas [4 [32 43]] INFO [2019-12-17 12:38:39,791] jepsen worker 64 - jepsen.util 10594 :fail :cas [21 [36 8]] :ValueNotSwapFrom INFO [2019-12-17 12:38:39,791] jepsen worker 64 - jepsen.util 10594 :invoke :write [21 17] INFO [2019-12-17 12:38:39,791] jepsen worker 74 - jepsen.util 13214 :ok :read [24 22] INFO [2019-12-17 12:38:39,791] jepsen worker 74 - jepsen.util 13214 :invoke :read [24 nil] INFO [2019-12-17 12:38:39,801] jepsen worker 46 - jepsen.util 13006 :ok :write [15 0] INFO [2019-12-17 12:38:39,801] jepsen worker 46 - jepsen.util 13006 :invoke :cas [15 [0 38]] INFO [2019-12-17 12:38:39,801] jepsen worker 16 - jepsen.util 19996 :info :write [5 2] :RequestCanceledException INFO [2019-12-17 12:38:39,801] jepsen worker 19 - jepsen.util 14239 :fail :read [6 nil] :RequestTimeoutException INFO [2019-12-17 12:38:39,801] jepsen worker 59 - jepsen.util 17429 :info :write [19 6] :RequestCanceledException INFO [2019-12-17 12:38:39,801] jepsen worker 85 - jepsen.util 18805 :info :cas [28 [36 15]] :RequestCanceledException INFO [2019-12-17 12:38:39,802] jepsen worker 19 - jepsen.util 14239 :invoke :read [6 nil] INFO [2019-12-17 12:38:39,802] jepsen worker 18 - jepsen.util 14058 :info :cas [6 [13 21]] :RequestCanceledException INFO [2019-12-17 12:38:39,803] jepsen worker 59 - jepsen.util 17519 :invoke :write [19 27] INFO [2019-12-17 12:38:39,803] jepsen worker 16 - jepsen.util 20086 :invoke :write [5 6] INFO [2019-12-17 12:38:39,803] jepsen worker 85 - jepsen.util 18895 :invoke :cas [28 [6 49]] INFO [2019-12-17 12:38:39,805] jepsen worker 18 - jepsen.util 14148 :invoke :read [6 nil] INFO [2019-12-17 12:38:39,811] jepsen worker 36 - jepsen.util 14076 :ok :read [12 43] INFO [2019-12-17 12:38:39,811] jepsen worker 36 - jepsen.util 14076 :invoke :read [12 nil] INFO [2019-12-17 12:38:39,819] jepsen worker 45 - jepsen.util 12915 :info :write [15 35] :RequestCanceledException INFO [2019-12-17 12:38:39,819] jepsen worker 72 - jepsen.util 13392 :info :cas [24 [45 12]] :RequestCanceledException INFO [2019-12-17 12:38:39,819] jepsen worker 10 - jepsen.util 13240 :info :write [3 40] :RequestCanceledException INFO [2019-12-17 12:38:39,819] jepsen worker 47 - jepsen.util 14627 :info :cas [15 [26 22]] :RequestCanceledException INFO [2019-12-17 12:38:39,819] jepsen worker 66 - jepsen.util 18606 :info :write [22 45] :RequestCanceledException INFO [2019-12-17 12:38:39,820] jepsen worker 10 - jepsen.util 13330 :invoke :read [3 nil] INFO [2019-12-17 12:38:39,820] jepsen worker 47 - jepsen.util 14717 :invoke :read [15 nil] INFO [2019-12-17 12:38:39,820] jepsen worker 66 - jepsen.util 18696 :invoke :read [22 nil] INFO [2019-12-17 12:38:39,820] jepsen worker 45 - jepsen.util 13005 :invoke :read [15 nil] INFO [2019-12-17 12:38:39,821] jepsen worker 72 - jepsen.util 13482 :invoke :write [24 15] INFO [2019-12-17 12:38:39,830] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:38:39,830] jepsen nemesis - couchbase.util Rebalance check count: 611 INFO [2019-12-17 12:38:39,830] jepsen nemesis - couchbase.util Rebalance stuck count: 592 INFO [2019-12-17 12:38:39,834] jepsen worker 33 - jepsen.util 18123 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:38:39,834] jepsen worker 77 - jepsen.util 19427 :info :write [25 10] :RequestCanceledException INFO [2019-12-17 12:38:39,834] jepsen worker 31 - jepsen.util 11551 :info :write [10 42] :RequestCanceledException INFO [2019-12-17 12:38:39,834] jepsen worker 22 - jepsen.util 18742 :info :write [7 40] :RequestCanceledException INFO [2019-12-17 12:38:39,834] jepsen worker 17 - jepsen.util 18197 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:38:39,834] jepsen worker 84 - jepsen.util 17724 :info :cas [28 [4 14]] :RequestCanceledException INFO [2019-12-17 12:38:39,835] jepsen worker 33 - jepsen.util 18123 :invoke :write [11 43] INFO [2019-12-17 12:38:39,835] jepsen worker 48 - jepsen.util 17058 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:38:39,835] jepsen worker 62 - jepsen.util 19052 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:38:39,835] jepsen worker 60 - jepsen.util 19410 :info :write [20 33] :RequestCanceledException INFO [2019-12-17 12:38:39,835] jepsen worker 17 - jepsen.util 18197 :invoke :write [5 5] INFO [2019-12-17 12:38:39,835] jepsen worker 21 - jepsen.util 17931 :info :write [7 0] :RequestCanceledException INFO [2019-12-17 12:38:39,835] jepsen worker 48 - jepsen.util 17058 :invoke :write [16 27] INFO [2019-12-17 12:38:39,835] jepsen worker 62 - jepsen.util 19052 :invoke :write [20 34] INFO [2019-12-17 12:38:39,836] jepsen worker 22 - jepsen.util 18832 :invoke :write [7 10] INFO [2019-12-17 12:38:39,836] jepsen worker 77 - jepsen.util 19517 :invoke :cas [25 [32 45]] INFO [2019-12-17 12:38:39,836] jepsen worker 84 - jepsen.util 17814 :invoke :read [28 nil] INFO [2019-12-17 12:38:39,836] jepsen worker 21 - jepsen.util 18021 :invoke :read [7 nil] INFO [2019-12-17 12:38:39,836] jepsen worker 31 - jepsen.util 11641 :invoke :cas [10 [47 19]] INFO [2019-12-17 12:38:39,837] jepsen worker 60 - jepsen.util 19500 :invoke :write [20 34] INFO [2019-12-17 12:38:39,849] jepsen worker 82 - jepsen.util 18442 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:38:39,849] jepsen worker 49 - jepsen.util 18319 :info :write [16 48] :RequestCanceledException INFO [2019-12-17 12:38:39,849] jepsen worker 86 - jepsen.util 19166 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:38:39,849] jepsen worker 78 - jepsen.util 13308 :info :write [26 23] :RequestCanceledException INFO [2019-12-17 12:38:39,849] jepsen worker 23 - jepsen.util 18563 :info :cas [7 [41 49]] :RequestCanceledException INFO [2019-12-17 12:38:39,849] jepsen worker 15 - jepsen.util 17385 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:38:39,849] jepsen worker 82 - jepsen.util 18442 :invoke :read [27 nil] INFO [2019-12-17 12:38:39,849] jepsen worker 86 - jepsen.util 19166 :invoke :cas [28 [29 20]] INFO [2019-12-17 12:38:39,849] jepsen worker 15 - jepsen.util 17385 :invoke :read [5 nil] INFO [2019-12-17 12:38:39,850] jepsen worker 49 - jepsen.util 18409 :invoke :read [16 nil] INFO [2019-12-17 12:38:39,850] jepsen worker 23 - jepsen.util 18653 :invoke :write [7 45] INFO [2019-12-17 12:38:39,850] jepsen worker 78 - jepsen.util 13398 :invoke :cas [26 [32 13]] INFO [2019-12-17 12:38:39,858] jepsen worker 74 - jepsen.util 13214 :ok :read [24 22] INFO [2019-12-17 12:38:39,858] jepsen worker 74 - jepsen.util 13214 :invoke :read [24 nil] INFO [2019-12-17 12:38:39,866] jepsen worker 58 - jepsen.util 18508 :info :write [19 11] :RequestCanceledException INFO [2019-12-17 12:38:39,866] jepsen worker 39 - jepsen.util 17769 :info :cas [13 [20 40]] :RequestCanceledException INFO [2019-12-17 12:38:39,866] jepsen worker 43 - jepsen.util 18583 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:38:39,866] jepsen worker 0 - jepsen.util 18810 :info :write [0 36] :RequestCanceledException INFO [2019-12-17 12:38:39,866] jepsen worker 8 - jepsen.util 18008 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:38:39,866] jepsen worker 43 - jepsen.util 18583 :invoke :write [14 14] INFO [2019-12-17 12:38:39,866] jepsen worker 8 - jepsen.util 18008 :invoke :cas [2 [37 6]] INFO [2019-12-17 12:38:39,867] jepsen worker 58 - jepsen.util 18598 :invoke :write [19 3] INFO [2019-12-17 12:38:39,867] jepsen worker 39 - jepsen.util 17859 :invoke :cas [13 [33 36]] INFO [2019-12-17 12:38:39,867] jepsen worker 0 - jepsen.util 18900 :invoke :cas [0 [3 37]] INFO [2019-12-17 12:38:39,879] jepsen worker 37 - jepsen.util 12907 :ok :write [12 20] INFO [2019-12-17 12:38:39,880] jepsen worker 37 - jepsen.util 12907 :invoke :read [12 nil] INFO [2019-12-17 12:38:39,883] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77742,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:38:39,883] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77742,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:38:39,883] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77745,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:38:39,883] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77745,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:38:39,883] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77746,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:38:39,883] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77746,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:38:39,888] jepsen worker 55 - jepsen.util 14095 :ok :read [18 19] INFO [2019-12-17 12:38:39,888] jepsen worker 55 - jepsen.util 14095 :invoke :write [18 30] INFO [2019-12-17 12:38:39,901] jepsen worker 42 - jepsen.util 19212 :info :cas [14 [41 4]] :RequestCanceledException INFO [2019-12-17 12:38:39,901] jepsen worker 68 - jepsen.util 17528 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:38:39,901] jepsen worker 68 - jepsen.util 17528 :invoke :cas [22 [21 32]] INFO [2019-12-17 12:38:39,902] jepsen worker 42 - jepsen.util 19302 :invoke :cas [14 [46 36]] INFO [2019-12-17 12:38:39,912] jepsen worker 45 - jepsen.util 13005 :ok :read [15 0] INFO [2019-12-17 12:38:39,912] jepsen worker 45 - jepsen.util 13005 :invoke :write [15 23] INFO [2019-12-17 12:38:39,920] jepsen worker 41 - jepsen.util 19661 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:38:39,920] jepsen worker 75 - jepsen.util 18075 :info :cas [25 [26 48]] :RequestCanceledException INFO [2019-12-17 12:38:39,920] jepsen worker 50 - jepsen.util 18140 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:38:39,920] jepsen worker 41 - jepsen.util 19661 :invoke :cas [13 [9 44]] INFO [2019-12-17 12:38:39,920] jepsen worker 50 - jepsen.util 18140 :invoke :cas [16 [44 46]] INFO [2019-12-17 12:38:39,923] jepsen worker 31 - jepsen.util 11641 :fail :cas [10 [47 19]] :ValueNotSwapFrom INFO [2019-12-17 12:38:39,923] jepsen worker 31 - jepsen.util 11641 :invoke :cas [10 [32 43]] INFO [2019-12-17 12:38:39,929] jepsen worker 75 - jepsen.util 18165 :invoke :read [25 nil] INFO [2019-12-17 12:38:39,936] jepsen worker 73 - jepsen.util 12313 :fail :read [24 nil] :RequestTimeoutException INFO [2019-12-17 12:38:39,936] jepsen worker 25 - jepsen.util 19465 :info :cas [8 [40 32]] :RequestCanceledException INFO [2019-12-17 12:38:39,936] jepsen worker 73 - jepsen.util 12313 :invoke :write [24 27] INFO [2019-12-17 12:38:39,937] jepsen worker 25 - jepsen.util 19555 :invoke :cas [8 [42 10]] INFO [2019-12-17 12:38:39,938] jepsen worker 30 - jepsen.util 12900 :ok :read [10 12] INFO [2019-12-17 12:38:39,938] jepsen worker 30 - jepsen.util 12900 :invoke :read [10 nil] INFO [2019-12-17 12:38:39,940] jepsen worker 19 - jepsen.util 14239 :ok :read [6 46] INFO [2019-12-17 12:38:39,940] jepsen worker 19 - jepsen.util 14239 :invoke :cas [6 [7 30]] INFO [2019-12-17 12:38:39,943] jepsen worker 10 - jepsen.util 13330 :ok :read [3 34] INFO [2019-12-17 12:38:39,943] jepsen worker 10 - jepsen.util 13330 :invoke :write [3 16] INFO [2019-12-17 12:38:39,948] jepsen worker 44 - jepsen.util 19124 :info :write [14 49] :RequestCanceledException INFO [2019-12-17 12:38:39,949] jepsen worker 44 - jepsen.util 19214 :invoke :write [14 8] INFO [2019-12-17 12:38:39,964] jepsen worker 38 - jepsen.util 14078 :fail :cas [12 [8 37]] :ValueNotSwapFrom INFO [2019-12-17 12:38:39,964] jepsen worker 38 - jepsen.util 14078 :invoke :read [12 nil] INFO [2019-12-17 12:38:39,992] jepsen worker 74 - jepsen.util 13214 :ok :read [24 22] INFO [2019-12-17 12:38:39,993] jepsen worker 74 - jepsen.util 13214 :invoke :cas [24 [22 1]] INFO [2019-12-17 12:38:39,994] jepsen worker 52 - jepsen.util 13012 :ok :read [17 42] INFO [2019-12-17 12:38:39,994] jepsen worker 52 - jepsen.util 13012 :invoke :write [17 27] INFO [2019-12-17 12:38:40,002] jepsen worker 1 - jepsen.util 18811 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:38:40,002] jepsen worker 1 - jepsen.util 18811 :invoke :cas [0 [18 7]] INFO [2019-12-17 12:38:40,015] jepsen worker 64 - jepsen.util 10594 :ok :write [21 17] INFO [2019-12-17 12:38:40,016] jepsen worker 64 - jepsen.util 10594 :invoke :write [21 19] INFO [2019-12-17 12:38:40,026] jepsen worker 88 - jepsen.util 13318 :fail :cas [29 [44 26]] :ValueNotSwapFrom INFO [2019-12-17 12:38:40,026] jepsen worker 88 - jepsen.util 13318 :invoke :read [29 nil] INFO [2019-12-17 12:38:40,035] jepsen worker 27 - jepsen.util 12267 :info :write [9 47] :RequestCanceledException INFO [2019-12-17 12:38:40,035] jepsen worker 87 - jepsen.util 13047 :info :write [29 18] :RequestCanceledException INFO [2019-12-17 12:38:40,035] jepsen worker 29 - jepsen.util 11639 :fail :read [9 nil] :RequestTimeoutException INFO [2019-12-17 12:38:40,036] jepsen worker 29 - jepsen.util 11639 :invoke :read [9 nil] INFO [2019-12-17 12:38:40,037] jepsen worker 87 - jepsen.util 13137 :invoke :write [29 33] INFO [2019-12-17 12:38:40,037] jepsen worker 27 - jepsen.util 12357 :invoke :write [9 4] INFO [2019-12-17 12:38:40,049] jepsen worker 18 - jepsen.util 14148 :ok :read [6 46] INFO [2019-12-17 12:38:40,049] jepsen worker 18 - jepsen.util 14148 :invoke :read [6 nil] INFO [2019-12-17 12:38:40,098] jepsen worker 47 - jepsen.util 14717 :ok :read [15 0] INFO [2019-12-17 12:38:40,098] jepsen worker 47 - jepsen.util 14717 :invoke :read [15 nil] INFO [2019-12-17 12:38:40,101] jepsen worker 6 - jepsen.util 18546 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:38:40,101] jepsen worker 6 - jepsen.util 18546 :invoke :write [2 30] INFO [2019-12-17 12:38:40,102] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77768,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:38:40,102] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77768,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:38:40,102] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77769,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:38:40,102] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77769,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:38:40,107] jepsen worker 30 - jepsen.util 12900 :ok :read [10 12] INFO [2019-12-17 12:38:40,107] jepsen worker 30 - jepsen.util 12900 :invoke :read [10 nil] INFO [2019-12-17 12:38:40,107] jepsen worker 14 - jepsen.util 13064 :fail :cas [4 [32 43]] :ValueNotSwapFrom INFO [2019-12-17 12:38:40,108] jepsen worker 14 - jepsen.util 13064 :invoke :read [4 nil] INFO [2019-12-17 12:38:40,112] jepsen worker 56 - jepsen.util 14186 :ok :write [18 37] INFO [2019-12-17 12:38:40,112] jepsen worker 56 - jepsen.util 14186 :invoke :write [18 36] INFO [2019-12-17 12:38:40,113] jepsen worker 5 - jepsen.util 13145 :fail :cas [1 [40 39]] :ValueNotSwapFrom INFO [2019-12-17 12:38:40,113] jepsen worker 5 - jepsen.util 13145 :invoke :read [1 nil] INFO [2019-12-17 12:38:40,129] jepsen worker 64 - jepsen.util 10594 :ok :write [21 19] INFO [2019-12-17 12:38:40,129] jepsen worker 64 - jepsen.util 10594 :invoke :read [21 nil] INFO [2019-12-17 12:38:40,135] jepsen worker 53 - jepsen.util 13193 :fail :cas [17 [34 41]] :ValueNotSwapFrom INFO [2019-12-17 12:38:40,135] jepsen worker 53 - jepsen.util 13193 :invoke :read [17 nil] INFO [2019-12-17 12:38:40,136] jepsen worker 71 - jepsen.util 14291 :fail :cas [23 [3 12]] :ValueNotSwapFrom INFO [2019-12-17 12:38:40,136] jepsen worker 71 - jepsen.util 14291 :invoke :read [23 nil] INFO [2019-12-17 12:38:40,144] jepsen worker 19 - jepsen.util 14239 :fail :cas [6 [7 30]] :ValueNotSwapFrom INFO [2019-12-17 12:38:40,144] jepsen worker 19 - jepsen.util 14239 :invoke :cas [6 [9 19]] INFO [2019-12-17 12:38:40,179] jepsen worker 47 - jepsen.util 14717 :ok :read [15 0] INFO [2019-12-17 12:38:40,179] jepsen worker 47 - jepsen.util 14717 :invoke :read [15 nil] INFO [2019-12-17 12:38:40,180] jepsen worker 64 - jepsen.util 10594 :ok :read [21 19] INFO [2019-12-17 12:38:40,180] jepsen worker 64 - jepsen.util 10594 :invoke :read [21 nil] INFO [2019-12-17 12:38:40,187] jepsen worker 19 - jepsen.util 14239 :fail :cas [6 [9 19]] :ValueNotSwapFrom INFO [2019-12-17 12:38:40,187] jepsen worker 19 - jepsen.util 14239 :invoke :read [6 nil] INFO [2019-12-17 12:38:40,197] jepsen worker 78 - jepsen.util 13398 :fail :cas [26 [32 13]] :ValueNotSwapFrom INFO [2019-12-17 12:38:40,197] jepsen worker 78 - jepsen.util 13398 :invoke :cas [26 [46 32]] INFO [2019-12-17 12:38:40,209] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77795,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:38:40,209] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77795,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:38:40,209] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77797,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:38:40,209] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77797,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:38:40,219] jepsen worker 34 - jepsen.util 19384 :info :write [11 33] :RequestCanceledException INFO [2019-12-17 12:38:40,221] jepsen worker 34 - jepsen.util 19474 :invoke :cas [11 [2 2]] INFO [2019-12-17 12:38:40,221] jepsen worker 36 - jepsen.util 14076 :ok :read [12 20] INFO [2019-12-17 12:38:40,221] jepsen worker 36 - jepsen.util 14076 :invoke :cas [12 [31 42]] INFO [2019-12-17 12:38:40,235] jepsen worker 4 - jepsen.util 12604 :info :write [1 26] :RequestCanceledException INFO [2019-12-17 12:38:40,235] jepsen worker 28 - jepsen.util 11458 :fail :read [9 nil] :RequestTimeoutException INFO [2019-12-17 12:38:40,235] jepsen worker 28 - jepsen.util 11458 :invoke :read [9 nil] INFO [2019-12-17 12:38:40,236] jepsen worker 4 - jepsen.util 12694 :invoke :cas [1 [10 4]] INFO [2019-12-17 12:38:40,239] jepsen worker 64 - jepsen.util 10594 :ok :read [21 19] INFO [2019-12-17 12:38:40,239] jepsen worker 64 - jepsen.util 10594 :invoke :write [21 29] INFO [2019-12-17 12:38:40,255] jepsen worker 13 - jepsen.util 12973 :fail :cas [4 [16 5]] :ValueNotSwapFrom INFO [2019-12-17 12:38:40,255] jepsen worker 13 - jepsen.util 12973 :invoke :cas [4 [39 47]] INFO [2019-12-17 12:38:40,258] jepsen worker 37 - jepsen.util 12907 :ok :read [12 20] INFO [2019-12-17 12:38:40,258] jepsen worker 37 - jepsen.util 12907 :invoke :write [12 2] INFO [2019-12-17 12:38:40,277] jepsen worker 14 - jepsen.util 13064 :ok :read [4 4] INFO [2019-12-17 12:38:40,278] jepsen worker 14 - jepsen.util 13064 :invoke :read [4 nil] INFO [2019-12-17 12:38:40,282] jepsen worker 47 - jepsen.util 14717 :ok :read [15 0] INFO [2019-12-17 12:38:40,282] jepsen worker 47 - jepsen.util 14717 :invoke :cas [15 [35 9]] INFO [2019-12-17 12:38:40,293] jepsen worker 70 - jepsen.util 13840 :fail :cas [23 [9 6]] :ValueNotSwapFrom INFO [2019-12-17 12:38:40,294] jepsen worker 70 - jepsen.util 13840 :invoke :write [23 48] INFO [2019-12-17 12:38:40,295] jepsen worker 81 - jepsen.util 17811 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:38:40,295] jepsen worker 81 - jepsen.util 17811 :invoke :write [27 36] INFO [2019-12-17 12:38:40,302] jepsen worker 19 - jepsen.util 14239 :ok :read [6 46] INFO [2019-12-17 12:38:40,302] jepsen worker 19 - jepsen.util 14239 :invoke :read [6 nil] INFO [2019-12-17 12:38:40,318] jepsen worker 53 - jepsen.util 13193 :ok :read [17 42] INFO [2019-12-17 12:38:40,318] jepsen worker 53 - jepsen.util 13193 :invoke :read [17 nil] INFO [2019-12-17 12:38:40,321] jepsen worker 71 - jepsen.util 14291 :ok :read [23 18] INFO [2019-12-17 12:38:40,321] jepsen worker 71 - jepsen.util 14291 :invoke :write [23 49] INFO [2019-12-17 12:38:40,323] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77803,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:38:40,323] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77803,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:38:40,323] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77805,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:38:40,323] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77805,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:38:40,335] jepsen worker 35 - jepsen.util 18665 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:38:40,335] jepsen worker 61 - jepsen.util 19681 :info :write [20 41] :RequestCanceledException INFO [2019-12-17 12:38:40,336] jepsen worker 35 - jepsen.util 18665 :invoke :write [11 2] INFO [2019-12-17 12:38:40,337] jepsen worker 61 - jepsen.util 19771 :invoke :cas [20 [9 31]] INFO [2019-12-17 12:38:40,349] jepsen worker 26 - jepsen.util 19286 :info :write [8 48] :RequestCanceledException INFO [2019-12-17 12:38:40,350] jepsen worker 26 - jepsen.util 19376 :invoke :write [8 8] INFO [2019-12-17 12:38:40,356] jepsen worker 47 - jepsen.util 14717 :fail :cas [15 [35 9]] :ValueNotSwapFrom INFO [2019-12-17 12:38:40,356] jepsen worker 47 - jepsen.util 14717 :invoke :read [15 nil] INFO [2019-12-17 12:38:40,362] jepsen worker 83 - jepsen.util 18263 :info :write [27 32] :RequestCanceledException INFO [2019-12-17 12:38:40,363] jepsen worker 83 - jepsen.util 18353 :invoke :read [27 nil] INFO [2019-12-17 12:38:40,365] jepsen worker 19 - jepsen.util 14239 :ok :read [6 46] INFO [2019-12-17 12:38:40,365] jepsen worker 19 - jepsen.util 14239 :invoke :write [6 0] INFO [2019-12-17 12:38:40,376] jepsen worker 47 - jepsen.util 14717 :ok :read [15 0] INFO [2019-12-17 12:38:40,376] jepsen worker 47 - jepsen.util 14717 :invoke :read [15 nil] INFO [2019-12-17 12:38:40,389] jepsen worker 64 - jepsen.util 10594 :ok :write [21 29] INFO [2019-12-17 12:38:40,389] jepsen worker 64 - jepsen.util 10594 :invoke :write [21 44] INFO [2019-12-17 12:38:40,406] jepsen worker 53 - jepsen.util 13193 :ok :read [17 42] INFO [2019-12-17 12:38:40,407] jepsen worker 53 - jepsen.util 13193 :invoke :write [17 23] INFO [2019-12-17 12:38:40,437] jepsen worker 88 - jepsen.util 13318 :ok :read [29 34] INFO [2019-12-17 12:38:40,437] jepsen worker 88 - jepsen.util 13318 :invoke :read [29 nil] INFO [2019-12-17 12:38:40,478] jepsen worker 36 - jepsen.util 14076 :fail :cas [12 [31 42]] :ValueNotSwapFrom INFO [2019-12-17 12:38:40,479] jepsen worker 36 - jepsen.util 14076 :invoke :cas [12 [22 49]] INFO [2019-12-17 12:38:40,486] jepsen worker 64 - jepsen.util 10594 :ok :write [21 44] INFO [2019-12-17 12:38:40,486] jepsen worker 64 - jepsen.util 10594 :invoke :cas [21 [36 1]] INFO [2019-12-17 12:38:40,508] jepsen worker 13 - jepsen.util 12973 :fail :cas [4 [39 47]] :ValueNotSwapFrom INFO [2019-12-17 12:38:40,508] jepsen worker 13 - jepsen.util 12973 :invoke :read [4 nil] INFO [2019-12-17 12:38:40,512] jepsen worker 72 - jepsen.util 13482 :ok :write [24 15] INFO [2019-12-17 12:38:40,513] jepsen worker 72 - jepsen.util 13482 :invoke :write [24 45] INFO [2019-12-17 12:38:40,515] jepsen worker 64 - jepsen.util 10594 :fail :cas [21 [36 1]] :ValueNotSwapFrom INFO [2019-12-17 12:38:40,515] jepsen worker 64 - jepsen.util 10594 :invoke :cas [21 [29 23]] INFO [2019-12-17 12:38:40,518] jepsen worker 79 - jepsen.util 13669 :ok :write [26 46] INFO [2019-12-17 12:38:40,518] jepsen worker 79 - jepsen.util 13669 :invoke :read [26 nil] INFO [2019-12-17 12:38:40,542] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77831,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:38:40,542] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77831,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:38:40,542] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77835,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:38:40,542] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77835,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:38:40,550] jepsen worker 12 - jepsen.util 13512 :ok :read [4 4] INFO [2019-12-17 12:38:40,550] jepsen worker 12 - jepsen.util 13512 :invoke :read [4 nil] INFO [2019-12-17 12:38:40,565] jepsen worker 32 - jepsen.util 13622 :info :cas [10 [48 18]] :RequestCanceledException INFO [2019-12-17 12:38:40,566] jepsen worker 32 - jepsen.util 13712 :invoke :write [10 25] INFO [2019-12-17 12:38:40,567] jepsen worker 64 - jepsen.util 10594 :fail :cas [21 [29 23]] :ValueNotSwapFrom INFO [2019-12-17 12:38:40,567] jepsen worker 64 - jepsen.util 10594 :invoke :read [21 nil] INFO [2019-12-17 12:38:40,596] jepsen worker 88 - jepsen.util 13318 :ok :read [29 18] INFO [2019-12-17 12:38:40,596] jepsen worker 88 - jepsen.util 13318 :invoke :write [29 0] INFO [2019-12-17 12:38:40,601] jepsen worker 64 - jepsen.util 10594 :ok :read [21 44] INFO [2019-12-17 12:38:40,602] jepsen worker 64 - jepsen.util 10594 :invoke :cas [21 [5 0]] INFO [2019-12-17 12:38:40,608] jepsen worker 36 - jepsen.util 14076 :fail :cas [12 [22 49]] :ValueNotSwapFrom INFO [2019-12-17 12:38:40,608] jepsen worker 36 - jepsen.util 14076 :invoke :write [12 42] INFO [2019-12-17 12:38:40,609] jepsen worker 27 - jepsen.util 12357 :ok :write [9 4] INFO [2019-12-17 12:38:40,609] jepsen worker 27 - jepsen.util 12357 :invoke :read [9 nil] INFO [2019-12-17 12:38:40,619] jepsen worker 54 - jepsen.util 14094 :info :write [18 45] :RequestCanceledException INFO [2019-12-17 12:38:40,620] jepsen worker 54 - jepsen.util 14184 :invoke :cas [18 [22 9]] INFO [2019-12-17 12:38:40,652] jepsen worker 13 - jepsen.util 12973 :ok :read [4 4] INFO [2019-12-17 12:38:40,652] jepsen worker 13 - jepsen.util 12973 :invoke :read [4 nil] INFO [2019-12-17 12:38:40,656] jepsen worker 64 - jepsen.util 10594 :fail :cas [21 [5 0]] :ValueNotSwapFrom INFO [2019-12-17 12:38:40,656] jepsen worker 64 - jepsen.util 10594 :invoke :write [21 20] INFO [2019-12-17 12:38:40,711] jepsen worker 53 - jepsen.util 13193 :ok :write [17 23] INFO [2019-12-17 12:38:40,711] jepsen worker 53 - jepsen.util 13193 :invoke :write [17 41] INFO [2019-12-17 12:38:40,715] jepsen worker 37 - jepsen.util 12907 :ok :write [12 2] INFO [2019-12-17 12:38:40,715] jepsen worker 37 - jepsen.util 12907 :invoke :read [12 nil] INFO [2019-12-17 12:38:40,718] jepsen worker 55 - jepsen.util 14095 :ok :write [18 30] INFO [2019-12-17 12:38:40,718] jepsen worker 55 - jepsen.util 14095 :invoke :write [18 11] INFO [2019-12-17 12:38:40,751] jepsen worker 88 - jepsen.util 13318 :ok :write [29 0] INFO [2019-12-17 12:38:40,751] jepsen worker 88 - jepsen.util 13318 :invoke :read [29 nil] INFO [2019-12-17 12:38:40,756] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77852,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:38:40,756] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77852,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:38:40,756] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77856,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:38:40,756] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77856,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:38:40,781] jepsen worker 14 - jepsen.util 13064 :ok :read [4 4] INFO [2019-12-17 12:38:40,781] jepsen worker 14 - jepsen.util 13064 :invoke :read [4 nil] INFO [2019-12-17 12:38:40,782] jepsen worker 64 - jepsen.util 10594 :ok :write [21 20] INFO [2019-12-17 12:38:40,782] jepsen worker 64 - jepsen.util 10594 :invoke :write [21 3] INFO [2019-12-17 12:38:40,838] jepsen worker 64 - jepsen.util 10594 :ok :write [21 3] INFO [2019-12-17 12:38:40,838] jepsen worker 64 - jepsen.util 10594 :invoke :cas [21 [47 29]] INFO [2019-12-17 12:38:40,848] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:38:40,848] jepsen nemesis - couchbase.util Rebalance check count: 612 INFO [2019-12-17 12:38:40,848] jepsen nemesis - couchbase.util Rebalance stuck count: 593 INFO [2019-12-17 12:38:40,857] jepsen worker 79 - jepsen.util 13669 :ok :read [26 46] INFO [2019-12-17 12:38:40,857] jepsen worker 79 - jepsen.util 13669 :invoke :cas [26 [31 7]] INFO [2019-12-17 12:38:40,860] jepsen worker 64 - jepsen.util 10594 :fail :cas [21 [47 29]] :ValueNotSwapFrom INFO [2019-12-17 12:38:40,860] jepsen worker 64 - jepsen.util 10594 :invoke :write [21 25] INFO [2019-12-17 12:38:40,873] jepsen worker 13 - jepsen.util 12973 :ok :read [4 4] INFO [2019-12-17 12:38:40,873] jepsen worker 13 - jepsen.util 12973 :invoke :write [4 2] INFO [2019-12-17 12:38:40,905] jepsen worker 64 - jepsen.util 10594 :ok :write [21 25] INFO [2019-12-17 12:38:40,905] jepsen worker 64 - jepsen.util 10594 :invoke :cas [21 [3 28]] INFO [2019-12-17 12:38:40,908] jepsen worker 54 - jepsen.util 14184 :fail :cas [18 [22 9]] :ValueNotSwapFrom INFO [2019-12-17 12:38:40,908] jepsen worker 54 - jepsen.util 14184 :invoke :cas [18 [7 38]] INFO [2019-12-17 12:38:40,933] jepsen worker 64 - jepsen.util 10594 :fail :cas [21 [3 28]] :ValueNotSwapFrom INFO [2019-12-17 12:38:40,933] jepsen worker 64 - jepsen.util 10594 :invoke :read [21 nil] INFO [2019-12-17 12:38:40,937] jepsen worker 37 - jepsen.util 12907 :ok :read [12 2] INFO [2019-12-17 12:38:40,937] jepsen worker 37 - jepsen.util 12907 :invoke :cas [12 [29 37]] INFO [2019-12-17 12:38:40,942] jepsen worker 45 - jepsen.util 13005 :ok :write [15 23] INFO [2019-12-17 12:38:40,942] jepsen worker 45 - jepsen.util 13005 :invoke :read [15 nil] INFO [2019-12-17 12:38:40,975] jepsen worker 10 - jepsen.util 13330 :ok :write [3 16] INFO [2019-12-17 12:38:40,975] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77872,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:38:40,975] jepsen worker 10 - jepsen.util 13330 :invoke :cas [3 [44 17]] INFO [2019-12-17 12:38:40,975] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77872,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:38:40,978] jepsen worker 64 - jepsen.util 10594 :ok :read [21 25] INFO [2019-12-17 12:38:40,978] jepsen worker 64 - jepsen.util 10594 :invoke :write [21 12] INFO [2019-12-17 12:38:40,978] jepsen worker 14 - jepsen.util 13064 :ok :read [4 4] INFO [2019-12-17 12:38:40,979] jepsen worker 14 - jepsen.util 13064 :invoke :write [4 3] INFO [2019-12-17 12:38:40,999] jepsen worker 69 - jepsen.util 13839 :ok :write [23 19] INFO [2019-12-17 12:38:41,000] jepsen worker 69 - jepsen.util 13839 :invoke :read [23 nil] INFO [2019-12-17 12:38:41,009] jepsen worker 28 - jepsen.util 11458 :ok :read [9 4] INFO [2019-12-17 12:38:41,009] jepsen worker 28 - jepsen.util 11458 :invoke :read [9 nil] INFO [2019-12-17 12:38:41,018] jepsen worker 45 - jepsen.util 13005 :ok :read [15 23] INFO [2019-12-17 12:38:41,018] jepsen worker 45 - jepsen.util 13005 :invoke :cas [15 [38 25]] INFO [2019-12-17 12:38:41,019] jepsen worker 79 - jepsen.util 13669 :fail :cas [26 [31 7]] :ValueNotSwapFrom INFO [2019-12-17 12:38:41,019] jepsen worker 79 - jepsen.util 13669 :invoke :cas [26 [5 27]] INFO [2019-12-17 12:38:41,035] jepsen worker 3 - jepsen.util 13053 :info :write [1 15] :RequestCanceledException INFO [2019-12-17 12:38:41,036] jepsen worker 51 - jepsen.util 12741 :info :cas [17 [27 0]] :RequestCanceledException INFO [2019-12-17 12:38:41,037] jepsen worker 3 - jepsen.util 13143 :invoke :cas [1 [3 27]] INFO [2019-12-17 12:38:41,037] jepsen worker 51 - jepsen.util 12831 :invoke :cas [17 [24 39]] INFO [2019-12-17 12:38:41,039] jepsen worker 52 - jepsen.util 13012 :ok :write [17 27] INFO [2019-12-17 12:38:41,039] jepsen worker 52 - jepsen.util 13012 :invoke :write [17 44] INFO [2019-12-17 12:38:41,041] jepsen worker 54 - jepsen.util 14184 :fail :cas [18 [7 38]] :ValueNotSwapFrom INFO [2019-12-17 12:38:41,041] jepsen worker 54 - jepsen.util 14184 :invoke :write [18 18] INFO [2019-12-17 12:38:41,046] jepsen worker 37 - jepsen.util 12907 :fail :cas [12 [29 37]] :ValueNotSwapFrom INFO [2019-12-17 12:38:41,046] jepsen worker 37 - jepsen.util 12907 :invoke :write [12 32] INFO [2019-12-17 12:38:41,059] jepsen worker 10 - jepsen.util 13330 :fail :cas [3 [44 17]] :ValueNotSwapFrom INFO [2019-12-17 12:38:41,059] jepsen worker 10 - jepsen.util 13330 :invoke :write [3 16] INFO [2019-12-17 12:38:41,063] jepsen worker 38 - jepsen.util 14078 :ok :read [12 2] INFO [2019-12-17 12:38:41,063] jepsen worker 38 - jepsen.util 14078 :invoke :write [12 34] INFO [2019-12-17 12:38:41,084] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77884,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:38:41,085] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77884,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:38:41,085] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77889,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:38:41,085] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77889,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:38:41,093] jepsen worker 28 - jepsen.util 11458 :ok :read [9 4] INFO [2019-12-17 12:38:41,093] jepsen worker 28 - jepsen.util 11458 :invoke :read [9 nil] INFO [2019-12-17 12:38:41,110] jepsen worker 45 - jepsen.util 13005 :fail :cas [15 [38 25]] :ValueNotSwapFrom INFO [2019-12-17 12:38:41,110] jepsen worker 45 - jepsen.util 13005 :invoke :cas [15 [2 8]] INFO [2019-12-17 12:38:41,114] jepsen worker 53 - jepsen.util 13193 :ok :write [17 41] INFO [2019-12-17 12:38:41,114] jepsen worker 53 - jepsen.util 13193 :invoke :read [17 nil] INFO [2019-12-17 12:38:41,130] jepsen worker 45 - jepsen.util 13005 :fail :cas [15 [2 8]] :ValueNotSwapFrom INFO [2019-12-17 12:38:41,130] jepsen worker 45 - jepsen.util 13005 :invoke :cas [15 [10 43]] INFO [2019-12-17 12:38:41,135] jepsen worker 20 - jepsen.util 13160 :info :write [6 10] :RequestCanceledException INFO [2019-12-17 12:38:41,136] jepsen worker 20 - jepsen.util 13250 :invoke :read [6 nil] INFO [2019-12-17 12:38:41,151] jepsen worker 56 - jepsen.util 14186 :ok :write [18 36] INFO [2019-12-17 12:38:41,152] jepsen worker 56 - jepsen.util 14186 :invoke :cas [18 [44 13]] INFO [2019-12-17 12:38:41,152] jepsen worker 36 - jepsen.util 14076 :ok :write [12 42] INFO [2019-12-17 12:38:41,152] jepsen worker 36 - jepsen.util 14076 :invoke :write [12 31] INFO [2019-12-17 12:38:41,153] jepsen worker 5 - jepsen.util 13145 :ok :read [1 12] INFO [2019-12-17 12:38:41,153] jepsen worker 5 - jepsen.util 13145 :invoke :cas [1 [19 21]] INFO [2019-12-17 12:38:41,163] jepsen worker 64 - jepsen.util 10594 :ok :write [21 12] INFO [2019-12-17 12:38:41,163] jepsen worker 64 - jepsen.util 10594 :invoke :write [21 12] INFO [2019-12-17 12:38:41,173] jepsen worker 19 - jepsen.util 14239 :ok :write [6 0] INFO [2019-12-17 12:38:41,173] jepsen worker 19 - jepsen.util 14239 :invoke :write [6 7] INFO [2019-12-17 12:38:41,179] jepsen worker 45 - jepsen.util 13005 :fail :cas [15 [10 43]] :ValueNotSwapFrom INFO [2019-12-17 12:38:41,179] jepsen worker 45 - jepsen.util 13005 :invoke :read [15 nil] INFO [2019-12-17 12:38:41,200] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77912,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:38:41,200] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77912,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:38:41,230] jepsen worker 14 - jepsen.util 13064 :ok :write [4 3] INFO [2019-12-17 12:38:41,231] jepsen worker 53 - jepsen.util 13193 :ok :read [17 41] INFO [2019-12-17 12:38:41,231] jepsen worker 14 - jepsen.util 13064 :invoke :cas [4 [34 42]] INFO [2019-12-17 12:38:41,231] jepsen worker 53 - jepsen.util 13193 :invoke :cas [17 [18 19]] INFO [2019-12-17 12:38:41,231] jepsen worker 4 - jepsen.util 12694 :fail :cas [1 [10 4]] :ValueNotSwapFrom INFO [2019-12-17 12:38:41,231] jepsen worker 4 - jepsen.util 12694 :invoke :write [1 13] INFO [2019-12-17 12:38:41,235] jepsen worker 80 - jepsen.util 12860 :fail :read [26 nil] :RequestTimeoutException INFO [2019-12-17 12:38:41,235] jepsen worker 80 - jepsen.util 12860 :invoke :read [26 nil] INFO [2019-12-17 12:38:41,255] jepsen worker 20 - jepsen.util 13250 :ok :read [6 46] INFO [2019-12-17 12:38:41,255] jepsen worker 20 - jepsen.util 13250 :invoke :cas [6 [17 41]] INFO [2019-12-17 12:38:41,269] jepsen worker 45 - jepsen.util 13005 :ok :read [15 23] INFO [2019-12-17 12:38:41,269] jepsen worker 45 - jepsen.util 13005 :invoke :write [15 39] INFO [2019-12-17 12:38:41,296] jepsen worker 53 - jepsen.util 13193 :fail :cas [17 [18 19]] :ValueNotSwapFrom INFO [2019-12-17 12:38:41,296] jepsen worker 53 - jepsen.util 13193 :invoke :cas [17 [1 4]] INFO [2019-12-17 12:38:41,309] jepsen worker 88 - jepsen.util 13318 :ok :read [29 33] INFO [2019-12-17 12:38:41,309] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77918,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:38:41,310] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77918,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:38:41,310] jepsen worker 88 - jepsen.util 13318 :invoke :cas [29 [24 19]] INFO [2019-12-17 12:38:41,310] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77922,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:38:41,310] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77922,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:38:41,310] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77923,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:38:41,310] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77923,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:38:41,316] jepsen worker 72 - jepsen.util 13482 :ok :write [24 45] INFO [2019-12-17 12:38:41,316] jepsen worker 72 - jepsen.util 13482 :invoke :write [24 48] INFO [2019-12-17 12:38:41,317] jepsen worker 87 - jepsen.util 13137 :ok :write [29 33] INFO [2019-12-17 12:38:41,318] jepsen worker 87 - jepsen.util 13137 :invoke :read [29 nil] INFO [2019-12-17 12:38:41,351] jepsen worker 79 - jepsen.util 13669 :fail :cas [26 [5 27]] :ValueNotSwapFrom INFO [2019-12-17 12:38:41,351] jepsen worker 79 - jepsen.util 13669 :invoke :cas [26 [36 20]] INFO [2019-12-17 12:38:41,357] jepsen worker 64 - jepsen.util 10594 :ok :write [21 12] INFO [2019-12-17 12:38:41,358] jepsen worker 64 - jepsen.util 10594 :invoke :write [21 31] INFO [2019-12-17 12:38:41,382] jepsen worker 53 - jepsen.util 13193 :fail :cas [17 [1 4]] :ValueNotSwapFrom INFO [2019-12-17 12:38:41,382] jepsen worker 53 - jepsen.util 13193 :invoke :write [17 29] INFO [2019-12-17 12:38:41,403] jepsen worker 10 - jepsen.util 13330 :ok :write [3 16] INFO [2019-12-17 12:38:41,403] jepsen worker 10 - jepsen.util 13330 :invoke :write [3 41] INFO [2019-12-17 12:38:41,405] jepsen worker 12 - jepsen.util 13512 :ok :read [4 3] INFO [2019-12-17 12:38:41,405] jepsen worker 12 - jepsen.util 13512 :invoke :read [4 nil] INFO [2019-12-17 12:38:41,457] jepsen worker 70 - jepsen.util 13840 :ok :write [23 48] INFO [2019-12-17 12:38:41,457] jepsen worker 70 - jepsen.util 13840 :invoke :cas [23 [47 48]] INFO [2019-12-17 12:38:41,461] jepsen worker 64 - jepsen.util 10594 :ok :write [21 31] INFO [2019-12-17 12:38:41,461] jepsen worker 64 - jepsen.util 10594 :invoke :cas [21 [25 19]] INFO [2019-12-17 12:38:41,477] jepsen worker 37 - jepsen.util 12907 :ok :write [12 32] INFO [2019-12-17 12:38:41,477] jepsen worker 37 - jepsen.util 12907 :invoke :read [12 nil] INFO [2019-12-17 12:38:41,479] jepsen worker 64 - jepsen.util 10594 :fail :cas [21 [25 19]] :ValueNotSwapFrom INFO [2019-12-17 12:38:41,479] jepsen worker 64 - jepsen.util 10594 :invoke :write [21 7] INFO [2019-12-17 12:38:41,480] jepsen worker 55 - jepsen.util 14095 :ok :write [18 11] INFO [2019-12-17 12:38:41,480] jepsen worker 55 - jepsen.util 14095 :invoke :cas [18 [13 36]] INFO [2019-12-17 12:38:41,514] jepsen worker 14 - jepsen.util 13064 :fail :cas [4 [34 42]] :ValueNotSwapFrom INFO [2019-12-17 12:38:41,514] jepsen worker 14 - jepsen.util 13064 :invoke :read [4 nil] INFO [2019-12-17 12:38:41,514] jepsen worker 87 - jepsen.util 13137 :ok :read [29 33] INFO [2019-12-17 12:38:41,514] jepsen worker 87 - jepsen.util 13137 :invoke :cas [29 [9 48]] INFO [2019-12-17 12:38:41,519] jepsen worker 11 - jepsen.util 13241 :fail :read [3 nil] :RequestTimeoutException INFO [2019-12-17 12:38:41,519] jepsen worker 11 - jepsen.util 13241 :invoke :cas [3 [34 12]] INFO [2019-12-17 12:38:41,526] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77940,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:38:41,526] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77940,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:38:41,538] jepsen worker 64 - jepsen.util 10594 :ok :write [21 7] INFO [2019-12-17 12:38:41,538] jepsen worker 64 - jepsen.util 10594 :invoke :cas [21 [28 3]] INFO [2019-12-17 12:38:41,542] jepsen worker 12 - jepsen.util 13512 :ok :read [4 3] INFO [2019-12-17 12:38:41,543] jepsen worker 12 - jepsen.util 13512 :invoke :cas [4 [36 40]] INFO [2019-12-17 12:38:41,546] jepsen worker 19 - jepsen.util 14239 :ok :write [6 7] INFO [2019-12-17 12:38:41,546] jepsen worker 19 - jepsen.util 14239 :invoke :write [6 0] INFO [2019-12-17 12:38:41,546] jepsen worker 45 - jepsen.util 13005 :ok :write [15 39] INFO [2019-12-17 12:38:41,546] jepsen worker 45 - jepsen.util 13005 :invoke :write [15 13] INFO [2019-12-17 12:38:41,555] jepsen worker 56 - jepsen.util 14186 :fail :cas [18 [44 13]] :ValueNotSwapFrom INFO [2019-12-17 12:38:41,555] jepsen worker 56 - jepsen.util 14186 :invoke :cas [18 [47 18]] INFO [2019-12-17 12:38:41,577] jepsen worker 11 - jepsen.util 13241 :fail :cas [3 [34 12]] :ValueNotSwapFrom INFO [2019-12-17 12:38:41,577] jepsen worker 11 - jepsen.util 13241 :invoke :write [3 45] INFO [2019-12-17 12:38:41,584] jepsen worker 88 - jepsen.util 13318 :fail :cas [29 [24 19]] :ValueNotSwapFrom INFO [2019-12-17 12:38:41,584] jepsen worker 88 - jepsen.util 13318 :invoke :write [29 4] INFO [2019-12-17 12:38:41,592] jepsen worker 79 - jepsen.util 13669 :fail :cas [26 [36 20]] :ValueNotSwapFrom INFO [2019-12-17 12:38:41,592] jepsen worker 79 - jepsen.util 13669 :invoke :write [26 44] INFO [2019-12-17 12:38:41,601] jepsen worker 65 - jepsen.util 11675 :info :write [21 2] :RequestCanceledException INFO [2019-12-17 12:38:41,602] jepsen worker 65 - jepsen.util 11765 :invoke :read [21 nil] INFO [2019-12-17 12:38:41,635] jepsen worker 9 - jepsen.util 13959 :fail :read [3 nil] :RequestTimeoutException INFO [2019-12-17 12:38:41,635] jepsen worker 89 - jepsen.util 13499 :info :write [29 24] :RequestCanceledException INFO [2019-12-17 12:38:41,635] jepsen worker 9 - jepsen.util 13959 :invoke :write [3 31] INFO [2019-12-17 12:38:41,636] jepsen worker 89 - jepsen.util 13589 :invoke :cas [29 [32 37]] INFO [2019-12-17 12:38:41,646] jepsen worker 78 - jepsen.util 13398 :ok :cas [26 [46 32]] INFO [2019-12-17 12:38:41,646] jepsen worker 78 - jepsen.util 13398 :invoke :read [26 nil] INFO [2019-12-17 12:38:41,649] jepsen worker 64 - jepsen.util 10594 :fail :cas [21 [28 3]] :ValueNotSwapFrom INFO [2019-12-17 12:38:41,649] jepsen worker 64 - jepsen.util 10594 :invoke :write [21 9] INFO [2019-12-17 12:38:41,663] jepsen worker 87 - jepsen.util 13137 :fail :cas [29 [9 48]] :ValueNotSwapFrom INFO [2019-12-17 12:38:41,664] jepsen worker 87 - jepsen.util 13137 :invoke :cas [29 [32 38]] INFO [2019-12-17 12:38:41,666] jepsen worker 69 - jepsen.util 13839 :ok :read [23 48] INFO [2019-12-17 12:38:41,666] jepsen worker 69 - jepsen.util 13839 :invoke :cas [23 [40 25]] INFO [2019-12-17 12:38:41,718] jepsen worker 14 - jepsen.util 13064 :ok :read [4 3] INFO [2019-12-17 12:38:41,718] jepsen worker 14 - jepsen.util 13064 :invoke :cas [4 [14 19]] INFO [2019-12-17 12:38:41,723] jepsen worker 64 - jepsen.util 10594 :ok :write [21 9] INFO [2019-12-17 12:38:41,723] jepsen worker 64 - jepsen.util 10594 :invoke :cas [21 [1 11]] INFO [2019-12-17 12:38:41,724] jepsen worker 51 - jepsen.util 12831 :fail :cas [17 [24 39]] :ValueNotSwapFrom INFO [2019-12-17 12:38:41,724] jepsen worker 51 - jepsen.util 12831 :invoke :read [17 nil] INFO [2019-12-17 12:38:41,735] jepsen worker 63 - jepsen.util 11763 :info :write [21 27] :RequestCanceledException INFO [2019-12-17 12:38:41,736] jepsen worker 64 - jepsen.util 10594 :fail :cas [21 [1 11]] :ValueNotSwapFrom INFO [2019-12-17 12:38:41,736] jepsen worker 64 - jepsen.util 10594 :invoke :write [21 30] INFO [2019-12-17 12:38:41,739] jepsen worker 63 - jepsen.util 11853 :invoke :read [21 nil] INFO [2019-12-17 12:38:41,748] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77965,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:38:41,749] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77965,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:38:41,749] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77969,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:38:41,749] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":77969,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:38:41,754] jepsen worker 12 - jepsen.util 13512 :fail :cas [4 [36 40]] :ValueNotSwapFrom INFO [2019-12-17 12:38:41,754] jepsen worker 12 - jepsen.util 13512 :invoke :cas [4 [18 11]] INFO [2019-12-17 12:38:41,755] jepsen worker 63 - jepsen.util 11853 :ok :read [21 9] INFO [2019-12-17 12:38:41,755] jepsen worker 63 - jepsen.util 11853 :invoke :write [21 10] INFO [2019-12-17 12:38:41,777] jepsen worker 3 - jepsen.util 13143 :fail :cas [1 [3 27]] :ValueNotSwapFrom INFO [2019-12-17 12:38:41,777] jepsen worker 3 - jepsen.util 13143 :invoke :cas [1 [37 32]] INFO [2019-12-17 12:38:41,780] jepsen worker 37 - jepsen.util 12907 :ok :read [12 32] INFO [2019-12-17 12:38:41,780] jepsen worker 37 - jepsen.util 12907 :invoke :cas [12 [46 47]] INFO [2019-12-17 12:38:41,795] jepsen worker 64 - jepsen.util 10594 :ok :write [21 30] INFO [2019-12-17 12:38:41,795] jepsen worker 64 - jepsen.util 10594 :invoke :write [21 0] INFO [2019-12-17 12:38:41,802] jepsen worker 67 - jepsen.util 18787 :info :cas [22 [39 0]] :RequestCanceledException INFO [2019-12-17 12:38:41,802] jepsen worker 24 - jepsen.util 17574 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:38:41,802] jepsen worker 24 - jepsen.util 17574 :invoke :write [8 32] INFO [2019-12-17 12:38:41,803] jepsen worker 67 - jepsen.util 18877 :invoke :write [22 34] INFO [2019-12-17 12:38:41,804] jepsen worker 80 - jepsen.util 12860 :ok :read [26 32] INFO [2019-12-17 12:38:41,804] jepsen worker 80 - jepsen.util 12860 :invoke :read [26 nil] INFO [2019-12-17 12:38:41,819] jepsen worker 59 - jepsen.util 17519 :info :write [19 27] :RequestCanceledException INFO [2019-12-17 12:38:41,819] jepsen worker 2 - jepsen.util 17732 :info :cas [0 [11 43]] :RequestCanceledException INFO [2019-12-17 12:38:41,820] jepsen worker 2 - jepsen.util 17822 :invoke :write [0 33] INFO [2019-12-17 12:38:41,820] jepsen worker 59 - jepsen.util 17609 :invoke :write [19 30] INFO [2019-12-17 12:38:41,828] jepsen worker 10 - jepsen.util 13330 :ok :write [3 41] INFO [2019-12-17 12:38:41,828] jepsen worker 72 - jepsen.util 13482 :ok :write [24 48] INFO [2019-12-17 12:38:41,828] jepsen worker 10 - jepsen.util 13330 :invoke :read [3 nil] INFO [2019-12-17 12:38:41,828] jepsen worker 72 - jepsen.util 13482 :invoke :cas [24 [15 14]] INFO [2019-12-17 12:38:41,833] jepsen worker 87 - jepsen.util 13137 :fail :cas [29 [32 38]] :ValueNotSwapFrom INFO [2019-12-17 12:38:41,833] jepsen worker 87 - jepsen.util 13137 :invoke :write [29 16] INFO [2019-12-17 12:38:41,835] jepsen worker 76 - jepsen.util 18706 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:38:41,835] jepsen worker 76 - jepsen.util 18706 :invoke :read [25 nil] INFO [2019-12-17 12:38:41,835] jepsen worker 7 - jepsen.util 18277 :info :cas [2 [39 43]] :RequestCanceledException INFO [2019-12-17 12:38:41,835] jepsen worker 16 - jepsen.util 20086 :info :write [5 6] :RequestCanceledException INFO [2019-12-17 12:38:41,836] jepsen worker 16 - jepsen.util 20176 :invoke :cas [5 [31 6]] INFO [2019-12-17 12:38:41,836] jepsen worker 7 - jepsen.util 18367 :invoke :cas [2 [36 35]] INFO [2019-12-17 12:38:41,850] jepsen worker 85 - jepsen.util 18895 :info :cas [28 [6 49]] :RequestCanceledException INFO [2019-12-17 12:38:41,850] jepsen worker 57 - jepsen.util 17967 :info :write [19 16] :RequestCanceledException INFO [2019-12-17 12:38:41,850] jepsen worker 40 - jepsen.util 18940 :info :cas [13 [8 2]] :RequestCanceledException INFO [2019-12-17 12:38:41,851] jepsen worker 40 - jepsen.util 19030 :invoke :cas [13 [20 26]] INFO [2019-12-17 12:38:41,851] jepsen worker 85 - jepsen.util 18985 :invoke :cas [28 [14 49]] INFO [2019-12-17 12:38:41,851] jepsen worker 57 - jepsen.util 18057 :invoke :write [19 2] INFO [2019-12-17 12:38:41,864] jepsen worker 70 - jepsen.util 13840 :fail :cas [23 [47 48]] :ValueNotSwapFrom INFO [2019-12-17 12:38:41,865] jepsen worker 70 - jepsen.util 13840 :invoke :cas [23 [38 24]] INFO [2019-12-17 12:38:41,866] jepsen worker 22 - jepsen.util 18832 :info :write [7 10] :RequestCanceledException INFO [2019-12-17 12:38:41,866] jepsen worker 49 - jepsen.util 18409 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:38:41,867] jepsen worker 49 - jepsen.util 18409 :invoke :read [16 nil] INFO [2019-12-17 12:38:41,867] jepsen worker 22 - jepsen.util 18922 :invoke :cas [7 [27 16]] INFO [2019-12-17 12:38:41,880] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:38:41,881] jepsen nemesis - couchbase.util Rebalance check count: 613 INFO [2019-12-17 12:38:41,881] jepsen nemesis - couchbase.util Rebalance stuck count: 594 INFO [2019-12-17 12:38:41,893] jepsen worker 10 - jepsen.util 13330 :ok :read [3 41] INFO [2019-12-17 12:38:41,893] jepsen worker 10 - jepsen.util 13330 :invoke :read [3 nil] INFO [2019-12-17 12:38:41,898] jepsen worker 72 - jepsen.util 13482 :fail :cas [24 [15 14]] :ValueNotSwapFrom INFO [2019-12-17 12:38:41,898] jepsen worker 72 - jepsen.util 13482 :invoke :write [24 10] INFO [2019-12-17 12:38:41,900] jepsen worker 84 - jepsen.util 17814 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:38:41,900] jepsen worker 84 - jepsen.util 17814 :invoke :cas [28 [17 44]] INFO [2019-12-17 12:38:41,900] jepsen worker 64 - jepsen.util 10594 :ok :write [21 0] INFO [2019-12-17 12:38:41,900] jepsen worker 64 - jepsen.util 10594 :invoke :cas [21 [29 12]] INFO [2019-12-17 12:38:41,906] jepsen worker 56 - jepsen.util 14186 :fail :cas [18 [47 18]] :ValueNotSwapFrom INFO [2019-12-17 12:38:41,906] jepsen worker 56 - jepsen.util 14186 :invoke :cas [18 [47 10]] INFO [2019-12-17 12:38:41,919] jepsen worker 21 - jepsen.util 18021 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:38:41,919] jepsen worker 66 - jepsen.util 18696 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:38:41,919] jepsen worker 58 - jepsen.util 18598 :info :write [19 3] :RequestCanceledException INFO [2019-12-17 12:38:41,919] jepsen worker 21 - jepsen.util 18021 :invoke :read [7 nil] INFO [2019-12-17 12:38:41,920] jepsen worker 66 - jepsen.util 18696 :invoke :cas [22 [10 33]] INFO [2019-12-17 12:38:41,921] jepsen worker 58 - jepsen.util 18688 :invoke :write [19 22] INFO [2019-12-17 12:38:41,926] jepsen worker 10 - jepsen.util 13330 :ok :read [3 41] INFO [2019-12-17 12:38:41,926] jepsen worker 10 - jepsen.util 13330 :invoke :read [3 nil] INFO [2019-12-17 12:38:41,933] jepsen worker 48 - jepsen.util 17058 :info :write [16 27] :RequestCanceledException INFO [2019-12-17 12:38:41,933] jepsen worker 33 - jepsen.util 18123 :info :write [11 43] :RequestCanceledException INFO [2019-12-17 12:38:41,933] jepsen worker 77 - jepsen.util 19517 :info :cas [25 [32 45]] :RequestCanceledException INFO [2019-12-17 12:38:41,933] jepsen worker 39 - jepsen.util 17859 :info :cas [13 [33 36]] :RequestCanceledException INFO [2019-12-17 12:38:41,933] jepsen worker 62 - jepsen.util 19052 :info :write [20 34] :RequestCanceledException INFO [2019-12-17 12:38:41,933] jepsen worker 31 - jepsen.util 11641 :info :cas [10 [32 43]] :RequestCanceledException INFO [2019-12-17 12:38:41,933] jepsen worker 17 - jepsen.util 18197 :info :write [5 5] :RequestCanceledException INFO [2019-12-17 12:38:41,933] jepsen worker 23 - jepsen.util 18653 :info :write [7 45] :RequestCanceledException INFO [2019-12-17 12:38:41,934] jepsen worker 62 - jepsen.util 19142 :invoke :read [20 nil] INFO [2019-12-17 12:38:41,934] jepsen worker 33 - jepsen.util 18213 :invoke :cas [11 [34 16]] INFO [2019-12-17 12:38:41,934] jepsen worker 31 - jepsen.util 11731 :invoke :read [10 nil] INFO [2019-12-17 12:38:41,934] jepsen worker 39 - jepsen.util 17949 :invoke :cas [13 [25 22]] INFO [2019-12-17 12:38:41,934] jepsen worker 48 - jepsen.util 17148 :invoke :read [16 nil] INFO [2019-12-17 12:38:41,935] jepsen worker 17 - jepsen.util 18287 :invoke :cas [5 [27 43]] INFO [2019-12-17 12:38:41,935] jepsen worker 23 - jepsen.util 18743 :invoke :write [7 11] INFO [2019-12-17 12:38:41,935] jepsen worker 77 - jepsen.util 19607 :invoke :cas [25 [36 2]] INFO [2019-12-17 12:38:41,948] jepsen worker 60 - jepsen.util 19500 :info :write [20 34] :RequestCanceledException INFO [2019-12-17 12:38:41,948] jepsen worker 82 - jepsen.util 18442 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:38:41,948] jepsen worker 86 - jepsen.util 19166 :info :cas [28 [29 20]] :RequestCanceledException INFO [2019-12-17 12:38:41,948] jepsen worker 15 - jepsen.util 17385 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:38:41,948] jepsen worker 0 - jepsen.util 18900 :info :cas [0 [3 37]] :RequestCanceledException INFO [2019-12-17 12:38:41,948] jepsen worker 82 - jepsen.util 18442 :invoke :cas [27 [18 47]] INFO [2019-12-17 12:38:41,948] jepsen worker 15 - jepsen.util 17385 :invoke :read [5 nil] INFO [2019-12-17 12:38:41,950] jepsen worker 86 - jepsen.util 19256 :invoke :cas [28 [31 46]] INFO [2019-12-17 12:38:41,950] jepsen worker 60 - jepsen.util 19590 :invoke :cas [20 [47 0]] INFO [2019-12-17 12:38:41,951] jepsen worker 0 - jepsen.util 18990 :invoke :read [0 nil] INFO [2019-12-17 12:38:41,962] jepsen worker 69 - jepsen.util 13839 :fail :cas [23 [40 25]] :ValueNotSwapFrom INFO [2019-12-17 12:38:41,963] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78002,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:38:41,963] jepsen worker 69 - jepsen.util 13839 :invoke :write [23 47] INFO [2019-12-17 12:38:41,963] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78002,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:38:41,963] jepsen worker 8 - jepsen.util 18008 :info :cas [2 [37 6]] :RequestCanceledException INFO [2019-12-17 12:38:41,963] jepsen worker 43 - jepsen.util 18583 :info :write [14 14] :RequestCanceledException INFO [2019-12-17 12:38:41,963] jepsen worker 42 - jepsen.util 19302 :info :cas [14 [46 36]] :RequestCanceledException INFO [2019-12-17 12:38:41,963] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78006,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:38:41,963] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78006,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:38:41,964] jepsen worker 8 - jepsen.util 18098 :invoke :read [2 nil] INFO [2019-12-17 12:38:41,964] jepsen worker 43 - jepsen.util 18673 :invoke :cas [14 [47 13]] INFO [2019-12-17 12:38:41,964] jepsen worker 42 - jepsen.util 19392 :invoke :cas [14 [42 22]] INFO [2019-12-17 12:38:41,977] jepsen worker 64 - jepsen.util 10594 :fail :cas [21 [29 12]] :ValueNotSwapFrom INFO [2019-12-17 12:38:41,977] jepsen worker 64 - jepsen.util 10594 :invoke :cas [21 [25 49]] INFO [2019-12-17 12:38:42,001] jepsen worker 68 - jepsen.util 17528 :info :cas [22 [21 32]] :RequestCanceledException INFO [2019-12-17 12:38:42,001] jepsen worker 25 - jepsen.util 19555 :info :cas [8 [42 10]] :RequestCanceledException INFO [2019-12-17 12:38:42,003] jepsen worker 25 - jepsen.util 19645 :invoke :cas [8 [26 28]] INFO [2019-12-17 12:38:42,003] jepsen worker 68 - jepsen.util 17618 :invoke :cas [22 [21 37]] INFO [2019-12-17 12:38:42,011] jepsen worker 10 - jepsen.util 13330 :ok :read [3 41] INFO [2019-12-17 12:38:42,011] jepsen worker 10 - jepsen.util 13330 :invoke :cas [3 [10 24]] INFO [2019-12-17 12:38:42,019] jepsen worker 50 - jepsen.util 18140 :info :cas [16 [44 46]] :RequestCanceledException INFO [2019-12-17 12:38:42,019] jepsen worker 41 - jepsen.util 19661 :info :cas [13 [9 44]] :RequestCanceledException INFO [2019-12-17 12:38:42,019] jepsen worker 44 - jepsen.util 19214 :info :write [14 8] :RequestCanceledException INFO [2019-12-17 12:38:42,021] jepsen worker 44 - jepsen.util 19304 :invoke :read [14 nil] INFO [2019-12-17 12:38:42,021] jepsen worker 50 - jepsen.util 18230 :invoke :cas [16 [21 23]] INFO [2019-12-17 12:38:42,021] jepsen worker 41 - jepsen.util 19751 :invoke :write [13 1] INFO [2019-12-17 12:38:42,027] jepsen worker 12 - jepsen.util 13512 :fail :cas [4 [18 11]] :ValueNotSwapFrom INFO [2019-12-17 12:38:42,027] jepsen worker 12 - jepsen.util 13512 :invoke :cas [4 [13 15]] INFO [2019-12-17 12:38:42,030] jepsen worker 89 - jepsen.util 13589 :fail :cas [29 [32 37]] :ValueNotSwapFrom INFO [2019-12-17 12:38:42,030] jepsen worker 89 - jepsen.util 13589 :invoke :write [29 39] INFO [2019-12-17 12:38:42,033] jepsen worker 64 - jepsen.util 10594 :fail :cas [21 [25 49]] :ValueNotSwapFrom INFO [2019-12-17 12:38:42,033] jepsen worker 64 - jepsen.util 10594 :invoke :write [21 11] INFO [2019-12-17 12:38:42,035] jepsen worker 75 - jepsen.util 18165 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:38:42,035] jepsen worker 74 - jepsen.util 13214 :info :cas [24 [22 1]] :RequestCanceledException INFO [2019-12-17 12:38:42,035] jepsen worker 73 - jepsen.util 12313 :info :write [24 27] :RequestCanceledException INFO [2019-12-17 12:38:42,035] jepsen worker 75 - jepsen.util 18165 :invoke :cas [25 [8 40]] INFO [2019-12-17 12:38:42,037] jepsen worker 74 - jepsen.util 13304 :invoke :write [24 47] INFO [2019-12-17 12:38:42,037] jepsen worker 73 - jepsen.util 12403 :invoke :read [24 nil] INFO [2019-12-17 12:38:42,053] jepsen worker 5 - jepsen.util 13145 :fail :cas [1 [19 21]] :ValueNotSwapFrom INFO [2019-12-17 12:38:42,053] jepsen worker 5 - jepsen.util 13145 :invoke :write [1 45] INFO [2019-12-17 12:38:42,064] jepsen worker 18 - jepsen.util 14148 :fail :read [6 nil] :RequestTimeoutException INFO [2019-12-17 12:38:42,064] jepsen worker 46 - jepsen.util 13006 :info :cas [15 [0 38]] :RequestCanceledException INFO [2019-12-17 12:38:42,064] jepsen worker 18 - jepsen.util 14148 :invoke :cas [6 [49 49]] INFO [2019-12-17 12:38:42,064] jepsen worker 3 - jepsen.util 13143 :fail :cas [1 [37 32]] :ValueNotSwapFrom INFO [2019-12-17 12:38:42,064] jepsen worker 3 - jepsen.util 13143 :invoke :read [1 nil] INFO [2019-12-17 12:38:42,065] jepsen worker 46 - jepsen.util 13096 :invoke :read [15 nil] INFO [2019-12-17 12:38:42,071] jepsen worker 31 - jepsen.util 11731 :ok :read [10 12] INFO [2019-12-17 12:38:42,071] jepsen worker 31 - jepsen.util 11731 :invoke :write [10 21] INFO [2019-12-17 12:38:42,073] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78037,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:38:42,073] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78037,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:38:42,073] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78041,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:38:42,073] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78041,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:38:42,075] jepsen worker 55 - jepsen.util 14095 :fail :cas [18 [13 36]] :ValueNotSwapFrom INFO [2019-12-17 12:38:42,075] jepsen worker 55 - jepsen.util 14095 :invoke :cas [18 [23 7]] INFO [2019-12-17 12:38:42,085] jepsen worker 80 - jepsen.util 12860 :ok :read [26 32] INFO [2019-12-17 12:38:42,085] jepsen worker 80 - jepsen.util 12860 :invoke :cas [26 [41 16]] INFO [2019-12-17 12:38:42,100] jepsen worker 78 - jepsen.util 13398 :ok :read [26 32] INFO [2019-12-17 12:38:42,100] jepsen worker 78 - jepsen.util 13398 :invoke :cas [26 [1 32]] INFO [2019-12-17 12:38:42,102] jepsen worker 1 - jepsen.util 18811 :info :cas [0 [18 7]] :RequestCanceledException INFO [2019-12-17 12:38:42,103] jepsen worker 1 - jepsen.util 18901 :invoke :cas [0 [40 6]] INFO [2019-12-17 12:38:42,110] jepsen worker 10 - jepsen.util 13330 :fail :cas [3 [10 24]] :ValueNotSwapFrom INFO [2019-12-17 12:38:42,110] jepsen worker 10 - jepsen.util 13330 :invoke :cas [3 [9 21]] INFO [2019-12-17 12:38:42,132] jepsen worker 73 - jepsen.util 12403 :ok :read [24 48] INFO [2019-12-17 12:38:42,132] jepsen worker 73 - jepsen.util 12403 :invoke :write [24 29] INFO [2019-12-17 12:38:42,135] jepsen worker 29 - jepsen.util 11639 :fail :read [9 nil] :RequestTimeoutException INFO [2019-12-17 12:38:42,135] jepsen worker 29 - jepsen.util 11639 :invoke :cas [9 [4 46]] INFO [2019-12-17 12:38:42,145] jepsen worker 70 - jepsen.util 13840 :fail :cas [23 [38 24]] :ValueNotSwapFrom INFO [2019-12-17 12:38:42,145] jepsen worker 70 - jepsen.util 13840 :invoke :cas [23 [27 8]] INFO [2019-12-17 12:38:42,152] jepsen worker 64 - jepsen.util 10594 :ok :write [21 11] INFO [2019-12-17 12:38:42,153] jepsen worker 64 - jepsen.util 10594 :invoke :write [21 31] INFO [2019-12-17 12:38:42,155] jepsen worker 51 - jepsen.util 12831 :ok :read [17 41] INFO [2019-12-17 12:38:42,155] jepsen worker 51 - jepsen.util 12831 :invoke :write [17 0] INFO [2019-12-17 12:38:42,166] jepsen worker 30 - jepsen.util 12900 :fail :read [10 nil] :RequestTimeoutException INFO [2019-12-17 12:38:42,166] jepsen worker 30 - jepsen.util 12900 :invoke :read [10 nil] INFO [2019-12-17 12:38:42,170] jepsen worker 56 - jepsen.util 14186 :fail :cas [18 [47 10]] :ValueNotSwapFrom INFO [2019-12-17 12:38:42,171] jepsen worker 56 - jepsen.util 14186 :invoke :write [18 43] INFO [2019-12-17 12:38:42,195] jepsen worker 30 - jepsen.util 12900 :ok :read [10 12] INFO [2019-12-17 12:38:42,195] jepsen worker 30 - jepsen.util 12900 :invoke :read [10 nil] INFO [2019-12-17 12:38:42,202] jepsen worker 6 - jepsen.util 18546 :info :write [2 30] :RequestCanceledException INFO [2019-12-17 12:38:42,203] jepsen worker 6 - jepsen.util 18636 :invoke :read [2 nil] INFO [2019-12-17 12:38:42,217] jepsen worker 30 - jepsen.util 12900 :ok :read [10 12] INFO [2019-12-17 12:38:42,217] jepsen worker 30 - jepsen.util 12900 :invoke :cas [10 [12 23]] INFO [2019-12-17 12:38:42,236] jepsen worker 78 - jepsen.util 13398 :fail :cas [26 [1 32]] :ValueNotSwapFrom INFO [2019-12-17 12:38:42,236] jepsen worker 78 - jepsen.util 13398 :invoke :cas [26 [13 13]] INFO [2019-12-17 12:38:42,258] jepsen worker 64 - jepsen.util 10594 :ok :write [21 31] INFO [2019-12-17 12:38:42,258] jepsen worker 64 - jepsen.util 10594 :invoke :cas [21 [16 15]] INFO [2019-12-17 12:38:42,266] jepsen worker 34 - jepsen.util 19474 :info :cas [11 [2 2]] :RequestCanceledException INFO [2019-12-17 12:38:42,267] jepsen worker 34 - jepsen.util 19564 :invoke :cas [11 [47 38]] INFO [2019-12-17 12:38:42,283] jepsen worker 64 - jepsen.util 10594 :fail :cas [21 [16 15]] :ValueNotSwapFrom INFO [2019-12-17 12:38:42,283] jepsen worker 64 - jepsen.util 10594 :invoke :cas [21 [15 40]] INFO [2019-12-17 12:38:42,289] jepsen worker 3 - jepsen.util 13143 :ok :read [1 12] INFO [2019-12-17 12:38:42,289] jepsen worker 3 - jepsen.util 13143 :invoke :write [1 35] INFO [2019-12-17 12:38:42,296] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78070,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:38:42,296] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78079,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:38:42,296] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78070,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:38:42,296] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78079,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:38:42,296] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78071,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:38:42,297] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78083,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:38:42,297] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78071,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:38:42,297] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78083,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:38:42,297] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78072,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:38:42,297] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78072,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:38:42,298] jepsen worker 64 - jepsen.util 10594 :fail :cas [21 [15 40]] :ValueNotSwapFrom INFO [2019-12-17 12:38:42,299] jepsen worker 64 - jepsen.util 10594 :invoke :write [21 31] INFO [2019-12-17 12:38:42,318] jepsen worker 80 - jepsen.util 12860 :fail :cas [26 [41 16]] :ValueNotSwapFrom INFO [2019-12-17 12:38:42,318] jepsen worker 80 - jepsen.util 12860 :invoke :read [26 nil] INFO [2019-12-17 12:38:42,339] jepsen worker 64 - jepsen.util 10594 :ok :write [21 31] INFO [2019-12-17 12:38:42,339] jepsen worker 64 - jepsen.util 10594 :invoke :cas [21 [26 34]] INFO [2019-12-17 12:38:42,349] jepsen worker 70 - jepsen.util 13840 :fail :cas [23 [27 8]] :ValueNotSwapFrom INFO [2019-12-17 12:38:42,349] jepsen worker 70 - jepsen.util 13840 :invoke :read [23 nil] INFO [2019-12-17 12:38:42,365] jepsen worker 64 - jepsen.util 10594 :fail :cas [21 [26 34]] :ValueNotSwapFrom INFO [2019-12-17 12:38:42,365] jepsen worker 64 - jepsen.util 10594 :invoke :cas [21 [7 29]] INFO [2019-12-17 12:38:42,380] jepsen worker 64 - jepsen.util 10594 :fail :cas [21 [7 29]] :ValueNotSwapFrom INFO [2019-12-17 12:38:42,381] jepsen worker 64 - jepsen.util 10594 :invoke :write [21 3] INFO [2019-12-17 12:38:42,381] jepsen worker 55 - jepsen.util 14095 :fail :cas [18 [23 7]] :ValueNotSwapFrom INFO [2019-12-17 12:38:42,381] jepsen worker 55 - jepsen.util 14095 :invoke :read [18 nil] INFO [2019-12-17 12:38:42,401] jepsen worker 81 - jepsen.util 17811 :info :write [27 36] :RequestCanceledException INFO [2019-12-17 12:38:42,401] jepsen worker 61 - jepsen.util 19771 :info :cas [20 [9 31]] :RequestCanceledException INFO [2019-12-17 12:38:42,401] jepsen worker 47 - jepsen.util 14717 :fail :read [15 nil] :RequestTimeoutException INFO [2019-12-17 12:38:42,401] jepsen worker 47 - jepsen.util 14717 :invoke :cas [15 [25 13]] INFO [2019-12-17 12:38:42,401] jepsen worker 81 - jepsen.util 17901 :invoke :write [27 28] INFO [2019-12-17 12:38:42,401] jepsen worker 61 - jepsen.util 19861 :invoke :cas [20 [40 14]] INFO [2019-12-17 12:38:42,419] jepsen worker 64 - jepsen.util 10594 :ok :write [21 3] INFO [2019-12-17 12:38:42,420] jepsen worker 26 - jepsen.util 19376 :info :write [8 8] :RequestCanceledException INFO [2019-12-17 12:38:42,420] jepsen worker 71 - jepsen.util 14291 :info :write [23 49] :RequestCanceledException INFO [2019-12-17 12:38:42,420] jepsen worker 64 - jepsen.util 10594 :invoke :cas [21 [41 9]] INFO [2019-12-17 12:38:42,420] jepsen worker 71 - jepsen.util 14381 :invoke :write [23 15] INFO [2019-12-17 12:38:42,421] jepsen worker 26 - jepsen.util 19466 :invoke :write [8 26] INFO [2019-12-17 12:38:42,433] jepsen worker 78 - jepsen.util 13398 :fail :cas [26 [13 13]] :ValueNotSwapFrom INFO [2019-12-17 12:38:42,433] jepsen worker 78 - jepsen.util 13398 :invoke :write [26 42] INFO [2019-12-17 12:38:42,434] jepsen worker 64 - jepsen.util 10594 :fail :cas [21 [41 9]] :ValueNotSwapFrom INFO [2019-12-17 12:38:42,434] jepsen worker 64 - jepsen.util 10594 :invoke :write [21 27] INFO [2019-12-17 12:38:42,435] jepsen worker 35 - jepsen.util 18665 :info :write [11 2] :RequestCanceledException INFO [2019-12-17 12:38:42,435] jepsen worker 83 - jepsen.util 18353 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:38:42,435] jepsen worker 83 - jepsen.util 18353 :invoke :cas [27 [21 6]] INFO [2019-12-17 12:38:42,436] jepsen worker 35 - jepsen.util 18755 :invoke :cas [11 [11 4]] INFO [2019-12-17 12:38:42,471] jepsen worker 64 - jepsen.util 10594 :ok :write [21 27] INFO [2019-12-17 12:38:42,472] jepsen worker 64 - jepsen.util 10594 :invoke :cas [21 [28 24]] INFO [2019-12-17 12:38:42,486] jepsen worker 64 - jepsen.util 10594 :fail :cas [21 [28 24]] :ValueNotSwapFrom INFO [2019-12-17 12:38:42,487] jepsen worker 64 - jepsen.util 10594 :invoke :write [21 22] INFO [2019-12-17 12:38:42,510] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78107,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:38:42,510] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78107,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:38:42,510] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78110,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:38:42,511] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78110,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:38:42,511] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78111,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:38:42,511] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78111,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:38:42,519] jepsen worker 31 - jepsen.util 11731 :ok :write [10 21] INFO [2019-12-17 12:38:42,519] jepsen worker 31 - jepsen.util 11731 :invoke :cas [10 [12 18]] INFO [2019-12-17 12:38:42,524] jepsen worker 45 - jepsen.util 13005 :ok :write [15 13] INFO [2019-12-17 12:38:42,524] jepsen worker 45 - jepsen.util 13005 :invoke :cas [15 [36 30]] INFO [2019-12-17 12:38:42,550] jepsen worker 45 - jepsen.util 13005 :fail :cas [15 [36 30]] :ValueNotSwapFrom INFO [2019-12-17 12:38:42,550] jepsen worker 45 - jepsen.util 13005 :invoke :write [15 36] INFO [2019-12-17 12:38:42,557] jepsen worker 72 - jepsen.util 13482 :ok :write [24 10] INFO [2019-12-17 12:38:42,557] jepsen worker 72 - jepsen.util 13482 :invoke :write [24 37] INFO [2019-12-17 12:38:42,557] jepsen worker 64 - jepsen.util 10594 :ok :write [21 22] INFO [2019-12-17 12:38:42,557] jepsen worker 64 - jepsen.util 10594 :invoke :read [21 nil] INFO [2019-12-17 12:38:42,567] jepsen worker 19 - jepsen.util 14239 :ok :write [6 0] INFO [2019-12-17 12:38:42,567] jepsen worker 19 - jepsen.util 14239 :invoke :read [6 nil] INFO [2019-12-17 12:38:42,572] jepsen worker 64 - jepsen.util 10594 :ok :read [21 22] INFO [2019-12-17 12:38:42,572] jepsen worker 64 - jepsen.util 10594 :invoke :read [21 nil] INFO [2019-12-17 12:38:42,593] jepsen worker 46 - jepsen.util 13096 :ok :read [15 13] INFO [2019-12-17 12:38:42,593] jepsen worker 46 - jepsen.util 13096 :invoke :cas [15 [48 22]] INFO [2019-12-17 12:38:42,603] jepsen worker 14 - jepsen.util 13064 :fail :cas [4 [14 19]] :ValueNotSwapFrom INFO [2019-12-17 12:38:42,603] jepsen worker 14 - jepsen.util 13064 :invoke :cas [4 [43 4]] INFO [2019-12-17 12:38:42,626] jepsen worker 12 - jepsen.util 13512 :fail :cas [4 [13 15]] :ValueNotSwapFrom INFO [2019-12-17 12:38:42,626] jepsen worker 12 - jepsen.util 13512 :invoke :write [4 41] INFO [2019-12-17 12:38:42,629] jepsen worker 37 - jepsen.util 12907 :fail :cas [12 [46 47]] :ValueNotSwapFrom INFO [2019-12-17 12:38:42,629] jepsen worker 37 - jepsen.util 12907 :invoke :write [12 7] INFO [2019-12-17 12:38:42,641] jepsen worker 64 - jepsen.util 10594 :ok :read [21 22] INFO [2019-12-17 12:38:42,642] jepsen worker 64 - jepsen.util 10594 :invoke :write [21 29] INFO [2019-12-17 12:38:42,648] jepsen worker 47 - jepsen.util 14717 :fail :cas [15 [25 13]] :ValueNotSwapFrom INFO [2019-12-17 12:38:42,649] jepsen worker 47 - jepsen.util 14717 :invoke :write [15 28] INFO [2019-12-17 12:38:42,649] jepsen worker 32 - jepsen.util 13712 :info :write [10 25] :RequestCanceledException INFO [2019-12-17 12:38:42,649] jepsen worker 27 - jepsen.util 12357 :fail :read [9 nil] :RequestTimeoutException INFO [2019-12-17 12:38:42,649] jepsen worker 27 - jepsen.util 12357 :invoke :read [9 nil] INFO [2019-12-17 12:38:42,650] jepsen worker 32 - jepsen.util 13802 :invoke :read [10 nil] INFO [2019-12-17 12:38:42,686] jepsen worker 19 - jepsen.util 14239 :ok :read [6 0] INFO [2019-12-17 12:38:42,686] jepsen worker 19 - jepsen.util 14239 :invoke :cas [6 [29 42]] INFO [2019-12-17 12:38:42,692] jepsen worker 29 - jepsen.util 11639 :ok :cas [9 [4 46]] INFO [2019-12-17 12:38:42,692] jepsen worker 29 - jepsen.util 11639 :invoke :read [9 nil] INFO [2019-12-17 12:38:42,724] jepsen worker 46 - jepsen.util 13096 :fail :cas [15 [48 22]] :ValueNotSwapFrom INFO [2019-12-17 12:38:42,724] jepsen worker 80 - jepsen.util 12860 :ok :read [26 32] INFO [2019-12-17 12:38:42,724] jepsen worker 46 - jepsen.util 13096 :invoke :write [15 36] INFO [2019-12-17 12:38:42,724] jepsen worker 80 - jepsen.util 12860 :invoke :write [26 15] INFO [2019-12-17 12:38:42,735] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78130,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:38:42,735] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78130,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:38:42,735] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78134,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:38:42,735] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78134,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:38:42,735] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78135,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:38:42,735] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78135,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:38:42,739] jepsen worker 70 - jepsen.util 13840 :ok :read [23 48] INFO [2019-12-17 12:38:42,739] jepsen worker 70 - jepsen.util 13840 :invoke :cas [23 [19 27]] INFO [2019-12-17 12:38:42,751] jepsen worker 64 - jepsen.util 10594 :ok :write [21 29] INFO [2019-12-17 12:38:42,751] jepsen worker 64 - jepsen.util 10594 :invoke :cas [21 [42 28]] INFO [2019-12-17 12:38:42,767] jepsen worker 55 - jepsen.util 14095 :ok :read [18 11] INFO [2019-12-17 12:38:42,767] jepsen worker 55 - jepsen.util 14095 :invoke :cas [18 [24 18]] INFO [2019-12-17 12:38:42,771] jepsen worker 64 - jepsen.util 10594 :fail :cas [21 [42 28]] :ValueNotSwapFrom INFO [2019-12-17 12:38:42,771] jepsen worker 64 - jepsen.util 10594 :invoke :write [21 29] INFO [2019-12-17 12:38:42,792] jepsen worker 73 - jepsen.util 12403 :ok :write [24 29] INFO [2019-12-17 12:38:42,793] jepsen worker 73 - jepsen.util 12403 :invoke :cas [24 [39 39]] INFO [2019-12-17 12:38:42,798] jepsen worker 47 - jepsen.util 14717 :ok :write [15 28] INFO [2019-12-17 12:38:42,798] jepsen worker 47 - jepsen.util 14717 :invoke :read [15 nil] INFO [2019-12-17 12:38:42,861] jepsen worker 31 - jepsen.util 11731 :fail :cas [10 [12 18]] :ValueNotSwapFrom INFO [2019-12-17 12:38:42,861] jepsen worker 31 - jepsen.util 11731 :invoke :read [10 nil] INFO [2019-12-17 12:38:42,870] jepsen worker 19 - jepsen.util 14239 :fail :cas [6 [29 42]] :ValueNotSwapFrom INFO [2019-12-17 12:38:42,870] jepsen worker 19 - jepsen.util 14239 :invoke :read [6 nil] INFO [2019-12-17 12:38:42,886] jepsen worker 64 - jepsen.util 10594 :ok :write [21 29] INFO [2019-12-17 12:38:42,887] jepsen worker 64 - jepsen.util 10594 :invoke :write [21 40] INFO [2019-12-17 12:38:42,895] jepsen worker 73 - jepsen.util 12403 :fail :cas [24 [39 39]] :ValueNotSwapFrom INFO [2019-12-17 12:38:42,895] jepsen worker 73 - jepsen.util 12403 :invoke :write [24 32] INFO [2019-12-17 12:38:42,903] jepsen worker 31 - jepsen.util 11731 :ok :read [10 21] INFO [2019-12-17 12:38:42,903] jepsen worker 31 - jepsen.util 11731 :invoke :read [10 nil] INFO [2019-12-17 12:38:42,915] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:38:42,915] jepsen nemesis - couchbase.util Rebalance check count: 614 INFO [2019-12-17 12:38:42,915] jepsen nemesis - couchbase.util Rebalance stuck count: 595 INFO [2019-12-17 12:38:42,920] jepsen worker 13 - jepsen.util 12973 :info :write [4 2] :RequestCanceledException INFO [2019-12-17 12:38:42,924] jepsen worker 13 - jepsen.util 13063 :invoke :write [4 34] INFO [2019-12-17 12:38:42,947] jepsen worker 46 - jepsen.util 13096 :ok :write [15 36] INFO [2019-12-17 12:38:42,947] jepsen worker 46 - jepsen.util 13096 :invoke :write [15 4] INFO [2019-12-17 12:38:42,970] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78152,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:38:42,970] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78152,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:38:42,970] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78153,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:38:42,970] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78153,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:38:42,984] jepsen worker 19 - jepsen.util 14239 :ok :read [6 0] INFO [2019-12-17 12:38:42,985] jepsen worker 19 - jepsen.util 14239 :invoke :cas [6 [4 25]] INFO [2019-12-17 12:38:43,000] jepsen worker 64 - jepsen.util 10594 :ok :write [21 40] INFO [2019-12-17 12:38:43,000] jepsen worker 64 - jepsen.util 10594 :invoke :read [21 nil] INFO [2019-12-17 12:38:43,034] jepsen worker 64 - jepsen.util 10594 :ok :read [21 40] INFO [2019-12-17 12:38:43,034] jepsen worker 64 - jepsen.util 10594 :invoke :write [21 40] INFO [2019-12-17 12:38:43,066] jepsen worker 54 - jepsen.util 14184 :info :write [18 18] :RequestCanceledException INFO [2019-12-17 12:38:43,067] jepsen worker 54 - jepsen.util 14274 :invoke :read [18 nil] INFO [2019-12-17 12:38:43,080] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78163,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:38:43,080] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78163,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:38:43,080] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78166,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:38:43,080] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78166,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:38:43,080] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78167,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:38:43,080] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78167,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:38:43,090] jepsen worker 64 - jepsen.util 10594 :ok :write [21 40] INFO [2019-12-17 12:38:43,090] jepsen worker 64 - jepsen.util 10594 :invoke :read [21 nil] INFO [2019-12-17 12:38:43,093] jepsen worker 14 - jepsen.util 13064 :fail :cas [4 [43 4]] :ValueNotSwapFrom INFO [2019-12-17 12:38:43,094] jepsen worker 14 - jepsen.util 13064 :invoke :write [4 43] INFO [2019-12-17 12:38:43,102] jepsen worker 52 - jepsen.util 13012 :info :write [17 44] :RequestCanceledException INFO [2019-12-17 12:38:43,102] jepsen worker 38 - jepsen.util 14078 :info :write [12 34] :RequestCanceledException INFO [2019-12-17 12:38:43,103] jepsen worker 52 - jepsen.util 13102 :invoke :cas [17 [46 5]] INFO [2019-12-17 12:38:43,110] jepsen worker 38 - jepsen.util 14168 :invoke :write [12 19] INFO [2019-12-17 12:38:43,111] jepsen worker 64 - jepsen.util 10594 :ok :read [21 40] INFO [2019-12-17 12:38:43,111] jepsen worker 64 - jepsen.util 10594 :invoke :cas [21 [29 14]] INFO [2019-12-17 12:38:43,114] jepsen worker 46 - jepsen.util 13096 :ok :write [15 4] INFO [2019-12-17 12:38:43,114] jepsen worker 46 - jepsen.util 13096 :invoke :write [15 33] INFO [2019-12-17 12:38:43,133] jepsen worker 19 - jepsen.util 14239 :fail :cas [6 [4 25]] :ValueNotSwapFrom INFO [2019-12-17 12:38:43,133] jepsen worker 19 - jepsen.util 14239 :invoke :write [6 28] INFO [2019-12-17 12:38:43,133] jepsen worker 28 - jepsen.util 11458 :fail :read [9 nil] :RequestTimeoutException INFO [2019-12-17 12:38:43,133] jepsen worker 28 - jepsen.util 11458 :invoke :read [9 nil] INFO [2019-12-17 12:38:43,148] jepsen worker 73 - jepsen.util 12403 :ok :write [24 32] INFO [2019-12-17 12:38:43,148] jepsen worker 73 - jepsen.util 12403 :invoke :read [24 nil] INFO [2019-12-17 12:38:43,149] jepsen worker 3 - jepsen.util 13143 :ok :write [1 35] INFO [2019-12-17 12:38:43,149] jepsen worker 3 - jepsen.util 13143 :invoke :cas [1 [48 35]] INFO [2019-12-17 12:38:43,154] jepsen worker 64 - jepsen.util 10594 :fail :cas [21 [29 14]] :ValueNotSwapFrom INFO [2019-12-17 12:38:43,154] jepsen worker 64 - jepsen.util 10594 :invoke :cas [21 [1 5]] INFO [2019-12-17 12:38:43,172] jepsen worker 70 - jepsen.util 13840 :fail :cas [23 [19 27]] :ValueNotSwapFrom INFO [2019-12-17 12:38:43,172] jepsen worker 70 - jepsen.util 13840 :invoke :write [23 10] INFO [2019-12-17 12:38:43,172] jepsen worker 28 - jepsen.util 11458 :ok :read [9 46] INFO [2019-12-17 12:38:43,172] jepsen worker 28 - jepsen.util 11458 :invoke :read [9 nil] INFO [2019-12-17 12:38:43,194] jepsen worker 73 - jepsen.util 12403 :ok :read [24 32] INFO [2019-12-17 12:38:43,194] jepsen worker 73 - jepsen.util 12403 :invoke :read [24 nil] INFO [2019-12-17 12:38:43,198] jepsen worker 89 - jepsen.util 13589 :ok :write [29 39] INFO [2019-12-17 12:38:43,198] jepsen worker 89 - jepsen.util 13589 :invoke :write [29 35] INFO [2019-12-17 12:38:43,219] jepsen worker 36 - jepsen.util 14076 :info :write [12 31] :RequestCanceledException INFO [2019-12-17 12:38:43,220] jepsen worker 36 - jepsen.util 14166 :invoke :read [12 nil] INFO [2019-12-17 12:38:43,223] jepsen worker 46 - jepsen.util 13096 :ok :write [15 33] INFO [2019-12-17 12:38:43,223] jepsen worker 46 - jepsen.util 13096 :invoke :cas [15 [4 41]] INFO [2019-12-17 12:38:43,267] jepsen worker 5 - jepsen.util 13145 :ok :write [1 45] INFO [2019-12-17 12:38:43,267] jepsen worker 5 - jepsen.util 13145 :invoke :read [1 nil] INFO [2019-12-17 12:38:43,270] jepsen worker 37 - jepsen.util 12907 :ok :write [12 7] INFO [2019-12-17 12:38:43,270] jepsen worker 37 - jepsen.util 12907 :invoke :write [12 39] INFO [2019-12-17 12:38:43,297] jepsen worker 64 - jepsen.util 10594 :fail :cas [21 [1 5]] :ValueNotSwapFrom INFO [2019-12-17 12:38:43,298] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78184,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:38:43,298] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78195,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:38:43,302] jepsen worker 55 - jepsen.util 14095 :fail :cas [18 [24 18]] :ValueNotSwapFrom INFO [2019-12-17 12:38:43,320] jepsen worker 20 - jepsen.util 13250 :info :cas [6 [17 41]] :RequestCanceledException INFO [2019-12-17 12:38:43,322] jepsen worker 28 - jepsen.util 11458 :ok :read [9 46] INFO [2019-12-17 12:38:43,335] jepsen worker 4 - jepsen.util 12694 :info :write [1 13] :RequestCanceledException INFO [2019-12-17 12:38:43,339] jepsen worker 73 - jepsen.util 12403 :ok :read [24 32] INFO [2019-12-17 12:38:43,366] jepsen worker 64 - jepsen.util 10594 :invoke :read [21 nil] INFO [2019-12-17 12:38:43,366] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78184,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:38:43,366] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78195,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:38:43,367] jepsen worker 55 - jepsen.util 14095 :invoke :write [18 28] INFO [2019-12-17 12:38:43,367] jepsen worker 28 - jepsen.util 11458 :invoke :write [9 47] INFO [2019-12-17 12:38:43,367] jepsen worker 73 - jepsen.util 12403 :invoke :write [24 13] INFO [2019-12-17 12:38:43,367] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78185,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:38:43,367] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78197,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:38:43,367] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78185,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:38:43,367] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78197,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:38:43,367] jepsen worker 4 - jepsen.util 12784 :invoke :read [1 nil] INFO [2019-12-17 12:38:43,367] jepsen worker 20 - jepsen.util 13340 :invoke :cas [6 [44 44]] INFO [2019-12-17 12:38:43,390] jepsen worker 20 - jepsen.util 13340 :fail :cas [6 [44 44]] :ValueNotSwapFrom INFO [2019-12-17 12:38:43,390] jepsen worker 20 - jepsen.util 13340 :invoke :cas [6 [36 24]] INFO [2019-12-17 12:38:43,429] jepsen worker 64 - jepsen.util 10594 :ok :read [21 40] INFO [2019-12-17 12:38:43,429] jepsen worker 64 - jepsen.util 10594 :invoke :cas [21 [2 20]] INFO [2019-12-17 12:38:43,435] jepsen worker 38 - jepsen.util 14168 :ok :write [12 19] INFO [2019-12-17 12:38:43,435] jepsen worker 38 - jepsen.util 14168 :invoke :cas [12 [24 46]] INFO [2019-12-17 12:38:43,437] jepsen worker 20 - jepsen.util 13340 :fail :cas [6 [36 24]] :ValueNotSwapFrom INFO [2019-12-17 12:38:43,437] jepsen worker 20 - jepsen.util 13340 :invoke :read [6 nil] INFO [2019-12-17 12:38:43,445] jepsen worker 56 - jepsen.util 14186 :ok :write [18 43] INFO [2019-12-17 12:38:43,445] jepsen worker 56 - jepsen.util 14186 :invoke :write [18 7] INFO [2019-12-17 12:38:43,449] jepsen worker 53 - jepsen.util 13193 :info :write [17 29] :RequestCanceledException INFO [2019-12-17 12:38:43,450] jepsen worker 53 - jepsen.util 13283 :invoke :write [17 8] INFO [2019-12-17 12:38:43,478] jepsen worker 64 - jepsen.util 10594 :fail :cas [21 [2 20]] :ValueNotSwapFrom INFO [2019-12-17 12:38:43,478] jepsen worker 64 - jepsen.util 10594 :invoke :write [21 25] INFO [2019-12-17 12:38:43,482] jepsen worker 20 - jepsen.util 13340 :ok :read [6 0] INFO [2019-12-17 12:38:43,485] jepsen worker 20 - jepsen.util 13340 :invoke :cas [6 [35 18]] INFO [2019-12-17 12:38:43,494] jepsen worker 3 - jepsen.util 13143 :fail :cas [1 [48 35]] :ValueNotSwapFrom INFO [2019-12-17 12:38:43,494] jepsen worker 3 - jepsen.util 13143 :invoke :cas [1 [20 8]] INFO [2019-12-17 12:38:43,501] jepsen worker 20 - jepsen.util 13340 :fail :cas [6 [35 18]] :ValueNotSwapFrom INFO [2019-12-17 12:38:43,501] jepsen worker 20 - jepsen.util 13340 :invoke :cas [6 [45 18]] INFO [2019-12-17 12:38:43,514] jepsen worker 20 - jepsen.util 13340 :fail :cas [6 [45 18]] :ValueNotSwapFrom INFO [2019-12-17 12:38:43,514] jepsen worker 20 - jepsen.util 13340 :invoke :write [6 38] INFO [2019-12-17 12:38:43,518] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78211,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:38:43,518] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78211,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:38:43,518] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78214,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:38:43,518] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78214,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:38:43,534] jepsen worker 64 - jepsen.util 10594 :ok :write [21 25] INFO [2019-12-17 12:38:43,534] jepsen worker 64 - jepsen.util 10594 :invoke :read [21 nil] INFO [2019-12-17 12:38:43,534] jepsen worker 19 - jepsen.util 14239 :ok :write [6 28] INFO [2019-12-17 12:38:43,535] jepsen worker 19 - jepsen.util 14239 :invoke :read [6 nil] INFO [2019-12-17 12:38:43,551] jepsen worker 64 - jepsen.util 10594 :ok :read [21 25] INFO [2019-12-17 12:38:43,551] jepsen worker 64 - jepsen.util 10594 :invoke :read [21 nil] INFO [2019-12-17 12:38:43,567] jepsen worker 73 - jepsen.util 12403 :ok :write [24 13] INFO [2019-12-17 12:38:43,567] jepsen worker 73 - jepsen.util 12403 :invoke :cas [24 [22 34]] INFO [2019-12-17 12:38:43,574] jepsen worker 19 - jepsen.util 14239 :ok :read [6 28] INFO [2019-12-17 12:38:43,574] jepsen worker 64 - jepsen.util 10594 :ok :read [21 25] INFO [2019-12-17 12:38:43,574] jepsen worker 19 - jepsen.util 14239 :invoke :read [6 nil] INFO [2019-12-17 12:38:43,574] jepsen worker 64 - jepsen.util 10594 :invoke :read [21 nil] INFO [2019-12-17 12:38:43,586] jepsen worker 71 - jepsen.util 14381 :ok :write [23 15] INFO [2019-12-17 12:38:43,586] jepsen worker 71 - jepsen.util 14381 :invoke :read [23 nil] INFO [2019-12-17 12:38:43,591] jepsen worker 36 - jepsen.util 14166 :ok :read [12 19] INFO [2019-12-17 12:38:43,591] jepsen worker 36 - jepsen.util 14166 :invoke :write [12 47] INFO [2019-12-17 12:38:43,613] jepsen worker 64 - jepsen.util 10594 :ok :read [21 25] INFO [2019-12-17 12:38:43,613] jepsen worker 64 - jepsen.util 10594 :invoke :write [21 49] INFO [2019-12-17 12:38:43,619] jepsen worker 11 - jepsen.util 13241 :info :write [3 45] :RequestCanceledException INFO [2019-12-17 12:38:43,619] jepsen worker 88 - jepsen.util 13318 :info :write [29 4] :RequestCanceledException INFO [2019-12-17 12:38:43,620] jepsen worker 88 - jepsen.util 13408 :invoke :read [29 nil] INFO [2019-12-17 12:38:43,620] jepsen worker 11 - jepsen.util 13331 :invoke :read [3 nil] INFO [2019-12-17 12:38:43,636] jepsen worker 65 - jepsen.util 11765 :fail :read [21 nil] :RequestTimeoutException INFO [2019-12-17 12:38:43,636] jepsen worker 65 - jepsen.util 11765 :invoke :read [21 nil] INFO [2019-12-17 12:38:43,645] jepsen worker 19 - jepsen.util 14239 :ok :read [6 28] INFO [2019-12-17 12:38:43,645] jepsen worker 19 - jepsen.util 14239 :invoke :read [6 nil] INFO [2019-12-17 12:38:43,657] jepsen worker 65 - jepsen.util 11765 :ok :read [21 25] INFO [2019-12-17 12:38:43,658] jepsen worker 65 - jepsen.util 11765 :invoke :cas [21 [0 42]] INFO [2019-12-17 12:38:43,662] jepsen worker 28 - jepsen.util 11458 :ok :write [9 47] INFO [2019-12-17 12:38:43,662] jepsen worker 28 - jepsen.util 11458 :invoke :write [9 2] INFO [2019-12-17 12:38:43,664] jepsen worker 19 - jepsen.util 14239 :ok :read [6 28] INFO [2019-12-17 12:38:43,664] jepsen worker 19 - jepsen.util 14239 :invoke :write [6 10] INFO [2019-12-17 12:38:43,666] jepsen worker 79 - jepsen.util 13669 :info :write [26 44] :RequestCanceledException INFO [2019-12-17 12:38:43,668] jepsen worker 79 - jepsen.util 13759 :invoke :cas [26 [33 41]] INFO [2019-12-17 12:38:43,701] jepsen worker 65 - jepsen.util 11765 :fail :cas [21 [0 42]] :ValueNotSwapFrom INFO [2019-12-17 12:38:43,701] jepsen worker 65 - jepsen.util 11765 :invoke :write [21 46] INFO [2019-12-17 12:38:43,714] jepsen worker 54 - jepsen.util 14274 :ok :read [18 43] INFO [2019-12-17 12:38:43,715] jepsen worker 54 - jepsen.util 14274 :invoke :write [18 6] INFO [2019-12-17 12:38:43,717] jepsen worker 38 - jepsen.util 14168 :fail :cas [12 [24 46]] :ValueNotSwapFrom INFO [2019-12-17 12:38:43,717] jepsen worker 38 - jepsen.util 14168 :invoke :cas [12 [44 9]] INFO [2019-12-17 12:38:43,727] jepsen worker 4 - jepsen.util 12784 :ok :read [1 45] INFO [2019-12-17 12:38:43,727] jepsen worker 4 - jepsen.util 12784 :invoke :write [1 10] INFO [2019-12-17 12:38:43,729] jepsen worker 64 - jepsen.util 10594 :ok :write [21 49] INFO [2019-12-17 12:38:43,729] jepsen worker 64 - jepsen.util 10594 :invoke :write [21 48] INFO [2019-12-17 12:38:43,735] jepsen worker 28 - jepsen.util 11458 :ok :write [9 2] INFO [2019-12-17 12:38:43,735] jepsen worker 28 - jepsen.util 11458 :invoke :write [9 0] INFO [2019-12-17 12:38:43,735] jepsen worker 9 - jepsen.util 13959 :info :write [3 31] :RequestCanceledException INFO [2019-12-17 12:38:43,736] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78240,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:38:43,736] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78240,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:38:43,736] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78242,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:38:43,736] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78242,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:38:43,737] jepsen worker 9 - jepsen.util 14049 :invoke :write [3 14] INFO [2019-12-17 12:38:43,750] jepsen worker 38 - jepsen.util 14168 :fail :cas [12 [44 9]] :ValueNotSwapFrom INFO [2019-12-17 12:38:43,750] jepsen worker 38 - jepsen.util 14168 :invoke :read [12 nil] INFO [2019-12-17 12:38:43,770] jepsen worker 38 - jepsen.util 14168 :ok :read [12 19] INFO [2019-12-17 12:38:43,770] jepsen worker 38 - jepsen.util 14168 :invoke :write [12 49] INFO [2019-12-17 12:38:43,796] jepsen worker 51 - jepsen.util 12831 :ok :write [17 0] INFO [2019-12-17 12:38:43,796] jepsen worker 51 - jepsen.util 12831 :invoke :cas [17 [48 12]] INFO [2019-12-17 12:38:43,808] jepsen worker 20 - jepsen.util 13340 :ok :write [6 38] INFO [2019-12-17 12:38:43,808] jepsen worker 20 - jepsen.util 13340 :invoke :write [6 11] INFO [2019-12-17 12:38:43,825] jepsen worker 64 - jepsen.util 10594 :ok :write [21 48] INFO [2019-12-17 12:38:43,825] jepsen worker 64 - jepsen.util 10594 :invoke :read [21 nil] INFO [2019-12-17 12:38:43,835] jepsen worker 67 - jepsen.util 18877 :info :write [22 34] :RequestCanceledException INFO [2019-12-17 12:38:43,837] jepsen worker 67 - jepsen.util 18967 :invoke :cas [22 [25 16]] INFO [2019-12-17 12:38:43,847] jepsen worker 28 - jepsen.util 11458 :ok :write [9 0] INFO [2019-12-17 12:38:43,847] jepsen worker 28 - jepsen.util 11458 :invoke :read [9 nil] INFO [2019-12-17 12:38:43,848] jepsen worker 52 - jepsen.util 13102 :fail :cas [17 [46 5]] :ValueNotSwapFrom INFO [2019-12-17 12:38:43,848] jepsen worker 52 - jepsen.util 13102 :invoke :cas [17 [39 47]] INFO [2019-12-17 12:38:43,849] jepsen worker 7 - jepsen.util 18367 :info :cas [2 [36 35]] :RequestCanceledException INFO [2019-12-17 12:38:43,851] jepsen worker 7 - jepsen.util 18457 :invoke :cas [2 [1 38]] INFO [2019-12-17 12:38:43,853] jepsen worker 64 - jepsen.util 10594 :ok :read [21 48] INFO [2019-12-17 12:38:43,853] jepsen worker 64 - jepsen.util 10594 :invoke :write [21 35] INFO [2019-12-17 12:38:43,866] jepsen worker 40 - jepsen.util 19030 :info :cas [13 [20 26]] :RequestCanceledException INFO [2019-12-17 12:38:43,866] jepsen worker 63 - jepsen.util 11853 :info :write [21 10] :RequestCanceledException INFO [2019-12-17 12:38:43,867] jepsen worker 40 - jepsen.util 19120 :invoke :write [13 35] INFO [2019-12-17 12:38:43,868] jepsen worker 63 - jepsen.util 11943 :invoke :read [21 nil] INFO [2019-12-17 12:38:43,880] jepsen worker 5 - jepsen.util 13145 :ok :read [1 45] INFO [2019-12-17 12:38:43,880] jepsen worker 5 - jepsen.util 13145 :invoke :cas [1 [10 42]] INFO [2019-12-17 12:38:43,890] jepsen worker 28 - jepsen.util 11458 :ok :read [9 0] INFO [2019-12-17 12:38:43,890] jepsen worker 28 - jepsen.util 11458 :invoke :read [9 nil] INFO [2019-12-17 12:38:43,899] jepsen worker 24 - jepsen.util 17574 :info :write [8 32] :RequestCanceledException INFO [2019-12-17 12:38:43,899] jepsen worker 2 - jepsen.util 17822 :info :write [0 33] :RequestCanceledException INFO [2019-12-17 12:38:43,899] jepsen worker 57 - jepsen.util 18057 :info :write [19 2] :RequestCanceledException INFO [2019-12-17 12:38:43,900] jepsen worker 57 - jepsen.util 18147 :invoke :cas [19 [31 19]] INFO [2019-12-17 12:38:43,901] jepsen worker 24 - jepsen.util 17664 :invoke :write [8 32] INFO [2019-12-17 12:38:43,901] jepsen worker 2 - jepsen.util 17912 :invoke :write [0 20] INFO [2019-12-17 12:38:43,915] jepsen worker 20 - jepsen.util 13340 :ok :write [6 11] INFO [2019-12-17 12:38:43,915] jepsen worker 20 - jepsen.util 13340 :invoke :cas [6 [17 9]] INFO [2019-12-17 12:38:43,917] jepsen worker 59 - jepsen.util 17609 :info :write [19 30] :RequestCanceledException INFO [2019-12-17 12:38:43,917] jepsen worker 22 - jepsen.util 18922 :info :cas [7 [27 16]] :RequestCanceledException INFO [2019-12-17 12:38:43,919] jepsen worker 22 - jepsen.util 19012 :invoke :cas [7 [42 38]] INFO [2019-12-17 12:38:43,919] jepsen worker 59 - jepsen.util 17699 :invoke :write [19 11] INFO [2019-12-17 12:38:43,919] jepsen worker 28 - jepsen.util 11458 :ok :read [9 0] INFO [2019-12-17 12:38:43,919] jepsen worker 28 - jepsen.util 11458 :invoke :read [9 nil] INFO [2019-12-17 12:38:43,921] jepsen worker 9 - jepsen.util 14049 :ok :write [3 14] INFO [2019-12-17 12:38:43,921] jepsen worker 9 - jepsen.util 14049 :invoke :cas [3 [45 16]] INFO [2019-12-17 12:38:43,924] jepsen worker 56 - jepsen.util 14186 :ok :write [18 7] INFO [2019-12-17 12:38:43,924] jepsen worker 56 - jepsen.util 14186 :invoke :read [18 nil] INFO [2019-12-17 12:38:43,928] jepsen worker 38 - jepsen.util 14168 :ok :write [12 49] INFO [2019-12-17 12:38:43,928] jepsen worker 38 - jepsen.util 14168 :invoke :read [12 nil] INFO [2019-12-17 12:38:43,931] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:38:43,932] jepsen nemesis - couchbase.util Rebalance check count: 615 INFO [2019-12-17 12:38:43,932] jepsen nemesis - couchbase.util Rebalance stuck count: 596 INFO [2019-12-17 12:38:43,933] jepsen worker 20 - jepsen.util 13340 :fail :cas [6 [17 9]] :ValueNotSwapFrom INFO [2019-12-17 12:38:43,933] jepsen worker 20 - jepsen.util 13340 :invoke :read [6 nil] INFO [2019-12-17 12:38:43,933] jepsen worker 76 - jepsen.util 18706 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:38:43,933] jepsen worker 87 - jepsen.util 13137 :info :write [29 16] :RequestCanceledException INFO [2019-12-17 12:38:43,933] jepsen worker 16 - jepsen.util 20176 :info :cas [5 [31 6]] :RequestCanceledException INFO [2019-12-17 12:38:43,933] jepsen worker 85 - jepsen.util 18985 :info :cas [28 [14 49]] :RequestCanceledException INFO [2019-12-17 12:38:43,933] jepsen worker 76 - jepsen.util 18706 :invoke :cas [25 [16 14]] INFO [2019-12-17 12:38:43,934] jepsen worker 16 - jepsen.util 20266 :invoke :write [5 49] INFO [2019-12-17 12:38:43,934] jepsen worker 87 - jepsen.util 13227 :invoke :write [29 39] INFO [2019-12-17 12:38:43,935] jepsen worker 85 - jepsen.util 19075 :invoke :read [28 nil] INFO [2019-12-17 12:38:43,949] jepsen worker 62 - jepsen.util 19142 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:38:43,949] jepsen worker 62 - jepsen.util 19142 :invoke :write [20 27] INFO [2019-12-17 12:38:43,955] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78266,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:38:43,955] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78266,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:38:43,955] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78269,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:38:43,956] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78269,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:38:43,956] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78270,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:38:43,956] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78270,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:38:43,963] jepsen worker 38 - jepsen.util 14168 :ok :read [12 49] INFO [2019-12-17 12:38:43,963] jepsen worker 38 - jepsen.util 14168 :invoke :read [12 nil] INFO [2019-12-17 12:38:43,967] jepsen worker 49 - jepsen.util 18409 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:38:43,967] jepsen worker 33 - jepsen.util 18213 :info :cas [11 [34 16]] :RequestCanceledException INFO [2019-12-17 12:38:43,967] jepsen worker 77 - jepsen.util 19607 :info :cas [25 [36 2]] :RequestCanceledException INFO [2019-12-17 12:38:43,967] jepsen worker 49 - jepsen.util 18409 :invoke :cas [16 [46 36]] INFO [2019-12-17 12:38:43,968] jepsen worker 77 - jepsen.util 19697 :invoke :cas [25 [48 41]] INFO [2019-12-17 12:38:43,968] jepsen worker 33 - jepsen.util 18303 :invoke :write [11 2] INFO [2019-12-17 12:38:43,969] jepsen worker 28 - jepsen.util 11458 :ok :read [9 0] INFO [2019-12-17 12:38:43,970] jepsen worker 28 - jepsen.util 11458 :invoke :cas [9 [3 3]] INFO [2019-12-17 12:38:43,978] jepsen worker 20 - jepsen.util 13340 :ok :read [6 11] INFO [2019-12-17 12:38:43,978] jepsen worker 20 - jepsen.util 13340 :invoke :write [6 26] INFO [2019-12-17 12:38:43,997] jepsen worker 38 - jepsen.util 14168 :ok :read [12 49] INFO [2019-12-17 12:38:43,997] jepsen worker 38 - jepsen.util 14168 :invoke :write [12 46] INFO [2019-12-17 12:38:44,002] jepsen worker 84 - jepsen.util 17814 :info :cas [28 [17 44]] :RequestCanceledException INFO [2019-12-17 12:38:44,002] jepsen worker 60 - jepsen.util 19590 :info :cas [20 [47 0]] :RequestCanceledException INFO [2019-12-17 12:38:44,002] jepsen worker 39 - jepsen.util 17949 :info :cas [13 [25 22]] :RequestCanceledException INFO [2019-12-17 12:38:44,006] jepsen worker 39 - jepsen.util 18039 :invoke :read [13 nil] INFO [2019-12-17 12:38:44,006] jepsen worker 84 - jepsen.util 17904 :invoke :cas [28 [34 40]] INFO [2019-12-17 12:38:44,006] jepsen worker 60 - jepsen.util 19680 :invoke :cas [20 [20 41]] INFO [2019-12-17 12:38:44,016] jepsen worker 9 - jepsen.util 14049 :fail :cas [3 [45 16]] :ValueNotSwapFrom INFO [2019-12-17 12:38:44,016] jepsen worker 9 - jepsen.util 14049 :invoke :write [3 38] INFO [2019-12-17 12:38:44,020] jepsen worker 21 - jepsen.util 18021 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:38:44,020] jepsen worker 66 - jepsen.util 18696 :info :cas [22 [10 33]] :RequestCanceledException INFO [2019-12-17 12:38:44,020] jepsen worker 48 - jepsen.util 17148 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:38:44,020] jepsen worker 0 - jepsen.util 18990 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:38:44,020] jepsen worker 21 - jepsen.util 18021 :invoke :write [7 40] INFO [2019-12-17 12:38:44,020] jepsen worker 48 - jepsen.util 17148 :invoke :read [16 nil] INFO [2019-12-17 12:38:44,020] jepsen worker 0 - jepsen.util 18990 :invoke :write [0 5] INFO [2019-12-17 12:38:44,021] jepsen worker 66 - jepsen.util 18786 :invoke :cas [22 [0 46]] INFO [2019-12-17 12:38:44,035] jepsen worker 58 - jepsen.util 18688 :info :write [19 22] :RequestCanceledException INFO [2019-12-17 12:38:44,035] jepsen worker 69 - jepsen.util 13839 :info :write [23 47] :RequestCanceledException INFO [2019-12-17 12:38:44,035] jepsen worker 25 - jepsen.util 19645 :info :cas [8 [26 28]] :RequestCanceledException INFO [2019-12-17 12:38:44,035] jepsen worker 86 - jepsen.util 19256 :info :cas [28 [31 46]] :RequestCanceledException INFO [2019-12-17 12:38:44,035] jepsen worker 17 - jepsen.util 18287 :info :cas [5 [27 43]] :RequestCanceledException INFO [2019-12-17 12:38:44,035] jepsen worker 50 - jepsen.util 18230 :info :cas [16 [21 23]] :RequestCanceledException INFO [2019-12-17 12:38:44,035] jepsen worker 8 - jepsen.util 18098 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:38:44,035] jepsen worker 8 - jepsen.util 18098 :invoke :cas [2 [2 35]] INFO [2019-12-17 12:38:44,036] jepsen worker 58 - jepsen.util 18778 :invoke :read [19 nil] INFO [2019-12-17 12:38:44,036] jepsen worker 69 - jepsen.util 13929 :invoke :cas [23 [26 9]] INFO [2019-12-17 12:38:44,036] jepsen worker 50 - jepsen.util 18320 :invoke :read [16 nil] INFO [2019-12-17 12:38:44,036] jepsen worker 86 - jepsen.util 19346 :invoke :cas [28 [33 49]] INFO [2019-12-17 12:38:44,036] jepsen worker 17 - jepsen.util 18377 :invoke :read [5 nil] INFO [2019-12-17 12:38:44,037] jepsen worker 25 - jepsen.util 19735 :invoke :write [8 26] INFO [2019-12-17 12:38:44,042] jepsen worker 71 - jepsen.util 14381 :ok :read [23 15] INFO [2019-12-17 12:38:44,042] jepsen worker 71 - jepsen.util 14381 :invoke :read [23 nil] INFO [2019-12-17 12:38:44,043] jepsen worker 23 - jepsen.util 18743 :info :write [7 11] :RequestCanceledException INFO [2019-12-17 12:38:44,043] jepsen worker 82 - jepsen.util 18442 :info :cas [27 [18 47]] :RequestCanceledException INFO [2019-12-17 12:38:44,043] jepsen worker 43 - jepsen.util 18673 :info :cas [14 [47 13]] :RequestCanceledException INFO [2019-12-17 12:38:44,043] jepsen worker 15 - jepsen.util 17385 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:38:44,044] jepsen worker 41 - jepsen.util 19751 :info :write [13 1] :RequestCanceledException INFO [2019-12-17 12:38:44,045] jepsen worker 23 - jepsen.util 18833 :invoke :read [7 nil] INFO [2019-12-17 12:38:44,045] jepsen worker 15 - jepsen.util 17385 :invoke :cas [5 [21 19]] INFO [2019-12-17 12:38:44,045] jepsen worker 82 - jepsen.util 18532 :invoke :cas [27 [48 45]] INFO [2019-12-17 12:38:44,045] jepsen worker 43 - jepsen.util 18763 :invoke :cas [14 [47 40]] INFO [2019-12-17 12:38:44,046] jepsen worker 41 - jepsen.util 19841 :invoke :read [13 nil] INFO [2019-12-17 12:38:44,046] jepsen worker 28 - jepsen.util 11458 :fail :cas [9 [3 3]] :ValueNotSwapFrom INFO [2019-12-17 12:38:44,046] jepsen worker 28 - jepsen.util 11458 :invoke :write [9 3] INFO [2019-12-17 12:38:44,055] jepsen worker 64 - jepsen.util 10594 :ok :write [21 35] INFO [2019-12-17 12:38:44,055] jepsen worker 64 - jepsen.util 10594 :invoke :write [21 17] INFO [2019-12-17 12:38:44,065] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78298,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:38:44,065] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78298,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:38:44,065] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78301,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:38:44,066] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78301,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:38:44,066] jepsen worker 74 - jepsen.util 13304 :info :write [24 47] :RequestCanceledException INFO [2019-12-17 12:38:44,066] jepsen worker 42 - jepsen.util 19392 :info :cas [14 [42 22]] :RequestCanceledException INFO [2019-12-17 12:38:44,067] jepsen worker 42 - jepsen.util 19482 :invoke :read [14 nil] INFO [2019-12-17 12:38:44,067] jepsen worker 74 - jepsen.util 13394 :invoke :write [24 47] INFO [2019-12-17 12:38:44,094] jepsen worker 28 - jepsen.util 11458 :ok :write [9 3] INFO [2019-12-17 12:38:44,094] jepsen worker 28 - jepsen.util 11458 :invoke :write [9 3] INFO [2019-12-17 12:38:44,101] jepsen worker 68 - jepsen.util 17618 :info :cas [22 [21 37]] :RequestCanceledException INFO [2019-12-17 12:38:44,102] jepsen worker 68 - jepsen.util 17708 :invoke :cas [22 [38 31]] INFO [2019-12-17 12:38:44,120] jepsen worker 44 - jepsen.util 19304 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:38:44,120] jepsen worker 1 - jepsen.util 18901 :info :cas [0 [40 6]] :RequestCanceledException INFO [2019-12-17 12:38:44,120] jepsen worker 44 - jepsen.util 19304 :invoke :read [14 nil] INFO [2019-12-17 12:38:44,121] jepsen worker 1 - jepsen.util 18991 :invoke :read [0 nil] INFO [2019-12-17 12:38:44,135] jepsen worker 75 - jepsen.util 18165 :info :cas [25 [8 40]] :RequestCanceledException INFO [2019-12-17 12:38:44,135] jepsen worker 10 - jepsen.util 13330 :info :cas [3 [9 21]] :RequestCanceledException INFO [2019-12-17 12:38:44,137] jepsen worker 75 - jepsen.util 18255 :invoke :write [25 42] INFO [2019-12-17 12:38:44,137] jepsen worker 10 - jepsen.util 13420 :invoke :write [3 11] INFO [2019-12-17 12:38:44,152] jepsen worker 9 - jepsen.util 14049 :ok :write [3 38] INFO [2019-12-17 12:38:44,152] jepsen worker 9 - jepsen.util 14049 :invoke :read [3 nil] INFO [2019-12-17 12:38:44,153] jepsen worker 20 - jepsen.util 13340 :ok :write [6 26] INFO [2019-12-17 12:38:44,153] jepsen worker 20 - jepsen.util 13340 :invoke :read [6 nil] INFO [2019-12-17 12:38:44,159] jepsen worker 88 - jepsen.util 13408 :ok :read [29 39] INFO [2019-12-17 12:38:44,159] jepsen worker 88 - jepsen.util 13408 :invoke :cas [29 [27 20]] INFO [2019-12-17 12:38:44,161] jepsen worker 18 - jepsen.util 14148 :info :cas [6 [49 49]] :RequestCanceledException INFO [2019-12-17 12:38:44,162] jepsen worker 18 - jepsen.util 14238 :invoke :cas [6 [3 49]] INFO [2019-12-17 12:38:44,185] jepsen worker 20 - jepsen.util 13340 :ok :read [6 26] INFO [2019-12-17 12:38:44,185] jepsen worker 20 - jepsen.util 13340 :invoke :cas [6 [36 13]] INFO [2019-12-17 12:38:44,190] jepsen worker 18 - jepsen.util 14238 :fail :cas [6 [3 49]] :ValueNotSwapFrom INFO [2019-12-17 12:38:44,190] jepsen worker 18 - jepsen.util 14238 :invoke :cas [6 [43 8]] INFO [2019-12-17 12:38:44,202] jepsen worker 3 - jepsen.util 13143 :fail :cas [1 [20 8]] :ValueNotSwapFrom INFO [2019-12-17 12:38:44,202] jepsen worker 3 - jepsen.util 13143 :invoke :write [1 10] INFO [2019-12-17 12:38:44,225] jepsen worker 20 - jepsen.util 13340 :fail :cas [6 [36 13]] :ValueNotSwapFrom INFO [2019-12-17 12:38:44,225] jepsen worker 20 - jepsen.util 13340 :invoke :write [6 0] INFO [2019-12-17 12:38:44,231] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78340,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:38:44,231] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78347,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:38:44,231] jepsen worker 6 - jepsen.util 18636 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:38:44,231] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78340,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:38:44,231] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78347,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:38:44,232] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78342,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:38:44,232] jepsen worker 6 - jepsen.util 18636 :invoke :cas [2 [37 20]] INFO [2019-12-17 12:38:44,232] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78352,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:38:44,232] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78342,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:38:44,233] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78352,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:38:44,234] jepsen worker 18 - jepsen.util 14238 :fail :cas [6 [43 8]] :ValueNotSwapFrom INFO [2019-12-17 12:38:44,234] jepsen worker 18 - jepsen.util 14238 :invoke :cas [6 [2 14]] INFO [2019-12-17 12:38:44,249] jepsen worker 64 - jepsen.util 10594 :ok :write [21 17] INFO [2019-12-17 12:38:44,249] jepsen worker 64 - jepsen.util 10594 :invoke :read [21 nil] INFO [2019-12-17 12:38:44,249] jepsen worker 51 - jepsen.util 12831 :fail :cas [17 [48 12]] :ValueNotSwapFrom INFO [2019-12-17 12:38:44,249] jepsen worker 51 - jepsen.util 12831 :invoke :write [17 18] INFO [2019-12-17 12:38:44,266] jepsen worker 30 - jepsen.util 12900 :info :cas [10 [12 23]] :RequestCanceledException INFO [2019-12-17 12:38:44,267] jepsen worker 52 - jepsen.util 13102 :fail :cas [17 [39 47]] :ValueNotSwapFrom INFO [2019-12-17 12:38:44,267] jepsen worker 52 - jepsen.util 13102 :invoke :read [17 nil] INFO [2019-12-17 12:38:44,267] jepsen worker 30 - jepsen.util 12990 :invoke :cas [10 [17 38]] INFO [2019-12-17 12:38:44,290] jepsen worker 18 - jepsen.util 14238 :fail :cas [6 [2 14]] :ValueNotSwapFrom INFO [2019-12-17 12:38:44,290] jepsen worker 18 - jepsen.util 14238 :invoke :cas [6 [45 12]] INFO [2019-12-17 12:38:44,303] jepsen worker 5 - jepsen.util 13145 :fail :cas [1 [10 42]] :ValueNotSwapFrom INFO [2019-12-17 12:38:44,303] jepsen worker 28 - jepsen.util 11458 :ok :write [9 3] INFO [2019-12-17 12:38:44,303] jepsen worker 5 - jepsen.util 13145 :invoke :cas [1 [33 48]] INFO [2019-12-17 12:38:44,303] jepsen worker 28 - jepsen.util 11458 :invoke :read [9 nil] INFO [2019-12-17 12:38:44,324] jepsen worker 18 - jepsen.util 14238 :fail :cas [6 [45 12]] :ValueNotSwapFrom INFO [2019-12-17 12:38:44,324] jepsen worker 18 - jepsen.util 14238 :invoke :write [6 5] INFO [2019-12-17 12:38:44,334] jepsen worker 28 - jepsen.util 11458 :ok :read [9 3] INFO [2019-12-17 12:38:44,334] jepsen worker 28 - jepsen.util 11458 :invoke :read [9 nil] INFO [2019-12-17 12:38:44,345] jepsen worker 28 - jepsen.util 11458 :ok :read [9 3] INFO [2019-12-17 12:38:44,346] jepsen worker 28 - jepsen.util 11458 :invoke :cas [9 [25 6]] INFO [2019-12-17 12:38:44,350] jepsen worker 34 - jepsen.util 19564 :info :cas [11 [47 38]] :RequestCanceledException INFO [2019-12-17 12:38:44,351] jepsen worker 34 - jepsen.util 19654 :invoke :cas [11 [0 22]] INFO [2019-12-17 12:38:44,366] jepsen worker 28 - jepsen.util 11458 :fail :cas [9 [25 6]] :ValueNotSwapFrom INFO [2019-12-17 12:38:44,366] jepsen worker 28 - jepsen.util 11458 :invoke :cas [9 [37 37]] INFO [2019-12-17 12:38:44,390] jepsen worker 71 - jepsen.util 14381 :ok :read [23 15] INFO [2019-12-17 12:38:44,390] jepsen worker 71 - jepsen.util 14381 :invoke :write [23 46] INFO [2019-12-17 12:38:44,392] jepsen worker 28 - jepsen.util 11458 :fail :cas [9 [37 37]] :ValueNotSwapFrom INFO [2019-12-17 12:38:44,392] jepsen worker 28 - jepsen.util 11458 :invoke :read [9 nil] INFO [2019-12-17 12:38:44,403] jepsen worker 28 - jepsen.util 11458 :ok :read [9 3] INFO [2019-12-17 12:38:44,403] jepsen worker 28 - jepsen.util 11458 :invoke :write [9 41] INFO [2019-12-17 12:38:44,436] jepsen worker 61 - jepsen.util 19861 :info :cas [20 [40 14]] :RequestCanceledException INFO [2019-12-17 12:38:44,437] jepsen worker 61 - jepsen.util 19951 :invoke :read [20 nil] INFO [2019-12-17 12:38:44,442] jepsen worker 79 - jepsen.util 13759 :fail :cas [26 [33 41]] :ValueNotSwapFrom INFO [2019-12-17 12:38:44,442] jepsen worker 79 - jepsen.util 13759 :invoke :read [26 nil] INFO [2019-12-17 12:38:44,456] jepsen worker 28 - jepsen.util 11458 :ok :write [9 41] INFO [2019-12-17 12:38:44,456] jepsen worker 28 - jepsen.util 11458 :invoke :write [9 42] INFO [2019-12-17 12:38:44,466] jepsen worker 81 - jepsen.util 17901 :info :write [27 28] :RequestCanceledException INFO [2019-12-17 12:38:44,468] jepsen worker 81 - jepsen.util 17991 :invoke :read [27 nil] INFO [2019-12-17 12:38:44,500] jepsen worker 78 - jepsen.util 13398 :info :write [26 42] :RequestCanceledException INFO [2019-12-17 12:38:44,501] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78372,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:38:44,501] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78372,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:38:44,501] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78376,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:38:44,501] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78376,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:38:44,502] jepsen worker 78 - jepsen.util 13488 :invoke :write [26 10] INFO [2019-12-17 12:38:44,515] jepsen worker 88 - jepsen.util 13408 :fail :cas [29 [27 20]] :ValueNotSwapFrom INFO [2019-12-17 12:38:44,515] jepsen worker 88 - jepsen.util 13408 :invoke :write [29 41] INFO [2019-12-17 12:38:44,515] jepsen worker 28 - jepsen.util 11458 :ok :write [9 42] INFO [2019-12-17 12:38:44,515] jepsen worker 28 - jepsen.util 11458 :invoke :cas [9 [38 18]] INFO [2019-12-17 12:38:44,520] jepsen worker 26 - jepsen.util 19466 :info :write [8 26] :RequestCanceledException INFO [2019-12-17 12:38:44,521] jepsen worker 26 - jepsen.util 19556 :invoke :read [8 nil] INFO [2019-12-17 12:38:44,535] jepsen worker 35 - jepsen.util 18755 :info :cas [11 [11 4]] :RequestCanceledException INFO [2019-12-17 12:38:44,535] jepsen worker 83 - jepsen.util 18353 :info :cas [27 [21 6]] :RequestCanceledException INFO [2019-12-17 12:38:44,535] jepsen worker 28 - jepsen.util 11458 :fail :cas [9 [38 18]] :ValueNotSwapFrom INFO [2019-12-17 12:38:44,535] jepsen worker 28 - jepsen.util 11458 :invoke :read [9 nil] INFO [2019-12-17 12:38:44,536] jepsen worker 35 - jepsen.util 18845 :invoke :read [11 nil] INFO [2019-12-17 12:38:44,536] jepsen worker 83 - jepsen.util 18443 :invoke :write [27 38] INFO [2019-12-17 12:38:44,555] jepsen worker 28 - jepsen.util 11458 :ok :read [9 42] INFO [2019-12-17 12:38:44,555] jepsen worker 28 - jepsen.util 11458 :invoke :read [9 nil] INFO [2019-12-17 12:38:44,576] jepsen worker 28 - jepsen.util 11458 :ok :read [9 42] INFO [2019-12-17 12:38:44,576] jepsen worker 28 - jepsen.util 11458 :invoke :read [9 nil] INFO [2019-12-17 12:38:44,588] jepsen worker 28 - jepsen.util 11458 :ok :read [9 42] INFO [2019-12-17 12:38:44,588] jepsen worker 28 - jepsen.util 11458 :invoke :read [9 nil] INFO [2019-12-17 12:38:44,601] jepsen worker 28 - jepsen.util 11458 :ok :read [9 42] INFO [2019-12-17 12:38:44,601] jepsen worker 28 - jepsen.util 11458 :invoke :cas [9 [20 30]] INFO [2019-12-17 12:38:44,602] jepsen worker 52 - jepsen.util 13102 :ok :read [17 0] INFO [2019-12-17 12:38:44,602] jepsen worker 52 - jepsen.util 13102 :invoke :cas [17 [46 37]] INFO [2019-12-17 12:38:44,617] jepsen worker 28 - jepsen.util 11458 :fail :cas [9 [20 30]] :ValueNotSwapFrom INFO [2019-12-17 12:38:44,617] jepsen worker 28 - jepsen.util 11458 :invoke :cas [9 [33 45]] INFO [2019-12-17 12:38:44,630] jepsen worker 28 - jepsen.util 11458 :fail :cas [9 [33 45]] :ValueNotSwapFrom INFO [2019-12-17 12:38:44,630] jepsen worker 28 - jepsen.util 11458 :invoke :write [9 5] INFO [2019-12-17 12:38:44,634] jepsen worker 45 - jepsen.util 13005 :info :write [15 36] :RequestCanceledException INFO [2019-12-17 12:38:44,635] jepsen worker 45 - jepsen.util 13095 :invoke :cas [15 [35 25]] INFO [2019-12-17 12:38:44,649] jepsen worker 72 - jepsen.util 13482 :info :write [24 37] :RequestCanceledException INFO [2019-12-17 12:38:44,650] jepsen worker 72 - jepsen.util 13572 :invoke :write [24 18] INFO [2019-12-17 12:38:44,651] jepsen worker 5 - jepsen.util 13145 :fail :cas [1 [33 48]] :ValueNotSwapFrom INFO [2019-12-17 12:38:44,651] jepsen worker 5 - jepsen.util 13145 :invoke :read [1 nil] INFO [2019-12-17 12:38:44,678] jepsen worker 28 - jepsen.util 11458 :ok :write [9 5] INFO [2019-12-17 12:38:44,678] jepsen worker 28 - jepsen.util 11458 :invoke :read [9 nil] INFO [2019-12-17 12:38:44,691] jepsen worker 28 - jepsen.util 11458 :ok :read [9 5] INFO [2019-12-17 12:38:44,691] jepsen worker 28 - jepsen.util 11458 :invoke :write [9 6] INFO [2019-12-17 12:38:44,721] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78397,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:38:44,721] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78397,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:38:44,721] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78401,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:38:44,721] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78401,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:38:44,733] jepsen worker 28 - jepsen.util 11458 :ok :write [9 6] INFO [2019-12-17 12:38:44,733] jepsen worker 28 - jepsen.util 11458 :invoke :cas [9 [29 29]] INFO [2019-12-17 12:38:44,734] jepsen worker 29 - jepsen.util 11639 :fail :read [9 nil] :RequestTimeoutException INFO [2019-12-17 12:38:44,734] jepsen worker 12 - jepsen.util 13512 :info :write [4 41] :RequestCanceledException INFO [2019-12-17 12:38:44,735] jepsen worker 29 - jepsen.util 11639 :invoke :write [9 34] INFO [2019-12-17 12:38:44,736] jepsen worker 12 - jepsen.util 13602 :invoke :read [4 nil] INFO [2019-12-17 12:38:44,750] jepsen worker 27 - jepsen.util 12357 :fail :read [9 nil] :RequestTimeoutException INFO [2019-12-17 12:38:44,750] jepsen worker 32 - jepsen.util 13802 :fail :read [10 nil] :RequestTimeoutException INFO [2019-12-17 12:38:44,751] jepsen worker 32 - jepsen.util 13802 :invoke :write [10 33] INFO [2019-12-17 12:38:44,751] jepsen worker 27 - jepsen.util 12357 :invoke :write [9 28] INFO [2019-12-17 12:38:44,756] jepsen worker 28 - jepsen.util 11458 :fail :cas [9 [29 29]] :ValueNotSwapFrom INFO [2019-12-17 12:38:44,756] jepsen worker 28 - jepsen.util 11458 :invoke :cas [9 [23 7]] INFO [2019-12-17 12:38:44,770] jepsen worker 56 - jepsen.util 14186 :ok :read [18 7] INFO [2019-12-17 12:38:44,770] jepsen worker 56 - jepsen.util 14186 :invoke :cas [18 [38 12]] INFO [2019-12-17 12:38:44,774] jepsen worker 28 - jepsen.util 11458 :fail :cas [9 [23 7]] :ValueNotSwapFrom INFO [2019-12-17 12:38:44,775] jepsen worker 28 - jepsen.util 11458 :invoke :read [9 nil] INFO [2019-12-17 12:38:44,789] jepsen worker 28 - jepsen.util 11458 :ok :read [9 6] INFO [2019-12-17 12:38:44,790] jepsen worker 28 - jepsen.util 11458 :invoke :write [9 20] INFO [2019-12-17 12:38:44,801] jepsen worker 69 - jepsen.util 13929 :fail :cas [23 [26 9]] :ValueNotSwapFrom INFO [2019-12-17 12:38:44,801] jepsen worker 69 - jepsen.util 13929 :invoke :cas [23 [14 10]] INFO [2019-12-17 12:38:44,802] jepsen worker 29 - jepsen.util 11639 :ok :write [9 34] INFO [2019-12-17 12:38:44,802] jepsen worker 29 - jepsen.util 11639 :invoke :read [9 nil] INFO [2019-12-17 12:38:44,818] jepsen worker 20 - jepsen.util 13340 :ok :write [6 0] INFO [2019-12-17 12:38:44,818] jepsen worker 20 - jepsen.util 13340 :invoke :write [6 37] INFO [2019-12-17 12:38:44,827] jepsen worker 10 - jepsen.util 13420 :ok :write [3 11] INFO [2019-12-17 12:38:44,828] jepsen worker 10 - jepsen.util 13420 :invoke :cas [3 [8 44]] INFO [2019-12-17 12:38:44,829] jepsen worker 74 - jepsen.util 13394 :ok :write [24 47] INFO [2019-12-17 12:38:44,829] jepsen worker 74 - jepsen.util 13394 :invoke :write [24 19] INFO [2019-12-17 12:38:44,831] jepsen worker 29 - jepsen.util 11639 :ok :read [9 34] INFO [2019-12-17 12:38:44,831] jepsen worker 29 - jepsen.util 11639 :invoke :read [9 nil] INFO [2019-12-17 12:38:44,835] jepsen worker 80 - jepsen.util 12860 :info :write [26 15] :RequestCanceledException INFO [2019-12-17 12:38:44,840] jepsen worker 80 - jepsen.util 12950 :invoke :write [26 48] INFO [2019-12-17 12:38:44,843] jepsen worker 79 - jepsen.util 13759 :ok :read [26 32] INFO [2019-12-17 12:38:44,843] jepsen worker 79 - jepsen.util 13759 :invoke :read [26 nil] INFO [2019-12-17 12:38:44,900] jepsen worker 47 - jepsen.util 14717 :fail :read [15 nil] :RequestTimeoutException INFO [2019-12-17 12:38:44,900] jepsen worker 47 - jepsen.util 14717 :invoke :write [15 37] INFO [2019-12-17 12:38:44,905] jepsen worker 64 - jepsen.util 10594 :ok :read [21 17] INFO [2019-12-17 12:38:44,905] jepsen worker 64 - jepsen.util 10594 :invoke :cas [21 [19 47]] INFO [2019-12-17 12:38:44,911] jepsen worker 28 - jepsen.util 11458 :ok :write [9 20] INFO [2019-12-17 12:38:44,911] jepsen worker 28 - jepsen.util 11458 :invoke :write [9 46] INFO [2019-12-17 12:38:44,913] jepsen worker 30 - jepsen.util 12990 :fail :cas [10 [17 38]] :ValueNotSwapFrom INFO [2019-12-17 12:38:44,913] jepsen worker 30 - jepsen.util 12990 :invoke :cas [10 [33 24]] INFO [2019-12-17 12:38:44,927] jepsen worker 29 - jepsen.util 11639 :ok :read [9 34] INFO [2019-12-17 12:38:44,927] jepsen worker 29 - jepsen.util 11639 :invoke :write [9 19] INFO [2019-12-17 12:38:44,935] jepsen worker 31 - jepsen.util 11731 :fail :read [10 nil] :RequestTimeoutException INFO [2019-12-17 12:38:44,935] jepsen worker 31 - jepsen.util 11731 :invoke :write [10 1] INFO [2019-12-17 12:38:44,939] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78424,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:38:44,939] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78424,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:38:44,940] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78428,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:38:44,940] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78428,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:38:44,963] jepsen worker 13 - jepsen.util 13063 :info :write [4 34] :RequestCanceledException INFO [2019-12-17 12:38:44,968] jepsen worker 13 - jepsen.util 13153 :invoke :write [4 3] INFO [2019-12-17 12:38:44,974] jepsen worker 10 - jepsen.util 13420 :fail :cas [3 [8 44]] :ValueNotSwapFrom INFO [2019-12-17 12:38:44,974] jepsen worker 10 - jepsen.util 13420 :invoke :read [3 nil] INFO [2019-12-17 12:38:44,988] jepsen worker 64 - jepsen.util 10594 :fail :cas [21 [19 47]] :ValueNotSwapFrom INFO [2019-12-17 12:38:44,988] jepsen worker 64 - jepsen.util 10594 :invoke :cas [21 [14 25]] INFO [2019-12-17 12:38:45,020] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:38:45,020] jepsen nemesis - couchbase.util Rebalance check count: 616 INFO [2019-12-17 12:38:45,020] jepsen nemesis - couchbase.util Rebalance stuck count: 597 INFO [2019-12-17 12:38:45,048] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78439,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:38:45,048] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78439,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:38:45,049] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78442,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:38:45,049] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78442,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:38:45,049] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78443,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:38:45,049] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78443,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:38:45,070] jepsen worker 32 - jepsen.util 13802 :ok :write [10 33] INFO [2019-12-17 12:38:45,071] jepsen worker 32 - jepsen.util 13802 :invoke :cas [10 [34 6]] INFO [2019-12-17 12:38:45,081] jepsen worker 10 - jepsen.util 13420 :ok :read [3 11] INFO [2019-12-17 12:38:45,081] jepsen worker 10 - jepsen.util 13420 :invoke :read [3 nil] INFO [2019-12-17 12:38:45,094] jepsen worker 64 - jepsen.util 10594 :fail :cas [21 [14 25]] :ValueNotSwapFrom INFO [2019-12-17 12:38:45,094] jepsen worker 64 - jepsen.util 10594 :invoke :cas [21 [0 25]] INFO [2019-12-17 12:38:45,127] jepsen worker 10 - jepsen.util 13420 :ok :read [3 11] INFO [2019-12-17 12:38:45,128] jepsen worker 10 - jepsen.util 13420 :invoke :read [3 nil] INFO [2019-12-17 12:38:45,167] jepsen worker 14 - jepsen.util 13064 :info :write [4 43] :RequestCanceledException INFO [2019-12-17 12:38:45,168] jepsen worker 14 - jepsen.util 13154 :invoke :cas [4 [8 9]] INFO [2019-12-17 12:38:45,175] jepsen worker 64 - jepsen.util 10594 :fail :cas [21 [0 25]] :ValueNotSwapFrom INFO [2019-12-17 12:38:45,175] jepsen worker 64 - jepsen.util 10594 :invoke :write [21 42] INFO [2019-12-17 12:38:45,181] jepsen worker 52 - jepsen.util 13102 :fail :cas [17 [46 37]] :ValueNotSwapFrom INFO [2019-12-17 12:38:45,181] jepsen worker 52 - jepsen.util 13102 :invoke :read [17 nil] INFO [2019-12-17 12:38:45,188] jepsen worker 10 - jepsen.util 13420 :ok :read [3 11] INFO [2019-12-17 12:38:45,189] jepsen worker 10 - jepsen.util 13420 :invoke :read [3 nil] INFO [2019-12-17 12:38:45,189] jepsen worker 27 - jepsen.util 12357 :ok :write [9 28] INFO [2019-12-17 12:38:45,190] jepsen worker 27 - jepsen.util 12357 :invoke :read [9 nil] INFO [2019-12-17 12:38:45,209] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78454,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:38:45,209] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78454,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:38:45,209] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78455,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:38:45,209] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78455,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:38:45,209] jepsen worker 56 - jepsen.util 14186 :fail :cas [18 [38 12]] :ValueNotSwapFrom INFO [2019-12-17 12:38:45,209] jepsen worker 56 - jepsen.util 14186 :invoke :cas [18 [23 24]] INFO [2019-12-17 12:38:45,225] jepsen worker 32 - jepsen.util 13802 :fail :cas [10 [34 6]] :ValueNotSwapFrom INFO [2019-12-17 12:38:45,225] jepsen worker 32 - jepsen.util 13802 :invoke :cas [10 [49 12]] INFO [2019-12-17 12:38:45,233] jepsen worker 70 - jepsen.util 13840 :info :write [23 10] :RequestCanceledException INFO [2019-12-17 12:38:45,237] jepsen worker 29 - jepsen.util 11639 :ok :write [9 19] INFO [2019-12-17 12:38:45,237] jepsen worker 29 - jepsen.util 11639 :invoke :write [9 42] INFO [2019-12-17 12:38:45,239] jepsen worker 70 - jepsen.util 13930 :invoke :cas [23 [19 35]] INFO [2019-12-17 12:38:45,247] jepsen worker 89 - jepsen.util 13589 :info :write [29 35] :RequestCanceledException INFO [2019-12-17 12:38:45,254] jepsen worker 89 - jepsen.util 13679 :invoke :write [29 12] INFO [2019-12-17 12:38:45,269] jepsen worker 27 - jepsen.util 12357 :ok :read [9 28] INFO [2019-12-17 12:38:45,269] jepsen worker 27 - jepsen.util 12357 :invoke :write [9 1] INFO [2019-12-17 12:38:45,272] jepsen worker 5 - jepsen.util 13145 :ok :read [1 45] INFO [2019-12-17 12:38:45,273] jepsen worker 5 - jepsen.util 13145 :invoke :read [1 nil] INFO [2019-12-17 12:38:45,293] jepsen worker 69 - jepsen.util 13929 :fail :cas [23 [14 10]] :ValueNotSwapFrom INFO [2019-12-17 12:38:45,293] jepsen worker 69 - jepsen.util 13929 :invoke :cas [23 [45 8]] INFO [2019-12-17 12:38:45,294] jepsen worker 32 - jepsen.util 13802 :fail :cas [10 [49 12]] :ValueNotSwapFrom INFO [2019-12-17 12:38:45,294] jepsen worker 32 - jepsen.util 13802 :invoke :cas [10 [32 14]] INFO [2019-12-17 12:38:45,302] jepsen worker 46 - jepsen.util 13096 :info :cas [15 [4 41]] :RequestCanceledException INFO [2019-12-17 12:38:45,303] jepsen worker 46 - jepsen.util 13186 :invoke :cas [15 [20 18]] INFO [2019-12-17 12:38:45,319] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78461,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:38:45,319] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78461,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:38:45,319] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78464,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:38:45,319] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78464,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:38:45,319] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78465,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:38:45,320] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78465,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:38:45,344] jepsen worker 79 - jepsen.util 13759 :ok :read [26 32] INFO [2019-12-17 12:38:45,344] jepsen worker 79 - jepsen.util 13759 :invoke :cas [26 [29 33]] INFO [2019-12-17 12:38:45,356] jepsen worker 29 - jepsen.util 11639 :ok :write [9 42] INFO [2019-12-17 12:38:45,356] jepsen worker 29 - jepsen.util 11639 :invoke :write [9 9] INFO [2019-12-17 12:38:45,366] jepsen worker 37 - jepsen.util 12907 :info :write [12 39] :RequestCanceledException INFO [2019-12-17 12:38:45,368] jepsen worker 37 - jepsen.util 12997 :invoke :write [12 24] INFO [2019-12-17 12:38:45,374] jepsen worker 64 - jepsen.util 10594 :ok :write [21 42] INFO [2019-12-17 12:38:45,374] jepsen worker 64 - jepsen.util 10594 :invoke :write [21 24] INFO [2019-12-17 12:38:45,398] jepsen worker 27 - jepsen.util 12357 :ok :write [9 1] INFO [2019-12-17 12:38:45,398] jepsen worker 27 - jepsen.util 12357 :invoke :write [9 49] INFO [2019-12-17 12:38:45,402] jepsen worker 55 - jepsen.util 14095 :info :write [18 28] :RequestCanceledException INFO [2019-12-17 12:38:45,406] jepsen worker 55 - jepsen.util 14185 :invoke :cas [18 [19 35]] INFO [2019-12-17 12:38:45,432] jepsen worker 10 - jepsen.util 13420 :ok :read [3 11] INFO [2019-12-17 12:38:45,432] jepsen worker 10 - jepsen.util 13420 :invoke :write [3 5] INFO [2019-12-17 12:38:45,457] jepsen worker 5 - jepsen.util 13145 :ok :read [1 45] INFO [2019-12-17 12:38:45,457] jepsen worker 5 - jepsen.util 13145 :invoke :read [1 nil] INFO [2019-12-17 12:38:45,467] jepsen worker 29 - jepsen.util 11639 :ok :write [9 9] INFO [2019-12-17 12:38:45,467] jepsen worker 29 - jepsen.util 11639 :invoke :read [9 nil] INFO [2019-12-17 12:38:45,475] jepsen worker 56 - jepsen.util 14186 :fail :cas [18 [23 24]] :ValueNotSwapFrom INFO [2019-12-17 12:38:45,475] jepsen worker 56 - jepsen.util 14186 :invoke :write [18 21] INFO [2019-12-17 12:38:45,476] jepsen worker 14 - jepsen.util 13154 :fail :cas [4 [8 9]] :ValueNotSwapFrom INFO [2019-12-17 12:38:45,476] jepsen worker 14 - jepsen.util 13154 :invoke :write [4 39] INFO [2019-12-17 12:38:45,488] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78483,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:38:45,489] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78483,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:38:45,489] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78486,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:38:45,489] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78486,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:38:45,494] jepsen worker 46 - jepsen.util 13186 :fail :cas [15 [20 18]] :ValueNotSwapFrom INFO [2019-12-17 12:38:45,494] jepsen worker 46 - jepsen.util 13186 :invoke :cas [15 [40 5]] INFO [2019-12-17 12:38:45,502] jepsen worker 32 - jepsen.util 13802 :fail :cas [10 [32 14]] :ValueNotSwapFrom INFO [2019-12-17 12:38:45,502] jepsen worker 32 - jepsen.util 13802 :invoke :read [10 nil] INFO [2019-12-17 12:38:45,517] jepsen worker 29 - jepsen.util 11639 :ok :read [9 9] INFO [2019-12-17 12:38:45,517] jepsen worker 29 - jepsen.util 11639 :invoke :write [9 16] INFO [2019-12-17 12:38:45,531] jepsen worker 64 - jepsen.util 10594 :ok :write [21 24] INFO [2019-12-17 12:38:45,531] jepsen worker 64 - jepsen.util 10594 :invoke :write [21 19] INFO [2019-12-17 12:38:45,534] jepsen worker 53 - jepsen.util 13283 :info :write [17 8] :RequestCanceledException INFO [2019-12-17 12:38:45,540] jepsen worker 53 - jepsen.util 13373 :invoke :write [17 40] INFO [2019-12-17 12:38:45,546] jepsen worker 12 - jepsen.util 13602 :ok :read [4 3] INFO [2019-12-17 12:38:45,546] jepsen worker 12 - jepsen.util 13602 :invoke :read [4 nil] INFO [2019-12-17 12:38:45,580] jepsen worker 27 - jepsen.util 12357 :ok :write [9 49] INFO [2019-12-17 12:38:45,580] jepsen worker 27 - jepsen.util 12357 :invoke :write [9 29] INFO [2019-12-17 12:38:45,581] jepsen worker 47 - jepsen.util 14717 :ok :write [15 37] INFO [2019-12-17 12:38:45,581] jepsen worker 20 - jepsen.util 13340 :ok :write [6 37] INFO [2019-12-17 12:38:45,581] jepsen worker 47 - jepsen.util 14717 :invoke :write [15 41] INFO [2019-12-17 12:38:45,581] jepsen worker 20 - jepsen.util 13340 :invoke :read [6 nil] INFO [2019-12-17 12:38:45,734] jepsen worker 19 - jepsen.util 14239 :info :write [6 10] :RequestCanceledException INFO [2019-12-17 12:38:45,735] jepsen worker 36 - jepsen.util 14166 :info :write [12 47] :RequestCanceledException INFO [2019-12-17 12:38:45,735] jepsen worker 73 - jepsen.util 12403 :info :cas [24 [22 34]] :RequestCanceledException INFO [2019-12-17 12:38:45,735] jepsen worker 65 - jepsen.util 11765 :info :write [21 46] :RequestCanceledException INFO [2019-12-17 12:38:45,735] jepsen worker 54 - jepsen.util 14274 :info :write [18 6] :RequestCanceledException INFO [2019-12-17 12:38:45,735] jepsen worker 29 - jepsen.util 11639 :ok :write [9 16] INFO [2019-12-17 12:38:45,735] jepsen worker 11 - jepsen.util 13331 :fail :read [3 nil] :RequestTimeoutException INFO [2019-12-17 12:38:45,736] jepsen worker 19 - jepsen.util 14329 :invoke :read [6 nil] INFO [2019-12-17 12:38:45,736] jepsen worker 29 - jepsen.util 11639 :invoke :write [9 10] INFO [2019-12-17 12:38:45,736] jepsen worker 36 - jepsen.util 14256 :invoke :write [12 28] INFO [2019-12-17 12:38:45,736] jepsen worker 11 - jepsen.util 13331 :invoke :write [3 30] INFO [2019-12-17 12:38:45,736] jepsen worker 65 - jepsen.util 11855 :invoke :cas [21 [42 24]] INFO [2019-12-17 12:38:45,736] jepsen worker 73 - jepsen.util 12493 :invoke :cas [24 [43 29]] INFO [2019-12-17 12:38:45,736] jepsen worker 54 - jepsen.util 14364 :invoke :write [18 36] INFO [2019-12-17 12:38:45,740] jepsen worker 4 - jepsen.util 12784 :info :write [1 10] :RequestCanceledException INFO [2019-12-17 12:38:45,741] jepsen worker 20 - jepsen.util 13340 :ok :read [6 37] INFO [2019-12-17 12:38:45,741] jepsen worker 20 - jepsen.util 13340 :invoke :write [6 37] INFO [2019-12-17 12:38:45,743] jepsen worker 4 - jepsen.util 12874 :invoke :read [1 nil] INFO [2019-12-17 12:38:45,756] jepsen worker 70 - jepsen.util 13930 :fail :cas [23 [19 35]] :ValueNotSwapFrom INFO [2019-12-17 12:38:45,756] jepsen worker 70 - jepsen.util 13930 :invoke :cas [23 [3 32]] INFO [2019-12-17 12:38:45,766] jepsen worker 19 - jepsen.util 14329 :ok :read [6 37] INFO [2019-12-17 12:38:45,766] jepsen worker 19 - jepsen.util 14329 :invoke :read [6 nil] INFO [2019-12-17 12:38:45,774] jepsen worker 12 - jepsen.util 13602 :ok :read [4 3] INFO [2019-12-17 12:38:45,774] jepsen worker 12 - jepsen.util 13602 :invoke :read [4 nil] INFO [2019-12-17 12:38:45,794] jepsen worker 19 - jepsen.util 14329 :ok :read [6 37] INFO [2019-12-17 12:38:45,794] jepsen worker 19 - jepsen.util 14329 :invoke :read [6 nil] INFO [2019-12-17 12:38:45,809] jepsen worker 55 - jepsen.util 14185 :fail :cas [18 [19 35]] :ValueNotSwapFrom INFO [2019-12-17 12:38:45,809] jepsen worker 55 - jepsen.util 14185 :invoke :write [18 40] INFO [2019-12-17 12:38:45,811] jepsen worker 73 - jepsen.util 12493 :fail :cas [24 [43 29]] :ValueNotSwapFrom INFO [2019-12-17 12:38:45,811] jepsen worker 73 - jepsen.util 12493 :invoke :read [24 nil] INFO [2019-12-17 12:38:45,822] jepsen worker 79 - jepsen.util 13759 :fail :cas [26 [29 33]] :ValueNotSwapFrom INFO [2019-12-17 12:38:45,822] jepsen worker 79 - jepsen.util 13759 :invoke :write [26 23] INFO [2019-12-17 12:38:45,823] jepsen worker 19 - jepsen.util 14329 :ok :read [6 37] INFO [2019-12-17 12:38:45,823] jepsen worker 19 - jepsen.util 14329 :invoke :cas [6 [47 5]] INFO [2019-12-17 12:38:45,836] jepsen worker 19 - jepsen.util 14329 :fail :cas [6 [47 5]] :ValueNotSwapFrom INFO [2019-12-17 12:38:45,836] jepsen worker 19 - jepsen.util 14329 :invoke :cas [6 [26 13]] INFO [2019-12-17 12:38:45,844] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78502,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:38:45,844] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78502,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:38:45,844] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78505,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:38:45,844] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78505,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:38:45,844] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78506,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:38:45,844] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78506,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:38:45,847] jepsen worker 19 - jepsen.util 14329 :fail :cas [6 [26 13]] :ValueNotSwapFrom INFO [2019-12-17 12:38:45,847] jepsen worker 19 - jepsen.util 14329 :invoke :write [6 43] INFO [2019-12-17 12:38:45,857] jepsen worker 69 - jepsen.util 13929 :fail :cas [23 [45 8]] :ValueNotSwapFrom INFO [2019-12-17 12:38:45,857] jepsen worker 69 - jepsen.util 13929 :invoke :read [23 nil] INFO [2019-12-17 12:38:45,866] jepsen worker 67 - jepsen.util 18967 :info :cas [22 [25 16]] :RequestCanceledException INFO [2019-12-17 12:38:45,867] jepsen worker 67 - jepsen.util 19057 :invoke :write [22 11] INFO [2019-12-17 12:38:45,902] jepsen worker 40 - jepsen.util 19120 :info :write [13 35] :RequestCanceledException INFO [2019-12-17 12:38:45,903] jepsen worker 40 - jepsen.util 19210 :invoke :read [13 nil] INFO [2019-12-17 12:38:45,916] jepsen worker 29 - jepsen.util 11639 :ok :write [9 10] INFO [2019-12-17 12:38:45,917] jepsen worker 29 - jepsen.util 11639 :invoke :write [9 10] INFO [2019-12-17 12:38:45,917] jepsen worker 63 - jepsen.util 11943 :fail :read [21 nil] :RequestTimeoutException INFO [2019-12-17 12:38:45,917] jepsen worker 63 - jepsen.util 11943 :invoke :write [21 40] INFO [2019-12-17 12:38:45,933] jepsen worker 22 - jepsen.util 19012 :info :cas [7 [42 38]] :RequestCanceledException INFO [2019-12-17 12:38:45,933] jepsen worker 57 - jepsen.util 18147 :info :cas [19 [31 19]] :RequestCanceledException INFO [2019-12-17 12:38:45,933] jepsen worker 7 - jepsen.util 18457 :info :cas [2 [1 38]] :RequestCanceledException INFO [2019-12-17 12:38:45,934] jepsen worker 57 - jepsen.util 18237 :invoke :read [19 nil] INFO [2019-12-17 12:38:45,934] jepsen worker 7 - jepsen.util 18547 :invoke :write [2 28] INFO [2019-12-17 12:38:45,938] jepsen worker 22 - jepsen.util 19102 :invoke :cas [7 [28 34]] INFO [2019-12-17 12:38:45,942] jepsen worker 73 - jepsen.util 12493 :ok :read [24 47] INFO [2019-12-17 12:38:45,942] jepsen worker 73 - jepsen.util 12493 :invoke :cas [24 [27 48]] INFO [2019-12-17 12:38:45,949] jepsen worker 85 - jepsen.util 19075 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:38:45,949] jepsen worker 24 - jepsen.util 17664 :info :write [8 32] :RequestCanceledException INFO [2019-12-17 12:38:45,949] jepsen worker 85 - jepsen.util 19075 :invoke :write [28 25] INFO [2019-12-17 12:38:45,950] jepsen worker 24 - jepsen.util 17754 :invoke :cas [8 [27 47]] INFO [2019-12-17 12:38:45,952] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78528,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:38:45,952] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78528,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:38:45,952] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78531,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:38:45,952] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78531,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:38:45,952] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78532,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:38:45,952] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78532,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:38:45,965] jepsen worker 10 - jepsen.util 13420 :ok :write [3 5] INFO [2019-12-17 12:38:45,965] jepsen worker 10 - jepsen.util 13420 :invoke :read [3 nil] INFO [2019-12-17 12:38:45,965] jepsen worker 70 - jepsen.util 13930 :fail :cas [23 [3 32]] :ValueNotSwapFrom INFO [2019-12-17 12:38:45,965] jepsen worker 70 - jepsen.util 13930 :invoke :write [23 21] INFO [2019-12-17 12:38:45,965] jepsen worker 2 - jepsen.util 17912 :info :write [0 20] :RequestCanceledException INFO [2019-12-17 12:38:45,967] jepsen worker 2 - jepsen.util 18002 :invoke :write [0 15] INFO [2019-12-17 12:38:45,985] jepsen worker 73 - jepsen.util 12493 :fail :cas [24 [27 48]] :ValueNotSwapFrom INFO [2019-12-17 12:38:45,986] jepsen worker 73 - jepsen.util 12493 :invoke :read [24 nil] INFO [2019-12-17 12:38:46,001] jepsen worker 59 - jepsen.util 17699 :info :write [19 11] :RequestCanceledException INFO [2019-12-17 12:38:46,003] jepsen worker 59 - jepsen.util 17789 :invoke :write [19 20] INFO [2019-12-17 12:38:46,020] jepsen worker 16 - jepsen.util 20266 :info :write [5 49] :RequestCanceledException INFO [2019-12-17 12:38:46,020] jepsen worker 84 - jepsen.util 17904 :info :cas [28 [34 40]] :RequestCanceledException INFO [2019-12-17 12:38:46,020] jepsen worker 73 - jepsen.util 12493 :ok :read [24 47] INFO [2019-12-17 12:38:46,020] jepsen worker 38 - jepsen.util 14168 :info :write [12 46] :RequestCanceledException INFO [2019-12-17 12:38:46,020] jepsen worker 73 - jepsen.util 12493 :invoke :write [24 42] INFO [2019-12-17 12:38:46,021] jepsen worker 38 - jepsen.util 14258 :invoke :read [12 nil] INFO [2019-12-17 12:38:46,021] jepsen worker 84 - jepsen.util 17994 :invoke :write [28 4] INFO [2019-12-17 12:38:46,021] jepsen worker 16 - jepsen.util 20356 :invoke :cas [5 [46 18]] INFO [2019-12-17 12:38:46,035] jepsen worker 76 - jepsen.util 18706 :info :cas [25 [16 14]] :RequestCanceledException INFO [2019-12-17 12:38:46,035] jepsen worker 87 - jepsen.util 13227 :info :write [29 39] :RequestCanceledException INFO [2019-12-17 12:38:46,035] jepsen worker 33 - jepsen.util 18303 :info :write [11 2] :RequestCanceledException INFO [2019-12-17 12:38:46,035] jepsen worker 60 - jepsen.util 19680 :info :cas [20 [20 41]] :RequestCanceledException INFO [2019-12-17 12:38:46,036] jepsen worker 60 - jepsen.util 19770 :invoke :write [20 9] INFO [2019-12-17 12:38:46,036] jepsen worker 87 - jepsen.util 13317 :invoke :read [29 nil] INFO [2019-12-17 12:38:46,036] jepsen worker 76 - jepsen.util 18796 :invoke :write [25 20] INFO [2019-12-17 12:38:46,038] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:38:46,038] jepsen nemesis - couchbase.util Rebalance check count: 617 INFO [2019-12-17 12:38:46,038] jepsen nemesis - couchbase.util Rebalance stuck count: 598 INFO [2019-12-17 12:38:46,040] jepsen worker 33 - jepsen.util 18393 :invoke :cas [11 [47 7]] INFO [2019-12-17 12:38:46,049] jepsen worker 62 - jepsen.util 19142 :info :write [20 27] :RequestCanceledException INFO [2019-12-17 12:38:46,049] jepsen worker 66 - jepsen.util 18786 :info :cas [22 [0 46]] :RequestCanceledException INFO [2019-12-17 12:38:46,050] jepsen worker 62 - jepsen.util 19232 :invoke :write [20 3] INFO [2019-12-17 12:38:46,050] jepsen worker 66 - jepsen.util 18876 :invoke :cas [22 [13 28]] INFO [2019-12-17 12:38:46,058] jepsen worker 47 - jepsen.util 14717 :ok :write [15 41] INFO [2019-12-17 12:38:46,058] jepsen worker 47 - jepsen.util 14717 :invoke :read [15 nil] INFO [2019-12-17 12:38:46,059] jepsen worker 20 - jepsen.util 13340 :ok :write [6 37] INFO [2019-12-17 12:38:46,059] jepsen worker 20 - jepsen.util 13340 :invoke :write [6 25] INFO [2019-12-17 12:38:46,061] jepsen worker 27 - jepsen.util 12357 :ok :write [9 29] INFO [2019-12-17 12:38:46,061] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78549,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:38:46,061] jepsen worker 27 - jepsen.util 12357 :invoke :write [9 49] INFO [2019-12-17 12:38:46,062] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78549,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:38:46,062] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78552,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:38:46,062] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78552,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:38:46,066] jepsen worker 77 - jepsen.util 19697 :info :cas [25 [48 41]] :RequestCanceledException INFO [2019-12-17 12:38:46,066] jepsen worker 23 - jepsen.util 18833 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:38:46,066] jepsen worker 49 - jepsen.util 18409 :info :cas [16 [46 36]] :RequestCanceledException INFO [2019-12-17 12:38:46,066] jepsen worker 58 - jepsen.util 18778 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:38:46,066] jepsen worker 23 - jepsen.util 18833 :invoke :read [7 nil] INFO [2019-12-17 12:38:46,067] jepsen worker 58 - jepsen.util 18778 :invoke :read [19 nil] INFO [2019-12-17 12:38:46,068] jepsen worker 49 - jepsen.util 18499 :invoke :read [16 nil] INFO [2019-12-17 12:38:46,068] jepsen worker 77 - jepsen.util 19787 :invoke :read [25 nil] INFO [2019-12-17 12:38:46,068] jepsen worker 64 - jepsen.util 10594 :ok :write [21 19] INFO [2019-12-17 12:38:46,068] jepsen worker 64 - jepsen.util 10594 :invoke :write [21 2] INFO [2019-12-17 12:38:46,091] jepsen worker 69 - jepsen.util 13929 :ok :read [23 15] INFO [2019-12-17 12:38:46,092] jepsen worker 69 - jepsen.util 13929 :invoke :write [23 22] INFO [2019-12-17 12:38:46,102] jepsen worker 39 - jepsen.util 18039 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:38:46,102] jepsen worker 50 - jepsen.util 18320 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:38:46,102] jepsen worker 43 - jepsen.util 18763 :info :cas [14 [47 40]] :RequestCanceledException INFO [2019-12-17 12:38:46,102] jepsen worker 39 - jepsen.util 18039 :invoke :cas [13 [25 21]] INFO [2019-12-17 12:38:46,102] jepsen worker 50 - jepsen.util 18320 :invoke :cas [16 [20 30]] INFO [2019-12-17 12:38:46,103] jepsen worker 43 - jepsen.util 18853 :invoke :read [14 nil] INFO [2019-12-17 12:38:46,119] jepsen worker 21 - jepsen.util 18021 :info :write [7 40] :RequestCanceledException INFO [2019-12-17 12:38:46,119] jepsen worker 48 - jepsen.util 17148 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:38:46,119] jepsen worker 0 - jepsen.util 18990 :info :write [0 5] :RequestCanceledException INFO [2019-12-17 12:38:46,119] jepsen worker 41 - jepsen.util 19841 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:38:46,119] jepsen worker 86 - jepsen.util 19346 :info :cas [28 [33 49]] :RequestCanceledException INFO [2019-12-17 12:38:46,120] jepsen worker 48 - jepsen.util 17148 :invoke :cas [16 [18 7]] INFO [2019-12-17 12:38:46,120] jepsen worker 41 - jepsen.util 19841 :invoke :cas [13 [31 8]] INFO [2019-12-17 12:38:46,121] jepsen worker 0 - jepsen.util 19080 :invoke :read [0 nil] INFO [2019-12-17 12:38:46,121] jepsen worker 21 - jepsen.util 18111 :invoke :read [7 nil] INFO [2019-12-17 12:38:46,121] jepsen worker 86 - jepsen.util 19436 :invoke :read [28 nil] INFO [2019-12-17 12:38:46,133] jepsen worker 47 - jepsen.util 14717 :ok :read [15 41] INFO [2019-12-17 12:38:46,133] jepsen worker 47 - jepsen.util 14717 :invoke :write [15 16] INFO [2019-12-17 12:38:46,135] jepsen worker 82 - jepsen.util 18532 :info :cas [27 [48 45]] :RequestCanceledException INFO [2019-12-17 12:38:46,135] jepsen worker 1 - jepsen.util 18991 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:38:46,135] jepsen worker 1 - jepsen.util 18991 :invoke :write [0 0] INFO [2019-12-17 12:38:46,135] jepsen worker 8 - jepsen.util 18098 :info :cas [2 [2 35]] :RequestCanceledException INFO [2019-12-17 12:38:46,135] jepsen worker 17 - jepsen.util 18377 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:38:46,135] jepsen worker 42 - jepsen.util 19482 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:38:46,135] jepsen worker 17 - jepsen.util 18377 :invoke :cas [5 [44 2]] INFO [2019-12-17 12:38:46,135] jepsen worker 42 - jepsen.util 19482 :invoke :read [14 nil] INFO [2019-12-17 12:38:46,136] jepsen worker 82 - jepsen.util 18622 :invoke :read [27 nil] INFO [2019-12-17 12:38:46,136] jepsen worker 8 - jepsen.util 18188 :invoke :cas [2 [26 37]] INFO [2019-12-17 12:38:46,149] jepsen worker 25 - jepsen.util 19735 :info :write [8 26] :RequestCanceledException INFO [2019-12-17 12:38:46,149] jepsen worker 15 - jepsen.util 17385 :info :cas [5 [21 19]] :RequestCanceledException INFO [2019-12-17 12:38:46,150] jepsen worker 15 - jepsen.util 17475 :invoke :write [5 33] INFO [2019-12-17 12:38:46,150] jepsen worker 25 - jepsen.util 19825 :invoke :cas [8 [21 28]] INFO [2019-12-17 12:38:46,166] jepsen worker 68 - jepsen.util 17708 :info :cas [22 [38 31]] :RequestCanceledException INFO [2019-12-17 12:38:46,167] jepsen worker 68 - jepsen.util 17798 :invoke :read [22 nil] INFO [2019-12-17 12:38:46,184] jepsen worker 63 - jepsen.util 11943 :ok :write [21 40] INFO [2019-12-17 12:38:46,184] jepsen worker 63 - jepsen.util 11943 :invoke :cas [21 [15 35]] INFO [2019-12-17 12:38:46,197] jepsen worker 27 - jepsen.util 12357 :ok :write [9 49] INFO [2019-12-17 12:38:46,197] jepsen worker 27 - jepsen.util 12357 :invoke :write [9 43] INFO [2019-12-17 12:38:46,199] jepsen worker 73 - jepsen.util 12493 :ok :write [24 42] INFO [2019-12-17 12:38:46,199] jepsen worker 73 - jepsen.util 12493 :invoke :cas [24 [16 3]] INFO [2019-12-17 12:38:46,202] jepsen worker 75 - jepsen.util 18255 :info :write [25 42] :RequestCanceledException INFO [2019-12-17 12:38:46,203] jepsen worker 75 - jepsen.util 18345 :invoke :cas [25 [31 46]] INFO [2019-12-17 12:38:46,219] jepsen worker 44 - jepsen.util 19304 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:38:46,220] jepsen worker 44 - jepsen.util 19304 :invoke :read [14 nil] INFO [2019-12-17 12:38:46,228] jepsen worker 79 - jepsen.util 13759 :ok :write [26 23] INFO [2019-12-17 12:38:46,228] jepsen worker 79 - jepsen.util 13759 :invoke :read [26 nil] INFO [2019-12-17 12:38:46,234] jepsen worker 3 - jepsen.util 13143 :info :write [1 10] :RequestCanceledException INFO [2019-12-17 12:38:46,236] jepsen worker 3 - jepsen.util 13233 :invoke :read [1 nil] INFO [2019-12-17 12:38:46,238] jepsen worker 73 - jepsen.util 12493 :fail :cas [24 [16 3]] :ValueNotSwapFrom INFO [2019-12-17 12:38:46,239] jepsen worker 73 - jepsen.util 12493 :invoke :cas [24 [3 15]] INFO [2019-12-17 12:38:46,249] jepsen worker 9 - jepsen.util 14049 :fail :read [3 nil] :RequestTimeoutException INFO [2019-12-17 12:38:46,249] jepsen worker 9 - jepsen.util 14049 :invoke :write [3 33] INFO [2019-12-17 12:38:46,265] jepsen worker 47 - jepsen.util 14717 :ok :write [15 16] INFO [2019-12-17 12:38:46,265] jepsen worker 20 - jepsen.util 13340 :ok :write [6 25] INFO [2019-12-17 12:38:46,265] jepsen worker 47 - jepsen.util 14717 :invoke :write [15 15] INFO [2019-12-17 12:38:46,265] jepsen worker 20 - jepsen.util 13340 :invoke :cas [6 [12 34]] INFO [2019-12-17 12:38:46,267] jepsen worker 64 - jepsen.util 10594 :ok :write [21 2] INFO [2019-12-17 12:38:46,267] jepsen worker 64 - jepsen.util 10594 :invoke :read [21 nil] INFO [2019-12-17 12:38:46,271] jepsen worker 73 - jepsen.util 12493 :fail :cas [24 [3 15]] :ValueNotSwapFrom INFO [2019-12-17 12:38:46,271] jepsen worker 73 - jepsen.util 12493 :invoke :read [24 nil] INFO [2019-12-17 12:38:46,279] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78593,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:38:46,279] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78602,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:38:46,279] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78593,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:38:46,279] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78602,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:38:46,280] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78596,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:38:46,280] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78606,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:38:46,280] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78596,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:38:46,280] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78606,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:38:46,289] jepsen worker 56 - jepsen.util 14186 :ok :write [18 21] INFO [2019-12-17 12:38:46,289] jepsen worker 56 - jepsen.util 14186 :invoke :read [18 nil] INFO [2019-12-17 12:38:46,302] jepsen worker 51 - jepsen.util 12831 :info :write [17 18] :RequestCanceledException INFO [2019-12-17 12:38:46,302] jepsen worker 51 - jepsen.util 12921 :invoke :cas [17 [33 40]] INFO [2019-12-17 12:38:46,310] jepsen worker 4 - jepsen.util 12874 :ok :read [1 45] INFO [2019-12-17 12:38:46,310] jepsen worker 4 - jepsen.util 12874 :invoke :read [1 nil] INFO [2019-12-17 12:38:46,319] jepsen worker 27 - jepsen.util 12357 :ok :write [9 43] INFO [2019-12-17 12:38:46,319] jepsen worker 27 - jepsen.util 12357 :invoke :write [9 11] INFO [2019-12-17 12:38:46,335] jepsen worker 6 - jepsen.util 18636 :info :cas [2 [37 20]] :RequestCanceledException INFO [2019-12-17 12:38:46,335] jepsen worker 18 - jepsen.util 14238 :info :write [6 5] :RequestCanceledException INFO [2019-12-17 12:38:46,336] jepsen worker 18 - jepsen.util 14328 :invoke :read [6 nil] INFO [2019-12-17 12:38:46,336] jepsen worker 6 - jepsen.util 18726 :invoke :cas [2 [23 48]] INFO [2019-12-17 12:38:46,351] jepsen worker 78 - jepsen.util 13488 :ok :write [26 10] INFO [2019-12-17 12:38:46,351] jepsen worker 78 - jepsen.util 13488 :invoke :read [26 nil] INFO [2019-12-17 12:38:46,357] jepsen worker 52 - jepsen.util 13102 :ok :read [17 18] INFO [2019-12-17 12:38:46,357] jepsen worker 52 - jepsen.util 13102 :invoke :write [17 33] INFO [2019-12-17 12:38:46,366] jepsen worker 34 - jepsen.util 19654 :info :cas [11 [0 22]] :RequestCanceledException INFO [2019-12-17 12:38:46,367] jepsen worker 34 - jepsen.util 19744 :invoke :write [11 9] INFO [2019-12-17 12:38:46,392] jepsen worker 20 - jepsen.util 13340 :fail :cas [6 [12 34]] :ValueNotSwapFrom INFO [2019-12-17 12:38:46,392] jepsen worker 20 - jepsen.util 13340 :invoke :read [6 nil] INFO [2019-12-17 12:38:46,406] jepsen worker 64 - jepsen.util 10594 :ok :read [21 2] INFO [2019-12-17 12:38:46,406] jepsen worker 64 - jepsen.util 10594 :invoke :cas [21 [42 6]] INFO [2019-12-17 12:38:46,412] jepsen worker 87 - jepsen.util 13317 :ok :read [29 39] INFO [2019-12-17 12:38:46,412] jepsen worker 87 - jepsen.util 13317 :invoke :write [29 12] INFO [2019-12-17 12:38:46,439] jepsen worker 73 - jepsen.util 12493 :ok :read [24 42] INFO [2019-12-17 12:38:46,439] jepsen worker 73 - jepsen.util 12493 :invoke :read [24 nil] INFO [2019-12-17 12:38:46,441] jepsen worker 5 - jepsen.util 13145 :ok :read [1 45] INFO [2019-12-17 12:38:46,441] jepsen worker 5 - jepsen.util 13145 :invoke :write [1 9] INFO [2019-12-17 12:38:46,463] jepsen worker 64 - jepsen.util 10594 :fail :cas [21 [42 6]] :ValueNotSwapFrom INFO [2019-12-17 12:38:46,466] jepsen worker 64 - jepsen.util 10594 :invoke :cas [21 [3 39]] INFO [2019-12-17 12:38:46,471] jepsen worker 12 - jepsen.util 13602 :ok :read [4 3] INFO [2019-12-17 12:38:46,471] jepsen worker 12 - jepsen.util 13602 :invoke :write [4 43] INFO [2019-12-17 12:38:46,499] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78628,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:38:46,499] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78628,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:38:46,499] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78631,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:38:46,499] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78631,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:38:46,499] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78632,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:38:46,499] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78632,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:38:46,501] jepsen worker 71 - jepsen.util 14381 :info :write [23 46] :RequestCanceledException INFO [2019-12-17 12:38:46,501] jepsen worker 81 - jepsen.util 17991 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:38:46,502] jepsen worker 81 - jepsen.util 17991 :invoke :cas [27 [13 42]] INFO [2019-12-17 12:38:46,502] jepsen worker 71 - jepsen.util 14471 :invoke :write [23 47] INFO [2019-12-17 12:38:46,510] jepsen worker 20 - jepsen.util 13340 :ok :read [6 25] INFO [2019-12-17 12:38:46,510] jepsen worker 20 - jepsen.util 13340 :invoke :read [6 nil] INFO [2019-12-17 12:38:46,533] jepsen worker 73 - jepsen.util 12493 :ok :read [24 42] INFO [2019-12-17 12:38:46,533] jepsen worker 73 - jepsen.util 12493 :invoke :cas [24 [5 37]] INFO [2019-12-17 12:38:46,536] jepsen worker 61 - jepsen.util 19951 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:38:46,536] jepsen worker 26 - jepsen.util 19556 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:38:46,536] jepsen worker 61 - jepsen.util 19951 :invoke :write [20 18] INFO [2019-12-17 12:38:46,536] jepsen worker 26 - jepsen.util 19556 :invoke :cas [8 [49 35]] INFO [2019-12-17 12:38:46,537] jepsen worker 3 - jepsen.util 13233 :ok :read [1 45] INFO [2019-12-17 12:38:46,537] jepsen worker 3 - jepsen.util 13233 :invoke :read [1 nil] INFO [2019-12-17 12:38:46,538] jepsen worker 38 - jepsen.util 14258 :ok :read [12 49] INFO [2019-12-17 12:38:46,538] jepsen worker 38 - jepsen.util 14258 :invoke :read [12 nil] INFO [2019-12-17 12:38:46,549] jepsen worker 35 - jepsen.util 18845 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:38:46,549] jepsen worker 35 - jepsen.util 18845 :invoke :cas [11 [21 42]] INFO [2019-12-17 12:38:46,560] jepsen worker 64 - jepsen.util 10594 :fail :cas [21 [3 39]] :ValueNotSwapFrom INFO [2019-12-17 12:38:46,561] jepsen worker 64 - jepsen.util 10594 :invoke :read [21 nil] INFO [2019-12-17 12:38:46,566] jepsen worker 83 - jepsen.util 18443 :info :write [27 38] :RequestCanceledException INFO [2019-12-17 12:38:46,567] jepsen worker 20 - jepsen.util 13340 :ok :read [6 25] INFO [2019-12-17 12:38:46,567] jepsen worker 20 - jepsen.util 13340 :invoke :cas [6 [42 2]] INFO [2019-12-17 12:38:46,567] jepsen worker 83 - jepsen.util 18533 :invoke :write [27 8] INFO [2019-12-17 12:38:46,585] jepsen worker 78 - jepsen.util 13488 :ok :read [26 10] INFO [2019-12-17 12:38:46,585] jepsen worker 78 - jepsen.util 13488 :invoke :cas [26 [36 24]] INFO [2019-12-17 12:38:46,586] jepsen worker 73 - jepsen.util 12493 :fail :cas [24 [5 37]] :ValueNotSwapFrom INFO [2019-12-17 12:38:46,586] jepsen worker 73 - jepsen.util 12493 :invoke :read [24 nil] INFO [2019-12-17 12:38:46,602] jepsen worker 88 - jepsen.util 13408 :info :write [29 41] :RequestCanceledException INFO [2019-12-17 12:38:46,603] jepsen worker 88 - jepsen.util 13498 :invoke :read [29 nil] INFO [2019-12-17 12:38:46,616] jepsen worker 20 - jepsen.util 13340 :fail :cas [6 [42 2]] :ValueNotSwapFrom INFO [2019-12-17 12:38:46,616] jepsen worker 20 - jepsen.util 13340 :invoke :cas [6 [43 8]] INFO [2019-12-17 12:38:46,633] jepsen worker 14 - jepsen.util 13154 :ok :write [4 39] INFO [2019-12-17 12:38:46,633] jepsen worker 14 - jepsen.util 13154 :invoke :read [4 nil] INFO [2019-12-17 12:38:46,636] jepsen worker 73 - jepsen.util 12493 :ok :read [24 42] INFO [2019-12-17 12:38:46,637] jepsen worker 73 - jepsen.util 12493 :invoke :write [24 20] INFO [2019-12-17 12:38:46,638] jepsen worker 38 - jepsen.util 14258 :ok :read [12 49] INFO [2019-12-17 12:38:46,638] jepsen worker 38 - jepsen.util 14258 :invoke :cas [12 [41 42]] INFO [2019-12-17 12:38:46,666] jepsen worker 64 - jepsen.util 10594 :ok :read [21 2] INFO [2019-12-17 12:38:46,666] jepsen worker 64 - jepsen.util 10594 :invoke :cas [21 [42 48]] INFO [2019-12-17 12:38:46,681] jepsen worker 78 - jepsen.util 13488 :fail :cas [26 [36 24]] :ValueNotSwapFrom INFO [2019-12-17 12:38:46,681] jepsen worker 78 - jepsen.util 13488 :invoke :read [26 nil] INFO [2019-12-17 12:38:46,692] jepsen worker 80 - jepsen.util 12950 :ok :write [26 48] INFO [2019-12-17 12:38:46,692] jepsen worker 80 - jepsen.util 12950 :invoke :read [26 nil] INFO [2019-12-17 12:38:46,697] jepsen worker 4 - jepsen.util 12874 :ok :read [1 45] INFO [2019-12-17 12:38:46,697] jepsen worker 4 - jepsen.util 12874 :invoke :read [1 nil] INFO [2019-12-17 12:38:46,702] jepsen worker 72 - jepsen.util 13572 :info :write [24 18] :RequestCanceledException INFO [2019-12-17 12:38:46,703] jepsen worker 72 - jepsen.util 13662 :invoke :cas [24 [39 22]] INFO [2019-12-17 12:38:46,718] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78657,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:38:46,718] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78657,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:38:46,719] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78661,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:38:46,719] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78661,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:38:46,727] jepsen worker 20 - jepsen.util 13340 :fail :cas [6 [43 8]] :ValueNotSwapFrom INFO [2019-12-17 12:38:46,727] jepsen worker 20 - jepsen.util 13340 :invoke :write [6 0] INFO [2019-12-17 12:38:46,735] jepsen worker 45 - jepsen.util 13095 :info :cas [15 [35 25]] :RequestCanceledException INFO [2019-12-17 12:38:46,736] jepsen worker 45 - jepsen.util 13185 :invoke :cas [15 [26 19]] INFO [2019-12-17 12:38:46,737] jepsen worker 79 - jepsen.util 13759 :ok :read [26 48] INFO [2019-12-17 12:38:46,737] jepsen worker 79 - jepsen.util 13759 :invoke :write [26 31] INFO [2019-12-17 12:38:46,745] jepsen worker 64 - jepsen.util 10594 :fail :cas [21 [42 48]] :ValueNotSwapFrom INFO [2019-12-17 12:38:46,745] jepsen worker 64 - jepsen.util 10594 :invoke :cas [21 [34 27]] INFO [2019-12-17 12:38:46,776] jepsen worker 45 - jepsen.util 13185 :fail :cas [15 [26 19]] :ValueNotSwapFrom INFO [2019-12-17 12:38:46,776] jepsen worker 45 - jepsen.util 13185 :invoke :read [15 nil] INFO [2019-12-17 12:38:46,791] jepsen worker 53 - jepsen.util 13373 :ok :write [17 40] INFO [2019-12-17 12:38:46,791] jepsen worker 53 - jepsen.util 13373 :invoke :write [17 16] INFO [2019-12-17 12:38:46,796] jepsen worker 64 - jepsen.util 10594 :fail :cas [21 [34 27]] :ValueNotSwapFrom INFO [2019-12-17 12:38:46,796] jepsen worker 64 - jepsen.util 10594 :invoke :write [21 21] INFO [2019-12-17 12:38:46,798] jepsen worker 12 - jepsen.util 13602 :ok :write [4 43] INFO [2019-12-17 12:38:46,799] jepsen worker 12 - jepsen.util 13602 :invoke :read [4 nil] INFO [2019-12-17 12:38:46,805] jepsen worker 5 - jepsen.util 13145 :ok :write [1 9] INFO [2019-12-17 12:38:46,805] jepsen worker 5 - jepsen.util 13145 :invoke :cas [1 [2 45]] INFO [2019-12-17 12:38:46,806] jepsen worker 87 - jepsen.util 13317 :ok :write [29 12] INFO [2019-12-17 12:38:46,806] jepsen worker 87 - jepsen.util 13317 :invoke :write [29 43] INFO [2019-12-17 12:38:46,812] jepsen worker 4 - jepsen.util 12874 :ok :read [1 9] INFO [2019-12-17 12:38:46,812] jepsen worker 4 - jepsen.util 12874 :invoke :write [1 15] INFO [2019-12-17 12:38:46,825] jepsen worker 45 - jepsen.util 13185 :ok :read [15 16] INFO [2019-12-17 12:38:46,826] jepsen worker 45 - jepsen.util 13185 :invoke :read [15 nil] INFO [2019-12-17 12:38:46,849] jepsen worker 74 - jepsen.util 13394 :info :write [24 19] :RequestCanceledException INFO [2019-12-17 12:38:46,852] jepsen worker 74 - jepsen.util 13484 :invoke :read [24 nil] INFO [2019-12-17 12:38:46,855] jepsen worker 38 - jepsen.util 14258 :fail :cas [12 [41 42]] :ValueNotSwapFrom INFO [2019-12-17 12:38:46,856] jepsen worker 38 - jepsen.util 14258 :invoke :cas [12 [20 48]] INFO [2019-12-17 12:38:46,864] jepsen worker 3 - jepsen.util 13233 :ok :read [1 9] INFO [2019-12-17 12:38:46,864] jepsen worker 3 - jepsen.util 13233 :invoke :cas [1 [14 47]] INFO [2019-12-17 12:38:46,870] jepsen worker 64 - jepsen.util 10594 :ok :write [21 21] INFO [2019-12-17 12:38:46,871] jepsen worker 64 - jepsen.util 10594 :invoke :cas [21 [40 10]] INFO [2019-12-17 12:38:46,871] jepsen worker 56 - jepsen.util 14186 :ok :read [18 21] INFO [2019-12-17 12:38:46,871] jepsen worker 56 - jepsen.util 14186 :invoke :cas [18 [41 5]] INFO [2019-12-17 12:38:46,883] jepsen worker 45 - jepsen.util 13185 :ok :read [15 16] INFO [2019-12-17 12:38:46,884] jepsen worker 45 - jepsen.util 13185 :invoke :write [15 8] INFO [2019-12-17 12:38:46,894] jepsen worker 64 - jepsen.util 10594 :fail :cas [21 [40 10]] :ValueNotSwapFrom INFO [2019-12-17 12:38:46,894] jepsen worker 64 - jepsen.util 10594 :invoke :write [21 23] INFO [2019-12-17 12:38:46,902] jepsen worker 78 - jepsen.util 13488 :ok :read [26 48] INFO [2019-12-17 12:38:46,902] jepsen worker 78 - jepsen.util 13488 :invoke :cas [26 [1 37]] INFO [2019-12-17 12:38:46,912] jepsen worker 14 - jepsen.util 13154 :ok :read [4 43] INFO [2019-12-17 12:38:46,912] jepsen worker 14 - jepsen.util 13154 :invoke :write [4 20] INFO [2019-12-17 12:38:46,925] jepsen worker 9 - jepsen.util 14049 :ok :write [3 33] INFO [2019-12-17 12:38:46,925] jepsen worker 9 - jepsen.util 14049 :invoke :write [3 38] INFO [2019-12-17 12:38:46,932] jepsen worker 51 - jepsen.util 12921 :fail :cas [17 [33 40]] :ValueNotSwapFrom INFO [2019-12-17 12:38:46,932] jepsen worker 51 - jepsen.util 12921 :invoke :write [17 12] INFO [2019-12-17 12:38:46,938] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78685,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:38:46,938] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78685,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:38:46,938] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78687,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:38:46,938] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78687,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:38:46,944] jepsen worker 12 - jepsen.util 13602 :ok :read [4 43] INFO [2019-12-17 12:38:46,944] jepsen worker 12 - jepsen.util 13602 :invoke :write [4 29] INFO [2019-12-17 12:38:46,948] jepsen worker 30 - jepsen.util 12990 :info :cas [10 [33 24]] :RequestCanceledException INFO [2019-12-17 12:38:46,949] jepsen worker 30 - jepsen.util 13080 :invoke :cas [10 [35 38]] INFO [2019-12-17 12:38:46,994] jepsen worker 64 - jepsen.util 10594 :ok :write [21 23] INFO [2019-12-17 12:38:46,994] jepsen worker 64 - jepsen.util 10594 :invoke :cas [21 [14 28]] INFO [2019-12-17 12:38:47,010] jepsen worker 64 - jepsen.util 10594 :fail :cas [21 [14 28]] :ValueNotSwapFrom INFO [2019-12-17 12:38:47,010] jepsen worker 64 - jepsen.util 10594 :invoke :read [21 nil] INFO [2019-12-17 12:38:47,025] jepsen worker 64 - jepsen.util 10594 :ok :read [21 23] INFO [2019-12-17 12:38:47,026] jepsen worker 64 - jepsen.util 10594 :invoke :read [21 nil] INFO [2019-12-17 12:38:47,035] jepsen worker 28 - jepsen.util 11458 :info :write [9 46] :RequestCanceledException INFO [2019-12-17 12:38:47,035] jepsen worker 31 - jepsen.util 11731 :info :write [10 1] :RequestCanceledException INFO [2019-12-17 12:38:47,036] jepsen worker 28 - jepsen.util 11548 :invoke :write [9 29] INFO [2019-12-17 12:38:47,036] jepsen worker 31 - jepsen.util 11821 :invoke :write [10 21] INFO [2019-12-17 12:38:47,043] jepsen worker 64 - jepsen.util 10594 :ok :read [21 23] INFO [2019-12-17 12:38:47,043] jepsen worker 64 - jepsen.util 10594 :invoke :read [21 nil] INFO [2019-12-17 12:38:47,046] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78699,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:38:47,046] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78699,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:38:47,046] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78703,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:38:47,046] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78703,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:38:47,047] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78704,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:38:47,047] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78704,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:38:47,049] jepsen worker 13 - jepsen.util 13153 :info :write [4 3] :RequestCanceledException INFO [2019-12-17 12:38:47,050] jepsen worker 13 - jepsen.util 13243 :invoke :cas [4 [19 32]] INFO [2019-12-17 12:38:47,061] jepsen worker 38 - jepsen.util 14258 :fail :cas [12 [20 48]] :ValueNotSwapFrom INFO [2019-12-17 12:38:47,061] jepsen worker 38 - jepsen.util 14258 :invoke :write [12 39] INFO [2019-12-17 12:38:47,063] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:38:47,063] jepsen nemesis - couchbase.util Rebalance check count: 618 INFO [2019-12-17 12:38:47,063] jepsen nemesis - couchbase.util Rebalance stuck count: 599 INFO [2019-12-17 12:38:47,083] jepsen worker 64 - jepsen.util 10594 :ok :read [21 23] INFO [2019-12-17 12:38:47,083] jepsen worker 78 - jepsen.util 13488 :fail :cas [26 [1 37]] :ValueNotSwapFrom INFO [2019-12-17 12:38:47,083] jepsen worker 64 - jepsen.util 10594 :invoke :write [21 12] INFO [2019-12-17 12:38:47,083] jepsen worker 78 - jepsen.util 13488 :invoke :read [26 nil] INFO [2019-12-17 12:38:47,085] jepsen worker 20 - jepsen.util 13340 :ok :write [6 0] INFO [2019-12-17 12:38:47,085] jepsen worker 20 - jepsen.util 13340 :invoke :write [6 22] INFO [2019-12-17 12:38:47,085] jepsen worker 47 - jepsen.util 14717 :ok :write [15 15] INFO [2019-12-17 12:38:47,085] jepsen worker 73 - jepsen.util 12493 :ok :write [24 20] INFO [2019-12-17 12:38:47,085] jepsen worker 47 - jepsen.util 14717 :invoke :write [15 16] INFO [2019-12-17 12:38:47,085] jepsen worker 73 - jepsen.util 12493 :invoke :cas [24 [5 33]] INFO [2019-12-17 12:38:47,113] jepsen worker 70 - jepsen.util 13930 :ok :write [23 21] INFO [2019-12-17 12:38:47,113] jepsen worker 70 - jepsen.util 13930 :invoke :read [23 nil] INFO [2019-12-17 12:38:47,115] jepsen worker 80 - jepsen.util 12950 :ok :read [26 48] INFO [2019-12-17 12:38:47,115] jepsen worker 80 - jepsen.util 12950 :invoke :cas [26 [23 24]] INFO [2019-12-17 12:38:47,128] jepsen worker 74 - jepsen.util 13484 :ok :read [24 20] INFO [2019-12-17 12:38:47,128] jepsen worker 74 - jepsen.util 13484 :invoke :cas [24 [9 36]] INFO [2019-12-17 12:38:47,186] jepsen worker 73 - jepsen.util 12493 :fail :cas [24 [5 33]] :ValueNotSwapFrom INFO [2019-12-17 12:38:47,186] jepsen worker 73 - jepsen.util 12493 :invoke :read [24 nil] INFO [2019-12-17 12:38:47,205] jepsen worker 80 - jepsen.util 12950 :fail :cas [26 [23 24]] :ValueNotSwapFrom INFO [2019-12-17 12:38:47,205] jepsen worker 80 - jepsen.util 12950 :invoke :write [26 27] INFO [2019-12-17 12:38:47,206] jepsen worker 72 - jepsen.util 13662 :fail :cas [24 [39 22]] :ValueNotSwapFrom INFO [2019-12-17 12:38:47,206] jepsen worker 72 - jepsen.util 13662 :invoke :read [24 nil] INFO [2019-12-17 12:38:47,233] jepsen worker 74 - jepsen.util 13484 :fail :cas [24 [9 36]] :ValueNotSwapFrom INFO [2019-12-17 12:38:47,233] jepsen worker 74 - jepsen.util 13484 :invoke :read [24 nil] INFO [2019-12-17 12:38:47,244] jepsen worker 5 - jepsen.util 13145 :fail :cas [1 [2 45]] :ValueNotSwapFrom INFO [2019-12-17 12:38:47,246] jepsen worker 5 - jepsen.util 13145 :invoke :cas [1 [46 31]] INFO [2019-12-17 12:38:47,251] jepsen worker 73 - jepsen.util 12493 :ok :read [24 20] INFO [2019-12-17 12:38:47,251] jepsen worker 73 - jepsen.util 12493 :invoke :cas [24 [7 47]] INFO [2019-12-17 12:38:47,265] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78721,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:38:47,265] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78721,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:38:47,265] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78722,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:38:47,265] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78722,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:38:47,266] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78731,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:38:47,266] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78731,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:38:47,282] jepsen worker 13 - jepsen.util 13243 :fail :cas [4 [19 32]] :ValueNotSwapFrom INFO [2019-12-17 12:38:47,283] jepsen worker 13 - jepsen.util 13243 :invoke :read [4 nil] INFO [2019-12-17 12:38:47,288] jepsen worker 74 - jepsen.util 13484 :ok :read [24 20] INFO [2019-12-17 12:38:47,288] jepsen worker 74 - jepsen.util 13484 :invoke :write [24 6] INFO [2019-12-17 12:38:47,302] jepsen worker 89 - jepsen.util 13679 :info :write [29 12] :RequestCanceledException INFO [2019-12-17 12:38:47,302] jepsen worker 28 - jepsen.util 11548 :ok :write [9 29] INFO [2019-12-17 12:38:47,302] jepsen worker 28 - jepsen.util 11548 :invoke :read [9 nil] INFO [2019-12-17 12:38:47,304] jepsen worker 89 - jepsen.util 13769 :invoke :write [29 9] INFO [2019-12-17 12:38:47,306] jepsen worker 54 - jepsen.util 14364 :ok :write [18 36] INFO [2019-12-17 12:38:47,306] jepsen worker 54 - jepsen.util 14364 :invoke :cas [18 [3 44]] INFO [2019-12-17 12:38:47,310] jepsen worker 73 - jepsen.util 12493 :fail :cas [24 [7 47]] :ValueNotSwapFrom INFO [2019-12-17 12:38:47,310] jepsen worker 73 - jepsen.util 12493 :invoke :read [24 nil] INFO [2019-12-17 12:38:47,313] jepsen worker 88 - jepsen.util 13498 :ok :read [29 12] INFO [2019-12-17 12:38:47,313] jepsen worker 88 - jepsen.util 13498 :invoke :cas [29 [15 36]] INFO [2019-12-17 12:38:47,323] jepsen worker 64 - jepsen.util 10594 :ok :write [21 12] INFO [2019-12-17 12:38:47,323] jepsen worker 64 - jepsen.util 10594 :invoke :read [21 nil] INFO [2019-12-17 12:38:47,338] jepsen worker 64 - jepsen.util 10594 :ok :read [21 12] INFO [2019-12-17 12:38:47,338] jepsen worker 64 - jepsen.util 10594 :invoke :write [21 10] INFO [2019-12-17 12:38:47,346] jepsen worker 28 - jepsen.util 11548 :ok :read [9 29] INFO [2019-12-17 12:38:47,346] jepsen worker 28 - jepsen.util 11548 :invoke :read [9 nil] INFO [2019-12-17 12:38:47,350] jepsen worker 5 - jepsen.util 13145 :fail :cas [1 [46 31]] :ValueNotSwapFrom INFO [2019-12-17 12:38:47,350] jepsen worker 5 - jepsen.util 13145 :invoke :write [1 42] INFO [2019-12-17 12:38:47,376] jepsen worker 13 - jepsen.util 13243 :ok :read [4 43] INFO [2019-12-17 12:38:47,376] jepsen worker 13 - jepsen.util 13243 :invoke :write [4 10] INFO [2019-12-17 12:38:47,380] jepsen worker 28 - jepsen.util 11548 :ok :read [9 29] INFO [2019-12-17 12:38:47,381] jepsen worker 28 - jepsen.util 11548 :invoke :cas [9 [48 0]] INFO [2019-12-17 12:38:47,383] jepsen worker 88 - jepsen.util 13498 :fail :cas [29 [15 36]] :ValueNotSwapFrom INFO [2019-12-17 12:38:47,384] jepsen worker 88 - jepsen.util 13498 :invoke :write [29 0] INFO [2019-12-17 12:38:47,397] jepsen worker 28 - jepsen.util 11548 :fail :cas [9 [48 0]] :ValueNotSwapFrom INFO [2019-12-17 12:38:47,397] jepsen worker 28 - jepsen.util 11548 :invoke :cas [9 [17 7]] INFO [2019-12-17 12:38:47,410] jepsen worker 31 - jepsen.util 11821 :ok :write [10 21] INFO [2019-12-17 12:38:47,410] jepsen worker 31 - jepsen.util 11821 :invoke :write [10 17] INFO [2019-12-17 12:38:47,410] jepsen worker 9 - jepsen.util 14049 :ok :write [3 38] INFO [2019-12-17 12:38:47,411] jepsen worker 9 - jepsen.util 14049 :invoke :write [3 39] INFO [2019-12-17 12:38:47,435] jepsen worker 37 - jepsen.util 12997 :info :write [12 24] :RequestCanceledException INFO [2019-12-17 12:38:47,436] jepsen worker 37 - jepsen.util 13087 :invoke :read [12 nil] INFO [2019-12-17 12:38:47,451] jepsen worker 28 - jepsen.util 11548 :fail :cas [9 [17 7]] :ValueNotSwapFrom INFO [2019-12-17 12:38:47,451] jepsen worker 28 - jepsen.util 11548 :invoke :write [9 0] INFO [2019-12-17 12:38:47,466] jepsen worker 36 - jepsen.util 14256 :ok :write [12 28] INFO [2019-12-17 12:38:47,466] jepsen worker 36 - jepsen.util 14256 :invoke :read [12 nil] INFO [2019-12-17 12:38:47,478] jepsen worker 56 - jepsen.util 14186 :fail :cas [18 [41 5]] :ValueNotSwapFrom INFO [2019-12-17 12:38:47,478] jepsen worker 56 - jepsen.util 14186 :invoke :write [18 21] INFO [2019-12-17 12:38:47,484] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78757,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:38:47,485] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78757,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:38:47,485] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78760,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:38:47,485] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78760,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:38:47,498] jepsen worker 64 - jepsen.util 10594 :ok :write [21 10] INFO [2019-12-17 12:38:47,498] jepsen worker 64 - jepsen.util 10594 :invoke :cas [21 [45 29]] INFO [2019-12-17 12:38:47,511] jepsen worker 79 - jepsen.util 13759 :ok :write [26 31] INFO [2019-12-17 12:38:47,511] jepsen worker 79 - jepsen.util 13759 :invoke :write [26 39] INFO [2019-12-17 12:38:47,520] jepsen worker 46 - jepsen.util 13186 :info :cas [15 [40 5]] :RequestCanceledException INFO [2019-12-17 12:38:47,522] jepsen worker 46 - jepsen.util 13276 :invoke :cas [15 [29 39]] INFO [2019-12-17 12:38:47,532] jepsen worker 64 - jepsen.util 10594 :fail :cas [21 [45 29]] :ValueNotSwapFrom INFO [2019-12-17 12:38:47,532] jepsen worker 64 - jepsen.util 10594 :invoke :read [21 nil] INFO [2019-12-17 12:38:47,549] jepsen worker 32 - jepsen.util 13802 :fail :read [10 nil] :RequestTimeoutException INFO [2019-12-17 12:38:47,549] jepsen worker 32 - jepsen.util 13802 :invoke :cas [10 [49 21]] INFO [2019-12-17 12:38:47,558] jepsen worker 47 - jepsen.util 14717 :ok :write [15 16] INFO [2019-12-17 12:38:47,558] jepsen worker 47 - jepsen.util 14717 :invoke :cas [15 [17 43]] INFO [2019-12-17 12:38:47,561] jepsen worker 78 - jepsen.util 13488 :ok :read [26 31] INFO [2019-12-17 12:38:47,561] jepsen worker 78 - jepsen.util 13488 :invoke :read [26 nil] INFO [2019-12-17 12:38:47,569] jepsen worker 52 - jepsen.util 13102 :ok :write [17 33] INFO [2019-12-17 12:38:47,569] jepsen worker 52 - jepsen.util 13102 :invoke :read [17 nil] INFO [2019-12-17 12:38:47,573] jepsen worker 70 - jepsen.util 13930 :ok :read [23 21] INFO [2019-12-17 12:38:47,573] jepsen worker 70 - jepsen.util 13930 :invoke :read [23 nil] INFO [2019-12-17 12:38:47,581] jepsen worker 28 - jepsen.util 11548 :ok :write [9 0] INFO [2019-12-17 12:38:47,582] jepsen worker 28 - jepsen.util 11548 :invoke :write [9 7] INFO [2019-12-17 12:38:47,582] jepsen worker 64 - jepsen.util 10594 :ok :read [21 10] INFO [2019-12-17 12:38:47,582] jepsen worker 64 - jepsen.util 10594 :invoke :read [21 nil] INFO [2019-12-17 12:38:47,616] jepsen worker 47 - jepsen.util 14717 :fail :cas [15 [17 43]] :ValueNotSwapFrom INFO [2019-12-17 12:38:47,616] jepsen worker 47 - jepsen.util 14717 :invoke :read [15 nil] INFO [2019-12-17 12:38:47,616] jepsen worker 37 - jepsen.util 13087 :ok :read [12 28] INFO [2019-12-17 12:38:47,616] jepsen worker 37 - jepsen.util 13087 :invoke :write [12 41] INFO [2019-12-17 12:38:47,661] jepsen worker 47 - jepsen.util 14717 :ok :read [15 16] INFO [2019-12-17 12:38:47,662] jepsen worker 47 - jepsen.util 14717 :invoke :write [15 37] INFO [2019-12-17 12:38:47,664] jepsen worker 74 - jepsen.util 13484 :ok :write [24 6] INFO [2019-12-17 12:38:47,665] jepsen worker 74 - jepsen.util 13484 :invoke :write [24 30] INFO [2019-12-17 12:38:47,667] jepsen worker 64 - jepsen.util 10594 :ok :read [21 10] INFO [2019-12-17 12:38:47,667] jepsen worker 64 - jepsen.util 10594 :invoke :read [21 nil] INFO [2019-12-17 12:38:47,697] jepsen worker 3 - jepsen.util 13233 :fail :cas [1 [14 47]] :ValueNotSwapFrom INFO [2019-12-17 12:38:47,697] jepsen worker 3 - jepsen.util 13233 :invoke :write [1 36] INFO [2019-12-17 12:38:47,701] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78781,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:38:47,701] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78781,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:38:47,701] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78783,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:38:47,701] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78783,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:38:47,702] jepsen worker 71 - jepsen.util 14471 :ok :write [23 47] INFO [2019-12-17 12:38:47,702] jepsen worker 71 - jepsen.util 14471 :invoke :read [23 nil] INFO [2019-12-17 12:38:47,705] jepsen worker 73 - jepsen.util 12493 :ok :read [24 6] INFO [2019-12-17 12:38:47,705] jepsen worker 73 - jepsen.util 12493 :invoke :read [24 nil] INFO [2019-12-17 12:38:47,722] jepsen worker 9 - jepsen.util 14049 :ok :write [3 39] INFO [2019-12-17 12:38:47,722] jepsen worker 9 - jepsen.util 14049 :invoke :cas [3 [19 44]] INFO [2019-12-17 12:38:47,741] jepsen worker 64 - jepsen.util 10594 :ok :read [21 10] INFO [2019-12-17 12:38:47,742] jepsen worker 64 - jepsen.util 10594 :invoke :cas [21 [33 46]] INFO [2019-12-17 12:38:47,744] jepsen worker 28 - jepsen.util 11548 :ok :write [9 7] INFO [2019-12-17 12:38:47,744] jepsen worker 28 - jepsen.util 11548 :invoke :write [9 40] INFO [2019-12-17 12:38:47,750] jepsen worker 70 - jepsen.util 13930 :ok :read [23 47] INFO [2019-12-17 12:38:47,750] jepsen worker 70 - jepsen.util 13930 :invoke :read [23 nil] INFO [2019-12-17 12:38:47,754] jepsen worker 32 - jepsen.util 13802 :fail :cas [10 [49 21]] :ValueNotSwapFrom INFO [2019-12-17 12:38:47,754] jepsen worker 32 - jepsen.util 13802 :invoke :cas [10 [30 40]] INFO [2019-12-17 12:38:47,761] jepsen worker 54 - jepsen.util 14364 :fail :cas [18 [3 44]] :ValueNotSwapFrom INFO [2019-12-17 12:38:47,761] jepsen worker 54 - jepsen.util 14364 :invoke :cas [18 [19 29]] INFO [2019-12-17 12:38:47,783] jepsen worker 64 - jepsen.util 10594 :fail :cas [21 [33 46]] :ValueNotSwapFrom INFO [2019-12-17 12:38:47,783] jepsen worker 64 - jepsen.util 10594 :invoke :write [21 12] INFO [2019-12-17 12:38:47,786] jepsen worker 9 - jepsen.util 14049 :fail :cas [3 [19 44]] :ValueNotSwapFrom INFO [2019-12-17 12:38:47,786] jepsen worker 9 - jepsen.util 14049 :invoke :write [3 0] INFO [2019-12-17 12:38:47,792] jepsen worker 32 - jepsen.util 13802 :fail :cas [10 [30 40]] :ValueNotSwapFrom INFO [2019-12-17 12:38:47,792] jepsen worker 32 - jepsen.util 13802 :invoke :cas [10 [23 46]] INFO [2019-12-17 12:38:47,807] jepsen worker 79 - jepsen.util 13759 :ok :write [26 39] INFO [2019-12-17 12:38:47,807] jepsen worker 79 - jepsen.util 13759 :invoke :write [26 31] INFO [2019-12-17 12:38:47,819] jepsen worker 65 - jepsen.util 11855 :info :cas [21 [42 24]] :RequestCanceledException INFO [2019-12-17 12:38:47,819] jepsen worker 11 - jepsen.util 13331 :info :write [3 30] :RequestCanceledException INFO [2019-12-17 12:38:47,820] jepsen worker 11 - jepsen.util 13421 :invoke :cas [3 [1 10]] INFO [2019-12-17 12:38:47,823] jepsen worker 65 - jepsen.util 11945 :invoke :read [21 nil] INFO [2019-12-17 12:38:47,826] jepsen worker 28 - jepsen.util 11548 :ok :write [9 40] INFO [2019-12-17 12:38:47,826] jepsen worker 28 - jepsen.util 11548 :invoke :read [9 nil] INFO [2019-12-17 12:38:47,843] jepsen worker 11 - jepsen.util 13421 :fail :cas [3 [1 10]] :ValueNotSwapFrom INFO [2019-12-17 12:38:47,844] jepsen worker 11 - jepsen.util 13421 :invoke :write [3 26] INFO [2019-12-17 12:38:47,846] jepsen worker 31 - jepsen.util 11821 :ok :write [10 17] INFO [2019-12-17 12:38:47,846] jepsen worker 31 - jepsen.util 11821 :invoke :read [10 nil] INFO [2019-12-17 12:38:47,847] jepsen worker 55 - jepsen.util 14185 :info :write [18 40] :RequestCanceledException INFO [2019-12-17 12:38:47,860] jepsen worker 55 - jepsen.util 14275 :invoke :read [18 nil] INFO [2019-12-17 12:38:47,861] jepsen worker 28 - jepsen.util 11548 :ok :read [9 40] INFO [2019-12-17 12:38:47,861] jepsen worker 28 - jepsen.util 11548 :invoke :read [9 nil] INFO [2019-12-17 12:38:47,883] jepsen worker 4 - jepsen.util 12874 :ok :write [1 15] INFO [2019-12-17 12:38:47,883] jepsen worker 4 - jepsen.util 12874 :invoke :write [1 35] INFO [2019-12-17 12:38:47,886] jepsen worker 64 - jepsen.util 10594 :ok :write [21 12] INFO [2019-12-17 12:38:47,887] jepsen worker 64 - jepsen.util 10594 :invoke :cas [21 [29 8]] INFO [2019-12-17 12:38:47,895] jepsen worker 71 - jepsen.util 14471 :ok :read [23 47] INFO [2019-12-17 12:38:47,895] jepsen worker 71 - jepsen.util 14471 :invoke :write [23 11] INFO [2019-12-17 12:38:47,904] jepsen worker 74 - jepsen.util 13484 :ok :write [24 30] INFO [2019-12-17 12:38:47,904] jepsen worker 47 - jepsen.util 14717 :ok :write [15 37] INFO [2019-12-17 12:38:47,904] jepsen worker 74 - jepsen.util 13484 :invoke :cas [24 [10 29]] INFO [2019-12-17 12:38:47,904] jepsen worker 47 - jepsen.util 14717 :invoke :read [15 nil] INFO [2019-12-17 12:38:47,919] jepsen worker 65 - jepsen.util 11945 :ok :read [21 12] INFO [2019-12-17 12:38:47,920] jepsen worker 65 - jepsen.util 11945 :invoke :cas [21 [2 21]] INFO [2019-12-17 12:38:47,920] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78808,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:38:47,920] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78808,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:38:47,920] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78811,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:38:47,920] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78811,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:38:47,920] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78812,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:38:47,921] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78812,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:38:47,923] jepsen worker 64 - jepsen.util 10594 :fail :cas [21 [29 8]] :ValueNotSwapFrom INFO [2019-12-17 12:38:47,923] jepsen worker 64 - jepsen.util 10594 :invoke :read [21 nil] INFO [2019-12-17 12:38:47,923] jepsen worker 54 - jepsen.util 14364 :fail :cas [18 [19 29]] :ValueNotSwapFrom INFO [2019-12-17 12:38:47,923] jepsen worker 54 - jepsen.util 14364 :invoke :read [18 nil] INFO [2019-12-17 12:38:47,935] jepsen worker 19 - jepsen.util 14329 :info :write [6 43] :RequestCanceledException INFO [2019-12-17 12:38:47,935] jepsen worker 29 - jepsen.util 11639 :info :write [9 10] :RequestCanceledException INFO [2019-12-17 12:38:47,935] jepsen worker 67 - jepsen.util 19057 :info :write [22 11] :RequestCanceledException INFO [2019-12-17 12:38:47,936] jepsen worker 67 - jepsen.util 19147 :invoke :write [22 2] INFO [2019-12-17 12:38:47,936] jepsen worker 29 - jepsen.util 11729 :invoke :write [9 19] INFO [2019-12-17 12:38:47,936] jepsen worker 19 - jepsen.util 14419 :invoke :cas [6 [35 6]] INFO [2019-12-17 12:38:47,947] jepsen worker 73 - jepsen.util 12493 :ok :read [24 30] INFO [2019-12-17 12:38:47,947] jepsen worker 73 - jepsen.util 12493 :invoke :write [24 41] INFO [2019-12-17 12:38:47,949] jepsen worker 22 - jepsen.util 19102 :info :cas [7 [28 34]] :RequestCanceledException INFO [2019-12-17 12:38:47,950] jepsen worker 22 - jepsen.util 19192 :invoke :write [7 3] INFO [2019-12-17 12:38:47,953] jepsen worker 31 - jepsen.util 11821 :ok :read [10 17] INFO [2019-12-17 12:38:47,954] jepsen worker 31 - jepsen.util 11821 :invoke :cas [10 [21 38]] INFO [2019-12-17 12:38:47,954] jepsen worker 47 - jepsen.util 14717 :ok :read [15 37] INFO [2019-12-17 12:38:47,957] jepsen worker 47 - jepsen.util 14717 :invoke :read [15 nil] INFO [2019-12-17 12:38:47,966] jepsen worker 24 - jepsen.util 17754 :info :cas [8 [27 47]] :RequestCanceledException INFO [2019-12-17 12:38:47,967] jepsen worker 24 - jepsen.util 17844 :invoke :read [8 nil] INFO [2019-12-17 12:38:47,972] jepsen worker 74 - jepsen.util 13484 :fail :cas [24 [10 29]] :ValueNotSwapFrom INFO [2019-12-17 12:38:47,972] jepsen worker 64 - jepsen.util 10594 :ok :read [21 12] INFO [2019-12-17 12:38:47,972] jepsen worker 74 - jepsen.util 13484 :invoke :read [24 nil] INFO [2019-12-17 12:38:47,972] jepsen worker 64 - jepsen.util 10594 :invoke :write [21 21] INFO [2019-12-17 12:38:48,001] jepsen worker 40 - jepsen.util 19210 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:38:48,001] jepsen worker 40 - jepsen.util 19210 :invoke :write [13 12] INFO [2019-12-17 12:38:48,012] jepsen worker 65 - jepsen.util 11945 :fail :cas [21 [2 21]] :ValueNotSwapFrom INFO [2019-12-17 12:38:48,012] jepsen worker 65 - jepsen.util 11945 :invoke :write [21 10] INFO [2019-12-17 12:38:48,020] jepsen worker 57 - jepsen.util 18237 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:38:48,020] jepsen worker 10 - jepsen.util 13420 :fail :read [3 nil] :RequestTimeoutException INFO [2019-12-17 12:38:48,020] jepsen worker 57 - jepsen.util 18237 :invoke :cas [19 [0 28]] INFO [2019-12-17 12:38:48,020] jepsen worker 10 - jepsen.util 13420 :invoke :cas [3 [16 23]] INFO [2019-12-17 12:38:48,023] jepsen worker 47 - jepsen.util 14717 :ok :read [15 37] INFO [2019-12-17 12:38:48,023] jepsen worker 47 - jepsen.util 14717 :invoke :read [15 nil] INFO [2019-12-17 12:38:48,030] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78832,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:38:48,030] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78832,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:38:48,030] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78835,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:38:48,030] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78835,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:38:48,033] jepsen worker 7 - jepsen.util 18547 :info :write [2 28] :RequestCanceledException INFO [2019-12-17 12:38:48,033] jepsen worker 2 - jepsen.util 18002 :info :write [0 15] :RequestCanceledException INFO [2019-12-17 12:38:48,034] jepsen worker 7 - jepsen.util 18637 :invoke :cas [2 [10 9]] INFO [2019-12-17 12:38:48,034] jepsen worker 2 - jepsen.util 18092 :invoke :read [0 nil] INFO [2019-12-17 12:38:48,045] jepsen worker 47 - jepsen.util 14717 :ok :read [15 37] INFO [2019-12-17 12:38:48,045] jepsen worker 47 - jepsen.util 14717 :invoke :write [15 15] INFO [2019-12-17 12:38:48,049] jepsen worker 16 - jepsen.util 20356 :info :cas [5 [46 18]] :RequestCanceledException INFO [2019-12-17 12:38:48,049] jepsen worker 85 - jepsen.util 19075 :info :write [28 25] :RequestCanceledException INFO [2019-12-17 12:38:48,050] jepsen worker 85 - jepsen.util 19165 :invoke :cas [28 [22 46]] INFO [2019-12-17 12:38:48,050] jepsen worker 16 - jepsen.util 20446 :invoke :cas [5 [48 38]] INFO [2019-12-17 12:38:48,057] jepsen worker 29 - jepsen.util 11729 :ok :write [9 19] INFO [2019-12-17 12:38:48,057] jepsen worker 29 - jepsen.util 11729 :invoke :cas [9 [3 41]] INFO [2019-12-17 12:38:48,059] jepsen worker 54 - jepsen.util 14364 :ok :read [18 36] INFO [2019-12-17 12:38:48,059] jepsen worker 54 - jepsen.util 14364 :invoke :cas [18 [39 43]] INFO [2019-12-17 12:38:48,066] jepsen worker 60 - jepsen.util 19770 :info :write [20 9] :RequestCanceledException INFO [2019-12-17 12:38:48,066] jepsen worker 60 - jepsen.util 19860 :invoke :cas [20 [44 22]] INFO [2019-12-17 12:38:48,076] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:38:48,076] jepsen nemesis - couchbase.util Rebalance check count: 619 INFO [2019-12-17 12:38:48,076] jepsen nemesis - couchbase.util Rebalance stuck count: 600 INFO [2019-12-17 12:38:48,076] jepsen nemesis - couchbase.util Rebalance stuck INFO [2019-12-17 12:38:48,082] jepsen worker 31 - jepsen.util 11821 :fail :cas [10 [21 38]] :ValueNotSwapFrom INFO [2019-12-17 12:38:48,082] jepsen worker 31 - jepsen.util 11821 :invoke :read [10 nil] ERROR [2019-12-17 12:38:48,090] jepsen nemesis - couchbase.core Caught exception in nemesis, aborting test. INFO [2019-12-17 12:38:48,096] jepsen worker 29 - jepsen.util 11729 :fail :cas [9 [3 41]] :ValueNotSwapFrom INFO [2019-12-17 12:38:48,096] jepsen worker 9 - jepsen.util 14049 :ok :write [3 0] INFO [2019-12-17 12:38:48,102] jepsen worker 76 - jepsen.util 18796 :info :write [25 20] :RequestCanceledException INFO [2019-12-17 12:38:48,102] jepsen worker 59 - jepsen.util 17789 :info :write [19 20] :RequestCanceledException INFO [2019-12-17 12:38:48,119] jepsen worker 47 - jepsen.util 14717 :ok :write [15 15] INFO [2019-12-17 12:38:48,119] jepsen worker 33 - jepsen.util 18393 :info :cas [11 [47 7]] :RequestCanceledException INFO [2019-12-17 12:38:48,135] jepsen worker 84 - jepsen.util 17994 :info :write [28 4] :RequestCanceledException INFO [2019-12-17 12:38:48,135] jepsen worker 77 - jepsen.util 19787 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:38:48,135] jepsen worker 62 - jepsen.util 19232 :info :write [20 3] :RequestCanceledException INFO [2019-12-17 12:38:48,135] jepsen worker 69 - jepsen.util 13929 :info :write [23 22] :RequestCanceledException INFO [2019-12-17 12:38:48,135] jepsen worker 21 - jepsen.util 18111 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:38:48,142] jepsen worker 64 - jepsen.util 10594 :ok :write [21 21] INFO [2019-12-17 12:38:48,148] jepsen worker 86 - jepsen.util 19436 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:38:48,148] jepsen worker 32 - jepsen.util 13802 :fail :cas [10 [23 46]] :ValueNotSwapFrom INFO [2019-12-17 12:38:48,148] jepsen worker 66 - jepsen.util 18876 :info :cas [22 [13 28]] :RequestCanceledException INFO [2019-12-17 12:38:48,165] jepsen worker 23 - jepsen.util 18833 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:38:48,165] jepsen worker 49 - jepsen.util 18499 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:38:48,165] jepsen worker 58 - jepsen.util 18778 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:38:48,165] jepsen worker 82 - jepsen.util 18622 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:38:48,169] jepsen worker 36 - jepsen.util 14256 :ok :read [12 28] INFO [2019-12-17 12:38:48,169] jepsen worker 65 - jepsen.util 11945 :ok :write [21 10] INFO [2019-12-17 12:38:48,171] jepsen worker 19 - jepsen.util 14419 :fail :cas [6 [35 6]] :ValueNotSwapFrom INFO [2019-12-17 12:38:48,171] jepsen worker 10 - jepsen.util 13420 :fail :cas [3 [16 23]] :ValueNotSwapFrom INFO [2019-12-17 12:38:48,192] jepsen worker 52 - jepsen.util 13102 :ok :read [17 33] INFO [2019-12-17 12:38:48,202] jepsen worker 15 - jepsen.util 17475 :info :write [5 33] :RequestCanceledException INFO [2019-12-17 12:38:48,202] jepsen worker 39 - jepsen.util 18039 :info :cas [13 [25 21]] :RequestCanceledException INFO [2019-12-17 12:38:48,202] jepsen worker 43 - jepsen.util 18853 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:38:48,202] jepsen worker 50 - jepsen.util 18320 :info :cas [16 [20 30]] :RequestCanceledException INFO [2019-12-17 12:38:48,206] jepsen worker 55 - jepsen.util 14275 :ok :read [18 36] INFO [2019-12-17 12:38:48,210] jepsen worker 54 - jepsen.util 14364 :fail :cas [18 [39 43]] :ValueNotSwapFrom INFO [2019-12-17 12:38:48,220] jepsen worker 0 - jepsen.util 19080 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:38:48,220] jepsen worker 41 - jepsen.util 19841 :info :cas [13 [31 8]] :RequestCanceledException INFO [2019-12-17 12:38:48,220] jepsen worker 48 - jepsen.util 17148 :info :cas [16 [18 7]] :RequestCanceledException INFO [2019-12-17 12:38:48,220] jepsen worker 25 - jepsen.util 19825 :info :cas [8 [21 28]] :RequestCanceledException INFO [2019-12-17 12:38:48,220] jepsen worker 63 - jepsen.util 11943 :info :cas [21 [15 35]] :RequestCanceledException WARN [2019-12-17 12:38:48,227] jepsen nemesis - jepsen.core Process :nemesis crashed java.lang.RuntimeException: Rebalance stuck at couchbase.util$wait_for_rebalance_complete.invokeStatic(util.clj:202) ~[classes/:na] at couchbase.util$wait_for_rebalance_complete.invoke(util.clj:188) ~[classes/:na] at couchbase.util$wait_for_rebalance_complete.invokeStatic(util.clj:189) ~[classes/:na] at couchbase.util$wait_for_rebalance_complete.invoke(util.clj:188) ~[classes/:na] at couchbase.nemesis$failover.invokeStatic(nemesis.clj:82) ~[classes/:na] at couchbase.nemesis$failover.invoke(nemesis.clj:70) ~[classes/:na] at couchbase.nemesis$couchbase$reify__1064.invoke_BANG_(nemesis.clj:386) ~[na:na] at jepsen.nemesis$invoke_compat_BANG_.invokeStatic(nemesis.clj:40) ~[jepsen-0.1.15.jar:na] at jepsen.nemesis$invoke_compat_BANG_.invoke(nemesis.clj:36) ~[jepsen-0.1.15.jar:na] at couchbase.core$_main$fn__1390$fn__1391.invoke(core.clj:379) ~[na:na] at jepsen.core$nemesis_invoke_op_BANG_$fn__5547.invoke(core.clj:256) ~[jepsen-0.1.15.jar:na] at jepsen.core$nemesis_invoke_op_BANG_.invokeStatic(core.clj:256) [jepsen-0.1.15.jar:na] at jepsen.core$nemesis_invoke_op_BANG_.invoke(core.clj:251) [jepsen-0.1.15.jar:na] at jepsen.core$nemesis_apply_op_BANG_.invokeStatic(core.clj:291) [jepsen-0.1.15.jar:na] at jepsen.core$nemesis_apply_op_BANG_.invoke(core.clj:283) [jepsen-0.1.15.jar:na] at jepsen.core.NemesisWorker.run_worker_BANG_(core.clj:407) [jepsen-0.1.15.jar:na] at jepsen.core$run_workers_BANG_$run__5531.invoke(core.clj:203) [jepsen-0.1.15.jar:na] at dom_top.core$real_pmap_helper$build_thread__214$fn__215.invoke(core.clj:146) [jepsen-0.1.15.jar:na] at clojure.lang.AFn.applyToHelper(AFn.java:152) [clojure-1.10.1.jar:na] at clojure.lang.AFn.applyTo(AFn.java:144) [clojure-1.10.1.jar:na] at clojure.core$apply.invokeStatic(core.clj:665) [clojure-1.10.1.jar:na] at clojure.core$with_bindings_STAR_.invokeStatic(core.clj:1973) [clojure-1.10.1.jar:na] at clojure.core$with_bindings_STAR_.doInvoke(core.clj:1973) [clojure-1.10.1.jar:na] at clojure.lang.RestFn.invoke(RestFn.java:425) [clojure-1.10.1.jar:na] at clojure.lang.AFn.applyToHelper(AFn.java:156) [clojure-1.10.1.jar:na] at clojure.lang.RestFn.applyTo(RestFn.java:132) [clojure-1.10.1.jar:na] at clojure.core$apply.invokeStatic(core.clj:669) [clojure-1.10.1.jar:na] at clojure.core$bound_fn_STAR_$fn__5749.doInvoke(core.clj:2003) [clojure-1.10.1.jar:na] at clojure.lang.RestFn.invoke(RestFn.java:397) [clojure-1.10.1.jar:na] at clojure.lang.AFn.run(AFn.java:22) [clojure-1.10.1.jar:na] at java.lang.Thread.run(Thread.java:748) [na:1.8.0_222] INFO [2019-12-17 12:38:48,231] jepsen nemesis - jepsen.util :nemesis :info :failover nil indeterminate: Rebalance stuck INFO [2019-12-17 12:38:48,234] jepsen worker 8 - jepsen.util 18188 :info :cas [2 [26 37]] :RequestCanceledException INFO [2019-12-17 12:38:48,234] jepsen worker 1 - jepsen.util 18991 :info :write [0 0] :RequestCanceledException INFO [2019-12-17 12:38:48,235] jepsen worker 17 - jepsen.util 18377 :info :cas [5 [44 2]] :RequestCanceledException INFO [2019-12-17 12:38:48,235] jepsen worker 68 - jepsen.util 17798 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:38:48,235] jepsen worker 42 - jepsen.util 19482 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:38:48,249] jepsen worker 75 - jepsen.util 18345 :info :cas [25 [31 46]] :RequestCanceledException INFO [2019-12-17 12:38:48,252] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78858,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:38:48,252] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78858,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:38:48,253] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78861,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:38:48,253] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78861,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:38:48,253] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78862,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:38:48,253] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78862,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:38:48,253] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78851,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:38:48,253] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78851,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:38:48,253] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78853,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:38:48,253] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78853,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:38:48,278] jepsen worker 78 - jepsen.util 13488 :ok :read [26 39] INFO [2019-12-17 12:38:48,320] jepsen worker 44 - jepsen.util 19304 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:38:48,349] jepsen worker 27 - jepsen.util 12357 :info :write [9 11] :RequestCanceledException INFO [2019-12-17 12:38:48,401] jepsen worker 18 - jepsen.util 14328 :fail :read [6 nil] :RequestTimeoutException INFO [2019-12-17 12:38:48,417] jepsen worker 6 - jepsen.util 18726 :info :cas [2 [23 48]] :RequestCanceledException INFO [2019-12-17 12:38:48,435] jepsen worker 34 - jepsen.util 19744 :info :write [11 9] :RequestCanceledException INFO [2019-12-17 12:38:48,458] jepsen worker 70 - jepsen.util 13930 :ok :read [23 47] INFO [2019-12-17 12:38:48,472] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78864,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:38:48,472] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78864,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:38:48,472] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78868,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:38:48,472] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78868,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:38:48,544] jepsen worker 56 - jepsen.util 14186 :ok :write [18 21] INFO [2019-12-17 12:38:48,568] jepsen worker 12 - jepsen.util 13602 :ok :write [4 29] INFO [2019-12-17 12:38:48,599] jepsen worker 81 - jepsen.util 17991 :info :cas [27 [13 42]] :RequestCanceledException INFO [2019-12-17 12:38:48,634] jepsen worker 26 - jepsen.util 19556 :info :cas [8 [49 35]] :RequestCanceledException INFO [2019-12-17 12:38:48,634] jepsen worker 61 - jepsen.util 19951 :info :write [20 18] :RequestCanceledException INFO [2019-12-17 12:38:48,649] jepsen worker 35 - jepsen.util 18845 :info :cas [11 [21 42]] :RequestCanceledException INFO [2019-12-17 12:38:48,666] jepsen worker 83 - jepsen.util 18533 :info :write [27 8] :RequestCanceledException INFO [2019-12-17 12:38:48,688] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78870,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:38:48,689] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78870,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:38:48,689] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78874,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:38:48,689] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78874,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:38:48,835] jepsen worker 87 - jepsen.util 13317 :info :write [29 43] :RequestCanceledException INFO [2019-12-17 12:38:48,864] jepsen worker 53 - jepsen.util 13373 :info :write [17 16] :RequestCanceledException INFO [2019-12-17 12:38:48,883] jepsen worker 51 - jepsen.util 12921 :ok :write [17 12] INFO [2019-12-17 12:38:48,896] jepsen worker 79 - jepsen.util 13759 :ok :write [26 31] INFO [2019-12-17 12:38:48,908] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78875,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:38:48,909] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78875,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:38:48,909] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78879,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:38:48,909] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78879,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:38:48,919] jepsen worker 45 - jepsen.util 13185 :info :write [15 8] :RequestCanceledException INFO [2019-12-17 12:38:48,967] jepsen worker 14 - jepsen.util 13154 :info :write [4 20] :RequestCanceledException INFO [2019-12-17 12:38:48,998] jepsen worker 13 - jepsen.util 13243 :ok :write [4 10] INFO [2019-12-17 12:38:49,013] jepsen worker 71 - jepsen.util 14471 :ok :write [23 11] INFO [2019-12-17 12:38:49,018] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78881,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:38:49,018] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78881,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:38:49,018] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78883,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:38:49,018] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78883,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:38:49,049] jepsen worker 30 - jepsen.util 13080 :info :cas [10 [35 38]] :RequestCanceledException INFO [2019-12-17 12:38:49,070] jepsen worker 37 - jepsen.util 13087 :ok :write [12 41] INFO [2019-12-17 12:38:49,119] jepsen worker 38 - jepsen.util 14258 :info :write [12 39] :RequestCanceledException INFO [2019-12-17 12:38:49,163] jepsen worker 89 - jepsen.util 13769 :ok :write [29 9] INFO [2019-12-17 12:38:49,167] jepsen worker 20 - jepsen.util 13340 :info :write [6 22] :RequestCanceledException INFO [2019-12-17 12:38:49,234] jepsen worker 5 - jepsen.util 13145 :ok :write [1 42] INFO [2019-12-17 12:38:49,235] jepsen worker 80 - jepsen.util 12950 :info :write [26 27] :RequestCanceledException INFO [2019-12-17 12:38:49,236] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78888,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:38:49,236] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78895,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:38:49,237] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78888,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:38:49,237] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78895,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:38:49,286] jepsen worker 88 - jepsen.util 13498 :ok :write [29 0] INFO [2019-12-17 12:38:49,302] jepsen worker 72 - jepsen.util 13662 :fail :read [24 nil] :RequestTimeoutException INFO [2019-12-17 12:38:49,331] jepsen worker 3 - jepsen.util 13233 :ok :write [1 36] INFO [2019-12-17 12:38:49,365] jepsen worker 4 - jepsen.util 12874 :ok :write [1 35] INFO [2019-12-17 12:38:49,455] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78903,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:38:49,455] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78903,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:38:49,549] jepsen worker 46 - jepsen.util 13276 :info :cas [15 [29 39]] :RequestCanceledException INFO [2019-12-17 12:38:49,667] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78908,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:38:49,667] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78908,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:38:49,866] jepsen worker 11 - jepsen.util 13421 :info :write [3 26] :RequestCanceledException INFO [2019-12-17 12:38:49,886] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78912,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:38:49,887] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78912,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:38:49,887] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78916,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:38:49,887] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78916,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:38:49,964] jepsen worker 22 - jepsen.util 19192 :info :write [7 3] :RequestCanceledException INFO [2019-12-17 12:38:49,964] jepsen worker 28 - jepsen.util 11548 :fail :read [9 nil] :RequestTimeoutException INFO [2019-12-17 12:38:50,013] jepsen worker 67 - jepsen.util 19147 :info :write [22 2] :RequestCanceledException INFO [2019-12-17 12:38:50,033] jepsen worker 73 - jepsen.util 12493 :info :write [24 41] :RequestCanceledException INFO [2019-12-17 12:38:50,033] jepsen worker 74 - jepsen.util 13484 :fail :read [24 nil] :RequestTimeoutException INFO [2019-12-17 12:38:50,033] jepsen worker 24 - jepsen.util 17844 :fail :read [8 nil] :RequestTimeoutException INFO [2019-12-17 12:38:50,102] jepsen worker 7 - jepsen.util 18637 :info :cas [2 [10 9]] :RequestCanceledException INFO [2019-12-17 12:38:50,102] jepsen worker 40 - jepsen.util 19210 :info :write [13 12] :RequestCanceledException INFO [2019-12-17 12:38:50,105] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78920,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:38:50,106] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78920,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:38:50,120] jepsen worker 57 - jepsen.util 18237 :info :cas [19 [0 28]] :RequestCanceledException INFO [2019-12-17 12:38:50,120] jepsen worker 2 - jepsen.util 18092 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:38:50,135] jepsen worker 31 - jepsen.util 11821 :fail :read [10 nil] :RequestTimeoutException INFO [2019-12-17 12:38:50,135] jepsen worker 85 - jepsen.util 19165 :info :cas [28 [22 46]] :RequestCanceledException INFO [2019-12-17 12:38:50,148] jepsen worker 16 - jepsen.util 20446 :info :cas [5 [48 38]] :RequestCanceledException INFO [2019-12-17 12:38:50,167] jepsen worker 60 - jepsen.util 19860 :info :cas [20 [44 22]] :RequestCanceledException INFO [2019-12-17 12:38:50,215] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78923,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:38:50,215] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78923,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:38:50,215] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78925,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:38:50,215] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78925,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:38:50,322] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78930,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:38:50,322] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78930,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:38:50,322] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78934,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:38:50,322] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78934,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:38:50,540] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78936,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:38:50,540] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78936,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:38:50,540] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78940,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:38:50,540] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78940,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:38:50,759] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78942,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:38:50,760] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78942,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:38:50,760] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78946,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:38:50,760] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78946,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:38:50,975] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78950,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:38:50,975] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78950,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:38:51,084] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78956,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:38:51,084] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78956,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:38:51,084] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78958,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:38:51,084] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78958,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:38:51,302] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78960,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:38:51,302] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78966,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:38:51,302] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78960,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:38:51,302] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78966,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:38:51,519] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78974,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:38:51,519] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78974,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:38:51,737] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78980,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:38:51,737] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78980,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:38:51,956] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78984,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:38:51,956] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78984,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:38:51,957] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78988,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:38:51,957] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78988,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:38:52,066] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78990,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:38:52,066] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78990,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:38:52,285] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78995,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:38:52,285] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79002,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:38:52,285] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78995,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:38:52,286] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79002,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:38:52,286] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78997,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:38:52,286] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79006,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:38:52,286] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":78997,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:38:52,286] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79006,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:38:52,504] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79008,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:38:52,504] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79008,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:38:52,504] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79012,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:38:52,504] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79012,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:38:52,721] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79014,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:38:52,721] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79014,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:38:52,721] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79018,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:38:52,721] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79018,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:38:52,940] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79021,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:38:52,940] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79021,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:38:53,050] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79026,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:38:53,050] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79026,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:38:53,050] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79030,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:38:53,050] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79030,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:38:53,267] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79039,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:38:53,267] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79032,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:38:53,267] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79039,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:38:53,267] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79032,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:38:53,486] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79046,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:38:53,486] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79046,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:38:53,696] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79052,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:38:53,696] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79052,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:38:53,914] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79057,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:38:53,914] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79057,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:38:53,914] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79060,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:38:53,914] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79060,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:38:54,021] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79064,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:38:54,021] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79064,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:38:54,233] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79074,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:38:54,233] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79067,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:38:54,233] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79074,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:38:54,233] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79067,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:38:54,233] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79078,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:38:54,233] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79069,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:38:54,233] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79078,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:38:54,233] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79069,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:38:54,450] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79080,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:38:54,450] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79080,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:38:54,450] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79084,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:38:54,450] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79084,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:38:54,669] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79086,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:38:54,669] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79086,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:38:54,669] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79090,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:38:54,669] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79090,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:38:54,891] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79093,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:38:54,891] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79093,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:38:55,105] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79097,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:38:55,105] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79097,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:38:55,105] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79102,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:38:55,105] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79102,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:38:55,214] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79104,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:38:55,214] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79104,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:38:55,324] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79111,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:38:55,324] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79111,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:38:55,540] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79118,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:38:55,760] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79123,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:38:55,760] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79123,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:38:55,871] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79127,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:38:55,871] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79127,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:38:55,871] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79131,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:38:55,872] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79131,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:38:56,088] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79133,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:38:56,088] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79133,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:38:56,304] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79145,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:38:56,304] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79138,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:38:56,305] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79145,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:38:56,305] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79149,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:38:56,305] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79138,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:38:56,305] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79149,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:38:56,305] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79140,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:38:56,305] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79140,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:38:56,523] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79150,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:38:56,523] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79150,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:38:56,523] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79155,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:38:56,523] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79155,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:38:56,739] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79157,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:38:56,739] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79157,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:38:56,739] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79161,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:38:56,739] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79161,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:38:56,958] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79165,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:38:56,959] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79165,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:38:57,068] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79168,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:38:57,068] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79168,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:38:57,068] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79173,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:38:57,068] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79173,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:38:57,283] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79175,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:38:57,283] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79182,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:38:57,283] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79175,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:38:57,283] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79182,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:38:57,501] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79189,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:38:57,501] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79189,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:38:57,718] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79195,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:38:57,718] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79195,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:38:57,936] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79199,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:38:57,936] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79199,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:38:57,936] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79203,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:38:57,936] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79203,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:38:58,042] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79207,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:38:58,042] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79207,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:38:58,244] jepsen nemesis - jepsen.util :nemesis :info :recover nil INFO [2019-12-17 12:38:58,260] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79217,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:38:58,260] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79210,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:38:58,260] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79217,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:38:58,260] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79210,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:38:58,260] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79221,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:38:58,260] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79212,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:38:58,260] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79212,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:38:58,260] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79221,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:38:58,370] jepsen nemesis - couchbase.nemesis No recovery necessary INFO [2019-12-17 12:38:58,371] jepsen nemesis - jepsen.util :nemesis :info :recover () INFO [2019-12-17 12:38:58,478] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79223,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:38:58,479] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79223,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:38:58,479] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79227,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:38:58,479] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79227,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:38:58,696] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79230,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:38:58,696] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79230,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:38:58,696] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79233,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:38:58,696] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79233,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:38:58,915] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79235,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:38:58,915] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79235,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:38:59,023] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79241,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:38:59,023] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79241,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:38:59,023] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79245,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:38:59,024] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79245,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:38:59,243] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79254,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:38:59,243] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79247,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:38:59,243] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79254,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:38:59,243] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79247,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:38:59,459] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79261,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:38:59,459] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79261,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:38:59,678] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79267,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:38:59,678] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79267,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:38:59,898] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79271,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:38:59,898] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79271,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:38:59,898] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79275,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:38:59,898] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79275,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:39:00,007] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79277,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:39:00,007] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79277,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:39:00,223] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79282,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:39:00,223] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79290,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:39:00,223] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79282,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:39:00,223] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79290,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:39:00,223] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79284,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:39:00,223] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79293,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:39:00,223] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79284,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:39:00,223] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79293,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:39:00,440] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79295,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:39:00,440] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79295,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:39:00,440] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79299,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:39:00,440] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79299,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:39:00,654] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79301,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:39:00,654] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79301,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:39:00,654] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79305,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:39:00,654] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79305,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:39:00,873] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79309,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:39:00,873] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79309,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:39:01,093] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79312,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:39:01,093] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79312,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:39:01,093] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79314,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:39:01,093] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79314,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:39:01,311] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79319,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:39:01,311] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79328,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:39:01,311] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79319,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:39:01,311] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79328,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:39:01,530] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79334,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:39:01,530] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79334,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:39:01,746] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79337,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:39:01,746] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79337,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:39:01,962] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79344,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:39:01,962] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79344,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:39:01,962] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79347,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:39:01,962] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79347,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:39:02,072] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79351,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:39:02,072] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79351,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:39:02,290] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79361,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:39:02,290] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79354,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:39:02,290] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79361,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:39:02,290] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79354,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:39:02,290] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79365,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:39:02,290] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79356,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:39:02,290] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79365,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:39:02,290] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79356,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:39:02,509] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79367,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:39:02,509] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79367,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:39:02,509] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79371,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:39:02,510] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79371,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:39:02,728] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79373,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:39:02,728] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79373,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:39:02,728] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79377,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:39:02,728] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79377,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:39:02,945] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79381,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:39:02,945] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79381,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:39:03,054] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79385,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:39:03,054] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79385,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:39:03,054] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79389,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:39:03,054] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79389,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:39:03,274] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79400,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:39:03,274] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79391,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:39:03,274] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79400,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:39:03,274] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79391,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:39:03,493] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79405,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:39:03,494] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79405,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:39:03,713] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79411,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:39:03,713] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79411,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:39:03,932] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79416,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:39:03,932] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79416,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:39:03,932] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79419,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:39:03,932] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79419,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:39:04,041] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79423,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:39:04,042] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79423,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:39:04,260] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79433,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:39:04,260] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79426,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:39:04,261] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79433,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:39:04,263] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79426,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:39:04,263] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79437,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:39:04,263] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79428,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:39:04,263] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79437,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:39:04,263] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79428,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:39:04,480] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79439,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:39:04,480] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79439,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:39:04,480] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79443,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:39:04,480] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79443,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:39:04,698] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79445,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:39:04,699] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79445,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:39:04,699] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79449,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:39:04,699] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79449,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:39:04,917] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79453,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:39:04,917] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79453,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:39:05,025] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79457,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:39:05,026] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79457,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:39:05,026] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79461,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:39:05,026] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79461,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:39:05,238] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79463,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:39:05,240] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79463,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:39:05,240] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79470,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:39:05,240] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79470,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:39:05,447] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79477,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:39:05,447] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79477,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:39:05,659] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79483,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:39:05,659] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79483,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:39:05,878] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79488,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:39:05,878] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79488,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:39:05,878] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79491,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:39:05,878] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79491,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:39:06,092] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79495,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:39:06,092] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79495,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:39:06,311] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79505,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:39:06,311] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79498,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:39:06,311] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79505,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:39:06,311] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79498,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:39:06,312] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79509,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:39:06,312] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79500,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:39:06,312] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79509,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:39:06,312] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79500,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:39:06,449] jepsen worker 9 - jepsen.core Tearing down worker 9 INFO [2019-12-17 12:39:06,449] jepsen worker 24 - jepsen.core Tearing down worker 24 INFO [2019-12-17 12:39:06,449] jepsen worker 12 - jepsen.core Tearing down worker 12 INFO [2019-12-17 12:39:06,449] jepsen worker 11 - jepsen.core Tearing down worker 11 INFO [2019-12-17 12:39:06,449] jepsen worker 10 - jepsen.core Tearing down worker 10 INFO [2019-12-17 12:39:06,449] jepsen worker 7 - jepsen.core Tearing down worker 7 INFO [2019-12-17 12:39:06,449] jepsen worker 16 - jepsen.core Tearing down worker 16 INFO [2019-12-17 12:39:06,449] jepsen worker 15 - jepsen.core Tearing down worker 15 INFO [2019-12-17 12:39:06,449] jepsen worker 2 - jepsen.core Tearing down worker 2 INFO [2019-12-17 12:39:06,449] jepsen worker 19 - jepsen.core Tearing down worker 19 INFO [2019-12-17 12:39:06,449] jepsen worker 22 - jepsen.core Tearing down worker 22 INFO [2019-12-17 12:39:06,449] jepsen worker 21 - jepsen.core Tearing down worker 21 INFO [2019-12-17 12:39:06,449] jepsen worker 0 - jepsen.core Tearing down worker 0 INFO [2019-12-17 12:39:06,449] jepsen worker 6 - jepsen.core Tearing down worker 6 INFO [2019-12-17 12:39:06,449] jepsen worker 18 - jepsen.core Tearing down worker 18 INFO [2019-12-17 12:39:06,449] jepsen worker 8 - jepsen.core Tearing down worker 8 INFO [2019-12-17 12:39:06,449] jepsen worker 13 - jepsen.core Tearing down worker 13 INFO [2019-12-17 12:39:06,449] jepsen worker 5 - jepsen.core Tearing down worker 5 INFO [2019-12-17 12:39:06,449] jepsen worker 1 - jepsen.core Tearing down worker 1 INFO [2019-12-17 12:39:06,449] jepsen worker 14 - jepsen.core Tearing down worker 14 INFO [2019-12-17 12:39:06,449] jepsen worker 25 - jepsen.core Tearing down worker 25 INFO [2019-12-17 12:39:06,449] jepsen worker 20 - jepsen.core Tearing down worker 20 INFO [2019-12-17 12:39:06,449] jepsen worker 4 - jepsen.core Tearing down worker 4 INFO [2019-12-17 12:39:06,449] jepsen worker 3 - jepsen.core Tearing down worker 3 INFO [2019-12-17 12:39:06,449] jepsen worker 23 - jepsen.core Tearing down worker 23 INFO [2019-12-17 12:39:06,449] jepsen worker 26 - jepsen.core Tearing down worker 26 INFO [2019-12-17 12:39:06,449] jepsen worker 27 - jepsen.core Tearing down worker 27 INFO [2019-12-17 12:39:06,449] jepsen worker 28 - jepsen.core Tearing down worker 28 INFO [2019-12-17 12:39:06,449] jepsen worker 30 - jepsen.core Tearing down worker 30 INFO [2019-12-17 12:39:06,449] jepsen worker 29 - jepsen.core Tearing down worker 29 INFO [2019-12-17 12:39:06,449] jepsen worker 31 - jepsen.core Tearing down worker 31 INFO [2019-12-17 12:39:06,452] jepsen worker 33 - jepsen.core Tearing down worker 33 INFO [2019-12-17 12:39:06,452] jepsen worker 35 - jepsen.core Tearing down worker 35 INFO [2019-12-17 12:39:06,452] jepsen worker 32 - jepsen.core Tearing down worker 32 INFO [2019-12-17 12:39:06,452] jepsen worker 36 - jepsen.core Tearing down worker 36 INFO [2019-12-17 12:39:06,452] jepsen worker 38 - jepsen.core Tearing down worker 38 INFO [2019-12-17 12:39:06,452] jepsen worker 39 - jepsen.core Tearing down worker 39 INFO [2019-12-17 12:39:06,452] jepsen worker 40 - jepsen.core Tearing down worker 40 INFO [2019-12-17 12:39:06,452] jepsen worker 17 - jepsen.core Tearing down worker 17 INFO [2019-12-17 12:39:06,452] jepsen worker 37 - jepsen.core Tearing down worker 37 INFO [2019-12-17 12:39:06,452] jepsen worker 41 - jepsen.core Tearing down worker 41 INFO [2019-12-17 12:39:06,453] jepsen worker 42 - jepsen.core Tearing down worker 42 INFO [2019-12-17 12:39:06,453] jepsen worker 43 - jepsen.core Tearing down worker 43 INFO [2019-12-17 12:39:06,453] jepsen worker 44 - jepsen.core Tearing down worker 44 INFO [2019-12-17 12:39:06,453] jepsen worker 45 - jepsen.core Tearing down worker 45 INFO [2019-12-17 12:39:06,453] jepsen worker 46 - jepsen.core Tearing down worker 46 INFO [2019-12-17 12:39:06,454] jepsen worker 47 - jepsen.core Tearing down worker 47 INFO [2019-12-17 12:39:06,454] jepsen nemesis - jepsen.core Tearing down nemesis INFO [2019-12-17 12:39:06,454] jepsen worker 48 - jepsen.core Tearing down worker 48 INFO [2019-12-17 12:39:06,454] jepsen worker 49 - jepsen.core Tearing down worker 49 INFO [2019-12-17 12:39:06,454] jepsen worker 50 - jepsen.core Tearing down worker 50 INFO [2019-12-17 12:39:06,454] jepsen worker 51 - jepsen.core Tearing down worker 51 INFO [2019-12-17 12:39:06,454] jepsen worker 52 - jepsen.core Tearing down worker 52 INFO [2019-12-17 12:39:06,454] jepsen worker 53 - jepsen.core Tearing down worker 53 INFO [2019-12-17 12:39:06,454] jepsen worker 55 - jepsen.core Tearing down worker 55 INFO [2019-12-17 12:39:06,454] jepsen worker 54 - jepsen.core Tearing down worker 54 INFO [2019-12-17 12:39:06,454] jepsen worker 34 - jepsen.core Tearing down worker 34 INFO [2019-12-17 12:39:06,454] jepsen worker 56 - jepsen.core Tearing down worker 56 INFO [2019-12-17 12:39:06,455] jepsen worker 57 - jepsen.core Tearing down worker 57 INFO [2019-12-17 12:39:06,455] jepsen worker 58 - jepsen.core Tearing down worker 58 INFO [2019-12-17 12:39:06,455] jepsen worker 59 - jepsen.core Tearing down worker 59 INFO [2019-12-17 12:39:06,456] jepsen worker 60 - jepsen.core Tearing down worker 60 INFO [2019-12-17 12:39:06,456] jepsen worker 61 - jepsen.core Tearing down worker 61 INFO [2019-12-17 12:39:06,456] jepsen worker 62 - jepsen.core Tearing down worker 62 INFO [2019-12-17 12:39:06,456] jepsen worker 63 - jepsen.core Tearing down worker 63 INFO [2019-12-17 12:39:06,456] jepsen worker 64 - jepsen.core Tearing down worker 64 INFO [2019-12-17 12:39:06,457] jepsen worker 65 - jepsen.core Tearing down worker 65 INFO [2019-12-17 12:39:06,457] jepsen worker 66 - jepsen.core Tearing down worker 66 INFO [2019-12-17 12:39:06,458] jepsen worker 69 - jepsen.core Tearing down worker 69 INFO [2019-12-17 12:39:06,458] jepsen worker 67 - jepsen.core Tearing down worker 67 INFO [2019-12-17 12:39:06,458] jepsen worker 68 - jepsen.core Tearing down worker 68 INFO [2019-12-17 12:39:06,458] jepsen worker 70 - jepsen.core Tearing down worker 70 INFO [2019-12-17 12:39:06,458] jepsen worker 71 - jepsen.core Tearing down worker 71 INFO [2019-12-17 12:39:06,459] jepsen worker 72 - jepsen.core Tearing down worker 72 INFO [2019-12-17 12:39:06,459] jepsen worker 73 - jepsen.core Tearing down worker 73 INFO [2019-12-17 12:39:06,460] jepsen worker 74 - jepsen.core Tearing down worker 74 INFO [2019-12-17 12:39:06,460] jepsen worker 75 - jepsen.core Tearing down worker 75 INFO [2019-12-17 12:39:06,461] jepsen worker 77 - jepsen.core Tearing down worker 77 INFO [2019-12-17 12:39:06,461] jepsen worker 76 - jepsen.core Tearing down worker 76 INFO [2019-12-17 12:39:06,461] jepsen worker 78 - jepsen.core Tearing down worker 78 INFO [2019-12-17 12:39:06,461] jepsen worker 79 - jepsen.core Tearing down worker 79 INFO [2019-12-17 12:39:06,461] jepsen worker 80 - jepsen.core Tearing down worker 80 INFO [2019-12-17 12:39:06,461] jepsen worker 82 - jepsen.core Tearing down worker 82 INFO [2019-12-17 12:39:06,461] jepsen worker 83 - jepsen.core Tearing down worker 83 INFO [2019-12-17 12:39:06,462] jepsen worker 81 - jepsen.core Tearing down worker 81 INFO [2019-12-17 12:39:06,462] jepsen worker 84 - jepsen.core Tearing down worker 84 INFO [2019-12-17 12:39:06,462] jepsen worker 85 - jepsen.core Tearing down worker 85 INFO [2019-12-17 12:39:06,462] jepsen worker 86 - jepsen.core Tearing down worker 86 INFO [2019-12-17 12:39:06,462] jepsen worker 87 - jepsen.core Tearing down worker 87 INFO [2019-12-17 12:39:06,462] jepsen worker 88 - jepsen.core Tearing down worker 88 INFO [2019-12-17 12:39:06,462] jepsen worker 89 - jepsen.core Tearing down worker 89 INFO [2019-12-17 12:39:06,530] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79511,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:39:06,531] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79511,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:39:06,531] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79515,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:39:06,531] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79515,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:39:06,641] cb-events - com.couchbase.core [com.couchbase.core][BucketClosedEvent][318µs] Closed bucket "default" {"coreId":2} INFO [2019-12-17 12:39:06,641] cb-events - com.couchbase.core [com.couchbase.core][BucketClosedEvent][23120µs] Closed bucket "default" {"coreId":1} INFO [2019-12-17 12:39:06,647] cb-events - com.couchbase.node [com.couchbase.node][NodeDisconnectedEvent][651µs] Node disconnected {"coreId":1,"managerPort":"8091","remote":"172.28.128.126"} INFO [2019-12-17 12:39:06,647] cb-events - com.couchbase.node [com.couchbase.node][NodeDisconnectedEvent][33µs] Node disconnected {"coreId":2,"managerPort":"8091","remote":"172.28.128.127"} INFO [2019-12-17 12:39:06,648] cb-events - com.couchbase.node [com.couchbase.node][NodeDisconnectedEvent][17µs] Node disconnected {"coreId":1,"managerPort":"8091","remote":"172.28.128.125"} INFO [2019-12-17 12:39:06,648] cb-events - com.couchbase.node [com.couchbase.node][NodeDisconnectedEvent][42µs] Node disconnected {"coreId":2,"managerPort":"8091","remote":"172.28.128.125"} INFO [2019-12-17 12:39:06,648] cb-events - com.couchbase.node [com.couchbase.node][NodeDisconnectedEvent][33µs] Node disconnected {"coreId":1,"managerPort":"8091","remote":"172.28.128.127"} INFO [2019-12-17 12:39:06,648] cb-events - com.couchbase.node [com.couchbase.node][NodeDisconnectedEvent][48µs] Node disconnected {"coreId":2,"managerPort":"8091","remote":"172.28.128.126"} INFO [2019-12-17 12:39:06,648] cb-events - com.couchbase.node [com.couchbase.node][NodeDisconnectedEvent][33µs] Node disconnected {"coreId":1,"managerPort":"8091","remote":"172.28.128.128"} INFO [2019-12-17 12:39:06,648] cb-events - com.couchbase.node [com.couchbase.node][NodeDisconnectedEvent][25µs] Node disconnected {"coreId":2,"managerPort":"8091","remote":"172.28.128.128"} INFO [2019-12-17 12:39:06,649] cb-events - com.couchbase.core [com.couchbase.core][ShutdownCompletedEvent][114958µs] Completed shutdown and closed all open buckets {"coreId":1} INFO [2019-12-17 12:39:06,749] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79519,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:39:06,749] cb-events - com.couchbase.core [com.couchbase.core][BucketClosedEvent][250µs] Closed bucket "default" {"coreId":6} INFO [2019-12-17 12:39:06,749] cb-events - com.couchbase.core [com.couchbase.core][BucketClosedEvent][1757µs] Closed bucket "default" {"coreId":3} INFO [2019-12-17 12:39:06,749] cb-events - com.couchbase.core [com.couchbase.core][BucketClosedEvent][265µs] Closed bucket "default" {"coreId":4} INFO [2019-12-17 12:39:06,749] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79519,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:39:06,749] cb-events - com.couchbase.node [com.couchbase.node][NodeDisconnectedEvent][32µs] Node disconnected {"coreId":3,"managerPort":"8091","remote":"172.28.128.126"} INFO [2019-12-17 12:39:06,749] cb-events - com.couchbase.node [com.couchbase.node][NodeDisconnectedEvent][87µs] Node disconnected {"coreId":6,"managerPort":"8091","remote":"172.28.128.125"} INFO [2019-12-17 12:39:06,749] cb-events - com.couchbase.node [com.couchbase.node][NodeDisconnectedEvent][47µs] Node disconnected {"coreId":4,"managerPort":"8091","remote":"172.28.128.127"} INFO [2019-12-17 12:39:06,750] cb-events - com.couchbase.node [com.couchbase.node][NodeDisconnectedEvent][21µs] Node disconnected {"coreId":3,"managerPort":"8091","remote":"172.28.128.125"} INFO [2019-12-17 12:39:06,750] cb-events - com.couchbase.node [com.couchbase.node][NodeDisconnectedEvent][20µs] Node disconnected {"coreId":4,"managerPort":"8091","remote":"172.28.128.125"} INFO [2019-12-17 12:39:06,750] cb-events - com.couchbase.node [com.couchbase.node][NodeDisconnectedEvent][30µs] Node disconnected {"coreId":3,"managerPort":"8091","remote":"172.28.128.127"} INFO [2019-12-17 12:39:06,750] cb-events - com.couchbase.node [com.couchbase.node][NodeDisconnectedEvent][743µs] Node disconnected {"coreId":4,"managerPort":"8091","remote":"172.28.128.126"} INFO [2019-12-17 12:39:06,750] cb-events - com.couchbase.node [com.couchbase.node][NodeDisconnectedEvent][38µs] Node disconnected {"coreId":6,"managerPort":"8091","remote":"172.28.128.126"} INFO [2019-12-17 12:39:06,750] cb-events - com.couchbase.node [com.couchbase.node][NodeDisconnectedEvent][505µs] Node disconnected {"coreId":4,"managerPort":"8091","remote":"172.28.128.128"} INFO [2019-12-17 12:39:06,750] cb-events - com.couchbase.node [com.couchbase.node][NodeDisconnectedEvent][667µs] Node disconnected {"coreId":3,"managerPort":"8091","remote":"172.28.128.128"} INFO [2019-12-17 12:39:06,750] cb-events - com.couchbase.node [com.couchbase.node][NodeDisconnectedEvent][35µs] Node disconnected {"coreId":6,"managerPort":"8091","remote":"172.28.128.127"} INFO [2019-12-17 12:39:06,750] cb-events - com.couchbase.core [com.couchbase.core][ShutdownCompletedEvent][17643µs] Completed shutdown and closed all open buckets {"coreId":4} INFO [2019-12-17 12:39:06,750] cb-events - com.couchbase.core [com.couchbase.core][ShutdownCompletedEvent][21958µs] Completed shutdown and closed all open buckets {"coreId":3} INFO [2019-12-17 12:39:06,751] cb-events - com.couchbase.node [com.couchbase.node][NodeDisconnectedEvent][52µs] Node disconnected {"coreId":6,"managerPort":"8091","remote":"172.28.128.128"} INFO [2019-12-17 12:39:06,749] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79521,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:39:06,751] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":79521,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:39:06,751] cb-events - com.couchbase.core [com.couchbase.core][BucketClosedEvent][217µs] Closed bucket "default" {"coreId":5} INFO [2019-12-17 12:39:06,751] cb-events - com.couchbase.node [com.couchbase.node][NodeDisconnectedEvent][29µs] Node disconnected {"coreId":5,"managerPort":"8091","remote":"172.28.128.128"} INFO [2019-12-17 12:39:06,752] cb-events - com.couchbase.node [com.couchbase.node][NodeDisconnectedEvent][27µs] Node disconnected {"coreId":5,"managerPort":"8091","remote":"172.28.128.125"} INFO [2019-12-17 12:39:06,752] cb-events - com.couchbase.node [com.couchbase.node][NodeDisconnectedEvent][36µs] Node disconnected {"coreId":5,"managerPort":"8091","remote":"172.28.128.126"} INFO [2019-12-17 12:39:06,752] cb-events - com.couchbase.node [com.couchbase.node][NodeDisconnectedEvent][59µs] Node disconnected {"coreId":5,"managerPort":"8091","remote":"172.28.128.127"} INFO [2019-12-17 12:39:06,752] cb-events - com.couchbase.core [com.couchbase.core][ShutdownCompletedEvent][23381µs] Completed shutdown and closed all open buckets {"coreId":5} INFO [2019-12-17 12:39:06,758] cb-events - com.couchbase.core [com.couchbase.core][ShutdownCompletedEvent][25491µs] Completed shutdown and closed all open buckets {"coreId":2} INFO [2019-12-17 12:39:06,861] cb-events - com.couchbase.core [com.couchbase.core][ShutdownCompletedEvent][24117µs] Completed shutdown and closed all open buckets {"coreId":6} INFO [2019-12-17 12:39:06,970] jepsen test runner - jepsen.core Run complete, writing INFO [2019-12-17 12:39:06,975] jepsen fressian - couchbase.core Not writing fressian INFO [2019-12-17 12:39:08,574] jepsen test runner - jepsen.core Analyzing... WARN [2019-12-17 12:39:08,615] jepsen test runner - jepsen.core DEPRECATED: Checker model is assigned to test, which is no longer supported. If the checker still needs a model, see `jepsen.checker` documentation for details. INFO [2019-12-17 12:39:08,910] clojure-agent-send-off-pool-6 - knossos.model.memo More than 1024 reachable models; not memoizing models for this search INFO [2019-12-17 12:39:08,958] clojure-agent-send-off-pool-12 - knossos.model.memo More than 1024 reachable models; not memoizing models for this search INFO [2019-12-17 12:39:08,987] clojure-agent-send-off-pool-18 - knossos.model.memo More than 1024 reachable models; not memoizing models for this search INFO [2019-12-17 12:39:08,990] clojure-agent-send-off-pool-17 - knossos.model.memo More than 1024 reachable models; not memoizing models for this search INFO [2019-12-17 12:39:09,014] clojure-agent-send-off-pool-16 - knossos.model.memo More than 1024 reachable models; not memoizing models for this search INFO [2019-12-17 12:39:09,047] clojure-agent-send-off-pool-4 - knossos.model.memo More than 1024 reachable models; not memoizing models for this search INFO [2019-12-17 12:39:09,060] clojure-agent-send-off-pool-11 - knossos.model.memo More than 1024 reachable models; not memoizing models for this search INFO [2019-12-17 12:39:09,096] Thread-177 - knossos.model.memo More than 1024 reachable models; not memoizing models for this search INFO [2019-12-17 12:39:09,116] Thread-171 - knossos.model.memo More than 1024 reachable models; not memoizing models for this search INFO [2019-12-17 12:39:09,119] Thread-174 - knossos.model.memo More than 1024 reachable models; not memoizing models for this search INFO [2019-12-17 12:39:09,120] Thread-180 - knossos.model.memo More than 1024 reachable models; not memoizing models for this search INFO [2019-12-17 12:39:09,129] Thread-183 - knossos.model.memo More than 1024 reachable models; not memoizing models for this search INFO [2019-12-17 12:39:09,135] Thread-168 - knossos.model.memo More than 1024 reachable models; not memoizing models for this search INFO [2019-12-17 12:39:09,154] clojure-agent-send-off-pool-3 - knossos.model.memo More than 1024 reachable models; not memoizing models for this search INFO [2019-12-17 12:39:09,164] Thread-186 - knossos.model.memo More than 1024 reachable models; not memoizing models for this search INFO [2019-12-17 12:39:09,185] clojure-agent-send-off-pool-7 - knossos.model.memo More than 1024 reachable models; not memoizing models for this search INFO [2019-12-17 12:39:09,201] clojure-agent-send-off-pool-15 - knossos.model.memo More than 1024 reachable models; not memoizing models for this search INFO [2019-12-17 12:39:09,318] clojure-agent-send-off-pool-5 - knossos.model.memo More than 1024 reachable models; not memoizing models for this search INFO [2019-12-17 12:39:09,325] clojure-agent-send-off-pool-2 - knossos.model.memo More than 1024 reachable models; not memoizing models for this search INFO [2019-12-17 12:39:09,337] clojure-agent-send-off-pool-14 - knossos.model.memo More than 1024 reachable models; not memoizing models for this search INFO [2019-12-17 12:39:09,485] Thread-189 - knossos.model.memo More than 1024 reachable models; not memoizing models for this search INFO [2019-12-17 12:39:09,550] Thread-192 - knossos.model.memo More than 1024 reachable models; not memoizing models for this search INFO [2019-12-17 12:39:09,604] Thread-195 - knossos.model.memo More than 1024 reachable models; not memoizing models for this search INFO [2019-12-17 12:39:09,685] clojure-agent-send-off-pool-10 - knossos.model.memo More than 1024 reachable models; not memoizing models for this search INFO [2019-12-17 12:39:09,788] Thread-201 - knossos.model.memo More than 1024 reachable models; not memoizing models for this search INFO [2019-12-17 12:39:09,823] Thread-198 - knossos.model.memo More than 1024 reachable models; not memoizing models for this search INFO [2019-12-17 12:39:09,840] Thread-204 - knossos.model.memo More than 1024 reachable models; not memoizing models for this search INFO [2019-12-17 12:39:11,056] Thread-207 - knossos.model.memo More than 1024 reachable models; not memoizing models for this search INFO [2019-12-17 12:39:14,063] knossos reporter - knossos.search {:linear {:space 4, :cost 7.85E+13, :op {:process 421, :type :ok, :f :read, :value 7, :index 2367, :time 24734641332, :cas 1576585664292388864}}, :wgl [:WGL]} INFO [2019-12-17 12:39:14,063] knossos reporter - knossos.search {:linear {:space 4, :cost 2.42E+16, :op {:process 311, :type :ok, :f :read, :value 9, :index 3441, :time 30964845173, :cas 1576585663783370752}}, :wgl [:WGL]} INFO [2019-12-17 12:39:14,076] knossos reporter - knossos.search {:linear {:space 2, :cost 1.62E+11, :op {:process 381, :type :ok, :f :read, :value 20, :index 2798, :time 27104314675, :cas 1576585664662929408}}, :wgl [:WGL]} INFO [2019-12-17 12:39:14,160] knossos reporter - knossos.search {:linear {:space 4, :cost 7.85E+13, :op {:process 373, :type :ok, :f :read, :value 4, :index 2791, :time 27050295842, :cas 1576585664984252416}}, :wgl [:WGL]} INFO [2019-12-17 12:39:14,232] knossos reporter - knossos.search {:linear {:space 2, :cost 2.44E+12, :op {:process 453, :type :ok, :f :read, :value 23, :index 3059, :time 28677692750, :cas 1576585665086095360}}, :wgl [:WGL]} INFO [2019-12-17 12:39:14,258] knossos reporter - knossos.search {:linear {:space 1154, :cost 1.41E+15, :op {:process 358, :type :ok, :f :read, :value 5, :index 3515, :time 31384389116, :cas 1576585665103593472}}, :wgl [:WGL]} INFO [2019-12-17 12:39:14,339] knossos reporter - knossos.search {:linear {:space 2, :cost 4.62E+18, :op {:process 524, :type :ok, :f :read, :value 35, :index 3747, :time 32707203288, :cas 1576585664480673792}}, :wgl [:WGL]} INFO [2019-12-17 12:39:14,339] knossos reporter - knossos.search {:linear {:space 4, :cost 2.30E+10, :op {:process 405, :type :ok, :f :read, :value 35, :index 2147, :time 23377858792, :cas 1576585664332693504}}, :wgl [:WGL]} INFO [2019-12-17 12:39:14,351] knossos reporter - knossos.search {:linear {:space 4, :cost 3.24E+11, :op {:process 289, :type :ok, :f :read, :value 45, :index 2351, :time 24641853933, :cas 1576585663882919936}}, :wgl [:WGL]} INFO [2019-12-17 12:39:17,200] clojure-agent-send-pool-4 - jepsen.independent Wrote /Users/richarddemellow/source/jepsen.couchbase/store/Couchbase/20191217T122702.000Z/independent/27/results.edn INFO [2019-12-17 12:39:17,251] clojure-agent-send-pool-12 - jepsen.independent Wrote /Users/richarddemellow/source/jepsen.couchbase/store/Couchbase/20191217T122702.000Z/independent/22/results.edn INFO [2019-12-17 12:39:17,301] clojure-agent-send-pool-8 - jepsen.independent Wrote /Users/richarddemellow/source/jepsen.couchbase/store/Couchbase/20191217T122702.000Z/independent/28/results.edn INFO [2019-12-17 12:39:17,367] clojure-agent-send-pool-1 - jepsen.independent Wrote /Users/richarddemellow/source/jepsen.couchbase/store/Couchbase/20191217T122702.000Z/independent/0/results.edn INFO [2019-12-17 12:39:17,391] clojure-agent-send-pool-8 - jepsen.independent Wrote /Users/richarddemellow/source/jepsen.couchbase/store/Couchbase/20191217T122702.000Z/independent/28/history.edn INFO [2019-12-17 12:39:17,421] clojure-agent-send-pool-4 - jepsen.independent Wrote /Users/richarddemellow/source/jepsen.couchbase/store/Couchbase/20191217T122702.000Z/independent/27/history.edn INFO [2019-12-17 12:39:17,427] clojure-agent-send-pool-12 - jepsen.independent Wrote /Users/richarddemellow/source/jepsen.couchbase/store/Couchbase/20191217T122702.000Z/independent/22/history.edn INFO [2019-12-17 12:39:17,883] clojure-agent-send-pool-1 - jepsen.independent Wrote /Users/richarddemellow/source/jepsen.couchbase/store/Couchbase/20191217T122702.000Z/independent/0/history.edn INFO [2019-12-17 12:39:17,969] clojure-agent-send-off-pool-13 - knossos.model.memo More than 1024 reachable models; not memoizing models for this search INFO [2019-12-17 12:39:18,084] Thread-242 - knossos.model.memo More than 1024 reachable models; not memoizing models for this search INFO [2019-12-17 12:39:18,117] clojure-agent-send-off-pool-43 - knossos.model.memo More than 1024 reachable models; not memoizing models for this search INFO [2019-12-17 12:39:18,120] clojure-agent-send-off-pool-24 - knossos.model.memo More than 1024 reachable models; not memoizing models for this search INFO [2019-12-17 12:39:18,550] clojure-agent-send-off-pool-73 - knossos.model.memo More than 1024 reachable models; not memoizing models for this search INFO [2019-12-17 12:39:18,556] Thread-244 - knossos.model.memo More than 1024 reachable models; not memoizing models for this search INFO [2019-12-17 12:39:18,626] Thread-246 - knossos.model.memo More than 1024 reachable models; not memoizing models for this search INFO [2019-12-17 12:39:18,813] Thread-248 - knossos.model.memo More than 1024 reachable models; not memoizing models for this search INFO [2019-12-17 12:39:19,233] knossos reporter - knossos.search {:linear {:space 2, :cost 3.92E+13, :op {:process 453, :type :ok, :f :read, :value 23, :index 3439, :time 30958589456, :cas 1576585665086095360}}, :wgl [:WGL]} INFO [2019-12-17 12:39:19,264] knossos reporter - knossos.search {:linear {:space 2, :cost 2.30E+17, :op {:process 562, :type :ok, :f :read, :value 20, :index 3393, :time 30602846742, :cas 1576585664662929408}}, :wgl [:WGL]} INFO [2019-12-17 12:39:19,292] knossos reporter - knossos.search {:linear {:space 4610, :cost 1.54E+18, :op {:process 447, :type :ok, :f :read, :value 5, :index 3759, :time 32801876250, :cas 1576585665103593472}}, :wgl [:WGL]} INFO [2019-12-17 12:39:19,348] knossos reporter - knossos.search {:linear {:space 2, :cost 9.73E+19, :op {:process 703, :type :ok, :f :read, :value 35, :index 3941, :time 33926763988, :cas 1576585664480673792}}, :wgl [:WGL]} INFO [2019-12-17 12:39:19,348] knossos reporter - knossos.search {:linear {:space 4, :cost 7.85E+13, :op {:process 316, :type :ok, :f :read, :value 35, :index 2475, :time 25300130017, :cas 1576585664332693504}}, :wgl [:WGL]} INFO [2019-12-17 12:39:19,359] knossos reporter - knossos.search {:linear {:space 4, :cost 1.34E+15, :op {:process 468, :type :ok, :f :read, :value 45, :index 2869, :time 27607385127, :cas 1576585663882919936}}, :wgl [:WGL]} INFO [2019-12-17 12:39:23,002] knossos reporter - knossos.search {:linear {:space 4, :cost 3.24E+11, :op {:process 345, :type :ok, :f :read, :value 2, :index 3101, :time 28839791206, :cas 1576585664725909504}}, :wgl [:WGL]} INFO [2019-12-17 12:39:23,129] knossos reporter - knossos.search {:linear {:space 8, :cost 4.60E+10, :op {:process 503, :type :ok, :f :read, :value 0, :index 2575, :time 25689476470, :cas 1576585664440238080}}, :wgl [:WGL]} INFO [2019-12-17 12:39:23,133] knossos reporter - knossos.search {:linear {:space 4, :cost 1.95E+20, :op {:process 549, :type :ok, :f :read, :value 14, :index 3673, :time 32207197026, :cas 1576585664669286400}}, :wgl [:WGL]} INFO [2019-12-17 12:39:24,417] knossos reporter - knossos.search {:linear {:space 1026, :cost 1.18E+20, :op {:process 487, :type :ok, :f :read, :value 12, :index 3553, :time 31598370823, :cas 1576585665026523136}}, :wgl [:WGL]} INFO [2019-12-17 12:39:28,012] knossos reporter - knossos.search {:linear {:space 2, :cost 4.62E+18, :op {:process 525, :type :ok, :f :read, :value 2, :index 4011, :time 34335986211, :cas 1576585664725909504}}, :wgl [:WGL]} INFO [2019-12-17 12:39:28,138] knossos reporter - knossos.search {:linear {:space 4, :cost 4.88E+12, :op {:process 412, :type :ok, :f :read, :value 0, :index 2917, :time 27821341935, :cas 1576585664440238080}}, :wgl [:WGL]} INFO [2019-12-17 12:39:29,428] knossos reporter - knossos.search {:linear {:space 2, :cost 3.92E+13, :op {:process 453, :type :ok, :f :read, :value 23, :index 3471, :time 31183190662, :cas 1576585665086095360}}, :wgl [:WGL]} INFO [2019-12-17 12:39:29,430] knossos reporter - knossos.search {:linear {:space 8, :cost 1.57E+14, :op {:process 464, :type :ok, :f :read, :value 4, :index 2855, :time 27510602026, :cas 1576585664984252416}}, :wgl [:WGL]} INFO [2019-12-17 12:39:29,454] knossos reporter - knossos.search {:linear {:space 8, :cost 1.57E+14, :op {:process 510, :type :ok, :f :read, :value 7, :index 2501, :time 25368425871, :cas 1576585664292388864}}, :wgl [:WGL]} INFO [2019-12-17 12:39:30,745] knossos reporter - knossos.search {:linear {:space 18434, :cost 1.98E+25, :op {:process 627, :type :invoke, :f :read, :value 5, :index 4542, :time 36909061570}}, :wgl [:WGL]} INFO [2019-12-17 12:39:34,466] knossos reporter - knossos.search {:linear {:space 2, :cost 2.15E+21, :op {:process 612, :type :ok, :f :read, :value 35, :index 4021, :time 34388132145, :cas 1576585664480673792}}, :wgl [:WGL]} INFO [2019-12-17 12:39:34,466] knossos reporter - knossos.search {:linear {:space 2, :cost 6.70E+14, :op {:process 544, :type :ok, :f :read, :value 23, :index 3549, :time 31572375139, :cas 1576585665086095360}}, :wgl [:WGL]} INFO [2019-12-17 12:39:34,466] knossos reporter - knossos.search {:linear {:space 4, :cost 9.25E+18, :op {:process 675, :type :ok, :f :read, :value 35, :index 3261, :time 29651039962, :cas 1576585664332693504}}, :wgl [:WGL]} INFO [2019-12-17 12:39:35,830] knossos reporter - knossos.search {:linear {:space 18434, :cost 4.56E+26, :op {:process 627, :type :ok, :f :read, :value 5, :index 4609, :time 37314865895, :cas 1576585665103593472}}, :wgl [:WGL]} INFO [2019-12-17 12:39:38,246] knossos reporter - knossos.search {:linear {:space 4, :cost 4.88E+12, :op {:process 412, :type :ok, :f :read, :value 0, :index 3031, :time 28417660318, :cas 1576585664440238080}}, :wgl [:WGL]} INFO [2019-12-17 12:39:39,485] knossos reporter - knossos.search {:linear {:space 4, :cost 9.25E+18, :op {:process 311, :type :ok, :f :read, :value 9, :index 3607, :time 31888223382, :cas 1576585663783370752}}, :wgl [:WGL]} INFO [2019-12-17 12:39:39,485] knossos reporter - knossos.search {:linear {:space 4, :cost 1.95E+20, :op {:process 558, :type :ok, :f :read, :value 45, :index 3493, :time 31265905734, :cas 1576585663882919936}}, :wgl [:WGL]} INFO [2019-12-17 12:39:43,247] knossos reporter - knossos.search {:linear {:space 4, :cost 4.88E+12, :op {:process 412, :type :ok, :f :read, :value 0, :index 3105, :time 28883947527, :cas 1576585664440238080}}, :wgl [:WGL]} INFO [2019-12-17 12:39:44,494] knossos reporter - knossos.search {:linear {:space 4, :cost 7.85E+13, :op {:process 464, :type :ok, :f :read, :value 4, :index 2925, :time 27848824086, :cas 1576585664984252416}}, :wgl [:WGL]} INFO [2019-12-17 12:39:44,494] knossos reporter - knossos.search {:linear {:space 4, :cost 4.61E+17, :op {:process 600, :type :ok, :f :read, :value 7, :index 2999, :time 28266722529, :cas 1576585664292388864}}, :wgl [:WGL]} INFO [2019-12-17 12:39:47,311] clojure-agent-send-pool-9 - jepsen.independent Wrote /Users/richarddemellow/source/jepsen.couchbase/store/Couchbase/20191217T122702.000Z/independent/24/results.edn INFO [2019-12-17 12:39:48,354] knossos reporter - knossos.search {:linear {:space 4, :cost 1.34E+15, :op {:process 593, :type :ok, :f :read, :value 0, :index 3263, :time 29653214943, :cas 1576585664440238080}}, :wgl [:WGL]} INFO [2019-12-17 12:39:48,478] clojure-agent-send-pool-9 - jepsen.independent Wrote /Users/richarddemellow/source/jepsen.couchbase/store/Couchbase/20191217T122702.000Z/independent/24/history.edn INFO [2019-12-17 12:39:48,961] clojure-agent-send-off-pool-47 - knossos.model.memo More than 1024 reachable models; not memoizing models for this search INFO [2019-12-17 12:39:49,017] Thread-250 - knossos.model.memo More than 1024 reachable models; not memoizing models for this search INFO [2019-12-17 12:39:49,630] knossos reporter - knossos.search {:linear {:space 2, :cost 6.70E+14, :op {:persist-to 0, :mutation-token Optional[mt{vbID=648, vbUUID=275963653234593, seqno=7, bucket=default}], :index 3649, :value 37, :time 32088266984, :process 453, :replicate-to 0, :type :ok, :cas 1576585682543771648, :f :write, :json true, :durability-level 1}}, :wgl [:WGL]} INFO [2019-12-17 12:39:54,210] knossos reporter - knossos.search {:linear {:space 2, :cost 1.21E+16, :op {:process 457, :type :ok, :f :read, :value 15, :index 2589, :time 25750582166, :cas 1576585664749699072}}, :wgl [:WGL]} INFO [2019-12-17 12:39:54,899] knossos reporter - knossos.search {:linear {:space 4, :cost 7.85E+13, :op {:process 464, :type :ok, :f :read, :value 4, :index 2975, :time 28107674949, :cas 1576585664984252416}}, :wgl [:WGL]} INFO [2019-12-17 12:39:55,063] knossos reporter - knossos.search {:linear {:space 16384, :cost 3.21E+17, :op {:process 275, :type :ok, :f :read, :value 37, :index 3771, :time 32914082673, :cas 1576585682543771648}}, :wgl [:WGL]} INFO [2019-12-17 12:39:56,902] clojure-agent-send-pool-4 - jepsen.independent Wrote /Users/richarddemellow/source/jepsen.couchbase/store/Couchbase/20191217T122702.000Z/independent/17/results.edn INFO [2019-12-17 12:39:57,594] clojure-agent-send-pool-4 - jepsen.independent Wrote /Users/richarddemellow/source/jepsen.couchbase/store/Couchbase/20191217T122702.000Z/independent/17/history.edn INFO [2019-12-17 12:39:58,088] clojure-agent-send-off-pool-52 - knossos.model.memo More than 1024 reachable models; not memoizing models for this search INFO [2019-12-17 12:39:58,230] Thread-252 - knossos.model.memo More than 1024 reachable models; not memoizing models for this search INFO [2019-12-17 12:39:59,224] knossos reporter - knossos.search {:linear {:space 3074, :cost 1.50E+23, :op {:process 548, :type :ok, :f :read, :value 15, :index 3107, :time 28885581742, :cas 1576585664749699072}}, :wgl [:WGL]} INFO [2019-12-17 12:40:00,086] knossos reporter - knossos.search {:linear {:space 8192, :cost 4.95E+19, :op {:process 453, :type :ok, :f :read, :value 37, :index 4067, :time 34686317804, :cas 1576585682543771648}}, :wgl [:WGL]} INFO [2019-12-17 12:40:02,050] clojure-agent-send-pool-11 - jepsen.independent Wrote /Users/richarddemellow/source/jepsen.couchbase/store/Couchbase/20191217T122702.000Z/independent/21/results.edn INFO [2019-12-17 12:40:03,093] knossos reporter - knossos.search {:linear {:space 2, :cost 3.92E+13, :op {:process 520, :type :ok, :f :read, :value 29, :index 2720, :time 26623966751, :cas 1576585664831750144}}, :wgl [:WGL]} INFO [2019-12-17 12:40:03,907] clojure-agent-send-pool-11 - jepsen.independent Wrote /Users/richarddemellow/source/jepsen.couchbase/store/Couchbase/20191217T122702.000Z/independent/21/history.edn INFO [2019-12-17 12:40:04,002] clojure-agent-send-off-pool-28 - knossos.model.memo More than 1024 reachable models; not memoizing models for this search INFO [2019-12-17 12:40:04,056] Thread-254 - knossos.model.memo More than 1024 reachable models; not memoizing models for this search INFO [2019-12-17 12:40:05,033] knossos reporter - knossos.search {:linear {:space 4, :cost 1.34E+15, :op {:process 464, :type :ok, :f :read, :value 4, :index 2995, :time 28256131345, :cas 1576585664984252416}}, :wgl [:WGL]} INFO [2019-12-17 12:40:05,129] knossos reporter - knossos.search {:linear {:space 8192, :cost 1.89E+22, :op {:process 453, :type :ok, :f :read, :value 37, :index 4153, :time 35159941090, :cas 1576585682543771648}}, :wgl [:WGL]} INFO [2019-12-17 12:40:05,257] knossos reporter - knossos.search {:linear {:space 32770, :cost 3.78E+21, :op {:persist-to 0, :index 3844, :value 39, :time 33331407226, :process 666, :replicate-to 0, :type :invoke, :f :write, :json true, :durability-level 1}}, :wgl [:WGL]} INFO [2019-12-17 12:40:08,101] knossos reporter - knossos.search {:linear {:space 1, :cost 6.05E+15, :op {:process 431, :type :ok, :f :read, :value 29, :index 3159, :time 29183945549, :cas 1576585664831750144}}, :wgl [:WGL]} INFO [2019-12-17 12:40:10,857] knossos reporter - knossos.search {:linear {:space 32770, :cost 1.59E+24, :op {:process 577, :type :ok, :f :read, :value 12, :index 3921, :time 33850211026, :cas 1576585665026523136}}, :wgl [:WGL]} INFO [2019-12-17 12:40:16,380] clojure-agent-send-pool-2 - jepsen.independent Wrote /Users/richarddemellow/source/jepsen.couchbase/store/Couchbase/20191217T122702.000Z/independent/13/results.edn INFO [2019-12-17 12:40:16,719] clojure-agent-send-pool-2 - jepsen.independent Wrote /Users/richarddemellow/source/jepsen.couchbase/store/Couchbase/20191217T122702.000Z/independent/13/history.edn INFO [2019-12-17 12:40:17,280] clojure-agent-send-off-pool-18 - knossos.model.memo More than 1024 reachable models; not memoizing models for this search INFO [2019-12-17 12:40:17,432] Thread-256 - knossos.model.memo More than 1024 reachable models; not memoizing models for this search INFO [2019-12-17 12:40:22,284] knossos reporter - knossos.search {:linear {:space 2, :cost 3.92E+13, :op {:process 483, :type :ok, :f :read, :value 7, :index 3127, :time 29059051513, :cas 1576585664942637056}}, :wgl [:WGL]} INFO [2019-12-17 12:40:27,681] knossos reporter - knossos.search {:linear {:space 2, :cost 2.30E+17, :op {:process 573, :type :ok, :f :read, :value 7, :index 3483, :time 31255545672, :cas 1576585664942637056}}, :wgl [:WGL]} INFO [2019-12-17 12:40:38,228] knossos reporter - knossos.search {:linear {:space 1, :cost 1.15E+17, :op {:process 431, :type :ok, :f :read, :value 29, :index 3357, :time 30310529582, :cas 1576585664831750144}}, :wgl [:WGL]} INFO [2019-12-17 12:40:40,378] knossos reporter - knossos.search {:linear {:space 4, :cost 2.42E+16, :op {:process 463, :type :ok, :f :read, :value 4, :index 3222, :time 29461808155, :cas 1576585664984252416}}, :wgl [:WGL]} INFO [2019-12-17 12:40:40,440] knossos reporter - knossos.search {:linear {:space 24578, :cost 1.20E+24, :op {:process 548, :type :ok, :f :read, :value 15, :index 3138, :time 29121352916, :cas 1576585664749699072}}, :wgl [:WGL]} INFO [2019-12-17 12:41:10,957] knossos reporter - knossos.search {:linear {:space 32768, :cost 7.57E+22, :op {:process 365, :type :ok, :f :read, :value 37, :index 4175, :time 35257797008, :cas 1576585682543771648}}, :wgl [:WGL]} INFO [2019-12-17 12:41:13,399] clojure-agent-send-pool-11 - jepsen.independent Wrote /Users/richarddemellow/source/jepsen.couchbase/store/Couchbase/20191217T122702.000Z/independent/19/results.edn INFO [2019-12-17 12:41:13,972] clojure-agent-send-pool-11 - jepsen.independent Wrote /Users/richarddemellow/source/jepsen.couchbase/store/Couchbase/20191217T122702.000Z/independent/19/history.edn INFO [2019-12-17 12:41:14,397] clojure-agent-send-off-pool-44 - knossos.model.memo More than 1024 reachable models; not memoizing models for this search INFO [2019-12-17 12:41:15,190] Thread-264 - knossos.model.memo More than 1024 reachable models; not memoizing models for this search INFO [2019-12-17 12:41:15,651] knossos reporter - knossos.search {:linear {:space 4, :cost 2.42E+16, :op {:process 383, :type :ok, :f :read, :value 20, :index 3443, :time 30966900682, :cas 1576585664662929408}}, :wgl [:WGL]} INFO [2019-12-17 12:41:22,663] knossos reporter - knossos.search {:linear {:space 24578, :cost 1.20E+24, :op {:process 548, :type :ok, :f :read, :value 15, :index 3289, :time 29725627319, :cas 1576585664749699072}}, :wgl [:WGL]} INFO [2019-12-17 12:41:45,025] knossos reporter - knossos.search {:linear {:space 2, :cost 2.30E+17, :op {:process 395, :type :ok, :f :read, :value 7, :index 3543, :time 31541447965, :cas 1576585664942637056}}, :wgl [:WGL]} INFO [2019-12-17 12:41:52,333] knossos reporter - knossos.search {:linear {:space 4, :cost 9.89E+22, :op {:process 550, :type :ok, :f :read, :value 14, :index 3827, :time 33213729436, :cas 1576585664669286400}}, :wgl [:WGL]} INFO [2019-12-17 12:41:55,163] knossos reporter - knossos.search {:linear {:space 1, :cost 2.31E+18, :op {:process 431, :type :ok, :f :read, :value 29, :index 3413, :time 30796201992, :cas 1576585664831750144}}, :wgl [:WGL]} INFO [2019-12-17 12:41:57,359] knossos reporter - knossos.search {:linear {:space 4, :cost 9.25E+18, :op {:process 462, :type :ok, :f :read, :value 4, :index 3403, :time 30655206002, :cas 1576585664984252416}}, :wgl [:WGL]} INFO [2019-12-17 12:42:02,992] clojure-agent-send-pool-12 - jepsen.independent Wrote /Users/richarddemellow/source/jepsen.couchbase/store/Couchbase/20191217T122702.000Z/independent/3/results.edn INFO [2019-12-17 12:42:04,303] clojure-agent-send-pool-12 - jepsen.independent Wrote /Users/richarddemellow/source/jepsen.couchbase/store/Couchbase/20191217T122702.000Z/independent/3/history.edn INFO [2019-12-17 12:42:04,682] clojure-agent-send-off-pool-57 - knossos.model.memo More than 1024 reachable models; not memoizing models for this search INFO [2019-12-17 12:42:04,725] Thread-266 - knossos.model.memo More than 1024 reachable models; not memoizing models for this search INFO [2019-12-17 12:42:07,363] knossos reporter - knossos.search {:linear {:space 2, :cost 4.62E+18, :op {:process 525, :type :ok, :f :read, :value 2, :index 4039, :time 34499225920, :cas 1576585664725909504}}, :wgl [:WGL]} INFO [2019-12-17 12:42:11,368] clojure-agent-send-pool-11 - jepsen.independent Wrote /Users/richarddemellow/source/jepsen.couchbase/store/Couchbase/20191217T122702.000Z/independent/9/results.edn INFO [2019-12-17 12:42:12,113] knossos reporter - knossos.search {:linear {:space 24578, :cost 6.08E+26, :op {:process 546, :type :ok, :f :read, :value 15, :index 3641, :time 32039152369, :cas 1576585664749699072}}, :wgl [:WGL]} INFO [2019-12-17 12:42:14,364] clojure-agent-send-pool-11 - jepsen.independent Wrote /Users/richarddemellow/source/jepsen.couchbase/store/Couchbase/20191217T122702.000Z/independent/9/history.edn INFO [2019-12-17 12:42:14,616] clojure-agent-send-off-pool-17 - knossos.model.memo More than 1024 reachable models; not memoizing models for this search INFO [2019-12-17 12:42:14,700] Thread-268 - knossos.model.memo More than 1024 reachable models; not memoizing models for this search INFO [2019-12-17 12:42:14,794] clojure-agent-send-pool-12 - jepsen.independent Wrote /Users/richarddemellow/source/jepsen.couchbase/store/Couchbase/20191217T122702.000Z/independent/5/results.edn INFO [2019-12-17 12:42:15,430] clojure-agent-send-pool-12 - jepsen.independent Wrote /Users/richarddemellow/source/jepsen.couchbase/store/Couchbase/20191217T122702.000Z/independent/5/history.edn INFO [2019-12-17 12:42:15,664] clojure-agent-send-off-pool-71 - knossos.model.memo More than 1024 reachable models; not memoizing models for this search INFO [2019-12-17 12:42:16,157] Thread-270 - knossos.model.memo More than 1024 reachable models; not memoizing models for this search INFO [2019-12-17 12:42:17,441] knossos reporter - knossos.search {:linear {:space 2, :cost 2.15E+21, :op {:process 473, :type :ok, :f :read, :value 20, :index 3950, :time 33963802749, :cas 1576585664662929408}}, :wgl [:WGL]} INFO [2019-12-17 12:42:19,630] knossos reporter - knossos.search {:linear {:space 132, :cost 2.59E+15, :op {:process 224, :type :ok, :f :read, :value 27, :index 2449, :time 25191862049, :cas 1576585663774130176}}, :wgl [:WGL]} INFO [2019-12-17 12:42:20,963] knossos reporter - knossos.search {:linear {:space 8, :cost 2.68E+15, :op {:process 438, :type :ok, :f :read, :value 47, :index 2755, :time 26814615710, :cas 1576585665376092160}}, :wgl [:WGL]} INFO [2019-12-17 12:42:25,802] knossos reporter - knossos.search {:linear {:space 32768, :cost 7.57E+22, :op {:process 453, :type :ok, :f :read, :value 37, :index 4261, :time 35666114621, :cas 1576585682543771648}}, :wgl [:WGL]} INFO [2019-12-17 12:42:31,596] knossos reporter - knossos.search {:linear {:space 12292, :cost 4.11E+18, :op {:process 224, :type :ok, :f :read, :value 27, :index 2731, :time 26687516836, :cas 1576585663774130176}}, :wgl [:WGL]} INFO [2019-12-17 12:42:46,852] knossos reporter - knossos.search {:linear {:space 8, :cost 1.85E+19, :op {:process 710, :type :ok, :f :read, :value 47, :index 3157, :time 29162870734, :cas 1576585665376092160}}, :wgl [:WGL]} INFO [2019-12-17 12:42:57,067] knossos reporter - knossos.search {:linear {:space 4, :cost 1.95E+20, :op {:process 601, :type :ok, :f :read, :value 7, :index 3315, :time 29960429512, :cas 1576585664292388864}}, :wgl [:WGL]} INFO [2019-12-17 12:42:57,271] knossos reporter - knossos.search {:linear {:space 24580, :cost 1.49E+20, :op {:process 224, :type :ok, :f :read, :value 27, :index 2969, :time 28067543755, :cas 1576585663774130176}}, :wgl [:WGL]} INFO [2019-12-17 12:43:23,773] knossos reporter - knossos.search {:linear {:space 2, :cost 2.15E+21, :op {:process 485, :type :ok, :f :read, :value 7, :index 3971, :time 34106176118, :cas 1576585664942637056}}, :wgl [:WGL]} INFO [2019-12-17 12:43:45,363] knossos reporter - knossos.search {:linear {:space 1, :cost 2.31E+18, :op {:process 431, :type :ok, :f :read, :value 29, :index 3559, :time 31624698647, :cas 1576585664831750144}}, :wgl [:WGL]} INFO [2019-12-17 12:44:42,123] knossos reporter - knossos.search {:linear {:space 49156, :cost 5.66E+21, :op {:process 224, :type :ok, :f :read, :value 27, :index 3041, :time 28532493580, :cas 1576585663774130176}}, :wgl [:WGL]} INFO [2019-12-17 12:48:20,632] clojure-agent-send-pool-6 - jepsen.independent Wrote /Users/richarddemellow/source/jepsen.couchbase/store/Couchbase/20191217T122702.000Z/independent/29/results.edn INFO [2019-12-17 12:48:22,282] clojure-agent-send-pool-6 - jepsen.independent Wrote /Users/richarddemellow/source/jepsen.couchbase/store/Couchbase/20191217T122702.000Z/independent/29/history.edn INFO [2019-12-17 12:48:24,660] clojure-agent-send-off-pool-133 - knossos.model.memo More than 1024 reachable models; not memoizing models for this search INFO [2019-12-17 12:48:24,724] Thread-328 - knossos.model.memo More than 1024 reachable models; not memoizing models for this search INFO [2019-12-17 12:48:35,602] knossos reporter - knossos.search {:linear {:space 16384, :cost 1.89E+21, :op {:process 634, :type :ok, :f :read, :value 37, :index 4271, :time 35706932914, :cas 1576585682543771648}}, :wgl [:WGL]} INFO [2019-12-17 12:48:41,392] clojure-agent-send-pool-6 - jepsen.independent Wrote /Users/richarddemellow/source/jepsen.couchbase/store/Couchbase/20191217T122702.000Z/independent/16/results.edn INFO [2019-12-17 12:48:42,302] clojure-agent-send-pool-6 - jepsen.independent Wrote /Users/richarddemellow/source/jepsen.couchbase/store/Couchbase/20191217T122702.000Z/independent/16/history.edn INFO [2019-12-17 12:48:46,013] clojure-agent-send-off-pool-136 - knossos.model.memo More than 1024 reachable models; not memoizing models for this search INFO [2019-12-17 12:48:46,788] Thread-330 - knossos.model.memo More than 1024 reachable models; not memoizing models for this search INFO [2019-12-17 12:48:51,788] knossos reporter - knossos.search {:linear {:space 2, :cost 9.73E+19, :op {:process 390, :type :ok, :f :read, :value 23, :index 3271, :time 29672168695, :cas 1576585664218202112}}, :wgl [:WGL]} INFO [2019-12-17 12:49:34,911] knossos reporter - knossos.search {:linear {:space 8192, :cost 3.99E+23, :op {:process 634, :type :ok, :f :read, :value 49, :index 5027, :time 40464944925, :cas 1576585690072088576}}, :wgl [:WGL]} INFO [2019-12-17 12:49:48,899] knossos reporter - knossos.search {:linear {:space 1, :cost 1.07E+21, :op {:process 699, :type :ok, :f :read, :value 29, :index 3825, :time 33203162662, :cas 1576585664831750144}}, :wgl [:WGL]} INFO [2019-12-17 12:52:54,481] knossos reporter - knossos.search {:linear {:space 2, :cost 9.73E+19, :op {:process 706, :type :ok, :f :read, :value 2, :index 4225, :time 35499662557, :cas 1576585664725909504}}, :wgl [:WGL]} INFO [2019-12-17 12:53:09,502] knossos reporter - knossos.search {:linear {:space 32, :cost 7.40E+19, :op {:process 439, :type :ok, :f :read, :value 47, :index 3207, :time 29421095830, :cas 1576585665376092160}}, :wgl [:WGL]} INFO [2019-12-17 12:54:13,047] knossos reporter - knossos.search {:linear {:space 98308, :cost 5.94E+20, :op {:persist-to 0, :index 3077, :value 25, :time 28743716757, :process 224, :replicate-to 0, :type :invoke, :f :write, :json true, :durability-level 1}}, :wgl [:WGL]} INFO [2019-12-17 12:54:20,323] knossos reporter - knossos.search {:linear {:space 98308, :cost 1.05E+26, :op {:process 763, :type :invoke, :f :read, :value 27, :index 3618, :time 31932475509}}, :wgl [:WGL]} INFO [2019-12-17 12:54:29,403] knossos reporter - knossos.search {:linear {:space 98308, :cost 5.85E+28, :op {:process 763, :type :ok, :f :read, :value 27, :index 3729, :time 32613705636, :cas 1576585663774130176}}, :wgl [:WGL]} INFO [2019-12-17 13:00:27,168] knossos reporter - knossos.search {:linear {:space 4, :cost 1.95E+20, :op {:process 390, :type :ok, :f :read, :value 23, :index 3281, :time 29701996368, :cas 1576585664218202112}}, :wgl [:WGL]} INFO [2019-12-17 13:01:00,762] clojure-agent-send-pool-7 - jepsen.independent Wrote /Users/richarddemellow/source/jepsen.couchbase/store/Couchbase/20191217T122702.000Z/independent/1/results.edn INFO [2019-12-17 13:01:03,423] clojure-agent-send-pool-7 - jepsen.independent Wrote /Users/richarddemellow/source/jepsen.couchbase/store/Couchbase/20191217T122702.000Z/independent/1/history.edn INFO [2019-12-17 13:01:09,202] clojure-agent-send-off-pool-195 - knossos.model.memo More than 1024 reachable models; not memoizing models for this search INFO [2019-12-17 13:01:09,691] Thread-399 - knossos.model.memo More than 1024 reachable models; not memoizing models for this search INFO [2019-12-17 13:01:14,375] knossos reporter - knossos.search {:linear {:space 2, :cost 6.70E+14, :op {:process 504, :type :ok, :f :read, :value 47, :index 2555, :time 25616432398, :cas 1576585664346914816}}, :wgl [:WGL]} INFO [2019-12-17 13:06:38,043] knossos reporter - knossos.search {:linear {:space 32, :cost 7.40E+19, :op {:process 528, :type :ok, :f :read, :value 47, :index 3227, :time 29491237503, :cas 1576585665376092160}}, :wgl [:WGL]} INFO [2019-12-17 13:06:47,833] knossos reporter - knossos.search {:linear {:space 131074, :cost 3.03E+23, :op {:persist-to 0, :index 3922, :value 6, :time 33850535466, :process 577, :replicate-to 0, :type :invoke, :f :write, :json true, :durability-level 1}}, :wgl [:WGL]} INFO [2019-12-17 13:06:59,911] knossos reporter - knossos.search {:linear {:space 131074, :cost 3.24E+27, :op {:process 756, :type :ok, :f :read, :value 12, :index 4283, :time 35802148195, :cas 1576585665026523136}}, :wgl [:WGL]} INFO [2019-12-17 13:07:09,289] knossos reporter - knossos.search {:linear {:space 2, :cost 6.70E+14, :op {:process 504, :type :ok, :f :read, :value 47, :index 2697, :time 26407342340, :cas 1576585664346914816}}, :wgl [:WGL]} INFO [2019-12-17 13:07:14,572] knossos reporter - knossos.search {:linear {:space 98306, :cost 1.05E+26, :op {:process 546, :type :invoke, :f :read, :value 15, :index 3642, :time 32039430475}}, :wgl [:WGL]} INFO [2019-12-17 13:07:21,982] knossos reporter - knossos.search {:linear {:space 98306, :cost 2.43E+27, :op {:process 546, :type :ok, :f :read, :value 15, :index 3779, :time 32960377471, :cas 1576585664749699072}}, :wgl [:WGL]} INFO [2019-12-17 13:13:05,840] knossos reporter - knossos.search {:linear {:space 2, :cost 6.70E+14, :op {:process 504, :type :ok, :f :read, :value 47, :index 2729, :time 26675328305, :cas 1576585664346914816}}, :wgl [:WGL]} INFO [2019-12-17 13:13:51,477] knossos reporter - knossos.search {:linear {:space 2, :cost 2.30E+17, :op {:process 416, :type :ok, :f :read, :value 47, :index 3115, :time 28937633149, :cas 1576585664346914816}}, :wgl [:WGL]} INFO [2019-12-17 13:16:54,348] knossos reporter - knossos.search {:linear {:space 4, :cost 1.95E+20, :op {:process 390, :type :ok, :f :read, :value 23, :index 3293, :time 29739299226, :cas 1576585664218202112}}, :wgl [:WGL]} INFO [2019-12-17 13:17:19,391] knossos reporter - knossos.search {:linear {:space 32, :cost 3.69E+18, :op {:process 710, :type :ok, :f :read, :value 47, :index 3279, :time 29698231264, :cas 1576585665376092160}}, :wgl [:WGL]} INFO [2019-12-17 13:17:25,799] knossos reporter - knossos.search {:linear {:space 8, :cost 1.85E+19, :op {:process 554, :type :ok, :f :read, :value 4, :index 3421, :time 30840240414, :cas 1576585664984252416}}, :wgl [:WGL]}